From 73df4f658ee7bd028f7ab216fe236e32b8b6fb06 Mon Sep 17 00:00:00 2001 From: James LaMar Date: Wed, 1 Mar 2023 16:34:51 -0800 Subject: [PATCH 001/114] Normalize naming and organization of rust crates --- .../{liballoc => alloc}/benches/btree/map.rs | 0 .../{liballoc => alloc}/benches/btree/mod.rs | 0 .../{liballoc => alloc}/benches/btree/set.rs | 0 .../lib/{liballoc => alloc}/benches/lib.rs | 0 .../benches/linked_list.rs | 0 .../lib/{liballoc => alloc}/benches/slice.rs | 0 .../lib/{liballoc => alloc}/benches/str.rs | 0 .../lib/{liballoc => alloc}/benches/string.rs | 0 .../lib/{liballoc => alloc}/benches/vec.rs | 0 .../{liballoc => alloc}/benches/vec_deque.rs | 0 .../benches/vec_deque_append.rs | 0 .../lib/{liballoc => alloc/src}/Cargo.toml | 0 crux-mir/lib/{liballoc => alloc/src}/alloc.rs | 0 .../{liballoc => alloc/src}/alloc/tests.rs | 0 .../lib/{liballoc => alloc/src}/borrow.rs | 0 crux-mir/lib/{liballoc => alloc/src}/boxed.rs | 0 .../src}/collections/binary_heap.rs | 0 .../src}/collections/btree/map.rs | 0 .../src}/collections/btree/mod.rs | 0 .../src}/collections/btree/navigate.rs | 0 .../src}/collections/btree/node.rs | 0 .../src}/collections/btree/search.rs | 0 .../src}/collections/btree/set.rs | 0 .../src}/collections/linked_list.rs | 0 .../src}/collections/linked_list/tests.rs | 0 .../src}/collections/mod.rs | 0 .../src}/collections/vec_deque.rs | 0 .../src}/collections/vec_deque/drain.rs | 0 .../src}/collections/vec_deque/tests.rs | 0 crux-mir/lib/{liballoc => alloc/src}/fmt.rs | 0 crux-mir/lib/{liballoc => alloc/src}/lib.rs | 0 .../lib/{liballoc => alloc/src}/macros.rs | 0 .../{liballoc => alloc/src}/prelude/mod.rs | 0 .../lib/{liballoc => alloc/src}/prelude/v1.rs | 0 .../lib/{liballoc => alloc/src}/raw_vec.rs | 0 .../{liballoc => alloc/src}/raw_vec/tests.rs | 0 crux-mir/lib/{liballoc => alloc/src}/rc.rs | 0 .../lib/{liballoc => alloc/src}/rc/tests.rs | 0 crux-mir/lib/{liballoc => alloc/src}/slice.rs | 0 crux-mir/lib/{liballoc => alloc/src}/str.rs | 0 .../lib/{liballoc => alloc/src}/string.rs | 0 crux-mir/lib/{liballoc => alloc/src}/sync.rs | 0 .../lib/{liballoc => alloc/src}/sync/tests.rs | 0 crux-mir/lib/{liballoc => alloc/src}/tests.rs | 0 crux-mir/lib/{liballoc => alloc/src}/vec.rs | 0 crux-mir/lib/{liballoc => alloc}/tests/arc.rs | 0 .../{liballoc => alloc}/tests/binary_heap.rs | 0 .../lib/{liballoc => alloc}/tests/boxed.rs | 0 .../{liballoc => alloc}/tests/btree/map.rs | 0 .../{liballoc => alloc}/tests/btree/mod.rs | 0 .../{liballoc => alloc}/tests/btree/set.rs | 0 .../lib/{liballoc => alloc}/tests/cow_str.rs | 0 crux-mir/lib/{liballoc => alloc}/tests/fmt.rs | 0 .../lib/{liballoc => alloc}/tests/heap.rs | 0 crux-mir/lib/{liballoc => alloc}/tests/lib.rs | 0 .../{liballoc => alloc}/tests/linked_list.rs | 0 crux-mir/lib/{liballoc => alloc}/tests/rc.rs | 0 .../lib/{liballoc => alloc}/tests/slice.rs | 0 crux-mir/lib/{liballoc => alloc}/tests/str.rs | 0 .../lib/{liballoc => alloc}/tests/string.rs | 0 crux-mir/lib/{liballoc => alloc}/tests/vec.rs | 0 .../{liballoc => alloc}/tests/vec_deque.rs | 0 .../Cargo.toml | 0 .../LICENSE.TXT | 0 .../PUBLISHING.md | 0 .../README.md | 0 .../build.rs | 0 .../aarch64-unknown-linux-gnu/Dockerfile | 0 .../arm-unknown-linux-gnueabi/Dockerfile | 0 .../arm-unknown-linux-gnueabihf/Dockerfile | 0 .../armv7-unknown-linux-gnueabihf/Dockerfile | 0 .../docker/i586-unknown-linux-gnu/Dockerfile | 0 .../docker/i686-unknown-linux-gnu/Dockerfile | 0 .../docker/mips-unknown-linux-gnu/Dockerfile | 0 .../mips64-unknown-linux-gnuabi64/Dockerfile | 0 .../Dockerfile | 0 .../mipsel-unknown-linux-gnu/Dockerfile | 0 .../powerpc-unknown-linux-gnu/Dockerfile | 0 .../powerpc64-unknown-linux-gnu/Dockerfile | 0 .../powerpc64le-unknown-linux-gnu/Dockerfile | 0 .../ci/docker/thumbv6m-none-eabi/Dockerfile | 0 .../ci/docker/thumbv7em-none-eabi/Dockerfile | 0 .../docker/thumbv7em-none-eabihf/Dockerfile | 0 .../ci/docker/thumbv7m-none-eabi/Dockerfile | 0 .../x86_64-unknown-linux-gnu/Dockerfile | 0 .../ci/run-docker.sh | 0 .../ci/run.sh | 0 .../crates/panic-handler/Cargo.toml | 0 .../crates/panic-handler/src/lib.rs | 0 .../examples/intrinsics.rs | 0 .../src/arm.rs | 0 .../src/arm_linux.rs | 0 .../src/float/add.rs | 0 .../src/float/cmp.rs | 0 .../src/float/conv.rs | 0 .../src/float/div.rs | 0 .../src/float/extend.rs | 0 .../src/float/mod.rs | 0 .../src/float/mul.rs | 0 .../src/float/pow.rs | 0 .../src/float/sub.rs | 0 .../src/int/addsub.rs | 0 .../src/int/mod.rs | 0 .../src/int/mul.rs | 0 .../src/int/sdiv.rs | 0 .../src/int/shift.rs | 0 .../src/int/udiv.rs | 0 .../src/lib.rs | 0 .../src/macros.rs | 0 .../src/math.rs | 0 .../src/mem.rs | 0 .../src/probestack.rs | 0 .../src/riscv32.rs | 0 .../src/x86.rs | 0 .../src/x86_64.rs | 0 .../test.c | 0 .../testcrate/Cargo.toml | 0 .../testcrate/build.rs | 0 .../testcrate/src/lib.rs | 0 .../testcrate/tests/aeabi_memclr.rs | 0 .../testcrate/tests/aeabi_memcpy.rs | 0 .../testcrate/tests/aeabi_memset.rs | 0 .../testcrate/tests/count_leading_zeros.rs | 0 .../testcrate/tests/generated.rs | 0 .../thumbv6m-linux-eabi.json | 0 .../thumbv7em-linux-eabi.json | 0 .../thumbv7em-linux-eabihf.json | 0 .../thumbv7m-linux-eabi.json | 0 crux-mir/lib/{libcore => core}/benches/any.rs | 0 .../lib/{libcore => core}/benches/ascii.rs | 0 .../{libcore => core}/benches/char/methods.rs | 0 .../lib/{libcore => core}/benches/char/mod.rs | 0 crux-mir/lib/{libcore => core}/benches/fmt.rs | 0 .../lib/{libcore => core}/benches/hash/mod.rs | 0 .../lib/{libcore => core}/benches/hash/sip.rs | 0 .../lib/{libcore => core}/benches/iter.rs | 0 crux-mir/lib/{libcore => core}/benches/lib.rs | 0 .../benches/num/dec2flt/mod.rs | 0 .../benches/num/flt2dec/mod.rs | 0 .../benches/num/flt2dec/strategy/dragon.rs | 0 .../benches/num/flt2dec/strategy/grisu.rs | 0 .../lib/{libcore => core}/benches/num/mod.rs | 0 crux-mir/lib/{libcore => core}/benches/ops.rs | 0 .../lib/{libcore => core}/benches/pattern.rs | 0 .../lib/{libcore => core}/benches/slice.rs | 0 crux-mir/lib/{libcore => core/src}/Cargo.toml | 0 crux-mir/lib/{libcore => core/src}/alloc.rs | 0 crux-mir/lib/{libcore => core/src}/any.rs | 0 .../lib/{libcore => core/src}/array/iter.rs | 0 .../lib/{libcore => core/src}/array/mod.rs | 0 crux-mir/lib/{libcore => core/src}/ascii.rs | 0 crux-mir/lib/{libcore => core/src}/bool.rs | 0 crux-mir/lib/{libcore => core/src}/borrow.rs | 0 crux-mir/lib/{libcore => core/src}/cell.rs | 0 .../lib/{libcore => core/src}/char/convert.rs | 0 .../lib/{libcore => core/src}/char/decode.rs | 0 .../lib/{libcore => core/src}/char/methods.rs | 0 .../lib/{libcore => core/src}/char/mod.rs | 0 crux-mir/lib/{libcore => core/src}/clone.rs | 0 crux-mir/lib/{libcore => core/src}/cmp.rs | 0 .../lib/{libcore => core/src}/convert/mod.rs | 0 .../lib/{libcore => core/src}/convert/num.rs | 0 .../lib/{libcore => core/src}/crucible/any.rs | 0 .../src}/crucible/concurrency.rs | 0 .../lib/{libcore => core/src}/crucible/mod.rs | 0 .../lib/{libcore => core/src}/crucible/ptr.rs | 0 crux-mir/lib/{libcore => core/src}/default.rs | 0 crux-mir/lib/{libcore => core/src}/ffi.rs | 0 .../lib/{libcore => core/src}/fmt/builders.rs | 0 .../lib/{libcore => core/src}/fmt/float.rs | 0 crux-mir/lib/{libcore => core/src}/fmt/mod.rs | 0 crux-mir/lib/{libcore => core/src}/fmt/num.rs | 0 .../lib/{libcore => core/src}/fmt/rt/v1.rs | 0 .../{libcore => core/src}/future/future.rs | 0 .../lib/{libcore => core/src}/future/mod.rs | 0 .../lib/{libcore => core/src}/hash/mod.rs | 0 .../lib/{libcore => core/src}/hash/sip.rs | 0 crux-mir/lib/{libcore => core/src}/hint.rs | 0 .../{libcore => core/src}/internal_macros.rs | 0 .../lib/{libcore => core/src}/intrinsics.rs | 0 .../src}/iter/adapters/chain.rs | 0 .../src}/iter/adapters/flatten.rs | 0 .../src}/iter/adapters/mod.rs | 0 .../src}/iter/adapters/zip.rs | 0 .../lib/{libcore => core/src}/iter/mod.rs | 0 .../lib/{libcore => core/src}/iter/range.rs | 0 .../lib/{libcore => core/src}/iter/sources.rs | 0 .../src}/iter/traits/accum.rs | 0 .../src}/iter/traits/collect.rs | 0 .../src}/iter/traits/double_ended.rs | 0 .../src}/iter/traits/exact_size.rs | 0 .../src}/iter/traits/iterator.rs | 0 .../src}/iter/traits/marker.rs | 0 .../{libcore => core/src}/iter/traits/mod.rs | 0 crux-mir/lib/{libcore => core/src}/lib.rs | 0 .../lib/{libcore => core/src}/macros/mod.rs | 0 .../lib/{libcore => core/src}/macros/panic.md | 0 crux-mir/lib/{libcore => core/src}/marker.rs | 0 .../src}/mem/manually_drop.rs | 0 .../{libcore => core/src}/mem/maybe_uninit.rs | 0 crux-mir/lib/{libcore => core/src}/mem/mod.rs | 0 .../lib/{libcore => core/src}/num/bignum.rs | 0 .../src}/num/dec2flt/algorithm.rs | 0 .../{libcore => core/src}/num/dec2flt/mod.rs | 0 .../{libcore => core/src}/num/dec2flt/num.rs | 0 .../src}/num/dec2flt/parse.rs | 0 .../src}/num/dec2flt/rawfp.rs | 0 .../src}/num/dec2flt/table.rs | 0 .../{libcore => core/src}/num/diy_float.rs | 0 crux-mir/lib/{libcore => core/src}/num/f32.rs | 0 crux-mir/lib/{libcore => core/src}/num/f64.rs | 0 .../src}/num/flt2dec/decoder.rs | 0 .../src}/num/flt2dec/estimator.rs | 0 .../{libcore => core/src}/num/flt2dec/mod.rs | 0 .../src}/num/flt2dec/strategy/dragon.rs | 0 .../src}/num/flt2dec/strategy/grisu.rs | 0 .../lib/{libcore => core/src}/num/i128.rs | 0 crux-mir/lib/{libcore => core/src}/num/i16.rs | 0 crux-mir/lib/{libcore => core/src}/num/i32.rs | 0 crux-mir/lib/{libcore => core/src}/num/i64.rs | 0 crux-mir/lib/{libcore => core/src}/num/i8.rs | 0 .../{libcore => core/src}/num/int_macros.rs | 0 .../lib/{libcore => core/src}/num/isize.rs | 0 crux-mir/lib/{libcore => core/src}/num/mod.rs | 0 .../lib/{libcore => core/src}/num/u128.rs | 0 crux-mir/lib/{libcore => core/src}/num/u16.rs | 0 crux-mir/lib/{libcore => core/src}/num/u32.rs | 0 crux-mir/lib/{libcore => core/src}/num/u64.rs | 0 crux-mir/lib/{libcore => core/src}/num/u8.rs | 0 .../lib/{libcore => core/src}/num/usize.rs | 0 .../lib/{libcore => core/src}/num/wrapping.rs | 0 .../lib/{libcore => core/src}/ops/arith.rs | 0 crux-mir/lib/{libcore => core/src}/ops/bit.rs | 0 .../lib/{libcore => core/src}/ops/deref.rs | 0 .../lib/{libcore => core/src}/ops/drop.rs | 0 .../lib/{libcore => core/src}/ops/function.rs | 0 .../{libcore => core/src}/ops/generator.rs | 0 .../lib/{libcore => core/src}/ops/index.rs | 0 crux-mir/lib/{libcore => core/src}/ops/mod.rs | 0 .../lib/{libcore => core/src}/ops/range.rs | 0 crux-mir/lib/{libcore => core/src}/ops/try.rs | 0 .../lib/{libcore => core/src}/ops/unsize.rs | 0 crux-mir/lib/{libcore => core/src}/option.rs | 0 crux-mir/lib/{libcore => core/src}/panic.rs | 0 .../lib/{libcore => core/src}/panicking.rs | 0 crux-mir/lib/{libcore => core/src}/pin.rs | 0 .../lib/{libcore => core/src}/prelude/mod.rs | 0 .../lib/{libcore => core/src}/prelude/v1.rs | 0 .../lib/{libcore => core/src}/primitive.rs | 0 .../{libcore => core/src}/ptr/const_ptr.rs | 0 crux-mir/lib/{libcore => core/src}/ptr/mod.rs | 0 .../lib/{libcore => core/src}/ptr/mut_ptr.rs | 0 .../lib/{libcore => core/src}/ptr/non_null.rs | 0 .../lib/{libcore => core/src}/ptr/unique.rs | 0 crux-mir/lib/{libcore => core/src}/raw.rs | 0 crux-mir/lib/{libcore => core/src}/result.rs | 0 .../lib/{libcore => core/src}/slice/memchr.rs | 0 .../lib/{libcore => core/src}/slice/mod.rs | 0 .../lib/{libcore => core/src}/slice/rotate.rs | 0 .../lib/{libcore => core/src}/slice/sort.rs | 0 .../lib/{libcore => core/src}/str/lossy.rs | 0 crux-mir/lib/{libcore => core/src}/str/mod.rs | 0 .../lib/{libcore => core/src}/str/pattern.rs | 0 .../lib/{libcore => core/src}/sync/atomic.rs | 0 .../lib/{libcore => core/src}/sync/mod.rs | 0 .../lib/{libcore => core/src}/task/mod.rs | 0 .../lib/{libcore => core/src}/task/poll.rs | 0 .../lib/{libcore => core/src}/task/wake.rs | 0 crux-mir/lib/{libcore => core/src}/time.rs | 0 crux-mir/lib/{libcore => core/src}/tuple.rs | 0 .../lib/{libcore => core/src}/unicode/mod.rs | 0 .../src}/unicode/printable.py | 0 .../src}/unicode/printable.rs | 0 .../src}/unicode/unicode_data.rs | 0 .../{libcore => core/src}/unicode/version.rs | 0 crux-mir/lib/{libcore => core/src}/unit.rs | 0 crux-mir/lib/{libcore => core}/tests/alloc.rs | 0 crux-mir/lib/{libcore => core}/tests/any.rs | 0 crux-mir/lib/{libcore => core}/tests/array.rs | 0 crux-mir/lib/{libcore => core}/tests/ascii.rs | 0 .../lib/{libcore => core}/tests/atomic.rs | 0 crux-mir/lib/{libcore => core}/tests/bool.rs | 0 crux-mir/lib/{libcore => core}/tests/cell.rs | 0 crux-mir/lib/{libcore => core}/tests/char.rs | 0 crux-mir/lib/{libcore => core}/tests/clone.rs | 0 crux-mir/lib/{libcore => core}/tests/cmp.rs | 0 .../{libcore => core}/tests/fmt/builders.rs | 0 .../lib/{libcore => core}/tests/fmt/float.rs | 0 .../lib/{libcore => core}/tests/fmt/mod.rs | 0 .../lib/{libcore => core}/tests/fmt/num.rs | 0 .../lib/{libcore => core}/tests/hash/mod.rs | 0 .../lib/{libcore => core}/tests/hash/sip.rs | 0 .../lib/{libcore => core}/tests/intrinsics.rs | 0 crux-mir/lib/{libcore => core}/tests/iter.rs | 0 crux-mir/lib/{libcore => core}/tests/lib.rs | 0 .../{libcore => core}/tests/manually_drop.rs | 0 crux-mir/lib/{libcore => core}/tests/mem.rs | 0 .../lib/{libcore => core}/tests/nonzero.rs | 0 .../lib/{libcore => core}/tests/num/bignum.rs | 0 .../tests/num/dec2flt/mod.rs | 0 .../tests/num/dec2flt/parse.rs | 0 .../tests/num/dec2flt/rawfp.rs | 0 .../tests/num/flt2dec/estimator.rs | 0 .../tests/num/flt2dec/mod.rs | 0 .../tests/num/flt2dec/random.rs | 0 .../tests/num/flt2dec/strategy/dragon.rs | 0 .../tests/num/flt2dec/strategy/grisu.rs | 0 .../lib/{libcore => core}/tests/num/i16.rs | 0 .../lib/{libcore => core}/tests/num/i32.rs | 0 .../lib/{libcore => core}/tests/num/i64.rs | 0 .../lib/{libcore => core}/tests/num/i8.rs | 0 .../{libcore => core}/tests/num/int_macros.rs | 0 .../lib/{libcore => core}/tests/num/mod.rs | 0 .../lib/{libcore => core}/tests/num/u16.rs | 0 .../lib/{libcore => core}/tests/num/u32.rs | 0 .../lib/{libcore => core}/tests/num/u64.rs | 0 .../lib/{libcore => core}/tests/num/u8.rs | 0 .../tests/num/uint_macros.rs | 0 crux-mir/lib/{libcore => core}/tests/ops.rs | 0 .../lib/{libcore => core}/tests/option.rs | 0 .../lib/{libcore => core}/tests/pattern.rs | 0 crux-mir/lib/{libcore => core}/tests/ptr.rs | 0 .../lib/{libcore => core}/tests/result.rs | 0 crux-mir/lib/{libcore => core}/tests/slice.rs | 0 crux-mir/lib/{libcore => core}/tests/str.rs | 0 .../lib/{libcore => core}/tests/str_lossy.rs | 0 crux-mir/lib/{libcore => core}/tests/time.rs | 0 crux-mir/lib/{libcore => core}/tests/tuple.rs | 0 .../Cargo.toml | 0 .../{libpanic_abort => panic_abort}/lib.rs | 0 crux-mir/lib/panic_abort/src/lib.rs | 126 ++++++++ .../Cargo.toml | 0 .../dummy.rs | 0 .../dwarf/eh.rs | 0 .../dwarf/mod.rs | 0 .../dwarf/tests.rs | 0 .../{libpanic_unwind => panic_unwind}/emcc.rs | 0 .../{libpanic_unwind => panic_unwind}/gcc.rs | 0 .../hermit.rs | 0 .../{libpanic_unwind => panic_unwind}/lib.rs | 0 .../{libpanic_unwind => panic_unwind}/miri.rs | 0 .../{libpanic_unwind => panic_unwind}/seh.rs | 0 .../lib/{libstd => std}/benches/hash/map.rs | 0 .../lib/{libstd => std}/benches/hash/mod.rs | 0 .../{libstd => std}/benches/hash/set_ops.rs | 0 crux-mir/lib/{libstd => std}/benches/lib.rs | 0 crux-mir/lib/{libstd => std/src}/Cargo.toml | 0 crux-mir/lib/{libstd => std/src}/alloc.rs | 0 crux-mir/lib/{libstd => std/src}/ascii.rs | 0 crux-mir/lib/{libstd => std/src}/backtrace.rs | 0 crux-mir/lib/{libstd => std/src}/build.rs | 0 .../src}/collections/hash/crucible_map.rs | 0 .../src}/collections/hash/map.rs | 0 .../src}/collections/hash/mod.rs | 0 .../src}/collections/hash/set.rs | 0 .../{libstd => std/src}/collections/mod.rs | 0 crux-mir/lib/{libstd => std/src}/env.rs | 0 crux-mir/lib/{libstd => std/src}/error.rs | 0 crux-mir/lib/{libstd => std/src}/f32.rs | 0 crux-mir/lib/{libstd => std/src}/f64.rs | 0 crux-mir/lib/{libstd => std/src}/ffi/c_str.rs | 0 crux-mir/lib/{libstd => std/src}/ffi/mod.rs | 0 .../lib/{libstd => std/src}/ffi/os_str.rs | 0 crux-mir/lib/{libstd => std/src}/fs.rs | 0 crux-mir/lib/{libstd => std/src}/future.rs | 0 .../lib/{libstd => std/src}/io/buffered.rs | 0 crux-mir/lib/{libstd => std/src}/io/cursor.rs | 0 crux-mir/lib/{libstd => std/src}/io/error.rs | 0 crux-mir/lib/{libstd => std/src}/io/impls.rs | 0 crux-mir/lib/{libstd => std/src}/io/lazy.rs | 0 crux-mir/lib/{libstd => std/src}/io/mod.rs | 0 .../lib/{libstd => std/src}/io/prelude.rs | 0 crux-mir/lib/{libstd => std/src}/io/stdio.rs | 0 crux-mir/lib/{libstd => std/src}/io/util.rs | 0 .../lib/{libstd => std/src}/keyword_docs.rs | 0 crux-mir/lib/{libstd => std/src}/lib.rs | 0 crux-mir/lib/{libstd => std/src}/macros.rs | 0 crux-mir/lib/{libstd => std/src}/memchr.rs | 0 crux-mir/lib/{libstd => std/src}/net/addr.rs | 0 crux-mir/lib/{libstd => std/src}/net/ip.rs | 0 crux-mir/lib/{libstd => std/src}/net/mod.rs | 0 .../lib/{libstd => std/src}/net/parser.rs | 0 crux-mir/lib/{libstd => std/src}/net/tcp.rs | 0 crux-mir/lib/{libstd => std/src}/net/test.rs | 0 crux-mir/lib/{libstd => std/src}/net/udp.rs | 0 crux-mir/lib/{libstd => std/src}/num.rs | 0 .../lib/{libstd => std/src}/os/android/fs.rs | 0 .../lib/{libstd => std/src}/os/android/mod.rs | 0 .../lib/{libstd => std/src}/os/android/raw.rs | 0 .../{libstd => std/src}/os/dragonfly/fs.rs | 0 .../{libstd => std/src}/os/dragonfly/mod.rs | 0 .../{libstd => std/src}/os/dragonfly/raw.rs | 0 .../{libstd => std/src}/os/emscripten/fs.rs | 0 .../{libstd => std/src}/os/emscripten/mod.rs | 0 .../{libstd => std/src}/os/emscripten/raw.rs | 0 .../src}/os/fortanix_sgx/mod.rs | 0 .../lib/{libstd => std/src}/os/freebsd/fs.rs | 0 .../lib/{libstd => std/src}/os/freebsd/mod.rs | 0 .../lib/{libstd => std/src}/os/freebsd/raw.rs | 0 .../lib/{libstd => std/src}/os/fuchsia/fs.rs | 0 .../lib/{libstd => std/src}/os/fuchsia/mod.rs | 0 .../lib/{libstd => std/src}/os/fuchsia/raw.rs | 0 .../lib/{libstd => std/src}/os/haiku/fs.rs | 0 .../lib/{libstd => std/src}/os/haiku/mod.rs | 0 .../lib/{libstd => std/src}/os/haiku/raw.rs | 0 crux-mir/lib/{libstd => std/src}/os/ios/fs.rs | 0 .../lib/{libstd => std/src}/os/ios/mod.rs | 0 .../lib/{libstd => std/src}/os/ios/raw.rs | 0 .../lib/{libstd => std/src}/os/linux/fs.rs | 0 .../lib/{libstd => std/src}/os/linux/mod.rs | 0 .../lib/{libstd => std/src}/os/linux/raw.rs | 0 .../lib/{libstd => std/src}/os/macos/fs.rs | 0 .../lib/{libstd => std/src}/os/macos/mod.rs | 0 .../lib/{libstd => std/src}/os/macos/raw.rs | 0 crux-mir/lib/{libstd => std/src}/os/mod.rs | 0 .../lib/{libstd => std/src}/os/netbsd/fs.rs | 0 .../lib/{libstd => std/src}/os/netbsd/mod.rs | 0 .../lib/{libstd => std/src}/os/netbsd/raw.rs | 0 .../lib/{libstd => std/src}/os/openbsd/fs.rs | 0 .../lib/{libstd => std/src}/os/openbsd/mod.rs | 0 .../lib/{libstd => std/src}/os/openbsd/raw.rs | 0 .../lib/{libstd => std/src}/os/raw/char.md | 0 .../lib/{libstd => std/src}/os/raw/double.md | 0 .../lib/{libstd => std/src}/os/raw/float.md | 0 .../lib/{libstd => std/src}/os/raw/int.md | 0 .../lib/{libstd => std/src}/os/raw/long.md | 0 .../{libstd => std/src}/os/raw/longlong.md | 0 .../lib/{libstd => std/src}/os/raw/mod.rs | 0 .../lib/{libstd => std/src}/os/raw/schar.md | 0 .../lib/{libstd => std/src}/os/raw/short.md | 0 .../lib/{libstd => std/src}/os/raw/uchar.md | 0 .../lib/{libstd => std/src}/os/raw/uint.md | 0 .../lib/{libstd => std/src}/os/raw/ulong.md | 0 .../{libstd => std/src}/os/raw/ulonglong.md | 0 .../lib/{libstd => std/src}/os/raw/ushort.md | 0 .../lib/{libstd => std/src}/os/redox/fs.rs | 0 .../lib/{libstd => std/src}/os/redox/mod.rs | 0 .../lib/{libstd => std/src}/os/redox/raw.rs | 0 .../lib/{libstd => std/src}/os/solaris/fs.rs | 0 .../lib/{libstd => std/src}/os/solaris/mod.rs | 0 .../lib/{libstd => std/src}/os/solaris/raw.rs | 0 .../lib/{libstd => std/src}/os/vxworks/fs.rs | 0 .../lib/{libstd => std/src}/os/vxworks/mod.rs | 0 .../lib/{libstd => std/src}/os/vxworks/raw.rs | 0 crux-mir/lib/{libstd => std/src}/os/wasi.rs | 0 crux-mir/lib/{libstd => std/src}/panic.rs | 0 crux-mir/lib/{libstd => std/src}/panicking.rs | 0 crux-mir/lib/{libstd => std/src}/path.rs | 0 .../lib/{libstd => std/src}/prelude/mod.rs | 0 .../lib/{libstd => std/src}/prelude/v1.rs | 0 .../lib/{libstd => std/src}/primitive_docs.rs | 0 crux-mir/lib/{libstd => std/src}/process.rs | 0 crux-mir/lib/{libstd => std/src}/rt.rs | 0 .../lib/{libstd => std/src}/sync/barrier.rs | 0 .../lib/{libstd => std/src}/sync/condvar.rs | 0 crux-mir/lib/{libstd => std/src}/sync/mod.rs | 0 .../{libstd => std/src}/sync/mpsc/blocking.rs | 0 .../src}/sync/mpsc/cache_aligned.rs | 0 .../lib/{libstd => std/src}/sync/mpsc/mod.rs | 0 .../src}/sync/mpsc/mpsc_queue.rs | 0 .../{libstd => std/src}/sync/mpsc/oneshot.rs | 0 .../{libstd => std/src}/sync/mpsc/shared.rs | 0 .../src}/sync/mpsc/spsc_queue.rs | 0 .../{libstd => std/src}/sync/mpsc/stream.rs | 0 .../lib/{libstd => std/src}/sync/mpsc/sync.rs | 0 .../lib/{libstd => std/src}/sync/mutex.rs | 0 crux-mir/lib/{libstd => std/src}/sync/once.rs | 0 .../lib/{libstd => std/src}/sync/rwlock.rs | 0 .../src}/sys/cloudabi/abi/bitflags.rs | 0 .../src}/sys/cloudabi/abi/cloudabi.rs | 0 .../src}/sys/cloudabi/abi/mod.rs | 0 .../{libstd => std/src}/sys/cloudabi/args.rs | 0 .../src}/sys/cloudabi/condvar.rs | 0 .../{libstd => std/src}/sys/cloudabi/io.rs | 0 .../{libstd => std/src}/sys/cloudabi/mod.rs | 0 .../{libstd => std/src}/sys/cloudabi/mutex.rs | 0 .../{libstd => std/src}/sys/cloudabi/os.rs | 0 .../src}/sys/cloudabi/rwlock.rs | 0 .../src}/sys/cloudabi/shims/args.rs | 0 .../src}/sys/cloudabi/shims/env.rs | 0 .../src}/sys/cloudabi/shims/fs.rs | 0 .../src}/sys/cloudabi/shims/mod.rs | 0 .../src}/sys/cloudabi/shims/net.rs | 0 .../src}/sys/cloudabi/shims/os.rs | 0 .../src}/sys/cloudabi/shims/pipe.rs | 0 .../src}/sys/cloudabi/shims/process.rs | 0 .../src}/sys/cloudabi/stack_overflow.rs | 0 .../{libstd => std/src}/sys/cloudabi/stdio.rs | 0 .../src}/sys/cloudabi/thread.rs | 0 .../{libstd => std/src}/sys/cloudabi/time.rs | 0 .../{libstd => std/src}/sys/crux/condvar.rs | 0 .../lib/{libstd => std/src}/sys/crux/mod.rs | 0 .../lib/{libstd => std/src}/sys/crux/mutex.rs | 0 .../{libstd => std/src}/sys/crux/rwlock.rs | 0 .../lib/{libstd => std/src}/sys/crux/time.rs | 0 .../{libstd => std/src}/sys/hermit/alloc.rs | 0 .../{libstd => std/src}/sys/hermit/args.rs | 0 .../{libstd => std/src}/sys/hermit/cmath.rs | 0 .../{libstd => std/src}/sys/hermit/condvar.rs | 0 .../lib/{libstd => std/src}/sys/hermit/env.rs | 0 .../src}/sys/hermit/fast_thread_local.rs | 0 .../lib/{libstd => std/src}/sys/hermit/fd.rs | 0 .../lib/{libstd => std/src}/sys/hermit/fs.rs | 0 .../lib/{libstd => std/src}/sys/hermit/io.rs | 0 .../{libstd => std/src}/sys/hermit/memchr.rs | 0 .../lib/{libstd => std/src}/sys/hermit/mod.rs | 0 .../{libstd => std/src}/sys/hermit/mutex.rs | 0 .../lib/{libstd => std/src}/sys/hermit/net.rs | 0 .../lib/{libstd => std/src}/sys/hermit/os.rs | 0 .../{libstd => std/src}/sys/hermit/path.rs | 0 .../{libstd => std/src}/sys/hermit/pipe.rs | 0 .../{libstd => std/src}/sys/hermit/process.rs | 0 .../{libstd => std/src}/sys/hermit/rwlock.rs | 0 .../src}/sys/hermit/stack_overflow.rs | 0 .../{libstd => std/src}/sys/hermit/stdio.rs | 0 .../{libstd => std/src}/sys/hermit/thread.rs | 0 .../src}/sys/hermit/thread_local.rs | 0 .../{libstd => std/src}/sys/hermit/time.rs | 0 crux-mir/lib/{libstd => std/src}/sys/mod.rs | 0 .../{libstd => std/src}/sys/sgx/abi/entry.S | 0 .../{libstd => std/src}/sys/sgx/abi/mem.rs | 0 .../{libstd => std/src}/sys/sgx/abi/mod.rs | 0 .../{libstd => std/src}/sys/sgx/abi/panic.rs | 0 .../{libstd => std/src}/sys/sgx/abi/reloc.rs | 0 .../{libstd => std/src}/sys/sgx/abi/thread.rs | 0 .../{libstd => std/src}/sys/sgx/abi/tls.rs | 0 .../src}/sys/sgx/abi/usercalls/alloc.rs | 0 .../src}/sys/sgx/abi/usercalls/mod.rs | 0 .../src}/sys/sgx/abi/usercalls/raw.rs | 0 .../lib/{libstd => std/src}/sys/sgx/alloc.rs | 0 .../lib/{libstd => std/src}/sys/sgx/args.rs | 0 .../lib/{libstd => std/src}/sys/sgx/cmath.rs | 0 .../{libstd => std/src}/sys/sgx/condvar.rs | 0 .../lib/{libstd => std/src}/sys/sgx/env.rs | 0 .../{libstd => std/src}/sys/sgx/ext/arch.rs | 0 .../{libstd => std/src}/sys/sgx/ext/ffi.rs | 0 .../lib/{libstd => std/src}/sys/sgx/ext/io.rs | 0 .../{libstd => std/src}/sys/sgx/ext/mod.rs | 0 .../lib/{libstd => std/src}/sys/sgx/fd.rs | 0 .../lib/{libstd => std/src}/sys/sgx/fs.rs | 0 .../lib/{libstd => std/src}/sys/sgx/io.rs | 0 .../lib/{libstd => std/src}/sys/sgx/memchr.rs | 0 .../lib/{libstd => std/src}/sys/sgx/mod.rs | 0 .../lib/{libstd => std/src}/sys/sgx/mutex.rs | 0 .../lib/{libstd => std/src}/sys/sgx/net.rs | 0 .../lib/{libstd => std/src}/sys/sgx/os.rs | 0 .../lib/{libstd => std/src}/sys/sgx/path.rs | 0 .../lib/{libstd => std/src}/sys/sgx/pipe.rs | 0 .../{libstd => std/src}/sys/sgx/process.rs | 0 .../lib/{libstd => std/src}/sys/sgx/rwlock.rs | 0 .../src}/sys/sgx/stack_overflow.rs | 0 .../lib/{libstd => std/src}/sys/sgx/stdio.rs | 0 .../lib/{libstd => std/src}/sys/sgx/thread.rs | 0 .../src}/sys/sgx/thread_local.rs | 0 .../lib/{libstd => std/src}/sys/sgx/time.rs | 0 .../{libstd => std/src}/sys/sgx/waitqueue.rs | 0 .../lib/{libstd => std/src}/sys/unix/alloc.rs | 0 .../{libstd => std/src}/sys/unix/android.rs | 0 .../lib/{libstd => std/src}/sys/unix/args.rs | 0 .../lib/{libstd => std/src}/sys/unix/cmath.rs | 0 .../{libstd => std/src}/sys/unix/condvar.rs | 0 .../lib/{libstd => std/src}/sys/unix/env.rs | 0 .../{libstd => std/src}/sys/unix/ext/ffi.rs | 0 .../{libstd => std/src}/sys/unix/ext/fs.rs | 0 .../{libstd => std/src}/sys/unix/ext/io.rs | 0 .../{libstd => std/src}/sys/unix/ext/mod.rs | 0 .../{libstd => std/src}/sys/unix/ext/net.rs | 0 .../src}/sys/unix/ext/process.rs | 0 .../{libstd => std/src}/sys/unix/ext/raw.rs | 0 .../src}/sys/unix/ext/thread.rs | 0 .../src}/sys/unix/fast_thread_local.rs | 0 .../lib/{libstd => std/src}/sys/unix/fd.rs | 0 .../lib/{libstd => std/src}/sys/unix/fs.rs | 0 .../lib/{libstd => std/src}/sys/unix/io.rs | 0 .../lib/{libstd => std/src}/sys/unix/l4re.rs | 0 .../{libstd => std/src}/sys/unix/memchr.rs | 0 .../lib/{libstd => std/src}/sys/unix/mod.rs | 0 .../lib/{libstd => std/src}/sys/unix/mutex.rs | 0 .../lib/{libstd => std/src}/sys/unix/net.rs | 0 .../lib/{libstd => std/src}/sys/unix/os.rs | 0 .../lib/{libstd => std/src}/sys/unix/path.rs | 0 .../lib/{libstd => std/src}/sys/unix/pipe.rs | 0 .../src}/sys/unix/process/mod.rs | 0 .../src}/sys/unix/process/process_common.rs | 0 .../src}/sys/unix/process/process_fuchsia.rs | 0 .../src}/sys/unix/process/process_unix.rs | 0 .../src}/sys/unix/process/zircon.rs | 0 .../lib/{libstd => std/src}/sys/unix/rand.rs | 0 .../{libstd => std/src}/sys/unix/rwlock.rs | 0 .../src}/sys/unix/stack_overflow.rs | 0 .../lib/{libstd => std/src}/sys/unix/stdio.rs | 0 .../{libstd => std/src}/sys/unix/thread.rs | 0 .../src}/sys/unix/thread_local.rs | 0 .../lib/{libstd => std/src}/sys/unix/time.rs | 0 .../lib/{libstd => std/src}/sys/unix/weak.rs | 0 .../{libstd => std/src}/sys/vxworks/alloc.rs | 0 .../{libstd => std/src}/sys/vxworks/args.rs | 0 .../{libstd => std/src}/sys/vxworks/cmath.rs | 0 .../src}/sys/vxworks/condvar.rs | 0 .../{libstd => std/src}/sys/vxworks/env.rs | 0 .../src}/sys/vxworks/ext/ffi.rs | 0 .../{libstd => std/src}/sys/vxworks/ext/fs.rs | 0 .../{libstd => std/src}/sys/vxworks/ext/io.rs | 0 .../src}/sys/vxworks/ext/mod.rs | 0 .../src}/sys/vxworks/ext/process.rs | 0 .../src}/sys/vxworks/ext/raw.rs | 0 .../src}/sys/vxworks/fast_thread_local.rs | 0 .../lib/{libstd => std/src}/sys/vxworks/fd.rs | 0 .../lib/{libstd => std/src}/sys/vxworks/fs.rs | 0 .../lib/{libstd => std/src}/sys/vxworks/io.rs | 0 .../{libstd => std/src}/sys/vxworks/memchr.rs | 0 .../{libstd => std/src}/sys/vxworks/mod.rs | 0 .../{libstd => std/src}/sys/vxworks/mutex.rs | 0 .../{libstd => std/src}/sys/vxworks/net.rs | 0 .../lib/{libstd => std/src}/sys/vxworks/os.rs | 0 .../{libstd => std/src}/sys/vxworks/path.rs | 0 .../{libstd => std/src}/sys/vxworks/pipe.rs | 0 .../src}/sys/vxworks/process/mod.rs | 0 .../sys/vxworks/process/process_common.rs | 0 .../sys/vxworks/process/process_vxworks.rs | 0 .../{libstd => std/src}/sys/vxworks/rand.rs | 0 .../{libstd => std/src}/sys/vxworks/rwlock.rs | 0 .../src}/sys/vxworks/stack_overflow.rs | 0 .../{libstd => std/src}/sys/vxworks/stdio.rs | 0 .../{libstd => std/src}/sys/vxworks/thread.rs | 0 .../src}/sys/vxworks/thread_local.rs | 0 .../{libstd => std/src}/sys/vxworks/time.rs | 0 .../lib/{libstd => std/src}/sys/wasi/alloc.rs | 0 .../lib/{libstd => std/src}/sys/wasi/args.rs | 0 .../lib/{libstd => std/src}/sys/wasi/env.rs | 0 .../{libstd => std/src}/sys/wasi/ext/ffi.rs | 0 .../{libstd => std/src}/sys/wasi/ext/fs.rs | 0 .../{libstd => std/src}/sys/wasi/ext/io.rs | 0 .../{libstd => std/src}/sys/wasi/ext/mod.rs | 0 .../lib/{libstd => std/src}/sys/wasi/fd.rs | 0 .../lib/{libstd => std/src}/sys/wasi/fs.rs | 0 .../lib/{libstd => std/src}/sys/wasi/io.rs | 0 .../lib/{libstd => std/src}/sys/wasi/mod.rs | 0 .../lib/{libstd => std/src}/sys/wasi/net.rs | 0 .../lib/{libstd => std/src}/sys/wasi/os.rs | 0 .../lib/{libstd => std/src}/sys/wasi/path.rs | 0 .../lib/{libstd => std/src}/sys/wasi/pipe.rs | 0 .../{libstd => std/src}/sys/wasi/process.rs | 0 .../lib/{libstd => std/src}/sys/wasi/stdio.rs | 0 .../{libstd => std/src}/sys/wasi/thread.rs | 0 .../lib/{libstd => std/src}/sys/wasi/time.rs | 0 .../lib/{libstd => std/src}/sys/wasm/alloc.rs | 0 .../lib/{libstd => std/src}/sys/wasm/args.rs | 0 .../lib/{libstd => std/src}/sys/wasm/cmath.rs | 0 .../{libstd => std/src}/sys/wasm/condvar.rs | 0 .../src}/sys/wasm/condvar_atomics.rs | 0 .../lib/{libstd => std/src}/sys/wasm/env.rs | 0 .../src}/sys/wasm/fast_thread_local.rs | 0 .../lib/{libstd => std/src}/sys/wasm/fs.rs | 0 .../lib/{libstd => std/src}/sys/wasm/io.rs | 0 .../{libstd => std/src}/sys/wasm/memchr.rs | 0 .../lib/{libstd => std/src}/sys/wasm/mod.rs | 0 .../lib/{libstd => std/src}/sys/wasm/mutex.rs | 0 .../src}/sys/wasm/mutex_atomics.rs | 0 .../lib/{libstd => std/src}/sys/wasm/net.rs | 0 .../lib/{libstd => std/src}/sys/wasm/os.rs | 0 .../lib/{libstd => std/src}/sys/wasm/path.rs | 0 .../lib/{libstd => std/src}/sys/wasm/pipe.rs | 0 .../{libstd => std/src}/sys/wasm/process.rs | 0 .../{libstd => std/src}/sys/wasm/rwlock.rs | 0 .../src}/sys/wasm/rwlock_atomics.rs | 0 .../src}/sys/wasm/stack_overflow.rs | 0 .../lib/{libstd => std/src}/sys/wasm/stdio.rs | 0 .../{libstd => std/src}/sys/wasm/thread.rs | 0 .../src}/sys/wasm/thread_local.rs | 0 .../lib/{libstd => std/src}/sys/wasm/time.rs | 0 .../{libstd => std/src}/sys/windows/alloc.rs | 0 .../{libstd => std/src}/sys/windows/args.rs | 0 .../lib/{libstd => std/src}/sys/windows/c.rs | 0 .../{libstd => std/src}/sys/windows/cmath.rs | 0 .../{libstd => std/src}/sys/windows/compat.rs | 0 .../src}/sys/windows/condvar.rs | 0 .../{libstd => std/src}/sys/windows/env.rs | 0 .../src}/sys/windows/ext/ffi.rs | 0 .../{libstd => std/src}/sys/windows/ext/fs.rs | 0 .../{libstd => std/src}/sys/windows/ext/io.rs | 0 .../src}/sys/windows/ext/mod.rs | 0 .../src}/sys/windows/ext/process.rs | 0 .../src}/sys/windows/ext/raw.rs | 0 .../src}/sys/windows/ext/thread.rs | 0 .../src}/sys/windows/fast_thread_local.rs | 0 .../lib/{libstd => std/src}/sys/windows/fs.rs | 0 .../{libstd => std/src}/sys/windows/handle.rs | 0 .../lib/{libstd => std/src}/sys/windows/io.rs | 0 .../{libstd => std/src}/sys/windows/memchr.rs | 0 .../{libstd => std/src}/sys/windows/mod.rs | 0 .../{libstd => std/src}/sys/windows/mutex.rs | 0 .../{libstd => std/src}/sys/windows/net.rs | 0 .../lib/{libstd => std/src}/sys/windows/os.rs | 0 .../{libstd => std/src}/sys/windows/os_str.rs | 0 .../{libstd => std/src}/sys/windows/path.rs | 0 .../{libstd => std/src}/sys/windows/pipe.rs | 0 .../src}/sys/windows/process.rs | 0 .../{libstd => std/src}/sys/windows/rand.rs | 0 .../{libstd => std/src}/sys/windows/rwlock.rs | 0 .../src}/sys/windows/stack_overflow.rs | 0 .../src}/sys/windows/stack_overflow_uwp.rs | 0 .../{libstd => std/src}/sys/windows/stdio.rs | 0 .../src}/sys/windows/stdio_uwp.rs | 0 .../{libstd => std/src}/sys/windows/thread.rs | 0 .../src}/sys/windows/thread_local.rs | 0 .../{libstd => std/src}/sys/windows/time.rs | 0 .../{libstd => std/src}/sys_common/alloc.rs | 0 .../src}/sys_common/at_exit_imp.rs | 0 .../src}/sys_common/backtrace.rs | 0 .../src}/sys_common/bytestring.rs | 0 .../{libstd => std/src}/sys_common/condvar.rs | 0 .../lib/{libstd => std/src}/sys_common/fs.rs | 0 .../lib/{libstd => std/src}/sys_common/io.rs | 0 .../lib/{libstd => std/src}/sys_common/mod.rs | 0 .../{libstd => std/src}/sys_common/mutex.rs | 0 .../lib/{libstd => std/src}/sys_common/net.rs | 0 .../src}/sys_common/os_str_bytes.rs | 0 .../{libstd => std/src}/sys_common/poison.rs | 0 .../{libstd => std/src}/sys_common/process.rs | 0 .../{libstd => std/src}/sys_common/remutex.rs | 0 .../{libstd => std/src}/sys_common/rwlock.rs | 0 .../{libstd => std/src}/sys_common/thread.rs | 0 .../src}/sys_common/thread_info.rs | 0 .../src}/sys_common/thread_local.rs | 0 .../{libstd => std/src}/sys_common/util.rs | 0 .../{libstd => std/src}/sys_common/wtf8.rs | 0 .../lib/{libstd => std/src}/thread/local.rs | 0 .../lib/{libstd => std/src}/thread/mod.rs | 0 crux-mir/lib/{libstd => std/src}/time.rs | 0 crux-mir/lib/{libstd => std}/tests/env.rs | 0 .../{libstd => std}/tests/run-time-detect.rs | 0 crux-mir/lib/{libterm => term}/Cargo.toml | 0 crux-mir/lib/{libterm => term/src}/lib.rs | 0 .../lib/{libterm => term/src}/terminfo/mod.rs | 0 .../{libterm => term/src}/terminfo/parm.rs | 0 .../src}/terminfo/parm/tests.rs | 0 .../src}/terminfo/parser/compiled.rs | 0 .../src}/terminfo/parser/compiled/tests.rs | 0 .../src}/terminfo/searcher.rs | 0 .../src}/terminfo/searcher/tests.rs | 0 crux-mir/lib/{libterm => term/src}/win.rs | 0 crux-mir/lib/{libtest => test}/Cargo.toml | 0 crux-mir/lib/{libtest => test/src}/bench.rs | 0 crux-mir/lib/{libtest => test/src}/cli.rs | 0 crux-mir/lib/{libtest => test/src}/console.rs | 0 crux-mir/lib/{libtest => test/src}/event.rs | 0 .../{libtest => test/src}/formatters/json.rs | 0 .../{libtest => test/src}/formatters/mod.rs | 0 .../src}/formatters/pretty.rs | 0 .../{libtest => test/src}/formatters/terse.rs | 0 .../src}/helpers/concurrency.rs | 0 .../src}/helpers/exit_code.rs | 0 .../{libtest => test/src}/helpers/isatty.rs | 0 .../{libtest => test/src}/helpers/metrics.rs | 0 .../lib/{libtest => test/src}/helpers/mod.rs | 0 .../lib/{libtest => test/src}/helpers/sink.rs | 0 crux-mir/lib/{libtest => test/src}/lib.rs | 0 crux-mir/lib/{libtest => test/src}/options.rs | 0 crux-mir/lib/{libtest => test/src}/stats.rs | 0 .../lib/{libtest => test/src}/stats/tests.rs | 0 .../lib/{libtest => test/src}/test_result.rs | 0 crux-mir/lib/{libtest => test/src}/tests.rs | 0 crux-mir/lib/{libtest => test/src}/time.rs | 0 crux-mir/lib/{libtest => test/src}/types.rs | 0 crux-mir/lib/{libunwind => unwind}/Cargo.toml | 0 crux-mir/lib/{libunwind => unwind}/build.rs | 0 crux-mir/lib/{libunwind => unwind}/lib.rs | 0 .../lib/{libunwind => unwind}/libunwind.rs | 0 crux-mir/lib/unwind/src/build.rs | 124 ++++++++ crux-mir/lib/unwind/src/lib.rs | 29 ++ crux-mir/lib/unwind/src/libunwind.rs | 279 ++++++++++++++++++ 772 files changed, 558 insertions(+) rename crux-mir/lib/{liballoc => alloc}/benches/btree/map.rs (100%) rename crux-mir/lib/{liballoc => alloc}/benches/btree/mod.rs (100%) rename crux-mir/lib/{liballoc => alloc}/benches/btree/set.rs (100%) rename crux-mir/lib/{liballoc => alloc}/benches/lib.rs (100%) rename crux-mir/lib/{liballoc => alloc}/benches/linked_list.rs (100%) rename crux-mir/lib/{liballoc => alloc}/benches/slice.rs (100%) rename crux-mir/lib/{liballoc => alloc}/benches/str.rs (100%) rename crux-mir/lib/{liballoc => alloc}/benches/string.rs (100%) rename crux-mir/lib/{liballoc => alloc}/benches/vec.rs (100%) rename crux-mir/lib/{liballoc => alloc}/benches/vec_deque.rs (100%) rename crux-mir/lib/{liballoc => alloc}/benches/vec_deque_append.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/Cargo.toml (100%) rename crux-mir/lib/{liballoc => alloc/src}/alloc.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/alloc/tests.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/borrow.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/boxed.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/collections/binary_heap.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/collections/btree/map.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/collections/btree/mod.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/collections/btree/navigate.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/collections/btree/node.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/collections/btree/search.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/collections/btree/set.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/collections/linked_list.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/collections/linked_list/tests.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/collections/mod.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/collections/vec_deque.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/collections/vec_deque/drain.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/collections/vec_deque/tests.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/fmt.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/lib.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/macros.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/prelude/mod.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/prelude/v1.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/raw_vec.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/raw_vec/tests.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/rc.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/rc/tests.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/slice.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/str.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/string.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/sync.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/sync/tests.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/tests.rs (100%) rename crux-mir/lib/{liballoc => alloc/src}/vec.rs (100%) rename crux-mir/lib/{liballoc => alloc}/tests/arc.rs (100%) rename crux-mir/lib/{liballoc => alloc}/tests/binary_heap.rs (100%) rename crux-mir/lib/{liballoc => alloc}/tests/boxed.rs (100%) rename crux-mir/lib/{liballoc => alloc}/tests/btree/map.rs (100%) rename crux-mir/lib/{liballoc => alloc}/tests/btree/mod.rs (100%) rename crux-mir/lib/{liballoc => alloc}/tests/btree/set.rs (100%) rename crux-mir/lib/{liballoc => alloc}/tests/cow_str.rs (100%) rename crux-mir/lib/{liballoc => alloc}/tests/fmt.rs (100%) rename crux-mir/lib/{liballoc => alloc}/tests/heap.rs (100%) rename crux-mir/lib/{liballoc => alloc}/tests/lib.rs (100%) rename crux-mir/lib/{liballoc => alloc}/tests/linked_list.rs (100%) rename crux-mir/lib/{liballoc => alloc}/tests/rc.rs (100%) rename crux-mir/lib/{liballoc => alloc}/tests/slice.rs (100%) rename crux-mir/lib/{liballoc => alloc}/tests/str.rs (100%) rename crux-mir/lib/{liballoc => alloc}/tests/string.rs (100%) rename crux-mir/lib/{liballoc => alloc}/tests/vec.rs (100%) rename crux-mir/lib/{liballoc => alloc}/tests/vec_deque.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/Cargo.toml (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/LICENSE.TXT (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/PUBLISHING.md (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/README.md (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/build.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/ci/docker/aarch64-unknown-linux-gnu/Dockerfile (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/ci/docker/arm-unknown-linux-gnueabi/Dockerfile (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/ci/docker/i586-unknown-linux-gnu/Dockerfile (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/ci/docker/i686-unknown-linux-gnu/Dockerfile (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/ci/docker/mips-unknown-linux-gnu/Dockerfile (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/ci/docker/mipsel-unknown-linux-gnu/Dockerfile (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/ci/docker/powerpc-unknown-linux-gnu/Dockerfile (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/ci/docker/thumbv6m-none-eabi/Dockerfile (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/ci/docker/thumbv7em-none-eabi/Dockerfile (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/ci/docker/thumbv7em-none-eabihf/Dockerfile (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/ci/docker/thumbv7m-none-eabi/Dockerfile (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/ci/docker/x86_64-unknown-linux-gnu/Dockerfile (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/ci/run-docker.sh (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/ci/run.sh (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/crates/panic-handler/Cargo.toml (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/crates/panic-handler/src/lib.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/examples/intrinsics.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/arm.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/arm_linux.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/float/add.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/float/cmp.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/float/conv.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/float/div.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/float/extend.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/float/mod.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/float/mul.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/float/pow.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/float/sub.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/int/addsub.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/int/mod.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/int/mul.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/int/sdiv.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/int/shift.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/int/udiv.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/lib.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/macros.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/math.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/mem.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/probestack.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/riscv32.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/x86.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/src/x86_64.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/test.c (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/testcrate/Cargo.toml (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/testcrate/build.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/testcrate/src/lib.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/testcrate/tests/aeabi_memclr.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/testcrate/tests/aeabi_memcpy.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/testcrate/tests/aeabi_memset.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/testcrate/tests/count_leading_zeros.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/testcrate/tests/generated.rs (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/thumbv6m-linux-eabi.json (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/thumbv7em-linux-eabi.json (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/thumbv7em-linux-eabihf.json (100%) rename crux-mir/lib/{compiler-builtins => compiler_builtins}/thumbv7m-linux-eabi.json (100%) rename crux-mir/lib/{libcore => core}/benches/any.rs (100%) rename crux-mir/lib/{libcore => core}/benches/ascii.rs (100%) rename crux-mir/lib/{libcore => core}/benches/char/methods.rs (100%) rename crux-mir/lib/{libcore => core}/benches/char/mod.rs (100%) rename crux-mir/lib/{libcore => core}/benches/fmt.rs (100%) rename crux-mir/lib/{libcore => core}/benches/hash/mod.rs (100%) rename crux-mir/lib/{libcore => core}/benches/hash/sip.rs (100%) rename crux-mir/lib/{libcore => core}/benches/iter.rs (100%) rename crux-mir/lib/{libcore => core}/benches/lib.rs (100%) rename crux-mir/lib/{libcore => core}/benches/num/dec2flt/mod.rs (100%) rename crux-mir/lib/{libcore => core}/benches/num/flt2dec/mod.rs (100%) rename crux-mir/lib/{libcore => core}/benches/num/flt2dec/strategy/dragon.rs (100%) rename crux-mir/lib/{libcore => core}/benches/num/flt2dec/strategy/grisu.rs (100%) rename crux-mir/lib/{libcore => core}/benches/num/mod.rs (100%) rename crux-mir/lib/{libcore => core}/benches/ops.rs (100%) rename crux-mir/lib/{libcore => core}/benches/pattern.rs (100%) rename crux-mir/lib/{libcore => core}/benches/slice.rs (100%) rename crux-mir/lib/{libcore => core/src}/Cargo.toml (100%) rename crux-mir/lib/{libcore => core/src}/alloc.rs (100%) rename crux-mir/lib/{libcore => core/src}/any.rs (100%) rename crux-mir/lib/{libcore => core/src}/array/iter.rs (100%) rename crux-mir/lib/{libcore => core/src}/array/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/ascii.rs (100%) rename crux-mir/lib/{libcore => core/src}/bool.rs (100%) rename crux-mir/lib/{libcore => core/src}/borrow.rs (100%) rename crux-mir/lib/{libcore => core/src}/cell.rs (100%) rename crux-mir/lib/{libcore => core/src}/char/convert.rs (100%) rename crux-mir/lib/{libcore => core/src}/char/decode.rs (100%) rename crux-mir/lib/{libcore => core/src}/char/methods.rs (100%) rename crux-mir/lib/{libcore => core/src}/char/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/clone.rs (100%) rename crux-mir/lib/{libcore => core/src}/cmp.rs (100%) rename crux-mir/lib/{libcore => core/src}/convert/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/convert/num.rs (100%) rename crux-mir/lib/{libcore => core/src}/crucible/any.rs (100%) rename crux-mir/lib/{libcore => core/src}/crucible/concurrency.rs (100%) rename crux-mir/lib/{libcore => core/src}/crucible/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/crucible/ptr.rs (100%) rename crux-mir/lib/{libcore => core/src}/default.rs (100%) rename crux-mir/lib/{libcore => core/src}/ffi.rs (100%) rename crux-mir/lib/{libcore => core/src}/fmt/builders.rs (100%) rename crux-mir/lib/{libcore => core/src}/fmt/float.rs (100%) rename crux-mir/lib/{libcore => core/src}/fmt/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/fmt/num.rs (100%) rename crux-mir/lib/{libcore => core/src}/fmt/rt/v1.rs (100%) rename crux-mir/lib/{libcore => core/src}/future/future.rs (100%) rename crux-mir/lib/{libcore => core/src}/future/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/hash/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/hash/sip.rs (100%) rename crux-mir/lib/{libcore => core/src}/hint.rs (100%) rename crux-mir/lib/{libcore => core/src}/internal_macros.rs (100%) rename crux-mir/lib/{libcore => core/src}/intrinsics.rs (100%) rename crux-mir/lib/{libcore => core/src}/iter/adapters/chain.rs (100%) rename crux-mir/lib/{libcore => core/src}/iter/adapters/flatten.rs (100%) rename crux-mir/lib/{libcore => core/src}/iter/adapters/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/iter/adapters/zip.rs (100%) rename crux-mir/lib/{libcore => core/src}/iter/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/iter/range.rs (100%) rename crux-mir/lib/{libcore => core/src}/iter/sources.rs (100%) rename crux-mir/lib/{libcore => core/src}/iter/traits/accum.rs (100%) rename crux-mir/lib/{libcore => core/src}/iter/traits/collect.rs (100%) rename crux-mir/lib/{libcore => core/src}/iter/traits/double_ended.rs (100%) rename crux-mir/lib/{libcore => core/src}/iter/traits/exact_size.rs (100%) rename crux-mir/lib/{libcore => core/src}/iter/traits/iterator.rs (100%) rename crux-mir/lib/{libcore => core/src}/iter/traits/marker.rs (100%) rename crux-mir/lib/{libcore => core/src}/iter/traits/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/lib.rs (100%) rename crux-mir/lib/{libcore => core/src}/macros/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/macros/panic.md (100%) rename crux-mir/lib/{libcore => core/src}/marker.rs (100%) rename crux-mir/lib/{libcore => core/src}/mem/manually_drop.rs (100%) rename crux-mir/lib/{libcore => core/src}/mem/maybe_uninit.rs (100%) rename crux-mir/lib/{libcore => core/src}/mem/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/bignum.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/dec2flt/algorithm.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/dec2flt/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/dec2flt/num.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/dec2flt/parse.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/dec2flt/rawfp.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/dec2flt/table.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/diy_float.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/f32.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/f64.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/flt2dec/decoder.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/flt2dec/estimator.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/flt2dec/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/flt2dec/strategy/dragon.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/flt2dec/strategy/grisu.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/i128.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/i16.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/i32.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/i64.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/i8.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/int_macros.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/isize.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/u128.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/u16.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/u32.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/u64.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/u8.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/usize.rs (100%) rename crux-mir/lib/{libcore => core/src}/num/wrapping.rs (100%) rename crux-mir/lib/{libcore => core/src}/ops/arith.rs (100%) rename crux-mir/lib/{libcore => core/src}/ops/bit.rs (100%) rename crux-mir/lib/{libcore => core/src}/ops/deref.rs (100%) rename crux-mir/lib/{libcore => core/src}/ops/drop.rs (100%) rename crux-mir/lib/{libcore => core/src}/ops/function.rs (100%) rename crux-mir/lib/{libcore => core/src}/ops/generator.rs (100%) rename crux-mir/lib/{libcore => core/src}/ops/index.rs (100%) rename crux-mir/lib/{libcore => core/src}/ops/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/ops/range.rs (100%) rename crux-mir/lib/{libcore => core/src}/ops/try.rs (100%) rename crux-mir/lib/{libcore => core/src}/ops/unsize.rs (100%) rename crux-mir/lib/{libcore => core/src}/option.rs (100%) rename crux-mir/lib/{libcore => core/src}/panic.rs (100%) rename crux-mir/lib/{libcore => core/src}/panicking.rs (100%) rename crux-mir/lib/{libcore => core/src}/pin.rs (100%) rename crux-mir/lib/{libcore => core/src}/prelude/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/prelude/v1.rs (100%) rename crux-mir/lib/{libcore => core/src}/primitive.rs (100%) rename crux-mir/lib/{libcore => core/src}/ptr/const_ptr.rs (100%) rename crux-mir/lib/{libcore => core/src}/ptr/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/ptr/mut_ptr.rs (100%) rename crux-mir/lib/{libcore => core/src}/ptr/non_null.rs (100%) rename crux-mir/lib/{libcore => core/src}/ptr/unique.rs (100%) rename crux-mir/lib/{libcore => core/src}/raw.rs (100%) rename crux-mir/lib/{libcore => core/src}/result.rs (100%) rename crux-mir/lib/{libcore => core/src}/slice/memchr.rs (100%) rename crux-mir/lib/{libcore => core/src}/slice/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/slice/rotate.rs (100%) rename crux-mir/lib/{libcore => core/src}/slice/sort.rs (100%) rename crux-mir/lib/{libcore => core/src}/str/lossy.rs (100%) rename crux-mir/lib/{libcore => core/src}/str/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/str/pattern.rs (100%) rename crux-mir/lib/{libcore => core/src}/sync/atomic.rs (100%) rename crux-mir/lib/{libcore => core/src}/sync/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/task/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/task/poll.rs (100%) rename crux-mir/lib/{libcore => core/src}/task/wake.rs (100%) rename crux-mir/lib/{libcore => core/src}/time.rs (100%) rename crux-mir/lib/{libcore => core/src}/tuple.rs (100%) rename crux-mir/lib/{libcore => core/src}/unicode/mod.rs (100%) rename crux-mir/lib/{libcore => core/src}/unicode/printable.py (100%) rename crux-mir/lib/{libcore => core/src}/unicode/printable.rs (100%) rename crux-mir/lib/{libcore => core/src}/unicode/unicode_data.rs (100%) rename crux-mir/lib/{libcore => core/src}/unicode/version.rs (100%) rename crux-mir/lib/{libcore => core/src}/unit.rs (100%) rename crux-mir/lib/{libcore => core}/tests/alloc.rs (100%) rename crux-mir/lib/{libcore => core}/tests/any.rs (100%) rename crux-mir/lib/{libcore => core}/tests/array.rs (100%) rename crux-mir/lib/{libcore => core}/tests/ascii.rs (100%) rename crux-mir/lib/{libcore => core}/tests/atomic.rs (100%) rename crux-mir/lib/{libcore => core}/tests/bool.rs (100%) rename crux-mir/lib/{libcore => core}/tests/cell.rs (100%) rename crux-mir/lib/{libcore => core}/tests/char.rs (100%) rename crux-mir/lib/{libcore => core}/tests/clone.rs (100%) rename crux-mir/lib/{libcore => core}/tests/cmp.rs (100%) rename crux-mir/lib/{libcore => core}/tests/fmt/builders.rs (100%) rename crux-mir/lib/{libcore => core}/tests/fmt/float.rs (100%) rename crux-mir/lib/{libcore => core}/tests/fmt/mod.rs (100%) rename crux-mir/lib/{libcore => core}/tests/fmt/num.rs (100%) rename crux-mir/lib/{libcore => core}/tests/hash/mod.rs (100%) rename crux-mir/lib/{libcore => core}/tests/hash/sip.rs (100%) rename crux-mir/lib/{libcore => core}/tests/intrinsics.rs (100%) rename crux-mir/lib/{libcore => core}/tests/iter.rs (100%) rename crux-mir/lib/{libcore => core}/tests/lib.rs (100%) rename crux-mir/lib/{libcore => core}/tests/manually_drop.rs (100%) rename crux-mir/lib/{libcore => core}/tests/mem.rs (100%) rename crux-mir/lib/{libcore => core}/tests/nonzero.rs (100%) rename crux-mir/lib/{libcore => core}/tests/num/bignum.rs (100%) rename crux-mir/lib/{libcore => core}/tests/num/dec2flt/mod.rs (100%) rename crux-mir/lib/{libcore => core}/tests/num/dec2flt/parse.rs (100%) rename crux-mir/lib/{libcore => core}/tests/num/dec2flt/rawfp.rs (100%) rename crux-mir/lib/{libcore => core}/tests/num/flt2dec/estimator.rs (100%) rename crux-mir/lib/{libcore => core}/tests/num/flt2dec/mod.rs (100%) rename crux-mir/lib/{libcore => core}/tests/num/flt2dec/random.rs (100%) rename crux-mir/lib/{libcore => core}/tests/num/flt2dec/strategy/dragon.rs (100%) rename crux-mir/lib/{libcore => core}/tests/num/flt2dec/strategy/grisu.rs (100%) rename crux-mir/lib/{libcore => core}/tests/num/i16.rs (100%) rename crux-mir/lib/{libcore => core}/tests/num/i32.rs (100%) rename crux-mir/lib/{libcore => core}/tests/num/i64.rs (100%) rename crux-mir/lib/{libcore => core}/tests/num/i8.rs (100%) rename crux-mir/lib/{libcore => core}/tests/num/int_macros.rs (100%) rename crux-mir/lib/{libcore => core}/tests/num/mod.rs (100%) rename crux-mir/lib/{libcore => core}/tests/num/u16.rs (100%) rename crux-mir/lib/{libcore => core}/tests/num/u32.rs (100%) rename crux-mir/lib/{libcore => core}/tests/num/u64.rs (100%) rename crux-mir/lib/{libcore => core}/tests/num/u8.rs (100%) rename crux-mir/lib/{libcore => core}/tests/num/uint_macros.rs (100%) rename crux-mir/lib/{libcore => core}/tests/ops.rs (100%) rename crux-mir/lib/{libcore => core}/tests/option.rs (100%) rename crux-mir/lib/{libcore => core}/tests/pattern.rs (100%) rename crux-mir/lib/{libcore => core}/tests/ptr.rs (100%) rename crux-mir/lib/{libcore => core}/tests/result.rs (100%) rename crux-mir/lib/{libcore => core}/tests/slice.rs (100%) rename crux-mir/lib/{libcore => core}/tests/str.rs (100%) rename crux-mir/lib/{libcore => core}/tests/str_lossy.rs (100%) rename crux-mir/lib/{libcore => core}/tests/time.rs (100%) rename crux-mir/lib/{libcore => core}/tests/tuple.rs (100%) rename crux-mir/lib/{libpanic_abort => panic_abort}/Cargo.toml (100%) rename crux-mir/lib/{libpanic_abort => panic_abort}/lib.rs (100%) create mode 100644 crux-mir/lib/panic_abort/src/lib.rs rename crux-mir/lib/{libpanic_unwind => panic_unwind}/Cargo.toml (100%) rename crux-mir/lib/{libpanic_unwind => panic_unwind}/dummy.rs (100%) rename crux-mir/lib/{libpanic_unwind => panic_unwind}/dwarf/eh.rs (100%) rename crux-mir/lib/{libpanic_unwind => panic_unwind}/dwarf/mod.rs (100%) rename crux-mir/lib/{libpanic_unwind => panic_unwind}/dwarf/tests.rs (100%) rename crux-mir/lib/{libpanic_unwind => panic_unwind}/emcc.rs (100%) rename crux-mir/lib/{libpanic_unwind => panic_unwind}/gcc.rs (100%) rename crux-mir/lib/{libpanic_unwind => panic_unwind}/hermit.rs (100%) rename crux-mir/lib/{libpanic_unwind => panic_unwind}/lib.rs (100%) rename crux-mir/lib/{libpanic_unwind => panic_unwind}/miri.rs (100%) rename crux-mir/lib/{libpanic_unwind => panic_unwind}/seh.rs (100%) rename crux-mir/lib/{libstd => std}/benches/hash/map.rs (100%) rename crux-mir/lib/{libstd => std}/benches/hash/mod.rs (100%) rename crux-mir/lib/{libstd => std}/benches/hash/set_ops.rs (100%) rename crux-mir/lib/{libstd => std}/benches/lib.rs (100%) rename crux-mir/lib/{libstd => std/src}/Cargo.toml (100%) rename crux-mir/lib/{libstd => std/src}/alloc.rs (100%) rename crux-mir/lib/{libstd => std/src}/ascii.rs (100%) rename crux-mir/lib/{libstd => std/src}/backtrace.rs (100%) rename crux-mir/lib/{libstd => std/src}/build.rs (100%) rename crux-mir/lib/{libstd => std/src}/collections/hash/crucible_map.rs (100%) rename crux-mir/lib/{libstd => std/src}/collections/hash/map.rs (100%) rename crux-mir/lib/{libstd => std/src}/collections/hash/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/collections/hash/set.rs (100%) rename crux-mir/lib/{libstd => std/src}/collections/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/env.rs (100%) rename crux-mir/lib/{libstd => std/src}/error.rs (100%) rename crux-mir/lib/{libstd => std/src}/f32.rs (100%) rename crux-mir/lib/{libstd => std/src}/f64.rs (100%) rename crux-mir/lib/{libstd => std/src}/ffi/c_str.rs (100%) rename crux-mir/lib/{libstd => std/src}/ffi/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/ffi/os_str.rs (100%) rename crux-mir/lib/{libstd => std/src}/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/future.rs (100%) rename crux-mir/lib/{libstd => std/src}/io/buffered.rs (100%) rename crux-mir/lib/{libstd => std/src}/io/cursor.rs (100%) rename crux-mir/lib/{libstd => std/src}/io/error.rs (100%) rename crux-mir/lib/{libstd => std/src}/io/impls.rs (100%) rename crux-mir/lib/{libstd => std/src}/io/lazy.rs (100%) rename crux-mir/lib/{libstd => std/src}/io/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/io/prelude.rs (100%) rename crux-mir/lib/{libstd => std/src}/io/stdio.rs (100%) rename crux-mir/lib/{libstd => std/src}/io/util.rs (100%) rename crux-mir/lib/{libstd => std/src}/keyword_docs.rs (100%) rename crux-mir/lib/{libstd => std/src}/lib.rs (100%) rename crux-mir/lib/{libstd => std/src}/macros.rs (100%) rename crux-mir/lib/{libstd => std/src}/memchr.rs (100%) rename crux-mir/lib/{libstd => std/src}/net/addr.rs (100%) rename crux-mir/lib/{libstd => std/src}/net/ip.rs (100%) rename crux-mir/lib/{libstd => std/src}/net/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/net/parser.rs (100%) rename crux-mir/lib/{libstd => std/src}/net/tcp.rs (100%) rename crux-mir/lib/{libstd => std/src}/net/test.rs (100%) rename crux-mir/lib/{libstd => std/src}/net/udp.rs (100%) rename crux-mir/lib/{libstd => std/src}/num.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/android/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/android/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/android/raw.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/dragonfly/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/dragonfly/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/dragonfly/raw.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/emscripten/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/emscripten/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/emscripten/raw.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/fortanix_sgx/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/freebsd/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/freebsd/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/freebsd/raw.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/fuchsia/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/fuchsia/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/fuchsia/raw.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/haiku/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/haiku/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/haiku/raw.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/ios/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/ios/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/ios/raw.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/linux/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/linux/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/linux/raw.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/macos/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/macos/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/macos/raw.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/netbsd/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/netbsd/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/netbsd/raw.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/openbsd/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/openbsd/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/openbsd/raw.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/raw/char.md (100%) rename crux-mir/lib/{libstd => std/src}/os/raw/double.md (100%) rename crux-mir/lib/{libstd => std/src}/os/raw/float.md (100%) rename crux-mir/lib/{libstd => std/src}/os/raw/int.md (100%) rename crux-mir/lib/{libstd => std/src}/os/raw/long.md (100%) rename crux-mir/lib/{libstd => std/src}/os/raw/longlong.md (100%) rename crux-mir/lib/{libstd => std/src}/os/raw/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/raw/schar.md (100%) rename crux-mir/lib/{libstd => std/src}/os/raw/short.md (100%) rename crux-mir/lib/{libstd => std/src}/os/raw/uchar.md (100%) rename crux-mir/lib/{libstd => std/src}/os/raw/uint.md (100%) rename crux-mir/lib/{libstd => std/src}/os/raw/ulong.md (100%) rename crux-mir/lib/{libstd => std/src}/os/raw/ulonglong.md (100%) rename crux-mir/lib/{libstd => std/src}/os/raw/ushort.md (100%) rename crux-mir/lib/{libstd => std/src}/os/redox/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/redox/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/redox/raw.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/solaris/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/solaris/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/solaris/raw.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/vxworks/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/vxworks/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/vxworks/raw.rs (100%) rename crux-mir/lib/{libstd => std/src}/os/wasi.rs (100%) rename crux-mir/lib/{libstd => std/src}/panic.rs (100%) rename crux-mir/lib/{libstd => std/src}/panicking.rs (100%) rename crux-mir/lib/{libstd => std/src}/path.rs (100%) rename crux-mir/lib/{libstd => std/src}/prelude/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/prelude/v1.rs (100%) rename crux-mir/lib/{libstd => std/src}/primitive_docs.rs (100%) rename crux-mir/lib/{libstd => std/src}/process.rs (100%) rename crux-mir/lib/{libstd => std/src}/rt.rs (100%) rename crux-mir/lib/{libstd => std/src}/sync/barrier.rs (100%) rename crux-mir/lib/{libstd => std/src}/sync/condvar.rs (100%) rename crux-mir/lib/{libstd => std/src}/sync/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sync/mpsc/blocking.rs (100%) rename crux-mir/lib/{libstd => std/src}/sync/mpsc/cache_aligned.rs (100%) rename crux-mir/lib/{libstd => std/src}/sync/mpsc/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sync/mpsc/mpsc_queue.rs (100%) rename crux-mir/lib/{libstd => std/src}/sync/mpsc/oneshot.rs (100%) rename crux-mir/lib/{libstd => std/src}/sync/mpsc/shared.rs (100%) rename crux-mir/lib/{libstd => std/src}/sync/mpsc/spsc_queue.rs (100%) rename crux-mir/lib/{libstd => std/src}/sync/mpsc/stream.rs (100%) rename crux-mir/lib/{libstd => std/src}/sync/mpsc/sync.rs (100%) rename crux-mir/lib/{libstd => std/src}/sync/mutex.rs (100%) rename crux-mir/lib/{libstd => std/src}/sync/once.rs (100%) rename crux-mir/lib/{libstd => std/src}/sync/rwlock.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/abi/bitflags.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/abi/cloudabi.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/abi/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/args.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/condvar.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/io.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/mutex.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/os.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/rwlock.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/shims/args.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/shims/env.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/shims/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/shims/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/shims/net.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/shims/os.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/shims/pipe.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/shims/process.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/stack_overflow.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/stdio.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/thread.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/cloudabi/time.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/crux/condvar.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/crux/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/crux/mutex.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/crux/rwlock.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/crux/time.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/alloc.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/args.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/cmath.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/condvar.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/env.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/fast_thread_local.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/fd.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/io.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/memchr.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/mutex.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/net.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/os.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/path.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/pipe.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/process.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/rwlock.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/stack_overflow.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/stdio.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/thread.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/thread_local.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/hermit/time.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/abi/entry.S (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/abi/mem.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/abi/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/abi/panic.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/abi/reloc.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/abi/thread.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/abi/tls.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/abi/usercalls/alloc.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/abi/usercalls/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/abi/usercalls/raw.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/alloc.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/args.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/cmath.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/condvar.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/env.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/ext/arch.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/ext/ffi.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/ext/io.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/ext/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/fd.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/io.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/memchr.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/mutex.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/net.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/os.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/path.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/pipe.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/process.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/rwlock.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/stack_overflow.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/stdio.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/thread.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/thread_local.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/time.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/sgx/waitqueue.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/alloc.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/android.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/args.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/cmath.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/condvar.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/env.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/ext/ffi.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/ext/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/ext/io.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/ext/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/ext/net.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/ext/process.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/ext/raw.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/ext/thread.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/fast_thread_local.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/fd.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/io.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/l4re.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/memchr.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/mutex.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/net.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/os.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/path.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/pipe.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/process/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/process/process_common.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/process/process_fuchsia.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/process/process_unix.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/process/zircon.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/rand.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/rwlock.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/stack_overflow.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/stdio.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/thread.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/thread_local.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/time.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/unix/weak.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/alloc.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/args.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/cmath.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/condvar.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/env.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/ext/ffi.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/ext/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/ext/io.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/ext/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/ext/process.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/ext/raw.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/fast_thread_local.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/fd.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/io.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/memchr.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/mutex.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/net.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/os.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/path.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/pipe.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/process/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/process/process_common.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/process/process_vxworks.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/rand.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/rwlock.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/stack_overflow.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/stdio.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/thread.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/thread_local.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/vxworks/time.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasi/alloc.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasi/args.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasi/env.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasi/ext/ffi.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasi/ext/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasi/ext/io.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasi/ext/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasi/fd.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasi/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasi/io.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasi/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasi/net.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasi/os.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasi/path.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasi/pipe.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasi/process.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasi/stdio.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasi/thread.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasi/time.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/alloc.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/args.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/cmath.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/condvar.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/condvar_atomics.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/env.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/fast_thread_local.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/io.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/memchr.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/mutex.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/mutex_atomics.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/net.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/os.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/path.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/pipe.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/process.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/rwlock.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/rwlock_atomics.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/stack_overflow.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/stdio.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/thread.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/thread_local.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/wasm/time.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/alloc.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/args.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/c.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/cmath.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/compat.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/condvar.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/env.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/ext/ffi.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/ext/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/ext/io.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/ext/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/ext/process.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/ext/raw.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/ext/thread.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/fast_thread_local.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/handle.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/io.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/memchr.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/mutex.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/net.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/os.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/os_str.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/path.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/pipe.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/process.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/rand.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/rwlock.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/stack_overflow.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/stack_overflow_uwp.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/stdio.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/stdio_uwp.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/thread.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/thread_local.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys/windows/time.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys_common/alloc.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys_common/at_exit_imp.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys_common/backtrace.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys_common/bytestring.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys_common/condvar.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys_common/fs.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys_common/io.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys_common/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys_common/mutex.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys_common/net.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys_common/os_str_bytes.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys_common/poison.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys_common/process.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys_common/remutex.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys_common/rwlock.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys_common/thread.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys_common/thread_info.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys_common/thread_local.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys_common/util.rs (100%) rename crux-mir/lib/{libstd => std/src}/sys_common/wtf8.rs (100%) rename crux-mir/lib/{libstd => std/src}/thread/local.rs (100%) rename crux-mir/lib/{libstd => std/src}/thread/mod.rs (100%) rename crux-mir/lib/{libstd => std/src}/time.rs (100%) rename crux-mir/lib/{libstd => std}/tests/env.rs (100%) rename crux-mir/lib/{libstd => std}/tests/run-time-detect.rs (100%) rename crux-mir/lib/{libterm => term}/Cargo.toml (100%) rename crux-mir/lib/{libterm => term/src}/lib.rs (100%) rename crux-mir/lib/{libterm => term/src}/terminfo/mod.rs (100%) rename crux-mir/lib/{libterm => term/src}/terminfo/parm.rs (100%) rename crux-mir/lib/{libterm => term/src}/terminfo/parm/tests.rs (100%) rename crux-mir/lib/{libterm => term/src}/terminfo/parser/compiled.rs (100%) rename crux-mir/lib/{libterm => term/src}/terminfo/parser/compiled/tests.rs (100%) rename crux-mir/lib/{libterm => term/src}/terminfo/searcher.rs (100%) rename crux-mir/lib/{libterm => term/src}/terminfo/searcher/tests.rs (100%) rename crux-mir/lib/{libterm => term/src}/win.rs (100%) rename crux-mir/lib/{libtest => test}/Cargo.toml (100%) rename crux-mir/lib/{libtest => test/src}/bench.rs (100%) rename crux-mir/lib/{libtest => test/src}/cli.rs (100%) rename crux-mir/lib/{libtest => test/src}/console.rs (100%) rename crux-mir/lib/{libtest => test/src}/event.rs (100%) rename crux-mir/lib/{libtest => test/src}/formatters/json.rs (100%) rename crux-mir/lib/{libtest => test/src}/formatters/mod.rs (100%) rename crux-mir/lib/{libtest => test/src}/formatters/pretty.rs (100%) rename crux-mir/lib/{libtest => test/src}/formatters/terse.rs (100%) rename crux-mir/lib/{libtest => test/src}/helpers/concurrency.rs (100%) rename crux-mir/lib/{libtest => test/src}/helpers/exit_code.rs (100%) rename crux-mir/lib/{libtest => test/src}/helpers/isatty.rs (100%) rename crux-mir/lib/{libtest => test/src}/helpers/metrics.rs (100%) rename crux-mir/lib/{libtest => test/src}/helpers/mod.rs (100%) rename crux-mir/lib/{libtest => test/src}/helpers/sink.rs (100%) rename crux-mir/lib/{libtest => test/src}/lib.rs (100%) rename crux-mir/lib/{libtest => test/src}/options.rs (100%) rename crux-mir/lib/{libtest => test/src}/stats.rs (100%) rename crux-mir/lib/{libtest => test/src}/stats/tests.rs (100%) rename crux-mir/lib/{libtest => test/src}/test_result.rs (100%) rename crux-mir/lib/{libtest => test/src}/tests.rs (100%) rename crux-mir/lib/{libtest => test/src}/time.rs (100%) rename crux-mir/lib/{libtest => test/src}/types.rs (100%) rename crux-mir/lib/{libunwind => unwind}/Cargo.toml (100%) rename crux-mir/lib/{libunwind => unwind}/build.rs (100%) rename crux-mir/lib/{libunwind => unwind}/lib.rs (100%) rename crux-mir/lib/{libunwind => unwind}/libunwind.rs (100%) create mode 100644 crux-mir/lib/unwind/src/build.rs create mode 100644 crux-mir/lib/unwind/src/lib.rs create mode 100644 crux-mir/lib/unwind/src/libunwind.rs diff --git a/crux-mir/lib/liballoc/benches/btree/map.rs b/crux-mir/lib/alloc/benches/btree/map.rs similarity index 100% rename from crux-mir/lib/liballoc/benches/btree/map.rs rename to crux-mir/lib/alloc/benches/btree/map.rs diff --git a/crux-mir/lib/liballoc/benches/btree/mod.rs b/crux-mir/lib/alloc/benches/btree/mod.rs similarity index 100% rename from crux-mir/lib/liballoc/benches/btree/mod.rs rename to crux-mir/lib/alloc/benches/btree/mod.rs diff --git a/crux-mir/lib/liballoc/benches/btree/set.rs b/crux-mir/lib/alloc/benches/btree/set.rs similarity index 100% rename from crux-mir/lib/liballoc/benches/btree/set.rs rename to crux-mir/lib/alloc/benches/btree/set.rs diff --git a/crux-mir/lib/liballoc/benches/lib.rs b/crux-mir/lib/alloc/benches/lib.rs similarity index 100% rename from crux-mir/lib/liballoc/benches/lib.rs rename to crux-mir/lib/alloc/benches/lib.rs diff --git a/crux-mir/lib/liballoc/benches/linked_list.rs b/crux-mir/lib/alloc/benches/linked_list.rs similarity index 100% rename from crux-mir/lib/liballoc/benches/linked_list.rs rename to crux-mir/lib/alloc/benches/linked_list.rs diff --git a/crux-mir/lib/liballoc/benches/slice.rs b/crux-mir/lib/alloc/benches/slice.rs similarity index 100% rename from crux-mir/lib/liballoc/benches/slice.rs rename to crux-mir/lib/alloc/benches/slice.rs diff --git a/crux-mir/lib/liballoc/benches/str.rs b/crux-mir/lib/alloc/benches/str.rs similarity index 100% rename from crux-mir/lib/liballoc/benches/str.rs rename to crux-mir/lib/alloc/benches/str.rs diff --git a/crux-mir/lib/liballoc/benches/string.rs b/crux-mir/lib/alloc/benches/string.rs similarity index 100% rename from crux-mir/lib/liballoc/benches/string.rs rename to crux-mir/lib/alloc/benches/string.rs diff --git a/crux-mir/lib/liballoc/benches/vec.rs b/crux-mir/lib/alloc/benches/vec.rs similarity index 100% rename from crux-mir/lib/liballoc/benches/vec.rs rename to crux-mir/lib/alloc/benches/vec.rs diff --git a/crux-mir/lib/liballoc/benches/vec_deque.rs b/crux-mir/lib/alloc/benches/vec_deque.rs similarity index 100% rename from crux-mir/lib/liballoc/benches/vec_deque.rs rename to crux-mir/lib/alloc/benches/vec_deque.rs diff --git a/crux-mir/lib/liballoc/benches/vec_deque_append.rs b/crux-mir/lib/alloc/benches/vec_deque_append.rs similarity index 100% rename from crux-mir/lib/liballoc/benches/vec_deque_append.rs rename to crux-mir/lib/alloc/benches/vec_deque_append.rs diff --git a/crux-mir/lib/liballoc/Cargo.toml b/crux-mir/lib/alloc/src/Cargo.toml similarity index 100% rename from crux-mir/lib/liballoc/Cargo.toml rename to crux-mir/lib/alloc/src/Cargo.toml diff --git a/crux-mir/lib/liballoc/alloc.rs b/crux-mir/lib/alloc/src/alloc.rs similarity index 100% rename from crux-mir/lib/liballoc/alloc.rs rename to crux-mir/lib/alloc/src/alloc.rs diff --git a/crux-mir/lib/liballoc/alloc/tests.rs b/crux-mir/lib/alloc/src/alloc/tests.rs similarity index 100% rename from crux-mir/lib/liballoc/alloc/tests.rs rename to crux-mir/lib/alloc/src/alloc/tests.rs diff --git a/crux-mir/lib/liballoc/borrow.rs b/crux-mir/lib/alloc/src/borrow.rs similarity index 100% rename from crux-mir/lib/liballoc/borrow.rs rename to crux-mir/lib/alloc/src/borrow.rs diff --git a/crux-mir/lib/liballoc/boxed.rs b/crux-mir/lib/alloc/src/boxed.rs similarity index 100% rename from crux-mir/lib/liballoc/boxed.rs rename to crux-mir/lib/alloc/src/boxed.rs diff --git a/crux-mir/lib/liballoc/collections/binary_heap.rs b/crux-mir/lib/alloc/src/collections/binary_heap.rs similarity index 100% rename from crux-mir/lib/liballoc/collections/binary_heap.rs rename to crux-mir/lib/alloc/src/collections/binary_heap.rs diff --git a/crux-mir/lib/liballoc/collections/btree/map.rs b/crux-mir/lib/alloc/src/collections/btree/map.rs similarity index 100% rename from crux-mir/lib/liballoc/collections/btree/map.rs rename to crux-mir/lib/alloc/src/collections/btree/map.rs diff --git a/crux-mir/lib/liballoc/collections/btree/mod.rs b/crux-mir/lib/alloc/src/collections/btree/mod.rs similarity index 100% rename from crux-mir/lib/liballoc/collections/btree/mod.rs rename to crux-mir/lib/alloc/src/collections/btree/mod.rs diff --git a/crux-mir/lib/liballoc/collections/btree/navigate.rs b/crux-mir/lib/alloc/src/collections/btree/navigate.rs similarity index 100% rename from crux-mir/lib/liballoc/collections/btree/navigate.rs rename to crux-mir/lib/alloc/src/collections/btree/navigate.rs diff --git a/crux-mir/lib/liballoc/collections/btree/node.rs b/crux-mir/lib/alloc/src/collections/btree/node.rs similarity index 100% rename from crux-mir/lib/liballoc/collections/btree/node.rs rename to crux-mir/lib/alloc/src/collections/btree/node.rs diff --git a/crux-mir/lib/liballoc/collections/btree/search.rs b/crux-mir/lib/alloc/src/collections/btree/search.rs similarity index 100% rename from crux-mir/lib/liballoc/collections/btree/search.rs rename to crux-mir/lib/alloc/src/collections/btree/search.rs diff --git a/crux-mir/lib/liballoc/collections/btree/set.rs b/crux-mir/lib/alloc/src/collections/btree/set.rs similarity index 100% rename from crux-mir/lib/liballoc/collections/btree/set.rs rename to crux-mir/lib/alloc/src/collections/btree/set.rs diff --git a/crux-mir/lib/liballoc/collections/linked_list.rs b/crux-mir/lib/alloc/src/collections/linked_list.rs similarity index 100% rename from crux-mir/lib/liballoc/collections/linked_list.rs rename to crux-mir/lib/alloc/src/collections/linked_list.rs diff --git a/crux-mir/lib/liballoc/collections/linked_list/tests.rs b/crux-mir/lib/alloc/src/collections/linked_list/tests.rs similarity index 100% rename from crux-mir/lib/liballoc/collections/linked_list/tests.rs rename to crux-mir/lib/alloc/src/collections/linked_list/tests.rs diff --git a/crux-mir/lib/liballoc/collections/mod.rs b/crux-mir/lib/alloc/src/collections/mod.rs similarity index 100% rename from crux-mir/lib/liballoc/collections/mod.rs rename to crux-mir/lib/alloc/src/collections/mod.rs diff --git a/crux-mir/lib/liballoc/collections/vec_deque.rs b/crux-mir/lib/alloc/src/collections/vec_deque.rs similarity index 100% rename from crux-mir/lib/liballoc/collections/vec_deque.rs rename to crux-mir/lib/alloc/src/collections/vec_deque.rs diff --git a/crux-mir/lib/liballoc/collections/vec_deque/drain.rs b/crux-mir/lib/alloc/src/collections/vec_deque/drain.rs similarity index 100% rename from crux-mir/lib/liballoc/collections/vec_deque/drain.rs rename to crux-mir/lib/alloc/src/collections/vec_deque/drain.rs diff --git a/crux-mir/lib/liballoc/collections/vec_deque/tests.rs b/crux-mir/lib/alloc/src/collections/vec_deque/tests.rs similarity index 100% rename from crux-mir/lib/liballoc/collections/vec_deque/tests.rs rename to crux-mir/lib/alloc/src/collections/vec_deque/tests.rs diff --git a/crux-mir/lib/liballoc/fmt.rs b/crux-mir/lib/alloc/src/fmt.rs similarity index 100% rename from crux-mir/lib/liballoc/fmt.rs rename to crux-mir/lib/alloc/src/fmt.rs diff --git a/crux-mir/lib/liballoc/lib.rs b/crux-mir/lib/alloc/src/lib.rs similarity index 100% rename from crux-mir/lib/liballoc/lib.rs rename to crux-mir/lib/alloc/src/lib.rs diff --git a/crux-mir/lib/liballoc/macros.rs b/crux-mir/lib/alloc/src/macros.rs similarity index 100% rename from crux-mir/lib/liballoc/macros.rs rename to crux-mir/lib/alloc/src/macros.rs diff --git a/crux-mir/lib/liballoc/prelude/mod.rs b/crux-mir/lib/alloc/src/prelude/mod.rs similarity index 100% rename from crux-mir/lib/liballoc/prelude/mod.rs rename to crux-mir/lib/alloc/src/prelude/mod.rs diff --git a/crux-mir/lib/liballoc/prelude/v1.rs b/crux-mir/lib/alloc/src/prelude/v1.rs similarity index 100% rename from crux-mir/lib/liballoc/prelude/v1.rs rename to crux-mir/lib/alloc/src/prelude/v1.rs diff --git a/crux-mir/lib/liballoc/raw_vec.rs b/crux-mir/lib/alloc/src/raw_vec.rs similarity index 100% rename from crux-mir/lib/liballoc/raw_vec.rs rename to crux-mir/lib/alloc/src/raw_vec.rs diff --git a/crux-mir/lib/liballoc/raw_vec/tests.rs b/crux-mir/lib/alloc/src/raw_vec/tests.rs similarity index 100% rename from crux-mir/lib/liballoc/raw_vec/tests.rs rename to crux-mir/lib/alloc/src/raw_vec/tests.rs diff --git a/crux-mir/lib/liballoc/rc.rs b/crux-mir/lib/alloc/src/rc.rs similarity index 100% rename from crux-mir/lib/liballoc/rc.rs rename to crux-mir/lib/alloc/src/rc.rs diff --git a/crux-mir/lib/liballoc/rc/tests.rs b/crux-mir/lib/alloc/src/rc/tests.rs similarity index 100% rename from crux-mir/lib/liballoc/rc/tests.rs rename to crux-mir/lib/alloc/src/rc/tests.rs diff --git a/crux-mir/lib/liballoc/slice.rs b/crux-mir/lib/alloc/src/slice.rs similarity index 100% rename from crux-mir/lib/liballoc/slice.rs rename to crux-mir/lib/alloc/src/slice.rs diff --git a/crux-mir/lib/liballoc/str.rs b/crux-mir/lib/alloc/src/str.rs similarity index 100% rename from crux-mir/lib/liballoc/str.rs rename to crux-mir/lib/alloc/src/str.rs diff --git a/crux-mir/lib/liballoc/string.rs b/crux-mir/lib/alloc/src/string.rs similarity index 100% rename from crux-mir/lib/liballoc/string.rs rename to crux-mir/lib/alloc/src/string.rs diff --git a/crux-mir/lib/liballoc/sync.rs b/crux-mir/lib/alloc/src/sync.rs similarity index 100% rename from crux-mir/lib/liballoc/sync.rs rename to crux-mir/lib/alloc/src/sync.rs diff --git a/crux-mir/lib/liballoc/sync/tests.rs b/crux-mir/lib/alloc/src/sync/tests.rs similarity index 100% rename from crux-mir/lib/liballoc/sync/tests.rs rename to crux-mir/lib/alloc/src/sync/tests.rs diff --git a/crux-mir/lib/liballoc/tests.rs b/crux-mir/lib/alloc/src/tests.rs similarity index 100% rename from crux-mir/lib/liballoc/tests.rs rename to crux-mir/lib/alloc/src/tests.rs diff --git a/crux-mir/lib/liballoc/vec.rs b/crux-mir/lib/alloc/src/vec.rs similarity index 100% rename from crux-mir/lib/liballoc/vec.rs rename to crux-mir/lib/alloc/src/vec.rs diff --git a/crux-mir/lib/liballoc/tests/arc.rs b/crux-mir/lib/alloc/tests/arc.rs similarity index 100% rename from crux-mir/lib/liballoc/tests/arc.rs rename to crux-mir/lib/alloc/tests/arc.rs diff --git a/crux-mir/lib/liballoc/tests/binary_heap.rs b/crux-mir/lib/alloc/tests/binary_heap.rs similarity index 100% rename from crux-mir/lib/liballoc/tests/binary_heap.rs rename to crux-mir/lib/alloc/tests/binary_heap.rs diff --git a/crux-mir/lib/liballoc/tests/boxed.rs b/crux-mir/lib/alloc/tests/boxed.rs similarity index 100% rename from crux-mir/lib/liballoc/tests/boxed.rs rename to crux-mir/lib/alloc/tests/boxed.rs diff --git a/crux-mir/lib/liballoc/tests/btree/map.rs b/crux-mir/lib/alloc/tests/btree/map.rs similarity index 100% rename from crux-mir/lib/liballoc/tests/btree/map.rs rename to crux-mir/lib/alloc/tests/btree/map.rs diff --git a/crux-mir/lib/liballoc/tests/btree/mod.rs b/crux-mir/lib/alloc/tests/btree/mod.rs similarity index 100% rename from crux-mir/lib/liballoc/tests/btree/mod.rs rename to crux-mir/lib/alloc/tests/btree/mod.rs diff --git a/crux-mir/lib/liballoc/tests/btree/set.rs b/crux-mir/lib/alloc/tests/btree/set.rs similarity index 100% rename from crux-mir/lib/liballoc/tests/btree/set.rs rename to crux-mir/lib/alloc/tests/btree/set.rs diff --git a/crux-mir/lib/liballoc/tests/cow_str.rs b/crux-mir/lib/alloc/tests/cow_str.rs similarity index 100% rename from crux-mir/lib/liballoc/tests/cow_str.rs rename to crux-mir/lib/alloc/tests/cow_str.rs diff --git a/crux-mir/lib/liballoc/tests/fmt.rs b/crux-mir/lib/alloc/tests/fmt.rs similarity index 100% rename from crux-mir/lib/liballoc/tests/fmt.rs rename to crux-mir/lib/alloc/tests/fmt.rs diff --git a/crux-mir/lib/liballoc/tests/heap.rs b/crux-mir/lib/alloc/tests/heap.rs similarity index 100% rename from crux-mir/lib/liballoc/tests/heap.rs rename to crux-mir/lib/alloc/tests/heap.rs diff --git a/crux-mir/lib/liballoc/tests/lib.rs b/crux-mir/lib/alloc/tests/lib.rs similarity index 100% rename from crux-mir/lib/liballoc/tests/lib.rs rename to crux-mir/lib/alloc/tests/lib.rs diff --git a/crux-mir/lib/liballoc/tests/linked_list.rs b/crux-mir/lib/alloc/tests/linked_list.rs similarity index 100% rename from crux-mir/lib/liballoc/tests/linked_list.rs rename to crux-mir/lib/alloc/tests/linked_list.rs diff --git a/crux-mir/lib/liballoc/tests/rc.rs b/crux-mir/lib/alloc/tests/rc.rs similarity index 100% rename from crux-mir/lib/liballoc/tests/rc.rs rename to crux-mir/lib/alloc/tests/rc.rs diff --git a/crux-mir/lib/liballoc/tests/slice.rs b/crux-mir/lib/alloc/tests/slice.rs similarity index 100% rename from crux-mir/lib/liballoc/tests/slice.rs rename to crux-mir/lib/alloc/tests/slice.rs diff --git a/crux-mir/lib/liballoc/tests/str.rs b/crux-mir/lib/alloc/tests/str.rs similarity index 100% rename from crux-mir/lib/liballoc/tests/str.rs rename to crux-mir/lib/alloc/tests/str.rs diff --git a/crux-mir/lib/liballoc/tests/string.rs b/crux-mir/lib/alloc/tests/string.rs similarity index 100% rename from crux-mir/lib/liballoc/tests/string.rs rename to crux-mir/lib/alloc/tests/string.rs diff --git a/crux-mir/lib/liballoc/tests/vec.rs b/crux-mir/lib/alloc/tests/vec.rs similarity index 100% rename from crux-mir/lib/liballoc/tests/vec.rs rename to crux-mir/lib/alloc/tests/vec.rs diff --git a/crux-mir/lib/liballoc/tests/vec_deque.rs b/crux-mir/lib/alloc/tests/vec_deque.rs similarity index 100% rename from crux-mir/lib/liballoc/tests/vec_deque.rs rename to crux-mir/lib/alloc/tests/vec_deque.rs diff --git a/crux-mir/lib/compiler-builtins/Cargo.toml b/crux-mir/lib/compiler_builtins/Cargo.toml similarity index 100% rename from crux-mir/lib/compiler-builtins/Cargo.toml rename to crux-mir/lib/compiler_builtins/Cargo.toml diff --git a/crux-mir/lib/compiler-builtins/LICENSE.TXT b/crux-mir/lib/compiler_builtins/LICENSE.TXT similarity index 100% rename from crux-mir/lib/compiler-builtins/LICENSE.TXT rename to crux-mir/lib/compiler_builtins/LICENSE.TXT diff --git a/crux-mir/lib/compiler-builtins/PUBLISHING.md b/crux-mir/lib/compiler_builtins/PUBLISHING.md similarity index 100% rename from crux-mir/lib/compiler-builtins/PUBLISHING.md rename to crux-mir/lib/compiler_builtins/PUBLISHING.md diff --git a/crux-mir/lib/compiler-builtins/README.md b/crux-mir/lib/compiler_builtins/README.md similarity index 100% rename from crux-mir/lib/compiler-builtins/README.md rename to crux-mir/lib/compiler_builtins/README.md diff --git a/crux-mir/lib/compiler-builtins/build.rs b/crux-mir/lib/compiler_builtins/build.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/build.rs rename to crux-mir/lib/compiler_builtins/build.rs diff --git a/crux-mir/lib/compiler-builtins/ci/docker/aarch64-unknown-linux-gnu/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/aarch64-unknown-linux-gnu/Dockerfile similarity index 100% rename from crux-mir/lib/compiler-builtins/ci/docker/aarch64-unknown-linux-gnu/Dockerfile rename to crux-mir/lib/compiler_builtins/ci/docker/aarch64-unknown-linux-gnu/Dockerfile diff --git a/crux-mir/lib/compiler-builtins/ci/docker/arm-unknown-linux-gnueabi/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/arm-unknown-linux-gnueabi/Dockerfile similarity index 100% rename from crux-mir/lib/compiler-builtins/ci/docker/arm-unknown-linux-gnueabi/Dockerfile rename to crux-mir/lib/compiler_builtins/ci/docker/arm-unknown-linux-gnueabi/Dockerfile diff --git a/crux-mir/lib/compiler-builtins/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile similarity index 100% rename from crux-mir/lib/compiler-builtins/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile rename to crux-mir/lib/compiler_builtins/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile diff --git a/crux-mir/lib/compiler-builtins/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile similarity index 100% rename from crux-mir/lib/compiler-builtins/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile rename to crux-mir/lib/compiler_builtins/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile diff --git a/crux-mir/lib/compiler-builtins/ci/docker/i586-unknown-linux-gnu/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/i586-unknown-linux-gnu/Dockerfile similarity index 100% rename from crux-mir/lib/compiler-builtins/ci/docker/i586-unknown-linux-gnu/Dockerfile rename to crux-mir/lib/compiler_builtins/ci/docker/i586-unknown-linux-gnu/Dockerfile diff --git a/crux-mir/lib/compiler-builtins/ci/docker/i686-unknown-linux-gnu/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/i686-unknown-linux-gnu/Dockerfile similarity index 100% rename from crux-mir/lib/compiler-builtins/ci/docker/i686-unknown-linux-gnu/Dockerfile rename to crux-mir/lib/compiler_builtins/ci/docker/i686-unknown-linux-gnu/Dockerfile diff --git a/crux-mir/lib/compiler-builtins/ci/docker/mips-unknown-linux-gnu/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/mips-unknown-linux-gnu/Dockerfile similarity index 100% rename from crux-mir/lib/compiler-builtins/ci/docker/mips-unknown-linux-gnu/Dockerfile rename to crux-mir/lib/compiler_builtins/ci/docker/mips-unknown-linux-gnu/Dockerfile diff --git a/crux-mir/lib/compiler-builtins/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile similarity index 100% rename from crux-mir/lib/compiler-builtins/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile rename to crux-mir/lib/compiler_builtins/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile diff --git a/crux-mir/lib/compiler-builtins/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile similarity index 100% rename from crux-mir/lib/compiler-builtins/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile rename to crux-mir/lib/compiler_builtins/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile diff --git a/crux-mir/lib/compiler-builtins/ci/docker/mipsel-unknown-linux-gnu/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/mipsel-unknown-linux-gnu/Dockerfile similarity index 100% rename from crux-mir/lib/compiler-builtins/ci/docker/mipsel-unknown-linux-gnu/Dockerfile rename to crux-mir/lib/compiler_builtins/ci/docker/mipsel-unknown-linux-gnu/Dockerfile diff --git a/crux-mir/lib/compiler-builtins/ci/docker/powerpc-unknown-linux-gnu/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/powerpc-unknown-linux-gnu/Dockerfile similarity index 100% rename from crux-mir/lib/compiler-builtins/ci/docker/powerpc-unknown-linux-gnu/Dockerfile rename to crux-mir/lib/compiler_builtins/ci/docker/powerpc-unknown-linux-gnu/Dockerfile diff --git a/crux-mir/lib/compiler-builtins/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile similarity index 100% rename from crux-mir/lib/compiler-builtins/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile rename to crux-mir/lib/compiler_builtins/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile diff --git a/crux-mir/lib/compiler-builtins/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile similarity index 100% rename from crux-mir/lib/compiler-builtins/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile rename to crux-mir/lib/compiler_builtins/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile diff --git a/crux-mir/lib/compiler-builtins/ci/docker/thumbv6m-none-eabi/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/thumbv6m-none-eabi/Dockerfile similarity index 100% rename from crux-mir/lib/compiler-builtins/ci/docker/thumbv6m-none-eabi/Dockerfile rename to crux-mir/lib/compiler_builtins/ci/docker/thumbv6m-none-eabi/Dockerfile diff --git a/crux-mir/lib/compiler-builtins/ci/docker/thumbv7em-none-eabi/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/thumbv7em-none-eabi/Dockerfile similarity index 100% rename from crux-mir/lib/compiler-builtins/ci/docker/thumbv7em-none-eabi/Dockerfile rename to crux-mir/lib/compiler_builtins/ci/docker/thumbv7em-none-eabi/Dockerfile diff --git a/crux-mir/lib/compiler-builtins/ci/docker/thumbv7em-none-eabihf/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/thumbv7em-none-eabihf/Dockerfile similarity index 100% rename from crux-mir/lib/compiler-builtins/ci/docker/thumbv7em-none-eabihf/Dockerfile rename to crux-mir/lib/compiler_builtins/ci/docker/thumbv7em-none-eabihf/Dockerfile diff --git a/crux-mir/lib/compiler-builtins/ci/docker/thumbv7m-none-eabi/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/thumbv7m-none-eabi/Dockerfile similarity index 100% rename from crux-mir/lib/compiler-builtins/ci/docker/thumbv7m-none-eabi/Dockerfile rename to crux-mir/lib/compiler_builtins/ci/docker/thumbv7m-none-eabi/Dockerfile diff --git a/crux-mir/lib/compiler-builtins/ci/docker/x86_64-unknown-linux-gnu/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/x86_64-unknown-linux-gnu/Dockerfile similarity index 100% rename from crux-mir/lib/compiler-builtins/ci/docker/x86_64-unknown-linux-gnu/Dockerfile rename to crux-mir/lib/compiler_builtins/ci/docker/x86_64-unknown-linux-gnu/Dockerfile diff --git a/crux-mir/lib/compiler-builtins/ci/run-docker.sh b/crux-mir/lib/compiler_builtins/ci/run-docker.sh similarity index 100% rename from crux-mir/lib/compiler-builtins/ci/run-docker.sh rename to crux-mir/lib/compiler_builtins/ci/run-docker.sh diff --git a/crux-mir/lib/compiler-builtins/ci/run.sh b/crux-mir/lib/compiler_builtins/ci/run.sh similarity index 100% rename from crux-mir/lib/compiler-builtins/ci/run.sh rename to crux-mir/lib/compiler_builtins/ci/run.sh diff --git a/crux-mir/lib/compiler-builtins/crates/panic-handler/Cargo.toml b/crux-mir/lib/compiler_builtins/crates/panic-handler/Cargo.toml similarity index 100% rename from crux-mir/lib/compiler-builtins/crates/panic-handler/Cargo.toml rename to crux-mir/lib/compiler_builtins/crates/panic-handler/Cargo.toml diff --git a/crux-mir/lib/compiler-builtins/crates/panic-handler/src/lib.rs b/crux-mir/lib/compiler_builtins/crates/panic-handler/src/lib.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/crates/panic-handler/src/lib.rs rename to crux-mir/lib/compiler_builtins/crates/panic-handler/src/lib.rs diff --git a/crux-mir/lib/compiler-builtins/examples/intrinsics.rs b/crux-mir/lib/compiler_builtins/examples/intrinsics.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/examples/intrinsics.rs rename to crux-mir/lib/compiler_builtins/examples/intrinsics.rs diff --git a/crux-mir/lib/compiler-builtins/src/arm.rs b/crux-mir/lib/compiler_builtins/src/arm.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/arm.rs rename to crux-mir/lib/compiler_builtins/src/arm.rs diff --git a/crux-mir/lib/compiler-builtins/src/arm_linux.rs b/crux-mir/lib/compiler_builtins/src/arm_linux.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/arm_linux.rs rename to crux-mir/lib/compiler_builtins/src/arm_linux.rs diff --git a/crux-mir/lib/compiler-builtins/src/float/add.rs b/crux-mir/lib/compiler_builtins/src/float/add.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/float/add.rs rename to crux-mir/lib/compiler_builtins/src/float/add.rs diff --git a/crux-mir/lib/compiler-builtins/src/float/cmp.rs b/crux-mir/lib/compiler_builtins/src/float/cmp.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/float/cmp.rs rename to crux-mir/lib/compiler_builtins/src/float/cmp.rs diff --git a/crux-mir/lib/compiler-builtins/src/float/conv.rs b/crux-mir/lib/compiler_builtins/src/float/conv.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/float/conv.rs rename to crux-mir/lib/compiler_builtins/src/float/conv.rs diff --git a/crux-mir/lib/compiler-builtins/src/float/div.rs b/crux-mir/lib/compiler_builtins/src/float/div.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/float/div.rs rename to crux-mir/lib/compiler_builtins/src/float/div.rs diff --git a/crux-mir/lib/compiler-builtins/src/float/extend.rs b/crux-mir/lib/compiler_builtins/src/float/extend.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/float/extend.rs rename to crux-mir/lib/compiler_builtins/src/float/extend.rs diff --git a/crux-mir/lib/compiler-builtins/src/float/mod.rs b/crux-mir/lib/compiler_builtins/src/float/mod.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/float/mod.rs rename to crux-mir/lib/compiler_builtins/src/float/mod.rs diff --git a/crux-mir/lib/compiler-builtins/src/float/mul.rs b/crux-mir/lib/compiler_builtins/src/float/mul.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/float/mul.rs rename to crux-mir/lib/compiler_builtins/src/float/mul.rs diff --git a/crux-mir/lib/compiler-builtins/src/float/pow.rs b/crux-mir/lib/compiler_builtins/src/float/pow.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/float/pow.rs rename to crux-mir/lib/compiler_builtins/src/float/pow.rs diff --git a/crux-mir/lib/compiler-builtins/src/float/sub.rs b/crux-mir/lib/compiler_builtins/src/float/sub.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/float/sub.rs rename to crux-mir/lib/compiler_builtins/src/float/sub.rs diff --git a/crux-mir/lib/compiler-builtins/src/int/addsub.rs b/crux-mir/lib/compiler_builtins/src/int/addsub.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/int/addsub.rs rename to crux-mir/lib/compiler_builtins/src/int/addsub.rs diff --git a/crux-mir/lib/compiler-builtins/src/int/mod.rs b/crux-mir/lib/compiler_builtins/src/int/mod.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/int/mod.rs rename to crux-mir/lib/compiler_builtins/src/int/mod.rs diff --git a/crux-mir/lib/compiler-builtins/src/int/mul.rs b/crux-mir/lib/compiler_builtins/src/int/mul.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/int/mul.rs rename to crux-mir/lib/compiler_builtins/src/int/mul.rs diff --git a/crux-mir/lib/compiler-builtins/src/int/sdiv.rs b/crux-mir/lib/compiler_builtins/src/int/sdiv.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/int/sdiv.rs rename to crux-mir/lib/compiler_builtins/src/int/sdiv.rs diff --git a/crux-mir/lib/compiler-builtins/src/int/shift.rs b/crux-mir/lib/compiler_builtins/src/int/shift.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/int/shift.rs rename to crux-mir/lib/compiler_builtins/src/int/shift.rs diff --git a/crux-mir/lib/compiler-builtins/src/int/udiv.rs b/crux-mir/lib/compiler_builtins/src/int/udiv.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/int/udiv.rs rename to crux-mir/lib/compiler_builtins/src/int/udiv.rs diff --git a/crux-mir/lib/compiler-builtins/src/lib.rs b/crux-mir/lib/compiler_builtins/src/lib.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/lib.rs rename to crux-mir/lib/compiler_builtins/src/lib.rs diff --git a/crux-mir/lib/compiler-builtins/src/macros.rs b/crux-mir/lib/compiler_builtins/src/macros.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/macros.rs rename to crux-mir/lib/compiler_builtins/src/macros.rs diff --git a/crux-mir/lib/compiler-builtins/src/math.rs b/crux-mir/lib/compiler_builtins/src/math.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/math.rs rename to crux-mir/lib/compiler_builtins/src/math.rs diff --git a/crux-mir/lib/compiler-builtins/src/mem.rs b/crux-mir/lib/compiler_builtins/src/mem.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/mem.rs rename to crux-mir/lib/compiler_builtins/src/mem.rs diff --git a/crux-mir/lib/compiler-builtins/src/probestack.rs b/crux-mir/lib/compiler_builtins/src/probestack.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/probestack.rs rename to crux-mir/lib/compiler_builtins/src/probestack.rs diff --git a/crux-mir/lib/compiler-builtins/src/riscv32.rs b/crux-mir/lib/compiler_builtins/src/riscv32.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/riscv32.rs rename to crux-mir/lib/compiler_builtins/src/riscv32.rs diff --git a/crux-mir/lib/compiler-builtins/src/x86.rs b/crux-mir/lib/compiler_builtins/src/x86.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/x86.rs rename to crux-mir/lib/compiler_builtins/src/x86.rs diff --git a/crux-mir/lib/compiler-builtins/src/x86_64.rs b/crux-mir/lib/compiler_builtins/src/x86_64.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/src/x86_64.rs rename to crux-mir/lib/compiler_builtins/src/x86_64.rs diff --git a/crux-mir/lib/compiler-builtins/test.c b/crux-mir/lib/compiler_builtins/test.c similarity index 100% rename from crux-mir/lib/compiler-builtins/test.c rename to crux-mir/lib/compiler_builtins/test.c diff --git a/crux-mir/lib/compiler-builtins/testcrate/Cargo.toml b/crux-mir/lib/compiler_builtins/testcrate/Cargo.toml similarity index 100% rename from crux-mir/lib/compiler-builtins/testcrate/Cargo.toml rename to crux-mir/lib/compiler_builtins/testcrate/Cargo.toml diff --git a/crux-mir/lib/compiler-builtins/testcrate/build.rs b/crux-mir/lib/compiler_builtins/testcrate/build.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/testcrate/build.rs rename to crux-mir/lib/compiler_builtins/testcrate/build.rs diff --git a/crux-mir/lib/compiler-builtins/testcrate/src/lib.rs b/crux-mir/lib/compiler_builtins/testcrate/src/lib.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/testcrate/src/lib.rs rename to crux-mir/lib/compiler_builtins/testcrate/src/lib.rs diff --git a/crux-mir/lib/compiler-builtins/testcrate/tests/aeabi_memclr.rs b/crux-mir/lib/compiler_builtins/testcrate/tests/aeabi_memclr.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/testcrate/tests/aeabi_memclr.rs rename to crux-mir/lib/compiler_builtins/testcrate/tests/aeabi_memclr.rs diff --git a/crux-mir/lib/compiler-builtins/testcrate/tests/aeabi_memcpy.rs b/crux-mir/lib/compiler_builtins/testcrate/tests/aeabi_memcpy.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/testcrate/tests/aeabi_memcpy.rs rename to crux-mir/lib/compiler_builtins/testcrate/tests/aeabi_memcpy.rs diff --git a/crux-mir/lib/compiler-builtins/testcrate/tests/aeabi_memset.rs b/crux-mir/lib/compiler_builtins/testcrate/tests/aeabi_memset.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/testcrate/tests/aeabi_memset.rs rename to crux-mir/lib/compiler_builtins/testcrate/tests/aeabi_memset.rs diff --git a/crux-mir/lib/compiler-builtins/testcrate/tests/count_leading_zeros.rs b/crux-mir/lib/compiler_builtins/testcrate/tests/count_leading_zeros.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/testcrate/tests/count_leading_zeros.rs rename to crux-mir/lib/compiler_builtins/testcrate/tests/count_leading_zeros.rs diff --git a/crux-mir/lib/compiler-builtins/testcrate/tests/generated.rs b/crux-mir/lib/compiler_builtins/testcrate/tests/generated.rs similarity index 100% rename from crux-mir/lib/compiler-builtins/testcrate/tests/generated.rs rename to crux-mir/lib/compiler_builtins/testcrate/tests/generated.rs diff --git a/crux-mir/lib/compiler-builtins/thumbv6m-linux-eabi.json b/crux-mir/lib/compiler_builtins/thumbv6m-linux-eabi.json similarity index 100% rename from crux-mir/lib/compiler-builtins/thumbv6m-linux-eabi.json rename to crux-mir/lib/compiler_builtins/thumbv6m-linux-eabi.json diff --git a/crux-mir/lib/compiler-builtins/thumbv7em-linux-eabi.json b/crux-mir/lib/compiler_builtins/thumbv7em-linux-eabi.json similarity index 100% rename from crux-mir/lib/compiler-builtins/thumbv7em-linux-eabi.json rename to crux-mir/lib/compiler_builtins/thumbv7em-linux-eabi.json diff --git a/crux-mir/lib/compiler-builtins/thumbv7em-linux-eabihf.json b/crux-mir/lib/compiler_builtins/thumbv7em-linux-eabihf.json similarity index 100% rename from crux-mir/lib/compiler-builtins/thumbv7em-linux-eabihf.json rename to crux-mir/lib/compiler_builtins/thumbv7em-linux-eabihf.json diff --git a/crux-mir/lib/compiler-builtins/thumbv7m-linux-eabi.json b/crux-mir/lib/compiler_builtins/thumbv7m-linux-eabi.json similarity index 100% rename from crux-mir/lib/compiler-builtins/thumbv7m-linux-eabi.json rename to crux-mir/lib/compiler_builtins/thumbv7m-linux-eabi.json diff --git a/crux-mir/lib/libcore/benches/any.rs b/crux-mir/lib/core/benches/any.rs similarity index 100% rename from crux-mir/lib/libcore/benches/any.rs rename to crux-mir/lib/core/benches/any.rs diff --git a/crux-mir/lib/libcore/benches/ascii.rs b/crux-mir/lib/core/benches/ascii.rs similarity index 100% rename from crux-mir/lib/libcore/benches/ascii.rs rename to crux-mir/lib/core/benches/ascii.rs diff --git a/crux-mir/lib/libcore/benches/char/methods.rs b/crux-mir/lib/core/benches/char/methods.rs similarity index 100% rename from crux-mir/lib/libcore/benches/char/methods.rs rename to crux-mir/lib/core/benches/char/methods.rs diff --git a/crux-mir/lib/libcore/benches/char/mod.rs b/crux-mir/lib/core/benches/char/mod.rs similarity index 100% rename from crux-mir/lib/libcore/benches/char/mod.rs rename to crux-mir/lib/core/benches/char/mod.rs diff --git a/crux-mir/lib/libcore/benches/fmt.rs b/crux-mir/lib/core/benches/fmt.rs similarity index 100% rename from crux-mir/lib/libcore/benches/fmt.rs rename to crux-mir/lib/core/benches/fmt.rs diff --git a/crux-mir/lib/libcore/benches/hash/mod.rs b/crux-mir/lib/core/benches/hash/mod.rs similarity index 100% rename from crux-mir/lib/libcore/benches/hash/mod.rs rename to crux-mir/lib/core/benches/hash/mod.rs diff --git a/crux-mir/lib/libcore/benches/hash/sip.rs b/crux-mir/lib/core/benches/hash/sip.rs similarity index 100% rename from crux-mir/lib/libcore/benches/hash/sip.rs rename to crux-mir/lib/core/benches/hash/sip.rs diff --git a/crux-mir/lib/libcore/benches/iter.rs b/crux-mir/lib/core/benches/iter.rs similarity index 100% rename from crux-mir/lib/libcore/benches/iter.rs rename to crux-mir/lib/core/benches/iter.rs diff --git a/crux-mir/lib/libcore/benches/lib.rs b/crux-mir/lib/core/benches/lib.rs similarity index 100% rename from crux-mir/lib/libcore/benches/lib.rs rename to crux-mir/lib/core/benches/lib.rs diff --git a/crux-mir/lib/libcore/benches/num/dec2flt/mod.rs b/crux-mir/lib/core/benches/num/dec2flt/mod.rs similarity index 100% rename from crux-mir/lib/libcore/benches/num/dec2flt/mod.rs rename to crux-mir/lib/core/benches/num/dec2flt/mod.rs diff --git a/crux-mir/lib/libcore/benches/num/flt2dec/mod.rs b/crux-mir/lib/core/benches/num/flt2dec/mod.rs similarity index 100% rename from crux-mir/lib/libcore/benches/num/flt2dec/mod.rs rename to crux-mir/lib/core/benches/num/flt2dec/mod.rs diff --git a/crux-mir/lib/libcore/benches/num/flt2dec/strategy/dragon.rs b/crux-mir/lib/core/benches/num/flt2dec/strategy/dragon.rs similarity index 100% rename from crux-mir/lib/libcore/benches/num/flt2dec/strategy/dragon.rs rename to crux-mir/lib/core/benches/num/flt2dec/strategy/dragon.rs diff --git a/crux-mir/lib/libcore/benches/num/flt2dec/strategy/grisu.rs b/crux-mir/lib/core/benches/num/flt2dec/strategy/grisu.rs similarity index 100% rename from crux-mir/lib/libcore/benches/num/flt2dec/strategy/grisu.rs rename to crux-mir/lib/core/benches/num/flt2dec/strategy/grisu.rs diff --git a/crux-mir/lib/libcore/benches/num/mod.rs b/crux-mir/lib/core/benches/num/mod.rs similarity index 100% rename from crux-mir/lib/libcore/benches/num/mod.rs rename to crux-mir/lib/core/benches/num/mod.rs diff --git a/crux-mir/lib/libcore/benches/ops.rs b/crux-mir/lib/core/benches/ops.rs similarity index 100% rename from crux-mir/lib/libcore/benches/ops.rs rename to crux-mir/lib/core/benches/ops.rs diff --git a/crux-mir/lib/libcore/benches/pattern.rs b/crux-mir/lib/core/benches/pattern.rs similarity index 100% rename from crux-mir/lib/libcore/benches/pattern.rs rename to crux-mir/lib/core/benches/pattern.rs diff --git a/crux-mir/lib/libcore/benches/slice.rs b/crux-mir/lib/core/benches/slice.rs similarity index 100% rename from crux-mir/lib/libcore/benches/slice.rs rename to crux-mir/lib/core/benches/slice.rs diff --git a/crux-mir/lib/libcore/Cargo.toml b/crux-mir/lib/core/src/Cargo.toml similarity index 100% rename from crux-mir/lib/libcore/Cargo.toml rename to crux-mir/lib/core/src/Cargo.toml diff --git a/crux-mir/lib/libcore/alloc.rs b/crux-mir/lib/core/src/alloc.rs similarity index 100% rename from crux-mir/lib/libcore/alloc.rs rename to crux-mir/lib/core/src/alloc.rs diff --git a/crux-mir/lib/libcore/any.rs b/crux-mir/lib/core/src/any.rs similarity index 100% rename from crux-mir/lib/libcore/any.rs rename to crux-mir/lib/core/src/any.rs diff --git a/crux-mir/lib/libcore/array/iter.rs b/crux-mir/lib/core/src/array/iter.rs similarity index 100% rename from crux-mir/lib/libcore/array/iter.rs rename to crux-mir/lib/core/src/array/iter.rs diff --git a/crux-mir/lib/libcore/array/mod.rs b/crux-mir/lib/core/src/array/mod.rs similarity index 100% rename from crux-mir/lib/libcore/array/mod.rs rename to crux-mir/lib/core/src/array/mod.rs diff --git a/crux-mir/lib/libcore/ascii.rs b/crux-mir/lib/core/src/ascii.rs similarity index 100% rename from crux-mir/lib/libcore/ascii.rs rename to crux-mir/lib/core/src/ascii.rs diff --git a/crux-mir/lib/libcore/bool.rs b/crux-mir/lib/core/src/bool.rs similarity index 100% rename from crux-mir/lib/libcore/bool.rs rename to crux-mir/lib/core/src/bool.rs diff --git a/crux-mir/lib/libcore/borrow.rs b/crux-mir/lib/core/src/borrow.rs similarity index 100% rename from crux-mir/lib/libcore/borrow.rs rename to crux-mir/lib/core/src/borrow.rs diff --git a/crux-mir/lib/libcore/cell.rs b/crux-mir/lib/core/src/cell.rs similarity index 100% rename from crux-mir/lib/libcore/cell.rs rename to crux-mir/lib/core/src/cell.rs diff --git a/crux-mir/lib/libcore/char/convert.rs b/crux-mir/lib/core/src/char/convert.rs similarity index 100% rename from crux-mir/lib/libcore/char/convert.rs rename to crux-mir/lib/core/src/char/convert.rs diff --git a/crux-mir/lib/libcore/char/decode.rs b/crux-mir/lib/core/src/char/decode.rs similarity index 100% rename from crux-mir/lib/libcore/char/decode.rs rename to crux-mir/lib/core/src/char/decode.rs diff --git a/crux-mir/lib/libcore/char/methods.rs b/crux-mir/lib/core/src/char/methods.rs similarity index 100% rename from crux-mir/lib/libcore/char/methods.rs rename to crux-mir/lib/core/src/char/methods.rs diff --git a/crux-mir/lib/libcore/char/mod.rs b/crux-mir/lib/core/src/char/mod.rs similarity index 100% rename from crux-mir/lib/libcore/char/mod.rs rename to crux-mir/lib/core/src/char/mod.rs diff --git a/crux-mir/lib/libcore/clone.rs b/crux-mir/lib/core/src/clone.rs similarity index 100% rename from crux-mir/lib/libcore/clone.rs rename to crux-mir/lib/core/src/clone.rs diff --git a/crux-mir/lib/libcore/cmp.rs b/crux-mir/lib/core/src/cmp.rs similarity index 100% rename from crux-mir/lib/libcore/cmp.rs rename to crux-mir/lib/core/src/cmp.rs diff --git a/crux-mir/lib/libcore/convert/mod.rs b/crux-mir/lib/core/src/convert/mod.rs similarity index 100% rename from crux-mir/lib/libcore/convert/mod.rs rename to crux-mir/lib/core/src/convert/mod.rs diff --git a/crux-mir/lib/libcore/convert/num.rs b/crux-mir/lib/core/src/convert/num.rs similarity index 100% rename from crux-mir/lib/libcore/convert/num.rs rename to crux-mir/lib/core/src/convert/num.rs diff --git a/crux-mir/lib/libcore/crucible/any.rs b/crux-mir/lib/core/src/crucible/any.rs similarity index 100% rename from crux-mir/lib/libcore/crucible/any.rs rename to crux-mir/lib/core/src/crucible/any.rs diff --git a/crux-mir/lib/libcore/crucible/concurrency.rs b/crux-mir/lib/core/src/crucible/concurrency.rs similarity index 100% rename from crux-mir/lib/libcore/crucible/concurrency.rs rename to crux-mir/lib/core/src/crucible/concurrency.rs diff --git a/crux-mir/lib/libcore/crucible/mod.rs b/crux-mir/lib/core/src/crucible/mod.rs similarity index 100% rename from crux-mir/lib/libcore/crucible/mod.rs rename to crux-mir/lib/core/src/crucible/mod.rs diff --git a/crux-mir/lib/libcore/crucible/ptr.rs b/crux-mir/lib/core/src/crucible/ptr.rs similarity index 100% rename from crux-mir/lib/libcore/crucible/ptr.rs rename to crux-mir/lib/core/src/crucible/ptr.rs diff --git a/crux-mir/lib/libcore/default.rs b/crux-mir/lib/core/src/default.rs similarity index 100% rename from crux-mir/lib/libcore/default.rs rename to crux-mir/lib/core/src/default.rs diff --git a/crux-mir/lib/libcore/ffi.rs b/crux-mir/lib/core/src/ffi.rs similarity index 100% rename from crux-mir/lib/libcore/ffi.rs rename to crux-mir/lib/core/src/ffi.rs diff --git a/crux-mir/lib/libcore/fmt/builders.rs b/crux-mir/lib/core/src/fmt/builders.rs similarity index 100% rename from crux-mir/lib/libcore/fmt/builders.rs rename to crux-mir/lib/core/src/fmt/builders.rs diff --git a/crux-mir/lib/libcore/fmt/float.rs b/crux-mir/lib/core/src/fmt/float.rs similarity index 100% rename from crux-mir/lib/libcore/fmt/float.rs rename to crux-mir/lib/core/src/fmt/float.rs diff --git a/crux-mir/lib/libcore/fmt/mod.rs b/crux-mir/lib/core/src/fmt/mod.rs similarity index 100% rename from crux-mir/lib/libcore/fmt/mod.rs rename to crux-mir/lib/core/src/fmt/mod.rs diff --git a/crux-mir/lib/libcore/fmt/num.rs b/crux-mir/lib/core/src/fmt/num.rs similarity index 100% rename from crux-mir/lib/libcore/fmt/num.rs rename to crux-mir/lib/core/src/fmt/num.rs diff --git a/crux-mir/lib/libcore/fmt/rt/v1.rs b/crux-mir/lib/core/src/fmt/rt/v1.rs similarity index 100% rename from crux-mir/lib/libcore/fmt/rt/v1.rs rename to crux-mir/lib/core/src/fmt/rt/v1.rs diff --git a/crux-mir/lib/libcore/future/future.rs b/crux-mir/lib/core/src/future/future.rs similarity index 100% rename from crux-mir/lib/libcore/future/future.rs rename to crux-mir/lib/core/src/future/future.rs diff --git a/crux-mir/lib/libcore/future/mod.rs b/crux-mir/lib/core/src/future/mod.rs similarity index 100% rename from crux-mir/lib/libcore/future/mod.rs rename to crux-mir/lib/core/src/future/mod.rs diff --git a/crux-mir/lib/libcore/hash/mod.rs b/crux-mir/lib/core/src/hash/mod.rs similarity index 100% rename from crux-mir/lib/libcore/hash/mod.rs rename to crux-mir/lib/core/src/hash/mod.rs diff --git a/crux-mir/lib/libcore/hash/sip.rs b/crux-mir/lib/core/src/hash/sip.rs similarity index 100% rename from crux-mir/lib/libcore/hash/sip.rs rename to crux-mir/lib/core/src/hash/sip.rs diff --git a/crux-mir/lib/libcore/hint.rs b/crux-mir/lib/core/src/hint.rs similarity index 100% rename from crux-mir/lib/libcore/hint.rs rename to crux-mir/lib/core/src/hint.rs diff --git a/crux-mir/lib/libcore/internal_macros.rs b/crux-mir/lib/core/src/internal_macros.rs similarity index 100% rename from crux-mir/lib/libcore/internal_macros.rs rename to crux-mir/lib/core/src/internal_macros.rs diff --git a/crux-mir/lib/libcore/intrinsics.rs b/crux-mir/lib/core/src/intrinsics.rs similarity index 100% rename from crux-mir/lib/libcore/intrinsics.rs rename to crux-mir/lib/core/src/intrinsics.rs diff --git a/crux-mir/lib/libcore/iter/adapters/chain.rs b/crux-mir/lib/core/src/iter/adapters/chain.rs similarity index 100% rename from crux-mir/lib/libcore/iter/adapters/chain.rs rename to crux-mir/lib/core/src/iter/adapters/chain.rs diff --git a/crux-mir/lib/libcore/iter/adapters/flatten.rs b/crux-mir/lib/core/src/iter/adapters/flatten.rs similarity index 100% rename from crux-mir/lib/libcore/iter/adapters/flatten.rs rename to crux-mir/lib/core/src/iter/adapters/flatten.rs diff --git a/crux-mir/lib/libcore/iter/adapters/mod.rs b/crux-mir/lib/core/src/iter/adapters/mod.rs similarity index 100% rename from crux-mir/lib/libcore/iter/adapters/mod.rs rename to crux-mir/lib/core/src/iter/adapters/mod.rs diff --git a/crux-mir/lib/libcore/iter/adapters/zip.rs b/crux-mir/lib/core/src/iter/adapters/zip.rs similarity index 100% rename from crux-mir/lib/libcore/iter/adapters/zip.rs rename to crux-mir/lib/core/src/iter/adapters/zip.rs diff --git a/crux-mir/lib/libcore/iter/mod.rs b/crux-mir/lib/core/src/iter/mod.rs similarity index 100% rename from crux-mir/lib/libcore/iter/mod.rs rename to crux-mir/lib/core/src/iter/mod.rs diff --git a/crux-mir/lib/libcore/iter/range.rs b/crux-mir/lib/core/src/iter/range.rs similarity index 100% rename from crux-mir/lib/libcore/iter/range.rs rename to crux-mir/lib/core/src/iter/range.rs diff --git a/crux-mir/lib/libcore/iter/sources.rs b/crux-mir/lib/core/src/iter/sources.rs similarity index 100% rename from crux-mir/lib/libcore/iter/sources.rs rename to crux-mir/lib/core/src/iter/sources.rs diff --git a/crux-mir/lib/libcore/iter/traits/accum.rs b/crux-mir/lib/core/src/iter/traits/accum.rs similarity index 100% rename from crux-mir/lib/libcore/iter/traits/accum.rs rename to crux-mir/lib/core/src/iter/traits/accum.rs diff --git a/crux-mir/lib/libcore/iter/traits/collect.rs b/crux-mir/lib/core/src/iter/traits/collect.rs similarity index 100% rename from crux-mir/lib/libcore/iter/traits/collect.rs rename to crux-mir/lib/core/src/iter/traits/collect.rs diff --git a/crux-mir/lib/libcore/iter/traits/double_ended.rs b/crux-mir/lib/core/src/iter/traits/double_ended.rs similarity index 100% rename from crux-mir/lib/libcore/iter/traits/double_ended.rs rename to crux-mir/lib/core/src/iter/traits/double_ended.rs diff --git a/crux-mir/lib/libcore/iter/traits/exact_size.rs b/crux-mir/lib/core/src/iter/traits/exact_size.rs similarity index 100% rename from crux-mir/lib/libcore/iter/traits/exact_size.rs rename to crux-mir/lib/core/src/iter/traits/exact_size.rs diff --git a/crux-mir/lib/libcore/iter/traits/iterator.rs b/crux-mir/lib/core/src/iter/traits/iterator.rs similarity index 100% rename from crux-mir/lib/libcore/iter/traits/iterator.rs rename to crux-mir/lib/core/src/iter/traits/iterator.rs diff --git a/crux-mir/lib/libcore/iter/traits/marker.rs b/crux-mir/lib/core/src/iter/traits/marker.rs similarity index 100% rename from crux-mir/lib/libcore/iter/traits/marker.rs rename to crux-mir/lib/core/src/iter/traits/marker.rs diff --git a/crux-mir/lib/libcore/iter/traits/mod.rs b/crux-mir/lib/core/src/iter/traits/mod.rs similarity index 100% rename from crux-mir/lib/libcore/iter/traits/mod.rs rename to crux-mir/lib/core/src/iter/traits/mod.rs diff --git a/crux-mir/lib/libcore/lib.rs b/crux-mir/lib/core/src/lib.rs similarity index 100% rename from crux-mir/lib/libcore/lib.rs rename to crux-mir/lib/core/src/lib.rs diff --git a/crux-mir/lib/libcore/macros/mod.rs b/crux-mir/lib/core/src/macros/mod.rs similarity index 100% rename from crux-mir/lib/libcore/macros/mod.rs rename to crux-mir/lib/core/src/macros/mod.rs diff --git a/crux-mir/lib/libcore/macros/panic.md b/crux-mir/lib/core/src/macros/panic.md similarity index 100% rename from crux-mir/lib/libcore/macros/panic.md rename to crux-mir/lib/core/src/macros/panic.md diff --git a/crux-mir/lib/libcore/marker.rs b/crux-mir/lib/core/src/marker.rs similarity index 100% rename from crux-mir/lib/libcore/marker.rs rename to crux-mir/lib/core/src/marker.rs diff --git a/crux-mir/lib/libcore/mem/manually_drop.rs b/crux-mir/lib/core/src/mem/manually_drop.rs similarity index 100% rename from crux-mir/lib/libcore/mem/manually_drop.rs rename to crux-mir/lib/core/src/mem/manually_drop.rs diff --git a/crux-mir/lib/libcore/mem/maybe_uninit.rs b/crux-mir/lib/core/src/mem/maybe_uninit.rs similarity index 100% rename from crux-mir/lib/libcore/mem/maybe_uninit.rs rename to crux-mir/lib/core/src/mem/maybe_uninit.rs diff --git a/crux-mir/lib/libcore/mem/mod.rs b/crux-mir/lib/core/src/mem/mod.rs similarity index 100% rename from crux-mir/lib/libcore/mem/mod.rs rename to crux-mir/lib/core/src/mem/mod.rs diff --git a/crux-mir/lib/libcore/num/bignum.rs b/crux-mir/lib/core/src/num/bignum.rs similarity index 100% rename from crux-mir/lib/libcore/num/bignum.rs rename to crux-mir/lib/core/src/num/bignum.rs diff --git a/crux-mir/lib/libcore/num/dec2flt/algorithm.rs b/crux-mir/lib/core/src/num/dec2flt/algorithm.rs similarity index 100% rename from crux-mir/lib/libcore/num/dec2flt/algorithm.rs rename to crux-mir/lib/core/src/num/dec2flt/algorithm.rs diff --git a/crux-mir/lib/libcore/num/dec2flt/mod.rs b/crux-mir/lib/core/src/num/dec2flt/mod.rs similarity index 100% rename from crux-mir/lib/libcore/num/dec2flt/mod.rs rename to crux-mir/lib/core/src/num/dec2flt/mod.rs diff --git a/crux-mir/lib/libcore/num/dec2flt/num.rs b/crux-mir/lib/core/src/num/dec2flt/num.rs similarity index 100% rename from crux-mir/lib/libcore/num/dec2flt/num.rs rename to crux-mir/lib/core/src/num/dec2flt/num.rs diff --git a/crux-mir/lib/libcore/num/dec2flt/parse.rs b/crux-mir/lib/core/src/num/dec2flt/parse.rs similarity index 100% rename from crux-mir/lib/libcore/num/dec2flt/parse.rs rename to crux-mir/lib/core/src/num/dec2flt/parse.rs diff --git a/crux-mir/lib/libcore/num/dec2flt/rawfp.rs b/crux-mir/lib/core/src/num/dec2flt/rawfp.rs similarity index 100% rename from crux-mir/lib/libcore/num/dec2flt/rawfp.rs rename to crux-mir/lib/core/src/num/dec2flt/rawfp.rs diff --git a/crux-mir/lib/libcore/num/dec2flt/table.rs b/crux-mir/lib/core/src/num/dec2flt/table.rs similarity index 100% rename from crux-mir/lib/libcore/num/dec2flt/table.rs rename to crux-mir/lib/core/src/num/dec2flt/table.rs diff --git a/crux-mir/lib/libcore/num/diy_float.rs b/crux-mir/lib/core/src/num/diy_float.rs similarity index 100% rename from crux-mir/lib/libcore/num/diy_float.rs rename to crux-mir/lib/core/src/num/diy_float.rs diff --git a/crux-mir/lib/libcore/num/f32.rs b/crux-mir/lib/core/src/num/f32.rs similarity index 100% rename from crux-mir/lib/libcore/num/f32.rs rename to crux-mir/lib/core/src/num/f32.rs diff --git a/crux-mir/lib/libcore/num/f64.rs b/crux-mir/lib/core/src/num/f64.rs similarity index 100% rename from crux-mir/lib/libcore/num/f64.rs rename to crux-mir/lib/core/src/num/f64.rs diff --git a/crux-mir/lib/libcore/num/flt2dec/decoder.rs b/crux-mir/lib/core/src/num/flt2dec/decoder.rs similarity index 100% rename from crux-mir/lib/libcore/num/flt2dec/decoder.rs rename to crux-mir/lib/core/src/num/flt2dec/decoder.rs diff --git a/crux-mir/lib/libcore/num/flt2dec/estimator.rs b/crux-mir/lib/core/src/num/flt2dec/estimator.rs similarity index 100% rename from crux-mir/lib/libcore/num/flt2dec/estimator.rs rename to crux-mir/lib/core/src/num/flt2dec/estimator.rs diff --git a/crux-mir/lib/libcore/num/flt2dec/mod.rs b/crux-mir/lib/core/src/num/flt2dec/mod.rs similarity index 100% rename from crux-mir/lib/libcore/num/flt2dec/mod.rs rename to crux-mir/lib/core/src/num/flt2dec/mod.rs diff --git a/crux-mir/lib/libcore/num/flt2dec/strategy/dragon.rs b/crux-mir/lib/core/src/num/flt2dec/strategy/dragon.rs similarity index 100% rename from crux-mir/lib/libcore/num/flt2dec/strategy/dragon.rs rename to crux-mir/lib/core/src/num/flt2dec/strategy/dragon.rs diff --git a/crux-mir/lib/libcore/num/flt2dec/strategy/grisu.rs b/crux-mir/lib/core/src/num/flt2dec/strategy/grisu.rs similarity index 100% rename from crux-mir/lib/libcore/num/flt2dec/strategy/grisu.rs rename to crux-mir/lib/core/src/num/flt2dec/strategy/grisu.rs diff --git a/crux-mir/lib/libcore/num/i128.rs b/crux-mir/lib/core/src/num/i128.rs similarity index 100% rename from crux-mir/lib/libcore/num/i128.rs rename to crux-mir/lib/core/src/num/i128.rs diff --git a/crux-mir/lib/libcore/num/i16.rs b/crux-mir/lib/core/src/num/i16.rs similarity index 100% rename from crux-mir/lib/libcore/num/i16.rs rename to crux-mir/lib/core/src/num/i16.rs diff --git a/crux-mir/lib/libcore/num/i32.rs b/crux-mir/lib/core/src/num/i32.rs similarity index 100% rename from crux-mir/lib/libcore/num/i32.rs rename to crux-mir/lib/core/src/num/i32.rs diff --git a/crux-mir/lib/libcore/num/i64.rs b/crux-mir/lib/core/src/num/i64.rs similarity index 100% rename from crux-mir/lib/libcore/num/i64.rs rename to crux-mir/lib/core/src/num/i64.rs diff --git a/crux-mir/lib/libcore/num/i8.rs b/crux-mir/lib/core/src/num/i8.rs similarity index 100% rename from crux-mir/lib/libcore/num/i8.rs rename to crux-mir/lib/core/src/num/i8.rs diff --git a/crux-mir/lib/libcore/num/int_macros.rs b/crux-mir/lib/core/src/num/int_macros.rs similarity index 100% rename from crux-mir/lib/libcore/num/int_macros.rs rename to crux-mir/lib/core/src/num/int_macros.rs diff --git a/crux-mir/lib/libcore/num/isize.rs b/crux-mir/lib/core/src/num/isize.rs similarity index 100% rename from crux-mir/lib/libcore/num/isize.rs rename to crux-mir/lib/core/src/num/isize.rs diff --git a/crux-mir/lib/libcore/num/mod.rs b/crux-mir/lib/core/src/num/mod.rs similarity index 100% rename from crux-mir/lib/libcore/num/mod.rs rename to crux-mir/lib/core/src/num/mod.rs diff --git a/crux-mir/lib/libcore/num/u128.rs b/crux-mir/lib/core/src/num/u128.rs similarity index 100% rename from crux-mir/lib/libcore/num/u128.rs rename to crux-mir/lib/core/src/num/u128.rs diff --git a/crux-mir/lib/libcore/num/u16.rs b/crux-mir/lib/core/src/num/u16.rs similarity index 100% rename from crux-mir/lib/libcore/num/u16.rs rename to crux-mir/lib/core/src/num/u16.rs diff --git a/crux-mir/lib/libcore/num/u32.rs b/crux-mir/lib/core/src/num/u32.rs similarity index 100% rename from crux-mir/lib/libcore/num/u32.rs rename to crux-mir/lib/core/src/num/u32.rs diff --git a/crux-mir/lib/libcore/num/u64.rs b/crux-mir/lib/core/src/num/u64.rs similarity index 100% rename from crux-mir/lib/libcore/num/u64.rs rename to crux-mir/lib/core/src/num/u64.rs diff --git a/crux-mir/lib/libcore/num/u8.rs b/crux-mir/lib/core/src/num/u8.rs similarity index 100% rename from crux-mir/lib/libcore/num/u8.rs rename to crux-mir/lib/core/src/num/u8.rs diff --git a/crux-mir/lib/libcore/num/usize.rs b/crux-mir/lib/core/src/num/usize.rs similarity index 100% rename from crux-mir/lib/libcore/num/usize.rs rename to crux-mir/lib/core/src/num/usize.rs diff --git a/crux-mir/lib/libcore/num/wrapping.rs b/crux-mir/lib/core/src/num/wrapping.rs similarity index 100% rename from crux-mir/lib/libcore/num/wrapping.rs rename to crux-mir/lib/core/src/num/wrapping.rs diff --git a/crux-mir/lib/libcore/ops/arith.rs b/crux-mir/lib/core/src/ops/arith.rs similarity index 100% rename from crux-mir/lib/libcore/ops/arith.rs rename to crux-mir/lib/core/src/ops/arith.rs diff --git a/crux-mir/lib/libcore/ops/bit.rs b/crux-mir/lib/core/src/ops/bit.rs similarity index 100% rename from crux-mir/lib/libcore/ops/bit.rs rename to crux-mir/lib/core/src/ops/bit.rs diff --git a/crux-mir/lib/libcore/ops/deref.rs b/crux-mir/lib/core/src/ops/deref.rs similarity index 100% rename from crux-mir/lib/libcore/ops/deref.rs rename to crux-mir/lib/core/src/ops/deref.rs diff --git a/crux-mir/lib/libcore/ops/drop.rs b/crux-mir/lib/core/src/ops/drop.rs similarity index 100% rename from crux-mir/lib/libcore/ops/drop.rs rename to crux-mir/lib/core/src/ops/drop.rs diff --git a/crux-mir/lib/libcore/ops/function.rs b/crux-mir/lib/core/src/ops/function.rs similarity index 100% rename from crux-mir/lib/libcore/ops/function.rs rename to crux-mir/lib/core/src/ops/function.rs diff --git a/crux-mir/lib/libcore/ops/generator.rs b/crux-mir/lib/core/src/ops/generator.rs similarity index 100% rename from crux-mir/lib/libcore/ops/generator.rs rename to crux-mir/lib/core/src/ops/generator.rs diff --git a/crux-mir/lib/libcore/ops/index.rs b/crux-mir/lib/core/src/ops/index.rs similarity index 100% rename from crux-mir/lib/libcore/ops/index.rs rename to crux-mir/lib/core/src/ops/index.rs diff --git a/crux-mir/lib/libcore/ops/mod.rs b/crux-mir/lib/core/src/ops/mod.rs similarity index 100% rename from crux-mir/lib/libcore/ops/mod.rs rename to crux-mir/lib/core/src/ops/mod.rs diff --git a/crux-mir/lib/libcore/ops/range.rs b/crux-mir/lib/core/src/ops/range.rs similarity index 100% rename from crux-mir/lib/libcore/ops/range.rs rename to crux-mir/lib/core/src/ops/range.rs diff --git a/crux-mir/lib/libcore/ops/try.rs b/crux-mir/lib/core/src/ops/try.rs similarity index 100% rename from crux-mir/lib/libcore/ops/try.rs rename to crux-mir/lib/core/src/ops/try.rs diff --git a/crux-mir/lib/libcore/ops/unsize.rs b/crux-mir/lib/core/src/ops/unsize.rs similarity index 100% rename from crux-mir/lib/libcore/ops/unsize.rs rename to crux-mir/lib/core/src/ops/unsize.rs diff --git a/crux-mir/lib/libcore/option.rs b/crux-mir/lib/core/src/option.rs similarity index 100% rename from crux-mir/lib/libcore/option.rs rename to crux-mir/lib/core/src/option.rs diff --git a/crux-mir/lib/libcore/panic.rs b/crux-mir/lib/core/src/panic.rs similarity index 100% rename from crux-mir/lib/libcore/panic.rs rename to crux-mir/lib/core/src/panic.rs diff --git a/crux-mir/lib/libcore/panicking.rs b/crux-mir/lib/core/src/panicking.rs similarity index 100% rename from crux-mir/lib/libcore/panicking.rs rename to crux-mir/lib/core/src/panicking.rs diff --git a/crux-mir/lib/libcore/pin.rs b/crux-mir/lib/core/src/pin.rs similarity index 100% rename from crux-mir/lib/libcore/pin.rs rename to crux-mir/lib/core/src/pin.rs diff --git a/crux-mir/lib/libcore/prelude/mod.rs b/crux-mir/lib/core/src/prelude/mod.rs similarity index 100% rename from crux-mir/lib/libcore/prelude/mod.rs rename to crux-mir/lib/core/src/prelude/mod.rs diff --git a/crux-mir/lib/libcore/prelude/v1.rs b/crux-mir/lib/core/src/prelude/v1.rs similarity index 100% rename from crux-mir/lib/libcore/prelude/v1.rs rename to crux-mir/lib/core/src/prelude/v1.rs diff --git a/crux-mir/lib/libcore/primitive.rs b/crux-mir/lib/core/src/primitive.rs similarity index 100% rename from crux-mir/lib/libcore/primitive.rs rename to crux-mir/lib/core/src/primitive.rs diff --git a/crux-mir/lib/libcore/ptr/const_ptr.rs b/crux-mir/lib/core/src/ptr/const_ptr.rs similarity index 100% rename from crux-mir/lib/libcore/ptr/const_ptr.rs rename to crux-mir/lib/core/src/ptr/const_ptr.rs diff --git a/crux-mir/lib/libcore/ptr/mod.rs b/crux-mir/lib/core/src/ptr/mod.rs similarity index 100% rename from crux-mir/lib/libcore/ptr/mod.rs rename to crux-mir/lib/core/src/ptr/mod.rs diff --git a/crux-mir/lib/libcore/ptr/mut_ptr.rs b/crux-mir/lib/core/src/ptr/mut_ptr.rs similarity index 100% rename from crux-mir/lib/libcore/ptr/mut_ptr.rs rename to crux-mir/lib/core/src/ptr/mut_ptr.rs diff --git a/crux-mir/lib/libcore/ptr/non_null.rs b/crux-mir/lib/core/src/ptr/non_null.rs similarity index 100% rename from crux-mir/lib/libcore/ptr/non_null.rs rename to crux-mir/lib/core/src/ptr/non_null.rs diff --git a/crux-mir/lib/libcore/ptr/unique.rs b/crux-mir/lib/core/src/ptr/unique.rs similarity index 100% rename from crux-mir/lib/libcore/ptr/unique.rs rename to crux-mir/lib/core/src/ptr/unique.rs diff --git a/crux-mir/lib/libcore/raw.rs b/crux-mir/lib/core/src/raw.rs similarity index 100% rename from crux-mir/lib/libcore/raw.rs rename to crux-mir/lib/core/src/raw.rs diff --git a/crux-mir/lib/libcore/result.rs b/crux-mir/lib/core/src/result.rs similarity index 100% rename from crux-mir/lib/libcore/result.rs rename to crux-mir/lib/core/src/result.rs diff --git a/crux-mir/lib/libcore/slice/memchr.rs b/crux-mir/lib/core/src/slice/memchr.rs similarity index 100% rename from crux-mir/lib/libcore/slice/memchr.rs rename to crux-mir/lib/core/src/slice/memchr.rs diff --git a/crux-mir/lib/libcore/slice/mod.rs b/crux-mir/lib/core/src/slice/mod.rs similarity index 100% rename from crux-mir/lib/libcore/slice/mod.rs rename to crux-mir/lib/core/src/slice/mod.rs diff --git a/crux-mir/lib/libcore/slice/rotate.rs b/crux-mir/lib/core/src/slice/rotate.rs similarity index 100% rename from crux-mir/lib/libcore/slice/rotate.rs rename to crux-mir/lib/core/src/slice/rotate.rs diff --git a/crux-mir/lib/libcore/slice/sort.rs b/crux-mir/lib/core/src/slice/sort.rs similarity index 100% rename from crux-mir/lib/libcore/slice/sort.rs rename to crux-mir/lib/core/src/slice/sort.rs diff --git a/crux-mir/lib/libcore/str/lossy.rs b/crux-mir/lib/core/src/str/lossy.rs similarity index 100% rename from crux-mir/lib/libcore/str/lossy.rs rename to crux-mir/lib/core/src/str/lossy.rs diff --git a/crux-mir/lib/libcore/str/mod.rs b/crux-mir/lib/core/src/str/mod.rs similarity index 100% rename from crux-mir/lib/libcore/str/mod.rs rename to crux-mir/lib/core/src/str/mod.rs diff --git a/crux-mir/lib/libcore/str/pattern.rs b/crux-mir/lib/core/src/str/pattern.rs similarity index 100% rename from crux-mir/lib/libcore/str/pattern.rs rename to crux-mir/lib/core/src/str/pattern.rs diff --git a/crux-mir/lib/libcore/sync/atomic.rs b/crux-mir/lib/core/src/sync/atomic.rs similarity index 100% rename from crux-mir/lib/libcore/sync/atomic.rs rename to crux-mir/lib/core/src/sync/atomic.rs diff --git a/crux-mir/lib/libcore/sync/mod.rs b/crux-mir/lib/core/src/sync/mod.rs similarity index 100% rename from crux-mir/lib/libcore/sync/mod.rs rename to crux-mir/lib/core/src/sync/mod.rs diff --git a/crux-mir/lib/libcore/task/mod.rs b/crux-mir/lib/core/src/task/mod.rs similarity index 100% rename from crux-mir/lib/libcore/task/mod.rs rename to crux-mir/lib/core/src/task/mod.rs diff --git a/crux-mir/lib/libcore/task/poll.rs b/crux-mir/lib/core/src/task/poll.rs similarity index 100% rename from crux-mir/lib/libcore/task/poll.rs rename to crux-mir/lib/core/src/task/poll.rs diff --git a/crux-mir/lib/libcore/task/wake.rs b/crux-mir/lib/core/src/task/wake.rs similarity index 100% rename from crux-mir/lib/libcore/task/wake.rs rename to crux-mir/lib/core/src/task/wake.rs diff --git a/crux-mir/lib/libcore/time.rs b/crux-mir/lib/core/src/time.rs similarity index 100% rename from crux-mir/lib/libcore/time.rs rename to crux-mir/lib/core/src/time.rs diff --git a/crux-mir/lib/libcore/tuple.rs b/crux-mir/lib/core/src/tuple.rs similarity index 100% rename from crux-mir/lib/libcore/tuple.rs rename to crux-mir/lib/core/src/tuple.rs diff --git a/crux-mir/lib/libcore/unicode/mod.rs b/crux-mir/lib/core/src/unicode/mod.rs similarity index 100% rename from crux-mir/lib/libcore/unicode/mod.rs rename to crux-mir/lib/core/src/unicode/mod.rs diff --git a/crux-mir/lib/libcore/unicode/printable.py b/crux-mir/lib/core/src/unicode/printable.py similarity index 100% rename from crux-mir/lib/libcore/unicode/printable.py rename to crux-mir/lib/core/src/unicode/printable.py diff --git a/crux-mir/lib/libcore/unicode/printable.rs b/crux-mir/lib/core/src/unicode/printable.rs similarity index 100% rename from crux-mir/lib/libcore/unicode/printable.rs rename to crux-mir/lib/core/src/unicode/printable.rs diff --git a/crux-mir/lib/libcore/unicode/unicode_data.rs b/crux-mir/lib/core/src/unicode/unicode_data.rs similarity index 100% rename from crux-mir/lib/libcore/unicode/unicode_data.rs rename to crux-mir/lib/core/src/unicode/unicode_data.rs diff --git a/crux-mir/lib/libcore/unicode/version.rs b/crux-mir/lib/core/src/unicode/version.rs similarity index 100% rename from crux-mir/lib/libcore/unicode/version.rs rename to crux-mir/lib/core/src/unicode/version.rs diff --git a/crux-mir/lib/libcore/unit.rs b/crux-mir/lib/core/src/unit.rs similarity index 100% rename from crux-mir/lib/libcore/unit.rs rename to crux-mir/lib/core/src/unit.rs diff --git a/crux-mir/lib/libcore/tests/alloc.rs b/crux-mir/lib/core/tests/alloc.rs similarity index 100% rename from crux-mir/lib/libcore/tests/alloc.rs rename to crux-mir/lib/core/tests/alloc.rs diff --git a/crux-mir/lib/libcore/tests/any.rs b/crux-mir/lib/core/tests/any.rs similarity index 100% rename from crux-mir/lib/libcore/tests/any.rs rename to crux-mir/lib/core/tests/any.rs diff --git a/crux-mir/lib/libcore/tests/array.rs b/crux-mir/lib/core/tests/array.rs similarity index 100% rename from crux-mir/lib/libcore/tests/array.rs rename to crux-mir/lib/core/tests/array.rs diff --git a/crux-mir/lib/libcore/tests/ascii.rs b/crux-mir/lib/core/tests/ascii.rs similarity index 100% rename from crux-mir/lib/libcore/tests/ascii.rs rename to crux-mir/lib/core/tests/ascii.rs diff --git a/crux-mir/lib/libcore/tests/atomic.rs b/crux-mir/lib/core/tests/atomic.rs similarity index 100% rename from crux-mir/lib/libcore/tests/atomic.rs rename to crux-mir/lib/core/tests/atomic.rs diff --git a/crux-mir/lib/libcore/tests/bool.rs b/crux-mir/lib/core/tests/bool.rs similarity index 100% rename from crux-mir/lib/libcore/tests/bool.rs rename to crux-mir/lib/core/tests/bool.rs diff --git a/crux-mir/lib/libcore/tests/cell.rs b/crux-mir/lib/core/tests/cell.rs similarity index 100% rename from crux-mir/lib/libcore/tests/cell.rs rename to crux-mir/lib/core/tests/cell.rs diff --git a/crux-mir/lib/libcore/tests/char.rs b/crux-mir/lib/core/tests/char.rs similarity index 100% rename from crux-mir/lib/libcore/tests/char.rs rename to crux-mir/lib/core/tests/char.rs diff --git a/crux-mir/lib/libcore/tests/clone.rs b/crux-mir/lib/core/tests/clone.rs similarity index 100% rename from crux-mir/lib/libcore/tests/clone.rs rename to crux-mir/lib/core/tests/clone.rs diff --git a/crux-mir/lib/libcore/tests/cmp.rs b/crux-mir/lib/core/tests/cmp.rs similarity index 100% rename from crux-mir/lib/libcore/tests/cmp.rs rename to crux-mir/lib/core/tests/cmp.rs diff --git a/crux-mir/lib/libcore/tests/fmt/builders.rs b/crux-mir/lib/core/tests/fmt/builders.rs similarity index 100% rename from crux-mir/lib/libcore/tests/fmt/builders.rs rename to crux-mir/lib/core/tests/fmt/builders.rs diff --git a/crux-mir/lib/libcore/tests/fmt/float.rs b/crux-mir/lib/core/tests/fmt/float.rs similarity index 100% rename from crux-mir/lib/libcore/tests/fmt/float.rs rename to crux-mir/lib/core/tests/fmt/float.rs diff --git a/crux-mir/lib/libcore/tests/fmt/mod.rs b/crux-mir/lib/core/tests/fmt/mod.rs similarity index 100% rename from crux-mir/lib/libcore/tests/fmt/mod.rs rename to crux-mir/lib/core/tests/fmt/mod.rs diff --git a/crux-mir/lib/libcore/tests/fmt/num.rs b/crux-mir/lib/core/tests/fmt/num.rs similarity index 100% rename from crux-mir/lib/libcore/tests/fmt/num.rs rename to crux-mir/lib/core/tests/fmt/num.rs diff --git a/crux-mir/lib/libcore/tests/hash/mod.rs b/crux-mir/lib/core/tests/hash/mod.rs similarity index 100% rename from crux-mir/lib/libcore/tests/hash/mod.rs rename to crux-mir/lib/core/tests/hash/mod.rs diff --git a/crux-mir/lib/libcore/tests/hash/sip.rs b/crux-mir/lib/core/tests/hash/sip.rs similarity index 100% rename from crux-mir/lib/libcore/tests/hash/sip.rs rename to crux-mir/lib/core/tests/hash/sip.rs diff --git a/crux-mir/lib/libcore/tests/intrinsics.rs b/crux-mir/lib/core/tests/intrinsics.rs similarity index 100% rename from crux-mir/lib/libcore/tests/intrinsics.rs rename to crux-mir/lib/core/tests/intrinsics.rs diff --git a/crux-mir/lib/libcore/tests/iter.rs b/crux-mir/lib/core/tests/iter.rs similarity index 100% rename from crux-mir/lib/libcore/tests/iter.rs rename to crux-mir/lib/core/tests/iter.rs diff --git a/crux-mir/lib/libcore/tests/lib.rs b/crux-mir/lib/core/tests/lib.rs similarity index 100% rename from crux-mir/lib/libcore/tests/lib.rs rename to crux-mir/lib/core/tests/lib.rs diff --git a/crux-mir/lib/libcore/tests/manually_drop.rs b/crux-mir/lib/core/tests/manually_drop.rs similarity index 100% rename from crux-mir/lib/libcore/tests/manually_drop.rs rename to crux-mir/lib/core/tests/manually_drop.rs diff --git a/crux-mir/lib/libcore/tests/mem.rs b/crux-mir/lib/core/tests/mem.rs similarity index 100% rename from crux-mir/lib/libcore/tests/mem.rs rename to crux-mir/lib/core/tests/mem.rs diff --git a/crux-mir/lib/libcore/tests/nonzero.rs b/crux-mir/lib/core/tests/nonzero.rs similarity index 100% rename from crux-mir/lib/libcore/tests/nonzero.rs rename to crux-mir/lib/core/tests/nonzero.rs diff --git a/crux-mir/lib/libcore/tests/num/bignum.rs b/crux-mir/lib/core/tests/num/bignum.rs similarity index 100% rename from crux-mir/lib/libcore/tests/num/bignum.rs rename to crux-mir/lib/core/tests/num/bignum.rs diff --git a/crux-mir/lib/libcore/tests/num/dec2flt/mod.rs b/crux-mir/lib/core/tests/num/dec2flt/mod.rs similarity index 100% rename from crux-mir/lib/libcore/tests/num/dec2flt/mod.rs rename to crux-mir/lib/core/tests/num/dec2flt/mod.rs diff --git a/crux-mir/lib/libcore/tests/num/dec2flt/parse.rs b/crux-mir/lib/core/tests/num/dec2flt/parse.rs similarity index 100% rename from crux-mir/lib/libcore/tests/num/dec2flt/parse.rs rename to crux-mir/lib/core/tests/num/dec2flt/parse.rs diff --git a/crux-mir/lib/libcore/tests/num/dec2flt/rawfp.rs b/crux-mir/lib/core/tests/num/dec2flt/rawfp.rs similarity index 100% rename from crux-mir/lib/libcore/tests/num/dec2flt/rawfp.rs rename to crux-mir/lib/core/tests/num/dec2flt/rawfp.rs diff --git a/crux-mir/lib/libcore/tests/num/flt2dec/estimator.rs b/crux-mir/lib/core/tests/num/flt2dec/estimator.rs similarity index 100% rename from crux-mir/lib/libcore/tests/num/flt2dec/estimator.rs rename to crux-mir/lib/core/tests/num/flt2dec/estimator.rs diff --git a/crux-mir/lib/libcore/tests/num/flt2dec/mod.rs b/crux-mir/lib/core/tests/num/flt2dec/mod.rs similarity index 100% rename from crux-mir/lib/libcore/tests/num/flt2dec/mod.rs rename to crux-mir/lib/core/tests/num/flt2dec/mod.rs diff --git a/crux-mir/lib/libcore/tests/num/flt2dec/random.rs b/crux-mir/lib/core/tests/num/flt2dec/random.rs similarity index 100% rename from crux-mir/lib/libcore/tests/num/flt2dec/random.rs rename to crux-mir/lib/core/tests/num/flt2dec/random.rs diff --git a/crux-mir/lib/libcore/tests/num/flt2dec/strategy/dragon.rs b/crux-mir/lib/core/tests/num/flt2dec/strategy/dragon.rs similarity index 100% rename from crux-mir/lib/libcore/tests/num/flt2dec/strategy/dragon.rs rename to crux-mir/lib/core/tests/num/flt2dec/strategy/dragon.rs diff --git a/crux-mir/lib/libcore/tests/num/flt2dec/strategy/grisu.rs b/crux-mir/lib/core/tests/num/flt2dec/strategy/grisu.rs similarity index 100% rename from crux-mir/lib/libcore/tests/num/flt2dec/strategy/grisu.rs rename to crux-mir/lib/core/tests/num/flt2dec/strategy/grisu.rs diff --git a/crux-mir/lib/libcore/tests/num/i16.rs b/crux-mir/lib/core/tests/num/i16.rs similarity index 100% rename from crux-mir/lib/libcore/tests/num/i16.rs rename to crux-mir/lib/core/tests/num/i16.rs diff --git a/crux-mir/lib/libcore/tests/num/i32.rs b/crux-mir/lib/core/tests/num/i32.rs similarity index 100% rename from crux-mir/lib/libcore/tests/num/i32.rs rename to crux-mir/lib/core/tests/num/i32.rs diff --git a/crux-mir/lib/libcore/tests/num/i64.rs b/crux-mir/lib/core/tests/num/i64.rs similarity index 100% rename from crux-mir/lib/libcore/tests/num/i64.rs rename to crux-mir/lib/core/tests/num/i64.rs diff --git a/crux-mir/lib/libcore/tests/num/i8.rs b/crux-mir/lib/core/tests/num/i8.rs similarity index 100% rename from crux-mir/lib/libcore/tests/num/i8.rs rename to crux-mir/lib/core/tests/num/i8.rs diff --git a/crux-mir/lib/libcore/tests/num/int_macros.rs b/crux-mir/lib/core/tests/num/int_macros.rs similarity index 100% rename from crux-mir/lib/libcore/tests/num/int_macros.rs rename to crux-mir/lib/core/tests/num/int_macros.rs diff --git a/crux-mir/lib/libcore/tests/num/mod.rs b/crux-mir/lib/core/tests/num/mod.rs similarity index 100% rename from crux-mir/lib/libcore/tests/num/mod.rs rename to crux-mir/lib/core/tests/num/mod.rs diff --git a/crux-mir/lib/libcore/tests/num/u16.rs b/crux-mir/lib/core/tests/num/u16.rs similarity index 100% rename from crux-mir/lib/libcore/tests/num/u16.rs rename to crux-mir/lib/core/tests/num/u16.rs diff --git a/crux-mir/lib/libcore/tests/num/u32.rs b/crux-mir/lib/core/tests/num/u32.rs similarity index 100% rename from crux-mir/lib/libcore/tests/num/u32.rs rename to crux-mir/lib/core/tests/num/u32.rs diff --git a/crux-mir/lib/libcore/tests/num/u64.rs b/crux-mir/lib/core/tests/num/u64.rs similarity index 100% rename from crux-mir/lib/libcore/tests/num/u64.rs rename to crux-mir/lib/core/tests/num/u64.rs diff --git a/crux-mir/lib/libcore/tests/num/u8.rs b/crux-mir/lib/core/tests/num/u8.rs similarity index 100% rename from crux-mir/lib/libcore/tests/num/u8.rs rename to crux-mir/lib/core/tests/num/u8.rs diff --git a/crux-mir/lib/libcore/tests/num/uint_macros.rs b/crux-mir/lib/core/tests/num/uint_macros.rs similarity index 100% rename from crux-mir/lib/libcore/tests/num/uint_macros.rs rename to crux-mir/lib/core/tests/num/uint_macros.rs diff --git a/crux-mir/lib/libcore/tests/ops.rs b/crux-mir/lib/core/tests/ops.rs similarity index 100% rename from crux-mir/lib/libcore/tests/ops.rs rename to crux-mir/lib/core/tests/ops.rs diff --git a/crux-mir/lib/libcore/tests/option.rs b/crux-mir/lib/core/tests/option.rs similarity index 100% rename from crux-mir/lib/libcore/tests/option.rs rename to crux-mir/lib/core/tests/option.rs diff --git a/crux-mir/lib/libcore/tests/pattern.rs b/crux-mir/lib/core/tests/pattern.rs similarity index 100% rename from crux-mir/lib/libcore/tests/pattern.rs rename to crux-mir/lib/core/tests/pattern.rs diff --git a/crux-mir/lib/libcore/tests/ptr.rs b/crux-mir/lib/core/tests/ptr.rs similarity index 100% rename from crux-mir/lib/libcore/tests/ptr.rs rename to crux-mir/lib/core/tests/ptr.rs diff --git a/crux-mir/lib/libcore/tests/result.rs b/crux-mir/lib/core/tests/result.rs similarity index 100% rename from crux-mir/lib/libcore/tests/result.rs rename to crux-mir/lib/core/tests/result.rs diff --git a/crux-mir/lib/libcore/tests/slice.rs b/crux-mir/lib/core/tests/slice.rs similarity index 100% rename from crux-mir/lib/libcore/tests/slice.rs rename to crux-mir/lib/core/tests/slice.rs diff --git a/crux-mir/lib/libcore/tests/str.rs b/crux-mir/lib/core/tests/str.rs similarity index 100% rename from crux-mir/lib/libcore/tests/str.rs rename to crux-mir/lib/core/tests/str.rs diff --git a/crux-mir/lib/libcore/tests/str_lossy.rs b/crux-mir/lib/core/tests/str_lossy.rs similarity index 100% rename from crux-mir/lib/libcore/tests/str_lossy.rs rename to crux-mir/lib/core/tests/str_lossy.rs diff --git a/crux-mir/lib/libcore/tests/time.rs b/crux-mir/lib/core/tests/time.rs similarity index 100% rename from crux-mir/lib/libcore/tests/time.rs rename to crux-mir/lib/core/tests/time.rs diff --git a/crux-mir/lib/libcore/tests/tuple.rs b/crux-mir/lib/core/tests/tuple.rs similarity index 100% rename from crux-mir/lib/libcore/tests/tuple.rs rename to crux-mir/lib/core/tests/tuple.rs diff --git a/crux-mir/lib/libpanic_abort/Cargo.toml b/crux-mir/lib/panic_abort/Cargo.toml similarity index 100% rename from crux-mir/lib/libpanic_abort/Cargo.toml rename to crux-mir/lib/panic_abort/Cargo.toml diff --git a/crux-mir/lib/libpanic_abort/lib.rs b/crux-mir/lib/panic_abort/lib.rs similarity index 100% rename from crux-mir/lib/libpanic_abort/lib.rs rename to crux-mir/lib/panic_abort/lib.rs diff --git a/crux-mir/lib/panic_abort/src/lib.rs b/crux-mir/lib/panic_abort/src/lib.rs new file mode 100644 index 000000000..f44a875c9 --- /dev/null +++ b/crux-mir/lib/panic_abort/src/lib.rs @@ -0,0 +1,126 @@ +//! Implementation of Rust panics via process aborts +//! +//! When compared to the implementation via unwinding, this crate is *much* +//! simpler! That being said, it's not quite as versatile, but here goes! + +#![no_std] +#![unstable(feature = "panic_abort", issue = "32837")] +#![doc( + html_root_url = "https://doc.rust-lang.org/nightly/", + issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/" +)] +#![panic_runtime] +#![allow(unused_features)] +#![feature(core_intrinsics)] +#![feature(libc)] +#![feature(nll)] +#![feature(panic_runtime)] +#![feature(staged_api)] +#![feature(rustc_attrs)] + +use core::any::Any; + +#[rustc_std_internal_symbol] +pub unsafe extern "C" fn __rust_panic_cleanup(_: *mut u8) -> *mut (dyn Any + Send + 'static) { + unreachable!() +} + +// "Leak" the payload and shim to the relevant abort on the platform in +// question. +// +// For Unix we just use `abort` from libc as it'll trigger debuggers, core +// dumps, etc, as one might expect. On Windows, however, the best option we have +// is the `__fastfail` intrinsics, but that's unfortunately not defined in LLVM, +// and the `RaiseFailFastException` function isn't available until Windows 7 +// which would break compat with XP. For now just use `intrinsics::abort` which +// will kill us with an illegal instruction, which will do a good enough job for +// now hopefully. +#[rustc_std_internal_symbol] +pub unsafe extern "C" fn __rust_start_panic(_payload: usize) -> u32 { + abort(); + + #[cfg(any(unix, target_os = "cloudabi"))] + unsafe fn abort() -> ! { + libc::abort(); + } + + #[cfg(any(windows, all(target_arch = "wasm32", not(target_os = "emscripten"))))] + unsafe fn abort() -> ! { + core::intrinsics::abort(); + } + + #[cfg(any(target_os = "hermit", all(target_vendor = "fortanix", target_env = "sgx")))] + unsafe fn abort() -> ! { + // call std::sys::abort_internal + extern "C" { + pub fn __rust_abort() -> !; + } + __rust_abort(); + } +} + +// This... is a bit of an oddity. The tl;dr; is that this is required to link +// correctly, the longer explanation is below. +// +// Right now the binaries of libcore/libstd that we ship are all compiled with +// `-C panic=unwind`. This is done to ensure that the binaries are maximally +// compatible with as many situations as possible. The compiler, however, +// requires a "personality function" for all functions compiled with `-C +// panic=unwind`. This personality function is hardcoded to the symbol +// `rust_eh_personality` and is defined by the `eh_personality` lang item. +// +// So... why not just define that lang item here? Good question! The way that +// panic runtimes are linked in is actually a little subtle in that they're +// "sort of" in the compiler's crate store, but only actually linked if another +// isn't actually linked. This ends up meaning that both this crate and the +// panic_unwind crate can appear in the compiler's crate store, and if both +// define the `eh_personality` lang item then that'll hit an error. +// +// To handle this the compiler only requires the `eh_personality` is defined if +// the panic runtime being linked in is the unwinding runtime, and otherwise +// it's not required to be defined (rightfully so). In this case, however, this +// library just defines this symbol so there's at least some personality +// somewhere. +// +// Essentially this symbol is just defined to get wired up to libcore/libstd +// binaries, but it should never be called as we don't link in an unwinding +// runtime at all. +pub mod personalities { + #[rustc_std_internal_symbol] + #[cfg(not(any( + all(target_arch = "wasm32", not(target_os = "emscripten"),), + all(target_os = "windows", target_env = "gnu", target_arch = "x86_64",), + )))] + pub extern "C" fn rust_eh_personality() {} + + // On x86_64-pc-windows-gnu we use our own personality function that needs + // to return `ExceptionContinueSearch` as we're passing on all our frames. + #[rustc_std_internal_symbol] + #[cfg(all(target_os = "windows", target_env = "gnu", target_arch = "x86_64"))] + pub extern "C" fn rust_eh_personality( + _record: usize, + _frame: usize, + _context: usize, + _dispatcher: usize, + ) -> u32 { + 1 // `ExceptionContinueSearch` + } + + // Similar to above, this corresponds to the `eh_unwind_resume` lang item + // that's only used on Windows currently. + // + // Note that we don't execute landing pads, so this is never called, so it's + // body is empty. + #[rustc_std_internal_symbol] + #[cfg(all(bootstrap, target_os = "windows", target_env = "gnu"))] + pub extern "C" fn rust_eh_unwind_resume() {} + + // These two are called by our startup objects on i686-pc-windows-gnu, but + // they don't need to do anything so the bodies are nops. + #[rustc_std_internal_symbol] + #[cfg(all(target_os = "windows", target_env = "gnu", target_arch = "x86"))] + pub extern "C" fn rust_eh_register_frames() {} + #[rustc_std_internal_symbol] + #[cfg(all(target_os = "windows", target_env = "gnu", target_arch = "x86"))] + pub extern "C" fn rust_eh_unregister_frames() {} +} diff --git a/crux-mir/lib/libpanic_unwind/Cargo.toml b/crux-mir/lib/panic_unwind/Cargo.toml similarity index 100% rename from crux-mir/lib/libpanic_unwind/Cargo.toml rename to crux-mir/lib/panic_unwind/Cargo.toml diff --git a/crux-mir/lib/libpanic_unwind/dummy.rs b/crux-mir/lib/panic_unwind/dummy.rs similarity index 100% rename from crux-mir/lib/libpanic_unwind/dummy.rs rename to crux-mir/lib/panic_unwind/dummy.rs diff --git a/crux-mir/lib/libpanic_unwind/dwarf/eh.rs b/crux-mir/lib/panic_unwind/dwarf/eh.rs similarity index 100% rename from crux-mir/lib/libpanic_unwind/dwarf/eh.rs rename to crux-mir/lib/panic_unwind/dwarf/eh.rs diff --git a/crux-mir/lib/libpanic_unwind/dwarf/mod.rs b/crux-mir/lib/panic_unwind/dwarf/mod.rs similarity index 100% rename from crux-mir/lib/libpanic_unwind/dwarf/mod.rs rename to crux-mir/lib/panic_unwind/dwarf/mod.rs diff --git a/crux-mir/lib/libpanic_unwind/dwarf/tests.rs b/crux-mir/lib/panic_unwind/dwarf/tests.rs similarity index 100% rename from crux-mir/lib/libpanic_unwind/dwarf/tests.rs rename to crux-mir/lib/panic_unwind/dwarf/tests.rs diff --git a/crux-mir/lib/libpanic_unwind/emcc.rs b/crux-mir/lib/panic_unwind/emcc.rs similarity index 100% rename from crux-mir/lib/libpanic_unwind/emcc.rs rename to crux-mir/lib/panic_unwind/emcc.rs diff --git a/crux-mir/lib/libpanic_unwind/gcc.rs b/crux-mir/lib/panic_unwind/gcc.rs similarity index 100% rename from crux-mir/lib/libpanic_unwind/gcc.rs rename to crux-mir/lib/panic_unwind/gcc.rs diff --git a/crux-mir/lib/libpanic_unwind/hermit.rs b/crux-mir/lib/panic_unwind/hermit.rs similarity index 100% rename from crux-mir/lib/libpanic_unwind/hermit.rs rename to crux-mir/lib/panic_unwind/hermit.rs diff --git a/crux-mir/lib/libpanic_unwind/lib.rs b/crux-mir/lib/panic_unwind/lib.rs similarity index 100% rename from crux-mir/lib/libpanic_unwind/lib.rs rename to crux-mir/lib/panic_unwind/lib.rs diff --git a/crux-mir/lib/libpanic_unwind/miri.rs b/crux-mir/lib/panic_unwind/miri.rs similarity index 100% rename from crux-mir/lib/libpanic_unwind/miri.rs rename to crux-mir/lib/panic_unwind/miri.rs diff --git a/crux-mir/lib/libpanic_unwind/seh.rs b/crux-mir/lib/panic_unwind/seh.rs similarity index 100% rename from crux-mir/lib/libpanic_unwind/seh.rs rename to crux-mir/lib/panic_unwind/seh.rs diff --git a/crux-mir/lib/libstd/benches/hash/map.rs b/crux-mir/lib/std/benches/hash/map.rs similarity index 100% rename from crux-mir/lib/libstd/benches/hash/map.rs rename to crux-mir/lib/std/benches/hash/map.rs diff --git a/crux-mir/lib/libstd/benches/hash/mod.rs b/crux-mir/lib/std/benches/hash/mod.rs similarity index 100% rename from crux-mir/lib/libstd/benches/hash/mod.rs rename to crux-mir/lib/std/benches/hash/mod.rs diff --git a/crux-mir/lib/libstd/benches/hash/set_ops.rs b/crux-mir/lib/std/benches/hash/set_ops.rs similarity index 100% rename from crux-mir/lib/libstd/benches/hash/set_ops.rs rename to crux-mir/lib/std/benches/hash/set_ops.rs diff --git a/crux-mir/lib/libstd/benches/lib.rs b/crux-mir/lib/std/benches/lib.rs similarity index 100% rename from crux-mir/lib/libstd/benches/lib.rs rename to crux-mir/lib/std/benches/lib.rs diff --git a/crux-mir/lib/libstd/Cargo.toml b/crux-mir/lib/std/src/Cargo.toml similarity index 100% rename from crux-mir/lib/libstd/Cargo.toml rename to crux-mir/lib/std/src/Cargo.toml diff --git a/crux-mir/lib/libstd/alloc.rs b/crux-mir/lib/std/src/alloc.rs similarity index 100% rename from crux-mir/lib/libstd/alloc.rs rename to crux-mir/lib/std/src/alloc.rs diff --git a/crux-mir/lib/libstd/ascii.rs b/crux-mir/lib/std/src/ascii.rs similarity index 100% rename from crux-mir/lib/libstd/ascii.rs rename to crux-mir/lib/std/src/ascii.rs diff --git a/crux-mir/lib/libstd/backtrace.rs b/crux-mir/lib/std/src/backtrace.rs similarity index 100% rename from crux-mir/lib/libstd/backtrace.rs rename to crux-mir/lib/std/src/backtrace.rs diff --git a/crux-mir/lib/libstd/build.rs b/crux-mir/lib/std/src/build.rs similarity index 100% rename from crux-mir/lib/libstd/build.rs rename to crux-mir/lib/std/src/build.rs diff --git a/crux-mir/lib/libstd/collections/hash/crucible_map.rs b/crux-mir/lib/std/src/collections/hash/crucible_map.rs similarity index 100% rename from crux-mir/lib/libstd/collections/hash/crucible_map.rs rename to crux-mir/lib/std/src/collections/hash/crucible_map.rs diff --git a/crux-mir/lib/libstd/collections/hash/map.rs b/crux-mir/lib/std/src/collections/hash/map.rs similarity index 100% rename from crux-mir/lib/libstd/collections/hash/map.rs rename to crux-mir/lib/std/src/collections/hash/map.rs diff --git a/crux-mir/lib/libstd/collections/hash/mod.rs b/crux-mir/lib/std/src/collections/hash/mod.rs similarity index 100% rename from crux-mir/lib/libstd/collections/hash/mod.rs rename to crux-mir/lib/std/src/collections/hash/mod.rs diff --git a/crux-mir/lib/libstd/collections/hash/set.rs b/crux-mir/lib/std/src/collections/hash/set.rs similarity index 100% rename from crux-mir/lib/libstd/collections/hash/set.rs rename to crux-mir/lib/std/src/collections/hash/set.rs diff --git a/crux-mir/lib/libstd/collections/mod.rs b/crux-mir/lib/std/src/collections/mod.rs similarity index 100% rename from crux-mir/lib/libstd/collections/mod.rs rename to crux-mir/lib/std/src/collections/mod.rs diff --git a/crux-mir/lib/libstd/env.rs b/crux-mir/lib/std/src/env.rs similarity index 100% rename from crux-mir/lib/libstd/env.rs rename to crux-mir/lib/std/src/env.rs diff --git a/crux-mir/lib/libstd/error.rs b/crux-mir/lib/std/src/error.rs similarity index 100% rename from crux-mir/lib/libstd/error.rs rename to crux-mir/lib/std/src/error.rs diff --git a/crux-mir/lib/libstd/f32.rs b/crux-mir/lib/std/src/f32.rs similarity index 100% rename from crux-mir/lib/libstd/f32.rs rename to crux-mir/lib/std/src/f32.rs diff --git a/crux-mir/lib/libstd/f64.rs b/crux-mir/lib/std/src/f64.rs similarity index 100% rename from crux-mir/lib/libstd/f64.rs rename to crux-mir/lib/std/src/f64.rs diff --git a/crux-mir/lib/libstd/ffi/c_str.rs b/crux-mir/lib/std/src/ffi/c_str.rs similarity index 100% rename from crux-mir/lib/libstd/ffi/c_str.rs rename to crux-mir/lib/std/src/ffi/c_str.rs diff --git a/crux-mir/lib/libstd/ffi/mod.rs b/crux-mir/lib/std/src/ffi/mod.rs similarity index 100% rename from crux-mir/lib/libstd/ffi/mod.rs rename to crux-mir/lib/std/src/ffi/mod.rs diff --git a/crux-mir/lib/libstd/ffi/os_str.rs b/crux-mir/lib/std/src/ffi/os_str.rs similarity index 100% rename from crux-mir/lib/libstd/ffi/os_str.rs rename to crux-mir/lib/std/src/ffi/os_str.rs diff --git a/crux-mir/lib/libstd/fs.rs b/crux-mir/lib/std/src/fs.rs similarity index 100% rename from crux-mir/lib/libstd/fs.rs rename to crux-mir/lib/std/src/fs.rs diff --git a/crux-mir/lib/libstd/future.rs b/crux-mir/lib/std/src/future.rs similarity index 100% rename from crux-mir/lib/libstd/future.rs rename to crux-mir/lib/std/src/future.rs diff --git a/crux-mir/lib/libstd/io/buffered.rs b/crux-mir/lib/std/src/io/buffered.rs similarity index 100% rename from crux-mir/lib/libstd/io/buffered.rs rename to crux-mir/lib/std/src/io/buffered.rs diff --git a/crux-mir/lib/libstd/io/cursor.rs b/crux-mir/lib/std/src/io/cursor.rs similarity index 100% rename from crux-mir/lib/libstd/io/cursor.rs rename to crux-mir/lib/std/src/io/cursor.rs diff --git a/crux-mir/lib/libstd/io/error.rs b/crux-mir/lib/std/src/io/error.rs similarity index 100% rename from crux-mir/lib/libstd/io/error.rs rename to crux-mir/lib/std/src/io/error.rs diff --git a/crux-mir/lib/libstd/io/impls.rs b/crux-mir/lib/std/src/io/impls.rs similarity index 100% rename from crux-mir/lib/libstd/io/impls.rs rename to crux-mir/lib/std/src/io/impls.rs diff --git a/crux-mir/lib/libstd/io/lazy.rs b/crux-mir/lib/std/src/io/lazy.rs similarity index 100% rename from crux-mir/lib/libstd/io/lazy.rs rename to crux-mir/lib/std/src/io/lazy.rs diff --git a/crux-mir/lib/libstd/io/mod.rs b/crux-mir/lib/std/src/io/mod.rs similarity index 100% rename from crux-mir/lib/libstd/io/mod.rs rename to crux-mir/lib/std/src/io/mod.rs diff --git a/crux-mir/lib/libstd/io/prelude.rs b/crux-mir/lib/std/src/io/prelude.rs similarity index 100% rename from crux-mir/lib/libstd/io/prelude.rs rename to crux-mir/lib/std/src/io/prelude.rs diff --git a/crux-mir/lib/libstd/io/stdio.rs b/crux-mir/lib/std/src/io/stdio.rs similarity index 100% rename from crux-mir/lib/libstd/io/stdio.rs rename to crux-mir/lib/std/src/io/stdio.rs diff --git a/crux-mir/lib/libstd/io/util.rs b/crux-mir/lib/std/src/io/util.rs similarity index 100% rename from crux-mir/lib/libstd/io/util.rs rename to crux-mir/lib/std/src/io/util.rs diff --git a/crux-mir/lib/libstd/keyword_docs.rs b/crux-mir/lib/std/src/keyword_docs.rs similarity index 100% rename from crux-mir/lib/libstd/keyword_docs.rs rename to crux-mir/lib/std/src/keyword_docs.rs diff --git a/crux-mir/lib/libstd/lib.rs b/crux-mir/lib/std/src/lib.rs similarity index 100% rename from crux-mir/lib/libstd/lib.rs rename to crux-mir/lib/std/src/lib.rs diff --git a/crux-mir/lib/libstd/macros.rs b/crux-mir/lib/std/src/macros.rs similarity index 100% rename from crux-mir/lib/libstd/macros.rs rename to crux-mir/lib/std/src/macros.rs diff --git a/crux-mir/lib/libstd/memchr.rs b/crux-mir/lib/std/src/memchr.rs similarity index 100% rename from crux-mir/lib/libstd/memchr.rs rename to crux-mir/lib/std/src/memchr.rs diff --git a/crux-mir/lib/libstd/net/addr.rs b/crux-mir/lib/std/src/net/addr.rs similarity index 100% rename from crux-mir/lib/libstd/net/addr.rs rename to crux-mir/lib/std/src/net/addr.rs diff --git a/crux-mir/lib/libstd/net/ip.rs b/crux-mir/lib/std/src/net/ip.rs similarity index 100% rename from crux-mir/lib/libstd/net/ip.rs rename to crux-mir/lib/std/src/net/ip.rs diff --git a/crux-mir/lib/libstd/net/mod.rs b/crux-mir/lib/std/src/net/mod.rs similarity index 100% rename from crux-mir/lib/libstd/net/mod.rs rename to crux-mir/lib/std/src/net/mod.rs diff --git a/crux-mir/lib/libstd/net/parser.rs b/crux-mir/lib/std/src/net/parser.rs similarity index 100% rename from crux-mir/lib/libstd/net/parser.rs rename to crux-mir/lib/std/src/net/parser.rs diff --git a/crux-mir/lib/libstd/net/tcp.rs b/crux-mir/lib/std/src/net/tcp.rs similarity index 100% rename from crux-mir/lib/libstd/net/tcp.rs rename to crux-mir/lib/std/src/net/tcp.rs diff --git a/crux-mir/lib/libstd/net/test.rs b/crux-mir/lib/std/src/net/test.rs similarity index 100% rename from crux-mir/lib/libstd/net/test.rs rename to crux-mir/lib/std/src/net/test.rs diff --git a/crux-mir/lib/libstd/net/udp.rs b/crux-mir/lib/std/src/net/udp.rs similarity index 100% rename from crux-mir/lib/libstd/net/udp.rs rename to crux-mir/lib/std/src/net/udp.rs diff --git a/crux-mir/lib/libstd/num.rs b/crux-mir/lib/std/src/num.rs similarity index 100% rename from crux-mir/lib/libstd/num.rs rename to crux-mir/lib/std/src/num.rs diff --git a/crux-mir/lib/libstd/os/android/fs.rs b/crux-mir/lib/std/src/os/android/fs.rs similarity index 100% rename from crux-mir/lib/libstd/os/android/fs.rs rename to crux-mir/lib/std/src/os/android/fs.rs diff --git a/crux-mir/lib/libstd/os/android/mod.rs b/crux-mir/lib/std/src/os/android/mod.rs similarity index 100% rename from crux-mir/lib/libstd/os/android/mod.rs rename to crux-mir/lib/std/src/os/android/mod.rs diff --git a/crux-mir/lib/libstd/os/android/raw.rs b/crux-mir/lib/std/src/os/android/raw.rs similarity index 100% rename from crux-mir/lib/libstd/os/android/raw.rs rename to crux-mir/lib/std/src/os/android/raw.rs diff --git a/crux-mir/lib/libstd/os/dragonfly/fs.rs b/crux-mir/lib/std/src/os/dragonfly/fs.rs similarity index 100% rename from crux-mir/lib/libstd/os/dragonfly/fs.rs rename to crux-mir/lib/std/src/os/dragonfly/fs.rs diff --git a/crux-mir/lib/libstd/os/dragonfly/mod.rs b/crux-mir/lib/std/src/os/dragonfly/mod.rs similarity index 100% rename from crux-mir/lib/libstd/os/dragonfly/mod.rs rename to crux-mir/lib/std/src/os/dragonfly/mod.rs diff --git a/crux-mir/lib/libstd/os/dragonfly/raw.rs b/crux-mir/lib/std/src/os/dragonfly/raw.rs similarity index 100% rename from crux-mir/lib/libstd/os/dragonfly/raw.rs rename to crux-mir/lib/std/src/os/dragonfly/raw.rs diff --git a/crux-mir/lib/libstd/os/emscripten/fs.rs b/crux-mir/lib/std/src/os/emscripten/fs.rs similarity index 100% rename from crux-mir/lib/libstd/os/emscripten/fs.rs rename to crux-mir/lib/std/src/os/emscripten/fs.rs diff --git a/crux-mir/lib/libstd/os/emscripten/mod.rs b/crux-mir/lib/std/src/os/emscripten/mod.rs similarity index 100% rename from crux-mir/lib/libstd/os/emscripten/mod.rs rename to crux-mir/lib/std/src/os/emscripten/mod.rs diff --git a/crux-mir/lib/libstd/os/emscripten/raw.rs b/crux-mir/lib/std/src/os/emscripten/raw.rs similarity index 100% rename from crux-mir/lib/libstd/os/emscripten/raw.rs rename to crux-mir/lib/std/src/os/emscripten/raw.rs diff --git a/crux-mir/lib/libstd/os/fortanix_sgx/mod.rs b/crux-mir/lib/std/src/os/fortanix_sgx/mod.rs similarity index 100% rename from crux-mir/lib/libstd/os/fortanix_sgx/mod.rs rename to crux-mir/lib/std/src/os/fortanix_sgx/mod.rs diff --git a/crux-mir/lib/libstd/os/freebsd/fs.rs b/crux-mir/lib/std/src/os/freebsd/fs.rs similarity index 100% rename from crux-mir/lib/libstd/os/freebsd/fs.rs rename to crux-mir/lib/std/src/os/freebsd/fs.rs diff --git a/crux-mir/lib/libstd/os/freebsd/mod.rs b/crux-mir/lib/std/src/os/freebsd/mod.rs similarity index 100% rename from crux-mir/lib/libstd/os/freebsd/mod.rs rename to crux-mir/lib/std/src/os/freebsd/mod.rs diff --git a/crux-mir/lib/libstd/os/freebsd/raw.rs b/crux-mir/lib/std/src/os/freebsd/raw.rs similarity index 100% rename from crux-mir/lib/libstd/os/freebsd/raw.rs rename to crux-mir/lib/std/src/os/freebsd/raw.rs diff --git a/crux-mir/lib/libstd/os/fuchsia/fs.rs b/crux-mir/lib/std/src/os/fuchsia/fs.rs similarity index 100% rename from crux-mir/lib/libstd/os/fuchsia/fs.rs rename to crux-mir/lib/std/src/os/fuchsia/fs.rs diff --git a/crux-mir/lib/libstd/os/fuchsia/mod.rs b/crux-mir/lib/std/src/os/fuchsia/mod.rs similarity index 100% rename from crux-mir/lib/libstd/os/fuchsia/mod.rs rename to crux-mir/lib/std/src/os/fuchsia/mod.rs diff --git a/crux-mir/lib/libstd/os/fuchsia/raw.rs b/crux-mir/lib/std/src/os/fuchsia/raw.rs similarity index 100% rename from crux-mir/lib/libstd/os/fuchsia/raw.rs rename to crux-mir/lib/std/src/os/fuchsia/raw.rs diff --git a/crux-mir/lib/libstd/os/haiku/fs.rs b/crux-mir/lib/std/src/os/haiku/fs.rs similarity index 100% rename from crux-mir/lib/libstd/os/haiku/fs.rs rename to crux-mir/lib/std/src/os/haiku/fs.rs diff --git a/crux-mir/lib/libstd/os/haiku/mod.rs b/crux-mir/lib/std/src/os/haiku/mod.rs similarity index 100% rename from crux-mir/lib/libstd/os/haiku/mod.rs rename to crux-mir/lib/std/src/os/haiku/mod.rs diff --git a/crux-mir/lib/libstd/os/haiku/raw.rs b/crux-mir/lib/std/src/os/haiku/raw.rs similarity index 100% rename from crux-mir/lib/libstd/os/haiku/raw.rs rename to crux-mir/lib/std/src/os/haiku/raw.rs diff --git a/crux-mir/lib/libstd/os/ios/fs.rs b/crux-mir/lib/std/src/os/ios/fs.rs similarity index 100% rename from crux-mir/lib/libstd/os/ios/fs.rs rename to crux-mir/lib/std/src/os/ios/fs.rs diff --git a/crux-mir/lib/libstd/os/ios/mod.rs b/crux-mir/lib/std/src/os/ios/mod.rs similarity index 100% rename from crux-mir/lib/libstd/os/ios/mod.rs rename to crux-mir/lib/std/src/os/ios/mod.rs diff --git a/crux-mir/lib/libstd/os/ios/raw.rs b/crux-mir/lib/std/src/os/ios/raw.rs similarity index 100% rename from crux-mir/lib/libstd/os/ios/raw.rs rename to crux-mir/lib/std/src/os/ios/raw.rs diff --git a/crux-mir/lib/libstd/os/linux/fs.rs b/crux-mir/lib/std/src/os/linux/fs.rs similarity index 100% rename from crux-mir/lib/libstd/os/linux/fs.rs rename to crux-mir/lib/std/src/os/linux/fs.rs diff --git a/crux-mir/lib/libstd/os/linux/mod.rs b/crux-mir/lib/std/src/os/linux/mod.rs similarity index 100% rename from crux-mir/lib/libstd/os/linux/mod.rs rename to crux-mir/lib/std/src/os/linux/mod.rs diff --git a/crux-mir/lib/libstd/os/linux/raw.rs b/crux-mir/lib/std/src/os/linux/raw.rs similarity index 100% rename from crux-mir/lib/libstd/os/linux/raw.rs rename to crux-mir/lib/std/src/os/linux/raw.rs diff --git a/crux-mir/lib/libstd/os/macos/fs.rs b/crux-mir/lib/std/src/os/macos/fs.rs similarity index 100% rename from crux-mir/lib/libstd/os/macos/fs.rs rename to crux-mir/lib/std/src/os/macos/fs.rs diff --git a/crux-mir/lib/libstd/os/macos/mod.rs b/crux-mir/lib/std/src/os/macos/mod.rs similarity index 100% rename from crux-mir/lib/libstd/os/macos/mod.rs rename to crux-mir/lib/std/src/os/macos/mod.rs diff --git a/crux-mir/lib/libstd/os/macos/raw.rs b/crux-mir/lib/std/src/os/macos/raw.rs similarity index 100% rename from crux-mir/lib/libstd/os/macos/raw.rs rename to crux-mir/lib/std/src/os/macos/raw.rs diff --git a/crux-mir/lib/libstd/os/mod.rs b/crux-mir/lib/std/src/os/mod.rs similarity index 100% rename from crux-mir/lib/libstd/os/mod.rs rename to crux-mir/lib/std/src/os/mod.rs diff --git a/crux-mir/lib/libstd/os/netbsd/fs.rs b/crux-mir/lib/std/src/os/netbsd/fs.rs similarity index 100% rename from crux-mir/lib/libstd/os/netbsd/fs.rs rename to crux-mir/lib/std/src/os/netbsd/fs.rs diff --git a/crux-mir/lib/libstd/os/netbsd/mod.rs b/crux-mir/lib/std/src/os/netbsd/mod.rs similarity index 100% rename from crux-mir/lib/libstd/os/netbsd/mod.rs rename to crux-mir/lib/std/src/os/netbsd/mod.rs diff --git a/crux-mir/lib/libstd/os/netbsd/raw.rs b/crux-mir/lib/std/src/os/netbsd/raw.rs similarity index 100% rename from crux-mir/lib/libstd/os/netbsd/raw.rs rename to crux-mir/lib/std/src/os/netbsd/raw.rs diff --git a/crux-mir/lib/libstd/os/openbsd/fs.rs b/crux-mir/lib/std/src/os/openbsd/fs.rs similarity index 100% rename from crux-mir/lib/libstd/os/openbsd/fs.rs rename to crux-mir/lib/std/src/os/openbsd/fs.rs diff --git a/crux-mir/lib/libstd/os/openbsd/mod.rs b/crux-mir/lib/std/src/os/openbsd/mod.rs similarity index 100% rename from crux-mir/lib/libstd/os/openbsd/mod.rs rename to crux-mir/lib/std/src/os/openbsd/mod.rs diff --git a/crux-mir/lib/libstd/os/openbsd/raw.rs b/crux-mir/lib/std/src/os/openbsd/raw.rs similarity index 100% rename from crux-mir/lib/libstd/os/openbsd/raw.rs rename to crux-mir/lib/std/src/os/openbsd/raw.rs diff --git a/crux-mir/lib/libstd/os/raw/char.md b/crux-mir/lib/std/src/os/raw/char.md similarity index 100% rename from crux-mir/lib/libstd/os/raw/char.md rename to crux-mir/lib/std/src/os/raw/char.md diff --git a/crux-mir/lib/libstd/os/raw/double.md b/crux-mir/lib/std/src/os/raw/double.md similarity index 100% rename from crux-mir/lib/libstd/os/raw/double.md rename to crux-mir/lib/std/src/os/raw/double.md diff --git a/crux-mir/lib/libstd/os/raw/float.md b/crux-mir/lib/std/src/os/raw/float.md similarity index 100% rename from crux-mir/lib/libstd/os/raw/float.md rename to crux-mir/lib/std/src/os/raw/float.md diff --git a/crux-mir/lib/libstd/os/raw/int.md b/crux-mir/lib/std/src/os/raw/int.md similarity index 100% rename from crux-mir/lib/libstd/os/raw/int.md rename to crux-mir/lib/std/src/os/raw/int.md diff --git a/crux-mir/lib/libstd/os/raw/long.md b/crux-mir/lib/std/src/os/raw/long.md similarity index 100% rename from crux-mir/lib/libstd/os/raw/long.md rename to crux-mir/lib/std/src/os/raw/long.md diff --git a/crux-mir/lib/libstd/os/raw/longlong.md b/crux-mir/lib/std/src/os/raw/longlong.md similarity index 100% rename from crux-mir/lib/libstd/os/raw/longlong.md rename to crux-mir/lib/std/src/os/raw/longlong.md diff --git a/crux-mir/lib/libstd/os/raw/mod.rs b/crux-mir/lib/std/src/os/raw/mod.rs similarity index 100% rename from crux-mir/lib/libstd/os/raw/mod.rs rename to crux-mir/lib/std/src/os/raw/mod.rs diff --git a/crux-mir/lib/libstd/os/raw/schar.md b/crux-mir/lib/std/src/os/raw/schar.md similarity index 100% rename from crux-mir/lib/libstd/os/raw/schar.md rename to crux-mir/lib/std/src/os/raw/schar.md diff --git a/crux-mir/lib/libstd/os/raw/short.md b/crux-mir/lib/std/src/os/raw/short.md similarity index 100% rename from crux-mir/lib/libstd/os/raw/short.md rename to crux-mir/lib/std/src/os/raw/short.md diff --git a/crux-mir/lib/libstd/os/raw/uchar.md b/crux-mir/lib/std/src/os/raw/uchar.md similarity index 100% rename from crux-mir/lib/libstd/os/raw/uchar.md rename to crux-mir/lib/std/src/os/raw/uchar.md diff --git a/crux-mir/lib/libstd/os/raw/uint.md b/crux-mir/lib/std/src/os/raw/uint.md similarity index 100% rename from crux-mir/lib/libstd/os/raw/uint.md rename to crux-mir/lib/std/src/os/raw/uint.md diff --git a/crux-mir/lib/libstd/os/raw/ulong.md b/crux-mir/lib/std/src/os/raw/ulong.md similarity index 100% rename from crux-mir/lib/libstd/os/raw/ulong.md rename to crux-mir/lib/std/src/os/raw/ulong.md diff --git a/crux-mir/lib/libstd/os/raw/ulonglong.md b/crux-mir/lib/std/src/os/raw/ulonglong.md similarity index 100% rename from crux-mir/lib/libstd/os/raw/ulonglong.md rename to crux-mir/lib/std/src/os/raw/ulonglong.md diff --git a/crux-mir/lib/libstd/os/raw/ushort.md b/crux-mir/lib/std/src/os/raw/ushort.md similarity index 100% rename from crux-mir/lib/libstd/os/raw/ushort.md rename to crux-mir/lib/std/src/os/raw/ushort.md diff --git a/crux-mir/lib/libstd/os/redox/fs.rs b/crux-mir/lib/std/src/os/redox/fs.rs similarity index 100% rename from crux-mir/lib/libstd/os/redox/fs.rs rename to crux-mir/lib/std/src/os/redox/fs.rs diff --git a/crux-mir/lib/libstd/os/redox/mod.rs b/crux-mir/lib/std/src/os/redox/mod.rs similarity index 100% rename from crux-mir/lib/libstd/os/redox/mod.rs rename to crux-mir/lib/std/src/os/redox/mod.rs diff --git a/crux-mir/lib/libstd/os/redox/raw.rs b/crux-mir/lib/std/src/os/redox/raw.rs similarity index 100% rename from crux-mir/lib/libstd/os/redox/raw.rs rename to crux-mir/lib/std/src/os/redox/raw.rs diff --git a/crux-mir/lib/libstd/os/solaris/fs.rs b/crux-mir/lib/std/src/os/solaris/fs.rs similarity index 100% rename from crux-mir/lib/libstd/os/solaris/fs.rs rename to crux-mir/lib/std/src/os/solaris/fs.rs diff --git a/crux-mir/lib/libstd/os/solaris/mod.rs b/crux-mir/lib/std/src/os/solaris/mod.rs similarity index 100% rename from crux-mir/lib/libstd/os/solaris/mod.rs rename to crux-mir/lib/std/src/os/solaris/mod.rs diff --git a/crux-mir/lib/libstd/os/solaris/raw.rs b/crux-mir/lib/std/src/os/solaris/raw.rs similarity index 100% rename from crux-mir/lib/libstd/os/solaris/raw.rs rename to crux-mir/lib/std/src/os/solaris/raw.rs diff --git a/crux-mir/lib/libstd/os/vxworks/fs.rs b/crux-mir/lib/std/src/os/vxworks/fs.rs similarity index 100% rename from crux-mir/lib/libstd/os/vxworks/fs.rs rename to crux-mir/lib/std/src/os/vxworks/fs.rs diff --git a/crux-mir/lib/libstd/os/vxworks/mod.rs b/crux-mir/lib/std/src/os/vxworks/mod.rs similarity index 100% rename from crux-mir/lib/libstd/os/vxworks/mod.rs rename to crux-mir/lib/std/src/os/vxworks/mod.rs diff --git a/crux-mir/lib/libstd/os/vxworks/raw.rs b/crux-mir/lib/std/src/os/vxworks/raw.rs similarity index 100% rename from crux-mir/lib/libstd/os/vxworks/raw.rs rename to crux-mir/lib/std/src/os/vxworks/raw.rs diff --git a/crux-mir/lib/libstd/os/wasi.rs b/crux-mir/lib/std/src/os/wasi.rs similarity index 100% rename from crux-mir/lib/libstd/os/wasi.rs rename to crux-mir/lib/std/src/os/wasi.rs diff --git a/crux-mir/lib/libstd/panic.rs b/crux-mir/lib/std/src/panic.rs similarity index 100% rename from crux-mir/lib/libstd/panic.rs rename to crux-mir/lib/std/src/panic.rs diff --git a/crux-mir/lib/libstd/panicking.rs b/crux-mir/lib/std/src/panicking.rs similarity index 100% rename from crux-mir/lib/libstd/panicking.rs rename to crux-mir/lib/std/src/panicking.rs diff --git a/crux-mir/lib/libstd/path.rs b/crux-mir/lib/std/src/path.rs similarity index 100% rename from crux-mir/lib/libstd/path.rs rename to crux-mir/lib/std/src/path.rs diff --git a/crux-mir/lib/libstd/prelude/mod.rs b/crux-mir/lib/std/src/prelude/mod.rs similarity index 100% rename from crux-mir/lib/libstd/prelude/mod.rs rename to crux-mir/lib/std/src/prelude/mod.rs diff --git a/crux-mir/lib/libstd/prelude/v1.rs b/crux-mir/lib/std/src/prelude/v1.rs similarity index 100% rename from crux-mir/lib/libstd/prelude/v1.rs rename to crux-mir/lib/std/src/prelude/v1.rs diff --git a/crux-mir/lib/libstd/primitive_docs.rs b/crux-mir/lib/std/src/primitive_docs.rs similarity index 100% rename from crux-mir/lib/libstd/primitive_docs.rs rename to crux-mir/lib/std/src/primitive_docs.rs diff --git a/crux-mir/lib/libstd/process.rs b/crux-mir/lib/std/src/process.rs similarity index 100% rename from crux-mir/lib/libstd/process.rs rename to crux-mir/lib/std/src/process.rs diff --git a/crux-mir/lib/libstd/rt.rs b/crux-mir/lib/std/src/rt.rs similarity index 100% rename from crux-mir/lib/libstd/rt.rs rename to crux-mir/lib/std/src/rt.rs diff --git a/crux-mir/lib/libstd/sync/barrier.rs b/crux-mir/lib/std/src/sync/barrier.rs similarity index 100% rename from crux-mir/lib/libstd/sync/barrier.rs rename to crux-mir/lib/std/src/sync/barrier.rs diff --git a/crux-mir/lib/libstd/sync/condvar.rs b/crux-mir/lib/std/src/sync/condvar.rs similarity index 100% rename from crux-mir/lib/libstd/sync/condvar.rs rename to crux-mir/lib/std/src/sync/condvar.rs diff --git a/crux-mir/lib/libstd/sync/mod.rs b/crux-mir/lib/std/src/sync/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sync/mod.rs rename to crux-mir/lib/std/src/sync/mod.rs diff --git a/crux-mir/lib/libstd/sync/mpsc/blocking.rs b/crux-mir/lib/std/src/sync/mpsc/blocking.rs similarity index 100% rename from crux-mir/lib/libstd/sync/mpsc/blocking.rs rename to crux-mir/lib/std/src/sync/mpsc/blocking.rs diff --git a/crux-mir/lib/libstd/sync/mpsc/cache_aligned.rs b/crux-mir/lib/std/src/sync/mpsc/cache_aligned.rs similarity index 100% rename from crux-mir/lib/libstd/sync/mpsc/cache_aligned.rs rename to crux-mir/lib/std/src/sync/mpsc/cache_aligned.rs diff --git a/crux-mir/lib/libstd/sync/mpsc/mod.rs b/crux-mir/lib/std/src/sync/mpsc/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sync/mpsc/mod.rs rename to crux-mir/lib/std/src/sync/mpsc/mod.rs diff --git a/crux-mir/lib/libstd/sync/mpsc/mpsc_queue.rs b/crux-mir/lib/std/src/sync/mpsc/mpsc_queue.rs similarity index 100% rename from crux-mir/lib/libstd/sync/mpsc/mpsc_queue.rs rename to crux-mir/lib/std/src/sync/mpsc/mpsc_queue.rs diff --git a/crux-mir/lib/libstd/sync/mpsc/oneshot.rs b/crux-mir/lib/std/src/sync/mpsc/oneshot.rs similarity index 100% rename from crux-mir/lib/libstd/sync/mpsc/oneshot.rs rename to crux-mir/lib/std/src/sync/mpsc/oneshot.rs diff --git a/crux-mir/lib/libstd/sync/mpsc/shared.rs b/crux-mir/lib/std/src/sync/mpsc/shared.rs similarity index 100% rename from crux-mir/lib/libstd/sync/mpsc/shared.rs rename to crux-mir/lib/std/src/sync/mpsc/shared.rs diff --git a/crux-mir/lib/libstd/sync/mpsc/spsc_queue.rs b/crux-mir/lib/std/src/sync/mpsc/spsc_queue.rs similarity index 100% rename from crux-mir/lib/libstd/sync/mpsc/spsc_queue.rs rename to crux-mir/lib/std/src/sync/mpsc/spsc_queue.rs diff --git a/crux-mir/lib/libstd/sync/mpsc/stream.rs b/crux-mir/lib/std/src/sync/mpsc/stream.rs similarity index 100% rename from crux-mir/lib/libstd/sync/mpsc/stream.rs rename to crux-mir/lib/std/src/sync/mpsc/stream.rs diff --git a/crux-mir/lib/libstd/sync/mpsc/sync.rs b/crux-mir/lib/std/src/sync/mpsc/sync.rs similarity index 100% rename from crux-mir/lib/libstd/sync/mpsc/sync.rs rename to crux-mir/lib/std/src/sync/mpsc/sync.rs diff --git a/crux-mir/lib/libstd/sync/mutex.rs b/crux-mir/lib/std/src/sync/mutex.rs similarity index 100% rename from crux-mir/lib/libstd/sync/mutex.rs rename to crux-mir/lib/std/src/sync/mutex.rs diff --git a/crux-mir/lib/libstd/sync/once.rs b/crux-mir/lib/std/src/sync/once.rs similarity index 100% rename from crux-mir/lib/libstd/sync/once.rs rename to crux-mir/lib/std/src/sync/once.rs diff --git a/crux-mir/lib/libstd/sync/rwlock.rs b/crux-mir/lib/std/src/sync/rwlock.rs similarity index 100% rename from crux-mir/lib/libstd/sync/rwlock.rs rename to crux-mir/lib/std/src/sync/rwlock.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/abi/bitflags.rs b/crux-mir/lib/std/src/sys/cloudabi/abi/bitflags.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/abi/bitflags.rs rename to crux-mir/lib/std/src/sys/cloudabi/abi/bitflags.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/abi/cloudabi.rs b/crux-mir/lib/std/src/sys/cloudabi/abi/cloudabi.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/abi/cloudabi.rs rename to crux-mir/lib/std/src/sys/cloudabi/abi/cloudabi.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/abi/mod.rs b/crux-mir/lib/std/src/sys/cloudabi/abi/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/abi/mod.rs rename to crux-mir/lib/std/src/sys/cloudabi/abi/mod.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/args.rs b/crux-mir/lib/std/src/sys/cloudabi/args.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/args.rs rename to crux-mir/lib/std/src/sys/cloudabi/args.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/condvar.rs b/crux-mir/lib/std/src/sys/cloudabi/condvar.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/condvar.rs rename to crux-mir/lib/std/src/sys/cloudabi/condvar.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/io.rs b/crux-mir/lib/std/src/sys/cloudabi/io.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/io.rs rename to crux-mir/lib/std/src/sys/cloudabi/io.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/mod.rs b/crux-mir/lib/std/src/sys/cloudabi/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/mod.rs rename to crux-mir/lib/std/src/sys/cloudabi/mod.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/mutex.rs b/crux-mir/lib/std/src/sys/cloudabi/mutex.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/mutex.rs rename to crux-mir/lib/std/src/sys/cloudabi/mutex.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/os.rs b/crux-mir/lib/std/src/sys/cloudabi/os.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/os.rs rename to crux-mir/lib/std/src/sys/cloudabi/os.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/rwlock.rs b/crux-mir/lib/std/src/sys/cloudabi/rwlock.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/rwlock.rs rename to crux-mir/lib/std/src/sys/cloudabi/rwlock.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/shims/args.rs b/crux-mir/lib/std/src/sys/cloudabi/shims/args.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/shims/args.rs rename to crux-mir/lib/std/src/sys/cloudabi/shims/args.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/shims/env.rs b/crux-mir/lib/std/src/sys/cloudabi/shims/env.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/shims/env.rs rename to crux-mir/lib/std/src/sys/cloudabi/shims/env.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/shims/fs.rs b/crux-mir/lib/std/src/sys/cloudabi/shims/fs.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/shims/fs.rs rename to crux-mir/lib/std/src/sys/cloudabi/shims/fs.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/shims/mod.rs b/crux-mir/lib/std/src/sys/cloudabi/shims/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/shims/mod.rs rename to crux-mir/lib/std/src/sys/cloudabi/shims/mod.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/shims/net.rs b/crux-mir/lib/std/src/sys/cloudabi/shims/net.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/shims/net.rs rename to crux-mir/lib/std/src/sys/cloudabi/shims/net.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/shims/os.rs b/crux-mir/lib/std/src/sys/cloudabi/shims/os.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/shims/os.rs rename to crux-mir/lib/std/src/sys/cloudabi/shims/os.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/shims/pipe.rs b/crux-mir/lib/std/src/sys/cloudabi/shims/pipe.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/shims/pipe.rs rename to crux-mir/lib/std/src/sys/cloudabi/shims/pipe.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/shims/process.rs b/crux-mir/lib/std/src/sys/cloudabi/shims/process.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/shims/process.rs rename to crux-mir/lib/std/src/sys/cloudabi/shims/process.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/stack_overflow.rs b/crux-mir/lib/std/src/sys/cloudabi/stack_overflow.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/stack_overflow.rs rename to crux-mir/lib/std/src/sys/cloudabi/stack_overflow.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/stdio.rs b/crux-mir/lib/std/src/sys/cloudabi/stdio.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/stdio.rs rename to crux-mir/lib/std/src/sys/cloudabi/stdio.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/thread.rs b/crux-mir/lib/std/src/sys/cloudabi/thread.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/thread.rs rename to crux-mir/lib/std/src/sys/cloudabi/thread.rs diff --git a/crux-mir/lib/libstd/sys/cloudabi/time.rs b/crux-mir/lib/std/src/sys/cloudabi/time.rs similarity index 100% rename from crux-mir/lib/libstd/sys/cloudabi/time.rs rename to crux-mir/lib/std/src/sys/cloudabi/time.rs diff --git a/crux-mir/lib/libstd/sys/crux/condvar.rs b/crux-mir/lib/std/src/sys/crux/condvar.rs similarity index 100% rename from crux-mir/lib/libstd/sys/crux/condvar.rs rename to crux-mir/lib/std/src/sys/crux/condvar.rs diff --git a/crux-mir/lib/libstd/sys/crux/mod.rs b/crux-mir/lib/std/src/sys/crux/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys/crux/mod.rs rename to crux-mir/lib/std/src/sys/crux/mod.rs diff --git a/crux-mir/lib/libstd/sys/crux/mutex.rs b/crux-mir/lib/std/src/sys/crux/mutex.rs similarity index 100% rename from crux-mir/lib/libstd/sys/crux/mutex.rs rename to crux-mir/lib/std/src/sys/crux/mutex.rs diff --git a/crux-mir/lib/libstd/sys/crux/rwlock.rs b/crux-mir/lib/std/src/sys/crux/rwlock.rs similarity index 100% rename from crux-mir/lib/libstd/sys/crux/rwlock.rs rename to crux-mir/lib/std/src/sys/crux/rwlock.rs diff --git a/crux-mir/lib/libstd/sys/crux/time.rs b/crux-mir/lib/std/src/sys/crux/time.rs similarity index 100% rename from crux-mir/lib/libstd/sys/crux/time.rs rename to crux-mir/lib/std/src/sys/crux/time.rs diff --git a/crux-mir/lib/libstd/sys/hermit/alloc.rs b/crux-mir/lib/std/src/sys/hermit/alloc.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/alloc.rs rename to crux-mir/lib/std/src/sys/hermit/alloc.rs diff --git a/crux-mir/lib/libstd/sys/hermit/args.rs b/crux-mir/lib/std/src/sys/hermit/args.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/args.rs rename to crux-mir/lib/std/src/sys/hermit/args.rs diff --git a/crux-mir/lib/libstd/sys/hermit/cmath.rs b/crux-mir/lib/std/src/sys/hermit/cmath.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/cmath.rs rename to crux-mir/lib/std/src/sys/hermit/cmath.rs diff --git a/crux-mir/lib/libstd/sys/hermit/condvar.rs b/crux-mir/lib/std/src/sys/hermit/condvar.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/condvar.rs rename to crux-mir/lib/std/src/sys/hermit/condvar.rs diff --git a/crux-mir/lib/libstd/sys/hermit/env.rs b/crux-mir/lib/std/src/sys/hermit/env.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/env.rs rename to crux-mir/lib/std/src/sys/hermit/env.rs diff --git a/crux-mir/lib/libstd/sys/hermit/fast_thread_local.rs b/crux-mir/lib/std/src/sys/hermit/fast_thread_local.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/fast_thread_local.rs rename to crux-mir/lib/std/src/sys/hermit/fast_thread_local.rs diff --git a/crux-mir/lib/libstd/sys/hermit/fd.rs b/crux-mir/lib/std/src/sys/hermit/fd.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/fd.rs rename to crux-mir/lib/std/src/sys/hermit/fd.rs diff --git a/crux-mir/lib/libstd/sys/hermit/fs.rs b/crux-mir/lib/std/src/sys/hermit/fs.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/fs.rs rename to crux-mir/lib/std/src/sys/hermit/fs.rs diff --git a/crux-mir/lib/libstd/sys/hermit/io.rs b/crux-mir/lib/std/src/sys/hermit/io.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/io.rs rename to crux-mir/lib/std/src/sys/hermit/io.rs diff --git a/crux-mir/lib/libstd/sys/hermit/memchr.rs b/crux-mir/lib/std/src/sys/hermit/memchr.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/memchr.rs rename to crux-mir/lib/std/src/sys/hermit/memchr.rs diff --git a/crux-mir/lib/libstd/sys/hermit/mod.rs b/crux-mir/lib/std/src/sys/hermit/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/mod.rs rename to crux-mir/lib/std/src/sys/hermit/mod.rs diff --git a/crux-mir/lib/libstd/sys/hermit/mutex.rs b/crux-mir/lib/std/src/sys/hermit/mutex.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/mutex.rs rename to crux-mir/lib/std/src/sys/hermit/mutex.rs diff --git a/crux-mir/lib/libstd/sys/hermit/net.rs b/crux-mir/lib/std/src/sys/hermit/net.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/net.rs rename to crux-mir/lib/std/src/sys/hermit/net.rs diff --git a/crux-mir/lib/libstd/sys/hermit/os.rs b/crux-mir/lib/std/src/sys/hermit/os.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/os.rs rename to crux-mir/lib/std/src/sys/hermit/os.rs diff --git a/crux-mir/lib/libstd/sys/hermit/path.rs b/crux-mir/lib/std/src/sys/hermit/path.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/path.rs rename to crux-mir/lib/std/src/sys/hermit/path.rs diff --git a/crux-mir/lib/libstd/sys/hermit/pipe.rs b/crux-mir/lib/std/src/sys/hermit/pipe.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/pipe.rs rename to crux-mir/lib/std/src/sys/hermit/pipe.rs diff --git a/crux-mir/lib/libstd/sys/hermit/process.rs b/crux-mir/lib/std/src/sys/hermit/process.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/process.rs rename to crux-mir/lib/std/src/sys/hermit/process.rs diff --git a/crux-mir/lib/libstd/sys/hermit/rwlock.rs b/crux-mir/lib/std/src/sys/hermit/rwlock.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/rwlock.rs rename to crux-mir/lib/std/src/sys/hermit/rwlock.rs diff --git a/crux-mir/lib/libstd/sys/hermit/stack_overflow.rs b/crux-mir/lib/std/src/sys/hermit/stack_overflow.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/stack_overflow.rs rename to crux-mir/lib/std/src/sys/hermit/stack_overflow.rs diff --git a/crux-mir/lib/libstd/sys/hermit/stdio.rs b/crux-mir/lib/std/src/sys/hermit/stdio.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/stdio.rs rename to crux-mir/lib/std/src/sys/hermit/stdio.rs diff --git a/crux-mir/lib/libstd/sys/hermit/thread.rs b/crux-mir/lib/std/src/sys/hermit/thread.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/thread.rs rename to crux-mir/lib/std/src/sys/hermit/thread.rs diff --git a/crux-mir/lib/libstd/sys/hermit/thread_local.rs b/crux-mir/lib/std/src/sys/hermit/thread_local.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/thread_local.rs rename to crux-mir/lib/std/src/sys/hermit/thread_local.rs diff --git a/crux-mir/lib/libstd/sys/hermit/time.rs b/crux-mir/lib/std/src/sys/hermit/time.rs similarity index 100% rename from crux-mir/lib/libstd/sys/hermit/time.rs rename to crux-mir/lib/std/src/sys/hermit/time.rs diff --git a/crux-mir/lib/libstd/sys/mod.rs b/crux-mir/lib/std/src/sys/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys/mod.rs rename to crux-mir/lib/std/src/sys/mod.rs diff --git a/crux-mir/lib/libstd/sys/sgx/abi/entry.S b/crux-mir/lib/std/src/sys/sgx/abi/entry.S similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/abi/entry.S rename to crux-mir/lib/std/src/sys/sgx/abi/entry.S diff --git a/crux-mir/lib/libstd/sys/sgx/abi/mem.rs b/crux-mir/lib/std/src/sys/sgx/abi/mem.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/abi/mem.rs rename to crux-mir/lib/std/src/sys/sgx/abi/mem.rs diff --git a/crux-mir/lib/libstd/sys/sgx/abi/mod.rs b/crux-mir/lib/std/src/sys/sgx/abi/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/abi/mod.rs rename to crux-mir/lib/std/src/sys/sgx/abi/mod.rs diff --git a/crux-mir/lib/libstd/sys/sgx/abi/panic.rs b/crux-mir/lib/std/src/sys/sgx/abi/panic.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/abi/panic.rs rename to crux-mir/lib/std/src/sys/sgx/abi/panic.rs diff --git a/crux-mir/lib/libstd/sys/sgx/abi/reloc.rs b/crux-mir/lib/std/src/sys/sgx/abi/reloc.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/abi/reloc.rs rename to crux-mir/lib/std/src/sys/sgx/abi/reloc.rs diff --git a/crux-mir/lib/libstd/sys/sgx/abi/thread.rs b/crux-mir/lib/std/src/sys/sgx/abi/thread.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/abi/thread.rs rename to crux-mir/lib/std/src/sys/sgx/abi/thread.rs diff --git a/crux-mir/lib/libstd/sys/sgx/abi/tls.rs b/crux-mir/lib/std/src/sys/sgx/abi/tls.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/abi/tls.rs rename to crux-mir/lib/std/src/sys/sgx/abi/tls.rs diff --git a/crux-mir/lib/libstd/sys/sgx/abi/usercalls/alloc.rs b/crux-mir/lib/std/src/sys/sgx/abi/usercalls/alloc.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/abi/usercalls/alloc.rs rename to crux-mir/lib/std/src/sys/sgx/abi/usercalls/alloc.rs diff --git a/crux-mir/lib/libstd/sys/sgx/abi/usercalls/mod.rs b/crux-mir/lib/std/src/sys/sgx/abi/usercalls/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/abi/usercalls/mod.rs rename to crux-mir/lib/std/src/sys/sgx/abi/usercalls/mod.rs diff --git a/crux-mir/lib/libstd/sys/sgx/abi/usercalls/raw.rs b/crux-mir/lib/std/src/sys/sgx/abi/usercalls/raw.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/abi/usercalls/raw.rs rename to crux-mir/lib/std/src/sys/sgx/abi/usercalls/raw.rs diff --git a/crux-mir/lib/libstd/sys/sgx/alloc.rs b/crux-mir/lib/std/src/sys/sgx/alloc.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/alloc.rs rename to crux-mir/lib/std/src/sys/sgx/alloc.rs diff --git a/crux-mir/lib/libstd/sys/sgx/args.rs b/crux-mir/lib/std/src/sys/sgx/args.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/args.rs rename to crux-mir/lib/std/src/sys/sgx/args.rs diff --git a/crux-mir/lib/libstd/sys/sgx/cmath.rs b/crux-mir/lib/std/src/sys/sgx/cmath.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/cmath.rs rename to crux-mir/lib/std/src/sys/sgx/cmath.rs diff --git a/crux-mir/lib/libstd/sys/sgx/condvar.rs b/crux-mir/lib/std/src/sys/sgx/condvar.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/condvar.rs rename to crux-mir/lib/std/src/sys/sgx/condvar.rs diff --git a/crux-mir/lib/libstd/sys/sgx/env.rs b/crux-mir/lib/std/src/sys/sgx/env.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/env.rs rename to crux-mir/lib/std/src/sys/sgx/env.rs diff --git a/crux-mir/lib/libstd/sys/sgx/ext/arch.rs b/crux-mir/lib/std/src/sys/sgx/ext/arch.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/ext/arch.rs rename to crux-mir/lib/std/src/sys/sgx/ext/arch.rs diff --git a/crux-mir/lib/libstd/sys/sgx/ext/ffi.rs b/crux-mir/lib/std/src/sys/sgx/ext/ffi.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/ext/ffi.rs rename to crux-mir/lib/std/src/sys/sgx/ext/ffi.rs diff --git a/crux-mir/lib/libstd/sys/sgx/ext/io.rs b/crux-mir/lib/std/src/sys/sgx/ext/io.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/ext/io.rs rename to crux-mir/lib/std/src/sys/sgx/ext/io.rs diff --git a/crux-mir/lib/libstd/sys/sgx/ext/mod.rs b/crux-mir/lib/std/src/sys/sgx/ext/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/ext/mod.rs rename to crux-mir/lib/std/src/sys/sgx/ext/mod.rs diff --git a/crux-mir/lib/libstd/sys/sgx/fd.rs b/crux-mir/lib/std/src/sys/sgx/fd.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/fd.rs rename to crux-mir/lib/std/src/sys/sgx/fd.rs diff --git a/crux-mir/lib/libstd/sys/sgx/fs.rs b/crux-mir/lib/std/src/sys/sgx/fs.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/fs.rs rename to crux-mir/lib/std/src/sys/sgx/fs.rs diff --git a/crux-mir/lib/libstd/sys/sgx/io.rs b/crux-mir/lib/std/src/sys/sgx/io.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/io.rs rename to crux-mir/lib/std/src/sys/sgx/io.rs diff --git a/crux-mir/lib/libstd/sys/sgx/memchr.rs b/crux-mir/lib/std/src/sys/sgx/memchr.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/memchr.rs rename to crux-mir/lib/std/src/sys/sgx/memchr.rs diff --git a/crux-mir/lib/libstd/sys/sgx/mod.rs b/crux-mir/lib/std/src/sys/sgx/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/mod.rs rename to crux-mir/lib/std/src/sys/sgx/mod.rs diff --git a/crux-mir/lib/libstd/sys/sgx/mutex.rs b/crux-mir/lib/std/src/sys/sgx/mutex.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/mutex.rs rename to crux-mir/lib/std/src/sys/sgx/mutex.rs diff --git a/crux-mir/lib/libstd/sys/sgx/net.rs b/crux-mir/lib/std/src/sys/sgx/net.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/net.rs rename to crux-mir/lib/std/src/sys/sgx/net.rs diff --git a/crux-mir/lib/libstd/sys/sgx/os.rs b/crux-mir/lib/std/src/sys/sgx/os.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/os.rs rename to crux-mir/lib/std/src/sys/sgx/os.rs diff --git a/crux-mir/lib/libstd/sys/sgx/path.rs b/crux-mir/lib/std/src/sys/sgx/path.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/path.rs rename to crux-mir/lib/std/src/sys/sgx/path.rs diff --git a/crux-mir/lib/libstd/sys/sgx/pipe.rs b/crux-mir/lib/std/src/sys/sgx/pipe.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/pipe.rs rename to crux-mir/lib/std/src/sys/sgx/pipe.rs diff --git a/crux-mir/lib/libstd/sys/sgx/process.rs b/crux-mir/lib/std/src/sys/sgx/process.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/process.rs rename to crux-mir/lib/std/src/sys/sgx/process.rs diff --git a/crux-mir/lib/libstd/sys/sgx/rwlock.rs b/crux-mir/lib/std/src/sys/sgx/rwlock.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/rwlock.rs rename to crux-mir/lib/std/src/sys/sgx/rwlock.rs diff --git a/crux-mir/lib/libstd/sys/sgx/stack_overflow.rs b/crux-mir/lib/std/src/sys/sgx/stack_overflow.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/stack_overflow.rs rename to crux-mir/lib/std/src/sys/sgx/stack_overflow.rs diff --git a/crux-mir/lib/libstd/sys/sgx/stdio.rs b/crux-mir/lib/std/src/sys/sgx/stdio.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/stdio.rs rename to crux-mir/lib/std/src/sys/sgx/stdio.rs diff --git a/crux-mir/lib/libstd/sys/sgx/thread.rs b/crux-mir/lib/std/src/sys/sgx/thread.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/thread.rs rename to crux-mir/lib/std/src/sys/sgx/thread.rs diff --git a/crux-mir/lib/libstd/sys/sgx/thread_local.rs b/crux-mir/lib/std/src/sys/sgx/thread_local.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/thread_local.rs rename to crux-mir/lib/std/src/sys/sgx/thread_local.rs diff --git a/crux-mir/lib/libstd/sys/sgx/time.rs b/crux-mir/lib/std/src/sys/sgx/time.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/time.rs rename to crux-mir/lib/std/src/sys/sgx/time.rs diff --git a/crux-mir/lib/libstd/sys/sgx/waitqueue.rs b/crux-mir/lib/std/src/sys/sgx/waitqueue.rs similarity index 100% rename from crux-mir/lib/libstd/sys/sgx/waitqueue.rs rename to crux-mir/lib/std/src/sys/sgx/waitqueue.rs diff --git a/crux-mir/lib/libstd/sys/unix/alloc.rs b/crux-mir/lib/std/src/sys/unix/alloc.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/alloc.rs rename to crux-mir/lib/std/src/sys/unix/alloc.rs diff --git a/crux-mir/lib/libstd/sys/unix/android.rs b/crux-mir/lib/std/src/sys/unix/android.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/android.rs rename to crux-mir/lib/std/src/sys/unix/android.rs diff --git a/crux-mir/lib/libstd/sys/unix/args.rs b/crux-mir/lib/std/src/sys/unix/args.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/args.rs rename to crux-mir/lib/std/src/sys/unix/args.rs diff --git a/crux-mir/lib/libstd/sys/unix/cmath.rs b/crux-mir/lib/std/src/sys/unix/cmath.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/cmath.rs rename to crux-mir/lib/std/src/sys/unix/cmath.rs diff --git a/crux-mir/lib/libstd/sys/unix/condvar.rs b/crux-mir/lib/std/src/sys/unix/condvar.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/condvar.rs rename to crux-mir/lib/std/src/sys/unix/condvar.rs diff --git a/crux-mir/lib/libstd/sys/unix/env.rs b/crux-mir/lib/std/src/sys/unix/env.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/env.rs rename to crux-mir/lib/std/src/sys/unix/env.rs diff --git a/crux-mir/lib/libstd/sys/unix/ext/ffi.rs b/crux-mir/lib/std/src/sys/unix/ext/ffi.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/ext/ffi.rs rename to crux-mir/lib/std/src/sys/unix/ext/ffi.rs diff --git a/crux-mir/lib/libstd/sys/unix/ext/fs.rs b/crux-mir/lib/std/src/sys/unix/ext/fs.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/ext/fs.rs rename to crux-mir/lib/std/src/sys/unix/ext/fs.rs diff --git a/crux-mir/lib/libstd/sys/unix/ext/io.rs b/crux-mir/lib/std/src/sys/unix/ext/io.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/ext/io.rs rename to crux-mir/lib/std/src/sys/unix/ext/io.rs diff --git a/crux-mir/lib/libstd/sys/unix/ext/mod.rs b/crux-mir/lib/std/src/sys/unix/ext/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/ext/mod.rs rename to crux-mir/lib/std/src/sys/unix/ext/mod.rs diff --git a/crux-mir/lib/libstd/sys/unix/ext/net.rs b/crux-mir/lib/std/src/sys/unix/ext/net.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/ext/net.rs rename to crux-mir/lib/std/src/sys/unix/ext/net.rs diff --git a/crux-mir/lib/libstd/sys/unix/ext/process.rs b/crux-mir/lib/std/src/sys/unix/ext/process.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/ext/process.rs rename to crux-mir/lib/std/src/sys/unix/ext/process.rs diff --git a/crux-mir/lib/libstd/sys/unix/ext/raw.rs b/crux-mir/lib/std/src/sys/unix/ext/raw.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/ext/raw.rs rename to crux-mir/lib/std/src/sys/unix/ext/raw.rs diff --git a/crux-mir/lib/libstd/sys/unix/ext/thread.rs b/crux-mir/lib/std/src/sys/unix/ext/thread.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/ext/thread.rs rename to crux-mir/lib/std/src/sys/unix/ext/thread.rs diff --git a/crux-mir/lib/libstd/sys/unix/fast_thread_local.rs b/crux-mir/lib/std/src/sys/unix/fast_thread_local.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/fast_thread_local.rs rename to crux-mir/lib/std/src/sys/unix/fast_thread_local.rs diff --git a/crux-mir/lib/libstd/sys/unix/fd.rs b/crux-mir/lib/std/src/sys/unix/fd.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/fd.rs rename to crux-mir/lib/std/src/sys/unix/fd.rs diff --git a/crux-mir/lib/libstd/sys/unix/fs.rs b/crux-mir/lib/std/src/sys/unix/fs.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/fs.rs rename to crux-mir/lib/std/src/sys/unix/fs.rs diff --git a/crux-mir/lib/libstd/sys/unix/io.rs b/crux-mir/lib/std/src/sys/unix/io.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/io.rs rename to crux-mir/lib/std/src/sys/unix/io.rs diff --git a/crux-mir/lib/libstd/sys/unix/l4re.rs b/crux-mir/lib/std/src/sys/unix/l4re.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/l4re.rs rename to crux-mir/lib/std/src/sys/unix/l4re.rs diff --git a/crux-mir/lib/libstd/sys/unix/memchr.rs b/crux-mir/lib/std/src/sys/unix/memchr.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/memchr.rs rename to crux-mir/lib/std/src/sys/unix/memchr.rs diff --git a/crux-mir/lib/libstd/sys/unix/mod.rs b/crux-mir/lib/std/src/sys/unix/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/mod.rs rename to crux-mir/lib/std/src/sys/unix/mod.rs diff --git a/crux-mir/lib/libstd/sys/unix/mutex.rs b/crux-mir/lib/std/src/sys/unix/mutex.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/mutex.rs rename to crux-mir/lib/std/src/sys/unix/mutex.rs diff --git a/crux-mir/lib/libstd/sys/unix/net.rs b/crux-mir/lib/std/src/sys/unix/net.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/net.rs rename to crux-mir/lib/std/src/sys/unix/net.rs diff --git a/crux-mir/lib/libstd/sys/unix/os.rs b/crux-mir/lib/std/src/sys/unix/os.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/os.rs rename to crux-mir/lib/std/src/sys/unix/os.rs diff --git a/crux-mir/lib/libstd/sys/unix/path.rs b/crux-mir/lib/std/src/sys/unix/path.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/path.rs rename to crux-mir/lib/std/src/sys/unix/path.rs diff --git a/crux-mir/lib/libstd/sys/unix/pipe.rs b/crux-mir/lib/std/src/sys/unix/pipe.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/pipe.rs rename to crux-mir/lib/std/src/sys/unix/pipe.rs diff --git a/crux-mir/lib/libstd/sys/unix/process/mod.rs b/crux-mir/lib/std/src/sys/unix/process/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/process/mod.rs rename to crux-mir/lib/std/src/sys/unix/process/mod.rs diff --git a/crux-mir/lib/libstd/sys/unix/process/process_common.rs b/crux-mir/lib/std/src/sys/unix/process/process_common.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/process/process_common.rs rename to crux-mir/lib/std/src/sys/unix/process/process_common.rs diff --git a/crux-mir/lib/libstd/sys/unix/process/process_fuchsia.rs b/crux-mir/lib/std/src/sys/unix/process/process_fuchsia.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/process/process_fuchsia.rs rename to crux-mir/lib/std/src/sys/unix/process/process_fuchsia.rs diff --git a/crux-mir/lib/libstd/sys/unix/process/process_unix.rs b/crux-mir/lib/std/src/sys/unix/process/process_unix.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/process/process_unix.rs rename to crux-mir/lib/std/src/sys/unix/process/process_unix.rs diff --git a/crux-mir/lib/libstd/sys/unix/process/zircon.rs b/crux-mir/lib/std/src/sys/unix/process/zircon.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/process/zircon.rs rename to crux-mir/lib/std/src/sys/unix/process/zircon.rs diff --git a/crux-mir/lib/libstd/sys/unix/rand.rs b/crux-mir/lib/std/src/sys/unix/rand.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/rand.rs rename to crux-mir/lib/std/src/sys/unix/rand.rs diff --git a/crux-mir/lib/libstd/sys/unix/rwlock.rs b/crux-mir/lib/std/src/sys/unix/rwlock.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/rwlock.rs rename to crux-mir/lib/std/src/sys/unix/rwlock.rs diff --git a/crux-mir/lib/libstd/sys/unix/stack_overflow.rs b/crux-mir/lib/std/src/sys/unix/stack_overflow.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/stack_overflow.rs rename to crux-mir/lib/std/src/sys/unix/stack_overflow.rs diff --git a/crux-mir/lib/libstd/sys/unix/stdio.rs b/crux-mir/lib/std/src/sys/unix/stdio.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/stdio.rs rename to crux-mir/lib/std/src/sys/unix/stdio.rs diff --git a/crux-mir/lib/libstd/sys/unix/thread.rs b/crux-mir/lib/std/src/sys/unix/thread.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/thread.rs rename to crux-mir/lib/std/src/sys/unix/thread.rs diff --git a/crux-mir/lib/libstd/sys/unix/thread_local.rs b/crux-mir/lib/std/src/sys/unix/thread_local.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/thread_local.rs rename to crux-mir/lib/std/src/sys/unix/thread_local.rs diff --git a/crux-mir/lib/libstd/sys/unix/time.rs b/crux-mir/lib/std/src/sys/unix/time.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/time.rs rename to crux-mir/lib/std/src/sys/unix/time.rs diff --git a/crux-mir/lib/libstd/sys/unix/weak.rs b/crux-mir/lib/std/src/sys/unix/weak.rs similarity index 100% rename from crux-mir/lib/libstd/sys/unix/weak.rs rename to crux-mir/lib/std/src/sys/unix/weak.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/alloc.rs b/crux-mir/lib/std/src/sys/vxworks/alloc.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/alloc.rs rename to crux-mir/lib/std/src/sys/vxworks/alloc.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/args.rs b/crux-mir/lib/std/src/sys/vxworks/args.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/args.rs rename to crux-mir/lib/std/src/sys/vxworks/args.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/cmath.rs b/crux-mir/lib/std/src/sys/vxworks/cmath.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/cmath.rs rename to crux-mir/lib/std/src/sys/vxworks/cmath.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/condvar.rs b/crux-mir/lib/std/src/sys/vxworks/condvar.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/condvar.rs rename to crux-mir/lib/std/src/sys/vxworks/condvar.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/env.rs b/crux-mir/lib/std/src/sys/vxworks/env.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/env.rs rename to crux-mir/lib/std/src/sys/vxworks/env.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/ext/ffi.rs b/crux-mir/lib/std/src/sys/vxworks/ext/ffi.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/ext/ffi.rs rename to crux-mir/lib/std/src/sys/vxworks/ext/ffi.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/ext/fs.rs b/crux-mir/lib/std/src/sys/vxworks/ext/fs.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/ext/fs.rs rename to crux-mir/lib/std/src/sys/vxworks/ext/fs.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/ext/io.rs b/crux-mir/lib/std/src/sys/vxworks/ext/io.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/ext/io.rs rename to crux-mir/lib/std/src/sys/vxworks/ext/io.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/ext/mod.rs b/crux-mir/lib/std/src/sys/vxworks/ext/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/ext/mod.rs rename to crux-mir/lib/std/src/sys/vxworks/ext/mod.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/ext/process.rs b/crux-mir/lib/std/src/sys/vxworks/ext/process.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/ext/process.rs rename to crux-mir/lib/std/src/sys/vxworks/ext/process.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/ext/raw.rs b/crux-mir/lib/std/src/sys/vxworks/ext/raw.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/ext/raw.rs rename to crux-mir/lib/std/src/sys/vxworks/ext/raw.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/fast_thread_local.rs b/crux-mir/lib/std/src/sys/vxworks/fast_thread_local.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/fast_thread_local.rs rename to crux-mir/lib/std/src/sys/vxworks/fast_thread_local.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/fd.rs b/crux-mir/lib/std/src/sys/vxworks/fd.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/fd.rs rename to crux-mir/lib/std/src/sys/vxworks/fd.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/fs.rs b/crux-mir/lib/std/src/sys/vxworks/fs.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/fs.rs rename to crux-mir/lib/std/src/sys/vxworks/fs.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/io.rs b/crux-mir/lib/std/src/sys/vxworks/io.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/io.rs rename to crux-mir/lib/std/src/sys/vxworks/io.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/memchr.rs b/crux-mir/lib/std/src/sys/vxworks/memchr.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/memchr.rs rename to crux-mir/lib/std/src/sys/vxworks/memchr.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/mod.rs b/crux-mir/lib/std/src/sys/vxworks/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/mod.rs rename to crux-mir/lib/std/src/sys/vxworks/mod.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/mutex.rs b/crux-mir/lib/std/src/sys/vxworks/mutex.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/mutex.rs rename to crux-mir/lib/std/src/sys/vxworks/mutex.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/net.rs b/crux-mir/lib/std/src/sys/vxworks/net.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/net.rs rename to crux-mir/lib/std/src/sys/vxworks/net.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/os.rs b/crux-mir/lib/std/src/sys/vxworks/os.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/os.rs rename to crux-mir/lib/std/src/sys/vxworks/os.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/path.rs b/crux-mir/lib/std/src/sys/vxworks/path.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/path.rs rename to crux-mir/lib/std/src/sys/vxworks/path.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/pipe.rs b/crux-mir/lib/std/src/sys/vxworks/pipe.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/pipe.rs rename to crux-mir/lib/std/src/sys/vxworks/pipe.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/process/mod.rs b/crux-mir/lib/std/src/sys/vxworks/process/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/process/mod.rs rename to crux-mir/lib/std/src/sys/vxworks/process/mod.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/process/process_common.rs b/crux-mir/lib/std/src/sys/vxworks/process/process_common.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/process/process_common.rs rename to crux-mir/lib/std/src/sys/vxworks/process/process_common.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/process/process_vxworks.rs b/crux-mir/lib/std/src/sys/vxworks/process/process_vxworks.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/process/process_vxworks.rs rename to crux-mir/lib/std/src/sys/vxworks/process/process_vxworks.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/rand.rs b/crux-mir/lib/std/src/sys/vxworks/rand.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/rand.rs rename to crux-mir/lib/std/src/sys/vxworks/rand.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/rwlock.rs b/crux-mir/lib/std/src/sys/vxworks/rwlock.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/rwlock.rs rename to crux-mir/lib/std/src/sys/vxworks/rwlock.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/stack_overflow.rs b/crux-mir/lib/std/src/sys/vxworks/stack_overflow.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/stack_overflow.rs rename to crux-mir/lib/std/src/sys/vxworks/stack_overflow.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/stdio.rs b/crux-mir/lib/std/src/sys/vxworks/stdio.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/stdio.rs rename to crux-mir/lib/std/src/sys/vxworks/stdio.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/thread.rs b/crux-mir/lib/std/src/sys/vxworks/thread.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/thread.rs rename to crux-mir/lib/std/src/sys/vxworks/thread.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/thread_local.rs b/crux-mir/lib/std/src/sys/vxworks/thread_local.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/thread_local.rs rename to crux-mir/lib/std/src/sys/vxworks/thread_local.rs diff --git a/crux-mir/lib/libstd/sys/vxworks/time.rs b/crux-mir/lib/std/src/sys/vxworks/time.rs similarity index 100% rename from crux-mir/lib/libstd/sys/vxworks/time.rs rename to crux-mir/lib/std/src/sys/vxworks/time.rs diff --git a/crux-mir/lib/libstd/sys/wasi/alloc.rs b/crux-mir/lib/std/src/sys/wasi/alloc.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasi/alloc.rs rename to crux-mir/lib/std/src/sys/wasi/alloc.rs diff --git a/crux-mir/lib/libstd/sys/wasi/args.rs b/crux-mir/lib/std/src/sys/wasi/args.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasi/args.rs rename to crux-mir/lib/std/src/sys/wasi/args.rs diff --git a/crux-mir/lib/libstd/sys/wasi/env.rs b/crux-mir/lib/std/src/sys/wasi/env.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasi/env.rs rename to crux-mir/lib/std/src/sys/wasi/env.rs diff --git a/crux-mir/lib/libstd/sys/wasi/ext/ffi.rs b/crux-mir/lib/std/src/sys/wasi/ext/ffi.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasi/ext/ffi.rs rename to crux-mir/lib/std/src/sys/wasi/ext/ffi.rs diff --git a/crux-mir/lib/libstd/sys/wasi/ext/fs.rs b/crux-mir/lib/std/src/sys/wasi/ext/fs.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasi/ext/fs.rs rename to crux-mir/lib/std/src/sys/wasi/ext/fs.rs diff --git a/crux-mir/lib/libstd/sys/wasi/ext/io.rs b/crux-mir/lib/std/src/sys/wasi/ext/io.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasi/ext/io.rs rename to crux-mir/lib/std/src/sys/wasi/ext/io.rs diff --git a/crux-mir/lib/libstd/sys/wasi/ext/mod.rs b/crux-mir/lib/std/src/sys/wasi/ext/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasi/ext/mod.rs rename to crux-mir/lib/std/src/sys/wasi/ext/mod.rs diff --git a/crux-mir/lib/libstd/sys/wasi/fd.rs b/crux-mir/lib/std/src/sys/wasi/fd.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasi/fd.rs rename to crux-mir/lib/std/src/sys/wasi/fd.rs diff --git a/crux-mir/lib/libstd/sys/wasi/fs.rs b/crux-mir/lib/std/src/sys/wasi/fs.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasi/fs.rs rename to crux-mir/lib/std/src/sys/wasi/fs.rs diff --git a/crux-mir/lib/libstd/sys/wasi/io.rs b/crux-mir/lib/std/src/sys/wasi/io.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasi/io.rs rename to crux-mir/lib/std/src/sys/wasi/io.rs diff --git a/crux-mir/lib/libstd/sys/wasi/mod.rs b/crux-mir/lib/std/src/sys/wasi/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasi/mod.rs rename to crux-mir/lib/std/src/sys/wasi/mod.rs diff --git a/crux-mir/lib/libstd/sys/wasi/net.rs b/crux-mir/lib/std/src/sys/wasi/net.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasi/net.rs rename to crux-mir/lib/std/src/sys/wasi/net.rs diff --git a/crux-mir/lib/libstd/sys/wasi/os.rs b/crux-mir/lib/std/src/sys/wasi/os.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasi/os.rs rename to crux-mir/lib/std/src/sys/wasi/os.rs diff --git a/crux-mir/lib/libstd/sys/wasi/path.rs b/crux-mir/lib/std/src/sys/wasi/path.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasi/path.rs rename to crux-mir/lib/std/src/sys/wasi/path.rs diff --git a/crux-mir/lib/libstd/sys/wasi/pipe.rs b/crux-mir/lib/std/src/sys/wasi/pipe.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasi/pipe.rs rename to crux-mir/lib/std/src/sys/wasi/pipe.rs diff --git a/crux-mir/lib/libstd/sys/wasi/process.rs b/crux-mir/lib/std/src/sys/wasi/process.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasi/process.rs rename to crux-mir/lib/std/src/sys/wasi/process.rs diff --git a/crux-mir/lib/libstd/sys/wasi/stdio.rs b/crux-mir/lib/std/src/sys/wasi/stdio.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasi/stdio.rs rename to crux-mir/lib/std/src/sys/wasi/stdio.rs diff --git a/crux-mir/lib/libstd/sys/wasi/thread.rs b/crux-mir/lib/std/src/sys/wasi/thread.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasi/thread.rs rename to crux-mir/lib/std/src/sys/wasi/thread.rs diff --git a/crux-mir/lib/libstd/sys/wasi/time.rs b/crux-mir/lib/std/src/sys/wasi/time.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasi/time.rs rename to crux-mir/lib/std/src/sys/wasi/time.rs diff --git a/crux-mir/lib/libstd/sys/wasm/alloc.rs b/crux-mir/lib/std/src/sys/wasm/alloc.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/alloc.rs rename to crux-mir/lib/std/src/sys/wasm/alloc.rs diff --git a/crux-mir/lib/libstd/sys/wasm/args.rs b/crux-mir/lib/std/src/sys/wasm/args.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/args.rs rename to crux-mir/lib/std/src/sys/wasm/args.rs diff --git a/crux-mir/lib/libstd/sys/wasm/cmath.rs b/crux-mir/lib/std/src/sys/wasm/cmath.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/cmath.rs rename to crux-mir/lib/std/src/sys/wasm/cmath.rs diff --git a/crux-mir/lib/libstd/sys/wasm/condvar.rs b/crux-mir/lib/std/src/sys/wasm/condvar.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/condvar.rs rename to crux-mir/lib/std/src/sys/wasm/condvar.rs diff --git a/crux-mir/lib/libstd/sys/wasm/condvar_atomics.rs b/crux-mir/lib/std/src/sys/wasm/condvar_atomics.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/condvar_atomics.rs rename to crux-mir/lib/std/src/sys/wasm/condvar_atomics.rs diff --git a/crux-mir/lib/libstd/sys/wasm/env.rs b/crux-mir/lib/std/src/sys/wasm/env.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/env.rs rename to crux-mir/lib/std/src/sys/wasm/env.rs diff --git a/crux-mir/lib/libstd/sys/wasm/fast_thread_local.rs b/crux-mir/lib/std/src/sys/wasm/fast_thread_local.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/fast_thread_local.rs rename to crux-mir/lib/std/src/sys/wasm/fast_thread_local.rs diff --git a/crux-mir/lib/libstd/sys/wasm/fs.rs b/crux-mir/lib/std/src/sys/wasm/fs.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/fs.rs rename to crux-mir/lib/std/src/sys/wasm/fs.rs diff --git a/crux-mir/lib/libstd/sys/wasm/io.rs b/crux-mir/lib/std/src/sys/wasm/io.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/io.rs rename to crux-mir/lib/std/src/sys/wasm/io.rs diff --git a/crux-mir/lib/libstd/sys/wasm/memchr.rs b/crux-mir/lib/std/src/sys/wasm/memchr.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/memchr.rs rename to crux-mir/lib/std/src/sys/wasm/memchr.rs diff --git a/crux-mir/lib/libstd/sys/wasm/mod.rs b/crux-mir/lib/std/src/sys/wasm/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/mod.rs rename to crux-mir/lib/std/src/sys/wasm/mod.rs diff --git a/crux-mir/lib/libstd/sys/wasm/mutex.rs b/crux-mir/lib/std/src/sys/wasm/mutex.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/mutex.rs rename to crux-mir/lib/std/src/sys/wasm/mutex.rs diff --git a/crux-mir/lib/libstd/sys/wasm/mutex_atomics.rs b/crux-mir/lib/std/src/sys/wasm/mutex_atomics.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/mutex_atomics.rs rename to crux-mir/lib/std/src/sys/wasm/mutex_atomics.rs diff --git a/crux-mir/lib/libstd/sys/wasm/net.rs b/crux-mir/lib/std/src/sys/wasm/net.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/net.rs rename to crux-mir/lib/std/src/sys/wasm/net.rs diff --git a/crux-mir/lib/libstd/sys/wasm/os.rs b/crux-mir/lib/std/src/sys/wasm/os.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/os.rs rename to crux-mir/lib/std/src/sys/wasm/os.rs diff --git a/crux-mir/lib/libstd/sys/wasm/path.rs b/crux-mir/lib/std/src/sys/wasm/path.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/path.rs rename to crux-mir/lib/std/src/sys/wasm/path.rs diff --git a/crux-mir/lib/libstd/sys/wasm/pipe.rs b/crux-mir/lib/std/src/sys/wasm/pipe.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/pipe.rs rename to crux-mir/lib/std/src/sys/wasm/pipe.rs diff --git a/crux-mir/lib/libstd/sys/wasm/process.rs b/crux-mir/lib/std/src/sys/wasm/process.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/process.rs rename to crux-mir/lib/std/src/sys/wasm/process.rs diff --git a/crux-mir/lib/libstd/sys/wasm/rwlock.rs b/crux-mir/lib/std/src/sys/wasm/rwlock.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/rwlock.rs rename to crux-mir/lib/std/src/sys/wasm/rwlock.rs diff --git a/crux-mir/lib/libstd/sys/wasm/rwlock_atomics.rs b/crux-mir/lib/std/src/sys/wasm/rwlock_atomics.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/rwlock_atomics.rs rename to crux-mir/lib/std/src/sys/wasm/rwlock_atomics.rs diff --git a/crux-mir/lib/libstd/sys/wasm/stack_overflow.rs b/crux-mir/lib/std/src/sys/wasm/stack_overflow.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/stack_overflow.rs rename to crux-mir/lib/std/src/sys/wasm/stack_overflow.rs diff --git a/crux-mir/lib/libstd/sys/wasm/stdio.rs b/crux-mir/lib/std/src/sys/wasm/stdio.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/stdio.rs rename to crux-mir/lib/std/src/sys/wasm/stdio.rs diff --git a/crux-mir/lib/libstd/sys/wasm/thread.rs b/crux-mir/lib/std/src/sys/wasm/thread.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/thread.rs rename to crux-mir/lib/std/src/sys/wasm/thread.rs diff --git a/crux-mir/lib/libstd/sys/wasm/thread_local.rs b/crux-mir/lib/std/src/sys/wasm/thread_local.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/thread_local.rs rename to crux-mir/lib/std/src/sys/wasm/thread_local.rs diff --git a/crux-mir/lib/libstd/sys/wasm/time.rs b/crux-mir/lib/std/src/sys/wasm/time.rs similarity index 100% rename from crux-mir/lib/libstd/sys/wasm/time.rs rename to crux-mir/lib/std/src/sys/wasm/time.rs diff --git a/crux-mir/lib/libstd/sys/windows/alloc.rs b/crux-mir/lib/std/src/sys/windows/alloc.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/alloc.rs rename to crux-mir/lib/std/src/sys/windows/alloc.rs diff --git a/crux-mir/lib/libstd/sys/windows/args.rs b/crux-mir/lib/std/src/sys/windows/args.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/args.rs rename to crux-mir/lib/std/src/sys/windows/args.rs diff --git a/crux-mir/lib/libstd/sys/windows/c.rs b/crux-mir/lib/std/src/sys/windows/c.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/c.rs rename to crux-mir/lib/std/src/sys/windows/c.rs diff --git a/crux-mir/lib/libstd/sys/windows/cmath.rs b/crux-mir/lib/std/src/sys/windows/cmath.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/cmath.rs rename to crux-mir/lib/std/src/sys/windows/cmath.rs diff --git a/crux-mir/lib/libstd/sys/windows/compat.rs b/crux-mir/lib/std/src/sys/windows/compat.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/compat.rs rename to crux-mir/lib/std/src/sys/windows/compat.rs diff --git a/crux-mir/lib/libstd/sys/windows/condvar.rs b/crux-mir/lib/std/src/sys/windows/condvar.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/condvar.rs rename to crux-mir/lib/std/src/sys/windows/condvar.rs diff --git a/crux-mir/lib/libstd/sys/windows/env.rs b/crux-mir/lib/std/src/sys/windows/env.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/env.rs rename to crux-mir/lib/std/src/sys/windows/env.rs diff --git a/crux-mir/lib/libstd/sys/windows/ext/ffi.rs b/crux-mir/lib/std/src/sys/windows/ext/ffi.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/ext/ffi.rs rename to crux-mir/lib/std/src/sys/windows/ext/ffi.rs diff --git a/crux-mir/lib/libstd/sys/windows/ext/fs.rs b/crux-mir/lib/std/src/sys/windows/ext/fs.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/ext/fs.rs rename to crux-mir/lib/std/src/sys/windows/ext/fs.rs diff --git a/crux-mir/lib/libstd/sys/windows/ext/io.rs b/crux-mir/lib/std/src/sys/windows/ext/io.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/ext/io.rs rename to crux-mir/lib/std/src/sys/windows/ext/io.rs diff --git a/crux-mir/lib/libstd/sys/windows/ext/mod.rs b/crux-mir/lib/std/src/sys/windows/ext/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/ext/mod.rs rename to crux-mir/lib/std/src/sys/windows/ext/mod.rs diff --git a/crux-mir/lib/libstd/sys/windows/ext/process.rs b/crux-mir/lib/std/src/sys/windows/ext/process.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/ext/process.rs rename to crux-mir/lib/std/src/sys/windows/ext/process.rs diff --git a/crux-mir/lib/libstd/sys/windows/ext/raw.rs b/crux-mir/lib/std/src/sys/windows/ext/raw.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/ext/raw.rs rename to crux-mir/lib/std/src/sys/windows/ext/raw.rs diff --git a/crux-mir/lib/libstd/sys/windows/ext/thread.rs b/crux-mir/lib/std/src/sys/windows/ext/thread.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/ext/thread.rs rename to crux-mir/lib/std/src/sys/windows/ext/thread.rs diff --git a/crux-mir/lib/libstd/sys/windows/fast_thread_local.rs b/crux-mir/lib/std/src/sys/windows/fast_thread_local.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/fast_thread_local.rs rename to crux-mir/lib/std/src/sys/windows/fast_thread_local.rs diff --git a/crux-mir/lib/libstd/sys/windows/fs.rs b/crux-mir/lib/std/src/sys/windows/fs.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/fs.rs rename to crux-mir/lib/std/src/sys/windows/fs.rs diff --git a/crux-mir/lib/libstd/sys/windows/handle.rs b/crux-mir/lib/std/src/sys/windows/handle.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/handle.rs rename to crux-mir/lib/std/src/sys/windows/handle.rs diff --git a/crux-mir/lib/libstd/sys/windows/io.rs b/crux-mir/lib/std/src/sys/windows/io.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/io.rs rename to crux-mir/lib/std/src/sys/windows/io.rs diff --git a/crux-mir/lib/libstd/sys/windows/memchr.rs b/crux-mir/lib/std/src/sys/windows/memchr.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/memchr.rs rename to crux-mir/lib/std/src/sys/windows/memchr.rs diff --git a/crux-mir/lib/libstd/sys/windows/mod.rs b/crux-mir/lib/std/src/sys/windows/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/mod.rs rename to crux-mir/lib/std/src/sys/windows/mod.rs diff --git a/crux-mir/lib/libstd/sys/windows/mutex.rs b/crux-mir/lib/std/src/sys/windows/mutex.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/mutex.rs rename to crux-mir/lib/std/src/sys/windows/mutex.rs diff --git a/crux-mir/lib/libstd/sys/windows/net.rs b/crux-mir/lib/std/src/sys/windows/net.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/net.rs rename to crux-mir/lib/std/src/sys/windows/net.rs diff --git a/crux-mir/lib/libstd/sys/windows/os.rs b/crux-mir/lib/std/src/sys/windows/os.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/os.rs rename to crux-mir/lib/std/src/sys/windows/os.rs diff --git a/crux-mir/lib/libstd/sys/windows/os_str.rs b/crux-mir/lib/std/src/sys/windows/os_str.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/os_str.rs rename to crux-mir/lib/std/src/sys/windows/os_str.rs diff --git a/crux-mir/lib/libstd/sys/windows/path.rs b/crux-mir/lib/std/src/sys/windows/path.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/path.rs rename to crux-mir/lib/std/src/sys/windows/path.rs diff --git a/crux-mir/lib/libstd/sys/windows/pipe.rs b/crux-mir/lib/std/src/sys/windows/pipe.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/pipe.rs rename to crux-mir/lib/std/src/sys/windows/pipe.rs diff --git a/crux-mir/lib/libstd/sys/windows/process.rs b/crux-mir/lib/std/src/sys/windows/process.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/process.rs rename to crux-mir/lib/std/src/sys/windows/process.rs diff --git a/crux-mir/lib/libstd/sys/windows/rand.rs b/crux-mir/lib/std/src/sys/windows/rand.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/rand.rs rename to crux-mir/lib/std/src/sys/windows/rand.rs diff --git a/crux-mir/lib/libstd/sys/windows/rwlock.rs b/crux-mir/lib/std/src/sys/windows/rwlock.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/rwlock.rs rename to crux-mir/lib/std/src/sys/windows/rwlock.rs diff --git a/crux-mir/lib/libstd/sys/windows/stack_overflow.rs b/crux-mir/lib/std/src/sys/windows/stack_overflow.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/stack_overflow.rs rename to crux-mir/lib/std/src/sys/windows/stack_overflow.rs diff --git a/crux-mir/lib/libstd/sys/windows/stack_overflow_uwp.rs b/crux-mir/lib/std/src/sys/windows/stack_overflow_uwp.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/stack_overflow_uwp.rs rename to crux-mir/lib/std/src/sys/windows/stack_overflow_uwp.rs diff --git a/crux-mir/lib/libstd/sys/windows/stdio.rs b/crux-mir/lib/std/src/sys/windows/stdio.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/stdio.rs rename to crux-mir/lib/std/src/sys/windows/stdio.rs diff --git a/crux-mir/lib/libstd/sys/windows/stdio_uwp.rs b/crux-mir/lib/std/src/sys/windows/stdio_uwp.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/stdio_uwp.rs rename to crux-mir/lib/std/src/sys/windows/stdio_uwp.rs diff --git a/crux-mir/lib/libstd/sys/windows/thread.rs b/crux-mir/lib/std/src/sys/windows/thread.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/thread.rs rename to crux-mir/lib/std/src/sys/windows/thread.rs diff --git a/crux-mir/lib/libstd/sys/windows/thread_local.rs b/crux-mir/lib/std/src/sys/windows/thread_local.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/thread_local.rs rename to crux-mir/lib/std/src/sys/windows/thread_local.rs diff --git a/crux-mir/lib/libstd/sys/windows/time.rs b/crux-mir/lib/std/src/sys/windows/time.rs similarity index 100% rename from crux-mir/lib/libstd/sys/windows/time.rs rename to crux-mir/lib/std/src/sys/windows/time.rs diff --git a/crux-mir/lib/libstd/sys_common/alloc.rs b/crux-mir/lib/std/src/sys_common/alloc.rs similarity index 100% rename from crux-mir/lib/libstd/sys_common/alloc.rs rename to crux-mir/lib/std/src/sys_common/alloc.rs diff --git a/crux-mir/lib/libstd/sys_common/at_exit_imp.rs b/crux-mir/lib/std/src/sys_common/at_exit_imp.rs similarity index 100% rename from crux-mir/lib/libstd/sys_common/at_exit_imp.rs rename to crux-mir/lib/std/src/sys_common/at_exit_imp.rs diff --git a/crux-mir/lib/libstd/sys_common/backtrace.rs b/crux-mir/lib/std/src/sys_common/backtrace.rs similarity index 100% rename from crux-mir/lib/libstd/sys_common/backtrace.rs rename to crux-mir/lib/std/src/sys_common/backtrace.rs diff --git a/crux-mir/lib/libstd/sys_common/bytestring.rs b/crux-mir/lib/std/src/sys_common/bytestring.rs similarity index 100% rename from crux-mir/lib/libstd/sys_common/bytestring.rs rename to crux-mir/lib/std/src/sys_common/bytestring.rs diff --git a/crux-mir/lib/libstd/sys_common/condvar.rs b/crux-mir/lib/std/src/sys_common/condvar.rs similarity index 100% rename from crux-mir/lib/libstd/sys_common/condvar.rs rename to crux-mir/lib/std/src/sys_common/condvar.rs diff --git a/crux-mir/lib/libstd/sys_common/fs.rs b/crux-mir/lib/std/src/sys_common/fs.rs similarity index 100% rename from crux-mir/lib/libstd/sys_common/fs.rs rename to crux-mir/lib/std/src/sys_common/fs.rs diff --git a/crux-mir/lib/libstd/sys_common/io.rs b/crux-mir/lib/std/src/sys_common/io.rs similarity index 100% rename from crux-mir/lib/libstd/sys_common/io.rs rename to crux-mir/lib/std/src/sys_common/io.rs diff --git a/crux-mir/lib/libstd/sys_common/mod.rs b/crux-mir/lib/std/src/sys_common/mod.rs similarity index 100% rename from crux-mir/lib/libstd/sys_common/mod.rs rename to crux-mir/lib/std/src/sys_common/mod.rs diff --git a/crux-mir/lib/libstd/sys_common/mutex.rs b/crux-mir/lib/std/src/sys_common/mutex.rs similarity index 100% rename from crux-mir/lib/libstd/sys_common/mutex.rs rename to crux-mir/lib/std/src/sys_common/mutex.rs diff --git a/crux-mir/lib/libstd/sys_common/net.rs b/crux-mir/lib/std/src/sys_common/net.rs similarity index 100% rename from crux-mir/lib/libstd/sys_common/net.rs rename to crux-mir/lib/std/src/sys_common/net.rs diff --git a/crux-mir/lib/libstd/sys_common/os_str_bytes.rs b/crux-mir/lib/std/src/sys_common/os_str_bytes.rs similarity index 100% rename from crux-mir/lib/libstd/sys_common/os_str_bytes.rs rename to crux-mir/lib/std/src/sys_common/os_str_bytes.rs diff --git a/crux-mir/lib/libstd/sys_common/poison.rs b/crux-mir/lib/std/src/sys_common/poison.rs similarity index 100% rename from crux-mir/lib/libstd/sys_common/poison.rs rename to crux-mir/lib/std/src/sys_common/poison.rs diff --git a/crux-mir/lib/libstd/sys_common/process.rs b/crux-mir/lib/std/src/sys_common/process.rs similarity index 100% rename from crux-mir/lib/libstd/sys_common/process.rs rename to crux-mir/lib/std/src/sys_common/process.rs diff --git a/crux-mir/lib/libstd/sys_common/remutex.rs b/crux-mir/lib/std/src/sys_common/remutex.rs similarity index 100% rename from crux-mir/lib/libstd/sys_common/remutex.rs rename to crux-mir/lib/std/src/sys_common/remutex.rs diff --git a/crux-mir/lib/libstd/sys_common/rwlock.rs b/crux-mir/lib/std/src/sys_common/rwlock.rs similarity index 100% rename from crux-mir/lib/libstd/sys_common/rwlock.rs rename to crux-mir/lib/std/src/sys_common/rwlock.rs diff --git a/crux-mir/lib/libstd/sys_common/thread.rs b/crux-mir/lib/std/src/sys_common/thread.rs similarity index 100% rename from crux-mir/lib/libstd/sys_common/thread.rs rename to crux-mir/lib/std/src/sys_common/thread.rs diff --git a/crux-mir/lib/libstd/sys_common/thread_info.rs b/crux-mir/lib/std/src/sys_common/thread_info.rs similarity index 100% rename from crux-mir/lib/libstd/sys_common/thread_info.rs rename to crux-mir/lib/std/src/sys_common/thread_info.rs diff --git a/crux-mir/lib/libstd/sys_common/thread_local.rs b/crux-mir/lib/std/src/sys_common/thread_local.rs similarity index 100% rename from crux-mir/lib/libstd/sys_common/thread_local.rs rename to crux-mir/lib/std/src/sys_common/thread_local.rs diff --git a/crux-mir/lib/libstd/sys_common/util.rs b/crux-mir/lib/std/src/sys_common/util.rs similarity index 100% rename from crux-mir/lib/libstd/sys_common/util.rs rename to crux-mir/lib/std/src/sys_common/util.rs diff --git a/crux-mir/lib/libstd/sys_common/wtf8.rs b/crux-mir/lib/std/src/sys_common/wtf8.rs similarity index 100% rename from crux-mir/lib/libstd/sys_common/wtf8.rs rename to crux-mir/lib/std/src/sys_common/wtf8.rs diff --git a/crux-mir/lib/libstd/thread/local.rs b/crux-mir/lib/std/src/thread/local.rs similarity index 100% rename from crux-mir/lib/libstd/thread/local.rs rename to crux-mir/lib/std/src/thread/local.rs diff --git a/crux-mir/lib/libstd/thread/mod.rs b/crux-mir/lib/std/src/thread/mod.rs similarity index 100% rename from crux-mir/lib/libstd/thread/mod.rs rename to crux-mir/lib/std/src/thread/mod.rs diff --git a/crux-mir/lib/libstd/time.rs b/crux-mir/lib/std/src/time.rs similarity index 100% rename from crux-mir/lib/libstd/time.rs rename to crux-mir/lib/std/src/time.rs diff --git a/crux-mir/lib/libstd/tests/env.rs b/crux-mir/lib/std/tests/env.rs similarity index 100% rename from crux-mir/lib/libstd/tests/env.rs rename to crux-mir/lib/std/tests/env.rs diff --git a/crux-mir/lib/libstd/tests/run-time-detect.rs b/crux-mir/lib/std/tests/run-time-detect.rs similarity index 100% rename from crux-mir/lib/libstd/tests/run-time-detect.rs rename to crux-mir/lib/std/tests/run-time-detect.rs diff --git a/crux-mir/lib/libterm/Cargo.toml b/crux-mir/lib/term/Cargo.toml similarity index 100% rename from crux-mir/lib/libterm/Cargo.toml rename to crux-mir/lib/term/Cargo.toml diff --git a/crux-mir/lib/libterm/lib.rs b/crux-mir/lib/term/src/lib.rs similarity index 100% rename from crux-mir/lib/libterm/lib.rs rename to crux-mir/lib/term/src/lib.rs diff --git a/crux-mir/lib/libterm/terminfo/mod.rs b/crux-mir/lib/term/src/terminfo/mod.rs similarity index 100% rename from crux-mir/lib/libterm/terminfo/mod.rs rename to crux-mir/lib/term/src/terminfo/mod.rs diff --git a/crux-mir/lib/libterm/terminfo/parm.rs b/crux-mir/lib/term/src/terminfo/parm.rs similarity index 100% rename from crux-mir/lib/libterm/terminfo/parm.rs rename to crux-mir/lib/term/src/terminfo/parm.rs diff --git a/crux-mir/lib/libterm/terminfo/parm/tests.rs b/crux-mir/lib/term/src/terminfo/parm/tests.rs similarity index 100% rename from crux-mir/lib/libterm/terminfo/parm/tests.rs rename to crux-mir/lib/term/src/terminfo/parm/tests.rs diff --git a/crux-mir/lib/libterm/terminfo/parser/compiled.rs b/crux-mir/lib/term/src/terminfo/parser/compiled.rs similarity index 100% rename from crux-mir/lib/libterm/terminfo/parser/compiled.rs rename to crux-mir/lib/term/src/terminfo/parser/compiled.rs diff --git a/crux-mir/lib/libterm/terminfo/parser/compiled/tests.rs b/crux-mir/lib/term/src/terminfo/parser/compiled/tests.rs similarity index 100% rename from crux-mir/lib/libterm/terminfo/parser/compiled/tests.rs rename to crux-mir/lib/term/src/terminfo/parser/compiled/tests.rs diff --git a/crux-mir/lib/libterm/terminfo/searcher.rs b/crux-mir/lib/term/src/terminfo/searcher.rs similarity index 100% rename from crux-mir/lib/libterm/terminfo/searcher.rs rename to crux-mir/lib/term/src/terminfo/searcher.rs diff --git a/crux-mir/lib/libterm/terminfo/searcher/tests.rs b/crux-mir/lib/term/src/terminfo/searcher/tests.rs similarity index 100% rename from crux-mir/lib/libterm/terminfo/searcher/tests.rs rename to crux-mir/lib/term/src/terminfo/searcher/tests.rs diff --git a/crux-mir/lib/libterm/win.rs b/crux-mir/lib/term/src/win.rs similarity index 100% rename from crux-mir/lib/libterm/win.rs rename to crux-mir/lib/term/src/win.rs diff --git a/crux-mir/lib/libtest/Cargo.toml b/crux-mir/lib/test/Cargo.toml similarity index 100% rename from crux-mir/lib/libtest/Cargo.toml rename to crux-mir/lib/test/Cargo.toml diff --git a/crux-mir/lib/libtest/bench.rs b/crux-mir/lib/test/src/bench.rs similarity index 100% rename from crux-mir/lib/libtest/bench.rs rename to crux-mir/lib/test/src/bench.rs diff --git a/crux-mir/lib/libtest/cli.rs b/crux-mir/lib/test/src/cli.rs similarity index 100% rename from crux-mir/lib/libtest/cli.rs rename to crux-mir/lib/test/src/cli.rs diff --git a/crux-mir/lib/libtest/console.rs b/crux-mir/lib/test/src/console.rs similarity index 100% rename from crux-mir/lib/libtest/console.rs rename to crux-mir/lib/test/src/console.rs diff --git a/crux-mir/lib/libtest/event.rs b/crux-mir/lib/test/src/event.rs similarity index 100% rename from crux-mir/lib/libtest/event.rs rename to crux-mir/lib/test/src/event.rs diff --git a/crux-mir/lib/libtest/formatters/json.rs b/crux-mir/lib/test/src/formatters/json.rs similarity index 100% rename from crux-mir/lib/libtest/formatters/json.rs rename to crux-mir/lib/test/src/formatters/json.rs diff --git a/crux-mir/lib/libtest/formatters/mod.rs b/crux-mir/lib/test/src/formatters/mod.rs similarity index 100% rename from crux-mir/lib/libtest/formatters/mod.rs rename to crux-mir/lib/test/src/formatters/mod.rs diff --git a/crux-mir/lib/libtest/formatters/pretty.rs b/crux-mir/lib/test/src/formatters/pretty.rs similarity index 100% rename from crux-mir/lib/libtest/formatters/pretty.rs rename to crux-mir/lib/test/src/formatters/pretty.rs diff --git a/crux-mir/lib/libtest/formatters/terse.rs b/crux-mir/lib/test/src/formatters/terse.rs similarity index 100% rename from crux-mir/lib/libtest/formatters/terse.rs rename to crux-mir/lib/test/src/formatters/terse.rs diff --git a/crux-mir/lib/libtest/helpers/concurrency.rs b/crux-mir/lib/test/src/helpers/concurrency.rs similarity index 100% rename from crux-mir/lib/libtest/helpers/concurrency.rs rename to crux-mir/lib/test/src/helpers/concurrency.rs diff --git a/crux-mir/lib/libtest/helpers/exit_code.rs b/crux-mir/lib/test/src/helpers/exit_code.rs similarity index 100% rename from crux-mir/lib/libtest/helpers/exit_code.rs rename to crux-mir/lib/test/src/helpers/exit_code.rs diff --git a/crux-mir/lib/libtest/helpers/isatty.rs b/crux-mir/lib/test/src/helpers/isatty.rs similarity index 100% rename from crux-mir/lib/libtest/helpers/isatty.rs rename to crux-mir/lib/test/src/helpers/isatty.rs diff --git a/crux-mir/lib/libtest/helpers/metrics.rs b/crux-mir/lib/test/src/helpers/metrics.rs similarity index 100% rename from crux-mir/lib/libtest/helpers/metrics.rs rename to crux-mir/lib/test/src/helpers/metrics.rs diff --git a/crux-mir/lib/libtest/helpers/mod.rs b/crux-mir/lib/test/src/helpers/mod.rs similarity index 100% rename from crux-mir/lib/libtest/helpers/mod.rs rename to crux-mir/lib/test/src/helpers/mod.rs diff --git a/crux-mir/lib/libtest/helpers/sink.rs b/crux-mir/lib/test/src/helpers/sink.rs similarity index 100% rename from crux-mir/lib/libtest/helpers/sink.rs rename to crux-mir/lib/test/src/helpers/sink.rs diff --git a/crux-mir/lib/libtest/lib.rs b/crux-mir/lib/test/src/lib.rs similarity index 100% rename from crux-mir/lib/libtest/lib.rs rename to crux-mir/lib/test/src/lib.rs diff --git a/crux-mir/lib/libtest/options.rs b/crux-mir/lib/test/src/options.rs similarity index 100% rename from crux-mir/lib/libtest/options.rs rename to crux-mir/lib/test/src/options.rs diff --git a/crux-mir/lib/libtest/stats.rs b/crux-mir/lib/test/src/stats.rs similarity index 100% rename from crux-mir/lib/libtest/stats.rs rename to crux-mir/lib/test/src/stats.rs diff --git a/crux-mir/lib/libtest/stats/tests.rs b/crux-mir/lib/test/src/stats/tests.rs similarity index 100% rename from crux-mir/lib/libtest/stats/tests.rs rename to crux-mir/lib/test/src/stats/tests.rs diff --git a/crux-mir/lib/libtest/test_result.rs b/crux-mir/lib/test/src/test_result.rs similarity index 100% rename from crux-mir/lib/libtest/test_result.rs rename to crux-mir/lib/test/src/test_result.rs diff --git a/crux-mir/lib/libtest/tests.rs b/crux-mir/lib/test/src/tests.rs similarity index 100% rename from crux-mir/lib/libtest/tests.rs rename to crux-mir/lib/test/src/tests.rs diff --git a/crux-mir/lib/libtest/time.rs b/crux-mir/lib/test/src/time.rs similarity index 100% rename from crux-mir/lib/libtest/time.rs rename to crux-mir/lib/test/src/time.rs diff --git a/crux-mir/lib/libtest/types.rs b/crux-mir/lib/test/src/types.rs similarity index 100% rename from crux-mir/lib/libtest/types.rs rename to crux-mir/lib/test/src/types.rs diff --git a/crux-mir/lib/libunwind/Cargo.toml b/crux-mir/lib/unwind/Cargo.toml similarity index 100% rename from crux-mir/lib/libunwind/Cargo.toml rename to crux-mir/lib/unwind/Cargo.toml diff --git a/crux-mir/lib/libunwind/build.rs b/crux-mir/lib/unwind/build.rs similarity index 100% rename from crux-mir/lib/libunwind/build.rs rename to crux-mir/lib/unwind/build.rs diff --git a/crux-mir/lib/libunwind/lib.rs b/crux-mir/lib/unwind/lib.rs similarity index 100% rename from crux-mir/lib/libunwind/lib.rs rename to crux-mir/lib/unwind/lib.rs diff --git a/crux-mir/lib/libunwind/libunwind.rs b/crux-mir/lib/unwind/libunwind.rs similarity index 100% rename from crux-mir/lib/libunwind/libunwind.rs rename to crux-mir/lib/unwind/libunwind.rs diff --git a/crux-mir/lib/unwind/src/build.rs b/crux-mir/lib/unwind/src/build.rs new file mode 100644 index 000000000..0628e5d2f --- /dev/null +++ b/crux-mir/lib/unwind/src/build.rs @@ -0,0 +1,124 @@ +use std::env; + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + let target = env::var("TARGET").expect("TARGET was not set"); + + if cfg!(feature = "llvm-libunwind") + && ((target.contains("linux") && !target.contains("musl")) || target.contains("fuchsia")) + { + // Build the unwinding from libunwind C/C++ source code. + llvm_libunwind::compile(); + } else if target.contains("linux") { + if target.contains("musl") { + // linking for musl is handled in lib.rs + llvm_libunwind::compile(); + } else if !target.contains("android") { + println!("cargo:rustc-link-lib=gcc_s"); + } + } else if target.contains("freebsd") { + println!("cargo:rustc-link-lib=gcc_s"); + } else if target.contains("rumprun") { + println!("cargo:rustc-link-lib=unwind"); + } else if target.contains("netbsd") { + println!("cargo:rustc-link-lib=gcc_s"); + } else if target.contains("openbsd") { + if target.contains("sparc64") { + println!("cargo:rustc-link-lib=gcc"); + } else { + println!("cargo:rustc-link-lib=c++abi"); + } + } else if target.contains("solaris") { + println!("cargo:rustc-link-lib=gcc_s"); + } else if target.contains("dragonfly") { + println!("cargo:rustc-link-lib=gcc_pic"); + } else if target.contains("pc-windows-gnu") { + // This is handled in the target spec with late_link_args_[static|dynamic] + + // cfg!(bootstrap) doesn't work in build scripts + if env::var("RUSTC_STAGE").ok() == Some("0".to_string()) { + println!("cargo:rustc-link-lib=static-nobundle=gcc_eh"); + println!("cargo:rustc-link-lib=static-nobundle=pthread"); + } + } else if target.contains("uwp-windows-gnu") { + println!("cargo:rustc-link-lib=unwind"); + } else if target.contains("fuchsia") { + println!("cargo:rustc-link-lib=unwind"); + } else if target.contains("haiku") { + println!("cargo:rustc-link-lib=gcc_s"); + } else if target.contains("redox") { + // redox is handled in lib.rs + } else if target.contains("cloudabi") { + println!("cargo:rustc-link-lib=unwind"); + } +} + +mod llvm_libunwind { + use std::env; + use std::path::Path; + + /// Compile the libunwind C/C++ source code. + pub fn compile() { + let target_env = env::var("CARGO_CFG_TARGET_ENV").unwrap(); + let target_vendor = env::var("CARGO_CFG_TARGET_VENDOR").unwrap(); + let target_endian_little = env::var("CARGO_CFG_TARGET_ENDIAN").unwrap() != "big"; + let cfg = &mut cc::Build::new(); + + cfg.cpp(true); + cfg.cpp_set_stdlib(None); + cfg.warnings(false); + + // libunwind expects a __LITTLE_ENDIAN__ macro to be set for LE archs, cf. #65765 + if target_endian_little { + cfg.define("__LITTLE_ENDIAN__", Some("1")); + } + + if target_env == "msvc" { + // Don't pull in extra libraries on MSVC + cfg.flag("/Zl"); + cfg.flag("/EHsc"); + cfg.define("_CRT_SECURE_NO_WARNINGS", None); + cfg.define("_LIBUNWIND_DISABLE_VISIBILITY_ANNOTATIONS", None); + } else { + cfg.flag("-std=c99"); + cfg.flag("-std=c++11"); + cfg.flag("-nostdinc++"); + cfg.flag("-fno-exceptions"); + cfg.flag("-fno-rtti"); + cfg.flag("-fstrict-aliasing"); + cfg.flag("-funwind-tables"); + } + + let mut unwind_sources = vec![ + "Unwind-EHABI.cpp", + "Unwind-seh.cpp", + "Unwind-sjlj.c", + "UnwindLevel1-gcc-ext.c", + "UnwindLevel1.c", + "UnwindRegistersRestore.S", + "UnwindRegistersSave.S", + "libunwind.cpp", + ]; + + if target_vendor == "apple" { + unwind_sources.push("Unwind_AppleExtras.cpp"); + } + + let root = Path::new("../llvm-project/libunwind"); + cfg.include(root.join("include")); + for src in unwind_sources { + cfg.file(root.join("src").join(src)); + } + + if target_env == "musl" { + // use the same C compiler command to compile C++ code so we do not need to setup the + // C++ compiler env variables on the builders + cfg.cpp(false); + // linking for musl is handled in lib.rs + cfg.cargo_metadata(false); + println!("cargo:rustc-link-search=native={}", env::var("OUT_DIR").unwrap()); + } + + cfg.compile("unwind"); + } +} diff --git a/crux-mir/lib/unwind/src/lib.rs b/crux-mir/lib/unwind/src/lib.rs new file mode 100644 index 000000000..18d41be77 --- /dev/null +++ b/crux-mir/lib/unwind/src/lib.rs @@ -0,0 +1,29 @@ +#![no_std] +#![unstable(feature = "panic_unwind", issue = "32837")] +#![feature(link_cfg)] +#![feature(nll)] +#![feature(staged_api)] +#![feature(unwind_attributes)] +#![feature(static_nobundle)] +#![cfg_attr(not(target_env = "msvc"), feature(libc))] + +cfg_if::cfg_if! { + if #[cfg(target_env = "msvc")] { + // no extra unwinder support needed + } else if #[cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))] { + // no unwinder on the system! + } else { + mod libunwind; + pub use libunwind::*; + } +} + +#[cfg(target_env = "musl")] +#[link(name = "unwind", kind = "static", cfg(target_feature = "crt-static"))] +#[link(name = "gcc_s", cfg(not(target_feature = "crt-static")))] +extern "C" {} + +#[cfg(target_os = "redox")] +#[link(name = "gcc_eh", kind = "static-nobundle", cfg(target_feature = "crt-static"))] +#[link(name = "gcc_s", cfg(not(target_feature = "crt-static")))] +extern "C" {} diff --git a/crux-mir/lib/unwind/src/libunwind.rs b/crux-mir/lib/unwind/src/libunwind.rs new file mode 100644 index 000000000..0c57861f7 --- /dev/null +++ b/crux-mir/lib/unwind/src/libunwind.rs @@ -0,0 +1,279 @@ +#![allow(nonstandard_style)] + +use libc::{c_int, c_void, uintptr_t}; + +#[repr(C)] +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum _Unwind_Reason_Code { + _URC_NO_REASON = 0, + _URC_FOREIGN_EXCEPTION_CAUGHT = 1, + _URC_FATAL_PHASE2_ERROR = 2, + _URC_FATAL_PHASE1_ERROR = 3, + _URC_NORMAL_STOP = 4, + _URC_END_OF_STACK = 5, + _URC_HANDLER_FOUND = 6, + _URC_INSTALL_CONTEXT = 7, + _URC_CONTINUE_UNWIND = 8, + _URC_FAILURE = 9, // used only by ARM EHABI +} +pub use _Unwind_Reason_Code::*; + +pub type _Unwind_Exception_Class = u64; +pub type _Unwind_Word = uintptr_t; +pub type _Unwind_Ptr = uintptr_t; +pub type _Unwind_Trace_Fn = + extern "C" fn(ctx: *mut _Unwind_Context, arg: *mut c_void) -> _Unwind_Reason_Code; + +#[cfg(target_arch = "x86")] +pub const unwinder_private_data_size: usize = 5; + +#[cfg(target_arch = "x86_64")] +pub const unwinder_private_data_size: usize = 6; + +#[cfg(all(target_arch = "arm", not(target_os = "ios")))] +pub const unwinder_private_data_size: usize = 20; + +#[cfg(all(target_arch = "arm", target_os = "ios"))] +pub const unwinder_private_data_size: usize = 5; + +#[cfg(target_arch = "aarch64")] +pub const unwinder_private_data_size: usize = 2; + +#[cfg(target_arch = "mips")] +pub const unwinder_private_data_size: usize = 2; + +#[cfg(target_arch = "mips64")] +pub const unwinder_private_data_size: usize = 2; + +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +pub const unwinder_private_data_size: usize = 2; + +#[cfg(target_arch = "s390x")] +pub const unwinder_private_data_size: usize = 2; + +#[cfg(target_arch = "sparc64")] +pub const unwinder_private_data_size: usize = 2; + +#[cfg(target_arch = "riscv64")] +pub const unwinder_private_data_size: usize = 2; + +#[cfg(target_os = "emscripten")] +pub const unwinder_private_data_size: usize = 20; + +#[cfg(all(target_arch = "hexagon", target_os = "linux"))] +pub const unwinder_private_data_size: usize = 35; + +#[repr(C)] +pub struct _Unwind_Exception { + pub exception_class: _Unwind_Exception_Class, + pub exception_cleanup: _Unwind_Exception_Cleanup_Fn, + pub private: [_Unwind_Word; unwinder_private_data_size], +} + +pub enum _Unwind_Context {} + +pub type _Unwind_Exception_Cleanup_Fn = + extern "C" fn(unwind_code: _Unwind_Reason_Code, exception: *mut _Unwind_Exception); +#[cfg_attr( + all(feature = "llvm-libunwind", any(target_os = "fuchsia", target_os = "linux")), + link(name = "unwind", kind = "static") +)] +extern "C" { + #[unwind(allowed)] + pub fn _Unwind_Resume(exception: *mut _Unwind_Exception) -> !; + pub fn _Unwind_DeleteException(exception: *mut _Unwind_Exception); + pub fn _Unwind_GetLanguageSpecificData(ctx: *mut _Unwind_Context) -> *mut c_void; + pub fn _Unwind_GetRegionStart(ctx: *mut _Unwind_Context) -> _Unwind_Ptr; + pub fn _Unwind_GetTextRelBase(ctx: *mut _Unwind_Context) -> _Unwind_Ptr; + pub fn _Unwind_GetDataRelBase(ctx: *mut _Unwind_Context) -> _Unwind_Ptr; +} + +cfg_if::cfg_if! { +if #[cfg(all(any(target_os = "ios", target_os = "netbsd", not(target_arch = "arm"))))] { + // Not ARM EHABI + #[repr(C)] + #[derive(Copy, Clone, PartialEq)] + pub enum _Unwind_Action { + _UA_SEARCH_PHASE = 1, + _UA_CLEANUP_PHASE = 2, + _UA_HANDLER_FRAME = 4, + _UA_FORCE_UNWIND = 8, + _UA_END_OF_STACK = 16, + } + pub use _Unwind_Action::*; + + #[cfg_attr(all(feature = "llvm-libunwind", + any(target_os = "fuchsia", target_os = "linux")), + link(name = "unwind", kind = "static"))] + extern "C" { + pub fn _Unwind_GetGR(ctx: *mut _Unwind_Context, reg_index: c_int) -> _Unwind_Word; + pub fn _Unwind_SetGR(ctx: *mut _Unwind_Context, reg_index: c_int, value: _Unwind_Word); + pub fn _Unwind_GetIP(ctx: *mut _Unwind_Context) -> _Unwind_Word; + pub fn _Unwind_SetIP(ctx: *mut _Unwind_Context, value: _Unwind_Word); + pub fn _Unwind_GetIPInfo(ctx: *mut _Unwind_Context, ip_before_insn: *mut c_int) + -> _Unwind_Word; + pub fn _Unwind_FindEnclosingFunction(pc: *mut c_void) -> *mut c_void; + } + +} else { + // ARM EHABI + #[repr(C)] + #[derive(Copy, Clone, PartialEq)] + pub enum _Unwind_State { + _US_VIRTUAL_UNWIND_FRAME = 0, + _US_UNWIND_FRAME_STARTING = 1, + _US_UNWIND_FRAME_RESUME = 2, + _US_ACTION_MASK = 3, + _US_FORCE_UNWIND = 8, + _US_END_OF_STACK = 16, + } + pub use _Unwind_State::*; + + #[repr(C)] + enum _Unwind_VRS_Result { + _UVRSR_OK = 0, + _UVRSR_NOT_IMPLEMENTED = 1, + _UVRSR_FAILED = 2, + } + #[repr(C)] + enum _Unwind_VRS_RegClass { + _UVRSC_CORE = 0, + _UVRSC_VFP = 1, + _UVRSC_FPA = 2, + _UVRSC_WMMXD = 3, + _UVRSC_WMMXC = 4, + } + use _Unwind_VRS_RegClass::*; + #[repr(C)] + enum _Unwind_VRS_DataRepresentation { + _UVRSD_UINT32 = 0, + _UVRSD_VFPX = 1, + _UVRSD_FPAX = 2, + _UVRSD_UINT64 = 3, + _UVRSD_FLOAT = 4, + _UVRSD_DOUBLE = 5, + } + use _Unwind_VRS_DataRepresentation::*; + + pub const UNWIND_POINTER_REG: c_int = 12; + pub const UNWIND_SP_REG: c_int = 13; + pub const UNWIND_IP_REG: c_int = 15; + + #[cfg_attr(all(feature = "llvm-libunwind", + any(target_os = "fuchsia", target_os = "linux")), + link(name = "unwind", kind = "static"))] + extern "C" { + fn _Unwind_VRS_Get(ctx: *mut _Unwind_Context, + regclass: _Unwind_VRS_RegClass, + regno: _Unwind_Word, + repr: _Unwind_VRS_DataRepresentation, + data: *mut c_void) + -> _Unwind_VRS_Result; + + fn _Unwind_VRS_Set(ctx: *mut _Unwind_Context, + regclass: _Unwind_VRS_RegClass, + regno: _Unwind_Word, + repr: _Unwind_VRS_DataRepresentation, + data: *mut c_void) + -> _Unwind_VRS_Result; + } + + // On Android or ARM/Linux, these are implemented as macros: + + pub unsafe fn _Unwind_GetGR(ctx: *mut _Unwind_Context, reg_index: c_int) -> _Unwind_Word { + let mut val: _Unwind_Word = 0; + _Unwind_VRS_Get(ctx, _UVRSC_CORE, reg_index as _Unwind_Word, _UVRSD_UINT32, + &mut val as *mut _ as *mut c_void); + val + } + + pub unsafe fn _Unwind_SetGR(ctx: *mut _Unwind_Context, reg_index: c_int, value: _Unwind_Word) { + let mut value = value; + _Unwind_VRS_Set(ctx, _UVRSC_CORE, reg_index as _Unwind_Word, _UVRSD_UINT32, + &mut value as *mut _ as *mut c_void); + } + + pub unsafe fn _Unwind_GetIP(ctx: *mut _Unwind_Context) + -> _Unwind_Word { + let val = _Unwind_GetGR(ctx, UNWIND_IP_REG); + (val & !1) as _Unwind_Word + } + + pub unsafe fn _Unwind_SetIP(ctx: *mut _Unwind_Context, + value: _Unwind_Word) { + // Propagate thumb bit to instruction pointer + let thumb_state = _Unwind_GetGR(ctx, UNWIND_IP_REG) & 1; + let value = value | thumb_state; + _Unwind_SetGR(ctx, UNWIND_IP_REG, value); + } + + pub unsafe fn _Unwind_GetIPInfo(ctx: *mut _Unwind_Context, + ip_before_insn: *mut c_int) + -> _Unwind_Word { + *ip_before_insn = 0; + _Unwind_GetIP(ctx) + } + + // This function also doesn't exist on Android or ARM/Linux, so make it a no-op + pub unsafe fn _Unwind_FindEnclosingFunction(pc: *mut c_void) -> *mut c_void { + pc + } +} +} // cfg_if! + +cfg_if::cfg_if! { +if #[cfg(not(all(target_os = "ios", target_arch = "arm")))] { + // Not 32-bit iOS + #[cfg_attr(all(feature = "llvm-libunwind", + any(target_os = "fuchsia", target_os = "linux")), + link(name = "unwind", kind = "static"))] + extern "C" { + #[unwind(allowed)] + pub fn _Unwind_RaiseException(exception: *mut _Unwind_Exception) -> _Unwind_Reason_Code; + pub fn _Unwind_Backtrace(trace: _Unwind_Trace_Fn, + trace_argument: *mut c_void) + -> _Unwind_Reason_Code; + } +} else { + // 32-bit iOS uses SjLj and does not provide _Unwind_Backtrace() + #[cfg_attr(all(feature = "llvm-libunwind", + any(target_os = "fuchsia", target_os = "linux")), + link(name = "unwind", kind = "static"))] + extern "C" { + #[unwind(allowed)] + pub fn _Unwind_SjLj_RaiseException(e: *mut _Unwind_Exception) -> _Unwind_Reason_Code; + } + + #[inline] + pub unsafe fn _Unwind_RaiseException(exc: *mut _Unwind_Exception) -> _Unwind_Reason_Code { + _Unwind_SjLj_RaiseException(exc) + } +} +} // cfg_if! + +cfg_if::cfg_if! { +if #[cfg(all(windows, target_arch = "x86_64", target_env = "gnu"))] { + // We declare these as opaque types. This is fine since you just need to + // pass them to _GCC_specific_handler and forget about them. + pub enum EXCEPTION_RECORD {} + pub type LPVOID = *mut c_void; + pub enum CONTEXT {} + pub enum DISPATCHER_CONTEXT {} + pub type EXCEPTION_DISPOSITION = c_int; + type PersonalityFn = unsafe extern "C" fn(version: c_int, + actions: _Unwind_Action, + exception_class: _Unwind_Exception_Class, + exception_object: *mut _Unwind_Exception, + context: *mut _Unwind_Context) + -> _Unwind_Reason_Code; + + extern "C" { + pub fn _GCC_specific_handler(exceptionRecord: *mut EXCEPTION_RECORD, + establisherFrame: LPVOID, + contextRecord: *mut CONTEXT, + dispatcherContext: *mut DISPATCHER_CONTEXT, + personality: PersonalityFn) + -> EXCEPTION_DISPOSITION; + } +} +} // cfg_if! From b121efe28fc3b7a96ec5c6be713eef73010a7ba6 Mon Sep 17 00:00:00 2001 From: James LaMar Date: Sat, 11 Mar 2023 10:39:32 -0800 Subject: [PATCH 002/114] WIP mir-json - updated libs - changed crux_mir to crux::mir in tests - new lib build script (build.sh) - changes to crux-mir to support changes to rustc (nightly-2023-01-22) --- crucible-mir/src/Mir/GenericOps.hs | 13 +- crucible-mir/src/Mir/JSON.hs | 49 +- crucible-mir/src/Mir/Mir.hs | 35 +- crucible-mir/src/Mir/PP.hs | 15 +- crucible-mir/src/Mir/Trans.hs | 28 +- crucible-mir/src/Mir/TransTy.hs | 4 +- crux-mir/build.sh | 68 + crux-mir/lib/addr2line/.cargo-ok | 1 + crux-mir/lib/addr2line/.cargo_vcs_info.json | 5 + crux-mir/lib/addr2line/.gitignore | 5 + crux-mir/lib/addr2line/CHANGELOG.md | 260 + crux-mir/lib/addr2line/Cargo.toml | 120 + crux-mir/lib/addr2line/Cargo.toml.orig | 75 + .../LICENSE-APACHE | 0 .../lib/{getopts => addr2line}/LICENSE-MIT | 2 +- crux-mir/lib/addr2line/README.md | 48 + crux-mir/lib/addr2line/bench.plot.r | 23 + crux-mir/lib/addr2line/benchmark.sh | 112 + crux-mir/lib/addr2line/coverage.sh | 5 + crux-mir/lib/addr2line/examples/addr2line.rs | 299 + crux-mir/lib/addr2line/rustfmt.toml | 1 + crux-mir/lib/addr2line/src/function.rs | 520 + crux-mir/lib/addr2line/src/lazy.rs | 29 + crux-mir/lib/addr2line/src/lib.rs | 1192 + crux-mir/lib/addr2line/tests/correctness.rs | 91 + .../lib/addr2line/tests/output_equivalence.rs | 145 + crux-mir/lib/addr2line/tests/parse.rs | 118 + crux-mir/lib/adler/.cargo-ok | 1 + crux-mir/lib/adler/.cargo_vcs_info.json | 5 + crux-mir/lib/adler/.github/workflows/ci.yml | 81 + crux-mir/lib/{stdarch => adler}/.gitignore | 4 +- crux-mir/lib/adler/CHANGELOG.md | 63 + crux-mir/lib/adler/Cargo.toml | 64 + crux-mir/lib/adler/Cargo.toml.orig | 68 + crux-mir/lib/adler/LICENSE-0BSD | 12 + crux-mir/lib/adler/LICENSE-APACHE | 201 + .../lib/{rustc-demangle => adler}/LICENSE-MIT | 2 - crux-mir/lib/adler/README.md | 39 + crux-mir/lib/adler/RELEASE_PROCESS.md | 13 + crux-mir/lib/adler/benches/bench.rs | 109 + crux-mir/lib/adler/src/algo.rs | 146 + crux-mir/lib/adler/src/lib.rs | 287 + crux-mir/lib/alloc/Cargo.toml | 37 + crux-mir/lib/alloc/benches/binary_heap.rs | 91 + crux-mir/lib/alloc/benches/btree/map.rs | 433 +- crux-mir/lib/alloc/benches/btree/set.rs | 80 +- crux-mir/lib/alloc/benches/lib.rs | 18 +- crux-mir/lib/alloc/benches/slice.rs | 25 +- crux-mir/lib/alloc/benches/str.rs | 65 +- crux-mir/lib/alloc/benches/vec.rs | 500 +- crux-mir/lib/alloc/benches/vec_deque.rs | 71 + crux-mir/lib/alloc/src/Cargo.toml | 37 - crux-mir/lib/alloc/src/alloc.rs | 362 +- crux-mir/lib/alloc/src/alloc/tests.rs | 13 +- crux-mir/lib/alloc/src/borrow.rs | 44 +- crux-mir/lib/alloc/src/boxed.rs | 1759 +- crux-mir/lib/alloc/src/boxed/thin.rs | 302 + .../{binary_heap.rs => binary_heap/mod.rs} | 745 +- .../collections/binary_heap/tests.rs} | 180 +- .../lib/alloc/src/collections/btree/append.rs | 107 + .../lib/alloc/src/collections/btree/borrow.rs | 47 + .../src/collections/btree/borrow/tests.rs | 19 + .../collections/btree/dedup_sorted_iter.rs | 49 + .../lib/alloc/src/collections/btree/fix.rs | 179 + .../lib/alloc/src/collections/btree/map.rs | 2334 +- .../alloc/src/collections/btree/map/entry.rs | 565 + .../alloc/src/collections/btree/map/tests.rs | 2338 + .../lib/alloc/src/collections/btree/mem.rs | 35 + .../alloc/src/collections/btree/merge_iter.rs | 98 + .../lib/alloc/src/collections/btree/mod.rs | 20 +- .../alloc/src/collections/btree/navigate.rs | 700 +- .../lib/alloc/src/collections/btree/node.rs | 1955 +- .../alloc/src/collections/btree/node/tests.rs | 103 + .../lib/alloc/src/collections/btree/remove.rs | 95 + .../lib/alloc/src/collections/btree/search.rs | 324 +- .../lib/alloc/src/collections/btree/set.rs | 903 +- .../collections/btree/set/tests.rs} | 410 +- .../alloc/src/collections/btree/set_val.rs | 29 + .../lib/alloc/src/collections/btree/split.rs | 73 + .../lib/alloc/src/collections/linked_list.rs | 499 +- .../src/collections/linked_list/tests.rs | 777 +- crux-mir/lib/alloc/src/collections/mod.rs | 80 +- .../lib/alloc/src/collections/vec_deque.rs | 2868 - .../alloc/src/collections/vec_deque/drain.rs | 213 +- .../src/collections/vec_deque/into_iter.rs | 76 + .../alloc/src/collections/vec_deque/iter.rs | 181 + .../src/collections/vec_deque/iter_mut.rs | 173 + .../alloc/src/collections/vec_deque/macros.rs | 19 + .../alloc/src/collections/vec_deque/mod.rs | 2909 + .../src/collections/vec_deque/spec_extend.rs | 123 + .../collections/vec_deque/spec_from_iter.rs | 33 + .../alloc/src/collections/vec_deque/tests.rs | 813 +- crux-mir/lib/alloc/src/ffi/c_str.rs | 1146 + crux-mir/lib/alloc/src/ffi/c_str/tests.rs | 228 + crux-mir/lib/alloc/src/ffi/mod.rs | 88 + crux-mir/lib/alloc/src/fmt.rs | 191 +- crux-mir/lib/alloc/src/lib.rs | 209 +- crux-mir/lib/alloc/src/macros.rs | 56 +- crux-mir/lib/alloc/src/prelude/mod.rs | 15 - crux-mir/lib/alloc/src/prelude/v1.rs | 14 - crux-mir/lib/alloc/src/raw_vec.rs | 784 +- crux-mir/lib/alloc/src/raw_vec/tests.rs | 113 +- crux-mir/lib/alloc/src/rc.rs | 1448 +- crux-mir/lib/alloc/src/rc/tests.rs | 143 +- crux-mir/lib/alloc/src/slice.rs | 549 +- crux-mir/lib/alloc/src/slice/tests.rs | 359 + crux-mir/lib/alloc/src/str.rs | 227 +- crux-mir/lib/alloc/src/string.rs | 1052 +- crux-mir/lib/alloc/src/sync.rs | 1529 +- crux-mir/lib/alloc/src/sync/tests.rs | 171 +- crux-mir/lib/alloc/src/task.rs | 153 + crux-mir/lib/alloc/src/testing/crash_test.rs | 119 + crux-mir/lib/alloc/src/testing/mod.rs | 3 + crux-mir/lib/alloc/src/testing/ord_chaos.rs | 81 + crux-mir/lib/alloc/src/testing/rng.rs | 28 + crux-mir/lib/alloc/src/tests.rs | 18 +- crux-mir/lib/alloc/src/vec.rs | 2966 - crux-mir/lib/alloc/src/vec/cow.rs | 53 + crux-mir/lib/alloc/src/vec/drain.rs | 255 + crux-mir/lib/alloc/src/vec/drain_filter.rs | 199 + .../lib/alloc/src/vec/in_place_collect.rs | 308 + crux-mir/lib/alloc/src/vec/in_place_drop.rs | 39 + crux-mir/lib/alloc/src/vec/into_iter.rs | 433 + crux-mir/lib/alloc/src/vec/is_zero.rs | 202 + crux-mir/lib/alloc/src/vec/mod.rs | 3306 + crux-mir/lib/alloc/src/vec/partial_eq.rs | 47 + crux-mir/lib/alloc/src/vec/set_len_on_drop.rs | 33 + crux-mir/lib/alloc/src/vec/spec_extend.rs | 57 + crux-mir/lib/alloc/src/vec/spec_from_elem.rs | 61 + crux-mir/lib/alloc/src/vec/spec_from_iter.rs | 64 + .../alloc/src/vec/spec_from_iter_nested.rs | 65 + crux-mir/lib/alloc/src/vec/splice.rs | 139 + crux-mir/lib/alloc/tests/arc.rs | 17 +- crux-mir/lib/alloc/tests/autotraits.rs | 293 + crux-mir/lib/alloc/tests/borrow.rs | 60 + crux-mir/lib/alloc/tests/boxed.rs | 184 +- crux-mir/lib/alloc/tests/btree/map.rs | 1064 - crux-mir/lib/alloc/tests/btree/mod.rs | 27 - crux-mir/lib/alloc/tests/btree_set_hash.rs | 29 + crux-mir/lib/alloc/tests/c_str.rs | 19 + crux-mir/lib/alloc/tests/const_fns.rs | 35 + crux-mir/lib/alloc/tests/fmt.rs | 319 +- crux-mir/lib/alloc/tests/heap.rs | 13 +- crux-mir/lib/alloc/tests/lib.rs | 53 +- crux-mir/lib/alloc/tests/linked_list.rs | 684 - crux-mir/lib/alloc/tests/rc.rs | 17 +- crux-mir/lib/alloc/tests/slice.rs | 630 +- crux-mir/lib/alloc/tests/str.rs | 562 +- crux-mir/lib/alloc/tests/string.rs | 311 +- crux-mir/lib/alloc/tests/thin_box.rs | 262 + crux-mir/lib/alloc/tests/vec.rs | 1462 +- crux-mir/lib/alloc/tests/vec_deque.rs | 417 +- .../lib/backtrace/.github/workflows/main.yml | 245 + crux-mir/lib/backtrace/Cargo.toml | 131 +- crux-mir/lib/backtrace/README.md | 12 +- crux-mir/lib/backtrace/benches/benchmarks.rs | 2 - crux-mir/lib/backtrace/build.rs | 41 + crux-mir/lib/backtrace/ci/android-sdk.sh | 9 - crux-mir/lib/backtrace/ci/debuglink-docker.sh | 29 + crux-mir/lib/backtrace/ci/debuglink.sh | 75 + .../docker/aarch64-linux-android/Dockerfile | 2 +- .../aarch64-unknown-linux-gnu/Dockerfile | 2 +- .../docker/arm-linux-androideabi/Dockerfile | 25 +- .../arm-unknown-linux-gnueabihf/Dockerfile | 2 +- .../docker/armv7-linux-androideabi/Dockerfile | 2 +- .../armv7-unknown-linux-gnueabihf/Dockerfile | 2 +- .../docker/i586-unknown-linux-gnu/Dockerfile | 2 +- .../ci/docker/i686-linux-android/Dockerfile | 2 +- .../docker/i686-unknown-linux-gnu/Dockerfile | 2 +- .../powerpc64-unknown-linux-gnu/Dockerfile | 2 +- .../docker/s390x-unknown-linux-gnu/Dockerfile | 17 + .../ci/docker/x86_64-linux-android/Dockerfile | 2 +- .../docker/x86_64-pc-windows-gnu/Dockerfile | 2 +- .../x86_64-unknown-linux-gnu/Dockerfile | 5 +- .../x86_64-unknown-linux-musl/Dockerfile | 2 +- crux-mir/lib/backtrace/ci/run-docker.sh | 1 + crux-mir/lib/backtrace/ci/run.sh | 1 + .../lib/backtrace/crates/as-if-std/Cargo.toml | 29 + .../lib/backtrace/crates/as-if-std/build.rs | 3 + .../lib/backtrace/crates/as-if-std/src/lib.rs | 21 + .../backtrace/crates/backtrace-sys/Cargo.toml | 23 - .../backtrace/crates/backtrace-sys/build.rs | 156 - .../backtrace/crates/backtrace-sys/src/lib.rs | 58 - .../crates/cpp_smoke_test/tests/smoke.rs | 1 - .../lib/backtrace/crates/debuglink/Cargo.toml | 7 + .../backtrace/crates/debuglink/src/main.rs | 34 + .../lib/backtrace/crates/dylib-dep/Cargo.toml | 10 + .../lib/backtrace/crates/dylib-dep/src/lib.rs | 14 + .../crates/line-tables-only/Cargo.toml | 21 + .../crates/line-tables-only/build.rs | 10 + .../crates/line-tables-only/src/callback.c | 14 + .../crates/line-tables-only/src/lib.rs | 57 + .../crates/macos_frames_test/Cargo.toml | 8 + .../crates/macos_frames_test/src/lib.rs | 1 + .../crates/macos_frames_test/tests/main.rs | 30 + .../crates/without_debuginfo/Cargo.toml | 13 +- .../crates/without_debuginfo/tests/smoke.rs | 33 +- crux-mir/lib/backtrace/examples/backtrace.rs | 2 - crux-mir/lib/backtrace/examples/raw.rs | 2 - .../backtrace-sys => }/src/android-api.c | 0 .../lib/backtrace/src/backtrace/dbghelp.rs | 88 +- .../lib/backtrace/src/backtrace/libunwind.rs | 224 +- crux-mir/lib/backtrace/src/backtrace/miri.rs | 109 + crux-mir/lib/backtrace/src/backtrace/mod.rs | 40 +- crux-mir/lib/backtrace/src/backtrace/noop.rs | 10 +- .../backtrace/src/backtrace/unix_backtrace.rs | 61 - crux-mir/lib/backtrace/src/capture.rs | 112 +- crux-mir/lib/backtrace/src/dbghelp.rs | 27 +- crux-mir/lib/backtrace/src/lib.rs | 98 +- crux-mir/lib/backtrace/src/print.rs | 73 +- crux-mir/lib/backtrace/src/print/fuchsia.rs | 18 +- .../src/symbolize/coresymbolication.rs | 277 - .../lib/backtrace/src/symbolize/dbghelp.rs | 42 +- .../lib/backtrace/src/symbolize/dladdr.rs | 112 - .../backtrace/src/symbolize/dladdr_resolve.rs | 50 - crux-mir/lib/backtrace/src/symbolize/gimli.rs | 675 +- .../lib/backtrace/src/symbolize/gimli/coff.rs | 108 + .../lib/backtrace/src/symbolize/gimli/elf.rs | 423 + .../symbolize/gimli/libs_dl_iterate_phdr.rs | 53 + .../src/symbolize/gimli/libs_haiku.rs | 48 + .../src/symbolize/gimli/libs_illumos.rs | 99 + .../src/symbolize/gimli/libs_libnx.rs | 27 + .../src/symbolize/gimli/libs_macos.rs | 146 + .../src/symbolize/gimli/libs_windows.rs | 89 + .../backtrace/src/symbolize/gimli/macho.rs | 324 + .../src/symbolize/gimli/mmap_fake.rs | 25 + .../src/symbolize/gimli/mmap_unix.rs | 44 + .../src/symbolize/gimli/mmap_windows.rs | 57 + .../backtrace/src/symbolize/gimli/stash.rs | 52 + .../backtrace/src/symbolize/libbacktrace.rs | 489 - crux-mir/lib/backtrace/src/symbolize/miri.rs | 56 + crux-mir/lib/backtrace/src/symbolize/mod.rs | 107 +- crux-mir/lib/backtrace/src/symbolize/noop.rs | 16 +- crux-mir/lib/backtrace/src/types.rs | 2 +- crux-mir/lib/backtrace/src/windows.rs | 60 +- .../lib/backtrace/tests/accuracy/auxiliary.rs | 1 - crux-mir/lib/backtrace/tests/accuracy/main.rs | 29 +- .../lib/backtrace/tests/concurrent-panics.rs | 8 +- crux-mir/lib/backtrace/tests/long_fn_name.rs | 4 +- .../lib/backtrace/tests/skip_inner_frames.rs | 5 - crux-mir/lib/backtrace/tests/smoke.rs | 169 +- crux-mir/lib/bigint/Cargo.toml | 31 - crux-mir/lib/bigint/LICENSE-MIT | 7 - crux-mir/lib/bigint/README.md | 57 - crux-mir/lib/bigint/benches/bigint.rs | 215 - crux-mir/lib/bigint/examples/modular.rs | 55 - crux-mir/lib/bigint/src/lib.rs | 17 - crux-mir/lib/bigint/src/uint.rs | 2126 - crux-mir/lib/byteorder/io.rs | 1605 - crux-mir/lib/byteorder/lib.rs | 3204 - crux-mir/lib/bytes.rs | 449 - crux-mir/lib/cfg_if/.cargo-ok | 1 + crux-mir/lib/cfg_if/.cargo_vcs_info.json | 5 + .../lib/cfg_if/.github/workflows/main.yml | 56 + crux-mir/lib/cfg_if/.gitignore | 2 + crux-mir/lib/cfg_if/Cargo.toml | 36 + .../Cargo.toml => cfg_if/Cargo.toml.orig} | 2 +- .../lib/{cfg-if => cfg_if}/LICENSE-APACHE | 0 .../backtrace-sys => cfg_if}/LICENSE-MIT | 0 crux-mir/lib/{cfg-if => cfg_if}/README.md | 0 crux-mir/lib/{cfg-if => cfg_if}/src/lib.rs | 12 +- .../lib/{cfg-if => cfg_if}/tests/xcrate.rs | 0 crux-mir/lib/compiler_builtins/.cargo-ok | 1 + .../compiler_builtins/.cargo_vcs_info.json | 6 + crux-mir/lib/compiler_builtins/Cargo.lock | 23 + crux-mir/lib/compiler_builtins/Cargo.toml | 110 +- .../lib/compiler_builtins/Cargo.toml.orig | 79 + .../{LICENSE.TXT => LICENSE.txt} | 0 crux-mir/lib/compiler_builtins/PUBLISHING.md | 16 - crux-mir/lib/compiler_builtins/README.md | 4 +- crux-mir/lib/compiler_builtins/build.rs | 243 +- .../aarch64-unknown-linux-gnu/Dockerfile | 10 - .../arm-unknown-linux-gnueabi/Dockerfile | 9 - .../arm-unknown-linux-gnueabihf/Dockerfile | 9 - .../armv7-unknown-linux-gnueabihf/Dockerfile | 9 - .../docker/i586-unknown-linux-gnu/Dockerfile | 4 - .../docker/i686-unknown-linux-gnu/Dockerfile | 4 - .../docker/mips-unknown-linux-gnu/Dockerfile | 12 - .../mips64-unknown-linux-gnuabi64/Dockerfile | 15 - .../Dockerfile | 14 - .../mipsel-unknown-linux-gnu/Dockerfile | 12 - .../powerpc-unknown-linux-gnu/Dockerfile | 12 - .../powerpc64-unknown-linux-gnu/Dockerfile | 13 - .../powerpc64le-unknown-linux-gnu/Dockerfile | 13 - .../ci/docker/thumbv6m-none-eabi/Dockerfile | 7 - .../ci/docker/thumbv7em-none-eabi/Dockerfile | 7 - .../docker/thumbv7em-none-eabihf/Dockerfile | 7 - .../ci/docker/thumbv7m-none-eabi/Dockerfile | 7 - .../x86_64-unknown-linux-gnu/Dockerfile | 4 - .../lib/compiler_builtins/ci/run-docker.sh | 38 - crux-mir/lib/compiler_builtins/ci/run.sh | 103 - .../crates/panic-handler/Cargo.toml | 6 - .../crates/panic-handler/src/lib.rs | 11 - .../compiler_builtins/examples/intrinsics.rs | 26 +- .../compiler_builtins/libm/src/math/acos.rs | 112 + .../compiler_builtins/libm/src/math/acosf.rs | 79 + .../compiler_builtins/libm/src/math/acosh.rs | 27 + .../compiler_builtins/libm/src/math/acoshf.rs | 26 + .../compiler_builtins/libm/src/math/asin.rs | 119 + .../compiler_builtins/libm/src/math/asinf.rs | 72 + .../compiler_builtins/libm/src/math/asinh.rs | 40 + .../compiler_builtins/libm/src/math/asinhf.rs | 39 + .../compiler_builtins/libm/src/math/atan.rs | 184 + .../compiler_builtins/libm/src/math/atan2.rs | 126 + .../compiler_builtins/libm/src/math/atan2f.rs | 91 + .../compiler_builtins/libm/src/math/atanf.rs | 112 + .../compiler_builtins/libm/src/math/atanh.rs | 37 + .../compiler_builtins/libm/src/math/atanhf.rs | 37 + .../compiler_builtins/libm/src/math/cbrt.rs | 113 + .../compiler_builtins/libm/src/math/cbrtf.rs | 75 + .../compiler_builtins/libm/src/math/ceil.rs | 82 + .../compiler_builtins/libm/src/math/ceilf.rs | 65 + .../libm/src/math/copysign.rs | 12 + .../libm/src/math/copysignf.rs | 12 + .../compiler_builtins/libm/src/math/cos.rs | 73 + .../compiler_builtins/libm/src/math/cosf.rs | 83 + .../compiler_builtins/libm/src/math/cosh.rs | 38 + .../compiler_builtins/libm/src/math/coshf.rs | 38 + .../compiler_builtins/libm/src/math/erf.rs | 318 + .../compiler_builtins/libm/src/math/erff.rs | 230 + .../compiler_builtins/libm/src/math/exp.rs | 154 + .../compiler_builtins/libm/src/math/exp10.rs | 22 + .../compiler_builtins/libm/src/math/exp10f.rs | 22 + .../compiler_builtins/libm/src/math/exp2.rs | 394 + .../compiler_builtins/libm/src/math/exp2f.rs | 135 + .../compiler_builtins/libm/src/math/expf.rs | 101 + .../compiler_builtins/libm/src/math/expm1.rs | 144 + .../compiler_builtins/libm/src/math/expm1f.rs | 134 + .../compiler_builtins/libm/src/math/expo2.rs | 14 + .../compiler_builtins/libm/src/math/fabs.rs | 41 + .../compiler_builtins/libm/src/math/fabsf.rs | 41 + .../compiler_builtins/libm/src/math/fdim.rs | 22 + .../compiler_builtins/libm/src/math/fdimf.rs | 22 + .../compiler_builtins/libm/src/math/fenv.rs | 27 + .../compiler_builtins/libm/src/math/floor.rs | 81 + .../compiler_builtins/libm/src/math/floorf.rs | 66 + .../compiler_builtins/libm/src/math/fma.rs | 243 + .../compiler_builtins/libm/src/math/fmaf.rs | 117 + .../compiler_builtins/libm/src/math/fmax.rs | 12 + .../compiler_builtins/libm/src/math/fmaxf.rs | 12 + .../compiler_builtins/libm/src/math/fmin.rs | 12 + .../compiler_builtins/libm/src/math/fminf.rs | 12 + .../compiler_builtins/libm/src/math/fmod.rs | 80 + .../compiler_builtins/libm/src/math/fmodf.rs | 89 + .../compiler_builtins/libm/src/math/frexp.rs | 20 + .../compiler_builtins/libm/src/math/frexpf.rs | 21 + .../compiler_builtins/libm/src/math/hypot.rs | 74 + .../compiler_builtins/libm/src/math/hypotf.rs | 43 + .../compiler_builtins/libm/src/math/ilogb.rs | 32 + .../compiler_builtins/libm/src/math/ilogbf.rs | 32 + .../lib/compiler_builtins/libm/src/math/j0.rs | 422 + .../compiler_builtins/libm/src/math/j0f.rs | 359 + .../lib/compiler_builtins/libm/src/math/j1.rs | 414 + .../compiler_builtins/libm/src/math/j1f.rs | 380 + .../lib/compiler_builtins/libm/src/math/jn.rs | 343 + .../compiler_builtins/libm/src/math/jnf.rs | 259 + .../compiler_builtins/libm/src/math/k_cos.rs | 62 + .../compiler_builtins/libm/src/math/k_cosf.rs | 29 + .../libm/src/math/k_expo2.rs | 14 + .../libm/src/math/k_expo2f.rs | 14 + .../compiler_builtins/libm/src/math/k_sin.rs | 57 + .../compiler_builtins/libm/src/math/k_sinf.rs | 30 + .../compiler_builtins/libm/src/math/k_tan.rs | 105 + .../compiler_builtins/libm/src/math/k_tanf.rs | 46 + .../compiler_builtins/libm/src/math/ldexp.rs | 4 + .../compiler_builtins/libm/src/math/ldexpf.rs | 4 + .../compiler_builtins/libm/src/math/lgamma.rs | 6 + .../libm/src/math/lgamma_r.rs | 320 + .../libm/src/math/lgammaf.rs | 6 + .../libm/src/math/lgammaf_r.rs | 255 + .../compiler_builtins/libm/src/math/log.rs | 117 + .../compiler_builtins/libm/src/math/log10.rs | 117 + .../compiler_builtins/libm/src/math/log10f.rs | 91 + .../compiler_builtins/libm/src/math/log1p.rs | 143 + .../compiler_builtins/libm/src/math/log1pf.rs | 98 + .../compiler_builtins/libm/src/math/log2.rs | 106 + .../compiler_builtins/libm/src/math/log2f.rs | 87 + .../compiler_builtins/libm/src/math/logf.rs | 65 + .../compiler_builtins/libm/src/math/mod.rs | 370 + .../compiler_builtins/libm/src/math/modf.rs | 34 + .../compiler_builtins/libm/src/math/modff.rs | 33 + .../libm/src/math/nextafter.rs | 37 + .../libm/src/math/nextafterf.rs | 37 + .../compiler_builtins/libm/src/math/pow.rs | 637 + .../compiler_builtins/libm/src/math/powf.rs | 342 + .../libm/src/math/rem_pio2.rs | 233 + .../libm/src/math/rem_pio2_large.rs | 470 + .../libm/src/math/rem_pio2f.rs | 67 + .../libm/src/math/remainder.rs | 5 + .../libm/src/math/remainderf.rs | 5 + .../compiler_builtins/libm/src/math/remquo.rs | 110 + .../libm/src/math/remquof.rs | 97 + .../compiler_builtins/libm/src/math/rint.rs | 48 + .../compiler_builtins/libm/src/math/rintf.rs | 48 + .../compiler_builtins/libm/src/math/round.rs | 28 + .../compiler_builtins/libm/src/math/roundf.rs | 30 + .../compiler_builtins/libm/src/math/scalbn.rs | 33 + .../libm/src/math/scalbnf.rs | 29 + .../compiler_builtins/libm/src/math/sin.rs | 88 + .../compiler_builtins/libm/src/math/sincos.rs | 134 + .../libm/src/math/sincosf.rs | 185 + .../compiler_builtins/libm/src/math/sinf.rs | 93 + .../compiler_builtins/libm/src/math/sinh.rs | 49 + .../compiler_builtins/libm/src/math/sinhf.rs | 30 + .../compiler_builtins/libm/src/math/sqrt.rs | 264 + .../compiler_builtins/libm/src/math/sqrtf.rs | 154 + .../compiler_builtins/libm/src/math/tan.rs | 70 + .../compiler_builtins/libm/src/math/tanf.rs | 78 + .../compiler_builtins/libm/src/math/tanh.rs | 53 + .../compiler_builtins/libm/src/math/tanhf.rs | 39 + .../compiler_builtins/libm/src/math/tgamma.rs | 208 + .../libm/src/math/tgammaf.rs | 6 + .../compiler_builtins/libm/src/math/trunc.rs | 40 + .../compiler_builtins/libm/src/math/truncf.rs | 42 + crux-mir/lib/compiler_builtins/src/arm.rs | 365 +- .../lib/compiler_builtins/src/arm_linux.rs | 133 +- .../lib/compiler_builtins/src/float/add.rs | 5 +- .../lib/compiler_builtins/src/float/cmp.rs | 40 +- .../lib/compiler_builtins/src/float/conv.rs | 492 +- .../lib/compiler_builtins/src/float/div.rs | 37 +- .../lib/compiler_builtins/src/float/mod.rs | 62 +- .../lib/compiler_builtins/src/float/mul.rs | 32 +- .../lib/compiler_builtins/src/float/pow.rs | 46 +- .../lib/compiler_builtins/src/float/trunc.rs | 125 + .../lib/compiler_builtins/src/int/addsub.rs | 84 +- .../src/int/leading_zeros.rs | 149 + crux-mir/lib/compiler_builtins/src/int/mod.rs | 490 +- crux-mir/lib/compiler_builtins/src/int/mul.rs | 172 +- .../lib/compiler_builtins/src/int/sdiv.rs | 240 +- .../lib/compiler_builtins/src/int/shift.rs | 106 +- .../src/int/specialized_div_rem/asymmetric.rs | 69 + .../int/specialized_div_rem/binary_long.rs | 548 + .../src/int/specialized_div_rem/delegate.rs | 319 + .../src/int/specialized_div_rem/mod.rs | 309 + .../src/int/specialized_div_rem/norm_shift.rs | 106 + .../src/int/specialized_div_rem/trifecta.rs | 386 + .../lib/compiler_builtins/src/int/udiv.rs | 286 +- crux-mir/lib/compiler_builtins/src/lib.rs | 31 +- crux-mir/lib/compiler_builtins/src/macros.rs | 257 +- crux-mir/lib/compiler_builtins/src/math.rs | 67 +- crux-mir/lib/compiler_builtins/src/mem.rs | 194 - .../lib/compiler_builtins/src/mem/impls.rs | 281 + crux-mir/lib/compiler_builtins/src/mem/mod.rs | 202 + .../lib/compiler_builtins/src/mem/x86_64.rs | 184 + .../lib/compiler_builtins/src/probestack.rs | 181 +- crux-mir/lib/compiler_builtins/src/riscv.rs | 50 + crux-mir/lib/compiler_builtins/src/riscv32.rs | 18 - crux-mir/lib/compiler_builtins/src/x86.rs | 131 +- crux-mir/lib/compiler_builtins/src/x86_64.rs | 144 +- crux-mir/lib/compiler_builtins/test.c | 11 - .../compiler_builtins/testcrate/Cargo.toml | 28 - .../lib/compiler_builtins/testcrate/build.rs | 1420 - .../compiler_builtins/testcrate/src/lib.rs | 1 - .../testcrate/tests/aeabi_memclr.rs | 61 - .../testcrate/tests/aeabi_memcpy.rs | 72 - .../testcrate/tests/aeabi_memset.rs | 241 - .../testcrate/tests/count_leading_zeros.rs | 25 - .../testcrate/tests/generated.rs | 38 - .../thumbv6m-linux-eabi.json | 28 - .../thumbv7em-linux-eabi.json | 27 - .../thumbv7em-linux-eabihf.json | 28 - .../thumbv7m-linux-eabi.json | 27 - crux-mir/lib/core/Cargo.toml | 35 + crux-mir/lib/core/benches/ascii.rs | 14 +- crux-mir/lib/core/benches/ascii/is_ascii.rs | 82 + crux-mir/lib/core/benches/char/methods.rs | 40 + crux-mir/lib/core/benches/fmt.rs | 40 + crux-mir/lib/core/benches/iter.rs | 96 +- crux-mir/lib/core/benches/lib.rs | 14 + crux-mir/lib/core/benches/num/dec2flt/mod.rs | 1 - crux-mir/lib/core/benches/num/flt2dec/mod.rs | 3 +- .../benches/num/flt2dec/strategy/dragon.rs | 50 +- .../benches/num/flt2dec/strategy/grisu.rs | 52 +- crux-mir/lib/core/benches/num/int_log/mod.rs | 58 + crux-mir/lib/core/benches/num/mod.rs | 1 + crux-mir/lib/core/benches/slice.rs | 100 +- crux-mir/lib/core/benches/str.rs | 10 + crux-mir/lib/core/benches/str/char_count.rs | 107 + crux-mir/lib/core/benches/str/corpora.rs | 88 + .../lib/core/primitive_docs/box_into_raw.md | 1 + crux-mir/lib/core/primitive_docs/fs_file.md | 1 + .../lib/core/primitive_docs/io_bufread.md | 1 + crux-mir/lib/core/primitive_docs/io_read.md | 1 + crux-mir/lib/core/primitive_docs/io_seek.md | 1 + crux-mir/lib/core/primitive_docs/io_write.md | 1 + .../core/primitive_docs/net_tosocketaddrs.md | 1 + .../lib/core/primitive_docs/process_exit.md | 1 + .../lib/core/primitive_docs/string_string.md | 1 + crux-mir/lib/core/src/Cargo.toml | 28 - crux-mir/lib/core/src/alloc.rs | 1007 - crux-mir/lib/core/src/alloc/global.rs | 277 + crux-mir/lib/core/src/alloc/layout.rs | 491 + crux-mir/lib/core/src/alloc/mod.rs | 421 + crux-mir/lib/core/src/any.rs | 841 +- crux-mir/lib/core/src/arch.rs | 30 + crux-mir/lib/core/src/array/equality.rs | 217 + crux-mir/lib/core/src/array/iter.rs | 434 +- crux-mir/lib/core/src/array/mod.rs | 989 +- crux-mir/lib/core/src/ascii.rs | 31 +- crux-mir/lib/core/src/asserting.rs | 109 + .../lib/core/src/async_iter/async_iter.rs | 111 + crux-mir/lib/core/src/async_iter/from_iter.rs | 38 + crux-mir/lib/core/src/async_iter/mod.rs | 128 + crux-mir/lib/core/src/bool.rs | 59 +- crux-mir/lib/core/src/borrow.rs | 33 +- crux-mir/lib/core/src/cell.rs | 828 +- crux-mir/lib/core/src/cell/lazy.rs | 110 + crux-mir/lib/core/src/cell/once.rs | 300 + crux-mir/lib/core/src/char/convert.rs | 265 +- crux-mir/lib/core/src/char/decode.rs | 101 +- crux-mir/lib/core/src/char/methods.rs | 821 +- crux-mir/lib/core/src/char/mod.rs | 155 +- crux-mir/lib/core/src/clone.rs | 62 +- crux-mir/lib/core/src/cmp.rs | 570 +- crux-mir/lib/core/src/const_closure.rs | 78 + crux-mir/lib/core/src/convert/mod.rs | 389 +- crux-mir/lib/core/src/convert/num.rs | 159 +- crux-mir/lib/core/src/default.rs | 79 +- crux-mir/lib/core/src/error.md | 137 + crux-mir/lib/core/src/error.rs | 511 + .../os/raw/char.md => core/src/ffi/c_char.md} | 9 +- .../double.md => core/src/ffi/c_double.md} | 7 +- .../raw/float.md => core/src/ffi/c_float.md} | 5 +- .../os/raw/int.md => core/src/ffi/c_int.md} | 4 +- .../os/raw/long.md => core/src/ffi/c_long.md} | 4 +- .../src/ffi/c_longlong.md} | 4 +- .../raw/schar.md => core/src/ffi/c_schar.md} | 3 +- .../raw/short.md => core/src/ffi/c_short.md} | 3 +- crux-mir/lib/core/src/ffi/c_str.rs | 666 + .../raw/uchar.md => core/src/ffi/c_uchar.md} | 3 +- .../os/raw/uint.md => core/src/ffi/c_uint.md} | 4 +- .../raw/ulong.md => core/src/ffi/c_ulong.md} | 4 +- .../src/ffi/c_ulonglong.md} | 4 +- .../ushort.md => core/src/ffi/c_ushort.md} | 3 +- crux-mir/lib/core/src/ffi/c_void.md | 16 + crux-mir/lib/core/src/{ffi.rs => ffi/mod.rs} | 320 +- crux-mir/lib/core/src/fmt/builders.rs | 107 +- crux-mir/lib/core/src/fmt/float.rs | 183 +- crux-mir/lib/core/src/fmt/mod.rs | 916 +- crux-mir/lib/core/src/fmt/nofloat.rs | 15 + crux-mir/lib/core/src/fmt/num.rs | 464 +- crux-mir/lib/core/src/fmt/rt/v1.rs | 4 + crux-mir/lib/core/src/future/future.rs | 27 +- crux-mir/lib/core/src/future/into_future.rs | 139 + crux-mir/lib/core/src/future/join.rs | 193 + crux-mir/lib/core/src/future/mod.rs | 82 +- crux-mir/lib/core/src/future/pending.rs | 58 + crux-mir/lib/core/src/future/poll_fn.rs | 66 + crux-mir/lib/core/src/future/ready.rs | 70 + crux-mir/lib/core/src/hash/mod.rs | 490 +- crux-mir/lib/core/src/hash/sip.rs | 120 +- crux-mir/lib/core/src/hint.rs | 405 +- crux-mir/lib/core/src/internal_macros.rs | 149 +- crux-mir/lib/core/src/intrinsics.rs | 2005 +- crux-mir/lib/core/src/intrinsics/mir.rs | 545 + .../core/src/iter/adapters/array_chunks.rs | 239 + .../core/src/iter/adapters/by_ref_sized.rs | 100 + crux-mir/lib/core/src/iter/adapters/chain.rs | 330 +- crux-mir/lib/core/src/iter/adapters/cloned.rs | 142 + crux-mir/lib/core/src/iter/adapters/copied.rs | 242 + crux-mir/lib/core/src/iter/adapters/cycle.rs | 108 + .../lib/core/src/iter/adapters/enumerate.rs | 266 + crux-mir/lib/core/src/iter/adapters/filter.rs | 152 + .../lib/core/src/iter/adapters/filter_map.rs | 149 + .../lib/core/src/iter/adapters/flatten.rs | 478 +- crux-mir/lib/core/src/iter/adapters/fuse.rs | 413 + .../lib/core/src/iter/adapters/inspect.rs | 166 + .../lib/core/src/iter/adapters/intersperse.rs | 187 + crux-mir/lib/core/src/iter/adapters/map.rs | 218 + .../lib/core/src/iter/adapters/map_while.rs | 88 + crux-mir/lib/core/src/iter/adapters/mod.rs | 2845 +- .../lib/core/src/iter/adapters/peekable.rs | 335 + crux-mir/lib/core/src/iter/adapters/rev.rs | 137 + crux-mir/lib/core/src/iter/adapters/scan.rs | 98 + crux-mir/lib/core/src/iter/adapters/skip.rs | 240 + .../lib/core/src/iter/adapters/skip_while.rs | 125 + .../lib/core/src/iter/adapters/step_by.rs | 235 + crux-mir/lib/core/src/iter/adapters/take.rs | 251 + .../lib/core/src/iter/adapters/take_while.rs | 126 + crux-mir/lib/core/src/iter/adapters/zip.rs | 478 +- crux-mir/lib/core/src/iter/mod.rs | 220 +- crux-mir/lib/core/src/iter/range.rs | 1195 +- crux-mir/lib/core/src/iter/sources.rs | 656 +- crux-mir/lib/core/src/iter/sources/empty.rs | 89 + crux-mir/lib/core/src/iter/sources/from_fn.rs | 78 + .../core/src/iter/sources/from_generator.rs | 58 + crux-mir/lib/core/src/iter/sources/once.rs | 99 + .../lib/core/src/iter/sources/once_with.rs | 121 + crux-mir/lib/core/src/iter/sources/repeat.rs | 129 + .../lib/core/src/iter/sources/repeat_n.rs | 195 + .../lib/core/src/iter/sources/repeat_with.rs | 123 + .../lib/core/src/iter/sources/successors.rs | 66 + crux-mir/lib/core/src/iter/traits/accum.rs | 114 +- crux-mir/lib/core/src/iter/traits/collect.rs | 160 +- .../lib/core/src/iter/traits/double_ended.rs | 159 +- .../lib/core/src/iter/traits/exact_size.rs | 47 +- crux-mir/lib/core/src/iter/traits/iterator.rs | 1548 +- crux-mir/lib/core/src/iter/traits/marker.rs | 58 +- crux-mir/lib/core/src/iter/traits/mod.rs | 20 +- crux-mir/lib/core/src/lib.rs | 334 +- crux-mir/lib/core/src/macros/mod.rs | 702 +- crux-mir/lib/core/src/macros/panic.md | 70 +- crux-mir/lib/core/src/marker.rs | 356 +- crux-mir/lib/core/src/mem/manually_drop.rs | 114 +- crux-mir/lib/core/src/mem/maybe_uninit.rs | 857 +- crux-mir/lib/core/src/mem/mod.rs | 569 +- crux-mir/lib/core/src/mem/transmutability.rs | 107 + crux-mir/lib/core/src/num/bignum.rs | 103 +- .../lib/core/src/num/dec2flt/algorithm.rs | 417 - crux-mir/lib/core/src/num/dec2flt/common.rs | 198 + crux-mir/lib/core/src/num/dec2flt/decimal.rs | 351 + crux-mir/lib/core/src/num/dec2flt/float.rs | 207 + crux-mir/lib/core/src/num/dec2flt/fpu.rs | 90 + crux-mir/lib/core/src/num/dec2flt/lemire.rs | 166 + crux-mir/lib/core/src/num/dec2flt/mod.rs | 247 +- crux-mir/lib/core/src/num/dec2flt/num.rs | 81 - crux-mir/lib/core/src/num/dec2flt/number.rs | 86 + crux-mir/lib/core/src/num/dec2flt/parse.rs | 305 +- crux-mir/lib/core/src/num/dec2flt/rawfp.rs | 363 - crux-mir/lib/core/src/num/dec2flt/slow.rs | 109 + crux-mir/lib/core/src/num/dec2flt/table.rs | 1937 +- crux-mir/lib/core/src/num/diy_float.rs | 2 +- crux-mir/lib/core/src/num/error.rs | 163 + crux-mir/lib/core/src/num/f32.rs | 901 +- crux-mir/lib/core/src/num/f64.rs | 897 +- crux-mir/lib/core/src/num/flt2dec/decoder.rs | 3 +- crux-mir/lib/core/src/num/flt2dec/mod.rs | 387 +- .../core/src/num/flt2dec/strategy/dragon.rs | 41 +- .../core/src/num/flt2dec/strategy/grisu.rs | 103 +- crux-mir/lib/core/src/num/fmt.rs | 108 + crux-mir/lib/core/src/num/i128.rs | 10 - crux-mir/lib/core/src/num/i16.rs | 10 - crux-mir/lib/core/src/num/i32.rs | 10 - crux-mir/lib/core/src/num/i64.rs | 10 - crux-mir/lib/core/src/num/i8.rs | 10 - crux-mir/lib/core/src/num/int_log10.rs | 140 + crux-mir/lib/core/src/num/int_macros.rs | 2839 +- crux-mir/lib/core/src/num/isize.rs | 10 - crux-mir/lib/core/src/num/mod.rs | 5181 +- crux-mir/lib/core/src/num/nonzero.rs | 1276 + crux-mir/lib/core/src/num/saturating.rs | 1081 + crux-mir/lib/core/src/num/shells/i128.rs | 13 + crux-mir/lib/core/src/num/shells/i16.rs | 13 + crux-mir/lib/core/src/num/shells/i32.rs | 13 + crux-mir/lib/core/src/num/shells/i64.rs | 13 + crux-mir/lib/core/src/num/shells/i8.rs | 13 + .../lib/core/src/num/shells/int_macros.rs | 44 + crux-mir/lib/core/src/num/shells/isize.rs | 13 + crux-mir/lib/core/src/num/shells/u128.rs | 13 + crux-mir/lib/core/src/num/shells/u16.rs | 13 + crux-mir/lib/core/src/num/shells/u32.rs | 13 + crux-mir/lib/core/src/num/shells/u64.rs | 13 + crux-mir/lib/core/src/num/shells/u8.rs | 13 + crux-mir/lib/core/src/num/shells/usize.rs | 13 + crux-mir/lib/core/src/num/u128.rs | 9 - crux-mir/lib/core/src/num/u16.rs | 10 - crux-mir/lib/core/src/num/u32.rs | 10 - crux-mir/lib/core/src/num/u64.rs | 10 - crux-mir/lib/core/src/num/u8.rs | 10 - crux-mir/lib/core/src/num/uint_macros.rs | 2442 + crux-mir/lib/core/src/num/usize.rs | 10 - crux-mir/lib/core/src/num/wrapping.rs | 1116 +- crux-mir/lib/core/src/ops/arith.rs | 232 +- crux-mir/lib/core/src/ops/bit.rs | 292 +- crux-mir/lib/core/src/ops/control_flow.rs | 304 + crux-mir/lib/core/src/ops/deref.rs | 28 +- crux-mir/lib/core/src/ops/drop.rs | 153 +- crux-mir/lib/core/src/ops/function.rs | 77 +- crux-mir/lib/core/src/ops/generator.rs | 5 +- crux-mir/lib/core/src/ops/index.rs | 25 +- crux-mir/lib/core/src/ops/index_range.rs | 171 + crux-mir/lib/core/src/ops/mod.rs | 54 +- crux-mir/lib/core/src/ops/range.rs | 300 +- crux-mir/lib/core/src/ops/try.rs | 57 - crux-mir/lib/core/src/ops/try_trait.rs | 424 + crux-mir/lib/core/src/ops/unsize.rs | 55 +- crux-mir/lib/core/src/option.rs | 1642 +- crux-mir/lib/core/src/panic.rs | 389 +- crux-mir/lib/core/src/panic/location.rs | 200 + crux-mir/lib/core/src/panic/panic_info.rs | 166 + crux-mir/lib/core/src/panic/unwind_safe.rs | 312 + crux-mir/lib/core/src/panicking.rs | 265 +- crux-mir/lib/core/src/pin.rs | 736 +- crux-mir/lib/core/src/prelude/mod.rs | 54 +- crux-mir/lib/core/src/prelude/v1.rs | 49 +- crux-mir/lib/core/src/primitive_docs.rs | 1575 + crux-mir/lib/core/src/ptr/alignment.rs | 332 + crux-mir/lib/core/src/ptr/const_ptr.rs | 1345 +- crux-mir/lib/core/src/ptr/metadata.rs | 273 + crux-mir/lib/core/src/ptr/mod.rs | 1460 +- crux-mir/lib/core/src/ptr/mut_ptr.rs | 1730 +- crux-mir/lib/core/src/ptr/non_null.rs | 651 +- crux-mir/lib/core/src/ptr/unique.rs | 68 +- crux-mir/lib/core/src/raw.rs | 85 - crux-mir/lib/core/src/result.rs | 1462 +- crux-mir/lib/core/src/slice/ascii.rs | 328 + crux-mir/lib/core/src/slice/cmp.rs | 260 + crux-mir/lib/core/src/slice/index.rs | 835 + crux-mir/lib/core/src/slice/iter.rs | 3409 + crux-mir/lib/core/src/slice/iter/macros.rs | 413 + crux-mir/lib/core/src/slice/memchr.rs | 93 +- crux-mir/lib/core/src/slice/mod.rs | 5844 +- crux-mir/lib/core/src/slice/raw.rs | 297 + crux-mir/lib/core/src/slice/rotate.rs | 102 +- crux-mir/lib/core/src/slice/sort.rs | 803 +- crux-mir/lib/core/src/slice/specialize.rs | 23 + crux-mir/lib/core/src/str/converts.rs | 203 + crux-mir/lib/core/src/str/count.rs | 136 + crux-mir/lib/core/src/str/error.rs | 155 + crux-mir/lib/core/src/str/iter.rs | 1516 + crux-mir/lib/core/src/str/lossy.rs | 317 +- crux-mir/lib/core/src/str/mod.rs | 3229 +- crux-mir/lib/core/src/str/pattern.rs | 607 +- crux-mir/lib/core/src/str/traits.rs | 608 + crux-mir/lib/core/src/str/validations.rs | 274 + crux-mir/lib/core/src/sync/atomic.rs | 3319 +- crux-mir/lib/core/src/sync/exclusive.rs | 180 + crux-mir/lib/core/src/sync/mod.rs | 3 + crux-mir/lib/core/src/task/mod.rs | 6 + crux-mir/lib/core/src/task/poll.rs | 238 +- crux-mir/lib/core/src/task/ready.rs | 114 + crux-mir/lib/core/src/task/wake.rs | 135 +- crux-mir/lib/core/src/time.rs | 899 +- crux-mir/lib/core/src/tuple.rs | 202 +- crux-mir/lib/core/src/unicode/mod.rs | 72 +- crux-mir/lib/core/src/unicode/printable.py | 13 +- crux-mir/lib/core/src/unicode/printable.rs | 305 +- crux-mir/lib/core/src/unicode/unicode_data.rs | 1605 +- crux-mir/lib/core/src/unicode/version.rs | 18 - crux-mir/lib/core/src/unit.rs | 2 +- crux-mir/lib/core/tests/alloc.rs | 66 +- crux-mir/lib/core/tests/any.rs | 117 +- crux-mir/lib/core/tests/array.rs | 535 +- crux-mir/lib/core/tests/ascii.rs | 138 +- crux-mir/lib/core/tests/asserting.rs | 37 + crux-mir/lib/core/tests/atomic.rs | 217 +- crux-mir/lib/core/tests/bool.rs | 98 + crux-mir/lib/core/tests/cell.rs | 118 +- crux-mir/lib/core/tests/char.rs | 63 +- crux-mir/lib/core/tests/clone.rs | 4 +- crux-mir/lib/core/tests/cmp.rs | 134 +- crux-mir/lib/core/tests/const_ptr.rs | 101 + crux-mir/lib/core/tests/convert.rs | 16 + crux-mir/lib/core/tests/fmt/builders.rs | 132 +- crux-mir/lib/core/tests/fmt/float.rs | 148 +- crux-mir/lib/core/tests/fmt/mod.rs | 8 +- crux-mir/lib/core/tests/fmt/num.rs | 7 +- crux-mir/lib/core/tests/future.rs | 128 + crux-mir/lib/core/tests/hash/mod.rs | 64 +- crux-mir/lib/core/tests/hash/sip.rs | 41 +- crux-mir/lib/core/tests/intrinsics.rs | 79 + crux-mir/lib/core/tests/iter.rs | 3014 - .../core/tests/iter/adapters/array_chunks.rs | 180 + .../core/tests/iter/adapters/by_ref_sized.rs | 20 + .../lib/core/tests/iter/adapters/chain.rs | 280 + .../lib/core/tests/iter/adapters/cloned.rs | 52 + .../lib/core/tests/iter/adapters/copied.rs | 18 + .../lib/core/tests/iter/adapters/cycle.rs | 31 + .../lib/core/tests/iter/adapters/enumerate.rs | 107 + .../lib/core/tests/iter/adapters/filter.rs | 52 + .../core/tests/iter/adapters/filter_map.rs | 50 + .../lib/core/tests/iter/adapters/flat_map.rs | 74 + .../lib/core/tests/iter/adapters/flatten.rs | 212 + crux-mir/lib/core/tests/iter/adapters/fuse.rs | 75 + .../lib/core/tests/iter/adapters/inspect.rs | 38 + .../core/tests/iter/adapters/intersperse.rs | 154 + crux-mir/lib/core/tests/iter/adapters/map.rs | 27 + crux-mir/lib/core/tests/iter/adapters/mod.rs | 209 + .../lib/core/tests/iter/adapters/peekable.rs | 272 + crux-mir/lib/core/tests/iter/adapters/scan.rs | 20 + crux-mir/lib/core/tests/iter/adapters/skip.rs | 234 + .../core/tests/iter/adapters/skip_while.rs | 50 + .../lib/core/tests/iter/adapters/step_by.rs | 246 + crux-mir/lib/core/tests/iter/adapters/take.rs | 168 + .../core/tests/iter/adapters/take_while.rs | 29 + crux-mir/lib/core/tests/iter/adapters/zip.rs | 315 + crux-mir/lib/core/tests/iter/mod.rs | 102 + crux-mir/lib/core/tests/iter/range.rs | 472 + crux-mir/lib/core/tests/iter/sources.rs | 157 + crux-mir/lib/core/tests/iter/traits/accum.rs | 66 + .../core/tests/iter/traits/double_ended.rs | 91 + .../lib/core/tests/iter/traits/iterator.rs | 593 + crux-mir/lib/core/tests/iter/traits/mod.rs | 4 + crux-mir/lib/core/tests/iter/traits/step.rs | 89 + crux-mir/lib/core/tests/lazy.rs | 144 + crux-mir/lib/core/tests/lib.rs | 134 +- crux-mir/lib/core/tests/macros.rs | 20 + crux-mir/lib/core/tests/manually_drop.rs | 8 + crux-mir/lib/core/tests/mem.rs | 259 +- crux-mir/lib/core/tests/nonzero.rs | 201 +- crux-mir/lib/core/tests/num/bignum.rs | 35 + crux-mir/lib/core/tests/num/const_from.rs | 25 + crux-mir/lib/core/tests/num/dec2flt/float.rs | 33 + crux-mir/lib/core/tests/num/dec2flt/lemire.rs | 53 + crux-mir/lib/core/tests/num/dec2flt/mod.rs | 29 +- crux-mir/lib/core/tests/num/dec2flt/parse.rs | 164 +- crux-mir/lib/core/tests/num/dec2flt/rawfp.rs | 177 - .../lib/core/tests/num/flt2dec/estimator.rs | 8 +- crux-mir/lib/core/tests/num/flt2dec/mod.rs | 328 +- crux-mir/lib/core/tests/num/flt2dec/random.rs | 66 +- .../core/tests/num/flt2dec/strategy/dragon.rs | 1 - .../core/tests/num/flt2dec/strategy/grisu.rs | 2 +- crux-mir/lib/core/tests/num/i128.rs | 1 + crux-mir/lib/core/tests/num/i16.rs | 2 +- crux-mir/lib/core/tests/num/i32.rs | 31 +- crux-mir/lib/core/tests/num/i64.rs | 2 +- crux-mir/lib/core/tests/num/i8.rs | 2 +- crux-mir/lib/core/tests/num/ieee754.rs | 158 + crux-mir/lib/core/tests/num/int_log.rs | 196 + crux-mir/lib/core/tests/num/int_macros.rs | 143 +- crux-mir/lib/core/tests/num/mod.rs | 257 +- crux-mir/lib/core/tests/num/nan.rs | 7 + crux-mir/lib/core/tests/num/ops.rs | 232 + crux-mir/lib/core/tests/num/u128.rs | 1 + crux-mir/lib/core/tests/num/u16.rs | 2 +- crux-mir/lib/core/tests/num/u32.rs | 2 +- crux-mir/lib/core/tests/num/u64.rs | 2 +- crux-mir/lib/core/tests/num/u8.rs | 2 +- crux-mir/lib/core/tests/num/uint_macros.rs | 102 +- crux-mir/lib/core/tests/num/wrapping.rs | 318 + crux-mir/lib/core/tests/ops.rs | 166 +- crux-mir/lib/core/tests/ops/control_flow.rs | 18 + crux-mir/lib/core/tests/option.rs | 247 +- crux-mir/lib/core/tests/panic.rs | 1 + crux-mir/lib/core/tests/panic/location.rs | 31 + crux-mir/lib/core/tests/pin.rs | 31 + crux-mir/lib/core/tests/pin_macro.rs | 33 + crux-mir/lib/core/tests/ptr.rs | 807 +- crux-mir/lib/core/tests/result.rs | 260 +- crux-mir/lib/core/tests/simd.rs | 14 + crux-mir/lib/core/tests/slice.rs | 1033 +- crux-mir/lib/core/tests/str.rs | 2 +- crux-mir/lib/core/tests/str_lossy.rs | 138 +- crux-mir/lib/core/tests/task.rs | 25 + crux-mir/lib/core/tests/time.rs | 232 +- crux-mir/lib/core/tests/tuple.rs | 13 +- crux-mir/lib/core/tests/unicode.rs | 5 + crux-mir/lib/core/tests/waker.rs | 22 + crux-mir/lib/crucible/lib.rs | 1 + crux-mir/lib/crucible/method_spec/mod.rs | 2 +- crux-mir/lib/getopts/Cargo.toml | 25 - crux-mir/lib/getopts/README.md | 15 - crux-mir/lib/getopts/src/lib.rs | 1142 - crux-mir/lib/getopts/src/tests/mod.rs | 1296 - crux-mir/lib/getopts/tests/smoke.rs | 8 - crux-mir/lib/gimli/.cargo-ok | 1 + crux-mir/lib/gimli/.cargo_vcs_info.json | 6 + crux-mir/lib/gimli/.gitignore | 4 + crux-mir/lib/gimli/CHANGELOG.md | 873 + crux-mir/lib/gimli/CONTRIBUTING.md | 137 + crux-mir/lib/gimli/Cargo.toml | 146 + crux-mir/lib/gimli/Cargo.toml.orig | 70 + .../lib/{getopts => gimli}/LICENSE-APACHE | 0 .../lib/{unicode-width => gimli}/LICENSE-MIT | 0 crux-mir/lib/gimli/README.md | 78 + crux-mir/lib/gimli/benches/bench.rs | 807 + crux-mir/lib/gimli/examples/dwarf-validate.rs | 267 + crux-mir/lib/gimli/examples/dwarfdump.rs | 2417 + crux-mir/lib/gimli/examples/simple.rs | 67 + crux-mir/lib/gimli/examples/simple_line.rs | 106 + crux-mir/lib/gimli/fixtures/self/README.md | 147 + crux-mir/lib/gimli/fixtures/self/debug_abbrev | Bin 0 -> 1865 bytes .../lib/gimli/fixtures/self/debug_aranges | Bin 0 -> 16304 bytes crux-mir/lib/gimli/fixtures/self/debug_info | Bin 0 -> 392832 bytes .../lib/gimli/fixtures/self/debug_inlined | Bin 0 -> 25062 bytes crux-mir/lib/gimli/fixtures/self/debug_line | Bin 0 -> 109251 bytes crux-mir/lib/gimli/fixtures/self/debug_loc | Bin 0 -> 283588 bytes .../lib/gimli/fixtures/self/debug_pubnames | Bin 0 -> 138556 bytes .../lib/gimli/fixtures/self/debug_pubtypes | Bin 0 -> 52984 bytes crux-mir/lib/gimli/fixtures/self/debug_ranges | Bin 0 -> 186016 bytes crux-mir/lib/gimli/fixtures/self/debug_str | Bin 0 -> 145794 bytes crux-mir/lib/gimli/fixtures/self/eh_frame | Bin 0 -> 147656 bytes crux-mir/lib/gimli/fixtures/self/eh_frame_hdr | Bin 0 -> 108732 bytes crux-mir/lib/gimli/rustfmt.toml | 0 crux-mir/lib/gimli/src/arch.rs | 603 + crux-mir/lib/gimli/src/common.rs | 363 + crux-mir/lib/gimli/src/constants.rs | 1425 + crux-mir/lib/gimli/src/endianity.rs | 256 + crux-mir/lib/gimli/src/leb128.rs | 612 + crux-mir/lib/gimli/src/lib.rs | 76 + crux-mir/lib/gimli/src/read/abbrev.rs | 996 + crux-mir/lib/gimli/src/read/addr.rs | 128 + crux-mir/lib/gimli/src/read/aranges.rs | 660 + crux-mir/lib/gimli/src/read/cfi.rs | 7585 + crux-mir/lib/gimli/src/read/dwarf.rs | 1143 + crux-mir/lib/gimli/src/read/endian_reader.rs | 639 + crux-mir/lib/gimli/src/read/endian_slice.rs | 350 + crux-mir/lib/gimli/src/read/index.rs | 535 + crux-mir/lib/gimli/src/read/line.rs | 3030 + crux-mir/lib/gimli/src/read/lists.rs | 68 + crux-mir/lib/gimli/src/read/loclists.rs | 1514 + crux-mir/lib/gimli/src/read/lookup.rs | 202 + crux-mir/lib/gimli/src/read/mod.rs | 821 + crux-mir/lib/gimli/src/read/op.rs | 4114 + crux-mir/lib/gimli/src/read/pubnames.rs | 141 + crux-mir/lib/gimli/src/read/pubtypes.rs | 141 + crux-mir/lib/gimli/src/read/reader.rs | 502 + crux-mir/lib/gimli/src/read/rnglists.rs | 1354 + crux-mir/lib/gimli/src/read/str.rs | 321 + crux-mir/lib/gimli/src/read/unit.rs | 6146 + crux-mir/lib/gimli/src/read/util.rs | 250 + crux-mir/lib/gimli/src/read/value.rs | 1621 + crux-mir/lib/gimli/src/test_util.rs | 53 + crux-mir/lib/gimli/src/write/abbrev.rs | 188 + crux-mir/lib/gimli/src/write/cfi.rs | 1025 + crux-mir/lib/gimli/src/write/dwarf.rs | 138 + crux-mir/lib/gimli/src/write/endian_vec.rs | 117 + crux-mir/lib/gimli/src/write/line.rs | 1960 + crux-mir/lib/gimli/src/write/loc.rs | 549 + crux-mir/lib/gimli/src/write/mod.rs | 425 + crux-mir/lib/gimli/src/write/op.rs | 1621 + crux-mir/lib/gimli/src/write/range.rs | 415 + crux-mir/lib/gimli/src/write/section.rs | 172 + crux-mir/lib/gimli/src/write/str.rs | 172 + crux-mir/lib/gimli/src/write/unit.rs | 3157 + crux-mir/lib/gimli/src/write/writer.rs | 497 + crux-mir/lib/gimli/tests/convert_self.rs | 158 + crux-mir/lib/gimli/tests/parse_self.rs | 431 + crux-mir/lib/hashbrown/.cargo-ok | 1 + crux-mir/lib/hashbrown/.cargo_vcs_info.json | 6 + crux-mir/lib/hashbrown/.gitignore | 3 + crux-mir/lib/hashbrown/CHANGELOG.md | 200 +- crux-mir/lib/hashbrown/Cargo.toml | 145 +- crux-mir/lib/hashbrown/Cargo.toml.orig | 60 + crux-mir/lib/hashbrown/README.md | 103 +- crux-mir/lib/hashbrown/benches/bench.rs | 103 +- .../benches/insert_unique_unchecked.rs | 32 + crux-mir/lib/hashbrown/bors.toml | 3 - crux-mir/lib/hashbrown/build.rs | 9 - crux-mir/lib/hashbrown/ci/miri.sh | 15 - crux-mir/lib/hashbrown/ci/run.sh | 47 - crux-mir/lib/hashbrown/ci/tools.sh | 45 - .../src/external_trait_impls/rayon/helpers.rs | 1 + .../src/external_trait_impls/rayon/map.rs | 234 +- .../src/external_trait_impls/rayon/raw.rs | 78 +- .../src/external_trait_impls/rayon/set.rs | 141 +- .../src/external_trait_impls/serde.rs | 1 + crux-mir/lib/hashbrown/src/lib.rs | 53 +- crux-mir/lib/hashbrown/src/macros.rs | 16 + crux-mir/lib/hashbrown/src/map.rs | 6839 +- crux-mir/lib/hashbrown/src/raw/alloc.rs | 73 + crux-mir/lib/hashbrown/src/raw/bitmask.rs | 16 +- crux-mir/lib/hashbrown/src/raw/generic.rs | 35 +- crux-mir/lib/hashbrown/src/raw/mod.rs | 2330 +- crux-mir/lib/hashbrown/src/raw/sse2.rs | 25 +- crux-mir/lib/hashbrown/src/rustc_entry.rs | 83 +- crux-mir/lib/hashbrown/src/scopeguard.rs | 37 +- crux-mir/lib/hashbrown/src/set.rs | 1150 +- crux-mir/lib/hashbrown/tests/rayon.rs | 70 +- crux-mir/lib/hashbrown/tests/serde.rs | 22 +- crux-mir/lib/hashbrown/tests/set.rs | 34 +- crux-mir/lib/int512.rs | 135 - crux-mir/lib/libc/.cargo-ok | 1 + crux-mir/lib/libc/.cargo_vcs_info.json | 6 + crux-mir/lib/libc/.gitignore | 4 + crux-mir/lib/libc/CONTRIBUTING.md | 51 +- crux-mir/lib/libc/Cargo.toml | 74 +- crux-mir/lib/libc/Cargo.toml.orig | 35 + crux-mir/lib/libc/LICENSE-APACHE | 25 - crux-mir/lib/libc/LICENSE-MIT | 2 +- crux-mir/lib/libc/README.md | 28 +- crux-mir/lib/libc/build.rs | 62 +- crux-mir/lib/libc/ci/README.md | 236 - crux-mir/lib/libc/ci/android-install-ndk.sh | 50 - crux-mir/lib/libc/ci/android-install-sdk.sh | 73 - crux-mir/lib/libc/ci/android-sysimage.sh | 56 - crux-mir/lib/libc/ci/azure-install-rust.yml | 82 - crux-mir/lib/libc/ci/azure-master.yml | 22 - crux-mir/lib/libc/ci/azure.yml | 215 - crux-mir/lib/libc/ci/build.sh | 267 - .../docker/aarch64-linux-android/Dockerfile | 45 - .../aarch64-unknown-linux-gnu/Dockerfile | 7 - .../aarch64-unknown-linux-musl/Dockerfile | 15 - .../docker/arm-linux-androideabi/Dockerfile | 45 - .../arm-unknown-linux-gnueabihf/Dockerfile | 7 - .../arm-unknown-linux-musleabihf/Dockerfile | 13 - .../asmjs-unknown-emscripten/Dockerfile | 21 - .../ci/docker/i686-linux-android/Dockerfile | 45 - .../docker/i686-unknown-linux-gnu/Dockerfile | 5 - .../docker/i686-unknown-linux-musl/Dockerfile | 12 - .../docker/mips-unknown-linux-gnu/Dockerfile | 11 - .../docker/mips-unknown-linux-musl/Dockerfile | 24 - .../mips64-unknown-linux-gnuabi64/Dockerfile | 11 - .../mips64-unknown-linux-muslabi64/Dockerfile | 15 - .../Dockerfile | 12 - .../Dockerfile | 15 - .../mipsel-unknown-linux-musl/Dockerfile | 24 - .../powerpc-unknown-linux-gnu/Dockerfile | 11 - .../powerpc64-unknown-linux-gnu/Dockerfile | 11 - .../powerpc64le-unknown-linux-gnu/Dockerfile | 11 - .../docker/s390x-unknown-linux-gnu/Dockerfile | 18 - .../sparc64-unknown-linux-gnu/Dockerfile | 18 - .../wasm32-unknown-emscripten/Dockerfile | 28 - .../wasm32-unknown-emscripten/node-wrapper.sh | 15 - .../lib/libc/ci/docker/wasm32-wasi/Dockerfile | 39 - .../lib/libc/ci/docker/wasm32-wasi/clang.sh | 2 - .../ci/docker/x86_64-linux-android/Dockerfile | 26 - .../docker/x86_64-rumprun-netbsd/Dockerfile | 10 - .../docker/x86_64-rumprun-netbsd/runtest.rs | 55 - .../x86_64-unknown-linux-gnu/Dockerfile | 9 - .../x86_64-unknown-linux-gnux32/Dockerfile | 5 - .../x86_64-unknown-linux-musl/Dockerfile | 10 - crux-mir/lib/libc/ci/dox.sh | 74 - crux-mir/lib/libc/ci/emscripten-entry.sh | 20 - crux-mir/lib/libc/ci/emscripten.sh | 52 - crux-mir/lib/libc/ci/install-musl.sh | 83 - .../ci/ios/deploy_and_run_on_ios_simulator.rs | 175 - crux-mir/lib/libc/ci/linux-s390x.sh | 20 - crux-mir/lib/libc/ci/linux-sparc64.sh | 19 - crux-mir/lib/libc/ci/run-docker.sh | 46 - crux-mir/lib/libc/ci/run-qemu.sh | 34 - crux-mir/lib/libc/ci/run.sh | 98 - crux-mir/lib/libc/ci/runtest-android.rs | 47 - crux-mir/lib/libc/ci/semver.sh | 77 - crux-mir/lib/libc/ci/style.rs | 208 - crux-mir/lib/libc/ci/style.sh | 19 - crux-mir/lib/libc/ci/switch.json | 37 - crux-mir/lib/libc/ci/sysinfo_guard.patch | 10 - crux-mir/lib/libc/ci/test-runner-linux | 25 - crux-mir/lib/libc/libc-test/Cargo.toml | 59 - crux-mir/lib/libc/libc-test/build.rs | 2717 - crux-mir/lib/libc/libc-test/src/cmsg.c | 28 - crux-mir/lib/libc/libc-test/src/errqueue.c | 10 - crux-mir/lib/libc/libc-test/test/cmsg.rs | 100 - crux-mir/lib/libc/libc-test/test/errqueue.rs | 22 - crux-mir/lib/libc/libc-test/test/linux_elf.rs | 12 - .../lib/libc/libc-test/test/linux_fcntl.rs | 12 - .../lib/libc/libc-test/test/linux_ipv6.rs | 12 - .../libc/libc-test/test/linux_strerror_r.rs | 12 - .../lib/libc/libc-test/test/linux_termios.rs | 12 - crux-mir/lib/libc/libc-test/test/main.rs | 6 - crux-mir/lib/libc/rustfmt.toml | 4 +- crux-mir/lib/libc/src/cloudabi/mod.rs | 375 - crux-mir/lib/libc/src/cloudabi/x86.rs | 4 - crux-mir/lib/libc/src/cloudabi/x86_64.rs | 4 - crux-mir/lib/libc/src/fixed_width_ints.rs | 79 + crux-mir/lib/libc/src/fuchsia/mod.rs | 760 +- crux-mir/lib/libc/src/hermit/mod.rs | 10 - crux-mir/lib/libc/src/lib.rs | 53 +- crux-mir/lib/libc/src/macros.rs | 110 +- crux-mir/lib/libc/src/psp.rs | 4174 + .../libc/src/{cloudabi => solid}/aarch64.rs | 4 +- .../lib/libc/src/{cloudabi => solid}/arm.rs | 4 +- crux-mir/lib/libc/src/solid/mod.rs | 904 + .../lib/libc/src/unix/bsd/apple/b32/mod.rs | 4 + .../src/unix/bsd/apple/b64/aarch64/align.rs | 56 + .../src/unix/bsd/apple/b64/aarch64/mod.rs | 14 + .../lib/libc/src/unix/bsd/apple/b64/mod.rs | 12 +- .../src/unix/bsd/apple/b64/x86_64/align.rs | 7 + .../libc/src/unix/bsd/apple/b64/x86_64/mod.rs | 180 + crux-mir/lib/libc/src/unix/bsd/apple/mod.rs | 2513 +- .../unix/bsd/freebsdlike/dragonfly/errno.rs | 1 + .../src/unix/bsd/freebsdlike/dragonfly/mod.rs | 702 +- .../unix/bsd/freebsdlike/freebsd/aarch64.rs | 148 +- .../src/unix/bsd/freebsdlike/freebsd/arm.rs | 3 + .../freebsd/freebsd11/{x86_64.rs => b64.rs} | 0 .../bsd/freebsdlike/freebsd/freebsd11/mod.rs | 272 +- .../bsd/freebsdlike/freebsd/freebsd12/b64.rs | 34 + .../bsd/freebsdlike/freebsd/freebsd12/mod.rs | 311 +- .../freebsdlike/freebsd/freebsd12/x86_64.rs | 39 +- .../bsd/freebsdlike/freebsd/freebsd13/b64.rs | 34 + .../bsd/freebsdlike/freebsd/freebsd13/mod.rs | 553 + .../freebsdlike/freebsd/freebsd13/x86_64.rs | 5 + .../bsd/freebsdlike/freebsd/freebsd14/b64.rs | 34 + .../bsd/freebsdlike/freebsd/freebsd14/mod.rs | 553 + .../freebsdlike/freebsd/freebsd14/x86_64.rs | 5 + .../src/unix/bsd/freebsdlike/freebsd/mod.rs | 3550 +- .../unix/bsd/freebsdlike/freebsd/powerpc.rs | 47 + .../unix/bsd/freebsdlike/freebsd/powerpc64.rs | 3 + .../unix/bsd/freebsdlike/freebsd/riscv64.rs | 154 + .../src/unix/bsd/freebsdlike/freebsd/x86.rs | 5 +- .../bsd/freebsdlike/freebsd/x86_64/align.rs | 190 + .../bsd/freebsdlike/freebsd/x86_64/mod.rs | 310 + .../lib/libc/src/unix/bsd/freebsdlike/mod.rs | 642 +- crux-mir/lib/libc/src/unix/bsd/mod.rs | 255 +- .../lib/libc/src/unix/bsd/netbsdlike/mod.rs | 162 +- .../src/unix/bsd/netbsdlike/netbsd/aarch64.rs | 81 + .../src/unix/bsd/netbsdlike/netbsd/mod.rs | 1163 +- .../src/unix/bsd/netbsdlike/netbsd/x86_64.rs | 17 + .../unix/bsd/netbsdlike/openbsd/aarch64.rs | 14 + .../src/unix/bsd/netbsdlike/openbsd/arm.rs | 16 + .../src/unix/bsd/netbsdlike/openbsd/mips64.rs | 8 + .../src/unix/bsd/netbsdlike/openbsd/mod.rs | 533 +- .../unix/bsd/netbsdlike/openbsd/powerpc.rs | 16 + .../unix/bsd/netbsdlike/openbsd/powerpc64.rs | 16 + .../unix/bsd/netbsdlike/openbsd/riscv64.rs | 16 + .../src/unix/bsd/netbsdlike/openbsd/x86_64.rs | 106 + crux-mir/lib/libc/src/unix/haiku/b32.rs | 17 + crux-mir/lib/libc/src/unix/haiku/b64.rs | 17 + crux-mir/lib/libc/src/unix/haiku/mod.rs | 907 +- crux-mir/lib/libc/src/unix/haiku/native.rs | 1366 + crux-mir/lib/libc/src/unix/haiku/x86_64.rs | 264 + crux-mir/lib/libc/src/unix/hermit/mod.rs | 51 +- .../src/unix/linux_like/android/b32/arm.rs | 18 +- .../src/unix/linux_like/android/b32/mod.rs | 23 +- .../unix/linux_like/android/b32/x86/mod.rs | 29 + .../linux_like/android/b64/aarch64/align.rs | 22 + .../linux_like/android/b64/aarch64/int128.rs | 7 + .../linux_like/android/b64/aarch64/mod.rs | 65 +- .../src/unix/linux_like/android/b64/mod.rs | 64 +- .../linux_like/android/b64/riscv64/align.rs | 7 + .../linux_like/android/b64/riscv64/mod.rs | 342 + .../unix/linux_like/android/b64/x86_64/mod.rs | 126 + .../libc/src/unix/linux_like/android/mod.rs | 1386 +- .../src/unix/linux_like/emscripten/align.rs | 4 +- .../src/unix/linux_like/emscripten/mod.rs | 369 +- .../libc/src/unix/linux_like/linux/align.rs | 46 +- .../unix/linux_like/linux/arch/generic/mod.rs | 287 + .../unix/linux_like/linux/arch/mips/mod.rs | 285 + .../src/unix/linux_like/linux/arch/mod.rs | 15 + .../unix/linux_like/linux/arch/powerpc/mod.rs | 240 + .../unix/linux_like/linux/arch/sparc/mod.rs | 228 + .../linux_like/linux/gnu/b32/arm/align.rs | 46 + .../unix/linux_like/linux/gnu/b32/arm/mod.rs | 153 +- .../linux_like/linux/gnu/b32/m68k/align.rs | 7 + .../unix/linux_like/linux/gnu/b32/m68k/mod.rs | 849 + .../unix/linux_like/linux/gnu/b32/mips/mod.rs | 139 +- .../src/unix/linux_like/linux/gnu/b32/mod.rs | 207 +- .../unix/linux_like/linux/gnu/b32/powerpc.rs | 108 +- .../linux_like/linux/gnu/b32/riscv32/align.rs | 44 + .../linux_like/linux/gnu/b32/riscv32/mod.rs | 775 + .../linux_like/linux/gnu/b32/sparc/mod.rs | 188 +- .../unix/linux_like/linux/gnu/b32/x86/mod.rs | 144 +- .../linux_like/linux/gnu/b64/aarch64/align.rs | 51 + .../linux_like/linux/gnu/b64/aarch64/ilp32.rs | 62 + .../linux/gnu/b64/aarch64/int128.rs | 7 + .../linux_like/linux/gnu/b64/aarch64/lp64.rs | 71 + .../linux_like/linux/gnu/b64/aarch64/mod.rs | 254 +- .../linux/gnu/b64/loongarch64/align.rs | 40 + .../linux/gnu/b64/loongarch64/mod.rs | 862 + .../linux_like/linux/gnu/b64/mips64/mod.rs | 147 +- .../src/unix/linux_like/linux/gnu/b64/mod.rs | 47 +- .../linux_like/linux/gnu/b64/powerpc64/mod.rs | 140 +- .../linux_like/linux/gnu/b64/riscv64/align.rs | 44 + .../linux_like/linux/gnu/b64/riscv64/mod.rs | 310 +- .../unix/linux_like/linux/gnu/b64/s390x.rs | 135 +- .../linux_like/linux/gnu/b64/sparc64/mod.rs | 128 +- .../linux_like/linux/gnu/b64/x86_64/align.rs | 17 + .../linux_like/linux/gnu/b64/x86_64/mod.rs | 172 +- .../linux/gnu/b64/x86_64/not_x32.rs | 29 +- .../linux_like/linux/gnu/b64/x86_64/x32.rs | 28 + .../libc/src/unix/linux_like/linux/gnu/mod.rs | 830 +- .../lib/libc/src/unix/linux_like/linux/mod.rs | 2166 +- .../unix/linux_like/linux/musl/b32/arm/mod.rs | 197 +- .../unix/linux_like/linux/musl/b32/hexagon.rs | 170 +- .../linux_like/linux/musl/b32/mips/mod.rs | 130 +- .../src/unix/linux_like/linux/musl/b32/mod.rs | 11 +- .../unix/linux_like/linux/musl/b32/powerpc.rs | 118 +- .../linux/musl/b32/riscv32/align.rs | 7 + .../linux_like/linux/musl/b32/riscv32/mod.rs | 808 + .../unix/linux_like/linux/musl/b32/x86/mod.rs | 138 +- .../linux/musl/b64/aarch64/align.rs | 35 + .../linux/musl/b64/aarch64/int128.rs | 7 + .../linux_like/linux/musl/b64/aarch64/mod.rs | 163 +- .../unix/linux_like/linux/musl/b64/mips64.rs | 145 +- .../src/unix/linux_like/linux/musl/b64/mod.rs | 19 +- .../linux_like/linux/musl/b64/powerpc64.rs | 109 +- .../linux/musl/b64/riscv64/align.rs | 44 + .../linux_like/linux/musl/b64/riscv64/mod.rs | 742 + .../unix/linux_like/linux/musl/b64/s390x.rs | 721 + .../linux_like/linux/musl/b64/x86_64/align.rs | 18 + .../linux_like/linux/musl/b64/x86_64/mod.rs | 129 +- .../src/unix/linux_like/linux/musl/mod.rs | 467 +- .../src/unix/linux_like/linux/no_align.rs | 19 + .../unix/linux_like/linux/non_exhaustive.rs | 9 + .../src/unix/linux_like/linux/uclibc/align.rs | 28 + .../linux}/uclibc/arm/align.rs | 0 .../{ => linux_like/linux}/uclibc/arm/mod.rs | 367 +- .../linux}/uclibc/arm/no_align.rs | 0 .../linux}/uclibc/mips/mips32/align.rs | 0 .../linux}/uclibc/mips/mips32/mod.rs | 54 +- .../linux}/uclibc/mips/mips32/no_align.rs | 0 .../linux}/uclibc/mips/mips64/align.rs | 0 .../linux}/uclibc/mips/mips64/mod.rs | 8 - .../linux}/uclibc/mips/mips64/no_align.rs | 0 .../{ => linux_like/linux}/uclibc/mips/mod.rs | 187 - .../src/unix/linux_like/linux/uclibc/mod.rs | 392 + .../{ => linux_like/linux}/uclibc/no_align.rs | 0 .../linux}/uclibc/x86_64/l4re.rs | 5 + .../linux}/uclibc/x86_64/mod.rs | 98 +- .../linux}/uclibc/x86_64/other.rs | 0 crux-mir/lib/libc/src/unix/linux_like/mod.rs | 698 +- crux-mir/lib/libc/src/unix/mod.rs | 609 +- .../lib/libc/src/unix/newlib/aarch64/mod.rs | 21 + crux-mir/lib/libc/src/unix/newlib/arm/mod.rs | 23 +- .../lib/libc/src/unix/newlib/espidf/mod.rs | 110 + crux-mir/lib/libc/src/unix/newlib/generic.rs | 27 + .../lib/libc/src/unix/newlib/horizon/mod.rs | 268 + crux-mir/lib/libc/src/unix/newlib/mod.rs | 246 +- crux-mir/lib/libc/src/unix/newlib/no_align.rs | 2 +- .../lib/libc/src/unix/newlib/powerpc/mod.rs | 16 + crux-mir/lib/libc/src/unix/redox/mod.rs | 375 +- crux-mir/lib/libc/src/unix/solarish/compat.rs | 224 +- .../lib/libc/src/unix/solarish/illumos.rs | 88 + crux-mir/lib/libc/src/unix/solarish/mod.rs | 1231 +- .../lib/libc/src/unix/solarish/solaris.rs | 101 + crux-mir/lib/libc/src/unix/solarish/x86.rs | 29 + crux-mir/lib/libc/src/unix/solarish/x86_64.rs | 190 + .../lib/libc/src/unix/solarish/x86_common.rs | 65 + crux-mir/lib/libc/src/unix/uclibc/align.rs | 66 - crux-mir/lib/libc/src/unix/uclibc/mod.rs | 2295 - .../lib/libc/src/unix/uclibc/x86_64/align.rs | 77 - .../libc/src/unix/uclibc/x86_64/no_align.rs | 59 - crux-mir/lib/libc/src/vxworks/mod.rs | 505 +- crux-mir/lib/libc/src/wasi.rs | 505 +- crux-mir/lib/libc/src/windows/gnu/mod.rs | 10 +- crux-mir/lib/libc/src/windows/mod.rs | 233 +- .../libc/src/windows/{msvc.rs => msvc/mod.rs} | 13 +- crux-mir/lib/memchr/.cargo-ok | 1 + crux-mir/lib/memchr/.cargo_vcs_info.json | 6 + crux-mir/lib/memchr/.gitignore | 10 + crux-mir/lib/memchr/.ignore | 1 + crux-mir/lib/memchr/COPYING | 3 + crux-mir/lib/memchr/Cargo.toml | 78 + crux-mir/lib/memchr/Cargo.toml.orig | 57 + .../lib/{byteorder => memchr}/LICENSE-MIT | 0 crux-mir/lib/memchr/README.md | 107 + crux-mir/lib/memchr/UNLICENSE | 24 + crux-mir/lib/memchr/build.rs | 88 + crux-mir/lib/memchr/rustfmt.toml | 2 + .../memchr/scripts/make-byte-frequency-table | 74 + crux-mir/lib/memchr/src/cow.rs | 97 + crux-mir/lib/memchr/src/lib.rs | 181 + crux-mir/lib/memchr/src/memchr/c.rs | 44 + crux-mir/lib/memchr/src/memchr/fallback.rs | 329 + crux-mir/lib/memchr/src/memchr/iter.rs | 173 + crux-mir/lib/memchr/src/memchr/mod.rs | 410 + crux-mir/lib/memchr/src/memchr/naive.rs | 25 + crux-mir/lib/memchr/src/memchr/x86/avx.rs | 755 + crux-mir/lib/memchr/src/memchr/x86/mod.rs | 148 + crux-mir/lib/memchr/src/memchr/x86/sse2.rs | 791 + crux-mir/lib/memchr/src/memchr/x86/sse42.rs | 72 + .../lib/memchr/src/memmem/byte_frequencies.rs | 258 + crux-mir/lib/memchr/src/memmem/genericsimd.rs | 266 + crux-mir/lib/memchr/src/memmem/mod.rs | 1321 + .../memchr/src/memmem/prefilter/fallback.rs | 122 + .../src/memmem/prefilter/genericsimd.rs | 207 + .../lib/memchr/src/memmem/prefilter/mod.rs | 570 + .../lib/memchr/src/memmem/prefilter/wasm.rs | 39 + .../memchr/src/memmem/prefilter/x86/avx.rs | 46 + .../memchr/src/memmem/prefilter/x86/mod.rs | 5 + .../memchr/src/memmem/prefilter/x86/sse.rs | 42 + crux-mir/lib/memchr/src/memmem/rabinkarp.rs | 233 + crux-mir/lib/memchr/src/memmem/rarebytes.rs | 136 + crux-mir/lib/memchr/src/memmem/twoway.rs | 878 + crux-mir/lib/memchr/src/memmem/util.rs | 88 + crux-mir/lib/memchr/src/memmem/vector.rs | 131 + crux-mir/lib/memchr/src/memmem/wasm.rs | 75 + crux-mir/lib/memchr/src/memmem/x86/avx.rs | 139 + crux-mir/lib/memchr/src/memmem/x86/mod.rs | 2 + crux-mir/lib/memchr/src/memmem/x86/sse.rs | 89 + crux-mir/lib/memchr/src/tests/memchr/iter.rs | 230 + .../lib/memchr/src/tests/memchr/memchr.rs | 134 + crux-mir/lib/memchr/src/tests/memchr/mod.rs | 7 + .../lib/memchr/src/tests/memchr/simple.rs | 23 + .../lib/memchr/src/tests/memchr/testdata.rs | 351 + crux-mir/lib/memchr/src/tests/mod.rs | 15 + .../memchr/src/tests/x86_64-soft_float.json | 15 + crux-mir/lib/miniz_oxide/.cargo-ok | 1 + crux-mir/lib/miniz_oxide/.cargo_vcs_info.json | 6 + crux-mir/lib/miniz_oxide/Cargo.toml | 55 + crux-mir/lib/miniz_oxide/Cargo.toml.orig | 36 + crux-mir/lib/miniz_oxide/LICENSE | 21 + crux-mir/lib/miniz_oxide/LICENSE-APACHE.md | 177 + crux-mir/lib/miniz_oxide/LICENSE-MIT.md | 21 + crux-mir/lib/miniz_oxide/LICENSE-ZLIB.md | 11 + crux-mir/lib/miniz_oxide/Readme.md | 35 + .../lib/miniz_oxide/src/deflate/buffer.rs | 58 + crux-mir/lib/miniz_oxide/src/deflate/core.rs | 2463 + crux-mir/lib/miniz_oxide/src/deflate/mod.rs | 227 + .../lib/miniz_oxide/src/deflate/stream.rs | 121 + crux-mir/lib/miniz_oxide/src/inflate/core.rs | 1931 + crux-mir/lib/miniz_oxide/src/inflate/mod.rs | 279 + .../miniz_oxide/src/inflate/output_buffer.rs | 60 + .../lib/miniz_oxide/src/inflate/stream.rs | 415 + crux-mir/lib/miniz_oxide/src/lib.rs | 208 + crux-mir/lib/miniz_oxide/src/shared.rs | 25 + crux-mir/lib/object/.cargo-ok | 1 + crux-mir/lib/object/.cargo_vcs_info.json | 6 + crux-mir/lib/object/.gitignore | 2 + crux-mir/lib/object/.gitmodules | 3 + crux-mir/lib/object/CHANGELOG.md | 408 + crux-mir/lib/object/Cargo.toml | 145 + crux-mir/lib/object/Cargo.toml.orig | 98 + .../{rustc-demangle => object}/LICENSE-APACHE | 0 crux-mir/lib/object/LICENSE-MIT | 25 + crux-mir/lib/object/README.md | 58 + crux-mir/lib/object/clippy.toml | 1 + crux-mir/lib/object/src/archive.rs | 39 + crux-mir/lib/object/src/common.rs | 450 + crux-mir/lib/object/src/elf.rs | 6404 + crux-mir/lib/object/src/endian.rs | 771 + crux-mir/lib/object/src/lib.rs | 112 + crux-mir/lib/object/src/macho.rs | 3307 + crux-mir/lib/object/src/pe.rs | 3035 + crux-mir/lib/object/src/pod.rs | 238 + crux-mir/lib/object/src/read/any.rs | 1251 + crux-mir/lib/object/src/read/archive.rs | 502 + crux-mir/lib/object/src/read/coff/comdat.rs | 167 + crux-mir/lib/object/src/read/coff/file.rs | 247 + crux-mir/lib/object/src/read/coff/mod.rs | 18 + .../lib/object/src/read/coff/relocation.rs | 91 + crux-mir/lib/object/src/read/coff/section.rs | 524 + crux-mir/lib/object/src/read/coff/symbol.rs | 518 + crux-mir/lib/object/src/read/elf/comdat.rs | 162 + .../lib/object/src/read/elf/compression.rs | 56 + crux-mir/lib/object/src/read/elf/dynamic.rs | 117 + crux-mir/lib/object/src/read/elf/file.rs | 882 + crux-mir/lib/object/src/read/elf/hash.rs | 220 + crux-mir/lib/object/src/read/elf/mod.rs | 39 + crux-mir/lib/object/src/read/elf/note.rs | 185 + .../lib/object/src/read/elf/relocation.rs | 539 + crux-mir/lib/object/src/read/elf/section.rs | 1090 + crux-mir/lib/object/src/read/elf/segment.rs | 333 + crux-mir/lib/object/src/read/elf/symbol.rs | 579 + crux-mir/lib/object/src/read/elf/version.rs | 421 + .../lib/object/src/read/macho/dyld_cache.rs | 343 + crux-mir/lib/object/src/read/macho/fat.rs | 120 + crux-mir/lib/object/src/read/macho/file.rs | 729 + .../lib/object/src/read/macho/load_command.rs | 348 + crux-mir/lib/object/src/read/macho/mod.rs | 30 + .../lib/object/src/read/macho/relocation.rs | 126 + crux-mir/lib/object/src/read/macho/section.rs | 384 + crux-mir/lib/object/src/read/macho/segment.rs | 303 + crux-mir/lib/object/src/read/macho/symbol.rs | 488 + crux-mir/lib/object/src/read/mod.rs | 710 + .../lib/object/src/read/pe/data_directory.rs | 185 + crux-mir/lib/object/src/read/pe/export.rs | 331 + crux-mir/lib/object/src/read/pe/file.rs | 1018 + crux-mir/lib/object/src/read/pe/import.rs | 218 + crux-mir/lib/object/src/read/pe/mod.rs | 34 + crux-mir/lib/object/src/read/pe/relocation.rs | 90 + crux-mir/lib/object/src/read/pe/resource.rs | 195 + crux-mir/lib/object/src/read/pe/rich.rs | 91 + crux-mir/lib/object/src/read/pe/section.rs | 436 + crux-mir/lib/object/src/read/read_cache.rs | 185 + crux-mir/lib/object/src/read/read_ref.rs | 137 + crux-mir/lib/object/src/read/traits.rs | 469 + crux-mir/lib/object/src/read/util.rs | 383 + crux-mir/lib/object/src/read/wasm.rs | 908 + crux-mir/lib/object/src/write/coff.rs | 712 + crux-mir/lib/object/src/write/elf/mod.rs | 9 + crux-mir/lib/object/src/write/elf/object.rs | 774 + crux-mir/lib/object/src/write/elf/writer.rs | 1955 + crux-mir/lib/object/src/write/macho.rs | 858 + crux-mir/lib/object/src/write/mod.rs | 918 + crux-mir/lib/object/src/write/pe.rs | 847 + crux-mir/lib/object/src/write/string.rs | 159 + crux-mir/lib/object/src/write/util.rs | 210 + crux-mir/lib/object/tests/integration.rs | 2 + crux-mir/lib/object/tests/parse_self.rs | 25 + crux-mir/lib/object/tests/read/coff.rs | 23 + crux-mir/lib/object/tests/read/mod.rs | 3 + crux-mir/lib/object/tests/round_trip/bss.rs | 255 + crux-mir/lib/object/tests/round_trip/coff.rs | 56 + .../lib/object/tests/round_trip/comdat.rs | 225 + .../lib/object/tests/round_trip/common.rs | 245 + crux-mir/lib/object/tests/round_trip/elf.rs | 218 + crux-mir/lib/object/tests/round_trip/macho.rs | 24 + crux-mir/lib/object/tests/round_trip/mod.rs | 448 + .../object/tests/round_trip/section_flags.rs | 90 + crux-mir/lib/object/tests/round_trip/tls.rs | 316 + crux-mir/lib/panic_abort/Cargo.toml | 11 +- crux-mir/lib/panic_abort/lib.rs | 126 - crux-mir/lib/panic_abort/src/android.rs | 49 + crux-mir/lib/panic_abort/src/lib.rs | 142 +- crux-mir/lib/panic_unwind/Cargo.toml | 15 +- crux-mir/lib/panic_unwind/{ => src}/dummy.rs | 4 +- crux-mir/lib/panic_unwind/{ => src}/emcc.rs | 85 +- crux-mir/lib/panic_unwind/src/gcc.rs | 113 + crux-mir/lib/panic_unwind/{ => src}/hermit.rs | 0 crux-mir/lib/panic_unwind/{ => src}/lib.rs | 56 +- crux-mir/lib/panic_unwind/{ => src}/miri.rs | 7 +- crux-mir/lib/panic_unwind/{ => src}/seh.rs | 72 +- .../.github/ISSUE_TEMPLATE/blank_issue.md | 4 + .../.github/ISSUE_TEMPLATE/bug_report.md | 50 + .../.github/ISSUE_TEMPLATE/config.yml | 10 + .../.github/ISSUE_TEMPLATE/feature_request.md | 14 + .../.github/PULL_REQUEST_TEMPLATE.md | 18 + .../portable-simd/.github/workflows/ci.yml | 260 + .../portable-simd/.github/workflows/doc.yml | 30 + crux-mir/lib/portable-simd/CONTRIBUTING.md | 32 + crux-mir/lib/portable-simd/Cargo.toml | 7 + crux-mir/lib/portable-simd/LICENSE-APACHE | 202 + crux-mir/lib/portable-simd/LICENSE-MIT | 19 + crux-mir/lib/portable-simd/README.md | 69 + crux-mir/lib/portable-simd/beginners-guide.md | 91 + .../portable-simd/crates/core_simd/Cargo.toml | 32 + .../crates/core_simd/LICENSE-APACHE | 202 + .../crates/core_simd/LICENSE-MIT | 19 + .../core_simd/examples/matrix_inversion.rs | 316 + .../crates/core_simd/examples/nbody.rs | 193 + .../core_simd/examples/spectral_norm.rs | 77 + .../crates/core_simd/src/core_simd_docs.md | 4 + .../crates/core_simd/src/elements.rs | 11 + .../crates/core_simd/src/elements/float.rs | 357 + .../crates/core_simd/src/elements/int.rs | 298 + .../crates/core_simd/src/elements/uint.rs | 139 + .../portable-simd/crates/core_simd/src/eq.rs | 73 + .../portable-simd/crates/core_simd/src/fmt.rs | 39 + .../crates/core_simd/src/intrinsics.rs | 153 + .../crates/core_simd/src/iter.rs | 58 + .../crates/core_simd/src/lane_count.rs | 46 + .../portable-simd/crates/core_simd/src/lib.rs | 22 + .../crates/core_simd/src/masks.rs | 595 + .../crates/core_simd/src/masks/bitmask.rs | 246 + .../crates/core_simd/src/masks/full_masks.rs | 323 + .../crates/core_simd/src/masks/to_bitmask.rs | 93 + .../portable-simd/crates/core_simd/src/mod.rs | 32 + .../portable-simd/crates/core_simd/src/ops.rs | 254 + .../crates/core_simd/src/ops/assign.rs | 124 + .../crates/core_simd/src/ops/deref.rs | 124 + .../crates/core_simd/src/ops/unary.rs | 78 + .../portable-simd/crates/core_simd/src/ord.rs | 213 + .../crates/core_simd/src/select.rs | 59 + .../crates/core_simd/src/swizzle.rs | 385 + .../crates/core_simd/src/to_bytes.rs | 41 + .../crates/core_simd/src/vector.rs | 742 + .../crates/core_simd/src/vector/float.rs | 24 + .../crates/core_simd/src/vector/int.rs | 63 + .../crates/core_simd/src/vector/ptr.rs | 51 + .../crates/core_simd/src/vector/uint.rs | 63 + .../crates/core_simd/src/vendor.rs | 31 + .../crates/core_simd/src/vendor/arm.rs | 76 + .../crates/core_simd/src/vendor/powerpc.rs | 11 + .../crates/core_simd/src/vendor/wasm32.rs | 30 + .../crates/core_simd/src/vendor/x86.rs | 63 + .../crates/core_simd/tests/autoderef.rs | 22 + .../crates/core_simd/tests/cast.rs | 37 + .../crates/core_simd/tests/f32_ops.rs | 5 + .../crates/core_simd/tests/f64_ops.rs | 5 + .../crates/core_simd/tests/i16_ops.rs | 5 + .../crates/core_simd/tests/i32_ops.rs | 5 + .../crates/core_simd/tests/i64_ops.rs | 5 + .../crates/core_simd/tests/i8_ops.rs | 5 + .../crates/core_simd/tests/isize_ops.rs | 5 + .../crates/core_simd/tests/mask_ops.rs | 3 + .../core_simd/tests/mask_ops_impl/mask16.rs | 4 + .../core_simd/tests/mask_ops_impl/mask32.rs | 4 + .../core_simd/tests/mask_ops_impl/mask64.rs | 3 + .../core_simd/tests/mask_ops_impl/mask8.rs | 3 + .../tests/mask_ops_impl/mask_macros.rs | 225 + .../core_simd/tests/mask_ops_impl/masksize.rs | 3 + .../core_simd/tests/mask_ops_impl/mod.rs | 9 + .../crates/core_simd/tests/masks.rs | 158 + .../crates/core_simd/tests/ops_macros.rs | 607 + .../crates/core_simd/tests/round.rs | 85 + .../crates/core_simd/tests/swizzle.rs | 62 + .../crates/core_simd/tests/to_bytes.rs | 14 + .../crates/core_simd/tests/u16_ops.rs | 5 + .../crates/core_simd/tests/u32_ops.rs | 5 + .../crates/core_simd/tests/u64_ops.rs | 5 + .../crates/core_simd/tests/u8_ops.rs | 5 + .../crates/core_simd/tests/usize_ops.rs | 5 + .../crates/core_simd/webdriver.json | 7 + .../portable-simd/crates/std_float/Cargo.toml | 13 + .../portable-simd/crates/std_float/src/lib.rs | 161 + .../crates/test_helpers/Cargo.toml | 10 + .../crates/test_helpers/src/array.rs | 97 + .../crates/test_helpers/src/biteq.rs | 106 + .../crates/test_helpers/src/lib.rs | 463 + .../crates/test_helpers/src/wasm.rs | 51 + crux-mir/lib/proc_macro/Cargo.toml | 11 + crux-mir/lib/proc_macro/src/bridge/arena.rs | 113 + crux-mir/lib/proc_macro/src/bridge/buffer.rs | 156 + crux-mir/lib/proc_macro/src/bridge/client.rs | 508 + crux-mir/lib/proc_macro/src/bridge/closure.rs | 32 + crux-mir/lib/proc_macro/src/bridge/fxhash.rs | 117 + crux-mir/lib/proc_macro/src/bridge/handle.rs | 75 + crux-mir/lib/proc_macro/src/bridge/mod.rs | 521 + crux-mir/lib/proc_macro/src/bridge/rpc.rs | 304 + .../lib/proc_macro/src/bridge/scoped_cell.rs | 81 + .../proc_macro/src/bridge/selfless_reify.rs | 84 + crux-mir/lib/proc_macro/src/bridge/server.rs | 372 + crux-mir/lib/proc_macro/src/bridge/symbol.rs | 205 + crux-mir/lib/proc_macro/src/diagnostic.rs | 175 + crux-mir/lib/proc_macro/src/lib.rs | 1521 + crux-mir/lib/proc_macro/src/quote.rs | 141 + crux-mir/lib/rustc-demangle/Cargo.toml | 12 - crux-mir/lib/rustc-demangle/README.md | 15 - crux-mir/lib/rustc-demangle/src/lib.rs | 350 - crux-mir/lib/rustc_demangle/.cargo-ok | 1 + .../lib/rustc_demangle/.cargo_vcs_info.json | 5 + .../lib/rustc_demangle/.github/dependabot.yml | 8 + .../rustc_demangle/.github/workflows/main.yml | 54 + crux-mir/lib/rustc_demangle/.gitignore | 2 + crux-mir/lib/rustc_demangle/Cargo.toml | 35 + crux-mir/lib/rustc_demangle/Cargo.toml.orig | 25 + .../LICENSE-APACHE | 0 .../{cfg-if => rustc_demangle}/LICENSE-MIT | 0 crux-mir/lib/rustc_demangle/README.md | 48 + crux-mir/lib/rustc_demangle/src/legacy.rs | 392 + crux-mir/lib/rustc_demangle/src/lib.rs | 493 + .../early-recursion-limit | 10 + crux-mir/lib/rustc_demangle/src/v0.rs | 1530 + .../lib/rustc_std_workspace_alloc/Cargo.toml | 14 + crux-mir/lib/rustc_std_workspace_alloc/lib.rs | 9 + .../lib/rustc_std_workspace_core/Cargo.toml | 14 + .../lib/rustc_std_workspace_core/README.md | 29 + crux-mir/lib/rustc_std_workspace_core/lib.rs | 4 + crux-mir/lib/std/Cargo.toml | 87 + crux-mir/lib/std/build.rs | 52 + .../lib/std/primitive_docs/box_into_raw.md | 1 + crux-mir/lib/std/primitive_docs/fs_file.md | 1 + crux-mir/lib/std/primitive_docs/io_bufread.md | 1 + crux-mir/lib/std/primitive_docs/io_read.md | 1 + crux-mir/lib/std/primitive_docs/io_seek.md | 1 + crux-mir/lib/std/primitive_docs/io_write.md | 1 + .../std/primitive_docs/net_tosocketaddrs.md | 1 + .../lib/std/primitive_docs/process_exit.md | 1 + .../lib/std/primitive_docs/string_string.md | 1 + crux-mir/lib/std/src/Cargo.toml | 77 - crux-mir/lib/std/src/alloc.rs | 263 +- crux-mir/lib/std/src/ascii.rs | 53 +- crux-mir/lib/std/src/backtrace.rs | 215 +- crux-mir/lib/std/src/backtrace/tests.rs | 95 + crux-mir/lib/std/src/build.rs | 58 - .../std/src/collections/hash/crucible_map.rs | 408 - crux-mir/lib/std/src/collections/hash/map.rs | 2729 +- .../lib/std/src/collections/hash/map/tests.rs | 1124 + crux-mir/lib/std/src/collections/hash/mod.rs | 2 - crux-mir/lib/std/src/collections/hash/set.rs | 1110 +- .../lib/std/src/collections/hash/set/tests.rs | 504 + crux-mir/lib/std/src/collections/mod.rs | 58 +- crux-mir/lib/std/src/env.rs | 516 +- crux-mir/lib/std/src/env/tests.rs | 102 + crux-mir/lib/std/src/error.rs | 1224 +- crux-mir/lib/std/src/error/tests.rs | 443 + crux-mir/lib/std/src/f32.rs | 913 +- crux-mir/lib/std/src/f32/tests.rs | 857 + crux-mir/lib/std/src/f64.rs | 902 +- crux-mir/lib/std/src/f64/tests.rs | 839 + crux-mir/lib/std/src/ffi/c_str.rs | 1536 - crux-mir/lib/std/src/ffi/mod.rs | 110 +- crux-mir/lib/std/src/ffi/os_str.rs | 751 +- crux-mir/lib/std/src/ffi/os_str/tests.rs | 179 + crux-mir/lib/std/src/fs.rs | 2089 +- crux-mir/lib/std/src/fs/tests.rs | 1597 + crux-mir/lib/std/src/future.rs | 111 - crux-mir/lib/std/src/io/buffered.rs | 1702 - crux-mir/lib/std/src/io/buffered/bufreader.rs | 504 + .../std/src/io/buffered/bufreader/buffer.rs | 122 + crux-mir/lib/std/src/io/buffered/bufwriter.rs | 674 + .../lib/std/src/io/buffered/linewriter.rs | 232 + .../lib/std/src/io/buffered/linewritershim.rs | 276 + crux-mir/lib/std/src/io/buffered/mod.rs | 196 + crux-mir/lib/std/src/io/buffered/tests.rs | 1065 + crux-mir/lib/std/src/io/copy.rs | 163 + crux-mir/lib/std/src/io/cursor.rs | 859 +- crux-mir/lib/std/src/io/cursor/tests.rs | 567 + crux-mir/lib/std/src/io/error.rs | 673 +- .../lib/std/src/io/error/repr_bitpacked.rs | 409 + .../lib/std/src/io/error/repr_unpacked.rs | 54 + crux-mir/lib/std/src/io/error/tests.rs | 194 + crux-mir/lib/std/src/io/impls.rs | 183 +- crux-mir/lib/std/src/io/impls/tests.rs | 57 + crux-mir/lib/std/src/io/lazy.rs | 63 - crux-mir/lib/std/src/io/mod.rs | 1590 +- crux-mir/lib/std/src/io/prelude.rs | 2 +- crux-mir/lib/std/src/io/readbuf.rs | 309 + crux-mir/lib/std/src/io/readbuf/tests.rs | 175 + crux-mir/lib/std/src/io/stdio.rs | 785 +- crux-mir/lib/std/src/io/stdio/tests.rs | 166 + crux-mir/lib/std/src/io/tests.rs | 624 + crux-mir/lib/std/src/io/util.rs | 258 +- crux-mir/lib/std/src/io/util/tests.rs | 147 + crux-mir/lib/std/src/keyword_docs.rs | 1393 +- crux-mir/lib/std/src/lib.rs | 514 +- crux-mir/lib/std/src/macros.rs | 133 +- crux-mir/lib/std/src/memchr.rs | 133 - crux-mir/lib/std/src/net/display_buffer.rs | 40 + crux-mir/lib/std/src/net/ip.rs | 2726 - crux-mir/lib/std/src/net/ip_addr.rs | 2102 + crux-mir/lib/std/src/net/ip_addr/tests.rs | 1039 + crux-mir/lib/std/src/net/mod.rs | 62 +- crux-mir/lib/std/src/net/parser.rs | 540 +- crux-mir/lib/std/src/net/parser/tests.rs | 149 + .../std/src/net/{addr.rs => socket_addr.rs} | 530 +- crux-mir/lib/std/src/net/socket_addr/tests.rs | 306 + crux-mir/lib/std/src/net/tcp.rs | 1137 +- crux-mir/lib/std/src/net/tcp/tests.rs | 876 + crux-mir/lib/std/src/net/udp.rs | 512 +- crux-mir/lib/std/src/net/udp/tests.rs | 365 + crux-mir/lib/std/src/num.rs | 271 +- crux-mir/lib/std/src/num/benches.rs | 9 + crux-mir/lib/std/src/num/tests.rs | 230 + crux-mir/lib/std/src/os/android/fs.rs | 8 +- crux-mir/lib/std/src/os/android/mod.rs | 1 + crux-mir/lib/std/src/os/android/net.rs | 9 + crux-mir/lib/std/src/os/android/raw.rs | 10 +- crux-mir/lib/std/src/os/dragonfly/fs.rs | 8 +- crux-mir/lib/std/src/os/dragonfly/raw.rs | 10 +- crux-mir/lib/std/src/os/emscripten/fs.rs | 8 +- crux-mir/lib/std/src/os/emscripten/raw.rs | 11 +- crux-mir/lib/std/src/os/espidf/fs.rs | 117 + crux-mir/lib/std/src/os/espidf/mod.rs | 6 + crux-mir/lib/std/src/os/espidf/raw.rs | 69 + crux-mir/lib/std/src/os/fd/mod.rs | 25 + crux-mir/lib/std/src/os/fd/net.rs | 46 + crux-mir/lib/std/src/os/fd/owned.rs | 456 + crux-mir/lib/std/src/os/fd/raw.rs | 253 + crux-mir/lib/std/src/os/fd/tests.rs | 53 + .../{sys/sgx/ext => os/fortanix_sgx}/arch.rs | 30 +- .../{sys/sgx/ext => os/fortanix_sgx}/ffi.rs | 5 +- .../{sys/sgx/ext => os/fortanix_sgx}/io.rs | 47 +- crux-mir/lib/std/src/os/fortanix_sgx/mod.rs | 11 +- crux-mir/lib/std/src/os/freebsd/fs.rs | 22 +- crux-mir/lib/std/src/os/freebsd/raw.rs | 10 +- crux-mir/lib/std/src/os/fuchsia/fs.rs | 2 +- crux-mir/lib/std/src/os/fuchsia/raw.rs | 10 +- crux-mir/lib/std/src/os/haiku/fs.rs | 8 +- crux-mir/lib/std/src/os/haiku/raw.rs | 7 + crux-mir/lib/std/src/os/hermit/ffi.rs | 41 + crux-mir/lib/std/src/os/hermit/mod.rs | 13 + crux-mir/lib/std/src/os/horizon/fs.rs | 95 + crux-mir/lib/std/src/os/horizon/mod.rs | 6 + crux-mir/lib/std/src/os/horizon/raw.rs | 70 + crux-mir/lib/std/src/os/illumos/fs.rs | 116 + crux-mir/lib/std/src/os/illumos/mod.rs | 6 + crux-mir/lib/std/src/os/illumos/raw.rs | 74 + crux-mir/lib/std/src/os/ios/fs.rs | 8 +- crux-mir/lib/std/src/os/ios/raw.rs | 10 +- crux-mir/lib/std/src/os/l4re/fs.rs | 382 + crux-mir/lib/std/src/os/l4re/mod.rs | 7 + crux-mir/lib/std/src/os/l4re/raw.rs | 365 + crux-mir/lib/std/src/os/linux/fs.rs | 39 +- crux-mir/lib/std/src/os/linux/mod.rs | 5 +- crux-mir/lib/std/src/os/linux/net.rs | 9 + crux-mir/lib/std/src/os/linux/process.rs | 165 + crux-mir/lib/std/src/os/linux/raw.rs | 83 +- crux-mir/lib/std/src/os/macos/fs.rs | 8 +- crux-mir/lib/std/src/os/macos/raw.rs | 10 +- crux-mir/lib/std/src/os/mod.rs | 143 +- crux-mir/lib/std/src/os/net/linux_ext/addr.rs | 64 + crux-mir/lib/std/src/os/net/linux_ext/mod.rs | 12 + crux-mir/lib/std/src/os/net/linux_ext/tcp.rs | 70 + .../lib/std/src/os/net/linux_ext/tests.rs | 28 + crux-mir/lib/std/src/os/net/mod.rs | 4 + crux-mir/lib/std/src/os/netbsd/fs.rs | 8 +- crux-mir/lib/std/src/os/netbsd/raw.rs | 10 +- crux-mir/lib/std/src/os/openbsd/fs.rs | 8 +- crux-mir/lib/std/src/os/openbsd/raw.rs | 10 +- crux-mir/lib/std/src/os/raw/mod.rs | 184 +- crux-mir/lib/std/src/os/raw/tests.rs | 15 + crux-mir/lib/std/src/os/redox/fs.rs | 18 +- crux-mir/lib/std/src/os/redox/raw.rs | 11 +- crux-mir/lib/std/src/os/solaris/fs.rs | 8 +- crux-mir/lib/std/src/os/solaris/raw.rs | 10 +- .../src/{sys/vxworks/ext => os/solid}/ffi.rs | 11 +- crux-mir/lib/std/src/os/solid/io.rs | 113 + .../src/{sys/vxworks/ext => os/solid}/mod.rs | 13 +- .../unix/ext/ffi.rs => os/unix/ffi/mod.rs} | 8 +- crux-mir/lib/std/src/os/unix/ffi/os_str.rs | 70 + .../std/src/{sys/unix/ext => os/unix}/fs.rs | 232 +- crux-mir/lib/std/src/os/unix/io/mod.rs | 85 + crux-mir/lib/std/src/os/unix/io/tests.rs | 11 + crux-mir/lib/std/src/os/unix/mod.rs | 128 + crux-mir/lib/std/src/os/unix/net/addr.rs | 295 + crux-mir/lib/std/src/os/unix/net/ancillary.rs | 674 + crux-mir/lib/std/src/os/unix/net/datagram.rs | 1012 + crux-mir/lib/std/src/os/unix/net/listener.rs | 391 + crux-mir/lib/std/src/os/unix/net/mod.rs | 26 + crux-mir/lib/std/src/os/unix/net/stream.rs | 736 + crux-mir/lib/std/src/os/unix/net/tests.rs | 759 + crux-mir/lib/std/src/os/unix/process.rs | 466 + crux-mir/lib/std/src/os/unix/raw.rs | 33 + .../src/{sys/unix/ext => os/unix}/thread.rs | 8 +- crux-mir/lib/std/src/os/unix/ucred.rs | 136 + crux-mir/lib/std/src/os/unix/ucred/tests.rs | 39 + crux-mir/lib/std/src/os/vxworks/fs.rs | 17 +- crux-mir/lib/std/src/os/vxworks/raw.rs | 3 + crux-mir/lib/std/src/os/wasi.rs | 6 - crux-mir/lib/std/src/os/wasi/ffi.rs | 11 + .../std/src/{sys/wasi/ext => os/wasi}/fs.rs | 227 +- crux-mir/lib/std/src/os/wasi/io/fd.rs | 9 + crux-mir/lib/std/src/os/wasi/io/fd/tests.rs | 11 + crux-mir/lib/std/src/os/wasi/io/mod.rs | 6 + crux-mir/lib/std/src/os/wasi/io/raw.rs | 20 + .../std/src/{sys/unix/ext => os/wasi}/mod.rs | 35 +- crux-mir/lib/std/src/os/wasi/net/mod.rs | 23 + crux-mir/lib/std/src/os/watchos/fs.rs | 142 + crux-mir/lib/std/src/os/watchos/mod.rs | 6 + crux-mir/lib/std/src/os/watchos/raw.rs | 83 + .../{sys/windows/ext => os/windows}/ffi.rs | 38 +- .../src/{sys/windows/ext => os/windows}/fs.rs | 90 +- crux-mir/lib/std/src/os/windows/io/handle.rs | 593 + crux-mir/lib/std/src/os/windows/io/mod.rs | 65 + crux-mir/lib/std/src/os/windows/io/raw.rs | 305 + crux-mir/lib/std/src/os/windows/io/socket.rs | 339 + crux-mir/lib/std/src/os/windows/io/tests.rs | 21 + .../{sys/windows/ext => os/windows}/mod.rs | 24 +- crux-mir/lib/std/src/os/windows/process.rs | 259 + .../{sys/windows/ext => os/windows}/raw.rs | 4 +- .../{sys/windows/ext => os/windows}/thread.rs | 10 +- crux-mir/lib/std/src/panic.rs | 505 +- crux-mir/lib/std/src/panic/tests.rs | 56 + crux-mir/lib/std/src/panicking.rs | 517 +- crux-mir/lib/std/src/path.rs | 2548 +- crux-mir/lib/std/src/path/tests.rs | 1878 + crux-mir/lib/std/src/personality.rs | 46 + .../src/personality}/dwarf/eh.rs | 65 +- .../src/personality}/dwarf/mod.rs | 4 +- .../src/personality}/dwarf/tests.rs | 0 crux-mir/lib/std/src/personality/emcc.rs | 20 + .../src/personality}/gcc.rs | 247 +- crux-mir/lib/std/src/prelude/mod.rs | 198 +- crux-mir/lib/std/src/prelude/v1.rs | 51 +- crux-mir/lib/std/src/primitive_docs.rs | 894 +- crux-mir/lib/std/src/process.rs | 1296 +- crux-mir/lib/std/src/process/tests.rs | 552 + crux-mir/lib/std/src/rt.rs | 160 +- crux-mir/lib/std/src/sync/barrier.rs | 81 +- crux-mir/lib/std/src/sync/barrier/tests.rs | 35 + crux-mir/lib/std/src/sync/condvar.rs | 361 +- crux-mir/lib/std/src/sync/condvar/tests.rs | 190 + crux-mir/lib/std/src/sync/lazy_lock.rs | 125 + crux-mir/lib/std/src/sync/lazy_lock/tests.rs | 149 + crux-mir/lib/std/src/sync/mod.rs | 20 +- crux-mir/lib/std/src/sync/mpmc/array.rs | 513 + crux-mir/lib/std/src/sync/mpmc/context.rs | 155 + crux-mir/lib/std/src/sync/mpmc/counter.rs | 137 + crux-mir/lib/std/src/sync/mpmc/error.rs | 46 + crux-mir/lib/std/src/sync/mpmc/list.rs | 638 + crux-mir/lib/std/src/sync/mpmc/mod.rs | 430 + crux-mir/lib/std/src/sync/mpmc/select.rs | 71 + crux-mir/lib/std/src/sync/mpmc/utils.rs | 143 + crux-mir/lib/std/src/sync/mpmc/waker.rs | 204 + crux-mir/lib/std/src/sync/mpmc/zero.rs | 318 + crux-mir/lib/std/src/sync/mpsc/blocking.rs | 79 - .../lib/std/src/sync/mpsc/cache_aligned.rs | 27 - crux-mir/lib/std/src/sync/mpsc/mod.rs | 2057 +- crux-mir/lib/std/src/sync/mpsc/mpsc_queue.rs | 165 - crux-mir/lib/std/src/sync/mpsc/oneshot.rs | 307 - crux-mir/lib/std/src/sync/mpsc/shared.rs | 492 - crux-mir/lib/std/src/sync/mpsc/spsc_queue.rs | 338 - crux-mir/lib/std/src/sync/mpsc/stream.rs | 454 - crux-mir/lib/std/src/sync/mpsc/sync.rs | 492 - crux-mir/lib/std/src/sync/mpsc/sync_tests.rs | 658 + crux-mir/lib/std/src/sync/mpsc/tests.rs | 723 + crux-mir/lib/std/src/sync/mutex.rs | 500 +- crux-mir/lib/std/src/sync/mutex/tests.rs | 238 + crux-mir/lib/std/src/sync/once.rs | 505 +- crux-mir/lib/std/src/sync/once/tests.rs | 116 + crux-mir/lib/std/src/sync/once_lock.rs | 458 + crux-mir/lib/std/src/sync/once_lock/tests.rs | 203 + .../std/src/{sys_common => sync}/poison.rs | 58 +- crux-mir/lib/std/src/sync/remutex.rs | 178 + crux-mir/lib/std/src/sync/remutex/tests.rs | 60 + crux-mir/lib/std/src/sync/rwlock.rs | 508 +- crux-mir/lib/std/src/sync/rwlock/tests.rs | 259 + .../lib/std/src/sys/cloudabi/abi/bitflags.rs | 47 - .../lib/std/src/sys/cloudabi/abi/cloudabi.rs | 2947 - crux-mir/lib/std/src/sys/cloudabi/abi/mod.rs | 3 - crux-mir/lib/std/src/sys/cloudabi/args.rs | 7 - crux-mir/lib/std/src/sys/cloudabi/condvar.rs | 152 - crux-mir/lib/std/src/sys/cloudabi/io.rs | 47 - crux-mir/lib/std/src/sys/cloudabi/mod.rs | 66 - crux-mir/lib/std/src/sys/cloudabi/mutex.rs | 153 - crux-mir/lib/std/src/sys/cloudabi/os.rs | 26 - crux-mir/lib/std/src/sys/cloudabi/rwlock.rs | 218 - .../lib/std/src/sys/cloudabi/shims/env.rs | 9 - crux-mir/lib/std/src/sys/cloudabi/shims/fs.rs | 300 - .../lib/std/src/sys/cloudabi/shims/mod.rs | 19 - .../lib/std/src/sys/cloudabi/shims/net.rs | 318 - crux-mir/lib/std/src/sys/cloudabi/shims/os.rs | 86 - .../lib/std/src/sys/cloudabi/shims/pipe.rs | 30 - .../lib/std/src/sys/cloudabi/shims/process.rs | 149 - .../std/src/sys/cloudabi/stack_overflow.rs | 13 - crux-mir/lib/std/src/sys/cloudabi/stdio.rs | 66 - crux-mir/lib/std/src/sys/cloudabi/thread.rs | 113 - crux-mir/lib/std/src/sys/cloudabi/time.rs | 82 - .../src/{sys_common => sys/common}/alloc.rs | 24 +- crux-mir/lib/std/src/sys/common/mod.rs | 17 + .../lib/std/src/sys/common/small_c_string.rs | 58 + crux-mir/lib/std/src/sys/common/tests.rs | 66 + crux-mir/lib/std/src/sys/crux/condvar.rs | 44 - crux-mir/lib/std/src/sys/crux/mod.rs | 4 - crux-mir/lib/std/src/sys/crux/mutex.rs | 94 - crux-mir/lib/std/src/sys/crux/rwlock.rs | 70 - crux-mir/lib/std/src/sys/crux/time.rs | 70 - crux-mir/lib/std/src/sys/hermit/args.rs | 87 +- crux-mir/lib/std/src/sys/hermit/cmath.rs | 29 - crux-mir/lib/std/src/sys/hermit/condvar.rs | 62 - .../std/src/sys/hermit/fast_thread_local.rs | 4 - crux-mir/lib/std/src/sys/hermit/fd.rs | 9 +- crux-mir/lib/std/src/sys/hermit/fs.rs | 132 +- crux-mir/lib/std/src/sys/hermit/futex.rs | 39 + crux-mir/lib/std/src/sys/hermit/mod.rs | 76 +- crux-mir/lib/std/src/sys/hermit/mutex.rs | 77 - crux-mir/lib/std/src/sys/hermit/net.rs | 319 +- crux-mir/lib/std/src/sys/hermit/os.rs | 25 +- crux-mir/lib/std/src/sys/hermit/path.rs | 19 - crux-mir/lib/std/src/sys/hermit/pipe.rs | 30 - crux-mir/lib/std/src/sys/hermit/process.rs | 149 - crux-mir/lib/std/src/sys/hermit/rwlock.rs | 49 - .../lib/std/src/sys/hermit/stack_overflow.rs | 13 - crux-mir/lib/std/src/sys/hermit/stdio.rs | 66 +- crux-mir/lib/std/src/sys/hermit/thread.rs | 65 +- .../lib/std/src/sys/hermit/thread_local.rs | 60 - .../std/src/sys/hermit/thread_local_dtor.rs | 36 + crux-mir/lib/std/src/sys/hermit/time.rs | 9 - crux-mir/lib/std/src/sys/itron/abi.rs | 197 + crux-mir/lib/std/src/sys/itron/condvar.rs | 292 + crux-mir/lib/std/src/sys/itron/error.rs | 159 + crux-mir/lib/std/src/sys/itron/mutex.rs | 85 + crux-mir/lib/std/src/sys/itron/spin.rs | 163 + crux-mir/lib/std/src/sys/itron/task.rs | 44 + crux-mir/lib/std/src/sys/itron/thread.rs | 368 + crux-mir/lib/std/src/sys/itron/time.rs | 114 + crux-mir/lib/std/src/sys/itron/time/tests.rs | 33 + crux-mir/lib/std/src/sys/itron/wait_flag.rs | 72 + crux-mir/lib/std/src/sys/mod.rs | 81 +- crux-mir/lib/std/src/sys/sgx/abi/entry.S | 54 +- crux-mir/lib/std/src/sys/sgx/abi/mem.rs | 66 +- crux-mir/lib/std/src/sys/sgx/abi/mod.rs | 28 +- crux-mir/lib/std/src/sys/sgx/abi/thread.rs | 8 +- crux-mir/lib/std/src/sys/sgx/abi/tls.rs | 241 - crux-mir/lib/std/src/sys/sgx/abi/tls/mod.rs | 133 + .../std/src/sys/sgx/abi/tls/sync_bitset.rs | 85 + .../src/sys/sgx/abi/tls/sync_bitset/tests.rs | 25 + .../std/src/sys/sgx/abi/usercalls/alloc.rs | 322 +- .../lib/std/src/sys/sgx/abi/usercalls/mod.rs | 108 +- .../lib/std/src/sys/sgx/abi/usercalls/raw.rs | 96 +- .../std/src/sys/sgx/abi/usercalls/tests.rs | 56 + crux-mir/lib/std/src/sys/sgx/alloc.rs | 66 +- crux-mir/lib/std/src/sys/sgx/args.rs | 11 +- crux-mir/lib/std/src/sys/sgx/cmath.rs | 31 - crux-mir/lib/std/src/sys/sgx/condvar.rs | 40 +- crux-mir/lib/std/src/sys/sgx/env.rs | 14 +- crux-mir/lib/std/src/sys/sgx/ext/mod.rs | 5 - crux-mir/lib/std/src/sys/sgx/fd.rs | 12 +- crux-mir/lib/std/src/sys/sgx/fs.rs | 300 - crux-mir/lib/std/src/sys/sgx/io.rs | 47 - crux-mir/lib/std/src/sys/sgx/mod.rs | 78 +- crux-mir/lib/std/src/sys/sgx/mutex.rs | 115 +- crux-mir/lib/std/src/sys/sgx/net.rs | 109 +- crux-mir/lib/std/src/sys/sgx/os.rs | 13 +- crux-mir/lib/std/src/sys/sgx/path.rs | 10 +- crux-mir/lib/std/src/sys/sgx/process.rs | 149 - crux-mir/lib/std/src/sys/sgx/rwlock.rs | 149 +- crux-mir/lib/std/src/sys/sgx/rwlock/tests.rs | 21 + .../lib/std/src/sys/sgx/stack_overflow.rs | 12 - crux-mir/lib/std/src/sys/sgx/stdio.rs | 18 +- crux-mir/lib/std/src/sys/sgx/thread.rs | 78 +- .../{thread_local.rs => thread_local_key.rs} | 5 - .../lib/std/src/sys/sgx/thread_parking.rs | 23 + crux-mir/lib/std/src/sys/sgx/time.rs | 8 - crux-mir/lib/std/src/sys/sgx/waitqueue.rs | 524 - crux-mir/lib/std/src/sys/sgx/waitqueue/mod.rs | 240 + .../std/src/sys/sgx/waitqueue/spin_mutex.rs | 80 + .../src/sys/sgx/waitqueue/spin_mutex/tests.rs | 23 + .../lib/std/src/sys/sgx/waitqueue/tests.rs | 20 + .../std/src/sys/sgx/waitqueue/unsafe_list.rs | 156 + .../sys/sgx/waitqueue/unsafe_list/tests.rs | 105 + crux-mir/lib/std/src/sys/solid/abi/fs.rs | 53 + crux-mir/lib/std/src/sys/solid/abi/mod.rs | 65 + crux-mir/lib/std/src/sys/solid/abi/sockets.rs | 277 + crux-mir/lib/std/src/sys/solid/alloc.rs | 32 + .../lib/std/src/sys/{vxworks => solid}/env.rs | 6 +- crux-mir/lib/std/src/sys/solid/error.rs | 55 + crux-mir/lib/std/src/sys/solid/fs.rs | 588 + .../lib/std/src/sys/{vxworks => solid}/io.rs | 8 +- .../std/src/sys/{vxworks => solid}/memchr.rs | 16 +- crux-mir/lib/std/src/sys/solid/mod.rs | 92 + crux-mir/lib/std/src/sys/solid/net.rs | 469 + crux-mir/lib/std/src/sys/solid/os.rs | 192 + crux-mir/lib/std/src/sys/solid/path.rs | 25 + crux-mir/lib/std/src/sys/solid/rwlock.rs | 93 + crux-mir/lib/std/src/sys/solid/stdio.rs | 80 + .../std/src/sys/solid/thread_local_dtor.rs | 50 + .../lib/std/src/sys/solid/thread_local_key.rs | 21 + crux-mir/lib/std/src/sys/solid/time.rs | 56 + crux-mir/lib/std/src/sys/unix/alloc.rs | 80 +- crux-mir/lib/std/src/sys/unix/android.rs | 94 +- crux-mir/lib/std/src/sys/unix/args.rs | 94 +- crux-mir/lib/std/src/sys/unix/cmath.rs | 55 +- crux-mir/lib/std/src/sys/unix/condvar.rs | 178 - crux-mir/lib/std/src/sys/unix/env.rs | 55 + crux-mir/lib/std/src/sys/unix/ext/io.rs | 124 - crux-mir/lib/std/src/sys/unix/ext/net.rs | 2006 - crux-mir/lib/std/src/sys/unix/ext/process.rs | 234 - crux-mir/lib/std/src/sys/unix/ext/raw.rs | 28 - crux-mir/lib/std/src/sys/unix/fd.rs | 352 +- crux-mir/lib/std/src/sys/unix/fd/tests.rs | 10 + crux-mir/lib/std/src/sys/unix/fs.rs | 1408 +- crux-mir/lib/std/src/sys/unix/futex.rs | 303 + crux-mir/lib/std/src/sys/unix/io.rs | 6 + crux-mir/lib/std/src/sys/unix/kernel_copy.rs | 690 + .../lib/std/src/sys/unix/kernel_copy/tests.rs | 270 + crux-mir/lib/std/src/sys/unix/l4re.rs | 88 +- .../std/src/sys/unix/locks/fuchsia_mutex.rs | 164 + .../std/src/sys/unix/locks/futex_condvar.rs | 56 + .../lib/std/src/sys/unix/locks/futex_mutex.rs | 96 + .../std/src/sys/unix/locks/futex_rwlock.rs | 320 + crux-mir/lib/std/src/sys/unix/locks/mod.rs | 31 + .../std/src/sys/unix/locks/pthread_condvar.rs | 192 + .../std/src/sys/unix/locks/pthread_mutex.rs | 131 + .../std/src/sys/unix/locks/pthread_rwlock.rs | 195 + crux-mir/lib/std/src/sys/unix/memchr.rs | 6 +- crux-mir/lib/std/src/sys/unix/mod.rs | 394 +- crux-mir/lib/std/src/sys/unix/mutex.rs | 131 - crux-mir/lib/std/src/sys/unix/net.rs | 333 +- crux-mir/lib/std/src/sys/unix/os.rs | 328 +- crux-mir/lib/std/src/sys/unix/os/tests.rs | 23 + .../os_str_bytes.rs => sys/unix/os_str.rs} | 165 +- crux-mir/lib/std/src/sys/unix/os_str/tests.rs | 18 + crux-mir/lib/std/src/sys/unix/path.rs | 46 +- crux-mir/lib/std/src/sys/unix/pipe.rs | 114 +- crux-mir/lib/std/src/sys/unix/process/mod.rs | 31 +- .../src/sys/unix/process/process_common.rs | 371 +- .../sys/unix/process/process_common/tests.rs | 161 + .../src/sys/unix/process/process_fuchsia.rs | 93 +- .../std/src/sys/unix/process/process_unix.rs | 585 +- .../sys/unix/process/process_unix/tests.rs | 62 + .../sys/unix/process/process_unsupported.rs | 122 + .../process/process_vxworks.rs | 108 +- .../lib/std/src/sys/unix/process/zircon.rs | 14 +- crux-mir/lib/std/src/sys/unix/rand.rs | 142 +- crux-mir/lib/std/src/sys/unix/rwlock.rs | 133 - .../lib/std/src/sys/unix/stack_overflow.rs | 103 +- crux-mir/lib/std/src/sys/unix/stdio.rs | 46 +- crux-mir/lib/std/src/sys/unix/thread.rs | 713 +- ...t_thread_local.rs => thread_local_dtor.rs} | 12 +- .../{thread_local.rs => thread_local_key.rs} | 5 - .../std/src/sys/unix/thread_parking/darwin.rs | 131 + .../std/src/sys/unix/thread_parking/mod.rs | 32 + .../std/src/sys/unix/thread_parking/netbsd.rs | 52 + .../src/sys/unix/thread_parking/pthread.rs | 271 + crux-mir/lib/std/src/sys/unix/time.rs | 376 +- crux-mir/lib/std/src/sys/unix/weak.rs | 171 +- crux-mir/lib/std/src/sys/unsupported/alloc.rs | 23 + .../{cloudabi/shims => unsupported}/args.rs | 17 +- .../lib/std/src/sys/unsupported/common.rs | 36 + crux-mir/lib/std/src/sys/unsupported/env.rs | 9 + .../std/src/sys/{wasm => unsupported}/fs.rs | 122 +- .../std/src/sys/{hermit => unsupported}/io.rs | 4 + .../{wasm => unsupported/locks}/condvar.rs | 18 +- .../lib/std/src/sys/unsupported/locks/mod.rs | 6 + .../std/src/sys/unsupported/locks/mutex.rs | 32 + .../std/src/sys/unsupported/locks/rwlock.rs | 65 + crux-mir/lib/std/src/sys/unsupported/mod.rs | 28 + .../std/src/sys/{wasm => unsupported}/net.rs | 155 +- crux-mir/lib/std/src/sys/unsupported/once.rs | 89 + .../std/src/sys/{wasm => unsupported}/os.rs | 34 +- .../std/src/sys/{sgx => unsupported}/pipe.rs | 25 +- .../src/sys/{wasi => unsupported}/process.rs | 99 +- .../src/sys/{wasm => unsupported}/stdio.rs | 12 +- .../lib/std/src/sys/unsupported/thread.rs | 46 + .../thread_local_dtor.rs} | 1 + .../src/sys/unsupported/thread_local_key.rs | 21 + .../std/src/sys/{wasm => unsupported}/time.rs | 12 +- crux-mir/lib/std/src/sys/vxworks/alloc.rs | 49 - crux-mir/lib/std/src/sys/vxworks/args.rs | 96 - crux-mir/lib/std/src/sys/vxworks/cmath.rs | 32 - crux-mir/lib/std/src/sys/vxworks/condvar.rs | 93 - crux-mir/lib/std/src/sys/vxworks/ext/fs.rs | 841 - crux-mir/lib/std/src/sys/vxworks/ext/io.rs | 189 - .../lib/std/src/sys/vxworks/ext/process.rs | 234 - crux-mir/lib/std/src/sys/vxworks/ext/raw.rs | 5 - .../std/src/sys/vxworks/fast_thread_local.rs | 11 - crux-mir/lib/std/src/sys/vxworks/fd.rs | 193 - crux-mir/lib/std/src/sys/vxworks/fs.rs | 615 - crux-mir/lib/std/src/sys/vxworks/mod.rs | 113 - crux-mir/lib/std/src/sys/vxworks/mutex.rs | 131 - crux-mir/lib/std/src/sys/vxworks/net.rs | 349 - crux-mir/lib/std/src/sys/vxworks/os.rs | 316 - crux-mir/lib/std/src/sys/vxworks/path.rs | 19 - crux-mir/lib/std/src/sys/vxworks/pipe.rs | 96 - .../lib/std/src/sys/vxworks/process/mod.rs | 7 - .../src/sys/vxworks/process/process_common.rs | 405 - crux-mir/lib/std/src/sys/vxworks/rand.rs | 37 - crux-mir/lib/std/src/sys/vxworks/rwlock.rs | 115 - .../lib/std/src/sys/vxworks/stack_overflow.rs | 38 - crux-mir/lib/std/src/sys/vxworks/stdio.rs | 69 - crux-mir/lib/std/src/sys/vxworks/thread.rs | 148 - .../lib/std/src/sys/vxworks/thread_local.rs | 34 - crux-mir/lib/std/src/sys/vxworks/time.rs | 199 - crux-mir/lib/std/src/sys/wasi/alloc.rs | 43 - crux-mir/lib/std/src/sys/wasi/args.rs | 23 +- crux-mir/lib/std/src/sys/wasi/ext/ffi.rs | 6 - crux-mir/lib/std/src/sys/wasi/ext/io.rs | 142 - crux-mir/lib/std/src/sys/wasi/ext/mod.rs | 22 - crux-mir/lib/std/src/sys/wasi/fd.rs | 181 +- crux-mir/lib/std/src/sys/wasi/fs.rs | 286 +- crux-mir/lib/std/src/sys/wasi/io.rs | 8 + crux-mir/lib/std/src/sys/wasi/mod.rs | 112 +- crux-mir/lib/std/src/sys/wasi/net.rs | 232 +- crux-mir/lib/std/src/sys/wasi/os.rs | 129 +- crux-mir/lib/std/src/sys/wasi/path.rs | 19 - crux-mir/lib/std/src/sys/wasi/pipe.rs | 30 - crux-mir/lib/std/src/sys/wasi/stdio.rs | 95 +- crux-mir/lib/std/src/sys/wasi/thread.rs | 23 +- crux-mir/lib/std/src/sys/wasi/time.rs | 16 +- crux-mir/lib/std/src/sys/wasm/alloc.rs | 22 +- crux-mir/lib/std/src/sys/wasm/args.rs | 46 - .../lib/std/src/sys/wasm/atomics/futex.rs | 34 + .../lib/std/src/sys/wasm/atomics/thread.rs | 55 + crux-mir/lib/std/src/sys/wasm/cmath.rs | 29 - .../lib/std/src/sys/wasm/condvar_atomics.rs | 94 - crux-mir/lib/std/src/sys/wasm/io.rs | 47 - crux-mir/lib/std/src/sys/wasm/memchr.rs | 1 - crux-mir/lib/std/src/sys/wasm/mod.rs | 100 +- crux-mir/lib/std/src/sys/wasm/mutex.rs | 66 - .../lib/std/src/sys/wasm/mutex_atomics.rs | 147 - crux-mir/lib/std/src/sys/wasm/path.rs | 19 - crux-mir/lib/std/src/sys/wasm/pipe.rs | 30 - crux-mir/lib/std/src/sys/wasm/process.rs | 149 - crux-mir/lib/std/src/sys/wasm/rwlock.rs | 69 - .../lib/std/src/sys/wasm/rwlock_atomics.rs | 144 - .../lib/std/src/sys/wasm/stack_overflow.rs | 11 - crux-mir/lib/std/src/sys/wasm/thread.rs | 98 - crux-mir/lib/std/src/sys/wasm/thread_local.rs | 26 - crux-mir/lib/std/src/sys/windows/alloc.rs | 240 +- .../lib/std/src/sys/windows/alloc/tests.rs | 9 + crux-mir/lib/std/src/sys/windows/args.rs | 436 +- .../lib/std/src/sys/windows/args/tests.rs | 91 + crux-mir/lib/std/src/sys/windows/c.rs | 820 +- crux-mir/lib/std/src/sys/windows/c/errors.rs | 1883 + crux-mir/lib/std/src/sys/windows/cmath.rs | 2 +- crux-mir/lib/std/src/sys/windows/compat.rs | 271 +- crux-mir/lib/std/src/sys/windows/ext/io.rs | 220 - .../lib/std/src/sys/windows/ext/process.rs | 113 - .../std/src/sys/windows/fast_thread_local.rs | 4 - crux-mir/lib/std/src/sys/windows/fs.rs | 793 +- crux-mir/lib/std/src/sys/windows/handle.rs | 295 +- .../lib/std/src/sys/windows/handle/tests.rs | 22 + crux-mir/lib/std/src/sys/windows/io.rs | 80 +- .../src/sys/windows/{ => locks}/condvar.rs | 20 +- crux-mir/lib/std/src/sys/windows/locks/mod.rs | 6 + .../lib/std/src/sys/windows/locks/mutex.rs | 54 + .../lib/std/src/sys/windows/locks/rwlock.rs | 40 + crux-mir/lib/std/src/sys/windows/mod.rs | 281 +- crux-mir/lib/std/src/sys/windows/mutex.rs | 184 - crux-mir/lib/std/src/sys/windows/net.rs | 412 +- crux-mir/lib/std/src/sys/windows/os.rs | 61 +- crux-mir/lib/std/src/sys/windows/os/tests.rs | 13 + crux-mir/lib/std/src/sys/windows/os_str.rs | 60 +- crux-mir/lib/std/src/sys/windows/path.rs | 377 +- .../lib/std/src/sys/windows/path/tests.rs | 137 + crux-mir/lib/std/src/sys/windows/pipe.rs | 205 +- crux-mir/lib/std/src/sys/windows/process.rs | 632 +- .../lib/std/src/sys/windows/process/tests.rs | 222 + crux-mir/lib/std/src/sys/windows/rand.rs | 121 +- crux-mir/lib/std/src/sys/windows/rwlock.rs | 44 - .../lib/std/src/sys/windows/stack_overflow.rs | 17 +- .../std/src/sys/windows/stack_overflow_uwp.rs | 2 - crux-mir/lib/std/src/sys/windows/stdio.rs | 264 +- crux-mir/lib/std/src/sys/windows/stdio_uwp.rs | 84 - crux-mir/lib/std/src/sys/windows/thread.rs | 50 +- .../std/src/sys/windows/thread_local_dtor.rs | 32 + .../{thread_local.rs => thread_local_key.rs} | 200 +- .../src/sys/windows/thread_local_key/tests.rs | 53 + .../lib/std/src/sys/windows/thread_parking.rs | 253 + crux-mir/lib/std/src/sys/windows/time.rs | 52 +- .../lib/std/src/sys_common/at_exit_imp.rs | 75 - crux-mir/lib/std/src/sys_common/backtrace.rs | 103 +- crux-mir/lib/std/src/sys_common/bytestring.rs | 46 - crux-mir/lib/std/src/sys_common/condvar.rs | 72 - crux-mir/lib/std/src/sys_common/fs.rs | 28 +- crux-mir/lib/std/src/sys_common/io.rs | 15 +- crux-mir/lib/std/src/sys_common/lazy_box.rs | 90 + crux-mir/lib/std/src/sys_common/memchr.rs | 51 + .../lib/std/src/sys_common/memchr/tests.rs | 86 + crux-mir/lib/std/src/sys_common/mod.rs | 104 +- crux-mir/lib/std/src/sys_common/mutex.rs | 100 - crux-mir/lib/std/src/sys_common/net.rs | 235 +- crux-mir/lib/std/src/sys_common/net/tests.rs | 19 + crux-mir/lib/std/src/sys_common/once/futex.rs | 134 + crux-mir/lib/std/src/sys_common/once/mod.rs | 34 + crux-mir/lib/std/src/sys_common/once/queue.rs | 283 + crux-mir/lib/std/src/sys_common/process.rs | 106 +- crux-mir/lib/std/src/sys_common/remutex.rs | 224 - crux-mir/lib/std/src/sys_common/rwlock.rs | 88 - crux-mir/lib/std/src/sys_common/tests.rs | 6 + crux-mir/lib/std/src/sys_common/thread.rs | 15 +- .../lib/std/src/sys_common/thread_info.rs | 27 +- .../std/src/sys_common/thread_local_dtor.rs | 49 + .../{thread_local.rs => thread_local_key.rs} | 138 +- .../src/sys_common/thread_local_key/tests.rs | 35 + .../src/sys_common/thread_parking/futex.rs | 97 + .../src/sys_common/thread_parking/generic.rs | 125 + .../std/src/sys_common/thread_parking/id.rs | 108 + .../std/src/sys_common/thread_parking/mod.rs | 29 + .../sys_common/thread_parking/wait_flag.rs | 102 + crux-mir/lib/std/src/sys_common/util.rs | 30 - crux-mir/lib/std/src/sys_common/wstr.rs | 59 + crux-mir/lib/std/src/sys_common/wtf8.rs | 605 +- crux-mir/lib/std/src/sys_common/wtf8/tests.rs | 666 + crux-mir/lib/std/src/thread/local.rs | 963 +- .../lib/std/src/thread/local/dynamic_tests.rs | 40 + crux-mir/lib/std/src/thread/local/tests.rs | 339 + crux-mir/lib/std/src/thread/mod.rs | 1242 +- crux-mir/lib/std/src/thread/scoped.rs | 343 + crux-mir/lib/std/src/thread/tests.rs | 403 + crux-mir/lib/std/src/time.rs | 439 +- crux-mir/lib/std/src/time/tests.rs | 245 + crux-mir/lib/std/tests/env.rs | 82 +- crux-mir/lib/std/tests/run-time-detect.rs | 144 +- crux-mir/lib/std/tests/thread.rs | 16 + crux-mir/lib/std_detect/Cargo.toml | 45 + .../lib/{bigint => std_detect}/LICENSE-APACHE | 2 +- crux-mir/lib/std_detect/LICENSE-MIT | 25 + crux-mir/lib/std_detect/README.md | 73 + .../lib/std_detect/src/detect/arch/aarch64.rs | 153 + .../lib/std_detect/src/detect/arch/arm.rs | 28 + .../lib/std_detect/src/detect/arch/mips.rs | 12 + .../lib/std_detect/src/detect/arch/mips64.rs | 12 + .../lib/std_detect/src/detect/arch/mod.rs | 56 + .../lib/std_detect/src/detect/arch/powerpc.rs | 16 + .../std_detect/src/detect/arch/powerpc64.rs | 16 + .../lib/std_detect/src/detect/arch/riscv.rs | 206 + .../lib/std_detect/src/detect/arch/x86.rs | 197 + crux-mir/lib/std_detect/src/detect/bit.rs | 9 + crux-mir/lib/std_detect/src/detect/cache.rs | 194 + crux-mir/lib/std_detect/src/detect/macros.rs | 166 + crux-mir/lib/std_detect/src/detect/mod.rs | 104 + .../lib/std_detect/src/detect/os/aarch64.rs | 104 + .../src/detect/os/freebsd/aarch64.rs | 21 + .../std_detect/src/detect/os/freebsd/arm.rs | 21 + .../src/detect/os/freebsd/auxvec.rs | 102 + .../std_detect/src/detect/os/freebsd/mod.rs | 22 + .../src/detect/os/freebsd/powerpc.rs | 21 + .../std_detect/src/detect/os/linux/aarch64.rs | 377 + .../lib/std_detect/src/detect/os/linux/arm.rs | 79 + .../std_detect/src/detect/os/linux/auxvec.rs | 389 + .../std_detect/src/detect/os/linux/cpuinfo.rs | 331 + .../std_detect/src/detect/os/linux/mips.rs | 25 + .../lib/std_detect/src/detect/os/linux/mod.rs | 64 + .../std_detect/src/detect/os/linux/powerpc.rs | 36 + .../std_detect/src/detect/os/linux/riscv.rs | 73 + .../lib/std_detect/src/detect/os/other.rs | 8 + .../src/detect/os/windows/aarch64.rs | 59 + crux-mir/lib/std_detect/src/detect/os/x86.rs | 273 + .../test_data/linux-artificial-aarch64.auxv | Bin 0 -> 336 bytes .../test_data/linux-empty-hwcap2-aarch64.auxv | Bin 0 -> 336 bytes .../test_data/linux-hwcap2-aarch64.auxv | Bin 0 -> 336 bytes .../test_data/linux-no-hwcap2-aarch64.auxv | Bin 0 -> 320 bytes .../src/detect/test_data/linux-rpi3.auxv | Bin 0 -> 160 bytes .../macos-virtualbox-linux-x86-4850HQ.auxv | Bin 0 -> 160 bytes crux-mir/lib/std_detect/src/lib.rs | 34 + .../lib/std_detect/tests/cpu-detection.rs | 164 + .../std_detect/tests/macro_trailing_commas.rs | 51 + crux-mir/lib/std_detect/tests/x86-specific.rs | 158 + crux-mir/lib/stdarch/.cirrus.yml | 6 +- .../lib/stdarch/.github/workflows/main.yml | 40 +- crux-mir/lib/stdarch/CONTRIBUTING.md | 29 +- crux-mir/lib/stdarch/Cargo.toml | 2 + crux-mir/lib/stdarch/README.md | 28 +- .../lib/stdarch/ci/android-install-sdk.sh | 4 +- .../docker/aarch64-linux-android/Dockerfile | 11 +- .../aarch64-unknown-linux-gnu/Dockerfile | 8 +- .../docker/arm-linux-androideabi/Dockerfile | 11 +- .../armv7-unknown-linux-gnueabihf/Dockerfile | 8 +- .../docker/i586-unknown-linux-gnu/Dockerfile | 2 +- .../docker/i686-unknown-linux-gnu/Dockerfile | 2 +- .../mipsel-unknown-linux-musl/Dockerfile | 2 +- .../riscv64gc-unknown-linux-gnu/Dockerfile | 10 + .../docker/wasm32-unknown-unknown/Dockerfile | 24 - .../wasm32-unknown-unknown/wasm-entrypoint.sh | 15 - .../stdarch/ci/docker/wasm32-wasi/Dockerfile | 17 + .../ci/docker/x86_64-linux-android/Dockerfile | 6 +- .../Dockerfile | 3 +- .../x86_64-unknown-linux-gnu/Dockerfile | 2 +- crux-mir/lib/stdarch/ci/dox.sh | 9 +- crux-mir/lib/stdarch/ci/run-docker.sh | 6 +- crux-mir/lib/stdarch/ci/run.sh | 63 +- .../crates/assert-instr-macro/Cargo.toml | 1 + .../crates/assert-instr-macro/build.rs | 2 +- .../crates/assert-instr-macro/src/lib.rs | 103 +- .../lib/stdarch/crates/core_arch/Cargo.toml | 10 +- .../lib/stdarch/crates/core_arch/MISSING.md | 116 + .../lib/stdarch/crates/core_arch/README.md | 8 - .../lib/stdarch/crates/core_arch/avx512bw.md | 764 + .../lib/stdarch/crates/core_arch/avx512f.md | 2633 + .../lib/stdarch/crates/core_arch/build.rs | 3 - .../crates/core_arch/src/aarch64/armclang.rs | 23 + .../crates/core_arch/src/aarch64/crc.rs | 102 +- .../crates/core_arch/src/aarch64/mod.rs | 16 +- .../crates/core_arch/src/aarch64/neon.rs | 1988 - .../core_arch/src/aarch64/neon/generated.rs | 28284 ++ .../crates/core_arch/src/aarch64/neon/mod.rs | 5482 + .../crates/core_arch/src/aarch64/prefetch.rs | 73 + .../core_arch/src/aarch64/test_support.rs | 184 + .../crates/core_arch/src/aarch64/tme.rs | 179 + .../crates/core_arch/src/arm/armclang.rs | 53 +- .../crates/core_arch/src/{acle => arm}/dsp.rs | 4 +- .../crates/core_arch/src/{acle => arm}/ex.rs | 50 +- .../stdarch/crates/core_arch/src/arm/mod.rs | 99 +- .../stdarch/crates/core_arch/src/arm/neon.rs | 2141 +- .../crates/core_arch/src/{acle => arm}/sat.rs | 0 .../core_arch/src/{acle => arm}/simd32.rs | 6 +- .../stdarch/crates/core_arch/src/arm/v7.rs | 1 - .../{acle => arm_shared}/barrier/common.rs | 0 .../src/{acle => arm_shared}/barrier/cp15.rs | 17 +- .../src/{acle => arm_shared}/barrier/mod.rs | 2 +- .../barrier/not_mclass.rs | 0 .../src/{acle => arm_shared}/barrier/v8.rs | 2 +- .../crates/core_arch/src/arm_shared/crc.rs | 133 + .../src/{aarch64 => arm_shared}/crypto.rs | 161 +- .../src/{acle => arm_shared}/hints.rs | 54 +- .../core_arch/src/{acle => arm_shared}/mod.rs | 80 +- .../src/arm_shared/neon/generated.rs | 44738 +++ .../src/arm_shared/neon/load_tests.rs | 206 + .../core_arch/src/arm_shared/neon/mod.rs | 12503 + .../arm_shared/neon/shift_and_insert_tests.rs | 93 + .../src/arm_shared/neon/store_tests.rs | 389 + .../neon}/table_lookup_tests.rs | 0 .../{acle => arm_shared}/registers/aarch32.rs | 0 .../src/{acle => arm_shared}/registers/mod.rs | 8 +- .../src/{acle => arm_shared}/registers/v6m.rs | 0 .../src/{acle => arm_shared}/registers/v7m.rs | 0 .../core_arch/src/arm_shared/test_support.rs | 836 + .../crates/core_arch/src/core_arch_docs.md | 32 +- .../lib/stdarch/crates/core_arch/src/lib.rs | 31 +- .../stdarch/crates/core_arch/src/macros.rs | 503 +- .../stdarch/crates/core_arch/src/mips/mod.rs | 4 + .../stdarch/crates/core_arch/src/mips/msa.rs | 2188 +- .../crates/core_arch/src/mips/msa/macros.rs | 4352 +- .../lib/stdarch/crates/core_arch/src/mod.rs | 204 +- .../crates/core_arch/src/powerpc/altivec.rs | 49 +- .../crates/core_arch/src/powerpc/vsx.rs | 25 +- .../crates/core_arch/src/riscv64/mod.rs | 49 + .../crates/core_arch/src/riscv_shared/mod.rs | 760 + .../crates/core_arch/src/riscv_shared/p.rs | 1061 + .../lib/stdarch/crates/core_arch/src/simd.rs | 1056 +- .../stdarch/crates/core_arch/src/simd_llvm.rs | 21 +- .../crates/core_arch/src/wasm32/atomic.rs | 73 +- .../crates/core_arch/src/wasm32/memory.rs | 44 +- .../crates/core_arch/src/wasm32/mod.rs | 15 +- .../crates/core_arch/src/wasm32/simd128.rs | 6728 +- .../stdarch/crates/core_arch/src/x86/adx.rs | 6 +- .../stdarch/crates/core_arch/src/x86/aes.rs | 18 +- .../stdarch/crates/core_arch/src/x86/avx.rs | 1020 +- .../stdarch/crates/core_arch/src/x86/avx2.rs | 1943 +- .../crates/core_arch/src/x86/avx512bf16.rs | 1573 + .../crates/core_arch/src/x86/avx512bitalg.rs | 760 + .../crates/core_arch/src/x86/avx512bw.rs | 18769 ++ .../crates/core_arch/src/x86/avx512cd.rs | 1170 + .../crates/core_arch/src/x86/avx512f.rs | 55898 +++- .../crates/core_arch/src/x86/avx512gfni.rs | 1492 + .../crates/core_arch/src/x86/avx512ifma.rs | 2 +- .../crates/core_arch/src/x86/avx512vaes.rs | 332 + .../crates/core_arch/src/x86/avx512vbmi.rs | 916 + .../crates/core_arch/src/x86/avx512vbmi2.rs | 4121 + .../crates/core_arch/src/x86/avx512vnni.rs | 939 + .../core_arch/src/x86/avx512vpclmulqdq.rs | 258 + .../core_arch/src/x86/avx512vpopcntdq.rs | 541 + .../stdarch/crates/core_arch/src/x86/bt.rs | 73 +- .../stdarch/crates/core_arch/src/x86/cpuid.rs | 76 +- .../crates/core_arch/src/x86/eflags.rs | 26 +- .../stdarch/crates/core_arch/src/x86/f16c.rs | 56 +- .../stdarch/crates/core_arch/src/x86/fma.rs | 2 +- .../stdarch/crates/core_arch/src/x86/fxsr.rs | 8 +- .../crates/core_arch/src/x86/macros.rs | 155 +- .../stdarch/crates/core_arch/src/x86/mmx.rs | 795 - .../stdarch/crates/core_arch/src/x86/mod.rs | 426 +- .../crates/core_arch/src/x86/pclmulqdq.rs | 34 +- .../stdarch/crates/core_arch/src/x86/rtm.rs | 22 +- .../stdarch/crates/core_arch/src/x86/sha.rs | 27 +- .../stdarch/crates/core_arch/src/x86/sse.rs | 1102 +- .../stdarch/crates/core_arch/src/x86/sse2.rs | 852 +- .../stdarch/crates/core_arch/src/x86/sse3.rs | 12 +- .../stdarch/crates/core_arch/src/x86/sse41.rs | 376 +- .../stdarch/crates/core_arch/src/x86/sse42.rs | 334 +- .../stdarch/crates/core_arch/src/x86/sse4a.rs | 2 +- .../stdarch/crates/core_arch/src/x86/ssse3.rs | 451 +- .../stdarch/crates/core_arch/src/x86/test.rs | 101 +- .../stdarch/crates/core_arch/src/x86/xsave.rs | 23 +- .../crates/core_arch/src/x86_64/adx.rs | 6 +- .../crates/core_arch/src/x86_64/avx.rs | 9 +- .../crates/core_arch/src/x86_64/avx2.rs | 12 +- .../crates/core_arch/src/x86_64/avx512f.rs | 12346 + .../stdarch/crates/core_arch/src/x86_64/bt.rs | 73 +- .../crates/core_arch/src/x86_64/cmpxchg16b.rs | 34 +- .../crates/core_arch/src/x86_64/fxsr.rs | 8 +- .../crates/core_arch/src/x86_64/macros.rs | 36 + .../crates/core_arch/src/x86_64/mod.rs | 6 + .../crates/core_arch/src/x86_64/sse.rs | 11 +- .../crates/core_arch/src/x86_64/sse2.rs | 2 +- .../crates/core_arch/src/x86_64/sse41.rs | 34 +- .../crates/core_arch/src/x86_64/sse42.rs | 2 +- .../crates/core_arch/src/x86_64/xsave.rs | 14 +- .../crates/core_arch/tests/cpu-detection.rs | 2 +- .../stdarch/crates/intrinsic-test/Cargo.toml | 17 + .../stdarch/crates/intrinsic-test/README.md | 24 + .../crates/intrinsic-test/missing_aarch64.txt | 96 + .../crates/intrinsic-test/missing_arm.txt | 334 + .../intrinsic-test/src/acle_csv_parser.rs | 319 + .../crates/intrinsic-test/src/argument.rs | 198 + .../crates/intrinsic-test/src/intrinsic.rs | 134 + .../stdarch/crates/intrinsic-test/src/main.rs | 501 + .../crates/intrinsic-test/src/types.rs | 436 + .../crates/intrinsic-test/src/values.rs | 125 + .../stdarch/crates/simd-test-macro/Cargo.toml | 1 + .../stdarch/crates/simd-test-macro/src/lib.rs | 34 +- .../lib/stdarch/crates/std_detect/Cargo.toml | 19 +- .../lib/stdarch/crates/std_detect/README.md | 8 - .../std_detect/src/detect/arch/aarch64.rs | 165 +- .../crates/std_detect/src/detect/arch/arm.rs | 9 + .../crates/std_detect/src/detect/arch/mips.rs | 1 + .../std_detect/src/detect/arch/mips64.rs | 1 + .../crates/std_detect/src/detect/arch/mod.rs | 56 + .../std_detect/src/detect/arch/powerpc.rs | 1 + .../std_detect/src/detect/arch/powerpc64.rs | 1 + .../std_detect/src/detect/arch/riscv.rs | 206 + .../crates/std_detect/src/detect/arch/x86.rs | 55 +- .../crates/std_detect/src/detect/bit.rs | 2 +- .../crates/std_detect/src/detect/cache.rs | 96 +- .../std_detect/src/detect/error_macros.rs | 150 - .../crates/std_detect/src/detect/macros.rs | 79 +- .../crates/std_detect/src/detect/mod.rs | 64 +- .../std_detect/src/detect/os/aarch64.rs | 36 +- .../src/detect/os/freebsd/aarch64.rs | 4 +- .../src/detect/os/freebsd/auxvec.rs | 12 +- .../std_detect/src/detect/os/freebsd/mod.rs | 12 +- .../std_detect/src/detect/os/linux/aarch64.rs | 323 +- .../std_detect/src/detect/os/linux/arm.rs | 37 +- .../std_detect/src/detect/os/linux/auxvec.rs | 179 +- .../std_detect/src/detect/os/linux/cpuinfo.rs | 49 +- .../std_detect/src/detect/os/linux/mod.rs | 54 +- .../std_detect/src/detect/os/linux/powerpc.rs | 5 +- .../std_detect/src/detect/os/linux/riscv.rs | 73 + .../crates/std_detect/src/detect/os/other.rs | 1 + .../src/detect/os/windows/aarch64.rs | 6 +- .../crates/std_detect/src/detect/os/x86.rs | 26 +- .../test_data/linux-artificial-aarch64.auxv | Bin 0 -> 336 bytes .../test_data/linux-empty-hwcap2-aarch64.auxv | Bin 0 -> 336 bytes .../test_data/linux-hwcap2-aarch64.auxv | Bin 0 -> 336 bytes .../test_data/linux-no-hwcap2-aarch64.auxv | Bin 0 -> 320 bytes .../detect/test_data/linux-x64-i7-6850k.auxv | Bin 304 -> 0 bytes .../lib/stdarch/crates/std_detect/src/lib.rs | 31 +- .../lib/stdarch/crates/std_detect/src/mod.rs | 5 - .../crates/std_detect/tests/cpu-detection.rs | 45 +- .../std_detect/tests/macro_trailing_commas.rs | 2 +- .../crates/std_detect/tests/x86-specific.rs | 2 +- .../lib/stdarch/crates/stdarch-gen/Cargo.toml | 9 + .../lib/stdarch/crates/stdarch-gen/README.md | 11 + .../lib/stdarch/crates/stdarch-gen/neon.spec | 7560 + .../stdarch/crates/stdarch-gen/src/main.rs | 3438 + .../stdarch/crates/stdarch-test/Cargo.toml | 14 +- .../crates/stdarch-test/src/disassembly.rs | 142 +- .../stdarch/crates/stdarch-test/src/lib.rs | 101 +- .../stdarch/crates/stdarch-test/src/wasm.rs | 48 +- .../crates/stdarch-verify/.gitattributes | 1 - .../stdarch/crates/stdarch-verify/src/lib.rs | 134 +- .../crates/stdarch-verify/tests/arm.rs | 292 +- .../crates/stdarch-verify/tests/mips.rs | 108 +- .../crates/stdarch-verify/tests/x86-intel.rs | 202 +- .../crates/stdarch-verify/x86-intel.xml | 211250 ++++++++------- crux-mir/lib/stdarch/examples/Cargo.toml | 11 +- crux-mir/lib/stdarch/examples/connect5.rs | 1272 + crux-mir/lib/stdarch/examples/hex.rs | 111 +- crux-mir/lib/stdarch/examples/wasm.rs | 6 +- crux-mir/lib/stdarch/rustfmt.toml | 0 crux-mir/lib/stdarch/triagebot.toml | 1 + crux-mir/lib/term/Cargo.toml | 12 - crux-mir/lib/term/src/lib.rs | 198 - crux-mir/lib/term/src/terminfo/mod.rs | 246 - crux-mir/lib/term/src/terminfo/parm.rs | 559 - crux-mir/lib/term/src/terminfo/parm/tests.rs | 138 - .../lib/term/src/terminfo/parser/compiled.rs | 336 - .../src/terminfo/parser/compiled/tests.rs | 8 - crux-mir/lib/term/src/terminfo/searcher.rs | 69 - .../lib/term/src/terminfo/searcher/tests.rs | 19 - crux-mir/lib/term/src/win.rs | 220 - crux-mir/lib/test/Cargo.toml | 31 - crux-mir/lib/test/src/bench.rs | 238 - crux-mir/lib/test/src/cli.rs | 431 - crux-mir/lib/test/src/console.rs | 293 - crux-mir/lib/test/src/event.rs | 36 - crux-mir/lib/test/src/formatters/json.rs | 246 - crux-mir/lib/test/src/formatters/mod.rs | 40 - crux-mir/lib/test/src/formatters/pretty.rs | 281 - crux-mir/lib/test/src/formatters/terse.rs | 258 - crux-mir/lib/test/src/helpers/concurrency.rs | 149 - crux-mir/lib/test/src/helpers/exit_code.rs | 20 - crux-mir/lib/test/src/helpers/isatty.rs | 34 - crux-mir/lib/test/src/helpers/metrics.rs | 50 - crux-mir/lib/test/src/helpers/mod.rs | 8 - crux-mir/lib/test/src/helpers/sink.rs | 24 - crux-mir/lib/test/src/lib.rs | 645 - crux-mir/lib/test/src/options.rs | 87 - crux-mir/lib/test/src/stats.rs | 319 - crux-mir/lib/test/src/stats/tests.rs | 592 - crux-mir/lib/test/src/test_result.rs | 115 - crux-mir/lib/test/src/tests.rs | 688 - crux-mir/lib/test/src/time.rs | 193 - crux-mir/lib/test/src/types.rs | 145 - crux-mir/lib/unicode-width/COPYRIGHT | 7 - crux-mir/lib/unicode-width/Cargo.toml | 29 - crux-mir/lib/unicode-width/README.md | 58 - crux-mir/lib/unicode-width/scripts/unicode.py | 321 - crux-mir/lib/unicode-width/src/lib.rs | 131 - crux-mir/lib/unicode-width/src/tables.rs | 284 - crux-mir/lib/unicode-width/src/tests.rs | 175 - crux-mir/lib/unwind/Cargo.toml | 24 +- crux-mir/lib/unwind/build.rs | 125 +- crux-mir/lib/unwind/lib.rs | 29 - crux-mir/lib/unwind/libunwind.rs | 279 - crux-mir/lib/unwind/src/build.rs | 124 - crux-mir/lib/unwind/src/lib.rs | 118 +- crux-mir/lib/unwind/src/libunwind.rs | 90 +- crux-mir/src/Mir/Generate.hs | 29 +- crux-mir/src/Mir/Language.hs | 6 +- crux-mir/test/conc_eval/array/arg.rs | 2 +- crux-mir/test/conc_eval/array/clone.rs | 2 +- crux-mir/test/conc_eval/array/const.rs | 4 +- crux-mir/test/conc_eval/array/const_impl.rs | 2 +- crux-mir/test/conc_eval/array/from_slice.rs | 2 +- crux-mir/test/conc_eval/array/iter.rs | 2 +- crux-mir/test/conc_eval/array/mk_and_proj.rs | 2 +- crux-mir/test/conc_eval/array/mut_arg.rs | 2 +- crux-mir/test/conc_eval/array/mut_index.rs | 2 +- crux-mir/test/conc_eval/array/wick1.rs | 2 +- crux-mir/test/conc_eval/array/wick2.rs | 2 +- crux-mir/test/conc_eval/array/wick3.rs | 6 +- crux-mir/test/conc_eval/box/mut.rs | 2 +- crux-mir/test/conc_eval/box/mut_ref.rs | 2 +- crux-mir/test/conc_eval/box/new.rs | 2 +- crux-mir/test/conc_eval/box/struct.rs | 2 +- crux-mir/test/conc_eval/box/unsize.rs | 2 +- crux-mir/test/conc_eval/cell/cell.rs | 2 +- crux-mir/test/conc_eval/cell/ref_cell.rs | 2 +- crux-mir/test/conc_eval/cell/ref_cell2.rs | 2 +- crux-mir/test/conc_eval/clos/as_fn_ptr.rs | 2 +- crux-mir/test/conc_eval/clos/conv_fnmut_fn.rs | 2 +- .../test/conc_eval/clos/conv_fnonce_fn.rs | 2 +- .../test/conc_eval/clos/conv_fnonce_fnmut.rs | 2 +- crux-mir/test/conc_eval/clos/direct_fn.rs | 2 +- crux-mir/test/conc_eval/clos/direct_fnmut.rs | 2 +- crux-mir/test/conc_eval/clos/direct_fnmut2.rs | 2 +- crux-mir/test/conc_eval/clos/direct_fnonce.rs | 2 +- .../test/conc_eval/clos/dispatch_fnmut.rs | 2 +- crux-mir/test/conc_eval/clos/fn_dyn.rs | 2 +- crux-mir/test/conc_eval/clos/fn_static.rs | 2 +- .../test/conc_eval/clos/fn_static_poly.rs | 2 +- crux-mir/test/conc_eval/clos/fnonce.rs | 2 +- crux-mir/test/conc_eval/clos/fnonce1.rs | 2 +- crux-mir/test/conc_eval/clos/fnptr_closure.rs | 2 +- crux-mir/test/conc_eval/clos/fnptr_fn.rs | 2 +- crux-mir/test/conc_eval/clos/fnptr_fnmut.rs | 2 +- crux-mir/test/conc_eval/clos/fnptr_fnonce.rs | 2 +- crux-mir/test/conc_eval/clos/fo.rs | 4 +- crux-mir/test/conc_eval/clos/poly.rs | 2 +- crux-mir/test/conc_eval/clos/promoted.rs | 2 +- crux-mir/test/conc_eval/clos/ref_fnmut.rs | 2 +- crux-mir/test/conc_eval/clos/unique_borrow.rs | 2 +- crux-mir/test/conc_eval/consts/enum_val.rs | 2 +- crux-mir/test/conc_eval/consts/fn_def.rs | 2 +- crux-mir/test/conc_eval/consts/local_key.rs | 2 +- crux-mir/test/conc_eval/consts/struct_unit.rs | 2 +- crux-mir/test/conc_eval/consts/struct_val.rs | 2 +- crux-mir/test/conc_eval/crypto/add.rs | 8 +- crux-mir/test/conc_eval/crypto/add_noL.rs | 10 +- crux-mir/test/conc_eval/dyn/assoc_ty.rs | 2 +- crux-mir/test/conc_eval/dyn/inherit.rs | 2 +- crux-mir/test/conc_eval/dyn/plain_trait.rs | 2 +- crux-mir/test/conc_eval/dyn/trait_param.rs | 2 +- crux-mir/test/conc_eval/enum/arg.rs | 2 +- crux-mir/test/conc_eval/enum/arg2.rs | 2 +- crux-mir/test/conc_eval/enum/cmp.rs | 4 +- crux-mir/test/conc_eval/enum/cow.rs | 2 +- crux-mir/test/conc_eval/enum/eq.rs | 5 +- crux-mir/test/conc_eval/enum/field_order.rs | 2 +- crux-mir/test/conc_eval/enum/inner.rs | 2 +- crux-mir/test/conc_eval/enum/match.rs | 2 +- crux-mir/test/conc_eval/enum/mixed_discrs.rs | 2 +- crux-mir/test/conc_eval/enum/ret.rs | 2 +- crux-mir/test/conc_eval/fnptr/call.rs | 2 +- crux-mir/test/conc_eval/fnptr/custom.rs | 2 +- crux-mir/test/conc_eval/fnptr/field.rs | 2 +- crux-mir/test/conc_eval/fnptr/make.rs | 2 +- .../test/conc_eval/hash_map/insert_get.rs | 2 +- .../test/conc_eval/hash_map/insert_iter.rs | 2 +- .../test/conc_eval/hash_map/insert_multi.rs | 2 +- .../test/conc_eval/hash_map/insert_remove.rs | 2 +- crux-mir/test/conc_eval/impl/self.rs | 2 +- crux-mir/test/conc_eval/impl/self_mut.rs | 2 +- crux-mir/test/conc_eval/impl/simple.rs | 2 +- crux-mir/test/conc_eval/intTest/test0038.rs | 2 +- crux-mir/test/conc_eval/intTest/test0039.rs | 2 +- crux-mir/test/conc_eval/io/cursor_read.rs | 2 +- crux-mir/test/conc_eval/io/cursor_write.rs | 2 +- crux-mir/test/conc_eval/io/cursor_write2.rs | 2 +- crux-mir/test/conc_eval/iter/cloned.rs | 2 +- crux-mir/test/conc_eval/iter/filter_chain.rs | 2 +- crux-mir/test/conc_eval/iter/for.rs | 2 +- crux-mir/test/conc_eval/iter/from_fn.rs | 2 +- crux-mir/test/conc_eval/iter/loop.rs | 4 +- crux-mir/test/conc_eval/iter/peek.rs | 2 +- crux-mir/test/conc_eval/iter/sum.rs | 2 +- crux-mir/test/conc_eval/iter/zip.rs | 2 +- crux-mir/test/conc_eval/mem/maybe_uninit.rs | 2 +- .../conc_eval/mem/maybe_uninit_array_cast.rs | 2 +- crux-mir/test/conc_eval/num/from_bytes.rs | 2 +- crux-mir/test/conc_eval/num/overflow.rs | 2 +- crux-mir/test/conc_eval/num/saturate.rs | 2 +- crux-mir/test/conc_eval/ops/arith1.rs | 2 +- crux-mir/test/conc_eval/ops/deref1.rs | 2 +- crux-mir/test/conc_eval/ops/deref2.rs | 2 +- crux-mir/test/conc_eval/ops/deref3.rs | 2 +- crux-mir/test/conc_eval/ops/index1.rs | 2 +- crux-mir/test/conc_eval/ops/index2.rs | 2 +- crux-mir/test/conc_eval/ops/index3.rs | 2 +- crux-mir/test/conc_eval/prim/add1.rs | 2 +- crux-mir/test/conc_eval/prim/bool.rs | 2 +- crux-mir/test/conc_eval/prim/char_from_u32.rs | 2 +- crux-mir/test/conc_eval/prim/div.rs | 2 +- crux-mir/test/conc_eval/prim/ffs.rs | 2 +- crux-mir/test/conc_eval/prim/ge.rs | 2 +- crux-mir/test/conc_eval/prim/lit.rs | 2 +- crux-mir/test/conc_eval/prim/litbstring.rs | 2 +- crux-mir/test/conc_eval/prim/litstring.rs | 2 +- crux-mir/test/conc_eval/prim/mut.rs | 8 +- crux-mir/test/conc_eval/prim/mut_arg.rs | 4 +- crux-mir/test/conc_eval/prim/shift1.rs | 2 +- crux-mir/test/conc_eval/prim/shift2.rs | 2 +- crux-mir/test/conc_eval/prim/shift3.rs | 2 +- crux-mir/test/conc_eval/prim/shift4.rs | 2 +- .../test/conc_eval/prim/shift_exceeding.rs | 2 +- crux-mir/test/conc_eval/prim/wrapping_sub.rs | 6 +- crux-mir/test/conc_eval/ptr/cast_eq.rs | 2 +- crux-mir/test/conc_eval/ptr/coerce_unsized.rs | 2 +- crux-mir/test/conc_eval/ptr/copy.rs | 2 +- crux-mir/test/conc_eval/ptr/dangling_eq.rs | 2 +- crux-mir/test/conc_eval/ptr/is_null.rs | 2 +- crux-mir/test/conc_eval/ptr/is_null_slice.rs | 2 +- crux-mir/test/conc_eval/ptr/null_eq.rs | 2 +- crux-mir/test/conc_eval/ptr/offset.rs | 2 +- crux-mir/test/conc_eval/ptr/offset_from.rs | 2 +- crux-mir/test/conc_eval/ptr/offset_mut.rs | 2 +- crux-mir/test/conc_eval/ptr/read_write.rs | 2 +- crux-mir/test/conc_eval/ptr/struct_eq.rs | 2 +- crux-mir/test/conc_eval/ptr/unsize_slice.rs | 2 +- crux-mir/test/conc_eval/ptr/valid_eq.rs | 2 +- crux-mir/test/conc_eval/refs/fn_ptr.rs | 2 +- crux-mir/test/conc_eval/refs/fn_ptr_mut.rs | 2 +- crux-mir/test/conc_eval/refs/imm_arg.rs | 2 +- crux-mir/test/conc_eval/refs/imm_raw.rs | 2 +- crux-mir/test/conc_eval/refs/imm_ref.rs | 2 +- crux-mir/test/conc_eval/refs/mut_arg.rs | 2 +- crux-mir/test/conc_eval/refs/mut_nested.rs | 2 +- crux-mir/test/conc_eval/refs/mut_raw.rs | 2 +- crux-mir/test/conc_eval/refs/mut_ref.rs | 2 +- .../test/conc_eval/refs/mut_tuple_field.rs | 2 +- crux-mir/test/conc_eval/refs/never.rs | 2 +- crux-mir/test/conc_eval/refs/never_mut.rs | 2 +- crux-mir/test/conc_eval/refs/promoted_imm.rs | 2 +- crux-mir/test/conc_eval/refs/promoted_mut.rs | 2 +- crux-mir/test/conc_eval/refs/static_mut.rs | 2 +- crux-mir/test/conc_eval/refs/temp.rs | 2 +- crux-mir/test/conc_eval/slice/eq.rs | 2 +- crux-mir/test/conc_eval/slice/get.rs | 6 +- crux-mir/test/conc_eval/slice/iter_mut.rs | 2 +- crux-mir/test/conc_eval/slice/last.rs | 2 +- crux-mir/test/conc_eval/slice/len.rs | 2 +- crux-mir/test/conc_eval/slice/mk_and_proj.rs | 2 +- crux-mir/test/conc_eval/slice/mut.rs | 2 +- crux-mir/test/conc_eval/slice/mut_range.rs | 2 +- crux-mir/test/conc_eval/slice/range_len.rs | 2 +- .../test/conc_eval/slice/range_len_mut.rs | 2 +- crux-mir/test/conc_eval/slice/swap.rs | 2 +- .../test/conc_eval/statics/promoted_fn.rs | 2 +- .../test/conc_eval/statics/promoted_static.rs | 2 +- crux-mir/test/conc_eval/stdlib/cvt.rs | 4 +- crux-mir/test/conc_eval/stdlib/default.rs | 2 +- .../test/conc_eval/stdlib/default_impl.rs | 6 +- crux-mir/test/conc_eval/stdlib/option.rs | 6 +- crux-mir/test/conc_eval/stdlib/option2.rs | 4 +- crux-mir/test/conc_eval/stdlib/option3.rs | 2 +- crux-mir/test/conc_eval/stdlib/poly.rs | 2 +- crux-mir/test/conc_eval/stdlib/range.rs | 6 +- crux-mir/test/conc_eval/stdlib/result.rs | 6 +- .../test/conc_eval/stdlib/result_interior.rs | 8 +- crux-mir/test/conc_eval/stdlib/teq.rs | 12 +- crux-mir/test/conc_eval/str/format.rs | 2 +- crux-mir/test/conc_eval/str/format_array.rs | 2 +- crux-mir/test/conc_eval/str/format_hex.rs | 2 +- crux-mir/test/conc_eval/str/format_int.rs | 2 +- crux-mir/test/conc_eval/str/format_struct.rs | 2 +- crux-mir/test/conc_eval/str/string_push.rs | 2 +- crux-mir/test/conc_eval/str/to_owned.rs | 2 +- crux-mir/test/conc_eval/struct/arg.rs | 2 +- crux-mir/test/conc_eval/struct/field_order.rs | 2 +- crux-mir/test/conc_eval/struct/proj.rs | 2 +- .../test/conc_eval/struct/repr_transparent.rs | 2 +- .../struct/repr_transparent_const.rs | 2 +- crux-mir/test/conc_eval/struct/ret.rs | 2 +- crux-mir/test/conc_eval/struct/tup.rs | 2 +- crux-mir/test/conc_eval/sync/arc.rs | 2 +- crux-mir/test/conc_eval/sync/arc_cell.rs | 2 +- crux-mir/test/conc_eval/sync/arc_clone.rs | 2 +- crux-mir/test/conc_eval/sync/atomic_add.rs | 2 +- crux-mir/test/conc_eval/sync/atomic_cxchg.rs | 2 +- crux-mir/test/conc_eval/sync/atomic_fence.rs | 2 +- crux-mir/test/conc_eval/sync/atomic_swap.rs | 2 +- crux-mir/test/conc_eval/sync/mutex.rs | 2 +- crux-mir/test/conc_eval/sync/mutex_multi.rs | 2 +- crux-mir/test/conc_eval/sync/rwlock.rs | 2 +- crux-mir/test/conc_eval/sync/rwlock_multi.rs | 2 +- crux-mir/test/conc_eval/time/instant.rs | 2 +- crux-mir/test/conc_eval/traits/assoc1.rs | 2 +- crux-mir/test/conc_eval/traits/assoc2.rs | 2 +- crux-mir/test/conc_eval/traits/assoc3.rs | 2 +- crux-mir/test/conc_eval/traits/basics1.rs | 2 +- crux-mir/test/conc_eval/traits/bounds1.rs | 2 +- crux-mir/test/conc_eval/traits/bounds2.rs | 2 +- crux-mir/test/conc_eval/traits/bounds3.rs | 2 +- crux-mir/test/conc_eval/traits/bounds4.rs | 2 +- crux-mir/test/conc_eval/traits/bounds5.rs | 2 +- crux-mir/test/conc_eval/traits/conv.rs | 4 +- crux-mir/test/conc_eval/traits/default.rs | 2 +- crux-mir/test/conc_eval/traits/dict_med.rs | 4 +- crux-mir/test/conc_eval/traits/dict_poly.rs | 2 +- .../test/conc_eval/traits/dict_polymem.rs | 6 +- crux-mir/test/conc_eval/traits/dict_simple.rs | 4 +- .../test/conc_eval/traits/dynamic_branch.rs | 2 +- crux-mir/test/conc_eval/traits/dynamic_med.rs | 2 +- .../test/conc_eval/traits/dynamic_poly.rs | 2 +- .../test/conc_eval/traits/dynamic_simple.rs | 2 +- crux-mir/test/conc_eval/traits/dynamic_two.rs | 2 +- crux-mir/test/conc_eval/traits/gen_trait.rs | 4 +- .../test/conc_eval/traits/gen_trait_poly.rs | 2 +- crux-mir/test/conc_eval/traits/generic1.rs | 2 +- crux-mir/test/conc_eval/traits/generic2.rs | 2 +- crux-mir/test/conc_eval/traits/generic3.rs | 2 +- crux-mir/test/conc_eval/traits/intoiter.rs | 8 +- crux-mir/test/conc_eval/traits/params.rs | 2 +- crux-mir/test/conc_eval/traits/static.rs | 2 +- crux-mir/test/conc_eval/traits/static_eq.rs | 2 +- crux-mir/test/conc_eval/traits/static_self.rs | 2 +- .../test/conc_eval/traits/static_three.rs | 2 +- crux-mir/test/conc_eval/traits/static_two.rs | 2 +- crux-mir/test/conc_eval/traits/subtrait.rs | 2 +- crux-mir/test/conc_eval/traits/subtrait2.rs | 2 +- crux-mir/test/conc_eval/traits/tyfam.rs | 10 +- crux-mir/test/conc_eval/traits/tyfam2.rs | 10 +- crux-mir/test/conc_eval/traits/tyfam3.rs | 8 +- crux-mir/test/conc_eval/traits/tyfam4.rs | 8 +- crux-mir/test/conc_eval/traits/tyfam5.rs | 4 +- crux-mir/test/conc_eval/tuple/clone.rs | 2 +- crux-mir/test/conc_eval/tuple/clone_from.rs | 2 +- crux-mir/test/conc_eval/tuple/clone_rec.rs | 2 +- crux-mir/test/conc_eval/tuple/clone_struct.rs | 2 +- crux-mir/test/conc_eval/vec/collect.rs | 2 +- crux-mir/test/conc_eval/vec/drop.rs | 2 +- crux-mir/test/conc_eval/vec/extend.rs | 2 +- .../test/conc_eval/vec/extend_trusted_len.rs | 2 +- crux-mir/test/conc_eval/vec/from_elem_zero.rs | 2 +- crux-mir/test/conc_eval/vec/push.rs | 2 +- crux-mir/test/conc_eval/vec/set_len.rs | 2 +- .../test/conc_eval/vec_deque/iter_clone.rs | 2 +- crux-mir/test/conc_eval/vec_deque/pop.rs | 2 +- crux-mir/test/conc_eval/vec_deque/push.rs | 2 +- crux-mir/test/conc_eval/vec_deque/retain.rs | 2 +- .../test/conc_eval/vec_deque/rotate_left.rs | 2 +- .../test/conc_eval/vec_deque/rotate_right.rs | 2 +- crux-mir/test/concurrency/atomic/atomic.rs | 2 +- .../test/concurrency/atomic/atomic_fetch.rs | 2 +- crux-mir/test/concurrency/mutex/mutex.rs | 4 +- .../test/concurrency/mutex/mutex_atomic.rs | 2 +- .../test/concurrency/mutex/thread_retval.rs | 2 +- .../concurrency/mutex/thread_retval_mutex.rs | 2 +- .../pthread-finding-k-matches.rs | 2 +- .../sv-benchmarks/c/pthread/bigshot.rs | 6 +- .../sv-benchmarks/c/pthread/fib_bench.rs | 4 +- .../sv-benchmarks/c/pthread/indexer.rs | 2 +- .../sv-benchmarks/c/pthread/queue.rs | 4 +- .../sv-benchmarks/c/pthread/reorder.rs | 4 +- .../sv-benchmarks/c/pthread/singleton.rs | 2 +- .../sv-benchmarks/c/pthread/stack.rs | 4 +- .../sv-benchmarks/c/pthread/stateful.rs | 4 +- .../sv-benchmarks/c/pthread/triangular.rs | 4 +- .../c/pthread/triangular_static.rs | 4 +- .../sv-benchmarks/c/pthread/twostage.rs | 2 +- crux-mir/test/coverage/coverage.rs | 2 +- crux-mir/test/coverage/coverage_cond.rs | 2 +- crux-mir/test/coverage/coverage_dual.rs | 4 +- crux-mir/test/coverage/coverage_loop.rs | 2 +- crux-mir/test/coverage/coverage_macro.rs | 2 +- crux-mir/test/coverage/coverage_match.rs | 2 +- crux-mir/test/coverage/coverage_mono.rs | 2 +- .../test/coverage/coverage_shortcircuit.rs | 2 +- crux-mir/test/coverage/coverage_try.rs | 2 +- .../test/symb_eval/alloc/out_of_bounds.rs | 2 +- crux-mir/test/symb_eval/alloc/uninit_read.rs | 2 +- crux-mir/test/symb_eval/alloc/valid_read.rs | 2 +- crux-mir/test/symb_eval/alloc/zero_length.rs | 2 +- crux-mir/test/symb_eval/any/downcast.rs | 2 +- crux-mir/test/symb_eval/any/downcast_fail.rs | 2 +- crux-mir/test/symb_eval/array/basic.rs | 2 +- crux-mir/test/symb_eval/array/mux_slice.rs | 2 +- crux-mir/test/symb_eval/array/slice.rs | 2 +- crux-mir/test/symb_eval/array/slice_mut.rs | 2 +- crux-mir/test/symb_eval/bitvector/arith.rs | 2 +- crux-mir/test/symb_eval/bitvector/cmp.rs | 2 +- crux-mir/test/symb_eval/bitvector/from_to.rs | 2 +- .../test/symb_eval/bitvector/leading_zeros.rs | 2 +- crux-mir/test/symb_eval/bitvector/literals.rs | 2 +- .../symb_eval/bitvector/overflowing_sub.rs | 2 +- crux-mir/test/symb_eval/bitvector/symbolic.rs | 2 +- crux-mir/test/symb_eval/byteorder/read.rs | 2 +- crux-mir/test/symb_eval/byteorder/write.rs | 2 +- crux-mir/test/symb_eval/bytes/extend_bytes.rs | 2 +- crux-mir/test/symb_eval/bytes/new.rs | 2 +- crux-mir/test/symb_eval/bytes/put.rs | 2 +- crux-mir/test/symb_eval/bytes/put_overflow.rs | 2 +- crux-mir/test/symb_eval/bytes/split_off.rs | 2 +- crux-mir/test/symb_eval/bytes/split_to.rs | 2 +- crux-mir/test/symb_eval/bytes/sym_len.rs | 2 +- crux-mir/test/symb_eval/concretize/array.rs | 2 +- crux-mir/test/symb_eval/concretize/assert.rs | 2 +- .../test/symb_eval/concretize/assert_ok.rs | 2 +- crux-mir/test/symb_eval/concretize/conc.rs | 2 +- crux-mir/test/symb_eval/concretize/no_conc.rs | 2 +- crux-mir/test/symb_eval/crux/early_fail.rs | 4 +- crux-mir/test/symb_eval/crux/fail_return.rs | 4 +- crux-mir/test/symb_eval/crux/mixed_fail.rs | 8 +- crux-mir/test/symb_eval/crux/multi.rs | 6 +- crux-mir/test/symb_eval/crypto/bytes.rs | 16 +- crux-mir/test/symb_eval/crypto/bytes2.rs | 18 +- crux-mir/test/symb_eval/crypto/double.rs | 2 +- crux-mir/test/symb_eval/crypto/ffs.rs | 2 +- crux-mir/test/symb_eval/enum/mux.rs | 2 +- crux-mir/test/symb_eval/fnptr/mux.rs | 2 +- crux-mir/test/symb_eval/io/vec_cursor_read.rs | 2 +- crux-mir/test/symb_eval/io/vec_write.rs | 2 +- crux-mir/test/symb_eval/mux/array.rs | 2 +- crux-mir/test/symb_eval/mux/array_mut.rs | 2 +- crux-mir/test/symb_eval/num/checked_add.rs | 2 +- crux-mir/test/symb_eval/num/checked_div.rs | 2 +- crux-mir/test/symb_eval/num/checked_mul.rs | 2 +- .../test/symb_eval/num/checked_mul_signed.rs | 2 +- .../test/symb_eval/overrides/bad_symb1.rs | 2 +- .../test/symb_eval/overrides/bad_symb2.rs | 2 +- .../test/symb_eval/overrides/override1.rs | 2 +- .../test/symb_eval/overrides/override2.rs | 2 +- .../test/symb_eval/overrides/override3.rs | 2 +- .../test/symb_eval/overrides/override4.rs | 2 +- .../test/symb_eval/overrides/override5.rs | 2 +- .../test/symb_eval/overrides/override_rust.rs | 2 +- crux-mir/test/symb_eval/refs/mux_init_imm.rs | 2 +- crux-mir/test/symb_eval/refs/mux_init_mut.rs | 2 +- crux-mir/test/symb_eval/scalar/test1.rs | 4 +- .../test/symb_eval/sym_bytes/construct.rs | 2 +- .../test/symb_eval/sym_bytes/deserialize.rs | 2 +- crux-mir/test/symb_eval/vec/clone.rs | 2 +- crux-mir/test/symb_eval/vec/into_iter.rs | 2 +- crux-mir/test/symb_eval/vec/macro.rs | 2 +- crux-mir/test/symb_eval/vec/sort_by_key.rs | 2 +- .../test/symb_eval/vector/as_mut_slice.rs | 2 +- crux-mir/test/symb_eval/vector/as_slice.rs | 2 +- crux-mir/test/symb_eval/vector/concat.rs | 2 +- .../test/symb_eval/vector/copy_from_slice.rs | 2 +- crux-mir/test/symb_eval/vector/mut.rs | 2 +- crux-mir/test/symb_eval/vector/new.rs | 2 +- crux-mir/test/symb_eval/vector/pop.rs | 2 +- crux-mir/test/symb_eval/vector/push.rs | 2 +- crux-mir/test/symb_eval/vector/replicate.rs | 2 +- crux-mir/test/symb_eval/vector/split_at.rs | 2 +- 2760 files changed, 726411 insertions(+), 261759 deletions(-) create mode 100644 crux-mir/build.sh create mode 100644 crux-mir/lib/addr2line/.cargo-ok create mode 100644 crux-mir/lib/addr2line/.cargo_vcs_info.json create mode 100644 crux-mir/lib/addr2line/.gitignore create mode 100644 crux-mir/lib/addr2line/CHANGELOG.md create mode 100644 crux-mir/lib/addr2line/Cargo.toml create mode 100644 crux-mir/lib/addr2line/Cargo.toml.orig rename crux-mir/lib/{backtrace/crates/backtrace-sys => addr2line}/LICENSE-APACHE (100%) rename crux-mir/lib/{getopts => addr2line}/LICENSE-MIT (95%) create mode 100644 crux-mir/lib/addr2line/README.md create mode 100644 crux-mir/lib/addr2line/bench.plot.r create mode 100755 crux-mir/lib/addr2line/benchmark.sh create mode 100644 crux-mir/lib/addr2line/coverage.sh create mode 100644 crux-mir/lib/addr2line/examples/addr2line.rs create mode 100644 crux-mir/lib/addr2line/rustfmt.toml create mode 100644 crux-mir/lib/addr2line/src/function.rs create mode 100644 crux-mir/lib/addr2line/src/lazy.rs create mode 100644 crux-mir/lib/addr2line/src/lib.rs create mode 100644 crux-mir/lib/addr2line/tests/correctness.rs create mode 100644 crux-mir/lib/addr2line/tests/output_equivalence.rs create mode 100644 crux-mir/lib/addr2line/tests/parse.rs create mode 100644 crux-mir/lib/adler/.cargo-ok create mode 100644 crux-mir/lib/adler/.cargo_vcs_info.json create mode 100644 crux-mir/lib/adler/.github/workflows/ci.yml rename crux-mir/lib/{stdarch => adler}/.gitignore (62%) create mode 100644 crux-mir/lib/adler/CHANGELOG.md create mode 100644 crux-mir/lib/adler/Cargo.toml create mode 100644 crux-mir/lib/adler/Cargo.toml.orig create mode 100644 crux-mir/lib/adler/LICENSE-0BSD create mode 100644 crux-mir/lib/adler/LICENSE-APACHE rename crux-mir/lib/{rustc-demangle => adler}/LICENSE-MIT (96%) create mode 100644 crux-mir/lib/adler/README.md create mode 100644 crux-mir/lib/adler/RELEASE_PROCESS.md create mode 100644 crux-mir/lib/adler/benches/bench.rs create mode 100644 crux-mir/lib/adler/src/algo.rs create mode 100644 crux-mir/lib/adler/src/lib.rs create mode 100644 crux-mir/lib/alloc/Cargo.toml create mode 100644 crux-mir/lib/alloc/benches/binary_heap.rs delete mode 100644 crux-mir/lib/alloc/src/Cargo.toml create mode 100644 crux-mir/lib/alloc/src/boxed/thin.rs rename crux-mir/lib/alloc/src/collections/{binary_heap.rs => binary_heap/mod.rs} (58%) rename crux-mir/lib/alloc/{tests/binary_heap.rs => src/collections/binary_heap/tests.rs} (71%) create mode 100644 crux-mir/lib/alloc/src/collections/btree/append.rs create mode 100644 crux-mir/lib/alloc/src/collections/btree/borrow.rs create mode 100644 crux-mir/lib/alloc/src/collections/btree/borrow/tests.rs create mode 100644 crux-mir/lib/alloc/src/collections/btree/dedup_sorted_iter.rs create mode 100644 crux-mir/lib/alloc/src/collections/btree/fix.rs create mode 100644 crux-mir/lib/alloc/src/collections/btree/map/entry.rs create mode 100644 crux-mir/lib/alloc/src/collections/btree/map/tests.rs create mode 100644 crux-mir/lib/alloc/src/collections/btree/mem.rs create mode 100644 crux-mir/lib/alloc/src/collections/btree/merge_iter.rs create mode 100644 crux-mir/lib/alloc/src/collections/btree/node/tests.rs create mode 100644 crux-mir/lib/alloc/src/collections/btree/remove.rs rename crux-mir/lib/alloc/{tests/btree/set.rs => src/collections/btree/set/tests.rs} (55%) create mode 100644 crux-mir/lib/alloc/src/collections/btree/set_val.rs create mode 100644 crux-mir/lib/alloc/src/collections/btree/split.rs delete mode 100644 crux-mir/lib/alloc/src/collections/vec_deque.rs create mode 100644 crux-mir/lib/alloc/src/collections/vec_deque/into_iter.rs create mode 100644 crux-mir/lib/alloc/src/collections/vec_deque/iter.rs create mode 100644 crux-mir/lib/alloc/src/collections/vec_deque/iter_mut.rs create mode 100644 crux-mir/lib/alloc/src/collections/vec_deque/macros.rs create mode 100644 crux-mir/lib/alloc/src/collections/vec_deque/mod.rs create mode 100644 crux-mir/lib/alloc/src/collections/vec_deque/spec_extend.rs create mode 100644 crux-mir/lib/alloc/src/collections/vec_deque/spec_from_iter.rs create mode 100644 crux-mir/lib/alloc/src/ffi/c_str.rs create mode 100644 crux-mir/lib/alloc/src/ffi/c_str/tests.rs create mode 100644 crux-mir/lib/alloc/src/ffi/mod.rs delete mode 100644 crux-mir/lib/alloc/src/prelude/mod.rs delete mode 100644 crux-mir/lib/alloc/src/prelude/v1.rs create mode 100644 crux-mir/lib/alloc/src/slice/tests.rs create mode 100644 crux-mir/lib/alloc/src/task.rs create mode 100644 crux-mir/lib/alloc/src/testing/crash_test.rs create mode 100644 crux-mir/lib/alloc/src/testing/mod.rs create mode 100644 crux-mir/lib/alloc/src/testing/ord_chaos.rs create mode 100644 crux-mir/lib/alloc/src/testing/rng.rs delete mode 100644 crux-mir/lib/alloc/src/vec.rs create mode 100644 crux-mir/lib/alloc/src/vec/cow.rs create mode 100644 crux-mir/lib/alloc/src/vec/drain.rs create mode 100644 crux-mir/lib/alloc/src/vec/drain_filter.rs create mode 100644 crux-mir/lib/alloc/src/vec/in_place_collect.rs create mode 100644 crux-mir/lib/alloc/src/vec/in_place_drop.rs create mode 100644 crux-mir/lib/alloc/src/vec/into_iter.rs create mode 100644 crux-mir/lib/alloc/src/vec/is_zero.rs create mode 100644 crux-mir/lib/alloc/src/vec/mod.rs create mode 100644 crux-mir/lib/alloc/src/vec/partial_eq.rs create mode 100644 crux-mir/lib/alloc/src/vec/set_len_on_drop.rs create mode 100644 crux-mir/lib/alloc/src/vec/spec_extend.rs create mode 100644 crux-mir/lib/alloc/src/vec/spec_from_elem.rs create mode 100644 crux-mir/lib/alloc/src/vec/spec_from_iter.rs create mode 100644 crux-mir/lib/alloc/src/vec/spec_from_iter_nested.rs create mode 100644 crux-mir/lib/alloc/src/vec/splice.rs create mode 100644 crux-mir/lib/alloc/tests/autotraits.rs create mode 100644 crux-mir/lib/alloc/tests/borrow.rs delete mode 100644 crux-mir/lib/alloc/tests/btree/map.rs delete mode 100644 crux-mir/lib/alloc/tests/btree/mod.rs create mode 100644 crux-mir/lib/alloc/tests/btree_set_hash.rs create mode 100644 crux-mir/lib/alloc/tests/c_str.rs create mode 100644 crux-mir/lib/alloc/tests/const_fns.rs create mode 100644 crux-mir/lib/alloc/tests/thin_box.rs create mode 100644 crux-mir/lib/backtrace/.github/workflows/main.yml create mode 100644 crux-mir/lib/backtrace/build.rs create mode 100755 crux-mir/lib/backtrace/ci/debuglink-docker.sh create mode 100755 crux-mir/lib/backtrace/ci/debuglink.sh create mode 100644 crux-mir/lib/backtrace/ci/docker/s390x-unknown-linux-gnu/Dockerfile create mode 100644 crux-mir/lib/backtrace/crates/as-if-std/Cargo.toml create mode 100644 crux-mir/lib/backtrace/crates/as-if-std/build.rs create mode 100644 crux-mir/lib/backtrace/crates/as-if-std/src/lib.rs delete mode 100644 crux-mir/lib/backtrace/crates/backtrace-sys/Cargo.toml delete mode 100644 crux-mir/lib/backtrace/crates/backtrace-sys/build.rs delete mode 100644 crux-mir/lib/backtrace/crates/backtrace-sys/src/lib.rs create mode 100644 crux-mir/lib/backtrace/crates/debuglink/Cargo.toml create mode 100644 crux-mir/lib/backtrace/crates/debuglink/src/main.rs create mode 100644 crux-mir/lib/backtrace/crates/dylib-dep/Cargo.toml create mode 100644 crux-mir/lib/backtrace/crates/dylib-dep/src/lib.rs create mode 100644 crux-mir/lib/backtrace/crates/line-tables-only/Cargo.toml create mode 100644 crux-mir/lib/backtrace/crates/line-tables-only/build.rs create mode 100644 crux-mir/lib/backtrace/crates/line-tables-only/src/callback.c create mode 100644 crux-mir/lib/backtrace/crates/line-tables-only/src/lib.rs create mode 100644 crux-mir/lib/backtrace/crates/macos_frames_test/Cargo.toml create mode 100644 crux-mir/lib/backtrace/crates/macos_frames_test/src/lib.rs create mode 100644 crux-mir/lib/backtrace/crates/macos_frames_test/tests/main.rs rename crux-mir/lib/backtrace/{crates/backtrace-sys => }/src/android-api.c (100%) create mode 100644 crux-mir/lib/backtrace/src/backtrace/miri.rs delete mode 100644 crux-mir/lib/backtrace/src/backtrace/unix_backtrace.rs delete mode 100644 crux-mir/lib/backtrace/src/symbolize/coresymbolication.rs delete mode 100644 crux-mir/lib/backtrace/src/symbolize/dladdr.rs delete mode 100644 crux-mir/lib/backtrace/src/symbolize/dladdr_resolve.rs create mode 100644 crux-mir/lib/backtrace/src/symbolize/gimli/coff.rs create mode 100644 crux-mir/lib/backtrace/src/symbolize/gimli/elf.rs create mode 100644 crux-mir/lib/backtrace/src/symbolize/gimli/libs_dl_iterate_phdr.rs create mode 100644 crux-mir/lib/backtrace/src/symbolize/gimli/libs_haiku.rs create mode 100644 crux-mir/lib/backtrace/src/symbolize/gimli/libs_illumos.rs create mode 100644 crux-mir/lib/backtrace/src/symbolize/gimli/libs_libnx.rs create mode 100644 crux-mir/lib/backtrace/src/symbolize/gimli/libs_macos.rs create mode 100644 crux-mir/lib/backtrace/src/symbolize/gimli/libs_windows.rs create mode 100644 crux-mir/lib/backtrace/src/symbolize/gimli/macho.rs create mode 100644 crux-mir/lib/backtrace/src/symbolize/gimli/mmap_fake.rs create mode 100644 crux-mir/lib/backtrace/src/symbolize/gimli/mmap_unix.rs create mode 100644 crux-mir/lib/backtrace/src/symbolize/gimli/mmap_windows.rs create mode 100644 crux-mir/lib/backtrace/src/symbolize/gimli/stash.rs delete mode 100644 crux-mir/lib/backtrace/src/symbolize/libbacktrace.rs create mode 100644 crux-mir/lib/backtrace/src/symbolize/miri.rs delete mode 100644 crux-mir/lib/bigint/Cargo.toml delete mode 100644 crux-mir/lib/bigint/LICENSE-MIT delete mode 100644 crux-mir/lib/bigint/README.md delete mode 100644 crux-mir/lib/bigint/benches/bigint.rs delete mode 100644 crux-mir/lib/bigint/examples/modular.rs delete mode 100644 crux-mir/lib/bigint/src/lib.rs delete mode 100644 crux-mir/lib/bigint/src/uint.rs delete mode 100644 crux-mir/lib/byteorder/io.rs delete mode 100644 crux-mir/lib/byteorder/lib.rs delete mode 100644 crux-mir/lib/bytes.rs create mode 100644 crux-mir/lib/cfg_if/.cargo-ok create mode 100644 crux-mir/lib/cfg_if/.cargo_vcs_info.json create mode 100644 crux-mir/lib/cfg_if/.github/workflows/main.yml create mode 100644 crux-mir/lib/cfg_if/.gitignore create mode 100644 crux-mir/lib/cfg_if/Cargo.toml rename crux-mir/lib/{cfg-if/Cargo.toml => cfg_if/Cargo.toml.orig} (97%) rename crux-mir/lib/{cfg-if => cfg_if}/LICENSE-APACHE (100%) rename crux-mir/lib/{backtrace/crates/backtrace-sys => cfg_if}/LICENSE-MIT (100%) rename crux-mir/lib/{cfg-if => cfg_if}/README.md (100%) rename crux-mir/lib/{cfg-if => cfg_if}/src/lib.rs (93%) rename crux-mir/lib/{cfg-if => cfg_if}/tests/xcrate.rs (100%) create mode 100644 crux-mir/lib/compiler_builtins/.cargo-ok create mode 100644 crux-mir/lib/compiler_builtins/.cargo_vcs_info.json create mode 100644 crux-mir/lib/compiler_builtins/Cargo.lock create mode 100644 crux-mir/lib/compiler_builtins/Cargo.toml.orig rename crux-mir/lib/compiler_builtins/{LICENSE.TXT => LICENSE.txt} (100%) delete mode 100644 crux-mir/lib/compiler_builtins/PUBLISHING.md delete mode 100644 crux-mir/lib/compiler_builtins/ci/docker/aarch64-unknown-linux-gnu/Dockerfile delete mode 100644 crux-mir/lib/compiler_builtins/ci/docker/arm-unknown-linux-gnueabi/Dockerfile delete mode 100644 crux-mir/lib/compiler_builtins/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile delete mode 100644 crux-mir/lib/compiler_builtins/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile delete mode 100644 crux-mir/lib/compiler_builtins/ci/docker/i586-unknown-linux-gnu/Dockerfile delete mode 100644 crux-mir/lib/compiler_builtins/ci/docker/i686-unknown-linux-gnu/Dockerfile delete mode 100644 crux-mir/lib/compiler_builtins/ci/docker/mips-unknown-linux-gnu/Dockerfile delete mode 100644 crux-mir/lib/compiler_builtins/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile delete mode 100644 crux-mir/lib/compiler_builtins/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile delete mode 100644 crux-mir/lib/compiler_builtins/ci/docker/mipsel-unknown-linux-gnu/Dockerfile delete mode 100644 crux-mir/lib/compiler_builtins/ci/docker/powerpc-unknown-linux-gnu/Dockerfile delete mode 100644 crux-mir/lib/compiler_builtins/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile delete mode 100644 crux-mir/lib/compiler_builtins/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile delete mode 100644 crux-mir/lib/compiler_builtins/ci/docker/thumbv6m-none-eabi/Dockerfile delete mode 100644 crux-mir/lib/compiler_builtins/ci/docker/thumbv7em-none-eabi/Dockerfile delete mode 100644 crux-mir/lib/compiler_builtins/ci/docker/thumbv7em-none-eabihf/Dockerfile delete mode 100644 crux-mir/lib/compiler_builtins/ci/docker/thumbv7m-none-eabi/Dockerfile delete mode 100644 crux-mir/lib/compiler_builtins/ci/docker/x86_64-unknown-linux-gnu/Dockerfile delete mode 100755 crux-mir/lib/compiler_builtins/ci/run-docker.sh delete mode 100755 crux-mir/lib/compiler_builtins/ci/run.sh delete mode 100644 crux-mir/lib/compiler_builtins/crates/panic-handler/Cargo.toml delete mode 100644 crux-mir/lib/compiler_builtins/crates/panic-handler/src/lib.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/acos.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/acosf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/acosh.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/acoshf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/asin.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/asinf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/asinh.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/asinhf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/atan.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/atan2.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/atan2f.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/atanf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/atanh.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/atanhf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/cbrt.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/cbrtf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/ceil.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/ceilf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/copysign.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/copysignf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/cos.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/cosf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/cosh.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/coshf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/erf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/erff.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/exp.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/exp10.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/exp10f.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/exp2.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/exp2f.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/expf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/expm1.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/expm1f.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/expo2.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/fabs.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/fabsf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/fdim.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/fdimf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/fenv.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/floor.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/floorf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/fma.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/fmaf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/fmax.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/fmaxf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/fmin.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/fminf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/fmod.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/fmodf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/frexp.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/frexpf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/hypot.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/hypotf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/ilogb.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/ilogbf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/j0.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/j0f.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/j1.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/j1f.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/jn.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/jnf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/k_cos.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/k_cosf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/k_expo2.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/k_expo2f.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/k_sin.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/k_sinf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/k_tan.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/k_tanf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/ldexp.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/ldexpf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/lgamma.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/lgamma_r.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/lgammaf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/lgammaf_r.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/log.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/log10.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/log10f.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/log1p.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/log1pf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/log2.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/log2f.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/logf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/mod.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/modf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/modff.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/nextafter.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/nextafterf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/pow.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/powf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/rem_pio2.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/rem_pio2_large.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/rem_pio2f.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/remainder.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/remainderf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/remquo.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/remquof.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/rint.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/rintf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/round.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/roundf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/scalbn.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/scalbnf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/sin.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/sincos.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/sincosf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/sinf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/sinh.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/sinhf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/sqrt.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/sqrtf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/tan.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/tanf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/tanh.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/tanhf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/tgamma.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/tgammaf.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/trunc.rs create mode 100644 crux-mir/lib/compiler_builtins/libm/src/math/truncf.rs create mode 100644 crux-mir/lib/compiler_builtins/src/float/trunc.rs create mode 100644 crux-mir/lib/compiler_builtins/src/int/leading_zeros.rs create mode 100644 crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/asymmetric.rs create mode 100644 crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/binary_long.rs create mode 100644 crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/delegate.rs create mode 100644 crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/mod.rs create mode 100644 crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/norm_shift.rs create mode 100644 crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/trifecta.rs delete mode 100644 crux-mir/lib/compiler_builtins/src/mem.rs create mode 100644 crux-mir/lib/compiler_builtins/src/mem/impls.rs create mode 100644 crux-mir/lib/compiler_builtins/src/mem/mod.rs create mode 100644 crux-mir/lib/compiler_builtins/src/mem/x86_64.rs create mode 100644 crux-mir/lib/compiler_builtins/src/riscv.rs delete mode 100644 crux-mir/lib/compiler_builtins/src/riscv32.rs delete mode 100644 crux-mir/lib/compiler_builtins/test.c delete mode 100644 crux-mir/lib/compiler_builtins/testcrate/Cargo.toml delete mode 100644 crux-mir/lib/compiler_builtins/testcrate/build.rs delete mode 100644 crux-mir/lib/compiler_builtins/testcrate/src/lib.rs delete mode 100644 crux-mir/lib/compiler_builtins/testcrate/tests/aeabi_memclr.rs delete mode 100644 crux-mir/lib/compiler_builtins/testcrate/tests/aeabi_memcpy.rs delete mode 100644 crux-mir/lib/compiler_builtins/testcrate/tests/aeabi_memset.rs delete mode 100644 crux-mir/lib/compiler_builtins/testcrate/tests/count_leading_zeros.rs delete mode 100644 crux-mir/lib/compiler_builtins/testcrate/tests/generated.rs delete mode 100644 crux-mir/lib/compiler_builtins/thumbv6m-linux-eabi.json delete mode 100644 crux-mir/lib/compiler_builtins/thumbv7em-linux-eabi.json delete mode 100644 crux-mir/lib/compiler_builtins/thumbv7em-linux-eabihf.json delete mode 100644 crux-mir/lib/compiler_builtins/thumbv7m-linux-eabi.json create mode 100644 crux-mir/lib/core/Cargo.toml create mode 100644 crux-mir/lib/core/benches/ascii/is_ascii.rs create mode 100644 crux-mir/lib/core/benches/num/int_log/mod.rs create mode 100644 crux-mir/lib/core/benches/str.rs create mode 100644 crux-mir/lib/core/benches/str/char_count.rs create mode 100644 crux-mir/lib/core/benches/str/corpora.rs create mode 100644 crux-mir/lib/core/primitive_docs/box_into_raw.md create mode 100644 crux-mir/lib/core/primitive_docs/fs_file.md create mode 100644 crux-mir/lib/core/primitive_docs/io_bufread.md create mode 100644 crux-mir/lib/core/primitive_docs/io_read.md create mode 100644 crux-mir/lib/core/primitive_docs/io_seek.md create mode 100644 crux-mir/lib/core/primitive_docs/io_write.md create mode 100644 crux-mir/lib/core/primitive_docs/net_tosocketaddrs.md create mode 100644 crux-mir/lib/core/primitive_docs/process_exit.md create mode 100644 crux-mir/lib/core/primitive_docs/string_string.md delete mode 100644 crux-mir/lib/core/src/Cargo.toml delete mode 100644 crux-mir/lib/core/src/alloc.rs create mode 100644 crux-mir/lib/core/src/alloc/global.rs create mode 100644 crux-mir/lib/core/src/alloc/layout.rs create mode 100644 crux-mir/lib/core/src/alloc/mod.rs create mode 100644 crux-mir/lib/core/src/arch.rs create mode 100644 crux-mir/lib/core/src/array/equality.rs create mode 100644 crux-mir/lib/core/src/asserting.rs create mode 100644 crux-mir/lib/core/src/async_iter/async_iter.rs create mode 100644 crux-mir/lib/core/src/async_iter/from_iter.rs create mode 100644 crux-mir/lib/core/src/async_iter/mod.rs create mode 100644 crux-mir/lib/core/src/cell/lazy.rs create mode 100644 crux-mir/lib/core/src/cell/once.rs create mode 100644 crux-mir/lib/core/src/const_closure.rs create mode 100644 crux-mir/lib/core/src/error.md create mode 100644 crux-mir/lib/core/src/error.rs rename crux-mir/lib/{std/src/os/raw/char.md => core/src/ffi/c_char.md} (58%) rename crux-mir/lib/{std/src/os/raw/double.md => core/src/ffi/c_double.md} (61%) rename crux-mir/lib/{std/src/os/raw/float.md => core/src/ffi/c_float.md} (62%) rename crux-mir/lib/{std/src/os/raw/int.md => core/src/ffi/c_int.md} (75%) rename crux-mir/lib/{std/src/os/raw/long.md => core/src/ffi/c_long.md} (81%) rename crux-mir/lib/{std/src/os/raw/longlong.md => core/src/ffi/c_longlong.md} (80%) rename crux-mir/lib/{std/src/os/raw/schar.md => core/src/ffi/c_schar.md} (75%) rename crux-mir/lib/{std/src/os/raw/short.md => core/src/ffi/c_short.md} (82%) create mode 100644 crux-mir/lib/core/src/ffi/c_str.rs rename crux-mir/lib/{std/src/os/raw/uchar.md => core/src/ffi/c_uchar.md} (75%) rename crux-mir/lib/{std/src/os/raw/uint.md => core/src/ffi/c_uint.md} (75%) rename crux-mir/lib/{std/src/os/raw/ulong.md => core/src/ffi/c_ulong.md} (79%) rename crux-mir/lib/{std/src/os/raw/ulonglong.md => core/src/ffi/c_ulonglong.md} (77%) rename crux-mir/lib/{std/src/os/raw/ushort.md => core/src/ffi/c_ushort.md} (79%) create mode 100644 crux-mir/lib/core/src/ffi/c_void.md rename crux-mir/lib/core/src/{ffi.rs => ffi/mod.rs} (51%) create mode 100644 crux-mir/lib/core/src/fmt/nofloat.rs create mode 100644 crux-mir/lib/core/src/future/into_future.rs create mode 100644 crux-mir/lib/core/src/future/join.rs create mode 100644 crux-mir/lib/core/src/future/pending.rs create mode 100644 crux-mir/lib/core/src/future/poll_fn.rs create mode 100644 crux-mir/lib/core/src/future/ready.rs create mode 100644 crux-mir/lib/core/src/intrinsics/mir.rs create mode 100644 crux-mir/lib/core/src/iter/adapters/array_chunks.rs create mode 100644 crux-mir/lib/core/src/iter/adapters/by_ref_sized.rs create mode 100644 crux-mir/lib/core/src/iter/adapters/cloned.rs create mode 100644 crux-mir/lib/core/src/iter/adapters/copied.rs create mode 100644 crux-mir/lib/core/src/iter/adapters/cycle.rs create mode 100644 crux-mir/lib/core/src/iter/adapters/enumerate.rs create mode 100644 crux-mir/lib/core/src/iter/adapters/filter.rs create mode 100644 crux-mir/lib/core/src/iter/adapters/filter_map.rs create mode 100644 crux-mir/lib/core/src/iter/adapters/fuse.rs create mode 100644 crux-mir/lib/core/src/iter/adapters/inspect.rs create mode 100644 crux-mir/lib/core/src/iter/adapters/intersperse.rs create mode 100644 crux-mir/lib/core/src/iter/adapters/map.rs create mode 100644 crux-mir/lib/core/src/iter/adapters/map_while.rs create mode 100644 crux-mir/lib/core/src/iter/adapters/peekable.rs create mode 100644 crux-mir/lib/core/src/iter/adapters/rev.rs create mode 100644 crux-mir/lib/core/src/iter/adapters/scan.rs create mode 100644 crux-mir/lib/core/src/iter/adapters/skip.rs create mode 100644 crux-mir/lib/core/src/iter/adapters/skip_while.rs create mode 100644 crux-mir/lib/core/src/iter/adapters/step_by.rs create mode 100644 crux-mir/lib/core/src/iter/adapters/take.rs create mode 100644 crux-mir/lib/core/src/iter/adapters/take_while.rs create mode 100644 crux-mir/lib/core/src/iter/sources/empty.rs create mode 100644 crux-mir/lib/core/src/iter/sources/from_fn.rs create mode 100644 crux-mir/lib/core/src/iter/sources/from_generator.rs create mode 100644 crux-mir/lib/core/src/iter/sources/once.rs create mode 100644 crux-mir/lib/core/src/iter/sources/once_with.rs create mode 100644 crux-mir/lib/core/src/iter/sources/repeat.rs create mode 100644 crux-mir/lib/core/src/iter/sources/repeat_n.rs create mode 100644 crux-mir/lib/core/src/iter/sources/repeat_with.rs create mode 100644 crux-mir/lib/core/src/iter/sources/successors.rs create mode 100644 crux-mir/lib/core/src/mem/transmutability.rs delete mode 100644 crux-mir/lib/core/src/num/dec2flt/algorithm.rs create mode 100644 crux-mir/lib/core/src/num/dec2flt/common.rs create mode 100644 crux-mir/lib/core/src/num/dec2flt/decimal.rs create mode 100644 crux-mir/lib/core/src/num/dec2flt/float.rs create mode 100644 crux-mir/lib/core/src/num/dec2flt/fpu.rs create mode 100644 crux-mir/lib/core/src/num/dec2flt/lemire.rs delete mode 100644 crux-mir/lib/core/src/num/dec2flt/num.rs create mode 100644 crux-mir/lib/core/src/num/dec2flt/number.rs delete mode 100644 crux-mir/lib/core/src/num/dec2flt/rawfp.rs create mode 100644 crux-mir/lib/core/src/num/dec2flt/slow.rs create mode 100644 crux-mir/lib/core/src/num/error.rs create mode 100644 crux-mir/lib/core/src/num/fmt.rs delete mode 100644 crux-mir/lib/core/src/num/i128.rs delete mode 100644 crux-mir/lib/core/src/num/i16.rs delete mode 100644 crux-mir/lib/core/src/num/i32.rs delete mode 100644 crux-mir/lib/core/src/num/i64.rs delete mode 100644 crux-mir/lib/core/src/num/i8.rs create mode 100644 crux-mir/lib/core/src/num/int_log10.rs delete mode 100644 crux-mir/lib/core/src/num/isize.rs create mode 100644 crux-mir/lib/core/src/num/nonzero.rs create mode 100644 crux-mir/lib/core/src/num/saturating.rs create mode 100644 crux-mir/lib/core/src/num/shells/i128.rs create mode 100644 crux-mir/lib/core/src/num/shells/i16.rs create mode 100644 crux-mir/lib/core/src/num/shells/i32.rs create mode 100644 crux-mir/lib/core/src/num/shells/i64.rs create mode 100644 crux-mir/lib/core/src/num/shells/i8.rs create mode 100644 crux-mir/lib/core/src/num/shells/int_macros.rs create mode 100644 crux-mir/lib/core/src/num/shells/isize.rs create mode 100644 crux-mir/lib/core/src/num/shells/u128.rs create mode 100644 crux-mir/lib/core/src/num/shells/u16.rs create mode 100644 crux-mir/lib/core/src/num/shells/u32.rs create mode 100644 crux-mir/lib/core/src/num/shells/u64.rs create mode 100644 crux-mir/lib/core/src/num/shells/u8.rs create mode 100644 crux-mir/lib/core/src/num/shells/usize.rs delete mode 100644 crux-mir/lib/core/src/num/u128.rs delete mode 100644 crux-mir/lib/core/src/num/u16.rs delete mode 100644 crux-mir/lib/core/src/num/u32.rs delete mode 100644 crux-mir/lib/core/src/num/u64.rs delete mode 100644 crux-mir/lib/core/src/num/u8.rs create mode 100644 crux-mir/lib/core/src/num/uint_macros.rs delete mode 100644 crux-mir/lib/core/src/num/usize.rs create mode 100644 crux-mir/lib/core/src/ops/control_flow.rs create mode 100644 crux-mir/lib/core/src/ops/index_range.rs delete mode 100644 crux-mir/lib/core/src/ops/try.rs create mode 100644 crux-mir/lib/core/src/ops/try_trait.rs create mode 100644 crux-mir/lib/core/src/panic/location.rs create mode 100644 crux-mir/lib/core/src/panic/panic_info.rs create mode 100644 crux-mir/lib/core/src/panic/unwind_safe.rs create mode 100644 crux-mir/lib/core/src/primitive_docs.rs create mode 100644 crux-mir/lib/core/src/ptr/alignment.rs create mode 100644 crux-mir/lib/core/src/ptr/metadata.rs delete mode 100644 crux-mir/lib/core/src/raw.rs create mode 100644 crux-mir/lib/core/src/slice/ascii.rs create mode 100644 crux-mir/lib/core/src/slice/cmp.rs create mode 100644 crux-mir/lib/core/src/slice/index.rs create mode 100644 crux-mir/lib/core/src/slice/iter.rs create mode 100644 crux-mir/lib/core/src/slice/iter/macros.rs create mode 100644 crux-mir/lib/core/src/slice/raw.rs create mode 100644 crux-mir/lib/core/src/slice/specialize.rs create mode 100644 crux-mir/lib/core/src/str/converts.rs create mode 100644 crux-mir/lib/core/src/str/count.rs create mode 100644 crux-mir/lib/core/src/str/error.rs create mode 100644 crux-mir/lib/core/src/str/iter.rs create mode 100644 crux-mir/lib/core/src/str/traits.rs create mode 100644 crux-mir/lib/core/src/str/validations.rs create mode 100644 crux-mir/lib/core/src/sync/exclusive.rs create mode 100644 crux-mir/lib/core/src/task/ready.rs delete mode 100644 crux-mir/lib/core/src/unicode/version.rs create mode 100644 crux-mir/lib/core/tests/asserting.rs create mode 100644 crux-mir/lib/core/tests/const_ptr.rs create mode 100644 crux-mir/lib/core/tests/convert.rs create mode 100644 crux-mir/lib/core/tests/future.rs delete mode 100644 crux-mir/lib/core/tests/iter.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/array_chunks.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/by_ref_sized.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/chain.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/cloned.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/copied.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/cycle.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/enumerate.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/filter.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/filter_map.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/flat_map.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/flatten.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/fuse.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/inspect.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/intersperse.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/map.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/mod.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/peekable.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/scan.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/skip.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/skip_while.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/step_by.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/take.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/take_while.rs create mode 100644 crux-mir/lib/core/tests/iter/adapters/zip.rs create mode 100644 crux-mir/lib/core/tests/iter/mod.rs create mode 100644 crux-mir/lib/core/tests/iter/range.rs create mode 100644 crux-mir/lib/core/tests/iter/sources.rs create mode 100644 crux-mir/lib/core/tests/iter/traits/accum.rs create mode 100644 crux-mir/lib/core/tests/iter/traits/double_ended.rs create mode 100644 crux-mir/lib/core/tests/iter/traits/iterator.rs create mode 100644 crux-mir/lib/core/tests/iter/traits/mod.rs create mode 100644 crux-mir/lib/core/tests/iter/traits/step.rs create mode 100644 crux-mir/lib/core/tests/lazy.rs create mode 100644 crux-mir/lib/core/tests/macros.rs create mode 100644 crux-mir/lib/core/tests/num/const_from.rs create mode 100644 crux-mir/lib/core/tests/num/dec2flt/float.rs create mode 100644 crux-mir/lib/core/tests/num/dec2flt/lemire.rs delete mode 100644 crux-mir/lib/core/tests/num/dec2flt/rawfp.rs create mode 100644 crux-mir/lib/core/tests/num/i128.rs create mode 100644 crux-mir/lib/core/tests/num/ieee754.rs create mode 100644 crux-mir/lib/core/tests/num/int_log.rs create mode 100644 crux-mir/lib/core/tests/num/nan.rs create mode 100644 crux-mir/lib/core/tests/num/ops.rs create mode 100644 crux-mir/lib/core/tests/num/u128.rs create mode 100644 crux-mir/lib/core/tests/num/wrapping.rs create mode 100644 crux-mir/lib/core/tests/ops/control_flow.rs create mode 100644 crux-mir/lib/core/tests/panic.rs create mode 100644 crux-mir/lib/core/tests/panic/location.rs create mode 100644 crux-mir/lib/core/tests/pin.rs create mode 100644 crux-mir/lib/core/tests/pin_macro.rs create mode 100644 crux-mir/lib/core/tests/simd.rs create mode 100644 crux-mir/lib/core/tests/task.rs create mode 100644 crux-mir/lib/core/tests/unicode.rs create mode 100644 crux-mir/lib/core/tests/waker.rs delete mode 100644 crux-mir/lib/getopts/Cargo.toml delete mode 100644 crux-mir/lib/getopts/README.md delete mode 100644 crux-mir/lib/getopts/src/lib.rs delete mode 100644 crux-mir/lib/getopts/src/tests/mod.rs delete mode 100644 crux-mir/lib/getopts/tests/smoke.rs create mode 100644 crux-mir/lib/gimli/.cargo-ok create mode 100644 crux-mir/lib/gimli/.cargo_vcs_info.json create mode 100644 crux-mir/lib/gimli/.gitignore create mode 100644 crux-mir/lib/gimli/CHANGELOG.md create mode 100644 crux-mir/lib/gimli/CONTRIBUTING.md create mode 100644 crux-mir/lib/gimli/Cargo.toml create mode 100644 crux-mir/lib/gimli/Cargo.toml.orig rename crux-mir/lib/{getopts => gimli}/LICENSE-APACHE (100%) rename crux-mir/lib/{unicode-width => gimli}/LICENSE-MIT (100%) create mode 100644 crux-mir/lib/gimli/README.md create mode 100644 crux-mir/lib/gimli/benches/bench.rs create mode 100644 crux-mir/lib/gimli/examples/dwarf-validate.rs create mode 100644 crux-mir/lib/gimli/examples/dwarfdump.rs create mode 100644 crux-mir/lib/gimli/examples/simple.rs create mode 100644 crux-mir/lib/gimli/examples/simple_line.rs create mode 100644 crux-mir/lib/gimli/fixtures/self/README.md create mode 100644 crux-mir/lib/gimli/fixtures/self/debug_abbrev create mode 100644 crux-mir/lib/gimli/fixtures/self/debug_aranges create mode 100644 crux-mir/lib/gimli/fixtures/self/debug_info create mode 100644 crux-mir/lib/gimli/fixtures/self/debug_inlined create mode 100644 crux-mir/lib/gimli/fixtures/self/debug_line create mode 100644 crux-mir/lib/gimli/fixtures/self/debug_loc create mode 100644 crux-mir/lib/gimli/fixtures/self/debug_pubnames create mode 100644 crux-mir/lib/gimli/fixtures/self/debug_pubtypes create mode 100644 crux-mir/lib/gimli/fixtures/self/debug_ranges create mode 100644 crux-mir/lib/gimli/fixtures/self/debug_str create mode 100644 crux-mir/lib/gimli/fixtures/self/eh_frame create mode 100644 crux-mir/lib/gimli/fixtures/self/eh_frame_hdr create mode 100644 crux-mir/lib/gimli/rustfmt.toml create mode 100644 crux-mir/lib/gimli/src/arch.rs create mode 100644 crux-mir/lib/gimli/src/common.rs create mode 100644 crux-mir/lib/gimli/src/constants.rs create mode 100644 crux-mir/lib/gimli/src/endianity.rs create mode 100644 crux-mir/lib/gimli/src/leb128.rs create mode 100644 crux-mir/lib/gimli/src/lib.rs create mode 100644 crux-mir/lib/gimli/src/read/abbrev.rs create mode 100644 crux-mir/lib/gimli/src/read/addr.rs create mode 100644 crux-mir/lib/gimli/src/read/aranges.rs create mode 100644 crux-mir/lib/gimli/src/read/cfi.rs create mode 100644 crux-mir/lib/gimli/src/read/dwarf.rs create mode 100644 crux-mir/lib/gimli/src/read/endian_reader.rs create mode 100644 crux-mir/lib/gimli/src/read/endian_slice.rs create mode 100644 crux-mir/lib/gimli/src/read/index.rs create mode 100644 crux-mir/lib/gimli/src/read/line.rs create mode 100644 crux-mir/lib/gimli/src/read/lists.rs create mode 100644 crux-mir/lib/gimli/src/read/loclists.rs create mode 100644 crux-mir/lib/gimli/src/read/lookup.rs create mode 100644 crux-mir/lib/gimli/src/read/mod.rs create mode 100644 crux-mir/lib/gimli/src/read/op.rs create mode 100644 crux-mir/lib/gimli/src/read/pubnames.rs create mode 100644 crux-mir/lib/gimli/src/read/pubtypes.rs create mode 100644 crux-mir/lib/gimli/src/read/reader.rs create mode 100644 crux-mir/lib/gimli/src/read/rnglists.rs create mode 100644 crux-mir/lib/gimli/src/read/str.rs create mode 100644 crux-mir/lib/gimli/src/read/unit.rs create mode 100644 crux-mir/lib/gimli/src/read/util.rs create mode 100644 crux-mir/lib/gimli/src/read/value.rs create mode 100644 crux-mir/lib/gimli/src/test_util.rs create mode 100644 crux-mir/lib/gimli/src/write/abbrev.rs create mode 100644 crux-mir/lib/gimli/src/write/cfi.rs create mode 100644 crux-mir/lib/gimli/src/write/dwarf.rs create mode 100644 crux-mir/lib/gimli/src/write/endian_vec.rs create mode 100644 crux-mir/lib/gimli/src/write/line.rs create mode 100644 crux-mir/lib/gimli/src/write/loc.rs create mode 100644 crux-mir/lib/gimli/src/write/mod.rs create mode 100644 crux-mir/lib/gimli/src/write/op.rs create mode 100644 crux-mir/lib/gimli/src/write/range.rs create mode 100644 crux-mir/lib/gimli/src/write/section.rs create mode 100644 crux-mir/lib/gimli/src/write/str.rs create mode 100644 crux-mir/lib/gimli/src/write/unit.rs create mode 100644 crux-mir/lib/gimli/src/write/writer.rs create mode 100644 crux-mir/lib/gimli/tests/convert_self.rs create mode 100755 crux-mir/lib/gimli/tests/parse_self.rs create mode 100644 crux-mir/lib/hashbrown/.cargo-ok create mode 100644 crux-mir/lib/hashbrown/.cargo_vcs_info.json create mode 100644 crux-mir/lib/hashbrown/.gitignore create mode 100644 crux-mir/lib/hashbrown/Cargo.toml.orig create mode 100644 crux-mir/lib/hashbrown/benches/insert_unique_unchecked.rs delete mode 100644 crux-mir/lib/hashbrown/bors.toml delete mode 100644 crux-mir/lib/hashbrown/build.rs delete mode 100644 crux-mir/lib/hashbrown/ci/miri.sh delete mode 100644 crux-mir/lib/hashbrown/ci/run.sh delete mode 100644 crux-mir/lib/hashbrown/ci/tools.sh create mode 100644 crux-mir/lib/hashbrown/src/raw/alloc.rs delete mode 100644 crux-mir/lib/int512.rs create mode 100644 crux-mir/lib/libc/.cargo-ok create mode 100644 crux-mir/lib/libc/.cargo_vcs_info.json create mode 100644 crux-mir/lib/libc/.gitignore create mode 100644 crux-mir/lib/libc/Cargo.toml.orig delete mode 100644 crux-mir/lib/libc/ci/README.md delete mode 100644 crux-mir/lib/libc/ci/android-install-ndk.sh delete mode 100644 crux-mir/lib/libc/ci/android-install-sdk.sh delete mode 100644 crux-mir/lib/libc/ci/android-sysimage.sh delete mode 100644 crux-mir/lib/libc/ci/azure-install-rust.yml delete mode 100644 crux-mir/lib/libc/ci/azure-master.yml delete mode 100644 crux-mir/lib/libc/ci/azure.yml delete mode 100644 crux-mir/lib/libc/ci/build.sh delete mode 100644 crux-mir/lib/libc/ci/docker/aarch64-linux-android/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/aarch64-unknown-linux-gnu/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/aarch64-unknown-linux-musl/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/arm-linux-androideabi/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/arm-unknown-linux-musleabihf/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/asmjs-unknown-emscripten/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/i686-linux-android/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/i686-unknown-linux-gnu/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/i686-unknown-linux-musl/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/mips-unknown-linux-gnu/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/mips-unknown-linux-musl/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/mips64-unknown-linux-muslabi64/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/mips64el-unknown-linux-muslabi64/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/mipsel-unknown-linux-musl/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/powerpc-unknown-linux-gnu/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/s390x-unknown-linux-gnu/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/sparc64-unknown-linux-gnu/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/wasm32-unknown-emscripten/Dockerfile delete mode 100755 crux-mir/lib/libc/ci/docker/wasm32-unknown-emscripten/node-wrapper.sh delete mode 100644 crux-mir/lib/libc/ci/docker/wasm32-wasi/Dockerfile delete mode 100755 crux-mir/lib/libc/ci/docker/wasm32-wasi/clang.sh delete mode 100644 crux-mir/lib/libc/ci/docker/x86_64-linux-android/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/x86_64-rumprun-netbsd/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/x86_64-rumprun-netbsd/runtest.rs delete mode 100644 crux-mir/lib/libc/ci/docker/x86_64-unknown-linux-gnu/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/x86_64-unknown-linux-gnux32/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/docker/x86_64-unknown-linux-musl/Dockerfile delete mode 100644 crux-mir/lib/libc/ci/dox.sh delete mode 100755 crux-mir/lib/libc/ci/emscripten-entry.sh delete mode 100644 crux-mir/lib/libc/ci/emscripten.sh delete mode 100644 crux-mir/lib/libc/ci/install-musl.sh delete mode 100644 crux-mir/lib/libc/ci/ios/deploy_and_run_on_ios_simulator.rs delete mode 100644 crux-mir/lib/libc/ci/linux-s390x.sh delete mode 100644 crux-mir/lib/libc/ci/linux-sparc64.sh delete mode 100755 crux-mir/lib/libc/ci/run-docker.sh delete mode 100644 crux-mir/lib/libc/ci/run-qemu.sh delete mode 100755 crux-mir/lib/libc/ci/run.sh delete mode 100644 crux-mir/lib/libc/ci/runtest-android.rs delete mode 100644 crux-mir/lib/libc/ci/semver.sh delete mode 100644 crux-mir/lib/libc/ci/style.rs delete mode 100644 crux-mir/lib/libc/ci/style.sh delete mode 100644 crux-mir/lib/libc/ci/switch.json delete mode 100644 crux-mir/lib/libc/ci/sysinfo_guard.patch delete mode 100755 crux-mir/lib/libc/ci/test-runner-linux delete mode 100644 crux-mir/lib/libc/libc-test/Cargo.toml delete mode 100644 crux-mir/lib/libc/libc-test/build.rs delete mode 100644 crux-mir/lib/libc/libc-test/src/cmsg.c delete mode 100644 crux-mir/lib/libc/libc-test/src/errqueue.c delete mode 100644 crux-mir/lib/libc/libc-test/test/cmsg.rs delete mode 100644 crux-mir/lib/libc/libc-test/test/errqueue.rs delete mode 100644 crux-mir/lib/libc/libc-test/test/linux_elf.rs delete mode 100644 crux-mir/lib/libc/libc-test/test/linux_fcntl.rs delete mode 100644 crux-mir/lib/libc/libc-test/test/linux_ipv6.rs delete mode 100644 crux-mir/lib/libc/libc-test/test/linux_strerror_r.rs delete mode 100644 crux-mir/lib/libc/libc-test/test/linux_termios.rs delete mode 100644 crux-mir/lib/libc/libc-test/test/main.rs delete mode 100644 crux-mir/lib/libc/src/cloudabi/mod.rs delete mode 100644 crux-mir/lib/libc/src/cloudabi/x86.rs delete mode 100644 crux-mir/lib/libc/src/cloudabi/x86_64.rs create mode 100644 crux-mir/lib/libc/src/psp.rs rename crux-mir/lib/libc/src/{cloudabi => solid}/aarch64.rs (76%) rename crux-mir/lib/libc/src/{cloudabi => solid}/arm.rs (76%) create mode 100644 crux-mir/lib/libc/src/solid/mod.rs create mode 100644 crux-mir/lib/libc/src/unix/bsd/apple/b64/aarch64/align.rs create mode 100644 crux-mir/lib/libc/src/unix/bsd/apple/b64/aarch64/mod.rs create mode 100644 crux-mir/lib/libc/src/unix/bsd/apple/b64/x86_64/align.rs create mode 100644 crux-mir/lib/libc/src/unix/bsd/apple/b64/x86_64/mod.rs rename crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd11/{x86_64.rs => b64.rs} (100%) create mode 100644 crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd12/b64.rs create mode 100644 crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd13/b64.rs create mode 100644 crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd13/mod.rs create mode 100644 crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd13/x86_64.rs create mode 100644 crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd14/b64.rs create mode 100644 crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd14/mod.rs create mode 100644 crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd14/x86_64.rs create mode 100644 crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/powerpc.rs create mode 100644 crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/riscv64.rs create mode 100644 crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/arm.rs create mode 100644 crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/mips64.rs create mode 100644 crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/powerpc.rs create mode 100644 crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/powerpc64.rs create mode 100644 crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/riscv64.rs create mode 100644 crux-mir/lib/libc/src/unix/haiku/native.rs create mode 100644 crux-mir/lib/libc/src/unix/haiku/x86_64.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/android/b64/aarch64/int128.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/android/b64/riscv64/align.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/android/b64/riscv64/mod.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/arch/generic/mod.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/arch/mips/mod.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/arch/mod.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/arch/powerpc/mod.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/arch/sparc/mod.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/m68k/align.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/m68k/mod.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/riscv32/align.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/riscv32/mod.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/aarch64/ilp32.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/aarch64/int128.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/aarch64/lp64.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/loongarch64/align.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/loongarch64/mod.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/riscv64/align.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/riscv32/align.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/riscv32/mod.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/aarch64/int128.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/riscv64/align.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/riscv64/mod.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/s390x.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/non_exhaustive.rs create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/align.rs rename crux-mir/lib/libc/src/unix/{ => linux_like/linux}/uclibc/arm/align.rs (100%) rename crux-mir/lib/libc/src/unix/{ => linux_like/linux}/uclibc/arm/mod.rs (74%) rename crux-mir/lib/libc/src/unix/{ => linux_like/linux}/uclibc/arm/no_align.rs (100%) rename crux-mir/lib/libc/src/unix/{ => linux_like/linux}/uclibc/mips/mips32/align.rs (100%) rename crux-mir/lib/libc/src/unix/{ => linux_like/linux}/uclibc/mips/mips32/mod.rs (94%) rename crux-mir/lib/libc/src/unix/{ => linux_like/linux}/uclibc/mips/mips32/no_align.rs (100%) rename crux-mir/lib/libc/src/unix/{ => linux_like/linux}/uclibc/mips/mips64/align.rs (100%) rename crux-mir/lib/libc/src/unix/{ => linux_like/linux}/uclibc/mips/mips64/mod.rs (96%) rename crux-mir/lib/libc/src/unix/{ => linux_like/linux}/uclibc/mips/mips64/no_align.rs (100%) rename crux-mir/lib/libc/src/unix/{ => linux_like/linux}/uclibc/mips/mod.rs (55%) create mode 100644 crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/mod.rs rename crux-mir/lib/libc/src/unix/{ => linux_like/linux}/uclibc/no_align.rs (100%) rename crux-mir/lib/libc/src/unix/{ => linux_like/linux}/uclibc/x86_64/l4re.rs (91%) rename crux-mir/lib/libc/src/unix/{ => linux_like/linux}/uclibc/x86_64/mod.rs (80%) rename crux-mir/lib/libc/src/unix/{ => linux_like/linux}/uclibc/x86_64/other.rs (100%) create mode 100644 crux-mir/lib/libc/src/unix/newlib/espidf/mod.rs create mode 100644 crux-mir/lib/libc/src/unix/newlib/generic.rs create mode 100644 crux-mir/lib/libc/src/unix/newlib/horizon/mod.rs create mode 100644 crux-mir/lib/libc/src/unix/newlib/powerpc/mod.rs create mode 100644 crux-mir/lib/libc/src/unix/solarish/illumos.rs create mode 100644 crux-mir/lib/libc/src/unix/solarish/solaris.rs create mode 100644 crux-mir/lib/libc/src/unix/solarish/x86.rs create mode 100644 crux-mir/lib/libc/src/unix/solarish/x86_64.rs create mode 100644 crux-mir/lib/libc/src/unix/solarish/x86_common.rs delete mode 100644 crux-mir/lib/libc/src/unix/uclibc/align.rs delete mode 100644 crux-mir/lib/libc/src/unix/uclibc/mod.rs delete mode 100644 crux-mir/lib/libc/src/unix/uclibc/x86_64/align.rs delete mode 100644 crux-mir/lib/libc/src/unix/uclibc/x86_64/no_align.rs mode change 100755 => 100644 crux-mir/lib/libc/src/vxworks/mod.rs rename crux-mir/lib/libc/src/windows/{msvc.rs => msvc/mod.rs} (58%) create mode 100644 crux-mir/lib/memchr/.cargo-ok create mode 100644 crux-mir/lib/memchr/.cargo_vcs_info.json create mode 100644 crux-mir/lib/memchr/.gitignore create mode 100644 crux-mir/lib/memchr/.ignore create mode 100644 crux-mir/lib/memchr/COPYING create mode 100644 crux-mir/lib/memchr/Cargo.toml create mode 100644 crux-mir/lib/memchr/Cargo.toml.orig rename crux-mir/lib/{byteorder => memchr}/LICENSE-MIT (100%) create mode 100644 crux-mir/lib/memchr/README.md create mode 100644 crux-mir/lib/memchr/UNLICENSE create mode 100644 crux-mir/lib/memchr/build.rs create mode 100644 crux-mir/lib/memchr/rustfmt.toml create mode 100755 crux-mir/lib/memchr/scripts/make-byte-frequency-table create mode 100644 crux-mir/lib/memchr/src/cow.rs create mode 100644 crux-mir/lib/memchr/src/lib.rs create mode 100644 crux-mir/lib/memchr/src/memchr/c.rs create mode 100644 crux-mir/lib/memchr/src/memchr/fallback.rs create mode 100644 crux-mir/lib/memchr/src/memchr/iter.rs create mode 100644 crux-mir/lib/memchr/src/memchr/mod.rs create mode 100644 crux-mir/lib/memchr/src/memchr/naive.rs create mode 100644 crux-mir/lib/memchr/src/memchr/x86/avx.rs create mode 100644 crux-mir/lib/memchr/src/memchr/x86/mod.rs create mode 100644 crux-mir/lib/memchr/src/memchr/x86/sse2.rs create mode 100644 crux-mir/lib/memchr/src/memchr/x86/sse42.rs create mode 100644 crux-mir/lib/memchr/src/memmem/byte_frequencies.rs create mode 100644 crux-mir/lib/memchr/src/memmem/genericsimd.rs create mode 100644 crux-mir/lib/memchr/src/memmem/mod.rs create mode 100644 crux-mir/lib/memchr/src/memmem/prefilter/fallback.rs create mode 100644 crux-mir/lib/memchr/src/memmem/prefilter/genericsimd.rs create mode 100644 crux-mir/lib/memchr/src/memmem/prefilter/mod.rs create mode 100644 crux-mir/lib/memchr/src/memmem/prefilter/wasm.rs create mode 100644 crux-mir/lib/memchr/src/memmem/prefilter/x86/avx.rs create mode 100644 crux-mir/lib/memchr/src/memmem/prefilter/x86/mod.rs create mode 100644 crux-mir/lib/memchr/src/memmem/prefilter/x86/sse.rs create mode 100644 crux-mir/lib/memchr/src/memmem/rabinkarp.rs create mode 100644 crux-mir/lib/memchr/src/memmem/rarebytes.rs create mode 100644 crux-mir/lib/memchr/src/memmem/twoway.rs create mode 100644 crux-mir/lib/memchr/src/memmem/util.rs create mode 100644 crux-mir/lib/memchr/src/memmem/vector.rs create mode 100644 crux-mir/lib/memchr/src/memmem/wasm.rs create mode 100644 crux-mir/lib/memchr/src/memmem/x86/avx.rs create mode 100644 crux-mir/lib/memchr/src/memmem/x86/mod.rs create mode 100644 crux-mir/lib/memchr/src/memmem/x86/sse.rs create mode 100644 crux-mir/lib/memchr/src/tests/memchr/iter.rs create mode 100644 crux-mir/lib/memchr/src/tests/memchr/memchr.rs create mode 100644 crux-mir/lib/memchr/src/tests/memchr/mod.rs create mode 100644 crux-mir/lib/memchr/src/tests/memchr/simple.rs create mode 100644 crux-mir/lib/memchr/src/tests/memchr/testdata.rs create mode 100644 crux-mir/lib/memchr/src/tests/mod.rs create mode 100644 crux-mir/lib/memchr/src/tests/x86_64-soft_float.json create mode 100644 crux-mir/lib/miniz_oxide/.cargo-ok create mode 100644 crux-mir/lib/miniz_oxide/.cargo_vcs_info.json create mode 100644 crux-mir/lib/miniz_oxide/Cargo.toml create mode 100644 crux-mir/lib/miniz_oxide/Cargo.toml.orig create mode 100644 crux-mir/lib/miniz_oxide/LICENSE create mode 100644 crux-mir/lib/miniz_oxide/LICENSE-APACHE.md create mode 100644 crux-mir/lib/miniz_oxide/LICENSE-MIT.md create mode 100644 crux-mir/lib/miniz_oxide/LICENSE-ZLIB.md create mode 100644 crux-mir/lib/miniz_oxide/Readme.md create mode 100644 crux-mir/lib/miniz_oxide/src/deflate/buffer.rs create mode 100644 crux-mir/lib/miniz_oxide/src/deflate/core.rs create mode 100644 crux-mir/lib/miniz_oxide/src/deflate/mod.rs create mode 100644 crux-mir/lib/miniz_oxide/src/deflate/stream.rs create mode 100644 crux-mir/lib/miniz_oxide/src/inflate/core.rs create mode 100644 crux-mir/lib/miniz_oxide/src/inflate/mod.rs create mode 100644 crux-mir/lib/miniz_oxide/src/inflate/output_buffer.rs create mode 100644 crux-mir/lib/miniz_oxide/src/inflate/stream.rs create mode 100644 crux-mir/lib/miniz_oxide/src/lib.rs create mode 100644 crux-mir/lib/miniz_oxide/src/shared.rs create mode 100644 crux-mir/lib/object/.cargo-ok create mode 100644 crux-mir/lib/object/.cargo_vcs_info.json create mode 100644 crux-mir/lib/object/.gitignore create mode 100644 crux-mir/lib/object/.gitmodules create mode 100644 crux-mir/lib/object/CHANGELOG.md create mode 100644 crux-mir/lib/object/Cargo.toml create mode 100755 crux-mir/lib/object/Cargo.toml.orig rename crux-mir/lib/{rustc-demangle => object}/LICENSE-APACHE (100%) create mode 100644 crux-mir/lib/object/LICENSE-MIT create mode 100644 crux-mir/lib/object/README.md create mode 100644 crux-mir/lib/object/clippy.toml create mode 100644 crux-mir/lib/object/src/archive.rs create mode 100644 crux-mir/lib/object/src/common.rs create mode 100644 crux-mir/lib/object/src/elf.rs create mode 100644 crux-mir/lib/object/src/endian.rs create mode 100644 crux-mir/lib/object/src/lib.rs create mode 100644 crux-mir/lib/object/src/macho.rs create mode 100644 crux-mir/lib/object/src/pe.rs create mode 100644 crux-mir/lib/object/src/pod.rs create mode 100644 crux-mir/lib/object/src/read/any.rs create mode 100644 crux-mir/lib/object/src/read/archive.rs create mode 100644 crux-mir/lib/object/src/read/coff/comdat.rs create mode 100644 crux-mir/lib/object/src/read/coff/file.rs create mode 100644 crux-mir/lib/object/src/read/coff/mod.rs create mode 100644 crux-mir/lib/object/src/read/coff/relocation.rs create mode 100644 crux-mir/lib/object/src/read/coff/section.rs create mode 100644 crux-mir/lib/object/src/read/coff/symbol.rs create mode 100644 crux-mir/lib/object/src/read/elf/comdat.rs create mode 100644 crux-mir/lib/object/src/read/elf/compression.rs create mode 100644 crux-mir/lib/object/src/read/elf/dynamic.rs create mode 100644 crux-mir/lib/object/src/read/elf/file.rs create mode 100644 crux-mir/lib/object/src/read/elf/hash.rs create mode 100644 crux-mir/lib/object/src/read/elf/mod.rs create mode 100644 crux-mir/lib/object/src/read/elf/note.rs create mode 100644 crux-mir/lib/object/src/read/elf/relocation.rs create mode 100644 crux-mir/lib/object/src/read/elf/section.rs create mode 100644 crux-mir/lib/object/src/read/elf/segment.rs create mode 100644 crux-mir/lib/object/src/read/elf/symbol.rs create mode 100644 crux-mir/lib/object/src/read/elf/version.rs create mode 100644 crux-mir/lib/object/src/read/macho/dyld_cache.rs create mode 100644 crux-mir/lib/object/src/read/macho/fat.rs create mode 100644 crux-mir/lib/object/src/read/macho/file.rs create mode 100644 crux-mir/lib/object/src/read/macho/load_command.rs create mode 100644 crux-mir/lib/object/src/read/macho/mod.rs create mode 100644 crux-mir/lib/object/src/read/macho/relocation.rs create mode 100644 crux-mir/lib/object/src/read/macho/section.rs create mode 100644 crux-mir/lib/object/src/read/macho/segment.rs create mode 100644 crux-mir/lib/object/src/read/macho/symbol.rs create mode 100644 crux-mir/lib/object/src/read/mod.rs create mode 100644 crux-mir/lib/object/src/read/pe/data_directory.rs create mode 100644 crux-mir/lib/object/src/read/pe/export.rs create mode 100644 crux-mir/lib/object/src/read/pe/file.rs create mode 100644 crux-mir/lib/object/src/read/pe/import.rs create mode 100644 crux-mir/lib/object/src/read/pe/mod.rs create mode 100644 crux-mir/lib/object/src/read/pe/relocation.rs create mode 100644 crux-mir/lib/object/src/read/pe/resource.rs create mode 100644 crux-mir/lib/object/src/read/pe/rich.rs create mode 100644 crux-mir/lib/object/src/read/pe/section.rs create mode 100644 crux-mir/lib/object/src/read/read_cache.rs create mode 100644 crux-mir/lib/object/src/read/read_ref.rs create mode 100644 crux-mir/lib/object/src/read/traits.rs create mode 100644 crux-mir/lib/object/src/read/util.rs create mode 100644 crux-mir/lib/object/src/read/wasm.rs create mode 100644 crux-mir/lib/object/src/write/coff.rs create mode 100644 crux-mir/lib/object/src/write/elf/mod.rs create mode 100644 crux-mir/lib/object/src/write/elf/object.rs create mode 100644 crux-mir/lib/object/src/write/elf/writer.rs create mode 100644 crux-mir/lib/object/src/write/macho.rs create mode 100644 crux-mir/lib/object/src/write/mod.rs create mode 100644 crux-mir/lib/object/src/write/pe.rs create mode 100644 crux-mir/lib/object/src/write/string.rs create mode 100644 crux-mir/lib/object/src/write/util.rs create mode 100644 crux-mir/lib/object/tests/integration.rs create mode 100644 crux-mir/lib/object/tests/parse_self.rs create mode 100644 crux-mir/lib/object/tests/read/coff.rs create mode 100644 crux-mir/lib/object/tests/read/mod.rs create mode 100644 crux-mir/lib/object/tests/round_trip/bss.rs create mode 100644 crux-mir/lib/object/tests/round_trip/coff.rs create mode 100644 crux-mir/lib/object/tests/round_trip/comdat.rs create mode 100644 crux-mir/lib/object/tests/round_trip/common.rs create mode 100644 crux-mir/lib/object/tests/round_trip/elf.rs create mode 100644 crux-mir/lib/object/tests/round_trip/macho.rs create mode 100644 crux-mir/lib/object/tests/round_trip/mod.rs create mode 100644 crux-mir/lib/object/tests/round_trip/section_flags.rs create mode 100644 crux-mir/lib/object/tests/round_trip/tls.rs delete mode 100644 crux-mir/lib/panic_abort/lib.rs create mode 100644 crux-mir/lib/panic_abort/src/android.rs rename crux-mir/lib/panic_unwind/{ => src}/dummy.rs (67%) rename crux-mir/lib/panic_unwind/{ => src}/emcc.rs (62%) create mode 100644 crux-mir/lib/panic_unwind/src/gcc.rs rename crux-mir/lib/panic_unwind/{ => src}/hermit.rs (100%) rename crux-mir/lib/panic_unwind/{ => src}/lib.rs (63%) rename crux-mir/lib/panic_unwind/{ => src}/miri.rs (79%) rename crux-mir/lib/panic_unwind/{ => src}/seh.rs (86%) create mode 100644 crux-mir/lib/portable-simd/.github/ISSUE_TEMPLATE/blank_issue.md create mode 100644 crux-mir/lib/portable-simd/.github/ISSUE_TEMPLATE/bug_report.md create mode 100644 crux-mir/lib/portable-simd/.github/ISSUE_TEMPLATE/config.yml create mode 100644 crux-mir/lib/portable-simd/.github/ISSUE_TEMPLATE/feature_request.md create mode 100644 crux-mir/lib/portable-simd/.github/PULL_REQUEST_TEMPLATE.md create mode 100644 crux-mir/lib/portable-simd/.github/workflows/ci.yml create mode 100644 crux-mir/lib/portable-simd/.github/workflows/doc.yml create mode 100644 crux-mir/lib/portable-simd/CONTRIBUTING.md create mode 100644 crux-mir/lib/portable-simd/Cargo.toml create mode 100644 crux-mir/lib/portable-simd/LICENSE-APACHE create mode 100644 crux-mir/lib/portable-simd/LICENSE-MIT create mode 100644 crux-mir/lib/portable-simd/README.md create mode 100644 crux-mir/lib/portable-simd/beginners-guide.md create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/Cargo.toml create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/LICENSE-APACHE create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/LICENSE-MIT create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/examples/matrix_inversion.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/examples/nbody.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/examples/spectral_norm.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/core_simd_docs.md create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/elements.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/elements/float.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/elements/int.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/elements/uint.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/eq.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/fmt.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/intrinsics.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/iter.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/lane_count.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/lib.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/masks.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/masks/bitmask.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/masks/full_masks.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/masks/to_bitmask.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/mod.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/ops.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/ops/assign.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/ops/deref.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/ops/unary.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/ord.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/select.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/swizzle.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/to_bytes.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/vector.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/vector/float.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/vector/int.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/vector/ptr.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/vector/uint.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/vendor.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/vendor/arm.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/vendor/powerpc.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/vendor/wasm32.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/src/vendor/x86.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/autoderef.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/cast.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/f32_ops.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/f64_ops.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/i16_ops.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/i32_ops.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/i64_ops.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/i8_ops.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/isize_ops.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mask16.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mask32.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mask64.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mask8.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mask_macros.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/masksize.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mod.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/masks.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/ops_macros.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/round.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/swizzle.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/to_bytes.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/u16_ops.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/u32_ops.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/u64_ops.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/u8_ops.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/tests/usize_ops.rs create mode 100644 crux-mir/lib/portable-simd/crates/core_simd/webdriver.json create mode 100644 crux-mir/lib/portable-simd/crates/std_float/Cargo.toml create mode 100644 crux-mir/lib/portable-simd/crates/std_float/src/lib.rs create mode 100644 crux-mir/lib/portable-simd/crates/test_helpers/Cargo.toml create mode 100644 crux-mir/lib/portable-simd/crates/test_helpers/src/array.rs create mode 100644 crux-mir/lib/portable-simd/crates/test_helpers/src/biteq.rs create mode 100644 crux-mir/lib/portable-simd/crates/test_helpers/src/lib.rs create mode 100644 crux-mir/lib/portable-simd/crates/test_helpers/src/wasm.rs create mode 100644 crux-mir/lib/proc_macro/Cargo.toml create mode 100644 crux-mir/lib/proc_macro/src/bridge/arena.rs create mode 100644 crux-mir/lib/proc_macro/src/bridge/buffer.rs create mode 100644 crux-mir/lib/proc_macro/src/bridge/client.rs create mode 100644 crux-mir/lib/proc_macro/src/bridge/closure.rs create mode 100644 crux-mir/lib/proc_macro/src/bridge/fxhash.rs create mode 100644 crux-mir/lib/proc_macro/src/bridge/handle.rs create mode 100644 crux-mir/lib/proc_macro/src/bridge/mod.rs create mode 100644 crux-mir/lib/proc_macro/src/bridge/rpc.rs create mode 100644 crux-mir/lib/proc_macro/src/bridge/scoped_cell.rs create mode 100644 crux-mir/lib/proc_macro/src/bridge/selfless_reify.rs create mode 100644 crux-mir/lib/proc_macro/src/bridge/server.rs create mode 100644 crux-mir/lib/proc_macro/src/bridge/symbol.rs create mode 100644 crux-mir/lib/proc_macro/src/diagnostic.rs create mode 100644 crux-mir/lib/proc_macro/src/lib.rs create mode 100644 crux-mir/lib/proc_macro/src/quote.rs delete mode 100644 crux-mir/lib/rustc-demangle/Cargo.toml delete mode 100644 crux-mir/lib/rustc-demangle/README.md delete mode 100644 crux-mir/lib/rustc-demangle/src/lib.rs create mode 100644 crux-mir/lib/rustc_demangle/.cargo-ok create mode 100644 crux-mir/lib/rustc_demangle/.cargo_vcs_info.json create mode 100644 crux-mir/lib/rustc_demangle/.github/dependabot.yml create mode 100644 crux-mir/lib/rustc_demangle/.github/workflows/main.yml create mode 100644 crux-mir/lib/rustc_demangle/.gitignore create mode 100644 crux-mir/lib/rustc_demangle/Cargo.toml create mode 100644 crux-mir/lib/rustc_demangle/Cargo.toml.orig rename crux-mir/lib/{unicode-width => rustc_demangle}/LICENSE-APACHE (100%) rename crux-mir/lib/{cfg-if => rustc_demangle}/LICENSE-MIT (100%) create mode 100644 crux-mir/lib/rustc_demangle/README.md create mode 100644 crux-mir/lib/rustc_demangle/src/legacy.rs create mode 100644 crux-mir/lib/rustc_demangle/src/lib.rs create mode 100644 crux-mir/lib/rustc_demangle/src/v0-large-test-symbols/early-recursion-limit create mode 100644 crux-mir/lib/rustc_demangle/src/v0.rs create mode 100644 crux-mir/lib/rustc_std_workspace_alloc/Cargo.toml create mode 100644 crux-mir/lib/rustc_std_workspace_alloc/lib.rs create mode 100644 crux-mir/lib/rustc_std_workspace_core/Cargo.toml create mode 100644 crux-mir/lib/rustc_std_workspace_core/README.md create mode 100644 crux-mir/lib/rustc_std_workspace_core/lib.rs create mode 100644 crux-mir/lib/std/Cargo.toml create mode 100644 crux-mir/lib/std/build.rs create mode 100644 crux-mir/lib/std/primitive_docs/box_into_raw.md create mode 100644 crux-mir/lib/std/primitive_docs/fs_file.md create mode 100644 crux-mir/lib/std/primitive_docs/io_bufread.md create mode 100644 crux-mir/lib/std/primitive_docs/io_read.md create mode 100644 crux-mir/lib/std/primitive_docs/io_seek.md create mode 100644 crux-mir/lib/std/primitive_docs/io_write.md create mode 100644 crux-mir/lib/std/primitive_docs/net_tosocketaddrs.md create mode 100644 crux-mir/lib/std/primitive_docs/process_exit.md create mode 100644 crux-mir/lib/std/primitive_docs/string_string.md delete mode 100644 crux-mir/lib/std/src/Cargo.toml create mode 100644 crux-mir/lib/std/src/backtrace/tests.rs delete mode 100644 crux-mir/lib/std/src/build.rs delete mode 100644 crux-mir/lib/std/src/collections/hash/crucible_map.rs create mode 100644 crux-mir/lib/std/src/collections/hash/map/tests.rs create mode 100644 crux-mir/lib/std/src/collections/hash/set/tests.rs create mode 100644 crux-mir/lib/std/src/env/tests.rs create mode 100644 crux-mir/lib/std/src/error/tests.rs create mode 100644 crux-mir/lib/std/src/f32/tests.rs create mode 100644 crux-mir/lib/std/src/f64/tests.rs delete mode 100644 crux-mir/lib/std/src/ffi/c_str.rs create mode 100644 crux-mir/lib/std/src/ffi/os_str/tests.rs create mode 100644 crux-mir/lib/std/src/fs/tests.rs delete mode 100644 crux-mir/lib/std/src/future.rs delete mode 100644 crux-mir/lib/std/src/io/buffered.rs create mode 100644 crux-mir/lib/std/src/io/buffered/bufreader.rs create mode 100644 crux-mir/lib/std/src/io/buffered/bufreader/buffer.rs create mode 100644 crux-mir/lib/std/src/io/buffered/bufwriter.rs create mode 100644 crux-mir/lib/std/src/io/buffered/linewriter.rs create mode 100644 crux-mir/lib/std/src/io/buffered/linewritershim.rs create mode 100644 crux-mir/lib/std/src/io/buffered/mod.rs create mode 100644 crux-mir/lib/std/src/io/buffered/tests.rs create mode 100644 crux-mir/lib/std/src/io/copy.rs create mode 100644 crux-mir/lib/std/src/io/cursor/tests.rs create mode 100644 crux-mir/lib/std/src/io/error/repr_bitpacked.rs create mode 100644 crux-mir/lib/std/src/io/error/repr_unpacked.rs create mode 100644 crux-mir/lib/std/src/io/error/tests.rs create mode 100644 crux-mir/lib/std/src/io/impls/tests.rs delete mode 100644 crux-mir/lib/std/src/io/lazy.rs create mode 100644 crux-mir/lib/std/src/io/readbuf.rs create mode 100644 crux-mir/lib/std/src/io/readbuf/tests.rs create mode 100644 crux-mir/lib/std/src/io/stdio/tests.rs create mode 100644 crux-mir/lib/std/src/io/tests.rs create mode 100644 crux-mir/lib/std/src/io/util/tests.rs delete mode 100644 crux-mir/lib/std/src/memchr.rs create mode 100644 crux-mir/lib/std/src/net/display_buffer.rs delete mode 100644 crux-mir/lib/std/src/net/ip.rs create mode 100644 crux-mir/lib/std/src/net/ip_addr.rs create mode 100644 crux-mir/lib/std/src/net/ip_addr/tests.rs create mode 100644 crux-mir/lib/std/src/net/parser/tests.rs rename crux-mir/lib/std/src/net/{addr.rs => socket_addr.rs} (66%) create mode 100644 crux-mir/lib/std/src/net/socket_addr/tests.rs create mode 100644 crux-mir/lib/std/src/net/tcp/tests.rs create mode 100644 crux-mir/lib/std/src/net/udp/tests.rs create mode 100644 crux-mir/lib/std/src/num/benches.rs create mode 100644 crux-mir/lib/std/src/num/tests.rs create mode 100644 crux-mir/lib/std/src/os/android/net.rs create mode 100644 crux-mir/lib/std/src/os/espidf/fs.rs create mode 100644 crux-mir/lib/std/src/os/espidf/mod.rs create mode 100644 crux-mir/lib/std/src/os/espidf/raw.rs create mode 100644 crux-mir/lib/std/src/os/fd/mod.rs create mode 100644 crux-mir/lib/std/src/os/fd/net.rs create mode 100644 crux-mir/lib/std/src/os/fd/owned.rs create mode 100644 crux-mir/lib/std/src/os/fd/raw.rs create mode 100644 crux-mir/lib/std/src/os/fd/tests.rs rename crux-mir/lib/std/src/{sys/sgx/ext => os/fortanix_sgx}/arch.rs (72%) rename crux-mir/lib/std/src/{sys/sgx/ext => os/fortanix_sgx}/ffi.rs (90%) rename crux-mir/lib/std/src/{sys/sgx/ext => os/fortanix_sgx}/io.rs (75%) create mode 100644 crux-mir/lib/std/src/os/hermit/ffi.rs create mode 100644 crux-mir/lib/std/src/os/hermit/mod.rs create mode 100644 crux-mir/lib/std/src/os/horizon/fs.rs create mode 100644 crux-mir/lib/std/src/os/horizon/mod.rs create mode 100644 crux-mir/lib/std/src/os/horizon/raw.rs create mode 100644 crux-mir/lib/std/src/os/illumos/fs.rs create mode 100644 crux-mir/lib/std/src/os/illumos/mod.rs create mode 100644 crux-mir/lib/std/src/os/illumos/raw.rs create mode 100644 crux-mir/lib/std/src/os/l4re/fs.rs create mode 100644 crux-mir/lib/std/src/os/l4re/mod.rs create mode 100644 crux-mir/lib/std/src/os/l4re/raw.rs create mode 100644 crux-mir/lib/std/src/os/linux/net.rs create mode 100644 crux-mir/lib/std/src/os/linux/process.rs create mode 100644 crux-mir/lib/std/src/os/net/linux_ext/addr.rs create mode 100644 crux-mir/lib/std/src/os/net/linux_ext/mod.rs create mode 100644 crux-mir/lib/std/src/os/net/linux_ext/tcp.rs create mode 100644 crux-mir/lib/std/src/os/net/linux_ext/tests.rs create mode 100644 crux-mir/lib/std/src/os/net/mod.rs create mode 100644 crux-mir/lib/std/src/os/raw/tests.rs rename crux-mir/lib/std/src/{sys/vxworks/ext => os/solid}/ffi.rs (74%) create mode 100644 crux-mir/lib/std/src/os/solid/io.rs rename crux-mir/lib/std/src/{sys/vxworks/ext => os/solid}/mod.rs (54%) rename crux-mir/lib/std/src/{sys/unix/ext/ffi.rs => os/unix/ffi/mod.rs} (82%) create mode 100644 crux-mir/lib/std/src/os/unix/ffi/os_str.rs rename crux-mir/lib/std/src/{sys/unix/ext => os/unix}/fs.rs (80%) create mode 100644 crux-mir/lib/std/src/os/unix/io/mod.rs create mode 100644 crux-mir/lib/std/src/os/unix/io/tests.rs create mode 100644 crux-mir/lib/std/src/os/unix/mod.rs create mode 100644 crux-mir/lib/std/src/os/unix/net/addr.rs create mode 100644 crux-mir/lib/std/src/os/unix/net/ancillary.rs create mode 100644 crux-mir/lib/std/src/os/unix/net/datagram.rs create mode 100644 crux-mir/lib/std/src/os/unix/net/listener.rs create mode 100644 crux-mir/lib/std/src/os/unix/net/mod.rs create mode 100644 crux-mir/lib/std/src/os/unix/net/stream.rs create mode 100644 crux-mir/lib/std/src/os/unix/net/tests.rs create mode 100644 crux-mir/lib/std/src/os/unix/process.rs create mode 100644 crux-mir/lib/std/src/os/unix/raw.rs rename crux-mir/lib/std/src/{sys/unix/ext => os/unix}/thread.rs (85%) create mode 100644 crux-mir/lib/std/src/os/unix/ucred.rs create mode 100644 crux-mir/lib/std/src/os/unix/ucred/tests.rs delete mode 100644 crux-mir/lib/std/src/os/wasi.rs create mode 100644 crux-mir/lib/std/src/os/wasi/ffi.rs rename crux-mir/lib/std/src/{sys/wasi/ext => os/wasi}/fs.rs (60%) create mode 100644 crux-mir/lib/std/src/os/wasi/io/fd.rs create mode 100644 crux-mir/lib/std/src/os/wasi/io/fd/tests.rs create mode 100644 crux-mir/lib/std/src/os/wasi/io/mod.rs create mode 100644 crux-mir/lib/std/src/os/wasi/io/raw.rs rename crux-mir/lib/std/src/{sys/unix/ext => os/wasi}/mod.rs (55%) create mode 100644 crux-mir/lib/std/src/os/wasi/net/mod.rs create mode 100644 crux-mir/lib/std/src/os/watchos/fs.rs create mode 100644 crux-mir/lib/std/src/os/watchos/mod.rs create mode 100644 crux-mir/lib/std/src/os/watchos/raw.rs rename crux-mir/lib/std/src/{sys/windows/ext => os/windows}/ffi.rs (80%) rename crux-mir/lib/std/src/{sys/windows/ext => os/windows}/fs.rs (84%) create mode 100644 crux-mir/lib/std/src/os/windows/io/handle.rs create mode 100644 crux-mir/lib/std/src/os/windows/io/mod.rs create mode 100644 crux-mir/lib/std/src/os/windows/io/raw.rs create mode 100644 crux-mir/lib/std/src/os/windows/io/socket.rs create mode 100644 crux-mir/lib/std/src/os/windows/io/tests.rs rename crux-mir/lib/std/src/{sys/windows/ext => os/windows}/mod.rs (72%) create mode 100644 crux-mir/lib/std/src/os/windows/process.rs rename crux-mir/lib/std/src/{sys/windows/ext => os/windows}/raw.rs (84%) rename crux-mir/lib/std/src/{sys/windows/ext => os/windows}/thread.rs (65%) create mode 100644 crux-mir/lib/std/src/panic/tests.rs create mode 100644 crux-mir/lib/std/src/path/tests.rs create mode 100644 crux-mir/lib/std/src/personality.rs rename crux-mir/lib/{panic_unwind => std/src/personality}/dwarf/eh.rs (74%) rename crux-mir/lib/{panic_unwind => std/src/personality}/dwarf/mod.rs (94%) rename crux-mir/lib/{panic_unwind => std/src/personality}/dwarf/tests.rs (100%) create mode 100644 crux-mir/lib/std/src/personality/emcc.rs rename crux-mir/lib/{panic_unwind => std/src/personality}/gcc.rs (52%) create mode 100644 crux-mir/lib/std/src/process/tests.rs create mode 100644 crux-mir/lib/std/src/sync/barrier/tests.rs create mode 100644 crux-mir/lib/std/src/sync/condvar/tests.rs create mode 100644 crux-mir/lib/std/src/sync/lazy_lock.rs create mode 100644 crux-mir/lib/std/src/sync/lazy_lock/tests.rs create mode 100644 crux-mir/lib/std/src/sync/mpmc/array.rs create mode 100644 crux-mir/lib/std/src/sync/mpmc/context.rs create mode 100644 crux-mir/lib/std/src/sync/mpmc/counter.rs create mode 100644 crux-mir/lib/std/src/sync/mpmc/error.rs create mode 100644 crux-mir/lib/std/src/sync/mpmc/list.rs create mode 100644 crux-mir/lib/std/src/sync/mpmc/mod.rs create mode 100644 crux-mir/lib/std/src/sync/mpmc/select.rs create mode 100644 crux-mir/lib/std/src/sync/mpmc/utils.rs create mode 100644 crux-mir/lib/std/src/sync/mpmc/waker.rs create mode 100644 crux-mir/lib/std/src/sync/mpmc/zero.rs delete mode 100644 crux-mir/lib/std/src/sync/mpsc/blocking.rs delete mode 100644 crux-mir/lib/std/src/sync/mpsc/cache_aligned.rs delete mode 100644 crux-mir/lib/std/src/sync/mpsc/mpsc_queue.rs delete mode 100644 crux-mir/lib/std/src/sync/mpsc/oneshot.rs delete mode 100644 crux-mir/lib/std/src/sync/mpsc/shared.rs delete mode 100644 crux-mir/lib/std/src/sync/mpsc/spsc_queue.rs delete mode 100644 crux-mir/lib/std/src/sync/mpsc/stream.rs delete mode 100644 crux-mir/lib/std/src/sync/mpsc/sync.rs create mode 100644 crux-mir/lib/std/src/sync/mpsc/sync_tests.rs create mode 100644 crux-mir/lib/std/src/sync/mpsc/tests.rs create mode 100644 crux-mir/lib/std/src/sync/mutex/tests.rs create mode 100644 crux-mir/lib/std/src/sync/once/tests.rs create mode 100644 crux-mir/lib/std/src/sync/once_lock.rs create mode 100644 crux-mir/lib/std/src/sync/once_lock/tests.rs rename crux-mir/lib/std/src/{sys_common => sync}/poison.rs (85%) create mode 100644 crux-mir/lib/std/src/sync/remutex.rs create mode 100644 crux-mir/lib/std/src/sync/remutex/tests.rs create mode 100644 crux-mir/lib/std/src/sync/rwlock/tests.rs delete mode 100644 crux-mir/lib/std/src/sys/cloudabi/abi/bitflags.rs delete mode 100644 crux-mir/lib/std/src/sys/cloudabi/abi/cloudabi.rs delete mode 100644 crux-mir/lib/std/src/sys/cloudabi/abi/mod.rs delete mode 100644 crux-mir/lib/std/src/sys/cloudabi/args.rs delete mode 100644 crux-mir/lib/std/src/sys/cloudabi/condvar.rs delete mode 100644 crux-mir/lib/std/src/sys/cloudabi/io.rs delete mode 100644 crux-mir/lib/std/src/sys/cloudabi/mod.rs delete mode 100644 crux-mir/lib/std/src/sys/cloudabi/mutex.rs delete mode 100644 crux-mir/lib/std/src/sys/cloudabi/os.rs delete mode 100644 crux-mir/lib/std/src/sys/cloudabi/rwlock.rs delete mode 100644 crux-mir/lib/std/src/sys/cloudabi/shims/env.rs delete mode 100644 crux-mir/lib/std/src/sys/cloudabi/shims/fs.rs delete mode 100644 crux-mir/lib/std/src/sys/cloudabi/shims/mod.rs delete mode 100644 crux-mir/lib/std/src/sys/cloudabi/shims/net.rs delete mode 100644 crux-mir/lib/std/src/sys/cloudabi/shims/os.rs delete mode 100644 crux-mir/lib/std/src/sys/cloudabi/shims/pipe.rs delete mode 100644 crux-mir/lib/std/src/sys/cloudabi/shims/process.rs delete mode 100644 crux-mir/lib/std/src/sys/cloudabi/stack_overflow.rs delete mode 100644 crux-mir/lib/std/src/sys/cloudabi/stdio.rs delete mode 100644 crux-mir/lib/std/src/sys/cloudabi/thread.rs delete mode 100644 crux-mir/lib/std/src/sys/cloudabi/time.rs rename crux-mir/lib/std/src/{sys_common => sys/common}/alloc.rs (70%) create mode 100644 crux-mir/lib/std/src/sys/common/mod.rs create mode 100644 crux-mir/lib/std/src/sys/common/small_c_string.rs create mode 100644 crux-mir/lib/std/src/sys/common/tests.rs delete mode 100644 crux-mir/lib/std/src/sys/crux/condvar.rs delete mode 100644 crux-mir/lib/std/src/sys/crux/mod.rs delete mode 100644 crux-mir/lib/std/src/sys/crux/mutex.rs delete mode 100644 crux-mir/lib/std/src/sys/crux/rwlock.rs delete mode 100644 crux-mir/lib/std/src/sys/crux/time.rs delete mode 100644 crux-mir/lib/std/src/sys/hermit/cmath.rs delete mode 100644 crux-mir/lib/std/src/sys/hermit/condvar.rs delete mode 100644 crux-mir/lib/std/src/sys/hermit/fast_thread_local.rs create mode 100644 crux-mir/lib/std/src/sys/hermit/futex.rs delete mode 100644 crux-mir/lib/std/src/sys/hermit/mutex.rs delete mode 100644 crux-mir/lib/std/src/sys/hermit/path.rs delete mode 100644 crux-mir/lib/std/src/sys/hermit/pipe.rs delete mode 100644 crux-mir/lib/std/src/sys/hermit/process.rs delete mode 100644 crux-mir/lib/std/src/sys/hermit/rwlock.rs delete mode 100644 crux-mir/lib/std/src/sys/hermit/stack_overflow.rs delete mode 100644 crux-mir/lib/std/src/sys/hermit/thread_local.rs create mode 100644 crux-mir/lib/std/src/sys/hermit/thread_local_dtor.rs create mode 100644 crux-mir/lib/std/src/sys/itron/abi.rs create mode 100644 crux-mir/lib/std/src/sys/itron/condvar.rs create mode 100644 crux-mir/lib/std/src/sys/itron/error.rs create mode 100644 crux-mir/lib/std/src/sys/itron/mutex.rs create mode 100644 crux-mir/lib/std/src/sys/itron/spin.rs create mode 100644 crux-mir/lib/std/src/sys/itron/task.rs create mode 100644 crux-mir/lib/std/src/sys/itron/thread.rs create mode 100644 crux-mir/lib/std/src/sys/itron/time.rs create mode 100644 crux-mir/lib/std/src/sys/itron/time/tests.rs create mode 100644 crux-mir/lib/std/src/sys/itron/wait_flag.rs delete mode 100644 crux-mir/lib/std/src/sys/sgx/abi/tls.rs create mode 100644 crux-mir/lib/std/src/sys/sgx/abi/tls/mod.rs create mode 100644 crux-mir/lib/std/src/sys/sgx/abi/tls/sync_bitset.rs create mode 100644 crux-mir/lib/std/src/sys/sgx/abi/tls/sync_bitset/tests.rs create mode 100644 crux-mir/lib/std/src/sys/sgx/abi/usercalls/tests.rs delete mode 100644 crux-mir/lib/std/src/sys/sgx/cmath.rs delete mode 100644 crux-mir/lib/std/src/sys/sgx/ext/mod.rs delete mode 100644 crux-mir/lib/std/src/sys/sgx/fs.rs delete mode 100644 crux-mir/lib/std/src/sys/sgx/io.rs delete mode 100644 crux-mir/lib/std/src/sys/sgx/process.rs create mode 100644 crux-mir/lib/std/src/sys/sgx/rwlock/tests.rs delete mode 100644 crux-mir/lib/std/src/sys/sgx/stack_overflow.rs rename crux-mir/lib/std/src/sys/sgx/{thread_local.rs => thread_local_key.rs} (86%) create mode 100644 crux-mir/lib/std/src/sys/sgx/thread_parking.rs delete mode 100644 crux-mir/lib/std/src/sys/sgx/waitqueue.rs create mode 100644 crux-mir/lib/std/src/sys/sgx/waitqueue/mod.rs create mode 100644 crux-mir/lib/std/src/sys/sgx/waitqueue/spin_mutex.rs create mode 100644 crux-mir/lib/std/src/sys/sgx/waitqueue/spin_mutex/tests.rs create mode 100644 crux-mir/lib/std/src/sys/sgx/waitqueue/tests.rs create mode 100644 crux-mir/lib/std/src/sys/sgx/waitqueue/unsafe_list.rs create mode 100644 crux-mir/lib/std/src/sys/sgx/waitqueue/unsafe_list/tests.rs create mode 100644 crux-mir/lib/std/src/sys/solid/abi/fs.rs create mode 100644 crux-mir/lib/std/src/sys/solid/abi/mod.rs create mode 100644 crux-mir/lib/std/src/sys/solid/abi/sockets.rs create mode 100644 crux-mir/lib/std/src/sys/solid/alloc.rs rename crux-mir/lib/std/src/sys/{vxworks => solid}/env.rs (60%) create mode 100644 crux-mir/lib/std/src/sys/solid/error.rs create mode 100644 crux-mir/lib/std/src/sys/solid/fs.rs rename crux-mir/lib/std/src/sys/{vxworks => solid}/io.rs (93%) rename crux-mir/lib/std/src/sys/{vxworks => solid}/memchr.rs (58%) create mode 100644 crux-mir/lib/std/src/sys/solid/mod.rs create mode 100644 crux-mir/lib/std/src/sys/solid/net.rs create mode 100644 crux-mir/lib/std/src/sys/solid/os.rs create mode 100644 crux-mir/lib/std/src/sys/solid/path.rs create mode 100644 crux-mir/lib/std/src/sys/solid/rwlock.rs create mode 100644 crux-mir/lib/std/src/sys/solid/stdio.rs create mode 100644 crux-mir/lib/std/src/sys/solid/thread_local_dtor.rs create mode 100644 crux-mir/lib/std/src/sys/solid/thread_local_key.rs create mode 100644 crux-mir/lib/std/src/sys/solid/time.rs delete mode 100644 crux-mir/lib/std/src/sys/unix/condvar.rs delete mode 100644 crux-mir/lib/std/src/sys/unix/ext/io.rs delete mode 100644 crux-mir/lib/std/src/sys/unix/ext/net.rs delete mode 100644 crux-mir/lib/std/src/sys/unix/ext/process.rs delete mode 100644 crux-mir/lib/std/src/sys/unix/ext/raw.rs create mode 100644 crux-mir/lib/std/src/sys/unix/fd/tests.rs create mode 100644 crux-mir/lib/std/src/sys/unix/futex.rs create mode 100644 crux-mir/lib/std/src/sys/unix/kernel_copy.rs create mode 100644 crux-mir/lib/std/src/sys/unix/kernel_copy/tests.rs create mode 100644 crux-mir/lib/std/src/sys/unix/locks/fuchsia_mutex.rs create mode 100644 crux-mir/lib/std/src/sys/unix/locks/futex_condvar.rs create mode 100644 crux-mir/lib/std/src/sys/unix/locks/futex_mutex.rs create mode 100644 crux-mir/lib/std/src/sys/unix/locks/futex_rwlock.rs create mode 100644 crux-mir/lib/std/src/sys/unix/locks/mod.rs create mode 100644 crux-mir/lib/std/src/sys/unix/locks/pthread_condvar.rs create mode 100644 crux-mir/lib/std/src/sys/unix/locks/pthread_mutex.rs create mode 100644 crux-mir/lib/std/src/sys/unix/locks/pthread_rwlock.rs delete mode 100644 crux-mir/lib/std/src/sys/unix/mutex.rs create mode 100644 crux-mir/lib/std/src/sys/unix/os/tests.rs rename crux-mir/lib/std/src/{sys_common/os_str_bytes.rs => sys/unix/os_str.rs} (56%) create mode 100644 crux-mir/lib/std/src/sys/unix/os_str/tests.rs create mode 100644 crux-mir/lib/std/src/sys/unix/process/process_common/tests.rs create mode 100644 crux-mir/lib/std/src/sys/unix/process/process_unix/tests.rs create mode 100644 crux-mir/lib/std/src/sys/unix/process/process_unsupported.rs rename crux-mir/lib/std/src/sys/{vxworks => unix}/process/process_vxworks.rs (63%) delete mode 100644 crux-mir/lib/std/src/sys/unix/rwlock.rs rename crux-mir/lib/std/src/sys/unix/{fast_thread_local.rs => thread_local_dtor.rs} (84%) rename crux-mir/lib/std/src/sys/unix/{thread_local.rs => thread_local_key.rs} (90%) create mode 100644 crux-mir/lib/std/src/sys/unix/thread_parking/darwin.rs create mode 100644 crux-mir/lib/std/src/sys/unix/thread_parking/mod.rs create mode 100644 crux-mir/lib/std/src/sys/unix/thread_parking/netbsd.rs create mode 100644 crux-mir/lib/std/src/sys/unix/thread_parking/pthread.rs create mode 100644 crux-mir/lib/std/src/sys/unsupported/alloc.rs rename crux-mir/lib/std/src/sys/{cloudabi/shims => unsupported}/args.rs (72%) create mode 100644 crux-mir/lib/std/src/sys/unsupported/common.rs create mode 100644 crux-mir/lib/std/src/sys/unsupported/env.rs rename crux-mir/lib/std/src/sys/{wasm => unsupported}/fs.rs (75%) rename crux-mir/lib/std/src/sys/{hermit => unsupported}/io.rs (94%) rename crux-mir/lib/std/src/sys/{wasm => unsupported/locks}/condvar.rs (51%) create mode 100644 crux-mir/lib/std/src/sys/unsupported/locks/mod.rs create mode 100644 crux-mir/lib/std/src/sys/unsupported/locks/mutex.rs create mode 100644 crux-mir/lib/std/src/sys/unsupported/locks/rwlock.rs create mode 100644 crux-mir/lib/std/src/sys/unsupported/mod.rs rename crux-mir/lib/std/src/sys/{wasm => unsupported}/net.rs (78%) create mode 100644 crux-mir/lib/std/src/sys/unsupported/once.rs rename crux-mir/lib/std/src/sys/{wasm => unsupported}/os.rs (68%) rename crux-mir/lib/std/src/sys/{sgx => unsupported}/pipe.rs (62%) rename crux-mir/lib/std/src/sys/{wasi => unsupported}/process.rs (59%) rename crux-mir/lib/std/src/sys/{wasm => unsupported}/stdio.rs (81%) create mode 100644 crux-mir/lib/std/src/sys/unsupported/thread.rs rename crux-mir/lib/std/src/sys/{wasm/fast_thread_local.rs => unsupported/thread_local_dtor.rs} (86%) create mode 100644 crux-mir/lib/std/src/sys/unsupported/thread_local_key.rs rename crux-mir/lib/std/src/sys/{wasm => unsupported}/time.rs (81%) delete mode 100644 crux-mir/lib/std/src/sys/vxworks/alloc.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/args.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/cmath.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/condvar.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/ext/fs.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/ext/io.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/ext/process.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/ext/raw.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/fast_thread_local.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/fd.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/fs.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/mod.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/mutex.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/net.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/os.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/path.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/pipe.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/process/mod.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/process/process_common.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/rand.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/rwlock.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/stack_overflow.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/stdio.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/thread.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/thread_local.rs delete mode 100644 crux-mir/lib/std/src/sys/vxworks/time.rs delete mode 100644 crux-mir/lib/std/src/sys/wasi/alloc.rs delete mode 100644 crux-mir/lib/std/src/sys/wasi/ext/ffi.rs delete mode 100644 crux-mir/lib/std/src/sys/wasi/ext/io.rs delete mode 100644 crux-mir/lib/std/src/sys/wasi/ext/mod.rs delete mode 100644 crux-mir/lib/std/src/sys/wasi/path.rs delete mode 100644 crux-mir/lib/std/src/sys/wasi/pipe.rs delete mode 100644 crux-mir/lib/std/src/sys/wasm/args.rs create mode 100644 crux-mir/lib/std/src/sys/wasm/atomics/futex.rs create mode 100644 crux-mir/lib/std/src/sys/wasm/atomics/thread.rs delete mode 100644 crux-mir/lib/std/src/sys/wasm/cmath.rs delete mode 100644 crux-mir/lib/std/src/sys/wasm/condvar_atomics.rs delete mode 100644 crux-mir/lib/std/src/sys/wasm/io.rs delete mode 100644 crux-mir/lib/std/src/sys/wasm/memchr.rs delete mode 100644 crux-mir/lib/std/src/sys/wasm/mutex.rs delete mode 100644 crux-mir/lib/std/src/sys/wasm/mutex_atomics.rs delete mode 100644 crux-mir/lib/std/src/sys/wasm/path.rs delete mode 100644 crux-mir/lib/std/src/sys/wasm/pipe.rs delete mode 100644 crux-mir/lib/std/src/sys/wasm/process.rs delete mode 100644 crux-mir/lib/std/src/sys/wasm/rwlock.rs delete mode 100644 crux-mir/lib/std/src/sys/wasm/rwlock_atomics.rs delete mode 100644 crux-mir/lib/std/src/sys/wasm/stack_overflow.rs delete mode 100644 crux-mir/lib/std/src/sys/wasm/thread.rs delete mode 100644 crux-mir/lib/std/src/sys/wasm/thread_local.rs create mode 100644 crux-mir/lib/std/src/sys/windows/alloc/tests.rs create mode 100644 crux-mir/lib/std/src/sys/windows/args/tests.rs create mode 100644 crux-mir/lib/std/src/sys/windows/c/errors.rs delete mode 100644 crux-mir/lib/std/src/sys/windows/ext/io.rs delete mode 100644 crux-mir/lib/std/src/sys/windows/ext/process.rs delete mode 100644 crux-mir/lib/std/src/sys/windows/fast_thread_local.rs create mode 100644 crux-mir/lib/std/src/sys/windows/handle/tests.rs rename crux-mir/lib/std/src/sys/windows/{ => locks}/condvar.rs (73%) create mode 100644 crux-mir/lib/std/src/sys/windows/locks/mod.rs create mode 100644 crux-mir/lib/std/src/sys/windows/locks/mutex.rs create mode 100644 crux-mir/lib/std/src/sys/windows/locks/rwlock.rs delete mode 100644 crux-mir/lib/std/src/sys/windows/mutex.rs create mode 100644 crux-mir/lib/std/src/sys/windows/os/tests.rs create mode 100644 crux-mir/lib/std/src/sys/windows/path/tests.rs create mode 100644 crux-mir/lib/std/src/sys/windows/process/tests.rs delete mode 100644 crux-mir/lib/std/src/sys/windows/rwlock.rs delete mode 100644 crux-mir/lib/std/src/sys/windows/stdio_uwp.rs create mode 100644 crux-mir/lib/std/src/sys/windows/thread_local_dtor.rs rename crux-mir/lib/std/src/sys/windows/{thread_local.rs => thread_local_key.rs} (59%) create mode 100644 crux-mir/lib/std/src/sys/windows/thread_local_key/tests.rs create mode 100644 crux-mir/lib/std/src/sys/windows/thread_parking.rs delete mode 100644 crux-mir/lib/std/src/sys_common/at_exit_imp.rs delete mode 100644 crux-mir/lib/std/src/sys_common/bytestring.rs delete mode 100644 crux-mir/lib/std/src/sys_common/condvar.rs create mode 100644 crux-mir/lib/std/src/sys_common/lazy_box.rs create mode 100644 crux-mir/lib/std/src/sys_common/memchr.rs create mode 100644 crux-mir/lib/std/src/sys_common/memchr/tests.rs delete mode 100644 crux-mir/lib/std/src/sys_common/mutex.rs create mode 100644 crux-mir/lib/std/src/sys_common/net/tests.rs create mode 100644 crux-mir/lib/std/src/sys_common/once/futex.rs create mode 100644 crux-mir/lib/std/src/sys_common/once/mod.rs create mode 100644 crux-mir/lib/std/src/sys_common/once/queue.rs delete mode 100644 crux-mir/lib/std/src/sys_common/remutex.rs delete mode 100644 crux-mir/lib/std/src/sys_common/rwlock.rs create mode 100644 crux-mir/lib/std/src/sys_common/tests.rs create mode 100644 crux-mir/lib/std/src/sys_common/thread_local_dtor.rs rename crux-mir/lib/std/src/sys_common/{thread_local.rs => thread_local_key.rs} (60%) create mode 100644 crux-mir/lib/std/src/sys_common/thread_local_key/tests.rs create mode 100644 crux-mir/lib/std/src/sys_common/thread_parking/futex.rs create mode 100644 crux-mir/lib/std/src/sys_common/thread_parking/generic.rs create mode 100644 crux-mir/lib/std/src/sys_common/thread_parking/id.rs create mode 100644 crux-mir/lib/std/src/sys_common/thread_parking/mod.rs create mode 100644 crux-mir/lib/std/src/sys_common/thread_parking/wait_flag.rs delete mode 100644 crux-mir/lib/std/src/sys_common/util.rs create mode 100644 crux-mir/lib/std/src/sys_common/wstr.rs create mode 100644 crux-mir/lib/std/src/sys_common/wtf8/tests.rs create mode 100644 crux-mir/lib/std/src/thread/local/dynamic_tests.rs create mode 100644 crux-mir/lib/std/src/thread/local/tests.rs create mode 100644 crux-mir/lib/std/src/thread/scoped.rs create mode 100644 crux-mir/lib/std/src/thread/tests.rs create mode 100644 crux-mir/lib/std/src/time/tests.rs create mode 100644 crux-mir/lib/std/tests/thread.rs create mode 100644 crux-mir/lib/std_detect/Cargo.toml rename crux-mir/lib/{bigint => std_detect}/LICENSE-APACHE (99%) create mode 100644 crux-mir/lib/std_detect/LICENSE-MIT create mode 100644 crux-mir/lib/std_detect/README.md create mode 100644 crux-mir/lib/std_detect/src/detect/arch/aarch64.rs create mode 100644 crux-mir/lib/std_detect/src/detect/arch/arm.rs create mode 100644 crux-mir/lib/std_detect/src/detect/arch/mips.rs create mode 100644 crux-mir/lib/std_detect/src/detect/arch/mips64.rs create mode 100644 crux-mir/lib/std_detect/src/detect/arch/mod.rs create mode 100644 crux-mir/lib/std_detect/src/detect/arch/powerpc.rs create mode 100644 crux-mir/lib/std_detect/src/detect/arch/powerpc64.rs create mode 100644 crux-mir/lib/std_detect/src/detect/arch/riscv.rs create mode 100644 crux-mir/lib/std_detect/src/detect/arch/x86.rs create mode 100644 crux-mir/lib/std_detect/src/detect/bit.rs create mode 100644 crux-mir/lib/std_detect/src/detect/cache.rs create mode 100644 crux-mir/lib/std_detect/src/detect/macros.rs create mode 100644 crux-mir/lib/std_detect/src/detect/mod.rs create mode 100644 crux-mir/lib/std_detect/src/detect/os/aarch64.rs create mode 100644 crux-mir/lib/std_detect/src/detect/os/freebsd/aarch64.rs create mode 100644 crux-mir/lib/std_detect/src/detect/os/freebsd/arm.rs create mode 100644 crux-mir/lib/std_detect/src/detect/os/freebsd/auxvec.rs create mode 100644 crux-mir/lib/std_detect/src/detect/os/freebsd/mod.rs create mode 100644 crux-mir/lib/std_detect/src/detect/os/freebsd/powerpc.rs create mode 100644 crux-mir/lib/std_detect/src/detect/os/linux/aarch64.rs create mode 100644 crux-mir/lib/std_detect/src/detect/os/linux/arm.rs create mode 100644 crux-mir/lib/std_detect/src/detect/os/linux/auxvec.rs create mode 100644 crux-mir/lib/std_detect/src/detect/os/linux/cpuinfo.rs create mode 100644 crux-mir/lib/std_detect/src/detect/os/linux/mips.rs create mode 100644 crux-mir/lib/std_detect/src/detect/os/linux/mod.rs create mode 100644 crux-mir/lib/std_detect/src/detect/os/linux/powerpc.rs create mode 100644 crux-mir/lib/std_detect/src/detect/os/linux/riscv.rs create mode 100644 crux-mir/lib/std_detect/src/detect/os/other.rs create mode 100644 crux-mir/lib/std_detect/src/detect/os/windows/aarch64.rs create mode 100644 crux-mir/lib/std_detect/src/detect/os/x86.rs create mode 100644 crux-mir/lib/std_detect/src/detect/test_data/linux-artificial-aarch64.auxv create mode 100644 crux-mir/lib/std_detect/src/detect/test_data/linux-empty-hwcap2-aarch64.auxv create mode 100644 crux-mir/lib/std_detect/src/detect/test_data/linux-hwcap2-aarch64.auxv create mode 100644 crux-mir/lib/std_detect/src/detect/test_data/linux-no-hwcap2-aarch64.auxv create mode 100644 crux-mir/lib/std_detect/src/detect/test_data/linux-rpi3.auxv create mode 100644 crux-mir/lib/std_detect/src/detect/test_data/macos-virtualbox-linux-x86-4850HQ.auxv create mode 100644 crux-mir/lib/std_detect/src/lib.rs create mode 100644 crux-mir/lib/std_detect/tests/cpu-detection.rs create mode 100644 crux-mir/lib/std_detect/tests/macro_trailing_commas.rs create mode 100644 crux-mir/lib/std_detect/tests/x86-specific.rs create mode 100644 crux-mir/lib/stdarch/ci/docker/riscv64gc-unknown-linux-gnu/Dockerfile delete mode 100644 crux-mir/lib/stdarch/ci/docker/wasm32-unknown-unknown/Dockerfile delete mode 100755 crux-mir/lib/stdarch/ci/docker/wasm32-unknown-unknown/wasm-entrypoint.sh create mode 100644 crux-mir/lib/stdarch/ci/docker/wasm32-wasi/Dockerfile create mode 100644 crux-mir/lib/stdarch/crates/core_arch/MISSING.md create mode 100644 crux-mir/lib/stdarch/crates/core_arch/avx512bw.md create mode 100644 crux-mir/lib/stdarch/crates/core_arch/avx512f.md delete mode 100644 crux-mir/lib/stdarch/crates/core_arch/build.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/aarch64/armclang.rs delete mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/aarch64/neon.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/aarch64/neon/generated.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/aarch64/neon/mod.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/aarch64/prefetch.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/aarch64/test_support.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/aarch64/tme.rs rename crux-mir/lib/stdarch/crates/core_arch/src/{acle => arm}/dsp.rs (99%) rename crux-mir/lib/stdarch/crates/core_arch/src/{acle => arm}/ex.rs (80%) rename crux-mir/lib/stdarch/crates/core_arch/src/{acle => arm}/sat.rs (100%) rename crux-mir/lib/stdarch/crates/core_arch/src/{acle => arm}/simd32.rs (99%) rename crux-mir/lib/stdarch/crates/core_arch/src/{acle => arm_shared}/barrier/common.rs (100%) rename crux-mir/lib/stdarch/crates/core_arch/src/{acle => arm_shared}/barrier/cp15.rs (60%) rename crux-mir/lib/stdarch/crates/core_arch/src/{acle => arm_shared}/barrier/mod.rs (99%) rename crux-mir/lib/stdarch/crates/core_arch/src/{acle => arm_shared}/barrier/not_mclass.rs (100%) rename crux-mir/lib/stdarch/crates/core_arch/src/{acle => arm_shared}/barrier/v8.rs (84%) create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/crc.rs rename crux-mir/lib/stdarch/crates/core_arch/src/{aarch64 => arm_shared}/crypto.rs (55%) rename crux-mir/lib/stdarch/crates/core_arch/src/{acle => arm_shared}/hints.rs (63%) rename crux-mir/lib/stdarch/crates/core_arch/src/{acle => arm_shared}/mod.rs (65%) create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/mod.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/shift_and_insert_tests.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/store_tests.rs rename crux-mir/lib/stdarch/crates/core_arch/src/{arm => arm_shared/neon}/table_lookup_tests.rs (100%) rename crux-mir/lib/stdarch/crates/core_arch/src/{acle => arm_shared}/registers/aarch32.rs (100%) rename crux-mir/lib/stdarch/crates/core_arch/src/{acle => arm_shared}/registers/mod.rs (83%) rename crux-mir/lib/stdarch/crates/core_arch/src/{acle => arm_shared}/registers/v6m.rs (100%) rename crux-mir/lib/stdarch/crates/core_arch/src/{acle => arm_shared}/registers/v7m.rs (100%) create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/test_support.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/riscv64/mod.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/riscv_shared/mod.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/riscv_shared/p.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512bf16.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512bitalg.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512bw.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512cd.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512gfni.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vaes.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vbmi.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vbmi2.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vnni.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vpclmulqdq.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vpopcntdq.rs delete mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/x86/mmx.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/x86_64/avx512f.rs create mode 100644 crux-mir/lib/stdarch/crates/core_arch/src/x86_64/macros.rs create mode 100644 crux-mir/lib/stdarch/crates/intrinsic-test/Cargo.toml create mode 100644 crux-mir/lib/stdarch/crates/intrinsic-test/README.md create mode 100644 crux-mir/lib/stdarch/crates/intrinsic-test/missing_aarch64.txt create mode 100644 crux-mir/lib/stdarch/crates/intrinsic-test/missing_arm.txt create mode 100644 crux-mir/lib/stdarch/crates/intrinsic-test/src/acle_csv_parser.rs create mode 100644 crux-mir/lib/stdarch/crates/intrinsic-test/src/argument.rs create mode 100644 crux-mir/lib/stdarch/crates/intrinsic-test/src/intrinsic.rs create mode 100644 crux-mir/lib/stdarch/crates/intrinsic-test/src/main.rs create mode 100644 crux-mir/lib/stdarch/crates/intrinsic-test/src/types.rs create mode 100644 crux-mir/lib/stdarch/crates/intrinsic-test/src/values.rs create mode 100644 crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/mod.rs create mode 100644 crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/riscv.rs delete mode 100644 crux-mir/lib/stdarch/crates/std_detect/src/detect/error_macros.rs create mode 100644 crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/riscv.rs create mode 100644 crux-mir/lib/stdarch/crates/std_detect/src/detect/test_data/linux-artificial-aarch64.auxv create mode 100644 crux-mir/lib/stdarch/crates/std_detect/src/detect/test_data/linux-empty-hwcap2-aarch64.auxv create mode 100644 crux-mir/lib/stdarch/crates/std_detect/src/detect/test_data/linux-hwcap2-aarch64.auxv create mode 100644 crux-mir/lib/stdarch/crates/std_detect/src/detect/test_data/linux-no-hwcap2-aarch64.auxv delete mode 100644 crux-mir/lib/stdarch/crates/std_detect/src/detect/test_data/linux-x64-i7-6850k.auxv delete mode 100644 crux-mir/lib/stdarch/crates/std_detect/src/mod.rs create mode 100644 crux-mir/lib/stdarch/crates/stdarch-gen/Cargo.toml create mode 100644 crux-mir/lib/stdarch/crates/stdarch-gen/README.md create mode 100644 crux-mir/lib/stdarch/crates/stdarch-gen/neon.spec create mode 100644 crux-mir/lib/stdarch/crates/stdarch-gen/src/main.rs delete mode 100644 crux-mir/lib/stdarch/crates/stdarch-verify/.gitattributes create mode 100644 crux-mir/lib/stdarch/examples/connect5.rs create mode 100644 crux-mir/lib/stdarch/rustfmt.toml create mode 100644 crux-mir/lib/stdarch/triagebot.toml delete mode 100644 crux-mir/lib/term/Cargo.toml delete mode 100644 crux-mir/lib/term/src/lib.rs delete mode 100644 crux-mir/lib/term/src/terminfo/mod.rs delete mode 100644 crux-mir/lib/term/src/terminfo/parm.rs delete mode 100644 crux-mir/lib/term/src/terminfo/parm/tests.rs delete mode 100644 crux-mir/lib/term/src/terminfo/parser/compiled.rs delete mode 100644 crux-mir/lib/term/src/terminfo/parser/compiled/tests.rs delete mode 100644 crux-mir/lib/term/src/terminfo/searcher.rs delete mode 100644 crux-mir/lib/term/src/terminfo/searcher/tests.rs delete mode 100644 crux-mir/lib/term/src/win.rs delete mode 100644 crux-mir/lib/test/Cargo.toml delete mode 100644 crux-mir/lib/test/src/bench.rs delete mode 100644 crux-mir/lib/test/src/cli.rs delete mode 100644 crux-mir/lib/test/src/console.rs delete mode 100644 crux-mir/lib/test/src/event.rs delete mode 100644 crux-mir/lib/test/src/formatters/json.rs delete mode 100644 crux-mir/lib/test/src/formatters/mod.rs delete mode 100644 crux-mir/lib/test/src/formatters/pretty.rs delete mode 100644 crux-mir/lib/test/src/formatters/terse.rs delete mode 100644 crux-mir/lib/test/src/helpers/concurrency.rs delete mode 100644 crux-mir/lib/test/src/helpers/exit_code.rs delete mode 100644 crux-mir/lib/test/src/helpers/isatty.rs delete mode 100644 crux-mir/lib/test/src/helpers/metrics.rs delete mode 100644 crux-mir/lib/test/src/helpers/mod.rs delete mode 100644 crux-mir/lib/test/src/helpers/sink.rs delete mode 100644 crux-mir/lib/test/src/lib.rs delete mode 100644 crux-mir/lib/test/src/options.rs delete mode 100644 crux-mir/lib/test/src/stats.rs delete mode 100644 crux-mir/lib/test/src/stats/tests.rs delete mode 100644 crux-mir/lib/test/src/test_result.rs delete mode 100644 crux-mir/lib/test/src/tests.rs delete mode 100644 crux-mir/lib/test/src/time.rs delete mode 100644 crux-mir/lib/test/src/types.rs delete mode 100644 crux-mir/lib/unicode-width/COPYRIGHT delete mode 100644 crux-mir/lib/unicode-width/Cargo.toml delete mode 100644 crux-mir/lib/unicode-width/README.md delete mode 100755 crux-mir/lib/unicode-width/scripts/unicode.py delete mode 100644 crux-mir/lib/unicode-width/src/lib.rs delete mode 100644 crux-mir/lib/unicode-width/src/tables.rs delete mode 100644 crux-mir/lib/unicode-width/src/tests.rs delete mode 100644 crux-mir/lib/unwind/lib.rs delete mode 100644 crux-mir/lib/unwind/libunwind.rs delete mode 100644 crux-mir/lib/unwind/src/build.rs diff --git a/crucible-mir/src/Mir/GenericOps.hs b/crucible-mir/src/Mir/GenericOps.hs index 61ed2530a..c03cf8972 100644 --- a/crucible-mir/src/Mir/GenericOps.hs +++ b/crucible-mir/src/Mir/GenericOps.hs @@ -72,19 +72,21 @@ adtIndices (Adt _aname _kind vars _ _ _ _) col = go 0 vars lastExplicit' = if isExplicit v then discr else lastExplicit in discr : go lastExplicit' vs - getDiscr _ (Variant name (Explicit did) _fields _knd) = case Map.lookup did (_functions col) of + getDiscr _ (Variant _ _ _ _ (Just i)) = i + + getDiscr _ (Variant name (Explicit did) _fields _knd _) = case Map.lookup did (_functions col) of Just fn -> case fn^.fbody.mblocks of - [ BasicBlock _info (BasicBlockData [Assign _lhs (Use (OpConstant (Constant _ty (ConstInt i)))) _loc] _term) ] -> + ( BasicBlock _info (BasicBlockData [Assign _lhs (Use (OpConstant (Constant _ty (ConstInt i)))) _loc] _term) ):_ -> fromIntegerLit i - _ -> error "enum discriminant constant should only have one basic block" + _ -> error ("enum discriminant constant should only have one basic block [variant id:" ++ show _aname ++ " discr index:" ++ show name ++ "]") Nothing -> error $ "cannot find discriminant constant " ++ show did ++ " for variant " ++ show name - getDiscr lastExplicit (Variant _vname (Relative i) _fields _kind) = + getDiscr lastExplicit (Variant _vname (Relative i) _fields _kind _) = lastExplicit + toInteger i - isExplicit (Variant _ (Explicit _) _ _) = True + isExplicit (Variant _ (Explicit _) _ _ _) = True isExplicit _ = False -------------------------------------------------------------------------------------- @@ -155,6 +157,7 @@ instance GenericOps Intrinsic instance GenericOps Instance instance GenericOps InstanceKind instance GenericOps NamedTy +instance GenericOps NonDivergingIntrinsic -- instances for newtypes -- we need the deriving strategy 'anyclass' to disambiguate diff --git a/crucible-mir/src/Mir/JSON.hs b/crucible-mir/src/Mir/JSON.hs index 07ff7c8e0..c835e2570 100644 --- a/crucible-mir/src/Mir/JSON.hs +++ b/crucible-mir/src/Mir/JSON.hs @@ -166,10 +166,15 @@ instance FromJSON CtorKind where parseJSON = withObject "CtorKind" $ \v -> case lookupKM "kind" v of Just (String "Fn") -> pure FnKind Just (String "Const") -> pure ConstKind - Just (String "Fictive") -> pure FictiveKind _ -> fail "unspported constructor kind" instance FromJSON Variant where - parseJSON = withObject "Variant" $ \v -> Variant <$> v .: "name" <*> v .: "discr" <*> v .: "fields" <*> v .: "ctor_kind" + parseJSON = withObject "Variant" $ \v -> + Variant <$> v .: "name" + <*> v .: "discr" + <*> v .: "fields" + <*> v .: "ctor_kind" + <*> do val <- v .:? "discr_value" + convertIntegerText `traverse` val instance FromJSON Field where parseJSON = withObject "Field" $ \v -> Field <$> v .: "name" <*> v .: "ty" @@ -251,6 +256,19 @@ instance FromJSON Statement where Just (String "StorageLive") -> StorageLive <$> v .: "slvar" Just (String "StorageDead") -> StorageDead <$> v .: "sdvar" Just (String "Nop") -> pure Nop + Just (String "Deinit") -> pure Deinit + Just (String "Intrinsic") -> do + kind <- v .: "intrinsic_kind" + ndi <- case kind of + "Assume" -> NDIAssume <$> v .: "operand" + "CopyNonOverlapping" -> + NDICopyNonOverlapping <$> v .: "src" + <*> v .: "dst" + <*> v .: "count" + _ -> fail $ "unknown Intrinsic kind" ++ kind + + return $ StmtIntrinsic ndi + k -> fail $ "kind not found for statement: " ++ show k @@ -290,6 +308,8 @@ instance FromJSON Rvalue where Just (String "UnaryOp") -> UnaryOp <$> v .: "uop" <*> v .: "op" Just (String "Discriminant") -> Discriminant <$> v .: "val" Just (String "Aggregate") -> Aggregate <$> v .: "akind" <*> v .: "ops" + Just (String "ShallowInitBox") -> ShallowInitBox <$> v .: "ptr" <*> v .: "ty" + Just (String "CopyForDeref") -> CopyForDeref <$> v .: "place" k -> fail $ "unsupported RValue " ++ show k instance FromJSON Terminator where @@ -322,7 +342,7 @@ instance FromJSON Operand where instance FromJSON NullOp where parseJSON = withObject "NullOp" $ \v -> case lookupKM "kind" v of Just (String "SizeOf") -> pure SizeOf - Just (String "Box") -> pure Box + Just (String "AlignOf") -> pure AlignOf x -> fail ("bad nullOp: " ++ show x) instance FromJSON BorrowKind where @@ -374,13 +394,23 @@ instance FromJSON Vtable where instance FromJSON CastKind where parseJSON = withObject "CastKind" $ \v -> case lookupKM "kind" v of - Just (String "Misc") -> pure Misc Just (String "Pointer(ReifyFnPointer)") -> pure ReifyFnPointer Just (String "Pointer(ClosureFnPointer(Normal))") -> pure ClosureFnPointer Just (String "Pointer(UnsafeFnPointer)") -> pure UnsafeFnPointer Just (String "Pointer(Unsize)") -> pure Unsize Just (String "Pointer(MutToConstPointer)") -> pure MutToConstPointer Just (String "UnsizeVtable") -> UnsizeVtable <$> v .: "vtable" + -- TODO: actually plumb this information through if it is relevant + -- instead of using Misc + Just (String "PointerExposeAddress") -> pure Misc + Just (String "PointerFromExposedAddress") -> pure Misc + Just (String "DynStar") -> pure Misc + Just (String "IntToInt") -> pure Misc + Just (String "FloatToInt") -> pure Misc + Just (String "FloatToFloat") -> pure Misc + Just (String "IntToFloat") -> pure Misc + Just (String "PtrToPtr") -> pure Misc + Just (String "FnPtrToPtr") -> pure Misc x -> fail ("bad CastKind: " ++ show x) instance FromJSON Constant where @@ -489,6 +519,17 @@ instance FromJSON RustcRenderedConst where val <- convertIntegerText =<< v .: "val" return $ ConstRawPtr val + Just (String "array") -> do + elems <- map (\(RustcRenderedConst val) -> val) <$> v .: "elements" + return $ ConstArray elems + + Just (String "tuple") -> do + elems <- map (\(RustcRenderedConst val) -> val) <$> v .: "elements" + return $ ConstArray elems + + o -> do + fail $ "parseJSON - bad rendered constant kind: " ++ show o + -- mir-json integers are expressed as strings of 128-bit unsigned values -- for example, -1 is displayed as "18446744073709551615" -- we need to parse this as a 128 unsigned bit Int value and then diff --git a/crucible-mir/src/Mir/Mir.hs b/crucible-mir/src/Mir/Mir.hs index 475cf431a..5219875d7 100644 --- a/crucible-mir/src/Mir/Mir.hs +++ b/crucible-mir/src/Mir/Mir.hs @@ -196,11 +196,10 @@ data VariantDiscr data CtorKind = FnKind | ConstKind - | FictiveKind deriving (Eq, Ord, Show, Generic) -data Variant = Variant {_vname :: DefId, _vdiscr :: VariantDiscr, _vfields :: [Field], _vctorkind :: CtorKind} +data Variant = Variant {_vname :: DefId, _vdiscr :: VariantDiscr, _vfields :: [Field], _vctorkind :: Maybe CtorKind, _discrval :: Maybe Integer } deriving (Eq, Ord,Show, Generic) @@ -284,6 +283,13 @@ data Statement = | StorageLive { _slv :: Var } | StorageDead { _sdv :: Var } | Nop + | Deinit + | StmtIntrinsic NonDivergingIntrinsic + deriving (Show,Eq, Ord, Generic) + +data NonDivergingIntrinsic = + NDIAssume Operand + | NDICopyNonOverlapping Operand Operand Operand deriving (Show,Eq, Ord, Generic) data PlaceElem = @@ -321,6 +327,8 @@ data Rvalue = | Discriminant { _dvar :: Lvalue } | Aggregate { _ak :: AggregateKind, _ops :: [Operand] } | RAdtAg AdtAg + | ShallowInitBox { _sibptr :: Operand, _sibty :: Ty } + | CopyForDeref Lvalue deriving (Show,Eq, Ord, Generic) data AdtAg = AdtAg { _agadt :: Adt, _avgariant :: Integer, _aops :: [Operand], _adtagty :: Ty } @@ -377,7 +385,7 @@ data Operand = data NullOp = SizeOf - | Box + | AlignOf deriving (Show,Eq, Ord, Generic) @@ -427,15 +435,16 @@ data Vtable = Vtable } deriving (Show, Eq, Ord, Generic) +-- TODO: add other castkinds (see json) data CastKind = - Misc - | ReifyFnPointer - | ClosureFnPointer - | UnsafeFnPointer - | Unsize - | UnsizeVtable VtableName - | MutToConstPointer - deriving (Show,Eq, Ord, Generic) + Misc + | ReifyFnPointer + | ClosureFnPointer + | UnsafeFnPointer + | Unsize + | UnsizeVtable VtableName + | MutToConstPointer + deriving (Show,Eq, Ord, Generic) data Constant = Constant Ty ConstVal deriving (Eq, Ord, Show, Generic) @@ -672,7 +681,7 @@ instance TypeOf Rvalue where in TyTuple [resTy, TyBool] typeOf (NullaryOp op ty) = case op of SizeOf -> TyUint USize - Box -> TyAdt (textId "type::adt") (textId "alloc::boxed::Box") (Substs [ty]) + AlignOf -> TyUint USize typeOf (UnaryOp op x) = let ty = typeOf x in case op of @@ -683,6 +692,8 @@ instance TypeOf Rvalue where typeOf (Aggregate AKTuple ops) = TyTuple $ map typeOf ops typeOf (Aggregate AKClosure ops) = TyClosure $ map typeOf ops typeOf (RAdtAg (AdtAg _ _ _ ty)) = ty + typeOf (ShallowInitBox _ ty) = ty + typeOf (CopyForDeref lv) = typeOf lv instance TypeOf Operand where typeOf (Move lv) = typeOf lv diff --git a/crucible-mir/src/Mir/PP.hs b/crucible-mir/src/Mir/PP.hs index 39357272d..a308b104c 100644 --- a/crucible-mir/src/Mir/PP.hs +++ b/crucible-mir/src/Mir/PP.hs @@ -108,7 +108,9 @@ instance Pretty CtorKind where pretty = viaShow instance Pretty Variant where - pretty (Variant nm dscr flds knd) = pretty_fn4 "Variant" nm dscr flds knd + pretty (Variant nm dscr flds knd mbVal) = + pretty "Variant" <> + tupled [pretty nm, pretty dscr, pretty flds, pretty knd, pretty mbVal] instance Pretty Field where pretty (Field nm ty) = pretty_fn2 "Field" nm ty @@ -164,6 +166,13 @@ instance Pretty Statement where pretty (StorageLive l) = pretty_fn1 "StorageLive" l <> semi pretty (StorageDead l) = pretty_fn1 "StorageDead" l <> semi pretty Nop = pretty "nop" <> semi + pretty Deinit = pretty "DeInit" + pretty (StmtIntrinsic (NDIAssume op)) = + pretty "Intrinsic" <> brackets (pretty "Assume") <> parens (pretty op) <> semi + pretty (StmtIntrinsic (NDICopyNonOverlapping o1 o2 o3)) = + pretty "Intrinsic" <> brackets (pretty "CopyNonOverlapping") + <> tupled (pretty <$> [o1, o2, o3]) + <> semi instance Pretty Lvalue where pretty (LBase base) = pretty base @@ -196,6 +205,8 @@ instance Pretty Rvalue where pretty (Discriminant a) = pretty_fn1 "Discriminant" a pretty (Aggregate a b) = pretty_fn2 "Aggregate" a b pretty (RAdtAg a) = pretty a + pretty (ShallowInitBox ptr ty) = pretty_fn2 "ShallowInitBox" ptr ty + pretty (CopyForDeref lv) = pretty_fn1 "CopyForDeref" lv instance Pretty AdtAg where pretty (AdtAg (Adt nm _kind _vs _ _ _ _) i ops _) = pretty_fn3 "AdtAg" nm i ops @@ -241,7 +252,7 @@ instance Pretty Constant where instance Pretty NullOp where pretty SizeOf = pretty "sizeof" - pretty Box = pretty "box" + pretty AlignOf = pretty "alignof" instance Pretty BorrowKind where pretty = viaShow diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index 589c4865e..f49ea71a3 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -578,19 +578,9 @@ transCheckedBinOp op a b = do -- Nullary ops in rust are used for resource allocation, so are not interpreted transNullaryOp :: M.NullOp -> M.Ty -> MirGenerator h s ret (MirExp s) -transNullaryOp M.Box ty = do - -- Box has special translation to ensure that its representation is just - -- an ordinary pointer. - Some tpr <- tyToReprM ty - ptr <- newMirRef tpr - maybeInitVal <- initialValue ty - case maybeInitVal of - Just (MirExp tpr' initVal) -> do - Refl <- testEqualityOrFail tpr tpr' $ - "bad initial value for box: expected " ++ show tpr ++ " but got " ++ show tpr' - writeMirRef ptr initVal - Nothing -> return () - return $ MirExp (MirReferenceRepr tpr) ptr +transNullaryOp M.AlignOf ty = do + -- TODO: return the actual alignment + return $ MirExp UsizeRepr $ R.App $ usizeLit 4 transNullaryOp M.SizeOf _ = do -- TODO: return the actual size, once mir-json exports size/layout info return $ MirExp UsizeRepr $ R.App $ usizeLit 1 @@ -983,6 +973,9 @@ evalRval rv@(M.RAdtAg (M.AdtAg adt agv ops ty)) = do mirFail $ "evalRval: Union types are unsupported, for " ++ show (adt ^. adtname) _ -> mirFail $ "evalRval: unsupported type for AdtAg: " ++ show ty +-- TODO: are these correct? +evalRval rv@(M.ShallowInitBox op ty) = evalOperand op +evalRval rv@(M.CopyForDeref lv) = evalLvalue lv evalLvalue :: HasCallStack => M.Lvalue -> MirGenerator h s ret (MirExp s) evalLvalue lv = evalPlace lv >>= readPlace @@ -1162,6 +1155,8 @@ transStatement (M.SetDiscriminant lv i) = do -- simultaneously), then we could remove AllocateEnum. ty -> mirFail $ "don't know how to set discriminant of " ++ show ty transStatement M.Nop = return () +transStatement M.Deinit = return () +transStatement (M.StmtIntrinsic _) = return () --TODO: is this true? -- | Add a new `BranchTransInfo` entry for the current function. Returns the -- index of the new entry. @@ -1464,10 +1459,15 @@ transTerminator (M.DropAndReplace dlv dop dtarg _ dropFn) _ = do transStatement (M.Assign dlv (M.Use dop) "") jumpToBlock dtarg -transTerminator (M.Call (M.OpConstant (M.Constant _ (M.ConstFunction funid))) cargs cretdest _) tr = do +-- transTerminator (M.Call (M.OpConstant (M.Constant _ (M.ConstFunction funid))) cargs cretdest _) tr = do +-- isCustom <- resolveCustom funid +-- doCall funid cargs cretdest tr -- cleanup ignored + +transTerminator (M.Call (M.OpConstant (M.Constant (M.TyFnDef funid) _)) cargs cretdest _) tr = do isCustom <- resolveCustom funid doCall funid cargs cretdest tr -- cleanup ignored + transTerminator (M.Call funcOp cargs cretdest _) tr = do func <- evalOperand funcOp ret <- callHandle func RustAbi Nothing cargs diff --git a/crucible-mir/src/Mir/TransTy.hs b/crucible-mir/src/Mir/TransTy.hs index 9c849f68d..2284abda9 100644 --- a/crucible-mir/src/Mir/TransTy.hs +++ b/crucible-mir/src/Mir/TransTy.hs @@ -325,7 +325,7 @@ reprTransparentFieldTy col adt = do variantFields :: TransTyConstraint => M.Collection -> M.Variant -> Some C.CtxRepr -variantFields col (M.Variant _vn _vd vfs _vct) = +variantFields col (M.Variant _vn _vd vfs _vct _mbVal) = tyReprListToCtx (map (mapSome fieldType . tyToFieldRepr col . (^. M.fty)) vfs) (\repr -> Some repr) @@ -360,7 +360,7 @@ tyToFieldRepr col ty | otherwise = viewSome (\tpr -> Some $ FieldRepr $ FkMaybe tpr) (tyToRepr col ty) variantFields' :: TransTyConstraint => M.Collection -> M.Variant -> Some FieldCtxRepr -variantFields' col (M.Variant _vn _vd vfs _vct) = +variantFields' col (M.Variant _vn _vd vfs _vct _mbVal) = fieldReprListToCtx (map (tyToFieldRepr col . (^. M.fty)) vfs) (\x -> Some x) diff --git a/crux-mir/build.sh b/crux-mir/build.sh new file mode 100644 index 000000000..deda4ca91 --- /dev/null +++ b/crux-mir/build.sh @@ -0,0 +1,68 @@ +set -e +echo 'Building core...' +mir-json lib/core/src/lib.rs --edition=2021 --crate-name core -L rlibs --out-dir rlibs --crate-type rlib + +echo 'Building rustc_std_workspace_core...' +mir-json lib/rustc_std_workspace_core/lib.rs --edition=2021 --crate-name rustc_std_workspace_core -L rlibs --out-dir rlibs --crate-type rlib --extern core=rlibs/libcore.rlib + +echo 'Building libc...' +mir-json lib/libc/src/lib.rs --crate-name libc -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="align"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="rustc-std-workspace-core"' --cfg freebsd11 --cfg libc_priv_mod_use --cfg libc_union --cfg libc_const_size_of --cfg libc_align --cfg libc_int128 --cfg libc_core_cvoid --cfg libc_packedN --cfg libc_cfg_target_vendor --cfg libc_non_exhaustive --cfg libc_ptr_addr_of --cfg libc_underscore_const_names --cfg libc_thread_local --cfg 'libc_const_extern_fn`' --extern rustc_std_workspace_core=rlibs/librustc_std_workspace_core.rlib + +echo 'Building compiler_builtins...' +mir-json lib/compiler_builtins/src/lib.rs --crate-name compiler_builtins -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="compiler-builtins"' --cfg 'feature="core"' --cfg 'feature="default"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="unstable"' --cfg 'feature="mem-unaligned"`' --extern core=rlibs/libcore.rlib + +#added crucible manually here +echo "Building crucible..." +mir-json lib/crucible/lib.rs --crate-name crucible --edition=2021 -L rlibs --out-dir rlibs --crate-type rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib + +echo 'Building alloc...' +mir-json lib/alloc/src/lib.rs --edition=2021 --crate-name alloc -L rlibs --out-dir rlibs --crate-type rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib + +echo 'Building cfg_if...' +mir-json lib/cfg_if/src/lib.rs --edition=2018 --crate-name cfg_if -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib + +echo 'Building memchr...' +mir-json lib/memchr/src/lib.rs --edition=2018 --crate-name memchr -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --cfg memchr_runtime_simd --cfg memchr_runtime_sse2 --cfg memchr_runtime_sse42 --cfg 'memchr_runtime_avx`' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib + +echo 'Building adler...' +mir-json lib/adler/src/lib.rs --crate-name adler -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib + +echo 'Building rustc_demangle...' +mir-json lib/rustc_demangle/src/lib.rs --crate-name rustc_demangle -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib + +echo 'Building unwind...' +mir-json lib/unwind/src/lib.rs --edition=2021 --crate-name unwind -L rlibs --out-dir rlibs --crate-type rlib --extern cfg_if=rlibs/libcfg_if.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern libc=rlibs/liblibc.rlib + +echo 'Building panic_unwind...' +mir-json lib/panic_unwind/src/lib.rs --edition=2021 --crate-name panic_unwind -L rlibs --out-dir rlibs --crate-type rlib --extern alloc=rlibs/liballoc.rlib --extern cfg_if=rlibs/libcfg_if.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern libc=rlibs/liblibc.rlib --extern unwind=rlibs/libunwind.rlib + +echo 'Building rustc_std_workspace_alloc...' +mir-json lib/rustc_std_workspace_alloc/lib.rs --edition=2021 --crate-name rustc_std_workspace_alloc -L rlibs --out-dir rlibs --crate-type rlib --extern alloc=rlibs/liballoc.rlib + +echo 'Building panic_abort...' +mir-json lib/panic_abort/src/lib.rs --edition=2021 --crate-name panic_abort -L rlibs --out-dir rlibs --crate-type rlib --extern alloc=rlibs/liballoc.rlib --extern cfg_if=rlibs/libcfg_if.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern libc=rlibs/liblibc.rlib + +echo 'Building gimli...' +mir-json lib/gimli/src/lib.rs --edition=2018 --crate-name gimli -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="read"' --cfg 'feature="read-core"' --cfg 'feature="rustc-dep-of-std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib + +echo 'Building std_detect...' +mir-json lib/std_detect/src/lib.rs --edition=2018 --crate-name std_detect -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="libc"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="std_detect_dlsym_getauxval"' --cfg 'feature="std_detect_file_io"' --extern cfg_if=rlibs/libcfg_if.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern libc=rlibs/liblibc.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib + +echo 'Building object...' +mir-json lib/object/src/lib.rs --edition=2018 --crate-name object -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="alloc"' --cfg 'feature="archive"' --cfg 'feature="coff"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="elf"' --cfg 'feature="macho"' --cfg 'feature="pe"' --cfg 'feature="read_core"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="unaligned"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern memchr=rlibs/libmemchr.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib + +echo 'Building miniz_oxide...' +mir-json lib/miniz_oxide/src/lib.rs --edition=2018 --crate-name miniz_oxide -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern adler=rlibs/libadler.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib + +echo 'Building hashbrown...' +mir-json lib/hashbrown/src/lib.rs --edition=2021 --crate-name hashbrown -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="nightly"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="rustc-internal-api"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib + +echo 'Building addr2line...' +mir-json lib/addr2line/src/lib.rs --crate-name addr2line -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern gimli=rlibs/libgimli.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib + +echo 'Building std...' +mir-json lib/std/src/lib.rs --edition=2021 --crate-name std -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="addr2line"' --cfg 'feature="backtrace"' --cfg 'feature="gimli-symbolize"' --cfg 'feature="miniz_oxide"' --cfg 'feature="object"' --cfg 'feature="panic_unwind"' --cfg 'feature="std_detect_dlsym_getauxval"' --cfg 'feature="std_detect_file_io"' --cfg 'backtrace_in_libstd`' --extern addr2line=rlibs/libaddr2line.rlib --extern alloc=rlibs/liballoc.rlib --extern cfg_if=rlibs/libcfg_if.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern hashbrown=rlibs/libhashbrown.rlib --extern libc=rlibs/liblibc.rlib --extern miniz_oxide=rlibs/libminiz_oxide.rlib --extern object=rlibs/libobject.rlib --extern panic_abort=rlibs/libpanic_abort.rlib --extern panic_unwind=rlibs/libpanic_unwind.rlib --extern rustc_demangle=rlibs/librustc_demangle.rlib --extern std_detect=rlibs/libstd_detect.rlib --extern unwind=rlibs/libunwind.rlib + +echo 'Building proc_macro...' +mir-json lib/proc_macro/src/lib.rs --edition=2021 --crate-name proc_macro -L rlibs --out-dir rlibs --crate-type rlib --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib + diff --git a/crux-mir/lib/addr2line/.cargo-ok b/crux-mir/lib/addr2line/.cargo-ok new file mode 100644 index 000000000..b5754e203 --- /dev/null +++ b/crux-mir/lib/addr2line/.cargo-ok @@ -0,0 +1 @@ +ok \ No newline at end of file diff --git a/crux-mir/lib/addr2line/.cargo_vcs_info.json b/crux-mir/lib/addr2line/.cargo_vcs_info.json new file mode 100644 index 000000000..1d8250b40 --- /dev/null +++ b/crux-mir/lib/addr2line/.cargo_vcs_info.json @@ -0,0 +1,5 @@ +{ + "git": { + "sha1": "028973a30a58cc993f13f55d49b17ffc99c7385c" + } +} diff --git a/crux-mir/lib/addr2line/.gitignore b/crux-mir/lib/addr2line/.gitignore new file mode 100644 index 000000000..8f5faf037 --- /dev/null +++ b/crux-mir/lib/addr2line/.gitignore @@ -0,0 +1,5 @@ +target +Cargo.lock +*.png +tmp.* +*~ diff --git a/crux-mir/lib/addr2line/CHANGELOG.md b/crux-mir/lib/addr2line/CHANGELOG.md new file mode 100644 index 000000000..914139400 --- /dev/null +++ b/crux-mir/lib/addr2line/CHANGELOG.md @@ -0,0 +1,260 @@ +## 0.17.0 (2021/10/24) + +### Breaking changes + +* Updated `gimli` and `object` dependencies. + +### Changed + +* Use `skip_attributes` to improve performance. + [#236](https://github.com/gimli-rs/addr2line/pull/236) + +-------------------------------------------------------------------------------- + +## 0.16.0 (2021/07/26) + +### Breaking changes + +* Updated `gimli` and `object` dependencies. + +-------------------------------------------------------------------------------- + +## 0.15.2 (2021/06/04) + +### Fixed + +* Allow `Context` to be `Send`. + [#219](https://github.com/gimli-rs/addr2line/pull/219) + +-------------------------------------------------------------------------------- + +## 0.15.1 (2021/05/02) + +### Fixed + +* Don't ignore aranges with address 0. + [#217](https://github.com/gimli-rs/addr2line/pull/217) + +-------------------------------------------------------------------------------- + +## 0.15.0 (2021/05/02) + +### Breaking changes + +* Updated `gimli` and `object` dependencies. + [#215](https://github.com/gimli-rs/addr2line/pull/215) + +* Added `debug_aranges` parameter to `Context::from_sections`. + [#200](https://github.com/gimli-rs/addr2line/pull/200) + +### Added + +* Added `.debug_aranges` support. + [#200](https://github.com/gimli-rs/addr2line/pull/200) + +* Added supplementary object file support. + [#208](https://github.com/gimli-rs/addr2line/pull/208) + +### Fixed + +* Fixed handling of Windows paths in locations. + [#209](https://github.com/gimli-rs/addr2line/pull/209) + +* examples/addr2line: Flush stdout after each response. + [#210](https://github.com/gimli-rs/addr2line/pull/210) + +* examples/addr2line: Avoid copying every section. + [#213](https://github.com/gimli-rs/addr2line/pull/213) + +-------------------------------------------------------------------------------- + +## 0.14.1 (2020/12/31) + +### Fixed + +* Fix location lookup for skeleton units. + [#201](https://github.com/gimli-rs/addr2line/pull/201) + +### Added + +* Added `Context::find_location_range`. + [#196](https://github.com/gimli-rs/addr2line/pull/196) + [#199](https://github.com/gimli-rs/addr2line/pull/199) + +-------------------------------------------------------------------------------- + +## 0.14.0 (2020/10/27) + +### Breaking changes + +* Updated `gimli` and `object` dependencies. + +### Fixed + +* Handle units that only have line information. + [#188](https://github.com/gimli-rs/addr2line/pull/188) + +* Handle DWARF units with version <= 4 and no `DW_AT_name`. + [#191](https://github.com/gimli-rs/addr2line/pull/191) + +* Fix handling of `DW_FORM_ref_addr`. + [#193](https://github.com/gimli-rs/addr2line/pull/193) + +-------------------------------------------------------------------------------- + +## 0.13.0 (2020/07/07) + +### Breaking changes + +* Updated `gimli` and `object` dependencies. + +* Added `rustc-dep-of-std` feature. + [#166](https://github.com/gimli-rs/addr2line/pull/166) + +### Changed + +* Improve performance by parsing function contents lazily. + [#178](https://github.com/gimli-rs/addr2line/pull/178) + +* Don't skip `.debug_info` and `.debug_line` entries with a zero address. + [#182](https://github.com/gimli-rs/addr2line/pull/182) + +-------------------------------------------------------------------------------- + +## 0.12.2 (2020/06/21) + +### Fixed + +* Avoid linear search for `DW_FORM_ref_addr`. + [#175](https://github.com/gimli-rs/addr2line/pull/175) + +-------------------------------------------------------------------------------- + +## 0.12.1 (2020/05/19) + +### Fixed + +* Handle units with overlapping address ranges. + [#163](https://github.com/gimli-rs/addr2line/pull/163) + +* Don't assert for functions with overlapping address ranges. + [#168](https://github.com/gimli-rs/addr2line/pull/168) + +-------------------------------------------------------------------------------- + +## 0.12.0 (2020/05/12) + +### Breaking changes + +* Updated `gimli` and `object` dependencies. + +* Added more optional features: `smallvec` and `fallible-iterator`. + [#160](https://github.com/gimli-rs/addr2line/pull/160) + +### Added + +* Added `Context::dwarf` and `Context::find_dwarf_unit`. + [#159](https://github.com/gimli-rs/addr2line/pull/159) + +### Changed + +* Removed `lazycell` dependency. + [#160](https://github.com/gimli-rs/addr2line/pull/160) + +-------------------------------------------------------------------------------- + +## 0.11.0 (2020/01/11) + +### Breaking changes + +* Updated `gimli` and `object` dependencies. + +* [#130](https://github.com/gimli-rs/addr2line/pull/130) + Changed `Location::file` from `Option` to `Option<&str>`. + This required adding lifetime parameters to `Location` and other structs that + contain it. + +* [#152](https://github.com/gimli-rs/addr2line/pull/152) + Changed `Location::line` and `Location::column` from `Option`to `Option`. + +* [#156](https://github.com/gimli-rs/addr2line/pull/156) + Deleted `alloc` feature, and fixed `no-std` builds with stable rust. + Removed default `Reader` parameter for `Context`, and added `ObjectContext` instead. + +### Added + +* [#134](https://github.com/gimli-rs/addr2line/pull/134) + Added `Context::from_dwarf`. + +### Changed + +* [#133](https://github.com/gimli-rs/addr2line/pull/133) + Fixed handling of units that can't be parsed. + +* [#155](https://github.com/gimli-rs/addr2line/pull/155) + Fixed `addr2line` output to match binutils. + +* [#130](https://github.com/gimli-rs/addr2line/pull/130) + Improved `.debug_line` parsing performance. + +* [#148](https://github.com/gimli-rs/addr2line/pull/148) + [#150](https://github.com/gimli-rs/addr2line/pull/150) + [#151](https://github.com/gimli-rs/addr2line/pull/151) + [#152](https://github.com/gimli-rs/addr2line/pull/152) + Improved `.debug_info` parsing performance. + +* [#137](https://github.com/gimli-rs/addr2line/pull/137) + [#138](https://github.com/gimli-rs/addr2line/pull/138) + [#139](https://github.com/gimli-rs/addr2line/pull/139) + [#140](https://github.com/gimli-rs/addr2line/pull/140) + [#146](https://github.com/gimli-rs/addr2line/pull/146) + Improved benchmarks. + +-------------------------------------------------------------------------------- + +## 0.10.0 (2019/07/07) + +### Breaking changes + +* [#127](https://github.com/gimli-rs/addr2line/pull/127) + Update `gimli`. + +-------------------------------------------------------------------------------- + +## 0.9.0 (2019/05/02) + +### Breaking changes + +* [#121](https://github.com/gimli-rs/addr2line/pull/121) + Update `gimli`, `object`, and `fallible-iterator` dependencies. + +### Added + +* [#121](https://github.com/gimli-rs/addr2line/pull/121) + Reexport `gimli`, `object`, and `fallible-iterator`. + +-------------------------------------------------------------------------------- + +## 0.8.0 (2019/02/06) + +### Breaking changes + +* [#107](https://github.com/gimli-rs/addr2line/pull/107) + Update `object` dependency to 0.11. This is part of the public API. + +### Added + +* [#101](https://github.com/gimli-rs/addr2line/pull/101) + Add `object` feature (enabled by default). Disable this feature to remove + the `object` dependency and `Context::new` API. + +* [#102](https://github.com/gimli-rs/addr2line/pull/102) + Add `std` (enabled by default) and `alloc` features. + +### Changed + +* [#108](https://github.com/gimli-rs/addr2line/issues/108) + `demangle` no longer ouputs the hash for rust symbols. + +* [#109](https://github.com/gimli-rs/addr2line/issues/109) + Set default `R` for `Context`. diff --git a/crux-mir/lib/addr2line/Cargo.toml b/crux-mir/lib/addr2line/Cargo.toml new file mode 100644 index 000000000..358995e53 --- /dev/null +++ b/crux-mir/lib/addr2line/Cargo.toml @@ -0,0 +1,120 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies +# +# If you believe there's an error in this file please file an +# issue against the rust-lang/cargo repository. If you're +# editing this file be aware that the upstream Cargo.toml +# will likely look very different (and much more reasonable) + +[package] +name = "addr2line" +version = "0.17.0" +exclude = ["/benches/*", "/fixtures/*", ".github"] +description = "A cross-platform symbolication library written in Rust, using `gimli`" +documentation = "https://docs.rs/addr2line" +readme = "./README.md" +keywords = ["DWARF", "debug", "elf", "symbolicate", "atos"] +categories = ["development-tools::debugging"] +license = "Apache-2.0 OR MIT" +repository = "https://github.com/gimli-rs/addr2line" +[profile.bench] +codegen-units = 1 +debug = true +split-debuginfo = "packed" + +[profile.dev] +split-debuginfo = "packed" + +[profile.release] +debug = true +split-debuginfo = "packed" + +[profile.test] +split-debuginfo = "packed" + +[[example]] +name = "addr2line" +required-features = ["std-object"] + +[[test]] +name = "output_equivalence" +harness = false +required-features = ["std-object"] + +[[test]] +name = "correctness" +required-features = ["default"] + +[[test]] +name = "parse" +required-features = ["std-object"] +[dependencies.alloc] +version = "1.0.0" +optional = true +package = "rustc-std-workspace-alloc" + +[dependencies.compiler_builtins] +version = "0.1.2" +optional = true + +[dependencies.core] +version = "1.0.0" +optional = true +package = "rustc-std-workspace-core" + +[dependencies.cpp_demangle] +version = "0.3" +optional = true +default-features = false + +[dependencies.fallible-iterator] +version = "0.2" +optional = true +default-features = false + +[dependencies.gimli] +version = "0.26" +features = ["read"] +default-features = false + +[dependencies.object] +version = "0.27.1" +features = ["read"] +optional = true +default-features = false + +[dependencies.rustc-demangle] +version = "0.1" +optional = true + +[dependencies.smallvec] +version = "1" +optional = true +default-features = false +[dev-dependencies.backtrace] +version = "0.3.13" + +[dev-dependencies.clap] +version = "2" + +[dev-dependencies.findshlibs] +version = "0.10" + +[dev-dependencies.memmap] +version = "0.7" + +[dev-dependencies.rustc-test] +version = "0.3" + +[dev-dependencies.typed-arena] +version = "2" + +[features] +default = ["rustc-demangle", "cpp_demangle", "std-object", "fallible-iterator", "smallvec"] +rustc-dep-of-std = ["core", "alloc", "compiler_builtins", "gimli/rustc-dep-of-std"] +std = ["gimli/std"] +std-object = ["std", "object", "object/std", "object/compression", "gimli/endian-reader"] diff --git a/crux-mir/lib/addr2line/Cargo.toml.orig b/crux-mir/lib/addr2line/Cargo.toml.orig new file mode 100644 index 000000000..29e4c3b96 --- /dev/null +++ b/crux-mir/lib/addr2line/Cargo.toml.orig @@ -0,0 +1,75 @@ +[package] +name = "addr2line" +version = "0.17.0" +description = "A cross-platform symbolication library written in Rust, using `gimli`" +documentation = "https://docs.rs/addr2line" +exclude = ["/benches/*", "/fixtures/*", ".github"] +keywords = ["DWARF", "debug", "elf", "symbolicate", "atos"] +categories = ["development-tools::debugging"] +license = "Apache-2.0 OR MIT" +readme = "./README.md" +repository = "https://github.com/gimli-rs/addr2line" + +[dependencies] +gimli = { version = "0.26", default-features = false, features = ["read"] } +fallible-iterator = { version = "0.2", default-features = false, optional = true } +object = { version = "0.27.1", default-features = false, features = ["read"], optional = true } +smallvec = { version = "1", default-features = false, optional = true } +rustc-demangle = { version = "0.1", optional = true } +cpp_demangle = { version = "0.3", default-features = false, optional = true } + +# Internal feature, only used when building as part of libstd, not part of the +# stable interface of this crate. +core = { version = '1.0.0', optional = true, package = 'rustc-std-workspace-core' } +alloc = { version = '1.0.0', optional = true, package = 'rustc-std-workspace-alloc' } +compiler_builtins = { version = '0.1.2', optional = true } + +[dev-dependencies] +memmap = "0.7" +clap = "2" +backtrace = "0.3.13" +findshlibs = "0.10" +rustc-test = "0.3" +auxiliary = { path = "tests/auxiliary" } +typed-arena = "2" + +[profile.dev] +split-debuginfo = 'packed' + +[profile.test] +split-debuginfo = 'packed' + +[profile.release] +debug = true +split-debuginfo = 'packed' + +[profile.bench] +debug = true +codegen-units = 1 +split-debuginfo = 'packed' + +[features] +default = ["rustc-demangle", "cpp_demangle", "std-object", "fallible-iterator", "smallvec"] +std = ["gimli/std"] +std-object = ["std", "object", "object/std", "object/compression", "gimli/endian-reader"] + +# Internal feature, only used when building as part of libstd, not part of the +# stable interface of this crate. +rustc-dep-of-std = ['core', 'alloc', 'compiler_builtins', 'gimli/rustc-dep-of-std'] + +[[test]] +name = "output_equivalence" +harness = false +required-features = ["std-object"] + +[[test]] +name = "correctness" +required-features = ["default"] + +[[test]] +name = "parse" +required-features = ["std-object"] + +[[example]] +name = "addr2line" +required-features = ["std-object"] diff --git a/crux-mir/lib/backtrace/crates/backtrace-sys/LICENSE-APACHE b/crux-mir/lib/addr2line/LICENSE-APACHE similarity index 100% rename from crux-mir/lib/backtrace/crates/backtrace-sys/LICENSE-APACHE rename to crux-mir/lib/addr2line/LICENSE-APACHE diff --git a/crux-mir/lib/getopts/LICENSE-MIT b/crux-mir/lib/addr2line/LICENSE-MIT similarity index 95% rename from crux-mir/lib/getopts/LICENSE-MIT rename to crux-mir/lib/addr2line/LICENSE-MIT index 39d4bdb5a..3a03f1f85 100644 --- a/crux-mir/lib/getopts/LICENSE-MIT +++ b/crux-mir/lib/addr2line/LICENSE-MIT @@ -1,4 +1,4 @@ -Copyright (c) 2014 The Rust Project Developers +Copyright (c) 2016-2018 The gimli Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated diff --git a/crux-mir/lib/addr2line/README.md b/crux-mir/lib/addr2line/README.md new file mode 100644 index 000000000..dc6cb9344 --- /dev/null +++ b/crux-mir/lib/addr2line/README.md @@ -0,0 +1,48 @@ +# addr2line + +[![](https://img.shields.io/crates/v/addr2line.svg)](https://crates.io/crates/addr2line) +[![](https://img.shields.io/docsrs/addr2line.svg)](https://docs.rs/addr2line) +[![Coverage Status](https://coveralls.io/repos/github/gimli-rs/addr2line/badge.svg?branch=master)](https://coveralls.io/github/gimli-rs/addr2line?branch=master) + +A cross-platform library for retrieving per-address debug information +from files with DWARF debug information. + +`addr2line` uses [`gimli`](https://github.com/gimli-rs/gimli) to parse +the debug information, and exposes an interface for finding +the source file, line number, and wrapping function for instruction +addresses within the target program. These lookups can either be +performed programmatically through `Context::find_location` and +`Context::find_frames`, or via the included example binary, +`addr2line` (named and modelled after the equivalent utility from +[GNU binutils](https://sourceware.org/binutils/docs/binutils/addr2line.html)). + +# Quickstart + - Add the [`addr2line` crate](https://crates.io/crates/addr2line) to your `Cargo.toml` + - Load the file and parse it with [`addr2line::object::read::File::parse`](https://docs.rs/object/*/object/read/struct.File.html#method.parse) + - Pass the parsed file to [`addr2line::Context::new` ](https://docs.rs/addr2line/*/addr2line/struct.Context.html#method.new) + - Use [`addr2line::Context::find_location`](https://docs.rs/addr2line/*/addr2line/struct.Context.html#method.find_location) + or [`addr2line::Context::find_frames`](https://docs.rs/addr2line/*/addr2line/struct.Context.html#method.find_frames) + to look up debug information for an address + +# Performance + +`addr2line` optimizes for speed over memory by caching parsed information. +The DWARF information is parsed lazily where possible. + +The library aims to perform similarly to equivalent existing tools such +as `addr2line` from binutils, `eu-addr2line` from elfutils, and +`llvm-symbolize` from the llvm project, and in the past some benchmarking +was done that indicates a comparable performance. + +## License + +Licensed under either of + + * Apache License, Version 2.0 ([`LICENSE-APACHE`](./LICENSE-APACHE) or https://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([`LICENSE-MIT`](./LICENSE-MIT) or https://opensource.org/licenses/MIT) + +at your option. + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in the work by you, as defined in the Apache-2.0 license, shall be +dual licensed as above, without any additional terms or conditions. diff --git a/crux-mir/lib/addr2line/bench.plot.r b/crux-mir/lib/addr2line/bench.plot.r new file mode 100644 index 000000000..ecbf24893 --- /dev/null +++ b/crux-mir/lib/addr2line/bench.plot.r @@ -0,0 +1,23 @@ +v <- read.table(file("stdin")) +t <- data.frame(prog=v[,1], funcs=(v[,2]=="func"), time=v[,3], mem=v[,4], stringsAsFactors=FALSE) + +t$prog <- as.character(t$prog) +t$prog[t$prog == "master"] <- "gimli-rs/addr2line" +t$funcs[t$funcs == TRUE] <- "With functions" +t$funcs[t$funcs == FALSE] <- "File/line only" +t$mem = t$mem / 1024.0 + +library(ggplot2) +p <- ggplot(data=t, aes(x=prog, y=time, fill=prog)) +p <- p + geom_bar(stat = "identity") +p <- p + facet_wrap(~ funcs) +p <- p + theme(axis.title.x=element_blank(), axis.text.x=element_blank(), axis.ticks.x=element_blank()) +p <- p + ylab("time (s)") + ggtitle("addr2line runtime") +ggsave('time.png',plot=p,width=10,height=6) + +p <- ggplot(data=t, aes(x=prog, y=mem, fill=prog)) +p <- p + geom_bar(stat = "identity") +p <- p + facet_wrap(~ funcs) +p <- p + theme(axis.title.x=element_blank(), axis.text.x=element_blank(), axis.ticks.x=element_blank()) +p <- p + ylab("memory (kB)") + ggtitle("addr2line memory usage") +ggsave('memory.png',plot=p,width=10,height=6) diff --git a/crux-mir/lib/addr2line/benchmark.sh b/crux-mir/lib/addr2line/benchmark.sh new file mode 100755 index 000000000..ca4c4f6ec --- /dev/null +++ b/crux-mir/lib/addr2line/benchmark.sh @@ -0,0 +1,112 @@ +#!/bin/bash +if [[ $# -le 1 ]]; then + echo "Usage: $0 [] REFS..." + exit 1 +fi +target="$1" +shift + +addresses="" +if [[ -e "$1" ]]; then + addresses="$1" + shift +fi + +# path to "us" +# readlink -f, but more portable: +dirname=$(perl -e 'use Cwd "abs_path";print abs_path(shift)' "$(dirname "$0")") + +# https://stackoverflow.com/a/2358432/472927 +{ + # compile all refs + pushd "$dirname" > /dev/null + # if the user has some local changes, preserve them + nstashed=$(git stash list | wc -l) + echo "==> Stashing any local modifications" + git stash --keep-index > /dev/null + popstash() { + # https://stackoverflow.com/q/24520791/472927 + if [[ "$(git stash list | wc -l)" -ne "$nstashed" ]]; then + echo "==> Restoring stashed state" + git stash pop > /dev/null + fi + } + # if the user has added stuff to the index, abort + if ! git diff-index --quiet HEAD --; then + echo "Refusing to overwrite outstanding git changes" + popstash + exit 2 + fi + current=$(git symbolic-ref --short HEAD) + for ref in "$@"; do + echo "==> Compiling $ref" + git checkout -q "$ref" + commit=$(git rev-parse HEAD) + fn="target/release/addr2line-$commit" + if [[ ! -e "$fn" ]]; then + cargo build --release --example addr2line + cp target/release/examples/addr2line "$fn" + fi + if [[ "$ref" != "$commit" ]]; then + ln -sfn "addr2line-$commit" target/release/addr2line-"$ref" + fi + done + git checkout -q "$current" + popstash + popd > /dev/null + + # get us some addresses to look up + if [[ -z "$addresses" ]]; then + echo "==> Looking for benchmarking addresses (this may take a while)" + addresses=$(mktemp tmp.XXXXXXXXXX) + objdump -C -x --disassemble -l "$target" \ + | grep -P '0[048]:' \ + | awk '{print $1}' \ + | sed 's/:$//' \ + > "$addresses" + echo " -> Addresses stored in $addresses; you should re-use it next time" + fi + + run() { + func="$1" + name="$2" + cmd="$3" + args="$4" + printf "%s\t%s\t" "$name" "$func" + if [[ "$cmd" =~ llvm-symbolizer ]]; then + /usr/bin/time -f '%e\t%M' "$cmd" $args -obj="$target" < "$addresses" 2>&1 >/dev/null + else + /usr/bin/time -f '%e\t%M' "$cmd" $args -e "$target" < "$addresses" 2>&1 >/dev/null + fi + } + + # run without functions + log1=$(mktemp tmp.XXXXXXXXXX) + echo "==> Benchmarking" + run nofunc binutils addr2line >> "$log1" + #run nofunc elfutils eu-addr2line >> "$log1" + run nofunc llvm-sym llvm-symbolizer -functions=none >> "$log1" + for ref in "$@"; do + run nofunc "$ref" "$dirname/target/release/addr2line-$ref" >> "$log1" + done + cat "$log1" | column -t + + # run with functions + log2=$(mktemp tmp.XXXXXXXXXX) + echo "==> Benchmarking with -f" + run func binutils addr2line "-f -i" >> "$log2" + #run func elfutils eu-addr2line "-f -i" >> "$log2" + run func llvm-sym llvm-symbolizer "-functions=linkage -demangle=0" >> "$log2" + for ref in "$@"; do + run func "$ref" "$dirname/target/release/addr2line-$ref" "-f -i" >> "$log2" + done + cat "$log2" | column -t + cat "$log2" >> "$log1"; rm "$log2" + + echo "==> Plotting" + Rscript --no-readline --no-restore --no-save "$dirname/bench.plot.r" < "$log1" + + echo "==> Cleaning up" + rm "$log1" + exit 0 +} diff --git a/crux-mir/lib/addr2line/coverage.sh b/crux-mir/lib/addr2line/coverage.sh new file mode 100644 index 000000000..892c0b7fa --- /dev/null +++ b/crux-mir/lib/addr2line/coverage.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# Run tarpaulin and pycobertura to generate coverage.html. + +cargo tarpaulin --skip-clean --out Xml +pycobertura show --format html --output coverage.html cobertura.xml diff --git a/crux-mir/lib/addr2line/examples/addr2line.rs b/crux-mir/lib/addr2line/examples/addr2line.rs new file mode 100644 index 000000000..4b228a706 --- /dev/null +++ b/crux-mir/lib/addr2line/examples/addr2line.rs @@ -0,0 +1,299 @@ +extern crate addr2line; +extern crate clap; +extern crate fallible_iterator; +extern crate gimli; +extern crate memmap; +extern crate object; +extern crate typed_arena; + +use std::borrow::Cow; +use std::fs::File; +use std::io::{BufRead, Lines, StdinLock, Write}; +use std::path::Path; + +use clap::{App, Arg, Values}; +use fallible_iterator::FallibleIterator; +use object::{Object, ObjectSection}; +use typed_arena::Arena; + +use addr2line::{Context, Location}; + +fn parse_uint_from_hex_string(string: &str) -> u64 { + if string.len() > 2 && string.starts_with("0x") { + u64::from_str_radix(&string[2..], 16).expect("Failed to parse address") + } else { + u64::from_str_radix(string, 16).expect("Failed to parse address") + } +} + +enum Addrs<'a> { + Args(Values<'a>), + Stdin(Lines>), +} + +impl<'a> Iterator for Addrs<'a> { + type Item = u64; + + fn next(&mut self) -> Option { + let text = match *self { + Addrs::Args(ref mut vals) => vals.next().map(Cow::from), + Addrs::Stdin(ref mut lines) => lines.next().map(Result::unwrap).map(Cow::from), + }; + text.as_ref() + .map(Cow::as_ref) + .map(parse_uint_from_hex_string) + } +} + +fn print_loc(loc: &Option, basenames: bool, llvm: bool) { + if let Some(ref loc) = *loc { + let file = loc.file.as_ref().unwrap(); + let path = if basenames { + Path::new(Path::new(file).file_name().unwrap()) + } else { + Path::new(file) + }; + print!("{}:", path.display()); + if llvm { + print!("{}:{}", loc.line.unwrap_or(0), loc.column.unwrap_or(0)); + } else if let Some(line) = loc.line { + print!("{}", line); + } else { + print!("?"); + } + println!(); + } else if llvm { + println!("??:0:0"); + } else { + println!("??:?"); + } +} + +fn print_function(name: &str, language: Option, demangle: bool) { + if demangle { + print!("{}", addr2line::demangle_auto(Cow::from(name), language)); + } else { + print!("{}", name); + } +} + +fn load_file_section<'input, 'arena, Endian: gimli::Endianity>( + id: gimli::SectionId, + file: &object::File<'input>, + endian: Endian, + arena_data: &'arena Arena>, +) -> Result, ()> { + // TODO: Unify with dwarfdump.rs in gimli. + let name = id.name(); + match file.section_by_name(name) { + Some(section) => match section.uncompressed_data().unwrap() { + Cow::Borrowed(b) => Ok(gimli::EndianSlice::new(b, endian)), + Cow::Owned(b) => Ok(gimli::EndianSlice::new(arena_data.alloc(b.into()), endian)), + }, + None => Ok(gimli::EndianSlice::new(&[][..], endian)), + } +} + +fn main() { + let matches = App::new("hardliner") + .version("0.1") + .about("A fast addr2line clone") + .arg( + Arg::with_name("exe") + .short("e") + .long("exe") + .value_name("filename") + .help( + "Specify the name of the executable for which addresses should be translated.", + ) + .required(true), + ) + .arg( + Arg::with_name("sup") + .long("sup") + .value_name("filename") + .help("Path to supplementary object file."), + ) + .arg( + Arg::with_name("functions") + .short("f") + .long("functions") + .help("Display function names as well as file and line number information."), + ) + .arg( + Arg::with_name("pretty") + .short("p") + .long("pretty-print") + .help( + "Make the output more human friendly: each location are printed on \ + one line.", + ), + ) + .arg(Arg::with_name("inlines").short("i").long("inlines").help( + "If the address belongs to a function that was inlined, the source \ + information for all enclosing scopes back to the first non-inlined \ + function will also be printed.", + )) + .arg( + Arg::with_name("addresses") + .short("a") + .long("addresses") + .help( + "Display the address before the function name, file and line \ + number information.", + ), + ) + .arg( + Arg::with_name("basenames") + .short("s") + .long("basenames") + .help("Display only the base of each file name."), + ) + .arg(Arg::with_name("demangle").short("C").long("demangle").help( + "Demangle function names. \ + Specifying a specific demangling style (like GNU addr2line) \ + is not supported. (TODO)", + )) + .arg( + Arg::with_name("llvm") + .long("llvm") + .help("Display output in the same format as llvm-symbolizer."), + ) + .arg( + Arg::with_name("addrs") + .takes_value(true) + .multiple(true) + .help("Addresses to use instead of reading from stdin."), + ) + .get_matches(); + + let arena_data = Arena::new(); + + let do_functions = matches.is_present("functions"); + let do_inlines = matches.is_present("inlines"); + let pretty = matches.is_present("pretty"); + let print_addrs = matches.is_present("addresses"); + let basenames = matches.is_present("basenames"); + let demangle = matches.is_present("demangle"); + let llvm = matches.is_present("llvm"); + let path = matches.value_of("exe").unwrap(); + + let file = File::open(path).unwrap(); + let map = unsafe { memmap::Mmap::map(&file).unwrap() }; + let object = &object::File::parse(&*map).unwrap(); + + let endian = if object.is_little_endian() { + gimli::RunTimeEndian::Little + } else { + gimli::RunTimeEndian::Big + }; + + let mut load_section = |id: gimli::SectionId| -> Result<_, _> { + load_file_section(id, object, endian, &arena_data) + }; + + let sup_map; + let sup_object = if let Some(sup_path) = matches.value_of("sup") { + let sup_file = File::open(sup_path).unwrap(); + sup_map = unsafe { memmap::Mmap::map(&sup_file).unwrap() }; + Some(object::File::parse(&*sup_map).unwrap()) + } else { + None + }; + + let symbols = object.symbol_map(); + let mut dwarf = gimli::Dwarf::load(&mut load_section).unwrap(); + if let Some(ref sup_object) = sup_object { + let mut load_sup_section = |id: gimli::SectionId| -> Result<_, _> { + load_file_section(id, sup_object, endian, &arena_data) + }; + dwarf.load_sup(&mut load_sup_section).unwrap(); + } + + let ctx = Context::from_dwarf(dwarf).unwrap(); + + let stdin = std::io::stdin(); + let addrs = matches + .values_of("addrs") + .map(Addrs::Args) + .unwrap_or_else(|| Addrs::Stdin(stdin.lock().lines())); + + for probe in addrs { + if print_addrs { + if llvm { + print!("0x{:x}", probe); + } else { + print!("0x{:016x}", probe); + } + if pretty { + print!(": "); + } else { + println!(); + } + } + + if do_functions || do_inlines { + let mut printed_anything = false; + let mut frames = ctx.find_frames(probe).unwrap().enumerate(); + while let Some((i, frame)) = frames.next().unwrap() { + if pretty && i != 0 { + print!(" (inlined by) "); + } + + if do_functions { + if let Some(func) = frame.function { + print_function(&func.raw_name().unwrap(), func.language, demangle); + } else if let Some(name) = symbols.get(probe).map(|x| x.name()) { + print_function(name, None, demangle); + } else { + print!("??"); + } + + if pretty { + print!(" at "); + } else { + println!(); + } + } + + print_loc(&frame.location, basenames, llvm); + + printed_anything = true; + + if !do_inlines { + break; + } + } + + if !printed_anything { + if do_functions { + if let Some(name) = symbols.get(probe).map(|x| x.name()) { + print_function(name, None, demangle); + } else { + print!("??"); + } + + if pretty { + print!(" at "); + } else { + println!(); + } + } + + if llvm { + println!("??:0:0"); + } else { + println!("??:?"); + } + } + } else { + let loc = ctx.find_location(probe).unwrap(); + print_loc(&loc, basenames, llvm); + } + + if llvm { + println!(); + } + std::io::stdout().flush().unwrap(); + } +} diff --git a/crux-mir/lib/addr2line/rustfmt.toml b/crux-mir/lib/addr2line/rustfmt.toml new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/crux-mir/lib/addr2line/rustfmt.toml @@ -0,0 +1 @@ + diff --git a/crux-mir/lib/addr2line/src/function.rs b/crux-mir/lib/addr2line/src/function.rs new file mode 100644 index 000000000..1589acdbe --- /dev/null +++ b/crux-mir/lib/addr2line/src/function.rs @@ -0,0 +1,520 @@ +use alloc::boxed::Box; +use alloc::vec::Vec; +use core::cmp::Ordering; +use core::iter; + +use crate::lazy::LazyCell; +use crate::maybe_small; +use crate::{Error, RangeAttributes, ResDwarf}; + +pub(crate) struct Functions { + /// List of all `DW_TAG_subprogram` details in the unit. + pub(crate) functions: Box< + [( + gimli::UnitOffset, + LazyCell, Error>>, + )], + >, + /// List of `DW_TAG_subprogram` address ranges in the unit. + pub(crate) addresses: Box<[FunctionAddress]>, +} + +/// A single address range for a function. +/// +/// It is possible for a function to have multiple address ranges; this +/// is handled by having multiple `FunctionAddress` entries with the same +/// `function` field. +pub(crate) struct FunctionAddress { + range: gimli::Range, + /// An index into `Functions::functions`. + pub(crate) function: usize, +} + +pub(crate) struct Function { + pub(crate) dw_die_offset: gimli::UnitOffset, + pub(crate) name: Option, + /// List of all `DW_TAG_inlined_subroutine` details in this function. + inlined_functions: Box<[InlinedFunction]>, + /// List of `DW_TAG_inlined_subroutine` address ranges in this function. + inlined_addresses: Box<[InlinedFunctionAddress]>, +} + +pub(crate) struct InlinedFunctionAddress { + range: gimli::Range, + call_depth: usize, + /// An index into `Function::inlined_functions`. + function: usize, +} + +pub(crate) struct InlinedFunction { + pub(crate) dw_die_offset: gimli::UnitOffset, + pub(crate) name: Option, + pub(crate) call_file: u64, + pub(crate) call_line: u32, + pub(crate) call_column: u32, +} + +impl Functions { + pub(crate) fn parse(unit: &gimli::Unit, dwarf: &ResDwarf) -> Result, Error> { + let mut functions = Vec::new(); + let mut addresses = Vec::new(); + let mut entries = unit.entries_raw(None)?; + while !entries.is_empty() { + let dw_die_offset = entries.next_offset(); + if let Some(abbrev) = entries.read_abbreviation()? { + if abbrev.tag() == gimli::DW_TAG_subprogram { + let mut ranges = RangeAttributes::default(); + for spec in abbrev.attributes() { + match entries.read_attribute(*spec) { + Ok(ref attr) => { + match attr.name() { + gimli::DW_AT_low_pc => { + if let gimli::AttributeValue::Addr(val) = attr.value() { + ranges.low_pc = Some(val); + } + } + gimli::DW_AT_high_pc => match attr.value() { + gimli::AttributeValue::Addr(val) => { + ranges.high_pc = Some(val) + } + gimli::AttributeValue::Udata(val) => { + ranges.size = Some(val) + } + _ => {} + }, + gimli::DW_AT_ranges => { + ranges.ranges_offset = dwarf + .sections + .attr_ranges_offset(unit, attr.value())?; + } + _ => {} + }; + } + Err(e) => return Err(e), + } + } + + let function_index = functions.len(); + if ranges.for_each_range(&dwarf.sections, unit, |range| { + addresses.push(FunctionAddress { + range, + function: function_index, + }); + })? { + functions.push((dw_die_offset, LazyCell::new())); + } + } else { + entries.skip_attributes(abbrev.attributes())?; + } + } + } + + // The binary search requires the addresses to be sorted. + // + // It also requires them to be non-overlapping. In practice, overlapping + // function ranges are unlikely, so we don't try to handle that yet. + // + // It's possible for multiple functions to have the same address range if the + // compiler can detect and remove functions with identical code. In that case + // we'll nondeterministically return one of them. + addresses.sort_by_key(|x| x.range.begin); + + Ok(Functions { + functions: functions.into_boxed_slice(), + addresses: addresses.into_boxed_slice(), + }) + } + + pub(crate) fn find_address(&self, probe: u64) -> Option { + self.addresses + .binary_search_by(|address| { + if probe < address.range.begin { + Ordering::Greater + } else if probe >= address.range.end { + Ordering::Less + } else { + Ordering::Equal + } + }) + .ok() + } + + pub(crate) fn parse_inlined_functions( + &self, + unit: &gimli::Unit, + dwarf: &ResDwarf, + ) -> Result<(), Error> { + for function in &*self.functions { + function + .1 + .borrow_with(|| Function::parse(function.0, unit, dwarf)) + .as_ref() + .map_err(Error::clone)?; + } + Ok(()) + } +} + +impl Function { + pub(crate) fn parse( + dw_die_offset: gimli::UnitOffset, + unit: &gimli::Unit, + dwarf: &ResDwarf, + ) -> Result { + let mut entries = unit.entries_raw(Some(dw_die_offset))?; + let depth = entries.next_depth(); + let abbrev = entries.read_abbreviation()?.unwrap(); + debug_assert_eq!(abbrev.tag(), gimli::DW_TAG_subprogram); + + let mut name = None; + for spec in abbrev.attributes() { + match entries.read_attribute(*spec) { + Ok(ref attr) => { + match attr.name() { + gimli::DW_AT_linkage_name | gimli::DW_AT_MIPS_linkage_name => { + if let Ok(val) = dwarf.sections.attr_string(unit, attr.value()) { + name = Some(val); + } + } + gimli::DW_AT_name => { + if name.is_none() { + name = dwarf.sections.attr_string(unit, attr.value()).ok(); + } + } + gimli::DW_AT_abstract_origin | gimli::DW_AT_specification => { + if name.is_none() { + name = name_attr(attr.value(), unit, dwarf, 16)?; + } + } + _ => {} + }; + } + Err(e) => return Err(e), + } + } + + let mut inlined_functions = Vec::new(); + let mut inlined_addresses = Vec::new(); + Function::parse_children( + &mut entries, + depth, + unit, + dwarf, + &mut inlined_functions, + &mut inlined_addresses, + 0, + )?; + + // Sort ranges in "breadth-first traversal order", i.e. first by call_depth + // and then by range.begin. This allows finding the range containing an + // address at a certain depth using binary search. + // Note: Using DFS order, i.e. ordering by range.begin first and then by + // call_depth, would not work! Consider the two examples + // "[0..10 at depth 0], [0..2 at depth 1], [6..8 at depth 1]" and + // "[0..5 at depth 0], [0..2 at depth 1], [5..10 at depth 0], [6..8 at depth 1]". + // In this example, if you want to look up address 7 at depth 0, and you + // encounter [0..2 at depth 1], are you before or after the target range? + // You don't know. + inlined_addresses.sort_by(|r1, r2| { + if r1.call_depth < r2.call_depth { + Ordering::Less + } else if r1.call_depth > r2.call_depth { + Ordering::Greater + } else if r1.range.begin < r2.range.begin { + Ordering::Less + } else if r1.range.begin > r2.range.begin { + Ordering::Greater + } else { + Ordering::Equal + } + }); + + Ok(Function { + dw_die_offset, + name, + inlined_functions: inlined_functions.into_boxed_slice(), + inlined_addresses: inlined_addresses.into_boxed_slice(), + }) + } + + fn parse_children( + entries: &mut gimli::EntriesRaw, + depth: isize, + unit: &gimli::Unit, + dwarf: &ResDwarf, + inlined_functions: &mut Vec>, + inlined_addresses: &mut Vec, + inlined_depth: usize, + ) -> Result<(), Error> { + loop { + let dw_die_offset = entries.next_offset(); + let next_depth = entries.next_depth(); + if next_depth <= depth { + return Ok(()); + } + if let Some(abbrev) = entries.read_abbreviation()? { + match abbrev.tag() { + gimli::DW_TAG_subprogram => { + Function::skip(entries, abbrev, next_depth)?; + } + gimli::DW_TAG_inlined_subroutine => { + InlinedFunction::parse( + dw_die_offset, + entries, + abbrev, + next_depth, + unit, + dwarf, + inlined_functions, + inlined_addresses, + inlined_depth, + )?; + } + _ => { + entries.skip_attributes(abbrev.attributes())?; + } + } + } + } + } + + fn skip( + entries: &mut gimli::EntriesRaw, + abbrev: &gimli::Abbreviation, + depth: isize, + ) -> Result<(), Error> { + // TODO: use DW_AT_sibling + entries.skip_attributes(abbrev.attributes())?; + while entries.next_depth() > depth { + if let Some(abbrev) = entries.read_abbreviation()? { + entries.skip_attributes(abbrev.attributes())?; + } + } + Ok(()) + } + + /// Build the list of inlined functions that contain `probe`. + pub(crate) fn find_inlined_functions( + &self, + probe: u64, + ) -> iter::Rev>> { + // `inlined_functions` is ordered from outside to inside. + let mut inlined_functions = maybe_small::Vec::new(); + let mut inlined_addresses = &self.inlined_addresses[..]; + loop { + let current_depth = inlined_functions.len(); + // Look up (probe, current_depth) in inline_ranges. + // `inlined_addresses` is sorted in "breadth-first traversal order", i.e. + // by `call_depth` first, and then by `range.begin`. See the comment at + // the sort call for more information about why. + let search = inlined_addresses.binary_search_by(|range| { + if range.call_depth > current_depth { + Ordering::Greater + } else if range.call_depth < current_depth { + Ordering::Less + } else if range.range.begin > probe { + Ordering::Greater + } else if range.range.end <= probe { + Ordering::Less + } else { + Ordering::Equal + } + }); + if let Ok(index) = search { + let function_index = inlined_addresses[index].function; + inlined_functions.push(&self.inlined_functions[function_index]); + inlined_addresses = &inlined_addresses[index + 1..]; + } else { + break; + } + } + inlined_functions.into_iter().rev() + } +} + +impl InlinedFunction { + fn parse( + dw_die_offset: gimli::UnitOffset, + entries: &mut gimli::EntriesRaw, + abbrev: &gimli::Abbreviation, + depth: isize, + unit: &gimli::Unit, + dwarf: &ResDwarf, + inlined_functions: &mut Vec>, + inlined_addresses: &mut Vec, + inlined_depth: usize, + ) -> Result<(), Error> { + let mut ranges = RangeAttributes::default(); + let mut name = None; + let mut call_file = 0; + let mut call_line = 0; + let mut call_column = 0; + for spec in abbrev.attributes() { + match entries.read_attribute(*spec) { + Ok(ref attr) => match attr.name() { + gimli::DW_AT_low_pc => { + if let gimli::AttributeValue::Addr(val) = attr.value() { + ranges.low_pc = Some(val); + } + } + gimli::DW_AT_high_pc => match attr.value() { + gimli::AttributeValue::Addr(val) => ranges.high_pc = Some(val), + gimli::AttributeValue::Udata(val) => ranges.size = Some(val), + _ => {} + }, + gimli::DW_AT_ranges => { + ranges.ranges_offset = + dwarf.sections.attr_ranges_offset(unit, attr.value())?; + } + gimli::DW_AT_linkage_name | gimli::DW_AT_MIPS_linkage_name => { + if let Ok(val) = dwarf.sections.attr_string(unit, attr.value()) { + name = Some(val); + } + } + gimli::DW_AT_name => { + if name.is_none() { + name = dwarf.sections.attr_string(unit, attr.value()).ok(); + } + } + gimli::DW_AT_abstract_origin | gimli::DW_AT_specification => { + if name.is_none() { + name = name_attr(attr.value(), unit, dwarf, 16)?; + } + } + gimli::DW_AT_call_file => { + if let gimli::AttributeValue::FileIndex(fi) = attr.value() { + call_file = fi; + } + } + gimli::DW_AT_call_line => { + call_line = attr.udata_value().unwrap_or(0) as u32; + } + gimli::DW_AT_call_column => { + call_column = attr.udata_value().unwrap_or(0) as u32; + } + _ => {} + }, + Err(e) => return Err(e), + } + } + + let function_index = inlined_functions.len(); + inlined_functions.push(InlinedFunction { + dw_die_offset, + name, + call_file, + call_line, + call_column, + }); + + ranges.for_each_range(&dwarf.sections, unit, |range| { + inlined_addresses.push(InlinedFunctionAddress { + range, + call_depth: inlined_depth, + function: function_index, + }); + })?; + + Function::parse_children( + entries, + depth, + unit, + dwarf, + inlined_functions, + inlined_addresses, + inlined_depth + 1, + ) + } +} + +fn name_attr( + attr: gimli::AttributeValue, + unit: &gimli::Unit, + dwarf: &ResDwarf, + recursion_limit: usize, +) -> Result, Error> +where + R: gimli::Reader, +{ + if recursion_limit == 0 { + return Ok(None); + } + + match attr { + gimli::AttributeValue::UnitRef(offset) => name_entry(unit, offset, dwarf, recursion_limit), + gimli::AttributeValue::DebugInfoRef(dr) => { + let res_unit = dwarf.find_unit(dr)?; + name_entry( + &res_unit.dw_unit, + gimli::UnitOffset(dr.0 - res_unit.offset.0), + dwarf, + recursion_limit, + ) + } + gimli::AttributeValue::DebugInfoRefSup(dr) => { + if let Some(sup_dwarf) = dwarf.sup.as_ref() { + let res_unit = sup_dwarf.find_unit(dr)?; + name_entry( + &res_unit.dw_unit, + gimli::UnitOffset(dr.0 - res_unit.offset.0), + sup_dwarf, + recursion_limit, + ) + } else { + Ok(None) + } + } + _ => Ok(None), + } +} + +fn name_entry( + unit: &gimli::Unit, + offset: gimli::UnitOffset, + dwarf: &ResDwarf, + recursion_limit: usize, +) -> Result, Error> +where + R: gimli::Reader, +{ + let mut entries = unit.entries_raw(Some(offset))?; + let abbrev = if let Some(abbrev) = entries.read_abbreviation()? { + abbrev + } else { + return Err(gimli::Error::NoEntryAtGivenOffset); + }; + + let mut name = None; + let mut next = None; + for spec in abbrev.attributes() { + match entries.read_attribute(*spec) { + Ok(ref attr) => match attr.name() { + gimli::DW_AT_linkage_name | gimli::DW_AT_MIPS_linkage_name => { + if let Ok(val) = dwarf.sections.attr_string(unit, attr.value()) { + return Ok(Some(val)); + } + } + gimli::DW_AT_name => { + if let Ok(val) = dwarf.sections.attr_string(unit, attr.value()) { + name = Some(val); + } + } + gimli::DW_AT_abstract_origin | gimli::DW_AT_specification => { + next = Some(attr.value()); + } + _ => {} + }, + Err(e) => return Err(e), + } + } + + if name.is_some() { + return Ok(name); + } + + if let Some(next) = next { + return name_attr(next, unit, dwarf, recursion_limit - 1); + } + + Ok(None) +} diff --git a/crux-mir/lib/addr2line/src/lazy.rs b/crux-mir/lib/addr2line/src/lazy.rs new file mode 100644 index 000000000..280c76b46 --- /dev/null +++ b/crux-mir/lib/addr2line/src/lazy.rs @@ -0,0 +1,29 @@ +use core::cell::UnsafeCell; + +pub struct LazyCell { + contents: UnsafeCell>, +} +impl LazyCell { + pub fn new() -> LazyCell { + LazyCell { + contents: UnsafeCell::new(None), + } + } + + pub fn borrow_with(&self, closure: impl FnOnce() -> T) -> &T { + unsafe { + // First check if we're already initialized... + let ptr = self.contents.get(); + if let Some(val) = &*ptr { + return val; + } + // Note that while we're executing `closure` our `borrow_with` may + // be called recursively. This means we need to check again after + // the closure has executed. For that we use the `get_or_insert` + // method which will only perform mutation if we aren't already + // `Some`. + let val = closure(); + (*ptr).get_or_insert(val) + } + } +} diff --git a/crux-mir/lib/addr2line/src/lib.rs b/crux-mir/lib/addr2line/src/lib.rs new file mode 100644 index 000000000..b46a98393 --- /dev/null +++ b/crux-mir/lib/addr2line/src/lib.rs @@ -0,0 +1,1192 @@ +//! This crate provides a cross-platform library and binary for translating addresses into +//! function names, file names and line numbers. Given an address in an executable or an +//! offset in a section of a relocatable object, it uses the debugging information to +//! figure out which file name and line number are associated with it. +//! +//! When used as a library, files must first be loaded using the +//! [`object`](https://github.com/gimli-rs/object) crate. +//! A context can then be created with [`Context::new`](./struct.Context.html#method.new). +//! The context caches some of the parsed information so that multiple lookups are +//! efficient. +//! Location information is obtained with +//! [`Context::find_location`](./struct.Context.html#method.find_location) or +//! [`Context::find_location_range`](./struct.Context.html#method.find_location_range). +//! Function information is obtained with +//! [`Context::find_frames`](./struct.Context.html#method.find_frames), which returns +//! a frame for each inline function. Each frame contains both name and location. +//! +//! The crate has an example CLI wrapper around the library which provides some of +//! the functionality of the `addr2line` command line tool distributed with [GNU +//! binutils](https://www.gnu.org/software/binutils/). +//! +//! Currently this library only provides information from the DWARF debugging information, +//! which is parsed using [`gimli`](https://github.com/gimli-rs/gimli). The example CLI +//! wrapper also uses symbol table information provided by the `object` crate. +#![deny(missing_docs)] +#![no_std] + +#[allow(unused_imports)] +#[macro_use] +extern crate alloc; + +#[cfg(feature = "cpp_demangle")] +extern crate cpp_demangle; +#[cfg(feature = "fallible-iterator")] +pub extern crate fallible_iterator; +pub extern crate gimli; +#[cfg(feature = "object")] +pub extern crate object; +#[cfg(feature = "rustc-demangle")] +extern crate rustc_demangle; + +use alloc::borrow::Cow; +use alloc::boxed::Box; +#[cfg(feature = "object")] +use alloc::rc::Rc; +use alloc::string::{String, ToString}; +use alloc::sync::Arc; +use alloc::vec::Vec; + +use core::cmp::{self, Ordering}; +use core::iter; +use core::mem; +use core::num::NonZeroU64; +use core::u64; + +use crate::function::{Function, Functions, InlinedFunction}; +use crate::lazy::LazyCell; + +#[cfg(feature = "smallvec")] +mod maybe_small { + pub type Vec = smallvec::SmallVec<[T; 16]>; + pub type IntoIter = smallvec::IntoIter<[T; 16]>; +} +#[cfg(not(feature = "smallvec"))] +mod maybe_small { + pub type Vec = alloc::vec::Vec; + pub type IntoIter = alloc::vec::IntoIter; +} + +mod function; +mod lazy; + +type Error = gimli::Error; + +/// The state necessary to perform address to line translation. +/// +/// Constructing a `Context` is somewhat costly, so users should aim to reuse `Context`s +/// when performing lookups for many addresses in the same executable. +pub struct Context { + dwarf: ResDwarf, +} + +/// The type of `Context` that supports the `new` method. +#[cfg(feature = "std-object")] +pub type ObjectContext = Context>; + +#[cfg(feature = "std-object")] +impl Context> { + /// Construct a new `Context`. + /// + /// The resulting `Context` uses `gimli::EndianRcSlice`. + /// This means it is not thread safe, has no lifetime constraints (since it copies + /// the input data), and works for any endianity. + /// + /// Performance sensitive applications may want to use `Context::from_dwarf` + /// with a more specialised `gimli::Reader` implementation. + #[inline] + pub fn new<'data: 'file, 'file, O: object::Object<'data, 'file>>( + file: &'file O, + ) -> Result { + Self::new_with_sup(file, None) + } + + /// Construct a new `Context`. + /// + /// Optionally also use a supplementary object file. + /// + /// The resulting `Context` uses `gimli::EndianRcSlice`. + /// This means it is not thread safe, has no lifetime constraints (since it copies + /// the input data), and works for any endianity. + /// + /// Performance sensitive applications may want to use `Context::from_dwarf_with_sup` + /// with a more specialised `gimli::Reader` implementation. + pub fn new_with_sup<'data: 'file, 'file, O: object::Object<'data, 'file>>( + file: &'file O, + sup_file: Option<&'file O>, + ) -> Result { + let endian = if file.is_little_endian() { + gimli::RunTimeEndian::Little + } else { + gimli::RunTimeEndian::Big + }; + + fn load_section<'data: 'file, 'file, O, Endian>( + id: gimli::SectionId, + file: &'file O, + endian: Endian, + ) -> Result, Error> + where + O: object::Object<'data, 'file>, + Endian: gimli::Endianity, + { + use object::ObjectSection; + + let data = file + .section_by_name(id.name()) + .and_then(|section| section.uncompressed_data().ok()) + .unwrap_or(Cow::Borrowed(&[])); + Ok(gimli::EndianRcSlice::new(Rc::from(&*data), endian)) + } + + let mut dwarf = gimli::Dwarf::load(|id| load_section(id, file, endian))?; + if let Some(sup_file) = sup_file { + dwarf.load_sup(|id| load_section(id, sup_file, endian))?; + } + Context::from_dwarf(dwarf) + } +} + +impl Context { + /// Construct a new `Context` from DWARF sections. + /// + /// This method does not support using a supplementary object file. + pub fn from_sections( + debug_abbrev: gimli::DebugAbbrev, + debug_addr: gimli::DebugAddr, + debug_aranges: gimli::DebugAranges, + debug_info: gimli::DebugInfo, + debug_line: gimli::DebugLine, + debug_line_str: gimli::DebugLineStr, + debug_ranges: gimli::DebugRanges, + debug_rnglists: gimli::DebugRngLists, + debug_str: gimli::DebugStr, + debug_str_offsets: gimli::DebugStrOffsets, + default_section: R, + ) -> Result { + Self::from_dwarf(gimli::Dwarf { + debug_abbrev, + debug_addr, + debug_aranges, + debug_info, + debug_line, + debug_line_str, + debug_str, + debug_str_offsets, + debug_types: default_section.clone().into(), + locations: gimli::LocationLists::new( + default_section.clone().into(), + default_section.clone().into(), + ), + ranges: gimli::RangeLists::new(debug_ranges, debug_rnglists), + file_type: gimli::DwarfFileType::Main, + sup: None, + }) + } + + /// Construct a new `Context` from an existing [`gimli::Dwarf`] object. + #[inline] + pub fn from_dwarf(sections: gimli::Dwarf) -> Result { + let mut dwarf = ResDwarf::parse(Arc::new(sections))?; + dwarf.sup = match dwarf.sections.sup.clone() { + Some(sup_sections) => Some(Box::new(ResDwarf::parse(sup_sections)?)), + None => None, + }; + Ok(Context { dwarf }) + } + + /// The dwarf sections associated with this `Context`. + pub fn dwarf(&self) -> &gimli::Dwarf { + &self.dwarf.sections + } + + /// Finds the CUs for the function address given. + /// + /// There might be multiple CUs whose range contains this address. + /// Weak symbols have shown up in the wild which cause this to happen + /// but otherwise this can happen if the CU has non-contiguous functions + /// but only reports a single range. + /// + /// Consequently we return an iterator for all CUs which may contain the + /// address, and the caller must check if there is actually a function or + /// location in the CU for that address. + fn find_units(&self, probe: u64) -> impl Iterator> { + self.find_units_range(probe, probe + 1) + .map(|(unit, _range)| unit) + } + + /// Finds the CUs covering the range of addresses given. + /// + /// The range is [low, high) (ie, the upper bound is exclusive). This can return multiple + /// ranges for the same unit. + #[inline] + fn find_units_range( + &self, + probe_low: u64, + probe_high: u64, + ) -> impl Iterator, &gimli::Range)> { + // First up find the position in the array which could have our function + // address. + let pos = match self + .dwarf + .unit_ranges + .binary_search_by_key(&probe_high, |i| i.range.begin) + { + // Although unlikely, we could find an exact match. + Ok(i) => i + 1, + // No exact match was found, but this probe would fit at slot `i`. + // This means that slot `i` is bigger than `probe`, along with all + // indices greater than `i`, so we need to search all previous + // entries. + Err(i) => i, + }; + + // Once we have our index we iterate backwards from that position + // looking for a matching CU. + self.dwarf.unit_ranges[..pos] + .iter() + .rev() + .take_while(move |i| { + // We know that this CU's start is beneath the probe already because + // of our sorted array. + debug_assert!(i.range.begin <= probe_high); + + // Each entry keeps track of the maximum end address seen so far, + // starting from the beginning of the array of unit ranges. We're + // iterating in reverse so if our probe is beyond the maximum range + // of this entry, then it's guaranteed to not fit in any prior + // entries, so we break out. + probe_low < i.max_end + }) + .filter_map(move |i| { + // If this CU doesn't actually contain this address, move to the + // next CU. + if probe_low >= i.range.end || probe_high <= i.range.begin { + return None; + } + Some((&self.dwarf.units[i.unit_id], &i.range)) + }) + } + + /// Find the DWARF unit corresponding to the given virtual memory address. + pub fn find_dwarf_unit(&self, probe: u64) -> Option<&gimli::Unit> { + for unit in self.find_units(probe) { + match unit.find_function_or_location(probe, &self.dwarf) { + Ok((Some(_), _)) | Ok((_, Some(_))) => return Some(&unit.dw_unit), + _ => {} + } + } + None + } + + /// Find the source file and line corresponding to the given virtual memory address. + pub fn find_location(&self, probe: u64) -> Result>, Error> { + for unit in self.find_units(probe) { + if let Some(location) = unit.find_location(probe, &self.dwarf.sections)? { + return Ok(Some(location)); + } + } + Ok(None) + } + + /// Return source file and lines for a range of addresses. For each location it also + /// returns the address and size of the range of the underlying instructions. + pub fn find_location_range( + &self, + probe_low: u64, + probe_high: u64, + ) -> Result, Error> { + LocationRangeIter::new(self, probe_low, probe_high) + } + + /// Return an iterator for the function frames corresponding to the given virtual + /// memory address. + /// + /// If the probe address is not for an inline function then only one frame is + /// returned. + /// + /// If the probe address is for an inline function then the first frame corresponds + /// to the innermost inline function. Subsequent frames contain the caller and call + /// location, until an non-inline caller is reached. + pub fn find_frames(&self, probe: u64) -> Result, Error> { + for unit in self.find_units(probe) { + match unit.find_function_or_location(probe, &self.dwarf)? { + (Some(function), location) => { + let inlined_functions = function.find_inlined_functions(probe); + return Ok(FrameIter(FrameIterState::Frames(FrameIterFrames { + unit, + sections: &self.dwarf.sections, + function, + inlined_functions, + next: location, + }))); + } + (None, Some(location)) => { + return Ok(FrameIter(FrameIterState::Location(Some(location)))); + } + _ => {} + } + } + Ok(FrameIter(FrameIterState::Empty)) + } + + /// Initialize all line data structures. This is used for benchmarks. + #[doc(hidden)] + pub fn parse_lines(&self) -> Result<(), Error> { + for unit in &self.dwarf.units { + unit.parse_lines(&self.dwarf.sections)?; + } + Ok(()) + } + + /// Initialize all function data structures. This is used for benchmarks. + #[doc(hidden)] + pub fn parse_functions(&self) -> Result<(), Error> { + for unit in &self.dwarf.units { + unit.parse_functions(&self.dwarf)?; + } + Ok(()) + } + + /// Initialize all inlined function data structures. This is used for benchmarks. + #[doc(hidden)] + pub fn parse_inlined_functions(&self) -> Result<(), Error> { + for unit in &self.dwarf.units { + unit.parse_inlined_functions(&self.dwarf)?; + } + Ok(()) + } +} + +struct UnitRange { + unit_id: usize, + max_end: u64, + range: gimli::Range, +} + +struct ResDwarf { + unit_ranges: Vec, + units: Vec>, + sections: Arc>, + sup: Option>>, +} + +impl ResDwarf { + fn parse(sections: Arc>) -> Result { + // Find all the references to compilation units in .debug_aranges. + // Note that we always also iterate through all of .debug_info to + // find compilation units, because .debug_aranges may be missing some. + let mut aranges = Vec::new(); + let mut headers = sections.debug_aranges.headers(); + while let Some(header) = headers.next()? { + aranges.push((header.debug_info_offset(), header.offset())); + } + aranges.sort_by_key(|i| i.0); + + let mut unit_ranges = Vec::new(); + let mut res_units = Vec::new(); + let mut units = sections.units(); + while let Some(header) = units.next()? { + let unit_id = res_units.len(); + let offset = match header.offset().as_debug_info_offset() { + Some(offset) => offset, + None => continue, + }; + // We mainly want compile units, but we may need to follow references to entries + // within other units for function names. We don't need anything from type units. + match header.type_() { + gimli::UnitType::Type { .. } | gimli::UnitType::SplitType { .. } => continue, + _ => {} + } + let dw_unit = match sections.unit(header) { + Ok(dw_unit) => dw_unit, + Err(_) => continue, + }; + + let mut lang = None; + { + let mut entries = dw_unit.entries_raw(None)?; + + let abbrev = match entries.read_abbreviation()? { + Some(abbrev) => abbrev, + None => continue, + }; + + let mut ranges = RangeAttributes::default(); + for spec in abbrev.attributes() { + let attr = entries.read_attribute(*spec)?; + match attr.name() { + gimli::DW_AT_low_pc => { + if let gimli::AttributeValue::Addr(val) = attr.value() { + ranges.low_pc = Some(val); + } + } + gimli::DW_AT_high_pc => match attr.value() { + gimli::AttributeValue::Addr(val) => ranges.high_pc = Some(val), + gimli::AttributeValue::Udata(val) => ranges.size = Some(val), + _ => {} + }, + gimli::DW_AT_ranges => { + ranges.ranges_offset = + sections.attr_ranges_offset(&dw_unit, attr.value())?; + } + gimli::DW_AT_language => { + if let gimli::AttributeValue::Language(val) = attr.value() { + lang = Some(val); + } + } + _ => {} + } + } + + // Find the address ranges for the CU, using in order of preference: + // - DW_AT_ranges + // - .debug_aranges + // - DW_AT_low_pc/DW_AT_high_pc + // + // Using DW_AT_ranges before .debug_aranges is possibly an arbitrary choice, + // but the feeling is that DW_AT_ranges is more likely to be reliable or complete + // if it is present. + // + // .debug_aranges must be used before DW_AT_low_pc/DW_AT_high_pc because + // it has been observed on macOS that DW_AT_ranges was not emitted even for + // discontiguous CUs. + let i = match ranges.ranges_offset { + Some(_) => None, + None => aranges.binary_search_by_key(&offset, |x| x.0).ok(), + }; + if let Some(mut i) = i { + // There should be only one set per CU, but in practice multiple + // sets have been observed. This is probably a compiler bug, but + // either way we need to handle it. + while i > 0 && aranges[i - 1].0 == offset { + i -= 1; + } + for (_, aranges_offset) in aranges[i..].iter().take_while(|x| x.0 == offset) { + let aranges_header = sections.debug_aranges.header(*aranges_offset)?; + let mut aranges = aranges_header.entries(); + while let Some(arange) = aranges.next()? { + if arange.length() != 0 { + unit_ranges.push(UnitRange { + range: arange.range(), + unit_id, + max_end: 0, + }); + } + } + } + } else { + ranges.for_each_range(§ions, &dw_unit, |range| { + unit_ranges.push(UnitRange { + range, + unit_id, + max_end: 0, + }); + })?; + } + } + + res_units.push(ResUnit { + offset, + dw_unit, + lang, + lines: LazyCell::new(), + funcs: LazyCell::new(), + }); + } + + // Sort this for faster lookup in `find_unit_and_address` below. + unit_ranges.sort_by_key(|i| i.range.begin); + + // Calculate the `max_end` field now that we've determined the order of + // CUs. + let mut max = 0; + for i in unit_ranges.iter_mut() { + max = max.max(i.range.end); + i.max_end = max; + } + + Ok(ResDwarf { + units: res_units, + unit_ranges, + sections, + sup: None, + }) + } + + fn find_unit(&self, offset: gimli::DebugInfoOffset) -> Result<&ResUnit, Error> { + match self + .units + .binary_search_by_key(&offset.0, |unit| unit.offset.0) + { + // There is never a DIE at the unit offset or before the first unit. + Ok(_) | Err(0) => Err(gimli::Error::NoEntryAtGivenOffset), + Err(i) => Ok(&self.units[i - 1]), + } + } +} + +struct Lines { + files: Box<[String]>, + sequences: Box<[LineSequence]>, +} + +struct LineSequence { + start: u64, + end: u64, + rows: Box<[LineRow]>, +} + +struct LineRow { + address: u64, + file_index: u64, + line: u32, + column: u32, +} + +struct ResUnit { + offset: gimli::DebugInfoOffset, + dw_unit: gimli::Unit, + lang: Option, + lines: LazyCell>, + funcs: LazyCell, Error>>, +} + +impl ResUnit { + fn parse_lines(&self, sections: &gimli::Dwarf) -> Result, Error> { + let ilnp = match self.dw_unit.line_program { + Some(ref ilnp) => ilnp, + None => return Ok(None), + }; + self.lines + .borrow_with(|| { + let mut sequences = Vec::new(); + let mut sequence_rows = Vec::::new(); + let mut rows = ilnp.clone().rows(); + while let Some((_, row)) = rows.next_row()? { + if row.end_sequence() { + if let Some(start) = sequence_rows.first().map(|x| x.address) { + let end = row.address(); + let mut rows = Vec::new(); + mem::swap(&mut rows, &mut sequence_rows); + sequences.push(LineSequence { + start, + end, + rows: rows.into_boxed_slice(), + }); + } + continue; + } + + let address = row.address(); + let file_index = row.file_index(); + let line = row.line().map(NonZeroU64::get).unwrap_or(0) as u32; + let column = match row.column() { + gimli::ColumnType::LeftEdge => 0, + gimli::ColumnType::Column(x) => x.get() as u32, + }; + + if let Some(last_row) = sequence_rows.last_mut() { + if last_row.address == address { + last_row.file_index = file_index; + last_row.line = line; + last_row.column = column; + continue; + } + } + + sequence_rows.push(LineRow { + address, + file_index, + line, + column, + }); + } + sequences.sort_by_key(|x| x.start); + + let mut files = Vec::new(); + let header = ilnp.header(); + match header.file(0) { + Some(file) => files.push(self.render_file(file, header, sections)?), + None => files.push(String::from("")), // DWARF version <= 4 may not have 0th index + } + let mut index = 1; + while let Some(file) = header.file(index) { + files.push(self.render_file(file, header, sections)?); + index += 1; + } + + Ok(Lines { + files: files.into_boxed_slice(), + sequences: sequences.into_boxed_slice(), + }) + }) + .as_ref() + .map(Some) + .map_err(Error::clone) + } + + fn parse_functions(&self, dwarf: &ResDwarf) -> Result<&Functions, Error> { + self.funcs + .borrow_with(|| Functions::parse(&self.dw_unit, dwarf)) + .as_ref() + .map_err(Error::clone) + } + + fn parse_inlined_functions(&self, dwarf: &ResDwarf) -> Result<(), Error> { + self.funcs + .borrow_with(|| Functions::parse(&self.dw_unit, dwarf)) + .as_ref() + .map_err(Error::clone)? + .parse_inlined_functions(&self.dw_unit, dwarf) + } + + fn find_location( + &self, + probe: u64, + sections: &gimli::Dwarf, + ) -> Result>, Error> { + if let Some(mut iter) = LocationRangeUnitIter::new(self, sections, probe, probe + 1)? { + match iter.next() { + None => Ok(None), + Some((_addr, _len, loc)) => Ok(Some(loc)), + } + } else { + Ok(None) + } + } + + #[inline] + fn find_location_range( + &self, + probe_low: u64, + probe_high: u64, + sections: &gimli::Dwarf, + ) -> Result>, Error> { + LocationRangeUnitIter::new(self, sections, probe_low, probe_high) + } + + fn find_function_or_location( + &self, + probe: u64, + dwarf: &ResDwarf, + ) -> Result<(Option<&Function>, Option>), Error> { + let functions = self.parse_functions(dwarf)?; + let function = match functions.find_address(probe) { + Some(address) => { + let function_index = functions.addresses[address].function; + let (offset, ref function) = functions.functions[function_index]; + Some( + function + .borrow_with(|| Function::parse(offset, &self.dw_unit, dwarf)) + .as_ref() + .map_err(Error::clone)?, + ) + } + None => None, + }; + let location = self.find_location(probe, &dwarf.sections)?; + Ok((function, location)) + } + + fn render_file( + &self, + file: &gimli::FileEntry, + header: &gimli::LineProgramHeader, + sections: &gimli::Dwarf, + ) -> Result { + let mut path = if let Some(ref comp_dir) = self.dw_unit.comp_dir { + comp_dir.to_string_lossy()?.into_owned() + } else { + String::new() + }; + + if let Some(directory) = file.directory(header) { + path_push( + &mut path, + sections + .attr_string(&self.dw_unit, directory)? + .to_string_lossy()? + .as_ref(), + ); + } + + path_push( + &mut path, + sections + .attr_string(&self.dw_unit, file.path_name())? + .to_string_lossy()? + .as_ref(), + ); + + Ok(path) + } +} + +/// Iterator over `Location`s in a range of addresses, returned by `Context::find_location_range`. +pub struct LocationRangeIter<'ctx, R: gimli::Reader> { + unit_iter: Box, &'ctx gimli::Range)> + 'ctx>, + iter: Option>, + + probe_low: u64, + probe_high: u64, + sections: &'ctx gimli::Dwarf, +} + +impl<'ctx, R: gimli::Reader> LocationRangeIter<'ctx, R> { + #[inline] + fn new(ctx: &'ctx Context, probe_low: u64, probe_high: u64) -> Result { + let sections = &ctx.dwarf.sections; + let unit_iter = ctx.find_units_range(probe_low, probe_high); + Ok(Self { + unit_iter: Box::new(unit_iter), + iter: None, + probe_low, + probe_high, + sections, + }) + } + + fn next_loc(&mut self) -> Result)>, Error> { + loop { + let iter = self.iter.take(); + match iter { + None => match self.unit_iter.next() { + Some((unit, range)) => { + self.iter = unit.find_location_range( + cmp::max(self.probe_low, range.begin), + cmp::min(self.probe_high, range.end), + self.sections, + )?; + } + None => return Ok(None), + }, + Some(mut iter) => { + if let item @ Some(_) = iter.next() { + self.iter = Some(iter); + return Ok(item); + } + } + } + } + } +} + +impl<'ctx, R> Iterator for LocationRangeIter<'ctx, R> +where + R: gimli::Reader + 'ctx, +{ + type Item = (u64, u64, Location<'ctx>); + + #[inline] + fn next(&mut self) -> Option { + match self.next_loc() { + Err(_) => None, + Ok(loc) => loc, + } + } +} + +#[cfg(feature = "fallible-iterator")] +impl<'ctx, R> fallible_iterator::FallibleIterator for LocationRangeIter<'ctx, R> +where + R: gimli::Reader + 'ctx, +{ + type Item = (u64, u64, Location<'ctx>); + type Error = Error; + + #[inline] + fn next(&mut self) -> Result, Self::Error> { + self.next_loc() + } +} + +struct LocationRangeUnitIter<'ctx> { + lines: &'ctx Lines, + seqs: &'ctx [LineSequence], + seq_idx: usize, + row_idx: usize, + probe_high: u64, +} + +impl<'ctx> LocationRangeUnitIter<'ctx> { + fn new( + resunit: &'ctx ResUnit, + sections: &gimli::Dwarf, + probe_low: u64, + probe_high: u64, + ) -> Result, Error> { + let lines = resunit.parse_lines(sections)?; + + if let Some(lines) = lines { + // Find index for probe_low. + let seq_idx = lines.sequences.binary_search_by(|sequence| { + if probe_low < sequence.start { + Ordering::Greater + } else if probe_low >= sequence.end { + Ordering::Less + } else { + Ordering::Equal + } + }); + let seq_idx = match seq_idx { + Ok(x) => x, + Err(0) => 0, // probe below sequence, but range could overlap + Err(_) => lines.sequences.len(), + }; + + let row_idx = if let Some(seq) = lines.sequences.get(seq_idx) { + let idx = seq.rows.binary_search_by(|row| row.address.cmp(&probe_low)); + let idx = match idx { + Ok(x) => x, + Err(0) => 0, // probe below sequence, but range could overlap + Err(x) => x - 1, + }; + idx + } else { + 0 + }; + + Ok(Some(Self { + lines, + seqs: &*lines.sequences, + seq_idx, + row_idx, + probe_high, + })) + } else { + Ok(None) + } + } +} + +impl<'ctx> Iterator for LocationRangeUnitIter<'ctx> { + type Item = (u64, u64, Location<'ctx>); + + fn next(&mut self) -> Option<(u64, u64, Location<'ctx>)> { + loop { + let seq = match self.seqs.get(self.seq_idx) { + Some(seq) => seq, + None => break, + }; + + if seq.start >= self.probe_high { + break; + } + + match seq.rows.get(self.row_idx) { + Some(row) => { + if row.address >= self.probe_high { + break; + } + + let file = self + .lines + .files + .get(row.file_index as usize) + .map(String::as_str); + let nextaddr = seq + .rows + .get(self.row_idx + 1) + .map(|row| row.address) + .unwrap_or(seq.end); + + let item = ( + row.address, + nextaddr - row.address, + Location { + file, + line: if row.line != 0 { Some(row.line) } else { None }, + column: if row.column != 0 { + Some(row.column) + } else { + None + }, + }, + ); + self.row_idx += 1; + + return Some(item); + } + None => { + self.seq_idx += 1; + self.row_idx = 0; + } + } + } + None + } +} + +fn path_push(path: &mut String, p: &str) { + if has_unix_root(p) || has_windows_root(p) { + *path = p.to_string(); + } else { + let dir_separator = if has_windows_root(path.as_str()) { + '\\' + } else { + '/' + }; + + if !path.ends_with(dir_separator) { + path.push(dir_separator); + } + *path += p; + } +} + +/// Check if the path in the given string has a unix style root +fn has_unix_root(p: &str) -> bool { + p.starts_with('/') +} + +/// Check if the path in the given string has a windows style root +fn has_windows_root(p: &str) -> bool { + p.starts_with('\\') || p.get(1..3) == Some(":\\") +} +struct RangeAttributes { + low_pc: Option, + high_pc: Option, + size: Option, + ranges_offset: Option::Offset>>, +} + +impl Default for RangeAttributes { + fn default() -> Self { + RangeAttributes { + low_pc: None, + high_pc: None, + size: None, + ranges_offset: None, + } + } +} + +impl RangeAttributes { + fn for_each_range( + &self, + sections: &gimli::Dwarf, + unit: &gimli::Unit, + mut f: F, + ) -> Result { + let mut added_any = false; + let mut add_range = |range: gimli::Range| { + if range.begin < range.end { + f(range); + added_any = true + } + }; + if let Some(ranges_offset) = self.ranges_offset { + let mut range_list = sections.ranges(unit, ranges_offset)?; + while let Some(range) = range_list.next()? { + add_range(range); + } + } else if let (Some(begin), Some(end)) = (self.low_pc, self.high_pc) { + add_range(gimli::Range { begin, end }); + } else if let (Some(begin), Some(size)) = (self.low_pc, self.size) { + add_range(gimli::Range { + begin, + end: begin + size, + }); + } + Ok(added_any) + } +} + +/// An iterator over function frames. +pub struct FrameIter<'ctx, R>(FrameIterState<'ctx, R>) +where + R: gimli::Reader + 'ctx; + +enum FrameIterState<'ctx, R> +where + R: gimli::Reader + 'ctx, +{ + Empty, + Location(Option>), + Frames(FrameIterFrames<'ctx, R>), +} + +struct FrameIterFrames<'ctx, R> +where + R: gimli::Reader + 'ctx, +{ + unit: &'ctx ResUnit, + sections: &'ctx gimli::Dwarf, + function: &'ctx Function, + inlined_functions: iter::Rev>>, + next: Option>, +} + +impl<'ctx, R> FrameIter<'ctx, R> +where + R: gimli::Reader + 'ctx, +{ + /// Advances the iterator and returns the next frame. + pub fn next(&mut self) -> Result>, Error> { + let frames = match &mut self.0 { + FrameIterState::Empty => return Ok(None), + FrameIterState::Location(location) => { + // We can't move out of a mutable reference, so use `take` instead. + let location = location.take(); + self.0 = FrameIterState::Empty; + return Ok(Some(Frame { + dw_die_offset: None, + function: None, + location, + })); + } + FrameIterState::Frames(frames) => frames, + }; + + let loc = frames.next.take(); + let func = match frames.inlined_functions.next() { + Some(func) => func, + None => { + let frame = Frame { + dw_die_offset: Some(frames.function.dw_die_offset), + function: frames.function.name.clone().map(|name| FunctionName { + name, + language: frames.unit.lang, + }), + location: loc, + }; + self.0 = FrameIterState::Empty; + return Ok(Some(frame)); + } + }; + + let mut next = Location { + file: None, + line: if func.call_line != 0 { + Some(func.call_line) + } else { + None + }, + column: if func.call_column != 0 { + Some(func.call_column) + } else { + None + }, + }; + if func.call_file != 0 { + if let Some(lines) = frames.unit.parse_lines(frames.sections)? { + next.file = lines.files.get(func.call_file as usize).map(String::as_str); + } + } + frames.next = Some(next); + + Ok(Some(Frame { + dw_die_offset: Some(func.dw_die_offset), + function: func.name.clone().map(|name| FunctionName { + name, + language: frames.unit.lang, + }), + location: loc, + })) + } +} + +#[cfg(feature = "fallible-iterator")] +impl<'ctx, R> fallible_iterator::FallibleIterator for FrameIter<'ctx, R> +where + R: gimli::Reader + 'ctx, +{ + type Item = Frame<'ctx, R>; + type Error = Error; + + #[inline] + fn next(&mut self) -> Result>, Error> { + self.next() + } +} + +/// A function frame. +pub struct Frame<'ctx, R: gimli::Reader> { + /// The DWARF unit offset corresponding to the DIE of the function. + pub dw_die_offset: Option>, + /// The name of the function. + pub function: Option>, + /// The source location corresponding to this frame. + pub location: Option>, +} + +/// A function name. +pub struct FunctionName { + /// The name of the function. + pub name: R, + /// The language of the compilation unit containing this function. + pub language: Option, +} + +impl FunctionName { + /// The raw name of this function before demangling. + pub fn raw_name(&self) -> Result, Error> { + self.name.to_string_lossy() + } + + /// The name of this function after demangling (if applicable). + pub fn demangle(&self) -> Result, Error> { + self.raw_name().map(|x| demangle_auto(x, self.language)) + } +} + +/// Demangle a symbol name using the demangling scheme for the given language. +/// +/// Returns `None` if demangling failed or is not required. +#[allow(unused_variables)] +pub fn demangle(name: &str, language: gimli::DwLang) -> Option { + match language { + #[cfg(feature = "rustc-demangle")] + gimli::DW_LANG_Rust => rustc_demangle::try_demangle(name) + .ok() + .as_ref() + .map(|x| format!("{:#}", x)), + #[cfg(feature = "cpp_demangle")] + gimli::DW_LANG_C_plus_plus + | gimli::DW_LANG_C_plus_plus_03 + | gimli::DW_LANG_C_plus_plus_11 + | gimli::DW_LANG_C_plus_plus_14 => cpp_demangle::Symbol::new(name) + .ok() + .and_then(|x| x.demangle(&Default::default()).ok()), + _ => None, + } +} + +/// Apply 'best effort' demangling of a symbol name. +/// +/// If `language` is given, then only the demangling scheme for that language +/// is used. +/// +/// If `language` is `None`, then heuristics are used to determine how to +/// demangle the name. Currently, these heuristics are very basic. +/// +/// If demangling fails or is not required, then `name` is returned unchanged. +pub fn demangle_auto(name: Cow, language: Option) -> Cow { + match language { + Some(language) => demangle(name.as_ref(), language), + None => demangle(name.as_ref(), gimli::DW_LANG_Rust) + .or_else(|| demangle(name.as_ref(), gimli::DW_LANG_C_plus_plus)), + } + .map(Cow::from) + .unwrap_or(name) +} + +/// A source location. +pub struct Location<'a> { + /// The file name. + pub file: Option<&'a str>, + /// The line number. + pub line: Option, + /// The column number. + pub column: Option, +} + +#[cfg(test)] +mod tests { + #[test] + fn context_is_send() { + fn assert_is_send() {} + assert_is_send::>>(); + } +} diff --git a/crux-mir/lib/addr2line/tests/correctness.rs b/crux-mir/lib/addr2line/tests/correctness.rs new file mode 100644 index 000000000..3f7b43373 --- /dev/null +++ b/crux-mir/lib/addr2line/tests/correctness.rs @@ -0,0 +1,91 @@ +extern crate addr2line; +extern crate fallible_iterator; +extern crate findshlibs; +extern crate gimli; +extern crate memmap; +extern crate object; + +use addr2line::Context; +use fallible_iterator::FallibleIterator; +use findshlibs::{IterationControl, SharedLibrary, TargetSharedLibrary}; +use object::Object; +use std::fs::File; + +fn find_debuginfo() -> memmap::Mmap { + let path = std::env::current_exe().unwrap(); + let file = File::open(&path).unwrap(); + let map = unsafe { memmap::Mmap::map(&file).unwrap() }; + let file = &object::File::parse(&*map).unwrap(); + if let Ok(uuid) = file.mach_uuid() { + for candidate in path.parent().unwrap().read_dir().unwrap() { + let path = candidate.unwrap().path(); + if !path.to_str().unwrap().ends_with(".dSYM") { + continue; + } + for candidate in path.join("Contents/Resources/DWARF").read_dir().unwrap() { + let path = candidate.unwrap().path(); + let file = File::open(&path).unwrap(); + let map = unsafe { memmap::Mmap::map(&file).unwrap() }; + let file = &object::File::parse(&*map).unwrap(); + if file.mach_uuid().unwrap() == uuid { + return map; + } + } + } + } + + return map; +} + +#[test] +fn correctness() { + let map = find_debuginfo(); + let file = &object::File::parse(&*map).unwrap(); + let ctx = Context::new(file).unwrap(); + + let mut bias = None; + TargetSharedLibrary::each(|lib| { + bias = Some(lib.virtual_memory_bias().0 as u64); + IterationControl::Break + }); + + let test = |sym: u64, expected_prefix: &str| { + let ip = sym.wrapping_sub(bias.unwrap()); + + let frames = ctx.find_frames(ip).unwrap(); + let frame = frames.last().unwrap().unwrap(); + let name = frame.function.as_ref().unwrap().demangle().unwrap(); + // Old rust versions generate DWARF with wrong linkage name, + // so only check the start. + if !name.starts_with(expected_prefix) { + panic!("incorrect name '{}', expected {:?}", name, expected_prefix); + } + }; + + test(test_function as u64, "correctness::test_function"); + test( + small::test_function as u64, + "correctness::small::test_function", + ); + test(auxiliary::foo as u64, "auxiliary::foo"); +} + +mod small { + pub fn test_function() { + println!("y"); + } +} + +fn test_function() { + println!("x"); +} + +#[test] +fn zero_function() { + let map = find_debuginfo(); + let file = &object::File::parse(&*map).unwrap(); + let ctx = Context::new(file).unwrap(); + for probe in 0..10 { + assert!(ctx.find_frames(probe).unwrap().count().unwrap() < 10); + } +} diff --git a/crux-mir/lib/addr2line/tests/output_equivalence.rs b/crux-mir/lib/addr2line/tests/output_equivalence.rs new file mode 100644 index 000000000..9dc366672 --- /dev/null +++ b/crux-mir/lib/addr2line/tests/output_equivalence.rs @@ -0,0 +1,145 @@ +extern crate backtrace; +extern crate findshlibs; +extern crate rustc_test as test; + +use std::env; +use std::ffi::OsStr; +use std::path::Path; +use std::process::Command; + +use backtrace::Backtrace; +use findshlibs::{IterationControl, SharedLibrary, TargetSharedLibrary}; +use test::{ShouldPanic, TestDesc, TestDescAndFn, TestFn, TestName}; + +fn make_trace() -> Vec { + fn foo() -> Backtrace { + bar() + } + #[inline(never)] + fn bar() -> Backtrace { + baz() + } + #[inline(always)] + fn baz() -> Backtrace { + Backtrace::new_unresolved() + } + + let mut base_addr = None; + TargetSharedLibrary::each(|lib| { + base_addr = Some(lib.virtual_memory_bias().0 as isize); + IterationControl::Break + }); + let addrfix = -base_addr.unwrap(); + + let trace = foo(); + trace + .frames() + .iter() + .take(5) + .map(|x| format!("{:p}", (x.ip() as *const u8).wrapping_offset(addrfix))) + .collect() +} + +fn run_cmd>(exe: P, me: &Path, flags: Option<&str>, trace: &str) -> String { + let mut cmd = Command::new(exe); + cmd.env("LC_ALL", "C"); // GNU addr2line is localized, we aren't + cmd.env("RUST_BACKTRACE", "1"); // if a child crashes, we want to know why + + if let Some(flags) = flags { + cmd.arg(flags); + } + cmd.arg("--exe").arg(me).arg(trace); + + let output = cmd.output().unwrap(); + + assert!(output.status.success()); + String::from_utf8(output.stdout).unwrap() +} + +fn run_test(flags: Option<&str>) { + let me = env::current_exe().unwrap(); + let mut exe = me.clone(); + assert!(exe.pop()); + if exe.file_name().unwrap().to_str().unwrap() == "deps" { + assert!(exe.pop()); + } + exe.push("examples"); + exe.push("addr2line"); + + assert!(exe.is_file()); + + let trace = make_trace(); + + // HACK: GNU addr2line has a bug where looking up multiple addresses can cause the second + // lookup to fail. Workaround by doing one address at a time. + for addr in &trace { + let theirs = run_cmd("addr2line", &me, flags, addr); + let ours = run_cmd(&exe, &me, flags, addr); + + // HACK: GNU addr2line does not tidy up paths properly, causing double slashes to be printed. + // We consider our behavior to be correct, so we fix their output to match ours. + let theirs = theirs.replace("//", "/"); + + assert!( + theirs == ours, + "Output not equivalent: + +$ addr2line {0} --exe {1} {2} +{4} +$ {3} {0} --exe {1} {2} +{5} + + +", + flags.unwrap_or(""), + me.display(), + trace.join(" "), + exe.display(), + theirs, + ours + ); + } +} + +static FLAGS: &'static str = "aipsf"; + +fn make_tests() -> Vec { + (0..(1 << FLAGS.len())) + .map(|bits| { + if bits == 0 { + None + } else { + let mut param = String::new(); + param.push('-'); + for (i, flag) in FLAGS.chars().enumerate() { + if (bits & (1 << i)) != 0 { + param.push(flag); + } + } + Some(param) + } + }) + .map(|param| TestDescAndFn { + desc: TestDesc { + name: TestName::DynTestName(format!( + "addr2line {}", + param.as_ref().map_or("", String::as_str) + )), + ignore: false, + should_panic: ShouldPanic::No, + allow_fail: false, + }, + testfn: TestFn::DynTestFn(Box::new(move || { + run_test(param.as_ref().map(String::as_str)) + })), + }) + .collect() +} + +fn main() { + if !cfg!(target_os = "linux") { + return; + } + let args: Vec<_> = env::args().collect(); + test::test_main(&args, make_tests()); +} diff --git a/crux-mir/lib/addr2line/tests/parse.rs b/crux-mir/lib/addr2line/tests/parse.rs new file mode 100644 index 000000000..91d66e382 --- /dev/null +++ b/crux-mir/lib/addr2line/tests/parse.rs @@ -0,0 +1,118 @@ +extern crate addr2line; +extern crate memmap; +extern crate object; + +use std::borrow::Cow; +use std::env; +use std::fs::File; +use std::path::{self, PathBuf}; + +use object::Object; + +fn release_fixture_path() -> PathBuf { + if let Ok(p) = env::var("ADDR2LINE_FIXTURE_PATH") { + return p.into(); + } + + let mut path = PathBuf::new(); + if let Ok(dir) = env::var("CARGO_MANIFEST_DIR") { + path.push(dir); + } + path.push("fixtures"); + path.push("addr2line-release"); + path +} + +fn with_file(target: &path::Path, f: F) { + let file = File::open(target).unwrap(); + let map = unsafe { memmap::Mmap::map(&file).unwrap() }; + let file = object::File::parse(&*map).unwrap(); + f(&file) +} + +fn dwarf_load<'a>(object: &object::File<'a>) -> gimli::Dwarf> { + let load_section = |id: gimli::SectionId| -> Result, gimli::Error> { + use object::ObjectSection; + + let data = object + .section_by_name(id.name()) + .and_then(|section| section.data().ok()) + .unwrap_or(&[][..]); + Ok(Cow::Borrowed(data)) + }; + gimli::Dwarf::load(&load_section).unwrap() +} + +fn dwarf_borrow<'a>( + dwarf: &'a gimli::Dwarf>, +) -> gimli::Dwarf> { + let borrow_section: &dyn for<'b> Fn( + &'b Cow<[u8]>, + ) -> gimli::EndianSlice<'b, gimli::LittleEndian> = + &|section| gimli::EndianSlice::new(&*section, gimli::LittleEndian); + dwarf.borrow(&borrow_section) +} + +#[test] +fn parse_base_rc() { + let target = release_fixture_path(); + + with_file(&target, |file| { + addr2line::ObjectContext::new(file).unwrap(); + }); +} + +#[test] +fn parse_base_slice() { + let target = release_fixture_path(); + + with_file(&target, |file| { + let dwarf = dwarf_load(file); + let dwarf = dwarf_borrow(&dwarf); + addr2line::Context::from_dwarf(dwarf).unwrap(); + }); +} + +#[test] +fn parse_lines_rc() { + let target = release_fixture_path(); + + with_file(&target, |file| { + let context = addr2line::ObjectContext::new(file).unwrap(); + context.parse_lines().unwrap(); + }); +} + +#[test] +fn parse_lines_slice() { + let target = release_fixture_path(); + + with_file(&target, |file| { + let dwarf = dwarf_load(file); + let dwarf = dwarf_borrow(&dwarf); + let context = addr2line::Context::from_dwarf(dwarf).unwrap(); + context.parse_lines().unwrap(); + }); +} + +#[test] +fn parse_functions_rc() { + let target = release_fixture_path(); + + with_file(&target, |file| { + let context = addr2line::ObjectContext::new(file).unwrap(); + context.parse_functions().unwrap(); + }); +} + +#[test] +fn parse_functions_slice() { + let target = release_fixture_path(); + + with_file(&target, |file| { + let dwarf = dwarf_load(file); + let dwarf = dwarf_borrow(&dwarf); + let context = addr2line::Context::from_dwarf(dwarf).unwrap(); + context.parse_functions().unwrap(); + }); +} diff --git a/crux-mir/lib/adler/.cargo-ok b/crux-mir/lib/adler/.cargo-ok new file mode 100644 index 000000000..b5754e203 --- /dev/null +++ b/crux-mir/lib/adler/.cargo-ok @@ -0,0 +1 @@ +ok \ No newline at end of file diff --git a/crux-mir/lib/adler/.cargo_vcs_info.json b/crux-mir/lib/adler/.cargo_vcs_info.json new file mode 100644 index 000000000..b40982aec --- /dev/null +++ b/crux-mir/lib/adler/.cargo_vcs_info.json @@ -0,0 +1,5 @@ +{ + "git": { + "sha1": "a94f525f62698d699d1fb3cc9112db8c35662b16" + } +} diff --git a/crux-mir/lib/adler/.github/workflows/ci.yml b/crux-mir/lib/adler/.github/workflows/ci.yml new file mode 100644 index 000000000..cf3189463 --- /dev/null +++ b/crux-mir/lib/adler/.github/workflows/ci.yml @@ -0,0 +1,81 @@ +name: CI + +on: + push: + branches: + - master + - staging + - trying + pull_request: + branches: + - master + +env: + CARGO_TERM_COLOR: always + RUSTFLAGS: "--deny warnings" + MSRV: 1.31.0 + NO_STD_TARGET: thumbv6m-none-eabi + +jobs: + test: + strategy: + matrix: + rust: + - stable + - nightly + os: + - ubuntu-latest + - macOS-latest + - windows-latest + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.rust }} + override: true + - name: Build + run: cargo build --all --all-targets + - name: Run tests + run: | + cargo test --all --all-targets + cargo test --all --no-default-features + + no-std: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + target: ${{ env.NO_STD_TARGET }} + - name: Build + run: cargo build --verbose --no-default-features --target ${{ env.NO_STD_TARGET }} + + msrv: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ env.MSRV }} + override: true + - name: Build + run: cargo build --verbose + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + components: rustfmt + - name: Check code formatting + run: cargo fmt -- --check diff --git a/crux-mir/lib/stdarch/.gitignore b/crux-mir/lib/adler/.gitignore similarity index 62% rename from crux-mir/lib/stdarch/.gitignore rename to crux-mir/lib/adler/.gitignore index c16125ed7..a9d37c560 100644 --- a/crux-mir/lib/stdarch/.gitignore +++ b/crux-mir/lib/adler/.gitignore @@ -1,4 +1,2 @@ -Cargo.lock -.*.swp target -tags \ No newline at end of file +Cargo.lock diff --git a/crux-mir/lib/adler/CHANGELOG.md b/crux-mir/lib/adler/CHANGELOG.md new file mode 100644 index 000000000..87994376e --- /dev/null +++ b/crux-mir/lib/adler/CHANGELOG.md @@ -0,0 +1,63 @@ +# Changelog + +## Unreleased + +No changes. + +## [1.0.2 - 2021-02-26](https://github.com/jonas-schievink/adler/releases/tag/v1.0.2) + +- Fix doctest on big-endian systems ([#9]). + +[#9]: https://github.com/jonas-schievink/adler/pull/9 + +## [1.0.1 - 2020-11-08](https://github.com/jonas-schievink/adler/releases/tag/v1.0.1) + +### Fixes + +- Fix documentation on docs.rs. + +## [1.0.0 - 2020-11-08](https://github.com/jonas-schievink/adler/releases/tag/v1.0.0) + +### Fixes + +- Fix `cargo test --no-default-features` ([#5]). + +### Improvements + +- Extended and clarified documentation. +- Added more rustdoc examples. +- Extended CI to test the crate with `--no-default-features`. + +### Breaking Changes + +- `adler32_reader` now takes its generic argument by value instead of as a `&mut`. +- Renamed `adler32_reader` to `adler32`. + +## [0.2.3 - 2020-07-11](https://github.com/jonas-schievink/adler/releases/tag/v0.2.3) + +- Process 4 Bytes at a time, improving performance by up to 50% ([#2]). + +## [0.2.2 - 2020-06-27](https://github.com/jonas-schievink/adler/releases/tag/v0.2.2) + +- Bump MSRV to 1.31.0. + +## [0.2.1 - 2020-06-27](https://github.com/jonas-schievink/adler/releases/tag/v0.2.1) + +- Add a few `#[inline]` annotations to small functions. +- Fix CI badge. +- Allow integration into libstd. + +## [0.2.0 - 2020-06-27](https://github.com/jonas-schievink/adler/releases/tag/v0.2.0) + +- Support `#![no_std]` when using `default-features = false`. +- Improve performance by around 7x. +- Support Rust 1.8.0. +- Improve API naming. + +## [0.1.0 - 2020-06-26](https://github.com/jonas-schievink/adler/releases/tag/v0.1.0) + +Initial release. + + +[#2]: https://github.com/jonas-schievink/adler/pull/2 +[#5]: https://github.com/jonas-schievink/adler/pull/5 diff --git a/crux-mir/lib/adler/Cargo.toml b/crux-mir/lib/adler/Cargo.toml new file mode 100644 index 000000000..1dacd2c70 --- /dev/null +++ b/crux-mir/lib/adler/Cargo.toml @@ -0,0 +1,64 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies +# +# If you believe there's an error in this file please file an +# issue against the rust-lang/cargo repository. If you're +# editing this file be aware that the upstream Cargo.toml +# will likely look very different (and much more reasonable) + +[package] +name = "adler" +version = "1.0.2" +authors = ["Jonas Schievink "] +description = "A simple clean-room implementation of the Adler-32 checksum" +documentation = "https://docs.rs/adler/" +readme = "README.md" +keywords = ["checksum", "integrity", "hash", "adler32", "zlib"] +categories = ["algorithms"] +license = "0BSD OR MIT OR Apache-2.0" +repository = "https://github.com/jonas-schievink/adler.git" +[package.metadata.docs.rs] +rustdoc-args = ["--cfg=docsrs"] + +[package.metadata.release] +no-dev-version = true +pre-release-commit-message = "Release {{version}}" +tag-message = "{{version}}" + +[[package.metadata.release.pre-release-replacements]] +file = "CHANGELOG.md" +replace = "## Unreleased\n\nNo changes.\n\n## [{{version}} - {{date}}](https://github.com/jonas-schievink/adler/releases/tag/v{{version}})\n" +search = "## Unreleased\n" + +[[package.metadata.release.pre-release-replacements]] +file = "README.md" +replace = "adler = \"{{version}}\"" +search = "adler = \"[a-z0-9\\\\.-]+\"" + +[[package.metadata.release.pre-release-replacements]] +file = "src/lib.rs" +replace = "https://docs.rs/adler/{{version}}" +search = "https://docs.rs/adler/[a-z0-9\\.-]+" + +[[bench]] +name = "bench" +harness = false +[dependencies.compiler_builtins] +version = "0.1.2" +optional = true + +[dependencies.core] +version = "1.0.0" +optional = true +package = "rustc-std-workspace-core" +[dev-dependencies.criterion] +version = "0.3.2" + +[features] +default = ["std"] +rustc-dep-of-std = ["core", "compiler_builtins"] +std = [] diff --git a/crux-mir/lib/adler/Cargo.toml.orig b/crux-mir/lib/adler/Cargo.toml.orig new file mode 100644 index 000000000..8c3865b1a --- /dev/null +++ b/crux-mir/lib/adler/Cargo.toml.orig @@ -0,0 +1,68 @@ +[package] +name = "adler" +version = "1.0.2" +authors = ["Jonas Schievink "] +description = "A simple clean-room implementation of the Adler-32 checksum" +documentation = "https://docs.rs/adler/" +repository = "https://github.com/jonas-schievink/adler.git" +keywords = ["checksum", "integrity", "hash", "adler32", "zlib"] +categories = ["algorithms"] +readme = "README.md" +license = "0BSD OR MIT OR Apache-2.0" + +[[bench]] +name = "bench" +harness = false + +[dependencies] +# Internal features, only used when building as part of libstd, not part of the +# stable interface of this crate. +core = { version = '1.0.0', optional = true, package = 'rustc-std-workspace-core' } +compiler_builtins = { version = '0.1.2', optional = true } + +[dev-dependencies] +criterion = "0.3.2" + +[features] +# Disable default features to enable `#![no_std]` support. +default = ["std"] +std = [] + +# Internal feature, only used when building as part of libstd, not part of the +# stable interface of this crate. +rustc-dep-of-std = ['core', 'compiler_builtins'] + + +[package.metadata.docs.rs] +rustdoc-args = [ "--cfg=docsrs" ] + +# cargo-release configuration +[package.metadata.release] +tag-message = "{{version}}" +no-dev-version = true +pre-release-commit-message = "Release {{version}}" + +# Change the changelog's `Unreleased` section to refer to this release and +# prepend a new `Unreleased` section +[[package.metadata.release.pre-release-replacements]] +file = "CHANGELOG.md" +search = "## Unreleased\n" +replace = """ +## Unreleased + +No changes. + +## [{{version}} - {{date}}](https://github.com/jonas-schievink/adler/releases/tag/v{{version}}) +""" + +# Bump the version inside the example manifest in `README.md` +[[package.metadata.release.pre-release-replacements]] +file = "README.md" +search = 'adler = "[a-z0-9\\.-]+"' +replace = 'adler = "{{version}}"' + +# Bump the version referenced by the `html_root_url` attribute in `lib.rs` +[[package.metadata.release.pre-release-replacements]] +file = "src/lib.rs" +search = "https://docs.rs/adler/[a-z0-9\\.-]+" +replace = "https://docs.rs/adler/{{version}}" diff --git a/crux-mir/lib/adler/LICENSE-0BSD b/crux-mir/lib/adler/LICENSE-0BSD new file mode 100644 index 000000000..89336aaca --- /dev/null +++ b/crux-mir/lib/adler/LICENSE-0BSD @@ -0,0 +1,12 @@ +Copyright (C) Jonas Schievink + +Permission to use, copy, modify, and/or distribute this software for +any purpose with or without fee is hereby granted. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN +AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/crux-mir/lib/adler/LICENSE-APACHE b/crux-mir/lib/adler/LICENSE-APACHE new file mode 100644 index 000000000..c98d27d4f --- /dev/null +++ b/crux-mir/lib/adler/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/LICENSE-2.0 + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/crux-mir/lib/rustc-demangle/LICENSE-MIT b/crux-mir/lib/adler/LICENSE-MIT similarity index 96% rename from crux-mir/lib/rustc-demangle/LICENSE-MIT rename to crux-mir/lib/adler/LICENSE-MIT index 39e0ed660..31aa79387 100644 --- a/crux-mir/lib/rustc-demangle/LICENSE-MIT +++ b/crux-mir/lib/adler/LICENSE-MIT @@ -1,5 +1,3 @@ -Copyright (c) 2014 Alex Crichton - Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the diff --git a/crux-mir/lib/adler/README.md b/crux-mir/lib/adler/README.md new file mode 100644 index 000000000..ba140d52a --- /dev/null +++ b/crux-mir/lib/adler/README.md @@ -0,0 +1,39 @@ +# Adler-32 checksums for Rust + +[![crates.io](https://img.shields.io/crates/v/adler.svg)](https://crates.io/crates/adler) +[![docs.rs](https://docs.rs/adler/badge.svg)](https://docs.rs/adler/) +![CI](https://github.com/jonas-schievink/adler/workflows/CI/badge.svg) + +This crate provides a simple implementation of the Adler-32 checksum, used in +the zlib compression format. + +Please refer to the [changelog](CHANGELOG.md) to see what changed in the last +releases. + +## Features + +- Permissively licensed (0BSD) clean-room implementation. +- Zero dependencies. +- Zero `unsafe`. +- Decent performance (3-4 GB/s). +- Supports `#![no_std]` (with `default-features = false`). + +## Usage + +Add an entry to your `Cargo.toml`: + +```toml +[dependencies] +adler = "1.0.2" +``` + +Check the [API Documentation](https://docs.rs/adler/) for how to use the +crate's functionality. + +## Rust version support + +Currently, this crate supports all Rust versions starting at Rust 1.31.0. + +Bumping the Minimum Supported Rust Version (MSRV) is *not* considered a breaking +change, but will not be done without good reasons. The latest 3 stable Rust +versions will always be supported no matter what. diff --git a/crux-mir/lib/adler/RELEASE_PROCESS.md b/crux-mir/lib/adler/RELEASE_PROCESS.md new file mode 100644 index 000000000..71a367341 --- /dev/null +++ b/crux-mir/lib/adler/RELEASE_PROCESS.md @@ -0,0 +1,13 @@ +# What to do to publish a new release + +1. Ensure all notable changes are in the changelog under "Unreleased". + +2. Execute `cargo release ` to bump version(s), tag and publish + everything. External subcommand, must be installed with `cargo install + cargo-release`. + + `` can be one of `major|minor|patch`. If this is the first release + (`0.1.0`), use `minor`, since the version starts out as `0.0.0`. + +3. Go to the GitHub releases, edit the just-pushed tag. Copy the release notes + from the changelog. diff --git a/crux-mir/lib/adler/benches/bench.rs b/crux-mir/lib/adler/benches/bench.rs new file mode 100644 index 000000000..0969f5948 --- /dev/null +++ b/crux-mir/lib/adler/benches/bench.rs @@ -0,0 +1,109 @@ +extern crate adler; +extern crate criterion; + +use adler::{adler32_slice, Adler32}; +use criterion::{criterion_group, criterion_main, Criterion, Throughput}; + +fn simple(c: &mut Criterion) { + { + const SIZE: usize = 100; + + let mut group = c.benchmark_group("simple-100b"); + group.throughput(Throughput::Bytes(SIZE as u64)); + group.bench_function("zeroes-100", |bencher| { + bencher.iter(|| { + adler32_slice(&[0; SIZE]); + }); + }); + group.bench_function("ones-100", |bencher| { + bencher.iter(|| { + adler32_slice(&[0xff; SIZE]); + }); + }); + } + + { + const SIZE: usize = 1024; + + let mut group = c.benchmark_group("simple-1k"); + group.throughput(Throughput::Bytes(SIZE as u64)); + + group.bench_function("zeroes-1k", |bencher| { + bencher.iter(|| { + adler32_slice(&[0; SIZE]); + }); + }); + + group.bench_function("ones-1k", |bencher| { + bencher.iter(|| { + adler32_slice(&[0xff; SIZE]); + }); + }); + } + + { + const SIZE: usize = 1024 * 1024; + + let mut group = c.benchmark_group("simple-1m"); + group.throughput(Throughput::Bytes(SIZE as u64)); + group.bench_function("zeroes-1m", |bencher| { + bencher.iter(|| { + adler32_slice(&[0; SIZE]); + }); + }); + + group.bench_function("ones-1m", |bencher| { + bencher.iter(|| { + adler32_slice(&[0xff; SIZE]); + }); + }); + } +} + +fn chunked(c: &mut Criterion) { + const SIZE: usize = 16 * 1024 * 1024; + + let data = vec![0xAB; SIZE]; + + let mut group = c.benchmark_group("chunked-16m"); + group.throughput(Throughput::Bytes(SIZE as u64)); + group.bench_function("5552", |bencher| { + bencher.iter(|| { + let mut h = Adler32::new(); + for chunk in data.chunks(5552) { + h.write_slice(chunk); + } + h.checksum() + }); + }); + group.bench_function("8k", |bencher| { + bencher.iter(|| { + let mut h = Adler32::new(); + for chunk in data.chunks(8 * 1024) { + h.write_slice(chunk); + } + h.checksum() + }); + }); + group.bench_function("64k", |bencher| { + bencher.iter(|| { + let mut h = Adler32::new(); + for chunk in data.chunks(64 * 1024) { + h.write_slice(chunk); + } + h.checksum() + }); + }); + group.bench_function("1m", |bencher| { + bencher.iter(|| { + let mut h = Adler32::new(); + for chunk in data.chunks(1024 * 1024) { + h.write_slice(chunk); + } + h.checksum() + }); + }); +} + +criterion_group!(benches, simple, chunked); +criterion_main!(benches); diff --git a/crux-mir/lib/adler/src/algo.rs b/crux-mir/lib/adler/src/algo.rs new file mode 100644 index 000000000..650cffa6c --- /dev/null +++ b/crux-mir/lib/adler/src/algo.rs @@ -0,0 +1,146 @@ +use crate::Adler32; +use std::ops::{AddAssign, MulAssign, RemAssign}; + +impl Adler32 { + pub(crate) fn compute(&mut self, bytes: &[u8]) { + // The basic algorithm is, for every byte: + // a = (a + byte) % MOD + // b = (b + a) % MOD + // where MOD = 65521. + // + // For efficiency, we can defer the `% MOD` operations as long as neither a nor b overflows: + // - Between calls to `write`, we ensure that a and b are always in range 0..MOD. + // - We use 32-bit arithmetic in this function. + // - Therefore, a and b must not increase by more than 2^32-MOD without performing a `% MOD` + // operation. + // + // According to Wikipedia, b is calculated as follows for non-incremental checksumming: + // b = n×D1 + (n−1)×D2 + (n−2)×D3 + ... + Dn + n*1 (mod 65521) + // Where n is the number of bytes and Di is the i-th Byte. We need to change this to account + // for the previous values of a and b, as well as treat every input Byte as being 255: + // b_inc = n×255 + (n-1)×255 + ... + 255 + n*65520 + // Or in other words: + // b_inc = n*65520 + n(n+1)/2*255 + // The max chunk size is thus the largest value of n so that b_inc <= 2^32-65521. + // 2^32-65521 = n*65520 + n(n+1)/2*255 + // Plugging this into an equation solver since I can't math gives n = 5552.18..., so 5552. + // + // On top of the optimization outlined above, the algorithm can also be parallelized with a + // bit more work: + // + // Note that b is a linear combination of a vector of input bytes (D1, ..., Dn). + // + // If we fix some value k Self { + U32X4([ + u32::from(bytes[0]), + u32::from(bytes[1]), + u32::from(bytes[2]), + u32::from(bytes[3]), + ]) + } +} + +impl AddAssign for U32X4 { + fn add_assign(&mut self, other: Self) { + for (s, o) in self.0.iter_mut().zip(other.0.iter()) { + *s += o; + } + } +} + +impl RemAssign for U32X4 { + fn rem_assign(&mut self, quotient: u32) { + for s in self.0.iter_mut() { + *s %= quotient; + } + } +} + +impl MulAssign for U32X4 { + fn mul_assign(&mut self, rhs: u32) { + for s in self.0.iter_mut() { + *s *= rhs; + } + } +} diff --git a/crux-mir/lib/adler/src/lib.rs b/crux-mir/lib/adler/src/lib.rs new file mode 100644 index 000000000..c7aa3805e --- /dev/null +++ b/crux-mir/lib/adler/src/lib.rs @@ -0,0 +1,287 @@ +//! Adler-32 checksum implementation. +//! +//! This implementation features: +//! +//! - Permissively licensed (0BSD) clean-room implementation. +//! - Zero dependencies. +//! - Zero `unsafe`. +//! - Decent performance (3-4 GB/s). +//! - `#![no_std]` support (with `default-features = false`). + +#![doc(html_root_url = "https://docs.rs/adler/1.0.2")] +// Deny a few warnings in doctests, since rustdoc `allow`s many warnings by default +#![doc(test(attr(deny(unused_imports, unused_must_use))))] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![warn(missing_debug_implementations)] +#![forbid(unsafe_code)] +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg(not(feature = "std"))] +extern crate core as std; + +mod algo; + +use std::hash::Hasher; + +#[cfg(feature = "std")] +use std::io::{self, BufRead}; + +/// Adler-32 checksum calculator. +/// +/// An instance of this type is equivalent to an Adler-32 checksum: It can be created in the default +/// state via [`new`] (or the provided `Default` impl), or from a precalculated checksum via +/// [`from_checksum`], and the currently stored checksum can be fetched via [`checksum`]. +/// +/// This type also implements `Hasher`, which makes it easy to calculate Adler-32 checksums of any +/// type that implements or derives `Hash`. This also allows using Adler-32 in a `HashMap`, although +/// that is not recommended (while every checksum is a hash function, they are not necessarily a +/// good one). +/// +/// # Examples +/// +/// Basic, piecewise checksum calculation: +/// +/// ``` +/// use adler::Adler32; +/// +/// let mut adler = Adler32::new(); +/// +/// adler.write_slice(&[0, 1, 2]); +/// adler.write_slice(&[3, 4, 5]); +/// +/// assert_eq!(adler.checksum(), 0x00290010); +/// ``` +/// +/// Using `Hash` to process structures: +/// +/// ``` +/// use std::hash::Hash; +/// use adler::Adler32; +/// +/// #[derive(Hash)] +/// struct Data { +/// byte: u8, +/// word: u16, +/// big: u64, +/// } +/// +/// let mut adler = Adler32::new(); +/// +/// let data = Data { byte: 0x1F, word: 0xABCD, big: !0 }; +/// data.hash(&mut adler); +/// +/// // hash value depends on architecture endianness +/// if cfg!(target_endian = "little") { +/// assert_eq!(adler.checksum(), 0x33410990); +/// } +/// if cfg!(target_endian = "big") { +/// assert_eq!(adler.checksum(), 0x331F0990); +/// } +/// +/// ``` +/// +/// [`new`]: #method.new +/// [`from_checksum`]: #method.from_checksum +/// [`checksum`]: #method.checksum +#[derive(Debug, Copy, Clone)] +pub struct Adler32 { + a: u16, + b: u16, +} + +impl Adler32 { + /// Creates a new Adler-32 instance with default state. + #[inline] + pub fn new() -> Self { + Self::default() + } + + /// Creates an `Adler32` instance from a precomputed Adler-32 checksum. + /// + /// This allows resuming checksum calculation without having to keep the `Adler32` instance + /// around. + /// + /// # Example + /// + /// ``` + /// # use adler::Adler32; + /// let parts = [ + /// "rust", + /// "acean", + /// ]; + /// let whole = adler::adler32_slice(b"rustacean"); + /// + /// let mut sum = Adler32::new(); + /// sum.write_slice(parts[0].as_bytes()); + /// let partial = sum.checksum(); + /// + /// // ...later + /// + /// let mut sum = Adler32::from_checksum(partial); + /// sum.write_slice(parts[1].as_bytes()); + /// assert_eq!(sum.checksum(), whole); + /// ``` + #[inline] + pub fn from_checksum(sum: u32) -> Self { + Adler32 { + a: sum as u16, + b: (sum >> 16) as u16, + } + } + + /// Returns the calculated checksum at this point in time. + #[inline] + pub fn checksum(&self) -> u32 { + (u32::from(self.b) << 16) | u32::from(self.a) + } + + /// Adds `bytes` to the checksum calculation. + /// + /// If efficiency matters, this should be called with Byte slices that contain at least a few + /// thousand Bytes. + pub fn write_slice(&mut self, bytes: &[u8]) { + self.compute(bytes); + } +} + +impl Default for Adler32 { + #[inline] + fn default() -> Self { + Adler32 { a: 1, b: 0 } + } +} + +impl Hasher for Adler32 { + #[inline] + fn finish(&self) -> u64 { + u64::from(self.checksum()) + } + + fn write(&mut self, bytes: &[u8]) { + self.write_slice(bytes); + } +} + +/// Calculates the Adler-32 checksum of a byte slice. +/// +/// This is a convenience function around the [`Adler32`] type. +/// +/// [`Adler32`]: struct.Adler32.html +pub fn adler32_slice(data: &[u8]) -> u32 { + let mut h = Adler32::new(); + h.write_slice(data); + h.checksum() +} + +/// Calculates the Adler-32 checksum of a `BufRead`'s contents. +/// +/// The passed `BufRead` implementor will be read until it reaches EOF (or until it reports an +/// error). +/// +/// If you only have a `Read` implementor, you can wrap it in `std::io::BufReader` before calling +/// this function. +/// +/// # Errors +/// +/// Any error returned by the reader are bubbled up by this function. +/// +/// # Examples +/// +/// ```no_run +/// # fn run() -> Result<(), Box> { +/// use adler::adler32; +/// +/// use std::fs::File; +/// use std::io::BufReader; +/// +/// let file = File::open("input.txt")?; +/// let mut file = BufReader::new(file); +/// +/// adler32(&mut file)?; +/// # Ok(()) } +/// # fn main() { run().unwrap() } +/// ``` +#[cfg(feature = "std")] +#[cfg_attr(docsrs, doc(cfg(feature = "std")))] +pub fn adler32(mut reader: R) -> io::Result { + let mut h = Adler32::new(); + loop { + let len = { + let buf = reader.fill_buf()?; + if buf.is_empty() { + return Ok(h.checksum()); + } + + h.write_slice(buf); + buf.len() + }; + reader.consume(len); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn zeroes() { + assert_eq!(adler32_slice(&[]), 1); + assert_eq!(adler32_slice(&[0]), 1 | 1 << 16); + assert_eq!(adler32_slice(&[0, 0]), 1 | 2 << 16); + assert_eq!(adler32_slice(&[0; 100]), 0x00640001); + assert_eq!(adler32_slice(&[0; 1024]), 0x04000001); + assert_eq!(adler32_slice(&[0; 1024 * 1024]), 0x00f00001); + } + + #[test] + fn ones() { + assert_eq!(adler32_slice(&[0xff; 1024]), 0x79a6fc2e); + assert_eq!(adler32_slice(&[0xff; 1024 * 1024]), 0x8e88ef11); + } + + #[test] + fn mixed() { + assert_eq!(adler32_slice(&[1]), 2 | 2 << 16); + assert_eq!(adler32_slice(&[40]), 41 | 41 << 16); + + assert_eq!(adler32_slice(&[0xA5; 1024 * 1024]), 0xd5009ab1); + } + + /// Example calculation from https://en.wikipedia.org/wiki/Adler-32. + #[test] + fn wiki() { + assert_eq!(adler32_slice(b"Wikipedia"), 0x11E60398); + } + + #[test] + fn resume() { + let mut adler = Adler32::new(); + adler.write_slice(&[0xff; 1024]); + let partial = adler.checksum(); + assert_eq!(partial, 0x79a6fc2e); // from above + adler.write_slice(&[0xff; 1024 * 1024 - 1024]); + assert_eq!(adler.checksum(), 0x8e88ef11); // from above + + // Make sure that we can resume computing from the partial checksum via `from_checksum`. + let mut adler = Adler32::from_checksum(partial); + adler.write_slice(&[0xff; 1024 * 1024 - 1024]); + assert_eq!(adler.checksum(), 0x8e88ef11); // from above + } + + #[cfg(feature = "std")] + #[test] + fn bufread() { + use std::io::BufReader; + fn test(data: &[u8], checksum: u32) { + // `BufReader` uses an 8 KB buffer, so this will test buffer refilling. + let mut buf = BufReader::new(data); + let real_sum = adler32(&mut buf).unwrap(); + assert_eq!(checksum, real_sum); + } + + test(&[], 1); + test(&[0; 1024], 0x04000001); + test(&[0; 1024 * 1024], 0x00f00001); + test(&[0xA5; 1024 * 1024], 0xd5009ab1); + } +} diff --git a/crux-mir/lib/alloc/Cargo.toml b/crux-mir/lib/alloc/Cargo.toml new file mode 100644 index 000000000..95c07abf7 --- /dev/null +++ b/crux-mir/lib/alloc/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "alloc" +version = "0.0.0" +license = "MIT OR Apache-2.0" +repository = "https://github.com/rust-lang/rust.git" +description = "The Rust core allocation and collections library" +autotests = false +autobenches = false +edition = "2021" + +[dependencies] +core = { path = "../core" } +compiler_builtins = { version = "0.1.40", features = ['rustc-dep-of-std'] } + +[dev-dependencies] +rand = { version = "0.8.5", default-features = false, features = ["alloc"] } +rand_xorshift = "0.3.0" + +[[test]] +name = "collectionstests" +path = "tests/lib.rs" + +[[bench]] +name = "collectionsbenches" +path = "benches/lib.rs" +test = true + +[[bench]] +name = "vec_deque_append_bench" +path = "benches/vec_deque_append.rs" +harness = false + +[features] +compiler-builtins-mem = ['compiler_builtins/mem'] +compiler-builtins-c = ["compiler_builtins/c"] +compiler-builtins-no-asm = ["compiler_builtins/no-asm"] +compiler-builtins-mangled-names = ["compiler_builtins/mangled-names"] diff --git a/crux-mir/lib/alloc/benches/binary_heap.rs b/crux-mir/lib/alloc/benches/binary_heap.rs new file mode 100644 index 000000000..917e71f25 --- /dev/null +++ b/crux-mir/lib/alloc/benches/binary_heap.rs @@ -0,0 +1,91 @@ +use std::collections::BinaryHeap; + +use rand::seq::SliceRandom; +use test::{black_box, Bencher}; + +#[bench] +fn bench_find_smallest_1000(b: &mut Bencher) { + let mut rng = crate::bench_rng(); + let mut vec: Vec = (0..100_000).collect(); + vec.shuffle(&mut rng); + + b.iter(|| { + let mut iter = vec.iter().copied(); + let mut heap: BinaryHeap<_> = iter.by_ref().take(1000).collect(); + + for x in iter { + let mut max = heap.peek_mut().unwrap(); + // This comparison should be true only 1% of the time. + // Unnecessary `sift_down`s will degrade performance + if x < *max { + *max = x; + } + } + + heap + }) +} + +#[bench] +fn bench_peek_mut_deref_mut(b: &mut Bencher) { + let mut bheap = BinaryHeap::from(vec![42]); + let vec: Vec = (0..1_000_000).collect(); + + b.iter(|| { + let vec = black_box(&vec); + let mut peek_mut = bheap.peek_mut().unwrap(); + // The compiler shouldn't be able to optimize away the `sift_down` + // assignment in `PeekMut`'s `DerefMut` implementation since + // the loop might not run. + for &i in vec.iter() { + *peek_mut = i; + } + // Remove the already minimal overhead of the sift_down + std::mem::forget(peek_mut); + }) +} + +#[bench] +fn bench_from_vec(b: &mut Bencher) { + let mut rng = crate::bench_rng(); + let mut vec: Vec = (0..100_000).collect(); + vec.shuffle(&mut rng); + + b.iter(|| BinaryHeap::from(vec.clone())) +} + +#[bench] +fn bench_into_sorted_vec(b: &mut Bencher) { + let bheap: BinaryHeap = (0..10_000).collect(); + + b.iter(|| bheap.clone().into_sorted_vec()) +} + +#[bench] +fn bench_push(b: &mut Bencher) { + let mut bheap = BinaryHeap::with_capacity(50_000); + let mut rng = crate::bench_rng(); + let mut vec: Vec = (0..50_000).collect(); + vec.shuffle(&mut rng); + + b.iter(|| { + for &i in vec.iter() { + bheap.push(i); + } + black_box(&mut bheap); + bheap.clear(); + }) +} + +#[bench] +fn bench_pop(b: &mut Bencher) { + let mut bheap = BinaryHeap::with_capacity(10_000); + + b.iter(|| { + bheap.extend((0..10_000).rev()); + black_box(&mut bheap); + while let Some(elem) = bheap.pop() { + black_box(elem); + } + }) +} diff --git a/crux-mir/lib/alloc/benches/btree/map.rs b/crux-mir/lib/alloc/benches/btree/map.rs index 83cdebf0e..1f6b87fb0 100644 --- a/crux-mir/lib/alloc/benches/btree/map.rs +++ b/crux-mir/lib/alloc/benches/btree/map.rs @@ -1,9 +1,9 @@ use std::collections::BTreeMap; use std::iter::Iterator; -use std::ops::Bound::{Excluded, Unbounded}; +use std::ops::RangeBounds; use std::vec::Vec; -use rand::{seq::SliceRandom, thread_rng, Rng}; +use rand::{seq::SliceRandom, Rng}; use test::{black_box, Bencher}; macro_rules! map_insert_rand_bench { @@ -13,7 +13,7 @@ macro_rules! map_insert_rand_bench { let n: usize = $n; let mut map = $map::new(); // setup - let mut rng = thread_rng(); + let mut rng = crate::bench_rng(); for _ in 0..n { let i = rng.gen::() % n; @@ -54,6 +54,50 @@ macro_rules! map_insert_seq_bench { }; } +macro_rules! map_from_iter_rand_bench { + ($name: ident, $n: expr, $map: ident) => { + #[bench] + pub fn $name(b: &mut Bencher) { + let n: usize = $n; + // setup + let mut rng = crate::bench_rng(); + let mut vec = Vec::with_capacity(n); + + for _ in 0..n { + let i = rng.gen::() % n; + vec.push((i, i)); + } + + // measure + b.iter(|| { + let map: $map<_, _> = vec.iter().copied().collect(); + black_box(map); + }); + } + }; +} + +macro_rules! map_from_iter_seq_bench { + ($name: ident, $n: expr, $map: ident) => { + #[bench] + pub fn $name(b: &mut Bencher) { + let n: usize = $n; + // setup + let mut vec = Vec::with_capacity(n); + + for i in 0..n { + vec.push((i, i)); + } + + // measure + b.iter(|| { + let map: $map<_, _> = vec.iter().copied().collect(); + black_box(map); + }); + } + }; +} + macro_rules! map_find_rand_bench { ($name: ident, $n: expr, $map: ident) => { #[bench] @@ -62,7 +106,7 @@ macro_rules! map_find_rand_bench { let n: usize = $n; // setup - let mut rng = thread_rng(); + let mut rng = crate::bench_rng(); let mut keys: Vec<_> = (0..n).map(|_| rng.gen::() % n).collect(); for &k in &keys { @@ -111,15 +155,21 @@ map_insert_rand_bench! {insert_rand_10_000, 10_000, BTreeMap} map_insert_seq_bench! {insert_seq_100, 100, BTreeMap} map_insert_seq_bench! {insert_seq_10_000, 10_000, BTreeMap} +map_from_iter_rand_bench! {from_iter_rand_100, 100, BTreeMap} +map_from_iter_rand_bench! {from_iter_rand_10_000, 10_000, BTreeMap} + +map_from_iter_seq_bench! {from_iter_seq_100, 100, BTreeMap} +map_from_iter_seq_bench! {from_iter_seq_10_000, 10_000, BTreeMap} + map_find_rand_bench! {find_rand_100, 100, BTreeMap} map_find_rand_bench! {find_rand_10_000, 10_000, BTreeMap} map_find_seq_bench! {find_seq_100, 100, BTreeMap} map_find_seq_bench! {find_seq_10_000, 10_000, BTreeMap} -fn bench_iter(b: &mut Bencher, size: i32) { +fn bench_iteration(b: &mut Bencher, size: i32) { let mut map = BTreeMap::::new(); - let mut rng = thread_rng(); + let mut rng = crate::bench_rng(); for _ in 0..size { map.insert(rng.gen(), rng.gen()); @@ -133,23 +183,23 @@ fn bench_iter(b: &mut Bencher, size: i32) { } #[bench] -pub fn iter_20(b: &mut Bencher) { - bench_iter(b, 20); +pub fn iteration_20(b: &mut Bencher) { + bench_iteration(b, 20); } #[bench] -pub fn iter_1000(b: &mut Bencher) { - bench_iter(b, 1000); +pub fn iteration_1000(b: &mut Bencher) { + bench_iteration(b, 1000); } #[bench] -pub fn iter_100000(b: &mut Bencher) { - bench_iter(b, 100000); +pub fn iteration_100000(b: &mut Bencher) { + bench_iteration(b, 100000); } -fn bench_iter_mut(b: &mut Bencher, size: i32) { +fn bench_iteration_mut(b: &mut Bencher, size: i32) { let mut map = BTreeMap::::new(); - let mut rng = thread_rng(); + let mut rng = crate::bench_rng(); for _ in 0..size { map.insert(rng.gen(), rng.gen()); @@ -163,21 +213,21 @@ fn bench_iter_mut(b: &mut Bencher, size: i32) { } #[bench] -pub fn iter_mut_20(b: &mut Bencher) { - bench_iter_mut(b, 20); +pub fn iteration_mut_20(b: &mut Bencher) { + bench_iteration_mut(b, 20); } #[bench] -pub fn iter_mut_1000(b: &mut Bencher) { - bench_iter_mut(b, 1000); +pub fn iteration_mut_1000(b: &mut Bencher) { + bench_iteration_mut(b, 1000); } #[bench] -pub fn iter_mut_100000(b: &mut Bencher) { - bench_iter_mut(b, 100000); +pub fn iteration_mut_100000(b: &mut Bencher) { + bench_iteration_mut(b, 100000); } -fn bench_first_and_last(b: &mut Bencher, size: i32) { +fn bench_first_and_last_nightly(b: &mut Bencher, size: i32) { let map: BTreeMap<_, _> = (0..size).map(|i| (i, i)).collect(); b.iter(|| { for _ in 0..10 { @@ -187,72 +237,349 @@ fn bench_first_and_last(b: &mut Bencher, size: i32) { }); } +fn bench_first_and_last_stable(b: &mut Bencher, size: i32) { + let map: BTreeMap<_, _> = (0..size).map(|i| (i, i)).collect(); + b.iter(|| { + for _ in 0..10 { + black_box(map.iter().next()); + black_box(map.iter().next_back()); + } + }); +} + #[bench] -pub fn first_and_last_0(b: &mut Bencher) { - bench_first_and_last(b, 0); +pub fn first_and_last_0_nightly(b: &mut Bencher) { + bench_first_and_last_nightly(b, 0); } #[bench] -pub fn first_and_last_100(b: &mut Bencher) { - bench_first_and_last(b, 100); +pub fn first_and_last_0_stable(b: &mut Bencher) { + bench_first_and_last_stable(b, 0); } #[bench] -pub fn first_and_last_10k(b: &mut Bencher) { - bench_first_and_last(b, 10_000); +pub fn first_and_last_100_nightly(b: &mut Bencher) { + bench_first_and_last_nightly(b, 100); } #[bench] -pub fn range_excluded_excluded(b: &mut Bencher) { - let size = 144; - let map: BTreeMap<_, _> = (0..size).map(|i| (i, i)).collect(); +pub fn first_and_last_100_stable(b: &mut Bencher) { + bench_first_and_last_stable(b, 100); +} + +#[bench] +pub fn first_and_last_10k_nightly(b: &mut Bencher) { + bench_first_and_last_nightly(b, 10_000); +} + +#[bench] +pub fn first_and_last_10k_stable(b: &mut Bencher) { + bench_first_and_last_stable(b, 10_000); +} + +const BENCH_RANGE_SIZE: i32 = 145; +const BENCH_RANGE_COUNT: i32 = BENCH_RANGE_SIZE * (BENCH_RANGE_SIZE - 1) / 2; + +fn bench_range(b: &mut Bencher, f: F) +where + F: Fn(i32, i32) -> R, + R: RangeBounds, +{ + let map: BTreeMap<_, _> = (0..BENCH_RANGE_SIZE).map(|i| (i, i)).collect(); b.iter(|| { - for first in 0..size { - for last in first + 1..size { - black_box(map.range((Excluded(first), Excluded(last)))); + let mut c = 0; + for i in 0..BENCH_RANGE_SIZE { + for j in i + 1..BENCH_RANGE_SIZE { + let _ = black_box(map.range(f(i, j))); + c += 1; } } + debug_assert_eq!(c, BENCH_RANGE_COUNT); }); } #[bench] -pub fn range_excluded_unbounded(b: &mut Bencher) { - let size = 144; +pub fn range_included_excluded(b: &mut Bencher) { + bench_range(b, |i, j| i..j); +} + +#[bench] +pub fn range_included_included(b: &mut Bencher) { + bench_range(b, |i, j| i..=j); +} + +#[bench] +pub fn range_included_unbounded(b: &mut Bencher) { + bench_range(b, |i, _| i..); +} + +#[bench] +pub fn range_unbounded_unbounded(b: &mut Bencher) { + bench_range(b, |_, _| ..); +} + +fn bench_iter(b: &mut Bencher, repeats: i32, size: i32) { let map: BTreeMap<_, _> = (0..size).map(|i| (i, i)).collect(); b.iter(|| { - for first in 0..size { - black_box(map.range((Excluded(first), Unbounded))); + for _ in 0..repeats { + let _ = black_box(map.iter()); } }); } +/// Contrast range_unbounded_unbounded with `iter()`. #[bench] -pub fn range_included_included(b: &mut Bencher) { - let size = 144; - let map: BTreeMap<_, _> = (0..size).map(|i| (i, i)).collect(); +pub fn range_unbounded_vs_iter(b: &mut Bencher) { + bench_iter(b, BENCH_RANGE_COUNT, BENCH_RANGE_SIZE); +} + +#[bench] +pub fn iter_0(b: &mut Bencher) { + bench_iter(b, 1_000, 0); +} + +#[bench] +pub fn iter_1(b: &mut Bencher) { + bench_iter(b, 1_000, 1); +} + +#[bench] +pub fn iter_100(b: &mut Bencher) { + bench_iter(b, 1_000, 100); +} + +#[bench] +pub fn iter_10k(b: &mut Bencher) { + bench_iter(b, 1_000, 10_000); +} + +#[bench] +pub fn iter_1m(b: &mut Bencher) { + bench_iter(b, 1_000, 1_000_000); +} + +const FAT: usize = 256; + +// The returned map has small keys and values. +// Benchmarks on it have a counterpart in set.rs with the same keys and no values at all. +fn slim_map(n: usize) -> BTreeMap { + (0..n).map(|i| (i, i)).collect::>() +} + +// The returned map has small keys and large values. +fn fat_val_map(n: usize) -> BTreeMap { + (0..n).map(|i| (i, [i; FAT])).collect::>() +} + +#[bench] +pub fn clone_slim_100(b: &mut Bencher) { + let src = slim_map(100); + b.iter(|| src.clone()) +} + +#[bench] +pub fn clone_slim_100_and_clear(b: &mut Bencher) { + let src = slim_map(100); + b.iter(|| src.clone().clear()) +} + +#[bench] +pub fn clone_slim_100_and_drain_all(b: &mut Bencher) { + let src = slim_map(100); + b.iter(|| src.clone().drain_filter(|_, _| true).count()) +} + +#[bench] +pub fn clone_slim_100_and_drain_half(b: &mut Bencher) { + let src = slim_map(100); b.iter(|| { - for first in 0..size { - for last in first..size { - black_box(map.range(first..=last)); - } + let mut map = src.clone(); + assert_eq!(map.drain_filter(|i, _| i % 2 == 0).count(), 100 / 2); + assert_eq!(map.len(), 100 / 2); + }) +} + +#[bench] +pub fn clone_slim_100_and_into_iter(b: &mut Bencher) { + let src = slim_map(100); + b.iter(|| src.clone().into_iter().count()) +} + +#[bench] +pub fn clone_slim_100_and_pop_all(b: &mut Bencher) { + let src = slim_map(100); + b.iter(|| { + let mut map = src.clone(); + while map.pop_first().is_some() {} + map + }); +} + +#[bench] +pub fn clone_slim_100_and_remove_all(b: &mut Bencher) { + let src = slim_map(100); + b.iter(|| { + let mut map = src.clone(); + while let Some(elt) = map.iter().map(|(&i, _)| i).next() { + let v = map.remove(&elt); + debug_assert!(v.is_some()); } + map }); } #[bench] -pub fn range_included_unbounded(b: &mut Bencher) { - let size = 144; - let map: BTreeMap<_, _> = (0..size).map(|i| (i, i)).collect(); +pub fn clone_slim_100_and_remove_half(b: &mut Bencher) { + let src = slim_map(100); + b.iter(|| { + let mut map = src.clone(); + for i in (0..100).step_by(2) { + let v = map.remove(&i); + debug_assert!(v.is_some()); + } + assert_eq!(map.len(), 100 / 2); + map + }) +} + +#[bench] +pub fn clone_slim_10k(b: &mut Bencher) { + let src = slim_map(10_000); + b.iter(|| src.clone()) +} + +#[bench] +pub fn clone_slim_10k_and_clear(b: &mut Bencher) { + let src = slim_map(10_000); + b.iter(|| src.clone().clear()) +} + +#[bench] +pub fn clone_slim_10k_and_drain_all(b: &mut Bencher) { + let src = slim_map(10_000); + b.iter(|| src.clone().drain_filter(|_, _| true).count()) +} + +#[bench] +pub fn clone_slim_10k_and_drain_half(b: &mut Bencher) { + let src = slim_map(10_000); + b.iter(|| { + let mut map = src.clone(); + assert_eq!(map.drain_filter(|i, _| i % 2 == 0).count(), 10_000 / 2); + assert_eq!(map.len(), 10_000 / 2); + }) +} + +#[bench] +pub fn clone_slim_10k_and_into_iter(b: &mut Bencher) { + let src = slim_map(10_000); + b.iter(|| src.clone().into_iter().count()) +} + +#[bench] +pub fn clone_slim_10k_and_pop_all(b: &mut Bencher) { + let src = slim_map(10_000); b.iter(|| { - for first in 0..size { - black_box(map.range(first..)); + let mut map = src.clone(); + while map.pop_first().is_some() {} + map + }); +} + +#[bench] +pub fn clone_slim_10k_and_remove_all(b: &mut Bencher) { + let src = slim_map(10_000); + b.iter(|| { + let mut map = src.clone(); + while let Some(elt) = map.iter().map(|(&i, _)| i).next() { + let v = map.remove(&elt); + debug_assert!(v.is_some()); } + map }); } #[bench] -pub fn range_unbounded_unbounded(b: &mut Bencher) { - let size = 144; - let map: BTreeMap<_, _> = (0..size).map(|i| (i, i)).collect(); - b.iter(|| map.range(..)); +pub fn clone_slim_10k_and_remove_half(b: &mut Bencher) { + let src = slim_map(10_000); + b.iter(|| { + let mut map = src.clone(); + for i in (0..10_000).step_by(2) { + let v = map.remove(&i); + debug_assert!(v.is_some()); + } + assert_eq!(map.len(), 10_000 / 2); + map + }) +} + +#[bench] +pub fn clone_fat_val_100(b: &mut Bencher) { + let src = fat_val_map(100); + b.iter(|| src.clone()) +} + +#[bench] +pub fn clone_fat_val_100_and_clear(b: &mut Bencher) { + let src = fat_val_map(100); + b.iter(|| src.clone().clear()) +} + +#[bench] +pub fn clone_fat_val_100_and_drain_all(b: &mut Bencher) { + let src = fat_val_map(100); + b.iter(|| src.clone().drain_filter(|_, _| true).count()) +} + +#[bench] +pub fn clone_fat_val_100_and_drain_half(b: &mut Bencher) { + let src = fat_val_map(100); + b.iter(|| { + let mut map = src.clone(); + assert_eq!(map.drain_filter(|i, _| i % 2 == 0).count(), 100 / 2); + assert_eq!(map.len(), 100 / 2); + }) +} + +#[bench] +pub fn clone_fat_val_100_and_into_iter(b: &mut Bencher) { + let src = fat_val_map(100); + b.iter(|| src.clone().into_iter().count()) +} + +#[bench] +pub fn clone_fat_val_100_and_pop_all(b: &mut Bencher) { + let src = fat_val_map(100); + b.iter(|| { + let mut map = src.clone(); + while map.pop_first().is_some() {} + map + }); +} + +#[bench] +pub fn clone_fat_val_100_and_remove_all(b: &mut Bencher) { + let src = fat_val_map(100); + b.iter(|| { + let mut map = src.clone(); + while let Some(elt) = map.iter().map(|(&i, _)| i).next() { + let v = map.remove(&elt); + debug_assert!(v.is_some()); + } + map + }); +} + +#[bench] +pub fn clone_fat_val_100_and_remove_half(b: &mut Bencher) { + let src = fat_val_map(100); + b.iter(|| { + let mut map = src.clone(); + for i in (0..100).step_by(2) { + let v = map.remove(&i); + debug_assert!(v.is_some()); + } + assert_eq!(map.len(), 100 / 2); + map + }) } diff --git a/crux-mir/lib/alloc/benches/btree/set.rs b/crux-mir/lib/alloc/benches/btree/set.rs index d9e75ab7f..3f4b0e0f1 100644 --- a/crux-mir/lib/alloc/benches/btree/set.rs +++ b/crux-mir/lib/alloc/benches/btree/set.rs @@ -1,10 +1,10 @@ use std::collections::BTreeSet; -use rand::{thread_rng, Rng}; +use rand::Rng; use test::Bencher; fn random(n: usize) -> BTreeSet { - let mut rng = thread_rng(); + let mut rng = crate::bench_rng(); let mut set = BTreeSet::new(); while set.len() < n { set.insert(rng.gen()); @@ -50,27 +50,47 @@ macro_rules! set_bench { }; } +fn slim_set(n: usize) -> BTreeSet { + (0..n).collect::>() +} + #[bench] pub fn clone_100(b: &mut Bencher) { - let src = pos(100); + let src = slim_set(100); b.iter(|| src.clone()) } #[bench] pub fn clone_100_and_clear(b: &mut Bencher) { - let src = pos(100); + let src = slim_set(100); b.iter(|| src.clone().clear()) } +#[bench] +pub fn clone_100_and_drain_all(b: &mut Bencher) { + let src = slim_set(100); + b.iter(|| src.clone().drain_filter(|_| true).count()) +} + +#[bench] +pub fn clone_100_and_drain_half(b: &mut Bencher) { + let src = slim_set(100); + b.iter(|| { + let mut set = src.clone(); + assert_eq!(set.drain_filter(|i| i % 2 == 0).count(), 100 / 2); + assert_eq!(set.len(), 100 / 2); + }) +} + #[bench] pub fn clone_100_and_into_iter(b: &mut Bencher) { - let src = pos(100); + let src = slim_set(100); b.iter(|| src.clone().into_iter().count()) } #[bench] pub fn clone_100_and_pop_all(b: &mut Bencher) { - let src = pos(100); + let src = slim_set(100); b.iter(|| { let mut set = src.clone(); while set.pop_first().is_some() {} @@ -80,11 +100,12 @@ pub fn clone_100_and_pop_all(b: &mut Bencher) { #[bench] pub fn clone_100_and_remove_all(b: &mut Bencher) { - let src = pos(100); + let src = slim_set(100); b.iter(|| { let mut set = src.clone(); while let Some(elt) = set.iter().copied().next() { - set.remove(&elt); + let ok = set.remove(&elt); + debug_assert!(ok); } set }); @@ -92,11 +113,12 @@ pub fn clone_100_and_remove_all(b: &mut Bencher) { #[bench] pub fn clone_100_and_remove_half(b: &mut Bencher) { - let src = pos(100); + let src = slim_set(100); b.iter(|| { let mut set = src.clone(); - for i in (2..=100 as i32).step_by(2) { - set.remove(&i); + for i in (0..100).step_by(2) { + let ok = set.remove(&i); + debug_assert!(ok); } assert_eq!(set.len(), 100 / 2); set @@ -105,25 +127,41 @@ pub fn clone_100_and_remove_half(b: &mut Bencher) { #[bench] pub fn clone_10k(b: &mut Bencher) { - let src = pos(10_000); + let src = slim_set(10_000); b.iter(|| src.clone()) } #[bench] pub fn clone_10k_and_clear(b: &mut Bencher) { - let src = pos(10_000); + let src = slim_set(10_000); b.iter(|| src.clone().clear()) } +#[bench] +pub fn clone_10k_and_drain_all(b: &mut Bencher) { + let src = slim_set(10_000); + b.iter(|| src.clone().drain_filter(|_| true).count()) +} + +#[bench] +pub fn clone_10k_and_drain_half(b: &mut Bencher) { + let src = slim_set(10_000); + b.iter(|| { + let mut set = src.clone(); + assert_eq!(set.drain_filter(|i| i % 2 == 0).count(), 10_000 / 2); + assert_eq!(set.len(), 10_000 / 2); + }) +} + #[bench] pub fn clone_10k_and_into_iter(b: &mut Bencher) { - let src = pos(10_000); + let src = slim_set(10_000); b.iter(|| src.clone().into_iter().count()) } #[bench] pub fn clone_10k_and_pop_all(b: &mut Bencher) { - let src = pos(10_000); + let src = slim_set(10_000); b.iter(|| { let mut set = src.clone(); while set.pop_first().is_some() {} @@ -133,11 +171,12 @@ pub fn clone_10k_and_pop_all(b: &mut Bencher) { #[bench] pub fn clone_10k_and_remove_all(b: &mut Bencher) { - let src = pos(10_000); + let src = slim_set(10_000); b.iter(|| { let mut set = src.clone(); while let Some(elt) = set.iter().copied().next() { - set.remove(&elt); + let ok = set.remove(&elt); + debug_assert!(ok); } set }); @@ -145,11 +184,12 @@ pub fn clone_10k_and_remove_all(b: &mut Bencher) { #[bench] pub fn clone_10k_and_remove_half(b: &mut Bencher) { - let src = pos(10_000); + let src = slim_set(10_000); b.iter(|| { let mut set = src.clone(); - for i in (2..=10_000 as i32).step_by(2) { - set.remove(&i); + for i in (0..10_000).step_by(2) { + let ok = set.remove(&i); + debug_assert!(ok); } assert_eq!(set.len(), 10_000 / 2); set diff --git a/crux-mir/lib/alloc/benches/lib.rs b/crux-mir/lib/alloc/benches/lib.rs index 951477a24..b25d63d83 100644 --- a/crux-mir/lib/alloc/benches/lib.rs +++ b/crux-mir/lib/alloc/benches/lib.rs @@ -1,9 +1,17 @@ -#![feature(map_first_last)] +// Disabling on android for the time being +// See https://github.com/rust-lang/rust/issues/73535#event-3477699747 +#![cfg(not(target_os = "android"))] +#![feature(btree_drain_filter)] +#![feature(iter_next_chunk)] #![feature(repr_simd)] +#![feature(slice_partition_dedup)] +#![feature(strict_provenance)] #![feature(test)] +#![deny(fuzzy_provenance_casts)] extern crate test; +mod binary_heap; mod btree; mod linked_list; mod slice; @@ -11,3 +19,11 @@ mod str; mod string; mod vec; mod vec_deque; + +/// Returns a `rand::Rng` seeded with a consistent seed. +/// +/// This is done to avoid introducing nondeterminism in benchmark results. +fn bench_rng() -> rand_xorshift::XorShiftRng { + const SEED: [u8; 16] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; + rand::SeedableRng::from_seed(SEED) +} diff --git a/crux-mir/lib/alloc/benches/slice.rs b/crux-mir/lib/alloc/benches/slice.rs index e20c04328..b62be9d39 100644 --- a/crux-mir/lib/alloc/benches/slice.rs +++ b/crux-mir/lib/alloc/benches/slice.rs @@ -1,8 +1,7 @@ use std::{mem, ptr}; -use rand::distributions::{Alphanumeric, Standard}; -use rand::{thread_rng, Rng, SeedableRng}; -use rand_xorshift::XorShiftRng; +use rand::distributions::{Alphanumeric, DistString, Standard}; +use rand::Rng; use test::{black_box, Bencher}; #[bench] @@ -152,7 +151,7 @@ fn zero_1kb_mut_iter(b: &mut Bencher) { #[bench] fn random_inserts(b: &mut Bencher) { - let mut rng = thread_rng(); + let mut rng = crate::bench_rng(); b.iter(|| { let mut v = vec![(0, 0); 30]; for _ in 0..100 { @@ -164,7 +163,7 @@ fn random_inserts(b: &mut Bencher) { #[bench] fn random_removes(b: &mut Bencher) { - let mut rng = thread_rng(); + let mut rng = crate::bench_rng(); b.iter(|| { let mut v = vec![(0, 0); 130]; for _ in 0..100 { @@ -182,20 +181,18 @@ fn gen_descending(len: usize) -> Vec { (0..len as u64).rev().collect() } -const SEED: [u8; 16] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; - fn gen_random(len: usize) -> Vec { - let mut rng = XorShiftRng::from_seed(SEED); + let mut rng = crate::bench_rng(); (&mut rng).sample_iter(&Standard).take(len).collect() } fn gen_random_bytes(len: usize) -> Vec { - let mut rng = XorShiftRng::from_seed(SEED); + let mut rng = crate::bench_rng(); (&mut rng).sample_iter(&Standard).take(len).collect() } fn gen_mostly_ascending(len: usize) -> Vec { - let mut rng = XorShiftRng::from_seed(SEED); + let mut rng = crate::bench_rng(); let mut v = gen_ascending(len); for _ in (0usize..).take_while(|x| x * x <= len) { let x = rng.gen::() % len; @@ -206,7 +203,7 @@ fn gen_mostly_ascending(len: usize) -> Vec { } fn gen_mostly_descending(len: usize) -> Vec { - let mut rng = XorShiftRng::from_seed(SEED); + let mut rng = crate::bench_rng(); let mut v = gen_descending(len); for _ in (0usize..).take_while(|x| x * x <= len) { let x = rng.gen::() % len; @@ -217,17 +214,17 @@ fn gen_mostly_descending(len: usize) -> Vec { } fn gen_strings(len: usize) -> Vec { - let mut rng = XorShiftRng::from_seed(SEED); + let mut rng = crate::bench_rng(); let mut v = vec![]; for _ in 0..len { let n = rng.gen::() % 20 + 1; - v.push((&mut rng).sample_iter(&Alphanumeric).take(n).collect()); + v.push(Alphanumeric.sample_string(&mut rng, n)); } v } fn gen_big_random(len: usize) -> Vec<[u64; 16]> { - let mut rng = XorShiftRng::from_seed(SEED); + let mut rng = crate::bench_rng(); (&mut rng).sample_iter(&Standard).map(|x| [x; 16]).take(len).collect() } diff --git a/crux-mir/lib/alloc/benches/str.rs b/crux-mir/lib/alloc/benches/str.rs index 391475bc0..54af389de 100644 --- a/crux-mir/lib/alloc/benches/str.rs +++ b/crux-mir/lib/alloc/benches/str.rs @@ -1,3 +1,4 @@ +use core::iter::Iterator; use test::{black_box, Bencher}; #[bench] @@ -122,14 +123,13 @@ fn bench_contains_short_short(b: &mut Bencher) { let haystack = "Lorem ipsum dolor sit amet, consectetur adipiscing elit."; let needle = "sit"; + b.bytes = haystack.len() as u64; b.iter(|| { - assert!(haystack.contains(needle)); + assert!(black_box(haystack).contains(black_box(needle))); }) } -#[bench] -fn bench_contains_short_long(b: &mut Bencher) { - let haystack = "\ +static LONG_HAYSTACK: &str = "\ Lorem ipsum dolor sit amet, consectetur adipiscing elit. Suspendisse quis lorem sit amet dolor \ ultricies condimentum. Praesent iaculis purus elit, ac malesuada quam malesuada in. Duis sed orci \ eros. Suspendisse sit amet magna mollis, mollis nunc luctus, imperdiet mi. Integer fringilla non \ @@ -164,10 +164,48 @@ feugiat. Etiam quis mauris vel risus luctus mattis a a nunc. Nullam orci quam, i vehicula in, porttitor ut nibh. Duis sagittis adipiscing nisl vitae congue. Donec mollis risus eu \ leo suscipit, varius porttitor nulla porta. Pellentesque ut sem nec nisi euismod vehicula. Nulla \ malesuada sollicitudin quam eu fermentum."; + +#[bench] +fn bench_contains_2b_repeated_long(b: &mut Bencher) { + let haystack = LONG_HAYSTACK; + let needle = "::"; + + b.bytes = haystack.len() as u64; + b.iter(|| { + assert!(!black_box(haystack).contains(black_box(needle))); + }) +} + +#[bench] +fn bench_contains_short_long(b: &mut Bencher) { + let haystack = LONG_HAYSTACK; let needle = "english"; + b.bytes = haystack.len() as u64; + b.iter(|| { + assert!(!black_box(haystack).contains(black_box(needle))); + }) +} + +#[bench] +fn bench_contains_16b_in_long(b: &mut Bencher) { + let haystack = LONG_HAYSTACK; + let needle = "english language"; + + b.bytes = haystack.len() as u64; + b.iter(|| { + assert!(!black_box(haystack).contains(black_box(needle))); + }) +} + +#[bench] +fn bench_contains_32b_in_long(b: &mut Bencher) { + let haystack = LONG_HAYSTACK; + let needle = "the english language sample text"; + + b.bytes = haystack.len() as u64; b.iter(|| { - assert!(!haystack.contains(needle)); + assert!(!black_box(haystack).contains(black_box(needle))); }) } @@ -176,8 +214,20 @@ fn bench_contains_bad_naive(b: &mut Bencher) { let haystack = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; let needle = "aaaaaaaab"; + b.bytes = haystack.len() as u64; + b.iter(|| { + assert!(!black_box(haystack).contains(black_box(needle))); + }) +} + +#[bench] +fn bench_contains_bad_simd(b: &mut Bencher) { + let haystack = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + let needle = "aaabaaaa"; + + b.bytes = haystack.len() as u64; b.iter(|| { - assert!(!haystack.contains(needle)); + assert!(!black_box(haystack).contains(black_box(needle))); }) } @@ -186,8 +236,9 @@ fn bench_contains_equal(b: &mut Bencher) { let haystack = "Lorem ipsum dolor sit amet, consectetur adipiscing elit."; let needle = "Lorem ipsum dolor sit amet, consectetur adipiscing elit."; + b.bytes = haystack.len() as u64; b.iter(|| { - assert!(haystack.contains(needle)); + assert!(black_box(haystack).contains(black_box(needle))); }) } diff --git a/crux-mir/lib/alloc/benches/vec.rs b/crux-mir/lib/alloc/benches/vec.rs index a3da9e80c..663f6b9dd 100644 --- a/crux-mir/lib/alloc/benches/vec.rs +++ b/crux-mir/lib/alloc/benches/vec.rs @@ -1,23 +1,16 @@ +use rand::RngCore; use std::iter::{repeat, FromIterator}; -use test::Bencher; +use test::{black_box, Bencher}; #[bench] fn bench_new(b: &mut Bencher) { - b.iter(|| { - let v: Vec = Vec::new(); - assert_eq!(v.len(), 0); - assert_eq!(v.capacity(), 0); - }) + b.iter(|| Vec::::new()) } fn do_bench_with_capacity(b: &mut Bencher, src_len: usize) { b.bytes = src_len as u64; - b.iter(|| { - let v: Vec = Vec::with_capacity(src_len); - assert_eq!(v.len(), 0); - assert_eq!(v.capacity(), src_len); - }) + b.iter(|| Vec::::with_capacity(src_len)) } #[bench] @@ -43,11 +36,7 @@ fn bench_with_capacity_1000(b: &mut Bencher) { fn do_bench_from_fn(b: &mut Bencher, src_len: usize) { b.bytes = src_len as u64; - b.iter(|| { - let dst = (0..src_len).collect::>(); - assert_eq!(dst.len(), src_len); - assert!(dst.iter().enumerate().all(|(i, x)| i == *x)); - }) + b.iter(|| (0..src_len).collect::>()) } #[bench] @@ -73,11 +62,7 @@ fn bench_from_fn_1000(b: &mut Bencher) { fn do_bench_from_elem(b: &mut Bencher, src_len: usize) { b.bytes = src_len as u64; - b.iter(|| { - let dst: Vec = repeat(5).take(src_len).collect(); - assert_eq!(dst.len(), src_len); - assert!(dst.iter().all(|x| *x == 5)); - }) + b.iter(|| repeat(5).take(src_len).collect::>()) } #[bench] @@ -105,11 +90,7 @@ fn do_bench_from_slice(b: &mut Bencher, src_len: usize) { b.bytes = src_len as u64; - b.iter(|| { - let dst = src.clone()[..].to_vec(); - assert_eq!(dst.len(), src_len); - assert!(dst.iter().enumerate().all(|(i, x)| i == *x)); - }); + b.iter(|| src.as_slice().to_vec()); } #[bench] @@ -138,9 +119,8 @@ fn do_bench_from_iter(b: &mut Bencher, src_len: usize) { b.bytes = src_len as u64; b.iter(|| { - let dst: Vec<_> = FromIterator::from_iter(src.clone()); - assert_eq!(dst.len(), src_len); - assert!(dst.iter().enumerate().all(|(i, x)| i == *x)); + let dst: Vec<_> = FromIterator::from_iter(src.iter().cloned()); + dst }); } @@ -173,8 +153,7 @@ fn do_bench_extend(b: &mut Bencher, dst_len: usize, src_len: usize) { b.iter(|| { let mut dst = dst.clone(); dst.extend(src.clone()); - assert_eq!(dst.len(), dst_len + src_len); - assert!(dst.iter().enumerate().all(|(i, x)| i == *x)); + dst }); } @@ -213,7 +192,7 @@ fn bench_extend_1000_1000(b: &mut Bencher) { do_bench_extend(b, 1000, 1000) } -fn do_bench_push_all(b: &mut Bencher, dst_len: usize, src_len: usize) { +fn do_bench_extend_from_slice(b: &mut Bencher, dst_len: usize, src_len: usize) { let dst: Vec<_> = FromIterator::from_iter(0..dst_len); let src: Vec<_> = FromIterator::from_iter(dst_len..dst_len + src_len); @@ -222,93 +201,57 @@ fn do_bench_push_all(b: &mut Bencher, dst_len: usize, src_len: usize) { b.iter(|| { let mut dst = dst.clone(); dst.extend_from_slice(&src); - assert_eq!(dst.len(), dst_len + src_len); - assert!(dst.iter().enumerate().all(|(i, x)| i == *x)); + dst }); } #[bench] -fn bench_push_all_0000_0000(b: &mut Bencher) { - do_bench_push_all(b, 0, 0) -} - -#[bench] -fn bench_push_all_0000_0010(b: &mut Bencher) { - do_bench_push_all(b, 0, 10) -} - -#[bench] -fn bench_push_all_0000_0100(b: &mut Bencher) { - do_bench_push_all(b, 0, 100) -} - -#[bench] -fn bench_push_all_0000_1000(b: &mut Bencher) { - do_bench_push_all(b, 0, 1000) -} - -#[bench] -fn bench_push_all_0010_0010(b: &mut Bencher) { - do_bench_push_all(b, 10, 10) -} - -#[bench] -fn bench_push_all_0100_0100(b: &mut Bencher) { - do_bench_push_all(b, 100, 100) -} - -#[bench] -fn bench_push_all_1000_1000(b: &mut Bencher) { - do_bench_push_all(b, 1000, 1000) -} - -fn do_bench_push_all_move(b: &mut Bencher, dst_len: usize, src_len: usize) { - let dst: Vec<_> = FromIterator::from_iter(0..dst_len); - let src: Vec<_> = FromIterator::from_iter(dst_len..dst_len + src_len); - - b.bytes = src_len as u64; +fn bench_extend_recycle(b: &mut Bencher) { + let mut data = vec![0; 1000]; b.iter(|| { - let mut dst = dst.clone(); - dst.extend(src.clone()); - assert_eq!(dst.len(), dst_len + src_len); - assert!(dst.iter().enumerate().all(|(i, x)| i == *x)); + let tmp = std::mem::take(&mut data); + let mut to_extend = black_box(Vec::new()); + to_extend.extend(tmp.into_iter()); + data = black_box(to_extend); }); + + black_box(data); } #[bench] -fn bench_push_all_move_0000_0000(b: &mut Bencher) { - do_bench_push_all_move(b, 0, 0) +fn bench_extend_from_slice_0000_0000(b: &mut Bencher) { + do_bench_extend_from_slice(b, 0, 0) } #[bench] -fn bench_push_all_move_0000_0010(b: &mut Bencher) { - do_bench_push_all_move(b, 0, 10) +fn bench_extend_from_slice_0000_0010(b: &mut Bencher) { + do_bench_extend_from_slice(b, 0, 10) } #[bench] -fn bench_push_all_move_0000_0100(b: &mut Bencher) { - do_bench_push_all_move(b, 0, 100) +fn bench_extend_from_slice_0000_0100(b: &mut Bencher) { + do_bench_extend_from_slice(b, 0, 100) } #[bench] -fn bench_push_all_move_0000_1000(b: &mut Bencher) { - do_bench_push_all_move(b, 0, 1000) +fn bench_extend_from_slice_0000_1000(b: &mut Bencher) { + do_bench_extend_from_slice(b, 0, 1000) } #[bench] -fn bench_push_all_move_0010_0010(b: &mut Bencher) { - do_bench_push_all_move(b, 10, 10) +fn bench_extend_from_slice_0010_0010(b: &mut Bencher) { + do_bench_extend_from_slice(b, 10, 10) } #[bench] -fn bench_push_all_move_0100_0100(b: &mut Bencher) { - do_bench_push_all_move(b, 100, 100) +fn bench_extend_from_slice_0100_0100(b: &mut Bencher) { + do_bench_extend_from_slice(b, 100, 100) } #[bench] -fn bench_push_all_move_1000_1000(b: &mut Bencher) { - do_bench_push_all_move(b, 1000, 1000) +fn bench_extend_from_slice_1000_1000(b: &mut Bencher) { + do_bench_extend_from_slice(b, 1000, 1000) } fn do_bench_clone(b: &mut Bencher, src_len: usize) { @@ -316,11 +259,7 @@ fn do_bench_clone(b: &mut Bencher, src_len: usize) { b.bytes = src_len as u64; - b.iter(|| { - let dst = src.clone(); - assert_eq!(dst.len(), src_len); - assert!(dst.iter().enumerate().all(|(i, x)| i == *x)); - }); + b.iter(|| src.clone()); } #[bench] @@ -354,10 +293,9 @@ fn do_bench_clone_from(b: &mut Bencher, times: usize, dst_len: usize, src_len: u for _ in 0..times { dst.clone_from(&src); - - assert_eq!(dst.len(), src_len); - assert!(dst.iter().enumerate().all(|(i, x)| dst_len + i == *x)); + dst = black_box(dst); } + dst }); } @@ -480,3 +418,367 @@ fn bench_clone_from_10_0100_0010(b: &mut Bencher) { fn bench_clone_from_10_1000_0100(b: &mut Bencher) { do_bench_clone_from(b, 10, 1000, 100) } + +macro_rules! bench_in_place { + ($($fname:ident, $type:ty, $count:expr, $init:expr);*) => { + $( + #[bench] + fn $fname(b: &mut Bencher) { + b.iter(|| { + let src: Vec<$type> = black_box(vec![$init; $count]); + src.into_iter() + .enumerate() + .map(|(idx, e)| idx as $type ^ e) + .collect::>() + }); + } + )+ + }; +} + +bench_in_place![ + bench_in_place_xxu8_0010_i0, u8, 10, 0; + bench_in_place_xxu8_0100_i0, u8, 100, 0; + bench_in_place_xxu8_1000_i0, u8, 1000, 0; + bench_in_place_xxu8_0010_i1, u8, 10, 1; + bench_in_place_xxu8_0100_i1, u8, 100, 1; + bench_in_place_xxu8_1000_i1, u8, 1000, 1; + bench_in_place_xu32_0010_i0, u32, 10, 0; + bench_in_place_xu32_0100_i0, u32, 100, 0; + bench_in_place_xu32_1000_i0, u32, 1000, 0; + bench_in_place_xu32_0010_i1, u32, 10, 1; + bench_in_place_xu32_0100_i1, u32, 100, 1; + bench_in_place_xu32_1000_i1, u32, 1000, 1; + bench_in_place_u128_0010_i0, u128, 10, 0; + bench_in_place_u128_0100_i0, u128, 100, 0; + bench_in_place_u128_1000_i0, u128, 1000, 0; + bench_in_place_u128_0010_i1, u128, 10, 1; + bench_in_place_u128_0100_i1, u128, 100, 1; + bench_in_place_u128_1000_i1, u128, 1000, 1 +]; + +#[bench] +fn bench_in_place_recycle(b: &mut Bencher) { + let mut data = vec![0; 1000]; + + b.iter(|| { + let tmp = std::mem::take(&mut data); + data = black_box( + tmp.into_iter() + .enumerate() + .map(|(idx, e)| idx.wrapping_add(e)) + .fuse() + .collect::>(), + ); + }); +} + +#[bench] +fn bench_in_place_zip_recycle(b: &mut Bencher) { + let mut data = vec![0u8; 1000]; + let mut rng = crate::bench_rng(); + let mut subst = vec![0u8; 1000]; + rng.fill_bytes(&mut subst[..]); + + b.iter(|| { + let tmp = std::mem::take(&mut data); + let mangled = tmp + .into_iter() + .zip(subst.iter().copied()) + .enumerate() + .map(|(i, (d, s))| d.wrapping_add(i as u8) ^ s) + .collect::>(); + data = black_box(mangled); + }); +} + +#[bench] +fn bench_in_place_zip_iter_mut(b: &mut Bencher) { + let mut data = vec![0u8; 256]; + let mut rng = crate::bench_rng(); + let mut subst = vec![0u8; 1000]; + rng.fill_bytes(&mut subst[..]); + + b.iter(|| { + data.iter_mut().enumerate().for_each(|(i, d)| { + *d = d.wrapping_add(i as u8) ^ subst[i]; + }); + }); + + black_box(data); +} + +pub fn vec_cast(input: Vec) -> Vec { + input.into_iter().map(|e| unsafe { std::mem::transmute_copy(&e) }).collect() +} + +#[bench] +fn bench_transmute(b: &mut Bencher) { + let mut vec = vec![10u32; 100]; + b.bytes = 800; // 2 casts x 4 bytes x 100 + b.iter(|| { + let v = std::mem::take(&mut vec); + let v = black_box(vec_cast::(v)); + let v = black_box(vec_cast::(v)); + vec = v; + }); +} + +#[derive(Clone)] +struct Droppable(usize); + +impl Drop for Droppable { + fn drop(&mut self) { + black_box(self); + } +} + +#[bench] +fn bench_in_place_collect_droppable(b: &mut Bencher) { + let v: Vec = std::iter::repeat_with(|| Droppable(0)).take(1000).collect(); + b.iter(|| { + v.clone() + .into_iter() + .skip(100) + .enumerate() + .map(|(i, e)| Droppable(i ^ e.0)) + .collect::>() + }) +} + +const LEN: usize = 16384; + +#[bench] +fn bench_chain_collect(b: &mut Bencher) { + let data = black_box([0; LEN]); + b.iter(|| data.iter().cloned().chain([1]).collect::>()); +} + +#[bench] +fn bench_chain_chain_collect(b: &mut Bencher) { + let data = black_box([0; LEN]); + b.iter(|| data.iter().cloned().chain([1]).chain([2]).collect::>()); +} + +#[bench] +fn bench_nest_chain_chain_collect(b: &mut Bencher) { + let data = black_box([0; LEN]); + b.iter(|| { + data.iter().cloned().chain([1].iter().chain([2].iter()).cloned()).collect::>() + }); +} + +#[bench] +fn bench_range_map_collect(b: &mut Bencher) { + b.iter(|| (0..LEN).map(|_| u32::default()).collect::>()); +} + +#[bench] +fn bench_chain_extend_ref(b: &mut Bencher) { + let data = black_box([0; LEN]); + b.iter(|| { + let mut v = Vec::::with_capacity(data.len() + 1); + v.extend(data.iter().chain([1].iter())); + v + }); +} + +#[bench] +fn bench_chain_extend_value(b: &mut Bencher) { + let data = black_box([0; LEN]); + b.iter(|| { + let mut v = Vec::::with_capacity(data.len() + 1); + v.extend(data.iter().cloned().chain(Some(1))); + v + }); +} + +#[bench] +fn bench_rev_1(b: &mut Bencher) { + let data = black_box([0; LEN]); + b.iter(|| { + let mut v = Vec::::new(); + v.extend(data.iter().rev()); + v + }); +} + +#[bench] +fn bench_rev_2(b: &mut Bencher) { + let data = black_box([0; LEN]); + b.iter(|| { + let mut v = Vec::::with_capacity(data.len()); + v.extend(data.iter().rev()); + v + }); +} + +#[bench] +fn bench_map_regular(b: &mut Bencher) { + let data = black_box([(0, 0); LEN]); + b.iter(|| { + let mut v = Vec::::new(); + v.extend(data.iter().map(|t| t.1)); + v + }); +} + +#[bench] +fn bench_map_fast(b: &mut Bencher) { + let data = black_box([(0, 0); LEN]); + b.iter(|| { + let mut result: Vec = Vec::with_capacity(data.len()); + for i in 0..data.len() { + unsafe { + *result.as_mut_ptr().add(i) = data[i].0; + result.set_len(i); + } + } + result + }); +} + +fn random_sorted_fill(mut seed: u32, buf: &mut [u32]) { + let mask = if buf.len() < 8192 { + 0xFF + } else if buf.len() < 200_000 { + 0xFFFF + } else { + 0xFFFF_FFFF + }; + + for item in buf.iter_mut() { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + + *item = seed & mask; + } + + buf.sort(); +} + +fn bench_vec_dedup_old(b: &mut Bencher, sz: usize) { + let mut template = vec![0u32; sz]; + b.bytes = std::mem::size_of_val(template.as_slice()) as u64; + random_sorted_fill(0x43, &mut template); + + let mut vec = template.clone(); + b.iter(|| { + let len = { + let (dedup, _) = vec.partition_dedup(); + dedup.len() + }; + vec.truncate(len); + + black_box(vec.first()); + vec.clear(); + vec.extend_from_slice(&template); + }); +} + +fn bench_vec_dedup_new(b: &mut Bencher, sz: usize) { + let mut template = vec![0u32; sz]; + b.bytes = std::mem::size_of_val(template.as_slice()) as u64; + random_sorted_fill(0x43, &mut template); + + let mut vec = template.clone(); + b.iter(|| { + vec.dedup(); + black_box(vec.first()); + vec.clear(); + vec.extend_from_slice(&template); + }); +} + +#[bench] +fn bench_dedup_old_100(b: &mut Bencher) { + bench_vec_dedup_old(b, 100); +} +#[bench] +fn bench_dedup_new_100(b: &mut Bencher) { + bench_vec_dedup_new(b, 100); +} + +#[bench] +fn bench_dedup_old_1000(b: &mut Bencher) { + bench_vec_dedup_old(b, 1000); +} +#[bench] +fn bench_dedup_new_1000(b: &mut Bencher) { + bench_vec_dedup_new(b, 1000); +} + +#[bench] +fn bench_dedup_old_10000(b: &mut Bencher) { + bench_vec_dedup_old(b, 10000); +} +#[bench] +fn bench_dedup_new_10000(b: &mut Bencher) { + bench_vec_dedup_new(b, 10000); +} + +#[bench] +fn bench_dedup_old_100000(b: &mut Bencher) { + bench_vec_dedup_old(b, 100000); +} +#[bench] +fn bench_dedup_new_100000(b: &mut Bencher) { + bench_vec_dedup_new(b, 100000); +} + +#[bench] +fn bench_flat_map_collect(b: &mut Bencher) { + let v = vec![777u32; 500000]; + b.iter(|| v.iter().flat_map(|color| color.rotate_left(8).to_be_bytes()).collect::>()); +} + +/// Reference benchmark that `retain` has to compete with. +#[bench] +fn bench_retain_iter_100000(b: &mut Bencher) { + let mut v = Vec::with_capacity(100000); + + b.iter(|| { + let mut tmp = std::mem::take(&mut v); + tmp.clear(); + tmp.extend(black_box(1..=100000)); + v = tmp.into_iter().filter(|x| x & 1 == 0).collect(); + }); +} + +#[bench] +fn bench_retain_100000(b: &mut Bencher) { + let mut v = Vec::with_capacity(100000); + + b.iter(|| { + v.clear(); + v.extend(black_box(1..=100000)); + v.retain(|x| x & 1 == 0) + }); +} + +#[bench] +fn bench_retain_whole_100000(b: &mut Bencher) { + let mut v = black_box(vec![826u32; 100000]); + b.iter(|| v.retain(|x| *x == 826u32)); +} + +#[bench] +fn bench_next_chunk(b: &mut Bencher) { + let v = vec![13u8; 2048]; + + b.iter(|| { + const CHUNK: usize = 8; + + let mut sum = [0u32; CHUNK]; + let mut iter = black_box(v.clone()).into_iter(); + + while let Ok(chunk) = iter.next_chunk::() { + for i in 0..CHUNK { + sum[i] += chunk[i] as u32; + } + } + + sum + }) +} diff --git a/crux-mir/lib/alloc/benches/vec_deque.rs b/crux-mir/lib/alloc/benches/vec_deque.rs index bf2dffd1e..7c78561eb 100644 --- a/crux-mir/lib/alloc/benches/vec_deque.rs +++ b/crux-mir/lib/alloc/benches/vec_deque.rs @@ -52,3 +52,74 @@ fn bench_try_fold(b: &mut Bencher) { b.iter(|| black_box(ring.iter().try_fold(0, |a, b| Some(a + b)))) } + +#[bench] +fn bench_from_array_1000(b: &mut Bencher) { + const N: usize = 1000; + let mut array: [usize; N] = [0; N]; + + for i in 0..N { + array[i] = i; + } + + b.iter(|| { + let deq: VecDeque<_> = array.into(); + black_box(deq); + }) +} + +#[bench] +fn bench_extend_bytes(b: &mut Bencher) { + let mut ring: VecDeque = VecDeque::with_capacity(1000); + let input: &[u8] = &[128; 512]; + + b.iter(|| { + ring.clear(); + ring.extend(black_box(input)); + }); +} + +#[bench] +fn bench_extend_vec(b: &mut Bencher) { + let mut ring: VecDeque = VecDeque::with_capacity(1000); + let input = vec![128; 512]; + + b.iter(|| { + ring.clear(); + + let input = input.clone(); + ring.extend(black_box(input)); + }); +} + +#[bench] +fn bench_extend_trustedlen(b: &mut Bencher) { + let mut ring: VecDeque = VecDeque::with_capacity(1000); + + b.iter(|| { + ring.clear(); + ring.extend(black_box(0..512)); + }); +} + +#[bench] +fn bench_extend_chained_trustedlen(b: &mut Bencher) { + let mut ring: VecDeque = VecDeque::with_capacity(1000); + + b.iter(|| { + ring.clear(); + ring.extend(black_box((0..256).chain(768..1024))); + }); +} + +#[bench] +fn bench_extend_chained_bytes(b: &mut Bencher) { + let mut ring: VecDeque = VecDeque::with_capacity(1000); + let input1: &[u16] = &[128; 256]; + let input2: &[u16] = &[255; 256]; + + b.iter(|| { + ring.clear(); + ring.extend(black_box(input1.iter().chain(input2.iter()))); + }); +} diff --git a/crux-mir/lib/alloc/src/Cargo.toml b/crux-mir/lib/alloc/src/Cargo.toml deleted file mode 100644 index 87df193c9..000000000 --- a/crux-mir/lib/alloc/src/Cargo.toml +++ /dev/null @@ -1,37 +0,0 @@ -[package] -authors = ["The Rust Project Developers"] -name = "alloc" -version = "0.0.0" -autotests = false -autobenches = false -edition = "2018" - -[lib] -name = "alloc" -path = "lib.rs" - -[dependencies] -core = { path = "../libcore" } -compiler_builtins = { version = "0.1.10", features = ['rustc-dep-of-std'] } - -[dev-dependencies] -rand = "0.7" -rand_xorshift = "0.2" - -[[test]] -name = "collectionstests" -path = "../liballoc/tests/lib.rs" - -[[bench]] -name = "collectionsbenches" -path = "../liballoc/benches/lib.rs" - -[[bench]] -name = "vec_deque_append_bench" -path = "../liballoc/benches/vec_deque_append.rs" -harness = false - -[features] -compiler-builtins-mem = ['compiler_builtins/mem'] -compiler-builtins-c = ["compiler_builtins/c"] -opaque-arc-atomics = [] diff --git a/crux-mir/lib/alloc/src/alloc.rs b/crux-mir/lib/alloc/src/alloc.rs index 703fb666e..3a797bd5e 100644 --- a/crux-mir/lib/alloc/src/alloc.rs +++ b/crux-mir/lib/alloc/src/alloc.rs @@ -2,47 +2,61 @@ #![stable(feature = "alloc_module", since = "1.28.0")] +#[cfg(not(test))] +use core::intrinsics; use core::intrinsics::{min_align_of_val, size_of_val}; -use core::ptr::{NonNull, Unique}; -use core::usize; + +use core::ptr::Unique; +#[cfg(not(test))] +use core::ptr::{self, NonNull}; #[stable(feature = "alloc_module", since = "1.28.0")] #[doc(inline)] pub use core::alloc::*; +use core::marker::Destruct; + #[cfg(test)] mod tests; extern "Rust" { - // These are the magic symbols to call the global allocator. rustc generates - // them from the `#[global_allocator]` attribute if there is one, or uses the - // default implementations in libstd (`__rdl_alloc` etc in `src/libstd/alloc.rs`) + // These are the magic symbols to call the global allocator. rustc generates + // them to call `__rg_alloc` etc. if there is a `#[global_allocator]` attribute + // (the code expanding that attribute macro generates those functions), or to call + // the default implementations in std (`__rdl_alloc` etc. in `library/std/src/alloc.rs`) // otherwise. + // The rustc fork of LLVM 14 and earlier also special-cases these function names to be able to optimize them + // like `malloc`, `realloc`, and `free`, respectively. #[rustc_allocator] - #[rustc_allocator_nounwind] + #[rustc_nounwind] fn __rust_alloc(size: usize, align: usize) -> *mut u8; - #[rustc_allocator_nounwind] + #[rustc_deallocator] + #[rustc_nounwind] fn __rust_dealloc(ptr: *mut u8, size: usize, align: usize); - #[rustc_allocator_nounwind] + #[rustc_reallocator] + #[rustc_nounwind] fn __rust_realloc(ptr: *mut u8, old_size: usize, align: usize, new_size: usize) -> *mut u8; - #[rustc_allocator_nounwind] + #[rustc_allocator_zeroed] + #[rustc_nounwind] fn __rust_alloc_zeroed(size: usize, align: usize) -> *mut u8; } /// The global memory allocator. /// -/// This type implements the [`AllocRef`] trait by forwarding calls +/// This type implements the [`Allocator`] trait by forwarding calls /// to the allocator registered with the `#[global_allocator]` attribute /// if there is one, or the `std` crate’s default. /// /// Note: while this type is unstable, the functionality it provides can be -/// accessed through the [free functions in `alloc`](index.html#functions). -/// -/// [`AllocRef`]: trait.AllocRef.html +/// accessed through the [free functions in `alloc`](self#functions). #[unstable(feature = "allocator_api", issue = "32838")] #[derive(Copy, Clone, Default, Debug)] +#[cfg(not(test))] pub struct Global; +#[cfg(test)] +pub use std::alloc::Global; + /// Allocate memory with the global allocator. /// /// This function forwards calls to the [`GlobalAlloc::alloc`] method @@ -50,24 +64,23 @@ pub struct Global; /// if there is one, or the `std` crate’s default. /// /// This function is expected to be deprecated in favor of the `alloc` method -/// of the [`Global`] type when it and the [`AllocRef`] trait become stable. +/// of the [`Global`] type when it and the [`Allocator`] trait become stable. /// /// # Safety /// /// See [`GlobalAlloc::alloc`]. /// -/// [`Global`]: struct.Global.html -/// [`AllocRef`]: trait.AllocRef.html -/// [`GlobalAlloc::alloc`]: trait.GlobalAlloc.html#tymethod.alloc -/// /// # Examples /// /// ``` -/// use std::alloc::{alloc, dealloc, Layout}; +/// use std::alloc::{alloc, dealloc, handle_alloc_error, Layout}; /// /// unsafe { /// let layout = Layout::new::(); /// let ptr = alloc(layout); +/// if ptr.is_null() { +/// handle_alloc_error(layout); +/// } /// /// *(ptr as *mut u16) = 42; /// assert_eq!(*(ptr as *mut u16), 42); @@ -76,9 +89,10 @@ pub struct Global; /// } /// ``` #[stable(feature = "global_alloc", since = "1.28.0")] +#[must_use = "losing the pointer will leak memory"] #[inline] pub unsafe fn alloc(layout: Layout) -> *mut u8 { - __rust_alloc(layout.size(), layout.align()) + unsafe { __rust_alloc(layout.size(), layout.align()) } } /// Deallocate memory with the global allocator. @@ -88,19 +102,15 @@ pub unsafe fn alloc(layout: Layout) -> *mut u8 { /// if there is one, or the `std` crate’s default. /// /// This function is expected to be deprecated in favor of the `dealloc` method -/// of the [`Global`] type when it and the [`AllocRef`] trait become stable. +/// of the [`Global`] type when it and the [`Allocator`] trait become stable. /// /// # Safety /// /// See [`GlobalAlloc::dealloc`]. -/// -/// [`Global`]: struct.Global.html -/// [`AllocRef`]: trait.AllocRef.html -/// [`GlobalAlloc::dealloc`]: trait.GlobalAlloc.html#tymethod.dealloc #[stable(feature = "global_alloc", since = "1.28.0")] #[inline] pub unsafe fn dealloc(ptr: *mut u8, layout: Layout) { - __rust_dealloc(ptr, layout.size(), layout.align()) + unsafe { __rust_dealloc(ptr, layout.size(), layout.align()) } } /// Reallocate memory with the global allocator. @@ -110,19 +120,16 @@ pub unsafe fn dealloc(ptr: *mut u8, layout: Layout) { /// if there is one, or the `std` crate’s default. /// /// This function is expected to be deprecated in favor of the `realloc` method -/// of the [`Global`] type when it and the [`AllocRef`] trait become stable. +/// of the [`Global`] type when it and the [`Allocator`] trait become stable. /// /// # Safety /// /// See [`GlobalAlloc::realloc`]. -/// -/// [`Global`]: struct.Global.html -/// [`AllocRef`]: trait.AllocRef.html -/// [`GlobalAlloc::realloc`]: trait.GlobalAlloc.html#method.realloc #[stable(feature = "global_alloc", since = "1.28.0")] +#[must_use = "losing the pointer will leak memory"] #[inline] pub unsafe fn realloc(ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - __rust_realloc(ptr, layout.size(), layout.align(), new_size) + unsafe { __rust_realloc(ptr, layout.size(), layout.align(), new_size) } } /// Allocate zero-initialized memory with the global allocator. @@ -132,16 +139,12 @@ pub unsafe fn realloc(ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 /// if there is one, or the `std` crate’s default. /// /// This function is expected to be deprecated in favor of the `alloc_zeroed` method -/// of the [`Global`] type when it and the [`AllocRef`] trait become stable. +/// of the [`Global`] type when it and the [`Allocator`] trait become stable. /// /// # Safety /// /// See [`GlobalAlloc::alloc_zeroed`]. /// -/// [`Global`]: struct.Global.html -/// [`AllocRef`]: trait.AllocRef.html -/// [`GlobalAlloc::alloc_zeroed`]: trait.GlobalAlloc.html#method.alloc_zeroed -/// /// # Examples /// /// ``` @@ -157,87 +160,203 @@ pub unsafe fn realloc(ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 /// } /// ``` #[stable(feature = "global_alloc", since = "1.28.0")] +#[must_use = "losing the pointer will leak memory"] #[inline] pub unsafe fn alloc_zeroed(layout: Layout) -> *mut u8 { - __rust_alloc_zeroed(layout.size(), layout.align()) + unsafe { __rust_alloc_zeroed(layout.size(), layout.align()) } } -#[unstable(feature = "allocator_api", issue = "32838")] -unsafe impl AllocRef for Global { +#[cfg(not(test))] +impl Global { #[inline] - fn alloc(&mut self, layout: Layout) -> Result<(NonNull, usize), AllocErr> { - if layout.size() == 0 { - Ok((layout.dangling(), 0)) - } else { - unsafe { NonNull::new(alloc(layout)).ok_or(AllocErr).map(|p| (p, layout.size())) } + fn alloc_impl(&self, layout: Layout, zeroed: bool) -> Result, AllocError> { + match layout.size() { + 0 => Ok(NonNull::slice_from_raw_parts(layout.dangling(), 0)), + // SAFETY: `layout` is non-zero in size, + size => unsafe { + let raw_ptr = if zeroed { alloc_zeroed(layout) } else { alloc(layout) }; + let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; + Ok(NonNull::slice_from_raw_parts(ptr, size)) + }, + } + } + + // SAFETY: Same as `Allocator::grow` + #[inline] + unsafe fn grow_impl( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + zeroed: bool, + ) -> Result, AllocError> { + debug_assert!( + new_layout.size() >= old_layout.size(), + "`new_layout.size()` must be greater than or equal to `old_layout.size()`" + ); + + match old_layout.size() { + 0 => self.alloc_impl(new_layout, zeroed), + + // SAFETY: `new_size` is non-zero as `old_size` is greater than or equal to `new_size` + // as required by safety conditions. Other conditions must be upheld by the caller + old_size if old_layout.align() == new_layout.align() => unsafe { + let new_size = new_layout.size(); + + // `realloc` probably checks for `new_size >= old_layout.size()` or something similar. + intrinsics::assume(new_size >= old_layout.size()); + + let raw_ptr = realloc(ptr.as_ptr(), old_layout, new_size); + let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; + if zeroed { + raw_ptr.add(old_size).write_bytes(0, new_size - old_size); + } + Ok(NonNull::slice_from_raw_parts(ptr, new_size)) + }, + + // SAFETY: because `new_layout.size()` must be greater than or equal to `old_size`, + // both the old and new memory allocation are valid for reads and writes for `old_size` + // bytes. Also, because the old allocation wasn't yet deallocated, it cannot overlap + // `new_ptr`. Thus, the call to `copy_nonoverlapping` is safe. The safety contract + // for `dealloc` must be upheld by the caller. + old_size => unsafe { + let new_ptr = self.alloc_impl(new_layout, zeroed)?; + ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_mut_ptr(), old_size); + self.deallocate(ptr, old_layout); + Ok(new_ptr) + }, } } +} +#[unstable(feature = "allocator_api", issue = "32838")] +#[cfg(not(test))] +unsafe impl Allocator for Global { #[inline] - unsafe fn dealloc(&mut self, ptr: NonNull, layout: Layout) { + fn allocate(&self, layout: Layout) -> Result, AllocError> { + self.alloc_impl(layout, false) + } + + #[inline] + fn allocate_zeroed(&self, layout: Layout) -> Result, AllocError> { + self.alloc_impl(layout, true) + } + + #[inline] + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { if layout.size() != 0 { - dealloc(ptr.as_ptr(), layout) + // SAFETY: `layout` is non-zero in size, + // other conditions must be upheld by the caller + unsafe { dealloc(ptr.as_ptr(), layout) } } } #[inline] - unsafe fn realloc( - &mut self, + unsafe fn grow( + &self, ptr: NonNull, - layout: Layout, - new_size: usize, - ) -> Result<(NonNull, usize), AllocErr> { - match (layout.size(), new_size) { - (0, 0) => Ok((layout.dangling(), 0)), - (0, _) => self.alloc(Layout::from_size_align_unchecked(new_size, layout.align())), - (_, 0) => { - self.dealloc(ptr, layout); - Ok((layout.dangling(), 0)) - } - (_, _) => NonNull::new(realloc(ptr.as_ptr(), layout, new_size)) - .ok_or(AllocErr) - .map(|p| (p, new_size)), - } + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + // SAFETY: all conditions must be upheld by the caller + unsafe { self.grow_impl(ptr, old_layout, new_layout, false) } } #[inline] - fn alloc_zeroed(&mut self, layout: Layout) -> Result<(NonNull, usize), AllocErr> { - if layout.size() == 0 { - Ok((layout.dangling(), 0)) - } else { - unsafe { - NonNull::new(alloc_zeroed(layout)).ok_or(AllocErr).map(|p| (p, layout.size())) - } + unsafe fn grow_zeroed( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + // SAFETY: all conditions must be upheld by the caller + unsafe { self.grow_impl(ptr, old_layout, new_layout, true) } + } + + #[inline] + unsafe fn shrink( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + debug_assert!( + new_layout.size() <= old_layout.size(), + "`new_layout.size()` must be smaller than or equal to `old_layout.size()`" + ); + + match new_layout.size() { + // SAFETY: conditions must be upheld by the caller + 0 => unsafe { + self.deallocate(ptr, old_layout); + Ok(NonNull::slice_from_raw_parts(new_layout.dangling(), 0)) + }, + + // SAFETY: `new_size` is non-zero. Other conditions must be upheld by the caller + new_size if old_layout.align() == new_layout.align() => unsafe { + // `realloc` probably checks for `new_size <= old_layout.size()` or something similar. + intrinsics::assume(new_size <= old_layout.size()); + + let raw_ptr = realloc(ptr.as_ptr(), old_layout, new_size); + let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; + Ok(NonNull::slice_from_raw_parts(ptr, new_size)) + }, + + // SAFETY: because `new_size` must be smaller than or equal to `old_layout.size()`, + // both the old and new memory allocation are valid for reads and writes for `new_size` + // bytes. Also, because the old allocation wasn't yet deallocated, it cannot overlap + // `new_ptr`. Thus, the call to `copy_nonoverlapping` is safe. The safety contract + // for `dealloc` must be upheld by the caller. + new_size => unsafe { + let new_ptr = self.allocate(new_layout)?; + ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_mut_ptr(), new_size); + self.deallocate(ptr, old_layout); + Ok(new_ptr) + }, } } } /// The allocator for unique pointers. -// This function must not unwind. If it does, MIR codegen will fail. -#[cfg(not(test))] +#[cfg(all(not(no_global_oom_handling), not(test)))] #[lang = "exchange_malloc"] #[inline] unsafe fn exchange_malloc(size: usize, align: usize) -> *mut u8 { - if size == 0 { - align as *mut u8 - } else { - let layout = Layout::from_size_align_unchecked(size, align); - match Global.alloc(layout) { - Ok((ptr, _)) => ptr.as_ptr(), - Err(_) => handle_alloc_error(layout), - } + let layout = unsafe { Layout::from_size_align_unchecked(size, align) }; + match Global.allocate(layout) { + Ok(ptr) => ptr.as_mut_ptr(), + Err(_) => handle_alloc_error(layout), } } #[cfg_attr(not(test), lang = "box_free")] #[inline] +#[rustc_const_unstable(feature = "const_box", issue = "92521")] // This signature has to be the same as `Box`, otherwise an ICE will happen. -// When an additional parameter to `Box` is added (like `A: AllocRef`), this has to be added here as +// When an additional parameter to `Box` is added (like `A: Allocator`), this has to be added here as // well. -// For example if `Box` is changed to `struct Box(Unique, A)`, -// this function has to be changed to `fn box_free(Unique, A)` as well. -pub(crate) unsafe fn box_free(_ptr: Unique) { - // Crux: currently we don't implement `deallocate`, so this is a no-op. +// For example if `Box` is changed to `struct Box(Unique, A)`, +// this function has to be changed to `fn box_free(Unique, A)` as well. +pub(crate) const unsafe fn box_free( + ptr: Unique, + alloc: A, +) { + unsafe { + let size = size_of_val(ptr.as_ref()); + let align = min_align_of_val(ptr.as_ref()); + let layout = Layout::from_size_align_unchecked(size, align); + alloc.deallocate(From::from(ptr.cast()), layout) + } +} + +// # Allocation error handler + +#[cfg(not(no_global_oom_handling))] +extern "Rust" { + // This is the magic symbol to call the global alloc error handler. rustc generates + // it to call `__rg_oom` if there is a `#[alloc_error_handler]`, or to call the + // default implementations below (`__rdl_oom`) otherwise. + fn __rust_alloc_error_handler(size: usize, align: usize) -> !; } /// Abort on memory allocation error or failure. @@ -253,11 +372,72 @@ pub(crate) unsafe fn box_free(_ptr: Unique) { /// [`set_alloc_error_hook`]: ../../std/alloc/fn.set_alloc_error_hook.html /// [`take_alloc_error_hook`]: ../../std/alloc/fn.take_alloc_error_hook.html #[stable(feature = "global_alloc", since = "1.28.0")] -#[rustc_allocator_nounwind] -pub fn handle_alloc_error(layout: Layout) -> ! { - extern "Rust" { - #[lang = "oom"] - fn oom_impl(layout: Layout) -> !; +#[rustc_const_unstable(feature = "const_alloc_error", issue = "92523")] +#[cfg(all(not(no_global_oom_handling), not(test)))] +#[cold] +pub const fn handle_alloc_error(layout: Layout) -> ! { + const fn ct_error(_: Layout) -> ! { + panic!("allocation failed"); + } + + fn rt_error(layout: Layout) -> ! { + unsafe { + __rust_alloc_error_handler(layout.size(), layout.align()); + } + } + + unsafe { core::intrinsics::const_eval_select((layout,), ct_error, rt_error) } +} + +// For alloc test `std::alloc::handle_alloc_error` can be used directly. +#[cfg(all(not(no_global_oom_handling), test))] +pub use std::alloc::handle_alloc_error; + +#[cfg(all(not(no_global_oom_handling), not(test)))] +#[doc(hidden)] +#[allow(unused_attributes)] +#[unstable(feature = "alloc_internals", issue = "none")] +pub mod __alloc_error_handler { + // called via generated `__rust_alloc_error_handler` if there is no + // `#[alloc_error_handler]`. + #[rustc_std_internal_symbol] + pub unsafe fn __rdl_oom(size: usize, _align: usize) -> ! { + extern "Rust" { + // This symbol is emitted by rustc next to __rust_alloc_error_handler. + // Its value depends on the -Zoom={panic,abort} compiler option. + static __rust_alloc_error_handler_should_panic: u8; + } + + #[allow(unused_unsafe)] + if unsafe { __rust_alloc_error_handler_should_panic != 0 } { + panic!("memory allocation of {size} bytes failed") + } else { + core::panicking::panic_nounwind_fmt(format_args!( + "memory allocation of {size} bytes failed" + )) + } + } +} + +/// Specialize clones into pre-allocated, uninitialized memory. +/// Used by `Box::clone` and `Rc`/`Arc::make_mut`. +pub(crate) trait WriteCloneIntoRaw: Sized { + unsafe fn write_clone_into_raw(&self, target: *mut Self); +} + +impl WriteCloneIntoRaw for T { + #[inline] + default unsafe fn write_clone_into_raw(&self, target: *mut Self) { + // Having allocated *first* may allow the optimizer to create + // the cloned value in-place, skipping the local and move. + unsafe { target.write(self.clone()) }; + } +} + +impl WriteCloneIntoRaw for T { + #[inline] + unsafe fn write_clone_into_raw(&self, target: *mut Self) { + // We can always copy in-place, without ever involving a local value. + unsafe { target.copy_from_nonoverlapping(self, 1) }; } - unsafe { oom_impl(layout) } } diff --git a/crux-mir/lib/alloc/src/alloc/tests.rs b/crux-mir/lib/alloc/src/alloc/tests.rs index 55944398e..1a5938fd3 100644 --- a/crux-mir/lib/alloc/src/alloc/tests.rs +++ b/crux-mir/lib/alloc/src/alloc/tests.rs @@ -8,23 +8,22 @@ use test::Bencher; fn allocate_zeroed() { unsafe { let layout = Layout::from_size_align(1024, 1).unwrap(); - let (ptr, _) = - Global.alloc_zeroed(layout.clone()).unwrap_or_else(|_| handle_alloc_error(layout)); + let ptr = + Global.allocate_zeroed(layout.clone()).unwrap_or_else(|_| handle_alloc_error(layout)); - let mut i = ptr.cast::().as_ptr(); + let mut i = ptr.as_non_null_ptr().as_ptr(); let end = i.add(layout.size()); while i < end { assert_eq!(*i, 0); - i = i.offset(1); + i = i.add(1); } - Global.dealloc(ptr, layout); + Global.deallocate(ptr.as_non_null_ptr(), layout); } } #[bench] -#[cfg_attr(miri, ignore)] // Miri does not support benchmarks fn alloc_owned_small(b: &mut Bencher) { b.iter(|| { - let _: Box<_> = box 10; + let _: Box<_> = Box::new(10); }) } diff --git a/crux-mir/lib/alloc/src/borrow.rs b/crux-mir/lib/alloc/src/borrow.rs index 51c233a21..83a138559 100644 --- a/crux-mir/lib/alloc/src/borrow.rs +++ b/crux-mir/lib/alloc/src/borrow.rs @@ -4,12 +4,15 @@ use core::cmp::Ordering; use core::hash::{Hash, Hasher}; -use core::ops::{Add, AddAssign, Deref}; +use core::ops::Deref; +#[cfg(not(no_global_oom_handling))] +use core::ops::{Add, AddAssign}; #[stable(feature = "rust1", since = "1.0.0")] pub use core::borrow::{Borrow, BorrowMut}; use crate::fmt; +#[cfg(not(no_global_oom_handling))] use crate::string::String; use Cow::*; @@ -18,7 +21,6 @@ use Cow::*; impl<'a, B: ?Sized> Borrow for Cow<'a, B> where B: ToOwned, - ::Owned: 'a, { fn borrow(&self) -> &B { &**self @@ -31,6 +33,7 @@ where /// implementing the `Clone` trait. But `Clone` works only for going from `&T` /// to `T`. The `ToOwned` trait generalizes `Clone` to construct owned data /// from any borrow of a given type. +#[cfg_attr(not(test), rustc_diagnostic_item = "ToOwned")] #[stable(feature = "rust1", since = "1.0.0")] pub trait ToOwned { /// The resulting type after obtaining ownership. @@ -56,21 +59,20 @@ pub trait ToOwned { /// Uses borrowed data to replace owned data, usually by cloning. /// - /// This is borrow-generalized version of `Clone::clone_from`. + /// This is borrow-generalized version of [`Clone::clone_from`]. /// /// # Examples /// /// Basic usage: /// /// ``` - /// # #![feature(toowned_clone_into)] /// let mut s: String = String::new(); /// "hello".clone_into(&mut s); /// /// let mut v: Vec = Vec::new(); /// [1, 2][..].clone_into(&mut v); /// ``` - #[unstable(feature = "toowned_clone_into", reason = "recently added", issue = "41263")] + #[stable(feature = "toowned_clone_into", since = "1.63.0")] fn clone_into(&self, target: &mut Self::Owned) { *target = self.to_owned(); } @@ -103,6 +105,11 @@ where /// is desired, `to_mut` will obtain a mutable reference to an owned /// value, cloning if necessary. /// +/// If you need reference-counting pointers, note that +/// [`Rc::make_mut`][crate::rc::Rc::make_mut] and +/// [`Arc::make_mut`][crate::sync::Arc::make_mut] can provide clone-on-write +/// functionality as well. +/// /// # Examples /// /// ``` @@ -152,7 +159,7 @@ where /// let readonly = [1, 2]; /// let borrowed = Items::new((&readonly[..]).into()); /// match borrowed { -/// Items { values: Cow::Borrowed(b) } => println!("borrowed {:?}", b), +/// Items { values: Cow::Borrowed(b) } => println!("borrowed {b:?}"), /// _ => panic!("expect borrowed value"), /// } /// @@ -161,13 +168,14 @@ where /// clone_on_write.values.to_mut().push(3); /// println!("clone_on_write = {:?}", clone_on_write.values); /// -/// // The data was mutated. Let check it out. +/// // The data was mutated. Let's check it out. /// match clone_on_write { /// Items { values: Cow::Owned(_) } => println!("clone_on_write contains owned data"), /// _ => panic!("expect owned data"), /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "Cow")] pub enum Cow<'a, B: ?Sized + 'a> where B: ToOwned, @@ -217,7 +225,8 @@ impl Cow<'_, B> { /// assert!(!bull.is_borrowed()); /// ``` #[unstable(feature = "cow_is_borrowed", issue = "65143")] - pub fn is_borrowed(&self) -> bool { + #[rustc_const_unstable(feature = "const_cow_is_borrowed", issue = "65143")] + pub const fn is_borrowed(&self) -> bool { match *self { Borrowed(_) => true, Owned(_) => false, @@ -239,7 +248,8 @@ impl Cow<'_, B> { /// assert!(!bull.is_owned()); /// ``` #[unstable(feature = "cow_is_borrowed", issue = "65143")] - pub fn is_owned(&self) -> bool { + #[rustc_const_unstable(feature = "const_cow_is_borrowed", issue = "65143")] + pub const fn is_owned(&self) -> bool { !self.is_borrowed() } @@ -280,8 +290,7 @@ impl Cow<'_, B> { /// /// # Examples /// - /// Calling `into_owned` on a `Cow::Borrowed` clones the underlying data - /// and becomes a `Cow::Owned`: + /// Calling `into_owned` on a `Cow::Borrowed` returns a clone of the borrowed data: /// /// ``` /// use std::borrow::Cow; @@ -295,7 +304,8 @@ impl Cow<'_, B> { /// ); /// ``` /// - /// Calling `into_owned` on a `Cow::Owned` is a no-op: + /// Calling `into_owned` on a `Cow::Owned` returns the owned data. The data is moved out of the + /// `Cow` without being cloned. /// /// ``` /// use std::borrow::Cow; @@ -318,7 +328,11 @@ impl Cow<'_, B> { } #[stable(feature = "rust1", since = "1.0.0")] -impl Deref for Cow<'_, B> { +#[rustc_const_unstable(feature = "const_deref", issue = "88955")] +impl const Deref for Cow<'_, B> +where + B::Owned: ~const Borrow, +{ type Target = B; fn deref(&self) -> &B { @@ -421,6 +435,7 @@ impl AsRef for Cow<'_, T> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "cow_add", since = "1.14.0")] impl<'a> Add<&'a str> for Cow<'a, str> { type Output = Cow<'a, str>; @@ -432,6 +447,7 @@ impl<'a> Add<&'a str> for Cow<'a, str> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "cow_add", since = "1.14.0")] impl<'a> Add> for Cow<'a, str> { type Output = Cow<'a, str>; @@ -443,6 +459,7 @@ impl<'a> Add> for Cow<'a, str> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "cow_add", since = "1.14.0")] impl<'a> AddAssign<&'a str> for Cow<'a, str> { fn add_assign(&mut self, rhs: &'a str) { @@ -459,6 +476,7 @@ impl<'a> AddAssign<&'a str> for Cow<'a, str> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "cow_add", since = "1.14.0")] impl<'a> AddAssign> for Cow<'a, str> { fn add_assign(&mut self, rhs: Cow<'a, str>) { diff --git a/crux-mir/lib/alloc/src/boxed.rs b/crux-mir/lib/alloc/src/boxed.rs index 36641284a..a563b2587 100644 --- a/crux-mir/lib/alloc/src/boxed.rs +++ b/crux-mir/lib/alloc/src/boxed.rs @@ -1,4 +1,4 @@ -//! A pointer type for heap allocation. +//! The `Box` type for heap allocation. //! //! [`Box`], casually referred to as a 'box', provides the simplest form of //! heap allocation in Rust. Boxes provide ownership for this allocation, and @@ -31,7 +31,7 @@ //! } //! //! let list: List = List::Cons(1, Box::new(List::Cons(2, Box::new(List::Nil)))); -//! println!("{:?}", list); +//! println!("{list:?}"); //! ``` //! //! This will print `Cons(1, Cons(2, Nil))`. @@ -62,6 +62,13 @@ //! T` obtained from [`Box::::into_raw`] may be deallocated using the //! [`Global`] allocator with [`Layout::for_value(&*value)`]. //! +//! For zero-sized values, the `Box` pointer still has to be [valid] for reads +//! and writes and sufficiently aligned. In particular, casting any aligned +//! non-zero integer literal to a raw pointer produces a valid pointer, but a +//! pointer pointing into previously allocated memory that since got freed is +//! not valid. The recommended way to build a Box to a ZST if `Box::new` cannot +//! be used is to use [`ptr::NonNull::dangling`]. +//! //! So long as `T: Sized`, a `Box` is guaranteed to be represented //! as a single pointer and is also ABI-compatible with C pointers //! (i.e. the C type `T*`). This means that if you have extern "C" @@ -77,7 +84,7 @@ //! /* Returns ownership to the caller */ //! struct Foo* foo_new(void); //! -//! /* Takes ownership from the caller; no-op when invoked with NULL */ +//! /* Takes ownership from the caller; no-op when invoked with null */ //! void foo_delete(struct Foo*); //! ``` //! @@ -115,49 +122,83 @@ //! definition is just using `T*` can lead to undefined behavior, as //! described in [rust-lang/unsafe-code-guidelines#198][ucg#198]. //! +//! # Considerations for unsafe code +//! +//! **Warning: This section is not normative and is subject to change, possibly +//! being relaxed in the future! It is a simplified summary of the rules +//! currently implemented in the compiler.** +//! +//! The aliasing rules for `Box` are the same as for `&mut T`. `Box` +//! asserts uniqueness over its content. Using raw pointers derived from a box +//! after that box has been mutated through, moved or borrowed as `&mut T` +//! is not allowed. For more guidance on working with box from unsafe code, see +//! [rust-lang/unsafe-code-guidelines#326][ucg#326]. +//! +//! //! [ucg#198]: https://github.com/rust-lang/unsafe-code-guidelines/issues/198 -//! [dereferencing]: ../../std/ops/trait.Deref.html -//! [`Box`]: struct.Box.html -//! [`Box`]: struct.Box.html -//! [`Box::::from_raw(value)`]: struct.Box.html#method.from_raw -//! [`Box::::into_raw`]: struct.Box.html#method.into_raw -//! [`Global`]: ../alloc/struct.Global.html -//! [`Layout`]: ../alloc/struct.Layout.html -//! [`Layout::for_value(&*value)`]: ../alloc/struct.Layout.html#method.for_value +//! [ucg#326]: https://github.com/rust-lang/unsafe-code-guidelines/issues/326 +//! [dereferencing]: core::ops::Deref +//! [`Box::::from_raw(value)`]: Box::from_raw +//! [`Global`]: crate::alloc::Global +//! [`Layout`]: crate::alloc::Layout +//! [`Layout::for_value(&*value)`]: crate::alloc::Layout::for_value +//! [valid]: ptr#safety #![stable(feature = "rust1", since = "1.0.0")] use core::any::Any; -use core::array::LengthAtMost32; +use core::async_iter::AsyncIterator; use core::borrow; use core::cmp::Ordering; use core::convert::{From, TryFrom}; +use core::error::Error; use core::fmt; use core::future::Future; use core::hash::{Hash, Hasher}; -use core::iter::{FromIterator, FusedIterator, Iterator}; -use core::marker::{Unpin, Unsize}; +#[cfg(not(no_global_oom_handling))] +use core::iter::FromIterator; +use core::iter::{FusedIterator, Iterator}; +use core::marker::Tuple; +use core::marker::{Destruct, Unpin, Unsize}; use core::mem; use core::ops::{ CoerceUnsized, Deref, DerefMut, DispatchFromDyn, Generator, GeneratorState, Receiver, }; use core::pin::Pin; -use core::ptr::{self, NonNull, Unique}; -use core::slice; +use core::ptr::{self, Unique}; use core::task::{Context, Poll}; -use crate::alloc::{self, AllocRef, Global}; +#[cfg(not(no_global_oom_handling))] +use crate::alloc::{handle_alloc_error, WriteCloneIntoRaw}; +use crate::alloc::{AllocError, Allocator, Global, Layout}; +#[cfg(not(no_global_oom_handling))] +use crate::borrow::Cow; use crate::raw_vec::RawVec; +#[cfg(not(no_global_oom_handling))] use crate::str::from_boxed_utf8_unchecked; +#[cfg(not(no_global_oom_handling))] +use crate::string::String; +#[cfg(not(no_global_oom_handling))] use crate::vec::Vec; -/// A pointer type for heap allocation. +#[unstable(feature = "thin_box", issue = "92791")] +pub use thin::ThinBox; + +mod thin; + +/// A pointer type that uniquely owns a heap allocation of type `T`. /// /// See the [module-level documentation](../../std/boxed/index.html) for more. #[lang = "owned_box"] #[fundamental] #[stable(feature = "rust1", since = "1.0.0")] -pub struct Box(Unique); +// The declaration of the `Box` struct must be kept in sync with the +// `alloc::alloc::box_free` function or ICEs will happen. See the comment +// on `box_free` for more details. +pub struct Box< + T: ?Sized, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator = Global, +>(Unique, A); impl Box { /// Allocates memory on the heap and then places `x` into it. @@ -169,10 +210,13 @@ impl Box { /// ``` /// let five = Box::new(5); /// ``` - #[stable(feature = "rust1", since = "1.0.0")] + #[cfg(all(not(no_global_oom_handling)))] #[inline(always)] - pub fn new(x: T) -> Box { - box x + #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + pub fn new(x: T) -> Self { + #[rustc_box] + Box::new(x) } /// Constructs a new box with uninitialized contents. @@ -193,17 +237,12 @@ impl Box { /// /// assert_eq!(*five, 5) /// ``` + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] + #[inline] pub fn new_uninit() -> Box> { - let layout = alloc::Layout::new::>(); - unsafe { - let ptr = if layout.size() == 0 { - NonNull::dangling() - } else { - Global.alloc(layout).unwrap_or_else(|_| alloc::handle_alloc_error(layout)).0.cast() - }; - Box::from_raw(ptr.as_ptr()) - } + Self::new_uninit_in(Global) } /// Constructs a new `Box` with uninitialized contents, with the memory @@ -223,22 +262,357 @@ impl Box { /// assert_eq!(*zero, 0) /// ``` /// - /// [zeroed]: ../../std/mem/union.MaybeUninit.html#method.zeroed + /// [zeroed]: mem::MaybeUninit::zeroed + #[cfg(not(no_global_oom_handling))] + #[inline] #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] pub fn new_zeroed() -> Box> { - unsafe { - let mut uninit = Self::new_uninit(); - ptr::write_bytes::(uninit.as_mut_ptr(), 0, 1); - uninit - } + Self::new_zeroed_in(Global) } - /// Constructs a new `Pin>`. If `T` does not implement `Unpin`, then + /// Constructs a new `Pin>`. If `T` does not implement [`Unpin`], then /// `x` will be pinned in memory and unable to be moved. + /// + /// Constructing and pinning of the `Box` can also be done in two steps: `Box::pin(x)` + /// does the same as [Box::into_pin]\([Box::new]\(x)). Consider using + /// [`into_pin`](Box::into_pin) if you already have a `Box`, or if you want to + /// construct a (pinned) `Box` in a different way than with [`Box::new`]. + #[cfg(not(no_global_oom_handling))] #[stable(feature = "pin", since = "1.33.0")] + #[must_use] #[inline(always)] pub fn pin(x: T) -> Pin> { - (box x).into() + (#[rustc_box] + Box::new(x)) + .into() + } + + /// Allocates memory on the heap then places `x` into it, + /// returning an error if the allocation fails + /// + /// This doesn't actually allocate if `T` is zero-sized. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api)] + /// + /// let five = Box::try_new(5)?; + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + #[unstable(feature = "allocator_api", issue = "32838")] + #[inline] + pub fn try_new(x: T) -> Result { + Self::try_new_in(x, Global) + } + + /// Constructs a new box with uninitialized contents on the heap, + /// returning an error if the allocation fails + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// + /// let mut five = Box::::try_new_uninit()?; + /// + /// let five = unsafe { + /// // Deferred initialization: + /// five.as_mut_ptr().write(5); + /// + /// five.assume_init() + /// }; + /// + /// assert_eq!(*five, 5); + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + #[unstable(feature = "allocator_api", issue = "32838")] + // #[unstable(feature = "new_uninit", issue = "63291")] + #[inline] + pub fn try_new_uninit() -> Result>, AllocError> { + Box::try_new_uninit_in(Global) + } + + /// Constructs a new `Box` with uninitialized contents, with the memory + /// being filled with `0` bytes on the heap + /// + /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage + /// of this method. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// + /// let zero = Box::::try_new_zeroed()?; + /// let zero = unsafe { zero.assume_init() }; + /// + /// assert_eq!(*zero, 0); + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + /// + /// [zeroed]: mem::MaybeUninit::zeroed + #[unstable(feature = "allocator_api", issue = "32838")] + // #[unstable(feature = "new_uninit", issue = "63291")] + #[inline] + pub fn try_new_zeroed() -> Result>, AllocError> { + Box::try_new_zeroed_in(Global) + } +} + +impl Box { + /// Allocates memory in the given allocator then places `x` into it. + /// + /// This doesn't actually allocate if `T` is zero-sized. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api)] + /// + /// use std::alloc::System; + /// + /// let five = Box::new_in(5, System); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[unstable(feature = "allocator_api", issue = "32838")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[must_use] + #[inline] + pub const fn new_in(x: T, alloc: A) -> Self + where + A: ~const Allocator + ~const Destruct, + { + let mut boxed = Self::new_uninit_in(alloc); + unsafe { + boxed.as_mut_ptr().write(x); + boxed.assume_init() + } + } + + /// Allocates memory in the given allocator then places `x` into it, + /// returning an error if the allocation fails + /// + /// This doesn't actually allocate if `T` is zero-sized. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api)] + /// + /// use std::alloc::System; + /// + /// let five = Box::try_new_in(5, System)?; + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + #[unstable(feature = "allocator_api", issue = "32838")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[inline] + pub const fn try_new_in(x: T, alloc: A) -> Result + where + T: ~const Destruct, + A: ~const Allocator + ~const Destruct, + { + let mut boxed = Self::try_new_uninit_in(alloc)?; + unsafe { + boxed.as_mut_ptr().write(x); + Ok(boxed.assume_init()) + } + } + + /// Constructs a new box with uninitialized contents in the provided allocator. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// + /// use std::alloc::System; + /// + /// let mut five = Box::::new_uninit_in(System); + /// + /// let five = unsafe { + /// // Deferred initialization: + /// five.as_mut_ptr().write(5); + /// + /// five.assume_init() + /// }; + /// + /// assert_eq!(*five, 5) + /// ``` + #[unstable(feature = "allocator_api", issue = "32838")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[cfg(not(no_global_oom_handling))] + #[must_use] + // #[unstable(feature = "new_uninit", issue = "63291")] + pub const fn new_uninit_in(alloc: A) -> Box, A> + where + A: ~const Allocator + ~const Destruct, + { + let layout = Layout::new::>(); + // NOTE: Prefer match over unwrap_or_else since closure sometimes not inlineable. + // That would make code size bigger. + match Box::try_new_uninit_in(alloc) { + Ok(m) => m, + Err(_) => handle_alloc_error(layout), + } + } + + /// Constructs a new box with uninitialized contents in the provided allocator, + /// returning an error if the allocation fails + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// + /// use std::alloc::System; + /// + /// let mut five = Box::::try_new_uninit_in(System)?; + /// + /// let five = unsafe { + /// // Deferred initialization: + /// five.as_mut_ptr().write(5); + /// + /// five.assume_init() + /// }; + /// + /// assert_eq!(*five, 5); + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + #[unstable(feature = "allocator_api", issue = "32838")] + // #[unstable(feature = "new_uninit", issue = "63291")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + pub const fn try_new_uninit_in(alloc: A) -> Result, A>, AllocError> + where + A: ~const Allocator + ~const Destruct, + { + let layout = Layout::new::>(); + let ptr = alloc.allocate(layout)?.cast(); + unsafe { Ok(Box::from_raw_in(ptr.as_ptr(), alloc)) } + } + + /// Constructs a new `Box` with uninitialized contents, with the memory + /// being filled with `0` bytes in the provided allocator. + /// + /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage + /// of this method. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// + /// use std::alloc::System; + /// + /// let zero = Box::::new_zeroed_in(System); + /// let zero = unsafe { zero.assume_init() }; + /// + /// assert_eq!(*zero, 0) + /// ``` + /// + /// [zeroed]: mem::MaybeUninit::zeroed + #[unstable(feature = "allocator_api", issue = "32838")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[cfg(not(no_global_oom_handling))] + // #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] + pub const fn new_zeroed_in(alloc: A) -> Box, A> + where + A: ~const Allocator + ~const Destruct, + { + let layout = Layout::new::>(); + // NOTE: Prefer match over unwrap_or_else since closure sometimes not inlineable. + // That would make code size bigger. + match Box::try_new_zeroed_in(alloc) { + Ok(m) => m, + Err(_) => handle_alloc_error(layout), + } + } + + /// Constructs a new `Box` with uninitialized contents, with the memory + /// being filled with `0` bytes in the provided allocator, + /// returning an error if the allocation fails, + /// + /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage + /// of this method. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// + /// use std::alloc::System; + /// + /// let zero = Box::::try_new_zeroed_in(System)?; + /// let zero = unsafe { zero.assume_init() }; + /// + /// assert_eq!(*zero, 0); + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + /// + /// [zeroed]: mem::MaybeUninit::zeroed + #[unstable(feature = "allocator_api", issue = "32838")] + // #[unstable(feature = "new_uninit", issue = "63291")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + pub const fn try_new_zeroed_in(alloc: A) -> Result, A>, AllocError> + where + A: ~const Allocator + ~const Destruct, + { + let layout = Layout::new::>(); + let ptr = alloc.allocate_zeroed(layout)?.cast(); + unsafe { Ok(Box::from_raw_in(ptr.as_ptr(), alloc)) } + } + + /// Constructs a new `Pin>`. If `T` does not implement [`Unpin`], then + /// `x` will be pinned in memory and unable to be moved. + /// + /// Constructing and pinning of the `Box` can also be done in two steps: `Box::pin_in(x, alloc)` + /// does the same as [Box::into_pin]\([Box::new_in]\(x, alloc)). Consider using + /// [`into_pin`](Box::into_pin) if you already have a `Box`, or if you want to + /// construct a (pinned) `Box` in a different way than with [`Box::new_in`]. + #[cfg(not(no_global_oom_handling))] + #[unstable(feature = "allocator_api", issue = "32838")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[must_use] + #[inline(always)] + pub const fn pin_in(x: T, alloc: A) -> Pin + where + A: 'static + ~const Allocator + ~const Destruct, + { + Self::into_pin(Self::new_in(x, alloc)) + } + + /// Converts a `Box` into a `Box<[T]>` + /// + /// This conversion does not allocate on the heap and happens in place. + #[unstable(feature = "box_into_boxed_slice", issue = "71582")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + pub const fn into_boxed_slice(boxed: Self) -> Box<[T], A> { + let (raw, alloc) = Box::into_raw_with_allocator(boxed); + unsafe { Box::from_raw_in(raw as *mut [T; 1], alloc) } + } + + /// Consumes the `Box`, returning the wrapped value. + /// + /// # Examples + /// + /// ``` + /// #![feature(box_into_inner)] + /// + /// let c = Box::new(5); + /// + /// assert_eq!(Box::into_inner(c), 5); + /// ``` + #[unstable(feature = "box_into_inner", issue = "80437")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[inline] + pub const fn into_inner(boxed: Self) -> T + where + Self: ~const Destruct, + { + *boxed } } @@ -263,22 +637,166 @@ impl Box<[T]> { /// /// assert_eq!(*values, [1, 2, 3]) /// ``` + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] pub fn new_uninit_slice(len: usize) -> Box<[mem::MaybeUninit]> { - let layout = alloc::Layout::array::>(len).unwrap(); + unsafe { RawVec::with_capacity(len).into_box(len) } + } + + /// Constructs a new boxed slice with uninitialized contents, with the memory + /// being filled with `0` bytes. + /// + /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage + /// of this method. + /// + /// # Examples + /// + /// ``` + /// #![feature(new_uninit)] + /// + /// let values = Box::<[u32]>::new_zeroed_slice(3); + /// let values = unsafe { values.assume_init() }; + /// + /// assert_eq!(*values, [0, 0, 0]) + /// ``` + /// + /// [zeroed]: mem::MaybeUninit::zeroed + #[cfg(not(no_global_oom_handling))] + #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] + pub fn new_zeroed_slice(len: usize) -> Box<[mem::MaybeUninit]> { + unsafe { RawVec::with_capacity_zeroed(len).into_box(len) } + } + + /// Constructs a new boxed slice with uninitialized contents. Returns an error if + /// the allocation fails + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// + /// let mut values = Box::<[u32]>::try_new_uninit_slice(3)?; + /// let values = unsafe { + /// // Deferred initialization: + /// values[0].as_mut_ptr().write(1); + /// values[1].as_mut_ptr().write(2); + /// values[2].as_mut_ptr().write(3); + /// values.assume_init() + /// }; + /// + /// assert_eq!(*values, [1, 2, 3]); + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + #[unstable(feature = "allocator_api", issue = "32838")] + #[inline] + pub fn try_new_uninit_slice(len: usize) -> Result]>, AllocError> { + unsafe { + let layout = match Layout::array::>(len) { + Ok(l) => l, + Err(_) => return Err(AllocError), + }; + let ptr = Global.allocate(layout)?; + Ok(RawVec::from_raw_parts_in(ptr.as_mut_ptr() as *mut _, len, Global).into_box(len)) + } + } + + /// Constructs a new boxed slice with uninitialized contents, with the memory + /// being filled with `0` bytes. Returns an error if the allocation fails + /// + /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage + /// of this method. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// + /// let values = Box::<[u32]>::try_new_zeroed_slice(3)?; + /// let values = unsafe { values.assume_init() }; + /// + /// assert_eq!(*values, [0, 0, 0]); + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + /// + /// [zeroed]: mem::MaybeUninit::zeroed + #[unstable(feature = "allocator_api", issue = "32838")] + #[inline] + pub fn try_new_zeroed_slice(len: usize) -> Result]>, AllocError> { unsafe { - let ptr = if layout.size() == 0 { - NonNull::dangling() - } else { - Global.alloc(layout).unwrap_or_else(|_| alloc::handle_alloc_error(layout)).0.cast() + let layout = match Layout::array::>(len) { + Ok(l) => l, + Err(_) => return Err(AllocError), }; - Box::from_raw(slice::from_raw_parts_mut(ptr.as_ptr(), len)) + let ptr = Global.allocate_zeroed(layout)?; + Ok(RawVec::from_raw_parts_in(ptr.as_mut_ptr() as *mut _, len, Global).into_box(len)) } } } -impl Box> { - /// Converts to `Box`. +impl Box<[T], A> { + /// Constructs a new boxed slice with uninitialized contents in the provided allocator. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// + /// use std::alloc::System; + /// + /// let mut values = Box::<[u32], _>::new_uninit_slice_in(3, System); + /// + /// let values = unsafe { + /// // Deferred initialization: + /// values[0].as_mut_ptr().write(1); + /// values[1].as_mut_ptr().write(2); + /// values[2].as_mut_ptr().write(3); + /// + /// values.assume_init() + /// }; + /// + /// assert_eq!(*values, [1, 2, 3]) + /// ``` + #[cfg(not(no_global_oom_handling))] + #[unstable(feature = "allocator_api", issue = "32838")] + // #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] + pub fn new_uninit_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit], A> { + unsafe { RawVec::with_capacity_in(len, alloc).into_box(len) } + } + + /// Constructs a new boxed slice with uninitialized contents in the provided allocator, + /// with the memory being filled with `0` bytes. + /// + /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage + /// of this method. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// + /// use std::alloc::System; + /// + /// let values = Box::<[u32], _>::new_zeroed_slice_in(3, System); + /// let values = unsafe { values.assume_init() }; + /// + /// assert_eq!(*values, [0, 0, 0]) + /// ``` + /// + /// [zeroed]: mem::MaybeUninit::zeroed + #[cfg(not(no_global_oom_handling))] + #[unstable(feature = "allocator_api", issue = "32838")] + // #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] + pub fn new_zeroed_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit], A> { + unsafe { RawVec::with_capacity_zeroed_in(len, alloc).into_box(len) } + } +} + +impl Box, A> { + /// Converts to `Box`. /// /// # Safety /// @@ -288,7 +806,7 @@ impl Box> { /// Calling this when the content is not yet fully initialized /// causes immediate undefined behavior. /// - /// [`MaybeUninit::assume_init`]: ../../std/mem/union.MaybeUninit.html#method.assume_init + /// [`MaybeUninit::assume_init`]: mem::MaybeUninit::assume_init /// /// # Examples /// @@ -307,14 +825,53 @@ impl Box> { /// assert_eq!(*five, 5) /// ``` #[unstable(feature = "new_uninit", issue = "63291")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[inline] + pub const unsafe fn assume_init(self) -> Box { + let (raw, alloc) = Box::into_raw_with_allocator(self); + unsafe { Box::from_raw_in(raw as *mut T, alloc) } + } + + /// Writes the value and converts to `Box`. + /// + /// This method converts the box similarly to [`Box::assume_init`] but + /// writes `value` into it before conversion thus guaranteeing safety. + /// In some scenarios use of this method may improve performance because + /// the compiler may be able to optimize copying from stack. + /// + /// # Examples + /// + /// ``` + /// #![feature(new_uninit)] + /// + /// let big_box = Box::<[usize; 1024]>::new_uninit(); + /// + /// let mut array = [0; 1024]; + /// for (i, place) in array.iter_mut().enumerate() { + /// *place = i; + /// } + /// + /// // The optimizer may be able to elide this copy, so previous code writes + /// // to heap directly. + /// let big_box = Box::write(big_box, array); + /// + /// for (i, x) in big_box.iter().enumerate() { + /// assert_eq!(*x, i); + /// } + /// ``` + #[unstable(feature = "new_uninit", issue = "63291")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] #[inline] - pub unsafe fn assume_init(self) -> Box { - Box::from_raw(Box::into_raw(self) as *mut T) + pub const fn write(mut boxed: Self, value: T) -> Box { + unsafe { + (*boxed).write(value); + boxed.assume_init() + } } } -impl Box<[mem::MaybeUninit]> { - /// Converts to `Box<[T]>`. +impl Box<[mem::MaybeUninit], A> { + /// Converts to `Box<[T], A>`. /// /// # Safety /// @@ -324,7 +881,7 @@ impl Box<[mem::MaybeUninit]> { /// Calling this when the content is not yet fully initialized /// causes immediate undefined behavior. /// - /// [`MaybeUninit::assume_init`]: ../../std/mem/union.MaybeUninit.html#method.assume_init + /// [`MaybeUninit::assume_init`]: mem::MaybeUninit::assume_init /// /// # Examples /// @@ -346,8 +903,9 @@ impl Box<[mem::MaybeUninit]> { /// ``` #[unstable(feature = "new_uninit", issue = "63291")] #[inline] - pub unsafe fn assume_init(self) -> Box<[T]> { - Box::from_raw(Box::into_raw(self) as *mut [T]) + pub unsafe fn assume_init(self) -> Box<[T], A> { + let (raw, alloc) = Box::into_raw_with_allocator(self); + unsafe { Box::from_raw_in(raw as *mut [T], alloc) } } } @@ -366,7 +924,10 @@ impl Box { /// memory problems. For example, a double-free may occur if the /// function is called twice on the same raw pointer. /// + /// The safety conditions are described in the [memory layout] section. + /// /// # Examples + /// /// Recreate a `Box` which was previously converted to a raw pointer /// using [`Box::into_raw`]: /// ``` @@ -380,18 +941,77 @@ impl Box { /// /// unsafe { /// let ptr = alloc(Layout::new::()) as *mut i32; - /// *ptr = 5; + /// // In general .write is required to avoid attempting to destruct + /// // the (uninitialized) previous contents of `ptr`, though for this + /// // simple example `*ptr = 5` would have worked as well. + /// ptr.write(5); /// let x = Box::from_raw(ptr); /// } /// ``` /// - /// [memory layout]: index.html#memory-layout - /// [`Layout`]: ../alloc/struct.Layout.html - /// [`Box::into_raw`]: struct.Box.html#method.into_raw + /// [memory layout]: self#memory-layout + /// [`Layout`]: crate::Layout #[stable(feature = "box_raw", since = "1.4.0")] #[inline] + #[must_use = "call `drop(Box::from_raw(ptr))` if you intend to drop the `Box`"] pub unsafe fn from_raw(raw: *mut T) -> Self { - Box(Unique::new_unchecked(raw)) + unsafe { Self::from_raw_in(raw, Global) } + } +} + +impl Box { + /// Constructs a box from a raw pointer in the given allocator. + /// + /// After calling this function, the raw pointer is owned by the + /// resulting `Box`. Specifically, the `Box` destructor will call + /// the destructor of `T` and free the allocated memory. For this + /// to be safe, the memory must have been allocated in accordance + /// with the [memory layout] used by `Box` . + /// + /// # Safety + /// + /// This function is unsafe because improper use may lead to + /// memory problems. For example, a double-free may occur if the + /// function is called twice on the same raw pointer. + /// + /// + /// # Examples + /// + /// Recreate a `Box` which was previously converted to a raw pointer + /// using [`Box::into_raw_with_allocator`]: + /// ``` + /// #![feature(allocator_api)] + /// + /// use std::alloc::System; + /// + /// let x = Box::new_in(5, System); + /// let (ptr, alloc) = Box::into_raw_with_allocator(x); + /// let x = unsafe { Box::from_raw_in(ptr, alloc) }; + /// ``` + /// Manually create a `Box` from scratch by using the system allocator: + /// ``` + /// #![feature(allocator_api, slice_ptr_get)] + /// + /// use std::alloc::{Allocator, Layout, System}; + /// + /// unsafe { + /// let ptr = System.allocate(Layout::new::())?.as_mut_ptr() as *mut i32; + /// // In general .write is required to avoid attempting to destruct + /// // the (uninitialized) previous contents of `ptr`, though for this + /// // simple example `*ptr = 5` would have worked as well. + /// ptr.write(5); + /// let x = Box::from_raw_in(ptr, System); + /// } + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + /// + /// [memory layout]: self#memory-layout + /// [`Layout`]: crate::Layout + #[unstable(feature = "allocator_api", issue = "32838")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[inline] + pub const unsafe fn from_raw_in(raw: *mut T, alloc: A) -> Self { + Box(unsafe { Unique::new_unchecked(raw) }, alloc) } /// Consumes the `Box`, returning a wrapped raw pointer. @@ -432,61 +1052,95 @@ impl Box { /// } /// ``` /// - /// [memory layout]: index.html#memory-layout - /// [`Box::from_raw`]: struct.Box.html#method.from_raw + /// [memory layout]: self#memory-layout #[stable(feature = "box_raw", since = "1.4.0")] #[inline] - pub fn into_raw(b: Box) -> *mut T { - Box::into_raw_non_null(b).as_ptr() + pub fn into_raw(b: Self) -> *mut T { + Self::into_raw_with_allocator(b).0 } - /// Consumes the `Box`, returning the wrapped pointer as `NonNull`. + /// Consumes the `Box`, returning a wrapped raw pointer and the allocator. + /// + /// The pointer will be properly aligned and non-null. /// /// After calling this function, the caller is responsible for the /// memory previously managed by the `Box`. In particular, the - /// caller should properly destroy `T` and release the memory. The - /// easiest way to do so is to convert the `NonNull` pointer - /// into a raw pointer and back into a `Box` with the [`Box::from_raw`] - /// function. + /// caller should properly destroy `T` and release the memory, taking + /// into account the [memory layout] used by `Box`. The easiest way to + /// do this is to convert the raw pointer back into a `Box` with the + /// [`Box::from_raw_in`] function, allowing the `Box` destructor to perform + /// the cleanup. /// /// Note: this is an associated function, which means that you have - /// to call it as `Box::into_raw_non_null(b)` - /// instead of `b.into_raw_non_null()`. This + /// to call it as `Box::into_raw_with_allocator(b)` instead of `b.into_raw_with_allocator()`. This /// is so that there is no conflict with a method on the inner type. /// - /// [`Box::from_raw`]: struct.Box.html#method.from_raw - /// /// # Examples + /// Converting the raw pointer back into a `Box` with [`Box::from_raw_in`] + /// for automatic cleanup: + /// ``` + /// #![feature(allocator_api)] /// + /// use std::alloc::System; + /// + /// let x = Box::new_in(String::from("Hello"), System); + /// let (ptr, alloc) = Box::into_raw_with_allocator(x); + /// let x = unsafe { Box::from_raw_in(ptr, alloc) }; + /// ``` + /// Manual cleanup by explicitly running the destructor and deallocating + /// the memory: /// ``` - /// #![feature(box_into_raw_non_null)] + /// #![feature(allocator_api)] /// - /// let x = Box::new(5); - /// let ptr = Box::into_raw_non_null(x); + /// use std::alloc::{Allocator, Layout, System}; + /// use std::ptr::{self, NonNull}; /// - /// // Clean up the memory by converting the NonNull pointer back - /// // into a Box and letting the Box be dropped. - /// let x = unsafe { Box::from_raw(ptr.as_ptr()) }; + /// let x = Box::new_in(String::from("Hello"), System); + /// let (ptr, alloc) = Box::into_raw_with_allocator(x); + /// unsafe { + /// ptr::drop_in_place(ptr); + /// let non_null = NonNull::new_unchecked(ptr); + /// alloc.deallocate(non_null.cast(), Layout::new::()); + /// } /// ``` - #[unstable(feature = "box_into_raw_non_null", issue = "47336")] + /// + /// [memory layout]: self#memory-layout + #[unstable(feature = "allocator_api", issue = "32838")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] #[inline] - pub fn into_raw_non_null(b: Box) -> NonNull { - Box::into_unique(b).into() + pub const fn into_raw_with_allocator(b: Self) -> (*mut T, A) { + let (leaked, alloc) = Box::into_unique(b); + (leaked.as_ptr(), alloc) } - #[unstable(feature = "ptr_internals", issue = "none", reason = "use into_raw_non_null instead")] + #[unstable( + feature = "ptr_internals", + issue = "none", + reason = "use `Box::leak(b).into()` or `Unique::from(Box::leak(b))` instead" + )] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] #[inline] #[doc(hidden)] - pub fn into_unique(b: Box) -> Unique { - let mut unique = b.0; - mem::forget(b); - // Box is kind-of a library type, but recognized as a "unique pointer" by - // Stacked Borrows. This function here corresponds to "reborrowing to - // a raw pointer", but there is no actual reborrow here -- so - // without some care, the pointer we are returning here still carries - // the tag of `b`, with `Unique` permission. - // We round-trip through a mutable reference to avoid that. - unsafe { Unique::new_unchecked(unique.as_mut() as *mut T) } + pub const fn into_unique(b: Self) -> (Unique, A) { + // Box is recognized as a "unique pointer" by Stacked Borrows, but internally it is a + // raw pointer for the type system. Turning it directly into a raw pointer would not be + // recognized as "releasing" the unique pointer to permit aliased raw accesses, + // so all raw pointer methods have to go through `Box::leak`. Turning *that* to a raw pointer + // behaves correctly. + let alloc = unsafe { ptr::read(&b.1) }; + (Unique::from(Box::leak(b)), alloc) + } + + /// Returns a reference to the underlying allocator. + /// + /// Note: this is an associated function, which means that you have + /// to call it as `Box::allocator(&b)` instead of `b.allocator()`. This + /// is so that there is no conflict with a method on the inner type. + #[unstable(feature = "allocator_api", issue = "32838")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[inline] + pub const fn allocator(b: &Self) -> &A { + &b.1 } /// Consumes and leaks the `Box`, returning a mutable reference, @@ -505,8 +1159,6 @@ impl Box { /// to call it as `Box::leak(b)` instead of `b.leak()`. This /// is so that there is no conflict with a method on the inner type. /// - /// [`Box::from_raw`]: struct.Box.html#method.from_raw - /// /// # Examples /// /// Simple usage: @@ -527,59 +1179,102 @@ impl Box { /// assert_eq!(*static_ref, [4, 2, 3]); /// ``` #[stable(feature = "box_leak", since = "1.26.0")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] #[inline] - pub fn leak<'a>(b: Box) -> &'a mut T + pub const fn leak<'a>(b: Self) -> &'a mut T where - T: 'a, // Technically not needed, but kept to be explicit. + A: 'a, { - unsafe { &mut *Box::into_raw(b) } + unsafe { &mut *mem::ManuallyDrop::new(b).0.as_ptr() } } - /// Converts a `Box` into a `Pin>` + /// Converts a `Box` into a `Pin>`. If `T` does not implement [`Unpin`], then + /// `*boxed` will be pinned in memory and unable to be moved. /// /// This conversion does not allocate on the heap and happens in place. /// /// This is also available via [`From`]. - #[unstable(feature = "box_into_pin", issue = "62370")] - pub fn into_pin(boxed: Box) -> Pin> { + /// + /// Constructing and pinning a `Box` with Box::into_pin([Box::new]\(x)) + /// can also be written more concisely using [Box::pin]\(x). + /// This `into_pin` method is useful if you already have a `Box`, or you are + /// constructing a (pinned) `Box` in a different way than with [`Box::new`]. + /// + /// # Notes + /// + /// It's not recommended that crates add an impl like `From> for Pin`, + /// as it'll introduce an ambiguity when calling `Pin::from`. + /// A demonstration of such a poor impl is shown below. + /// + /// ```compile_fail + /// # use std::pin::Pin; + /// struct Foo; // A type defined in this crate. + /// impl From> for Pin { + /// fn from(_: Box<()>) -> Pin { + /// Pin::new(Foo) + /// } + /// } + /// + /// let foo = Box::new(()); + /// let bar = Pin::from(foo); + /// ``` + #[stable(feature = "box_into_pin", since = "1.63.0")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + pub const fn into_pin(boxed: Self) -> Pin + where + A: 'static, + { // It's not possible to move or replace the insides of a `Pin>` - // when `T: !Unpin`, so it's safe to pin it directly without any + // when `T: !Unpin`, so it's safe to pin it directly without any // additional requirements. unsafe { Pin::new_unchecked(boxed) } } } #[stable(feature = "rust1", since = "1.0.0")] -unsafe impl<#[may_dangle] T: ?Sized> Drop for Box { +unsafe impl<#[may_dangle] T: ?Sized, A: Allocator> Drop for Box { fn drop(&mut self) { // FIXME: Do nothing, drop is currently performed by compiler. } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl Default for Box { /// Creates a `Box`, with the `Default` value for T. - fn default() -> Box { - box Default::default() + fn default() -> Self { + #[rustc_box] + Box::new(T::default()) } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] -impl Default for Box<[T]> { - fn default() -> Box<[T]> { - Box::<[T; 0]>::new([]) +#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] +impl const Default for Box<[T]> { + fn default() -> Self { + let ptr: Unique<[T]> = Unique::<[T; 0]>::dangling(); + Box(ptr, Global) } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "default_box_extra", since = "1.17.0")] -impl Default for Box { - fn default() -> Box { - unsafe { from_boxed_utf8_unchecked(Default::default()) } +#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] +impl const Default for Box { + fn default() -> Self { + // SAFETY: This is the same as `Unique::cast` but with an unsized `U = str`. + let ptr: Unique = unsafe { + let bytes: Unique<[u8]> = Unique::<[u8; 0]>::dangling(); + Unique::new_unchecked(bytes.as_ptr() as *mut str) + }; + Box(ptr, Global) } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] -impl Clone for Box { +impl Clone for Box { /// Returns a new box with a `clone()` of this box's contents. /// /// # Examples @@ -594,10 +1289,14 @@ impl Clone for Box { /// // But they are unique objects /// assert_ne!(&*x as *const i32, &*y as *const i32); /// ``` - #[rustfmt::skip] #[inline] - fn clone(&self) -> Box { - box { (**self).clone() } + fn clone(&self) -> Self { + // Pre-allocate memory to allow writing the cloned value directly. + let mut boxed = Self::new_uninit_in(self.1.clone()); + unsafe { + (**self).write_clone_into_raw(boxed.as_mut_ptr()); + boxed.assume_init() + } } /// Copies `source`'s contents into `self` without creating a new allocation. @@ -618,11 +1317,12 @@ impl Clone for Box { /// assert_eq!(yp, &*y); /// ``` #[inline] - fn clone_from(&mut self, source: &Box) { + fn clone_from(&mut self, source: &Self) { (**self).clone_from(&(**source)); } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "box_slice_clone", since = "1.3.0")] impl Clone for Box { fn clone(&self) -> Self { @@ -633,58 +1333,58 @@ impl Clone for Box { } #[stable(feature = "rust1", since = "1.0.0")] -impl PartialEq for Box { +impl PartialEq for Box { #[inline] - fn eq(&self, other: &Box) -> bool { + fn eq(&self, other: &Self) -> bool { PartialEq::eq(&**self, &**other) } #[inline] - fn ne(&self, other: &Box) -> bool { + fn ne(&self, other: &Self) -> bool { PartialEq::ne(&**self, &**other) } } #[stable(feature = "rust1", since = "1.0.0")] -impl PartialOrd for Box { +impl PartialOrd for Box { #[inline] - fn partial_cmp(&self, other: &Box) -> Option { + fn partial_cmp(&self, other: &Self) -> Option { PartialOrd::partial_cmp(&**self, &**other) } #[inline] - fn lt(&self, other: &Box) -> bool { + fn lt(&self, other: &Self) -> bool { PartialOrd::lt(&**self, &**other) } #[inline] - fn le(&self, other: &Box) -> bool { + fn le(&self, other: &Self) -> bool { PartialOrd::le(&**self, &**other) } #[inline] - fn ge(&self, other: &Box) -> bool { + fn ge(&self, other: &Self) -> bool { PartialOrd::ge(&**self, &**other) } #[inline] - fn gt(&self, other: &Box) -> bool { + fn gt(&self, other: &Self) -> bool { PartialOrd::gt(&**self, &**other) } } #[stable(feature = "rust1", since = "1.0.0")] -impl Ord for Box { +impl Ord for Box { #[inline] - fn cmp(&self, other: &Box) -> Ordering { + fn cmp(&self, other: &Self) -> Ordering { Ord::cmp(&**self, &**other) } } #[stable(feature = "rust1", since = "1.0.0")] -impl Eq for Box {} +impl Eq for Box {} #[stable(feature = "rust1", since = "1.0.0")] -impl Hash for Box { +impl Hash for Box { fn hash(&self, state: &mut H) { (**self).hash(state); } } #[stable(feature = "indirect_hasher_impl", since = "1.22.0")] -impl Hasher for Box { +impl Hasher for Box { fn finish(&self) -> u64 { (**self).finish() } @@ -727,16 +1427,24 @@ impl Hasher for Box { fn write_isize(&mut self, i: isize) { (**self).write_isize(i) } + fn write_length_prefix(&mut self, len: usize) { + (**self).write_length_prefix(len) + } + fn write_str(&mut self, s: &str) { + (**self).write_str(s) + } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "from_for_ptrs", since = "1.6.0")] impl From for Box { - /// Converts a generic type `T` into a `Box` + /// Converts a `T` into a `Box` /// /// The conversion allocates on the heap and moves `t` /// from the stack into it. /// /// # Examples + /// /// ```rust /// let x = 5; /// let boxed = Box::new(5); @@ -749,21 +1457,34 @@ impl From for Box { } #[stable(feature = "pin", since = "1.33.0")] -impl From> for Pin> { - /// Converts a `Box` into a `Pin>` +#[rustc_const_unstable(feature = "const_box", issue = "92521")] +impl const From> for Pin> +where + A: 'static, +{ + /// Converts a `Box` into a `Pin>`. If `T` does not implement [`Unpin`], then + /// `*boxed` will be pinned in memory and unable to be moved. /// /// This conversion does not allocate on the heap and happens in place. - fn from(boxed: Box) -> Self { + /// + /// This is also available via [`Box::into_pin`]. + /// + /// Constructing and pinning a `Box` with >>::from([Box::new]\(x)) + /// can also be written more concisely using [Box::pin]\(x). + /// This `From` implementation is useful if you already have a `Box`, or you are + /// constructing a (pinned) `Box` in a different way than with [`Box::new`]. + fn from(boxed: Box) -> Self { Box::into_pin(boxed) } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "box_from_slice", since = "1.17.0")] impl From<&[T]> for Box<[T]> { /// Converts a `&[T]` into a `Box<[T]>` /// /// This conversion allocates on the heap - /// and performs a copy of `slice`. + /// and performs a copy of `slice` and its contents. /// /// # Examples /// ```rust @@ -771,79 +1492,266 @@ impl From<&[T]> for Box<[T]> { /// let slice: &[u8] = &[104, 101, 108, 108, 111]; /// let boxed_slice: Box<[u8]> = Box::from(slice); /// - /// println!("{:?}", boxed_slice); + /// println!("{boxed_slice:?}"); /// ``` fn from(slice: &[T]) -> Box<[T]> { let len = slice.len(); let buf = RawVec::with_capacity(len); unsafe { ptr::copy_nonoverlapping(slice.as_ptr(), buf.ptr(), len); - buf.into_box() + buf.into_box(slice.len()).assume_init() + } + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "box_from_cow", since = "1.45.0")] +impl From> for Box<[T]> { + /// Converts a `Cow<'_, [T]>` into a `Box<[T]>` + /// + /// When `cow` is the `Cow::Borrowed` variant, this + /// conversion allocates on the heap and copies the + /// underlying slice. Otherwise, it will try to reuse the owned + /// `Vec`'s allocation. + #[inline] + fn from(cow: Cow<'_, [T]>) -> Box<[T]> { + match cow { + Cow::Borrowed(slice) => Box::from(slice), + Cow::Owned(slice) => Box::from(slice), } } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "box_from_slice", since = "1.17.0")] impl From<&str> for Box { /// Converts a `&str` into a `Box` /// - /// This conversion allocates on the heap - /// and performs a copy of `s`. + /// This conversion allocates on the heap + /// and performs a copy of `s`. + /// + /// # Examples + /// + /// ```rust + /// let boxed: Box = Box::from("hello"); + /// println!("{boxed}"); + /// ``` + #[inline] + fn from(s: &str) -> Box { + unsafe { from_boxed_utf8_unchecked(Box::from(s.as_bytes())) } + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "box_from_cow", since = "1.45.0")] +impl From> for Box { + /// Converts a `Cow<'_, str>` into a `Box` + /// + /// When `cow` is the `Cow::Borrowed` variant, this + /// conversion allocates on the heap and copies the + /// underlying `str`. Otherwise, it will try to reuse the owned + /// `String`'s allocation. + /// + /// # Examples + /// + /// ```rust + /// use std::borrow::Cow; + /// + /// let unboxed = Cow::Borrowed("hello"); + /// let boxed: Box = Box::from(unboxed); + /// println!("{boxed}"); + /// ``` + /// + /// ```rust + /// # use std::borrow::Cow; + /// let unboxed = Cow::Owned("hello".to_string()); + /// let boxed: Box = Box::from(unboxed); + /// println!("{boxed}"); + /// ``` + #[inline] + fn from(cow: Cow<'_, str>) -> Box { + match cow { + Cow::Borrowed(s) => Box::from(s), + Cow::Owned(s) => Box::from(s), + } + } +} + +#[stable(feature = "boxed_str_conv", since = "1.19.0")] +impl From> for Box<[u8], A> { + /// Converts a `Box` into a `Box<[u8]>` + /// + /// This conversion does not allocate on the heap and happens in place. + /// + /// # Examples + /// ```rust + /// // create a Box which will be used to create a Box<[u8]> + /// let boxed: Box = Box::from("hello"); + /// let boxed_str: Box<[u8]> = Box::from(boxed); + /// + /// // create a &[u8] which will be used to create a Box<[u8]> + /// let slice: &[u8] = &[104, 101, 108, 108, 111]; + /// let boxed_slice = Box::from(slice); + /// + /// assert_eq!(boxed_slice, boxed_str); + /// ``` + #[inline] + fn from(s: Box) -> Self { + let (raw, alloc) = Box::into_raw_with_allocator(s); + unsafe { Box::from_raw_in(raw as *mut [u8], alloc) } + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "box_from_array", since = "1.45.0")] +impl From<[T; N]> for Box<[T]> { + /// Converts a `[T; N]` into a `Box<[T]>` + /// + /// This conversion moves the array to newly heap-allocated memory. + /// + /// # Examples + /// + /// ```rust + /// let boxed: Box<[u8]> = Box::from([4, 2]); + /// println!("{boxed:?}"); + /// ``` + fn from(array: [T; N]) -> Box<[T]> { + #[rustc_box] + Box::new(array) + } +} + +/// Casts a boxed slice to a boxed array. +/// +/// # Safety +/// +/// `boxed_slice.len()` must be exactly `N`. +unsafe fn boxed_slice_as_array_unchecked( + boxed_slice: Box<[T], A>, +) -> Box<[T; N], A> { + debug_assert_eq!(boxed_slice.len(), N); + + let (ptr, alloc) = Box::into_raw_with_allocator(boxed_slice); + // SAFETY: Pointer and allocator came from an existing box, + // and our safety condition requires that the length is exactly `N` + unsafe { Box::from_raw_in(ptr as *mut [T; N], alloc) } +} + +#[stable(feature = "boxed_slice_try_from", since = "1.43.0")] +impl TryFrom> for Box<[T; N]> { + type Error = Box<[T]>; + + /// Attempts to convert a `Box<[T]>` into a `Box<[T; N]>`. + /// + /// The conversion occurs in-place and does not require a + /// new memory allocation. + /// + /// # Errors + /// + /// Returns the old `Box<[T]>` in the `Err` variant if + /// `boxed_slice.len()` does not equal `N`. + fn try_from(boxed_slice: Box<[T]>) -> Result { + if boxed_slice.len() == N { + Ok(unsafe { boxed_slice_as_array_unchecked(boxed_slice) }) + } else { + Err(boxed_slice) + } + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "boxed_array_try_from_vec", since = "1.66.0")] +impl TryFrom> for Box<[T; N]> { + type Error = Vec; + + /// Attempts to convert a `Vec` into a `Box<[T; N]>`. + /// + /// Like [`Vec::into_boxed_slice`], this is in-place if `vec.capacity() == N`, + /// but will require a reallocation otherwise. + /// + /// # Errors + /// + /// Returns the original `Vec` in the `Err` variant if + /// `boxed_slice.len()` does not equal `N`. + /// + /// # Examples + /// + /// This can be used with [`vec!`] to create an array on the heap: + /// + /// ``` + /// let state: Box<[f32; 100]> = vec![1.0; 100].try_into().unwrap(); + /// assert_eq!(state.len(), 100); + /// ``` + fn try_from(vec: Vec) -> Result { + if vec.len() == N { + let boxed_slice = vec.into_boxed_slice(); + Ok(unsafe { boxed_slice_as_array_unchecked(boxed_slice) }) + } else { + Err(vec) + } + } +} + +impl Box { + /// Attempt to downcast the box to a concrete type. + /// + /// # Examples + /// + /// ``` + /// use std::any::Any; + /// + /// fn print_if_string(value: Box) { + /// if let Ok(string) = value.downcast::() { + /// println!("String ({}): {}", string.len(), string); + /// } + /// } /// - /// # Examples - /// ```rust - /// let boxed: Box = Box::from("hello"); - /// println!("{}", boxed); + /// let my_string = "Hello World".to_string(); + /// print_if_string(Box::new(my_string)); + /// print_if_string(Box::new(0i8)); /// ``` #[inline] - fn from(s: &str) -> Box { - unsafe { from_boxed_utf8_unchecked(Box::from(s.as_bytes())) } + #[stable(feature = "rust1", since = "1.0.0")] + pub fn downcast(self) -> Result, Self> { + if self.is::() { unsafe { Ok(self.downcast_unchecked::()) } } else { Err(self) } } -} -#[stable(feature = "boxed_str_conv", since = "1.19.0")] -impl From> for Box<[u8]> { - /// Converts a `Box>` into a `Box<[u8]>` + /// Downcasts the box to a concrete type. /// - /// This conversion does not allocate on the heap and happens in place. + /// For a safe alternative see [`downcast`]. /// /// # Examples - /// ```rust - /// // create a Box which will be used to create a Box<[u8]> - /// let boxed: Box = Box::from("hello"); - /// let boxed_str: Box<[u8]> = Box::from(boxed); /// - /// // create a &[u8] which will be used to create a Box<[u8]> - /// let slice: &[u8] = &[104, 101, 108, 108, 111]; - /// let boxed_slice = Box::from(slice); + /// ``` + /// #![feature(downcast_unchecked)] /// - /// assert_eq!(boxed_slice, boxed_str); + /// use std::any::Any; + /// + /// let x: Box = Box::new(1_usize); + /// + /// unsafe { + /// assert_eq!(*x.downcast_unchecked::(), 1); + /// } /// ``` + /// + /// # Safety + /// + /// The contained value must be of type `T`. Calling this method + /// with the incorrect type is *undefined behavior*. + /// + /// [`downcast`]: Self::downcast #[inline] - fn from(s: Box) -> Self { - unsafe { Box::from_raw(Box::into_raw(s) as *mut [u8]) } - } -} - -#[stable(feature = "boxed_slice_try_from", since = "1.43.0")] -impl TryFrom> for Box<[T; N]> -where - [T; N]: LengthAtMost32, -{ - type Error = Box<[T]>; - - fn try_from(boxed_slice: Box<[T]>) -> Result { - if boxed_slice.len() == N { - Ok(unsafe { Box::from_raw(Box::into_raw(boxed_slice) as *mut [T; N]) }) - } else { - Err(boxed_slice) + #[unstable(feature = "downcast_unchecked", issue = "90850")] + pub unsafe fn downcast_unchecked(self) -> Box { + debug_assert!(self.is::()); + unsafe { + let (raw, alloc): (*mut dyn Any, _) = Box::into_raw_with_allocator(self); + Box::from_raw_in(raw as *mut T, alloc) } } } -impl Box { - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] +impl Box { /// Attempt to downcast the box to a concrete type. /// /// # Examples @@ -851,7 +1759,7 @@ impl Box { /// ``` /// use std::any::Any; /// - /// fn print_if_string(value: Box) { + /// fn print_if_string(value: Box) { /// if let Ok(string) = value.downcast::() { /// println!("String ({}): {}", string.len(), string); /// } @@ -861,21 +1769,48 @@ impl Box { /// print_if_string(Box::new(my_string)); /// print_if_string(Box::new(0i8)); /// ``` - pub fn downcast(self) -> Result, Box> { - if self.is::() { - unsafe { - let raw: *mut dyn Any = Box::into_raw(self); - Ok(Box::from_raw(raw as *mut T)) - } - } else { - Err(self) + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn downcast(self) -> Result, Self> { + if self.is::() { unsafe { Ok(self.downcast_unchecked::()) } } else { Err(self) } + } + + /// Downcasts the box to a concrete type. + /// + /// For a safe alternative see [`downcast`]. + /// + /// # Examples + /// + /// ``` + /// #![feature(downcast_unchecked)] + /// + /// use std::any::Any; + /// + /// let x: Box = Box::new(1_usize); + /// + /// unsafe { + /// assert_eq!(*x.downcast_unchecked::(), 1); + /// } + /// ``` + /// + /// # Safety + /// + /// The contained value must be of type `T`. Calling this method + /// with the incorrect type is *undefined behavior*. + /// + /// [`downcast`]: Self::downcast + #[inline] + #[unstable(feature = "downcast_unchecked", issue = "90850")] + pub unsafe fn downcast_unchecked(self) -> Box { + debug_assert!(self.is::()); + unsafe { + let (raw, alloc): (*mut (dyn Any + Send), _) = Box::into_raw_with_allocator(self); + Box::from_raw_in(raw as *mut T, alloc) } } } -impl Box { - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] +impl Box { /// Attempt to downcast the box to a concrete type. /// /// # Examples @@ -883,7 +1818,7 @@ impl Box { /// ``` /// use std::any::Any; /// - /// fn print_if_string(value: Box) { + /// fn print_if_string(value: Box) { /// if let Ok(string) = value.downcast::() { /// println!("String ({}): {}", string.len(), string); /// } @@ -893,30 +1828,64 @@ impl Box { /// print_if_string(Box::new(my_string)); /// print_if_string(Box::new(0i8)); /// ``` - pub fn downcast(self) -> Result, Box> { - >::downcast(self).map_err(|s| unsafe { - // reapply the Send marker - Box::from_raw(Box::into_raw(s) as *mut (dyn Any + Send)) - }) + #[inline] + #[stable(feature = "box_send_sync_any_downcast", since = "1.51.0")] + pub fn downcast(self) -> Result, Self> { + if self.is::() { unsafe { Ok(self.downcast_unchecked::()) } } else { Err(self) } + } + + /// Downcasts the box to a concrete type. + /// + /// For a safe alternative see [`downcast`]. + /// + /// # Examples + /// + /// ``` + /// #![feature(downcast_unchecked)] + /// + /// use std::any::Any; + /// + /// let x: Box = Box::new(1_usize); + /// + /// unsafe { + /// assert_eq!(*x.downcast_unchecked::(), 1); + /// } + /// ``` + /// + /// # Safety + /// + /// The contained value must be of type `T`. Calling this method + /// with the incorrect type is *undefined behavior*. + /// + /// [`downcast`]: Self::downcast + #[inline] + #[unstable(feature = "downcast_unchecked", issue = "90850")] + pub unsafe fn downcast_unchecked(self) -> Box { + debug_assert!(self.is::()); + unsafe { + let (raw, alloc): (*mut (dyn Any + Send + Sync), _) = + Box::into_raw_with_allocator(self); + Box::from_raw_in(raw as *mut T, alloc) + } } } #[stable(feature = "rust1", since = "1.0.0")] -impl fmt::Display for Box { +impl fmt::Display for Box { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&**self, f) } } #[stable(feature = "rust1", since = "1.0.0")] -impl fmt::Debug for Box { +impl fmt::Debug for Box { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Debug::fmt(&**self, f) } } #[stable(feature = "rust1", since = "1.0.0")] -impl fmt::Pointer for Box { +impl fmt::Pointer for Box { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // It's not possible to extract the inner Uniq directly from the Box, // instead we cast it to a *const which aliases the Unique @@ -926,7 +1895,8 @@ impl fmt::Pointer for Box { } #[stable(feature = "rust1", since = "1.0.0")] -impl Deref for Box { +#[rustc_const_unstable(feature = "const_box", issue = "92521")] +impl const Deref for Box { type Target = T; fn deref(&self) -> &T { @@ -935,17 +1905,18 @@ impl Deref for Box { } #[stable(feature = "rust1", since = "1.0.0")] -impl DerefMut for Box { +#[rustc_const_unstable(feature = "const_box", issue = "92521")] +impl const DerefMut for Box { fn deref_mut(&mut self) -> &mut T { &mut **self } } #[unstable(feature = "receiver_trait", issue = "none")] -impl Receiver for Box {} +impl Receiver for Box {} #[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for Box { +impl Iterator for Box { type Item = I::Item; fn next(&mut self) -> Option { (**self).next() @@ -966,7 +1937,7 @@ trait BoxIter { fn last(self) -> Option; } -impl BoxIter for Box { +impl BoxIter for Box { type Item = I::Item; default fn last(self) -> Option { #[inline] @@ -981,14 +1952,14 @@ impl BoxIter for Box { /// Specialization for sized `I`s that uses `I`s implementation of `last()` /// instead of the default. #[stable(feature = "rust1", since = "1.0.0")] -impl BoxIter for Box { +impl BoxIter for Box { fn last(self) -> Option { (*self).last() } } #[stable(feature = "rust1", since = "1.0.0")] -impl DoubleEndedIterator for Box { +impl DoubleEndedIterator for Box { fn next_back(&mut self) -> Option { (**self).next_back() } @@ -997,7 +1968,7 @@ impl DoubleEndedIterator for Box { } } #[stable(feature = "rust1", since = "1.0.0")] -impl ExactSizeIterator for Box { +impl ExactSizeIterator for Box { fn len(&self) -> usize { (**self).len() } @@ -1007,74 +1978,85 @@ impl ExactSizeIterator for Box { } #[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Box {} +impl FusedIterator for Box {} #[stable(feature = "boxed_closure_impls", since = "1.35.0")] -impl + ?Sized> FnOnce for Box { - type Output = >::Output; +impl + ?Sized, A: Allocator> FnOnce for Box { + type Output = >::Output; - extern "rust-call" fn call_once(self, args: A) -> Self::Output { - >::call_once(*self, args) + extern "rust-call" fn call_once(self, args: Args) -> Self::Output { + >::call_once(*self, args) } } #[stable(feature = "boxed_closure_impls", since = "1.35.0")] -impl + ?Sized> FnMut for Box { - extern "rust-call" fn call_mut(&mut self, args: A) -> Self::Output { - >::call_mut(self, args) +impl + ?Sized, A: Allocator> FnMut for Box { + extern "rust-call" fn call_mut(&mut self, args: Args) -> Self::Output { + >::call_mut(self, args) } } #[stable(feature = "boxed_closure_impls", since = "1.35.0")] -impl + ?Sized> Fn for Box { - extern "rust-call" fn call(&self, args: A) -> Self::Output { - >::call(self, args) +impl + ?Sized, A: Allocator> Fn for Box { + extern "rust-call" fn call(&self, args: Args) -> Self::Output { + >::call(self, args) } } -#[unstable(feature = "coerce_unsized", issue = "27732")] -impl, U: ?Sized> CoerceUnsized> for Box {} +#[unstable(feature = "coerce_unsized", issue = "18598")] +impl, U: ?Sized, A: Allocator> CoerceUnsized> for Box {} #[unstable(feature = "dispatch_from_dyn", issue = "none")] -impl, U: ?Sized> DispatchFromDyn> for Box {} +impl, U: ?Sized> DispatchFromDyn> for Box {} +#[cfg(not(no_global_oom_handling))] #[stable(feature = "boxed_slice_from_iter", since = "1.32.0")] -impl FromIterator for Box<[A]> { - fn from_iter>(iter: T) -> Self { +impl FromIterator for Box<[I]> { + fn from_iter>(iter: T) -> Self { iter.into_iter().collect::>().into_boxed_slice() } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "box_slice_clone", since = "1.3.0")] -impl Clone for Box<[T]> { +impl Clone for Box<[T], A> { fn clone(&self) -> Self { - self.to_vec().into_boxed_slice() + let alloc = Box::allocator(self).clone(); + self.to_vec_in(alloc).into_boxed_slice() + } + + fn clone_from(&mut self, other: &Self) { + if self.len() == other.len() { + self.clone_from_slice(&other); + } else { + *self = other.clone(); + } } } #[stable(feature = "box_borrow", since = "1.1.0")] -impl borrow::Borrow for Box { +impl borrow::Borrow for Box { fn borrow(&self) -> &T { &**self } } #[stable(feature = "box_borrow", since = "1.1.0")] -impl borrow::BorrowMut for Box { +impl borrow::BorrowMut for Box { fn borrow_mut(&mut self) -> &mut T { &mut **self } } #[stable(since = "1.5.0", feature = "smart_ptr_as_ref")] -impl AsRef for Box { +impl AsRef for Box { fn as_ref(&self) -> &T { &**self } } #[stable(since = "1.5.0", feature = "smart_ptr_as_ref")] -impl AsMut for Box { +impl AsMut for Box { fn as_mut(&mut self) -> &mut T { &mut **self } @@ -1103,10 +2085,13 @@ impl AsMut for Box { * could have a method to project a Pin from it. */ #[stable(feature = "pin", since = "1.33.0")] -impl Unpin for Box {} +impl Unpin for Box where A: 'static {} #[unstable(feature = "generator_trait", issue = "43122")] -impl + Unpin, R> Generator for Box { +impl + Unpin, R, A: Allocator> Generator for Box +where + A: 'static, +{ type Yield = G::Yield; type Return = G::Return; @@ -1116,7 +2101,10 @@ impl + Unpin, R> Generator for Box { } #[unstable(feature = "generator_trait", issue = "43122")] -impl, R> Generator for Pin> { +impl, R, A: Allocator> Generator for Pin> +where + A: 'static, +{ type Yield = G::Yield; type Return = G::Return; @@ -1126,10 +2114,315 @@ impl, R> Generator for Pin> { } #[stable(feature = "futures_api", since = "1.36.0")] -impl Future for Box { +impl Future for Box +where + A: 'static, +{ type Output = F::Output; fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { F::poll(Pin::new(&mut *self), cx) } } + +#[unstable(feature = "async_iterator", issue = "79024")] +impl AsyncIterator for Box { + type Item = S::Item; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut **self).poll_next(cx) + } + + fn size_hint(&self) -> (usize, Option) { + (**self).size_hint() + } +} + +impl dyn Error { + #[inline] + #[stable(feature = "error_downcast", since = "1.3.0")] + #[rustc_allow_incoherent_impl] + /// Attempts to downcast the box to a concrete type. + pub fn downcast(self: Box) -> Result, Box> { + if self.is::() { + unsafe { + let raw: *mut dyn Error = Box::into_raw(self); + Ok(Box::from_raw(raw as *mut T)) + } + } else { + Err(self) + } + } +} + +impl dyn Error + Send { + #[inline] + #[stable(feature = "error_downcast", since = "1.3.0")] + #[rustc_allow_incoherent_impl] + /// Attempts to downcast the box to a concrete type. + pub fn downcast(self: Box) -> Result, Box> { + let err: Box = self; + ::downcast(err).map_err(|s| unsafe { + // Reapply the `Send` marker. + mem::transmute::, Box>(s) + }) + } +} + +impl dyn Error + Send + Sync { + #[inline] + #[stable(feature = "error_downcast", since = "1.3.0")] + #[rustc_allow_incoherent_impl] + /// Attempts to downcast the box to a concrete type. + pub fn downcast(self: Box) -> Result, Box> { + let err: Box = self; + ::downcast(err).map_err(|s| unsafe { + // Reapply the `Send + Sync` marker. + mem::transmute::, Box>(s) + }) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, E: Error + 'a> From for Box { + /// Converts a type of [`Error`] into a box of dyn [`Error`]. + /// + /// # Examples + /// + /// ``` + /// use std::error::Error; + /// use std::fmt; + /// use std::mem; + /// + /// #[derive(Debug)] + /// struct AnError; + /// + /// impl fmt::Display for AnError { + /// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// write!(f, "An error") + /// } + /// } + /// + /// impl Error for AnError {} + /// + /// let an_error = AnError; + /// assert!(0 == mem::size_of_val(&an_error)); + /// let a_boxed_error = Box::::from(an_error); + /// assert!(mem::size_of::>() == mem::size_of_val(&a_boxed_error)) + /// ``` + fn from(err: E) -> Box { + Box::new(err) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, E: Error + Send + Sync + 'a> From for Box { + /// Converts a type of [`Error`] + [`Send`] + [`Sync`] into a box of + /// dyn [`Error`] + [`Send`] + [`Sync`]. + /// + /// # Examples + /// + /// ``` + /// use std::error::Error; + /// use std::fmt; + /// use std::mem; + /// + /// #[derive(Debug)] + /// struct AnError; + /// + /// impl fmt::Display for AnError { + /// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// write!(f, "An error") + /// } + /// } + /// + /// impl Error for AnError {} + /// + /// unsafe impl Send for AnError {} + /// + /// unsafe impl Sync for AnError {} + /// + /// let an_error = AnError; + /// assert!(0 == mem::size_of_val(&an_error)); + /// let a_boxed_error = Box::::from(an_error); + /// assert!( + /// mem::size_of::>() == mem::size_of_val(&a_boxed_error)) + /// ``` + fn from(err: E) -> Box { + Box::new(err) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +impl From for Box { + /// Converts a [`String`] into a box of dyn [`Error`] + [`Send`] + [`Sync`]. + /// + /// # Examples + /// + /// ``` + /// use std::error::Error; + /// use std::mem; + /// + /// let a_string_error = "a string error".to_string(); + /// let a_boxed_error = Box::::from(a_string_error); + /// assert!( + /// mem::size_of::>() == mem::size_of_val(&a_boxed_error)) + /// ``` + #[inline] + fn from(err: String) -> Box { + struct StringError(String); + + impl Error for StringError { + #[allow(deprecated)] + fn description(&self) -> &str { + &self.0 + } + } + + impl fmt::Display for StringError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } + } + + // Purposefully skip printing "StringError(..)" + impl fmt::Debug for StringError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self.0, f) + } + } + + Box::new(StringError(err)) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "string_box_error", since = "1.6.0")] +impl From for Box { + /// Converts a [`String`] into a box of dyn [`Error`]. + /// + /// # Examples + /// + /// ``` + /// use std::error::Error; + /// use std::mem; + /// + /// let a_string_error = "a string error".to_string(); + /// let a_boxed_error = Box::::from(a_string_error); + /// assert!(mem::size_of::>() == mem::size_of_val(&a_boxed_error)) + /// ``` + fn from(str_err: String) -> Box { + let err1: Box = From::from(str_err); + let err2: Box = err1; + err2 + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a> From<&str> for Box { + /// Converts a [`str`] into a box of dyn [`Error`] + [`Send`] + [`Sync`]. + /// + /// [`str`]: prim@str + /// + /// # Examples + /// + /// ``` + /// use std::error::Error; + /// use std::mem; + /// + /// let a_str_error = "a str error"; + /// let a_boxed_error = Box::::from(a_str_error); + /// assert!( + /// mem::size_of::>() == mem::size_of_val(&a_boxed_error)) + /// ``` + #[inline] + fn from(err: &str) -> Box { + From::from(String::from(err)) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "string_box_error", since = "1.6.0")] +impl From<&str> for Box { + /// Converts a [`str`] into a box of dyn [`Error`]. + /// + /// [`str`]: prim@str + /// + /// # Examples + /// + /// ``` + /// use std::error::Error; + /// use std::mem; + /// + /// let a_str_error = "a str error"; + /// let a_boxed_error = Box::::from(a_str_error); + /// assert!(mem::size_of::>() == mem::size_of_val(&a_boxed_error)) + /// ``` + fn from(err: &str) -> Box { + From::from(String::from(err)) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "cow_box_error", since = "1.22.0")] +impl<'a, 'b> From> for Box { + /// Converts a [`Cow`] into a box of dyn [`Error`] + [`Send`] + [`Sync`]. + /// + /// # Examples + /// + /// ``` + /// use std::error::Error; + /// use std::mem; + /// use std::borrow::Cow; + /// + /// let a_cow_str_error = Cow::from("a str error"); + /// let a_boxed_error = Box::::from(a_cow_str_error); + /// assert!( + /// mem::size_of::>() == mem::size_of_val(&a_boxed_error)) + /// ``` + fn from(err: Cow<'b, str>) -> Box { + From::from(String::from(err)) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "cow_box_error", since = "1.22.0")] +impl<'a> From> for Box { + /// Converts a [`Cow`] into a box of dyn [`Error`]. + /// + /// # Examples + /// + /// ``` + /// use std::error::Error; + /// use std::mem; + /// use std::borrow::Cow; + /// + /// let a_cow_str_error = Cow::from("a str error"); + /// let a_boxed_error = Box::::from(a_cow_str_error); + /// assert!(mem::size_of::>() == mem::size_of_val(&a_boxed_error)) + /// ``` + fn from(err: Cow<'a, str>) -> Box { + From::from(String::from(err)) + } +} + +#[stable(feature = "box_error", since = "1.8.0")] +impl core::error::Error for Box { + #[allow(deprecated, deprecated_in_future)] + fn description(&self) -> &str { + core::error::Error::description(&**self) + } + + #[allow(deprecated)] + fn cause(&self) -> Option<&dyn core::error::Error> { + core::error::Error::cause(&**self) + } + + fn source(&self) -> Option<&(dyn core::error::Error + 'static)> { + core::error::Error::source(&**self) + } +} diff --git a/crux-mir/lib/alloc/src/boxed/thin.rs b/crux-mir/lib/alloc/src/boxed/thin.rs new file mode 100644 index 000000000..c1a82e452 --- /dev/null +++ b/crux-mir/lib/alloc/src/boxed/thin.rs @@ -0,0 +1,302 @@ +// Based on +// https://github.com/matthieu-m/rfc2580/blob/b58d1d3cba0d4b5e859d3617ea2d0943aaa31329/examples/thin.rs +// by matthieu-m +use crate::alloc::{self, Layout, LayoutError}; +use core::error::Error; +use core::fmt::{self, Debug, Display, Formatter}; +use core::marker::PhantomData; +#[cfg(not(no_global_oom_handling))] +use core::marker::Unsize; +use core::mem; +use core::ops::{Deref, DerefMut}; +use core::ptr::Pointee; +use core::ptr::{self, NonNull}; + +/// ThinBox. +/// +/// A thin pointer for heap allocation, regardless of T. +/// +/// # Examples +/// +/// ``` +/// #![feature(thin_box)] +/// use std::boxed::ThinBox; +/// +/// let five = ThinBox::new(5); +/// let thin_slice = ThinBox::<[i32]>::new_unsize([1, 2, 3, 4]); +/// +/// use std::mem::{size_of, size_of_val}; +/// let size_of_ptr = size_of::<*const ()>(); +/// assert_eq!(size_of_ptr, size_of_val(&five)); +/// assert_eq!(size_of_ptr, size_of_val(&thin_slice)); +/// ``` +#[unstable(feature = "thin_box", issue = "92791")] +pub struct ThinBox { + // This is essentially `WithHeader<::Metadata>`, + // but that would be invariant in `T`, and we want covariance. + ptr: WithOpaqueHeader, + _marker: PhantomData, +} + +/// `ThinBox` is `Send` if `T` is `Send` because the data is owned. +#[unstable(feature = "thin_box", issue = "92791")] +unsafe impl Send for ThinBox {} + +/// `ThinBox` is `Sync` if `T` is `Sync` because the data is owned. +#[unstable(feature = "thin_box", issue = "92791")] +unsafe impl Sync for ThinBox {} + +#[unstable(feature = "thin_box", issue = "92791")] +impl ThinBox { + /// Moves a type to the heap with its `Metadata` stored in the heap allocation instead of on + /// the stack. + /// + /// # Examples + /// + /// ``` + /// #![feature(thin_box)] + /// use std::boxed::ThinBox; + /// + /// let five = ThinBox::new(5); + /// ``` + #[cfg(not(no_global_oom_handling))] + pub fn new(value: T) -> Self { + let meta = ptr::metadata(&value); + let ptr = WithOpaqueHeader::new(meta, value); + ThinBox { ptr, _marker: PhantomData } + } +} + +#[unstable(feature = "thin_box", issue = "92791")] +impl ThinBox { + /// Moves a type to the heap with its `Metadata` stored in the heap allocation instead of on + /// the stack. + /// + /// # Examples + /// + /// ``` + /// #![feature(thin_box)] + /// use std::boxed::ThinBox; + /// + /// let thin_slice = ThinBox::<[i32]>::new_unsize([1, 2, 3, 4]); + /// ``` + #[cfg(not(no_global_oom_handling))] + pub fn new_unsize(value: T) -> Self + where + T: Unsize, + { + let meta = ptr::metadata(&value as &Dyn); + let ptr = WithOpaqueHeader::new(meta, value); + ThinBox { ptr, _marker: PhantomData } + } +} + +#[unstable(feature = "thin_box", issue = "92791")] +impl Debug for ThinBox { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + Debug::fmt(self.deref(), f) + } +} + +#[unstable(feature = "thin_box", issue = "92791")] +impl Display for ThinBox { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + Display::fmt(self.deref(), f) + } +} + +#[unstable(feature = "thin_box", issue = "92791")] +impl Deref for ThinBox { + type Target = T; + + fn deref(&self) -> &T { + let value = self.data(); + let metadata = self.meta(); + let pointer = ptr::from_raw_parts(value as *const (), metadata); + unsafe { &*pointer } + } +} + +#[unstable(feature = "thin_box", issue = "92791")] +impl DerefMut for ThinBox { + fn deref_mut(&mut self) -> &mut T { + let value = self.data(); + let metadata = self.meta(); + let pointer = ptr::from_raw_parts_mut::(value as *mut (), metadata); + unsafe { &mut *pointer } + } +} + +#[unstable(feature = "thin_box", issue = "92791")] +impl Drop for ThinBox { + fn drop(&mut self) { + unsafe { + let value = self.deref_mut(); + let value = value as *mut T; + self.with_header().drop::(value); + } + } +} + +#[unstable(feature = "thin_box", issue = "92791")] +impl ThinBox { + fn meta(&self) -> ::Metadata { + // Safety: + // - NonNull and valid. + unsafe { *self.with_header().header() } + } + + fn data(&self) -> *mut u8 { + self.with_header().value() + } + + fn with_header(&self) -> &WithHeader<::Metadata> { + // SAFETY: both types are transparent to `NonNull` + unsafe { &*((&self.ptr) as *const WithOpaqueHeader as *const WithHeader<_>) } + } +} + +/// A pointer to type-erased data, guaranteed to either be: +/// 1. `NonNull::dangling()`, in the case where both the pointee (`T`) and +/// metadata (`H`) are ZSTs. +/// 2. A pointer to a valid `T` that has a header `H` directly before the +/// pointed-to location. +#[repr(transparent)] +struct WithHeader(NonNull, PhantomData); + +/// An opaque representation of `WithHeader` to avoid the +/// projection invariance of `::Metadata`. +#[repr(transparent)] +struct WithOpaqueHeader(NonNull); + +impl WithOpaqueHeader { + #[cfg(not(no_global_oom_handling))] + fn new(header: H, value: T) -> Self { + let ptr = WithHeader::new(header, value); + Self(ptr.0) + } +} + +impl WithHeader { + #[cfg(not(no_global_oom_handling))] + fn new(header: H, value: T) -> WithHeader { + let value_layout = Layout::new::(); + let Ok((layout, value_offset)) = Self::alloc_layout(value_layout) else { + // We pass an empty layout here because we do not know which layout caused the + // arithmetic overflow in `Layout::extend` and `handle_alloc_error` takes `Layout` as + // its argument rather than `Result`, also this function has been + // stable since 1.28 ._. + // + // On the other hand, look at this gorgeous turbofish! + alloc::handle_alloc_error(Layout::new::<()>()); + }; + + unsafe { + // Note: It's UB to pass a layout with a zero size to `alloc::alloc`, so + // we use `layout.dangling()` for this case, which should have a valid + // alignment for both `T` and `H`. + let ptr = if layout.size() == 0 { + // Some paranoia checking, mostly so that the ThinBox tests are + // more able to catch issues. + debug_assert!( + value_offset == 0 && mem::size_of::() == 0 && mem::size_of::() == 0 + ); + layout.dangling() + } else { + let ptr = alloc::alloc(layout); + if ptr.is_null() { + alloc::handle_alloc_error(layout); + } + // Safety: + // - The size is at least `aligned_header_size`. + let ptr = ptr.add(value_offset) as *mut _; + + NonNull::new_unchecked(ptr) + }; + + let result = WithHeader(ptr, PhantomData); + ptr::write(result.header(), header); + ptr::write(result.value().cast(), value); + + result + } + } + + // Safety: + // - Assumes that either `value` can be dereferenced, or is the + // `NonNull::dangling()` we use when both `T` and `H` are ZSTs. + unsafe fn drop(&self, value: *mut T) { + struct DropGuard { + ptr: NonNull, + value_layout: Layout, + _marker: PhantomData, + } + + impl Drop for DropGuard { + fn drop(&mut self) { + unsafe { + // SAFETY: Layout must have been computable if we're in drop + let (layout, value_offset) = + WithHeader::::alloc_layout(self.value_layout).unwrap_unchecked(); + + // Note: Don't deallocate if the layout size is zero, because the pointer + // didn't come from the allocator. + if layout.size() != 0 { + alloc::dealloc(self.ptr.as_ptr().sub(value_offset), layout); + } else { + debug_assert!( + value_offset == 0 + && mem::size_of::() == 0 + && self.value_layout.size() == 0 + ); + } + } + } + } + + unsafe { + // `_guard` will deallocate the memory when dropped, even if `drop_in_place` unwinds. + let _guard = DropGuard { + ptr: self.0, + value_layout: Layout::for_value_raw(value), + _marker: PhantomData::, + }; + + // We only drop the value because the Pointee trait requires that the metadata is copy + // aka trivially droppable. + ptr::drop_in_place::(value); + } + } + + fn header(&self) -> *mut H { + // Safety: + // - At least `size_of::()` bytes are allocated ahead of the pointer. + // - We know that H will be aligned because the middle pointer is aligned to the greater + // of the alignment of the header and the data and the header size includes the padding + // needed to align the header. Subtracting the header size from the aligned data pointer + // will always result in an aligned header pointer, it just may not point to the + // beginning of the allocation. + let hp = unsafe { self.0.as_ptr().sub(Self::header_size()) as *mut H }; + debug_assert!(hp.is_aligned()); + hp + } + + fn value(&self) -> *mut u8 { + self.0.as_ptr() + } + + const fn header_size() -> usize { + mem::size_of::() + } + + fn alloc_layout(value_layout: Layout) -> Result<(Layout, usize), LayoutError> { + Layout::new::().extend(value_layout) + } +} + +#[unstable(feature = "thin_box", issue = "92791")] +impl Error for ThinBox { + fn source(&self) -> Option<&(dyn Error + 'static)> { + self.deref().source() + } +} diff --git a/crux-mir/lib/alloc/src/collections/binary_heap.rs b/crux-mir/lib/alloc/src/collections/binary_heap/mod.rs similarity index 58% rename from crux-mir/lib/alloc/src/collections/binary_heap.rs rename to crux-mir/lib/alloc/src/collections/binary_heap/mod.rs index 9908a3049..0b73b1af4 100644 --- a/crux-mir/lib/alloc/src/collections/binary_heap.rs +++ b/crux-mir/lib/alloc/src/collections/binary_heap/mod.rs @@ -1,10 +1,10 @@ //! A priority queue implemented with a binary heap. //! -//! Insertion and popping the largest element have `O(log n)` time complexity. -//! Checking the largest element is `O(1)`. Converting a vector to a binary heap -//! can be done in-place, and has `O(n)` complexity. A binary heap can also be -//! converted to a sorted vector in-place, allowing it to be used for an `O(n -//! log n)` in-place heapsort. +//! Insertion and popping the largest element have *O*(log(*n*)) time complexity. +//! Checking the largest element is *O*(1). Converting a vector to a binary heap +//! can be done in-place, and has *O*(*n*) complexity. A binary heap can also be +//! converted to a sorted vector in-place, allowing it to be used for an *O*(*n* * log(*n*)) +//! in-place heapsort. //! //! # Examples //! @@ -12,15 +12,13 @@ //! to solve the [shortest path problem][sssp] on a [directed graph][dir_graph]. //! It shows how to use [`BinaryHeap`] with custom types. //! -//! [dijkstra]: http://en.wikipedia.org/wiki/Dijkstra%27s_algorithm -//! [sssp]: http://en.wikipedia.org/wiki/Shortest_path_problem -//! [dir_graph]: http://en.wikipedia.org/wiki/Directed_graph -//! [`BinaryHeap`]: struct.BinaryHeap.html +//! [dijkstra]: https://en.wikipedia.org/wiki/Dijkstra%27s_algorithm +//! [sssp]: https://en.wikipedia.org/wiki/Shortest_path_problem +//! [dir_graph]: https://en.wikipedia.org/wiki/Directed_graph //! //! ``` //! use std::cmp::Ordering; //! use std::collections::BinaryHeap; -//! use std::usize; //! //! #[derive(Copy, Clone, Eq, PartialEq)] //! struct State { @@ -32,7 +30,7 @@ //! // Explicitly implement the trait so the queue becomes a min-heap //! // instead of a max-heap. //! impl Ord for State { -//! fn cmp(&self, other: &State) -> Ordering { +//! fn cmp(&self, other: &Self) -> Ordering { //! // Notice that the we flip the ordering on costs. //! // In case of a tie we compare positions - this step is necessary //! // to make implementations of `PartialEq` and `Ord` consistent. @@ -43,12 +41,12 @@ //! //! // `PartialOrd` needs to be implemented as well. //! impl PartialOrd for State { -//! fn partial_cmp(&self, other: &State) -> Option { +//! fn partial_cmp(&self, other: &Self) -> Option { //! Some(self.cmp(other)) //! } //! } //! -//! // Each node is represented as an `usize`, for a shorter implementation. +//! // Each node is represented as a `usize`, for a shorter implementation. //! struct Edge { //! node: usize, //! cost: usize, @@ -146,24 +144,41 @@ #![stable(feature = "rust1", since = "1.0.0")] use core::fmt; -use core::iter::{FromIterator, FusedIterator, TrustedLen}; -use core::mem::{self, size_of, swap, ManuallyDrop}; +use core::iter::{FromIterator, FusedIterator, InPlaceIterable, SourceIter, TrustedLen}; +use core::mem::{self, swap, ManuallyDrop}; +use core::num::NonZeroUsize; use core::ops::{Deref, DerefMut}; use core::ptr; +use crate::collections::TryReserveError; use crate::slice; -use crate::vec::{self, Vec}; +use crate::vec::{self, AsVecIntoIter, Vec}; use super::SpecExtend; +#[cfg(test)] +mod tests; + /// A priority queue implemented with a binary heap. /// /// This will be a max-heap. /// /// It is a logic error for an item to be modified in such a way that the -/// item's ordering relative to any other item, as determined by the `Ord` +/// item's ordering relative to any other item, as determined by the [`Ord`] /// trait, changes while it is in the heap. This is normally only possible -/// through `Cell`, `RefCell`, global state, I/O, or unsafe code. +/// through interior mutability, global state, I/O, or unsafe code. The +/// behavior resulting from such a logic error is not specified, but will +/// be encapsulated to the `BinaryHeap` that observed the logic error and not +/// result in undefined behavior. This could include panics, incorrect results, +/// aborts, memory leaks, and non-termination. +/// +/// As long as no elements change their relative order while being in the heap +/// as described above, the API of `BinaryHeap` guarantees that the heap +/// invariant remains intact i.e. its methods all behave as documented. For +/// example if a method is documented as iterating in sorted order, that's +/// guaranteed to work as long as elements in the heap have not changed order, +/// even in the presence of closures getting unwinded out of, iterators getting +/// leaked, and similar foolishness. /// /// # Examples /// @@ -192,7 +207,7 @@ use super::SpecExtend; /// // We can iterate over the items in the heap, although they are returned in /// // a random order. /// for x in &heap { -/// println!("{}", x); +/// println!("{x}"); /// } /// /// // If we instead pop these scores, they should come back in order. @@ -208,9 +223,17 @@ use super::SpecExtend; /// assert!(heap.is_empty()) /// ``` /// +/// A `BinaryHeap` with a known list of items can be initialized from an array: +/// +/// ``` +/// use std::collections::BinaryHeap; +/// +/// let heap = BinaryHeap::from([1, 5, 2]); +/// ``` +/// /// ## Min-heap /// -/// Either `std::cmp::Reverse` or a custom `Ord` implementation can be used to +/// Either [`core::cmp::Reverse`] or a custom [`Ord`] implementation can be used to /// make `BinaryHeap` a min-heap. This makes `heap.pop()` return the smallest /// value instead of the greatest one. /// @@ -234,18 +257,23 @@ use super::SpecExtend; /// /// # Time complexity /// -/// | [push] | [pop] | [peek]/[peek\_mut] | -/// |--------|----------|--------------------| -/// | O(1)~ | O(log n) | O(1) | +/// | [push] | [pop] | [peek]/[peek\_mut] | +/// |---------|---------------|--------------------| +/// | *O*(1)~ | *O*(log(*n*)) | *O*(1) | /// /// The value for `push` is an expected cost; the method documentation gives a /// more detailed analysis. /// -/// [push]: #method.push -/// [pop]: #method.pop -/// [peek]: #method.peek -/// [peek\_mut]: #method.peek_mut +/// [`core::cmp::Reverse`]: core::cmp::Reverse +/// [`Ord`]: core::cmp::Ord +/// [`Cell`]: core::cell::Cell +/// [`RefCell`]: core::cell::RefCell +/// [push]: BinaryHeap::push +/// [pop]: BinaryHeap::pop +/// [peek]: BinaryHeap::peek +/// [peek\_mut]: BinaryHeap::peek_mut #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "BinaryHeap")] pub struct BinaryHeap { data: Vec, } @@ -256,12 +284,13 @@ pub struct BinaryHeap { /// This `struct` is created by the [`peek_mut`] method on [`BinaryHeap`]. See /// its documentation for more. /// -/// [`peek_mut`]: struct.BinaryHeap.html#method.peek_mut -/// [`BinaryHeap`]: struct.BinaryHeap.html +/// [`peek_mut`]: BinaryHeap::peek_mut #[stable(feature = "binary_heap_peek_mut", since = "1.12.0")] pub struct PeekMut<'a, T: 'a + Ord> { heap: &'a mut BinaryHeap, - sift: bool, + // If a set_len + sift_down are required, this is Some. If a &mut T has not + // yet been exposed to peek_mut()'s caller, it's None. + original_len: Option, } #[stable(feature = "collection_debug", since = "1.17.0")] @@ -274,8 +303,16 @@ impl fmt::Debug for PeekMut<'_, T> { #[stable(feature = "binary_heap_peek_mut", since = "1.12.0")] impl Drop for PeekMut<'_, T> { fn drop(&mut self) { - if self.sift { - self.heap.sift_down(0); + if let Some(original_len) = self.original_len { + // SAFETY: That's how many elements were in the Vec at the time of + // the PeekMut::deref_mut call, and therefore also at the time of + // the BinaryHeap::peek_mut call. Since the PeekMut did not end up + // getting leaked, we are now undoing the leak amplification that + // the DerefMut prepared for. + unsafe { self.heap.data.set_len(original_len.get()) }; + + // SAFETY: PeekMut is only instantiated for non-empty heaps. + unsafe { self.heap.sift_down(0) }; } } } @@ -294,6 +331,26 @@ impl Deref for PeekMut<'_, T> { impl DerefMut for PeekMut<'_, T> { fn deref_mut(&mut self) -> &mut T { debug_assert!(!self.heap.is_empty()); + + let len = self.heap.len(); + if len > 1 { + // Here we preemptively leak all the rest of the underlying vector + // after the currently max element. If the caller mutates the &mut T + // we're about to give them, and then leaks the PeekMut, all these + // elements will remain leaked. If they don't leak the PeekMut, then + // either Drop or PeekMut::pop will un-leak the vector elements. + // + // This is technique is described throughout several other places in + // the standard library as "leak amplification". + unsafe { + // SAFETY: len > 1 so len != 0. + self.original_len = Some(NonZeroUsize::new_unchecked(len)); + // SAFETY: len > 1 so all this does for now is leak elements, + // which is safe. + self.heap.data.set_len(1); + } + } + // SAFE: PeekMut is only instantiated for non-empty heaps unsafe { self.heap.data.get_unchecked_mut(0) } } @@ -303,9 +360,16 @@ impl<'a, T: Ord> PeekMut<'a, T> { /// Removes the peeked value from the heap and returns it. #[stable(feature = "binary_heap_peek_mut_pop", since = "1.18.0")] pub fn pop(mut this: PeekMut<'a, T>) -> T { - let value = this.heap.pop().unwrap(); - this.sift = false; - value + if let Some(original_len) = this.original_len.take() { + // SAFETY: This is how many elements were in the Vec at the time of + // the BinaryHeap::peek_mut call. + unsafe { this.heap.data.set_len(original_len.get()) }; + + // Unlike in Drop, here we don't also need to do a sift_down even if + // the caller could've mutated the element. It is removed from the + // heap on the next line and pop() is not sensitive to its value. + } + this.heap.pop().unwrap() } } @@ -349,14 +413,16 @@ impl BinaryHeap { /// heap.push(4); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] pub fn new() -> BinaryHeap { BinaryHeap { data: vec![] } } - /// Creates an empty `BinaryHeap` with a specific capacity. - /// This preallocates enough memory for `capacity` elements, - /// so that the `BinaryHeap` does not have to be reallocated - /// until it contains at least that many values. + /// Creates an empty `BinaryHeap` with at least the specified capacity. + /// + /// The binary heap will be able to hold at least `capacity` elements without + /// reallocating. This method is allowed to allocate for more elements than + /// `capacity`. If `capacity` is 0, the binary heap will not allocate. /// /// # Examples /// @@ -368,6 +434,7 @@ impl BinaryHeap { /// heap.push(4); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] pub fn with_capacity(capacity: usize) -> BinaryHeap { BinaryHeap { data: Vec::with_capacity(capacity) } } @@ -375,8 +442,9 @@ impl BinaryHeap { /// Returns a mutable reference to the greatest item in the binary heap, or /// `None` if it is empty. /// - /// Note: If the `PeekMut` value is leaked, the heap may be in an - /// inconsistent state. + /// Note: If the `PeekMut` value is leaked, some heap elements might get + /// leaked along with it, but the remaining elements will remain a valid + /// heap. /// /// # Examples /// @@ -399,10 +467,11 @@ impl BinaryHeap { /// /// # Time complexity /// - /// Cost is O(1) in the worst case. + /// If the item is modified then the worst case time complexity is *O*(log(*n*)), + /// otherwise it's *O*(1). #[stable(feature = "binary_heap_peek_mut", since = "1.12.0")] pub fn peek_mut(&mut self) -> Option> { - if self.is_empty() { None } else { Some(PeekMut { heap: self, sift: true }) } + if self.is_empty() { None } else { Some(PeekMut { heap: self, original_len: None }) } } /// Removes the greatest item from the binary heap and returns it, or `None` if it @@ -414,7 +483,7 @@ impl BinaryHeap { /// /// ``` /// use std::collections::BinaryHeap; - /// let mut heap = BinaryHeap::from(vec![1, 3]); + /// let mut heap = BinaryHeap::from([1, 3]); /// /// assert_eq!(heap.pop(), Some(3)); /// assert_eq!(heap.pop(), Some(1)); @@ -423,14 +492,14 @@ impl BinaryHeap { /// /// # Time complexity /// - /// The worst case cost of `pop` on a heap containing *n* elements is O(log - /// n). + /// The worst case cost of `pop` on a heap containing *n* elements is *O*(log(*n*)). #[stable(feature = "rust1", since = "1.0.0")] pub fn pop(&mut self) -> Option { self.data.pop().map(|mut item| { if !self.is_empty() { swap(&mut item, &mut self.data[0]); - self.sift_down_to_bottom(0); + // SAFETY: !self.is_empty() means that self.len() > 0 + unsafe { self.sift_down_to_bottom(0) }; } item }) @@ -457,22 +526,24 @@ impl BinaryHeap { /// /// The expected cost of `push`, averaged over every possible ordering of /// the elements being pushed, and over a sufficiently large number of - /// pushes, is O(1). This is the most meaningful cost metric when pushing + /// pushes, is *O*(1). This is the most meaningful cost metric when pushing /// elements that are *not* already in any sorted pattern. /// /// The time complexity degrades if elements are pushed in predominantly /// ascending order. In the worst case, elements are pushed in ascending - /// sorted order and the amortized cost per push is O(log n) against a heap + /// sorted order and the amortized cost per push is *O*(log(*n*)) against a heap /// containing *n* elements. /// - /// The worst case cost of a *single* call to `push` is O(n). The worst case + /// The worst case cost of a *single* call to `push` is *O*(*n*). The worst case /// occurs when capacity is exhausted and needs a resize. The resize cost /// has been amortized in the previous figures. #[stable(feature = "rust1", since = "1.0.0")] pub fn push(&mut self, item: T) { let old_len = self.len(); self.data.push(item); - self.sift_up(0, old_len); + // SAFETY: Since we pushed a new item it means that + // old_len = self.len() - 1 < self.len() + unsafe { self.sift_up(0, old_len) }; } /// Consumes the `BinaryHeap` and returns a vector in sorted @@ -485,20 +556,31 @@ impl BinaryHeap { /// ``` /// use std::collections::BinaryHeap; /// - /// let mut heap = BinaryHeap::from(vec![1, 2, 4, 5, 7]); + /// let mut heap = BinaryHeap::from([1, 2, 4, 5, 7]); /// heap.push(6); /// heap.push(3); /// /// let vec = heap.into_sorted_vec(); /// assert_eq!(vec, [1, 2, 3, 4, 5, 6, 7]); /// ``` + #[must_use = "`self` will be dropped if the result is not used"] #[stable(feature = "binary_heap_extras_15", since = "1.5.0")] pub fn into_sorted_vec(mut self) -> Vec { let mut end = self.len(); while end > 1 { end -= 1; - self.data.swap(0, end); - self.sift_down_range(0, end); + // SAFETY: `end` goes from `self.len() - 1` to 1 (both included), + // so it's always a valid index to access. + // It is safe to access index 0 (i.e. `ptr`), because + // 1 <= end < self.len(), which means self.len() >= 2. + unsafe { + let ptr = self.data.as_mut_ptr(); + ptr::swap(ptr, ptr.add(end)); + } + // SAFETY: `end` goes from `self.len() - 1` to 1 (both included) so: + // 0 < 1 <= end <= self.len() - 1 < self.len() + // Which means 0 < end and end < self.len(). + unsafe { self.sift_down_range(0, end) }; } self.into_vec() } @@ -511,47 +593,84 @@ impl BinaryHeap { // the hole is filled back at the end of its scope, even on panic. // Using a hole reduces the constant factor compared to using swaps, // which involves twice as many moves. - fn sift_up(&mut self, start: usize, pos: usize) -> usize { - unsafe { - // Take out the value at `pos` and create a hole. - let mut hole = Hole::new(&mut self.data, pos); - - while hole.pos() > start { - let parent = (hole.pos() - 1) / 2; - if hole.element() <= hole.get(parent) { - break; - } - hole.move_to(parent); + + /// # Safety + /// + /// The caller must guarantee that `pos < self.len()`. + unsafe fn sift_up(&mut self, start: usize, pos: usize) -> usize { + // Take out the value at `pos` and create a hole. + // SAFETY: The caller guarantees that pos < self.len() + let mut hole = unsafe { Hole::new(&mut self.data, pos) }; + + while hole.pos() > start { + let parent = (hole.pos() - 1) / 2; + + // SAFETY: hole.pos() > start >= 0, which means hole.pos() > 0 + // and so hole.pos() - 1 can't underflow. + // This guarantees that parent < hole.pos() so + // it's a valid index and also != hole.pos(). + if hole.element() <= unsafe { hole.get(parent) } { + break; } - hole.pos() + + // SAFETY: Same as above + unsafe { hole.move_to(parent) }; } + + hole.pos() } /// Take an element at `pos` and move it down the heap, /// while its children are larger. - fn sift_down_range(&mut self, pos: usize, end: usize) { - unsafe { - let mut hole = Hole::new(&mut self.data, pos); - let mut child = 2 * pos + 1; - while child < end { - let right = child + 1; - // compare with the greater of the two children - if right < end && hole.get(child) <= hole.get(right) { - child = right; - } - // if we are already in order, stop. - if hole.element() >= hole.get(child) { - break; - } - hole.move_to(child); - child = 2 * hole.pos() + 1; + /// + /// # Safety + /// + /// The caller must guarantee that `pos < end <= self.len()`. + unsafe fn sift_down_range(&mut self, pos: usize, end: usize) { + // SAFETY: The caller guarantees that pos < end <= self.len(). + let mut hole = unsafe { Hole::new(&mut self.data, pos) }; + let mut child = 2 * hole.pos() + 1; + + // Loop invariant: child == 2 * hole.pos() + 1. + while child <= end.saturating_sub(2) { + // compare with the greater of the two children + // SAFETY: child < end - 1 < self.len() and + // child + 1 < end <= self.len(), so they're valid indexes. + // child == 2 * hole.pos() + 1 != hole.pos() and + // child + 1 == 2 * hole.pos() + 2 != hole.pos(). + // FIXME: 2 * hole.pos() + 1 or 2 * hole.pos() + 2 could overflow + // if T is a ZST + child += unsafe { hole.get(child) <= hole.get(child + 1) } as usize; + + // if we are already in order, stop. + // SAFETY: child is now either the old child or the old child+1 + // We already proven that both are < self.len() and != hole.pos() + if hole.element() >= unsafe { hole.get(child) } { + return; } + + // SAFETY: same as above. + unsafe { hole.move_to(child) }; + child = 2 * hole.pos() + 1; + } + + // SAFETY: && short circuit, which means that in the + // second condition it's already true that child == end - 1 < self.len(). + if child == end - 1 && hole.element() < unsafe { hole.get(child) } { + // SAFETY: child is already proven to be a valid index and + // child == 2 * hole.pos() + 1 != hole.pos(). + unsafe { hole.move_to(child) }; } } - fn sift_down(&mut self, pos: usize) { + /// # Safety + /// + /// The caller must guarantee that `pos < self.len()`. + unsafe fn sift_down(&mut self, pos: usize) { let len = self.len(); - self.sift_down_range(pos, len); + // SAFETY: pos < len is guaranteed by the caller and + // obviously len = self.len() <= self.len(). + unsafe { self.sift_down_range(pos, len) }; } /// Take an element at `pos` and move it all the way down the heap, @@ -559,31 +678,91 @@ impl BinaryHeap { /// /// Note: This is faster when the element is known to be large / should /// be closer to the bottom. - fn sift_down_to_bottom(&mut self, mut pos: usize) { + /// + /// # Safety + /// + /// The caller must guarantee that `pos < self.len()`. + unsafe fn sift_down_to_bottom(&mut self, mut pos: usize) { let end = self.len(); let start = pos; - unsafe { - let mut hole = Hole::new(&mut self.data, pos); - let mut child = 2 * pos + 1; - while child < end { - let right = child + 1; - // compare with the greater of the two children - if right < end && hole.get(child) <= hole.get(right) { - child = right; - } - hole.move_to(child); - child = 2 * hole.pos() + 1; + + // SAFETY: The caller guarantees that pos < self.len(). + let mut hole = unsafe { Hole::new(&mut self.data, pos) }; + let mut child = 2 * hole.pos() + 1; + + // Loop invariant: child == 2 * hole.pos() + 1. + while child <= end.saturating_sub(2) { + // SAFETY: child < end - 1 < self.len() and + // child + 1 < end <= self.len(), so they're valid indexes. + // child == 2 * hole.pos() + 1 != hole.pos() and + // child + 1 == 2 * hole.pos() + 2 != hole.pos(). + // FIXME: 2 * hole.pos() + 1 or 2 * hole.pos() + 2 could overflow + // if T is a ZST + child += unsafe { hole.get(child) <= hole.get(child + 1) } as usize; + + // SAFETY: Same as above + unsafe { hole.move_to(child) }; + child = 2 * hole.pos() + 1; + } + + if child == end - 1 { + // SAFETY: child == end - 1 < self.len(), so it's a valid index + // and child == 2 * hole.pos() + 1 != hole.pos(). + unsafe { hole.move_to(child) }; + } + pos = hole.pos(); + drop(hole); + + // SAFETY: pos is the position in the hole and was already proven + // to be a valid index. + unsafe { self.sift_up(start, pos) }; + } + + /// Rebuild assuming data[0..start] is still a proper heap. + fn rebuild_tail(&mut self, start: usize) { + if start == self.len() { + return; + } + + let tail_len = self.len() - start; + + #[inline(always)] + fn log2_fast(x: usize) -> usize { + (usize::BITS - x.leading_zeros() - 1) as usize + } + + // `rebuild` takes O(self.len()) operations + // and about 2 * self.len() comparisons in the worst case + // while repeating `sift_up` takes O(tail_len * log(start)) operations + // and about 1 * tail_len * log_2(start) comparisons in the worst case, + // assuming start >= tail_len. For larger heaps, the crossover point + // no longer follows this reasoning and was determined empirically. + let better_to_rebuild = if start < tail_len { + true + } else if self.len() <= 2048 { + 2 * self.len() < tail_len * log2_fast(start) + } else { + 2 * self.len() < tail_len * 11 + }; + + if better_to_rebuild { + self.rebuild(); + } else { + for i in start..self.len() { + // SAFETY: The index `i` is always less than self.len(). + unsafe { self.sift_up(0, i) }; } - pos = hole.pos; } - self.sift_up(start, pos); } fn rebuild(&mut self) { let mut n = self.len() / 2; while n > 0 { n -= 1; - self.sift_down(n); + // SAFETY: n starts from self.len() / 2 and goes down to 0. + // The only case when !(n < self.len()) is if + // self.len() == 0, but it's ruled out by the loop condition. + unsafe { self.sift_down(n) }; } } @@ -596,11 +775,8 @@ impl BinaryHeap { /// ``` /// use std::collections::BinaryHeap; /// - /// let v = vec![-10, 1, 2, 3, 3]; - /// let mut a = BinaryHeap::from(v); - /// - /// let v = vec![-20, 5, 43]; - /// let mut b = BinaryHeap::from(v); + /// let mut a = BinaryHeap::from([-10, 1, 2, 3, 3]); + /// let mut b = BinaryHeap::from([-20, 5, 43]); /// /// a.append(&mut b); /// @@ -613,39 +789,22 @@ impl BinaryHeap { swap(self, other); } - if other.is_empty() { - return; - } - - #[inline(always)] - fn log2_fast(x: usize) -> usize { - 8 * size_of::() - (x.leading_zeros() as usize) - 1 - } + let start = self.data.len(); - // `rebuild` takes O(len1 + len2) operations - // and about 2 * (len1 + len2) comparisons in the worst case - // while `extend` takes O(len2 * log_2(len1)) operations - // and about 1 * len2 * log_2(len1) comparisons in the worst case, - // assuming len1 >= len2. - #[inline] - fn better_to_rebuild(len1: usize, len2: usize) -> bool { - 2 * (len1 + len2) < len2 * log2_fast(len1) - } + self.data.append(&mut other.data); - if better_to_rebuild(self.len(), other.len()) { - self.data.append(&mut other.data); - self.rebuild(); - } else { - self.extend(other.drain()); - } + self.rebuild_tail(start); } - /// Returns an iterator which retrieves elements in heap order. - /// The retrieved elements are removed from the original heap. - /// The remaining elements will be removed on drop in heap order. + /// Clears the binary heap, returning an iterator over the removed elements + /// in heap order. If the iterator is dropped before being fully consumed, + /// it drops the remaining elements in heap order. + /// + /// The returned iterator keeps a mutable borrow on the heap to optimize + /// its implementation. /// /// Note: - /// * `.drain_sorted()` is O(n lg n); much slower than `.drain()`. + /// * `.drain_sorted()` is *O*(*n* \* log(*n*)); much slower than `.drain()`. /// You should use the latter for most cases. /// /// # Examples @@ -656,7 +815,7 @@ impl BinaryHeap { /// #![feature(binary_heap_drain_sorted)] /// use std::collections::BinaryHeap; /// - /// let mut heap = BinaryHeap::from(vec![1, 2, 3, 4, 5]); + /// let mut heap = BinaryHeap::from([1, 2, 3, 4, 5]); /// assert_eq!(heap.len(), 5); /// /// drop(heap.drain_sorted()); // removes all elements in heap order @@ -667,6 +826,44 @@ impl BinaryHeap { pub fn drain_sorted(&mut self) -> DrainSorted<'_, T> { DrainSorted { inner: self } } + + /// Retains only the elements specified by the predicate. + /// + /// In other words, remove all elements `e` for which `f(&e)` returns + /// `false`. The elements are visited in unsorted (and unspecified) order. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(binary_heap_retain)] + /// use std::collections::BinaryHeap; + /// + /// let mut heap = BinaryHeap::from([-10, -5, 1, 2, 4, 13]); + /// + /// heap.retain(|x| x % 2 == 0); // only keep even numbers + /// + /// assert_eq!(heap.into_sorted_vec(), [-10, 2, 4]) + /// ``` + #[unstable(feature = "binary_heap_retain", issue = "71503")] + pub fn retain(&mut self, mut f: F) + where + F: FnMut(&T) -> bool, + { + let mut first_removed = self.len(); + let mut i = 0; + self.data.retain(|e| { + let keep = f(e); + if !keep && i < first_removed { + first_removed = i; + } + i += 1; + keep + }); + // data[0..first_removed] is untouched, so we only need to rebuild the tail: + self.rebuild_tail(first_removed); + } } impl BinaryHeap { @@ -679,11 +876,11 @@ impl BinaryHeap { /// /// ``` /// use std::collections::BinaryHeap; - /// let heap = BinaryHeap::from(vec![1, 2, 3, 4]); + /// let heap = BinaryHeap::from([1, 2, 3, 4]); /// /// // Print 1, 2, 3, 4 in arbitrary order /// for x in heap.iter() { - /// println!("{}", x); + /// println!("{x}"); /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] @@ -701,9 +898,9 @@ impl BinaryHeap { /// ``` /// #![feature(binary_heap_into_iter_sorted)] /// use std::collections::BinaryHeap; - /// let heap = BinaryHeap::from(vec![1, 2, 3, 4, 5]); + /// let heap = BinaryHeap::from([1, 2, 3, 4, 5]); /// - /// assert_eq!(heap.into_iter_sorted().take(2).collect::>(), vec![5, 4]); + /// assert_eq!(heap.into_iter_sorted().take(2).collect::>(), [5, 4]); /// ``` #[unstable(feature = "binary_heap_into_iter_sorted", issue = "59278")] pub fn into_iter_sorted(self) -> IntoIterSorted { @@ -730,7 +927,8 @@ impl BinaryHeap { /// /// # Time complexity /// - /// Cost is O(1) in the worst case. + /// Cost is *O*(1) in the worst case. + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn peek(&self) -> Option<&T> { self.data.get(0) @@ -748,21 +946,24 @@ impl BinaryHeap { /// assert!(heap.capacity() >= 100); /// heap.push(4); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn capacity(&self) -> usize { self.data.capacity() } - /// Reserves the minimum capacity for exactly `additional` more elements to be inserted in the - /// given `BinaryHeap`. Does nothing if the capacity is already sufficient. + /// Reserves the minimum capacity for at least `additional` elements more than + /// the current length. Unlike [`reserve`], this will not + /// deliberately over-allocate to speculatively avoid frequent allocations. + /// After calling `reserve_exact`, capacity will be greater than or equal to + /// `self.len() + additional`. Does nothing if the capacity is already + /// sufficient. /// - /// Note that the allocator may give the collection more space than it requests. Therefore - /// capacity can not be relied upon to be precisely minimal. Prefer [`reserve`] if future - /// insertions are expected. + /// [`reserve`]: BinaryHeap::reserve /// /// # Panics /// - /// Panics if the new capacity overflows `usize`. + /// Panics if the new capacity overflows [`usize`]. /// /// # Examples /// @@ -776,18 +977,21 @@ impl BinaryHeap { /// heap.push(4); /// ``` /// - /// [`reserve`]: #method.reserve + /// [`reserve`]: BinaryHeap::reserve #[stable(feature = "rust1", since = "1.0.0")] pub fn reserve_exact(&mut self, additional: usize) { self.data.reserve_exact(additional); } - /// Reserves capacity for at least `additional` more elements to be inserted in the - /// `BinaryHeap`. The collection may reserve more space to avoid frequent reallocations. + /// Reserves capacity for at least `additional` elements more than the + /// current length. The allocator may reserve more space to speculatively + /// avoid frequent allocations. After calling `reserve`, + /// capacity will be greater than or equal to `self.len() + additional`. + /// Does nothing if capacity is already sufficient. /// /// # Panics /// - /// Panics if the new capacity overflows `usize`. + /// Panics if the new capacity overflows [`usize`]. /// /// # Examples /// @@ -805,6 +1009,84 @@ impl BinaryHeap { self.data.reserve(additional); } + /// Tries to reserve the minimum capacity for at least `additional` elements + /// more than the current length. Unlike [`try_reserve`], this will not + /// deliberately over-allocate to speculatively avoid frequent allocations. + /// After calling `try_reserve_exact`, capacity will be greater than or + /// equal to `self.len() + additional` if it returns `Ok(())`. + /// Does nothing if the capacity is already sufficient. + /// + /// Note that the allocator may give the collection more space than it + /// requests. Therefore, capacity can not be relied upon to be precisely + /// minimal. Prefer [`try_reserve`] if future insertions are expected. + /// + /// [`try_reserve`]: BinaryHeap::try_reserve + /// + /// # Errors + /// + /// If the capacity overflows, or the allocator reports a failure, then an error + /// is returned. + /// + /// # Examples + /// + /// ``` + /// use std::collections::BinaryHeap; + /// use std::collections::TryReserveError; + /// + /// fn find_max_slow(data: &[u32]) -> Result, TryReserveError> { + /// let mut heap = BinaryHeap::new(); + /// + /// // Pre-reserve the memory, exiting if we can't + /// heap.try_reserve_exact(data.len())?; + /// + /// // Now we know this can't OOM in the middle of our complex work + /// heap.extend(data.iter()); + /// + /// Ok(heap.pop()) + /// } + /// # find_max_slow(&[1, 2, 3]).expect("why is the test harness OOMing on 12 bytes?"); + /// ``` + #[stable(feature = "try_reserve_2", since = "1.63.0")] + pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.data.try_reserve_exact(additional) + } + + /// Tries to reserve capacity for at least `additional` elements more than the + /// current length. The allocator may reserve more space to speculatively + /// avoid frequent allocations. After calling `try_reserve`, capacity will be + /// greater than or equal to `self.len() + additional` if it returns + /// `Ok(())`. Does nothing if capacity is already sufficient. This method + /// preserves the contents even if an error occurs. + /// + /// # Errors + /// + /// If the capacity overflows, or the allocator reports a failure, then an error + /// is returned. + /// + /// # Examples + /// + /// ``` + /// use std::collections::BinaryHeap; + /// use std::collections::TryReserveError; + /// + /// fn find_max_slow(data: &[u32]) -> Result, TryReserveError> { + /// let mut heap = BinaryHeap::new(); + /// + /// // Pre-reserve the memory, exiting if we can't + /// heap.try_reserve(data.len())?; + /// + /// // Now we know this can't OOM in the middle of our complex work + /// heap.extend(data.iter()); + /// + /// Ok(heap.pop()) + /// } + /// # find_max_slow(&[1, 2, 3]).expect("why is the test harness OOMing on 12 bytes?"); + /// ``` + #[stable(feature = "try_reserve_2", since = "1.63.0")] + pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.data.try_reserve(additional) + } + /// Discards as much additional capacity as possible. /// /// # Examples @@ -829,13 +1111,11 @@ impl BinaryHeap { /// The capacity will remain at least as large as both the length /// and the supplied value. /// - /// Panics if the current capacity is smaller than the supplied - /// minimum capacity. + /// If the current capacity is less than the lower limit, this is a no-op. /// /// # Examples /// /// ``` - /// #![feature(shrink_to)] /// use std::collections::BinaryHeap; /// let mut heap: BinaryHeap = BinaryHeap::with_capacity(100); /// @@ -844,11 +1124,33 @@ impl BinaryHeap { /// assert!(heap.capacity() >= 10); /// ``` #[inline] - #[unstable(feature = "shrink_to", reason = "new API", issue = "56431")] + #[stable(feature = "shrink_to", since = "1.56.0")] pub fn shrink_to(&mut self, min_capacity: usize) { self.data.shrink_to(min_capacity) } + /// Returns a slice of all values in the underlying vector, in arbitrary + /// order. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(binary_heap_as_slice)] + /// use std::collections::BinaryHeap; + /// use std::io::{self, Write}; + /// + /// let heap = BinaryHeap::from([1, 2, 3, 4, 5, 6, 7]); + /// + /// io::sink().write(heap.as_slice()).unwrap(); + /// ``` + #[must_use] + #[unstable(feature = "binary_heap_as_slice", issue = "83659")] + pub fn as_slice(&self) -> &[T] { + self.data.as_slice() + } + /// Consumes the `BinaryHeap` and returns the underlying vector /// in arbitrary order. /// @@ -858,14 +1160,15 @@ impl BinaryHeap { /// /// ``` /// use std::collections::BinaryHeap; - /// let heap = BinaryHeap::from(vec![1, 2, 3, 4, 5, 6, 7]); + /// let heap = BinaryHeap::from([1, 2, 3, 4, 5, 6, 7]); /// let vec = heap.into_vec(); /// /// // Will print in some order /// for x in vec { - /// println!("{}", x); + /// println!("{x}"); /// } /// ``` + #[must_use = "`self` will be dropped if the result is not used"] #[stable(feature = "binary_heap_extras_15", since = "1.5.0")] pub fn into_vec(self) -> Vec { self.into() @@ -879,10 +1182,11 @@ impl BinaryHeap { /// /// ``` /// use std::collections::BinaryHeap; - /// let heap = BinaryHeap::from(vec![1, 3]); + /// let heap = BinaryHeap::from([1, 3]); /// /// assert_eq!(heap.len(), 2); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn len(&self) -> usize { self.data.len() @@ -906,14 +1210,18 @@ impl BinaryHeap { /// /// assert!(!heap.is_empty()); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn is_empty(&self) -> bool { self.len() == 0 } - /// Clears the binary heap, returning an iterator over the removed elements. + /// Clears the binary heap, returning an iterator over the removed elements + /// in arbitrary order. If the iterator is dropped before being fully + /// consumed, it drops the remaining elements in arbitrary order. /// - /// The elements are removed in arbitrary order. + /// The returned iterator keeps a mutable borrow on the heap to optimize + /// its implementation. /// /// # Examples /// @@ -921,12 +1229,12 @@ impl BinaryHeap { /// /// ``` /// use std::collections::BinaryHeap; - /// let mut heap = BinaryHeap::from(vec![1, 3]); + /// let mut heap = BinaryHeap::from([1, 3]); /// /// assert!(!heap.is_empty()); /// /// for x in heap.drain() { - /// println!("{}", x); + /// println!("{x}"); /// } /// /// assert!(heap.is_empty()); @@ -945,7 +1253,7 @@ impl BinaryHeap { /// /// ``` /// use std::collections::BinaryHeap; - /// let mut heap = BinaryHeap::from(vec![1, 3]); + /// let mut heap = BinaryHeap::from([1, 3]); /// /// assert!(!heap.is_empty()); /// @@ -977,7 +1285,7 @@ impl<'a, T> Hole<'a, T> { unsafe fn new(data: &'a mut [T], pos: usize) -> Self { debug_assert!(pos < data.len()); // SAFE: pos should be inside the slice - let elt = ptr::read(data.get_unchecked(pos)); + let elt = unsafe { ptr::read(data.get_unchecked(pos)) }; Hole { data, elt: ManuallyDrop::new(elt), pos } } @@ -999,7 +1307,7 @@ impl<'a, T> Hole<'a, T> { unsafe fn get(&self, index: usize) -> &T { debug_assert!(index != self.pos); debug_assert!(index < self.data.len()); - self.data.get_unchecked(index) + unsafe { self.data.get_unchecked(index) } } /// Move hole to new location @@ -1009,9 +1317,12 @@ impl<'a, T> Hole<'a, T> { unsafe fn move_to(&mut self, index: usize) { debug_assert!(index != self.pos); debug_assert!(index < self.data.len()); - let index_ptr: *const _ = self.data.get_unchecked(index); - let hole_ptr = self.data.get_unchecked_mut(self.pos); - ptr::copy_nonoverlapping(index_ptr, hole_ptr, 1); + unsafe { + let ptr = self.data.as_mut_ptr(); + let index_ptr: *const _ = ptr.add(index); + let hole_ptr = ptr.add(self.pos); + ptr::copy_nonoverlapping(index_ptr, hole_ptr, 1); + } self.pos = index; } } @@ -1029,11 +1340,11 @@ impl Drop for Hole<'_, T> { /// An iterator over the elements of a `BinaryHeap`. /// -/// This `struct` is created by the [`iter`] method on [`BinaryHeap`]. See its +/// This `struct` is created by [`BinaryHeap::iter()`]. See its /// documentation for more. /// -/// [`iter`]: struct.BinaryHeap.html#method.iter -/// [`BinaryHeap`]: struct.BinaryHeap.html +/// [`iter`]: BinaryHeap::iter +#[must_use = "iterators are lazy and do nothing unless consumed"] #[stable(feature = "rust1", since = "1.0.0")] pub struct Iter<'a, T: 'a> { iter: slice::Iter<'a, T>, @@ -1094,11 +1405,11 @@ impl FusedIterator for Iter<'_, T> {} /// An owning iterator over the elements of a `BinaryHeap`. /// -/// This `struct` is created by the [`into_iter`] method on [`BinaryHeap`] -/// (provided by the `IntoIterator` trait). See its documentation for more. +/// This `struct` is created by [`BinaryHeap::into_iter()`] +/// (provided by the [`IntoIterator`] trait). See its documentation for more. /// -/// [`into_iter`]: struct.BinaryHeap.html#method.into_iter -/// [`BinaryHeap`]: struct.BinaryHeap.html +/// [`into_iter`]: BinaryHeap::into_iter +/// [`IntoIterator`]: core::iter::IntoIterator #[stable(feature = "rust1", since = "1.0.0")] #[derive(Clone)] pub struct IntoIter { @@ -1145,6 +1456,32 @@ impl ExactSizeIterator for IntoIter { #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for IntoIter {} +// In addition to the SAFETY invariants of the following three unsafe traits +// also refer to the vec::in_place_collect module documentation to get an overview +#[unstable(issue = "none", feature = "inplace_iteration")] +#[doc(hidden)] +unsafe impl SourceIter for IntoIter { + type Source = IntoIter; + + #[inline] + unsafe fn as_inner(&mut self) -> &mut Self::Source { + self + } +} + +#[unstable(issue = "none", feature = "inplace_iteration")] +#[doc(hidden)] +unsafe impl InPlaceIterable for IntoIter {} + +unsafe impl AsVecIntoIter for IntoIter { + type Item = I; + + fn as_into_iter(&mut self) -> &mut vec::IntoIter { + &mut self.iter + } +} + +#[must_use = "iterators are lazy and do nothing unless consumed"] #[unstable(feature = "binary_heap_into_iter_sorted", issue = "59278")] #[derive(Clone, Debug)] pub struct IntoIterSorted { @@ -1178,11 +1515,10 @@ unsafe impl TrustedLen for IntoIterSorted {} /// A draining iterator over the elements of a `BinaryHeap`. /// -/// This `struct` is created by the [`drain`] method on [`BinaryHeap`]. See its +/// This `struct` is created by [`BinaryHeap::drain()`]. See its /// documentation for more. /// -/// [`drain`]: struct.BinaryHeap.html#method.drain -/// [`BinaryHeap`]: struct.BinaryHeap.html +/// [`drain`]: BinaryHeap::drain #[stable(feature = "drain", since = "1.6.0")] #[derive(Debug)] pub struct Drain<'a, T: 'a> { @@ -1224,11 +1560,10 @@ impl FusedIterator for Drain<'_, T> {} /// A draining iterator over the elements of a `BinaryHeap`. /// -/// This `struct` is created by the [`drain_sorted`] method on [`BinaryHeap`]. See its +/// This `struct` is created by [`BinaryHeap::drain_sorted()`]. See its /// documentation for more. /// -/// [`drain_sorted`]: struct.BinaryHeap.html#method.drain_sorted -/// [`BinaryHeap`]: struct.BinaryHeap.html +/// [`drain_sorted`]: BinaryHeap::drain_sorted #[unstable(feature = "binary_heap_drain_sorted", issue = "59278")] #[derive(Debug)] pub struct DrainSorted<'a, T: Ord> { @@ -1243,7 +1578,7 @@ impl<'a, T: Ord> Drop for DrainSorted<'a, T> { impl<'r, 'a, T: Ord> Drop for DropGuard<'r, 'a, T> { fn drop(&mut self) { - while let Some(_) = self.0.inner.pop() {} + while self.0.inner.pop().is_some() {} } } @@ -1284,7 +1619,7 @@ unsafe impl TrustedLen for DrainSorted<'_, T> {} impl From> for BinaryHeap { /// Converts a `Vec` into a `BinaryHeap`. /// - /// This conversion happens in-place, and has `O(n)` time complexity. + /// This conversion happens in-place, and has *O*(*n*) time complexity. fn from(vec: Vec) -> BinaryHeap { let mut heap = BinaryHeap { data: vec }; heap.rebuild(); @@ -1292,8 +1627,28 @@ impl From> for BinaryHeap { } } +#[stable(feature = "std_collections_from_array", since = "1.56.0")] +impl From<[T; N]> for BinaryHeap { + /// ``` + /// use std::collections::BinaryHeap; + /// + /// let mut h1 = BinaryHeap::from([1, 4, 2, 3]); + /// let mut h2: BinaryHeap<_> = [1, 4, 2, 3].into(); + /// while let Some((a, b)) = h1.pop().zip(h2.pop()) { + /// assert_eq!(a, b); + /// } + /// ``` + fn from(arr: [T; N]) -> Self { + Self::from_iter(arr) + } +} + #[stable(feature = "binary_heap_extras_15", since = "1.5.0")] impl From> for Vec { + /// Converts a `BinaryHeap` into a `Vec`. + /// + /// This conversion requires no data movement or allocation, and has + /// constant time complexity. fn from(heap: BinaryHeap) -> Vec { heap.data } @@ -1321,12 +1676,12 @@ impl IntoIterator for BinaryHeap { /// /// ``` /// use std::collections::BinaryHeap; - /// let heap = BinaryHeap::from(vec![1, 2, 3, 4]); + /// let heap = BinaryHeap::from([1, 2, 3, 4]); /// /// // Print 1, 2, 3, 4 in arbitrary order /// for x in heap.into_iter() { /// // x has type i32, not &i32 - /// println!("{}", x); + /// println!("{x}"); /// } /// ``` fn into_iter(self) -> IntoIter { @@ -1350,6 +1705,16 @@ impl Extend for BinaryHeap { fn extend>(&mut self, iter: I) { >::spec_extend(self, iter); } + + #[inline] + fn extend_one(&mut self, item: T) { + self.push(item); + } + + #[inline] + fn extend_reserve(&mut self, additional: usize) { + self.reserve(additional); + } } impl> SpecExtend for BinaryHeap { @@ -1358,6 +1723,14 @@ impl> SpecExtend for BinaryHeap { } } +impl SpecExtend> for BinaryHeap { + fn spec_extend(&mut self, ref mut other: Vec) { + let start = self.data.len(); + self.data.append(other); + self.rebuild_tail(start); + } +} + impl SpecExtend> for BinaryHeap { fn spec_extend(&mut self, ref mut other: BinaryHeap) { self.append(other); @@ -1380,4 +1753,14 @@ impl<'a, T: 'a + Ord + Copy> Extend<&'a T> for BinaryHeap { fn extend>(&mut self, iter: I) { self.extend(iter.into_iter().cloned()); } + + #[inline] + fn extend_one(&mut self, &item: &'a T) { + self.push(item); + } + + #[inline] + fn extend_reserve(&mut self, additional: usize) { + self.reserve(additional); + } } diff --git a/crux-mir/lib/alloc/tests/binary_heap.rs b/crux-mir/lib/alloc/src/collections/binary_heap/tests.rs similarity index 71% rename from crux-mir/lib/alloc/tests/binary_heap.rs rename to crux-mir/lib/alloc/src/collections/binary_heap/tests.rs index be5516f54..ffbb6c80a 100644 --- a/crux-mir/lib/alloc/tests/binary_heap.rs +++ b/crux-mir/lib/alloc/src/collections/binary_heap/tests.rs @@ -1,8 +1,9 @@ -use std::collections::binary_heap::{Drain, PeekMut}; -use std::collections::BinaryHeap; +use super::*; +use crate::boxed::Box; +use crate::testing::crash_test::{CrashTestDummy, Panic}; +use core::mem; use std::iter::TrustedLen; use std::panic::{catch_unwind, AssertUnwindSafe}; -use std::sync::atomic::{AtomicU32, Ordering}; #[test] fn test_iterator() { @@ -146,6 +147,24 @@ fn test_peek_mut() { assert_eq!(heap.peek(), Some(&9)); } +#[test] +fn test_peek_mut_leek() { + let data = vec![4, 2, 7]; + let mut heap = BinaryHeap::from(data); + let mut max = heap.peek_mut().unwrap(); + *max = -1; + + // The PeekMut object's Drop impl would have been responsible for moving the + // -1 out of the max position of the BinaryHeap, but we don't run it. + mem::forget(max); + + // Absent some mitigation like leak amplification, the -1 would incorrectly + // end up in the last position of the returned Vec, with the rest of the + // heap's original contents in front of it in sorted order. + let sorted_vec = heap.into_sorted_vec(); + assert!(sorted_vec.is_sorted(), "{:?}", sorted_vec); +} + #[test] fn test_peek_mut_pop() { let data = vec![2, 4, 6, 2, 1, 8, 10, 3, 5, 7, 0, 9, 1]; @@ -183,22 +202,22 @@ fn test_push() { #[test] fn test_push_unique() { - let mut heap = BinaryHeap::>::from(vec![box 2, box 4, box 9]); + let mut heap = BinaryHeap::>::from(vec![Box::new(2), Box::new(4), Box::new(9)]); assert_eq!(heap.len(), 3); assert!(**heap.peek().unwrap() == 9); - heap.push(box 11); + heap.push(Box::new(11)); assert_eq!(heap.len(), 4); assert!(**heap.peek().unwrap() == 11); - heap.push(box 5); + heap.push(Box::new(5)); assert_eq!(heap.len(), 5); assert!(**heap.peek().unwrap() == 11); - heap.push(box 27); + heap.push(Box::new(27)); assert_eq!(heap.len(), 6); assert!(**heap.peek().unwrap() == 27); - heap.push(box 3); + heap.push(Box::new(3)); assert_eq!(heap.len(), 7); assert!(**heap.peek().unwrap() == 27); - heap.push(box 103); + heap.push(Box::new(103)); assert_eq!(heap.len(), 8); assert!(**heap.peek().unwrap() == 103); } @@ -230,6 +249,18 @@ fn test_to_vec() { check_to_vec(vec![5, 4, 3, 2, 1, 5, 4, 3, 2, 1, 5, 4, 3, 2, 1]); } +#[test] +fn test_in_place_iterator_specialization() { + let src: Vec = vec![1, 2, 3]; + let src_ptr = src.as_ptr(); + let heap: BinaryHeap<_> = src.into_iter().map(std::convert::identity).collect(); + let heap_ptr = heap.iter().next().unwrap() as *const usize; + assert_eq!(src_ptr, heap_ptr); + let sink: Vec<_> = heap.into_iter().map(std::convert::identity).collect(); + let sink_ptr = sink.as_ptr(); + assert_eq!(heap_ptr, sink_ptr); +} + #[test] fn test_empty_pop() { let mut heap = BinaryHeap::::new(); @@ -279,33 +310,83 @@ fn test_drain_sorted() { #[test] fn test_drain_sorted_leak() { - static DROPS: AtomicU32 = AtomicU32::new(0); - - #[derive(Clone, PartialEq, Eq, PartialOrd, Ord)] - struct D(u32, bool); - - impl Drop for D { - fn drop(&mut self) { - DROPS.fetch_add(1, Ordering::SeqCst); - - if self.1 { - panic!("panic in `drop`"); - } - } - } - + let d0 = CrashTestDummy::new(0); + let d1 = CrashTestDummy::new(1); + let d2 = CrashTestDummy::new(2); + let d3 = CrashTestDummy::new(3); + let d4 = CrashTestDummy::new(4); + let d5 = CrashTestDummy::new(5); let mut q = BinaryHeap::from(vec![ - D(0, false), - D(1, false), - D(2, false), - D(3, true), - D(4, false), - D(5, false), + d0.spawn(Panic::Never), + d1.spawn(Panic::Never), + d2.spawn(Panic::Never), + d3.spawn(Panic::InDrop), + d4.spawn(Panic::Never), + d5.spawn(Panic::Never), ]); - catch_unwind(AssertUnwindSafe(|| drop(q.drain_sorted()))).ok(); + catch_unwind(AssertUnwindSafe(|| drop(q.drain_sorted()))).unwrap_err(); - assert_eq!(DROPS.load(Ordering::SeqCst), 6); + assert_eq!(d0.dropped(), 1); + assert_eq!(d1.dropped(), 1); + assert_eq!(d2.dropped(), 1); + assert_eq!(d3.dropped(), 1); + assert_eq!(d4.dropped(), 1); + assert_eq!(d5.dropped(), 1); + assert!(q.is_empty()); +} + +#[test] +fn test_drain_forget() { + let a = CrashTestDummy::new(0); + let b = CrashTestDummy::new(1); + let c = CrashTestDummy::new(2); + let mut q = + BinaryHeap::from(vec![a.spawn(Panic::Never), b.spawn(Panic::Never), c.spawn(Panic::Never)]); + + catch_unwind(AssertUnwindSafe(|| { + let mut it = q.drain(); + it.next(); + mem::forget(it); + })) + .unwrap(); + // Behaviour after leaking is explicitly unspecified and order is arbitrary, + // so it's fine if these start failing, but probably worth knowing. + assert!(q.is_empty()); + assert_eq!(a.dropped() + b.dropped() + c.dropped(), 1); + assert_eq!(a.dropped(), 0); + assert_eq!(b.dropped(), 0); + assert_eq!(c.dropped(), 1); + drop(q); + assert_eq!(a.dropped(), 0); + assert_eq!(b.dropped(), 0); + assert_eq!(c.dropped(), 1); +} + +#[test] +fn test_drain_sorted_forget() { + let a = CrashTestDummy::new(0); + let b = CrashTestDummy::new(1); + let c = CrashTestDummy::new(2); + let mut q = + BinaryHeap::from(vec![a.spawn(Panic::Never), b.spawn(Panic::Never), c.spawn(Panic::Never)]); + + catch_unwind(AssertUnwindSafe(|| { + let mut it = q.drain_sorted(); + it.next(); + mem::forget(it); + })) + .unwrap(); + // Behaviour after leaking is explicitly unspecified, + // so it's fine if these start failing, but probably worth knowing. + assert_eq!(q.len(), 2); + assert_eq!(a.dropped(), 0); + assert_eq!(b.dropped(), 0); + assert_eq!(c.dropped(), 1); + drop(q); + assert_eq!(a.dropped(), 1); + assert_eq!(b.dropped(), 1); + assert_eq!(c.dropped(), 1); } #[test] @@ -372,17 +453,38 @@ fn assert_covariance() { } } +#[test] +fn test_retain() { + let mut a = BinaryHeap::from(vec![100, 10, 50, 1, 2, 20, 30]); + a.retain(|&x| x != 2); + + // Check that 20 moved into 10's place. + assert_eq!(a.clone().into_vec(), [100, 20, 50, 1, 10, 30]); + + a.retain(|_| true); + + assert_eq!(a.clone().into_vec(), [100, 20, 50, 1, 10, 30]); + + a.retain(|&x| x < 50); + + assert_eq!(a.clone().into_vec(), [30, 20, 10, 1]); + + a.retain(|_| false); + + assert!(a.is_empty()); +} + // old binaryheap failed this test // // Integrity means that all elements are present after a comparison panics, -// even if the order may not be correct. +// even if the order might not be correct. // // Destructors must be called exactly once per element. // FIXME: re-enable emscripten once it can unwind again #[test] #[cfg(not(target_os = "emscripten"))] fn panic_safe() { - use rand::{seq::SliceRandom, thread_rng}; + use rand::seq::SliceRandom; use std::cmp; use std::panic::{self, AssertUnwindSafe}; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -407,18 +509,16 @@ fn panic_safe() { self.0.partial_cmp(&other.0) } } - let mut rng = thread_rng(); + let mut rng = crate::test_helpers::test_rng(); const DATASZ: usize = 32; - #[cfg(not(miri))] // Miri is too slow - const NTEST: usize = 10; - #[cfg(miri)] - const NTEST: usize = 1; + // Miri is too slow + let ntest = if cfg!(miri) { 1 } else { 10 }; // don't use 0 in the data -- we want to catch the zeroed-out case. let data = (1..=DATASZ).collect::>(); // since it's a fuzzy test, run several tries. - for _ in 0..NTEST { + for _ in 0..ntest { for i in 1..=DATASZ { DROP_COUNTER.store(0, Ordering::SeqCst); diff --git a/crux-mir/lib/alloc/src/collections/btree/append.rs b/crux-mir/lib/alloc/src/collections/btree/append.rs new file mode 100644 index 000000000..b6989afb6 --- /dev/null +++ b/crux-mir/lib/alloc/src/collections/btree/append.rs @@ -0,0 +1,107 @@ +use super::merge_iter::MergeIterInner; +use super::node::{self, Root}; +use core::alloc::Allocator; +use core::iter::FusedIterator; + +impl Root { + /// Appends all key-value pairs from the union of two ascending iterators, + /// incrementing a `length` variable along the way. The latter makes it + /// easier for the caller to avoid a leak when a drop handler panicks. + /// + /// If both iterators produce the same key, this method drops the pair from + /// the left iterator and appends the pair from the right iterator. + /// + /// If you want the tree to end up in a strictly ascending order, like for + /// a `BTreeMap`, both iterators should produce keys in strictly ascending + /// order, each greater than all keys in the tree, including any keys + /// already in the tree upon entry. + pub fn append_from_sorted_iters( + &mut self, + left: I, + right: I, + length: &mut usize, + alloc: A, + ) where + K: Ord, + I: Iterator + FusedIterator, + { + // We prepare to merge `left` and `right` into a sorted sequence in linear time. + let iter = MergeIter(MergeIterInner::new(left, right)); + + // Meanwhile, we build a tree from the sorted sequence in linear time. + self.bulk_push(iter, length, alloc) + } + + /// Pushes all key-value pairs to the end of the tree, incrementing a + /// `length` variable along the way. The latter makes it easier for the + /// caller to avoid a leak when the iterator panicks. + pub fn bulk_push(&mut self, iter: I, length: &mut usize, alloc: A) + where + I: Iterator, + { + let mut cur_node = self.borrow_mut().last_leaf_edge().into_node(); + // Iterate through all key-value pairs, pushing them into nodes at the right level. + for (key, value) in iter { + // Try to push key-value pair into the current leaf node. + if cur_node.len() < node::CAPACITY { + cur_node.push(key, value); + } else { + // No space left, go up and push there. + let mut open_node; + let mut test_node = cur_node.forget_type(); + loop { + match test_node.ascend() { + Ok(parent) => { + let parent = parent.into_node(); + if parent.len() < node::CAPACITY { + // Found a node with space left, push here. + open_node = parent; + break; + } else { + // Go up again. + test_node = parent.forget_type(); + } + } + Err(_) => { + // We are at the top, create a new root node and push there. + open_node = self.push_internal_level(alloc.clone()); + break; + } + } + } + + // Push key-value pair and new right subtree. + let tree_height = open_node.height() - 1; + let mut right_tree = Root::new(alloc.clone()); + for _ in 0..tree_height { + right_tree.push_internal_level(alloc.clone()); + } + open_node.push(key, value, right_tree); + + // Go down to the right-most leaf again. + cur_node = open_node.forget_type().last_leaf_edge().into_node(); + } + + // Increment length every iteration, to make sure the map drops + // the appended elements even if advancing the iterator panicks. + *length += 1; + } + self.fix_right_border_of_plentiful(); + } +} + +// An iterator for merging two sorted sequences into one +struct MergeIter>(MergeIterInner); + +impl Iterator for MergeIter +where + I: Iterator + FusedIterator, +{ + type Item = (K, V); + + /// If two keys are equal, returns the key-value pair from the right source. + fn next(&mut self) -> Option<(K, V)> { + let (a_next, b_next) = self.0.nexts(|a: &(K, V), b: &(K, V)| K::cmp(&a.0, &b.0)); + b_next.or(a_next) + } +} diff --git a/crux-mir/lib/alloc/src/collections/btree/borrow.rs b/crux-mir/lib/alloc/src/collections/btree/borrow.rs new file mode 100644 index 000000000..016f139a5 --- /dev/null +++ b/crux-mir/lib/alloc/src/collections/btree/borrow.rs @@ -0,0 +1,47 @@ +use core::marker::PhantomData; +use core::ptr::NonNull; + +/// Models a reborrow of some unique reference, when you know that the reborrow +/// and all its descendants (i.e., all pointers and references derived from it) +/// will not be used any more at some point, after which you want to use the +/// original unique reference again. +/// +/// The borrow checker usually handles this stacking of borrows for you, but +/// some control flows that accomplish this stacking are too complicated for +/// the compiler to follow. A `DormantMutRef` allows you to check borrowing +/// yourself, while still expressing its stacked nature, and encapsulating +/// the raw pointer code needed to do this without undefined behavior. +pub struct DormantMutRef<'a, T> { + ptr: NonNull, + _marker: PhantomData<&'a mut T>, +} + +unsafe impl<'a, T> Sync for DormantMutRef<'a, T> where &'a mut T: Sync {} +unsafe impl<'a, T> Send for DormantMutRef<'a, T> where &'a mut T: Send {} + +impl<'a, T> DormantMutRef<'a, T> { + /// Capture a unique borrow, and immediately reborrow it. For the compiler, + /// the lifetime of the new reference is the same as the lifetime of the + /// original reference, but you promise to use it for a shorter period. + pub fn new(t: &'a mut T) -> (&'a mut T, Self) { + let ptr = NonNull::from(t); + // SAFETY: we hold the borrow throughout 'a via `_marker`, and we expose + // only this reference, so it is unique. + let new_ref = unsafe { &mut *ptr.as_ptr() }; + (new_ref, Self { ptr, _marker: PhantomData }) + } + + /// Revert to the unique borrow initially captured. + /// + /// # Safety + /// + /// The reborrow must have ended, i.e., the reference returned by `new` and + /// all pointers and references derived from it, must not be used anymore. + pub unsafe fn awaken(self) -> &'a mut T { + // SAFETY: our own safety conditions imply this reference is again unique. + unsafe { &mut *self.ptr.as_ptr() } + } +} + +#[cfg(test)] +mod tests; diff --git a/crux-mir/lib/alloc/src/collections/btree/borrow/tests.rs b/crux-mir/lib/alloc/src/collections/btree/borrow/tests.rs new file mode 100644 index 000000000..56a8434fc --- /dev/null +++ b/crux-mir/lib/alloc/src/collections/btree/borrow/tests.rs @@ -0,0 +1,19 @@ +use super::DormantMutRef; + +#[test] +fn test_borrow() { + let mut data = 1; + let mut stack = vec![]; + let mut rr = &mut data; + for factor in [2, 3, 7].iter() { + let (r, dormant_r) = DormantMutRef::new(rr); + rr = r; + assert_eq!(*rr, 1); + stack.push((factor, dormant_r)); + } + while let Some((factor, dormant_r)) = stack.pop() { + let r = unsafe { dormant_r.awaken() }; + *r *= factor; + } + assert_eq!(data, 42); +} diff --git a/crux-mir/lib/alloc/src/collections/btree/dedup_sorted_iter.rs b/crux-mir/lib/alloc/src/collections/btree/dedup_sorted_iter.rs new file mode 100644 index 000000000..17ee78045 --- /dev/null +++ b/crux-mir/lib/alloc/src/collections/btree/dedup_sorted_iter.rs @@ -0,0 +1,49 @@ +use core::iter::Peekable; + +/// A iterator for deduping the key of a sorted iterator. +/// When encountering the duplicated key, only the last key-value pair is yielded. +/// +/// Used by [`BTreeMap::bulk_build_from_sorted_iter`][1]. +/// +/// [1]: crate::collections::BTreeMap::bulk_build_from_sorted_iter +pub struct DedupSortedIter +where + I: Iterator, +{ + iter: Peekable, +} + +impl DedupSortedIter +where + I: Iterator, +{ + pub fn new(iter: I) -> Self { + Self { iter: iter.peekable() } + } +} + +impl Iterator for DedupSortedIter +where + K: Eq, + I: Iterator, +{ + type Item = (K, V); + + fn next(&mut self) -> Option<(K, V)> { + loop { + let next = match self.iter.next() { + Some(next) => next, + None => return None, + }; + + let peeked = match self.iter.peek() { + Some(peeked) => peeked, + None => return Some(next), + }; + + if next.0 != peeked.0 { + return Some(next); + } + } + } +} diff --git a/crux-mir/lib/alloc/src/collections/btree/fix.rs b/crux-mir/lib/alloc/src/collections/btree/fix.rs new file mode 100644 index 000000000..91b612180 --- /dev/null +++ b/crux-mir/lib/alloc/src/collections/btree/fix.rs @@ -0,0 +1,179 @@ +use super::map::MIN_LEN; +use super::node::{marker, ForceResult::*, Handle, LeftOrRight::*, NodeRef, Root}; +use core::alloc::Allocator; + +impl<'a, K: 'a, V: 'a> NodeRef, K, V, marker::LeafOrInternal> { + /// Stocks up a possibly underfull node by merging with or stealing from a + /// sibling. If successful but at the cost of shrinking the parent node, + /// returns that shrunk parent node. Returns an `Err` if the node is + /// an empty root. + fn fix_node_through_parent( + self, + alloc: A, + ) -> Result, K, V, marker::Internal>>, Self> { + let len = self.len(); + if len >= MIN_LEN { + Ok(None) + } else { + match self.choose_parent_kv() { + Ok(Left(mut left_parent_kv)) => { + if left_parent_kv.can_merge() { + let parent = left_parent_kv.merge_tracking_parent(alloc); + Ok(Some(parent)) + } else { + left_parent_kv.bulk_steal_left(MIN_LEN - len); + Ok(None) + } + } + Ok(Right(mut right_parent_kv)) => { + if right_parent_kv.can_merge() { + let parent = right_parent_kv.merge_tracking_parent(alloc); + Ok(Some(parent)) + } else { + right_parent_kv.bulk_steal_right(MIN_LEN - len); + Ok(None) + } + } + Err(root) => { + if len > 0 { + Ok(None) + } else { + Err(root) + } + } + } + } + } +} + +impl<'a, K: 'a, V: 'a> NodeRef, K, V, marker::LeafOrInternal> { + /// Stocks up a possibly underfull node, and if that causes its parent node + /// to shrink, stocks up the parent, recursively. + /// Returns `true` if it fixed the tree, `false` if it couldn't because the + /// root node became empty. + /// + /// This method does not expect ancestors to already be underfull upon entry + /// and panics if it encounters an empty ancestor. + pub fn fix_node_and_affected_ancestors(mut self, alloc: A) -> bool { + loop { + match self.fix_node_through_parent(alloc.clone()) { + Ok(Some(parent)) => self = parent.forget_type(), + Ok(None) => return true, + Err(_) => return false, + } + } + } +} + +impl Root { + /// Removes empty levels on the top, but keeps an empty leaf if the entire tree is empty. + pub fn fix_top(&mut self, alloc: A) { + while self.height() > 0 && self.len() == 0 { + self.pop_internal_level(alloc.clone()); + } + } + + /// Stocks up or merge away any underfull nodes on the right border of the + /// tree. The other nodes, those that are not the root nor a rightmost edge, + /// must already have at least MIN_LEN elements. + pub fn fix_right_border(&mut self, alloc: A) { + self.fix_top(alloc.clone()); + if self.len() > 0 { + self.borrow_mut().last_kv().fix_right_border_of_right_edge(alloc.clone()); + self.fix_top(alloc); + } + } + + /// The symmetric clone of `fix_right_border`. + pub fn fix_left_border(&mut self, alloc: A) { + self.fix_top(alloc.clone()); + if self.len() > 0 { + self.borrow_mut().first_kv().fix_left_border_of_left_edge(alloc.clone()); + self.fix_top(alloc); + } + } + + /// Stocks up any underfull nodes on the right border of the tree. + /// The other nodes, those that are neither the root nor a rightmost edge, + /// must be prepared to have up to MIN_LEN elements stolen. + pub fn fix_right_border_of_plentiful(&mut self) { + let mut cur_node = self.borrow_mut(); + while let Internal(internal) = cur_node.force() { + // Check if right-most child is underfull. + let mut last_kv = internal.last_kv().consider_for_balancing(); + debug_assert!(last_kv.left_child_len() >= MIN_LEN * 2); + let right_child_len = last_kv.right_child_len(); + if right_child_len < MIN_LEN { + // We need to steal. + last_kv.bulk_steal_left(MIN_LEN - right_child_len); + } + + // Go further down. + cur_node = last_kv.into_right_child(); + } + } +} + +impl<'a, K: 'a, V: 'a> Handle, K, V, marker::LeafOrInternal>, marker::KV> { + fn fix_left_border_of_left_edge(mut self, alloc: A) { + while let Internal(internal_kv) = self.force() { + self = internal_kv.fix_left_child(alloc.clone()).first_kv(); + debug_assert!(self.reborrow().into_node().len() > MIN_LEN); + } + } + + fn fix_right_border_of_right_edge(mut self, alloc: A) { + while let Internal(internal_kv) = self.force() { + self = internal_kv.fix_right_child(alloc.clone()).last_kv(); + debug_assert!(self.reborrow().into_node().len() > MIN_LEN); + } + } +} + +impl<'a, K: 'a, V: 'a> Handle, K, V, marker::Internal>, marker::KV> { + /// Stocks up the left child, assuming the right child isn't underfull, and + /// provisions an extra element to allow merging its children in turn + /// without becoming underfull. + /// Returns the left child. + fn fix_left_child( + self, + alloc: A, + ) -> NodeRef, K, V, marker::LeafOrInternal> { + let mut internal_kv = self.consider_for_balancing(); + let left_len = internal_kv.left_child_len(); + debug_assert!(internal_kv.right_child_len() >= MIN_LEN); + if internal_kv.can_merge() { + internal_kv.merge_tracking_child(alloc) + } else { + // `MIN_LEN + 1` to avoid readjust if merge happens on the next level. + let count = (MIN_LEN + 1).saturating_sub(left_len); + if count > 0 { + internal_kv.bulk_steal_right(count); + } + internal_kv.into_left_child() + } + } + + /// Stocks up the right child, assuming the left child isn't underfull, and + /// provisions an extra element to allow merging its children in turn + /// without becoming underfull. + /// Returns wherever the right child ended up. + fn fix_right_child( + self, + alloc: A, + ) -> NodeRef, K, V, marker::LeafOrInternal> { + let mut internal_kv = self.consider_for_balancing(); + let right_len = internal_kv.right_child_len(); + debug_assert!(internal_kv.left_child_len() >= MIN_LEN); + if internal_kv.can_merge() { + internal_kv.merge_tracking_child(alloc) + } else { + // `MIN_LEN + 1` to avoid readjust if merge happens on the next level. + let count = (MIN_LEN + 1).saturating_sub(right_len); + if count > 0 { + internal_kv.bulk_steal_left(count); + } + internal_kv.into_right_child() + } + } +} diff --git a/crux-mir/lib/alloc/src/collections/btree/map.rs b/crux-mir/lib/alloc/src/collections/btree/map.rs index 3ba7befc0..1d9c4460e 100644 --- a/crux-mir/lib/alloc/src/collections/btree/map.rs +++ b/crux-mir/lib/alloc/src/collections/btree/map.rs @@ -1,21 +1,43 @@ +use crate::vec::Vec; use core::borrow::Borrow; use core::cmp::Ordering; -use core::fmt::Debug; +use core::fmt::{self, Debug}; use core::hash::{Hash, Hasher}; -use core::iter::{FromIterator, FusedIterator, Peekable}; +use core::iter::{FromIterator, FusedIterator}; use core::marker::PhantomData; -use core::ops::Bound::{Excluded, Included, Unbounded}; +use core::mem::{self, ManuallyDrop}; use core::ops::{Index, RangeBounds}; -use core::{fmt, mem, ptr}; +use core::ptr; -use super::node::{self, marker, ForceResult::*, Handle, InsertResult::*, NodeRef}; -use super::search::{self, SearchResult::*}; -use super::unwrap_unchecked; +use crate::alloc::{Allocator, Global}; + +use super::borrow::DormantMutRef; +use super::dedup_sorted_iter::DedupSortedIter; +use super::navigate::{LazyLeafRange, LeafRange}; +use super::node::{self, marker, ForceResult::*, Handle, NodeRef, Root}; +use super::search::SearchResult::*; +use super::set_val::SetValZST; + +mod entry; + +#[stable(feature = "rust1", since = "1.0.0")] +pub use entry::{Entry, OccupiedEntry, OccupiedError, VacantEntry}; use Entry::*; -use UnderflowResult::*; -/// A map based on a B-Tree. +/// Minimum number of elements in a node that is not a root. +/// We might temporarily have fewer elements during methods. +pub(super) const MIN_LEN: usize = node::MIN_LEN_AFTER_SPLIT; + +// A tree in a `BTreeMap` is a tree in the `node` module with additional invariants: +// - Keys must appear in ascending order (according to the key's type). +// - Every non-leaf node contains at least 1 element (has at least 2 children). +// - Every non-root node contains at least MIN_LEN elements. +// +// An empty map is represented either by the absence of a root node or by a +// root node that is an empty leaf. + +/// An ordered map based on a [B-Tree]. /// /// B-Trees represent a fundamental compromise between cache-efficiency and actually minimizing /// the amount of work performed in a search. In theory, a binary search tree (BST) is the optimal @@ -24,8 +46,8 @@ use UnderflowResult::*; /// is done is *very* inefficient for modern computer architectures. In particular, every element /// is stored in its own individually heap-allocated node. This means that every single insertion /// triggers a heap-allocation, and every single comparison should be a cache-miss. Since these -/// are both notably expensive things to do in practice, we are forced to at very least reconsider -/// the BST strategy. +/// are both notably expensive things to do in practice, we are forced to, at the very least, +/// reconsider the BST strategy. /// /// A B-Tree instead makes each node contain B-1 to 2B-1 elements in a contiguous array. By doing /// this, we reduce the number of allocations by a factor of B, and improve cache efficiency in @@ -39,16 +61,23 @@ use UnderflowResult::*; /// performance on *small* nodes of elements which are cheap to compare. However in the future we /// would like to further explore choosing the optimal search strategy based on the choice of B, /// and possibly other factors. Using linear search, searching for a random element is expected -/// to take O(B logBn) comparisons, which is generally worse than a BST. In practice, +/// to take B * log(n) comparisons, which is generally worse than a BST. In practice, /// however, performance is excellent. /// /// It is a logic error for a key to be modified in such a way that the key's ordering relative to /// any other key, as determined by the [`Ord`] trait, changes while it is in the map. This is /// normally only possible through [`Cell`], [`RefCell`], global state, I/O, or unsafe code. +/// The behavior resulting from such a logic error is not specified, but will be encapsulated to the +/// `BTreeMap` that observed the logic error and not result in undefined behavior. This could +/// include panics, incorrect results, aborts, memory leaks, and non-termination. /// -/// [`Ord`]: ../../std/cmp/trait.Ord.html -/// [`Cell`]: ../../std/cell/struct.Cell.html -/// [`RefCell`]: ../../std/cell/struct.RefCell.html +/// Iterators obtained from functions such as [`BTreeMap::iter`], [`BTreeMap::values`], or +/// [`BTreeMap::keys`] produce their items in order by key, and take worst-case logarithmic and +/// amortized constant time per item returned. +/// +/// [B-Tree]: https://en.wikipedia.org/wiki/B-tree +/// [`Cell`]: core::cell::Cell +/// [`RefCell`]: core::cell::RefCell /// /// # Examples /// @@ -78,8 +107,8 @@ use UnderflowResult::*; /// let to_find = ["Up!", "Office Space"]; /// for movie in &to_find { /// match movie_reviews.get(movie) { -/// Some(review) => println!("{}: {}", movie, review), -/// None => println!("{} is unreviewed.", movie) +/// Some(review) => println!("{movie}: {review}"), +/// None => println!("{movie} is unreviewed.") /// } /// } /// @@ -88,13 +117,27 @@ use UnderflowResult::*; /// /// // iterate over everything. /// for (movie, review) in &movie_reviews { -/// println!("{}: \"{}\"", movie, review); +/// println!("{movie}: \"{review}\""); /// } /// ``` /// -/// `BTreeMap` also implements an [`Entry API`](#method.entry), which allows -/// for more complex methods of getting, setting, updating and removing keys and -/// their values: +/// A `BTreeMap` with a known list of items can be initialized from an array: +/// +/// ``` +/// use std::collections::BTreeMap; +/// +/// let solar_distance = BTreeMap::from([ +/// ("Mercury", 0.4), +/// ("Venus", 0.7), +/// ("Earth", 1.0), +/// ("Mars", 1.5), +/// ]); +/// ``` +/// +/// `BTreeMap` implements an [`Entry API`], which allows for complex +/// methods of getting, setting, updating and removing keys and their values: +/// +/// [`Entry API`]: BTreeMap::entry /// /// ``` /// use std::collections::BTreeMap; @@ -119,39 +162,69 @@ use UnderflowResult::*; /// // update a key, guarding against the key possibly not being set /// let stat = player_stats.entry("attack").or_insert(100); /// *stat += random_stat_buff(); +/// +/// // modify an entry before an insert with in-place mutation +/// player_stats.entry("mana").and_modify(|mana| *mana += 200).or_insert(100); /// ``` #[stable(feature = "rust1", since = "1.0.0")] -pub struct BTreeMap { - root: Option>, +#[cfg_attr(not(test), rustc_diagnostic_item = "BTreeMap")] +#[rustc_insignificant_dtor] +pub struct BTreeMap< + K, + V, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + Clone = Global, +> { + root: Option>, length: usize, + /// `ManuallyDrop` to control drop order (needs to be dropped after all the nodes). + pub(super) alloc: ManuallyDrop, + // For dropck; the `Box` avoids making the `Unpin` impl more strict than before + _marker: PhantomData>, } #[stable(feature = "btree_drop", since = "1.7.0")] -unsafe impl<#[may_dangle] K, #[may_dangle] V> Drop for BTreeMap { +unsafe impl<#[may_dangle] K, #[may_dangle] V, A: Allocator + Clone> Drop for BTreeMap { fn drop(&mut self) { - unsafe { - drop(ptr::read(self).into_iter()); - } + drop(unsafe { ptr::read(self) }.into_iter()) } } +// FIXME: This implementation is "wrong", but changing it would be a breaking change. +// (The bounds of the automatic `UnwindSafe` implementation have been like this since Rust 1.50.) +// Maybe we can fix it nonetheless with a crater run, or if the `UnwindSafe` +// traits are deprecated, or disarmed (no longer causing hard errors) in the future. +#[stable(feature = "btree_unwindsafe", since = "1.64.0")] +impl core::panic::UnwindSafe for BTreeMap +where + A: core::panic::UnwindSafe, + K: core::panic::RefUnwindSafe, + V: core::panic::RefUnwindSafe, +{ +} + #[stable(feature = "rust1", since = "1.0.0")] -impl Clone for BTreeMap { - fn clone(&self) -> BTreeMap { - fn clone_subtree<'a, K: Clone, V: Clone>( - node: node::NodeRef, K, V, marker::LeafOrInternal>, - ) -> BTreeMap +impl Clone for BTreeMap { + fn clone(&self) -> BTreeMap { + fn clone_subtree<'a, K: Clone, V: Clone, A: Allocator + Clone>( + node: NodeRef, K, V, marker::LeafOrInternal>, + alloc: A, + ) -> BTreeMap where K: 'a, V: 'a, { match node.force() { Leaf(leaf) => { - let mut out_tree = BTreeMap { root: Some(node::Root::new_leaf()), length: 0 }; + let mut out_tree = BTreeMap { + root: Some(Root::new(alloc.clone())), + length: 0, + alloc: ManuallyDrop::new(alloc), + _marker: PhantomData, + }; { - let root = out_tree.root.as_mut().unwrap(); - let mut out_node = match root.as_mut().force() { + let root = out_tree.root.as_mut().unwrap(); // unwrap succeeds because we just wrapped + let mut out_node = match root.borrow_mut().force() { Leaf(leaf) => leaf, Internal(_) => unreachable!(), }; @@ -169,15 +242,12 @@ impl Clone for BTreeMap { out_tree } Internal(internal) => { - let mut out_tree = clone_subtree(internal.first_edge().descend()); - out_tree.ensure_root_is_owned(); + let mut out_tree = + clone_subtree(internal.first_edge().descend(), alloc.clone()); { - // Ideally we'd use the return of ensure_root_is_owned - // instead of re-unwrapping here but unfortunately that - // borrows all of out_tree and we need access to the - // length below. - let mut out_node = out_tree.root.as_mut().unwrap().push_level(); + let out_root = out_tree.root.as_mut().unwrap(); + let mut out_node = out_root.push_internal_level(alloc.clone()); let mut in_edge = internal.first_edge(); while let Ok(kv) = in_edge.right_kv() { let (k, v) = kv.into_kv(); @@ -185,18 +255,22 @@ impl Clone for BTreeMap { let k = (*k).clone(); let v = (*v).clone(); - let subtree = clone_subtree(in_edge.descend()); + let subtree = clone_subtree(in_edge.descend(), alloc.clone()); // We can't destructure subtree directly // because BTreeMap implements Drop let (subroot, sublength) = unsafe { + let subtree = ManuallyDrop::new(subtree); let root = ptr::read(&subtree.root); let length = subtree.length; - mem::forget(subtree); (root, length) }; - out_node.push(k, v, subroot.unwrap_or_else(|| node::Root::new_leaf())); + out_node.push( + k, + v, + subroot.unwrap_or_else(|| Root::new(alloc.clone())), + ); out_tree.length += 1 + sublength; } } @@ -207,69 +281,14 @@ impl Clone for BTreeMap { } if self.is_empty() { - // Ideally we'd call `BTreeMap::new` here, but that has the `K: - // Ord` constraint, which this method lacks. - BTreeMap { root: None, length: 0 } - } else { - clone_subtree(self.root.as_ref().unwrap().as_ref()) - } - } - - fn clone_from(&mut self, other: &Self) { - BTreeClone::clone_from(self, other); - } -} - -trait BTreeClone { - fn clone_from(&mut self, other: &Self); -} - -impl BTreeClone for BTreeMap { - default fn clone_from(&mut self, other: &Self) { - *self = other.clone(); - } -} - -impl BTreeClone for BTreeMap { - fn clone_from(&mut self, other: &Self) { - // This truncates `self` to `other.len()` by calling `split_off` on - // the first key after `other.len()` elements if it exists. - let split_off_key = if self.len() > other.len() { - let diff = self.len() - other.len(); - if diff <= other.len() { - self.iter().nth_back(diff - 1).map(|pair| (*pair.0).clone()) - } else { - self.iter().nth(other.len()).map(|pair| (*pair.0).clone()) - } + BTreeMap::new_in((*self.alloc).clone()) } else { - None - }; - if let Some(key) = split_off_key { - self.split_off(&key); - } - - let mut siter = self.range_mut(..); - let mut oiter = other.iter(); - // After truncation, `self` is at most as long as `other` so this loop - // replaces every key-value pair in `self`. Since `oiter` is in sorted - // order and the structure of the `BTreeMap` stays the same, - // the BTree invariants are maintained at the end of the loop. - while !siter.is_empty() { - if let Some((ok, ov)) = oiter.next() { - // SAFETY: This is safe because `siter` is nonempty. - let (sk, sv) = unsafe { siter.next_unchecked() }; - sk.clone_from(ok); - sv.clone_from(ov); - } else { - break; - } + clone_subtree(self.root.as_ref().unwrap().reborrow(), (*self.alloc).clone()) // unwrap succeeds because not empty } - // If `other` is longer than `self`, the remaining elements are inserted. - self.extend(oiter.map(|(k, v)| ((*k).clone(), (*v).clone()))); } } -impl super::Recover for BTreeMap +impl super::Recover for BTreeMap where K: Borrow + Ord, Q: Ord, @@ -277,30 +296,46 @@ where type Key = K; fn get(&self, key: &Q) -> Option<&K> { - match search::search_tree(self.root.as_ref()?.as_ref(), key) { + let root_node = self.root.as_ref()?.reborrow(); + match root_node.search_tree(key) { Found(handle) => Some(handle.into_kv().0), GoDown(_) => None, } } fn take(&mut self, key: &Q) -> Option { - match search::search_tree(self.root.as_mut()?.as_mut(), key) { + let (map, dormant_map) = DormantMutRef::new(self); + let root_node = map.root.as_mut()?.borrow_mut(); + match root_node.search_tree(key) { Found(handle) => Some( - OccupiedEntry { handle, length: &mut self.length, _marker: PhantomData } - .remove_kv() - .0, + OccupiedEntry { + handle, + dormant_map, + alloc: (*map.alloc).clone(), + _marker: PhantomData, + } + .remove_kv() + .0, ), GoDown(_) => None, } } fn replace(&mut self, key: K) -> Option { - self.ensure_root_is_owned(); - match search::search_tree::, K, (), K>(self.root.as_mut()?.as_mut(), &key) { - Found(handle) => Some(mem::replace(handle.into_kv_mut().0, key)), + let (map, dormant_map) = DormantMutRef::new(self); + let root_node = + map.root.get_or_insert_with(|| Root::new((*map.alloc).clone())).borrow_mut(); + match root_node.search_tree::(&key) { + Found(mut kv) => Some(mem::replace(kv.key_mut(), key)), GoDown(handle) => { - VacantEntry { key, handle, length: &mut self.length, _marker: PhantomData } - .insert(()); + VacantEntry { + key, + handle: Some(handle), + dormant_map, + alloc: (*map.alloc).clone(), + _marker: PhantomData, + } + .insert(SetValZST::default()); None } } @@ -312,11 +347,11 @@ where /// This `struct` is created by the [`iter`] method on [`BTreeMap`]. See its /// documentation for more. /// -/// [`iter`]: struct.BTreeMap.html#method.iter -/// [`BTreeMap`]: struct.BTreeMap.html +/// [`iter`]: BTreeMap::iter +#[must_use = "iterators are lazy and do nothing unless consumed"] #[stable(feature = "rust1", since = "1.0.0")] pub struct Iter<'a, K: 'a, V: 'a> { - range: Range<'a, K, V>, + range: LazyLeafRange, K, V>, length: usize, } @@ -332,37 +367,57 @@ impl fmt::Debug for Iter<'_, K, V> { /// This `struct` is created by the [`iter_mut`] method on [`BTreeMap`]. See its /// documentation for more. /// -/// [`iter_mut`]: struct.BTreeMap.html#method.iter_mut -/// [`BTreeMap`]: struct.BTreeMap.html +/// [`iter_mut`]: BTreeMap::iter_mut #[stable(feature = "rust1", since = "1.0.0")] -#[derive(Debug)] pub struct IterMut<'a, K: 'a, V: 'a> { - range: RangeMut<'a, K, V>, + range: LazyLeafRange, K, V>, length: usize, + + // Be invariant in `K` and `V` + _marker: PhantomData<&'a mut (K, V)>, +} + +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "collection_debug", since = "1.17.0")] +impl fmt::Debug for IterMut<'_, K, V> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let range = Iter { range: self.range.reborrow(), length: self.length }; + f.debug_list().entries(range).finish() + } } /// An owning iterator over the entries of a `BTreeMap`. /// /// This `struct` is created by the [`into_iter`] method on [`BTreeMap`] -/// (provided by the `IntoIterator` trait). See its documentation for more. +/// (provided by the [`IntoIterator`] trait). See its documentation for more. /// -/// [`into_iter`]: struct.BTreeMap.html#method.into_iter -/// [`BTreeMap`]: struct.BTreeMap.html +/// [`into_iter`]: IntoIterator::into_iter +/// [`IntoIterator`]: core::iter::IntoIterator #[stable(feature = "rust1", since = "1.0.0")] -pub struct IntoIter { - front: Option, marker::Edge>>, - back: Option, marker::Edge>>, +#[rustc_insignificant_dtor] +pub struct IntoIter< + K, + V, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + Clone = Global, +> { + range: LazyLeafRange, length: usize, + /// The BTreeMap will outlive this IntoIter so we don't care about drop order for `alloc`. + alloc: A, +} + +impl IntoIter { + /// Returns an iterator of references over the remaining items. + #[inline] + pub(super) fn iter(&self) -> Iter<'_, K, V> { + Iter { range: self.range.reborrow(), length: self.length } + } } #[stable(feature = "collection_debug", since = "1.17.0")] -impl fmt::Debug for IntoIter { +impl Debug for IntoIter { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let range = Range { - front: self.front.as_ref().map(|f| f.reborrow()), - back: self.back.as_ref().map(|b| b.reborrow()), - }; - f.debug_list().entries(range).finish() + f.debug_list().entries(self.iter()).finish() } } @@ -371,10 +426,10 @@ impl fmt::Debug for IntoIter { /// This `struct` is created by the [`keys`] method on [`BTreeMap`]. See its /// documentation for more. /// -/// [`keys`]: struct.BTreeMap.html#method.keys -/// [`BTreeMap`]: struct.BTreeMap.html +/// [`keys`]: BTreeMap::keys +#[must_use = "iterators are lazy and do nothing unless consumed"] #[stable(feature = "rust1", since = "1.0.0")] -pub struct Keys<'a, K: 'a, V: 'a> { +pub struct Keys<'a, K, V> { inner: Iter<'a, K, V>, } @@ -390,10 +445,10 @@ impl fmt::Debug for Keys<'_, K, V> { /// This `struct` is created by the [`values`] method on [`BTreeMap`]. See its /// documentation for more. /// -/// [`values`]: struct.BTreeMap.html#method.values -/// [`BTreeMap`]: struct.BTreeMap.html +/// [`values`]: BTreeMap::values +#[must_use = "iterators are lazy and do nothing unless consumed"] #[stable(feature = "rust1", since = "1.0.0")] -pub struct Values<'a, K: 'a, V: 'a> { +pub struct Values<'a, K, V> { inner: Iter<'a, K, V>, } @@ -409,138 +464,108 @@ impl fmt::Debug for Values<'_, K, V> { /// This `struct` is created by the [`values_mut`] method on [`BTreeMap`]. See its /// documentation for more. /// -/// [`values_mut`]: struct.BTreeMap.html#method.values_mut -/// [`BTreeMap`]: struct.BTreeMap.html +/// [`values_mut`]: BTreeMap::values_mut +#[must_use = "iterators are lazy and do nothing unless consumed"] #[stable(feature = "map_values_mut", since = "1.10.0")] -#[derive(Debug)] -pub struct ValuesMut<'a, K: 'a, V: 'a> { +pub struct ValuesMut<'a, K, V> { inner: IterMut<'a, K, V>, } -/// An iterator over a sub-range of entries in a `BTreeMap`. -/// -/// This `struct` is created by the [`range`] method on [`BTreeMap`]. See its -/// documentation for more. -/// -/// [`range`]: struct.BTreeMap.html#method.range -/// [`BTreeMap`]: struct.BTreeMap.html -#[stable(feature = "btree_range", since = "1.17.0")] -pub struct Range<'a, K: 'a, V: 'a> { - front: Option, K, V, marker::Leaf>, marker::Edge>>, - back: Option, K, V, marker::Leaf>, marker::Edge>>, -} - -#[stable(feature = "collection_debug", since = "1.17.0")] -impl fmt::Debug for Range<'_, K, V> { +#[stable(feature = "map_values_mut", since = "1.10.0")] +impl fmt::Debug for ValuesMut<'_, K, V> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_list().entries(self.clone()).finish() + f.debug_list().entries(self.inner.iter().map(|(_, val)| val)).finish() } } -/// A mutable iterator over a sub-range of entries in a `BTreeMap`. +/// An owning iterator over the keys of a `BTreeMap`. /// -/// This `struct` is created by the [`range_mut`] method on [`BTreeMap`]. See its -/// documentation for more. +/// This `struct` is created by the [`into_keys`] method on [`BTreeMap`]. +/// See its documentation for more. /// -/// [`range_mut`]: struct.BTreeMap.html#method.range_mut -/// [`BTreeMap`]: struct.BTreeMap.html -#[stable(feature = "btree_range", since = "1.17.0")] -pub struct RangeMut<'a, K: 'a, V: 'a> { - front: Option, K, V, marker::Leaf>, marker::Edge>>, - back: Option, K, V, marker::Leaf>, marker::Edge>>, - - // Be invariant in `K` and `V` - _marker: PhantomData<&'a mut (K, V)>, +/// [`into_keys`]: BTreeMap::into_keys +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +pub struct IntoKeys { + inner: IntoIter, } -#[stable(feature = "collection_debug", since = "1.17.0")] -impl fmt::Debug for RangeMut<'_, K, V> { +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +impl fmt::Debug for IntoKeys { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let range = Range { - front: self.front.as_ref().map(|f| f.reborrow()), - back: self.back.as_ref().map(|b| b.reborrow()), - }; - f.debug_list().entries(range).finish() + f.debug_list().entries(self.inner.iter().map(|(key, _)| key)).finish() } } -/// A view into a single entry in a map, which may either be vacant or occupied. +/// An owning iterator over the values of a `BTreeMap`. /// -/// This `enum` is constructed from the [`entry`] method on [`BTreeMap`]. +/// This `struct` is created by the [`into_values`] method on [`BTreeMap`]. +/// See its documentation for more. /// -/// [`BTreeMap`]: struct.BTreeMap.html -/// [`entry`]: struct.BTreeMap.html#method.entry -#[stable(feature = "rust1", since = "1.0.0")] -pub enum Entry<'a, K: 'a, V: 'a> { - /// A vacant entry. - #[stable(feature = "rust1", since = "1.0.0")] - Vacant(#[stable(feature = "rust1", since = "1.0.0")] VacantEntry<'a, K, V>), - - /// An occupied entry. - #[stable(feature = "rust1", since = "1.0.0")] - Occupied(#[stable(feature = "rust1", since = "1.0.0")] OccupiedEntry<'a, K, V>), -} - -#[stable(feature = "debug_btree_map", since = "1.12.0")] -impl Debug for Entry<'_, K, V> { +/// [`into_values`]: BTreeMap::into_values +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +pub struct IntoValues< + K, + V, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + Clone = Global, +> { + inner: IntoIter, +} + +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +impl fmt::Debug for IntoValues { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match *self { - Vacant(ref v) => f.debug_tuple("Entry").field(v).finish(), - Occupied(ref o) => f.debug_tuple("Entry").field(o).finish(), - } + f.debug_list().entries(self.inner.iter().map(|(_, val)| val)).finish() } } -/// A view into a vacant entry in a `BTreeMap`. -/// It is part of the [`Entry`] enum. +/// An iterator over a sub-range of entries in a `BTreeMap`. /// -/// [`Entry`]: enum.Entry.html -#[stable(feature = "rust1", since = "1.0.0")] -pub struct VacantEntry<'a, K: 'a, V: 'a> { - key: K, - handle: Handle, K, V, marker::Leaf>, marker::Edge>, - length: &'a mut usize, - - // Be invariant in `K` and `V` - _marker: PhantomData<&'a mut (K, V)>, +/// This `struct` is created by the [`range`] method on [`BTreeMap`]. See its +/// documentation for more. +/// +/// [`range`]: BTreeMap::range +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "btree_range", since = "1.17.0")] +pub struct Range<'a, K: 'a, V: 'a> { + inner: LeafRange, K, V>, } -#[stable(feature = "debug_btree_map", since = "1.12.0")] -impl Debug for VacantEntry<'_, K, V> { +#[stable(feature = "collection_debug", since = "1.17.0")] +impl fmt::Debug for Range<'_, K, V> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("VacantEntry").field(self.key()).finish() + f.debug_list().entries(self.clone()).finish() } } -/// A view into an occupied entry in a `BTreeMap`. -/// It is part of the [`Entry`] enum. +/// A mutable iterator over a sub-range of entries in a `BTreeMap`. /// -/// [`Entry`]: enum.Entry.html -#[stable(feature = "rust1", since = "1.0.0")] -pub struct OccupiedEntry<'a, K: 'a, V: 'a> { - handle: Handle, K, V, marker::LeafOrInternal>, marker::KV>, - - length: &'a mut usize, +/// This `struct` is created by the [`range_mut`] method on [`BTreeMap`]. See its +/// documentation for more. +/// +/// [`range_mut`]: BTreeMap::range_mut +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "btree_range", since = "1.17.0")] +pub struct RangeMut<'a, K: 'a, V: 'a> { + inner: LeafRange, K, V>, // Be invariant in `K` and `V` _marker: PhantomData<&'a mut (K, V)>, } -#[stable(feature = "debug_btree_map", since = "1.12.0")] -impl Debug for OccupiedEntry<'_, K, V> { +#[stable(feature = "collection_debug", since = "1.17.0")] +impl fmt::Debug for RangeMut<'_, K, V> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("OccupiedEntry").field("key", self.key()).field("value", self.get()).finish() + let range = Range { inner: self.inner.reborrow() }; + f.debug_list().entries(range).finish() } } -// An iterator for merging two sorted sequences into one -struct MergeIter> { - left: Peekable, - right: Peekable, -} - -impl BTreeMap { - /// Makes a new empty BTreeMap with a reasonable choice for B. +impl BTreeMap { + /// Makes a new, empty `BTreeMap`. + /// + /// Does not allocate anything on its own. /// /// # Examples /// @@ -555,10 +580,14 @@ impl BTreeMap { /// map.insert(1, "a"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn new() -> BTreeMap { - BTreeMap { root: None, length: 0 } + #[rustc_const_stable(feature = "const_btree_new", since = "1.66.0")] + #[must_use] + pub const fn new() -> BTreeMap { + BTreeMap { root: None, length: 0, alloc: ManuallyDrop::new(Global), _marker: PhantomData } } +} +impl BTreeMap { /// Clears the map, removing all elements. /// /// # Examples @@ -575,9 +604,39 @@ impl BTreeMap { /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn clear(&mut self) { - *self = BTreeMap::new(); + // avoid moving the allocator + mem::drop(BTreeMap { + root: mem::replace(&mut self.root, None), + length: mem::replace(&mut self.length, 0), + alloc: self.alloc.clone(), + _marker: PhantomData, + }); + } + + /// Makes a new empty BTreeMap with a reasonable choice for B. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// # #![feature(allocator_api)] + /// # #![feature(btreemap_alloc)] + /// use std::collections::BTreeMap; + /// use std::alloc::Global; + /// + /// let mut map = BTreeMap::new_in(Global); + /// + /// // entries can now be inserted into the empty map + /// map.insert(1, "a"); + /// ``` + #[unstable(feature = "btreemap_alloc", issue = "32838")] + pub fn new_in(alloc: A) -> BTreeMap { + BTreeMap { root: None, length: 0, alloc: ManuallyDrop::new(alloc), _marker: PhantomData } } +} +impl BTreeMap { /// Returns a reference to the value corresponding to the key. /// /// The key may be any borrowed form of the map's key type, but the ordering @@ -598,10 +657,11 @@ impl BTreeMap { #[stable(feature = "rust1", since = "1.0.0")] pub fn get(&self, key: &Q) -> Option<&V> where - K: Borrow, + K: Borrow + Ord, Q: Ord, { - match search::search_tree(self.root.as_ref()?.as_ref(), key) { + let root_node = self.root.as_ref()?.reborrow(); + match root_node.search_tree(key) { Found(handle) => Some(handle.into_kv().1), GoDown(_) => None, } @@ -625,10 +685,11 @@ impl BTreeMap { #[stable(feature = "map_get_key_value", since = "1.40.0")] pub fn get_key_value(&self, k: &Q) -> Option<(&K, &V)> where - K: Borrow, + K: Borrow + Ord, Q: Ord, { - match search::search_tree(self.root.as_ref()?.as_ref(), k) { + let root_node = self.root.as_ref()?.reborrow(); + match root_node.search_tree(k) { Found(handle) => Some(handle.into_kv()), GoDown(_) => None, } @@ -642,7 +703,6 @@ impl BTreeMap { /// Basic usage: /// /// ``` - /// #![feature(map_first_last)] /// use std::collections::BTreeMap; /// /// let mut map = BTreeMap::new(); @@ -651,14 +711,13 @@ impl BTreeMap { /// map.insert(2, "a"); /// assert_eq!(map.first_key_value(), Some((&1, &"b"))); /// ``` - #[unstable(feature = "map_first_last", issue = "62924")] - pub fn first_key_value(&self) -> Option<(&K, &V)> + #[stable(feature = "map_first_last", since = "1.66.0")] + pub fn first_key_value(&self) -> Option<(&K, &V)> where - T: Ord, - K: Borrow, + K: Ord, { - let front = self.root.as_ref()?.as_ref().first_leaf_edge(); - front.right_kv().ok().map(Handle::into_kv) + let root_node = self.root.as_ref()?.reborrow(); + root_node.first_leaf_edge().right_kv().ok().map(Handle::into_kv) } /// Returns the first entry in the map for in-place manipulation. @@ -666,36 +725,60 @@ impl BTreeMap { /// /// # Examples /// - /// Contrived way to `clear` a map: + /// ``` + /// use std::collections::BTreeMap; + /// + /// let mut map = BTreeMap::new(); + /// map.insert(1, "a"); + /// map.insert(2, "b"); + /// if let Some(mut entry) = map.first_entry() { + /// if *entry.key() > 0 { + /// entry.insert("first"); + /// } + /// } + /// assert_eq!(*map.get(&1).unwrap(), "first"); + /// assert_eq!(*map.get(&2).unwrap(), "b"); + /// ``` + #[stable(feature = "map_first_last", since = "1.66.0")] + pub fn first_entry(&mut self) -> Option> + where + K: Ord, + { + let (map, dormant_map) = DormantMutRef::new(self); + let root_node = map.root.as_mut()?.borrow_mut(); + let kv = root_node.first_leaf_edge().right_kv().ok()?; + Some(OccupiedEntry { + handle: kv.forget_node_type(), + dormant_map, + alloc: (*map.alloc).clone(), + _marker: PhantomData, + }) + } + + /// Removes and returns the first element in the map. + /// The key of this element is the minimum key that was in the map. + /// + /// # Examples + /// + /// Draining elements in ascending order, while keeping a usable map each iteration. /// /// ``` - /// #![feature(map_first_last)] /// use std::collections::BTreeMap; /// /// let mut map = BTreeMap::new(); /// map.insert(1, "a"); /// map.insert(2, "b"); - /// while let Some(entry) = map.first_entry() { - /// let (key, val) = entry.remove_entry(); - /// assert!(!map.contains_key(&key)); + /// while let Some((key, _val)) = map.pop_first() { + /// assert!(map.iter().all(|(k, _v)| *k > key)); /// } + /// assert!(map.is_empty()); /// ``` - #[unstable(feature = "map_first_last", issue = "62924")] - pub fn first_entry(&mut self) -> Option> + #[stable(feature = "map_first_last", since = "1.66.0")] + pub fn pop_first(&mut self) -> Option<(K, V)> where - T: Ord, - K: Borrow, + K: Ord, { - let front = self.root.as_mut()?.as_mut().first_leaf_edge(); - if let Ok(kv) = front.right_kv() { - Some(OccupiedEntry { - handle: kv.forget_node_type(), - length: &mut self.length, - _marker: PhantomData, - }) - } else { - None - } + self.first_entry().map(|entry| entry.remove_entry()) } /// Returns the last key-value pair in the map. @@ -706,7 +789,6 @@ impl BTreeMap { /// Basic usage: /// /// ``` - /// #![feature(map_first_last)] /// use std::collections::BTreeMap; /// /// let mut map = BTreeMap::new(); @@ -714,14 +796,13 @@ impl BTreeMap { /// map.insert(2, "a"); /// assert_eq!(map.last_key_value(), Some((&2, &"a"))); /// ``` - #[unstable(feature = "map_first_last", issue = "62924")] - pub fn last_key_value(&self) -> Option<(&K, &V)> + #[stable(feature = "map_first_last", since = "1.66.0")] + pub fn last_key_value(&self) -> Option<(&K, &V)> where - T: Ord, - K: Borrow, + K: Ord, { - let back = self.root.as_ref()?.as_ref().last_leaf_edge(); - back.left_kv().ok().map(Handle::into_kv) + let root_node = self.root.as_ref()?.reborrow(); + root_node.last_leaf_edge().left_kv().ok().map(Handle::into_kv) } /// Returns the last entry in the map for in-place manipulation. @@ -729,36 +810,60 @@ impl BTreeMap { /// /// # Examples /// - /// Contrived way to `clear` a map: + /// ``` + /// use std::collections::BTreeMap; + /// + /// let mut map = BTreeMap::new(); + /// map.insert(1, "a"); + /// map.insert(2, "b"); + /// if let Some(mut entry) = map.last_entry() { + /// if *entry.key() > 0 { + /// entry.insert("last"); + /// } + /// } + /// assert_eq!(*map.get(&1).unwrap(), "a"); + /// assert_eq!(*map.get(&2).unwrap(), "last"); + /// ``` + #[stable(feature = "map_first_last", since = "1.66.0")] + pub fn last_entry(&mut self) -> Option> + where + K: Ord, + { + let (map, dormant_map) = DormantMutRef::new(self); + let root_node = map.root.as_mut()?.borrow_mut(); + let kv = root_node.last_leaf_edge().left_kv().ok()?; + Some(OccupiedEntry { + handle: kv.forget_node_type(), + dormant_map, + alloc: (*map.alloc).clone(), + _marker: PhantomData, + }) + } + + /// Removes and returns the last element in the map. + /// The key of this element is the maximum key that was in the map. + /// + /// # Examples + /// + /// Draining elements in descending order, while keeping a usable map each iteration. /// /// ``` - /// #![feature(map_first_last)] /// use std::collections::BTreeMap; /// /// let mut map = BTreeMap::new(); /// map.insert(1, "a"); /// map.insert(2, "b"); - /// while let Some(entry) = map.last_entry() { - /// let (key, val) = entry.remove_entry(); - /// assert!(!map.contains_key(&key)); + /// while let Some((key, _val)) = map.pop_last() { + /// assert!(map.iter().all(|(k, _v)| *k < key)); /// } + /// assert!(map.is_empty()); /// ``` - #[unstable(feature = "map_first_last", issue = "62924")] - pub fn last_entry(&mut self) -> Option> + #[stable(feature = "map_first_last", since = "1.66.0")] + pub fn pop_last(&mut self) -> Option<(K, V)> where - T: Ord, - K: Borrow, + K: Ord, { - let back = self.root.as_mut()?.as_mut().last_leaf_edge(); - if let Ok(kv) = back.left_kv() { - Some(OccupiedEntry { - handle: kv.forget_node_type(), - length: &mut self.length, - _marker: PhantomData, - }) - } else { - None - } + self.last_entry().map(|entry| entry.remove_entry()) } /// Returns `true` if the map contains a value for the specified key. @@ -781,7 +886,7 @@ impl BTreeMap { #[stable(feature = "rust1", since = "1.0.0")] pub fn contains_key(&self, key: &Q) -> bool where - K: Borrow, + K: Borrow + Ord, Q: Ord, { self.get(key).is_some() @@ -810,11 +915,12 @@ impl BTreeMap { #[stable(feature = "rust1", since = "1.0.0")] pub fn get_mut(&mut self, key: &Q) -> Option<&mut V> where - K: Borrow, + K: Borrow + Ord, Q: Ord, { - match search::search_tree(self.root.as_mut()?.as_mut(), key) { - Found(handle) => Some(handle.into_kv_mut().1), + let root_node = self.root.as_mut()?.borrow_mut(); + match root_node.search_tree(key) { + Found(handle) => Some(handle.into_val_mut()), GoDown(_) => None, } } @@ -846,7 +952,10 @@ impl BTreeMap { /// assert_eq!(map[&37], "c"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn insert(&mut self, key: K, value: V) -> Option { + pub fn insert(&mut self, key: K, value: V) -> Option + where + K: Ord, + { match self.entry(key) { Occupied(mut entry) => Some(entry.insert(value)), Vacant(entry) => { @@ -856,6 +965,40 @@ impl BTreeMap { } } + /// Tries to insert a key-value pair into the map, and returns + /// a mutable reference to the value in the entry. + /// + /// If the map already had this key present, nothing is updated, and + /// an error containing the occupied entry and the value is returned. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(map_try_insert)] + /// + /// use std::collections::BTreeMap; + /// + /// let mut map = BTreeMap::new(); + /// assert_eq!(map.try_insert(37, "a").unwrap(), &"a"); + /// + /// let err = map.try_insert(37, "b").unwrap_err(); + /// assert_eq!(err.entry.key(), &37); + /// assert_eq!(err.entry.get(), &"a"); + /// assert_eq!(err.value, "b"); + /// ``` + #[unstable(feature = "map_try_insert", issue = "82766")] + pub fn try_insert(&mut self, key: K, value: V) -> Result<&mut V, OccupiedError<'_, K, V, A>> + where + K: Ord, + { + match self.entry(key) { + Occupied(entry) => Err(OccupiedError { entry, value }), + Vacant(entry) => Ok(entry.insert(value)), + } + } + /// Removes a key from the map, returning the value at the key if the key /// was previously in the map. /// @@ -877,7 +1020,7 @@ impl BTreeMap { #[stable(feature = "rust1", since = "1.0.0")] pub fn remove(&mut self, key: &Q) -> Option where - K: Borrow, + K: Borrow + Ord, Q: Ord, { self.remove_entry(key).map(|(_, v)| v) @@ -894,7 +1037,6 @@ impl BTreeMap { /// Basic usage: /// /// ``` - /// #![feature(btreemap_remove_entry)] /// use std::collections::BTreeMap; /// /// let mut map = BTreeMap::new(); @@ -902,22 +1044,57 @@ impl BTreeMap { /// assert_eq!(map.remove_entry(&1), Some((1, "a"))); /// assert_eq!(map.remove_entry(&1), None); /// ``` - #[unstable(feature = "btreemap_remove_entry", issue = "66714")] + #[stable(feature = "btreemap_remove_entry", since = "1.45.0")] pub fn remove_entry(&mut self, key: &Q) -> Option<(K, V)> where - K: Borrow, + K: Borrow + Ord, Q: Ord, { - match search::search_tree(self.root.as_mut()?.as_mut(), key) { + let (map, dormant_map) = DormantMutRef::new(self); + let root_node = map.root.as_mut()?.borrow_mut(); + match root_node.search_tree(key) { Found(handle) => Some( - OccupiedEntry { handle, length: &mut self.length, _marker: PhantomData } - .remove_entry(), + OccupiedEntry { + handle, + dormant_map, + alloc: (*map.alloc).clone(), + _marker: PhantomData, + } + .remove_entry(), ), GoDown(_) => None, } } - /// Moves all elements from `other` into `Self`, leaving `other` empty. + /// Retains only the elements specified by the predicate. + /// + /// In other words, remove all pairs `(k, v)` for which `f(&k, &mut v)` returns `false`. + /// The elements are visited in ascending key order. + /// + /// # Examples + /// + /// ``` + /// use std::collections::BTreeMap; + /// + /// let mut map: BTreeMap = (0..8).map(|x| (x, x*10)).collect(); + /// // Keep only the elements with even-numbered keys. + /// map.retain(|&k, _| k % 2 == 0); + /// assert!(map.into_iter().eq(vec![(0, 0), (2, 20), (4, 40), (6, 60)])); + /// ``` + #[inline] + #[stable(feature = "btree_retain", since = "1.53.0")] + pub fn retain(&mut self, mut f: F) + where + K: Ord, + F: FnMut(&K, &mut V) -> bool, + { + self.drain_filter(|k, v| !f(k, v)); + } + + /// Moves all elements from `other` into `self`, leaving `other` empty. + /// + /// If a key from `other` is already present in `self`, the respective + /// value from `self` will be overwritten with the respective value from `other`. /// /// # Examples /// @@ -927,10 +1104,10 @@ impl BTreeMap { /// let mut a = BTreeMap::new(); /// a.insert(1, "a"); /// a.insert(2, "b"); - /// a.insert(3, "c"); + /// a.insert(3, "c"); // Note: Key (3) also present in b. /// /// let mut b = BTreeMap::new(); - /// b.insert(3, "d"); + /// b.insert(3, "d"); // Note: Key (3) also present in a. /// b.insert(4, "e"); /// b.insert(5, "f"); /// @@ -941,12 +1118,16 @@ impl BTreeMap { /// /// assert_eq!(a[&1], "a"); /// assert_eq!(a[&2], "b"); - /// assert_eq!(a[&3], "d"); + /// assert_eq!(a[&3], "d"); // Note: "c" has been overwritten. /// assert_eq!(a[&4], "e"); /// assert_eq!(a[&5], "f"); /// ``` #[stable(feature = "btree_append", since = "1.11.0")] - pub fn append(&mut self, other: &mut Self) { + pub fn append(&mut self, other: &mut Self) + where + K: Ord, + A: Clone, + { // Do we have to append anything at all? if other.is_empty() { return; @@ -958,14 +1139,15 @@ impl BTreeMap { return; } - // First, we merge `self` and `other` into a sorted sequence in linear time. - let self_iter = mem::take(self).into_iter(); - let other_iter = mem::take(other).into_iter(); - let iter = MergeIter { left: self_iter.peekable(), right: other_iter.peekable() }; - - // Second, we build a tree from the sorted sequence in linear time. - self.from_sorted_iter(iter); - self.fix_right_edge(); + let self_iter = mem::replace(self, Self::new_in((*self.alloc).clone())).into_iter(); + let other_iter = mem::replace(other, Self::new_in((*self.alloc).clone())).into_iter(); + let root = self.root.get_or_insert_with(|| Root::new((*self.alloc).clone())); + root.append_from_sorted_iters( + self_iter, + other_iter, + &mut self.length, + (*self.alloc).clone(), + ) } /// Constructs a double-ended iterator over a sub-range of elements in the map. @@ -993,7 +1175,7 @@ impl BTreeMap { /// map.insert(5, "b"); /// map.insert(8, "c"); /// for (&key, &value) in map.range((Included(&4), Included(&8))) { - /// println!("{}: {}", key, value); + /// println!("{key}: {value}"); /// } /// assert_eq!(Some((&5, &"b")), map.range(4..).next()); /// ``` @@ -1001,17 +1183,13 @@ impl BTreeMap { pub fn range(&self, range: R) -> Range<'_, K, V> where T: Ord, - K: Borrow, + K: Borrow + Ord, R: RangeBounds, { if let Some(root) = &self.root { - let root1 = root.as_ref(); - let root2 = root.as_ref(); - let (f, b) = range_search(root1, root2, range); - - Range { front: Some(f), back: Some(b) } + Range { inner: root.reborrow().range_search(range) } } else { - Range { front: None, back: None } + Range { inner: LeafRange::none() } } } @@ -1034,32 +1212,26 @@ impl BTreeMap { /// ``` /// use std::collections::BTreeMap; /// - /// let mut map: BTreeMap<&str, i32> = ["Alice", "Bob", "Carol", "Cheryl"] - /// .iter() - /// .map(|&s| (s, 0)) - /// .collect(); + /// let mut map: BTreeMap<&str, i32> = + /// [("Alice", 0), ("Bob", 0), ("Carol", 0), ("Cheryl", 0)].into(); /// for (_, balance) in map.range_mut("B".."Cheryl") { /// *balance += 100; /// } /// for (name, balance) in &map { - /// println!("{} => {}", name, balance); + /// println!("{name} => {balance}"); /// } /// ``` #[stable(feature = "btree_range", since = "1.17.0")] pub fn range_mut(&mut self, range: R) -> RangeMut<'_, K, V> where T: Ord, - K: Borrow, + K: Borrow + Ord, R: RangeBounds, { if let Some(root) = &mut self.root { - let root1 = root.as_mut(); - let root2 = unsafe { ptr::read(&root1) }; - let (f, b) = range_search(root1, root2, range); - - RangeMut { front: Some(f), back: Some(b), _marker: PhantomData } + RangeMut { inner: root.borrow_valmut().range_search(range), _marker: PhantomData } } else { - RangeMut { front: None, back: None, _marker: PhantomData } + RangeMut { inner: LeafRange::none(), _marker: PhantomData } } } @@ -1075,94 +1247,43 @@ impl BTreeMap { /// let mut count: BTreeMap<&str, usize> = BTreeMap::new(); /// /// // count the number of occurrences of letters in the vec - /// for x in vec!["a","b","a","c","a","b"] { - /// *count.entry(x).or_insert(0) += 1; + /// for x in ["a", "b", "a", "c", "a", "b"] { + /// count.entry(x).and_modify(|curr| *curr += 1).or_insert(1); /// } /// /// assert_eq!(count["a"], 3); + /// assert_eq!(count["b"], 2); + /// assert_eq!(count["c"], 1); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn entry(&mut self, key: K) -> Entry<'_, K, V> { - // FIXME(@porglezomp) Avoid allocating if we don't insert - self.ensure_root_is_owned(); - match search::search_tree(self.root.as_mut().unwrap().as_mut(), &key) { - Found(handle) => { - Occupied(OccupiedEntry { handle, length: &mut self.length, _marker: PhantomData }) - } - GoDown(handle) => { - Vacant(VacantEntry { key, handle, length: &mut self.length, _marker: PhantomData }) - } - } - } - - fn from_sorted_iter>(&mut self, iter: I) { - self.ensure_root_is_owned(); - let mut cur_node = self.root.as_mut().unwrap().as_mut().last_leaf_edge().into_node(); - // Iterate through all key-value pairs, pushing them into nodes at the right level. - for (key, value) in iter { - // Try to push key-value pair into the current leaf node. - if cur_node.len() < node::CAPACITY { - cur_node.push(key, value); - } else { - // No space left, go up and push there. - let mut open_node; - let mut test_node = cur_node.forget_type(); - loop { - match test_node.ascend() { - Ok(parent) => { - let parent = parent.into_node(); - if parent.len() < node::CAPACITY { - // Found a node with space left, push here. - open_node = parent; - break; - } else { - // Go up again. - test_node = parent.forget_type(); - } - } - Err(node) => { - // We are at the top, create a new root node and push there. - open_node = node.into_root_mut().push_level(); - break; - } - } - } - - // Push key-value pair and new right subtree. - let tree_height = open_node.height() - 1; - let mut right_tree = node::Root::new_leaf(); - for _ in 0..tree_height { - right_tree.push_level(); - } - open_node.push(key, value, right_tree); - - // Go down to the right-most leaf again. - cur_node = open_node.forget_type().last_leaf_edge().into_node(); - } - - self.length += 1; - } - } - - fn fix_right_edge(&mut self) { - // Handle underfull nodes, start from the top. - let mut cur_node = self.root.as_mut().unwrap().as_mut(); - while let Internal(internal) = cur_node.force() { - // Check if right-most child is underfull. - let mut last_edge = internal.last_edge(); - let right_child_len = last_edge.reborrow().descend().len(); - if right_child_len < node::MIN_LEN { - // We need to steal. - let mut last_kv = match last_edge.left_kv() { - Ok(left) => left, - Err(_) => unreachable!(), - }; - last_kv.bulk_steal_left(node::MIN_LEN - right_child_len); - last_edge = last_kv.right_edge(); - } - - // Go further down. - cur_node = last_edge.descend(); + pub fn entry(&mut self, key: K) -> Entry<'_, K, V, A> + where + K: Ord, + { + let (map, dormant_map) = DormantMutRef::new(self); + match map.root { + None => Vacant(VacantEntry { + key, + handle: None, + dormant_map, + alloc: (*map.alloc).clone(), + _marker: PhantomData, + }), + Some(ref mut root) => match root.borrow_mut().search_tree(&key) { + Found(handle) => Occupied(OccupiedEntry { + handle, + dormant_map, + alloc: (*map.alloc).clone(), + _marker: PhantomData, + }), + GoDown(handle) => Vacant(VacantEntry { + key, + handle: Some(handle), + dormant_map, + alloc: (*map.alloc).clone(), + _marker: PhantomData, + }), + }, } } @@ -1198,160 +1319,157 @@ impl BTreeMap { #[stable(feature = "btree_split_off", since = "1.11.0")] pub fn split_off(&mut self, key: &Q) -> Self where - K: Borrow, + K: Borrow + Ord, + A: Clone, { if self.is_empty() { - return Self::new(); + return Self::new_in((*self.alloc).clone()); } let total_num = self.len(); + let left_root = self.root.as_mut().unwrap(); // unwrap succeeds because not empty + + let right_root = left_root.split_off(key, (*self.alloc).clone()); - let mut right = Self::new(); - let right_root = right.ensure_root_is_owned(); - for _ in 0..(self.root.as_ref().unwrap().as_ref().height()) { - right_root.push_level(); + let (new_left_len, right_len) = Root::calc_split_length(total_num, &left_root, &right_root); + self.length = new_left_len; + + BTreeMap { + root: Some(right_root), + length: right_len, + alloc: self.alloc.clone(), + _marker: PhantomData, } + } - { - let mut left_node = self.root.as_mut().unwrap().as_mut(); - let mut right_node = right.root.as_mut().unwrap().as_mut(); - - loop { - let mut split_edge = match search::search_node(left_node, key) { - // key is going to the right tree - Found(handle) => handle.left_edge(), - GoDown(handle) => handle, - }; - - split_edge.move_suffix(&mut right_node); - - match (split_edge.force(), right_node.force()) { - (Internal(edge), Internal(node)) => { - left_node = edge.descend(); - right_node = node.first_edge().descend(); - } - (Leaf(_), Leaf(_)) => { - break; - } - _ => { - unreachable!(); - } - } - } - } - - self.fix_right_border(); - right.fix_left_border(); - - if self.root.as_ref().unwrap().as_ref().height() - < right.root.as_ref().unwrap().as_ref().height() - { - self.recalc_length(); - right.length = total_num - self.len(); - } else { - right.recalc_length(); - self.length = total_num - right.len(); - } - - right + /// Creates an iterator that visits all elements (key-value pairs) in + /// ascending key order and uses a closure to determine if an element should + /// be removed. If the closure returns `true`, the element is removed from + /// the map and yielded. If the closure returns `false`, or panics, the + /// element remains in the map and will not be yielded. + /// + /// The iterator also lets you mutate the value of each element in the + /// closure, regardless of whether you choose to keep or remove it. + /// + /// If the iterator is only partially consumed or not consumed at all, each + /// of the remaining elements is still subjected to the closure, which may + /// change its value and, by returning `true`, have the element removed and + /// dropped. + /// + /// It is unspecified how many more elements will be subjected to the + /// closure if a panic occurs in the closure, or a panic occurs while + /// dropping an element, or if the `DrainFilter` value is leaked. + /// + /// # Examples + /// + /// Splitting a map into even and odd keys, reusing the original map: + /// + /// ``` + /// #![feature(btree_drain_filter)] + /// use std::collections::BTreeMap; + /// + /// let mut map: BTreeMap = (0..8).map(|x| (x, x)).collect(); + /// let evens: BTreeMap<_, _> = map.drain_filter(|k, _v| k % 2 == 0).collect(); + /// let odds = map; + /// assert_eq!(evens.keys().copied().collect::>(), [0, 2, 4, 6]); + /// assert_eq!(odds.keys().copied().collect::>(), [1, 3, 5, 7]); + /// ``` + #[unstable(feature = "btree_drain_filter", issue = "70530")] + pub fn drain_filter(&mut self, pred: F) -> DrainFilter<'_, K, V, F, A> + where + K: Ord, + F: FnMut(&K, &mut V) -> bool, + { + let (inner, alloc) = self.drain_filter_inner(); + DrainFilter { pred, inner, alloc } } - /// Calculates the number of elements if it is incorrect. - fn recalc_length(&mut self) { - fn dfs<'a, K, V>(node: NodeRef, K, V, marker::LeafOrInternal>) -> usize - where - K: 'a, - V: 'a, - { - let mut res = node.len(); - - if let Internal(node) = node.force() { - let mut edge = node.first_edge(); - loop { - res += dfs(edge.reborrow().descend()); - match edge.right_kv() { - Ok(right_kv) => { - edge = right_kv.right_edge(); - } - Err(_) => { - break; - } - } - } - } - - res + pub(super) fn drain_filter_inner(&mut self) -> (DrainFilterInner<'_, K, V>, A) + where + K: Ord, + { + if let Some(root) = self.root.as_mut() { + let (root, dormant_root) = DormantMutRef::new(root); + let front = root.borrow_mut().first_leaf_edge(); + ( + DrainFilterInner { + length: &mut self.length, + dormant_root: Some(dormant_root), + cur_leaf_edge: Some(front), + }, + (*self.alloc).clone(), + ) + } else { + ( + DrainFilterInner { + length: &mut self.length, + dormant_root: None, + cur_leaf_edge: None, + }, + (*self.alloc).clone(), + ) } - - self.length = dfs(self.root.as_ref().unwrap().as_ref()); } - /// Removes empty levels on the top. - fn fix_top(&mut self) { - loop { - { - let node = self.root.as_ref().unwrap().as_ref(); - if node.height() == 0 || node.len() > 0 { - break; - } - } - self.root.as_mut().unwrap().pop_level(); - } + /// Creates a consuming iterator visiting all the keys, in sorted order. + /// The map cannot be used after calling this. + /// The iterator element type is `K`. + /// + /// # Examples + /// + /// ``` + /// use std::collections::BTreeMap; + /// + /// let mut a = BTreeMap::new(); + /// a.insert(2, "b"); + /// a.insert(1, "a"); + /// + /// let keys: Vec = a.into_keys().collect(); + /// assert_eq!(keys, [1, 2]); + /// ``` + #[inline] + #[stable(feature = "map_into_keys_values", since = "1.54.0")] + pub fn into_keys(self) -> IntoKeys { + IntoKeys { inner: self.into_iter() } } - fn fix_right_border(&mut self) { - self.fix_top(); - - { - let mut cur_node = self.root.as_mut().unwrap().as_mut(); - - while let Internal(node) = cur_node.force() { - let mut last_kv = node.last_kv(); - - if last_kv.can_merge() { - cur_node = last_kv.merge().descend(); - } else { - let right_len = last_kv.reborrow().right_edge().descend().len(); - // `MINLEN + 1` to avoid readjust if merge happens on the next level. - if right_len < node::MIN_LEN + 1 { - last_kv.bulk_steal_left(node::MIN_LEN + 1 - right_len); - } - cur_node = last_kv.right_edge().descend(); - } - } - } - - self.fix_top(); + /// Creates a consuming iterator visiting all the values, in order by key. + /// The map cannot be used after calling this. + /// The iterator element type is `V`. + /// + /// # Examples + /// + /// ``` + /// use std::collections::BTreeMap; + /// + /// let mut a = BTreeMap::new(); + /// a.insert(1, "hello"); + /// a.insert(2, "goodbye"); + /// + /// let values: Vec<&str> = a.into_values().collect(); + /// assert_eq!(values, ["hello", "goodbye"]); + /// ``` + #[inline] + #[stable(feature = "map_into_keys_values", since = "1.54.0")] + pub fn into_values(self) -> IntoValues { + IntoValues { inner: self.into_iter() } } - /// The symmetric clone of `fix_right_border`. - fn fix_left_border(&mut self) { - self.fix_top(); - - { - let mut cur_node = self.root.as_mut().unwrap().as_mut(); - - while let Internal(node) = cur_node.force() { - let mut first_kv = node.first_kv(); - - if first_kv.can_merge() { - cur_node = first_kv.merge().descend(); - } else { - let left_len = first_kv.reborrow().left_edge().descend().len(); - if left_len < node::MIN_LEN + 1 { - first_kv.bulk_steal_right(node::MIN_LEN + 1 - left_len); - } - cur_node = first_kv.left_edge().descend(); - } - } - } - - self.fix_top(); + /// Makes a `BTreeMap` from a sorted iterator. + pub(crate) fn bulk_build_from_sorted_iter(iter: I, alloc: A) -> Self + where + K: Ord, + I: IntoIterator, + { + let mut root = Root::new(alloc.clone()); + let mut length = 0; + root.bulk_push(DedupSortedIter::new(iter.into_iter()), &mut length, alloc.clone()); + BTreeMap { root: Some(root), length, alloc: ManuallyDrop::new(alloc), _marker: PhantomData } } } #[stable(feature = "rust1", since = "1.0.0")] -impl<'a, K: 'a, V: 'a> IntoIterator for &'a BTreeMap { +impl<'a, K, V, A: Allocator + Clone> IntoIterator for &'a BTreeMap { type Item = (&'a K, &'a V); type IntoIter = Iter<'a, K, V>; @@ -1369,7 +1487,7 @@ impl<'a, K: 'a, V: 'a> Iterator for Iter<'a, K, V> { None } else { self.length -= 1; - unsafe { Some(self.range.next_unchecked()) } + Some(unsafe { self.range.next_unchecked() }) } } @@ -1380,6 +1498,14 @@ impl<'a, K: 'a, V: 'a> Iterator for Iter<'a, K, V> { fn last(mut self) -> Option<(&'a K, &'a V)> { self.next_back() } + + fn min(mut self) -> Option<(&'a K, &'a V)> { + self.next() + } + + fn max(mut self) -> Option<(&'a K, &'a V)> { + self.next_back() + } } #[stable(feature = "fused", since = "1.26.0")] @@ -1392,7 +1518,7 @@ impl<'a, K: 'a, V: 'a> DoubleEndedIterator for Iter<'a, K, V> { None } else { self.length -= 1; - unsafe { Some(self.range.next_back_unchecked()) } + Some(unsafe { self.range.next_back_unchecked() }) } } } @@ -1412,7 +1538,7 @@ impl Clone for Iter<'_, K, V> { } #[stable(feature = "rust1", since = "1.0.0")] -impl<'a, K: 'a, V: 'a> IntoIterator for &'a mut BTreeMap { +impl<'a, K, V, A: Allocator + Clone> IntoIterator for &'a mut BTreeMap { type Item = (&'a K, &'a mut V); type IntoIter = IterMut<'a, K, V>; @@ -1422,7 +1548,7 @@ impl<'a, K: 'a, V: 'a> IntoIterator for &'a mut BTreeMap { } #[stable(feature = "rust1", since = "1.0.0")] -impl<'a, K: 'a, V: 'a> Iterator for IterMut<'a, K, V> { +impl<'a, K, V> Iterator for IterMut<'a, K, V> { type Item = (&'a K, &'a mut V); fn next(&mut self) -> Option<(&'a K, &'a mut V)> { @@ -1430,8 +1556,7 @@ impl<'a, K: 'a, V: 'a> Iterator for IterMut<'a, K, V> { None } else { self.length -= 1; - let (k, v) = unsafe { self.range.next_unchecked() }; - Some((k, v)) // coerce k from `&mut K` to `&K` + Some(unsafe { self.range.next_unchecked() }) } } @@ -1442,17 +1567,24 @@ impl<'a, K: 'a, V: 'a> Iterator for IterMut<'a, K, V> { fn last(mut self) -> Option<(&'a K, &'a mut V)> { self.next_back() } + + fn min(mut self) -> Option<(&'a K, &'a mut V)> { + self.next() + } + + fn max(mut self) -> Option<(&'a K, &'a mut V)> { + self.next_back() + } } #[stable(feature = "rust1", since = "1.0.0")] -impl<'a, K: 'a, V: 'a> DoubleEndedIterator for IterMut<'a, K, V> { +impl<'a, K, V> DoubleEndedIterator for IterMut<'a, K, V> { fn next_back(&mut self) -> Option<(&'a K, &'a mut V)> { if self.length == 0 { None } else { self.length -= 1; - let (k, v) = unsafe { self.range.next_back_unchecked() }; - Some((k, v)) // coerce k from `&mut K` to `&K` + Some(unsafe { self.range.next_back_unchecked() }) } } } @@ -1467,82 +1599,102 @@ impl ExactSizeIterator for IterMut<'_, K, V> { #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for IterMut<'_, K, V> {} +impl<'a, K, V> IterMut<'a, K, V> { + /// Returns an iterator of references over the remaining items. + #[inline] + pub(super) fn iter(&self) -> Iter<'_, K, V> { + Iter { range: self.range.reborrow(), length: self.length } + } +} + #[stable(feature = "rust1", since = "1.0.0")] -impl IntoIterator for BTreeMap { +impl IntoIterator for BTreeMap { type Item = (K, V); - type IntoIter = IntoIter; - - fn into_iter(self) -> IntoIter { - if self.root.is_none() { - mem::forget(self); - return IntoIter { front: None, back: None, length: 0 }; - } + type IntoIter = IntoIter; - let root1 = unsafe { unwrap_unchecked(ptr::read(&self.root)).into_ref() }; - let root2 = unsafe { unwrap_unchecked(ptr::read(&self.root)).into_ref() }; - let len = self.length; - mem::forget(self); + fn into_iter(self) -> IntoIter { + let mut me = ManuallyDrop::new(self); + if let Some(root) = me.root.take() { + let full_range = root.into_dying().full_range(); - IntoIter { - front: Some(root1.first_leaf_edge()), - back: Some(root2.last_leaf_edge()), - length: len, + IntoIter { + range: full_range, + length: me.length, + alloc: unsafe { ManuallyDrop::take(&mut me.alloc) }, + } + } else { + IntoIter { + range: LazyLeafRange::none(), + length: 0, + alloc: unsafe { ManuallyDrop::take(&mut me.alloc) }, + } } } } #[stable(feature = "btree_drop", since = "1.7.0")] -impl Drop for IntoIter { +impl Drop for IntoIter { fn drop(&mut self) { - struct DropGuard<'a, K, V>(&'a mut IntoIter); + struct DropGuard<'a, K, V, A: Allocator + Clone>(&'a mut IntoIter); - impl<'a, K, V> Drop for DropGuard<'a, K, V> { + impl<'a, K, V, A: Allocator + Clone> Drop for DropGuard<'a, K, V, A> { fn drop(&mut self) { // Continue the same loop we perform below. This only runs when unwinding, so we // don't have to care about panics this time (they'll abort). - while let Some(_) = self.0.next() {} - - unsafe { - let mut node = - unwrap_unchecked(ptr::read(&self.0.front)).into_node().forget_type(); - while let Some(parent) = node.deallocate_and_ascend() { - node = parent.into_node().forget_type(); - } + while let Some(kv) = self.0.dying_next() { + // SAFETY: we consume the dying handle immediately. + unsafe { kv.drop_key_val() }; } } } - while let Some(pair) = self.next() { + while let Some(kv) = self.dying_next() { let guard = DropGuard(self); - drop(pair); + // SAFETY: we don't touch the tree before consuming the dying handle. + unsafe { kv.drop_key_val() }; mem::forget(guard); } - - unsafe { - if let Some(front) = ptr::read(&self.front) { - let mut node = front.into_node().forget_type(); - // Most of the nodes have been deallocated while traversing - // but one pile from a leaf up to the root is left standing. - while let Some(parent) = node.deallocate_and_ascend() { - node = parent.into_node().forget_type(); - } - } - } } } -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for IntoIter { - type Item = (K, V); +impl IntoIter { + /// Core of a `next` method returning a dying KV handle, + /// invalidated by further calls to this function and some others. + fn dying_next( + &mut self, + ) -> Option, marker::KV>> { + if self.length == 0 { + self.range.deallocating_end(self.alloc.clone()); + None + } else { + self.length -= 1; + Some(unsafe { self.range.deallocating_next_unchecked(self.alloc.clone()) }) + } + } - fn next(&mut self) -> Option<(K, V)> { + /// Core of a `next_back` method returning a dying KV handle, + /// invalidated by further calls to this function and some others. + fn dying_next_back( + &mut self, + ) -> Option, marker::KV>> { if self.length == 0 { + self.range.deallocating_end(self.alloc.clone()); None } else { self.length -= 1; - Some(unsafe { self.front.as_mut().unwrap().next_unchecked() }) + Some(unsafe { self.range.deallocating_next_back_unchecked(self.alloc.clone()) }) } } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for IntoIter { + type Item = (K, V); + + fn next(&mut self) -> Option<(K, V)> { + // SAFETY: we consume the dying handle immediately. + self.dying_next().map(unsafe { |kv| kv.into_key_val() }) + } fn size_hint(&self) -> (usize, Option) { (self.length, Some(self.length)) @@ -1550,26 +1702,22 @@ impl Iterator for IntoIter { } #[stable(feature = "rust1", since = "1.0.0")] -impl DoubleEndedIterator for IntoIter { +impl DoubleEndedIterator for IntoIter { fn next_back(&mut self) -> Option<(K, V)> { - if self.length == 0 { - None - } else { - self.length -= 1; - Some(unsafe { self.back.as_mut().unwrap().next_back_unchecked() }) - } + // SAFETY: we consume the dying handle immediately. + self.dying_next_back().map(unsafe { |kv| kv.into_key_val() }) } } #[stable(feature = "rust1", since = "1.0.0")] -impl ExactSizeIterator for IntoIter { +impl ExactSizeIterator for IntoIter { fn len(&self) -> usize { self.length } } #[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for IntoIter {} +impl FusedIterator for IntoIter {} #[stable(feature = "rust1", since = "1.0.0")] impl<'a, K, V> Iterator for Keys<'a, K, V> { @@ -1586,6 +1734,14 @@ impl<'a, K, V> Iterator for Keys<'a, K, V> { fn last(mut self) -> Option<&'a K> { self.next_back() } + + fn min(mut self) -> Option<&'a K> { + self.next() + } + + fn max(mut self) -> Option<&'a K> { + self.next_back() + } } #[stable(feature = "rust1", since = "1.0.0")] @@ -1653,17 +1809,140 @@ impl Clone for Values<'_, K, V> { } } +/// An iterator produced by calling `drain_filter` on BTreeMap. +#[unstable(feature = "btree_drain_filter", issue = "70530")] +pub struct DrainFilter< + 'a, + K, + V, + F, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + Clone = Global, +> where + F: 'a + FnMut(&K, &mut V) -> bool, +{ + pred: F, + inner: DrainFilterInner<'a, K, V>, + /// The BTreeMap will outlive this IntoIter so we don't care about drop order for `alloc`. + alloc: A, +} +/// Most of the implementation of DrainFilter are generic over the type +/// of the predicate, thus also serving for BTreeSet::DrainFilter. +pub(super) struct DrainFilterInner<'a, K, V> { + /// Reference to the length field in the borrowed map, updated live. + length: &'a mut usize, + /// Buried reference to the root field in the borrowed map. + /// Wrapped in `Option` to allow drop handler to `take` it. + dormant_root: Option>>, + /// Contains a leaf edge preceding the next element to be returned, or the last leaf edge. + /// Empty if the map has no root, if iteration went beyond the last leaf edge, + /// or if a panic occurred in the predicate. + cur_leaf_edge: Option, K, V, marker::Leaf>, marker::Edge>>, +} + +#[unstable(feature = "btree_drain_filter", issue = "70530")] +impl Drop for DrainFilter<'_, K, V, F, A> +where + F: FnMut(&K, &mut V) -> bool, +{ + fn drop(&mut self) { + self.for_each(drop); + } +} + +#[unstable(feature = "btree_drain_filter", issue = "70530")] +impl fmt::Debug for DrainFilter<'_, K, V, F> +where + K: fmt::Debug, + V: fmt::Debug, + F: FnMut(&K, &mut V) -> bool, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("DrainFilter").field(&self.inner.peek()).finish() + } +} + +#[unstable(feature = "btree_drain_filter", issue = "70530")] +impl Iterator for DrainFilter<'_, K, V, F, A> +where + F: FnMut(&K, &mut V) -> bool, +{ + type Item = (K, V); + + fn next(&mut self) -> Option<(K, V)> { + self.inner.next(&mut self.pred, self.alloc.clone()) + } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} + +impl<'a, K, V> DrainFilterInner<'a, K, V> { + /// Allow Debug implementations to predict the next element. + pub(super) fn peek(&self) -> Option<(&K, &V)> { + let edge = self.cur_leaf_edge.as_ref()?; + edge.reborrow().next_kv().ok().map(Handle::into_kv) + } + + /// Implementation of a typical `DrainFilter::next` method, given the predicate. + pub(super) fn next(&mut self, pred: &mut F, alloc: A) -> Option<(K, V)> + where + F: FnMut(&K, &mut V) -> bool, + { + while let Ok(mut kv) = self.cur_leaf_edge.take()?.next_kv() { + let (k, v) = kv.kv_mut(); + if pred(k, v) { + *self.length -= 1; + let (kv, pos) = kv.remove_kv_tracking( + || { + // SAFETY: we will touch the root in a way that will not + // invalidate the position returned. + let root = unsafe { self.dormant_root.take().unwrap().awaken() }; + root.pop_internal_level(alloc.clone()); + self.dormant_root = Some(DormantMutRef::new(root).1); + }, + alloc.clone(), + ); + self.cur_leaf_edge = Some(pos); + return Some(kv); + } + self.cur_leaf_edge = Some(kv.next_leaf_edge()); + } + None + } + + /// Implementation of a typical `DrainFilter::size_hint` method. + pub(super) fn size_hint(&self) -> (usize, Option) { + // In most of the btree iterators, `self.length` is the number of elements + // yet to be visited. Here, it includes elements that were visited and that + // the predicate decided not to drain. Making this upper bound more tight + // during iteration would require an extra field. + (0, Some(*self.length)) + } +} + +#[unstable(feature = "btree_drain_filter", issue = "70530")] +impl FusedIterator for DrainFilter<'_, K, V, F> where F: FnMut(&K, &mut V) -> bool {} + #[stable(feature = "btree_range", since = "1.17.0")] impl<'a, K, V> Iterator for Range<'a, K, V> { type Item = (&'a K, &'a V); fn next(&mut self) -> Option<(&'a K, &'a V)> { - if self.is_empty() { None } else { unsafe { Some(self.next_unchecked()) } } + self.inner.next_checked() } fn last(mut self) -> Option<(&'a K, &'a V)> { self.next_back() } + + fn min(mut self) -> Option<(&'a K, &'a V)> { + self.next() + } + + fn max(mut self) -> Option<(&'a K, &'a V)> { + self.next_back() + } } #[stable(feature = "map_values_mut", since = "1.10.0")] @@ -1700,26 +1979,86 @@ impl ExactSizeIterator for ValuesMut<'_, K, V> { #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for ValuesMut<'_, K, V> {} -impl<'a, K, V> Range<'a, K, V> { - fn is_empty(&self) -> bool { - self.front == self.back +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +impl Iterator for IntoKeys { + type Item = K; + + fn next(&mut self) -> Option { + self.inner.next().map(|(k, _)| k) + } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } + + fn last(mut self) -> Option { + self.next_back() + } + + fn min(mut self) -> Option { + self.next() } - unsafe fn next_unchecked(&mut self) -> (&'a K, &'a V) { - unwrap_unchecked(self.front.as_mut()).next_unchecked() + fn max(mut self) -> Option { + self.next_back() } } -#[stable(feature = "btree_range", since = "1.17.0")] -impl<'a, K, V> DoubleEndedIterator for Range<'a, K, V> { - fn next_back(&mut self) -> Option<(&'a K, &'a V)> { - if self.is_empty() { None } else { Some(unsafe { self.next_back_unchecked() }) } +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +impl DoubleEndedIterator for IntoKeys { + fn next_back(&mut self) -> Option { + self.inner.next_back().map(|(k, _)| k) + } +} + +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +impl ExactSizeIterator for IntoKeys { + fn len(&self) -> usize { + self.inner.len() + } +} + +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +impl FusedIterator for IntoKeys {} + +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +impl Iterator for IntoValues { + type Item = V; + + fn next(&mut self) -> Option { + self.inner.next().map(|(_, v)| v) + } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } + + fn last(mut self) -> Option { + self.next_back() + } +} + +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +impl DoubleEndedIterator for IntoValues { + fn next_back(&mut self) -> Option { + self.inner.next_back().map(|(_, v)| v) + } +} + +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +impl ExactSizeIterator for IntoValues { + fn len(&self) -> usize { + self.inner.len() } } -impl<'a, K, V> Range<'a, K, V> { - unsafe fn next_back_unchecked(&mut self) -> (&'a K, &'a V) { - unwrap_unchecked(self.back.as_mut()).next_back_unchecked() +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +impl FusedIterator for IntoValues {} + +#[stable(feature = "btree_range", since = "1.17.0")] +impl<'a, K, V> DoubleEndedIterator for Range<'a, K, V> { + fn next_back(&mut self) -> Option<(&'a K, &'a V)> { + self.inner.next_back_checked() } } @@ -1729,7 +2068,7 @@ impl FusedIterator for Range<'_, K, V> {} #[stable(feature = "btree_range", since = "1.17.0")] impl Clone for Range<'_, K, V> { fn clone(&self) -> Self { - Range { front: self.front, back: self.back } + Range { inner: self.inner.clone() } } } @@ -1738,79 +2077,80 @@ impl<'a, K, V> Iterator for RangeMut<'a, K, V> { type Item = (&'a K, &'a mut V); fn next(&mut self) -> Option<(&'a K, &'a mut V)> { - if self.is_empty() { - None - } else { - let (k, v) = unsafe { self.next_unchecked() }; - Some((k, v)) // coerce k from `&mut K` to `&K` - } + self.inner.next_checked() } fn last(mut self) -> Option<(&'a K, &'a mut V)> { self.next_back() } -} -impl<'a, K, V> RangeMut<'a, K, V> { - fn is_empty(&self) -> bool { - self.front == self.back + fn min(mut self) -> Option<(&'a K, &'a mut V)> { + self.next() } - unsafe fn next_unchecked(&mut self) -> (&'a mut K, &'a mut V) { - unwrap_unchecked(self.front.as_mut()).next_unchecked() + fn max(mut self) -> Option<(&'a K, &'a mut V)> { + self.next_back() } } #[stable(feature = "btree_range", since = "1.17.0")] impl<'a, K, V> DoubleEndedIterator for RangeMut<'a, K, V> { fn next_back(&mut self) -> Option<(&'a K, &'a mut V)> { - if self.is_empty() { - None - } else { - let (k, v) = unsafe { self.next_back_unchecked() }; - Some((k, v)) // coerce k from `&mut K` to `&K` - } + self.inner.next_back_checked() } } #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for RangeMut<'_, K, V> {} -impl<'a, K, V> RangeMut<'a, K, V> { - unsafe fn next_back_unchecked(&mut self) -> (&'a mut K, &'a mut V) { - unwrap_unchecked(self.back.as_mut()).next_back_unchecked() - } -} - #[stable(feature = "rust1", since = "1.0.0")] impl FromIterator<(K, V)> for BTreeMap { fn from_iter>(iter: T) -> BTreeMap { - let mut map = BTreeMap::new(); - map.extend(iter); - map + let mut inputs: Vec<_> = iter.into_iter().collect(); + + if inputs.is_empty() { + return BTreeMap::new(); + } + + // use stable sort to preserve the insertion order. + inputs.sort_by(|a, b| a.0.cmp(&b.0)); + BTreeMap::bulk_build_from_sorted_iter(inputs, Global) } } #[stable(feature = "rust1", since = "1.0.0")] -impl Extend<(K, V)> for BTreeMap { +impl Extend<(K, V)> for BTreeMap { #[inline] fn extend>(&mut self, iter: T) { iter.into_iter().for_each(move |(k, v)| { self.insert(k, v); }); } + + #[inline] + fn extend_one(&mut self, (k, v): (K, V)) { + self.insert(k, v); + } } #[stable(feature = "extend_ref", since = "1.2.0")] -impl<'a, K: Ord + Copy, V: Copy> Extend<(&'a K, &'a V)> for BTreeMap { +impl<'a, K: Ord + Copy, V: Copy, A: Allocator + Clone> Extend<(&'a K, &'a V)> + for BTreeMap +{ fn extend>(&mut self, iter: I) { self.extend(iter.into_iter().map(|(&key, &value)| (key, value))); } + + #[inline] + fn extend_one(&mut self, (&k, &v): (&'a K, &'a V)) { + self.insert(k, v); + } } #[stable(feature = "rust1", since = "1.0.0")] -impl Hash for BTreeMap { +impl Hash for BTreeMap { fn hash(&self, state: &mut H) { + state.write_length_prefix(self.len()); for elt in self { elt.hash(state); } @@ -1818,50 +2158,50 @@ impl Hash for BTreeMap { } #[stable(feature = "rust1", since = "1.0.0")] -impl Default for BTreeMap { - /// Creates an empty `BTreeMap`. +impl Default for BTreeMap { + /// Creates an empty `BTreeMap`. fn default() -> BTreeMap { BTreeMap::new() } } #[stable(feature = "rust1", since = "1.0.0")] -impl PartialEq for BTreeMap { - fn eq(&self, other: &BTreeMap) -> bool { +impl PartialEq for BTreeMap { + fn eq(&self, other: &BTreeMap) -> bool { self.len() == other.len() && self.iter().zip(other).all(|(a, b)| a == b) } } #[stable(feature = "rust1", since = "1.0.0")] -impl Eq for BTreeMap {} +impl Eq for BTreeMap {} #[stable(feature = "rust1", since = "1.0.0")] -impl PartialOrd for BTreeMap { +impl PartialOrd for BTreeMap { #[inline] - fn partial_cmp(&self, other: &BTreeMap) -> Option { + fn partial_cmp(&self, other: &BTreeMap) -> Option { self.iter().partial_cmp(other.iter()) } } #[stable(feature = "rust1", since = "1.0.0")] -impl Ord for BTreeMap { +impl Ord for BTreeMap { #[inline] - fn cmp(&self, other: &BTreeMap) -> Ordering { + fn cmp(&self, other: &BTreeMap) -> Ordering { self.iter().cmp(other.iter()) } } #[stable(feature = "rust1", since = "1.0.0")] -impl Debug for BTreeMap { +impl Debug for BTreeMap { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_map().entries(self.iter()).finish() } } #[stable(feature = "rust1", since = "1.0.0")] -impl Index<&Q> for BTreeMap +impl Index<&Q> for BTreeMap where - K: Borrow, + K: Borrow + Ord, Q: Ord, { type Output = V; @@ -1877,96 +2217,29 @@ where } } -fn range_search>( - root1: NodeRef, - root2: NodeRef, - range: R, -) -> ( - Handle, marker::Edge>, - Handle, marker::Edge>, -) -where - Q: Ord, - K: Borrow, -{ - match (range.start_bound(), range.end_bound()) { - (Excluded(s), Excluded(e)) if s == e => { - panic!("range start and end are equal and excluded in BTreeMap") - } - (Included(s), Included(e)) - | (Included(s), Excluded(e)) - | (Excluded(s), Included(e)) - | (Excluded(s), Excluded(e)) - if s > e => - { - panic!("range start is greater than range end in BTreeMap") +#[stable(feature = "std_collections_from_array", since = "1.56.0")] +impl From<[(K, V); N]> for BTreeMap { + /// Converts a `[(K, V); N]` into a `BTreeMap<(K, V)>`. + /// + /// ``` + /// use std::collections::BTreeMap; + /// + /// let map1 = BTreeMap::from([(1, 2), (3, 4)]); + /// let map2: BTreeMap<_, _> = [(1, 2), (3, 4)].into(); + /// assert_eq!(map1, map2); + /// ``` + fn from(mut arr: [(K, V); N]) -> Self { + if N == 0 { + return BTreeMap::new(); } - _ => {} - }; - - let mut min_node = root1; - let mut max_node = root2; - let mut min_found = false; - let mut max_found = false; - - loop { - let front = match (min_found, range.start_bound()) { - (false, Included(key)) => match search::search_node(min_node, key) { - Found(kv) => { - min_found = true; - kv.left_edge() - } - GoDown(edge) => edge, - }, - (false, Excluded(key)) => match search::search_node(min_node, key) { - Found(kv) => { - min_found = true; - kv.right_edge() - } - GoDown(edge) => edge, - }, - (true, Included(_)) => min_node.last_edge(), - (true, Excluded(_)) => min_node.first_edge(), - (_, Unbounded) => min_node.first_edge(), - }; - - let back = match (max_found, range.end_bound()) { - (false, Included(key)) => match search::search_node(max_node, key) { - Found(kv) => { - max_found = true; - kv.right_edge() - } - GoDown(edge) => edge, - }, - (false, Excluded(key)) => match search::search_node(max_node, key) { - Found(kv) => { - max_found = true; - kv.left_edge() - } - GoDown(edge) => edge, - }, - (true, Included(_)) => max_node.first_edge(), - (true, Excluded(_)) => max_node.last_edge(), - (_, Unbounded) => max_node.last_edge(), - }; - if front.partial_cmp(&back) == Some(Ordering::Greater) { - panic!("Ord is ill-defined in BTreeMap range"); - } - match (front.force(), back.force()) { - (Leaf(f), Leaf(b)) => { - return (f, b); - } - (Internal(min_int), Internal(max_int)) => { - min_node = min_int.descend(); - max_node = max_int.descend(); - } - _ => unreachable!("BTreeMap has different depths"), - }; + // use stable sort to preserve the insertion order. + arr.sort_by(|a, b| a.0.cmp(&b.0)); + BTreeMap::bulk_build_from_sorted_iter(arr, Global) } } -impl BTreeMap { +impl BTreeMap { /// Gets an iterator over the entries of the map, sorted by key. /// /// # Examples @@ -1982,7 +2255,7 @@ impl BTreeMap { /// map.insert(1, "a"); /// /// for (key, value) in map.iter() { - /// println!("{}: {}", key, value); + /// println!("{key}: {value}"); /// } /// /// let (first_key, first_value) = map.iter().next().unwrap(); @@ -1990,12 +2263,12 @@ impl BTreeMap { /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn iter(&self) -> Iter<'_, K, V> { - Iter { - range: Range { - front: self.root.as_ref().map(|r| r.as_ref().first_leaf_edge()), - back: self.root.as_ref().map(|r| r.as_ref().last_leaf_edge()), - }, - length: self.length, + if let Some(root) = &self.root { + let full_range = root.reborrow().full_range(); + + Iter { range: full_range, length: self.length } + } else { + Iter { range: LazyLeafRange::none(), length: 0 } } } @@ -2008,10 +2281,11 @@ impl BTreeMap { /// ``` /// use std::collections::BTreeMap; /// - /// let mut map = BTreeMap::new(); - /// map.insert("a", 1); - /// map.insert("b", 2); - /// map.insert("c", 3); + /// let mut map = BTreeMap::from([ + /// ("a", 1), + /// ("b", 2), + /// ("c", 3), + /// ]); /// /// // add 10 to the value if the key isn't "a" /// for (key, value) in map.iter_mut() { @@ -2022,19 +2296,12 @@ impl BTreeMap { /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn iter_mut(&mut self) -> IterMut<'_, K, V> { - IterMut { - range: if let Some(root) = &mut self.root { - let root1 = root.as_mut(); - let root2 = unsafe { ptr::read(&root1) }; - RangeMut { - front: Some(root1.first_leaf_edge()), - back: Some(root2.last_leaf_edge()), - _marker: PhantomData, - } - } else { - RangeMut { front: None, back: None, _marker: PhantomData } - }, - length: self.length, + if let Some(root) = &mut self.root { + let full_range = root.borrow_valmut().full_range(); + + IterMut { range: full_range, length: self.length, _marker: PhantomData } + } else { + IterMut { range: LazyLeafRange::none(), length: 0, _marker: PhantomData } } } @@ -2120,8 +2387,14 @@ impl BTreeMap { /// a.insert(1, "a"); /// assert_eq!(a.len(), 1); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] - pub fn len(&self) -> usize { + #[rustc_const_unstable( + feature = "const_btree_len", + issue = "71835", + implied_by = "const_btree_new" + )] + pub const fn len(&self) -> usize { self.length } @@ -2139,500 +2412,17 @@ impl BTreeMap { /// a.insert(1, "a"); /// assert!(!a.is_empty()); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] - pub fn is_empty(&self) -> bool { + #[rustc_const_unstable( + feature = "const_btree_len", + issue = "71835", + implied_by = "const_btree_new" + )] + pub const fn is_empty(&self) -> bool { self.len() == 0 } - - /// If the root node is the empty (non-allocated) root node, allocate our - /// own node. - fn ensure_root_is_owned(&mut self) -> &mut node::Root { - self.root.get_or_insert_with(|| node::Root::new_leaf()) - } -} - -impl<'a, K: Ord, V> Entry<'a, K, V> { - /// Ensures a value is in the entry by inserting the default if empty, and returns - /// a mutable reference to the value in the entry. - /// - /// # Examples - /// - /// ``` - /// use std::collections::BTreeMap; - /// - /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); - /// map.entry("poneyland").or_insert(12); - /// - /// assert_eq!(map["poneyland"], 12); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn or_insert(self, default: V) -> &'a mut V { - match self { - Occupied(entry) => entry.into_mut(), - Vacant(entry) => entry.insert(default), - } - } - - /// Ensures a value is in the entry by inserting the result of the default function if empty, - /// and returns a mutable reference to the value in the entry. - /// - /// # Examples - /// - /// ``` - /// use std::collections::BTreeMap; - /// - /// let mut map: BTreeMap<&str, String> = BTreeMap::new(); - /// let s = "hoho".to_string(); - /// - /// map.entry("poneyland").or_insert_with(|| s); - /// - /// assert_eq!(map["poneyland"], "hoho".to_string()); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn or_insert_with V>(self, default: F) -> &'a mut V { - match self { - Occupied(entry) => entry.into_mut(), - Vacant(entry) => entry.insert(default()), - } - } - - /// Returns a reference to this entry's key. - /// - /// # Examples - /// - /// ``` - /// use std::collections::BTreeMap; - /// - /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); - /// assert_eq!(map.entry("poneyland").key(), &"poneyland"); - /// ``` - #[stable(feature = "map_entry_keys", since = "1.10.0")] - pub fn key(&self) -> &K { - match *self { - Occupied(ref entry) => entry.key(), - Vacant(ref entry) => entry.key(), - } - } - - /// Provides in-place mutable access to an occupied entry before any - /// potential inserts into the map. - /// - /// # Examples - /// - /// ``` - /// use std::collections::BTreeMap; - /// - /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); - /// - /// map.entry("poneyland") - /// .and_modify(|e| { *e += 1 }) - /// .or_insert(42); - /// assert_eq!(map["poneyland"], 42); - /// - /// map.entry("poneyland") - /// .and_modify(|e| { *e += 1 }) - /// .or_insert(42); - /// assert_eq!(map["poneyland"], 43); - /// ``` - #[stable(feature = "entry_and_modify", since = "1.26.0")] - pub fn and_modify(self, f: F) -> Self - where - F: FnOnce(&mut V), - { - match self { - Occupied(mut entry) => { - f(entry.get_mut()); - Occupied(entry) - } - Vacant(entry) => Vacant(entry), - } - } -} - -impl<'a, K: Ord, V: Default> Entry<'a, K, V> { - #[stable(feature = "entry_or_default", since = "1.28.0")] - /// Ensures a value is in the entry by inserting the default value if empty, - /// and returns a mutable reference to the value in the entry. - /// - /// # Examples - /// - /// ``` - /// use std::collections::BTreeMap; - /// - /// let mut map: BTreeMap<&str, Option> = BTreeMap::new(); - /// map.entry("poneyland").or_default(); - /// - /// assert_eq!(map["poneyland"], None); - /// ``` - pub fn or_default(self) -> &'a mut V { - match self { - Occupied(entry) => entry.into_mut(), - Vacant(entry) => entry.insert(Default::default()), - } - } } -impl<'a, K: Ord, V> VacantEntry<'a, K, V> { - /// Gets a reference to the key that would be used when inserting a value - /// through the VacantEntry. - /// - /// # Examples - /// - /// ``` - /// use std::collections::BTreeMap; - /// - /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); - /// assert_eq!(map.entry("poneyland").key(), &"poneyland"); - /// ``` - #[stable(feature = "map_entry_keys", since = "1.10.0")] - pub fn key(&self) -> &K { - &self.key - } - - /// Take ownership of the key. - /// - /// # Examples - /// - /// ``` - /// use std::collections::BTreeMap; - /// use std::collections::btree_map::Entry; - /// - /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); - /// - /// if let Entry::Vacant(v) = map.entry("poneyland") { - /// v.into_key(); - /// } - /// ``` - #[stable(feature = "map_entry_recover_keys2", since = "1.12.0")] - pub fn into_key(self) -> K { - self.key - } - - /// Sets the value of the entry with the `VacantEntry`'s key, - /// and returns a mutable reference to it. - /// - /// # Examples - /// - /// ``` - /// use std::collections::BTreeMap; - /// - /// let mut count: BTreeMap<&str, usize> = BTreeMap::new(); - /// - /// // count the number of occurrences of letters in the vec - /// for x in vec!["a","b","a","c","a","b"] { - /// *count.entry(x).or_insert(0) += 1; - /// } - /// - /// assert_eq!(count["a"], 3); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn insert(self, value: V) -> &'a mut V { - *self.length += 1; - - let out_ptr; - - let mut ins_k; - let mut ins_v; - let mut ins_edge; - - let mut cur_parent = match self.handle.insert(self.key, value) { - (Fit(handle), _) => return handle.into_kv_mut().1, - (Split(left, k, v, right), ptr) => { - ins_k = k; - ins_v = v; - ins_edge = right; - out_ptr = ptr; - left.ascend().map_err(|n| n.into_root_mut()) - } - }; - - loop { - match cur_parent { - Ok(parent) => match parent.insert(ins_k, ins_v, ins_edge) { - Fit(_) => return unsafe { &mut *out_ptr }, - Split(left, k, v, right) => { - ins_k = k; - ins_v = v; - ins_edge = right; - cur_parent = left.ascend().map_err(|n| n.into_root_mut()); - } - }, - Err(root) => { - root.push_level().push(ins_k, ins_v, ins_edge); - return unsafe { &mut *out_ptr }; - } - } - } - } -} - -impl<'a, K: Ord, V> OccupiedEntry<'a, K, V> { - /// Gets a reference to the key in the entry. - /// - /// # Examples - /// - /// ``` - /// use std::collections::BTreeMap; - /// - /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); - /// map.entry("poneyland").or_insert(12); - /// assert_eq!(map.entry("poneyland").key(), &"poneyland"); - /// ``` - #[stable(feature = "map_entry_keys", since = "1.10.0")] - pub fn key(&self) -> &K { - self.handle.reborrow().into_kv().0 - } - - /// Take ownership of the key and value from the map. - /// - /// # Examples - /// - /// ``` - /// use std::collections::BTreeMap; - /// use std::collections::btree_map::Entry; - /// - /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); - /// map.entry("poneyland").or_insert(12); - /// - /// if let Entry::Occupied(o) = map.entry("poneyland") { - /// // We delete the entry from the map. - /// o.remove_entry(); - /// } - /// - /// // If now try to get the value, it will panic: - /// // println!("{}", map["poneyland"]); - /// ``` - #[stable(feature = "map_entry_recover_keys2", since = "1.12.0")] - pub fn remove_entry(self) -> (K, V) { - self.remove_kv() - } - - /// Gets a reference to the value in the entry. - /// - /// # Examples - /// - /// ``` - /// use std::collections::BTreeMap; - /// use std::collections::btree_map::Entry; - /// - /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); - /// map.entry("poneyland").or_insert(12); - /// - /// if let Entry::Occupied(o) = map.entry("poneyland") { - /// assert_eq!(o.get(), &12); - /// } - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn get(&self) -> &V { - self.handle.reborrow().into_kv().1 - } - - /// Gets a mutable reference to the value in the entry. - /// - /// If you need a reference to the `OccupiedEntry` that may outlive the - /// destruction of the `Entry` value, see [`into_mut`]. - /// - /// [`into_mut`]: #method.into_mut - /// - /// # Examples - /// - /// ``` - /// use std::collections::BTreeMap; - /// use std::collections::btree_map::Entry; - /// - /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); - /// map.entry("poneyland").or_insert(12); - /// - /// assert_eq!(map["poneyland"], 12); - /// if let Entry::Occupied(mut o) = map.entry("poneyland") { - /// *o.get_mut() += 10; - /// assert_eq!(*o.get(), 22); - /// - /// // We can use the same Entry multiple times. - /// *o.get_mut() += 2; - /// } - /// assert_eq!(map["poneyland"], 24); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn get_mut(&mut self) -> &mut V { - self.handle.kv_mut().1 - } - - /// Converts the entry into a mutable reference to its value. - /// - /// If you need multiple references to the `OccupiedEntry`, see [`get_mut`]. - /// - /// [`get_mut`]: #method.get_mut - /// - /// # Examples - /// - /// ``` - /// use std::collections::BTreeMap; - /// use std::collections::btree_map::Entry; - /// - /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); - /// map.entry("poneyland").or_insert(12); - /// - /// assert_eq!(map["poneyland"], 12); - /// if let Entry::Occupied(o) = map.entry("poneyland") { - /// *o.into_mut() += 10; - /// } - /// assert_eq!(map["poneyland"], 22); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn into_mut(self) -> &'a mut V { - self.handle.into_kv_mut().1 - } - - /// Sets the value of the entry with the `OccupiedEntry`'s key, - /// and returns the entry's old value. - /// - /// # Examples - /// - /// ``` - /// use std::collections::BTreeMap; - /// use std::collections::btree_map::Entry; - /// - /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); - /// map.entry("poneyland").or_insert(12); - /// - /// if let Entry::Occupied(mut o) = map.entry("poneyland") { - /// assert_eq!(o.insert(15), 12); - /// } - /// assert_eq!(map["poneyland"], 15); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn insert(&mut self, value: V) -> V { - mem::replace(self.get_mut(), value) - } - - /// Takes the value of the entry out of the map, and returns it. - /// - /// # Examples - /// - /// ``` - /// use std::collections::BTreeMap; - /// use std::collections::btree_map::Entry; - /// - /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); - /// map.entry("poneyland").or_insert(12); - /// - /// if let Entry::Occupied(o) = map.entry("poneyland") { - /// assert_eq!(o.remove(), 12); - /// } - /// // If we try to get "poneyland"'s value, it'll panic: - /// // println!("{}", map["poneyland"]); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn remove(self) -> V { - self.remove_kv().1 - } - - fn remove_kv(self) -> (K, V) { - *self.length -= 1; - - let (small_leaf, old_key, old_val) = match self.handle.force() { - Leaf(leaf) => { - let (hole, old_key, old_val) = leaf.remove(); - (hole.into_node(), old_key, old_val) - } - Internal(mut internal) => { - let key_loc = internal.kv_mut().0 as *mut K; - let val_loc = internal.kv_mut().1 as *mut V; - - let to_remove = internal.right_edge().descend().first_leaf_edge().right_kv().ok(); - let to_remove = unsafe { unwrap_unchecked(to_remove) }; - - let (hole, key, val) = to_remove.remove(); - - let old_key = unsafe { mem::replace(&mut *key_loc, key) }; - let old_val = unsafe { mem::replace(&mut *val_loc, val) }; - - (hole.into_node(), old_key, old_val) - } - }; - - // Handle underflow - let mut cur_node = small_leaf.forget_type(); - while cur_node.len() < node::MIN_LEN { - match handle_underfull_node(cur_node) { - AtRoot => break, - EmptyParent(_) => unreachable!(), - Merged(parent) => { - if parent.len() == 0 { - // We must be at the root - parent.into_root_mut().pop_level(); - break; - } else { - cur_node = parent.forget_type(); - } - } - Stole(_) => break, - } - } - - (old_key, old_val) - } -} - -enum UnderflowResult<'a, K, V> { - AtRoot, - EmptyParent(NodeRef, K, V, marker::Internal>), - Merged(NodeRef, K, V, marker::Internal>), - Stole(NodeRef, K, V, marker::Internal>), -} - -fn handle_underfull_node( - node: NodeRef, K, V, marker::LeafOrInternal>, -) -> UnderflowResult<'_, K, V> { - let parent = if let Ok(parent) = node.ascend() { - parent - } else { - return AtRoot; - }; - - let (is_left, mut handle) = match parent.left_kv() { - Ok(left) => (true, left), - Err(parent) => match parent.right_kv() { - Ok(right) => (false, right), - Err(parent) => { - return EmptyParent(parent.into_node()); - } - }, - }; - - if handle.can_merge() { - Merged(handle.merge().into_node()) - } else { - if is_left { - handle.steal_left(); - } else { - handle.steal_right(); - } - Stole(handle.into_node()) - } -} - -impl> Iterator for MergeIter { - type Item = (K, V); - - fn next(&mut self) -> Option<(K, V)> { - let res = match (self.left.peek(), self.right.peek()) { - (Some(&(ref left_key, _)), Some(&(ref right_key, _))) => left_key.cmp(right_key), - (Some(_), None) => Ordering::Less, - (None, Some(_)) => Ordering::Greater, - (None, None) => return None, - }; - - // Check which elements comes first and only advance the corresponding iterator. - // If two keys are equal, take the value from `right`. - match res { - Ordering::Less => self.left.next(), - Ordering::Greater => self.right.next(), - Ordering::Equal => { - self.left.next(); - self.right.next() - } - } - } -} +#[cfg(test)] +mod tests; diff --git a/crux-mir/lib/alloc/src/collections/btree/map/entry.rs b/crux-mir/lib/alloc/src/collections/btree/map/entry.rs new file mode 100644 index 000000000..370b58864 --- /dev/null +++ b/crux-mir/lib/alloc/src/collections/btree/map/entry.rs @@ -0,0 +1,565 @@ +use core::fmt::{self, Debug}; +use core::marker::PhantomData; +use core::mem; + +use crate::alloc::{Allocator, Global}; + +use super::super::borrow::DormantMutRef; +use super::super::node::{marker, Handle, NodeRef}; +use super::BTreeMap; + +use Entry::*; + +/// A view into a single entry in a map, which may either be vacant or occupied. +/// +/// This `enum` is constructed from the [`entry`] method on [`BTreeMap`]. +/// +/// [`entry`]: BTreeMap::entry +#[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "BTreeEntry")] +pub enum Entry< + 'a, + K: 'a, + V: 'a, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + Clone = Global, +> { + /// A vacant entry. + #[stable(feature = "rust1", since = "1.0.0")] + Vacant(#[stable(feature = "rust1", since = "1.0.0")] VacantEntry<'a, K, V, A>), + + /// An occupied entry. + #[stable(feature = "rust1", since = "1.0.0")] + Occupied(#[stable(feature = "rust1", since = "1.0.0")] OccupiedEntry<'a, K, V, A>), +} + +#[stable(feature = "debug_btree_map", since = "1.12.0")] +impl Debug for Entry<'_, K, V, A> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + Vacant(ref v) => f.debug_tuple("Entry").field(v).finish(), + Occupied(ref o) => f.debug_tuple("Entry").field(o).finish(), + } + } +} + +/// A view into a vacant entry in a `BTreeMap`. +/// It is part of the [`Entry`] enum. +#[stable(feature = "rust1", since = "1.0.0")] +pub struct VacantEntry< + 'a, + K, + V, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + Clone = Global, +> { + pub(super) key: K, + /// `None` for a (empty) map without root + pub(super) handle: Option, K, V, marker::Leaf>, marker::Edge>>, + pub(super) dormant_map: DormantMutRef<'a, BTreeMap>, + + /// The BTreeMap will outlive this IntoIter so we don't care about drop order for `alloc`. + pub(super) alloc: A, + + // Be invariant in `K` and `V` + pub(super) _marker: PhantomData<&'a mut (K, V)>, +} + +#[stable(feature = "debug_btree_map", since = "1.12.0")] +impl Debug for VacantEntry<'_, K, V, A> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("VacantEntry").field(self.key()).finish() + } +} + +/// A view into an occupied entry in a `BTreeMap`. +/// It is part of the [`Entry`] enum. +#[stable(feature = "rust1", since = "1.0.0")] +pub struct OccupiedEntry< + 'a, + K, + V, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + Clone = Global, +> { + pub(super) handle: Handle, K, V, marker::LeafOrInternal>, marker::KV>, + pub(super) dormant_map: DormantMutRef<'a, BTreeMap>, + + /// The BTreeMap will outlive this IntoIter so we don't care about drop order for `alloc`. + pub(super) alloc: A, + + // Be invariant in `K` and `V` + pub(super) _marker: PhantomData<&'a mut (K, V)>, +} + +#[stable(feature = "debug_btree_map", since = "1.12.0")] +impl Debug for OccupiedEntry<'_, K, V, A> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OccupiedEntry").field("key", self.key()).field("value", self.get()).finish() + } +} + +/// The error returned by [`try_insert`](BTreeMap::try_insert) when the key already exists. +/// +/// Contains the occupied entry, and the value that was not inserted. +#[unstable(feature = "map_try_insert", issue = "82766")] +pub struct OccupiedError<'a, K: 'a, V: 'a, A: Allocator + Clone = Global> { + /// The entry in the map that was already occupied. + pub entry: OccupiedEntry<'a, K, V, A>, + /// The value which was not inserted, because the entry was already occupied. + pub value: V, +} + +#[unstable(feature = "map_try_insert", issue = "82766")] +impl Debug for OccupiedError<'_, K, V, A> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OccupiedError") + .field("key", self.entry.key()) + .field("old_value", self.entry.get()) + .field("new_value", &self.value) + .finish() + } +} + +#[unstable(feature = "map_try_insert", issue = "82766")] +impl<'a, K: Debug + Ord, V: Debug, A: Allocator + Clone> fmt::Display + for OccupiedError<'a, K, V, A> +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "failed to insert {:?}, key {:?} already exists with value {:?}", + self.value, + self.entry.key(), + self.entry.get(), + ) + } +} + +#[unstable(feature = "map_try_insert", issue = "82766")] +impl<'a, K: core::fmt::Debug + Ord, V: core::fmt::Debug> core::error::Error + for crate::collections::btree_map::OccupiedError<'a, K, V> +{ + #[allow(deprecated)] + fn description(&self) -> &str { + "key already exists" + } +} + +impl<'a, K: Ord, V, A: Allocator + Clone> Entry<'a, K, V, A> { + /// Ensures a value is in the entry by inserting the default if empty, and returns + /// a mutable reference to the value in the entry. + /// + /// # Examples + /// + /// ``` + /// use std::collections::BTreeMap; + /// + /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); + /// map.entry("poneyland").or_insert(12); + /// + /// assert_eq!(map["poneyland"], 12); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn or_insert(self, default: V) -> &'a mut V { + match self { + Occupied(entry) => entry.into_mut(), + Vacant(entry) => entry.insert(default), + } + } + + /// Ensures a value is in the entry by inserting the result of the default function if empty, + /// and returns a mutable reference to the value in the entry. + /// + /// # Examples + /// + /// ``` + /// use std::collections::BTreeMap; + /// + /// let mut map: BTreeMap<&str, String> = BTreeMap::new(); + /// let s = "hoho".to_string(); + /// + /// map.entry("poneyland").or_insert_with(|| s); + /// + /// assert_eq!(map["poneyland"], "hoho".to_string()); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn or_insert_with V>(self, default: F) -> &'a mut V { + match self { + Occupied(entry) => entry.into_mut(), + Vacant(entry) => entry.insert(default()), + } + } + + /// Ensures a value is in the entry by inserting, if empty, the result of the default function. + /// This method allows for generating key-derived values for insertion by providing the default + /// function a reference to the key that was moved during the `.entry(key)` method call. + /// + /// The reference to the moved key is provided so that cloning or copying the key is + /// unnecessary, unlike with `.or_insert_with(|| ... )`. + /// + /// # Examples + /// + /// ``` + /// use std::collections::BTreeMap; + /// + /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); + /// + /// map.entry("poneyland").or_insert_with_key(|key| key.chars().count()); + /// + /// assert_eq!(map["poneyland"], 9); + /// ``` + #[inline] + #[stable(feature = "or_insert_with_key", since = "1.50.0")] + pub fn or_insert_with_key V>(self, default: F) -> &'a mut V { + match self { + Occupied(entry) => entry.into_mut(), + Vacant(entry) => { + let value = default(entry.key()); + entry.insert(value) + } + } + } + + /// Returns a reference to this entry's key. + /// + /// # Examples + /// + /// ``` + /// use std::collections::BTreeMap; + /// + /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); + /// assert_eq!(map.entry("poneyland").key(), &"poneyland"); + /// ``` + #[stable(feature = "map_entry_keys", since = "1.10.0")] + pub fn key(&self) -> &K { + match *self { + Occupied(ref entry) => entry.key(), + Vacant(ref entry) => entry.key(), + } + } + + /// Provides in-place mutable access to an occupied entry before any + /// potential inserts into the map. + /// + /// # Examples + /// + /// ``` + /// use std::collections::BTreeMap; + /// + /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); + /// + /// map.entry("poneyland") + /// .and_modify(|e| { *e += 1 }) + /// .or_insert(42); + /// assert_eq!(map["poneyland"], 42); + /// + /// map.entry("poneyland") + /// .and_modify(|e| { *e += 1 }) + /// .or_insert(42); + /// assert_eq!(map["poneyland"], 43); + /// ``` + #[stable(feature = "entry_and_modify", since = "1.26.0")] + pub fn and_modify(self, f: F) -> Self + where + F: FnOnce(&mut V), + { + match self { + Occupied(mut entry) => { + f(entry.get_mut()); + Occupied(entry) + } + Vacant(entry) => Vacant(entry), + } + } +} + +impl<'a, K: Ord, V: Default, A: Allocator + Clone> Entry<'a, K, V, A> { + #[stable(feature = "entry_or_default", since = "1.28.0")] + /// Ensures a value is in the entry by inserting the default value if empty, + /// and returns a mutable reference to the value in the entry. + /// + /// # Examples + /// + /// ``` + /// use std::collections::BTreeMap; + /// + /// let mut map: BTreeMap<&str, Option> = BTreeMap::new(); + /// map.entry("poneyland").or_default(); + /// + /// assert_eq!(map["poneyland"], None); + /// ``` + pub fn or_default(self) -> &'a mut V { + match self { + Occupied(entry) => entry.into_mut(), + Vacant(entry) => entry.insert(Default::default()), + } + } +} + +impl<'a, K: Ord, V, A: Allocator + Clone> VacantEntry<'a, K, V, A> { + /// Gets a reference to the key that would be used when inserting a value + /// through the VacantEntry. + /// + /// # Examples + /// + /// ``` + /// use std::collections::BTreeMap; + /// + /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); + /// assert_eq!(map.entry("poneyland").key(), &"poneyland"); + /// ``` + #[stable(feature = "map_entry_keys", since = "1.10.0")] + pub fn key(&self) -> &K { + &self.key + } + + /// Take ownership of the key. + /// + /// # Examples + /// + /// ``` + /// use std::collections::BTreeMap; + /// use std::collections::btree_map::Entry; + /// + /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); + /// + /// if let Entry::Vacant(v) = map.entry("poneyland") { + /// v.into_key(); + /// } + /// ``` + #[stable(feature = "map_entry_recover_keys2", since = "1.12.0")] + pub fn into_key(self) -> K { + self.key + } + + /// Sets the value of the entry with the `VacantEntry`'s key, + /// and returns a mutable reference to it. + /// + /// # Examples + /// + /// ``` + /// use std::collections::BTreeMap; + /// use std::collections::btree_map::Entry; + /// + /// let mut map: BTreeMap<&str, u32> = BTreeMap::new(); + /// + /// if let Entry::Vacant(o) = map.entry("poneyland") { + /// o.insert(37); + /// } + /// assert_eq!(map["poneyland"], 37); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn insert(self, value: V) -> &'a mut V { + let out_ptr = match self.handle { + None => { + // SAFETY: There is no tree yet so no reference to it exists. + let map = unsafe { self.dormant_map.awaken() }; + let mut root = NodeRef::new_leaf(self.alloc.clone()); + let val_ptr = root.borrow_mut().push(self.key, value) as *mut V; + map.root = Some(root.forget_type()); + map.length = 1; + val_ptr + } + Some(handle) => match handle.insert_recursing(self.key, value, self.alloc.clone()) { + (None, val_ptr) => { + // SAFETY: We have consumed self.handle. + let map = unsafe { self.dormant_map.awaken() }; + map.length += 1; + val_ptr + } + (Some(ins), val_ptr) => { + drop(ins.left); + // SAFETY: We have consumed self.handle and dropped the + // remaining reference to the tree, ins.left. + let map = unsafe { self.dormant_map.awaken() }; + let root = map.root.as_mut().unwrap(); // same as ins.left + root.push_internal_level(self.alloc).push(ins.kv.0, ins.kv.1, ins.right); + map.length += 1; + val_ptr + } + }, + }; + // Now that we have finished growing the tree using borrowed references, + // dereference the pointer to a part of it, that we picked up along the way. + unsafe { &mut *out_ptr } + } +} + +impl<'a, K: Ord, V, A: Allocator + Clone> OccupiedEntry<'a, K, V, A> { + /// Gets a reference to the key in the entry. + /// + /// # Examples + /// + /// ``` + /// use std::collections::BTreeMap; + /// + /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); + /// map.entry("poneyland").or_insert(12); + /// assert_eq!(map.entry("poneyland").key(), &"poneyland"); + /// ``` + #[must_use] + #[stable(feature = "map_entry_keys", since = "1.10.0")] + pub fn key(&self) -> &K { + self.handle.reborrow().into_kv().0 + } + + /// Take ownership of the key and value from the map. + /// + /// # Examples + /// + /// ``` + /// use std::collections::BTreeMap; + /// use std::collections::btree_map::Entry; + /// + /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); + /// map.entry("poneyland").or_insert(12); + /// + /// if let Entry::Occupied(o) = map.entry("poneyland") { + /// // We delete the entry from the map. + /// o.remove_entry(); + /// } + /// + /// // If now try to get the value, it will panic: + /// // println!("{}", map["poneyland"]); + /// ``` + #[stable(feature = "map_entry_recover_keys2", since = "1.12.0")] + pub fn remove_entry(self) -> (K, V) { + self.remove_kv() + } + + /// Gets a reference to the value in the entry. + /// + /// # Examples + /// + /// ``` + /// use std::collections::BTreeMap; + /// use std::collections::btree_map::Entry; + /// + /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); + /// map.entry("poneyland").or_insert(12); + /// + /// if let Entry::Occupied(o) = map.entry("poneyland") { + /// assert_eq!(o.get(), &12); + /// } + /// ``` + #[must_use] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn get(&self) -> &V { + self.handle.reborrow().into_kv().1 + } + + /// Gets a mutable reference to the value in the entry. + /// + /// If you need a reference to the `OccupiedEntry` that may outlive the + /// destruction of the `Entry` value, see [`into_mut`]. + /// + /// [`into_mut`]: OccupiedEntry::into_mut + /// + /// # Examples + /// + /// ``` + /// use std::collections::BTreeMap; + /// use std::collections::btree_map::Entry; + /// + /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); + /// map.entry("poneyland").or_insert(12); + /// + /// assert_eq!(map["poneyland"], 12); + /// if let Entry::Occupied(mut o) = map.entry("poneyland") { + /// *o.get_mut() += 10; + /// assert_eq!(*o.get(), 22); + /// + /// // We can use the same Entry multiple times. + /// *o.get_mut() += 2; + /// } + /// assert_eq!(map["poneyland"], 24); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn get_mut(&mut self) -> &mut V { + self.handle.kv_mut().1 + } + + /// Converts the entry into a mutable reference to its value. + /// + /// If you need multiple references to the `OccupiedEntry`, see [`get_mut`]. + /// + /// [`get_mut`]: OccupiedEntry::get_mut + /// + /// # Examples + /// + /// ``` + /// use std::collections::BTreeMap; + /// use std::collections::btree_map::Entry; + /// + /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); + /// map.entry("poneyland").or_insert(12); + /// + /// assert_eq!(map["poneyland"], 12); + /// if let Entry::Occupied(o) = map.entry("poneyland") { + /// *o.into_mut() += 10; + /// } + /// assert_eq!(map["poneyland"], 22); + /// ``` + #[must_use = "`self` will be dropped if the result is not used"] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn into_mut(self) -> &'a mut V { + self.handle.into_val_mut() + } + + /// Sets the value of the entry with the `OccupiedEntry`'s key, + /// and returns the entry's old value. + /// + /// # Examples + /// + /// ``` + /// use std::collections::BTreeMap; + /// use std::collections::btree_map::Entry; + /// + /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); + /// map.entry("poneyland").or_insert(12); + /// + /// if let Entry::Occupied(mut o) = map.entry("poneyland") { + /// assert_eq!(o.insert(15), 12); + /// } + /// assert_eq!(map["poneyland"], 15); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn insert(&mut self, value: V) -> V { + mem::replace(self.get_mut(), value) + } + + /// Takes the value of the entry out of the map, and returns it. + /// + /// # Examples + /// + /// ``` + /// use std::collections::BTreeMap; + /// use std::collections::btree_map::Entry; + /// + /// let mut map: BTreeMap<&str, usize> = BTreeMap::new(); + /// map.entry("poneyland").or_insert(12); + /// + /// if let Entry::Occupied(o) = map.entry("poneyland") { + /// assert_eq!(o.remove(), 12); + /// } + /// // If we try to get "poneyland"'s value, it'll panic: + /// // println!("{}", map["poneyland"]); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn remove(self) -> V { + self.remove_kv().1 + } + + // Body of `remove_entry`, probably separate because the name reflects the returned pair. + pub(super) fn remove_kv(self) -> (K, V) { + let mut emptied_internal_root = false; + let (old_kv, _) = + self.handle.remove_kv_tracking(|| emptied_internal_root = true, self.alloc.clone()); + // SAFETY: we consumed the intermediate root borrow, `self.handle`. + let map = unsafe { self.dormant_map.awaken() }; + map.length -= 1; + if emptied_internal_root { + let root = map.root.as_mut().unwrap(); + root.pop_internal_level(self.alloc); + } + old_kv + } +} diff --git a/crux-mir/lib/alloc/src/collections/btree/map/tests.rs b/crux-mir/lib/alloc/src/collections/btree/map/tests.rs new file mode 100644 index 000000000..700b1463b --- /dev/null +++ b/crux-mir/lib/alloc/src/collections/btree/map/tests.rs @@ -0,0 +1,2338 @@ +use super::Entry::{Occupied, Vacant}; +use super::*; +use crate::boxed::Box; +use crate::fmt::Debug; +use crate::rc::Rc; +use crate::string::{String, ToString}; +use crate::testing::crash_test::{CrashTestDummy, Panic}; +use crate::testing::ord_chaos::{Cyclic3, Governed, Governor}; +use crate::testing::rng::DeterministicRng; +use crate::vec::Vec; +use std::cmp::Ordering; +use std::convert::TryFrom; +use std::iter::{self, FromIterator}; +use std::mem; +use std::ops::Bound::{self, Excluded, Included, Unbounded}; +use std::ops::RangeBounds; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; + +// Minimum number of elements to insert, to guarantee a tree with 2 levels, +// i.e., a tree who's root is an internal node at height 1, with edges to leaf nodes. +// It's not the minimum size: removing an element from such a tree does not always reduce height. +const MIN_INSERTS_HEIGHT_1: usize = node::CAPACITY + 1; + +// Minimum number of elements to insert in ascending order, to guarantee a tree with 3 levels, +// i.e., a tree who's root is an internal node at height 2, with edges to more internal nodes. +// It's not the minimum size: removing an element from such a tree does not always reduce height. +const MIN_INSERTS_HEIGHT_2: usize = 89; + +// Gathers all references from a mutable iterator and makes sure Miri notices if +// using them is dangerous. +fn test_all_refs<'a, T: 'a>(dummy: &mut T, iter: impl Iterator) { + // Gather all those references. + let mut refs: Vec<&mut T> = iter.collect(); + // Use them all. Twice, to be sure we got all interleavings. + for r in refs.iter_mut() { + mem::swap(dummy, r); + } + for r in refs { + mem::swap(dummy, r); + } +} + +impl BTreeMap { + // Panics if the map (or the code navigating it) is corrupted. + fn check_invariants(&self) { + if let Some(root) = &self.root { + let root_node = root.reborrow(); + + // Check the back pointers top-down, before we attempt to rely on + // more serious navigation code. + assert!(root_node.ascend().is_err()); + root_node.assert_back_pointers(); + + // Check consistency of `length` with what navigation code encounters. + assert_eq!(self.length, root_node.calc_length()); + + // Lastly, check the invariant causing the least harm. + root_node.assert_min_len(if root_node.height() > 0 { 1 } else { 0 }); + } else { + assert_eq!(self.length, 0); + } + + // Check that `assert_strictly_ascending` will encounter all keys. + assert_eq!(self.length, self.keys().count()); + } + + // Panics if the map is corrupted or if the keys are not in strictly + // ascending order, in the current opinion of the `Ord` implementation. + // If the `Ord` implementation violates transitivity, this method does not + // guarantee that all keys are unique, just that adjacent keys are unique. + fn check(&self) + where + K: Debug + Ord, + { + self.check_invariants(); + self.assert_strictly_ascending(); + } + + // Returns the height of the root, if any. + fn height(&self) -> Option { + self.root.as_ref().map(node::Root::height) + } + + fn dump_keys(&self) -> String + where + K: Debug, + { + if let Some(root) = self.root.as_ref() { + root.reborrow().dump_keys() + } else { + String::from("not yet allocated") + } + } + + // Panics if the keys are not in strictly ascending order. + fn assert_strictly_ascending(&self) + where + K: Debug + Ord, + { + let mut keys = self.keys(); + if let Some(mut previous) = keys.next() { + for next in keys { + assert!(previous < next, "{:?} >= {:?}", previous, next); + previous = next; + } + } + } + + // Transform the tree to minimize wasted space, obtaining fewer nodes that + // are mostly filled up to their capacity. The same compact tree could have + // been obtained by inserting keys in a shrewd order. + fn compact(&mut self) + where + K: Ord, + { + let iter = mem::take(self).into_iter(); + if !iter.is_empty() { + self.root.insert(Root::new(*self.alloc)).bulk_push(iter, &mut self.length, *self.alloc); + } + } +} + +impl<'a, K: 'a, V: 'a> NodeRef, K, V, marker::LeafOrInternal> { + fn assert_min_len(self, min_len: usize) { + assert!(self.len() >= min_len, "node len {} < {}", self.len(), min_len); + if let node::ForceResult::Internal(node) = self.force() { + for idx in 0..=node.len() { + let edge = unsafe { Handle::new_edge(node, idx) }; + edge.descend().assert_min_len(MIN_LEN); + } + } + } +} + +// Tests our value of MIN_INSERTS_HEIGHT_2. Failure may mean you just need to +// adapt that value to match a change in node::CAPACITY or the choices made +// during insertion, otherwise other test cases may fail or be less useful. +#[test] +fn test_levels() { + let mut map = BTreeMap::new(); + map.check(); + assert_eq!(map.height(), None); + assert_eq!(map.len(), 0); + + map.insert(0, ()); + while map.height() == Some(0) { + let last_key = *map.last_key_value().unwrap().0; + map.insert(last_key + 1, ()); + } + map.check(); + // Structure: + // - 1 element in internal root node with 2 children + // - 6 elements in left leaf child + // - 5 elements in right leaf child + assert_eq!(map.height(), Some(1)); + assert_eq!(map.len(), MIN_INSERTS_HEIGHT_1, "{}", map.dump_keys()); + + while map.height() == Some(1) { + let last_key = *map.last_key_value().unwrap().0; + map.insert(last_key + 1, ()); + } + map.check(); + // Structure: + // - 1 element in internal root node with 2 children + // - 6 elements in left internal child with 7 grandchildren + // - 42 elements in left child's 7 grandchildren with 6 elements each + // - 5 elements in right internal child with 6 grandchildren + // - 30 elements in right child's 5 first grandchildren with 6 elements each + // - 5 elements in right child's last grandchild + assert_eq!(map.height(), Some(2)); + assert_eq!(map.len(), MIN_INSERTS_HEIGHT_2, "{}", map.dump_keys()); +} + +// Ensures the testing infrastructure usually notices order violations. +#[test] +#[should_panic] +fn test_check_ord_chaos() { + let gov = Governor::new(); + let map = BTreeMap::from([(Governed(1, &gov), ()), (Governed(2, &gov), ())]); + gov.flip(); + map.check(); +} + +// Ensures the testing infrastructure doesn't always mind order violations. +#[test] +fn test_check_invariants_ord_chaos() { + let gov = Governor::new(); + let map = BTreeMap::from([(Governed(1, &gov), ()), (Governed(2, &gov), ())]); + gov.flip(); + map.check_invariants(); +} + +#[test] +fn test_basic_large() { + let mut map = BTreeMap::new(); + // Miri is too slow + let size = if cfg!(miri) { MIN_INSERTS_HEIGHT_2 } else { 10000 }; + let size = size + (size % 2); // round up to even number + assert_eq!(map.len(), 0); + + for i in 0..size { + assert_eq!(map.insert(i, 10 * i), None); + assert_eq!(map.len(), i + 1); + } + + assert_eq!(map.first_key_value(), Some((&0, &0))); + assert_eq!(map.last_key_value(), Some((&(size - 1), &(10 * (size - 1))))); + assert_eq!(map.first_entry().unwrap().key(), &0); + assert_eq!(map.last_entry().unwrap().key(), &(size - 1)); + + for i in 0..size { + assert_eq!(map.get(&i).unwrap(), &(i * 10)); + } + + for i in size..size * 2 { + assert_eq!(map.get(&i), None); + } + + for i in 0..size { + assert_eq!(map.insert(i, 100 * i), Some(10 * i)); + assert_eq!(map.len(), size); + } + + for i in 0..size { + assert_eq!(map.get(&i).unwrap(), &(i * 100)); + } + + for i in 0..size / 2 { + assert_eq!(map.remove(&(i * 2)), Some(i * 200)); + assert_eq!(map.len(), size - i - 1); + } + + for i in 0..size / 2 { + assert_eq!(map.get(&(2 * i)), None); + assert_eq!(map.get(&(2 * i + 1)).unwrap(), &(i * 200 + 100)); + } + + for i in 0..size / 2 { + assert_eq!(map.remove(&(2 * i)), None); + assert_eq!(map.remove(&(2 * i + 1)), Some(i * 200 + 100)); + assert_eq!(map.len(), size / 2 - i - 1); + } + map.check(); +} + +#[test] +fn test_basic_small() { + let mut map = BTreeMap::new(); + // Empty, root is absent (None): + assert_eq!(map.remove(&1), None); + assert_eq!(map.len(), 0); + assert_eq!(map.get(&1), None); + assert_eq!(map.get_mut(&1), None); + assert_eq!(map.first_key_value(), None); + assert_eq!(map.last_key_value(), None); + assert_eq!(map.keys().count(), 0); + assert_eq!(map.values().count(), 0); + assert_eq!(map.range(..).next(), None); + assert_eq!(map.range(..1).next(), None); + assert_eq!(map.range(1..).next(), None); + assert_eq!(map.range(1..=1).next(), None); + assert_eq!(map.range(1..2).next(), None); + assert_eq!(map.height(), None); + assert_eq!(map.insert(1, 1), None); + assert_eq!(map.height(), Some(0)); + map.check(); + + // 1 key-value pair: + assert_eq!(map.len(), 1); + assert_eq!(map.get(&1), Some(&1)); + assert_eq!(map.get_mut(&1), Some(&mut 1)); + assert_eq!(map.first_key_value(), Some((&1, &1))); + assert_eq!(map.last_key_value(), Some((&1, &1))); + assert_eq!(map.keys().collect::>(), vec![&1]); + assert_eq!(map.values().collect::>(), vec![&1]); + assert_eq!(map.insert(1, 2), Some(1)); + assert_eq!(map.len(), 1); + assert_eq!(map.get(&1), Some(&2)); + assert_eq!(map.get_mut(&1), Some(&mut 2)); + assert_eq!(map.first_key_value(), Some((&1, &2))); + assert_eq!(map.last_key_value(), Some((&1, &2))); + assert_eq!(map.keys().collect::>(), vec![&1]); + assert_eq!(map.values().collect::>(), vec![&2]); + assert_eq!(map.insert(2, 4), None); + assert_eq!(map.height(), Some(0)); + map.check(); + + // 2 key-value pairs: + assert_eq!(map.len(), 2); + assert_eq!(map.get(&2), Some(&4)); + assert_eq!(map.get_mut(&2), Some(&mut 4)); + assert_eq!(map.first_key_value(), Some((&1, &2))); + assert_eq!(map.last_key_value(), Some((&2, &4))); + assert_eq!(map.keys().collect::>(), vec![&1, &2]); + assert_eq!(map.values().collect::>(), vec![&2, &4]); + assert_eq!(map.remove(&1), Some(2)); + assert_eq!(map.height(), Some(0)); + map.check(); + + // 1 key-value pair: + assert_eq!(map.len(), 1); + assert_eq!(map.get(&1), None); + assert_eq!(map.get_mut(&1), None); + assert_eq!(map.get(&2), Some(&4)); + assert_eq!(map.get_mut(&2), Some(&mut 4)); + assert_eq!(map.first_key_value(), Some((&2, &4))); + assert_eq!(map.last_key_value(), Some((&2, &4))); + assert_eq!(map.keys().collect::>(), vec![&2]); + assert_eq!(map.values().collect::>(), vec![&4]); + assert_eq!(map.remove(&2), Some(4)); + assert_eq!(map.height(), Some(0)); + map.check(); + + // Empty but root is owned (Some(...)): + assert_eq!(map.len(), 0); + assert_eq!(map.get(&1), None); + assert_eq!(map.get_mut(&1), None); + assert_eq!(map.first_key_value(), None); + assert_eq!(map.last_key_value(), None); + assert_eq!(map.keys().count(), 0); + assert_eq!(map.values().count(), 0); + assert_eq!(map.range(..).next(), None); + assert_eq!(map.range(..1).next(), None); + assert_eq!(map.range(1..).next(), None); + assert_eq!(map.range(1..=1).next(), None); + assert_eq!(map.range(1..2).next(), None); + assert_eq!(map.remove(&1), None); + assert_eq!(map.height(), Some(0)); + map.check(); +} + +#[test] +fn test_iter() { + // Miri is too slow + let size = if cfg!(miri) { 200 } else { 10000 }; + let mut map = BTreeMap::from_iter((0..size).map(|i| (i, i))); + + fn test(size: usize, mut iter: T) + where + T: Iterator, + { + for i in 0..size { + assert_eq!(iter.size_hint(), (size - i, Some(size - i))); + assert_eq!(iter.next().unwrap(), (i, i)); + } + assert_eq!(iter.size_hint(), (0, Some(0))); + assert_eq!(iter.next(), None); + } + test(size, map.iter().map(|(&k, &v)| (k, v))); + test(size, map.iter_mut().map(|(&k, &mut v)| (k, v))); + test(size, map.into_iter()); +} + +#[test] +fn test_iter_rev() { + // Miri is too slow + let size = if cfg!(miri) { 200 } else { 10000 }; + let mut map = BTreeMap::from_iter((0..size).map(|i| (i, i))); + + fn test(size: usize, mut iter: T) + where + T: Iterator, + { + for i in 0..size { + assert_eq!(iter.size_hint(), (size - i, Some(size - i))); + assert_eq!(iter.next().unwrap(), (size - i - 1, size - i - 1)); + } + assert_eq!(iter.size_hint(), (0, Some(0))); + assert_eq!(iter.next(), None); + } + test(size, map.iter().rev().map(|(&k, &v)| (k, v))); + test(size, map.iter_mut().rev().map(|(&k, &mut v)| (k, v))); + test(size, map.into_iter().rev()); +} + +// Specifically tests iter_mut's ability to mutate the value of pairs in-line. +fn do_test_iter_mut_mutation(size: usize) +where + T: Copy + Debug + Ord + TryFrom, + >::Error: Debug, +{ + let zero = T::try_from(0).unwrap(); + let mut map = BTreeMap::from_iter((0..size).map(|i| (T::try_from(i).unwrap(), zero))); + + // Forward and backward iteration sees enough pairs (also tested elsewhere) + assert_eq!(map.iter_mut().count(), size); + assert_eq!(map.iter_mut().rev().count(), size); + + // Iterate forwards, trying to mutate to unique values + for (i, (k, v)) in map.iter_mut().enumerate() { + assert_eq!(*k, T::try_from(i).unwrap()); + assert_eq!(*v, zero); + *v = T::try_from(i + 1).unwrap(); + } + + // Iterate backwards, checking that mutations succeeded and trying to mutate again + for (i, (k, v)) in map.iter_mut().rev().enumerate() { + assert_eq!(*k, T::try_from(size - i - 1).unwrap()); + assert_eq!(*v, T::try_from(size - i).unwrap()); + *v = T::try_from(2 * size - i).unwrap(); + } + + // Check that backward mutations succeeded + for (i, (k, v)) in map.iter_mut().enumerate() { + assert_eq!(*k, T::try_from(i).unwrap()); + assert_eq!(*v, T::try_from(size + i + 1).unwrap()); + } + map.check(); +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, PartialOrd, Ord)] +#[repr(align(32))] +struct Align32(usize); + +impl TryFrom for Align32 { + type Error = (); + + fn try_from(s: usize) -> Result { + Ok(Align32(s)) + } +} + +#[test] +fn test_iter_mut_mutation() { + // Check many alignments and trees with roots at various heights. + do_test_iter_mut_mutation::(0); + do_test_iter_mut_mutation::(1); + do_test_iter_mut_mutation::(MIN_INSERTS_HEIGHT_1); + do_test_iter_mut_mutation::(MIN_INSERTS_HEIGHT_2); + do_test_iter_mut_mutation::(1); + do_test_iter_mut_mutation::(MIN_INSERTS_HEIGHT_1); + do_test_iter_mut_mutation::(MIN_INSERTS_HEIGHT_2); + do_test_iter_mut_mutation::(1); + do_test_iter_mut_mutation::(MIN_INSERTS_HEIGHT_1); + do_test_iter_mut_mutation::(MIN_INSERTS_HEIGHT_2); + do_test_iter_mut_mutation::(1); + do_test_iter_mut_mutation::(MIN_INSERTS_HEIGHT_1); + do_test_iter_mut_mutation::(MIN_INSERTS_HEIGHT_2); + do_test_iter_mut_mutation::(1); + do_test_iter_mut_mutation::(MIN_INSERTS_HEIGHT_1); + do_test_iter_mut_mutation::(MIN_INSERTS_HEIGHT_2); + do_test_iter_mut_mutation::(1); + do_test_iter_mut_mutation::(MIN_INSERTS_HEIGHT_1); + do_test_iter_mut_mutation::(MIN_INSERTS_HEIGHT_2); +} + +#[test] +fn test_values_mut() { + let mut a = BTreeMap::from_iter((0..MIN_INSERTS_HEIGHT_2).map(|i| (i, i))); + test_all_refs(&mut 13, a.values_mut()); + a.check(); +} + +#[test] +fn test_values_mut_mutation() { + let mut a = BTreeMap::new(); + a.insert(1, String::from("hello")); + a.insert(2, String::from("goodbye")); + + for value in a.values_mut() { + value.push_str("!"); + } + + let values = Vec::from_iter(a.values().cloned()); + assert_eq!(values, [String::from("hello!"), String::from("goodbye!")]); + a.check(); +} + +#[test] +fn test_iter_entering_root_twice() { + let mut map = BTreeMap::from([(0, 0), (1, 1)]); + let mut it = map.iter_mut(); + let front = it.next().unwrap(); + let back = it.next_back().unwrap(); + assert_eq!(front, (&0, &mut 0)); + assert_eq!(back, (&1, &mut 1)); + *front.1 = 24; + *back.1 = 42; + assert_eq!(front, (&0, &mut 24)); + assert_eq!(back, (&1, &mut 42)); + assert_eq!(it.next(), None); + assert_eq!(it.next_back(), None); + map.check(); +} + +#[test] +fn test_iter_descending_to_same_node_twice() { + let mut map = BTreeMap::from_iter((0..MIN_INSERTS_HEIGHT_1).map(|i| (i, i))); + let mut it = map.iter_mut(); + // Descend into first child. + let front = it.next().unwrap(); + // Descend into first child again, after running through second child. + while it.next_back().is_some() {} + // Check immutable access. + assert_eq!(front, (&0, &mut 0)); + // Perform mutable access. + *front.1 = 42; + map.check(); +} + +#[test] +fn test_iter_mixed() { + // Miri is too slow + let size = if cfg!(miri) { 200 } else { 10000 }; + + let mut map = BTreeMap::from_iter((0..size).map(|i| (i, i))); + + fn test(size: usize, mut iter: T) + where + T: Iterator + DoubleEndedIterator, + { + for i in 0..size / 4 { + assert_eq!(iter.size_hint(), (size - i * 2, Some(size - i * 2))); + assert_eq!(iter.next().unwrap(), (i, i)); + assert_eq!(iter.next_back().unwrap(), (size - i - 1, size - i - 1)); + } + for i in size / 4..size * 3 / 4 { + assert_eq!(iter.size_hint(), (size * 3 / 4 - i, Some(size * 3 / 4 - i))); + assert_eq!(iter.next().unwrap(), (i, i)); + } + assert_eq!(iter.size_hint(), (0, Some(0))); + assert_eq!(iter.next(), None); + } + test(size, map.iter().map(|(&k, &v)| (k, v))); + test(size, map.iter_mut().map(|(&k, &mut v)| (k, v))); + test(size, map.into_iter()); +} + +#[test] +fn test_iter_min_max() { + let mut a = BTreeMap::new(); + assert_eq!(a.iter().min(), None); + assert_eq!(a.iter().max(), None); + assert_eq!(a.iter_mut().min(), None); + assert_eq!(a.iter_mut().max(), None); + assert_eq!(a.range(..).min(), None); + assert_eq!(a.range(..).max(), None); + assert_eq!(a.range_mut(..).min(), None); + assert_eq!(a.range_mut(..).max(), None); + assert_eq!(a.keys().min(), None); + assert_eq!(a.keys().max(), None); + assert_eq!(a.values().min(), None); + assert_eq!(a.values().max(), None); + assert_eq!(a.values_mut().min(), None); + assert_eq!(a.values_mut().max(), None); + a.insert(1, 42); + a.insert(2, 24); + assert_eq!(a.iter().min(), Some((&1, &42))); + assert_eq!(a.iter().max(), Some((&2, &24))); + assert_eq!(a.iter_mut().min(), Some((&1, &mut 42))); + assert_eq!(a.iter_mut().max(), Some((&2, &mut 24))); + assert_eq!(a.range(..).min(), Some((&1, &42))); + assert_eq!(a.range(..).max(), Some((&2, &24))); + assert_eq!(a.range_mut(..).min(), Some((&1, &mut 42))); + assert_eq!(a.range_mut(..).max(), Some((&2, &mut 24))); + assert_eq!(a.keys().min(), Some(&1)); + assert_eq!(a.keys().max(), Some(&2)); + assert_eq!(a.values().min(), Some(&24)); + assert_eq!(a.values().max(), Some(&42)); + assert_eq!(a.values_mut().min(), Some(&mut 24)); + assert_eq!(a.values_mut().max(), Some(&mut 42)); + a.check(); +} + +fn range_keys(map: &BTreeMap, range: impl RangeBounds) -> Vec { + Vec::from_iter(map.range(range).map(|(&k, &v)| { + assert_eq!(k, v); + k + })) +} + +#[test] +fn test_range_small() { + let size = 4; + + let all = Vec::from_iter(1..=size); + let (first, last) = (vec![all[0]], vec![all[size as usize - 1]]); + let map = BTreeMap::from_iter(all.iter().copied().map(|i| (i, i))); + + assert_eq!(range_keys(&map, (Excluded(0), Excluded(size + 1))), all); + assert_eq!(range_keys(&map, (Excluded(0), Included(size + 1))), all); + assert_eq!(range_keys(&map, (Excluded(0), Included(size))), all); + assert_eq!(range_keys(&map, (Excluded(0), Unbounded)), all); + assert_eq!(range_keys(&map, (Included(0), Excluded(size + 1))), all); + assert_eq!(range_keys(&map, (Included(0), Included(size + 1))), all); + assert_eq!(range_keys(&map, (Included(0), Included(size))), all); + assert_eq!(range_keys(&map, (Included(0), Unbounded)), all); + assert_eq!(range_keys(&map, (Included(1), Excluded(size + 1))), all); + assert_eq!(range_keys(&map, (Included(1), Included(size + 1))), all); + assert_eq!(range_keys(&map, (Included(1), Included(size))), all); + assert_eq!(range_keys(&map, (Included(1), Unbounded)), all); + assert_eq!(range_keys(&map, (Unbounded, Excluded(size + 1))), all); + assert_eq!(range_keys(&map, (Unbounded, Included(size + 1))), all); + assert_eq!(range_keys(&map, (Unbounded, Included(size))), all); + assert_eq!(range_keys(&map, ..), all); + + assert_eq!(range_keys(&map, (Excluded(0), Excluded(1))), vec![]); + assert_eq!(range_keys(&map, (Excluded(0), Included(0))), vec![]); + assert_eq!(range_keys(&map, (Included(0), Included(0))), vec![]); + assert_eq!(range_keys(&map, (Included(0), Excluded(1))), vec![]); + assert_eq!(range_keys(&map, (Unbounded, Excluded(1))), vec![]); + assert_eq!(range_keys(&map, (Unbounded, Included(0))), vec![]); + assert_eq!(range_keys(&map, (Excluded(0), Excluded(2))), first); + assert_eq!(range_keys(&map, (Excluded(0), Included(1))), first); + assert_eq!(range_keys(&map, (Included(0), Excluded(2))), first); + assert_eq!(range_keys(&map, (Included(0), Included(1))), first); + assert_eq!(range_keys(&map, (Included(1), Excluded(2))), first); + assert_eq!(range_keys(&map, (Included(1), Included(1))), first); + assert_eq!(range_keys(&map, (Unbounded, Excluded(2))), first); + assert_eq!(range_keys(&map, (Unbounded, Included(1))), first); + assert_eq!(range_keys(&map, (Excluded(size - 1), Excluded(size + 1))), last); + assert_eq!(range_keys(&map, (Excluded(size - 1), Included(size + 1))), last); + assert_eq!(range_keys(&map, (Excluded(size - 1), Included(size))), last); + assert_eq!(range_keys(&map, (Excluded(size - 1), Unbounded)), last); + assert_eq!(range_keys(&map, (Included(size), Excluded(size + 1))), last); + assert_eq!(range_keys(&map, (Included(size), Included(size + 1))), last); + assert_eq!(range_keys(&map, (Included(size), Included(size))), last); + assert_eq!(range_keys(&map, (Included(size), Unbounded)), last); + assert_eq!(range_keys(&map, (Excluded(size), Excluded(size + 1))), vec![]); + assert_eq!(range_keys(&map, (Excluded(size), Included(size))), vec![]); + assert_eq!(range_keys(&map, (Excluded(size), Unbounded)), vec![]); + assert_eq!(range_keys(&map, (Included(size + 1), Excluded(size + 1))), vec![]); + assert_eq!(range_keys(&map, (Included(size + 1), Included(size + 1))), vec![]); + assert_eq!(range_keys(&map, (Included(size + 1), Unbounded)), vec![]); + + assert_eq!(range_keys(&map, ..3), vec![1, 2]); + assert_eq!(range_keys(&map, 3..), vec![3, 4]); + assert_eq!(range_keys(&map, 2..=3), vec![2, 3]); +} + +#[test] +fn test_range_height_1() { + // Tests tree with a root and 2 leaves. We test around the middle of the + // keys because one of those is the single key in the root node. + let map = BTreeMap::from_iter((0..MIN_INSERTS_HEIGHT_1 as i32).map(|i| (i, i))); + let middle = MIN_INSERTS_HEIGHT_1 as i32 / 2; + for root in middle - 2..=middle + 2 { + assert_eq!(range_keys(&map, (Excluded(root), Excluded(root + 1))), vec![]); + assert_eq!(range_keys(&map, (Excluded(root), Included(root + 1))), vec![root + 1]); + assert_eq!(range_keys(&map, (Included(root), Excluded(root + 1))), vec![root]); + assert_eq!(range_keys(&map, (Included(root), Included(root + 1))), vec![root, root + 1]); + + assert_eq!(range_keys(&map, (Excluded(root - 1), Excluded(root))), vec![]); + assert_eq!(range_keys(&map, (Included(root - 1), Excluded(root))), vec![root - 1]); + assert_eq!(range_keys(&map, (Excluded(root - 1), Included(root))), vec![root]); + assert_eq!(range_keys(&map, (Included(root - 1), Included(root))), vec![root - 1, root]); + } +} + +#[test] +fn test_range_large() { + let size = 200; + + let all = Vec::from_iter(1..=size); + let (first, last) = (vec![all[0]], vec![all[size as usize - 1]]); + let map = BTreeMap::from_iter(all.iter().copied().map(|i| (i, i))); + + assert_eq!(range_keys(&map, (Excluded(0), Excluded(size + 1))), all); + assert_eq!(range_keys(&map, (Excluded(0), Included(size + 1))), all); + assert_eq!(range_keys(&map, (Excluded(0), Included(size))), all); + assert_eq!(range_keys(&map, (Excluded(0), Unbounded)), all); + assert_eq!(range_keys(&map, (Included(0), Excluded(size + 1))), all); + assert_eq!(range_keys(&map, (Included(0), Included(size + 1))), all); + assert_eq!(range_keys(&map, (Included(0), Included(size))), all); + assert_eq!(range_keys(&map, (Included(0), Unbounded)), all); + assert_eq!(range_keys(&map, (Included(1), Excluded(size + 1))), all); + assert_eq!(range_keys(&map, (Included(1), Included(size + 1))), all); + assert_eq!(range_keys(&map, (Included(1), Included(size))), all); + assert_eq!(range_keys(&map, (Included(1), Unbounded)), all); + assert_eq!(range_keys(&map, (Unbounded, Excluded(size + 1))), all); + assert_eq!(range_keys(&map, (Unbounded, Included(size + 1))), all); + assert_eq!(range_keys(&map, (Unbounded, Included(size))), all); + assert_eq!(range_keys(&map, ..), all); + + assert_eq!(range_keys(&map, (Excluded(0), Excluded(1))), vec![]); + assert_eq!(range_keys(&map, (Excluded(0), Included(0))), vec![]); + assert_eq!(range_keys(&map, (Included(0), Included(0))), vec![]); + assert_eq!(range_keys(&map, (Included(0), Excluded(1))), vec![]); + assert_eq!(range_keys(&map, (Unbounded, Excluded(1))), vec![]); + assert_eq!(range_keys(&map, (Unbounded, Included(0))), vec![]); + assert_eq!(range_keys(&map, (Excluded(0), Excluded(2))), first); + assert_eq!(range_keys(&map, (Excluded(0), Included(1))), first); + assert_eq!(range_keys(&map, (Included(0), Excluded(2))), first); + assert_eq!(range_keys(&map, (Included(0), Included(1))), first); + assert_eq!(range_keys(&map, (Included(1), Excluded(2))), first); + assert_eq!(range_keys(&map, (Included(1), Included(1))), first); + assert_eq!(range_keys(&map, (Unbounded, Excluded(2))), first); + assert_eq!(range_keys(&map, (Unbounded, Included(1))), first); + assert_eq!(range_keys(&map, (Excluded(size - 1), Excluded(size + 1))), last); + assert_eq!(range_keys(&map, (Excluded(size - 1), Included(size + 1))), last); + assert_eq!(range_keys(&map, (Excluded(size - 1), Included(size))), last); + assert_eq!(range_keys(&map, (Excluded(size - 1), Unbounded)), last); + assert_eq!(range_keys(&map, (Included(size), Excluded(size + 1))), last); + assert_eq!(range_keys(&map, (Included(size), Included(size + 1))), last); + assert_eq!(range_keys(&map, (Included(size), Included(size))), last); + assert_eq!(range_keys(&map, (Included(size), Unbounded)), last); + assert_eq!(range_keys(&map, (Excluded(size), Excluded(size + 1))), vec![]); + assert_eq!(range_keys(&map, (Excluded(size), Included(size))), vec![]); + assert_eq!(range_keys(&map, (Excluded(size), Unbounded)), vec![]); + assert_eq!(range_keys(&map, (Included(size + 1), Excluded(size + 1))), vec![]); + assert_eq!(range_keys(&map, (Included(size + 1), Included(size + 1))), vec![]); + assert_eq!(range_keys(&map, (Included(size + 1), Unbounded)), vec![]); + + fn check<'a, L, R>(lhs: L, rhs: R) + where + L: IntoIterator, + R: IntoIterator, + { + assert_eq!(Vec::from_iter(lhs), Vec::from_iter(rhs)); + } + + check(map.range(..=100), map.range(..101)); + check(map.range(5..=8), vec![(&5, &5), (&6, &6), (&7, &7), (&8, &8)]); + check(map.range(-1..=2), vec![(&1, &1), (&2, &2)]); +} + +#[test] +fn test_range_inclusive_max_value() { + let max = usize::MAX; + let map = BTreeMap::from([(max, 0)]); + assert_eq!(Vec::from_iter(map.range(max..=max)), &[(&max, &0)]); +} + +#[test] +fn test_range_equal_empty_cases() { + let map = BTreeMap::from_iter((0..5).map(|i| (i, i))); + assert_eq!(map.range((Included(2), Excluded(2))).next(), None); + assert_eq!(map.range((Excluded(2), Included(2))).next(), None); +} + +#[test] +#[should_panic] +fn test_range_equal_excluded() { + let map = BTreeMap::from_iter((0..5).map(|i| (i, i))); + let _ = map.range((Excluded(2), Excluded(2))); +} + +#[test] +#[should_panic] +fn test_range_backwards_1() { + let map = BTreeMap::from_iter((0..5).map(|i| (i, i))); + let _ = map.range((Included(3), Included(2))); +} + +#[test] +#[should_panic] +fn test_range_backwards_2() { + let map = BTreeMap::from_iter((0..5).map(|i| (i, i))); + let _ = map.range((Included(3), Excluded(2))); +} + +#[test] +#[should_panic] +fn test_range_backwards_3() { + let map = BTreeMap::from_iter((0..5).map(|i| (i, i))); + let _ = map.range((Excluded(3), Included(2))); +} + +#[test] +#[should_panic] +fn test_range_backwards_4() { + let map = BTreeMap::from_iter((0..5).map(|i| (i, i))); + let _ = map.range((Excluded(3), Excluded(2))); +} + +#[test] +fn test_range_finding_ill_order_in_map() { + let mut map = BTreeMap::new(); + map.insert(Cyclic3::B, ()); + // Lacking static_assert, call `range` conditionally, to emphasise that + // we cause a different panic than `test_range_backwards_1` does. + // A more refined `should_panic` would be welcome. + if Cyclic3::C < Cyclic3::A { + let _ = map.range(Cyclic3::C..=Cyclic3::A); + } +} + +#[test] +fn test_range_finding_ill_order_in_range_ord() { + // Has proper order the first time asked, then flips around. + struct EvilTwin(i32); + + impl PartialOrd for EvilTwin { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } + } + + static COMPARES: AtomicUsize = AtomicUsize::new(0); + impl Ord for EvilTwin { + fn cmp(&self, other: &Self) -> Ordering { + let ord = self.0.cmp(&other.0); + if COMPARES.fetch_add(1, SeqCst) > 0 { ord.reverse() } else { ord } + } + } + + impl PartialEq for EvilTwin { + fn eq(&self, other: &Self) -> bool { + self.0.eq(&other.0) + } + } + + impl Eq for EvilTwin {} + + #[derive(PartialEq, Eq, PartialOrd, Ord)] + struct CompositeKey(i32, EvilTwin); + + impl Borrow for CompositeKey { + fn borrow(&self) -> &EvilTwin { + &self.1 + } + } + + let map = BTreeMap::from_iter((0..12).map(|i| (CompositeKey(i, EvilTwin(i)), ()))); + let _ = map.range(EvilTwin(5)..=EvilTwin(7)); +} + +#[test] +fn test_range_1000() { + // Miri is too slow + let size = if cfg!(miri) { MIN_INSERTS_HEIGHT_2 as u32 } else { 1000 }; + let map = BTreeMap::from_iter((0..size).map(|i| (i, i))); + + fn test(map: &BTreeMap, size: u32, min: Bound<&u32>, max: Bound<&u32>) { + let mut kvs = map.range((min, max)).map(|(&k, &v)| (k, v)); + let mut pairs = (0..size).map(|i| (i, i)); + + for (kv, pair) in kvs.by_ref().zip(pairs.by_ref()) { + assert_eq!(kv, pair); + } + assert_eq!(kvs.next(), None); + assert_eq!(pairs.next(), None); + } + test(&map, size, Included(&0), Excluded(&size)); + test(&map, size, Unbounded, Excluded(&size)); + test(&map, size, Included(&0), Included(&(size - 1))); + test(&map, size, Unbounded, Included(&(size - 1))); + test(&map, size, Included(&0), Unbounded); + test(&map, size, Unbounded, Unbounded); +} + +#[test] +fn test_range_borrowed_key() { + let mut map = BTreeMap::new(); + map.insert("aardvark".to_string(), 1); + map.insert("baboon".to_string(), 2); + map.insert("coyote".to_string(), 3); + map.insert("dingo".to_string(), 4); + // NOTE: would like to use simply "b".."d" here... + let mut iter = map.range::((Included("b"), Excluded("d"))); + assert_eq!(iter.next(), Some((&"baboon".to_string(), &2))); + assert_eq!(iter.next(), Some((&"coyote".to_string(), &3))); + assert_eq!(iter.next(), None); +} + +#[test] +fn test_range() { + let size = 200; + // Miri is too slow + let step = if cfg!(miri) { 66 } else { 1 }; + let map = BTreeMap::from_iter((0..size).map(|i| (i, i))); + + for i in (0..size).step_by(step) { + for j in (i..size).step_by(step) { + let mut kvs = map.range((Included(&i), Included(&j))).map(|(&k, &v)| (k, v)); + let mut pairs = (i..=j).map(|i| (i, i)); + + for (kv, pair) in kvs.by_ref().zip(pairs.by_ref()) { + assert_eq!(kv, pair); + } + assert_eq!(kvs.next(), None); + assert_eq!(pairs.next(), None); + } + } +} + +#[test] +fn test_range_mut() { + let size = 200; + // Miri is too slow + let step = if cfg!(miri) { 66 } else { 1 }; + let mut map = BTreeMap::from_iter((0..size).map(|i| (i, i))); + + for i in (0..size).step_by(step) { + for j in (i..size).step_by(step) { + let mut kvs = map.range_mut((Included(&i), Included(&j))).map(|(&k, &mut v)| (k, v)); + let mut pairs = (i..=j).map(|i| (i, i)); + + for (kv, pair) in kvs.by_ref().zip(pairs.by_ref()) { + assert_eq!(kv, pair); + } + assert_eq!(kvs.next(), None); + assert_eq!(pairs.next(), None); + } + } + map.check(); +} + +#[should_panic(expected = "range start is greater than range end in BTreeMap")] +#[test] +fn test_range_panic_1() { + let mut map = BTreeMap::new(); + map.insert(3, "a"); + map.insert(5, "b"); + map.insert(8, "c"); + + let _invalid_range = map.range((Included(&8), Included(&3))); +} + +#[should_panic(expected = "range start and end are equal and excluded in BTreeMap")] +#[test] +fn test_range_panic_2() { + let mut map = BTreeMap::new(); + map.insert(3, "a"); + map.insert(5, "b"); + map.insert(8, "c"); + + let _invalid_range = map.range((Excluded(&5), Excluded(&5))); +} + +#[should_panic(expected = "range start and end are equal and excluded in BTreeMap")] +#[test] +fn test_range_panic_3() { + let mut map: BTreeMap = BTreeMap::new(); + map.insert(3, ()); + map.insert(5, ()); + map.insert(8, ()); + + let _invalid_range = map.range((Excluded(&5), Excluded(&5))); +} + +#[test] +fn test_retain() { + let mut map = BTreeMap::from_iter((0..100).map(|x| (x, x * 10))); + + map.retain(|&k, _| k % 2 == 0); + assert_eq!(map.len(), 50); + assert_eq!(map[&2], 20); + assert_eq!(map[&4], 40); + assert_eq!(map[&6], 60); +} + +mod test_drain_filter { + use super::*; + + #[test] + fn empty() { + let mut map: BTreeMap = BTreeMap::new(); + map.drain_filter(|_, _| unreachable!("there's nothing to decide on")); + assert_eq!(map.height(), None); + map.check(); + } + + // Explicitly consumes the iterator, where most test cases drop it instantly. + #[test] + fn consumed_keeping_all() { + let pairs = (0..3).map(|i| (i, i)); + let mut map = BTreeMap::from_iter(pairs); + assert!(map.drain_filter(|_, _| false).eq(iter::empty())); + map.check(); + } + + // Explicitly consumes the iterator, where most test cases drop it instantly. + #[test] + fn consumed_removing_all() { + let pairs = (0..3).map(|i| (i, i)); + let mut map = BTreeMap::from_iter(pairs.clone()); + assert!(map.drain_filter(|_, _| true).eq(pairs)); + assert!(map.is_empty()); + map.check(); + } + + // Explicitly consumes the iterator and modifies values through it. + #[test] + fn mutating_and_keeping() { + let pairs = (0..3).map(|i| (i, i)); + let mut map = BTreeMap::from_iter(pairs); + assert!( + map.drain_filter(|_, v| { + *v += 6; + false + }) + .eq(iter::empty()) + ); + assert!(map.keys().copied().eq(0..3)); + assert!(map.values().copied().eq(6..9)); + map.check(); + } + + // Explicitly consumes the iterator and modifies values through it. + #[test] + fn mutating_and_removing() { + let pairs = (0..3).map(|i| (i, i)); + let mut map = BTreeMap::from_iter(pairs); + assert!( + map.drain_filter(|_, v| { + *v += 6; + true + }) + .eq((0..3).map(|i| (i, i + 6))) + ); + assert!(map.is_empty()); + map.check(); + } + + #[test] + fn underfull_keeping_all() { + let pairs = (0..3).map(|i| (i, i)); + let mut map = BTreeMap::from_iter(pairs); + map.drain_filter(|_, _| false); + assert!(map.keys().copied().eq(0..3)); + map.check(); + } + + #[test] + fn underfull_removing_one() { + let pairs = (0..3).map(|i| (i, i)); + for doomed in 0..3 { + let mut map = BTreeMap::from_iter(pairs.clone()); + map.drain_filter(|i, _| *i == doomed); + assert_eq!(map.len(), 2); + map.check(); + } + } + + #[test] + fn underfull_keeping_one() { + let pairs = (0..3).map(|i| (i, i)); + for sacred in 0..3 { + let mut map = BTreeMap::from_iter(pairs.clone()); + map.drain_filter(|i, _| *i != sacred); + assert!(map.keys().copied().eq(sacred..=sacred)); + map.check(); + } + } + + #[test] + fn underfull_removing_all() { + let pairs = (0..3).map(|i| (i, i)); + let mut map = BTreeMap::from_iter(pairs); + map.drain_filter(|_, _| true); + assert!(map.is_empty()); + map.check(); + } + + #[test] + fn height_0_keeping_all() { + let pairs = (0..node::CAPACITY).map(|i| (i, i)); + let mut map = BTreeMap::from_iter(pairs); + map.drain_filter(|_, _| false); + assert!(map.keys().copied().eq(0..node::CAPACITY)); + map.check(); + } + + #[test] + fn height_0_removing_one() { + let pairs = (0..node::CAPACITY).map(|i| (i, i)); + for doomed in 0..node::CAPACITY { + let mut map = BTreeMap::from_iter(pairs.clone()); + map.drain_filter(|i, _| *i == doomed); + assert_eq!(map.len(), node::CAPACITY - 1); + map.check(); + } + } + + #[test] + fn height_0_keeping_one() { + let pairs = (0..node::CAPACITY).map(|i| (i, i)); + for sacred in 0..node::CAPACITY { + let mut map = BTreeMap::from_iter(pairs.clone()); + map.drain_filter(|i, _| *i != sacred); + assert!(map.keys().copied().eq(sacred..=sacred)); + map.check(); + } + } + + #[test] + fn height_0_removing_all() { + let pairs = (0..node::CAPACITY).map(|i| (i, i)); + let mut map = BTreeMap::from_iter(pairs); + map.drain_filter(|_, _| true); + assert!(map.is_empty()); + map.check(); + } + + #[test] + fn height_0_keeping_half() { + let mut map = BTreeMap::from_iter((0..16).map(|i| (i, i))); + assert_eq!(map.drain_filter(|i, _| *i % 2 == 0).count(), 8); + assert_eq!(map.len(), 8); + map.check(); + } + + #[test] + fn height_1_removing_all() { + let pairs = (0..MIN_INSERTS_HEIGHT_1).map(|i| (i, i)); + let mut map = BTreeMap::from_iter(pairs); + map.drain_filter(|_, _| true); + assert!(map.is_empty()); + map.check(); + } + + #[test] + fn height_1_removing_one() { + let pairs = (0..MIN_INSERTS_HEIGHT_1).map(|i| (i, i)); + for doomed in 0..MIN_INSERTS_HEIGHT_1 { + let mut map = BTreeMap::from_iter(pairs.clone()); + map.drain_filter(|i, _| *i == doomed); + assert_eq!(map.len(), MIN_INSERTS_HEIGHT_1 - 1); + map.check(); + } + } + + #[test] + fn height_1_keeping_one() { + let pairs = (0..MIN_INSERTS_HEIGHT_1).map(|i| (i, i)); + for sacred in 0..MIN_INSERTS_HEIGHT_1 { + let mut map = BTreeMap::from_iter(pairs.clone()); + map.drain_filter(|i, _| *i != sacred); + assert!(map.keys().copied().eq(sacred..=sacred)); + map.check(); + } + } + + #[test] + fn height_2_removing_one() { + let pairs = (0..MIN_INSERTS_HEIGHT_2).map(|i| (i, i)); + for doomed in (0..MIN_INSERTS_HEIGHT_2).step_by(12) { + let mut map = BTreeMap::from_iter(pairs.clone()); + map.drain_filter(|i, _| *i == doomed); + assert_eq!(map.len(), MIN_INSERTS_HEIGHT_2 - 1); + map.check(); + } + } + + #[test] + fn height_2_keeping_one() { + let pairs = (0..MIN_INSERTS_HEIGHT_2).map(|i| (i, i)); + for sacred in (0..MIN_INSERTS_HEIGHT_2).step_by(12) { + let mut map = BTreeMap::from_iter(pairs.clone()); + map.drain_filter(|i, _| *i != sacred); + assert!(map.keys().copied().eq(sacred..=sacred)); + map.check(); + } + } + + #[test] + fn height_2_removing_all() { + let pairs = (0..MIN_INSERTS_HEIGHT_2).map(|i| (i, i)); + let mut map = BTreeMap::from_iter(pairs); + map.drain_filter(|_, _| true); + assert!(map.is_empty()); + map.check(); + } + + #[test] + fn drop_panic_leak() { + let a = CrashTestDummy::new(0); + let b = CrashTestDummy::new(1); + let c = CrashTestDummy::new(2); + let mut map = BTreeMap::new(); + map.insert(a.spawn(Panic::Never), ()); + map.insert(b.spawn(Panic::InDrop), ()); + map.insert(c.spawn(Panic::Never), ()); + + catch_unwind(move || drop(map.drain_filter(|dummy, _| dummy.query(true)))).unwrap_err(); + + assert_eq!(a.queried(), 1); + assert_eq!(b.queried(), 1); + assert_eq!(c.queried(), 0); + assert_eq!(a.dropped(), 1); + assert_eq!(b.dropped(), 1); + assert_eq!(c.dropped(), 1); + } + + #[test] + fn pred_panic_leak() { + let a = CrashTestDummy::new(0); + let b = CrashTestDummy::new(1); + let c = CrashTestDummy::new(2); + let mut map = BTreeMap::new(); + map.insert(a.spawn(Panic::Never), ()); + map.insert(b.spawn(Panic::InQuery), ()); + map.insert(c.spawn(Panic::InQuery), ()); + + catch_unwind(AssertUnwindSafe(|| drop(map.drain_filter(|dummy, _| dummy.query(true))))) + .unwrap_err(); + + assert_eq!(a.queried(), 1); + assert_eq!(b.queried(), 1); + assert_eq!(c.queried(), 0); + assert_eq!(a.dropped(), 1); + assert_eq!(b.dropped(), 0); + assert_eq!(c.dropped(), 0); + assert_eq!(map.len(), 2); + assert_eq!(map.first_entry().unwrap().key().id(), 1); + assert_eq!(map.last_entry().unwrap().key().id(), 2); + map.check(); + } + + // Same as above, but attempt to use the iterator again after the panic in the predicate + #[test] + fn pred_panic_reuse() { + let a = CrashTestDummy::new(0); + let b = CrashTestDummy::new(1); + let c = CrashTestDummy::new(2); + let mut map = BTreeMap::new(); + map.insert(a.spawn(Panic::Never), ()); + map.insert(b.spawn(Panic::InQuery), ()); + map.insert(c.spawn(Panic::InQuery), ()); + + { + let mut it = map.drain_filter(|dummy, _| dummy.query(true)); + catch_unwind(AssertUnwindSafe(|| while it.next().is_some() {})).unwrap_err(); + // Iterator behaviour after a panic is explicitly unspecified, + // so this is just the current implementation: + let result = catch_unwind(AssertUnwindSafe(|| it.next())); + assert!(matches!(result, Ok(None))); + } + + assert_eq!(a.queried(), 1); + assert_eq!(b.queried(), 1); + assert_eq!(c.queried(), 0); + assert_eq!(a.dropped(), 1); + assert_eq!(b.dropped(), 0); + assert_eq!(c.dropped(), 0); + assert_eq!(map.len(), 2); + assert_eq!(map.first_entry().unwrap().key().id(), 1); + assert_eq!(map.last_entry().unwrap().key().id(), 2); + map.check(); + } +} + +#[test] +fn test_borrow() { + // make sure these compile -- using the Borrow trait + { + let mut map = BTreeMap::new(); + map.insert("0".to_string(), 1); + assert_eq!(map["0"], 1); + } + + { + let mut map = BTreeMap::new(); + map.insert(Box::new(0), 1); + assert_eq!(map[&0], 1); + } + + { + let mut map = BTreeMap::new(); + map.insert(Box::new([0, 1]) as Box<[i32]>, 1); + assert_eq!(map[&[0, 1][..]], 1); + } + + { + let mut map = BTreeMap::new(); + map.insert(Rc::new(0), 1); + assert_eq!(map[&0], 1); + } + + #[allow(dead_code)] + fn get(v: &BTreeMap, ()>, t: &T) { + let _ = v.get(t); + } + + #[allow(dead_code)] + fn get_mut(v: &mut BTreeMap, ()>, t: &T) { + let _ = v.get_mut(t); + } + + #[allow(dead_code)] + fn get_key_value(v: &BTreeMap, ()>, t: &T) { + let _ = v.get_key_value(t); + } + + #[allow(dead_code)] + fn contains_key(v: &BTreeMap, ()>, t: &T) { + let _ = v.contains_key(t); + } + + #[allow(dead_code)] + fn range(v: &BTreeMap, ()>, t: T) { + let _ = v.range(t..); + } + + #[allow(dead_code)] + fn range_mut(v: &mut BTreeMap, ()>, t: T) { + let _ = v.range_mut(t..); + } + + #[allow(dead_code)] + fn remove(v: &mut BTreeMap, ()>, t: &T) { + v.remove(t); + } + + #[allow(dead_code)] + fn remove_entry(v: &mut BTreeMap, ()>, t: &T) { + v.remove_entry(t); + } + + #[allow(dead_code)] + fn split_off(v: &mut BTreeMap, ()>, t: &T) { + v.split_off(t); + } +} + +#[test] +fn test_entry() { + let xs = [(1, 10), (2, 20), (3, 30), (4, 40), (5, 50), (6, 60)]; + + let mut map = BTreeMap::from(xs); + + // Existing key (insert) + match map.entry(1) { + Vacant(_) => unreachable!(), + Occupied(mut view) => { + assert_eq!(view.get(), &10); + assert_eq!(view.insert(100), 10); + } + } + assert_eq!(map.get(&1).unwrap(), &100); + assert_eq!(map.len(), 6); + + // Existing key (update) + match map.entry(2) { + Vacant(_) => unreachable!(), + Occupied(mut view) => { + let v = view.get_mut(); + *v *= 10; + } + } + assert_eq!(map.get(&2).unwrap(), &200); + assert_eq!(map.len(), 6); + map.check(); + + // Existing key (take) + match map.entry(3) { + Vacant(_) => unreachable!(), + Occupied(view) => { + assert_eq!(view.remove(), 30); + } + } + assert_eq!(map.get(&3), None); + assert_eq!(map.len(), 5); + map.check(); + + // Inexistent key (insert) + match map.entry(10) { + Occupied(_) => unreachable!(), + Vacant(view) => { + assert_eq!(*view.insert(1000), 1000); + } + } + assert_eq!(map.get(&10).unwrap(), &1000); + assert_eq!(map.len(), 6); + map.check(); +} + +#[test] +fn test_extend_ref() { + let mut a = BTreeMap::new(); + a.insert(1, "one"); + let mut b = BTreeMap::new(); + b.insert(2, "two"); + b.insert(3, "three"); + + a.extend(&b); + + assert_eq!(a.len(), 3); + assert_eq!(a[&1], "one"); + assert_eq!(a[&2], "two"); + assert_eq!(a[&3], "three"); + a.check(); +} + +#[test] +fn test_zst() { + let mut m = BTreeMap::new(); + assert_eq!(m.len(), 0); + + assert_eq!(m.insert((), ()), None); + assert_eq!(m.len(), 1); + + assert_eq!(m.insert((), ()), Some(())); + assert_eq!(m.len(), 1); + assert_eq!(m.iter().count(), 1); + + m.clear(); + assert_eq!(m.len(), 0); + + for _ in 0..100 { + m.insert((), ()); + } + + assert_eq!(m.len(), 1); + assert_eq!(m.iter().count(), 1); + m.check(); +} + +// This test's only purpose is to ensure that zero-sized keys with nonsensical orderings +// do not cause segfaults when used with zero-sized values. All other map behavior is +// undefined. +#[test] +fn test_bad_zst() { + #[derive(Clone, Copy, Debug)] + struct Bad; + + impl PartialEq for Bad { + fn eq(&self, _: &Self) -> bool { + false + } + } + + impl Eq for Bad {} + + impl PartialOrd for Bad { + fn partial_cmp(&self, _: &Self) -> Option { + Some(Ordering::Less) + } + } + + impl Ord for Bad { + fn cmp(&self, _: &Self) -> Ordering { + Ordering::Less + } + } + + let mut m = BTreeMap::new(); + + for _ in 0..100 { + m.insert(Bad, Bad); + } + m.check(); +} + +#[test] +fn test_clear() { + let mut map = BTreeMap::new(); + for &len in &[MIN_INSERTS_HEIGHT_1, MIN_INSERTS_HEIGHT_2, 0, node::CAPACITY] { + for i in 0..len { + map.insert(i, ()); + } + assert_eq!(map.len(), len); + map.clear(); + map.check(); + assert_eq!(map.height(), None); + } +} + +#[test] +fn test_clear_drop_panic_leak() { + let a = CrashTestDummy::new(0); + let b = CrashTestDummy::new(1); + let c = CrashTestDummy::new(2); + + let mut map = BTreeMap::new(); + map.insert(a.spawn(Panic::Never), ()); + map.insert(b.spawn(Panic::InDrop), ()); + map.insert(c.spawn(Panic::Never), ()); + + catch_unwind(AssertUnwindSafe(|| map.clear())).unwrap_err(); + assert_eq!(a.dropped(), 1); + assert_eq!(b.dropped(), 1); + assert_eq!(c.dropped(), 1); + assert_eq!(map.len(), 0); + + drop(map); + assert_eq!(a.dropped(), 1); + assert_eq!(b.dropped(), 1); + assert_eq!(c.dropped(), 1); +} + +#[test] +fn test_clone() { + let mut map = BTreeMap::new(); + let size = MIN_INSERTS_HEIGHT_1; + assert_eq!(map.len(), 0); + + for i in 0..size { + assert_eq!(map.insert(i, 10 * i), None); + assert_eq!(map.len(), i + 1); + map.check(); + assert_eq!(map, map.clone()); + } + + for i in 0..size { + assert_eq!(map.insert(i, 100 * i), Some(10 * i)); + assert_eq!(map.len(), size); + map.check(); + assert_eq!(map, map.clone()); + } + + for i in 0..size / 2 { + assert_eq!(map.remove(&(i * 2)), Some(i * 200)); + assert_eq!(map.len(), size - i - 1); + map.check(); + assert_eq!(map, map.clone()); + } + + for i in 0..size / 2 { + assert_eq!(map.remove(&(2 * i)), None); + assert_eq!(map.remove(&(2 * i + 1)), Some(i * 200 + 100)); + assert_eq!(map.len(), size / 2 - i - 1); + map.check(); + assert_eq!(map, map.clone()); + } + + // Test a tree with 2 semi-full levels and a tree with 3 levels. + map = BTreeMap::from_iter((1..MIN_INSERTS_HEIGHT_2).map(|i| (i, i))); + assert_eq!(map.len(), MIN_INSERTS_HEIGHT_2 - 1); + assert_eq!(map, map.clone()); + map.insert(0, 0); + assert_eq!(map.len(), MIN_INSERTS_HEIGHT_2); + assert_eq!(map, map.clone()); + map.check(); +} + +fn test_clone_panic_leak(size: usize) { + for i in 0..size { + let dummies = Vec::from_iter((0..size).map(|id| CrashTestDummy::new(id))); + let map = BTreeMap::from_iter(dummies.iter().map(|dummy| { + let panic = if dummy.id == i { Panic::InClone } else { Panic::Never }; + (dummy.spawn(panic), ()) + })); + + catch_unwind(|| map.clone()).unwrap_err(); + for d in &dummies { + assert_eq!(d.cloned(), if d.id <= i { 1 } else { 0 }, "id={}/{}", d.id, i); + assert_eq!(d.dropped(), if d.id < i { 1 } else { 0 }, "id={}/{}", d.id, i); + } + assert_eq!(map.len(), size); + + drop(map); + for d in &dummies { + assert_eq!(d.cloned(), if d.id <= i { 1 } else { 0 }, "id={}/{}", d.id, i); + assert_eq!(d.dropped(), if d.id < i { 2 } else { 1 }, "id={}/{}", d.id, i); + } + } +} + +#[test] +fn test_clone_panic_leak_height_0() { + test_clone_panic_leak(3) +} + +#[test] +fn test_clone_panic_leak_height_1() { + test_clone_panic_leak(MIN_INSERTS_HEIGHT_1) +} + +#[test] +fn test_clone_from() { + let mut map1 = BTreeMap::new(); + let max_size = MIN_INSERTS_HEIGHT_1; + + // Range to max_size inclusive, because i is the size of map1 being tested. + for i in 0..=max_size { + let mut map2 = BTreeMap::new(); + for j in 0..i { + let mut map1_copy = map2.clone(); + map1_copy.clone_from(&map1); // small cloned from large + assert_eq!(map1_copy, map1); + let mut map2_copy = map1.clone(); + map2_copy.clone_from(&map2); // large cloned from small + assert_eq!(map2_copy, map2); + map2.insert(100 * j + 1, 2 * j + 1); + } + map2.clone_from(&map1); // same length + map2.check(); + assert_eq!(map2, map1); + map1.insert(i, 10 * i); + map1.check(); + } +} + +#[allow(dead_code)] +fn assert_covariance() { + fn map_key<'new>(v: BTreeMap<&'static str, ()>) -> BTreeMap<&'new str, ()> { + v + } + fn map_val<'new>(v: BTreeMap<(), &'static str>) -> BTreeMap<(), &'new str> { + v + } + + fn iter_key<'a, 'new>(v: Iter<'a, &'static str, ()>) -> Iter<'a, &'new str, ()> { + v + } + fn iter_val<'a, 'new>(v: Iter<'a, (), &'static str>) -> Iter<'a, (), &'new str> { + v + } + + fn into_iter_key<'new>(v: IntoIter<&'static str, ()>) -> IntoIter<&'new str, ()> { + v + } + fn into_iter_val<'new>(v: IntoIter<(), &'static str>) -> IntoIter<(), &'new str> { + v + } + + fn into_keys_key<'new>(v: IntoKeys<&'static str, ()>) -> IntoKeys<&'new str, ()> { + v + } + fn into_keys_val<'new>(v: IntoKeys<(), &'static str>) -> IntoKeys<(), &'new str> { + v + } + + fn into_values_key<'new>(v: IntoValues<&'static str, ()>) -> IntoValues<&'new str, ()> { + v + } + fn into_values_val<'new>(v: IntoValues<(), &'static str>) -> IntoValues<(), &'new str> { + v + } + + fn range_key<'a, 'new>(v: Range<'a, &'static str, ()>) -> Range<'a, &'new str, ()> { + v + } + fn range_val<'a, 'new>(v: Range<'a, (), &'static str>) -> Range<'a, (), &'new str> { + v + } + + fn keys_key<'a, 'new>(v: Keys<'a, &'static str, ()>) -> Keys<'a, &'new str, ()> { + v + } + fn keys_val<'a, 'new>(v: Keys<'a, (), &'static str>) -> Keys<'a, (), &'new str> { + v + } + + fn values_key<'a, 'new>(v: Values<'a, &'static str, ()>) -> Values<'a, &'new str, ()> { + v + } + fn values_val<'a, 'new>(v: Values<'a, (), &'static str>) -> Values<'a, (), &'new str> { + v + } +} + +#[allow(dead_code)] +fn assert_sync() { + fn map(v: &BTreeMap) -> impl Sync + '_ { + v + } + + fn into_iter(v: BTreeMap) -> impl Sync { + v.into_iter() + } + + fn into_keys(v: BTreeMap) -> impl Sync { + v.into_keys() + } + + fn into_values(v: BTreeMap) -> impl Sync { + v.into_values() + } + + fn drain_filter(v: &mut BTreeMap) -> impl Sync + '_ { + v.drain_filter(|_, _| false) + } + + fn iter(v: &BTreeMap) -> impl Sync + '_ { + v.iter() + } + + fn iter_mut(v: &mut BTreeMap) -> impl Sync + '_ { + v.iter_mut() + } + + fn keys(v: &BTreeMap) -> impl Sync + '_ { + v.keys() + } + + fn values(v: &BTreeMap) -> impl Sync + '_ { + v.values() + } + + fn values_mut(v: &mut BTreeMap) -> impl Sync + '_ { + v.values_mut() + } + + fn range(v: &BTreeMap) -> impl Sync + '_ { + v.range(..) + } + + fn range_mut(v: &mut BTreeMap) -> impl Sync + '_ { + v.range_mut(..) + } + + fn entry(v: &mut BTreeMap) -> impl Sync + '_ { + v.entry(Default::default()) + } + + fn occupied_entry(v: &mut BTreeMap) -> impl Sync + '_ { + match v.entry(Default::default()) { + Occupied(entry) => entry, + _ => unreachable!(), + } + } + + fn vacant_entry(v: &mut BTreeMap) -> impl Sync + '_ { + match v.entry(Default::default()) { + Vacant(entry) => entry, + _ => unreachable!(), + } + } +} + +#[allow(dead_code)] +fn assert_send() { + fn map(v: BTreeMap) -> impl Send { + v + } + + fn into_iter(v: BTreeMap) -> impl Send { + v.into_iter() + } + + fn into_keys(v: BTreeMap) -> impl Send { + v.into_keys() + } + + fn into_values(v: BTreeMap) -> impl Send { + v.into_values() + } + + fn drain_filter(v: &mut BTreeMap) -> impl Send + '_ { + v.drain_filter(|_, _| false) + } + + fn iter(v: &BTreeMap) -> impl Send + '_ { + v.iter() + } + + fn iter_mut(v: &mut BTreeMap) -> impl Send + '_ { + v.iter_mut() + } + + fn keys(v: &BTreeMap) -> impl Send + '_ { + v.keys() + } + + fn values(v: &BTreeMap) -> impl Send + '_ { + v.values() + } + + fn values_mut(v: &mut BTreeMap) -> impl Send + '_ { + v.values_mut() + } + + fn range(v: &BTreeMap) -> impl Send + '_ { + v.range(..) + } + + fn range_mut(v: &mut BTreeMap) -> impl Send + '_ { + v.range_mut(..) + } + + fn entry(v: &mut BTreeMap) -> impl Send + '_ { + v.entry(Default::default()) + } + + fn occupied_entry(v: &mut BTreeMap) -> impl Send + '_ { + match v.entry(Default::default()) { + Occupied(entry) => entry, + _ => unreachable!(), + } + } + + fn vacant_entry(v: &mut BTreeMap) -> impl Send + '_ { + match v.entry(Default::default()) { + Vacant(entry) => entry, + _ => unreachable!(), + } + } +} + +#[test] +fn test_ord_absence() { + fn map(mut map: BTreeMap) { + let _ = map.is_empty(); + let _ = map.len(); + map.clear(); + let _ = map.iter(); + let _ = map.iter_mut(); + let _ = map.keys(); + let _ = map.values(); + let _ = map.values_mut(); + if true { + let _ = map.into_values(); + } else if true { + let _ = map.into_iter(); + } else { + let _ = map.into_keys(); + } + } + + fn map_debug(mut map: BTreeMap) { + format!("{map:?}"); + format!("{:?}", map.iter()); + format!("{:?}", map.iter_mut()); + format!("{:?}", map.keys()); + format!("{:?}", map.values()); + format!("{:?}", map.values_mut()); + if true { + format!("{:?}", map.into_iter()); + } else if true { + format!("{:?}", map.into_keys()); + } else { + format!("{:?}", map.into_values()); + } + } + + fn map_clone(mut map: BTreeMap) { + map.clone_from(&map.clone()); + } + + #[derive(Debug, Clone)] + struct NonOrd; + map(BTreeMap::::new()); + map_debug(BTreeMap::::new()); + map_clone(BTreeMap::::default()); +} + +#[test] +fn test_occupied_entry_key() { + let mut a = BTreeMap::new(); + let key = "hello there"; + let value = "value goes here"; + assert_eq!(a.height(), None); + a.insert(key, value); + assert_eq!(a.len(), 1); + assert_eq!(a[key], value); + + match a.entry(key) { + Vacant(_) => panic!(), + Occupied(e) => assert_eq!(key, *e.key()), + } + assert_eq!(a.len(), 1); + assert_eq!(a[key], value); + a.check(); +} + +#[test] +fn test_vacant_entry_key() { + let mut a = BTreeMap::new(); + let key = "hello there"; + let value = "value goes here"; + + assert_eq!(a.height(), None); + match a.entry(key) { + Occupied(_) => unreachable!(), + Vacant(e) => { + assert_eq!(key, *e.key()); + e.insert(value); + } + } + assert_eq!(a.len(), 1); + assert_eq!(a[key], value); + a.check(); +} + +#[test] +fn test_vacant_entry_no_insert() { + let mut a = BTreeMap::<&str, ()>::new(); + let key = "hello there"; + + // Non-allocated + assert_eq!(a.height(), None); + match a.entry(key) { + Occupied(_) => unreachable!(), + Vacant(e) => assert_eq!(key, *e.key()), + } + // Ensures the tree has no root. + assert_eq!(a.height(), None); + a.check(); + + // Allocated but still empty + a.insert(key, ()); + a.remove(&key); + assert_eq!(a.height(), Some(0)); + assert!(a.is_empty()); + match a.entry(key) { + Occupied(_) => unreachable!(), + Vacant(e) => assert_eq!(key, *e.key()), + } + // Ensures the allocated root is not changed. + assert_eq!(a.height(), Some(0)); + assert!(a.is_empty()); + a.check(); +} + +#[test] +fn test_first_last_entry() { + let mut a = BTreeMap::new(); + assert!(a.first_entry().is_none()); + assert!(a.last_entry().is_none()); + a.insert(1, 42); + assert_eq!(a.first_entry().unwrap().key(), &1); + assert_eq!(a.last_entry().unwrap().key(), &1); + a.insert(2, 24); + assert_eq!(a.first_entry().unwrap().key(), &1); + assert_eq!(a.last_entry().unwrap().key(), &2); + a.insert(0, 6); + assert_eq!(a.first_entry().unwrap().key(), &0); + assert_eq!(a.last_entry().unwrap().key(), &2); + let (k1, v1) = a.first_entry().unwrap().remove_entry(); + assert_eq!(k1, 0); + assert_eq!(v1, 6); + let (k2, v2) = a.last_entry().unwrap().remove_entry(); + assert_eq!(k2, 2); + assert_eq!(v2, 24); + assert_eq!(a.first_entry().unwrap().key(), &1); + assert_eq!(a.last_entry().unwrap().key(), &1); + a.check(); +} + +#[test] +fn test_pop_first_last() { + let mut map = BTreeMap::new(); + assert_eq!(map.pop_first(), None); + assert_eq!(map.pop_last(), None); + + map.insert(1, 10); + map.insert(2, 20); + map.insert(3, 30); + map.insert(4, 40); + + assert_eq!(map.len(), 4); + + let (key, val) = map.pop_first().unwrap(); + assert_eq!(key, 1); + assert_eq!(val, 10); + assert_eq!(map.len(), 3); + + let (key, val) = map.pop_first().unwrap(); + assert_eq!(key, 2); + assert_eq!(val, 20); + assert_eq!(map.len(), 2); + let (key, val) = map.pop_last().unwrap(); + assert_eq!(key, 4); + assert_eq!(val, 40); + assert_eq!(map.len(), 1); + + map.insert(5, 50); + map.insert(6, 60); + assert_eq!(map.len(), 3); + + let (key, val) = map.pop_first().unwrap(); + assert_eq!(key, 3); + assert_eq!(val, 30); + assert_eq!(map.len(), 2); + + let (key, val) = map.pop_last().unwrap(); + assert_eq!(key, 6); + assert_eq!(val, 60); + assert_eq!(map.len(), 1); + + let (key, val) = map.pop_last().unwrap(); + assert_eq!(key, 5); + assert_eq!(val, 50); + assert_eq!(map.len(), 0); + + assert_eq!(map.pop_first(), None); + assert_eq!(map.pop_last(), None); + + map.insert(7, 70); + map.insert(8, 80); + + let (key, val) = map.pop_last().unwrap(); + assert_eq!(key, 8); + assert_eq!(val, 80); + assert_eq!(map.len(), 1); + + let (key, val) = map.pop_last().unwrap(); + assert_eq!(key, 7); + assert_eq!(val, 70); + assert_eq!(map.len(), 0); + + assert_eq!(map.pop_first(), None); + assert_eq!(map.pop_last(), None); +} + +#[test] +fn test_get_key_value() { + let mut map = BTreeMap::new(); + + assert!(map.is_empty()); + assert_eq!(map.get_key_value(&1), None); + assert_eq!(map.get_key_value(&2), None); + + map.insert(1, 10); + map.insert(2, 20); + map.insert(3, 30); + + assert_eq!(map.len(), 3); + assert_eq!(map.get_key_value(&1), Some((&1, &10))); + assert_eq!(map.get_key_value(&3), Some((&3, &30))); + assert_eq!(map.get_key_value(&4), None); + + map.remove(&3); + + assert_eq!(map.len(), 2); + assert_eq!(map.get_key_value(&3), None); + assert_eq!(map.get_key_value(&2), Some((&2, &20))); +} + +#[test] +fn test_insert_into_full_height_0() { + let size = node::CAPACITY; + for pos in 0..=size { + let mut map = BTreeMap::from_iter((0..size).map(|i| (i * 2 + 1, ()))); + assert!(map.insert(pos * 2, ()).is_none()); + map.check(); + } +} + +#[test] +fn test_insert_into_full_height_1() { + let size = node::CAPACITY + 1 + node::CAPACITY; + for pos in 0..=size { + let mut map = BTreeMap::from_iter((0..size).map(|i| (i * 2 + 1, ()))); + map.compact(); + let root_node = map.root.as_ref().unwrap().reborrow(); + assert_eq!(root_node.len(), 1); + assert_eq!(root_node.first_leaf_edge().into_node().len(), node::CAPACITY); + assert_eq!(root_node.last_leaf_edge().into_node().len(), node::CAPACITY); + + assert!(map.insert(pos * 2, ()).is_none()); + map.check(); + } +} + +#[test] +fn test_try_insert() { + let mut map = BTreeMap::new(); + + assert!(map.is_empty()); + + assert_eq!(map.try_insert(1, 10).unwrap(), &10); + assert_eq!(map.try_insert(2, 20).unwrap(), &20); + + let err = map.try_insert(2, 200).unwrap_err(); + assert_eq!(err.entry.key(), &2); + assert_eq!(err.entry.get(), &20); + assert_eq!(err.value, 200); +} + +macro_rules! create_append_test { + ($name:ident, $len:expr) => { + #[test] + fn $name() { + let mut a = BTreeMap::new(); + for i in 0..8 { + a.insert(i, i); + } + + let mut b = BTreeMap::new(); + for i in 5..$len { + b.insert(i, 2 * i); + } + + a.append(&mut b); + + assert_eq!(a.len(), $len); + assert_eq!(b.len(), 0); + + for i in 0..$len { + if i < 5 { + assert_eq!(a[&i], i); + } else { + assert_eq!(a[&i], 2 * i); + } + } + + a.check(); + assert_eq!(a.remove(&($len - 1)), Some(2 * ($len - 1))); + assert_eq!(a.insert($len - 1, 20), None); + a.check(); + } + }; +} + +// These are mostly for testing the algorithm that "fixes" the right edge after insertion. +// Single node. +create_append_test!(test_append_9, 9); +// Two leafs that don't need fixing. +create_append_test!(test_append_17, 17); +// Two leafs where the second one ends up underfull and needs stealing at the end. +create_append_test!(test_append_14, 14); +// Two leafs where the second one ends up empty because the insertion finished at the root. +create_append_test!(test_append_12, 12); +// Three levels; insertion finished at the root. +create_append_test!(test_append_144, 144); +// Three levels; insertion finished at leaf while there is an empty node on the second level. +create_append_test!(test_append_145, 145); +// Tests for several randomly chosen sizes. +create_append_test!(test_append_170, 170); +create_append_test!(test_append_181, 181); +#[cfg(not(miri))] // Miri is too slow +create_append_test!(test_append_239, 239); +#[cfg(not(miri))] // Miri is too slow +create_append_test!(test_append_1700, 1700); + +#[test] +fn test_append_drop_leak() { + let a = CrashTestDummy::new(0); + let b = CrashTestDummy::new(1); + let c = CrashTestDummy::new(2); + let mut left = BTreeMap::new(); + let mut right = BTreeMap::new(); + left.insert(a.spawn(Panic::Never), ()); + left.insert(b.spawn(Panic::InDrop), ()); // first duplicate key, dropped during append + left.insert(c.spawn(Panic::Never), ()); + right.insert(b.spawn(Panic::Never), ()); + right.insert(c.spawn(Panic::Never), ()); + + catch_unwind(move || left.append(&mut right)).unwrap_err(); + assert_eq!(a.dropped(), 1); + assert_eq!(b.dropped(), 1); // should be 2 were it not for Rust issue #47949 + assert_eq!(c.dropped(), 2); +} + +#[test] +fn test_append_ord_chaos() { + let mut map1 = BTreeMap::new(); + map1.insert(Cyclic3::A, ()); + map1.insert(Cyclic3::B, ()); + let mut map2 = BTreeMap::new(); + map2.insert(Cyclic3::A, ()); + map2.insert(Cyclic3::B, ()); + map2.insert(Cyclic3::C, ()); // lands first, before A + map2.insert(Cyclic3::B, ()); // lands first, before C + map1.check(); + map2.check(); // keys are not unique but still strictly ascending + assert_eq!(map1.len(), 2); + assert_eq!(map2.len(), 4); + map1.append(&mut map2); + assert_eq!(map1.len(), 5); + assert_eq!(map2.len(), 0); + map1.check(); + map2.check(); +} + +fn rand_data(len: usize) -> Vec<(u32, u32)> { + let mut rng = DeterministicRng::new(); + Vec::from_iter((0..len).map(|_| (rng.next(), rng.next()))) +} + +#[test] +fn test_split_off_empty_right() { + let mut data = rand_data(173); + + let mut map = BTreeMap::from_iter(data.clone()); + let right = map.split_off(&(data.iter().max().unwrap().0 + 1)); + map.check(); + right.check(); + + data.sort(); + assert!(map.into_iter().eq(data)); + assert!(right.into_iter().eq(None)); +} + +#[test] +fn test_split_off_empty_left() { + let mut data = rand_data(314); + + let mut map = BTreeMap::from_iter(data.clone()); + let right = map.split_off(&data.iter().min().unwrap().0); + map.check(); + right.check(); + + data.sort(); + assert!(map.into_iter().eq(None)); + assert!(right.into_iter().eq(data)); +} + +// In a tree with 3 levels, if all but a part of the first leaf node is split off, +// make sure fix_top eliminates both top levels. +#[test] +fn test_split_off_tiny_left_height_2() { + let pairs = (0..MIN_INSERTS_HEIGHT_2).map(|i| (i, i)); + let mut left = BTreeMap::from_iter(pairs.clone()); + let right = left.split_off(&1); + left.check(); + right.check(); + assert_eq!(left.len(), 1); + assert_eq!(right.len(), MIN_INSERTS_HEIGHT_2 - 1); + assert_eq!(*left.first_key_value().unwrap().0, 0); + assert_eq!(*right.first_key_value().unwrap().0, 1); +} + +// In a tree with 3 levels, if only part of the last leaf node is split off, +// make sure fix_top eliminates both top levels. +#[test] +fn test_split_off_tiny_right_height_2() { + let pairs = (0..MIN_INSERTS_HEIGHT_2).map(|i| (i, i)); + let last = MIN_INSERTS_HEIGHT_2 - 1; + let mut left = BTreeMap::from_iter(pairs.clone()); + assert_eq!(*left.last_key_value().unwrap().0, last); + let right = left.split_off(&last); + left.check(); + right.check(); + assert_eq!(left.len(), MIN_INSERTS_HEIGHT_2 - 1); + assert_eq!(right.len(), 1); + assert_eq!(*left.last_key_value().unwrap().0, last - 1); + assert_eq!(*right.last_key_value().unwrap().0, last); +} + +#[test] +fn test_split_off_halfway() { + let mut rng = DeterministicRng::new(); + for &len in &[node::CAPACITY, 25, 50, 75, 100] { + let mut data = Vec::from_iter((0..len).map(|_| (rng.next(), ()))); + // Insertion in non-ascending order creates some variation in node length. + let mut map = BTreeMap::from_iter(data.iter().copied()); + data.sort(); + let small_keys = data.iter().take(len / 2).map(|kv| kv.0); + let large_keys = data.iter().skip(len / 2).map(|kv| kv.0); + let split_key = large_keys.clone().next().unwrap(); + let right = map.split_off(&split_key); + map.check(); + right.check(); + assert!(map.keys().copied().eq(small_keys)); + assert!(right.keys().copied().eq(large_keys)); + } +} + +#[test] +fn test_split_off_large_random_sorted() { + // Miri is too slow + let mut data = if cfg!(miri) { rand_data(529) } else { rand_data(1529) }; + // special case with maximum height. + data.sort(); + + let mut map = BTreeMap::from_iter(data.clone()); + let key = data[data.len() / 2].0; + let right = map.split_off(&key); + map.check(); + right.check(); + + assert!(map.into_iter().eq(data.clone().into_iter().filter(|x| x.0 < key))); + assert!(right.into_iter().eq(data.into_iter().filter(|x| x.0 >= key))); +} + +#[test] +fn test_into_iter_drop_leak_height_0() { + let a = CrashTestDummy::new(0); + let b = CrashTestDummy::new(1); + let c = CrashTestDummy::new(2); + let d = CrashTestDummy::new(3); + let e = CrashTestDummy::new(4); + let mut map = BTreeMap::new(); + map.insert("a", a.spawn(Panic::Never)); + map.insert("b", b.spawn(Panic::Never)); + map.insert("c", c.spawn(Panic::Never)); + map.insert("d", d.spawn(Panic::InDrop)); + map.insert("e", e.spawn(Panic::Never)); + + catch_unwind(move || drop(map.into_iter())).unwrap_err(); + + assert_eq!(a.dropped(), 1); + assert_eq!(b.dropped(), 1); + assert_eq!(c.dropped(), 1); + assert_eq!(d.dropped(), 1); + assert_eq!(e.dropped(), 1); +} + +#[test] +fn test_into_iter_drop_leak_height_1() { + let size = MIN_INSERTS_HEIGHT_1; + for panic_point in vec![0, 1, size - 2, size - 1] { + let dummies = Vec::from_iter((0..size).map(|i| CrashTestDummy::new(i))); + let map = BTreeMap::from_iter((0..size).map(|i| { + let panic = if i == panic_point { Panic::InDrop } else { Panic::Never }; + (dummies[i].spawn(Panic::Never), dummies[i].spawn(panic)) + })); + catch_unwind(move || drop(map.into_iter())).unwrap_err(); + for i in 0..size { + assert_eq!(dummies[i].dropped(), 2); + } + } +} + +#[test] +fn test_into_keys() { + let map = BTreeMap::from([(1, 'a'), (2, 'b'), (3, 'c')]); + let keys = Vec::from_iter(map.into_keys()); + + assert_eq!(keys.len(), 3); + assert!(keys.contains(&1)); + assert!(keys.contains(&2)); + assert!(keys.contains(&3)); +} + +#[test] +fn test_into_values() { + let map = BTreeMap::from([(1, 'a'), (2, 'b'), (3, 'c')]); + let values = Vec::from_iter(map.into_values()); + + assert_eq!(values.len(), 3); + assert!(values.contains(&'a')); + assert!(values.contains(&'b')); + assert!(values.contains(&'c')); +} + +#[test] +fn test_insert_remove_intertwined() { + let loops = if cfg!(miri) { 100 } else { 1_000_000 }; + let mut map = BTreeMap::new(); + let mut i = 1; + let offset = 165; // somewhat arbitrarily chosen to cover some code paths + for _ in 0..loops { + i = (i + offset) & 0xFF; + map.insert(i, i); + map.remove(&(0xFF - i)); + } + map.check(); +} + +#[test] +fn test_insert_remove_intertwined_ord_chaos() { + let loops = if cfg!(miri) { 100 } else { 1_000_000 }; + let gov = Governor::new(); + let mut map = BTreeMap::new(); + let mut i = 1; + let offset = 165; // more arbitrarily copied from above + for _ in 0..loops { + i = (i + offset) & 0xFF; + map.insert(Governed(i, &gov), ()); + map.remove(&Governed(0xFF - i, &gov)); + gov.flip(); + } + map.check_invariants(); +} + +#[test] +fn from_array() { + let map = BTreeMap::from([(1, 2), (3, 4)]); + let unordered_duplicates = BTreeMap::from([(3, 4), (1, 2), (1, 2)]); + assert_eq!(map, unordered_duplicates); +} diff --git a/crux-mir/lib/alloc/src/collections/btree/mem.rs b/crux-mir/lib/alloc/src/collections/btree/mem.rs new file mode 100644 index 000000000..e1363d1ae --- /dev/null +++ b/crux-mir/lib/alloc/src/collections/btree/mem.rs @@ -0,0 +1,35 @@ +use core::intrinsics; +use core::mem; +use core::ptr; + +/// This replaces the value behind the `v` unique reference by calling the +/// relevant function. +/// +/// If a panic occurs in the `change` closure, the entire process will be aborted. +#[allow(dead_code)] // keep as illustration and for future use +#[inline] +pub fn take_mut(v: &mut T, change: impl FnOnce(T) -> T) { + replace(v, |value| (change(value), ())) +} + +/// This replaces the value behind the `v` unique reference by calling the +/// relevant function, and returns a result obtained along the way. +/// +/// If a panic occurs in the `change` closure, the entire process will be aborted. +#[inline] +pub fn replace(v: &mut T, change: impl FnOnce(T) -> (T, R)) -> R { + struct PanicGuard; + impl Drop for PanicGuard { + fn drop(&mut self) { + intrinsics::abort() + } + } + let guard = PanicGuard; + let value = unsafe { ptr::read(v) }; + let (new_value, ret) = change(value); + unsafe { + ptr::write(v, new_value); + } + mem::forget(guard); + ret +} diff --git a/crux-mir/lib/alloc/src/collections/btree/merge_iter.rs b/crux-mir/lib/alloc/src/collections/btree/merge_iter.rs new file mode 100644 index 000000000..7f23d93b9 --- /dev/null +++ b/crux-mir/lib/alloc/src/collections/btree/merge_iter.rs @@ -0,0 +1,98 @@ +use core::cmp::Ordering; +use core::fmt::{self, Debug}; +use core::iter::FusedIterator; + +/// Core of an iterator that merges the output of two strictly ascending iterators, +/// for instance a union or a symmetric difference. +pub struct MergeIterInner { + a: I, + b: I, + peeked: Option>, +} + +/// Benchmarks faster than wrapping both iterators in a Peekable, +/// probably because we can afford to impose a FusedIterator bound. +#[derive(Clone, Debug)] +enum Peeked { + A(I::Item), + B(I::Item), +} + +impl Clone for MergeIterInner +where + I: Clone, + I::Item: Clone, +{ + fn clone(&self) -> Self { + Self { a: self.a.clone(), b: self.b.clone(), peeked: self.peeked.clone() } + } +} + +impl Debug for MergeIterInner +where + I: Debug, + I::Item: Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("MergeIterInner").field(&self.a).field(&self.b).field(&self.peeked).finish() + } +} + +impl MergeIterInner { + /// Creates a new core for an iterator merging a pair of sources. + pub fn new(a: I, b: I) -> Self { + MergeIterInner { a, b, peeked: None } + } + + /// Returns the next pair of items stemming from the pair of sources + /// being merged. If both returned options contain a value, that value + /// is equal and occurs in both sources. If one of the returned options + /// contains a value, that value doesn't occur in the other source (or + /// the sources are not strictly ascending). If neither returned option + /// contains a value, iteration has finished and subsequent calls will + /// return the same empty pair. + pub fn nexts Ordering>( + &mut self, + cmp: Cmp, + ) -> (Option, Option) + where + I: FusedIterator, + { + let mut a_next; + let mut b_next; + match self.peeked.take() { + Some(Peeked::A(next)) => { + a_next = Some(next); + b_next = self.b.next(); + } + Some(Peeked::B(next)) => { + b_next = Some(next); + a_next = self.a.next(); + } + None => { + a_next = self.a.next(); + b_next = self.b.next(); + } + } + if let (Some(ref a1), Some(ref b1)) = (&a_next, &b_next) { + match cmp(a1, b1) { + Ordering::Less => self.peeked = b_next.take().map(Peeked::B), + Ordering::Greater => self.peeked = a_next.take().map(Peeked::A), + Ordering::Equal => (), + } + } + (a_next, b_next) + } + + /// Returns a pair of upper bounds for the `size_hint` of the final iterator. + pub fn lens(&self) -> (usize, usize) + where + I: ExactSizeIterator, + { + match self.peeked { + Some(Peeked::A(_)) => (1 + self.a.len(), self.b.len()), + Some(Peeked::B(_)) => (self.a.len(), 1 + self.b.len()), + _ => (self.a.len(), self.b.len()), + } + } +} diff --git a/crux-mir/lib/alloc/src/collections/btree/mod.rs b/crux-mir/lib/alloc/src/collections/btree/mod.rs index fb5825ee2..7552f2fc0 100644 --- a/crux-mir/lib/alloc/src/collections/btree/mod.rs +++ b/crux-mir/lib/alloc/src/collections/btree/mod.rs @@ -1,8 +1,17 @@ +mod append; +mod borrow; +mod dedup_sorted_iter; +mod fix; pub mod map; +mod mem; +mod merge_iter; mod navigate; mod node; +mod remove; mod search; pub mod set; +mod set_val; +mod split; #[doc(hidden)] trait Recover { @@ -12,14 +21,3 @@ trait Recover { fn take(&mut self, key: &Q) -> Option; fn replace(&mut self, key: Self::Key) -> Option; } - -#[inline(always)] -pub unsafe fn unwrap_unchecked(val: Option) -> T { - val.unwrap_or_else(|| { - if cfg!(debug_assertions) { - panic!("'unchecked' unwrap on None in BTreeMap"); - } else { - core::intrinsics::unreachable(); - } - }) -} diff --git a/crux-mir/lib/alloc/src/collections/btree/navigate.rs b/crux-mir/lib/alloc/src/collections/btree/navigate.rs index 5e8dcf247..1e33c1e64 100644 --- a/crux-mir/lib/alloc/src/collections/btree/navigate.rs +++ b/crux-mir/lib/alloc/src/collections/btree/navigate.rs @@ -1,9 +1,367 @@ +use core::borrow::Borrow; +use core::hint; +use core::ops::RangeBounds; use core::ptr; use super::node::{marker, ForceResult::*, Handle, NodeRef}; -use super::unwrap_unchecked; -impl Handle, marker::Edge> { +use crate::alloc::Allocator; +// `front` and `back` are always both `None` or both `Some`. +pub struct LeafRange { + front: Option, marker::Edge>>, + back: Option, marker::Edge>>, +} + +impl<'a, K: 'a, V: 'a> Clone for LeafRange, K, V> { + fn clone(&self) -> Self { + LeafRange { front: self.front.clone(), back: self.back.clone() } + } +} + +impl LeafRange { + pub fn none() -> Self { + LeafRange { front: None, back: None } + } + + fn is_empty(&self) -> bool { + self.front == self.back + } + + /// Temporarily takes out another, immutable equivalent of the same range. + pub fn reborrow(&self) -> LeafRange, K, V> { + LeafRange { + front: self.front.as_ref().map(|f| f.reborrow()), + back: self.back.as_ref().map(|b| b.reborrow()), + } + } +} + +impl<'a, K, V> LeafRange, K, V> { + #[inline] + pub fn next_checked(&mut self) -> Option<(&'a K, &'a V)> { + self.perform_next_checked(|kv| kv.into_kv()) + } + + #[inline] + pub fn next_back_checked(&mut self) -> Option<(&'a K, &'a V)> { + self.perform_next_back_checked(|kv| kv.into_kv()) + } +} + +impl<'a, K, V> LeafRange, K, V> { + #[inline] + pub fn next_checked(&mut self) -> Option<(&'a K, &'a mut V)> { + self.perform_next_checked(|kv| unsafe { ptr::read(kv) }.into_kv_valmut()) + } + + #[inline] + pub fn next_back_checked(&mut self) -> Option<(&'a K, &'a mut V)> { + self.perform_next_back_checked(|kv| unsafe { ptr::read(kv) }.into_kv_valmut()) + } +} + +impl LeafRange { + /// If possible, extract some result from the following KV and move to the edge beyond it. + fn perform_next_checked(&mut self, f: F) -> Option + where + F: Fn(&Handle, marker::KV>) -> R, + { + if self.is_empty() { + None + } else { + super::mem::replace(self.front.as_mut().unwrap(), |front| { + let kv = front.next_kv().ok().unwrap(); + let result = f(&kv); + (kv.next_leaf_edge(), Some(result)) + }) + } + } + + /// If possible, extract some result from the preceding KV and move to the edge beyond it. + fn perform_next_back_checked(&mut self, f: F) -> Option + where + F: Fn(&Handle, marker::KV>) -> R, + { + if self.is_empty() { + None + } else { + super::mem::replace(self.back.as_mut().unwrap(), |back| { + let kv = back.next_back_kv().ok().unwrap(); + let result = f(&kv); + (kv.next_back_leaf_edge(), Some(result)) + }) + } + } +} + +enum LazyLeafHandle { + Root(NodeRef), // not yet descended + Edge(Handle, marker::Edge>), +} + +impl<'a, K: 'a, V: 'a> Clone for LazyLeafHandle, K, V> { + fn clone(&self) -> Self { + match self { + LazyLeafHandle::Root(root) => LazyLeafHandle::Root(*root), + LazyLeafHandle::Edge(edge) => LazyLeafHandle::Edge(*edge), + } + } +} + +impl LazyLeafHandle { + fn reborrow(&self) -> LazyLeafHandle, K, V> { + match self { + LazyLeafHandle::Root(root) => LazyLeafHandle::Root(root.reborrow()), + LazyLeafHandle::Edge(edge) => LazyLeafHandle::Edge(edge.reborrow()), + } + } +} + +// `front` and `back` are always both `None` or both `Some`. +pub struct LazyLeafRange { + front: Option>, + back: Option>, +} + +impl<'a, K: 'a, V: 'a> Clone for LazyLeafRange, K, V> { + fn clone(&self) -> Self { + LazyLeafRange { front: self.front.clone(), back: self.back.clone() } + } +} + +impl LazyLeafRange { + pub fn none() -> Self { + LazyLeafRange { front: None, back: None } + } + + /// Temporarily takes out another, immutable equivalent of the same range. + pub fn reborrow(&self) -> LazyLeafRange, K, V> { + LazyLeafRange { + front: self.front.as_ref().map(|f| f.reborrow()), + back: self.back.as_ref().map(|b| b.reborrow()), + } + } +} + +impl<'a, K, V> LazyLeafRange, K, V> { + #[inline] + pub unsafe fn next_unchecked(&mut self) -> (&'a K, &'a V) { + unsafe { self.init_front().unwrap().next_unchecked() } + } + + #[inline] + pub unsafe fn next_back_unchecked(&mut self) -> (&'a K, &'a V) { + unsafe { self.init_back().unwrap().next_back_unchecked() } + } +} + +impl<'a, K, V> LazyLeafRange, K, V> { + #[inline] + pub unsafe fn next_unchecked(&mut self) -> (&'a K, &'a mut V) { + unsafe { self.init_front().unwrap().next_unchecked() } + } + + #[inline] + pub unsafe fn next_back_unchecked(&mut self) -> (&'a K, &'a mut V) { + unsafe { self.init_back().unwrap().next_back_unchecked() } + } +} + +impl LazyLeafRange { + fn take_front( + &mut self, + ) -> Option, marker::Edge>> { + match self.front.take()? { + LazyLeafHandle::Root(root) => Some(root.first_leaf_edge()), + LazyLeafHandle::Edge(edge) => Some(edge), + } + } + + #[inline] + pub unsafe fn deallocating_next_unchecked( + &mut self, + alloc: A, + ) -> Handle, marker::KV> { + debug_assert!(self.front.is_some()); + let front = self.init_front().unwrap(); + unsafe { front.deallocating_next_unchecked(alloc) } + } + + #[inline] + pub unsafe fn deallocating_next_back_unchecked( + &mut self, + alloc: A, + ) -> Handle, marker::KV> { + debug_assert!(self.back.is_some()); + let back = self.init_back().unwrap(); + unsafe { back.deallocating_next_back_unchecked(alloc) } + } + + #[inline] + pub fn deallocating_end(&mut self, alloc: A) { + if let Some(front) = self.take_front() { + front.deallocating_end(alloc) + } + } +} + +impl LazyLeafRange { + fn init_front( + &mut self, + ) -> Option<&mut Handle, marker::Edge>> { + if let Some(LazyLeafHandle::Root(root)) = &self.front { + self.front = Some(LazyLeafHandle::Edge(unsafe { ptr::read(root) }.first_leaf_edge())); + } + match &mut self.front { + None => None, + Some(LazyLeafHandle::Edge(edge)) => Some(edge), + // SAFETY: the code above would have replaced it. + Some(LazyLeafHandle::Root(_)) => unsafe { hint::unreachable_unchecked() }, + } + } + + fn init_back( + &mut self, + ) -> Option<&mut Handle, marker::Edge>> { + if let Some(LazyLeafHandle::Root(root)) = &self.back { + self.back = Some(LazyLeafHandle::Edge(unsafe { ptr::read(root) }.last_leaf_edge())); + } + match &mut self.back { + None => None, + Some(LazyLeafHandle::Edge(edge)) => Some(edge), + // SAFETY: the code above would have replaced it. + Some(LazyLeafHandle::Root(_)) => unsafe { hint::unreachable_unchecked() }, + } + } +} + +impl NodeRef { + /// Finds the distinct leaf edges delimiting a specified range in a tree. + /// + /// If such distinct edges exist, returns them in ascending order, meaning + /// that a non-zero number of calls to `next_unchecked` on the `front` of + /// the result and/or calls to `next_back_unchecked` on the `back` of the + /// result will eventually reach the same edge. + /// + /// If there are no such edges, i.e., if the tree contains no key within + /// the range, returns an empty `front` and `back`. + /// + /// # Safety + /// Unless `BorrowType` is `Immut`, do not use the handles to visit the same + /// KV twice. + unsafe fn find_leaf_edges_spanning_range( + self, + range: R, + ) -> LeafRange + where + Q: Ord, + K: Borrow, + R: RangeBounds, + { + match self.search_tree_for_bifurcation(&range) { + Err(_) => LeafRange::none(), + Ok(( + node, + lower_edge_idx, + upper_edge_idx, + mut lower_child_bound, + mut upper_child_bound, + )) => { + let mut lower_edge = unsafe { Handle::new_edge(ptr::read(&node), lower_edge_idx) }; + let mut upper_edge = unsafe { Handle::new_edge(node, upper_edge_idx) }; + loop { + match (lower_edge.force(), upper_edge.force()) { + (Leaf(f), Leaf(b)) => return LeafRange { front: Some(f), back: Some(b) }, + (Internal(f), Internal(b)) => { + (lower_edge, lower_child_bound) = + f.descend().find_lower_bound_edge(lower_child_bound); + (upper_edge, upper_child_bound) = + b.descend().find_upper_bound_edge(upper_child_bound); + } + _ => unreachable!("BTreeMap has different depths"), + } + } + } + } + } +} + +fn full_range( + root1: NodeRef, + root2: NodeRef, +) -> LazyLeafRange { + LazyLeafRange { + front: Some(LazyLeafHandle::Root(root1)), + back: Some(LazyLeafHandle::Root(root2)), + } +} + +impl<'a, K: 'a, V: 'a> NodeRef, K, V, marker::LeafOrInternal> { + /// Finds the pair of leaf edges delimiting a specific range in a tree. + /// + /// The result is meaningful only if the tree is ordered by key, like the tree + /// in a `BTreeMap` is. + pub fn range_search(self, range: R) -> LeafRange, K, V> + where + Q: ?Sized + Ord, + K: Borrow, + R: RangeBounds, + { + // SAFETY: our borrow type is immutable. + unsafe { self.find_leaf_edges_spanning_range(range) } + } + + /// Finds the pair of leaf edges delimiting an entire tree. + pub fn full_range(self) -> LazyLeafRange, K, V> { + full_range(self, self) + } +} + +impl<'a, K: 'a, V: 'a> NodeRef, K, V, marker::LeafOrInternal> { + /// Splits a unique reference into a pair of leaf edges delimiting a specified range. + /// The result are non-unique references allowing (some) mutation, which must be used + /// carefully. + /// + /// The result is meaningful only if the tree is ordered by key, like the tree + /// in a `BTreeMap` is. + /// + /// # Safety + /// Do not use the duplicate handles to visit the same KV twice. + pub fn range_search(self, range: R) -> LeafRange, K, V> + where + Q: ?Sized + Ord, + K: Borrow, + R: RangeBounds, + { + unsafe { self.find_leaf_edges_spanning_range(range) } + } + + /// Splits a unique reference into a pair of leaf edges delimiting the full range of the tree. + /// The results are non-unique references allowing mutation (of values only), so must be used + /// with care. + pub fn full_range(self) -> LazyLeafRange, K, V> { + // We duplicate the root NodeRef here -- we will never visit the same KV + // twice, and never end up with overlapping value references. + let self2 = unsafe { ptr::read(&self) }; + full_range(self, self2) + } +} + +impl NodeRef { + /// Splits a unique reference into a pair of leaf edges delimiting the full range of the tree. + /// The results are non-unique references allowing massively destructive mutation, so must be + /// used with the utmost care. + pub fn full_range(self) -> LazyLeafRange { + // We duplicate the root NodeRef here -- we will never access it in a way + // that overlaps references obtained from the root. + let self2 = unsafe { ptr::read(&self) }; + full_range(self, self2) + } +} + +impl + Handle, marker::Edge> +{ /// Given a leaf edge handle, returns [`Result::Ok`] with a handle to the neighboring KV /// on the right side, which is either in the same leaf node or in an ancestor node. /// If the leaf edge is the last one in the tree, returns [`Result::Err`] with the root node. @@ -16,10 +374,10 @@ impl Handle, marker::E let mut edge = self.forget_node_type(); loop { edge = match edge.right_kv() { - Ok(internal_kv) => return Ok(internal_kv), + Ok(kv) => return Ok(kv), Err(last_edge) => match last_edge.into_node().ascend() { Ok(parent_edge) => parent_edge.forget_node_type(), - Err(root) => return Err(root.forget_type()), + Err(root) => return Err(root), }, } } @@ -28,7 +386,7 @@ impl Handle, marker::E /// Given a leaf edge handle, returns [`Result::Ok`] with a handle to the neighboring KV /// on the left side, which is either in the same leaf node or in an ancestor node. /// If the leaf edge is the first one in the tree, returns [`Result::Err`] with the root node. - pub fn next_back_kv( + fn next_back_kv( self, ) -> Result< Handle, marker::KV>, @@ -37,158 +395,223 @@ impl Handle, marker::E let mut edge = self.forget_node_type(); loop { edge = match edge.left_kv() { - Ok(internal_kv) => return Ok(internal_kv), + Ok(kv) => return Ok(kv), Err(last_edge) => match last_edge.into_node().ascend() { Ok(parent_edge) => parent_edge.forget_node_type(), - Err(root) => return Err(root.forget_type()), + Err(root) => return Err(root), }, } } } } -macro_rules! def_next_kv_uncheched_dealloc { - { unsafe fn $name:ident : $adjacent_kv:ident } => { - /// Given a leaf edge handle into an owned tree, returns a handle to the next KV, - /// while deallocating any node left behind. - /// Unsafe for two reasons: - /// - The caller must ensure that the leaf edge is not the last one in the tree. - /// - The node pointed at by the given handle, and its ancestors, may be deallocated, - /// while the reference to those nodes in the surviving ancestors is left dangling; - /// thus using the returned handle to navigate further is dangerous. - unsafe fn $name ( - leaf_edge: Handle, marker::Edge>, - ) -> Handle, marker::KV> { - let mut edge = leaf_edge.forget_node_type(); - loop { - edge = match edge.$adjacent_kv() { - Ok(internal_kv) => return internal_kv, - Err(last_edge) => { - let parent_edge = last_edge.into_node().deallocate_and_ascend(); - unwrap_unchecked(parent_edge).forget_node_type() +impl + Handle, marker::Edge> +{ + /// Given an internal edge handle, returns [`Result::Ok`] with a handle to the neighboring KV + /// on the right side, which is either in the same internal node or in an ancestor node. + /// If the internal edge is the last one in the tree, returns [`Result::Err`] with the root node. + fn next_kv( + self, + ) -> Result< + Handle, marker::KV>, + NodeRef, + > { + let mut edge = self; + loop { + edge = match edge.right_kv() { + Ok(internal_kv) => return Ok(internal_kv), + Err(last_edge) => match last_edge.into_node().ascend() { + Ok(parent_edge) => parent_edge, + Err(root) => return Err(root), + }, + } + } + } +} + +impl Handle, marker::Edge> { + /// Given a leaf edge handle into a dying tree, returns the next leaf edge + /// on the right side, and the key-value pair in between, if they exist. + /// + /// If the given edge is the last one in a leaf, this method deallocates + /// the leaf, as well as any ancestor nodes whose last edge was reached. + /// This implies that if no more key-value pair follows, the entire tree + /// will have been deallocated and there is nothing left to return. + /// + /// # Safety + /// - The given edge must not have been previously returned by counterpart + /// `deallocating_next_back`. + /// - The returned KV handle is only valid to access the key and value, + /// and only valid until the next call to a `deallocating_` method. + unsafe fn deallocating_next( + self, + alloc: A, + ) -> Option<(Self, Handle, marker::KV>)> + { + let mut edge = self.forget_node_type(); + loop { + edge = match edge.right_kv() { + Ok(kv) => return Some((unsafe { ptr::read(&kv) }.next_leaf_edge(), kv)), + Err(last_edge) => { + match unsafe { last_edge.into_node().deallocate_and_ascend(alloc.clone()) } { + Some(parent_edge) => parent_edge.forget_node_type(), + None => return None, } } } } - }; -} + } -def_next_kv_uncheched_dealloc! {unsafe fn next_kv_unchecked_dealloc: right_kv} -def_next_kv_uncheched_dealloc! {unsafe fn next_back_kv_unchecked_dealloc: left_kv} + /// Given a leaf edge handle into a dying tree, returns the next leaf edge + /// on the left side, and the key-value pair in between, if they exist. + /// + /// If the given edge is the first one in a leaf, this method deallocates + /// the leaf, as well as any ancestor nodes whose first edge was reached. + /// This implies that if no more key-value pair follows, the entire tree + /// will have been deallocated and there is nothing left to return. + /// + /// # Safety + /// - The given edge must not have been previously returned by counterpart + /// `deallocating_next`. + /// - The returned KV handle is only valid to access the key and value, + /// and only valid until the next call to a `deallocating_` method. + unsafe fn deallocating_next_back( + self, + alloc: A, + ) -> Option<(Self, Handle, marker::KV>)> + { + let mut edge = self.forget_node_type(); + loop { + edge = match edge.left_kv() { + Ok(kv) => return Some((unsafe { ptr::read(&kv) }.next_back_leaf_edge(), kv)), + Err(last_edge) => { + match unsafe { last_edge.into_node().deallocate_and_ascend(alloc.clone()) } { + Some(parent_edge) => parent_edge.forget_node_type(), + None => return None, + } + } + } + } + } -/// This replaces the value behind the `v` unique reference by calling the -/// relevant function. -/// -/// Safety: The change closure must not panic. -#[inline] -unsafe fn replace(v: &mut T, change: impl FnOnce(T) -> (T, R)) -> R { - let value = ptr::read(v); - let (new_value, ret) = change(value); - ptr::write(v, new_value); - ret + /// Deallocates a pile of nodes from the leaf up to the root. + /// This is the only way to deallocate the remainder of a tree after + /// `deallocating_next` and `deallocating_next_back` have been nibbling at + /// both sides of the tree, and have hit the same edge. As it is intended + /// only to be called when all keys and values have been returned, + /// no cleanup is done on any of the keys or values. + fn deallocating_end(self, alloc: A) { + let mut edge = self.forget_node_type(); + while let Some(parent_edge) = + unsafe { edge.into_node().deallocate_and_ascend(alloc.clone()) } + { + edge = parent_edge.forget_node_type(); + } + } } impl<'a, K, V> Handle, K, V, marker::Leaf>, marker::Edge> { /// Moves the leaf edge handle to the next leaf edge and returns references to the /// key and value in between. - /// Unsafe because the caller must ensure that the leaf edge is not the last one in the tree. - pub unsafe fn next_unchecked(&mut self) -> (&'a K, &'a V) { - replace(self, |leaf_edge| { - let kv = leaf_edge.next_kv(); - let kv = unwrap_unchecked(kv.ok()); + /// + /// # Safety + /// There must be another KV in the direction travelled. + unsafe fn next_unchecked(&mut self) -> (&'a K, &'a V) { + super::mem::replace(self, |leaf_edge| { + let kv = leaf_edge.next_kv().ok().unwrap(); (kv.next_leaf_edge(), kv.into_kv()) }) } /// Moves the leaf edge handle to the previous leaf edge and returns references to the /// key and value in between. - /// Unsafe because the caller must ensure that the leaf edge is not the first one in the tree. - pub unsafe fn next_back_unchecked(&mut self) -> (&'a K, &'a V) { - replace(self, |leaf_edge| { - let kv = leaf_edge.next_back_kv(); - let kv = unwrap_unchecked(kv.ok()); + /// + /// # Safety + /// There must be another KV in the direction travelled. + unsafe fn next_back_unchecked(&mut self) -> (&'a K, &'a V) { + super::mem::replace(self, |leaf_edge| { + let kv = leaf_edge.next_back_kv().ok().unwrap(); (kv.next_back_leaf_edge(), kv.into_kv()) }) } } -impl<'a, K, V> Handle, K, V, marker::Leaf>, marker::Edge> { +impl<'a, K, V> Handle, K, V, marker::Leaf>, marker::Edge> { /// Moves the leaf edge handle to the next leaf edge and returns references to the /// key and value in between. - /// Unsafe for two reasons: - /// - The caller must ensure that the leaf edge is not the last one in the tree. - /// - Using the updated handle may well invalidate the returned references. - pub unsafe fn next_unchecked(&mut self) -> (&'a mut K, &'a mut V) { - let kv = replace(self, |leaf_edge| { - let kv = leaf_edge.next_kv(); - let kv = unwrap_unchecked(kv.ok()); - (ptr::read(&kv).next_leaf_edge(), kv) + /// + /// # Safety + /// There must be another KV in the direction travelled. + unsafe fn next_unchecked(&mut self) -> (&'a K, &'a mut V) { + let kv = super::mem::replace(self, |leaf_edge| { + let kv = leaf_edge.next_kv().ok().unwrap(); + (unsafe { ptr::read(&kv) }.next_leaf_edge(), kv) }); - // Doing the descend (and perhaps another move) invalidates the references - // returned by `into_kv_mut`, so we have to do this last. - kv.into_kv_mut() + // Doing this last is faster, according to benchmarks. + kv.into_kv_valmut() } /// Moves the leaf edge handle to the previous leaf and returns references to the /// key and value in between. - /// Unsafe for two reasons: - /// - The caller must ensure that the leaf edge is not the first one in the tree. - /// - Using the updated handle may well invalidate the returned references. - pub unsafe fn next_back_unchecked(&mut self) -> (&'a mut K, &'a mut V) { - let kv = replace(self, |leaf_edge| { - let kv = leaf_edge.next_back_kv(); - let kv = unwrap_unchecked(kv.ok()); - (ptr::read(&kv).next_back_leaf_edge(), kv) + /// + /// # Safety + /// There must be another KV in the direction travelled. + unsafe fn next_back_unchecked(&mut self) -> (&'a K, &'a mut V) { + let kv = super::mem::replace(self, |leaf_edge| { + let kv = leaf_edge.next_back_kv().ok().unwrap(); + (unsafe { ptr::read(&kv) }.next_back_leaf_edge(), kv) }); - // Doing the descend (and perhaps another move) invalidates the references - // returned by `into_kv_mut`, so we have to do this last. - kv.into_kv_mut() + // Doing this last is faster, according to benchmarks. + kv.into_kv_valmut() } } -impl Handle, marker::Edge> { +impl Handle, marker::Edge> { /// Moves the leaf edge handle to the next leaf edge and returns the key and value - /// in between, while deallocating any node left behind. - /// Unsafe for three reasons: - /// - The caller must ensure that the leaf edge is not the last one in the tree - /// and is not a handle previously resulting from counterpart `next_back_unchecked`. - /// - If the leaf edge is the last edge of a node, that node and possibly ancestors - /// will be deallocated, while the reference to those nodes in the surviving ancestor - /// is left dangling; thus further use of the leaf edge handle is dangerous. - /// It is, however, safe to call this method again on the updated handle. - /// if the two preconditions above hold. - /// - Using the updated handle may well invalidate the returned references. - pub unsafe fn next_unchecked(&mut self) -> (K, V) { - replace(self, |leaf_edge| { - let kv = next_kv_unchecked_dealloc(leaf_edge); - let k = ptr::read(kv.reborrow().into_kv().0); - let v = ptr::read(kv.reborrow().into_kv().1); - (kv.next_leaf_edge(), (k, v)) + /// in between, deallocating any node left behind while leaving the corresponding + /// edge in its parent node dangling. + /// + /// # Safety + /// - There must be another KV in the direction travelled. + /// - That KV was not previously returned by counterpart + /// `deallocating_next_back_unchecked` on any copy of the handles + /// being used to traverse the tree. + /// + /// The only safe way to proceed with the updated handle is to compare it, drop it, + /// or call this method or counterpart `deallocating_next_back_unchecked` again. + unsafe fn deallocating_next_unchecked( + &mut self, + alloc: A, + ) -> Handle, marker::KV> { + super::mem::replace(self, |leaf_edge| unsafe { + leaf_edge.deallocating_next(alloc).unwrap() }) } - /// Moves the leaf edge handle to the previous leaf edge and returns the key - /// and value in between, while deallocating any node left behind. - /// Unsafe for three reasons: - /// - The caller must ensure that the leaf edge is not the first one in the tree - /// and is not a handle previously resulting from counterpart `next_unchecked`. - /// - If the lead edge is the first edge of a node, that node and possibly ancestors - /// will be deallocated, while the reference to those nodes in the surviving ancestor - /// is left dangling; thus further use of the leaf edge handle is dangerous. - /// It is, however, safe to call this method again on the updated handle. - /// if the two preconditions above hold. - /// - Using the updated handle may well invalidate the returned references. - pub unsafe fn next_back_unchecked(&mut self) -> (K, V) { - replace(self, |leaf_edge| { - let kv = next_back_kv_unchecked_dealloc(leaf_edge); - let k = ptr::read(kv.reborrow().into_kv().0); - let v = ptr::read(kv.reborrow().into_kv().1); - (kv.next_back_leaf_edge(), (k, v)) + /// Moves the leaf edge handle to the previous leaf edge and returns the key and value + /// in between, deallocating any node left behind while leaving the corresponding + /// edge in its parent node dangling. + /// + /// # Safety + /// - There must be another KV in the direction travelled. + /// - That leaf edge was not previously returned by counterpart + /// `deallocating_next_unchecked` on any copy of the handles + /// being used to traverse the tree. + /// + /// The only safe way to proceed with the updated handle is to compare it, drop it, + /// or call this method or counterpart `deallocating_next_unchecked` again. + unsafe fn deallocating_next_back_unchecked( + &mut self, + alloc: A, + ) -> Handle, marker::KV> { + super::mem::replace(self, |leaf_edge| unsafe { + leaf_edge.deallocating_next_back(alloc).unwrap() }) } } -impl NodeRef { +impl NodeRef { /// Returns the leftmost leaf edge in or underneath a node - in other words, the edge /// you need first when navigating forward (or last when navigating backward). #[inline] @@ -216,7 +639,62 @@ impl NodeRef { } } -impl Handle, marker::KV> { +pub enum Position { + Leaf(NodeRef), + Internal(NodeRef), + InternalKV(Handle, marker::KV>), +} + +impl<'a, K: 'a, V: 'a> NodeRef, K, V, marker::LeafOrInternal> { + /// Visits leaf nodes and internal KVs in order of ascending keys, and also + /// visits internal nodes as a whole in a depth first order, meaning that + /// internal nodes precede their individual KVs and their child nodes. + pub fn visit_nodes_in_order(self, mut visit: F) + where + F: FnMut(Position, K, V>), + { + match self.force() { + Leaf(leaf) => visit(Position::Leaf(leaf)), + Internal(internal) => { + visit(Position::Internal(internal)); + let mut edge = internal.first_edge(); + loop { + edge = match edge.descend().force() { + Leaf(leaf) => { + visit(Position::Leaf(leaf)); + match edge.next_kv() { + Ok(kv) => { + visit(Position::InternalKV(kv)); + kv.right_edge() + } + Err(_) => return, + } + } + Internal(internal) => { + visit(Position::Internal(internal)); + internal.first_edge() + } + } + } + } + } + } + + /// Calculates the number of elements in a (sub)tree. + pub fn calc_length(self) -> usize { + let mut result = 0; + self.visit_nodes_in_order(|pos| match pos { + Position::Leaf(node) => result += node.len(), + Position::Internal(node) => result += node.len(), + Position::InternalKV(_) => (), + }); + result + } +} + +impl + Handle, marker::KV> +{ /// Returns the leaf edge closest to a KV for forward navigation. pub fn next_leaf_edge(self) -> Handle, marker::Edge> { match self.force() { @@ -229,9 +707,7 @@ impl Handle, } /// Returns the leaf edge closest to a KV for backward navigation. - pub fn next_back_leaf_edge( - self, - ) -> Handle, marker::Edge> { + fn next_back_leaf_edge(self) -> Handle, marker::Edge> { match self.force() { Leaf(leaf_kv) => leaf_kv.left_edge(), Internal(internal_kv) => { diff --git a/crux-mir/lib/alloc/src/collections/btree/node.rs b/crux-mir/lib/alloc/src/collections/btree/node.rs index 6ebb98c42..691246644 100644 --- a/crux-mir/lib/alloc/src/collections/btree/node.rs +++ b/crux-mir/lib/alloc/src/collections/btree/node.rs @@ -9,11 +9,8 @@ // struct Node { // keys: [K; 2 * B - 1], // vals: [V; 2 * B - 1], -// edges: if height > 0 { -// [Box>; 2 * B] -// } else { () }, -// parent: *const Node, -// parent_idx: u16, +// edges: [if height > 0 { Box> } else { () }; 2 * B], +// parent: Option<(NonNull>, u16)>, // len: u16, // } // ``` @@ -28,28 +25,31 @@ // // - Trees must have uniform depth/height. This means that every path down to a leaf from a // given node has exactly the same length. -// - A node of length `n` has `n` keys, `n` values, and (in an internal node) `n + 1` edges. -// This implies that even an empty internal node has at least one edge. +// - A node of length `n` has `n` keys, `n` values, and `n + 1` edges. +// This implies that even an empty node has at least one edge. +// For a leaf node, "having an edge" only means we can identify a position in the node, +// since leaf edges are empty and need no data representation. In an internal node, +// an edge both identifies a position and contains a pointer to a child node. -use core::cmp::Ordering; use core::marker::PhantomData; use core::mem::{self, MaybeUninit}; -use core::ptr::{self, NonNull, Unique}; -use core::slice; +use core::ptr::{self, NonNull}; +use core::slice::SliceIndex; -use crate::alloc::{AllocRef, Global, Layout}; +use crate::alloc::{Allocator, Layout}; use crate::boxed::Box; const B: usize = 6; -pub const MIN_LEN: usize = B - 1; pub const CAPACITY: usize = 2 * B - 1; +pub const MIN_LEN_AFTER_SPLIT: usize = B - 1; +const KV_IDX_CENTER: usize = B - 1; +const EDGE_IDX_LEFT_OF_CENTER: usize = B - 1; +const EDGE_IDX_RIGHT_OF_CENTER: usize = B; -/// The underlying representation of leaf nodes. -#[repr(C)] +/// The underlying representation of leaf nodes and part of the representation of internal nodes. struct LeafNode { - /// We use `*const` as opposed to `*mut` so as to be covariant in `K` and `V`. - /// This either points to an actual node or is null. - parent: *const InternalNode, + /// We want to be covariant in `K` and `V`. + parent: Option>>, /// This node's index into the parent node's `edges` array. /// `*node.parent.edges[node.parent_idx]` should be the same thing as `node`. @@ -57,9 +57,6 @@ struct LeafNode { parent_idx: MaybeUninit, /// The number of keys and values this node stores. - /// - /// This next to `parent_idx` to encourage the compiler to join `len` and - /// `parent_idx` into the same 32-bit word, reducing space overhead. len: u16, /// The arrays storing the actual data of the node. Only the first `len` elements of each @@ -69,45 +66,57 @@ struct LeafNode { } impl LeafNode { - /// Creates a new `LeafNode`. Unsafe because all nodes should really be hidden behind - /// `BoxedNode`, preventing accidental dropping of uninitialized keys and values. - unsafe fn new() -> Self { - LeafNode { - // As a general policy, we leave fields uninitialized if they can be, as this should - // be both slightly faster and easier to track in Valgrind. - keys: [MaybeUninit::UNINIT; CAPACITY], - vals: [MaybeUninit::UNINIT; CAPACITY], - parent: ptr::null(), - parent_idx: MaybeUninit::uninit(), - len: 0, + /// Initializes a new `LeafNode` in-place. + unsafe fn init(this: *mut Self) { + // As a general policy, we leave fields uninitialized if they can be, as this should + // be both slightly faster and easier to track in Valgrind. + unsafe { + // parent_idx, keys, and vals are all MaybeUninit + ptr::addr_of_mut!((*this).parent).write(None); + ptr::addr_of_mut!((*this).len).write(0); + } + } + + /// Creates a new boxed `LeafNode`. + fn new(alloc: A) -> Box { + unsafe { + let mut leaf = Box::new_uninit_in(alloc); + LeafNode::init(leaf.as_mut_ptr()); + leaf.assume_init() } } } /// The underlying representation of internal nodes. As with `LeafNode`s, these should be hidden /// behind `BoxedNode`s to prevent dropping uninitialized keys and values. Any pointer to an -/// `InternalNode` can be directly casted to a pointer to the underlying `LeafNode` portion of the +/// `InternalNode` can be directly cast to a pointer to the underlying `LeafNode` portion of the /// node, allowing code to act on leaf and internal nodes generically without having to even check /// which of the two a pointer is pointing at. This property is enabled by the use of `repr(C)`. #[repr(C)] +// gdb_providers.py uses this type name for introspection. struct InternalNode { data: LeafNode, /// The pointers to the children of this node. `len + 1` of these are considered - /// initialized and valid. + /// initialized and valid, except that near the end, while the tree is held + /// through borrow type `Dying`, some of these pointers are dangling. edges: [MaybeUninit>; 2 * B], } impl InternalNode { - /// Creates a new `InternalNode`. + /// Creates a new boxed `InternalNode`. /// - /// This is unsafe for two reasons. First, it returns an `InternalNode` by value, risking - /// dropping of uninitialized fields. Second, an invariant of internal nodes is that `len + 1` - /// edges are initialized and valid, meaning that even when the node is empty (having a - /// `len` of 0), there must be one initialized and valid edge. This function does not set up + /// # Safety + /// An invariant of internal nodes is that they have at least one + /// initialized and valid edge. This function does not set up /// such an edge. - unsafe fn new() -> Self { - InternalNode { data: LeafNode::new(), edges: [MaybeUninit::UNINIT; 2 * B] } + unsafe fn new(alloc: A) -> Box { + unsafe { + let mut node = Box::::new_uninit_in(alloc); + // We only need to initialize the data; the edges are MaybeUninit. + LeafNode::init(ptr::addr_of_mut!((*node.as_mut_ptr()).data)); + node.assume_init() + } } } @@ -116,148 +125,78 @@ impl InternalNode { /// /// However, `BoxedNode` contains no information as to which of the two types /// of nodes it actually contains, and, partially due to this lack of information, -/// has no destructor. -struct BoxedNode { - ptr: Unique>, -} - -impl BoxedNode { - fn from_leaf(node: Box>) -> Self { - BoxedNode { ptr: Box::into_unique(node) } - } - - fn from_internal(node: Box>) -> Self { - BoxedNode { ptr: Box::into_unique(node).cast() } - } - - unsafe fn from_ptr(ptr: NonNull>) -> Self { - BoxedNode { ptr: Unique::from(ptr) } - } - - fn as_ptr(&self) -> NonNull> { - NonNull::from(self.ptr) - } -} - -/// An owned tree. -/// -/// Note that this does not have a destructor, and must be cleaned up manually. -pub struct Root { - node: BoxedNode, - /// The number of levels below the root node. - height: usize, -} - -unsafe impl Sync for Root {} -unsafe impl Send for Root {} - -impl Root { - /// Returns a new owned tree, with its own root node that is initially empty. - pub fn new_leaf() -> Self { - Root { node: BoxedNode::from_leaf(Box::new(unsafe { LeafNode::new() })), height: 0 } - } - - pub fn as_ref(&self) -> NodeRef, K, V, marker::LeafOrInternal> { - NodeRef { - height: self.height, - node: self.node.as_ptr(), - root: self as *const _ as *mut _, - _marker: PhantomData, - } - } - - pub fn as_mut(&mut self) -> NodeRef, K, V, marker::LeafOrInternal> { - NodeRef { - height: self.height, - node: self.node.as_ptr(), - root: self as *mut _, - _marker: PhantomData, - } - } - - pub fn into_ref(self) -> NodeRef { - NodeRef { - height: self.height, - node: self.node.as_ptr(), - root: ptr::null_mut(), // FIXME: Is there anything better to do here? - _marker: PhantomData, - } - } - - /// Adds a new internal node with a single edge, pointing to the previous root, and make that - /// new node the root. This increases the height by 1 and is the opposite of `pop_level`. - pub fn push_level(&mut self) -> NodeRef, K, V, marker::Internal> { - let mut new_node = Box::new(unsafe { InternalNode::new() }); - new_node.edges[0].write(unsafe { BoxedNode::from_ptr(self.node.as_ptr()) }); - - self.node = BoxedNode::from_internal(new_node); - self.height += 1; - - let mut ret = NodeRef { - height: self.height, - node: self.node.as_ptr(), - root: self as *mut _, - _marker: PhantomData, - }; - - unsafe { - ret.reborrow_mut().first_edge().correct_parent_link(); - } - - ret - } - - /// Removes the root node, using its first child as the new root. This cannot be called when - /// the tree consists only of a leaf node. As it is intended only to be called when the root - /// has only one edge, no cleanup is done on any of the other children of the root. - /// This decreases the height by 1 and is the opposite of `push_level`. - pub fn pop_level(&mut self) { - assert!(self.height > 0); - - let top = self.node.ptr; - - self.node = unsafe { - BoxedNode::from_ptr( - self.as_mut().cast_unchecked::().first_edge().descend().node, - ) - }; - self.height -= 1; - unsafe { - (*self.as_mut().as_leaf_mut()).parent = ptr::null(); - } - - unsafe { - Global.dealloc(NonNull::from(top).cast(), Layout::new::>()); - } - } -} +/// is not a separate type and has no destructor. +type BoxedNode = NonNull>; // N.B. `NodeRef` is always covariant in `K` and `V`, even when the `BorrowType` // is `Mut`. This is technically wrong, but cannot result in any unsafety due to // internal use of `NodeRef` because we stay completely generic over `K` and `V`. // However, whenever a public type wraps `NodeRef`, make sure that it has the // correct variance. +/// /// A reference to a node. /// /// This type has a number of parameters that controls how it acts: -/// - `BorrowType`: This can be `Immut<'a>` or `Mut<'a>` for some `'a` or `Owned`. -/// When this is `Immut<'a>`, the `NodeRef` acts roughly like `&'a Node`, -/// when this is `Mut<'a>`, the `NodeRef` acts roughly like `&'a mut Node`, -/// and when this is `Owned`, the `NodeRef` acts roughly like `Box`. -/// - `K` and `V`: These control what types of things are stored in the nodes. +/// - `BorrowType`: A dummy type that describes the kind of borrow and carries a lifetime. +/// - When this is `Immut<'a>`, the `NodeRef` acts roughly like `&'a Node`. +/// - When this is `ValMut<'a>`, the `NodeRef` acts roughly like `&'a Node` +/// with respect to keys and tree structure, but also allows many +/// mutable references to values throughout the tree to coexist. +/// - When this is `Mut<'a>`, the `NodeRef` acts roughly like `&'a mut Node`, +/// although insert methods allow a mutable pointer to a value to coexist. +/// - When this is `Owned`, the `NodeRef` acts roughly like `Box`, +/// but does not have a destructor, and must be cleaned up manually. +/// - When this is `Dying`, the `NodeRef` still acts roughly like `Box`, +/// but has methods to destroy the tree bit by bit, and ordinary methods, +/// while not marked as unsafe to call, can invoke UB if called incorrectly. +/// Since any `NodeRef` allows navigating through the tree, `BorrowType` +/// effectively applies to the entire tree, not just to the node itself. +/// - `K` and `V`: These are the types of keys and values stored in the nodes. /// - `Type`: This can be `Leaf`, `Internal`, or `LeafOrInternal`. When this is /// `Leaf`, the `NodeRef` points to a leaf node, when this is `Internal` the /// `NodeRef` points to an internal node, and when this is `LeafOrInternal` the /// `NodeRef` could be pointing to either type of node. +/// `Type` is named `NodeType` when used outside `NodeRef`. +/// +/// Both `BorrowType` and `NodeType` restrict what methods we implement, to +/// exploit static type safety. There are limitations in the way we can apply +/// such restrictions: +/// - For each type parameter, we can only define a method either generically +/// or for one particular type. For example, we cannot define a method like +/// `into_kv` generically for all `BorrowType`, or once for all types that +/// carry a lifetime, because we want it to return `&'a` references. +/// Therefore, we define it only for the least powerful type `Immut<'a>`. +/// - We cannot get implicit coercion from say `Mut<'a>` to `Immut<'a>`. +/// Therefore, we have to explicitly call `reborrow` on a more powerful +/// `NodeRef` in order to reach a method like `into_kv`. +/// +/// All methods on `NodeRef` that return some kind of reference, either: +/// - Take `self` by value, and return the lifetime carried by `BorrowType`. +/// Sometimes, to invoke such a method, we need to call `reborrow_mut`. +/// - Take `self` by reference, and (implicitly) return that reference's +/// lifetime, instead of the lifetime carried by `BorrowType`. That way, +/// the borrow checker guarantees that the `NodeRef` remains borrowed as long +/// as the returned reference is used. +/// The methods supporting insert bend this rule by returning a raw pointer, +/// i.e., a reference without any lifetime. pub struct NodeRef { - /// The number of levels below the node. + /// The number of levels that the node and the level of leaves are apart, a + /// constant of the node that cannot be entirely described by `Type`, and that + /// the node itself does not store. We only need to store the height of the root + /// node, and derive every other node's height from it. + /// Must be zero if `Type` is `Leaf` and non-zero if `Type` is `Internal`. height: usize, + /// The pointer to the leaf or internal node. The definition of `InternalNode` + /// ensures that the pointer is valid either way. node: NonNull>, - // `root` is null unless the borrow type is `Mut` - root: *const Root, _marker: PhantomData<(BorrowType, Type)>, } +/// The root node of an owned tree. +/// +/// Note that this does not have a destructor, and must be cleaned up manually. +pub type Root = NodeRef; + impl<'a, K: 'a, V: 'a, Type> Copy for NodeRef, K, V, Type> {} impl<'a, K: 'a, V: 'a, Type> Clone for NodeRef, K, V, Type> { fn clone(&self) -> Self { @@ -267,95 +206,133 @@ impl<'a, K: 'a, V: 'a, Type> Clone for NodeRef, K, V, Type> { unsafe impl Sync for NodeRef {} -unsafe impl<'a, K: Sync + 'a, V: Sync + 'a, Type> Send for NodeRef, K, V, Type> {} -unsafe impl<'a, K: Send + 'a, V: Send + 'a, Type> Send for NodeRef, K, V, Type> {} +unsafe impl Send for NodeRef, K, V, Type> {} +unsafe impl Send for NodeRef, K, V, Type> {} +unsafe impl Send for NodeRef, K, V, Type> {} unsafe impl Send for NodeRef {} +unsafe impl Send for NodeRef {} + +impl NodeRef { + pub fn new_leaf(alloc: A) -> Self { + Self::from_new_leaf(LeafNode::new(alloc)) + } + + fn from_new_leaf(leaf: Box, A>) -> Self { + NodeRef { height: 0, node: NonNull::from(Box::leak(leaf)), _marker: PhantomData } + } +} + +impl NodeRef { + fn new_internal(child: Root, alloc: A) -> Self { + let mut new_node = unsafe { InternalNode::new(alloc) }; + new_node.edges[0].write(child.node); + unsafe { NodeRef::from_new_internal(new_node, child.height + 1) } + } + + /// # Safety + /// `height` must not be zero. + unsafe fn from_new_internal( + internal: Box, A>, + height: usize, + ) -> Self { + debug_assert!(height > 0); + let node = NonNull::from(Box::leak(internal)).cast(); + let mut this = NodeRef { height, node, _marker: PhantomData }; + this.borrow_mut().correct_all_childrens_parent_links(); + this + } +} impl NodeRef { - fn as_internal(&self) -> &InternalNode { - unsafe { &*(self.node.as_ptr() as *mut InternalNode) } + /// Unpack a node reference that was packed as `NodeRef::parent`. + fn from_internal(node: NonNull>, height: usize) -> Self { + debug_assert!(height > 0); + NodeRef { height, node: node.cast(), _marker: PhantomData } + } +} + +impl NodeRef { + /// Exposes the data of an internal node. + /// + /// Returns a raw ptr to avoid invalidating other references to this node. + fn as_internal_ptr(this: &Self) -> *mut InternalNode { + // SAFETY: the static node type is `Internal`. + this.node.as_ptr() as *mut InternalNode } } impl<'a, K, V> NodeRef, K, V, marker::Internal> { + /// Borrows exclusive access to the data of an internal node. fn as_internal_mut(&mut self) -> &mut InternalNode { - unsafe { &mut *(self.node.as_ptr() as *mut InternalNode) } + let ptr = Self::as_internal_ptr(self); + unsafe { &mut *ptr } } } impl NodeRef { - /// Finds the length of the node. This is the number of keys or values. In an - /// internal node, the number of edges is `len() + 1`. - /// For any node, the number of possible edge handles is also `len() + 1`. + /// Finds the length of the node. This is the number of keys or values. + /// The number of edges is `len() + 1`. /// Note that, despite being safe, calling this function can have the side effect /// of invalidating mutable references that unsafe code has created. pub fn len(&self) -> usize { - self.as_leaf().len as usize + // Crucially, we only access the `len` field here. If BorrowType is marker::ValMut, + // there might be outstanding mutable references to values that we must not invalidate. + unsafe { usize::from((*Self::as_leaf_ptr(self)).len) } } - /// Returns the height of this node in the whole tree. Zero height denotes the - /// leaf level. + /// Returns the number of levels that the node and leaves are apart. Zero + /// height means the node is a leaf itself. If you picture trees with the + /// root on top, the number says at which elevation the node appears. + /// If you picture trees with leaves on top, the number says how high + /// the tree extends above the node. pub fn height(&self) -> usize { self.height } - /// Removes any static information about whether this node is a `Leaf` or an - /// `Internal` node. - pub fn forget_type(self) -> NodeRef { - NodeRef { height: self.height, node: self.node, root: self.root, _marker: PhantomData } - } - /// Temporarily takes out another, immutable reference to the same node. - fn reborrow(&self) -> NodeRef, K, V, Type> { - NodeRef { height: self.height, node: self.node, root: self.root, _marker: PhantomData } + pub fn reborrow(&self) -> NodeRef, K, V, Type> { + NodeRef { height: self.height, node: self.node, _marker: PhantomData } } - /// Exposes the leaf "portion" of any leaf or internal node. - /// If the node is a leaf, this function simply opens up its data. - /// If the node is an internal node, so not a leaf, it does have all the data a leaf has - /// (header, keys and values), and this function exposes that. - fn as_leaf(&self) -> &LeafNode { + /// Exposes the leaf portion of any leaf or internal node. + /// + /// Returns a raw ptr to avoid invalidating other references to this node. + fn as_leaf_ptr(this: &Self) -> *mut LeafNode { // The node must be valid for at least the LeafNode portion. // This is not a reference in the NodeRef type because we don't know if // it should be unique or shared. - unsafe { self.node.as_ref() } - } - - /// Borrows a view into the keys stored in the node. - pub fn keys(&self) -> &[K] { - self.reborrow().into_key_slice() - } - - /// Borrows a view into the values stored in the node. - fn vals(&self) -> &[V] { - self.reborrow().into_val_slice() + this.node.as_ptr() } +} +impl NodeRef { /// Finds the parent of the current node. Returns `Ok(handle)` if the current /// node actually has a parent, where `handle` points to the edge of the parent /// that points to the current node. Returns `Err(self)` if the current node has /// no parent, giving back the original `NodeRef`. /// + /// The method name assumes you picture trees with the root node on top. + /// /// `edge.descend().ascend().unwrap()` and `node.ascend().unwrap().descend()` should /// both, upon success, do nothing. pub fn ascend( self, ) -> Result, marker::Edge>, Self> { - let parent_as_leaf = self.as_leaf().parent as *const LeafNode; - if let Some(non_zero) = NonNull::new(parent_as_leaf as *mut _) { - Ok(Handle { - node: NodeRef { - height: self.height + 1, - node: non_zero, - root: self.root, - _marker: PhantomData, - }, - idx: unsafe { usize::from(*self.as_leaf().parent_idx.as_ptr()) }, + const { + assert!(BorrowType::TRAVERSAL_PERMIT); + } + + // We need to use raw pointers to nodes because, if BorrowType is marker::ValMut, + // there might be outstanding mutable references to values that we must not invalidate. + let leaf_ptr: *const _ = Self::as_leaf_ptr(&self); + unsafe { (*leaf_ptr).parent } + .as_ref() + .map(|parent| Handle { + node: NodeRef::from_internal(*parent, self.height + 1), + idx: unsafe { usize::from((*leaf_ptr).parent_idx.assume_init()) }, _marker: PhantomData, }) - } else { - Err(self) - } + .ok_or(self) } pub fn first_edge(self) -> Handle { @@ -382,290 +359,320 @@ impl NodeRef { } } -impl NodeRef { +impl NodeRef { + /// Could be a public implementation of PartialEq, but only used in this module. + fn eq(&self, other: &Self) -> bool { + let Self { node, height, _marker } = self; + if node.eq(&other.node) { + debug_assert_eq!(*height, other.height); + true + } else { + false + } + } +} + +impl<'a, K: 'a, V: 'a, Type> NodeRef, K, V, Type> { + /// Exposes the leaf portion of any leaf or internal node in an immutable tree. + fn into_leaf(self) -> &'a LeafNode { + let ptr = Self::as_leaf_ptr(&self); + // SAFETY: there can be no mutable references into this tree borrowed as `Immut`. + unsafe { &*ptr } + } + + /// Borrows a view into the keys stored in the node. + pub fn keys(&self) -> &[K] { + let leaf = self.into_leaf(); + unsafe { + MaybeUninit::slice_assume_init_ref(leaf.keys.get_unchecked(..usize::from(leaf.len))) + } + } +} + +impl NodeRef { /// Similar to `ascend`, gets a reference to a node's parent node, but also - /// deallocate the current node in the process. This is unsafe because the + /// deallocates the current node in the process. This is unsafe because the /// current node will still be accessible despite being deallocated. - pub unsafe fn deallocate_and_ascend( + pub unsafe fn deallocate_and_ascend( self, - ) -> Option, marker::Edge>> { + alloc: A, + ) -> Option, marker::Edge>> { let height = self.height; let node = self.node; let ret = self.ascend().ok(); - Global.dealloc( - node.cast(), - if height > 0 { - Layout::new::>() - } else { - Layout::new::>() - }, - ); + unsafe { + alloc.deallocate( + node.cast(), + if height > 0 { + Layout::new::>() + } else { + Layout::new::>() + }, + ); + } ret } } impl<'a, K, V, Type> NodeRef, K, V, Type> { - /// Unsafely asserts to the compiler some static information about whether this - /// node is a `Leaf`. - unsafe fn cast_unchecked(&mut self) -> NodeRef, K, V, NewType> { - NodeRef { height: self.height, node: self.node, root: self.root, _marker: PhantomData } - } - - /// Temporarily takes out another, mutable reference to the same node. Beware, as - /// this method is very dangerous, doubly so since it may not immediately appear + /// Temporarily takes out another mutable reference to the same node. Beware, as + /// this method is very dangerous, doubly so since it might not immediately appear /// dangerous. /// - /// Because mutable pointers can roam anywhere around the tree and can even (through - /// `into_root_mut`) mess with the root of the tree, the result of `reborrow_mut` - /// can easily be used to make the original mutable pointer dangling, or, in the case - /// of a reborrowed handle, out of bounds. - // FIXME(@gereeter) consider adding yet another type parameter to `NodeRef` that restricts - // the use of `ascend` and `into_root_mut` on reborrowed pointers, preventing this unsafety. + /// Because mutable pointers can roam anywhere around the tree, the returned + /// pointer can easily be used to make the original pointer dangling, out of + /// bounds, or invalid under stacked borrow rules. + // FIXME(@gereeter) consider adding yet another type parameter to `NodeRef` + // that restricts the use of navigation methods on reborrowed pointers, + // preventing this unsafety. unsafe fn reborrow_mut(&mut self) -> NodeRef, K, V, Type> { - NodeRef { height: self.height, node: self.node, root: self.root, _marker: PhantomData } + NodeRef { height: self.height, node: self.node, _marker: PhantomData } } - /// Exposes the leaf "portion" of any leaf or internal node for writing. - /// If the node is a leaf, this function simply opens up its data. - /// If the node is an internal node, so not a leaf, it does have all the data a leaf has - /// (header, keys and values), and this function exposes that. - /// - /// Returns a raw ptr to avoid asserting exclusive access to the entire node. - fn as_leaf_mut(&mut self) -> *mut LeafNode { - self.node.as_ptr() + /// Borrows exclusive access to the leaf portion of a leaf or internal node. + fn as_leaf_mut(&mut self) -> &mut LeafNode { + let ptr = Self::as_leaf_ptr(self); + // SAFETY: we have exclusive access to the entire node. + unsafe { &mut *ptr } } - fn keys_mut(&mut self) -> &mut [K] { + /// Offers exclusive access to the leaf portion of a leaf or internal node. + fn into_leaf_mut(mut self) -> &'a mut LeafNode { + let ptr = Self::as_leaf_ptr(&mut self); + // SAFETY: we have exclusive access to the entire node. + unsafe { &mut *ptr } + } +} + +impl NodeRef { + /// Borrows exclusive access to the leaf portion of a dying leaf or internal node. + fn as_leaf_dying(&mut self) -> &mut LeafNode { + let ptr = Self::as_leaf_ptr(self); + // SAFETY: we have exclusive access to the entire node. + unsafe { &mut *ptr } + } +} + +impl<'a, K: 'a, V: 'a, Type> NodeRef, K, V, Type> { + /// Borrows exclusive access to an element of the key storage area. + /// + /// # Safety + /// `index` is in bounds of 0..CAPACITY + unsafe fn key_area_mut(&mut self, index: I) -> &mut Output + where + I: SliceIndex<[MaybeUninit], Output = Output>, + { // SAFETY: the caller will not be able to call further methods on self // until the key slice reference is dropped, as we have unique access // for the lifetime of the borrow. - unsafe { self.reborrow_mut().into_key_slice_mut() } + unsafe { self.as_leaf_mut().keys.as_mut_slice().get_unchecked_mut(index) } } - fn vals_mut(&mut self) -> &mut [V] { + /// Borrows exclusive access to an element or slice of the node's value storage area. + /// + /// # Safety + /// `index` is in bounds of 0..CAPACITY + unsafe fn val_area_mut(&mut self, index: I) -> &mut Output + where + I: SliceIndex<[MaybeUninit], Output = Output>, + { // SAFETY: the caller will not be able to call further methods on self // until the value slice reference is dropped, as we have unique access // for the lifetime of the borrow. - unsafe { self.reborrow_mut().into_val_slice_mut() } + unsafe { self.as_leaf_mut().vals.as_mut_slice().get_unchecked_mut(index) } } } -impl<'a, K: 'a, V: 'a, Type> NodeRef, K, V, Type> { - fn into_key_slice(self) -> &'a [K] { - unsafe { slice::from_raw_parts(MaybeUninit::first_ptr(&self.as_leaf().keys), self.len()) } - } - - fn into_val_slice(self) -> &'a [V] { - unsafe { slice::from_raw_parts(MaybeUninit::first_ptr(&self.as_leaf().vals), self.len()) } +impl<'a, K: 'a, V: 'a> NodeRef, K, V, marker::Internal> { + /// Borrows exclusive access to an element or slice of the node's storage area for edge contents. + /// + /// # Safety + /// `index` is in bounds of 0..CAPACITY + 1 + unsafe fn edge_area_mut(&mut self, index: I) -> &mut Output + where + I: SliceIndex<[MaybeUninit>], Output = Output>, + { + // SAFETY: the caller will not be able to call further methods on self + // until the edge slice reference is dropped, as we have unique access + // for the lifetime of the borrow. + unsafe { self.as_internal_mut().edges.as_mut_slice().get_unchecked_mut(index) } } +} - fn into_slices(self) -> (&'a [K], &'a [V]) { - // SAFETY: equivalent to reborrow() except not requiring Type: 'a - let k = unsafe { ptr::read(&self) }; - (k.into_key_slice(), self.into_val_slice()) +impl<'a, K, V, Type> NodeRef, K, V, Type> { + /// # Safety + /// - The node has more than `idx` initialized elements. + unsafe fn into_key_val_mut_at(mut self, idx: usize) -> (&'a K, &'a mut V) { + // We only create a reference to the one element we are interested in, + // to avoid aliasing with outstanding references to other elements, + // in particular, those returned to the caller in earlier iterations. + let leaf = Self::as_leaf_ptr(&mut self); + let keys = unsafe { ptr::addr_of!((*leaf).keys) }; + let vals = unsafe { ptr::addr_of_mut!((*leaf).vals) }; + // We must coerce to unsized array pointers because of Rust issue #74679. + let keys: *const [_] = keys; + let vals: *mut [_] = vals; + let key = unsafe { (&*keys.get_unchecked(idx)).assume_init_ref() }; + let val = unsafe { (&mut *vals.get_unchecked_mut(idx)).assume_init_mut() }; + (key, val) } } impl<'a, K: 'a, V: 'a, Type> NodeRef, K, V, Type> { - /// Gets a mutable reference to the root itself. This is useful primarily when the - /// height of the tree needs to be adjusted. Never call this on a reborrowed pointer. - pub fn into_root_mut(self) -> &'a mut Root { - unsafe { &mut *(self.root as *mut Root) } - } - - fn into_key_slice_mut(mut self) -> &'a mut [K] { - // SAFETY: The keys of a node must always be initialized up to length. - unsafe { - slice::from_raw_parts_mut( - MaybeUninit::first_ptr_mut(&mut (*self.as_leaf_mut()).keys), - self.len(), - ) - } + /// Borrows exclusive access to the length of the node. + pub fn len_mut(&mut self) -> &mut u16 { + &mut self.as_leaf_mut().len } +} - fn into_val_slice_mut(mut self) -> &'a mut [V] { - // SAFETY: The values of a node must always be initialized up to length. - unsafe { - slice::from_raw_parts_mut( - MaybeUninit::first_ptr_mut(&mut (*self.as_leaf_mut()).vals), - self.len(), - ) +impl<'a, K, V> NodeRef, K, V, marker::Internal> { + /// # Safety + /// Every item returned by `range` is a valid edge index for the node. + unsafe fn correct_childrens_parent_links>(&mut self, range: R) { + for i in range { + debug_assert!(i <= self.len()); + unsafe { Handle::new_edge(self.reborrow_mut(), i) }.correct_parent_link(); } } - fn into_slices_mut(mut self) -> (&'a mut [K], &'a mut [V]) { - // We cannot use the getters here, because calling the second one - // invalidates the reference returned by the first. - // More precisely, it is the call to `len` that is the culprit, - // because that creates a shared reference to the header, which *can* - // overlap with the keys (and even the values, for ZST keys). + fn correct_all_childrens_parent_links(&mut self) { let len = self.len(); - let leaf = self.as_leaf_mut(); - // SAFETY: The keys and values of a node must always be initialized up to length. - let keys = unsafe { - slice::from_raw_parts_mut(MaybeUninit::first_ptr_mut(&mut (*leaf).keys), len) - }; - let vals = unsafe { - slice::from_raw_parts_mut(MaybeUninit::first_ptr_mut(&mut (*leaf).vals), len) - }; - (keys, vals) + unsafe { self.correct_childrens_parent_links(0..=len) }; } } -impl<'a, K, V> NodeRef, K, V, marker::Leaf> { - /// Adds a key/value pair the end of the node. - pub fn push(&mut self, key: K, val: V) { - assert!(self.len() < CAPACITY); - - let idx = self.len(); - - unsafe { - ptr::write(self.keys_mut().get_unchecked_mut(idx), key); - ptr::write(self.vals_mut().get_unchecked_mut(idx), val); +impl<'a, K: 'a, V: 'a> NodeRef, K, V, marker::LeafOrInternal> { + /// Sets the node's link to its parent edge, + /// without invalidating other references to the node. + fn set_parent_link(&mut self, parent: NonNull>, parent_idx: usize) { + let leaf = Self::as_leaf_ptr(self); + unsafe { (*leaf).parent = Some(parent) }; + unsafe { (*leaf).parent_idx.write(parent_idx as u16) }; + } +} - (*self.as_leaf_mut()).len += 1; - } +impl NodeRef { + /// Clears the root's link to its parent edge. + fn clear_parent_link(&mut self) { + let mut root_node = self.borrow_mut(); + let leaf = root_node.as_leaf_mut(); + leaf.parent = None; } +} - /// Adds a key/value pair to the beginning of the node. - pub fn push_front(&mut self, key: K, val: V) { - assert!(self.len() < CAPACITY); +impl NodeRef { + /// Returns a new owned tree, with its own root node that is initially empty. + pub fn new(alloc: A) -> Self { + NodeRef::new_leaf(alloc).forget_type() + } - unsafe { - slice_insert(self.keys_mut(), 0, key); - slice_insert(self.vals_mut(), 0, val); + /// Adds a new internal node with a single edge pointing to the previous root node, + /// make that new node the root node, and return it. This increases the height by 1 + /// and is the opposite of `pop_internal_level`. + pub fn push_internal_level( + &mut self, + alloc: A, + ) -> NodeRef, K, V, marker::Internal> { + super::mem::take_mut(self, |old_root| NodeRef::new_internal(old_root, alloc).forget_type()); - (*self.as_leaf_mut()).len += 1; - } + // `self.borrow_mut()`, except that we just forgot we're internal now: + NodeRef { height: self.height, node: self.node, _marker: PhantomData } } -} -impl<'a, K, V> NodeRef, K, V, marker::Internal> { - /// Adds a key/value pair and an edge to go to the right of that pair to - /// the end of the node. - pub fn push(&mut self, key: K, val: V, edge: Root) { - assert!(edge.height == self.height - 1); - assert!(self.len() < CAPACITY); + /// Removes the internal root node, using its first child as the new root node. + /// As it is intended only to be called when the root node has only one child, + /// no cleanup is done on any of the keys, values and other children. + /// This decreases the height by 1 and is the opposite of `push_internal_level`. + /// + /// Requires exclusive access to the `NodeRef` object but not to the root node; + /// it will not invalidate other handles or references to the root node. + /// + /// Panics if there is no internal level, i.e., if the root node is a leaf. + pub fn pop_internal_level(&mut self, alloc: A) { + assert!(self.height > 0); - let idx = self.len(); + let top = self.node; - unsafe { - ptr::write(self.keys_mut().get_unchecked_mut(idx), key); - ptr::write(self.vals_mut().get_unchecked_mut(idx), val); - self.as_internal_mut().edges.get_unchecked_mut(idx + 1).write(edge.node); - - (*self.as_leaf_mut()).len += 1; + // SAFETY: we asserted to be internal. + let internal_self = unsafe { self.borrow_mut().cast_to_internal_unchecked() }; + // SAFETY: we borrowed `self` exclusively and its borrow type is exclusive. + let internal_node = unsafe { &mut *NodeRef::as_internal_ptr(&internal_self) }; + // SAFETY: the first edge is always initialized. + self.node = unsafe { internal_node.edges[0].assume_init_read() }; + self.height -= 1; + self.clear_parent_link(); - Handle::new_edge(self.reborrow_mut(), idx + 1).correct_parent_link(); + unsafe { + alloc.deallocate(top.cast(), Layout::new::>()); } } +} - // Unsafe because 'first' and 'after_last' must be in range - unsafe fn correct_childrens_parent_links(&mut self, first: usize, after_last: usize) { - debug_assert!(first <= self.len()); - debug_assert!(after_last <= self.len() + 1); - for i in first..after_last { - Handle::new_edge(self.reborrow_mut(), i).correct_parent_link(); - } +impl NodeRef { + /// Mutably borrows the owned root node. Unlike `reborrow_mut`, this is safe + /// because the return value cannot be used to destroy the root, and there + /// cannot be other references to the tree. + pub fn borrow_mut(&mut self) -> NodeRef, K, V, Type> { + NodeRef { height: self.height, node: self.node, _marker: PhantomData } } - fn correct_all_childrens_parent_links(&mut self) { - let len = self.len(); - unsafe { self.correct_childrens_parent_links(0, len + 1) }; + /// Slightly mutably borrows the owned root node. + pub fn borrow_valmut(&mut self) -> NodeRef, K, V, Type> { + NodeRef { height: self.height, node: self.node, _marker: PhantomData } } - /// Adds a key/value pair and an edge to go to the left of that pair to - /// the beginning of the node. - pub fn push_front(&mut self, key: K, val: V, edge: Root) { - assert!(edge.height == self.height - 1); - assert!(self.len() < CAPACITY); + /// Irreversibly transitions to a reference that permits traversal and offers + /// destructive methods and little else. + pub fn into_dying(self) -> NodeRef { + NodeRef { height: self.height, node: self.node, _marker: PhantomData } + } +} +impl<'a, K: 'a, V: 'a> NodeRef, K, V, marker::Leaf> { + /// Adds a key-value pair to the end of the node, and returns + /// the mutable reference of the inserted value. + pub fn push(&mut self, key: K, val: V) -> &mut V { + let len = self.len_mut(); + let idx = usize::from(*len); + assert!(idx < CAPACITY); + *len += 1; unsafe { - slice_insert(self.keys_mut(), 0, key); - slice_insert(self.vals_mut(), 0, val); - slice_insert( - slice::from_raw_parts_mut( - MaybeUninit::first_ptr_mut(&mut self.as_internal_mut().edges), - self.len() + 1, - ), - 0, - edge.node, - ); - - (*self.as_leaf_mut()).len += 1; - - self.correct_all_childrens_parent_links(); + self.key_area_mut(idx).write(key); + self.val_area_mut(idx).write(val) } } } -impl<'a, K, V> NodeRef, K, V, marker::LeafOrInternal> { - /// Removes a key/value pair from the end of this node. If this is an internal node, - /// also removes the edge that was to the right of that pair. - pub fn pop(&mut self) -> (K, V, Option>) { - assert!(self.len() > 0); - - let idx = self.len() - 1; +impl<'a, K: 'a, V: 'a> NodeRef, K, V, marker::Internal> { + /// Adds a key-value pair, and an edge to go to the right of that pair, + /// to the end of the node. + pub fn push(&mut self, key: K, val: V, edge: Root) { + assert!(edge.height == self.height - 1); + let len = self.len_mut(); + let idx = usize::from(*len); + assert!(idx < CAPACITY); + *len += 1; unsafe { - let key = ptr::read(self.keys().get_unchecked(idx)); - let val = ptr::read(self.vals().get_unchecked(idx)); - let edge = match self.reborrow_mut().force() { - ForceResult::Leaf(_) => None, - ForceResult::Internal(internal) => { - let edge = - ptr::read(internal.as_internal().edges.get_unchecked(idx + 1).as_ptr()); - let mut new_root = Root { node: edge, height: internal.height - 1 }; - (*new_root.as_mut().as_leaf_mut()).parent = ptr::null(); - Some(new_root) - } - }; - - (*self.as_leaf_mut()).len -= 1; - (key, val, edge) + self.key_area_mut(idx).write(key); + self.val_area_mut(idx).write(val); + self.edge_area_mut(idx + 1).write(edge.node); + Handle::new_edge(self.reborrow_mut(), idx + 1).correct_parent_link(); } } +} - /// Removes a key/value pair from the beginning of this node. If this is an internal node, - /// also removes the edge that was to the left of that pair. - pub fn pop_front(&mut self) -> (K, V, Option>) { - assert!(self.len() > 0); - - let old_len = self.len(); - - unsafe { - let key = slice_remove(self.keys_mut(), 0); - let val = slice_remove(self.vals_mut(), 0); - let edge = match self.reborrow_mut().force() { - ForceResult::Leaf(_) => None, - ForceResult::Internal(mut internal) => { - let edge = slice_remove( - slice::from_raw_parts_mut( - MaybeUninit::first_ptr_mut(&mut internal.as_internal_mut().edges), - old_len + 1, - ), - 0, - ); - - let mut new_root = Root { node: edge, height: internal.height - 1 }; - (*new_root.as_mut().as_leaf_mut()).parent = ptr::null(); - - for i in 0..old_len { - Handle::new_edge(internal.reborrow_mut(), i).correct_parent_link(); - } - - Some(new_root) - } - }; - - (*self.as_leaf_mut()).len -= 1; - - (key, val, edge) - } +impl NodeRef { + /// Removes any static information asserting that this node is a `Leaf` node. + pub fn forget_type(self) -> NodeRef { + NodeRef { height: self.height, node: self.node, _marker: PhantomData } } +} - fn into_kv_pointers_mut(mut self) -> (*mut K, *mut V) { - (self.keys_mut().as_mut_ptr(), self.vals_mut().as_mut_ptr()) +impl NodeRef { + /// Removes any static information asserting that this node is an `Internal` node. + pub fn forget_type(self) -> NodeRef { + NodeRef { height: self.height, node: self.node, _marker: PhantomData } } } @@ -681,26 +688,38 @@ impl NodeRef { ForceResult::Leaf(NodeRef { height: self.height, node: self.node, - root: self.root, _marker: PhantomData, }) } else { ForceResult::Internal(NodeRef { height: self.height, node: self.node, - root: self.root, _marker: PhantomData, }) } } } -/// A reference to a specific key/value pair or edge within a node. The `Node` parameter -/// must be a `NodeRef`, while the `Type` can either be `KV` (signifying a handle on a key/value +impl<'a, K, V> NodeRef, K, V, marker::LeafOrInternal> { + /// Unsafely asserts to the compiler the static information that this node is a `Leaf`. + unsafe fn cast_to_leaf_unchecked(self) -> NodeRef, K, V, marker::Leaf> { + debug_assert!(self.height == 0); + NodeRef { height: self.height, node: self.node, _marker: PhantomData } + } + + /// Unsafely asserts to the compiler the static information that this node is an `Internal`. + unsafe fn cast_to_internal_unchecked(self) -> NodeRef, K, V, marker::Internal> { + debug_assert!(self.height > 0); + NodeRef { height: self.height, node: self.node, _marker: PhantomData } + } +} + +/// A reference to a specific key-value pair or edge within a node. The `Node` parameter +/// must be a `NodeRef`, while the `Type` can either be `KV` (signifying a handle on a key-value /// pair) or `Edge` (signifying a handle on an edge). /// /// Note that even `Leaf` nodes can have `Edge` handles. Instead of representing a pointer to -/// a child node, these represent the spaces where child pointers would go between the key/value +/// a child node, these represent the spaces where child pointers would go between the key-value /// pairs. For example, in a node with length 2, there would be 3 possible edge locations - one /// to the left of the node, one between the two pairs, and one at the right of the node. pub struct Handle { @@ -719,14 +738,19 @@ impl Clone for Handle { } impl Handle { - /// Retrieves the node that contains the edge of key/value pair this handle points to. + /// Retrieves the node that contains the edge or key-value pair this handle points to. pub fn into_node(self) -> Node { self.node } + + /// Returns the position of this handle in the node. + pub fn idx(&self) -> usize { + self.idx + } } impl Handle, marker::KV> { - /// Creates a new handle to a key/value pair in `node`. + /// Creates a new handle to a key-value pair in `node`. /// Unsafe because the caller must ensure that `idx < node.len()`. pub unsafe fn new_kv(node: NodeRef, idx: usize) -> Self { debug_assert!(idx < node.len()); @@ -747,22 +771,15 @@ impl PartialEq for Handle, HandleType> { fn eq(&self, other: &Self) -> bool { - self.node.node == other.node.node && self.idx == other.idx - } -} - -impl PartialOrd - for Handle, HandleType> -{ - fn partial_cmp(&self, other: &Self) -> Option { - if self.node.node == other.node.node { Some(self.idx.cmp(&other.idx)) } else { None } + let Self { node, idx, _marker } = self; + node.eq(&other.node) && *idx == other.idx } } impl Handle, HandleType> { - /// Temporarily takes out another, immutable handle on the same location. + /// Temporarily takes out another immutable handle on the same location. pub fn reborrow(&self) -> Handle, K, V, NodeType>, HandleType> { // We can't use Handle::new_kv or Handle::new_edge because we don't know our type Handle { node: self.node.reborrow(), idx: self.idx, _marker: PhantomData } @@ -770,21 +787,16 @@ impl } impl<'a, K, V, NodeType, HandleType> Handle, K, V, NodeType>, HandleType> { - /// Temporarily takes out another, mutable handle on the same location. Beware, as - /// this method is very dangerous, doubly so since it may not immediately appear + /// Temporarily takes out another mutable handle on the same location. Beware, as + /// this method is very dangerous, doubly so since it might not immediately appear /// dangerous. /// - /// Because mutable pointers can roam anywhere around the tree and can even (through - /// `into_root_mut`) mess with the root of the tree, the result of `reborrow_mut` - /// can easily be used to make the original mutable pointer dangling, or, in the case - /// of a reborrowed handle, out of bounds. - // FIXME(@gereeter) consider adding yet another type parameter to `NodeRef` that restricts - // the use of `ascend` and `into_root_mut` on reborrowed pointers, preventing this unsafety. + /// For details, see `NodeRef::reborrow_mut`. pub unsafe fn reborrow_mut( &mut self, ) -> Handle, K, V, NodeType>, HandleType> { // We can't use Handle::new_kv or Handle::new_edge because we don't know our type - Handle { node: self.node.reborrow_mut(), idx: self.idx, _marker: PhantomData } + Handle { node: unsafe { self.node.reborrow_mut() }, idx: self.idx, _marker: PhantomData } } } @@ -814,534 +826,712 @@ impl Handle, mar } } -impl<'a, K, V> Handle, K, V, marker::Leaf>, marker::Edge> { - /// Inserts a new key/value pair between the key/value pairs to the right and left of +pub enum LeftOrRight { + Left(T), + Right(T), +} + +/// Given an edge index where we want to insert into a node filled to capacity, +/// computes a sensible KV index of a split point and where to perform the insertion. +/// The goal of the split point is for its key and value to end up in a parent node; +/// the keys, values and edges to the left of the split point become the left child; +/// the keys, values and edges to the right of the split point become the right child. +fn splitpoint(edge_idx: usize) -> (usize, LeftOrRight) { + debug_assert!(edge_idx <= CAPACITY); + // Rust issue #74834 tries to explain these symmetric rules. + match edge_idx { + 0..EDGE_IDX_LEFT_OF_CENTER => (KV_IDX_CENTER - 1, LeftOrRight::Left(edge_idx)), + EDGE_IDX_LEFT_OF_CENTER => (KV_IDX_CENTER, LeftOrRight::Left(edge_idx)), + EDGE_IDX_RIGHT_OF_CENTER => (KV_IDX_CENTER, LeftOrRight::Right(0)), + _ => (KV_IDX_CENTER + 1, LeftOrRight::Right(edge_idx - (KV_IDX_CENTER + 1 + 1))), + } +} + +impl<'a, K: 'a, V: 'a> Handle, K, V, marker::Leaf>, marker::Edge> { + /// Inserts a new key-value pair between the key-value pairs to the right and left of /// this edge. This method assumes that there is enough space in the node for the new /// pair to fit. /// /// The returned pointer points to the inserted value. fn insert_fit(&mut self, key: K, val: V) -> *mut V { - // Necessary for correctness, but in a private module debug_assert!(self.node.len() < CAPACITY); + let new_len = self.node.len() + 1; unsafe { - slice_insert(self.node.keys_mut(), self.idx, key); - slice_insert(self.node.vals_mut(), self.idx, val); - - (*self.node.as_leaf_mut()).len += 1; + slice_insert(self.node.key_area_mut(..new_len), self.idx, key); + slice_insert(self.node.val_area_mut(..new_len), self.idx, val); + *self.node.len_mut() = new_len as u16; - self.node.vals_mut().get_unchecked_mut(self.idx) + self.node.val_area_mut(self.idx).assume_init_mut() } } +} - /// Inserts a new key/value pair between the key/value pairs to the right and left of +impl<'a, K: 'a, V: 'a> Handle, K, V, marker::Leaf>, marker::Edge> { + /// Inserts a new key-value pair between the key-value pairs to the right and left of /// this edge. This method splits the node if there isn't enough room. /// /// The returned pointer points to the inserted value. - pub fn insert(mut self, key: K, val: V) -> (InsertResult<'a, K, V, marker::Leaf>, *mut V) { + fn insert( + mut self, + key: K, + val: V, + alloc: A, + ) -> (Option>, *mut V) { if self.node.len() < CAPACITY { - let ptr = self.insert_fit(key, val); - let kv = unsafe { Handle::new_kv(self.node, self.idx) }; - (InsertResult::Fit(kv), ptr) + let val_ptr = self.insert_fit(key, val); + (None, val_ptr) } else { - let middle = unsafe { Handle::new_kv(self.node, B) }; - let (mut left, k, v, mut right) = middle.split(); - let ptr = if self.idx <= B { - unsafe { Handle::new_edge(left.reborrow_mut(), self.idx).insert_fit(key, val) } - } else { - unsafe { - Handle::new_edge( - right.as_mut().cast_unchecked::(), - self.idx - (B + 1), - ) - .insert_fit(key, val) - } + let (middle_kv_idx, insertion) = splitpoint(self.idx); + let middle = unsafe { Handle::new_kv(self.node, middle_kv_idx) }; + let mut result = middle.split(alloc); + let mut insertion_edge = match insertion { + LeftOrRight::Left(insert_idx) => unsafe { + Handle::new_edge(result.left.reborrow_mut(), insert_idx) + }, + LeftOrRight::Right(insert_idx) => unsafe { + Handle::new_edge(result.right.borrow_mut(), insert_idx) + }, }; - (InsertResult::Split(left, k, v, right), ptr) + let val_ptr = insertion_edge.insert_fit(key, val); + (Some(result), val_ptr) } } } impl<'a, K, V> Handle, K, V, marker::Internal>, marker::Edge> { - /// Fixes the parent pointer and index in the child node below this edge. This is useful - /// when the ordering of edges has been changed, such as in the various `insert` methods. - fn correct_parent_link(mut self) { - let idx = self.idx as u16; - let ptr = self.node.as_internal_mut() as *mut _; + /// Fixes the parent pointer and index in the child node that this edge + /// links to. This is useful when the ordering of edges has been changed, + fn correct_parent_link(self) { + // Create backpointer without invalidating other references to the node. + let ptr = unsafe { NonNull::new_unchecked(NodeRef::as_internal_ptr(&self.node)) }; + let idx = self.idx; let mut child = self.descend(); - unsafe { - (*child.as_leaf_mut()).parent = ptr; - (*child.as_leaf_mut()).parent_idx.write(idx); - } - } - - /// Unsafely asserts to the compiler some static information about whether the underlying - /// node of this handle is a `Leaf`. - unsafe fn cast_unchecked( - &mut self, - ) -> Handle, K, V, NewType>, marker::Edge> { - Handle::new_edge(self.node.cast_unchecked(), self.idx) + child.set_parent_link(ptr, idx); } +} - /// Inserts a new key/value pair and an edge that will go to the right of that new pair - /// between this edge and the key/value pair to the right of this edge. This method assumes +impl<'a, K: 'a, V: 'a> Handle, K, V, marker::Internal>, marker::Edge> { + /// Inserts a new key-value pair and an edge that will go to the right of that new pair + /// between this edge and the key-value pair to the right of this edge. This method assumes /// that there is enough space in the node for the new pair to fit. fn insert_fit(&mut self, key: K, val: V, edge: Root) { - // Necessary for correctness, but in an internal module debug_assert!(self.node.len() < CAPACITY); debug_assert!(edge.height == self.node.height - 1); + let new_len = self.node.len() + 1; unsafe { - // This cast is a lie, but it allows us to reuse the key/value insertion logic. - self.cast_unchecked::().insert_fit(key, val); - - slice_insert( - slice::from_raw_parts_mut( - MaybeUninit::first_ptr_mut(&mut self.node.as_internal_mut().edges), - self.node.len(), - ), - self.idx + 1, - edge.node, - ); + slice_insert(self.node.key_area_mut(..new_len), self.idx, key); + slice_insert(self.node.val_area_mut(..new_len), self.idx, val); + slice_insert(self.node.edge_area_mut(..new_len + 1), self.idx + 1, edge.node); + *self.node.len_mut() = new_len as u16; - for i in (self.idx + 1)..(self.node.len() + 1) { - Handle::new_edge(self.node.reborrow_mut(), i).correct_parent_link(); - } + self.node.correct_childrens_parent_links(self.idx + 1..new_len + 1); } } - /// Inserts a new key/value pair and an edge that will go to the right of that new pair - /// between this edge and the key/value pair to the right of this edge. This method splits + /// Inserts a new key-value pair and an edge that will go to the right of that new pair + /// between this edge and the key-value pair to the right of this edge. This method splits /// the node if there isn't enough room. - pub fn insert( + fn insert( mut self, key: K, val: V, edge: Root, - ) -> InsertResult<'a, K, V, marker::Internal> { + alloc: A, + ) -> Option> { assert!(edge.height == self.node.height - 1); if self.node.len() < CAPACITY { self.insert_fit(key, val, edge); - let kv = unsafe { Handle::new_kv(self.node, self.idx) }; - InsertResult::Fit(kv) + None } else { - let middle = unsafe { Handle::new_kv(self.node, B) }; - let (mut left, k, v, mut right) = middle.split(); - if self.idx <= B { - unsafe { - Handle::new_edge(left.reborrow_mut(), self.idx).insert_fit(key, val, edge); - } - } else { - unsafe { - Handle::new_edge( - right.as_mut().cast_unchecked::(), - self.idx - (B + 1), - ) - .insert_fit(key, val, edge); + let (middle_kv_idx, insertion) = splitpoint(self.idx); + let middle = unsafe { Handle::new_kv(self.node, middle_kv_idx) }; + let mut result = middle.split(alloc); + let mut insertion_edge = match insertion { + LeftOrRight::Left(insert_idx) => unsafe { + Handle::new_edge(result.left.reborrow_mut(), insert_idx) + }, + LeftOrRight::Right(insert_idx) => unsafe { + Handle::new_edge(result.right.borrow_mut(), insert_idx) + }, + }; + insertion_edge.insert_fit(key, val, edge); + Some(result) + } + } +} + +impl<'a, K: 'a, V: 'a> Handle, K, V, marker::Leaf>, marker::Edge> { + /// Inserts a new key-value pair between the key-value pairs to the right and left of + /// this edge. This method splits the node if there isn't enough room, and tries to + /// insert the split off portion into the parent node recursively, until the root is reached. + /// + /// If the returned result is some `SplitResult`, the `left` field will be the root node. + /// The returned pointer points to the inserted value, which in the case of `SplitResult` + /// is in the `left` or `right` tree. + pub fn insert_recursing( + self, + key: K, + value: V, + alloc: A, + ) -> (Option>, *mut V) { + let (mut split, val_ptr) = match self.insert(key, value, alloc.clone()) { + (None, val_ptr) => return (None, val_ptr), + (Some(split), val_ptr) => (split.forget_node_type(), val_ptr), + }; + + loop { + split = match split.left.ascend() { + Ok(parent) => { + match parent.insert(split.kv.0, split.kv.1, split.right, alloc.clone()) { + None => return (None, val_ptr), + Some(split) => split.forget_node_type(), + } } - } - InsertResult::Split(left, k, v, right) + Err(root) => return (Some(SplitResult { left: root, ..split }), val_ptr), + }; } } } -impl Handle, marker::Edge> { +impl + Handle, marker::Edge> +{ /// Finds the node pointed to by this edge. /// + /// The method name assumes you picture trees with the root node on top. + /// /// `edge.descend().ascend().unwrap()` and `node.ascend().unwrap().descend()` should /// both, upon success, do nothing. pub fn descend(self) -> NodeRef { - NodeRef { - height: self.node.height - 1, - node: unsafe { - (&*self.node.as_internal().edges.get_unchecked(self.idx).as_ptr()).as_ptr() - }, - root: self.node.root, - _marker: PhantomData, + const { + assert!(BorrowType::TRAVERSAL_PERMIT); } + + // We need to use raw pointers to nodes because, if BorrowType is + // marker::ValMut, there might be outstanding mutable references to + // values that we must not invalidate. There's no worry accessing the + // height field because that value is copied. Beware that, once the + // node pointer is dereferenced, we access the edges array with a + // reference (Rust issue #73987) and invalidate any other references + // to or inside the array, should any be around. + let parent_ptr = NodeRef::as_internal_ptr(&self.node); + let node = unsafe { (*parent_ptr).edges.get_unchecked(self.idx).assume_init_read() }; + NodeRef { node, height: self.node.height - 1, _marker: PhantomData } } } impl<'a, K: 'a, V: 'a, NodeType> Handle, K, V, NodeType>, marker::KV> { pub fn into_kv(self) -> (&'a K, &'a V) { - unsafe { - let (keys, vals) = self.node.into_slices(); - (keys.get_unchecked(self.idx), vals.get_unchecked(self.idx)) - } + debug_assert!(self.idx < self.node.len()); + let leaf = self.node.into_leaf(); + let k = unsafe { leaf.keys.get_unchecked(self.idx).assume_init_ref() }; + let v = unsafe { leaf.vals.get_unchecked(self.idx).assume_init_ref() }; + (k, v) } } impl<'a, K: 'a, V: 'a, NodeType> Handle, K, V, NodeType>, marker::KV> { - pub fn into_kv_mut(self) -> (&'a mut K, &'a mut V) { - unsafe { - let (keys, vals) = self.node.into_slices_mut(); - (keys.get_unchecked_mut(self.idx), vals.get_unchecked_mut(self.idx)) - } + pub fn key_mut(&mut self) -> &mut K { + unsafe { self.node.key_area_mut(self.idx).assume_init_mut() } + } + + pub fn into_val_mut(self) -> &'a mut V { + debug_assert!(self.idx < self.node.len()); + let leaf = self.node.into_leaf_mut(); + unsafe { leaf.vals.get_unchecked_mut(self.idx).assume_init_mut() } + } +} + +impl<'a, K, V, NodeType> Handle, K, V, NodeType>, marker::KV> { + pub fn into_kv_valmut(self) -> (&'a K, &'a mut V) { + unsafe { self.node.into_key_val_mut_at(self.idx) } } } -impl<'a, K, V, NodeType> Handle, K, V, NodeType>, marker::KV> { +impl<'a, K: 'a, V: 'a, NodeType> Handle, K, V, NodeType>, marker::KV> { pub fn kv_mut(&mut self) -> (&mut K, &mut V) { + debug_assert!(self.idx < self.node.len()); + // We cannot call separate key and value methods, because calling the second one + // invalidates the reference returned by the first. unsafe { - let (keys, vals) = self.node.reborrow_mut().into_slices_mut(); - (keys.get_unchecked_mut(self.idx), vals.get_unchecked_mut(self.idx)) + let leaf = self.node.as_leaf_mut(); + let key = leaf.keys.get_unchecked_mut(self.idx).assume_init_mut(); + let val = leaf.vals.get_unchecked_mut(self.idx).assume_init_mut(); + (key, val) } } + + /// Replaces the key and value that the KV handle refers to. + pub fn replace_kv(&mut self, k: K, v: V) -> (K, V) { + let (key, val) = self.kv_mut(); + (mem::replace(key, k), mem::replace(val, v)) + } } -impl<'a, K, V> Handle, K, V, marker::Leaf>, marker::KV> { - /// Splits the underlying node into three parts: - /// - /// - The node is truncated to only contain the key/value pairs to the right of - /// this handle. - /// - The key and value pointed to by this handle and extracted. - /// - All the key/value pairs to the right of this handle are put into a newly - /// allocated node. - pub fn split(mut self) -> (NodeRef, K, V, marker::Leaf>, K, V, Root) { +impl Handle, marker::KV> { + /// Extracts the key and value that the KV handle refers to. + /// # Safety + /// The node that the handle refers to must not yet have been deallocated. + pub unsafe fn into_key_val(mut self) -> (K, V) { + debug_assert!(self.idx < self.node.len()); + let leaf = self.node.as_leaf_dying(); unsafe { - let mut new_node = Box::new(LeafNode::new()); + let key = leaf.keys.get_unchecked_mut(self.idx).assume_init_read(); + let val = leaf.vals.get_unchecked_mut(self.idx).assume_init_read(); + (key, val) + } + } - let k = ptr::read(self.node.keys().get_unchecked(self.idx)); - let v = ptr::read(self.node.vals().get_unchecked(self.idx)); + /// Drops the key and value that the KV handle refers to. + /// # Safety + /// The node that the handle refers to must not yet have been deallocated. + #[inline] + pub unsafe fn drop_key_val(mut self) { + debug_assert!(self.idx < self.node.len()); + let leaf = self.node.as_leaf_dying(); + unsafe { + leaf.keys.get_unchecked_mut(self.idx).assume_init_drop(); + leaf.vals.get_unchecked_mut(self.idx).assume_init_drop(); + } + } +} - let new_len = self.node.len() - self.idx - 1; +impl<'a, K: 'a, V: 'a, NodeType> Handle, K, V, NodeType>, marker::KV> { + /// Helps implementations of `split` for a particular `NodeType`, + /// by taking care of leaf data. + fn split_leaf_data(&mut self, new_node: &mut LeafNode) -> (K, V) { + debug_assert!(self.idx < self.node.len()); + let old_len = self.node.len(); + let new_len = old_len - self.idx - 1; + new_node.len = new_len as u16; + unsafe { + let k = self.node.key_area_mut(self.idx).assume_init_read(); + let v = self.node.val_area_mut(self.idx).assume_init_read(); - ptr::copy_nonoverlapping( - self.node.keys().as_ptr().add(self.idx + 1), - new_node.keys.as_mut_ptr() as *mut K, - new_len, + move_to_slice( + self.node.key_area_mut(self.idx + 1..old_len), + &mut new_node.keys[..new_len], ); - ptr::copy_nonoverlapping( - self.node.vals().as_ptr().add(self.idx + 1), - new_node.vals.as_mut_ptr() as *mut V, - new_len, + move_to_slice( + self.node.val_area_mut(self.idx + 1..old_len), + &mut new_node.vals[..new_len], ); - (*self.node.as_leaf_mut()).len = self.idx as u16; - new_node.len = new_len as u16; - - (self.node, k, v, Root { node: BoxedNode::from_leaf(new_node), height: 0 }) + *self.node.len_mut() = self.idx as u16; + (k, v) } } +} + +impl<'a, K: 'a, V: 'a> Handle, K, V, marker::Leaf>, marker::KV> { + /// Splits the underlying node into three parts: + /// + /// - The node is truncated to only contain the key-value pairs to the left of + /// this handle. + /// - The key and value pointed to by this handle are extracted. + /// - All the key-value pairs to the right of this handle are put into a newly + /// allocated node. + pub fn split(mut self, alloc: A) -> SplitResult<'a, K, V, marker::Leaf> { + let mut new_node = LeafNode::new(alloc); + + let kv = self.split_leaf_data(&mut new_node); + + let right = NodeRef::from_new_leaf(new_node); + SplitResult { left: self.node, kv, right } + } - /// Removes the key/value pair pointed to by this handle and returns it, along with the edge - /// between the now adjacent key/value pairs (if any) to the left and right of this handle. + /// Removes the key-value pair pointed to by this handle and returns it, along with the edge + /// that the key-value pair collapsed into. pub fn remove( mut self, - ) -> (Handle, K, V, marker::Leaf>, marker::Edge>, K, V) { + ) -> ((K, V), Handle, K, V, marker::Leaf>, marker::Edge>) { + let old_len = self.node.len(); unsafe { - let k = slice_remove(self.node.keys_mut(), self.idx); - let v = slice_remove(self.node.vals_mut(), self.idx); - (*self.node.as_leaf_mut()).len -= 1; - (self.left_edge(), k, v) + let k = slice_remove(self.node.key_area_mut(..old_len), self.idx); + let v = slice_remove(self.node.val_area_mut(..old_len), self.idx); + *self.node.len_mut() = (old_len - 1) as u16; + ((k, v), self.left_edge()) } } } -impl<'a, K, V> Handle, K, V, marker::Internal>, marker::KV> { +impl<'a, K: 'a, V: 'a> Handle, K, V, marker::Internal>, marker::KV> { /// Splits the underlying node into three parts: /// - /// - The node is truncated to only contain the edges and key/value pairs to the - /// right of this handle. - /// - The key and value pointed to by this handle and extracted. - /// - All the edges and key/value pairs to the right of this handle are put into + /// - The node is truncated to only contain the edges and key-value pairs to the + /// left of this handle. + /// - The key and value pointed to by this handle are extracted. + /// - All the edges and key-value pairs to the right of this handle are put into /// a newly allocated node. - pub fn split(mut self) -> (NodeRef, K, V, marker::Internal>, K, V, Root) { + pub fn split( + mut self, + alloc: A, + ) -> SplitResult<'a, K, V, marker::Internal> { + let old_len = self.node.len(); unsafe { - let mut new_node = Box::new(InternalNode::new()); - - let k = ptr::read(self.node.keys().get_unchecked(self.idx)); - let v = ptr::read(self.node.vals().get_unchecked(self.idx)); + let mut new_node = InternalNode::new(alloc); + let kv = self.split_leaf_data(&mut new_node.data); + let new_len = usize::from(new_node.data.len); + move_to_slice( + self.node.edge_area_mut(self.idx + 1..old_len + 1), + &mut new_node.edges[..new_len + 1], + ); let height = self.node.height; - let new_len = self.node.len() - self.idx - 1; + let right = NodeRef::from_new_internal(new_node, height); - ptr::copy_nonoverlapping( - self.node.keys().as_ptr().add(self.idx + 1), - new_node.data.keys.as_mut_ptr() as *mut K, - new_len, - ); - ptr::copy_nonoverlapping( - self.node.vals().as_ptr().add(self.idx + 1), - new_node.data.vals.as_mut_ptr() as *mut V, - new_len, - ); - ptr::copy_nonoverlapping( - self.node.as_internal().edges.as_ptr().add(self.idx + 1), - new_node.edges.as_mut_ptr(), - new_len + 1, - ); - - (*self.node.as_leaf_mut()).len = self.idx as u16; - new_node.data.len = new_len as u16; + SplitResult { left: self.node, kv, right } + } + } +} - let mut new_root = Root { node: BoxedNode::from_internal(new_node), height }; +/// Represents a session for evaluating and performing a balancing operation +/// around an internal key-value pair. +pub struct BalancingContext<'a, K, V> { + parent: Handle, K, V, marker::Internal>, marker::KV>, + left_child: NodeRef, K, V, marker::LeafOrInternal>, + right_child: NodeRef, K, V, marker::LeafOrInternal>, +} - for i in 0..(new_len + 1) { - Handle::new_edge(new_root.as_mut().cast_unchecked(), i).correct_parent_link(); - } +impl<'a, K, V> Handle, K, V, marker::Internal>, marker::KV> { + pub fn consider_for_balancing(self) -> BalancingContext<'a, K, V> { + let self1 = unsafe { ptr::read(&self) }; + let self2 = unsafe { ptr::read(&self) }; + BalancingContext { + parent: self, + left_child: self1.left_edge().descend(), + right_child: self2.right_edge().descend(), + } + } +} - (self.node, k, v, new_root) +impl<'a, K, V> NodeRef, K, V, marker::LeafOrInternal> { + /// Chooses a balancing context involving the node as a child, thus between + /// the KV immediately to the left or to the right in the parent node. + /// Returns an `Err` if there is no parent. + /// Panics if the parent is empty. + /// + /// Prefers the left side, to be optimal if the given node is somehow + /// underfull, meaning here only that it has fewer elements than its left + /// sibling and than its right sibling, if they exist. In that case, + /// merging with the left sibling is faster, since we only need to move + /// the node's N elements, instead of shifting them to the right and moving + /// more than N elements in front. Stealing from the left sibling is also + /// typically faster, since we only need to shift the node's N elements to + /// the right, instead of shifting at least N of the sibling's elements to + /// the left. + pub fn choose_parent_kv(self) -> Result>, Self> { + match unsafe { ptr::read(&self) }.ascend() { + Ok(parent_edge) => match parent_edge.left_kv() { + Ok(left_parent_kv) => Ok(LeftOrRight::Left(BalancingContext { + parent: unsafe { ptr::read(&left_parent_kv) }, + left_child: left_parent_kv.left_edge().descend(), + right_child: self, + })), + Err(parent_edge) => match parent_edge.right_kv() { + Ok(right_parent_kv) => Ok(LeftOrRight::Right(BalancingContext { + parent: unsafe { ptr::read(&right_parent_kv) }, + left_child: self, + right_child: right_parent_kv.right_edge().descend(), + })), + Err(_) => unreachable!("empty internal node"), + }, + }, + Err(root) => Err(root), } } +} - /// Returns `true` if it is valid to call `.merge()`, i.e., whether there is enough room in - /// a node to hold the combination of the nodes to the left and right of this handle along - /// with the key/value pair at this handle. +impl<'a, K, V> BalancingContext<'a, K, V> { + pub fn left_child_len(&self) -> usize { + self.left_child.len() + } + + pub fn right_child_len(&self) -> usize { + self.right_child.len() + } + + pub fn into_left_child(self) -> NodeRef, K, V, marker::LeafOrInternal> { + self.left_child + } + + pub fn into_right_child(self) -> NodeRef, K, V, marker::LeafOrInternal> { + self.right_child + } + + /// Returns whether merging is possible, i.e., whether there is enough room + /// in a node to combine the central KV with both adjacent child nodes. pub fn can_merge(&self) -> bool { - (self.reborrow().left_edge().descend().len() - + self.reborrow().right_edge().descend().len() - + 1) - <= CAPACITY + self.left_child.len() + 1 + self.right_child.len() <= CAPACITY } +} - /// Combines the node immediately to the left of this handle, the key/value pair pointed - /// to by this handle, and the node immediately to the right of this handle into one new - /// child of the underlying node, returning an edge referencing that new child. - /// - /// Assumes that this edge `.can_merge()`. - pub fn merge( - mut self, - ) -> Handle, K, V, marker::Internal>, marker::Edge> { - let self1 = unsafe { ptr::read(&self) }; - let self2 = unsafe { ptr::read(&self) }; - let mut left_node = self1.left_edge().descend(); - let left_len = left_node.len(); - let mut right_node = self2.right_edge().descend(); +impl<'a, K: 'a, V: 'a> BalancingContext<'a, K, V> { + /// Performs a merge and lets a closure decide what to return. + fn do_merge< + F: FnOnce( + NodeRef, K, V, marker::Internal>, + NodeRef, K, V, marker::LeafOrInternal>, + ) -> R, + R, + A: Allocator, + >( + self, + result: F, + alloc: A, + ) -> R { + let Handle { node: mut parent_node, idx: parent_idx, _marker } = self.parent; + let old_parent_len = parent_node.len(); + let mut left_node = self.left_child; + let old_left_len = left_node.len(); + let mut right_node = self.right_child; let right_len = right_node.len(); + let new_left_len = old_left_len + 1 + right_len; - // necessary for correctness, but in a private module - assert!(left_len + right_len < CAPACITY); + assert!(new_left_len <= CAPACITY); unsafe { - ptr::write( - left_node.keys_mut().get_unchecked_mut(left_len), - slice_remove(self.node.keys_mut(), self.idx), - ); - ptr::copy_nonoverlapping( - right_node.keys().as_ptr(), - left_node.keys_mut().as_mut_ptr().add(left_len + 1), - right_len, - ); - ptr::write( - left_node.vals_mut().get_unchecked_mut(left_len), - slice_remove(self.node.vals_mut(), self.idx), + *left_node.len_mut() = new_left_len as u16; + + let parent_key = slice_remove(parent_node.key_area_mut(..old_parent_len), parent_idx); + left_node.key_area_mut(old_left_len).write(parent_key); + move_to_slice( + right_node.key_area_mut(..right_len), + left_node.key_area_mut(old_left_len + 1..new_left_len), ); - ptr::copy_nonoverlapping( - right_node.vals().as_ptr(), - left_node.vals_mut().as_mut_ptr().add(left_len + 1), - right_len, + + let parent_val = slice_remove(parent_node.val_area_mut(..old_parent_len), parent_idx); + left_node.val_area_mut(old_left_len).write(parent_val); + move_to_slice( + right_node.val_area_mut(..right_len), + left_node.val_area_mut(old_left_len + 1..new_left_len), ); - slice_remove(&mut self.node.as_internal_mut().edges, self.idx + 1); - for i in self.idx + 1..self.node.len() { - Handle::new_edge(self.node.reborrow_mut(), i).correct_parent_link(); - } - (*self.node.as_leaf_mut()).len -= 1; - - (*left_node.as_leaf_mut()).len += right_len as u16 + 1; - - if self.node.height > 1 { - ptr::copy_nonoverlapping( - right_node.cast_unchecked().as_internal().edges.as_ptr(), - left_node - .cast_unchecked() - .as_internal_mut() - .edges - .as_mut_ptr() - .add(left_len + 1), - right_len + 1, + slice_remove(&mut parent_node.edge_area_mut(..old_parent_len + 1), parent_idx + 1); + parent_node.correct_childrens_parent_links(parent_idx + 1..old_parent_len); + *parent_node.len_mut() -= 1; + + if parent_node.height > 1 { + // SAFETY: the height of the nodes being merged is one below the height + // of the node of this edge, thus above zero, so they are internal. + let mut left_node = left_node.reborrow_mut().cast_to_internal_unchecked(); + let mut right_node = right_node.cast_to_internal_unchecked(); + move_to_slice( + right_node.edge_area_mut(..right_len + 1), + left_node.edge_area_mut(old_left_len + 1..new_left_len + 1), ); - for i in left_len + 1..left_len + right_len + 2 { - Handle::new_edge(left_node.cast_unchecked().reborrow_mut(), i) - .correct_parent_link(); - } + left_node.correct_childrens_parent_links(old_left_len + 1..new_left_len + 1); - Global.dealloc(right_node.node.cast(), Layout::new::>()); + alloc.deallocate(right_node.node.cast(), Layout::new::>()); } else { - Global.dealloc(right_node.node.cast(), Layout::new::>()); + alloc.deallocate(right_node.node.cast(), Layout::new::>()); } - - Handle::new_edge(self.node, self.idx) } + result(parent_node, left_node) } - /// This removes a key/value pair from the left child and places it in the key/value storage - /// pointed to by this handle while pushing the old key/value pair of this handle into the right - /// child. - pub fn steal_left(&mut self) { - unsafe { - let (k, v, edge) = self.reborrow_mut().left_edge().descend().pop(); - - let k = mem::replace(self.reborrow_mut().into_kv_mut().0, k); - let v = mem::replace(self.reborrow_mut().into_kv_mut().1, v); + /// Merges the parent's key-value pair and both adjacent child nodes into + /// the left child node and returns the shrunk parent node. + /// + /// Panics unless we `.can_merge()`. + pub fn merge_tracking_parent( + self, + alloc: A, + ) -> NodeRef, K, V, marker::Internal> { + self.do_merge(|parent, _child| parent, alloc) + } - match self.reborrow_mut().right_edge().descend().force() { - ForceResult::Leaf(mut leaf) => leaf.push_front(k, v), - ForceResult::Internal(mut internal) => internal.push_front(k, v, edge.unwrap()), - } - } + /// Merges the parent's key-value pair and both adjacent child nodes into + /// the left child node and returns that child node. + /// + /// Panics unless we `.can_merge()`. + pub fn merge_tracking_child( + self, + alloc: A, + ) -> NodeRef, K, V, marker::LeafOrInternal> { + self.do_merge(|_parent, child| child, alloc) } - /// This removes a key/value pair from the right child and places it in the key/value storage - /// pointed to by this handle while pushing the old key/value pair of this handle into the left - /// child. - pub fn steal_right(&mut self) { - unsafe { - let (k, v, edge) = self.reborrow_mut().right_edge().descend().pop_front(); + /// Merges the parent's key-value pair and both adjacent child nodes into + /// the left child node and returns the edge handle in that child node + /// where the tracked child edge ended up, + /// + /// Panics unless we `.can_merge()`. + pub fn merge_tracking_child_edge( + self, + track_edge_idx: LeftOrRight, + alloc: A, + ) -> Handle, K, V, marker::LeafOrInternal>, marker::Edge> { + let old_left_len = self.left_child.len(); + let right_len = self.right_child.len(); + assert!(match track_edge_idx { + LeftOrRight::Left(idx) => idx <= old_left_len, + LeftOrRight::Right(idx) => idx <= right_len, + }); + let child = self.merge_tracking_child(alloc); + let new_idx = match track_edge_idx { + LeftOrRight::Left(idx) => idx, + LeftOrRight::Right(idx) => old_left_len + 1 + idx, + }; + unsafe { Handle::new_edge(child, new_idx) } + } - let k = mem::replace(self.reborrow_mut().into_kv_mut().0, k); - let v = mem::replace(self.reborrow_mut().into_kv_mut().1, v); + /// Removes a key-value pair from the left child and places it in the key-value storage + /// of the parent, while pushing the old parent key-value pair into the right child. + /// Returns a handle to the edge in the right child corresponding to where the original + /// edge specified by `track_right_edge_idx` ended up. + pub fn steal_left( + mut self, + track_right_edge_idx: usize, + ) -> Handle, K, V, marker::LeafOrInternal>, marker::Edge> { + self.bulk_steal_left(1); + unsafe { Handle::new_edge(self.right_child, 1 + track_right_edge_idx) } + } - match self.reborrow_mut().left_edge().descend().force() { - ForceResult::Leaf(mut leaf) => leaf.push(k, v), - ForceResult::Internal(mut internal) => internal.push(k, v, edge.unwrap()), - } - } + /// Removes a key-value pair from the right child and places it in the key-value storage + /// of the parent, while pushing the old parent key-value pair onto the left child. + /// Returns a handle to the edge in the left child specified by `track_left_edge_idx`, + /// which didn't move. + pub fn steal_right( + mut self, + track_left_edge_idx: usize, + ) -> Handle, K, V, marker::LeafOrInternal>, marker::Edge> { + self.bulk_steal_right(1); + unsafe { Handle::new_edge(self.left_child, track_left_edge_idx) } } /// This does stealing similar to `steal_left` but steals multiple elements at once. pub fn bulk_steal_left(&mut self, count: usize) { + assert!(count > 0); unsafe { - let mut left_node = ptr::read(self).left_edge().descend(); - let left_len = left_node.len(); - let mut right_node = ptr::read(self).right_edge().descend(); - let right_len = right_node.len(); + let left_node = &mut self.left_child; + let old_left_len = left_node.len(); + let right_node = &mut self.right_child; + let old_right_len = right_node.len(); // Make sure that we may steal safely. - assert!(right_len + count <= CAPACITY); - assert!(left_len >= count); + assert!(old_right_len + count <= CAPACITY); + assert!(old_left_len >= count); - let new_left_len = left_len - count; + let new_left_len = old_left_len - count; + let new_right_len = old_right_len + count; + *left_node.len_mut() = new_left_len as u16; + *right_node.len_mut() = new_right_len as u16; - // Move data. + // Move leaf data. { - let left_kv = left_node.reborrow_mut().into_kv_pointers_mut(); - let right_kv = right_node.reborrow_mut().into_kv_pointers_mut(); - let parent_kv = { - let kv = self.reborrow_mut().into_kv_mut(); - (kv.0 as *mut K, kv.1 as *mut V) - }; - // Make room for stolen elements in the right child. - ptr::copy(right_kv.0, right_kv.0.add(count), right_len); - ptr::copy(right_kv.1, right_kv.1.add(count), right_len); + slice_shr(right_node.key_area_mut(..new_right_len), count); + slice_shr(right_node.val_area_mut(..new_right_len), count); // Move elements from the left child to the right one. - move_kv(left_kv, new_left_len + 1, right_kv, 0, count - 1); - - // Move parent's key/value pair to the right child. - move_kv(parent_kv, 0, right_kv, count - 1, 1); + move_to_slice( + left_node.key_area_mut(new_left_len + 1..old_left_len), + right_node.key_area_mut(..count - 1), + ); + move_to_slice( + left_node.val_area_mut(new_left_len + 1..old_left_len), + right_node.val_area_mut(..count - 1), + ); // Move the left-most stolen pair to the parent. - move_kv(left_kv, new_left_len, parent_kv, 0, 1); - } + let k = left_node.key_area_mut(new_left_len).assume_init_read(); + let v = left_node.val_area_mut(new_left_len).assume_init_read(); + let (k, v) = self.parent.replace_kv(k, v); - (*left_node.reborrow_mut().as_leaf_mut()).len -= count as u16; - (*right_node.reborrow_mut().as_leaf_mut()).len += count as u16; + // Move parent's key-value pair to the right child. + right_node.key_area_mut(count - 1).write(k); + right_node.val_area_mut(count - 1).write(v); + } - match (left_node.force(), right_node.force()) { - (ForceResult::Internal(left), ForceResult::Internal(mut right)) => { + match (left_node.reborrow_mut().force(), right_node.reborrow_mut().force()) { + (ForceResult::Internal(mut left), ForceResult::Internal(mut right)) => { // Make room for stolen edges. - let right_edges = right.reborrow_mut().as_internal_mut().edges.as_mut_ptr(); - ptr::copy(right_edges, right_edges.add(count), right_len + 1); - right.correct_childrens_parent_links(count, count + right_len + 1); + slice_shr(right.edge_area_mut(..new_right_len + 1), count); - move_edges(left, new_left_len + 1, right, 0, count); + // Steal edges. + move_to_slice( + left.edge_area_mut(new_left_len + 1..old_left_len + 1), + right.edge_area_mut(..count), + ); + + right.correct_childrens_parent_links(0..new_right_len + 1); } (ForceResult::Leaf(_), ForceResult::Leaf(_)) => {} - _ => { - unreachable!(); - } + _ => unreachable!(), } } } /// The symmetric clone of `bulk_steal_left`. pub fn bulk_steal_right(&mut self, count: usize) { + assert!(count > 0); unsafe { - let mut left_node = ptr::read(self).left_edge().descend(); - let left_len = left_node.len(); - let mut right_node = ptr::read(self).right_edge().descend(); - let right_len = right_node.len(); + let left_node = &mut self.left_child; + let old_left_len = left_node.len(); + let right_node = &mut self.right_child; + let old_right_len = right_node.len(); // Make sure that we may steal safely. - assert!(left_len + count <= CAPACITY); - assert!(right_len >= count); + assert!(old_left_len + count <= CAPACITY); + assert!(old_right_len >= count); - let new_right_len = right_len - count; + let new_left_len = old_left_len + count; + let new_right_len = old_right_len - count; + *left_node.len_mut() = new_left_len as u16; + *right_node.len_mut() = new_right_len as u16; - // Move data. + // Move leaf data. { - let left_kv = left_node.reborrow_mut().into_kv_pointers_mut(); - let right_kv = right_node.reborrow_mut().into_kv_pointers_mut(); - let parent_kv = { - let kv = self.reborrow_mut().into_kv_mut(); - (kv.0 as *mut K, kv.1 as *mut V) - }; + // Move the right-most stolen pair to the parent. + let k = right_node.key_area_mut(count - 1).assume_init_read(); + let v = right_node.val_area_mut(count - 1).assume_init_read(); + let (k, v) = self.parent.replace_kv(k, v); - // Move parent's key/value pair to the left child. - move_kv(parent_kv, 0, left_kv, left_len, 1); + // Move parent's key-value pair to the left child. + left_node.key_area_mut(old_left_len).write(k); + left_node.val_area_mut(old_left_len).write(v); // Move elements from the right child to the left one. - move_kv(right_kv, 0, left_kv, left_len + 1, count - 1); - - // Move the right-most stolen pair to the parent. - move_kv(right_kv, count - 1, parent_kv, 0, 1); + move_to_slice( + right_node.key_area_mut(..count - 1), + left_node.key_area_mut(old_left_len + 1..new_left_len), + ); + move_to_slice( + right_node.val_area_mut(..count - 1), + left_node.val_area_mut(old_left_len + 1..new_left_len), + ); - // Fix right indexing - ptr::copy(right_kv.0.add(count), right_kv.0, new_right_len); - ptr::copy(right_kv.1.add(count), right_kv.1, new_right_len); + // Fill gap where stolen elements used to be. + slice_shl(right_node.key_area_mut(..old_right_len), count); + slice_shl(right_node.val_area_mut(..old_right_len), count); } - (*left_node.reborrow_mut().as_leaf_mut()).len += count as u16; - (*right_node.reborrow_mut().as_leaf_mut()).len -= count as u16; + match (left_node.reborrow_mut().force(), right_node.reborrow_mut().force()) { + (ForceResult::Internal(mut left), ForceResult::Internal(mut right)) => { + // Steal edges. + move_to_slice( + right.edge_area_mut(..count), + left.edge_area_mut(old_left_len + 1..new_left_len + 1), + ); - match (left_node.force(), right_node.force()) { - (ForceResult::Internal(left), ForceResult::Internal(mut right)) => { - move_edges(right.reborrow_mut(), 0, left, left_len + 1, count); + // Fill gap where stolen edges used to be. + slice_shl(right.edge_area_mut(..old_right_len + 1), count); - // Fix right indexing. - let right_edges = right.reborrow_mut().as_internal_mut().edges.as_mut_ptr(); - ptr::copy(right_edges.add(count), right_edges, new_right_len + 1); - right.correct_childrens_parent_links(0, new_right_len + 1); + left.correct_childrens_parent_links(old_left_len + 1..new_left_len + 1); + right.correct_childrens_parent_links(0..new_right_len + 1); } (ForceResult::Leaf(_), ForceResult::Leaf(_)) => {} - _ => { - unreachable!(); - } + _ => unreachable!(), } } } } -unsafe fn move_kv( - source: (*mut K, *mut V), - source_offset: usize, - dest: (*mut K, *mut V), - dest_offset: usize, - count: usize, -) { - ptr::copy_nonoverlapping(source.0.add(source_offset), dest.0.add(dest_offset), count); - ptr::copy_nonoverlapping(source.1.add(source_offset), dest.1.add(dest_offset), count); -} - -// Source and destination must have the same height. -unsafe fn move_edges( - mut source: NodeRef, K, V, marker::Internal>, - source_offset: usize, - mut dest: NodeRef, K, V, marker::Internal>, - dest_offset: usize, - count: usize, -) { - let source_ptr = source.as_internal_mut().edges.as_mut_ptr(); - let dest_ptr = dest.as_internal_mut().edges.as_mut_ptr(); - ptr::copy_nonoverlapping(source_ptr.add(source_offset), dest_ptr.add(dest_offset), count); - dest.correct_childrens_parent_links(dest_offset, dest_offset + count); -} - impl Handle, marker::Edge> { pub fn forget_node_type( self, @@ -1366,15 +1556,13 @@ impl Handle, marker::K } } -impl - Handle, HandleType> -{ +impl Handle, Type> { /// Checks whether the underlying node is an `Internal` node or a `Leaf` node. pub fn force( self, ) -> ForceResult< - Handle, HandleType>, - Handle, HandleType>, + Handle, Type>, + Handle, Type>, > { match self.node.force() { ForceResult::Leaf(node) => { @@ -1387,6 +1575,16 @@ impl } } +impl<'a, K, V, Type> Handle, K, V, marker::LeafOrInternal>, Type> { + /// Unsafely asserts to the compiler the static information that the handle's node is a `Leaf`. + pub unsafe fn cast_to_leaf_unchecked( + self, + ) -> Handle, K, V, marker::Leaf>, Type> { + let node = unsafe { self.node.cast_to_leaf_unchecked() }; + Handle { node, idx: self.idx, _marker: PhantomData } + } +} + impl<'a, K, V> Handle, K, V, marker::LeafOrInternal>, marker::Edge> { /// Move the suffix after `self` from one node to another one. `right` must be empty. /// The first edge of `right` remains unchanged. @@ -1395,32 +1593,38 @@ impl<'a, K, V> Handle, K, V, marker::LeafOrInternal>, ma right: &mut NodeRef, K, V, marker::LeafOrInternal>, ) { unsafe { - let left_new_len = self.idx; + let new_left_len = self.idx; let mut left_node = self.reborrow_mut().into_node(); + let old_left_len = left_node.len(); - let right_new_len = left_node.len() - left_new_len; + let new_right_len = old_left_len - new_left_len; let mut right_node = right.reborrow_mut(); assert!(right_node.len() == 0); assert!(left_node.height == right_node.height); - if right_new_len > 0 { - let left_kv = left_node.reborrow_mut().into_kv_pointers_mut(); - let right_kv = right_node.reborrow_mut().into_kv_pointers_mut(); - - move_kv(left_kv, left_new_len, right_kv, 0, right_new_len); - - (*left_node.reborrow_mut().as_leaf_mut()).len = left_new_len as u16; - (*right_node.reborrow_mut().as_leaf_mut()).len = right_new_len as u16; + if new_right_len > 0 { + *left_node.len_mut() = new_left_len as u16; + *right_node.len_mut() = new_right_len as u16; + move_to_slice( + left_node.key_area_mut(new_left_len..old_left_len), + right_node.key_area_mut(..new_right_len), + ); + move_to_slice( + left_node.val_area_mut(new_left_len..old_left_len), + right_node.val_area_mut(..new_right_len), + ); match (left_node.force(), right_node.force()) { - (ForceResult::Internal(left), ForceResult::Internal(right)) => { - move_edges(left, left_new_len + 1, right, 1, right_new_len); + (ForceResult::Internal(mut left), ForceResult::Internal(mut right)) => { + move_to_slice( + left.edge_area_mut(new_left_len + 1..old_left_len + 1), + right.edge_area_mut(1..new_right_len + 1), + ); + right.correct_childrens_parent_links(1..new_right_len + 1); } (ForceResult::Leaf(_), ForceResult::Leaf(_)) => {} - _ => { - unreachable!(); - } + _ => unreachable!(), } } } @@ -1432,9 +1636,26 @@ pub enum ForceResult { Internal(Internal), } -pub enum InsertResult<'a, K, V, Type> { - Fit(Handle, K, V, Type>, marker::KV>), - Split(NodeRef, K, V, Type>, K, V, Root), +/// Result of insertion, when a node needed to expand beyond its capacity. +pub struct SplitResult<'a, K, V, NodeType> { + // Altered node in existing tree with elements and edges that belong to the left of `kv`. + pub left: NodeRef, K, V, NodeType>, + // Some key and value that existed before and were split off, to be inserted elsewhere. + pub kv: (K, V), + // Owned, unattached, new node with elements and edges that belong to the right of `kv`. + pub right: NodeRef, +} + +impl<'a, K, V> SplitResult<'a, K, V, marker::Leaf> { + pub fn forget_node_type(self) -> SplitResult<'a, K, V, marker::LeafOrInternal> { + SplitResult { left: self.left.forget_type(), kv: self.kv, right: self.right.forget_type() } + } +} + +impl<'a, K, V> SplitResult<'a, K, V, marker::Internal> { + pub fn forget_node_type(self) -> SplitResult<'a, K, V, marker::LeafOrInternal> { + SplitResult { left: self.left.forget_type(), kv: self.kv, right: self.right.forget_type() } + } } pub mod marker { @@ -1445,20 +1666,96 @@ pub mod marker { pub enum LeafOrInternal {} pub enum Owned {} + pub enum Dying {} pub struct Immut<'a>(PhantomData<&'a ()>); pub struct Mut<'a>(PhantomData<&'a mut ()>); + pub struct ValMut<'a>(PhantomData<&'a mut ()>); + + pub trait BorrowType { + /// If node references of this borrow type allow traversing to other + /// nodes in the tree, this constant is set to `true`. It can be used + /// for a compile-time assertion. + const TRAVERSAL_PERMIT: bool = true; + } + impl BorrowType for Owned { + /// Reject traversal, because it isn't needed. Instead traversal + /// happens using the result of `borrow_mut`. + /// By disabling traversal, and only creating new references to roots, + /// we know that every reference of the `Owned` type is to a root node. + const TRAVERSAL_PERMIT: bool = false; + } + impl BorrowType for Dying {} + impl<'a> BorrowType for Immut<'a> {} + impl<'a> BorrowType for Mut<'a> {} + impl<'a> BorrowType for ValMut<'a> {} pub enum KV {} pub enum Edge {} } -unsafe fn slice_insert(slice: &mut [T], idx: usize, val: T) { - ptr::copy(slice.as_ptr().add(idx), slice.as_mut_ptr().add(idx + 1), slice.len() - idx); - ptr::write(slice.get_unchecked_mut(idx), val); +/// Inserts a value into a slice of initialized elements followed by one uninitialized element. +/// +/// # Safety +/// The slice has more than `idx` elements. +unsafe fn slice_insert(slice: &mut [MaybeUninit], idx: usize, val: T) { + unsafe { + let len = slice.len(); + debug_assert!(len > idx); + let slice_ptr = slice.as_mut_ptr(); + if len > idx + 1 { + ptr::copy(slice_ptr.add(idx), slice_ptr.add(idx + 1), len - idx - 1); + } + (*slice_ptr.add(idx)).write(val); + } +} + +/// Removes and returns a value from a slice of all initialized elements, leaving behind one +/// trailing uninitialized element. +/// +/// # Safety +/// The slice has more than `idx` elements. +unsafe fn slice_remove(slice: &mut [MaybeUninit], idx: usize) -> T { + unsafe { + let len = slice.len(); + debug_assert!(idx < len); + let slice_ptr = slice.as_mut_ptr(); + let ret = (*slice_ptr.add(idx)).assume_init_read(); + ptr::copy(slice_ptr.add(idx + 1), slice_ptr.add(idx), len - idx - 1); + ret + } } -unsafe fn slice_remove(slice: &mut [T], idx: usize) -> T { - let ret = ptr::read(slice.get_unchecked(idx)); - ptr::copy(slice.as_ptr().add(idx + 1), slice.as_mut_ptr().add(idx), slice.len() - idx - 1); - ret +/// Shifts the elements in a slice `distance` positions to the left. +/// +/// # Safety +/// The slice has at least `distance` elements. +unsafe fn slice_shl(slice: &mut [MaybeUninit], distance: usize) { + unsafe { + let slice_ptr = slice.as_mut_ptr(); + ptr::copy(slice_ptr.add(distance), slice_ptr, slice.len() - distance); + } } + +/// Shifts the elements in a slice `distance` positions to the right. +/// +/// # Safety +/// The slice has at least `distance` elements. +unsafe fn slice_shr(slice: &mut [MaybeUninit], distance: usize) { + unsafe { + let slice_ptr = slice.as_mut_ptr(); + ptr::copy(slice_ptr, slice_ptr.add(distance), slice.len() - distance); + } +} + +/// Moves all values from a slice of initialized elements to a slice +/// of uninitialized elements, leaving behind `src` as all uninitialized. +/// Works like `dst.copy_from_slice(src)` but does not require `T` to be `Copy`. +fn move_to_slice(src: &mut [MaybeUninit], dst: &mut [MaybeUninit]) { + assert!(src.len() == dst.len()); + unsafe { + ptr::copy_nonoverlapping(src.as_ptr(), dst.as_mut_ptr(), src.len()); + } +} + +#[cfg(test)] +mod tests; diff --git a/crux-mir/lib/alloc/src/collections/btree/node/tests.rs b/crux-mir/lib/alloc/src/collections/btree/node/tests.rs new file mode 100644 index 000000000..64bce0ff8 --- /dev/null +++ b/crux-mir/lib/alloc/src/collections/btree/node/tests.rs @@ -0,0 +1,103 @@ +use super::super::navigate; +use super::*; +use crate::alloc::Global; +use crate::fmt::Debug; +use crate::string::String; + +impl<'a, K: 'a, V: 'a> NodeRef, K, V, marker::LeafOrInternal> { + // Asserts that the back pointer in each reachable node points to its parent. + pub fn assert_back_pointers(self) { + if let ForceResult::Internal(node) = self.force() { + for idx in 0..=node.len() { + let edge = unsafe { Handle::new_edge(node, idx) }; + let child = edge.descend(); + assert!(child.ascend().ok() == Some(edge)); + child.assert_back_pointers(); + } + } + } + + // Renders a multi-line display of the keys in order and in tree hierarchy, + // picturing the tree growing sideways from its root on the left to its + // leaves on the right. + pub fn dump_keys(self) -> String + where + K: Debug, + { + let mut result = String::new(); + self.visit_nodes_in_order(|pos| match pos { + navigate::Position::Leaf(leaf) => { + let depth = self.height(); + let indent = " ".repeat(depth); + result += &format!("\n{}{:?}", indent, leaf.keys()); + } + navigate::Position::Internal(_) => {} + navigate::Position::InternalKV(kv) => { + let depth = self.height() - kv.into_node().height(); + let indent = " ".repeat(depth); + result += &format!("\n{}{:?}", indent, kv.into_kv().0); + } + }); + result + } +} + +#[test] +fn test_splitpoint() { + for idx in 0..=CAPACITY { + let (middle_kv_idx, insertion) = splitpoint(idx); + + // Simulate performing the split: + let mut left_len = middle_kv_idx; + let mut right_len = CAPACITY - middle_kv_idx - 1; + match insertion { + LeftOrRight::Left(edge_idx) => { + assert!(edge_idx <= left_len); + left_len += 1; + } + LeftOrRight::Right(edge_idx) => { + assert!(edge_idx <= right_len); + right_len += 1; + } + } + assert!(left_len >= MIN_LEN_AFTER_SPLIT); + assert!(right_len >= MIN_LEN_AFTER_SPLIT); + assert!(left_len + right_len == CAPACITY); + } +} + +#[test] +fn test_partial_eq() { + let mut root1 = NodeRef::new_leaf(Global); + root1.borrow_mut().push(1, ()); + let mut root1 = NodeRef::new_internal(root1.forget_type(), Global).forget_type(); + let root2 = Root::new(Global); + root1.reborrow().assert_back_pointers(); + root2.reborrow().assert_back_pointers(); + + let leaf_edge_1a = root1.reborrow().first_leaf_edge().forget_node_type(); + let leaf_edge_1b = root1.reborrow().last_leaf_edge().forget_node_type(); + let top_edge_1 = root1.reborrow().first_edge(); + let top_edge_2 = root2.reborrow().first_edge(); + + assert!(leaf_edge_1a == leaf_edge_1a); + assert!(leaf_edge_1a != leaf_edge_1b); + assert!(leaf_edge_1a != top_edge_1); + assert!(leaf_edge_1a != top_edge_2); + assert!(top_edge_1 == top_edge_1); + assert!(top_edge_1 != top_edge_2); + + root1.pop_internal_level(Global); + unsafe { root1.into_dying().deallocate_and_ascend(Global) }; + unsafe { root2.into_dying().deallocate_and_ascend(Global) }; +} + +#[test] +#[cfg(target_arch = "x86_64")] +#[cfg_attr(miri, ignore)] // We'd like to run Miri with layout randomization +fn test_sizes() { + assert_eq!(core::mem::size_of::>(), 16); + assert_eq!(core::mem::size_of::>(), 16 + CAPACITY * 2 * 8); + assert_eq!(core::mem::size_of::>(), 16 + (CAPACITY + 1) * 8); + assert_eq!(core::mem::size_of::>(), 16 + (CAPACITY * 3 + 1) * 8); +} diff --git a/crux-mir/lib/alloc/src/collections/btree/remove.rs b/crux-mir/lib/alloc/src/collections/btree/remove.rs new file mode 100644 index 000000000..090429925 --- /dev/null +++ b/crux-mir/lib/alloc/src/collections/btree/remove.rs @@ -0,0 +1,95 @@ +use super::map::MIN_LEN; +use super::node::{marker, ForceResult::*, Handle, LeftOrRight::*, NodeRef}; +use core::alloc::Allocator; + +impl<'a, K: 'a, V: 'a> Handle, K, V, marker::LeafOrInternal>, marker::KV> { + /// Removes a key-value pair from the tree, and returns that pair, as well as + /// the leaf edge corresponding to that former pair. It's possible this empties + /// a root node that is internal, which the caller should pop from the map + /// holding the tree. The caller should also decrement the map's length. + pub fn remove_kv_tracking( + self, + handle_emptied_internal_root: F, + alloc: A, + ) -> ((K, V), Handle, K, V, marker::Leaf>, marker::Edge>) { + match self.force() { + Leaf(node) => node.remove_leaf_kv(handle_emptied_internal_root, alloc), + Internal(node) => node.remove_internal_kv(handle_emptied_internal_root, alloc), + } + } +} + +impl<'a, K: 'a, V: 'a> Handle, K, V, marker::Leaf>, marker::KV> { + fn remove_leaf_kv( + self, + handle_emptied_internal_root: F, + alloc: A, + ) -> ((K, V), Handle, K, V, marker::Leaf>, marker::Edge>) { + let (old_kv, mut pos) = self.remove(); + let len = pos.reborrow().into_node().len(); + if len < MIN_LEN { + let idx = pos.idx(); + // We have to temporarily forget the child type, because there is no + // distinct node type for the immediate parents of a leaf. + let new_pos = match pos.into_node().forget_type().choose_parent_kv() { + Ok(Left(left_parent_kv)) => { + debug_assert!(left_parent_kv.right_child_len() == MIN_LEN - 1); + if left_parent_kv.can_merge() { + left_parent_kv.merge_tracking_child_edge(Right(idx), alloc.clone()) + } else { + debug_assert!(left_parent_kv.left_child_len() > MIN_LEN); + left_parent_kv.steal_left(idx) + } + } + Ok(Right(right_parent_kv)) => { + debug_assert!(right_parent_kv.left_child_len() == MIN_LEN - 1); + if right_parent_kv.can_merge() { + right_parent_kv.merge_tracking_child_edge(Left(idx), alloc.clone()) + } else { + debug_assert!(right_parent_kv.right_child_len() > MIN_LEN); + right_parent_kv.steal_right(idx) + } + } + Err(pos) => unsafe { Handle::new_edge(pos, idx) }, + }; + // SAFETY: `new_pos` is the leaf we started from or a sibling. + pos = unsafe { new_pos.cast_to_leaf_unchecked() }; + + // Only if we merged, the parent (if any) has shrunk, but skipping + // the following step otherwise does not pay off in benchmarks. + // + // SAFETY: We won't destroy or rearrange the leaf where `pos` is at + // by handling its parent recursively; at worst we will destroy or + // rearrange the parent through the grandparent, thus change the + // link to the parent inside the leaf. + if let Ok(parent) = unsafe { pos.reborrow_mut() }.into_node().ascend() { + if !parent.into_node().forget_type().fix_node_and_affected_ancestors(alloc) { + handle_emptied_internal_root(); + } + } + } + (old_kv, pos) + } +} + +impl<'a, K: 'a, V: 'a> Handle, K, V, marker::Internal>, marker::KV> { + fn remove_internal_kv( + self, + handle_emptied_internal_root: F, + alloc: A, + ) -> ((K, V), Handle, K, V, marker::Leaf>, marker::Edge>) { + // Remove an adjacent KV from its leaf and then put it back in place of + // the element we were asked to remove. Prefer the left adjacent KV, + // for the reasons listed in `choose_parent_kv`. + let left_leaf_kv = self.left_edge().descend().last_leaf_edge().left_kv(); + let left_leaf_kv = unsafe { left_leaf_kv.ok().unwrap_unchecked() }; + let (left_kv, left_hole) = left_leaf_kv.remove_leaf_kv(handle_emptied_internal_root, alloc); + + // The internal node may have been stolen from or merged. Go back right + // to find where the original KV ended up. + let mut internal = unsafe { left_hole.next_kv().ok().unwrap_unchecked() }; + let old_kv = internal.replace_kv(left_kv.0, left_kv.1); + let pos = internal.next_leaf_edge(); + (old_kv, pos) + } +} diff --git a/crux-mir/lib/alloc/src/collections/btree/search.rs b/crux-mir/lib/alloc/src/collections/btree/search.rs index 4e80f7f21..ad3522b4e 100644 --- a/crux-mir/lib/alloc/src/collections/btree/search.rs +++ b/crux-mir/lib/alloc/src/collections/btree/search.rs @@ -1,83 +1,285 @@ use core::borrow::Borrow; use core::cmp::Ordering; +use core::ops::{Bound, RangeBounds}; use super::node::{marker, ForceResult::*, Handle, NodeRef}; +use SearchBound::*; use SearchResult::*; +pub enum SearchBound { + /// An inclusive bound to look for, just like `Bound::Included(T)`. + Included(T), + /// An exclusive bound to look for, just like `Bound::Excluded(T)`. + Excluded(T), + /// An unconditional inclusive bound, just like `Bound::Unbounded`. + AllIncluded, + /// An unconditional exclusive bound. + AllExcluded, +} + +impl SearchBound { + pub fn from_range(range_bound: Bound) -> Self { + match range_bound { + Bound::Included(t) => Included(t), + Bound::Excluded(t) => Excluded(t), + Bound::Unbounded => AllIncluded, + } + } +} + pub enum SearchResult { Found(Handle, marker::KV>), GoDown(Handle, marker::Edge>), } -/// Looks up a given key in a (sub)tree headed by the given node, recursively. -/// Returns a `Found` with the handle of the matching KV, if any. Otherwise, -/// returns a `GoDown` with the handle of the possible leaf edge where the key -/// belongs. -pub fn search_tree( - mut node: NodeRef, - key: &Q, -) -> SearchResult -where - Q: Ord, - K: Borrow, -{ - loop { - match search_node(node, key) { - Found(handle) => return Found(handle), - GoDown(handle) => match handle.force() { - Leaf(leaf) => return GoDown(leaf), - Internal(internal) => { - node = internal.descend(); - continue; +pub enum IndexResult { + KV(usize), + Edge(usize), +} + +impl NodeRef { + /// Looks up a given key in a (sub)tree headed by the node, recursively. + /// Returns a `Found` with the handle of the matching KV, if any. Otherwise, + /// returns a `GoDown` with the handle of the leaf edge where the key belongs. + /// + /// The result is meaningful only if the tree is ordered by key, like the tree + /// in a `BTreeMap` is. + pub fn search_tree( + mut self, + key: &Q, + ) -> SearchResult + where + Q: Ord, + K: Borrow, + { + loop { + self = match self.search_node(key) { + Found(handle) => return Found(handle), + GoDown(handle) => match handle.force() { + Leaf(leaf) => return GoDown(leaf), + Internal(internal) => internal.descend(), + }, + } + } + } + + /// Descends to the nearest node where the edge matching the lower bound + /// of the range is different from the edge matching the upper bound, i.e., + /// the nearest node that has at least one key contained in the range. + /// + /// If found, returns an `Ok` with that node, the strictly ascending pair of + /// edge indices in the node delimiting the range, and the corresponding + /// pair of bounds for continuing the search in the child nodes, in case + /// the node is internal. + /// + /// If not found, returns an `Err` with the leaf edge matching the entire + /// range. + /// + /// As a diagnostic service, panics if the range specifies impossible bounds. + /// + /// The result is meaningful only if the tree is ordered by key. + pub fn search_tree_for_bifurcation<'r, Q: ?Sized, R>( + mut self, + range: &'r R, + ) -> Result< + ( + NodeRef, + usize, + usize, + SearchBound<&'r Q>, + SearchBound<&'r Q>, + ), + Handle, marker::Edge>, + > + where + Q: Ord, + K: Borrow, + R: RangeBounds, + { + // Determine if map or set is being searched + let is_set = ::is_set_val(); + + // Inlining these variables should be avoided. We assume the bounds reported by `range` + // remain the same, but an adversarial implementation could change between calls (#81138). + let (start, end) = (range.start_bound(), range.end_bound()); + match (start, end) { + (Bound::Excluded(s), Bound::Excluded(e)) if s == e => { + if is_set { + panic!("range start and end are equal and excluded in BTreeSet") + } else { + panic!("range start and end are equal and excluded in BTreeMap") } - }, + } + (Bound::Included(s) | Bound::Excluded(s), Bound::Included(e) | Bound::Excluded(e)) + if s > e => + { + if is_set { + panic!("range start is greater than range end in BTreeSet") + } else { + panic!("range start is greater than range end in BTreeMap") + } + } + _ => {} + } + let mut lower_bound = SearchBound::from_range(start); + let mut upper_bound = SearchBound::from_range(end); + loop { + let (lower_edge_idx, lower_child_bound) = self.find_lower_bound_index(lower_bound); + let (upper_edge_idx, upper_child_bound) = + unsafe { self.find_upper_bound_index(upper_bound, lower_edge_idx) }; + if lower_edge_idx < upper_edge_idx { + return Ok(( + self, + lower_edge_idx, + upper_edge_idx, + lower_child_bound, + upper_child_bound, + )); + } + debug_assert_eq!(lower_edge_idx, upper_edge_idx); + let common_edge = unsafe { Handle::new_edge(self, lower_edge_idx) }; + match common_edge.force() { + Leaf(common_edge) => return Err(common_edge), + Internal(common_edge) => { + self = common_edge.descend(); + lower_bound = lower_child_bound; + upper_bound = upper_child_bound; + } + } } } -} -/// Looks up a given key in a given node, without recursion. -/// Returns a `Found` with the handle of the matching KV, if any. Otherwise, -/// returns a `GoDown` with the handle of the edge where the key might be found. -/// If the node is a leaf, a `GoDown` edge is not an actual edge but a possible edge. -pub fn search_node( - node: NodeRef, - key: &Q, -) -> SearchResult -where - Q: Ord, - K: Borrow, -{ - match search_linear(&node, key) { - (idx, true) => Found(unsafe { Handle::new_kv(node, idx) }), - (idx, false) => SearchResult::GoDown(unsafe { Handle::new_edge(node, idx) }), + /// Finds an edge in the node delimiting the lower bound of a range. + /// Also returns the lower bound to be used for continuing the search in + /// the matching child node, if `self` is an internal node. + /// + /// The result is meaningful only if the tree is ordered by key. + pub fn find_lower_bound_edge<'r, Q>( + self, + bound: SearchBound<&'r Q>, + ) -> (Handle, SearchBound<&'r Q>) + where + Q: ?Sized + Ord, + K: Borrow, + { + let (edge_idx, bound) = self.find_lower_bound_index(bound); + let edge = unsafe { Handle::new_edge(self, edge_idx) }; + (edge, bound) + } + + /// Clone of `find_lower_bound_edge` for the upper bound. + pub fn find_upper_bound_edge<'r, Q>( + self, + bound: SearchBound<&'r Q>, + ) -> (Handle, SearchBound<&'r Q>) + where + Q: ?Sized + Ord, + K: Borrow, + { + let (edge_idx, bound) = unsafe { self.find_upper_bound_index(bound, 0) }; + let edge = unsafe { Handle::new_edge(self, edge_idx) }; + (edge, bound) } } -/// Returns the index in the node at which the key (or an equivalent) exists -/// or could exist, and whether it exists in the node itself. If it doesn't -/// exist in the node itself, it may exist in the subtree with that index -/// (if the node has subtrees). If the key doesn't exist in node or subtree, -/// the returned index is the position or subtree where the key belongs. -fn search_linear( - node: &NodeRef, - key: &Q, -) -> (usize, bool) -where - Q: Ord, - K: Borrow, -{ - // This function is defined over all borrow types (immutable, mutable, owned). - // Using `keys()` is fine here even if BorrowType is mutable, as all we return - // is an index -- not a reference. - let len = node.len(); - let keys = node.keys(); - for (i, k) in keys.iter().enumerate() { - match key.cmp(k.borrow()) { - Ordering::Greater => {} - Ordering::Equal => return (i, true), - Ordering::Less => return (i, false), +impl NodeRef { + /// Looks up a given key in the node, without recursion. + /// Returns a `Found` with the handle of the matching KV, if any. Otherwise, + /// returns a `GoDown` with the handle of the edge where the key might be found + /// (if the node is internal) or where the key can be inserted. + /// + /// The result is meaningful only if the tree is ordered by key, like the tree + /// in a `BTreeMap` is. + pub fn search_node(self, key: &Q) -> SearchResult + where + Q: Ord, + K: Borrow, + { + match unsafe { self.find_key_index(key, 0) } { + IndexResult::KV(idx) => Found(unsafe { Handle::new_kv(self, idx) }), + IndexResult::Edge(idx) => GoDown(unsafe { Handle::new_edge(self, idx) }), + } + } + + /// Returns either the KV index in the node at which the key (or an equivalent) + /// exists, or the edge index where the key belongs, starting from a particular index. + /// + /// The result is meaningful only if the tree is ordered by key, like the tree + /// in a `BTreeMap` is. + /// + /// # Safety + /// `start_index` must be a valid edge index for the node. + unsafe fn find_key_index(&self, key: &Q, start_index: usize) -> IndexResult + where + Q: Ord, + K: Borrow, + { + let node = self.reborrow(); + let keys = node.keys(); + debug_assert!(start_index <= keys.len()); + for (offset, k) in unsafe { keys.get_unchecked(start_index..) }.iter().enumerate() { + match key.cmp(k.borrow()) { + Ordering::Greater => {} + Ordering::Equal => return IndexResult::KV(start_index + offset), + Ordering::Less => return IndexResult::Edge(start_index + offset), + } + } + IndexResult::Edge(keys.len()) + } + + /// Finds an edge index in the node delimiting the lower bound of a range. + /// Also returns the lower bound to be used for continuing the search in + /// the matching child node, if `self` is an internal node. + /// + /// The result is meaningful only if the tree is ordered by key. + fn find_lower_bound_index<'r, Q>( + &self, + bound: SearchBound<&'r Q>, + ) -> (usize, SearchBound<&'r Q>) + where + Q: ?Sized + Ord, + K: Borrow, + { + match bound { + Included(key) => match unsafe { self.find_key_index(key, 0) } { + IndexResult::KV(idx) => (idx, AllExcluded), + IndexResult::Edge(idx) => (idx, bound), + }, + Excluded(key) => match unsafe { self.find_key_index(key, 0) } { + IndexResult::KV(idx) => (idx + 1, AllIncluded), + IndexResult::Edge(idx) => (idx, bound), + }, + AllIncluded => (0, AllIncluded), + AllExcluded => (self.len(), AllExcluded), + } + } + + /// Mirror image of `find_lower_bound_index` for the upper bound, + /// with an additional parameter to skip part of the key array. + /// + /// # Safety + /// `start_index` must be a valid edge index for the node. + unsafe fn find_upper_bound_index<'r, Q>( + &self, + bound: SearchBound<&'r Q>, + start_index: usize, + ) -> (usize, SearchBound<&'r Q>) + where + Q: ?Sized + Ord, + K: Borrow, + { + match bound { + Included(key) => match unsafe { self.find_key_index(key, start_index) } { + IndexResult::KV(idx) => (idx + 1, AllExcluded), + IndexResult::Edge(idx) => (idx, bound), + }, + Excluded(key) => match unsafe { self.find_key_index(key, start_index) } { + IndexResult::KV(idx) => (idx, AllIncluded), + IndexResult::Edge(idx) => (idx, bound), + }, + AllIncluded => (self.len(), AllIncluded), + AllExcluded => (start_index, AllExcluded), } } - (len, false) } diff --git a/crux-mir/lib/alloc/src/collections/btree/set.rs b/crux-mir/lib/alloc/src/collections/btree/set.rs index b100ce754..4ddb21192 100644 --- a/crux-mir/lib/alloc/src/collections/btree/set.rs +++ b/crux-mir/lib/alloc/src/collections/btree/set.rs @@ -1,19 +1,26 @@ // This is pretty much entirely stolen from TreeSet, since BTreeMap has an identical interface // to TreeMap +use crate::vec::Vec; use core::borrow::Borrow; -use core::cmp::Ordering::{Equal, Greater, Less}; +use core::cmp::Ordering::{self, Equal, Greater, Less}; use core::cmp::{max, min}; use core::fmt::{self, Debug}; +use core::hash::{Hash, Hasher}; use core::iter::{FromIterator, FusedIterator, Peekable}; +use core::mem::ManuallyDrop; use core::ops::{BitAnd, BitOr, BitXor, RangeBounds, Sub}; +use super::map::{BTreeMap, Keys}; +use super::merge_iter::MergeIterInner; +use super::set_val::SetValZST; use super::Recover; -use crate::collections::btree_map::{self, BTreeMap, Keys}; + +use crate::alloc::{Allocator, Global}; // FIXME(conventions): implement bounded iterators -/// A set based on a B-Tree. +/// An ordered set based on a B-Tree. /// /// See [`BTreeMap`]'s documentation for a detailed discussion of this collection's performance /// benefits and drawbacks. @@ -21,11 +28,16 @@ use crate::collections::btree_map::{self, BTreeMap, Keys}; /// It is a logic error for an item to be modified in such a way that the item's ordering relative /// to any other item, as determined by the [`Ord`] trait, changes while it is in the set. This is /// normally only possible through [`Cell`], [`RefCell`], global state, I/O, or unsafe code. +/// The behavior resulting from such a logic error is not specified, but will be encapsulated to the +/// `BTreeSet` that observed the logic error and not result in undefined behavior. This could +/// include panics, incorrect results, aborts, memory leaks, and non-termination. +/// +/// Iterators returned by [`BTreeSet::iter`] produce their items in order, and take worst-case +/// logarithmic and amortized constant time per item returned. /// -/// [`BTreeMap`]: struct.BTreeMap.html -/// [`Ord`]: ../../std/cmp/trait.Ord.html -/// [`Cell`]: ../../std/cell/struct.Cell.html -/// [`RefCell`]: ../../std/cell/struct.RefCell.html +/// [`Ord`]: core::cmp::Ord +/// [`Cell`]: core::cell::Cell +/// [`RefCell`]: core::cell::RefCell /// /// # Examples /// @@ -53,17 +65,59 @@ use crate::collections::btree_map::{self, BTreeMap, Keys}; /// /// // Iterate over everything. /// for book in &books { -/// println!("{}", book); +/// println!("{book}"); /// } /// ``` -#[derive(Hash, PartialEq, Eq, Ord, PartialOrd)] +/// +/// A `BTreeSet` with a known list of items can be initialized from an array: +/// +/// ``` +/// use std::collections::BTreeSet; +/// +/// let set = BTreeSet::from([1, 2, 3]); +/// ``` +#[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "BTreeSet")] +pub struct BTreeSet< + T, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + Clone = Global, +> { + map: BTreeMap, +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Hash for BTreeSet { + fn hash(&self, state: &mut H) { + self.map.hash(state) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialEq for BTreeSet { + fn eq(&self, other: &BTreeSet) -> bool { + self.map.eq(&other.map) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Eq for BTreeSet {} + #[stable(feature = "rust1", since = "1.0.0")] -pub struct BTreeSet { - map: BTreeMap, +impl PartialOrd for BTreeSet { + fn partial_cmp(&self, other: &BTreeSet) -> Option { + self.map.partial_cmp(&other.map) + } } #[stable(feature = "rust1", since = "1.0.0")] -impl Clone for BTreeSet { +impl Ord for BTreeSet { + fn cmp(&self, other: &BTreeSet) -> Ordering { + self.map.cmp(&other.map) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Clone for BTreeSet { fn clone(&self) -> Self { BTreeSet { map: self.map.clone() } } @@ -78,11 +132,11 @@ impl Clone for BTreeSet { /// This `struct` is created by the [`iter`] method on [`BTreeSet`]. /// See its documentation for more. /// -/// [`BTreeSet`]: struct.BTreeSet.html -/// [`iter`]: struct.BTreeSet.html#method.iter +/// [`iter`]: BTreeSet::iter +#[must_use = "iterators are lazy and do nothing unless consumed"] #[stable(feature = "rust1", since = "1.0.0")] pub struct Iter<'a, T: 'a> { - iter: Keys<'a, T, ()>, + iter: Keys<'a, T, SetValZST>, } #[stable(feature = "collection_debug", since = "1.17.0")] @@ -95,14 +149,17 @@ impl fmt::Debug for Iter<'_, T> { /// An owning iterator over the items of a `BTreeSet`. /// /// This `struct` is created by the [`into_iter`] method on [`BTreeSet`] -/// (provided by the `IntoIterator` trait). See its documentation for more. +/// (provided by the [`IntoIterator`] trait). See its documentation for more. /// -/// [`BTreeSet`]: struct.BTreeSet.html -/// [`into_iter`]: struct.BTreeSet.html#method.into_iter +/// [`into_iter`]: BTreeSet#method.into_iter +/// [`IntoIterator`]: core::iter::IntoIterator #[stable(feature = "rust1", since = "1.0.0")] #[derive(Debug)] -pub struct IntoIter { - iter: btree_map::IntoIter, +pub struct IntoIter< + T, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + Clone = Global, +> { + iter: super::map::IntoIter, } /// An iterator over a sub-range of items in a `BTreeSet`. @@ -110,83 +167,12 @@ pub struct IntoIter { /// This `struct` is created by the [`range`] method on [`BTreeSet`]. /// See its documentation for more. /// -/// [`BTreeSet`]: struct.BTreeSet.html -/// [`range`]: struct.BTreeSet.html#method.range +/// [`range`]: BTreeSet::range +#[must_use = "iterators are lazy and do nothing unless consumed"] #[derive(Debug)] #[stable(feature = "btree_range", since = "1.17.0")] pub struct Range<'a, T: 'a> { - iter: btree_map::Range<'a, T, ()>, -} - -/// Core of SymmetricDifference and Union. -/// More efficient than btree.map.MergeIter, -/// and crucially for SymmetricDifference, nexts() reports on both sides. -#[derive(Clone)] -struct MergeIterInner -where - I: Iterator, - I::Item: Copy, -{ - a: I, - b: I, - peeked: Option>, -} - -#[derive(Copy, Clone, Debug)] -enum MergeIterPeeked { - A(I::Item), - B(I::Item), -} - -impl MergeIterInner -where - I: ExactSizeIterator + FusedIterator, - I::Item: Copy + Ord, -{ - fn new(a: I, b: I) -> Self { - MergeIterInner { a, b, peeked: None } - } - - fn nexts(&mut self) -> (Option, Option) { - let mut a_next = match self.peeked { - Some(MergeIterPeeked::A(next)) => Some(next), - _ => self.a.next(), - }; - let mut b_next = match self.peeked { - Some(MergeIterPeeked::B(next)) => Some(next), - _ => self.b.next(), - }; - let ord = match (a_next, b_next) { - (None, None) => Equal, - (_, None) => Less, - (None, _) => Greater, - (Some(a1), Some(b1)) => a1.cmp(&b1), - }; - self.peeked = match ord { - Less => b_next.take().map(MergeIterPeeked::B), - Equal => None, - Greater => a_next.take().map(MergeIterPeeked::A), - }; - (a_next, b_next) - } - - fn lens(&self) -> (usize, usize) { - match self.peeked { - Some(MergeIterPeeked::A(_)) => (1 + self.a.len(), self.b.len()), - Some(MergeIterPeeked::B(_)) => (self.a.len(), 1 + self.b.len()), - _ => (self.a.len(), self.b.len()), - } - } -} - -impl Debug for MergeIterInner -where - I: Iterator + Debug, - I::Item: Copy + Debug, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("MergeIterInner").field(&self.a).field(&self.b).finish() - } + iter: super::map::Range<'a, T, SetValZST>, } /// A lazy iterator producing elements in the difference of `BTreeSet`s. @@ -194,14 +180,18 @@ where /// This `struct` is created by the [`difference`] method on [`BTreeSet`]. /// See its documentation for more. /// -/// [`BTreeSet`]: struct.BTreeSet.html -/// [`difference`]: struct.BTreeSet.html#method.difference +/// [`difference`]: BTreeSet::difference +#[must_use = "this returns the difference as an iterator, \ + without modifying either input set"] #[stable(feature = "rust1", since = "1.0.0")] -pub struct Difference<'a, T: 'a> { - inner: DifferenceInner<'a, T>, +pub struct Difference< + 'a, + T: 'a, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + Clone = Global, +> { + inner: DifferenceInner<'a, T, A>, } -#[derive(Debug)] -enum DifferenceInner<'a, T: 'a> { +enum DifferenceInner<'a, T: 'a, A: Allocator + Clone> { Stitch { // iterate all of `self` and some of `other`, spotting matches along the way self_iter: Iter<'a, T>, @@ -210,13 +200,32 @@ enum DifferenceInner<'a, T: 'a> { Search { // iterate `self`, look up in `other` self_iter: Iter<'a, T>, - other_set: &'a BTreeSet, + other_set: &'a BTreeSet, }, - Iterate(Iter<'a, T>), // simply produce all values in `self` + Iterate(Iter<'a, T>), // simply produce all elements in `self` +} + +// Explicit Debug impl necessary because of issue #26925 +impl Debug for DifferenceInner<'_, T, A> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + DifferenceInner::Stitch { self_iter, other_iter } => f + .debug_struct("Stitch") + .field("self_iter", self_iter) + .field("other_iter", other_iter) + .finish(), + DifferenceInner::Search { self_iter, other_set } => f + .debug_struct("Search") + .field("self_iter", self_iter) + .field("other_iter", other_set) + .finish(), + DifferenceInner::Iterate(x) => f.debug_tuple("Iterate").field(x).finish(), + } + } } #[stable(feature = "collection_debug", since = "1.17.0")] -impl fmt::Debug for Difference<'_, T> { +impl fmt::Debug for Difference<'_, T, A> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("Difference").field(&self.inner).finish() } @@ -227,8 +236,9 @@ impl fmt::Debug for Difference<'_, T> { /// This `struct` is created by the [`symmetric_difference`] method on /// [`BTreeSet`]. See its documentation for more. /// -/// [`BTreeSet`]: struct.BTreeSet.html -/// [`symmetric_difference`]: struct.BTreeSet.html#method.symmetric_difference +/// [`symmetric_difference`]: BTreeSet::symmetric_difference +#[must_use = "this returns the difference as an iterator, \ + without modifying either input set"] #[stable(feature = "rust1", since = "1.0.0")] pub struct SymmetricDifference<'a, T: 'a>(MergeIterInner>); @@ -244,14 +254,18 @@ impl fmt::Debug for SymmetricDifference<'_, T> { /// This `struct` is created by the [`intersection`] method on [`BTreeSet`]. /// See its documentation for more. /// -/// [`BTreeSet`]: struct.BTreeSet.html -/// [`intersection`]: struct.BTreeSet.html#method.intersection +/// [`intersection`]: BTreeSet::intersection +#[must_use = "this returns the intersection as an iterator, \ + without modifying either input set"] #[stable(feature = "rust1", since = "1.0.0")] -pub struct Intersection<'a, T: 'a> { - inner: IntersectionInner<'a, T>, +pub struct Intersection< + 'a, + T: 'a, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + Clone = Global, +> { + inner: IntersectionInner<'a, T, A>, } -#[derive(Debug)] -enum IntersectionInner<'a, T: 'a> { +enum IntersectionInner<'a, T: 'a, A: Allocator + Clone> { Stitch { // iterate similarly sized sets jointly, spotting matches along the way a: Iter<'a, T>, @@ -260,13 +274,30 @@ enum IntersectionInner<'a, T: 'a> { Search { // iterate a small set, look up in the large set small_iter: Iter<'a, T>, - large_set: &'a BTreeSet, + large_set: &'a BTreeSet, }, - Answer(Option<&'a T>), // return a specific value or emptiness + Answer(Option<&'a T>), // return a specific element or emptiness +} + +// Explicit Debug impl necessary because of issue #26925 +impl Debug for IntersectionInner<'_, T, A> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + IntersectionInner::Stitch { a, b } => { + f.debug_struct("Stitch").field("a", a).field("b", b).finish() + } + IntersectionInner::Search { small_iter, large_set } => f + .debug_struct("Search") + .field("small_iter", small_iter) + .field("large_set", large_set) + .finish(), + IntersectionInner::Answer(x) => f.debug_tuple("Answer").field(x).finish(), + } + } } #[stable(feature = "collection_debug", since = "1.17.0")] -impl fmt::Debug for Intersection<'_, T> { +impl Debug for Intersection<'_, T, A> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("Intersection").field(&self.inner).finish() } @@ -277,8 +308,9 @@ impl fmt::Debug for Intersection<'_, T> { /// This `struct` is created by the [`union`] method on [`BTreeSet`]. /// See its documentation for more. /// -/// [`BTreeSet`]: struct.BTreeSet.html -/// [`union`]: struct.BTreeSet.html#method.union +/// [`union`]: BTreeSet::union +#[must_use = "this returns the union as an iterator, \ + without modifying either input set"] #[stable(feature = "rust1", since = "1.0.0")] pub struct Union<'a, T: 'a>(MergeIterInner>); @@ -292,13 +324,15 @@ impl fmt::Debug for Union<'_, T> { // This constant is used by functions that compare two sets. // It estimates the relative size at which searching performs better // than iterating, based on the benchmarks in -// https://github.com/ssomers/rust_bench_btreeset_intersection; +// https://github.com/ssomers/rust_bench_btreeset_intersection. // It's used to divide rather than multiply sizes, to rule out overflow, // and it's a power of two to make that division cheap. const ITER_PERFORMANCE_TIPPING_SIZE_DIFF: usize = 16; -impl BTreeSet { - /// Makes a new `BTreeSet` with a reasonable choice of B. +impl BTreeSet { + /// Makes a new, empty `BTreeSet`. + /// + /// Does not allocate anything on its own. /// /// # Examples /// @@ -309,9 +343,31 @@ impl BTreeSet { /// let mut set: BTreeSet = BTreeSet::new(); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn new() -> BTreeSet { + #[rustc_const_stable(feature = "const_btree_new", since = "1.66.0")] + #[must_use] + pub const fn new() -> BTreeSet { BTreeSet { map: BTreeMap::new() } } +} + +impl BTreeSet { + /// Makes a new `BTreeSet` with a reasonable choice of B. + /// + /// # Examples + /// + /// ``` + /// # #![allow(unused_mut)] + /// # #![feature(allocator_api)] + /// # #![feature(btreemap_alloc)] + /// use std::collections::BTreeSet; + /// use std::alloc::Global; + /// + /// let mut set: BTreeSet = BTreeSet::new_in(Global); + /// ``` + #[unstable(feature = "btreemap_alloc", issue = "32838")] + pub fn new_in(alloc: A) -> BTreeSet { + BTreeSet { map: BTreeMap::new_in(alloc) } + } /// Constructs a double-ended iterator over a sub-range of elements in the set. /// The simplest way is to use the range syntax `min..max`, thus `range(min..max)` will @@ -320,6 +376,11 @@ impl BTreeSet { /// `range((Excluded(4), Included(10)))` will yield a left-exclusive, right-inclusive /// range from 4 to 10. /// + /// # Panics + /// + /// Panics if range `start > end`. + /// Panics if range `start == end` and both bounds are `Excluded`. + /// /// # Examples /// /// ``` @@ -331,7 +392,7 @@ impl BTreeSet { /// set.insert(5); /// set.insert(8); /// for &elem in set.range((Included(&4), Included(&8))) { - /// println!("{}", elem); + /// println!("{elem}"); /// } /// assert_eq!(Some(&5), set.range(4..).next()); /// ``` @@ -339,14 +400,14 @@ impl BTreeSet { pub fn range(&self, range: R) -> Range<'_, T> where K: Ord, - T: Borrow, + T: Borrow + Ord, R: RangeBounds, { Range { iter: self.map.range(range) } } - /// Visits the values representing the difference, - /// i.e., the values that are in `self` but not in `other`, + /// Visits the elements representing the difference, + /// i.e., the elements that are in `self` but not in `other`, /// in ascending order. /// /// # Examples @@ -366,7 +427,10 @@ impl BTreeSet { /// assert_eq!(diff, [1]); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn difference<'a>(&'a self, other: &'a BTreeSet) -> Difference<'a, T> { + pub fn difference<'a>(&'a self, other: &'a BTreeSet) -> Difference<'a, T, A> + where + T: Ord, + { let (self_min, self_max) = if let (Some(self_min), Some(self_max)) = (self.first(), self.last()) { (self_min, self_max) @@ -403,8 +467,8 @@ impl BTreeSet { } } - /// Visits the values representing the symmetric difference, - /// i.e., the values that are in `self` or in `other` but not in both, + /// Visits the elements representing the symmetric difference, + /// i.e., the elements that are in `self` or in `other` but not in both, /// in ascending order. /// /// # Examples @@ -426,13 +490,16 @@ impl BTreeSet { #[stable(feature = "rust1", since = "1.0.0")] pub fn symmetric_difference<'a>( &'a self, - other: &'a BTreeSet, - ) -> SymmetricDifference<'a, T> { + other: &'a BTreeSet, + ) -> SymmetricDifference<'a, T> + where + T: Ord, + { SymmetricDifference(MergeIterInner::new(self.iter(), other.iter())) } - /// Visits the values representing the intersection, - /// i.e., the values that are both in `self` and `other`, + /// Visits the elements representing the intersection, + /// i.e., the elements that are both in `self` and `other`, /// in ascending order. /// /// # Examples @@ -452,7 +519,10 @@ impl BTreeSet { /// assert_eq!(intersection, [2]); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn intersection<'a>(&'a self, other: &'a BTreeSet) -> Intersection<'a, T> { + pub fn intersection<'a>(&'a self, other: &'a BTreeSet) -> Intersection<'a, T, A> + where + T: Ord, + { let (self_min, self_max) = if let (Some(self_min), Some(self_max)) = (self.first(), self.last()) { (self_min, self_max) @@ -481,8 +551,8 @@ impl BTreeSet { } } - /// Visits the values representing the union, - /// i.e., all the values in `self` or `other`, without duplicates, + /// Visits the elements representing the union, + /// i.e., all the elements in `self` or `other`, without duplicates, /// in ascending order. /// /// # Examples @@ -500,11 +570,14 @@ impl BTreeSet { /// assert_eq!(union, [1, 2]); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn union<'a>(&'a self, other: &'a BTreeSet) -> Union<'a, T> { + pub fn union<'a>(&'a self, other: &'a BTreeSet) -> Union<'a, T> + where + T: Ord, + { Union(MergeIterInner::new(self.iter(), other.iter())) } - /// Clears the set, removing all values. + /// Clears the set, removing all elements. /// /// # Examples /// @@ -517,53 +590,57 @@ impl BTreeSet { /// assert!(v.is_empty()); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn clear(&mut self) { + pub fn clear(&mut self) + where + A: Clone, + { self.map.clear() } - /// Returns `true` if the set contains a value. + /// Returns `true` if the set contains an element equal to the value. /// - /// The value may be any borrowed form of the set's value type, + /// The value may be any borrowed form of the set's element type, /// but the ordering on the borrowed form *must* match the - /// ordering on the value type. + /// ordering on the element type. /// /// # Examples /// /// ``` /// use std::collections::BTreeSet; /// - /// let set: BTreeSet<_> = [1, 2, 3].iter().cloned().collect(); + /// let set = BTreeSet::from([1, 2, 3]); /// assert_eq!(set.contains(&1), true); /// assert_eq!(set.contains(&4), false); /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn contains(&self, value: &Q) -> bool where - T: Borrow, + T: Borrow + Ord, Q: Ord, { self.map.contains_key(value) } - /// Returns a reference to the value in the set, if any, that is equal to the given value. + /// Returns a reference to the element in the set, if any, that is equal to + /// the value. /// - /// The value may be any borrowed form of the set's value type, + /// The value may be any borrowed form of the set's element type, /// but the ordering on the borrowed form *must* match the - /// ordering on the value type. + /// ordering on the element type. /// /// # Examples /// /// ``` /// use std::collections::BTreeSet; /// - /// let set: BTreeSet<_> = [1, 2, 3].iter().cloned().collect(); + /// let set = BTreeSet::from([1, 2, 3]); /// assert_eq!(set.get(&2), Some(&2)); /// assert_eq!(set.get(&4), None); /// ``` #[stable(feature = "set_recovery", since = "1.9.0")] pub fn get(&self, value: &Q) -> Option<&T> where - T: Borrow, + T: Borrow + Ord, Q: Ord, { Recover::get(&self.map, value) @@ -577,7 +654,7 @@ impl BTreeSet { /// ``` /// use std::collections::BTreeSet; /// - /// let a: BTreeSet<_> = [1, 2, 3].iter().cloned().collect(); + /// let a = BTreeSet::from([1, 2, 3]); /// let mut b = BTreeSet::new(); /// /// assert_eq!(a.is_disjoint(&b), true); @@ -586,20 +663,24 @@ impl BTreeSet { /// b.insert(1); /// assert_eq!(a.is_disjoint(&b), false); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] - pub fn is_disjoint(&self, other: &BTreeSet) -> bool { + pub fn is_disjoint(&self, other: &BTreeSet) -> bool + where + T: Ord, + { self.intersection(other).next().is_none() } /// Returns `true` if the set is a subset of another, - /// i.e., `other` contains at least all the values in `self`. + /// i.e., `other` contains at least all the elements in `self`. /// /// # Examples /// /// ``` /// use std::collections::BTreeSet; /// - /// let sup: BTreeSet<_> = [1, 2, 3].iter().cloned().collect(); + /// let sup = BTreeSet::from([1, 2, 3]); /// let mut set = BTreeSet::new(); /// /// assert_eq!(set.is_subset(&sup), true); @@ -608,8 +689,12 @@ impl BTreeSet { /// set.insert(4); /// assert_eq!(set.is_subset(&sup), false); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] - pub fn is_subset(&self, other: &BTreeSet) -> bool { + pub fn is_subset(&self, other: &BTreeSet) -> bool + where + T: Ord, + { // Same result as self.difference(other).next().is_none() // but the code below is faster (hugely in some cases). if self.len() > other.len() { @@ -665,14 +750,14 @@ impl BTreeSet { } /// Returns `true` if the set is a superset of another, - /// i.e., `self` contains at least all the values in `other`. + /// i.e., `self` contains at least all the elements in `other`. /// /// # Examples /// /// ``` /// use std::collections::BTreeSet; /// - /// let sub: BTreeSet<_> = [1, 2].iter().cloned().collect(); + /// let sub = BTreeSet::from([1, 2]); /// let mut set = BTreeSet::new(); /// /// assert_eq!(set.is_superset(&sub), false); @@ -684,64 +769,73 @@ impl BTreeSet { /// set.insert(2); /// assert_eq!(set.is_superset(&sub), true); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] - pub fn is_superset(&self, other: &BTreeSet) -> bool { + pub fn is_superset(&self, other: &BTreeSet) -> bool + where + T: Ord, + { other.is_subset(self) } - /// Returns a reference to the first value in the set, if any. - /// This value is always the minimum of all values in the set. + /// Returns a reference to the first element in the set, if any. + /// This element is always the minimum of all elements in the set. /// /// # Examples /// /// Basic usage: /// /// ``` - /// #![feature(map_first_last)] /// use std::collections::BTreeSet; /// - /// let mut map = BTreeSet::new(); - /// assert_eq!(map.first(), None); - /// map.insert(1); - /// assert_eq!(map.first(), Some(&1)); - /// map.insert(2); - /// assert_eq!(map.first(), Some(&1)); + /// let mut set = BTreeSet::new(); + /// assert_eq!(set.first(), None); + /// set.insert(1); + /// assert_eq!(set.first(), Some(&1)); + /// set.insert(2); + /// assert_eq!(set.first(), Some(&1)); /// ``` - #[unstable(feature = "map_first_last", issue = "62924")] - pub fn first(&self) -> Option<&T> { + #[must_use] + #[stable(feature = "map_first_last", since = "1.66.0")] + pub fn first(&self) -> Option<&T> + where + T: Ord, + { self.map.first_key_value().map(|(k, _)| k) } - /// Returns a reference to the last value in the set, if any. - /// This value is always the maximum of all values in the set. + /// Returns a reference to the last element in the set, if any. + /// This element is always the maximum of all elements in the set. /// /// # Examples /// /// Basic usage: /// /// ``` - /// #![feature(map_first_last)] /// use std::collections::BTreeSet; /// - /// let mut map = BTreeSet::new(); - /// assert_eq!(map.first(), None); - /// map.insert(1); - /// assert_eq!(map.last(), Some(&1)); - /// map.insert(2); - /// assert_eq!(map.last(), Some(&2)); + /// let mut set = BTreeSet::new(); + /// assert_eq!(set.last(), None); + /// set.insert(1); + /// assert_eq!(set.last(), Some(&1)); + /// set.insert(2); + /// assert_eq!(set.last(), Some(&2)); /// ``` - #[unstable(feature = "map_first_last", issue = "62924")] - pub fn last(&self) -> Option<&T> { + #[must_use] + #[stable(feature = "map_first_last", since = "1.66.0")] + pub fn last(&self) -> Option<&T> + where + T: Ord, + { self.map.last_key_value().map(|(k, _)| k) } - /// Removes the first value from the set and returns it, if any. - /// The first value is always the minimum value in the set. + /// Removes the first element from the set and returns it, if any. + /// The first element is always the minimum element in the set. /// /// # Examples /// /// ``` - /// #![feature(map_first_last)] /// use std::collections::BTreeSet; /// /// let mut set = BTreeSet::new(); @@ -752,18 +846,20 @@ impl BTreeSet { /// } /// assert!(set.is_empty()); /// ``` - #[unstable(feature = "map_first_last", issue = "62924")] - pub fn pop_first(&mut self) -> Option { - self.map.first_entry().map(|entry| entry.remove_entry().0) + #[stable(feature = "map_first_last", since = "1.66.0")] + pub fn pop_first(&mut self) -> Option + where + T: Ord, + { + self.map.pop_first().map(|kv| kv.0) } - /// Removes the last value from the set and returns it, if any. - /// The last value is always the maximum value in the set. + /// Removes the last element from the set and returns it, if any. + /// The last element is always the maximum element in the set. /// /// # Examples /// /// ``` - /// #![feature(map_first_last)] /// use std::collections::BTreeSet; /// /// let mut set = BTreeSet::new(); @@ -774,17 +870,24 @@ impl BTreeSet { /// } /// assert!(set.is_empty()); /// ``` - #[unstable(feature = "map_first_last", issue = "62924")] - pub fn pop_last(&mut self) -> Option { - self.map.last_entry().map(|entry| entry.remove_entry().0) + #[stable(feature = "map_first_last", since = "1.66.0")] + pub fn pop_last(&mut self) -> Option + where + T: Ord, + { + self.map.pop_last().map(|kv| kv.0) } /// Adds a value to the set. /// - /// If the set did not have this value present, `true` is returned. + /// Returns whether the value was newly inserted. That is: /// - /// If the set did have this value present, `false` is returned, and the - /// entry is not updated. See the [module-level documentation] for more. + /// - If the set did not previously contain an equal value, `true` is + /// returned. + /// - If the set already contained an equal value, `false` is returned, and + /// the entry is not updated. + /// + /// See the [module-level documentation] for more. /// /// [module-level documentation]: index.html#insert-and-complex-keys /// @@ -800,12 +903,15 @@ impl BTreeSet { /// assert_eq!(set.len(), 1); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn insert(&mut self, value: T) -> bool { - self.map.insert(value, ()).is_none() + pub fn insert(&mut self, value: T) -> bool + where + T: Ord, + { + self.map.insert(value, SetValZST::default()).is_none() } - /// Adds a value to the set, replacing the existing value, if any, that is equal to the given - /// one. Returns the replaced value. + /// Adds a value to the set, replacing the existing element, if any, that is + /// equal to the value. Returns the replaced element. /// /// # Examples /// @@ -820,16 +926,19 @@ impl BTreeSet { /// assert_eq!(set.get(&[][..]).unwrap().capacity(), 10); /// ``` #[stable(feature = "set_recovery", since = "1.9.0")] - pub fn replace(&mut self, value: T) -> Option { + pub fn replace(&mut self, value: T) -> Option + where + T: Ord, + { Recover::replace(&mut self.map, value) } - /// Removes a value from the set. Returns whether the value was - /// present in the set. + /// If the set contains an element equal to the value, removes it from the + /// set and drops it. Returns whether such an element was present. /// - /// The value may be any borrowed form of the set's value type, + /// The value may be any borrowed form of the set's element type, /// but the ordering on the borrowed form *must* match the - /// ordering on the value type. + /// ordering on the element type. /// /// # Examples /// @@ -845,37 +954,62 @@ impl BTreeSet { #[stable(feature = "rust1", since = "1.0.0")] pub fn remove(&mut self, value: &Q) -> bool where - T: Borrow, + T: Borrow + Ord, Q: Ord, { self.map.remove(value).is_some() } - /// Removes and returns the value in the set, if any, that is equal to the given one. + /// Removes and returns the element in the set, if any, that is equal to + /// the value. /// - /// The value may be any borrowed form of the set's value type, + /// The value may be any borrowed form of the set's element type, /// but the ordering on the borrowed form *must* match the - /// ordering on the value type. + /// ordering on the element type. /// /// # Examples /// /// ``` /// use std::collections::BTreeSet; /// - /// let mut set: BTreeSet<_> = [1, 2, 3].iter().cloned().collect(); + /// let mut set = BTreeSet::from([1, 2, 3]); /// assert_eq!(set.take(&2), Some(2)); /// assert_eq!(set.take(&2), None); /// ``` #[stable(feature = "set_recovery", since = "1.9.0")] pub fn take(&mut self, value: &Q) -> Option where - T: Borrow, + T: Borrow + Ord, Q: Ord, { Recover::take(&mut self.map, value) } - /// Moves all elements from `other` into `Self`, leaving `other` empty. + /// Retains only the elements specified by the predicate. + /// + /// In other words, remove all elements `e` for which `f(&e)` returns `false`. + /// The elements are visited in ascending order. + /// + /// # Examples + /// + /// ``` + /// use std::collections::BTreeSet; + /// + /// let mut set = BTreeSet::from([1, 2, 3, 4, 5, 6]); + /// // Keep only the even numbers. + /// set.retain(|&k| k % 2 == 0); + /// assert!(set.iter().eq([2, 4, 6].iter())); + /// ``` + #[stable(feature = "btree_retain", since = "1.53.0")] + pub fn retain(&mut self, mut f: F) + where + T: Ord, + F: FnMut(&T) -> bool, + { + self.drain_filter(|v| !f(v)); + } + + /// Moves all elements from `other` into `self`, leaving `other` empty. /// /// # Examples /// @@ -904,12 +1038,16 @@ impl BTreeSet { /// assert!(a.contains(&5)); /// ``` #[stable(feature = "btree_append", since = "1.11.0")] - pub fn append(&mut self, other: &mut Self) { + pub fn append(&mut self, other: &mut Self) + where + T: Ord, + A: Clone, + { self.map.append(&mut other.map); } - /// Splits the collection into two at the given key. Returns everything after the given key, - /// including the key. + /// Splits the collection into two at the value. Returns a new collection + /// with all elements greater than or equal to the value. /// /// # Examples /// @@ -938,23 +1076,62 @@ impl BTreeSet { /// assert!(b.contains(&41)); /// ``` #[stable(feature = "btree_split_off", since = "1.11.0")] - pub fn split_off(&mut self, key: &Q) -> Self + pub fn split_off(&mut self, value: &Q) -> Self where - T: Borrow, + T: Borrow + Ord, + A: Clone, { - BTreeSet { map: self.map.split_off(key) } + BTreeSet { map: self.map.split_off(value) } } -} -impl BTreeSet { - /// Gets an iterator that visits the values in the `BTreeSet` in ascending order. + /// Creates an iterator that visits all elements in ascending order and + /// uses a closure to determine if an element should be removed. + /// + /// If the closure returns `true`, the element is removed from the set and + /// yielded. If the closure returns `false`, or panics, the element remains + /// in the set and will not be yielded. + /// + /// If the iterator is only partially consumed or not consumed at all, each + /// of the remaining elements is still subjected to the closure and removed + /// and dropped if it returns `true`. + /// + /// It is unspecified how many more elements will be subjected to the + /// closure if a panic occurs in the closure, or if a panic occurs while + /// dropping an element, or if the `DrainFilter` itself is leaked. /// /// # Examples /// + /// Splitting a set into even and odd values, reusing the original set: + /// /// ``` + /// #![feature(btree_drain_filter)] /// use std::collections::BTreeSet; /// - /// let set: BTreeSet = [1, 2, 3].iter().cloned().collect(); + /// let mut set: BTreeSet = (0..8).collect(); + /// let evens: BTreeSet<_> = set.drain_filter(|v| v % 2 == 0).collect(); + /// let odds = set; + /// assert_eq!(evens.into_iter().collect::>(), vec![0, 2, 4, 6]); + /// assert_eq!(odds.into_iter().collect::>(), vec![1, 3, 5, 7]); + /// ``` + #[unstable(feature = "btree_drain_filter", issue = "70530")] + pub fn drain_filter<'a, F>(&'a mut self, pred: F) -> DrainFilter<'a, T, F, A> + where + T: Ord, + F: 'a + FnMut(&T) -> bool, + { + let (inner, alloc) = self.map.drain_filter_inner(); + DrainFilter { pred, inner, alloc } + } + + /// Gets an iterator that visits the elements in the `BTreeSet` in ascending + /// order. + /// + /// # Examples + /// + /// ``` + /// use std::collections::BTreeSet; + /// + /// let set = BTreeSet::from([1, 2, 3]); /// let mut set_iter = set.iter(); /// assert_eq!(set_iter.next(), Some(&1)); /// assert_eq!(set_iter.next(), Some(&2)); @@ -967,7 +1144,7 @@ impl BTreeSet { /// ``` /// use std::collections::BTreeSet; /// - /// let set: BTreeSet = [3, 1, 2].iter().cloned().collect(); + /// let set = BTreeSet::from([3, 1, 2]); /// let mut set_iter = set.iter(); /// assert_eq!(set_iter.next(), Some(&1)); /// assert_eq!(set_iter.next(), Some(&2)); @@ -991,8 +1168,14 @@ impl BTreeSet { /// v.insert(1); /// assert_eq!(v.len(), 1); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] - pub fn len(&self) -> usize { + #[rustc_const_unstable( + feature = "const_btree_len", + issue = "71835", + implied_by = "const_btree_new" + )] + pub const fn len(&self) -> usize { self.map.len() } @@ -1008,8 +1191,14 @@ impl BTreeSet { /// v.insert(1); /// assert!(!v.is_empty()); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] - pub fn is_empty(&self) -> bool { + #[rustc_const_unstable( + feature = "const_btree_len", + issue = "71835", + implied_by = "const_btree_new" + )] + pub const fn is_empty(&self) -> bool { self.len() == 0 } } @@ -1017,16 +1206,54 @@ impl BTreeSet { #[stable(feature = "rust1", since = "1.0.0")] impl FromIterator for BTreeSet { fn from_iter>(iter: I) -> BTreeSet { - let mut set = BTreeSet::new(); - set.extend(iter); - set + let mut inputs: Vec<_> = iter.into_iter().collect(); + + if inputs.is_empty() { + return BTreeSet::new(); + } + + // use stable sort to preserve the insertion order. + inputs.sort(); + BTreeSet::from_sorted_iter(inputs.into_iter(), Global) + } +} + +impl BTreeSet { + fn from_sorted_iter>(iter: I, alloc: A) -> BTreeSet { + let iter = iter.map(|k| (k, SetValZST::default())); + let map = BTreeMap::bulk_build_from_sorted_iter(iter, alloc); + BTreeSet { map } + } +} + +#[stable(feature = "std_collections_from_array", since = "1.56.0")] +impl From<[T; N]> for BTreeSet { + /// Converts a `[T; N]` into a `BTreeSet`. + /// + /// ``` + /// use std::collections::BTreeSet; + /// + /// let set1 = BTreeSet::from([1, 2, 3, 4]); + /// let set2: BTreeSet<_> = [1, 2, 3, 4].into(); + /// assert_eq!(set1, set2); + /// ``` + fn from(mut arr: [T; N]) -> Self { + if N == 0 { + return BTreeSet::new(); + } + + // use stable sort to preserve the insertion order. + arr.sort(); + let iter = IntoIterator::into_iter(arr).map(|k| (k, SetValZST::default())); + let map = BTreeMap::bulk_build_from_sorted_iter(iter, Global); + BTreeSet { map } } } #[stable(feature = "rust1", since = "1.0.0")] -impl IntoIterator for BTreeSet { +impl IntoIterator for BTreeSet { type Item = T; - type IntoIter = IntoIter; + type IntoIter = IntoIter; /// Gets an iterator for moving out the `BTreeSet`'s contents. /// @@ -1035,18 +1262,18 @@ impl IntoIterator for BTreeSet { /// ``` /// use std::collections::BTreeSet; /// - /// let set: BTreeSet = [1, 2, 3, 4].iter().cloned().collect(); + /// let set = BTreeSet::from([1, 2, 3, 4]); /// /// let v: Vec<_> = set.into_iter().collect(); /// assert_eq!(v, [1, 2, 3, 4]); /// ``` - fn into_iter(self) -> IntoIter { + fn into_iter(self) -> IntoIter { IntoIter { iter: self.map.into_iter() } } } #[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T> IntoIterator for &'a BTreeSet { +impl<'a, T, A: Allocator + Clone> IntoIterator for &'a BTreeSet { type Item = &'a T; type IntoIter = Iter<'a, T>; @@ -1055,34 +1282,106 @@ impl<'a, T> IntoIterator for &'a BTreeSet { } } +/// An iterator produced by calling `drain_filter` on BTreeSet. +#[unstable(feature = "btree_drain_filter", issue = "70530")] +pub struct DrainFilter< + 'a, + T, + F, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + Clone = Global, +> where + T: 'a, + F: 'a + FnMut(&T) -> bool, +{ + pred: F, + inner: super::map::DrainFilterInner<'a, T, SetValZST>, + /// The BTreeMap will outlive this IntoIter so we don't care about drop order for `alloc`. + alloc: A, +} + +#[unstable(feature = "btree_drain_filter", issue = "70530")] +impl Drop for DrainFilter<'_, T, F, A> +where + F: FnMut(&T) -> bool, +{ + fn drop(&mut self) { + self.for_each(drop); + } +} + +#[unstable(feature = "btree_drain_filter", issue = "70530")] +impl fmt::Debug for DrainFilter<'_, T, F, A> +where + T: fmt::Debug, + F: FnMut(&T) -> bool, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("DrainFilter").field(&self.inner.peek().map(|(k, _)| k)).finish() + } +} + +#[unstable(feature = "btree_drain_filter", issue = "70530")] +impl<'a, T, F, A: Allocator + Clone> Iterator for DrainFilter<'_, T, F, A> +where + F: 'a + FnMut(&T) -> bool, +{ + type Item = T; + + fn next(&mut self) -> Option { + let pred = &mut self.pred; + let mut mapped_pred = |k: &T, _v: &mut SetValZST| pred(k); + self.inner.next(&mut mapped_pred, self.alloc.clone()).map(|(k, _)| k) + } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} + +#[unstable(feature = "btree_drain_filter", issue = "70530")] +impl FusedIterator for DrainFilter<'_, T, F, A> where + F: FnMut(&T) -> bool +{ +} + #[stable(feature = "rust1", since = "1.0.0")] -impl Extend for BTreeSet { +impl Extend for BTreeSet { #[inline] fn extend>(&mut self, iter: Iter) { iter.into_iter().for_each(move |elem| { self.insert(elem); }); } + + #[inline] + fn extend_one(&mut self, elem: T) { + self.insert(elem); + } } #[stable(feature = "extend_ref", since = "1.2.0")] -impl<'a, T: 'a + Ord + Copy> Extend<&'a T> for BTreeSet { +impl<'a, T: 'a + Ord + Copy, A: Allocator + Clone> Extend<&'a T> for BTreeSet { fn extend>(&mut self, iter: I) { self.extend(iter.into_iter().cloned()); } + + #[inline] + fn extend_one(&mut self, &elem: &'a T) { + self.insert(elem); + } } #[stable(feature = "rust1", since = "1.0.0")] -impl Default for BTreeSet { - /// Makes an empty `BTreeSet` with a reasonable choice of B. +impl Default for BTreeSet { + /// Creates an empty `BTreeSet`. fn default() -> BTreeSet { BTreeSet::new() } } #[stable(feature = "rust1", since = "1.0.0")] -impl Sub<&BTreeSet> for &BTreeSet { - type Output = BTreeSet; +impl Sub<&BTreeSet> for &BTreeSet { + type Output = BTreeSet; /// Returns the difference of `self` and `rhs` as a new `BTreeSet`. /// @@ -1091,21 +1390,23 @@ impl Sub<&BTreeSet> for &BTreeSet { /// ``` /// use std::collections::BTreeSet; /// - /// let a: BTreeSet<_> = vec![1, 2, 3].into_iter().collect(); - /// let b: BTreeSet<_> = vec![3, 4, 5].into_iter().collect(); + /// let a = BTreeSet::from([1, 2, 3]); + /// let b = BTreeSet::from([3, 4, 5]); /// /// let result = &a - &b; - /// let result_vec: Vec<_> = result.into_iter().collect(); - /// assert_eq!(result_vec, [1, 2]); + /// assert_eq!(result, BTreeSet::from([1, 2])); /// ``` - fn sub(self, rhs: &BTreeSet) -> BTreeSet { - self.difference(rhs).cloned().collect() + fn sub(self, rhs: &BTreeSet) -> BTreeSet { + BTreeSet::from_sorted_iter( + self.difference(rhs).cloned(), + ManuallyDrop::into_inner(self.map.alloc.clone()), + ) } } #[stable(feature = "rust1", since = "1.0.0")] -impl BitXor<&BTreeSet> for &BTreeSet { - type Output = BTreeSet; +impl BitXor<&BTreeSet> for &BTreeSet { + type Output = BTreeSet; /// Returns the symmetric difference of `self` and `rhs` as a new `BTreeSet`. /// @@ -1114,21 +1415,23 @@ impl BitXor<&BTreeSet> for &BTreeSet { /// ``` /// use std::collections::BTreeSet; /// - /// let a: BTreeSet<_> = vec![1, 2, 3].into_iter().collect(); - /// let b: BTreeSet<_> = vec![2, 3, 4].into_iter().collect(); + /// let a = BTreeSet::from([1, 2, 3]); + /// let b = BTreeSet::from([2, 3, 4]); /// /// let result = &a ^ &b; - /// let result_vec: Vec<_> = result.into_iter().collect(); - /// assert_eq!(result_vec, [1, 4]); + /// assert_eq!(result, BTreeSet::from([1, 4])); /// ``` - fn bitxor(self, rhs: &BTreeSet) -> BTreeSet { - self.symmetric_difference(rhs).cloned().collect() + fn bitxor(self, rhs: &BTreeSet) -> BTreeSet { + BTreeSet::from_sorted_iter( + self.symmetric_difference(rhs).cloned(), + ManuallyDrop::into_inner(self.map.alloc.clone()), + ) } } #[stable(feature = "rust1", since = "1.0.0")] -impl BitAnd<&BTreeSet> for &BTreeSet { - type Output = BTreeSet; +impl BitAnd<&BTreeSet> for &BTreeSet { + type Output = BTreeSet; /// Returns the intersection of `self` and `rhs` as a new `BTreeSet`. /// @@ -1137,21 +1440,23 @@ impl BitAnd<&BTreeSet> for &BTreeSet { /// ``` /// use std::collections::BTreeSet; /// - /// let a: BTreeSet<_> = vec![1, 2, 3].into_iter().collect(); - /// let b: BTreeSet<_> = vec![2, 3, 4].into_iter().collect(); + /// let a = BTreeSet::from([1, 2, 3]); + /// let b = BTreeSet::from([2, 3, 4]); /// /// let result = &a & &b; - /// let result_vec: Vec<_> = result.into_iter().collect(); - /// assert_eq!(result_vec, [2, 3]); + /// assert_eq!(result, BTreeSet::from([2, 3])); /// ``` - fn bitand(self, rhs: &BTreeSet) -> BTreeSet { - self.intersection(rhs).cloned().collect() + fn bitand(self, rhs: &BTreeSet) -> BTreeSet { + BTreeSet::from_sorted_iter( + self.intersection(rhs).cloned(), + ManuallyDrop::into_inner(self.map.alloc.clone()), + ) } } #[stable(feature = "rust1", since = "1.0.0")] -impl BitOr<&BTreeSet> for &BTreeSet { - type Output = BTreeSet; +impl BitOr<&BTreeSet> for &BTreeSet { + type Output = BTreeSet; /// Returns the union of `self` and `rhs` as a new `BTreeSet`. /// @@ -1160,20 +1465,22 @@ impl BitOr<&BTreeSet> for &BTreeSet { /// ``` /// use std::collections::BTreeSet; /// - /// let a: BTreeSet<_> = vec![1, 2, 3].into_iter().collect(); - /// let b: BTreeSet<_> = vec![3, 4, 5].into_iter().collect(); + /// let a = BTreeSet::from([1, 2, 3]); + /// let b = BTreeSet::from([3, 4, 5]); /// /// let result = &a | &b; - /// let result_vec: Vec<_> = result.into_iter().collect(); - /// assert_eq!(result_vec, [1, 2, 3, 4, 5]); + /// assert_eq!(result, BTreeSet::from([1, 2, 3, 4, 5])); /// ``` - fn bitor(self, rhs: &BTreeSet) -> BTreeSet { - self.union(rhs).cloned().collect() + fn bitor(self, rhs: &BTreeSet) -> BTreeSet { + BTreeSet::from_sorted_iter( + self.union(rhs).cloned(), + ManuallyDrop::into_inner(self.map.alloc.clone()), + ) } } #[stable(feature = "rust1", since = "1.0.0")] -impl Debug for BTreeSet { +impl Debug for BTreeSet { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_set().entries(self.iter()).finish() } @@ -1192,12 +1499,22 @@ impl<'a, T> Iterator for Iter<'a, T> { fn next(&mut self) -> Option<&'a T> { self.iter.next() } + fn size_hint(&self) -> (usize, Option) { self.iter.size_hint() } + fn last(mut self) -> Option<&'a T> { self.next_back() } + + fn min(mut self) -> Option<&'a T> { + self.next() + } + + fn max(mut self) -> Option<&'a T> { + self.next_back() + } } #[stable(feature = "rust1", since = "1.0.0")] impl<'a, T> DoubleEndedIterator for Iter<'a, T> { @@ -1216,31 +1533,32 @@ impl ExactSizeIterator for Iter<'_, T> { impl FusedIterator for Iter<'_, T> {} #[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for IntoIter { +impl Iterator for IntoIter { type Item = T; fn next(&mut self) -> Option { self.iter.next().map(|(k, _)| k) } + fn size_hint(&self) -> (usize, Option) { self.iter.size_hint() } } #[stable(feature = "rust1", since = "1.0.0")] -impl DoubleEndedIterator for IntoIter { +impl DoubleEndedIterator for IntoIter { fn next_back(&mut self) -> Option { self.iter.next_back().map(|(k, _)| k) } } #[stable(feature = "rust1", since = "1.0.0")] -impl ExactSizeIterator for IntoIter { +impl ExactSizeIterator for IntoIter { fn len(&self) -> usize { self.iter.len() } } #[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for IntoIter {} +impl FusedIterator for IntoIter {} #[stable(feature = "btree_range", since = "1.17.0")] impl Clone for Range<'_, T> { @@ -1260,6 +1578,14 @@ impl<'a, T> Iterator for Range<'a, T> { fn last(mut self) -> Option<&'a T> { self.next_back() } + + fn min(mut self) -> Option<&'a T> { + self.next() + } + + fn max(mut self) -> Option<&'a T> { + self.next_back() + } } #[stable(feature = "btree_range", since = "1.17.0")] @@ -1273,7 +1599,7 @@ impl<'a, T> DoubleEndedIterator for Range<'a, T> { impl FusedIterator for Range<'_, T> {} #[stable(feature = "rust1", since = "1.0.0")] -impl Clone for Difference<'_, T> { +impl Clone for Difference<'_, T, A> { fn clone(&self) -> Self { Difference { inner: match &self.inner { @@ -1290,7 +1616,7 @@ impl Clone for Difference<'_, T> { } } #[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T: Ord> Iterator for Difference<'a, T> { +impl<'a, T: Ord, A: Allocator + Clone> Iterator for Difference<'a, T, A> { type Item = &'a T; fn next(&mut self) -> Option<&'a T> { @@ -1330,10 +1656,14 @@ impl<'a, T: Ord> Iterator for Difference<'a, T> { }; (self_len.saturating_sub(other_len), Some(self_len)) } + + fn min(mut self) -> Option<&'a T> { + self.next() + } } #[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Difference<'_, T> {} +impl FusedIterator for Difference<'_, T, A> {} #[stable(feature = "rust1", since = "1.0.0")] impl Clone for SymmetricDifference<'_, T> { @@ -1347,7 +1677,7 @@ impl<'a, T: Ord> Iterator for SymmetricDifference<'a, T> { fn next(&mut self) -> Option<&'a T> { loop { - let (a_next, b_next) = self.0.nexts(); + let (a_next, b_next) = self.0.nexts(Self::Item::cmp); if a_next.and(b_next).is_none() { return a_next.or(b_next); } @@ -1357,17 +1687,21 @@ impl<'a, T: Ord> Iterator for SymmetricDifference<'a, T> { fn size_hint(&self) -> (usize, Option) { let (a_len, b_len) = self.0.lens(); // No checked_add, because even if a and b refer to the same set, - // and T is an empty type, the storage overhead of sets limits + // and T is a zero-sized type, the storage overhead of sets limits // the number of elements to less than half the range of usize. (0, Some(a_len + b_len)) } + + fn min(mut self) -> Option<&'a T> { + self.next() + } } #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for SymmetricDifference<'_, T> {} #[stable(feature = "rust1", since = "1.0.0")] -impl Clone for Intersection<'_, T> { +impl Clone for Intersection<'_, T, A> { fn clone(&self) -> Self { Intersection { inner: match &self.inner { @@ -1383,7 +1717,7 @@ impl Clone for Intersection<'_, T> { } } #[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T: Ord> Iterator for Intersection<'a, T> { +impl<'a, T: Ord, A: Allocator + Clone> Iterator for Intersection<'a, T, A> { type Item = &'a T; fn next(&mut self) -> Option<&'a T> { @@ -1417,10 +1751,14 @@ impl<'a, T: Ord> Iterator for Intersection<'a, T> { IntersectionInner::Answer(Some(_)) => (1, Some(1)), } } + + fn min(mut self) -> Option<&'a T> { + self.next() + } } #[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Intersection<'_, T> {} +impl FusedIterator for Intersection<'_, T, A> {} #[stable(feature = "rust1", since = "1.0.0")] impl Clone for Union<'_, T> { @@ -1433,7 +1771,7 @@ impl<'a, T: Ord> Iterator for Union<'a, T> { type Item = &'a T; fn next(&mut self) -> Option<&'a T> { - let (a_next, b_next) = self.0.nexts(); + let (a_next, b_next) = self.0.nexts(Self::Item::cmp); a_next.or(b_next) } @@ -1442,7 +1780,14 @@ impl<'a, T: Ord> Iterator for Union<'a, T> { // No checked_add - see SymmetricDifference::size_hint. (max(a_len, b_len), Some(a_len + b_len)) } + + fn min(mut self) -> Option<&'a T> { + self.next() + } } #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for Union<'_, T> {} + +#[cfg(test)] +mod tests; diff --git a/crux-mir/lib/alloc/tests/btree/set.rs b/crux-mir/lib/alloc/src/collections/btree/set/tests.rs similarity index 55% rename from crux-mir/lib/alloc/tests/btree/set.rs rename to crux-mir/lib/alloc/src/collections/btree/set/tests.rs index 1a2b62d02..7b8d41a60 100644 --- a/crux-mir/lib/alloc/tests/btree/set.rs +++ b/crux-mir/lib/alloc/src/collections/btree/set/tests.rs @@ -1,7 +1,12 @@ -use std::collections::BTreeSet; +use super::*; +use crate::testing::crash_test::{CrashTestDummy, Panic}; +use crate::testing::rng::DeterministicRng; +use crate::vec::Vec; +use std::cmp::Ordering; +use std::hash::{Hash, Hasher}; use std::iter::FromIterator; - -use super::DeterministicRng; +use std::ops::Bound::{Excluded, Included}; +use std::panic::{catch_unwind, AssertUnwindSafe}; #[test] fn test_clone_eq() { @@ -14,21 +19,34 @@ fn test_clone_eq() { } #[test] -fn test_hash() { - use crate::hash; - - let mut x = BTreeSet::new(); - let mut y = BTreeSet::new(); - - x.insert(1); - x.insert(2); - x.insert(3); - - y.insert(3); - y.insert(2); - y.insert(1); - - assert_eq!(hash(&x), hash(&y)); +fn test_iter_min_max() { + let mut a = BTreeSet::new(); + assert_eq!(a.iter().min(), None); + assert_eq!(a.iter().max(), None); + assert_eq!(a.range(..).min(), None); + assert_eq!(a.range(..).max(), None); + assert_eq!(a.difference(&BTreeSet::new()).min(), None); + assert_eq!(a.difference(&BTreeSet::new()).max(), None); + assert_eq!(a.intersection(&a).min(), None); + assert_eq!(a.intersection(&a).max(), None); + assert_eq!(a.symmetric_difference(&BTreeSet::new()).min(), None); + assert_eq!(a.symmetric_difference(&BTreeSet::new()).max(), None); + assert_eq!(a.union(&a).min(), None); + assert_eq!(a.union(&a).max(), None); + a.insert(1); + a.insert(2); + assert_eq!(a.iter().min(), Some(&1)); + assert_eq!(a.iter().max(), Some(&2)); + assert_eq!(a.range(..).min(), Some(&1)); + assert_eq!(a.range(..).max(), Some(&2)); + assert_eq!(a.difference(&BTreeSet::new()).min(), Some(&1)); + assert_eq!(a.difference(&BTreeSet::new()).max(), Some(&2)); + assert_eq!(a.intersection(&a).min(), Some(&1)); + assert_eq!(a.intersection(&a).max(), Some(&2)); + assert_eq!(a.symmetric_difference(&BTreeSet::new()).min(), Some(&1)); + assert_eq!(a.symmetric_difference(&BTreeSet::new()).max(), Some(&2)); + assert_eq!(a.union(&a).min(), Some(&1)); + assert_eq!(a.union(&a).max(), Some(&2)); } fn check(a: &[i32], b: &[i32], expected: &[i32], f: F) @@ -74,7 +92,7 @@ fn test_intersection() { return; } - let large = (0..100).collect::>(); + let large = Vec::from_iter(0..100); check_intersection(&[], &large, &[]); check_intersection(&large, &[], &[]); check_intersection(&[-1], &large, &[]); @@ -90,8 +108,8 @@ fn test_intersection() { #[test] fn test_intersection_size_hint() { - let x: BTreeSet = [3, 4].iter().copied().collect(); - let y: BTreeSet = [1, 2, 3].iter().copied().collect(); + let x = BTreeSet::from([3, 4]); + let y = BTreeSet::from([1, 2, 3]); let mut iter = x.intersection(&y); assert_eq!(iter.size_hint(), (1, Some(1))); assert_eq!(iter.next(), Some(&3)); @@ -128,7 +146,7 @@ fn test_difference() { return; } - let large = (0..100).collect::>(); + let large = Vec::from_iter(0..100); check_difference(&[], &large, &[]); check_difference(&[-1], &large, &[-1]); check_difference(&[0], &large, &[]); @@ -142,43 +160,43 @@ fn test_difference() { #[test] fn test_difference_size_hint() { - let s246: BTreeSet = [2, 4, 6].iter().copied().collect(); - let s23456: BTreeSet = (2..=6).collect(); + let s246 = BTreeSet::from([2, 4, 6]); + let s23456 = BTreeSet::from_iter(2..=6); let mut iter = s246.difference(&s23456); assert_eq!(iter.size_hint(), (0, Some(3))); assert_eq!(iter.next(), None); - let s12345: BTreeSet = (1..=5).collect(); + let s12345 = BTreeSet::from_iter(1..=5); iter = s246.difference(&s12345); assert_eq!(iter.size_hint(), (0, Some(3))); assert_eq!(iter.next(), Some(&6)); assert_eq!(iter.size_hint(), (0, Some(0))); assert_eq!(iter.next(), None); - let s34567: BTreeSet = (3..=7).collect(); + let s34567 = BTreeSet::from_iter(3..=7); iter = s246.difference(&s34567); assert_eq!(iter.size_hint(), (0, Some(3))); assert_eq!(iter.next(), Some(&2)); assert_eq!(iter.size_hint(), (0, Some(2))); assert_eq!(iter.next(), None); - let s1: BTreeSet = (-9..=1).collect(); + let s1 = BTreeSet::from_iter(-9..=1); iter = s246.difference(&s1); assert_eq!(iter.size_hint(), (3, Some(3))); - let s2: BTreeSet = (-9..=2).collect(); + let s2 = BTreeSet::from_iter(-9..=2); iter = s246.difference(&s2); assert_eq!(iter.size_hint(), (2, Some(2))); assert_eq!(iter.next(), Some(&4)); assert_eq!(iter.size_hint(), (1, Some(1))); - let s23: BTreeSet = (2..=3).collect(); + let s23 = BTreeSet::from([2, 3]); iter = s246.difference(&s23); assert_eq!(iter.size_hint(), (1, Some(3))); assert_eq!(iter.next(), Some(&4)); assert_eq!(iter.size_hint(), (1, Some(1))); - let s4: BTreeSet = (4..=4).collect(); + let s4 = BTreeSet::from([4]); iter = s246.difference(&s4); assert_eq!(iter.size_hint(), (2, Some(3))); assert_eq!(iter.next(), Some(&2)); @@ -187,19 +205,19 @@ fn test_difference_size_hint() { assert_eq!(iter.size_hint(), (0, Some(0))); assert_eq!(iter.next(), None); - let s56: BTreeSet = (5..=6).collect(); + let s56 = BTreeSet::from([5, 6]); iter = s246.difference(&s56); assert_eq!(iter.size_hint(), (1, Some(3))); assert_eq!(iter.next(), Some(&2)); assert_eq!(iter.size_hint(), (0, Some(2))); - let s6: BTreeSet = (6..=19).collect(); + let s6 = BTreeSet::from_iter(6..=19); iter = s246.difference(&s6); assert_eq!(iter.size_hint(), (2, Some(2))); assert_eq!(iter.next(), Some(&2)); assert_eq!(iter.size_hint(), (1, Some(1))); - let s7: BTreeSet = (7..=19).collect(); + let s7 = BTreeSet::from_iter(7..=19); iter = s246.difference(&s7); assert_eq!(iter.size_hint(), (3, Some(3))); } @@ -218,8 +236,8 @@ fn test_symmetric_difference() { #[test] fn test_symmetric_difference_size_hint() { - let x: BTreeSet = [2, 4].iter().copied().collect(); - let y: BTreeSet = [1, 2, 3].iter().copied().collect(); + let x = BTreeSet::from([2, 4]); + let y = BTreeSet::from([1, 2, 3]); let mut iter = x.symmetric_difference(&y); assert_eq!(iter.size_hint(), (0, Some(5))); assert_eq!(iter.next(), Some(&1)); @@ -246,8 +264,8 @@ fn test_union() { #[test] fn test_union_size_hint() { - let x: BTreeSet = [2, 4].iter().copied().collect(); - let y: BTreeSet = [1, 2, 3].iter().copied().collect(); + let x = BTreeSet::from([2, 4]); + let y = BTreeSet::from([1, 2, 3]); let mut iter = x.union(&y); assert_eq!(iter.size_hint(), (3, Some(5))); assert_eq!(iter.next(), Some(&1)); @@ -259,8 +277,8 @@ fn test_union_size_hint() { #[test] // Only tests the simple function definition with respect to intersection fn test_is_disjoint() { - let one = [1].iter().collect::>(); - let two = [2].iter().collect::>(); + let one = BTreeSet::from([1]); + let two = BTreeSet::from([2]); assert!(one.is_disjoint(&two)); } @@ -268,8 +286,8 @@ fn test_is_disjoint() { // Also implicitly tests the trivial function definition of is_superset fn test_is_subset() { fn is_subset(a: &[i32], b: &[i32]) -> bool { - let set_a = a.iter().collect::>(); - let set_b = b.iter().collect::>(); + let set_a = BTreeSet::from_iter(a.iter()); + let set_b = BTreeSet::from_iter(b.iter()); set_a.is_subset(&set_b) } @@ -293,7 +311,7 @@ fn test_is_subset() { return; } - let large = (0..100).collect::>(); + let large = Vec::from_iter(0..100); assert_eq!(is_subset(&[], &large), true); assert_eq!(is_subset(&large, &[]), false); assert_eq!(is_subset(&[-1], &large), false); @@ -302,6 +320,106 @@ fn test_is_subset() { assert_eq!(is_subset(&[99, 100], &large), false); } +#[test] +fn test_is_superset() { + fn is_superset(a: &[i32], b: &[i32]) -> bool { + let set_a = BTreeSet::from_iter(a.iter()); + let set_b = BTreeSet::from_iter(b.iter()); + set_a.is_superset(&set_b) + } + + assert_eq!(is_superset(&[], &[]), true); + assert_eq!(is_superset(&[], &[1, 2]), false); + assert_eq!(is_superset(&[0], &[1, 2]), false); + assert_eq!(is_superset(&[1], &[1, 2]), false); + assert_eq!(is_superset(&[4], &[1, 2]), false); + assert_eq!(is_superset(&[1, 4], &[1, 2]), false); + assert_eq!(is_superset(&[1, 2], &[1, 2]), true); + assert_eq!(is_superset(&[1, 2, 3], &[1, 3]), true); + assert_eq!(is_superset(&[1, 2, 3], &[]), true); + assert_eq!(is_superset(&[-1, 1, 2, 3], &[-1, 3]), true); + + if cfg!(miri) { + // Miri is too slow + return; + } + + let large = Vec::from_iter(0..100); + assert_eq!(is_superset(&[], &large), false); + assert_eq!(is_superset(&large, &[]), true); + assert_eq!(is_superset(&large, &[1]), true); + assert_eq!(is_superset(&large, &[50, 99]), true); + assert_eq!(is_superset(&large, &[100]), false); + assert_eq!(is_superset(&large, &[0, 99]), true); + assert_eq!(is_superset(&[-1], &large), false); + assert_eq!(is_superset(&[0], &large), false); + assert_eq!(is_superset(&[99, 100], &large), false); +} + +#[test] +fn test_retain() { + let mut set = BTreeSet::from([1, 2, 3, 4, 5, 6]); + set.retain(|&k| k % 2 == 0); + assert_eq!(set.len(), 3); + assert!(set.contains(&2)); + assert!(set.contains(&4)); + assert!(set.contains(&6)); +} + +#[test] +fn test_drain_filter() { + let mut x = BTreeSet::from([1]); + let mut y = BTreeSet::from([1]); + + x.drain_filter(|_| true); + y.drain_filter(|_| false); + assert_eq!(x.len(), 0); + assert_eq!(y.len(), 1); +} + +#[test] +fn test_drain_filter_drop_panic_leak() { + let a = CrashTestDummy::new(0); + let b = CrashTestDummy::new(1); + let c = CrashTestDummy::new(2); + let mut set = BTreeSet::new(); + set.insert(a.spawn(Panic::Never)); + set.insert(b.spawn(Panic::InDrop)); + set.insert(c.spawn(Panic::Never)); + + catch_unwind(move || drop(set.drain_filter(|dummy| dummy.query(true)))).ok(); + + assert_eq!(a.queried(), 1); + assert_eq!(b.queried(), 1); + assert_eq!(c.queried(), 0); + assert_eq!(a.dropped(), 1); + assert_eq!(b.dropped(), 1); + assert_eq!(c.dropped(), 1); +} + +#[test] +fn test_drain_filter_pred_panic_leak() { + let a = CrashTestDummy::new(0); + let b = CrashTestDummy::new(1); + let c = CrashTestDummy::new(2); + let mut set = BTreeSet::new(); + set.insert(a.spawn(Panic::Never)); + set.insert(b.spawn(Panic::InQuery)); + set.insert(c.spawn(Panic::InQuery)); + + catch_unwind(AssertUnwindSafe(|| drop(set.drain_filter(|dummy| dummy.query(true))))).ok(); + + assert_eq!(a.queried(), 1); + assert_eq!(b.queried(), 1); + assert_eq!(c.queried(), 0); + assert_eq!(a.dropped(), 1); + assert_eq!(b.dropped(), 0); + assert_eq!(c.dropped(), 0); + assert_eq!(set.len(), 2); + assert_eq!(set.first().unwrap().id(), 1); + assert_eq!(set.last().unwrap().id(), 2); +} + #[test] fn test_clear() { let mut x = BTreeSet::new(); @@ -310,6 +428,26 @@ fn test_clear() { x.clear(); assert!(x.is_empty()); } +#[test] +fn test_remove() { + let mut x = BTreeSet::new(); + assert!(x.is_empty()); + + x.insert(1); + x.insert(2); + x.insert(3); + x.insert(4); + + assert_eq!(x.remove(&2), true); + assert_eq!(x.remove(&0), false); + assert_eq!(x.remove(&5), false); + assert_eq!(x.remove(&1), true); + assert_eq!(x.remove(&2), false); + assert_eq!(x.remove(&3), true); + assert_eq!(x.remove(&4), true); + assert_eq!(x.remove(&4), false); + assert!(x.is_empty()); +} #[test] fn test_zip() { @@ -335,7 +473,7 @@ fn test_zip() { fn test_from_iter() { let xs = [1, 2, 3, 4, 5, 6, 7, 8, 9]; - let set: BTreeSet<_> = xs.iter().cloned().collect(); + let set = BTreeSet::from_iter(xs.iter()); for x in &xs { assert!(set.contains(x)); @@ -350,10 +488,10 @@ fn test_show() { set.insert(1); set.insert(2); - let set_str = format!("{:?}", set); + let set_str = format!("{set:?}"); assert_eq!(set_str, "{1, 2}"); - assert_eq!(format!("{:?}", empty), "{}"); + assert_eq!(format!("{empty:?}"), "{}"); } #[test] @@ -386,8 +524,6 @@ fn test_extend_ref() { #[test] fn test_recovery() { - use std::cmp::Ordering; - #[derive(Debug)] struct Foo(&'static str, i32); @@ -433,11 +569,8 @@ fn test_recovery() { assert_eq!(s.iter().next(), None); } -#[test] #[allow(dead_code)] -fn test_variance() { - use std::collections::btree_set::{IntoIter, Iter, Range}; - +fn assert_covariance() { fn set<'new>(v: BTreeSet<&'static str>) -> BTreeSet<&'new str> { v } @@ -450,6 +583,143 @@ fn test_variance() { fn range<'a, 'new>(v: Range<'a, &'static str>) -> Range<'a, &'new str> { v } + // not applied to Difference, Intersection, SymmetricDifference, Union +} + +#[allow(dead_code)] +fn assert_sync() { + fn set(v: &BTreeSet) -> impl Sync + '_ { + v + } + + fn iter(v: &BTreeSet) -> impl Sync + '_ { + v.iter() + } + + fn into_iter(v: BTreeSet) -> impl Sync { + v.into_iter() + } + + fn range(v: &BTreeSet) -> impl Sync + '_ { + v.range(..) + } + + fn drain_filter(v: &mut BTreeSet) -> impl Sync + '_ { + v.drain_filter(|_| false) + } + + fn difference(v: &BTreeSet) -> impl Sync + '_ { + v.difference(&v) + } + + fn intersection(v: &BTreeSet) -> impl Sync + '_ { + v.intersection(&v) + } + + fn symmetric_difference(v: &BTreeSet) -> impl Sync + '_ { + v.symmetric_difference(&v) + } + + fn union(v: &BTreeSet) -> impl Sync + '_ { + v.union(&v) + } +} + +#[allow(dead_code)] +fn assert_send() { + fn set(v: BTreeSet) -> impl Send { + v + } + + fn iter(v: &BTreeSet) -> impl Send + '_ { + v.iter() + } + + fn into_iter(v: BTreeSet) -> impl Send { + v.into_iter() + } + + fn range(v: &BTreeSet) -> impl Send + '_ { + v.range(..) + } + + fn drain_filter(v: &mut BTreeSet) -> impl Send + '_ { + v.drain_filter(|_| false) + } + + fn difference(v: &BTreeSet) -> impl Send + '_ { + v.difference(&v) + } + + fn intersection(v: &BTreeSet) -> impl Send + '_ { + v.intersection(&v) + } + + fn symmetric_difference(v: &BTreeSet) -> impl Send + '_ { + v.symmetric_difference(&v) + } + + fn union(v: &BTreeSet) -> impl Send + '_ { + v.union(&v) + } +} + +#[allow(dead_code)] +// Check that the member-like functions conditionally provided by #[derive()] +// are not overridden by genuine member functions with a different signature. +fn assert_derives() { + fn hash(v: BTreeSet, state: &mut H) { + v.hash(state); + // Tested much more thoroughly outside the crate in btree_set_hash.rs + } + fn eq(v: BTreeSet) { + let _ = v.eq(&v); + } + fn ne(v: BTreeSet) { + let _ = v.ne(&v); + } + fn cmp(v: BTreeSet) { + let _ = v.cmp(&v); + } + fn min(v: BTreeSet, w: BTreeSet) { + let _ = v.min(w); + } + fn max(v: BTreeSet, w: BTreeSet) { + let _ = v.max(w); + } + fn clamp(v: BTreeSet, w: BTreeSet, x: BTreeSet) { + let _ = v.clamp(w, x); + } + fn partial_cmp(v: &BTreeSet) { + let _ = v.partial_cmp(&v); + } +} + +#[test] +fn test_ord_absence() { + fn set(mut set: BTreeSet) { + let _ = set.is_empty(); + let _ = set.len(); + set.clear(); + let _ = set.iter(); + let _ = set.into_iter(); + } + + fn set_debug(set: BTreeSet) { + format!("{set:?}"); + format!("{:?}", set.iter()); + format!("{:?}", set.into_iter()); + } + + fn set_clone(mut set: BTreeSet) { + set.clone_from(&set.clone()); + } + + #[derive(Debug, Clone)] + struct NonOrd; + set(BTreeSet::::new()); + set_debug(BTreeSet::::new()); + set_clone(BTreeSet::::default()); } #[test] @@ -509,6 +779,9 @@ fn test_first_last() { assert_eq!(a.pop_last(), None); } +// Unlike the function with the same name in map/tests, returns no values. +// Which also means it returns different predetermined pseudo-random keys, +// and the test cases using this function explore slightly different trees. fn rand_data(len: usize) -> Vec { let mut rng = DeterministicRng::new(); Vec::from_iter((0..len).map(|_| rng.next())) @@ -540,10 +813,8 @@ fn test_split_off_empty_left() { #[test] fn test_split_off_large_random_sorted() { - #[cfg(not(miri))] // Miri is too slow - let mut data = rand_data(1529); - #[cfg(miri)] - let mut data = rand_data(529); + // Miri is too slow + let mut data = if cfg!(miri) { rand_data(529) } else { rand_data(1529) }; // special case with maximum height. data.sort(); @@ -554,3 +825,32 @@ fn test_split_off_large_random_sorted() { assert!(set.into_iter().eq(data.clone().into_iter().filter(|x| *x < key))); assert!(right.into_iter().eq(data.into_iter().filter(|x| *x >= key))); } + +#[test] +fn from_array() { + let set = BTreeSet::from([1, 2, 3, 4]); + let unordered_duplicates = BTreeSet::from([4, 1, 4, 3, 2]); + assert_eq!(set, unordered_duplicates); +} + +#[should_panic(expected = "range start is greater than range end in BTreeSet")] +#[test] +fn test_range_panic_1() { + let mut set = BTreeSet::new(); + set.insert(3); + set.insert(5); + set.insert(8); + + let _invalid_range = set.range((Included(&8), Included(&3))); +} + +#[should_panic(expected = "range start and end are equal and excluded in BTreeSet")] +#[test] +fn test_range_panic_2() { + let mut set = BTreeSet::new(); + set.insert(3); + set.insert(5); + set.insert(8); + + let _invalid_range = set.range((Excluded(&5), Excluded(&5))); +} diff --git a/crux-mir/lib/alloc/src/collections/btree/set_val.rs b/crux-mir/lib/alloc/src/collections/btree/set_val.rs new file mode 100644 index 000000000..80c459bcf --- /dev/null +++ b/crux-mir/lib/alloc/src/collections/btree/set_val.rs @@ -0,0 +1,29 @@ +/// Zero-Sized Type (ZST) for internal `BTreeSet` values. +/// Used instead of `()` to differentiate between: +/// * `BTreeMap` (possible user-defined map) +/// * `BTreeMap` (internal set representation) +#[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Clone, Default)] +pub struct SetValZST; + +/// A trait to differentiate between `BTreeMap` and `BTreeSet` values. +/// Returns `true` only for type `SetValZST`, `false` for all other types (blanket implementation). +/// `TypeId` requires a `'static` lifetime, use of this trait avoids that restriction. +/// +/// [`TypeId`]: std::any::TypeId +pub trait IsSetVal { + fn is_set_val() -> bool; +} + +// Blanket implementation +impl IsSetVal for V { + default fn is_set_val() -> bool { + false + } +} + +// Specialization +impl IsSetVal for SetValZST { + fn is_set_val() -> bool { + true + } +} diff --git a/crux-mir/lib/alloc/src/collections/btree/split.rs b/crux-mir/lib/alloc/src/collections/btree/split.rs new file mode 100644 index 000000000..638dc98fc --- /dev/null +++ b/crux-mir/lib/alloc/src/collections/btree/split.rs @@ -0,0 +1,73 @@ +use super::node::{ForceResult::*, Root}; +use super::search::SearchResult::*; +use core::alloc::Allocator; +use core::borrow::Borrow; + +impl Root { + /// Calculates the length of both trees that result from splitting up + /// a given number of distinct key-value pairs. + pub fn calc_split_length( + total_num: usize, + root_a: &Root, + root_b: &Root, + ) -> (usize, usize) { + let (length_a, length_b); + if root_a.height() < root_b.height() { + length_a = root_a.reborrow().calc_length(); + length_b = total_num - length_a; + debug_assert_eq!(length_b, root_b.reborrow().calc_length()); + } else { + length_b = root_b.reborrow().calc_length(); + length_a = total_num - length_b; + debug_assert_eq!(length_a, root_a.reborrow().calc_length()); + } + (length_a, length_b) + } + + /// Split off a tree with key-value pairs at and after the given key. + /// The result is meaningful only if the tree is ordered by key, + /// and if the ordering of `Q` corresponds to that of `K`. + /// If `self` respects all `BTreeMap` tree invariants, then both + /// `self` and the returned tree will respect those invariants. + pub fn split_off(&mut self, key: &Q, alloc: A) -> Self + where + K: Borrow, + { + let left_root = self; + let mut right_root = Root::new_pillar(left_root.height(), alloc.clone()); + let mut left_node = left_root.borrow_mut(); + let mut right_node = right_root.borrow_mut(); + + loop { + let mut split_edge = match left_node.search_node(key) { + // key is going to the right tree + Found(kv) => kv.left_edge(), + GoDown(edge) => edge, + }; + + split_edge.move_suffix(&mut right_node); + + match (split_edge.force(), right_node.force()) { + (Internal(edge), Internal(node)) => { + left_node = edge.descend(); + right_node = node.first_edge().descend(); + } + (Leaf(_), Leaf(_)) => break, + _ => unreachable!(), + } + } + + left_root.fix_right_border(alloc.clone()); + right_root.fix_left_border(alloc); + right_root + } + + /// Creates a tree consisting of empty nodes. + fn new_pillar(height: usize, alloc: A) -> Self { + let mut root = Root::new(alloc.clone()); + for _ in 0..height { + root.push_internal_level(alloc.clone()); + } + root + } +} diff --git a/crux-mir/lib/alloc/src/collections/linked_list.rs b/crux-mir/lib/alloc/src/collections/linked_list.rs index 53d4f7239..f2f5dffc2 100644 --- a/crux-mir/lib/alloc/src/collections/linked_list.rs +++ b/crux-mir/lib/alloc/src/collections/linked_list.rs @@ -7,8 +7,8 @@ //! array-based containers are generally faster, //! more memory efficient, and make better use of CPU cache. //! -//! [`Vec`]: ../../vec/struct.Vec.html -//! [`VecDeque`]: ../vec_deque/struct.VecDeque.html +//! [`Vec`]: crate::vec::Vec +//! [`VecDeque`]: super::vec_deque::VecDeque #![stable(feature = "rust1", since = "1.0.0")] @@ -31,10 +31,22 @@ mod tests; /// The `LinkedList` allows pushing and popping elements at either end /// in constant time. /// -/// NOTE: It is almost always better to use `Vec` or `VecDeque` because +/// A `LinkedList` with a known list of items can be initialized from an array: +/// ``` +/// use std::collections::LinkedList; +/// +/// let list = LinkedList::from([1, 2, 3]); +/// ``` +/// +/// NOTE: It is almost always better to use [`Vec`] or [`VecDeque`] because /// array-based containers are generally faster, /// more memory efficient, and make better use of CPU cache. +/// +/// [`Vec`]: crate::vec::Vec +/// [`VecDeque`]: super::vec_deque::VecDeque #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "LinkedList")] +#[rustc_insignificant_dtor] pub struct LinkedList { head: Option>>, tail: Option>>, @@ -50,11 +62,9 @@ struct Node { /// An iterator over the elements of a `LinkedList`. /// -/// This `struct` is created by the [`iter`] method on [`LinkedList`]. See its +/// This `struct` is created by [`LinkedList::iter()`]. See its /// documentation for more. -/// -/// [`iter`]: struct.LinkedList.html#method.iter -/// [`LinkedList`]: struct.LinkedList.html +#[must_use = "iterators are lazy and do nothing unless consumed"] #[stable(feature = "rust1", since = "1.0.0")] pub struct Iter<'a, T: 'a> { head: Option>>, @@ -66,7 +76,15 @@ pub struct Iter<'a, T: 'a> { #[stable(feature = "collection_debug", since = "1.17.0")] impl fmt::Debug for Iter<'_, T> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("Iter").field(&self.len).finish() + f.debug_tuple("Iter") + .field(&*mem::ManuallyDrop::new(LinkedList { + head: self.head, + tail: self.tail, + len: self.len, + marker: PhantomData, + })) + .field(&self.len) + .finish() } } @@ -80,36 +98,39 @@ impl Clone for Iter<'_, T> { /// A mutable iterator over the elements of a `LinkedList`. /// -/// This `struct` is created by the [`iter_mut`] method on [`LinkedList`]. See its +/// This `struct` is created by [`LinkedList::iter_mut()`]. See its /// documentation for more. -/// -/// [`iter_mut`]: struct.LinkedList.html#method.iter_mut -/// [`LinkedList`]: struct.LinkedList.html +#[must_use = "iterators are lazy and do nothing unless consumed"] #[stable(feature = "rust1", since = "1.0.0")] pub struct IterMut<'a, T: 'a> { - // We do *not* exclusively own the entire list here, references to node's `element` - // have been handed out by the iterator! So be careful when using this; the methods - // called must be aware that there can be aliasing pointers to `element`. - list: &'a mut LinkedList, head: Option>>, tail: Option>>, len: usize, + marker: PhantomData<&'a mut Node>, } #[stable(feature = "collection_debug", since = "1.17.0")] impl fmt::Debug for IterMut<'_, T> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("IterMut").field(&self.list).field(&self.len).finish() + f.debug_tuple("IterMut") + .field(&*mem::ManuallyDrop::new(LinkedList { + head: self.head, + tail: self.tail, + len: self.len, + marker: PhantomData, + })) + .field(&self.len) + .finish() } } /// An owning iterator over the elements of a `LinkedList`. /// /// This `struct` is created by the [`into_iter`] method on [`LinkedList`] -/// (provided by the `IntoIterator` trait). See its documentation for more. +/// (provided by the [`IntoIterator`] trait). See its documentation for more. /// -/// [`into_iter`]: struct.LinkedList.html#method.into_iter -/// [`LinkedList`]: struct.LinkedList.html +/// [`into_iter`]: LinkedList::into_iter +/// [`IntoIterator`]: core::iter::IntoIterator #[derive(Clone)] #[stable(feature = "rust1", since = "1.0.0")] pub struct IntoIter { @@ -143,7 +164,7 @@ impl LinkedList { unsafe { node.next = self.head; node.prev = None; - let node = Some(Box::into_raw_non_null(node)); + let node = Some(Box::leak(node).into()); match self.head { None => self.tail = node, @@ -184,7 +205,7 @@ impl LinkedList { unsafe { node.next = None; node.prev = self.tail; - let node = Some(Box::into_raw_non_null(node)); + let node = Some(Box::leak(node).into()); match self.tail { None => self.head = node, @@ -225,17 +246,17 @@ impl LinkedList { /// maintain validity of aliasing pointers. #[inline] unsafe fn unlink_node(&mut self, mut node: NonNull>) { - let node = node.as_mut(); // this one is ours now, we can create an &mut. + let node = unsafe { node.as_mut() }; // this one is ours now, we can create an &mut. // Not creating new mutable (unique!) references overlapping `element`. match node.prev { - Some(prev) => (*prev.as_ptr()).next = node.next, + Some(prev) => unsafe { (*prev.as_ptr()).next = node.next }, // this node is the head node None => self.head = node.next, }; match node.next { - Some(next) => (*next.as_ptr()).prev = node.prev, + Some(next) => unsafe { (*next.as_ptr()).prev = node.prev }, // this node is the tail node None => self.tail = node.prev, }; @@ -258,17 +279,23 @@ impl LinkedList { // This method takes care not to create multiple mutable references to whole nodes at the same time, // to maintain validity of aliasing pointers into `element`. if let Some(mut existing_prev) = existing_prev { - existing_prev.as_mut().next = Some(splice_start); + unsafe { + existing_prev.as_mut().next = Some(splice_start); + } } else { self.head = Some(splice_start); } if let Some(mut existing_next) = existing_next { - existing_next.as_mut().prev = Some(splice_end); + unsafe { + existing_next.as_mut().prev = Some(splice_end); + } } else { self.tail = Some(splice_end); } - splice_start.as_mut().prev = existing_prev; - splice_end.as_mut().next = existing_next; + unsafe { + splice_start.as_mut().prev = existing_prev; + splice_end.as_mut().next = existing_next; + } self.len += splice_length; } @@ -280,7 +307,10 @@ impl LinkedList { let tail = self.tail.take(); let len = mem::replace(&mut self.len, 0); if let Some(head) = head { - let tail = tail.unwrap_or_else(|| unsafe { core::hint::unreachable_unchecked() }); + // SAFETY: In a LinkedList, either both the head and tail are None because + // the list is empty, or both head and tail are Some because the list is populated. + // Since we have verified the head is Some, we are sure the tail is Some too. + let tail = unsafe { tail.unwrap_unchecked() }; Some((head, tail, len)) } else { None @@ -297,9 +327,13 @@ impl LinkedList { if let Some(mut split_node) = split_node { let first_part_head; let first_part_tail; - first_part_tail = split_node.as_mut().prev.take(); + unsafe { + first_part_tail = split_node.as_mut().prev.take(); + } if let Some(mut tail) = first_part_tail { - tail.as_mut().next = None; + unsafe { + tail.as_mut().next = None; + } first_part_head = self.head; } else { first_part_head = None; @@ -333,9 +367,13 @@ impl LinkedList { if let Some(mut split_node) = split_node { let second_part_head; let second_part_tail; - second_part_head = split_node.as_mut().next.take(); + unsafe { + second_part_head = split_node.as_mut().next.take(); + } if let Some(mut head) = second_part_head { - head.as_mut().prev = None; + unsafe { + head.as_mut().prev = None; + } second_part_tail = self.tail; } else { second_part_tail = None; @@ -379,8 +417,9 @@ impl LinkedList { /// let list: LinkedList = LinkedList::new(); /// ``` #[inline] - #[rustc_const_stable(feature = "const_linked_list_new", since = "1.32.0")] + #[rustc_const_stable(feature = "const_linked_list_new", since = "1.39.0")] #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] pub const fn new() -> Self { LinkedList { head: None, tail: None, len: 0, marker: PhantomData } } @@ -390,7 +429,7 @@ impl LinkedList { /// This reuses all the nodes from `other` and moves them into `self`. After /// this operation, `other` becomes empty. /// - /// This operation should compute in O(1) time and O(1) memory. + /// This operation should compute in *O*(1) time and *O*(1) memory. /// /// # Examples /// @@ -434,27 +473,6 @@ impl LinkedList { } } - /// Moves all elements from `other` to the begin of the list. - #[unstable(feature = "linked_list_prepend", issue = "none")] - pub fn prepend(&mut self, other: &mut Self) { - match self.head { - None => mem::swap(self, other), - Some(mut head) => { - // `as_mut` is okay here because we have exclusive access to the entirety - // of both lists. - if let Some(mut other_tail) = other.tail.take() { - unsafe { - head.as_mut().prev = Some(other_tail); - other_tail.as_mut().next = Some(head); - } - - self.head = other.head.take(); - self.len += mem::replace(&mut other.len, 0); - } - } - } - } - /// Provides a forward iterator. /// /// # Examples @@ -506,13 +524,14 @@ impl LinkedList { #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn iter_mut(&mut self) -> IterMut<'_, T> { - IterMut { head: self.head, tail: self.tail, len: self.len, list: self } + IterMut { head: self.head, tail: self.tail, len: self.len, marker: PhantomData } } /// Provides a cursor at the front element. /// /// The cursor is pointing to the "ghost" non-element if the list is empty. #[inline] + #[must_use] #[unstable(feature = "linked_list_cursors", issue = "58533")] pub fn cursor_front(&self) -> Cursor<'_, T> { Cursor { index: 0, current: self.head, list: self } @@ -522,6 +541,7 @@ impl LinkedList { /// /// The cursor is pointing to the "ghost" non-element if the list is empty. #[inline] + #[must_use] #[unstable(feature = "linked_list_cursors", issue = "58533")] pub fn cursor_front_mut(&mut self) -> CursorMut<'_, T> { CursorMut { index: 0, current: self.head, list: self } @@ -531,6 +551,7 @@ impl LinkedList { /// /// The cursor is pointing to the "ghost" non-element if the list is empty. #[inline] + #[must_use] #[unstable(feature = "linked_list_cursors", issue = "58533")] pub fn cursor_back(&self) -> Cursor<'_, T> { Cursor { index: self.len.checked_sub(1).unwrap_or(0), current: self.tail, list: self } @@ -540,6 +561,7 @@ impl LinkedList { /// /// The cursor is pointing to the "ghost" non-element if the list is empty. #[inline] + #[must_use] #[unstable(feature = "linked_list_cursors", issue = "58533")] pub fn cursor_back_mut(&mut self) -> CursorMut<'_, T> { CursorMut { index: self.len.checked_sub(1).unwrap_or(0), current: self.tail, list: self } @@ -547,7 +569,7 @@ impl LinkedList { /// Returns `true` if the `LinkedList` is empty. /// - /// This operation should compute in O(1) time. + /// This operation should compute in *O*(1) time. /// /// # Examples /// @@ -561,6 +583,7 @@ impl LinkedList { /// assert!(!dl.is_empty()); /// ``` #[inline] + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn is_empty(&self) -> bool { self.head.is_none() @@ -568,7 +591,7 @@ impl LinkedList { /// Returns the length of the `LinkedList`. /// - /// This operation should compute in O(1) time. + /// This operation should compute in *O*(1) time. /// /// # Examples /// @@ -587,6 +610,7 @@ impl LinkedList { /// assert_eq!(dl.len(), 3); /// ``` #[inline] + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn len(&self) -> usize { self.len @@ -594,7 +618,7 @@ impl LinkedList { /// Removes all elements from the `LinkedList`. /// - /// This operation should compute in O(n) time. + /// This operation should compute in *O*(*n*) time. /// /// # Examples /// @@ -621,6 +645,8 @@ impl LinkedList { /// Returns `true` if the `LinkedList` contains an element equal to the /// given value. /// + /// This operation should compute linearly in *O*(*n*) time. + /// /// # Examples /// /// ``` @@ -646,6 +672,8 @@ impl LinkedList { /// Provides a reference to the front element, or `None` if the list is /// empty. /// + /// This operation should compute in *O*(1) time. + /// /// # Examples /// /// ``` @@ -658,6 +686,7 @@ impl LinkedList { /// assert_eq!(dl.front(), Some(&1)); /// ``` #[inline] + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn front(&self) -> Option<&T> { unsafe { self.head.as_ref().map(|node| &node.as_ref().element) } @@ -666,6 +695,8 @@ impl LinkedList { /// Provides a mutable reference to the front element, or `None` if the list /// is empty. /// + /// This operation should compute in *O*(1) time. + /// /// # Examples /// /// ``` @@ -684,6 +715,7 @@ impl LinkedList { /// assert_eq!(dl.front(), Some(&5)); /// ``` #[inline] + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn front_mut(&mut self) -> Option<&mut T> { unsafe { self.head.as_mut().map(|node| &mut node.as_mut().element) } @@ -692,6 +724,8 @@ impl LinkedList { /// Provides a reference to the back element, or `None` if the list is /// empty. /// + /// This operation should compute in *O*(1) time. + /// /// # Examples /// /// ``` @@ -704,6 +738,7 @@ impl LinkedList { /// assert_eq!(dl.back(), Some(&1)); /// ``` #[inline] + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn back(&self) -> Option<&T> { unsafe { self.tail.as_ref().map(|node| &node.as_ref().element) } @@ -712,6 +747,8 @@ impl LinkedList { /// Provides a mutable reference to the back element, or `None` if the list /// is empty. /// + /// This operation should compute in *O*(1) time. + /// /// # Examples /// /// ``` @@ -737,7 +774,7 @@ impl LinkedList { /// Adds an element first in the list. /// - /// This operation should compute in O(1) time. + /// This operation should compute in *O*(1) time. /// /// # Examples /// @@ -754,13 +791,13 @@ impl LinkedList { /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn push_front(&mut self, elt: T) { - self.push_front_node(box Node::new(elt)); + self.push_front_node(Box::new(Node::new(elt))); } /// Removes the first element and returns it, or `None` if the list is /// empty. /// - /// This operation should compute in O(1) time. + /// This operation should compute in *O*(1) time. /// /// # Examples /// @@ -783,7 +820,7 @@ impl LinkedList { /// Appends an element to the back of a list. /// - /// This operation should compute in O(1) time. + /// This operation should compute in *O*(1) time. /// /// # Examples /// @@ -797,13 +834,13 @@ impl LinkedList { /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn push_back(&mut self, elt: T) { - self.push_back_node(box Node::new(elt)); + self.push_back_node(Box::new(Node::new(elt))); } /// Removes the last element from a list and returns it, or `None` if /// it is empty. /// - /// This operation should compute in O(1) time. + /// This operation should compute in *O*(1) time. /// /// # Examples /// @@ -824,7 +861,7 @@ impl LinkedList { /// Splits the list into two at the given index. Returns everything after the given index, /// including the index. /// - /// This operation should compute in O(n) time. + /// This operation should compute in *O*(*n*) time. /// /// # Panics /// @@ -880,7 +917,7 @@ impl LinkedList { /// Removes the element at the given index and returns it. /// - /// This operation should compute in O(n) time. + /// This operation should compute in *O*(*n*) time. /// /// # Panics /// Panics if at >= len @@ -972,7 +1009,7 @@ unsafe impl<#[may_dangle] T> Drop for LinkedList { fn drop(&mut self) { // Continue the same loop we do below. This only runs when a destructor has // panicked. If another one panics this will abort. - while let Some(_) = self.0.pop_front_node() {} + while self.0.pop_front_node().is_some() {} } } @@ -1092,95 +1129,6 @@ impl ExactSizeIterator for IterMut<'_, T> {} #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for IterMut<'_, T> {} -impl IterMut<'_, T> { - /// Inserts the given element just after the element most recently returned by `.next()`. - /// The inserted element does not appear in the iteration. - /// - /// # Examples - /// - /// ``` - /// #![feature(linked_list_extras)] - /// - /// use std::collections::LinkedList; - /// - /// let mut list: LinkedList<_> = vec![1, 3, 4].into_iter().collect(); - /// - /// { - /// let mut it = list.iter_mut(); - /// assert_eq!(it.next().unwrap(), &1); - /// // insert `2` after `1` - /// it.insert_next(2); - /// } - /// { - /// let vec: Vec<_> = list.into_iter().collect(); - /// assert_eq!(vec, [1, 2, 3, 4]); - /// } - /// ``` - #[inline] - #[unstable( - feature = "linked_list_extras", - reason = "this is probably better handled by a cursor type -- we'll see", - issue = "27794" - )] - pub fn insert_next(&mut self, element: T) { - match self.head { - // `push_back` is okay with aliasing `element` references - None => self.list.push_back(element), - Some(head) => unsafe { - let prev = match head.as_ref().prev { - // `push_front` is okay with aliasing nodes - None => return self.list.push_front(element), - Some(prev) => prev, - }; - - let node = Some(Box::into_raw_non_null(box Node { - next: Some(head), - prev: Some(prev), - element, - })); - - // Not creating references to entire nodes to not invalidate the - // reference to `element` we handed to the user. - (*prev.as_ptr()).next = node; - (*head.as_ptr()).prev = node; - - self.list.len += 1; - }, - } - } - - /// Provides a reference to the next element, without changing the iterator. - /// - /// # Examples - /// - /// ``` - /// #![feature(linked_list_extras)] - /// - /// use std::collections::LinkedList; - /// - /// let mut list: LinkedList<_> = vec![1, 2, 3].into_iter().collect(); - /// - /// let mut it = list.iter_mut(); - /// assert_eq!(it.next().unwrap(), &1); - /// assert_eq!(it.peek_next().unwrap(), &2); - /// // We just peeked at 2, so it was not consumed from the iterator. - /// assert_eq!(it.next().unwrap(), &2); - /// ``` - #[inline] - #[unstable( - feature = "linked_list_extras", - reason = "this is probably better handled by a cursor type -- we'll see", - issue = "27794" - )] - pub fn peek_next(&mut self) -> Option<&mut T> { - if self.len == 0 { - None - } else { - unsafe { self.head.as_mut().map(|node| &mut node.as_mut().element) } - } - } -} - /// A cursor over a `LinkedList`. /// /// A `Cursor` is like an iterator, except that it can freely seek back-and-forth. @@ -1197,6 +1145,14 @@ pub struct Cursor<'a, T: 'a> { list: &'a LinkedList, } +#[unstable(feature = "linked_list_cursors", issue = "58533")] +impl Clone for Cursor<'_, T> { + fn clone(&self) -> Self { + let Cursor { index, current, list } = *self; + Cursor { index, current, list } + } +} + #[unstable(feature = "linked_list_cursors", issue = "58533")] impl fmt::Debug for Cursor<'_, T> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -1233,6 +1189,7 @@ impl<'a, T> Cursor<'a, T> { /// /// This returns `None` if the cursor is currently pointing to the /// "ghost" non-element. + #[must_use] #[unstable(feature = "linked_list_cursors", issue = "58533")] pub fn index(&self) -> Option { let _ = self.current?; @@ -1287,6 +1244,7 @@ impl<'a, T> Cursor<'a, T> { /// /// This returns `None` if the cursor is currently pointing to the /// "ghost" non-element. + #[must_use] #[unstable(feature = "linked_list_cursors", issue = "58533")] pub fn current(&self) -> Option<&'a T> { unsafe { self.current.map(|current| &(*current.as_ptr()).element) } @@ -1297,6 +1255,7 @@ impl<'a, T> Cursor<'a, T> { /// If the cursor is pointing to the "ghost" non-element then this returns /// the first element of the `LinkedList`. If it is pointing to the last /// element of the `LinkedList` then this returns `None`. + #[must_use] #[unstable(feature = "linked_list_cursors", issue = "58533")] pub fn peek_next(&self) -> Option<&'a T> { unsafe { @@ -1313,6 +1272,7 @@ impl<'a, T> Cursor<'a, T> { /// If the cursor is pointing to the "ghost" non-element then this returns /// the last element of the `LinkedList`. If it is pointing to the first /// element of the `LinkedList` then this returns `None`. + #[must_use] #[unstable(feature = "linked_list_cursors", issue = "58533")] pub fn peek_prev(&self) -> Option<&'a T> { unsafe { @@ -1323,6 +1283,22 @@ impl<'a, T> Cursor<'a, T> { prev.map(|prev| &(*prev.as_ptr()).element) } } + + /// Provides a reference to the front element of the cursor's parent list, + /// or None if the list is empty. + #[must_use] + #[unstable(feature = "linked_list_cursors", issue = "58533")] + pub fn front(&self) -> Option<&'a T> { + self.list.front() + } + + /// Provides a reference to the back element of the cursor's parent list, + /// or None if the list is empty. + #[must_use] + #[unstable(feature = "linked_list_cursors", issue = "58533")] + pub fn back(&self) -> Option<&'a T> { + self.list.back() + } } impl<'a, T> CursorMut<'a, T> { @@ -1330,6 +1306,7 @@ impl<'a, T> CursorMut<'a, T> { /// /// This returns `None` if the cursor is currently pointing to the /// "ghost" non-element. + #[must_use] #[unstable(feature = "linked_list_cursors", issue = "58533")] pub fn index(&self) -> Option { let _ = self.current?; @@ -1384,6 +1361,7 @@ impl<'a, T> CursorMut<'a, T> { /// /// This returns `None` if the cursor is currently pointing to the /// "ghost" non-element. + #[must_use] #[unstable(feature = "linked_list_cursors", issue = "58533")] pub fn current(&mut self) -> Option<&mut T> { unsafe { self.current.map(|current| &mut (*current.as_ptr()).element) } @@ -1426,6 +1404,7 @@ impl<'a, T> CursorMut<'a, T> { /// The lifetime of the returned `Cursor` is bound to that of the /// `CursorMut`, which means it cannot outlive the `CursorMut` and that the /// `CursorMut` is frozen for the lifetime of the `Cursor`. + #[must_use] #[unstable(feature = "linked_list_cursors", issue = "58533")] pub fn as_cursor(&self) -> Cursor<'_, T> { Cursor { list: self.list, current: self.current, index: self.index } @@ -1442,7 +1421,7 @@ impl<'a, T> CursorMut<'a, T> { #[unstable(feature = "linked_list_cursors", issue = "58533")] pub fn insert_after(&mut self, item: T) { unsafe { - let spliced_node = Box::into_raw_non_null(Box::new(Node::new(item))); + let spliced_node = Box::leak(Box::new(Node::new(item))).into(); let node_next = match self.current { None => self.list.head, Some(node) => node.as_ref().next, @@ -1462,7 +1441,7 @@ impl<'a, T> CursorMut<'a, T> { #[unstable(feature = "linked_list_cursors", issue = "58533")] pub fn insert_before(&mut self, item: T) { unsafe { - let spliced_node = Box::into_raw_non_null(Box::new(Node::new(item))); + let spliced_node = Box::leak(Box::new(Node::new(item))).into(); let node_prev = match self.current { None => self.list.tail, Some(node) => node.as_ref().prev, @@ -1490,6 +1469,31 @@ impl<'a, T> CursorMut<'a, T> { } } + /// Removes the current element from the `LinkedList` without deallocating the list node. + /// + /// The node that was removed is returned as a new `LinkedList` containing only this node. + /// The cursor is moved to point to the next element in the current `LinkedList`. + /// + /// If the cursor is currently pointing to the "ghost" non-element then no element + /// is removed and `None` is returned. + #[unstable(feature = "linked_list_cursors", issue = "58533")] + pub fn remove_current_as_list(&mut self) -> Option> { + let mut unlinked_node = self.current?; + unsafe { + self.current = unlinked_node.as_ref().next; + self.list.unlink_node(unlinked_node); + + unlinked_node.as_mut().prev = None; + unlinked_node.as_mut().next = None; + Some(LinkedList { + head: Some(unlinked_node), + tail: Some(unlinked_node), + len: 1, + marker: PhantomData, + }) + } + } + /// Inserts the elements from the given `LinkedList` after the current one. /// /// If the cursor is pointing at the "ghost" non-element then the new elements are @@ -1561,6 +1565,139 @@ impl<'a, T> CursorMut<'a, T> { self.index = 0; unsafe { self.list.split_off_before_node(self.current, split_off_idx) } } + + /// Appends an element to the front of the cursor's parent list. The node + /// that the cursor points to is unchanged, even if it is the "ghost" node. + /// + /// This operation should compute in *O*(1) time. + // `push_front` continues to point to "ghost" when it adds a node to mimic + // the behavior of `insert_before` on an empty list. + #[unstable(feature = "linked_list_cursors", issue = "58533")] + pub fn push_front(&mut self, elt: T) { + // Safety: We know that `push_front` does not change the position in + // memory of other nodes. This ensures that `self.current` remains + // valid. + self.list.push_front(elt); + self.index += 1; + } + + /// Appends an element to the back of the cursor's parent list. The node + /// that the cursor points to is unchanged, even if it is the "ghost" node. + /// + /// This operation should compute in *O*(1) time. + #[unstable(feature = "linked_list_cursors", issue = "58533")] + pub fn push_back(&mut self, elt: T) { + // Safety: We know that `push_back` does not change the position in + // memory of other nodes. This ensures that `self.current` remains + // valid. + self.list.push_back(elt); + if self.current().is_none() { + // The index of "ghost" is the length of the list, so we just need + // to increment self.index to reflect the new length of the list. + self.index += 1; + } + } + + /// Removes the first element from the cursor's parent list and returns it, + /// or None if the list is empty. The element the cursor points to remains + /// unchanged, unless it was pointing to the front element. In that case, it + /// points to the new front element. + /// + /// This operation should compute in *O*(1) time. + #[unstable(feature = "linked_list_cursors", issue = "58533")] + pub fn pop_front(&mut self) -> Option { + // We can't check if current is empty, we must check the list directly. + // It is possible for `self.current == None` and the list to be + // non-empty. + if self.list.is_empty() { + None + } else { + // We can't point to the node that we pop. Copying the behavior of + // `remove_current`, we move on to the next node in the sequence. + // If the list is of length 1 then we end pointing to the "ghost" + // node at index 0, which is expected. + if self.list.head == self.current { + self.move_next(); + } else { + self.index -= 1; + } + self.list.pop_front() + } + } + + /// Removes the last element from the cursor's parent list and returns it, + /// or None if the list is empty. The element the cursor points to remains + /// unchanged, unless it was pointing to the back element. In that case, it + /// points to the "ghost" element. + /// + /// This operation should compute in *O*(1) time. + #[unstable(feature = "linked_list_cursors", issue = "58533")] + pub fn pop_back(&mut self) -> Option { + if self.list.is_empty() { + None + } else { + if self.list.tail == self.current { + // The index now reflects the length of the list. It was the + // length of the list minus 1, but now the list is 1 smaller. No + // change is needed for `index`. + self.current = None; + } else if self.current.is_none() { + self.index = self.list.len - 1; + } + self.list.pop_back() + } + } + + /// Provides a reference to the front element of the cursor's parent list, + /// or None if the list is empty. + #[must_use] + #[unstable(feature = "linked_list_cursors", issue = "58533")] + pub fn front(&self) -> Option<&T> { + self.list.front() + } + + /// Provides a mutable reference to the front element of the cursor's + /// parent list, or None if the list is empty. + #[must_use] + #[unstable(feature = "linked_list_cursors", issue = "58533")] + pub fn front_mut(&mut self) -> Option<&mut T> { + self.list.front_mut() + } + + /// Provides a reference to the back element of the cursor's parent list, + /// or None if the list is empty. + #[must_use] + #[unstable(feature = "linked_list_cursors", issue = "58533")] + pub fn back(&self) -> Option<&T> { + self.list.back() + } + + /// Provides a mutable reference to back element of the cursor's parent + /// list, or `None` if the list is empty. + /// + /// # Examples + /// Building and mutating a list with a cursor, then getting the back element: + /// ``` + /// #![feature(linked_list_cursors)] + /// use std::collections::LinkedList; + /// let mut dl = LinkedList::new(); + /// dl.push_front(3); + /// dl.push_front(2); + /// dl.push_front(1); + /// let mut cursor = dl.cursor_front_mut(); + /// *cursor.current().unwrap() = 99; + /// *cursor.back_mut().unwrap() = 0; + /// let mut contents = dl.into_iter(); + /// assert_eq!(contents.next(), Some(99)); + /// assert_eq!(contents.next(), Some(2)); + /// assert_eq!(contents.next(), Some(0)); + /// assert_eq!(contents.next(), None); + /// ``` + #[must_use] + #[unstable(feature = "linked_list_cursors", issue = "58533")] + pub fn back_mut(&mut self) -> Option<&mut T> { + self.list.back_mut() + } } /// An iterator produced by calling `drain_filter` on LinkedList. @@ -1717,6 +1854,11 @@ impl Extend for LinkedList { fn extend>(&mut self, iter: I) { >::spec_extend(self, iter); } + + #[inline] + fn extend_one(&mut self, elem: T) { + self.push_back(elem); + } } impl SpecExtend for LinkedList { @@ -1736,6 +1878,11 @@ impl<'a, T: 'a + Copy> Extend<&'a T> for LinkedList { fn extend>(&mut self, iter: I) { self.extend(iter.into_iter().cloned()); } + + #[inline] + fn extend_one(&mut self, &elem: &'a T) { + self.push_back(elem); + } } #[stable(feature = "rust1", since = "1.0.0")] @@ -1797,13 +1944,29 @@ impl fmt::Debug for LinkedList { #[stable(feature = "rust1", since = "1.0.0")] impl Hash for LinkedList { fn hash(&self, state: &mut H) { - self.len().hash(state); + state.write_length_prefix(self.len()); for elt in self { elt.hash(state); } } } +#[stable(feature = "std_collections_from_array", since = "1.56.0")] +impl From<[T; N]> for LinkedList { + /// Converts a `[T; N]` into a `LinkedList`. + /// + /// ``` + /// use std::collections::LinkedList; + /// + /// let list1 = LinkedList::from([1, 2, 3, 4]); + /// let list2: LinkedList<_> = [1, 2, 3, 4].into(); + /// assert_eq!(list1, list2); + /// ``` + fn from(arr: [T; N]) -> Self { + Self::from_iter(arr) + } +} + // Ensure that `LinkedList` and its read-only iterators are covariant in their type parameters. #[allow(dead_code)] fn assert_covariance() { @@ -1835,3 +1998,15 @@ unsafe impl Send for IterMut<'_, T> {} #[stable(feature = "rust1", since = "1.0.0")] unsafe impl Sync for IterMut<'_, T> {} + +#[unstable(feature = "linked_list_cursors", issue = "58533")] +unsafe impl Send for Cursor<'_, T> {} + +#[unstable(feature = "linked_list_cursors", issue = "58533")] +unsafe impl Sync for Cursor<'_, T> {} + +#[unstable(feature = "linked_list_cursors", issue = "58533")] +unsafe impl Send for CursorMut<'_, T> {} + +#[unstable(feature = "linked_list_cursors", issue = "58533")] +unsafe impl Sync for CursorMut<'_, T> {} diff --git a/crux-mir/lib/alloc/src/collections/linked_list/tests.rs b/crux-mir/lib/alloc/src/collections/linked_list/tests.rs index 085f734ed..04594d55b 100644 --- a/crux-mir/lib/alloc/src/collections/linked_list/tests.rs +++ b/crux-mir/lib/alloc/src/collections/linked_list/tests.rs @@ -1,9 +1,55 @@ use super::*; +use crate::testing::crash_test::{CrashTestDummy, Panic}; +use crate::vec::Vec; +use std::panic::{catch_unwind, AssertUnwindSafe}; use std::thread; -use std::vec::Vec; -use rand::{thread_rng, RngCore}; +use rand::RngCore; + +#[test] +fn test_basic() { + let mut m = LinkedList::>::new(); + assert_eq!(m.pop_front(), None); + assert_eq!(m.pop_back(), None); + assert_eq!(m.pop_front(), None); + m.push_front(Box::new(1)); + assert_eq!(m.pop_front(), Some(Box::new(1))); + m.push_back(Box::new(2)); + m.push_back(Box::new(3)); + assert_eq!(m.len(), 2); + assert_eq!(m.pop_front(), Some(Box::new(2))); + assert_eq!(m.pop_front(), Some(Box::new(3))); + assert_eq!(m.len(), 0); + assert_eq!(m.pop_front(), None); + m.push_back(Box::new(1)); + m.push_back(Box::new(3)); + m.push_back(Box::new(5)); + m.push_back(Box::new(7)); + assert_eq!(m.pop_front(), Some(Box::new(1))); + + let mut n = LinkedList::new(); + n.push_front(2); + n.push_front(3); + { + assert_eq!(n.front().unwrap(), &3); + let x = n.front_mut().unwrap(); + assert_eq!(*x, 3); + *x = 0; + } + { + assert_eq!(n.back().unwrap(), &2); + let y = n.back_mut().unwrap(); + assert_eq!(*y, 2); + *y = 1; + } + assert_eq!(n.pop_front(), Some(0)); + assert_eq!(n.pop_front(), Some(1)); +} + +fn generate_test() -> LinkedList { + list_from(&[0, 1, 2, 3, 4, 5, 6]) +} fn list_from(v: &[T]) -> LinkedList { v.iter().cloned().collect() @@ -110,6 +156,123 @@ fn test_append() { check_links(&n); } +#[test] +fn test_iterator() { + let m = generate_test(); + for (i, elt) in m.iter().enumerate() { + assert_eq!(i as i32, *elt); + } + let mut n = LinkedList::new(); + assert_eq!(n.iter().next(), None); + n.push_front(4); + let mut it = n.iter(); + assert_eq!(it.size_hint(), (1, Some(1))); + assert_eq!(it.next().unwrap(), &4); + assert_eq!(it.size_hint(), (0, Some(0))); + assert_eq!(it.next(), None); +} + +#[test] +fn test_iterator_clone() { + let mut n = LinkedList::new(); + n.push_back(2); + n.push_back(3); + n.push_back(4); + let mut it = n.iter(); + it.next(); + let mut jt = it.clone(); + assert_eq!(it.next(), jt.next()); + assert_eq!(it.next_back(), jt.next_back()); + assert_eq!(it.next(), jt.next()); +} + +#[test] +fn test_iterator_double_end() { + let mut n = LinkedList::new(); + assert_eq!(n.iter().next(), None); + n.push_front(4); + n.push_front(5); + n.push_front(6); + let mut it = n.iter(); + assert_eq!(it.size_hint(), (3, Some(3))); + assert_eq!(it.next().unwrap(), &6); + assert_eq!(it.size_hint(), (2, Some(2))); + assert_eq!(it.next_back().unwrap(), &4); + assert_eq!(it.size_hint(), (1, Some(1))); + assert_eq!(it.next_back().unwrap(), &5); + assert_eq!(it.next_back(), None); + assert_eq!(it.next(), None); +} + +#[test] +fn test_rev_iter() { + let m = generate_test(); + for (i, elt) in m.iter().rev().enumerate() { + assert_eq!((6 - i) as i32, *elt); + } + let mut n = LinkedList::new(); + assert_eq!(n.iter().rev().next(), None); + n.push_front(4); + let mut it = n.iter().rev(); + assert_eq!(it.size_hint(), (1, Some(1))); + assert_eq!(it.next().unwrap(), &4); + assert_eq!(it.size_hint(), (0, Some(0))); + assert_eq!(it.next(), None); +} + +#[test] +fn test_mut_iter() { + let mut m = generate_test(); + let mut len = m.len(); + for (i, elt) in m.iter_mut().enumerate() { + assert_eq!(i as i32, *elt); + len -= 1; + } + assert_eq!(len, 0); + let mut n = LinkedList::new(); + assert!(n.iter_mut().next().is_none()); + n.push_front(4); + n.push_back(5); + let mut it = n.iter_mut(); + assert_eq!(it.size_hint(), (2, Some(2))); + assert!(it.next().is_some()); + assert!(it.next().is_some()); + assert_eq!(it.size_hint(), (0, Some(0))); + assert!(it.next().is_none()); +} + +#[test] +fn test_iterator_mut_double_end() { + let mut n = LinkedList::new(); + assert!(n.iter_mut().next_back().is_none()); + n.push_front(4); + n.push_front(5); + n.push_front(6); + let mut it = n.iter_mut(); + assert_eq!(it.size_hint(), (3, Some(3))); + assert_eq!(*it.next().unwrap(), 6); + assert_eq!(it.size_hint(), (2, Some(2))); + assert_eq!(*it.next_back().unwrap(), 4); + assert_eq!(it.size_hint(), (1, Some(1))); + assert_eq!(*it.next_back().unwrap(), 5); + assert!(it.next_back().is_none()); + assert!(it.next().is_none()); +} + +#[test] +fn test_mut_rev_iter() { + let mut m = generate_test(); + for (i, elt) in m.iter_mut().rev().enumerate() { + assert_eq!((6 - i) as i32, *elt); + } + let mut n = LinkedList::new(); + assert!(n.iter_mut().rev().next().is_none()); + n.push_front(4); + let mut it = n.iter_mut().rev(); + assert!(it.next().is_some()); + assert!(it.next().is_none()); +} + #[test] fn test_clone_from() { // Short cloned from long @@ -153,36 +316,8 @@ fn test_clone_from() { } } -#[test] -fn test_insert_prev() { - let mut m = list_from(&[0, 2, 4, 6, 8]); - let len = m.len(); - { - let mut it = m.iter_mut(); - it.insert_next(-2); - loop { - match it.next() { - None => break, - Some(elt) => { - it.insert_next(*elt + 1); - match it.peek_next() { - Some(x) => assert_eq!(*x, *elt + 2), - None => assert_eq!(8, *elt), - } - } - } - } - it.insert_next(0); - it.insert_next(1); - } - check_links(&m); - assert_eq!(m.len(), 3 + len * 2); - assert_eq!(m.into_iter().collect::>(), [-2, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1]); -} - #[test] #[cfg_attr(target_os = "emscripten", ignore)] -#[cfg_attr(miri, ignore)] // Miri does not support threads fn test_send() { let n = list_from(&[1, 2, 3]); thread::spawn(move || { @@ -196,13 +331,60 @@ fn test_send() { } #[test] -fn test_fuzz() { - for _ in 0..25 { - fuzz_test(3); - fuzz_test(16); - #[cfg(not(miri))] // Miri is too slow - fuzz_test(189); - } +fn test_eq() { + let mut n = list_from(&[]); + let mut m = list_from(&[]); + assert!(n == m); + n.push_front(1); + assert!(n != m); + m.push_back(1); + assert!(n == m); + + let n = list_from(&[2, 3, 4]); + let m = list_from(&[1, 2, 3]); + assert!(n != m); +} + +#[test] +fn test_ord() { + let n = list_from(&[]); + let m = list_from(&[1, 2, 3]); + assert!(n < m); + assert!(m > n); + assert!(n <= n); + assert!(n >= n); +} + +#[test] +fn test_ord_nan() { + let nan = 0.0f64 / 0.0; + let n = list_from(&[nan]); + let m = list_from(&[nan]); + assert!(!(n < m)); + assert!(!(n > m)); + assert!(!(n <= m)); + assert!(!(n >= m)); + + let n = list_from(&[nan]); + let one = list_from(&[1.0f64]); + assert!(!(n < one)); + assert!(!(n > one)); + assert!(!(n <= one)); + assert!(!(n >= one)); + + let u = list_from(&[1.0f64, 2.0, nan]); + let v = list_from(&[1.0f64, 2.0, 3.0]); + assert!(!(u < v)); + assert!(!(u > v)); + assert!(!(u <= v)); + assert!(!(u >= v)); + + let s = list_from(&[1.0f64, 2.0, 4.0, 2.0]); + let t = list_from(&[1.0f64, 2.0, 3.0, 2.0]); + assert!(!(s < t)); + assert!(s > one); + assert!(!(s <= one)); + assert!(s >= one); } #[test] @@ -243,12 +425,68 @@ fn test_split_off() { } } -fn fuzz_test(sz: i32) { +#[test] +fn test_split_off_2() { + // singleton + { + let mut m = LinkedList::new(); + m.push_back(1); + + let p = m.split_off(0); + assert_eq!(m.len(), 0); + assert_eq!(p.len(), 1); + assert_eq!(p.back(), Some(&1)); + assert_eq!(p.front(), Some(&1)); + } + + // not singleton, forwards + { + let u = vec![1, 2, 3, 4, 5]; + let mut m = list_from(&u); + let mut n = m.split_off(2); + assert_eq!(m.len(), 2); + assert_eq!(n.len(), 3); + for elt in 1..3 { + assert_eq!(m.pop_front(), Some(elt)); + } + for elt in 3..6 { + assert_eq!(n.pop_front(), Some(elt)); + } + } + // not singleton, backwards + { + let u = vec![1, 2, 3, 4, 5]; + let mut m = list_from(&u); + let mut n = m.split_off(4); + assert_eq!(m.len(), 4); + assert_eq!(n.len(), 1); + for elt in 1..5 { + assert_eq!(m.pop_front(), Some(elt)); + } + for elt in 5..6 { + assert_eq!(n.pop_front(), Some(elt)); + } + } + + // no-op on the last index + { + let mut m = LinkedList::new(); + m.push_back(1); + + let p = m.split_off(1); + assert_eq!(m.len(), 1); + assert_eq!(p.len(), 0); + assert_eq!(m.back(), Some(&1)); + assert_eq!(m.front(), Some(&1)); + } +} + +fn fuzz_test(sz: i32, rng: &mut impl RngCore) { let mut m: LinkedList<_> = LinkedList::new(); let mut v = vec![]; for i in 0..sz { check_links(&m); - let r: u8 = thread_rng().next_u32() as u8; + let r: u8 = rng.next_u32() as u8; match r % 6 { 0 => { m.pop_back(); @@ -281,6 +519,26 @@ fn fuzz_test(sz: i32) { assert_eq!(i, v.len()); } +#[test] +fn test_fuzz() { + let mut rng = crate::test_helpers::test_rng(); + for _ in 0..25 { + fuzz_test(3, &mut rng); + fuzz_test(16, &mut rng); + #[cfg(not(miri))] // Miri is too slow + fuzz_test(189, &mut rng); + } +} + +#[test] +fn test_show() { + let list: LinkedList<_> = (0..10).collect(); + assert_eq!(format!("{list:?}"), "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]"); + + let list: LinkedList<_> = ["just", "one", "test", "more"].into_iter().collect(); + assert_eq!(format!("{list:?}"), "[\"just\", \"one\", \"test\", \"more\"]"); +} + #[test] fn drain_filter_test() { let mut m: LinkedList = LinkedList::new(); @@ -456,3 +714,444 @@ fn test_cursor_mut_insert() { check_links(&m); assert_eq!(m.iter().cloned().collect::>(), &[200, 201, 202, 203, 1, 100, 101]); } + +#[test] +fn test_cursor_push_front_back() { + let mut ll: LinkedList = LinkedList::new(); + ll.extend(&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + let mut c = ll.cursor_front_mut(); + assert_eq!(c.current(), Some(&mut 1)); + assert_eq!(c.index(), Some(0)); + c.push_front(0); + assert_eq!(c.current(), Some(&mut 1)); + assert_eq!(c.peek_prev(), Some(&mut 0)); + assert_eq!(c.index(), Some(1)); + c.push_back(11); + drop(c); + let p = ll.cursor_back().front().unwrap(); + assert_eq!(p, &0); + assert_eq!(ll, (0..12).collect()); + check_links(&ll); +} + +#[test] +fn test_cursor_pop_front_back() { + let mut ll: LinkedList = LinkedList::new(); + ll.extend(&[1, 2, 3, 4, 5, 6]); + let mut c = ll.cursor_back_mut(); + assert_eq!(c.pop_front(), Some(1)); + c.move_prev(); + c.move_prev(); + c.move_prev(); + assert_eq!(c.pop_back(), Some(6)); + let c = c.as_cursor(); + assert_eq!(c.front(), Some(&2)); + assert_eq!(c.back(), Some(&5)); + assert_eq!(c.index(), Some(1)); + drop(c); + assert_eq!(ll, (2..6).collect()); + check_links(&ll); + let mut c = ll.cursor_back_mut(); + assert_eq!(c.current(), Some(&mut 5)); + assert_eq!(c.index, 3); + assert_eq!(c.pop_back(), Some(5)); + assert_eq!(c.current(), None); + assert_eq!(c.index, 3); + assert_eq!(c.pop_back(), Some(4)); + assert_eq!(c.current(), None); + assert_eq!(c.index, 2); +} + +#[test] +fn test_extend_ref() { + let mut a = LinkedList::new(); + a.push_back(1); + + a.extend(&[2, 3, 4]); + + assert_eq!(a.len(), 4); + assert_eq!(a, list_from(&[1, 2, 3, 4])); + + let mut b = LinkedList::new(); + b.push_back(5); + b.push_back(6); + a.extend(&b); + + assert_eq!(a.len(), 6); + assert_eq!(a, list_from(&[1, 2, 3, 4, 5, 6])); +} + +#[test] +fn test_extend() { + let mut a = LinkedList::new(); + a.push_back(1); + a.extend(vec![2, 3, 4]); // uses iterator + + assert_eq!(a.len(), 4); + assert!(a.iter().eq(&[1, 2, 3, 4])); + + let b: LinkedList<_> = [5, 6, 7].into_iter().collect(); + a.extend(b); // specializes to `append` + + assert_eq!(a.len(), 7); + assert!(a.iter().eq(&[1, 2, 3, 4, 5, 6, 7])); +} + +#[test] +fn test_contains() { + let mut l = LinkedList::new(); + l.extend(&[2, 3, 4]); + + assert!(l.contains(&3)); + assert!(!l.contains(&1)); + + l.clear(); + + assert!(!l.contains(&3)); +} + +#[test] +fn drain_filter_empty() { + let mut list: LinkedList = LinkedList::new(); + + { + let mut iter = list.drain_filter(|_| true); + assert_eq!(iter.size_hint(), (0, Some(0))); + assert_eq!(iter.next(), None); + assert_eq!(iter.size_hint(), (0, Some(0))); + assert_eq!(iter.next(), None); + assert_eq!(iter.size_hint(), (0, Some(0))); + } + + assert_eq!(list.len(), 0); + assert_eq!(list.into_iter().collect::>(), vec![]); +} + +#[test] +fn drain_filter_zst() { + let mut list: LinkedList<_> = [(), (), (), (), ()].into_iter().collect(); + let initial_len = list.len(); + let mut count = 0; + + { + let mut iter = list.drain_filter(|_| true); + assert_eq!(iter.size_hint(), (0, Some(initial_len))); + while let Some(_) = iter.next() { + count += 1; + assert_eq!(iter.size_hint(), (0, Some(initial_len - count))); + } + assert_eq!(iter.size_hint(), (0, Some(0))); + assert_eq!(iter.next(), None); + assert_eq!(iter.size_hint(), (0, Some(0))); + } + + assert_eq!(count, initial_len); + assert_eq!(list.len(), 0); + assert_eq!(list.into_iter().collect::>(), vec![]); +} + +#[test] +fn drain_filter_false() { + let mut list: LinkedList<_> = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10].into_iter().collect(); + + let initial_len = list.len(); + let mut count = 0; + + { + let mut iter = list.drain_filter(|_| false); + assert_eq!(iter.size_hint(), (0, Some(initial_len))); + for _ in iter.by_ref() { + count += 1; + } + assert_eq!(iter.size_hint(), (0, Some(0))); + assert_eq!(iter.next(), None); + assert_eq!(iter.size_hint(), (0, Some(0))); + } + + assert_eq!(count, 0); + assert_eq!(list.len(), initial_len); + assert_eq!(list.into_iter().collect::>(), vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); +} + +#[test] +fn drain_filter_true() { + let mut list: LinkedList<_> = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10].into_iter().collect(); + + let initial_len = list.len(); + let mut count = 0; + + { + let mut iter = list.drain_filter(|_| true); + assert_eq!(iter.size_hint(), (0, Some(initial_len))); + while let Some(_) = iter.next() { + count += 1; + assert_eq!(iter.size_hint(), (0, Some(initial_len - count))); + } + assert_eq!(iter.size_hint(), (0, Some(0))); + assert_eq!(iter.next(), None); + assert_eq!(iter.size_hint(), (0, Some(0))); + } + + assert_eq!(count, initial_len); + assert_eq!(list.len(), 0); + assert_eq!(list.into_iter().collect::>(), vec![]); +} + +#[test] +fn drain_filter_complex() { + { + // [+xxx++++++xxxxx++++x+x++] + let mut list = [ + 1, 2, 4, 6, 7, 9, 11, 13, 15, 17, 18, 20, 22, 24, 26, 27, 29, 31, 33, 34, 35, 36, 37, + 39, + ] + .into_iter() + .collect::>(); + + let removed = list.drain_filter(|x| *x % 2 == 0).collect::>(); + assert_eq!(removed.len(), 10); + assert_eq!(removed, vec![2, 4, 6, 18, 20, 22, 24, 26, 34, 36]); + + assert_eq!(list.len(), 14); + assert_eq!( + list.into_iter().collect::>(), + vec![1, 7, 9, 11, 13, 15, 17, 27, 29, 31, 33, 35, 37, 39] + ); + } + + { + // [xxx++++++xxxxx++++x+x++] + let mut list = + [2, 4, 6, 7, 9, 11, 13, 15, 17, 18, 20, 22, 24, 26, 27, 29, 31, 33, 34, 35, 36, 37, 39] + .into_iter() + .collect::>(); + + let removed = list.drain_filter(|x| *x % 2 == 0).collect::>(); + assert_eq!(removed.len(), 10); + assert_eq!(removed, vec![2, 4, 6, 18, 20, 22, 24, 26, 34, 36]); + + assert_eq!(list.len(), 13); + assert_eq!( + list.into_iter().collect::>(), + vec![7, 9, 11, 13, 15, 17, 27, 29, 31, 33, 35, 37, 39] + ); + } + + { + // [xxx++++++xxxxx++++x+x] + let mut list = + [2, 4, 6, 7, 9, 11, 13, 15, 17, 18, 20, 22, 24, 26, 27, 29, 31, 33, 34, 35, 36] + .into_iter() + .collect::>(); + + let removed = list.drain_filter(|x| *x % 2 == 0).collect::>(); + assert_eq!(removed.len(), 10); + assert_eq!(removed, vec![2, 4, 6, 18, 20, 22, 24, 26, 34, 36]); + + assert_eq!(list.len(), 11); + assert_eq!( + list.into_iter().collect::>(), + vec![7, 9, 11, 13, 15, 17, 27, 29, 31, 33, 35] + ); + } + + { + // [xxxxxxxxxx+++++++++++] + let mut list = [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19] + .into_iter() + .collect::>(); + + let removed = list.drain_filter(|x| *x % 2 == 0).collect::>(); + assert_eq!(removed.len(), 10); + assert_eq!(removed, vec![2, 4, 6, 8, 10, 12, 14, 16, 18, 20]); + + assert_eq!(list.len(), 10); + assert_eq!(list.into_iter().collect::>(), vec![1, 3, 5, 7, 9, 11, 13, 15, 17, 19]); + } + + { + // [+++++++++++xxxxxxxxxx] + let mut list = [1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20] + .into_iter() + .collect::>(); + + let removed = list.drain_filter(|x| *x % 2 == 0).collect::>(); + assert_eq!(removed.len(), 10); + assert_eq!(removed, vec![2, 4, 6, 8, 10, 12, 14, 16, 18, 20]); + + assert_eq!(list.len(), 10); + assert_eq!(list.into_iter().collect::>(), vec![1, 3, 5, 7, 9, 11, 13, 15, 17, 19]); + } +} + +#[test] +fn drain_filter_drop_panic_leak() { + let d0 = CrashTestDummy::new(0); + let d1 = CrashTestDummy::new(1); + let d2 = CrashTestDummy::new(2); + let d3 = CrashTestDummy::new(3); + let d4 = CrashTestDummy::new(4); + let d5 = CrashTestDummy::new(5); + let d6 = CrashTestDummy::new(6); + let d7 = CrashTestDummy::new(7); + let mut q = LinkedList::new(); + q.push_back(d3.spawn(Panic::Never)); + q.push_back(d4.spawn(Panic::Never)); + q.push_back(d5.spawn(Panic::Never)); + q.push_back(d6.spawn(Panic::Never)); + q.push_back(d7.spawn(Panic::Never)); + q.push_front(d2.spawn(Panic::Never)); + q.push_front(d1.spawn(Panic::InDrop)); + q.push_front(d0.spawn(Panic::Never)); + + catch_unwind(AssertUnwindSafe(|| drop(q.drain_filter(|_| true)))).unwrap_err(); + + assert_eq!(d0.dropped(), 1); + assert_eq!(d1.dropped(), 1); + assert_eq!(d2.dropped(), 1); + assert_eq!(d3.dropped(), 1); + assert_eq!(d4.dropped(), 1); + assert_eq!(d5.dropped(), 1); + assert_eq!(d6.dropped(), 1); + assert_eq!(d7.dropped(), 1); + assert!(q.is_empty()); +} + +#[test] +fn drain_filter_pred_panic_leak() { + static mut DROPS: i32 = 0; + + #[derive(Debug)] + struct D(u32); + + impl Drop for D { + fn drop(&mut self) { + unsafe { + DROPS += 1; + } + } + } + + let mut q = LinkedList::new(); + q.push_back(D(3)); + q.push_back(D(4)); + q.push_back(D(5)); + q.push_back(D(6)); + q.push_back(D(7)); + q.push_front(D(2)); + q.push_front(D(1)); + q.push_front(D(0)); + + catch_unwind(AssertUnwindSafe(|| { + drop(q.drain_filter(|item| if item.0 >= 2 { panic!() } else { true })) + })) + .ok(); + + assert_eq!(unsafe { DROPS }, 2); // 0 and 1 + assert_eq!(q.len(), 6); +} + +#[test] +fn test_drop() { + static mut DROPS: i32 = 0; + struct Elem; + impl Drop for Elem { + fn drop(&mut self) { + unsafe { + DROPS += 1; + } + } + } + + let mut ring = LinkedList::new(); + ring.push_back(Elem); + ring.push_front(Elem); + ring.push_back(Elem); + ring.push_front(Elem); + drop(ring); + + assert_eq!(unsafe { DROPS }, 4); +} + +#[test] +fn test_drop_with_pop() { + static mut DROPS: i32 = 0; + struct Elem; + impl Drop for Elem { + fn drop(&mut self) { + unsafe { + DROPS += 1; + } + } + } + + let mut ring = LinkedList::new(); + ring.push_back(Elem); + ring.push_front(Elem); + ring.push_back(Elem); + ring.push_front(Elem); + + drop(ring.pop_back()); + drop(ring.pop_front()); + assert_eq!(unsafe { DROPS }, 2); + + drop(ring); + assert_eq!(unsafe { DROPS }, 4); +} + +#[test] +fn test_drop_clear() { + static mut DROPS: i32 = 0; + struct Elem; + impl Drop for Elem { + fn drop(&mut self) { + unsafe { + DROPS += 1; + } + } + } + + let mut ring = LinkedList::new(); + ring.push_back(Elem); + ring.push_front(Elem); + ring.push_back(Elem); + ring.push_front(Elem); + ring.clear(); + assert_eq!(unsafe { DROPS }, 4); + + drop(ring); + assert_eq!(unsafe { DROPS }, 4); +} + +#[test] +fn test_drop_panic() { + static mut DROPS: i32 = 0; + + struct D(bool); + + impl Drop for D { + fn drop(&mut self) { + unsafe { + DROPS += 1; + } + + if self.0 { + panic!("panic in `drop`"); + } + } + } + + let mut q = LinkedList::new(); + q.push_back(D(false)); + q.push_back(D(false)); + q.push_back(D(false)); + q.push_back(D(false)); + q.push_back(D(false)); + q.push_front(D(false)); + q.push_front(D(false)); + q.push_front(D(true)); + + catch_unwind(move || drop(q)).ok(); + + assert_eq!(unsafe { DROPS }, 8); +} diff --git a/crux-mir/lib/alloc/src/collections/mod.rs b/crux-mir/lib/alloc/src/collections/mod.rs index 6b21e54f6..3e0b0f735 100644 --- a/crux-mir/lib/alloc/src/collections/mod.rs +++ b/crux-mir/lib/alloc/src/collections/mod.rs @@ -2,52 +2,88 @@ #![stable(feature = "rust1", since = "1.0.0")] +#[cfg(not(no_global_oom_handling))] pub mod binary_heap; +#[cfg(not(no_global_oom_handling))] mod btree; +#[cfg(not(no_global_oom_handling))] pub mod linked_list; +#[cfg(not(no_global_oom_handling))] pub mod vec_deque; +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] pub mod btree_map { - //! A map based on a B-Tree. + //! An ordered map based on a B-Tree. #[stable(feature = "rust1", since = "1.0.0")] pub use super::btree::map::*; } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] pub mod btree_set { - //! A set based on a B-Tree. + //! An ordered set based on a B-Tree. #[stable(feature = "rust1", since = "1.0.0")] pub use super::btree::set::*; } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] #[doc(no_inline)] pub use binary_heap::BinaryHeap; +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] #[doc(no_inline)] pub use btree_map::BTreeMap; +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] #[doc(no_inline)] pub use btree_set::BTreeSet; +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] #[doc(no_inline)] pub use linked_list::LinkedList; +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] #[doc(no_inline)] pub use vec_deque::VecDeque; -use crate::alloc::{Layout, LayoutErr}; +use crate::alloc::{Layout, LayoutError}; use core::fmt::Display; /// The error type for `try_reserve` methods. #[derive(Clone, PartialEq, Eq, Debug)] -#[unstable(feature = "try_reserve", reason = "new API", issue = "48043")] -pub enum TryReserveError { +#[stable(feature = "try_reserve", since = "1.57.0")] +pub struct TryReserveError { + kind: TryReserveErrorKind, +} + +impl TryReserveError { + /// Details about the allocation that caused the error + #[inline] + #[must_use] + #[unstable( + feature = "try_reserve_kind", + reason = "Uncertain how much info should be exposed", + issue = "48043" + )] + pub fn kind(&self) -> TryReserveErrorKind { + self.kind.clone() + } +} + +/// Details of the allocation that caused a `TryReserveError` +#[derive(Clone, PartialEq, Eq, Debug)] +#[unstable( + feature = "try_reserve_kind", + reason = "Uncertain how much info should be exposed", + issue = "48043" +)] +pub enum TryReserveErrorKind { /// Error due to the computed capacity exceeding the collection's maximum /// (usually `isize::MAX` bytes). CapacityOverflow, @@ -70,26 +106,41 @@ pub enum TryReserveError { }, } -#[unstable(feature = "try_reserve", reason = "new API", issue = "48043")] -impl From for TryReserveError { +#[unstable( + feature = "try_reserve_kind", + reason = "Uncertain how much info should be exposed", + issue = "48043" +)] +impl From for TryReserveError { #[inline] - fn from(_: LayoutErr) -> Self { - TryReserveError::CapacityOverflow + fn from(kind: TryReserveErrorKind) -> Self { + Self { kind } } } -#[unstable(feature = "try_reserve", reason = "new API", issue = "48043")] +#[unstable(feature = "try_reserve_kind", reason = "new API", issue = "48043")] +impl From for TryReserveErrorKind { + /// Always evaluates to [`TryReserveErrorKind::CapacityOverflow`]. + #[inline] + fn from(_: LayoutError) -> Self { + TryReserveErrorKind::CapacityOverflow + } +} + +#[stable(feature = "try_reserve", since = "1.57.0")] impl Display for TryReserveError { fn fmt( &self, fmt: &mut core::fmt::Formatter<'_>, ) -> core::result::Result<(), core::fmt::Error> { fmt.write_str("memory allocation failed")?; - let reason = match &self { - TryReserveError::CapacityOverflow => { + let reason = match self.kind { + TryReserveErrorKind::CapacityOverflow => { " because the computed capacity exceeded the collection's maximum" } - TryReserveError::AllocError { .. } => " because the memory allocator returned a error", + TryReserveErrorKind::AllocError { .. } => { + " because the memory allocator returned an error" + } }; fmt.write_str(reason) } @@ -101,3 +152,6 @@ trait SpecExtend { /// Extends `self` with the contents of the given iterator. fn spec_extend(&mut self, iter: I); } + +#[stable(feature = "try_reserve", since = "1.57.0")] +impl core::error::Error for TryReserveError {} diff --git a/crux-mir/lib/alloc/src/collections/vec_deque.rs b/crux-mir/lib/alloc/src/collections/vec_deque.rs deleted file mode 100644 index 9d56f1770..000000000 --- a/crux-mir/lib/alloc/src/collections/vec_deque.rs +++ /dev/null @@ -1,2868 +0,0 @@ -//! A double-ended queue implemented with a growable ring buffer. -//! -//! This queue has `O(1)` amortized inserts and removals from both ends of the -//! container. It also has `O(1)` indexing like a vector. The contained elements -//! are not required to be copyable, and the queue will be sendable if the -//! contained type is sendable. - -#![stable(feature = "rust1", since = "1.0.0")] - -use core::array::LengthAtMost32; -use core::cmp::{self, Ordering}; -use core::fmt; -use core::hash::{Hash, Hasher}; -use core::iter::{once, repeat_with, FromIterator, FusedIterator}; -use core::mem::{self, replace}; -use core::ops::Bound::{Excluded, Included, Unbounded}; -use core::ops::{Index, IndexMut, RangeBounds, Try}; -use core::ptr::{self, NonNull}; -use core::slice; - -use crate::collections::TryReserveError; -use crate::raw_vec::RawVec; -use crate::vec::Vec; - -#[stable(feature = "drain", since = "1.6.0")] -pub use self::drain::Drain; - -mod drain; - -#[cfg(test)] -mod tests; - -const INITIAL_CAPACITY: usize = 7; // 2^3 - 1 -const MINIMUM_CAPACITY: usize = 1; // 2 - 1 -#[cfg(target_pointer_width = "16")] -const MAXIMUM_ZST_CAPACITY: usize = 1 << (16 - 1); // Largest possible power of two -#[cfg(target_pointer_width = "32")] -const MAXIMUM_ZST_CAPACITY: usize = 1 << (32 - 1); // Largest possible power of two -#[cfg(target_pointer_width = "64")] -const MAXIMUM_ZST_CAPACITY: usize = 1 << (64 - 1); // Largest possible power of two - -/// A double-ended queue implemented with a growable ring buffer. -/// -/// The "default" usage of this type as a queue is to use [`push_back`] to add to -/// the queue, and [`pop_front`] to remove from the queue. [`extend`] and [`append`] -/// push onto the back in this manner, and iterating over `VecDeque` goes front -/// to back. -/// -/// [`push_back`]: #method.push_back -/// [`pop_front`]: #method.pop_front -/// [`extend`]: #method.extend -/// [`append`]: #method.append -#[stable(feature = "rust1", since = "1.0.0")] -pub struct VecDeque { - // tail and head are pointers into the buffer. Tail always points - // to the first element that could be read, Head always points - // to where data should be written. - // If tail == head the buffer is empty. The length of the ringbuffer - // is defined as the distance between the two. - tail: usize, - head: usize, - buf: RawVec, -} - -/// PairSlices pairs up equal length slice parts of two deques -/// -/// For example, given deques "A" and "B" with the following division into slices: -/// -/// A: [0 1 2] [3 4 5] -/// B: [a b] [c d e] -/// -/// It produces the following sequence of matching slices: -/// -/// ([0 1], [a b]) -/// ([2], [c]) -/// ([3 4], [d e]) -/// -/// and the uneven remainder of either A or B is skipped. -struct PairSlices<'a, 'b, T> { - a0: &'a mut [T], - a1: &'a mut [T], - b0: &'b [T], - b1: &'b [T], -} - -impl<'a, 'b, T> PairSlices<'a, 'b, T> { - fn from(to: &'a mut VecDeque, from: &'b VecDeque) -> Self { - let (a0, a1) = to.as_mut_slices(); - let (b0, b1) = from.as_slices(); - PairSlices { a0, a1, b0, b1 } - } - - fn has_remainder(&self) -> bool { - !self.b0.is_empty() - } - - fn remainder(self) -> impl Iterator { - once(self.b0).chain(once(self.b1)) - } -} - -impl<'a, 'b, T> Iterator for PairSlices<'a, 'b, T> { - type Item = (&'a mut [T], &'b [T]); - fn next(&mut self) -> Option { - // Get next part length - let part = cmp::min(self.a0.len(), self.b0.len()); - if part == 0 { - return None; - } - let (p0, p1) = replace(&mut self.a0, &mut []).split_at_mut(part); - let (q0, q1) = self.b0.split_at(part); - - // Move a1 into a0, if it's empty (and b1, b0 the same way). - self.a0 = p1; - self.b0 = q1; - if self.a0.is_empty() { - self.a0 = replace(&mut self.a1, &mut []); - } - if self.b0.is_empty() { - self.b0 = replace(&mut self.b1, &[]); - } - Some((p0, q0)) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Clone for VecDeque { - fn clone(&self) -> VecDeque { - self.iter().cloned().collect() - } - - fn clone_from(&mut self, other: &Self) { - self.truncate(other.len()); - - let mut iter = PairSlices::from(self, other); - while let Some((dst, src)) = iter.next() { - dst.clone_from_slice(&src); - } - - if iter.has_remainder() { - for remainder in iter.remainder() { - self.extend(remainder.iter().cloned()); - } - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -unsafe impl<#[may_dangle] T> Drop for VecDeque { - fn drop(&mut self) { - /// Runs the destructor for all items in the slice when it gets dropped (normally or - /// during unwinding). - struct Dropper<'a, T>(&'a mut [T]); - - impl<'a, T> Drop for Dropper<'a, T> { - fn drop(&mut self) { - unsafe { - ptr::drop_in_place(self.0); - } - } - } - - let (front, back) = self.as_mut_slices(); - unsafe { - let _back_dropper = Dropper(back); - // use drop for [T] - ptr::drop_in_place(front); - } - // RawVec handles deallocation - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Default for VecDeque { - /// Creates an empty `VecDeque`. - #[inline] - fn default() -> VecDeque { - VecDeque::new() - } -} - -impl VecDeque { - /// Marginally more convenient - #[inline] - fn ptr(&self) -> *mut T { - self.buf.ptr() - } - - /// Marginally more convenient - #[inline] - fn cap(&self) -> usize { - if mem::size_of::() == 0 { - // For zero sized types, we are always at maximum capacity - MAXIMUM_ZST_CAPACITY - } else { - self.buf.capacity() - } - } - - /// Turn ptr into a slice - #[inline] - unsafe fn buffer_as_slice(&self) -> &[T] { - slice::from_raw_parts(self.ptr(), self.cap()) - } - - /// Turn ptr into a mut slice - #[inline] - unsafe fn buffer_as_mut_slice(&mut self) -> &mut [T] { - slice::from_raw_parts_mut(self.ptr(), self.cap()) - } - - /// Moves an element out of the buffer - #[inline] - unsafe fn buffer_read(&mut self, off: usize) -> T { - ptr::read(self.ptr().add(off)) - } - - /// Writes an element into the buffer, moving it. - #[inline] - unsafe fn buffer_write(&mut self, off: usize, value: T) { - ptr::write(self.ptr().add(off), value); - } - - /// Returns `true` if the buffer is at full capacity. - #[inline] - fn is_full(&self) -> bool { - self.cap() - self.len() == 1 - } - - /// Returns the index in the underlying buffer for a given logical element - /// index. - #[inline] - fn wrap_index(&self, idx: usize) -> usize { - wrap_index(idx, self.cap()) - } - - /// Returns the index in the underlying buffer for a given logical element - /// index + addend. - #[inline] - fn wrap_add(&self, idx: usize, addend: usize) -> usize { - wrap_index(idx.wrapping_add(addend), self.cap()) - } - - /// Returns the index in the underlying buffer for a given logical element - /// index - subtrahend. - #[inline] - fn wrap_sub(&self, idx: usize, subtrahend: usize) -> usize { - wrap_index(idx.wrapping_sub(subtrahend), self.cap()) - } - - /// Copies a contiguous block of memory len long from src to dst - #[inline] - unsafe fn copy(&self, dst: usize, src: usize, len: usize) { - debug_assert!( - dst + len <= self.cap(), - "cpy dst={} src={} len={} cap={}", - dst, - src, - len, - self.cap() - ); - debug_assert!( - src + len <= self.cap(), - "cpy dst={} src={} len={} cap={}", - dst, - src, - len, - self.cap() - ); - ptr::copy(self.ptr().add(src), self.ptr().add(dst), len); - } - - /// Copies a contiguous block of memory len long from src to dst - #[inline] - unsafe fn copy_nonoverlapping(&self, dst: usize, src: usize, len: usize) { - debug_assert!( - dst + len <= self.cap(), - "cno dst={} src={} len={} cap={}", - dst, - src, - len, - self.cap() - ); - debug_assert!( - src + len <= self.cap(), - "cno dst={} src={} len={} cap={}", - dst, - src, - len, - self.cap() - ); - ptr::copy_nonoverlapping(self.ptr().add(src), self.ptr().add(dst), len); - } - - /// Copies a potentially wrapping block of memory len long from src to dest. - /// (abs(dst - src) + len) must be no larger than cap() (There must be at - /// most one continuous overlapping region between src and dest). - unsafe fn wrap_copy(&self, dst: usize, src: usize, len: usize) { - #[allow(dead_code)] - fn diff(a: usize, b: usize) -> usize { - if a <= b { b - a } else { a - b } - } - debug_assert!( - cmp::min(diff(dst, src), self.cap() - diff(dst, src)) + len <= self.cap(), - "wrc dst={} src={} len={} cap={}", - dst, - src, - len, - self.cap() - ); - - if src == dst || len == 0 { - return; - } - - let dst_after_src = self.wrap_sub(dst, src) < len; - - let src_pre_wrap_len = self.cap() - src; - let dst_pre_wrap_len = self.cap() - dst; - let src_wraps = src_pre_wrap_len < len; - let dst_wraps = dst_pre_wrap_len < len; - - match (dst_after_src, src_wraps, dst_wraps) { - (_, false, false) => { - // src doesn't wrap, dst doesn't wrap - // - // S . . . - // 1 [_ _ A A B B C C _] - // 2 [_ _ A A A A B B _] - // D . . . - // - self.copy(dst, src, len); - } - (false, false, true) => { - // dst before src, src doesn't wrap, dst wraps - // - // S . . . - // 1 [A A B B _ _ _ C C] - // 2 [A A B B _ _ _ A A] - // 3 [B B B B _ _ _ A A] - // . . D . - // - self.copy(dst, src, dst_pre_wrap_len); - self.copy(0, src + dst_pre_wrap_len, len - dst_pre_wrap_len); - } - (true, false, true) => { - // src before dst, src doesn't wrap, dst wraps - // - // S . . . - // 1 [C C _ _ _ A A B B] - // 2 [B B _ _ _ A A B B] - // 3 [B B _ _ _ A A A A] - // . . D . - // - self.copy(0, src + dst_pre_wrap_len, len - dst_pre_wrap_len); - self.copy(dst, src, dst_pre_wrap_len); - } - (false, true, false) => { - // dst before src, src wraps, dst doesn't wrap - // - // . . S . - // 1 [C C _ _ _ A A B B] - // 2 [C C _ _ _ B B B B] - // 3 [C C _ _ _ B B C C] - // D . . . - // - self.copy(dst, src, src_pre_wrap_len); - self.copy(dst + src_pre_wrap_len, 0, len - src_pre_wrap_len); - } - (true, true, false) => { - // src before dst, src wraps, dst doesn't wrap - // - // . . S . - // 1 [A A B B _ _ _ C C] - // 2 [A A A A _ _ _ C C] - // 3 [C C A A _ _ _ C C] - // D . . . - // - self.copy(dst + src_pre_wrap_len, 0, len - src_pre_wrap_len); - self.copy(dst, src, src_pre_wrap_len); - } - (false, true, true) => { - // dst before src, src wraps, dst wraps - // - // . . . S . - // 1 [A B C D _ E F G H] - // 2 [A B C D _ E G H H] - // 3 [A B C D _ E G H A] - // 4 [B C C D _ E G H A] - // . . D . . - // - debug_assert!(dst_pre_wrap_len > src_pre_wrap_len); - let delta = dst_pre_wrap_len - src_pre_wrap_len; - self.copy(dst, src, src_pre_wrap_len); - self.copy(dst + src_pre_wrap_len, 0, delta); - self.copy(0, delta, len - dst_pre_wrap_len); - } - (true, true, true) => { - // src before dst, src wraps, dst wraps - // - // . . S . . - // 1 [A B C D _ E F G H] - // 2 [A A B D _ E F G H] - // 3 [H A B D _ E F G H] - // 4 [H A B D _ E F F G] - // . . . D . - // - debug_assert!(src_pre_wrap_len > dst_pre_wrap_len); - let delta = src_pre_wrap_len - dst_pre_wrap_len; - self.copy(delta, 0, len - src_pre_wrap_len); - self.copy(0, self.cap() - delta, delta); - self.copy(dst, src, dst_pre_wrap_len); - } - } - } - - /// Frobs the head and tail sections around to handle the fact that we - /// just reallocated. Unsafe because it trusts old_capacity. - #[inline] - unsafe fn handle_capacity_increase(&mut self, old_capacity: usize) { - let new_capacity = self.cap(); - - // Move the shortest contiguous section of the ring buffer - // T H - // [o o o o o o o . ] - // T H - // A [o o o o o o o . . . . . . . . . ] - // H T - // [o o . o o o o o ] - // T H - // B [. . . o o o o o o o . . . . . . ] - // H T - // [o o o o o . o o ] - // H T - // C [o o o o o . . . . . . . . . o o ] - - if self.tail <= self.head { - // A - // Nop - } else if self.head < old_capacity - self.tail { - // B - self.copy_nonoverlapping(old_capacity, 0, self.head); - self.head += old_capacity; - debug_assert!(self.head > self.tail); - } else { - // C - let new_tail = new_capacity - (old_capacity - self.tail); - self.copy_nonoverlapping(new_tail, self.tail, old_capacity - self.tail); - self.tail = new_tail; - debug_assert!(self.head < self.tail); - } - debug_assert!(self.head < self.cap()); - debug_assert!(self.tail < self.cap()); - debug_assert!(self.cap().count_ones() == 1); - } -} - -impl VecDeque { - /// Creates an empty `VecDeque`. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let vector: VecDeque = VecDeque::new(); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn new() -> VecDeque { - VecDeque::with_capacity(INITIAL_CAPACITY) - } - - /// Creates an empty `VecDeque` with space for at least `capacity` elements. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let vector: VecDeque = VecDeque::with_capacity(10); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn with_capacity(capacity: usize) -> VecDeque { - // +1 since the ringbuffer always leaves one space empty - let cap = cmp::max(capacity + 1, MINIMUM_CAPACITY + 1).next_power_of_two(); - assert!(cap > capacity, "capacity overflow"); - - VecDeque { tail: 0, head: 0, buf: RawVec::with_capacity(cap) } - } - - /// Retrieves an element in the `VecDeque` by index. - /// - /// Element at index 0 is the front of the queue. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf = VecDeque::new(); - /// buf.push_back(3); - /// buf.push_back(4); - /// buf.push_back(5); - /// assert_eq!(buf.get(1), Some(&4)); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn get(&self, index: usize) -> Option<&T> { - if index < self.len() { - let idx = self.wrap_add(self.tail, index); - unsafe { Some(&*self.ptr().add(idx)) } - } else { - None - } - } - - /// Retrieves an element in the `VecDeque` mutably by index. - /// - /// Element at index 0 is the front of the queue. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf = VecDeque::new(); - /// buf.push_back(3); - /// buf.push_back(4); - /// buf.push_back(5); - /// if let Some(elem) = buf.get_mut(1) { - /// *elem = 7; - /// } - /// - /// assert_eq!(buf[1], 7); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn get_mut(&mut self, index: usize) -> Option<&mut T> { - if index < self.len() { - let idx = self.wrap_add(self.tail, index); - unsafe { Some(&mut *self.ptr().add(idx)) } - } else { - None - } - } - - /// Swaps elements at indices `i` and `j`. - /// - /// `i` and `j` may be equal. - /// - /// Element at index 0 is the front of the queue. - /// - /// # Panics - /// - /// Panics if either index is out of bounds. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf = VecDeque::new(); - /// buf.push_back(3); - /// buf.push_back(4); - /// buf.push_back(5); - /// assert_eq!(buf, [3, 4, 5]); - /// buf.swap(0, 2); - /// assert_eq!(buf, [5, 4, 3]); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn swap(&mut self, i: usize, j: usize) { - assert!(i < self.len()); - assert!(j < self.len()); - let ri = self.wrap_add(self.tail, i); - let rj = self.wrap_add(self.tail, j); - unsafe { ptr::swap(self.ptr().add(ri), self.ptr().add(rj)) } - } - - /// Returns the number of elements the `VecDeque` can hold without - /// reallocating. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let buf: VecDeque = VecDeque::with_capacity(10); - /// assert!(buf.capacity() >= 10); - /// ``` - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn capacity(&self) -> usize { - self.cap() - 1 - } - - /// Reserves the minimum capacity for exactly `additional` more elements to be inserted in the - /// given `VecDeque`. Does nothing if the capacity is already sufficient. - /// - /// Note that the allocator may give the collection more space than it requests. Therefore - /// capacity can not be relied upon to be precisely minimal. Prefer [`reserve`] if future - /// insertions are expected. - /// - /// # Panics - /// - /// Panics if the new capacity overflows `usize`. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf: VecDeque = vec![1].into_iter().collect(); - /// buf.reserve_exact(10); - /// assert!(buf.capacity() >= 11); - /// ``` - /// - /// [`reserve`]: #method.reserve - #[stable(feature = "rust1", since = "1.0.0")] - pub fn reserve_exact(&mut self, additional: usize) { - self.reserve(additional); - } - - /// Reserves capacity for at least `additional` more elements to be inserted in the given - /// `VecDeque`. The collection may reserve more space to avoid frequent reallocations. - /// - /// # Panics - /// - /// Panics if the new capacity overflows `usize`. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf: VecDeque = vec![1].into_iter().collect(); - /// buf.reserve(10); - /// assert!(buf.capacity() >= 11); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn reserve(&mut self, additional: usize) { - let old_cap = self.cap(); - let used_cap = self.len() + 1; - let new_cap = used_cap - .checked_add(additional) - .and_then(|needed_cap| needed_cap.checked_next_power_of_two()) - .expect("capacity overflow"); - - if new_cap > old_cap { - self.buf.reserve_exact(used_cap, new_cap - used_cap); - unsafe { - self.handle_capacity_increase(old_cap); - } - } - } - - /// Tries to reserves the minimum capacity for exactly `additional` more elements to - /// be inserted in the given `VecDeque`. After calling `reserve_exact`, - /// capacity will be greater than or equal to `self.len() + additional`. - /// Does nothing if the capacity is already sufficient. - /// - /// Note that the allocator may give the collection more space than it - /// requests. Therefore, capacity can not be relied upon to be precisely - /// minimal. Prefer `reserve` if future insertions are expected. - /// - /// # Errors - /// - /// If the capacity overflows, or the allocator reports a failure, then an error - /// is returned. - /// - /// # Examples - /// - /// ``` - /// #![feature(try_reserve)] - /// use std::collections::TryReserveError; - /// use std::collections::VecDeque; - /// - /// fn process_data(data: &[u32]) -> Result, TryReserveError> { - /// let mut output = VecDeque::new(); - /// - /// // Pre-reserve the memory, exiting if we can't - /// output.try_reserve_exact(data.len())?; - /// - /// // Now we know this can't OOM in the middle of our complex work - /// output.extend(data.iter().map(|&val| { - /// val * 2 + 5 // very complicated - /// })); - /// - /// Ok(output) - /// } - /// # process_data(&[1, 2, 3]).expect("why is the test harness OOMing on 12 bytes?"); - /// ``` - #[unstable(feature = "try_reserve", reason = "new API", issue = "48043")] - pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> { - self.try_reserve(additional) - } - - /// Tries to reserve capacity for at least `additional` more elements to be inserted - /// in the given `VecDeque`. The collection may reserve more space to avoid - /// frequent reallocations. After calling `reserve`, capacity will be - /// greater than or equal to `self.len() + additional`. Does nothing if - /// capacity is already sufficient. - /// - /// # Errors - /// - /// If the capacity overflows, or the allocator reports a failure, then an error - /// is returned. - /// - /// # Examples - /// - /// ``` - /// #![feature(try_reserve)] - /// use std::collections::TryReserveError; - /// use std::collections::VecDeque; - /// - /// fn process_data(data: &[u32]) -> Result, TryReserveError> { - /// let mut output = VecDeque::new(); - /// - /// // Pre-reserve the memory, exiting if we can't - /// output.try_reserve(data.len())?; - /// - /// // Now we know this can't OOM in the middle of our complex work - /// output.extend(data.iter().map(|&val| { - /// val * 2 + 5 // very complicated - /// })); - /// - /// Ok(output) - /// } - /// # process_data(&[1, 2, 3]).expect("why is the test harness OOMing on 12 bytes?"); - /// ``` - #[unstable(feature = "try_reserve", reason = "new API", issue = "48043")] - pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { - let old_cap = self.cap(); - let used_cap = self.len() + 1; - let new_cap = used_cap - .checked_add(additional) - .and_then(|needed_cap| needed_cap.checked_next_power_of_two()) - .ok_or(TryReserveError::CapacityOverflow)?; - - if new_cap > old_cap { - self.buf.try_reserve_exact(used_cap, new_cap - used_cap)?; - unsafe { - self.handle_capacity_increase(old_cap); - } - } - Ok(()) - } - - /// Shrinks the capacity of the `VecDeque` as much as possible. - /// - /// It will drop down as close as possible to the length but the allocator may still inform the - /// `VecDeque` that there is space for a few more elements. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf = VecDeque::with_capacity(15); - /// buf.extend(0..4); - /// assert_eq!(buf.capacity(), 15); - /// buf.shrink_to_fit(); - /// assert!(buf.capacity() >= 4); - /// ``` - #[stable(feature = "deque_extras_15", since = "1.5.0")] - pub fn shrink_to_fit(&mut self) { - self.shrink_to(0); - } - - /// Shrinks the capacity of the `VecDeque` with a lower bound. - /// - /// The capacity will remain at least as large as both the length - /// and the supplied value. - /// - /// Panics if the current capacity is smaller than the supplied - /// minimum capacity. - /// - /// # Examples - /// - /// ``` - /// #![feature(shrink_to)] - /// use std::collections::VecDeque; - /// - /// let mut buf = VecDeque::with_capacity(15); - /// buf.extend(0..4); - /// assert_eq!(buf.capacity(), 15); - /// buf.shrink_to(6); - /// assert!(buf.capacity() >= 6); - /// buf.shrink_to(0); - /// assert!(buf.capacity() >= 4); - /// ``` - #[unstable(feature = "shrink_to", reason = "new API", issue = "56431")] - pub fn shrink_to(&mut self, min_capacity: usize) { - assert!(self.capacity() >= min_capacity, "Tried to shrink to a larger capacity"); - - // +1 since the ringbuffer always leaves one space empty - // len + 1 can't overflow for an existing, well-formed ringbuffer. - let target_cap = cmp::max(cmp::max(min_capacity, self.len()) + 1, MINIMUM_CAPACITY + 1) - .next_power_of_two(); - - if target_cap < self.cap() { - // There are three cases of interest: - // All elements are out of desired bounds - // Elements are contiguous, and head is out of desired bounds - // Elements are discontiguous, and tail is out of desired bounds - // - // At all other times, element positions are unaffected. - // - // Indicates that elements at the head should be moved. - let head_outside = self.head == 0 || self.head >= target_cap; - // Move elements from out of desired bounds (positions after target_cap) - if self.tail >= target_cap && head_outside { - // T H - // [. . . . . . . . o o o o o o o . ] - // T H - // [o o o o o o o . ] - unsafe { - self.copy_nonoverlapping(0, self.tail, self.len()); - } - self.head = self.len(); - self.tail = 0; - } else if self.tail != 0 && self.tail < target_cap && head_outside { - // T H - // [. . . o o o o o o o . . . . . . ] - // H T - // [o o . o o o o o ] - let len = self.wrap_sub(self.head, target_cap); - unsafe { - self.copy_nonoverlapping(0, target_cap, len); - } - self.head = len; - debug_assert!(self.head < self.tail); - } else if self.tail >= target_cap { - // H T - // [o o o o o . . . . . . . . . o o ] - // H T - // [o o o o o . o o ] - debug_assert!(self.wrap_sub(self.head, 1) < target_cap); - let len = self.cap() - self.tail; - let new_tail = target_cap - len; - unsafe { - self.copy_nonoverlapping(new_tail, self.tail, len); - } - self.tail = new_tail; - debug_assert!(self.head < self.tail); - } - - self.buf.shrink_to_fit(target_cap); - - debug_assert!(self.head < self.cap()); - debug_assert!(self.tail < self.cap()); - debug_assert!(self.cap().count_ones() == 1); - } - } - - /// Shortens the `VecDeque`, keeping the first `len` elements and dropping - /// the rest. - /// - /// If `len` is greater than the `VecDeque`'s current length, this has no - /// effect. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf = VecDeque::new(); - /// buf.push_back(5); - /// buf.push_back(10); - /// buf.push_back(15); - /// assert_eq!(buf, [5, 10, 15]); - /// buf.truncate(1); - /// assert_eq!(buf, [5]); - /// ``` - #[stable(feature = "deque_extras", since = "1.16.0")] - pub fn truncate(&mut self, len: usize) { - /// Runs the destructor for all items in the slice when it gets dropped (normally or - /// during unwinding). - struct Dropper<'a, T>(&'a mut [T]); - - impl<'a, T> Drop for Dropper<'a, T> { - fn drop(&mut self) { - unsafe { - ptr::drop_in_place(self.0); - } - } - } - - // Safe because: - // - // * Any slice passed to `drop_in_place` is valid; the second case has - // `len <= front.len()` and returning on `len > self.len()` ensures - // `begin <= back.len()` in the first case - // * The head of the VecDeque is moved before calling `drop_in_place`, - // so no value is dropped twice if `drop_in_place` panics - unsafe { - if len > self.len() { - return; - } - let num_dropped = self.len() - len; - let (front, back) = self.as_mut_slices(); - if len > front.len() { - let begin = len - front.len(); - let drop_back = back.get_unchecked_mut(begin..) as *mut _; - self.head = self.wrap_sub(self.head, num_dropped); - ptr::drop_in_place(drop_back); - } else { - let drop_back = back as *mut _; - let drop_front = front.get_unchecked_mut(len..) as *mut _; - self.head = self.wrap_sub(self.head, num_dropped); - - // Make sure the second half is dropped even when a destructor - // in the first one panics. - let _back_dropper = Dropper(&mut *drop_back); - ptr::drop_in_place(drop_front); - } - } - } - - /// Returns a front-to-back iterator. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf = VecDeque::new(); - /// buf.push_back(5); - /// buf.push_back(3); - /// buf.push_back(4); - /// let b: &[_] = &[&5, &3, &4]; - /// let c: Vec<&i32> = buf.iter().collect(); - /// assert_eq!(&c[..], b); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn iter(&self) -> Iter<'_, T> { - Iter { tail: self.tail, head: self.head, ring: unsafe { self.buffer_as_slice() } } - } - - /// Returns a front-to-back iterator that returns mutable references. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf = VecDeque::new(); - /// buf.push_back(5); - /// buf.push_back(3); - /// buf.push_back(4); - /// for num in buf.iter_mut() { - /// *num = *num - 2; - /// } - /// let b: &[_] = &[&mut 3, &mut 1, &mut 2]; - /// assert_eq!(&buf.iter_mut().collect::>()[..], b); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn iter_mut(&mut self) -> IterMut<'_, T> { - IterMut { tail: self.tail, head: self.head, ring: unsafe { self.buffer_as_mut_slice() } } - } - - /// Returns a pair of slices which contain, in order, the contents of the - /// `VecDeque`. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut vector = VecDeque::new(); - /// - /// vector.push_back(0); - /// vector.push_back(1); - /// vector.push_back(2); - /// - /// assert_eq!(vector.as_slices(), (&[0, 1, 2][..], &[][..])); - /// - /// vector.push_front(10); - /// vector.push_front(9); - /// - /// assert_eq!(vector.as_slices(), (&[9, 10][..], &[0, 1, 2][..])); - /// ``` - #[inline] - #[stable(feature = "deque_extras_15", since = "1.5.0")] - pub fn as_slices(&self) -> (&[T], &[T]) { - unsafe { - let buf = self.buffer_as_slice(); - RingSlices::ring_slices(buf, self.head, self.tail) - } - } - - /// Returns a pair of slices which contain, in order, the contents of the - /// `VecDeque`. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut vector = VecDeque::new(); - /// - /// vector.push_back(0); - /// vector.push_back(1); - /// - /// vector.push_front(10); - /// vector.push_front(9); - /// - /// vector.as_mut_slices().0[0] = 42; - /// vector.as_mut_slices().1[0] = 24; - /// assert_eq!(vector.as_slices(), (&[42, 10][..], &[24, 1][..])); - /// ``` - #[inline] - #[stable(feature = "deque_extras_15", since = "1.5.0")] - pub fn as_mut_slices(&mut self) -> (&mut [T], &mut [T]) { - unsafe { - let head = self.head; - let tail = self.tail; - let buf = self.buffer_as_mut_slice(); - RingSlices::ring_slices(buf, head, tail) - } - } - - /// Returns the number of elements in the `VecDeque`. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut v = VecDeque::new(); - /// assert_eq!(v.len(), 0); - /// v.push_back(1); - /// assert_eq!(v.len(), 1); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn len(&self) -> usize { - count(self.tail, self.head, self.cap()) - } - - /// Returns `true` if the `VecDeque` is empty. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut v = VecDeque::new(); - /// assert!(v.is_empty()); - /// v.push_front(1); - /// assert!(!v.is_empty()); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn is_empty(&self) -> bool { - self.tail == self.head - } - - /// Creates a draining iterator that removes the specified range in the - /// `VecDeque` and yields the removed items. - /// - /// Note 1: The element range is removed even if the iterator is not - /// consumed until the end. - /// - /// Note 2: It is unspecified how many elements are removed from the deque, - /// if the `Drain` value is not dropped, but the borrow it holds expires - /// (e.g., due to `mem::forget`). - /// - /// # Panics - /// - /// Panics if the starting point is greater than the end point or if - /// the end point is greater than the length of the vector. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut v: VecDeque<_> = vec![1, 2, 3].into_iter().collect(); - /// let drained = v.drain(2..).collect::>(); - /// assert_eq!(drained, [3]); - /// assert_eq!(v, [1, 2]); - /// - /// // A full range clears all contents - /// v.drain(..); - /// assert!(v.is_empty()); - /// ``` - #[inline] - #[stable(feature = "drain", since = "1.6.0")] - pub fn drain(&mut self, range: R) -> Drain<'_, T> - where - R: RangeBounds, - { - // Memory safety - // - // When the Drain is first created, the source deque is shortened to - // make sure no uninitialized or moved-from elements are accessible at - // all if the Drain's destructor never gets to run. - // - // Drain will ptr::read out the values to remove. - // When finished, the remaining data will be copied back to cover the hole, - // and the head/tail values will be restored correctly. - // - let len = self.len(); - let start = match range.start_bound() { - Included(&n) => n, - Excluded(&n) => n + 1, - Unbounded => 0, - }; - let end = match range.end_bound() { - Included(&n) => n + 1, - Excluded(&n) => n, - Unbounded => len, - }; - assert!(start <= end, "drain lower bound was too large"); - assert!(end <= len, "drain upper bound was too large"); - - // The deque's elements are parted into three segments: - // * self.tail -> drain_tail - // * drain_tail -> drain_head - // * drain_head -> self.head - // - // T = self.tail; H = self.head; t = drain_tail; h = drain_head - // - // We store drain_tail as self.head, and drain_head and self.head as - // after_tail and after_head respectively on the Drain. This also - // truncates the effective array such that if the Drain is leaked, we - // have forgotten about the potentially moved values after the start of - // the drain. - // - // T t h H - // [. . . o o x x o o . . .] - // - let drain_tail = self.wrap_add(self.tail, start); - let drain_head = self.wrap_add(self.tail, end); - let head = self.head; - - // "forget" about the values after the start of the drain until after - // the drain is complete and the Drain destructor is run. - self.head = drain_tail; - - Drain { - deque: NonNull::from(&mut *self), - after_tail: drain_head, - after_head: head, - iter: Iter { - tail: drain_tail, - head: drain_head, - // Crucially, we only create shared references from `self` here and read from - // it. We do not write to `self` nor reborrow to a mutable reference. - // Hence the raw pointer we created above, for `deque`, remains valid. - ring: unsafe { self.buffer_as_slice() }, - }, - } - } - - /// Clears the `VecDeque`, removing all values. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut v = VecDeque::new(); - /// v.push_back(1); - /// v.clear(); - /// assert!(v.is_empty()); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - #[inline] - pub fn clear(&mut self) { - self.truncate(0); - } - - /// Returns `true` if the `VecDeque` contains an element equal to the - /// given value. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut vector: VecDeque = VecDeque::new(); - /// - /// vector.push_back(0); - /// vector.push_back(1); - /// - /// assert_eq!(vector.contains(&1), true); - /// assert_eq!(vector.contains(&10), false); - /// ``` - #[stable(feature = "vec_deque_contains", since = "1.12.0")] - pub fn contains(&self, x: &T) -> bool - where - T: PartialEq, - { - let (a, b) = self.as_slices(); - a.contains(x) || b.contains(x) - } - - /// Provides a reference to the front element, or `None` if the `VecDeque` is - /// empty. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut d = VecDeque::new(); - /// assert_eq!(d.front(), None); - /// - /// d.push_back(1); - /// d.push_back(2); - /// assert_eq!(d.front(), Some(&1)); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn front(&self) -> Option<&T> { - if !self.is_empty() { Some(&self[0]) } else { None } - } - - /// Provides a mutable reference to the front element, or `None` if the - /// `VecDeque` is empty. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut d = VecDeque::new(); - /// assert_eq!(d.front_mut(), None); - /// - /// d.push_back(1); - /// d.push_back(2); - /// match d.front_mut() { - /// Some(x) => *x = 9, - /// None => (), - /// } - /// assert_eq!(d.front(), Some(&9)); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn front_mut(&mut self) -> Option<&mut T> { - if !self.is_empty() { Some(&mut self[0]) } else { None } - } - - /// Provides a reference to the back element, or `None` if the `VecDeque` is - /// empty. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut d = VecDeque::new(); - /// assert_eq!(d.back(), None); - /// - /// d.push_back(1); - /// d.push_back(2); - /// assert_eq!(d.back(), Some(&2)); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn back(&self) -> Option<&T> { - if !self.is_empty() { Some(&self[self.len() - 1]) } else { None } - } - - /// Provides a mutable reference to the back element, or `None` if the - /// `VecDeque` is empty. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut d = VecDeque::new(); - /// assert_eq!(d.back(), None); - /// - /// d.push_back(1); - /// d.push_back(2); - /// match d.back_mut() { - /// Some(x) => *x = 9, - /// None => (), - /// } - /// assert_eq!(d.back(), Some(&9)); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn back_mut(&mut self) -> Option<&mut T> { - let len = self.len(); - if !self.is_empty() { Some(&mut self[len - 1]) } else { None } - } - - /// Removes the first element and returns it, or `None` if the `VecDeque` is - /// empty. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut d = VecDeque::new(); - /// d.push_back(1); - /// d.push_back(2); - /// - /// assert_eq!(d.pop_front(), Some(1)); - /// assert_eq!(d.pop_front(), Some(2)); - /// assert_eq!(d.pop_front(), None); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn pop_front(&mut self) -> Option { - if self.is_empty() { - None - } else { - let tail = self.tail; - self.tail = self.wrap_add(self.tail, 1); - unsafe { Some(self.buffer_read(tail)) } - } - } - - /// Removes the last element from the `VecDeque` and returns it, or `None` if - /// it is empty. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf = VecDeque::new(); - /// assert_eq!(buf.pop_back(), None); - /// buf.push_back(1); - /// buf.push_back(3); - /// assert_eq!(buf.pop_back(), Some(3)); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn pop_back(&mut self) -> Option { - if self.is_empty() { - None - } else { - self.head = self.wrap_sub(self.head, 1); - let head = self.head; - unsafe { Some(self.buffer_read(head)) } - } - } - - /// Prepends an element to the `VecDeque`. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut d = VecDeque::new(); - /// d.push_front(1); - /// d.push_front(2); - /// assert_eq!(d.front(), Some(&2)); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn push_front(&mut self, value: T) { - self.grow_if_necessary(); - - self.tail = self.wrap_sub(self.tail, 1); - let tail = self.tail; - unsafe { - self.buffer_write(tail, value); - } - } - - /// Appends an element to the back of the `VecDeque`. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf = VecDeque::new(); - /// buf.push_back(1); - /// buf.push_back(3); - /// assert_eq!(3, *buf.back().unwrap()); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn push_back(&mut self, value: T) { - self.grow_if_necessary(); - - let head = self.head; - self.head = self.wrap_add(self.head, 1); - unsafe { self.buffer_write(head, value) } - } - - #[inline] - fn is_contiguous(&self) -> bool { - self.tail <= self.head - } - - /// Removes an element from anywhere in the `VecDeque` and returns it, - /// replacing it with the first element. - /// - /// This does not preserve ordering, but is O(1). - /// - /// Returns `None` if `index` is out of bounds. - /// - /// Element at index 0 is the front of the queue. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf = VecDeque::new(); - /// assert_eq!(buf.swap_remove_front(0), None); - /// buf.push_back(1); - /// buf.push_back(2); - /// buf.push_back(3); - /// assert_eq!(buf, [1, 2, 3]); - /// - /// assert_eq!(buf.swap_remove_front(2), Some(3)); - /// assert_eq!(buf, [2, 1]); - /// ``` - #[stable(feature = "deque_extras_15", since = "1.5.0")] - pub fn swap_remove_front(&mut self, index: usize) -> Option { - let length = self.len(); - if length > 0 && index < length && index != 0 { - self.swap(index, 0); - } else if index >= length { - return None; - } - self.pop_front() - } - - /// Removes an element from anywhere in the `VecDeque` and returns it, replacing it with the - /// last element. - /// - /// This does not preserve ordering, but is O(1). - /// - /// Returns `None` if `index` is out of bounds. - /// - /// Element at index 0 is the front of the queue. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf = VecDeque::new(); - /// assert_eq!(buf.swap_remove_back(0), None); - /// buf.push_back(1); - /// buf.push_back(2); - /// buf.push_back(3); - /// assert_eq!(buf, [1, 2, 3]); - /// - /// assert_eq!(buf.swap_remove_back(0), Some(1)); - /// assert_eq!(buf, [3, 2]); - /// ``` - #[stable(feature = "deque_extras_15", since = "1.5.0")] - pub fn swap_remove_back(&mut self, index: usize) -> Option { - let length = self.len(); - if length > 0 && index < length - 1 { - self.swap(index, length - 1); - } else if index >= length { - return None; - } - self.pop_back() - } - - /// Inserts an element at `index` within the `VecDeque`, shifting all elements with indices - /// greater than or equal to `index` towards the back. - /// - /// Element at index 0 is the front of the queue. - /// - /// # Panics - /// - /// Panics if `index` is greater than `VecDeque`'s length - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut vec_deque = VecDeque::new(); - /// vec_deque.push_back('a'); - /// vec_deque.push_back('b'); - /// vec_deque.push_back('c'); - /// assert_eq!(vec_deque, &['a', 'b', 'c']); - /// - /// vec_deque.insert(1, 'd'); - /// assert_eq!(vec_deque, &['a', 'd', 'b', 'c']); - /// ``` - #[stable(feature = "deque_extras_15", since = "1.5.0")] - pub fn insert(&mut self, index: usize, value: T) { - assert!(index <= self.len(), "index out of bounds"); - self.grow_if_necessary(); - - // Move the least number of elements in the ring buffer and insert - // the given object - // - // At most len/2 - 1 elements will be moved. O(min(n, n-i)) - // - // There are three main cases: - // Elements are contiguous - // - special case when tail is 0 - // Elements are discontiguous and the insert is in the tail section - // Elements are discontiguous and the insert is in the head section - // - // For each of those there are two more cases: - // Insert is closer to tail - // Insert is closer to head - // - // Key: H - self.head - // T - self.tail - // o - Valid element - // I - Insertion element - // A - The element that should be after the insertion point - // M - Indicates element was moved - - let idx = self.wrap_add(self.tail, index); - - let distance_to_tail = index; - let distance_to_head = self.len() - index; - - let contiguous = self.is_contiguous(); - - match (contiguous, distance_to_tail <= distance_to_head, idx >= self.tail) { - (true, true, _) if index == 0 => { - // push_front - // - // T - // I H - // [A o o o o o o . . . . . . . . .] - // - // H T - // [A o o o o o o o . . . . . I] - // - - self.tail = self.wrap_sub(self.tail, 1); - } - (true, true, _) => { - unsafe { - // contiguous, insert closer to tail: - // - // T I H - // [. . . o o A o o o o . . . . . .] - // - // T H - // [. . o o I A o o o o . . . . . .] - // M M - // - // contiguous, insert closer to tail and tail is 0: - // - // - // T I H - // [o o A o o o o . . . . . . . . .] - // - // H T - // [o I A o o o o o . . . . . . . o] - // M M - - let new_tail = self.wrap_sub(self.tail, 1); - - self.copy(new_tail, self.tail, 1); - // Already moved the tail, so we only copy `index - 1` elements. - self.copy(self.tail, self.tail + 1, index - 1); - - self.tail = new_tail; - } - } - (true, false, _) => { - unsafe { - // contiguous, insert closer to head: - // - // T I H - // [. . . o o o o A o o . . . . . .] - // - // T H - // [. . . o o o o I A o o . . . . .] - // M M M - - self.copy(idx + 1, idx, self.head - idx); - self.head = self.wrap_add(self.head, 1); - } - } - (false, true, true) => { - unsafe { - // discontiguous, insert closer to tail, tail section: - // - // H T I - // [o o o o o o . . . . . o o A o o] - // - // H T - // [o o o o o o . . . . o o I A o o] - // M M - - self.copy(self.tail - 1, self.tail, index); - self.tail -= 1; - } - } - (false, false, true) => { - unsafe { - // discontiguous, insert closer to head, tail section: - // - // H T I - // [o o . . . . . . . o o o o o A o] - // - // H T - // [o o o . . . . . . o o o o o I A] - // M M M M - - // copy elements up to new head - self.copy(1, 0, self.head); - - // copy last element into empty spot at bottom of buffer - self.copy(0, self.cap() - 1, 1); - - // move elements from idx to end forward not including ^ element - self.copy(idx + 1, idx, self.cap() - 1 - idx); - - self.head += 1; - } - } - (false, true, false) if idx == 0 => { - unsafe { - // discontiguous, insert is closer to tail, head section, - // and is at index zero in the internal buffer: - // - // I H T - // [A o o o o o o o o o . . . o o o] - // - // H T - // [A o o o o o o o o o . . o o o I] - // M M M - - // copy elements up to new tail - self.copy(self.tail - 1, self.tail, self.cap() - self.tail); - - // copy last element into empty spot at bottom of buffer - self.copy(self.cap() - 1, 0, 1); - - self.tail -= 1; - } - } - (false, true, false) => { - unsafe { - // discontiguous, insert closer to tail, head section: - // - // I H T - // [o o o A o o o o o o . . . o o o] - // - // H T - // [o o I A o o o o o o . . o o o o] - // M M M M M M - - // copy elements up to new tail - self.copy(self.tail - 1, self.tail, self.cap() - self.tail); - - // copy last element into empty spot at bottom of buffer - self.copy(self.cap() - 1, 0, 1); - - // move elements from idx-1 to end forward not including ^ element - self.copy(0, 1, idx - 1); - - self.tail -= 1; - } - } - (false, false, false) => { - unsafe { - // discontiguous, insert closer to head, head section: - // - // I H T - // [o o o o A o o . . . . . . o o o] - // - // H T - // [o o o o I A o o . . . . . o o o] - // M M M - - self.copy(idx + 1, idx, self.head - idx); - self.head += 1; - } - } - } - - // tail might've been changed so we need to recalculate - let new_idx = self.wrap_add(self.tail, index); - unsafe { - self.buffer_write(new_idx, value); - } - } - - /// Removes and returns the element at `index` from the `VecDeque`. - /// Whichever end is closer to the removal point will be moved to make - /// room, and all the affected elements will be moved to new positions. - /// Returns `None` if `index` is out of bounds. - /// - /// Element at index 0 is the front of the queue. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf = VecDeque::new(); - /// buf.push_back(1); - /// buf.push_back(2); - /// buf.push_back(3); - /// assert_eq!(buf, [1, 2, 3]); - /// - /// assert_eq!(buf.remove(1), Some(2)); - /// assert_eq!(buf, [1, 3]); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn remove(&mut self, index: usize) -> Option { - if self.is_empty() || self.len() <= index { - return None; - } - - // There are three main cases: - // Elements are contiguous - // Elements are discontiguous and the removal is in the tail section - // Elements are discontiguous and the removal is in the head section - // - special case when elements are technically contiguous, - // but self.head = 0 - // - // For each of those there are two more cases: - // Insert is closer to tail - // Insert is closer to head - // - // Key: H - self.head - // T - self.tail - // o - Valid element - // x - Element marked for removal - // R - Indicates element that is being removed - // M - Indicates element was moved - - let idx = self.wrap_add(self.tail, index); - - let elem = unsafe { Some(self.buffer_read(idx)) }; - - let distance_to_tail = index; - let distance_to_head = self.len() - index; - - let contiguous = self.is_contiguous(); - - match (contiguous, distance_to_tail <= distance_to_head, idx >= self.tail) { - (true, true, _) => { - unsafe { - // contiguous, remove closer to tail: - // - // T R H - // [. . . o o x o o o o . . . . . .] - // - // T H - // [. . . . o o o o o o . . . . . .] - // M M - - self.copy(self.tail + 1, self.tail, index); - self.tail += 1; - } - } - (true, false, _) => { - unsafe { - // contiguous, remove closer to head: - // - // T R H - // [. . . o o o o x o o . . . . . .] - // - // T H - // [. . . o o o o o o . . . . . . .] - // M M - - self.copy(idx, idx + 1, self.head - idx - 1); - self.head -= 1; - } - } - (false, true, true) => { - unsafe { - // discontiguous, remove closer to tail, tail section: - // - // H T R - // [o o o o o o . . . . . o o x o o] - // - // H T - // [o o o o o o . . . . . . o o o o] - // M M - - self.copy(self.tail + 1, self.tail, index); - self.tail = self.wrap_add(self.tail, 1); - } - } - (false, false, false) => { - unsafe { - // discontiguous, remove closer to head, head section: - // - // R H T - // [o o o o x o o . . . . . . o o o] - // - // H T - // [o o o o o o . . . . . . . o o o] - // M M - - self.copy(idx, idx + 1, self.head - idx - 1); - self.head -= 1; - } - } - (false, false, true) => { - unsafe { - // discontiguous, remove closer to head, tail section: - // - // H T R - // [o o o . . . . . . o o o o o x o] - // - // H T - // [o o . . . . . . . o o o o o o o] - // M M M M - // - // or quasi-discontiguous, remove next to head, tail section: - // - // H T R - // [. . . . . . . . . o o o o o x o] - // - // T H - // [. . . . . . . . . o o o o o o .] - // M - - // draw in elements in the tail section - self.copy(idx, idx + 1, self.cap() - idx - 1); - - // Prevents underflow. - if self.head != 0 { - // copy first element into empty spot - self.copy(self.cap() - 1, 0, 1); - - // move elements in the head section backwards - self.copy(0, 1, self.head - 1); - } - - self.head = self.wrap_sub(self.head, 1); - } - } - (false, true, false) => { - unsafe { - // discontiguous, remove closer to tail, head section: - // - // R H T - // [o o x o o o o o o o . . . o o o] - // - // H T - // [o o o o o o o o o o . . . . o o] - // M M M M M - - // draw in elements up to idx - self.copy(1, 0, idx); - - // copy last element into empty spot - self.copy(0, self.cap() - 1, 1); - - // move elements from tail to end forward, excluding the last one - self.copy(self.tail + 1, self.tail, self.cap() - self.tail - 1); - - self.tail = self.wrap_add(self.tail, 1); - } - } - } - - elem - } - - /// Splits the `VecDeque` into two at the given index. - /// - /// Returns a newly allocated `VecDeque`. `self` contains elements `[0, at)`, - /// and the returned `VecDeque` contains elements `[at, len)`. - /// - /// Note that the capacity of `self` does not change. - /// - /// Element at index 0 is the front of the queue. - /// - /// # Panics - /// - /// Panics if `at > len`. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf: VecDeque<_> = vec![1,2,3].into_iter().collect(); - /// let buf2 = buf.split_off(1); - /// assert_eq!(buf, [1]); - /// assert_eq!(buf2, [2, 3]); - /// ``` - #[inline] - #[stable(feature = "split_off", since = "1.4.0")] - pub fn split_off(&mut self, at: usize) -> Self { - let len = self.len(); - assert!(at <= len, "`at` out of bounds"); - - let other_len = len - at; - let mut other = VecDeque::with_capacity(other_len); - - unsafe { - let (first_half, second_half) = self.as_slices(); - - let first_len = first_half.len(); - let second_len = second_half.len(); - if at < first_len { - // `at` lies in the first half. - let amount_in_first = first_len - at; - - ptr::copy_nonoverlapping(first_half.as_ptr().add(at), other.ptr(), amount_in_first); - - // just take all of the second half. - ptr::copy_nonoverlapping( - second_half.as_ptr(), - other.ptr().add(amount_in_first), - second_len, - ); - } else { - // `at` lies in the second half, need to factor in the elements we skipped - // in the first half. - let offset = at - first_len; - let amount_in_second = second_len - offset; - ptr::copy_nonoverlapping( - second_half.as_ptr().add(offset), - other.ptr(), - amount_in_second, - ); - } - } - - // Cleanup where the ends of the buffers are - self.head = self.wrap_sub(self.head, other_len); - other.head = other.wrap_index(other_len); - - other - } - - /// Moves all the elements of `other` into `self`, leaving `other` empty. - /// - /// # Panics - /// - /// Panics if the new number of elements in self overflows a `usize`. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf: VecDeque<_> = vec![1, 2].into_iter().collect(); - /// let mut buf2: VecDeque<_> = vec![3, 4].into_iter().collect(); - /// buf.append(&mut buf2); - /// assert_eq!(buf, [1, 2, 3, 4]); - /// assert_eq!(buf2, []); - /// ``` - #[inline] - #[stable(feature = "append", since = "1.4.0")] - pub fn append(&mut self, other: &mut Self) { - // naive impl - self.extend(other.drain(..)); - } - - /// Retains only the elements specified by the predicate. - /// - /// In other words, remove all elements `e` such that `f(&e)` returns false. - /// This method operates in place, visiting each element exactly once in the - /// original order, and preserves the order of the retained elements. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf = VecDeque::new(); - /// buf.extend(1..5); - /// buf.retain(|&x| x % 2 == 0); - /// assert_eq!(buf, [2, 4]); - /// ``` - /// - /// The exact order may be useful for tracking external state, like an index. - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf = VecDeque::new(); - /// buf.extend(1..6); - /// - /// let keep = [false, true, true, false, true]; - /// let mut i = 0; - /// buf.retain(|_| (keep[i], i += 1).0); - /// assert_eq!(buf, [2, 3, 5]); - /// ``` - #[stable(feature = "vec_deque_retain", since = "1.4.0")] - pub fn retain(&mut self, mut f: F) - where - F: FnMut(&T) -> bool, - { - let len = self.len(); - let mut del = 0; - for i in 0..len { - if !f(&self[i]) { - del += 1; - } else if del > 0 { - self.swap(i - del, i); - } - } - if del > 0 { - self.truncate(len - del); - } - } - - // This may panic or abort - #[inline] - fn grow_if_necessary(&mut self) { - if self.is_full() { - let old_cap = self.cap(); - self.buf.double(); - unsafe { - self.handle_capacity_increase(old_cap); - } - debug_assert!(!self.is_full()); - } - } - - /// Modifies the `VecDeque` in-place so that `len()` is equal to `new_len`, - /// either by removing excess elements from the back or by appending - /// elements generated by calling `generator` to the back. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf = VecDeque::new(); - /// buf.push_back(5); - /// buf.push_back(10); - /// buf.push_back(15); - /// assert_eq!(buf, [5, 10, 15]); - /// - /// buf.resize_with(5, Default::default); - /// assert_eq!(buf, [5, 10, 15, 0, 0]); - /// - /// buf.resize_with(2, || unreachable!()); - /// assert_eq!(buf, [5, 10]); - /// - /// let mut state = 100; - /// buf.resize_with(5, || { state += 1; state }); - /// assert_eq!(buf, [5, 10, 101, 102, 103]); - /// ``` - #[stable(feature = "vec_resize_with", since = "1.33.0")] - pub fn resize_with(&mut self, new_len: usize, generator: impl FnMut() -> T) { - let len = self.len(); - - if new_len > len { - self.extend(repeat_with(generator).take(new_len - len)) - } else { - self.truncate(new_len); - } - } - - /// Rotates the double-ended queue `mid` places to the left. - /// - /// Equivalently, - /// - Rotates item `mid` into the first position. - /// - Pops the first `mid` items and pushes them to the end. - /// - Rotates `len() - mid` places to the right. - /// - /// # Panics - /// - /// If `mid` is greater than `len()`. Note that `mid == len()` - /// does _not_ panic and is a no-op rotation. - /// - /// # Complexity - /// - /// Takes `O(min(mid, len() - mid))` time and no extra space. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf: VecDeque<_> = (0..10).collect(); - /// - /// buf.rotate_left(3); - /// assert_eq!(buf, [3, 4, 5, 6, 7, 8, 9, 0, 1, 2]); - /// - /// for i in 1..10 { - /// assert_eq!(i * 3 % 10, buf[0]); - /// buf.rotate_left(3); - /// } - /// assert_eq!(buf, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); - /// ``` - #[stable(feature = "vecdeque_rotate", since = "1.36.0")] - pub fn rotate_left(&mut self, mid: usize) { - assert!(mid <= self.len()); - let k = self.len() - mid; - if mid <= k { - unsafe { self.rotate_left_inner(mid) } - } else { - unsafe { self.rotate_right_inner(k) } - } - } - - /// Rotates the double-ended queue `k` places to the right. - /// - /// Equivalently, - /// - Rotates the first item into position `k`. - /// - Pops the last `k` items and pushes them to the front. - /// - Rotates `len() - k` places to the left. - /// - /// # Panics - /// - /// If `k` is greater than `len()`. Note that `k == len()` - /// does _not_ panic and is a no-op rotation. - /// - /// # Complexity - /// - /// Takes `O(min(k, len() - k))` time and no extra space. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf: VecDeque<_> = (0..10).collect(); - /// - /// buf.rotate_right(3); - /// assert_eq!(buf, [7, 8, 9, 0, 1, 2, 3, 4, 5, 6]); - /// - /// for i in 1..10 { - /// assert_eq!(0, buf[i * 3 % 10]); - /// buf.rotate_right(3); - /// } - /// assert_eq!(buf, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); - /// ``` - #[stable(feature = "vecdeque_rotate", since = "1.36.0")] - pub fn rotate_right(&mut self, k: usize) { - assert!(k <= self.len()); - let mid = self.len() - k; - if k <= mid { - unsafe { self.rotate_right_inner(k) } - } else { - unsafe { self.rotate_left_inner(mid) } - } - } - - // Safety: the following two methods require that the rotation amount - // be less than half the length of the deque. - // - // `wrap_copy` requires that `min(x, cap() - x) + copy_len <= cap()`, - // but than `min` is never more than half the capacity, regardless of x, - // so it's sound to call here because we're calling with something - // less than half the length, which is never above half the capacity. - - unsafe fn rotate_left_inner(&mut self, mid: usize) { - debug_assert!(mid * 2 <= self.len()); - self.wrap_copy(self.head, self.tail, mid); - self.head = self.wrap_add(self.head, mid); - self.tail = self.wrap_add(self.tail, mid); - } - - unsafe fn rotate_right_inner(&mut self, k: usize) { - debug_assert!(k * 2 <= self.len()); - self.head = self.wrap_sub(self.head, k); - self.tail = self.wrap_sub(self.tail, k); - self.wrap_copy(self.tail, self.head, k); - } -} - -impl VecDeque { - /// Modifies the `VecDeque` in-place so that `len()` is equal to new_len, - /// either by removing excess elements from the back or by appending clones of `value` - /// to the back. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// let mut buf = VecDeque::new(); - /// buf.push_back(5); - /// buf.push_back(10); - /// buf.push_back(15); - /// assert_eq!(buf, [5, 10, 15]); - /// - /// buf.resize(2, 0); - /// assert_eq!(buf, [5, 10]); - /// - /// buf.resize(5, 20); - /// assert_eq!(buf, [5, 10, 20, 20, 20]); - /// ``` - #[stable(feature = "deque_extras", since = "1.16.0")] - pub fn resize(&mut self, new_len: usize, value: T) { - self.resize_with(new_len, || value.clone()); - } -} - -/// Returns the index in the underlying buffer for a given logical element index. -#[inline] -fn wrap_index(index: usize, size: usize) -> usize { - // size is always a power of 2 - debug_assert!(size.is_power_of_two()); - index & (size - 1) -} - -/// Returns the two slices that cover the `VecDeque`'s valid range -trait RingSlices: Sized { - fn slice(self, from: usize, to: usize) -> Self; - fn split_at(self, i: usize) -> (Self, Self); - - fn ring_slices(buf: Self, head: usize, tail: usize) -> (Self, Self) { - let contiguous = tail <= head; - if contiguous { - let (empty, buf) = buf.split_at(0); - (buf.slice(tail, head), empty) - } else { - let (mid, right) = buf.split_at(tail); - let (left, _) = mid.split_at(head); - (right, left) - } - } -} - -impl RingSlices for &[T] { - fn slice(self, from: usize, to: usize) -> Self { - &self[from..to] - } - fn split_at(self, i: usize) -> (Self, Self) { - (*self).split_at(i) - } -} - -impl RingSlices for &mut [T] { - fn slice(self, from: usize, to: usize) -> Self { - &mut self[from..to] - } - fn split_at(self, i: usize) -> (Self, Self) { - (*self).split_at_mut(i) - } -} - -/// Calculate the number of elements left to be read in the buffer -#[inline] -fn count(tail: usize, head: usize, size: usize) -> usize { - // size is always a power of 2 - (head.wrapping_sub(tail)) & (size - 1) -} - -/// An iterator over the elements of a `VecDeque`. -/// -/// This `struct` is created by the [`iter`] method on [`VecDeque`]. See its -/// documentation for more. -/// -/// [`iter`]: struct.VecDeque.html#method.iter -/// [`VecDeque`]: struct.VecDeque.html -#[stable(feature = "rust1", since = "1.0.0")] -pub struct Iter<'a, T: 'a> { - ring: &'a [T], - tail: usize, - head: usize, -} - -#[stable(feature = "collection_debug", since = "1.17.0")] -impl fmt::Debug for Iter<'_, T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let (front, back) = RingSlices::ring_slices(self.ring, self.head, self.tail); - f.debug_tuple("Iter").field(&front).field(&back).finish() - } -} - -// FIXME(#26925) Remove in favor of `#[derive(Clone)]` -#[stable(feature = "rust1", since = "1.0.0")] -impl Clone for Iter<'_, T> { - fn clone(&self) -> Self { - Iter { ring: self.ring, tail: self.tail, head: self.head } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T> Iterator for Iter<'a, T> { - type Item = &'a T; - - #[inline] - fn next(&mut self) -> Option<&'a T> { - if self.tail == self.head { - return None; - } - let tail = self.tail; - self.tail = wrap_index(self.tail.wrapping_add(1), self.ring.len()); - unsafe { Some(self.ring.get_unchecked(tail)) } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let len = count(self.tail, self.head, self.ring.len()); - (len, Some(len)) - } - - fn fold(self, mut accum: Acc, mut f: F) -> Acc - where - F: FnMut(Acc, Self::Item) -> Acc, - { - let (front, back) = RingSlices::ring_slices(self.ring, self.head, self.tail); - accum = front.iter().fold(accum, &mut f); - back.iter().fold(accum, &mut f) - } - - fn try_fold(&mut self, init: B, mut f: F) -> R - where - Self: Sized, - F: FnMut(B, Self::Item) -> R, - R: Try, - { - let (mut iter, final_res); - if self.tail <= self.head { - // single slice self.ring[self.tail..self.head] - iter = self.ring[self.tail..self.head].iter(); - final_res = iter.try_fold(init, &mut f); - } else { - // two slices: self.ring[self.tail..], self.ring[..self.head] - let (front, back) = self.ring.split_at(self.tail); - let mut back_iter = back.iter(); - let res = back_iter.try_fold(init, &mut f); - let len = self.ring.len(); - self.tail = (self.ring.len() - back_iter.len()) & (len - 1); - iter = front[..self.head].iter(); - final_res = iter.try_fold(res?, &mut f); - } - self.tail = self.head - iter.len(); - final_res - } - - fn nth(&mut self, n: usize) -> Option { - if n >= count(self.tail, self.head, self.ring.len()) { - self.tail = self.head; - None - } else { - self.tail = wrap_index(self.tail.wrapping_add(n), self.ring.len()); - self.next() - } - } - - #[inline] - fn last(mut self) -> Option<&'a T> { - self.next_back() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T> DoubleEndedIterator for Iter<'a, T> { - #[inline] - fn next_back(&mut self) -> Option<&'a T> { - if self.tail == self.head { - return None; - } - self.head = wrap_index(self.head.wrapping_sub(1), self.ring.len()); - unsafe { Some(self.ring.get_unchecked(self.head)) } - } - - fn rfold(self, mut accum: Acc, mut f: F) -> Acc - where - F: FnMut(Acc, Self::Item) -> Acc, - { - let (front, back) = RingSlices::ring_slices(self.ring, self.head, self.tail); - accum = back.iter().rfold(accum, &mut f); - front.iter().rfold(accum, &mut f) - } - - fn try_rfold(&mut self, init: B, mut f: F) -> R - where - Self: Sized, - F: FnMut(B, Self::Item) -> R, - R: Try, - { - let (mut iter, final_res); - if self.tail <= self.head { - // single slice self.ring[self.tail..self.head] - iter = self.ring[self.tail..self.head].iter(); - final_res = iter.try_rfold(init, &mut f); - } else { - // two slices: self.ring[self.tail..], self.ring[..self.head] - let (front, back) = self.ring.split_at(self.tail); - let mut front_iter = front[..self.head].iter(); - let res = front_iter.try_rfold(init, &mut f); - self.head = front_iter.len(); - iter = back.iter(); - final_res = iter.try_rfold(res?, &mut f); - } - self.head = self.tail + iter.len(); - final_res - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ExactSizeIterator for Iter<'_, T> { - fn is_empty(&self) -> bool { - self.head == self.tail - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Iter<'_, T> {} - -/// A mutable iterator over the elements of a `VecDeque`. -/// -/// This `struct` is created by the [`iter_mut`] method on [`VecDeque`]. See its -/// documentation for more. -/// -/// [`iter_mut`]: struct.VecDeque.html#method.iter_mut -/// [`VecDeque`]: struct.VecDeque.html -#[stable(feature = "rust1", since = "1.0.0")] -pub struct IterMut<'a, T: 'a> { - ring: &'a mut [T], - tail: usize, - head: usize, -} - -#[stable(feature = "collection_debug", since = "1.17.0")] -impl fmt::Debug for IterMut<'_, T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let (front, back) = RingSlices::ring_slices(&*self.ring, self.head, self.tail); - f.debug_tuple("IterMut").field(&front).field(&back).finish() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T> Iterator for IterMut<'a, T> { - type Item = &'a mut T; - - #[inline] - fn next(&mut self) -> Option<&'a mut T> { - if self.tail == self.head { - return None; - } - let tail = self.tail; - self.tail = wrap_index(self.tail.wrapping_add(1), self.ring.len()); - - unsafe { - let elem = self.ring.get_unchecked_mut(tail); - Some(&mut *(elem as *mut _)) - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let len = count(self.tail, self.head, self.ring.len()); - (len, Some(len)) - } - - fn fold(self, mut accum: Acc, mut f: F) -> Acc - where - F: FnMut(Acc, Self::Item) -> Acc, - { - let (front, back) = RingSlices::ring_slices(self.ring, self.head, self.tail); - accum = front.iter_mut().fold(accum, &mut f); - back.iter_mut().fold(accum, &mut f) - } - - fn nth(&mut self, n: usize) -> Option { - if n >= count(self.tail, self.head, self.ring.len()) { - self.tail = self.head; - None - } else { - self.tail = wrap_index(self.tail.wrapping_add(n), self.ring.len()); - self.next() - } - } - - #[inline] - fn last(mut self) -> Option<&'a mut T> { - self.next_back() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T> DoubleEndedIterator for IterMut<'a, T> { - #[inline] - fn next_back(&mut self) -> Option<&'a mut T> { - if self.tail == self.head { - return None; - } - self.head = wrap_index(self.head.wrapping_sub(1), self.ring.len()); - - unsafe { - let elem = self.ring.get_unchecked_mut(self.head); - Some(&mut *(elem as *mut _)) - } - } - - fn rfold(self, mut accum: Acc, mut f: F) -> Acc - where - F: FnMut(Acc, Self::Item) -> Acc, - { - let (front, back) = RingSlices::ring_slices(self.ring, self.head, self.tail); - accum = back.iter_mut().rfold(accum, &mut f); - front.iter_mut().rfold(accum, &mut f) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ExactSizeIterator for IterMut<'_, T> { - fn is_empty(&self) -> bool { - self.head == self.tail - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for IterMut<'_, T> {} - -/// An owning iterator over the elements of a `VecDeque`. -/// -/// This `struct` is created by the [`into_iter`] method on [`VecDeque`] -/// (provided by the `IntoIterator` trait). See its documentation for more. -/// -/// [`into_iter`]: struct.VecDeque.html#method.into_iter -/// [`VecDeque`]: struct.VecDeque.html -#[derive(Clone)] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct IntoIter { - inner: VecDeque, -} - -#[stable(feature = "collection_debug", since = "1.17.0")] -impl fmt::Debug for IntoIter { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("IntoIter").field(&self.inner).finish() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for IntoIter { - type Item = T; - - #[inline] - fn next(&mut self) -> Option { - self.inner.pop_front() - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let len = self.inner.len(); - (len, Some(len)) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl DoubleEndedIterator for IntoIter { - #[inline] - fn next_back(&mut self) -> Option { - self.inner.pop_back() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ExactSizeIterator for IntoIter { - fn is_empty(&self) -> bool { - self.inner.is_empty() - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for IntoIter {} - -#[stable(feature = "rust1", since = "1.0.0")] -impl PartialEq for VecDeque { - fn eq(&self, other: &VecDeque) -> bool { - if self.len() != other.len() { - return false; - } - let (sa, sb) = self.as_slices(); - let (oa, ob) = other.as_slices(); - if sa.len() == oa.len() { - sa == oa && sb == ob - } else if sa.len() < oa.len() { - // Always divisible in three sections, for example: - // self: [a b c|d e f] - // other: [0 1 2 3|4 5] - // front = 3, mid = 1, - // [a b c] == [0 1 2] && [d] == [3] && [e f] == [4 5] - let front = sa.len(); - let mid = oa.len() - front; - - let (oa_front, oa_mid) = oa.split_at(front); - let (sb_mid, sb_back) = sb.split_at(mid); - debug_assert_eq!(sa.len(), oa_front.len()); - debug_assert_eq!(sb_mid.len(), oa_mid.len()); - debug_assert_eq!(sb_back.len(), ob.len()); - sa == oa_front && sb_mid == oa_mid && sb_back == ob - } else { - let front = oa.len(); - let mid = sa.len() - front; - - let (sa_front, sa_mid) = sa.split_at(front); - let (ob_mid, ob_back) = ob.split_at(mid); - debug_assert_eq!(sa_front.len(), oa.len()); - debug_assert_eq!(sa_mid.len(), ob_mid.len()); - debug_assert_eq!(sb.len(), ob_back.len()); - sa_front == oa && sa_mid == ob_mid && sb == ob_back - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Eq for VecDeque {} - -macro_rules! __impl_slice_eq1 { - ([$($vars:tt)*] $lhs:ty, $rhs:ty, $($constraints:tt)*) => { - #[stable(feature = "vec_deque_partial_eq_slice", since = "1.17.0")] - impl PartialEq<$rhs> for $lhs - where - A: PartialEq, - $($constraints)* - { - fn eq(&self, other: &$rhs) -> bool { - if self.len() != other.len() { - return false; - } - let (sa, sb) = self.as_slices(); - let (oa, ob) = other[..].split_at(sa.len()); - sa == oa && sb == ob - } - } - } -} - -__impl_slice_eq1! { [] VecDeque, Vec, } -__impl_slice_eq1! { [] VecDeque, &[B], } -__impl_slice_eq1! { [] VecDeque, &mut [B], } -__impl_slice_eq1! { [const N: usize] VecDeque, [B; N], [B; N]: LengthAtMost32 } -__impl_slice_eq1! { [const N: usize] VecDeque, &[B; N], [B; N]: LengthAtMost32 } -__impl_slice_eq1! { [const N: usize] VecDeque, &mut [B; N], [B; N]: LengthAtMost32 } - -#[stable(feature = "rust1", since = "1.0.0")] -impl PartialOrd for VecDeque { - fn partial_cmp(&self, other: &VecDeque) -> Option { - self.iter().partial_cmp(other.iter()) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Ord for VecDeque { - #[inline] - fn cmp(&self, other: &VecDeque) -> Ordering { - self.iter().cmp(other.iter()) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Hash for VecDeque { - fn hash(&self, state: &mut H) { - self.len().hash(state); - let (a, b) = self.as_slices(); - Hash::hash_slice(a, state); - Hash::hash_slice(b, state); - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Index for VecDeque { - type Output = A; - - #[inline] - fn index(&self, index: usize) -> &A { - self.get(index).expect("Out of bounds access") - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl IndexMut for VecDeque { - #[inline] - fn index_mut(&mut self, index: usize) -> &mut A { - self.get_mut(index).expect("Out of bounds access") - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl FromIterator for VecDeque { - fn from_iter>(iter: T) -> VecDeque { - let iterator = iter.into_iter(); - let (lower, _) = iterator.size_hint(); - let mut deq = VecDeque::with_capacity(lower); - deq.extend(iterator); - deq - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl IntoIterator for VecDeque { - type Item = T; - type IntoIter = IntoIter; - - /// Consumes the `VecDeque` into a front-to-back iterator yielding elements by - /// value. - fn into_iter(self) -> IntoIter { - IntoIter { inner: self } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T> IntoIterator for &'a VecDeque { - type Item = &'a T; - type IntoIter = Iter<'a, T>; - - fn into_iter(self) -> Iter<'a, T> { - self.iter() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T> IntoIterator for &'a mut VecDeque { - type Item = &'a mut T; - type IntoIter = IterMut<'a, T>; - - fn into_iter(self) -> IterMut<'a, T> { - self.iter_mut() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Extend for VecDeque { - fn extend>(&mut self, iter: T) { - // This function should be the moral equivalent of: - // - // for item in iter.into_iter() { - // self.push_back(item); - // } - let mut iter = iter.into_iter(); - while let Some(element) = iter.next() { - if self.len() == self.capacity() { - let (lower, _) = iter.size_hint(); - self.reserve(lower.saturating_add(1)); - } - - let head = self.head; - self.head = self.wrap_add(self.head, 1); - unsafe { - self.buffer_write(head, element); - } - } - } -} - -#[stable(feature = "extend_ref", since = "1.2.0")] -impl<'a, T: 'a + Copy> Extend<&'a T> for VecDeque { - fn extend>(&mut self, iter: I) { - self.extend(iter.into_iter().cloned()); - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl fmt::Debug for VecDeque { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_list().entries(self).finish() - } -} - -#[stable(feature = "vecdeque_vec_conversions", since = "1.10.0")] -impl From> for VecDeque { - /// Turn a [`Vec`] into a [`VecDeque`]. - /// - /// [`Vec`]: crate::vec::Vec - /// [`VecDeque`]: crate::collections::VecDeque - /// - /// This avoids reallocating where possible, but the conditions for that are - /// strict, and subject to change, and so shouldn't be relied upon unless the - /// `Vec` came from `From>` and hasn't been reallocated. - fn from(mut other: Vec) -> Self { - unsafe { - let other_buf = other.as_mut_ptr(); - let mut buf = RawVec::from_raw_parts(other_buf, other.capacity()); - let len = other.len(); - mem::forget(other); - - // We need to extend the buf if it's not a power of two, too small - // or doesn't have at least one free space - if !buf.capacity().is_power_of_two() - || (buf.capacity() < (MINIMUM_CAPACITY + 1)) - || (buf.capacity() == len) - { - let cap = cmp::max(buf.capacity() + 1, MINIMUM_CAPACITY + 1).next_power_of_two(); - buf.reserve_exact(len, cap - len); - } - - VecDeque { tail: 0, head: len, buf } - } - } -} - -#[stable(feature = "vecdeque_vec_conversions", since = "1.10.0")] -impl From> for Vec { - /// Turn a [`VecDeque`] into a [`Vec`]. - /// - /// [`Vec`]: crate::vec::Vec - /// [`VecDeque`]: crate::collections::VecDeque - /// - /// This never needs to re-allocate, but does need to do O(n) data movement if - /// the circular buffer doesn't happen to be at the beginning of the allocation. - /// - /// # Examples - /// - /// ``` - /// use std::collections::VecDeque; - /// - /// // This one is O(1). - /// let deque: VecDeque<_> = (1..5).collect(); - /// let ptr = deque.as_slices().0.as_ptr(); - /// let vec = Vec::from(deque); - /// assert_eq!(vec, [1, 2, 3, 4]); - /// assert_eq!(vec.as_ptr(), ptr); - /// - /// // This one needs data rearranging. - /// let mut deque: VecDeque<_> = (1..5).collect(); - /// deque.push_front(9); - /// deque.push_front(8); - /// let ptr = deque.as_slices().1.as_ptr(); - /// let vec = Vec::from(deque); - /// assert_eq!(vec, [8, 9, 1, 2, 3, 4]); - /// assert_eq!(vec.as_ptr(), ptr); - /// ``` - fn from(other: VecDeque) -> Self { - unsafe { - let buf = other.buf.ptr(); - let len = other.len(); - let tail = other.tail; - let head = other.head; - let cap = other.cap(); - - // Need to move the ring to the front of the buffer, as vec will expect this. - if other.is_contiguous() { - ptr::copy(buf.add(tail), buf, len); - } else { - if (tail - head) >= cmp::min(cap - tail, head) { - // There is enough free space in the centre for the shortest block so we can - // do this in at most three copy moves. - if (cap - tail) > head { - // right hand block is the long one; move that enough for the left - ptr::copy(buf.add(tail), buf.add(tail - head), cap - tail); - // copy left in the end - ptr::copy(buf, buf.add(cap - head), head); - // shift the new thing to the start - ptr::copy(buf.add(tail - head), buf, len); - } else { - // left hand block is the long one, we can do it in two! - ptr::copy(buf, buf.add(cap - tail), head); - ptr::copy(buf.add(tail), buf, cap - tail); - } - } else { - // Need to use N swaps to move the ring - // We can use the space at the end of the ring as a temp store - - let mut left_edge: usize = 0; - let mut right_edge: usize = tail; - - // The general problem looks like this - // GHIJKLM...ABCDEF - before any swaps - // ABCDEFM...GHIJKL - after 1 pass of swaps - // ABCDEFGHIJM...KL - swap until the left edge reaches the temp store - // - then restart the algorithm with a new (smaller) store - // Sometimes the temp store is reached when the right edge is at the end - // of the buffer - this means we've hit the right order with fewer swaps! - // E.g - // EF..ABCD - // ABCDEF.. - after four only swaps we've finished - - while left_edge < len && right_edge != cap { - let mut right_offset = 0; - for i in left_edge..right_edge { - right_offset = (i - left_edge) % (cap - right_edge); - let src: isize = (right_edge + right_offset) as isize; - ptr::swap(buf.add(i), buf.offset(src)); - } - let n_ops = right_edge - left_edge; - left_edge += n_ops; - right_edge += right_offset + 1; - } - } - } - let out = Vec::from_raw_parts(buf, len, cap); - mem::forget(other); - out - } - } -} diff --git a/crux-mir/lib/alloc/src/collections/vec_deque/drain.rs b/crux-mir/lib/alloc/src/collections/vec_deque/drain.rs index 1ae94de75..89feb361d 100644 --- a/crux-mir/lib/alloc/src/collections/vec_deque/drain.rs +++ b/crux-mir/lib/alloc/src/collections/vec_deque/drain.rs @@ -1,126 +1,231 @@ use core::iter::FusedIterator; -use core::ptr::{self, NonNull}; -use core::{fmt, mem}; +use core::marker::PhantomData; +use core::mem::{self, SizedTypeProperties}; +use core::ptr::NonNull; +use core::{fmt, ptr}; -use super::{count, Iter, VecDeque}; +use crate::alloc::{Allocator, Global}; + +use super::VecDeque; /// A draining iterator over the elements of a `VecDeque`. /// /// This `struct` is created by the [`drain`] method on [`VecDeque`]. See its /// documentation for more. /// -/// [`drain`]: struct.VecDeque.html#method.drain -/// [`VecDeque`]: struct.VecDeque.html +/// [`drain`]: VecDeque::drain #[stable(feature = "drain", since = "1.6.0")] -pub struct Drain<'a, T: 'a> { - pub(crate) after_tail: usize, - pub(crate) after_head: usize, - pub(crate) iter: Iter<'a, T>, - pub(crate) deque: NonNull>, +pub struct Drain< + 'a, + T: 'a, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator = Global, +> { + // We can't just use a &mut VecDeque, as that would make Drain invariant over T + // and we want it to be covariant instead + deque: NonNull>, + // drain_start is stored in deque.len + drain_len: usize, + // index into the logical array, not the physical one (always lies in [0..deque.len)) + idx: usize, + // number of elements after the drain range + tail_len: usize, + remaining: usize, + // Needed to make Drain covariant over T + _marker: PhantomData<&'a T>, +} + +impl<'a, T, A: Allocator> Drain<'a, T, A> { + pub(super) unsafe fn new( + deque: &'a mut VecDeque, + drain_start: usize, + drain_len: usize, + ) -> Self { + let orig_len = mem::replace(&mut deque.len, drain_start); + let tail_len = orig_len - drain_start - drain_len; + Drain { + deque: NonNull::from(deque), + drain_len, + idx: drain_start, + tail_len, + remaining: drain_len, + _marker: PhantomData, + } + } + + // Only returns pointers to the slices, as that's + // all we need to drop them. May only be called if `self.remaining != 0`. + unsafe fn as_slices(&self) -> (*mut [T], *mut [T]) { + unsafe { + let deque = self.deque.as_ref(); + // FIXME: This is doing almost exactly the same thing as the else branch in `VecDeque::slice_ranges`. + // Unfortunately, we can't just call `slice_ranges` here, as the deque's `len` is currently + // just `drain_start`, so the range check would (almost) always panic. Between temporarily + // adjusting the deques `len` to call `slice_ranges`, and just copy pasting the `slice_ranges` + // implementation, this seemed like the less hacky solution, though it might be good to + // find a better one in the future. + + // because `self.remaining != 0`, we know that `self.idx < deque.original_len`, so it's a valid + // logical index. + let wrapped_start = deque.to_physical_idx(self.idx); + + let head_len = deque.capacity() - wrapped_start; + + let (a_range, b_range) = if head_len >= self.remaining { + (wrapped_start..wrapped_start + self.remaining, 0..0) + } else { + let tail_len = self.remaining - head_len; + (wrapped_start..deque.capacity(), 0..tail_len) + }; + + // SAFETY: the range `self.idx..self.idx+self.remaining` lies strictly inside + // the range `0..deque.original_len`. because of this, and because of the fact + // that we acquire `a_range` and `b_range` exactly like `slice_ranges` would, + // it's guaranteed that `a_range` and `b_range` represent valid ranges into + // the deques buffer. + (deque.buffer_range(a_range), deque.buffer_range(b_range)) + } + } } #[stable(feature = "collection_debug", since = "1.17.0")] -impl fmt::Debug for Drain<'_, T> { +impl fmt::Debug for Drain<'_, T, A> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("Drain") - .field(&self.after_tail) - .field(&self.after_head) - .field(&self.iter) + .field(&self.drain_len) + .field(&self.idx) + .field(&self.tail_len) + .field(&self.remaining) .finish() } } #[stable(feature = "drain", since = "1.6.0")] -unsafe impl Sync for Drain<'_, T> {} +unsafe impl Sync for Drain<'_, T, A> {} #[stable(feature = "drain", since = "1.6.0")] -unsafe impl Send for Drain<'_, T> {} +unsafe impl Send for Drain<'_, T, A> {} #[stable(feature = "drain", since = "1.6.0")] -impl Drop for Drain<'_, T> { +impl Drop for Drain<'_, T, A> { fn drop(&mut self) { - struct DropGuard<'r, 'a, T>(&'r mut Drain<'a, T>); + struct DropGuard<'r, 'a, T, A: Allocator>(&'r mut Drain<'a, T, A>); - impl<'r, 'a, T> Drop for DropGuard<'r, 'a, T> { + impl<'r, 'a, T, A: Allocator> Drop for DropGuard<'r, 'a, T, A> { fn drop(&mut self) { - self.0.for_each(drop); + if self.0.remaining != 0 { + unsafe { + // SAFETY: We just checked that `self.remaining != 0`. + let (front, back) = self.0.as_slices(); + ptr::drop_in_place(front); + ptr::drop_in_place(back); + } + } let source_deque = unsafe { self.0.deque.as_mut() }; - // T = source_deque_tail; H = source_deque_head; t = drain_tail; h = drain_head - // - // T t h H - // [. . . o o x x o o . . .] - // - let orig_tail = source_deque.tail; - let drain_tail = source_deque.head; - let drain_head = self.0.after_tail; - let orig_head = self.0.after_head; + let drain_start = source_deque.len(); + let drain_len = self.0.drain_len; + let drain_end = drain_start + drain_len; - let tail_len = count(orig_tail, drain_tail, source_deque.cap()); - let head_len = count(drain_head, orig_head, source_deque.cap()); + let orig_len = self.0.tail_len + drain_end; + + if T::IS_ZST { + // no need to copy around any memory if T is a ZST + source_deque.len = orig_len - drain_len; + return; + } - // Restore the original head value - source_deque.head = orig_head; + let head_len = drain_start; + let tail_len = self.0.tail_len; - match (tail_len, head_len) { + match (head_len, tail_len) { (0, 0) => { source_deque.head = 0; - source_deque.tail = 0; + source_deque.len = 0; } (0, _) => { - source_deque.tail = drain_head; + source_deque.head = source_deque.to_physical_idx(drain_len); + source_deque.len = orig_len - drain_len; } (_, 0) => { - source_deque.head = drain_tail; + source_deque.len = orig_len - drain_len; } _ => unsafe { - if tail_len <= head_len { - source_deque.tail = source_deque.wrap_sub(drain_head, tail_len); - source_deque.wrap_copy(source_deque.tail, orig_tail, tail_len); + if head_len <= tail_len { + source_deque.wrap_copy( + source_deque.head, + source_deque.to_physical_idx(drain_len), + head_len, + ); + source_deque.head = source_deque.to_physical_idx(drain_len); + source_deque.len = orig_len - drain_len; } else { - source_deque.head = source_deque.wrap_add(drain_tail, head_len); - source_deque.wrap_copy(drain_tail, drain_head, head_len); + source_deque.wrap_copy( + source_deque.to_physical_idx(head_len + drain_len), + source_deque.to_physical_idx(head_len), + tail_len, + ); + source_deque.len = orig_len - drain_len; } }, } } } - while let Some(item) = self.next() { - let guard = DropGuard(self); - drop(item); - mem::forget(guard); + let guard = DropGuard(self); + if guard.0.remaining != 0 { + unsafe { + // SAFETY: We just checked that `self.remaining != 0`. + let (front, back) = guard.0.as_slices(); + // since idx is a logical index, we don't need to worry about wrapping. + guard.0.idx += front.len(); + guard.0.remaining -= front.len(); + ptr::drop_in_place(front); + guard.0.remaining = 0; + ptr::drop_in_place(back); + } } - DropGuard(self); + // Dropping `guard` handles moving the remaining elements into place. } } #[stable(feature = "drain", since = "1.6.0")] -impl Iterator for Drain<'_, T> { +impl Iterator for Drain<'_, T, A> { type Item = T; #[inline] fn next(&mut self) -> Option { - self.iter.next().map(|elt| unsafe { ptr::read(elt) }) + if self.remaining == 0 { + return None; + } + let wrapped_idx = unsafe { self.deque.as_ref().to_physical_idx(self.idx) }; + self.idx += 1; + self.remaining -= 1; + Some(unsafe { self.deque.as_mut().buffer_read(wrapped_idx) }) } #[inline] fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() + let len = self.remaining; + (len, Some(len)) } } #[stable(feature = "drain", since = "1.6.0")] -impl DoubleEndedIterator for Drain<'_, T> { +impl DoubleEndedIterator for Drain<'_, T, A> { #[inline] fn next_back(&mut self) -> Option { - self.iter.next_back().map(|elt| unsafe { ptr::read(elt) }) + if self.remaining == 0 { + return None; + } + self.remaining -= 1; + let wrapped_idx = unsafe { self.deque.as_ref().to_physical_idx(self.idx + self.remaining) }; + Some(unsafe { self.deque.as_mut().buffer_read(wrapped_idx) }) } } #[stable(feature = "drain", since = "1.6.0")] -impl ExactSizeIterator for Drain<'_, T> {} +impl ExactSizeIterator for Drain<'_, T, A> {} #[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Drain<'_, T> {} +impl FusedIterator for Drain<'_, T, A> {} diff --git a/crux-mir/lib/alloc/src/collections/vec_deque/into_iter.rs b/crux-mir/lib/alloc/src/collections/vec_deque/into_iter.rs new file mode 100644 index 000000000..e54880e86 --- /dev/null +++ b/crux-mir/lib/alloc/src/collections/vec_deque/into_iter.rs @@ -0,0 +1,76 @@ +use core::fmt; +use core::iter::{FusedIterator, TrustedLen}; + +use crate::alloc::{Allocator, Global}; + +use super::VecDeque; + +/// An owning iterator over the elements of a `VecDeque`. +/// +/// This `struct` is created by the [`into_iter`] method on [`VecDeque`] +/// (provided by the [`IntoIterator`] trait). See its documentation for more. +/// +/// [`into_iter`]: VecDeque::into_iter +/// [`IntoIterator`]: core::iter::IntoIterator +#[derive(Clone)] +#[stable(feature = "rust1", since = "1.0.0")] +pub struct IntoIter< + T, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator = Global, +> { + inner: VecDeque, +} + +impl IntoIter { + pub(super) fn new(inner: VecDeque) -> Self { + IntoIter { inner } + } + + pub(super) fn into_vecdeque(self) -> VecDeque { + self.inner + } +} + +#[stable(feature = "collection_debug", since = "1.17.0")] +impl fmt::Debug for IntoIter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("IntoIter").field(&self.inner).finish() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for IntoIter { + type Item = T; + + #[inline] + fn next(&mut self) -> Option { + self.inner.pop_front() + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let len = self.inner.len(); + (len, Some(len)) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl DoubleEndedIterator for IntoIter { + #[inline] + fn next_back(&mut self) -> Option { + self.inner.pop_back() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ExactSizeIterator for IntoIter { + fn is_empty(&self) -> bool { + self.inner.is_empty() + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for IntoIter {} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for IntoIter {} diff --git a/crux-mir/lib/alloc/src/collections/vec_deque/iter.rs b/crux-mir/lib/alloc/src/collections/vec_deque/iter.rs new file mode 100644 index 000000000..d9f393714 --- /dev/null +++ b/crux-mir/lib/alloc/src/collections/vec_deque/iter.rs @@ -0,0 +1,181 @@ +use core::iter::{FusedIterator, TrustedLen, TrustedRandomAccess, TrustedRandomAccessNoCoerce}; +use core::ops::Try; +use core::{fmt, mem, slice}; + +/// An iterator over the elements of a `VecDeque`. +/// +/// This `struct` is created by the [`iter`] method on [`super::VecDeque`]. See its +/// documentation for more. +/// +/// [`iter`]: super::VecDeque::iter +#[stable(feature = "rust1", since = "1.0.0")] +pub struct Iter<'a, T: 'a> { + i1: slice::Iter<'a, T>, + i2: slice::Iter<'a, T>, +} + +impl<'a, T> Iter<'a, T> { + pub(super) fn new(i1: slice::Iter<'a, T>, i2: slice::Iter<'a, T>) -> Self { + Self { i1, i2 } + } +} + +#[stable(feature = "collection_debug", since = "1.17.0")] +impl fmt::Debug for Iter<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("Iter").field(&self.i1.as_slice()).field(&self.i2.as_slice()).finish() + } +} + +// FIXME(#26925) Remove in favor of `#[derive(Clone)]` +#[stable(feature = "rust1", since = "1.0.0")] +impl Clone for Iter<'_, T> { + fn clone(&self) -> Self { + Iter { i1: self.i1.clone(), i2: self.i2.clone() } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T> Iterator for Iter<'a, T> { + type Item = &'a T; + + #[inline] + fn next(&mut self) -> Option<&'a T> { + match self.i1.next() { + Some(val) => Some(val), + None => { + // most of the time, the iterator will either always + // call next(), or always call next_back(). By swapping + // the iterators once the first one is empty, we ensure + // that the first branch is taken as often as possible, + // without sacrificing correctness, as i1 is empty anyways + mem::swap(&mut self.i1, &mut self.i2); + self.i1.next() + } + } + } + + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + let m = match self.i1.advance_by(n) { + Ok(_) => return Ok(()), + Err(m) => m, + }; + mem::swap(&mut self.i1, &mut self.i2); + self.i1.advance_by(n - m).map_err(|o| o + m) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let len = self.len(); + (len, Some(len)) + } + + fn fold(self, accum: Acc, mut f: F) -> Acc + where + F: FnMut(Acc, Self::Item) -> Acc, + { + let accum = self.i1.fold(accum, &mut f); + self.i2.fold(accum, &mut f) + } + + fn try_fold(&mut self, init: B, mut f: F) -> R + where + F: FnMut(B, Self::Item) -> R, + R: Try, + { + let acc = self.i1.try_fold(init, &mut f)?; + self.i2.try_fold(acc, &mut f) + } + + #[inline] + fn last(mut self) -> Option<&'a T> { + self.next_back() + } + + #[inline] + unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> Self::Item { + // Safety: The TrustedRandomAccess contract requires that callers only pass an index + // that is in bounds. + unsafe { + let i1_len = self.i1.len(); + if idx < i1_len { + self.i1.__iterator_get_unchecked(idx) + } else { + self.i2.__iterator_get_unchecked(idx - i1_len) + } + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T> DoubleEndedIterator for Iter<'a, T> { + #[inline] + fn next_back(&mut self) -> Option<&'a T> { + match self.i2.next_back() { + Some(val) => Some(val), + None => { + // most of the time, the iterator will either always + // call next(), or always call next_back(). By swapping + // the iterators once the second one is empty, we ensure + // that the first branch is taken as often as possible, + // without sacrificing correctness, as i2 is empty anyways + mem::swap(&mut self.i1, &mut self.i2); + self.i2.next_back() + } + } + } + + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + let m = match self.i2.advance_back_by(n) { + Ok(_) => return Ok(()), + Err(m) => m, + }; + + mem::swap(&mut self.i1, &mut self.i2); + self.i2.advance_back_by(n - m).map_err(|o| m + o) + } + + fn rfold(self, accum: Acc, mut f: F) -> Acc + where + F: FnMut(Acc, Self::Item) -> Acc, + { + let accum = self.i2.rfold(accum, &mut f); + self.i1.rfold(accum, &mut f) + } + + fn try_rfold(&mut self, init: B, mut f: F) -> R + where + F: FnMut(B, Self::Item) -> R, + R: Try, + { + let acc = self.i2.try_rfold(init, &mut f)?; + self.i1.try_rfold(acc, &mut f) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ExactSizeIterator for Iter<'_, T> { + fn len(&self) -> usize { + self.i1.len() + self.i2.len() + } + + fn is_empty(&self) -> bool { + self.i1.is_empty() && self.i2.is_empty() + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Iter<'_, T> {} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for Iter<'_, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl TrustedRandomAccess for Iter<'_, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl TrustedRandomAccessNoCoerce for Iter<'_, T> { + const MAY_HAVE_SIDE_EFFECT: bool = false; +} diff --git a/crux-mir/lib/alloc/src/collections/vec_deque/iter_mut.rs b/crux-mir/lib/alloc/src/collections/vec_deque/iter_mut.rs new file mode 100644 index 000000000..2c59d95cd --- /dev/null +++ b/crux-mir/lib/alloc/src/collections/vec_deque/iter_mut.rs @@ -0,0 +1,173 @@ +use core::iter::{FusedIterator, TrustedLen, TrustedRandomAccess, TrustedRandomAccessNoCoerce}; +use core::ops::Try; +use core::{fmt, mem, slice}; + +/// A mutable iterator over the elements of a `VecDeque`. +/// +/// This `struct` is created by the [`iter_mut`] method on [`super::VecDeque`]. See its +/// documentation for more. +/// +/// [`iter_mut`]: super::VecDeque::iter_mut +#[stable(feature = "rust1", since = "1.0.0")] +pub struct IterMut<'a, T: 'a> { + i1: slice::IterMut<'a, T>, + i2: slice::IterMut<'a, T>, +} + +impl<'a, T> IterMut<'a, T> { + pub(super) fn new(i1: slice::IterMut<'a, T>, i2: slice::IterMut<'a, T>) -> Self { + Self { i1, i2 } + } +} + +#[stable(feature = "collection_debug", since = "1.17.0")] +impl fmt::Debug for IterMut<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("IterMut").field(&self.i1.as_slice()).field(&self.i2.as_slice()).finish() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T> Iterator for IterMut<'a, T> { + type Item = &'a mut T; + + #[inline] + fn next(&mut self) -> Option<&'a mut T> { + match self.i1.next() { + Some(val) => Some(val), + None => { + // most of the time, the iterator will either always + // call next(), or always call next_back(). By swapping + // the iterators once the first one is empty, we ensure + // that the first branch is taken as often as possible, + // without sacrificing correctness, as i1 is empty anyways + mem::swap(&mut self.i1, &mut self.i2); + self.i1.next() + } + } + } + + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + let m = match self.i1.advance_by(n) { + Ok(_) => return Ok(()), + Err(m) => m, + }; + mem::swap(&mut self.i1, &mut self.i2); + self.i1.advance_by(n - m).map_err(|o| o + m) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let len = self.len(); + (len, Some(len)) + } + + fn fold(self, accum: Acc, mut f: F) -> Acc + where + F: FnMut(Acc, Self::Item) -> Acc, + { + let accum = self.i1.fold(accum, &mut f); + self.i2.fold(accum, &mut f) + } + + fn try_fold(&mut self, init: B, mut f: F) -> R + where + F: FnMut(B, Self::Item) -> R, + R: Try, + { + let acc = self.i1.try_fold(init, &mut f)?; + self.i2.try_fold(acc, &mut f) + } + + #[inline] + fn last(mut self) -> Option<&'a mut T> { + self.next_back() + } + + #[inline] + unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> Self::Item { + // Safety: The TrustedRandomAccess contract requires that callers only pass an index + // that is in bounds. + unsafe { + let i1_len = self.i1.len(); + if idx < i1_len { + self.i1.__iterator_get_unchecked(idx) + } else { + self.i2.__iterator_get_unchecked(idx - i1_len) + } + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T> DoubleEndedIterator for IterMut<'a, T> { + #[inline] + fn next_back(&mut self) -> Option<&'a mut T> { + match self.i2.next_back() { + Some(val) => Some(val), + None => { + // most of the time, the iterator will either always + // call next(), or always call next_back(). By swapping + // the iterators once the first one is empty, we ensure + // that the first branch is taken as often as possible, + // without sacrificing correctness, as i2 is empty anyways + mem::swap(&mut self.i1, &mut self.i2); + self.i2.next_back() + } + } + } + + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + let m = match self.i2.advance_back_by(n) { + Ok(_) => return Ok(()), + Err(m) => m, + }; + + mem::swap(&mut self.i1, &mut self.i2); + self.i2.advance_back_by(n - m).map_err(|o| m + o) + } + + fn rfold(self, accum: Acc, mut f: F) -> Acc + where + F: FnMut(Acc, Self::Item) -> Acc, + { + let accum = self.i2.rfold(accum, &mut f); + self.i1.rfold(accum, &mut f) + } + + fn try_rfold(&mut self, init: B, mut f: F) -> R + where + F: FnMut(B, Self::Item) -> R, + R: Try, + { + let acc = self.i2.try_rfold(init, &mut f)?; + self.i1.try_rfold(acc, &mut f) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ExactSizeIterator for IterMut<'_, T> { + fn len(&self) -> usize { + self.i1.len() + self.i2.len() + } + + fn is_empty(&self) -> bool { + self.i1.is_empty() && self.i2.is_empty() + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for IterMut<'_, T> {} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for IterMut<'_, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl TrustedRandomAccess for IterMut<'_, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl TrustedRandomAccessNoCoerce for IterMut<'_, T> { + const MAY_HAVE_SIDE_EFFECT: bool = false; +} diff --git a/crux-mir/lib/alloc/src/collections/vec_deque/macros.rs b/crux-mir/lib/alloc/src/collections/vec_deque/macros.rs new file mode 100644 index 000000000..5c7913073 --- /dev/null +++ b/crux-mir/lib/alloc/src/collections/vec_deque/macros.rs @@ -0,0 +1,19 @@ +macro_rules! __impl_slice_eq1 { + ([$($vars:tt)*] $lhs:ty, $rhs:ty, $($constraints:tt)*) => { + #[stable(feature = "vec_deque_partial_eq_slice", since = "1.17.0")] + impl PartialEq<$rhs> for $lhs + where + T: PartialEq, + $($constraints)* + { + fn eq(&self, other: &$rhs) -> bool { + if self.len() != other.len() { + return false; + } + let (sa, sb) = self.as_slices(); + let (oa, ob) = other[..].split_at(sa.len()); + sa == oa && sb == ob + } + } + } +} diff --git a/crux-mir/lib/alloc/src/collections/vec_deque/mod.rs b/crux-mir/lib/alloc/src/collections/vec_deque/mod.rs new file mode 100644 index 000000000..451e4936b --- /dev/null +++ b/crux-mir/lib/alloc/src/collections/vec_deque/mod.rs @@ -0,0 +1,2909 @@ +//! A double-ended queue (deque) implemented with a growable ring buffer. +//! +//! This queue has *O*(1) amortized inserts and removals from both ends of the +//! container. It also has *O*(1) indexing like a vector. The contained elements +//! are not required to be copyable, and the queue will be sendable if the +//! contained type is sendable. + +#![stable(feature = "rust1", since = "1.0.0")] + +use core::cmp::{self, Ordering}; +use core::fmt; +use core::hash::{Hash, Hasher}; +use core::iter::{repeat_n, repeat_with, ByRefSized, FromIterator}; +use core::mem::{ManuallyDrop, SizedTypeProperties}; +use core::ops::{Index, IndexMut, Range, RangeBounds}; +use core::ptr; +use core::slice; + +// This is used in a bunch of intra-doc links. +// FIXME: For some reason, `#[cfg(doc)]` wasn't sufficient, resulting in +// failures in linkchecker even though rustdoc built the docs just fine. +#[allow(unused_imports)] +use core::mem; + +use crate::alloc::{Allocator, Global}; +use crate::collections::TryReserveError; +use crate::collections::TryReserveErrorKind; +use crate::raw_vec::RawVec; +use crate::vec::Vec; + +#[macro_use] +mod macros; + +#[stable(feature = "drain", since = "1.6.0")] +pub use self::drain::Drain; + +mod drain; + +#[stable(feature = "rust1", since = "1.0.0")] +pub use self::iter_mut::IterMut; + +mod iter_mut; + +#[stable(feature = "rust1", since = "1.0.0")] +pub use self::into_iter::IntoIter; + +mod into_iter; + +#[stable(feature = "rust1", since = "1.0.0")] +pub use self::iter::Iter; + +mod iter; + +use self::spec_extend::SpecExtend; + +mod spec_extend; + +use self::spec_from_iter::SpecFromIter; + +mod spec_from_iter; + +#[cfg(test)] +mod tests; + +/// A double-ended queue implemented with a growable ring buffer. +/// +/// The "default" usage of this type as a queue is to use [`push_back`] to add to +/// the queue, and [`pop_front`] to remove from the queue. [`extend`] and [`append`] +/// push onto the back in this manner, and iterating over `VecDeque` goes front +/// to back. +/// +/// A `VecDeque` with a known list of items can be initialized from an array: +/// +/// ``` +/// use std::collections::VecDeque; +/// +/// let deq = VecDeque::from([-1, 0, 1]); +/// ``` +/// +/// Since `VecDeque` is a ring buffer, its elements are not necessarily contiguous +/// in memory. If you want to access the elements as a single slice, such as for +/// efficient sorting, you can use [`make_contiguous`]. It rotates the `VecDeque` +/// so that its elements do not wrap, and returns a mutable slice to the +/// now-contiguous element sequence. +/// +/// [`push_back`]: VecDeque::push_back +/// [`pop_front`]: VecDeque::pop_front +/// [`extend`]: VecDeque::extend +/// [`append`]: VecDeque::append +/// [`make_contiguous`]: VecDeque::make_contiguous +#[cfg_attr(not(test), rustc_diagnostic_item = "VecDeque")] +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_insignificant_dtor] +pub struct VecDeque< + T, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator = Global, +> { + // `self[0]`, if it exists, is `buf[head]`. + // `head < buf.capacity()`, unless `buf.capacity() == 0` when `head == 0`. + head: usize, + // the number of initialized elements, starting from the one at `head` and potentially wrapping around. + // if `len == 0`, the exact value of `head` is unimportant. + // if `T` is zero-Sized, then `self.len <= usize::MAX`, otherwise `self.len <= isize::MAX as usize`. + len: usize, + buf: RawVec, +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Clone for VecDeque { + fn clone(&self) -> Self { + let mut deq = Self::with_capacity_in(self.len(), self.allocator().clone()); + deq.extend(self.iter().cloned()); + deq + } + + fn clone_from(&mut self, other: &Self) { + self.clear(); + self.extend(other.iter().cloned()); + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +unsafe impl<#[may_dangle] T, A: Allocator> Drop for VecDeque { + fn drop(&mut self) { + /// Runs the destructor for all items in the slice when it gets dropped (normally or + /// during unwinding). + struct Dropper<'a, T>(&'a mut [T]); + + impl<'a, T> Drop for Dropper<'a, T> { + fn drop(&mut self) { + unsafe { + ptr::drop_in_place(self.0); + } + } + } + + let (front, back) = self.as_mut_slices(); + unsafe { + let _back_dropper = Dropper(back); + // use drop for [T] + ptr::drop_in_place(front); + } + // RawVec handles deallocation + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Default for VecDeque { + /// Creates an empty deque. + #[inline] + fn default() -> VecDeque { + VecDeque::new() + } +} + +impl VecDeque { + /// Marginally more convenient + #[inline] + fn ptr(&self) -> *mut T { + self.buf.ptr() + } + + /// Moves an element out of the buffer + #[inline] + unsafe fn buffer_read(&mut self, off: usize) -> T { + unsafe { ptr::read(self.ptr().add(off)) } + } + + /// Writes an element into the buffer, moving it. + #[inline] + unsafe fn buffer_write(&mut self, off: usize, value: T) { + unsafe { + ptr::write(self.ptr().add(off), value); + } + } + + /// Returns a slice pointer into the buffer. + /// `range` must lie inside `0..self.capacity()`. + #[inline] + unsafe fn buffer_range(&self, range: Range) -> *mut [T] { + unsafe { + ptr::slice_from_raw_parts_mut(self.ptr().add(range.start), range.end - range.start) + } + } + + /// Returns `true` if the buffer is at full capacity. + #[inline] + fn is_full(&self) -> bool { + self.len == self.capacity() + } + + /// Returns the index in the underlying buffer for a given logical element + /// index + addend. + #[inline] + fn wrap_add(&self, idx: usize, addend: usize) -> usize { + wrap_index(idx.wrapping_add(addend), self.capacity()) + } + + #[inline] + fn to_physical_idx(&self, idx: usize) -> usize { + self.wrap_add(self.head, idx) + } + + /// Returns the index in the underlying buffer for a given logical element + /// index - subtrahend. + #[inline] + fn wrap_sub(&self, idx: usize, subtrahend: usize) -> usize { + wrap_index(idx.wrapping_sub(subtrahend).wrapping_add(self.capacity()), self.capacity()) + } + + /// Copies a contiguous block of memory len long from src to dst + #[inline] + unsafe fn copy(&mut self, src: usize, dst: usize, len: usize) { + debug_assert!( + dst + len <= self.capacity(), + "cpy dst={} src={} len={} cap={}", + dst, + src, + len, + self.capacity() + ); + debug_assert!( + src + len <= self.capacity(), + "cpy dst={} src={} len={} cap={}", + dst, + src, + len, + self.capacity() + ); + unsafe { + ptr::copy(self.ptr().add(src), self.ptr().add(dst), len); + } + } + + /// Copies a contiguous block of memory len long from src to dst + #[inline] + unsafe fn copy_nonoverlapping(&mut self, src: usize, dst: usize, len: usize) { + debug_assert!( + dst + len <= self.capacity(), + "cno dst={} src={} len={} cap={}", + dst, + src, + len, + self.capacity() + ); + debug_assert!( + src + len <= self.capacity(), + "cno dst={} src={} len={} cap={}", + dst, + src, + len, + self.capacity() + ); + unsafe { + ptr::copy_nonoverlapping(self.ptr().add(src), self.ptr().add(dst), len); + } + } + + /// Copies a potentially wrapping block of memory len long from src to dest. + /// (abs(dst - src) + len) must be no larger than capacity() (There must be at + /// most one continuous overlapping region between src and dest). + unsafe fn wrap_copy(&mut self, src: usize, dst: usize, len: usize) { + debug_assert!( + cmp::min(src.abs_diff(dst), self.capacity() - src.abs_diff(dst)) + len + <= self.capacity(), + "wrc dst={} src={} len={} cap={}", + dst, + src, + len, + self.capacity() + ); + + // If T is a ZST, don't do any copying. + if T::IS_ZST || src == dst || len == 0 { + return; + } + + let dst_after_src = self.wrap_sub(dst, src) < len; + + let src_pre_wrap_len = self.capacity() - src; + let dst_pre_wrap_len = self.capacity() - dst; + let src_wraps = src_pre_wrap_len < len; + let dst_wraps = dst_pre_wrap_len < len; + + match (dst_after_src, src_wraps, dst_wraps) { + (_, false, false) => { + // src doesn't wrap, dst doesn't wrap + // + // S . . . + // 1 [_ _ A A B B C C _] + // 2 [_ _ A A A A B B _] + // D . . . + // + unsafe { + self.copy(src, dst, len); + } + } + (false, false, true) => { + // dst before src, src doesn't wrap, dst wraps + // + // S . . . + // 1 [A A B B _ _ _ C C] + // 2 [A A B B _ _ _ A A] + // 3 [B B B B _ _ _ A A] + // . . D . + // + unsafe { + self.copy(src, dst, dst_pre_wrap_len); + self.copy(src + dst_pre_wrap_len, 0, len - dst_pre_wrap_len); + } + } + (true, false, true) => { + // src before dst, src doesn't wrap, dst wraps + // + // S . . . + // 1 [C C _ _ _ A A B B] + // 2 [B B _ _ _ A A B B] + // 3 [B B _ _ _ A A A A] + // . . D . + // + unsafe { + self.copy(src + dst_pre_wrap_len, 0, len - dst_pre_wrap_len); + self.copy(src, dst, dst_pre_wrap_len); + } + } + (false, true, false) => { + // dst before src, src wraps, dst doesn't wrap + // + // . . S . + // 1 [C C _ _ _ A A B B] + // 2 [C C _ _ _ B B B B] + // 3 [C C _ _ _ B B C C] + // D . . . + // + unsafe { + self.copy(src, dst, src_pre_wrap_len); + self.copy(0, dst + src_pre_wrap_len, len - src_pre_wrap_len); + } + } + (true, true, false) => { + // src before dst, src wraps, dst doesn't wrap + // + // . . S . + // 1 [A A B B _ _ _ C C] + // 2 [A A A A _ _ _ C C] + // 3 [C C A A _ _ _ C C] + // D . . . + // + unsafe { + self.copy(0, dst + src_pre_wrap_len, len - src_pre_wrap_len); + self.copy(src, dst, src_pre_wrap_len); + } + } + (false, true, true) => { + // dst before src, src wraps, dst wraps + // + // . . . S . + // 1 [A B C D _ E F G H] + // 2 [A B C D _ E G H H] + // 3 [A B C D _ E G H A] + // 4 [B C C D _ E G H A] + // . . D . . + // + debug_assert!(dst_pre_wrap_len > src_pre_wrap_len); + let delta = dst_pre_wrap_len - src_pre_wrap_len; + unsafe { + self.copy(src, dst, src_pre_wrap_len); + self.copy(0, dst + src_pre_wrap_len, delta); + self.copy(delta, 0, len - dst_pre_wrap_len); + } + } + (true, true, true) => { + // src before dst, src wraps, dst wraps + // + // . . S . . + // 1 [A B C D _ E F G H] + // 2 [A A B D _ E F G H] + // 3 [H A B D _ E F G H] + // 4 [H A B D _ E F F G] + // . . . D . + // + debug_assert!(src_pre_wrap_len > dst_pre_wrap_len); + let delta = src_pre_wrap_len - dst_pre_wrap_len; + unsafe { + self.copy(0, delta, len - src_pre_wrap_len); + self.copy(self.capacity() - delta, 0, delta); + self.copy(src, dst, dst_pre_wrap_len); + } + } + } + } + + /// Copies all values from `src` to `dst`, wrapping around if needed. + /// Assumes capacity is sufficient. + #[inline] + unsafe fn copy_slice(&mut self, dst: usize, src: &[T]) { + debug_assert!(src.len() <= self.capacity()); + let head_room = self.capacity() - dst; + if src.len() <= head_room { + unsafe { + ptr::copy_nonoverlapping(src.as_ptr(), self.ptr().add(dst), src.len()); + } + } else { + let (left, right) = src.split_at(head_room); + unsafe { + ptr::copy_nonoverlapping(left.as_ptr(), self.ptr().add(dst), left.len()); + ptr::copy_nonoverlapping(right.as_ptr(), self.ptr(), right.len()); + } + } + } + + /// Writes all values from `iter` to `dst`. + /// + /// # Safety + /// + /// Assumes no wrapping around happens. + /// Assumes capacity is sufficient. + #[inline] + unsafe fn write_iter( + &mut self, + dst: usize, + iter: impl Iterator, + written: &mut usize, + ) { + iter.enumerate().for_each(|(i, element)| unsafe { + self.buffer_write(dst + i, element); + *written += 1; + }); + } + + /// Writes all values from `iter` to `dst`, wrapping + /// at the end of the buffer and returns the number + /// of written values. + /// + /// # Safety + /// + /// Assumes that `iter` yields at most `len` items. + /// Assumes capacity is sufficient. + unsafe fn write_iter_wrapping( + &mut self, + dst: usize, + mut iter: impl Iterator, + len: usize, + ) -> usize { + struct Guard<'a, T, A: Allocator> { + deque: &'a mut VecDeque, + written: usize, + } + + impl<'a, T, A: Allocator> Drop for Guard<'a, T, A> { + fn drop(&mut self) { + self.deque.len += self.written; + } + } + + let head_room = self.capacity() - dst; + + let mut guard = Guard { deque: self, written: 0 }; + + if head_room >= len { + unsafe { guard.deque.write_iter(dst, iter, &mut guard.written) }; + } else { + unsafe { + guard.deque.write_iter( + dst, + ByRefSized(&mut iter).take(head_room), + &mut guard.written, + ); + guard.deque.write_iter(0, iter, &mut guard.written) + }; + } + + guard.written + } + + /// Frobs the head and tail sections around to handle the fact that we + /// just reallocated. Unsafe because it trusts old_capacity. + #[inline] + unsafe fn handle_capacity_increase(&mut self, old_capacity: usize) { + let new_capacity = self.capacity(); + debug_assert!(new_capacity >= old_capacity); + + // Move the shortest contiguous section of the ring buffer + // + // H := head + // L := last element (`self.to_physical_idx(self.len - 1)`) + // + // H L + // [o o o o o o o . ] + // H L + // A [o o o o o o o . . . . . . . . . ] + // L H + // [o o o o o o o o ] + // H L + // B [. . . o o o o o o o . . . . . . ] + // L H + // [o o o o o o o o ] + // L H + // C [o o o o o . . . . . . . . . o o ] + + // can't use is_contiguous() because the capacity is already updated. + if self.head <= old_capacity - self.len { + // A + // Nop + } else { + let head_len = old_capacity - self.head; + let tail_len = self.len - head_len; + if head_len > tail_len && new_capacity - old_capacity >= tail_len { + // B + unsafe { + self.copy_nonoverlapping(0, old_capacity, tail_len); + } + } else { + // C + let new_head = new_capacity - head_len; + unsafe { + // can't use copy_nonoverlapping here, because if e.g. head_len = 2 + // and new_capacity = old_capacity + 1, then the heads overlap. + self.copy(self.head, new_head, head_len); + } + self.head = new_head; + } + } + debug_assert!(self.head < self.capacity() || self.capacity() == 0); + } +} + +impl VecDeque { + /// Creates an empty deque. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let deque: VecDeque = VecDeque::new(); + /// ``` + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_vec_deque_new", since = "CURRENT_RUSTC_VERSION")] + #[must_use] + pub const fn new() -> VecDeque { + // FIXME: This should just be `VecDeque::new_in(Global)` once that hits stable. + VecDeque { head: 0, len: 0, buf: RawVec::NEW } + } + + /// Creates an empty deque with space for at least `capacity` elements. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let deque: VecDeque = VecDeque::with_capacity(10); + /// ``` + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + pub fn with_capacity(capacity: usize) -> VecDeque { + Self::with_capacity_in(capacity, Global) + } +} + +impl VecDeque { + /// Creates an empty deque. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let deque: VecDeque = VecDeque::new(); + /// ``` + #[inline] + #[unstable(feature = "allocator_api", issue = "32838")] + pub const fn new_in(alloc: A) -> VecDeque { + VecDeque { head: 0, len: 0, buf: RawVec::new_in(alloc) } + } + + /// Creates an empty deque with space for at least `capacity` elements. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let deque: VecDeque = VecDeque::with_capacity(10); + /// ``` + #[unstable(feature = "allocator_api", issue = "32838")] + pub fn with_capacity_in(capacity: usize, alloc: A) -> VecDeque { + VecDeque { head: 0, len: 0, buf: RawVec::with_capacity_in(capacity, alloc) } + } + + /// Creates a `VecDeque` from a raw allocation, when the initialized + /// part of that allocation forms a *contiguous* subslice thereof. + /// + /// For use by `vec::IntoIter::into_vecdeque` + /// + /// # Safety + /// + /// All the usual requirements on the allocated memory like in + /// `Vec::from_raw_parts_in`, but takes a *range* of elements that are + /// initialized rather than only supporting `0..len`. Requires that + /// `initialized.start` ≤ `initialized.end` ≤ `capacity`. + #[inline] + pub(crate) unsafe fn from_contiguous_raw_parts_in( + ptr: *mut T, + initialized: Range, + capacity: usize, + alloc: A, + ) -> Self { + debug_assert!(initialized.start <= initialized.end); + debug_assert!(initialized.end <= capacity); + + // SAFETY: Our safety precondition guarantees the range length won't wrap, + // and that the allocation is valid for use in `RawVec`. + unsafe { + VecDeque { + head: initialized.start, + len: initialized.end.unchecked_sub(initialized.start), + buf: RawVec::from_raw_parts_in(ptr, capacity, alloc), + } + } + } + + /// Provides a reference to the element at the given index. + /// + /// Element at index 0 is the front of the queue. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf = VecDeque::new(); + /// buf.push_back(3); + /// buf.push_back(4); + /// buf.push_back(5); + /// buf.push_back(6); + /// assert_eq!(buf.get(1), Some(&4)); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn get(&self, index: usize) -> Option<&T> { + if index < self.len { + let idx = self.to_physical_idx(index); + unsafe { Some(&*self.ptr().add(idx)) } + } else { + None + } + } + + /// Provides a mutable reference to the element at the given index. + /// + /// Element at index 0 is the front of the queue. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf = VecDeque::new(); + /// buf.push_back(3); + /// buf.push_back(4); + /// buf.push_back(5); + /// buf.push_back(6); + /// assert_eq!(buf[1], 4); + /// if let Some(elem) = buf.get_mut(1) { + /// *elem = 7; + /// } + /// assert_eq!(buf[1], 7); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn get_mut(&mut self, index: usize) -> Option<&mut T> { + if index < self.len { + let idx = self.to_physical_idx(index); + unsafe { Some(&mut *self.ptr().add(idx)) } + } else { + None + } + } + + /// Swaps elements at indices `i` and `j`. + /// + /// `i` and `j` may be equal. + /// + /// Element at index 0 is the front of the queue. + /// + /// # Panics + /// + /// Panics if either index is out of bounds. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf = VecDeque::new(); + /// buf.push_back(3); + /// buf.push_back(4); + /// buf.push_back(5); + /// assert_eq!(buf, [3, 4, 5]); + /// buf.swap(0, 2); + /// assert_eq!(buf, [5, 4, 3]); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn swap(&mut self, i: usize, j: usize) { + assert!(i < self.len()); + assert!(j < self.len()); + let ri = self.to_physical_idx(i); + let rj = self.to_physical_idx(j); + unsafe { ptr::swap(self.ptr().add(ri), self.ptr().add(rj)) } + } + + /// Returns the number of elements the deque can hold without + /// reallocating. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let buf: VecDeque = VecDeque::with_capacity(10); + /// assert!(buf.capacity() >= 10); + /// ``` + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn capacity(&self) -> usize { + if T::IS_ZST { usize::MAX } else { self.buf.capacity() } + } + + /// Reserves the minimum capacity for at least `additional` more elements to be inserted in the + /// given deque. Does nothing if the capacity is already sufficient. + /// + /// Note that the allocator may give the collection more space than it requests. Therefore + /// capacity can not be relied upon to be precisely minimal. Prefer [`reserve`] if future + /// insertions are expected. + /// + /// # Panics + /// + /// Panics if the new capacity overflows `usize`. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf: VecDeque = [1].into(); + /// buf.reserve_exact(10); + /// assert!(buf.capacity() >= 11); + /// ``` + /// + /// [`reserve`]: VecDeque::reserve + #[stable(feature = "rust1", since = "1.0.0")] + pub fn reserve_exact(&mut self, additional: usize) { + let new_cap = self.len.checked_add(additional).expect("capacity overflow"); + let old_cap = self.capacity(); + + if new_cap > old_cap { + self.buf.reserve_exact(self.len, additional); + unsafe { + self.handle_capacity_increase(old_cap); + } + } + } + + /// Reserves capacity for at least `additional` more elements to be inserted in the given + /// deque. The collection may reserve more space to speculatively avoid frequent reallocations. + /// + /// # Panics + /// + /// Panics if the new capacity overflows `usize`. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf: VecDeque = [1].into(); + /// buf.reserve(10); + /// assert!(buf.capacity() >= 11); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn reserve(&mut self, additional: usize) { + let new_cap = self.len.checked_add(additional).expect("capacity overflow"); + let old_cap = self.capacity(); + + if new_cap > old_cap { + // we don't need to reserve_exact(), as the size doesn't have + // to be a power of 2. + self.buf.reserve(self.len, additional); + unsafe { + self.handle_capacity_increase(old_cap); + } + } + } + + /// Tries to reserve the minimum capacity for at least `additional` more elements to + /// be inserted in the given deque. After calling `try_reserve_exact`, + /// capacity will be greater than or equal to `self.len() + additional` if + /// it returns `Ok(())`. Does nothing if the capacity is already sufficient. + /// + /// Note that the allocator may give the collection more space than it + /// requests. Therefore, capacity can not be relied upon to be precisely + /// minimal. Prefer [`try_reserve`] if future insertions are expected. + /// + /// [`try_reserve`]: VecDeque::try_reserve + /// + /// # Errors + /// + /// If the capacity overflows `usize`, or the allocator reports a failure, then an error + /// is returned. + /// + /// # Examples + /// + /// ``` + /// use std::collections::TryReserveError; + /// use std::collections::VecDeque; + /// + /// fn process_data(data: &[u32]) -> Result, TryReserveError> { + /// let mut output = VecDeque::new(); + /// + /// // Pre-reserve the memory, exiting if we can't + /// output.try_reserve_exact(data.len())?; + /// + /// // Now we know this can't OOM(Out-Of-Memory) in the middle of our complex work + /// output.extend(data.iter().map(|&val| { + /// val * 2 + 5 // very complicated + /// })); + /// + /// Ok(output) + /// } + /// # process_data(&[1, 2, 3]).expect("why is the test harness OOMing on 12 bytes?"); + /// ``` + #[stable(feature = "try_reserve", since = "1.57.0")] + pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> { + let new_cap = + self.len.checked_add(additional).ok_or(TryReserveErrorKind::CapacityOverflow)?; + let old_cap = self.capacity(); + + if new_cap > old_cap { + self.buf.try_reserve_exact(self.len, additional)?; + unsafe { + self.handle_capacity_increase(old_cap); + } + } + Ok(()) + } + + /// Tries to reserve capacity for at least `additional` more elements to be inserted + /// in the given deque. The collection may reserve more space to speculatively avoid + /// frequent reallocations. After calling `try_reserve`, capacity will be + /// greater than or equal to `self.len() + additional` if it returns + /// `Ok(())`. Does nothing if capacity is already sufficient. This method + /// preserves the contents even if an error occurs. + /// + /// # Errors + /// + /// If the capacity overflows `usize`, or the allocator reports a failure, then an error + /// is returned. + /// + /// # Examples + /// + /// ``` + /// use std::collections::TryReserveError; + /// use std::collections::VecDeque; + /// + /// fn process_data(data: &[u32]) -> Result, TryReserveError> { + /// let mut output = VecDeque::new(); + /// + /// // Pre-reserve the memory, exiting if we can't + /// output.try_reserve(data.len())?; + /// + /// // Now we know this can't OOM in the middle of our complex work + /// output.extend(data.iter().map(|&val| { + /// val * 2 + 5 // very complicated + /// })); + /// + /// Ok(output) + /// } + /// # process_data(&[1, 2, 3]).expect("why is the test harness OOMing on 12 bytes?"); + /// ``` + #[stable(feature = "try_reserve", since = "1.57.0")] + pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { + let new_cap = + self.len.checked_add(additional).ok_or(TryReserveErrorKind::CapacityOverflow)?; + let old_cap = self.capacity(); + + if new_cap > old_cap { + self.buf.try_reserve(self.len, additional)?; + unsafe { + self.handle_capacity_increase(old_cap); + } + } + Ok(()) + } + + /// Shrinks the capacity of the deque as much as possible. + /// + /// It will drop down as close as possible to the length but the allocator may still inform the + /// deque that there is space for a few more elements. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf = VecDeque::with_capacity(15); + /// buf.extend(0..4); + /// assert_eq!(buf.capacity(), 15); + /// buf.shrink_to_fit(); + /// assert!(buf.capacity() >= 4); + /// ``` + #[stable(feature = "deque_extras_15", since = "1.5.0")] + pub fn shrink_to_fit(&mut self) { + self.shrink_to(0); + } + + /// Shrinks the capacity of the deque with a lower bound. + /// + /// The capacity will remain at least as large as both the length + /// and the supplied value. + /// + /// If the current capacity is less than the lower limit, this is a no-op. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf = VecDeque::with_capacity(15); + /// buf.extend(0..4); + /// assert_eq!(buf.capacity(), 15); + /// buf.shrink_to(6); + /// assert!(buf.capacity() >= 6); + /// buf.shrink_to(0); + /// assert!(buf.capacity() >= 4); + /// ``` + #[stable(feature = "shrink_to", since = "1.56.0")] + pub fn shrink_to(&mut self, min_capacity: usize) { + let target_cap = min_capacity.max(self.len); + + // never shrink ZSTs + if T::IS_ZST || self.capacity() <= target_cap { + return; + } + + if target_cap < self.capacity() { + // There are three cases of interest: + // All elements are out of desired bounds + // Elements are contiguous, and head is out of desired bounds + // Elements are discontiguous, and tail is out of desired bounds + // + // At all other times, element positions are unaffected. + // + // Indicates that elements at the head should be moved. + + let tail_outside = (target_cap + 1..=self.capacity()).contains(&(self.head + self.len)); + // Move elements from out of desired bounds (positions after target_cap) + if self.len == 0 { + self.head = 0; + } else if self.head >= target_cap && tail_outside { + // H := head + // L := last element + // H L + // [. . . . . . . . o o o o o o o . ] + // H L + // [o o o o o o o . ] + unsafe { + // nonoverlapping because self.head >= target_cap >= self.len + self.copy_nonoverlapping(self.head, 0, self.len); + } + self.head = 0; + } else if self.head < target_cap && tail_outside { + // H := head + // L := last element + // H L + // [. . . o o o o o o o . . . . . . ] + // L H + // [o o . o o o o o ] + let len = self.head + self.len - target_cap; + unsafe { + self.copy_nonoverlapping(target_cap, 0, len); + } + } else if self.head >= target_cap { + // H := head + // L := last element + // L H + // [o o o o o . . . . . . . . . o o ] + // L H + // [o o o o o . o o ] + let len = self.capacity() - self.head; + let new_head = target_cap - len; + unsafe { + // can't use copy_nonoverlapping here for the same reason + // as in `handle_capacity_increase()` + self.copy(self.head, new_head, len); + } + self.head = new_head; + } + + self.buf.shrink_to_fit(target_cap); + + debug_assert!(self.head < self.capacity() || self.capacity() == 0); + debug_assert!(self.len <= self.capacity()); + } + } + + /// Shortens the deque, keeping the first `len` elements and dropping + /// the rest. + /// + /// If `len` is greater than the deque's current length, this has no + /// effect. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf = VecDeque::new(); + /// buf.push_back(5); + /// buf.push_back(10); + /// buf.push_back(15); + /// assert_eq!(buf, [5, 10, 15]); + /// buf.truncate(1); + /// assert_eq!(buf, [5]); + /// ``` + #[stable(feature = "deque_extras", since = "1.16.0")] + pub fn truncate(&mut self, len: usize) { + /// Runs the destructor for all items in the slice when it gets dropped (normally or + /// during unwinding). + struct Dropper<'a, T>(&'a mut [T]); + + impl<'a, T> Drop for Dropper<'a, T> { + fn drop(&mut self) { + unsafe { + ptr::drop_in_place(self.0); + } + } + } + + // Safe because: + // + // * Any slice passed to `drop_in_place` is valid; the second case has + // `len <= front.len()` and returning on `len > self.len()` ensures + // `begin <= back.len()` in the first case + // * The head of the VecDeque is moved before calling `drop_in_place`, + // so no value is dropped twice if `drop_in_place` panics + unsafe { + if len >= self.len { + return; + } + + let (front, back) = self.as_mut_slices(); + if len > front.len() { + let begin = len - front.len(); + let drop_back = back.get_unchecked_mut(begin..) as *mut _; + self.len = len; + ptr::drop_in_place(drop_back); + } else { + let drop_back = back as *mut _; + let drop_front = front.get_unchecked_mut(len..) as *mut _; + self.len = len; + + // Make sure the second half is dropped even when a destructor + // in the first one panics. + let _back_dropper = Dropper(&mut *drop_back); + ptr::drop_in_place(drop_front); + } + } + } + + /// Returns a reference to the underlying allocator. + #[unstable(feature = "allocator_api", issue = "32838")] + #[inline] + pub fn allocator(&self) -> &A { + self.buf.allocator() + } + + /// Returns a front-to-back iterator. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf = VecDeque::new(); + /// buf.push_back(5); + /// buf.push_back(3); + /// buf.push_back(4); + /// let b: &[_] = &[&5, &3, &4]; + /// let c: Vec<&i32> = buf.iter().collect(); + /// assert_eq!(&c[..], b); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn iter(&self) -> Iter<'_, T> { + let (a, b) = self.as_slices(); + Iter::new(a.iter(), b.iter()) + } + + /// Returns a front-to-back iterator that returns mutable references. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf = VecDeque::new(); + /// buf.push_back(5); + /// buf.push_back(3); + /// buf.push_back(4); + /// for num in buf.iter_mut() { + /// *num = *num - 2; + /// } + /// let b: &[_] = &[&mut 3, &mut 1, &mut 2]; + /// assert_eq!(&buf.iter_mut().collect::>()[..], b); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn iter_mut(&mut self) -> IterMut<'_, T> { + let (a, b) = self.as_mut_slices(); + IterMut::new(a.iter_mut(), b.iter_mut()) + } + + /// Returns a pair of slices which contain, in order, the contents of the + /// deque. + /// + /// If [`make_contiguous`] was previously called, all elements of the + /// deque will be in the first slice and the second slice will be empty. + /// + /// [`make_contiguous`]: VecDeque::make_contiguous + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut deque = VecDeque::new(); + /// + /// deque.push_back(0); + /// deque.push_back(1); + /// deque.push_back(2); + /// + /// assert_eq!(deque.as_slices(), (&[0, 1, 2][..], &[][..])); + /// + /// deque.push_front(10); + /// deque.push_front(9); + /// + /// assert_eq!(deque.as_slices(), (&[9, 10][..], &[0, 1, 2][..])); + /// ``` + #[inline] + #[stable(feature = "deque_extras_15", since = "1.5.0")] + pub fn as_slices(&self) -> (&[T], &[T]) { + let (a_range, b_range) = self.slice_ranges(..); + // SAFETY: `slice_ranges` always returns valid ranges into + // the physical buffer. + unsafe { (&*self.buffer_range(a_range), &*self.buffer_range(b_range)) } + } + + /// Returns a pair of slices which contain, in order, the contents of the + /// deque. + /// + /// If [`make_contiguous`] was previously called, all elements of the + /// deque will be in the first slice and the second slice will be empty. + /// + /// [`make_contiguous`]: VecDeque::make_contiguous + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut deque = VecDeque::new(); + /// + /// deque.push_back(0); + /// deque.push_back(1); + /// + /// deque.push_front(10); + /// deque.push_front(9); + /// + /// deque.as_mut_slices().0[0] = 42; + /// deque.as_mut_slices().1[0] = 24; + /// assert_eq!(deque.as_slices(), (&[42, 10][..], &[24, 1][..])); + /// ``` + #[inline] + #[stable(feature = "deque_extras_15", since = "1.5.0")] + pub fn as_mut_slices(&mut self) -> (&mut [T], &mut [T]) { + let (a_range, b_range) = self.slice_ranges(..); + // SAFETY: `slice_ranges` always returns valid ranges into + // the physical buffer. + unsafe { (&mut *self.buffer_range(a_range), &mut *self.buffer_range(b_range)) } + } + + /// Returns the number of elements in the deque. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut deque = VecDeque::new(); + /// assert_eq!(deque.len(), 0); + /// deque.push_back(1); + /// assert_eq!(deque.len(), 1); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn len(&self) -> usize { + self.len + } + + /// Returns `true` if the deque is empty. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut deque = VecDeque::new(); + /// assert!(deque.is_empty()); + /// deque.push_front(1); + /// assert!(!deque.is_empty()); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Given a range into the logical buffer of the deque, this function + /// return two ranges into the physical buffer that correspond to + /// the given range. + fn slice_ranges(&self, range: R) -> (Range, Range) + where + R: RangeBounds, + { + let Range { start, end } = slice::range(range, ..self.len); + let len = end - start; + + if len == 0 { + (0..0, 0..0) + } else { + // `slice::range` guarantees that `start <= end <= self.len`. + // because `len != 0`, we know that `start < end`, so `start < self.len` + // and the indexing is valid. + let wrapped_start = self.to_physical_idx(start); + + // this subtraction can never overflow because `wrapped_start` is + // at most `self.capacity()` (and if `self.capacity != 0`, then `wrapped_start` is strictly less + // than `self.capacity`). + let head_len = self.capacity() - wrapped_start; + + if head_len >= len { + // we know that `len + wrapped_start <= self.capacity <= usize::MAX`, so this addition can't overflow + (wrapped_start..wrapped_start + len, 0..0) + } else { + // can't overflow because of the if condition + let tail_len = len - head_len; + (wrapped_start..self.capacity(), 0..tail_len) + } + } + } + + /// Creates an iterator that covers the specified range in the deque. + /// + /// # Panics + /// + /// Panics if the starting point is greater than the end point or if + /// the end point is greater than the length of the deque. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let deque: VecDeque<_> = [1, 2, 3].into(); + /// let range = deque.range(2..).copied().collect::>(); + /// assert_eq!(range, [3]); + /// + /// // A full range covers all contents + /// let all = deque.range(..); + /// assert_eq!(all.len(), 3); + /// ``` + #[inline] + #[stable(feature = "deque_range", since = "1.51.0")] + pub fn range(&self, range: R) -> Iter<'_, T> + where + R: RangeBounds, + { + let (a_range, b_range) = self.slice_ranges(range); + // SAFETY: The ranges returned by `slice_ranges` + // are valid ranges into the physical buffer, so + // it's ok to pass them to `buffer_range` and + // dereference the result. + let a = unsafe { &*self.buffer_range(a_range) }; + let b = unsafe { &*self.buffer_range(b_range) }; + Iter::new(a.iter(), b.iter()) + } + + /// Creates an iterator that covers the specified mutable range in the deque. + /// + /// # Panics + /// + /// Panics if the starting point is greater than the end point or if + /// the end point is greater than the length of the deque. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut deque: VecDeque<_> = [1, 2, 3].into(); + /// for v in deque.range_mut(2..) { + /// *v *= 2; + /// } + /// assert_eq!(deque, [1, 2, 6]); + /// + /// // A full range covers all contents + /// for v in deque.range_mut(..) { + /// *v *= 2; + /// } + /// assert_eq!(deque, [2, 4, 12]); + /// ``` + #[inline] + #[stable(feature = "deque_range", since = "1.51.0")] + pub fn range_mut(&mut self, range: R) -> IterMut<'_, T> + where + R: RangeBounds, + { + let (a_range, b_range) = self.slice_ranges(range); + // SAFETY: The ranges returned by `slice_ranges` + // are valid ranges into the physical buffer, so + // it's ok to pass them to `buffer_range` and + // dereference the result. + let a = unsafe { &mut *self.buffer_range(a_range) }; + let b = unsafe { &mut *self.buffer_range(b_range) }; + IterMut::new(a.iter_mut(), b.iter_mut()) + } + + /// Removes the specified range from the deque in bulk, returning all + /// removed elements as an iterator. If the iterator is dropped before + /// being fully consumed, it drops the remaining removed elements. + /// + /// The returned iterator keeps a mutable borrow on the queue to optimize + /// its implementation. + /// + /// + /// # Panics + /// + /// Panics if the starting point is greater than the end point or if + /// the end point is greater than the length of the deque. + /// + /// # Leaking + /// + /// If the returned iterator goes out of scope without being dropped (due to + /// [`mem::forget`], for example), the deque may have lost and leaked + /// elements arbitrarily, including elements outside the range. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut deque: VecDeque<_> = [1, 2, 3].into(); + /// let drained = deque.drain(2..).collect::>(); + /// assert_eq!(drained, [3]); + /// assert_eq!(deque, [1, 2]); + /// + /// // A full range clears all contents, like `clear()` does + /// deque.drain(..); + /// assert!(deque.is_empty()); + /// ``` + #[inline] + #[stable(feature = "drain", since = "1.6.0")] + pub fn drain(&mut self, range: R) -> Drain<'_, T, A> + where + R: RangeBounds, + { + // Memory safety + // + // When the Drain is first created, the source deque is shortened to + // make sure no uninitialized or moved-from elements are accessible at + // all if the Drain's destructor never gets to run. + // + // Drain will ptr::read out the values to remove. + // When finished, the remaining data will be copied back to cover the hole, + // and the head/tail values will be restored correctly. + // + let Range { start, end } = slice::range(range, ..self.len); + let drain_start = start; + let drain_len = end - start; + + // The deque's elements are parted into three segments: + // * 0 -> drain_start + // * drain_start -> drain_start+drain_len + // * drain_start+drain_len -> self.len + // + // H = self.head; T = self.head+self.len; t = drain_start+drain_len; h = drain_head + // + // We store drain_start as self.len, and drain_len and self.len as + // drain_len and orig_len respectively on the Drain. This also + // truncates the effective array such that if the Drain is leaked, we + // have forgotten about the potentially moved values after the start of + // the drain. + // + // H h t T + // [. . . o o x x o o . . .] + // + // "forget" about the values after the start of the drain until after + // the drain is complete and the Drain destructor is run. + + unsafe { Drain::new(self, drain_start, drain_len) } + } + + /// Clears the deque, removing all values. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut deque = VecDeque::new(); + /// deque.push_back(1); + /// deque.clear(); + /// assert!(deque.is_empty()); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[inline] + pub fn clear(&mut self) { + self.truncate(0); + // Not strictly necessary, but leaves things in a more consistent/predictable state. + self.head = 0; + } + + /// Returns `true` if the deque contains an element equal to the + /// given value. + /// + /// This operation is *O*(*n*). + /// + /// Note that if you have a sorted `VecDeque`, [`binary_search`] may be faster. + /// + /// [`binary_search`]: VecDeque::binary_search + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut deque: VecDeque = VecDeque::new(); + /// + /// deque.push_back(0); + /// deque.push_back(1); + /// + /// assert_eq!(deque.contains(&1), true); + /// assert_eq!(deque.contains(&10), false); + /// ``` + #[stable(feature = "vec_deque_contains", since = "1.12.0")] + pub fn contains(&self, x: &T) -> bool + where + T: PartialEq, + { + let (a, b) = self.as_slices(); + a.contains(x) || b.contains(x) + } + + /// Provides a reference to the front element, or `None` if the deque is + /// empty. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut d = VecDeque::new(); + /// assert_eq!(d.front(), None); + /// + /// d.push_back(1); + /// d.push_back(2); + /// assert_eq!(d.front(), Some(&1)); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn front(&self) -> Option<&T> { + self.get(0) + } + + /// Provides a mutable reference to the front element, or `None` if the + /// deque is empty. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut d = VecDeque::new(); + /// assert_eq!(d.front_mut(), None); + /// + /// d.push_back(1); + /// d.push_back(2); + /// match d.front_mut() { + /// Some(x) => *x = 9, + /// None => (), + /// } + /// assert_eq!(d.front(), Some(&9)); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn front_mut(&mut self) -> Option<&mut T> { + self.get_mut(0) + } + + /// Provides a reference to the back element, or `None` if the deque is + /// empty. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut d = VecDeque::new(); + /// assert_eq!(d.back(), None); + /// + /// d.push_back(1); + /// d.push_back(2); + /// assert_eq!(d.back(), Some(&2)); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn back(&self) -> Option<&T> { + self.get(self.len.wrapping_sub(1)) + } + + /// Provides a mutable reference to the back element, or `None` if the + /// deque is empty. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut d = VecDeque::new(); + /// assert_eq!(d.back(), None); + /// + /// d.push_back(1); + /// d.push_back(2); + /// match d.back_mut() { + /// Some(x) => *x = 9, + /// None => (), + /// } + /// assert_eq!(d.back(), Some(&9)); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn back_mut(&mut self) -> Option<&mut T> { + self.get_mut(self.len.wrapping_sub(1)) + } + + /// Removes the first element and returns it, or `None` if the deque is + /// empty. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut d = VecDeque::new(); + /// d.push_back(1); + /// d.push_back(2); + /// + /// assert_eq!(d.pop_front(), Some(1)); + /// assert_eq!(d.pop_front(), Some(2)); + /// assert_eq!(d.pop_front(), None); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn pop_front(&mut self) -> Option { + if self.is_empty() { + None + } else { + let old_head = self.head; + self.head = self.to_physical_idx(1); + self.len -= 1; + Some(unsafe { self.buffer_read(old_head) }) + } + } + + /// Removes the last element from the deque and returns it, or `None` if + /// it is empty. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf = VecDeque::new(); + /// assert_eq!(buf.pop_back(), None); + /// buf.push_back(1); + /// buf.push_back(3); + /// assert_eq!(buf.pop_back(), Some(3)); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn pop_back(&mut self) -> Option { + if self.is_empty() { + None + } else { + self.len -= 1; + Some(unsafe { self.buffer_read(self.to_physical_idx(self.len)) }) + } + } + + /// Prepends an element to the deque. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut d = VecDeque::new(); + /// d.push_front(1); + /// d.push_front(2); + /// assert_eq!(d.front(), Some(&2)); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn push_front(&mut self, value: T) { + if self.is_full() { + self.grow(); + } + + self.head = self.wrap_sub(self.head, 1); + self.len += 1; + + unsafe { + self.buffer_write(self.head, value); + } + } + + /// Appends an element to the back of the deque. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf = VecDeque::new(); + /// buf.push_back(1); + /// buf.push_back(3); + /// assert_eq!(3, *buf.back().unwrap()); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn push_back(&mut self, value: T) { + if self.is_full() { + self.grow(); + } + + unsafe { self.buffer_write(self.to_physical_idx(self.len), value) } + self.len += 1; + } + + #[inline] + fn is_contiguous(&self) -> bool { + // Do the calculation like this to avoid overflowing if len + head > usize::MAX + self.head <= self.capacity() - self.len + } + + /// Removes an element from anywhere in the deque and returns it, + /// replacing it with the first element. + /// + /// This does not preserve ordering, but is *O*(1). + /// + /// Returns `None` if `index` is out of bounds. + /// + /// Element at index 0 is the front of the queue. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf = VecDeque::new(); + /// assert_eq!(buf.swap_remove_front(0), None); + /// buf.push_back(1); + /// buf.push_back(2); + /// buf.push_back(3); + /// assert_eq!(buf, [1, 2, 3]); + /// + /// assert_eq!(buf.swap_remove_front(2), Some(3)); + /// assert_eq!(buf, [2, 1]); + /// ``` + #[stable(feature = "deque_extras_15", since = "1.5.0")] + pub fn swap_remove_front(&mut self, index: usize) -> Option { + let length = self.len; + if index < length && index != 0 { + self.swap(index, 0); + } else if index >= length { + return None; + } + self.pop_front() + } + + /// Removes an element from anywhere in the deque and returns it, + /// replacing it with the last element. + /// + /// This does not preserve ordering, but is *O*(1). + /// + /// Returns `None` if `index` is out of bounds. + /// + /// Element at index 0 is the front of the queue. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf = VecDeque::new(); + /// assert_eq!(buf.swap_remove_back(0), None); + /// buf.push_back(1); + /// buf.push_back(2); + /// buf.push_back(3); + /// assert_eq!(buf, [1, 2, 3]); + /// + /// assert_eq!(buf.swap_remove_back(0), Some(1)); + /// assert_eq!(buf, [3, 2]); + /// ``` + #[stable(feature = "deque_extras_15", since = "1.5.0")] + pub fn swap_remove_back(&mut self, index: usize) -> Option { + let length = self.len; + if length > 0 && index < length - 1 { + self.swap(index, length - 1); + } else if index >= length { + return None; + } + self.pop_back() + } + + /// Inserts an element at `index` within the deque, shifting all elements + /// with indices greater than or equal to `index` towards the back. + /// + /// Element at index 0 is the front of the queue. + /// + /// # Panics + /// + /// Panics if `index` is greater than deque's length + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut vec_deque = VecDeque::new(); + /// vec_deque.push_back('a'); + /// vec_deque.push_back('b'); + /// vec_deque.push_back('c'); + /// assert_eq!(vec_deque, &['a', 'b', 'c']); + /// + /// vec_deque.insert(1, 'd'); + /// assert_eq!(vec_deque, &['a', 'd', 'b', 'c']); + /// ``` + #[stable(feature = "deque_extras_15", since = "1.5.0")] + pub fn insert(&mut self, index: usize, value: T) { + assert!(index <= self.len(), "index out of bounds"); + if self.is_full() { + self.grow(); + } + + let k = self.len - index; + if k < index { + // `index + 1` can't overflow, because if index was usize::MAX, then either the + // assert would've failed, or the deque would've tried to grow past usize::MAX + // and panicked. + unsafe { + // see `remove()` for explanation why this wrap_copy() call is safe. + self.wrap_copy(self.to_physical_idx(index), self.to_physical_idx(index + 1), k); + self.buffer_write(self.to_physical_idx(index), value); + self.len += 1; + } + } else { + let old_head = self.head; + self.head = self.wrap_sub(self.head, 1); + unsafe { + self.wrap_copy(old_head, self.head, index); + self.buffer_write(self.to_physical_idx(index), value); + self.len += 1; + } + } + } + + /// Removes and returns the element at `index` from the deque. + /// Whichever end is closer to the removal point will be moved to make + /// room, and all the affected elements will be moved to new positions. + /// Returns `None` if `index` is out of bounds. + /// + /// Element at index 0 is the front of the queue. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf = VecDeque::new(); + /// buf.push_back(1); + /// buf.push_back(2); + /// buf.push_back(3); + /// assert_eq!(buf, [1, 2, 3]); + /// + /// assert_eq!(buf.remove(1), Some(2)); + /// assert_eq!(buf, [1, 3]); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn remove(&mut self, index: usize) -> Option { + if self.len <= index { + return None; + } + + let wrapped_idx = self.to_physical_idx(index); + + let elem = unsafe { Some(self.buffer_read(wrapped_idx)) }; + + let k = self.len - index - 1; + // safety: due to the nature of the if-condition, whichever wrap_copy gets called, + // its length argument will be at most `self.len / 2`, so there can't be more than + // one overlapping area. + if k < index { + unsafe { self.wrap_copy(self.wrap_add(wrapped_idx, 1), wrapped_idx, k) }; + self.len -= 1; + } else { + let old_head = self.head; + self.head = self.to_physical_idx(1); + unsafe { self.wrap_copy(old_head, self.head, index) }; + self.len -= 1; + } + + elem + } + + /// Splits the deque into two at the given index. + /// + /// Returns a newly allocated `VecDeque`. `self` contains elements `[0, at)`, + /// and the returned deque contains elements `[at, len)`. + /// + /// Note that the capacity of `self` does not change. + /// + /// Element at index 0 is the front of the queue. + /// + /// # Panics + /// + /// Panics if `at > len`. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf: VecDeque<_> = [1, 2, 3].into(); + /// let buf2 = buf.split_off(1); + /// assert_eq!(buf, [1]); + /// assert_eq!(buf2, [2, 3]); + /// ``` + #[inline] + #[must_use = "use `.truncate()` if you don't need the other half"] + #[stable(feature = "split_off", since = "1.4.0")] + pub fn split_off(&mut self, at: usize) -> Self + where + A: Clone, + { + let len = self.len; + assert!(at <= len, "`at` out of bounds"); + + let other_len = len - at; + let mut other = VecDeque::with_capacity_in(other_len, self.allocator().clone()); + + unsafe { + let (first_half, second_half) = self.as_slices(); + + let first_len = first_half.len(); + let second_len = second_half.len(); + if at < first_len { + // `at` lies in the first half. + let amount_in_first = first_len - at; + + ptr::copy_nonoverlapping(first_half.as_ptr().add(at), other.ptr(), amount_in_first); + + // just take all of the second half. + ptr::copy_nonoverlapping( + second_half.as_ptr(), + other.ptr().add(amount_in_first), + second_len, + ); + } else { + // `at` lies in the second half, need to factor in the elements we skipped + // in the first half. + let offset = at - first_len; + let amount_in_second = second_len - offset; + ptr::copy_nonoverlapping( + second_half.as_ptr().add(offset), + other.ptr(), + amount_in_second, + ); + } + } + + // Cleanup where the ends of the buffers are + self.len = at; + other.len = other_len; + + other + } + + /// Moves all the elements of `other` into `self`, leaving `other` empty. + /// + /// # Panics + /// + /// Panics if the new number of elements in self overflows a `usize`. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf: VecDeque<_> = [1, 2].into(); + /// let mut buf2: VecDeque<_> = [3, 4].into(); + /// buf.append(&mut buf2); + /// assert_eq!(buf, [1, 2, 3, 4]); + /// assert_eq!(buf2, []); + /// ``` + #[inline] + #[stable(feature = "append", since = "1.4.0")] + pub fn append(&mut self, other: &mut Self) { + if T::IS_ZST { + self.len += other.len; + other.len = 0; + other.head = 0; + return; + } + + self.reserve(other.len); + unsafe { + let (left, right) = other.as_slices(); + self.copy_slice(self.to_physical_idx(self.len), left); + // no overflow, because self.capacity() >= old_cap + left.len() >= self.len + left.len() + self.copy_slice(self.to_physical_idx(self.len + left.len()), right); + } + // SAFETY: Update pointers after copying to avoid leaving doppelganger + // in case of panics. + self.len += other.len; + // Now that we own its values, forget everything in `other`. + other.len = 0; + other.head = 0; + } + + /// Retains only the elements specified by the predicate. + /// + /// In other words, remove all elements `e` for which `f(&e)` returns false. + /// This method operates in place, visiting each element exactly once in the + /// original order, and preserves the order of the retained elements. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf = VecDeque::new(); + /// buf.extend(1..5); + /// buf.retain(|&x| x % 2 == 0); + /// assert_eq!(buf, [2, 4]); + /// ``` + /// + /// Because the elements are visited exactly once in the original order, + /// external state may be used to decide which elements to keep. + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf = VecDeque::new(); + /// buf.extend(1..6); + /// + /// let keep = [false, true, true, false, true]; + /// let mut iter = keep.iter(); + /// buf.retain(|_| *iter.next().unwrap()); + /// assert_eq!(buf, [2, 3, 5]); + /// ``` + #[stable(feature = "vec_deque_retain", since = "1.4.0")] + pub fn retain(&mut self, mut f: F) + where + F: FnMut(&T) -> bool, + { + self.retain_mut(|elem| f(elem)); + } + + /// Retains only the elements specified by the predicate. + /// + /// In other words, remove all elements `e` for which `f(&e)` returns false. + /// This method operates in place, visiting each element exactly once in the + /// original order, and preserves the order of the retained elements. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf = VecDeque::new(); + /// buf.extend(1..5); + /// buf.retain_mut(|x| if *x % 2 == 0 { + /// *x += 1; + /// true + /// } else { + /// false + /// }); + /// assert_eq!(buf, [3, 5]); + /// ``` + #[stable(feature = "vec_retain_mut", since = "1.61.0")] + pub fn retain_mut(&mut self, mut f: F) + where + F: FnMut(&mut T) -> bool, + { + let len = self.len; + let mut idx = 0; + let mut cur = 0; + + // Stage 1: All values are retained. + while cur < len { + if !f(&mut self[cur]) { + cur += 1; + break; + } + cur += 1; + idx += 1; + } + // Stage 2: Swap retained value into current idx. + while cur < len { + if !f(&mut self[cur]) { + cur += 1; + continue; + } + + self.swap(idx, cur); + cur += 1; + idx += 1; + } + // Stage 3: Truncate all values after idx. + if cur != idx { + self.truncate(idx); + } + } + + // Double the buffer size. This method is inline(never), so we expect it to only + // be called in cold paths. + // This may panic or abort + #[inline(never)] + fn grow(&mut self) { + // Extend or possibly remove this assertion when valid use-cases for growing the + // buffer without it being full emerge + debug_assert!(self.is_full()); + let old_cap = self.capacity(); + self.buf.reserve_for_push(old_cap); + unsafe { + self.handle_capacity_increase(old_cap); + } + debug_assert!(!self.is_full()); + } + + /// Modifies the deque in-place so that `len()` is equal to `new_len`, + /// either by removing excess elements from the back or by appending + /// elements generated by calling `generator` to the back. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf = VecDeque::new(); + /// buf.push_back(5); + /// buf.push_back(10); + /// buf.push_back(15); + /// assert_eq!(buf, [5, 10, 15]); + /// + /// buf.resize_with(5, Default::default); + /// assert_eq!(buf, [5, 10, 15, 0, 0]); + /// + /// buf.resize_with(2, || unreachable!()); + /// assert_eq!(buf, [5, 10]); + /// + /// let mut state = 100; + /// buf.resize_with(5, || { state += 1; state }); + /// assert_eq!(buf, [5, 10, 101, 102, 103]); + /// ``` + #[stable(feature = "vec_resize_with", since = "1.33.0")] + pub fn resize_with(&mut self, new_len: usize, generator: impl FnMut() -> T) { + let len = self.len; + + if new_len > len { + self.extend(repeat_with(generator).take(new_len - len)) + } else { + self.truncate(new_len); + } + } + + /// Rearranges the internal storage of this deque so it is one contiguous + /// slice, which is then returned. + /// + /// This method does not allocate and does not change the order of the + /// inserted elements. As it returns a mutable slice, this can be used to + /// sort a deque. + /// + /// Once the internal storage is contiguous, the [`as_slices`] and + /// [`as_mut_slices`] methods will return the entire contents of the + /// deque in a single slice. + /// + /// [`as_slices`]: VecDeque::as_slices + /// [`as_mut_slices`]: VecDeque::as_mut_slices + /// + /// # Examples + /// + /// Sorting the content of a deque. + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf = VecDeque::with_capacity(15); + /// + /// buf.push_back(2); + /// buf.push_back(1); + /// buf.push_front(3); + /// + /// // sorting the deque + /// buf.make_contiguous().sort(); + /// assert_eq!(buf.as_slices(), (&[1, 2, 3] as &[_], &[] as &[_])); + /// + /// // sorting it in reverse order + /// buf.make_contiguous().sort_by(|a, b| b.cmp(a)); + /// assert_eq!(buf.as_slices(), (&[3, 2, 1] as &[_], &[] as &[_])); + /// ``` + /// + /// Getting immutable access to the contiguous slice. + /// + /// ```rust + /// use std::collections::VecDeque; + /// + /// let mut buf = VecDeque::new(); + /// + /// buf.push_back(2); + /// buf.push_back(1); + /// buf.push_front(3); + /// + /// buf.make_contiguous(); + /// if let (slice, &[]) = buf.as_slices() { + /// // we can now be sure that `slice` contains all elements of the deque, + /// // while still having immutable access to `buf`. + /// assert_eq!(buf.len(), slice.len()); + /// assert_eq!(slice, &[3, 2, 1] as &[_]); + /// } + /// ``` + #[stable(feature = "deque_make_contiguous", since = "1.48.0")] + pub fn make_contiguous(&mut self) -> &mut [T] { + if T::IS_ZST { + self.head = 0; + } + + if self.is_contiguous() { + unsafe { return slice::from_raw_parts_mut(self.ptr().add(self.head), self.len) } + } + + let &mut Self { head, len, .. } = self; + let ptr = self.ptr(); + let cap = self.capacity(); + + let free = cap - len; + let head_len = cap - head; + let tail = len - head_len; + let tail_len = tail; + + if free >= head_len { + // there is enough free space to copy the head in one go, + // this means that we first shift the tail backwards, and then + // copy the head to the correct position. + // + // from: DEFGH....ABC + // to: ABCDEFGH.... + unsafe { + self.copy(0, head_len, tail_len); + // ...DEFGH.ABC + self.copy_nonoverlapping(head, 0, head_len); + // ABCDEFGH.... + } + + self.head = 0; + } else if free >= tail_len { + // there is enough free space to copy the tail in one go, + // this means that we first shift the head forwards, and then + // copy the tail to the correct position. + // + // from: FGH....ABCDE + // to: ...ABCDEFGH. + unsafe { + self.copy(head, tail, head_len); + // FGHABCDE.... + self.copy_nonoverlapping(0, tail + head_len, tail_len); + // ...ABCDEFGH. + } + + self.head = tail; + } else { + // `free` is smaller than both `head_len` and `tail_len`. + // the general algorithm for this first moves the slices + // right next to each other and then uses `slice::rotate` + // to rotate them into place: + // + // initially: HIJK..ABCDEFG + // step 1: ..HIJKABCDEFG + // step 2: ..ABCDEFGHIJK + // + // or: + // + // initially: FGHIJK..ABCDE + // step 1: FGHIJKABCDE.. + // step 2: ABCDEFGHIJK.. + + // pick the shorter of the 2 slices to reduce the amount + // of memory that needs to be moved around. + if head_len > tail_len { + // tail is shorter, so: + // 1. copy tail forwards + // 2. rotate used part of the buffer + // 3. update head to point to the new beginning (which is just `free`) + + unsafe { + // if there is no free space in the buffer, then the slices are already + // right next to each other and we don't need to move any memory. + if free != 0 { + // because we only move the tail forward as much as there's free space + // behind it, we don't overwrite any elements of the head slice, and + // the slices end up right next to each other. + self.copy(0, free, tail_len); + } + + // We just copied the tail right next to the head slice, + // so all of the elements in the range are initialized + let slice = &mut *self.buffer_range(free..self.capacity()); + + // because the deque wasn't contiguous, we know that `tail_len < self.len == slice.len()`, + // so this will never panic. + slice.rotate_left(tail_len); + + // the used part of the buffer now is `free..self.capacity()`, so set + // `head` to the beginning of that range. + self.head = free; + } + } else { + // head is shorter so: + // 1. copy head backwards + // 2. rotate used part of the buffer + // 3. update head to point to the new beginning (which is the beginning of the buffer) + + unsafe { + // if there is no free space in the buffer, then the slices are already + // right next to each other and we don't need to move any memory. + if free != 0 { + // copy the head slice to lie right behind the tail slice. + self.copy(self.head, tail_len, head_len); + } + + // because we copied the head slice so that both slices lie right + // next to each other, all the elements in the range are initialized. + let slice = &mut *self.buffer_range(0..self.len); + + // because the deque wasn't contiguous, we know that `head_len < self.len == slice.len()` + // so this will never panic. + slice.rotate_right(head_len); + + // the used part of the buffer now is `0..self.len`, so set + // `head` to the beginning of that range. + self.head = 0; + } + } + } + + unsafe { slice::from_raw_parts_mut(ptr.add(self.head), self.len) } + } + + /// Rotates the double-ended queue `mid` places to the left. + /// + /// Equivalently, + /// - Rotates item `mid` into the first position. + /// - Pops the first `mid` items and pushes them to the end. + /// - Rotates `len() - mid` places to the right. + /// + /// # Panics + /// + /// If `mid` is greater than `len()`. Note that `mid == len()` + /// does _not_ panic and is a no-op rotation. + /// + /// # Complexity + /// + /// Takes `*O*(min(mid, len() - mid))` time and no extra space. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf: VecDeque<_> = (0..10).collect(); + /// + /// buf.rotate_left(3); + /// assert_eq!(buf, [3, 4, 5, 6, 7, 8, 9, 0, 1, 2]); + /// + /// for i in 1..10 { + /// assert_eq!(i * 3 % 10, buf[0]); + /// buf.rotate_left(3); + /// } + /// assert_eq!(buf, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// ``` + #[stable(feature = "vecdeque_rotate", since = "1.36.0")] + pub fn rotate_left(&mut self, mid: usize) { + assert!(mid <= self.len()); + let k = self.len - mid; + if mid <= k { + unsafe { self.rotate_left_inner(mid) } + } else { + unsafe { self.rotate_right_inner(k) } + } + } + + /// Rotates the double-ended queue `k` places to the right. + /// + /// Equivalently, + /// - Rotates the first item into position `k`. + /// - Pops the last `k` items and pushes them to the front. + /// - Rotates `len() - k` places to the left. + /// + /// # Panics + /// + /// If `k` is greater than `len()`. Note that `k == len()` + /// does _not_ panic and is a no-op rotation. + /// + /// # Complexity + /// + /// Takes `*O*(min(k, len() - k))` time and no extra space. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf: VecDeque<_> = (0..10).collect(); + /// + /// buf.rotate_right(3); + /// assert_eq!(buf, [7, 8, 9, 0, 1, 2, 3, 4, 5, 6]); + /// + /// for i in 1..10 { + /// assert_eq!(0, buf[i * 3 % 10]); + /// buf.rotate_right(3); + /// } + /// assert_eq!(buf, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// ``` + #[stable(feature = "vecdeque_rotate", since = "1.36.0")] + pub fn rotate_right(&mut self, k: usize) { + assert!(k <= self.len()); + let mid = self.len - k; + if k <= mid { + unsafe { self.rotate_right_inner(k) } + } else { + unsafe { self.rotate_left_inner(mid) } + } + } + + // SAFETY: the following two methods require that the rotation amount + // be less than half the length of the deque. + // + // `wrap_copy` requires that `min(x, capacity() - x) + copy_len <= capacity()`, + // but then `min` is never more than half the capacity, regardless of x, + // so it's sound to call here because we're calling with something + // less than half the length, which is never above half the capacity. + + unsafe fn rotate_left_inner(&mut self, mid: usize) { + debug_assert!(mid * 2 <= self.len()); + unsafe { + self.wrap_copy(self.head, self.to_physical_idx(self.len), mid); + } + self.head = self.to_physical_idx(mid); + } + + unsafe fn rotate_right_inner(&mut self, k: usize) { + debug_assert!(k * 2 <= self.len()); + self.head = self.wrap_sub(self.head, k); + unsafe { + self.wrap_copy(self.to_physical_idx(self.len), self.head, k); + } + } + + /// Binary searches this `VecDeque` for a given element. + /// This behaves similarly to [`contains`] if this `VecDeque` is sorted. + /// + /// If the value is found then [`Result::Ok`] is returned, containing the + /// index of the matching element. If there are multiple matches, then any + /// one of the matches could be returned. If the value is not found then + /// [`Result::Err`] is returned, containing the index where a matching + /// element could be inserted while maintaining sorted order. + /// + /// See also [`binary_search_by`], [`binary_search_by_key`], and [`partition_point`]. + /// + /// [`contains`]: VecDeque::contains + /// [`binary_search_by`]: VecDeque::binary_search_by + /// [`binary_search_by_key`]: VecDeque::binary_search_by_key + /// [`partition_point`]: VecDeque::partition_point + /// + /// # Examples + /// + /// Looks up a series of four elements. The first is found, with a + /// uniquely determined position; the second and third are not + /// found; the fourth could match any position in `[1, 4]`. + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let deque: VecDeque<_> = [0, 1, 1, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55].into(); + /// + /// assert_eq!(deque.binary_search(&13), Ok(9)); + /// assert_eq!(deque.binary_search(&4), Err(7)); + /// assert_eq!(deque.binary_search(&100), Err(13)); + /// let r = deque.binary_search(&1); + /// assert!(matches!(r, Ok(1..=4))); + /// ``` + /// + /// If you want to insert an item to a sorted deque, while maintaining + /// sort order, consider using [`partition_point`]: + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut deque: VecDeque<_> = [0, 1, 1, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55].into(); + /// let num = 42; + /// let idx = deque.partition_point(|&x| x < num); + /// // The above is equivalent to `let idx = deque.binary_search(&num).unwrap_or_else(|x| x);` + /// deque.insert(idx, num); + /// assert_eq!(deque, &[0, 1, 1, 1, 1, 2, 3, 5, 8, 13, 21, 34, 42, 55]); + /// ``` + #[stable(feature = "vecdeque_binary_search", since = "1.54.0")] + #[inline] + pub fn binary_search(&self, x: &T) -> Result + where + T: Ord, + { + self.binary_search_by(|e| e.cmp(x)) + } + + /// Binary searches this `VecDeque` with a comparator function. + /// This behaves similarly to [`contains`] if this `VecDeque` is sorted. + /// + /// The comparator function should implement an order consistent + /// with the sort order of the deque, returning an order code that + /// indicates whether its argument is `Less`, `Equal` or `Greater` + /// than the desired target. + /// + /// If the value is found then [`Result::Ok`] is returned, containing the + /// index of the matching element. If there are multiple matches, then any + /// one of the matches could be returned. If the value is not found then + /// [`Result::Err`] is returned, containing the index where a matching + /// element could be inserted while maintaining sorted order. + /// + /// See also [`binary_search`], [`binary_search_by_key`], and [`partition_point`]. + /// + /// [`contains`]: VecDeque::contains + /// [`binary_search`]: VecDeque::binary_search + /// [`binary_search_by_key`]: VecDeque::binary_search_by_key + /// [`partition_point`]: VecDeque::partition_point + /// + /// # Examples + /// + /// Looks up a series of four elements. The first is found, with a + /// uniquely determined position; the second and third are not + /// found; the fourth could match any position in `[1, 4]`. + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let deque: VecDeque<_> = [0, 1, 1, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55].into(); + /// + /// assert_eq!(deque.binary_search_by(|x| x.cmp(&13)), Ok(9)); + /// assert_eq!(deque.binary_search_by(|x| x.cmp(&4)), Err(7)); + /// assert_eq!(deque.binary_search_by(|x| x.cmp(&100)), Err(13)); + /// let r = deque.binary_search_by(|x| x.cmp(&1)); + /// assert!(matches!(r, Ok(1..=4))); + /// ``` + #[stable(feature = "vecdeque_binary_search", since = "1.54.0")] + pub fn binary_search_by<'a, F>(&'a self, mut f: F) -> Result + where + F: FnMut(&'a T) -> Ordering, + { + let (front, back) = self.as_slices(); + let cmp_back = back.first().map(|elem| f(elem)); + + if let Some(Ordering::Equal) = cmp_back { + Ok(front.len()) + } else if let Some(Ordering::Less) = cmp_back { + back.binary_search_by(f).map(|idx| idx + front.len()).map_err(|idx| idx + front.len()) + } else { + front.binary_search_by(f) + } + } + + /// Binary searches this `VecDeque` with a key extraction function. + /// This behaves similarly to [`contains`] if this `VecDeque` is sorted. + /// + /// Assumes that the deque is sorted by the key, for instance with + /// [`make_contiguous().sort_by_key()`] using the same key extraction function. + /// + /// If the value is found then [`Result::Ok`] is returned, containing the + /// index of the matching element. If there are multiple matches, then any + /// one of the matches could be returned. If the value is not found then + /// [`Result::Err`] is returned, containing the index where a matching + /// element could be inserted while maintaining sorted order. + /// + /// See also [`binary_search`], [`binary_search_by`], and [`partition_point`]. + /// + /// [`contains`]: VecDeque::contains + /// [`make_contiguous().sort_by_key()`]: VecDeque::make_contiguous + /// [`binary_search`]: VecDeque::binary_search + /// [`binary_search_by`]: VecDeque::binary_search_by + /// [`partition_point`]: VecDeque::partition_point + /// + /// # Examples + /// + /// Looks up a series of four elements in a slice of pairs sorted by + /// their second elements. The first is found, with a uniquely + /// determined position; the second and third are not found; the + /// fourth could match any position in `[1, 4]`. + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let deque: VecDeque<_> = [(0, 0), (2, 1), (4, 1), (5, 1), + /// (3, 1), (1, 2), (2, 3), (4, 5), (5, 8), (3, 13), + /// (1, 21), (2, 34), (4, 55)].into(); + /// + /// assert_eq!(deque.binary_search_by_key(&13, |&(a, b)| b), Ok(9)); + /// assert_eq!(deque.binary_search_by_key(&4, |&(a, b)| b), Err(7)); + /// assert_eq!(deque.binary_search_by_key(&100, |&(a, b)| b), Err(13)); + /// let r = deque.binary_search_by_key(&1, |&(a, b)| b); + /// assert!(matches!(r, Ok(1..=4))); + /// ``` + #[stable(feature = "vecdeque_binary_search", since = "1.54.0")] + #[inline] + pub fn binary_search_by_key<'a, B, F>(&'a self, b: &B, mut f: F) -> Result + where + F: FnMut(&'a T) -> B, + B: Ord, + { + self.binary_search_by(|k| f(k).cmp(b)) + } + + /// Returns the index of the partition point according to the given predicate + /// (the index of the first element of the second partition). + /// + /// The deque is assumed to be partitioned according to the given predicate. + /// This means that all elements for which the predicate returns true are at the start of the deque + /// and all elements for which the predicate returns false are at the end. + /// For example, `[7, 15, 3, 5, 4, 12, 6]` is partitioned under the predicate `x % 2 != 0` + /// (all odd numbers are at the start, all even at the end). + /// + /// If the deque is not partitioned, the returned result is unspecified and meaningless, + /// as this method performs a kind of binary search. + /// + /// See also [`binary_search`], [`binary_search_by`], and [`binary_search_by_key`]. + /// + /// [`binary_search`]: VecDeque::binary_search + /// [`binary_search_by`]: VecDeque::binary_search_by + /// [`binary_search_by_key`]: VecDeque::binary_search_by_key + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let deque: VecDeque<_> = [1, 2, 3, 3, 5, 6, 7].into(); + /// let i = deque.partition_point(|&x| x < 5); + /// + /// assert_eq!(i, 4); + /// assert!(deque.iter().take(i).all(|&x| x < 5)); + /// assert!(deque.iter().skip(i).all(|&x| !(x < 5))); + /// ``` + /// + /// If you want to insert an item to a sorted deque, while maintaining + /// sort order: + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut deque: VecDeque<_> = [0, 1, 1, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55].into(); + /// let num = 42; + /// let idx = deque.partition_point(|&x| x < num); + /// deque.insert(idx, num); + /// assert_eq!(deque, &[0, 1, 1, 1, 1, 2, 3, 5, 8, 13, 21, 34, 42, 55]); + /// ``` + #[stable(feature = "vecdeque_binary_search", since = "1.54.0")] + pub fn partition_point

AsyncIterator for Pin

+where + P: DerefMut, + P::Target: AsyncIterator, +{ + type Item = ::Item; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + ::poll_next(self.as_deref_mut(), cx) + } + + fn size_hint(&self) -> (usize, Option) { + (**self).size_hint() + } +} diff --git a/crux-mir/lib/core/src/async_iter/from_iter.rs b/crux-mir/lib/core/src/async_iter/from_iter.rs new file mode 100644 index 000000000..3180187af --- /dev/null +++ b/crux-mir/lib/core/src/async_iter/from_iter.rs @@ -0,0 +1,38 @@ +use crate::pin::Pin; + +use crate::async_iter::AsyncIterator; +use crate::task::{Context, Poll}; + +/// An async iterator that was created from iterator. +/// +/// This async iterator is created by the [`from_iter`] function. +/// See it documentation for more. +/// +/// [`from_iter`]: fn.from_iter.html +#[unstable(feature = "async_iter_from_iter", issue = "81798")] +#[derive(Clone, Debug)] +pub struct FromIter { + iter: I, +} + +#[unstable(feature = "async_iter_from_iter", issue = "81798")] +impl Unpin for FromIter {} + +/// Converts an iterator into an async iterator. +#[unstable(feature = "async_iter_from_iter", issue = "81798")] +pub fn from_iter(iter: I) -> FromIter { + FromIter { iter: iter.into_iter() } +} + +#[unstable(feature = "async_iter_from_iter", issue = "81798")] +impl AsyncIterator for FromIter { + type Item = I::Item; + + fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(self.iter.next()) + } + + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } +} diff --git a/crux-mir/lib/core/src/async_iter/mod.rs b/crux-mir/lib/core/src/async_iter/mod.rs new file mode 100644 index 000000000..0c6f63771 --- /dev/null +++ b/crux-mir/lib/core/src/async_iter/mod.rs @@ -0,0 +1,128 @@ +//! Composable asynchronous iteration. +//! +//! If you've found yourself with an asynchronous collection of some kind, +//! and needed to perform an operation on the elements of said collection, +//! you'll quickly run into 'async iterators'. Async Iterators are heavily used in +//! idiomatic asynchronous Rust code, so it's worth becoming familiar with them. +//! +//! Before explaining more, let's talk about how this module is structured: +//! +//! # Organization +//! +//! This module is largely organized by type: +//! +//! * [Traits] are the core portion: these traits define what kind of async iterators +//! exist and what you can do with them. The methods of these traits are worth +//! putting some extra study time into. +//! * Functions provide some helpful ways to create some basic async iterators. +//! * Structs are often the return types of the various methods on this +//! module's traits. You'll usually want to look at the method that creates +//! the `struct`, rather than the `struct` itself. For more detail about why, +//! see '[Implementing Async Iterator](#implementing-async-iterator)'. +//! +//! [Traits]: #traits +//! +//! That's it! Let's dig into async iterators. +//! +//! # Async Iterators +//! +//! The heart and soul of this module is the [`AsyncIterator`] trait. The core of +//! [`AsyncIterator`] looks like this: +//! +//! ``` +//! # use core::task::{Context, Poll}; +//! # use core::pin::Pin; +//! trait AsyncIterator { +//! type Item; +//! fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll>; +//! } +//! ``` +//! +//! Unlike `Iterator`, `AsyncIterator` makes a distinction between the [`poll_next`] +//! method which is used when implementing an `AsyncIterator`, and a (to-be-implemented) +//! `next` method which is used when consuming an async iterator. Consumers of `AsyncIterator` +//! only need to consider `next`, which when called, returns a future which +//! yields `Option`. +//! +//! The future returned by `next` will yield `Some(Item)` as long as there are +//! elements, and once they've all been exhausted, will yield `None` to indicate +//! that iteration is finished. If we're waiting on something asynchronous to +//! resolve, the future will wait until the async iterator is ready to yield again. +//! +//! Individual async iterators may choose to resume iteration, and so calling `next` +//! again may or may not eventually yield `Some(Item)` again at some point. +//! +//! [`AsyncIterator`]'s full definition includes a number of other methods as well, +//! but they are default methods, built on top of [`poll_next`], and so you get +//! them for free. +//! +//! [`Poll`]: super::task::Poll +//! [`poll_next`]: AsyncIterator::poll_next +//! +//! # Implementing Async Iterator +//! +//! Creating an async iterator of your own involves two steps: creating a `struct` to +//! hold the async iterator's state, and then implementing [`AsyncIterator`] for that +//! `struct`. +//! +//! Let's make an async iterator named `Counter` which counts from `1` to `5`: +//! +//! ```no_run +//! #![feature(async_iterator)] +//! # use core::async_iter::AsyncIterator; +//! # use core::task::{Context, Poll}; +//! # use core::pin::Pin; +//! +//! // First, the struct: +//! +//! /// An async iterator which counts from one to five +//! struct Counter { +//! count: usize, +//! } +//! +//! // we want our count to start at one, so let's add a new() method to help. +//! // This isn't strictly necessary, but is convenient. Note that we start +//! // `count` at zero, we'll see why in `poll_next()`'s implementation below. +//! impl Counter { +//! fn new() -> Counter { +//! Counter { count: 0 } +//! } +//! } +//! +//! // Then, we implement `AsyncIterator` for our `Counter`: +//! +//! impl AsyncIterator for Counter { +//! // we will be counting with usize +//! type Item = usize; +//! +//! // poll_next() is the only required method +//! fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { +//! // Increment our count. This is why we started at zero. +//! self.count += 1; +//! +//! // Check to see if we've finished counting or not. +//! if self.count < 6 { +//! Poll::Ready(Some(self.count)) +//! } else { +//! Poll::Ready(None) +//! } +//! } +//! } +//! ``` +//! +//! # Laziness +//! +//! Async iterators are *lazy*. This means that just creating an async iterator doesn't +//! _do_ a whole lot. Nothing really happens until you call `poll_next`. This is +//! sometimes a source of confusion when creating an async iterator solely for its side +//! effects. The compiler will warn us about this kind of behavior: +//! +//! ```text +//! warning: unused result that must be used: async iterators do nothing unless polled +//! ``` + +mod async_iter; +mod from_iter; + +pub use async_iter::AsyncIterator; +pub use from_iter::{from_iter, FromIter}; diff --git a/crux-mir/lib/core/src/bool.rs b/crux-mir/lib/core/src/bool.rs index 6e0865e86..db1c505ba 100644 --- a/crux-mir/lib/core/src/bool.rs +++ b/crux-mir/lib/core/src/bool.rs @@ -1,36 +1,73 @@ //! impl bool {} -#[lang = "bool"] +use crate::marker::Destruct; + impl bool { - /// Returns `Some(t)` if the `bool` is `true`, or `None` otherwise. + /// Returns `Some(t)` if the `bool` is [`true`](../std/keyword.true.html), + /// or `None` otherwise. + /// + /// Arguments passed to `then_some` are eagerly evaluated; if you are + /// passing the result of a function call, it is recommended to use + /// [`then`], which is lazily evaluated. + /// + /// [`then`]: bool::then /// /// # Examples /// /// ``` - /// #![feature(bool_to_option)] - /// /// assert_eq!(false.then_some(0), None); /// assert_eq!(true.then_some(0), Some(0)); /// ``` - #[unstable(feature = "bool_to_option", issue = "64260")] + /// + /// ``` + /// let mut a = 0; + /// let mut function_with_side_effects = || { a += 1; }; + /// + /// true.then_some(function_with_side_effects()); + /// false.then_some(function_with_side_effects()); + /// + /// // `a` is incremented twice because the value passed to `then_some` is + /// // evaluated eagerly. + /// assert_eq!(a, 2); + /// ``` + #[stable(feature = "bool_to_option", since = "1.62.0")] + #[rustc_const_unstable(feature = "const_bool_to_option", issue = "91917")] #[inline] - pub fn then_some(self, t: T) -> Option { + pub const fn then_some(self, t: T) -> Option + where + T: ~const Destruct, + { if self { Some(t) } else { None } } - /// Returns `Some(f())` if the `bool` is `true`, or `None` otherwise. + /// Returns `Some(f())` if the `bool` is [`true`](../std/keyword.true.html), + /// or `None` otherwise. /// /// # Examples /// /// ``` - /// #![feature(bool_to_option)] - /// /// assert_eq!(false.then(|| 0), None); /// assert_eq!(true.then(|| 0), Some(0)); /// ``` - #[unstable(feature = "bool_to_option", issue = "64260")] + /// + /// ``` + /// let mut a = 0; + /// + /// true.then(|| { a += 1; }); + /// false.then(|| { a += 1; }); + /// + /// // `a` is incremented once because the closure is evaluated lazily by + /// // `then`. + /// assert_eq!(a, 1); + /// ``` + #[stable(feature = "lazy_bool_to_option", since = "1.50.0")] + #[rustc_const_unstable(feature = "const_bool_to_option", issue = "91917")] #[inline] - pub fn then T>(self, f: F) -> Option { + pub const fn then(self, f: F) -> Option + where + F: ~const FnOnce() -> T, + F: ~const Destruct, + { if self { Some(f()) } else { None } } } diff --git a/crux-mir/lib/core/src/borrow.rs b/crux-mir/lib/core/src/borrow.rs index 3e533255b..4a8302ee4 100644 --- a/crux-mir/lib/core/src/borrow.rs +++ b/crux-mir/lib/core/src/borrow.rs @@ -1,4 +1,4 @@ -//! A module for working with borrowed data. +//! Utilities for working with borrowed data. #![stable(feature = "rust1", since = "1.0.0")] @@ -26,7 +26,7 @@ /// to be modified, it can additionally implement [`BorrowMut`]. /// /// Further, when providing implementations for additional traits, it needs -/// to be considered whether they should behave identical to those of the +/// to be considered whether they should behave identically to those of the /// underlying type as a consequence of acting as a representation of that /// underlying type. Generic code typically uses `Borrow` when it relies /// on the identical behavior of these additional trait implementations. @@ -40,14 +40,11 @@ /// provide a reference to related type `T`, it is often better to use /// [`AsRef`] as more types can safely implement it. /// -/// [`AsRef`]: ../../std/convert/trait.AsRef.html -/// [`BorrowMut`]: trait.BorrowMut.html /// [`Box`]: ../../std/boxed/struct.Box.html /// [`Mutex`]: ../../std/sync/struct.Mutex.html /// [`Rc`]: ../../std/rc/struct.Rc.html -/// [`str`]: ../../std/primitive.str.html /// [`String`]: ../../std/string/struct.String.html -/// [`borrow`]: #tymethod.borrow +/// [`borrow`]: Borrow::borrow /// /// # Examples /// @@ -152,11 +149,12 @@ /// If it wants to allow others access to the underlying `str`, it can do /// that via `AsRef` which doesn’t carry any extra requirements. /// -/// [`Hash`]: ../../std/hash/trait.Hash.html +/// [`Hash`]: crate::hash::Hash /// [`HashMap`]: ../../std/collections/struct.HashMap.html /// [`String`]: ../../std/string/struct.String.html -/// [`str`]: ../../std/primitive.str.html #[stable(feature = "rust1", since = "1.0.0")] +#[rustc_diagnostic_item = "Borrow"] +#[const_trait] pub trait Borrow { /// Immutably borrows from an owned value. /// @@ -186,9 +184,8 @@ pub trait Borrow { /// As a companion to [`Borrow`] this trait allows a type to borrow as /// an underlying type by providing a mutable reference. See [`Borrow`] /// for more information on borrowing as another type. -/// -/// [`Borrow`]: trait.Borrow.html #[stable(feature = "rust1", since = "1.0.0")] +#[const_trait] pub trait BorrowMut: Borrow { /// Mutably borrows from an owned value. /// @@ -210,35 +207,41 @@ pub trait BorrowMut: Borrow { } #[stable(feature = "rust1", since = "1.0.0")] -impl Borrow for T { +#[rustc_const_unstable(feature = "const_borrow", issue = "91522")] +impl const Borrow for T { + #[rustc_diagnostic_item = "noop_method_borrow"] fn borrow(&self) -> &T { self } } #[stable(feature = "rust1", since = "1.0.0")] -impl BorrowMut for T { +#[rustc_const_unstable(feature = "const_borrow", issue = "91522")] +impl const BorrowMut for T { fn borrow_mut(&mut self) -> &mut T { self } } #[stable(feature = "rust1", since = "1.0.0")] -impl Borrow for &T { +#[rustc_const_unstable(feature = "const_borrow", issue = "91522")] +impl const Borrow for &T { fn borrow(&self) -> &T { &**self } } #[stable(feature = "rust1", since = "1.0.0")] -impl Borrow for &mut T { +#[rustc_const_unstable(feature = "const_borrow", issue = "91522")] +impl const Borrow for &mut T { fn borrow(&self) -> &T { &**self } } #[stable(feature = "rust1", since = "1.0.0")] -impl BorrowMut for &mut T { +#[rustc_const_unstable(feature = "const_borrow", issue = "91522")] +impl const BorrowMut for &mut T { fn borrow_mut(&mut self) -> &mut T { &mut **self } diff --git a/crux-mir/lib/core/src/cell.rs b/crux-mir/lib/core/src/cell.rs index a922d4f11..129213fde 100644 --- a/crux-mir/lib/core/src/cell.rs +++ b/crux-mir/lib/core/src/cell.rs @@ -11,12 +11,10 @@ //! mutate it. //! //! Shareable mutable containers exist to permit mutability in a controlled manner, even in the -//! presence of aliasing. Both `Cell` and `RefCell` allow doing this in a single-threaded +//! presence of aliasing. Both [`Cell`] and [`RefCell`] allow doing this in a single-threaded //! way. However, neither `Cell` nor `RefCell` are thread safe (they do not implement -//! `Sync`). If you need to do aliasing and mutation between multiple threads it is possible to -//! use [`Mutex`](../../std/sync/struct.Mutex.html), -//! [`RwLock`](../../std/sync/struct.RwLock.html) or -//! [`atomic`](../../core/sync/atomic/index.html) types. +//! [`Sync`]). If you need to do aliasing and mutation between multiple threads it is possible to +//! use [`Mutex`], [`RwLock`] or [`atomic`] types. //! //! Values of the `Cell` and `RefCell` types may be mutated through shared references (i.e. //! the common `&T` type), whereas most Rust types can only be mutated through unique (`&mut T`) @@ -28,13 +26,14 @@ //! one must use the `RefCell` type, acquiring a write lock before mutating. `Cell` provides //! methods to retrieve and change the current interior value: //! -//! - For types that implement `Copy`, the `get` method retrieves the current interior value. -//! - For types that implement `Default`, the `take` method replaces the current interior value -//! with `Default::default()` and returns the replaced value. -//! - For all types, the `replace` method replaces the current interior value and returns the -//! replaced value and the `into_inner` method consumes the `Cell` and returns the interior -//! value. Additionally, the `set` method replaces the interior value, dropping the replaced -//! value. +//! - For types that implement [`Copy`], the [`get`](Cell::get) method retrieves the current +//! interior value. +//! - For types that implement [`Default`], the [`take`](Cell::take) method replaces the current +//! interior value with [`Default::default()`] and returns the replaced value. +//! - For all types, the [`replace`](Cell::replace) method replaces the current interior value and +//! returns the replaced value and the [`into_inner`](Cell::into_inner) method consumes the +//! `Cell` and returns the interior value. Additionally, the [`set`](Cell::set) method +//! replaces the interior value, dropping the replaced value. //! //! `RefCell` uses Rust's lifetimes to implement 'dynamic borrowing', a process whereby one can //! claim temporary, exclusive, mutable access to the inner value. Borrows for `RefCell`s are @@ -54,12 +53,12 @@ //! //! * Introducing mutability 'inside' of something immutable //! * Implementation details of logically-immutable methods. -//! * Mutating implementations of `Clone`. +//! * Mutating implementations of [`Clone`]. //! //! ## Introducing mutability 'inside' of something immutable //! -//! Many shared smart pointer types, including `Rc` and `Arc`, provide containers that can be -//! cloned and shared between multiple parties. Because the contained values may be +//! Many shared smart pointer types, including [`Rc`] and [`Arc`], provide containers that can +//! be cloned and shared between multiple parties. Because the contained values may be //! multiply-aliased, they can only be borrowed with `&`, not `&mut`. Without cells it would be //! impossible to mutate data inside of these smart pointers at all. //! @@ -86,12 +85,12 @@ //! // of scope then the subsequent borrow would cause a dynamic thread panic. //! // This is the major hazard of using `RefCell`. //! let total: i32 = shared_map.borrow().values().sum(); -//! println!("{}", total); +//! println!("{total}"); //! } //! ``` //! //! Note that this example uses `Rc` and not `Arc`. `RefCell`s are for single-threaded -//! scenarios. Consider using `RwLock` or `Mutex` if you need shared mutability in a +//! scenarios. Consider using [`RwLock`] or [`Mutex`] if you need shared mutability in a //! multi-threaded situation. //! //! ## Implementation details of logically-immutable methods @@ -127,16 +126,15 @@ //! ## Mutating implementations of `Clone` //! //! This is simply a special - but common - case of the previous: hiding mutability for operations -//! that appear to be immutable. The `clone` method is expected to not change the source value, and -//! is declared to take `&self`, not `&mut self`. Therefore, any mutation that happens in the -//! `clone` method must use cell types. For example, `Rc` maintains its reference counts within a -//! `Cell`. +//! that appear to be immutable. The [`clone`](Clone::clone) method is expected to not change the +//! source value, and is declared to take `&self`, not `&mut self`. Therefore, any mutation that +//! happens in the `clone` method must use cell types. For example, [`Rc`] maintains its +//! reference counts within a `Cell`. //! //! ``` -//! #![feature(core_intrinsics)] //! use std::cell::Cell; //! use std::ptr::NonNull; -//! use std::intrinsics::abort; +//! use std::process::abort; //! use std::marker::PhantomData; //! //! struct Rc { @@ -173,7 +171,7 @@ //! .strong //! .set(self.strong() //! .checked_add(1) -//! .unwrap_or_else(|| unsafe { abort() })); +//! .unwrap_or_else(|| abort() )); //! } //! } //! @@ -186,15 +184,28 @@ //! } //! ``` //! +//! [`Arc`]: ../../std/sync/struct.Arc.html +//! [`Rc`]: ../../std/rc/struct.Rc.html +//! [`RwLock`]: ../../std/sync/struct.RwLock.html +//! [`Mutex`]: ../../std/sync/struct.Mutex.html +//! [`atomic`]: crate::sync::atomic #![stable(feature = "rust1", since = "1.0.0")] use crate::cmp::Ordering; use crate::fmt::{self, Debug, Display}; -use crate::marker::Unsize; +use crate::marker::{PhantomData, Unsize}; use crate::mem; use crate::ops::{CoerceUnsized, Deref, DerefMut}; -use crate::ptr; +use crate::ptr::{self, NonNull}; + +mod lazy; +mod once; + +#[unstable(feature = "once_cell", issue = "74465")] +pub use lazy::LazyCell; +#[unstable(feature = "once_cell", issue = "74465")] +pub use once::OnceCell; /// A mutable memory location. /// @@ -227,7 +238,7 @@ use crate::ptr; /// assert_eq!(my_struct.special_field.get(), new_value); /// ``` /// -/// See the [module-level documentation](index.html) for more. +/// See the [module-level documentation](self) for more. #[stable(feature = "rust1", since = "1.0.0")] #[repr(transparent)] pub struct Cell { @@ -237,6 +248,11 @@ pub struct Cell { #[stable(feature = "rust1", since = "1.0.0")] unsafe impl Send for Cell where T: Send {} +// Note that this negative impl isn't strictly necessary for correctness, +// as `Cell` wraps `UnsafeCell`, which is itself `!Sync`. +// However, given how important `Cell`'s `!Sync`-ness is, +// having an explicit negative impl is nice for documentation purposes +// and results in nicer error messages. #[stable(feature = "rust1", since = "1.0.0")] impl !Sync for Cell {} @@ -305,7 +321,9 @@ impl Ord for Cell { } #[stable(feature = "cell_from", since = "1.12.0")] -impl From for Cell { +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const From for Cell { + /// Creates a new `Cell` containing the given value. fn from(t: T) -> Cell { Cell::new(t) } @@ -322,7 +340,7 @@ impl Cell { /// let c = Cell::new(5); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_cell_new", since = "1.32.0")] + #[rustc_const_stable(feature = "const_cell_new", since = "1.24.0")] #[inline] pub const fn new(value: T) -> Cell { Cell { value: UnsafeCell::new(value) } @@ -346,7 +364,7 @@ impl Cell { drop(old); } - /// Swaps the values of two Cells. + /// Swaps the values of two `Cell`s. /// Difference with `std::mem::swap` is that this function doesn't require `&mut` reference. /// /// # Examples @@ -375,7 +393,7 @@ impl Cell { } } - /// Replaces the contained value, and returns it. + /// Replaces the contained value with `val`, and returns the old contained value. /// /// # Examples /// @@ -387,6 +405,7 @@ impl Cell { /// assert_eq!(cell.replace(10), 5); /// assert_eq!(cell.get(), 10); /// ``` + #[inline] #[stable(feature = "move_cell", since = "1.17.0")] pub fn replace(&self, val: T) -> T { // SAFETY: This can cause data races if called from a separate thread, @@ -407,7 +426,8 @@ impl Cell { /// assert_eq!(five, 5); /// ``` #[stable(feature = "move_cell", since = "1.17.0")] - pub fn into_inner(self) -> T { + #[rustc_const_unstable(feature = "const_cell_into_inner", issue = "78729")] + pub const fn into_inner(self) -> T { self.value.into_inner() } } @@ -484,6 +504,13 @@ impl Cell { /// This call borrows `Cell` mutably (at compile-time) which guarantees /// that we possess the only reference. /// + /// However be cautious: this method expects `self` to be mutable, which is + /// generally not the case when using a `Cell`. If you require interior + /// mutability by reference, consider using `RefCell` which provides + /// run-time checked mutable borrows through its [`borrow_mut`] method. + /// + /// [`borrow_mut`]: RefCell::borrow_mut() + /// /// # Examples /// /// ``` @@ -497,10 +524,7 @@ impl Cell { #[inline] #[stable(feature = "cell_get_mut", since = "1.11.0")] pub fn get_mut(&mut self) -> &mut T { - // SAFETY: This can cause data races if called from a separate thread, - // but `Cell` is `!Sync` so this won't happen, and `&mut` guarantees - // unique access. - unsafe { &mut *self.value.get() } + self.value.get_mut() } /// Returns a `&Cell` from a `&mut T` @@ -544,7 +568,7 @@ impl Cell { } } -#[unstable(feature = "coerce_unsized", issue = "27732")] +#[unstable(feature = "coerce_unsized", issue = "18598")] impl, U> CoerceUnsized> for Cell {} impl Cell<[T]> { @@ -568,25 +592,59 @@ impl Cell<[T]> { } } +impl Cell<[T; N]> { + /// Returns a `&[Cell; N]` from a `&Cell<[T; N]>` + /// + /// # Examples + /// + /// ``` + /// #![feature(as_array_of_cells)] + /// use std::cell::Cell; + /// + /// let mut array: [i32; 3] = [1, 2, 3]; + /// let cell_array: &Cell<[i32; 3]> = Cell::from_mut(&mut array); + /// let array_cell: &[Cell; 3] = cell_array.as_array_of_cells(); + /// ``` + #[unstable(feature = "as_array_of_cells", issue = "88248")] + pub fn as_array_of_cells(&self) -> &[Cell; N] { + // SAFETY: `Cell` has the same memory layout as `T`. + unsafe { &*(self as *const Cell<[T; N]> as *const [Cell; N]) } + } +} + /// A mutable memory location with dynamically checked borrow rules /// -/// See the [module-level documentation](index.html) for more. +/// See the [module-level documentation](self) for more. +#[cfg_attr(not(test), rustc_diagnostic_item = "RefCell")] #[stable(feature = "rust1", since = "1.0.0")] pub struct RefCell { borrow: Cell, + // Stores the location of the earliest currently active borrow. + // This gets updated whenever we go from having zero borrows + // to having a single borrow. When a borrow occurs, this gets included + // in the generated `BorrowError/`BorrowMutError` + #[cfg(feature = "debug_refcell")] + borrowed_at: Cell>>, value: UnsafeCell, } -/// An error returned by [`RefCell::try_borrow`](struct.RefCell.html#method.try_borrow). +/// An error returned by [`RefCell::try_borrow`]. #[stable(feature = "try_borrow", since = "1.13.0")] +#[non_exhaustive] pub struct BorrowError { - _private: (), + #[cfg(feature = "debug_refcell")] + location: &'static crate::panic::Location<'static>, } #[stable(feature = "try_borrow", since = "1.13.0")] impl Debug for BorrowError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("BorrowError").finish() + let mut builder = f.debug_struct("BorrowError"); + + #[cfg(feature = "debug_refcell")] + builder.field("location", self.location); + + builder.finish() } } @@ -597,16 +655,23 @@ impl Display for BorrowError { } } -/// An error returned by [`RefCell::try_borrow_mut`](struct.RefCell.html#method.try_borrow_mut). +/// An error returned by [`RefCell::try_borrow_mut`]. #[stable(feature = "try_borrow", since = "1.13.0")] +#[non_exhaustive] pub struct BorrowMutError { - _private: (), + #[cfg(feature = "debug_refcell")] + location: &'static crate::panic::Location<'static>, } #[stable(feature = "try_borrow", since = "1.13.0")] impl Debug for BorrowMutError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("BorrowMutError").finish() + let mut builder = f.debug_struct("BorrowMutError"); + + #[cfg(feature = "debug_refcell")] + builder.field("location", self.location); + + builder.finish() } } @@ -654,10 +719,15 @@ impl RefCell { /// let c = RefCell::new(5); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_refcell_new", since = "1.32.0")] + #[rustc_const_stable(feature = "const_refcell_new", since = "1.24.0")] #[inline] pub const fn new(value: T) -> RefCell { - RefCell { value: UnsafeCell::new(value), borrow: Cell::new(UNUSED) } + RefCell { + value: UnsafeCell::new(value), + borrow: Cell::new(UNUSED), + #[cfg(feature = "debug_refcell")] + borrowed_at: Cell::new(None), + } } /// Consumes the `RefCell`, returning the wrapped value. @@ -672,12 +742,11 @@ impl RefCell { /// let five = c.into_inner(); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_cell_into_inner", issue = "78729")] #[inline] - pub fn into_inner(self) -> T { + pub const fn into_inner(self) -> T { // Since this function takes `self` (the `RefCell`) by value, the // compiler statically verifies that it is not currently borrowed. - // Therefore the following assertion is just a `debug_assert!`. - debug_assert!(self.borrow.get() == UNUSED); self.value.into_inner() } @@ -701,6 +770,7 @@ impl RefCell { /// ``` #[inline] #[stable(feature = "refcell_replace", since = "1.24.0")] + #[track_caller] pub fn replace(&self, t: T) -> T { mem::replace(&mut *self.borrow_mut(), t) } @@ -723,6 +793,7 @@ impl RefCell { /// ``` #[inline] #[stable(feature = "refcell_replace_swap", since = "1.35.0")] + #[track_caller] pub fn replace_with T>(&self, f: F) -> T { let mut_borrow = &mut *self.borrow_mut(); let replacement = f(mut_borrow); @@ -736,7 +807,8 @@ impl RefCell { /// /// # Panics /// - /// Panics if the value in either `RefCell` is currently borrowed. + /// Panics if the value in either `RefCell` is currently borrowed, or + /// if `self` and `other` point to the same `RefCell`. /// /// # Examples /// @@ -779,21 +851,17 @@ impl RefCell { /// /// An example of panic: /// - /// ``` + /// ```should_panic /// use std::cell::RefCell; - /// use std::thread; - /// - /// let result = thread::spawn(move || { - /// let c = RefCell::new(5); - /// let m = c.borrow_mut(); /// - /// let b = c.borrow(); // this causes a panic - /// }).join(); + /// let c = RefCell::new(5); /// - /// assert!(result.is_err()); + /// let m = c.borrow_mut(); + /// let b = c.borrow(); // this causes a panic /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] + #[track_caller] pub fn borrow(&self) -> Ref<'_, T> { self.try_borrow().expect("already mutably borrowed") } @@ -825,12 +893,29 @@ impl RefCell { /// ``` #[stable(feature = "try_borrow", since = "1.13.0")] #[inline] + #[cfg_attr(feature = "debug_refcell", track_caller)] pub fn try_borrow(&self) -> Result, BorrowError> { match BorrowRef::new(&self.borrow) { - // SAFETY: `BorrowRef` ensures that there is only immutable access - // to the value while borrowed. - Some(b) => Ok(Ref { value: unsafe { &*self.value.get() }, borrow: b }), - None => Err(BorrowError { _private: () }), + Some(b) => { + #[cfg(feature = "debug_refcell")] + { + // `borrowed_at` is always the *first* active borrow + if b.borrow.get() == 1 { + self.borrowed_at.set(Some(crate::panic::Location::caller())); + } + } + + // SAFETY: `BorrowRef` ensures that there is only immutable access + // to the value while borrowed. + let value = unsafe { NonNull::new_unchecked(self.value.get()) }; + Ok(Ref { value, borrow: b }) + } + None => Err(BorrowError { + // If a borrow occurred, then we must already have an outstanding borrow, + // so `borrowed_at` will be `Some` + #[cfg(feature = "debug_refcell")] + location: self.borrowed_at.get().unwrap(), + }), } } @@ -850,30 +935,26 @@ impl RefCell { /// ``` /// use std::cell::RefCell; /// - /// let c = RefCell::new(5); + /// let c = RefCell::new("hello".to_owned()); /// - /// *c.borrow_mut() = 7; + /// *c.borrow_mut() = "bonjour".to_owned(); /// - /// assert_eq!(*c.borrow(), 7); + /// assert_eq!(&*c.borrow(), "bonjour"); /// ``` /// /// An example of panic: /// - /// ``` + /// ```should_panic /// use std::cell::RefCell; - /// use std::thread; /// - /// let result = thread::spawn(move || { - /// let c = RefCell::new(5); - /// let m = c.borrow(); - /// - /// let b = c.borrow_mut(); // this causes a panic - /// }).join(); + /// let c = RefCell::new(5); + /// let m = c.borrow(); /// - /// assert!(result.is_err()); + /// let b = c.borrow_mut(); // this causes a panic /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] + #[track_caller] pub fn borrow_mut(&self) -> RefMut<'_, T> { self.try_borrow_mut().expect("already borrowed") } @@ -902,11 +983,25 @@ impl RefCell { /// ``` #[stable(feature = "try_borrow", since = "1.13.0")] #[inline] + #[cfg_attr(feature = "debug_refcell", track_caller)] pub fn try_borrow_mut(&self) -> Result, BorrowMutError> { match BorrowRefMut::new(&self.borrow) { - // SAFETY: `BorrowRef` guarantees unique access. - Some(b) => Ok(RefMut { value: unsafe { &mut *self.value.get() }, borrow: b }), - None => Err(BorrowMutError { _private: () }), + Some(b) => { + #[cfg(feature = "debug_refcell")] + { + self.borrowed_at.set(Some(crate::panic::Location::caller())); + } + + // SAFETY: `BorrowRefMut` guarantees unique access. + let value = unsafe { NonNull::new_unchecked(self.value.get()) }; + Ok(RefMut { value, borrow: b, marker: PhantomData }) + } + None => Err(BorrowMutError { + // If a borrow occurred, then we must already have an outstanding borrow, + // so `borrowed_at` will be `Some` + #[cfg(feature = "debug_refcell")] + location: self.borrowed_at.get().unwrap(), + }), } } @@ -929,17 +1024,20 @@ impl RefCell { /// Returns a mutable reference to the underlying data. /// - /// This call borrows `RefCell` mutably (at compile-time) so there is no - /// need for dynamic checks. + /// Since this method borrows `RefCell` mutably, it is statically guaranteed + /// that no borrows to the underlying data exist. The dynamic checks inherent + /// in [`borrow_mut`] and most other methods of `RefCell` are therefore + /// unnecessary. /// - /// However be cautious: this method expects `self` to be mutable, which is - /// generally not the case when using a `RefCell`. Take a look at the - /// [`borrow_mut`] method instead if `self` isn't mutable. + /// This method can only be called if `RefCell` can be mutably borrowed, + /// which in general is only the case directly after the `RefCell` has + /// been created. In these situations, skipping the aforementioned dynamic + /// borrowing checks may yield better ergonomics and runtime-performance. /// - /// Also, please be aware that this method is only for special circumstances and is usually - /// not what you want. In case of doubt, use [`borrow_mut`] instead. + /// In most situations where `RefCell` is used, it can't be borrowed mutably. + /// Use [`borrow_mut`] to get mutable access to the underlying data then. /// - /// [`borrow_mut`]: #method.borrow_mut + /// [`borrow_mut`]: RefCell::borrow_mut() /// /// # Examples /// @@ -954,8 +1052,7 @@ impl RefCell { #[inline] #[stable(feature = "cell_get_mut", since = "1.11.0")] pub fn get_mut(&mut self) -> &mut T { - // SAFETY: `&mut` guarantees unique access. - unsafe { &mut *self.value.get() } + self.value.get_mut() } /// Undo the effect of leaked guards on the borrow state of the `RefCell`. @@ -964,7 +1061,7 @@ impl RefCell { /// ensure no borrows exist and then resets the state tracking shared borrows. This is relevant /// if some `Ref` or `RefMut` borrows have been leaked. /// - /// [`get_mut`]: #method.get_mut + /// [`get_mut`]: RefCell::get_mut() /// /// # Examples /// @@ -1016,13 +1113,47 @@ impl RefCell { #[inline] pub unsafe fn try_borrow_unguarded(&self) -> Result<&T, BorrowError> { if !is_writing(self.borrow.get()) { - Ok(&*self.value.get()) + // SAFETY: We check that nobody is actively writing now, but it is + // the caller's responsibility to ensure that nobody writes until + // the returned reference is no longer in use. + // Also, `self.value.get()` refers to the value owned by `self` + // and is thus guaranteed to be valid for the lifetime of `self`. + Ok(unsafe { &*self.value.get() }) } else { - Err(BorrowError { _private: () }) + Err(BorrowError { + // If a borrow occurred, then we must already have an outstanding borrow, + // so `borrowed_at` will be `Some` + #[cfg(feature = "debug_refcell")] + location: self.borrowed_at.get().unwrap(), + }) } } } +impl RefCell { + /// Takes the wrapped value, leaving `Default::default()` in its place. + /// + /// # Panics + /// + /// Panics if the value is currently borrowed. + /// + /// # Examples + /// + /// ``` + /// use std::cell::RefCell; + /// + /// let c = RefCell::new(5); + /// let five = c.take(); + /// + /// assert_eq!(five, 5); + /// assert_eq!(c.into_inner(), 0); + /// ``` + #[stable(feature = "refcell_take", since = "1.50.0")] + pub fn take(&self) -> T { + self.replace(Default::default()) + } +} + #[stable(feature = "rust1", since = "1.0.0")] unsafe impl Send for RefCell where T: Send {} @@ -1035,9 +1166,19 @@ impl Clone for RefCell { /// /// Panics if the value is currently mutably borrowed. #[inline] + #[track_caller] fn clone(&self) -> RefCell { RefCell::new(self.borrow().clone()) } + + /// # Panics + /// + /// Panics if `other` is currently mutably borrowed. + #[inline] + #[track_caller] + fn clone_from(&mut self, other: &Self) { + self.get_mut().clone_from(&other.borrow()) + } } #[stable(feature = "rust1", since = "1.0.0")] @@ -1053,7 +1194,7 @@ impl Default for RefCell { impl PartialEq for RefCell { /// # Panics /// - /// Panics if the value in either `RefCell` is currently borrowed. + /// Panics if the value in either `RefCell` is currently mutably borrowed. #[inline] fn eq(&self, other: &RefCell) -> bool { *self.borrow() == *other.borrow() @@ -1067,7 +1208,7 @@ impl Eq for RefCell {} impl PartialOrd for RefCell { /// # Panics /// - /// Panics if the value in either `RefCell` is currently borrowed. + /// Panics if the value in either `RefCell` is currently mutably borrowed. #[inline] fn partial_cmp(&self, other: &RefCell) -> Option { self.borrow().partial_cmp(&*other.borrow()) @@ -1075,7 +1216,7 @@ impl PartialOrd for RefCell { /// # Panics /// - /// Panics if the value in either `RefCell` is currently borrowed. + /// Panics if the value in either `RefCell` is currently mutably borrowed. #[inline] fn lt(&self, other: &RefCell) -> bool { *self.borrow() < *other.borrow() @@ -1083,7 +1224,7 @@ impl PartialOrd for RefCell { /// # Panics /// - /// Panics if the value in either `RefCell` is currently borrowed. + /// Panics if the value in either `RefCell` is currently mutably borrowed. #[inline] fn le(&self, other: &RefCell) -> bool { *self.borrow() <= *other.borrow() @@ -1091,7 +1232,7 @@ impl PartialOrd for RefCell { /// # Panics /// - /// Panics if the value in either `RefCell` is currently borrowed. + /// Panics if the value in either `RefCell` is currently mutably borrowed. #[inline] fn gt(&self, other: &RefCell) -> bool { *self.borrow() > *other.borrow() @@ -1099,7 +1240,7 @@ impl PartialOrd for RefCell { /// # Panics /// - /// Panics if the value in either `RefCell` is currently borrowed. + /// Panics if the value in either `RefCell` is currently mutably borrowed. #[inline] fn ge(&self, other: &RefCell) -> bool { *self.borrow() >= *other.borrow() @@ -1110,7 +1251,7 @@ impl PartialOrd for RefCell { impl Ord for RefCell { /// # Panics /// - /// Panics if the value in either `RefCell` is currently borrowed. + /// Panics if the value in either `RefCell` is currently mutably borrowed. #[inline] fn cmp(&self, other: &RefCell) -> Ordering { self.borrow().cmp(&*other.borrow()) @@ -1118,13 +1259,15 @@ impl Ord for RefCell { } #[stable(feature = "cell_from", since = "1.12.0")] -impl From for RefCell { +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const From for RefCell { + /// Creates a new `RefCell` containing the given value. fn from(t: T) -> RefCell { RefCell::new(t) } } -#[unstable(feature = "coerce_unsized", issue = "27732")] +#[unstable(feature = "coerce_unsized", issue = "18598")] impl, U> CoerceUnsized> for RefCell {} struct BorrowRef<'b> { @@ -1139,8 +1282,8 @@ impl<'b> BorrowRef<'b> { // Incrementing borrow can result in a non-reading value (<= 0) in these cases: // 1. It was < 0, i.e. there are writing borrows, so we can't allow a read borrow // due to Rust's reference aliasing rules - // 2. It was isize::max_value() (the max amount of reading borrows) and it overflowed - // into isize::min_value() (the max amount of writing borrows) so we can't allow + // 2. It was isize::MAX (the max amount of reading borrows) and it overflowed + // into isize::MIN (the max amount of writing borrows) so we can't allow // an additional read borrow because isize can't represent so many read borrows // (this can only happen if you mem::forget more than a small constant amount of // `Ref`s, which is not good practice) @@ -1148,7 +1291,7 @@ impl<'b> BorrowRef<'b> { } else { // Incrementing borrow can result in a reading value (> 0) in these cases: // 1. It was = 0, i.e. it wasn't borrowed, and we are taking the first read borrow - // 2. It was > 0 and < isize::max_value(), i.e. there were read borrows, and isize + // 2. It was > 0 and < isize::MAX, i.e. there were read borrows, and isize // is large enough to represent having one more read borrow borrow.set(b); Some(BorrowRef { borrow }) @@ -1174,7 +1317,7 @@ impl Clone for BorrowRef<'_> { debug_assert!(is_reading(borrow)); // Prevent the borrow counter from overflowing into // a writing borrow. - assert!(borrow != isize::max_value()); + assert!(borrow != isize::MAX); self.borrow.set(borrow + 1); BorrowRef { borrow: self.borrow } } @@ -1183,10 +1326,14 @@ impl Clone for BorrowRef<'_> { /// Wraps a borrowed reference to a value in a `RefCell` box. /// A wrapper type for an immutably borrowed value from a `RefCell`. /// -/// See the [module-level documentation](index.html) for more. +/// See the [module-level documentation](self) for more. #[stable(feature = "rust1", since = "1.0.0")] +#[must_not_suspend = "holding a Ref across suspend points can cause BorrowErrors"] pub struct Ref<'b, T: ?Sized + 'b> { - value: &'b T, + // NB: we use a pointer instead of `&'b T` to avoid `noalias` violations, because a + // `Ref` argument doesn't hold immutability for its whole scope, only until it drops. + // `NonNull` is also covariant over `T`, just like we would have with `&T`. + value: NonNull, borrow: BorrowRef<'b>, } @@ -1196,7 +1343,8 @@ impl Deref for Ref<'_, T> { #[inline] fn deref(&self) -> &T { - self.value + // SAFETY: the value is accessible as long as we hold our borrow. + unsafe { self.value.as_ref() } } } @@ -1210,6 +1358,7 @@ impl<'b, T: ?Sized> Ref<'b, T> { /// with the widespread use of `r.borrow().clone()` to clone the contents of /// a `RefCell`. #[stable(feature = "cell_extras", since = "1.15.0")] + #[must_use] #[inline] pub fn clone(orig: &Ref<'b, T>) -> Ref<'b, T> { Ref { value: orig.value, borrow: orig.borrow.clone() } @@ -1239,7 +1388,39 @@ impl<'b, T: ?Sized> Ref<'b, T> { where F: FnOnce(&T) -> &U, { - Ref { value: f(orig.value), borrow: orig.borrow } + Ref { value: NonNull::from(f(&*orig)), borrow: orig.borrow } + } + + /// Makes a new `Ref` for an optional component of the borrowed data. The + /// original guard is returned as an `Err(..)` if the closure returns + /// `None`. + /// + /// The `RefCell` is already immutably borrowed, so this cannot fail. + /// + /// This is an associated function that needs to be used as + /// `Ref::filter_map(...)`. A method would interfere with methods of the same + /// name on the contents of a `RefCell` used through `Deref`. + /// + /// # Examples + /// + /// ``` + /// use std::cell::{RefCell, Ref}; + /// + /// let c = RefCell::new(vec![1, 2, 3]); + /// let b1: Ref> = c.borrow(); + /// let b2: Result, _> = Ref::filter_map(b1, |v| v.get(1)); + /// assert_eq!(*b2.unwrap(), 2); + /// ``` + #[stable(feature = "cell_filter_map", since = "1.63.0")] + #[inline] + pub fn filter_map(orig: Ref<'b, T>, f: F) -> Result, Self> + where + F: FnOnce(&T) -> Option<&U>, + { + match f(&*orig) { + Some(value) => Ok(Ref { value: NonNull::from(value), borrow: orig.borrow }), + None => Err(orig), + } } /// Splits a `Ref` into multiple `Ref`s for different components of the @@ -1268,9 +1449,12 @@ impl<'b, T: ?Sized> Ref<'b, T> { where F: FnOnce(&T) -> (&U, &V), { - let (a, b) = f(orig.value); + let (a, b) = f(&*orig); let borrow = orig.borrow.clone(); - (Ref { value: a, borrow }, Ref { value: b, borrow: orig.borrow }) + ( + Ref { value: NonNull::from(a), borrow }, + Ref { value: NonNull::from(b), borrow: orig.borrow }, + ) } /// Convert into a reference to the underlying data. @@ -1304,17 +1488,18 @@ impl<'b, T: ?Sized> Ref<'b, T> { // unique reference to the borrowed RefCell. No further mutable references can be created // from the original cell. mem::forget(orig.borrow); - orig.value + // SAFETY: after forgetting, we can form a reference for the rest of lifetime `'b`. + unsafe { orig.value.as_ref() } } } -#[unstable(feature = "coerce_unsized", issue = "27732")] +#[unstable(feature = "coerce_unsized", issue = "18598")] impl<'b, T: ?Sized + Unsize, U: ?Sized> CoerceUnsized> for Ref<'b, T> {} #[stable(feature = "std_guard_impls", since = "1.20.0")] impl fmt::Display for Ref<'_, T> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.value.fmt(f) + (**self).fmt(f) } } @@ -1344,13 +1529,58 @@ impl<'b, T: ?Sized> RefMut<'b, T> { /// ``` #[stable(feature = "cell_map", since = "1.8.0")] #[inline] - pub fn map(orig: RefMut<'b, T>, f: F) -> RefMut<'b, U> + pub fn map(mut orig: RefMut<'b, T>, f: F) -> RefMut<'b, U> where F: FnOnce(&mut T) -> &mut U, { - // FIXME(nll-rfc#40): fix borrow-check - let RefMut { value, borrow } = orig; - RefMut { value: f(value), borrow } + let value = NonNull::from(f(&mut *orig)); + RefMut { value, borrow: orig.borrow, marker: PhantomData } + } + + /// Makes a new `RefMut` for an optional component of the borrowed data. The + /// original guard is returned as an `Err(..)` if the closure returns + /// `None`. + /// + /// The `RefCell` is already mutably borrowed, so this cannot fail. + /// + /// This is an associated function that needs to be used as + /// `RefMut::filter_map(...)`. A method would interfere with methods of the + /// same name on the contents of a `RefCell` used through `Deref`. + /// + /// # Examples + /// + /// ``` + /// use std::cell::{RefCell, RefMut}; + /// + /// let c = RefCell::new(vec![1, 2, 3]); + /// + /// { + /// let b1: RefMut> = c.borrow_mut(); + /// let mut b2: Result, _> = RefMut::filter_map(b1, |v| v.get_mut(1)); + /// + /// if let Ok(mut b2) = b2 { + /// *b2 += 2; + /// } + /// } + /// + /// assert_eq!(*c.borrow(), vec![1, 4, 3]); + /// ``` + #[stable(feature = "cell_filter_map", since = "1.63.0")] + #[inline] + pub fn filter_map(mut orig: RefMut<'b, T>, f: F) -> Result, Self> + where + F: FnOnce(&mut T) -> Option<&mut U>, + { + // SAFETY: function holds onto an exclusive reference for the duration + // of its call through `orig`, and the pointer is only de-referenced + // inside of the function call never allowing the exclusive reference to + // escape. + match f(&mut *orig) { + Some(value) => { + Ok(RefMut { value: NonNull::from(value), borrow: orig.borrow, marker: PhantomData }) + } + None => Err(orig), + } } /// Splits a `RefMut` into multiple `RefMut`s for different components of the @@ -1381,15 +1611,18 @@ impl<'b, T: ?Sized> RefMut<'b, T> { #[stable(feature = "refcell_map_split", since = "1.35.0")] #[inline] pub fn map_split( - orig: RefMut<'b, T>, + mut orig: RefMut<'b, T>, f: F, ) -> (RefMut<'b, U>, RefMut<'b, V>) where F: FnOnce(&mut T) -> (&mut U, &mut V), { - let (a, b) = f(orig.value); let borrow = orig.borrow.clone(); - (RefMut { value: a, borrow }, RefMut { value: b, borrow: orig.borrow }) + let (a, b) = f(&mut *orig); + ( + RefMut { value: NonNull::from(a), borrow, marker: PhantomData }, + RefMut { value: NonNull::from(b), borrow: orig.borrow, marker: PhantomData }, + ) } /// Convert into a mutable reference to the underlying data. @@ -1415,14 +1648,15 @@ impl<'b, T: ?Sized> RefMut<'b, T> { /// assert!(cell.try_borrow_mut().is_err()); /// ``` #[unstable(feature = "cell_leak", issue = "69099")] - pub fn leak(orig: RefMut<'b, T>) -> &'b mut T { + pub fn leak(mut orig: RefMut<'b, T>) -> &'b mut T { // By forgetting this BorrowRefMut we ensure that the borrow counter in the RefCell can't // go back to UNUSED within the lifetime `'b`. Resetting the reference tracking state would // require a unique reference to the borrowed RefCell. No further references can be created // from the original cell within that lifetime, making the current borrow the only // reference for the remaining lifetime. mem::forget(orig.borrow); - orig.value + // SAFETY: after forgetting, we can form a reference for the rest of lifetime `'b`. + unsafe { orig.value.as_mut() } } } @@ -1465,7 +1699,7 @@ impl<'b> BorrowRefMut<'b> { let borrow = self.borrow.get(); debug_assert!(is_writing(borrow)); // Prevent the borrow counter from underflowing. - assert!(borrow != isize::min_value()); + assert!(borrow != isize::MIN); self.borrow.set(borrow - 1); BorrowRefMut { borrow: self.borrow } } @@ -1473,11 +1707,16 @@ impl<'b> BorrowRefMut<'b> { /// A wrapper type for a mutably borrowed value from a `RefCell`. /// -/// See the [module-level documentation](index.html) for more. +/// See the [module-level documentation](self) for more. #[stable(feature = "rust1", since = "1.0.0")] +#[must_not_suspend = "holding a RefMut across suspend points can cause BorrowErrors"] pub struct RefMut<'b, T: ?Sized + 'b> { - value: &'b mut T, + // NB: we use a pointer instead of `&'b mut T` to avoid `noalias` violations, because a + // `RefMut` argument doesn't hold exclusivity for its whole scope, only until it drops. + value: NonNull, borrow: BorrowRefMut<'b>, + // `NonNull` is covariant over `T`, so we need to reintroduce invariance. + marker: PhantomData<&'b mut T>, } #[stable(feature = "rust1", since = "1.0.0")] @@ -1486,7 +1725,8 @@ impl Deref for RefMut<'_, T> { #[inline] fn deref(&self) -> &T { - self.value + // SAFETY: the value is accessible as long as we hold our borrow. + unsafe { self.value.as_ref() } } } @@ -1494,48 +1734,62 @@ impl Deref for RefMut<'_, T> { impl DerefMut for RefMut<'_, T> { #[inline] fn deref_mut(&mut self) -> &mut T { - self.value + // SAFETY: the value is accessible as long as we hold our borrow. + unsafe { self.value.as_mut() } } } -#[unstable(feature = "coerce_unsized", issue = "27732")] +#[unstable(feature = "coerce_unsized", issue = "18598")] impl<'b, T: ?Sized + Unsize, U: ?Sized> CoerceUnsized> for RefMut<'b, T> {} #[stable(feature = "std_guard_impls", since = "1.20.0")] impl fmt::Display for RefMut<'_, T> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.value.fmt(f) + (**self).fmt(f) } } /// The core primitive for interior mutability in Rust. /// -/// `UnsafeCell` is a type that wraps some `T` and indicates unsafe interior operations on the -/// wrapped type. Types with an `UnsafeCell` field are considered to have an 'unsafe interior'. -/// The `UnsafeCell` type is the only legal way to obtain aliasable data that is considered -/// mutable. In general, transmuting an `&T` type into an `&mut T` is considered undefined behavior. +/// If you have a reference `&T`, then normally in Rust the compiler performs optimizations based on +/// the knowledge that `&T` points to immutable data. Mutating that data, for example through an +/// alias or by transmuting an `&T` into an `&mut T`, is considered undefined behavior. +/// `UnsafeCell` opts-out of the immutability guarantee for `&T`: a shared reference +/// `&UnsafeCell` may point to data that is being mutated. This is called "interior mutability". +/// +/// All other types that allow internal mutability, such as `Cell` and `RefCell`, internally +/// use `UnsafeCell` to wrap their data. +/// +/// Note that only the immutability guarantee for shared references is affected by `UnsafeCell`. The +/// uniqueness guarantee for mutable references is unaffected. There is *no* legal way to obtain +/// aliasing `&mut`, not even with `UnsafeCell`. /// -/// If you have a reference `&SomeStruct`, then normally in Rust all fields of `SomeStruct` are -/// immutable. The compiler makes optimizations based on the knowledge that `&T` is not mutably -/// aliased or mutated, and that `&mut T` is unique. `UnsafeCell` is the only core language -/// feature to work around the restriction that `&T` may not be mutated. All other types that -/// allow internal mutability, such as `Cell` and `RefCell`, use `UnsafeCell` to wrap their -/// internal data. There is *no* legal way to obtain aliasing `&mut`, not even with `UnsafeCell`. +/// The `UnsafeCell` API itself is technically very simple: [`.get()`] gives you a raw pointer +/// `*mut T` to its contents. It is up to _you_ as the abstraction designer to use that raw pointer +/// correctly. /// -/// The `UnsafeCell` API itself is technically very simple: it gives you a raw pointer `*mut T` to -/// its contents. It is up to _you_ as the abstraction designer to use that raw pointer correctly. +/// [`.get()`]: `UnsafeCell::get` /// /// The precise Rust aliasing rules are somewhat in flux, but the main points are not contentious: /// -/// - If you create a safe reference with lifetime `'a` (either a `&T` or `&mut T` -/// reference) that is accessible by safe code (for example, because you returned it), -/// then you must not access the data in any way that contradicts that reference for the -/// remainder of `'a`. For example, this means that if you take the `*mut T` from an -/// `UnsafeCell` and cast it to an `&T`, then the data in `T` must remain immutable -/// (modulo any `UnsafeCell` data found within `T`, of course) until that reference's -/// lifetime expires. Similarly, if you create a `&mut T` reference that is released to -/// safe code, then you must not access the data within the `UnsafeCell` until that -/// reference expires. +/// - If you create a safe reference with lifetime `'a` (either a `&T` or `&mut T` reference), then +/// you must not access the data in any way that contradicts that reference for the remainder of +/// `'a`. For example, this means that if you take the `*mut T` from an `UnsafeCell` and cast it +/// to an `&T`, then the data in `T` must remain immutable (modulo any `UnsafeCell` data found +/// within `T`, of course) until that reference's lifetime expires. Similarly, if you create a `&mut +/// T` reference that is released to safe code, then you must not access the data within the +/// `UnsafeCell` until that reference expires. +/// +/// - For both `&T` without `UnsafeCell<_>` and `&mut T`, you must also not deallocate the data +/// until the reference expires. As a special exception, given an `&T`, any part of it that is +/// inside an `UnsafeCell<_>` may be deallocated during the lifetime of the reference, after the +/// last time the reference is used (dereferenced or reborrowed). Since you cannot deallocate a part +/// of what a reference points to, this means the memory an `&T` points to can be deallocated only if +/// *every part of it* (including padding) is inside an `UnsafeCell`. +/// +/// However, whenever a `&UnsafeCell` is constructed or dereferenced, it must still point to +/// live memory and the compiler is allowed to insert spurious reads if it can prove that this +/// memory has not yet been deallocated. /// /// - At all times, you must avoid data races. If multiple threads have access to /// the same `UnsafeCell`, then any writes must have a proper happens-before relation to all other @@ -1550,26 +1804,128 @@ impl fmt::Display for RefMut<'_, T> { /// 2. A `&mut T` reference may be released to safe code provided neither other `&mut T` nor `&T` /// co-exist with it. A `&mut T` must always be unique. /// -/// Note that while mutating or mutably aliasing the contents of an `&UnsafeCell` is -/// ok (provided you enforce the invariants some other way), it is still undefined behavior -/// to have multiple `&mut UnsafeCell` aliases. +/// Note that whilst mutating the contents of an `&UnsafeCell` (even while other +/// `&UnsafeCell` references alias the cell) is +/// ok (provided you enforce the above invariants some other way), it is still undefined behavior +/// to have multiple `&mut UnsafeCell` aliases. That is, `UnsafeCell` is a wrapper +/// designed to have a special interaction with _shared_ accesses (_i.e._, through an +/// `&UnsafeCell<_>` reference); there is no magic whatsoever when dealing with _exclusive_ +/// accesses (_e.g._, through an `&mut UnsafeCell<_>`): neither the cell nor the wrapped value +/// may be aliased for the duration of that `&mut` borrow. +/// This is showcased by the [`.get_mut()`] accessor, which is a _safe_ getter that yields +/// a `&mut T`. +/// +/// [`.get_mut()`]: `UnsafeCell::get_mut` +/// +/// # Memory layout +/// +/// `UnsafeCell` has the same in-memory representation as its inner type `T`. A consequence +/// of this guarantee is that it is possible to convert between `T` and `UnsafeCell`. +/// Special care has to be taken when converting a nested `T` inside of an `Outer` type +/// to an `Outer>` type: this is not sound when the `Outer` type enables [niche] +/// optimizations. For example, the type `Option>` is typically 8 bytes large on +/// 64-bit platforms, but the type `Option>>` takes up 16 bytes of space. +/// Therefore this is not a valid conversion, despite `NonNull` and `UnsafeCell>>` +/// having the same memory layout. This is because `UnsafeCell` disables niche optimizations in +/// order to avoid its interior mutability property from spreading from `T` into the `Outer` type, +/// thus this can cause distortions in the type size in these cases. +/// +/// Note that the only valid way to obtain a `*mut T` pointer to the contents of a +/// _shared_ `UnsafeCell` is through [`.get()`] or [`.raw_get()`]. A `&mut T` reference +/// can be obtained by either dereferencing this pointer or by calling [`.get_mut()`] +/// on an _exclusive_ `UnsafeCell`. Even though `T` and `UnsafeCell` have the +/// same memory layout, the following is not allowed and undefined behavior: +/// +/// ```rust,no_run +/// # use std::cell::UnsafeCell; +/// unsafe fn not_allowed(ptr: &UnsafeCell) -> &mut T { +/// let t = ptr as *const UnsafeCell as *mut T; +/// // This is undefined behavior, because the `*mut T` pointer +/// // was not obtained through `.get()` nor `.raw_get()`: +/// unsafe { &mut *t } +/// } +/// ``` +/// +/// Instead, do this: +/// +/// ```rust +/// # use std::cell::UnsafeCell; +/// // Safety: the caller must ensure that there are no references that +/// // point to the *contents* of the `UnsafeCell`. +/// unsafe fn get_mut(ptr: &UnsafeCell) -> &mut T { +/// unsafe { &mut *ptr.get() } +/// } +/// ``` +/// +/// Converting in the other direction from a `&mut T` +/// to an `&UnsafeCell` is allowed: +/// +/// ```rust +/// # use std::cell::UnsafeCell; +/// fn get_shared(ptr: &mut T) -> &UnsafeCell { +/// let t = ptr as *mut T as *const UnsafeCell; +/// // SAFETY: `T` and `UnsafeCell` have the same memory layout +/// unsafe { &*t } +/// } +/// ``` +/// +/// [niche]: https://rust-lang.github.io/unsafe-code-guidelines/glossary.html#niche +/// [`.raw_get()`]: `UnsafeCell::raw_get` /// /// # Examples /// +/// Here is an example showcasing how to soundly mutate the contents of an `UnsafeCell<_>` despite +/// there being multiple references aliasing the cell: +/// /// ``` /// use std::cell::UnsafeCell; /// -/// # #[allow(dead_code)] -/// struct NotThreadSafe { -/// value: UnsafeCell, +/// let x: UnsafeCell = 42.into(); +/// // Get multiple / concurrent / shared references to the same `x`. +/// let (p1, p2): (&UnsafeCell, &UnsafeCell) = (&x, &x); +/// +/// unsafe { +/// // SAFETY: within this scope there are no other references to `x`'s contents, +/// // so ours is effectively unique. +/// let p1_exclusive: &mut i32 = &mut *p1.get(); // -- borrow --+ +/// *p1_exclusive += 27; // | +/// } // <---------- cannot go beyond this point -------------------+ +/// +/// unsafe { +/// // SAFETY: within this scope nobody expects to have exclusive access to `x`'s contents, +/// // so we can have multiple shared accesses concurrently. +/// let p2_shared: &i32 = &*p2.get(); +/// assert_eq!(*p2_shared, 42 + 27); +/// let p1_shared: &i32 = &*p1.get(); +/// assert_eq!(*p1_shared, *p2_shared); /// } +/// ``` +/// +/// The following example showcases the fact that exclusive access to an `UnsafeCell` +/// implies exclusive access to its `T`: +/// +/// ```rust +/// #![forbid(unsafe_code)] // with exclusive accesses, +/// // `UnsafeCell` is a transparent no-op wrapper, +/// // so no need for `unsafe` here. +/// use std::cell::UnsafeCell; /// -/// unsafe impl Sync for NotThreadSafe {} +/// let mut x: UnsafeCell = 42.into(); +/// +/// // Get a compile-time-checked unique reference to `x`. +/// let p_unique: &mut UnsafeCell = &mut x; +/// // With an exclusive reference, we can mutate the contents for free. +/// *p_unique.get_mut() = 0; +/// // Or, equivalently: +/// x = UnsafeCell::new(0); +/// +/// // When we own the value, we can extract the contents for free. +/// let contents: i32 = x.into_inner(); +/// assert_eq!(contents, 0); /// ``` #[lang = "unsafe_cell"] #[stable(feature = "rust1", since = "1.0.0")] #[repr(transparent)] -#[repr(no_niche)] // rust-lang/rust#68303. pub struct UnsafeCell { value: T, } @@ -1581,7 +1937,7 @@ impl UnsafeCell { /// Constructs a new instance of `UnsafeCell` which will wrap the specified /// value. /// - /// All access to the inner value through methods is `unsafe`. + /// All access to the inner value through `&UnsafeCell` requires `unsafe` code. /// /// # Examples /// @@ -1592,7 +1948,7 @@ impl UnsafeCell { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[rustc_const_stable(feature = "const_unsafe_cell_new", since = "1.32.0")] - #[inline] + #[inline(always)] pub const fn new(value: T) -> UnsafeCell { UnsafeCell { value } } @@ -1608,9 +1964,10 @@ impl UnsafeCell { /// /// let five = uc.into_inner(); /// ``` - #[inline] + #[inline(always)] #[stable(feature = "rust1", since = "1.0.0")] - pub fn into_inner(self) -> T { + #[rustc_const_unstable(feature = "const_cell_into_inner", issue = "78729")] + pub const fn into_inner(self) -> T { self.value } } @@ -1632,18 +1989,40 @@ impl UnsafeCell { /// /// let five = uc.get(); /// ``` - #[inline] + #[inline(always)] #[stable(feature = "rust1", since = "1.0.0")] #[rustc_const_stable(feature = "const_unsafecell_get", since = "1.32.0")] pub const fn get(&self) -> *mut T { // We can just cast the pointer from `UnsafeCell` to `T` because of - // #[repr(transparent)]. This exploits libstd's special status, there is + // #[repr(transparent)]. This exploits std's special status, there is // no guarantee for user code that this will work in future versions of the compiler! self as *const UnsafeCell as *const T as *mut T } + /// Returns a mutable reference to the underlying data. + /// + /// This call borrows the `UnsafeCell` mutably (at compile-time) which + /// guarantees that we possess the only reference. + /// + /// # Examples + /// + /// ``` + /// use std::cell::UnsafeCell; + /// + /// let mut c = UnsafeCell::new(5); + /// *c.get_mut() += 1; + /// + /// assert_eq!(*c.get_mut(), 6); + /// ``` + #[inline(always)] + #[stable(feature = "unsafe_cell_get_mut", since = "1.50.0")] + #[rustc_const_unstable(feature = "const_unsafecell_get_mut", issue = "88836")] + pub const fn get_mut(&mut self) -> &mut T { + &mut self.value + } + /// Gets a mutable pointer to the wrapped value. - /// The difference to [`get`] is that this function accepts a raw pointer, + /// The difference from [`get`] is that this function accepts a raw pointer, /// which is useful to avoid the creation of temporary references. /// /// The result can be cast to a pointer of any kind. @@ -1651,7 +2030,7 @@ impl UnsafeCell { /// when casting to `&mut T`, and ensure that there are no mutations /// or mutable aliases going on when casting to `&T`. /// - /// [`get`]: #method.get + /// [`get`]: UnsafeCell::get() /// /// # Examples /// @@ -1659,7 +2038,6 @@ impl UnsafeCell { /// calling `get` would require creating a reference to uninitialized data: /// /// ``` - /// #![feature(unsafe_cell_raw_get)] /// use std::cell::UnsafeCell; /// use std::mem::MaybeUninit; /// @@ -1669,11 +2047,12 @@ impl UnsafeCell { /// /// assert_eq!(uc.into_inner(), 5); /// ``` - #[inline] - #[unstable(feature = "unsafe_cell_raw_get", issue = "66358")] + #[inline(always)] + #[stable(feature = "unsafe_cell_raw_get", since = "1.56.0")] + #[rustc_const_stable(feature = "unsafe_cell_raw_get", since = "1.56.0")] pub const fn raw_get(this: *const Self) -> *mut T { // We can just cast the pointer from `UnsafeCell` to `T` because of - // #[repr(transparent)]. This exploits libstd's special status, there is + // #[repr(transparent)]. This exploits std's special status, there is // no guarantee for user code that this will work in future versions of the compiler! this as *const T as *mut T } @@ -1688,18 +2067,117 @@ impl Default for UnsafeCell { } #[stable(feature = "cell_from", since = "1.12.0")] -impl From for UnsafeCell { +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const From for UnsafeCell { + /// Creates a new `UnsafeCell` containing the given value. fn from(t: T) -> UnsafeCell { UnsafeCell::new(t) } } -#[unstable(feature = "coerce_unsized", issue = "27732")] +#[unstable(feature = "coerce_unsized", issue = "18598")] impl, U> CoerceUnsized> for UnsafeCell {} +/// [`UnsafeCell`], but [`Sync`]. +/// +/// This is just an `UnsafeCell`, except it implements `Sync` +/// if `T` implements `Sync`. +/// +/// `UnsafeCell` doesn't implement `Sync`, to prevent accidental mis-use. +/// You can use `SyncUnsafeCell` instead of `UnsafeCell` to allow it to be +/// shared between threads, if that's intentional. +/// Providing proper synchronization is still the task of the user, +/// making this type just as unsafe to use. +/// +/// See [`UnsafeCell`] for details. +#[unstable(feature = "sync_unsafe_cell", issue = "95439")] +#[repr(transparent)] +pub struct SyncUnsafeCell { + value: UnsafeCell, +} + +#[unstable(feature = "sync_unsafe_cell", issue = "95439")] +unsafe impl Sync for SyncUnsafeCell {} + +#[unstable(feature = "sync_unsafe_cell", issue = "95439")] +impl SyncUnsafeCell { + /// Constructs a new instance of `SyncUnsafeCell` which will wrap the specified value. + #[inline] + pub const fn new(value: T) -> Self { + Self { value: UnsafeCell { value } } + } + + /// Unwraps the value. + #[inline] + pub const fn into_inner(self) -> T { + self.value.into_inner() + } +} + +#[unstable(feature = "sync_unsafe_cell", issue = "95439")] +impl SyncUnsafeCell { + /// Gets a mutable pointer to the wrapped value. + /// + /// This can be cast to a pointer of any kind. + /// Ensure that the access is unique (no active references, mutable or not) + /// when casting to `&mut T`, and ensure that there are no mutations + /// or mutable aliases going on when casting to `&T` + #[inline] + pub const fn get(&self) -> *mut T { + self.value.get() + } + + /// Returns a mutable reference to the underlying data. + /// + /// This call borrows the `SyncUnsafeCell` mutably (at compile-time) which + /// guarantees that we possess the only reference. + #[inline] + pub const fn get_mut(&mut self) -> &mut T { + self.value.get_mut() + } + + /// Gets a mutable pointer to the wrapped value. + /// + /// See [`UnsafeCell::get`] for details. + #[inline] + pub const fn raw_get(this: *const Self) -> *mut T { + // We can just cast the pointer from `SyncUnsafeCell` to `T` because + // of #[repr(transparent)] on both SyncUnsafeCell and UnsafeCell. + // See UnsafeCell::raw_get. + this as *const T as *mut T + } +} + +#[unstable(feature = "sync_unsafe_cell", issue = "95439")] +impl Default for SyncUnsafeCell { + /// Creates an `SyncUnsafeCell`, with the `Default` value for T. + fn default() -> SyncUnsafeCell { + SyncUnsafeCell::new(Default::default()) + } +} + +#[unstable(feature = "sync_unsafe_cell", issue = "95439")] +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const From for SyncUnsafeCell { + /// Creates a new `SyncUnsafeCell` containing the given value. + fn from(t: T) -> SyncUnsafeCell { + SyncUnsafeCell::new(t) + } +} + +#[unstable(feature = "coerce_unsized", issue = "18598")] +//#[unstable(feature = "sync_unsafe_cell", issue = "95439")] +impl, U> CoerceUnsized> for SyncUnsafeCell {} + #[allow(unused)] -fn assert_coerce_unsized(a: UnsafeCell<&i32>, b: Cell<&i32>, c: RefCell<&i32>) { +fn assert_coerce_unsized( + a: UnsafeCell<&i32>, + b: SyncUnsafeCell<&i32>, + c: Cell<&i32>, + d: RefCell<&i32>, +) { let _: UnsafeCell<&dyn Send> = a; - let _: Cell<&dyn Send> = b; - let _: RefCell<&dyn Send> = c; + let _: SyncUnsafeCell<&dyn Send> = b; + let _: Cell<&dyn Send> = c; + let _: RefCell<&dyn Send> = d; } diff --git a/crux-mir/lib/core/src/cell/lazy.rs b/crux-mir/lib/core/src/cell/lazy.rs new file mode 100644 index 000000000..65d12c25c --- /dev/null +++ b/crux-mir/lib/core/src/cell/lazy.rs @@ -0,0 +1,110 @@ +use crate::cell::{Cell, OnceCell}; +use crate::fmt; +use crate::ops::Deref; + +/// A value which is initialized on the first access. +/// +/// For a thread-safe version of this struct, see [`std::sync::LazyLock`]. +/// +/// [`std::sync::LazyLock`]: ../../std/sync/struct.LazyLock.html +/// +/// # Examples +/// +/// ``` +/// #![feature(once_cell)] +/// +/// use std::cell::LazyCell; +/// +/// let lazy: LazyCell = LazyCell::new(|| { +/// println!("initializing"); +/// 92 +/// }); +/// println!("ready"); +/// println!("{}", *lazy); +/// println!("{}", *lazy); +/// +/// // Prints: +/// // ready +/// // initializing +/// // 92 +/// // 92 +/// ``` +#[unstable(feature = "once_cell", issue = "74465")] +pub struct LazyCell T> { + cell: OnceCell, + init: Cell>, +} + +impl T> LazyCell { + /// Creates a new lazy value with the given initializing function. + /// + /// # Examples + /// + /// ``` + /// #![feature(once_cell)] + /// + /// use std::cell::LazyCell; + /// + /// let hello = "Hello, World!".to_string(); + /// + /// let lazy = LazyCell::new(|| hello.to_uppercase()); + /// + /// assert_eq!(&*lazy, "HELLO, WORLD!"); + /// ``` + #[inline] + #[unstable(feature = "once_cell", issue = "74465")] + pub const fn new(init: F) -> LazyCell { + LazyCell { cell: OnceCell::new(), init: Cell::new(Some(init)) } + } + + /// Forces the evaluation of this lazy value and returns a reference to + /// the result. + /// + /// This is equivalent to the `Deref` impl, but is explicit. + /// + /// # Examples + /// + /// ``` + /// #![feature(once_cell)] + /// + /// use std::cell::LazyCell; + /// + /// let lazy = LazyCell::new(|| 92); + /// + /// assert_eq!(LazyCell::force(&lazy), &92); + /// assert_eq!(&*lazy, &92); + /// ``` + #[inline] + #[unstable(feature = "once_cell", issue = "74465")] + pub fn force(this: &LazyCell) -> &T { + this.cell.get_or_init(|| match this.init.take() { + Some(f) => f(), + None => panic!("`Lazy` instance has previously been poisoned"), + }) + } +} + +#[unstable(feature = "once_cell", issue = "74465")] +impl T> Deref for LazyCell { + type Target = T; + #[inline] + fn deref(&self) -> &T { + LazyCell::force(self) + } +} + +#[unstable(feature = "once_cell", issue = "74465")] +impl Default for LazyCell { + /// Creates a new lazy value using `Default` as the initializing function. + #[inline] + fn default() -> LazyCell { + LazyCell::new(T::default) + } +} + +#[unstable(feature = "once_cell", issue = "74465")] +impl fmt::Debug for LazyCell { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Lazy").field("cell", &self.cell).field("init", &"..").finish() + } +} diff --git a/crux-mir/lib/core/src/cell/once.rs b/crux-mir/lib/core/src/cell/once.rs new file mode 100644 index 000000000..7757068a4 --- /dev/null +++ b/crux-mir/lib/core/src/cell/once.rs @@ -0,0 +1,300 @@ +use crate::cell::UnsafeCell; +use crate::fmt; +use crate::mem; + +/// A cell which can be written to only once. +/// +/// Unlike [`RefCell`], a `OnceCell` only provides shared `&T` references to its value. +/// Unlike [`Cell`], a `OnceCell` doesn't require copying or replacing the value to access it. +/// +/// For a thread-safe version of this struct, see [`std::sync::OnceLock`]. +/// +/// [`RefCell`]: crate::cell::RefCell +/// [`Cell`]: crate::cell::Cell +/// [`std::sync::OnceLock`]: ../../std/sync/struct.OnceLock.html +/// +/// # Examples +/// +/// ``` +/// #![feature(once_cell)] +/// +/// use std::cell::OnceCell; +/// +/// let cell = OnceCell::new(); +/// assert!(cell.get().is_none()); +/// +/// let value: &String = cell.get_or_init(|| { +/// "Hello, World!".to_string() +/// }); +/// assert_eq!(value, "Hello, World!"); +/// assert!(cell.get().is_some()); +/// ``` +#[unstable(feature = "once_cell", issue = "74465")] +pub struct OnceCell { + // Invariant: written to at most once. + inner: UnsafeCell>, +} + +impl OnceCell { + /// Creates a new empty cell. + #[inline] + #[must_use] + #[unstable(feature = "once_cell", issue = "74465")] + pub const fn new() -> OnceCell { + OnceCell { inner: UnsafeCell::new(None) } + } + + /// Gets the reference to the underlying value. + /// + /// Returns `None` if the cell is empty. + #[inline] + #[unstable(feature = "once_cell", issue = "74465")] + pub fn get(&self) -> Option<&T> { + // SAFETY: Safe due to `inner`'s invariant + unsafe { &*self.inner.get() }.as_ref() + } + + /// Gets the mutable reference to the underlying value. + /// + /// Returns `None` if the cell is empty. + #[inline] + #[unstable(feature = "once_cell", issue = "74465")] + pub fn get_mut(&mut self) -> Option<&mut T> { + self.inner.get_mut().as_mut() + } + + /// Sets the contents of the cell to `value`. + /// + /// # Errors + /// + /// This method returns `Ok(())` if the cell was empty and `Err(value)` if + /// it was full. + /// + /// # Examples + /// + /// ``` + /// #![feature(once_cell)] + /// + /// use std::cell::OnceCell; + /// + /// let cell = OnceCell::new(); + /// assert!(cell.get().is_none()); + /// + /// assert_eq!(cell.set(92), Ok(())); + /// assert_eq!(cell.set(62), Err(62)); + /// + /// assert!(cell.get().is_some()); + /// ``` + #[inline] + #[unstable(feature = "once_cell", issue = "74465")] + pub fn set(&self, value: T) -> Result<(), T> { + // SAFETY: Safe because we cannot have overlapping mutable borrows + let slot = unsafe { &*self.inner.get() }; + if slot.is_some() { + return Err(value); + } + + // SAFETY: This is the only place where we set the slot, no races + // due to reentrancy/concurrency are possible, and we've + // checked that slot is currently `None`, so this write + // maintains the `inner`'s invariant. + let slot = unsafe { &mut *self.inner.get() }; + *slot = Some(value); + Ok(()) + } + + /// Gets the contents of the cell, initializing it with `f` + /// if the cell was empty. + /// + /// # Panics + /// + /// If `f` panics, the panic is propagated to the caller, and the cell + /// remains uninitialized. + /// + /// It is an error to reentrantly initialize the cell from `f`. Doing + /// so results in a panic. + /// + /// # Examples + /// + /// ``` + /// #![feature(once_cell)] + /// + /// use std::cell::OnceCell; + /// + /// let cell = OnceCell::new(); + /// let value = cell.get_or_init(|| 92); + /// assert_eq!(value, &92); + /// let value = cell.get_or_init(|| unreachable!()); + /// assert_eq!(value, &92); + /// ``` + #[inline] + #[unstable(feature = "once_cell", issue = "74465")] + pub fn get_or_init(&self, f: F) -> &T + where + F: FnOnce() -> T, + { + match self.get_or_try_init(|| Ok::(f())) { + Ok(val) => val, + } + } + + /// Gets the contents of the cell, initializing it with `f` if + /// the cell was empty. If the cell was empty and `f` failed, an + /// error is returned. + /// + /// # Panics + /// + /// If `f` panics, the panic is propagated to the caller, and the cell + /// remains uninitialized. + /// + /// It is an error to reentrantly initialize the cell from `f`. Doing + /// so results in a panic. + /// + /// # Examples + /// + /// ``` + /// #![feature(once_cell)] + /// + /// use std::cell::OnceCell; + /// + /// let cell = OnceCell::new(); + /// assert_eq!(cell.get_or_try_init(|| Err(())), Err(())); + /// assert!(cell.get().is_none()); + /// let value = cell.get_or_try_init(|| -> Result { + /// Ok(92) + /// }); + /// assert_eq!(value, Ok(&92)); + /// assert_eq!(cell.get(), Some(&92)) + /// ``` + #[unstable(feature = "once_cell", issue = "74465")] + pub fn get_or_try_init(&self, f: F) -> Result<&T, E> + where + F: FnOnce() -> Result, + { + if let Some(val) = self.get() { + return Ok(val); + } + /// Avoid inlining the initialization closure into the common path that fetches + /// the already initialized value + #[cold] + fn outlined_call(f: F) -> Result + where + F: FnOnce() -> Result, + { + f() + } + let val = outlined_call(f)?; + // Note that *some* forms of reentrant initialization might lead to + // UB (see `reentrant_init` test). I believe that just removing this + // `assert`, while keeping `set/get` would be sound, but it seems + // better to panic, rather than to silently use an old value. + assert!(self.set(val).is_ok(), "reentrant init"); + Ok(self.get().unwrap()) + } + + /// Consumes the cell, returning the wrapped value. + /// + /// Returns `None` if the cell was empty. + /// + /// # Examples + /// + /// ``` + /// #![feature(once_cell)] + /// + /// use std::cell::OnceCell; + /// + /// let cell: OnceCell = OnceCell::new(); + /// assert_eq!(cell.into_inner(), None); + /// + /// let cell = OnceCell::new(); + /// cell.set("hello".to_string()).unwrap(); + /// assert_eq!(cell.into_inner(), Some("hello".to_string())); + /// ``` + #[inline] + #[unstable(feature = "once_cell", issue = "74465")] + pub fn into_inner(self) -> Option { + // Because `into_inner` takes `self` by value, the compiler statically verifies + // that it is not currently borrowed. So it is safe to move out `Option`. + self.inner.into_inner() + } + + /// Takes the value out of this `OnceCell`, moving it back to an uninitialized state. + /// + /// Has no effect and returns `None` if the `OnceCell` hasn't been initialized. + /// + /// Safety is guaranteed by requiring a mutable reference. + /// + /// # Examples + /// + /// ``` + /// #![feature(once_cell)] + /// + /// use std::cell::OnceCell; + /// + /// let mut cell: OnceCell = OnceCell::new(); + /// assert_eq!(cell.take(), None); + /// + /// let mut cell = OnceCell::new(); + /// cell.set("hello".to_string()).unwrap(); + /// assert_eq!(cell.take(), Some("hello".to_string())); + /// assert_eq!(cell.get(), None); + /// ``` + #[inline] + #[unstable(feature = "once_cell", issue = "74465")] + pub fn take(&mut self) -> Option { + mem::take(self).into_inner() + } +} + +#[unstable(feature = "once_cell", issue = "74465")] +impl Default for OnceCell { + #[inline] + fn default() -> Self { + Self::new() + } +} + +#[unstable(feature = "once_cell", issue = "74465")] +impl fmt::Debug for OnceCell { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.get() { + Some(v) => f.debug_tuple("OnceCell").field(v).finish(), + None => f.write_str("OnceCell(Uninit)"), + } + } +} + +#[unstable(feature = "once_cell", issue = "74465")] +impl Clone for OnceCell { + #[inline] + fn clone(&self) -> OnceCell { + let res = OnceCell::new(); + if let Some(value) = self.get() { + match res.set(value.clone()) { + Ok(()) => (), + Err(_) => unreachable!(), + } + } + res + } +} + +#[unstable(feature = "once_cell", issue = "74465")] +impl PartialEq for OnceCell { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.get() == other.get() + } +} + +#[unstable(feature = "once_cell", issue = "74465")] +impl Eq for OnceCell {} + +#[unstable(feature = "once_cell", issue = "74465")] +impl const From for OnceCell { + /// Creates a new `OnceCell` which already contains the given `value`. + #[inline] + fn from(value: T) -> Self { + OnceCell { inner: UnsafeCell::new(Some(value)) } + } +} diff --git a/crux-mir/lib/core/src/char/convert.rs b/crux-mir/lib/core/src/char/convert.rs index 315020bac..f1a51a550 100644 --- a/crux-mir/lib/core/src/char/convert.rs +++ b/crux-mir/lib/core/src/char/convert.rs @@ -1,109 +1,33 @@ //! Character conversions. +use crate::char::TryFromCharError; use crate::convert::TryFrom; use crate::fmt; use crate::mem::transmute; use crate::str::FromStr; -use super::MAX; - -/// Converts a `u32` to a `char`. -/// -/// Note that all [`char`]s are valid [`u32`]s, and can be cast to one with -/// `as`: -/// -/// ``` -/// let c = '💯'; -/// let i = c as u32; -/// -/// assert_eq!(128175, i); -/// ``` -/// -/// However, the reverse is not true: not all valid [`u32`]s are valid -/// [`char`]s. `from_u32()` will return `None` if the input is not a valid value -/// for a [`char`]. -/// -/// [`char`]: ../../std/primitive.char.html -/// [`u32`]: ../../std/primitive.u32.html -/// -/// For an unsafe version of this function which ignores these checks, see -/// [`from_u32_unchecked`]. -/// -/// [`from_u32_unchecked`]: fn.from_u32_unchecked.html -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// use std::char; -/// -/// let c = char::from_u32(0x2764); -/// -/// assert_eq!(Some('❤'), c); -/// ``` -/// -/// Returning `None` when the input is not a valid [`char`]: -/// -/// ``` -/// use std::char; -/// -/// let c = char::from_u32(0x110000); -/// -/// assert_eq!(None, c); -/// ``` +/// Converts a `u32` to a `char`. See [`char::from_u32`]. +#[must_use] #[inline] -#[stable(feature = "rust1", since = "1.0.0")] -pub fn from_u32(i: u32) -> Option { - char::try_from(i).ok() +pub(super) const fn from_u32(i: u32) -> Option { + // FIXME: once Result::ok is const fn, use it here + match char_try_from_u32(i) { + Ok(c) => Some(c), + Err(_) => None, + } } -/// Converts a `u32` to a `char`, ignoring validity. -/// -/// Note that all [`char`]s are valid [`u32`]s, and can be cast to one with -/// `as`: -/// -/// ``` -/// let c = '💯'; -/// let i = c as u32; -/// -/// assert_eq!(128175, i); -/// ``` -/// -/// However, the reverse is not true: not all valid [`u32`]s are valid -/// [`char`]s. `from_u32_unchecked()` will ignore this, and blindly cast to -/// [`char`], possibly creating an invalid one. -/// -/// [`char`]: ../../std/primitive.char.html -/// [`u32`]: ../../std/primitive.u32.html -/// -/// # Safety -/// -/// This function is unsafe, as it may construct invalid `char` values. -/// -/// For a safe version of this function, see the [`from_u32`] function. -/// -/// [`from_u32`]: fn.from_u32.html -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// use std::char; -/// -/// let c = unsafe { char::from_u32_unchecked(0x2764) }; -/// -/// assert_eq!('❤', c); -/// ``` +/// Converts a `u32` to a `char`, ignoring validity. See [`char::from_u32_unchecked`]. #[inline] -#[stable(feature = "char_from_unchecked", since = "1.5.0")] -pub unsafe fn from_u32_unchecked(i: u32) -> char { - transmute(i) +#[must_use] +pub(super) const unsafe fn from_u32_unchecked(i: u32) -> char { + // SAFETY: the caller must guarantee that `i` is a valid char value. + if cfg!(debug_assertions) { char::from_u32(i).unwrap() } else { unsafe { transmute(i) } } } #[stable(feature = "char_convert", since = "1.13.0")] -impl From for u32 { +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const From for u32 { /// Converts a [`char`] into a [`u32`]. /// /// # Examples @@ -121,6 +45,64 @@ impl From for u32 { } } +#[stable(feature = "more_char_conversions", since = "1.51.0")] +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const From for u64 { + /// Converts a [`char`] into a [`u64`]. + /// + /// # Examples + /// + /// ``` + /// use std::mem; + /// + /// let c = '👤'; + /// let u = u64::from(c); + /// assert!(8 == mem::size_of_val(&u)) + /// ``` + #[inline] + fn from(c: char) -> Self { + // The char is casted to the value of the code point, then zero-extended to 64 bit. + // See [https://doc.rust-lang.org/reference/expressions/operator-expr.html#semantics] + c as u64 + } +} + +#[stable(feature = "more_char_conversions", since = "1.51.0")] +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const From for u128 { + /// Converts a [`char`] into a [`u128`]. + /// + /// # Examples + /// + /// ``` + /// use std::mem; + /// + /// let c = '⚙'; + /// let u = u128::from(c); + /// assert!(16 == mem::size_of_val(&u)) + /// ``` + #[inline] + fn from(c: char) -> Self { + // The char is casted to the value of the code point, then zero-extended to 128 bit. + // See [https://doc.rust-lang.org/reference/expressions/operator-expr.html#semantics] + c as u128 + } +} + +/// Map `char` with code point in U+0000..=U+00FF to byte in 0x00..=0xFF with same value, failing +/// if the code point is greater than U+00FF. +/// +/// See [`impl From for char`](char#impl-From-for-char) for details on the encoding. +#[stable(feature = "u8_from_char", since = "1.59.0")] +impl TryFrom for u8 { + type Error = TryFromCharError; + + #[inline] + fn try_from(c: char) -> Result { + u8::try_from(u32::from(c)).map_err(|_| TryFromCharError(())) + } +} + /// Maps a byte in 0x00..=0xFF to a `char` whose code point has the same value, in U+0000..=U+00FF. /// /// Unicode is designed such that this effectively decodes bytes @@ -140,7 +122,8 @@ impl From for u32 { /// for a superset of Windows-1252 that fills the remaining blanks with corresponding /// C0 and C1 control codes. #[stable(feature = "char_convert", since = "1.13.0")] -impl From for char { +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const From for char { /// Converts a [`u8`] into a [`char`]. /// /// # Examples @@ -159,6 +142,8 @@ impl From for char { } /// An error which can be returned when parsing a char. +/// +/// This `struct` is created when using the [`char::from_str`] method. #[stable(feature = "char_from_str", since = "1.20.0")] #[derive(Clone, Debug, PartialEq, Eq)] pub struct ParseCharError { @@ -208,22 +193,43 @@ impl FromStr for char { } } +#[inline] +const fn char_try_from_u32(i: u32) -> Result { + // This is an optimized version of the check + // (i > MAX as u32) || (i >= 0xD800 && i <= 0xDFFF), + // which can also be written as + // i >= 0x110000 || (i >= 0xD800 && i < 0xE000). + // + // The XOR with 0xD800 permutes the ranges such that 0xD800..0xE000 is + // mapped to 0x0000..0x0800, while keeping all the high bits outside 0xFFFF the same. + // In particular, numbers >= 0x110000 stay in this range. + // + // Subtracting 0x800 causes 0x0000..0x0800 to wrap, meaning that a single + // unsigned comparison against 0x110000 - 0x800 will detect both the wrapped + // surrogate range as well as the numbers originally larger than 0x110000. + // + if (i ^ 0xD800).wrapping_sub(0x800) >= 0x110000 - 0x800 { + Err(CharTryFromError(())) + } else { + // SAFETY: checked that it's a legal unicode value + Ok(unsafe { transmute(i) }) + } +} + #[stable(feature = "try_from", since = "1.34.0")] impl TryFrom for char { type Error = CharTryFromError; #[inline] fn try_from(i: u32) -> Result { - if (i > MAX as u32) || (i >= 0xD800 && i <= 0xDFFF) { - Err(CharTryFromError(())) - } else { - // SAFETY: checked that it's a legal unicode value - Ok(unsafe { from_u32_unchecked(i) }) - } + char_try_from_u32(i) } } -/// The error type returned when a conversion from u32 to char fails. +/// The error type returned when a conversion from [`prim@u32`] to [`prim@char`] fails. +/// +/// This `struct` is created by the [`char::try_from`](char#impl-TryFrom-for-char) method. +/// See its documentation for more. #[stable(feature = "try_from", since = "1.34.0")] #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub struct CharTryFromError(()); @@ -235,63 +241,10 @@ impl fmt::Display for CharTryFromError { } } -/// Converts a digit in the given radix to a `char`. -/// -/// A 'radix' here is sometimes also called a 'base'. A radix of two -/// indicates a binary number, a radix of ten, decimal, and a radix of -/// sixteen, hexadecimal, to give some common values. Arbitrary -/// radices are supported. -/// -/// `from_digit()` will return `None` if the input is not a digit in -/// the given radix. -/// -/// # Panics -/// -/// Panics if given a radix larger than 36. -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// use std::char; -/// -/// let c = char::from_digit(4, 10); -/// -/// assert_eq!(Some('4'), c); -/// -/// // Decimal 11 is a single digit in base 16 -/// let c = char::from_digit(11, 16); -/// -/// assert_eq!(Some('b'), c); -/// ``` -/// -/// Returning `None` when the input is not a digit: -/// -/// ``` -/// use std::char; -/// -/// let c = char::from_digit(20, 10); -/// -/// assert_eq!(None, c); -/// ``` -/// -/// Passing a large radix, causing a panic: -/// -/// ``` -/// use std::thread; -/// use std::char; -/// -/// let result = thread::spawn(|| { -/// // this panics -/// let c = char::from_digit(1, 37); -/// }).join(); -/// -/// assert!(result.is_err()); -/// ``` +/// Converts a digit in the given radix to a `char`. See [`char::from_digit`]. #[inline] -#[stable(feature = "rust1", since = "1.0.0")] -pub fn from_digit(num: u32, radix: u32) -> Option { +#[must_use] +pub(super) const fn from_digit(num: u32, radix: u32) -> Option { if radix > 36 { panic!("from_digit: radix is too high (maximum 36)"); } diff --git a/crux-mir/lib/core/src/char/decode.rs b/crux-mir/lib/core/src/char/decode.rs index 5e7784730..eeb088030 100644 --- a/crux-mir/lib/core/src/char/decode.rs +++ b/crux-mir/lib/core/src/char/decode.rs @@ -1,10 +1,16 @@ //! UTF-8 and UTF-16 decoding iterators +use crate::error::Error; use crate::fmt; use super::from_u32_unchecked; /// An iterator that decodes UTF-16 encoded code points from an iterator of `u16`s. +/// +/// This `struct` is created by the [`decode_utf16`] method on [`char`]. See its +/// documentation for more. +/// +/// [`decode_utf16`]: char::decode_utf16 #[stable(feature = "decode_utf16", since = "1.9.0")] #[derive(Clone, Debug)] pub struct DecodeUtf16 @@ -16,6 +22,8 @@ where } /// An error that can be returned when decoding UTF-16 code points. +/// +/// This `struct` is created when using the [`DecodeUtf16`] type. #[stable(feature = "decode_utf16", since = "1.9.0")] #[derive(Debug, Clone, Eq, PartialEq)] pub struct DecodeUtf16Error { @@ -23,54 +31,9 @@ pub struct DecodeUtf16Error { } /// Creates an iterator over the UTF-16 encoded code points in `iter`, -/// returning unpaired surrogates as `Err`s. -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// use std::char::decode_utf16; -/// -/// // 𝄞music -/// let v = [ -/// 0xD834, 0xDD1E, 0x006d, 0x0075, 0x0073, 0xDD1E, 0x0069, 0x0063, 0xD834, -/// ]; -/// -/// assert_eq!( -/// decode_utf16(v.iter().cloned()) -/// .map(|r| r.map_err(|e| e.unpaired_surrogate())) -/// .collect::>(), -/// vec![ -/// Ok('𝄞'), -/// Ok('m'), Ok('u'), Ok('s'), -/// Err(0xDD1E), -/// Ok('i'), Ok('c'), -/// Err(0xD834) -/// ] -/// ); -/// ``` -/// -/// A lossy decoder can be obtained by replacing `Err` results with the replacement character: -/// -/// ``` -/// use std::char::{decode_utf16, REPLACEMENT_CHARACTER}; -/// -/// // 𝄞music -/// let v = [ -/// 0xD834, 0xDD1E, 0x006d, 0x0075, 0x0073, 0xDD1E, 0x0069, 0x0063, 0xD834, -/// ]; -/// -/// assert_eq!( -/// decode_utf16(v.iter().cloned()) -/// .map(|r| r.unwrap_or(REPLACEMENT_CHARACTER)) -/// .collect::(), -/// "𝄞mus�ic�" -/// ); -/// ``` -#[stable(feature = "decode_utf16", since = "1.9.0")] +/// returning unpaired surrogates as `Err`s. See [`char::decode_utf16`]. #[inline] -pub fn decode_utf16>(iter: I) -> DecodeUtf16 { +pub(super) fn decode_utf16>(iter: I) -> DecodeUtf16 { DecodeUtf16 { iter: iter.into_iter(), buf: None } } @@ -84,7 +47,7 @@ impl> Iterator for DecodeUtf16 { None => self.iter.next()?, }; - if u < 0xD800 || 0xDFFF < u { + if !u.is_utf16_surrogate() { // SAFETY: not a surrogate Some(Ok(unsafe { from_u32_unchecked(u as u32) })) } else if u >= 0xDC00 { @@ -104,7 +67,7 @@ impl> Iterator for DecodeUtf16 { } // all ok, so lets decode it. - let c = (((u - 0xD800) as u32) << 10 | (u2 - 0xDC00) as u32) + 0x1_0000; + let c = (((u & 0x3ff) as u32) << 10 | (u2 & 0x3ff) as u32) + 0x1_0000; // SAFETY: we checked that it's a legal unicode value Some(Ok(unsafe { from_u32_unchecked(c) })) } @@ -113,14 +76,40 @@ impl> Iterator for DecodeUtf16 { #[inline] fn size_hint(&self) -> (usize, Option) { let (low, high) = self.iter.size_hint(); - // we could be entirely valid surrogates (2 elements per - // char), or entirely non-surrogates (1 element per char) - (low / 2, high) + + let (low_buf, high_buf) = match self.buf { + // buf is empty, no additional elements from it. + None => (0, 0), + // `u` is a non surrogate, so it's always an additional character. + Some(u) if !u.is_utf16_surrogate() => (1, 1), + // `u` is a leading surrogate (it can never be a trailing surrogate and + // it's a surrogate due to the previous branch) and `self.iter` is empty. + // + // `u` can't be paired, since the `self.iter` is empty, + // so it will always become an additional element (error). + Some(_u) if high == Some(0) => (1, 1), + // `u` is a leading surrogate and `iter` may be non-empty. + // + // `u` can either pair with a trailing surrogate, in which case no additional elements + // are produced, or it can become an error, in which case it's an additional character (error). + Some(_u) => (0, 1), + }; + + // `self.iter` could contain entirely valid surrogates (2 elements per + // char), or entirely non-surrogates (1 element per char). + // + // On odd lower bound, at least one element must stay unpaired + // (with other elements from `self.iter`), so we round up. + let low = low.div_ceil(2) + low_buf; + let high = high.and_then(|h| h.checked_add(high_buf)); + + (low, high) } } impl DecodeUtf16Error { /// Returns the unpaired surrogate which caused this error. + #[must_use] #[stable(feature = "decode_utf16", since = "1.9.0")] pub fn unpaired_surrogate(&self) -> u16 { self.code @@ -133,3 +122,11 @@ impl fmt::Display for DecodeUtf16Error { write!(f, "unpaired surrogate found: {:x}", self.code) } } + +#[stable(feature = "decode_utf16", since = "1.9.0")] +impl Error for DecodeUtf16Error { + #[allow(deprecated)] + fn description(&self) -> &str { + "unpaired surrogate found" + } +} diff --git a/crux-mir/lib/core/src/char/methods.rs b/crux-mir/lib/core/src/char/methods.rs index 302400744..3e7383b4c 100644 --- a/crux-mir/lib/core/src/char/methods.rs +++ b/crux-mir/lib/core/src/char/methods.rs @@ -7,8 +7,247 @@ use crate::unicode::{self, conversions}; use super::*; -#[lang = "char"] impl char { + /// The highest valid code point a `char` can have, `'\u{10FFFF}'`. + /// + /// # Examples + /// + /// ``` + /// # fn something_which_returns_char() -> char { 'a' } + /// let c: char = something_which_returns_char(); + /// assert!(c <= char::MAX); + /// + /// let value_at_max = char::MAX as u32; + /// assert_eq!(char::from_u32(value_at_max), Some('\u{10FFFF}')); + /// assert_eq!(char::from_u32(value_at_max + 1), None); + /// ``` + #[stable(feature = "assoc_char_consts", since = "1.52.0")] + pub const MAX: char = '\u{10ffff}'; + + /// `U+FFFD REPLACEMENT CHARACTER` (�) is used in Unicode to represent a + /// decoding error. + /// + /// It can occur, for example, when giving ill-formed UTF-8 bytes to + /// [`String::from_utf8_lossy`](../std/string/struct.String.html#method.from_utf8_lossy). + #[stable(feature = "assoc_char_consts", since = "1.52.0")] + pub const REPLACEMENT_CHARACTER: char = '\u{FFFD}'; + + /// The version of [Unicode](https://www.unicode.org/) that the Unicode parts of + /// `char` and `str` methods are based on. + /// + /// New versions of Unicode are released regularly and subsequently all methods + /// in the standard library depending on Unicode are updated. Therefore the + /// behavior of some `char` and `str` methods and the value of this constant + /// changes over time. This is *not* considered to be a breaking change. + /// + /// The version numbering scheme is explained in + /// [Unicode 11.0 or later, Section 3.1 Versions of the Unicode Standard](https://www.unicode.org/versions/Unicode11.0.0/ch03.pdf#page=4). + #[stable(feature = "assoc_char_consts", since = "1.52.0")] + pub const UNICODE_VERSION: (u8, u8, u8) = crate::unicode::UNICODE_VERSION; + + /// Creates an iterator over the UTF-16 encoded code points in `iter`, + /// returning unpaired surrogates as `Err`s. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use std::char::decode_utf16; + /// + /// // 𝄞music + /// let v = [ + /// 0xD834, 0xDD1E, 0x006d, 0x0075, 0x0073, 0xDD1E, 0x0069, 0x0063, 0xD834, + /// ]; + /// + /// assert_eq!( + /// decode_utf16(v) + /// .map(|r| r.map_err(|e| e.unpaired_surrogate())) + /// .collect::>(), + /// vec![ + /// Ok('𝄞'), + /// Ok('m'), Ok('u'), Ok('s'), + /// Err(0xDD1E), + /// Ok('i'), Ok('c'), + /// Err(0xD834) + /// ] + /// ); + /// ``` + /// + /// A lossy decoder can be obtained by replacing `Err` results with the replacement character: + /// + /// ``` + /// use std::char::{decode_utf16, REPLACEMENT_CHARACTER}; + /// + /// // 𝄞music + /// let v = [ + /// 0xD834, 0xDD1E, 0x006d, 0x0075, 0x0073, 0xDD1E, 0x0069, 0x0063, 0xD834, + /// ]; + /// + /// assert_eq!( + /// decode_utf16(v) + /// .map(|r| r.unwrap_or(REPLACEMENT_CHARACTER)) + /// .collect::(), + /// "𝄞mus�ic�" + /// ); + /// ``` + #[stable(feature = "assoc_char_funcs", since = "1.52.0")] + #[inline] + pub fn decode_utf16>(iter: I) -> DecodeUtf16 { + super::decode::decode_utf16(iter) + } + + /// Converts a `u32` to a `char`. + /// + /// Note that all `char`s are valid [`u32`]s, and can be cast to one with + /// [`as`](../std/keyword.as.html): + /// + /// ``` + /// let c = '💯'; + /// let i = c as u32; + /// + /// assert_eq!(128175, i); + /// ``` + /// + /// However, the reverse is not true: not all valid [`u32`]s are valid + /// `char`s. `from_u32()` will return `None` if the input is not a valid value + /// for a `char`. + /// + /// For an unsafe version of this function which ignores these checks, see + /// [`from_u32_unchecked`]. + /// + /// [`from_u32_unchecked`]: #method.from_u32_unchecked + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use std::char; + /// + /// let c = char::from_u32(0x2764); + /// + /// assert_eq!(Some('❤'), c); + /// ``` + /// + /// Returning `None` when the input is not a valid `char`: + /// + /// ``` + /// use std::char; + /// + /// let c = char::from_u32(0x110000); + /// + /// assert_eq!(None, c); + /// ``` + #[stable(feature = "assoc_char_funcs", since = "1.52.0")] + #[rustc_const_stable(feature = "const_char_convert", since = "1.67.0")] + #[must_use] + #[inline] + pub const fn from_u32(i: u32) -> Option { + super::convert::from_u32(i) + } + + /// Converts a `u32` to a `char`, ignoring validity. + /// + /// Note that all `char`s are valid [`u32`]s, and can be cast to one with + /// `as`: + /// + /// ``` + /// let c = '💯'; + /// let i = c as u32; + /// + /// assert_eq!(128175, i); + /// ``` + /// + /// However, the reverse is not true: not all valid [`u32`]s are valid + /// `char`s. `from_u32_unchecked()` will ignore this, and blindly cast to + /// `char`, possibly creating an invalid one. + /// + /// # Safety + /// + /// This function is unsafe, as it may construct invalid `char` values. + /// + /// For a safe version of this function, see the [`from_u32`] function. + /// + /// [`from_u32`]: #method.from_u32 + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use std::char; + /// + /// let c = unsafe { char::from_u32_unchecked(0x2764) }; + /// + /// assert_eq!('❤', c); + /// ``` + #[stable(feature = "assoc_char_funcs", since = "1.52.0")] + #[rustc_const_unstable(feature = "const_char_from_u32_unchecked", issue = "89259")] + #[must_use] + #[inline] + pub const unsafe fn from_u32_unchecked(i: u32) -> char { + // SAFETY: the safety contract must be upheld by the caller. + unsafe { super::convert::from_u32_unchecked(i) } + } + + /// Converts a digit in the given radix to a `char`. + /// + /// A 'radix' here is sometimes also called a 'base'. A radix of two + /// indicates a binary number, a radix of ten, decimal, and a radix of + /// sixteen, hexadecimal, to give some common values. Arbitrary + /// radices are supported. + /// + /// `from_digit()` will return `None` if the input is not a digit in + /// the given radix. + /// + /// # Panics + /// + /// Panics if given a radix larger than 36. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use std::char; + /// + /// let c = char::from_digit(4, 10); + /// + /// assert_eq!(Some('4'), c); + /// + /// // Decimal 11 is a single digit in base 16 + /// let c = char::from_digit(11, 16); + /// + /// assert_eq!(Some('b'), c); + /// ``` + /// + /// Returning `None` when the input is not a digit: + /// + /// ``` + /// use std::char; + /// + /// let c = char::from_digit(20, 10); + /// + /// assert_eq!(None, c); + /// ``` + /// + /// Passing a large radix, causing a panic: + /// + /// ```should_panic + /// use std::char; + /// + /// // this panics + /// let _c = char::from_digit(1, 37); + /// ``` + #[stable(feature = "assoc_char_funcs", since = "1.52.0")] + #[rustc_const_stable(feature = "const_char_convert", since = "1.67.0")] + #[must_use] + #[inline] + pub const fn from_digit(num: u32, radix: u32) -> Option { + super::convert::from_digit(num, radix) + } + /// Checks if a `char` is a digit in the given radix. /// /// A 'radix' here is sometimes also called a 'base'. A radix of two @@ -16,7 +255,7 @@ impl char { /// sixteen, hexadecimal, to give some common values. Arbitrary /// radices are supported. /// - /// Compared to `is_numeric()`, this function only recognizes the characters + /// Compared to [`is_numeric()`], this function only recognizes the characters /// `0-9`, `a-z` and `A-Z`. /// /// 'Digit' is defined to be only the following characters: @@ -25,9 +264,9 @@ impl char { /// * `a-z` /// * `A-Z` /// - /// For a more comprehensive understanding of 'digit', see [`is_numeric`][is_numeric]. + /// For a more comprehensive understanding of 'digit', see [`is_numeric()`]. /// - /// [is_numeric]: #method.is_numeric + /// [`is_numeric()`]: #method.is_numeric /// /// # Panics /// @@ -45,15 +284,9 @@ impl char { /// /// Passing a large radix, causing a panic: /// - /// ``` - /// use std::thread; - /// - /// let result = thread::spawn(|| { - /// // this panics - /// '1'.is_digit(37); - /// }).join(); - /// - /// assert!(result.is_err()); + /// ```should_panic + /// // this panics + /// '1'.is_digit(37); /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -100,37 +333,28 @@ impl char { /// /// Passing a large radix, causing a panic: /// - /// ``` - /// use std::thread; - /// - /// let result = thread::spawn(|| { - /// '1'.to_digit(37); - /// }).join(); - /// - /// assert!(result.is_err()); + /// ```should_panic + /// // this panics + /// let _ = '1'.to_digit(37); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_char_convert", since = "1.67.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[inline] - pub fn to_digit(self, radix: u32) -> Option { - assert!(radix <= 36, "to_digit: radix is too high (maximum 36)"); - - // the code is split up here to improve execution speed for cases where - // the `radix` is constant and 10 or smaller - let val = if radix <= 10 { - match self { - '0'..='9' => self as u32 - '0' as u32, - _ => return None, - } - } else { - match self { - '0'..='9' => self as u32 - '0' as u32, - 'a'..='z' => self as u32 - 'a' as u32 + 10, - 'A'..='Z' => self as u32 - 'A' as u32 + 10, - _ => return None, + pub const fn to_digit(self, radix: u32) -> Option { + // If not a digit, a number greater than radix will be created. + let mut digit = (self as u32).wrapping_sub('0' as u32); + if radix > 10 { + assert!(radix <= 36, "to_digit: radix is too high (maximum 36)"); + if digit < 10 { + return Some(digit); } - }; - - if val < radix { Some(val) } else { None } + // Force the 6th bit to be set to ensure ascii is lower case. + digit = (self as u32 | 0b10_0000).wrapping_sub('a' as u32).saturating_add(10); + } + // FIXME: once then_some is const fn, use it here + if digit < radix { Some(digit) } else { None } } /// Returns an iterator that yields the hexadecimal Unicode escape of a @@ -145,7 +369,7 @@ impl char { /// /// ``` /// for c in '❤'.escape_unicode() { - /// print!("{}", c); + /// print!("{c}"); /// } /// println!(); /// ``` @@ -162,11 +386,13 @@ impl char { /// println!("\\u{{2764}}"); /// ``` /// - /// Using `to_string`: + /// Using [`to_string`](../std/string/trait.ToString.html#tymethod.to_string): /// /// ``` /// assert_eq!('❤'.escape_unicode().to_string(), "\\u{2764}"); /// ``` + #[must_use = "this returns the escaped char as an iterator, \ + without modifying the original"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn escape_unicode(self) -> EscapeUnicode { @@ -187,16 +413,21 @@ impl char { } /// An extended version of `escape_debug` that optionally permits escaping - /// Extended Grapheme codepoints. This allows us to format characters like - /// nonspacing marks better when they're at the start of a string. + /// Extended Grapheme codepoints, single quotes, and double quotes. This + /// allows us to format characters like nonspacing marks better when they're + /// at the start of a string, and allows escaping single quotes in + /// characters, and double quotes in strings. #[inline] - pub(crate) fn escape_debug_ext(self, escape_grapheme_extended: bool) -> EscapeDebug { + pub(crate) fn escape_debug_ext(self, args: EscapeDebugExtArgs) -> EscapeDebug { let init_state = match self { + '\0' => EscapeDefaultState::Backslash('0'), '\t' => EscapeDefaultState::Backslash('t'), '\r' => EscapeDefaultState::Backslash('r'), '\n' => EscapeDefaultState::Backslash('n'), - '\\' | '\'' | '"' => EscapeDefaultState::Backslash(self), - _ if escape_grapheme_extended && self.is_grapheme_extended() => { + '\\' => EscapeDefaultState::Backslash(self), + '"' if args.escape_double_quote => EscapeDefaultState::Backslash(self), + '\'' if args.escape_single_quote => EscapeDefaultState::Backslash(self), + _ if args.escape_grapheme_extended && self.is_grapheme_extended() => { EscapeDefaultState::Unicode(self.escape_unicode()) } _ if is_printable(self) => EscapeDefaultState::Char(self), @@ -208,7 +439,7 @@ impl char { /// Returns an iterator that yields the literal escape code of a character /// as `char`s. /// - /// This will escape the characters similar to the `Debug` implementations + /// This will escape the characters similar to the [`Debug`](core::fmt::Debug) implementations /// of `str` or `char`. /// /// # Examples @@ -217,7 +448,7 @@ impl char { /// /// ``` /// for c in '\n'.escape_debug() { - /// print!("{}", c); + /// print!("{c}"); /// } /// println!(); /// ``` @@ -234,15 +465,17 @@ impl char { /// println!("\\n"); /// ``` /// - /// Using `to_string`: + /// Using [`to_string`](../std/string/trait.ToString.html#tymethod.to_string): /// /// ``` /// assert_eq!('\n'.escape_debug().to_string(), "\\n"); /// ``` + #[must_use = "this returns the escaped char as an iterator, \ + without modifying the original"] #[stable(feature = "char_escape_debug", since = "1.20.0")] #[inline] pub fn escape_debug(self) -> EscapeDebug { - self.escape_debug_ext(true) + self.escape_debug_ext(EscapeDebugExtArgs::ESCAPE_ALL) } /// Returns an iterator that yields the literal escape code of a character @@ -261,9 +494,9 @@ impl char { /// * Any character in the 'printable ASCII' range `0x20` .. `0x7e` /// inclusive is not escaped. /// * All other characters are given hexadecimal Unicode escapes; see - /// [`escape_unicode`][escape_unicode]. + /// [`escape_unicode`]. /// - /// [escape_unicode]: #method.escape_unicode + /// [`escape_unicode`]: #method.escape_unicode /// /// # Examples /// @@ -271,7 +504,7 @@ impl char { /// /// ``` /// for c in '"'.escape_default() { - /// print!("{}", c); + /// print!("{c}"); /// } /// println!(); /// ``` @@ -282,18 +515,19 @@ impl char { /// println!("{}", '"'.escape_default()); /// ``` /// - /// /// Both are equivalent to: /// /// ``` /// println!("\\\""); /// ``` /// - /// Using `to_string`: + /// Using [`to_string`](../std/string/trait.ToString.html#tymethod.to_string): /// /// ``` /// assert_eq!('"'.escape_default().to_string(), "\\\""); /// ``` + #[must_use = "this returns the escaped char as an iterator, \ + without modifying the original"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn escape_default(self) -> EscapeDefault { @@ -354,27 +588,24 @@ impl char { /// assert_eq!(len, tokyo.len()); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_char_len_utf", since = "1.52.0")] #[inline] - pub fn len_utf8(self) -> usize { - let code = self as u32; - if code < MAX_ONE_B { - 1 - } else if code < MAX_TWO_B { - 2 - } else if code < MAX_THREE_B { - 3 - } else { - 4 - } + pub const fn len_utf8(self) -> usize { + len_utf8(self as u32) } /// Returns the number of 16-bit code units this `char` would need if /// encoded in UTF-16. /// - /// See the documentation for [`len_utf8`] for more explanation of this + /// That number of code units is always either 1 or 2, for unicode scalar values in + /// the [basic multilingual plane] or [supplementary planes] respectively. + /// + /// See the documentation for [`len_utf8()`] for more explanation of this /// concept. This function is a mirror, but for UTF-16 instead of UTF-8. /// - /// [`len_utf8`]: #method.len_utf8 + /// [basic multilingual plane]: http://www.unicode.org/glossary/#basic_multilingual_plane + /// [supplementary planes]: http://www.unicode.org/glossary/#supplementary_planes + /// [`len_utf8()`]: #method.len_utf8 /// /// # Examples /// @@ -388,8 +619,9 @@ impl char { /// assert_eq!(len, 2); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_char_len_utf", since = "1.52.0")] #[inline] - pub fn len_utf16(self) -> usize { + pub const fn len_utf16(self) -> usize { let ch = self as u32; if (ch & 0xFFFF) == ch { 1 } else { 2 } } @@ -418,51 +650,17 @@ impl char { /// /// A buffer that's too small: /// - /// ``` - /// use std::thread; - /// - /// let result = thread::spawn(|| { - /// let mut b = [0; 1]; - /// - /// // this panics - /// 'ß'.encode_utf8(&mut b); - /// }).join(); + /// ```should_panic + /// let mut b = [0; 1]; /// - /// assert!(result.is_err()); + /// // this panics + /// 'ß'.encode_utf8(&mut b); /// ``` #[stable(feature = "unicode_encode_char", since = "1.15.0")] #[inline] pub fn encode_utf8(self, dst: &mut [u8]) -> &mut str { - let code = self as u32; - let len = self.len_utf8(); - match (len, &mut dst[..]) { - (1, [a, ..]) => { - *a = code as u8; - } - (2, [a, b, ..]) => { - *a = (code >> 6 & 0x1F) as u8 | TAG_TWO_B; - *b = (code & 0x3F) as u8 | TAG_CONT; - } - (3, [a, b, c, ..]) => { - *a = (code >> 12 & 0x0F) as u8 | TAG_THREE_B; - *b = (code >> 6 & 0x3F) as u8 | TAG_CONT; - *c = (code & 0x3F) as u8 | TAG_CONT; - } - (4, [a, b, c, d, ..]) => { - *a = (code >> 18 & 0x07) as u8 | TAG_FOUR_B; - *b = (code >> 12 & 0x3F) as u8 | TAG_CONT; - *c = (code >> 6 & 0x3F) as u8 | TAG_CONT; - *d = (code & 0x3F) as u8 | TAG_CONT; - } - _ => panic!( - "encode_utf8: need {} bytes to encode U+{:X}, but the buffer has {}", - len, - code, - dst.len(), - ), - }; - // SAFETY: We just wrote UTF-8 content in, so converting to str is fine. - unsafe { from_utf8_unchecked_mut(&mut dst[..len]) } + // SAFETY: `char` is not a surrogate, so this is valid UTF-8. + unsafe { from_utf8_unchecked_mut(encode_utf8_raw(self as u32, dst)) } } /// Encodes this character as UTF-16 into the provided `u16` buffer, @@ -487,43 +685,16 @@ impl char { /// /// A buffer that's too small: /// - /// ``` - /// use std::thread; - /// - /// let result = thread::spawn(|| { - /// let mut b = [0; 1]; - /// - /// // this panics - /// '𝕊'.encode_utf16(&mut b); - /// }).join(); + /// ```should_panic + /// let mut b = [0; 1]; /// - /// assert!(result.is_err()); + /// // this panics + /// '𝕊'.encode_utf16(&mut b); /// ``` #[stable(feature = "unicode_encode_char", since = "1.15.0")] #[inline] pub fn encode_utf16(self, dst: &mut [u16]) -> &mut [u16] { - let mut code = self as u32; - // SAFETY: each arm checks whether there are enough bits to write into - unsafe { - if (code & 0xFFFF) == code && !dst.is_empty() { - // The BMP falls through (assuming non-surrogate, as it should) - *dst.get_unchecked_mut(0) = code as u16; - slice::from_raw_parts_mut(dst.as_mut_ptr(), 1) - } else if dst.len() >= 2 { - // Supplementary planes break into surrogates. - code -= 0x1_0000; - *dst.get_unchecked_mut(0) = 0xD800 | ((code >> 10) as u16); - *dst.get_unchecked_mut(1) = 0xDC00 | ((code as u16) & 0x3FF); - slice::from_raw_parts_mut(dst.as_mut_ptr(), 2) - } else { - panic!( - "encode_utf16: need {} units to encode U+{:X}, but the buffer has {}", - from_u32_unchecked(code).len_utf16(), - code, - dst.len(), - ) - } - } + encode_utf16_raw(self as u32, dst) } /// Returns `true` if this `char` has the `Alphabetic` property. @@ -547,6 +718,7 @@ impl char { /// // love is many things, but it is not alphabetic /// assert!(!c.is_alphabetic()); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn is_alphabetic(self) -> bool { @@ -575,12 +747,23 @@ impl char { /// assert!(!'A'.is_lowercase()); /// assert!(!'Δ'.is_lowercase()); /// - /// // The various Chinese scripts do not have case, and so: + /// // The various Chinese scripts and punctuation do not have case, and so: /// assert!(!'中'.is_lowercase()); + /// assert!(!' '.is_lowercase()); + /// ``` + /// + /// In a const context: + /// /// ``` + /// #![feature(const_unicode_case_lookup)] + /// const CAPITAL_DELTA_IS_LOWERCASE: bool = 'Δ'.is_lowercase(); + /// assert!(!CAPITAL_DELTA_IS_LOWERCASE); + /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_unicode_case_lookup", issue = "101400")] #[inline] - pub fn is_lowercase(self) -> bool { + pub const fn is_lowercase(self) -> bool { match self { 'a'..='z' => true, c => c > '\x7f' && unicode::Lowercase(c), @@ -606,12 +789,23 @@ impl char { /// assert!('A'.is_uppercase()); /// assert!('Δ'.is_uppercase()); /// - /// // The various Chinese scripts do not have case, and so: + /// // The various Chinese scripts and punctuation do not have case, and so: /// assert!(!'中'.is_uppercase()); + /// assert!(!' '.is_uppercase()); /// ``` + /// + /// In a const context: + /// + /// ``` + /// #![feature(const_unicode_case_lookup)] + /// const CAPITAL_DELTA_IS_UPPERCASE: bool = 'Δ'.is_uppercase(); + /// assert!(CAPITAL_DELTA_IS_UPPERCASE); + /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_unicode_case_lookup", issue = "101400")] #[inline] - pub fn is_uppercase(self) -> bool { + pub const fn is_uppercase(self) -> bool { match self { 'A'..='Z' => true, c => c > '\x7f' && unicode::Uppercase(c), @@ -632,11 +826,15 @@ impl char { /// ``` /// assert!(' '.is_whitespace()); /// + /// // line break + /// assert!('\n'.is_whitespace()); + /// /// // a non-breaking space /// assert!('\u{A0}'.is_whitespace()); /// /// assert!(!'越'.is_whitespace()); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn is_whitespace(self) -> bool { @@ -665,6 +863,7 @@ impl char { /// assert!('و'.is_alphanumeric()); /// assert!('藏'.is_alphanumeric()); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn is_alphanumeric(self) -> bool { @@ -690,6 +889,7 @@ impl char { /// assert!('œ'.is_control()); /// assert!(!'q'.is_control()); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn is_control(self) -> bool { @@ -705,6 +905,7 @@ impl char { /// [uax29]: https://www.unicode.org/reports/tr29/ /// [ucd]: https://www.unicode.org/reports/tr44/ /// [`DerivedCoreProperties.txt`]: https://www.unicode.org/Public/UCD/latest/ucd/DerivedCoreProperties.txt + #[must_use] #[inline] pub(crate) fn is_grapheme_extended(self) -> bool { unicode::Grapheme_Extend(self) @@ -716,6 +917,14 @@ impl char { /// characters, and `No` for other numeric characters) are specified in the [Unicode Character /// Database][ucd] [`UnicodeData.txt`]. /// + /// This method doesn't cover everything that could be considered a number, e.g. ideographic numbers like '三'. + /// If you want everything including characters with overlapping purposes then you might want to use + /// a unicode or language-processing library that exposes the appropriate character properties instead + /// of looking at the unicode categories. + /// + /// If you want to parse ASCII decimal digits (0-9) or ASCII base-N, use + /// `is_ascii_digit` or `is_digit` instead. + /// /// [Unicode Standard]: https://www.unicode.org/versions/latest/ /// [ucd]: https://www.unicode.org/reports/tr44/ /// [`UnicodeData.txt`]: https://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt @@ -733,7 +942,9 @@ impl char { /// assert!(!'K'.is_numeric()); /// assert!(!'و'.is_numeric()); /// assert!(!'藏'.is_numeric()); + /// assert!(!'三'.is_numeric()); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn is_numeric(self) -> bool { @@ -773,7 +984,7 @@ impl char { /// /// ``` /// for c in 'İ'.to_lowercase() { - /// print!("{}", c); + /// print!("{c}"); /// } /// println!(); /// ``` @@ -790,7 +1001,7 @@ impl char { /// println!("i\u{307}"); /// ``` /// - /// Using `to_string`: + /// Using [`to_string`](../std/string/trait.ToString.html#tymethod.to_string): /// /// ``` /// assert_eq!('C'.to_lowercase().to_string(), "c"); @@ -802,6 +1013,8 @@ impl char { /// // convert into themselves. /// assert_eq!('山'.to_lowercase().to_string(), "山"); /// ``` + #[must_use = "this returns the lowercase character as a new iterator, \ + without modifying the original"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn to_lowercase(self) -> ToLowercase { @@ -811,7 +1024,7 @@ impl char { /// Returns an iterator that yields the uppercase mapping of this `char` as one or more /// `char`s. /// - /// If this `char` does not have a uppercase mapping, the iterator yields the same `char`. + /// If this `char` does not have an uppercase mapping, the iterator yields the same `char`. /// /// If this `char` has a one-to-one uppercase mapping given by the [Unicode Character /// Database][ucd] [`UnicodeData.txt`], the iterator yields that `char`. @@ -838,7 +1051,7 @@ impl char { /// /// ``` /// for c in 'ß'.to_uppercase() { - /// print!("{}", c); + /// print!("{c}"); /// } /// println!(); /// ``` @@ -855,7 +1068,7 @@ impl char { /// println!("SS"); /// ``` /// - /// Using `to_string`: + /// Using [`to_string`](../std/string/trait.ToString.html#tymethod.to_string): /// /// ``` /// assert_eq!('c'.to_uppercase().to_string(), "C"); @@ -892,6 +1105,8 @@ impl char { /// ``` /// /// holds across languages. + #[must_use = "this returns the uppercase character as a new iterator, \ + without modifying the original"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn to_uppercase(self) -> ToUppercase { @@ -909,8 +1124,9 @@ impl char { /// assert!(ascii.is_ascii()); /// assert!(!non_ascii.is_ascii()); /// ``` + #[must_use] #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] - #[rustc_const_stable(feature = "const_ascii_methods_on_intrinsics", since = "1.32.0")] + #[rustc_const_stable(feature = "const_char_is_ascii", since = "1.32.0")] #[inline] pub const fn is_ascii(&self) -> bool { *self as u32 <= 0x7F @@ -921,10 +1137,10 @@ impl char { /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', /// but non-ASCII letters are unchanged. /// - /// To uppercase the value in-place, use [`make_ascii_uppercase`]. + /// To uppercase the value in-place, use [`make_ascii_uppercase()`]. /// /// To uppercase ASCII characters in addition to non-ASCII characters, use - /// [`to_uppercase`]. + /// [`to_uppercase()`]. /// /// # Examples /// @@ -936,12 +1152,18 @@ impl char { /// assert_eq!('❤', non_ascii.to_ascii_uppercase()); /// ``` /// - /// [`make_ascii_uppercase`]: #method.make_ascii_uppercase - /// [`to_uppercase`]: #method.to_uppercase + /// [`make_ascii_uppercase()`]: #method.make_ascii_uppercase + /// [`to_uppercase()`]: #method.to_uppercase + #[must_use = "to uppercase the value in-place, use `make_ascii_uppercase()`"] #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] + #[rustc_const_stable(feature = "const_ascii_methods_on_intrinsics", since = "1.52.0")] #[inline] - pub fn to_ascii_uppercase(&self) -> char { - if self.is_ascii() { (*self as u8).to_ascii_uppercase() as char } else { *self } + pub const fn to_ascii_uppercase(&self) -> char { + if self.is_ascii_lowercase() { + (*self as u8).ascii_change_case_unchecked() as char + } else { + *self + } } /// Makes a copy of the value in its ASCII lower case equivalent. @@ -949,10 +1171,10 @@ impl char { /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', /// but non-ASCII letters are unchanged. /// - /// To lowercase the value in-place, use [`make_ascii_lowercase`]. + /// To lowercase the value in-place, use [`make_ascii_lowercase()`]. /// /// To lowercase ASCII characters in addition to non-ASCII characters, use - /// [`to_lowercase`]. + /// [`to_lowercase()`]. /// /// # Examples /// @@ -964,17 +1186,23 @@ impl char { /// assert_eq!('❤', non_ascii.to_ascii_lowercase()); /// ``` /// - /// [`make_ascii_lowercase`]: #method.make_ascii_lowercase - /// [`to_lowercase`]: #method.to_lowercase + /// [`make_ascii_lowercase()`]: #method.make_ascii_lowercase + /// [`to_lowercase()`]: #method.to_lowercase + #[must_use = "to lowercase the value in-place, use `make_ascii_lowercase()`"] #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] + #[rustc_const_stable(feature = "const_ascii_methods_on_intrinsics", since = "1.52.0")] #[inline] - pub fn to_ascii_lowercase(&self) -> char { - if self.is_ascii() { (*self as u8).to_ascii_lowercase() as char } else { *self } + pub const fn to_ascii_lowercase(&self) -> char { + if self.is_ascii_uppercase() { + (*self as u8).ascii_change_case_unchecked() as char + } else { + *self + } } /// Checks that two values are an ASCII case-insensitive match. /// - /// Equivalent to `to_ascii_lowercase(a) == to_ascii_lowercase(b)`. + /// Equivalent to [to_ascii_lowercase]\(a) == [to_ascii_lowercase]\(b). /// /// # Examples /// @@ -987,9 +1215,12 @@ impl char { /// assert!(upper_a.eq_ignore_ascii_case(&upper_a)); /// assert!(!upper_a.eq_ignore_ascii_case(&lower_z)); /// ``` + /// + /// [to_ascii_lowercase]: #method.to_ascii_lowercase #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] + #[rustc_const_stable(feature = "const_ascii_methods_on_intrinsics", since = "1.52.0")] #[inline] - pub fn eq_ignore_ascii_case(&self, other: &char) -> bool { + pub const fn eq_ignore_ascii_case(&self, other: &char) -> bool { self.to_ascii_lowercase() == other.to_ascii_lowercase() } @@ -999,7 +1230,7 @@ impl char { /// but non-ASCII letters are unchanged. /// /// To return a new uppercased value without modifying the existing one, use - /// [`to_ascii_uppercase`]. + /// [`to_ascii_uppercase()`]. /// /// # Examples /// @@ -1011,7 +1242,7 @@ impl char { /// assert_eq!('A', ascii); /// ``` /// - /// [`to_ascii_uppercase`]: #method.to_ascii_uppercase + /// [`to_ascii_uppercase()`]: #method.to_ascii_uppercase #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] #[inline] pub fn make_ascii_uppercase(&mut self) { @@ -1024,7 +1255,7 @@ impl char { /// but non-ASCII letters are unchanged. /// /// To return a new lowercased value without modifying the existing one, use - /// [`to_ascii_lowercase`]. + /// [`to_ascii_lowercase()`]. /// /// # Examples /// @@ -1036,7 +1267,7 @@ impl char { /// assert_eq!('a', ascii); /// ``` /// - /// [`to_ascii_lowercase`]: #method.to_ascii_lowercase + /// [`to_ascii_lowercase()`]: #method.to_ascii_lowercase #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] #[inline] pub fn make_ascii_lowercase(&mut self) { @@ -1059,7 +1290,7 @@ impl char { /// let percent = '%'; /// let space = ' '; /// let lf = '\n'; - /// let esc: char = 0x1b_u8.into(); + /// let esc = '\x1b'; /// /// assert!(uppercase_a.is_ascii_alphabetic()); /// assert!(uppercase_g.is_ascii_alphabetic()); @@ -1071,14 +1302,12 @@ impl char { /// assert!(!lf.is_ascii_alphabetic()); /// assert!(!esc.is_ascii_alphabetic()); /// ``` + #[must_use] #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] - #[rustc_const_unstable(feature = "const_ascii_ctype_on_intrinsics", issue = "68983")] + #[rustc_const_stable(feature = "const_ascii_ctype_on_intrinsics", since = "1.47.0")] #[inline] pub const fn is_ascii_alphabetic(&self) -> bool { - match *self { - 'A'..='Z' | 'a'..='z' => true, - _ => false, - } + matches!(*self, 'A'..='Z' | 'a'..='z') } /// Checks if the value is an ASCII uppercase character: @@ -1095,7 +1324,7 @@ impl char { /// let percent = '%'; /// let space = ' '; /// let lf = '\n'; - /// let esc: char = 0x1b_u8.into(); + /// let esc = '\x1b'; /// /// assert!(uppercase_a.is_ascii_uppercase()); /// assert!(uppercase_g.is_ascii_uppercase()); @@ -1107,14 +1336,12 @@ impl char { /// assert!(!lf.is_ascii_uppercase()); /// assert!(!esc.is_ascii_uppercase()); /// ``` + #[must_use] #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] - #[rustc_const_unstable(feature = "const_ascii_ctype_on_intrinsics", issue = "68983")] + #[rustc_const_stable(feature = "const_ascii_ctype_on_intrinsics", since = "1.47.0")] #[inline] pub const fn is_ascii_uppercase(&self) -> bool { - match *self { - 'A'..='Z' => true, - _ => false, - } + matches!(*self, 'A'..='Z') } /// Checks if the value is an ASCII lowercase character: @@ -1131,7 +1358,7 @@ impl char { /// let percent = '%'; /// let space = ' '; /// let lf = '\n'; - /// let esc: char = 0x1b_u8.into(); + /// let esc = '\x1b'; /// /// assert!(!uppercase_a.is_ascii_lowercase()); /// assert!(!uppercase_g.is_ascii_lowercase()); @@ -1143,14 +1370,12 @@ impl char { /// assert!(!lf.is_ascii_lowercase()); /// assert!(!esc.is_ascii_lowercase()); /// ``` + #[must_use] #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] - #[rustc_const_unstable(feature = "const_ascii_ctype_on_intrinsics", issue = "68983")] + #[rustc_const_stable(feature = "const_ascii_ctype_on_intrinsics", since = "1.47.0")] #[inline] pub const fn is_ascii_lowercase(&self) -> bool { - match *self { - 'a'..='z' => true, - _ => false, - } + matches!(*self, 'a'..='z') } /// Checks if the value is an ASCII alphanumeric character: @@ -1170,7 +1395,7 @@ impl char { /// let percent = '%'; /// let space = ' '; /// let lf = '\n'; - /// let esc: char = 0x1b_u8.into(); + /// let esc = '\x1b'; /// /// assert!(uppercase_a.is_ascii_alphanumeric()); /// assert!(uppercase_g.is_ascii_alphanumeric()); @@ -1182,14 +1407,12 @@ impl char { /// assert!(!lf.is_ascii_alphanumeric()); /// assert!(!esc.is_ascii_alphanumeric()); /// ``` + #[must_use] #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] - #[rustc_const_unstable(feature = "const_ascii_ctype_on_intrinsics", issue = "68983")] + #[rustc_const_stable(feature = "const_ascii_ctype_on_intrinsics", since = "1.47.0")] #[inline] pub const fn is_ascii_alphanumeric(&self) -> bool { - match *self { - '0'..='9' | 'A'..='Z' | 'a'..='z' => true, - _ => false, - } + matches!(*self, '0'..='9' | 'A'..='Z' | 'a'..='z') } /// Checks if the value is an ASCII decimal digit: @@ -1206,7 +1429,7 @@ impl char { /// let percent = '%'; /// let space = ' '; /// let lf = '\n'; - /// let esc: char = 0x1b_u8.into(); + /// let esc = '\x1b'; /// /// assert!(!uppercase_a.is_ascii_digit()); /// assert!(!uppercase_g.is_ascii_digit()); @@ -1218,14 +1441,44 @@ impl char { /// assert!(!lf.is_ascii_digit()); /// assert!(!esc.is_ascii_digit()); /// ``` + #[must_use] #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] - #[rustc_const_unstable(feature = "const_ascii_ctype_on_intrinsics", issue = "68983")] + #[rustc_const_stable(feature = "const_ascii_ctype_on_intrinsics", since = "1.47.0")] #[inline] pub const fn is_ascii_digit(&self) -> bool { - match *self { - '0'..='9' => true, - _ => false, - } + matches!(*self, '0'..='9') + } + + /// Checks if the value is an ASCII octal digit: + /// U+0030 '0' ..= U+0037 '7'. + /// + /// # Examples + /// + /// ``` + /// #![feature(is_ascii_octdigit)] + /// + /// let uppercase_a = 'A'; + /// let a = 'a'; + /// let zero = '0'; + /// let seven = '7'; + /// let nine = '9'; + /// let percent = '%'; + /// let lf = '\n'; + /// + /// assert!(!uppercase_a.is_ascii_octdigit()); + /// assert!(!a.is_ascii_octdigit()); + /// assert!(zero.is_ascii_octdigit()); + /// assert!(seven.is_ascii_octdigit()); + /// assert!(!nine.is_ascii_octdigit()); + /// assert!(!percent.is_ascii_octdigit()); + /// assert!(!lf.is_ascii_octdigit()); + /// ``` + #[must_use] + #[unstable(feature = "is_ascii_octdigit", issue = "101288")] + #[rustc_const_unstable(feature = "is_ascii_octdigit", issue = "101288")] + #[inline] + pub const fn is_ascii_octdigit(&self) -> bool { + matches!(*self, '0'..='7') } /// Checks if the value is an ASCII hexadecimal digit: @@ -1245,7 +1498,7 @@ impl char { /// let percent = '%'; /// let space = ' '; /// let lf = '\n'; - /// let esc: char = 0x1b_u8.into(); + /// let esc = '\x1b'; /// /// assert!(uppercase_a.is_ascii_hexdigit()); /// assert!(!uppercase_g.is_ascii_hexdigit()); @@ -1257,14 +1510,12 @@ impl char { /// assert!(!lf.is_ascii_hexdigit()); /// assert!(!esc.is_ascii_hexdigit()); /// ``` + #[must_use] #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] - #[rustc_const_unstable(feature = "const_ascii_ctype_on_intrinsics", issue = "68983")] + #[rustc_const_stable(feature = "const_ascii_ctype_on_intrinsics", since = "1.47.0")] #[inline] pub const fn is_ascii_hexdigit(&self) -> bool { - match *self { - '0'..='9' | 'A'..='F' | 'a'..='f' => true, - _ => false, - } + matches!(*self, '0'..='9' | 'A'..='F' | 'a'..='f') } /// Checks if the value is an ASCII punctuation character: @@ -1285,7 +1536,7 @@ impl char { /// let percent = '%'; /// let space = ' '; /// let lf = '\n'; - /// let esc: char = 0x1b_u8.into(); + /// let esc = '\x1b'; /// /// assert!(!uppercase_a.is_ascii_punctuation()); /// assert!(!uppercase_g.is_ascii_punctuation()); @@ -1297,14 +1548,12 @@ impl char { /// assert!(!lf.is_ascii_punctuation()); /// assert!(!esc.is_ascii_punctuation()); /// ``` + #[must_use] #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] - #[rustc_const_unstable(feature = "const_ascii_ctype_on_intrinsics", issue = "68983")] + #[rustc_const_stable(feature = "const_ascii_ctype_on_intrinsics", since = "1.47.0")] #[inline] pub const fn is_ascii_punctuation(&self) -> bool { - match *self { - '!'..='/' | ':'..='@' | '['..='`' | '{'..='~' => true, - _ => false, - } + matches!(*self, '!'..='/' | ':'..='@' | '['..='`' | '{'..='~') } /// Checks if the value is an ASCII graphic character: @@ -1321,7 +1570,7 @@ impl char { /// let percent = '%'; /// let space = ' '; /// let lf = '\n'; - /// let esc: char = 0x1b_u8.into(); + /// let esc = '\x1b'; /// /// assert!(uppercase_a.is_ascii_graphic()); /// assert!(uppercase_g.is_ascii_graphic()); @@ -1333,14 +1582,12 @@ impl char { /// assert!(!lf.is_ascii_graphic()); /// assert!(!esc.is_ascii_graphic()); /// ``` + #[must_use] #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] - #[rustc_const_unstable(feature = "const_ascii_ctype_on_intrinsics", issue = "68983")] + #[rustc_const_stable(feature = "const_ascii_ctype_on_intrinsics", since = "1.47.0")] #[inline] pub const fn is_ascii_graphic(&self) -> bool { - match *self { - '!'..='~' => true, - _ => false, - } + matches!(*self, '!'..='~') } /// Checks if the value is an ASCII whitespace character: @@ -1360,8 +1607,8 @@ impl char { /// before using this function. /// /// [infra-aw]: https://infra.spec.whatwg.org/#ascii-whitespace - /// [pct]: http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap07.html#tag_07_03_01 - /// [bfs]: http://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_05 + /// [pct]: https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap07.html#tag_07_03_01 + /// [bfs]: https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_05 /// /// # Examples /// @@ -1374,7 +1621,7 @@ impl char { /// let percent = '%'; /// let space = ' '; /// let lf = '\n'; - /// let esc: char = 0x1b_u8.into(); + /// let esc = '\x1b'; /// /// assert!(!uppercase_a.is_ascii_whitespace()); /// assert!(!uppercase_g.is_ascii_whitespace()); @@ -1386,14 +1633,12 @@ impl char { /// assert!(lf.is_ascii_whitespace()); /// assert!(!esc.is_ascii_whitespace()); /// ``` + #[must_use] #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] - #[rustc_const_unstable(feature = "const_ascii_ctype_on_intrinsics", issue = "68983")] + #[rustc_const_stable(feature = "const_ascii_ctype_on_intrinsics", since = "1.47.0")] #[inline] pub const fn is_ascii_whitespace(&self) -> bool { - match *self { - '\t' | '\n' | '\x0C' | '\r' | ' ' => true, - _ => false, - } + matches!(*self, '\t' | '\n' | '\x0C' | '\r' | ' ') } /// Checks if the value is an ASCII control character: @@ -1412,7 +1657,7 @@ impl char { /// let percent = '%'; /// let space = ' '; /// let lf = '\n'; - /// let esc: char = 0x1b_u8.into(); + /// let esc = '\x1b'; /// /// assert!(!uppercase_a.is_ascii_control()); /// assert!(!uppercase_g.is_ascii_control()); @@ -1424,13 +1669,127 @@ impl char { /// assert!(lf.is_ascii_control()); /// assert!(esc.is_ascii_control()); /// ``` + #[must_use] #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] - #[rustc_const_unstable(feature = "const_ascii_ctype_on_intrinsics", issue = "68983")] + #[rustc_const_stable(feature = "const_ascii_ctype_on_intrinsics", since = "1.47.0")] #[inline] pub const fn is_ascii_control(&self) -> bool { - match *self { - '\0'..='\x1F' | '\x7F' => true, - _ => false, + matches!(*self, '\0'..='\x1F' | '\x7F') + } +} + +pub(crate) struct EscapeDebugExtArgs { + /// Escape Extended Grapheme codepoints? + pub(crate) escape_grapheme_extended: bool, + + /// Escape single quotes? + pub(crate) escape_single_quote: bool, + + /// Escape double quotes? + pub(crate) escape_double_quote: bool, +} + +impl EscapeDebugExtArgs { + pub(crate) const ESCAPE_ALL: Self = Self { + escape_grapheme_extended: true, + escape_single_quote: true, + escape_double_quote: true, + }; +} + +#[inline] +const fn len_utf8(code: u32) -> usize { + if code < MAX_ONE_B { + 1 + } else if code < MAX_TWO_B { + 2 + } else if code < MAX_THREE_B { + 3 + } else { + 4 + } +} + +/// Encodes a raw u32 value as UTF-8 into the provided byte buffer, +/// and then returns the subslice of the buffer that contains the encoded character. +/// +/// Unlike `char::encode_utf8`, this method also handles codepoints in the surrogate range. +/// (Creating a `char` in the surrogate range is UB.) +/// The result is valid [generalized UTF-8] but not valid UTF-8. +/// +/// [generalized UTF-8]: https://simonsapin.github.io/wtf-8/#generalized-utf8 +/// +/// # Panics +/// +/// Panics if the buffer is not large enough. +/// A buffer of length four is large enough to encode any `char`. +#[unstable(feature = "char_internals", reason = "exposed only for libstd", issue = "none")] +#[doc(hidden)] +#[inline] +pub fn encode_utf8_raw(code: u32, dst: &mut [u8]) -> &mut [u8] { + let len = len_utf8(code); + match (len, &mut dst[..]) { + (1, [a, ..]) => { + *a = code as u8; + } + (2, [a, b, ..]) => { + *a = (code >> 6 & 0x1F) as u8 | TAG_TWO_B; + *b = (code & 0x3F) as u8 | TAG_CONT; + } + (3, [a, b, c, ..]) => { + *a = (code >> 12 & 0x0F) as u8 | TAG_THREE_B; + *b = (code >> 6 & 0x3F) as u8 | TAG_CONT; + *c = (code & 0x3F) as u8 | TAG_CONT; + } + (4, [a, b, c, d, ..]) => { + *a = (code >> 18 & 0x07) as u8 | TAG_FOUR_B; + *b = (code >> 12 & 0x3F) as u8 | TAG_CONT; + *c = (code >> 6 & 0x3F) as u8 | TAG_CONT; + *d = (code & 0x3F) as u8 | TAG_CONT; + } + _ => panic!( + "encode_utf8: need {} bytes to encode U+{:X}, but the buffer has {}", + len, + code, + dst.len(), + ), + }; + &mut dst[..len] +} + +/// Encodes a raw u32 value as UTF-16 into the provided `u16` buffer, +/// and then returns the subslice of the buffer that contains the encoded character. +/// +/// Unlike `char::encode_utf16`, this method also handles codepoints in the surrogate range. +/// (Creating a `char` in the surrogate range is UB.) +/// +/// # Panics +/// +/// Panics if the buffer is not large enough. +/// A buffer of length 2 is large enough to encode any `char`. +#[unstable(feature = "char_internals", reason = "exposed only for libstd", issue = "none")] +#[doc(hidden)] +#[inline] +pub fn encode_utf16_raw(mut code: u32, dst: &mut [u16]) -> &mut [u16] { + // SAFETY: each arm checks whether there are enough bits to write into + unsafe { + if (code & 0xFFFF) == code && !dst.is_empty() { + // The BMP falls through + *dst.get_unchecked_mut(0) = code as u16; + slice::from_raw_parts_mut(dst.as_mut_ptr(), 1) + } else if dst.len() >= 2 { + // Supplementary planes break into surrogates. + code -= 0x1_0000; + *dst.get_unchecked_mut(0) = 0xD800 | ((code >> 10) as u16); + *dst.get_unchecked_mut(1) = 0xDC00 | ((code as u16) & 0x3FF); + slice::from_raw_parts_mut(dst.as_mut_ptr(), 2) + } else { + panic!( + "encode_utf16: need {} units to encode U+{:X}, but the buffer has {}", + from_u32_unchecked(code).len_utf16(), + code, + dst.len(), + ) } } } diff --git a/crux-mir/lib/core/src/char/mod.rs b/crux-mir/lib/core/src/char/mod.rs index cf5576e54..af98059cf 100644 --- a/crux-mir/lib/core/src/char/mod.rs +++ b/crux-mir/lib/core/src/char/mod.rs @@ -1,16 +1,17 @@ -//! A character type. +//! Utilities for the `char` primitive type. +//! +//! *[See also the `char` primitive type](primitive@char).* //! //! The `char` type represents a single character. More specifically, since //! 'character' isn't a well-defined concept in Unicode, `char` is a '[Unicode //! scalar value]', which is similar to, but not the same as, a '[Unicode code //! point]'. //! -//! [Unicode scalar value]: http://www.unicode.org/glossary/#unicode_scalar_value -//! [Unicode code point]: http://www.unicode.org/glossary/#code_point +//! [Unicode scalar value]: https://www.unicode.org/glossary/#unicode_scalar_value +//! [Unicode code point]: https://www.unicode.org/glossary/#code_point //! //! This module exists for technical reasons, the primary documentation for -//! `char` is directly on [the `char` primitive type](../../std/primitive.char.html) -//! itself. +//! `char` is directly on [the `char` primitive type][char] itself. //! //! This module is the home of the iterator implementations for the iterators //! implemented on `char`, as well as some useful constants and conversion @@ -24,26 +25,25 @@ mod decode; mod methods; // stable re-exports -#[stable(feature = "char_from_unchecked", since = "1.5.0")] -pub use self::convert::from_u32_unchecked; #[stable(feature = "try_from", since = "1.34.0")] pub use self::convert::CharTryFromError; #[stable(feature = "char_from_str", since = "1.20.0")] pub use self::convert::ParseCharError; -#[stable(feature = "rust1", since = "1.0.0")] -pub use self::convert::{from_digit, from_u32}; #[stable(feature = "decode_utf16", since = "1.9.0")] -pub use self::decode::{decode_utf16, DecodeUtf16, DecodeUtf16Error}; +pub use self::decode::{DecodeUtf16, DecodeUtf16Error}; -// unstable re-exports -#[unstable(feature = "unicode_version", issue = "49726")] -pub use crate::unicode::version::UnicodeVersion; -#[unstable(feature = "unicode_version", issue = "49726")] -pub use crate::unicode::UNICODE_VERSION; +// perma-unstable re-exports +#[unstable(feature = "char_internals", reason = "exposed only for libstd", issue = "none")] +pub use self::methods::encode_utf16_raw; +#[unstable(feature = "char_internals", reason = "exposed only for libstd", issue = "none")] +pub use self::methods::encode_utf8_raw; +use crate::error::Error; use crate::fmt::{self, Write}; use crate::iter::FusedIterator; +pub(crate) use self::methods::EscapeDebugExtArgs; + // UTF-8 ranges and tags for encoding characters const TAG_CONT: u8 = 0b1000_0000; const TAG_TWO_B: u8 = 0b1100_0000; @@ -86,25 +86,56 @@ const MAX_THREE_B: u32 = 0x10000; Cn Unassigned a reserved unassigned code point or a noncharacter */ -/// The highest valid code point a `char` can have. -/// -/// A [`char`] is a [Unicode Scalar Value], which means that it is a [Code -/// Point], but only ones within a certain range. `MAX` is the highest valid -/// code point that's a valid [Unicode Scalar Value]. -/// -/// [`char`]: ../../std/primitive.char.html -/// [Unicode Scalar Value]: http://www.unicode.org/glossary/#unicode_scalar_value -/// [Code Point]: http://www.unicode.org/glossary/#code_point +/// The highest valid code point a `char` can have, `'\u{10FFFF}'`. Use [`char::MAX`] instead. #[stable(feature = "rust1", since = "1.0.0")] -pub const MAX: char = '\u{10ffff}'; +pub const MAX: char = char::MAX; /// `U+FFFD REPLACEMENT CHARACTER` (�) is used in Unicode to represent a -/// decoding error. -/// -/// It can occur, for example, when giving ill-formed UTF-8 bytes to -/// [`String::from_utf8_lossy`](../../std/string/struct.String.html#method.from_utf8_lossy). +/// decoding error. Use [`char::REPLACEMENT_CHARACTER`] instead. +#[stable(feature = "decode_utf16", since = "1.9.0")] +pub const REPLACEMENT_CHARACTER: char = char::REPLACEMENT_CHARACTER; + +/// The version of [Unicode](https://www.unicode.org/) that the Unicode parts of +/// `char` and `str` methods are based on. Use [`char::UNICODE_VERSION`] instead. +#[stable(feature = "unicode_version", since = "1.45.0")] +pub const UNICODE_VERSION: (u8, u8, u8) = char::UNICODE_VERSION; + +/// Creates an iterator over the UTF-16 encoded code points in `iter`, returning +/// unpaired surrogates as `Err`s. Use [`char::decode_utf16`] instead. #[stable(feature = "decode_utf16", since = "1.9.0")] -pub const REPLACEMENT_CHARACTER: char = '\u{FFFD}'; +#[inline] +pub fn decode_utf16>(iter: I) -> DecodeUtf16 { + self::decode::decode_utf16(iter) +} + +/// Converts a `u32` to a `char`. Use [`char::from_u32`] instead. +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_const_stable(feature = "const_char_convert", since = "1.67.0")] +#[must_use] +#[inline] +pub const fn from_u32(i: u32) -> Option { + self::convert::from_u32(i) +} + +/// Converts a `u32` to a `char`, ignoring validity. Use [`char::from_u32_unchecked`]. +/// instead. +#[stable(feature = "char_from_unchecked", since = "1.5.0")] +#[rustc_const_unstable(feature = "const_char_from_u32_unchecked", issue = "89259")] +#[must_use] +#[inline] +pub const unsafe fn from_u32_unchecked(i: u32) -> char { + // SAFETY: the safety contract must be upheld by the caller. + unsafe { self::convert::from_u32_unchecked(i) } +} + +/// Converts a digit in the given radix to a `char`. Use [`char::from_digit`] instead. +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_const_stable(feature = "const_char_convert", since = "1.67.0")] +#[must_use] +#[inline] +pub const fn from_digit(num: u32, radix: u32) -> Option { + self::convert::from_digit(num, radix) +} /// Returns an iterator that yields the hexadecimal Unicode escape of a /// character, as `char`s. @@ -112,8 +143,7 @@ pub const REPLACEMENT_CHARACTER: char = '\u{FFFD}'; /// This `struct` is created by the [`escape_unicode`] method on [`char`]. See /// its documentation for more. /// -/// [`escape_unicode`]: ../../std/primitive.char.html#method.escape_unicode -/// [`char`]: ../../std/primitive.char.html +/// [`escape_unicode`]: char::escape_unicode #[derive(Clone, Debug)] #[stable(feature = "rust1", since = "1.0.0")] pub struct EscapeUnicode { @@ -234,8 +264,7 @@ impl fmt::Display for EscapeUnicode { /// This `struct` is created by the [`escape_default`] method on [`char`]. See /// its documentation for more. /// -/// [`escape_default`]: ../../std/primitive.char.html#method.escape_default -/// [`char`]: ../../std/primitive.char.html +/// [`escape_default`]: char::escape_default #[derive(Clone, Debug)] #[stable(feature = "rust1", since = "1.0.0")] pub struct EscapeDefault { @@ -343,8 +372,7 @@ impl fmt::Display for EscapeDefault { /// This `struct` is created by the [`escape_debug`] method on [`char`]. See its /// documentation for more. /// -/// [`escape_debug`]: ../../std/primitive.char.html#method.escape_debug -/// [`char`]: ../../std/primitive.char.html +/// [`escape_debug`]: char::escape_debug #[stable(feature = "char_escape_debug", since = "1.20.0")] #[derive(Clone, Debug)] pub struct EscapeDebug(EscapeDefault); @@ -378,8 +406,7 @@ impl fmt::Display for EscapeDebug { /// This `struct` is created by the [`to_lowercase`] method on [`char`]. See /// its documentation for more. /// -/// [`to_lowercase`]: ../../std/primitive.char.html#method.to_lowercase -/// [`char`]: ../../std/primitive.char.html +/// [`to_lowercase`]: char::to_lowercase #[stable(feature = "rust1", since = "1.0.0")] #[derive(Debug, Clone)] pub struct ToLowercase(CaseMappingIter); @@ -395,6 +422,13 @@ impl Iterator for ToLowercase { } } +#[stable(feature = "case_mapping_double_ended", since = "1.59.0")] +impl DoubleEndedIterator for ToLowercase { + fn next_back(&mut self) -> Option { + self.0.next_back() + } +} + #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for ToLowercase {} @@ -406,8 +440,7 @@ impl ExactSizeIterator for ToLowercase {} /// This `struct` is created by the [`to_uppercase`] method on [`char`]. See /// its documentation for more. /// -/// [`to_uppercase`]: ../../std/primitive.char.html#method.to_uppercase -/// [`char`]: ../../std/primitive.char.html +/// [`to_uppercase`]: char::to_uppercase #[stable(feature = "rust1", since = "1.0.0")] #[derive(Debug, Clone)] pub struct ToUppercase(CaseMappingIter); @@ -423,6 +456,13 @@ impl Iterator for ToUppercase { } } +#[stable(feature = "case_mapping_double_ended", since = "1.59.0")] +impl DoubleEndedIterator for ToUppercase { + fn next_back(&mut self) -> Option { + self.0.next_back() + } +} + #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for ToUppercase {} @@ -482,6 +522,26 @@ impl Iterator for CaseMappingIter { } } +impl DoubleEndedIterator for CaseMappingIter { + fn next_back(&mut self) -> Option { + match *self { + CaseMappingIter::Three(a, b, c) => { + *self = CaseMappingIter::Two(a, b); + Some(c) + } + CaseMappingIter::Two(b, c) => { + *self = CaseMappingIter::One(b); + Some(c) + } + CaseMappingIter::One(c) => { + *self = CaseMappingIter::Zero; + Some(c) + } + CaseMappingIter::Zero => None, + } + } +} + impl fmt::Display for CaseMappingIter { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match *self { @@ -513,3 +573,18 @@ impl fmt::Display for ToUppercase { fmt::Display::fmt(&self.0, f) } } + +/// The error type returned when a checked char conversion fails. +#[stable(feature = "u8_from_char", since = "1.59.0")] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct TryFromCharError(pub(crate) ()); + +#[stable(feature = "u8_from_char", since = "1.59.0")] +impl fmt::Display for TryFromCharError { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + "unicode code point out of range".fmt(fmt) + } +} + +#[stable(feature = "u8_from_char", since = "1.59.0")] +impl Error for TryFromCharError {} diff --git a/crux-mir/lib/core/src/clone.rs b/crux-mir/lib/core/src/clone.rs index 9a412e572..398437d9a 100644 --- a/crux-mir/lib/core/src/clone.rs +++ b/crux-mir/lib/core/src/clone.rs @@ -7,11 +7,9 @@ //! contain owned boxes or implement [`Drop`]), so the compiler considers //! them cheap and safe to copy. For other types copies must be made //! explicitly, by convention implementing the [`Clone`] trait and calling -//! the [`clone`][clone] method. +//! the [`clone`] method. //! -//! [`Clone`]: trait.Clone.html -//! [clone]: trait.Clone.html#tymethod.clone -//! [`Drop`]: ../../std/ops/trait.Drop.html +//! [`clone`]: Clone::clone //! //! Basic usage example: //! @@ -38,9 +36,11 @@ #![stable(feature = "rust1", since = "1.0.0")] +use crate::marker::Destruct; + /// A common trait for the ability to explicitly duplicate an object. /// -/// Differs from [`Copy`] in that [`Copy`] is implicit and extremely inexpensive, while +/// Differs from [`Copy`] in that [`Copy`] is implicit and an inexpensive bit-wise copy, while /// `Clone` is always explicit and may or may not be expensive. In order to enforce /// these characteristics, Rust does not allow you to reimplement [`Copy`], but you /// may reimplement `Clone` and run arbitrary code. @@ -51,7 +51,9 @@ /// ## Derivable /// /// This trait can be used with `#[derive]` if all fields are `Clone`. The `derive`d -/// implementation of [`clone`] calls [`clone`] on each field. +/// implementation of [`Clone`] calls [`clone`] on each field. +/// +/// [`clone`]: Clone::clone /// /// For a generic struct, `#[derive]` implements `Clone` conditionally by adding bound `Clone` on /// generic parameters. @@ -74,9 +76,6 @@ /// An example is a generic struct holding a function pointer. In this case, the /// implementation of `Clone` cannot be `derive`d, but can be implemented as: /// -/// [`Copy`]: ../../std/marker/trait.Copy.html -/// [`clone`]: trait.Clone.html#tymethod.clone -/// /// ``` /// struct Generate(fn() -> T); /// @@ -96,8 +95,6 @@ /// /// * Function item types (i.e., the distinct types defined for each function) /// * Function pointer types (e.g., `fn() -> i32`) -/// * Array types, for all sizes, if the item type also implements `Clone` (e.g., `[i32; 123456]`) -/// * Tuple types, if each component also implements `Clone` (e.g., `()`, `(i32, bool)`) /// * Closure types, if they capture no value from the environment /// or if all such captured values implement `Clone` themselves. /// Note that variables captured by shared reference always implement `Clone` @@ -107,12 +104,16 @@ /// [impls]: #implementors #[stable(feature = "rust1", since = "1.0.0")] #[lang = "clone"] +#[rustc_diagnostic_item = "Clone"] +#[rustc_trivial_field_reads] +#[const_trait] pub trait Clone: Sized { /// Returns a copy of the value. /// /// # Examples /// /// ``` + /// # #![allow(noop_method_call)] /// let hello = "Hello"; // &str implements Clone /// /// assert_eq!("Hello", hello.clone()); @@ -128,7 +129,10 @@ pub trait Clone: Sized { /// allocations. #[inline] #[stable(feature = "rust1", since = "1.0.0")] - fn clone_from(&mut self, source: &Self) { + fn clone_from(&mut self, source: &Self) + where + Self: ~const Destruct, + { *self = source.clone() } } @@ -169,17 +173,18 @@ pub struct AssertParamIsCopy { /// Implementations of `Clone` for primitive types. /// /// Implementations that cannot be described in Rust -/// are implemented in `SelectionContext::copy_clone_conditions()` in librustc. +/// are implemented in `traits::SelectionContext::copy_clone_conditions()` +/// in `rustc_trait_selection`. mod impls { - use super::Clone; macro_rules! impl_clone { ($($t:ty)*) => { $( #[stable(feature = "rust1", since = "1.0.0")] - impl Clone for $t { - #[inline] + #[rustc_const_unstable(feature = "const_clone", issue = "91805")] + impl const Clone for $t { + #[inline(always)] fn clone(&self) -> Self { *self } @@ -196,7 +201,8 @@ mod impls { } #[unstable(feature = "never_type", issue = "35121")] - impl Clone for ! { + #[rustc_const_unstable(feature = "const_clone", issue = "91805")] + impl const Clone for ! { #[inline] fn clone(&self) -> Self { *self @@ -204,27 +210,35 @@ mod impls { } #[stable(feature = "rust1", since = "1.0.0")] - impl Clone for *const T { - #[inline] + #[rustc_const_unstable(feature = "const_clone", issue = "91805")] + impl const Clone for *const T { + #[inline(always)] fn clone(&self) -> Self { *self } } #[stable(feature = "rust1", since = "1.0.0")] - impl Clone for *mut T { - #[inline] + #[rustc_const_unstable(feature = "const_clone", issue = "91805")] + impl const Clone for *mut T { + #[inline(always)] fn clone(&self) -> Self { *self } } - // Shared references can be cloned, but mutable references *cannot*! + /// Shared references can be cloned, but mutable references *cannot*! #[stable(feature = "rust1", since = "1.0.0")] - impl Clone for &T { - #[inline] + #[rustc_const_unstable(feature = "const_clone", issue = "91805")] + impl const Clone for &T { + #[inline(always)] + #[rustc_diagnostic_item = "noop_method_clone"] fn clone(&self) -> Self { *self } } + + /// Shared references can be cloned, but mutable references *cannot*! + #[stable(feature = "rust1", since = "1.0.0")] + impl !Clone for &mut T {} } diff --git a/crux-mir/lib/core/src/cmp.rs b/crux-mir/lib/core/src/cmp.rs index 604be7d5f..a7d6fec7d 100644 --- a/crux-mir/lib/core/src/cmp.rs +++ b/crux-mir/lib/core/src/cmp.rs @@ -1,6 +1,6 @@ -//! Functionality for ordering and comparison. +//! Utilities for comparing and ordering values. //! -//! This module contains various tools for ordering and comparing values. In +//! This module contains various tools for comparing and ordering values. In //! summary: //! //! * [`Eq`] and [`PartialEq`] are traits that allow you to define total and @@ -17,53 +17,61 @@ //! //! For more details, see the respective documentation of each item in the list. //! -//! [`Eq`]: trait.Eq.html -//! [`PartialEq`]: trait.PartialEq.html -//! [`Ord`]: trait.Ord.html -//! [`PartialOrd`]: trait.PartialOrd.html -//! [`Ordering`]: enum.Ordering.html -//! [`Reverse`]: struct.Reverse.html -//! [`max`]: fn.max.html -//! [`min`]: fn.min.html +//! [`max`]: Ord::max +//! [`min`]: Ord::min #![stable(feature = "rust1", since = "1.0.0")] +use crate::const_closure::ConstFnMutClosure; +use crate::marker::Destruct; + use self::Ordering::*; -/// Trait for equality comparisons which are [partial equivalence -/// relations](http://en.wikipedia.org/wiki/Partial_equivalence_relation). +/// Trait for equality comparisons. +/// +/// `x.eq(y)` can also be written `x == y`, and `x.ne(y)` can be written `x != y`. +/// We use the easier-to-read infix notation in the remainder of this documentation. /// /// This trait allows for partial equality, for types that do not have a full /// equivalence relation. For example, in floating point numbers `NaN != NaN`, -/// so floating point types implement `PartialEq` but not [`Eq`]. +/// so floating point types implement `PartialEq` but not [`trait@Eq`]. +/// Formally speaking, when `Rhs == Self`, this trait corresponds to a [partial equivalence +/// relation](https://en.wikipedia.org/wiki/Partial_equivalence_relation). /// -/// Formally, the equality must be (for all `a`, `b` and `c`): +/// Implementations must ensure that `eq` and `ne` are consistent with each other: /// -/// - symmetric: `a == b` implies `b == a`; and -/// - transitive: `a == b` and `b == c` implies `a == c`. +/// - `a != b` if and only if `!(a == b)`. +/// +/// The default implementation of `ne` provides this consistency and is almost +/// always sufficient. It should not be overridden without very good reason. +/// +/// If [`PartialOrd`] or [`Ord`] are also implemented for `Self` and `Rhs`, their methods must also +/// be consistent with `PartialEq` (see the documentation of those traits for the exact +/// requirements). It's easy to accidentally make them disagree by deriving some of the traits and +/// manually implementing others. /// -/// Note that these requirements mean that the trait itself must be implemented -/// symmetrically and transitively: if `T: PartialEq` and `U: PartialEq` -/// then `U: PartialEq` and `T: PartialEq`. +/// The equality relation `==` must satisfy the following conditions +/// (for all `a`, `b`, `c` of type `A`, `B`, `C`): +/// +/// - **Symmetric**: if `A: PartialEq` and `B: PartialEq`, then **`a == b` +/// implies `b == a`**; and +/// +/// - **Transitive**: if `A: PartialEq` and `B: PartialEq` and `A: +/// PartialEq`, then **`a == b` and `b == c` implies `a == c`**. +/// +/// Note that the `B: PartialEq` (symmetric) and `A: PartialEq` +/// (transitive) impls are not forced to exist, but these requirements apply +/// whenever they do exist. /// /// ## Derivable /// /// This trait can be used with `#[derive]`. When `derive`d on structs, two /// instances are equal if all fields are equal, and not equal if any fields -/// are not equal. When `derive`d on enums, each variant is equal to itself -/// and not equal to the other variants. +/// are not equal. When `derive`d on enums, two instances are equal if they +/// are the same variant and all fields are equal. /// /// ## How can I implement `PartialEq`? /// -/// `PartialEq` only requires the [`eq`] method to be implemented; [`ne`] is defined -/// in terms of it by default. Any manual implementation of [`ne`] *must* respect -/// the rule that [`eq`] is a strict inverse of [`ne`]; that is, `!(a == b)` if and -/// only if `a != b`. -/// -/// Implementations of `PartialEq`, [`PartialOrd`], and [`Ord`] *must* agree with -/// each other. It's easy to accidentally make them disagree by deriving some -/// of the traits and manually implementing others. -/// /// An example implementation for a domain in which two books are considered /// the same book if their ISBN matches, even if the formats differ: /// @@ -199,8 +207,11 @@ use self::Ordering::*; #[doc(alias = "!=")] #[rustc_on_unimplemented( message = "can't compare `{Self}` with `{Rhs}`", - label = "no implementation for `{Self} == {Rhs}`" + label = "no implementation for `{Self} == {Rhs}`", + append_const_msg )] +#[const_trait] +#[rustc_diagnostic_item = "PartialEq"] pub trait PartialEq { /// This method tests for `self` and `other` values to be equal, and is used /// by `==`. @@ -208,7 +219,8 @@ pub trait PartialEq { #[stable(feature = "rust1", since = "1.0.0")] fn eq(&self, other: &Rhs) -> bool; - /// This method tests for `!=`. + /// This method tests for `!=`. The default implementation is almost always + /// sufficient, and should not be overridden without very good reason. #[inline] #[must_use] #[stable(feature = "rust1", since = "1.0.0")] @@ -267,6 +279,7 @@ pub macro PartialEq($item:item) { #[doc(alias = "==")] #[doc(alias = "!=")] #[stable(feature = "rust1", since = "1.0.0")] +#[rustc_diagnostic_item = "Eq"] pub trait Eq: PartialEq { // this method is used solely by #[deriving] to assert // that every component of a type implements #[deriving] @@ -276,6 +289,7 @@ pub trait Eq: PartialEq { // // This should never be implemented by hand. #[doc(hidden)] + #[no_coverage] // rust-lang/rust#84605 #[inline] #[stable(feature = "rust1", since = "1.0.0")] fn assert_receiver_is_total_eq(&self) {} @@ -284,7 +298,7 @@ pub trait Eq: PartialEq { /// Derive macro generating an impl of the trait `Eq`. #[rustc_builtin_macro] #[stable(feature = "builtin_macro_prelude", since = "1.38.0")] -#[allow_internal_unstable(core_intrinsics, derive_eq, structural_match)] +#[allow_internal_unstable(core_intrinsics, derive_eq, structural_match, no_coverage)] pub macro Eq($item:item) { /* compiler built-in */ } @@ -316,8 +330,10 @@ pub struct AssertParamIsEq { /// let result = 2.cmp(&1); /// assert_eq!(Ordering::Greater, result); /// ``` -#[derive(Clone, Copy, PartialEq, Debug, Hash)] +#[derive(Clone, Copy, Eq, Debug, Hash)] +#[derive_const(PartialOrd, Ord, PartialEq)] #[stable(feature = "rust1", since = "1.0.0")] +#[repr(i8)] pub enum Ordering { /// An ordering where a compared value is less than another. #[stable(feature = "rust1", since = "1.0.0")] @@ -331,6 +347,120 @@ pub enum Ordering { } impl Ordering { + /// Returns `true` if the ordering is the `Equal` variant. + /// + /// # Examples + /// + /// ``` + /// use std::cmp::Ordering; + /// + /// assert_eq!(Ordering::Less.is_eq(), false); + /// assert_eq!(Ordering::Equal.is_eq(), true); + /// assert_eq!(Ordering::Greater.is_eq(), false); + /// ``` + #[inline] + #[must_use] + #[rustc_const_stable(feature = "ordering_helpers", since = "1.53.0")] + #[stable(feature = "ordering_helpers", since = "1.53.0")] + pub const fn is_eq(self) -> bool { + matches!(self, Equal) + } + + /// Returns `true` if the ordering is not the `Equal` variant. + /// + /// # Examples + /// + /// ``` + /// use std::cmp::Ordering; + /// + /// assert_eq!(Ordering::Less.is_ne(), true); + /// assert_eq!(Ordering::Equal.is_ne(), false); + /// assert_eq!(Ordering::Greater.is_ne(), true); + /// ``` + #[inline] + #[must_use] + #[rustc_const_stable(feature = "ordering_helpers", since = "1.53.0")] + #[stable(feature = "ordering_helpers", since = "1.53.0")] + pub const fn is_ne(self) -> bool { + !matches!(self, Equal) + } + + /// Returns `true` if the ordering is the `Less` variant. + /// + /// # Examples + /// + /// ``` + /// use std::cmp::Ordering; + /// + /// assert_eq!(Ordering::Less.is_lt(), true); + /// assert_eq!(Ordering::Equal.is_lt(), false); + /// assert_eq!(Ordering::Greater.is_lt(), false); + /// ``` + #[inline] + #[must_use] + #[rustc_const_stable(feature = "ordering_helpers", since = "1.53.0")] + #[stable(feature = "ordering_helpers", since = "1.53.0")] + pub const fn is_lt(self) -> bool { + matches!(self, Less) + } + + /// Returns `true` if the ordering is the `Greater` variant. + /// + /// # Examples + /// + /// ``` + /// use std::cmp::Ordering; + /// + /// assert_eq!(Ordering::Less.is_gt(), false); + /// assert_eq!(Ordering::Equal.is_gt(), false); + /// assert_eq!(Ordering::Greater.is_gt(), true); + /// ``` + #[inline] + #[must_use] + #[rustc_const_stable(feature = "ordering_helpers", since = "1.53.0")] + #[stable(feature = "ordering_helpers", since = "1.53.0")] + pub const fn is_gt(self) -> bool { + matches!(self, Greater) + } + + /// Returns `true` if the ordering is either the `Less` or `Equal` variant. + /// + /// # Examples + /// + /// ``` + /// use std::cmp::Ordering; + /// + /// assert_eq!(Ordering::Less.is_le(), true); + /// assert_eq!(Ordering::Equal.is_le(), true); + /// assert_eq!(Ordering::Greater.is_le(), false); + /// ``` + #[inline] + #[must_use] + #[rustc_const_stable(feature = "ordering_helpers", since = "1.53.0")] + #[stable(feature = "ordering_helpers", since = "1.53.0")] + pub const fn is_le(self) -> bool { + !matches!(self, Greater) + } + + /// Returns `true` if the ordering is either the `Greater` or `Equal` variant. + /// + /// # Examples + /// + /// ``` + /// use std::cmp::Ordering; + /// + /// assert_eq!(Ordering::Less.is_ge(), false); + /// assert_eq!(Ordering::Equal.is_ge(), true); + /// assert_eq!(Ordering::Greater.is_ge(), true); + /// ``` + #[inline] + #[must_use] + #[rustc_const_stable(feature = "ordering_helpers", since = "1.53.0")] + #[stable(feature = "ordering_helpers", since = "1.53.0")] + pub const fn is_ge(self) -> bool { + !matches!(self, Less) + } + /// Reverses the `Ordering`. /// /// * `Less` becomes `Greater`. @@ -362,8 +492,9 @@ impl Ordering { /// ``` #[inline] #[must_use] + #[rustc_const_stable(feature = "const_ordering", since = "1.48.0")] #[stable(feature = "rust1", since = "1.0.0")] - pub fn reverse(self) -> Ordering { + pub const fn reverse(self) -> Ordering { match self { Less => Greater, Equal => Equal, @@ -400,8 +531,9 @@ impl Ordering { /// ``` #[inline] #[must_use] + #[rustc_const_stable(feature = "const_ordering", since = "1.48.0")] #[stable(feature = "ordering_chaining", since = "1.17.0")] - pub fn then(self, other: Ordering) -> Ordering { + pub const fn then(self, other: Ordering) -> Ordering { match self { Equal => other, _ => self, @@ -431,7 +563,7 @@ impl Ordering { /// assert_eq!(result, Ordering::Equal); /// /// let x: (i64, i64, i64) = (1, 2, 7); - /// let y: (i64, i64, i64) = (1, 5, 3); + /// let y: (i64, i64, i64) = (1, 5, 3); /// let result = x.0.cmp(&y.0).then_with(|| x.1.cmp(&y.1)).then_with(|| x.2.cmp(&y.2)); /// /// assert_eq!(result, Ordering::Less); @@ -463,12 +595,14 @@ impl Ordering { /// v.sort_by_key(|&num| (num > 3, Reverse(num))); /// assert_eq!(v, vec![3, 2, 1, 6, 5, 4]); /// ``` -#[derive(PartialEq, Eq, Debug, Copy, Clone, Default, Hash)] +#[derive(PartialEq, Eq, Debug, Copy, Default, Hash)] #[stable(feature = "reverse_cmp_key", since = "1.19.0")] +#[repr(transparent)] pub struct Reverse(#[stable(feature = "reverse_cmp_key", since = "1.19.0")] pub T); #[stable(feature = "reverse_cmp_key", since = "1.19.0")] -impl PartialOrd for Reverse { +#[rustc_const_unstable(feature = "const_cmp", issue = "92391")] +impl const PartialOrd for Reverse { #[inline] fn partial_cmp(&self, other: &Reverse) -> Option { other.0.partial_cmp(&self.0) @@ -500,18 +634,85 @@ impl Ord for Reverse { } } +#[stable(feature = "reverse_cmp_key", since = "1.19.0")] +impl Clone for Reverse { + #[inline] + fn clone(&self) -> Reverse { + Reverse(self.0.clone()) + } + + #[inline] + fn clone_from(&mut self, other: &Self) { + self.0.clone_from(&other.0) + } +} + /// Trait for types that form a [total order](https://en.wikipedia.org/wiki/Total_order). /// -/// An order is a total order if it is (for all `a`, `b` and `c`): +/// Implementations must be consistent with the [`PartialOrd`] implementation, and ensure +/// `max`, `min`, and `clamp` are consistent with `cmp`: +/// +/// - `partial_cmp(a, b) == Some(cmp(a, b))`. +/// - `max(a, b) == max_by(a, b, cmp)` (ensured by the default implementation). +/// - `min(a, b) == min_by(a, b, cmp)` (ensured by the default implementation). +/// - For `a.clamp(min, max)`, see the [method docs](#method.clamp) +/// (ensured by the default implementation). +/// +/// It's easy to accidentally make `cmp` and `partial_cmp` disagree by +/// deriving some of the traits and manually implementing others. +/// +/// ## Corollaries +/// +/// From the above and the requirements of `PartialOrd`, it follows that `<` defines a strict total order. +/// This means that for all `a`, `b` and `c`: /// -/// - total and asymmetric: exactly one of `a < b`, `a == b` or `a > b` is true; and -/// - transitive, `a < b` and `b < c` implies `a < c`. The same must hold for both `==` and `>`. +/// - exactly one of `a < b`, `a == b` or `a > b` is true; and +/// - `<` is transitive: `a < b` and `b < c` implies `a < c`. The same must hold for both `==` and `>`. /// /// ## Derivable /// -/// This trait can be used with `#[derive]`. When `derive`d on structs, it will produce a -/// lexicographic ordering based on the top-to-bottom declaration order of the struct's members. -/// When `derive`d on enums, variants are ordered by their top-to-bottom declaration order. +/// This trait can be used with `#[derive]`. +/// +/// When `derive`d on structs, it will produce a +/// [lexicographic](https://en.wikipedia.org/wiki/Lexicographic_order) ordering +/// based on the top-to-bottom declaration order of the struct's members. +/// +/// When `derive`d on enums, variants are ordered by their discriminants. +/// By default, the discriminant is smallest for variants at the top, and +/// largest for variants at the bottom. Here's an example: +/// +/// ``` +/// #[derive(PartialEq, Eq, PartialOrd, Ord)] +/// enum E { +/// Top, +/// Bottom, +/// } +/// +/// assert!(E::Top < E::Bottom); +/// ``` +/// +/// However, manually setting the discriminants can override this default +/// behavior: +/// +/// ``` +/// #[derive(PartialEq, Eq, PartialOrd, Ord)] +/// enum E { +/// Top = 2, +/// Bottom = 1, +/// } +/// +/// assert!(E::Bottom < E::Top); +/// ``` +/// +/// ## Lexicographical comparison +/// +/// Lexicographical comparison is an operation with the following properties: +/// - Two sequences are compared element by element. +/// - The first mismatching element defines which sequence is lexicographically less or greater than the other. +/// - If one sequence is a prefix of another, the shorter sequence is lexicographically less than the other. +/// - If two sequence have equivalent elements and are of the same length, then the sequences are lexicographically equal. +/// - An empty sequence is lexicographically less than any non-empty sequence. +/// - Two empty sequences are lexicographically equal. /// /// ## How can I implement `Ord`? /// @@ -520,12 +721,6 @@ impl Ord for Reverse { /// Then you must define an implementation for [`cmp`]. You may find it useful to use /// [`cmp`] on your type's fields. /// -/// Implementations of [`PartialEq`], [`PartialOrd`], and `Ord` *must* -/// agree with each other. That is, `a.cmp(b) == Ordering::Equal` if -/// and only if `a == b` and `Some(a.cmp(b)) == a.partial_cmp(b)` for -/// all `a` and `b`. It's easy to accidentally make them disagree by -/// deriving some of the traits and manually implementing others. -/// /// Here's an example where you want to sort people by height only, disregarding `id` /// and `name`: /// @@ -564,6 +759,8 @@ impl Ord for Reverse { #[doc(alias = "<=")] #[doc(alias = ">=")] #[stable(feature = "rust1", since = "1.0.0")] +#[rustc_diagnostic_item = "Ord"] +#[const_trait] pub trait Ord: Eq + PartialOrd { /// This method returns an [`Ordering`] between `self` and `other`. /// @@ -599,8 +796,18 @@ pub trait Ord: Eq + PartialOrd { fn max(self, other: Self) -> Self where Self: Sized, + Self: ~const Destruct, { - max_by(self, other, Ord::cmp) + #[cfg(not(bootstrap))] + { + max_by(self, other, Ord::cmp) + } + + #[cfg(bootstrap)] + match self.cmp(&other) { + Ordering::Less | Ordering::Equal => other, + Ordering::Greater => self, + } } /// Compares and returns the minimum of two values. @@ -619,8 +826,18 @@ pub trait Ord: Eq + PartialOrd { fn min(self, other: Self) -> Self where Self: Sized, + Self: ~const Destruct, { - min_by(self, other, Ord::cmp) + #[cfg(not(bootstrap))] + { + min_by(self, other, Ord::cmp) + } + + #[cfg(bootstrap)] + match self.cmp(&other) { + Ordering::Less | Ordering::Equal => self, + Ordering::Greater => other, + } } /// Restrict a value to a certain interval. @@ -635,17 +852,17 @@ pub trait Ord: Eq + PartialOrd { /// # Examples /// /// ``` - /// #![feature(clamp)] - /// /// assert!((-3).clamp(-2, 1) == -2); /// assert!(0.clamp(-2, 1) == 0); /// assert!(2.clamp(-2, 1) == 1); /// ``` #[must_use] - #[unstable(feature = "clamp", issue = "44095")] + #[stable(feature = "clamp", since = "1.50.0")] fn clamp(self, min: Self, max: Self) -> Self where Self: Sized, + Self: ~const Destruct, + Self: ~const PartialOrd, { assert!(min <= max); if self < min { @@ -666,41 +883,80 @@ pub macro Ord($item:item) { /* compiler built-in */ } -#[stable(feature = "rust1", since = "1.0.0")] -impl Eq for Ordering {} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Ord for Ordering { - #[inline] - fn cmp(&self, other: &Ordering) -> Ordering { - (*self as i32).cmp(&(*other as i32)) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl PartialOrd for Ordering { - #[inline] - fn partial_cmp(&self, other: &Ordering) -> Option { - (*self as i32).partial_cmp(&(*other as i32)) - } -} - -/// Trait for values that can be compared for a sort-order. +/// Trait for types that form a [partial order](https://en.wikipedia.org/wiki/Partial_order). +/// +/// The `lt`, `le`, `gt`, and `ge` methods of this trait can be called using +/// the `<`, `<=`, `>`, and `>=` operators, respectively. +/// +/// The methods of this trait must be consistent with each other and with those of [`PartialEq`]. +/// The following conditions must hold: +/// +/// 1. `a == b` if and only if `partial_cmp(a, b) == Some(Equal)`. +/// 2. `a < b` if and only if `partial_cmp(a, b) == Some(Less)` +/// 3. `a > b` if and only if `partial_cmp(a, b) == Some(Greater)` +/// 4. `a <= b` if and only if `a < b || a == b` +/// 5. `a >= b` if and only if `a > b || a == b` +/// 6. `a != b` if and only if `!(a == b)`. +/// +/// Conditions 2–5 above are ensured by the default implementation. +/// Condition 6 is already ensured by [`PartialEq`]. +/// +/// If [`Ord`] is also implemented for `Self` and `Rhs`, it must also be consistent with +/// `partial_cmp` (see the documentation of that trait for the exact requirements). It's +/// easy to accidentally make them disagree by deriving some of the traits and manually +/// implementing others. /// /// The comparison must satisfy, for all `a`, `b` and `c`: /// -/// - asymmetry: if `a < b` then `!(a > b)`, as well as `a > b` implying `!(a < b)`; and /// - transitivity: `a < b` and `b < c` implies `a < c`. The same must hold for both `==` and `>`. +/// - duality: `a < b` if and only if `b > a`. /// /// Note that these requirements mean that the trait itself must be implemented symmetrically and /// transitively: if `T: PartialOrd` and `U: PartialOrd` then `U: PartialOrd` and `T: /// PartialOrd`. /// +/// ## Corollaries +/// +/// The following corollaries follow from the above requirements: +/// +/// - irreflexivity of `<` and `>`: `!(a < a)`, `!(a > a)` +/// - transitivity of `>`: if `a > b` and `b > c` then `a > c` +/// - duality of `partial_cmp`: `partial_cmp(a, b) == partial_cmp(b, a).map(Ordering::reverse)` +/// /// ## Derivable /// -/// This trait can be used with `#[derive]`. When `derive`d on structs, it will produce a -/// lexicographic ordering based on the top-to-bottom declaration order of the struct's members. -/// When `derive`d on enums, variants are ordered by their top-to-bottom declaration order. +/// This trait can be used with `#[derive]`. +/// +/// When `derive`d on structs, it will produce a +/// [lexicographic](https://en.wikipedia.org/wiki/Lexicographic_order) ordering +/// based on the top-to-bottom declaration order of the struct's members. +/// +/// When `derive`d on enums, variants are ordered by their discriminants. +/// By default, the discriminant is smallest for variants at the top, and +/// largest for variants at the bottom. Here's an example: +/// +/// ``` +/// #[derive(PartialEq, PartialOrd)] +/// enum E { +/// Top, +/// Bottom, +/// } +/// +/// assert!(E::Top < E::Bottom); +/// ``` +/// +/// However, manually setting the discriminants can override this default +/// behavior: +/// +/// ``` +/// #[derive(PartialEq, PartialOrd)] +/// enum E { +/// Top = 2, +/// Bottom = 1, +/// } +/// +/// assert!(E::Bottom < E::Top); +/// ``` /// /// ## How can I implement `PartialOrd`? /// @@ -713,10 +969,6 @@ impl PartialOrd for Ordering { /// /// `PartialOrd` requires your type to be [`PartialEq`]. /// -/// Implementations of [`PartialEq`], `PartialOrd`, and [`Ord`] *must* agree with each other. It's -/// easy to accidentally make them disagree by deriving some of the traits and manually -/// implementing others. -/// /// If your type is [`Ord`], you can implement [`partial_cmp`] by using [`cmp`]: /// /// ``` @@ -730,19 +982,19 @@ impl PartialOrd for Ordering { /// } /// /// impl PartialOrd for Person { -/// fn partial_cmp(&self, other: &Person) -> Option { +/// fn partial_cmp(&self, other: &Self) -> Option { /// Some(self.cmp(other)) /// } /// } /// /// impl Ord for Person { -/// fn cmp(&self, other: &Person) -> Ordering { +/// fn cmp(&self, other: &Self) -> Ordering { /// self.height.cmp(&other.height) /// } /// } /// /// impl PartialEq for Person { -/// fn eq(&self, other: &Person) -> bool { +/// fn eq(&self, other: &Self) -> bool { /// self.height == other.height /// } /// } @@ -777,8 +1029,8 @@ impl PartialOrd for Ordering { /// # Examples /// /// ``` -/// let x : u32 = 0; -/// let y : u32 = 1; +/// let x: u32 = 0; +/// let y: u32 = 1; /// /// assert_eq!(x < y, true); /// assert_eq!(x.lt(&y), true); @@ -794,8 +1046,11 @@ impl PartialOrd for Ordering { #[doc(alias = ">=")] #[rustc_on_unimplemented( message = "can't compare `{Self}` with `{Rhs}`", - label = "no implementation for `{Self} < {Rhs}` and `{Self} > {Rhs}`" + label = "no implementation for `{Self} < {Rhs}` and `{Self} > {Rhs}`", + append_const_msg )] +#[const_trait] +#[rustc_diagnostic_item = "PartialOrd"] pub trait PartialOrd: PartialEq { /// This method returns an ordering between `self` and `other` values if one exists. /// @@ -817,7 +1072,7 @@ pub trait PartialOrd: PartialEq { /// When comparison is impossible: /// /// ``` - /// let result = std::f64::NAN.partial_cmp(&1.0); + /// let result = f64::NAN.partial_cmp(&1.0); /// assert_eq!(result, None); /// ``` #[must_use] @@ -858,7 +1113,7 @@ pub trait PartialOrd: PartialEq { #[must_use] #[stable(feature = "rust1", since = "1.0.0")] fn le(&self, other: &Rhs) -> bool { - matches!(self.partial_cmp(other), Some(Less) | Some(Equal)) + matches!(self.partial_cmp(other), Some(Less | Equal)) } /// This method tests greater than (for `self` and `other`) and is used by the `>` operator. @@ -895,7 +1150,7 @@ pub trait PartialOrd: PartialEq { #[must_use] #[stable(feature = "rust1", since = "1.0.0")] fn ge(&self, other: &Rhs) -> bool { - matches!(self.partial_cmp(other), Some(Greater) | Some(Equal)) + matches!(self.partial_cmp(other), Some(Greater | Equal)) } } @@ -924,7 +1179,9 @@ pub macro PartialOrd($item:item) { #[inline] #[must_use] #[stable(feature = "rust1", since = "1.0.0")] -pub fn min(v1: T, v2: T) -> T { +#[rustc_const_unstable(feature = "const_cmp", issue = "92391")] +#[cfg_attr(not(test), rustc_diagnostic_item = "cmp_min")] +pub const fn min(v1: T, v2: T) -> T { v1.min(v2) } @@ -935,8 +1192,6 @@ pub fn min(v1: T, v2: T) -> T { /// # Examples /// /// ``` -/// #![feature(cmp_min_max_by)] -/// /// use std::cmp; /// /// assert_eq!(cmp::min_by(-2, 1, |x: &i32, y: &i32| x.abs().cmp(&y.abs())), 1); @@ -944,8 +1199,13 @@ pub fn min(v1: T, v2: T) -> T { /// ``` #[inline] #[must_use] -#[unstable(feature = "cmp_min_max_by", issue = "64460")] -pub fn min_by Ordering>(v1: T, v2: T, compare: F) -> T { +#[stable(feature = "cmp_min_max_by", since = "1.53.0")] +#[rustc_const_unstable(feature = "const_cmp", issue = "92391")] +pub const fn min_by Ordering>(v1: T, v2: T, compare: F) -> T +where + T: ~const Destruct, + F: ~const Destruct, +{ match compare(&v1, &v2) { Ordering::Less | Ordering::Equal => v1, Ordering::Greater => v2, @@ -959,8 +1219,6 @@ pub fn min_by Ordering>(v1: T, v2: T, compare: F) -> T { /// # Examples /// /// ``` -/// #![feature(cmp_min_max_by)] -/// /// use std::cmp; /// /// assert_eq!(cmp::min_by_key(-2, 1, |x: &i32| x.abs()), 1); @@ -968,9 +1226,31 @@ pub fn min_by Ordering>(v1: T, v2: T, compare: F) -> T { /// ``` #[inline] #[must_use] -#[unstable(feature = "cmp_min_max_by", issue = "64460")] -pub fn min_by_key K, K: Ord>(v1: T, v2: T, mut f: F) -> T { - min_by(v1, v2, |v1, v2| f(v1).cmp(&f(v2))) +#[stable(feature = "cmp_min_max_by", since = "1.53.0")] +#[rustc_const_unstable(feature = "const_cmp", issue = "92391")] +pub const fn min_by_key K, K: ~const Ord>(v1: T, v2: T, mut f: F) -> T +where + T: ~const Destruct, + F: ~const Destruct, + K: ~const Destruct, +{ + cfg_if! { + if #[cfg(bootstrap)] { + const fn imp K, K: ~const Ord>( + f: &mut F, + (v1, v2): (&T, &T), + ) -> Ordering + where + T: ~const Destruct, + K: ~const Destruct, + { + f(v1).cmp(&f(v2)) + } + min_by(v1, v2, ConstFnMutClosure::new(&mut f, imp)) + } else { + min_by(v1, v2, const |v1, v2| f(v1).cmp(&f(v2))) + } + } } /// Compares and returns the maximum of two values. @@ -990,7 +1270,9 @@ pub fn min_by_key K, K: Ord>(v1: T, v2: T, mut f: F) -> T { #[inline] #[must_use] #[stable(feature = "rust1", since = "1.0.0")] -pub fn max(v1: T, v2: T) -> T { +#[rustc_const_unstable(feature = "const_cmp", issue = "92391")] +#[cfg_attr(not(test), rustc_diagnostic_item = "cmp_max")] +pub const fn max(v1: T, v2: T) -> T { v1.max(v2) } @@ -1001,8 +1283,6 @@ pub fn max(v1: T, v2: T) -> T { /// # Examples /// /// ``` -/// #![feature(cmp_min_max_by)] -/// /// use std::cmp; /// /// assert_eq!(cmp::max_by(-2, 1, |x: &i32, y: &i32| x.abs().cmp(&y.abs())), -2); @@ -1010,8 +1290,13 @@ pub fn max(v1: T, v2: T) -> T { /// ``` #[inline] #[must_use] -#[unstable(feature = "cmp_min_max_by", issue = "64460")] -pub fn max_by Ordering>(v1: T, v2: T, compare: F) -> T { +#[stable(feature = "cmp_min_max_by", since = "1.53.0")] +#[rustc_const_unstable(feature = "const_cmp", issue = "92391")] +pub const fn max_by Ordering>(v1: T, v2: T, compare: F) -> T +where + T: ~const Destruct, + F: ~const Destruct, +{ match compare(&v1, &v2) { Ordering::Less | Ordering::Equal => v2, Ordering::Greater => v1, @@ -1025,8 +1310,6 @@ pub fn max_by Ordering>(v1: T, v2: T, compare: F) -> T { /// # Examples /// /// ``` -/// #![feature(cmp_min_max_by)] -/// /// use std::cmp; /// /// assert_eq!(cmp::max_by_key(-2, 1, |x: &i32| x.abs()), -2); @@ -1034,9 +1317,25 @@ pub fn max_by Ordering>(v1: T, v2: T, compare: F) -> T { /// ``` #[inline] #[must_use] -#[unstable(feature = "cmp_min_max_by", issue = "64460")] -pub fn max_by_key K, K: Ord>(v1: T, v2: T, mut f: F) -> T { - max_by(v1, v2, |v1, v2| f(v1).cmp(&f(v2))) +#[stable(feature = "cmp_min_max_by", since = "1.53.0")] +#[rustc_const_unstable(feature = "const_cmp", issue = "92391")] +pub const fn max_by_key K, K: ~const Ord>(v1: T, v2: T, mut f: F) -> T +where + T: ~const Destruct, + F: ~const Destruct, + K: ~const Destruct, +{ + const fn imp K, K: ~const Ord>( + f: &mut F, + (v1, v2): (&T, &T), + ) -> Ordering + where + T: ~const Destruct, + K: ~const Destruct, + { + f(v1).cmp(&f(v2)) + } + max_by(v1, v2, ConstFnMutClosure::new(&mut f, imp)) } // Implementation of PartialEq, Eq, PartialOrd and Ord for primitive types @@ -1047,7 +1346,8 @@ mod impls { macro_rules! partial_eq_impl { ($($t:ty)*) => ($( #[stable(feature = "rust1", since = "1.0.0")] - impl PartialEq for $t { + #[rustc_const_unstable(feature = "const_cmp", issue = "92391")] + impl const PartialEq for $t { #[inline] fn eq(&self, other: &$t) -> bool { (*self) == (*other) } #[inline] @@ -1057,7 +1357,8 @@ mod impls { } #[stable(feature = "rust1", since = "1.0.0")] - impl PartialEq for () { + #[rustc_const_unstable(feature = "const_cmp", issue = "92391")] + impl const PartialEq for () { #[inline] fn eq(&self, _other: &()) -> bool { true @@ -1084,10 +1385,11 @@ mod impls { macro_rules! partial_ord_impl { ($($t:ty)*) => ($( #[stable(feature = "rust1", since = "1.0.0")] - impl PartialOrd for $t { + #[rustc_const_unstable(feature = "const_cmp", issue = "92391")] + impl const PartialOrd for $t { #[inline] fn partial_cmp(&self, other: &$t) -> Option { - match (self <= other, self >= other) { + match (*self <= *other, *self >= *other) { (false, false) => None, (false, true) => Some(Greater), (true, false) => Some(Less), @@ -1107,7 +1409,8 @@ mod impls { } #[stable(feature = "rust1", since = "1.0.0")] - impl PartialOrd for () { + #[rustc_const_unstable(feature = "const_cmp", issue = "92391")] + impl const PartialOrd for () { #[inline] fn partial_cmp(&self, _: &()) -> Option { Some(Equal) @@ -1115,10 +1418,11 @@ mod impls { } #[stable(feature = "rust1", since = "1.0.0")] - impl PartialOrd for bool { + #[rustc_const_unstable(feature = "const_cmp", issue = "92391")] + impl const PartialOrd for bool { #[inline] fn partial_cmp(&self, other: &bool) -> Option { - (*self as u8).partial_cmp(&(*other as u8)) + Some(self.cmp(other)) } } @@ -1127,7 +1431,8 @@ mod impls { macro_rules! ord_impl { ($($t:ty)*) => ($( #[stable(feature = "rust1", since = "1.0.0")] - impl PartialOrd for $t { + #[rustc_const_unstable(feature = "const_cmp", issue = "92391")] + impl const PartialOrd for $t { #[inline] fn partial_cmp(&self, other: &$t) -> Option { Some(self.cmp(other)) @@ -1143,7 +1448,8 @@ mod impls { } #[stable(feature = "rust1", since = "1.0.0")] - impl Ord for $t { + #[rustc_const_unstable(feature = "const_cmp", issue = "92391")] + impl const Ord for $t { #[inline] fn cmp(&self, other: &$t) -> Ordering { // The order here is important to generate more optimal assembly. @@ -1157,7 +1463,8 @@ mod impls { } #[stable(feature = "rust1", since = "1.0.0")] - impl Ord for () { + #[rustc_const_unstable(feature = "const_cmp", issue = "92391")] + impl const Ord for () { #[inline] fn cmp(&self, _other: &()) -> Ordering { Equal @@ -1165,7 +1472,8 @@ mod impls { } #[stable(feature = "rust1", since = "1.0.0")] - impl Ord for bool { + #[rustc_const_unstable(feature = "const_cmp", issue = "92391")] + impl const Ord for bool { #[inline] fn cmp(&self, other: &bool) -> Ordering { // Casting to i8's and converting the difference to an Ordering generates @@ -1184,7 +1492,8 @@ mod impls { ord_impl! { char usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } #[unstable(feature = "never_type", issue = "35121")] - impl PartialEq for ! { + #[rustc_const_unstable(feature = "const_cmp", issue = "92391")] + impl const PartialEq for ! { fn eq(&self, _: &!) -> bool { *self } @@ -1194,14 +1503,16 @@ mod impls { impl Eq for ! {} #[unstable(feature = "never_type", issue = "35121")] - impl PartialOrd for ! { + #[rustc_const_unstable(feature = "const_cmp", issue = "92391")] + impl const PartialOrd for ! { fn partial_cmp(&self, _: &!) -> Option { *self } } #[unstable(feature = "never_type", issue = "35121")] - impl Ord for ! { + #[rustc_const_unstable(feature = "const_cmp", issue = "92391")] + impl const Ord for ! { fn cmp(&self, _: &!) -> Ordering { *self } @@ -1210,9 +1521,10 @@ mod impls { // & pointers #[stable(feature = "rust1", since = "1.0.0")] - impl PartialEq<&B> for &A + #[rustc_const_unstable(feature = "const_cmp", issue = "92391")] + impl const PartialEq<&B> for &A where - A: PartialEq, + A: ~const PartialEq, { #[inline] fn eq(&self, other: &&B) -> bool { diff --git a/crux-mir/lib/core/src/const_closure.rs b/crux-mir/lib/core/src/const_closure.rs new file mode 100644 index 000000000..97900a486 --- /dev/null +++ b/crux-mir/lib/core/src/const_closure.rs @@ -0,0 +1,78 @@ +use crate::marker::Destruct; +use crate::marker::Tuple; + +/// Struct representing a closure with mutably borrowed data. +/// +/// Example: +/// ```no_build +/// #![feature(const_mut_refs)] +/// use crate::const_closure::ConstFnMutClosure; +/// const fn imp(state: &mut i32, (arg,): (i32,)) -> i32 { +/// *state += arg; +/// *state +/// } +/// let mut i = 5; +/// let mut cl = ConstFnMutClosure::new(&mut i, imp); +/// +/// assert!(7 == cl(2)); +/// assert!(8 == cl(1)); +/// ``` +pub(crate) struct ConstFnMutClosure { + /// The Data captured by the Closure. + /// Must be either a (mutable) reference or a tuple of (mutable) references. + pub data: CapturedData, + /// The Function of the Closure, must be: Fn(CapturedData, ClosureArgs) -> ClosureReturn + pub func: Function, +} +impl<'a, CapturedData: ?Sized, Function> ConstFnMutClosure<&'a mut CapturedData, Function> { + /// Function for creating a new closure. + /// + /// `data` is the a mutable borrow of data that is captured from the environment. + /// If you want Data to be a tuple of mutable Borrows, the struct must be constructed manually. + /// + /// `func` is the function of the closure, it gets the data and a tuple of the arguments closure + /// and return the return value of the closure. + pub(crate) const fn new( + data: &'a mut CapturedData, + func: Function, + ) -> Self + where + Function: ~const Fn(&mut CapturedData, ClosureArguments) -> ClosureReturnValue, + { + Self { data, func } + } +} + +macro_rules! impl_fn_mut_tuple { + ($($var:ident)*) => { + #[allow(unused_parens)] + impl<'a, $($var,)* ClosureArguments: Tuple, Function, ClosureReturnValue> const + FnOnce for ConstFnMutClosure<($(&'a mut $var),*), Function> + where + Function: ~const Fn(($(&mut $var),*), ClosureArguments) -> ClosureReturnValue+ ~const Destruct, + { + type Output = ClosureReturnValue; + + extern "rust-call" fn call_once(mut self, args: ClosureArguments) -> Self::Output { + self.call_mut(args) + } + } + #[allow(unused_parens)] + impl<'a, $($var,)* ClosureArguments: Tuple, Function, ClosureReturnValue> const + FnMut for ConstFnMutClosure<($(&'a mut $var),*), Function> + where + Function: ~const Fn(($(&mut $var),*), ClosureArguments)-> ClosureReturnValue, + { + extern "rust-call" fn call_mut(&mut self, args: ClosureArguments) -> Self::Output { + #[allow(non_snake_case)] + let ($($var),*) = &mut self.data; + (self.func)(($($var),*), args) + } + } + }; +} +impl_fn_mut_tuple!(A); +impl_fn_mut_tuple!(A B); +impl_fn_mut_tuple!(A B C); +impl_fn_mut_tuple!(A B C D); +impl_fn_mut_tuple!(A B C D E); diff --git a/crux-mir/lib/core/src/convert/mod.rs b/crux-mir/lib/core/src/convert/mod.rs index 47ab8715c..f95b880df 100644 --- a/crux-mir/lib/core/src/convert/mod.rs +++ b/crux-mir/lib/core/src/convert/mod.rs @@ -18,29 +18,26 @@ //! [`TryFrom`][`TryFrom`] rather than [`Into`][`Into`] or [`TryInto`][`TryInto`], //! as [`From`] and [`TryFrom`] provide greater flexibility and offer //! equivalent [`Into`] or [`TryInto`] implementations for free, thanks to a -//! blanket implementation in the standard library. Only implement [`Into`] or [`TryInto`] -//! when a conversion to a type outside the current crate is required. +//! blanket implementation in the standard library. When targeting a version prior to Rust 1.41, it +//! may be necessary to implement [`Into`] or [`TryInto`] directly when converting to a type +//! outside the current crate. //! //! # Generic Implementations //! //! - [`AsRef`] and [`AsMut`] auto-dereference if the inner type is a reference +//! (but not generally for all [dereferenceable types][core::ops::Deref]) //! - [`From`]` for T` implies [`Into`]` for U` //! - [`TryFrom`]` for T` implies [`TryInto`]` for U` //! - [`From`] and [`Into`] are reflexive, which means that all types can //! `into` themselves and `from` themselves //! //! See each trait for usage examples. -//! -//! [`Into`]: trait.Into.html -//! [`From`]: trait.From.html -//! [`TryFrom`]: trait.TryFrom.html -//! [`TryInto`]: trait.TryInto.html -//! [`AsRef`]: trait.AsRef.html -//! [`AsMut`]: trait.AsMut.html #![stable(feature = "rust1", since = "1.0.0")] +use crate::error::Error; use crate::fmt; +use crate::hash::{Hash, Hasher}; mod num; @@ -96,13 +93,13 @@ pub use num::FloatToInt; /// ```rust /// use std::convert::identity; /// -/// let iter = vec![Some(1), None, Some(3)].into_iter(); +/// let iter = [Some(1), None, Some(3)].into_iter(); /// let filtered = iter.filter_map(identity).collect::>(); /// assert_eq!(vec![1, 3], filtered); /// ``` #[stable(feature = "convert_id", since = "1.33.0")] #[rustc_const_stable(feature = "const_identity", since = "1.33.0")] -#[inline] +#[inline(always)] pub const fn identity(x: T) -> T { x } @@ -113,11 +110,13 @@ pub const fn identity(x: T) -> T { /// If you need to do a costly conversion it is better to implement [`From`] with type /// `&T` or write a custom function. /// -/// `AsRef` has the same signature as [`Borrow`], but [`Borrow`] is different in few aspects: +/// # Relation to `Borrow` +/// +/// `AsRef` has the same signature as [`Borrow`], but [`Borrow`] is different in a few aspects: /// /// - Unlike `AsRef`, [`Borrow`] has a blanket impl for any `T`, and can be used to accept either -/// a reference or a value. -/// - [`Borrow`] also requires that [`Hash`], [`Eq`] and [`Ord`] for borrowed value are +/// a reference or a value. (See also note on `AsRef`'s reflexibility below.) +/// - [`Borrow`] also requires that [`Hash`], [`Eq`] and [`Ord`] for a borrowed value are /// equivalent to those of the owned value. For this reason, if you want to /// borrow only a single field of a struct you can implement `AsRef`, but not [`Borrow`]. /// @@ -126,9 +125,66 @@ pub const fn identity(x: T) -> T { /// /// # Generic Implementations /// -/// - `AsRef` auto-dereferences if the inner type is a reference or a mutable -/// reference (e.g.: `foo.as_ref()` will work the same if `foo` has type -/// `&mut Foo` or `&&mut Foo`) +/// `AsRef` auto-dereferences if the inner type is a reference or a mutable reference +/// (e.g.: `foo.as_ref()` will work the same if `foo` has type `&mut Foo` or `&&mut Foo`). +/// +/// Note that due to historic reasons, the above currently does not hold generally for all +/// [dereferenceable types], e.g. `foo.as_ref()` will *not* work the same as +/// `Box::new(foo).as_ref()`. Instead, many smart pointers provide an `as_ref` implementation which +/// simply returns a reference to the [pointed-to value] (but do not perform a cheap +/// reference-to-reference conversion for that value). However, [`AsRef::as_ref`] should not be +/// used for the sole purpose of dereferencing; instead ['`Deref` coercion'] can be used: +/// +/// [dereferenceable types]: core::ops::Deref +/// [pointed-to value]: core::ops::Deref::Target +/// ['`Deref` coercion']: core::ops::Deref#more-on-deref-coercion +/// +/// ``` +/// let x = Box::new(5i32); +/// // Avoid this: +/// // let y: &i32 = x.as_ref(); +/// // Better just write: +/// let y: &i32 = &x; +/// ``` +/// +/// Types which implement [`Deref`] should consider implementing `AsRef` as follows: +/// +/// [`Deref`]: core::ops::Deref +/// +/// ``` +/// # use core::ops::Deref; +/// # struct SomeType; +/// # impl Deref for SomeType { +/// # type Target = [u8]; +/// # fn deref(&self) -> &[u8] { +/// # &[] +/// # } +/// # } +/// impl AsRef for SomeType +/// where +/// T: ?Sized, +/// ::Target: AsRef, +/// { +/// fn as_ref(&self) -> &T { +/// self.deref().as_ref() +/// } +/// } +/// ``` +/// +/// # Reflexivity +/// +/// Ideally, `AsRef` would be reflexive, i.e. there would be an `impl AsRef for T` +/// with [`as_ref`] simply returning its argument unchanged. +/// Such a blanket implementation is currently *not* provided due to technical restrictions of +/// Rust's type system (it would be overlapping with another existing blanket implementation for +/// `&T where T: AsRef` which allows `AsRef` to auto-dereference, see "Generic Implementations" +/// above). +/// +/// [`as_ref`]: AsRef::as_ref +/// +/// A trivial implementation of `AsRef for T` must be added explicitly for a particular type `T` +/// where needed or desired. Note, however, that not all types from `std` contain such an +/// implementation, and those cannot be added by external code due to orphan rules. /// /// # Examples /// @@ -139,13 +195,10 @@ pub const fn identity(x: T) -> T { /// want to accept all references that can be converted to [`&str`] as an argument. /// Since both [`String`] and [`&str`] implement `AsRef` we can accept both as input argument. /// -/// [`Option`]: ../../std/option/enum.Option.html -/// [`Result`]: ../../std/result/enum.Result.html -/// [`Borrow`]: ../../std/borrow/trait.Borrow.html -/// [`Hash`]: ../../std/hash/trait.Hash.html -/// [`Eq`]: ../../std/cmp/trait.Eq.html -/// [`Ord`]: ../../std/cmp/trait.Ord.html -/// [`&str`]: ../../std/primitive.str.html +/// [`&str`]: primitive@str +/// [`Borrow`]: crate::borrow::Borrow +/// [`Eq`]: crate::cmp::Eq +/// [`Ord`]: crate::cmp::Ord /// [`String`]: ../../std/string/struct.String.html /// /// ``` @@ -160,8 +213,10 @@ pub const fn identity(x: T) -> T { /// is_hello(s); /// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "AsRef")] +#[const_trait] pub trait AsRef { - /// Performs the conversion. + /// Converts this type into a shared reference of the (usually inferred) input type. #[stable(feature = "rust1", since = "1.0.0")] fn as_ref(&self) -> &T; } @@ -175,37 +230,145 @@ pub trait AsRef { /// **Note: This trait must not fail**. If the conversion can fail, use a /// dedicated method which returns an [`Option`] or a [`Result`]. /// -/// [`Option`]: ../../std/option/enum.Option.html -/// [`Result`]: ../../std/result/enum.Result.html -/// /// # Generic Implementations /// -/// - `AsMut` auto-dereferences if the inner type is a mutable reference -/// (e.g.: `foo.as_mut()` will work the same if `foo` has type `&mut Foo` -/// or `&mut &mut Foo`) +/// `AsMut` auto-dereferences if the inner type is a mutable reference +/// (e.g.: `foo.as_mut()` will work the same if `foo` has type `&mut Foo` or `&mut &mut Foo`). +/// +/// Note that due to historic reasons, the above currently does not hold generally for all +/// [mutably dereferenceable types], e.g. `foo.as_mut()` will *not* work the same as +/// `Box::new(foo).as_mut()`. Instead, many smart pointers provide an `as_mut` implementation which +/// simply returns a reference to the [pointed-to value] (but do not perform a cheap +/// reference-to-reference conversion for that value). However, [`AsMut::as_mut`] should not be +/// used for the sole purpose of mutable dereferencing; instead ['`Deref` coercion'] can be used: +/// +/// [mutably dereferenceable types]: core::ops::DerefMut +/// [pointed-to value]: core::ops::Deref::Target +/// ['`Deref` coercion']: core::ops::DerefMut#more-on-deref-coercion +/// +/// ``` +/// let mut x = Box::new(5i32); +/// // Avoid this: +/// // let y: &mut i32 = x.as_mut(); +/// // Better just write: +/// let y: &mut i32 = &mut x; +/// ``` +/// +/// Types which implement [`DerefMut`] should consider to add an implementation of `AsMut` as +/// follows: +/// +/// [`DerefMut`]: core::ops::DerefMut +/// +/// ``` +/// # use core::ops::{Deref, DerefMut}; +/// # struct SomeType; +/// # impl Deref for SomeType { +/// # type Target = [u8]; +/// # fn deref(&self) -> &[u8] { +/// # &[] +/// # } +/// # } +/// # impl DerefMut for SomeType { +/// # fn deref_mut(&mut self) -> &mut [u8] { +/// # &mut [] +/// # } +/// # } +/// impl AsMut for SomeType +/// where +/// ::Target: AsMut, +/// { +/// fn as_mut(&mut self) -> &mut T { +/// self.deref_mut().as_mut() +/// } +/// } +/// ``` +/// +/// # Reflexivity +/// +/// Ideally, `AsMut` would be reflexive, i.e. there would be an `impl AsMut for T` +/// with [`as_mut`] simply returning its argument unchanged. +/// Such a blanket implementation is currently *not* provided due to technical restrictions of +/// Rust's type system (it would be overlapping with another existing blanket implementation for +/// `&mut T where T: AsMut` which allows `AsMut` to auto-dereference, see "Generic +/// Implementations" above). +/// +/// [`as_mut`]: AsMut::as_mut +/// +/// A trivial implementation of `AsMut for T` must be added explicitly for a particular type `T` +/// where needed or desired. Note, however, that not all types from `std` contain such an +/// implementation, and those cannot be added by external code due to orphan rules. /// /// # Examples /// -/// Using `AsMut` as trait bound for a generic function we can accept all mutable references -/// that can be converted to type `&mut T`. Because [`Box`] implements `AsMut` we can -/// write a function `add_one` that takes all arguments that can be converted to `&mut u64`. -/// Because [`Box`] implements `AsMut`, `add_one` accepts arguments of type -/// `&mut Box` as well: +/// Using `AsMut` as trait bound for a generic function, we can accept all mutable references that +/// can be converted to type `&mut T`. Unlike [dereference], which has a single [target type], +/// there can be multiple implementations of `AsMut` for a type. In particular, `Vec` implements +/// both `AsMut>` and `AsMut<[T]>`. +/// +/// In the following, the example functions `caesar` and `null_terminate` provide a generic +/// interface which work with any type that can be converted by cheap mutable-to-mutable conversion +/// into a byte slice (`[u8]`) or byte vector (`Vec`), respectively. +/// +/// [dereference]: core::ops::DerefMut +/// [target type]: core::ops::Deref::Target /// /// ``` -/// fn add_one>(num: &mut T) { -/// *num.as_mut() += 1; +/// struct Document { +/// info: String, +/// content: Vec, +/// } +/// +/// impl AsMut for Document +/// where +/// Vec: AsMut, +/// { +/// fn as_mut(&mut self) -> &mut T { +/// self.content.as_mut() +/// } +/// } +/// +/// fn caesar>(data: &mut T, key: u8) { +/// for byte in data.as_mut() { +/// *byte = byte.wrapping_add(key); +/// } +/// } +/// +/// fn null_terminate>>(data: &mut T) { +/// // Using a non-generic inner function, which contains most of the +/// // functionality, helps to minimize monomorphization overhead. +/// fn doit(data: &mut Vec) { +/// let len = data.len(); +/// if len == 0 || data[len-1] != 0 { +/// data.push(0); +/// } +/// } +/// doit(data.as_mut()); /// } /// -/// let mut boxed_num = Box::new(0); -/// add_one(&mut boxed_num); -/// assert_eq!(*boxed_num, 1); +/// fn main() { +/// let mut v: Vec = vec![1, 2, 3]; +/// caesar(&mut v, 5); +/// assert_eq!(v, [6, 7, 8]); +/// null_terminate(&mut v); +/// assert_eq!(v, [6, 7, 8, 0]); +/// let mut doc = Document { +/// info: String::from("Example"), +/// content: vec![17, 19, 8], +/// }; +/// caesar(&mut doc, 1); +/// assert_eq!(doc.content, [18, 20, 9]); +/// null_terminate(&mut doc); +/// assert_eq!(doc.content, [18, 20, 9, 0]); +/// } /// ``` /// -/// [`Box`]: ../../std/boxed/struct.Box.html +/// Note, however, that APIs don't need to be generic. In many cases taking a `&mut [u8]` or +/// `&mut Vec`, for example, is the better choice (callers need to pass the correct type then). #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "AsMut")] +#[const_trait] pub trait AsMut { - /// Performs the conversion. + /// Converts this type into a mutable reference of the (usually inferred) input type. #[stable(feature = "rust1", since = "1.0.0")] fn as_mut(&mut self) -> &mut T; } @@ -276,16 +439,14 @@ pub trait AsMut { /// is_hello(s); /// ``` /// -/// [`TryInto`]: trait.TryInto.html -/// [`Option`]: ../../std/option/enum.Option.html -/// [`Result`]: ../../std/result/enum.Result.html /// [`String`]: ../../std/string/struct.String.html -/// [`From`]: trait.From.html -/// [`Into`]: trait.Into.html /// [`Vec`]: ../../std/vec/struct.Vec.html +#[rustc_diagnostic_item = "Into"] #[stable(feature = "rust1", since = "1.0.0")] +#[const_trait] pub trait Into: Sized { - /// Performs the conversion. + /// Converts this type into the (usually inferred) input type. + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] fn into(self) -> T; } @@ -297,8 +458,10 @@ pub trait Into: Sized { /// because implementing `From` automatically provides one with an implementation of [`Into`] /// thanks to the blanket implementation in the standard library. /// -/// Only implement [`Into`] if a conversion to a type outside the current crate is required. -/// `From` cannot do these type of conversions because of Rust's orphaning rules. +/// Only implement [`Into`] when targeting a version prior to Rust 1.41 and converting to a type +/// outside the current crate. +/// `From` was not able to do these types of conversions in earlier versions because of Rust's +/// orphaning rules. /// See [`Into`] for more details. /// /// Prefer using [`Into`] over using `From` when specifying trait bounds on a generic function. @@ -310,7 +473,8 @@ pub trait Into: Sized { /// that encapsulate multiple error types. See the "Examples" section and [the book][book] for more /// details. /// -/// **Note: This trait must not fail**. If the conversion can fail, use [`TryFrom`]. +/// **Note: This trait must not fail**. The `From` trait is intended for perfect conversions. +/// If the conversion can fail or is not perfect, use [`TryFrom`]. /// /// # Generic Implementations /// @@ -366,22 +530,22 @@ pub trait Into: Sized { /// } /// ``` /// -/// [`TryFrom`]: trait.TryFrom.html -/// [`Option`]: ../../std/option/enum.Option.html -/// [`Result`]: ../../std/result/enum.Result.html /// [`String`]: ../../std/string/struct.String.html -/// [`Into`]: trait.Into.html -/// [`from`]: trait.From.html#tymethod.from +/// [`from`]: From::from /// [book]: ../../book/ch09-00-error-handling.html +#[rustc_diagnostic_item = "From"] #[stable(feature = "rust1", since = "1.0.0")] #[rustc_on_unimplemented(on( all(_Self = "&str", T = "std::string::String"), note = "to coerce a `{T}` into a `{Self}`, use `&*` as a prefix", ))] +#[const_trait] pub trait From: Sized { - /// Performs the conversion. + /// Converts to this type from the input type. + #[lang = "from"] + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] - fn from(_: T) -> Self; + fn from(value: T) -> Self; } /// An attempted conversion that consumes `self`, which may or may not be @@ -398,10 +562,9 @@ pub trait From: Sized { /// /// This suffers the same restrictions and reasoning as implementing /// [`Into`], see there for details. -/// -/// [`TryFrom`]: trait.TryFrom.html -/// [`Into`]: trait.Into.html +#[rustc_diagnostic_item = "TryInto"] #[stable(feature = "try_from", since = "1.34.0")] +#[const_trait] pub trait TryInto: Sized { /// The type returned in the event of a conversion error. #[stable(feature = "try_from", since = "1.34.0")] @@ -439,8 +602,6 @@ pub trait TryInto: Sized { /// `TryFrom` can be implemented as follows: /// /// ``` -/// use std::convert::TryFrom; -/// /// struct GreaterThanZero(i32); /// /// impl TryFrom for GreaterThanZero { @@ -448,7 +609,7 @@ pub trait TryInto: Sized { /// /// fn try_from(value: i32) -> Result { /// if value <= 0 { -/// Err("GreaterThanZero only accepts value superior than zero!") +/// Err("GreaterThanZero only accepts values greater than zero!") /// } else { /// Ok(GreaterThanZero(value)) /// } @@ -461,8 +622,6 @@ pub trait TryInto: Sized { /// As described, [`i32`] implements `TryFrom<`[`i64`]`>`: /// /// ``` -/// use std::convert::TryFrom; -/// /// let big_number = 1_000_000_000_000i64; /// // Silently truncates `big_number`, requires detecting /// // and handling the truncation after the fact. @@ -479,12 +638,10 @@ pub trait TryInto: Sized { /// assert!(try_successful_smaller_number.is_ok()); /// ``` /// -/// [`try_from`]: trait.TryFrom.html#tymethod.try_from -/// [`TryInto`]: trait.TryInto.html -/// [`i32::MAX`]: ../../std/i32/constant.MAX.html -/// [`!`]: ../../std/primitive.never.html -/// [`Infallible`]: enum.Infallible.html +/// [`try_from`]: TryFrom::try_from +#[rustc_diagnostic_item = "TryFrom"] #[stable(feature = "try_from", since = "1.34.0")] +#[const_trait] pub trait TryFrom: Sized { /// The type returned in the event of a conversion error. #[stable(feature = "try_from", since = "1.34.0")] @@ -501,10 +658,12 @@ pub trait TryFrom: Sized { // As lifts over & #[stable(feature = "rust1", since = "1.0.0")] -impl AsRef for &T +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const AsRef for &T where - T: AsRef, + T: ~const AsRef, { + #[inline] fn as_ref(&self) -> &U { >::as_ref(*self) } @@ -512,10 +671,12 @@ where // As lifts over &mut #[stable(feature = "rust1", since = "1.0.0")] -impl AsRef for &mut T +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const AsRef for &mut T where - T: AsRef, + T: ~const AsRef, { + #[inline] fn as_ref(&self) -> &U { >::as_ref(*self) } @@ -531,10 +692,12 @@ where // AsMut lifts over &mut #[stable(feature = "rust1", since = "1.0.0")] -impl AsMut for &mut T +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const AsMut for &mut T where - T: AsMut, + T: ~const AsMut, { + #[inline] fn as_mut(&mut self) -> &mut U { (*self).as_mut() } @@ -550,10 +713,15 @@ where // From implies Into #[stable(feature = "rust1", since = "1.0.0")] -impl Into for T +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const Into for T where - U: From, + U: ~const From, { + /// Calls `U::from(self)`. + /// + /// That is, this conversion is whatever the implementation of + /// [From]<T> for U chooses to do. fn into(self) -> U { U::from(self) } @@ -561,7 +729,10 @@ where // From (and thus Into) is reflexive #[stable(feature = "rust1", since = "1.0.0")] -impl From for T { +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const From for T { + /// Returns the argument unchanged. + #[inline(always)] fn from(t: T) -> T { t } @@ -576,7 +747,8 @@ impl From for T { #[allow(unused_attributes)] // FIXME(#58633): do a principled fix instead. #[rustc_reservation_impl = "permitting this impl would forbid us from adding \ `impl From for T` later; see rust-lang/rust#64715 for details"] -impl From for T { +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const From for T { fn from(t: !) -> T { t } @@ -584,9 +756,10 @@ impl From for T { // TryFrom implies TryInto #[stable(feature = "try_from", since = "1.34.0")] -impl TryInto for T +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const TryInto for T where - U: TryFrom, + U: ~const TryFrom, { type Error = U::Error; @@ -598,9 +771,10 @@ where // Infallible conversions are semantically equivalent to fallible conversions // with an uninhabited error type. #[stable(feature = "try_from", since = "1.34.0")] -impl TryFrom for T +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const TryFrom for T where - U: Into, + U: ~const Into, { type Error = Infallible; @@ -615,6 +789,7 @@ where #[stable(feature = "rust1", since = "1.0.0")] impl AsRef<[T]> for [T] { + #[inline(always)] fn as_ref(&self) -> &[T] { self } @@ -622,6 +797,7 @@ impl AsRef<[T]> for [T] { #[stable(feature = "rust1", since = "1.0.0")] impl AsMut<[T]> for [T] { + #[inline(always)] fn as_mut(&mut self) -> &mut [T] { self } @@ -629,12 +805,20 @@ impl AsMut<[T]> for [T] { #[stable(feature = "rust1", since = "1.0.0")] impl AsRef for str { - #[inline] + #[inline(always)] fn as_ref(&self) -> &str { self } } +#[stable(feature = "as_mut_str_for_str", since = "1.51.0")] +impl AsMut for str { + #[inline(always)] + fn as_mut(&mut self) -> &mut str { + self + } +} + //////////////////////////////////////////////////////////////////////////////// // THE NO-ERROR ERROR TYPE //////////////////////////////////////////////////////////////////////////////// @@ -670,10 +854,9 @@ impl AsRef for str { /// /// … and eventually deprecate `Infallible`. /// -/// /// However there is one case where `!` syntax can be used -/// before `!` is stabilized as a full-fleged type: in the position of a function’s return type. -/// Specifically, it is possible implementations for two different function pointer types: +/// before `!` is stabilized as a full-fledged type: in the position of a function’s return type. +/// Specifically, it is possible to have implementations for two different function pointer types: /// /// ``` /// trait MyTrait {} @@ -685,18 +868,13 @@ impl AsRef for str { /// However when `Infallible` becomes an alias for the never type, /// the two `impl`s will start to overlap /// and therefore will be disallowed by the language’s trait coherence rules. -/// -/// [`Ok`]: ../result/enum.Result.html#variant.Ok -/// [`Result`]: ../result/enum.Result.html -/// [`TryFrom`]: trait.TryFrom.html -/// [`Into`]: trait.Into.html -/// [never]: ../../std/primitive.never.html #[stable(feature = "convert_infallible", since = "1.34.0")] #[derive(Copy)] pub enum Infallible {} #[stable(feature = "convert_infallible", since = "1.34.0")] -impl Clone for Infallible { +#[rustc_const_unstable(feature = "const_clone", issue = "91805")] +impl const Clone for Infallible { fn clone(&self) -> Infallible { match *self {} } @@ -716,6 +894,13 @@ impl fmt::Display for Infallible { } } +#[stable(feature = "str_parse_error2", since = "1.8.0")] +impl Error for Infallible { + fn description(&self) -> &str { + match *self {} + } +} + #[stable(feature = "convert_infallible", since = "1.34.0")] impl PartialEq for Infallible { fn eq(&self, _: &Infallible) -> bool { @@ -741,8 +926,16 @@ impl Ord for Infallible { } #[stable(feature = "convert_infallible", since = "1.34.0")] -impl From for Infallible { +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const From for Infallible { fn from(x: !) -> Self { x } } + +#[stable(feature = "convert_infallible_hash", since = "1.44.0")] +impl Hash for Infallible { + fn hash(&self, _: &mut H) { + match *self {} + } +} diff --git a/crux-mir/lib/core/src/convert/num.rs b/crux-mir/lib/core/src/convert/num.rs index 752199c94..45e2f711c 100644 --- a/crux-mir/lib/core/src/convert/num.rs +++ b/crux-mir/lib/core/src/convert/num.rs @@ -9,13 +9,13 @@ mod private { pub trait Sealed {} } -/// Supporting trait for inherent methods of `f32` and `f64` such as `round_unchecked_to`. +/// Supporting trait for inherent methods of `f32` and `f64` such as `to_int_unchecked`. /// Typically doesn’t need to be used directly. #[unstable(feature = "convert_float_to_int", issue = "67057")] pub trait FloatToInt: private::Sealed + Sized { - #[unstable(feature = "float_approx_unchecked_to", issue = "67058")] + #[unstable(feature = "convert_float_to_int", issue = "67057")] #[doc(hidden)] - unsafe fn approx_unchecked(self) -> Int; + unsafe fn to_int_unchecked(self) -> Int; } macro_rules! impl_float_to_int { @@ -25,10 +25,10 @@ macro_rules! impl_float_to_int { $( #[unstable(feature = "convert_float_to_int", issue = "67057")] impl FloatToInt<$Int> for $Float { - #[doc(hidden)] #[inline] - unsafe fn approx_unchecked(self) -> $Int { - crate::intrinsics::float_to_int_approx_unchecked(self) + unsafe fn to_int_unchecked(self) -> $Int { + // SAFETY: the safety contract must be upheld by the caller. + unsafe { crate::intrinsics::float_to_int_unchecked(self) } } } )+ @@ -44,9 +44,12 @@ impl_float_to_int!(f64 => u8 u16 u32 u64 u128 usize i8 i16 i32 i64 i128 isize); macro_rules! impl_from { ($Small: ty, $Large: ty, #[$attr:meta], $doc: expr) => { #[$attr] - #[doc = $doc] - impl From<$Small> for $Large { - #[inline] + #[rustc_const_unstable(feature = "const_num_from_num", issue = "87852")] + impl const From<$Small> for $Large { + // Rustdocs on the impl block show a "[+] show undocumented items" toggle. + // Rustdocs on functions do not. + #[doc = $doc] + #[inline(always)] fn from(small: $Small) -> Self { small as Self } @@ -142,7 +145,7 @@ impl_from! { i16, isize, #[stable(feature = "lossless_iusize_conv", since = "1.2 // CHERI proposes 256-bit “capabilities”. Unclear if this would be relevant to usize/isize. // https://www.cl.cam.ac.uk/research/security/ctsrd/pdfs/20171017a-cheri-poster.pdf -// http://www.csl.sri.com/users/neumann/2012resolve-cheri.pdf +// https://www.csl.sri.com/users/neumann/2012resolve-cheri.pdf // Note: integers can only be represented with full precision in a float if // they fit in the significand, which is 24 bits in f32 and 53 bits in f64. @@ -165,11 +168,32 @@ impl_from! { u32, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0" // Float -> Float impl_from! { f32, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0")] } +// bool -> Float +#[stable(feature = "float_from_bool", since = "CURRENT_RUSTC_VERSION")] +#[rustc_const_unstable(feature = "const_num_from_num", issue = "87852")] +impl const From for f32 { + /// Converts `bool` to `f32` losslessly. + #[inline] + fn from(small: bool) -> Self { + small as u8 as Self + } +} +#[stable(feature = "float_from_bool", since = "CURRENT_RUSTC_VERSION")] +#[rustc_const_unstable(feature = "const_num_from_num", issue = "87852")] +impl const From for f64 { + /// Converts `bool` to `f64` losslessly. + #[inline] + fn from(small: bool) -> Self { + small as u8 as Self + } +} + // no possible bounds violation macro_rules! try_from_unbounded { ($source:ty, $($target:ty),*) => {$( #[stable(feature = "try_from", since = "1.34.0")] - impl TryFrom<$source> for $target { + #[rustc_const_unstable(feature = "const_num_from_num", issue = "87852")] + impl const TryFrom<$source> for $target { type Error = TryFromIntError; /// Try to create the target number type from a source @@ -187,7 +211,8 @@ macro_rules! try_from_unbounded { macro_rules! try_from_lower_bounded { ($source:ty, $($target:ty),*) => {$( #[stable(feature = "try_from", since = "1.34.0")] - impl TryFrom<$source> for $target { + #[rustc_const_unstable(feature = "const_num_from_num", issue = "87852")] + impl const TryFrom<$source> for $target { type Error = TryFromIntError; /// Try to create the target number type from a source @@ -209,7 +234,8 @@ macro_rules! try_from_lower_bounded { macro_rules! try_from_upper_bounded { ($source:ty, $($target:ty),*) => {$( #[stable(feature = "try_from", since = "1.34.0")] - impl TryFrom<$source> for $target { + #[rustc_const_unstable(feature = "const_num_from_num", issue = "87852")] + impl const TryFrom<$source> for $target { type Error = TryFromIntError; /// Try to create the target number type from a source @@ -217,7 +243,7 @@ macro_rules! try_from_upper_bounded { /// is outside of the range of the target type. #[inline] fn try_from(u: $source) -> Result { - if u > (Self::max_value() as $source) { + if u > (Self::MAX as $source) { Err(TryFromIntError(())) } else { Ok(u as Self) @@ -231,7 +257,8 @@ macro_rules! try_from_upper_bounded { macro_rules! try_from_both_bounded { ($source:ty, $($target:ty),*) => {$( #[stable(feature = "try_from", since = "1.34.0")] - impl TryFrom<$source> for $target { + #[rustc_const_unstable(feature = "const_num_from_num", issue = "87852")] + impl const TryFrom<$source> for $target { type Error = TryFromIntError; /// Try to create the target number type from a source @@ -239,8 +266,8 @@ macro_rules! try_from_both_bounded { /// is outside of the range of the target type. #[inline] fn try_from(u: $source) -> Result { - let min = Self::min_value() as $source; - let max = Self::max_value() as $source; + let min = Self::MIN as $source; + let max = Self::MAX as $source; if u < min || u > max { Err(TryFromIntError(())) } else { @@ -382,13 +409,16 @@ use crate::num::NonZeroUsize; macro_rules! nzint_impl_from { ($Small: ty, $Large: ty, #[$attr:meta], $doc: expr) => { #[$attr] - #[doc = $doc] - impl From<$Small> for $Large { + #[rustc_const_unstable(feature = "const_num_from_num", issue = "87852")] + impl const From<$Small> for $Large { + // Rustdocs on the impl block show a "[+] show undocumented items" toggle. + // Rustdocs on functions do not. + #[doc = $doc] #[inline] fn from(small: $Small) -> Self { // SAFETY: input type guarantees the value is non-zero unsafe { - Self::new_unchecked(small.get().into()) + Self::new_unchecked(From::from(small.get())) } } } @@ -445,3 +475,92 @@ nzint_impl_from! { NonZeroU16, NonZeroI128, #[stable(feature = "nz_int_conv", si nzint_impl_from! { NonZeroU32, NonZeroI64, #[stable(feature = "nz_int_conv", since = "1.41.0")] } nzint_impl_from! { NonZeroU32, NonZeroI128, #[stable(feature = "nz_int_conv", since = "1.41.0")] } nzint_impl_from! { NonZeroU64, NonZeroI128, #[stable(feature = "nz_int_conv", since = "1.41.0")] } + +macro_rules! nzint_impl_try_from_int { + ($Int: ty, $NonZeroInt: ty, #[$attr:meta], $doc: expr) => { + #[$attr] + impl TryFrom<$Int> for $NonZeroInt { + type Error = TryFromIntError; + + // Rustdocs on the impl block show a "[+] show undocumented items" toggle. + // Rustdocs on functions do not. + #[doc = $doc] + #[inline] + fn try_from(value: $Int) -> Result { + Self::new(value).ok_or(TryFromIntError(())) + } + } + }; + ($Int: ty, $NonZeroInt: ty, #[$attr:meta]) => { + nzint_impl_try_from_int!($Int, + $NonZeroInt, + #[$attr], + concat!("Attempts to convert `", + stringify!($Int), + "` to `", + stringify!($NonZeroInt), + "`.")); + } +} + +// Int -> Non-zero Int +nzint_impl_try_from_int! { u8, NonZeroU8, #[stable(feature = "nzint_try_from_int_conv", since = "1.46.0")] } +nzint_impl_try_from_int! { u16, NonZeroU16, #[stable(feature = "nzint_try_from_int_conv", since = "1.46.0")] } +nzint_impl_try_from_int! { u32, NonZeroU32, #[stable(feature = "nzint_try_from_int_conv", since = "1.46.0")] } +nzint_impl_try_from_int! { u64, NonZeroU64, #[stable(feature = "nzint_try_from_int_conv", since = "1.46.0")] } +nzint_impl_try_from_int! { u128, NonZeroU128, #[stable(feature = "nzint_try_from_int_conv", since = "1.46.0")] } +nzint_impl_try_from_int! { usize, NonZeroUsize, #[stable(feature = "nzint_try_from_int_conv", since = "1.46.0")] } +nzint_impl_try_from_int! { i8, NonZeroI8, #[stable(feature = "nzint_try_from_int_conv", since = "1.46.0")] } +nzint_impl_try_from_int! { i16, NonZeroI16, #[stable(feature = "nzint_try_from_int_conv", since = "1.46.0")] } +nzint_impl_try_from_int! { i32, NonZeroI32, #[stable(feature = "nzint_try_from_int_conv", since = "1.46.0")] } +nzint_impl_try_from_int! { i64, NonZeroI64, #[stable(feature = "nzint_try_from_int_conv", since = "1.46.0")] } +nzint_impl_try_from_int! { i128, NonZeroI128, #[stable(feature = "nzint_try_from_int_conv", since = "1.46.0")] } +nzint_impl_try_from_int! { isize, NonZeroIsize, #[stable(feature = "nzint_try_from_int_conv", since = "1.46.0")] } + +macro_rules! nzint_impl_try_from_nzint { + ($From:ty => $To:ty, $doc: expr) => { + #[stable(feature = "nzint_try_from_nzint_conv", since = "1.49.0")] + impl TryFrom<$From> for $To { + type Error = TryFromIntError; + + // Rustdocs on the impl block show a "[+] show undocumented items" toggle. + // Rustdocs on functions do not. + #[doc = $doc] + #[inline] + fn try_from(value: $From) -> Result { + TryFrom::try_from(value.get()).map(|v| { + // SAFETY: $From is a NonZero type, so v is not zero. + unsafe { Self::new_unchecked(v) } + }) + } + } + }; + ($To:ty: $($From: ty),*) => {$( + nzint_impl_try_from_nzint!( + $From => $To, + concat!( + "Attempts to convert `", + stringify!($From), + "` to `", + stringify!($To), + "`.", + ) + ); + )*}; +} + +// Non-zero int -> non-zero unsigned int +nzint_impl_try_from_nzint! { NonZeroU8: NonZeroI8, NonZeroU16, NonZeroI16, NonZeroU32, NonZeroI32, NonZeroU64, NonZeroI64, NonZeroU128, NonZeroI128, NonZeroUsize, NonZeroIsize } +nzint_impl_try_from_nzint! { NonZeroU16: NonZeroI8, NonZeroI16, NonZeroU32, NonZeroI32, NonZeroU64, NonZeroI64, NonZeroU128, NonZeroI128, NonZeroUsize, NonZeroIsize } +nzint_impl_try_from_nzint! { NonZeroU32: NonZeroI8, NonZeroI16, NonZeroI32, NonZeroU64, NonZeroI64, NonZeroU128, NonZeroI128, NonZeroUsize, NonZeroIsize } +nzint_impl_try_from_nzint! { NonZeroU64: NonZeroI8, NonZeroI16, NonZeroI32, NonZeroI64, NonZeroU128, NonZeroI128, NonZeroUsize, NonZeroIsize } +nzint_impl_try_from_nzint! { NonZeroU128: NonZeroI8, NonZeroI16, NonZeroI32, NonZeroI64, NonZeroI128, NonZeroUsize, NonZeroIsize } +nzint_impl_try_from_nzint! { NonZeroUsize: NonZeroI8, NonZeroI16, NonZeroU32, NonZeroI32, NonZeroU64, NonZeroI64, NonZeroU128, NonZeroI128, NonZeroIsize } + +// Non-zero int -> non-zero signed int +nzint_impl_try_from_nzint! { NonZeroI8: NonZeroU8, NonZeroU16, NonZeroI16, NonZeroU32, NonZeroI32, NonZeroU64, NonZeroI64, NonZeroU128, NonZeroI128, NonZeroUsize, NonZeroIsize } +nzint_impl_try_from_nzint! { NonZeroI16: NonZeroU16, NonZeroU32, NonZeroI32, NonZeroU64, NonZeroI64, NonZeroU128, NonZeroI128, NonZeroUsize, NonZeroIsize } +nzint_impl_try_from_nzint! { NonZeroI32: NonZeroU32, NonZeroU64, NonZeroI64, NonZeroU128, NonZeroI128, NonZeroUsize, NonZeroIsize } +nzint_impl_try_from_nzint! { NonZeroI64: NonZeroU64, NonZeroU128, NonZeroI128, NonZeroUsize, NonZeroIsize } +nzint_impl_try_from_nzint! { NonZeroI128: NonZeroU128, NonZeroUsize, NonZeroIsize } +nzint_impl_try_from_nzint! { NonZeroIsize: NonZeroU16, NonZeroU32, NonZeroI32, NonZeroU64, NonZeroI64, NonZeroU128, NonZeroI128, NonZeroUsize } diff --git a/crux-mir/lib/core/src/default.rs b/crux-mir/lib/core/src/default.rs index 15ac3aea8..d96b53de0 100644 --- a/crux-mir/lib/core/src/default.rs +++ b/crux-mir/lib/core/src/default.rs @@ -1,4 +1,4 @@ -//! The `Default` trait for types which may have meaningful default values. +//! The `Default` trait for types with a default value. #![stable(feature = "rust1", since = "1.0.0")] @@ -52,9 +52,26 @@ /// This trait can be used with `#[derive]` if all of the type's fields implement /// `Default`. When `derive`d, it will use the default value for each field's type. /// +/// ### `enum`s +/// +/// When using `#[derive(Default)]` on an `enum`, you need to choose which unit variant will be +/// default. You do this by placing the `#[default]` attribute on the variant. +/// +/// ``` +/// #[derive(Default)] +/// enum Kind { +/// #[default] +/// A, +/// B, +/// C, +/// } +/// ``` +/// +/// You cannot use the `#[default]` attribute on non-unit or non-exhaustive variants. +/// /// ## How can I implement `Default`? /// -/// Provides an implementation for the `default()` method that returns the value of +/// Provide an implementation for the `default()` method that returns the value of /// your type that should be the default: /// /// ``` @@ -80,7 +97,9 @@ /// bar: f32, /// } /// ``` +#[cfg_attr(not(test), rustc_diagnostic_item = "Default")] #[stable(feature = "rust1", since = "1.0.0")] +#[const_trait] pub trait Default: Sized { /// Returns the "default value" for a type. /// @@ -115,8 +134,53 @@ pub trait Default: Sized { fn default() -> Self; } +/// Return the default value of a type according to the `Default` trait. +/// +/// The type to return is inferred from context; this is equivalent to +/// `Default::default()` but shorter to type. +/// +/// For example: +/// ``` +/// #![feature(default_free_fn)] +/// +/// use std::default::default; +/// +/// #[derive(Default)] +/// struct AppConfig { +/// foo: FooConfig, +/// bar: BarConfig, +/// } +/// +/// #[derive(Default)] +/// struct FooConfig { +/// foo: i32, +/// } +/// +/// #[derive(Default)] +/// struct BarConfig { +/// bar: f32, +/// baz: u8, +/// } +/// +/// fn main() { +/// let options = AppConfig { +/// foo: default(), +/// bar: BarConfig { +/// bar: 10.1, +/// ..default() +/// }, +/// }; +/// } +/// ``` +#[unstable(feature = "default_free_fn", issue = "73014")] +#[must_use] +#[inline] +pub fn default() -> T { + Default::default() +} + /// Derive macro generating an impl of the trait `Default`. -#[rustc_builtin_macro] +#[rustc_builtin_macro(Default, attributes(default))] #[stable(feature = "builtin_macro_prelude", since = "1.38.0")] #[allow_internal_unstable(core_intrinsics)] pub macro Default($item:item) { @@ -126,12 +190,15 @@ pub macro Default($item:item) { macro_rules! default_impl { ($t:ty, $v:expr, $doc:tt) => { #[stable(feature = "rust1", since = "1.0.0")] - impl Default for $t { + #[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] + impl const Default for $t { #[inline] #[doc = $doc] - fn default() -> $t { $v } + fn default() -> $t { + $v + } } - } + }; } default_impl! { (), (), "Returns the default value of `()`" } diff --git a/crux-mir/lib/core/src/error.md b/crux-mir/lib/core/src/error.md new file mode 100644 index 000000000..78808d489 --- /dev/null +++ b/crux-mir/lib/core/src/error.md @@ -0,0 +1,137 @@ +Interfaces for working with Errors. + +# Error Handling In Rust + +The Rust language provides two complementary systems for constructing / +representing, reporting, propagating, reacting to, and discarding errors. +These responsibilities are collectively known as "error handling." The +components of the first system, the panic runtime and interfaces, are most +commonly used to represent bugs that have been detected in your program. The +components of the second system, `Result`, the error traits, and user +defined types, are used to represent anticipated runtime failure modes of +your program. + +## The Panic Interfaces + +The following are the primary interfaces of the panic system and the +responsibilities they cover: + +* [`panic!`] and [`panic_any`] (Constructing, Propagated automatically) +* [`PanicInfo`] (Reporting) +* [`set_hook`], [`take_hook`], and [`#[panic_handler]`][panic-handler] (Reporting) +* [`catch_unwind`] and [`resume_unwind`] (Discarding, Propagating) + +The following are the primary interfaces of the error system and the +responsibilities they cover: + +* [`Result`] (Propagating, Reacting) +* The [`Error`] trait (Reporting) +* User defined types (Constructing / Representing) +* [`match`] and [`downcast`] (Reacting) +* The question mark operator ([`?`]) (Propagating) +* The partially stable [`Try`] traits (Propagating, Constructing) +* [`Termination`] (Reporting) + +## Converting Errors into Panics + +The panic and error systems are not entirely distinct. Often times errors +that are anticipated runtime failures in an API might instead represent bugs +to a caller. For these situations the standard library provides APIs for +constructing panics with an `Error` as it's source. + +* [`Result::unwrap`] +* [`Result::expect`] + +These functions are equivalent, they either return the inner value if the +`Result` is `Ok` or panic if the `Result` is `Err` printing the inner error +as the source. The only difference between them is that with `expect` you +provide a panic error message to be printed alongside the source, whereas +`unwrap` has a default message indicating only that you unwrapped an `Err`. + +Of the two, `expect` is generally preferred since its `msg` field allows you +to convey your intent and assumptions which makes tracking down the source +of a panic easier. `unwrap` on the other hand can still be a good fit in +situations where you can trivially show that a piece of code will never +panic, such as `"127.0.0.1".parse::().unwrap()` or early +prototyping. + +# Common Message Styles + +There are two common styles for how people word `expect` messages. Using +the message to present information to users encountering a panic +("expect as error message") or using the message to present information +to developers debugging the panic ("expect as precondition"). + +In the former case the expect message is used to describe the error that +has occurred which is considered a bug. Consider the following example: + +```should_panic +// Read environment variable, panic if it is not present +let path = std::env::var("IMPORTANT_PATH").unwrap(); +``` + +In the "expect as error message" style we would use expect to describe +that the environment variable was not set when it should have been: + +```should_panic +let path = std::env::var("IMPORTANT_PATH") + .expect("env variable `IMPORTANT_PATH` is not set"); +``` + +In the "expect as precondition" style, we would instead describe the +reason we _expect_ the `Result` should be `Ok`. With this style we would +prefer to write: + +```should_panic +let path = std::env::var("IMPORTANT_PATH") + .expect("env variable `IMPORTANT_PATH` should be set by `wrapper_script.sh`"); +``` + +The "expect as error message" style does not work as well with the +default output of the std panic hooks, and often ends up repeating +information that is already communicated by the source error being +unwrapped: + +```text +thread 'main' panicked at 'env variable `IMPORTANT_PATH` is not set: NotPresent', src/main.rs:4:6 +``` + +In this example we end up mentioning that an env variable is not set, +followed by our source message that says the env is not present, the +only additional information we're communicating is the name of the +environment variable being checked. + +The "expect as precondition" style instead focuses on source code +readability, making it easier to understand what must have gone wrong in +situations where panics are being used to represent bugs exclusively. +Also, by framing our expect in terms of what "SHOULD" have happened to +prevent the source error, we end up introducing new information that is +independent from our source error. + +```text +thread 'main' panicked at 'env variable `IMPORTANT_PATH` should be set by `wrapper_script.sh`: NotPresent', src/main.rs:4:6 +``` + +In this example we are communicating not only the name of the +environment variable that should have been set, but also an explanation +for why it should have been set, and we let the source error display as +a clear contradiction to our expectation. + +**Hint**: If you're having trouble remembering how to phrase +expect-as-precondition style error messages remember to focus on the word +"should" as in "env variable should be set by blah" or "the given binary +should be available and executable by the current user". + +[`panic_any`]: ../../std/panic/fn.panic_any.html +[`PanicInfo`]: crate::panic::PanicInfo +[`catch_unwind`]: ../../std/panic/fn.catch_unwind.html +[`resume_unwind`]: ../../std/panic/fn.resume_unwind.html +[`downcast`]: crate::error::Error +[`Termination`]: ../../std/process/trait.Termination.html +[`Try`]: crate::ops::Try +[panic hook]: ../../std/panic/fn.set_hook.html +[`set_hook`]: ../../std/panic/fn.set_hook.html +[`take_hook`]: ../../std/panic/fn.take_hook.html +[panic-handler]: +[`match`]: ../../std/keyword.match.html +[`?`]: ../../std/result/index.html#the-question-mark-operator- diff --git a/crux-mir/lib/core/src/error.rs b/crux-mir/lib/core/src/error.rs new file mode 100644 index 000000000..7152300ab --- /dev/null +++ b/crux-mir/lib/core/src/error.rs @@ -0,0 +1,511 @@ +#![doc = include_str!("error.md")] +#![unstable(feature = "error_in_core", issue = "103765")] + +#[cfg(test)] +mod tests; + +use crate::any::{Demand, Provider, TypeId}; +use crate::fmt::{Debug, Display}; + +/// `Error` is a trait representing the basic expectations for error values, +/// i.e., values of type `E` in [`Result`]. +/// +/// Errors must describe themselves through the [`Display`] and [`Debug`] +/// traits. Error messages are typically concise lowercase sentences without +/// trailing punctuation: +/// +/// ``` +/// let err = "NaN".parse::().unwrap_err(); +/// assert_eq!(err.to_string(), "invalid digit found in string"); +/// ``` +/// +/// Errors may provide cause information. [`Error::source()`] is generally +/// used when errors cross "abstraction boundaries". If one module must report +/// an error that is caused by an error from a lower-level module, it can allow +/// accessing that error via [`Error::source()`]. This makes it possible for the +/// high-level module to provide its own errors while also revealing some of the +/// implementation for debugging. +#[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "Error")] +#[rustc_has_incoherent_inherent_impls] +pub trait Error: Debug + Display { + /// The lower-level source of this error, if any. + /// + /// # Examples + /// + /// ``` + /// use std::error::Error; + /// use std::fmt; + /// + /// #[derive(Debug)] + /// struct SuperError { + /// source: SuperErrorSideKick, + /// } + /// + /// impl fmt::Display for SuperError { + /// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// write!(f, "SuperError is here!") + /// } + /// } + /// + /// impl Error for SuperError { + /// fn source(&self) -> Option<&(dyn Error + 'static)> { + /// Some(&self.source) + /// } + /// } + /// + /// #[derive(Debug)] + /// struct SuperErrorSideKick; + /// + /// impl fmt::Display for SuperErrorSideKick { + /// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// write!(f, "SuperErrorSideKick is here!") + /// } + /// } + /// + /// impl Error for SuperErrorSideKick {} + /// + /// fn get_super_error() -> Result<(), SuperError> { + /// Err(SuperError { source: SuperErrorSideKick }) + /// } + /// + /// fn main() { + /// match get_super_error() { + /// Err(e) => { + /// println!("Error: {e}"); + /// println!("Caused by: {}", e.source().unwrap()); + /// } + /// _ => println!("No error"), + /// } + /// } + /// ``` + #[stable(feature = "error_source", since = "1.30.0")] + fn source(&self) -> Option<&(dyn Error + 'static)> { + None + } + + /// Gets the `TypeId` of `self`. + #[doc(hidden)] + #[unstable( + feature = "error_type_id", + reason = "this is memory-unsafe to override in user code", + issue = "60784" + )] + fn type_id(&self, _: private::Internal) -> TypeId + where + Self: 'static, + { + TypeId::of::() + } + + /// ``` + /// if let Err(e) = "xc".parse::() { + /// // Print `e` itself, no need for description(). + /// eprintln!("Error: {e}"); + /// } + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[deprecated(since = "1.42.0", note = "use the Display impl or to_string()")] + fn description(&self) -> &str { + "description() is deprecated; use Display" + } + + #[stable(feature = "rust1", since = "1.0.0")] + #[deprecated( + since = "1.33.0", + note = "replaced by Error::source, which can support downcasting" + )] + #[allow(missing_docs)] + fn cause(&self) -> Option<&dyn Error> { + self.source() + } + + /// Provides type based access to context intended for error reports. + /// + /// Used in conjunction with [`Demand::provide_value`] and [`Demand::provide_ref`] to extract + /// references to member variables from `dyn Error` trait objects. + /// + /// # Example + /// + /// ```rust + /// #![feature(provide_any)] + /// #![feature(error_generic_member_access)] + /// use core::fmt; + /// use core::any::Demand; + /// + /// #[derive(Debug)] + /// struct MyBacktrace { + /// // ... + /// } + /// + /// impl MyBacktrace { + /// fn new() -> MyBacktrace { + /// // ... + /// # MyBacktrace {} + /// } + /// } + /// + /// #[derive(Debug)] + /// struct SourceError { + /// // ... + /// } + /// + /// impl fmt::Display for SourceError { + /// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// write!(f, "Example Source Error") + /// } + /// } + /// + /// impl std::error::Error for SourceError {} + /// + /// #[derive(Debug)] + /// struct Error { + /// source: SourceError, + /// backtrace: MyBacktrace, + /// } + /// + /// impl fmt::Display for Error { + /// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// write!(f, "Example Error") + /// } + /// } + /// + /// impl std::error::Error for Error { + /// fn provide<'a>(&'a self, demand: &mut Demand<'a>) { + /// demand + /// .provide_ref::(&self.backtrace) + /// .provide_ref::(&self.source); + /// } + /// } + /// + /// fn main() { + /// let backtrace = MyBacktrace::new(); + /// let source = SourceError {}; + /// let error = Error { source, backtrace }; + /// let dyn_error = &error as &dyn std::error::Error; + /// let backtrace_ref = dyn_error.request_ref::().unwrap(); + /// + /// assert!(core::ptr::eq(&error.backtrace, backtrace_ref)); + /// } + /// ``` + #[unstable(feature = "error_generic_member_access", issue = "99301")] + #[allow(unused_variables)] + fn provide<'a>(&'a self, demand: &mut Demand<'a>) {} +} + +#[unstable(feature = "error_generic_member_access", issue = "99301")] +impl Provider for E +where + E: Error + ?Sized, +{ + fn provide<'a>(&'a self, demand: &mut Demand<'a>) { + self.provide(demand) + } +} + +mod private { + // This is a hack to prevent `type_id` from being overridden by `Error` + // implementations, since that can enable unsound downcasting. + #[unstable(feature = "error_type_id", issue = "60784")] + #[derive(Debug)] + pub struct Internal; +} + +#[unstable(feature = "never_type", issue = "35121")] +impl Error for ! {} + +impl<'a> dyn Error + 'a { + /// Request a reference of type `T` as context about this error. + #[unstable(feature = "error_generic_member_access", issue = "99301")] + pub fn request_ref(&'a self) -> Option<&'a T> { + core::any::request_ref(self) + } + + /// Request a value of type `T` as context about this error. + #[unstable(feature = "error_generic_member_access", issue = "99301")] + pub fn request_value(&'a self) -> Option { + core::any::request_value(self) + } +} + +// Copied from `any.rs`. +impl dyn Error + 'static { + /// Returns `true` if the inner type is the same as `T`. + #[stable(feature = "error_downcast", since = "1.3.0")] + #[inline] + pub fn is(&self) -> bool { + // Get `TypeId` of the type this function is instantiated with. + let t = TypeId::of::(); + + // Get `TypeId` of the type in the trait object (`self`). + let concrete = self.type_id(private::Internal); + + // Compare both `TypeId`s on equality. + t == concrete + } + + /// Returns some reference to the inner value if it is of type `T`, or + /// `None` if it isn't. + #[stable(feature = "error_downcast", since = "1.3.0")] + #[inline] + pub fn downcast_ref(&self) -> Option<&T> { + if self.is::() { + // SAFETY: `is` ensures this type cast is correct + unsafe { Some(&*(self as *const dyn Error as *const T)) } + } else { + None + } + } + + /// Returns some mutable reference to the inner value if it is of type `T`, or + /// `None` if it isn't. + #[stable(feature = "error_downcast", since = "1.3.0")] + #[inline] + pub fn downcast_mut(&mut self) -> Option<&mut T> { + if self.is::() { + // SAFETY: `is` ensures this type cast is correct + unsafe { Some(&mut *(self as *mut dyn Error as *mut T)) } + } else { + None + } + } +} + +impl dyn Error + 'static + Send { + /// Forwards to the method defined on the type `dyn Error`. + #[stable(feature = "error_downcast", since = "1.3.0")] + #[inline] + pub fn is(&self) -> bool { + ::is::(self) + } + + /// Forwards to the method defined on the type `dyn Error`. + #[stable(feature = "error_downcast", since = "1.3.0")] + #[inline] + pub fn downcast_ref(&self) -> Option<&T> { + ::downcast_ref::(self) + } + + /// Forwards to the method defined on the type `dyn Error`. + #[stable(feature = "error_downcast", since = "1.3.0")] + #[inline] + pub fn downcast_mut(&mut self) -> Option<&mut T> { + ::downcast_mut::(self) + } + + /// Request a reference of type `T` as context about this error. + #[unstable(feature = "error_generic_member_access", issue = "99301")] + pub fn request_ref(&self) -> Option<&T> { + ::request_ref(self) + } + + /// Request a value of type `T` as context about this error. + #[unstable(feature = "error_generic_member_access", issue = "99301")] + pub fn request_value(&self) -> Option { + ::request_value(self) + } +} + +impl dyn Error + 'static + Send + Sync { + /// Forwards to the method defined on the type `dyn Error`. + #[stable(feature = "error_downcast", since = "1.3.0")] + #[inline] + pub fn is(&self) -> bool { + ::is::(self) + } + + /// Forwards to the method defined on the type `dyn Error`. + #[stable(feature = "error_downcast", since = "1.3.0")] + #[inline] + pub fn downcast_ref(&self) -> Option<&T> { + ::downcast_ref::(self) + } + + /// Forwards to the method defined on the type `dyn Error`. + #[stable(feature = "error_downcast", since = "1.3.0")] + #[inline] + pub fn downcast_mut(&mut self) -> Option<&mut T> { + ::downcast_mut::(self) + } + + /// Request a reference of type `T` as context about this error. + #[unstable(feature = "error_generic_member_access", issue = "99301")] + pub fn request_ref(&self) -> Option<&T> { + ::request_ref(self) + } + + /// Request a value of type `T` as context about this error. + #[unstable(feature = "error_generic_member_access", issue = "99301")] + pub fn request_value(&self) -> Option { + ::request_value(self) + } +} + +impl dyn Error { + /// Returns an iterator starting with the current error and continuing with + /// recursively calling [`Error::source`]. + /// + /// If you want to omit the current error and only use its sources, + /// use `skip(1)`. + /// + /// # Examples + /// + /// ``` + /// #![feature(error_iter)] + /// use std::error::Error; + /// use std::fmt; + /// + /// #[derive(Debug)] + /// struct A; + /// + /// #[derive(Debug)] + /// struct B(Option>); + /// + /// impl fmt::Display for A { + /// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// write!(f, "A") + /// } + /// } + /// + /// impl fmt::Display for B { + /// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// write!(f, "B") + /// } + /// } + /// + /// impl Error for A {} + /// + /// impl Error for B { + /// fn source(&self) -> Option<&(dyn Error + 'static)> { + /// self.0.as_ref().map(|e| e.as_ref()) + /// } + /// } + /// + /// let b = B(Some(Box::new(A))); + /// + /// // let err : Box = b.into(); // or + /// let err = &b as &(dyn Error); + /// + /// let mut iter = err.sources(); + /// + /// assert_eq!("B".to_string(), iter.next().unwrap().to_string()); + /// assert_eq!("A".to_string(), iter.next().unwrap().to_string()); + /// assert!(iter.next().is_none()); + /// assert!(iter.next().is_none()); + /// ``` + #[unstable(feature = "error_iter", issue = "58520")] + #[inline] + pub fn sources(&self) -> Source<'_> { + // You may think this method would be better in the Error trait, and you'd be right. + // Unfortunately that doesn't work, not because of the object safety rules but because we + // save a reference to self in Sources below as a trait object. If this method was + // declared in Error, then self would have the type &T where T is some concrete type which + // implements Error. We would need to coerce self to have type &dyn Error, but that requires + // that Self has a known size (i.e., Self: Sized). We can't put that bound on Error + // since that would forbid Error trait objects, and we can't put that bound on the method + // because that means the method can't be called on trait objects (we'd also need the + // 'static bound, but that isn't allowed because methods with bounds on Self other than + // Sized are not object-safe). Requiring an Unsize bound is not backwards compatible. + + Source { current: Some(self) } + } +} + +/// An iterator over an [`Error`] and its sources. +/// +/// If you want to omit the initial error and only process +/// its sources, use `skip(1)`. +#[unstable(feature = "error_iter", issue = "58520")] +#[derive(Clone, Debug)] +pub struct Source<'a> { + current: Option<&'a (dyn Error + 'static)>, +} + +#[unstable(feature = "error_iter", issue = "58520")] +impl<'a> Iterator for Source<'a> { + type Item = &'a (dyn Error + 'static); + + fn next(&mut self) -> Option { + let current = self.current; + self.current = self.current.and_then(Error::source); + current + } +} + +#[stable(feature = "error_by_ref", since = "1.51.0")] +impl<'a, T: Error + ?Sized> Error for &'a T { + #[allow(deprecated, deprecated_in_future)] + fn description(&self) -> &str { + Error::description(&**self) + } + + #[allow(deprecated)] + fn cause(&self) -> Option<&dyn Error> { + Error::cause(&**self) + } + + fn source(&self) -> Option<&(dyn Error + 'static)> { + Error::source(&**self) + } + + fn provide<'b>(&'b self, demand: &mut Demand<'b>) { + Error::provide(&**self, demand); + } +} + +#[stable(feature = "fmt_error", since = "1.11.0")] +impl Error for crate::fmt::Error { + #[allow(deprecated)] + fn description(&self) -> &str { + "an error occurred when formatting an argument" + } +} + +#[stable(feature = "try_borrow", since = "1.13.0")] +impl Error for crate::cell::BorrowError { + #[allow(deprecated)] + fn description(&self) -> &str { + "already mutably borrowed" + } +} + +#[stable(feature = "try_borrow", since = "1.13.0")] +impl Error for crate::cell::BorrowMutError { + #[allow(deprecated)] + fn description(&self) -> &str { + "already borrowed" + } +} + +#[stable(feature = "try_from", since = "1.34.0")] +impl Error for crate::char::CharTryFromError { + #[allow(deprecated)] + fn description(&self) -> &str { + "converted integer out of range for `char`" + } +} + +#[stable(feature = "char_from_str", since = "1.20.0")] +impl Error for crate::char::ParseCharError { + #[allow(deprecated)] + fn description(&self) -> &str { + self.__description() + } +} + +#[stable(feature = "duration_checked_float", since = "1.66.0")] +impl Error for crate::time::TryFromFloatSecsError {} + +#[stable(feature = "frombyteswithnulerror_impls", since = "1.17.0")] +impl Error for crate::ffi::FromBytesWithNulError { + #[allow(deprecated)] + fn description(&self) -> &str { + self.__description() + } +} + +#[unstable(feature = "cstr_from_bytes_until_nul", issue = "95027")] +impl Error for crate::ffi::FromBytesUntilNulError {} + +#[unstable(feature = "get_many_mut", issue = "104642")] +impl Error for crate::slice::GetManyMutError {} diff --git a/crux-mir/lib/std/src/os/raw/char.md b/crux-mir/lib/core/src/ffi/c_char.md similarity index 58% rename from crux-mir/lib/std/src/os/raw/char.md rename to crux-mir/lib/core/src/ffi/c_char.md index 9a55767d9..b262a3663 100644 --- a/crux-mir/lib/std/src/os/raw/char.md +++ b/crux-mir/lib/core/src/ffi/c_char.md @@ -1,11 +1,8 @@ Equivalent to C's `char` type. -[C's `char` type] is completely unlike [Rust's `char` type]; while Rust's type represents a unicode scalar value, C's `char` type is just an ordinary integer. This type will always be either [`i8`] or [`u8`], as the type is defined as being one byte long. +[C's `char` type] is completely unlike [Rust's `char` type]; while Rust's type represents a unicode scalar value, C's `char` type is just an ordinary integer. On modern architectures this type will always be either [`i8`] or [`u8`], as they use byte-addresses memory with 8-bit bytes. -C chars are most commonly used to make C strings. Unlike Rust, where the length of a string is included alongside the string, C strings mark the end of a string with the character `'\0'`. See [`CStr`] for more information. +C chars are most commonly used to make C strings. Unlike Rust, where the length of a string is included alongside the string, C strings mark the end of a string with the character `'\0'`. See `CStr` for more information. [C's `char` type]: https://en.wikipedia.org/wiki/C_data_types#Basic_types -[Rust's `char` type]: ../../primitive.char.html -[`CStr`]: ../../ffi/struct.CStr.html -[`i8`]: ../../primitive.i8.html -[`u8`]: ../../primitive.u8.html +[Rust's `char` type]: char diff --git a/crux-mir/lib/std/src/os/raw/double.md b/crux-mir/lib/core/src/ffi/c_double.md similarity index 61% rename from crux-mir/lib/std/src/os/raw/double.md rename to crux-mir/lib/core/src/ffi/c_double.md index 6818dada3..d49e29b6e 100644 --- a/crux-mir/lib/std/src/os/raw/double.md +++ b/crux-mir/lib/core/src/ffi/c_double.md @@ -1,7 +1,6 @@ Equivalent to C's `double` type. -This type will almost always be [`f64`], which is guaranteed to be an [IEEE-754 double-precision float] in Rust. That said, the standard technically only guarantees that it be a floating-point number with at least the precision of a [`float`], and it may be `f32` or something entirely different from the IEEE-754 standard. +This type will almost always be [`f64`], which is guaranteed to be an [IEEE 754 double-precision float] in Rust. That said, the standard technically only guarantees that it be a floating-point number with at least the precision of a [`float`], and it may be `f32` or something entirely different from the IEEE-754 standard. -[IEEE-754 double-precision float]: https://en.wikipedia.org/wiki/IEEE_754 -[`float`]: type.c_float.html -[`f64`]: ../../primitive.f64.html +[IEEE 754 double-precision float]: https://en.wikipedia.org/wiki/IEEE_754 +[`float`]: c_float diff --git a/crux-mir/lib/std/src/os/raw/float.md b/crux-mir/lib/core/src/ffi/c_float.md similarity index 62% rename from crux-mir/lib/std/src/os/raw/float.md rename to crux-mir/lib/core/src/ffi/c_float.md index 57d1071d0..36374ef43 100644 --- a/crux-mir/lib/std/src/os/raw/float.md +++ b/crux-mir/lib/core/src/ffi/c_float.md @@ -1,6 +1,5 @@ Equivalent to C's `float` type. -This type will almost always be [`f32`], which is guaranteed to be an [IEEE-754 single-precision float] in Rust. That said, the standard technically only guarantees that it be a floating-point number, and it may have less precision than `f32` or not follow the IEEE-754 standard at all. +This type will almost always be [`f32`], which is guaranteed to be an [IEEE 754 single-precision float] in Rust. That said, the standard technically only guarantees that it be a floating-point number, and it may have less precision than `f32` or not follow the IEEE-754 standard at all. -[IEEE-754 single-precision float]: https://en.wikipedia.org/wiki/IEEE_754 -[`f32`]: ../../primitive.f32.html +[IEEE 754 single-precision float]: https://en.wikipedia.org/wiki/IEEE_754 diff --git a/crux-mir/lib/std/src/os/raw/int.md b/crux-mir/lib/core/src/ffi/c_int.md similarity index 75% rename from crux-mir/lib/std/src/os/raw/int.md rename to crux-mir/lib/core/src/ffi/c_int.md index a0d25fd21..8062ff230 100644 --- a/crux-mir/lib/std/src/os/raw/int.md +++ b/crux-mir/lib/core/src/ffi/c_int.md @@ -2,6 +2,4 @@ Equivalent to C's `signed int` (`int`) type. This type will almost always be [`i32`], but may differ on some esoteric systems. The C standard technically only requires that this type be a signed integer that is at least the size of a [`short`]; some systems define it as an [`i16`], for example. -[`short`]: type.c_short.html -[`i32`]: ../../primitive.i32.html -[`i16`]: ../../primitive.i16.html +[`short`]: c_short diff --git a/crux-mir/lib/std/src/os/raw/long.md b/crux-mir/lib/core/src/ffi/c_long.md similarity index 81% rename from crux-mir/lib/std/src/os/raw/long.md rename to crux-mir/lib/core/src/ffi/c_long.md index c620b4028..cc160783f 100644 --- a/crux-mir/lib/std/src/os/raw/long.md +++ b/crux-mir/lib/core/src/ffi/c_long.md @@ -2,6 +2,4 @@ Equivalent to C's `signed long` (`long`) type. This type will always be [`i32`] or [`i64`]. Most notably, many Linux-based systems assume an `i64`, but Windows assumes `i32`. The C standard technically only requires that this type be a signed integer that is at least 32 bits and at least the size of an [`int`], although in practice, no system would have a `long` that is neither an `i32` nor `i64`. -[`int`]: type.c_int.html -[`i32`]: ../../primitive.i32.html -[`i64`]: ../../primitive.i64.html +[`int`]: c_int diff --git a/crux-mir/lib/std/src/os/raw/longlong.md b/crux-mir/lib/core/src/ffi/c_longlong.md similarity index 80% rename from crux-mir/lib/std/src/os/raw/longlong.md rename to crux-mir/lib/core/src/ffi/c_longlong.md index ab3d64365..49c61bd61 100644 --- a/crux-mir/lib/std/src/os/raw/longlong.md +++ b/crux-mir/lib/core/src/ffi/c_longlong.md @@ -2,6 +2,4 @@ Equivalent to C's `signed long long` (`long long`) type. This type will almost always be [`i64`], but may differ on some systems. The C standard technically only requires that this type be a signed integer that is at least 64 bits and at least the size of a [`long`], although in practice, no system would have a `long long` that is not an `i64`, as most systems do not have a standardised [`i128`] type. -[`long`]: type.c_int.html -[`i64`]: ../../primitive.i64.html -[`i128`]: ../../primitive.i128.html +[`long`]: c_int diff --git a/crux-mir/lib/std/src/os/raw/schar.md b/crux-mir/lib/core/src/ffi/c_schar.md similarity index 75% rename from crux-mir/lib/std/src/os/raw/schar.md rename to crux-mir/lib/core/src/ffi/c_schar.md index 6aa8b1211..69879c9f1 100644 --- a/crux-mir/lib/std/src/os/raw/schar.md +++ b/crux-mir/lib/core/src/ffi/c_schar.md @@ -2,5 +2,4 @@ Equivalent to C's `signed char` type. This type will always be [`i8`], but is included for completeness. It is defined as being a signed integer the same size as a C [`char`]. -[`char`]: type.c_char.html -[`i8`]: ../../primitive.i8.html +[`char`]: c_char diff --git a/crux-mir/lib/std/src/os/raw/short.md b/crux-mir/lib/core/src/ffi/c_short.md similarity index 82% rename from crux-mir/lib/std/src/os/raw/short.md rename to crux-mir/lib/core/src/ffi/c_short.md index be92c6c10..3d1e53d13 100644 --- a/crux-mir/lib/std/src/os/raw/short.md +++ b/crux-mir/lib/core/src/ffi/c_short.md @@ -2,5 +2,4 @@ Equivalent to C's `signed short` (`short`) type. This type will almost always be [`i16`], but may differ on some esoteric systems. The C standard technically only requires that this type be a signed integer with at least 16 bits; some systems may define it as `i32`, for example. -[`char`]: type.c_char.html -[`i16`]: ../../primitive.i16.html +[`char`]: c_char diff --git a/crux-mir/lib/core/src/ffi/c_str.rs b/crux-mir/lib/core/src/ffi/c_str.rs new file mode 100644 index 000000000..15dd9ea7e --- /dev/null +++ b/crux-mir/lib/core/src/ffi/c_str.rs @@ -0,0 +1,666 @@ +use crate::cmp::Ordering; +use crate::ffi::c_char; +use crate::fmt; +use crate::intrinsics; +use crate::ops; +use crate::slice; +use crate::slice::memchr; +use crate::str; + +/// Representation of a borrowed C string. +/// +/// This type represents a borrowed reference to a nul-terminated +/// array of bytes. It can be constructed safely from a &[[u8]] +/// slice, or unsafely from a raw `*const c_char`. It can then be +/// converted to a Rust &[str] by performing UTF-8 validation, or +/// into an owned [`CString`]. +/// +/// `&CStr` is to [`CString`] as &[str] is to [`String`]: the former +/// in each pair are borrowed references; the latter are owned +/// strings. +/// +/// Note that this structure is **not** `repr(C)` and is not recommended to be +/// placed in the signatures of FFI functions. Instead, safe wrappers of FFI +/// functions may leverage the unsafe [`CStr::from_ptr`] constructor to provide +/// a safe interface to other consumers. +/// +/// [`CString`]: ../../std/ffi/struct.CString.html +/// [`String`]: ../../std/string/struct.String.html +/// +/// # Examples +/// +/// Inspecting a foreign C string: +/// +/// ```ignore (extern-declaration) +/// use std::ffi::CStr; +/// use std::os::raw::c_char; +/// +/// extern "C" { fn my_string() -> *const c_char; } +/// +/// unsafe { +/// let slice = CStr::from_ptr(my_string()); +/// println!("string buffer size without nul terminator: {}", slice.to_bytes().len()); +/// } +/// ``` +/// +/// Passing a Rust-originating C string: +/// +/// ```ignore (extern-declaration) +/// use std::ffi::{CString, CStr}; +/// use std::os::raw::c_char; +/// +/// fn work(data: &CStr) { +/// extern "C" { fn work_with(data: *const c_char); } +/// +/// unsafe { work_with(data.as_ptr()) } +/// } +/// +/// let s = CString::new("data data data data").expect("CString::new failed"); +/// work(&s); +/// ``` +/// +/// Converting a foreign C string into a Rust `String`: +/// +/// ```ignore (extern-declaration) +/// use std::ffi::CStr; +/// use std::os::raw::c_char; +/// +/// extern "C" { fn my_string() -> *const c_char; } +/// +/// fn my_string_safe() -> String { +/// let cstr = unsafe { CStr::from_ptr(my_string()) }; +/// // Get copy-on-write Cow<'_, str>, then guarantee a freshly-owned String allocation +/// String::from_utf8_lossy(cstr.to_bytes()).to_string() +/// } +/// +/// println!("string: {}", my_string_safe()); +/// ``` +/// +/// [str]: prim@str "str" +#[derive(Hash)] +#[cfg_attr(not(test), rustc_diagnostic_item = "CStr")] +#[stable(feature = "core_c_str", since = "1.64.0")] +#[rustc_has_incoherent_inherent_impls] +// FIXME: +// `fn from` in `impl From<&CStr> for Box` current implementation relies +// on `CStr` being layout-compatible with `[u8]`. +// When attribute privacy is implemented, `CStr` should be annotated as `#[repr(transparent)]`. +// Anyway, `CStr` representation and layout are considered implementation detail, are +// not documented and must not be relied upon. +pub struct CStr { + // FIXME: this should not be represented with a DST slice but rather with + // just a raw `c_char` along with some form of marker to make + // this an unsized type. Essentially `sizeof(&CStr)` should be the + // same as `sizeof(&c_char)` but `CStr` should be an unsized type. + inner: [c_char], +} + +/// An error indicating that a nul byte was not in the expected position. +/// +/// The slice used to create a [`CStr`] must have one and only one nul byte, +/// positioned at the end. +/// +/// This error is created by the [`CStr::from_bytes_with_nul`] method. +/// See its documentation for more. +/// +/// # Examples +/// +/// ``` +/// use std::ffi::{CStr, FromBytesWithNulError}; +/// +/// let _: FromBytesWithNulError = CStr::from_bytes_with_nul(b"f\0oo").unwrap_err(); +/// ``` +#[derive(Clone, PartialEq, Eq, Debug)] +#[stable(feature = "core_c_str", since = "1.64.0")] +pub struct FromBytesWithNulError { + kind: FromBytesWithNulErrorKind, +} + +#[derive(Clone, PartialEq, Eq, Debug)] +enum FromBytesWithNulErrorKind { + InteriorNul(usize), + NotNulTerminated, +} + +impl FromBytesWithNulError { + const fn interior_nul(pos: usize) -> FromBytesWithNulError { + FromBytesWithNulError { kind: FromBytesWithNulErrorKind::InteriorNul(pos) } + } + const fn not_nul_terminated() -> FromBytesWithNulError { + FromBytesWithNulError { kind: FromBytesWithNulErrorKind::NotNulTerminated } + } + + #[doc(hidden)] + #[unstable(feature = "cstr_internals", issue = "none")] + pub fn __description(&self) -> &str { + match self.kind { + FromBytesWithNulErrorKind::InteriorNul(..) => { + "data provided contains an interior nul byte" + } + FromBytesWithNulErrorKind::NotNulTerminated => "data provided is not nul terminated", + } + } +} + +/// An error indicating that no nul byte was present. +/// +/// A slice used to create a [`CStr`] must contain a nul byte somewhere +/// within the slice. +/// +/// This error is created by the [`CStr::from_bytes_until_nul`] method. +/// +#[derive(Clone, PartialEq, Eq, Debug)] +#[unstable(feature = "cstr_from_bytes_until_nul", issue = "95027")] +pub struct FromBytesUntilNulError(()); + +#[unstable(feature = "cstr_from_bytes_until_nul", issue = "95027")] +impl fmt::Display for FromBytesUntilNulError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "data provided does not contain a nul") + } +} + +#[stable(feature = "cstr_debug", since = "1.3.0")] +impl fmt::Debug for CStr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "\"{}\"", self.to_bytes().escape_ascii()) + } +} + +#[stable(feature = "cstr_default", since = "1.10.0")] +impl Default for &CStr { + fn default() -> Self { + const SLICE: &[c_char] = &[0]; + // SAFETY: `SLICE` is indeed pointing to a valid nul-terminated string. + unsafe { CStr::from_ptr(SLICE.as_ptr()) } + } +} + +#[stable(feature = "frombyteswithnulerror_impls", since = "1.17.0")] +impl fmt::Display for FromBytesWithNulError { + #[allow(deprecated, deprecated_in_future)] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.__description())?; + if let FromBytesWithNulErrorKind::InteriorNul(pos) = self.kind { + write!(f, " at byte pos {pos}")?; + } + Ok(()) + } +} + +impl CStr { + /// Wraps a raw C string with a safe C string wrapper. + /// + /// This function will wrap the provided `ptr` with a `CStr` wrapper, which + /// allows inspection and interoperation of non-owned C strings. The total + /// size of the raw C string must be smaller than `isize::MAX` **bytes** + /// in memory due to calling the `slice::from_raw_parts` function. + /// + /// # Safety + /// + /// * The memory pointed to by `ptr` must contain a valid nul terminator at the + /// end of the string. + /// + /// * `ptr` must be [valid] for reads of bytes up to and including the null terminator. + /// This means in particular: + /// + /// * The entire memory range of this `CStr` must be contained within a single allocated object! + /// * `ptr` must be non-null even for a zero-length cstr. + /// + /// * The memory referenced by the returned `CStr` must not be mutated for + /// the duration of lifetime `'a`. + /// + /// > **Note**: This operation is intended to be a 0-cost cast but it is + /// > currently implemented with an up-front calculation of the length of + /// > the string. This is not guaranteed to always be the case. + /// + /// # Caveat + /// + /// The lifetime for the returned slice is inferred from its usage. To prevent accidental misuse, + /// it's suggested to tie the lifetime to whichever source lifetime is safe in the context, + /// such as by providing a helper function taking the lifetime of a host value for the slice, + /// or by explicit annotation. + /// + /// # Examples + /// + /// ```ignore (extern-declaration) + /// use std::ffi::{c_char, CStr}; + /// + /// extern "C" { + /// fn my_string() -> *const c_char; + /// } + /// + /// unsafe { + /// let slice = CStr::from_ptr(my_string()); + /// println!("string returned: {}", slice.to_str().unwrap()); + /// } + /// ``` + /// + /// ``` + /// #![feature(const_cstr_methods)] + /// + /// use std::ffi::{c_char, CStr}; + /// + /// const HELLO_PTR: *const c_char = { + /// const BYTES: &[u8] = b"Hello, world!\0"; + /// BYTES.as_ptr().cast() + /// }; + /// const HELLO: &CStr = unsafe { CStr::from_ptr(HELLO_PTR) }; + /// ``` + /// + /// [valid]: core::ptr#safety + #[inline] + #[must_use] + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_cstr_methods", issue = "101719")] + pub const unsafe fn from_ptr<'a>(ptr: *const c_char) -> &'a CStr { + // SAFETY: The caller has provided a pointer that points to a valid C + // string with a NUL terminator of size less than `isize::MAX`, whose + // content remain valid and doesn't change for the lifetime of the + // returned `CStr`. + // + // Thus computing the length is fine (a NUL byte exists), the call to + // from_raw_parts is safe because we know the length is at most `isize::MAX`, meaning + // the call to `from_bytes_with_nul_unchecked` is correct. + // + // The cast from c_char to u8 is ok because a c_char is always one byte. + unsafe { + const fn strlen_ct(s: *const c_char) -> usize { + let mut len = 0; + + // SAFETY: Outer caller has provided a pointer to a valid C string. + while unsafe { *s.add(len) } != 0 { + len += 1; + } + + len + } + + fn strlen_rt(s: *const c_char) -> usize { + extern "C" { + /// Provided by libc or compiler_builtins. + fn strlen(s: *const c_char) -> usize; + } + + // SAFETY: Outer caller has provided a pointer to a valid C string. + unsafe { strlen(s) } + } + + let len = intrinsics::const_eval_select((ptr,), strlen_ct, strlen_rt); + Self::from_bytes_with_nul_unchecked(slice::from_raw_parts(ptr.cast(), len + 1)) + } + } + + /// Creates a C string wrapper from a byte slice. + /// + /// This method will create a `CStr` from any byte slice that contains at + /// least one nul byte. The caller does not need to know or specify where + /// the nul byte is located. + /// + /// If the first byte is a nul character, this method will return an + /// empty `CStr`. If multiple nul characters are present, the `CStr` will + /// end at the first one. + /// + /// If the slice only has a single nul byte at the end, this method is + /// equivalent to [`CStr::from_bytes_with_nul`]. + /// + /// # Examples + /// ``` + /// #![feature(cstr_from_bytes_until_nul)] + /// + /// use std::ffi::CStr; + /// + /// let mut buffer = [0u8; 16]; + /// unsafe { + /// // Here we might call an unsafe C function that writes a string + /// // into the buffer. + /// let buf_ptr = buffer.as_mut_ptr(); + /// buf_ptr.write_bytes(b'A', 8); + /// } + /// // Attempt to extract a C nul-terminated string from the buffer. + /// let c_str = CStr::from_bytes_until_nul(&buffer[..]).unwrap(); + /// assert_eq!(c_str.to_str().unwrap(), "AAAAAAAA"); + /// ``` + /// + #[unstable(feature = "cstr_from_bytes_until_nul", issue = "95027")] + #[rustc_const_unstable(feature = "cstr_from_bytes_until_nul", issue = "95027")] + pub const fn from_bytes_until_nul(bytes: &[u8]) -> Result<&CStr, FromBytesUntilNulError> { + let nul_pos = memchr::memchr(0, bytes); + match nul_pos { + Some(nul_pos) => { + let subslice = &bytes[..nul_pos + 1]; + // SAFETY: We know there is a nul byte at nul_pos, so this slice + // (ending at the nul byte) is a well-formed C string. + Ok(unsafe { CStr::from_bytes_with_nul_unchecked(subslice) }) + } + None => Err(FromBytesUntilNulError(())), + } + } + + /// Creates a C string wrapper from a byte slice. + /// + /// This function will cast the provided `bytes` to a `CStr` + /// wrapper after ensuring that the byte slice is nul-terminated + /// and does not contain any interior nul bytes. + /// + /// If the nul byte may not be at the end, + /// [`CStr::from_bytes_until_nul`] can be used instead. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::CStr; + /// + /// let cstr = CStr::from_bytes_with_nul(b"hello\0"); + /// assert!(cstr.is_ok()); + /// ``` + /// + /// Creating a `CStr` without a trailing nul terminator is an error: + /// + /// ``` + /// use std::ffi::CStr; + /// + /// let cstr = CStr::from_bytes_with_nul(b"hello"); + /// assert!(cstr.is_err()); + /// ``` + /// + /// Creating a `CStr` with an interior nul byte is an error: + /// + /// ``` + /// use std::ffi::CStr; + /// + /// let cstr = CStr::from_bytes_with_nul(b"he\0llo\0"); + /// assert!(cstr.is_err()); + /// ``` + #[stable(feature = "cstr_from_bytes", since = "1.10.0")] + #[rustc_const_unstable(feature = "const_cstr_methods", issue = "101719")] + pub const fn from_bytes_with_nul(bytes: &[u8]) -> Result<&Self, FromBytesWithNulError> { + let nul_pos = memchr::memchr(0, bytes); + match nul_pos { + Some(nul_pos) if nul_pos + 1 == bytes.len() => { + // SAFETY: We know there is only one nul byte, at the end + // of the byte slice. + Ok(unsafe { Self::from_bytes_with_nul_unchecked(bytes) }) + } + Some(nul_pos) => Err(FromBytesWithNulError::interior_nul(nul_pos)), + None => Err(FromBytesWithNulError::not_nul_terminated()), + } + } + + /// Unsafely creates a C string wrapper from a byte slice. + /// + /// This function will cast the provided `bytes` to a `CStr` wrapper without + /// performing any sanity checks. + /// + /// # Safety + /// The provided slice **must** be nul-terminated and not contain any interior + /// nul bytes. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::{CStr, CString}; + /// + /// unsafe { + /// let cstring = CString::new("hello").expect("CString::new failed"); + /// let cstr = CStr::from_bytes_with_nul_unchecked(cstring.to_bytes_with_nul()); + /// assert_eq!(cstr, &*cstring); + /// } + /// ``` + #[inline] + #[must_use] + #[stable(feature = "cstr_from_bytes", since = "1.10.0")] + #[rustc_const_stable(feature = "const_cstr_unchecked", since = "1.59.0")] + #[rustc_allow_const_fn_unstable(const_eval_select)] + pub const unsafe fn from_bytes_with_nul_unchecked(bytes: &[u8]) -> &CStr { + #[inline] + fn rt_impl(bytes: &[u8]) -> &CStr { + // Chance at catching some UB at runtime with debug builds. + debug_assert!(!bytes.is_empty() && bytes[bytes.len() - 1] == 0); + + // SAFETY: Casting to CStr is safe because its internal representation + // is a [u8] too (safe only inside std). + // Dereferencing the obtained pointer is safe because it comes from a + // reference. Making a reference is then safe because its lifetime + // is bound by the lifetime of the given `bytes`. + unsafe { &*(bytes as *const [u8] as *const CStr) } + } + + const fn const_impl(bytes: &[u8]) -> &CStr { + // Saturating so that an empty slice panics in the assert with a good + // message, not here due to underflow. + let mut i = bytes.len().saturating_sub(1); + assert!(!bytes.is_empty() && bytes[i] == 0, "input was not nul-terminated"); + + // Ending null byte exists, skip to the rest. + while i != 0 { + i -= 1; + let byte = bytes[i]; + assert!(byte != 0, "input contained interior nul"); + } + + // SAFETY: See `rt_impl` cast. + unsafe { &*(bytes as *const [u8] as *const CStr) } + } + + // SAFETY: The const and runtime versions have identical behavior + // unless the safety contract of `from_bytes_with_nul_unchecked` is + // violated, which is UB. + unsafe { intrinsics::const_eval_select((bytes,), const_impl, rt_impl) } + } + + /// Returns the inner pointer to this C string. + /// + /// The returned pointer will be valid for as long as `self` is, and points + /// to a contiguous region of memory terminated with a 0 byte to represent + /// the end of the string. + /// + /// **WARNING** + /// + /// The returned pointer is read-only; writing to it (including passing it + /// to C code that writes to it) causes undefined behavior. + /// + /// It is your responsibility to make sure that the underlying memory is not + /// freed too early. For example, the following code will cause undefined + /// behavior when `ptr` is used inside the `unsafe` block: + /// + /// ```no_run + /// # #![allow(unused_must_use)] #![allow(temporary_cstring_as_ptr)] + /// use std::ffi::CString; + /// + /// let ptr = CString::new("Hello").expect("CString::new failed").as_ptr(); + /// unsafe { + /// // `ptr` is dangling + /// *ptr; + /// } + /// ``` + /// + /// This happens because the pointer returned by `as_ptr` does not carry any + /// lifetime information and the `CString` is deallocated immediately after + /// the `CString::new("Hello").expect("CString::new failed").as_ptr()` + /// expression is evaluated. + /// To fix the problem, bind the `CString` to a local variable: + /// + /// ```no_run + /// # #![allow(unused_must_use)] + /// use std::ffi::CString; + /// + /// let hello = CString::new("Hello").expect("CString::new failed"); + /// let ptr = hello.as_ptr(); + /// unsafe { + /// // `ptr` is valid because `hello` is in scope + /// *ptr; + /// } + /// ``` + /// + /// This way, the lifetime of the `CString` in `hello` encompasses + /// the lifetime of `ptr` and the `unsafe` block. + #[inline] + #[must_use] + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_str_as_ptr", since = "1.32.0")] + pub const fn as_ptr(&self) -> *const c_char { + self.inner.as_ptr() + } + + /// Returns `true` if `self.to_bytes()` has a length of 0. + /// + /// # Examples + /// + /// ``` + /// #![feature(cstr_is_empty)] + /// + /// use std::ffi::CStr; + /// # use std::ffi::FromBytesWithNulError; + /// + /// # fn main() { test().unwrap(); } + /// # fn test() -> Result<(), FromBytesWithNulError> { + /// let cstr = CStr::from_bytes_with_nul(b"foo\0")?; + /// assert!(!cstr.is_empty()); + /// + /// let empty_cstr = CStr::from_bytes_with_nul(b"\0")?; + /// assert!(empty_cstr.is_empty()); + /// # Ok(()) + /// # } + /// ``` + #[inline] + #[unstable(feature = "cstr_is_empty", issue = "102444")] + pub const fn is_empty(&self) -> bool { + // SAFETY: We know there is at least one byte; for empty strings it + // is the NUL terminator. + (unsafe { self.inner.get_unchecked(0) }) == &0 + } + + /// Converts this C string to a byte slice. + /// + /// The returned slice will **not** contain the trailing nul terminator that this C + /// string has. + /// + /// > **Note**: This method is currently implemented as a constant-time + /// > cast, but it is planned to alter its definition in the future to + /// > perform the length calculation whenever this method is called. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::CStr; + /// + /// let cstr = CStr::from_bytes_with_nul(b"foo\0").expect("CStr::from_bytes_with_nul failed"); + /// assert_eq!(cstr.to_bytes(), b"foo"); + /// ``` + #[inline] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_cstr_methods", issue = "101719")] + pub const fn to_bytes(&self) -> &[u8] { + let bytes = self.to_bytes_with_nul(); + // SAFETY: to_bytes_with_nul returns slice with length at least 1 + unsafe { bytes.get_unchecked(..bytes.len() - 1) } + } + + /// Converts this C string to a byte slice containing the trailing 0 byte. + /// + /// This function is the equivalent of [`CStr::to_bytes`] except that it + /// will retain the trailing nul terminator instead of chopping it off. + /// + /// > **Note**: This method is currently implemented as a 0-cost cast, but + /// > it is planned to alter its definition in the future to perform the + /// > length calculation whenever this method is called. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::CStr; + /// + /// let cstr = CStr::from_bytes_with_nul(b"foo\0").expect("CStr::from_bytes_with_nul failed"); + /// assert_eq!(cstr.to_bytes_with_nul(), b"foo\0"); + /// ``` + #[inline] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_cstr_methods", issue = "101719")] + pub const fn to_bytes_with_nul(&self) -> &[u8] { + // SAFETY: Transmuting a slice of `c_char`s to a slice of `u8`s + // is safe on all supported targets. + unsafe { &*(&self.inner as *const [c_char] as *const [u8]) } + } + + /// Yields a &[str] slice if the `CStr` contains valid UTF-8. + /// + /// If the contents of the `CStr` are valid UTF-8 data, this + /// function will return the corresponding &[str] slice. Otherwise, + /// it will return an error with details of where UTF-8 validation failed. + /// + /// [str]: prim@str "str" + /// + /// # Examples + /// + /// ``` + /// use std::ffi::CStr; + /// + /// let cstr = CStr::from_bytes_with_nul(b"foo\0").expect("CStr::from_bytes_with_nul failed"); + /// assert_eq!(cstr.to_str(), Ok("foo")); + /// ``` + #[stable(feature = "cstr_to_str", since = "1.4.0")] + #[rustc_const_unstable(feature = "const_cstr_methods", issue = "101719")] + pub const fn to_str(&self) -> Result<&str, str::Utf8Error> { + // N.B., when `CStr` is changed to perform the length check in `.to_bytes()` + // instead of in `from_ptr()`, it may be worth considering if this should + // be rewritten to do the UTF-8 check inline with the length calculation + // instead of doing it afterwards. + str::from_utf8(self.to_bytes()) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialEq for CStr { + fn eq(&self, other: &CStr) -> bool { + self.to_bytes().eq(other.to_bytes()) + } +} +#[stable(feature = "rust1", since = "1.0.0")] +impl Eq for CStr {} +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialOrd for CStr { + fn partial_cmp(&self, other: &CStr) -> Option { + self.to_bytes().partial_cmp(&other.to_bytes()) + } +} +#[stable(feature = "rust1", since = "1.0.0")] +impl Ord for CStr { + fn cmp(&self, other: &CStr) -> Ordering { + self.to_bytes().cmp(&other.to_bytes()) + } +} + +#[stable(feature = "cstr_range_from", since = "1.47.0")] +impl ops::Index> for CStr { + type Output = CStr; + + fn index(&self, index: ops::RangeFrom) -> &CStr { + let bytes = self.to_bytes_with_nul(); + // we need to manually check the starting index to account for the null + // byte, since otherwise we could get an empty string that doesn't end + // in a null. + if index.start < bytes.len() { + // SAFETY: Non-empty tail of a valid `CStr` is still a valid `CStr`. + unsafe { CStr::from_bytes_with_nul_unchecked(&bytes[index.start..]) } + } else { + panic!( + "index out of bounds: the len is {} but the index is {}", + bytes.len(), + index.start + ); + } + } +} + +#[stable(feature = "cstring_asref", since = "1.7.0")] +impl AsRef for CStr { + #[inline] + fn as_ref(&self) -> &CStr { + self + } +} diff --git a/crux-mir/lib/std/src/os/raw/uchar.md b/crux-mir/lib/core/src/ffi/c_uchar.md similarity index 75% rename from crux-mir/lib/std/src/os/raw/uchar.md rename to crux-mir/lib/core/src/ffi/c_uchar.md index b6ca711f8..b633bb7f8 100644 --- a/crux-mir/lib/std/src/os/raw/uchar.md +++ b/crux-mir/lib/core/src/ffi/c_uchar.md @@ -2,5 +2,4 @@ Equivalent to C's `unsigned char` type. This type will always be [`u8`], but is included for completeness. It is defined as being an unsigned integer the same size as a C [`char`]. -[`char`]: type.c_char.html -[`u8`]: ../../primitive.u8.html +[`char`]: c_char diff --git a/crux-mir/lib/std/src/os/raw/uint.md b/crux-mir/lib/core/src/ffi/c_uint.md similarity index 75% rename from crux-mir/lib/std/src/os/raw/uint.md rename to crux-mir/lib/core/src/ffi/c_uint.md index 6f7013a8a..f3abea359 100644 --- a/crux-mir/lib/std/src/os/raw/uint.md +++ b/crux-mir/lib/core/src/ffi/c_uint.md @@ -2,6 +2,4 @@ Equivalent to C's `unsigned int` type. This type will almost always be [`u32`], but may differ on some esoteric systems. The C standard technically only requires that this type be an unsigned integer with the same size as an [`int`]; some systems define it as a [`u16`], for example. -[`int`]: type.c_int.html -[`u32`]: ../../primitive.u32.html -[`u16`]: ../../primitive.u16.html +[`int`]: c_int diff --git a/crux-mir/lib/std/src/os/raw/ulong.md b/crux-mir/lib/core/src/ffi/c_ulong.md similarity index 79% rename from crux-mir/lib/std/src/os/raw/ulong.md rename to crux-mir/lib/core/src/ffi/c_ulong.md index c35039508..4ab304e65 100644 --- a/crux-mir/lib/std/src/os/raw/ulong.md +++ b/crux-mir/lib/core/src/ffi/c_ulong.md @@ -2,6 +2,4 @@ Equivalent to C's `unsigned long` type. This type will always be [`u32`] or [`u64`]. Most notably, many Linux-based systems assume an `u64`, but Windows assumes `u32`. The C standard technically only requires that this type be an unsigned integer with the size of a [`long`], although in practice, no system would have a `ulong` that is neither a `u32` nor `u64`. -[`long`]: type.c_long.html -[`u32`]: ../../primitive.u32.html -[`u64`]: ../../primitive.u64.html +[`long`]: c_long diff --git a/crux-mir/lib/std/src/os/raw/ulonglong.md b/crux-mir/lib/core/src/ffi/c_ulonglong.md similarity index 77% rename from crux-mir/lib/std/src/os/raw/ulonglong.md rename to crux-mir/lib/core/src/ffi/c_ulonglong.md index c41faf74c..a27d70e17 100644 --- a/crux-mir/lib/std/src/os/raw/ulonglong.md +++ b/crux-mir/lib/core/src/ffi/c_ulonglong.md @@ -2,6 +2,4 @@ Equivalent to C's `unsigned long long` type. This type will almost always be [`u64`], but may differ on some systems. The C standard technically only requires that this type be an unsigned integer with the size of a [`long long`], although in practice, no system would have a `long long` that is not a `u64`, as most systems do not have a standardised [`u128`] type. -[`long long`]: type.c_longlong.html -[`u64`]: ../../primitive.u64.html -[`u128`]: ../../primitive.u128.html +[`long long`]: c_longlong diff --git a/crux-mir/lib/std/src/os/raw/ushort.md b/crux-mir/lib/core/src/ffi/c_ushort.md similarity index 79% rename from crux-mir/lib/std/src/os/raw/ushort.md rename to crux-mir/lib/core/src/ffi/c_ushort.md index d364abb3c..6928e51b3 100644 --- a/crux-mir/lib/std/src/os/raw/ushort.md +++ b/crux-mir/lib/core/src/ffi/c_ushort.md @@ -2,5 +2,4 @@ Equivalent to C's `unsigned short` type. This type will almost always be [`u16`], but may differ on some esoteric systems. The C standard technically only requires that this type be an unsigned integer with the same size as a [`short`]. -[`short`]: type.c_short.html -[`u16`]: ../../primitive.u16.html +[`short`]: c_short diff --git a/crux-mir/lib/core/src/ffi/c_void.md b/crux-mir/lib/core/src/ffi/c_void.md new file mode 100644 index 000000000..ee7403aa0 --- /dev/null +++ b/crux-mir/lib/core/src/ffi/c_void.md @@ -0,0 +1,16 @@ +Equivalent to C's `void` type when used as a [pointer]. + +In essence, `*const c_void` is equivalent to C's `const void*` +and `*mut c_void` is equivalent to C's `void*`. That said, this is +*not* the same as C's `void` return type, which is Rust's `()` type. + +To model pointers to opaque types in FFI, until `extern type` is +stabilized, it is recommended to use a newtype wrapper around an empty +byte array. See the [Nomicon] for details. + +One could use `std::os::raw::c_void` if they want to support old Rust +compiler down to 1.1.0. After Rust 1.30.0, it was re-exported by +this definition. For more information, please read [RFC 2521]. + +[Nomicon]: https://doc.rust-lang.org/nomicon/ffi.html#representing-opaque-structs +[RFC 2521]: https://github.com/rust-lang/rfcs/blob/master/text/2521-c_void-reunification.md diff --git a/crux-mir/lib/core/src/ffi.rs b/crux-mir/lib/core/src/ffi/mod.rs similarity index 51% rename from crux-mir/lib/core/src/ffi.rs rename to crux-mir/lib/core/src/ffi/mod.rs index 6277da4f1..76daceecd 100644 --- a/crux-mir/lib/core/src/ffi.rs +++ b/crux-mir/lib/core/src/ffi/mod.rs @@ -1,29 +1,194 @@ +//! Platform-specific types, as defined by C. +//! +//! Code that interacts via FFI will almost certainly be using the +//! base types provided by C, which aren't nearly as nicely defined +//! as Rust's primitive types. This module provides types which will +//! match those defined by C, so that code that interacts with C will +//! refer to the correct types. + #![stable(feature = "", since = "1.30.0")] #![allow(non_camel_case_types)] -//! Utilities related to FFI bindings. - use crate::fmt; use crate::marker::PhantomData; +use crate::num::*; use crate::ops::{Deref, DerefMut}; -/// Equivalent to C's `void` type when used as a [pointer]. -/// -/// In essence, `*const c_void` is equivalent to C's `const void*` -/// and `*mut c_void` is equivalent to C's `void*`. That said, this is -/// *not* the same as C's `void` return type, which is Rust's `()` type. +#[stable(feature = "core_c_str", since = "1.64.0")] +pub use self::c_str::{CStr, FromBytesUntilNulError, FromBytesWithNulError}; + +mod c_str; + +macro_rules! type_alias_no_nz { + { + $Docfile:tt, $Alias:ident = $Real:ty; + $( $Cfg:tt )* + } => { + #[doc = include_str!($Docfile)] + $( $Cfg )* + #[stable(feature = "core_ffi_c", since = "1.64.0")] + pub type $Alias = $Real; + } +} + +// To verify that the NonZero types in this file's macro invocations correspond +// +// perl -n < library/std/src/os/raw/mod.rs -e 'next unless m/type_alias\!/; die "$_ ?" unless m/, (c_\w+) = (\w+), NonZero_(\w+) = NonZero(\w+)/; die "$_ ?" unless $3 eq $1 and $4 eq ucfirst $2' +// +// NB this does not check that the main c_* types are right. + +macro_rules! type_alias { + { + $Docfile:tt, $Alias:ident = $Real:ty, $NZAlias:ident = $NZReal:ty; + $( $Cfg:tt )* + } => { + type_alias_no_nz! { $Docfile, $Alias = $Real; $( $Cfg )* } + + #[doc = concat!("Type alias for `NonZero` version of [`", stringify!($Alias), "`]")] + #[unstable(feature = "raw_os_nonzero", issue = "82363")] + $( $Cfg )* + pub type $NZAlias = $NZReal; + } +} + +type_alias! { "c_char.md", c_char = c_char_definition::c_char, NonZero_c_char = c_char_definition::NonZero_c_char; +// Make this type alias appear cfg-dependent so that Clippy does not suggest +// replacing `0 as c_char` with `0_i8`/`0_u8`. This #[cfg(all())] can be removed +// after the false positive in https://github.com/rust-lang/rust-clippy/issues/8093 +// is fixed. +#[cfg(all())] +#[doc(cfg(all()))] } + +type_alias! { "c_schar.md", c_schar = i8, NonZero_c_schar = NonZeroI8; } +type_alias! { "c_uchar.md", c_uchar = u8, NonZero_c_uchar = NonZeroU8; } +type_alias! { "c_short.md", c_short = i16, NonZero_c_short = NonZeroI16; } +type_alias! { "c_ushort.md", c_ushort = u16, NonZero_c_ushort = NonZeroU16; } + +type_alias! { "c_int.md", c_int = c_int_definition::c_int, NonZero_c_int = c_int_definition::NonZero_c_int; +#[doc(cfg(all()))] } +type_alias! { "c_uint.md", c_uint = c_int_definition::c_uint, NonZero_c_uint = c_int_definition::NonZero_c_uint; +#[doc(cfg(all()))] } + +type_alias! { "c_long.md", c_long = c_long_definition::c_long, NonZero_c_long = c_long_definition::NonZero_c_long; +#[doc(cfg(all()))] } +type_alias! { "c_ulong.md", c_ulong = c_long_definition::c_ulong, NonZero_c_ulong = c_long_definition::NonZero_c_ulong; +#[doc(cfg(all()))] } + +type_alias! { "c_longlong.md", c_longlong = i64, NonZero_c_longlong = NonZeroI64; } +type_alias! { "c_ulonglong.md", c_ulonglong = u64, NonZero_c_ulonglong = NonZeroU64; } + +type_alias_no_nz! { "c_float.md", c_float = f32; } +type_alias_no_nz! { "c_double.md", c_double = f64; } + +/// Equivalent to C's `size_t` type, from `stddef.h` (or `cstddef` for C++). /// -/// To model pointers to opaque types in FFI, until `extern type` is -/// stabilized, it is recommended to use a newtype wrapper around an empty -/// byte array. See the [Nomicon] for details. +/// This type is currently always [`usize`], however in the future there may be +/// platforms where this is not the case. +#[unstable(feature = "c_size_t", issue = "88345")] +pub type c_size_t = usize; + +/// Equivalent to C's `ptrdiff_t` type, from `stddef.h` (or `cstddef` for C++). /// -/// One could use `std::os::raw::c_void` if they want to support old Rust -/// compiler down to 1.1.0. After Rust 1.30.0, it was re-exported by -/// this definition. For more information, please read [RFC 2521]. +/// This type is currently always [`isize`], however in the future there may be +/// platforms where this is not the case. +#[unstable(feature = "c_size_t", issue = "88345")] +pub type c_ptrdiff_t = isize; + +/// Equivalent to C's `ssize_t` (on POSIX) or `SSIZE_T` (on Windows) type. /// -/// [pointer]: ../../std/primitive.pointer.html -/// [Nomicon]: https://doc.rust-lang.org/nomicon/ffi.html#representing-opaque-structs -/// [RFC 2521]: https://github.com/rust-lang/rfcs/blob/master/text/2521-c_void-reunification.md +/// This type is currently always [`isize`], however in the future there may be +/// platforms where this is not the case. +#[unstable(feature = "c_size_t", issue = "88345")] +pub type c_ssize_t = isize; + +mod c_char_definition { + cfg_if! { + // These are the targets on which c_char is unsigned. + if #[cfg(any( + all( + target_os = "linux", + any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "hexagon", + target_arch = "powerpc", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "riscv64", + target_arch = "riscv32" + ) + ), + all(target_os = "android", any(target_arch = "aarch64", target_arch = "arm")), + all(target_os = "l4re", target_arch = "x86_64"), + all( + any(target_os = "freebsd", target_os = "openbsd"), + any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc", + target_arch = "powerpc64", + target_arch = "riscv64" + ) + ), + all( + target_os = "netbsd", + any(target_arch = "aarch64", target_arch = "arm", target_arch = "powerpc") + ), + all( + target_os = "vxworks", + any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc64", + target_arch = "powerpc" + ) + ), + all(target_os = "fuchsia", target_arch = "aarch64"), + target_os = "horizon" + ))] { + pub type c_char = u8; + pub type NonZero_c_char = crate::num::NonZeroU8; + } else { + // On every other target, c_char is signed. + pub type c_char = i8; + pub type NonZero_c_char = crate::num::NonZeroI8; + } + } +} + +mod c_int_definition { + cfg_if! { + if #[cfg(any(target_arch = "avr", target_arch = "msp430"))] { + pub type c_int = i16; + pub type NonZero_c_int = crate::num::NonZeroI16; + pub type c_uint = u16; + pub type NonZero_c_uint = crate::num::NonZeroU16; + } else { + pub type c_int = i32; + pub type NonZero_c_int = crate::num::NonZeroI32; + pub type c_uint = u32; + pub type NonZero_c_uint = crate::num::NonZeroU32; + } + } +} + +mod c_long_definition { + cfg_if! { + if #[cfg(all(target_pointer_width = "64", not(windows)))] { + pub type c_long = i64; + pub type NonZero_c_long = crate::num::NonZeroI64; + pub type c_ulong = u64; + pub type NonZero_c_ulong = crate::num::NonZeroU64; + } else { + // The minimal size of `long` in the C standard is 32 bits + pub type c_long = i32; + pub type NonZero_c_long = crate::num::NonZeroI32; + pub type c_ulong = u32; + pub type NonZero_c_ulong = crate::num::NonZeroU32; + } + } +} + // N.B., for LLVM to recognize the void pointer type and by extension // functions like malloc(), we need to have it represented as i8* in // LLVM bitcode. The enum used here ensures this and prevents misuse @@ -32,6 +197,7 @@ use crate::ops::{Deref, DerefMut}; // otherwise and we need at least one variant as otherwise the enum // would be uninhabited and at least dereferencing such pointers would // be UB. +#[doc = include_str!("c_void.md")] #[repr(u8)] #[stable(feature = "core_c_void", since = "1.30.0")] pub enum c_void { @@ -54,18 +220,24 @@ pub enum c_void { #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for c_void { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("c_void") + f.debug_struct("c_void").finish() } } /// Basic implementation of a `va_list`. // The name is WIP, using `VaListImpl` for now. #[cfg(any( - all(not(target_arch = "aarch64"), not(target_arch = "powerpc"), not(target_arch = "x86_64")), - all(target_arch = "aarch64", target_os = "ios"), - target_arch = "wasm32", + all( + not(target_arch = "aarch64"), + not(target_arch = "powerpc"), + not(target_arch = "s390x"), + not(target_arch = "x86_64") + ), + all(target_arch = "aarch64", any(target_os = "macos", target_os = "ios")), + target_family = "wasm", target_arch = "asmjs", - windows + target_os = "uefi", + windows, ))] #[repr(transparent)] #[unstable( @@ -84,11 +256,17 @@ pub struct VaListImpl<'f> { } #[cfg(any( - all(not(target_arch = "aarch64"), not(target_arch = "powerpc"), not(target_arch = "x86_64")), - all(target_arch = "aarch64", target_os = "ios"), - target_arch = "wasm32", + all( + not(target_arch = "aarch64"), + not(target_arch = "powerpc"), + not(target_arch = "s390x"), + not(target_arch = "x86_64") + ), + all(target_arch = "aarch64", any(target_os = "macos", target_os = "ios")), + target_family = "wasm", target_arch = "asmjs", - windows + target_os = "uefi", + windows, ))] #[unstable( feature = "c_variadic", @@ -107,7 +285,12 @@ impl<'f> fmt::Debug for VaListImpl<'f> { /// /// [AArch64 Procedure Call Standard]: /// http://infocenter.arm.com/help/topic/com.arm.doc.ihi0055b/IHI0055B_aapcs64.pdf -#[cfg(all(target_arch = "aarch64", not(target_os = "ios"), not(windows)))] +#[cfg(all( + target_arch = "aarch64", + not(any(target_os = "macos", target_os = "ios")), + not(target_os = "uefi"), + not(windows), +))] #[repr(C)] #[derive(Debug)] #[unstable( @@ -127,7 +310,7 @@ pub struct VaListImpl<'f> { } /// PowerPC ABI implementation of a `va_list`. -#[cfg(all(target_arch = "powerpc", not(windows)))] +#[cfg(all(target_arch = "powerpc", not(target_os = "uefi"), not(windows)))] #[repr(C)] #[derive(Debug)] #[unstable( @@ -146,8 +329,27 @@ pub struct VaListImpl<'f> { _marker: PhantomData<&'f mut &'f c_void>, } +/// s390x ABI implementation of a `va_list`. +#[cfg(target_arch = "s390x")] +#[repr(C)] +#[derive(Debug)] +#[unstable( + feature = "c_variadic", + reason = "the `c_variadic` feature has not been properly tested on \ + all supported platforms", + issue = "44930" +)] +#[lang = "va_list"] +pub struct VaListImpl<'f> { + gpr: i64, + fpr: i64, + overflow_arg_area: *mut c_void, + reg_save_area: *mut c_void, + _marker: PhantomData<&'f mut &'f c_void>, +} + /// x86_64 ABI implementation of a `va_list`. -#[cfg(all(target_arch = "x86_64", not(windows)))] +#[cfg(all(target_arch = "x86_64", not(target_os = "uefi"), not(windows)))] #[repr(C)] #[derive(Debug)] #[unstable( @@ -179,21 +381,29 @@ pub struct VaList<'a, 'f: 'a> { all( not(target_arch = "aarch64"), not(target_arch = "powerpc"), + not(target_arch = "s390x"), not(target_arch = "x86_64") ), - all(target_arch = "aarch64", target_os = "ios"), - target_arch = "wasm32", + all(target_arch = "aarch64", any(target_os = "macos", target_os = "ios")), + target_family = "wasm", target_arch = "asmjs", - windows + target_os = "uefi", + windows, ))] inner: VaListImpl<'f>, #[cfg(all( - any(target_arch = "aarch64", target_arch = "powerpc", target_arch = "x86_64"), - any(not(target_arch = "aarch64"), not(target_os = "ios")), - not(target_arch = "wasm32"), + any( + target_arch = "aarch64", + target_arch = "powerpc", + target_arch = "s390x", + target_arch = "x86_64" + ), + any(not(target_arch = "aarch64"), not(any(target_os = "macos", target_os = "ios"))), + not(target_family = "wasm"), not(target_arch = "asmjs"), - not(windows) + not(target_os = "uefi"), + not(windows), ))] inner: &'a mut VaListImpl<'f>, @@ -201,11 +411,17 @@ pub struct VaList<'a, 'f: 'a> { } #[cfg(any( - all(not(target_arch = "aarch64"), not(target_arch = "powerpc"), not(target_arch = "x86_64")), - all(target_arch = "aarch64", target_os = "ios"), - target_arch = "wasm32", + all( + not(target_arch = "aarch64"), + not(target_arch = "powerpc"), + not(target_arch = "s390x"), + not(target_arch = "x86_64") + ), + all(target_arch = "aarch64", any(target_os = "macos", target_os = "ios")), + target_family = "wasm", target_arch = "asmjs", - windows + target_os = "uefi", + windows, ))] #[unstable( feature = "c_variadic", @@ -222,11 +438,17 @@ impl<'f> VaListImpl<'f> { } #[cfg(all( - any(target_arch = "aarch64", target_arch = "powerpc", target_arch = "x86_64"), - any(not(target_arch = "aarch64"), not(target_os = "ios")), - not(target_arch = "wasm32"), + any( + target_arch = "aarch64", + target_arch = "powerpc", + target_arch = "s390x", + target_arch = "x86_64" + ), + any(not(target_arch = "aarch64"), not(any(target_os = "macos", target_os = "ios"))), + not(target_family = "wasm"), not(target_arch = "asmjs"), - not(windows) + not(target_os = "uefi"), + not(windows), ))] #[unstable( feature = "c_variadic", @@ -280,9 +502,7 @@ impl<'a, 'f: 'a> DerefMut for VaList<'a, 'f> { // within a private module. Once RFC 2145 has been implemented look into // improving this. mod sealed_trait { - /// Trait which whitelists the allowed types to be used with [VaList::arg] - /// - /// [VaList::va_arg]: struct.VaList.html#method.arg + /// Trait which permits the allowed types to be used with [super::VaListImpl::arg]. #[unstable( feature = "c_variadic", reason = "the `c_variadic` feature has not been properly tested on \ @@ -333,7 +553,8 @@ impl<'f> VaListImpl<'f> { /// Advance to the next arg. #[inline] pub unsafe fn arg(&mut self) -> T { - va_arg(self) + // SAFETY: the caller must uphold the safety contract for `va_arg`. + unsafe { va_arg(self) } } /// Copies the `va_list` at the current location. @@ -343,7 +564,10 @@ impl<'f> VaListImpl<'f> { { let mut ap = self.clone(); let ret = f(ap.as_va_list()); - va_end(&mut ap); + // SAFETY: the caller must uphold the safety contract for `va_end`. + unsafe { + va_end(&mut ap); + } ret } } diff --git a/crux-mir/lib/core/src/fmt/builders.rs b/crux-mir/lib/core/src/fmt/builders.rs index 63866a5d1..7da49b04a 100644 --- a/crux-mir/lib/core/src/fmt/builders.rs +++ b/crux-mir/lib/core/src/fmt/builders.rs @@ -1,4 +1,6 @@ -use crate::fmt; +#![allow(unused_imports)] + +use crate::fmt::{self, Debug, Formatter}; struct PadAdapter<'buf, 'state> { buf: &'buf mut (dyn fmt::Write + 'buf), @@ -21,46 +23,31 @@ impl<'buf, 'state> PadAdapter<'buf, 'state> { slot: &'slot mut Option, state: &'state mut PadAdapterState, ) -> fmt::Formatter<'slot> { - fmt.wrap_buf(move |buf| { - *slot = Some(PadAdapter { buf, state }); - slot.as_mut().unwrap() - }) + fmt.wrap_buf(move |buf| slot.insert(PadAdapter { buf, state })) } } impl fmt::Write for PadAdapter<'_, '_> { - fn write_str(&mut self, mut s: &str) -> fmt::Result { - while !s.is_empty() { + fn write_str(&mut self, s: &str) -> fmt::Result { + for s in s.split_inclusive('\n') { if self.state.on_newline { self.buf.write_str(" ")?; } - let split = match s.find('\n') { - Some(pos) => { - self.state.on_newline = true; - pos + 1 - } - None => { - self.state.on_newline = false; - s.len() - } - }; - self.buf.write_str(&s[..split])?; - s = &s[split..]; + self.state.on_newline = s.ends_with('\n'); + self.buf.write_str(s)?; } Ok(()) } } -/// A struct to help with [`fmt::Debug`](trait.Debug.html) implementations. +/// A struct to help with [`fmt::Debug`](Debug) implementations. /// /// This is useful when you wish to output a formatted struct as a part of your -/// [`Debug::fmt`](trait.Debug.html#tymethod.fmt) implementation. +/// [`Debug::fmt`] implementation. /// -/// This can be constructed by the -/// [`Formatter::debug_struct`](struct.Formatter.html#method.debug_struct) -/// method. +/// This can be constructed by the [`Formatter::debug_struct`] method. /// /// # Examples /// @@ -141,7 +128,7 @@ impl<'a, 'b: 'a> DebugStruct<'a, 'b> { } let mut slot = None; let mut state = Default::default(); - let mut writer = PadAdapter::wrap(&mut self.fmt, &mut slot, &mut state); + let mut writer = PadAdapter::wrap(self.fmt, &mut slot, &mut state); writer.write_str(name)?; writer.write_str(": ")?; value.fmt(&mut writer)?; @@ -165,7 +152,6 @@ impl<'a, 'b: 'a> DebugStruct<'a, 'b> { /// # Examples /// /// ``` - /// # #![feature(debug_non_exhaustive)] /// use std::fmt; /// /// struct Bar { @@ -186,31 +172,22 @@ impl<'a, 'b: 'a> DebugStruct<'a, 'b> { /// "Bar { bar: 10, .. }", /// ); /// ``` - #[unstable(feature = "debug_non_exhaustive", issue = "67364")] + #[stable(feature = "debug_non_exhaustive", since = "1.53.0")] pub fn finish_non_exhaustive(&mut self) -> fmt::Result { self.result = self.result.and_then(|_| { - // Draw non-exhaustive dots (`..`), and open brace if necessary (no fields). - if self.is_pretty() { - if !self.has_fields { - self.fmt.write_str(" {\n")?; - } - let mut slot = None; - let mut state = Default::default(); - let mut writer = PadAdapter::wrap(&mut self.fmt, &mut slot, &mut state); - writer.write_str("..\n")?; - } else { - if self.has_fields { - self.fmt.write_str(", ..")?; + if self.has_fields { + if self.is_pretty() { + let mut slot = None; + let mut state = Default::default(); + let mut writer = PadAdapter::wrap(self.fmt, &mut slot, &mut state); + writer.write_str("..\n")?; + self.fmt.write_str("}") } else { - self.fmt.write_str(" { ..")?; + self.fmt.write_str(", .. }") } - } - if self.is_pretty() { - self.fmt.write_str("}")? } else { - self.fmt.write_str(" }")?; + self.fmt.write_str(" { .. }") } - Ok(()) }); self.result } @@ -257,14 +234,12 @@ impl<'a, 'b: 'a> DebugStruct<'a, 'b> { } } -/// A struct to help with [`fmt::Debug`](trait.Debug.html) implementations. +/// A struct to help with [`fmt::Debug`](Debug) implementations. /// /// This is useful when you wish to output a formatted tuple as a part of your -/// [`Debug::fmt`](trait.Debug.html#tymethod.fmt) implementation. +/// [`Debug::fmt`] implementation. /// -/// This can be constructed by the -/// [`Formatter::debug_tuple`](struct.Formatter.html#method.debug_tuple) -/// method. +/// This can be constructed by the [`Formatter::debug_tuple`] method. /// /// # Examples /// @@ -338,7 +313,7 @@ impl<'a, 'b: 'a> DebugTuple<'a, 'b> { } let mut slot = None; let mut state = Default::default(); - let mut writer = PadAdapter::wrap(&mut self.fmt, &mut slot, &mut state); + let mut writer = PadAdapter::wrap(self.fmt, &mut slot, &mut state); value.fmt(&mut writer)?; writer.write_str(",\n") } else { @@ -409,7 +384,7 @@ impl<'a, 'b: 'a> DebugInner<'a, 'b> { } let mut slot = None; let mut state = Default::default(); - let mut writer = PadAdapter::wrap(&mut self.fmt, &mut slot, &mut state); + let mut writer = PadAdapter::wrap(self.fmt, &mut slot, &mut state); entry.fmt(&mut writer)?; writer.write_str(",\n") } else { @@ -428,14 +403,12 @@ impl<'a, 'b: 'a> DebugInner<'a, 'b> { } } -/// A struct to help with [`fmt::Debug`](trait.Debug.html) implementations. +/// A struct to help with [`fmt::Debug`](Debug) implementations. /// /// This is useful when you wish to output a formatted set of items as a part -/// of your [`Debug::fmt`](trait.Debug.html#tymethod.fmt) implementation. +/// of your [`Debug::fmt`] implementation. /// -/// This can be constructed by the -/// [`Formatter::debug_set`](struct.Formatter.html#method.debug_set) -/// method. +/// This can be constructed by the [`Formatter::debug_set`] method. /// /// # Examples /// @@ -560,14 +533,12 @@ impl<'a, 'b: 'a> DebugSet<'a, 'b> { } } -/// A struct to help with [`fmt::Debug`](trait.Debug.html) implementations. +/// A struct to help with [`fmt::Debug`](Debug) implementations. /// /// This is useful when you wish to output a formatted list of items as a part -/// of your [`Debug::fmt`](trait.Debug.html#tymethod.fmt) implementation. +/// of your [`Debug::fmt`] implementation. /// -/// This can be constructed by the -/// [`Formatter::debug_list`](struct.Formatter.html#method.debug_list) -/// method. +/// This can be constructed by the [`Formatter::debug_list`] method. /// /// # Examples /// @@ -692,14 +663,12 @@ impl<'a, 'b: 'a> DebugList<'a, 'b> { } } -/// A struct to help with [`fmt::Debug`](trait.Debug.html) implementations. +/// A struct to help with [`fmt::Debug`](Debug) implementations. /// /// This is useful when you wish to output a formatted map as a part of your -/// [`Debug::fmt`](trait.Debug.html#tymethod.fmt) implementation. +/// [`Debug::fmt`] implementation. /// -/// This can be constructed by the -/// [`Formatter::debug_map`](struct.Formatter.html#method.debug_map) -/// method. +/// This can be constructed by the [`Formatter::debug_map`] method. /// /// # Examples /// @@ -810,7 +779,7 @@ impl<'a, 'b: 'a> DebugMap<'a, 'b> { } let mut slot = None; self.state = Default::default(); - let mut writer = PadAdapter::wrap(&mut self.fmt, &mut slot, &mut self.state); + let mut writer = PadAdapter::wrap(self.fmt, &mut slot, &mut self.state); key.fmt(&mut writer)?; writer.write_str(": ")?; } else { @@ -866,7 +835,7 @@ impl<'a, 'b: 'a> DebugMap<'a, 'b> { if self.is_pretty() { let mut slot = None; - let mut writer = PadAdapter::wrap(&mut self.fmt, &mut slot, &mut self.state); + let mut writer = PadAdapter::wrap(self.fmt, &mut slot, &mut self.state); value.fmt(&mut writer)?; writer.write_str(",\n")?; } else { diff --git a/crux-mir/lib/core/src/fmt/float.rs b/crux-mir/lib/core/src/fmt/float.rs index 52d8349bc..89d5fac30 100644 --- a/crux-mir/lib/core/src/fmt/float.rs +++ b/crux-mir/lib/core/src/fmt/float.rs @@ -1,6 +1,27 @@ use crate::fmt::{Debug, Display, Formatter, LowerExp, Result, UpperExp}; use crate::mem::MaybeUninit; use crate::num::flt2dec; +use crate::num::fmt as numfmt; + +#[doc(hidden)] +trait GeneralFormat: PartialOrd { + /// Determines if a value should use exponential based on its magnitude, given the precondition + /// that it will not be rounded any further before it is displayed. + fn already_rounded_value_should_use_exponential(&self) -> bool; +} + +macro_rules! impl_general_format { + ($($t:ident)*) => { + $(impl GeneralFormat for $t { + fn already_rounded_value_should_use_exponential(&self) -> bool { + let abs = $t::abs_private(*self); + (abs != 0.0 && abs < 1e-4) || abs >= 1e+16 + } + })* + } +} + +impl_general_format! { f32 f64 } // Don't inline this so callers don't use the stack space this function // requires unless they have to. @@ -14,25 +35,17 @@ fn float_to_decimal_common_exact( where T: flt2dec::DecodableFloat, { - // SAFETY: Possible undefined behavior, see FIXME(#53491) - unsafe { - let mut buf = MaybeUninit::<[u8; 1024]>::uninit(); // enough for f32 and f64 - let mut parts = MaybeUninit::<[flt2dec::Part<'_>; 4]>::uninit(); - // FIXME(#53491): This is calling `get_mut` on an uninitialized - // `MaybeUninit` (here and elsewhere in this file). Revisit this once - // we decided whether that is valid or not. - // We can do this only because we are libstd and coupled to the compiler. - // (FWIW, using `freeze` would not be enough; `flt2dec::Part` is an enum!) - let formatted = flt2dec::to_exact_fixed_str( - flt2dec::strategy::grisu::format_exact, - *num, - sign, - precision, - buf.get_mut(), - parts.get_mut(), - ); - fmt.pad_formatted_parts(&formatted) - } + let mut buf: [MaybeUninit; 1024] = MaybeUninit::uninit_array(); // enough for f32 and f64 + let mut parts: [MaybeUninit>; 4] = MaybeUninit::uninit_array(); + let formatted = flt2dec::to_exact_fixed_str( + flt2dec::strategy::grisu::format_exact, + *num, + sign, + precision, + &mut buf, + &mut parts, + ); + fmt.pad_formatted_parts(&formatted) } // Don't inline this so callers that call both this and the above won't wind @@ -47,45 +60,34 @@ fn float_to_decimal_common_shortest( where T: flt2dec::DecodableFloat, { - // SAFETY: Possible undefined behavior, see FIXME(#53491) - unsafe { - // enough for f32 and f64 - let mut buf = MaybeUninit::<[u8; flt2dec::MAX_SIG_DIGITS]>::uninit(); - let mut parts = MaybeUninit::<[flt2dec::Part<'_>; 4]>::uninit(); - // FIXME(#53491) - let formatted = flt2dec::to_shortest_str( - flt2dec::strategy::grisu::format_shortest, - *num, - sign, - precision, - buf.get_mut(), - parts.get_mut(), - ); - fmt.pad_formatted_parts(&formatted) - } + // enough for f32 and f64 + let mut buf: [MaybeUninit; flt2dec::MAX_SIG_DIGITS] = MaybeUninit::uninit_array(); + let mut parts: [MaybeUninit>; 4] = MaybeUninit::uninit_array(); + let formatted = flt2dec::to_shortest_str( + flt2dec::strategy::grisu::format_shortest, + *num, + sign, + precision, + &mut buf, + &mut parts, + ); + fmt.pad_formatted_parts(&formatted) } -// Common code of floating point Debug and Display. -fn float_to_decimal_common( - fmt: &mut Formatter<'_>, - num: &T, - negative_zero: bool, - min_precision: usize, -) -> Result +fn float_to_decimal_display(fmt: &mut Formatter<'_>, num: &T) -> Result where T: flt2dec::DecodableFloat, { let force_sign = fmt.sign_plus(); - let sign = match (force_sign, negative_zero) { - (false, false) => flt2dec::Sign::Minus, - (false, true) => flt2dec::Sign::MinusRaw, - (true, false) => flt2dec::Sign::MinusPlus, - (true, true) => flt2dec::Sign::MinusPlusRaw, + let sign = match force_sign { + false => flt2dec::Sign::Minus, + true => flt2dec::Sign::MinusPlus, }; if let Some(precision) = fmt.precision { float_to_decimal_common_exact(fmt, num, sign, precision) } else { + let min_precision = 0; float_to_decimal_common_shortest(fmt, num, sign, min_precision) } } @@ -103,22 +105,18 @@ fn float_to_exponential_common_exact( where T: flt2dec::DecodableFloat, { - // SAFETY: Possible undefined behavior, see FIXME(#53491) - unsafe { - let mut buf = MaybeUninit::<[u8; 1024]>::uninit(); // enough for f32 and f64 - let mut parts = MaybeUninit::<[flt2dec::Part<'_>; 6]>::uninit(); - // FIXME(#53491) - let formatted = flt2dec::to_exact_exp_str( - flt2dec::strategy::grisu::format_exact, - *num, - sign, - precision, - upper, - buf.get_mut(), - parts.get_mut(), - ); - fmt.pad_formatted_parts(&formatted) - } + let mut buf: [MaybeUninit; 1024] = MaybeUninit::uninit_array(); // enough for f32 and f64 + let mut parts: [MaybeUninit>; 6] = MaybeUninit::uninit_array(); + let formatted = flt2dec::to_exact_exp_str( + flt2dec::strategy::grisu::format_exact, + *num, + sign, + precision, + upper, + &mut buf, + &mut parts, + ); + fmt.pad_formatted_parts(&formatted) } // Don't inline this so callers that call both this and the above won't wind @@ -133,23 +131,19 @@ fn float_to_exponential_common_shortest( where T: flt2dec::DecodableFloat, { - // SAFETY: Possible undefined behavior, see FIXME(#53491) - unsafe { - // enough for f32 and f64 - let mut buf = MaybeUninit::<[u8; flt2dec::MAX_SIG_DIGITS]>::uninit(); - let mut parts = MaybeUninit::<[flt2dec::Part<'_>; 6]>::uninit(); - // FIXME(#53491) - let formatted = flt2dec::to_shortest_exp_str( - flt2dec::strategy::grisu::format_shortest, - *num, - sign, - (0, 0), - upper, - buf.get_mut(), - parts.get_mut(), - ); - fmt.pad_formatted_parts(&formatted) - } + // enough for f32 and f64 + let mut buf: [MaybeUninit; flt2dec::MAX_SIG_DIGITS] = MaybeUninit::uninit_array(); + let mut parts: [MaybeUninit>; 6] = MaybeUninit::uninit_array(); + let formatted = flt2dec::to_shortest_exp_str( + flt2dec::strategy::grisu::format_shortest, + *num, + sign, + (0, 0), + upper, + &mut buf, + &mut parts, + ); + fmt.pad_formatted_parts(&formatted) } // Common code of floating point LowerExp and UpperExp. @@ -171,19 +165,44 @@ where } } +fn float_to_general_debug(fmt: &mut Formatter<'_>, num: &T) -> Result +where + T: flt2dec::DecodableFloat + GeneralFormat, +{ + let force_sign = fmt.sign_plus(); + let sign = match force_sign { + false => flt2dec::Sign::Minus, + true => flt2dec::Sign::MinusPlus, + }; + + if let Some(precision) = fmt.precision { + // this behavior of {:.PREC?} predates exponential formatting for {:?} + float_to_decimal_common_exact(fmt, num, sign, precision) + } else { + // since there is no precision, there will be no rounding + if num.already_rounded_value_should_use_exponential() { + let upper = false; + float_to_exponential_common_shortest(fmt, num, sign, upper) + } else { + let min_precision = 1; + float_to_decimal_common_shortest(fmt, num, sign, min_precision) + } + } +} + macro_rules! floating { ($ty:ident) => { #[stable(feature = "rust1", since = "1.0.0")] impl Debug for $ty { fn fmt(&self, fmt: &mut Formatter<'_>) -> Result { - float_to_decimal_common(fmt, self, true, 1) + float_to_general_debug(fmt, self) } } #[stable(feature = "rust1", since = "1.0.0")] impl Display for $ty { fn fmt(&self, fmt: &mut Formatter<'_>) -> Result { - float_to_decimal_common(fmt, self, false, 0) + float_to_decimal_display(fmt, self) } } diff --git a/crux-mir/lib/core/src/fmt/mod.rs b/crux-mir/lib/core/src/fmt/mod.rs index 17bb47a27..2a7ec544f 100644 --- a/crux-mir/lib/core/src/fmt/mod.rs +++ b/crux-mir/lib/core/src/fmt/mod.rs @@ -2,22 +2,27 @@ #![stable(feature = "rust1", since = "1.0.0")] -use crate::cell::{Cell, Ref, RefCell, RefMut, UnsafeCell}; -use crate::crucible::any::Any; +use crate::cell::{Cell, Ref, RefCell, RefMut, SyncUnsafeCell, UnsafeCell}; +use crate::char::EscapeDebugExtArgs; +use crate::iter; use crate::marker::PhantomData; use crate::mem; -use crate::num::flt2dec; +use crate::num::fmt as numfmt; use crate::ops::Deref; use crate::result; use crate::str; mod builders; +#[cfg(not(no_fp_fmt_parse))] mod float; +#[cfg(no_fp_fmt_parse)] +mod nofloat; mod num; #[stable(feature = "fmt_flags_align", since = "1.28.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "Alignment")] /// Possible alignments returned by `Formatter::align` -#[derive(Debug)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum Alignment { #[stable(feature = "fmt_flags_align", since = "1.28.0")] /// Indication that contents should be left-aligned. @@ -61,7 +66,7 @@ pub mod rt { /// /// let pythagorean_triple = Triangle { a: 3.0, b: 4.0, c: 5.0 }; /// -/// assert_eq!(format!("{}", pythagorean_triple), "(3, 4, 5)"); +/// assert_eq!(format!("{pythagorean_triple}"), "(3, 4, 5)"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub type Result = result::Result<(), Error>; @@ -93,18 +98,14 @@ pub type Result = result::Result<(), Error>; #[derive(Copy, Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] pub struct Error; -/// A collection of methods that are required to format a message into a stream. +/// A trait for writing or formatting into Unicode-accepting buffers or streams. /// -/// This trait is the type which this modules requires when formatting -/// information. This is similar to the standard library's [`io::Write`] trait, -/// but it is only intended for use in libcore. +/// This trait only accepts UTF-8–encoded data and is not [flushable]. If you only +/// want to accept Unicode and you don't need flushing, you should implement this trait; +/// otherwise you should implement [`std::io::Write`]. /// -/// This trait should generally not be implemented by consumers of the standard -/// library. The [`write!`] macro accepts an instance of [`io::Write`], and the -/// [`io::Write`] trait is favored over implementing this trait. -/// -/// [`write!`]: ../../std/macro.write.html -/// [`io::Write`]: ../../std/io/trait.Write.html +/// [`std::io::Write`]: ../../std/io/trait.Write.html +/// [flushable]: ../../std/io/trait.Write.html#tymethod.flush #[stable(feature = "rust1", since = "1.0.0")] pub trait Write { /// Writes a string slice into this writer, returning whether the write @@ -118,7 +119,9 @@ pub trait Write { /// /// This function will return an instance of [`Error`] on error. /// - /// [`Error`]: struct.Error.html + /// The purpose of std::fmt::Error is to abort the formatting operation when the underlying + /// destination encounters some error preventing it from accepting more text; it should + /// generally be propagated rather than handled, at least when implementing formatting traits. /// /// # Examples /// @@ -147,9 +150,6 @@ pub trait Write { /// /// This function will return an instance of [`Error`] on error. /// - /// [`char`]: ../../std/primitive.char.html - /// [`Error`]: struct.Error.html - /// /// # Examples /// /// ``` @@ -174,7 +174,10 @@ pub trait Write { /// This method should generally not be invoked manually, but rather through /// the [`write!`] macro itself. /// - /// [`write!`]: ../../std/macro.write.html + /// # Errors + /// + /// This function will return an instance of [`Error`] on error. Please see + /// [write_str](Write::write_str) for details. /// /// # Examples /// @@ -182,7 +185,7 @@ pub trait Write { /// use std::fmt::{Error, Write}; /// /// fn writer(f: &mut W, s: &str) -> Result<(), Error> { - /// f.write_fmt(format_args!("{}", s)) + /// f.write_fmt(format_args!("{s}")) /// } /// /// let mut buf = String::new(); @@ -219,9 +222,6 @@ impl Write for &mut W { /// To interact with a `Formatter`, you'll call various methods to change the /// various options related to formatting. For examples, please see the /// documentation of the methods defined on `Formatter` below. -/// -/// [`Debug`]: trait.Debug.html -/// [`Display`]: trait.Display.html #[allow(missing_debug_implementations)] #[stable(feature = "rust1", since = "1.0.0")] pub struct Formatter<'a> { @@ -234,6 +234,28 @@ pub struct Formatter<'a> { buf: &'a mut (dyn Write + 'a), } +impl<'a> Formatter<'a> { + /// Creates a new formatter with default settings. + /// + /// This can be used as a micro-optimization in cases where a full `Arguments` + /// structure (as created by `format_args!`) is not necessary; `Arguments` + /// is a little more expensive to use in simple formatting scenarios. + /// + /// Currently not intended for use outside of the standard library. + #[unstable(feature = "fmt_internals", reason = "internal to standard library", issue = "none")] + #[doc(hidden)] + pub fn new(buf: &'a mut (dyn Write + 'a)) -> Formatter<'a> { + Formatter { + flags: 0, + fill: ' ', + align: rt::v1::Alignment::Unknown, + width: None, + precision: None, + buf, + } + } +} + // NB. Argument is essentially an optimized partially applied formatting function, // equivalent to `exists T.(&T, fn(&T, &mut Formatter<'_>) -> Result`. @@ -250,17 +272,28 @@ extern "C" { #[unstable(feature = "fmt_internals", reason = "internal to format_args!", issue = "none")] #[doc(hidden)] pub struct ArgumentV1<'a> { - value: Any, - formatter: Any, - dispatch: fn(Any, Any, &mut Formatter<'_>) -> Result, - _marker: PhantomData<&'a Opaque>, + value: &'a Opaque, + formatter: fn(&Opaque, &mut Formatter<'_>) -> Result, } -fn dispatch(value: Any, formatter: Any, fmt: &mut Formatter<'_>) -> Result { - unsafe { - let value = value.downcast::<&T>(); - let formatter = formatter.downcast::) -> Result>(); - formatter(value, fmt) +/// This struct represents the unsafety of constructing an `Arguments`. +/// It exists, rather than an unsafe function, in order to simplify the expansion +/// of `format_args!(..)` and reduce the scope of the `unsafe` block. +#[allow(missing_debug_implementations)] +#[doc(hidden)] +#[unstable(feature = "fmt_internals", reason = "internal to format_args!", issue = "none")] +pub struct UnsafeArg { + _private: (), +} + +impl UnsafeArg { + /// See documentation where `UnsafeArg` is required to know when it is safe to + /// create and use `UnsafeArg`. + #[doc(hidden)] + #[unstable(feature = "fmt_internals", reason = "internal to format_args!", issue = "none")] + #[inline(always)] + pub unsafe fn new() -> Self { + Self { _private: () } } } @@ -286,29 +319,62 @@ static USIZE_MARKER: fn(&usize, &mut Formatter<'_>) -> Result = |ptr, _| { loop {} }; +macro_rules! arg_new { + ($f: ident, $t: ident) => { + #[doc(hidden)] + #[unstable(feature = "fmt_internals", reason = "internal to format_args!", issue = "none")] + #[inline] + pub fn $f<'b, T: $t>(x: &'b T) -> ArgumentV1<'_> { + Self::new(x, $t::fmt) + } + }; +} + +#[rustc_diagnostic_item = "ArgumentV1Methods"] impl<'a> ArgumentV1<'a> { #[doc(hidden)] #[unstable(feature = "fmt_internals", reason = "internal to format_args!", issue = "none")] - pub fn new<'b, T>(x: &'b T, - f: fn(&T, &mut Formatter<'_>) -> Result) -> ArgumentV1<'b> { - ArgumentV1 { - value: Any::new(x), - formatter: Any::new(f), - dispatch: dispatch::, - _marker: PhantomData, - } - } + #[inline] + pub fn new<'b, T>(x: &'b T, f: fn(&T, &mut Formatter<'_>) -> Result) -> ArgumentV1<'b> { + // SAFETY: `mem::transmute(x)` is safe because + // 1. `&'b T` keeps the lifetime it originated with `'b` + // (so as to not have an unbounded lifetime) + // 2. `&'b T` and `&'b Opaque` have the same memory layout + // (when `T` is `Sized`, as it is here) + // `mem::transmute(f)` is safe since `fn(&T, &mut Formatter<'_>) -> Result` + // and `fn(&Opaque, &mut Formatter<'_>) -> Result` have the same ABI + // (as long as `T` is `Sized`) + unsafe { ArgumentV1 { formatter: mem::transmute(f), value: mem::transmute(x) } } + } + + arg_new!(new_display, Display); + arg_new!(new_debug, Debug); + arg_new!(new_octal, Octal); + arg_new!(new_lower_hex, LowerHex); + arg_new!(new_upper_hex, UpperHex); + arg_new!(new_pointer, Pointer); + arg_new!(new_binary, Binary); + arg_new!(new_lower_exp, LowerExp); + arg_new!(new_upper_exp, UpperExp); #[doc(hidden)] #[unstable(feature = "fmt_internals", reason = "internal to format_args!", issue = "none")] pub fn from_usize(x: &usize) -> ArgumentV1<'_> { - Self::new(x, Display::fmt) + ArgumentV1::new(x, USIZE_MARKER) } fn as_usize(&self) -> Option { - // According to the comment on `USIZE_MARKER`, this method is only ever called on arguments - // that have been statically checked to be `usize`. - Some(unsafe { *self.value.downcast::<&usize>() }) + // We are type punning a bit here: USIZE_MARKER only takes an &usize but + // formatter takes an &Opaque. Rust understandably doesn't think we should compare + // the function pointers if they don't have the same signature, so we cast to + // usizes to tell it that we just want to compare addresses. + if self.formatter as usize == USIZE_MARKER as usize { + // SAFETY: The `formatter` field is only set to USIZE_MARKER if + // the value is a usize, so this is safe + Some(unsafe { *(self.value as *const _ as *const usize) }) + } else { + None + } } } @@ -329,23 +395,32 @@ impl<'a> Arguments<'a> { #[doc(hidden)] #[inline] #[unstable(feature = "fmt_internals", reason = "internal to format_args!", issue = "none")] - pub fn new_v1(pieces: &'a [&'a str], args: &'a [ArgumentV1<'a>]) -> Arguments<'a> { + #[rustc_const_unstable(feature = "const_fmt_arguments_new", issue = "none")] + pub const fn new_v1(pieces: &'a [&'static str], args: &'a [ArgumentV1<'a>]) -> Arguments<'a> { + if pieces.len() < args.len() || pieces.len() > args.len() + 1 { + panic!("invalid args"); + } Arguments { pieces, fmt: None, args } } /// This function is used to specify nonstandard formatting parameters. - /// The `pieces` array must be at least as long as `fmt` to construct - /// a valid Arguments structure. Also, any `Count` within `fmt` that is - /// `CountIsParam` or `CountIsNextParam` has to point to an argument - /// created with `argumentusize`. However, failing to do so doesn't cause - /// unsafety, but will ignore invalid . + /// + /// An `UnsafeArg` is required because the following invariants must be held + /// in order for this function to be safe: + /// 1. The `pieces` slice must be at least as long as `fmt`. + /// 2. Every [`rt::v1::Argument::position`] value within `fmt` must be a + /// valid index of `args`. + /// 3. Every [`rt::v1::Count::Param`] within `fmt` must contain a valid index of + /// `args`. #[doc(hidden)] #[inline] #[unstable(feature = "fmt_internals", reason = "internal to format_args!", issue = "none")] - pub fn new_v1_formatted( - pieces: &'a [&'a str], + #[rustc_const_unstable(feature = "const_fmt_arguments_new", issue = "none")] + pub const fn new_v1_formatted( + pieces: &'a [&'static str], args: &'a [ArgumentV1<'a>], fmt: &'a [rt::v1::Argument], + _unsafe_arg: UnsafeArg, ) -> Arguments<'a> { Arguments { pieces, fmt: Some(fmt), args } } @@ -362,7 +437,7 @@ impl<'a> Arguments<'a> { if self.args.is_empty() { pieces_length - } else if self.pieces[0] == "" && pieces_length < 16 { + } else if !self.pieces.is_empty() && self.pieces[0].is_empty() && pieces_length < 16 { // If the format string starts with an argument, // don't preallocate anything, unless length // of pieces is significant. @@ -383,7 +458,7 @@ impl<'a> Arguments<'a> { /// /// The [`format_args!`] macro will safely create an instance of this structure. /// The macro validates the format string at compile-time so usage of the -/// [`write`] and [`format`] functions can be safely performed. +/// [`write()`] and [`format()`] functions can be safely performed. /// /// You can use the `Arguments<'a>` that [`format_args!`] returns in `Debug` /// and `Display` contexts as seen below. The example also shows that `Debug` @@ -397,14 +472,13 @@ impl<'a> Arguments<'a> { /// assert_eq!(display, debug); /// ``` /// -/// [`format_args!`]: ../../std/macro.format_args.html -/// [`format`]: ../../std/fmt/fn.format.html -/// [`write`]: ../../std/fmt/fn.write.html +/// [`format()`]: ../../std/fmt/fn.format.html #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "Arguments")] #[derive(Copy, Clone)] pub struct Arguments<'a> { // Format string pieces to print. - pieces: &'a [&'a str], + pieces: &'a [&'static str], // Placeholder specs, or `None` if all specs are default (as in "{}{}"). fmt: Option<&'a [rt::v1::Argument]>, @@ -414,6 +488,45 @@ pub struct Arguments<'a> { args: &'a [ArgumentV1<'a>], } +impl<'a> Arguments<'a> { + /// Get the formatted string, if it has no arguments to be formatted. + /// + /// This can be used to avoid allocations in the most trivial case. + /// + /// # Examples + /// + /// ```rust + /// use std::fmt::Arguments; + /// + /// fn write_str(_: &str) { /* ... */ } + /// + /// fn write_fmt(args: &Arguments) { + /// if let Some(s) = args.as_str() { + /// write_str(s) + /// } else { + /// write_str(&args.to_string()); + /// } + /// } + /// ``` + /// + /// ```rust + /// assert_eq!(format_args!("hello").as_str(), Some("hello")); + /// assert_eq!(format_args!("").as_str(), Some("")); + /// assert_eq!(format_args!("{}", 1).as_str(), None); + /// ``` + #[stable(feature = "fmt_as_str", since = "1.52.0")] + #[rustc_const_unstable(feature = "const_arguments_as_str", issue = "103900")] + #[must_use] + #[inline] + pub const fn as_str(&self) -> Option<&'static str> { + match (self.pieces, self.args) { + ([], []) => Some(""), + ([s], []) => Some(s), + _ => None, + } + } +} + #[stable(feature = "rust1", since = "1.0.0")] impl Debug for Arguments<'_> { fn fmt(&self, fmt: &mut Formatter<'_>) -> Result { @@ -446,6 +559,13 @@ impl Display for Arguments<'_> { /// `enum`s, it will use the name of the variant and, if applicable, `(`, then the /// `Debug` values of the fields, then `)`. /// +/// # Stability +/// +/// Derived `Debug` formats are not stable, and so may change with future Rust +/// versions. Additionally, `Debug` implementations of types provided by the +/// standard library (`std`, `core`, `alloc`, etc.) are not stable, and +/// may also change with future Rust versions. +/// /// # Examples /// /// Deriving an implementation: @@ -459,7 +579,7 @@ impl Display for Arguments<'_> { /// /// let origin = Point { x: 0, y: 0 }; /// -/// assert_eq!(format!("The origin is: {:?}", origin), "The origin is: Point { x: 0, y: 0 }"); +/// assert_eq!(format!("The origin is: {origin:?}"), "The origin is: Point { x: 0, y: 0 }"); /// ``` /// /// Manually implementing: @@ -483,18 +603,36 @@ impl Display for Arguments<'_> { /// /// let origin = Point { x: 0, y: 0 }; /// -/// assert_eq!(format!("The origin is: {:?}", origin), "The origin is: Point { x: 0, y: 0 }"); +/// assert_eq!(format!("The origin is: {origin:?}"), "The origin is: Point { x: 0, y: 0 }"); /// ``` /// /// There are a number of helper methods on the [`Formatter`] struct to help you with manual /// implementations, such as [`debug_struct`]. /// +/// [`debug_struct`]: Formatter::debug_struct +/// +/// Types that do not wish to use the standard suite of debug representations +/// provided by the `Formatter` trait (`debug_struct`, `debug_tuple`, +/// `debug_list`, `debug_set`, `debug_map`) can do something totally custom by +/// manually writing an arbitrary representation to the `Formatter`. +/// +/// ``` +/// # use std::fmt; +/// # struct Point { +/// # x: i32, +/// # y: i32, +/// # } +/// # +/// impl fmt::Debug for Point { +/// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +/// write!(f, "Point [{} {}]", self.x, self.y) +/// } +/// } +/// ``` +/// /// `Debug` implementations using either `derive` or the debug builder API /// on [`Formatter`] support pretty-printing using the alternate flag: `{:#?}`. /// -/// [`debug_struct`]: ../../std/fmt/struct.Formatter.html#method.debug_struct -/// [`Formatter`]: ../../std/fmt/struct.Formatter.html -/// /// Pretty-printing with `#?`: /// /// ``` @@ -506,7 +644,7 @@ impl Display for Arguments<'_> { /// /// let origin = Point { x: 0, y: 0 }; /// -/// assert_eq!(format!("The origin is: {:#?}", origin), +/// assert_eq!(format!("The origin is: {origin:#?}"), /// "The origin is: Point { /// x: 0, /// y: 0, @@ -518,13 +656,14 @@ impl Display for Arguments<'_> { on( crate_local, label = "`{Self}` cannot be formatted using `{{:?}}`", - note = "add `#[derive(Debug)]` or manually implement `{Debug}`" + note = "add `#[derive(Debug)]` to `{Self}` or manually `impl {Debug} for {Self}`" ), message = "`{Self}` doesn't implement `{Debug}`", label = "`{Self}` cannot be formatted using `{{:?}}` because it doesn't implement `{Debug}`" )] #[doc(alias = "{:?}")] -#[rustc_diagnostic_item = "debug_trait"] +#[rustc_diagnostic_item = "Debug"] +#[rustc_trivial_field_reads] pub trait Debug { /// Formats the value using the given formatter. /// @@ -548,9 +687,9 @@ pub trait Debug { /// } /// /// let position = Position { longitude: 1.987, latitude: 2.983 }; - /// assert_eq!(format!("{:?}", position), "(1.987, 2.983)"); + /// assert_eq!(format!("{position:?}"), "(1.987, 2.983)"); /// - /// assert_eq!(format!("{:#?}", position), "( + /// assert_eq!(format!("{position:#?}"), "( /// 1.987, /// 2.983, /// )"); @@ -564,7 +703,7 @@ pub(crate) mod macros { /// Derive macro generating an impl of the trait `Debug`. #[rustc_builtin_macro] #[stable(feature = "builtin_macro_prelude", since = "1.38.0")] - #[allow_internal_unstable(core_intrinsics)] + #[allow_internal_unstable(core_intrinsics, fmt_helpers_for_derive)] pub macro Debug($item:item) { /* compiler built-in */ } @@ -575,14 +714,19 @@ pub use macros::Debug; /// Format trait for an empty format, `{}`. /// -/// `Display` is similar to [`Debug`][debug], but `Display` is for user-facing -/// output, and so cannot be derived. +/// Implementing this trait for a type will automatically implement the +/// [`ToString`][tostring] trait for the type, allowing the usage +/// of the [`.to_string()`][tostring_function] method. Prefer implementing +/// the `Display` trait for a type, rather than [`ToString`][tostring]. /// -/// [debug]: trait.Debug.html +/// `Display` is similar to [`Debug`], but `Display` is for user-facing +/// output, and so cannot be derived. /// /// For more information on formatters, see [the module-level documentation][module]. /// /// [module]: ../../std/fmt/index.html +/// [tostring]: ../../std/string/trait.ToString.html +/// [tostring_function]: ../../std/string/trait.ToString.html#tymethod.to_string /// /// # Examples /// @@ -604,11 +748,11 @@ pub use macros::Debug; /// /// let origin = Point { x: 0, y: 0 }; /// -/// assert_eq!(format!("The origin is: {}", origin), "The origin is: (0, 0)"); +/// assert_eq!(format!("The origin is: {origin}"), "The origin is: (0, 0)"); /// ``` #[rustc_on_unimplemented( on( - _Self = "std::path::Path", + any(_Self = "std::path::Path", _Self = "std::path::PathBuf"), label = "`{Self}` cannot be formatted with the default formatter; call `.display()` on it", note = "call `.display()` or `.to_string_lossy()` to safely print paths, \ as they may contain non-Unicode data" @@ -618,6 +762,7 @@ pub use macros::Debug; note = "in format strings you may be able to use `{{:?}}` (or {{:#?}} for pretty-print) instead" )] #[doc(alias = "{}")] +#[rustc_diagnostic_item = "Display"] #[stable(feature = "rust1", since = "1.0.0")] pub trait Display { /// Formats the value using the given formatter. @@ -665,8 +810,8 @@ pub trait Display { /// ``` /// let x = 42; // 42 is '52' in octal /// -/// assert_eq!(format!("{:o}", x), "52"); -/// assert_eq!(format!("{:#o}", x), "0o52"); +/// assert_eq!(format!("{x:o}"), "52"); +/// assert_eq!(format!("{x:#o}"), "0o52"); /// /// assert_eq!(format!("{:o}", -16), "37777777760"); /// ``` @@ -688,9 +833,9 @@ pub trait Display { /// /// let l = Length(9); /// -/// assert_eq!(format!("l as octal is: {:o}", l), "l as octal is: 11"); +/// assert_eq!(format!("l as octal is: {l:o}"), "l as octal is: 11"); /// -/// assert_eq!(format!("l as octal is: {:#06o}", l), "l as octal is: 0o0011"); +/// assert_eq!(format!("l as octal is: {l:#06o}"), "l as octal is: 0o0011"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub trait Octal { @@ -710,6 +855,8 @@ pub trait Octal { /// /// For more information on formatters, see [the module-level documentation][module]. /// +/// [module]: ../../std/fmt/index.html +/// /// # Examples /// /// Basic usage with [`i32`]: @@ -717,8 +864,8 @@ pub trait Octal { /// ``` /// let x = 42; // 42 is '101010' in binary /// -/// assert_eq!(format!("{:b}", x), "101010"); -/// assert_eq!(format!("{:#b}", x), "0b101010"); +/// assert_eq!(format!("{x:b}"), "101010"); +/// assert_eq!(format!("{x:#b}"), "0b101010"); /// /// assert_eq!(format!("{:b}", -16), "11111111111111111111111111110000"); /// ``` @@ -740,19 +887,13 @@ pub trait Octal { /// /// let l = Length(107); /// -/// assert_eq!(format!("l as binary is: {:b}", l), "l as binary is: 1101011"); +/// assert_eq!(format!("l as binary is: {l:b}"), "l as binary is: 1101011"); /// /// assert_eq!( -/// format!("l as binary is: {:#032b}", l), +/// format!("l as binary is: {l:#032b}"), /// "l as binary is: 0b000000000000000000000001101011" /// ); /// ``` -/// -/// [module]: ../../std/fmt/index.html -/// [`i8`]: ../../std/primitive.i8.html -/// [`i128`]: ../../std/primitive.i128.html -/// [`isize`]: ../../std/primitive.isize.html -/// [`i32`]: ../../std/primitive.i32.html #[stable(feature = "rust1", since = "1.0.0")] pub trait Binary { /// Formats the value using the given formatter. @@ -781,8 +922,8 @@ pub trait Binary { /// ``` /// let x = 42; // 42 is '2a' in hex /// -/// assert_eq!(format!("{:x}", x), "2a"); -/// assert_eq!(format!("{:#x}", x), "0x2a"); +/// assert_eq!(format!("{x:x}"), "2a"); +/// assert_eq!(format!("{x:#x}"), "0x2a"); /// /// assert_eq!(format!("{:x}", -16), "fffffff0"); /// ``` @@ -804,9 +945,9 @@ pub trait Binary { /// /// let l = Length(9); /// -/// assert_eq!(format!("l as hex is: {:x}", l), "l as hex is: 9"); +/// assert_eq!(format!("l as hex is: {l:x}"), "l as hex is: 9"); /// -/// assert_eq!(format!("l as hex is: {:#010x}", l), "l as hex is: 0x00000009"); +/// assert_eq!(format!("l as hex is: {l:#010x}"), "l as hex is: 0x00000009"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub trait LowerHex { @@ -836,8 +977,8 @@ pub trait LowerHex { /// ``` /// let x = 42; // 42 is '2A' in hex /// -/// assert_eq!(format!("{:X}", x), "2A"); -/// assert_eq!(format!("{:#X}", x), "0x2A"); +/// assert_eq!(format!("{x:X}"), "2A"); +/// assert_eq!(format!("{x:#X}"), "0x2A"); /// /// assert_eq!(format!("{:X}", -16), "FFFFFFF0"); /// ``` @@ -857,11 +998,11 @@ pub trait LowerHex { /// } /// } /// -/// let l = Length(i32::max_value()); +/// let l = Length(i32::MAX); /// -/// assert_eq!(format!("l as hex is: {:X}", l), "l as hex is: 7FFFFFFF"); +/// assert_eq!(format!("l as hex is: {l:X}"), "l as hex is: 7FFFFFFF"); /// -/// assert_eq!(format!("l as hex is: {:#010X}", l), "l as hex is: 0x7FFFFFFF"); +/// assert_eq!(format!("l as hex is: {l:#010X}"), "l as hex is: 0x7FFFFFFF"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub trait UpperHex { @@ -886,7 +1027,7 @@ pub trait UpperHex { /// ``` /// let x = &42; /// -/// let address = format!("{:p}", x); // this produces something like '0x7f06092ac6d0' +/// let address = format!("{x:p}"); // this produces something like '0x7f06092ac6d0' /// ``` /// /// Implementing `Pointer` on a type: @@ -907,13 +1048,14 @@ pub trait UpperHex { /// /// let l = Length(42); /// -/// println!("l is in memory here: {:p}", l); +/// println!("l is in memory here: {l:p}"); /// -/// let l_ptr = format!("{:018p}", l); +/// let l_ptr = format!("{l:018p}"); /// assert_eq!(l_ptr.len(), 18); /// assert_eq!(&l_ptr[..2], "0x"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[rustc_diagnostic_item = "Pointer"] pub trait Pointer { /// Formats the value using the given formatter. #[stable(feature = "rust1", since = "1.0.0")] @@ -935,7 +1077,7 @@ pub trait Pointer { /// ``` /// let x = 42.0; // 42.0 is '4.2e1' in scientific notation /// -/// assert_eq!(format!("{:e}", x), "4.2e1"); +/// assert_eq!(format!("{x:e}"), "4.2e1"); /// ``` /// /// Implementing `LowerExp` on a type: @@ -955,12 +1097,12 @@ pub trait Pointer { /// let l = Length(100); /// /// assert_eq!( -/// format!("l in scientific notation is: {:e}", l), +/// format!("l in scientific notation is: {l:e}"), /// "l in scientific notation is: 1e2" /// ); /// /// assert_eq!( -/// format!("l in scientific notation is: {:05e}", l), +/// format!("l in scientific notation is: {l:05e}"), /// "l in scientific notation is: 001e2" /// ); /// ``` @@ -986,7 +1128,7 @@ pub trait LowerExp { /// ``` /// let x = 42.0; // 42.0 is '4.2E1' in scientific notation /// -/// assert_eq!(format!("{:E}", x), "4.2E1"); +/// assert_eq!(format!("{x:E}"), "4.2E1"); /// ``` /// /// Implementing `UpperExp` on a type: @@ -1006,12 +1148,12 @@ pub trait LowerExp { /// let l = Length(100); /// /// assert_eq!( -/// format!("l in scientific notation is: {:E}", l), +/// format!("l in scientific notation is: {l:E}"), /// "l in scientific notation is: 1E2" /// ); /// /// assert_eq!( -/// format!("l in scientific notation is: {:05E}", l), +/// format!("l in scientific notation is: {l:05E}"), /// "l in scientific notation is: 001E2" /// ); /// ``` @@ -1052,35 +1194,39 @@ pub trait UpperExp { /// assert_eq!(output, "Hello world!"); /// ``` /// -/// [`write!`]: ../../std/macro.write.html +/// [`write!`]: crate::write! #[stable(feature = "rust1", since = "1.0.0")] pub fn write(output: &mut dyn Write, args: Arguments<'_>) -> Result { - let mut formatter = Formatter { - flags: 0, - width: None, - precision: None, - buf: output, - align: rt::v1::Alignment::Unknown, - fill: ' ', - }; - + let mut formatter = Formatter::new(output); let mut idx = 0; match args.fmt { None => { // We can use default formatting parameters for all arguments. - for (arg, piece) in args.args.iter().zip(args.pieces.iter()) { - formatter.buf.write_str(*piece)?; - (arg.dispatch)(arg.value, arg.formatter, &mut formatter)?; + for (i, arg) in args.args.iter().enumerate() { + // SAFETY: args.args and args.pieces come from the same Arguments, + // which guarantees the indexes are always within bounds. + let piece = unsafe { args.pieces.get_unchecked(i) }; + if !piece.is_empty() { + formatter.buf.write_str(*piece)?; + } + (arg.formatter)(arg.value, &mut formatter)?; idx += 1; } } Some(fmt) => { // Every spec has a corresponding argument that is preceded by // a string piece. - for (arg, piece) in fmt.iter().zip(args.pieces.iter()) { - formatter.buf.write_str(*piece)?; - run(&mut formatter, arg, &args.args)?; + for (i, arg) in fmt.iter().enumerate() { + // SAFETY: fmt and args.pieces come from the same Arguments, + // which guarantees the indexes are always within bounds. + let piece = unsafe { args.pieces.get_unchecked(i) }; + if !piece.is_empty() { + formatter.buf.write_str(*piece)?; + } + // SAFETY: arg and args.args come from the same Arguments, + // which guarantees the indexes are always within bounds. + unsafe { run(&mut formatter, arg, args.args) }?; idx += 1; } } @@ -1094,31 +1240,43 @@ pub fn write(output: &mut dyn Write, args: Arguments<'_>) -> Result { Ok(()) } -fn run(fmt: &mut Formatter<'_>, arg: &rt::v1::Argument, args: &[ArgumentV1<'_>]) -> Result { +unsafe fn run(fmt: &mut Formatter<'_>, arg: &rt::v1::Argument, args: &[ArgumentV1<'_>]) -> Result { fmt.fill = arg.format.fill; fmt.align = arg.format.align; fmt.flags = arg.format.flags; - fmt.width = getcount(args, &arg.format.width); - fmt.precision = getcount(args, &arg.format.precision); + // SAFETY: arg and args come from the same Arguments, + // which guarantees the indexes are always within bounds. + unsafe { + fmt.width = getcount(args, &arg.format.width); + fmt.precision = getcount(args, &arg.format.precision); + } // Extract the correct argument - let value = args[arg.position]; + debug_assert!(arg.position < args.len()); + // SAFETY: arg and args come from the same Arguments, + // which guarantees its index is always within bounds. + let value = unsafe { args.get_unchecked(arg.position) }; // Then actually do some printing - (value.dispatch)(value.value, value.formatter, fmt) + (value.formatter)(value.value, fmt) } -fn getcount(args: &[ArgumentV1<'_>], cnt: &rt::v1::Count) -> Option { +unsafe fn getcount(args: &[ArgumentV1<'_>], cnt: &rt::v1::Count) -> Option { match *cnt { rt::v1::Count::Is(n) => Some(n), rt::v1::Count::Implied => None, - rt::v1::Count::Param(i) => args[i].as_usize(), + rt::v1::Count::Param(i) => { + debug_assert!(i < args.len()); + // SAFETY: cnt and args come from the same Arguments, + // which guarantees this index is always within bounds. + unsafe { args.get_unchecked(i).as_usize() } + } } } /// Padding after the end of something. Returned by `Formatter::padding`. #[must_use = "don't forget to write the post padding"] -struct PostPadding { +pub(crate) struct PostPadding { fill: char, padding: usize, } @@ -1129,9 +1287,9 @@ impl PostPadding { } /// Write this post padding. - fn write(self, buf: &mut dyn Write) -> Result { + pub(crate) fn write(self, f: &mut Formatter<'_>) -> Result { for _ in 0..self.padding { - buf.write_char(self.fill)?; + f.buf.write_char(self.fill)?; } Ok(()) } @@ -1178,7 +1336,7 @@ impl<'a> Formatter<'a> { /// ``` /// use std::fmt; /// - /// struct Foo { nb: i32 }; + /// struct Foo { nb: i32 } /// /// impl Foo { /// fn new(nb: i32) -> Foo { @@ -1193,14 +1351,15 @@ impl<'a> Formatter<'a> { /// // We need to remove "-" from the number output. /// let tmp = self.nb.abs().to_string(); /// - /// formatter.pad_integral(self.nb > 0, "Foo ", &tmp) + /// formatter.pad_integral(self.nb >= 0, "Foo ", &tmp) /// } /// } /// - /// assert_eq!(&format!("{}", Foo::new(2)), "2"); - /// assert_eq!(&format!("{}", Foo::new(-1)), "-1"); - /// assert_eq!(&format!("{:#}", Foo::new(-1)), "-Foo 1"); - /// assert_eq!(&format!("{:0>#8}", Foo::new(-1)), "00-Foo 1"); + /// assert_eq!(format!("{}", Foo::new(2)), "2"); + /// assert_eq!(format!("{}", Foo::new(-1)), "-1"); + /// assert_eq!(format!("{}", Foo::new(0)), "0"); + /// assert_eq!(format!("{:#}", Foo::new(-1)), "-Foo 1"); + /// assert_eq!(format!("{:0>#8}", Foo::new(-1)), "00-Foo 1"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn pad_integral(&mut self, is_nonnegative: bool, prefix: &str, buf: &str) -> Result { @@ -1253,7 +1412,7 @@ impl<'a> Formatter<'a> { write_prefix(self, sign, prefix)?; let post_padding = self.padding(min - width, rt::v1::Alignment::Right)?; self.buf.write_str(buf)?; - post_padding.write(self.buf)?; + post_padding.write(self)?; self.fill = old_fill; self.align = old_align; Ok(()) @@ -1263,7 +1422,7 @@ impl<'a> Formatter<'a> { let post_padding = self.padding(min - width, rt::v1::Alignment::Right)?; write_prefix(self, sign, prefix)?; self.buf.write_str(buf)?; - post_padding.write(self.buf) + post_padding.write(self) } } } @@ -1293,8 +1452,8 @@ impl<'a> Formatter<'a> { /// } /// } /// - /// assert_eq!(&format!("{:<4}", Foo), "Foo "); - /// assert_eq!(&format!("{:0>4}", Foo), "0Foo"); + /// assert_eq!(format!("{Foo:<4}"), "Foo "); + /// assert_eq!(format!("{Foo:0>4}"), "0Foo"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn pad(&mut self, s: &str) -> Result { @@ -1313,7 +1472,7 @@ impl<'a> Formatter<'a> { // we know that it can't panic. Use `get` + `unwrap_or` to avoid // `unsafe` and otherwise don't emit any panic-related code // here. - s.get(..i).unwrap_or(&s) + s.get(..i).unwrap_or(s) } else { &s } @@ -1325,16 +1484,21 @@ impl<'a> Formatter<'a> { // If we're under the maximum length, and there's no minimum length // requirements, then we can just emit the string None => self.buf.write_str(s), - // If we're under the maximum width, check if we're over the minimum - // width, if so it's as easy as just emitting the string. - Some(width) if s.chars().count() >= width => self.buf.write_str(s), - // If we're under both the maximum and the minimum width, then fill - // up the minimum width with the specified string + some alignment. Some(width) => { - let align = rt::v1::Alignment::Left; - let post_padding = self.padding(width - s.chars().count(), align)?; - self.buf.write_str(s)?; - post_padding.write(self.buf) + let chars_count = s.chars().count(); + // If we're under the maximum width, check if we're over the minimum + // width, if so it's as easy as just emitting the string. + if chars_count >= width { + self.buf.write_str(s) + } + // If we're under both the maximum and the minimum width, then fill + // up the minimum width with the specified string + some alignment. + else { + let align = rt::v1::Alignment::Left; + let post_padding = self.padding(width - chars_count, align)?; + self.buf.write_str(s)?; + post_padding.write(self) + } } } } @@ -1342,7 +1506,7 @@ impl<'a> Formatter<'a> { /// Write the pre-padding and return the unwritten post-padding. Callers are /// responsible for ensuring post-padding is written after the thing that is /// being padded. - fn padding( + pub(crate) fn padding( &mut self, padding: usize, default: rt::v1::Alignment, @@ -1368,7 +1532,7 @@ impl<'a> Formatter<'a> { /// Takes the formatted parts and applies the padding. /// Assumes that the caller already has rendered the parts with required precision, /// so that `self.precision` can be ignored. - fn pad_formatted_parts(&mut self, formatted: &flt2dec::Formatted<'_>) -> Result { + fn pad_formatted_parts(&mut self, formatted: &numfmt::Formatted<'_>) -> Result { if let Some(mut width) = self.width { // for the sign-aware zero padding, we render the sign first and // behave as if we had no sign from the beginning. @@ -1397,7 +1561,7 @@ impl<'a> Formatter<'a> { } else { let post_padding = self.padding(width - len, align)?; self.write_formatted_parts(&formatted)?; - post_padding.write(self.buf) + post_padding.write(self) }; self.fill = old_fill; self.align = old_align; @@ -1408,14 +1572,14 @@ impl<'a> Formatter<'a> { } } - fn write_formatted_parts(&mut self, formatted: &flt2dec::Formatted<'_>) -> Result { + fn write_formatted_parts(&mut self, formatted: &numfmt::Formatted<'_>) -> Result { fn write_bytes(buf: &mut dyn Write, s: &[u8]) -> Result { - // SAFETY: This is used for `flt2dec::Part::Num` and `flt2dec::Part::Copy`. - // It's safe to use for `flt2dec::Part::Num` since every char `c` is between + // SAFETY: This is used for `numfmt::Part::Num` and `numfmt::Part::Copy`. + // It's safe to use for `numfmt::Part::Num` since every char `c` is between // `b'0'` and `b'9'`, which means `s` is valid UTF-8. - // It's also probably safe in practice to use for `flt2dec::Part::Copy(buf)` + // It's also probably safe in practice to use for `numfmt::Part::Copy(buf)` // since `buf` should be plain ASCII, but it's possible for someone to pass - // in a bad value for `buf` into `flt2dec::to_shortest_str` since it is a + // in a bad value for `buf` into `numfmt::to_shortest_str` since it is a // public function. // FIXME: Determine whether this could result in UB. buf.write_str(unsafe { str::from_utf8_unchecked(s) }) @@ -1426,7 +1590,7 @@ impl<'a> Formatter<'a> { } for part in formatted.parts { match *part { - flt2dec::Part::Zero(mut nzeroes) => { + numfmt::Part::Zero(mut nzeroes) => { const ZEROES: &str = // 64 zeroes "0000000000000000000000000000000000000000000000000000000000000000"; while nzeroes > ZEROES.len() { @@ -1437,7 +1601,7 @@ impl<'a> Formatter<'a> { self.buf.write_str(&ZEROES[..nzeroes])?; } } - flt2dec::Part::Num(mut v) => { + numfmt::Part::Num(mut v) => { let mut s = [0; 5]; let len = part.len(); for c in s[..len].iter_mut().rev() { @@ -1446,7 +1610,7 @@ impl<'a> Formatter<'a> { } write_bytes(self.buf, &s[..len])?; } - flt2dec::Part::Copy(buf) => { + numfmt::Part::Copy(buf) => { write_bytes(self.buf, buf)?; } } @@ -1472,8 +1636,8 @@ impl<'a> Formatter<'a> { /// } /// } /// - /// assert_eq!(&format!("{}", Foo), "Foo"); - /// assert_eq!(&format!("{:0>8}", Foo), "Foo"); + /// assert_eq!(format!("{Foo}"), "Foo"); + /// assert_eq!(format!("{Foo:0>8}"), "Foo"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn write_str(&mut self, data: &str) -> Result { @@ -1495,8 +1659,8 @@ impl<'a> Formatter<'a> { /// } /// } /// - /// assert_eq!(&format!("{}", Foo(-1)), "Foo -1"); - /// assert_eq!(&format!("{:0>8}", Foo(2)), "Foo 2"); + /// assert_eq!(format!("{}", Foo(-1)), "Foo -1"); + /// assert_eq!(format!("{:0>8}", Foo(2)), "Foo 2"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn write_fmt(&mut self, fmt: Arguments<'_>) -> Result { @@ -1504,11 +1668,12 @@ impl<'a> Formatter<'a> { } /// Flags for formatting + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_deprecated( + #[deprecated( since = "1.24.0", - reason = "use the `sign_plus`, `sign_minus`, `alternate`, \ - or `sign_aware_zero_pad` methods instead" + note = "use the `sign_plus`, `sign_minus`, `alternate`, \ + or `sign_aware_zero_pad` methods instead" )] pub fn flags(&self) -> u32 { self.flags @@ -1528,19 +1693,20 @@ impl<'a> Formatter<'a> { /// let c = formatter.fill(); /// if let Some(width) = formatter.width() { /// for _ in 0..width { - /// write!(formatter, "{}", c)?; + /// write!(formatter, "{c}")?; /// } /// Ok(()) /// } else { - /// write!(formatter, "{}", c) + /// write!(formatter, "{c}") /// } /// } /// } /// - /// // We set alignment to the left with ">". - /// assert_eq!(&format!("{:G>3}", Foo), "GGG"); - /// assert_eq!(&format!("{:t>6}", Foo), "tttttt"); + /// // We set alignment to the right with ">". + /// assert_eq!(format!("{Foo:G>3}"), "GGG"); + /// assert_eq!(format!("{Foo:t>6}"), "tttttt"); /// ``` + #[must_use] #[stable(feature = "fmt_flags", since = "1.5.0")] pub fn fill(&self) -> char { self.fill @@ -1568,15 +1734,16 @@ impl<'a> Formatter<'a> { /// } else { /// "into the void" /// }; - /// write!(formatter, "{}", s) + /// write!(formatter, "{s}") /// } /// } /// - /// assert_eq!(&format!("{:<}", Foo), "left"); - /// assert_eq!(&format!("{:>}", Foo), "right"); - /// assert_eq!(&format!("{:^}", Foo), "center"); - /// assert_eq!(&format!("{}", Foo), "into the void"); + /// assert_eq!(format!("{Foo:<}"), "left"); + /// assert_eq!(format!("{Foo:>}"), "right"); + /// assert_eq!(format!("{Foo:^}"), "center"); + /// assert_eq!(format!("{Foo}"), "into the void"); /// ``` + #[must_use] #[stable(feature = "fmt_flags_align", since = "1.28.0")] pub fn align(&self) -> Option { match self.align { @@ -1600,7 +1767,7 @@ impl<'a> Formatter<'a> { /// fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { /// if let Some(width) = formatter.width() { /// // If we received a width, we use it - /// write!(formatter, "{:width$}", &format!("Foo({})", self.0), width = width) + /// write!(formatter, "{:width$}", format!("Foo({})", self.0), width = width) /// } else { /// // Otherwise we do nothing special /// write!(formatter, "Foo({})", self.0) @@ -1608,15 +1775,17 @@ impl<'a> Formatter<'a> { /// } /// } /// - /// assert_eq!(&format!("{:10}", Foo(23)), "Foo(23) "); - /// assert_eq!(&format!("{}", Foo(23)), "Foo(23)"); + /// assert_eq!(format!("{:10}", Foo(23)), "Foo(23) "); + /// assert_eq!(format!("{}", Foo(23)), "Foo(23)"); /// ``` + #[must_use] #[stable(feature = "fmt_flags", since = "1.5.0")] pub fn width(&self) -> Option { self.width } - /// Optionally specified precision for numeric types. + /// Optionally specified precision for numeric types. Alternatively, the + /// maximum width for string types. /// /// # Examples /// @@ -1637,9 +1806,10 @@ impl<'a> Formatter<'a> { /// } /// } /// - /// assert_eq!(&format!("{:.4}", Foo(23.2)), "Foo(23.2000)"); - /// assert_eq!(&format!("{}", Foo(23.2)), "Foo(23.20)"); + /// assert_eq!(format!("{:.4}", Foo(23.2)), "Foo(23.2000)"); + /// assert_eq!(format!("{}", Foo(23.2)), "Foo(23.20)"); /// ``` + #[must_use] #[stable(feature = "fmt_flags", since = "1.5.0")] pub fn precision(&self) -> Option { self.precision @@ -1660,16 +1830,18 @@ impl<'a> Formatter<'a> { /// write!(formatter, /// "Foo({}{})", /// if self.0 < 0 { '-' } else { '+' }, - /// self.0) + /// self.0.abs()) /// } else { /// write!(formatter, "Foo({})", self.0) /// } /// } /// } /// - /// assert_eq!(&format!("{:+}", Foo(23)), "Foo(+23)"); - /// assert_eq!(&format!("{}", Foo(23)), "Foo(23)"); + /// assert_eq!(format!("{:+}", Foo(23)), "Foo(+23)"); + /// assert_eq!(format!("{:+}", Foo(-23)), "Foo(-23)"); + /// assert_eq!(format!("{}", Foo(23)), "Foo(23)"); /// ``` + #[must_use] #[stable(feature = "fmt_flags", since = "1.5.0")] pub fn sign_plus(&self) -> bool { self.flags & (1 << FlagV1::SignPlus as u32) != 0 @@ -1695,9 +1867,10 @@ impl<'a> Formatter<'a> { /// } /// } /// - /// assert_eq!(&format!("{:-}", Foo(23)), "-Foo(23)"); - /// assert_eq!(&format!("{}", Foo(23)), "Foo(23)"); + /// assert_eq!(format!("{:-}", Foo(23)), "-Foo(23)"); + /// assert_eq!(format!("{}", Foo(23)), "Foo(23)"); /// ``` + #[must_use] #[stable(feature = "fmt_flags", since = "1.5.0")] pub fn sign_minus(&self) -> bool { self.flags & (1 << FlagV1::SignMinus as u32) != 0 @@ -1722,9 +1895,10 @@ impl<'a> Formatter<'a> { /// } /// } /// - /// assert_eq!(&format!("{:#}", Foo(23)), "Foo(23)"); - /// assert_eq!(&format!("{}", Foo(23)), "23"); + /// assert_eq!(format!("{:#}", Foo(23)), "Foo(23)"); + /// assert_eq!(format!("{}", Foo(23)), "23"); /// ``` + #[must_use] #[stable(feature = "fmt_flags", since = "1.5.0")] pub fn alternate(&self) -> bool { self.flags & (1 << FlagV1::Alternate as u32) != 0 @@ -1748,8 +1922,9 @@ impl<'a> Formatter<'a> { /// } /// } /// - /// assert_eq!(&format!("{:04}", Foo(23)), "23"); + /// assert_eq!(format!("{:04}", Foo(23)), "23"); /// ``` + #[must_use] #[stable(feature = "fmt_flags", since = "1.5.0")] pub fn sign_aware_zero_pad(&self) -> bool { self.flags & (1 << FlagV1::SignAwareZeroPad as u32) != 0 @@ -1768,8 +1943,7 @@ impl<'a> Formatter<'a> { /// Creates a [`DebugStruct`] builder designed to assist with creation of /// [`fmt::Debug`] implementations for structs. /// - /// [`DebugStruct`]: ../../std/fmt/struct.DebugStruct.html - /// [`fmt::Debug`]: ../../std/fmt/trait.Debug.html + /// [`fmt::Debug`]: self::Debug /// /// # Examples /// @@ -1807,6 +1981,129 @@ impl<'a> Formatter<'a> { builders::debug_struct_new(self, name) } + /// Used to shrink `derive(Debug)` code, for faster compilation and smaller binaries. + /// `debug_struct_fields_finish` is more general, but this is faster for 1 field. + #[doc(hidden)] + #[unstable(feature = "fmt_helpers_for_derive", issue = "none")] + pub fn debug_struct_field1_finish<'b>( + &'b mut self, + name: &str, + name1: &str, + value1: &dyn Debug, + ) -> Result { + let mut builder = builders::debug_struct_new(self, name); + builder.field(name1, value1); + builder.finish() + } + + /// Used to shrink `derive(Debug)` code, for faster compilation and smaller binaries. + /// `debug_struct_fields_finish` is more general, but this is faster for 2 fields. + #[doc(hidden)] + #[unstable(feature = "fmt_helpers_for_derive", issue = "none")] + pub fn debug_struct_field2_finish<'b>( + &'b mut self, + name: &str, + name1: &str, + value1: &dyn Debug, + name2: &str, + value2: &dyn Debug, + ) -> Result { + let mut builder = builders::debug_struct_new(self, name); + builder.field(name1, value1); + builder.field(name2, value2); + builder.finish() + } + + /// Used to shrink `derive(Debug)` code, for faster compilation and smaller binaries. + /// `debug_struct_fields_finish` is more general, but this is faster for 3 fields. + #[doc(hidden)] + #[unstable(feature = "fmt_helpers_for_derive", issue = "none")] + pub fn debug_struct_field3_finish<'b>( + &'b mut self, + name: &str, + name1: &str, + value1: &dyn Debug, + name2: &str, + value2: &dyn Debug, + name3: &str, + value3: &dyn Debug, + ) -> Result { + let mut builder = builders::debug_struct_new(self, name); + builder.field(name1, value1); + builder.field(name2, value2); + builder.field(name3, value3); + builder.finish() + } + + /// Used to shrink `derive(Debug)` code, for faster compilation and smaller binaries. + /// `debug_struct_fields_finish` is more general, but this is faster for 4 fields. + #[doc(hidden)] + #[unstable(feature = "fmt_helpers_for_derive", issue = "none")] + pub fn debug_struct_field4_finish<'b>( + &'b mut self, + name: &str, + name1: &str, + value1: &dyn Debug, + name2: &str, + value2: &dyn Debug, + name3: &str, + value3: &dyn Debug, + name4: &str, + value4: &dyn Debug, + ) -> Result { + let mut builder = builders::debug_struct_new(self, name); + builder.field(name1, value1); + builder.field(name2, value2); + builder.field(name3, value3); + builder.field(name4, value4); + builder.finish() + } + + /// Used to shrink `derive(Debug)` code, for faster compilation and smaller binaries. + /// `debug_struct_fields_finish` is more general, but this is faster for 5 fields. + #[doc(hidden)] + #[unstable(feature = "fmt_helpers_for_derive", issue = "none")] + pub fn debug_struct_field5_finish<'b>( + &'b mut self, + name: &str, + name1: &str, + value1: &dyn Debug, + name2: &str, + value2: &dyn Debug, + name3: &str, + value3: &dyn Debug, + name4: &str, + value4: &dyn Debug, + name5: &str, + value5: &dyn Debug, + ) -> Result { + let mut builder = builders::debug_struct_new(self, name); + builder.field(name1, value1); + builder.field(name2, value2); + builder.field(name3, value3); + builder.field(name4, value4); + builder.field(name5, value5); + builder.finish() + } + + /// Used to shrink `derive(Debug)` code, for faster compilation and smaller binaries. + /// For the cases not covered by `debug_struct_field[12345]_finish`. + #[doc(hidden)] + #[unstable(feature = "fmt_helpers_for_derive", issue = "none")] + pub fn debug_struct_fields_finish<'b>( + &'b mut self, + name: &str, + names: &[&str], + values: &[&dyn Debug], + ) -> Result { + assert_eq!(names.len(), values.len()); + let mut builder = builders::debug_struct_new(self, name); + for (name, value) in iter::zip(names, values) { + builder.field(name, value); + } + builder.finish() + } + /// Creates a `DebugTuple` builder designed to assist with creation of /// `fmt::Debug` implementations for tuple structs. /// @@ -1838,6 +2135,108 @@ impl<'a> Formatter<'a> { builders::debug_tuple_new(self, name) } + /// Used to shrink `derive(Debug)` code, for faster compilation and smaller binaries. + /// `debug_tuple_fields_finish` is more general, but this is faster for 1 field. + #[doc(hidden)] + #[unstable(feature = "fmt_helpers_for_derive", issue = "none")] + pub fn debug_tuple_field1_finish<'b>(&'b mut self, name: &str, value1: &dyn Debug) -> Result { + let mut builder = builders::debug_tuple_new(self, name); + builder.field(value1); + builder.finish() + } + + /// Used to shrink `derive(Debug)` code, for faster compilation and smaller binaries. + /// `debug_tuple_fields_finish` is more general, but this is faster for 2 fields. + #[doc(hidden)] + #[unstable(feature = "fmt_helpers_for_derive", issue = "none")] + pub fn debug_tuple_field2_finish<'b>( + &'b mut self, + name: &str, + value1: &dyn Debug, + value2: &dyn Debug, + ) -> Result { + let mut builder = builders::debug_tuple_new(self, name); + builder.field(value1); + builder.field(value2); + builder.finish() + } + + /// Used to shrink `derive(Debug)` code, for faster compilation and smaller binaries. + /// `debug_tuple_fields_finish` is more general, but this is faster for 3 fields. + #[doc(hidden)] + #[unstable(feature = "fmt_helpers_for_derive", issue = "none")] + pub fn debug_tuple_field3_finish<'b>( + &'b mut self, + name: &str, + value1: &dyn Debug, + value2: &dyn Debug, + value3: &dyn Debug, + ) -> Result { + let mut builder = builders::debug_tuple_new(self, name); + builder.field(value1); + builder.field(value2); + builder.field(value3); + builder.finish() + } + + /// Used to shrink `derive(Debug)` code, for faster compilation and smaller binaries. + /// `debug_tuple_fields_finish` is more general, but this is faster for 4 fields. + #[doc(hidden)] + #[unstable(feature = "fmt_helpers_for_derive", issue = "none")] + pub fn debug_tuple_field4_finish<'b>( + &'b mut self, + name: &str, + value1: &dyn Debug, + value2: &dyn Debug, + value3: &dyn Debug, + value4: &dyn Debug, + ) -> Result { + let mut builder = builders::debug_tuple_new(self, name); + builder.field(value1); + builder.field(value2); + builder.field(value3); + builder.field(value4); + builder.finish() + } + + /// Used to shrink `derive(Debug)` code, for faster compilation and smaller binaries. + /// `debug_tuple_fields_finish` is more general, but this is faster for 5 fields. + #[doc(hidden)] + #[unstable(feature = "fmt_helpers_for_derive", issue = "none")] + pub fn debug_tuple_field5_finish<'b>( + &'b mut self, + name: &str, + value1: &dyn Debug, + value2: &dyn Debug, + value3: &dyn Debug, + value4: &dyn Debug, + value5: &dyn Debug, + ) -> Result { + let mut builder = builders::debug_tuple_new(self, name); + builder.field(value1); + builder.field(value2); + builder.field(value3); + builder.field(value4); + builder.field(value5); + builder.finish() + } + + /// Used to shrink `derive(Debug)` code, for faster compilation and smaller binaries. + /// For the cases not covered by `debug_tuple_field[12345]_finish`. + #[doc(hidden)] + #[unstable(feature = "fmt_helpers_for_derive", issue = "none")] + pub fn debug_tuple_fields_finish<'b>( + &'b mut self, + name: &str, + values: &[&dyn Debug], + ) -> Result { + let mut builder = builders::debug_tuple_new(self, name); + for value in values { + builder.field(value); + } + builder.finish() + } + /// Creates a `DebugList` builder designed to assist with creation of /// `fmt::Debug` implementations for list-like structures. /// @@ -1880,7 +2279,7 @@ impl<'a> Formatter<'a> { /// assert_eq!(format!("{:?}", Foo(vec![10, 11])), "{10, 11}"); /// ``` /// - /// [`format_args!`]: ../../std/macro.format_args.html + /// [`format_args!`]: crate::format_args /// /// In this more complex example, we use [`format_args!`] and `.debug_set()` /// to build a list of match arms: @@ -2022,7 +2421,11 @@ impl Debug for str { f.write_char('"')?; let mut from = 0; for (i, c) in self.char_indices() { - let esc = c.escape_debug(); + let esc = c.escape_debug_ext(EscapeDebugExtArgs { + escape_grapheme_extended: true, + escape_single_quote: false, + escape_double_quote: true, + }); // If char needs escaping, flush backlog so far and write, else skip if esc.len() != 1 { f.write_str(&self[from..i])?; @@ -2048,7 +2451,11 @@ impl Display for str { impl Debug for char { fn fmt(&self, f: &mut Formatter<'_>) -> Result { f.write_char('\'')?; - for c in self.escape_debug() { + for c in self.escape_debug_ext(EscapeDebugExtArgs { + escape_grapheme_extended: true, + escape_single_quote: true, + escape_double_quote: false, + }) { f.write_char(c)? } f.write_char('\'') @@ -2069,29 +2476,41 @@ impl Display for char { #[stable(feature = "rust1", since = "1.0.0")] impl Pointer for *const T { fn fmt(&self, f: &mut Formatter<'_>) -> Result { - let old_width = f.width; - let old_flags = f.flags; - - // The alternate flag is already treated by LowerHex as being special- - // it denotes whether to prefix with 0x. We use it to work out whether - // or not to zero extend, and then unconditionally set it to get the - // prefix. - if f.alternate() { - f.flags |= 1 << (FlagV1::SignAwareZeroPad as u32); - - if f.width.is_none() { - f.width = Some(((mem::size_of::() * 8) / 4) + 2); - } - } - f.flags |= 1 << (FlagV1::Alternate as u32); + // Cast is needed here because `.expose_addr()` requires `T: Sized`. + pointer_fmt_inner((*self as *const ()).expose_addr(), f) + } +} - let ret = LowerHex::fmt(&(*self as *const () as usize), f); +/// Since the formatting will be identical for all pointer types, use a non-monomorphized +/// implementation for the actual formatting to reduce the amount of codegen work needed. +/// +/// This uses `ptr_addr: usize` and not `ptr: *const ()` to be able to use this for +/// `fn(...) -> ...` without using [problematic] "Oxford Casts". +/// +/// [problematic]: https://github.com/rust-lang/rust/issues/95489 +pub(crate) fn pointer_fmt_inner(ptr_addr: usize, f: &mut Formatter<'_>) -> Result { + let old_width = f.width; + let old_flags = f.flags; - f.width = old_width; - f.flags = old_flags; + // The alternate flag is already treated by LowerHex as being special- + // it denotes whether to prefix with 0x. We use it to work out whether + // or not to zero extend, and then unconditionally set it to get the + // prefix. + if f.alternate() { + f.flags |= 1 << (FlagV1::SignAwareZeroPad as u32); - ret + if f.width.is_none() { + f.width = Some((usize::BITS / 4) as usize + 2); + } } + f.flags |= 1 << (FlagV1::Alternate as u32); + + let ret = LowerHex::fmt(&ptr_addr, f); + + f.width = old_width; + f.flags = old_flags; + + ret } #[stable(feature = "rust1", since = "1.0.0")] @@ -2137,29 +2556,46 @@ macro_rules! peel { macro_rules! tuple { () => (); ( $($name:ident,)+ ) => ( - #[stable(feature = "rust1", since = "1.0.0")] - impl<$($name:Debug),+> Debug for ($($name,)+) where last_type!($($name,)+): ?Sized { - #[allow(non_snake_case, unused_assignments)] - fn fmt(&self, f: &mut Formatter<'_>) -> Result { - let mut builder = f.debug_tuple(""); - let ($(ref $name,)+) = *self; - $( - builder.field(&$name); - )+ - - builder.finish() + maybe_tuple_doc! { + $($name)+ @ + #[stable(feature = "rust1", since = "1.0.0")] + impl<$($name:Debug),+> Debug for ($($name,)+) where last_type!($($name,)+): ?Sized { + #[allow(non_snake_case, unused_assignments)] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + let mut builder = f.debug_tuple(""); + let ($(ref $name,)+) = *self; + $( + builder.field(&$name); + )+ + + builder.finish() + } } } peel! { $($name,)+ } ) } +macro_rules! maybe_tuple_doc { + ($a:ident @ #[$meta:meta] $item:item) => { + #[doc(fake_variadic)] + #[doc = "This trait is implemented for tuples up to twelve items long."] + #[$meta] + $item + }; + ($a:ident $($rest_a:ident)+ @ #[$meta:meta] $item:item) => { + #[doc(hidden)] + #[$meta] + $item + }; +} + macro_rules! last_type { ($a:ident,) => { $a }; ($a:ident, $($rest_a:ident,)+) => { last_type!($($rest_a,)+) }; } -tuple! { T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, } +tuple! { E, D, C, B, A, Z, Y, X, W, V, U, T, } #[stable(feature = "rust1", since = "1.0.0")] impl Debug for [T] { @@ -2178,7 +2614,7 @@ impl Debug for () { #[stable(feature = "rust1", since = "1.0.0")] impl Debug for PhantomData { fn fmt(&self, f: &mut Formatter<'_>) -> Result { - f.pad("PhantomData") + write!(f, "PhantomData<{}>", crate::any::type_name::()) } } @@ -2226,11 +2662,19 @@ impl Debug for RefMut<'_, T> { } #[stable(feature = "core_impl_debug", since = "1.9.0")] -impl Debug for UnsafeCell { +impl Debug for UnsafeCell { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + f.debug_struct("UnsafeCell").finish_non_exhaustive() + } +} + +#[unstable(feature = "sync_unsafe_cell", issue = "95439")] +impl Debug for SyncUnsafeCell { fn fmt(&self, f: &mut Formatter<'_>) -> Result { - f.pad("UnsafeCell") + f.debug_struct("SyncUnsafeCell").finish_non_exhaustive() } } -// If you expected tests to be here, look instead at the ui/ifmt.rs test, +// If you expected tests to be here, look instead at the core/tests/fmt.rs file, // it's a lot easier than creating all of the rt::Piece structures here. +// There are also tests in the alloc crate, for those that need allocations. diff --git a/crux-mir/lib/core/src/fmt/nofloat.rs b/crux-mir/lib/core/src/fmt/nofloat.rs new file mode 100644 index 000000000..cfb94cd9d --- /dev/null +++ b/crux-mir/lib/core/src/fmt/nofloat.rs @@ -0,0 +1,15 @@ +use crate::fmt::{Debug, Formatter, Result}; + +macro_rules! floating { + ($ty:ident) => { + #[stable(feature = "rust1", since = "1.0.0")] + impl Debug for $ty { + fn fmt(&self, _fmt: &mut Formatter<'_>) -> Result { + panic!("floating point support is turned off"); + } + } + }; +} + +floating! { f32 } +floating! { f64 } diff --git a/crux-mir/lib/core/src/fmt/num.rs b/crux-mir/lib/core/src/fmt/num.rs index 630aa3e8a..d8365ae9b 100644 --- a/crux-mir/lib/core/src/fmt/num.rs +++ b/crux-mir/lib/core/src/fmt/num.rs @@ -2,14 +2,14 @@ use crate::fmt; use crate::mem::MaybeUninit; -use crate::num::flt2dec; +use crate::num::fmt as numfmt; use crate::ops::{Div, Rem, Sub}; use crate::ptr; use crate::slice; use crate::str; #[doc(hidden)] -trait Int: +trait DisplayInt: PartialEq + PartialOrd + Div + Rem + Sub + Copy { fn zero() -> Self; @@ -21,22 +21,39 @@ trait Int: fn to_u128(&self) -> u128; } -macro_rules! doit { - ($($t:ident)*) => ($(impl Int for $t { - fn zero() -> Self { 0 } - fn from_u8(u: u8) -> Self { u as Self } - fn to_u8(&self) -> u8 { *self as u8 } - fn to_u16(&self) -> u16 { *self as u16 } - fn to_u32(&self) -> u32 { *self as u32 } - fn to_u64(&self) -> u64 { *self as u64 } - fn to_u128(&self) -> u128 { *self as u128 } - })*) +macro_rules! impl_int { + ($($t:ident)*) => ( + $(impl DisplayInt for $t { + fn zero() -> Self { 0 } + fn from_u8(u: u8) -> Self { u as Self } + fn to_u8(&self) -> u8 { *self as u8 } + fn to_u16(&self) -> u16 { *self as u16 } + fn to_u32(&self) -> u32 { *self as u32 } + fn to_u64(&self) -> u64 { *self as u64 } + fn to_u128(&self) -> u128 { *self as u128 } + })* + ) } -doit! { i8 i16 i32 i64 i128 isize u8 u16 u32 u64 u128 usize } +macro_rules! impl_uint { + ($($t:ident)*) => ( + $(impl DisplayInt for $t { + fn zero() -> Self { 0 } + fn from_u8(u: u8) -> Self { u as Self } + fn to_u8(&self) -> u8 { *self as u8 } + fn to_u16(&self) -> u16 { *self as u16 } + fn to_u32(&self) -> u32 { *self as u32 } + fn to_u64(&self) -> u64 { *self as u64 } + fn to_u128(&self) -> u128 { *self as u128 } + })* + ) +} + +impl_int! { i8 i16 i32 i64 i128 isize } +impl_uint! { u8 u16 u32 u64 u128 usize } /// A type that represents a specific radix #[doc(hidden)] -trait GenericRadix { +trait GenericRadix: Sized { /// The number of digits. const BASE: u8; @@ -47,12 +64,12 @@ trait GenericRadix { fn digit(x: u8) -> u8; /// Format an integer using the radix using a formatter. - fn fmt_int(&self, mut x: T, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fn fmt_int(&self, mut x: T, f: &mut fmt::Formatter<'_>) -> fmt::Result { // The radix can be as low as 2, so we need a buffer of at least 128 // characters for a base 2 number. let zero = T::zero(); let is_nonnegative = x >= zero; - let mut buf = [0; 128]; + let mut buf = [MaybeUninit::::uninit(); 128]; let mut curr = buf.len(); let base = T::from_u8(Self::BASE); if is_nonnegative { @@ -61,7 +78,7 @@ trait GenericRadix { for byte in buf.iter_mut().rev() { let n = x % base; // Get the current place value. x = x / base; // Deaccumulate the number. - *byte = Self::digit(n.to_u8()); // Store the digit in the buffer. + byte.write(Self::digit(n.to_u8())); // Store the digit in the buffer. curr -= 1; if x == zero { // No more digits left to accumulate. @@ -73,7 +90,7 @@ trait GenericRadix { for byte in buf.iter_mut().rev() { let n = zero - (x % base); // Get the current place value. x = x / base; // Deaccumulate the number. - *byte = Self::digit(n.to_u8()); // Store the digit in the buffer. + byte.write(Self::digit(n.to_u8())); // Store the digit in the buffer. curr -= 1; if x == zero { // No more digits left to accumulate. @@ -85,7 +102,10 @@ trait GenericRadix { // SAFETY: The only chars in `buf` are created by `Self::digit` which are assumed to be // valid UTF-8 let buf = unsafe { - str::from_utf8_unchecked(buf) + str::from_utf8_unchecked(slice::from_raw_parts( + MaybeUninit::slice_as_ptr(buf), + buf.len(), + )) }; f.pad_integral(is_nonnegative, Self::PREFIX, buf) } @@ -124,13 +144,11 @@ macro_rules! radix { radix! { Binary, 2, "0b", x @ 0 ..= 1 => b'0' + x } radix! { Octal, 8, "0o", x @ 0 ..= 7 => b'0' + x } -radix! { LowerHex, 16, "0x", x @ 0 ..= 9 => b'0' + x, -x @ 10 ..= 15 => b'a' + (x - 10) } -radix! { UpperHex, 16, "0x", x @ 0 ..= 9 => b'0' + x, -x @ 10 ..= 15 => b'A' + (x - 10) } +radix! { LowerHex, 16, "0x", x @ 0 ..= 9 => b'0' + x, x @ 10 ..= 15 => b'a' + (x - 10) } +radix! { UpperHex, 16, "0x", x @ 0 ..= 9 => b'0' + x, x @ 10 ..= 15 => b'A' + (x - 10) } macro_rules! int_base { - ($Trait:ident for $T:ident as $U:ident -> $Radix:ident) => { + (fmt::$Trait:ident for $T:ident as $U:ident -> $Radix:ident) => { #[stable(feature = "rust1", since = "1.0.0")] impl fmt::$Trait for $T { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -140,8 +158,27 @@ macro_rules! int_base { }; } +macro_rules! integer { + ($Int:ident, $Uint:ident) => { + int_base! { fmt::Binary for $Int as $Uint -> Binary } + int_base! { fmt::Octal for $Int as $Uint -> Octal } + int_base! { fmt::LowerHex for $Int as $Uint -> LowerHex } + int_base! { fmt::UpperHex for $Int as $Uint -> UpperHex } + + int_base! { fmt::Binary for $Uint as $Uint -> Binary } + int_base! { fmt::Octal for $Uint as $Uint -> Octal } + int_base! { fmt::LowerHex for $Uint as $Uint -> LowerHex } + int_base! { fmt::UpperHex for $Uint as $Uint -> UpperHex } + }; +} +integer! { isize, usize } +integer! { i8, u8 } +integer! { i16, u16 } +integer! { i32, u32 } +integer! { i64, u64 } +integer! { i128, u128 } macro_rules! debug { - ($T:ident) => { + ($($T:ident)*) => {$( #[stable(feature = "rust1", since = "1.0.0")] impl fmt::Debug for $T { #[inline] @@ -155,31 +192,14 @@ macro_rules! debug { } } } - }; + )*}; } - -macro_rules! integer { - ($Int:ident, $Uint:ident) => { - int_base! { Binary for $Int as $Uint -> Binary } - int_base! { Octal for $Int as $Uint -> Octal } - int_base! { LowerHex for $Int as $Uint -> LowerHex } - int_base! { UpperHex for $Int as $Uint -> UpperHex } - debug! { $Int } - - int_base! { Binary for $Uint as $Uint -> Binary } - int_base! { Octal for $Uint as $Uint -> Octal } - int_base! { LowerHex for $Uint as $Uint -> LowerHex } - int_base! { UpperHex for $Uint as $Uint -> UpperHex } - debug! { $Uint } - }; +debug! { + i8 i16 i32 i64 i128 isize + u8 u16 u32 u64 u128 usize } -integer! { isize, usize } -integer! { i8, u8 } -integer! { i16, u16 } -integer! { i32, u32 } -integer! { i64, u64 } -integer! { i128, u128 } +// 2 digit decimal look up table static DEC_DIGITS_LUT: &[u8; 200] = b"0001020304050607080910111213141516171819\ 2021222324252627282930313233343536373839\ 4041424344454647484950515253545556575859\ @@ -189,47 +209,84 @@ static DEC_DIGITS_LUT: &[u8; 200] = b"0001020304050607080910111213141516171819\ macro_rules! impl_Display { ($($t:ident),* as $u:ident via $conv_fn:ident named $name:ident) => { fn $name(mut n: $u, is_nonnegative: bool, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let mut buf = [0; 39]; - let mut curr = buf.len() as usize; - let lut = DEC_DIGITS_LUT; + // 2^128 is about 3*10^38, so 39 gives an extra byte of space + let mut buf = [MaybeUninit::::uninit(); 39]; + let mut curr = buf.len(); + let buf_ptr = MaybeUninit::slice_as_mut_ptr(&mut buf); + let lut_ptr = DEC_DIGITS_LUT.as_ptr(); - // decode 2 more chars, if > 2 chars - while n >= 100 { - let d1 = ((n % 100) as usize) << 1; - n /= 100; - curr -= 2; - buf[curr .. curr + 2].copy_from_slice(&lut[d1 .. d1 + 2]); - } + // SAFETY: Since `d1` and `d2` are always less than or equal to `198`, we + // can copy from `lut_ptr[d1..d1 + 1]` and `lut_ptr[d2..d2 + 1]`. To show + // that it's OK to copy into `buf_ptr`, notice that at the beginning + // `curr == buf.len() == 39 > log(n)` since `n < 2^128 < 10^39`, and at + // each step this is kept the same as `n` is divided. Since `n` is always + // non-negative, this means that `curr > 0` so `buf_ptr[curr..curr + 1]` + // is safe to access. + unsafe { + // need at least 16 bits for the 4-characters-at-a-time to work. + assert!(crate::mem::size_of::<$u>() >= 2); + + // eagerly decode 4 characters at a time + while n >= 10000 { + let rem = (n % 10000) as usize; + n /= 10000; + + let d1 = (rem / 100) << 1; + let d2 = (rem % 100) << 1; + curr -= 4; + + // We are allowed to copy to `buf_ptr[curr..curr + 3]` here since + // otherwise `curr < 0`. But then `n` was originally at least `10000^10` + // which is `10^40 > 2^128 > n`. + ptr::copy_nonoverlapping(lut_ptr.add(d1), buf_ptr.add(curr), 2); + ptr::copy_nonoverlapping(lut_ptr.add(d2), buf_ptr.add(curr + 2), 2); + } - // decode last 1 or 2 chars - if n < 10 { - curr -= 1; - buf[curr] = (n as u8) + b'0'; - } else { - let d1 = (n as usize) << 1; - curr -= 2; - buf[curr .. curr + 2].copy_from_slice(&lut[d1 .. d1 + 2]); + // if we reach here numbers are <= 9999, so at most 4 chars long + let mut n = n as usize; // possibly reduce 64bit math + + // decode 2 more chars, if > 2 chars + if n >= 100 { + let d1 = (n % 100) << 1; + n /= 100; + curr -= 2; + ptr::copy_nonoverlapping(lut_ptr.add(d1), buf_ptr.add(curr), 2); + } + + // decode last 1 or 2 chars + if n < 10 { + curr -= 1; + *buf_ptr.add(curr) = (n as u8) + b'0'; + } else { + let d1 = n << 1; + curr -= 2; + ptr::copy_nonoverlapping(lut_ptr.add(d1), buf_ptr.add(curr), 2); + } } - let buf_slice = unsafe { str::from_utf8_unchecked(&buf[curr..]) }; + // SAFETY: `curr` > 0 (since we made `buf` large enough), and all the chars are valid + // UTF-8 since `DEC_DIGITS_LUT` is + let buf_slice = unsafe { + str::from_utf8_unchecked( + slice::from_raw_parts(buf_ptr.add(curr), buf.len() - curr)) + }; f.pad_integral(is_nonnegative, "", buf_slice) } - $( - #[stable(feature = "rust1", since = "1.0.0")] - impl fmt::Display for $t { - #[allow(unused_comparisons)] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let is_nonnegative = *self >= 0; - let n = if is_nonnegative { - self.$conv_fn() - } else { - // convert the negative num to positive by summing 1 to it's 2 complement - (!self.$conv_fn()).wrapping_add(1) - }; - $name(n, is_nonnegative, f) - } - })* + $(#[stable(feature = "rust1", since = "1.0.0")] + impl fmt::Display for $t { + #[allow(unused_comparisons)] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let is_nonnegative = *self >= 0; + let n = if is_nonnegative { + self.$conv_fn() + } else { + // convert the negative num to positive by summing 1 to it's 2 complement + (!self.$conv_fn()).wrapping_add(1) + }; + $name(n, is_nonnegative, f) + } + })* }; } @@ -248,7 +305,6 @@ macro_rules! impl_Exp { n /= 10; exponent += 1; } - let trailing_zeros = exponent; let (added_precision, subtracted_precision) = match f.precision() { Some(fmt_prec) => { @@ -261,10 +317,10 @@ macro_rules! impl_Exp { } (fmt_prec.saturating_sub(prec), prec.saturating_sub(fmt_prec)) } - None => (0,0) + None => (0, 0) }; for _ in 1..subtracted_precision { - n/=10; + n /= 10; exponent += 1; } if subtracted_precision != 0 { @@ -276,25 +332,25 @@ macro_rules! impl_Exp { n += 1; } } - (n, exponent, trailing_zeros, added_precision) + (n, exponent, exponent, added_precision) }; // 39 digits (worst case u128) + . = 40 // Since `curr` always decreases by the number of digits copied, this means // that `curr >= 0`. let mut buf = [MaybeUninit::::uninit(); 40]; - let mut curr = buf.len() as isize; //index for buf - let buf_ptr = MaybeUninit::first_ptr_mut(&mut buf); + let mut curr = buf.len(); //index for buf + let buf_ptr = MaybeUninit::slice_as_mut_ptr(&mut buf); let lut_ptr = DEC_DIGITS_LUT.as_ptr(); // decode 2 chars at a time while n >= 100 { - let d1 = ((n % 100) as isize) << 1; + let d1 = ((n % 100) as usize) << 1; curr -= 2; // SAFETY: `d1 <= 198`, so we can copy from `lut_ptr[d1..d1 + 2]` since // `DEC_DIGITS_LUT` has a length of 200. unsafe { - ptr::copy_nonoverlapping(lut_ptr.offset(d1), buf_ptr.offset(curr), 2); + ptr::copy_nonoverlapping(lut_ptr.add(d1), buf_ptr.add(curr), 2); } n /= 100; exponent += 2; @@ -306,7 +362,7 @@ macro_rules! impl_Exp { curr -= 1; // SAFETY: Safe since `40 > curr >= 0` (see comment) unsafe { - *buf_ptr.offset(curr) = (n as u8 % 10_u8) + b'0'; + *buf_ptr.add(curr) = (n as u8 % 10_u8) + b'0'; } n /= 10; exponent += 1; @@ -316,7 +372,7 @@ macro_rules! impl_Exp { curr -= 1; // SAFETY: Safe since `40 > curr >= 0` unsafe { - *buf_ptr.offset(curr) = b'.'; + *buf_ptr.add(curr) = b'.'; } } @@ -324,34 +380,34 @@ macro_rules! impl_Exp { let buf_slice = unsafe { // decode last character curr -= 1; - *buf_ptr.offset(curr) = (n as u8) + b'0'; + *buf_ptr.add(curr) = (n as u8) + b'0'; let len = buf.len() - curr as usize; - slice::from_raw_parts(buf_ptr.offset(curr), len) + slice::from_raw_parts(buf_ptr.add(curr), len) }; // stores 'e' (or 'E') and the up to 2-digit exponent let mut exp_buf = [MaybeUninit::::uninit(); 3]; - let exp_ptr = MaybeUninit::first_ptr_mut(&mut exp_buf); + let exp_ptr = MaybeUninit::slice_as_mut_ptr(&mut exp_buf); // SAFETY: In either case, `exp_buf` is written within bounds and `exp_ptr[..len]` // is contained within `exp_buf` since `len <= 3`. let exp_slice = unsafe { - *exp_ptr.offset(0) = if upper {b'E'} else {b'e'}; + *exp_ptr.add(0) = if upper { b'E' } else { b'e' }; let len = if exponent < 10 { - *exp_ptr.offset(1) = (exponent as u8) + b'0'; + *exp_ptr.add(1) = (exponent as u8) + b'0'; 2 } else { let off = exponent << 1; - ptr::copy_nonoverlapping(lut_ptr.offset(off), exp_ptr.offset(1), 2); + ptr::copy_nonoverlapping(lut_ptr.add(off), exp_ptr.add(1), 2); 3 }; slice::from_raw_parts(exp_ptr, len) }; let parts = &[ - flt2dec::Part::Copy(buf_slice), - flt2dec::Part::Zero(added_precision), - flt2dec::Part::Copy(exp_slice) + numfmt::Part::Copy(buf_slice), + numfmt::Part::Zero(added_precision), + numfmt::Part::Copy(exp_slice) ]; let sign = if !is_nonnegative { "-" @@ -360,7 +416,7 @@ macro_rules! impl_Exp { } else { "" }; - let formatted = flt2dec::Formatted{sign, parts}; + let formatted = numfmt::Formatted{sign, parts}; f.pad_formatted_parts(&formatted) } @@ -420,6 +476,208 @@ mod imp { impl_Exp!(i8, u8, i16, u16, i32, u32, isize, usize as u32 via to_u32 named exp_u32); impl_Exp!(i64, u64 as u64 via to_u64 named exp_u64); } - -impl_Display!(i128, u128 as u128 via to_u128 named fmt_u128); impl_Exp!(i128, u128 as u128 via to_u128 named exp_u128); + +/// Helper function for writing a u64 into `buf` going from last to first, with `curr`. +fn parse_u64_into(mut n: u64, buf: &mut [MaybeUninit; N], curr: &mut usize) { + let buf_ptr = MaybeUninit::slice_as_mut_ptr(buf); + let lut_ptr = DEC_DIGITS_LUT.as_ptr(); + assert!(*curr > 19); + + // SAFETY: + // Writes at most 19 characters into the buffer. Guaranteed that any ptr into LUT is at most + // 198, so will never OOB. There is a check above that there are at least 19 characters + // remaining. + unsafe { + if n >= 1e16 as u64 { + let to_parse = n % 1e16 as u64; + n /= 1e16 as u64; + + // Some of these are nops but it looks more elegant this way. + let d1 = ((to_parse / 1e14 as u64) % 100) << 1; + let d2 = ((to_parse / 1e12 as u64) % 100) << 1; + let d3 = ((to_parse / 1e10 as u64) % 100) << 1; + let d4 = ((to_parse / 1e8 as u64) % 100) << 1; + let d5 = ((to_parse / 1e6 as u64) % 100) << 1; + let d6 = ((to_parse / 1e4 as u64) % 100) << 1; + let d7 = ((to_parse / 1e2 as u64) % 100) << 1; + let d8 = ((to_parse / 1e0 as u64) % 100) << 1; + + *curr -= 16; + + ptr::copy_nonoverlapping(lut_ptr.add(d1 as usize), buf_ptr.add(*curr + 0), 2); + ptr::copy_nonoverlapping(lut_ptr.add(d2 as usize), buf_ptr.add(*curr + 2), 2); + ptr::copy_nonoverlapping(lut_ptr.add(d3 as usize), buf_ptr.add(*curr + 4), 2); + ptr::copy_nonoverlapping(lut_ptr.add(d4 as usize), buf_ptr.add(*curr + 6), 2); + ptr::copy_nonoverlapping(lut_ptr.add(d5 as usize), buf_ptr.add(*curr + 8), 2); + ptr::copy_nonoverlapping(lut_ptr.add(d6 as usize), buf_ptr.add(*curr + 10), 2); + ptr::copy_nonoverlapping(lut_ptr.add(d7 as usize), buf_ptr.add(*curr + 12), 2); + ptr::copy_nonoverlapping(lut_ptr.add(d8 as usize), buf_ptr.add(*curr + 14), 2); + } + if n >= 1e8 as u64 { + let to_parse = n % 1e8 as u64; + n /= 1e8 as u64; + + // Some of these are nops but it looks more elegant this way. + let d1 = ((to_parse / 1e6 as u64) % 100) << 1; + let d2 = ((to_parse / 1e4 as u64) % 100) << 1; + let d3 = ((to_parse / 1e2 as u64) % 100) << 1; + let d4 = ((to_parse / 1e0 as u64) % 100) << 1; + *curr -= 8; + + ptr::copy_nonoverlapping(lut_ptr.add(d1 as usize), buf_ptr.add(*curr + 0), 2); + ptr::copy_nonoverlapping(lut_ptr.add(d2 as usize), buf_ptr.add(*curr + 2), 2); + ptr::copy_nonoverlapping(lut_ptr.add(d3 as usize), buf_ptr.add(*curr + 4), 2); + ptr::copy_nonoverlapping(lut_ptr.add(d4 as usize), buf_ptr.add(*curr + 6), 2); + } + // `n` < 1e8 < (1 << 32) + let mut n = n as u32; + if n >= 1e4 as u32 { + let to_parse = n % 1e4 as u32; + n /= 1e4 as u32; + + let d1 = (to_parse / 100) << 1; + let d2 = (to_parse % 100) << 1; + *curr -= 4; + + ptr::copy_nonoverlapping(lut_ptr.add(d1 as usize), buf_ptr.add(*curr + 0), 2); + ptr::copy_nonoverlapping(lut_ptr.add(d2 as usize), buf_ptr.add(*curr + 2), 2); + } + + // `n` < 1e4 < (1 << 16) + let mut n = n as u16; + if n >= 100 { + let d1 = (n % 100) << 1; + n /= 100; + *curr -= 2; + ptr::copy_nonoverlapping(lut_ptr.add(d1 as usize), buf_ptr.add(*curr), 2); + } + + // decode last 1 or 2 chars + if n < 10 { + *curr -= 1; + *buf_ptr.add(*curr) = (n as u8) + b'0'; + } else { + let d1 = n << 1; + *curr -= 2; + ptr::copy_nonoverlapping(lut_ptr.add(d1 as usize), buf_ptr.add(*curr), 2); + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Display for u128 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt_u128(*self, true, f) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Display for i128 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let is_nonnegative = *self >= 0; + let n = if is_nonnegative { + self.to_u128() + } else { + // convert the negative num to positive by summing 1 to it's 2 complement + (!self.to_u128()).wrapping_add(1) + }; + fmt_u128(n, is_nonnegative, f) + } +} + +/// Specialized optimization for u128. Instead of taking two items at a time, it splits +/// into at most 2 u64s, and then chunks by 10e16, 10e8, 10e4, 10e2, and then 10e1. +/// It also has to handle 1 last item, as 10^40 > 2^128 > 10^39, whereas +/// 10^20 > 2^64 > 10^19. +fn fmt_u128(n: u128, is_nonnegative: bool, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // 2^128 is about 3*10^38, so 39 gives an extra byte of space + let mut buf = [MaybeUninit::::uninit(); 39]; + let mut curr = buf.len(); + + let (n, rem) = udiv_1e19(n); + parse_u64_into(rem, &mut buf, &mut curr); + + if n != 0 { + // 0 pad up to point + let target = buf.len() - 19; + // SAFETY: Guaranteed that we wrote at most 19 bytes, and there must be space + // remaining since it has length 39 + unsafe { + ptr::write_bytes( + MaybeUninit::slice_as_mut_ptr(&mut buf).add(target), + b'0', + curr - target, + ); + } + curr = target; + + let (n, rem) = udiv_1e19(n); + parse_u64_into(rem, &mut buf, &mut curr); + // Should this following branch be annotated with unlikely? + if n != 0 { + let target = buf.len() - 38; + // The raw `buf_ptr` pointer is only valid until `buf` is used the next time, + // buf `buf` is not used in this scope so we are good. + let buf_ptr = MaybeUninit::slice_as_mut_ptr(&mut buf); + // SAFETY: At this point we wrote at most 38 bytes, pad up to that point, + // There can only be at most 1 digit remaining. + unsafe { + ptr::write_bytes(buf_ptr.add(target), b'0', curr - target); + curr = target - 1; + *buf_ptr.add(curr) = (n as u8) + b'0'; + } + } + } + + // SAFETY: `curr` > 0 (since we made `buf` large enough), and all the chars are valid + // UTF-8 since `DEC_DIGITS_LUT` is + let buf_slice = unsafe { + str::from_utf8_unchecked(slice::from_raw_parts( + MaybeUninit::slice_as_mut_ptr(&mut buf).add(curr), + buf.len() - curr, + )) + }; + f.pad_integral(is_nonnegative, "", buf_slice) +} + +/// Partition of `n` into n > 1e19 and rem <= 1e19 +/// +/// Integer division algorithm is based on the following paper: +/// +/// T. Granlund and P. Montgomery, “Division by Invariant Integers Using Multiplication” +/// in Proc. of the SIGPLAN94 Conference on Programming Language Design and +/// Implementation, 1994, pp. 61–72 +/// +fn udiv_1e19(n: u128) -> (u128, u64) { + const DIV: u64 = 1e19 as u64; + const FACTOR: u128 = 156927543384667019095894735580191660403; + + let quot = if n < 1 << 83 { + ((n >> 19) as u64 / (DIV >> 19)) as u128 + } else { + u128_mulhi(n, FACTOR) >> 62 + }; + + let rem = (n - quot * DIV as u128) as u64; + (quot, rem) +} + +/// Multiply unsigned 128 bit integers, return upper 128 bits of the result +#[inline] +fn u128_mulhi(x: u128, y: u128) -> u128 { + let x_lo = x as u64; + let x_hi = (x >> 64) as u64; + let y_lo = y as u64; + let y_hi = (y >> 64) as u64; + + // handle possibility of overflow + let carry = (x_lo as u128 * y_lo as u128) >> 64; + let m = x_lo as u128 * y_hi as u128 + carry; + let high1 = m >> 64; + + let m_lo = m as u64; + let high2 = (x_hi as u128 * y_lo as u128 + m_lo as u128) >> 64; + + x_hi as u128 * y_hi as u128 + high1 + high2 +} diff --git a/crux-mir/lib/core/src/fmt/rt/v1.rs b/crux-mir/lib/core/src/fmt/rt/v1.rs index f6460470b..37202b277 100644 --- a/crux-mir/lib/core/src/fmt/rt/v1.rs +++ b/crux-mir/lib/core/src/fmt/rt/v1.rs @@ -33,9 +33,13 @@ pub enum Alignment { Unknown, } +/// Used by [width](https://doc.rust-lang.org/std/fmt/#width) and [precision](https://doc.rust-lang.org/std/fmt/#precision) specifiers. #[derive(Copy, Clone)] pub enum Count { + /// Specified with a literal number, stores the value Is(usize), + /// Specified using `$` and `*` syntaxes, stores the index into `args` Param(usize), + /// Not specified Implied, } diff --git a/crux-mir/lib/core/src/future/future.rs b/crux-mir/lib/core/src/future/future.rs index 00a171e6b..8c7111cb3 100644 --- a/crux-mir/lib/core/src/future/future.rs +++ b/crux-mir/lib/core/src/future/future.rs @@ -5,9 +5,9 @@ use crate::ops; use crate::pin::Pin; use crate::task::{Context, Poll}; -/// A future represents an asynchronous computation. +/// A future represents an asynchronous computation obtained by use of [`async`]. /// -/// A future is a value that may not have finished computing yet. This kind of +/// A future is a value that might not have finished computing yet. This kind of /// "asynchronous value" makes it possible for a thread to continue doing useful /// work while it waits for the value to become available. /// @@ -23,13 +23,21 @@ use crate::task::{Context, Poll}; /// When using a future, you generally won't call `poll` directly, but instead /// `.await` the value. /// -/// [`Waker`]: ../task/struct.Waker.html +/// [`async`]: ../../std/keyword.async.html +/// [`Waker`]: crate::task::Waker +#[doc(notable_trait)] #[must_use = "futures do nothing unless you `.await` or poll them"] #[stable(feature = "futures_api", since = "1.36.0")] #[lang = "future_trait"] +#[rustc_on_unimplemented( + label = "`{Self}` is not a future", + message = "`{Self}` is not a future", + note = "{Self} must be a future or must implement `IntoFuture` to be awaited" +)] pub trait Future { /// The type of value produced on completion. #[stable(feature = "futures_api", since = "1.36.0")] + #[rustc_diagnostic_item = "FutureOutput"] type Output; /// Attempt to resolve the future to a final value, registering @@ -89,11 +97,10 @@ pub trait Future { /// (memory corruption, incorrect use of `unsafe` functions, or the like), /// regardless of the future's state. /// - /// [`Poll::Pending`]: ../task/enum.Poll.html#variant.Pending - /// [`Poll::Ready(val)`]: ../task/enum.Poll.html#variant.Ready - /// [`Context`]: ../task/struct.Context.html - /// [`Waker`]: ../task/struct.Waker.html - /// [`Waker::wake`]: ../task/struct.Waker.html#method.wake + /// [`Poll::Ready(val)`]: Poll::Ready + /// [`Waker`]: crate::task::Waker + /// [`Waker::wake`]: crate::task::Waker::wake + #[lang = "poll"] #[stable(feature = "futures_api", since = "1.36.0")] fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll; } @@ -110,11 +117,11 @@ impl Future for &mut F { #[stable(feature = "futures_api", since = "1.36.0")] impl

Future for Pin

where - P: Unpin + ops::DerefMut, + P: ops::DerefMut, { type Output = <

::Target as Future>::Output; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - Pin::get_mut(self).as_mut().poll(cx) + ::poll(self.as_deref_mut(), cx) } } diff --git a/crux-mir/lib/core/src/future/into_future.rs b/crux-mir/lib/core/src/future/into_future.rs new file mode 100644 index 000000000..649b43387 --- /dev/null +++ b/crux-mir/lib/core/src/future/into_future.rs @@ -0,0 +1,139 @@ +use crate::future::Future; + +/// Conversion into a `Future`. +/// +/// By implementing `IntoFuture` for a type, you define how it will be +/// converted to a future. +/// +/// # `.await` desugaring +/// +/// The `.await` keyword desugars into a call to `IntoFuture::into_future` +/// first before polling the future to completion. `IntoFuture` is implemented +/// for all `T: Future` which means the `into_future` method will be available +/// on all futures. +/// +/// ```no_run +/// use std::future::IntoFuture; +/// +/// # async fn foo() { +/// let v = async { "meow" }; +/// let mut fut = v.into_future(); +/// assert_eq!("meow", fut.await); +/// # } +/// ``` +/// +/// # Async builders +/// +/// When implementing futures manually there will often be a choice between +/// implementing `Future` or `IntoFuture` for a type. Implementing `Future` is a +/// good choice in most cases. But implementing `IntoFuture` is most useful when +/// implementing "async builder" types, which allow their values to be modified +/// multiple times before being `.await`ed. +/// +/// ```rust +/// use std::future::{ready, Ready, IntoFuture}; +/// +/// /// Eventually multiply two numbers +/// pub struct Multiply { +/// num: u16, +/// factor: u16, +/// } +/// +/// impl Multiply { +/// /// Construct a new instance of `Multiply`. +/// pub fn new(num: u16, factor: u16) -> Self { +/// Self { num, factor } +/// } +/// +/// /// Set the number to multiply by the factor. +/// pub fn number(mut self, num: u16) -> Self { +/// self.num = num; +/// self +/// } +/// +/// /// Set the factor to multiply the number with. +/// pub fn factor(mut self, factor: u16) -> Self { +/// self.factor = factor; +/// self +/// } +/// } +/// +/// impl IntoFuture for Multiply { +/// type Output = u16; +/// type IntoFuture = Ready; +/// +/// fn into_future(self) -> Self::IntoFuture { +/// ready(self.num * self.factor) +/// } +/// } +/// +/// // NOTE: Rust does not yet have an `async fn main` function, that functionality +/// // currently only exists in the ecosystem. +/// async fn run() { +/// let num = Multiply::new(0, 0) // initialize the builder to number: 0, factor: 0 +/// .number(2) // change the number to 2 +/// .factor(2) // change the factor to 2 +/// .await; // convert to future and .await +/// +/// assert_eq!(num, 4); +/// } +/// ``` +/// +/// # Usage in trait bounds +/// +/// Using `IntoFuture` in trait bounds allows a function to be generic over both +/// `Future` and `IntoFuture`. This is convenient for users of the function, so +/// when they are using it they don't have to make an extra call to +/// `IntoFuture::into_future` to obtain an instance of `Future`: +/// +/// ```rust +/// use std::future::IntoFuture; +/// +/// /// Convert the output of a future to a string. +/// async fn fut_to_string(fut: Fut) -> String +/// where +/// Fut: IntoFuture, +/// Fut::Output: std::fmt::Debug, +/// { +/// format!("{:?}", fut.await) +/// } +/// ``` +#[stable(feature = "into_future", since = "1.64.0")] +pub trait IntoFuture { + /// The output that the future will produce on completion. + #[stable(feature = "into_future", since = "1.64.0")] + type Output; + + /// Which kind of future are we turning this into? + #[stable(feature = "into_future", since = "1.64.0")] + type IntoFuture: Future; + + /// Creates a future from a value. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ```no_run + /// use std::future::IntoFuture; + /// + /// # async fn foo() { + /// let v = async { "meow" }; + /// let mut fut = v.into_future(); + /// assert_eq!("meow", fut.await); + /// # } + /// ``` + #[stable(feature = "into_future", since = "1.64.0")] + #[lang = "into_future"] + fn into_future(self) -> Self::IntoFuture; +} + +#[stable(feature = "into_future", since = "1.64.0")] +impl IntoFuture for F { + type Output = F::Output; + type IntoFuture = F; + + fn into_future(self) -> Self::IntoFuture { + self + } +} diff --git a/crux-mir/lib/core/src/future/join.rs b/crux-mir/lib/core/src/future/join.rs new file mode 100644 index 000000000..35f0dea06 --- /dev/null +++ b/crux-mir/lib/core/src/future/join.rs @@ -0,0 +1,193 @@ +#![allow(unused_imports, unused_macros)] // items are used by the macro + +use crate::cell::UnsafeCell; +use crate::future::{poll_fn, Future}; +use crate::mem; +use crate::pin::Pin; +use crate::task::{Context, Poll}; + +/// Polls multiple futures simultaneously, returning a tuple +/// of all results once complete. +/// +/// While `join!(a, b).await` is similar to `(a.await, b.await)`, +/// `join!` polls both futures concurrently and is therefore more efficient. +/// +/// # Examples +/// +/// ``` +/// #![feature(future_join)] +/// +/// use std::future::join; +/// +/// async fn one() -> usize { 1 } +/// async fn two() -> usize { 2 } +/// +/// # let _ = async { +/// let x = join!(one(), two()).await; +/// assert_eq!(x, (1, 2)); +/// # }; +/// ``` +/// +/// `join!` is variadic, so you can pass any number of futures: +/// +/// ``` +/// #![feature(future_join)] +/// +/// use std::future::join; +/// +/// async fn one() -> usize { 1 } +/// async fn two() -> usize { 2 } +/// async fn three() -> usize { 3 } +/// +/// # let _ = async { +/// let x = join!(one(), two(), three()).await; +/// assert_eq!(x, (1, 2, 3)); +/// # }; +/// ``` +#[unstable(feature = "future_join", issue = "91642")] +pub macro join( $($fut:expr),+ $(,)? ) { + // Funnel through an internal macro not to leak implementation details. + join_internal! { + current_position: [] + futures_and_positions: [] + munching: [ $($fut)+ ] + } +} + +// FIXME(danielhenrymantilla): a private macro should need no stability guarantee. +#[unstable(feature = "future_join", issue = "91642")] +/// To be able to *name* the i-th future in the tuple (say we want the .4-th), +/// the following trick will be used: `let (_, _, _, _, it, ..) = tuple;` +/// In order to do that, we need to generate a `i`-long repetition of `_`, +/// for each i-th fut. Hence the recursive muncher approach. +macro join_internal { + // Recursion step: map each future with its "position" (underscore count). + ( + // Accumulate a token for each future that has been expanded: "_ _ _". + current_position: [ + $($underscores:tt)* + ] + // Accumulate Futures and their positions in the tuple: `_0th () _1st ( _ ) …`. + futures_and_positions: [ + $($acc:tt)* + ] + // Munch one future. + munching: [ + $current:tt + $($rest:tt)* + ] + ) => ( + join_internal! { + current_position: [ + $($underscores)* + _ + ] + futures_and_positions: [ + $($acc)* + $current ( $($underscores)* ) + ] + munching: [ + $($rest)* + ] + } + ), + + // End of recursion: generate the output future. + ( + current_position: $_:tt + futures_and_positions: [ + $( + $fut_expr:tt ( $($pos:tt)* ) + )* + ] + // Nothing left to munch. + munching: [] + ) => ( + match ( $( MaybeDone::Future($fut_expr), )* ) { futures => async { + let mut futures = futures; + // SAFETY: this is `pin_mut!`. + let mut futures = unsafe { Pin::new_unchecked(&mut futures) }; + poll_fn(move |cx| { + let mut done = true; + // For each `fut`, pin-project to it, and poll it. + $( + // SAFETY: pinning projection + let fut = unsafe { + futures.as_mut().map_unchecked_mut(|it| { + let ( $($pos,)* fut, .. ) = it; + fut + }) + }; + // Despite how tempting it may be to `let () = fut.poll(cx).ready()?;` + // doing so would defeat the point of `join!`: to start polling eagerly all + // of the futures, to allow parallelizing the waits. + done &= fut.poll(cx).is_ready(); + )* + if !done { + return Poll::Pending; + } + // All ready; time to extract all the outputs. + + // SAFETY: `.take_output()` does not break the `Pin` invariants for that `fut`. + let futures = unsafe { + futures.as_mut().get_unchecked_mut() + }; + Poll::Ready( + ($( + { + let ( $($pos,)* fut, .. ) = &mut *futures; + fut.take_output().unwrap() + } + ),*) // <- no trailing comma since we don't want 1-tuples. + ) + }).await + }} + ), +} + +/// Future used by `join!` that stores it's output to +/// be later taken and doesn't panic when polled after ready. +/// +/// This type is public in a private module for use by the macro. +#[allow(missing_debug_implementations)] +#[unstable(feature = "future_join", issue = "91642")] +pub enum MaybeDone { + Future(F), + Done(F::Output), + Taken, +} + +#[unstable(feature = "future_join", issue = "91642")] +impl MaybeDone { + pub fn take_output(&mut self) -> Option { + match *self { + MaybeDone::Done(_) => match mem::replace(self, Self::Taken) { + MaybeDone::Done(val) => Some(val), + _ => unreachable!(), + }, + _ => None, + } + } +} + +#[unstable(feature = "future_join", issue = "91642")] +impl Future for MaybeDone { + type Output = (); + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + // SAFETY: pinning in structural for `f` + unsafe { + // Do not mix match ergonomics with unsafe. + match *self.as_mut().get_unchecked_mut() { + MaybeDone::Future(ref mut f) => { + let val = Pin::new_unchecked(f).poll(cx).ready()?; + self.set(Self::Done(val)); + } + MaybeDone::Done(_) => {} + MaybeDone::Taken => unreachable!(), + } + } + + Poll::Ready(()) + } +} diff --git a/crux-mir/lib/core/src/future/mod.rs b/crux-mir/lib/core/src/future/mod.rs index 8dfda7a4a..c4fb36209 100644 --- a/crux-mir/lib/core/src/future/mod.rs +++ b/crux-mir/lib/core/src/future/mod.rs @@ -1,39 +1,59 @@ #![stable(feature = "futures_api", since = "1.36.0")] -//! Asynchronous values. +//! Asynchronous basic functionality. +//! +//! Please see the fundamental [`async`] and [`await`] keywords and the [async book] +//! for more information on asynchronous programming in Rust. +//! +//! [`async`]: ../../std/keyword.async.html +//! [`await`]: ../../std/keyword.await.html +//! [async book]: https://rust-lang.github.io/async-book/ -#[cfg(not(bootstrap))] -use crate::{ - ops::{Generator, GeneratorState}, - pin::Pin, - ptr::NonNull, - task::{Context, Poll}, -}; +use crate::ptr::NonNull; +use crate::task::Context; mod future; +mod into_future; +mod join; +mod pending; +mod poll_fn; +mod ready; + #[stable(feature = "futures_api", since = "1.36.0")] pub use self::future::Future; +#[unstable(feature = "future_join", issue = "91642")] +pub use self::join::join; + +#[stable(feature = "into_future", since = "1.64.0")] +pub use into_future::IntoFuture; + +#[stable(feature = "future_readiness_fns", since = "1.48.0")] +pub use pending::{pending, Pending}; +#[stable(feature = "future_readiness_fns", since = "1.48.0")] +pub use ready::{ready, Ready}; + +#[stable(feature = "future_poll_fn", since = "1.64.0")] +pub use poll_fn::{poll_fn, PollFn}; + /// This type is needed because: /// /// a) Generators cannot implement `for<'a, 'b> Generator<&'a mut Context<'b>>`, so we need to pass -/// a raw pointer (see https://github.com/rust-lang/rust/issues/68923). +/// a raw pointer (see ). /// b) Raw pointers and `NonNull` aren't `Send` or `Sync`, so that would make every single future /// non-Send/Sync as well, and we don't want that. /// /// It also simplifies the HIR lowering of `.await`. +#[lang = "ResumeTy"] #[doc(hidden)] #[unstable(feature = "gen_future", issue = "50547")] -#[cfg(not(bootstrap))] #[derive(Debug, Copy, Clone)] pub struct ResumeTy(NonNull>); #[unstable(feature = "gen_future", issue = "50547")] -#[cfg(not(bootstrap))] unsafe impl Send for ResumeTy {} #[unstable(feature = "gen_future", issue = "50547")] -#[cfg(not(bootstrap))] unsafe impl Sync for ResumeTy {} /// Wrap a generator in a future. @@ -43,12 +63,19 @@ unsafe impl Sync for ResumeTy {} // This is `const` to avoid extra errors after we recover from `const async fn` #[doc(hidden)] #[unstable(feature = "gen_future", issue = "50547")] -#[cfg(not(bootstrap))] +#[rustc_const_unstable(feature = "gen_future", issue = "50547")] #[inline] pub const fn from_generator(gen: T) -> impl Future where - T: Generator, + T: crate::ops::Generator, { + use crate::{ + ops::{Generator, GeneratorState}, + pin::Pin, + task::Poll, + }; + + #[rustc_diagnostic_item = "gen_future"] struct GenFuture>(T); // We rely on the fact that async/await futures are immovable in order to create @@ -57,8 +84,9 @@ where impl> Future for GenFuture { type Output = T::Return; + #[track_caller] fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - // Safety: Safe because we're !Unpin + !Drop, and this is just a field projection. + // SAFETY: Safe because we're !Unpin + !Drop, and this is just a field projection. let gen = unsafe { Pin::map_unchecked_mut(self, |s| &mut s.0) }; // Resume the generator, turning the `&mut Context` into a `NonNull` raw pointer. The @@ -73,13 +101,25 @@ where GenFuture(gen) } +#[lang = "get_context"] #[doc(hidden)] #[unstable(feature = "gen_future", issue = "50547")] -#[cfg(not(bootstrap))] +#[must_use] #[inline] -pub unsafe fn poll_with_context(f: Pin<&mut F>, mut cx: ResumeTy) -> Poll -where - F: Future, -{ - F::poll(f, cx.0.as_mut()) +pub unsafe fn get_context<'a, 'b>(cx: ResumeTy) -> &'a mut Context<'b> { + // SAFETY: the caller must guarantee that `cx.0` is a valid pointer + // that fulfills all the requirements for a mutable reference. + unsafe { &mut *cx.0.as_ptr().cast() } +} + +// FIXME(swatinem): This fn is currently needed to work around shortcomings +// in type and lifetime inference. +// See the comment at the bottom of `LoweringContext::make_async_expr` and +// . +#[doc(hidden)] +#[unstable(feature = "gen_future", issue = "50547")] +#[inline] +#[lang = "identity_future"] +pub const fn identity_future>(f: Fut) -> Fut { + f } diff --git a/crux-mir/lib/core/src/future/pending.rs b/crux-mir/lib/core/src/future/pending.rs new file mode 100644 index 000000000..2877e66ec --- /dev/null +++ b/crux-mir/lib/core/src/future/pending.rs @@ -0,0 +1,58 @@ +use crate::fmt::{self, Debug}; +use crate::future::Future; +use crate::marker; +use crate::pin::Pin; +use crate::task::{Context, Poll}; + +/// Creates a future which never resolves, representing a computation that never +/// finishes. +/// +/// This `struct` is created by [`pending()`]. See its +/// documentation for more. +#[stable(feature = "future_readiness_fns", since = "1.48.0")] +#[must_use = "futures do nothing unless you `.await` or poll them"] +pub struct Pending { + _data: marker::PhantomData T>, +} + +/// Creates a future which never resolves, representing a computation that never +/// finishes. +/// +/// # Examples +/// +/// ```no_run +/// use std::future; +/// +/// # async fn run() { +/// let future = future::pending(); +/// let () = future.await; +/// unreachable!(); +/// # } +/// ``` +#[stable(feature = "future_readiness_fns", since = "1.48.0")] +pub fn pending() -> Pending { + Pending { _data: marker::PhantomData } +} + +#[stable(feature = "future_readiness_fns", since = "1.48.0")] +impl Future for Pending { + type Output = T; + + fn poll(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll { + Poll::Pending + } +} + +#[stable(feature = "future_readiness_fns", since = "1.48.0")] +impl Debug for Pending { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Pending").finish() + } +} + +#[stable(feature = "future_readiness_fns", since = "1.48.0")] +impl Clone for Pending { + fn clone(&self) -> Self { + pending() + } +} diff --git a/crux-mir/lib/core/src/future/poll_fn.rs b/crux-mir/lib/core/src/future/poll_fn.rs new file mode 100644 index 000000000..90cb79739 --- /dev/null +++ b/crux-mir/lib/core/src/future/poll_fn.rs @@ -0,0 +1,66 @@ +use crate::fmt; +use crate::future::Future; +use crate::pin::Pin; +use crate::task::{Context, Poll}; + +/// Creates a future that wraps a function returning [`Poll`]. +/// +/// Polling the future delegates to the wrapped function. If the returned future is pinned, then the +/// captured environment of the wrapped function is also pinned in-place, so as long as the closure +/// does not move out of its captures it can soundly create pinned references to them. +/// +/// # Examples +/// +/// ``` +/// # async fn run() { +/// use core::future::poll_fn; +/// use std::task::{Context, Poll}; +/// +/// fn read_line(_cx: &mut Context<'_>) -> Poll { +/// Poll::Ready("Hello, World!".into()) +/// } +/// +/// let read_future = poll_fn(read_line); +/// assert_eq!(read_future.await, "Hello, World!".to_owned()); +/// # } +/// ``` +#[stable(feature = "future_poll_fn", since = "1.64.0")] +pub fn poll_fn(f: F) -> PollFn +where + F: FnMut(&mut Context<'_>) -> Poll, +{ + PollFn { f } +} + +/// A Future that wraps a function returning [`Poll`]. +/// +/// This `struct` is created by [`poll_fn()`]. See its +/// documentation for more. +#[must_use = "futures do nothing unless you `.await` or poll them"] +#[stable(feature = "future_poll_fn", since = "1.64.0")] +pub struct PollFn { + f: F, +} + +#[stable(feature = "future_poll_fn", since = "1.64.0")] +impl Unpin for PollFn {} + +#[stable(feature = "future_poll_fn", since = "1.64.0")] +impl fmt::Debug for PollFn { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PollFn").finish() + } +} + +#[stable(feature = "future_poll_fn", since = "1.64.0")] +impl Future for PollFn +where + F: FnMut(&mut Context<'_>) -> Poll, +{ + type Output = T; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + // SAFETY: We are not moving out of the pinned field. + (unsafe { &mut self.get_unchecked_mut().f })(cx) + } +} diff --git a/crux-mir/lib/core/src/future/ready.rs b/crux-mir/lib/core/src/future/ready.rs new file mode 100644 index 000000000..a07b63fb6 --- /dev/null +++ b/crux-mir/lib/core/src/future/ready.rs @@ -0,0 +1,70 @@ +use crate::future::Future; +use crate::pin::Pin; +use crate::task::{Context, Poll}; + +/// A future that is immediately ready with a value. +/// +/// This `struct` is created by [`ready()`]. See its +/// documentation for more. +#[stable(feature = "future_readiness_fns", since = "1.48.0")] +#[derive(Debug, Clone)] +#[must_use = "futures do nothing unless you `.await` or poll them"] +pub struct Ready(Option); + +#[stable(feature = "future_readiness_fns", since = "1.48.0")] +impl Unpin for Ready {} + +#[stable(feature = "future_readiness_fns", since = "1.48.0")] +impl Future for Ready { + type Output = T; + + #[inline] + fn poll(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll { + Poll::Ready(self.0.take().expect("`Ready` polled after completion")) + } +} + +impl Ready { + /// Consumes the `Ready`, returning the wrapped value. + /// + /// # Panics + /// + /// Will panic if this [`Ready`] was already polled to completion. + /// + /// # Examples + /// + /// ``` + /// #![feature(ready_into_inner)] + /// use std::future; + /// + /// let a = future::ready(1); + /// assert_eq!(a.into_inner(), 1); + /// ``` + #[unstable(feature = "ready_into_inner", issue = "101196")] + #[must_use] + #[inline] + pub fn into_inner(self) -> T { + self.0.expect("Called `into_inner()` on `Ready` after completion") + } +} + +/// Creates a future that is immediately ready with a value. +/// +/// Futures created through this function are functionally similar to those +/// created through `async {}`. The main difference is that futures created +/// through this function are named and implement `Unpin`. +/// +/// # Examples +/// +/// ``` +/// use std::future; +/// +/// # async fn run() { +/// let a = future::ready(1); +/// assert_eq!(a.await, 1); +/// # } +/// ``` +#[stable(feature = "future_readiness_fns", since = "1.48.0")] +pub fn ready(t: T) -> Ready { + Ready(Some(t)) +} diff --git a/crux-mir/lib/core/src/hash/mod.rs b/crux-mir/lib/core/src/hash/mod.rs index 2a7fa58dd..71a0d1825 100644 --- a/crux-mir/lib/core/src/hash/mod.rs +++ b/crux-mir/lib/core/src/hash/mod.rs @@ -1,7 +1,13 @@ //! Generic hashing support. //! -//! This module provides a generic way to compute the hash of a value. The -//! simplest way to make a type hashable is to use `#[derive(Hash)]`: +//! This module provides a generic way to compute the [hash] of a value. +//! Hashes are most commonly used with [`HashMap`] and [`HashSet`]. +//! +//! [hash]: https://en.wikipedia.org/wiki/Hash_function +//! [`HashMap`]: ../../std/collections/struct.HashMap.html +//! [`HashSet`]: ../../std/collections/struct.HashSet.html +//! +//! The simplest way to make a type hashable is to use `#[derive(Hash)]`: //! //! # Examples //! @@ -39,8 +45,6 @@ //! If you need more control over how a value is hashed, you need to implement //! the [`Hash`] trait: //! -//! [`Hash`]: trait.Hash.html -//! //! ```rust //! use std::collections::hash_map::DefaultHasher; //! use std::hash::{Hash, Hasher}; @@ -79,12 +83,11 @@ //! } //! ``` -// ignore-tidy-undocumented-unsafe - #![stable(feature = "rust1", since = "1.0.0")] use crate::fmt; -use crate::marker; +use crate::intrinsics::const_eval_select; +use crate::marker::{self, Destruct}; #[stable(feature = "rust1", since = "1.0.0")] #[allow(deprecated)] @@ -151,12 +154,37 @@ mod sip; /// Thankfully, you won't need to worry about upholding this property when /// deriving both [`Eq`] and `Hash` with `#[derive(PartialEq, Eq, Hash)]`. /// -/// [`Eq`]: ../../std/cmp/trait.Eq.html -/// [`Hasher`]: trait.Hasher.html +/// ## Prefix collisions +/// +/// Implementations of `hash` should ensure that the data they +/// pass to the `Hasher` are prefix-free. That is, +/// unequal values should cause two different sequences of values to be written, +/// and neither of the two sequences should be a prefix of the other. +/// +/// For example, the standard implementation of [`Hash` for `&str`][impl] passes an extra +/// `0xFF` byte to the `Hasher` so that the values `("ab", "c")` and `("a", +/// "bc")` hash differently. +/// +/// ## Portability +/// +/// Due to differences in endianness and type sizes, data fed by `Hash` to a `Hasher` +/// should not be considered portable across platforms. Additionally the data passed by most +/// standard library types should not be considered stable between compiler versions. +/// +/// This means tests shouldn't probe hard-coded hash values or data fed to a `Hasher` and +/// instead should check consistency with `Eq`. +/// +/// Serialization formats intended to be portable between platforms or compiler versions should +/// either avoid encoding hashes or only rely on `Hash` and `Hasher` implementations that +/// provide additional guarantees. +/// /// [`HashMap`]: ../../std/collections/struct.HashMap.html /// [`HashSet`]: ../../std/collections/struct.HashSet.html -/// [`hash`]: #tymethod.hash +/// [`hash`]: Hash::hash +/// [impl]: ../../std/primitive.str.html#impl-Hash-for-str #[stable(feature = "rust1", since = "1.0.0")] +#[rustc_diagnostic_item = "Hash"] +#[const_trait] pub trait Hash { /// Feeds this value into the given [`Hasher`]. /// @@ -170,13 +198,26 @@ pub trait Hash { /// 7920.hash(&mut hasher); /// println!("Hash is {:x}!", hasher.finish()); /// ``` - /// - /// [`Hasher`]: trait.Hasher.html #[stable(feature = "rust1", since = "1.0.0")] - fn hash(&self, state: &mut H); + fn hash(&self, state: &mut H); /// Feeds a slice of this type into the given [`Hasher`]. /// + /// This method is meant as a convenience, but its implementation is + /// also explicitly left unspecified. It isn't guaranteed to be + /// equivalent to repeated calls of [`hash`] and implementations of + /// [`Hash`] should keep that in mind and call [`hash`] themselves + /// if the slice isn't treated as a whole unit in the [`PartialEq`] + /// implementation. + /// + /// For example, a [`VecDeque`] implementation might naïvely call + /// [`as_slices`] and then [`hash_slice`] on each slice, but this + /// is wrong since the two slices can change with a call to + /// [`make_contiguous`] without affecting the [`PartialEq`] + /// result. Since these slices aren't treated as singular + /// units, and instead part of a larger deque, this method cannot + /// be used. + /// /// # Examples /// /// ``` @@ -189,15 +230,31 @@ pub trait Hash { /// println!("Hash is {:x}!", hasher.finish()); /// ``` /// - /// [`Hasher`]: trait.Hasher.html + /// [`VecDeque`]: ../../std/collections/struct.VecDeque.html + /// [`as_slices`]: ../../std/collections/struct.VecDeque.html#method.as_slices + /// [`make_contiguous`]: ../../std/collections/struct.VecDeque.html#method.make_contiguous + /// [`hash`]: Hash::hash + /// [`hash_slice`]: Hash::hash_slice #[stable(feature = "hash_slice", since = "1.3.0")] - fn hash_slice(data: &[Self], state: &mut H) + fn hash_slice(data: &[Self], state: &mut H) where Self: Sized, { - for piece in data { - piece.hash(state); + //FIXME(const_trait_impl): revert to only a for loop + fn rt(data: &[T], state: &mut H) { + for piece in data { + piece.hash(state) + } } + const fn ct(data: &[T], state: &mut H) { + let mut i = 0; + while i < data.len() { + data[i].hash(state); + i += 1; + } + } + // SAFETY: same behavior, CT just uses while instead of for + unsafe { const_eval_select((data, state), ct, rt) }; } } @@ -225,6 +282,30 @@ pub use macros::Hash; /// instance (with [`write`] and [`write_u8`] etc.). Most of the time, `Hasher` /// instances are used in conjunction with the [`Hash`] trait. /// +/// This trait provides no guarantees about how the various `write_*` methods are +/// defined and implementations of [`Hash`] should not assume that they work one +/// way or another. You cannot assume, for example, that a [`write_u32`] call is +/// equivalent to four calls of [`write_u8`]. Nor can you assume that adjacent +/// `write` calls are merged, so it's possible, for example, that +/// ``` +/// # fn foo(hasher: &mut impl std::hash::Hasher) { +/// hasher.write(&[1, 2]); +/// hasher.write(&[3, 4, 5, 6]); +/// # } +/// ``` +/// and +/// ``` +/// # fn foo(hasher: &mut impl std::hash::Hasher) { +/// hasher.write(&[1, 2, 3, 4]); +/// hasher.write(&[5, 6]); +/// # } +/// ``` +/// end up producing different hashes. +/// +/// Thus to produce the same hash value, [`Hash`] implementations must ensure +/// for equivalent items that exactly the same sequence of calls is made -- the +/// same methods with the same parameters in the same order. +/// /// # Examples /// /// ``` @@ -241,11 +322,12 @@ pub use macros::Hash; /// println!("Hash is {:x}!", hasher.finish()); /// ``` /// -/// [`Hash`]: trait.Hash.html -/// [`finish`]: #tymethod.finish -/// [`write`]: #tymethod.write -/// [`write_u8`]: #method.write_u8 +/// [`finish`]: Hasher::finish +/// [`write`]: Hasher::write +/// [`write_u8`]: Hasher::write_u8 +/// [`write_u32`]: Hasher::write_u32 #[stable(feature = "rust1", since = "1.0.0")] +#[const_trait] pub trait Hasher { /// Returns the hash value for the values written so far. /// @@ -266,7 +348,7 @@ pub trait Hasher { /// println!("Hash is {:x}!", hasher.finish()); /// ``` /// - /// [`write`]: #tymethod.write + /// [`write`]: Hasher::write #[stable(feature = "rust1", since = "1.0.0")] fn finish(&self) -> u64; @@ -285,6 +367,12 @@ pub trait Hasher { /// /// println!("Hash is {:x}!", hasher.finish()); /// ``` + /// + /// # Note to Implementers + /// + /// You generally should not do length-prefixing as part of implementing + /// this method. It's up to the [`Hash`] implementation to call + /// [`Hasher::write_length_prefix`] before sequences that need it. #[stable(feature = "rust1", since = "1.0.0")] fn write(&mut self, bytes: &[u8]); @@ -335,36 +423,158 @@ pub trait Hasher { #[inline] #[stable(feature = "hasher_write", since = "1.3.0")] fn write_i16(&mut self, i: i16) { - self.write(&i.to_ne_bytes()) + self.write_u16(i as u16) } /// Writes a single `i32` into this hasher. #[inline] #[stable(feature = "hasher_write", since = "1.3.0")] fn write_i32(&mut self, i: i32) { - self.write(&i.to_ne_bytes()) + self.write_u32(i as u32) } /// Writes a single `i64` into this hasher. #[inline] #[stable(feature = "hasher_write", since = "1.3.0")] fn write_i64(&mut self, i: i64) { - self.write(&i.to_ne_bytes()) + self.write_u64(i as u64) } /// Writes a single `i128` into this hasher. #[inline] #[stable(feature = "i128", since = "1.26.0")] fn write_i128(&mut self, i: i128) { - self.write(&i.to_ne_bytes()) + self.write_u128(i as u128) } /// Writes a single `isize` into this hasher. #[inline] #[stable(feature = "hasher_write", since = "1.3.0")] fn write_isize(&mut self, i: isize) { - self.write(&i.to_ne_bytes()) + self.write_usize(i as usize) + } + + /// Writes a length prefix into this hasher, as part of being prefix-free. + /// + /// If you're implementing [`Hash`] for a custom collection, call this before + /// writing its contents to this `Hasher`. That way + /// `(collection![1, 2, 3], collection![4, 5])` and + /// `(collection![1, 2], collection![3, 4, 5])` will provide different + /// sequences of values to the `Hasher` + /// + /// The `impl Hash for [T]` includes a call to this method, so if you're + /// hashing a slice (or array or vector) via its `Hash::hash` method, + /// you should **not** call this yourself. + /// + /// This method is only for providing domain separation. If you want to + /// hash a `usize` that represents part of the *data*, then it's important + /// that you pass it to [`Hasher::write_usize`] instead of to this method. + /// + /// # Examples + /// + /// ``` + /// #![feature(hasher_prefixfree_extras)] + /// # // Stubs to make the `impl` below pass the compiler + /// # struct MyCollection(Option); + /// # impl MyCollection { + /// # fn len(&self) -> usize { todo!() } + /// # } + /// # impl<'a, T> IntoIterator for &'a MyCollection { + /// # type Item = T; + /// # type IntoIter = std::iter::Empty; + /// # fn into_iter(self) -> Self::IntoIter { todo!() } + /// # } + /// + /// use std::hash::{Hash, Hasher}; + /// impl Hash for MyCollection { + /// fn hash(&self, state: &mut H) { + /// state.write_length_prefix(self.len()); + /// for elt in self { + /// elt.hash(state); + /// } + /// } + /// } + /// ``` + /// + /// # Note to Implementers + /// + /// If you've decided that your `Hasher` is willing to be susceptible to + /// Hash-DoS attacks, then you might consider skipping hashing some or all + /// of the `len` provided in the name of increased performance. + #[inline] + #[unstable(feature = "hasher_prefixfree_extras", issue = "96762")] + fn write_length_prefix(&mut self, len: usize) { + self.write_usize(len); + } + + /// Writes a single `str` into this hasher. + /// + /// If you're implementing [`Hash`], you generally do not need to call this, + /// as the `impl Hash for str` does, so you should prefer that instead. + /// + /// This includes the domain separator for prefix-freedom, so you should + /// **not** call `Self::write_length_prefix` before calling this. + /// + /// # Note to Implementers + /// + /// There are at least two reasonable default ways to implement this. + /// Which one will be the default is not yet decided, so for now + /// you probably want to override it specifically. + /// + /// ## The general answer + /// + /// It's always correct to implement this with a length prefix: + /// + /// ``` + /// # #![feature(hasher_prefixfree_extras)] + /// # struct Foo; + /// # impl std::hash::Hasher for Foo { + /// # fn finish(&self) -> u64 { unimplemented!() } + /// # fn write(&mut self, _bytes: &[u8]) { unimplemented!() } + /// fn write_str(&mut self, s: &str) { + /// self.write_length_prefix(s.len()); + /// self.write(s.as_bytes()); + /// } + /// # } + /// ``` + /// + /// And, if your `Hasher` works in `usize` chunks, this is likely a very + /// efficient way to do it, as anything more complicated may well end up + /// slower than just running the round with the length. + /// + /// ## If your `Hasher` works byte-wise + /// + /// One nice thing about `str` being UTF-8 is that the `b'\xFF'` byte + /// never happens. That means that you can append that to the byte stream + /// being hashed and maintain prefix-freedom: + /// + /// ``` + /// # #![feature(hasher_prefixfree_extras)] + /// # struct Foo; + /// # impl std::hash::Hasher for Foo { + /// # fn finish(&self) -> u64 { unimplemented!() } + /// # fn write(&mut self, _bytes: &[u8]) { unimplemented!() } + /// fn write_str(&mut self, s: &str) { + /// self.write(s.as_bytes()); + /// self.write_u8(0xff); + /// } + /// # } + /// ``` + /// + /// This does require that your implementation not add extra padding, and + /// thus generally requires that you maintain a buffer, running a round + /// only once that buffer is full (or `finish` is called). + /// + /// That's because if `write` pads data out to a fixed chunk size, it's + /// likely that it does it in such a way that `"a"` and `"a\x00"` would + /// end up hashing the same sequence of things, introducing conflicts. + #[inline] + #[unstable(feature = "hasher_prefixfree_extras", issue = "96762")] + fn write_str(&mut self, s: &str) { + self.write(s.as_bytes()); + self.write_u8(0xff); } } #[stable(feature = "indirect_hasher_impl", since = "1.22.0")] -impl Hasher for &mut H { +#[rustc_const_unstable(feature = "const_hash", issue = "104061")] +impl const Hasher for &mut H { fn finish(&self) -> u64 { (**self).finish() } @@ -407,6 +617,12 @@ impl Hasher for &mut H { fn write_isize(&mut self, i: isize) { (**self).write_isize(i) } + fn write_length_prefix(&mut self, len: usize) { + (**self).write_length_prefix(len) + } + fn write_str(&mut self, s: &str) { + (**self).write_str(s) + } } /// A trait for creating instances of [`Hasher`]. @@ -435,10 +651,10 @@ impl Hasher for &mut H { /// assert_eq!(hasher_1.finish(), hasher_2.finish()); /// ``` /// -/// [`build_hasher`]: #tymethod.build_hasher -/// [`Hasher`]: trait.Hasher.html +/// [`build_hasher`]: BuildHasher::build_hasher /// [`HashMap`]: ../../std/collections/struct.HashMap.html #[stable(since = "1.7.0", feature = "build_hasher")] +#[const_trait] pub trait BuildHasher { /// Type of the hasher that will be created. #[stable(since = "1.7.0", feature = "build_hasher")] @@ -458,10 +674,56 @@ pub trait BuildHasher { /// let s = RandomState::new(); /// let new_s = s.build_hasher(); /// ``` - /// - /// [`Hasher`]: trait.Hasher.html #[stable(since = "1.7.0", feature = "build_hasher")] fn build_hasher(&self) -> Self::Hasher; + + /// Calculates the hash of a single value. + /// + /// This is intended as a convenience for code which *consumes* hashes, such + /// as the implementation of a hash table or in unit tests that check + /// whether a custom [`Hash`] implementation behaves as expected. + /// + /// This must not be used in any code which *creates* hashes, such as in an + /// implementation of [`Hash`]. The way to create a combined hash of + /// multiple values is to call [`Hash::hash`] multiple times using the same + /// [`Hasher`], not to call this method repeatedly and combine the results. + /// + /// # Example + /// + /// ``` + /// #![feature(build_hasher_simple_hash_one)] + /// + /// use std::cmp::{max, min}; + /// use std::hash::{BuildHasher, Hash, Hasher}; + /// struct OrderAmbivalentPair(T, T); + /// impl Hash for OrderAmbivalentPair { + /// fn hash(&self, hasher: &mut H) { + /// min(&self.0, &self.1).hash(hasher); + /// max(&self.0, &self.1).hash(hasher); + /// } + /// } + /// + /// // Then later, in a `#[test]` for the type... + /// let bh = std::collections::hash_map::RandomState::new(); + /// assert_eq!( + /// bh.hash_one(OrderAmbivalentPair(1, 2)), + /// bh.hash_one(OrderAmbivalentPair(2, 1)) + /// ); + /// assert_eq!( + /// bh.hash_one(OrderAmbivalentPair(10, 2)), + /// bh.hash_one(&OrderAmbivalentPair(2, 10)) + /// ); + /// ``` + #[unstable(feature = "build_hasher_simple_hash_one", issue = "86161")] + fn hash_one(&self, x: T) -> u64 + where + Self: Sized, + Self::Hasher: ~const Hasher + ~const Destruct, + { + let mut hasher = self.build_hasher(); + x.hash(&mut hasher); + hasher.finish() + } } /// Used to create a default [`BuildHasher`] instance for types that implement @@ -472,7 +734,7 @@ pub trait BuildHasher { /// defined. /// /// Any `BuildHasherDefault` is [zero-sized]. It can be created with -/// [`default`][method.Default]. When using `BuildHasherDefault` with [`HashMap`] or +/// [`default`][method.default]. When using `BuildHasherDefault` with [`HashMap`] or /// [`HashSet`], this doesn't need to be done, since they implement appropriate /// [`Default`] instances themselves. /// @@ -505,25 +767,23 @@ pub trait BuildHasher { /// let hash_map = HashMap::::default(); /// ``` /// -/// [`BuildHasher`]: trait.BuildHasher.html -/// [`Default`]: ../default/trait.Default.html -/// [method.default]: #method.default -/// [`Hasher`]: trait.Hasher.html +/// [method.default]: BuildHasherDefault::default /// [`HashMap`]: ../../std/collections/struct.HashMap.html /// [`HashSet`]: ../../std/collections/struct.HashSet.html /// [zero-sized]: https://doc.rust-lang.org/nomicon/exotic-sizes.html#zero-sized-types-zsts #[stable(since = "1.7.0", feature = "build_hasher")] -pub struct BuildHasherDefault(marker::PhantomData); +pub struct BuildHasherDefault(marker::PhantomData H>); #[stable(since = "1.9.0", feature = "core_impl_debug")] impl fmt::Debug for BuildHasherDefault { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("BuildHasherDefault") + f.debug_struct("BuildHasherDefault").finish() } } #[stable(since = "1.7.0", feature = "build_hasher")] -impl BuildHasher for BuildHasherDefault { +#[rustc_const_unstable(feature = "const_hash", issue = "104061")] +impl const BuildHasher for BuildHasherDefault { type Hasher = H; fn build_hasher(&self) -> H { @@ -539,7 +799,8 @@ impl Clone for BuildHasherDefault { } #[stable(since = "1.7.0", feature = "build_hasher")] -impl Default for BuildHasherDefault { +#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] +impl const Default for BuildHasherDefault { fn default() -> BuildHasherDefault { BuildHasherDefault(marker::PhantomData) } @@ -564,14 +825,21 @@ mod impls { macro_rules! impl_write { ($(($ty:ident, $meth:ident),)*) => {$( #[stable(feature = "rust1", since = "1.0.0")] - impl Hash for $ty { - fn hash(&self, state: &mut H) { + #[rustc_const_unstable(feature = "const_hash", issue = "104061")] + impl const Hash for $ty { + #[inline] + fn hash(&self, state: &mut H) { state.$meth(*self) } - fn hash_slice(data: &[$ty], state: &mut H) { + #[inline] + fn hash_slice(data: &[$ty], state: &mut H) { let newlen = data.len() * mem::size_of::<$ty>(); let ptr = data.as_ptr() as *const u8; + // SAFETY: `ptr` is valid and aligned, as this macro is only used + // for numeric primitives which have no padding. The new slice only + // spans across `data` and is never mutated, and its total size is the + // same as the original `data` so it can't be over `isize::MAX`. state.write(unsafe { slice::from_raw_parts(ptr, newlen) }) } } @@ -594,30 +862,37 @@ mod impls { } #[stable(feature = "rust1", since = "1.0.0")] - impl Hash for bool { - fn hash(&self, state: &mut H) { + #[rustc_const_unstable(feature = "const_hash", issue = "104061")] + impl const Hash for bool { + #[inline] + fn hash(&self, state: &mut H) { state.write_u8(*self as u8) } } #[stable(feature = "rust1", since = "1.0.0")] - impl Hash for char { - fn hash(&self, state: &mut H) { + #[rustc_const_unstable(feature = "const_hash", issue = "104061")] + impl const Hash for char { + #[inline] + fn hash(&self, state: &mut H) { state.write_u32(*self as u32) } } #[stable(feature = "rust1", since = "1.0.0")] - impl Hash for str { - fn hash(&self, state: &mut H) { - state.write(self.as_bytes()); - state.write_u8(0xff) + #[rustc_const_unstable(feature = "const_hash", issue = "104061")] + impl const Hash for str { + #[inline] + fn hash(&self, state: &mut H) { + state.write_str(self); } } #[stable(feature = "never_hash", since = "1.29.0")] - impl Hash for ! { - fn hash(&self, _: &mut H) { + #[rustc_const_unstable(feature = "const_hash", issue = "104061")] + impl const Hash for ! { + #[inline] + fn hash(&self, _: &mut H) { *self } } @@ -625,91 +900,108 @@ mod impls { macro_rules! impl_hash_tuple { () => ( #[stable(feature = "rust1", since = "1.0.0")] - impl Hash for () { - fn hash(&self, _state: &mut H) {} + #[rustc_const_unstable(feature = "const_hash", issue = "104061")] + impl const Hash for () { + #[inline] + fn hash(&self, _state: &mut H) {} } ); ( $($name:ident)+) => ( - #[stable(feature = "rust1", since = "1.0.0")] - impl<$($name: Hash),+> Hash for ($($name,)+) where last_type!($($name,)+): ?Sized { - #[allow(non_snake_case)] - fn hash(&self, state: &mut S) { - let ($(ref $name,)+) = *self; - $($name.hash(state);)+ + maybe_tuple_doc! { + $($name)+ @ + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_hash", issue = "104061")] + impl<$($name: ~const Hash),+> const Hash for ($($name,)+) where last_type!($($name,)+): ?Sized { + #[allow(non_snake_case)] + #[inline] + fn hash(&self, state: &mut S) { + let ($(ref $name,)+) = *self; + $($name.hash(state);)+ + } } } ); } + macro_rules! maybe_tuple_doc { + ($a:ident @ #[$meta:meta] $item:item) => { + #[doc(fake_variadic)] + #[doc = "This trait is implemented for tuples up to twelve items long."] + #[$meta] + $item + }; + ($a:ident $($rest_a:ident)+ @ #[$meta:meta] $item:item) => { + #[doc(hidden)] + #[$meta] + $item + }; + } + macro_rules! last_type { ($a:ident,) => { $a }; ($a:ident, $($rest_a:ident,)+) => { last_type!($($rest_a,)+) }; } impl_hash_tuple! {} - impl_hash_tuple! { A } - impl_hash_tuple! { A B } - impl_hash_tuple! { A B C } - impl_hash_tuple! { A B C D } - impl_hash_tuple! { A B C D E } - impl_hash_tuple! { A B C D E F } - impl_hash_tuple! { A B C D E F G } - impl_hash_tuple! { A B C D E F G H } - impl_hash_tuple! { A B C D E F G H I } - impl_hash_tuple! { A B C D E F G H I J } - impl_hash_tuple! { A B C D E F G H I J K } - impl_hash_tuple! { A B C D E F G H I J K L } + impl_hash_tuple! { T } + impl_hash_tuple! { T B } + impl_hash_tuple! { T B C } + impl_hash_tuple! { T B C D } + impl_hash_tuple! { T B C D E } + impl_hash_tuple! { T B C D E F } + impl_hash_tuple! { T B C D E F G } + impl_hash_tuple! { T B C D E F G H } + impl_hash_tuple! { T B C D E F G H I } + impl_hash_tuple! { T B C D E F G H I J } + impl_hash_tuple! { T B C D E F G H I J K } + impl_hash_tuple! { T B C D E F G H I J K L } #[stable(feature = "rust1", since = "1.0.0")] - impl Hash for [T] { - fn hash(&self, state: &mut H) { - self.len().hash(state); + #[rustc_const_unstable(feature = "const_hash", issue = "104061")] + impl const Hash for [T] { + #[inline] + fn hash(&self, state: &mut H) { + state.write_length_prefix(self.len()); Hash::hash_slice(self, state) } } #[stable(feature = "rust1", since = "1.0.0")] - impl Hash for &T { - fn hash(&self, state: &mut H) { + #[rustc_const_unstable(feature = "const_hash", issue = "104061")] + impl const Hash for &T { + #[inline] + fn hash(&self, state: &mut H) { (**self).hash(state); } } #[stable(feature = "rust1", since = "1.0.0")] - impl Hash for &mut T { - fn hash(&self, state: &mut H) { + #[rustc_const_unstable(feature = "const_hash", issue = "104061")] + impl const Hash for &mut T { + #[inline] + fn hash(&self, state: &mut H) { (**self).hash(state); } } #[stable(feature = "rust1", since = "1.0.0")] impl Hash for *const T { + #[inline] fn hash(&self, state: &mut H) { - if mem::size_of::() == mem::size_of::() { - // Thin pointer - state.write_usize(*self as *const () as usize); - } else { - // Fat pointer - let (a, b) = unsafe { *(self as *const Self as *const (usize, usize)) }; - state.write_usize(a); - state.write_usize(b); - } + let (address, metadata) = self.to_raw_parts(); + state.write_usize(address.addr()); + metadata.hash(state); } } #[stable(feature = "rust1", since = "1.0.0")] impl Hash for *mut T { + #[inline] fn hash(&self, state: &mut H) { - if mem::size_of::() == mem::size_of::() { - // Thin pointer - state.write_usize(*self as *const () as usize); - } else { - // Fat pointer - let (a, b) = unsafe { *(self as *const Self as *const (usize, usize)) }; - state.write_usize(a); - state.write_usize(b); - } + let (address, metadata) = self.to_raw_parts(); + state.write_usize(address.addr()); + metadata.hash(state); } } } diff --git a/crux-mir/lib/core/src/hash/sip.rs b/crux-mir/lib/core/src/hash/sip.rs index adfbe2435..7f8287bf5 100644 --- a/crux-mir/lib/core/src/hash/sip.rs +++ b/crux-mir/lib/core/src/hash/sip.rs @@ -1,7 +1,5 @@ //! An implementation of SipHash. -// ignore-tidy-undocumented-unsafe - #![allow(deprecated)] // the types in this module are deprecated use crate::cmp; @@ -16,10 +14,7 @@ use crate::ptr; /// /// See: #[unstable(feature = "hashmap_internals", issue = "none")] -#[rustc_deprecated( - since = "1.13.0", - reason = "use `std::collections::hash_map::DefaultHasher` instead" -)] +#[deprecated(since = "1.13.0", note = "use `std::collections::hash_map::DefaultHasher` instead")] #[derive(Debug, Clone, Default)] #[doc(hidden)] pub struct SipHasher13 { @@ -30,10 +25,7 @@ pub struct SipHasher13 { /// /// See: #[unstable(feature = "hashmap_internals", issue = "none")] -#[rustc_deprecated( - since = "1.13.0", - reason = "use `std::collections::hash_map::DefaultHasher` instead" -)] +#[deprecated(since = "1.13.0", note = "use `std::collections::hash_map::DefaultHasher` instead")] #[derive(Debug, Clone, Default)] struct SipHasher24 { hasher: Hasher, @@ -45,17 +37,14 @@ struct SipHasher24 { /// /// SipHash is a general-purpose hashing function: it runs at a good /// speed (competitive with Spooky and City) and permits strong _keyed_ -/// hashing. This lets you key your hashtables from a strong RNG, such as -/// [`rand::os::OsRng`](https://doc.rust-lang.org/rand/rand/os/struct.OsRng.html). +/// hashing. This lets you key your hash tables from a strong RNG, such as +/// [`rand::os::OsRng`](https://docs.rs/rand/latest/rand/rngs/struct.OsRng.html). /// /// Although the SipHash algorithm is considered to be generally strong, /// it is not intended for cryptographic purposes. As such, all /// cryptographic uses of this implementation are _strongly discouraged_. #[stable(feature = "rust1", since = "1.0.0")] -#[rustc_deprecated( - since = "1.13.0", - reason = "use `std::collections::hash_map::DefaultHasher` instead" -)] +#[deprecated(since = "1.13.0", note = "use `std::collections::hash_map::DefaultHasher` instead")] #[derive(Debug, Clone, Default)] pub struct SipHasher(SipHasher24); @@ -107,13 +96,14 @@ macro_rules! compress { /// `copy_nonoverlapping` to let the compiler generate the most efficient way /// to load it from a possibly unaligned address. /// -/// Unsafe because: unchecked indexing at i..i+size_of(int_ty) +/// Safety: this performs unchecked indexing of `$buf` at +/// `$i..$i+size_of::<$int_ty>()`, so that must be in-bounds. macro_rules! load_int_le { ($buf:expr, $i:expr, $int_ty:ident) => {{ debug_assert!($i + mem::size_of::<$int_ty>() <= $buf.len()); let mut data = 0 as $int_ty; ptr::copy_nonoverlapping( - $buf.get_unchecked($i), + $buf.as_ptr().add($i), &mut data as *mut _ as *mut u8, mem::size_of::<$int_ty>(), ); @@ -125,25 +115,31 @@ macro_rules! load_int_le { /// `copy_nonoverlapping` calls that occur (via `load_int_le!`) all have fixed /// sizes and avoid calling `memcpy`, which is good for speed. /// -/// Unsafe because: unchecked indexing at start..start+len +/// Safety: this performs unchecked indexing of `buf` at `start..start+len`, so +/// that must be in-bounds. #[inline] -unsafe fn u8to64_le(buf: &[u8], start: usize, len: usize) -> u64 { +const unsafe fn u8to64_le(buf: &[u8], start: usize, len: usize) -> u64 { debug_assert!(len < 8); let mut i = 0; // current byte index (from LSB) in the output u64 let mut out = 0; if i + 3 < len { - out = load_int_le!(buf, start + i, u32) as u64; + // SAFETY: `i` cannot be greater than `len`, and the caller must guarantee + // that the index start..start+len is in bounds. + out = unsafe { load_int_le!(buf, start + i, u32) } as u64; i += 4; } if i + 1 < len { - out |= (load_int_le!(buf, start + i, u16) as u64) << (i * 8); + // SAFETY: same as above. + out |= (unsafe { load_int_le!(buf, start + i, u16) } as u64) << (i * 8); i += 2 } if i < len { - out |= (*buf.get_unchecked(start + i) as u64) << (i * 8); + // SAFETY: same as above. + out |= (unsafe { *buf.get_unchecked(start + i) } as u64) << (i * 8); i += 1; } - debug_assert_eq!(i, len); + //FIXME(fee1-dead): use debug_assert_eq + debug_assert!(i == len); out } @@ -151,22 +147,26 @@ impl SipHasher { /// Creates a new `SipHasher` with the two initial keys set to 0. #[inline] #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_deprecated( + #[deprecated( since = "1.13.0", - reason = "use `std::collections::hash_map::DefaultHasher` instead" + note = "use `std::collections::hash_map::DefaultHasher` instead" )] - pub fn new() -> SipHasher { + #[rustc_const_unstable(feature = "const_hash", issue = "104061")] + #[must_use] + pub const fn new() -> SipHasher { SipHasher::new_with_keys(0, 0) } /// Creates a `SipHasher` that is keyed off the provided keys. #[inline] #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_deprecated( + #[deprecated( since = "1.13.0", - reason = "use `std::collections::hash_map::DefaultHasher` instead" + note = "use `std::collections::hash_map::DefaultHasher` instead" )] - pub fn new_with_keys(key0: u64, key1: u64) -> SipHasher { + #[rustc_const_unstable(feature = "const_hash", issue = "104061")] + #[must_use] + pub const fn new_with_keys(key0: u64, key1: u64) -> SipHasher { SipHasher(SipHasher24 { hasher: Hasher::new_with_keys(key0, key1) }) } } @@ -175,29 +175,31 @@ impl SipHasher13 { /// Creates a new `SipHasher13` with the two initial keys set to 0. #[inline] #[unstable(feature = "hashmap_internals", issue = "none")] - #[rustc_deprecated( + #[deprecated( since = "1.13.0", - reason = "use `std::collections::hash_map::DefaultHasher` instead" + note = "use `std::collections::hash_map::DefaultHasher` instead" )] - pub fn new() -> SipHasher13 { + #[rustc_const_unstable(feature = "const_hash", issue = "104061")] + pub const fn new() -> SipHasher13 { SipHasher13::new_with_keys(0, 0) } /// Creates a `SipHasher13` that is keyed off the provided keys. #[inline] #[unstable(feature = "hashmap_internals", issue = "none")] - #[rustc_deprecated( + #[deprecated( since = "1.13.0", - reason = "use `std::collections::hash_map::DefaultHasher` instead" + note = "use `std::collections::hash_map::DefaultHasher` instead" )] - pub fn new_with_keys(key0: u64, key1: u64) -> SipHasher13 { + #[rustc_const_unstable(feature = "const_hash", issue = "104061")] + pub const fn new_with_keys(key0: u64, key1: u64) -> SipHasher13 { SipHasher13 { hasher: Hasher::new_with_keys(key0, key1) } } } impl Hasher { #[inline] - fn new_with_keys(key0: u64, key1: u64) -> Hasher { + const fn new_with_keys(key0: u64, key1: u64) -> Hasher { let mut state = Hasher { k0: key0, k1: key1, @@ -212,7 +214,7 @@ impl Hasher { } #[inline] - fn reset(&mut self) { + const fn reset(&mut self) { self.length = 0; self.state.v0 = self.k0 ^ 0x736f6d6570736575; self.state.v1 = self.k1 ^ 0x646f72616e646f6d; @@ -223,12 +225,18 @@ impl Hasher { } #[stable(feature = "rust1", since = "1.0.0")] -impl super::Hasher for SipHasher { +#[rustc_const_unstable(feature = "const_hash", issue = "104061")] +impl const super::Hasher for SipHasher { #[inline] fn write(&mut self, msg: &[u8]) { self.0.hasher.write(msg) } + #[inline] + fn write_str(&mut self, s: &str) { + self.0.hasher.write_str(s); + } + #[inline] fn finish(&self) -> u64 { self.0.hasher.finish() @@ -236,19 +244,25 @@ impl super::Hasher for SipHasher { } #[unstable(feature = "hashmap_internals", issue = "none")] -impl super::Hasher for SipHasher13 { +#[rustc_const_unstable(feature = "const_hash", issue = "104061")] +impl const super::Hasher for SipHasher13 { #[inline] fn write(&mut self, msg: &[u8]) { self.hasher.write(msg) } + #[inline] + fn write_str(&mut self, s: &str) { + self.hasher.write_str(s); + } + #[inline] fn finish(&self) -> u64 { self.hasher.finish() } } -impl super::Hasher for Hasher { +impl const super::Hasher for Hasher { // Note: no integer hashing methods (`write_u*`, `write_i*`) are defined // for this type. We could add them, copy the `short_write` implementation // in librustc_data_structures/sip128.rs, and add `write_u*`/`write_i*` @@ -265,6 +279,7 @@ impl super::Hasher for Hasher { if self.ntail != 0 { needed = 8 - self.ntail; + // SAFETY: `cmp::min(length, needed)` is guaranteed to not be over `length` self.tail |= unsafe { u8to64_le(msg, 0, cmp::min(length, needed)) } << (8 * self.ntail); if length < needed { self.ntail += length; @@ -279,10 +294,13 @@ impl super::Hasher for Hasher { // Buffered tail is now flushed, process new input. let len = length - needed; - let left = len & 0x7; + let left = len & 0x7; // len % 8 let mut i = needed; while i < len - left { + // SAFETY: because `len - left` is the biggest multiple of 8 under + // `len`, and because `i` starts at `needed` where `len` is `length - needed`, + // `i + 8` is guaranteed to be less than or equal to `length`. let mi = unsafe { load_int_le!(msg, i, u64) }; self.state.v3 ^= mi; @@ -292,10 +310,21 @@ impl super::Hasher for Hasher { i += 8; } + // SAFETY: `i` is now `needed + len.div_euclid(8) * 8`, + // so `i + left` = `needed + len` = `length`, which is by + // definition equal to `msg.len()`. self.tail = unsafe { u8to64_le(msg, i, left) }; self.ntail = left; } + #[inline] + fn write_str(&mut self, s: &str) { + // This hasher works byte-wise, and `0xFF` cannot show up in a `str`, + // so just hashing the one extra byte is enough to be prefix-free. + self.write(s.as_bytes()); + self.write_u8(0xFF); + } + #[inline] fn finish(&self) -> u64 { let mut state = self.state; @@ -313,7 +342,7 @@ impl super::Hasher for Hasher { } } -impl Clone for Hasher { +impl const Clone for Hasher { #[inline] fn clone(&self) -> Hasher { Hasher { @@ -337,6 +366,7 @@ impl Default for Hasher { } #[doc(hidden)] +#[const_trait] trait Sip { fn c_rounds(_: &mut State); fn d_rounds(_: &mut State); @@ -345,7 +375,7 @@ trait Sip { #[derive(Debug, Clone, Default)] struct Sip13Rounds; -impl Sip for Sip13Rounds { +impl const Sip for Sip13Rounds { #[inline] fn c_rounds(state: &mut State) { compress!(state); @@ -362,7 +392,7 @@ impl Sip for Sip13Rounds { #[derive(Debug, Clone, Default)] struct Sip24Rounds; -impl Sip for Sip24Rounds { +impl const Sip for Sip24Rounds { #[inline] fn c_rounds(state: &mut State) { compress!(state); diff --git a/crux-mir/lib/core/src/hint.rs b/crux-mir/lib/core/src/hint.rs index 6cbd26a78..5a76e8669 100644 --- a/crux-mir/lib/core/src/hint.rs +++ b/crux-mir/lib/core/src/hint.rs @@ -1,34 +1,82 @@ #![stable(feature = "core_hint", since = "1.27.0")] //! Hints to compiler that affects how code should be emitted or optimized. - -// ignore-tidy-undocumented-unsafe +//! Hints may be compile time or runtime. use crate::intrinsics; -/// Informs the compiler that this point in the code is not reachable, enabling -/// further optimizations. +/// Informs the compiler that the site which is calling this function is not +/// reachable, possibly enabling further optimizations. /// /// # Safety /// -/// Reaching this function is completely *undefined behavior* (UB). In -/// particular, the compiler assumes that all UB must never happen, and -/// therefore will eliminate all branches that reach to a call to -/// `unreachable_unchecked()`. +/// Reaching this function is *Undefined Behavior*. /// -/// Like all instances of UB, if this assumption turns out to be wrong, i.e., the -/// `unreachable_unchecked()` call is actually reachable among all possible -/// control flow, the compiler will apply the wrong optimization strategy, and -/// may sometimes even corrupt seemingly unrelated code, causing -/// difficult-to-debug problems. +/// As the compiler assumes that all forms of Undefined Behavior can never +/// happen, it will eliminate all branches in the surrounding code that it can +/// determine will invariably lead to a call to `unreachable_unchecked()`. /// -/// Use this function only when you can prove that the code will never call it. -/// Otherwise, consider using the [`unreachable!`] macro, which does not allow -/// optimizations but will panic when executed. +/// If the assumptions embedded in using this function turn out to be wrong - +/// that is, if the site which is calling `unreachable_unchecked()` is actually +/// reachable at runtime - the compiler may have generated nonsensical machine +/// instructions for this situation, including in seemingly unrelated code, +/// causing difficult-to-debug problems. /// -/// [`unreachable!`]: ../macro.unreachable.html +/// Use this function sparingly. Consider using the [`unreachable!`] macro, +/// which may prevent some optimizations but will safely panic in case it is +/// actually reached at runtime. Benchmark your code to find out if using +/// `unreachable_unchecked()` comes with a performance benefit. /// -/// # Example +/// # Examples +/// +/// `unreachable_unchecked()` can be used in situations where the compiler +/// can't prove invariants that were previously established. Such situations +/// have a higher chance of occurring if those invariants are upheld by +/// external code that the compiler can't analyze. +/// ``` +/// fn prepare_inputs(divisors: &mut Vec) { +/// // Note to future-self when making changes: The invariant established +/// // here is NOT checked in `do_computation()`; if this changes, you HAVE +/// // to change `do_computation()`. +/// divisors.retain(|divisor| *divisor != 0) +/// } +/// +/// /// # Safety +/// /// All elements of `divisor` must be non-zero. +/// unsafe fn do_computation(i: u32, divisors: &[u32]) -> u32 { +/// divisors.iter().fold(i, |acc, divisor| { +/// // Convince the compiler that a division by zero can't happen here +/// // and a check is not needed below. +/// if *divisor == 0 { +/// // Safety: `divisor` can't be zero because of `prepare_inputs`, +/// // but the compiler does not know about this. We *promise* +/// // that we always call `prepare_inputs`. +/// std::hint::unreachable_unchecked() +/// } +/// // The compiler would normally introduce a check here that prevents +/// // a division by zero. However, if `divisor` was zero, the branch +/// // above would reach what we explicitly marked as unreachable. +/// // The compiler concludes that `divisor` can't be zero at this point +/// // and removes the - now proven useless - check. +/// acc / divisor +/// }) +/// } +/// +/// let mut divisors = vec![2, 0, 4]; +/// prepare_inputs(&mut divisors); +/// let result = unsafe { +/// // Safety: prepare_inputs() guarantees that divisors is non-zero +/// do_computation(100, &divisors) +/// }; +/// assert_eq!(result, 12); +/// +/// ``` +/// +/// While using `unreachable_unchecked()` is perfectly sound in the following +/// example, the compiler is able to prove that a division by zero is not +/// possible. Benchmarking reveals that `unreachable_unchecked()` provides +/// no benefit over using [`unreachable!`], while the latter does not introduce +/// the possibility of Undefined Behavior. /// /// ``` /// fn div_1(a: u32, b: u32) -> u32 { @@ -43,37 +91,102 @@ use crate::intrinsics; /// /// assert_eq!(div_1(7, 0), 7); /// assert_eq!(div_1(9, 1), 4); -/// assert_eq!(div_1(11, std::u32::MAX), 0); +/// assert_eq!(div_1(11, u32::MAX), 0); /// ``` #[inline] #[stable(feature = "unreachable", since = "1.27.0")] -pub unsafe fn unreachable_unchecked() -> ! { - intrinsics::unreachable() +#[rustc_const_stable(feature = "const_unreachable_unchecked", since = "1.57.0")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces +pub const unsafe fn unreachable_unchecked() -> ! { + // SAFETY: the safety contract for `intrinsics::unreachable` must + // be upheld by the caller. + unsafe { + intrinsics::assert_unsafe_precondition!("hint::unreachable_unchecked must never be reached", () => false); + intrinsics::unreachable() + } } -/// Emits a machine instruction hinting to the processor that it is running in busy-wait -/// spin-loop ("spin lock"). +/// Emits a machine instruction to signal the processor that it is running in +/// a busy-wait spin-loop ("spin lock"). /// -/// For a discussion of different locking strategies and their trade-offs, see -/// [`core::sync::atomic::spin_loop_hint`]. +/// Upon receiving the spin-loop signal the processor can optimize its behavior by, +/// for example, saving power or switching hyper-threads. /// -/// **Note**: On platforms that do not support receiving spin-loop hints this function does not -/// do anything at all. +/// This function is different from [`thread::yield_now`] which directly +/// yields to the system's scheduler, whereas `spin_loop` does not interact +/// with the operating system. /// -/// [`core::sync::atomic::spin_loop_hint`]: ../sync/atomic/fn.spin_loop_hint.html -#[inline] -#[unstable(feature = "renamed_spin_loop", issue = "55002")] +/// A common use case for `spin_loop` is implementing bounded optimistic +/// spinning in a CAS loop in synchronization primitives. To avoid problems +/// like priority inversion, it is strongly recommended that the spin loop is +/// terminated after a finite amount of iterations and an appropriate blocking +/// syscall is made. +/// +/// **Note**: On platforms that do not support receiving spin-loop hints this +/// function does not do anything at all. +/// +/// # Examples +/// +/// ``` +/// use std::sync::atomic::{AtomicBool, Ordering}; +/// use std::sync::Arc; +/// use std::{hint, thread}; +/// +/// // A shared atomic value that threads will use to coordinate +/// let live = Arc::new(AtomicBool::new(false)); +/// +/// // In a background thread we'll eventually set the value +/// let bg_work = { +/// let live = live.clone(); +/// thread::spawn(move || { +/// // Do some work, then make the value live +/// do_some_work(); +/// live.store(true, Ordering::Release); +/// }) +/// }; +/// +/// // Back on our current thread, we wait for the value to be set +/// while !live.load(Ordering::Acquire) { +/// // The spin loop is a hint to the CPU that we're waiting, but probably +/// // not for very long +/// hint::spin_loop(); +/// } +/// +/// // The value is now set +/// # fn do_some_work() {} +/// do_some_work(); +/// bg_work.join()?; +/// # Ok::<(), Box>(()) +/// ``` +/// +/// [`thread::yield_now`]: ../../std/thread/fn.yield_now.html +#[inline(always)] +#[stable(feature = "renamed_spin_loop", since = "1.49.0")] pub fn spin_loop() { - #[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), target_feature = "sse2"))] + #[cfg(target_arch = "x86")] { - #[cfg(target_arch = "x86")] + // SAFETY: the `cfg` attr ensures that we only execute this on x86 targets. + unsafe { crate::arch::x86::_mm_pause() }; + } + + #[cfg(target_arch = "x86_64")] + { + // SAFETY: the `cfg` attr ensures that we only execute this on x86_64 targets. + unsafe { crate::arch::x86_64::_mm_pause() }; + } + + // RISC-V platform spin loop hint implementation + { + // RISC-V RV32 and RV64 share the same PAUSE instruction, but they are located in different + // modules in `core::arch`. + // In this case, here we call `pause` function in each core arch module. + #[cfg(target_arch = "riscv32")] { - unsafe { crate::arch::x86::_mm_pause() }; + crate::arch::riscv32::pause(); } - - #[cfg(target_arch = "x86_64")] + #[cfg(target_arch = "riscv64")] { - unsafe { crate::arch::x86_64::_mm_pause() }; + crate::arch::riscv64::pause(); } } @@ -81,10 +194,13 @@ pub fn spin_loop() { { #[cfg(target_arch = "aarch64")] { - unsafe { crate::arch::aarch64::__yield() }; + // SAFETY: the `cfg` attr ensures that we only execute this on aarch64 targets. + unsafe { crate::arch::aarch64::__isb(crate::arch::aarch64::SY) }; } #[cfg(target_arch = "arm")] { + // SAFETY: the `cfg` attr ensures that we only execute this on arm targets + // with support for the v6 feature. unsafe { crate::arch::arm::__yield() }; } } @@ -93,27 +209,212 @@ pub fn spin_loop() { /// An identity function that *__hints__* to the compiler to be maximally pessimistic about what /// `black_box` could do. /// -/// [`std::convert::identity`]: https://doc.rust-lang.org/core/convert/fn.identity.html -/// /// Unlike [`std::convert::identity`], a Rust compiler is encouraged to assume that `black_box` can -/// use `x` in any possible valid way that Rust code is allowed to without introducing undefined +/// use `dummy` in any possible valid way that Rust code is allowed to without introducing undefined /// behavior in the calling code. This property makes `black_box` useful for writing code in which /// certain optimizations are not desired, such as benchmarks. /// /// Note however, that `black_box` is only (and can only be) provided on a "best-effort" basis. The /// extent to which it can block optimisations may vary depending upon the platform and code-gen /// backend used. Programs cannot rely on `black_box` for *correctness* in any way. +/// +/// [`std::convert::identity`]: crate::convert::identity +/// +/// # When is this useful? +/// +/// First and foremost: `black_box` does _not_ guarantee any exact behavior and, in some cases, may +/// do nothing at all. As such, it **must not be relied upon to control critical program behavior.** +/// This _immediately_ precludes any direct use of this function for cryptographic or security +/// purposes. +/// +/// While not suitable in those mission-critical cases, `back_box`'s functionality can generally be +/// relied upon for benchmarking, and should be used there. It will try to ensure that the +/// compiler doesn't optimize away part of the intended test code based on context. For +/// example: +/// +/// ``` +/// fn contains(haystack: &[&str], needle: &str) -> bool { +/// haystack.iter().any(|x| x == &needle) +/// } +/// +/// pub fn benchmark() { +/// let haystack = vec!["abc", "def", "ghi", "jkl", "mno"]; +/// let needle = "ghi"; +/// for _ in 0..10 { +/// contains(&haystack, needle); +/// } +/// } +/// ``` +/// +/// The compiler could theoretically make optimizations like the following: +/// +/// - `needle` and `haystack` are always the same, move the call to `contains` outside the loop and +/// delete the loop +/// - Inline `contains` +/// - `needle` and `haystack` have values known at compile time, `contains` is always true. Remove +/// the call and replace with `true` +/// - Nothing is done with the result of `contains`: delete this function call entirely +/// - `benchmark` now has no purpose: delete this function +/// +/// It is not likely that all of the above happens, but the compiler is definitely able to make some +/// optimizations that could result in a very inaccurate benchmark. This is where `black_box` comes +/// in: +/// +/// ``` +/// use std::hint::black_box; +/// +/// // Same `contains` function +/// fn contains(haystack: &[&str], needle: &str) -> bool { +/// haystack.iter().any(|x| x == &needle) +/// } +/// +/// pub fn benchmark() { +/// let haystack = vec!["abc", "def", "ghi", "jkl", "mno"]; +/// let needle = "ghi"; +/// for _ in 0..10 { +/// // Adjust our benchmark loop contents +/// black_box(contains(black_box(&haystack), black_box(needle))); +/// } +/// } +/// ``` +/// +/// This essentially tells the compiler to block optimizations across any calls to `black_box`. So, +/// it now: +/// +/// - Treats both arguments to `contains` as unpredictable: the body of `contains` can no longer be +/// optimized based on argument values +/// - Treats the call to `contains` and its result as volatile: the body of `benchmark` cannot +/// optimize this away +/// +/// This makes our benchmark much more realistic to how the function would be used in situ, where +/// arguments are usually not known at compile time and the result is used in some way. #[inline] -#[unstable(feature = "test", issue = "50297")] -#[allow(unreachable_code)] // this makes #[cfg] a bit easier below. -pub fn black_box(dummy: T) -> T { - // We need to "use" the argument in some way LLVM can't introspect, and on - // targets that support it we can typically leverage inline assembly to do - // this. LLVM's interpretation of inline assembly is that it's, well, a black - // box. This isn't the greatest implementation since it probably deoptimizes - // more than we want, but it's so far good enough. - unsafe { - asm!("" : : "r"(&dummy)); - dummy - } +#[stable(feature = "bench_black_box", since = "1.66.0")] +#[rustc_const_unstable(feature = "const_black_box", issue = "none")] +pub const fn black_box(dummy: T) -> T { + crate::intrinsics::black_box(dummy) +} + +/// An identity function that causes an `unused_must_use` warning to be +/// triggered if the given value is not used (returned, stored in a variable, +/// etc) by the caller. +/// +/// This is primarily intended for use in macro-generated code, in which a +/// [`#[must_use]` attribute][must_use] either on a type or a function would not +/// be convenient. +/// +/// [must_use]: https://doc.rust-lang.org/reference/attributes/diagnostics.html#the-must_use-attribute +/// +/// # Example +/// +/// ``` +/// #![feature(hint_must_use)] +/// +/// use core::fmt; +/// +/// pub struct Error(/* ... */); +/// +/// #[macro_export] +/// macro_rules! make_error { +/// ($($args:expr),*) => { +/// core::hint::must_use({ +/// let error = $crate::make_error(core::format_args!($($args),*)); +/// error +/// }) +/// }; +/// } +/// +/// // Implementation detail of make_error! macro. +/// #[doc(hidden)] +/// pub fn make_error(args: fmt::Arguments<'_>) -> Error { +/// Error(/* ... */) +/// } +/// +/// fn demo() -> Option { +/// if true { +/// // Oops, meant to write `return Some(make_error!("..."));` +/// Some(make_error!("...")); +/// } +/// None +/// } +/// # +/// # // Make rustdoc not wrap the whole snippet in fn main, so that $crate::make_error works +/// # fn main() {} +/// ``` +/// +/// In the above example, we'd like an `unused_must_use` lint to apply to the +/// value created by `make_error!`. However, neither `#[must_use]` on a struct +/// nor `#[must_use]` on a function is appropriate here, so the macro expands +/// using `core::hint::must_use` instead. +/// +/// - We wouldn't want `#[must_use]` on the `struct Error` because that would +/// make the following unproblematic code trigger a warning: +/// +/// ``` +/// # struct Error; +/// # +/// fn f(arg: &str) -> Result<(), Error> +/// # { Ok(()) } +/// +/// #[test] +/// fn t() { +/// // Assert that `f` returns error if passed an empty string. +/// // A value of type `Error` is unused here but that's not a problem. +/// f("").unwrap_err(); +/// } +/// ``` +/// +/// - Using `#[must_use]` on `fn make_error` can't help because the return value +/// *is* used, as the right-hand side of a `let` statement. The `let` +/// statement looks useless but is in fact necessary for ensuring that +/// temporaries within the `format_args` expansion are not kept alive past the +/// creation of the `Error`, as keeping them alive past that point can cause +/// autotrait issues in async code: +/// +/// ``` +/// # #![feature(hint_must_use)] +/// # +/// # struct Error; +/// # +/// # macro_rules! make_error { +/// # ($($args:expr),*) => { +/// # core::hint::must_use({ +/// # // If `let` isn't used, then `f()` produces a non-Send future. +/// # let error = make_error(core::format_args!($($args),*)); +/// # error +/// # }) +/// # }; +/// # } +/// # +/// # fn make_error(args: core::fmt::Arguments<'_>) -> Error { +/// # Error +/// # } +/// # +/// async fn f() { +/// // Using `let` inside the make_error expansion causes temporaries like +/// // `unsync()` to drop at the semicolon of that `let` statement, which +/// // is prior to the await point. They would otherwise stay around until +/// // the semicolon on *this* statement, which is after the await point, +/// // and the enclosing Future would not implement Send. +/// log(make_error!("look: {:p}", unsync())).await; +/// } +/// +/// async fn log(error: Error) {/* ... */} +/// +/// // Returns something without a Sync impl. +/// fn unsync() -> *const () { +/// 0 as *const () +/// } +/// # +/// # fn test() { +/// # fn assert_send(_: impl Send) {} +/// # assert_send(f()); +/// # } +/// ``` +#[unstable(feature = "hint_must_use", issue = "94745")] +#[rustc_const_unstable(feature = "hint_must_use", issue = "94745")] +#[must_use] // <-- :) +#[inline(always)] +pub const fn must_use(value: T) -> T { + value } diff --git a/crux-mir/lib/core/src/internal_macros.rs b/crux-mir/lib/core/src/internal_macros.rs index 4ea7dfc07..5d4c9ba73 100644 --- a/crux-mir/lib/core/src/internal_macros.rs +++ b/crux-mir/lib/core/src/internal_macros.rs @@ -1,10 +1,23 @@ // implements the unary operator "op &T" // based on "op T" where T is expected to be `Copy`able macro_rules! forward_ref_unop { - (impl $imp:ident, $method:ident for $t:ty) => { - forward_ref_unop!(impl $imp, $method for $t, + (impl const $imp:ident, $method:ident for $t:ty) => { + forward_ref_unop!(impl const $imp, $method for $t, #[stable(feature = "rust1", since = "1.0.0")]); }; + // Equivalent to the non-const version, with the addition of `rustc_const_unstable` + (impl const $imp:ident, $method:ident for $t:ty, #[$attr:meta]) => { + #[$attr] + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const $imp for &$t { + type Output = <$t as $imp>::Output; + + #[inline] + fn $method(self) -> <$t as $imp>::Output { + $imp::$method(*self) + } + } + }; (impl $imp:ident, $method:ident for $t:ty, #[$attr:meta]) => { #[$attr] impl $imp for &$t { @@ -21,10 +34,45 @@ macro_rules! forward_ref_unop { // implements binary operators "&T op U", "T op &U", "&T op &U" // based on "T op U" where T and U are expected to be `Copy`able macro_rules! forward_ref_binop { - (impl $imp:ident, $method:ident for $t:ty, $u:ty) => { - forward_ref_binop!(impl $imp, $method for $t, $u, + (impl const $imp:ident, $method:ident for $t:ty, $u:ty) => { + forward_ref_binop!(impl const $imp, $method for $t, $u, #[stable(feature = "rust1", since = "1.0.0")]); }; + // Equivalent to the non-const version, with the addition of `rustc_const_unstable` + (impl const $imp:ident, $method:ident for $t:ty, $u:ty, #[$attr:meta]) => { + #[$attr] + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl<'a> const $imp<$u> for &'a $t { + type Output = <$t as $imp<$u>>::Output; + + #[inline] + fn $method(self, other: $u) -> <$t as $imp<$u>>::Output { + $imp::$method(*self, other) + } + } + + #[$attr] + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const $imp<&$u> for $t { + type Output = <$t as $imp<$u>>::Output; + + #[inline] + fn $method(self, other: &$u) -> <$t as $imp<$u>>::Output { + $imp::$method(self, *other) + } + } + + #[$attr] + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const $imp<&$u> for &$t { + type Output = <$t as $imp<$u>>::Output; + + #[inline] + fn $method(self, other: &$u) -> <$t as $imp<$u>>::Output { + $imp::$method(*self, *other) + } + } + }; (impl $imp:ident, $method:ident for $t:ty, $u:ty, #[$attr:meta]) => { #[$attr] impl<'a> $imp<$u> for &'a $t { @@ -65,6 +113,21 @@ macro_rules! forward_ref_op_assign { forward_ref_op_assign!(impl $imp, $method for $t, $u, #[stable(feature = "op_assign_builtins_by_ref", since = "1.22.0")]); }; + (impl const $imp:ident, $method:ident for $t:ty, $u:ty) => { + forward_ref_op_assign!(impl const $imp, $method for $t, $u, + #[stable(feature = "op_assign_builtins_by_ref", since = "1.22.0")]); + }; + // Equivalent to the non-const version, with the addition of `rustc_const_unstable` + (impl const $imp:ident, $method:ident for $t:ty, $u:ty, #[$attr:meta]) => { + #[$attr] + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const $imp<&$u> for $t { + #[inline] + fn $method(&mut self, other: &$u) { + $imp::$method(self, *other); + } + } + }; (impl $imp:ident, $method:ident for $t:ty, $u:ty, #[$attr:meta]) => { #[$attr] impl $imp<&$u> for $t { @@ -77,7 +140,6 @@ macro_rules! forward_ref_op_assign { } /// Create a zero-size type similar to a closure type, but named. -#[unstable(feature = "std_internals", issue = "none")] macro_rules! impl_fn_for_zst { ($( $( #[$attr: meta] )* @@ -117,3 +179,80 @@ macro_rules! impl_fn_for_zst { )+ } } + +/// A macro for defining `#[cfg]` if-else statements. +/// +/// `cfg_if` is similar to the `if/elif` C preprocessor macro by allowing definition of a cascade +/// of `#[cfg]` cases, emitting the implementation which matches first. +/// +/// This allows you to conveniently provide a long list `#[cfg]`'d blocks of code without having to +/// rewrite each clause multiple times. +/// +/// # Example +/// +/// ```ignore(cannot-test-this-because-non-exported-macro) +/// cfg_if! { +/// if #[cfg(unix)] { +/// fn foo() { /* unix specific functionality */ } +/// } else if #[cfg(target_pointer_width = "32")] { +/// fn foo() { /* non-unix, 32-bit functionality */ } +/// } else { +/// fn foo() { /* fallback implementation */ } +/// } +/// } +/// +/// # fn main() {} +/// ``` +// This is a copy of `cfg_if!` from the `cfg_if` crate. +// The recursive invocations should use $crate if this is ever exported. +macro_rules! cfg_if { + // match if/else chains with a final `else` + ( + $( + if #[cfg( $i_meta:meta )] { $( $i_tokens:tt )* } + ) else+ + else { $( $e_tokens:tt )* } + ) => { + cfg_if! { + @__items () ; + $( + (( $i_meta ) ( $( $i_tokens )* )) , + )+ + (() ( $( $e_tokens )* )) , + } + }; + + // Internal and recursive macro to emit all the items + // + // Collects all the previous cfgs in a list at the beginning, so they can be + // negated. After the semicolon is all the remaining items. + (@__items ( $( $_:meta , )* ) ; ) => {}; + ( + @__items ( $( $no:meta , )* ) ; + (( $( $yes:meta )? ) ( $( $tokens:tt )* )) , + $( $rest:tt , )* + ) => { + // Emit all items within one block, applying an appropriate #[cfg]. The + // #[cfg] will require all `$yes` matchers specified and must also negate + // all previous matchers. + #[cfg(all( + $( $yes , )? + not(any( $( $no ),* )) + ))] + cfg_if! { @__identity $( $tokens )* } + + // Recurse to emit all other items in `$rest`, and when we do so add all + // our `$yes` matchers to the list of `$no` matchers as future emissions + // will have to negate everything we just matched as well. + cfg_if! { + @__items ( $( $no , )* $( $yes , )? ) ; + $( $rest , )* + } + }; + + // Internal macro to make __apply work out right for different match types, + // because of how macros match/expand stuff. + (@__identity $( $tokens:tt )* ) => { + $( $tokens )* + }; +} diff --git a/crux-mir/lib/core/src/intrinsics.rs b/crux-mir/lib/core/src/intrinsics.rs index f7ecbd80c..a315a28fb 100644 --- a/crux-mir/lib/core/src/intrinsics.rs +++ b/crux-mir/lib/core/src/intrinsics.rs @@ -1,7 +1,7 @@ //! Compiler intrinsics. //! -//! The corresponding definitions are in `librustc_codegen_llvm/intrinsic.rs`. -//! The corresponding const implementations are in `librustc_mir/interpret/intrinsics.rs` +//! The corresponding definitions are in . +//! The corresponding const implementations are in . //! //! # Const intrinsics //! @@ -9,13 +9,13 @@ //! This includes changes in the stability of the constness. //! //! In order to make an intrinsic usable at compile-time, one needs to copy the implementation -//! from https://github.com/rust-lang/miri/blob/master/src/shims/intrinsics.rs to -//! `librustc_mir/interpret/intrinsics.rs` and add a -//! `#[rustc_const_unstable(feature = "foo", issue = "01234")]` to the intrinsic. +//! from to +//! and add a +//! `#[rustc_const_unstable(feature = "const_such_and_such", issue = "01234")]` to the intrinsic declaration. //! //! If an intrinsic is supposed to be used from a `const fn` with a `rustc_const_stable` attribute, //! the intrinsic's attribute must be `rustc_const_stable`, too. Such a change should not be done -//! without T-lang consulation, because it bakes a feature into the language that cannot be +//! without T-lang consultation, because it bakes a feature into the language that cannot be //! replicated in user code without compiler support. //! //! # Volatiles @@ -25,7 +25,7 @@ //! across other volatile intrinsics. See the LLVM documentation on //! [[volatile]]. //! -//! [volatile]: http://llvm.org/docs/LangRef.html#volatile-memory-accesses +//! [volatile]: https://llvm.org/docs/LangRef.html#volatile-memory-accesses //! //! # Atomics //! @@ -33,7 +33,7 @@ //! words, with multiple possible memory orderings. They obey the same //! semantics as C++11. See the LLVM documentation on [[atomics]]. //! -//! [atomics]: http://llvm.org/docs/Atomics.html +//! [atomics]: https://llvm.org/docs/Atomics.html //! //! A quick refresher on memory ordering: //! @@ -54,14 +54,26 @@ )] #![allow(missing_docs)] +use crate::marker::DiscriminantKind; +use crate::marker::Tuple; use crate::mem; +#[cfg(not(bootstrap))] +pub mod mir; + +// These imports are used for simplifying intra-doc links +#[allow(unused_imports)] +#[cfg(all(target_has_atomic = "8", target_has_atomic = "32", target_has_atomic = "ptr"))] +use crate::sync::atomic::{self, AtomicBool, AtomicI32, AtomicIsize, AtomicU32, Ordering}; + #[stable(feature = "drop_in_place", since = "1.8.0")] -#[rustc_deprecated( - reason = "no longer an intrinsic - use `ptr::drop_in_place` directly", - since = "1.18.0" -)] -pub use crate::ptr::drop_in_place; +#[rustc_allowed_through_unstable_modules] +#[deprecated(note = "no longer an intrinsic - use `ptr::drop_in_place` directly", since = "1.52.0")] +#[inline] +pub unsafe fn drop_in_place(to_drop: *mut T) { + // SAFETY: see `ptr::drop_in_place` + unsafe { crate::ptr::drop_in_place(to_drop) } +} extern "rust-intrinsic" { // N.B., these intrinsics take raw pointers because they mutate aliased @@ -70,778 +82,618 @@ extern "rust-intrinsic" { /// Stores a value if the current value is the same as the `old` value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `compare_exchange` method by passing - /// [`Ordering::SeqCst`](../../std/sync/atomic/enum.Ordering.html) - /// as both the `success` and `failure` parameters. For example, - /// [`AtomicBool::compare_exchange`][compare_exchange]. - /// - /// [compare_exchange]: ../../std/sync/atomic/struct.AtomicBool.html#method.compare_exchange - pub fn atomic_cxchg(dst: *mut T, old: T, src: T) -> (T, bool); + /// [`atomic`] types via the `compare_exchange` method by passing + /// [`Ordering::Relaxed`] as both the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange`]. + pub fn atomic_cxchg_relaxed_relaxed(dst: *mut T, old: T, src: T) -> (T, bool); /// Stores a value if the current value is the same as the `old` value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `compare_exchange` method by passing - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html) - /// as both the `success` and `failure` parameters. For example, - /// [`AtomicBool::compare_exchange`][compare_exchange]. - /// - /// [compare_exchange]: ../../std/sync/atomic/struct.AtomicBool.html#method.compare_exchange - pub fn atomic_cxchg_acq(dst: *mut T, old: T, src: T) -> (T, bool); + /// [`atomic`] types via the `compare_exchange` method by passing + /// [`Ordering::Relaxed`] and [`Ordering::Acquire`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange`]. + pub fn atomic_cxchg_relaxed_acquire(dst: *mut T, old: T, src: T) -> (T, bool); /// Stores a value if the current value is the same as the `old` value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `compare_exchange` method by passing - /// [`Ordering::Release`](../../std/sync/atomic/enum.Ordering.html) - /// as the `success` and - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html) - /// as the `failure` parameters. For example, - /// [`AtomicBool::compare_exchange`][compare_exchange]. - /// - /// [compare_exchange]: ../../std/sync/atomic/struct.AtomicBool.html#method.compare_exchange - pub fn atomic_cxchg_rel(dst: *mut T, old: T, src: T) -> (T, bool); + /// [`atomic`] types via the `compare_exchange` method by passing + /// [`Ordering::Relaxed`] and [`Ordering::SeqCst`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange`]. + pub fn atomic_cxchg_relaxed_seqcst(dst: *mut T, old: T, src: T) -> (T, bool); /// Stores a value if the current value is the same as the `old` value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `compare_exchange` method by passing - /// [`Ordering::AcqRel`](../../std/sync/atomic/enum.Ordering.html) - /// as the `success` and - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html) - /// as the `failure` parameters. For example, - /// [`AtomicBool::compare_exchange`][compare_exchange]. + /// [`atomic`] types via the `compare_exchange` method by passing + /// [`Ordering::Acquire`] and [`Ordering::Relaxed`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange`]. + pub fn atomic_cxchg_acquire_relaxed(dst: *mut T, old: T, src: T) -> (T, bool); + /// Stores a value if the current value is the same as the `old` value. /// - /// [compare_exchange]: ../../std/sync/atomic/struct.AtomicBool.html#method.compare_exchange - pub fn atomic_cxchg_acqrel(dst: *mut T, old: T, src: T) -> (T, bool); + /// The stabilized version of this intrinsic is available on the + /// [`atomic`] types via the `compare_exchange` method by passing + /// [`Ordering::Acquire`] as both the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange`]. + pub fn atomic_cxchg_acquire_acquire(dst: *mut T, old: T, src: T) -> (T, bool); /// Stores a value if the current value is the same as the `old` value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `compare_exchange` method by passing - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html) - /// as both the `success` and `failure` parameters. For example, - /// [`AtomicBool::compare_exchange`][compare_exchange]. + /// [`atomic`] types via the `compare_exchange` method by passing + /// [`Ordering::Acquire`] and [`Ordering::SeqCst`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange`]. + pub fn atomic_cxchg_acquire_seqcst(dst: *mut T, old: T, src: T) -> (T, bool); + /// Stores a value if the current value is the same as the `old` value. /// - /// [compare_exchange]: ../../std/sync/atomic/struct.AtomicBool.html#method.compare_exchange - pub fn atomic_cxchg_relaxed(dst: *mut T, old: T, src: T) -> (T, bool); + /// The stabilized version of this intrinsic is available on the + /// [`atomic`] types via the `compare_exchange` method by passing + /// [`Ordering::Release`] and [`Ordering::Relaxed`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange`]. + pub fn atomic_cxchg_release_relaxed(dst: *mut T, old: T, src: T) -> (T, bool); /// Stores a value if the current value is the same as the `old` value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `compare_exchange` method by passing - /// [`Ordering::SeqCst`](../../std/sync/atomic/enum.Ordering.html) - /// as the `success` and - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html) - /// as the `failure` parameters. For example, - /// [`AtomicBool::compare_exchange`][compare_exchange]. + /// [`atomic`] types via the `compare_exchange` method by passing + /// [`Ordering::Release`] and [`Ordering::Acquire`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange`]. + pub fn atomic_cxchg_release_acquire(dst: *mut T, old: T, src: T) -> (T, bool); + /// Stores a value if the current value is the same as the `old` value. /// - /// [compare_exchange]: ../../std/sync/atomic/struct.AtomicBool.html#method.compare_exchange - pub fn atomic_cxchg_failrelaxed(dst: *mut T, old: T, src: T) -> (T, bool); + /// The stabilized version of this intrinsic is available on the + /// [`atomic`] types via the `compare_exchange` method by passing + /// [`Ordering::Release`] and [`Ordering::SeqCst`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange`]. + pub fn atomic_cxchg_release_seqcst(dst: *mut T, old: T, src: T) -> (T, bool); /// Stores a value if the current value is the same as the `old` value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `compare_exchange` method by passing - /// [`Ordering::SeqCst`](../../std/sync/atomic/enum.Ordering.html) - /// as the `success` and - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html) - /// as the `failure` parameters. For example, - /// [`AtomicBool::compare_exchange`][compare_exchange]. + /// [`atomic`] types via the `compare_exchange` method by passing + /// [`Ordering::AcqRel`] and [`Ordering::Relaxed`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange`]. + pub fn atomic_cxchg_acqrel_relaxed(dst: *mut T, old: T, src: T) -> (T, bool); + /// Stores a value if the current value is the same as the `old` value. /// - /// [compare_exchange]: ../../std/sync/atomic/struct.AtomicBool.html#method.compare_exchange - pub fn atomic_cxchg_failacq(dst: *mut T, old: T, src: T) -> (T, bool); + /// The stabilized version of this intrinsic is available on the + /// [`atomic`] types via the `compare_exchange` method by passing + /// [`Ordering::AcqRel`] and [`Ordering::Acquire`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange`]. + pub fn atomic_cxchg_acqrel_acquire(dst: *mut T, old: T, src: T) -> (T, bool); /// Stores a value if the current value is the same as the `old` value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `compare_exchange` method by passing - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html) - /// as the `success` and - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html) - /// as the `failure` parameters. For example, - /// [`AtomicBool::compare_exchange`][compare_exchange]. + /// [`atomic`] types via the `compare_exchange` method by passing + /// [`Ordering::AcqRel`] and [`Ordering::SeqCst`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange`]. + pub fn atomic_cxchg_acqrel_seqcst(dst: *mut T, old: T, src: T) -> (T, bool); + /// Stores a value if the current value is the same as the `old` value. /// - /// [compare_exchange]: ../../std/sync/atomic/struct.AtomicBool.html#method.compare_exchange - pub fn atomic_cxchg_acq_failrelaxed(dst: *mut T, old: T, src: T) -> (T, bool); + /// The stabilized version of this intrinsic is available on the + /// [`atomic`] types via the `compare_exchange` method by passing + /// [`Ordering::SeqCst`] and [`Ordering::Relaxed`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange`]. + pub fn atomic_cxchg_seqcst_relaxed(dst: *mut T, old: T, src: T) -> (T, bool); /// Stores a value if the current value is the same as the `old` value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `compare_exchange` method by passing - /// [`Ordering::AcqRel`](../../std/sync/atomic/enum.Ordering.html) - /// as the `success` and - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html) - /// as the `failure` parameters. For example, - /// [`AtomicBool::compare_exchange`][compare_exchange]. + /// [`atomic`] types via the `compare_exchange` method by passing + /// [`Ordering::SeqCst`] and [`Ordering::Acquire`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange`]. + pub fn atomic_cxchg_seqcst_acquire(dst: *mut T, old: T, src: T) -> (T, bool); + /// Stores a value if the current value is the same as the `old` value. /// - /// [compare_exchange]: ../../std/sync/atomic/struct.AtomicBool.html#method.compare_exchange - pub fn atomic_cxchg_acqrel_failrelaxed(dst: *mut T, old: T, src: T) -> (T, bool); + /// The stabilized version of this intrinsic is available on the + /// [`atomic`] types via the `compare_exchange` method by passing + /// [`Ordering::SeqCst`] as both the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange`]. + pub fn atomic_cxchg_seqcst_seqcst(dst: *mut T, old: T, src: T) -> (T, bool); /// Stores a value if the current value is the same as the `old` value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `compare_exchange_weak` method by passing - /// [`Ordering::SeqCst`](../../std/sync/atomic/enum.Ordering.html) - /// as both the `success` and `failure` parameters. For example, - /// [`AtomicBool::compare_exchange_weak`][cew]. - /// - /// [cew]: ../../std/sync/atomic/struct.AtomicBool.html#method.compare_exchange_weak - pub fn atomic_cxchgweak(dst: *mut T, old: T, src: T) -> (T, bool); + /// [`atomic`] types via the `compare_exchange_weak` method by passing + /// [`Ordering::Relaxed`] as both the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange_weak`]. + pub fn atomic_cxchgweak_relaxed_relaxed(dst: *mut T, old: T, src: T) -> (T, bool); /// Stores a value if the current value is the same as the `old` value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `compare_exchange_weak` method by passing - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html) - /// as both the `success` and `failure` parameters. For example, - /// [`AtomicBool::compare_exchange_weak`][cew]. - /// - /// [cew]: ../../std/sync/atomic/struct.AtomicBool.html#method.compare_exchange_weak - pub fn atomic_cxchgweak_acq(dst: *mut T, old: T, src: T) -> (T, bool); + /// [`atomic`] types via the `compare_exchange_weak` method by passing + /// [`Ordering::Relaxed`] and [`Ordering::Acquire`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange_weak`]. + pub fn atomic_cxchgweak_relaxed_acquire(dst: *mut T, old: T, src: T) -> (T, bool); /// Stores a value if the current value is the same as the `old` value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `compare_exchange_weak` method by passing - /// [`Ordering::Release`](../../std/sync/atomic/enum.Ordering.html) - /// as the `success` and - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html) - /// as the `failure` parameters. For example, - /// [`AtomicBool::compare_exchange_weak`][cew]. - /// - /// [cew]: ../../std/sync/atomic/struct.AtomicBool.html#method.compare_exchange_weak - pub fn atomic_cxchgweak_rel(dst: *mut T, old: T, src: T) -> (T, bool); + /// [`atomic`] types via the `compare_exchange_weak` method by passing + /// [`Ordering::Relaxed`] and [`Ordering::SeqCst`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange_weak`]. + pub fn atomic_cxchgweak_relaxed_seqcst(dst: *mut T, old: T, src: T) -> (T, bool); /// Stores a value if the current value is the same as the `old` value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `compare_exchange_weak` method by passing - /// [`Ordering::AcqRel`](../../std/sync/atomic/enum.Ordering.html) - /// as the `success` and - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html) - /// as the `failure` parameters. For example, - /// [`AtomicBool::compare_exchange_weak`][cew]. + /// [`atomic`] types via the `compare_exchange_weak` method by passing + /// [`Ordering::Acquire`] and [`Ordering::Relaxed`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange_weak`]. + pub fn atomic_cxchgweak_acquire_relaxed(dst: *mut T, old: T, src: T) -> (T, bool); + /// Stores a value if the current value is the same as the `old` value. /// - /// [cew]: ../../std/sync/atomic/struct.AtomicBool.html#method.compare_exchange_weak - pub fn atomic_cxchgweak_acqrel(dst: *mut T, old: T, src: T) -> (T, bool); + /// The stabilized version of this intrinsic is available on the + /// [`atomic`] types via the `compare_exchange_weak` method by passing + /// [`Ordering::Acquire`] as both the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange_weak`]. + pub fn atomic_cxchgweak_acquire_acquire(dst: *mut T, old: T, src: T) -> (T, bool); /// Stores a value if the current value is the same as the `old` value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `compare_exchange_weak` method by passing - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html) - /// as both the `success` and `failure` parameters. For example, - /// [`AtomicBool::compare_exchange_weak`][cew]. + /// [`atomic`] types via the `compare_exchange_weak` method by passing + /// [`Ordering::Acquire`] and [`Ordering::SeqCst`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange_weak`]. + pub fn atomic_cxchgweak_acquire_seqcst(dst: *mut T, old: T, src: T) -> (T, bool); + /// Stores a value if the current value is the same as the `old` value. /// - /// [cew]: ../../std/sync/atomic/struct.AtomicBool.html#method.compare_exchange_weak - pub fn atomic_cxchgweak_relaxed(dst: *mut T, old: T, src: T) -> (T, bool); + /// The stabilized version of this intrinsic is available on the + /// [`atomic`] types via the `compare_exchange_weak` method by passing + /// [`Ordering::Release`] and [`Ordering::Relaxed`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange_weak`]. + pub fn atomic_cxchgweak_release_relaxed(dst: *mut T, old: T, src: T) -> (T, bool); /// Stores a value if the current value is the same as the `old` value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `compare_exchange_weak` method by passing - /// [`Ordering::SeqCst`](../../std/sync/atomic/enum.Ordering.html) - /// as the `success` and - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html) - /// as the `failure` parameters. For example, - /// [`AtomicBool::compare_exchange_weak`][cew]. + /// [`atomic`] types via the `compare_exchange_weak` method by passing + /// [`Ordering::Release`] and [`Ordering::Acquire`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange_weak`]. + pub fn atomic_cxchgweak_release_acquire(dst: *mut T, old: T, src: T) -> (T, bool); + /// Stores a value if the current value is the same as the `old` value. /// - /// [cew]: ../../std/sync/atomic/struct.AtomicBool.html#method.compare_exchange_weak - pub fn atomic_cxchgweak_failrelaxed(dst: *mut T, old: T, src: T) -> (T, bool); + /// The stabilized version of this intrinsic is available on the + /// [`atomic`] types via the `compare_exchange_weak` method by passing + /// [`Ordering::Release`] and [`Ordering::SeqCst`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange_weak`]. + pub fn atomic_cxchgweak_release_seqcst(dst: *mut T, old: T, src: T) -> (T, bool); /// Stores a value if the current value is the same as the `old` value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `compare_exchange_weak` method by passing - /// [`Ordering::SeqCst`](../../std/sync/atomic/enum.Ordering.html) - /// as the `success` and - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html) - /// as the `failure` parameters. For example, - /// [`AtomicBool::compare_exchange_weak`][cew]. + /// [`atomic`] types via the `compare_exchange_weak` method by passing + /// [`Ordering::AcqRel`] and [`Ordering::Relaxed`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange_weak`]. + pub fn atomic_cxchgweak_acqrel_relaxed(dst: *mut T, old: T, src: T) -> (T, bool); + /// Stores a value if the current value is the same as the `old` value. /// - /// [cew]: ../../std/sync/atomic/struct.AtomicBool.html#method.compare_exchange_weak - pub fn atomic_cxchgweak_failacq(dst: *mut T, old: T, src: T) -> (T, bool); + /// The stabilized version of this intrinsic is available on the + /// [`atomic`] types via the `compare_exchange_weak` method by passing + /// [`Ordering::AcqRel`] and [`Ordering::Acquire`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange_weak`]. + pub fn atomic_cxchgweak_acqrel_acquire(dst: *mut T, old: T, src: T) -> (T, bool); /// Stores a value if the current value is the same as the `old` value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `compare_exchange_weak` method by passing - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html) - /// as the `success` and - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html) - /// as the `failure` parameters. For example, - /// [`AtomicBool::compare_exchange_weak`][cew]. + /// [`atomic`] types via the `compare_exchange_weak` method by passing + /// [`Ordering::AcqRel`] and [`Ordering::SeqCst`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange_weak`]. + pub fn atomic_cxchgweak_acqrel_seqcst(dst: *mut T, old: T, src: T) -> (T, bool); + /// Stores a value if the current value is the same as the `old` value. /// - /// [cew]: ../../std/sync/atomic/struct.AtomicBool.html#method.compare_exchange_weak - pub fn atomic_cxchgweak_acq_failrelaxed(dst: *mut T, old: T, src: T) -> (T, bool); + /// The stabilized version of this intrinsic is available on the + /// [`atomic`] types via the `compare_exchange_weak` method by passing + /// [`Ordering::SeqCst`] and [`Ordering::Relaxed`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange_weak`]. + pub fn atomic_cxchgweak_seqcst_relaxed(dst: *mut T, old: T, src: T) -> (T, bool); /// Stores a value if the current value is the same as the `old` value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `compare_exchange_weak` method by passing - /// [`Ordering::AcqRel`](../../std/sync/atomic/enum.Ordering.html) - /// as the `success` and - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html) - /// as the `failure` parameters. For example, - /// [`AtomicBool::compare_exchange_weak`][cew]. + /// [`atomic`] types via the `compare_exchange_weak` method by passing + /// [`Ordering::SeqCst`] and [`Ordering::Acquire`] as the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange_weak`]. + pub fn atomic_cxchgweak_seqcst_acquire(dst: *mut T, old: T, src: T) -> (T, bool); + /// Stores a value if the current value is the same as the `old` value. /// - /// [cew]: ../../std/sync/atomic/struct.AtomicBool.html#method.compare_exchange_weak - pub fn atomic_cxchgweak_acqrel_failrelaxed(dst: *mut T, old: T, src: T) -> (T, bool); + /// The stabilized version of this intrinsic is available on the + /// [`atomic`] types via the `compare_exchange_weak` method by passing + /// [`Ordering::SeqCst`] as both the success and failure parameters. + /// For example, [`AtomicBool::compare_exchange_weak`]. + pub fn atomic_cxchgweak_seqcst_seqcst(dst: *mut T, old: T, src: T) -> (T, bool); /// Loads the current value of the pointer. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `load` method by passing - /// [`Ordering::SeqCst`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::load`](../../std/sync/atomic/struct.AtomicBool.html#method.load). - pub fn atomic_load(src: *const T) -> T; + /// [`atomic`] types via the `load` method by passing + /// [`Ordering::SeqCst`] as the `order`. For example, [`AtomicBool::load`]. + pub fn atomic_load_seqcst(src: *const T) -> T; /// Loads the current value of the pointer. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `load` method by passing - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::load`](../../std/sync/atomic/struct.AtomicBool.html#method.load). - pub fn atomic_load_acq(src: *const T) -> T; + /// [`atomic`] types via the `load` method by passing + /// [`Ordering::Acquire`] as the `order`. For example, [`AtomicBool::load`]. + pub fn atomic_load_acquire(src: *const T) -> T; /// Loads the current value of the pointer. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `load` method by passing - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::load`](../../std/sync/atomic/struct.AtomicBool.html#method.load). - pub fn atomic_load_relaxed(src: *const T) -> T; - pub fn atomic_load_unordered(src: *const T) -> T; + /// [`atomic`] types via the `load` method by passing + /// [`Ordering::Relaxed`] as the `order`. For example, [`AtomicBool::load`]. + pub fn atomic_load_relaxed(src: *const T) -> T; + pub fn atomic_load_unordered(src: *const T) -> T; /// Stores the value at the specified memory location. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `store` method by passing - /// [`Ordering::SeqCst`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::store`](../../std/sync/atomic/struct.AtomicBool.html#method.store). - pub fn atomic_store(dst: *mut T, val: T); + /// [`atomic`] types via the `store` method by passing + /// [`Ordering::SeqCst`] as the `order`. For example, [`AtomicBool::store`]. + pub fn atomic_store_seqcst(dst: *mut T, val: T); /// Stores the value at the specified memory location. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `store` method by passing - /// [`Ordering::Release`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::store`](../../std/sync/atomic/struct.AtomicBool.html#method.store). - pub fn atomic_store_rel(dst: *mut T, val: T); + /// [`atomic`] types via the `store` method by passing + /// [`Ordering::Release`] as the `order`. For example, [`AtomicBool::store`]. + pub fn atomic_store_release(dst: *mut T, val: T); /// Stores the value at the specified memory location. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `store` method by passing - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::store`](../../std/sync/atomic/struct.AtomicBool.html#method.store). - pub fn atomic_store_relaxed(dst: *mut T, val: T); - pub fn atomic_store_unordered(dst: *mut T, val: T); + /// [`atomic`] types via the `store` method by passing + /// [`Ordering::Relaxed`] as the `order`. For example, [`AtomicBool::store`]. + pub fn atomic_store_relaxed(dst: *mut T, val: T); + pub fn atomic_store_unordered(dst: *mut T, val: T); /// Stores the value at the specified memory location, returning the old value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `swap` method by passing - /// [`Ordering::SeqCst`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::swap`](../../std/sync/atomic/struct.AtomicBool.html#method.swap). - pub fn atomic_xchg(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `swap` method by passing + /// [`Ordering::SeqCst`] as the `order`. For example, [`AtomicBool::swap`]. + pub fn atomic_xchg_seqcst(dst: *mut T, src: T) -> T; /// Stores the value at the specified memory location, returning the old value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `swap` method by passing - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::swap`](../../std/sync/atomic/struct.AtomicBool.html#method.swap). - pub fn atomic_xchg_acq(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `swap` method by passing + /// [`Ordering::Acquire`] as the `order`. For example, [`AtomicBool::swap`]. + pub fn atomic_xchg_acquire(dst: *mut T, src: T) -> T; /// Stores the value at the specified memory location, returning the old value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `swap` method by passing - /// [`Ordering::Release`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::swap`](../../std/sync/atomic/struct.AtomicBool.html#method.swap). - pub fn atomic_xchg_rel(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `swap` method by passing + /// [`Ordering::Release`] as the `order`. For example, [`AtomicBool::swap`]. + pub fn atomic_xchg_release(dst: *mut T, src: T) -> T; /// Stores the value at the specified memory location, returning the old value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `swap` method by passing - /// [`Ordering::AcqRel`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::swap`](../../std/sync/atomic/struct.AtomicBool.html#method.swap). - pub fn atomic_xchg_acqrel(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `swap` method by passing + /// [`Ordering::AcqRel`] as the `order`. For example, [`AtomicBool::swap`]. + pub fn atomic_xchg_acqrel(dst: *mut T, src: T) -> T; /// Stores the value at the specified memory location, returning the old value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `swap` method by passing - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::swap`](../../std/sync/atomic/struct.AtomicBool.html#method.swap). - pub fn atomic_xchg_relaxed(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `swap` method by passing + /// [`Ordering::Relaxed`] as the `order`. For example, [`AtomicBool::swap`]. + pub fn atomic_xchg_relaxed(dst: *mut T, src: T) -> T; /// Adds to the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_add` method by passing - /// [`Ordering::SeqCst`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicIsize::fetch_add`](../../std/sync/atomic/struct.AtomicIsize.html#method.fetch_add). - pub fn atomic_xadd(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_add` method by passing + /// [`Ordering::SeqCst`] as the `order`. For example, [`AtomicIsize::fetch_add`]. + pub fn atomic_xadd_seqcst(dst: *mut T, src: T) -> T; /// Adds to the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_add` method by passing - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicIsize::fetch_add`](../../std/sync/atomic/struct.AtomicIsize.html#method.fetch_add). - pub fn atomic_xadd_acq(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_add` method by passing + /// [`Ordering::Acquire`] as the `order`. For example, [`AtomicIsize::fetch_add`]. + pub fn atomic_xadd_acquire(dst: *mut T, src: T) -> T; /// Adds to the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_add` method by passing - /// [`Ordering::Release`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicIsize::fetch_add`](../../std/sync/atomic/struct.AtomicIsize.html#method.fetch_add). - pub fn atomic_xadd_rel(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_add` method by passing + /// [`Ordering::Release`] as the `order`. For example, [`AtomicIsize::fetch_add`]. + pub fn atomic_xadd_release(dst: *mut T, src: T) -> T; /// Adds to the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_add` method by passing - /// [`Ordering::AcqRel`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicIsize::fetch_add`](../../std/sync/atomic/struct.AtomicIsize.html#method.fetch_add). - pub fn atomic_xadd_acqrel(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_add` method by passing + /// [`Ordering::AcqRel`] as the `order`. For example, [`AtomicIsize::fetch_add`]. + pub fn atomic_xadd_acqrel(dst: *mut T, src: T) -> T; /// Adds to the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_add` method by passing - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicIsize::fetch_add`](../../std/sync/atomic/struct.AtomicIsize.html#method.fetch_add). - pub fn atomic_xadd_relaxed(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_add` method by passing + /// [`Ordering::Relaxed`] as the `order`. For example, [`AtomicIsize::fetch_add`]. + pub fn atomic_xadd_relaxed(dst: *mut T, src: T) -> T; /// Subtract from the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_sub` method by passing - /// [`Ordering::SeqCst`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicIsize::fetch_sub`](../../std/sync/atomic/struct.AtomicIsize.html#method.fetch_sub). - pub fn atomic_xsub(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_sub` method by passing + /// [`Ordering::SeqCst`] as the `order`. For example, [`AtomicIsize::fetch_sub`]. + pub fn atomic_xsub_seqcst(dst: *mut T, src: T) -> T; /// Subtract from the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_sub` method by passing - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicIsize::fetch_sub`](../../std/sync/atomic/struct.AtomicIsize.html#method.fetch_sub). - pub fn atomic_xsub_acq(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_sub` method by passing + /// [`Ordering::Acquire`] as the `order`. For example, [`AtomicIsize::fetch_sub`]. + pub fn atomic_xsub_acquire(dst: *mut T, src: T) -> T; /// Subtract from the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_sub` method by passing - /// [`Ordering::Release`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicIsize::fetch_sub`](../../std/sync/atomic/struct.AtomicIsize.html#method.fetch_sub). - pub fn atomic_xsub_rel(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_sub` method by passing + /// [`Ordering::Release`] as the `order`. For example, [`AtomicIsize::fetch_sub`]. + pub fn atomic_xsub_release(dst: *mut T, src: T) -> T; /// Subtract from the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_sub` method by passing - /// [`Ordering::AcqRel`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicIsize::fetch_sub`](../../std/sync/atomic/struct.AtomicIsize.html#method.fetch_sub). - pub fn atomic_xsub_acqrel(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_sub` method by passing + /// [`Ordering::AcqRel`] as the `order`. For example, [`AtomicIsize::fetch_sub`]. + pub fn atomic_xsub_acqrel(dst: *mut T, src: T) -> T; /// Subtract from the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_sub` method by passing - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicIsize::fetch_sub`](../../std/sync/atomic/struct.AtomicIsize.html#method.fetch_sub). - pub fn atomic_xsub_relaxed(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_sub` method by passing + /// [`Ordering::Relaxed`] as the `order`. For example, [`AtomicIsize::fetch_sub`]. + pub fn atomic_xsub_relaxed(dst: *mut T, src: T) -> T; /// Bitwise and with the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_and` method by passing - /// [`Ordering::SeqCst`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::fetch_and`](../../std/sync/atomic/struct.AtomicBool.html#method.fetch_and). - pub fn atomic_and(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_and` method by passing + /// [`Ordering::SeqCst`] as the `order`. For example, [`AtomicBool::fetch_and`]. + pub fn atomic_and_seqcst(dst: *mut T, src: T) -> T; /// Bitwise and with the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_and` method by passing - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::fetch_and`](../../std/sync/atomic/struct.AtomicBool.html#method.fetch_and). - pub fn atomic_and_acq(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_and` method by passing + /// [`Ordering::Acquire`] as the `order`. For example, [`AtomicBool::fetch_and`]. + pub fn atomic_and_acquire(dst: *mut T, src: T) -> T; /// Bitwise and with the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_and` method by passing - /// [`Ordering::Release`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::fetch_and`](../../std/sync/atomic/struct.AtomicBool.html#method.fetch_and). - pub fn atomic_and_rel(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_and` method by passing + /// [`Ordering::Release`] as the `order`. For example, [`AtomicBool::fetch_and`]. + pub fn atomic_and_release(dst: *mut T, src: T) -> T; /// Bitwise and with the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_and` method by passing - /// [`Ordering::AcqRel`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::fetch_and`](../../std/sync/atomic/struct.AtomicBool.html#method.fetch_and). - pub fn atomic_and_acqrel(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_and` method by passing + /// [`Ordering::AcqRel`] as the `order`. For example, [`AtomicBool::fetch_and`]. + pub fn atomic_and_acqrel(dst: *mut T, src: T) -> T; /// Bitwise and with the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_and` method by passing - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::fetch_and`](../../std/sync/atomic/struct.AtomicBool.html#method.fetch_and). - pub fn atomic_and_relaxed(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_and` method by passing + /// [`Ordering::Relaxed`] as the `order`. For example, [`AtomicBool::fetch_and`]. + pub fn atomic_and_relaxed(dst: *mut T, src: T) -> T; /// Bitwise nand with the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic::AtomicBool` type via the `fetch_nand` method by passing - /// [`Ordering::SeqCst`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::fetch_nand`](../../std/sync/atomic/struct.AtomicBool.html#method.fetch_nand). - pub fn atomic_nand(dst: *mut T, src: T) -> T; + /// [`AtomicBool`] type via the `fetch_nand` method by passing + /// [`Ordering::SeqCst`] as the `order`. For example, [`AtomicBool::fetch_nand`]. + pub fn atomic_nand_seqcst(dst: *mut T, src: T) -> T; /// Bitwise nand with the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic::AtomicBool` type via the `fetch_nand` method by passing - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::fetch_nand`](../../std/sync/atomic/struct.AtomicBool.html#method.fetch_nand). - pub fn atomic_nand_acq(dst: *mut T, src: T) -> T; + /// [`AtomicBool`] type via the `fetch_nand` method by passing + /// [`Ordering::Acquire`] as the `order`. For example, [`AtomicBool::fetch_nand`]. + pub fn atomic_nand_acquire(dst: *mut T, src: T) -> T; /// Bitwise nand with the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic::AtomicBool` type via the `fetch_nand` method by passing - /// [`Ordering::Release`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::fetch_nand`](../../std/sync/atomic/struct.AtomicBool.html#method.fetch_nand). - pub fn atomic_nand_rel(dst: *mut T, src: T) -> T; + /// [`AtomicBool`] type via the `fetch_nand` method by passing + /// [`Ordering::Release`] as the `order`. For example, [`AtomicBool::fetch_nand`]. + pub fn atomic_nand_release(dst: *mut T, src: T) -> T; /// Bitwise nand with the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic::AtomicBool` type via the `fetch_nand` method by passing - /// [`Ordering::AcqRel`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::fetch_nand`](../../std/sync/atomic/struct.AtomicBool.html#method.fetch_nand). - pub fn atomic_nand_acqrel(dst: *mut T, src: T) -> T; + /// [`AtomicBool`] type via the `fetch_nand` method by passing + /// [`Ordering::AcqRel`] as the `order`. For example, [`AtomicBool::fetch_nand`]. + pub fn atomic_nand_acqrel(dst: *mut T, src: T) -> T; /// Bitwise nand with the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic::AtomicBool` type via the `fetch_nand` method by passing - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::fetch_nand`](../../std/sync/atomic/struct.AtomicBool.html#method.fetch_nand). - pub fn atomic_nand_relaxed(dst: *mut T, src: T) -> T; + /// [`AtomicBool`] type via the `fetch_nand` method by passing + /// [`Ordering::Relaxed`] as the `order`. For example, [`AtomicBool::fetch_nand`]. + pub fn atomic_nand_relaxed(dst: *mut T, src: T) -> T; /// Bitwise or with the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_or` method by passing - /// [`Ordering::SeqCst`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::fetch_or`](../../std/sync/atomic/struct.AtomicBool.html#method.fetch_or). - pub fn atomic_or(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_or` method by passing + /// [`Ordering::SeqCst`] as the `order`. For example, [`AtomicBool::fetch_or`]. + pub fn atomic_or_seqcst(dst: *mut T, src: T) -> T; /// Bitwise or with the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_or` method by passing - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::fetch_or`](../../std/sync/atomic/struct.AtomicBool.html#method.fetch_or). - pub fn atomic_or_acq(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_or` method by passing + /// [`Ordering::Acquire`] as the `order`. For example, [`AtomicBool::fetch_or`]. + pub fn atomic_or_acquire(dst: *mut T, src: T) -> T; /// Bitwise or with the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_or` method by passing - /// [`Ordering::Release`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::fetch_or`](../../std/sync/atomic/struct.AtomicBool.html#method.fetch_or). - pub fn atomic_or_rel(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_or` method by passing + /// [`Ordering::Release`] as the `order`. For example, [`AtomicBool::fetch_or`]. + pub fn atomic_or_release(dst: *mut T, src: T) -> T; /// Bitwise or with the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_or` method by passing - /// [`Ordering::AcqRel`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::fetch_or`](../../std/sync/atomic/struct.AtomicBool.html#method.fetch_or). - pub fn atomic_or_acqrel(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_or` method by passing + /// [`Ordering::AcqRel`] as the `order`. For example, [`AtomicBool::fetch_or`]. + pub fn atomic_or_acqrel(dst: *mut T, src: T) -> T; /// Bitwise or with the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_or` method by passing - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::fetch_or`](../../std/sync/atomic/struct.AtomicBool.html#method.fetch_or). - pub fn atomic_or_relaxed(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_or` method by passing + /// [`Ordering::Relaxed`] as the `order`. For example, [`AtomicBool::fetch_or`]. + pub fn atomic_or_relaxed(dst: *mut T, src: T) -> T; /// Bitwise xor with the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_xor` method by passing - /// [`Ordering::SeqCst`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::fetch_xor`](../../std/sync/atomic/struct.AtomicBool.html#method.fetch_xor). - pub fn atomic_xor(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_xor` method by passing + /// [`Ordering::SeqCst`] as the `order`. For example, [`AtomicBool::fetch_xor`]. + pub fn atomic_xor_seqcst(dst: *mut T, src: T) -> T; /// Bitwise xor with the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_xor` method by passing - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::fetch_xor`](../../std/sync/atomic/struct.AtomicBool.html#method.fetch_xor). - pub fn atomic_xor_acq(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_xor` method by passing + /// [`Ordering::Acquire`] as the `order`. For example, [`AtomicBool::fetch_xor`]. + pub fn atomic_xor_acquire(dst: *mut T, src: T) -> T; /// Bitwise xor with the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_xor` method by passing - /// [`Ordering::Release`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::fetch_xor`](../../std/sync/atomic/struct.AtomicBool.html#method.fetch_xor). - pub fn atomic_xor_rel(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_xor` method by passing + /// [`Ordering::Release`] as the `order`. For example, [`AtomicBool::fetch_xor`]. + pub fn atomic_xor_release(dst: *mut T, src: T) -> T; /// Bitwise xor with the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_xor` method by passing - /// [`Ordering::AcqRel`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::fetch_xor`](../../std/sync/atomic/struct.AtomicBool.html#method.fetch_xor). - pub fn atomic_xor_acqrel(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_xor` method by passing + /// [`Ordering::AcqRel`] as the `order`. For example, [`AtomicBool::fetch_xor`]. + pub fn atomic_xor_acqrel(dst: *mut T, src: T) -> T; /// Bitwise xor with the current value, returning the previous value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` types via the `fetch_xor` method by passing - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html) - /// as the `order`. For example, - /// [`AtomicBool::fetch_xor`](../../std/sync/atomic/struct.AtomicBool.html#method.fetch_xor). - pub fn atomic_xor_relaxed(dst: *mut T, src: T) -> T; + /// [`atomic`] types via the `fetch_xor` method by passing + /// [`Ordering::Relaxed`] as the `order`. For example, [`AtomicBool::fetch_xor`]. + pub fn atomic_xor_relaxed(dst: *mut T, src: T) -> T; /// Maximum with the current value using a signed comparison. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` signed integer types via the `fetch_max` method by passing - /// [`Ordering::SeqCst`](../../std/sync/atomic/enum.Ordering.html#variant.SeqCst) - /// as the `order`. For example, - /// [`AtomicI32::fetch_max`](../../std/sync/atomic/struct.AtomicI32.html#method.fetch_max). - pub fn atomic_max(dst: *mut T, src: T) -> T; + /// [`atomic`] signed integer types via the `fetch_max` method by passing + /// [`Ordering::SeqCst`] as the `order`. For example, [`AtomicI32::fetch_max`]. + pub fn atomic_max_seqcst(dst: *mut T, src: T) -> T; /// Maximum with the current value using a signed comparison. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` signed integer types via the `fetch_max` method by passing - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html#variant.Acquire) - /// as the `order`. For example, - /// [`AtomicI32::fetch_max`](../../std/sync/atomic/struct.AtomicI32.html#method.fetch_max). - pub fn atomic_max_acq(dst: *mut T, src: T) -> T; + /// [`atomic`] signed integer types via the `fetch_max` method by passing + /// [`Ordering::Acquire`] as the `order`. For example, [`AtomicI32::fetch_max`]. + pub fn atomic_max_acquire(dst: *mut T, src: T) -> T; /// Maximum with the current value using a signed comparison. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` signed integer types via the `fetch_max` method by passing - /// [`Ordering::Release`](../../std/sync/atomic/enum.Ordering.html#variant.Release) - /// as the `order`. For example, - /// [`AtomicI32::fetch_max`](../../std/sync/atomic/struct.AtomicI32.html#method.fetch_max). - pub fn atomic_max_rel(dst: *mut T, src: T) -> T; + /// [`atomic`] signed integer types via the `fetch_max` method by passing + /// [`Ordering::Release`] as the `order`. For example, [`AtomicI32::fetch_max`]. + pub fn atomic_max_release(dst: *mut T, src: T) -> T; /// Maximum with the current value using a signed comparison. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` signed integer types via the `fetch_max` method by passing - /// [`Ordering::AcqRel`](../../std/sync/atomic/enum.Ordering.html#variant.AcqRel) - /// as the `order`. For example, - /// [`AtomicI32::fetch_max`](../../std/sync/atomic/struct.AtomicI32.html#method.fetch_max). - pub fn atomic_max_acqrel(dst: *mut T, src: T) -> T; + /// [`atomic`] signed integer types via the `fetch_max` method by passing + /// [`Ordering::AcqRel`] as the `order`. For example, [`AtomicI32::fetch_max`]. + pub fn atomic_max_acqrel(dst: *mut T, src: T) -> T; /// Maximum with the current value. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` signed integer types via the `fetch_max` method by passing - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html#variant.Relaxed) - /// as the `order`. For example, - /// [`AtomicI32::fetch_max`](../../std/sync/atomic/struct.AtomicI32.html#method.fetch_max). - pub fn atomic_max_relaxed(dst: *mut T, src: T) -> T; + /// [`atomic`] signed integer types via the `fetch_max` method by passing + /// [`Ordering::Relaxed`] as the `order`. For example, [`AtomicI32::fetch_max`]. + pub fn atomic_max_relaxed(dst: *mut T, src: T) -> T; /// Minimum with the current value using a signed comparison. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` signed integer types via the `fetch_min` method by passing - /// [`Ordering::SeqCst`](../../std/sync/atomic/enum.Ordering.html#variant.SeqCst) - /// as the `order`. For example, - /// [`AtomicI32::fetch_min`](../../std/sync/atomic/struct.AtomicI32.html#method.fetch_min). - pub fn atomic_min(dst: *mut T, src: T) -> T; + /// [`atomic`] signed integer types via the `fetch_min` method by passing + /// [`Ordering::SeqCst`] as the `order`. For example, [`AtomicI32::fetch_min`]. + pub fn atomic_min_seqcst(dst: *mut T, src: T) -> T; /// Minimum with the current value using a signed comparison. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` signed integer types via the `fetch_min` method by passing - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html#variant.Acquire) - /// as the `order`. For example, - /// [`AtomicI32::fetch_min`](../../std/sync/atomic/struct.AtomicI32.html#method.fetch_min). - pub fn atomic_min_acq(dst: *mut T, src: T) -> T; + /// [`atomic`] signed integer types via the `fetch_min` method by passing + /// [`Ordering::Acquire`] as the `order`. For example, [`AtomicI32::fetch_min`]. + pub fn atomic_min_acquire(dst: *mut T, src: T) -> T; /// Minimum with the current value using a signed comparison. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` signed integer types via the `fetch_min` method by passing - /// [`Ordering::Release`](../../std/sync/atomic/enum.Ordering.html#variant.Release) - /// as the `order`. For example, - /// [`AtomicI32::fetch_min`](../../std/sync/atomic/struct.AtomicI32.html#method.fetch_min). - pub fn atomic_min_rel(dst: *mut T, src: T) -> T; + /// [`atomic`] signed integer types via the `fetch_min` method by passing + /// [`Ordering::Release`] as the `order`. For example, [`AtomicI32::fetch_min`]. + pub fn atomic_min_release(dst: *mut T, src: T) -> T; /// Minimum with the current value using a signed comparison. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` signed integer types via the `fetch_min` method by passing - /// [`Ordering::AcqRel`](../../std/sync/atomic/enum.Ordering.html#variant.AcqRel) - /// as the `order`. For example, - /// [`AtomicI32::fetch_min`](../../std/sync/atomic/struct.AtomicI32.html#method.fetch_min). - pub fn atomic_min_acqrel(dst: *mut T, src: T) -> T; + /// [`atomic`] signed integer types via the `fetch_min` method by passing + /// [`Ordering::AcqRel`] as the `order`. For example, [`AtomicI32::fetch_min`]. + pub fn atomic_min_acqrel(dst: *mut T, src: T) -> T; /// Minimum with the current value using a signed comparison. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` signed integer types via the `fetch_min` method by passing - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html#variant.Relaxed) - /// as the `order`. For example, - /// [`AtomicI32::fetch_min`](../../std/sync/atomic/struct.AtomicI32.html#method.fetch_min). - pub fn atomic_min_relaxed(dst: *mut T, src: T) -> T; + /// [`atomic`] signed integer types via the `fetch_min` method by passing + /// [`Ordering::Relaxed`] as the `order`. For example, [`AtomicI32::fetch_min`]. + pub fn atomic_min_relaxed(dst: *mut T, src: T) -> T; /// Minimum with the current value using an unsigned comparison. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` unsigned integer types via the `fetch_min` method by passing - /// [`Ordering::SeqCst`](../../std/sync/atomic/enum.Ordering.html#variant.SeqCst) - /// as the `order`. For example, - /// [`AtomicU32::fetch_min`](../../std/sync/atomic/struct.AtomicU32.html#method.fetch_min). - pub fn atomic_umin(dst: *mut T, src: T) -> T; + /// [`atomic`] unsigned integer types via the `fetch_min` method by passing + /// [`Ordering::SeqCst`] as the `order`. For example, [`AtomicU32::fetch_min`]. + pub fn atomic_umin_seqcst(dst: *mut T, src: T) -> T; /// Minimum with the current value using an unsigned comparison. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` unsigned integer types via the `fetch_min` method by passing - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html#variant.Acquire) - /// as the `order`. For example, - /// [`AtomicU32::fetch_min`](../../std/sync/atomic/struct.AtomicU32.html#method.fetch_min). - pub fn atomic_umin_acq(dst: *mut T, src: T) -> T; + /// [`atomic`] unsigned integer types via the `fetch_min` method by passing + /// [`Ordering::Acquire`] as the `order`. For example, [`AtomicU32::fetch_min`]. + pub fn atomic_umin_acquire(dst: *mut T, src: T) -> T; /// Minimum with the current value using an unsigned comparison. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` unsigned integer types via the `fetch_min` method by passing - /// [`Ordering::Release`](../../std/sync/atomic/enum.Ordering.html#variant.Release) - /// as the `order`. For example, - /// [`AtomicU32::fetch_min`](../../std/sync/atomic/struct.AtomicU32.html#method.fetch_min). - pub fn atomic_umin_rel(dst: *mut T, src: T) -> T; + /// [`atomic`] unsigned integer types via the `fetch_min` method by passing + /// [`Ordering::Release`] as the `order`. For example, [`AtomicU32::fetch_min`]. + pub fn atomic_umin_release(dst: *mut T, src: T) -> T; /// Minimum with the current value using an unsigned comparison. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` unsigned integer types via the `fetch_min` method by passing - /// [`Ordering::AcqRel`](../../std/sync/atomic/enum.Ordering.html#variant.AcqRel) - /// as the `order`. For example, - /// [`AtomicU32::fetch_min`](../../std/sync/atomic/struct.AtomicU32.html#method.fetch_min). - pub fn atomic_umin_acqrel(dst: *mut T, src: T) -> T; + /// [`atomic`] unsigned integer types via the `fetch_min` method by passing + /// [`Ordering::AcqRel`] as the `order`. For example, [`AtomicU32::fetch_min`]. + pub fn atomic_umin_acqrel(dst: *mut T, src: T) -> T; /// Minimum with the current value using an unsigned comparison. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` unsigned integer types via the `fetch_min` method by passing - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html#variant.Relaxed) - /// as the `order`. For example, - /// [`AtomicU32::fetch_min`](../../std/sync/atomic/struct.AtomicU32.html#method.fetch_min). - pub fn atomic_umin_relaxed(dst: *mut T, src: T) -> T; + /// [`atomic`] unsigned integer types via the `fetch_min` method by passing + /// [`Ordering::Relaxed`] as the `order`. For example, [`AtomicU32::fetch_min`]. + pub fn atomic_umin_relaxed(dst: *mut T, src: T) -> T; /// Maximum with the current value using an unsigned comparison. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` unsigned integer types via the `fetch_max` method by passing - /// [`Ordering::SeqCst`](../../std/sync/atomic/enum.Ordering.html#variant.SeqCst) - /// as the `order`. For example, - /// [`AtomicU32::fetch_max`](../../std/sync/atomic/struct.AtomicU32.html#method.fetch_max). - pub fn atomic_umax(dst: *mut T, src: T) -> T; + /// [`atomic`] unsigned integer types via the `fetch_max` method by passing + /// [`Ordering::SeqCst`] as the `order`. For example, [`AtomicU32::fetch_max`]. + pub fn atomic_umax_seqcst(dst: *mut T, src: T) -> T; /// Maximum with the current value using an unsigned comparison. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` unsigned integer types via the `fetch_max` method by passing - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html#variant.Acquire) - /// as the `order`. For example, - /// [`AtomicU32::fetch_max`](../../std/sync/atomic/struct.AtomicU32.html#method.fetch_max). - pub fn atomic_umax_acq(dst: *mut T, src: T) -> T; + /// [`atomic`] unsigned integer types via the `fetch_max` method by passing + /// [`Ordering::Acquire`] as the `order`. For example, [`AtomicU32::fetch_max`]. + pub fn atomic_umax_acquire(dst: *mut T, src: T) -> T; /// Maximum with the current value using an unsigned comparison. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` unsigned integer types via the `fetch_max` method by passing - /// [`Ordering::Release`](../../std/sync/atomic/enum.Ordering.html#variant.Release) - /// as the `order`. For example, - /// [`AtomicU32::fetch_max`](../../std/sync/atomic/struct.AtomicU32.html#method.fetch_max). - pub fn atomic_umax_rel(dst: *mut T, src: T) -> T; + /// [`atomic`] unsigned integer types via the `fetch_max` method by passing + /// [`Ordering::Release`] as the `order`. For example, [`AtomicU32::fetch_max`]. + pub fn atomic_umax_release(dst: *mut T, src: T) -> T; /// Maximum with the current value using an unsigned comparison. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` unsigned integer types via the `fetch_max` method by passing - /// [`Ordering::AcqRel`](../../std/sync/atomic/enum.Ordering.html#variant.AcqRel) - /// as the `order`. For example, - /// [`AtomicU32::fetch_max`](../../std/sync/atomic/struct.AtomicU32.html#method.fetch_max). - pub fn atomic_umax_acqrel(dst: *mut T, src: T) -> T; + /// [`atomic`] unsigned integer types via the `fetch_max` method by passing + /// [`Ordering::AcqRel`] as the `order`. For example, [`AtomicU32::fetch_max`]. + pub fn atomic_umax_acqrel(dst: *mut T, src: T) -> T; /// Maximum with the current value using an unsigned comparison. /// /// The stabilized version of this intrinsic is available on the - /// `std::sync::atomic` unsigned integer types via the `fetch_max` method by passing - /// [`Ordering::Relaxed`](../../std/sync/atomic/enum.Ordering.html#variant.Relaxed) - /// as the `order`. For example, - /// [`AtomicU32::fetch_max`](../../std/sync/atomic/struct.AtomicU32.html#method.fetch_max). - pub fn atomic_umax_relaxed(dst: *mut T, src: T) -> T; - - /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction - /// if supported; otherwise, it is a no-op. - /// Prefetches have no effect on the behavior of the program but can change its performance - /// characteristics. - /// - /// The `locality` argument must be a constant integer and is a temporal locality specifier - /// ranging from (0) - no locality, to (3) - extremely local keep in cache - pub fn prefetch_read_data(data: *const T, locality: i32); - /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction - /// if supported; otherwise, it is a no-op. - /// Prefetches have no effect on the behavior of the program but can change its performance - /// characteristics. - /// - /// The `locality` argument must be a constant integer and is a temporal locality specifier - /// ranging from (0) - no locality, to (3) - extremely local keep in cache - pub fn prefetch_write_data(data: *const T, locality: i32); - /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction - /// if supported; otherwise, it is a no-op. - /// Prefetches have no effect on the behavior of the program but can change its performance - /// characteristics. - /// - /// The `locality` argument must be a constant integer and is a temporal locality specifier - /// ranging from (0) - no locality, to (3) - extremely local keep in cache - pub fn prefetch_read_instruction(data: *const T, locality: i32); - /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction - /// if supported; otherwise, it is a no-op. - /// Prefetches have no effect on the behavior of the program but can change its performance - /// characteristics. - /// - /// The `locality` argument must be a constant integer and is a temporal locality specifier - /// ranging from (0) - no locality, to (3) - extremely local keep in cache - pub fn prefetch_write_instruction(data: *const T, locality: i32); -} - -extern "rust-intrinsic" { + /// [`atomic`] unsigned integer types via the `fetch_max` method by passing + /// [`Ordering::Relaxed`] as the `order`. For example, [`AtomicU32::fetch_max`]. + pub fn atomic_umax_relaxed(dst: *mut T, src: T) -> T; /// An atomic fence. /// /// The stabilized version of this intrinsic is available in - /// [`std::sync::atomic::fence`](../../std/sync/atomic/fn.fence.html) - /// by passing - /// [`Ordering::SeqCst`](../../std/sync/atomic/enum.Ordering.html#variant.SeqCst) + /// [`atomic::fence`] by passing [`Ordering::SeqCst`] /// as the `order`. - pub fn atomic_fence(); + pub fn atomic_fence_seqcst(); /// An atomic fence. /// /// The stabilized version of this intrinsic is available in - /// [`std::sync::atomic::fence`](../../std/sync/atomic/fn.fence.html) - /// by passing - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html#variant.Acquire) + /// [`atomic::fence`] by passing [`Ordering::Acquire`] /// as the `order`. - pub fn atomic_fence_acq(); + pub fn atomic_fence_acquire(); /// An atomic fence. /// /// The stabilized version of this intrinsic is available in - /// [`std::sync::atomic::fence`](../../std/sync/atomic/fn.fence.html) - /// by passing - /// [`Ordering::Release`](../../std/sync/atomic/enum.Ordering.html#variant.Release) + /// [`atomic::fence`] by passing [`Ordering::Release`] /// as the `order`. - pub fn atomic_fence_rel(); + pub fn atomic_fence_release(); /// An atomic fence. /// /// The stabilized version of this intrinsic is available in - /// [`std::sync::atomic::fence`](../../std/sync/atomic/fn.fence.html) - /// by passing - /// [`Ordering::AcqRel`](../../std/sync/atomic/enum.Ordering.html#variant.AcqRel) + /// [`atomic::fence`] by passing [`Ordering::AcqRel`] /// as the `order`. pub fn atomic_fence_acqrel(); @@ -853,11 +705,9 @@ extern "rust-intrinsic" { /// such as when interacting with signal handlers. /// /// The stabilized version of this intrinsic is available in - /// [`std::sync::atomic::compiler_fence`](../../std/sync/atomic/fn.compiler_fence.html) - /// by passing - /// [`Ordering::SeqCst`](../../std/sync/atomic/enum.Ordering.html#variant.SeqCst) + /// [`atomic::compiler_fence`] by passing [`Ordering::SeqCst`] /// as the `order`. - pub fn atomic_singlethreadfence(); + pub fn atomic_singlethreadfence_seqcst(); /// A compiler-only memory barrier. /// /// Memory accesses will never be reordered across this barrier by the @@ -866,11 +716,9 @@ extern "rust-intrinsic" { /// such as when interacting with signal handlers. /// /// The stabilized version of this intrinsic is available in - /// [`std::sync::atomic::compiler_fence`](../../std/sync/atomic/fn.compiler_fence.html) - /// by passing - /// [`Ordering::Acquire`](../../std/sync/atomic/enum.Ordering.html#variant.Acquire) + /// [`atomic::compiler_fence`] by passing [`Ordering::Acquire`] /// as the `order`. - pub fn atomic_singlethreadfence_acq(); + pub fn atomic_singlethreadfence_acquire(); /// A compiler-only memory barrier. /// /// Memory accesses will never be reordered across this barrier by the @@ -879,11 +727,9 @@ extern "rust-intrinsic" { /// such as when interacting with signal handlers. /// /// The stabilized version of this intrinsic is available in - /// [`std::sync::atomic::compiler_fence`](../../std/sync/atomic/fn.compiler_fence.html) - /// by passing - /// [`Ordering::Release`](../../std/sync/atomic/enum.Ordering.html#variant.Release) + /// [`atomic::compiler_fence`] by passing [`Ordering::Release`] /// as the `order`. - pub fn atomic_singlethreadfence_rel(); + pub fn atomic_singlethreadfence_release(); /// A compiler-only memory barrier. /// /// Memory accesses will never be reordered across this barrier by the @@ -892,12 +738,51 @@ extern "rust-intrinsic" { /// such as when interacting with signal handlers. /// /// The stabilized version of this intrinsic is available in - /// [`std::sync::atomic::compiler_fence`](../../std/sync/atomic/fn.compiler_fence.html) - /// by passing - /// [`Ordering::AcqRel`](../../std/sync/atomic/enum.Ordering.html#variant.AcqRel) + /// [`atomic::compiler_fence`] by passing [`Ordering::AcqRel`] /// as the `order`. pub fn atomic_singlethreadfence_acqrel(); + /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction + /// if supported; otherwise, it is a no-op. + /// Prefetches have no effect on the behavior of the program but can change its performance + /// characteristics. + /// + /// The `locality` argument must be a constant integer and is a temporal locality specifier + /// ranging from (0) - no locality, to (3) - extremely local keep in cache. + /// + /// This intrinsic does not have a stable counterpart. + pub fn prefetch_read_data(data: *const T, locality: i32); + /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction + /// if supported; otherwise, it is a no-op. + /// Prefetches have no effect on the behavior of the program but can change its performance + /// characteristics. + /// + /// The `locality` argument must be a constant integer and is a temporal locality specifier + /// ranging from (0) - no locality, to (3) - extremely local keep in cache. + /// + /// This intrinsic does not have a stable counterpart. + pub fn prefetch_write_data(data: *const T, locality: i32); + /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction + /// if supported; otherwise, it is a no-op. + /// Prefetches have no effect on the behavior of the program but can change its performance + /// characteristics. + /// + /// The `locality` argument must be a constant integer and is a temporal locality specifier + /// ranging from (0) - no locality, to (3) - extremely local keep in cache. + /// + /// This intrinsic does not have a stable counterpart. + pub fn prefetch_read_instruction(data: *const T, locality: i32); + /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction + /// if supported; otherwise, it is a no-op. + /// Prefetches have no effect on the behavior of the program but can change its performance + /// characteristics. + /// + /// The `locality` argument must be a constant integer and is a temporal locality specifier + /// ranging from (0) - no locality, to (3) - extremely local keep in cache. + /// + /// This intrinsic does not have a stable counterpart. + pub fn prefetch_write_instruction(data: *const T, locality: i32); + /// Magic intrinsic that derives its meaning from attributes /// attached to the function. /// @@ -905,23 +790,38 @@ extern "rust-intrinsic" { /// that `rustc_peek(potentially_uninitialized)` would actually /// double-check that dataflow did indeed compute that it is /// uninitialized at that point in the control flow. + /// + /// This intrinsic should not be used outside of the compiler. + #[rustc_safe_intrinsic] pub fn rustc_peek(_: T) -> T; /// Aborts the execution of the process. /// - /// The stabilized version of this intrinsic is - /// [`std::process::abort`](../../std/process/fn.abort.html) + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// + /// [`std::process::abort`](../../std/process/fn.abort.html) is to be preferred if possible, + /// as its behavior is more user-friendly and more stable. + /// + /// The current implementation of `intrinsics::abort` is to invoke an invalid instruction, + /// on most platforms. + /// On Unix, the + /// process will probably terminate with a signal like `SIGABRT`, `SIGILL`, `SIGTRAP`, `SIGSEGV` or + /// `SIGBUS`. The precise behaviour is not guaranteed and not stable. + #[rustc_safe_intrinsic] pub fn abort() -> !; - /// Tells LLVM that this point in the code is not reachable, enabling - /// further optimizations. + /// Informs the optimizer that this point in the code is not reachable, + /// enabling further optimizations. /// /// N.B., this is very different from the `unreachable!()` macro: Unlike the /// macro, which panics when it is executed, it is *undefined behavior* to /// reach code marked with this function. /// - /// The stabilized version of this intrinsic is - /// [`std::hint::unreachable_unchecked`](../../std/hint/fn.unreachable_unchecked.html). + /// The stabilized version of this intrinsic is [`core::hint::unreachable_unchecked`]. + #[rustc_const_stable(feature = "const_unreachable_unchecked", since = "1.57.0")] pub fn unreachable() -> !; /// Informs the optimizer that a condition is always true. @@ -932,121 +832,195 @@ extern "rust-intrinsic" { /// with optimization of surrounding code and reduce performance. It should /// not be used if the invariant can be discovered by the optimizer on its /// own, or if it does not enable any significant optimizations. + /// + /// This intrinsic does not have a stable counterpart. + #[rustc_const_unstable(feature = "const_assume", issue = "76972")] pub fn assume(b: bool); /// Hints to the compiler that branch condition is likely to be true. /// Returns the value passed to it. /// /// Any use other than with `if` statements will probably not have an effect. + /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// + /// This intrinsic does not have a stable counterpart. + #[rustc_const_unstable(feature = "const_likely", issue = "none")] + #[rustc_safe_intrinsic] pub fn likely(b: bool) -> bool; /// Hints to the compiler that branch condition is likely to be false. /// Returns the value passed to it. /// /// Any use other than with `if` statements will probably not have an effect. + /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// + /// This intrinsic does not have a stable counterpart. + #[rustc_const_unstable(feature = "const_likely", issue = "none")] + #[rustc_safe_intrinsic] pub fn unlikely(b: bool) -> bool; /// Executes a breakpoint trap, for inspection by a debugger. + /// + /// This intrinsic does not have a stable counterpart. pub fn breakpoint(); /// The size of a type in bytes. /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// /// More specifically, this is the offset in bytes between successive /// items of the same type, including alignment padding. /// - /// The stabilized version of this intrinsic is - /// [`std::mem::size_of`](../../std/mem/fn.size_of.html). + /// The stabilized version of this intrinsic is [`core::mem::size_of`]. #[rustc_const_stable(feature = "const_size_of", since = "1.40.0")] + #[rustc_safe_intrinsic] pub fn size_of() -> usize; - /// Moves a value to an uninitialized memory location. - /// - /// Drop glue is not run on the destination. - /// - /// The stabilized version of this intrinsic is - /// [`std::ptr::write`](../../std/ptr/fn.write.html). - pub fn move_val_init(dst: *mut T, src: T); - /// The minimum alignment of a type. /// - /// The stabilized version of this intrinsic is - /// [`std::mem::align_of`](../../std/mem/fn.align_of.html). + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// + /// The stabilized version of this intrinsic is [`core::mem::align_of`]. #[rustc_const_stable(feature = "const_min_align_of", since = "1.40.0")] + #[rustc_safe_intrinsic] pub fn min_align_of() -> usize; - #[rustc_const_unstable(feature = "const_pref_align_of", issue = "none")] + /// The preferred alignment of a type. + /// + /// This intrinsic does not have a stable counterpart. + /// It's "tracking issue" is [#91971](https://github.com/rust-lang/rust/issues/91971). + #[rustc_const_unstable(feature = "const_pref_align_of", issue = "91971")] pub fn pref_align_of() -> usize; /// The size of the referenced value in bytes. /// - /// The stabilized version of this intrinsic is - /// [`std::mem::size_of_val`](../../std/mem/fn.size_of_val.html). - pub fn size_of_val(_: &T) -> usize; - /// The minimum alignment of the type of the value that `val` points to. + /// The stabilized version of this intrinsic is [`mem::size_of_val`]. + #[rustc_const_unstable(feature = "const_size_of_val", issue = "46571")] + pub fn size_of_val(_: *const T) -> usize; + /// The required alignment of the referenced value. /// - /// The stabilized version of this intrinsic is - /// [`std::mem::min_align_of_val`](../../std/mem/fn.min_align_of_val.html). - pub fn min_align_of_val(_: &T) -> usize; + /// The stabilized version of this intrinsic is [`core::mem::align_of_val`]. + #[rustc_const_unstable(feature = "const_align_of_val", issue = "46571")] + pub fn min_align_of_val(_: *const T) -> usize; /// Gets a static string slice containing the name of a type. /// - /// The stabilized version of this intrinsic is - /// [`std::any::type_name`](../../std/any/fn.type_name.html) - #[rustc_const_unstable(feature = "const_type_name", issue = "none")] + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// + /// The stabilized version of this intrinsic is [`core::any::type_name`]. + #[rustc_const_unstable(feature = "const_type_name", issue = "63084")] + #[rustc_safe_intrinsic] pub fn type_name() -> &'static str; /// Gets an identifier which is globally unique to the specified type. This /// function will return the same value for a type regardless of whichever /// crate it is invoked in. /// - /// The stabilized version of this intrinsic is - /// [`std::any::TypeId::of`](../../std/any/struct.TypeId.html#method.of) - #[rustc_const_unstable(feature = "const_type_id", issue = "none")] + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// + /// The stabilized version of this intrinsic is [`core::any::TypeId::of`]. + #[rustc_const_unstable(feature = "const_type_id", issue = "77125")] + #[rustc_safe_intrinsic] pub fn type_id() -> u64; /// A guard for unsafe functions that cannot ever be executed if `T` is uninhabited: /// This will statically either panic, or do nothing. - #[cfg(bootstrap)] - pub fn panic_if_uninhabited(); - - /// A guard for unsafe functions that cannot ever be executed if `T` is uninhabited: - /// This will statically either panic, or do nothing. - #[cfg(not(bootstrap))] + /// + /// This intrinsic does not have a stable counterpart. + #[rustc_const_stable(feature = "const_assert_type", since = "1.59.0")] + #[rustc_safe_intrinsic] pub fn assert_inhabited(); /// A guard for unsafe functions that cannot ever be executed if `T` does not permit /// zero-initialization: This will statically either panic, or do nothing. - #[cfg(not(bootstrap))] + /// + /// This intrinsic does not have a stable counterpart. + #[rustc_const_unstable(feature = "const_assert_type2", issue = "none")] + #[rustc_safe_intrinsic] pub fn assert_zero_valid(); - /// A guard for unsafe functions that cannot ever be executed if `T` has invalid - /// bit patterns: This will statically either panic, or do nothing. + /// A guard for `std::mem::uninitialized`. This will statically either panic, or do nothing. + /// + /// This intrinsic does not have a stable counterpart. + #[rustc_const_unstable(feature = "const_assert_type2", issue = "none")] + #[rustc_safe_intrinsic] #[cfg(not(bootstrap))] - pub fn assert_uninit_valid(); + pub fn assert_mem_uninitialized_valid(); /// Gets a reference to a static `Location` indicating where it was called. - #[rustc_const_unstable(feature = "const_caller_location", issue = "47809")] + /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// + /// Consider using [`core::panic::Location::caller`] instead. + #[rustc_const_unstable(feature = "const_caller_location", issue = "76156")] + #[rustc_safe_intrinsic] pub fn caller_location() -> &'static crate::panic::Location<'static>; /// Moves a value out of scope without running drop glue. - /// This exists solely for `mem::forget_unsized`; normal `forget` uses `ManuallyDrop` instead. + /// + /// This exists solely for [`mem::forget_unsized`]; normal `forget` uses + /// `ManuallyDrop` instead. + /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + #[rustc_const_unstable(feature = "const_intrinsic_forget", issue = "none")] + #[rustc_safe_intrinsic] pub fn forget(_: T); /// Reinterprets the bits of a value of one type as another type. /// - /// Both types must have the same size. Neither the original, nor the result, - /// may be an [invalid value](../../nomicon/what-unsafe-does.html). + /// Both types must have the same size. Compilation will fail if this is not guaranteed. /// /// `transmute` is semantically equivalent to a bitwise move of one type /// into another. It copies the bits from the source value into the - /// destination value, then forgets the original. It's equivalent to C's - /// `memcpy` under the hood, just like `transmute_copy`. - /// - /// `transmute` is **incredibly** unsafe. There are a vast number of ways to - /// cause [undefined behavior][ub] with this function. `transmute` should be - /// the absolute last resort. - /// - /// The [nomicon](../../nomicon/transmutes.html) has additional - /// documentation. + /// destination value, then forgets the original. Note that source and destination + /// are passed by-value, which means if `Src` or `Dst` contain padding, that padding + /// is *not* guaranteed to be preserved by `transmute`. + /// + /// Both the argument and the result must be [valid](../../nomicon/what-unsafe-does.html) at + /// their given type. Violating this condition leads to [undefined behavior][ub]. The compiler + /// will generate code *assuming that you, the programmer, ensure that there will never be + /// undefined behavior*. It is therefore your responsibility to guarantee that every value + /// passed to `transmute` is valid at both types `Src` and `Dst`. Failing to uphold this condition + /// may lead to unexpected and unstable compilation results. This makes `transmute` **incredibly + /// unsafe**. `transmute` should be the absolute last resort. + /// + /// Transmuting pointers to integers in a `const` context is [undefined behavior][ub]. + /// Any attempt to use the resulting value for integer operations will abort const-evaluation. + /// (And even outside `const`, such transmutation is touching on many unspecified aspects of the + /// Rust memory model and should be avoided. See below for alternatives.) + /// + /// Because `transmute` is a by-value operation, alignment of the *transmuted values + /// themselves* is not a concern. As with any other function, the compiler already ensures + /// both `Src` and `Dst` are properly aligned. However, when transmuting values that *point + /// elsewhere* (such as pointers, references, boxes…), the caller has to ensure proper + /// alignment of the pointed-to values. + /// + /// The [nomicon](../../nomicon/transmutes.html) has additional documentation. /// /// [ub]: ../../reference/behavior-considered-undefined.html /// @@ -1061,6 +1035,9 @@ extern "rust-intrinsic" { /// fn foo() -> i32 { /// 0 /// } + /// // Crucially, we `as`-cast to a raw pointer before `transmute`ing to a function pointer. + /// // This avoids an integer-to-pointer `transmute`, which can be problematic. + /// // Transmuting between raw pointers and function pointers (i.e., two pointer types) is fine. /// let pointer = foo as *const (); /// let function = unsafe { /// std::mem::transmute::<*const (), fn() -> i32>(pointer) @@ -1089,9 +1066,27 @@ extern "rust-intrinsic" { /// Below are common applications of `transmute` which can be replaced with safer /// constructs. /// - /// Turning a pointer into a `usize`: + /// Turning raw bytes (`&[u8]`) into `u32`, `f64`, etc.: + /// + /// ``` + /// let raw_bytes = [0x78, 0x56, 0x34, 0x12]; + /// + /// let num = unsafe { + /// std::mem::transmute::<[u8; 4], u32>(raw_bytes) + /// }; /// + /// // use `u32::from_ne_bytes` instead + /// let num = u32::from_ne_bytes(raw_bytes); + /// // or use `u32::from_le_bytes` or `u32::from_be_bytes` to specify the endianness + /// let num = u32::from_le_bytes(raw_bytes); + /// assert_eq!(num, 0x12345678); + /// let num = u32::from_be_bytes(raw_bytes); + /// assert_eq!(num, 0x78563412); /// ``` + /// + /// Turning a pointer into a `usize`: + /// + /// ```no_run /// let ptr = &0; /// let ptr_num_transmute = unsafe { /// std::mem::transmute::<&i32, usize>(ptr) @@ -1101,6 +1096,16 @@ extern "rust-intrinsic" { /// let ptr_num_cast = ptr as *const i32 as usize; /// ``` /// + /// Note that using `transmute` to turn a pointer to a `usize` is (as noted above) [undefined + /// behavior][ub] in `const` contexts. Also outside of consts, this operation might not behave + /// as expected -- this is touching on many unspecified aspects of the Rust memory model. + /// Depending on what the code is doing, the following alternatives are preferable to + /// pointer-to-integer transmutation: + /// - If the code just wants to store data of arbitrary type in some buffer and needs to pick a + /// type for that buffer, it can use [`MaybeUninit`][mem::MaybeUninit]. + /// - If the code actually wants to work on the address the pointer points to, it can use `as` + /// casts or [`ptr.addr()`][pointer::addr]. + /// /// Turning a `*mut T` into an `&mut T`: /// /// ``` @@ -1126,7 +1131,7 @@ extern "rust-intrinsic" { /// let val_casts = unsafe { &mut *(ptr as *mut i32 as *mut u32) }; /// ``` /// - /// Turning an `&str` into an `&[u8]`: + /// Turning an `&str` into a `&[u8]`: /// /// ``` /// // this is not a good way to do this. @@ -1142,7 +1147,13 @@ extern "rust-intrinsic" { /// assert_eq!(b"Rust", &[82, 117, 115, 116]); /// ``` /// - /// Turning a `Vec<&T>` into a `Vec>`: + /// Turning a `Vec<&T>` into a `Vec>`. + /// + /// To transmute the inner type of the contents of a container, you must make sure to not + /// violate any of the container's invariants. For `Vec`, this means that both the size + /// *and alignment* of the inner types have to match. Other containers might rely on the + /// size of the type, alignment, or even the `TypeId`, in which case transmuting wouldn't + /// be possible at all without violating the container invariants. /// /// ``` /// let store = [0, 1, 2, 3]; @@ -1168,14 +1179,11 @@ extern "rust-intrinsic" { /// /// let v_clone = v_orig.clone(); /// - /// // The no-copy, unsafe way, still using transmute, but not relying on the data layout. - /// // Like the first approach, this reuses the `Vec` internals. - /// // Therefore, the new inner type must have the - /// // exact same size, *and the same alignment*, as the old type. - /// // The same caveats exist for this method as transmute, for - /// // the original inner type (`&i32`) to the converted inner type - /// // (`Option<&i32>`), so read the nomicon pages linked above and also - /// // consult the [`from_raw_parts`] documentation. + /// // This is the proper no-copy, unsafe way of "transmuting" a `Vec`, without relying on the + /// // data layout. Instead of literally calling `transmute`, we perform a pointer cast, but + /// // in terms of converting the original inner type (`&i32`) to the new one (`Option<&i32>`), + /// // this has all the same caveats. Besides the information provided above, also consult the + /// // [`from_raw_parts`] documentation. /// let v_from_raw = unsafe { // FIXME Update this when vec_into_raw_parts is stabilized /// // Ensure the original vector is not dropped. @@ -1201,14 +1209,14 @@ extern "rust-intrinsic" { /// assert!(mid <= len); /// unsafe { /// let slice2 = mem::transmute::<&mut [T], &mut [T]>(slice); - /// // first: transmute is not typesafe; all it checks is that T and + /// // first: transmute is not type safe; all it checks is that T and /// // U are of the same size. Second, right here, you have two /// // mutable references pointing to the same memory. /// (&mut slice[0..mid], &mut slice2[mid..len]) /// } /// } /// - /// // This gets rid of the typesafety problems; `&mut *` will *only* give + /// // This gets rid of the type safety problems; `&mut *` will *only* give /// // you an `&mut T` from an `&mut T` or `*mut T`. /// fn split_at_mut_casts(slice: &mut [T], mid: usize) /// -> (&mut [T], &mut [T]) { @@ -1241,20 +1249,27 @@ extern "rust-intrinsic" { /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_transmute", issue = "53605")] - pub fn transmute(e: T) -> U; + #[rustc_allowed_through_unstable_modules] + #[rustc_const_stable(feature = "const_transmute", since = "1.56.0")] + #[rustc_diagnostic_item = "transmute"] + pub fn transmute(src: Src) -> Dst; /// Returns `true` if the actual type given as `T` requires drop /// glue; returns `false` if the actual type provided for `T` /// implements `Copy`. /// /// If the actual type neither requires drop glue nor implements - /// `Copy`, then may return `true` or `false`. + /// `Copy`, then the return value of this function is unspecified. /// - /// The stabilized version of this intrinsic is - /// [`std::mem::needs_drop`](../../std/mem/fn.needs_drop.html). + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// + /// The stabilized version of this intrinsic is [`mem::needs_drop`](crate::mem::needs_drop). #[rustc_const_stable(feature = "const_needs_drop", since = "1.40.0")] - pub fn needs_drop() -> bool; + #[rustc_safe_intrinsic] + pub fn needs_drop() -> bool; /// Calculates the offset from a pointer. /// @@ -1268,8 +1283,9 @@ extern "rust-intrinsic" { /// bounds or arithmetic overflow occurs then any further use of the /// returned value will result in undefined behavior. /// - /// The stabilized version of this intrinsic is - /// [`std::pointer::offset`](../../std/primitive.pointer.html#method.offset). + /// The stabilized version of this intrinsic is [`pointer::offset`]. + #[must_use = "returns a new pointer rather than modifying its argument"] + #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] pub fn offset(dst: *const T, offset: isize) -> *const T; /// Calculates the offset from a pointer, potentially wrapping. @@ -1284,245 +1300,289 @@ extern "rust-intrinsic" { /// object, and it wraps with two's complement arithmetic. The resulting /// value is not necessarily valid to be used to actually access memory. /// - /// The stabilized version of this intrinsic is - /// [`std::pointer::wrapping_offset`](../../std/primitive.pointer.html#method.wrapping_offset). + /// The stabilized version of this intrinsic is [`pointer::wrapping_offset`]. + #[must_use = "returns a new pointer rather than modifying its argument"] + #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] pub fn arith_offset(dst: *const T, offset: isize) -> *const T; + /// Masks out bits of the pointer according to a mask. + /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// + /// Consider using [`pointer::mask`] instead. + #[rustc_safe_intrinsic] + pub fn ptr_mask(ptr: *const T, mask: usize) -> *const T; + /// Equivalent to the appropriate `llvm.memcpy.p0i8.0i8.*` intrinsic, with /// a size of `count` * `size_of::()` and an alignment of /// `min_align_of::()` /// /// The volatile parameter is set to `true`, so it will not be optimized out /// unless size is equal to zero. + /// + /// This intrinsic does not have a stable counterpart. pub fn volatile_copy_nonoverlapping_memory(dst: *mut T, src: *const T, count: usize); /// Equivalent to the appropriate `llvm.memmove.p0i8.0i8.*` intrinsic, with - /// a size of `count` * `size_of::()` and an alignment of + /// a size of `count * size_of::()` and an alignment of /// `min_align_of::()` /// /// The volatile parameter is set to `true`, so it will not be optimized out /// unless size is equal to zero. + /// + /// This intrinsic does not have a stable counterpart. pub fn volatile_copy_memory(dst: *mut T, src: *const T, count: usize); /// Equivalent to the appropriate `llvm.memset.p0i8.*` intrinsic, with a - /// size of `count` * `size_of::()` and an alignment of + /// size of `count * size_of::()` and an alignment of /// `min_align_of::()`. /// /// The volatile parameter is set to `true`, so it will not be optimized out /// unless size is equal to zero. + /// + /// This intrinsic does not have a stable counterpart. pub fn volatile_set_memory(dst: *mut T, val: u8, count: usize); /// Performs a volatile load from the `src` pointer. /// - /// The stabilized version of this intrinsic is - /// [`std::ptr::read_volatile`](../../std/ptr/fn.read_volatile.html). + /// The stabilized version of this intrinsic is [`core::ptr::read_volatile`]. pub fn volatile_load(src: *const T) -> T; /// Performs a volatile store to the `dst` pointer. /// - /// The stabilized version of this intrinsic is - /// [`std::ptr::write_volatile`](../../std/ptr/fn.write_volatile.html). + /// The stabilized version of this intrinsic is [`core::ptr::write_volatile`]. pub fn volatile_store(dst: *mut T, val: T); /// Performs a volatile load from the `src` pointer /// The pointer is not required to be aligned. + /// + /// This intrinsic does not have a stable counterpart. pub fn unaligned_volatile_load(src: *const T) -> T; /// Performs a volatile store to the `dst` pointer. /// The pointer is not required to be aligned. + /// + /// This intrinsic does not have a stable counterpart. pub fn unaligned_volatile_store(dst: *mut T, val: T); /// Returns the square root of an `f32` /// /// The stabilized version of this intrinsic is - /// [`std::f32::sqrt`](../../std/primitive.f32.html#method.sqrt) + /// [`f32::sqrt`](../../std/primitive.f32.html#method.sqrt) pub fn sqrtf32(x: f32) -> f32; /// Returns the square root of an `f64` /// /// The stabilized version of this intrinsic is - /// [`std::f64::sqrt`](../../std/primitive.f64.html#method.sqrt) + /// [`f64::sqrt`](../../std/primitive.f64.html#method.sqrt) pub fn sqrtf64(x: f64) -> f64; /// Raises an `f32` to an integer power. /// /// The stabilized version of this intrinsic is - /// [`std::f32::powi`](../../std/primitive.f32.html#method.powi) + /// [`f32::powi`](../../std/primitive.f32.html#method.powi) pub fn powif32(a: f32, x: i32) -> f32; /// Raises an `f64` to an integer power. /// /// The stabilized version of this intrinsic is - /// [`std::f64::powi`](../../std/primitive.f64.html#method.powi) + /// [`f64::powi`](../../std/primitive.f64.html#method.powi) pub fn powif64(a: f64, x: i32) -> f64; /// Returns the sine of an `f32`. /// /// The stabilized version of this intrinsic is - /// [`std::f32::sin`](../../std/primitive.f32.html#method.sin) + /// [`f32::sin`](../../std/primitive.f32.html#method.sin) pub fn sinf32(x: f32) -> f32; /// Returns the sine of an `f64`. /// /// The stabilized version of this intrinsic is - /// [`std::f64::sin`](../../std/primitive.f64.html#method.sin) + /// [`f64::sin`](../../std/primitive.f64.html#method.sin) pub fn sinf64(x: f64) -> f64; /// Returns the cosine of an `f32`. /// /// The stabilized version of this intrinsic is - /// [`std::f32::cos`](../../std/primitive.f32.html#method.cos) + /// [`f32::cos`](../../std/primitive.f32.html#method.cos) pub fn cosf32(x: f32) -> f32; /// Returns the cosine of an `f64`. /// /// The stabilized version of this intrinsic is - /// [`std::f64::cos`](../../std/primitive.f64.html#method.cos) + /// [`f64::cos`](../../std/primitive.f64.html#method.cos) pub fn cosf64(x: f64) -> f64; /// Raises an `f32` to an `f32` power. /// /// The stabilized version of this intrinsic is - /// [`std::f32::powf`](../../std/primitive.f32.html#method.powf) + /// [`f32::powf`](../../std/primitive.f32.html#method.powf) pub fn powf32(a: f32, x: f32) -> f32; /// Raises an `f64` to an `f64` power. /// /// The stabilized version of this intrinsic is - /// [`std::f64::powf`](../../std/primitive.f64.html#method.powf) + /// [`f64::powf`](../../std/primitive.f64.html#method.powf) pub fn powf64(a: f64, x: f64) -> f64; /// Returns the exponential of an `f32`. /// /// The stabilized version of this intrinsic is - /// [`std::f32::exp`](../../std/primitive.f32.html#method.exp) + /// [`f32::exp`](../../std/primitive.f32.html#method.exp) pub fn expf32(x: f32) -> f32; /// Returns the exponential of an `f64`. /// /// The stabilized version of this intrinsic is - /// [`std::f64::exp`](../../std/primitive.f64.html#method.exp) + /// [`f64::exp`](../../std/primitive.f64.html#method.exp) pub fn expf64(x: f64) -> f64; /// Returns 2 raised to the power of an `f32`. /// /// The stabilized version of this intrinsic is - /// [`std::f32::exp2`](../../std/primitive.f32.html#method.exp2) + /// [`f32::exp2`](../../std/primitive.f32.html#method.exp2) pub fn exp2f32(x: f32) -> f32; /// Returns 2 raised to the power of an `f64`. /// /// The stabilized version of this intrinsic is - /// [`std::f64::exp2`](../../std/primitive.f64.html#method.exp2) + /// [`f64::exp2`](../../std/primitive.f64.html#method.exp2) pub fn exp2f64(x: f64) -> f64; /// Returns the natural logarithm of an `f32`. /// /// The stabilized version of this intrinsic is - /// [`std::f32::ln`](../../std/primitive.f32.html#method.ln) + /// [`f32::ln`](../../std/primitive.f32.html#method.ln) pub fn logf32(x: f32) -> f32; /// Returns the natural logarithm of an `f64`. /// /// The stabilized version of this intrinsic is - /// [`std::f64::ln`](../../std/primitive.f64.html#method.ln) + /// [`f64::ln`](../../std/primitive.f64.html#method.ln) pub fn logf64(x: f64) -> f64; /// Returns the base 10 logarithm of an `f32`. /// /// The stabilized version of this intrinsic is - /// [`std::f32::log10`](../../std/primitive.f32.html#method.log10) + /// [`f32::log10`](../../std/primitive.f32.html#method.log10) pub fn log10f32(x: f32) -> f32; /// Returns the base 10 logarithm of an `f64`. /// /// The stabilized version of this intrinsic is - /// [`std::f64::log10`](../../std/primitive.f64.html#method.log10) + /// [`f64::log10`](../../std/primitive.f64.html#method.log10) pub fn log10f64(x: f64) -> f64; /// Returns the base 2 logarithm of an `f32`. /// /// The stabilized version of this intrinsic is - /// [`std::f32::log2`](../../std/primitive.f32.html#method.log2) + /// [`f32::log2`](../../std/primitive.f32.html#method.log2) pub fn log2f32(x: f32) -> f32; /// Returns the base 2 logarithm of an `f64`. /// /// The stabilized version of this intrinsic is - /// [`std::f64::log2`](../../std/primitive.f64.html#method.log2) + /// [`f64::log2`](../../std/primitive.f64.html#method.log2) pub fn log2f64(x: f64) -> f64; /// Returns `a * b + c` for `f32` values. /// /// The stabilized version of this intrinsic is - /// [`std::f32::mul_add`](../../std/primitive.f32.html#method.mul_add) + /// [`f32::mul_add`](../../std/primitive.f32.html#method.mul_add) pub fn fmaf32(a: f32, b: f32, c: f32) -> f32; /// Returns `a * b + c` for `f64` values. /// /// The stabilized version of this intrinsic is - /// [`std::f64::mul_add`](../../std/primitive.f64.html#method.mul_add) + /// [`f64::mul_add`](../../std/primitive.f64.html#method.mul_add) pub fn fmaf64(a: f64, b: f64, c: f64) -> f64; /// Returns the absolute value of an `f32`. /// /// The stabilized version of this intrinsic is - /// [`std::f32::abs`](../../std/primitive.f32.html#method.abs) + /// [`f32::abs`](../../std/primitive.f32.html#method.abs) pub fn fabsf32(x: f32) -> f32; /// Returns the absolute value of an `f64`. /// /// The stabilized version of this intrinsic is - /// [`std::f64::abs`](../../std/primitive.f64.html#method.abs) + /// [`f64::abs`](../../std/primitive.f64.html#method.abs) pub fn fabsf64(x: f64) -> f64; /// Returns the minimum of two `f32` values. /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// /// The stabilized version of this intrinsic is - /// [`std::f32::min`](../../std/primitive.f32.html#method.min) + /// [`f32::min`] + #[rustc_safe_intrinsic] pub fn minnumf32(x: f32, y: f32) -> f32; /// Returns the minimum of two `f64` values. /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// /// The stabilized version of this intrinsic is - /// [`std::f64::min`](../../std/primitive.f64.html#method.min) + /// [`f64::min`] + #[rustc_safe_intrinsic] pub fn minnumf64(x: f64, y: f64) -> f64; /// Returns the maximum of two `f32` values. /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// /// The stabilized version of this intrinsic is - /// [`std::f32::max`](../../std/primitive.f32.html#method.max) + /// [`f32::max`] + #[rustc_safe_intrinsic] pub fn maxnumf32(x: f32, y: f32) -> f32; /// Returns the maximum of two `f64` values. /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// /// The stabilized version of this intrinsic is - /// [`std::f64::max`](../../std/primitive.f64.html#method.max) + /// [`f64::max`] + #[rustc_safe_intrinsic] pub fn maxnumf64(x: f64, y: f64) -> f64; /// Copies the sign from `y` to `x` for `f32` values. /// /// The stabilized version of this intrinsic is - /// [`std::f32::copysign`](../../std/primitive.f32.html#method.copysign) + /// [`f32::copysign`](../../std/primitive.f32.html#method.copysign) pub fn copysignf32(x: f32, y: f32) -> f32; /// Copies the sign from `y` to `x` for `f64` values. /// /// The stabilized version of this intrinsic is - /// [`std::f64::copysign`](../../std/primitive.f64.html#method.copysign) + /// [`f64::copysign`](../../std/primitive.f64.html#method.copysign) pub fn copysignf64(x: f64, y: f64) -> f64; /// Returns the largest integer less than or equal to an `f32`. /// /// The stabilized version of this intrinsic is - /// [`std::f32::floor`](../../std/primitive.f32.html#method.floor) + /// [`f32::floor`](../../std/primitive.f32.html#method.floor) pub fn floorf32(x: f32) -> f32; /// Returns the largest integer less than or equal to an `f64`. /// /// The stabilized version of this intrinsic is - /// [`std::f64::floor`](../../std/primitive.f64.html#method.floor) + /// [`f64::floor`](../../std/primitive.f64.html#method.floor) pub fn floorf64(x: f64) -> f64; /// Returns the smallest integer greater than or equal to an `f32`. /// /// The stabilized version of this intrinsic is - /// [`std::f32::ceil`](../../std/primitive.f32.html#method.ceil) + /// [`f32::ceil`](../../std/primitive.f32.html#method.ceil) pub fn ceilf32(x: f32) -> f32; /// Returns the smallest integer greater than or equal to an `f64`. /// /// The stabilized version of this intrinsic is - /// [`std::f64::ceil`](../../std/primitive.f64.html#method.ceil) + /// [`f64::ceil`](../../std/primitive.f64.html#method.ceil) pub fn ceilf64(x: f64) -> f64; /// Returns the integer part of an `f32`. /// /// The stabilized version of this intrinsic is - /// [`std::f32::trunc`](../../std/primitive.f32.html#method.trunc) + /// [`f32::trunc`](../../std/primitive.f32.html#method.trunc) pub fn truncf32(x: f32) -> f32; /// Returns the integer part of an `f64`. /// /// The stabilized version of this intrinsic is - /// [`std::f64::trunc`](../../std/primitive.f64.html#method.trunc) + /// [`f64::trunc`](../../std/primitive.f64.html#method.trunc) pub fn truncf64(x: f64) -> f64; /// Returns the nearest integer to an `f32`. May raise an inexact floating-point exception @@ -1533,59 +1593,85 @@ extern "rust-intrinsic" { pub fn rintf64(x: f64) -> f64; /// Returns the nearest integer to an `f32`. + /// + /// This intrinsic does not have a stable counterpart. pub fn nearbyintf32(x: f32) -> f32; /// Returns the nearest integer to an `f64`. + /// + /// This intrinsic does not have a stable counterpart. pub fn nearbyintf64(x: f64) -> f64; /// Returns the nearest integer to an `f32`. Rounds half-way cases away from zero. /// /// The stabilized version of this intrinsic is - /// [`std::f32::round`](../../std/primitive.f32.html#method.round) + /// [`f32::round`](../../std/primitive.f32.html#method.round) pub fn roundf32(x: f32) -> f32; /// Returns the nearest integer to an `f64`. Rounds half-way cases away from zero. /// /// The stabilized version of this intrinsic is - /// [`std::f64::round`](../../std/primitive.f64.html#method.round) + /// [`f64::round`](../../std/primitive.f64.html#method.round) pub fn roundf64(x: f64) -> f64; /// Float addition that allows optimizations based on algebraic rules. /// May assume inputs are finite. - pub fn fadd_fast(a: T, b: T) -> T; + /// + /// This intrinsic does not have a stable counterpart. + pub fn fadd_fast(a: T, b: T) -> T; /// Float subtraction that allows optimizations based on algebraic rules. /// May assume inputs are finite. - pub fn fsub_fast(a: T, b: T) -> T; + /// + /// This intrinsic does not have a stable counterpart. + pub fn fsub_fast(a: T, b: T) -> T; /// Float multiplication that allows optimizations based on algebraic rules. /// May assume inputs are finite. - pub fn fmul_fast(a: T, b: T) -> T; + /// + /// This intrinsic does not have a stable counterpart. + pub fn fmul_fast(a: T, b: T) -> T; /// Float division that allows optimizations based on algebraic rules. /// May assume inputs are finite. - pub fn fdiv_fast(a: T, b: T) -> T; + /// + /// This intrinsic does not have a stable counterpart. + pub fn fdiv_fast(a: T, b: T) -> T; /// Float remainder that allows optimizations based on algebraic rules. /// May assume inputs are finite. - pub fn frem_fast(a: T, b: T) -> T; + /// + /// This intrinsic does not have a stable counterpart. + pub fn frem_fast(a: T, b: T) -> T; /// Convert with LLVM’s fptoui/fptosi, which may return undef for values out of range /// () - /// This is under stabilization at - pub fn float_to_int_approx_unchecked(value: Float) -> Int; + /// + /// Stabilized as [`f32::to_int_unchecked`] and [`f64::to_int_unchecked`]. + pub fn float_to_int_unchecked(value: Float) -> Int; /// Returns the number of bits set in an integer type `T` /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// /// The stabilized versions of this intrinsic are available on the integer /// primitives via the `count_ones` method. For example, - /// [`std::u32::count_ones`](../../std/primitive.u32.html#method.count_ones) + /// [`u32::count_ones`] #[rustc_const_stable(feature = "const_ctpop", since = "1.40.0")] - pub fn ctpop(x: T) -> T; + #[rustc_safe_intrinsic] + pub fn ctpop(x: T) -> T; /// Returns the number of leading unset bits (zeroes) in an integer type `T`. /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// /// The stabilized versions of this intrinsic are available on the integer /// primitives via the `leading_zeros` method. For example, - /// [`std::u32::leading_zeros`](../../std/primitive.u32.html#method.leading_zeros) + /// [`u32::leading_zeros`] /// /// # Examples /// @@ -1611,11 +1697,14 @@ extern "rust-intrinsic" { /// assert_eq!(num_leading, 16); /// ``` #[rustc_const_stable(feature = "const_ctlz", since = "1.40.0")] - pub fn ctlz(x: T) -> T; + #[rustc_safe_intrinsic] + pub fn ctlz(x: T) -> T; /// Like `ctlz`, but extra-unsafe as it returns `undef` when /// given an `x` with value `0`. /// + /// This intrinsic does not have a stable counterpart. + /// /// # Examples /// /// ``` @@ -1627,14 +1716,19 @@ extern "rust-intrinsic" { /// let num_leading = unsafe { ctlz_nonzero(x) }; /// assert_eq!(num_leading, 3); /// ``` - #[rustc_const_unstable(feature = "constctlz", issue = "none")] - pub fn ctlz_nonzero(x: T) -> T; + #[rustc_const_stable(feature = "constctlz", since = "1.50.0")] + pub fn ctlz_nonzero(x: T) -> T; /// Returns the number of trailing unset bits (zeroes) in an integer type `T`. /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// /// The stabilized versions of this intrinsic are available on the integer /// primitives via the `trailing_zeros` method. For example, - /// [`std::u32::trailing_zeros`](../../std/primitive.u32.html#method.trailing_zeros) + /// [`u32::trailing_zeros`] /// /// # Examples /// @@ -1660,11 +1754,14 @@ extern "rust-intrinsic" { /// assert_eq!(num_trailing, 16); /// ``` #[rustc_const_stable(feature = "const_cttz", since = "1.40.0")] - pub fn cttz(x: T) -> T; + #[rustc_safe_intrinsic] + pub fn cttz(x: T) -> T; /// Like `cttz`, but extra-unsafe as it returns `undef` when /// given an `x` with value `0`. /// + /// This intrinsic does not have a stable counterpart. + /// /// # Examples /// /// ``` @@ -1676,162 +1773,261 @@ extern "rust-intrinsic" { /// let num_trailing = unsafe { cttz_nonzero(x) }; /// assert_eq!(num_trailing, 3); /// ``` - #[rustc_const_unstable(feature = "const_cttz", issue = "none")] - pub fn cttz_nonzero(x: T) -> T; + #[rustc_const_stable(feature = "const_cttz_nonzero", since = "1.53.0")] + pub fn cttz_nonzero(x: T) -> T; /// Reverses the bytes in an integer type `T`. /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// /// The stabilized versions of this intrinsic are available on the integer /// primitives via the `swap_bytes` method. For example, - /// [`std::u32::swap_bytes`](../../std/primitive.u32.html#method.swap_bytes) + /// [`u32::swap_bytes`] #[rustc_const_stable(feature = "const_bswap", since = "1.40.0")] - pub fn bswap(x: T) -> T; + #[rustc_safe_intrinsic] + pub fn bswap(x: T) -> T; /// Reverses the bits in an integer type `T`. /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// /// The stabilized versions of this intrinsic are available on the integer /// primitives via the `reverse_bits` method. For example, - /// [`std::u32::reverse_bits`](../../std/primitive.u32.html#method.reverse_bits) + /// [`u32::reverse_bits`] #[rustc_const_stable(feature = "const_bitreverse", since = "1.40.0")] - pub fn bitreverse(x: T) -> T; + #[rustc_safe_intrinsic] + pub fn bitreverse(x: T) -> T; /// Performs checked integer addition. /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// /// The stabilized versions of this intrinsic are available on the integer /// primitives via the `overflowing_add` method. For example, - /// [`std::u32::overflowing_add`](../../std/primitive.u32.html#method.overflowing_add) + /// [`u32::overflowing_add`] #[rustc_const_stable(feature = "const_int_overflow", since = "1.40.0")] - pub fn add_with_overflow(x: T, y: T) -> (T, bool); + #[rustc_safe_intrinsic] + pub fn add_with_overflow(x: T, y: T) -> (T, bool); /// Performs checked integer subtraction /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// /// The stabilized versions of this intrinsic are available on the integer /// primitives via the `overflowing_sub` method. For example, - /// [`std::u32::overflowing_sub`](../../std/primitive.u32.html#method.overflowing_sub) + /// [`u32::overflowing_sub`] #[rustc_const_stable(feature = "const_int_overflow", since = "1.40.0")] - pub fn sub_with_overflow(x: T, y: T) -> (T, bool); + #[rustc_safe_intrinsic] + pub fn sub_with_overflow(x: T, y: T) -> (T, bool); /// Performs checked integer multiplication /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// /// The stabilized versions of this intrinsic are available on the integer /// primitives via the `overflowing_mul` method. For example, - /// [`std::u32::overflowing_mul`](../../std/primitive.u32.html#method.overflowing_mul) + /// [`u32::overflowing_mul`] #[rustc_const_stable(feature = "const_int_overflow", since = "1.40.0")] - pub fn mul_with_overflow(x: T, y: T) -> (T, bool); + #[rustc_safe_intrinsic] + pub fn mul_with_overflow(x: T, y: T) -> (T, bool); /// Performs an exact division, resulting in undefined behavior where - /// `x % y != 0` or `y == 0` or `x == T::min_value() && y == -1` - pub fn exact_div(x: T, y: T) -> T; + /// `x % y != 0` or `y == 0` or `x == T::MIN && y == -1` + /// + /// This intrinsic does not have a stable counterpart. + #[rustc_const_unstable(feature = "const_exact_div", issue = "none")] + pub fn exact_div(x: T, y: T) -> T; /// Performs an unchecked division, resulting in undefined behavior - /// where y = 0 or x = `T::min_value()` and y = -1 + /// where `y == 0` or `x == T::MIN && y == -1` /// - /// The stabilized versions of this intrinsic are available on the integer + /// Safe wrappers for this intrinsic are available on the integer /// primitives via the `checked_div` method. For example, - /// [`std::u32::checked_div`](../../std/primitive.u32.html#method.checked_div) - #[rustc_const_unstable(feature = "const_int_unchecked_arith", issue = "none")] - pub fn unchecked_div(x: T, y: T) -> T; + /// [`u32::checked_div`] + #[rustc_const_stable(feature = "const_int_unchecked_div", since = "1.52.0")] + pub fn unchecked_div(x: T, y: T) -> T; /// Returns the remainder of an unchecked division, resulting in - /// undefined behavior where y = 0 or x = `T::min_value()` and y = -1 + /// undefined behavior when `y == 0` or `x == T::MIN && y == -1` /// - /// The stabilized versions of this intrinsic are available on the integer + /// Safe wrappers for this intrinsic are available on the integer /// primitives via the `checked_rem` method. For example, - /// [`std::u32::checked_rem`](../../std/primitive.u32.html#method.checked_rem) - #[rustc_const_unstable(feature = "const_int_unchecked_arith", issue = "none")] - pub fn unchecked_rem(x: T, y: T) -> T; + /// [`u32::checked_rem`] + #[rustc_const_stable(feature = "const_int_unchecked_rem", since = "1.52.0")] + pub fn unchecked_rem(x: T, y: T) -> T; /// Performs an unchecked left shift, resulting in undefined behavior when - /// y < 0 or y >= N, where N is the width of T in bits. + /// `y < 0` or `y >= N`, where N is the width of T in bits. /// - /// The stabilized versions of this intrinsic are available on the integer + /// Safe wrappers for this intrinsic are available on the integer /// primitives via the `checked_shl` method. For example, - /// [`std::u32::checked_shl`](../../std/primitive.u32.html#method.checked_shl) + /// [`u32::checked_shl`] #[rustc_const_stable(feature = "const_int_unchecked", since = "1.40.0")] - pub fn unchecked_shl(x: T, y: T) -> T; + pub fn unchecked_shl(x: T, y: T) -> T; /// Performs an unchecked right shift, resulting in undefined behavior when - /// y < 0 or y >= N, where N is the width of T in bits. + /// `y < 0` or `y >= N`, where N is the width of T in bits. /// - /// The stabilized versions of this intrinsic are available on the integer + /// Safe wrappers for this intrinsic are available on the integer /// primitives via the `checked_shr` method. For example, - /// [`std::u32::checked_shr`](../../std/primitive.u32.html#method.checked_shr) + /// [`u32::checked_shr`] #[rustc_const_stable(feature = "const_int_unchecked", since = "1.40.0")] - pub fn unchecked_shr(x: T, y: T) -> T; + pub fn unchecked_shr(x: T, y: T) -> T; /// Returns the result of an unchecked addition, resulting in - /// undefined behavior when `x + y > T::max_value()` or `x + y < T::min_value()`. + /// undefined behavior when `x + y > T::MAX` or `x + y < T::MIN`. + /// + /// This intrinsic does not have a stable counterpart. #[rustc_const_unstable(feature = "const_int_unchecked_arith", issue = "none")] - pub fn unchecked_add(x: T, y: T) -> T; + pub fn unchecked_add(x: T, y: T) -> T; /// Returns the result of an unchecked subtraction, resulting in - /// undefined behavior when `x - y > T::max_value()` or `x - y < T::min_value()`. + /// undefined behavior when `x - y > T::MAX` or `x - y < T::MIN`. + /// + /// This intrinsic does not have a stable counterpart. #[rustc_const_unstable(feature = "const_int_unchecked_arith", issue = "none")] - pub fn unchecked_sub(x: T, y: T) -> T; + pub fn unchecked_sub(x: T, y: T) -> T; /// Returns the result of an unchecked multiplication, resulting in - /// undefined behavior when `x * y > T::max_value()` or `x * y < T::min_value()`. + /// undefined behavior when `x * y > T::MAX` or `x * y < T::MIN`. + /// + /// This intrinsic does not have a stable counterpart. #[rustc_const_unstable(feature = "const_int_unchecked_arith", issue = "none")] - pub fn unchecked_mul(x: T, y: T) -> T; + pub fn unchecked_mul(x: T, y: T) -> T; /// Performs rotate left. /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// /// The stabilized versions of this intrinsic are available on the integer /// primitives via the `rotate_left` method. For example, - /// [`std::u32::rotate_left`](../../std/primitive.u32.html#method.rotate_left) + /// [`u32::rotate_left`] #[rustc_const_stable(feature = "const_int_rotate", since = "1.40.0")] - pub fn rotate_left(x: T, y: T) -> T; + #[rustc_safe_intrinsic] + pub fn rotate_left(x: T, y: T) -> T; /// Performs rotate right. /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// /// The stabilized versions of this intrinsic are available on the integer /// primitives via the `rotate_right` method. For example, - /// [`std::u32::rotate_right`](../../std/primitive.u32.html#method.rotate_right) + /// [`u32::rotate_right`] #[rustc_const_stable(feature = "const_int_rotate", since = "1.40.0")] - pub fn rotate_right(x: T, y: T) -> T; + #[rustc_safe_intrinsic] + pub fn rotate_right(x: T, y: T) -> T; /// Returns (a + b) mod 2N, where N is the width of T in bits. /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// /// The stabilized versions of this intrinsic are available on the integer - /// primitives via the `checked_add` method. For example, - /// [`std::u32::checked_add`](../../std/primitive.u32.html#method.checked_add) + /// primitives via the `wrapping_add` method. For example, + /// [`u32::wrapping_add`] #[rustc_const_stable(feature = "const_int_wrapping", since = "1.40.0")] - pub fn wrapping_add(a: T, b: T) -> T; + #[rustc_safe_intrinsic] + pub fn wrapping_add(a: T, b: T) -> T; /// Returns (a - b) mod 2N, where N is the width of T in bits. /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// /// The stabilized versions of this intrinsic are available on the integer - /// primitives via the `checked_sub` method. For example, - /// [`std::u32::checked_sub`](../../std/primitive.u32.html#method.checked_sub) + /// primitives via the `wrapping_sub` method. For example, + /// [`u32::wrapping_sub`] #[rustc_const_stable(feature = "const_int_wrapping", since = "1.40.0")] - pub fn wrapping_sub(a: T, b: T) -> T; + #[rustc_safe_intrinsic] + pub fn wrapping_sub(a: T, b: T) -> T; /// Returns (a * b) mod 2N, where N is the width of T in bits. /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// /// The stabilized versions of this intrinsic are available on the integer - /// primitives via the `checked_mul` method. For example, - /// [`std::u32::checked_mul`](../../std/primitive.u32.html#method.checked_mul) + /// primitives via the `wrapping_mul` method. For example, + /// [`u32::wrapping_mul`] #[rustc_const_stable(feature = "const_int_wrapping", since = "1.40.0")] - pub fn wrapping_mul(a: T, b: T) -> T; + #[rustc_safe_intrinsic] + pub fn wrapping_mul(a: T, b: T) -> T; - /// Computes `a + b`, while saturating at numeric bounds. + /// Computes `a + b`, saturating at numeric bounds. + /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. /// /// The stabilized versions of this intrinsic are available on the integer /// primitives via the `saturating_add` method. For example, - /// [`std::u32::saturating_add`](../../std/primitive.u32.html#method.saturating_add) + /// [`u32::saturating_add`] #[rustc_const_stable(feature = "const_int_saturating", since = "1.40.0")] - pub fn saturating_add(a: T, b: T) -> T; - /// Computes `a - b`, while saturating at numeric bounds. + #[rustc_safe_intrinsic] + pub fn saturating_add(a: T, b: T) -> T; + /// Computes `a - b`, saturating at numeric bounds. + /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. /// /// The stabilized versions of this intrinsic are available on the integer /// primitives via the `saturating_sub` method. For example, - /// [`std::u32::saturating_sub`](../../std/primitive.u32.html#method.saturating_sub) + /// [`u32::saturating_sub`] #[rustc_const_stable(feature = "const_int_saturating", since = "1.40.0")] - pub fn saturating_sub(a: T, b: T) -> T; + #[rustc_safe_intrinsic] + pub fn saturating_sub(a: T, b: T) -> T; - /// Returns the value of the discriminant for the variant in 'v', - /// cast to a `u64`; if `T` has no discriminant, returns 0. + /// Returns the value of the discriminant for the variant in 'v'; + /// if `T` has no discriminant, returns `0`. /// - /// The stabilized version of this intrinsic is - /// [`std::mem::discriminant`](../../std/mem/fn.discriminant.html) + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// + /// The stabilized version of this intrinsic is [`core::mem::discriminant`]. #[rustc_const_unstable(feature = "const_discriminant", issue = "69821")] - pub fn discriminant_value(v: &T) -> u64; + #[rustc_safe_intrinsic] + pub fn discriminant_value(v: &T) -> ::Discriminant; + + /// Returns the number of variants of the type `T` cast to a `usize`; + /// if `T` has no variants, returns `0`. Uninhabited variants will be counted. + /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// + /// The to-be-stabilized version of this intrinsic is [`mem::variant_count`]. + #[rustc_const_unstable(feature = "variant_count", issue = "73662")] + #[rustc_safe_intrinsic] + pub fn variant_count() -> usize; /// Rust's "try catch" construct which invokes the function pointer `try_fn` /// with the data pointer `data`. @@ -1840,26 +2036,153 @@ extern "rust-intrinsic" { /// takes the data pointer and a pointer to the target-specific exception /// object that was caught. For more information see the compiler's /// source as well as std's catch implementation. - #[cfg(not(bootstrap))] pub fn r#try(try_fn: fn(*mut u8), data: *mut u8, catch_fn: fn(*mut u8, *mut u8)) -> i32; - #[cfg(bootstrap)] - pub fn r#try(f: fn(*mut u8), data: *mut u8, local_ptr: *mut u8) -> i32; /// Emits a `!nontemporal` store according to LLVM (see their docs). /// Probably will never become stable. pub fn nontemporal_store(ptr: *mut T, val: T); /// See documentation of `<*const T>::offset_from` for details. - #[rustc_const_unstable(feature = "const_ptr_offset_from", issue = "none")] + #[rustc_const_stable(feature = "const_ptr_offset_from", since = "1.65.0")] pub fn ptr_offset_from(ptr: *const T, base: *const T) -> isize; - /// Internal hook used by Miri to implement unwinding. - /// ICEs when encountered during non-Miri codegen. + /// See documentation of `<*const T>::sub_ptr` for details. + #[rustc_const_unstable(feature = "const_ptr_sub_ptr", issue = "95892")] + pub fn ptr_offset_from_unsigned(ptr: *const T, base: *const T) -> usize; + + /// See documentation of `<*const T>::guaranteed_eq` for details. + /// Returns `2` if the result is unknown. + /// Returns `1` if the pointers are guaranteed equal + /// Returns `0` if the pointers are guaranteed inequal + /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + #[rustc_const_unstable(feature = "const_raw_ptr_comparison", issue = "53020")] + #[rustc_safe_intrinsic] + pub fn ptr_guaranteed_cmp(ptr: *const T, other: *const T) -> u8; + + /// Allocates a block of memory at compile time. + /// At runtime, just returns a null pointer. + /// + /// # Safety + /// + /// - The `align` argument must be a power of two. + /// - At compile time, a compile error occurs if this constraint is violated. + /// - At runtime, it is not checked. + #[rustc_const_unstable(feature = "const_heap", issue = "79597")] + pub fn const_allocate(size: usize, align: usize) -> *mut u8; + + /// Deallocates a memory which allocated by `intrinsics::const_allocate` at compile time. + /// At runtime, does nothing. + /// + /// # Safety + /// + /// - The `align` argument must be a power of two. + /// - At compile time, a compile error occurs if this constraint is violated. + /// - At runtime, it is not checked. + /// - If the `ptr` is created in an another const, this intrinsic doesn't deallocate it. + /// - If the `ptr` is pointing to a local variable, this intrinsic doesn't deallocate it. + #[rustc_const_unstable(feature = "const_heap", issue = "79597")] + pub fn const_deallocate(ptr: *mut u8, size: usize, align: usize); + + /// Determines whether the raw bytes of the two values are equal. + /// + /// This is particularly handy for arrays, since it allows things like just + /// comparing `i96`s instead of forcing `alloca`s for `[6 x i16]`. + /// + /// Above some backend-decided threshold this will emit calls to `memcmp`, + /// like slice equality does, instead of causing massive code size. + /// + /// # Safety + /// + /// It's UB to call this if any of the *bytes* in `*a` or `*b` are uninitialized or carry a + /// pointer value. + /// Note that this is a stricter criterion than just the *values* being + /// fully-initialized: if `T` has padding, it's UB to call this intrinsic. + /// + /// (The implementation is allowed to branch on the results of comparisons, + /// which is UB if any of their inputs are `undef`.) + #[rustc_const_unstable(feature = "const_intrinsic_raw_eq", issue = "none")] + pub fn raw_eq(a: &T, b: &T) -> bool; + + /// See documentation of [`std::hint::black_box`] for details. + /// + /// [`std::hint::black_box`]: crate::hint::black_box + #[rustc_const_unstable(feature = "const_black_box", issue = "none")] + #[rustc_safe_intrinsic] + pub fn black_box(dummy: T) -> T; + + /// `ptr` must point to a vtable. + /// The intrinsic will return the size stored in that vtable. + pub fn vtable_size(ptr: *const ()) -> usize; + + /// `ptr` must point to a vtable. + /// The intrinsic will return the alignment stored in that vtable. + pub fn vtable_align(ptr: *const ()) -> usize; + + /// Selects which function to call depending on the context. + /// + /// If this function is evaluated at compile-time, then a call to this + /// intrinsic will be replaced with a call to `called_in_const`. It gets + /// replaced with a call to `called_at_rt` otherwise. + /// + /// # Type Requirements /// - /// The `payload` ptr here will be exactly the one `do_catch` gets passed by `try`. + /// The two functions must be both function items. They cannot be function + /// pointers or closures. The first function must be a `const fn`. /// - /// Perma-unstable: do not use. - pub fn miri_start_panic(payload: *mut u8) -> !; + /// `arg` will be the tupled arguments that will be passed to either one of + /// the two functions, therefore, both functions must accept the same type of + /// arguments. Both functions must return RET. + /// + /// # Safety + /// + /// The two functions must behave observably equivalent. Safe code in other + /// crates may assume that calling a `const fn` at compile-time and at run-time + /// produces the same result. A function that produces a different result when + /// evaluated at run-time, or has any other observable side-effects, is + /// *unsound*. + /// + /// Here is an example of how this could cause a problem: + /// ```no_run + /// #![feature(const_eval_select)] + /// #![feature(core_intrinsics)] + /// use std::hint::unreachable_unchecked; + /// use std::intrinsics::const_eval_select; + /// + /// // Crate A + /// pub const fn inconsistent() -> i32 { + /// fn runtime() -> i32 { 1 } + /// const fn compiletime() -> i32 { 2 } + /// + /// unsafe { + // // ⚠ This code violates the required equivalence of `compiletime` + /// // and `runtime`. + /// const_eval_select((), compiletime, runtime) + /// } + /// } + /// + /// // Crate B + /// const X: i32 = inconsistent(); + /// let x = inconsistent(); + /// if x != X { unsafe { unreachable_unchecked(); }} + /// ``` + /// + /// This code causes Undefined Behavior when being run, since the + /// `unreachable_unchecked` is actually being reached. The bug is in *crate A*, + /// which violates the principle that a `const fn` must behave the same at + /// compile-time and at run-time. The unsafe code in crate B is fine. + #[rustc_const_unstable(feature = "const_eval_select", issue = "none")] + pub fn const_eval_select( + arg: ARG, + called_in_const: F, + called_at_rt: G, + ) -> RET + where + G: FnOnce, + F: FnOnce; } // Some functions are defined here because they accidentally got made @@ -1867,17 +2190,71 @@ extern "rust-intrinsic" { // (`transmute` also falls into this category, but it cannot be wrapped due to the // check that `T` and `U` have the same size.) +/// Check that the preconditions of an unsafe function are followed, if debug_assertions are on, +/// and only at runtime. +/// +/// This macro should be called as `assert_unsafe_precondition!([Generics](name: Type) => Expression)` +/// where the names specified will be moved into the macro as captured variables, and defines an item +/// to call `const_eval_select` on. The tokens inside the square brackets are used to denote generics +/// for the function declaractions and can be omitted if there is no generics. +/// +/// # Safety +/// +/// Invoking this macro is only sound if the following code is already UB when the passed +/// expression evaluates to false. +/// +/// This macro expands to a check at runtime if debug_assertions is set. It has no effect at +/// compile time, but the semantics of the contained `const_eval_select` must be the same at +/// runtime and at compile time. Thus if the expression evaluates to false, this macro produces +/// different behavior at compile time and at runtime, and invoking it is incorrect. +/// +/// So in a sense it is UB if this macro is useful, but we expect callers of `unsafe fn` to make +/// the occasional mistake, and this check should help them figure things out. +#[allow_internal_unstable(const_eval_select)] // permit this to be called in stably-const fn +macro_rules! assert_unsafe_precondition { + ($name:expr, $([$($tt:tt)*])?($($i:ident:$ty:ty),*$(,)?) => $e:expr) => { + if cfg!(debug_assertions) { + // allow non_snake_case to allow capturing const generics + #[allow(non_snake_case)] + #[inline(always)] + fn runtime$(<$($tt)*>)?($($i:$ty),*) { + if !$e { + // don't unwind to reduce impact on code size + ::core::panicking::panic_nounwind( + concat!("unsafe precondition(s) violated: ", $name) + ); + } + } + #[allow(non_snake_case)] + const fn comptime$(<$($tt)*>)?($(_:$ty),*) {} + + ::core::intrinsics::const_eval_select(($($i,)*), comptime, runtime); + } + }; +} +pub(crate) use assert_unsafe_precondition; + /// Checks whether `ptr` is properly aligned with respect to /// `align_of::()`. pub(crate) fn is_aligned_and_not_null(ptr: *const T) -> bool { - !ptr.is_null() && ptr as usize % mem::align_of::() == 0 + !ptr.is_null() && ptr.is_aligned() +} + +/// Checks whether an allocation of `len` instances of `T` exceeds +/// the maximum allowed allocation size. +pub(crate) fn is_valid_allocation_size(len: usize) -> bool { + let max_len = const { + let size = crate::mem::size_of::(); + if size == 0 { usize::MAX } else { isize::MAX as usize / size } + }; + len <= max_len } /// Checks whether the regions of memory starting at `src` and `dst` of size /// `count * size_of::()` do *not* overlap. pub(crate) fn is_nonoverlapping(src: *const T, dst: *const T, count: usize) -> bool { - let src_usize = src as usize; - let dst_usize = dst as usize; + let src_usize = src.addr(); + let dst_usize = dst.addr(); let size = mem::size_of::().checked_mul(count).unwrap(); let diff = if src_usize > dst_usize { src_usize - dst_usize } else { dst_usize - src_usize }; // If the absolute distance between the ptrs is at least as big as the size of the buffer, @@ -1893,7 +2270,9 @@ pub(crate) fn is_nonoverlapping(src: *const T, dst: *const T, count: usize) - /// `copy_nonoverlapping` is semantically equivalent to C's [`memcpy`], but /// with the argument order swapped. /// -/// [`copy`]: ./fn.copy.html +/// The copy is "untyped" in the sense that data may be uninitialized or otherwise violate the +/// requirements of `T`. The initialization state is preserved exactly. +/// /// [`memcpy`]: https://en.cppreference.com/w/c/string/byte/memcpy /// /// # Safety @@ -1916,12 +2295,11 @@ pub(crate) fn is_nonoverlapping(src: *const T, dst: *const T, count: usize) - /// [violate memory safety][read-ownership]. /// /// Note that even if the effectively copied size (`count * size_of::()`) is -/// `0`, the pointers must be non-NULL and properly aligned. +/// `0`, the pointers must be non-null and properly aligned. /// -/// [`Copy`]: ../marker/trait.Copy.html -/// [`read`]: ../ptr/fn.read.html -/// [read-ownership]: ../ptr/fn.read.html#ownership-of-the-returned-value -/// [valid]: ../ptr/index.html#safety +/// [`read`]: crate::ptr::read +/// [read-ownership]: crate::ptr::read#ownership-of-the-returned-value +/// [valid]: crate::ptr#safety /// /// # Examples /// @@ -1939,9 +2317,9 @@ pub(crate) fn is_nonoverlapping(src: *const T, dst: *const T, count: usize) - /// dst.reserve(src_len); /// /// unsafe { -/// // The call to offset is always safe because `Vec` will never +/// // The call to add is always safe because `Vec` will never /// // allocate more than `isize::MAX` bytes. -/// let dst_ptr = dst.as_mut_ptr().offset(dst_len as isize); +/// let dst_ptr = dst.as_mut_ptr().add(dst_len); /// let src_ptr = src.as_ptr(); /// /// // Truncate `src` without dropping its contents. We do this first, @@ -1970,16 +2348,29 @@ pub(crate) fn is_nonoverlapping(src: *const T, dst: *const T, count: usize) - /// [`Vec::append`]: ../../std/vec/struct.Vec.html#method.append #[doc(alias = "memcpy")] #[stable(feature = "rust1", since = "1.0.0")] +#[rustc_allowed_through_unstable_modules] +#[rustc_const_stable(feature = "const_intrinsic_copy", since = "1.63.0")] #[inline] -pub unsafe fn copy_nonoverlapping(src: *const T, dst: *mut T, count: usize) { +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces +pub const unsafe fn copy_nonoverlapping(src: *const T, dst: *mut T, count: usize) { extern "rust-intrinsic" { - fn copy_nonoverlapping(src: *const T, dst: *mut T, count: usize); + #[rustc_const_stable(feature = "const_intrinsic_copy", since = "1.63.0")] + pub fn copy_nonoverlapping(src: *const T, dst: *mut T, count: usize); } - debug_assert!(is_aligned_and_not_null(src), "attempt to copy from unaligned or null pointer"); - debug_assert!(is_aligned_and_not_null(dst), "attempt to copy to unaligned or null pointer"); - debug_assert!(is_nonoverlapping(src, dst, count), "attempt to copy to overlapping memory"); - copy_nonoverlapping(src, dst, count) + // SAFETY: the safety contract for `copy_nonoverlapping` must be + // upheld by the caller. + unsafe { + assert_unsafe_precondition!( + "ptr::copy_nonoverlapping requires that both pointer arguments are aligned and non-null \ + and the specified memory ranges do not overlap", + [T](src: *const T, dst: *mut T, count: usize) => + is_aligned_and_not_null(src) + && is_aligned_and_not_null(dst) + && is_nonoverlapping(src, dst, count) + ); + copy_nonoverlapping(src, dst, count) + } } /// Copies `count * size_of::()` bytes from `src` to `dst`. The source @@ -1992,7 +2383,9 @@ pub unsafe fn copy_nonoverlapping(src: *const T, dst: *mut T, count: usize) { /// order swapped. Copying takes place as if the bytes were copied from `src` /// to a temporary array and then copied from the array to `dst`. /// -/// [`copy_nonoverlapping`]: ./fn.copy_nonoverlapping.html +/// The copy is "untyped" in the sense that data may be uninitialized or otherwise violate the +/// requirements of `T`. The initialization state is preserved exactly. +/// /// [`memmove`]: https://en.cppreference.com/w/c/string/byte/memmove /// /// # Safety @@ -2011,12 +2404,11 @@ pub unsafe fn copy_nonoverlapping(src: *const T, dst: *mut T, count: usize) { /// [violate memory safety][read-ownership]. /// /// Note that even if the effectively copied size (`count * size_of::()`) is -/// `0`, the pointers must be non-NULL and properly aligned. +/// `0`, the pointers must be non-null and properly aligned. /// -/// [`Copy`]: ../marker/trait.Copy.html -/// [`read`]: ../ptr/fn.read.html -/// [read-ownership]: ../ptr/fn.read.html#ownership-of-the-returned-value -/// [valid]: ../ptr/index.html#safety +/// [`read`]: crate::ptr::read +/// [read-ownership]: crate::ptr::read#ownership-of-the-returned-value +/// [valid]: crate::ptr#safety /// /// # Examples /// @@ -2025,25 +2417,46 @@ pub unsafe fn copy_nonoverlapping(src: *const T, dst: *mut T, count: usize) { /// ``` /// use std::ptr; /// +/// /// # Safety +/// /// +/// /// * `ptr` must be correctly aligned for its type and non-zero. +/// /// * `ptr` must be valid for reads of `elts` contiguous elements of type `T`. +/// /// * Those elements must not be used after calling this function unless `T: Copy`. /// # #[allow(dead_code)] /// unsafe fn from_buf_raw(ptr: *const T, elts: usize) -> Vec { /// let mut dst = Vec::with_capacity(elts); -/// dst.set_len(elts); +/// +/// // SAFETY: Our precondition ensures the source is aligned and valid, +/// // and `Vec::with_capacity` ensures that we have usable space to write them. /// ptr::copy(ptr, dst.as_mut_ptr(), elts); +/// +/// // SAFETY: We created it with this much capacity earlier, +/// // and the previous `copy` has initialized these elements. +/// dst.set_len(elts); /// dst /// } /// ``` #[doc(alias = "memmove")] #[stable(feature = "rust1", since = "1.0.0")] +#[rustc_allowed_through_unstable_modules] +#[rustc_const_stable(feature = "const_intrinsic_copy", since = "1.63.0")] #[inline] -pub unsafe fn copy(src: *const T, dst: *mut T, count: usize) { +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces +pub const unsafe fn copy(src: *const T, dst: *mut T, count: usize) { extern "rust-intrinsic" { + #[rustc_const_stable(feature = "const_intrinsic_copy", since = "1.63.0")] fn copy(src: *const T, dst: *mut T, count: usize); } - debug_assert!(is_aligned_and_not_null(src), "attempt to copy from unaligned or null pointer"); - debug_assert!(is_aligned_and_not_null(dst), "attempt to copy to unaligned or null pointer"); - copy(src, dst, count) + // SAFETY: the safety contract for `copy` must be upheld by the caller. + unsafe { + assert_unsafe_precondition!( + "ptr::copy requires that both pointer arguments are aligned aligned and non-null", + [T](src: *const T, dst: *mut T) => + is_aligned_and_not_null(src) && is_aligned_and_not_null(dst) + ); + copy(src, dst, count) + } } /// Sets `count * size_of::()` bytes of memory starting at `dst` to @@ -2062,15 +2475,24 @@ pub unsafe fn copy(src: *const T, dst: *mut T, count: usize) { /// /// * `dst` must be properly aligned. /// -/// Additionally, the caller must ensure that writing `count * -/// size_of::()` bytes to the given region of memory results in a valid -/// value of `T`. Using a region of memory typed as a `T` that contains an -/// invalid value of `T` is undefined behavior. -/// /// Note that even if the effectively copied size (`count * size_of::()`) is -/// `0`, the pointer must be non-NULL and properly aligned. +/// `0`, the pointer must be non-null and properly aligned. +/// +/// Additionally, note that changing `*dst` in this way can easily lead to undefined behavior (UB) +/// later if the written bytes are not a valid representation of some `T`. For instance, the +/// following is an **incorrect** use of this function: +/// +/// ```rust,no_run +/// unsafe { +/// let mut value: u8 = 0; +/// let ptr: *mut bool = &mut value as *mut u8 as *mut bool; +/// let _bool = ptr.read(); // This is fine, `ptr` points to a valid `bool`. +/// ptr.write_bytes(42u8, 1); // This function itself does not cause UB... +/// let _bool = ptr.read(); // ...but it makes this operation UB! ⚠️ +/// } +/// ``` /// -/// [valid]: ../ptr/index.html#safety +/// [valid]: crate::ptr#safety /// /// # Examples /// @@ -2086,45 +2508,24 @@ pub unsafe fn copy(src: *const T, dst: *mut T, count: usize) { /// } /// assert_eq!(vec, [0xfefefefe, 0xfefefefe, 0, 0]); /// ``` -/// -/// Creating an invalid value: -/// -/// ``` -/// use std::ptr; -/// -/// let mut v = Box::new(0i32); -/// -/// unsafe { -/// // Leaks the previously held value by overwriting the `Box` with -/// // a null pointer. -/// ptr::write_bytes(&mut v as *mut Box, 0, 1); -/// } -/// -/// // At this point, using or dropping `v` results in undefined behavior. -/// // drop(v); // ERROR -/// -/// // Even leaking `v` "uses" it, and hence is undefined behavior. -/// // mem::forget(v); // ERROR -/// -/// // In fact, `v` is invalid according to basic type layout invariants, so *any* -/// // operation touching it is undefined behavior. -/// // let v2 = v; // ERROR -/// -/// unsafe { -/// // Let us instead put in a valid value -/// ptr::write(&mut v as *mut Box, Box::new(42i32)); -/// } -/// -/// // Now the box is fine -/// assert_eq!(*v, 42); -/// ``` +#[doc(alias = "memset")] #[stable(feature = "rust1", since = "1.0.0")] +#[rustc_allowed_through_unstable_modules] +#[rustc_const_unstable(feature = "const_ptr_write", issue = "86302")] #[inline] -pub unsafe fn write_bytes(dst: *mut T, val: u8, count: usize) { +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces +pub const unsafe fn write_bytes(dst: *mut T, val: u8, count: usize) { extern "rust-intrinsic" { + #[rustc_const_unstable(feature = "const_ptr_write", issue = "86302")] fn write_bytes(dst: *mut T, val: u8, count: usize); } - debug_assert!(is_aligned_and_not_null(dst), "attempt to write to unaligned or null pointer"); - write_bytes(dst, val, count) + // SAFETY: the safety contract for `write_bytes` must be upheld by the caller. + unsafe { + assert_unsafe_precondition!( + "ptr::write_bytes requires that the destination pointer is aligned and non-null", + [T](dst: *mut T) => is_aligned_and_not_null(dst) + ); + write_bytes(dst, val, count) + } } diff --git a/crux-mir/lib/core/src/intrinsics/mir.rs b/crux-mir/lib/core/src/intrinsics/mir.rs new file mode 100644 index 000000000..e3157b669 --- /dev/null +++ b/crux-mir/lib/core/src/intrinsics/mir.rs @@ -0,0 +1,545 @@ +//! Rustc internal tooling for hand-writing MIR. +//! +//! If for some reasons you are not writing rustc tests and have found yourself considering using +//! this feature, turn back. This is *exceptionally* unstable. There is no attempt at all to make +//! anything work besides those things which the rustc test suite happened to need. If you make a +//! typo you'll probably ICE. Really, this is not the solution to your problems. Consider instead +//! supporting the [stable MIR project group](https://github.com/rust-lang/project-stable-mir). +//! +//! The documentation for this module describes how to use this feature. If you are interested in +//! hacking on the implementation, most of that documentation lives at +//! `rustc_mir_building/src/build/custom/mod.rs`. +//! +//! Typical usage will look like this: +//! +//! ```rust +//! #![feature(core_intrinsics, custom_mir)] +//! +//! extern crate core; +//! use core::intrinsics::mir::*; +//! +//! #[custom_mir(dialect = "built")] +//! pub fn simple(x: i32) -> i32 { +//! mir!( +//! let temp2: i32; +//! +//! { +//! let temp1 = x; +//! Goto(my_second_block) +//! } +//! +//! my_second_block = { +//! temp2 = Move(temp1); +//! RET = temp2; +//! Return() +//! } +//! ) +//! } +//! ``` +//! +//! The `custom_mir` attribute tells the compiler to treat the function as being custom MIR. This +//! attribute only works on functions - there is no way to insert custom MIR into the middle of +//! another function. The `dialect` and `phase` parameters indicate which [version of MIR][dialect +//! docs] you are inserting here. Generally you'll want to use `#![custom_mir(dialect = "built")]` +//! if you want your MIR to be modified by the full MIR pipeline, or `#![custom_mir(dialect = +//! "runtime", phase = "optimized")] if you don't. +//! +//! [dialect docs]: +//! https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/enum.MirPhase.html +//! +//! The input to the [`mir!`] macro is: +//! +//! - A possibly empty list of local declarations. Locals can also be declared inline on +//! assignments via `let`. Type inference generally works. Shadowing does not. +//! - A list of basic blocks. The first of these is the start block and is where execution begins. +//! All blocks other than the start block need to be given a name, so that they can be referred +//! to later. +//! - Each block is a list of semicolon terminated statements, followed by a terminator. The +//! syntax for the various statements and terminators is designed to be as similar as possible +//! to the syntax for analogous concepts in native Rust. See below for a list. +//! +//! # Examples +//! +#![cfg_attr(bootstrap, doc = "```rust,compile_fail")] +#![cfg_attr(not(bootstrap), doc = "```rust")] +//! #![feature(core_intrinsics, custom_mir)] +//! +//! extern crate core; +//! use core::intrinsics::mir::*; +//! +//! #[custom_mir(dialect = "built")] +//! pub fn choose_load(a: &i32, b: &i32, c: bool) -> i32 { +//! mir!( +//! { +//! match c { +//! true => t, +//! _ => f, +//! } +//! } +//! +//! t = { +//! let temp = a; +//! Goto(load_and_exit) +//! } +//! +//! f = { +//! temp = b; +//! Goto(load_and_exit) +//! } +//! +//! load_and_exit = { +//! RET = *temp; +//! Return() +//! } +//! ) +//! } +//! +//! #[custom_mir(dialect = "built")] +//! fn unwrap_unchecked(opt: Option) -> T { +//! mir!({ +//! RET = Move(Field(Variant(opt, 1), 0)); +//! Return() +//! }) +//! } +//! +//! #[custom_mir(dialect = "runtime", phase = "optimized")] +//! fn push_and_pop(v: &mut Vec, value: T) { +//! mir!( +//! let unused; +//! let popped; +//! +//! { +//! Call(unused, pop, Vec::push(v, value)) +//! } +//! +//! pop = { +//! Call(popped, drop, Vec::pop(v)) +//! } +//! +//! drop = { +//! Drop(popped, ret) +//! } +//! +//! ret = { +//! Return() +//! } +//! ) +//! } +//! ``` +//! +//! We can also set off compilation failures that happen in sufficiently late stages of the +//! compiler: +//! +//! ```rust,compile_fail +//! #![feature(core_intrinsics, custom_mir)] +//! +//! extern crate core; +//! use core::intrinsics::mir::*; +//! +//! #[custom_mir(dialect = "built")] +//! fn borrow_error(should_init: bool) -> i32 { +//! mir!( +//! let temp: i32; +//! +//! { +//! match should_init { +//! true => init, +//! _ => use_temp, +//! } +//! } +//! +//! init = { +//! temp = 0; +//! Goto(use_temp) +//! } +//! +//! use_temp = { +//! RET = temp; +//! Return() +//! } +//! ) +//! } +//! ``` +//! +//! ```text +//! error[E0381]: used binding is possibly-uninitialized +//! --> test.rs:24:13 +//! | +//! 8 | / mir!( +//! 9 | | let temp: i32; +//! 10 | | +//! 11 | | { +//! ... | +//! 19 | | temp = 0; +//! | | -------- binding initialized here in some conditions +//! ... | +//! 24 | | RET = temp; +//! | | ^^^^^^^^^^ value used here but it is possibly-uninitialized +//! 25 | | Return() +//! 26 | | } +//! 27 | | ) +//! | |_____- binding declared here but left uninitialized +//! +//! error: aborting due to previous error +//! +//! For more information about this error, try `rustc --explain E0381`. +//! ``` +//! +//! # Syntax +//! +//! The lists below are an exhaustive description of how various MIR constructs can be created. +//! Anything missing from the list should be assumed to not be supported, PRs welcome. +//! +//! #### Locals +//! +//! - The `_0` return local can always be accessed via `RET`. +//! - Arguments can be accessed via their regular name. +//! - All other locals need to be declared with `let` somewhere and then can be accessed by name. +//! +//! #### Places +//! - Locals implicit convert to places. +//! - Field accesses, derefs, and indexing work normally. +//! - Fields in variants can be accessed via the [`Variant`] and [`Field`] associated functions, +//! see their documentation for details. +//! +//! #### Operands +//! - Places implicitly convert to `Copy` operands. +//! - `Move` operands can be created via [`Move`]. +//! - Const blocks, literals, named constants, and const params all just work. +//! - [`Static`] and [`StaticMut`] can be used to create `&T` and `*mut T`s to statics. These are +//! constants in MIR and the only way to access statics. +//! +//! #### Statements +//! - Assign statements work via normal Rust assignment. +//! - [`Retag`] statements have an associated function. +//! +//! #### Rvalues +//! +//! - Operands implicitly convert to `Use` rvalues. +//! - `&`, `&mut`, `addr_of!`, and `addr_of_mut!` all work to create their associated rvalue. +//! - [`Discriminant`] has an associated function. +//! +//! #### Terminators +//! +//! Custom MIR does not currently support cleanup blocks or non-trivial unwind paths. As such, there +//! are no resume and abort terminators, and terminators that might unwind do not have any way to +//! indicate the unwind block. +//! +//! - [`Goto`], [`Return`], [`Unreachable`], [`Drop`](Drop()), and [`DropAndReplace`] have associated functions. +//! - `match some_int_operand` becomes a `SwitchInt`. Each arm should be `literal => basic_block` +//! - The exception is the last arm, which must be `_ => basic_block` and corresponds to the +//! otherwise branch. +//! - [`Call`] has an associated function as well. The third argument of this function is a normal +//! function call expresion, for example `my_other_function(a, 5)`. +//! + +#![unstable( + feature = "custom_mir", + reason = "MIR is an implementation detail and extremely unstable", + issue = "none" +)] +#![allow(unused_variables, non_snake_case, missing_debug_implementations)] + +/// Type representing basic blocks. +/// +/// All terminators will have this type as a return type. It helps achieve some type safety. +pub struct BasicBlock; + +macro_rules! define { + ($name:literal, $( #[ $meta:meta ] )* fn $($sig:tt)*) => { + #[rustc_diagnostic_item = $name] + $( #[ $meta ] )* + pub fn $($sig)* { panic!() } + } +} + +define!("mir_return", fn Return() -> BasicBlock); +define!("mir_goto", fn Goto(destination: BasicBlock) -> BasicBlock); +define!("mir_unreachable", fn Unreachable() -> BasicBlock); +define!("mir_drop", fn Drop(place: T, goto: BasicBlock)); +define!("mir_drop_and_replace", fn DropAndReplace(place: T, value: T, goto: BasicBlock)); +define!("mir_call", fn Call(place: T, goto: BasicBlock, call: T)); +define!("mir_storage_live", fn StorageLive(local: T)); +define!("mir_storage_dead", fn StorageDead(local: T)); +define!("mir_retag", fn Retag(place: T)); +define!("mir_move", fn Move(place: T) -> T); +define!("mir_static", fn Static(s: T) -> &'static T); +define!("mir_static_mut", fn StaticMut(s: T) -> *mut T); +define!( + "mir_discriminant", + /// Gets the discriminant of a place. + fn Discriminant(place: T) -> ::Discriminant +); +define!("mir_set_discriminant", fn SetDiscriminant(place: T, index: u32)); +define!( + "mir_field", + /// Access the field with the given index of some place. + /// + /// This only makes sense to use in conjunction with [`Variant`]. If the type you are looking to + /// access the field of does not have variants, you can use normal field projection syntax. + /// + /// There is no proper way to do a place projection to a variant in Rust, and so these two + /// functions are a workaround. You can access a field of a variant via `Field(Variant(place, + /// var_idx), field_idx)`, where `var_idx` and `field_idx` are appropriate literals. Some + /// caveats: + /// + /// - The return type of `Variant` is always `()`. Don't worry about that, the correct MIR will + /// still be generated. + /// - In some situations, the return type of `Field` cannot be inferred. You may need to + /// annotate it on the function in these cases. + /// - Since `Field` is a function call which is not a place expression, using this on the left + /// hand side of an expression is rejected by the compiler. [`place!`] is a macro provided to + /// work around that issue. Wrap the left hand side of an assignment in the macro to convince + /// the compiler that it's ok. + /// + /// # Examples + /// + #[cfg_attr(bootstrap, doc = "```rust,compile_fail")] + #[cfg_attr(not(bootstrap), doc = "```rust")] + /// #![feature(custom_mir, core_intrinsics)] + /// + /// extern crate core; + /// use core::intrinsics::mir::*; + /// + /// #[custom_mir(dialect = "built")] + /// fn unwrap_deref(opt: Option<&i32>) -> i32 { + /// mir!({ + /// RET = *Field::<&i32>(Variant(opt, 1), 0); + /// Return() + /// }) + /// } + /// + /// #[custom_mir(dialect = "built")] + /// fn set(opt: &mut Option) { + /// mir!({ + /// place!(Field(Variant(*opt, 1), 0)) = 5; + /// Return() + /// }) + /// } + /// ``` + fn Field(place: (), field: u32) -> F +); +define!( + "mir_variant", + /// Adds a variant projection with the given index to the place. + /// + /// See [`Field`] for documentation. + fn Variant(place: T, index: u32) -> () +); +define!( + "mir_make_place", + #[doc(hidden)] + fn __internal_make_place(place: T) -> *mut T +); + +/// Macro for generating custom MIR. +/// +/// See the module documentation for syntax details. This macro is not magic - it only transforms +/// your MIR into something that is easier to parse in the compiler. +#[rustc_macro_transparency = "transparent"] +pub macro mir { + ( + $(let $local_decl:ident $(: $local_decl_ty:ty)? ;)* + + { + $($entry:tt)* + } + + $( + $block_name:ident = { + $($block:tt)* + } + )* + ) => {{ + // First, we declare all basic blocks. + $( + let $block_name: ::core::intrinsics::mir::BasicBlock; + )* + + { + // Now all locals + #[allow(non_snake_case)] + let RET; + $( + let $local_decl $(: $local_decl_ty)? ; + )* + + ::core::intrinsics::mir::__internal_extract_let!($($entry)*); + $( + ::core::intrinsics::mir::__internal_extract_let!($($block)*); + )* + + { + // Finally, the contents of the basic blocks + ::core::intrinsics::mir::__internal_remove_let!({ + {} + { $($entry)* } + }); + $( + ::core::intrinsics::mir::__internal_remove_let!({ + {} + { $($block)* } + }); + )* + + RET + } + } + }} +} + +/// Helper macro that allows you to treat a value expression like a place expression. +/// +/// See the documentation on [`Variant`] for why this is necessary and how to use it. +pub macro place($e:expr) { + (*::core::intrinsics::mir::__internal_make_place($e)) +} + +/// Helper macro that extracts the `let` declarations out of a bunch of statements. +/// +/// This macro is written using the "statement muncher" strategy. Each invocation parses the first +/// statement out of the input, does the appropriate thing with it, and then recursively calls the +/// same macro on the remainder of the input. +#[doc(hidden)] +pub macro __internal_extract_let { + // If it's a `let` like statement, keep the `let` + ( + let $var:ident $(: $ty:ty)? = $expr:expr; $($rest:tt)* + ) => { + let $var $(: $ty)?; + ::core::intrinsics::mir::__internal_extract_let!($($rest)*); + }, + // Due to #86730, we have to handle const blocks separately + ( + let $var:ident $(: $ty:ty)? = const $block:block; $($rest:tt)* + ) => { + let $var $(: $ty)?; + ::core::intrinsics::mir::__internal_extract_let!($($rest)*); + }, + // Otherwise, output nothing + ( + $stmt:stmt; $($rest:tt)* + ) => { + ::core::intrinsics::mir::__internal_extract_let!($($rest)*); + }, + ( + $expr:expr + ) => {} +} + +/// Helper macro that removes the `let` declarations from a bunch of statements. +/// +/// Because expression position macros cannot expand to statements + expressions, we need to be +/// slightly creative here. The general strategy is also statement munching as above, but the output +/// of the macro is "stored" in the subsequent macro invocation. Easiest understood via example: +/// ```text +/// invoke!( +/// { +/// { +/// x = 5; +/// } +/// { +/// let d = e; +/// Call() +/// } +/// } +/// ) +/// ``` +/// becomes +/// ```text +/// invoke!( +/// { +/// { +/// x = 5; +/// d = e; +/// } +/// { +/// Call() +/// } +/// } +/// ) +/// ``` +#[doc(hidden)] +pub macro __internal_remove_let { + // If it's a `let` like statement, remove the `let` + ( + { + { + $($already_parsed:tt)* + } + { + let $var:ident $(: $ty:ty)? = $expr:expr; + $($rest:tt)* + } + } + ) => { ::core::intrinsics::mir::__internal_remove_let!( + { + { + $($already_parsed)* + $var = $expr; + } + { + $($rest)* + } + } + )}, + // Due to #86730 , we have to handle const blocks separately + ( + { + { + $($already_parsed:tt)* + } + { + let $var:ident $(: $ty:ty)? = const $block:block; + $($rest:tt)* + } + } + ) => { ::core::intrinsics::mir::__internal_remove_let!( + { + { + $($already_parsed)* + $var = const $block; + } + { + $($rest)* + } + } + )}, + // Otherwise, keep going + ( + { + { + $($already_parsed:tt)* + } + { + $stmt:stmt; + $($rest:tt)* + } + } + ) => { ::core::intrinsics::mir::__internal_remove_let!( + { + { + $($already_parsed)* + $stmt; + } + { + $($rest)* + } + } + )}, + ( + { + { + $($already_parsed:tt)* + } + { + $expr:expr + } + } + ) => { + { + $($already_parsed)* + $expr + } + }, +} diff --git a/crux-mir/lib/core/src/iter/adapters/array_chunks.rs b/crux-mir/lib/core/src/iter/adapters/array_chunks.rs new file mode 100644 index 000000000..5e4211058 --- /dev/null +++ b/crux-mir/lib/core/src/iter/adapters/array_chunks.rs @@ -0,0 +1,239 @@ +use crate::array; +use crate::const_closure::ConstFnMutClosure; +use crate::iter::{ByRefSized, FusedIterator, Iterator, TrustedRandomAccessNoCoerce}; +use crate::mem::{self, MaybeUninit}; +use crate::ops::{ControlFlow, NeverShortCircuit, Try}; + +/// An iterator over `N` elements of the iterator at a time. +/// +/// The chunks do not overlap. If `N` does not divide the length of the +/// iterator, then the last up to `N-1` elements will be omitted. +/// +/// This `struct` is created by the [`array_chunks`][Iterator::array_chunks] +/// method on [`Iterator`]. See its documentation for more. +#[derive(Debug, Clone)] +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[unstable(feature = "iter_array_chunks", reason = "recently added", issue = "100450")] +pub struct ArrayChunks { + iter: I, + remainder: Option>, +} + +impl ArrayChunks +where + I: Iterator, +{ + #[track_caller] + pub(in crate::iter) fn new(iter: I) -> Self { + assert!(N != 0, "chunk size must be non-zero"); + Self { iter, remainder: None } + } + + /// Returns an iterator over the remaining elements of the original iterator + /// that are not going to be returned by this iterator. The returned + /// iterator will yield at most `N-1` elements. + #[unstable(feature = "iter_array_chunks", reason = "recently added", issue = "100450")] + #[inline] + pub fn into_remainder(self) -> Option> { + self.remainder + } +} + +#[unstable(feature = "iter_array_chunks", reason = "recently added", issue = "100450")] +impl Iterator for ArrayChunks +where + I: Iterator, +{ + type Item = [I::Item; N]; + + #[inline] + fn next(&mut self) -> Option { + self.try_for_each(ControlFlow::Break).break_value() + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let (lower, upper) = self.iter.size_hint(); + + (lower / N, upper.map(|n| n / N)) + } + + #[inline] + fn count(self) -> usize { + self.iter.count() / N + } + + fn try_fold(&mut self, init: B, mut f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: Try, + { + let mut acc = init; + loop { + match self.iter.next_chunk() { + Ok(chunk) => acc = f(acc, chunk)?, + Err(remainder) => { + // Make sure to not override `self.remainder` with an empty array + // when `next` is called after `ArrayChunks` exhaustion. + self.remainder.get_or_insert(remainder); + + break try { acc }; + } + } + } + } + + fn fold(self, init: B, f: F) -> B + where + Self: Sized, + F: FnMut(B, Self::Item) -> B, + { + ::fold(self, init, f) + } +} + +#[unstable(feature = "iter_array_chunks", reason = "recently added", issue = "100450")] +impl DoubleEndedIterator for ArrayChunks +where + I: DoubleEndedIterator + ExactSizeIterator, +{ + #[inline] + fn next_back(&mut self) -> Option { + self.try_rfold((), |(), x| ControlFlow::Break(x)).break_value() + } + + fn try_rfold(&mut self, init: B, mut f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: Try, + { + // We are iterating from the back we need to first handle the remainder. + self.next_back_remainder(); + + let mut acc = init; + let mut iter = ByRefSized(&mut self.iter).rev(); + + // NB remainder is handled by `next_back_remainder`, so + // `next_chunk` can't return `Err` with non-empty remainder + // (assuming correct `I as ExactSizeIterator` impl). + while let Ok(mut chunk) = iter.next_chunk() { + // FIXME: do not do double reverse + // (we could instead add `next_chunk_back` for example) + chunk.reverse(); + acc = f(acc, chunk)? + } + + try { acc } + } + + impl_fold_via_try_fold! { rfold -> try_rfold } +} + +impl ArrayChunks +where + I: DoubleEndedIterator + ExactSizeIterator, +{ + /// Updates `self.remainder` such that `self.iter.len` is divisible by `N`. + fn next_back_remainder(&mut self) { + // Make sure to not override `self.remainder` with an empty array + // when `next_back` is called after `ArrayChunks` exhaustion. + if self.remainder.is_some() { + return; + } + + // We use the `ExactSizeIterator` implementation of the underlying + // iterator to know how many remaining elements there are. + let rem = self.iter.len() % N; + + // Take the last `rem` elements out of `self.iter`. + let mut remainder = + // SAFETY: `unwrap_err` always succeeds because x % N < N for all x. + unsafe { self.iter.by_ref().rev().take(rem).next_chunk().unwrap_err_unchecked() }; + + // We used `.rev()` above, so we need to re-reverse the reminder + remainder.as_mut_slice().reverse(); + self.remainder = Some(remainder); + } +} + +#[unstable(feature = "iter_array_chunks", reason = "recently added", issue = "100450")] +impl FusedIterator for ArrayChunks where I: FusedIterator {} + +#[unstable(feature = "iter_array_chunks", reason = "recently added", issue = "100450")] +impl ExactSizeIterator for ArrayChunks +where + I: ExactSizeIterator, +{ + #[inline] + fn len(&self) -> usize { + self.iter.len() / N + } + + #[inline] + fn is_empty(&self) -> bool { + self.iter.len() < N + } +} + +trait SpecFold: Iterator { + fn fold(self, init: B, f: F) -> B + where + Self: Sized, + F: FnMut(B, Self::Item) -> B; +} + +impl SpecFold for ArrayChunks +where + I: Iterator, +{ + #[inline] + default fn fold(mut self, init: B, mut f: F) -> B + where + Self: Sized, + F: FnMut(B, Self::Item) -> B, + { + let fold = ConstFnMutClosure::new(&mut f, NeverShortCircuit::wrap_mut_2_imp); + self.try_fold(init, fold).0 + } +} + +impl SpecFold for ArrayChunks +where + I: Iterator + TrustedRandomAccessNoCoerce, +{ + #[inline] + fn fold(mut self, init: B, mut f: F) -> B + where + Self: Sized, + F: FnMut(B, Self::Item) -> B, + { + let mut accum = init; + let inner_len = self.iter.size(); + let mut i = 0; + // Use a while loop because (0..len).step_by(N) doesn't optimize well. + while inner_len - i >= N { + let mut chunk = MaybeUninit::uninit_array(); + let mut guard = array::Guard { array_mut: &mut chunk, initialized: 0 }; + while guard.initialized < N { + // SAFETY: The method consumes the iterator and the loop condition ensures that + // all accesses are in bounds and only happen once. + unsafe { + let idx = i + guard.initialized; + guard.push_unchecked(self.iter.__iterator_get_unchecked(idx)); + } + } + mem::forget(guard); + // SAFETY: The loop above initialized all elements + let chunk = unsafe { MaybeUninit::array_assume_init(chunk) }; + accum = f(accum, chunk); + i += N; + } + + // unlike try_fold this method does not need to take care of the remainder + // since `self` will be dropped + + accum + } +} diff --git a/crux-mir/lib/core/src/iter/adapters/by_ref_sized.rs b/crux-mir/lib/core/src/iter/adapters/by_ref_sized.rs new file mode 100644 index 000000000..1945e402f --- /dev/null +++ b/crux-mir/lib/core/src/iter/adapters/by_ref_sized.rs @@ -0,0 +1,100 @@ +use crate::{ + const_closure::ConstFnMutClosure, + ops::{NeverShortCircuit, Try}, +}; + +/// Like `Iterator::by_ref`, but requiring `Sized` so it can forward generics. +/// +/// Ideally this will no longer be required, eventually, but as can be seen in +/// the benchmarks (as of Feb 2022 at least) `by_ref` can have performance cost. +#[unstable(feature = "std_internals", issue = "none")] +#[derive(Debug)] +pub struct ByRefSized<'a, I>(pub &'a mut I); + +// The following implementations use UFCS-style, rather than trusting autoderef, +// to avoid accidentally calling the `&mut Iterator` implementations. + +#[unstable(feature = "std_internals", issue = "none")] +impl Iterator for ByRefSized<'_, I> { + type Item = I::Item; + + #[inline] + fn next(&mut self) -> Option { + I::next(self.0) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + I::size_hint(self.0) + } + + #[inline] + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + I::advance_by(self.0, n) + } + + #[inline] + fn nth(&mut self, n: usize) -> Option { + I::nth(self.0, n) + } + + #[inline] + fn fold(self, init: B, mut f: F) -> B + where + F: FnMut(B, Self::Item) -> B, + { + // `fold` needs ownership, so this can't forward directly. + I::try_fold(self.0, init, ConstFnMutClosure::new(&mut f, NeverShortCircuit::wrap_mut_2_imp)) + .0 + } + + #[inline] + fn try_fold(&mut self, init: B, f: F) -> R + where + F: FnMut(B, Self::Item) -> R, + R: Try, + { + I::try_fold(self.0, init, f) + } +} + +#[unstable(feature = "std_internals", issue = "none")] +impl DoubleEndedIterator for ByRefSized<'_, I> { + #[inline] + fn next_back(&mut self) -> Option { + I::next_back(self.0) + } + + #[inline] + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + I::advance_back_by(self.0, n) + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option { + I::nth_back(self.0, n) + } + + #[inline] + fn rfold(self, init: B, mut f: F) -> B + where + F: FnMut(B, Self::Item) -> B, + { + // `rfold` needs ownership, so this can't forward directly. + I::try_rfold( + self.0, + init, + ConstFnMutClosure::new(&mut f, NeverShortCircuit::wrap_mut_2_imp), + ) + .0 + } + + #[inline] + fn try_rfold(&mut self, init: B, f: F) -> R + where + F: FnMut(B, Self::Item) -> R, + R: Try, + { + I::try_rfold(self.0, init, f) + } +} diff --git a/crux-mir/lib/core/src/iter/adapters/chain.rs b/crux-mir/lib/core/src/iter/adapters/chain.rs index 3611a1aad..60eb3a6da 100644 --- a/crux-mir/lib/core/src/iter/adapters/chain.rs +++ b/crux-mir/lib/core/src/iter/adapters/chain.rs @@ -1,52 +1,42 @@ +use crate::iter::{DoubleEndedIterator, FusedIterator, Iterator, TrustedLen}; use crate::ops::Try; -use crate::usize; - -use super::super::{DoubleEndedIterator, FusedIterator, Iterator, TrustedLen}; /// An iterator that links two iterators together, in a chain. /// -/// This `struct` is created by the [`chain`] method on [`Iterator`]. See its -/// documentation for more. +/// This `struct` is created by [`Iterator::chain`]. See its documentation +/// for more. +/// +/// # Examples +/// +/// ``` +/// use std::iter::Chain; +/// use std::slice::Iter; /// -/// [`chain`]: trait.Iterator.html#method.chain -/// [`Iterator`]: trait.Iterator.html +/// let a1 = [1, 2, 3]; +/// let a2 = [4, 5, 6]; +/// let iter: Chain, Iter<_>> = a1.iter().chain(a2.iter()); +/// ``` #[derive(Clone, Debug)] #[must_use = "iterators are lazy and do nothing unless consumed"] #[stable(feature = "rust1", since = "1.0.0")] pub struct Chain { - a: A, - b: B, - state: ChainState, + // These are "fused" with `Option` so we don't need separate state to track which part is + // already exhausted, and we may also get niche layout for `None`. We don't use the real `Fuse` + // adapter because its specialization for `FusedIterator` unconditionally descends into the + // iterator, and that could be expensive to keep revisiting stuff like nested chains. It also + // hurts compiler performance to add more iterator layers to `Chain`. + // + // Only the "first" iterator is actually set `None` when exhausted, depending on whether you + // iterate forward or backward. If you mix directions, then both sides may be `None`. + a: Option, + b: Option, } impl Chain { pub(in super::super) fn new(a: A, b: B) -> Chain { - Chain { a, b, state: ChainState::Both } + Chain { a: Some(a), b: Some(b) } } } -// The iterator protocol specifies that iteration ends with the return value -// `None` from `.next()` (or `.next_back()`) and it is unspecified what -// further calls return. The chain adaptor must account for this since it uses -// two subiterators. -// -// It uses three states: -// -// - Both: `a` and `b` are remaining -// - Front: `a` remaining -// - Back: `b` remaining -// -// The fourth state (neither iterator is remaining) only occurs after Chain has -// returned None once, so we don't need to store this state. -#[derive(Clone, Debug)] -enum ChainState { - // both front and back iterator are remaining - Both, - // only front is remaining - Front, - // only back is remaining - Back, -} - #[stable(feature = "rust1", since = "1.0.0")] impl Iterator for Chain where @@ -57,88 +47,91 @@ where #[inline] fn next(&mut self) -> Option { - match self.state { - ChainState::Both => match self.a.next() { - elt @ Some(..) => elt, - None => { - self.state = ChainState::Back; - self.b.next() - } - }, - ChainState::Front => self.a.next(), - ChainState::Back => self.b.next(), - } + and_then_or_clear(&mut self.a, Iterator::next).or_else(|| self.b.as_mut()?.next()) } #[inline] #[rustc_inherit_overflow_checks] fn count(self) -> usize { - match self.state { - ChainState::Both => self.a.count() + self.b.count(), - ChainState::Front => self.a.count(), - ChainState::Back => self.b.count(), - } + let a_count = match self.a { + Some(a) => a.count(), + None => 0, + }; + let b_count = match self.b { + Some(b) => b.count(), + None => 0, + }; + a_count + b_count } - fn try_fold(&mut self, init: Acc, mut f: F) -> R + fn try_fold(&mut self, mut acc: Acc, mut f: F) -> R where Self: Sized, F: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { - let mut accum = init; - match self.state { - ChainState::Both | ChainState::Front => { - accum = self.a.try_fold(accum, &mut f)?; - if let ChainState::Both = self.state { - self.state = ChainState::Back; - } - } - _ => {} + if let Some(ref mut a) = self.a { + acc = a.try_fold(acc, &mut f)?; + self.a = None; } - if let ChainState::Back = self.state { - accum = self.b.try_fold(accum, &mut f)?; + if let Some(ref mut b) = self.b { + acc = b.try_fold(acc, f)?; + // we don't fuse the second iterator } - Try::from_ok(accum) + try { acc } } - fn fold(self, init: Acc, mut f: F) -> Acc + fn fold(self, mut acc: Acc, mut f: F) -> Acc where F: FnMut(Acc, Self::Item) -> Acc, { - let mut accum = init; - match self.state { - ChainState::Both | ChainState::Front => { - accum = self.a.fold(accum, &mut f); + if let Some(a) = self.a { + acc = a.fold(acc, &mut f); + } + if let Some(b) = self.b { + acc = b.fold(acc, f); + } + acc + } + + #[inline] + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + let mut rem = n; + + if let Some(ref mut a) = self.a { + match a.advance_by(rem) { + Ok(()) => return Ok(()), + Err(k) => rem -= k, } - _ => {} + self.a = None; } - match self.state { - ChainState::Both | ChainState::Back => { - accum = self.b.fold(accum, &mut f); + + if let Some(ref mut b) = self.b { + match b.advance_by(rem) { + Ok(()) => return Ok(()), + Err(k) => rem -= k, } - _ => {} + // we don't fuse the second iterator } - accum + + if rem == 0 { Ok(()) } else { Err(n - rem) } } #[inline] - fn nth(&mut self, mut n: usize) -> Option { - match self.state { - ChainState::Both | ChainState::Front => { - for x in self.a.by_ref() { - if n == 0 { - return Some(x); - } - n -= 1; - } - if let ChainState::Both = self.state { - self.state = ChainState::Back; - } + fn nth(&mut self, mut n: usize) -> Option { + if let Some(ref mut a) = self.a { + match a.advance_by(n) { + Ok(()) => match a.next() { + None => n = 0, + x => return x, + }, + Err(k) => n -= k, } - ChainState::Back => {} + + self.a = None; } - if let ChainState::Back = self.state { self.b.nth(n) } else { None } + + self.b.as_mut()?.nth(n) } #[inline] @@ -146,39 +139,24 @@ where where P: FnMut(&Self::Item) -> bool, { - match self.state { - ChainState::Both => match self.a.find(&mut predicate) { - None => { - self.state = ChainState::Back; - self.b.find(predicate) - } - v => v, - }, - ChainState::Front => self.a.find(predicate), - ChainState::Back => self.b.find(predicate), - } + and_then_or_clear(&mut self.a, |a| a.find(&mut predicate)) + .or_else(|| self.b.as_mut()?.find(predicate)) } #[inline] fn last(self) -> Option { - match self.state { - ChainState::Both => { - // Must exhaust a before b. - let a_last = self.a.last(); - let b_last = self.b.last(); - b_last.or(a_last) - } - ChainState::Front => self.a.last(), - ChainState::Back => self.b.last(), - } + // Must exhaust a before b. + let a_last = self.a.and_then(Iterator::last); + let b_last = self.b.and_then(Iterator::last); + b_last.or(a_last) } #[inline] fn size_hint(&self) -> (usize, Option) { - match self.state { - ChainState::Both => { - let (a_lower, a_upper) = self.a.size_hint(); - let (b_lower, b_upper) = self.b.size_hint(); + match self { + Chain { a: Some(a), b: Some(b) } => { + let (a_lower, a_upper) = a.size_hint(); + let (b_lower, b_upper) = b.size_hint(); let lower = a_lower.saturating_add(b_lower); @@ -189,8 +167,9 @@ where (lower, upper) } - ChainState::Front => self.a.size_hint(), - ChainState::Back => self.b.size_hint(), + Chain { a: Some(a), b: None } => a.size_hint(), + Chain { a: None, b: Some(b) } => b.size_hint(), + Chain { a: None, b: None } => (0, Some(0)), } } } @@ -203,78 +182,86 @@ where { #[inline] fn next_back(&mut self) -> Option { - match self.state { - ChainState::Both => match self.b.next_back() { - elt @ Some(..) => elt, - None => { - self.state = ChainState::Front; - self.a.next_back() - } - }, - ChainState::Front => self.a.next_back(), - ChainState::Back => self.b.next_back(), + and_then_or_clear(&mut self.b, |b| b.next_back()).or_else(|| self.a.as_mut()?.next_back()) + } + + #[inline] + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + let mut rem = n; + + if let Some(ref mut b) = self.b { + match b.advance_back_by(rem) { + Ok(()) => return Ok(()), + Err(k) => rem -= k, + } + self.b = None; } + + if let Some(ref mut a) = self.a { + match a.advance_back_by(rem) { + Ok(()) => return Ok(()), + Err(k) => rem -= k, + } + // we don't fuse the second iterator + } + + if rem == 0 { Ok(()) } else { Err(n - rem) } } #[inline] - fn nth_back(&mut self, mut n: usize) -> Option { - match self.state { - ChainState::Both | ChainState::Back => { - for x in self.b.by_ref().rev() { - if n == 0 { - return Some(x); - } - n -= 1; - } - if let ChainState::Both = self.state { - self.state = ChainState::Front; - } + fn nth_back(&mut self, mut n: usize) -> Option { + if let Some(ref mut b) = self.b { + match b.advance_back_by(n) { + Ok(()) => match b.next_back() { + None => n = 0, + x => return x, + }, + Err(k) => n -= k, } - ChainState::Front => {} + + self.b = None; } - if let ChainState::Front = self.state { self.a.nth_back(n) } else { None } + + self.a.as_mut()?.nth_back(n) + } + + #[inline] + fn rfind

(&mut self, mut predicate: P) -> Option + where + P: FnMut(&Self::Item) -> bool, + { + and_then_or_clear(&mut self.b, |b| b.rfind(&mut predicate)) + .or_else(|| self.a.as_mut()?.rfind(predicate)) } - fn try_rfold(&mut self, init: Acc, mut f: F) -> R + fn try_rfold(&mut self, mut acc: Acc, mut f: F) -> R where Self: Sized, F: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { - let mut accum = init; - match self.state { - ChainState::Both | ChainState::Back => { - accum = self.b.try_rfold(accum, &mut f)?; - if let ChainState::Both = self.state { - self.state = ChainState::Front; - } - } - _ => {} + if let Some(ref mut b) = self.b { + acc = b.try_rfold(acc, &mut f)?; + self.b = None; } - if let ChainState::Front = self.state { - accum = self.a.try_rfold(accum, &mut f)?; + if let Some(ref mut a) = self.a { + acc = a.try_rfold(acc, f)?; + // we don't fuse the second iterator } - Try::from_ok(accum) + try { acc } } - fn rfold(self, init: Acc, mut f: F) -> Acc + fn rfold(self, mut acc: Acc, mut f: F) -> Acc where F: FnMut(Acc, Self::Item) -> Acc, { - let mut accum = init; - match self.state { - ChainState::Both | ChainState::Back => { - accum = self.b.rfold(accum, &mut f); - } - _ => {} + if let Some(b) = self.b { + acc = b.rfold(acc, &mut f); } - match self.state { - ChainState::Both | ChainState::Front => { - accum = self.a.rfold(accum, &mut f); - } - _ => {} + if let Some(a) = self.a { + acc = a.rfold(acc, f); } - accum + acc } } @@ -294,3 +281,12 @@ where B: TrustedLen, { } + +#[inline] +fn and_then_or_clear(opt: &mut Option, f: impl FnOnce(&mut T) -> Option) -> Option { + let x = f(opt.as_mut()?); + if x.is_none() { + *opt = None; + } + x +} diff --git a/crux-mir/lib/core/src/iter/adapters/cloned.rs b/crux-mir/lib/core/src/iter/adapters/cloned.rs new file mode 100644 index 000000000..aba24a79d --- /dev/null +++ b/crux-mir/lib/core/src/iter/adapters/cloned.rs @@ -0,0 +1,142 @@ +use crate::iter::adapters::{ + zip::try_get_unchecked, TrustedRandomAccess, TrustedRandomAccessNoCoerce, +}; +use crate::iter::{FusedIterator, TrustedLen}; +use crate::ops::Try; + +/// An iterator that clones the elements of an underlying iterator. +/// +/// This `struct` is created by the [`cloned`] method on [`Iterator`]. See its +/// documentation for more. +/// +/// [`cloned`]: Iterator::cloned +/// [`Iterator`]: trait.Iterator.html +#[stable(feature = "iter_cloned", since = "1.1.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[derive(Clone, Debug)] +pub struct Cloned { + it: I, +} + +impl Cloned { + pub(in crate::iter) fn new(it: I) -> Cloned { + Cloned { it } + } +} + +fn clone_try_fold(mut f: impl FnMut(Acc, T) -> R) -> impl FnMut(Acc, &T) -> R { + move |acc, elt| f(acc, elt.clone()) +} + +#[stable(feature = "iter_cloned", since = "1.1.0")] +impl<'a, I, T: 'a> Iterator for Cloned +where + I: Iterator, + T: Clone, +{ + type Item = T; + + fn next(&mut self) -> Option { + self.it.next().cloned() + } + + fn size_hint(&self) -> (usize, Option) { + self.it.size_hint() + } + + fn try_fold(&mut self, init: B, f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: Try, + { + self.it.try_fold(init, clone_try_fold(f)) + } + + fn fold(self, init: Acc, f: F) -> Acc + where + F: FnMut(Acc, Self::Item) -> Acc, + { + self.it.map(T::clone).fold(init, f) + } + + unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> T + where + Self: TrustedRandomAccessNoCoerce, + { + // SAFETY: the caller must uphold the contract for + // `Iterator::__iterator_get_unchecked`. + unsafe { try_get_unchecked(&mut self.it, idx).clone() } + } +} + +#[stable(feature = "iter_cloned", since = "1.1.0")] +impl<'a, I, T: 'a> DoubleEndedIterator for Cloned +where + I: DoubleEndedIterator, + T: Clone, +{ + fn next_back(&mut self) -> Option { + self.it.next_back().cloned() + } + + fn try_rfold(&mut self, init: B, f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: Try, + { + self.it.try_rfold(init, clone_try_fold(f)) + } + + fn rfold(self, init: Acc, f: F) -> Acc + where + F: FnMut(Acc, Self::Item) -> Acc, + { + self.it.map(T::clone).rfold(init, f) + } +} + +#[stable(feature = "iter_cloned", since = "1.1.0")] +impl<'a, I, T: 'a> ExactSizeIterator for Cloned +where + I: ExactSizeIterator, + T: Clone, +{ + fn len(&self) -> usize { + self.it.len() + } + + fn is_empty(&self) -> bool { + self.it.is_empty() + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl<'a, I, T: 'a> FusedIterator for Cloned +where + I: FusedIterator, + T: Clone, +{ +} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl TrustedRandomAccess for Cloned where I: TrustedRandomAccess {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl TrustedRandomAccessNoCoerce for Cloned +where + I: TrustedRandomAccessNoCoerce, +{ + const MAY_HAVE_SIDE_EFFECT: bool = true; +} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl<'a, I, T: 'a> TrustedLen for Cloned +where + I: TrustedLen, + T: Clone, +{ +} diff --git a/crux-mir/lib/core/src/iter/adapters/copied.rs b/crux-mir/lib/core/src/iter/adapters/copied.rs new file mode 100644 index 000000000..62d3afb81 --- /dev/null +++ b/crux-mir/lib/core/src/iter/adapters/copied.rs @@ -0,0 +1,242 @@ +use crate::iter::adapters::{ + zip::try_get_unchecked, TrustedRandomAccess, TrustedRandomAccessNoCoerce, +}; +use crate::iter::{FusedIterator, TrustedLen}; +use crate::mem::MaybeUninit; +use crate::mem::SizedTypeProperties; +use crate::ops::Try; +use crate::{array, ptr}; + +/// An iterator that copies the elements of an underlying iterator. +/// +/// This `struct` is created by the [`copied`] method on [`Iterator`]. See its +/// documentation for more. +/// +/// [`copied`]: Iterator::copied +/// [`Iterator`]: trait.Iterator.html +#[stable(feature = "iter_copied", since = "1.36.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[derive(Clone, Debug)] +pub struct Copied { + it: I, +} + +impl Copied { + pub(in crate::iter) fn new(it: I) -> Copied { + Copied { it } + } +} + +fn copy_fold(mut f: impl FnMut(Acc, T) -> Acc) -> impl FnMut(Acc, &T) -> Acc { + move |acc, &elt| f(acc, elt) +} + +fn copy_try_fold(mut f: impl FnMut(Acc, T) -> R) -> impl FnMut(Acc, &T) -> R { + move |acc, &elt| f(acc, elt) +} + +#[stable(feature = "iter_copied", since = "1.36.0")] +impl<'a, I, T: 'a> Iterator for Copied +where + I: Iterator, + T: Copy, +{ + type Item = T; + + fn next(&mut self) -> Option { + self.it.next().copied() + } + + fn next_chunk( + &mut self, + ) -> Result<[Self::Item; N], array::IntoIter> + where + Self: Sized, + { + >::spec_next_chunk(&mut self.it) + } + + fn size_hint(&self) -> (usize, Option) { + self.it.size_hint() + } + + fn try_fold(&mut self, init: B, f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: Try, + { + self.it.try_fold(init, copy_try_fold(f)) + } + + fn fold(self, init: Acc, f: F) -> Acc + where + F: FnMut(Acc, Self::Item) -> Acc, + { + self.it.fold(init, copy_fold(f)) + } + + fn nth(&mut self, n: usize) -> Option { + self.it.nth(n).copied() + } + + fn last(self) -> Option { + self.it.last().copied() + } + + fn count(self) -> usize { + self.it.count() + } + + #[inline] + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + self.it.advance_by(n) + } + + unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> T + where + Self: TrustedRandomAccessNoCoerce, + { + // SAFETY: the caller must uphold the contract for + // `Iterator::__iterator_get_unchecked`. + *unsafe { try_get_unchecked(&mut self.it, idx) } + } +} + +#[stable(feature = "iter_copied", since = "1.36.0")] +impl<'a, I, T: 'a> DoubleEndedIterator for Copied +where + I: DoubleEndedIterator, + T: Copy, +{ + fn next_back(&mut self) -> Option { + self.it.next_back().copied() + } + + fn try_rfold(&mut self, init: B, f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: Try, + { + self.it.try_rfold(init, copy_try_fold(f)) + } + + fn rfold(self, init: Acc, f: F) -> Acc + where + F: FnMut(Acc, Self::Item) -> Acc, + { + self.it.rfold(init, copy_fold(f)) + } + + #[inline] + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + self.it.advance_back_by(n) + } +} + +#[stable(feature = "iter_copied", since = "1.36.0")] +impl<'a, I, T: 'a> ExactSizeIterator for Copied +where + I: ExactSizeIterator, + T: Copy, +{ + fn len(&self) -> usize { + self.it.len() + } + + fn is_empty(&self) -> bool { + self.it.is_empty() + } +} + +#[stable(feature = "iter_copied", since = "1.36.0")] +impl<'a, I, T: 'a> FusedIterator for Copied +where + I: FusedIterator, + T: Copy, +{ +} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl TrustedRandomAccess for Copied where I: TrustedRandomAccess {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl TrustedRandomAccessNoCoerce for Copied +where + I: TrustedRandomAccessNoCoerce, +{ + const MAY_HAVE_SIDE_EFFECT: bool = I::MAY_HAVE_SIDE_EFFECT; +} + +#[stable(feature = "iter_copied", since = "1.36.0")] +unsafe impl<'a, I, T: 'a> TrustedLen for Copied +where + I: TrustedLen, + T: Copy, +{ +} + +trait SpecNextChunk<'a, const N: usize, T: 'a>: Iterator +where + T: Copy, +{ + fn spec_next_chunk(&mut self) -> Result<[T; N], array::IntoIter>; +} + +impl<'a, const N: usize, I, T: 'a> SpecNextChunk<'a, N, T> for I +where + I: Iterator, + T: Copy, +{ + default fn spec_next_chunk(&mut self) -> Result<[T; N], array::IntoIter> { + array::iter_next_chunk(&mut self.map(|e| *e)) + } +} + +impl<'a, const N: usize, T: 'a> SpecNextChunk<'a, N, T> for crate::slice::Iter<'a, T> +where + T: Copy, +{ + fn spec_next_chunk(&mut self) -> Result<[T; N], array::IntoIter> { + let mut raw_array = MaybeUninit::uninit_array(); + + let len = self.len(); + + if T::IS_ZST { + if len < N { + let _ = self.advance_by(len); + // SAFETY: ZSTs can be conjured ex nihilo; only the amount has to be correct + return Err(unsafe { array::IntoIter::new_unchecked(raw_array, 0..len) }); + } + + let _ = self.advance_by(N); + // SAFETY: ditto + return Ok(unsafe { MaybeUninit::array_assume_init(raw_array) }); + } + + if len < N { + // SAFETY: `len` indicates that this many elements are available and we just checked that + // it fits into the array. + unsafe { + ptr::copy_nonoverlapping( + self.as_ref().as_ptr(), + raw_array.as_mut_ptr() as *mut T, + len, + ); + let _ = self.advance_by(len); + return Err(array::IntoIter::new_unchecked(raw_array, 0..len)); + } + } + + // SAFETY: `len` is larger than the array size. Copy a fixed amount here to fully initialize + // the array. + unsafe { + ptr::copy_nonoverlapping(self.as_ref().as_ptr(), raw_array.as_mut_ptr() as *mut T, N); + let _ = self.advance_by(N); + Ok(MaybeUninit::array_assume_init(raw_array)) + } + } +} diff --git a/crux-mir/lib/core/src/iter/adapters/cycle.rs b/crux-mir/lib/core/src/iter/adapters/cycle.rs new file mode 100644 index 000000000..02b593907 --- /dev/null +++ b/crux-mir/lib/core/src/iter/adapters/cycle.rs @@ -0,0 +1,108 @@ +use crate::{iter::FusedIterator, ops::Try}; + +/// An iterator that repeats endlessly. +/// +/// This `struct` is created by the [`cycle`] method on [`Iterator`]. See its +/// documentation for more. +/// +/// [`cycle`]: Iterator::cycle +/// [`Iterator`]: trait.Iterator.html +#[derive(Clone, Debug)] +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "rust1", since = "1.0.0")] +pub struct Cycle { + orig: I, + iter: I, +} + +impl Cycle { + pub(in crate::iter) fn new(iter: I) -> Cycle { + Cycle { orig: iter.clone(), iter } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for Cycle +where + I: Clone + Iterator, +{ + type Item = ::Item; + + #[inline] + fn next(&mut self) -> Option<::Item> { + match self.iter.next() { + None => { + self.iter = self.orig.clone(); + self.iter.next() + } + y => y, + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + // the cycle iterator is either empty or infinite + match self.orig.size_hint() { + sz @ (0, Some(0)) => sz, + (0, _) => (0, None), + _ => (usize::MAX, None), + } + } + + #[inline] + fn try_fold(&mut self, mut acc: Acc, mut f: F) -> R + where + F: FnMut(Acc, Self::Item) -> R, + R: Try, + { + // fully iterate the current iterator. this is necessary because + // `self.iter` may be empty even when `self.orig` isn't + acc = self.iter.try_fold(acc, &mut f)?; + self.iter = self.orig.clone(); + + // complete a full cycle, keeping track of whether the cycled + // iterator is empty or not. we need to return early in case + // of an empty iterator to prevent an infinite loop + let mut is_empty = true; + acc = self.iter.try_fold(acc, |acc, x| { + is_empty = false; + f(acc, x) + })?; + + if is_empty { + return try { acc }; + } + + loop { + self.iter = self.orig.clone(); + acc = self.iter.try_fold(acc, &mut f)?; + } + } + + #[inline] + #[rustc_inherit_overflow_checks] + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + let mut rem = n; + match self.iter.advance_by(rem) { + ret @ Ok(_) => return ret, + Err(advanced) => rem -= advanced, + } + + while rem > 0 { + self.iter = self.orig.clone(); + match self.iter.advance_by(rem) { + ret @ Ok(_) => return ret, + Err(0) => return Err(n - rem), + Err(advanced) => rem -= advanced, + } + } + + Ok(()) + } + + // No `fold` override, because `fold` doesn't make much sense for `Cycle`, + // and we can't do anything better than the default. +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Cycle where I: Clone + Iterator {} diff --git a/crux-mir/lib/core/src/iter/adapters/enumerate.rs b/crux-mir/lib/core/src/iter/adapters/enumerate.rs new file mode 100644 index 000000000..14a126951 --- /dev/null +++ b/crux-mir/lib/core/src/iter/adapters/enumerate.rs @@ -0,0 +1,266 @@ +use crate::iter::adapters::{ + zip::try_get_unchecked, SourceIter, TrustedRandomAccess, TrustedRandomAccessNoCoerce, +}; +use crate::iter::{FusedIterator, InPlaceIterable, TrustedLen}; +use crate::ops::Try; + +/// An iterator that yields the current count and the element during iteration. +/// +/// This `struct` is created by the [`enumerate`] method on [`Iterator`]. See its +/// documentation for more. +/// +/// [`enumerate`]: Iterator::enumerate +/// [`Iterator`]: trait.Iterator.html +#[derive(Clone, Debug)] +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "rust1", since = "1.0.0")] +pub struct Enumerate { + iter: I, + count: usize, +} +impl Enumerate { + pub(in crate::iter) fn new(iter: I) -> Enumerate { + Enumerate { iter, count: 0 } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for Enumerate +where + I: Iterator, +{ + type Item = (usize, ::Item); + + /// # Overflow Behavior + /// + /// The method does no guarding against overflows, so enumerating more than + /// `usize::MAX` elements either produces the wrong result or panics. If + /// debug assertions are enabled, a panic is guaranteed. + /// + /// # Panics + /// + /// Might panic if the index of the element overflows a `usize`. + #[inline] + #[rustc_inherit_overflow_checks] + fn next(&mut self) -> Option<(usize, ::Item)> { + let a = self.iter.next()?; + let i = self.count; + self.count += 1; + Some((i, a)) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } + + #[inline] + #[rustc_inherit_overflow_checks] + fn nth(&mut self, n: usize) -> Option<(usize, I::Item)> { + let a = self.iter.nth(n)?; + let i = self.count + n; + self.count = i + 1; + Some((i, a)) + } + + #[inline] + fn count(self) -> usize { + self.iter.count() + } + + #[inline] + fn try_fold(&mut self, init: Acc, fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + { + #[inline] + fn enumerate<'a, T, Acc, R>( + count: &'a mut usize, + mut fold: impl FnMut(Acc, (usize, T)) -> R + 'a, + ) -> impl FnMut(Acc, T) -> R + 'a { + #[rustc_inherit_overflow_checks] + move |acc, item| { + let acc = fold(acc, (*count, item)); + *count += 1; + acc + } + } + + self.iter.try_fold(init, enumerate(&mut self.count, fold)) + } + + #[inline] + fn fold(self, init: Acc, fold: Fold) -> Acc + where + Fold: FnMut(Acc, Self::Item) -> Acc, + { + #[inline] + fn enumerate( + mut count: usize, + mut fold: impl FnMut(Acc, (usize, T)) -> Acc, + ) -> impl FnMut(Acc, T) -> Acc { + #[rustc_inherit_overflow_checks] + move |acc, item| { + let acc = fold(acc, (count, item)); + count += 1; + acc + } + } + + self.iter.fold(init, enumerate(self.count, fold)) + } + + #[inline] + #[rustc_inherit_overflow_checks] + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + match self.iter.advance_by(n) { + ret @ Ok(_) => { + self.count += n; + ret + } + ret @ Err(advanced) => { + self.count += advanced; + ret + } + } + } + + #[rustc_inherit_overflow_checks] + #[inline] + unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> ::Item + where + Self: TrustedRandomAccessNoCoerce, + { + // SAFETY: the caller must uphold the contract for + // `Iterator::__iterator_get_unchecked`. + let value = unsafe { try_get_unchecked(&mut self.iter, idx) }; + (self.count + idx, value) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl DoubleEndedIterator for Enumerate +where + I: ExactSizeIterator + DoubleEndedIterator, +{ + #[inline] + fn next_back(&mut self) -> Option<(usize, ::Item)> { + let a = self.iter.next_back()?; + let len = self.iter.len(); + // Can safely add, `ExactSizeIterator` promises that the number of + // elements fits into a `usize`. + Some((self.count + len, a)) + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option<(usize, ::Item)> { + let a = self.iter.nth_back(n)?; + let len = self.iter.len(); + // Can safely add, `ExactSizeIterator` promises that the number of + // elements fits into a `usize`. + Some((self.count + len, a)) + } + + #[inline] + fn try_rfold(&mut self, init: Acc, fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + { + // Can safely add and subtract the count, as `ExactSizeIterator` promises + // that the number of elements fits into a `usize`. + fn enumerate( + mut count: usize, + mut fold: impl FnMut(Acc, (usize, T)) -> R, + ) -> impl FnMut(Acc, T) -> R { + move |acc, item| { + count -= 1; + fold(acc, (count, item)) + } + } + + let count = self.count + self.iter.len(); + self.iter.try_rfold(init, enumerate(count, fold)) + } + + #[inline] + fn rfold(self, init: Acc, fold: Fold) -> Acc + where + Fold: FnMut(Acc, Self::Item) -> Acc, + { + // Can safely add and subtract the count, as `ExactSizeIterator` promises + // that the number of elements fits into a `usize`. + fn enumerate( + mut count: usize, + mut fold: impl FnMut(Acc, (usize, T)) -> Acc, + ) -> impl FnMut(Acc, T) -> Acc { + move |acc, item| { + count -= 1; + fold(acc, (count, item)) + } + } + + let count = self.count + self.iter.len(); + self.iter.rfold(init, enumerate(count, fold)) + } + + #[inline] + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + // we do not need to update the count since that only tallies the number of items + // consumed from the front. consuming items from the back can never reduce that. + self.iter.advance_back_by(n) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ExactSizeIterator for Enumerate +where + I: ExactSizeIterator, +{ + fn len(&self) -> usize { + self.iter.len() + } + + fn is_empty(&self) -> bool { + self.iter.is_empty() + } +} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl TrustedRandomAccess for Enumerate where I: TrustedRandomAccess {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl TrustedRandomAccessNoCoerce for Enumerate +where + I: TrustedRandomAccessNoCoerce, +{ + const MAY_HAVE_SIDE_EFFECT: bool = I::MAY_HAVE_SIDE_EFFECT; +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Enumerate where I: FusedIterator {} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for Enumerate where I: TrustedLen {} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl SourceIter for Enumerate +where + I: SourceIter, +{ + type Source = I::Source; + + #[inline] + unsafe fn as_inner(&mut self) -> &mut I::Source { + // SAFETY: unsafe function forwarding to unsafe function with the same requirements + unsafe { SourceIter::as_inner(&mut self.iter) } + } +} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for Enumerate {} diff --git a/crux-mir/lib/core/src/iter/adapters/filter.rs b/crux-mir/lib/core/src/iter/adapters/filter.rs new file mode 100644 index 000000000..a0afaa326 --- /dev/null +++ b/crux-mir/lib/core/src/iter/adapters/filter.rs @@ -0,0 +1,152 @@ +use crate::fmt; +use crate::iter::{adapters::SourceIter, FusedIterator, InPlaceIterable}; +use crate::ops::Try; + +/// An iterator that filters the elements of `iter` with `predicate`. +/// +/// This `struct` is created by the [`filter`] method on [`Iterator`]. See its +/// documentation for more. +/// +/// [`filter`]: Iterator::filter +/// [`Iterator`]: trait.Iterator.html +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "rust1", since = "1.0.0")] +#[derive(Clone)] +pub struct Filter { + // Used for `SplitWhitespace` and `SplitAsciiWhitespace` `as_str` methods + pub(crate) iter: I, + predicate: P, +} +impl Filter { + pub(in crate::iter) fn new(iter: I, predicate: P) -> Filter { + Filter { iter, predicate } + } +} + +#[stable(feature = "core_impl_debug", since = "1.9.0")] +impl fmt::Debug for Filter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Filter").field("iter", &self.iter).finish() + } +} + +fn filter_fold( + mut predicate: impl FnMut(&T) -> bool, + mut fold: impl FnMut(Acc, T) -> Acc, +) -> impl FnMut(Acc, T) -> Acc { + move |acc, item| if predicate(&item) { fold(acc, item) } else { acc } +} + +fn filter_try_fold<'a, T, Acc, R: Try>( + predicate: &'a mut impl FnMut(&T) -> bool, + mut fold: impl FnMut(Acc, T) -> R + 'a, +) -> impl FnMut(Acc, T) -> R + 'a { + move |acc, item| if predicate(&item) { fold(acc, item) } else { try { acc } } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for Filter +where + P: FnMut(&I::Item) -> bool, +{ + type Item = I::Item; + + #[inline] + fn next(&mut self) -> Option { + self.iter.find(&mut self.predicate) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let (_, upper) = self.iter.size_hint(); + (0, upper) // can't know a lower bound, due to the predicate + } + + // this special case allows the compiler to make `.filter(_).count()` + // branchless. Barring perfect branch prediction (which is unattainable in + // the general case), this will be much faster in >90% of cases (containing + // virtually all real workloads) and only a tiny bit slower in the rest. + // + // Having this specialization thus allows us to write `.filter(p).count()` + // where we would otherwise write `.map(|x| p(x) as usize).sum()`, which is + // less readable and also less backwards-compatible to Rust before 1.10. + // + // Using the branchless version will also simplify the LLVM byte code, thus + // leaving more budget for LLVM optimizations. + #[inline] + fn count(self) -> usize { + #[inline] + fn to_usize(mut predicate: impl FnMut(&T) -> bool) -> impl FnMut(T) -> usize { + move |x| predicate(&x) as usize + } + + self.iter.map(to_usize(self.predicate)).sum() + } + + #[inline] + fn try_fold(&mut self, init: Acc, fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + { + self.iter.try_fold(init, filter_try_fold(&mut self.predicate, fold)) + } + + #[inline] + fn fold(self, init: Acc, fold: Fold) -> Acc + where + Fold: FnMut(Acc, Self::Item) -> Acc, + { + self.iter.fold(init, filter_fold(self.predicate, fold)) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl DoubleEndedIterator for Filter +where + P: FnMut(&I::Item) -> bool, +{ + #[inline] + fn next_back(&mut self) -> Option { + self.iter.rfind(&mut self.predicate) + } + + #[inline] + fn try_rfold(&mut self, init: Acc, fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + { + self.iter.try_rfold(init, filter_try_fold(&mut self.predicate, fold)) + } + + #[inline] + fn rfold(self, init: Acc, fold: Fold) -> Acc + where + Fold: FnMut(Acc, Self::Item) -> Acc, + { + self.iter.rfold(init, filter_fold(self.predicate, fold)) + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Filter where P: FnMut(&I::Item) -> bool {} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl SourceIter for Filter +where + I: SourceIter, +{ + type Source = I::Source; + + #[inline] + unsafe fn as_inner(&mut self) -> &mut I::Source { + // SAFETY: unsafe function forwarding to unsafe function with the same requirements + unsafe { SourceIter::as_inner(&mut self.iter) } + } +} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for Filter where P: FnMut(&I::Item) -> bool {} diff --git a/crux-mir/lib/core/src/iter/adapters/filter_map.rs b/crux-mir/lib/core/src/iter/adapters/filter_map.rs new file mode 100644 index 000000000..e0d665c9e --- /dev/null +++ b/crux-mir/lib/core/src/iter/adapters/filter_map.rs @@ -0,0 +1,149 @@ +use crate::fmt; +use crate::iter::{adapters::SourceIter, FusedIterator, InPlaceIterable}; +use crate::ops::{ControlFlow, Try}; + +/// An iterator that uses `f` to both filter and map elements from `iter`. +/// +/// This `struct` is created by the [`filter_map`] method on [`Iterator`]. See its +/// documentation for more. +/// +/// [`filter_map`]: Iterator::filter_map +/// [`Iterator`]: trait.Iterator.html +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "rust1", since = "1.0.0")] +#[derive(Clone)] +pub struct FilterMap { + iter: I, + f: F, +} +impl FilterMap { + pub(in crate::iter) fn new(iter: I, f: F) -> FilterMap { + FilterMap { iter, f } + } +} + +#[stable(feature = "core_impl_debug", since = "1.9.0")] +impl fmt::Debug for FilterMap { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("FilterMap").field("iter", &self.iter).finish() + } +} + +fn filter_map_fold( + mut f: impl FnMut(T) -> Option, + mut fold: impl FnMut(Acc, B) -> Acc, +) -> impl FnMut(Acc, T) -> Acc { + move |acc, item| match f(item) { + Some(x) => fold(acc, x), + None => acc, + } +} + +fn filter_map_try_fold<'a, T, B, Acc, R: Try>( + f: &'a mut impl FnMut(T) -> Option, + mut fold: impl FnMut(Acc, B) -> R + 'a, +) -> impl FnMut(Acc, T) -> R + 'a { + move |acc, item| match f(item) { + Some(x) => fold(acc, x), + None => try { acc }, + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for FilterMap +where + F: FnMut(I::Item) -> Option, +{ + type Item = B; + + #[inline] + fn next(&mut self) -> Option { + self.iter.find_map(&mut self.f) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let (_, upper) = self.iter.size_hint(); + (0, upper) // can't know a lower bound, due to the predicate + } + + #[inline] + fn try_fold(&mut self, init: Acc, fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + { + self.iter.try_fold(init, filter_map_try_fold(&mut self.f, fold)) + } + + #[inline] + fn fold(self, init: Acc, fold: Fold) -> Acc + where + Fold: FnMut(Acc, Self::Item) -> Acc, + { + self.iter.fold(init, filter_map_fold(self.f, fold)) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl DoubleEndedIterator for FilterMap +where + F: FnMut(I::Item) -> Option, +{ + #[inline] + fn next_back(&mut self) -> Option { + #[inline] + fn find( + f: &mut impl FnMut(T) -> Option, + ) -> impl FnMut((), T) -> ControlFlow + '_ { + move |(), x| match f(x) { + Some(x) => ControlFlow::Break(x), + None => ControlFlow::CONTINUE, + } + } + + self.iter.try_rfold((), find(&mut self.f)).break_value() + } + + #[inline] + fn try_rfold(&mut self, init: Acc, fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + { + self.iter.try_rfold(init, filter_map_try_fold(&mut self.f, fold)) + } + + #[inline] + fn rfold(self, init: Acc, fold: Fold) -> Acc + where + Fold: FnMut(Acc, Self::Item) -> Acc, + { + self.iter.rfold(init, filter_map_fold(self.f, fold)) + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for FilterMap where F: FnMut(I::Item) -> Option {} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl SourceIter for FilterMap +where + I: SourceIter, +{ + type Source = I::Source; + + #[inline] + unsafe fn as_inner(&mut self) -> &mut I::Source { + // SAFETY: unsafe function forwarding to unsafe function with the same requirements + unsafe { SourceIter::as_inner(&mut self.iter) } + } +} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for FilterMap where + F: FnMut(I::Item) -> Option +{ +} diff --git a/crux-mir/lib/core/src/iter/adapters/flatten.rs b/crux-mir/lib/core/src/iter/adapters/flatten.rs index 4202e5244..307016c26 100644 --- a/crux-mir/lib/core/src/iter/adapters/flatten.rs +++ b/crux-mir/lib/core/src/iter/adapters/flatten.rs @@ -1,24 +1,20 @@ use crate::fmt; -use crate::ops::Try; - -use super::super::{DoubleEndedIterator, Fuse, FusedIterator, Iterator}; -use super::Map; +use crate::iter::{DoubleEndedIterator, Fuse, FusedIterator, Iterator, Map, TrustedLen}; +use crate::ops::{ControlFlow, Try}; /// An iterator that maps each element to an iterator, and yields the elements /// of the produced iterators. /// -/// This `struct` is created by the [`flat_map`] method on [`Iterator`]. See its -/// documentation for more. -/// -/// [`flat_map`]: trait.Iterator.html#method.flat_map -/// [`Iterator`]: trait.Iterator.html +/// This `struct` is created by [`Iterator::flat_map`]. See its documentation +/// for more. #[must_use = "iterators are lazy and do nothing unless consumed"] #[stable(feature = "rust1", since = "1.0.0")] pub struct FlatMap { inner: FlattenCompat, ::IntoIter>, } + impl U> FlatMap { - pub(in super::super) fn new(iter: I, f: F) -> FlatMap { + pub(in crate::iter) fn new(iter: I, f: F) -> FlatMap { FlatMap { inner: FlattenCompat::new(iter.map(f)) } } } @@ -65,7 +61,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { self.inner.try_fold(init, fold) } @@ -77,6 +73,21 @@ where { self.inner.fold(init, fold) } + + #[inline] + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + self.inner.advance_by(n) + } + + #[inline] + fn count(self) -> usize { + self.inner.count() + } + + #[inline] + fn last(self) -> Option { + self.inner.last() + } } #[stable(feature = "rust1", since = "1.0.0")] @@ -95,7 +106,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { self.inner.try_rfold(init, fold) } @@ -107,6 +118,11 @@ where { self.inner.rfold(init, fold) } + + #[inline] + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + self.inner.advance_back_by(n) + } } #[stable(feature = "fused", since = "1.26.0")] @@ -118,14 +134,37 @@ where { } +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for FlatMap +where + I: TrustedLen, + F: FnMut(I::Item) -> [T; N], +{ +} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl<'a, T, I, F, const N: usize> TrustedLen for FlatMap +where + I: TrustedLen, + F: FnMut(I::Item) -> &'a [T; N], +{ +} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl<'a, T, I, F, const N: usize> TrustedLen for FlatMap +where + I: TrustedLen, + F: FnMut(I::Item) -> &'a mut [T; N], +{ +} + /// An iterator that flattens one level of nesting in an iterator of things /// that can be turned into iterators. /// /// This `struct` is created by the [`flatten`] method on [`Iterator`]. See its /// documentation for more. /// -/// [`flatten`]: trait.Iterator.html#method.flatten -/// [`Iterator`]: trait.Iterator.html +/// [`flatten`]: Iterator::flatten() #[must_use = "iterators are lazy and do nothing unless consumed"] #[stable(feature = "iterator_flatten", since = "1.29.0")] pub struct Flatten> { @@ -183,7 +222,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { self.inner.try_fold(init, fold) } @@ -195,6 +234,21 @@ where { self.inner.fold(init, fold) } + + #[inline] + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + self.inner.advance_by(n) + } + + #[inline] + fn count(self) -> usize { + self.inner.count() + } + + #[inline] + fn last(self) -> Option { + self.inner.last() + } } #[stable(feature = "iterator_flatten", since = "1.29.0")] @@ -213,7 +267,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { self.inner.try_rfold(init, fold) } @@ -225,6 +279,11 @@ where { self.inner.rfold(init, fold) } + + #[inline] + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + self.inner.advance_back_by(n) + } } #[stable(feature = "iterator_flatten", since = "1.29.0")] @@ -235,6 +294,14 @@ where { } +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for Flatten +where + I: TrustedLen, + ::Item: TrustedConstSize, +{ +} + /// Real logic of both `Flatten` and `FlatMap` which simply delegate to /// this type. #[derive(Clone, Debug)] @@ -253,6 +320,144 @@ where } } +impl FlattenCompat +where + I: Iterator>, +{ + /// Folds the inner iterators into an accumulator by applying an operation. + /// + /// Folds over the inner iterators, not over their elements. Is used by the `fold`, `count`, + /// and `last` methods. + #[inline] + fn iter_fold(self, mut acc: Acc, mut fold: Fold) -> Acc + where + Fold: FnMut(Acc, U) -> Acc, + { + #[inline] + fn flatten( + fold: &mut impl FnMut(Acc, T::IntoIter) -> Acc, + ) -> impl FnMut(Acc, T) -> Acc + '_ { + move |acc, iter| fold(acc, iter.into_iter()) + } + + if let Some(iter) = self.frontiter { + acc = fold(acc, iter); + } + + acc = self.iter.fold(acc, flatten(&mut fold)); + + if let Some(iter) = self.backiter { + acc = fold(acc, iter); + } + + acc + } + + /// Folds over the inner iterators as long as the given function returns successfully, + /// always storing the most recent inner iterator in `self.frontiter`. + /// + /// Folds over the inner iterators, not over their elements. Is used by the `try_fold` and + /// `advance_by` methods. + #[inline] + fn iter_try_fold(&mut self, mut acc: Acc, mut fold: Fold) -> R + where + Fold: FnMut(Acc, &mut U) -> R, + R: Try, + { + #[inline] + fn flatten<'a, T: IntoIterator, Acc, R: Try>( + frontiter: &'a mut Option, + fold: &'a mut impl FnMut(Acc, &mut T::IntoIter) -> R, + ) -> impl FnMut(Acc, T) -> R + 'a { + move |acc, iter| fold(acc, frontiter.insert(iter.into_iter())) + } + + if let Some(iter) = &mut self.frontiter { + acc = fold(acc, iter)?; + } + self.frontiter = None; + + acc = self.iter.try_fold(acc, flatten(&mut self.frontiter, &mut fold))?; + self.frontiter = None; + + if let Some(iter) = &mut self.backiter { + acc = fold(acc, iter)?; + } + self.backiter = None; + + try { acc } + } +} + +impl FlattenCompat +where + I: DoubleEndedIterator>, +{ + /// Folds the inner iterators into an accumulator by applying an operation, starting form the + /// back. + /// + /// Folds over the inner iterators, not over their elements. Is used by the `rfold` method. + #[inline] + fn iter_rfold(self, mut acc: Acc, mut fold: Fold) -> Acc + where + Fold: FnMut(Acc, U) -> Acc, + { + #[inline] + fn flatten( + fold: &mut impl FnMut(Acc, T::IntoIter) -> Acc, + ) -> impl FnMut(Acc, T) -> Acc + '_ { + move |acc, iter| fold(acc, iter.into_iter()) + } + + if let Some(iter) = self.backiter { + acc = fold(acc, iter); + } + + acc = self.iter.rfold(acc, flatten(&mut fold)); + + if let Some(iter) = self.frontiter { + acc = fold(acc, iter); + } + + acc + } + + /// Folds over the inner iterators in reverse order as long as the given function returns + /// successfully, always storing the most recent inner iterator in `self.backiter`. + /// + /// Folds over the inner iterators, not over their elements. Is used by the `try_rfold` and + /// `advance_back_by` methods. + #[inline] + fn iter_try_rfold(&mut self, mut acc: Acc, mut fold: Fold) -> R + where + Fold: FnMut(Acc, &mut U) -> R, + R: Try, + { + #[inline] + fn flatten<'a, T: IntoIterator, Acc, R: Try>( + backiter: &'a mut Option, + fold: &'a mut impl FnMut(Acc, &mut T::IntoIter) -> R, + ) -> impl FnMut(Acc, T) -> R + 'a { + move |acc, iter| fold(acc, backiter.insert(iter.into_iter())) + } + + if let Some(iter) = &mut self.backiter { + acc = fold(acc, iter)?; + } + self.backiter = None; + + acc = self.iter.try_rfold(acc, flatten(&mut self.backiter, &mut fold))?; + self.backiter = None; + + if let Some(iter) = &mut self.frontiter { + acc = fold(acc, iter)?; + } + self.frontiter = None; + + try { acc } + } +} + impl Iterator for FlattenCompat where I: Iterator>, @@ -263,14 +468,11 @@ where #[inline] fn next(&mut self) -> Option { loop { - if let Some(ref mut inner) = self.frontiter { - match inner.next() { - None => self.frontiter = None, - elt @ Some(_) => return elt, - } + if let elt @ Some(_) = and_then_or_clear(&mut self.frontiter, Iterator::next) { + return elt; } match self.iter.next() { - None => return self.backiter.as_mut()?.next(), + None => return and_then_or_clear(&mut self.backiter, Iterator::next), Some(inner) => self.frontiter = Some(inner.into_iter()), } } @@ -281,6 +483,17 @@ where let (flo, fhi) = self.frontiter.as_ref().map_or((0, Some(0)), U::size_hint); let (blo, bhi) = self.backiter.as_ref().map_or((0, Some(0)), U::size_hint); let lo = flo.saturating_add(blo); + + if let Some(fixed_size) = <::Item as ConstSizeIntoIterator>::size() { + let (lower, upper) = self.iter.size_hint(); + + let lower = lower.saturating_mul(fixed_size).saturating_add(lo); + let upper = + try { fhi?.checked_add(bhi?)?.checked_add(fixed_size.checked_mul(upper?)?)? }; + + return (lower, upper); + } + match (self.iter.size_hint(), fhi, bhi) { ((0, Some(0)), Some(a), Some(b)) => (lo, a.checked_add(b)), _ => (lo, None), @@ -288,58 +501,74 @@ where } #[inline] - fn try_fold(&mut self, mut init: Acc, mut fold: Fold) -> R + fn try_fold(&mut self, init: Acc, fold: Fold) -> R where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { #[inline] - fn flatten<'a, T: IntoIterator, Acc, R: Try>( - frontiter: &'a mut Option, - fold: &'a mut impl FnMut(Acc, T::Item) -> R, - ) -> impl FnMut(Acc, T) -> R + 'a { - move |acc, x| { - let mut mid = x.into_iter(); - let r = mid.try_fold(acc, &mut *fold); - *frontiter = Some(mid); - r - } + fn flatten>( + mut fold: impl FnMut(Acc, U::Item) -> R, + ) -> impl FnMut(Acc, &mut U) -> R { + move |acc, iter| iter.try_fold(acc, &mut fold) } - if let Some(ref mut front) = self.frontiter { - init = front.try_fold(init, &mut fold)?; + self.iter_try_fold(init, flatten(fold)) + } + + #[inline] + fn fold(self, init: Acc, fold: Fold) -> Acc + where + Fold: FnMut(Acc, Self::Item) -> Acc, + { + #[inline] + fn flatten( + mut fold: impl FnMut(Acc, U::Item) -> Acc, + ) -> impl FnMut(Acc, U) -> Acc { + move |acc, iter| iter.fold(acc, &mut fold) } - self.frontiter = None; - init = self.iter.try_fold(init, flatten(&mut self.frontiter, &mut fold))?; - self.frontiter = None; + self.iter_fold(init, flatten(fold)) + } - if let Some(ref mut back) = self.backiter { - init = back.try_fold(init, &mut fold)?; + #[inline] + #[rustc_inherit_overflow_checks] + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + #[inline] + #[rustc_inherit_overflow_checks] + fn advance(n: usize, iter: &mut U) -> ControlFlow<(), usize> { + match iter.advance_by(n) { + Ok(()) => ControlFlow::BREAK, + Err(advanced) => ControlFlow::Continue(n - advanced), + } } - self.backiter = None; - Try::from_ok(init) + match self.iter_try_fold(n, advance) { + ControlFlow::Continue(remaining) if remaining > 0 => Err(n - remaining), + _ => Ok(()), + } } #[inline] - fn fold(self, init: Acc, ref mut fold: Fold) -> Acc - where - Fold: FnMut(Acc, Self::Item) -> Acc, - { + fn count(self) -> usize { #[inline] - fn flatten( - fold: &mut impl FnMut(Acc, U::Item) -> Acc, - ) -> impl FnMut(Acc, U) -> Acc + '_ { - move |acc, iter| iter.fold(acc, &mut *fold) + #[rustc_inherit_overflow_checks] + fn count(acc: usize, iter: U) -> usize { + acc + iter.count() + } + + self.iter_fold(0, count) + } + + #[inline] + fn last(self) -> Option { + #[inline] + fn last(last: Option, iter: U) -> Option { + iter.last().or(last) } - self.frontiter - .into_iter() - .chain(self.iter.map(IntoIterator::into_iter)) - .chain(self.backiter) - .fold(init, flatten(fold)) + self.iter_fold(None, last) } } @@ -351,74 +580,121 @@ where #[inline] fn next_back(&mut self) -> Option { loop { - if let Some(ref mut inner) = self.backiter { - match inner.next_back() { - None => self.backiter = None, - elt @ Some(_) => return elt, - } + if let elt @ Some(_) = and_then_or_clear(&mut self.backiter, |b| b.next_back()) { + return elt; } match self.iter.next_back() { - None => return self.frontiter.as_mut()?.next_back(), - next => self.backiter = next.map(IntoIterator::into_iter), + None => return and_then_or_clear(&mut self.frontiter, |f| f.next_back()), + Some(inner) => self.backiter = Some(inner.into_iter()), } } } #[inline] - fn try_rfold(&mut self, mut init: Acc, mut fold: Fold) -> R + fn try_rfold(&mut self, init: Acc, fold: Fold) -> R where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { #[inline] - fn flatten<'a, T: IntoIterator, Acc, R: Try>( - backiter: &'a mut Option, - fold: &'a mut impl FnMut(Acc, T::Item) -> R, - ) -> impl FnMut(Acc, T) -> R + 'a - where - T::IntoIter: DoubleEndedIterator, - { - move |acc, x| { - let mut mid = x.into_iter(); - let r = mid.try_rfold(acc, &mut *fold); - *backiter = Some(mid); - r - } - } - - if let Some(ref mut back) = self.backiter { - init = back.try_rfold(init, &mut fold)?; - } - self.backiter = None; - - init = self.iter.try_rfold(init, flatten(&mut self.backiter, &mut fold))?; - self.backiter = None; - - if let Some(ref mut front) = self.frontiter { - init = front.try_rfold(init, &mut fold)?; + fn flatten>( + mut fold: impl FnMut(Acc, U::Item) -> R, + ) -> impl FnMut(Acc, &mut U) -> R { + move |acc, iter| iter.try_rfold(acc, &mut fold) } - self.frontiter = None; - Try::from_ok(init) + self.iter_try_rfold(init, flatten(fold)) } #[inline] - fn rfold(self, init: Acc, ref mut fold: Fold) -> Acc + fn rfold(self, init: Acc, fold: Fold) -> Acc where Fold: FnMut(Acc, Self::Item) -> Acc, { #[inline] fn flatten( - fold: &mut impl FnMut(Acc, U::Item) -> Acc, - ) -> impl FnMut(Acc, U) -> Acc + '_ { - move |acc, iter| iter.rfold(acc, &mut *fold) + mut fold: impl FnMut(Acc, U::Item) -> Acc, + ) -> impl FnMut(Acc, U) -> Acc { + move |acc, iter| iter.rfold(acc, &mut fold) + } + + self.iter_rfold(init, flatten(fold)) + } + + #[inline] + #[rustc_inherit_overflow_checks] + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + #[inline] + #[rustc_inherit_overflow_checks] + fn advance(n: usize, iter: &mut U) -> ControlFlow<(), usize> { + match iter.advance_back_by(n) { + Ok(()) => ControlFlow::BREAK, + Err(advanced) => ControlFlow::Continue(n - advanced), + } } - self.frontiter - .into_iter() - .chain(self.iter.map(IntoIterator::into_iter)) - .chain(self.backiter) - .rfold(init, flatten(fold)) + match self.iter_try_rfold(n, advance) { + ControlFlow::Continue(remaining) if remaining > 0 => Err(n - remaining), + _ => Ok(()), + } + } +} + +trait ConstSizeIntoIterator: IntoIterator { + // FIXME(#31844): convert to an associated const once specialization supports that + fn size() -> Option; +} + +impl ConstSizeIntoIterator for T +where + T: IntoIterator, +{ + #[inline] + default fn size() -> Option { + None + } +} + +impl ConstSizeIntoIterator for [T; N] { + #[inline] + fn size() -> Option { + Some(N) + } +} + +impl ConstSizeIntoIterator for &[T; N] { + #[inline] + fn size() -> Option { + Some(N) + } +} + +impl ConstSizeIntoIterator for &mut [T; N] { + #[inline] + fn size() -> Option { + Some(N) + } +} + +#[doc(hidden)] +#[unstable(feature = "std_internals", issue = "none")] +// FIXME(#20400): Instead of this helper trait there should be multiple impl TrustedLen for Flatten<> +// blocks with different bounds on Iterator::Item but the compiler erroneously considers them overlapping +pub unsafe trait TrustedConstSize: IntoIterator {} + +#[unstable(feature = "std_internals", issue = "none")] +unsafe impl TrustedConstSize for [T; N] {} +#[unstable(feature = "std_internals", issue = "none")] +unsafe impl TrustedConstSize for &'_ [T; N] {} +#[unstable(feature = "std_internals", issue = "none")] +unsafe impl TrustedConstSize for &'_ mut [T; N] {} + +#[inline] +fn and_then_or_clear(opt: &mut Option, f: impl FnOnce(&mut T) -> Option) -> Option { + let x = f(opt.as_mut()?); + if x.is_none() { + *opt = None; } + x } diff --git a/crux-mir/lib/core/src/iter/adapters/fuse.rs b/crux-mir/lib/core/src/iter/adapters/fuse.rs new file mode 100644 index 000000000..c93144542 --- /dev/null +++ b/crux-mir/lib/core/src/iter/adapters/fuse.rs @@ -0,0 +1,413 @@ +use crate::intrinsics; +use crate::iter::adapters::zip::try_get_unchecked; +use crate::iter::{ + DoubleEndedIterator, ExactSizeIterator, FusedIterator, TrustedLen, TrustedRandomAccess, + TrustedRandomAccessNoCoerce, +}; +use crate::ops::Try; + +/// An iterator that yields `None` forever after the underlying iterator +/// yields `None` once. +/// +/// This `struct` is created by [`Iterator::fuse`]. See its documentation +/// for more. +#[derive(Clone, Debug)] +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "rust1", since = "1.0.0")] +pub struct Fuse { + // NOTE: for `I: FusedIterator`, we never bother setting `None`, but + // we still have to be prepared for that state due to variance. + // See rust-lang/rust#85863 + iter: Option, +} +impl Fuse { + pub(in crate::iter) fn new(iter: I) -> Fuse { + Fuse { iter: Some(iter) } + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Fuse where I: Iterator {} + +// Any specialized implementation here is made internal +// to avoid exposing default fns outside this trait. +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for Fuse +where + I: Iterator, +{ + type Item = ::Item; + + #[inline] + fn next(&mut self) -> Option { + FuseImpl::next(self) + } + + #[inline] + fn nth(&mut self, n: usize) -> Option { + FuseImpl::nth(self, n) + } + + #[inline] + fn last(self) -> Option { + match self.iter { + Some(iter) => iter.last(), + None => None, + } + } + + #[inline] + fn count(self) -> usize { + match self.iter { + Some(iter) => iter.count(), + None => 0, + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + match self.iter { + Some(ref iter) => iter.size_hint(), + None => (0, Some(0)), + } + } + + #[inline] + fn try_fold(&mut self, acc: Acc, fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + { + FuseImpl::try_fold(self, acc, fold) + } + + #[inline] + fn fold(self, mut acc: Acc, fold: Fold) -> Acc + where + Fold: FnMut(Acc, Self::Item) -> Acc, + { + if let Some(iter) = self.iter { + acc = iter.fold(acc, fold); + } + acc + } + + #[inline] + fn find

(&mut self, predicate: P) -> Option + where + P: FnMut(&Self::Item) -> bool, + { + FuseImpl::find(self, predicate) + } + + #[inline] + unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> Self::Item + where + Self: TrustedRandomAccessNoCoerce, + { + match self.iter { + // SAFETY: the caller must uphold the contract for + // `Iterator::__iterator_get_unchecked`. + Some(ref mut iter) => unsafe { try_get_unchecked(iter, idx) }, + // SAFETY: the caller asserts there is an item at `i`, so we're not exhausted. + None => unsafe { intrinsics::unreachable() }, + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl DoubleEndedIterator for Fuse +where + I: DoubleEndedIterator, +{ + #[inline] + fn next_back(&mut self) -> Option<::Item> { + FuseImpl::next_back(self) + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option<::Item> { + FuseImpl::nth_back(self, n) + } + + #[inline] + fn try_rfold(&mut self, acc: Acc, fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + { + FuseImpl::try_rfold(self, acc, fold) + } + + #[inline] + fn rfold(self, mut acc: Acc, fold: Fold) -> Acc + where + Fold: FnMut(Acc, Self::Item) -> Acc, + { + if let Some(iter) = self.iter { + acc = iter.rfold(acc, fold); + } + acc + } + + #[inline] + fn rfind

(&mut self, predicate: P) -> Option + where + P: FnMut(&Self::Item) -> bool, + { + FuseImpl::rfind(self, predicate) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ExactSizeIterator for Fuse +where + I: ExactSizeIterator, +{ + fn len(&self) -> usize { + match self.iter { + Some(ref iter) => iter.len(), + None => 0, + } + } + + fn is_empty(&self) -> bool { + match self.iter { + Some(ref iter) => iter.is_empty(), + None => true, + } + } +} + +#[unstable(feature = "trusted_len", issue = "37572")] +// SAFETY: `TrustedLen` requires that an accurate length is reported via `size_hint()`. As `Fuse` +// is just forwarding this to the wrapped iterator `I` this property is preserved and it is safe to +// implement `TrustedLen` here. +unsafe impl TrustedLen for Fuse where I: TrustedLen {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +// SAFETY: `TrustedRandomAccess` requires that `size_hint()` must be exact and cheap to call, and +// `Iterator::__iterator_get_unchecked()` must be implemented accordingly. +// +// This is safe to implement as `Fuse` is just forwarding these to the wrapped iterator `I`, which +// preserves these properties. +unsafe impl TrustedRandomAccess for Fuse where I: TrustedRandomAccess {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl TrustedRandomAccessNoCoerce for Fuse +where + I: TrustedRandomAccessNoCoerce, +{ + const MAY_HAVE_SIDE_EFFECT: bool = I::MAY_HAVE_SIDE_EFFECT; +} + +/// Fuse specialization trait +/// +/// We only need to worry about `&mut self` methods, which +/// may exhaust the iterator without consuming it. +#[doc(hidden)] +trait FuseImpl { + type Item; + + // Functions specific to any normal Iterators + fn next(&mut self) -> Option; + fn nth(&mut self, n: usize) -> Option; + fn try_fold(&mut self, acc: Acc, fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try; + fn find

(&mut self, predicate: P) -> Option + where + P: FnMut(&Self::Item) -> bool; + + // Functions specific to DoubleEndedIterators + fn next_back(&mut self) -> Option + where + I: DoubleEndedIterator; + fn nth_back(&mut self, n: usize) -> Option + where + I: DoubleEndedIterator; + fn try_rfold(&mut self, acc: Acc, fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + I: DoubleEndedIterator; + fn rfind

(&mut self, predicate: P) -> Option + where + P: FnMut(&Self::Item) -> bool, + I: DoubleEndedIterator; +} + +/// General `Fuse` impl which sets `iter = None` when exhausted. +#[doc(hidden)] +impl FuseImpl for Fuse +where + I: Iterator, +{ + type Item = ::Item; + + #[inline] + default fn next(&mut self) -> Option<::Item> { + and_then_or_clear(&mut self.iter, Iterator::next) + } + + #[inline] + default fn nth(&mut self, n: usize) -> Option { + and_then_or_clear(&mut self.iter, |iter| iter.nth(n)) + } + + #[inline] + default fn try_fold(&mut self, mut acc: Acc, fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + { + if let Some(ref mut iter) = self.iter { + acc = iter.try_fold(acc, fold)?; + self.iter = None; + } + try { acc } + } + + #[inline] + default fn find

(&mut self, predicate: P) -> Option + where + P: FnMut(&Self::Item) -> bool, + { + and_then_or_clear(&mut self.iter, |iter| iter.find(predicate)) + } + + #[inline] + default fn next_back(&mut self) -> Option<::Item> + where + I: DoubleEndedIterator, + { + and_then_or_clear(&mut self.iter, |iter| iter.next_back()) + } + + #[inline] + default fn nth_back(&mut self, n: usize) -> Option<::Item> + where + I: DoubleEndedIterator, + { + and_then_or_clear(&mut self.iter, |iter| iter.nth_back(n)) + } + + #[inline] + default fn try_rfold(&mut self, mut acc: Acc, fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + I: DoubleEndedIterator, + { + if let Some(ref mut iter) = self.iter { + acc = iter.try_rfold(acc, fold)?; + self.iter = None; + } + try { acc } + } + + #[inline] + default fn rfind

(&mut self, predicate: P) -> Option + where + P: FnMut(&Self::Item) -> bool, + I: DoubleEndedIterator, + { + and_then_or_clear(&mut self.iter, |iter| iter.rfind(predicate)) + } +} + +/// Specialized `Fuse` impl which doesn't bother clearing `iter` when exhausted. +/// However, we must still be prepared for the possibility that it was already cleared! +#[doc(hidden)] +impl FuseImpl for Fuse +where + I: FusedIterator, +{ + #[inline] + fn next(&mut self) -> Option<::Item> { + self.iter.as_mut()?.next() + } + + #[inline] + fn nth(&mut self, n: usize) -> Option { + self.iter.as_mut()?.nth(n) + } + + #[inline] + fn try_fold(&mut self, mut acc: Acc, fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + { + if let Some(ref mut iter) = self.iter { + acc = iter.try_fold(acc, fold)?; + } + try { acc } + } + + #[inline] + fn find

(&mut self, predicate: P) -> Option + where + P: FnMut(&Self::Item) -> bool, + { + self.iter.as_mut()?.find(predicate) + } + + #[inline] + fn next_back(&mut self) -> Option<::Item> + where + I: DoubleEndedIterator, + { + self.iter.as_mut()?.next_back() + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option<::Item> + where + I: DoubleEndedIterator, + { + self.iter.as_mut()?.nth_back(n) + } + + #[inline] + fn try_rfold(&mut self, mut acc: Acc, fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + I: DoubleEndedIterator, + { + if let Some(ref mut iter) = self.iter { + acc = iter.try_rfold(acc, fold)?; + } + try { acc } + } + + #[inline] + fn rfind

(&mut self, predicate: P) -> Option + where + P: FnMut(&Self::Item) -> bool, + I: DoubleEndedIterator, + { + self.iter.as_mut()?.rfind(predicate) + } +} + +#[inline] +fn and_then_or_clear(opt: &mut Option, f: impl FnOnce(&mut T) -> Option) -> Option { + let x = f(opt.as_mut()?); + if x.is_none() { + *opt = None; + } + x +} diff --git a/crux-mir/lib/core/src/iter/adapters/inspect.rs b/crux-mir/lib/core/src/iter/adapters/inspect.rs new file mode 100644 index 000000000..19839fdfe --- /dev/null +++ b/crux-mir/lib/core/src/iter/adapters/inspect.rs @@ -0,0 +1,166 @@ +use crate::fmt; +use crate::iter::{adapters::SourceIter, FusedIterator, InPlaceIterable}; +use crate::ops::Try; + +/// An iterator that calls a function with a reference to each element before +/// yielding it. +/// +/// This `struct` is created by the [`inspect`] method on [`Iterator`]. See its +/// documentation for more. +/// +/// [`inspect`]: Iterator::inspect +/// [`Iterator`]: trait.Iterator.html +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "rust1", since = "1.0.0")] +#[derive(Clone)] +pub struct Inspect { + iter: I, + f: F, +} +impl Inspect { + pub(in crate::iter) fn new(iter: I, f: F) -> Inspect { + Inspect { iter, f } + } +} + +#[stable(feature = "core_impl_debug", since = "1.9.0")] +impl fmt::Debug for Inspect { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Inspect").field("iter", &self.iter).finish() + } +} + +impl Inspect +where + F: FnMut(&I::Item), +{ + #[inline] + fn do_inspect(&mut self, elt: Option) -> Option { + if let Some(ref a) = elt { + (self.f)(a); + } + + elt + } +} + +fn inspect_fold( + mut f: impl FnMut(&T), + mut fold: impl FnMut(Acc, T) -> Acc, +) -> impl FnMut(Acc, T) -> Acc { + move |acc, item| { + f(&item); + fold(acc, item) + } +} + +fn inspect_try_fold<'a, T, Acc, R>( + f: &'a mut impl FnMut(&T), + mut fold: impl FnMut(Acc, T) -> R + 'a, +) -> impl FnMut(Acc, T) -> R + 'a { + move |acc, item| { + f(&item); + fold(acc, item) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for Inspect +where + F: FnMut(&I::Item), +{ + type Item = I::Item; + + #[inline] + fn next(&mut self) -> Option { + let next = self.iter.next(); + self.do_inspect(next) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } + + #[inline] + fn try_fold(&mut self, init: Acc, fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + { + self.iter.try_fold(init, inspect_try_fold(&mut self.f, fold)) + } + + #[inline] + fn fold(self, init: Acc, fold: Fold) -> Acc + where + Fold: FnMut(Acc, Self::Item) -> Acc, + { + self.iter.fold(init, inspect_fold(self.f, fold)) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl DoubleEndedIterator for Inspect +where + F: FnMut(&I::Item), +{ + #[inline] + fn next_back(&mut self) -> Option { + let next = self.iter.next_back(); + self.do_inspect(next) + } + + #[inline] + fn try_rfold(&mut self, init: Acc, fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + { + self.iter.try_rfold(init, inspect_try_fold(&mut self.f, fold)) + } + + #[inline] + fn rfold(self, init: Acc, fold: Fold) -> Acc + where + Fold: FnMut(Acc, Self::Item) -> Acc, + { + self.iter.rfold(init, inspect_fold(self.f, fold)) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ExactSizeIterator for Inspect +where + F: FnMut(&I::Item), +{ + fn len(&self) -> usize { + self.iter.len() + } + + fn is_empty(&self) -> bool { + self.iter.is_empty() + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Inspect where F: FnMut(&I::Item) {} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl SourceIter for Inspect +where + I: SourceIter, +{ + type Source = I::Source; + + #[inline] + unsafe fn as_inner(&mut self) -> &mut I::Source { + // SAFETY: unsafe function forwarding to unsafe function with the same requirements + unsafe { SourceIter::as_inner(&mut self.iter) } + } +} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for Inspect where F: FnMut(&I::Item) {} diff --git a/crux-mir/lib/core/src/iter/adapters/intersperse.rs b/crux-mir/lib/core/src/iter/adapters/intersperse.rs new file mode 100644 index 000000000..d8bbd424c --- /dev/null +++ b/crux-mir/lib/core/src/iter/adapters/intersperse.rs @@ -0,0 +1,187 @@ +use super::Peekable; + +/// An iterator adapter that places a separator between all elements. +/// +/// This `struct` is created by [`Iterator::intersperse`]. See its documentation +/// for more information. +#[unstable(feature = "iter_intersperse", reason = "recently added", issue = "79524")] +#[derive(Debug, Clone)] +pub struct Intersperse +where + I::Item: Clone, +{ + separator: I::Item, + iter: Peekable, + needs_sep: bool, +} + +impl Intersperse +where + I::Item: Clone, +{ + pub(in crate::iter) fn new(iter: I, separator: I::Item) -> Self { + Self { iter: iter.peekable(), separator, needs_sep: false } + } +} + +#[unstable(feature = "iter_intersperse", reason = "recently added", issue = "79524")] +impl Iterator for Intersperse +where + I: Iterator, + I::Item: Clone, +{ + type Item = I::Item; + + #[inline] + fn next(&mut self) -> Option { + if self.needs_sep && self.iter.peek().is_some() { + self.needs_sep = false; + Some(self.separator.clone()) + } else { + self.needs_sep = true; + self.iter.next() + } + } + + fn fold(self, init: B, f: F) -> B + where + Self: Sized, + F: FnMut(B, Self::Item) -> B, + { + let separator = self.separator; + intersperse_fold(self.iter, init, f, move || separator.clone(), self.needs_sep) + } + + fn size_hint(&self) -> (usize, Option) { + intersperse_size_hint(&self.iter, self.needs_sep) + } +} + +/// An iterator adapter that places a separator between all elements. +/// +/// This `struct` is created by [`Iterator::intersperse_with`]. See its +/// documentation for more information. +#[unstable(feature = "iter_intersperse", reason = "recently added", issue = "79524")] +pub struct IntersperseWith +where + I: Iterator, +{ + separator: G, + iter: Peekable, + needs_sep: bool, +} + +#[unstable(feature = "iter_intersperse", reason = "recently added", issue = "79524")] +impl crate::fmt::Debug for IntersperseWith +where + I: Iterator + crate::fmt::Debug, + I::Item: crate::fmt::Debug, + G: crate::fmt::Debug, +{ + fn fmt(&self, f: &mut crate::fmt::Formatter<'_>) -> crate::fmt::Result { + f.debug_struct("IntersperseWith") + .field("separator", &self.separator) + .field("iter", &self.iter) + .field("needs_sep", &self.needs_sep) + .finish() + } +} + +#[unstable(feature = "iter_intersperse", reason = "recently added", issue = "79524")] +impl crate::clone::Clone for IntersperseWith +where + I: Iterator + crate::clone::Clone, + I::Item: crate::clone::Clone, + G: Clone, +{ + fn clone(&self) -> Self { + IntersperseWith { + separator: self.separator.clone(), + iter: self.iter.clone(), + needs_sep: self.needs_sep.clone(), + } + } +} + +impl IntersperseWith +where + I: Iterator, + G: FnMut() -> I::Item, +{ + pub(in crate::iter) fn new(iter: I, separator: G) -> Self { + Self { iter: iter.peekable(), separator, needs_sep: false } + } +} + +#[unstable(feature = "iter_intersperse", reason = "recently added", issue = "79524")] +impl Iterator for IntersperseWith +where + I: Iterator, + G: FnMut() -> I::Item, +{ + type Item = I::Item; + + #[inline] + fn next(&mut self) -> Option { + if self.needs_sep && self.iter.peek().is_some() { + self.needs_sep = false; + Some((self.separator)()) + } else { + self.needs_sep = true; + self.iter.next() + } + } + + fn fold(self, init: B, f: F) -> B + where + Self: Sized, + F: FnMut(B, Self::Item) -> B, + { + intersperse_fold(self.iter, init, f, self.separator, self.needs_sep) + } + + fn size_hint(&self) -> (usize, Option) { + intersperse_size_hint(&self.iter, self.needs_sep) + } +} + +fn intersperse_size_hint(iter: &I, needs_sep: bool) -> (usize, Option) +where + I: Iterator, +{ + let (lo, hi) = iter.size_hint(); + let next_is_elem = !needs_sep; + ( + lo.saturating_sub(next_is_elem as usize).saturating_add(lo), + hi.and_then(|hi| hi.saturating_sub(next_is_elem as usize).checked_add(hi)), + ) +} + +fn intersperse_fold( + mut iter: I, + init: B, + mut f: F, + mut separator: G, + needs_sep: bool, +) -> B +where + I: Iterator, + F: FnMut(B, I::Item) -> B, + G: FnMut() -> I::Item, +{ + let mut accum = init; + + if !needs_sep { + if let Some(x) = iter.next() { + accum = f(accum, x); + } else { + return accum; + } + } + + iter.fold(accum, |mut accum, x| { + accum = f(accum, separator()); + accum = f(accum, x); + accum + }) +} diff --git a/crux-mir/lib/core/src/iter/adapters/map.rs b/crux-mir/lib/core/src/iter/adapters/map.rs new file mode 100644 index 000000000..9e25dbe46 --- /dev/null +++ b/crux-mir/lib/core/src/iter/adapters/map.rs @@ -0,0 +1,218 @@ +use crate::fmt; +use crate::iter::adapters::{ + zip::try_get_unchecked, SourceIter, TrustedRandomAccess, TrustedRandomAccessNoCoerce, +}; +use crate::iter::{FusedIterator, InPlaceIterable, TrustedLen}; +use crate::ops::Try; + +/// An iterator that maps the values of `iter` with `f`. +/// +/// This `struct` is created by the [`map`] method on [`Iterator`]. See its +/// documentation for more. +/// +/// [`map`]: Iterator::map +/// [`Iterator`]: trait.Iterator.html +/// +/// # Notes about side effects +/// +/// The [`map`] iterator implements [`DoubleEndedIterator`], meaning that +/// you can also [`map`] backwards: +/// +/// ```rust +/// let v: Vec = [1, 2, 3].into_iter().map(|x| x + 1).rev().collect(); +/// +/// assert_eq!(v, [4, 3, 2]); +/// ``` +/// +/// [`DoubleEndedIterator`]: trait.DoubleEndedIterator.html +/// +/// But if your closure has state, iterating backwards may act in a way you do +/// not expect. Let's go through an example. First, in the forward direction: +/// +/// ```rust +/// let mut c = 0; +/// +/// for pair in ['a', 'b', 'c'].into_iter() +/// .map(|letter| { c += 1; (letter, c) }) { +/// println!("{pair:?}"); +/// } +/// ``` +/// +/// This will print `('a', 1), ('b', 2), ('c', 3)`. +/// +/// Now consider this twist where we add a call to `rev`. This version will +/// print `('c', 1), ('b', 2), ('a', 3)`. Note that the letters are reversed, +/// but the values of the counter still go in order. This is because `map()` is +/// still being called lazily on each item, but we are popping items off the +/// back of the vector now, instead of shifting them from the front. +/// +/// ```rust +/// let mut c = 0; +/// +/// for pair in ['a', 'b', 'c'].into_iter() +/// .map(|letter| { c += 1; (letter, c) }) +/// .rev() { +/// println!("{pair:?}"); +/// } +/// ``` +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "rust1", since = "1.0.0")] +#[derive(Clone)] +pub struct Map { + // Used for `SplitWhitespace` and `SplitAsciiWhitespace` `as_str` methods + pub(crate) iter: I, + f: F, +} + +impl Map { + pub(in crate::iter) fn new(iter: I, f: F) -> Map { + Map { iter, f } + } +} + +#[stable(feature = "core_impl_debug", since = "1.9.0")] +impl fmt::Debug for Map { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Map").field("iter", &self.iter).finish() + } +} + +fn map_fold( + mut f: impl FnMut(T) -> B, + mut g: impl FnMut(Acc, B) -> Acc, +) -> impl FnMut(Acc, T) -> Acc { + move |acc, elt| g(acc, f(elt)) +} + +fn map_try_fold<'a, T, B, Acc, R>( + f: &'a mut impl FnMut(T) -> B, + mut g: impl FnMut(Acc, B) -> R + 'a, +) -> impl FnMut(Acc, T) -> R + 'a { + move |acc, elt| g(acc, f(elt)) +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for Map +where + F: FnMut(I::Item) -> B, +{ + type Item = B; + + #[inline] + fn next(&mut self) -> Option { + self.iter.next().map(&mut self.f) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } + + fn try_fold(&mut self, init: Acc, g: G) -> R + where + Self: Sized, + G: FnMut(Acc, Self::Item) -> R, + R: Try, + { + self.iter.try_fold(init, map_try_fold(&mut self.f, g)) + } + + fn fold(self, init: Acc, g: G) -> Acc + where + G: FnMut(Acc, Self::Item) -> Acc, + { + self.iter.fold(init, map_fold(self.f, g)) + } + + #[inline] + unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> B + where + Self: TrustedRandomAccessNoCoerce, + { + // SAFETY: the caller must uphold the contract for + // `Iterator::__iterator_get_unchecked`. + unsafe { (self.f)(try_get_unchecked(&mut self.iter, idx)) } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl DoubleEndedIterator for Map +where + F: FnMut(I::Item) -> B, +{ + #[inline] + fn next_back(&mut self) -> Option { + self.iter.next_back().map(&mut self.f) + } + + fn try_rfold(&mut self, init: Acc, g: G) -> R + where + Self: Sized, + G: FnMut(Acc, Self::Item) -> R, + R: Try, + { + self.iter.try_rfold(init, map_try_fold(&mut self.f, g)) + } + + fn rfold(self, init: Acc, g: G) -> Acc + where + G: FnMut(Acc, Self::Item) -> Acc, + { + self.iter.rfold(init, map_fold(self.f, g)) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ExactSizeIterator for Map +where + F: FnMut(I::Item) -> B, +{ + fn len(&self) -> usize { + self.iter.len() + } + + fn is_empty(&self) -> bool { + self.iter.is_empty() + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Map where F: FnMut(I::Item) -> B {} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for Map +where + I: TrustedLen, + F: FnMut(I::Item) -> B, +{ +} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl TrustedRandomAccess for Map where I: TrustedRandomAccess {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl TrustedRandomAccessNoCoerce for Map +where + I: TrustedRandomAccessNoCoerce, +{ + const MAY_HAVE_SIDE_EFFECT: bool = true; +} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl SourceIter for Map +where + I: SourceIter, +{ + type Source = I::Source; + + #[inline] + unsafe fn as_inner(&mut self) -> &mut I::Source { + // SAFETY: unsafe function forwarding to unsafe function with the same requirements + unsafe { SourceIter::as_inner(&mut self.iter) } + } +} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for Map where F: FnMut(I::Item) -> B {} diff --git a/crux-mir/lib/core/src/iter/adapters/map_while.rs b/crux-mir/lib/core/src/iter/adapters/map_while.rs new file mode 100644 index 000000000..fbdeca4d4 --- /dev/null +++ b/crux-mir/lib/core/src/iter/adapters/map_while.rs @@ -0,0 +1,88 @@ +use crate::fmt; +use crate::iter::{adapters::SourceIter, InPlaceIterable}; +use crate::ops::{ControlFlow, Try}; + +/// An iterator that only accepts elements while `predicate` returns `Some(_)`. +/// +/// This `struct` is created by the [`map_while`] method on [`Iterator`]. See its +/// documentation for more. +/// +/// [`map_while`]: Iterator::map_while +/// [`Iterator`]: trait.Iterator.html +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "iter_map_while", since = "1.57.0")] +#[derive(Clone)] +pub struct MapWhile { + iter: I, + predicate: P, +} + +impl MapWhile { + pub(in crate::iter) fn new(iter: I, predicate: P) -> MapWhile { + MapWhile { iter, predicate } + } +} + +#[stable(feature = "iter_map_while", since = "1.57.0")] +impl fmt::Debug for MapWhile { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("MapWhile").field("iter", &self.iter).finish() + } +} + +#[stable(feature = "iter_map_while", since = "1.57.0")] +impl Iterator for MapWhile +where + P: FnMut(I::Item) -> Option, +{ + type Item = B; + + #[inline] + fn next(&mut self) -> Option { + let x = self.iter.next()?; + (self.predicate)(x) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let (_, upper) = self.iter.size_hint(); + (0, upper) // can't know a lower bound, due to the predicate + } + + #[inline] + fn try_fold(&mut self, init: Acc, mut fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + { + let Self { iter, predicate } = self; + iter.try_fold(init, |acc, x| match predicate(x) { + Some(item) => ControlFlow::from_try(fold(acc, item)), + None => ControlFlow::Break(try { acc }), + }) + .into_try() + } + + impl_fold_via_try_fold! { fold -> try_fold } +} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl SourceIter for MapWhile +where + I: SourceIter, +{ + type Source = I::Source; + + #[inline] + unsafe fn as_inner(&mut self) -> &mut I::Source { + // SAFETY: unsafe function forwarding to unsafe function with the same requirements + unsafe { SourceIter::as_inner(&mut self.iter) } + } +} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for MapWhile where + P: FnMut(I::Item) -> Option +{ +} diff --git a/crux-mir/lib/core/src/iter/adapters/mod.rs b/crux-mir/lib/core/src/iter/adapters/mod.rs index 3c0ddcb2b..8cc2b7cec 100644 --- a/crux-mir/lib/core/src/iter/adapters/mod.rs +++ b/crux-mir/lib/core/src/iter/adapters/mod.rs @@ -1,2727 +1,230 @@ -use crate::cmp; -use crate::fmt; -use crate::intrinsics; -use crate::ops::{Add, AddAssign, Try}; -use crate::usize; - -use super::{from_fn, LoopState}; -use super::{DoubleEndedIterator, ExactSizeIterator, FusedIterator, Iterator, TrustedLen}; +use crate::iter::{InPlaceIterable, Iterator}; +use crate::ops::{ChangeOutputType, ControlFlow, FromResidual, Residual, Try}; +mod array_chunks; +mod by_ref_sized; mod chain; +mod cloned; +mod copied; +mod cycle; +mod enumerate; +mod filter; +mod filter_map; mod flatten; +mod fuse; +mod inspect; +mod intersperse; +mod map; +mod map_while; +mod peekable; +mod rev; +mod scan; +mod skip; +mod skip_while; +mod step_by; +mod take; +mod take_while; mod zip; -pub use self::chain::Chain; -#[stable(feature = "rust1", since = "1.0.0")] -pub use self::flatten::{FlatMap, Flatten}; -pub(crate) use self::zip::TrustedRandomAccess; -pub use self::zip::Zip; - -/// A double-ended iterator with the direction inverted. -/// -/// This `struct` is created by the [`rev`] method on [`Iterator`]. See its -/// documentation for more. -/// -/// [`rev`]: trait.Iterator.html#method.rev -/// [`Iterator`]: trait.Iterator.html -#[derive(Clone, Debug)] -#[must_use = "iterators are lazy and do nothing unless consumed"] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct Rev { - iter: T, -} -impl Rev { - pub(super) fn new(iter: T) -> Rev { - Rev { iter } - } -} - #[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for Rev -where - I: DoubleEndedIterator, -{ - type Item = ::Item; - - #[inline] - fn next(&mut self) -> Option<::Item> { - self.iter.next_back() - } - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() - } - - #[inline] - fn nth(&mut self, n: usize) -> Option<::Item> { - self.iter.nth_back(n) - } - - fn try_fold(&mut self, init: B, f: F) -> R - where - Self: Sized, - F: FnMut(B, Self::Item) -> R, - R: Try, - { - self.iter.try_rfold(init, f) - } +pub use self::{ + chain::Chain, cycle::Cycle, enumerate::Enumerate, filter::Filter, filter_map::FilterMap, + flatten::FlatMap, fuse::Fuse, inspect::Inspect, map::Map, peekable::Peekable, rev::Rev, + scan::Scan, skip::Skip, skip_while::SkipWhile, take::Take, take_while::TakeWhile, zip::Zip, +}; - fn fold(self, init: Acc, f: F) -> Acc - where - F: FnMut(Acc, Self::Item) -> Acc, - { - self.iter.rfold(init, f) - } +#[unstable(feature = "iter_array_chunks", reason = "recently added", issue = "100450")] +pub use self::array_chunks::ArrayChunks; - #[inline] - fn find

(&mut self, predicate: P) -> Option - where - P: FnMut(&Self::Item) -> bool, - { - self.iter.rfind(predicate) - } -} +#[unstable(feature = "std_internals", issue = "none")] +pub use self::by_ref_sized::ByRefSized; -#[stable(feature = "rust1", since = "1.0.0")] -impl DoubleEndedIterator for Rev -where - I: DoubleEndedIterator, -{ - #[inline] - fn next_back(&mut self) -> Option<::Item> { - self.iter.next() - } +#[stable(feature = "iter_cloned", since = "1.1.0")] +pub use self::cloned::Cloned; - #[inline] - fn nth_back(&mut self, n: usize) -> Option<::Item> { - self.iter.nth(n) - } +#[stable(feature = "iterator_step_by", since = "1.28.0")] +pub use self::step_by::StepBy; - fn try_rfold(&mut self, init: B, f: F) -> R - where - Self: Sized, - F: FnMut(B, Self::Item) -> R, - R: Try, - { - self.iter.try_fold(init, f) - } +#[stable(feature = "iterator_flatten", since = "1.29.0")] +pub use self::flatten::Flatten; - fn rfold(self, init: Acc, f: F) -> Acc - where - F: FnMut(Acc, Self::Item) -> Acc, - { - self.iter.fold(init, f) - } +#[stable(feature = "iter_copied", since = "1.36.0")] +pub use self::copied::Copied; - fn rfind

(&mut self, predicate: P) -> Option - where - P: FnMut(&Self::Item) -> bool, - { - self.iter.find(predicate) - } -} +#[unstable(feature = "iter_intersperse", reason = "recently added", issue = "79524")] +pub use self::intersperse::{Intersperse, IntersperseWith}; -#[stable(feature = "rust1", since = "1.0.0")] -impl ExactSizeIterator for Rev -where - I: ExactSizeIterator + DoubleEndedIterator, -{ - fn len(&self) -> usize { - self.iter.len() - } +#[stable(feature = "iter_map_while", since = "1.57.0")] +pub use self::map_while::MapWhile; - fn is_empty(&self) -> bool { - self.iter.is_empty() - } -} +#[unstable(feature = "trusted_random_access", issue = "none")] +pub use self::zip::TrustedRandomAccess; -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Rev where I: FusedIterator + DoubleEndedIterator {} +#[unstable(feature = "trusted_random_access", issue = "none")] +pub use self::zip::TrustedRandomAccessNoCoerce; -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for Rev where I: TrustedLen + DoubleEndedIterator {} +#[stable(feature = "iter_zip", since = "1.59.0")] +pub use self::zip::zip; -/// An iterator that copies the elements of an underlying iterator. +/// This trait provides transitive access to source-stage in an iterator-adapter pipeline +/// under the conditions that +/// * the iterator source `S` itself implements `SourceIter` +/// * there is a delegating implementation of this trait for each adapter in the pipeline between +/// the source and the pipeline consumer. /// -/// This `struct` is created by the [`copied`] method on [`Iterator`]. See its -/// documentation for more. +/// When the source is an owning iterator struct (commonly called `IntoIter`) then +/// this can be useful for specializing [`FromIterator`] implementations or recovering the +/// remaining elements after an iterator has been partially exhausted. /// -/// [`copied`]: trait.Iterator.html#method.copied -/// [`Iterator`]: trait.Iterator.html -#[stable(feature = "iter_copied", since = "1.36.0")] -#[must_use = "iterators are lazy and do nothing unless consumed"] -#[derive(Clone, Debug)] -pub struct Copied { - it: I, -} - -impl Copied { - pub(super) fn new(it: I) -> Copied { - Copied { it } - } -} - -fn copy_fold(mut f: impl FnMut(Acc, T) -> Acc) -> impl FnMut(Acc, &T) -> Acc { - move |acc, &elt| f(acc, elt) -} - -fn copy_try_fold(mut f: impl FnMut(Acc, T) -> R) -> impl FnMut(Acc, &T) -> R { - move |acc, &elt| f(acc, elt) -} - -#[stable(feature = "iter_copied", since = "1.36.0")] -impl<'a, I, T: 'a> Iterator for Copied -where - I: Iterator, - T: Copy, -{ - type Item = T; - - fn next(&mut self) -> Option { - self.it.next().copied() - } - - fn size_hint(&self) -> (usize, Option) { - self.it.size_hint() - } - - fn try_fold(&mut self, init: B, f: F) -> R - where - Self: Sized, - F: FnMut(B, Self::Item) -> R, - R: Try, - { - self.it.try_fold(init, copy_try_fold(f)) - } - - fn fold(self, init: Acc, f: F) -> Acc - where - F: FnMut(Acc, Self::Item) -> Acc, - { - self.it.fold(init, copy_fold(f)) - } - - fn nth(&mut self, n: usize) -> Option { - self.it.nth(n).copied() - } - - fn last(self) -> Option { - self.it.last().copied() - } - - fn count(self) -> usize { - self.it.count() - } -} - -#[stable(feature = "iter_copied", since = "1.36.0")] -impl<'a, I, T: 'a> DoubleEndedIterator for Copied -where - I: DoubleEndedIterator, - T: Copy, -{ - fn next_back(&mut self) -> Option { - self.it.next_back().copied() - } - - fn try_rfold(&mut self, init: B, f: F) -> R - where - Self: Sized, - F: FnMut(B, Self::Item) -> R, - R: Try, - { - self.it.try_rfold(init, copy_try_fold(f)) - } - - fn rfold(self, init: Acc, f: F) -> Acc - where - F: FnMut(Acc, Self::Item) -> Acc, - { - self.it.rfold(init, copy_fold(f)) - } -} - -#[stable(feature = "iter_copied", since = "1.36.0")] -impl<'a, I, T: 'a> ExactSizeIterator for Copied -where - I: ExactSizeIterator, - T: Copy, -{ - fn len(&self) -> usize { - self.it.len() - } - - fn is_empty(&self) -> bool { - self.it.is_empty() - } -} - -#[stable(feature = "iter_copied", since = "1.36.0")] -impl<'a, I, T: 'a> FusedIterator for Copied -where - I: FusedIterator, - T: Copy, -{ -} - -#[doc(hidden)] -unsafe impl<'a, I, T: 'a> TrustedRandomAccess for Copied -where - I: TrustedRandomAccess, - T: Copy, -{ - unsafe fn get_unchecked(&mut self, i: usize) -> Self::Item { - *self.it.get_unchecked(i) - } - - #[inline] - fn may_have_side_effect() -> bool { - I::may_have_side_effect() - } -} - -#[stable(feature = "iter_copied", since = "1.36.0")] -unsafe impl<'a, I, T: 'a> TrustedLen for Copied -where - I: TrustedLen, - T: Copy, -{ -} - -/// An iterator that clones the elements of an underlying iterator. +/// Note that implementations do not necessarily have to provide access to the inner-most +/// source of a pipeline. A stateful intermediate adapter might eagerly evaluate a part +/// of the pipeline and expose its internal storage as source. /// -/// This `struct` is created by the [`cloned`] method on [`Iterator`]. See its -/// documentation for more. +/// The trait is unsafe because implementers must uphold additional safety properties. +/// See [`as_inner`] for details. /// -/// [`cloned`]: trait.Iterator.html#method.cloned -/// [`Iterator`]: trait.Iterator.html -#[stable(feature = "iter_cloned", since = "1.1.0")] -#[must_use = "iterators are lazy and do nothing unless consumed"] -#[derive(Clone, Debug)] -pub struct Cloned { - it: I, -} -impl Cloned { - pub(super) fn new(it: I) -> Cloned { - Cloned { it } - } -} - -fn clone_try_fold(mut f: impl FnMut(Acc, T) -> R) -> impl FnMut(Acc, &T) -> R { - move |acc, elt| f(acc, elt.clone()) -} - -#[stable(feature = "iter_cloned", since = "1.1.0")] -impl<'a, I, T: 'a> Iterator for Cloned -where - I: Iterator, - T: Clone, -{ - type Item = T; - - fn next(&mut self) -> Option { - self.it.next().cloned() - } - - fn size_hint(&self) -> (usize, Option) { - self.it.size_hint() - } - - fn try_fold(&mut self, init: B, f: F) -> R - where - Self: Sized, - F: FnMut(B, Self::Item) -> R, - R: Try, - { - self.it.try_fold(init, clone_try_fold(f)) - } - - fn fold(self, init: Acc, f: F) -> Acc - where - F: FnMut(Acc, Self::Item) -> Acc, - { - self.it.map(T::clone).fold(init, f) - } -} - -#[stable(feature = "iter_cloned", since = "1.1.0")] -impl<'a, I, T: 'a> DoubleEndedIterator for Cloned -where - I: DoubleEndedIterator, - T: Clone, -{ - fn next_back(&mut self) -> Option { - self.it.next_back().cloned() - } - - fn try_rfold(&mut self, init: B, f: F) -> R - where - Self: Sized, - F: FnMut(B, Self::Item) -> R, - R: Try, - { - self.it.try_rfold(init, clone_try_fold(f)) - } - - fn rfold(self, init: Acc, f: F) -> Acc - where - F: FnMut(Acc, Self::Item) -> Acc, - { - self.it.map(T::clone).rfold(init, f) - } -} - -#[stable(feature = "iter_cloned", since = "1.1.0")] -impl<'a, I, T: 'a> ExactSizeIterator for Cloned -where - I: ExactSizeIterator, - T: Clone, -{ - fn len(&self) -> usize { - self.it.len() - } - - fn is_empty(&self) -> bool { - self.it.is_empty() - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl<'a, I, T: 'a> FusedIterator for Cloned -where - I: FusedIterator, - T: Clone, -{ -} - +/// The primary use of this trait is in-place iteration. Refer to the [`vec::in_place_collect`] +/// module documentation for more information. +/// +/// [`vec::in_place_collect`]: ../../../../alloc/vec/in_place_collect/index.html +/// +/// # Examples +/// +/// Retrieving a partially consumed source: +/// +/// ``` +/// # #![feature(inplace_iteration)] +/// # use std::iter::SourceIter; +/// +/// let mut iter = vec![9, 9, 9].into_iter().map(|i| i * i); +/// let _ = iter.next(); +/// let mut remainder = std::mem::replace(unsafe { iter.as_inner() }, Vec::new().into_iter()); +/// println!("n = {} elements remaining", remainder.len()); +/// ``` +/// +/// [`FromIterator`]: crate::iter::FromIterator +/// [`as_inner`]: SourceIter::as_inner +#[unstable(issue = "none", feature = "inplace_iteration")] #[doc(hidden)] -unsafe impl<'a, I, T: 'a> TrustedRandomAccess for Cloned -where - I: TrustedRandomAccess, - T: Clone, -{ - default unsafe fn get_unchecked(&mut self, i: usize) -> Self::Item { - self.it.get_unchecked(i).clone() - } +#[rustc_specialization_trait] +pub unsafe trait SourceIter { + /// A source stage in an iterator pipeline. + type Source; - #[inline] - default fn may_have_side_effect() -> bool { - true - } + /// Retrieve the source of an iterator pipeline. + /// + /// # Safety + /// + /// Implementations of must return the same mutable reference for their lifetime, unless + /// replaced by a caller. + /// Callers may only replace the reference when they stopped iteration and drop the + /// iterator pipeline after extracting the source. + /// + /// This means iterator adapters can rely on the source not changing during + /// iteration but they cannot rely on it in their Drop implementations. + /// + /// Implementing this method means adapters relinquish private-only access to their + /// source and can only rely on guarantees made based on method receiver types. + /// The lack of restricted access also requires that adapters must uphold the source's + /// public API even when they have access to its internals. + /// + /// Callers in turn must expect the source to be in any state that is consistent with + /// its public API since adapters sitting between it and the source have the same + /// access. In particular an adapter may have consumed more elements than strictly necessary. + /// + /// The overall goal of these requirements is to let the consumer of a pipeline use + /// * whatever remains in the source after iteration has stopped + /// * the memory that has become unused by advancing a consuming iterator + /// + /// [`next()`]: Iterator::next() + unsafe fn as_inner(&mut self) -> &mut Self::Source; } -#[doc(hidden)] -unsafe impl<'a, I, T: 'a> TrustedRandomAccess for Cloned -where - I: TrustedRandomAccess, - T: Copy, -{ - unsafe fn get_unchecked(&mut self, i: usize) -> Self::Item { - *self.it.get_unchecked(i) - } - - #[inline] - fn may_have_side_effect() -> bool { - I::may_have_side_effect() - } +/// An iterator adapter that produces output as long as the underlying +/// iterator produces values where `Try::branch` says to `ControlFlow::Continue`. +/// +/// If a `ControlFlow::Break` is encountered, the iterator stops and the +/// residual is stored. +pub(crate) struct GenericShunt<'a, I, R> { + iter: I, + residual: &'a mut Option, } -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl<'a, I, T: 'a> TrustedLen for Cloned +/// Process the given iterator as if it yielded a the item's `Try::Output` +/// type instead. Any `Try::Residual`s encountered will stop the inner iterator +/// and be propagated back to the overall result. +pub(crate) fn try_process(iter: I, mut f: F) -> ChangeOutputType where - I: TrustedLen, - T: Clone, + I: Iterator>, + for<'a> F: FnMut(GenericShunt<'a, I, R>) -> U, + R: Residual, { -} - -/// An iterator that repeats endlessly. -/// -/// This `struct` is created by the [`cycle`] method on [`Iterator`]. See its -/// documentation for more. -/// -/// [`cycle`]: trait.Iterator.html#method.cycle -/// [`Iterator`]: trait.Iterator.html -#[derive(Clone, Debug)] -#[must_use = "iterators are lazy and do nothing unless consumed"] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct Cycle { - orig: I, - iter: I, -} -impl Cycle { - pub(super) fn new(iter: I) -> Cycle { - Cycle { orig: iter.clone(), iter } + let mut residual = None; + let shunt = GenericShunt { iter, residual: &mut residual }; + let value = f(shunt); + match residual { + Some(r) => FromResidual::from_residual(r), + None => Try::from_output(value), } } -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for Cycle +impl Iterator for GenericShunt<'_, I, R> where - I: Clone + Iterator, + I: Iterator>, { - type Item = ::Item; + type Item = ::Output; - #[inline] - fn next(&mut self) -> Option<::Item> { - match self.iter.next() { - None => { - self.iter = self.orig.clone(); - self.iter.next() - } - y => y, - } + fn next(&mut self) -> Option { + self.try_for_each(ControlFlow::Break).break_value() } - #[inline] fn size_hint(&self) -> (usize, Option) { - // the cycle iterator is either empty or infinite - match self.orig.size_hint() { - sz @ (0, Some(0)) => sz, - (0, _) => (0, None), - _ => (usize::MAX, None), + if self.residual.is_some() { + (0, Some(0)) + } else { + let (_, upper) = self.iter.size_hint(); + (0, upper) } } - #[inline] - fn try_fold(&mut self, mut acc: Acc, mut f: F) -> R + fn try_fold(&mut self, init: B, mut f: F) -> T where - F: FnMut(Acc, Self::Item) -> R, - R: Try, + F: FnMut(B, Self::Item) -> T, + T: Try, { - // fully iterate the current iterator. this is necessary because - // `self.iter` may be empty even when `self.orig` isn't - acc = self.iter.try_fold(acc, &mut f)?; - self.iter = self.orig.clone(); - - // complete a full cycle, keeping track of whether the cycled - // iterator is empty or not. we need to return early in case - // of an empty iterator to prevent an infinite loop - let mut is_empty = true; - acc = self.iter.try_fold(acc, |acc, x| { - is_empty = false; - f(acc, x) - })?; - - if is_empty { - return Try::from_ok(acc); - } - - loop { - self.iter = self.orig.clone(); - acc = self.iter.try_fold(acc, &mut f)?; - } + self.iter + .try_fold(init, |acc, x| match Try::branch(x) { + ControlFlow::Continue(x) => ControlFlow::from_try(f(acc, x)), + ControlFlow::Break(r) => { + *self.residual = Some(r); + ControlFlow::Break(try { acc }) + } + }) + .into_try() } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Cycle where I: Clone + Iterator {} -/// An iterator for stepping iterators by a custom amount. -/// -/// This `struct` is created by the [`step_by`] method on [`Iterator`]. See -/// its documentation for more. -/// -/// [`step_by`]: trait.Iterator.html#method.step_by -/// [`Iterator`]: trait.Iterator.html -#[must_use = "iterators are lazy and do nothing unless consumed"] -#[stable(feature = "iterator_step_by", since = "1.28.0")] -#[derive(Clone, Debug)] -pub struct StepBy { - iter: I, - step: usize, - first_take: bool, -} -impl StepBy { - pub(super) fn new(iter: I, step: usize) -> StepBy { - assert!(step != 0); - StepBy { iter, step: step - 1, first_take: true } - } + impl_fold_via_try_fold! { fold -> try_fold } } -#[stable(feature = "iterator_step_by", since = "1.28.0")] -impl Iterator for StepBy +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl SourceIter for GenericShunt<'_, I, R> where - I: Iterator, + I: SourceIter, { - type Item = I::Item; + type Source = I::Source; #[inline] - fn next(&mut self) -> Option { - if self.first_take { - self.first_take = false; - self.iter.next() - } else { - self.iter.nth(self.step) - } + unsafe fn as_inner(&mut self) -> &mut Self::Source { + // SAFETY: unsafe function forwarding to unsafe function with the same requirements + unsafe { SourceIter::as_inner(&mut self.iter) } } +} - #[inline] - fn size_hint(&self) -> (usize, Option) { - #[inline] - fn first_size(step: usize) -> impl Fn(usize) -> usize { - move |n| if n == 0 { 0 } else { 1 + (n - 1) / (step + 1) } - } - - #[inline] - fn other_size(step: usize) -> impl Fn(usize) -> usize { - move |n| n / (step + 1) - } - - let (low, high) = self.iter.size_hint(); - - if self.first_take { - let f = first_size(self.step); - (f(low), high.map(f)) - } else { - let f = other_size(self.step); - (f(low), high.map(f)) - } - } - - #[inline] - fn nth(&mut self, mut n: usize) -> Option { - if self.first_take { - self.first_take = false; - let first = self.iter.next(); - if n == 0 { - return first; - } - n -= 1; - } - // n and self.step are indices, we need to add 1 to get the amount of elements - // When calling `.nth`, we need to subtract 1 again to convert back to an index - // step + 1 can't overflow because `.step_by` sets `self.step` to `step - 1` - let mut step = self.step + 1; - // n + 1 could overflow - // thus, if n is usize::MAX, instead of adding one, we call .nth(step) - if n == usize::MAX { - self.iter.nth(step - 1); - } else { - n += 1; - } - - // overflow handling - loop { - let mul = n.checked_mul(step); - { - if intrinsics::likely(mul.is_some()) { - return self.iter.nth(mul.unwrap() - 1); - } - } - let div_n = usize::MAX / n; - let div_step = usize::MAX / step; - let nth_n = div_n * n; - let nth_step = div_step * step; - let nth = if nth_n > nth_step { - step -= div_n; - nth_n - } else { - n -= div_step; - nth_step - }; - self.iter.nth(nth - 1); - } - } - - fn try_fold(&mut self, mut acc: Acc, mut f: F) -> R - where - F: FnMut(Acc, Self::Item) -> R, - R: Try, - { - #[inline] - fn nth(iter: &mut I, step: usize) -> impl FnMut() -> Option + '_ { - move || iter.nth(step) - } - - if self.first_take { - self.first_take = false; - match self.iter.next() { - None => return Try::from_ok(acc), - Some(x) => acc = f(acc, x)?, - } - } - from_fn(nth(&mut self.iter, self.step)).try_fold(acc, f) - } -} - -impl StepBy -where - I: ExactSizeIterator, -{ - // The zero-based index starting from the end of the iterator of the - // last element. Used in the `DoubleEndedIterator` implementation. - fn next_back_index(&self) -> usize { - let rem = self.iter.len() % (self.step + 1); - if self.first_take { - if rem == 0 { self.step } else { rem - 1 } - } else { - rem - } - } -} - -#[stable(feature = "double_ended_step_by_iterator", since = "1.38.0")] -impl DoubleEndedIterator for StepBy -where - I: DoubleEndedIterator + ExactSizeIterator, -{ - #[inline] - fn next_back(&mut self) -> Option { - self.iter.nth_back(self.next_back_index()) - } - - #[inline] - fn nth_back(&mut self, n: usize) -> Option { - // `self.iter.nth_back(usize::MAX)` does the right thing here when `n` - // is out of bounds because the length of `self.iter` does not exceed - // `usize::MAX` (because `I: ExactSizeIterator`) and `nth_back` is - // zero-indexed - let n = n.saturating_mul(self.step + 1).saturating_add(self.next_back_index()); - self.iter.nth_back(n) - } - - fn try_rfold(&mut self, init: Acc, mut f: F) -> R - where - F: FnMut(Acc, Self::Item) -> R, - R: Try, - { - #[inline] - fn nth_back( - iter: &mut I, - step: usize, - ) -> impl FnMut() -> Option + '_ { - move || iter.nth_back(step) - } - - match self.next_back() { - None => Try::from_ok(init), - Some(x) => { - let acc = f(init, x)?; - from_fn(nth_back(&mut self.iter, self.step)).try_fold(acc, f) - } - } - } -} - -// StepBy can only make the iterator shorter, so the len will still fit. -#[stable(feature = "iterator_step_by", since = "1.28.0")] -impl ExactSizeIterator for StepBy where I: ExactSizeIterator {} - -/// An iterator that maps the values of `iter` with `f`. -/// -/// This `struct` is created by the [`map`] method on [`Iterator`]. See its -/// documentation for more. -/// -/// [`map`]: trait.Iterator.html#method.map -/// [`Iterator`]: trait.Iterator.html -/// -/// # Notes about side effects -/// -/// The [`map`] iterator implements [`DoubleEndedIterator`], meaning that -/// you can also [`map`] backwards: -/// -/// ```rust -/// let v: Vec = vec![1, 2, 3].into_iter().map(|x| x + 1).rev().collect(); -/// -/// assert_eq!(v, [4, 3, 2]); -/// ``` -/// -/// [`DoubleEndedIterator`]: trait.DoubleEndedIterator.html -/// -/// But if your closure has state, iterating backwards may act in a way you do -/// not expect. Let's go through an example. First, in the forward direction: -/// -/// ```rust -/// let mut c = 0; -/// -/// for pair in vec!['a', 'b', 'c'].into_iter() -/// .map(|letter| { c += 1; (letter, c) }) { -/// println!("{:?}", pair); -/// } -/// ``` -/// -/// This will print "('a', 1), ('b', 2), ('c', 3)". -/// -/// Now consider this twist where we add a call to `rev`. This version will -/// print `('c', 1), ('b', 2), ('a', 3)`. Note that the letters are reversed, -/// but the values of the counter still go in order. This is because `map()` is -/// still being called lazily on each item, but we are popping items off the -/// back of the vector now, instead of shifting them from the front. -/// -/// ```rust -/// let mut c = 0; -/// -/// for pair in vec!['a', 'b', 'c'].into_iter() -/// .map(|letter| { c += 1; (letter, c) }) -/// .rev() { -/// println!("{:?}", pair); -/// } -/// ``` -#[must_use = "iterators are lazy and do nothing unless consumed"] -#[stable(feature = "rust1", since = "1.0.0")] -#[derive(Clone)] -pub struct Map { - iter: I, - f: F, -} -impl Map { - pub(super) fn new(iter: I, f: F) -> Map { - Map { iter, f } - } -} - -#[stable(feature = "core_impl_debug", since = "1.9.0")] -impl fmt::Debug for Map { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Map").field("iter", &self.iter).finish() - } -} - -fn map_fold( - mut f: impl FnMut(T) -> B, - mut g: impl FnMut(Acc, B) -> Acc, -) -> impl FnMut(Acc, T) -> Acc { - move |acc, elt| g(acc, f(elt)) -} - -fn map_try_fold<'a, T, B, Acc, R>( - f: &'a mut impl FnMut(T) -> B, - mut g: impl FnMut(Acc, B) -> R + 'a, -) -> impl FnMut(Acc, T) -> R + 'a { - move |acc, elt| g(acc, f(elt)) -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for Map -where - F: FnMut(I::Item) -> B, -{ - type Item = B; - - #[inline] - fn next(&mut self) -> Option { - self.iter.next().map(&mut self.f) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() - } - - fn try_fold(&mut self, init: Acc, g: G) -> R - where - Self: Sized, - G: FnMut(Acc, Self::Item) -> R, - R: Try, - { - self.iter.try_fold(init, map_try_fold(&mut self.f, g)) - } - - fn fold(self, init: Acc, g: G) -> Acc - where - G: FnMut(Acc, Self::Item) -> Acc, - { - self.iter.fold(init, map_fold(self.f, g)) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl DoubleEndedIterator for Map -where - F: FnMut(I::Item) -> B, -{ - #[inline] - fn next_back(&mut self) -> Option { - self.iter.next_back().map(&mut self.f) - } - - fn try_rfold(&mut self, init: Acc, g: G) -> R - where - Self: Sized, - G: FnMut(Acc, Self::Item) -> R, - R: Try, - { - self.iter.try_rfold(init, map_try_fold(&mut self.f, g)) - } - - fn rfold(self, init: Acc, g: G) -> Acc - where - G: FnMut(Acc, Self::Item) -> Acc, - { - self.iter.rfold(init, map_fold(self.f, g)) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ExactSizeIterator for Map -where - F: FnMut(I::Item) -> B, -{ - fn len(&self) -> usize { - self.iter.len() - } - - fn is_empty(&self) -> bool { - self.iter.is_empty() - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Map where F: FnMut(I::Item) -> B {} - -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for Map -where - I: TrustedLen, - F: FnMut(I::Item) -> B, -{ -} - -#[doc(hidden)] -unsafe impl TrustedRandomAccess for Map -where - I: TrustedRandomAccess, - F: FnMut(I::Item) -> B, -{ - unsafe fn get_unchecked(&mut self, i: usize) -> Self::Item { - (self.f)(self.iter.get_unchecked(i)) - } - #[inline] - fn may_have_side_effect() -> bool { - true - } -} - -/// An iterator that filters the elements of `iter` with `predicate`. -/// -/// This `struct` is created by the [`filter`] method on [`Iterator`]. See its -/// documentation for more. -/// -/// [`filter`]: trait.Iterator.html#method.filter -/// [`Iterator`]: trait.Iterator.html -#[must_use = "iterators are lazy and do nothing unless consumed"] -#[stable(feature = "rust1", since = "1.0.0")] -#[derive(Clone)] -pub struct Filter { - iter: I, - predicate: P, -} -impl Filter { - pub(super) fn new(iter: I, predicate: P) -> Filter { - Filter { iter, predicate } - } -} - -#[stable(feature = "core_impl_debug", since = "1.9.0")] -impl fmt::Debug for Filter { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Filter").field("iter", &self.iter).finish() - } -} - -fn filter_fold( - mut predicate: impl FnMut(&T) -> bool, - mut fold: impl FnMut(Acc, T) -> Acc, -) -> impl FnMut(Acc, T) -> Acc { - move |acc, item| if predicate(&item) { fold(acc, item) } else { acc } -} - -fn filter_try_fold<'a, T, Acc, R: Try>( - predicate: &'a mut impl FnMut(&T) -> bool, - mut fold: impl FnMut(Acc, T) -> R + 'a, -) -> impl FnMut(Acc, T) -> R + 'a { - move |acc, item| if predicate(&item) { fold(acc, item) } else { R::from_ok(acc) } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for Filter -where - P: FnMut(&I::Item) -> bool, -{ - type Item = I::Item; - - #[inline] - fn next(&mut self) -> Option { - self.iter.find(&mut self.predicate) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let (_, upper) = self.iter.size_hint(); - (0, upper) // can't know a lower bound, due to the predicate - } - - // this special case allows the compiler to make `.filter(_).count()` - // branchless. Barring perfect branch prediction (which is unattainable in - // the general case), this will be much faster in >90% of cases (containing - // virtually all real workloads) and only a tiny bit slower in the rest. - // - // Having this specialization thus allows us to write `.filter(p).count()` - // where we would otherwise write `.map(|x| p(x) as usize).sum()`, which is - // less readable and also less backwards-compatible to Rust before 1.10. - // - // Using the branchless version will also simplify the LLVM byte code, thus - // leaving more budget for LLVM optimizations. - #[inline] - fn count(self) -> usize { - #[inline] - fn to_usize(mut predicate: impl FnMut(&T) -> bool) -> impl FnMut(T) -> usize { - move |x| predicate(&x) as usize - } - - self.iter.map(to_usize(self.predicate)).sum() - } - - #[inline] - fn try_fold(&mut self, init: Acc, fold: Fold) -> R - where - Self: Sized, - Fold: FnMut(Acc, Self::Item) -> R, - R: Try, - { - self.iter.try_fold(init, filter_try_fold(&mut self.predicate, fold)) - } - - #[inline] - fn fold(self, init: Acc, fold: Fold) -> Acc - where - Fold: FnMut(Acc, Self::Item) -> Acc, - { - self.iter.fold(init, filter_fold(self.predicate, fold)) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl DoubleEndedIterator for Filter -where - P: FnMut(&I::Item) -> bool, -{ - #[inline] - fn next_back(&mut self) -> Option { - self.iter.rfind(&mut self.predicate) - } - - #[inline] - fn try_rfold(&mut self, init: Acc, fold: Fold) -> R - where - Self: Sized, - Fold: FnMut(Acc, Self::Item) -> R, - R: Try, - { - self.iter.try_rfold(init, filter_try_fold(&mut self.predicate, fold)) - } - - #[inline] - fn rfold(self, init: Acc, fold: Fold) -> Acc - where - Fold: FnMut(Acc, Self::Item) -> Acc, - { - self.iter.rfold(init, filter_fold(self.predicate, fold)) - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Filter where P: FnMut(&I::Item) -> bool {} - -/// An iterator that uses `f` to both filter and map elements from `iter`. -/// -/// This `struct` is created by the [`filter_map`] method on [`Iterator`]. See its -/// documentation for more. -/// -/// [`filter_map`]: trait.Iterator.html#method.filter_map -/// [`Iterator`]: trait.Iterator.html -#[must_use = "iterators are lazy and do nothing unless consumed"] -#[stable(feature = "rust1", since = "1.0.0")] -#[derive(Clone)] -pub struct FilterMap { - iter: I, - f: F, -} -impl FilterMap { - pub(super) fn new(iter: I, f: F) -> FilterMap { - FilterMap { iter, f } - } -} - -#[stable(feature = "core_impl_debug", since = "1.9.0")] -impl fmt::Debug for FilterMap { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("FilterMap").field("iter", &self.iter).finish() - } -} - -fn filter_map_fold( - mut f: impl FnMut(T) -> Option, - mut fold: impl FnMut(Acc, B) -> Acc, -) -> impl FnMut(Acc, T) -> Acc { - move |acc, item| match f(item) { - Some(x) => fold(acc, x), - None => acc, - } -} - -fn filter_map_try_fold<'a, T, B, Acc, R: Try>( - f: &'a mut impl FnMut(T) -> Option, - mut fold: impl FnMut(Acc, B) -> R + 'a, -) -> impl FnMut(Acc, T) -> R + 'a { - move |acc, item| match f(item) { - Some(x) => fold(acc, x), - None => R::from_ok(acc), - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for FilterMap -where - F: FnMut(I::Item) -> Option, -{ - type Item = B; - - #[inline] - fn next(&mut self) -> Option { - self.iter.find_map(&mut self.f) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let (_, upper) = self.iter.size_hint(); - (0, upper) // can't know a lower bound, due to the predicate - } - - #[inline] - fn try_fold(&mut self, init: Acc, fold: Fold) -> R - where - Self: Sized, - Fold: FnMut(Acc, Self::Item) -> R, - R: Try, - { - self.iter.try_fold(init, filter_map_try_fold(&mut self.f, fold)) - } - - #[inline] - fn fold(self, init: Acc, fold: Fold) -> Acc - where - Fold: FnMut(Acc, Self::Item) -> Acc, - { - self.iter.fold(init, filter_map_fold(self.f, fold)) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl DoubleEndedIterator for FilterMap -where - F: FnMut(I::Item) -> Option, -{ - #[inline] - fn next_back(&mut self) -> Option { - #[inline] - fn find( - f: &mut impl FnMut(T) -> Option, - ) -> impl FnMut((), T) -> LoopState<(), B> + '_ { - move |(), x| match f(x) { - Some(x) => LoopState::Break(x), - None => LoopState::Continue(()), - } - } - - self.iter.try_rfold((), find(&mut self.f)).break_value() - } - - #[inline] - fn try_rfold(&mut self, init: Acc, fold: Fold) -> R - where - Self: Sized, - Fold: FnMut(Acc, Self::Item) -> R, - R: Try, - { - self.iter.try_rfold(init, filter_map_try_fold(&mut self.f, fold)) - } - - #[inline] - fn rfold(self, init: Acc, fold: Fold) -> Acc - where - Fold: FnMut(Acc, Self::Item) -> Acc, - { - self.iter.rfold(init, filter_map_fold(self.f, fold)) - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for FilterMap where F: FnMut(I::Item) -> Option {} - -/// An iterator that yields the current count and the element during iteration. -/// -/// This `struct` is created by the [`enumerate`] method on [`Iterator`]. See its -/// documentation for more. -/// -/// [`enumerate`]: trait.Iterator.html#method.enumerate -/// [`Iterator`]: trait.Iterator.html -#[derive(Clone, Debug)] -#[must_use = "iterators are lazy and do nothing unless consumed"] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct Enumerate { - iter: I, - count: usize, -} -impl Enumerate { - pub(super) fn new(iter: I) -> Enumerate { - Enumerate { iter, count: 0 } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for Enumerate -where - I: Iterator, -{ - type Item = (usize, ::Item); - - /// # Overflow Behavior - /// - /// The method does no guarding against overflows, so enumerating more than - /// `usize::MAX` elements either produces the wrong result or panics. If - /// debug assertions are enabled, a panic is guaranteed. - /// - /// # Panics - /// - /// Might panic if the index of the element overflows a `usize`. - #[inline] - fn next(&mut self) -> Option<(usize, ::Item)> { - let a = self.iter.next()?; - let i = self.count; - // Possible undefined overflow. - AddAssign::add_assign(&mut self.count, 1); - Some((i, a)) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() - } - - #[inline] - fn nth(&mut self, n: usize) -> Option<(usize, I::Item)> { - let a = self.iter.nth(n)?; - // Possible undefined overflow. - let i = Add::add(self.count, n); - self.count = Add::add(i, 1); - Some((i, a)) - } - - #[inline] - fn count(self) -> usize { - self.iter.count() - } - - #[inline] - fn try_fold(&mut self, init: Acc, fold: Fold) -> R - where - Self: Sized, - Fold: FnMut(Acc, Self::Item) -> R, - R: Try, - { - #[inline] - fn enumerate<'a, T, Acc, R>( - count: &'a mut usize, - mut fold: impl FnMut(Acc, (usize, T)) -> R + 'a, - ) -> impl FnMut(Acc, T) -> R + 'a { - move |acc, item| { - let acc = fold(acc, (*count, item)); - // Possible undefined overflow. - AddAssign::add_assign(count, 1); - acc - } - } - - self.iter.try_fold(init, enumerate(&mut self.count, fold)) - } - - #[inline] - fn fold(self, init: Acc, fold: Fold) -> Acc - where - Fold: FnMut(Acc, Self::Item) -> Acc, - { - #[inline] - fn enumerate( - mut count: usize, - mut fold: impl FnMut(Acc, (usize, T)) -> Acc, - ) -> impl FnMut(Acc, T) -> Acc { - move |acc, item| { - let acc = fold(acc, (count, item)); - // Possible undefined overflow. - AddAssign::add_assign(&mut count, 1); - acc - } - } - - self.iter.fold(init, enumerate(self.count, fold)) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl DoubleEndedIterator for Enumerate -where - I: ExactSizeIterator + DoubleEndedIterator, -{ - #[inline] - fn next_back(&mut self) -> Option<(usize, ::Item)> { - let a = self.iter.next_back()?; - let len = self.iter.len(); - // Can safely add, `ExactSizeIterator` promises that the number of - // elements fits into a `usize`. - Some((self.count + len, a)) - } - - #[inline] - fn nth_back(&mut self, n: usize) -> Option<(usize, ::Item)> { - let a = self.iter.nth_back(n)?; - let len = self.iter.len(); - // Can safely add, `ExactSizeIterator` promises that the number of - // elements fits into a `usize`. - Some((self.count + len, a)) - } - - #[inline] - fn try_rfold(&mut self, init: Acc, fold: Fold) -> R - where - Self: Sized, - Fold: FnMut(Acc, Self::Item) -> R, - R: Try, - { - // Can safely add and subtract the count, as `ExactSizeIterator` promises - // that the number of elements fits into a `usize`. - fn enumerate( - mut count: usize, - mut fold: impl FnMut(Acc, (usize, T)) -> R, - ) -> impl FnMut(Acc, T) -> R { - move |acc, item| { - count -= 1; - fold(acc, (count, item)) - } - } - - let count = self.count + self.iter.len(); - self.iter.try_rfold(init, enumerate(count, fold)) - } - - #[inline] - fn rfold(self, init: Acc, fold: Fold) -> Acc - where - Fold: FnMut(Acc, Self::Item) -> Acc, - { - // Can safely add and subtract the count, as `ExactSizeIterator` promises - // that the number of elements fits into a `usize`. - fn enumerate( - mut count: usize, - mut fold: impl FnMut(Acc, (usize, T)) -> Acc, - ) -> impl FnMut(Acc, T) -> Acc { - move |acc, item| { - count -= 1; - fold(acc, (count, item)) - } - } - - let count = self.count + self.iter.len(); - self.iter.rfold(init, enumerate(count, fold)) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ExactSizeIterator for Enumerate -where - I: ExactSizeIterator, -{ - fn len(&self) -> usize { - self.iter.len() - } - - fn is_empty(&self) -> bool { - self.iter.is_empty() - } -} - -#[doc(hidden)] -unsafe impl TrustedRandomAccess for Enumerate -where - I: TrustedRandomAccess, -{ - unsafe fn get_unchecked(&mut self, i: usize) -> (usize, I::Item) { - (self.count + i, self.iter.get_unchecked(i)) - } - - fn may_have_side_effect() -> bool { - I::may_have_side_effect() - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Enumerate where I: FusedIterator {} - -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for Enumerate where I: TrustedLen {} - -/// An iterator with a `peek()` that returns an optional reference to the next -/// element. -/// -/// This `struct` is created by the [`peekable`] method on [`Iterator`]. See its -/// documentation for more. -/// -/// [`peekable`]: trait.Iterator.html#method.peekable -/// [`Iterator`]: trait.Iterator.html -#[derive(Clone, Debug)] -#[must_use = "iterators are lazy and do nothing unless consumed"] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct Peekable { - iter: I, - /// Remember a peeked value, even if it was None. - peeked: Option>, -} -impl Peekable { - pub(super) fn new(iter: I) -> Peekable { - Peekable { iter, peeked: None } - } -} - -// Peekable must remember if a None has been seen in the `.peek()` method. -// It ensures that `.peek(); .peek();` or `.peek(); .next();` only advances the -// underlying iterator at most once. This does not by itself make the iterator -// fused. -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for Peekable { - type Item = I::Item; - - #[inline] - fn next(&mut self) -> Option { - match self.peeked.take() { - Some(v) => v, - None => self.iter.next(), - } - } - - #[inline] - #[rustc_inherit_overflow_checks] - fn count(mut self) -> usize { - match self.peeked.take() { - Some(None) => 0, - Some(Some(_)) => 1 + self.iter.count(), - None => self.iter.count(), - } - } - - #[inline] - fn nth(&mut self, n: usize) -> Option { - match self.peeked.take() { - Some(None) => None, - Some(v @ Some(_)) if n == 0 => v, - Some(Some(_)) => self.iter.nth(n - 1), - None => self.iter.nth(n), - } - } - - #[inline] - fn last(mut self) -> Option { - let peek_opt = match self.peeked.take() { - Some(None) => return None, - Some(v) => v, - None => None, - }; - self.iter.last().or(peek_opt) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let peek_len = match self.peeked { - Some(None) => return (0, Some(0)), - Some(Some(_)) => 1, - None => 0, - }; - let (lo, hi) = self.iter.size_hint(); - let lo = lo.saturating_add(peek_len); - let hi = match hi { - Some(x) => x.checked_add(peek_len), - None => None, - }; - (lo, hi) - } - - #[inline] - fn try_fold(&mut self, init: B, mut f: F) -> R - where - Self: Sized, - F: FnMut(B, Self::Item) -> R, - R: Try, - { - let acc = match self.peeked.take() { - Some(None) => return Try::from_ok(init), - Some(Some(v)) => f(init, v)?, - None => init, - }; - self.iter.try_fold(acc, f) - } - - #[inline] - fn fold(self, init: Acc, mut fold: Fold) -> Acc - where - Fold: FnMut(Acc, Self::Item) -> Acc, - { - let acc = match self.peeked { - Some(None) => return init, - Some(Some(v)) => fold(init, v), - None => init, - }; - self.iter.fold(acc, fold) - } -} - -#[stable(feature = "double_ended_peek_iterator", since = "1.38.0")] -impl DoubleEndedIterator for Peekable -where - I: DoubleEndedIterator, -{ - #[inline] - fn next_back(&mut self) -> Option { - match self.peeked.as_mut() { - Some(v @ Some(_)) => self.iter.next_back().or_else(|| v.take()), - Some(None) => None, - None => self.iter.next_back(), - } - } - - #[inline] - fn try_rfold(&mut self, init: B, mut f: F) -> R - where - Self: Sized, - F: FnMut(B, Self::Item) -> R, - R: Try, - { - match self.peeked.take() { - Some(None) => Try::from_ok(init), - Some(Some(v)) => match self.iter.try_rfold(init, &mut f).into_result() { - Ok(acc) => f(acc, v), - Err(e) => { - self.peeked = Some(Some(v)); - Try::from_error(e) - } - }, - None => self.iter.try_rfold(init, f), - } - } - - #[inline] - fn rfold(self, init: Acc, mut fold: Fold) -> Acc - where - Fold: FnMut(Acc, Self::Item) -> Acc, - { - match self.peeked { - Some(None) => init, - Some(Some(v)) => { - let acc = self.iter.rfold(init, &mut fold); - fold(acc, v) - } - None => self.iter.rfold(init, fold), - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ExactSizeIterator for Peekable {} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Peekable {} - -impl Peekable { - /// Returns a reference to the next() value without advancing the iterator. - /// - /// Like [`next`], if there is a value, it is wrapped in a `Some(T)`. - /// But if the iteration is over, `None` is returned. - /// - /// [`next`]: trait.Iterator.html#tymethod.next - /// - /// Because `peek()` returns a reference, and many iterators iterate over - /// references, there can be a possibly confusing situation where the - /// return value is a double reference. You can see this effect in the - /// examples below. - /// - /// # Examples - /// - /// Basic usage: - /// - /// ``` - /// let xs = [1, 2, 3]; - /// - /// let mut iter = xs.iter().peekable(); - /// - /// // peek() lets us see into the future - /// assert_eq!(iter.peek(), Some(&&1)); - /// assert_eq!(iter.next(), Some(&1)); - /// - /// assert_eq!(iter.next(), Some(&2)); - /// - /// // The iterator does not advance even if we `peek` multiple times - /// assert_eq!(iter.peek(), Some(&&3)); - /// assert_eq!(iter.peek(), Some(&&3)); - /// - /// assert_eq!(iter.next(), Some(&3)); - /// - /// // After the iterator is finished, so is `peek()` - /// assert_eq!(iter.peek(), None); - /// assert_eq!(iter.next(), None); - /// ``` - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn peek(&mut self) -> Option<&I::Item> { - let iter = &mut self.iter; - self.peeked.get_or_insert_with(|| iter.next()).as_ref() - } -} - -/// An iterator that rejects elements while `predicate` returns `true`. -/// -/// This `struct` is created by the [`skip_while`] method on [`Iterator`]. See its -/// documentation for more. -/// -/// [`skip_while`]: trait.Iterator.html#method.skip_while -/// [`Iterator`]: trait.Iterator.html -#[must_use = "iterators are lazy and do nothing unless consumed"] -#[stable(feature = "rust1", since = "1.0.0")] -#[derive(Clone)] -pub struct SkipWhile { - iter: I, - flag: bool, - predicate: P, -} -impl SkipWhile { - pub(super) fn new(iter: I, predicate: P) -> SkipWhile { - SkipWhile { iter, flag: false, predicate } - } -} - -#[stable(feature = "core_impl_debug", since = "1.9.0")] -impl fmt::Debug for SkipWhile { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SkipWhile").field("iter", &self.iter).field("flag", &self.flag).finish() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for SkipWhile -where - P: FnMut(&I::Item) -> bool, -{ - type Item = I::Item; - - #[inline] - fn next(&mut self) -> Option { - fn check<'a, T>( - flag: &'a mut bool, - pred: &'a mut impl FnMut(&T) -> bool, - ) -> impl FnMut(&T) -> bool + 'a { - move |x| { - if *flag || !pred(x) { - *flag = true; - true - } else { - false - } - } - } - - let flag = &mut self.flag; - let pred = &mut self.predicate; - self.iter.find(check(flag, pred)) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let (_, upper) = self.iter.size_hint(); - (0, upper) // can't know a lower bound, due to the predicate - } - - #[inline] - fn try_fold(&mut self, mut init: Acc, mut fold: Fold) -> R - where - Self: Sized, - Fold: FnMut(Acc, Self::Item) -> R, - R: Try, - { - if !self.flag { - match self.next() { - Some(v) => init = fold(init, v)?, - None => return Try::from_ok(init), - } - } - self.iter.try_fold(init, fold) - } - - #[inline] - fn fold(mut self, mut init: Acc, mut fold: Fold) -> Acc - where - Fold: FnMut(Acc, Self::Item) -> Acc, - { - if !self.flag { - match self.next() { - Some(v) => init = fold(init, v), - None => return init, - } - } - self.iter.fold(init, fold) - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for SkipWhile -where - I: FusedIterator, - P: FnMut(&I::Item) -> bool, -{ -} - -/// An iterator that only accepts elements while `predicate` returns `true`. -/// -/// This `struct` is created by the [`take_while`] method on [`Iterator`]. See its -/// documentation for more. -/// -/// [`take_while`]: trait.Iterator.html#method.take_while -/// [`Iterator`]: trait.Iterator.html -#[must_use = "iterators are lazy and do nothing unless consumed"] -#[stable(feature = "rust1", since = "1.0.0")] -#[derive(Clone)] -pub struct TakeWhile { - iter: I, - flag: bool, - predicate: P, -} -impl TakeWhile { - pub(super) fn new(iter: I, predicate: P) -> TakeWhile { - TakeWhile { iter, flag: false, predicate } - } -} - -#[stable(feature = "core_impl_debug", since = "1.9.0")] -impl fmt::Debug for TakeWhile { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("TakeWhile").field("iter", &self.iter).field("flag", &self.flag).finish() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for TakeWhile -where - P: FnMut(&I::Item) -> bool, -{ - type Item = I::Item; - - #[inline] - fn next(&mut self) -> Option { - if self.flag { - None - } else { - let x = self.iter.next()?; - if (self.predicate)(&x) { - Some(x) - } else { - self.flag = true; - None - } - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - if self.flag { - (0, Some(0)) - } else { - let (_, upper) = self.iter.size_hint(); - (0, upper) // can't know a lower bound, due to the predicate - } - } - - #[inline] - fn try_fold(&mut self, init: Acc, fold: Fold) -> R - where - Self: Sized, - Fold: FnMut(Acc, Self::Item) -> R, - R: Try, - { - fn check<'a, T, Acc, R: Try>( - flag: &'a mut bool, - p: &'a mut impl FnMut(&T) -> bool, - mut fold: impl FnMut(Acc, T) -> R + 'a, - ) -> impl FnMut(Acc, T) -> LoopState + 'a { - move |acc, x| { - if p(&x) { - LoopState::from_try(fold(acc, x)) - } else { - *flag = true; - LoopState::Break(Try::from_ok(acc)) - } - } - } - - if self.flag { - Try::from_ok(init) - } else { - let flag = &mut self.flag; - let p = &mut self.predicate; - self.iter.try_fold(init, check(flag, p, fold)).into_try() - } - } -} - -/// An iterator that only accepts elements while `predicate` returns `Some(_)`. -/// -/// This `struct` is created by the [`map_while`] method on [`Iterator`]. See its -/// documentation for more. -/// -/// [`map_while`]: trait.Iterator.html#method.map_while -/// [`Iterator`]: trait.Iterator.html -#[must_use = "iterators are lazy and do nothing unless consumed"] -#[unstable(feature = "iter_map_while", reason = "recently added", issue = "68537")] -#[derive(Clone)] -pub struct MapWhile { - iter: I, - finished: bool, - predicate: P, -} - -impl MapWhile { - pub(super) fn new(iter: I, predicate: P) -> MapWhile { - MapWhile { iter, finished: false, predicate } - } -} - -#[unstable(feature = "iter_map_while", reason = "recently added", issue = "68537")] -impl fmt::Debug for MapWhile { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("MapWhile").field("iter", &self.iter).field("flag", &self.finished).finish() - } -} - -#[unstable(feature = "iter_map_while", reason = "recently added", issue = "68537")] -impl Iterator for MapWhile -where - P: FnMut(I::Item) -> Option, -{ - type Item = B; - - #[inline] - fn next(&mut self) -> Option { - if self.finished { - None - } else { - let x = self.iter.next()?; - let ret = (self.predicate)(x); - self.finished = ret.is_none(); - ret - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - if self.finished { - (0, Some(0)) - } else { - let (_, upper) = self.iter.size_hint(); - (0, upper) // can't know a lower bound, due to the predicate - } - } - - #[inline] - fn try_fold(&mut self, init: Acc, fold: Fold) -> R - where - Self: Sized, - Fold: FnMut(Acc, Self::Item) -> R, - R: Try, - { - fn check<'a, B, T, Acc, R: Try>( - flag: &'a mut bool, - p: &'a mut impl FnMut(T) -> Option, - mut fold: impl FnMut(Acc, B) -> R + 'a, - ) -> impl FnMut(Acc, T) -> LoopState + 'a { - move |acc, x| match p(x) { - Some(item) => LoopState::from_try(fold(acc, item)), - None => { - *flag = true; - LoopState::Break(Try::from_ok(acc)) - } - } - } - - if self.finished { - Try::from_ok(init) - } else { - let flag = &mut self.finished; - let p = &mut self.predicate; - self.iter.try_fold(init, check(flag, p, fold)).into_try() - } - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for TakeWhile -where - I: FusedIterator, - P: FnMut(&I::Item) -> bool, -{ -} - -/// An iterator that skips over `n` elements of `iter`. -/// -/// This `struct` is created by the [`skip`] method on [`Iterator`]. See its -/// documentation for more. -/// -/// [`skip`]: trait.Iterator.html#method.skip -/// [`Iterator`]: trait.Iterator.html -#[derive(Clone, Debug)] -#[must_use = "iterators are lazy and do nothing unless consumed"] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct Skip { - iter: I, - n: usize, -} -impl Skip { - pub(super) fn new(iter: I, n: usize) -> Skip { - Skip { iter, n } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for Skip -where - I: Iterator, -{ - type Item = ::Item; - - #[inline] - fn next(&mut self) -> Option { - if self.n == 0 { - self.iter.next() - } else { - let old_n = self.n; - self.n = 0; - self.iter.nth(old_n) - } - } - - #[inline] - fn nth(&mut self, n: usize) -> Option { - // Can't just add n + self.n due to overflow. - if self.n > 0 { - let to_skip = self.n; - self.n = 0; - // nth(n) skips n+1 - self.iter.nth(to_skip - 1)?; - } - self.iter.nth(n) - } - - #[inline] - fn count(mut self) -> usize { - if self.n > 0 { - // nth(n) skips n+1 - if self.iter.nth(self.n - 1).is_none() { - return 0; - } - } - self.iter.count() - } - - #[inline] - fn last(mut self) -> Option { - if self.n > 0 { - // nth(n) skips n+1 - self.iter.nth(self.n - 1)?; - } - self.iter.last() - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let (lower, upper) = self.iter.size_hint(); - - let lower = lower.saturating_sub(self.n); - let upper = match upper { - Some(x) => Some(x.saturating_sub(self.n)), - None => None, - }; - - (lower, upper) - } - - #[inline] - fn try_fold(&mut self, init: Acc, fold: Fold) -> R - where - Self: Sized, - Fold: FnMut(Acc, Self::Item) -> R, - R: Try, - { - let n = self.n; - self.n = 0; - if n > 0 { - // nth(n) skips n+1 - if self.iter.nth(n - 1).is_none() { - return Try::from_ok(init); - } - } - self.iter.try_fold(init, fold) - } - - #[inline] - fn fold(mut self, init: Acc, fold: Fold) -> Acc - where - Fold: FnMut(Acc, Self::Item) -> Acc, - { - if self.n > 0 { - // nth(n) skips n+1 - if self.iter.nth(self.n - 1).is_none() { - return init; - } - } - self.iter.fold(init, fold) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ExactSizeIterator for Skip where I: ExactSizeIterator {} - -#[stable(feature = "double_ended_skip_iterator", since = "1.9.0")] -impl DoubleEndedIterator for Skip -where - I: DoubleEndedIterator + ExactSizeIterator, -{ - fn next_back(&mut self) -> Option { - if self.len() > 0 { self.iter.next_back() } else { None } - } - - #[inline] - fn nth_back(&mut self, n: usize) -> Option { - let len = self.len(); - if n < len { - self.iter.nth_back(n) - } else { - if len > 0 { - // consume the original iterator - self.iter.nth_back(len - 1); - } - None - } - } - - fn try_rfold(&mut self, init: Acc, fold: Fold) -> R - where - Self: Sized, - Fold: FnMut(Acc, Self::Item) -> R, - R: Try, - { - fn check>( - mut n: usize, - mut fold: impl FnMut(Acc, T) -> R, - ) -> impl FnMut(Acc, T) -> LoopState { - move |acc, x| { - n -= 1; - let r = fold(acc, x); - if n == 0 { LoopState::Break(r) } else { LoopState::from_try(r) } - } - } - - let n = self.len(); - if n == 0 { - Try::from_ok(init) - } else { - self.iter.try_rfold(init, check(n, fold)).into_try() - } - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Skip where I: FusedIterator {} - -/// An iterator that only iterates over the first `n` iterations of `iter`. -/// -/// This `struct` is created by the [`take`] method on [`Iterator`]. See its -/// documentation for more. -/// -/// [`take`]: trait.Iterator.html#method.take -/// [`Iterator`]: trait.Iterator.html -#[derive(Clone, Debug)] -#[must_use = "iterators are lazy and do nothing unless consumed"] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct Take { - pub(super) iter: I, - pub(super) n: usize, -} -impl Take { - pub(super) fn new(iter: I, n: usize) -> Take { - Take { iter, n } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for Take -where - I: Iterator, -{ - type Item = ::Item; - - #[inline] - fn next(&mut self) -> Option<::Item> { - if self.n != 0 { - self.n -= 1; - self.iter.next() - } else { - None - } - } - - #[inline] - fn nth(&mut self, n: usize) -> Option { - if self.n > n { - self.n -= n + 1; - self.iter.nth(n) - } else { - if self.n > 0 { - self.iter.nth(self.n - 1); - self.n = 0; - } - None - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - if self.n == 0 { - return (0, Some(0)); - } - - let (lower, upper) = self.iter.size_hint(); - - let lower = cmp::min(lower, self.n); - - let upper = match upper { - Some(x) if x < self.n => Some(x), - _ => Some(self.n), - }; - - (lower, upper) - } - - #[inline] - fn try_fold(&mut self, init: Acc, fold: Fold) -> R - where - Self: Sized, - Fold: FnMut(Acc, Self::Item) -> R, - R: Try, - { - fn check<'a, T, Acc, R: Try>( - n: &'a mut usize, - mut fold: impl FnMut(Acc, T) -> R + 'a, - ) -> impl FnMut(Acc, T) -> LoopState + 'a { - move |acc, x| { - *n -= 1; - let r = fold(acc, x); - if *n == 0 { LoopState::Break(r) } else { LoopState::from_try(r) } - } - } - - if self.n == 0 { - Try::from_ok(init) - } else { - let n = &mut self.n; - self.iter.try_fold(init, check(n, fold)).into_try() - } - } -} - -#[stable(feature = "double_ended_take_iterator", since = "1.38.0")] -impl DoubleEndedIterator for Take -where - I: DoubleEndedIterator + ExactSizeIterator, -{ - #[inline] - fn next_back(&mut self) -> Option { - if self.n == 0 { - None - } else { - let n = self.n; - self.n -= 1; - self.iter.nth_back(self.iter.len().saturating_sub(n)) - } - } - - #[inline] - fn nth_back(&mut self, n: usize) -> Option { - let len = self.iter.len(); - if self.n > n { - let m = len.saturating_sub(self.n) + n; - self.n -= n + 1; - self.iter.nth_back(m) - } else { - if len > 0 { - self.iter.nth_back(len - 1); - } - None - } - } - - #[inline] - fn try_rfold(&mut self, init: Acc, fold: Fold) -> R - where - Self: Sized, - Fold: FnMut(Acc, Self::Item) -> R, - R: Try, - { - if self.n == 0 { - Try::from_ok(init) - } else { - let len = self.iter.len(); - if len > self.n && self.iter.nth_back(len - self.n - 1).is_none() { - Try::from_ok(init) - } else { - self.iter.try_rfold(init, fold) - } - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ExactSizeIterator for Take where I: ExactSizeIterator {} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Take where I: FusedIterator {} - -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for Take {} - -/// An iterator to maintain state while iterating another iterator. -/// -/// This `struct` is created by the [`scan`] method on [`Iterator`]. See its -/// documentation for more. -/// -/// [`scan`]: trait.Iterator.html#method.scan -/// [`Iterator`]: trait.Iterator.html -#[must_use = "iterators are lazy and do nothing unless consumed"] -#[stable(feature = "rust1", since = "1.0.0")] -#[derive(Clone)] -pub struct Scan { - iter: I, - f: F, - state: St, -} -impl Scan { - pub(super) fn new(iter: I, state: St, f: F) -> Scan { - Scan { iter, state, f } - } -} - -#[stable(feature = "core_impl_debug", since = "1.9.0")] -impl fmt::Debug for Scan { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Scan").field("iter", &self.iter).field("state", &self.state).finish() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for Scan -where - I: Iterator, - F: FnMut(&mut St, I::Item) -> Option, -{ - type Item = B; - - #[inline] - fn next(&mut self) -> Option { - let a = self.iter.next()?; - (self.f)(&mut self.state, a) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let (_, upper) = self.iter.size_hint(); - (0, upper) // can't know a lower bound, due to the scan function - } - - #[inline] - fn try_fold(&mut self, init: Acc, fold: Fold) -> R - where - Self: Sized, - Fold: FnMut(Acc, Self::Item) -> R, - R: Try, - { - fn scan<'a, T, St, B, Acc, R: Try>( - state: &'a mut St, - f: &'a mut impl FnMut(&mut St, T) -> Option, - mut fold: impl FnMut(Acc, B) -> R + 'a, - ) -> impl FnMut(Acc, T) -> LoopState + 'a { - move |acc, x| match f(state, x) { - None => LoopState::Break(Try::from_ok(acc)), - Some(x) => LoopState::from_try(fold(acc, x)), - } - } - - let state = &mut self.state; - let f = &mut self.f; - self.iter.try_fold(init, scan(state, f, fold)).into_try() - } -} - -/// An iterator that yields `None` forever after the underlying iterator -/// yields `None` once. -/// -/// This `struct` is created by the [`fuse`] method on [`Iterator`]. See its -/// documentation for more. -/// -/// [`fuse`]: trait.Iterator.html#method.fuse -/// [`Iterator`]: trait.Iterator.html -#[derive(Clone, Debug)] -#[must_use = "iterators are lazy and do nothing unless consumed"] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct Fuse { - iter: I, - done: bool, -} -impl Fuse { - pub(super) fn new(iter: I) -> Fuse { - Fuse { iter, done: false } - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Fuse where I: Iterator {} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for Fuse -where - I: Iterator, -{ - type Item = ::Item; - - #[inline] - default fn next(&mut self) -> Option<::Item> { - if self.done { - None - } else { - let next = self.iter.next(); - self.done = next.is_none(); - next - } - } - - #[inline] - default fn nth(&mut self, n: usize) -> Option { - if self.done { - None - } else { - let nth = self.iter.nth(n); - self.done = nth.is_none(); - nth - } - } - - #[inline] - default fn last(self) -> Option { - if self.done { None } else { self.iter.last() } - } - - #[inline] - default fn count(self) -> usize { - if self.done { 0 } else { self.iter.count() } - } - - #[inline] - default fn size_hint(&self) -> (usize, Option) { - if self.done { (0, Some(0)) } else { self.iter.size_hint() } - } - - #[inline] - default fn try_fold(&mut self, init: Acc, fold: Fold) -> R - where - Self: Sized, - Fold: FnMut(Acc, Self::Item) -> R, - R: Try, - { - if self.done { - Try::from_ok(init) - } else { - let acc = self.iter.try_fold(init, fold)?; - self.done = true; - Try::from_ok(acc) - } - } - - #[inline] - default fn fold(self, init: Acc, fold: Fold) -> Acc - where - Fold: FnMut(Acc, Self::Item) -> Acc, - { - if self.done { init } else { self.iter.fold(init, fold) } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl DoubleEndedIterator for Fuse -where - I: DoubleEndedIterator, -{ - #[inline] - default fn next_back(&mut self) -> Option<::Item> { - if self.done { - None - } else { - let next = self.iter.next_back(); - self.done = next.is_none(); - next - } - } - - #[inline] - default fn nth_back(&mut self, n: usize) -> Option<::Item> { - if self.done { - None - } else { - let nth = self.iter.nth_back(n); - self.done = nth.is_none(); - nth - } - } - - #[inline] - default fn try_rfold(&mut self, init: Acc, fold: Fold) -> R - where - Self: Sized, - Fold: FnMut(Acc, Self::Item) -> R, - R: Try, - { - if self.done { - Try::from_ok(init) - } else { - let acc = self.iter.try_rfold(init, fold)?; - self.done = true; - Try::from_ok(acc) - } - } - - #[inline] - default fn rfold(self, init: Acc, fold: Fold) -> Acc - where - Fold: FnMut(Acc, Self::Item) -> Acc, - { - if self.done { init } else { self.iter.rfold(init, fold) } - } -} - -unsafe impl TrustedRandomAccess for Fuse -where - I: TrustedRandomAccess, -{ - unsafe fn get_unchecked(&mut self, i: usize) -> I::Item { - self.iter.get_unchecked(i) - } - - fn may_have_side_effect() -> bool { - I::may_have_side_effect() - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl Iterator for Fuse -where - I: FusedIterator, -{ - #[inline] - fn next(&mut self) -> Option<::Item> { - self.iter.next() - } - - #[inline] - fn nth(&mut self, n: usize) -> Option { - self.iter.nth(n) - } - - #[inline] - fn last(self) -> Option { - self.iter.last() - } - - #[inline] - fn count(self) -> usize { - self.iter.count() - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() - } - - #[inline] - fn try_fold(&mut self, init: Acc, fold: Fold) -> R - where - Self: Sized, - Fold: FnMut(Acc, Self::Item) -> R, - R: Try, - { - self.iter.try_fold(init, fold) - } - - #[inline] - fn fold(self, init: Acc, fold: Fold) -> Acc - where - Fold: FnMut(Acc, Self::Item) -> Acc, - { - self.iter.fold(init, fold) - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl DoubleEndedIterator for Fuse -where - I: DoubleEndedIterator + FusedIterator, -{ - #[inline] - fn next_back(&mut self) -> Option<::Item> { - self.iter.next_back() - } - - #[inline] - fn nth_back(&mut self, n: usize) -> Option<::Item> { - self.iter.nth_back(n) - } - - #[inline] - fn try_rfold(&mut self, init: Acc, fold: Fold) -> R - where - Self: Sized, - Fold: FnMut(Acc, Self::Item) -> R, - R: Try, - { - self.iter.try_rfold(init, fold) - } - - #[inline] - fn rfold(self, init: Acc, fold: Fold) -> Acc - where - Fold: FnMut(Acc, Self::Item) -> Acc, - { - self.iter.rfold(init, fold) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ExactSizeIterator for Fuse -where - I: ExactSizeIterator, -{ - fn len(&self) -> usize { - self.iter.len() - } - - fn is_empty(&self) -> bool { - self.iter.is_empty() - } -} - -/// An iterator that calls a function with a reference to each element before -/// yielding it. -/// -/// This `struct` is created by the [`inspect`] method on [`Iterator`]. See its -/// documentation for more. -/// -/// [`inspect`]: trait.Iterator.html#method.inspect -/// [`Iterator`]: trait.Iterator.html -#[must_use = "iterators are lazy and do nothing unless consumed"] -#[stable(feature = "rust1", since = "1.0.0")] -#[derive(Clone)] -pub struct Inspect { - iter: I, - f: F, -} -impl Inspect { - pub(super) fn new(iter: I, f: F) -> Inspect { - Inspect { iter, f } - } -} - -#[stable(feature = "core_impl_debug", since = "1.9.0")] -impl fmt::Debug for Inspect { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Inspect").field("iter", &self.iter).finish() - } -} - -impl Inspect -where - F: FnMut(&I::Item), -{ - #[inline] - fn do_inspect(&mut self, elt: Option) -> Option { - if let Some(ref a) = elt { - (self.f)(a); - } - - elt - } -} - -fn inspect_fold( - mut f: impl FnMut(&T), - mut fold: impl FnMut(Acc, T) -> Acc, -) -> impl FnMut(Acc, T) -> Acc { - move |acc, item| { - f(&item); - fold(acc, item) - } -} - -fn inspect_try_fold<'a, T, Acc, R>( - f: &'a mut impl FnMut(&T), - mut fold: impl FnMut(Acc, T) -> R + 'a, -) -> impl FnMut(Acc, T) -> R + 'a { - move |acc, item| { - f(&item); - fold(acc, item) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for Inspect -where - F: FnMut(&I::Item), -{ - type Item = I::Item; - - #[inline] - fn next(&mut self) -> Option { - let next = self.iter.next(); - self.do_inspect(next) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() - } - - #[inline] - fn try_fold(&mut self, init: Acc, fold: Fold) -> R - where - Self: Sized, - Fold: FnMut(Acc, Self::Item) -> R, - R: Try, - { - self.iter.try_fold(init, inspect_try_fold(&mut self.f, fold)) - } - - #[inline] - fn fold(self, init: Acc, fold: Fold) -> Acc - where - Fold: FnMut(Acc, Self::Item) -> Acc, - { - self.iter.fold(init, inspect_fold(self.f, fold)) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl DoubleEndedIterator for Inspect -where - F: FnMut(&I::Item), -{ - #[inline] - fn next_back(&mut self) -> Option { - let next = self.iter.next_back(); - self.do_inspect(next) - } - - #[inline] - fn try_rfold(&mut self, init: Acc, fold: Fold) -> R - where - Self: Sized, - Fold: FnMut(Acc, Self::Item) -> R, - R: Try, - { - self.iter.try_rfold(init, inspect_try_fold(&mut self.f, fold)) - } - - #[inline] - fn rfold(self, init: Acc, fold: Fold) -> Acc - where - Fold: FnMut(Acc, Self::Item) -> Acc, - { - self.iter.rfold(init, inspect_fold(self.f, fold)) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ExactSizeIterator for Inspect -where - F: FnMut(&I::Item), -{ - fn len(&self) -> usize { - self.iter.len() - } - - fn is_empty(&self) -> bool { - self.iter.is_empty() - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Inspect where F: FnMut(&I::Item) {} - -/// An iterator adapter that produces output as long as the underlying -/// iterator produces `Result::Ok` values. -/// -/// If an error is encountered, the iterator stops and the error is -/// stored. -pub(crate) struct ResultShunt<'a, I, E> { - iter: I, - error: &'a mut Result<(), E>, -} - -/// Process the given iterator as if it yielded a `T` instead of a -/// `Result`. Any errors will stop the inner iterator and -/// the overall result will be an error. -pub(crate) fn process_results(iter: I, mut f: F) -> Result -where - I: Iterator>, - for<'a> F: FnMut(ResultShunt<'a, I, E>) -> U, +// SAFETY: GenericShunt::next calls `I::try_for_each`, which has to advance `iter` +// in order to return `Some(_)`. Since `iter` has type `I: InPlaceIterable` it's +// guaranteed that at least one item will be moved out from the underlying source. +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for GenericShunt<'_, I, R> where + I: Iterator> + InPlaceIterable { - let mut error = Ok(()); - let shunt = ResultShunt { iter, error: &mut error }; - let value = f(shunt); - error.map(|()| value) -} - -impl Iterator for ResultShunt<'_, I, E> -where - I: Iterator>, -{ - type Item = T; - - fn next(&mut self) -> Option { - self.find(|_| true) - } - - fn size_hint(&self) -> (usize, Option) { - if self.error.is_err() { - (0, Some(0)) - } else { - let (_, upper) = self.iter.size_hint(); - (0, upper) - } - } - - fn try_fold(&mut self, init: B, mut f: F) -> R - where - F: FnMut(B, Self::Item) -> R, - R: Try, - { - let error = &mut *self.error; - self.iter - .try_fold(init, |acc, x| match x { - Ok(x) => LoopState::from_try(f(acc, x)), - Err(e) => { - *error = Err(e); - LoopState::Break(Try::from_ok(acc)) - } - }) - .into_try() - } } diff --git a/crux-mir/lib/core/src/iter/adapters/peekable.rs b/crux-mir/lib/core/src/iter/adapters/peekable.rs new file mode 100644 index 000000000..20aca323b --- /dev/null +++ b/crux-mir/lib/core/src/iter/adapters/peekable.rs @@ -0,0 +1,335 @@ +use crate::iter::{adapters::SourceIter, FusedIterator, TrustedLen}; +use crate::ops::{ControlFlow, Try}; + +/// An iterator with a `peek()` that returns an optional reference to the next +/// element. +/// +/// This `struct` is created by the [`peekable`] method on [`Iterator`]. See its +/// documentation for more. +/// +/// [`peekable`]: Iterator::peekable +/// [`Iterator`]: trait.Iterator.html +#[derive(Clone, Debug)] +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "rust1", since = "1.0.0")] +pub struct Peekable { + iter: I, + /// Remember a peeked value, even if it was None. + peeked: Option>, +} + +impl Peekable { + pub(in crate::iter) fn new(iter: I) -> Peekable { + Peekable { iter, peeked: None } + } +} + +// Peekable must remember if a None has been seen in the `.peek()` method. +// It ensures that `.peek(); .peek();` or `.peek(); .next();` only advances the +// underlying iterator at most once. This does not by itself make the iterator +// fused. +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for Peekable { + type Item = I::Item; + + #[inline] + fn next(&mut self) -> Option { + match self.peeked.take() { + Some(v) => v, + None => self.iter.next(), + } + } + + #[inline] + #[rustc_inherit_overflow_checks] + fn count(mut self) -> usize { + match self.peeked.take() { + Some(None) => 0, + Some(Some(_)) => 1 + self.iter.count(), + None => self.iter.count(), + } + } + + #[inline] + fn nth(&mut self, n: usize) -> Option { + match self.peeked.take() { + Some(None) => None, + Some(v @ Some(_)) if n == 0 => v, + Some(Some(_)) => self.iter.nth(n - 1), + None => self.iter.nth(n), + } + } + + #[inline] + fn last(mut self) -> Option { + let peek_opt = match self.peeked.take() { + Some(None) => return None, + Some(v) => v, + None => None, + }; + self.iter.last().or(peek_opt) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let peek_len = match self.peeked { + Some(None) => return (0, Some(0)), + Some(Some(_)) => 1, + None => 0, + }; + let (lo, hi) = self.iter.size_hint(); + let lo = lo.saturating_add(peek_len); + let hi = match hi { + Some(x) => x.checked_add(peek_len), + None => None, + }; + (lo, hi) + } + + #[inline] + fn try_fold(&mut self, init: B, mut f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: Try, + { + let acc = match self.peeked.take() { + Some(None) => return try { init }, + Some(Some(v)) => f(init, v)?, + None => init, + }; + self.iter.try_fold(acc, f) + } + + #[inline] + fn fold(self, init: Acc, mut fold: Fold) -> Acc + where + Fold: FnMut(Acc, Self::Item) -> Acc, + { + let acc = match self.peeked { + Some(None) => return init, + Some(Some(v)) => fold(init, v), + None => init, + }; + self.iter.fold(acc, fold) + } +} + +#[stable(feature = "double_ended_peek_iterator", since = "1.38.0")] +impl DoubleEndedIterator for Peekable +where + I: DoubleEndedIterator, +{ + #[inline] + fn next_back(&mut self) -> Option { + match self.peeked.as_mut() { + Some(v @ Some(_)) => self.iter.next_back().or_else(|| v.take()), + Some(None) => None, + None => self.iter.next_back(), + } + } + + #[inline] + fn try_rfold(&mut self, init: B, mut f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: Try, + { + match self.peeked.take() { + Some(None) => try { init }, + Some(Some(v)) => match self.iter.try_rfold(init, &mut f).branch() { + ControlFlow::Continue(acc) => f(acc, v), + ControlFlow::Break(r) => { + self.peeked = Some(Some(v)); + R::from_residual(r) + } + }, + None => self.iter.try_rfold(init, f), + } + } + + #[inline] + fn rfold(self, init: Acc, mut fold: Fold) -> Acc + where + Fold: FnMut(Acc, Self::Item) -> Acc, + { + match self.peeked { + Some(None) => init, + Some(Some(v)) => { + let acc = self.iter.rfold(init, &mut fold); + fold(acc, v) + } + None => self.iter.rfold(init, fold), + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ExactSizeIterator for Peekable {} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Peekable {} + +impl Peekable { + /// Returns a reference to the next() value without advancing the iterator. + /// + /// Like [`next`], if there is a value, it is wrapped in a `Some(T)`. + /// But if the iteration is over, `None` is returned. + /// + /// [`next`]: Iterator::next + /// + /// Because `peek()` returns a reference, and many iterators iterate over + /// references, there can be a possibly confusing situation where the + /// return value is a double reference. You can see this effect in the + /// examples below. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// let xs = [1, 2, 3]; + /// + /// let mut iter = xs.iter().peekable(); + /// + /// // peek() lets us see into the future + /// assert_eq!(iter.peek(), Some(&&1)); + /// assert_eq!(iter.next(), Some(&1)); + /// + /// assert_eq!(iter.next(), Some(&2)); + /// + /// // The iterator does not advance even if we `peek` multiple times + /// assert_eq!(iter.peek(), Some(&&3)); + /// assert_eq!(iter.peek(), Some(&&3)); + /// + /// assert_eq!(iter.next(), Some(&3)); + /// + /// // After the iterator is finished, so is `peek()` + /// assert_eq!(iter.peek(), None); + /// assert_eq!(iter.next(), None); + /// ``` + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn peek(&mut self) -> Option<&I::Item> { + let iter = &mut self.iter; + self.peeked.get_or_insert_with(|| iter.next()).as_ref() + } + + /// Returns a mutable reference to the next() value without advancing the iterator. + /// + /// Like [`next`], if there is a value, it is wrapped in a `Some(T)`. + /// But if the iteration is over, `None` is returned. + /// + /// Because `peek_mut()` returns a reference, and many iterators iterate over + /// references, there can be a possibly confusing situation where the + /// return value is a double reference. You can see this effect in the examples + /// below. + /// + /// [`next`]: Iterator::next + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// let mut iter = [1, 2, 3].iter().peekable(); + /// + /// // Like with `peek()`, we can see into the future without advancing the iterator. + /// assert_eq!(iter.peek_mut(), Some(&mut &1)); + /// assert_eq!(iter.peek_mut(), Some(&mut &1)); + /// assert_eq!(iter.next(), Some(&1)); + /// + /// // Peek into the iterator and set the value behind the mutable reference. + /// if let Some(p) = iter.peek_mut() { + /// assert_eq!(*p, &2); + /// *p = &5; + /// } + /// + /// // The value we put in reappears as the iterator continues. + /// assert_eq!(iter.collect::>(), vec![&5, &3]); + /// ``` + #[inline] + #[stable(feature = "peekable_peek_mut", since = "1.53.0")] + pub fn peek_mut(&mut self) -> Option<&mut I::Item> { + let iter = &mut self.iter; + self.peeked.get_or_insert_with(|| iter.next()).as_mut() + } + + /// Consume and return the next value of this iterator if a condition is true. + /// + /// If `func` returns `true` for the next value of this iterator, consume and return it. + /// Otherwise, return `None`. + /// + /// # Examples + /// Consume a number if it's equal to 0. + /// ``` + /// let mut iter = (0..5).peekable(); + /// // The first item of the iterator is 0; consume it. + /// assert_eq!(iter.next_if(|&x| x == 0), Some(0)); + /// // The next item returned is now 1, so `consume` will return `false`. + /// assert_eq!(iter.next_if(|&x| x == 0), None); + /// // `next_if` saves the value of the next item if it was not equal to `expected`. + /// assert_eq!(iter.next(), Some(1)); + /// ``` + /// + /// Consume any number less than 10. + /// ``` + /// let mut iter = (1..20).peekable(); + /// // Consume all numbers less than 10 + /// while iter.next_if(|&x| x < 10).is_some() {} + /// // The next value returned will be 10 + /// assert_eq!(iter.next(), Some(10)); + /// ``` + #[stable(feature = "peekable_next_if", since = "1.51.0")] + pub fn next_if(&mut self, func: impl FnOnce(&I::Item) -> bool) -> Option { + match self.next() { + Some(matched) if func(&matched) => Some(matched), + other => { + // Since we called `self.next()`, we consumed `self.peeked`. + assert!(self.peeked.is_none()); + self.peeked = Some(other); + None + } + } + } + + /// Consume and return the next item if it is equal to `expected`. + /// + /// # Example + /// Consume a number if it's equal to 0. + /// ``` + /// let mut iter = (0..5).peekable(); + /// // The first item of the iterator is 0; consume it. + /// assert_eq!(iter.next_if_eq(&0), Some(0)); + /// // The next item returned is now 1, so `consume` will return `false`. + /// assert_eq!(iter.next_if_eq(&0), None); + /// // `next_if_eq` saves the value of the next item if it was not equal to `expected`. + /// assert_eq!(iter.next(), Some(1)); + /// ``` + #[stable(feature = "peekable_next_if", since = "1.51.0")] + pub fn next_if_eq(&mut self, expected: &T) -> Option + where + T: ?Sized, + I::Item: PartialEq, + { + self.next_if(|next| next == expected) + } +} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for Peekable where I: TrustedLen {} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl SourceIter for Peekable +where + I: SourceIter, +{ + type Source = I::Source; + + #[inline] + unsafe fn as_inner(&mut self) -> &mut I::Source { + // SAFETY: unsafe function forwarding to unsafe function with the same requirements + unsafe { SourceIter::as_inner(&mut self.iter) } + } +} diff --git a/crux-mir/lib/core/src/iter/adapters/rev.rs b/crux-mir/lib/core/src/iter/adapters/rev.rs new file mode 100644 index 000000000..139fb7bbd --- /dev/null +++ b/crux-mir/lib/core/src/iter/adapters/rev.rs @@ -0,0 +1,137 @@ +use crate::iter::{FusedIterator, TrustedLen}; +use crate::ops::Try; + +/// A double-ended iterator with the direction inverted. +/// +/// This `struct` is created by the [`rev`] method on [`Iterator`]. See its +/// documentation for more. +/// +/// [`rev`]: Iterator::rev +/// [`Iterator`]: trait.Iterator.html +#[derive(Clone, Debug)] +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "rust1", since = "1.0.0")] +pub struct Rev { + iter: T, +} + +impl Rev { + pub(in crate::iter) fn new(iter: T) -> Rev { + Rev { iter } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for Rev +where + I: DoubleEndedIterator, +{ + type Item = ::Item; + + #[inline] + fn next(&mut self) -> Option<::Item> { + self.iter.next_back() + } + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } + + #[inline] + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + self.iter.advance_back_by(n) + } + + #[inline] + fn nth(&mut self, n: usize) -> Option<::Item> { + self.iter.nth_back(n) + } + + fn try_fold(&mut self, init: B, f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: Try, + { + self.iter.try_rfold(init, f) + } + + fn fold(self, init: Acc, f: F) -> Acc + where + F: FnMut(Acc, Self::Item) -> Acc, + { + self.iter.rfold(init, f) + } + + #[inline] + fn find

(&mut self, predicate: P) -> Option + where + P: FnMut(&Self::Item) -> bool, + { + self.iter.rfind(predicate) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl DoubleEndedIterator for Rev +where + I: DoubleEndedIterator, +{ + #[inline] + fn next_back(&mut self) -> Option<::Item> { + self.iter.next() + } + + #[inline] + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + self.iter.advance_by(n) + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option<::Item> { + self.iter.nth(n) + } + + fn try_rfold(&mut self, init: B, f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: Try, + { + self.iter.try_fold(init, f) + } + + fn rfold(self, init: Acc, f: F) -> Acc + where + F: FnMut(Acc, Self::Item) -> Acc, + { + self.iter.fold(init, f) + } + + fn rfind

(&mut self, predicate: P) -> Option + where + P: FnMut(&Self::Item) -> bool, + { + self.iter.find(predicate) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ExactSizeIterator for Rev +where + I: ExactSizeIterator + DoubleEndedIterator, +{ + fn len(&self) -> usize { + self.iter.len() + } + + fn is_empty(&self) -> bool { + self.iter.is_empty() + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Rev where I: FusedIterator + DoubleEndedIterator {} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for Rev where I: TrustedLen + DoubleEndedIterator {} diff --git a/crux-mir/lib/core/src/iter/adapters/scan.rs b/crux-mir/lib/core/src/iter/adapters/scan.rs new file mode 100644 index 000000000..62470512c --- /dev/null +++ b/crux-mir/lib/core/src/iter/adapters/scan.rs @@ -0,0 +1,98 @@ +use crate::fmt; +use crate::iter::{adapters::SourceIter, InPlaceIterable}; +use crate::ops::{ControlFlow, Try}; + +/// An iterator to maintain state while iterating another iterator. +/// +/// This `struct` is created by the [`scan`] method on [`Iterator`]. See its +/// documentation for more. +/// +/// [`scan`]: Iterator::scan +/// [`Iterator`]: trait.Iterator.html +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "rust1", since = "1.0.0")] +#[derive(Clone)] +pub struct Scan { + iter: I, + f: F, + state: St, +} + +impl Scan { + pub(in crate::iter) fn new(iter: I, state: St, f: F) -> Scan { + Scan { iter, state, f } + } +} + +#[stable(feature = "core_impl_debug", since = "1.9.0")] +impl fmt::Debug for Scan { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Scan").field("iter", &self.iter).field("state", &self.state).finish() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for Scan +where + I: Iterator, + F: FnMut(&mut St, I::Item) -> Option, +{ + type Item = B; + + #[inline] + fn next(&mut self) -> Option { + let a = self.iter.next()?; + (self.f)(&mut self.state, a) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let (_, upper) = self.iter.size_hint(); + (0, upper) // can't know a lower bound, due to the scan function + } + + #[inline] + fn try_fold(&mut self, init: Acc, fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + { + fn scan<'a, T, St, B, Acc, R: Try>( + state: &'a mut St, + f: &'a mut impl FnMut(&mut St, T) -> Option, + mut fold: impl FnMut(Acc, B) -> R + 'a, + ) -> impl FnMut(Acc, T) -> ControlFlow + 'a { + move |acc, x| match f(state, x) { + None => ControlFlow::Break(try { acc }), + Some(x) => ControlFlow::from_try(fold(acc, x)), + } + } + + let state = &mut self.state; + let f = &mut self.f; + self.iter.try_fold(init, scan(state, f, fold)).into_try() + } + + impl_fold_via_try_fold! { fold -> try_fold } +} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl SourceIter for Scan +where + I: SourceIter, +{ + type Source = I::Source; + + #[inline] + unsafe fn as_inner(&mut self) -> &mut I::Source { + // SAFETY: unsafe function forwarding to unsafe function with the same requirements + unsafe { SourceIter::as_inner(&mut self.iter) } + } +} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for Scan where + F: FnMut(&mut St, I::Item) -> Option +{ +} diff --git a/crux-mir/lib/core/src/iter/adapters/skip.rs b/crux-mir/lib/core/src/iter/adapters/skip.rs new file mode 100644 index 000000000..c6334880d --- /dev/null +++ b/crux-mir/lib/core/src/iter/adapters/skip.rs @@ -0,0 +1,240 @@ +use crate::intrinsics::unlikely; +use crate::iter::{adapters::SourceIter, FusedIterator, InPlaceIterable}; +use crate::ops::{ControlFlow, Try}; + +/// An iterator that skips over `n` elements of `iter`. +/// +/// This `struct` is created by the [`skip`] method on [`Iterator`]. See its +/// documentation for more. +/// +/// [`skip`]: Iterator::skip +/// [`Iterator`]: trait.Iterator.html +#[derive(Clone, Debug)] +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "rust1", since = "1.0.0")] +pub struct Skip { + iter: I, + n: usize, +} + +impl Skip { + pub(in crate::iter) fn new(iter: I, n: usize) -> Skip { + Skip { iter, n } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for Skip +where + I: Iterator, +{ + type Item = ::Item; + + #[inline] + fn next(&mut self) -> Option { + if unlikely(self.n > 0) { + self.iter.nth(crate::mem::take(&mut self.n)) + } else { + self.iter.next() + } + } + + #[inline] + fn nth(&mut self, n: usize) -> Option { + if self.n > 0 { + let skip: usize = crate::mem::take(&mut self.n); + // Checked add to handle overflow case. + let n = match skip.checked_add(n) { + Some(nth) => nth, + None => { + // In case of overflow, load skip value, before loading `n`. + // Because the amount of elements to iterate is beyond `usize::MAX`, this + // is split into two `nth` calls where the `skip` `nth` call is discarded. + self.iter.nth(skip - 1)?; + n + } + }; + // Load nth element including skip. + self.iter.nth(n) + } else { + self.iter.nth(n) + } + } + + #[inline] + fn count(mut self) -> usize { + if self.n > 0 { + // nth(n) skips n+1 + if self.iter.nth(self.n - 1).is_none() { + return 0; + } + } + self.iter.count() + } + + #[inline] + fn last(mut self) -> Option { + if self.n > 0 { + // nth(n) skips n+1 + self.iter.nth(self.n - 1)?; + } + self.iter.last() + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let (lower, upper) = self.iter.size_hint(); + + let lower = lower.saturating_sub(self.n); + let upper = match upper { + Some(x) => Some(x.saturating_sub(self.n)), + None => None, + }; + + (lower, upper) + } + + #[inline] + fn try_fold(&mut self, init: Acc, fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + { + let n = self.n; + self.n = 0; + if n > 0 { + // nth(n) skips n+1 + if self.iter.nth(n - 1).is_none() { + return try { init }; + } + } + self.iter.try_fold(init, fold) + } + + #[inline] + fn fold(mut self, init: Acc, fold: Fold) -> Acc + where + Fold: FnMut(Acc, Self::Item) -> Acc, + { + if self.n > 0 { + // nth(n) skips n+1 + if self.iter.nth(self.n - 1).is_none() { + return init; + } + } + self.iter.fold(init, fold) + } + + #[inline] + #[rustc_inherit_overflow_checks] + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + let mut rem = n; + let step_one = self.n.saturating_add(rem); + + match self.iter.advance_by(step_one) { + Ok(_) => { + rem -= step_one - self.n; + self.n = 0; + } + Err(advanced) => { + let advanced_without_skip = advanced.saturating_sub(self.n); + self.n = self.n.saturating_sub(advanced); + return if n == 0 { Ok(()) } else { Err(advanced_without_skip) }; + } + } + + // step_one calculation may have saturated + if unlikely(rem > 0) { + return match self.iter.advance_by(rem) { + ret @ Ok(_) => ret, + Err(advanced) => { + rem -= advanced; + Err(n - rem) + } + }; + } + + Ok(()) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ExactSizeIterator for Skip where I: ExactSizeIterator {} + +#[stable(feature = "double_ended_skip_iterator", since = "1.9.0")] +impl DoubleEndedIterator for Skip +where + I: DoubleEndedIterator + ExactSizeIterator, +{ + fn next_back(&mut self) -> Option { + if self.len() > 0 { self.iter.next_back() } else { None } + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option { + let len = self.len(); + if n < len { + self.iter.nth_back(n) + } else { + if len > 0 { + // consume the original iterator + self.iter.nth_back(len - 1); + } + None + } + } + + fn try_rfold(&mut self, init: Acc, fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + { + fn check>( + mut n: usize, + mut fold: impl FnMut(Acc, T) -> R, + ) -> impl FnMut(Acc, T) -> ControlFlow { + move |acc, x| { + n -= 1; + let r = fold(acc, x); + if n == 0 { ControlFlow::Break(r) } else { ControlFlow::from_try(r) } + } + } + + let n = self.len(); + if n == 0 { try { init } } else { self.iter.try_rfold(init, check(n, fold)).into_try() } + } + + impl_fold_via_try_fold! { rfold -> try_rfold } + + #[inline] + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + let min = crate::cmp::min(self.len(), n); + return match self.iter.advance_back_by(min) { + ret @ Ok(_) if n <= min => ret, + Ok(_) => Err(min), + _ => panic!("ExactSizeIterator contract violation"), + }; + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Skip where I: FusedIterator {} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl SourceIter for Skip +where + I: SourceIter, +{ + type Source = I::Source; + + #[inline] + unsafe fn as_inner(&mut self) -> &mut I::Source { + // SAFETY: unsafe function forwarding to unsafe function with the same requirements + unsafe { SourceIter::as_inner(&mut self.iter) } + } +} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for Skip {} diff --git a/crux-mir/lib/core/src/iter/adapters/skip_while.rs b/crux-mir/lib/core/src/iter/adapters/skip_while.rs new file mode 100644 index 000000000..f29661779 --- /dev/null +++ b/crux-mir/lib/core/src/iter/adapters/skip_while.rs @@ -0,0 +1,125 @@ +use crate::fmt; +use crate::iter::{adapters::SourceIter, FusedIterator, InPlaceIterable}; +use crate::ops::Try; + +/// An iterator that rejects elements while `predicate` returns `true`. +/// +/// This `struct` is created by the [`skip_while`] method on [`Iterator`]. See its +/// documentation for more. +/// +/// [`skip_while`]: Iterator::skip_while +/// [`Iterator`]: trait.Iterator.html +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "rust1", since = "1.0.0")] +#[derive(Clone)] +pub struct SkipWhile { + iter: I, + flag: bool, + predicate: P, +} + +impl SkipWhile { + pub(in crate::iter) fn new(iter: I, predicate: P) -> SkipWhile { + SkipWhile { iter, flag: false, predicate } + } +} + +#[stable(feature = "core_impl_debug", since = "1.9.0")] +impl fmt::Debug for SkipWhile { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SkipWhile").field("iter", &self.iter).field("flag", &self.flag).finish() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for SkipWhile +where + P: FnMut(&I::Item) -> bool, +{ + type Item = I::Item; + + #[inline] + fn next(&mut self) -> Option { + fn check<'a, T>( + flag: &'a mut bool, + pred: &'a mut impl FnMut(&T) -> bool, + ) -> impl FnMut(&T) -> bool + 'a { + move |x| { + if *flag || !pred(x) { + *flag = true; + true + } else { + false + } + } + } + + let flag = &mut self.flag; + let pred = &mut self.predicate; + self.iter.find(check(flag, pred)) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let (_, upper) = self.iter.size_hint(); + (0, upper) // can't know a lower bound, due to the predicate + } + + #[inline] + fn try_fold(&mut self, mut init: Acc, mut fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + { + if !self.flag { + match self.next() { + Some(v) => init = fold(init, v)?, + None => return try { init }, + } + } + self.iter.try_fold(init, fold) + } + + #[inline] + fn fold(mut self, mut init: Acc, mut fold: Fold) -> Acc + where + Fold: FnMut(Acc, Self::Item) -> Acc, + { + if !self.flag { + match self.next() { + Some(v) => init = fold(init, v), + None => return init, + } + } + self.iter.fold(init, fold) + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for SkipWhile +where + I: FusedIterator, + P: FnMut(&I::Item) -> bool, +{ +} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl SourceIter for SkipWhile +where + I: SourceIter, +{ + type Source = I::Source; + + #[inline] + unsafe fn as_inner(&mut self) -> &mut I::Source { + // SAFETY: unsafe function forwarding to unsafe function with the same requirements + unsafe { SourceIter::as_inner(&mut self.iter) } + } +} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for SkipWhile where + F: FnMut(&I::Item) -> bool +{ +} diff --git a/crux-mir/lib/core/src/iter/adapters/step_by.rs b/crux-mir/lib/core/src/iter/adapters/step_by.rs new file mode 100644 index 000000000..4252c34a0 --- /dev/null +++ b/crux-mir/lib/core/src/iter/adapters/step_by.rs @@ -0,0 +1,235 @@ +use crate::{intrinsics, iter::from_fn, ops::Try}; + +/// An iterator for stepping iterators by a custom amount. +/// +/// This `struct` is created by the [`step_by`] method on [`Iterator`]. See +/// its documentation for more. +/// +/// [`step_by`]: Iterator::step_by +/// [`Iterator`]: trait.Iterator.html +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "iterator_step_by", since = "1.28.0")] +#[derive(Clone, Debug)] +pub struct StepBy { + iter: I, + step: usize, + first_take: bool, +} + +impl StepBy { + pub(in crate::iter) fn new(iter: I, step: usize) -> StepBy { + assert!(step != 0); + StepBy { iter, step: step - 1, first_take: true } + } +} + +#[stable(feature = "iterator_step_by", since = "1.28.0")] +impl Iterator for StepBy +where + I: Iterator, +{ + type Item = I::Item; + + #[inline] + fn next(&mut self) -> Option { + if self.first_take { + self.first_take = false; + self.iter.next() + } else { + self.iter.nth(self.step) + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + #[inline] + fn first_size(step: usize) -> impl Fn(usize) -> usize { + move |n| if n == 0 { 0 } else { 1 + (n - 1) / (step + 1) } + } + + #[inline] + fn other_size(step: usize) -> impl Fn(usize) -> usize { + move |n| n / (step + 1) + } + + let (low, high) = self.iter.size_hint(); + + if self.first_take { + let f = first_size(self.step); + (f(low), high.map(f)) + } else { + let f = other_size(self.step); + (f(low), high.map(f)) + } + } + + #[inline] + fn nth(&mut self, mut n: usize) -> Option { + if self.first_take { + self.first_take = false; + let first = self.iter.next(); + if n == 0 { + return first; + } + n -= 1; + } + // n and self.step are indices, we need to add 1 to get the amount of elements + // When calling `.nth`, we need to subtract 1 again to convert back to an index + // step + 1 can't overflow because `.step_by` sets `self.step` to `step - 1` + let mut step = self.step + 1; + // n + 1 could overflow + // thus, if n is usize::MAX, instead of adding one, we call .nth(step) + if n == usize::MAX { + self.iter.nth(step - 1); + } else { + n += 1; + } + + // overflow handling + loop { + let mul = n.checked_mul(step); + { + if intrinsics::likely(mul.is_some()) { + return self.iter.nth(mul.unwrap() - 1); + } + } + let div_n = usize::MAX / n; + let div_step = usize::MAX / step; + let nth_n = div_n * n; + let nth_step = div_step * step; + let nth = if nth_n > nth_step { + step -= div_n; + nth_n + } else { + n -= div_step; + nth_step + }; + self.iter.nth(nth - 1); + } + } + + fn try_fold(&mut self, mut acc: Acc, mut f: F) -> R + where + F: FnMut(Acc, Self::Item) -> R, + R: Try, + { + #[inline] + fn nth(iter: &mut I, step: usize) -> impl FnMut() -> Option + '_ { + move || iter.nth(step) + } + + if self.first_take { + self.first_take = false; + match self.iter.next() { + None => return try { acc }, + Some(x) => acc = f(acc, x)?, + } + } + from_fn(nth(&mut self.iter, self.step)).try_fold(acc, f) + } + + fn fold(mut self, mut acc: Acc, mut f: F) -> Acc + where + F: FnMut(Acc, Self::Item) -> Acc, + { + #[inline] + fn nth(iter: &mut I, step: usize) -> impl FnMut() -> Option + '_ { + move || iter.nth(step) + } + + if self.first_take { + self.first_take = false; + match self.iter.next() { + None => return acc, + Some(x) => acc = f(acc, x), + } + } + from_fn(nth(&mut self.iter, self.step)).fold(acc, f) + } +} + +impl StepBy +where + I: ExactSizeIterator, +{ + // The zero-based index starting from the end of the iterator of the + // last element. Used in the `DoubleEndedIterator` implementation. + fn next_back_index(&self) -> usize { + let rem = self.iter.len() % (self.step + 1); + if self.first_take { + if rem == 0 { self.step } else { rem - 1 } + } else { + rem + } + } +} + +#[stable(feature = "double_ended_step_by_iterator", since = "1.38.0")] +impl DoubleEndedIterator for StepBy +where + I: DoubleEndedIterator + ExactSizeIterator, +{ + #[inline] + fn next_back(&mut self) -> Option { + self.iter.nth_back(self.next_back_index()) + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option { + // `self.iter.nth_back(usize::MAX)` does the right thing here when `n` + // is out of bounds because the length of `self.iter` does not exceed + // `usize::MAX` (because `I: ExactSizeIterator`) and `nth_back` is + // zero-indexed + let n = n.saturating_mul(self.step + 1).saturating_add(self.next_back_index()); + self.iter.nth_back(n) + } + + fn try_rfold(&mut self, init: Acc, mut f: F) -> R + where + F: FnMut(Acc, Self::Item) -> R, + R: Try, + { + #[inline] + fn nth_back( + iter: &mut I, + step: usize, + ) -> impl FnMut() -> Option + '_ { + move || iter.nth_back(step) + } + + match self.next_back() { + None => try { init }, + Some(x) => { + let acc = f(init, x)?; + from_fn(nth_back(&mut self.iter, self.step)).try_fold(acc, f) + } + } + } + + #[inline] + fn rfold(mut self, init: Acc, mut f: F) -> Acc + where + Self: Sized, + F: FnMut(Acc, Self::Item) -> Acc, + { + #[inline] + fn nth_back( + iter: &mut I, + step: usize, + ) -> impl FnMut() -> Option + '_ { + move || iter.nth_back(step) + } + + match self.next_back() { + None => init, + Some(x) => { + let acc = f(init, x); + from_fn(nth_back(&mut self.iter, self.step)).fold(acc, f) + } + } + } +} + +// StepBy can only make the iterator shorter, so the len will still fit. +#[stable(feature = "iterator_step_by", since = "1.28.0")] +impl ExactSizeIterator for StepBy where I: ExactSizeIterator {} diff --git a/crux-mir/lib/core/src/iter/adapters/take.rs b/crux-mir/lib/core/src/iter/adapters/take.rs new file mode 100644 index 000000000..d947c7b0e --- /dev/null +++ b/crux-mir/lib/core/src/iter/adapters/take.rs @@ -0,0 +1,251 @@ +use crate::cmp; +use crate::iter::{adapters::SourceIter, FusedIterator, InPlaceIterable, TrustedLen}; +use crate::ops::{ControlFlow, Try}; + +/// An iterator that only iterates over the first `n` iterations of `iter`. +/// +/// This `struct` is created by the [`take`] method on [`Iterator`]. See its +/// documentation for more. +/// +/// [`take`]: Iterator::take +/// [`Iterator`]: trait.Iterator.html +#[derive(Clone, Debug)] +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "rust1", since = "1.0.0")] +pub struct Take { + iter: I, + n: usize, +} + +impl Take { + pub(in crate::iter) fn new(iter: I, n: usize) -> Take { + Take { iter, n } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for Take +where + I: Iterator, +{ + type Item = ::Item; + + #[inline] + fn next(&mut self) -> Option<::Item> { + if self.n != 0 { + self.n -= 1; + self.iter.next() + } else { + None + } + } + + #[inline] + fn nth(&mut self, n: usize) -> Option { + if self.n > n { + self.n -= n + 1; + self.iter.nth(n) + } else { + if self.n > 0 { + self.iter.nth(self.n - 1); + self.n = 0; + } + None + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + if self.n == 0 { + return (0, Some(0)); + } + + let (lower, upper) = self.iter.size_hint(); + + let lower = cmp::min(lower, self.n); + + let upper = match upper { + Some(x) if x < self.n => Some(x), + _ => Some(self.n), + }; + + (lower, upper) + } + + #[inline] + fn try_fold(&mut self, init: Acc, fold: Fold) -> R + where + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + { + fn check<'a, T, Acc, R: Try>( + n: &'a mut usize, + mut fold: impl FnMut(Acc, T) -> R + 'a, + ) -> impl FnMut(Acc, T) -> ControlFlow + 'a { + move |acc, x| { + *n -= 1; + let r = fold(acc, x); + if *n == 0 { ControlFlow::Break(r) } else { ControlFlow::from_try(r) } + } + } + + if self.n == 0 { + try { init } + } else { + let n = &mut self.n; + self.iter.try_fold(init, check(n, fold)).into_try() + } + } + + impl_fold_via_try_fold! { fold -> try_fold } + + #[inline] + fn for_each(mut self, f: F) { + // The default implementation would use a unit accumulator, so we can + // avoid a stateful closure by folding over the remaining number + // of items we wish to return instead. + fn check<'a, Item>( + mut action: impl FnMut(Item) + 'a, + ) -> impl FnMut(usize, Item) -> Option + 'a { + move |more, x| { + action(x); + more.checked_sub(1) + } + } + + let remaining = self.n; + if remaining > 0 { + self.iter.try_fold(remaining - 1, check(f)); + } + } + + #[inline] + #[rustc_inherit_overflow_checks] + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + let min = self.n.min(n); + match self.iter.advance_by(min) { + Ok(_) => { + self.n -= min; + if min < n { Err(min) } else { Ok(()) } + } + ret @ Err(advanced) => { + self.n -= advanced; + ret + } + } + } +} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl SourceIter for Take +where + I: SourceIter, +{ + type Source = I::Source; + + #[inline] + unsafe fn as_inner(&mut self) -> &mut I::Source { + // SAFETY: unsafe function forwarding to unsafe function with the same requirements + unsafe { SourceIter::as_inner(&mut self.iter) } + } +} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for Take {} + +#[stable(feature = "double_ended_take_iterator", since = "1.38.0")] +impl DoubleEndedIterator for Take +where + I: DoubleEndedIterator + ExactSizeIterator, +{ + #[inline] + fn next_back(&mut self) -> Option { + if self.n == 0 { + None + } else { + let n = self.n; + self.n -= 1; + self.iter.nth_back(self.iter.len().saturating_sub(n)) + } + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option { + let len = self.iter.len(); + if self.n > n { + let m = len.saturating_sub(self.n) + n; + self.n -= n + 1; + self.iter.nth_back(m) + } else { + if len > 0 { + self.iter.nth_back(len - 1); + } + None + } + } + + #[inline] + fn try_rfold(&mut self, init: Acc, fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + { + if self.n == 0 { + try { init } + } else { + let len = self.iter.len(); + if len > self.n && self.iter.nth_back(len - self.n - 1).is_none() { + try { init } + } else { + self.iter.try_rfold(init, fold) + } + } + } + + #[inline] + fn rfold(mut self, init: Acc, fold: Fold) -> Acc + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> Acc, + { + if self.n == 0 { + init + } else { + let len = self.iter.len(); + if len > self.n && self.iter.nth_back(len - self.n - 1).is_none() { + init + } else { + self.iter.rfold(init, fold) + } + } + } + + #[inline] + #[rustc_inherit_overflow_checks] + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + // The amount by which the inner iterator needs to be shortened for it to be + // at most as long as the take() amount. + let trim_inner = self.iter.len().saturating_sub(self.n); + // The amount we need to advance inner to fulfill the caller's request. + // take(), advance_by() and len() all can be at most usize, so we don't have to worry + // about having to advance more than usize::MAX here. + let advance_by = trim_inner.saturating_add(n); + + let advanced = match self.iter.advance_back_by(advance_by) { + Ok(_) => advance_by - trim_inner, + Err(advanced) => advanced - trim_inner, + }; + self.n -= advanced; + return if advanced < n { Err(advanced) } else { Ok(()) }; + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ExactSizeIterator for Take where I: ExactSizeIterator {} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Take where I: FusedIterator {} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for Take {} diff --git a/crux-mir/lib/core/src/iter/adapters/take_while.rs b/crux-mir/lib/core/src/iter/adapters/take_while.rs new file mode 100644 index 000000000..ec66dc3ae --- /dev/null +++ b/crux-mir/lib/core/src/iter/adapters/take_while.rs @@ -0,0 +1,126 @@ +use crate::fmt; +use crate::iter::{adapters::SourceIter, FusedIterator, InPlaceIterable}; +use crate::ops::{ControlFlow, Try}; + +/// An iterator that only accepts elements while `predicate` returns `true`. +/// +/// This `struct` is created by the [`take_while`] method on [`Iterator`]. See its +/// documentation for more. +/// +/// [`take_while`]: Iterator::take_while +/// [`Iterator`]: trait.Iterator.html +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "rust1", since = "1.0.0")] +#[derive(Clone)] +pub struct TakeWhile { + iter: I, + flag: bool, + predicate: P, +} + +impl TakeWhile { + pub(in crate::iter) fn new(iter: I, predicate: P) -> TakeWhile { + TakeWhile { iter, flag: false, predicate } + } +} + +#[stable(feature = "core_impl_debug", since = "1.9.0")] +impl fmt::Debug for TakeWhile { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TakeWhile").field("iter", &self.iter).field("flag", &self.flag).finish() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for TakeWhile +where + P: FnMut(&I::Item) -> bool, +{ + type Item = I::Item; + + #[inline] + fn next(&mut self) -> Option { + if self.flag { + None + } else { + let x = self.iter.next()?; + if (self.predicate)(&x) { + Some(x) + } else { + self.flag = true; + None + } + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + if self.flag { + (0, Some(0)) + } else { + let (_, upper) = self.iter.size_hint(); + (0, upper) // can't know a lower bound, due to the predicate + } + } + + #[inline] + fn try_fold(&mut self, init: Acc, fold: Fold) -> R + where + Self: Sized, + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + { + fn check<'a, T, Acc, R: Try>( + flag: &'a mut bool, + p: &'a mut impl FnMut(&T) -> bool, + mut fold: impl FnMut(Acc, T) -> R + 'a, + ) -> impl FnMut(Acc, T) -> ControlFlow + 'a { + move |acc, x| { + if p(&x) { + ControlFlow::from_try(fold(acc, x)) + } else { + *flag = true; + ControlFlow::Break(try { acc }) + } + } + } + + if self.flag { + try { init } + } else { + let flag = &mut self.flag; + let p = &mut self.predicate; + self.iter.try_fold(init, check(flag, p, fold)).into_try() + } + } + + impl_fold_via_try_fold! { fold -> try_fold } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for TakeWhile +where + I: FusedIterator, + P: FnMut(&I::Item) -> bool, +{ +} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl SourceIter for TakeWhile +where + I: SourceIter, +{ + type Source = I::Source; + + #[inline] + unsafe fn as_inner(&mut self) -> &mut I::Source { + // SAFETY: unsafe function forwarding to unsafe function with the same requirements + unsafe { SourceIter::as_inner(&mut self.iter) } + } +} + +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for TakeWhile where + F: FnMut(&I::Item) -> bool +{ +} diff --git a/crux-mir/lib/core/src/iter/adapters/zip.rs b/crux-mir/lib/core/src/iter/adapters/zip.rs index b13e12e2e..8153c8cfe 100644 --- a/crux-mir/lib/core/src/iter/adapters/zip.rs +++ b/crux-mir/lib/core/src/iter/adapters/zip.rs @@ -1,28 +1,25 @@ -// ignore-tidy-undocumented-unsafe - use crate::cmp; - -use super::super::{DoubleEndedIterator, ExactSizeIterator, FusedIterator, Iterator, TrustedLen}; +use crate::fmt::{self, Debug}; +use crate::iter::{DoubleEndedIterator, ExactSizeIterator, FusedIterator, Iterator}; +use crate::iter::{InPlaceIterable, SourceIter, TrustedLen}; /// An iterator that iterates two other iterators simultaneously. /// -/// This `struct` is created by the [`zip`] method on [`Iterator`]. See its -/// documentation for more. -/// -/// [`zip`]: trait.Iterator.html#method.zip -/// [`Iterator`]: trait.Iterator.html -#[derive(Clone, Debug)] +/// This `struct` is created by [`zip`] or [`Iterator::zip`]. +/// See their documentation for more. +#[derive(Clone)] #[must_use = "iterators are lazy and do nothing unless consumed"] #[stable(feature = "rust1", since = "1.0.0")] pub struct Zip { a: A, b: B, - // index and len are only used by the specialized version of zip + // index, len and a_len are only used by the specialized version of zip index: usize, len: usize, + a_len: usize, } impl Zip { - pub(in super::super) fn new(a: A, b: B) -> Zip { + pub(in crate::iter) fn new(a: A, b: B) -> Zip { ZipImpl::new(a, b) } fn super_nth(&mut self, mut n: usize) -> Option<(A::Item, B::Item)> { @@ -36,6 +33,44 @@ impl Zip { } } +/// Converts the arguments to iterators and zips them. +/// +/// See the documentation of [`Iterator::zip`] for more. +/// +/// # Examples +/// +/// ``` +/// use std::iter::zip; +/// +/// let xs = [1, 2, 3]; +/// let ys = [4, 5, 6]; +/// +/// let mut iter = zip(xs, ys); +/// +/// assert_eq!(iter.next().unwrap(), (1, 4)); +/// assert_eq!(iter.next().unwrap(), (2, 5)); +/// assert_eq!(iter.next().unwrap(), (3, 6)); +/// assert!(iter.next().is_none()); +/// +/// // Nested zips are also possible: +/// let zs = [7, 8, 9]; +/// +/// let mut iter = zip(zip(xs, ys), zs); +/// +/// assert_eq!(iter.next().unwrap(), ((1, 4), 7)); +/// assert_eq!(iter.next().unwrap(), ((2, 5), 8)); +/// assert_eq!(iter.next().unwrap(), ((3, 6), 9)); +/// assert!(iter.next().is_none()); +/// ``` +#[stable(feature = "iter_zip", since = "1.59.0")] +pub fn zip(a: A, b: B) -> Zip +where + A: IntoIterator, + B: IntoIterator, +{ + ZipImpl::new(a.into_iter(), b.into_iter()) +} + #[stable(feature = "rust1", since = "1.0.0")] impl Iterator for Zip where @@ -58,6 +93,16 @@ where fn nth(&mut self, n: usize) -> Option { ZipImpl::nth(self, n) } + + #[inline] + unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> Self::Item + where + Self: TrustedRandomAccessNoCoerce, + { + // SAFETY: `ZipImpl::__iterator_get_unchecked` has same safety + // requirements as `Iterator::__iterator_get_unchecked`. + unsafe { ZipImpl::get_unchecked(self, idx) } + } } #[stable(feature = "rust1", since = "1.0.0")] @@ -84,6 +129,69 @@ trait ZipImpl { where A: DoubleEndedIterator + ExactSizeIterator, B: DoubleEndedIterator + ExactSizeIterator; + // This has the same safety requirements as `Iterator::__iterator_get_unchecked` + unsafe fn get_unchecked(&mut self, idx: usize) -> ::Item + where + Self: Iterator + TrustedRandomAccessNoCoerce; +} + +// Work around limitations of specialization, requiring `default` impls to be repeated +// in intermediary impls. +macro_rules! zip_impl_general_defaults { + () => { + default fn new(a: A, b: B) -> Self { + Zip { + a, + b, + index: 0, // unused + len: 0, // unused + a_len: 0, // unused + } + } + + #[inline] + default fn next(&mut self) -> Option<(A::Item, B::Item)> { + let x = self.a.next()?; + let y = self.b.next()?; + Some((x, y)) + } + + #[inline] + default fn nth(&mut self, n: usize) -> Option { + self.super_nth(n) + } + + #[inline] + default fn next_back(&mut self) -> Option<(A::Item, B::Item)> + where + A: DoubleEndedIterator + ExactSizeIterator, + B: DoubleEndedIterator + ExactSizeIterator, + { + // The function body below only uses `self.a/b.len()` and `self.a/b.next_back()` + // and doesn’t call `next_back` too often, so this implementation is safe in + // the `TrustedRandomAccessNoCoerce` specialization + + let a_sz = self.a.len(); + let b_sz = self.b.len(); + if a_sz != b_sz { + // Adjust a, b to equal length + if a_sz > b_sz { + for _ in 0..a_sz - b_sz { + self.a.next_back(); + } + } else { + for _ in 0..b_sz - a_sz { + self.b.next_back(); + } + } + } + match (self.a.next_back(), self.b.next_back()) { + (Some(x), Some(y)) => Some((x, y)), + (None, None) => None, + _ => unreachable!(), + } + } + }; } // General Zip impl @@ -94,53 +202,8 @@ where B: Iterator, { type Item = (A::Item, B::Item); - default fn new(a: A, b: B) -> Self { - Zip { - a, - b, - index: 0, // unused - len: 0, // unused - } - } - #[inline] - default fn next(&mut self) -> Option<(A::Item, B::Item)> { - let x = self.a.next()?; - let y = self.b.next()?; - Some((x, y)) - } - - #[inline] - default fn nth(&mut self, n: usize) -> Option { - self.super_nth(n) - } - - #[inline] - default fn next_back(&mut self) -> Option<(A::Item, B::Item)> - where - A: DoubleEndedIterator + ExactSizeIterator, - B: DoubleEndedIterator + ExactSizeIterator, - { - let a_sz = self.a.len(); - let b_sz = self.b.len(); - if a_sz != b_sz { - // Adjust a, b to equal length - if a_sz > b_sz { - for _ in 0..a_sz - b_sz { - self.a.next_back(); - } - } else { - for _ in 0..b_sz - a_sz { - self.b.next_back(); - } - } - } - match (self.a.next_back(), self.b.next_back()) { - (Some(x), Some(y)) => Some((x, y)), - (None, None) => None, - _ => unreachable!(), - } - } + zip_impl_general_defaults! {} #[inline] default fn size_hint(&self) -> (usize, Option) { @@ -158,31 +221,71 @@ where (lower, upper) } + + default unsafe fn get_unchecked(&mut self, _idx: usize) -> ::Item + where + Self: TrustedRandomAccessNoCoerce, + { + unreachable!("Always specialized"); + } } #[doc(hidden)] impl ZipImpl for Zip where - A: TrustedRandomAccess, - B: TrustedRandomAccess, + A: TrustedRandomAccessNoCoerce + Iterator, + B: TrustedRandomAccessNoCoerce + Iterator, +{ + zip_impl_general_defaults! {} + + #[inline] + default fn size_hint(&self) -> (usize, Option) { + let size = cmp::min(self.a.size(), self.b.size()); + (size, Some(size)) + } + + #[inline] + unsafe fn get_unchecked(&mut self, idx: usize) -> ::Item { + let idx = self.index + idx; + // SAFETY: the caller must uphold the contract for + // `Iterator::__iterator_get_unchecked`. + unsafe { (self.a.__iterator_get_unchecked(idx), self.b.__iterator_get_unchecked(idx)) } + } +} + +#[doc(hidden)] +impl ZipImpl for Zip +where + A: TrustedRandomAccess + Iterator, + B: TrustedRandomAccess + Iterator, { fn new(a: A, b: B) -> Self { - let len = cmp::min(a.len(), b.len()); - Zip { a, b, index: 0, len } + let a_len = a.size(); + let len = cmp::min(a_len, b.size()); + Zip { a, b, index: 0, len, a_len } } #[inline] fn next(&mut self) -> Option<(A::Item, B::Item)> { if self.index < self.len { let i = self.index; + // since get_unchecked executes code which can panic we increment the counters beforehand + // so that the same index won't be accessed twice, as required by TrustedRandomAccess self.index += 1; - unsafe { Some((self.a.get_unchecked(i), self.b.get_unchecked(i))) } - } else if A::may_have_side_effect() && self.index < self.a.len() { - // match the base implementation's potential side effects + // SAFETY: `i` is smaller than `self.len`, thus smaller than `self.a.len()` and `self.b.len()` unsafe { - self.a.get_unchecked(self.index); + Some((self.a.__iterator_get_unchecked(i), self.b.__iterator_get_unchecked(i))) } + } else if A::MAY_HAVE_SIDE_EFFECT && self.index < self.a_len { + let i = self.index; + // as above, increment before executing code that may panic self.index += 1; + self.len += 1; + // match the base implementation's potential side effects + // SAFETY: we just checked that `i` < `self.a.len()` + unsafe { + self.a.__iterator_get_unchecked(i); + } None } else { None @@ -201,15 +304,21 @@ where let end = self.index + delta; while self.index < end { let i = self.index; + // since get_unchecked executes code which can panic we increment the counters beforehand + // so that the same index won't be accessed twice, as required by TrustedRandomAccess self.index += 1; - if A::may_have_side_effect() { + if A::MAY_HAVE_SIDE_EFFECT { + // SAFETY: the usage of `cmp::min` to calculate `delta` + // ensures that `end` is smaller than or equal to `self.len`, + // so `i` is also smaller than `self.len`. unsafe { - self.a.get_unchecked(i); + self.a.__iterator_get_unchecked(i); } } - if B::may_have_side_effect() { + if B::MAY_HAVE_SIDE_EFFECT { + // SAFETY: same as above. unsafe { - self.b.get_unchecked(i); + self.b.__iterator_get_unchecked(i); } } } @@ -223,27 +332,42 @@ where A: DoubleEndedIterator + ExactSizeIterator, B: DoubleEndedIterator + ExactSizeIterator, { - // Adjust a, b to equal length - if A::may_have_side_effect() { - let sz = self.a.len(); - if sz > self.len { - for _ in 0..sz - cmp::max(self.len, self.index) { - self.a.next_back(); + if A::MAY_HAVE_SIDE_EFFECT || B::MAY_HAVE_SIDE_EFFECT { + let sz_a = self.a.size(); + let sz_b = self.b.size(); + // Adjust a, b to equal length, make sure that only the first call + // of `next_back` does this, otherwise we will break the restriction + // on calls to `self.next_back()` after calling `get_unchecked()`. + if sz_a != sz_b { + let sz_a = self.a.size(); + if A::MAY_HAVE_SIDE_EFFECT && sz_a > self.len { + for _ in 0..sz_a - self.len { + // since next_back() may panic we increment the counters beforehand + // to keep Zip's state in sync with the underlying iterator source + self.a_len -= 1; + self.a.next_back(); + } + debug_assert_eq!(self.a_len, self.len); } - } - } - if B::may_have_side_effect() { - let sz = self.b.len(); - if sz > self.len { - for _ in 0..sz - self.len { - self.b.next_back(); + let sz_b = self.b.size(); + if B::MAY_HAVE_SIDE_EFFECT && sz_b > self.len { + for _ in 0..sz_b - self.len { + self.b.next_back(); + } } } } if self.index < self.len { + // since get_unchecked executes code which can panic we increment the counters beforehand + // so that the same index won't be accessed twice, as required by TrustedRandomAccess self.len -= 1; + self.a_len -= 1; let i = self.len; - unsafe { Some((self.a.get_unchecked(i), self.b.get_unchecked(i))) } + // SAFETY: `i` is smaller than the previous value of `self.len`, + // which is also smaller than or equal to `self.a.len()` and `self.b.len()` + unsafe { + Some((self.a.__iterator_get_unchecked(i), self.b.__iterator_get_unchecked(i))) + } } else { None } @@ -259,18 +383,22 @@ where } #[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] unsafe impl TrustedRandomAccess for Zip where A: TrustedRandomAccess, B: TrustedRandomAccess, { - unsafe fn get_unchecked(&mut self, i: usize) -> (A::Item, B::Item) { - (self.a.get_unchecked(i), self.b.get_unchecked(i)) - } +} - fn may_have_side_effect() -> bool { - A::may_have_side_effect() || B::may_have_side_effect() - } +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl TrustedRandomAccessNoCoerce for Zip +where + A: TrustedRandomAccessNoCoerce, + B: TrustedRandomAccessNoCoerce, +{ + const MAY_HAVE_SIDE_EFFECT: bool = A::MAY_HAVE_SIDE_EFFECT || B::MAY_HAVE_SIDE_EFFECT; } #[stable(feature = "fused", since = "1.26.0")] @@ -289,19 +417,169 @@ where { } +// Arbitrarily selects the left side of the zip iteration as extractable "source" +// it would require negative trait bounds to be able to try both +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl SourceIter for Zip +where + A: SourceIter, +{ + type Source = A::Source; + + #[inline] + unsafe fn as_inner(&mut self) -> &mut A::Source { + // SAFETY: unsafe function forwarding to unsafe function with the same requirements + unsafe { SourceIter::as_inner(&mut self.a) } + } +} + +// Since SourceIter forwards the left hand side we do the same here +#[unstable(issue = "none", feature = "inplace_iteration")] +unsafe impl InPlaceIterable for Zip {} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Debug for Zip { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + ZipFmt::fmt(self, f) + } +} + +trait ZipFmt { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result; +} + +impl ZipFmt for Zip { + default fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Zip").field("a", &self.a).field("b", &self.b).finish() + } +} + +impl ZipFmt + for Zip +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // It's *not safe* to call fmt on the contained iterators, since once + // we start iterating they're in strange, potentially unsafe, states. + f.debug_struct("Zip").finish() + } +} + /// An iterator whose items are random-accessible efficiently /// /// # Safety /// -/// The iterator's .len() and size_hint() must be exact. -/// `.len()` must be cheap to call. +/// The iterator's `size_hint` must be exact and cheap to call. +/// +/// `TrustedRandomAccessNoCoerce::size` may not be overridden. +/// +/// All subtypes and all supertypes of `Self` must also implement `TrustedRandomAccess`. +/// In particular, this means that types with non-invariant parameters usually can not have +/// an impl for `TrustedRandomAccess` that depends on any trait bounds on such parameters, except +/// for bounds that come from the respective struct/enum definition itself, or bounds involving +/// traits that themselves come with a guarantee similar to this one. +/// +/// If `Self: ExactSizeIterator` then `self.len()` must always produce results consistent +/// with `self.size()`. /// -/// .get_unchecked() must return distinct mutable references for distinct -/// indices (if applicable), and must return a valid reference if index is in -/// 0..self.len(). -pub(crate) unsafe trait TrustedRandomAccess: ExactSizeIterator { - unsafe fn get_unchecked(&mut self, i: usize) -> Self::Item; - /// Returns `true` if getting an iterator element may have - /// side effects. Remember to take inner iterators into account. - fn may_have_side_effect() -> bool; +/// If `Self: Iterator`, then `::__iterator_get_unchecked(&mut self, idx)` +/// must be safe to call provided the following conditions are met. +/// +/// 1. `0 <= idx` and `idx < self.size()`. +/// 2. If `Self: !Clone`, then `self.__iterator_get_unchecked(idx)` is never called with the same +/// index on `self` more than once. +/// 3. After `self.__iterator_get_unchecked(idx)` has been called, then `self.next_back()` will +/// only be called at most `self.size() - idx - 1` times. If `Self: Clone` and `self` is cloned, +/// then this number is calculated for `self` and its clone individually, +/// but `self.next_back()` calls that happened before the cloning count for both `self` and the clone. +/// 4. After `self.__iterator_get_unchecked(idx)` has been called, then only the following methods +/// will be called on `self` or on any new clones of `self`: +/// * `std::clone::Clone::clone` +/// * `std::iter::Iterator::size_hint` +/// * `std::iter::DoubleEndedIterator::next_back` +/// * `std::iter::ExactSizeIterator::len` +/// * `std::iter::Iterator::__iterator_get_unchecked` +/// * `std::iter::TrustedRandomAccessNoCoerce::size` +/// 5. If `T` is a subtype of `Self`, then `self` is allowed to be coerced +/// to `T`. If `self` is coerced to `T` after `self.__iterator_get_unchecked(idx)` has already +/// been called, then no methods except for the ones listed under 4. are allowed to be called +/// on the resulting value of type `T`, either. Multiple such coercion steps are allowed. +/// Regarding 2. and 3., the number of times `__iterator_get_unchecked(idx)` or `next_back()` is +/// called on `self` and the resulting value of type `T` (and on further coercion results with +/// sub-subtypes) are added together and their sums must not exceed the specified bounds. +/// +/// Further, given that these conditions are met, it must guarantee that: +/// +/// * It does not change the value returned from `size_hint` +/// * It must be safe to call the methods listed above on `self` after calling +/// `self.__iterator_get_unchecked(idx)`, assuming that the required traits are implemented. +/// * It must also be safe to drop `self` after calling `self.__iterator_get_unchecked(idx)`. +/// * If `T` is a subtype of `Self`, then it must be safe to coerce `self` to `T`. +// +// FIXME: Clarify interaction with SourceIter/InPlaceIterable. Calling `SourceIter::as_inner` +// after `__iterator_get_unchecked` is supposed to be allowed. +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +#[rustc_specialization_trait] +pub unsafe trait TrustedRandomAccess: TrustedRandomAccessNoCoerce {} + +/// Like [`TrustedRandomAccess`] but without any of the requirements / guarantees around +/// coercions to subtypes after `__iterator_get_unchecked` (they aren’t allowed here!), and +/// without the requirement that subtypes / supertypes implement `TrustedRandomAccessNoCoerce`. +/// +/// This trait was created in PR #85874 to fix soundness issue #85873 without performance regressions. +/// It is subject to change as we might want to build a more generally useful (for performance +/// optimizations) and more sophisticated trait or trait hierarchy that replaces or extends +/// [`TrustedRandomAccess`] and `TrustedRandomAccessNoCoerce`. +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +#[rustc_specialization_trait] +pub unsafe trait TrustedRandomAccessNoCoerce: Sized { + // Convenience method. + fn size(&self) -> usize + where + Self: Iterator, + { + self.size_hint().0 + } + /// `true` if getting an iterator element may have side effects. + /// Remember to take inner iterators into account. + const MAY_HAVE_SIDE_EFFECT: bool; +} + +/// Like `Iterator::__iterator_get_unchecked`, but doesn't require the compiler to +/// know that `U: TrustedRandomAccess`. +/// +/// ## Safety +/// +/// Same requirements calling `get_unchecked` directly. +#[doc(hidden)] +#[inline] +pub(in crate::iter::adapters) unsafe fn try_get_unchecked(it: &mut I, idx: usize) -> I::Item +where + I: Iterator, +{ + // SAFETY: the caller must uphold the contract for + // `Iterator::__iterator_get_unchecked`. + unsafe { it.try_get_unchecked(idx) } +} + +unsafe trait SpecTrustedRandomAccess: Iterator { + /// If `Self: TrustedRandomAccess`, it must be safe to call + /// `Iterator::__iterator_get_unchecked(self, index)`. + unsafe fn try_get_unchecked(&mut self, index: usize) -> Self::Item; +} + +unsafe impl SpecTrustedRandomAccess for I { + default unsafe fn try_get_unchecked(&mut self, _: usize) -> Self::Item { + panic!("Should only be called on TrustedRandomAccess iterators"); + } +} + +unsafe impl SpecTrustedRandomAccess for I { + #[inline] + unsafe fn try_get_unchecked(&mut self, index: usize) -> Self::Item { + // SAFETY: the caller must uphold the contract for + // `Iterator::__iterator_get_unchecked`. + unsafe { self.__iterator_get_unchecked(index) } + } } diff --git a/crux-mir/lib/core/src/iter/mod.rs b/crux-mir/lib/core/src/iter/mod.rs index 080b70c63..bb35d50b4 100644 --- a/crux-mir/lib/core/src/iter/mod.rs +++ b/crux-mir/lib/core/src/iter/mod.rs @@ -39,11 +39,11 @@ //! ``` //! //! An iterator has a method, [`next`], which when called, returns an -//! [`Option`]``. [`next`] will return `Some(Item)` as long as there +//! [Option]\. Calling [`next`] will return [`Some(Item)`] as long as there //! are elements, and once they've all been exhausted, will return `None` to //! indicate that iteration is finished. Individual iterators may choose to //! resume iteration, and so calling [`next`] again may or may not eventually -//! start returning `Some(Item)` again at some point (for example, see [`TryIter`]). +//! start returning [`Some(Item)`] again at some point (for example, see [`TryIter`]). //! //! [`Iterator`]'s full definition includes a number of other methods as well, //! but they are default methods, built on top of [`next`], and so you get @@ -53,9 +53,8 @@ //! more complex forms of processing. See the [Adapters](#adapters) section //! below for more details. //! -//! [`Iterator`]: trait.Iterator.html -//! [`next`]: trait.Iterator.html#tymethod.next -//! [`Option`]: ../../std/option/enum.Option.html +//! [`Some(Item)`]: Some +//! [`next`]: Iterator::next //! [`TryIter`]: ../../std/sync/mpsc/struct.TryIter.html //! //! # The three forms of iteration @@ -72,9 +71,9 @@ //! # Implementing Iterator //! //! Creating an iterator of your own involves two steps: creating a `struct` to -//! hold the iterator's state, and then `impl`ementing [`Iterator`] for that -//! `struct`. This is why there are so many `struct`s in this module: there is -//! one for each iterator and iterator adapter. +//! hold the iterator's state, and then implementing [`Iterator`] for that `struct`. +//! This is why there are so many `struct`s in this module: there is one for +//! each iterator and iterator adapter. //! //! Let's make an iterator named `Counter` which counts from `1` to `5`: //! @@ -136,7 +135,7 @@ //! methods like `nth` and `fold` if an iterator can compute them more efficiently without calling //! `next`. //! -//! # for Loops and IntoIterator +//! # `for` loops and `IntoIterator` //! //! Rust's `for` loop syntax is actually sugar for iterators. Here's a basic //! example of `for`: @@ -145,7 +144,7 @@ //! let values = vec![1, 2, 3, 4, 5]; //! //! for x in values { -//! println!("{}", x); +//! println!("{x}"); //! } //! ``` //! @@ -159,14 +158,13 @@ //! Let's take a look at that `for` loop again, and what the compiler converts //! it into: //! -//! [`IntoIterator`]: trait.IntoIterator.html -//! [`into_iter`]: trait.IntoIterator.html#tymethod.into_iter +//! [`into_iter`]: IntoIterator::into_iter //! //! ``` //! let values = vec![1, 2, 3, 4, 5]; //! //! for x in values { -//! println!("{}", x); +//! println!("{x}"); //! } //! ``` //! @@ -183,7 +181,7 @@ //! None => break, //! }; //! let x = next; -//! let () = { println!("{}", x); }; +//! let () = { println!("{x}"); }; //! }, //! }; //! result @@ -208,6 +206,50 @@ //! 2. If you're creating a collection, implementing [`IntoIterator`] for it //! will allow your collection to be used with the `for` loop. //! +//! # Iterating by reference +//! +//! Since [`into_iter()`] takes `self` by value, using a `for` loop to iterate +//! over a collection consumes that collection. Often, you may want to iterate +//! over a collection without consuming it. Many collections offer methods that +//! provide iterators over references, conventionally called `iter()` and +//! `iter_mut()` respectively: +//! +//! ``` +//! let mut values = vec![41]; +//! for x in values.iter_mut() { +//! *x += 1; +//! } +//! for x in values.iter() { +//! assert_eq!(*x, 42); +//! } +//! assert_eq!(values.len(), 1); // `values` is still owned by this function. +//! ``` +//! +//! If a collection type `C` provides `iter()`, it usually also implements +//! `IntoIterator` for `&C`, with an implementation that just calls `iter()`. +//! Likewise, a collection `C` that provides `iter_mut()` generally implements +//! `IntoIterator` for `&mut C` by delegating to `iter_mut()`. This enables a +//! convenient shorthand: +//! +//! ``` +//! let mut values = vec![41]; +//! for x in &mut values { // same as `values.iter_mut()` +//! *x += 1; +//! } +//! for x in &values { // same as `values.iter()` +//! assert_eq!(*x, 42); +//! } +//! assert_eq!(values.len(), 1); +//! ``` +//! +//! While many collections offer `iter()`, not all offer `iter_mut()`. For +//! example, mutating the keys of a [`HashSet`] could put the collection +//! into an inconsistent state if the key hashes change, so this collection +//! only offers `iter()`. +//! +//! [`into_iter()`]: IntoIterator::into_iter +//! [`HashSet`]: ../../std/collections/struct.HashSet.html +//! //! # Adapters //! //! Functions which take an [`Iterator`] and return another [`Iterator`] are @@ -222,9 +264,9 @@ //! across versions of Rust, so you should avoid relying on the exact values //! returned by an iterator which panicked. //! -//! [`map`]: trait.Iterator.html#method.map -//! [`take`]: trait.Iterator.html#method.take -//! [`filter`]: trait.Iterator.html#method.filter +//! [`map`]: Iterator::map +//! [`take`]: Iterator::take +//! [`filter`]: Iterator::filter //! //! # Laziness //! @@ -237,7 +279,7 @@ //! ``` //! # #![allow(unused_must_use)] //! let v = vec![1, 2, 3, 4, 5]; -//! v.iter().map(|x| println!("{}", x)); +//! v.iter().map(|x| println!("{x}")); //! ``` //! //! This will not print any values, as we only created an iterator, rather than @@ -254,20 +296,20 @@ //! ``` //! let v = vec![1, 2, 3, 4, 5]; //! -//! v.iter().for_each(|x| println!("{}", x)); +//! v.iter().for_each(|x| println!("{x}")); //! // or //! for x in &v { -//! println!("{}", x); +//! println!("{x}"); //! } //! ``` //! -//! [`map`]: trait.Iterator.html#method.map -//! [`for_each`]: trait.Iterator.html#method.for_each +//! [`map`]: Iterator::map +//! [`for_each`]: Iterator::for_each //! //! Another common way to evaluate an iterator is to use the [`collect`] //! method to produce a new collection. //! -//! [`collect`]: trait.Iterator.html#method.collect +//! [`collect`]: Iterator::collect //! //! # Infinity //! @@ -286,14 +328,14 @@ //! let five_numbers = numbers.take(5); //! //! for number in five_numbers { -//! println!("{}", number); +//! println!("{number}"); //! } //! ``` //! //! This will print the numbers `0` through `4`, each on their own line. //! //! Bear in mind that methods on infinite iterators, even those for which a -//! result can be determined mathematically in finite time, may not terminate. +//! result can be determined mathematically in finite time, might not terminate. //! Specifically, methods such as [`min`], which in the general case require //! traversing every element in the iterator, are likely not to return //! successfully for any infinite iterators. @@ -302,15 +344,36 @@ //! let ones = std::iter::repeat(1); //! let least = ones.min().unwrap(); // Oh no! An infinite loop! //! // `ones.min()` causes an infinite loop, so we won't reach this point! -//! println!("The smallest number one is {}.", least); +//! println!("The smallest number one is {least}."); //! ``` //! -//! [`take`]: trait.Iterator.html#method.take -//! [`min`]: trait.Iterator.html#method.min +//! [`take`]: Iterator::take +//! [`min`]: Iterator::min #![stable(feature = "rust1", since = "1.0.0")] -use crate::ops::Try; +// This needs to be up here in order to be usable in the child modules +macro_rules! impl_fold_via_try_fold { + (fold -> try_fold) => { + impl_fold_via_try_fold! { @internal fold -> try_fold } + }; + (rfold -> try_rfold) => { + impl_fold_via_try_fold! { @internal rfold -> try_rfold } + }; + (@internal $fold:ident -> $try_fold:ident) => { + #[inline] + fn $fold(mut self, init: AAA, mut fold: FFF) -> AAA + where + FFF: FnMut(AAA, Self::Item) -> AAA, + { + use crate::const_closure::ConstFnMutClosure; + use crate::ops::NeverShortCircuit; + + let fold = ConstFnMutClosure::new(&mut fold, NeverShortCircuit::wrap_mut_2_imp); + self.$try_fold(init, fold).0 + } + }; +} #[stable(feature = "rust1", since = "1.0.0")] pub use self::traits::Iterator; @@ -322,6 +385,12 @@ pub use self::traits::Iterator; )] pub use self::range::Step; +#[unstable( + feature = "iter_from_generator", + issue = "43122", + reason = "generators are unstable" +)] +pub use self::sources::from_generator; #[stable(feature = "iter_empty", since = "1.2.0")] pub use self::sources::{empty, Empty}; #[stable(feature = "iter_from_fn", since = "1.34.0")] @@ -332,6 +401,8 @@ pub use self::sources::{once, Once}; pub use self::sources::{once_with, OnceWith}; #[stable(feature = "rust1", since = "1.0.0")] pub use self::sources::{repeat, Repeat}; +#[unstable(feature = "iter_repeat_n", issue = "104434")] +pub use self::sources::{repeat_n, RepeatN}; #[stable(feature = "iterator_repeat_with", since = "1.28.0")] pub use self::sources::{repeat_with, RepeatWith}; #[stable(feature = "iter_successors", since = "1.34.0")] @@ -339,87 +410,50 @@ pub use self::sources::{successors, Successors}; #[stable(feature = "fused", since = "1.26.0")] pub use self::traits::FusedIterator; +#[unstable(issue = "none", feature = "inplace_iteration")] +pub use self::traits::InPlaceIterable; #[unstable(feature = "trusted_len", issue = "37572")] pub use self::traits::TrustedLen; +#[unstable(feature = "trusted_step", issue = "85731")] +pub use self::traits::TrustedStep; #[stable(feature = "rust1", since = "1.0.0")] -pub use self::traits::{DoubleEndedIterator, Extend, FromIterator, IntoIterator}; -#[stable(feature = "rust1", since = "1.0.0")] -pub use self::traits::{ExactSizeIterator, Product, Sum}; +pub use self::traits::{ + DoubleEndedIterator, ExactSizeIterator, Extend, FromIterator, IntoIterator, Product, Sum, +}; +#[stable(feature = "iter_zip", since = "1.59.0")] +pub use self::adapters::zip; +#[unstable(feature = "iter_array_chunks", reason = "recently added", issue = "100450")] +pub use self::adapters::ArrayChunks; +#[unstable(feature = "std_internals", issue = "none")] +pub use self::adapters::ByRefSized; #[stable(feature = "iter_cloned", since = "1.1.0")] pub use self::adapters::Cloned; #[stable(feature = "iter_copied", since = "1.36.0")] pub use self::adapters::Copied; #[stable(feature = "iterator_flatten", since = "1.29.0")] pub use self::adapters::Flatten; -#[unstable(feature = "iter_map_while", reason = "recently added", issue = "68537")] +#[stable(feature = "iter_map_while", since = "1.57.0")] pub use self::adapters::MapWhile; +#[unstable(feature = "inplace_iteration", issue = "none")] +pub use self::adapters::SourceIter; #[stable(feature = "iterator_step_by", since = "1.28.0")] pub use self::adapters::StepBy; +#[unstable(feature = "trusted_random_access", issue = "none")] +pub use self::adapters::TrustedRandomAccess; +#[unstable(feature = "trusted_random_access", issue = "none")] +pub use self::adapters::TrustedRandomAccessNoCoerce; #[stable(feature = "rust1", since = "1.0.0")] -pub use self::adapters::{Chain, Cycle, Enumerate, Filter, FilterMap, Map, Rev, Zip}; -#[stable(feature = "rust1", since = "1.0.0")] -pub use self::adapters::{FlatMap, Peekable, Scan, Skip, SkipWhile, Take, TakeWhile}; -#[stable(feature = "rust1", since = "1.0.0")] -pub use self::adapters::{Fuse, Inspect}; +pub use self::adapters::{ + Chain, Cycle, Enumerate, Filter, FilterMap, FlatMap, Fuse, Inspect, Map, Peekable, Rev, Scan, + Skip, SkipWhile, Take, TakeWhile, Zip, +}; +#[unstable(feature = "iter_intersperse", reason = "recently added", issue = "79524")] +pub use self::adapters::{Intersperse, IntersperseWith}; -pub(crate) use self::adapters::{process_results, TrustedRandomAccess}; +pub(crate) use self::adapters::try_process; mod adapters; mod range; mod sources; mod traits; - -/// Used to make try_fold closures more like normal loops -#[derive(PartialEq)] -enum LoopState { - Continue(C), - Break(B), -} - -impl Try for LoopState { - type Ok = C; - type Error = B; - #[inline] - fn into_result(self) -> Result { - match self { - LoopState::Continue(y) => Ok(y), - LoopState::Break(x) => Err(x), - } - } - #[inline] - fn from_error(v: Self::Error) -> Self { - LoopState::Break(v) - } - #[inline] - fn from_ok(v: Self::Ok) -> Self { - LoopState::Continue(v) - } -} - -impl LoopState { - #[inline] - fn break_value(self) -> Option { - match self { - LoopState::Continue(..) => None, - LoopState::Break(x) => Some(x), - } - } -} - -impl LoopState { - #[inline] - fn from_try(r: R) -> Self { - match Try::into_result(r) { - Ok(v) => LoopState::Continue(v), - Err(v) => LoopState::Break(Try::from_error(v)), - } - } - #[inline] - fn into_try(self) -> R { - match self { - LoopState::Continue(v) => Try::from_ok(v), - LoopState::Break(v) => v, - } - } -} diff --git a/crux-mir/lib/core/src/iter/range.rs b/crux-mir/lib/core/src/iter/range.rs index 28fbd00f3..b5739f2f3 100644 --- a/crux-mir/lib/core/src/iter/range.rs +++ b/crux-mir/lib/core/src/iter/range.rs @@ -1,51 +1,184 @@ +use crate::char; use crate::convert::TryFrom; use crate::mem; -use crate::ops::{self, Add, Sub, Try}; -use crate::usize; - -use super::{FusedIterator, TrustedLen}; +use crate::ops::{self, Try}; + +use super::{ + FusedIterator, TrustedLen, TrustedRandomAccess, TrustedRandomAccessNoCoerce, TrustedStep, +}; + +// Safety: All invariants are upheld. +macro_rules! unsafe_impl_trusted_step { + ($($type:ty)*) => {$( + #[unstable(feature = "trusted_step", issue = "85731")] + unsafe impl TrustedStep for $type {} + )*}; +} +unsafe_impl_trusted_step![char i8 i16 i32 i64 i128 isize u8 u16 u32 u64 u128 usize]; -/// Objects that can be stepped over in both directions. +/// Objects that have a notion of *successor* and *predecessor* operations. /// -/// The `steps_between` function provides a way to efficiently compare -/// two `Step` objects. -#[unstable( - feature = "step_trait", - reason = "likely to be replaced by finer-grained traits", - issue = "42168" -)] +/// The *successor* operation moves towards values that compare greater. +/// The *predecessor* operation moves towards values that compare lesser. +#[unstable(feature = "step_trait", reason = "recently redesigned", issue = "42168")] pub trait Step: Clone + PartialOrd + Sized { - /// Returns the number of steps between two step objects. The count is - /// inclusive of `start` and exclusive of `end`. + /// Returns the number of *successor* steps required to get from `start` to `end`. + /// + /// Returns `None` if the number of steps would overflow `usize` + /// (or is infinite, or if `end` would never be reached). + /// + /// # Invariants /// - /// Returns `None` if it is not possible to calculate `steps_between` - /// without overflow. + /// For any `a`, `b`, and `n`: + /// + /// * `steps_between(&a, &b) == Some(n)` if and only if `Step::forward_checked(&a, n) == Some(b)` + /// * `steps_between(&a, &b) == Some(n)` if and only if `Step::backward_checked(&b, n) == Some(a)` + /// * `steps_between(&a, &b) == Some(n)` only if `a <= b` + /// * Corollary: `steps_between(&a, &b) == Some(0)` if and only if `a == b` + /// * Note that `a <= b` does _not_ imply `steps_between(&a, &b) != None`; + /// this is the case when it would require more than `usize::MAX` steps to get to `b` + /// * `steps_between(&a, &b) == None` if `a > b` fn steps_between(start: &Self, end: &Self) -> Option; - /// Replaces this step with `1`, returning a clone of itself. + /// Returns the value that would be obtained by taking the *successor* + /// of `self` `count` times. + /// + /// If this would overflow the range of values supported by `Self`, returns `None`. + /// + /// # Invariants + /// + /// For any `a`, `n`, and `m`: /// - /// The output of this method should always be greater than the output of replace_zero. - fn replace_one(&mut self) -> Self; + /// * `Step::forward_checked(a, n).and_then(|x| Step::forward_checked(x, m)) == Step::forward_checked(a, m).and_then(|x| Step::forward_checked(x, n))` + /// + /// For any `a`, `n`, and `m` where `n + m` does not overflow: + /// + /// * `Step::forward_checked(a, n).and_then(|x| Step::forward_checked(x, m)) == Step::forward_checked(a, n + m)` + /// + /// For any `a` and `n`: + /// + /// * `Step::forward_checked(a, n) == (0..n).try_fold(a, |x, _| Step::forward_checked(&x, 1))` + /// * Corollary: `Step::forward_checked(&a, 0) == Some(a)` + fn forward_checked(start: Self, count: usize) -> Option; - /// Replaces this step with `0`, returning a clone of itself. + /// Returns the value that would be obtained by taking the *successor* + /// of `self` `count` times. + /// + /// If this would overflow the range of values supported by `Self`, + /// this function is allowed to panic, wrap, or saturate. + /// The suggested behavior is to panic when debug assertions are enabled, + /// and to wrap or saturate otherwise. + /// + /// Unsafe code should not rely on the correctness of behavior after overflow. + /// + /// # Invariants /// - /// The output of this method should always be less than the output of replace_one. - fn replace_zero(&mut self) -> Self; + /// For any `a`, `n`, and `m`, where no overflow occurs: + /// + /// * `Step::forward(Step::forward(a, n), m) == Step::forward(a, n + m)` + /// + /// For any `a` and `n`, where no overflow occurs: + /// + /// * `Step::forward_checked(a, n) == Some(Step::forward(a, n))` + /// * `Step::forward(a, n) == (0..n).fold(a, |x, _| Step::forward(x, 1))` + /// * Corollary: `Step::forward(a, 0) == a` + /// * `Step::forward(a, n) >= a` + /// * `Step::backward(Step::forward(a, n), n) == a` + fn forward(start: Self, count: usize) -> Self { + Step::forward_checked(start, count).expect("overflow in `Step::forward`") + } - /// Adds one to this step, returning the result. - fn add_one(&self) -> Self; + /// Returns the value that would be obtained by taking the *successor* + /// of `self` `count` times. + /// + /// # Safety + /// + /// It is undefined behavior for this operation to overflow the + /// range of values supported by `Self`. If you cannot guarantee that this + /// will not overflow, use `forward` or `forward_checked` instead. + /// + /// # Invariants + /// + /// For any `a`: + /// + /// * if there exists `b` such that `b > a`, it is safe to call `Step::forward_unchecked(a, 1)` + /// * if there exists `b`, `n` such that `steps_between(&a, &b) == Some(n)`, + /// it is safe to call `Step::forward_unchecked(a, m)` for any `m <= n`. + /// + /// For any `a` and `n`, where no overflow occurs: + /// + /// * `Step::forward_unchecked(a, n)` is equivalent to `Step::forward(a, n)` + unsafe fn forward_unchecked(start: Self, count: usize) -> Self { + Step::forward(start, count) + } - /// Subtracts one to this step, returning the result. - fn sub_one(&self) -> Self; + /// Returns the value that would be obtained by taking the *predecessor* + /// of `self` `count` times. + /// + /// If this would overflow the range of values supported by `Self`, returns `None`. + /// + /// # Invariants + /// + /// For any `a`, `n`, and `m`: + /// + /// * `Step::backward_checked(a, n).and_then(|x| Step::backward_checked(x, m)) == n.checked_add(m).and_then(|x| Step::backward_checked(a, x))` + /// * `Step::backward_checked(a, n).and_then(|x| Step::backward_checked(x, m)) == try { Step::backward_checked(a, n.checked_add(m)?) }` + /// + /// For any `a` and `n`: + /// + /// * `Step::backward_checked(a, n) == (0..n).try_fold(a, |x, _| Step::backward_checked(&x, 1))` + /// * Corollary: `Step::backward_checked(&a, 0) == Some(a)` + fn backward_checked(start: Self, count: usize) -> Option; - /// Adds a `usize`, returning `None` on overflow. - fn add_usize(&self, n: usize) -> Option; + /// Returns the value that would be obtained by taking the *predecessor* + /// of `self` `count` times. + /// + /// If this would overflow the range of values supported by `Self`, + /// this function is allowed to panic, wrap, or saturate. + /// The suggested behavior is to panic when debug assertions are enabled, + /// and to wrap or saturate otherwise. + /// + /// Unsafe code should not rely on the correctness of behavior after overflow. + /// + /// # Invariants + /// + /// For any `a`, `n`, and `m`, where no overflow occurs: + /// + /// * `Step::backward(Step::backward(a, n), m) == Step::backward(a, n + m)` + /// + /// For any `a` and `n`, where no overflow occurs: + /// + /// * `Step::backward_checked(a, n) == Some(Step::backward(a, n))` + /// * `Step::backward(a, n) == (0..n).fold(a, |x, _| Step::backward(x, 1))` + /// * Corollary: `Step::backward(a, 0) == a` + /// * `Step::backward(a, n) <= a` + /// * `Step::forward(Step::backward(a, n), n) == a` + fn backward(start: Self, count: usize) -> Self { + Step::backward_checked(start, count).expect("overflow in `Step::backward`") + } - /// Subtracts a `usize`, returning `None` on underflow. - fn sub_usize(&self, n: usize) -> Option { - // this default implementation makes the addition of `sub_usize` a non-breaking change - let _ = n; - unimplemented!() + /// Returns the value that would be obtained by taking the *predecessor* + /// of `self` `count` times. + /// + /// # Safety + /// + /// It is undefined behavior for this operation to overflow the + /// range of values supported by `Self`. If you cannot guarantee that this + /// will not overflow, use `backward` or `backward_checked` instead. + /// + /// # Invariants + /// + /// For any `a`: + /// + /// * if there exists `b` such that `b < a`, it is safe to call `Step::backward_unchecked(a, 1)` + /// * if there exists `b`, `n` such that `steps_between(&b, &a) == Some(n)`, + /// it is safe to call `Step::backward_unchecked(a, m)` for any `m <= n`. + /// + /// For any `a` and `n`, where no overflow occurs: + /// + /// * `Step::backward_unchecked(a, n)` is equivalent to `Step::backward(a, n)` + unsafe fn backward_unchecked(start: Self, count: usize) -> Self { + Step::backward(start, count) } } @@ -53,127 +186,303 @@ pub trait Step: Clone + PartialOrd + Sized { macro_rules! step_identical_methods { () => { #[inline] - fn replace_one(&mut self) -> Self { - mem::replace(self, 1) + unsafe fn forward_unchecked(start: Self, n: usize) -> Self { + // SAFETY: the caller has to guarantee that `start + n` doesn't overflow. + unsafe { start.unchecked_add(n as Self) } } #[inline] - fn replace_zero(&mut self) -> Self { - mem::replace(self, 0) + unsafe fn backward_unchecked(start: Self, n: usize) -> Self { + // SAFETY: the caller has to guarantee that `start - n` doesn't overflow. + unsafe { start.unchecked_sub(n as Self) } } #[inline] - fn add_one(&self) -> Self { - Add::add(*self, 1) + #[allow(arithmetic_overflow)] + #[rustc_inherit_overflow_checks] + fn forward(start: Self, n: usize) -> Self { + // In debug builds, trigger a panic on overflow. + // This should optimize completely out in release builds. + if Self::forward_checked(start, n).is_none() { + let _ = Self::MAX + 1; + } + // Do wrapping math to allow e.g. `Step::forward(-128i8, 255)`. + start.wrapping_add(n as Self) } #[inline] - fn sub_one(&self) -> Self { - Sub::sub(*self, 1) + #[allow(arithmetic_overflow)] + #[rustc_inherit_overflow_checks] + fn backward(start: Self, n: usize) -> Self { + // In debug builds, trigger a panic on overflow. + // This should optimize completely out in release builds. + if Self::backward_checked(start, n).is_none() { + let _ = Self::MIN - 1; + } + // Do wrapping math to allow e.g. `Step::backward(127i8, 255)`. + start.wrapping_sub(n as Self) } - } + }; } -macro_rules! step_impl_unsigned { - ($($t:ty)*) => ($( - #[unstable(feature = "step_trait", - reason = "likely to be replaced by finer-grained traits", - issue = "42168")] - impl Step for $t { - #[inline] - fn steps_between(start: &$t, end: &$t) -> Option { - if *start < *end { - usize::try_from(*end - *start).ok() - } else { - Some(0) +macro_rules! step_integer_impls { + { + narrower than or same width as usize: + $( [ $u_narrower:ident $i_narrower:ident ] ),+; + wider than usize: + $( [ $u_wider:ident $i_wider:ident ] ),+; + } => { + $( + #[allow(unreachable_patterns)] + #[unstable(feature = "step_trait", reason = "recently redesigned", issue = "42168")] + impl Step for $u_narrower { + step_identical_methods!(); + + #[inline] + fn steps_between(start: &Self, end: &Self) -> Option { + if *start <= *end { + // This relies on $u_narrower <= usize + Some((*end - *start) as usize) + } else { + None + } } - } - #[inline] - #[allow(unreachable_patterns)] - fn add_usize(&self, n: usize) -> Option { - match <$t>::try_from(n) { - Ok(n_as_t) => self.checked_add(n_as_t), - Err(_) => None, + #[inline] + fn forward_checked(start: Self, n: usize) -> Option { + match Self::try_from(n) { + Ok(n) => start.checked_add(n), + Err(_) => None, // if n is out of range, `unsigned_start + n` is too + } + } + + #[inline] + fn backward_checked(start: Self, n: usize) -> Option { + match Self::try_from(n) { + Ok(n) => start.checked_sub(n), + Err(_) => None, // if n is out of range, `unsigned_start - n` is too + } } } - #[inline] #[allow(unreachable_patterns)] - fn sub_usize(&self, n: usize) -> Option { - match <$t>::try_from(n) { - Ok(n_as_t) => self.checked_sub(n_as_t), - Err(_) => None, + #[unstable(feature = "step_trait", reason = "recently redesigned", issue = "42168")] + impl Step for $i_narrower { + step_identical_methods!(); + + #[inline] + fn steps_between(start: &Self, end: &Self) -> Option { + if *start <= *end { + // This relies on $i_narrower <= usize + // + // Casting to isize extends the width but preserves the sign. + // Use wrapping_sub in isize space and cast to usize to compute + // the difference that might not fit inside the range of isize. + Some((*end as isize).wrapping_sub(*start as isize) as usize) + } else { + None + } } - } - step_identical_methods!(); - } - )*) -} -macro_rules! step_impl_signed { - ($( [$t:ty : $unsigned:ty] )*) => ($( - #[unstable(feature = "step_trait", - reason = "likely to be replaced by finer-grained traits", - issue = "42168")] - impl Step for $t { - #[inline] - fn steps_between(start: &$t, end: &$t) -> Option { - if *start < *end { - // Use .wrapping_sub and cast to unsigned to compute the - // difference that may not fit inside the range of $t. - usize::try_from(end.wrapping_sub(*start) as $unsigned).ok() - } else { - Some(0) + #[inline] + fn forward_checked(start: Self, n: usize) -> Option { + match $u_narrower::try_from(n) { + Ok(n) => { + // Wrapping handles cases like + // `Step::forward(-120_i8, 200) == Some(80_i8)`, + // even though 200 is out of range for i8. + let wrapped = start.wrapping_add(n as Self); + if wrapped >= start { + Some(wrapped) + } else { + None // Addition overflowed + } + } + // If n is out of range of e.g. u8, + // then it is bigger than the entire range for i8 is wide + // so `any_i8 + n` necessarily overflows i8. + Err(_) => None, + } + } + + #[inline] + fn backward_checked(start: Self, n: usize) -> Option { + match $u_narrower::try_from(n) { + Ok(n) => { + // Wrapping handles cases like + // `Step::forward(-120_i8, 200) == Some(80_i8)`, + // even though 200 is out of range for i8. + let wrapped = start.wrapping_sub(n as Self); + if wrapped <= start { + Some(wrapped) + } else { + None // Subtraction overflowed + } + } + // If n is out of range of e.g. u8, + // then it is bigger than the entire range for i8 is wide + // so `any_i8 - n` necessarily overflows i8. + Err(_) => None, + } } } + )+ - #[inline] + $( #[allow(unreachable_patterns)] - fn add_usize(&self, n: usize) -> Option { - match <$unsigned>::try_from(n) { - Ok(n_as_unsigned) => { - // Wrapping in unsigned space handles cases like - // `-120_i8.add_usize(200) == Some(80_i8)`, - // even though 200_usize is out of range for i8. - let wrapped = (*self as $unsigned).wrapping_add(n_as_unsigned) as $t; - if wrapped >= *self { - Some(wrapped) - } else { - None // Addition overflowed - } + #[unstable(feature = "step_trait", reason = "recently redesigned", issue = "42168")] + impl Step for $u_wider { + step_identical_methods!(); + + #[inline] + fn steps_between(start: &Self, end: &Self) -> Option { + if *start <= *end { + usize::try_from(*end - *start).ok() + } else { + None } - Err(_) => None, + } + + #[inline] + fn forward_checked(start: Self, n: usize) -> Option { + start.checked_add(n as Self) + } + + #[inline] + fn backward_checked(start: Self, n: usize) -> Option { + start.checked_sub(n as Self) } } - #[inline] #[allow(unreachable_patterns)] - fn sub_usize(&self, n: usize) -> Option { - match <$unsigned>::try_from(n) { - Ok(n_as_unsigned) => { - // Wrapping in unsigned space handles cases like - // `80_i8.sub_usize(200) == Some(-120_i8)`, - // even though 200_usize is out of range for i8. - let wrapped = (*self as $unsigned).wrapping_sub(n_as_unsigned) as $t; - if wrapped <= *self { - Some(wrapped) - } else { - None // Subtraction underflowed + #[unstable(feature = "step_trait", reason = "recently redesigned", issue = "42168")] + impl Step for $i_wider { + step_identical_methods!(); + + #[inline] + fn steps_between(start: &Self, end: &Self) -> Option { + if *start <= *end { + match end.checked_sub(*start) { + Some(result) => usize::try_from(result).ok(), + // If the difference is too big for e.g. i128, + // it's also gonna be too big for usize with fewer bits. + None => None, } + } else { + None } - Err(_) => None, + } + + #[inline] + fn forward_checked(start: Self, n: usize) -> Option { + start.checked_add(n as Self) + } + + #[inline] + fn backward_checked(start: Self, n: usize) -> Option { + start.checked_sub(n as Self) } } + )+ + }; +} - step_identical_methods!(); - } - )*) +#[cfg(target_pointer_width = "64")] +step_integer_impls! { + narrower than or same width as usize: [u8 i8], [u16 i16], [u32 i32], [u64 i64], [usize isize]; + wider than usize: [u128 i128]; +} + +#[cfg(target_pointer_width = "32")] +step_integer_impls! { + narrower than or same width as usize: [u8 i8], [u16 i16], [u32 i32], [usize isize]; + wider than usize: [u64 i64], [u128 i128]; } -step_impl_unsigned!(usize u8 u16 u32 u64 u128); -step_impl_signed!([isize: usize][i8: u8][i16: u16]); -step_impl_signed!([i32: u32][i64: u64][i128: u128]); +#[cfg(target_pointer_width = "16")] +step_integer_impls! { + narrower than or same width as usize: [u8 i8], [u16 i16], [usize isize]; + wider than usize: [u32 i32], [u64 i64], [u128 i128]; +} + +#[unstable(feature = "step_trait", reason = "recently redesigned", issue = "42168")] +impl Step for char { + #[inline] + fn steps_between(&start: &char, &end: &char) -> Option { + let start = start as u32; + let end = end as u32; + if start <= end { + let count = end - start; + if start < 0xD800 && 0xE000 <= end { + usize::try_from(count - 0x800).ok() + } else { + usize::try_from(count).ok() + } + } else { + None + } + } + + #[inline] + fn forward_checked(start: char, count: usize) -> Option { + let start = start as u32; + let mut res = Step::forward_checked(start, count)?; + if start < 0xD800 && 0xD800 <= res { + res = Step::forward_checked(res, 0x800)?; + } + if res <= char::MAX as u32 { + // SAFETY: res is a valid unicode scalar + // (below 0x110000 and not in 0xD800..0xE000) + Some(unsafe { char::from_u32_unchecked(res) }) + } else { + None + } + } + + #[inline] + fn backward_checked(start: char, count: usize) -> Option { + let start = start as u32; + let mut res = Step::backward_checked(start, count)?; + if start >= 0xE000 && 0xE000 > res { + res = Step::backward_checked(res, 0x800)?; + } + // SAFETY: res is a valid unicode scalar + // (below 0x110000 and not in 0xD800..0xE000) + Some(unsafe { char::from_u32_unchecked(res) }) + } + + #[inline] + unsafe fn forward_unchecked(start: char, count: usize) -> char { + let start = start as u32; + // SAFETY: the caller must guarantee that this doesn't overflow + // the range of values for a char. + let mut res = unsafe { Step::forward_unchecked(start, count) }; + if start < 0xD800 && 0xD800 <= res { + // SAFETY: the caller must guarantee that this doesn't overflow + // the range of values for a char. + res = unsafe { Step::forward_unchecked(res, 0x800) }; + } + // SAFETY: because of the previous contract, this is guaranteed + // by the caller to be a valid char. + unsafe { char::from_u32_unchecked(res) } + } + + #[inline] + unsafe fn backward_unchecked(start: char, count: usize) -> char { + let start = start as u32; + // SAFETY: the caller must guarantee that this doesn't overflow + // the range of values for a char. + let mut res = unsafe { Step::backward_unchecked(start, count) }; + if start >= 0xE000 && 0xE000 > res { + // SAFETY: the caller must guarantee that this doesn't overflow + // the range of values for a char. + res = unsafe { Step::backward_unchecked(res, 0x800) }; + } + // SAFETY: because of the previous contract, this is guaranteed + // by the caller to be a valid char. + unsafe { char::from_u32_unchecked(res) } + } +} macro_rules! range_exact_iter_impl { ($($t:ty)*) => ($( @@ -182,62 +491,148 @@ macro_rules! range_exact_iter_impl { )*) } -macro_rules! range_incl_exact_iter_impl { +/// Safety: This macro must only be used on types that are `Copy` and result in ranges +/// which have an exact `size_hint()` where the upper bound must not be `None`. +macro_rules! unsafe_range_trusted_random_access_impl { ($($t:ty)*) => ($( - #[stable(feature = "inclusive_range", since = "1.26.0")] - impl ExactSizeIterator for ops::RangeInclusive<$t> { } + #[doc(hidden)] + #[unstable(feature = "trusted_random_access", issue = "none")] + unsafe impl TrustedRandomAccess for ops::Range<$t> {} + + #[doc(hidden)] + #[unstable(feature = "trusted_random_access", issue = "none")] + unsafe impl TrustedRandomAccessNoCoerce for ops::Range<$t> { + const MAY_HAVE_SIDE_EFFECT: bool = false; + } )*) } -macro_rules! range_trusted_len_impl { +macro_rules! range_incl_exact_iter_impl { ($($t:ty)*) => ($( - #[unstable(feature = "trusted_len", issue = "37572")] - unsafe impl TrustedLen for ops::Range<$t> { } + #[stable(feature = "inclusive_range", since = "1.26.0")] + impl ExactSizeIterator for ops::RangeInclusive<$t> { } )*) } -macro_rules! range_incl_trusted_len_impl { - ($($t:ty)*) => ($( - #[unstable(feature = "trusted_len", issue = "37572")] - unsafe impl TrustedLen for ops::RangeInclusive<$t> { } - )*) +/// Specialization implementations for `Range`. +trait RangeIteratorImpl { + type Item; + + // Iterator + fn spec_next(&mut self) -> Option; + fn spec_nth(&mut self, n: usize) -> Option; + fn spec_advance_by(&mut self, n: usize) -> Result<(), usize>; + + // DoubleEndedIterator + fn spec_next_back(&mut self) -> Option; + fn spec_nth_back(&mut self, n: usize) -> Option; + fn spec_advance_back_by(&mut self, n: usize) -> Result<(), usize>; } -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for ops::Range { +impl RangeIteratorImpl for ops::Range { type Item = A; #[inline] - fn next(&mut self) -> Option { + default fn spec_next(&mut self) -> Option { if self.start < self.end { - // We check for overflow here, even though it can't actually - // happen. Adding this check does however help llvm vectorize loops - // for some ranges that don't get vectorized otherwise, - // and this won't actually result in an extra check in an optimized build. - if let Some(mut n) = self.start.add_usize(1) { - mem::swap(&mut n, &mut self.start); - Some(n) - } else { - None + let n = + Step::forward_checked(self.start.clone(), 1).expect("`Step` invariants not upheld"); + Some(mem::replace(&mut self.start, n)) + } else { + None + } + } + + #[inline] + default fn spec_nth(&mut self, n: usize) -> Option { + if let Some(plus_n) = Step::forward_checked(self.start.clone(), n) { + if plus_n < self.end { + self.start = + Step::forward_checked(plus_n.clone(), 1).expect("`Step` invariants not upheld"); + return Some(plus_n); } + } + + self.start = self.end.clone(); + None + } + + #[inline] + default fn spec_advance_by(&mut self, n: usize) -> Result<(), usize> { + let available = if self.start <= self.end { + Step::steps_between(&self.start, &self.end).unwrap_or(usize::MAX) + } else { + 0 + }; + + let taken = available.min(n); + + self.start = + Step::forward_checked(self.start.clone(), taken).expect("`Step` invariants not upheld"); + + if taken < n { Err(taken) } else { Ok(()) } + } + + #[inline] + default fn spec_next_back(&mut self) -> Option { + if self.start < self.end { + self.end = + Step::backward_checked(self.end.clone(), 1).expect("`Step` invariants not upheld"); + Some(self.end.clone()) } else { None } } #[inline] - fn size_hint(&self) -> (usize, Option) { - match Step::steps_between(&self.start, &self.end) { - Some(hint) => (hint, Some(hint)), - None => (usize::MAX, None), + default fn spec_nth_back(&mut self, n: usize) -> Option { + if let Some(minus_n) = Step::backward_checked(self.end.clone(), n) { + if minus_n > self.start { + self.end = + Step::backward_checked(minus_n, 1).expect("`Step` invariants not upheld"); + return Some(self.end.clone()); + } } + + self.end = self.start.clone(); + None } #[inline] - fn nth(&mut self, n: usize) -> Option { - if let Some(plus_n) = self.start.add_usize(n) { + default fn spec_advance_back_by(&mut self, n: usize) -> Result<(), usize> { + let available = if self.start <= self.end { + Step::steps_between(&self.start, &self.end).unwrap_or(usize::MAX) + } else { + 0 + }; + + let taken = available.min(n); + + self.end = + Step::backward_checked(self.end.clone(), taken).expect("`Step` invariants not upheld"); + + if taken < n { Err(taken) } else { Ok(()) } + } +} + +impl RangeIteratorImpl for ops::Range { + #[inline] + fn spec_next(&mut self) -> Option { + if self.start < self.end { + // SAFETY: just checked precondition + let n = unsafe { Step::forward_unchecked(self.start.clone(), 1) }; + Some(mem::replace(&mut self.start, n)) + } else { + None + } + } + + #[inline] + fn spec_nth(&mut self, n: usize) -> Option { + if let Some(plus_n) = Step::forward_checked(self.start.clone(), n) { if plus_n < self.end { - self.start = plus_n.add_one(); + // SAFETY: just checked precondition + self.start = unsafe { Step::forward_unchecked(plus_n.clone(), 1) }; return Some(plus_n); } } @@ -246,6 +641,91 @@ impl Iterator for ops::Range { None } + #[inline] + fn spec_advance_by(&mut self, n: usize) -> Result<(), usize> { + let available = if self.start <= self.end { + Step::steps_between(&self.start, &self.end).unwrap_or(usize::MAX) + } else { + 0 + }; + + let taken = available.min(n); + + // SAFETY: the conditions above ensure that the count is in bounds. If start <= end + // then steps_between either returns a bound to which we clamp or returns None which + // together with the initial inequality implies more than usize::MAX steps. + // Otherwise 0 is returned which always safe to use. + self.start = unsafe { Step::forward_unchecked(self.start.clone(), taken) }; + + if taken < n { Err(taken) } else { Ok(()) } + } + + #[inline] + fn spec_next_back(&mut self) -> Option { + if self.start < self.end { + // SAFETY: just checked precondition + self.end = unsafe { Step::backward_unchecked(self.end.clone(), 1) }; + Some(self.end.clone()) + } else { + None + } + } + + #[inline] + fn spec_nth_back(&mut self, n: usize) -> Option { + if let Some(minus_n) = Step::backward_checked(self.end.clone(), n) { + if minus_n > self.start { + // SAFETY: just checked precondition + self.end = unsafe { Step::backward_unchecked(minus_n, 1) }; + return Some(self.end.clone()); + } + } + + self.end = self.start.clone(); + None + } + + #[inline] + fn spec_advance_back_by(&mut self, n: usize) -> Result<(), usize> { + let available = if self.start <= self.end { + Step::steps_between(&self.start, &self.end).unwrap_or(usize::MAX) + } else { + 0 + }; + + let taken = available.min(n); + + // SAFETY: same as the spec_advance_by() implementation + self.end = unsafe { Step::backward_unchecked(self.end.clone(), taken) }; + + if taken < n { Err(taken) } else { Ok(()) } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for ops::Range { + type Item = A; + + #[inline] + fn next(&mut self) -> Option { + self.spec_next() + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + if self.start < self.end { + let hint = Step::steps_between(&self.start, &self.end); + (hint.unwrap_or(usize::MAX), hint) + } else { + (0, Some(0)) + } + } + + #[inline] + fn nth(&mut self, n: usize) -> Option { + self.spec_nth(n) + } + #[inline] fn last(mut self) -> Option { self.next_back() @@ -260,48 +740,118 @@ impl Iterator for ops::Range { fn max(mut self) -> Option { self.next_back() } + + #[inline] + fn is_sorted(self) -> bool { + true + } + + #[inline] + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + self.spec_advance_by(n) + } + + #[inline] + unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> Self::Item + where + Self: TrustedRandomAccessNoCoerce, + { + // SAFETY: The TrustedRandomAccess contract requires that callers only pass an index + // that is in bounds. + // Additionally Self: TrustedRandomAccess is only implemented for Copy types + // which means even repeated reads of the same index would be safe. + unsafe { Step::forward_unchecked(self.start.clone(), idx) } + } } // These macros generate `ExactSizeIterator` impls for various range types. -// Range<{u,i}64> and RangeInclusive<{u,i}{32,64,size}> are excluded -// because they cannot guarantee having a length <= usize::MAX, which is -// required by ExactSizeIterator. -range_exact_iter_impl!(usize u8 u16 u32 isize i8 i16 i32); -range_incl_exact_iter_impl!(u8 u16 i8 i16); - -// These macros generate `TrustedLen` impls. // -// They need to guarantee that .size_hint() is either exact, or that -// the upper bound is None when it does not fit the type limits. -range_trusted_len_impl!(usize isize u8 i8 u16 i16 u32 i32 u64 i64 u128 i128); -range_incl_trusted_len_impl!(usize isize u8 i8 u16 i16 u32 i32 u64 i64 u128 i128); +// * `ExactSizeIterator::len` is required to always return an exact `usize`, +// so no range can be longer than `usize::MAX`. +// * For integer types in `Range<_>` this is the case for types narrower than or as wide as `usize`. +// For integer types in `RangeInclusive<_>` +// this is the case for types *strictly narrower* than `usize` +// since e.g. `(0..=u64::MAX).len()` would be `u64::MAX + 1`. +range_exact_iter_impl! { + usize u8 u16 + isize i8 i16 + + // These are incorrect per the reasoning above, + // but removing them would be a breaking change as they were stabilized in Rust 1.0.0. + // So e.g. `(0..66_000_u32).len()` for example will compile without error or warnings + // on 16-bit platforms, but continue to give a wrong result. + u32 + i32 +} + +unsafe_range_trusted_random_access_impl! { + usize u8 u16 + isize i8 i16 +} + +#[cfg(target_pointer_width = "32")] +unsafe_range_trusted_random_access_impl! { + u32 i32 +} + +#[cfg(target_pointer_width = "64")] +unsafe_range_trusted_random_access_impl! { + u32 i32 + u64 i64 +} + +range_incl_exact_iter_impl! { + u8 + i8 + + // These are incorrect per the reasoning above, + // but removing them would be a breaking change as they were stabilized in Rust 1.26.0. + // So e.g. `(0..=u16::MAX).len()` for example will compile without error or warnings + // on 16-bit platforms, but continue to give a wrong result. + u16 + i16 +} #[stable(feature = "rust1", since = "1.0.0")] impl DoubleEndedIterator for ops::Range { #[inline] fn next_back(&mut self) -> Option { - if self.start < self.end { - self.end = self.end.sub_one(); - Some(self.end.clone()) - } else { - None - } + self.spec_next_back() } #[inline] fn nth_back(&mut self, n: usize) -> Option { - if let Some(minus_n) = self.end.sub_usize(n) { - if minus_n > self.start { - self.end = minus_n.sub_one(); - return Some(self.end.clone()); - } - } + self.spec_nth_back(n) + } - self.end = self.start.clone(); - None + #[inline] + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + self.spec_advance_back_by(n) } } +// Safety: +// The following invariants for `Step::steps_between` exist: +// +// > * `steps_between(&a, &b) == Some(n)` only if `a <= b` +// > * Note that `a <= b` does _not_ imply `steps_between(&a, &b) != None`; +// > this is the case when it would require more than `usize::MAX` steps to +// > get to `b` +// > * `steps_between(&a, &b) == None` if `a > b` +// +// The first invariant is what is generally required for `TrustedLen` to be +// sound. The note addendum satisfies an additional `TrustedLen` invariant. +// +// > The upper bound must only be `None` if the actual iterator length is larger +// > than `usize::MAX` +// +// The second invariant logically follows the first so long as the `PartialOrd` +// implementation is correct; regardless it is explicitly stated. If `a < b` +// then `(0, Some(0))` is returned by `ops::Range::size_hint`. As such +// the second invariant is upheld. +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for ops::Range {} + #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for ops::Range {} @@ -311,9 +861,8 @@ impl Iterator for ops::RangeFrom { #[inline] fn next(&mut self) -> Option { - let mut n = self.start.add_one(); - mem::swap(&mut n, &mut self.start); - Some(n) + let n = Step::forward(self.start.clone(), 1); + Some(mem::replace(&mut self.start, n)) } #[inline] @@ -323,30 +872,143 @@ impl Iterator for ops::RangeFrom { #[inline] fn nth(&mut self, n: usize) -> Option { - let plus_n = self.start.add_usize(n).expect("overflow in RangeFrom::nth"); - self.start = plus_n.add_one(); + let plus_n = Step::forward(self.start.clone(), n); + self.start = Step::forward(plus_n.clone(), 1); Some(plus_n) } } +// Safety: See above implementation for `ops::Range` +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for ops::RangeFrom {} + #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for ops::RangeFrom {} -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for ops::RangeFrom {} +trait RangeInclusiveIteratorImpl { + type Item; -#[stable(feature = "inclusive_range", since = "1.26.0")] -impl Iterator for ops::RangeInclusive { + // Iterator + fn spec_next(&mut self) -> Option; + fn spec_try_fold(&mut self, init: B, f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: Try; + + // DoubleEndedIterator + fn spec_next_back(&mut self) -> Option; + fn spec_try_rfold(&mut self, init: B, f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: Try; +} + +impl RangeInclusiveIteratorImpl for ops::RangeInclusive { type Item = A; #[inline] - fn next(&mut self) -> Option { + default fn spec_next(&mut self) -> Option { + if self.is_empty() { + return None; + } + let is_iterating = self.start < self.end; + Some(if is_iterating { + let n = + Step::forward_checked(self.start.clone(), 1).expect("`Step` invariants not upheld"); + mem::replace(&mut self.start, n) + } else { + self.exhausted = true; + self.start.clone() + }) + } + + #[inline] + default fn spec_try_fold(&mut self, init: B, mut f: F) -> R + where + Self: Sized, + F: FnMut(B, A) -> R, + R: Try, + { + if self.is_empty() { + return try { init }; + } + + let mut accum = init; + + while self.start < self.end { + let n = + Step::forward_checked(self.start.clone(), 1).expect("`Step` invariants not upheld"); + let n = mem::replace(&mut self.start, n); + accum = f(accum, n)?; + } + + self.exhausted = true; + + if self.start == self.end { + accum = f(accum, self.start.clone())?; + } + + try { accum } + } + + #[inline] + default fn spec_next_back(&mut self) -> Option { + if self.is_empty() { + return None; + } + let is_iterating = self.start < self.end; + Some(if is_iterating { + let n = + Step::backward_checked(self.end.clone(), 1).expect("`Step` invariants not upheld"); + mem::replace(&mut self.end, n) + } else { + self.exhausted = true; + self.end.clone() + }) + } + + #[inline] + default fn spec_try_rfold(&mut self, init: B, mut f: F) -> R + where + Self: Sized, + F: FnMut(B, A) -> R, + R: Try, + { + if self.is_empty() { + return try { init }; + } + + let mut accum = init; + + while self.start < self.end { + let n = + Step::backward_checked(self.end.clone(), 1).expect("`Step` invariants not upheld"); + let n = mem::replace(&mut self.end, n); + accum = f(accum, n)?; + } + + self.exhausted = true; + + if self.start == self.end { + accum = f(accum, self.start.clone())?; + } + + try { accum } + } +} + +impl RangeInclusiveIteratorImpl for ops::RangeInclusive { + #[inline] + fn spec_next(&mut self) -> Option { if self.is_empty() { return None; } let is_iterating = self.start < self.end; Some(if is_iterating { - let n = self.start.add_one(); + // SAFETY: just checked precondition + let n = unsafe { Step::forward_unchecked(self.start.clone(), 1) }; mem::replace(&mut self.start, n) } else { self.exhausted = true; @@ -354,6 +1016,90 @@ impl Iterator for ops::RangeInclusive { }) } + #[inline] + fn spec_try_fold(&mut self, init: B, mut f: F) -> R + where + Self: Sized, + F: FnMut(B, T) -> R, + R: Try, + { + if self.is_empty() { + return try { init }; + } + + let mut accum = init; + + while self.start < self.end { + // SAFETY: just checked precondition + let n = unsafe { Step::forward_unchecked(self.start.clone(), 1) }; + let n = mem::replace(&mut self.start, n); + accum = f(accum, n)?; + } + + self.exhausted = true; + + if self.start == self.end { + accum = f(accum, self.start.clone())?; + } + + try { accum } + } + + #[inline] + fn spec_next_back(&mut self) -> Option { + if self.is_empty() { + return None; + } + let is_iterating = self.start < self.end; + Some(if is_iterating { + // SAFETY: just checked precondition + let n = unsafe { Step::backward_unchecked(self.end.clone(), 1) }; + mem::replace(&mut self.end, n) + } else { + self.exhausted = true; + self.end.clone() + }) + } + + #[inline] + fn spec_try_rfold(&mut self, init: B, mut f: F) -> R + where + Self: Sized, + F: FnMut(B, T) -> R, + R: Try, + { + if self.is_empty() { + return try { init }; + } + + let mut accum = init; + + while self.start < self.end { + // SAFETY: just checked precondition + let n = unsafe { Step::backward_unchecked(self.end.clone(), 1) }; + let n = mem::replace(&mut self.end, n); + accum = f(accum, n)?; + } + + self.exhausted = true; + + if self.start == self.end { + accum = f(accum, self.start.clone())?; + } + + try { accum } + } +} + +#[stable(feature = "inclusive_range", since = "1.26.0")] +impl Iterator for ops::RangeInclusive { + type Item = A; + + #[inline] + fn next(&mut self) -> Option { + self.spec_next() + } + #[inline] fn size_hint(&self) -> (usize, Option) { if self.is_empty() { @@ -372,12 +1118,12 @@ impl Iterator for ops::RangeInclusive { return None; } - if let Some(plus_n) = self.start.add_usize(n) { + if let Some(plus_n) = Step::forward_checked(self.start.clone(), n) { use crate::cmp::Ordering::*; match plus_n.partial_cmp(&self.end) { Some(Less) => { - self.start = plus_n.add_one(); + self.start = Step::forward(plus_n.clone(), 1); return Some(plus_n); } Some(Equal) => { @@ -395,33 +1141,17 @@ impl Iterator for ops::RangeInclusive { } #[inline] - fn try_fold(&mut self, init: B, mut f: F) -> R + fn try_fold(&mut self, init: B, f: F) -> R where Self: Sized, F: FnMut(B, Self::Item) -> R, - R: Try, + R: Try, { - if self.is_empty() { - return Try::from_ok(init); - } - - let mut accum = init; - - while self.start < self.end { - let n = self.start.add_one(); - let n = mem::replace(&mut self.start, n); - accum = f(accum, n)?; - } - - self.exhausted = true; - - if self.start == self.end { - accum = f(accum, self.start.clone())?; - } - - Try::from_ok(accum) + self.spec_try_fold(init, f) } + impl_fold_via_try_fold! { fold -> try_fold } + #[inline] fn last(mut self) -> Option { self.next_back() @@ -436,23 +1166,18 @@ impl Iterator for ops::RangeInclusive { fn max(mut self) -> Option { self.next_back() } + + #[inline] + fn is_sorted(self) -> bool { + true + } } #[stable(feature = "inclusive_range", since = "1.26.0")] impl DoubleEndedIterator for ops::RangeInclusive { #[inline] fn next_back(&mut self) -> Option { - if self.is_empty() { - return None; - } - let is_iterating = self.start < self.end; - Some(if is_iterating { - let n = self.end.sub_one(); - mem::replace(&mut self.end, n) - } else { - self.exhausted = true; - self.end.clone() - }) + self.spec_next_back() } #[inline] @@ -461,12 +1186,12 @@ impl DoubleEndedIterator for ops::RangeInclusive { return None; } - if let Some(minus_n) = self.end.sub_usize(n) { + if let Some(minus_n) = Step::backward_checked(self.end.clone(), n) { use crate::cmp::Ordering::*; match minus_n.partial_cmp(&self.start) { Some(Greater) => { - self.end = minus_n.sub_one(); + self.end = Step::backward(minus_n.clone(), 1); return Some(minus_n); } Some(Equal) => { @@ -484,33 +1209,21 @@ impl DoubleEndedIterator for ops::RangeInclusive { } #[inline] - fn try_rfold(&mut self, init: B, mut f: F) -> R + fn try_rfold(&mut self, init: B, f: F) -> R where Self: Sized, F: FnMut(B, Self::Item) -> R, - R: Try, + R: Try, { - if self.is_empty() { - return Try::from_ok(init); - } - - let mut accum = init; - - while self.start < self.end { - let n = self.end.sub_one(); - let n = mem::replace(&mut self.end, n); - accum = f(accum, n)?; - } - - self.exhausted = true; - - if self.start == self.end { - accum = f(accum, self.start.clone())?; - } - - Try::from_ok(accum) + self.spec_try_rfold(init, f) } + + impl_fold_via_try_fold! { rfold -> try_rfold } } +// Safety: See above implementation for `ops::Range` +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for ops::RangeInclusive {} + #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for ops::RangeInclusive {} diff --git a/crux-mir/lib/core/src/iter/sources.rs b/crux-mir/lib/core/src/iter/sources.rs index a1d4e1b31..3ec426a3a 100644 --- a/crux-mir/lib/core/src/iter/sources.rs +++ b/crux-mir/lib/core/src/iter/sources.rs @@ -1,644 +1,40 @@ -use crate::fmt; -use crate::marker; -use crate::usize; - -use super::{FusedIterator, TrustedLen}; - -/// An iterator that repeats an element endlessly. -/// -/// This `struct` is created by the [`repeat`] function. See its documentation for more. -/// -/// [`repeat`]: fn.repeat.html -#[derive(Clone, Debug)] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct Repeat { - element: A, -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for Repeat { - type Item = A; - - #[inline] - fn next(&mut self) -> Option { - Some(self.element.clone()) - } - #[inline] - fn size_hint(&self) -> (usize, Option) { - (usize::MAX, None) - } -} +mod empty; +mod from_fn; +mod from_generator; +mod once; +mod once_with; +mod repeat; +mod repeat_n; +mod repeat_with; +mod successors; #[stable(feature = "rust1", since = "1.0.0")] -impl DoubleEndedIterator for Repeat { - #[inline] - fn next_back(&mut self) -> Option { - Some(self.element.clone()) - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Repeat {} - -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for Repeat {} - -/// Creates a new iterator that endlessly repeats a single element. -/// -/// The `repeat()` function repeats a single value over and over again. -/// -/// Infinite iterators like `repeat()` are often used with adapters like -/// [`take`], in order to make them finite. -/// -/// [`take`]: trait.Iterator.html#method.take -/// -/// If the element type of the iterator you need does not implement `Clone`, -/// or if you do not want to keep the repeated element in memory, you can -/// instead use the [`repeat_with`] function. -/// -/// [`repeat_with`]: fn.repeat_with.html -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// use std::iter; -/// -/// // the number four 4ever: -/// let mut fours = iter::repeat(4); -/// -/// assert_eq!(Some(4), fours.next()); -/// assert_eq!(Some(4), fours.next()); -/// assert_eq!(Some(4), fours.next()); -/// assert_eq!(Some(4), fours.next()); -/// assert_eq!(Some(4), fours.next()); -/// -/// // yup, still four -/// assert_eq!(Some(4), fours.next()); -/// ``` -/// -/// Going finite with [`take`]: -/// -/// ``` -/// use std::iter; -/// -/// // that last example was too many fours. Let's only have four fours. -/// let mut four_fours = iter::repeat(4).take(4); -/// -/// assert_eq!(Some(4), four_fours.next()); -/// assert_eq!(Some(4), four_fours.next()); -/// assert_eq!(Some(4), four_fours.next()); -/// assert_eq!(Some(4), four_fours.next()); -/// -/// // ... and now we're done -/// assert_eq!(None, four_fours.next()); -/// ``` -#[inline] -#[stable(feature = "rust1", since = "1.0.0")] -pub fn repeat(elt: T) -> Repeat { - Repeat { element: elt } -} - -/// An iterator that repeats elements of type `A` endlessly by -/// applying the provided closure `F: FnMut() -> A`. -/// -/// This `struct` is created by the [`repeat_with`] function. -/// See its documentation for more. -/// -/// [`repeat_with`]: fn.repeat_with.html -#[derive(Copy, Clone, Debug)] -#[stable(feature = "iterator_repeat_with", since = "1.28.0")] -pub struct RepeatWith { - repeater: F, -} - -#[stable(feature = "iterator_repeat_with", since = "1.28.0")] -impl A> Iterator for RepeatWith { - type Item = A; - - #[inline] - fn next(&mut self) -> Option { - Some((self.repeater)()) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - (usize::MAX, None) - } -} - -#[stable(feature = "iterator_repeat_with", since = "1.28.0")] -impl A> FusedIterator for RepeatWith {} - -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl A> TrustedLen for RepeatWith {} - -/// Creates a new iterator that repeats elements of type `A` endlessly by -/// applying the provided closure, the repeater, `F: FnMut() -> A`. -/// -/// The `repeat_with()` function calls the repeater over and over again. -/// -/// Infinite iterators like `repeat_with()` are often used with adapters like -/// [`take`], in order to make them finite. -/// -/// [`take`]: trait.Iterator.html#method.take -/// -/// If the element type of the iterator you need implements `Clone`, and -/// it is OK to keep the source element in memory, you should instead use -/// the [`repeat`] function. -/// -/// [`repeat`]: fn.repeat.html -/// -/// An iterator produced by `repeat_with()` is not a `DoubleEndedIterator`. -/// If you need `repeat_with()` to return a `DoubleEndedIterator`, -/// please open a GitHub issue explaining your use case. -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// use std::iter; -/// -/// // let's assume we have some value of a type that is not `Clone` -/// // or which don't want to have in memory just yet because it is expensive: -/// #[derive(PartialEq, Debug)] -/// struct Expensive; -/// -/// // a particular value forever: -/// let mut things = iter::repeat_with(|| Expensive); -/// -/// assert_eq!(Some(Expensive), things.next()); -/// assert_eq!(Some(Expensive), things.next()); -/// assert_eq!(Some(Expensive), things.next()); -/// assert_eq!(Some(Expensive), things.next()); -/// assert_eq!(Some(Expensive), things.next()); -/// ``` -/// -/// Using mutation and going finite: -/// -/// ```rust -/// use std::iter; -/// -/// // From the zeroth to the third power of two: -/// let mut curr = 1; -/// let mut pow2 = iter::repeat_with(|| { let tmp = curr; curr *= 2; tmp }) -/// .take(4); -/// -/// assert_eq!(Some(1), pow2.next()); -/// assert_eq!(Some(2), pow2.next()); -/// assert_eq!(Some(4), pow2.next()); -/// assert_eq!(Some(8), pow2.next()); -/// -/// // ... and now we're done -/// assert_eq!(None, pow2.next()); -/// ``` -#[inline] -#[stable(feature = "iterator_repeat_with", since = "1.28.0")] -pub fn repeat_with A>(repeater: F) -> RepeatWith { - RepeatWith { repeater } -} +pub use self::repeat::{repeat, Repeat}; -/// An iterator that yields nothing. -/// -/// This `struct` is created by the [`empty`] function. See its documentation for more. -/// -/// [`empty`]: fn.empty.html #[stable(feature = "iter_empty", since = "1.2.0")] -pub struct Empty(marker::PhantomData); - -#[stable(feature = "iter_empty_send_sync", since = "1.42.0")] -unsafe impl Send for Empty {} -#[stable(feature = "iter_empty_send_sync", since = "1.42.0")] -unsafe impl Sync for Empty {} - -#[stable(feature = "core_impl_debug", since = "1.9.0")] -impl fmt::Debug for Empty { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("Empty") - } -} - -#[stable(feature = "iter_empty", since = "1.2.0")] -impl Iterator for Empty { - type Item = T; - - fn next(&mut self) -> Option { - None - } - - fn size_hint(&self) -> (usize, Option) { - (0, Some(0)) - } -} - -#[stable(feature = "iter_empty", since = "1.2.0")] -impl DoubleEndedIterator for Empty { - fn next_back(&mut self) -> Option { - None - } -} - -#[stable(feature = "iter_empty", since = "1.2.0")] -impl ExactSizeIterator for Empty { - fn len(&self) -> usize { - 0 - } -} - -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for Empty {} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Empty {} - -// not #[derive] because that adds a Clone bound on T, -// which isn't necessary. -#[stable(feature = "iter_empty", since = "1.2.0")] -impl Clone for Empty { - fn clone(&self) -> Empty { - Empty(marker::PhantomData) - } -} - -// not #[derive] because that adds a Default bound on T, -// which isn't necessary. -#[stable(feature = "iter_empty", since = "1.2.0")] -impl Default for Empty { - fn default() -> Empty { - Empty(marker::PhantomData) - } -} - -/// Creates an iterator that yields nothing. -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// use std::iter; -/// -/// // this could have been an iterator over i32, but alas, it's just not. -/// let mut nope = iter::empty::(); -/// -/// assert_eq!(None, nope.next()); -/// ``` -#[stable(feature = "iter_empty", since = "1.2.0")] -#[rustc_const_stable(feature = "const_iter_empty", since = "1.32.0")] -pub const fn empty() -> Empty { - Empty(marker::PhantomData) -} - -/// An iterator that yields an element exactly once. -/// -/// This `struct` is created by the [`once`] function. See its documentation for more. -/// -/// [`once`]: fn.once.html -#[derive(Clone, Debug)] -#[stable(feature = "iter_once", since = "1.2.0")] -pub struct Once { - inner: crate::option::IntoIter, -} +pub use self::empty::{empty, Empty}; #[stable(feature = "iter_once", since = "1.2.0")] -impl Iterator for Once { - type Item = T; +pub use self::once::{once, Once}; - fn next(&mut self) -> Option { - self.inner.next() - } +#[unstable(feature = "iter_repeat_n", issue = "104434")] +pub use self::repeat_n::{repeat_n, RepeatN}; - fn size_hint(&self) -> (usize, Option) { - self.inner.size_hint() - } -} - -#[stable(feature = "iter_once", since = "1.2.0")] -impl DoubleEndedIterator for Once { - fn next_back(&mut self) -> Option { - self.inner.next_back() - } -} - -#[stable(feature = "iter_once", since = "1.2.0")] -impl ExactSizeIterator for Once { - fn len(&self) -> usize { - self.inner.len() - } -} - -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for Once {} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Once {} - -/// Creates an iterator that yields an element exactly once. -/// -/// This is commonly used to adapt a single value into a [`chain`] of other -/// kinds of iteration. Maybe you have an iterator that covers almost -/// everything, but you need an extra special case. Maybe you have a function -/// which works on iterators, but you only need to process one value. -/// -/// [`chain`]: trait.Iterator.html#method.chain -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// use std::iter; -/// -/// // one is the loneliest number -/// let mut one = iter::once(1); -/// -/// assert_eq!(Some(1), one.next()); -/// -/// // just one, that's all we get -/// assert_eq!(None, one.next()); -/// ``` -/// -/// Chaining together with another iterator. Let's say that we want to iterate -/// over each file of the `.foo` directory, but also a configuration file, -/// `.foorc`: -/// -/// ```no_run -/// use std::iter; -/// use std::fs; -/// use std::path::PathBuf; -/// -/// let dirs = fs::read_dir(".foo").unwrap(); -/// -/// // we need to convert from an iterator of DirEntry-s to an iterator of -/// // PathBufs, so we use map -/// let dirs = dirs.map(|file| file.unwrap().path()); -/// -/// // now, our iterator just for our config file -/// let config = iter::once(PathBuf::from(".foorc")); -/// -/// // chain the two iterators together into one big iterator -/// let files = dirs.chain(config); -/// -/// // this will give us all of the files in .foo as well as .foorc -/// for f in files { -/// println!("{:?}", f); -/// } -/// ``` -#[stable(feature = "iter_once", since = "1.2.0")] -pub fn once(value: T) -> Once { - Once { inner: Some(value).into_iter() } -} - -/// An iterator that yields a single element of type `A` by -/// applying the provided closure `F: FnOnce() -> A`. -/// -/// This `struct` is created by the [`once_with`] function. -/// See its documentation for more. -/// -/// [`once_with`]: fn.once_with.html -#[derive(Clone, Debug)] -#[stable(feature = "iter_once_with", since = "1.43.0")] -pub struct OnceWith { - gen: Option, -} - -#[stable(feature = "iter_once_with", since = "1.43.0")] -impl A> Iterator for OnceWith { - type Item = A; - - #[inline] - fn next(&mut self) -> Option { - let f = self.gen.take()?; - Some(f()) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.gen.iter().size_hint() - } -} - -#[stable(feature = "iter_once_with", since = "1.43.0")] -impl A> DoubleEndedIterator for OnceWith { - fn next_back(&mut self) -> Option { - self.next() - } -} - -#[stable(feature = "iter_once_with", since = "1.43.0")] -impl A> ExactSizeIterator for OnceWith { - fn len(&self) -> usize { - self.gen.iter().len() - } -} - -#[stable(feature = "iter_once_with", since = "1.43.0")] -impl A> FusedIterator for OnceWith {} - -#[stable(feature = "iter_once_with", since = "1.43.0")] -unsafe impl A> TrustedLen for OnceWith {} - -/// Creates an iterator that lazily generates a value exactly once by invoking -/// the provided closure. -/// -/// This is commonly used to adapt a single value generator into a [`chain`] of -/// other kinds of iteration. Maybe you have an iterator that covers almost -/// everything, but you need an extra special case. Maybe you have a function -/// which works on iterators, but you only need to process one value. -/// -/// Unlike [`once`], this function will lazily generate the value on request. -/// -/// [`once`]: fn.once.html -/// [`chain`]: trait.Iterator.html#method.chain -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// use std::iter; -/// -/// // one is the loneliest number -/// let mut one = iter::once_with(|| 1); -/// -/// assert_eq!(Some(1), one.next()); -/// -/// // just one, that's all we get -/// assert_eq!(None, one.next()); -/// ``` -/// -/// Chaining together with another iterator. Let's say that we want to iterate -/// over each file of the `.foo` directory, but also a configuration file, -/// `.foorc`: -/// -/// ```no_run -/// use std::iter; -/// use std::fs; -/// use std::path::PathBuf; -/// -/// let dirs = fs::read_dir(".foo").unwrap(); -/// -/// // we need to convert from an iterator of DirEntry-s to an iterator of -/// // PathBufs, so we use map -/// let dirs = dirs.map(|file| file.unwrap().path()); -/// -/// // now, our iterator just for our config file -/// let config = iter::once_with(|| PathBuf::from(".foorc")); -/// -/// // chain the two iterators together into one big iterator -/// let files = dirs.chain(config); -/// -/// // this will give us all of the files in .foo as well as .foorc -/// for f in files { -/// println!("{:?}", f); -/// } -/// ``` -#[inline] -#[stable(feature = "iter_once_with", since = "1.43.0")] -pub fn once_with A>(gen: F) -> OnceWith { - OnceWith { gen: Some(gen) } -} - -/// Creates a new iterator where each iteration calls the provided closure -/// `F: FnMut() -> Option`. -/// -/// This allows creating a custom iterator with any behavior -/// without using the more verbose syntax of creating a dedicated type -/// and implementing the `Iterator` trait for it. -/// -/// Note that the `FromFn` iterator doesn’t make assumptions about the behavior of the closure, -/// and therefore conservatively does not implement [`FusedIterator`], -/// or override [`Iterator::size_hint`] from its default `(0, None)`. -/// -/// [`FusedIterator`]: trait.FusedIterator.html -/// [`Iterator::size_hint`]: trait.Iterator.html#method.size_hint -/// -/// The closure can use captures and its environment to track state across iterations. Depending on -/// how the iterator is used, this may require specifying the `move` keyword on the closure. -/// -/// # Examples -/// -/// Let’s re-implement the counter iterator from [module-level documentation]: -/// -/// [module-level documentation]: index.html -/// -/// ``` -/// let mut count = 0; -/// let counter = std::iter::from_fn(move || { -/// // Increment our count. This is why we started at zero. -/// count += 1; -/// -/// // Check to see if we've finished counting or not. -/// if count < 6 { -/// Some(count) -/// } else { -/// None -/// } -/// }); -/// assert_eq!(counter.collect::>(), &[1, 2, 3, 4, 5]); -/// ``` -#[inline] -#[stable(feature = "iter_from_fn", since = "1.34.0")] -pub fn from_fn(f: F) -> FromFn -where - F: FnMut() -> Option, -{ - FromFn(f) -} - -/// An iterator where each iteration calls the provided closure `F: FnMut() -> Option`. -/// -/// This `struct` is created by the [`iter::from_fn`] function. -/// See its documentation for more. -/// -/// [`iter::from_fn`]: fn.from_fn.html -#[derive(Clone)] -#[stable(feature = "iter_from_fn", since = "1.34.0")] -pub struct FromFn(F); - -#[stable(feature = "iter_from_fn", since = "1.34.0")] -impl Iterator for FromFn -where - F: FnMut() -> Option, -{ - type Item = T; - - #[inline] - fn next(&mut self) -> Option { - (self.0)() - } -} +#[stable(feature = "iterator_repeat_with", since = "1.28.0")] +pub use self::repeat_with::{repeat_with, RepeatWith}; #[stable(feature = "iter_from_fn", since = "1.34.0")] -impl fmt::Debug for FromFn { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("FromFn").finish() - } -} - -/// Creates a new iterator where each successive item is computed based on the preceding one. -/// -/// The iterator starts with the given first item (if any) -/// and calls the given `FnMut(&T) -> Option` closure to compute each item’s successor. -/// -/// ``` -/// use std::iter::successors; -/// -/// let powers_of_10 = successors(Some(1_u16), |n| n.checked_mul(10)); -/// assert_eq!(powers_of_10.collect::>(), &[1, 10, 100, 1_000, 10_000]); -/// ``` -#[stable(feature = "iter_successors", since = "1.34.0")] -pub fn successors(first: Option, succ: F) -> Successors -where - F: FnMut(&T) -> Option, -{ - // If this function returned `impl Iterator` - // it could be based on `unfold` and not need a dedicated type. - // However having a named `Successors` type allows it to be `Clone` when `T` and `F` are. - Successors { next: first, succ } -} - -/// An new iterator where each successive item is computed based on the preceding one. -/// -/// This `struct` is created by the [`successors`] function. -/// See its documentation for more. -/// -/// [`successors`]: fn.successors.html -#[derive(Clone)] -#[stable(feature = "iter_successors", since = "1.34.0")] -pub struct Successors { - next: Option, - succ: F, -} +pub use self::from_fn::{from_fn, FromFn}; -#[stable(feature = "iter_successors", since = "1.34.0")] -impl Iterator for Successors -where - F: FnMut(&T) -> Option, -{ - type Item = T; - - #[inline] - fn next(&mut self) -> Option { - let item = self.next.take()?; - self.next = (self.succ)(&item); - Some(item) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - if self.next.is_some() { (1, None) } else { (0, Some(0)) } - } -} +#[unstable( + feature = "iter_from_generator", + issue = "43122", + reason = "generators are unstable" +)] +pub use self::from_generator::from_generator; #[stable(feature = "iter_successors", since = "1.34.0")] -impl FusedIterator for Successors where F: FnMut(&T) -> Option {} +pub use self::successors::{successors, Successors}; -#[stable(feature = "iter_successors", since = "1.34.0")] -impl fmt::Debug for Successors { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Successors").field("next", &self.next).finish() - } -} +#[stable(feature = "iter_once_with", since = "1.43.0")] +pub use self::once_with::{once_with, OnceWith}; diff --git a/crux-mir/lib/core/src/iter/sources/empty.rs b/crux-mir/lib/core/src/iter/sources/empty.rs new file mode 100644 index 000000000..617dfd123 --- /dev/null +++ b/crux-mir/lib/core/src/iter/sources/empty.rs @@ -0,0 +1,89 @@ +use crate::fmt; +use crate::iter::{FusedIterator, TrustedLen}; +use crate::marker; + +/// Creates an iterator that yields nothing. +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// use std::iter; +/// +/// // this could have been an iterator over i32, but alas, it's just not. +/// let mut nope = iter::empty::(); +/// +/// assert_eq!(None, nope.next()); +/// ``` +#[stable(feature = "iter_empty", since = "1.2.0")] +#[rustc_const_stable(feature = "const_iter_empty", since = "1.32.0")] +pub const fn empty() -> Empty { + Empty(marker::PhantomData) +} + +/// An iterator that yields nothing. +/// +/// This `struct` is created by the [`empty()`] function. See its documentation for more. +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "iter_empty", since = "1.2.0")] +pub struct Empty(marker::PhantomData T>); + +#[stable(feature = "core_impl_debug", since = "1.9.0")] +impl fmt::Debug for Empty { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Empty").finish() + } +} + +#[stable(feature = "iter_empty", since = "1.2.0")] +impl Iterator for Empty { + type Item = T; + + fn next(&mut self) -> Option { + None + } + + fn size_hint(&self) -> (usize, Option) { + (0, Some(0)) + } +} + +#[stable(feature = "iter_empty", since = "1.2.0")] +impl DoubleEndedIterator for Empty { + fn next_back(&mut self) -> Option { + None + } +} + +#[stable(feature = "iter_empty", since = "1.2.0")] +impl ExactSizeIterator for Empty { + fn len(&self) -> usize { + 0 + } +} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for Empty {} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Empty {} + +// not #[derive] because that adds a Clone bound on T, +// which isn't necessary. +#[stable(feature = "iter_empty", since = "1.2.0")] +impl Clone for Empty { + fn clone(&self) -> Empty { + Empty(marker::PhantomData) + } +} + +// not #[derive] because that adds a Default bound on T, +// which isn't necessary. +#[stable(feature = "iter_empty", since = "1.2.0")] +#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] +impl const Default for Empty { + fn default() -> Empty { + Empty(marker::PhantomData) + } +} diff --git a/crux-mir/lib/core/src/iter/sources/from_fn.rs b/crux-mir/lib/core/src/iter/sources/from_fn.rs new file mode 100644 index 000000000..3cd383047 --- /dev/null +++ b/crux-mir/lib/core/src/iter/sources/from_fn.rs @@ -0,0 +1,78 @@ +use crate::fmt; + +/// Creates a new iterator where each iteration calls the provided closure +/// `F: FnMut() -> Option`. +/// +/// This allows creating a custom iterator with any behavior +/// without using the more verbose syntax of creating a dedicated type +/// and implementing the [`Iterator`] trait for it. +/// +/// Note that the `FromFn` iterator doesn’t make assumptions about the behavior of the closure, +/// and therefore conservatively does not implement [`FusedIterator`], +/// or override [`Iterator::size_hint()`] from its default `(0, None)`. +/// +/// The closure can use captures and its environment to track state across iterations. Depending on +/// how the iterator is used, this may require specifying the [`move`] keyword on the closure. +/// +/// [`move`]: ../../std/keyword.move.html +/// [`FusedIterator`]: crate::iter::FusedIterator +/// +/// # Examples +/// +/// Let’s re-implement the counter iterator from [module-level documentation]: +/// +/// [module-level documentation]: crate::iter +/// +/// ``` +/// let mut count = 0; +/// let counter = std::iter::from_fn(move || { +/// // Increment our count. This is why we started at zero. +/// count += 1; +/// +/// // Check to see if we've finished counting or not. +/// if count < 6 { +/// Some(count) +/// } else { +/// None +/// } +/// }); +/// assert_eq!(counter.collect::>(), &[1, 2, 3, 4, 5]); +/// ``` +#[inline] +#[stable(feature = "iter_from_fn", since = "1.34.0")] +pub fn from_fn(f: F) -> FromFn +where + F: FnMut() -> Option, +{ + FromFn(f) +} + +/// An iterator where each iteration calls the provided closure `F: FnMut() -> Option`. +/// +/// This `struct` is created by the [`iter::from_fn()`] function. +/// See its documentation for more. +/// +/// [`iter::from_fn()`]: from_fn +#[derive(Clone)] +#[stable(feature = "iter_from_fn", since = "1.34.0")] +pub struct FromFn(F); + +#[stable(feature = "iter_from_fn", since = "1.34.0")] +impl Iterator for FromFn +where + F: FnMut() -> Option, +{ + type Item = T; + + #[inline] + fn next(&mut self) -> Option { + (self.0)() + } +} + +#[stable(feature = "iter_from_fn", since = "1.34.0")] +impl fmt::Debug for FromFn { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("FromFn").finish() + } +} diff --git a/crux-mir/lib/core/src/iter/sources/from_generator.rs b/crux-mir/lib/core/src/iter/sources/from_generator.rs new file mode 100644 index 000000000..4cbe731b2 --- /dev/null +++ b/crux-mir/lib/core/src/iter/sources/from_generator.rs @@ -0,0 +1,58 @@ +use crate::fmt; +use crate::ops::{Generator, GeneratorState}; +use crate::pin::Pin; + +/// Creates a new iterator where each iteration calls the provided generator. +/// +/// Similar to [`iter::from_fn`]. +/// +/// [`iter::from_fn`]: crate::iter::from_fn +/// +/// # Examples +/// +/// ``` +/// #![feature(generators)] +/// #![feature(iter_from_generator)] +/// +/// let it = std::iter::from_generator(|| { +/// yield 1; +/// yield 2; +/// yield 3; +/// }); +/// let v: Vec<_> = it.collect(); +/// assert_eq!(v, [1, 2, 3]); +/// ``` +#[inline] +#[unstable(feature = "iter_from_generator", issue = "43122", reason = "generators are unstable")] +pub fn from_generator + Unpin>(generator: G) -> FromGenerator { + FromGenerator(generator) +} + +/// An iterator over the values yielded by an underlying generator. +/// +/// This `struct` is created by the [`iter::from_generator()`] function. See its documentation for +/// more. +/// +/// [`iter::from_generator()`]: from_generator +#[unstable(feature = "iter_from_generator", issue = "43122", reason = "generators are unstable")] +#[derive(Clone)] +pub struct FromGenerator(G); + +#[unstable(feature = "iter_from_generator", issue = "43122", reason = "generators are unstable")] +impl + Unpin> Iterator for FromGenerator { + type Item = G::Yield; + + fn next(&mut self) -> Option { + match Pin::new(&mut self.0).resume(()) { + GeneratorState::Yielded(n) => Some(n), + GeneratorState::Complete(()) => None, + } + } +} + +#[unstable(feature = "iter_from_generator", issue = "43122", reason = "generators are unstable")] +impl fmt::Debug for FromGenerator { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("FromGenerator").finish() + } +} diff --git a/crux-mir/lib/core/src/iter/sources/once.rs b/crux-mir/lib/core/src/iter/sources/once.rs new file mode 100644 index 000000000..6e9ed0d3c --- /dev/null +++ b/crux-mir/lib/core/src/iter/sources/once.rs @@ -0,0 +1,99 @@ +use crate::iter::{FusedIterator, TrustedLen}; + +/// Creates an iterator that yields an element exactly once. +/// +/// This is commonly used to adapt a single value into a [`chain()`] of other +/// kinds of iteration. Maybe you have an iterator that covers almost +/// everything, but you need an extra special case. Maybe you have a function +/// which works on iterators, but you only need to process one value. +/// +/// [`chain()`]: Iterator::chain +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// use std::iter; +/// +/// // one is the loneliest number +/// let mut one = iter::once(1); +/// +/// assert_eq!(Some(1), one.next()); +/// +/// // just one, that's all we get +/// assert_eq!(None, one.next()); +/// ``` +/// +/// Chaining together with another iterator. Let's say that we want to iterate +/// over each file of the `.foo` directory, but also a configuration file, +/// `.foorc`: +/// +/// ```no_run +/// use std::iter; +/// use std::fs; +/// use std::path::PathBuf; +/// +/// let dirs = fs::read_dir(".foo").unwrap(); +/// +/// // we need to convert from an iterator of DirEntry-s to an iterator of +/// // PathBufs, so we use map +/// let dirs = dirs.map(|file| file.unwrap().path()); +/// +/// // now, our iterator just for our config file +/// let config = iter::once(PathBuf::from(".foorc")); +/// +/// // chain the two iterators together into one big iterator +/// let files = dirs.chain(config); +/// +/// // this will give us all of the files in .foo as well as .foorc +/// for f in files { +/// println!("{f:?}"); +/// } +/// ``` +#[stable(feature = "iter_once", since = "1.2.0")] +pub fn once(value: T) -> Once { + Once { inner: Some(value).into_iter() } +} + +/// An iterator that yields an element exactly once. +/// +/// This `struct` is created by the [`once()`] function. See its documentation for more. +#[derive(Clone, Debug)] +#[stable(feature = "iter_once", since = "1.2.0")] +pub struct Once { + inner: crate::option::IntoIter, +} + +#[stable(feature = "iter_once", since = "1.2.0")] +impl Iterator for Once { + type Item = T; + + fn next(&mut self) -> Option { + self.inner.next() + } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} + +#[stable(feature = "iter_once", since = "1.2.0")] +impl DoubleEndedIterator for Once { + fn next_back(&mut self) -> Option { + self.inner.next_back() + } +} + +#[stable(feature = "iter_once", since = "1.2.0")] +impl ExactSizeIterator for Once { + fn len(&self) -> usize { + self.inner.len() + } +} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for Once {} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Once {} diff --git a/crux-mir/lib/core/src/iter/sources/once_with.rs b/crux-mir/lib/core/src/iter/sources/once_with.rs new file mode 100644 index 000000000..080ae27a3 --- /dev/null +++ b/crux-mir/lib/core/src/iter/sources/once_with.rs @@ -0,0 +1,121 @@ +use crate::fmt; +use crate::iter::{FusedIterator, TrustedLen}; + +/// Creates an iterator that lazily generates a value exactly once by invoking +/// the provided closure. +/// +/// This is commonly used to adapt a single value generator into a [`chain()`] of +/// other kinds of iteration. Maybe you have an iterator that covers almost +/// everything, but you need an extra special case. Maybe you have a function +/// which works on iterators, but you only need to process one value. +/// +/// Unlike [`once()`], this function will lazily generate the value on request. +/// +/// [`chain()`]: Iterator::chain +/// [`once()`]: crate::iter::once +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// use std::iter; +/// +/// // one is the loneliest number +/// let mut one = iter::once_with(|| 1); +/// +/// assert_eq!(Some(1), one.next()); +/// +/// // just one, that's all we get +/// assert_eq!(None, one.next()); +/// ``` +/// +/// Chaining together with another iterator. Let's say that we want to iterate +/// over each file of the `.foo` directory, but also a configuration file, +/// `.foorc`: +/// +/// ```no_run +/// use std::iter; +/// use std::fs; +/// use std::path::PathBuf; +/// +/// let dirs = fs::read_dir(".foo").unwrap(); +/// +/// // we need to convert from an iterator of DirEntry-s to an iterator of +/// // PathBufs, so we use map +/// let dirs = dirs.map(|file| file.unwrap().path()); +/// +/// // now, our iterator just for our config file +/// let config = iter::once_with(|| PathBuf::from(".foorc")); +/// +/// // chain the two iterators together into one big iterator +/// let files = dirs.chain(config); +/// +/// // this will give us all of the files in .foo as well as .foorc +/// for f in files { +/// println!("{f:?}"); +/// } +/// ``` +#[inline] +#[stable(feature = "iter_once_with", since = "1.43.0")] +pub fn once_with A>(gen: F) -> OnceWith { + OnceWith { gen: Some(gen) } +} + +/// An iterator that yields a single element of type `A` by +/// applying the provided closure `F: FnOnce() -> A`. +/// +/// This `struct` is created by the [`once_with()`] function. +/// See its documentation for more. +#[derive(Clone)] +#[stable(feature = "iter_once_with", since = "1.43.0")] +pub struct OnceWith { + gen: Option, +} + +#[stable(feature = "iter_once_with_debug", since = "CURRENT_RUSTC_VERSION")] +impl fmt::Debug for OnceWith { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.gen.is_some() { + f.write_str("OnceWith(Some(_))") + } else { + f.write_str("OnceWith(None)") + } + } +} + +#[stable(feature = "iter_once_with", since = "1.43.0")] +impl A> Iterator for OnceWith { + type Item = A; + + #[inline] + fn next(&mut self) -> Option { + let f = self.gen.take()?; + Some(f()) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.gen.iter().size_hint() + } +} + +#[stable(feature = "iter_once_with", since = "1.43.0")] +impl A> DoubleEndedIterator for OnceWith { + fn next_back(&mut self) -> Option { + self.next() + } +} + +#[stable(feature = "iter_once_with", since = "1.43.0")] +impl A> ExactSizeIterator for OnceWith { + fn len(&self) -> usize { + self.gen.iter().len() + } +} + +#[stable(feature = "iter_once_with", since = "1.43.0")] +impl A> FusedIterator for OnceWith {} + +#[stable(feature = "iter_once_with", since = "1.43.0")] +unsafe impl A> TrustedLen for OnceWith {} diff --git a/crux-mir/lib/core/src/iter/sources/repeat.rs b/crux-mir/lib/core/src/iter/sources/repeat.rs new file mode 100644 index 000000000..733142ed0 --- /dev/null +++ b/crux-mir/lib/core/src/iter/sources/repeat.rs @@ -0,0 +1,129 @@ +use crate::iter::{FusedIterator, TrustedLen}; + +/// Creates a new iterator that endlessly repeats a single element. +/// +/// The `repeat()` function repeats a single value over and over again. +/// +/// Infinite iterators like `repeat()` are often used with adapters like +/// [`Iterator::take()`], in order to make them finite. +/// +/// If the element type of the iterator you need does not implement `Clone`, +/// or if you do not want to keep the repeated element in memory, you can +/// instead use the [`repeat_with()`] function. +/// +/// [`repeat_with()`]: crate::iter::repeat_with +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// use std::iter; +/// +/// // the number four 4ever: +/// let mut fours = iter::repeat(4); +/// +/// assert_eq!(Some(4), fours.next()); +/// assert_eq!(Some(4), fours.next()); +/// assert_eq!(Some(4), fours.next()); +/// assert_eq!(Some(4), fours.next()); +/// assert_eq!(Some(4), fours.next()); +/// +/// // yup, still four +/// assert_eq!(Some(4), fours.next()); +/// ``` +/// +/// Going finite with [`Iterator::take()`]: +/// +/// ``` +/// use std::iter; +/// +/// // that last example was too many fours. Let's only have four fours. +/// let mut four_fours = iter::repeat(4).take(4); +/// +/// assert_eq!(Some(4), four_fours.next()); +/// assert_eq!(Some(4), four_fours.next()); +/// assert_eq!(Some(4), four_fours.next()); +/// assert_eq!(Some(4), four_fours.next()); +/// +/// // ... and now we're done +/// assert_eq!(None, four_fours.next()); +/// ``` +#[inline] +#[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "iter_repeat")] +pub fn repeat(elt: T) -> Repeat { + Repeat { element: elt } +} + +/// An iterator that repeats an element endlessly. +/// +/// This `struct` is created by the [`repeat()`] function. See its documentation for more. +#[derive(Clone, Debug)] +#[stable(feature = "rust1", since = "1.0.0")] +pub struct Repeat { + element: A, +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for Repeat { + type Item = A; + + #[inline] + fn next(&mut self) -> Option { + Some(self.element.clone()) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + (usize::MAX, None) + } + + #[inline] + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + // Advancing an infinite iterator of a single element is a no-op. + let _ = n; + Ok(()) + } + + #[inline] + fn nth(&mut self, n: usize) -> Option { + let _ = n; + Some(self.element.clone()) + } + + fn last(self) -> Option { + loop {} + } + + fn count(self) -> usize { + loop {} + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl DoubleEndedIterator for Repeat { + #[inline] + fn next_back(&mut self) -> Option { + Some(self.element.clone()) + } + + #[inline] + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + // Advancing an infinite iterator of a single element is a no-op. + let _ = n; + Ok(()) + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option { + let _ = n; + Some(self.element.clone()) + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Repeat {} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for Repeat {} diff --git a/crux-mir/lib/core/src/iter/sources/repeat_n.rs b/crux-mir/lib/core/src/iter/sources/repeat_n.rs new file mode 100644 index 000000000..dc61d6065 --- /dev/null +++ b/crux-mir/lib/core/src/iter/sources/repeat_n.rs @@ -0,0 +1,195 @@ +use crate::iter::{FusedIterator, TrustedLen}; +use crate::mem::ManuallyDrop; + +/// Creates a new iterator that repeats a single element a given number of times. +/// +/// The `repeat_n()` function repeats a single value exactly `n` times. +/// +/// This is very similar to using [`repeat()`] with [`Iterator::take()`], +/// but there are two differences: +/// - `repeat_n()` can return the original value, rather than always cloning. +/// - `repeat_n()` produces an [`ExactSizeIterator`]. +/// +/// [`repeat()`]: crate::iter::repeat +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// #![feature(iter_repeat_n)] +/// use std::iter; +/// +/// // four of the number four: +/// let mut four_fours = iter::repeat_n(4, 4); +/// +/// assert_eq!(Some(4), four_fours.next()); +/// assert_eq!(Some(4), four_fours.next()); +/// assert_eq!(Some(4), four_fours.next()); +/// assert_eq!(Some(4), four_fours.next()); +/// +/// // no more fours +/// assert_eq!(None, four_fours.next()); +/// ``` +/// +/// For non-`Copy` types, +/// +/// ``` +/// #![feature(iter_repeat_n)] +/// use std::iter; +/// +/// let v: Vec = Vec::with_capacity(123); +/// let mut it = iter::repeat_n(v, 5); +/// +/// for i in 0..4 { +/// // It starts by cloning things +/// let cloned = it.next().unwrap(); +/// assert_eq!(cloned.len(), 0); +/// assert_eq!(cloned.capacity(), 0); +/// } +/// +/// // ... but the last item is the original one +/// let last = it.next().unwrap(); +/// assert_eq!(last.len(), 0); +/// assert_eq!(last.capacity(), 123); +/// +/// // ... and now we're done +/// assert_eq!(None, it.next()); +/// ``` +#[inline] +#[unstable(feature = "iter_repeat_n", issue = "104434")] +#[doc(hidden)] // waiting on ACP#120 to decide whether to expose publicly +pub fn repeat_n(element: T, count: usize) -> RepeatN { + let mut element = ManuallyDrop::new(element); + + if count == 0 { + // SAFETY: we definitely haven't dropped it yet, since we only just got + // passed it in, and because the count is zero the instance we're about + // to create won't drop it, so to avoid leaking we need to now. + unsafe { ManuallyDrop::drop(&mut element) }; + } + + RepeatN { element, count } +} + +/// An iterator that repeats an element an exact number of times. +/// +/// This `struct` is created by the [`repeat_n()`] function. +/// See its documentation for more. +#[derive(Clone, Debug)] +#[unstable(feature = "iter_repeat_n", issue = "104434")] +#[doc(hidden)] // waiting on ACP#120 to decide whether to expose publicly +pub struct RepeatN { + count: usize, + // Invariant: has been dropped iff count == 0. + element: ManuallyDrop, +} + +impl RepeatN { + /// If we haven't already dropped the element, return it in an option. + /// + /// Clears the count so it won't be dropped again later. + #[inline] + fn take_element(&mut self) -> Option { + if self.count > 0 { + self.count = 0; + // SAFETY: We just set count to zero so it won't be dropped again, + // and it used to be non-zero so it hasn't already been dropped. + unsafe { Some(ManuallyDrop::take(&mut self.element)) } + } else { + None + } + } +} + +#[unstable(feature = "iter_repeat_n", issue = "104434")] +impl Drop for RepeatN { + fn drop(&mut self) { + self.take_element(); + } +} + +#[unstable(feature = "iter_repeat_n", issue = "104434")] +impl Iterator for RepeatN { + type Item = A; + + #[inline] + fn next(&mut self) -> Option { + if self.count == 0 { + return None; + } + + self.count -= 1; + Some(if self.count == 0 { + // SAFETY: the check above ensured that the count used to be non-zero, + // so element hasn't been dropped yet, and we just lowered the count to + // zero so it won't be dropped later, and thus it's okay to take it here. + unsafe { ManuallyDrop::take(&mut self.element) } + } else { + A::clone(&self.element) + }) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let len = self.len(); + (len, Some(len)) + } + + #[inline] + fn advance_by(&mut self, skip: usize) -> Result<(), usize> { + let len = self.count; + + if skip >= len { + self.take_element(); + } + + if skip > len { + Err(len) + } else { + self.count = len - skip; + Ok(()) + } + } + + #[inline] + fn last(mut self) -> Option { + self.take_element() + } + + #[inline] + fn count(self) -> usize { + self.len() + } +} + +#[unstable(feature = "iter_repeat_n", issue = "104434")] +impl ExactSizeIterator for RepeatN { + fn len(&self) -> usize { + self.count + } +} + +#[unstable(feature = "iter_repeat_n", issue = "104434")] +impl DoubleEndedIterator for RepeatN { + #[inline] + fn next_back(&mut self) -> Option { + self.next() + } + + #[inline] + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + self.advance_by(n) + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option { + self.nth(n) + } +} + +#[unstable(feature = "iter_repeat_n", issue = "104434")] +impl FusedIterator for RepeatN {} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for RepeatN {} diff --git a/crux-mir/lib/core/src/iter/sources/repeat_with.rs b/crux-mir/lib/core/src/iter/sources/repeat_with.rs new file mode 100644 index 000000000..20420a3ad --- /dev/null +++ b/crux-mir/lib/core/src/iter/sources/repeat_with.rs @@ -0,0 +1,123 @@ +use crate::fmt; +use crate::iter::{FusedIterator, TrustedLen}; +use crate::ops::Try; + +/// Creates a new iterator that repeats elements of type `A` endlessly by +/// applying the provided closure, the repeater, `F: FnMut() -> A`. +/// +/// The `repeat_with()` function calls the repeater over and over again. +/// +/// Infinite iterators like `repeat_with()` are often used with adapters like +/// [`Iterator::take()`], in order to make them finite. +/// +/// If the element type of the iterator you need implements [`Clone`], and +/// it is OK to keep the source element in memory, you should instead use +/// the [`repeat()`] function. +/// +/// An iterator produced by `repeat_with()` is not a [`DoubleEndedIterator`]. +/// If you need `repeat_with()` to return a [`DoubleEndedIterator`], +/// please open a GitHub issue explaining your use case. +/// +/// [`repeat()`]: crate::iter::repeat +/// [`DoubleEndedIterator`]: crate::iter::DoubleEndedIterator +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// use std::iter; +/// +/// // let's assume we have some value of a type that is not `Clone` +/// // or which we don't want to have in memory just yet because it is expensive: +/// #[derive(PartialEq, Debug)] +/// struct Expensive; +/// +/// // a particular value forever: +/// let mut things = iter::repeat_with(|| Expensive); +/// +/// assert_eq!(Some(Expensive), things.next()); +/// assert_eq!(Some(Expensive), things.next()); +/// assert_eq!(Some(Expensive), things.next()); +/// assert_eq!(Some(Expensive), things.next()); +/// assert_eq!(Some(Expensive), things.next()); +/// ``` +/// +/// Using mutation and going finite: +/// +/// ```rust +/// use std::iter; +/// +/// // From the zeroth to the third power of two: +/// let mut curr = 1; +/// let mut pow2 = iter::repeat_with(|| { let tmp = curr; curr *= 2; tmp }) +/// .take(4); +/// +/// assert_eq!(Some(1), pow2.next()); +/// assert_eq!(Some(2), pow2.next()); +/// assert_eq!(Some(4), pow2.next()); +/// assert_eq!(Some(8), pow2.next()); +/// +/// // ... and now we're done +/// assert_eq!(None, pow2.next()); +/// ``` +#[inline] +#[stable(feature = "iterator_repeat_with", since = "1.28.0")] +pub fn repeat_with A>(repeater: F) -> RepeatWith { + RepeatWith { repeater } +} + +/// An iterator that repeats elements of type `A` endlessly by +/// applying the provided closure `F: FnMut() -> A`. +/// +/// This `struct` is created by the [`repeat_with()`] function. +/// See its documentation for more. +#[derive(Copy, Clone)] +#[stable(feature = "iterator_repeat_with", since = "1.28.0")] +pub struct RepeatWith { + repeater: F, +} + +#[stable(feature = "iterator_repeat_with_debug", since = "CURRENT_RUSTC_VERSION")] +impl fmt::Debug for RepeatWith { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RepeatWith").finish_non_exhaustive() + } +} + +#[stable(feature = "iterator_repeat_with", since = "1.28.0")] +impl A> Iterator for RepeatWith { + type Item = A; + + #[inline] + fn next(&mut self) -> Option { + Some((self.repeater)()) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + (usize::MAX, None) + } + + #[inline] + fn try_fold(&mut self, mut init: Acc, mut fold: Fold) -> R + where + Fold: FnMut(Acc, Self::Item) -> R, + R: Try, + { + // This override isn't strictly needed, but avoids the need to optimize + // away the `next`-always-returns-`Some` and emphasizes that the `?` + // is the only way to exit the loop. + + loop { + let item = (self.repeater)(); + init = fold(init, item)?; + } + } +} + +#[stable(feature = "iterator_repeat_with", since = "1.28.0")] +impl A> FusedIterator for RepeatWith {} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl A> TrustedLen for RepeatWith {} diff --git a/crux-mir/lib/core/src/iter/sources/successors.rs b/crux-mir/lib/core/src/iter/sources/successors.rs new file mode 100644 index 000000000..99f058a90 --- /dev/null +++ b/crux-mir/lib/core/src/iter/sources/successors.rs @@ -0,0 +1,66 @@ +use crate::{fmt, iter::FusedIterator}; + +/// Creates a new iterator where each successive item is computed based on the preceding one. +/// +/// The iterator starts with the given first item (if any) +/// and calls the given `FnMut(&T) -> Option` closure to compute each item’s successor. +/// +/// ``` +/// use std::iter::successors; +/// +/// let powers_of_10 = successors(Some(1_u16), |n| n.checked_mul(10)); +/// assert_eq!(powers_of_10.collect::>(), &[1, 10, 100, 1_000, 10_000]); +/// ``` +#[stable(feature = "iter_successors", since = "1.34.0")] +pub fn successors(first: Option, succ: F) -> Successors +where + F: FnMut(&T) -> Option, +{ + // If this function returned `impl Iterator` + // it could be based on `unfold` and not need a dedicated type. + // However having a named `Successors` type allows it to be `Clone` when `T` and `F` are. + Successors { next: first, succ } +} + +/// An new iterator where each successive item is computed based on the preceding one. +/// +/// This `struct` is created by the [`iter::successors()`] function. +/// See its documentation for more. +/// +/// [`iter::successors()`]: successors +#[derive(Clone)] +#[stable(feature = "iter_successors", since = "1.34.0")] +pub struct Successors { + next: Option, + succ: F, +} + +#[stable(feature = "iter_successors", since = "1.34.0")] +impl Iterator for Successors +where + F: FnMut(&T) -> Option, +{ + type Item = T; + + #[inline] + fn next(&mut self) -> Option { + let item = self.next.take()?; + self.next = (self.succ)(&item); + Some(item) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + if self.next.is_some() { (1, None) } else { (0, Some(0)) } + } +} + +#[stable(feature = "iter_successors", since = "1.34.0")] +impl FusedIterator for Successors where F: FnMut(&T) -> Option {} + +#[stable(feature = "iter_successors", since = "1.34.0")] +impl fmt::Debug for Successors { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Successors").field("next", &self.next).finish() + } +} diff --git a/crux-mir/lib/core/src/iter/traits/accum.rs b/crux-mir/lib/core/src/iter/traits/accum.rs index 55f30794a..e31669b39 100644 --- a/crux-mir/lib/core/src/iter/traits/accum.rs +++ b/crux-mir/lib/core/src/iter/traits/accum.rs @@ -1,18 +1,19 @@ use crate::iter; use crate::num::Wrapping; -use crate::ops::{Add, Mul}; /// Trait to represent types that can be created by summing up an iterator. /// -/// This trait is used to implement the [`sum`] method on iterators. Types which -/// implement the trait can be generated by the [`sum`] method. Like -/// [`FromIterator`] this trait should rarely be called directly and instead -/// interacted with through [`Iterator::sum`]. +/// This trait is used to implement [`Iterator::sum()`]. Types which implement +/// this trait can be generated by using the [`sum()`] method on an iterator. +/// Like [`FromIterator`], this trait should rarely be called directly. /// -/// [`sum`]: ../../std/iter/trait.Sum.html#tymethod.sum -/// [`FromIterator`]: ../../std/iter/trait.FromIterator.html -/// [`Iterator::sum`]: ../../std/iter/trait.Iterator.html#method.sum +/// [`sum()`]: Iterator::sum +/// [`FromIterator`]: iter::FromIterator #[stable(feature = "iter_arith_traits", since = "1.12.0")] +#[rustc_on_unimplemented( + message = "a value of type `{Self}` cannot be made by summing an iterator over elements of type `{A}`", + label = "value of type `{Self}` cannot be made by summing a `std::iter::Iterator`" +)] pub trait Sum: Sized { /// Method which takes an iterator and generates `Self` from the elements by /// "summing up" the items. @@ -23,15 +24,17 @@ pub trait Sum: Sized { /// Trait to represent types that can be created by multiplying elements of an /// iterator. /// -/// This trait is used to implement the [`product`] method on iterators. Types -/// which implement the trait can be generated by the [`product`] method. Like -/// [`FromIterator`] this trait should rarely be called directly and instead -/// interacted with through [`Iterator::product`]. +/// This trait is used to implement [`Iterator::product()`]. Types which implement +/// this trait can be generated by using the [`product()`] method on an iterator. +/// Like [`FromIterator`], this trait should rarely be called directly. /// -/// [`product`]: ../../std/iter/trait.Product.html#tymethod.product -/// [`FromIterator`]: ../../std/iter/trait.FromIterator.html -/// [`Iterator::product`]: ../../std/iter/trait.Iterator.html#method.product +/// [`product()`]: Iterator::product +/// [`FromIterator`]: iter::FromIterator #[stable(feature = "iter_arith_traits", since = "1.12.0")] +#[rustc_on_unimplemented( + message = "a value of type `{Self}` cannot be made by multiplying all elements of type `{A}` from an iterator", + label = "value of type `{Self}` cannot be made by multiplying all elements from a `std::iter::Iterator`" +)] pub trait Product: Sized { /// Method which takes an iterator and generates `Self` from the elements by /// multiplying the items. @@ -39,34 +42,49 @@ pub trait Product: Sized { fn product>(iter: I) -> Self; } -// N.B., explicitly use Add and Mul here to inherit overflow checks macro_rules! integer_sum_product { (@impls $zero:expr, $one:expr, #[$attr:meta], $($a:ty)*) => ($( #[$attr] impl Sum for $a { fn sum>(iter: I) -> Self { - iter.fold($zero, Add::add) + iter.fold( + $zero, + #[rustc_inherit_overflow_checks] + |a, b| a + b, + ) } } #[$attr] impl Product for $a { fn product>(iter: I) -> Self { - iter.fold($one, Mul::mul) + iter.fold( + $one, + #[rustc_inherit_overflow_checks] + |a, b| a * b, + ) } } #[$attr] impl<'a> Sum<&'a $a> for $a { fn sum>(iter: I) -> Self { - iter.fold($zero, Add::add) + iter.fold( + $zero, + #[rustc_inherit_overflow_checks] + |a, b| a + b, + ) } } #[$attr] impl<'a> Product<&'a $a> for $a { fn product>(iter: I) -> Self { - iter.fold($one, Mul::mul) + iter.fold( + $one, + #[rustc_inherit_overflow_checks] + |a, b| a * b, + ) } } )*); @@ -85,28 +103,44 @@ macro_rules! float_sum_product { #[stable(feature = "iter_arith_traits", since = "1.12.0")] impl Sum for $a { fn sum>(iter: I) -> Self { - iter.fold(0.0, Add::add) + iter.fold( + 0.0, + #[rustc_inherit_overflow_checks] + |a, b| a + b, + ) } } #[stable(feature = "iter_arith_traits", since = "1.12.0")] impl Product for $a { fn product>(iter: I) -> Self { - iter.fold(1.0, Mul::mul) + iter.fold( + 1.0, + #[rustc_inherit_overflow_checks] + |a, b| a * b, + ) } } #[stable(feature = "iter_arith_traits", since = "1.12.0")] impl<'a> Sum<&'a $a> for $a { fn sum>(iter: I) -> Self { - iter.fold(0.0, Add::add) + iter.fold( + 0.0, + #[rustc_inherit_overflow_checks] + |a, b| a + b, + ) } } #[stable(feature = "iter_arith_traits", since = "1.12.0")] impl<'a> Product<&'a $a> for $a { fn product>(iter: I) -> Self { - iter.fold(1.0, Mul::mul) + iter.fold( + 1.0, + #[rustc_inherit_overflow_checks] + |a, b| a * b, + ) } } )*) @@ -120,9 +154,9 @@ impl Sum> for Result where T: Sum, { - /// Takes each element in the `Iterator`: if it is an `Err`, no further - /// elements are taken, and the `Err` is returned. Should no `Err` occur, - /// the sum of all elements is returned. + /// Takes each element in the [`Iterator`]: if it is an [`Err`], no further + /// elements are taken, and the [`Err`] is returned. Should no [`Err`] + /// occur, the sum of all elements is returned. /// /// # Examples /// @@ -141,7 +175,7 @@ where where I: Iterator>, { - iter::process_results(iter, |i| i.sum()) + iter::try_process(iter, |i| i.sum()) } } @@ -150,14 +184,14 @@ impl Product> for Result where T: Product, { - /// Takes each element in the `Iterator`: if it is an `Err`, no further - /// elements are taken, and the `Err` is returned. Should no `Err` occur, - /// the product of all elements is returned. + /// Takes each element in the [`Iterator`]: if it is an [`Err`], no further + /// elements are taken, and the [`Err`] is returned. Should no [`Err`] + /// occur, the product of all elements is returned. fn product(iter: I) -> Result where I: Iterator>, { - iter::process_results(iter, |i| i.product()) + iter::try_process(iter, |i| i.product()) } } @@ -166,9 +200,9 @@ impl Sum> for Option where T: Sum, { - /// Takes each element in the `Iterator`: if it is a `None`, no further - /// elements are taken, and the `None` is returned. Should no `None` occur, - /// the sum of all elements is returned. + /// Takes each element in the [`Iterator`]: if it is a [`None`], no further + /// elements are taken, and the [`None`] is returned. Should no [`None`] + /// occur, the sum of all elements is returned. /// /// # Examples /// @@ -184,7 +218,7 @@ where where I: Iterator>, { - iter.map(|x| x.ok_or(())).sum::>().ok() + iter::try_process(iter, |i| i.sum()) } } @@ -193,13 +227,13 @@ impl Product> for Option where T: Product, { - /// Takes each element in the `Iterator`: if it is a `None`, no further - /// elements are taken, and the `None` is returned. Should no `None` occur, - /// the product of all elements is returned. + /// Takes each element in the [`Iterator`]: if it is a [`None`], no further + /// elements are taken, and the [`None`] is returned. Should no [`None`] + /// occur, the product of all elements is returned. fn product(iter: I) -> Option where I: Iterator>, { - iter.map(|x| x.ok_or(())).product::>().ok() + iter::try_process(iter, |i| i.product()) } } diff --git a/crux-mir/lib/core/src/iter/traits/collect.rs b/crux-mir/lib/core/src/iter/traits/collect.rs index f21ab8dbc..e099700e3 100644 --- a/crux-mir/lib/core/src/iter/traits/collect.rs +++ b/crux-mir/lib/core/src/iter/traits/collect.rs @@ -1,28 +1,22 @@ -/// Conversion from an `Iterator`. +/// Conversion from an [`Iterator`]. /// /// By implementing `FromIterator` for a type, you define how it will be /// created from an iterator. This is common for types which describe a /// collection of some kind. /// -/// `FromIterator`'s [`from_iter`] is rarely called explicitly, and is instead -/// used through [`Iterator`]'s [`collect`] method. See [`collect`]'s -/// documentation for more examples. -/// -/// [`from_iter`]: #tymethod.from_iter -/// [`Iterator`]: trait.Iterator.html -/// [`collect`]: trait.Iterator.html#method.collect +/// If you want to create a collection from the contents of an iterator, the +/// [`Iterator::collect()`] method is preferred. However, when you need to +/// specify the container type, [`FromIterator::from_iter()`] can be more +/// readable than using a turbofish (e.g. `::>()`). See the +/// [`Iterator::collect()`] documentation for more examples of its use. /// /// See also: [`IntoIterator`]. /// -/// [`IntoIterator`]: trait.IntoIterator.html -/// /// # Examples /// /// Basic usage: /// /// ``` -/// use std::iter::FromIterator; -/// /// let five_fives = std::iter::repeat(5).take(5); /// /// let v = Vec::from_iter(five_fives); @@ -30,7 +24,7 @@ /// assert_eq!(v, vec![5, 5, 5, 5, 5]); /// ``` /// -/// Using [`collect`] to implicitly use `FromIterator`: +/// Using [`Iterator::collect()`] to implicitly use `FromIterator`: /// /// ``` /// let five_fives = std::iter::repeat(5).take(5); @@ -40,11 +34,20 @@ /// assert_eq!(v, vec![5, 5, 5, 5, 5]); /// ``` /// -/// Implementing `FromIterator` for your type: +/// Using [`FromIterator::from_iter()`] as a more readable alternative to +/// [`Iterator::collect()`]: /// /// ``` -/// use std::iter::FromIterator; +/// use std::collections::VecDeque; +/// let first = (0..10).collect::>(); +/// let second = VecDeque::from_iter(0..10); /// +/// assert_eq!(first, second); +/// ``` +/// +/// Implementing `FromIterator` for your type: +/// +/// ``` /// // A sample collection, that's just a wrapper over Vec /// #[derive(Debug)] /// struct MyCollection(Vec); @@ -91,24 +94,43 @@ /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[rustc_on_unimplemented( + on( + _Self = "[{A}]", + message = "a slice of type `{Self}` cannot be built since `{Self}` has no definite size", + label = "try explicitly collecting into a `Vec<{A}>`", + ), + on( + all(A = "{integer}", any(_Self = "[{integral}]",)), + message = "a slice of type `{Self}` cannot be built since `{Self}` has no definite size", + label = "try explicitly collecting into a `Vec<{A}>`", + ), + on( + _Self = "[{A}; _]", + message = "an array of type `{Self}` cannot be built directly from an iterator", + label = "try collecting into a `Vec<{A}>`, then using `.try_into()`", + ), + on( + all(A = "{integer}", any(_Self = "[{integral}; _]",)), + message = "an array of type `{Self}` cannot be built directly from an iterator", + label = "try collecting into a `Vec<{A}>`, then using `.try_into()`", + ), message = "a value of type `{Self}` cannot be built from an iterator \ over elements of type `{A}`", label = "value of type `{Self}` cannot be built from `std::iter::Iterator`" )] +#[rustc_diagnostic_item = "FromIterator"] pub trait FromIterator: Sized { /// Creates a value from an iterator. /// /// See the [module-level documentation] for more. /// - /// [module-level documentation]: index.html + /// [module-level documentation]: crate::iter /// /// # Examples /// /// Basic usage: /// /// ``` - /// use std::iter::FromIterator; - /// /// let five_fives = std::iter::repeat(5).take(5); /// /// let v = Vec::from_iter(five_fives); @@ -119,25 +141,23 @@ pub trait FromIterator: Sized { fn from_iter>(iter: T) -> Self; } -/// Conversion into an `Iterator`. +/// Conversion into an [`Iterator`]. /// /// By implementing `IntoIterator` for a type, you define how it will be /// converted to an iterator. This is common for types which describe a /// collection of some kind. /// /// One benefit of implementing `IntoIterator` is that your type will [work -/// with Rust's `for` loop syntax](index.html#for-loops-and-intoiterator). +/// with Rust's `for` loop syntax](crate::iter#for-loops-and-intoiterator). /// /// See also: [`FromIterator`]. /// -/// [`FromIterator`]: trait.FromIterator.html -/// /// # Examples /// /// Basic usage: /// /// ``` -/// let v = vec![1, 2, 3]; +/// let v = [1, 2, 3]; /// let mut iter = v.into_iter(); /// /// assert_eq!(Some(1), iter.next()); @@ -201,12 +221,14 @@ pub trait FromIterator: Sized { /// { /// collection /// .into_iter() -/// .map(|item| format!("{:?}", item)) +/// .map(|item| format!("{item:?}")) /// .collect() /// } /// ``` #[rustc_diagnostic_item = "IntoIterator"] +#[rustc_skip_array_during_method_dispatch] #[stable(feature = "rust1", since = "1.0.0")] +#[const_trait] pub trait IntoIterator { /// The type of the elements being iterated over. #[stable(feature = "rust1", since = "1.0.0")] @@ -220,14 +242,14 @@ pub trait IntoIterator { /// /// See the [module-level documentation] for more. /// - /// [module-level documentation]: index.html + /// [module-level documentation]: crate::iter /// /// # Examples /// /// Basic usage: /// /// ``` - /// let v = vec![1, 2, 3]; + /// let v = [1, 2, 3]; /// let mut iter = v.into_iter(); /// /// assert_eq!(Some(1), iter.next()); @@ -235,15 +257,18 @@ pub trait IntoIterator { /// assert_eq!(Some(3), iter.next()); /// assert_eq!(None, iter.next()); /// ``` + #[lang = "into_iter"] #[stable(feature = "rust1", since = "1.0.0")] fn into_iter(self) -> Self::IntoIter; } +#[rustc_const_unstable(feature = "const_intoiterator_identity", issue = "90603")] #[stable(feature = "rust1", since = "1.0.0")] -impl IntoIterator for I { +impl const IntoIterator for I { type Item = I::Item; type IntoIter = I; + #[inline] fn into_iter(self) -> I { self } @@ -316,16 +341,16 @@ impl IntoIterator for I { /// c.extend(vec![1, 2, 3]); /// /// // we've added these elements onto the end -/// assert_eq!("MyCollection([5, 6, 7, 1, 2, 3])", format!("{:?}", c)); +/// assert_eq!("MyCollection([5, 6, 7, 1, 2, 3])", format!("{c:?}")); /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub trait Extend { /// Extends a collection with the contents of an iterator. /// - /// As this is the only method for this trait, the [trait-level] docs + /// As this is the only required method for this trait, the [trait-level] docs /// contain more details. /// - /// [trait-level]: trait.Extend.html + /// [trait-level]: Extend /// /// # Examples /// @@ -341,6 +366,20 @@ pub trait Extend { /// ``` #[stable(feature = "rust1", since = "1.0.0")] fn extend>(&mut self, iter: T); + + /// Extends a collection with exactly one element. + #[unstable(feature = "extend_one", issue = "72631")] + fn extend_one(&mut self, item: A) { + self.extend(Some(item)); + } + + /// Reserves capacity in a collection for the given number of additional elements. + /// + /// The default implementation does nothing. + #[unstable(feature = "extend_one", issue = "72631")] + fn extend_reserve(&mut self, additional: usize) { + let _ = additional; + } } #[stable(feature = "extend_for_unit", since = "1.28.0")] @@ -348,4 +387,65 @@ impl Extend<()> for () { fn extend>(&mut self, iter: T) { iter.into_iter().for_each(drop) } + fn extend_one(&mut self, _item: ()) {} +} + +#[stable(feature = "extend_for_tuple", since = "1.56.0")] +impl Extend<(A, B)> for (ExtendA, ExtendB) +where + ExtendA: Extend, + ExtendB: Extend, +{ + /// Allows to `extend` a tuple of collections that also implement `Extend`. + /// + /// See also: [`Iterator::unzip`] + /// + /// # Examples + /// ``` + /// let mut tuple = (vec![0], vec![1]); + /// tuple.extend([(2, 3), (4, 5), (6, 7)]); + /// assert_eq!(tuple.0, [0, 2, 4, 6]); + /// assert_eq!(tuple.1, [1, 3, 5, 7]); + /// + /// // also allows for arbitrarily nested tuples as elements + /// let mut nested_tuple = (vec![1], (vec![2], vec![3])); + /// nested_tuple.extend([(4, (5, 6)), (7, (8, 9))]); + /// + /// let (a, (b, c)) = nested_tuple; + /// assert_eq!(a, [1, 4, 7]); + /// assert_eq!(b, [2, 5, 8]); + /// assert_eq!(c, [3, 6, 9]); + /// ``` + fn extend>(&mut self, into_iter: T) { + let (a, b) = self; + let iter = into_iter.into_iter(); + + fn extend<'a, A, B>( + a: &'a mut impl Extend, + b: &'a mut impl Extend, + ) -> impl FnMut((), (A, B)) + 'a { + move |(), (t, u)| { + a.extend_one(t); + b.extend_one(u); + } + } + + let (lower_bound, _) = iter.size_hint(); + if lower_bound > 0 { + a.extend_reserve(lower_bound); + b.extend_reserve(lower_bound); + } + + iter.fold((), extend(a, b)); + } + + fn extend_one(&mut self, item: (A, B)) { + self.0.extend_one(item.0); + self.1.extend_one(item.1); + } + + fn extend_reserve(&mut self, additional: usize) { + self.0.extend_reserve(additional); + self.1.extend_reserve(additional); + } } diff --git a/crux-mir/lib/core/src/iter/traits/double_ended.rs b/crux-mir/lib/core/src/iter/traits/double_ended.rs index 104724d9f..bdf94c792 100644 --- a/crux-mir/lib/core/src/iter/traits/double_ended.rs +++ b/crux-mir/lib/core/src/iter/traits/double_ended.rs @@ -1,5 +1,4 @@ -use crate::iter::LoopState; -use crate::ops::Try; +use crate::ops::{ControlFlow, Try}; /// An iterator able to yield elements from both ends. /// @@ -11,11 +10,12 @@ use crate::ops::Try; /// and do not cross: iteration is over when they meet in the middle. /// /// In a similar fashion to the [`Iterator`] protocol, once a -/// `DoubleEndedIterator` returns `None` from a `next_back()`, calling it again -/// may or may not ever return `Some` again. `next()` and `next_back()` are -/// interchangeable for this purpose. +/// `DoubleEndedIterator` returns [`None`] from a [`next_back()`], calling it +/// again may or may not ever return [`Some`] again. [`next()`] and +/// [`next_back()`] are interchangeable for this purpose. /// -/// [`Iterator`]: trait.Iterator.html +/// [`next_back()`]: DoubleEndedIterator::next_back +/// [`next()`]: Iterator::next /// /// # Examples /// @@ -36,6 +36,7 @@ use crate::ops::Try; /// assert_eq!(None, iter.next_back()); /// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "DoubleEndedIterator")] pub trait DoubleEndedIterator: Iterator { /// Removes and returns an element from the end of the iterator. /// @@ -43,7 +44,7 @@ pub trait DoubleEndedIterator: Iterator { /// /// The [trait-level] docs contain more details. /// - /// [trait-level]: trait.DoubleEndedIterator.html + /// [trait-level]: DoubleEndedIterator /// /// # Examples /// @@ -63,25 +64,94 @@ pub trait DoubleEndedIterator: Iterator { /// assert_eq!(None, iter.next()); /// assert_eq!(None, iter.next_back()); /// ``` + /// + /// # Remarks + /// + /// The elements yielded by `DoubleEndedIterator`'s methods may differ from + /// the ones yielded by [`Iterator`]'s methods: + /// + /// ``` + /// let vec = vec![(1, 'a'), (1, 'b'), (1, 'c'), (2, 'a'), (2, 'b')]; + /// let uniq_by_fst_comp = || { + /// let mut seen = std::collections::HashSet::new(); + /// vec.iter().copied().filter(move |x| seen.insert(x.0)) + /// }; + /// + /// assert_eq!(uniq_by_fst_comp().last(), Some((2, 'a'))); + /// assert_eq!(uniq_by_fst_comp().next_back(), Some((2, 'b'))); + /// + /// assert_eq!( + /// uniq_by_fst_comp().fold(vec![], |mut v, x| {v.push(x); v}), + /// vec![(1, 'a'), (2, 'a')] + /// ); + /// assert_eq!( + /// uniq_by_fst_comp().rfold(vec![], |mut v, x| {v.push(x); v}), + /// vec![(2, 'b'), (1, 'c')] + /// ); + /// ``` #[stable(feature = "rust1", since = "1.0.0")] fn next_back(&mut self) -> Option; + /// Advances the iterator from the back by `n` elements. + /// + /// `advance_back_by` is the reverse version of [`advance_by`]. This method will + /// eagerly skip `n` elements starting from the back by calling [`next_back`] up + /// to `n` times until [`None`] is encountered. + /// + /// `advance_back_by(n)` will return [`Ok(())`] if the iterator successfully advances by + /// `n` elements, or [`Err(k)`] if [`None`] is encountered, where `k` is the number of + /// elements the iterator is advanced by before running out of elements (i.e. the length + /// of the iterator). Note that `k` is always less than `n`. + /// + /// Calling `advance_back_by(0)` can do meaningful work, for example [`Flatten`] can advance its + /// outer iterator until it finds an inner iterator that is not empty, which then often + /// allows it to return a more accurate `size_hint()` than in its initial state. + /// + /// [`advance_by`]: Iterator::advance_by + /// [`Flatten`]: crate::iter::Flatten + /// [`next_back`]: DoubleEndedIterator::next_back + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(iter_advance_by)] + /// + /// let a = [3, 4, 5, 6]; + /// let mut iter = a.iter(); + /// + /// assert_eq!(iter.advance_back_by(2), Ok(())); + /// assert_eq!(iter.next_back(), Some(&4)); + /// assert_eq!(iter.advance_back_by(0), Ok(())); + /// assert_eq!(iter.advance_back_by(100), Err(1)); // only `&3` was skipped + /// ``` + /// + /// [`Ok(())`]: Ok + /// [`Err(k)`]: Err + #[inline] + #[unstable(feature = "iter_advance_by", reason = "recently added", issue = "77404")] + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + for i in 0..n { + self.next_back().ok_or(i)?; + } + Ok(()) + } + /// Returns the `n`th element from the end of the iterator. /// - /// This is essentially the reversed version of [`nth`]. Although like most indexing - /// operations, the count starts from zero, so `nth_back(0)` returns the first value from - /// the end, `nth_back(1)` the second, and so on. + /// This is essentially the reversed version of [`Iterator::nth()`]. + /// Although like most indexing operations, the count starts from zero, so + /// `nth_back(0)` returns the first value from the end, `nth_back(1)` the + /// second, and so on. /// /// Note that all elements between the end and the returned element will be /// consumed, including the returned element. This also means that calling /// `nth_back(0)` multiple times on the same iterator will return different /// elements. /// - /// `nth_back()` will return [`None`] if `n` is greater than or equal to the length of the - /// iterator. - /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// [`nth`]: ../../std/iter/trait.Iterator.html#method.nth + /// `nth_back()` will return [`None`] if `n` is greater than or equal to the + /// length of the iterator. /// /// # Examples /// @@ -111,20 +181,13 @@ pub trait DoubleEndedIterator: Iterator { /// ``` #[inline] #[stable(feature = "iter_nth_back", since = "1.37.0")] - fn nth_back(&mut self, mut n: usize) -> Option { - for x in self.rev() { - if n == 0 { - return Some(x); - } - n -= 1; - } - None + fn nth_back(&mut self, n: usize) -> Option { + self.advance_back_by(n).ok()?; + self.next_back() } - /// This is the reverse version of [`try_fold()`]: it takes elements - /// starting from the back of the iterator. - /// - /// [`try_fold()`]: trait.Iterator.html#method.try_fold + /// This is the reverse version of [`Iterator::try_fold()`]: it takes + /// elements starting from the back of the iterator. /// /// # Examples /// @@ -159,20 +222,20 @@ pub trait DoubleEndedIterator: Iterator { where Self: Sized, F: FnMut(B, Self::Item) -> R, - R: Try, + R: Try, { let mut accum = init; while let Some(x) = self.next_back() { accum = f(accum, x)?; } - Try::from_ok(accum) + try { accum } } /// An iterator method that reduces the iterator's elements to a single, /// final value, starting from the back. /// - /// This is the reverse version of [`fold()`]: it takes elements starting from - /// the back of the iterator. + /// This is the reverse version of [`Iterator::fold()`]: it takes elements + /// starting from the back of the iterator. /// /// `rfold()` takes two arguments: an initial value, and a closure with two /// arguments: an 'accumulator', and an element. The closure returns the value that @@ -189,7 +252,10 @@ pub trait DoubleEndedIterator: Iterator { /// Folding is useful whenever you have a collection of something, and want /// to produce a single value from it. /// - /// [`fold()`]: trait.Iterator.html#method.fold + /// Note: `rfold()` combines elements in a *right-associative* fashion. For associative + /// operators like `+`, the order the elements are combined in is not important, but for non-associative + /// operators like `-` the order will affect the final result. + /// For a *left-associative* version of `rfold()`, see [`Iterator::fold()`]. /// /// # Examples /// @@ -205,7 +271,8 @@ pub trait DoubleEndedIterator: Iterator { /// assert_eq!(sum, 6); /// ``` /// - /// This example builds a string, starting with an initial value + /// This example demonstrates the right-associative nature of `rfold()`: + /// it builds a string, starting with an initial value /// and continuing with each element from the back until the front: /// /// ``` @@ -214,24 +281,24 @@ pub trait DoubleEndedIterator: Iterator { /// let zero = "0".to_string(); /// /// let result = numbers.iter().rfold(zero, |acc, &x| { - /// format!("({} + {})", x, acc) + /// format!("({x} + {acc})") /// }); /// /// assert_eq!(result, "(1 + (2 + (3 + (4 + (5 + 0)))))"); /// ``` + #[doc(alias = "foldr")] #[inline] #[stable(feature = "iter_rfold", since = "1.27.0")] - fn rfold(mut self, accum: B, f: F) -> B + fn rfold(mut self, init: B, mut f: F) -> B where Self: Sized, F: FnMut(B, Self::Item) -> B, { - #[inline] - fn ok(mut f: impl FnMut(B, T) -> B) -> impl FnMut(B, T) -> Result { - move |acc, x| Ok(f(acc, x)) + let mut accum = init; + while let Some(x) = self.next_back() { + accum = f(accum, x); } - - self.try_rfold(accum, ok(f)).unwrap() + accum } /// Searches for an element of an iterator from the back that satisfies a predicate. @@ -249,8 +316,7 @@ pub trait DoubleEndedIterator: Iterator { /// argument is a double reference. You can see this effect in the /// examples below, with `&&x`. /// - /// [`Some(element)`]: ../../std/option/enum.Option.html#variant.Some - /// [`None`]: ../../std/option/enum.Option.html#variant.None + /// [`Some(element)`]: Some /// /// # Examples /// @@ -284,11 +350,9 @@ pub trait DoubleEndedIterator: Iterator { P: FnMut(&Self::Item) -> bool, { #[inline] - fn check( - mut predicate: impl FnMut(&T) -> bool, - ) -> impl FnMut((), T) -> LoopState<(), T> { + fn check(mut predicate: impl FnMut(&T) -> bool) -> impl FnMut((), T) -> ControlFlow { move |(), x| { - if predicate(&x) { LoopState::Break(x) } else { LoopState::Continue(()) } + if predicate(&x) { ControlFlow::Break(x) } else { ControlFlow::CONTINUE } } } @@ -301,6 +365,9 @@ impl<'a, I: DoubleEndedIterator + ?Sized> DoubleEndedIterator for &'a mut I { fn next_back(&mut self) -> Option { (**self).next_back() } + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + (**self).advance_back_by(n) + } fn nth_back(&mut self, n: usize) -> Option { (**self).nth_back(n) } diff --git a/crux-mir/lib/core/src/iter/traits/exact_size.rs b/crux-mir/lib/core/src/iter/traits/exact_size.rs index ad87d0958..1757e37ec 100644 --- a/crux-mir/lib/core/src/iter/traits/exact_size.rs +++ b/crux-mir/lib/core/src/iter/traits/exact_size.rs @@ -6,17 +6,20 @@ /// backwards, a good start is to know where the end is. /// /// When implementing an `ExactSizeIterator`, you must also implement -/// [`Iterator`]. When doing so, the implementation of [`size_hint`] *must* -/// return the exact size of the iterator. -/// -/// [`Iterator`]: trait.Iterator.html -/// [`size_hint`]: trait.Iterator.html#method.size_hint +/// [`Iterator`]. When doing so, the implementation of [`Iterator::size_hint`] +/// *must* return the exact size of the iterator. /// /// The [`len`] method has a default implementation, so you usually shouldn't /// implement it. However, you may be able to provide a more performant /// implementation than the default, so overriding it in this case makes sense. /// -/// [`len`]: #method.len +/// Note that this trait is a safe trait and as such does *not* and *cannot* +/// guarantee that the returned length is correct. This means that `unsafe` +/// code **must not** rely on the correctness of [`Iterator::size_hint`]. The +/// unstable and unsafe [`TrustedLen`](super::marker::TrustedLen) trait gives +/// this additional guarantee. +/// +/// [`len`]: ExactSizeIterator::len /// /// # Examples /// @@ -29,10 +32,10 @@ /// assert_eq!(5, five.len()); /// ``` /// -/// In the [module level docs][moddocs], we implemented an [`Iterator`], -/// `Counter`. Let's implement `ExactSizeIterator` for it as well: +/// In the [module-level docs], we implemented an [`Iterator`], `Counter`. +/// Let's implement `ExactSizeIterator` for it as well: /// -/// [moddocs]: index.html +/// [module-level docs]: crate::iter /// /// ``` /// # struct Counter { @@ -63,26 +66,28 @@ /// /// // And now we can use it! /// -/// let counter = Counter::new(); +/// let mut counter = Counter::new(); /// /// assert_eq!(5, counter.len()); +/// let _ = counter.next(); +/// assert_eq!(4, counter.len()); /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub trait ExactSizeIterator: Iterator { - /// Returns the exact length of the iterator. + /// Returns the exact remaining length of the iterator. /// /// The implementation ensures that the iterator will return exactly `len()` - /// more times a `Some(T)` value, before returning `None`. + /// more times a [`Some(T)`] value, before returning [`None`]. /// This method has a default implementation, so you usually should not /// implement it directly. However, if you can provide a more efficient /// implementation, you can do so. See the [trait-level] docs for an /// example. /// - /// This function has the same safety guarantees as the [`size_hint`] - /// function. + /// This function has the same safety guarantees as the + /// [`Iterator::size_hint`] function. /// - /// [trait-level]: trait.ExactSizeIterator.html - /// [`size_hint`]: trait.Iterator.html#method.size_hint + /// [trait-level]: ExactSizeIterator + /// [`Some(T)`]: Some /// /// # Examples /// @@ -90,9 +95,11 @@ pub trait ExactSizeIterator: Iterator { /// /// ``` /// // a finite range knows exactly how many times it will iterate - /// let five = 0..5; + /// let mut range = 0..5; /// - /// assert_eq!(5, five.len()); + /// assert_eq!(5, range.len()); + /// let _ = range.next(); + /// assert_eq!(4, range.len()); /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] @@ -108,8 +115,8 @@ pub trait ExactSizeIterator: Iterator { /// Returns `true` if the iterator is empty. /// - /// This method has a default implementation using `self.len()`, so you - /// don't need to implement it yourself. + /// This method has a default implementation using + /// [`ExactSizeIterator::len()`], so you don't need to implement it yourself. /// /// # Examples /// diff --git a/crux-mir/lib/core/src/iter/traits/iterator.rs b/crux-mir/lib/core/src/iter/traits/iterator.rs index e2ebef9c6..a4a665d48 100644 --- a/crux-mir/lib/core/src/iter/traits/iterator.rs +++ b/crux-mir/lib/core/src/iter/traits/iterator.rs @@ -1,64 +1,29 @@ -// ignore-tidy-filelength -// This file almost exclusively consists of the definition of `Iterator`. We -// can't split that into multiple files. - +use crate::array; use crate::cmp::{self, Ordering}; -use crate::ops::{Add, Try}; +use crate::ops::{ChangeOutputType, ControlFlow, FromResidual, Residual, Try}; -use super::super::LoopState; -use super::super::{Chain, Cloned, Copied, Cycle, Enumerate, Filter, FilterMap, Fuse}; +use super::super::try_process; +use super::super::ByRefSized; +use super::super::TrustedRandomAccessNoCoerce; +use super::super::{ArrayChunks, Chain, Cloned, Copied, Cycle, Enumerate, Filter, FilterMap, Fuse}; use super::super::{FlatMap, Flatten}; -use super::super::{FromIterator, Product, Sum, Zip}; +use super::super::{FromIterator, Intersperse, IntersperseWith, Product, Sum, Zip}; use super::super::{ Inspect, Map, MapWhile, Peekable, Rev, Scan, Skip, SkipWhile, StepBy, Take, TakeWhile, }; fn _assert_is_object_safe(_: &dyn Iterator) {} -/// An interface for dealing with iterators. +/// A trait for dealing with iterators. /// /// This is the main iterator trait. For more about the concept of iterators /// generally, please see the [module-level documentation]. In particular, you /// may want to know how to [implement `Iterator`][impl]. /// -/// [module-level documentation]: index.html -/// [impl]: index.html#implementing-iterator +/// [module-level documentation]: crate::iter +/// [impl]: crate::iter#implementing-iterator #[stable(feature = "rust1", since = "1.0.0")] #[rustc_on_unimplemented( - on( - _Self = "[std::ops::Range; 1]", - label = "if you meant to iterate between two values, remove the square brackets", - note = "`[start..end]` is an array of one `Range`; you might have meant to have a `Range` \ - without the brackets: `start..end`" - ), - on( - _Self = "[std::ops::RangeFrom; 1]", - label = "if you meant to iterate from a value onwards, remove the square brackets", - note = "`[start..]` is an array of one `RangeFrom`; you might have meant to have a \ - `RangeFrom` without the brackets: `start..`, keeping in mind that iterating over an \ - unbounded iterator will run forever unless you `break` or `return` from within the \ - loop" - ), - on( - _Self = "[std::ops::RangeTo; 1]", - label = "if you meant to iterate until a value, remove the square brackets and add a \ - starting value", - note = "`[..end]` is an array of one `RangeTo`; you might have meant to have a bounded \ - `Range` without the brackets: `0..end`" - ), - on( - _Self = "[std::ops::RangeInclusive; 1]", - label = "if you meant to iterate between two values, remove the square brackets", - note = "`[start..=end]` is an array of one `RangeInclusive`; you might have meant to have a \ - `RangeInclusive` without the brackets: `start..=end`" - ), - on( - _Self = "[std::ops::RangeToInclusive; 1]", - label = "if you meant to iterate until a value (including it), remove the square brackets \ - and add a starting value", - note = "`[..=end]` is an array of one `RangeToInclusive`; you might have meant to have a \ - bounded `RangeInclusive` without the brackets: `0..=end`" - ), on( _Self = "std::ops::RangeTo", label = "if you meant to iterate until a value, add a starting value", @@ -71,6 +36,15 @@ fn _assert_is_object_safe(_: &dyn Iterator) {} note = "`..=end` is a `RangeToInclusive`, which cannot be iterated on; you might have meant \ to have a bounded `RangeInclusive`: `0..=end`" ), + on( + _Self = "[]", + label = "`{Self}` is not an iterator; try calling `.into_iter()` or `.iter()`" + ), + on(_Self = "&[]", label = "`{Self}` is not an iterator; try calling `.iter()`"), + on( + _Self = "std::vec::Vec", + label = "`{Self}` is not an iterator; try calling `.into_iter()` or `.iter()`" + ), on( _Self = "&str", label = "`{Self}` is not an iterator; try calling `.chars()` or `.bytes()`" @@ -80,21 +54,24 @@ fn _assert_is_object_safe(_: &dyn Iterator) {} label = "`{Self}` is not an iterator; try calling `.chars()` or `.bytes()`" ), on( - _Self = "[]", - label = "borrow the array with `&` or call `.iter()` on it to iterate over it", - note = "arrays are not iterators, but slices like the following are: `&[1, 2, 3]`" + _Self = "{integral}", + note = "if you want to iterate between `start` until a value `end`, use the exclusive range \ + syntax `start..end` or the inclusive range syntax `start..=end`" ), on( - _Self = "{integral}", + _Self = "{float}", note = "if you want to iterate between `start` until a value `end`, use the exclusive range \ syntax `start..end` or the inclusive range syntax `start..=end`" ), label = "`{Self}` is not an iterator", message = "`{Self}` is not an iterator" )] +#[doc(notable_trait)] +#[rustc_diagnostic_item = "Iterator"] #[must_use = "iterators are lazy and do nothing unless consumed"] pub trait Iterator { /// The type of the elements being iterated over. + #[rustc_diagnostic_item = "IteratorItem"] #[stable(feature = "rust1", since = "1.0.0")] type Item; @@ -105,8 +82,7 @@ pub trait Iterator { /// again may or may not eventually start returning [`Some(Item)`] again at some /// point. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// [`Some(Item)`]: ../../std/option/enum.Option.html#variant.Some + /// [`Some(Item)`]: Some /// /// # Examples /// @@ -129,15 +105,57 @@ pub trait Iterator { /// assert_eq!(None, iter.next()); /// assert_eq!(None, iter.next()); /// ``` + #[lang = "next"] #[stable(feature = "rust1", since = "1.0.0")] fn next(&mut self) -> Option; + /// Advances the iterator and returns an array containing the next `N` values. + /// + /// If there are not enough elements to fill the array then `Err` is returned + /// containing an iterator over the remaining elements. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(iter_next_chunk)] + /// + /// let mut iter = "lorem".chars(); + /// + /// assert_eq!(iter.next_chunk().unwrap(), ['l', 'o']); // N is inferred as 2 + /// assert_eq!(iter.next_chunk().unwrap(), ['r', 'e', 'm']); // N is inferred as 3 + /// assert_eq!(iter.next_chunk::<4>().unwrap_err().as_slice(), &[]); // N is explicitly 4 + /// ``` + /// + /// Split a string and get the first three items. + /// + /// ``` + /// #![feature(iter_next_chunk)] + /// + /// let quote = "not all those who wander are lost"; + /// let [first, second, third] = quote.split_whitespace().next_chunk().unwrap(); + /// assert_eq!(first, "not"); + /// assert_eq!(second, "all"); + /// assert_eq!(third, "those"); + /// ``` + #[inline] + #[unstable(feature = "iter_next_chunk", reason = "recently added", issue = "98326")] + fn next_chunk( + &mut self, + ) -> Result<[Self::Item; N], array::IntoIter> + where + Self: Sized, + { + array::iter_next_chunk(self) + } + /// Returns the bounds on the remaining length of the iterator. /// /// Specifically, `size_hint()` returns a tuple where the first element /// is the lower bound, and the second element is the upper bound. /// - /// The second half of the tuple that is returned is an [`Option`]`<`[`usize`]`>`. + /// The second half of the tuple that is returned is an [Option]<[usize]>. /// A [`None`] here means that either there is no known upper bound, or the /// upper bound is larger than [`usize`]. /// @@ -156,28 +174,26 @@ pub trait Iterator { /// That said, the implementation should provide a correct estimation, /// because otherwise it would be a violation of the trait's protocol. /// - /// The default implementation returns `(0, `[`None`]`)` which is correct for any + /// The default implementation returns (0, [None]) which is correct for any /// iterator. /// - /// [`usize`]: ../../std/primitive.usize.html - /// [`Option`]: ../../std/option/enum.Option.html - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// /// # Examples /// /// Basic usage: /// /// ``` /// let a = [1, 2, 3]; - /// let iter = a.iter(); + /// let mut iter = a.iter(); /// /// assert_eq!((3, Some(3)), iter.size_hint()); + /// let _ = iter.next(); + /// assert_eq!((2, Some(2)), iter.size_hint()); /// ``` /// /// A more complex example: /// /// ``` - /// // The even numbers from zero to ten. + /// // The even numbers in the range of zero to nine. /// let iter = (0..10).filter(|x| x % 2 == 0); /// /// // We might iterate from zero to ten times. Knowing that it's five @@ -198,7 +214,7 @@ pub trait Iterator { /// // and the maximum possible lower bound /// let iter = 0..; /// - /// assert_eq!((usize::max_value(), None), iter.size_hint()); + /// assert_eq!((usize::MAX, None), iter.size_hint()); /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] @@ -212,9 +228,7 @@ pub trait Iterator { /// returning the number of times it saw [`Some`]. Note that [`next`] has to be /// called at least once even if the iterator does not have any elements. /// - /// [`next`]: #tymethod.next - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// [`Some`]: ../../std/option/enum.Option.html#variant.Some + /// [`next`]: Iterator::next /// /// # Overflow Behavior /// @@ -228,8 +242,6 @@ pub trait Iterator { /// This function might panic if the iterator has more than [`usize::MAX`] /// elements. /// - /// [`usize::MAX`]: ../../std/usize/constant.MAX.html - /// /// # Examples /// /// Basic usage: @@ -247,13 +259,11 @@ pub trait Iterator { where Self: Sized, { - #[inline] - fn add1(count: usize, _: T) -> usize { - // Might overflow. - Add::add(count, 1) - } - - self.fold(0, add1) + self.fold( + 0, + #[rustc_inherit_overflow_checks] + |count, _| count + 1, + ) } /// Consumes the iterator, returning the last element. @@ -262,8 +272,6 @@ pub trait Iterator { /// doing so, it keeps track of the current element. After [`None`] is /// returned, `last()` will then return the last element it saw. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// /// # Examples /// /// Basic usage: @@ -289,6 +297,47 @@ pub trait Iterator { self.fold(None, some) } + /// Advances the iterator by `n` elements. + /// + /// This method will eagerly skip `n` elements by calling [`next`] up to `n` + /// times until [`None`] is encountered. + /// + /// `advance_by(n)` will return [`Ok(())`][Ok] if the iterator successfully advances by + /// `n` elements, or [`Err(k)`][Err] if [`None`] is encountered, where `k` is the number + /// of elements the iterator is advanced by before running out of elements (i.e. the + /// length of the iterator). Note that `k` is always less than `n`. + /// + /// Calling `advance_by(0)` can do meaningful work, for example [`Flatten`] + /// can advance its outer iterator until it finds an inner iterator that is not empty, which + /// then often allows it to return a more accurate `size_hint()` than in its initial state. + /// + /// [`Flatten`]: crate::iter::Flatten + /// [`next`]: Iterator::next + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(iter_advance_by)] + /// + /// let a = [1, 2, 3, 4]; + /// let mut iter = a.iter(); + /// + /// assert_eq!(iter.advance_by(2), Ok(())); + /// assert_eq!(iter.next(), Some(&3)); + /// assert_eq!(iter.advance_by(0), Ok(())); + /// assert_eq!(iter.advance_by(100), Err(1)); // only `&4` was skipped + /// ``` + #[inline] + #[unstable(feature = "iter_advance_by", reason = "recently added", issue = "77404")] + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + for i in 0..n { + self.next().ok_or(i)?; + } + Ok(()) + } + /// Returns the `n`th element of the iterator. /// /// Like most indexing operations, the count starts from zero, so `nth(0)` @@ -302,8 +351,6 @@ pub trait Iterator { /// `nth()` will return [`None`] if `n` is greater than or equal to the length of the /// iterator. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// /// # Examples /// /// Basic usage: @@ -332,14 +379,9 @@ pub trait Iterator { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - fn nth(&mut self, mut n: usize) -> Option { - for x in self { - if n == 0 { - return Some(x); - } - n -= 1; - } - None + fn nth(&mut self, n: usize) -> Option { + self.advance_by(n).ok()?; + self.next() } /// Creates an iterator starting at the same point, but stepping by @@ -349,21 +391,22 @@ pub trait Iterator { /// regardless of the step given. /// /// Note 2: The time at which ignored elements are pulled is not fixed. - /// `StepBy` behaves like the sequence `next(), nth(step-1), nth(step-1), …`, - /// but is also free to behave like the sequence - /// `advance_n_and_return_first(step), advance_n_and_return_first(step), …` + /// `StepBy` behaves like the sequence `self.next()`, `self.nth(step-1)`, + /// `self.nth(step-1)`, …, but is also free to behave like the sequence + /// `advance_n_and_return_first(&mut self, step)`, + /// `advance_n_and_return_first(&mut self, step)`, … /// Which way is used may change for some iterators for performance reasons. /// The second way will advance the iterator earlier and may consume more items. /// /// `advance_n_and_return_first` is the equivalent of: /// ``` - /// fn advance_n_and_return_first(iter: &mut I, total_step: usize) -> Option + /// fn advance_n_and_return_first(iter: &mut I, n: usize) -> Option /// where /// I: Iterator, /// { /// let next = iter.next(); - /// if total_step > 1 { - /// iter.nth(total_step-2); + /// if n > 1 { + /// iter.nth(n - 2); /// } /// next /// } @@ -455,9 +498,7 @@ pub trait Iterator { /// } /// ``` /// - /// [`once`]: fn.once.html - /// [`Iterator`]: trait.Iterator.html - /// [`IntoIterator`]: trait.IntoIterator.html + /// [`once`]: crate::iter::once /// [`OsStr`]: ../../std/ffi/struct.OsStr.html #[inline] #[stable(feature = "rust1", since = "1.0.0")] @@ -478,8 +519,14 @@ pub trait Iterator { /// In other words, it zips two iterators together, into a single one. /// /// If either iterator returns [`None`], [`next`] from the zipped iterator - /// will return [`None`]. If the first iterator returns [`None`], `zip` will - /// short-circuit and `next` will not be called on the second iterator. + /// will return [`None`]. + /// If the zipped iterator has no more elements to return then each further attempt to advance + /// it will first try to advance the first iterator at most one time and if it still yielded an item + /// try to advance the second iterator at most one time. + /// + /// To 'undo' the result of zipping up two iterators, see [`unzip`]. + /// + /// [`unzip`]: Iterator::unzip /// /// # Examples /// @@ -502,9 +549,6 @@ pub trait Iterator { /// [`Iterator`] itself. For example, slices (`&[T]`) implement /// [`IntoIterator`], and so can be passed to `zip()` directly: /// - /// [`IntoIterator`]: trait.IntoIterator.html - /// [`Iterator`]: trait.Iterator.html - /// /// ``` /// let s1 = &[1, 2, 3]; /// let s2 = &[4, 5, 6]; @@ -536,9 +580,44 @@ pub trait Iterator { /// assert_eq!((2, 'o'), zipper[2]); /// ``` /// - /// [`enumerate`]: trait.Iterator.html#method.enumerate - /// [`next`]: ../../std/iter/trait.Iterator.html#tymethod.next - /// [`None`]: ../../std/option/enum.Option.html#variant.None + /// If both iterators have roughly equivalent syntax, it may be more readable to use [`zip`]: + /// + /// ``` + /// use std::iter::zip; + /// + /// let a = [1, 2, 3]; + /// let b = [2, 3, 4]; + /// + /// let mut zipped = zip( + /// a.into_iter().map(|x| x * 2).skip(1), + /// b.into_iter().map(|x| x * 2).skip(1), + /// ); + /// + /// assert_eq!(zipped.next(), Some((4, 6))); + /// assert_eq!(zipped.next(), Some((6, 8))); + /// assert_eq!(zipped.next(), None); + /// ``` + /// + /// compared to: + /// + /// ``` + /// # let a = [1, 2, 3]; + /// # let b = [2, 3, 4]; + /// # + /// let mut zipped = a + /// .into_iter() + /// .map(|x| x * 2) + /// .skip(1) + /// .zip(b.into_iter().map(|x| x * 2).skip(1)); + /// # + /// # assert_eq!(zipped.next(), Some((4, 6))); + /// # assert_eq!(zipped.next(), Some((6, 8))); + /// # assert_eq!(zipped.next(), None); + /// ``` + /// + /// [`enumerate`]: Iterator::enumerate + /// [`next`]: Iterator::next + /// [`zip`]: crate::iter::zip #[inline] #[stable(feature = "rust1", since = "1.0.0")] fn zip(self, other: U) -> Zip @@ -549,6 +628,106 @@ pub trait Iterator { Zip::new(self, other.into_iter()) } + /// Creates a new iterator which places a copy of `separator` between adjacent + /// items of the original iterator. + /// + /// In case `separator` does not implement [`Clone`] or needs to be + /// computed every time, use [`intersperse_with`]. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(iter_intersperse)] + /// + /// let mut a = [0, 1, 2].iter().intersperse(&100); + /// assert_eq!(a.next(), Some(&0)); // The first element from `a`. + /// assert_eq!(a.next(), Some(&100)); // The separator. + /// assert_eq!(a.next(), Some(&1)); // The next element from `a`. + /// assert_eq!(a.next(), Some(&100)); // The separator. + /// assert_eq!(a.next(), Some(&2)); // The last element from `a`. + /// assert_eq!(a.next(), None); // The iterator is finished. + /// ``` + /// + /// `intersperse` can be very useful to join an iterator's items using a common element: + /// ``` + /// #![feature(iter_intersperse)] + /// + /// let hello = ["Hello", "World", "!"].iter().copied().intersperse(" ").collect::(); + /// assert_eq!(hello, "Hello World !"); + /// ``` + /// + /// [`Clone`]: crate::clone::Clone + /// [`intersperse_with`]: Iterator::intersperse_with + #[inline] + #[unstable(feature = "iter_intersperse", reason = "recently added", issue = "79524")] + fn intersperse(self, separator: Self::Item) -> Intersperse + where + Self: Sized, + Self::Item: Clone, + { + Intersperse::new(self, separator) + } + + /// Creates a new iterator which places an item generated by `separator` + /// between adjacent items of the original iterator. + /// + /// The closure will be called exactly once each time an item is placed + /// between two adjacent items from the underlying iterator; specifically, + /// the closure is not called if the underlying iterator yields less than + /// two items and after the last item is yielded. + /// + /// If the iterator's item implements [`Clone`], it may be easier to use + /// [`intersperse`]. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(iter_intersperse)] + /// + /// #[derive(PartialEq, Debug)] + /// struct NotClone(usize); + /// + /// let v = [NotClone(0), NotClone(1), NotClone(2)]; + /// let mut it = v.into_iter().intersperse_with(|| NotClone(99)); + /// + /// assert_eq!(it.next(), Some(NotClone(0))); // The first element from `v`. + /// assert_eq!(it.next(), Some(NotClone(99))); // The separator. + /// assert_eq!(it.next(), Some(NotClone(1))); // The next element from `v`. + /// assert_eq!(it.next(), Some(NotClone(99))); // The separator. + /// assert_eq!(it.next(), Some(NotClone(2))); // The last element from `v`. + /// assert_eq!(it.next(), None); // The iterator is finished. + /// ``` + /// + /// `intersperse_with` can be used in situations where the separator needs + /// to be computed: + /// ``` + /// #![feature(iter_intersperse)] + /// + /// let src = ["Hello", "to", "all", "people", "!!"].iter().copied(); + /// + /// // The closure mutably borrows its context to generate an item. + /// let mut happy_emojis = [" ❤️ ", " 😀 "].iter().copied(); + /// let separator = || happy_emojis.next().unwrap_or(" 🦀 "); + /// + /// let result = src.intersperse_with(separator).collect::(); + /// assert_eq!(result, "Hello ❤️ to 😀 all 🦀 people 🦀 !!"); + /// ``` + /// [`Clone`]: crate::clone::Clone + /// [`intersperse`]: Iterator::intersperse + #[inline] + #[unstable(feature = "iter_intersperse", reason = "recently added", issue = "79524")] + fn intersperse_with(self, separator: G) -> IntersperseWith + where + Self: Sized, + G: FnMut() -> Self::Item, + { + IntersperseWith::new(self, separator) + } + /// Takes a closure and creates an iterator which calls that closure on each /// element. /// @@ -567,7 +746,7 @@ pub trait Iterator { /// more idiomatic to use [`for`] than `map()`. /// /// [`for`]: ../../book/ch03-05-control-flow.html#looping-through-a-collection-with-for - /// [`FnMut`]: ../../std/ops/trait.FnMut.html + /// [`FnMut`]: crate::ops::FnMut /// /// # Examples /// @@ -589,13 +768,13 @@ pub trait Iterator { /// ``` /// # #![allow(unused_must_use)] /// // don't do this: - /// (0..5).map(|x| println!("{}", x)); + /// (0..5).map(|x| println!("{x}")); /// /// // it won't even execute, as it is lazy. Rust will warn you about this. /// /// // Instead, use for: /// for x in 0..5 { - /// println!("{}", x); + /// println!("{x}"); /// } /// ``` #[inline] @@ -615,7 +794,7 @@ pub trait Iterator { /// more idiomatic to use a `for` loop, but `for_each` may be more legible /// when processing items at the end of longer iterator chains. In some /// cases `for_each` may also be faster than a loop, because it will use - /// internal iteration on adaptors like `Chain`. + /// internal iteration on adapters like `Chain`. /// /// [`for`]: ../../book/ch03-05-control-flow.html#looping-through-a-collection-with-for /// @@ -630,7 +809,7 @@ pub trait Iterator { /// (0..5).map(|x| x * 2 + 1) /// .for_each(move |x| tx.send(x).unwrap()); /// - /// let v: Vec<_> = rx.iter().collect(); + /// let v: Vec<_> = rx.iter().collect(); /// assert_eq!(v, vec![1, 3, 5, 7, 9]); /// ``` /// @@ -641,7 +820,7 @@ pub trait Iterator { /// (0..5).flat_map(|x| x * 100 .. x * 110) /// .enumerate() /// .filter(|&(i, x)| (i + x) % 3 == 0) - /// .for_each(|(i, x)| println!("{}:{}", i, x)); + /// .for_each(|(i, x)| println!("{i}:{x}")); /// ``` #[inline] #[stable(feature = "iterator_for_each", since = "1.21.0")] @@ -661,11 +840,9 @@ pub trait Iterator { /// Creates an iterator which uses a closure to determine if an element /// should be yielded. /// - /// The closure must return `true` or `false`. `filter()` creates an - /// iterator which calls this closure on each element. If the closure - /// returns `true`, then the element is returned. If the closure returns - /// `false`, it will try again, and call the closure on the next element, - /// seeing if it passes the test. + /// Given an element the closure must return `true` or `false`. The returned + /// iterator will yield only the elements for which the closure returns + /// true. /// /// # Examples /// @@ -732,35 +909,26 @@ pub trait Iterator { /// Creates an iterator that both filters and maps. /// - /// The closure must return an [`Option`]. `filter_map` creates an - /// iterator which calls this closure on each element. If the closure - /// returns [`Some(element)`][`Some`], then that element is returned. If the - /// closure returns [`None`], it will try again, and call the closure on the - /// next element, seeing if it will return [`Some`]. - /// - /// Why `filter_map` and not just [`filter`] and [`map`]? The key is in this - /// part: + /// The returned iterator yields only the `value`s for which the supplied + /// closure returns `Some(value)`. /// - /// [`filter`]: #method.filter - /// [`map`]: #method.map + /// `filter_map` can be used to make chains of [`filter`] and [`map`] more + /// concise. The example below shows how a `map().filter().map()` can be + /// shortened to a single call to `filter_map`. /// - /// > If the closure returns [`Some(element)`][`Some`], then that element is returned. - /// - /// In other words, it removes the [`Option`] layer automatically. If your - /// mapping is already returning an [`Option`] and you want to skip over - /// [`None`]s, then `filter_map` is much, much nicer to use. + /// [`filter`]: Iterator::filter + /// [`map`]: Iterator::map /// /// # Examples /// /// Basic usage: /// /// ``` - /// let a = ["1", "lol", "3", "NaN", "5"]; + /// let a = ["1", "two", "NaN", "four", "5"]; /// /// let mut iter = a.iter().filter_map(|s| s.parse().ok()); /// /// assert_eq!(iter.next(), Some(1)); - /// assert_eq!(iter.next(), Some(3)); /// assert_eq!(iter.next(), Some(5)); /// assert_eq!(iter.next(), None); /// ``` @@ -768,17 +936,12 @@ pub trait Iterator { /// Here's the same example, but with [`filter`] and [`map`]: /// /// ``` - /// let a = ["1", "lol", "3", "NaN", "5"]; + /// let a = ["1", "two", "NaN", "four", "5"]; /// let mut iter = a.iter().map(|s| s.parse()).filter(|s| s.is_ok()).map(|s| s.unwrap()); /// assert_eq!(iter.next(), Some(1)); - /// assert_eq!(iter.next(), Some(3)); /// assert_eq!(iter.next(), Some(5)); /// assert_eq!(iter.next(), None); /// ``` - /// - /// [`Option`]: ../../std/option/enum.Option.html - /// [`Some`]: ../../std/option/enum.Option.html#variant.Some - /// [`None`]: ../../std/option/enum.Option.html#variant.None #[inline] #[stable(feature = "rust1", since = "1.0.0")] fn filter_map(self, f: F) -> FilterMap @@ -811,9 +974,7 @@ pub trait Iterator { /// The returned iterator might panic if the to-be-returned index would /// overflow a [`usize`]. /// - /// [`usize::MAX`]: ../../std/usize/constant.MAX.html - /// [`usize`]: ../../std/primitive.usize.html - /// [`zip`]: #method.zip + /// [`zip`]: Iterator::zip /// /// # Examples /// @@ -836,20 +997,16 @@ pub trait Iterator { Enumerate::new(self) } - /// Creates an iterator which can use `peek` to look at the next element of - /// the iterator without consuming it. + /// Creates an iterator which can use the [`peek`] and [`peek_mut`] methods + /// to look at the next element of the iterator without consuming it. See + /// their documentation for more information. /// - /// Adds a [`peek`] method to an iterator. See its documentation for - /// more information. + /// Note that the underlying iterator is still advanced when [`peek`] or + /// [`peek_mut`] are called for the first time: In order to retrieve the + /// next element, [`next`] is called on the underlying iterator, hence any + /// side effects (i.e. anything other than fetching the next value) of + /// the [`next`] method will occur. /// - /// Note that the underlying iterator is still advanced when [`peek`] is - /// called for the first time: In order to retrieve the next element, - /// [`next`] is called on the underlying iterator, hence any side effects (i.e. - /// anything other than fetching the next value) of the [`next`] method - /// will occur. - /// - /// [`peek`]: struct.Peekable.html#method.peek - /// [`next`]: ../../std/iter/trait.Iterator.html#tymethod.next /// /// # Examples /// @@ -876,6 +1033,32 @@ pub trait Iterator { /// assert_eq!(iter.peek(), None); /// assert_eq!(iter.next(), None); /// ``` + /// + /// Using [`peek_mut`] to mutate the next item without advancing the + /// iterator: + /// + /// ``` + /// let xs = [1, 2, 3]; + /// + /// let mut iter = xs.iter().peekable(); + /// + /// // `peek_mut()` lets us see into the future + /// assert_eq!(iter.peek_mut(), Some(&mut &1)); + /// assert_eq!(iter.peek_mut(), Some(&mut &1)); + /// assert_eq!(iter.next(), Some(&1)); + /// + /// if let Some(mut p) = iter.peek_mut() { + /// assert_eq!(*p, &2); + /// // put a value into the iterator + /// *p = &1000; + /// } + /// + /// // The value reappears as the iterator continues + /// assert_eq!(iter.collect::>(), vec![&1000, &3]); + /// ``` + /// [`peek`]: Peekable::peek + /// [`peek_mut`]: Peekable::peek_mut + /// [`next`]: Iterator::next #[inline] #[stable(feature = "rust1", since = "1.0.0")] fn peekable(self) -> Peekable @@ -887,7 +1070,7 @@ pub trait Iterator { /// Creates an iterator that [`skip`]s elements based on a predicate. /// - /// [`skip`]: #method.skip + /// [`skip`]: Iterator::skip /// /// `skip_while()` takes a closure as an argument. It will call this /// closure on each element of the iterator, and ignore elements @@ -912,7 +1095,7 @@ pub trait Iterator { /// /// Because the closure passed to `skip_while()` takes a reference, and many /// iterators iterate over references, this leads to a possibly confusing - /// situation, where the type of the closure is a double reference: + /// situation, where the type of the closure argument is a double reference: /// /// ``` /// let a = [-1, 0, 1]; @@ -941,6 +1124,7 @@ pub trait Iterator { /// assert_eq!(iter.next(), None); /// ``` #[inline] + #[doc(alias = "drop_while")] #[stable(feature = "rust1", since = "1.0.0")] fn skip_while

(self, predicate: P) -> SkipWhile where @@ -1037,15 +1221,11 @@ pub trait Iterator { /// closure on each element of the iterator, and yield elements /// while it returns [`Some(_)`][`Some`]. /// - /// After [`None`] is returned, `map_while()`'s job is over, and the - /// rest of the elements are ignored. - /// /// # Examples /// /// Basic usage: /// /// ``` - /// #![feature(iter_map_while)] /// let a = [-1i32, 4, 0, 1]; /// /// let mut iter = a.iter().map_while(|x| 16i32.checked_div(*x)); @@ -1057,8 +1237,8 @@ pub trait Iterator { /// /// Here's the same example, but with [`take_while`] and [`map`]: /// - /// [`take_while`]: #method.take_while - /// [`map`]: #method.map + /// [`take_while`]: Iterator::take_while + /// [`map`]: Iterator::map /// /// ``` /// let a = [-1i32, 4, 0, 1]; @@ -1076,18 +1256,14 @@ pub trait Iterator { /// Stopping after an initial [`None`]: /// /// ``` - /// #![feature(iter_map_while)] - /// use std::convert::TryFrom; - /// - /// let a = [0, -1, 1, -2]; - /// - /// let mut iter = a.iter().map_while(|x| u32::try_from(*x).ok()); + /// let a = [0, 1, 2, -3, 4, 5, -6]; /// - /// assert_eq!(iter.next(), Some(0u32)); + /// let iter = a.iter().map_while(|x| u32::try_from(*x).ok()); + /// let vec = iter.collect::>(); /// - /// // We have more elements that are fit in u32, but since we already - /// // got a None, map_while() isn't used any more - /// assert_eq!(iter.next(), None); + /// // We have more elements which could fit in u32 (4, 5), but `map_while` returned `None` for `-3` + /// // (as the `predicate` returned `None`) and `collect` stops at the first `None` encountered. + /// assert_eq!(vec, vec![0, 1, 2]); /// ``` /// /// Because `map_while()` needs to look at the value in order to see if it @@ -1095,9 +1271,6 @@ pub trait Iterator { /// removed: /// /// ``` - /// #![feature(iter_map_while)] - /// use std::convert::TryFrom; - /// /// let a = [1, 2, -3, 4]; /// let mut iter = a.iter(); /// @@ -1115,10 +1288,13 @@ pub trait Iterator { /// The `-3` is no longer there, because it was consumed in order to see if /// the iteration should stop, but wasn't placed back into the iterator. /// - /// [`Some`]: ../../std/option/enum.Option.html#variant.Some - /// [`None`]: ../../std/option/enum.Option.html#variant.None + /// Note that unlike [`take_while`] this iterator is **not** fused. + /// It is also not specified what this iterator returns after the first [`None`] is returned. + /// If you need fused iterator, use [`fuse`]. + /// + /// [`fuse`]: Iterator::fuse #[inline] - #[unstable(feature = "iter_map_while", reason = "recently added", issue = "68537")] + #[stable(feature = "iter_map_while", since = "1.57.0")] fn map_while(self, predicate: P) -> MapWhile where Self: Sized, @@ -1129,7 +1305,11 @@ pub trait Iterator { /// Creates an iterator that skips the first `n` elements. /// - /// After they have been consumed, the rest of the elements are yielded. + /// `skip(n)` skips elements until `n` elements are skipped or the end of the + /// iterator is reached (whichever happens first). After that, all the remaining + /// elements are yielded. In particular, if the original iterator is too short, + /// then the returned iterator is empty. + /// /// Rather than overriding this method directly, instead override the `nth` method. /// /// # Examples @@ -1153,7 +1333,14 @@ pub trait Iterator { Skip::new(self, n) } - /// Creates an iterator that yields its first `n` elements. + /// Creates an iterator that yields the first `n` elements, or fewer + /// if the underlying iterator ends sooner. + /// + /// `take(n)` yields elements until `n` elements are yielded or the end of + /// the iterator is reached (whichever happens first). + /// The returned iterator is a prefix of length `n` if the original iterator + /// contains at least `n` elements, otherwise it contains all of the + /// (fewer than `n`) elements of the original iterator. /// /// # Examples /// @@ -1179,6 +1366,17 @@ pub trait Iterator { /// assert_eq!(iter.next(), Some(2)); /// assert_eq!(iter.next(), None); /// ``` + /// + /// If less than `n` elements are available, + /// `take` will limit itself to the size of the underlying iterator: + /// + /// ``` + /// let v = [1, 2]; + /// let mut iter = v.into_iter().take(5); + /// assert_eq!(iter.next(), Some(1)); + /// assert_eq!(iter.next(), Some(2)); + /// assert_eq!(iter.next(), None); + /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] fn take(self, n: usize) -> Take @@ -1188,10 +1386,10 @@ pub trait Iterator { Take::new(self, n) } - /// An iterator adaptor similar to [`fold`] that holds internal state and - /// produces a new iterator. + /// An iterator adapter which, like [`fold`], holds internal state, but + /// unlike [`fold`], produces a new iterator. /// - /// [`fold`]: #method.fold + /// [`fold`]: Iterator::fold /// /// `scan()` takes two arguments: an initial value which seeds the internal /// state, and a closure with two arguments, the first being a mutable @@ -1201,22 +1399,25 @@ pub trait Iterator { /// /// On iteration, the closure will be applied to each element of the /// iterator and the return value from the closure, an [`Option`], is - /// yielded by the iterator. - /// - /// [`Option`]: ../../std/option/enum.Option.html + /// returned by the `next` method. Thus the closure can return + /// `Some(value)` to yield `value`, or `None` to end the iteration. /// /// # Examples /// /// Basic usage: /// /// ``` - /// let a = [1, 2, 3]; + /// let a = [1, 2, 3, 4]; /// /// let mut iter = a.iter().scan(1, |state, &x| { - /// // each iteration, we'll multiply the state by the element + /// // each iteration, we'll multiply the state by the element ... /// *state = *state * x; /// - /// // then, we'll yield the negation of the state + /// // ... and terminate if the state exceeds 6 + /// if *state > 6 { + /// return None; + /// } + /// // ... else yield the negation of the state /// Some(-*state) /// }); /// @@ -1249,8 +1450,8 @@ pub trait Iterator { /// one item for each element, and `flat_map()`'s closure returns an /// iterator for each element. /// - /// [`map`]: #method.map - /// [`flatten`]: #method.flatten + /// [`map`]: Iterator::map + /// [`flatten`]: Iterator::flatten /// /// # Examples /// @@ -1318,7 +1519,19 @@ pub trait Iterator { /// assert_eq!(merged, "alphabetagamma"); /// ``` /// - /// Flattening once only removes one level of nesting: + /// Flattening works on any `IntoIterator` type, including `Option` and `Result`: + /// + /// ``` + /// let options = vec![Some(123), Some(321), None, Some(231)]; + /// let flattened_options: Vec<_> = options.into_iter().flatten().collect(); + /// assert_eq!(flattened_options, vec![123, 321, 231]); + /// + /// let results = vec![Ok(123), Ok(321), Err(456), Ok(231)]; + /// let flattened_results: Vec<_> = results.into_iter().flatten().collect(); + /// assert_eq!(flattened_results, vec![123, 321, 231]); + /// ``` + /// + /// Flattening only removes one level of nesting at a time: /// /// ``` /// let d3 = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]]; @@ -1332,11 +1545,11 @@ pub trait Iterator { /// /// Here we see that `flatten()` does not perform a "deep" flatten. /// Instead, only one level of nesting is removed. That is, if you - /// `flatten()` a three-dimensional array the result will be + /// `flatten()` a three-dimensional array, the result will be /// two-dimensional and not one-dimensional. To get a one-dimensional /// structure, you have to `flatten()` again. /// - /// [`flat_map()`]: #method.flat_map + /// [`flat_map()`]: Iterator::flat_map #[inline] #[stable(feature = "iterator_flatten", since = "1.29.0")] fn flatten(self) -> Flatten @@ -1353,8 +1566,12 @@ pub trait Iterator { /// [`Some(T)`] again. `fuse()` adapts an iterator, ensuring that after a /// [`None`] is given, it will always return [`None`] forever. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// [`Some(T)`]: ../../std/option/enum.Option.html#variant.Some + /// Note that the [`Fuse`] wrapper is a no-op on iterators that implement + /// the [`FusedIterator`] trait. `fuse()` may therefore behave incorrectly + /// if the [`FusedIterator`] trait is improperly implemented. + /// + /// [`Some(T)`]: Some + /// [`FusedIterator`]: crate::iter::FusedIterator /// /// # Examples /// @@ -1434,17 +1651,17 @@ pub trait Iterator { /// .filter(|x| x % 2 == 0) /// .fold(0, |sum, i| sum + i); /// - /// println!("{}", sum); + /// println!("{sum}"); /// /// // let's add some inspect() calls to investigate what's happening /// let sum = a.iter() /// .cloned() - /// .inspect(|x| println!("about to filter: {}", x)) + /// .inspect(|x| println!("about to filter: {x}")) /// .filter(|x| x % 2 == 0) - /// .inspect(|x| println!("made it through filter: {}", x)) + /// .inspect(|x| println!("made it through filter: {x}")) /// .fold(0, |sum, i| sum + i); /// - /// println!("{}", sum); + /// println!("{sum}"); /// ``` /// /// This will print: @@ -1470,13 +1687,13 @@ pub trait Iterator { /// .map(|line| line.parse::()) /// .inspect(|num| { /// if let Err(ref e) = *num { - /// println!("Parsing error: {}", e); + /// println!("Parsing error: {e}"); /// } /// }) /// .filter_map(Result::ok) /// .sum(); /// - /// println!("Sum: {}", sum); + /// println!("Sum: {sum}"); /// ``` /// /// This will print: @@ -1497,7 +1714,7 @@ pub trait Iterator { /// Borrows an iterator, rather than consuming it. /// - /// This is useful to allow applying iterator adaptors while still + /// This is useful to allow applying iterator adapters while still /// retaining ownership of the original iterator. /// /// # Examples @@ -1505,31 +1722,16 @@ pub trait Iterator { /// Basic usage: /// /// ``` - /// let a = [1, 2, 3]; - /// - /// let iter = a.iter(); - /// - /// let sum: i32 = iter.take(5).fold(0, |acc, i| acc + i ); + /// let mut words = ["hello", "world", "of", "Rust"].into_iter(); /// - /// assert_eq!(sum, 6); - /// - /// // if we try to use iter again, it won't work. The following line - /// // gives "error: use of moved value: `iter` - /// // assert_eq!(iter.next(), None); - /// - /// // let's try that again - /// let a = [1, 2, 3]; + /// // Take the first two words. + /// let hello_world: Vec<_> = words.by_ref().take(2).collect(); + /// assert_eq!(hello_world, vec!["hello", "world"]); /// - /// let mut iter = a.iter(); - /// - /// // instead, we add in a .by_ref() - /// let sum: i32 = iter.by_ref().take(2).fold(0, |acc, i| acc + i ); - /// - /// assert_eq!(sum, 3); - /// - /// // now this is just fine: - /// assert_eq!(iter.next(), Some(&3)); - /// assert_eq!(iter.next(), None); + /// // Collect the rest of the words. + /// // We can only do this because we used `by_ref` earlier. + /// let of_rust: Vec<_> = words.collect(); + /// assert_eq!(of_rust, vec!["of", "Rust"]); /// ``` #[stable(feature = "rust1", since = "1.0.0")] fn by_ref(&mut self) -> &mut Self @@ -1549,11 +1751,10 @@ pub trait Iterator { /// collection into another. You take a collection, call [`iter`] on it, /// do a bunch of transformations, and then `collect()` at the end. /// - /// One of the keys to `collect()`'s power is that many things you might - /// not think of as 'collections' actually are. For example, a [`String`] - /// is a collection of [`char`]s. And a collection of - /// [`Result`][`Result`] can be thought of as single - /// [`Result`]`, E>`. See the examples below for more. + /// `collect()` can also create instances of types that are not typical + /// collections. For example, a [`String`] can be built from [`char`]s, + /// and an iterator of [`Result`][`Result`] items can be collected + /// into `Result, E>`. See the examples below for more. /// /// Because `collect()` is so general, it can cause problems with type /// inference. As such, `collect()` is one of the few times you'll see @@ -1645,13 +1846,13 @@ pub trait Iterator { /// assert_eq!(Ok(vec![1, 3]), result); /// ``` /// - /// [`iter`]: ../../std/iter/trait.Iterator.html#tymethod.next + /// [`iter`]: Iterator::next /// [`String`]: ../../std/string/struct.String.html - /// [`char`]: ../../std/primitive.char.html - /// [`Result`]: ../../std/result/enum.Result.html + /// [`char`]: type@char #[inline] #[stable(feature = "rust1", since = "1.0.0")] #[must_use = "if you really need to exhaust the iterator, consider `.for_each(drop)` instead"] + #[cfg_attr(not(test), rustc_diagnostic_item = "iterator_collect_fn")] fn collect>(self) -> B where Self: Sized, @@ -1659,6 +1860,158 @@ pub trait Iterator { FromIterator::from_iter(self) } + /// Fallibly transforms an iterator into a collection, short circuiting if + /// a failure is encountered. + /// + /// `try_collect()` is a variation of [`collect()`][`collect`] that allows fallible + /// conversions during collection. Its main use case is simplifying conversions from + /// iterators yielding [`Option`][`Option`] into `Option>`, or similarly for other [`Try`] + /// types (e.g. [`Result`]). + /// + /// Importantly, `try_collect()` doesn't require that the outer [`Try`] type also implements [`FromIterator`]; + /// only the inner type produced on `Try::Output` must implement it. Concretely, + /// this means that collecting into `ControlFlow<_, Vec>` is valid because `Vec` implements + /// [`FromIterator`], even though [`ControlFlow`] doesn't. + /// + /// Also, if a failure is encountered during `try_collect()`, the iterator is still valid and + /// may continue to be used, in which case it will continue iterating starting after the element that + /// triggered the failure. See the last example below for an example of how this works. + /// + /// # Examples + /// Successfully collecting an iterator of `Option` into `Option>`: + /// ``` + /// #![feature(iterator_try_collect)] + /// + /// let u = vec![Some(1), Some(2), Some(3)]; + /// let v = u.into_iter().try_collect::>(); + /// assert_eq!(v, Some(vec![1, 2, 3])); + /// ``` + /// + /// Failing to collect in the same way: + /// ``` + /// #![feature(iterator_try_collect)] + /// + /// let u = vec![Some(1), Some(2), None, Some(3)]; + /// let v = u.into_iter().try_collect::>(); + /// assert_eq!(v, None); + /// ``` + /// + /// A similar example, but with `Result`: + /// ``` + /// #![feature(iterator_try_collect)] + /// + /// let u: Vec> = vec![Ok(1), Ok(2), Ok(3)]; + /// let v = u.into_iter().try_collect::>(); + /// assert_eq!(v, Ok(vec![1, 2, 3])); + /// + /// let u = vec![Ok(1), Ok(2), Err(()), Ok(3)]; + /// let v = u.into_iter().try_collect::>(); + /// assert_eq!(v, Err(())); + /// ``` + /// + /// Finally, even [`ControlFlow`] works, despite the fact that it + /// doesn't implement [`FromIterator`]. Note also that the iterator can + /// continue to be used, even if a failure is encountered: + /// + /// ``` + /// #![feature(iterator_try_collect)] + /// + /// use core::ops::ControlFlow::{Break, Continue}; + /// + /// let u = [Continue(1), Continue(2), Break(3), Continue(4), Continue(5)]; + /// let mut it = u.into_iter(); + /// + /// let v = it.try_collect::>(); + /// assert_eq!(v, Break(3)); + /// + /// let v = it.try_collect::>(); + /// assert_eq!(v, Continue(vec![4, 5])); + /// ``` + /// + /// [`collect`]: Iterator::collect + #[inline] + #[unstable(feature = "iterator_try_collect", issue = "94047")] + fn try_collect(&mut self) -> ChangeOutputType + where + Self: Sized, + ::Item: Try, + <::Item as Try>::Residual: Residual, + B: FromIterator<::Output>, + { + try_process(ByRefSized(self), |i| i.collect()) + } + + /// Collects all the items from an iterator into a collection. + /// + /// This method consumes the iterator and adds all its items to the + /// passed collection. The collection is then returned, so the call chain + /// can be continued. + /// + /// This is useful when you already have a collection and wants to add + /// the iterator items to it. + /// + /// This method is a convenience method to call [Extend::extend](trait.Extend.html), + /// but instead of being called on a collection, it's called on an iterator. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(iter_collect_into)] + /// + /// let a = [1, 2, 3]; + /// let mut vec: Vec:: = vec![0, 1]; + /// + /// a.iter().map(|&x| x * 2).collect_into(&mut vec); + /// a.iter().map(|&x| x * 10).collect_into(&mut vec); + /// + /// assert_eq!(vec![0, 1, 2, 4, 6, 10, 20, 30], vec); + /// ``` + /// + /// `Vec` can have a manual set capacity to avoid reallocating it: + /// + /// ``` + /// #![feature(iter_collect_into)] + /// + /// let a = [1, 2, 3]; + /// let mut vec: Vec:: = Vec::with_capacity(6); + /// + /// a.iter().map(|&x| x * 2).collect_into(&mut vec); + /// a.iter().map(|&x| x * 10).collect_into(&mut vec); + /// + /// assert_eq!(6, vec.capacity()); + /// println!("{:?}", vec); + /// ``` + /// + /// The returned mutable reference can be used to continue the call chain: + /// + /// ``` + /// #![feature(iter_collect_into)] + /// + /// let a = [1, 2, 3]; + /// let mut vec: Vec:: = Vec::with_capacity(6); + /// + /// let count = a.iter().collect_into(&mut vec).iter().count(); + /// + /// assert_eq!(count, vec.len()); + /// println!("Vec len is {}", count); + /// + /// let count = a.iter().collect_into(&mut vec).iter().count(); + /// + /// assert_eq!(count, vec.len()); + /// println!("Vec len now is {}", count); + /// ``` + #[inline] + #[unstable(feature = "iter_collect_into", reason = "new API", issue = "94780")] + fn collect_into>(self, collection: &mut E) -> &mut E + where + Self: Sized, + { + collection.extend(self); + collection + } + /// Consumes an iterator, creating two collections from it. /// /// The predicate passed to `partition()` can return `true`, or `false`. @@ -1667,8 +2020,8 @@ pub trait Iterator { /// /// See also [`is_partitioned()`] and [`partition_in_place()`]. /// - /// [`is_partitioned()`]: #method.is_partitioned - /// [`partition_in_place()`]: #method.partition_in_place + /// [`is_partitioned()`]: Iterator::is_partitioned + /// [`partition_in_place()`]: Iterator::partition_in_place /// /// # Examples /// @@ -1677,9 +2030,9 @@ pub trait Iterator { /// ``` /// let a = [1, 2, 3]; /// - /// let (even, odd): (Vec, Vec) = a - /// .iter() - /// .partition(|&n| n % 2 == 0); + /// let (even, odd): (Vec<_>, Vec<_>) = a + /// .into_iter() + /// .partition(|n| n % 2 == 0); /// /// assert_eq!(even, vec![2]); /// assert_eq!(odd, vec![1, 3]); @@ -1696,12 +2049,12 @@ pub trait Iterator { mut f: impl FnMut(&T) -> bool + 'a, left: &'a mut B, right: &'a mut B, - ) -> impl FnMut(T) + 'a { - move |x| { + ) -> impl FnMut((), T) + 'a { + move |(), x| { if f(&x) { - left.extend(Some(x)); + left.extend_one(x); } else { - right.extend(Some(x)); + right.extend_one(x); } } } @@ -1709,7 +2062,7 @@ pub trait Iterator { let mut left: B = Default::default(); let mut right: B = Default::default(); - self.for_each(extend(f, &mut left, &mut right)); + self.fold((), extend(f, &mut left, &mut right)); (left, right) } @@ -1720,10 +2073,17 @@ pub trait Iterator { /// /// The relative order of partitioned items is not maintained. /// + /// # Current implementation + /// + /// Current algorithms tries finding the first element for which the predicate evaluates + /// to false, and the last element for which it evaluates to true and repeatedly swaps them. + /// + /// Time complexity: *O*(*n*) + /// /// See also [`is_partitioned()`] and [`partition()`]. /// - /// [`is_partitioned()`]: #method.is_partitioned - /// [`partition()`]: #method.partition + /// [`is_partitioned()`]: Iterator::is_partitioned + /// [`partition()`]: Iterator::partition /// /// # Examples /// @@ -1785,8 +2145,8 @@ pub trait Iterator { /// /// See also [`partition()`] and [`partition_in_place()`]. /// - /// [`partition()`]: #method.partition - /// [`partition_in_place()`]: #method.partition_in_place + /// [`partition()`]: Iterator::partition + /// [`partition_in_place()`]: Iterator::partition_in_place /// /// # Examples /// @@ -1825,7 +2185,7 @@ pub trait Iterator { /// /// # Note to Implementors /// - /// Most of the other (forward) methods have default implementations in + /// Several of the other (forward) methods have default implementations in /// terms of this one, so try to implement this explicitly if it can /// do something better than the default `for` loop implementation. /// @@ -1864,19 +2224,44 @@ pub trait Iterator { /// assert_eq!(it.len(), 2); /// assert_eq!(it.next(), Some(&40)); /// ``` + /// + /// While you cannot `break` from a closure, the [`ControlFlow`] type allows + /// a similar idea: + /// + /// ``` + /// use std::ops::ControlFlow; + /// + /// let triangular = (1..30).try_fold(0_i8, |prev, x| { + /// if let Some(next) = prev.checked_add(x) { + /// ControlFlow::Continue(next) + /// } else { + /// ControlFlow::Break(prev) + /// } + /// }); + /// assert_eq!(triangular, ControlFlow::Break(120)); + /// + /// let triangular = (1..30).try_fold(0_u64, |prev, x| { + /// if let Some(next) = prev.checked_add(x) { + /// ControlFlow::Continue(next) + /// } else { + /// ControlFlow::Break(prev) + /// } + /// }); + /// assert_eq!(triangular, ControlFlow::Continue(435)); + /// ``` #[inline] #[stable(feature = "iterator_try_fold", since = "1.27.0")] fn try_fold(&mut self, init: B, mut f: F) -> R where Self: Sized, F: FnMut(B, Self::Item) -> R, - R: Try, + R: Try, { let mut accum = init; while let Some(x) = self.next() { accum = f(accum, x)?; } - Try::from_ok(accum) + try { accum } } /// An iterator method that applies a fallible function to each item in the @@ -1885,8 +2270,8 @@ pub trait Iterator { /// This can also be thought of as the fallible form of [`for_each()`] /// or as the stateless version of [`try_fold()`]. /// - /// [`for_each()`]: #method.for_each - /// [`try_fold()`]: #method.try_fold + /// [`for_each()`]: Iterator::for_each + /// [`try_fold()`]: Iterator::try_fold /// /// # Examples /// @@ -1897,7 +2282,7 @@ pub trait Iterator { /// /// let data = ["no_tea.txt", "stale_bread.json", "torrential_rain.png"]; /// - /// let res = data.iter().try_for_each(|x| writeln!(stdout(), "{}", x)); + /// let res = data.iter().try_for_each(|x| writeln!(stdout(), "{x}")); /// assert!(res.is_ok()); /// /// let mut it = data.iter().cloned(); @@ -1906,13 +2291,29 @@ pub trait Iterator { /// // It short-circuited, so the remaining items are still in the iterator: /// assert_eq!(it.next(), Some("stale_bread.json")); /// ``` + /// + /// The [`ControlFlow`] type can be used with this method for the situations + /// in which you'd use `break` and `continue` in a normal loop: + /// + /// ``` + /// use std::ops::ControlFlow; + /// + /// let r = (2..100).try_for_each(|x| { + /// if 323 % x == 0 { + /// return ControlFlow::Break(x) + /// } + /// + /// ControlFlow::Continue(()) + /// }); + /// assert_eq!(r, ControlFlow::Break(17)); + /// ``` #[inline] #[stable(feature = "iterator_try_fold", since = "1.27.0")] fn try_for_each(&mut self, f: F) -> R where Self: Sized, F: FnMut(Self::Item) -> R, - R: Try, + R: Try, { #[inline] fn call(mut f: impl FnMut(T) -> R) -> impl FnMut((), T) -> R { @@ -1922,7 +2323,8 @@ pub trait Iterator { self.try_fold((), call(f)) } - /// An iterator method that applies a function, producing a single, final value. + /// Folds every element into an accumulator by applying an operation, + /// returning the final result. /// /// `fold()` takes two arguments: an initial value, and a closure with two /// arguments: an 'accumulator', and an element. The closure returns the value that @@ -1940,9 +2342,26 @@ pub trait Iterator { /// to produce a single value from it. /// /// Note: `fold()`, and similar methods that traverse the entire iterator, - /// may not terminate for infinite iterators, even on traits for which a + /// might not terminate for infinite iterators, even on traits for which a /// result is determinable in finite time. /// + /// Note: [`reduce()`] can be used to use the first element as the initial + /// value, if the accumulator type and item type is the same. + /// + /// Note: `fold()` combines elements in a *left-associative* fashion. For associative + /// operators like `+`, the order the elements are combined in is not important, but for non-associative + /// operators like `-` the order will affect the final result. + /// For a *right-associative* version of `fold()`, see [`DoubleEndedIterator::rfold()`]. + /// + /// # Note to Implementors + /// + /// Several of the other (forward) methods have default implementations in + /// terms of this one, so try to implement this explicitly if it can + /// do something better than the default `for` loop implementation. + /// + /// In particular, try to have this call `fold()` on the internal parts + /// from which this iterator is composed. + /// /// # Examples /// /// Basic usage: @@ -1967,6 +2386,21 @@ pub trait Iterator { /// /// And so, our final result, `6`. /// + /// This example demonstrates the left-associative nature of `fold()`: + /// it builds a string, starting with an initial value + /// and continuing with each element from the front until the back: + /// + /// ``` + /// let numbers = [1, 2, 3, 4, 5]; + /// + /// let zero = "0".to_string(); + /// + /// let result = numbers.iter().fold(zero, |acc, &x| { + /// format!("({acc} + {x})") + /// }); + /// + /// assert_eq!(result, "(((((0 + 1) + 2) + 3) + 4) + 5)"); + /// ``` /// It's common for people who haven't used iterators a lot to /// use a `for` loop with a list of things to build up a result. Those /// can be turned into `fold()`s: @@ -1989,19 +2423,135 @@ pub trait Iterator { /// // they're the same /// assert_eq!(result, result2); /// ``` + /// + /// [`reduce()`]: Iterator::reduce + #[doc(alias = "inject", alias = "foldl")] #[inline] #[stable(feature = "rust1", since = "1.0.0")] - fn fold(mut self, init: B, f: F) -> B + fn fold(mut self, init: B, mut f: F) -> B where Self: Sized, F: FnMut(B, Self::Item) -> B, { - #[inline] - fn ok(mut f: impl FnMut(B, T) -> B) -> impl FnMut(B, T) -> Result { - move |acc, x| Ok(f(acc, x)) + let mut accum = init; + while let Some(x) = self.next() { + accum = f(accum, x); } + accum + } - self.try_fold(init, ok(f)).unwrap() + /// Reduces the elements to a single one, by repeatedly applying a reducing + /// operation. + /// + /// If the iterator is empty, returns [`None`]; otherwise, returns the + /// result of the reduction. + /// + /// The reducing function is a closure with two arguments: an 'accumulator', and an element. + /// For iterators with at least one element, this is the same as [`fold()`] + /// with the first element of the iterator as the initial accumulator value, folding + /// every subsequent element into it. + /// + /// [`fold()`]: Iterator::fold + /// + /// # Example + /// + /// ``` + /// let reduced: i32 = (1..10).reduce(|acc, e| acc + e).unwrap(); + /// assert_eq!(reduced, 45); + /// + /// // Which is equivalent to doing it with `fold`: + /// let folded: i32 = (1..10).fold(0, |acc, e| acc + e); + /// assert_eq!(reduced, folded); + /// ``` + #[inline] + #[stable(feature = "iterator_fold_self", since = "1.51.0")] + fn reduce(mut self, f: F) -> Option + where + Self: Sized, + F: FnMut(Self::Item, Self::Item) -> Self::Item, + { + let first = self.next()?; + Some(self.fold(first, f)) + } + + /// Reduces the elements to a single one by repeatedly applying a reducing operation. If the + /// closure returns a failure, the failure is propagated back to the caller immediately. + /// + /// The return type of this method depends on the return type of the closure. If the closure + /// returns `Result`, then this function will return `Result, + /// E>`. If the closure returns `Option`, then this function will return + /// `Option>`. + /// + /// When called on an empty iterator, this function will return either `Some(None)` or + /// `Ok(None)` depending on the type of the provided closure. + /// + /// For iterators with at least one element, this is essentially the same as calling + /// [`try_fold()`] with the first element of the iterator as the initial accumulator value. + /// + /// [`try_fold()`]: Iterator::try_fold + /// + /// # Examples + /// + /// Safely calculate the sum of a series of numbers: + /// + /// ``` + /// #![feature(iterator_try_reduce)] + /// + /// let numbers: Vec = vec![10, 20, 5, 23, 0]; + /// let sum = numbers.into_iter().try_reduce(|x, y| x.checked_add(y)); + /// assert_eq!(sum, Some(Some(58))); + /// ``` + /// + /// Determine when a reduction short circuited: + /// + /// ``` + /// #![feature(iterator_try_reduce)] + /// + /// let numbers = vec![1, 2, 3, usize::MAX, 4, 5]; + /// let sum = numbers.into_iter().try_reduce(|x, y| x.checked_add(y)); + /// assert_eq!(sum, None); + /// ``` + /// + /// Determine when a reduction was not performed because there are no elements: + /// + /// ``` + /// #![feature(iterator_try_reduce)] + /// + /// let numbers: Vec = Vec::new(); + /// let sum = numbers.into_iter().try_reduce(|x, y| x.checked_add(y)); + /// assert_eq!(sum, Some(None)); + /// ``` + /// + /// Use a [`Result`] instead of an [`Option`]: + /// + /// ``` + /// #![feature(iterator_try_reduce)] + /// + /// let numbers = vec!["1", "2", "3", "4", "5"]; + /// let max: Result, ::Err> = + /// numbers.into_iter().try_reduce(|x, y| { + /// if x.parse::()? > y.parse::()? { Ok(x) } else { Ok(y) } + /// }); + /// assert_eq!(max, Ok(Some("5"))); + /// ``` + #[inline] + #[unstable(feature = "iterator_try_reduce", reason = "new API", issue = "87053")] + fn try_reduce(&mut self, f: F) -> ChangeOutputType> + where + Self: Sized, + F: FnMut(Self::Item, Self::Item) -> R, + R: Try, + R::Residual: Residual>, + { + let first = match self.next() { + Some(i) => i, + None => return Try::from_output(None), + }; + + match self.try_fold(first, f).branch() { + ControlFlow::Break(r) => FromResidual::from_residual(r), + ControlFlow::Continue(i) => Try::from_output(Some(i)), + } } /// Tests if every element of the iterator matches a predicate. @@ -2049,12 +2599,12 @@ pub trait Iterator { F: FnMut(Self::Item) -> bool, { #[inline] - fn check(mut f: impl FnMut(T) -> bool) -> impl FnMut((), T) -> LoopState<(), ()> { + fn check(mut f: impl FnMut(T) -> bool) -> impl FnMut((), T) -> ControlFlow<()> { move |(), x| { - if f(x) { LoopState::Continue(()) } else { LoopState::Break(()) } + if f(x) { ControlFlow::CONTINUE } else { ControlFlow::BREAK } } } - self.try_fold((), check(f)) == LoopState::Continue(()) + self.try_fold((), check(f)) == ControlFlow::CONTINUE } /// Tests if any element of the iterator matches a predicate. @@ -2102,13 +2652,13 @@ pub trait Iterator { F: FnMut(Self::Item) -> bool, { #[inline] - fn check(mut f: impl FnMut(T) -> bool) -> impl FnMut((), T) -> LoopState<(), ()> { + fn check(mut f: impl FnMut(T) -> bool) -> impl FnMut((), T) -> ControlFlow<()> { move |(), x| { - if f(x) { LoopState::Break(()) } else { LoopState::Continue(()) } + if f(x) { ControlFlow::BREAK } else { ControlFlow::CONTINUE } } } - self.try_fold((), check(f)) == LoopState::Break(()) + self.try_fold((), check(f)) == ControlFlow::BREAK } /// Searches for an element of an iterator that satisfies a predicate. @@ -2126,8 +2676,10 @@ pub trait Iterator { /// argument is a double reference. You can see this effect in the /// examples below, with `&&x`. /// - /// [`Some(element)`]: ../../std/option/enum.Option.html#variant.Some - /// [`None`]: ../../std/option/enum.Option.html#variant.None + /// If you need the index of the element, see [`position()`]. + /// + /// [`Some(element)`]: Some + /// [`position()`]: Iterator::position /// /// # Examples /// @@ -2163,11 +2715,9 @@ pub trait Iterator { P: FnMut(&Self::Item) -> bool, { #[inline] - fn check( - mut predicate: impl FnMut(&T) -> bool, - ) -> impl FnMut((), T) -> LoopState<(), T> { + fn check(mut predicate: impl FnMut(&T) -> bool) -> impl FnMut((), T) -> ControlFlow { move |(), x| { - if predicate(&x) { LoopState::Break(x) } else { LoopState::Continue(()) } + if predicate(&x) { ControlFlow::Break(x) } else { ControlFlow::CONTINUE } } } @@ -2179,7 +2729,6 @@ pub trait Iterator { /// /// `iter.find_map(f)` is equivalent to `iter.filter_map(f).next()`. /// - /// /// # Examples /// /// ``` @@ -2197,10 +2746,10 @@ pub trait Iterator { F: FnMut(Self::Item) -> Option, { #[inline] - fn check(mut f: impl FnMut(T) -> Option) -> impl FnMut((), T) -> LoopState<(), B> { + fn check(mut f: impl FnMut(T) -> Option) -> impl FnMut((), T) -> ControlFlow { move |(), x| match f(x) { - Some(x) => LoopState::Break(x), - None => LoopState::Continue(()), + Some(x) => ControlFlow::Break(x), + None => ControlFlow::CONTINUE, } } @@ -2208,7 +2757,11 @@ pub trait Iterator { } /// Applies function to the elements of iterator and returns - /// the first non-none result or the first error. + /// the first true result or the first error. + /// + /// The return type of this method depends on the return type of the closure. + /// If you return `Result` from the closure, you'll get a `Result, E>`. + /// If you return `Option` from the closure, you'll get an `Option>`. /// /// # Examples /// @@ -2227,21 +2780,48 @@ pub trait Iterator { /// let result = a.iter().try_find(|&&s| is_my_num(s, 5)); /// assert!(result.is_err()); /// ``` + /// + /// This also supports other types which implement `Try`, not just `Result`. + /// ``` + /// #![feature(try_find)] + /// + /// use std::num::NonZeroU32; + /// let a = [3, 5, 7, 4, 9, 0, 11]; + /// let result = a.iter().try_find(|&&x| NonZeroU32::new(x).map(|y| y.is_power_of_two())); + /// assert_eq!(result, Some(Some(&4))); + /// let result = a.iter().take(3).try_find(|&&x| NonZeroU32::new(x).map(|y| y.is_power_of_two())); + /// assert_eq!(result, Some(None)); + /// let result = a.iter().rev().try_find(|&&x| NonZeroU32::new(x).map(|y| y.is_power_of_two())); + /// assert_eq!(result, None); + /// ``` #[inline] #[unstable(feature = "try_find", reason = "new API", issue = "63178")] - fn try_find(&mut self, mut f: F) -> Result, E> + fn try_find(&mut self, f: F) -> ChangeOutputType> where Self: Sized, F: FnMut(&Self::Item) -> R, - R: Try, + R: Try, + R::Residual: Residual>, { - self.try_for_each(move |x| match f(&x).into_result() { - Ok(false) => LoopState::Continue(()), - Ok(true) => LoopState::Break(Ok(x)), - Err(x) => LoopState::Break(Err(x)), - }) - .break_value() - .transpose() + #[inline] + fn check( + mut f: impl FnMut(&I) -> V, + ) -> impl FnMut((), I) -> ControlFlow + where + V: Try, + R: Residual>, + { + move |(), x| match f(&x).branch() { + ControlFlow::Continue(false) => ControlFlow::CONTINUE, + ControlFlow::Continue(true) => ControlFlow::Break(Try::from_output(Some(x))), + ControlFlow::Break(r) => ControlFlow::Break(FromResidual::from_residual(r)), + } + } + + match self.try_fold((), check(f)) { + ControlFlow::Break(x) => x, + ControlFlow::Continue(()) => Try::from_output(None), + } } /// Searches for an element in an iterator, returning its index. @@ -2266,9 +2846,7 @@ pub trait Iterator { /// This function might panic if the iterator has more than `usize::MAX` /// non-matching elements. /// - /// [`Some(index)`]: ../../std/option/enum.Option.html#variant.Some - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// [`usize::MAX`]: ../../std/usize/constant.MAX.html + /// [`Some(index)`]: Some /// /// # Examples /// @@ -2308,10 +2886,10 @@ pub trait Iterator { #[inline] fn check( mut predicate: impl FnMut(T) -> bool, - ) -> impl FnMut(usize, T) -> LoopState { - // The addition might panic on overflow + ) -> impl FnMut(usize, T) -> ControlFlow { + #[rustc_inherit_overflow_checks] move |i, x| { - if predicate(x) { LoopState::Break(i) } else { LoopState::Continue(Add::add(i, 1)) } + if predicate(x) { ControlFlow::Break(i) } else { ControlFlow::Continue(i + 1) } } } @@ -2329,8 +2907,7 @@ pub trait Iterator { /// `rposition()` is short-circuiting; in other words, it will stop /// processing as soon as it finds a `true`. /// - /// [`Some(index)`]: ../../std/option/enum.Option.html#variant.Some - /// [`None`]: ../../std/option/enum.Option.html#variant.None + /// [`Some(index)`]: Some /// /// # Examples /// @@ -2347,14 +2924,14 @@ pub trait Iterator { /// Stopping at the first `true`: /// /// ``` - /// let a = [1, 2, 3]; + /// let a = [-1, 2, 3, 4]; /// /// let mut iter = a.iter(); /// - /// assert_eq!(iter.rposition(|&x| x == 2), Some(1)); + /// assert_eq!(iter.rposition(|&x| x >= 2), Some(3)); /// /// // we can still use `iter`, as there are more elements. - /// assert_eq!(iter.next(), Some(&1)); + /// assert_eq!(iter.next(), Some(&-1)); /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] @@ -2368,10 +2945,10 @@ pub trait Iterator { #[inline] fn check( mut predicate: impl FnMut(T) -> bool, - ) -> impl FnMut(usize, T) -> LoopState { + ) -> impl FnMut(usize, T) -> ControlFlow { move |i, x| { let i = i - 1; - if predicate(x) { LoopState::Break(i) } else { LoopState::Continue(i) } + if predicate(x) { ControlFlow::Break(i) } else { ControlFlow::Continue(i) } } } @@ -2384,7 +2961,17 @@ pub trait Iterator { /// If several elements are equally maximum, the last element is /// returned. If the iterator is empty, [`None`] is returned. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None + /// Note that [`f32`]/[`f64`] doesn't implement [`Ord`] due to NaN being + /// incomparable. You can work around this by using [`Iterator::reduce`]: + /// ``` + /// assert_eq!( + /// [2.4, f32::NAN, 1.3] + /// .into_iter() + /// .reduce(f32::max) + /// .unwrap(), + /// 2.4 + /// ); + /// ``` /// /// # Examples /// @@ -2409,10 +2996,20 @@ pub trait Iterator { /// Returns the minimum element of an iterator. /// - /// If several elements are equally minimum, the first element is - /// returned. If the iterator is empty, [`None`] is returned. + /// If several elements are equally minimum, the first element is returned. + /// If the iterator is empty, [`None`] is returned. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None + /// Note that [`f32`]/[`f64`] doesn't implement [`Ord`] due to NaN being + /// incomparable. You can work around this by using [`Iterator::reduce`]: + /// ``` + /// assert_eq!( + /// [2.4, f32::NAN, 1.3] + /// .into_iter() + /// .reduce(f32::min) + /// .unwrap(), + /// 1.3 + /// ); + /// ``` /// /// # Examples /// @@ -2441,8 +3038,6 @@ pub trait Iterator { /// If several elements are equally maximum, the last element is /// returned. If the iterator is empty, [`None`] is returned. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// /// # Examples /// /// ``` @@ -2476,8 +3071,6 @@ pub trait Iterator { /// If several elements are equally maximum, the last element is /// returned. If the iterator is empty, [`None`] is returned. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// /// # Examples /// /// ``` @@ -2496,7 +3089,7 @@ pub trait Iterator { move |x, y| cmp::max_by(x, y, &mut compare) } - fold1(self, fold(compare)) + self.reduce(fold(compare)) } /// Returns the element that gives the minimum value from the @@ -2505,8 +3098,6 @@ pub trait Iterator { /// If several elements are equally minimum, the first element is /// returned. If the iterator is empty, [`None`] is returned. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// /// # Examples /// /// ``` @@ -2540,8 +3131,6 @@ pub trait Iterator { /// If several elements are equally minimum, the first element is /// returned. If the iterator is empty, [`None`] is returned. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// /// # Examples /// /// ``` @@ -2560,7 +3149,7 @@ pub trait Iterator { move |x, y| cmp::min_by(x, y, &mut compare) } - fold1(self, fold(compare)) + self.reduce(fold(compare)) } /// Reverses an iterator's direction. @@ -2571,8 +3160,6 @@ pub trait Iterator { /// This is only possible if the iterator has an end, so `rev()` only /// works on [`DoubleEndedIterator`]s. /// - /// [`DoubleEndedIterator`]: trait.DoubleEndedIterator.html - /// /// # Examples /// /// ``` @@ -2587,6 +3174,7 @@ pub trait Iterator { /// assert_eq!(iter.next(), None); /// ``` #[inline] + #[doc(alias = "reverse")] #[stable(feature = "rust1", since = "1.0.0")] fn rev(self) -> Rev where @@ -2603,19 +3191,27 @@ pub trait Iterator { /// /// This function is, in some sense, the opposite of [`zip`]. /// - /// [`zip`]: #method.zip + /// [`zip`]: Iterator::zip /// /// # Examples /// /// Basic usage: /// /// ``` - /// let a = [(1, 2), (3, 4)]; + /// let a = [(1, 2), (3, 4), (5, 6)]; /// /// let (left, right): (Vec<_>, Vec<_>) = a.iter().cloned().unzip(); /// - /// assert_eq!(left, [1, 3]); - /// assert_eq!(right, [2, 4]); + /// assert_eq!(left, [1, 3, 5]); + /// assert_eq!(right, [2, 4, 6]); + /// + /// // you can also unzip multiple nested tuples at once + /// let a = [(1, (2, 3)), (4, (5, 6))]; + /// + /// let (x, (y, z)): (Vec<_>, (Vec<_>, Vec<_>)) = a.iter().cloned().unzip(); + /// assert_eq!(x, [1, 4]); + /// assert_eq!(y, [2, 5]); + /// assert_eq!(z, [3, 6]); /// ``` #[stable(feature = "rust1", since = "1.0.0")] fn unzip(self) -> (FromA, FromB) @@ -2624,22 +3220,9 @@ pub trait Iterator { FromB: Default + Extend, Self: Sized + Iterator, { - fn extend<'a, A, B>( - ts: &'a mut impl Extend, - us: &'a mut impl Extend, - ) -> impl FnMut((A, B)) + 'a { - move |(t, u)| { - ts.extend(Some(t)); - us.extend(Some(u)); - } - } - - let mut ts: FromA = Default::default(); - let mut us: FromB = Default::default(); - - self.for_each(extend(&mut ts, &mut us)); - - (ts, us) + let mut unzipped: (FromA, FromB) = Default::default(); + unzipped.extend(self); + unzipped } /// Creates an iterator which copies all of its elements. @@ -2654,12 +3237,12 @@ pub trait Iterator { /// ``` /// let a = [1, 2, 3]; /// - /// let v_cloned: Vec<_> = a.iter().copied().collect(); + /// let v_copied: Vec<_> = a.iter().copied().collect(); /// /// // copied is the same as .map(|&x| x) /// let v_map: Vec<_> = a.iter().map(|&x| x).collect(); /// - /// assert_eq!(v_cloned, vec![1, 2, 3]); + /// assert_eq!(v_copied, vec![1, 2, 3]); /// assert_eq!(v_map, vec![1, 2, 3]); /// ``` #[stable(feature = "iter_copied", since = "1.36.0")] @@ -2676,7 +3259,11 @@ pub trait Iterator { /// This is useful when you have an iterator over `&T`, but you need an /// iterator over `T`. /// - /// [`clone`]: ../../std/clone/trait.Clone.html#tymethod.clone + /// There is no guarantee whatsoever about the `clone` method actually + /// being called *or* optimized away. So code should not depend on + /// either. + /// + /// [`clone`]: Clone::clone /// /// # Examples /// @@ -2693,6 +3280,18 @@ pub trait Iterator { /// assert_eq!(v_cloned, vec![1, 2, 3]); /// assert_eq!(v_map, vec![1, 2, 3]); /// ``` + /// + /// To get the best performance, try to clone late: + /// + /// ``` + /// let a = [vec![0_u8, 1, 2], vec![3, 4], vec![23]]; + /// // don't do this: + /// let slower: Vec<_> = a.iter().cloned().filter(|s| s.len() == 1).collect(); + /// assert_eq!(&[vec![23]], &slower[..]); + /// // instead call `cloned` late + /// let faster: Vec<_> = a.iter().filter(|s| s.len() == 1).cloned().collect(); + /// assert_eq!(&[vec![23]], &faster[..]); + /// ``` #[stable(feature = "rust1", since = "1.0.0")] fn cloned<'a, T: 'a>(self) -> Cloned where @@ -2706,9 +3305,8 @@ pub trait Iterator { /// /// Instead of stopping at [`None`], the iterator will instead start again, /// from the beginning. After iterating again, it will start at the - /// beginning again. And again. And again. Forever. - /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None + /// beginning again. And again. And again. Forever. Note that in case the + /// original iterator is empty, the resulting iterator will also be empty. /// /// # Examples /// @@ -2736,6 +3334,49 @@ pub trait Iterator { Cycle::new(self) } + /// Returns an iterator over `N` elements of the iterator at a time. + /// + /// The chunks do not overlap. If `N` does not divide the length of the + /// iterator, then the last up to `N-1` elements will be omitted and can be + /// retrieved from the [`.into_remainder()`][ArrayChunks::into_remainder] + /// function of the iterator. + /// + /// # Panics + /// + /// Panics if `N` is 0. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(iter_array_chunks)] + /// + /// let mut iter = "lorem".chars().array_chunks(); + /// assert_eq!(iter.next(), Some(['l', 'o'])); + /// assert_eq!(iter.next(), Some(['r', 'e'])); + /// assert_eq!(iter.next(), None); + /// assert_eq!(iter.into_remainder().unwrap().as_slice(), &['m']); + /// ``` + /// + /// ``` + /// #![feature(iter_array_chunks)] + /// + /// let data = [1, 1, 2, -2, 6, 0, 3, 1]; + /// // ^-----^ ^------^ + /// for [x, y, z] in data.iter().array_chunks() { + /// assert_eq!(x + y + z, 4); + /// } + /// ``` + #[track_caller] + #[unstable(feature = "iter_array_chunks", reason = "recently added", issue = "100450")] + fn array_chunks(self) -> ArrayChunks + where + Self: Sized, + { + ArrayChunks::new(self) + } + /// Sums the elements of an iterator. /// /// Takes each element, adds them together, and returns the result. @@ -2796,7 +3437,7 @@ pub trait Iterator { Product::product(self) } - /// Lexicographically compares the elements of this `Iterator` with those + /// [Lexicographically](Ord#lexicographical-comparison) compares the elements of this [`Iterator`] with those /// of another. /// /// # Examples @@ -2818,7 +3459,7 @@ pub trait Iterator { self.cmp_by(other, |x, y| x.cmp(&y)) } - /// Lexicographically compares the elements of this `Iterator` with those + /// [Lexicographically](Ord#lexicographical-comparison) compares the elements of this [`Iterator`] with those /// of another with respect to the specified comparison function. /// /// # Examples @@ -2838,39 +3479,30 @@ pub trait Iterator { /// assert_eq!(xs.iter().cmp_by(&ys, |&x, &y| (2 * x).cmp(&y)), Ordering::Greater); /// ``` #[unstable(feature = "iter_order_by", issue = "64295")] - fn cmp_by(mut self, other: I, mut cmp: F) -> Ordering + fn cmp_by(self, other: I, cmp: F) -> Ordering where Self: Sized, I: IntoIterator, F: FnMut(Self::Item, I::Item) -> Ordering, { - let mut other = other.into_iter(); - - loop { - let x = match self.next() { - None => { - if other.next().is_none() { - return Ordering::Equal; - } else { - return Ordering::Less; - } - } - Some(val) => val, - }; - - let y = match other.next() { - None => return Ordering::Greater, - Some(val) => val, - }; - - match cmp(x, y) { - Ordering::Equal => (), - non_eq => return non_eq, + #[inline] + fn compare(mut cmp: F) -> impl FnMut(X, Y) -> ControlFlow + where + F: FnMut(X, Y) -> Ordering, + { + move |x, y| match cmp(x, y) { + Ordering::Equal => ControlFlow::CONTINUE, + non_eq => ControlFlow::Break(non_eq), } } + + match iter_compare(self, other.into_iter(), compare(cmp)) { + ControlFlow::Continue(ord) => ord, + ControlFlow::Break(ord) => ord, + } } - /// Lexicographically compares the elements of this `Iterator` with those + /// [Lexicographically](Ord#lexicographical-comparison) compares the elements of this [`Iterator`] with those /// of another. /// /// # Examples @@ -2882,7 +3514,7 @@ pub trait Iterator { /// assert_eq!([1.].iter().partial_cmp([1., 2.].iter()), Some(Ordering::Less)); /// assert_eq!([1., 2.].iter().partial_cmp([1.].iter()), Some(Ordering::Greater)); /// - /// assert_eq!([std::f64::NAN].iter().partial_cmp([1.].iter()), None); + /// assert_eq!([f64::NAN].iter().partial_cmp([1.].iter()), None); /// ``` #[stable(feature = "iter_order", since = "1.5.0")] fn partial_cmp(self, other: I) -> Option @@ -2894,7 +3526,7 @@ pub trait Iterator { self.partial_cmp_by(other, |x, y| x.partial_cmp(&y)) } - /// Lexicographically compares the elements of this `Iterator` with those + /// [Lexicographically](Ord#lexicographical-comparison) compares the elements of this [`Iterator`] with those /// of another with respect to the specified comparison function. /// /// # Examples @@ -2923,39 +3555,30 @@ pub trait Iterator { /// ); /// ``` #[unstable(feature = "iter_order_by", issue = "64295")] - fn partial_cmp_by(mut self, other: I, mut partial_cmp: F) -> Option + fn partial_cmp_by(self, other: I, partial_cmp: F) -> Option where Self: Sized, I: IntoIterator, F: FnMut(Self::Item, I::Item) -> Option, { - let mut other = other.into_iter(); - - loop { - let x = match self.next() { - None => { - if other.next().is_none() { - return Some(Ordering::Equal); - } else { - return Some(Ordering::Less); - } - } - Some(val) => val, - }; - - let y = match other.next() { - None => return Some(Ordering::Greater), - Some(val) => val, - }; - - match partial_cmp(x, y) { - Some(Ordering::Equal) => (), - non_eq => return non_eq, + #[inline] + fn compare(mut partial_cmp: F) -> impl FnMut(X, Y) -> ControlFlow> + where + F: FnMut(X, Y) -> Option, + { + move |x, y| match partial_cmp(x, y) { + Some(Ordering::Equal) => ControlFlow::CONTINUE, + non_eq => ControlFlow::Break(non_eq), } } + + match iter_compare(self, other.into_iter(), compare(partial_cmp)) { + ControlFlow::Continue(ord) => Some(ord), + ControlFlow::Break(ord) => ord, + } } - /// Determines if the elements of this `Iterator` are equal to those of + /// Determines if the elements of this [`Iterator`] are equal to those of /// another. /// /// # Examples @@ -2974,7 +3597,7 @@ pub trait Iterator { self.eq_by(other, |x, y| x == y) } - /// Determines if the elements of this `Iterator` are equal to those of + /// Determines if the elements of this [`Iterator`] are equal to those of /// another with respect to the specified equality function. /// /// # Examples @@ -2990,32 +3613,29 @@ pub trait Iterator { /// assert!(xs.iter().eq_by(&ys, |&x, &y| x * x == y)); /// ``` #[unstable(feature = "iter_order_by", issue = "64295")] - fn eq_by(mut self, other: I, mut eq: F) -> bool + fn eq_by(self, other: I, eq: F) -> bool where Self: Sized, I: IntoIterator, F: FnMut(Self::Item, I::Item) -> bool, { - let mut other = other.into_iter(); - - loop { - let x = match self.next() { - None => return other.next().is_none(), - Some(val) => val, - }; - - let y = match other.next() { - None => return false, - Some(val) => val, - }; - - if !eq(x, y) { - return false; + #[inline] + fn compare(mut eq: F) -> impl FnMut(X, Y) -> ControlFlow<()> + where + F: FnMut(X, Y) -> bool, + { + move |x, y| { + if eq(x, y) { ControlFlow::CONTINUE } else { ControlFlow::BREAK } } } + + match iter_compare(self, other.into_iter(), compare(eq)) { + ControlFlow::Continue(ord) => ord == Ordering::Equal, + ControlFlow::Break(()) => false, + } } - /// Determines if the elements of this `Iterator` are unequal to those of + /// Determines if the elements of this [`Iterator`] are unequal to those of /// another. /// /// # Examples @@ -3034,7 +3654,7 @@ pub trait Iterator { !self.eq(other) } - /// Determines if the elements of this `Iterator` are lexicographically + /// Determines if the elements of this [`Iterator`] are [lexicographically](Ord#lexicographical-comparison) /// less than those of another. /// /// # Examples @@ -3043,6 +3663,7 @@ pub trait Iterator { /// assert_eq!([1].iter().lt([1].iter()), false); /// assert_eq!([1].iter().lt([1, 2].iter()), true); /// assert_eq!([1, 2].iter().lt([1].iter()), false); + /// assert_eq!([1, 2].iter().lt([1, 2].iter()), false); /// ``` #[stable(feature = "iter_order", since = "1.5.0")] fn lt(self, other: I) -> bool @@ -3054,7 +3675,7 @@ pub trait Iterator { self.partial_cmp(other) == Some(Ordering::Less) } - /// Determines if the elements of this `Iterator` are lexicographically + /// Determines if the elements of this [`Iterator`] are [lexicographically](Ord#lexicographical-comparison) /// less or equal to those of another. /// /// # Examples @@ -3063,6 +3684,7 @@ pub trait Iterator { /// assert_eq!([1].iter().le([1].iter()), true); /// assert_eq!([1].iter().le([1, 2].iter()), true); /// assert_eq!([1, 2].iter().le([1].iter()), false); + /// assert_eq!([1, 2].iter().le([1, 2].iter()), true); /// ``` #[stable(feature = "iter_order", since = "1.5.0")] fn le(self, other: I) -> bool @@ -3071,10 +3693,10 @@ pub trait Iterator { Self::Item: PartialOrd, Self: Sized, { - matches!(self.partial_cmp(other), Some(Ordering::Less) | Some(Ordering::Equal)) + matches!(self.partial_cmp(other), Some(Ordering::Less | Ordering::Equal)) } - /// Determines if the elements of this `Iterator` are lexicographically + /// Determines if the elements of this [`Iterator`] are [lexicographically](Ord#lexicographical-comparison) /// greater than those of another. /// /// # Examples @@ -3083,6 +3705,7 @@ pub trait Iterator { /// assert_eq!([1].iter().gt([1].iter()), false); /// assert_eq!([1].iter().gt([1, 2].iter()), false); /// assert_eq!([1, 2].iter().gt([1].iter()), true); + /// assert_eq!([1, 2].iter().gt([1, 2].iter()), false); /// ``` #[stable(feature = "iter_order", since = "1.5.0")] fn gt(self, other: I) -> bool @@ -3094,7 +3717,7 @@ pub trait Iterator { self.partial_cmp(other) == Some(Ordering::Greater) } - /// Determines if the elements of this `Iterator` are lexicographically + /// Determines if the elements of this [`Iterator`] are [lexicographically](Ord#lexicographical-comparison) /// greater than or equal to those of another. /// /// # Examples @@ -3103,6 +3726,7 @@ pub trait Iterator { /// assert_eq!([1].iter().ge([1].iter()), true); /// assert_eq!([1].iter().ge([1, 2].iter()), false); /// assert_eq!([1, 2].iter().ge([1].iter()), true); + /// assert_eq!([1, 2].iter().ge([1, 2].iter()), true); /// ``` #[stable(feature = "iter_order", since = "1.5.0")] fn ge(self, other: I) -> bool @@ -3111,7 +3735,7 @@ pub trait Iterator { Self::Item: PartialOrd, Self: Sized, { - matches!(self.partial_cmp(other), Some(Ordering::Greater) | Some(Ordering::Equal)) + matches!(self.partial_cmp(other), Some(Ordering::Greater | Ordering::Equal)) } /// Checks if the elements of this iterator are sorted. @@ -3132,7 +3756,7 @@ pub trait Iterator { /// assert!(![1, 3, 2, 4].iter().is_sorted()); /// assert!([0].iter().is_sorted()); /// assert!(std::iter::empty::().is_sorted()); - /// assert!(![0.0, 1.0, std::f32::NAN].iter().is_sorted()); + /// assert!(![0.0, 1.0, f32::NAN].iter().is_sorted()); /// ``` #[inline] #[unstable(feature = "is_sorted", reason = "new API", issue = "53485")] @@ -3159,29 +3783,36 @@ pub trait Iterator { /// assert!(![1, 3, 2, 4].iter().is_sorted_by(|a, b| a.partial_cmp(b))); /// assert!([0].iter().is_sorted_by(|a, b| a.partial_cmp(b))); /// assert!(std::iter::empty::().is_sorted_by(|a, b| a.partial_cmp(b))); - /// assert!(![0.0, 1.0, std::f32::NAN].iter().is_sorted_by(|a, b| a.partial_cmp(b))); + /// assert!(![0.0, 1.0, f32::NAN].iter().is_sorted_by(|a, b| a.partial_cmp(b))); /// ``` /// - /// [`is_sorted`]: trait.Iterator.html#method.is_sorted + /// [`is_sorted`]: Iterator::is_sorted #[unstable(feature = "is_sorted", reason = "new API", issue = "53485")] - fn is_sorted_by(mut self, mut compare: F) -> bool + fn is_sorted_by(mut self, compare: F) -> bool where Self: Sized, F: FnMut(&Self::Item, &Self::Item) -> Option, { + #[inline] + fn check<'a, T>( + last: &'a mut T, + mut compare: impl FnMut(&T, &T) -> Option + 'a, + ) -> impl FnMut(T) -> bool + 'a { + move |curr| { + if let Some(Ordering::Greater) | None = compare(&last, &curr) { + return false; + } + *last = curr; + true + } + } + let mut last = match self.next() { Some(e) => e, None => return true, }; - while let Some(curr) = self.next() { - if let Some(Ordering::Greater) | None = compare(&last, &curr) { - return false; - } - last = curr; - } - - true + self.all(check(&mut last, compare)) } /// Checks if the elements of this iterator are sorted using the given key extraction @@ -3191,7 +3822,7 @@ pub trait Iterator { /// the elements, as determined by `f`. Apart from that, it's equivalent to [`is_sorted`]; see /// its documentation for more information. /// - /// [`is_sorted`]: trait.Iterator.html#method.is_sorted + /// [`is_sorted`]: Iterator::is_sorted /// /// # Examples /// @@ -3211,31 +3842,74 @@ pub trait Iterator { { self.map(f).is_sorted() } + + /// See [TrustedRandomAccess][super::super::TrustedRandomAccess] + // The unusual name is to avoid name collisions in method resolution + // see #76479. + #[inline] + #[doc(hidden)] + #[unstable(feature = "trusted_random_access", issue = "none")] + unsafe fn __iterator_get_unchecked(&mut self, _idx: usize) -> Self::Item + where + Self: TrustedRandomAccessNoCoerce, + { + unreachable!("Always specialized"); + } } -/// Fold an iterator without having to provide an initial value. +/// Compares two iterators element-wise using the given function. +/// +/// If `ControlFlow::CONTINUE` is returned from the function, the comparison moves on to the next +/// elements of both iterators. Returning `ControlFlow::Break(x)` short-circuits the iteration and +/// returns `ControlFlow::Break(x)`. If one of the iterators runs out of elements, +/// `ControlFlow::Continue(ord)` is returned where `ord` is the result of comparing the lengths of +/// the iterators. +/// +/// Isolates the logic shared by ['cmp_by'](Iterator::cmp_by), +/// ['partial_cmp_by'](Iterator::partial_cmp_by), and ['eq_by'](Iterator::eq_by). #[inline] -fn fold1(mut it: I, f: F) -> Option +fn iter_compare(mut a: A, mut b: B, f: F) -> ControlFlow where - I: Iterator, - F: FnMut(I::Item, I::Item) -> I::Item, + A: Iterator, + B: Iterator, + F: FnMut(A::Item, B::Item) -> ControlFlow, { - // start with the first element as our selection. This avoids - // having to use `Option`s inside the loop, translating to a - // sizeable performance gain (6x in one case). - let first = it.next()?; - Some(it.fold(first, f)) + #[inline] + fn compare<'a, B, X, T>( + b: &'a mut B, + mut f: impl FnMut(X, B::Item) -> ControlFlow + 'a, + ) -> impl FnMut(X) -> ControlFlow> + 'a + where + B: Iterator, + { + move |x| match b.next() { + None => ControlFlow::Break(ControlFlow::Continue(Ordering::Greater)), + Some(y) => f(x, y).map_break(ControlFlow::Break), + } + } + + match a.try_for_each(compare(&mut b, f)) { + ControlFlow::Continue(()) => ControlFlow::Continue(match b.next() { + None => Ordering::Equal, + Some(_) => Ordering::Less, + }), + ControlFlow::Break(x) => x, + } } #[stable(feature = "rust1", since = "1.0.0")] impl Iterator for &mut I { type Item = I::Item; + #[inline] fn next(&mut self) -> Option { (**self).next() } fn size_hint(&self) -> (usize, Option) { (**self).size_hint() } + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + (**self).advance_by(n) + } fn nth(&mut self, n: usize) -> Option { (**self).nth(n) } diff --git a/crux-mir/lib/core/src/iter/traits/marker.rs b/crux-mir/lib/core/src/iter/traits/marker.rs index 404cc8449..da7537457 100644 --- a/crux-mir/lib/core/src/iter/traits/marker.rs +++ b/crux-mir/lib/core/src/iter/traits/marker.rs @@ -1,18 +1,19 @@ +use crate::iter::Step; + /// An iterator that always continues to yield `None` when exhausted. /// /// Calling next on a fused iterator that has returned `None` once is guaranteed /// to return [`None`] again. This trait should be implemented by all iterators -/// that behave this way because it allows optimizing [`Iterator::fuse`]. +/// that behave this way because it allows optimizing [`Iterator::fuse()`]. /// /// Note: In general, you should not use `FusedIterator` in generic bounds if -/// you need a fused iterator. Instead, you should just call [`Iterator::fuse`] +/// you need a fused iterator. Instead, you should just call [`Iterator::fuse()`] /// on the iterator. If the iterator is already fused, the additional [`Fuse`] /// wrapper will be a no-op with no performance penalty. /// -/// [`None`]: ../../std/option/enum.Option.html#variant.None -/// [`Iterator::fuse`]: ../../std/iter/trait.Iterator.html#method.fuse -/// [`Fuse`]: ../../std/iter/struct.Fuse.html +/// [`Fuse`]: crate::iter::Fuse #[stable(feature = "fused", since = "1.26.0")] +#[rustc_unsafe_specialization_marker] pub trait FusedIterator: Iterator {} #[stable(feature = "fused", since = "1.26.0")] @@ -24,21 +25,54 @@ impl FusedIterator for &mut I {} /// (lower bound is equal to upper bound), or the upper bound is [`None`]. /// The upper bound must only be [`None`] if the actual iterator length is /// larger than [`usize::MAX`]. In that case, the lower bound must be -/// [`usize::MAX`], resulting in a [`.size_hint`] of `(usize::MAX, None)`. +/// [`usize::MAX`], resulting in an [`Iterator::size_hint()`] of +/// `(usize::MAX, None)`. /// /// The iterator must produce exactly the number of elements it reported /// or diverge before reaching the end. /// /// # Safety /// -/// This trait must only be implemented when the contract is upheld. -/// Consumers of this trait must inspect [`.size_hint`]’s upper bound. -/// -/// [`None`]: ../../std/option/enum.Option.html#variant.None -/// [`usize::MAX`]: ../../std/usize/constant.MAX.html -/// [`.size_hint`]: ../../std/iter/trait.Iterator.html#method.size_hint +/// This trait must only be implemented when the contract is upheld. Consumers +/// of this trait must inspect [`Iterator::size_hint()`]’s upper bound. #[unstable(feature = "trusted_len", issue = "37572")] +#[rustc_unsafe_specialization_marker] pub unsafe trait TrustedLen: Iterator {} #[unstable(feature = "trusted_len", issue = "37572")] unsafe impl TrustedLen for &mut I {} + +/// An iterator that when yielding an item will have taken at least one element +/// from its underlying [`SourceIter`]. +/// +/// Calling any method that advances the iterator, e.g. [`next()`] or [`try_fold()`], +/// guarantees that for each step at least one value of the iterator's underlying source +/// has been moved out and the result of the iterator chain could be inserted +/// in its place, assuming structural constraints of the source allow such an insertion. +/// In other words this trait indicates that an iterator pipeline can be collected in place. +/// +/// The primary use of this trait is in-place iteration. Refer to the [`vec::in_place_collect`] +/// module documentation for more information. +/// +/// [`vec::in_place_collect`]: ../../../../alloc/vec/in_place_collect/index.html +/// [`SourceIter`]: crate::iter::SourceIter +/// [`next()`]: Iterator::next +/// [`try_fold()`]: Iterator::try_fold +#[unstable(issue = "none", feature = "inplace_iteration")] +#[doc(hidden)] +pub unsafe trait InPlaceIterable: Iterator {} + +/// A type that upholds all invariants of [`Step`]. +/// +/// The invariants of [`Step::steps_between()`] are a superset of the invariants +/// of [`TrustedLen`]. As such, [`TrustedLen`] is implemented for all range +/// types with the same generic type argument. +/// +/// # Safety +/// +/// The implementation of [`Step`] for the given type must guarantee all +/// invariants of all methods are upheld. See the [`Step`] trait's documentation +/// for details. Consumers are free to rely on the invariants in unsafe code. +#[unstable(feature = "trusted_step", issue = "85731")] +#[rustc_specialization_trait] +pub unsafe trait TrustedStep: Step {} diff --git a/crux-mir/lib/core/src/iter/traits/mod.rs b/crux-mir/lib/core/src/iter/traits/mod.rs index efd1580a5..ed0fb634d 100644 --- a/crux-mir/lib/core/src/iter/traits/mod.rs +++ b/crux-mir/lib/core/src/iter/traits/mod.rs @@ -5,11 +5,17 @@ mod exact_size; mod iterator; mod marker; -pub use self::accum::{Product, Sum}; -pub use self::collect::{Extend, FromIterator, IntoIterator}; -pub use self::double_ended::DoubleEndedIterator; -pub use self::exact_size::ExactSizeIterator; #[stable(feature = "rust1", since = "1.0.0")] -pub use self::iterator::Iterator; -#[stable(feature = "rust1", since = "1.0.0")] -pub use self::marker::{FusedIterator, TrustedLen}; +pub use self::{ + accum::{Product, Sum}, + collect::{Extend, FromIterator, IntoIterator}, + double_ended::DoubleEndedIterator, + exact_size::ExactSizeIterator, + iterator::Iterator, + marker::{FusedIterator, TrustedLen}, +}; + +#[unstable(issue = "none", feature = "inplace_iteration")] +pub use self::marker::InPlaceIterable; +#[unstable(feature = "trusted_step", issue = "85731")] +pub use self::marker::TrustedStep; diff --git a/crux-mir/lib/core/src/lib.rs b/crux-mir/lib/core/src/lib.rs index 4c36ed4e7..b85c89525 100644 --- a/crux-mir/lib/core/src/lib.rs +++ b/crux-mir/lib/core/src/lib.rs @@ -20,7 +20,7 @@ // FIXME: Fill me in with more detail when the interface settles //! This library is built on the assumption of a few existing symbols: //! -//! * `memcpy`, `memcmp`, `memset` - These are core memory routines which are +//! * `memcpy`, `memcmp`, `memset`, `strlen` - These are core memory routines which are //! often generated by LLVM. Additionally, this library can make explicit //! calls to these functions. Their signatures are the same as found in C. //! These functions are often provided by the system libc, but can also be @@ -38,110 +38,221 @@ //! which do not trigger a panic can be assured that this function is never //! called. The `lang` attribute is called `eh_personality`. -// Since libcore defines many fundamental lang items, all tests live in a -// separate crate, libcoretest, to avoid bizarre issues. +// Since core defines many fundamental lang items, all tests live in a +// separate crate, libcoretest (library/core/tests), to avoid bizarre issues. // // Here we explicitly #[cfg]-out this whole crate when testing. If we don't do // this, both the generated test artifact and the linked libtest (which -// transitively includes libcore) will both define the same set of lang items, +// transitively includes core) will both define the same set of lang items, // and this will cause the E0152 "found duplicate lang item" error. See // discussion in #50466 for details. // // This cfg won't affect doc tests. #![cfg(not(test))] +// To run core tests without x.py without ending up with two copies of core, Miri needs to be +// able to "empty" this crate. See . +// rustc itself never sets the feature, so this line has no affect there. +#![cfg(any(not(feature = "miri-test-libstd"), test, doctest))] #![stable(feature = "core", since = "1.6.0")] #![doc( - html_root_url = "https://doc.rust-lang.org/nightly/", html_playground_url = "https://play.rust-lang.org/", issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/", test(no_crate_inject, attr(deny(warnings))), test(attr(allow(dead_code, deprecated, unused_variables, unused_mut))) )] +#![doc(cfg_hide( + not(test), + any(not(feature = "miri-test-libstd"), test, doctest), + no_fp_fmt_parse, + target_pointer_width = "16", + target_pointer_width = "32", + target_pointer_width = "64", + target_has_atomic = "8", + target_has_atomic = "16", + target_has_atomic = "32", + target_has_atomic = "64", + target_has_atomic = "ptr", + target_has_atomic_equal_alignment = "8", + target_has_atomic_equal_alignment = "16", + target_has_atomic_equal_alignment = "32", + target_has_atomic_equal_alignment = "64", + target_has_atomic_equal_alignment = "ptr", + target_has_atomic_load_store = "8", + target_has_atomic_load_store = "16", + target_has_atomic_load_store = "32", + target_has_atomic_load_store = "64", + target_has_atomic_load_store = "ptr", +))] #![no_core] +#![rustc_coherence_is_core] +// +// Lints: +#![deny(rust_2021_incompatible_or_patterns)] +#![deny(unsafe_op_in_unsafe_fn)] +#![deny(fuzzy_provenance_casts)] #![warn(deprecated_in_future)] -#![warn(missing_docs)] #![warn(missing_debug_implementations)] -#![deny(intra_doc_link_resolution_failure)] // rustdoc is run without -D warnings +#![warn(missing_docs)] #![allow(explicit_outlives_requirements)] #![allow(incomplete_features)] -#![feature(allow_internal_unstable)] -#![feature(arbitrary_self_types)] -#![feature(asm)] -#![feature(bound_cloned)] -#![feature(cfg_target_has_atomic)] -#![feature(concat_idents)] -#![feature(const_ascii_ctype_on_intrinsics)] +// +// Library features: +#![feature(const_align_offset)] +#![feature(const_align_of_val)] +#![feature(const_align_of_val_raw)] #![feature(const_alloc_layout)] +#![feature(const_arguments_as_str)] +#![feature(const_array_into_iter_constructors)] +#![feature(const_bigint_helper_methods)] +#![feature(const_black_box)] +#![feature(const_caller_location)] +#![feature(const_cell_into_inner)] +#![feature(const_char_from_u32_unchecked)] +#![feature(const_clone)] +#![feature(const_cmp)] #![feature(const_discriminant)] -#![feature(const_if_match)] -#![feature(const_loop)] -#![feature(const_checked_int_methods)] -#![feature(const_euclidean_int_methods)] -#![feature(const_overflowing_int_methods)] -#![feature(const_saturating_int_methods)] +#![feature(const_eval_select)] +#![feature(const_exact_div)] +#![feature(const_float_bits_conv)] +#![feature(const_float_classify)] +#![feature(const_fmt_arguments_new)] +#![feature(const_hash)] +#![feature(const_heap)] +#![feature(const_convert)] +#![feature(const_index_range_slice_index)] +#![feature(const_inherent_unchecked_arith)] #![feature(const_int_unchecked_arith)] -#![feature(const_int_pow)] -#![feature(constctlz)] -#![feature(const_panic)] -#![feature(const_fn_union)] -#![feature(const_generics)] -#![feature(const_ptr_offset_from)] -#![feature(const_result)] +#![feature(const_intrinsic_forget)] +#![feature(const_likely)] +#![feature(const_maybe_uninit_uninit_array)] +#![feature(const_maybe_uninit_as_mut_ptr)] +#![feature(const_maybe_uninit_assume_init)] +#![feature(const_nonnull_new)] +#![feature(const_num_from_num)] +#![feature(const_ops)] +#![feature(const_option)] +#![feature(const_option_ext)] +#![feature(const_pin)] +#![feature(const_pointer_is_aligned)] +#![feature(const_ptr_sub_ptr)] +#![feature(const_replace)] +#![feature(const_result_drop)] +#![feature(const_ptr_as_ref)] +#![feature(const_ptr_is_null)] +#![feature(const_ptr_read)] +#![feature(const_ptr_write)] +#![feature(const_raw_ptr_comparison)] +#![feature(const_size_of_val)] +#![feature(const_size_of_val_raw)] +#![feature(const_slice_from_raw_parts_mut)] +#![feature(const_slice_ptr_len)] +#![feature(const_slice_split_at_mut)] +#![feature(const_str_from_utf8_unchecked_mut)] +#![feature(const_swap)] +#![feature(const_trait_impl)] +#![feature(const_try)] +#![feature(const_type_id)] #![feature(const_type_name)] -#![feature(custom_inner_attributes)] +#![feature(const_default_impls)] +#![feature(const_unicode_case_lookup)] +#![feature(const_unsafecell_get_mut)] +#![feature(const_waker)] +#![feature(core_panic)] +#![feature(char_indices_offset)] +#![feature(duration_consts_float)] +#![feature(maybe_uninit_uninit_array)] +#![feature(ptr_alignment_type)] +#![feature(ptr_metadata)] +#![feature(set_ptr_value)] +#![feature(slice_ptr_get)] +#![feature(slice_split_at_unchecked)] +#![feature(str_internals)] +#![feature(str_split_remainder)] +#![feature(str_split_inclusive_remainder)] +#![feature(strict_provenance)] +#![feature(utf16_extra)] +#![feature(utf16_extra_const)] +#![feature(variant_count)] +#![feature(const_array_from_ref)] +#![feature(const_slice_from_ref)] +#![feature(const_slice_index)] +#![feature(const_is_char_boundary)] +#![feature(const_cstr_methods)] +#![feature(is_ascii_octdigit)] +// +// Language features: +#![feature(abi_unadjusted)] +#![feature(adt_const_params)] +#![feature(allow_internal_unsafe)] +#![feature(allow_internal_unstable)] +#![feature(associated_type_bounds)] +#![feature(auto_traits)] +#![feature(c_unwind)] +#![feature(cfg_sanitize)] +#![feature(cfg_target_has_atomic)] +#![feature(cfg_target_has_atomic_equal_alignment)] +#![cfg_attr(not(bootstrap), feature(const_closures))] +#![feature(const_fn_floating_point_arithmetic)] +#![feature(const_mut_refs)] +#![feature(const_precise_live_drops)] +#![feature(const_refs_to_cell)] #![feature(decl_macro)] +#![feature(deprecated_suggestion)] +#![feature(derive_const)] #![feature(doc_cfg)] +#![feature(doc_notable_trait)] +#![feature(rustdoc_internals)] +#![feature(exhaustive_patterns)] +#![feature(doc_cfg_hide)] #![feature(extern_types)] #![feature(fundamental)] +#![feature(if_let_guard)] +#![feature(inline_const)] +#![feature(intra_doc_pointers)] #![feature(intrinsics)] -#![feature(try_find)] -#![feature(is_sorted)] #![feature(lang_items)] #![feature(link_llvm_intrinsics)] +#![feature(macro_metavar_expr)] +#![feature(min_specialization)] +#![feature(must_not_suspend)] +#![feature(negative_impls)] #![feature(never_type)] -#![feature(nll)] -#![feature(exhaustive_patterns)] #![feature(no_core)] -#![feature(optin_builtin_traits)] +#![feature(no_coverage)] // rust-lang/rust#84605 +#![feature(platform_intrinsics)] #![feature(prelude_import)] -#![feature(repr_simd, platform_intrinsics)] +#![feature(repr_simd)] +#![feature(rustc_allow_const_fn_unstable)] #![feature(rustc_attrs)] #![feature(simd_ffi)] -#![feature(specialization)] #![feature(staged_api)] -#![feature(std_internals)] #![feature(stmt_expr_attributes)] -#![feature(track_caller)] +#![feature(target_feature_11)] +#![feature(trait_alias)] #![feature(transparent_unions)] +#![feature(try_blocks)] #![feature(unboxed_closures)] -#![feature(unsized_locals)] -#![feature(untagged_unions)] -#![feature(unwind_attributes)] -#![feature(doc_alias)] -#![feature(mmx_target_feature)] -#![feature(tbm_target_feature)] -#![feature(sse4a_target_feature)] +#![feature(unsized_fn_params)] +#![feature(asm_const)] +#![feature(const_transmute_copy)] +// +// Target features: #![feature(arm_target_feature)] -#![feature(powerpc_target_feature)] -#![feature(mips_target_feature)] -#![feature(aarch64_target_feature)] -#![feature(wasm_target_feature)] #![feature(avx512_target_feature)] #![feature(cmpxchg16b_target_feature)] -#![feature(rtm_target_feature)] -#![feature(f16c_target_feature)] #![feature(hexagon_target_feature)] -#![feature(const_transmute)] -#![feature(structural_match)] -#![feature(abi_unadjusted)] -#![feature(adx_target_feature)] -#![feature(maybe_uninit_slice)] -#![feature(external_doc)] -#![feature(associated_type_bounds)] -#![feature(const_type_id)] -#![feature(const_caller_location)] -#![feature(option_zip)] -#![feature(no_niche)] // rust-lang/rust#68303 +#![feature(mips_target_feature)] +#![feature(powerpc_target_feature)] +#![feature(riscv_target_feature)] +#![feature(rtm_target_feature)] +#![feature(sse4a_target_feature)] +#![feature(tbm_target_feature)] +#![feature(wasm_target_feature)] +#![cfg_attr(bootstrap, feature(f16c_target_feature))] + +// allow using `core::` in intra-doc links +#[allow(unused_extern_crates)] +extern crate self as core; #[prelude_import] #[allow(unused)] @@ -151,37 +262,47 @@ use prelude::v1::*; #[macro_use] mod macros; +// We don't export this through #[macro_export] for now, to avoid breakage. +// See https://github.com/rust-lang/rust/issues/82913 +#[cfg(not(test))] +#[unstable(feature = "assert_matches", issue = "82775")] +/// Unstable module containing the unstable `assert_matches` macro. +pub mod assert_matches { + #[unstable(feature = "assert_matches", issue = "82775")] + pub use crate::macros::{assert_matches, debug_assert_matches}; +} + #[macro_use] mod internal_macros; -#[path = "num/int_macros.rs"] +#[path = "num/shells/int_macros.rs"] #[macro_use] mod int_macros; -#[path = "num/i128.rs"] +#[path = "num/shells/i128.rs"] pub mod i128; -#[path = "num/i16.rs"] +#[path = "num/shells/i16.rs"] pub mod i16; -#[path = "num/i32.rs"] +#[path = "num/shells/i32.rs"] pub mod i32; -#[path = "num/i64.rs"] +#[path = "num/shells/i64.rs"] pub mod i64; -#[path = "num/i8.rs"] +#[path = "num/shells/i8.rs"] pub mod i8; -#[path = "num/isize.rs"] +#[path = "num/shells/isize.rs"] pub mod isize; -#[path = "num/u128.rs"] +#[path = "num/shells/u128.rs"] pub mod u128; -#[path = "num/u16.rs"] +#[path = "num/shells/u16.rs"] pub mod u16; -#[path = "num/u32.rs"] +#[path = "num/shells/u32.rs"] pub mod u32; -#[path = "num/u64.rs"] +#[path = "num/shells/u64.rs"] pub mod u64; -#[path = "num/u8.rs"] +#[path = "num/shells/u8.rs"] pub mod u8; -#[path = "num/usize.rs"] +#[path = "num/shells/usize.rs"] pub mod usize; #[path = "num/f32.rs"] @@ -192,7 +313,7 @@ pub mod f64; #[macro_use] pub mod num; -/* The libcore prelude, not as all-encompassing as the libstd prelude */ +/* The core prelude, not as all-encompassing as the std prelude */ pub mod prelude; @@ -206,50 +327,42 @@ pub mod ptr; /* Core language traits */ pub mod borrow; -#[cfg(not(test))] // See #65860 pub mod clone; -#[cfg(not(test))] // See #65860 pub mod cmp; pub mod convert; -#[cfg(not(test))] // See #65860 pub mod default; -#[cfg(not(test))] // See #65860 +pub mod error; pub mod marker; pub mod ops; /* Core types and methods on primitives */ pub mod any; -#[cfg(not(test))] // See #65860 pub mod array; pub mod ascii; +pub mod asserting; +#[unstable(feature = "async_iterator", issue = "79024")] +pub mod async_iter; pub mod cell; pub mod char; pub mod ffi; -#[cfg(not(test))] // See #65860 pub mod iter; pub mod option; pub mod panic; pub mod panicking; -#[cfg(not(test))] // See #65860 pub mod pin; -pub mod raw; pub mod result; pub mod sync; -#[cfg(not(test))] // See #65860 pub mod fmt; -#[cfg(not(test))] // See #65860 pub mod hash; pub mod slice; -#[cfg(not(test))] // See #65860 pub mod str; pub mod time; pub mod unicode; /* Async */ -#[cfg(not(test))] // See #65860 pub mod future; pub mod task; @@ -262,22 +375,55 @@ mod bool; mod tuple; mod unit; +mod const_closure; + #[stable(feature = "core_primitive", since = "1.43.0")] pub mod primitive; -// Pull in the `core_arch` crate directly into libcore. The contents of +// Pull in the `core_arch` crate directly into core. The contents of // `core_arch` are in a different repository: rust-lang/stdarch. // -// `core_arch` depends on libcore, but the contents of this module are +// `core_arch` depends on core, but the contents of this module are // set up in such a way that directly pulling it here works such that the -// crate uses the this crate as its libcore. -#[path = "../stdarch/crates/core_arch/src/mod.rs"] -#[allow(missing_docs, missing_debug_implementations, dead_code, unused_imports)] +// crate uses the this crate as its core. +#[path = "../../stdarch/crates/core_arch/src/mod.rs"] +#[allow( + missing_docs, + missing_debug_implementations, + dead_code, + unused_imports, + unsafe_op_in_unsafe_fn +)] +#[allow(rustdoc::bare_urls)] +// FIXME: This annotation should be moved into rust-lang/stdarch after clashing_extern_declarations is +// merged. It currently cannot because bootstrap fails as the lint hasn't been defined yet. +#[allow(clashing_extern_declarations)] #[unstable(feature = "stdsimd", issue = "48556")] mod core_arch; #[stable(feature = "simd_arch", since = "1.27.0")] -pub use core_arch::arch; +pub mod arch; + +// Pull in the `core_simd` crate directly into core. The contents of +// `core_simd` are in a different repository: rust-lang/portable-simd. +// +// `core_simd` depends on core, but the contents of this module are +// set up in such a way that directly pulling it here works such that the +// crate uses this crate as its core. +#[path = "../../portable-simd/crates/core_simd/src/mod.rs"] +#[allow(missing_debug_implementations, dead_code, unsafe_op_in_unsafe_fn, unused_unsafe)] +#[allow(rustdoc::bare_urls)] +#[unstable(feature = "portable_simd", issue = "86656")] +mod core_simd; + +#[doc = include_str!("../../portable-simd/crates/core_simd/src/core_simd_docs.md")] +#[unstable(feature = "portable_simd", issue = "86656")] +pub mod simd { + #[unstable(feature = "portable_simd", issue = "86656")] + pub use crate::core_simd::simd::*; +} + +include!("primitive_docs.rs"); #[unstable(feature = "crucible_intrinsics", issue = "none")] -pub mod crucible; +pub mod crucible; \ No newline at end of file diff --git a/crux-mir/lib/core/src/macros/mod.rs b/crux-mir/lib/core/src/macros/mod.rs index a0873fe6b..3b026bc0e 100644 --- a/crux-mir/lib/core/src/macros/mod.rs +++ b/crux-mir/lib/core/src/macros/mod.rs @@ -1,44 +1,15 @@ -#[cfg(bootstrap)] -#[doc(include = "panic.md")] +#[doc = include_str!("panic.md")] #[macro_export] -#[allow_internal_unstable(core_panic, track_caller)] +#[rustc_builtin_macro(core_panic)] +#[allow_internal_unstable(edition_panic)] #[stable(feature = "core", since = "1.6.0")] +#[rustc_diagnostic_item = "core_panic_macro"] macro_rules! panic { - () => ( - $crate::panic!("explicit panic") - ); - ($msg:expr) => ( - $crate::panicking::panic($msg) - ); - ($msg:expr,) => ( - $crate::panic!($msg) - ); - ($fmt:expr, $($arg:tt)+) => ( - $crate::panicking::panic_fmt( - $crate::format_args!($fmt, $($arg)+), - $crate::panic::Location::caller(), - ) - ); -} - -#[cfg(not(bootstrap))] -#[doc(include = "panic.md")] -#[macro_export] -#[allow_internal_unstable(core_panic, track_caller)] -#[stable(feature = "core", since = "1.6.0")] -macro_rules! panic { - () => ( - $crate::panic!("explicit panic") - ); - ($msg:expr) => ( - $crate::panicking::panic($msg) - ); - ($msg:expr,) => ( - $crate::panic!($msg) - ); - ($fmt:expr, $($arg:tt)+) => ( - $crate::panicking::panic_fmt($crate::format_args!($fmt, $($arg)+)) - ); + // Expands to either `$crate::panic::panic_2015` or `$crate::panic::panic_2021` + // depending on the edition of the caller. + ($($arg:tt)*) => { + /* compiler built-in */ + }; } /// Asserts that two expressions are equal to each other (using [`PartialEq`]). @@ -49,9 +20,6 @@ macro_rules! panic { /// Like [`assert!`], this macro has a second form, where a custom /// panic message can be provided. /// -/// [`PartialEq`]: cmp/trait.PartialEq.html -/// [`assert!`]: macro.assert.html -/// /// # Examples /// /// ``` @@ -63,39 +31,35 @@ macro_rules! panic { /// ``` #[macro_export] #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "assert_eq_macro")] +#[allow_internal_unstable(core_panic)] macro_rules! assert_eq { - ($left:expr, $right:expr) => ({ + ($left:expr, $right:expr $(,)?) => { match (&$left, &$right) { (left_val, right_val) => { if !(*left_val == *right_val) { + let kind = $crate::panicking::AssertKind::Eq; // The reborrows below are intentional. Without them, the stack slot for the // borrow is initialized even before the values are compared, leading to a // noticeable slow down. - panic!(r#"assertion failed: `(left == right)` - left: `{:?}`, - right: `{:?}`"#, &*left_val, &*right_val) + $crate::panicking::assert_failed(kind, &*left_val, &*right_val, $crate::option::Option::None); } } } - }); - ($left:expr, $right:expr,) => ({ - $crate::assert_eq!($left, $right) - }); - ($left:expr, $right:expr, $($arg:tt)+) => ({ - match (&($left), &($right)) { + }; + ($left:expr, $right:expr, $($arg:tt)+) => { + match (&$left, &$right) { (left_val, right_val) => { if !(*left_val == *right_val) { + let kind = $crate::panicking::AssertKind::Eq; // The reborrows below are intentional. Without them, the stack slot for the // borrow is initialized even before the values are compared, leading to a // noticeable slow down. - panic!(r#"assertion failed: `(left == right)` - left: `{:?}`, - right: `{:?}`: {}"#, &*left_val, &*right_val, - $crate::format_args!($($arg)+)) + $crate::panicking::assert_failed(kind, &*left_val, &*right_val, $crate::option::Option::Some($crate::format_args!($($arg)+))); } } } - }); + }; } /// Asserts that two expressions are not equal to each other (using [`PartialEq`]). @@ -106,9 +70,6 @@ macro_rules! assert_eq { /// Like [`assert!`], this macro has a second form, where a custom /// panic message can be provided. /// -/// [`PartialEq`]: cmp/trait.PartialEq.html -/// [`assert!`]: macro.assert.html -/// /// # Examples /// /// ``` @@ -120,39 +81,91 @@ macro_rules! assert_eq { /// ``` #[macro_export] #[stable(feature = "assert_ne", since = "1.13.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "assert_ne_macro")] +#[allow_internal_unstable(core_panic)] macro_rules! assert_ne { - ($left:expr, $right:expr) => ({ + ($left:expr, $right:expr $(,)?) => { match (&$left, &$right) { (left_val, right_val) => { if *left_val == *right_val { + let kind = $crate::panicking::AssertKind::Ne; // The reborrows below are intentional. Without them, the stack slot for the // borrow is initialized even before the values are compared, leading to a // noticeable slow down. - panic!(r#"assertion failed: `(left != right)` - left: `{:?}`, - right: `{:?}`"#, &*left_val, &*right_val) + $crate::panicking::assert_failed(kind, &*left_val, &*right_val, $crate::option::Option::None); } } } - }); - ($left:expr, $right:expr,) => { - $crate::assert_ne!($left, $right) }; - ($left:expr, $right:expr, $($arg:tt)+) => ({ + ($left:expr, $right:expr, $($arg:tt)+) => { match (&($left), &($right)) { (left_val, right_val) => { if *left_val == *right_val { + let kind = $crate::panicking::AssertKind::Ne; // The reborrows below are intentional. Without them, the stack slot for the // borrow is initialized even before the values are compared, leading to a // noticeable slow down. - panic!(r#"assertion failed: `(left != right)` - left: `{:?}`, - right: `{:?}`: {}"#, &*left_val, &*right_val, - $crate::format_args!($($arg)+)) + $crate::panicking::assert_failed(kind, &*left_val, &*right_val, $crate::option::Option::Some($crate::format_args!($($arg)+))); } } } - }); + }; +} + +/// Asserts that an expression matches any of the given patterns. +/// +/// Like in a `match` expression, the pattern can be optionally followed by `if` +/// and a guard expression that has access to names bound by the pattern. +/// +/// On panic, this macro will print the value of the expression with its +/// debug representation. +/// +/// Like [`assert!`], this macro has a second form, where a custom +/// panic message can be provided. +/// +/// # Examples +/// +/// ``` +/// #![feature(assert_matches)] +/// +/// use std::assert_matches::assert_matches; +/// +/// let a = 1u32.checked_add(2); +/// let b = 1u32.checked_sub(2); +/// assert_matches!(a, Some(_)); +/// assert_matches!(b, None); +/// +/// let c = Ok("abc".to_string()); +/// assert_matches!(c, Ok(x) | Err(x) if x.len() < 100); +/// ``` +#[unstable(feature = "assert_matches", issue = "82775")] +#[allow_internal_unstable(core_panic)] +#[rustc_macro_transparency = "semitransparent"] +pub macro assert_matches { + ($left:expr, $(|)? $( $pattern:pat_param )|+ $( if $guard: expr )? $(,)?) => { + match $left { + $( $pattern )|+ $( if $guard )? => {} + ref left_val => { + $crate::panicking::assert_matches_failed( + left_val, + $crate::stringify!($($pattern)|+ $(if $guard)?), + $crate::option::Option::None + ); + } + } + }, + ($left:expr, $(|)? $( $pattern:pat_param )|+ $( if $guard: expr )?, $($arg:tt)+) => { + match $left { + $( $pattern )|+ $( if $guard )? => {} + ref left_val => { + $crate::panicking::assert_matches_failed( + left_val, + $crate::stringify!($($pattern)|+ $(if $guard)?), + $crate::option::Option::Some($crate::format_args!($($arg)+)) + ); + } + } + }, } /// Asserts that a boolean expression is `true` at runtime. @@ -175,13 +188,10 @@ macro_rules! assert_ne { /// An unchecked assertion allows a program in an inconsistent state to keep /// running, which might have unexpected consequences but does not introduce /// unsafety as long as this only happens in safe code. The performance cost -/// of assertions, is however, not measurable in general. Replacing [`assert!`] +/// of assertions, however, is not measurable in general. Replacing [`assert!`] /// with `debug_assert!` is thus only encouraged after thorough profiling, and /// more importantly, only in safe code! /// -/// [`panic!`]: macro.panic.html -/// [`assert!`]: macro.assert.html -/// /// # Examples /// /// ``` @@ -201,8 +211,14 @@ macro_rules! assert_ne { /// ``` #[macro_export] #[stable(feature = "rust1", since = "1.0.0")] +#[rustc_diagnostic_item = "debug_assert_macro"] +#[allow_internal_unstable(edition_panic)] macro_rules! debug_assert { - ($($arg:tt)*) => (if $crate::cfg!(debug_assertions) { $crate::assert!($($arg)*); }) + ($($arg:tt)*) => { + if $crate::cfg!(debug_assertions) { + $crate::assert!($($arg)*); + } + }; } /// Asserts that two expressions are equal to each other. @@ -217,8 +233,6 @@ macro_rules! debug_assert { /// expensive to be present in a release build but may be helpful during /// development. The result of expanding `debug_assert_eq!` is always type checked. /// -/// [`assert_eq!`]: ../std/macro.assert_eq.html -/// /// # Examples /// /// ``` @@ -228,8 +242,13 @@ macro_rules! debug_assert { /// ``` #[macro_export] #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "debug_assert_eq_macro")] macro_rules! debug_assert_eq { - ($($arg:tt)*) => (if $crate::cfg!(debug_assertions) { $crate::assert_eq!($($arg)*); }) + ($($arg:tt)*) => { + if $crate::cfg!(debug_assertions) { + $crate::assert_eq!($($arg)*); + } + }; } /// Asserts that two expressions are not equal to each other. @@ -244,8 +263,6 @@ macro_rules! debug_assert_eq { /// expensive to be present in a release build but may be helpful during /// development. The result of expanding `debug_assert_ne!` is always type checked. /// -/// [`assert_ne!`]: ../std/macro.assert_ne.html -/// /// # Examples /// /// ``` @@ -255,8 +272,54 @@ macro_rules! debug_assert_eq { /// ``` #[macro_export] #[stable(feature = "assert_ne", since = "1.13.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "debug_assert_ne_macro")] macro_rules! debug_assert_ne { - ($($arg:tt)*) => (if $crate::cfg!(debug_assertions) { $crate::assert_ne!($($arg)*); }) + ($($arg:tt)*) => { + if $crate::cfg!(debug_assertions) { + $crate::assert_ne!($($arg)*); + } + }; +} + +/// Asserts that an expression matches any of the given patterns. +/// +/// Like in a `match` expression, the pattern can be optionally followed by `if` +/// and a guard expression that has access to names bound by the pattern. +/// +/// On panic, this macro will print the value of the expression with its +/// debug representation. +/// +/// Unlike [`assert_matches!`], `debug_assert_matches!` statements are only +/// enabled in non optimized builds by default. An optimized build will not +/// execute `debug_assert_matches!` statements unless `-C debug-assertions` is +/// passed to the compiler. This makes `debug_assert_matches!` useful for +/// checks that are too expensive to be present in a release build but may be +/// helpful during development. The result of expanding `debug_assert_matches!` +/// is always type checked. +/// +/// # Examples +/// +/// ``` +/// #![feature(assert_matches)] +/// +/// use std::assert_matches::debug_assert_matches; +/// +/// let a = 1u32.checked_add(2); +/// let b = 1u32.checked_sub(2); +/// debug_assert_matches!(a, Some(_)); +/// debug_assert_matches!(b, None); +/// +/// let c = Ok("abc".to_string()); +/// debug_assert_matches!(c, Ok(x) | Err(x) if x.len() < 100); +/// ``` +#[macro_export] +#[unstable(feature = "assert_matches", issue = "82775")] +#[allow_internal_unstable(assert_matches)] +#[rustc_macro_transparency = "semitransparent"] +pub macro debug_assert_matches($($arg:tt)*) { + if $crate::cfg!(debug_assertions) { + $crate::assert_matches::assert_matches!($($arg)*); + } } /// Returns whether the given expression matches any of the given patterns. @@ -275,21 +338,24 @@ macro_rules! debug_assert_ne { /// ``` #[macro_export] #[stable(feature = "matches_macro", since = "1.42.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "matches_macro")] macro_rules! matches { - ($expression:expr, $( $pattern:pat )|+ $( if $guard: expr )?) => { + ($expression:expr, $(|)? $( $pattern:pat_param )|+ $( if $guard: expr )? $(,)?) => { match $expression { $( $pattern )|+ $( if $guard )? => true, _ => false } - } + }; } /// Unwraps a result or propagates its error. /// -/// The `?` operator was added to replace `try!` and should be used instead. -/// Furthermore, `try` is a reserved word in Rust 2018, so if you must use -/// it, you will need to use the [raw-identifier syntax][ris]: `r#try`. +/// The [`?` operator][propagating-errors] was added to replace `try!` +/// and should be used instead. Furthermore, `try` is a reserved word +/// in Rust 2018, so if you must use it, you will need to use the +/// [raw-identifier syntax][ris]: `r#try`. /// +/// [propagating-errors]: https://doc.rust-lang.org/book/ch09-02-recoverable-errors-with-result.html#a-shortcut-for-propagating-errors-the--operator /// [ris]: https://doc.rust-lang.org/nightly/rust-by-example/compatibility/raw_identifiers.html /// /// `try!` matches the given [`Result`]. In case of the `Ok` variant, the @@ -303,8 +369,6 @@ macro_rules! matches { /// Because of the early return, `try!` can only be used in functions that /// return [`Result`]. /// -/// [`Result`]: ../std/result/enum.Result.html -/// /// # Examples /// /// ``` @@ -348,10 +412,10 @@ macro_rules! matches { /// ``` #[macro_export] #[stable(feature = "rust1", since = "1.0.0")] -#[rustc_deprecated(since = "1.39.0", reason = "use the `?` operator instead")] +#[deprecated(since = "1.39.0", note = "use the `?` operator instead")] #[doc(alias = "?")] macro_rules! r#try { - ($expr:expr) => { + ($expr:expr $(,)?) => { match $expr { $crate::result::Result::Ok(val) => val, $crate::result::Result::Err(err) => { @@ -359,26 +423,23 @@ macro_rules! r#try { } } }; - ($expr:expr,) => { - $crate::r#try!($expr) - }; } /// Writes formatted data into a buffer. /// -/// This macro accepts a format string, a list of arguments, and a 'writer'. Arguments will be +/// This macro accepts a 'writer', a format string, and a list of arguments. Arguments will be /// formatted according to the specified format string and the result will be passed to the writer. /// The writer may be any value with a `write_fmt` method; generally this comes from an -/// implementation of either the [`std::fmt::Write`] or the [`std::io::Write`] trait. The macro -/// returns whatever the `write_fmt` method returns; commonly a [`std::fmt::Result`], or an +/// implementation of either the [`fmt::Write`] or the [`io::Write`] trait. The macro +/// returns whatever the `write_fmt` method returns; commonly a [`fmt::Result`], or an /// [`io::Result`]. /// /// See [`std::fmt`] for more information on the format string syntax. /// /// [`std::fmt`]: ../std/fmt/index.html -/// [`std::fmt::Write`]: ../std/fmt/trait.Write.html -/// [`std::io::Write`]: ../std/io/trait.Write.html -/// [`std::fmt::Result`]: ../std/fmt/type.Result.html +/// [`fmt::Write`]: crate::fmt::Write +/// [`io::Write`]: ../std/io/trait.Write.html +/// [`fmt::Result`]: crate::fmt::Result /// [`io::Result`]: ../std/io/type.Result.html /// /// # Examples @@ -398,11 +459,12 @@ macro_rules! r#try { /// /// A module can import both `std::fmt::Write` and `std::io::Write` and call `write!` on objects /// implementing either, as objects do not typically implement both. However, the module must -/// import the traits qualified so their names do not conflict: +/// avoid conflict between the trait names, such as by importing them as `_` or otherwise renaming +/// them: /// /// ``` -/// use std::fmt::Write as FmtWrite; -/// use std::io::Write as IoWrite; +/// use std::fmt::Write as _; +/// use std::io::Write as _; /// /// fn main() -> Result<(), Box> { /// let mut s = String::new(); @@ -415,6 +477,23 @@ macro_rules! r#try { /// } /// ``` /// +/// If you also need the trait names themselves, such as to implement one or both on your types, +/// import the containing module and then name them with a prefix: +/// +/// ``` +/// # #![allow(unused_imports)] +/// use std::fmt::{self, Write as _}; +/// use std::io::{self, Write as _}; +/// +/// struct Example; +/// +/// impl fmt::Write for Example { +/// fn write_str(&mut self, _s: &str) -> core::fmt::Result { +/// unimplemented!(); +/// } +/// } +/// ``` +/// /// Note: This macro can be used in `no_std` setups as well. /// In a `no_std` setup you are responsible for the implementation details of the components. /// @@ -435,8 +514,11 @@ macro_rules! r#try { /// ``` #[macro_export] #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "write_macro")] macro_rules! write { - ($dst:expr, $($arg:tt)*) => ($dst.write_fmt($crate::format_args!($($arg)*))) + ($dst:expr, $($arg:tt)*) => { + $dst.write_fmt($crate::format_args!($($arg)*)) + }; } /// Write formatted data into a buffer, with a newline appended. @@ -447,10 +529,8 @@ macro_rules! write { /// For more information, see [`write!`]. For information on the format string syntax, see /// [`std::fmt`]. /// -/// [`write!`]: macro.write.html /// [`std::fmt`]: ../std/fmt/index.html /// -/// /// # Examples /// /// ``` @@ -466,38 +546,17 @@ macro_rules! write { /// Ok(()) /// } /// ``` -/// -/// A module can import both `std::fmt::Write` and `std::io::Write` and call `write!` on objects -/// implementing either, as objects do not typically implement both. However, the module must -/// import the traits qualified so their names do not conflict: -/// -/// ``` -/// use std::fmt::Write as FmtWrite; -/// use std::io::Write as IoWrite; -/// -/// fn main() -> Result<(), Box> { -/// let mut s = String::new(); -/// let mut v = Vec::new(); -/// -/// writeln!(&mut s, "{} {}", "abc", 123)?; // uses fmt::Write::write_fmt -/// writeln!(&mut v, "s = {:?}", s)?; // uses io::Write::write_fmt -/// assert_eq!(v, b"s = \"abc 123\\n\"\n"); -/// Ok(()) -/// } -/// ``` #[macro_export] #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "writeln_macro")] #[allow_internal_unstable(format_args_nl)] macro_rules! writeln { - ($dst:expr) => ( + ($dst:expr $(,)?) => { $crate::write!($dst, "\n") - ); - ($dst:expr,) => ( - $crate::writeln!($dst) - ); - ($dst:expr, $($arg:tt)*) => ( + }; + ($dst:expr, $($arg:tt)*) => { $dst.write_fmt($crate::format_args_nl!($($arg)*)) - ); + }; } /// Indicates unreachable code. @@ -515,15 +574,14 @@ macro_rules! writeln { /// The unsafe counterpart of this macro is the [`unreachable_unchecked`] function, which /// will cause undefined behavior if the code is reached. /// -/// [`panic!`]: ../std/macro.panic.html -/// [`unreachable_unchecked`]: ../std/hint/fn.unreachable_unchecked.html -/// [`std::hint`]: ../std/hint/index.html +/// [`unreachable_unchecked`]: crate::hint::unreachable_unchecked /// /// # Panics /// -/// This will always [`panic!`] +/// This will always [`panic!`] because `unreachable!` is just a shorthand for `panic!` with a +/// fixed, specific message. /// -/// [`panic!`]: ../std/macro.panic.html +/// Like `panic!`, this macro has a second form for displaying custom values. /// /// # Examples /// @@ -550,43 +608,41 @@ macro_rules! writeln { /// if 3*i < i { panic!("u32 overflow"); } /// if x < 3*i { return i-1; } /// } -/// unreachable!(); +/// unreachable!("The loop should always return"); /// } /// ``` #[macro_export] +#[rustc_builtin_macro(unreachable)] +#[allow_internal_unstable(edition_panic)] #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "unreachable_macro")] macro_rules! unreachable { - () => ({ - panic!("internal error: entered unreachable code") - }); - ($msg:expr) => ({ - $crate::unreachable!("{}", $msg) - }); - ($msg:expr,) => ({ - $crate::unreachable!($msg) - }); - ($fmt:expr, $($arg:tt)*) => ({ - panic!($crate::concat!("internal error: entered unreachable code: ", $fmt), $($arg)*) - }); + // Expands to either `$crate::panic::unreachable_2015` or `$crate::panic::unreachable_2021` + // depending on the edition of the caller. + ($($arg:tt)*) => { + /* compiler built-in */ + }; } /// Indicates unimplemented code by panicking with a message of "not implemented". /// /// This allows your code to type-check, which is useful if you are prototyping or -/// implementing a trait that requires multiple methods which you don't plan of using all of. +/// implementing a trait that requires multiple methods which you don't plan to use all of. /// -/// The difference between `unimplemented!` and [`todo!`](macro.todo.html) is that while `todo!` +/// The difference between `unimplemented!` and [`todo!`] is that while `todo!` /// conveys an intent of implementing the functionality later and the message is "not yet /// implemented", `unimplemented!` makes no such claims. Its message is "not implemented". /// Also some IDEs will mark `todo!`s. /// /// # Panics /// -/// This will always [panic!](macro.panic.html) because `unimplemented!` is just a -/// shorthand for `panic!` with a fixed, specific message. +/// This will always [`panic!`] because `unimplemented!` is just a shorthand for `panic!` with a +/// fixed, specific message. /// /// Like `panic!`, this macro has a second form for displaying custom values. /// +/// [`todo!`]: crate::todo +/// /// # Examples /// /// Say we have a trait `Foo`: @@ -643,9 +699,15 @@ macro_rules! unreachable { /// ``` #[macro_export] #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "unimplemented_macro")] +#[allow_internal_unstable(core_panic)] macro_rules! unimplemented { - () => (panic!("not implemented")); - ($($arg:tt)+) => (panic!("not implemented: {}", $crate::format_args!($($arg)+))); + () => { + $crate::panicking::panic("not implemented") + }; + ($($arg:tt)+) => { + $crate::panic!("not implemented: {}", $crate::format_args!($($arg)+)) + }; } /// Indicates unfinished code. @@ -658,11 +720,9 @@ macro_rules! unimplemented { /// implemented", `unimplemented!` makes no such claims. Its message is "not implemented". /// Also some IDEs will mark `todo!`s. /// -/// [`unimplemented!`]: macro.unimplemented.html -/// /// # Panics /// -/// This will always [panic!](macro.panic.html) +/// This will always [`panic!`]. /// /// # Examples /// @@ -706,9 +766,15 @@ macro_rules! unimplemented { /// ``` #[macro_export] #[stable(feature = "todo_macro", since = "1.40.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "todo_macro")] +#[allow_internal_unstable(core_panic)] macro_rules! todo { - () => (panic!("not yet implemented")); - ($($arg:tt)+) => (panic!("not yet implemented: {}", $crate::format_args!($($arg)+))); + () => { + $crate::panicking::panic("not yet implemented") + }; + ($($arg:tt)+) => { + $crate::panic!("not yet implemented: {}", $crate::format_args!($($arg)+)) + }; } /// Definitions of built-in macros. @@ -728,7 +794,7 @@ pub(crate) mod builtin { /// /// Two such examples are macros and `#[cfg]` environments. /// - /// Emit better compiler error if a macro is passed invalid values. Without the final branch, + /// Emit a better compiler error if a macro is passed invalid values. Without the final branch, /// the compiler would still emit an error, but the error's message would not mention the two /// valid values. /// @@ -745,20 +811,17 @@ pub(crate) mod builtin { /// // ^ will fail at compile time with message "This macro only accepts `foo` or `bar`" /// ``` /// - /// Emit compiler error if one of a number of features isn't available. + /// Emit a compiler error if one of a number of features isn't available. /// /// ```compile_fail /// #[cfg(not(any(feature = "foo", feature = "bar")))] /// compile_error!("Either feature \"foo\" or \"bar\" must be enabled for this crate."); /// ``` - /// - /// [`panic!`]: ../std/macro.panic.html #[stable(feature = "compile_error_macro", since = "1.20.0")] #[rustc_builtin_macro] #[macro_export] macro_rules! compile_error { - ($msg:expr) => {{ /* compiler built-in */ }}; - ($msg:expr,) => {{ /* compiler built-in */ }}; + ($msg:expr $(,)?) => {{ /* compiler built-in */ }}; } /// Constructs parameters for the other string-formatting macros. @@ -790,12 +853,11 @@ pub(crate) mod builtin { /// /// For more information, see the documentation in [`std::fmt`]. /// - /// [`Display`]: ../std/fmt/trait.Display.html - /// [`Debug`]: ../std/fmt/trait.Debug.html - /// [`fmt::Arguments`]: ../std/fmt/struct.Arguments.html + /// [`Display`]: crate::fmt::Display + /// [`Debug`]: crate::fmt::Debug + /// [`fmt::Arguments`]: crate::fmt::Arguments /// [`std::fmt`]: ../std/fmt/index.html /// [`format!`]: ../std/macro.format.html - /// [`write!`]: ../std/macro.write.html /// [`println!`]: ../std/macro.println.html /// /// # Examples @@ -807,6 +869,8 @@ pub(crate) mod builtin { /// assert_eq!(s, format!("hello {}", "world")); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[cfg_attr(not(test), rustc_diagnostic_item = "format_args_macro")] + #[allow_internal_unsafe] #[allow_internal_unstable(fmt_internals)] #[rustc_builtin_macro] #[macro_export] @@ -815,7 +879,21 @@ pub(crate) mod builtin { ($fmt:expr, $($args:tt)*) => {{ /* compiler built-in */ }}; } - /// Same as `format_args`, but adds a newline in the end. + /// Same as [`format_args`], but can be used in some const contexts. + /// + /// This macro is used by the panic macros for the `const_panic` feature. + /// + /// This macro will be removed once `format_args` is allowed in const contexts. + #[unstable(feature = "const_format_args", issue = "none")] + #[allow_internal_unstable(fmt_internals, const_fmt_arguments_new)] + #[rustc_builtin_macro] + #[macro_export] + macro_rules! const_format_args { + ($fmt:expr) => {{ /* compiler built-in */ }}; + ($fmt:expr, $($args:tt)*) => {{ /* compiler built-in */ }}; + } + + /// Same as [`format_args`], but adds a newline in the end. #[unstable( feature = "format_args_nl", issue = "none", @@ -833,19 +911,20 @@ pub(crate) mod builtin { /// Inspects an environment variable at compile time. /// /// This macro will expand to the value of the named environment variable at - /// compile time, yielding an expression of type `&'static str`. + /// compile time, yielding an expression of type `&'static str`. Use + /// [`std::env::var`] instead if you want to read the value at runtime. + /// + /// [`std::env::var`]: ../std/env/fn.var.html /// /// If the environment variable is not defined, then a compilation error /// will be emitted. To not emit a compile error, use the [`option_env!`] /// macro instead. /// - /// [`option_env!`]: ../std/macro.option_env.html - /// /// # Examples /// /// ``` /// let path: &'static str = env!("PATH"); - /// println!("the $PATH variable at the time of compiling was: {}", path); + /// println!("the $PATH variable at the time of compiling was: {path}"); /// ``` /// /// You can customize the error message by passing a string as the second @@ -865,8 +944,8 @@ pub(crate) mod builtin { #[rustc_builtin_macro] #[macro_export] macro_rules! env { - ($name:expr) => {{ /* compiler built-in */ }}; - ($name:expr,) => {{ /* compiler built-in */ }}; + ($name:expr $(,)?) => {{ /* compiler built-in */ }}; + ($name:expr, $error_msg:expr $(,)?) => {{ /* compiler built-in */ }}; } /// Optionally inspects an environment variable at compile time. @@ -875,25 +954,25 @@ pub(crate) mod builtin { /// expand into an expression of type `Option<&'static str>` whose value is /// `Some` of the value of the environment variable. If the environment /// variable is not present, then this will expand to `None`. See - /// [`Option`][option] for more information on this type. + /// [`Option`][Option] for more information on this type. Use + /// [`std::env::var`] instead if you want to read the value at runtime. + /// + /// [`std::env::var`]: ../std/env/fn.var.html /// /// A compile time error is never emitted when using this macro regardless /// of whether the environment variable is present or not. /// - /// [option]: ../std/option/enum.Option.html - /// /// # Examples /// /// ``` /// let key: Option<&'static str> = option_env!("SECRET_KEY"); - /// println!("the secret key might be: {:?}", key); + /// println!("the secret key might be: {key:?}"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[rustc_builtin_macro] #[macro_export] macro_rules! option_env { - ($name:expr) => {{ /* compiler built-in */ }}; - ($name:expr,) => {{ /* compiler built-in */ }}; + ($name:expr $(,)?) => {{ /* compiler built-in */ }}; } /// Concatenates identifiers into one identifier. @@ -928,8 +1007,34 @@ pub(crate) mod builtin { #[rustc_builtin_macro] #[macro_export] macro_rules! concat_idents { - ($($e:ident),+) => {{ /* compiler built-in */ }}; - ($($e:ident,)+) => {{ /* compiler built-in */ }}; + ($($e:ident),+ $(,)?) => {{ /* compiler built-in */ }}; + } + + /// Concatenates literals into a byte slice. + /// + /// This macro takes any number of comma-separated literals, and concatenates them all into + /// one, yielding an expression of type `&[u8; _]`, which represents all of the literals + /// concatenated left-to-right. The literals passed can be any combination of: + /// + /// - byte literals (`b'r'`) + /// - byte strings (`b"Rust"`) + /// - arrays of bytes/numbers (`[b'A', 66, b'C']`) + /// + /// # Examples + /// + /// ``` + /// #![feature(concat_bytes)] + /// + /// # fn main() { + /// let s: &[u8; 6] = concat_bytes!(b'A', b"BC", [68, b'E', 70]); + /// assert_eq!(s, b"ABCDEF"); + /// # } + /// ``` + #[unstable(feature = "concat_bytes", issue = "87555")] + #[rustc_builtin_macro] + #[macro_export] + macro_rules! concat_bytes { + ($($e:literal),+ $(,)?) => {{ /* compiler built-in */ }}; } /// Concatenates literals into a static string slice. @@ -951,8 +1056,7 @@ pub(crate) mod builtin { #[rustc_builtin_macro] #[macro_export] macro_rules! concat { - ($($e:expr),*) => {{ /* compiler built-in */ }}; - ($($e:expr,)*) => {{ /* compiler built-in */ }}; + ($($e:expr),* $(,)?) => {{ /* compiler built-in */ }}; } /// Expands to the line number on which it was invoked. @@ -967,14 +1071,11 @@ pub(crate) mod builtin { /// but rather the first macro invocation leading up to the invocation /// of the `line!` macro. /// - /// [`column!`]: macro.column.html - /// [`file!`]: macro.file.html - /// /// # Examples /// /// ``` /// let current_line = line!(); - /// println!("defined on line: {}", current_line); + /// println!("defined on line: {current_line}"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[rustc_builtin_macro] @@ -997,14 +1098,23 @@ pub(crate) mod builtin { /// but rather the first macro invocation leading up to the invocation /// of the `column!` macro. /// - /// [`line!`]: macro.line.html - /// [`file!`]: macro.file.html - /// /// # Examples /// /// ``` /// let current_col = column!(); - /// println!("defined on column: {}", current_col); + /// println!("defined on column: {current_col}"); + /// ``` + /// + /// `column!` counts Unicode code points, not bytes or graphemes. As a result, the first two + /// invocations return the same value, but the third does not. + /// + /// ``` + /// let a = ("foobar", column!()).1; + /// let b = ("人之初性本善", column!()).1; + /// let c = ("f̅o̅o̅b̅a̅r̅", column!()).1; // Uses combining overline (U+0305) + /// + /// assert_eq!(a, b); + /// assert_ne!(b, c); /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[rustc_builtin_macro] @@ -1020,20 +1130,16 @@ pub(crate) mod builtin { /// With [`line!`] and [`column!`], these macros provide debugging information for /// developers about the location within the source. /// - /// /// The expanded expression has type `&'static str`, and the returned file /// is not the invocation of the `file!` macro itself, but rather the /// first macro invocation leading up to the invocation of the `file!` /// macro. /// - /// [`line!`]: macro.line.html - /// [`column!`]: macro.column.html - /// /// # Examples /// /// ``` /// let this_file = file!(); - /// println!("defined in file: {}", this_file); + /// println!("defined in file: {this_file}"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[rustc_builtin_macro] @@ -1068,10 +1174,12 @@ pub(crate) mod builtin { }; } - /// Includes a utf8-encoded file as a string. + /// Includes a UTF-8 encoded file as a string. /// - /// The file is located relative to the current file. (similarly to how - /// modules are found) + /// The file is located relative to the current file (similarly to how + /// modules are found). The provided path is interpreted in a platform-specific + /// way at compile time. So, for instance, an invocation with a Windows path + /// containing backslashes `\` would not compile correctly on Unix. /// /// This macro will yield an expression of type `&'static str` which is the /// contents of the file. @@ -1093,7 +1201,7 @@ pub(crate) mod builtin { /// fn main() { /// let my_str = include_str!("spanish.in"); /// assert_eq!(my_str, "adiós\n"); - /// print!("{}", my_str); + /// print!("{my_str}"); /// } /// ``` /// @@ -1101,15 +1209,17 @@ pub(crate) mod builtin { #[stable(feature = "rust1", since = "1.0.0")] #[rustc_builtin_macro] #[macro_export] + #[cfg_attr(not(test), rustc_diagnostic_item = "include_str_macro")] macro_rules! include_str { - ($file:expr) => {{ /* compiler built-in */ }}; - ($file:expr,) => {{ /* compiler built-in */ }}; + ($file:expr $(,)?) => {{ /* compiler built-in */ }}; } /// Includes a file as a reference to a byte array. /// - /// The file is located relative to the current file. (similarly to how - /// modules are found) + /// The file is located relative to the current file (similarly to how + /// modules are found). The provided path is interpreted in a platform-specific + /// way at compile time. So, for instance, an invocation with a Windows path + /// containing backslashes `\` would not compile correctly on Unix. /// /// This macro will yield an expression of type `&'static [u8; N]` which is /// the contents of the file. @@ -1139,9 +1249,9 @@ pub(crate) mod builtin { #[stable(feature = "rust1", since = "1.0.0")] #[rustc_builtin_macro] #[macro_export] + #[cfg_attr(not(test), rustc_diagnostic_item = "include_bytes_macro")] macro_rules! include_bytes { - ($file:expr) => {{ /* compiler built-in */ }}; - ($file:expr,) => {{ /* compiler built-in */ }}; + ($file:expr $(,)?) => {{ /* compiler built-in */ }}; } /// Expands to a string that represents the current module path. @@ -1179,6 +1289,10 @@ pub(crate) mod builtin { /// The syntax given to this macro is the same syntax as the [`cfg`] /// attribute. /// + /// `cfg!`, unlike `#[cfg]`, does not remove any code and only evaluates to true or false. For + /// example, all blocks in an if/else expression need to be valid when `cfg!` is used for + /// the condition, regardless of what `cfg!` is evaluating. + /// /// [`cfg`]: ../reference/conditional-compilation.html#the-cfg-attribute /// /// # Examples @@ -1201,20 +1315,41 @@ pub(crate) mod builtin { /// Parses a file as an expression or an item according to the context. /// - /// The file is located relative to the current file (similarly to how - /// modules are found). + /// **Warning**: For multi-file Rust projects, the `include!` macro is probably not what you + /// are looking for. Usually, multi-file Rust projects use + /// [modules](https://doc.rust-lang.org/reference/items/modules.html). Multi-file projects and + /// modules are explained in the Rust-by-Example book + /// [here](https://doc.rust-lang.org/rust-by-example/mod/split.html) and the module system is + /// explained in the Rust Book + /// [here](https://doc.rust-lang.org/book/ch07-02-defining-modules-to-control-scope-and-privacy.html). + /// + /// The included file is placed in the surrounding code + /// [unhygienically](https://doc.rust-lang.org/reference/macros-by-example.html#hygiene). If + /// the included file is parsed as an expression and variables or functions share names across + /// both files, it could result in variables or functions being different from what the + /// included file expected. + /// + /// The included file is located relative to the current file (similarly to how modules are + /// found). The provided path is interpreted in a platform-specific way at compile time. So, + /// for instance, an invocation with a Windows path containing backslashes `\` would not + /// compile correctly on Unix. + /// + /// # Uses /// - /// Using this macro is often a bad idea, because if the file is - /// parsed as an expression, it is going to be placed in the - /// surrounding code unhygienically. This could result in variables - /// or functions being different from what the file expected if - /// there are variables or functions that have the same name in - /// the current file. + /// The `include!` macro is primarily used for two purposes. It is used to include + /// documentation that is written in a separate file and it is used to include [build artifacts + /// usually as a result from the `build.rs` + /// script](https://doc.rust-lang.org/cargo/reference/build-scripts.html#outputs-of-the-build-script). + /// + /// When using the `include` macro to include stretches of documentation, remember that the + /// included file still needs to be a valid rust syntax. It is also possible to + /// use the [`include_str`] macro as `#![doc = include_str!("...")]` (at the module level) or + /// `#[doc = include_str!("...")]` (at the item level) to include documentation from a plain + /// text or markdown file. /// /// # Examples /// - /// Assume there are two files in the same directory with the following - /// contents: + /// Assume there are two files in the same directory with the following contents: /// /// File 'monkeys.in': /// @@ -1232,7 +1367,7 @@ pub(crate) mod builtin { /// fn main() { /// let my_string = include!("monkeys.in"); /// assert_eq!("🙈🙊🙉🙈🙊🙉", my_string); - /// println!("{}", my_string); + /// println!("{my_string}"); /// } /// ``` /// @@ -1242,8 +1377,7 @@ pub(crate) mod builtin { #[rustc_builtin_macro] #[macro_export] macro_rules! include { - ($file:expr) => {{ /* compiler built-in */ }}; - ($file:expr,) => {{ /* compiler built-in */ }}; + ($file:expr $(,)?) => {{ /* compiler built-in */ }}; } /// Asserts that a boolean expression is `true` at runtime. @@ -1257,7 +1391,7 @@ pub(crate) mod builtin { /// be disabled. See [`debug_assert!`] for assertions that are not enabled in /// release builds by default. /// - /// Unsafe code relies on `assert!` to enforce run-time invariants that, if + /// Unsafe code may rely on `assert!` to enforce run-time invariants that, if /// violated could lead to unsafety. /// /// Other use-cases of `assert!` include testing and enforcing run-time @@ -1267,10 +1401,9 @@ pub(crate) mod builtin { /// /// This macro has a second form, where a custom panic message can /// be provided with or without arguments for formatting. See [`std::fmt`] - /// for syntax for this form. + /// for syntax for this form. Expressions used as format arguments will only + /// be evaluated if the assertion fails. /// - /// [`panic!`]: macro.panic.html - /// [`debug_assert!`]: macro.debug_assert.html /// [`std::fmt`]: ../std/fmt/index.html /// /// # Examples @@ -1294,48 +1427,13 @@ pub(crate) mod builtin { #[stable(feature = "rust1", since = "1.0.0")] #[rustc_builtin_macro] #[macro_export] + #[rustc_diagnostic_item = "assert_macro"] + #[allow_internal_unstable(core_panic, edition_panic)] macro_rules! assert { - ($cond:expr) => {{ /* compiler built-in */ }}; - ($cond:expr,) => {{ /* compiler built-in */ }}; + ($cond:expr $(,)?) => {{ /* compiler built-in */ }}; ($cond:expr, $($arg:tt)+) => {{ /* compiler built-in */ }}; } - /// Inline assembly. - /// - /// Read the [unstable book] for the usage. - /// - /// [unstable book]: ../unstable-book/library-features/asm.html - #[unstable( - feature = "asm", - issue = "29722", - reason = "inline assembly is not stable enough for use and is subject to change" - )] - #[rustc_builtin_macro] - #[macro_export] - macro_rules! asm { - ("assembly template" - : $("output"(operand),)* - : $("input"(operand),)* - : $("clobbers",)* - : $("options",)*) => { - /* compiler built-in */ - }; - } - - /// Module-level inline assembly. - #[unstable( - feature = "global_asm", - issue = "35119", - reason = "`global_asm!` is not stable enough for use and is subject to change" - )] - #[rustc_builtin_macro] - #[macro_export] - macro_rules! global_asm { - ("assembly") => { - /* compiler built-in */ - }; - } - /// Prints passed tokens into the standard output. #[unstable( feature = "log_syntax", @@ -1363,7 +1461,34 @@ pub(crate) mod builtin { (false) => {{ /* compiler built-in */ }}; } + /// Attribute macro used to apply derive macros. + /// + /// See [the reference] for more info. + /// + /// [the reference]: ../../../reference/attributes/derive.html + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_builtin_macro] + pub macro derive($item:item) { + /* compiler built-in */ + } + + /// Attribute macro used to apply derive macros for implementing traits + /// in a const context. + /// + /// See [the reference] for more info. + /// + /// [the reference]: ../../../reference/attributes/derive.html + #[unstable(feature = "derive_const", issue = "none")] + #[rustc_builtin_macro] + pub macro derive_const($item:item) { + /* compiler built-in */ + } + /// Attribute macro applied to a function to turn it into a unit test. + /// + /// See [the reference] for more info. + /// + /// [the reference]: ../../../reference/attributes/testing.html#the-test-attribute #[stable(feature = "rust1", since = "1.0.0")] #[allow_internal_unstable(test, rustc_attrs)] #[rustc_builtin_macro] @@ -1397,6 +1522,8 @@ pub(crate) mod builtin { } /// Attribute macro applied to a static to register it as a global allocator. + /// + /// See also [`std::alloc::GlobalAlloc`](../../../std/alloc/trait.GlobalAlloc.html). #[stable(feature = "global_allocator", since = "1.28.0")] #[allow_internal_unstable(rustc_attrs)] #[rustc_builtin_macro] @@ -1404,8 +1531,17 @@ pub(crate) mod builtin { /* compiler built-in */ } + /// Attribute macro applied to a function to register it as a handler for allocation failure. + /// + /// See also [`std::alloc::handle_alloc_error`](../../../std/alloc/fn.handle_alloc_error.html). + #[unstable(feature = "alloc_error_handler", issue = "51540")] + #[allow_internal_unstable(rustc_attrs)] + #[rustc_builtin_macro] + pub macro alloc_error_handler($item:item) { + /* compiler built-in */ + } + /// Keeps the item it's applied to if the passed path is accessible, and removes it otherwise. - #[cfg(not(bootstrap))] #[unstable( feature = "cfg_accessible", issue = "64797", @@ -1416,10 +1552,34 @@ pub(crate) mod builtin { /* compiler built-in */ } + /// Expands all `#[cfg]` and `#[cfg_attr]` attributes in the code fragment it's applied to. + #[unstable( + feature = "cfg_eval", + issue = "82679", + reason = "`cfg_eval` is a recently implemented feature" + )] + #[rustc_builtin_macro] + pub macro cfg_eval($($tt:tt)*) { + /* compiler built-in */ + } + + /// Unstable placeholder for type ascription. + #[rustc_builtin_macro] + #[unstable( + feature = "type_ascription", + issue = "23416", + reason = "placeholder syntax for type ascription" + )] + pub macro type_ascribe($expr:expr, $ty:ty) { + /* compiler built-in */ + } + /// Unstable implementation detail of the `rustc` compiler, do not use. #[rustc_builtin_macro] #[stable(feature = "rust1", since = "1.0.0")] - #[allow_internal_unstable(core_intrinsics, libstd_sys_internals)] + #[allow_internal_unstable(core_intrinsics, libstd_sys_internals, rt)] + #[deprecated(since = "1.52.0", note = "rustc-serialize is deprecated and no longer supported")] + #[doc(hidden)] // While technically stable, using it is unstable, and deprecated. Hide it. pub macro RustcDecodable($item:item) { /* compiler built-in */ } @@ -1427,7 +1587,9 @@ pub(crate) mod builtin { /// Unstable implementation detail of the `rustc` compiler, do not use. #[rustc_builtin_macro] #[stable(feature = "rust1", since = "1.0.0")] - #[allow_internal_unstable(core_intrinsics)] + #[allow_internal_unstable(core_intrinsics, rt)] + #[deprecated(since = "1.52.0", note = "rustc-serialize is deprecated and no longer supported")] + #[doc(hidden)] // While technically stable, using it is unstable, and deprecated. Hide it. pub macro RustcEncodable($item:item) { /* compiler built-in */ } diff --git a/crux-mir/lib/core/src/macros/panic.md b/crux-mir/lib/core/src/macros/panic.md index 3ecfc43be..98fb7e9e4 100644 --- a/crux-mir/lib/core/src/macros/panic.md +++ b/crux-mir/lib/core/src/macros/panic.md @@ -1,35 +1,63 @@ Panics the current thread. This allows a program to terminate immediately and provide feedback -to the caller of the program. `panic!` should be used when a program reaches -an unrecoverable state. +to the caller of the program. This macro is the perfect way to assert conditions in example code and in -tests. `panic!` is closely tied with the `unwrap` method of both [`Option`] -and [`Result`][runwrap] enums. Both implementations call `panic!` when they are set -to None or Err variants. +tests. `panic!` is closely tied with the `unwrap` method of both +[`Option`][ounwrap] and [`Result`][runwrap] enums. Both implementations call +`panic!` when they are set to [`None`] or [`Err`] variants. -This macro is used to inject panic into a Rust thread, causing the thread to -panic entirely. Each thread's panic can be reaped as the `Box` type, -and the single-argument form of the `panic!` macro will be the value which -is transmitted. +When using `panic!()` you can specify a string payload, that is built using +the [`format!`] syntax. That payload is used when injecting the panic into +the calling Rust thread, causing the thread to panic entirely. -[`Result`] enum is often a better solution for recovering from errors than -using the `panic!` macro. This macro should be used to avoid proceeding using -incorrect values, such as from external sources. Detailed information about -error handling is found in the [book]. - -The multi-argument form of this macro panics with a string and has the -[`format!`] syntax for building a string. +The behavior of the default `std` hook, i.e. the code that runs directly +after the panic is invoked, is to print the message payload to +`stderr` along with the file/line/column information of the `panic!()` +call. You can override the panic hook using [`std::panic::set_hook()`]. +Inside the hook a panic can be accessed as a `&dyn Any + Send`, +which contains either a `&str` or `String` for regular `panic!()` invocations. +To panic with a value of another other type, [`panic_any`] can be used. See also the macro [`compile_error!`], for raising errors during compilation. -[runwrap]: ../std/result/enum.Result.html#method.unwrap -[`Option`]: ../std/option/enum.Option.html#method.unwrap -[`Result`]: ../std/result/enum.Result.html +# When to use `panic!` vs `Result` + +The Rust language provides two complementary systems for constructing / +representing, reporting, propagating, reacting to, and discarding errors. These +responsibilities are collectively known as "error handling." `panic!` and +`Result` are similar in that they are each the primary interface of their +respective error handling systems; however, the meaning these interfaces attach +to their errors and the responsibilities they fulfill within their respective +error handling systems differ. + +The `panic!` macro is used to construct errors that represent a bug that has +been detected in your program. With `panic!` you provide a message that +describes the bug and the language then constructs an error with that message, +reports it, and propagates it for you. + +`Result` on the other hand is used to wrap other types that represent either +the successful result of some computation, `Ok(T)`, or error types that +represent an anticipated runtime failure mode of that computation, `Err(E)`. +`Result` is used alongside user defined types which represent the various +anticipated runtime failure modes that the associated computation could +encounter. `Result` must be propagated manually, often with the the help of the +`?` operator and `Try` trait, and they must be reported manually, often with +the help of the `Error` trait. + +For more detailed information about error handling check out the [book] or the +[`std::result`] module docs. + +[ounwrap]: Option::unwrap +[runwrap]: Result::unwrap +[`std::panic::set_hook()`]: ../std/panic/fn.set_hook.html +[`panic_any`]: ../std/panic/fn.panic_any.html +[`Box`]: ../std/boxed/struct.Box.html +[`Any`]: crate::any::Any [`format!`]: ../std/macro.format.html -[`compile_error!`]: ../std/macro.compile_error.html [book]: ../book/ch09-00-error-handling.html +[`std::result`]: ../std/result/index.html # Current implementation @@ -42,6 +70,6 @@ program with code `101`. # #![allow(unreachable_code)] panic!(); panic!("this is a terrible mistake!"); -panic!(4); // panic with the value of 4 to be collected elsewhere panic!("this is a {} {message}", "fancy", message = "message"); +std::panic::panic_any(4); // panic with the value of 4 to be collected elsewhere ``` diff --git a/crux-mir/lib/core/src/marker.rs b/crux-mir/lib/core/src/marker.rs index 2b908f07a..1326fc9ab 100644 --- a/crux-mir/lib/core/src/marker.rs +++ b/crux-mir/lib/core/src/marker.rs @@ -8,6 +8,7 @@ use crate::cell::UnsafeCell; use crate::cmp; +use crate::fmt::Debug; use crate::hash::Hash; use crate::hash::Hasher; @@ -29,7 +30,7 @@ use crate::hash::Hasher; /// [arc]: ../../std/sync/struct.Arc.html /// [ub]: ../../reference/behavior-considered-undefined.html #[stable(feature = "rust1", since = "1.0.0")] -#[cfg_attr(not(test), rustc_diagnostic_item = "send_trait")] +#[cfg_attr(not(test), rustc_diagnostic_item = "Send")] #[rustc_on_unimplemented( message = "`{Self}` cannot be sent between threads safely", label = "`{Self}` cannot be sent between threads safely" @@ -43,6 +44,12 @@ impl !Send for *const T {} #[stable(feature = "rust1", since = "1.0.0")] impl !Send for *mut T {} +// Most instances arise automatically, but this instance is needed to link up `T: Sync` with +// `&T: Send` (and it also removes the unsound default instance `T Send` -> `&T: Send` that would +// otherwise exist). +#[stable(feature = "rust1", since = "1.0.0")] +unsafe impl Send for &T {} + /// Types with a constant size known at compile time. /// /// All type parameters have an implicit bound of `Sized`. The special syntax @@ -80,17 +87,16 @@ impl !Send for *mut T {} /// ``` /// /// [trait object]: ../../book/ch17-02-trait-objects.html +#[doc(alias = "?", alias = "?Sized")] #[stable(feature = "rust1", since = "1.0.0")] #[lang = "sized"] #[rustc_on_unimplemented( - on(parent_trait = "std::path::Path", label = "borrow the `Path` instead"), message = "the size for values of type `{Self}` cannot be known at compilation time", - label = "doesn't have a size known at compile-time", - note = "to learn more, visit " + label = "doesn't have a size known at compile-time" )] #[fundamental] // for Default, for example, which requires that `[T]: !Default` be evaluatable -#[cfg_attr(not(bootstrap), rustc_specialization_trait)] +#[rustc_specialization_trait] +#[rustc_deny_explicit_impl] pub trait Sized { // Empty. } @@ -101,29 +107,28 @@ pub trait Sized { /// `Unsize`. /// /// All implementations of `Unsize` are provided automatically by the compiler. -/// -/// `Unsize` is implemented for: -/// -/// - `[T; N]` is `Unsize<[T]>` -/// - `T` is `Unsize` when `T: Trait` -/// - `Foo<..., T, ...>` is `Unsize>` if: -/// - `T: Unsize` -/// - Foo is a struct -/// - Only the last field of `Foo` has a type involving `T` -/// - `T` is not part of the type of any other fields -/// - `Bar: Unsize>`, if the last field of `Foo` has type `Bar` -/// -/// `Unsize` is used along with [`ops::CoerceUnsized`][coerceunsized] to allow -/// "user-defined" containers such as [`rc::Rc`][rc] to contain dynamically-sized +/// Those implementations are: +/// +/// - Arrays `[T; N]` implement `Unsize<[T]>`. +/// - Types implementing a trait `Trait` also implement `Unsize`. +/// - Structs `Foo<..., T, ...>` implement `Unsize>` if all of these conditions +/// are met: +/// - `T: Unsize`. +/// - Only the last field of `Foo` has a type involving `T`. +/// - `Bar: Unsize>`, where `Bar` stands for the actual type of that last field. +/// +/// `Unsize` is used along with [`ops::CoerceUnsized`] to allow +/// "user-defined" containers such as [`Rc`] to contain dynamically-sized /// types. See the [DST coercion RFC][RFC982] and [the nomicon entry on coercion][nomicon-coerce] /// for more details. /// -/// [coerceunsized]: ../ops/trait.CoerceUnsized.html -/// [rc]: ../../std/rc/struct.Rc.html +/// [`ops::CoerceUnsized`]: crate::ops::CoerceUnsized +/// [`Rc`]: ../../std/rc/struct.Rc.html /// [RFC982]: https://github.com/rust-lang/rfcs/blob/master/text/0982-dst-coercion.md /// [nomicon-coerce]: ../../nomicon/coercions.html -#[unstable(feature = "unsize", issue = "27732")] +#[unstable(feature = "unsize", issue = "18598")] #[lang = "unsize"] +#[rustc_deny_explicit_impl] pub trait Unsize { // Empty. } @@ -158,18 +163,18 @@ pub trait StructuralPartialEq { /// Required trait for constants used in pattern matches. /// /// Any type that derives `Eq` automatically implements this trait, *regardless* -/// of whether its type-parameters implement `Eq`. +/// of whether its type parameters implement `Eq`. /// -/// This is a hack to workaround a limitation in our type-system. +/// This is a hack to work around a limitation in our type system. /// -/// Background: +/// # Background /// /// We want to require that types of consts used in pattern matches /// have the attribute `#[derive(PartialEq, Eq)]`. /// /// In a more ideal world, we could check that requirement by just checking that -/// the given type implements both (1.) the `StructuralPartialEq` trait *and* -/// (2.) the `Eq` trait. However, you can have ADTs that *do* `derive(PartialEq, Eq)`, +/// the given type implements both the `StructuralPartialEq` trait *and* +/// the `Eq` trait. However, you can have ADTs that *do* `derive(PartialEq, Eq)`, /// and be a case that we want the compiler to accept, and yet the constant's /// type fails to implement `Eq`. /// @@ -178,8 +183,11 @@ pub trait StructuralPartialEq { /// ```rust /// #[derive(PartialEq, Eq)] /// struct Wrap(X); +/// /// fn higher_order(_: &()) { } +/// /// const CFN: Wrap = Wrap(higher_order); +/// /// fn main() { /// match CFN { /// CFN => {} @@ -220,7 +228,7 @@ pub trait StructuralEq { /// /// // `x` has moved into `y`, and so cannot be used /// -/// // println!("{:?}", x); // error: use of moved value +/// // println!("{x:?}"); // error: use of moved value /// ``` /// /// However, if a type implements `Copy`, it instead has 'copy semantics': @@ -237,7 +245,7 @@ pub trait StructuralEq { /// /// // `y` is a copy of `x` /// -/// println!("{:?}", x); // A-OK! +/// println!("{x:?}"); // A-OK! /// ``` /// /// It's important to note that in these two examples, the only difference is whether you @@ -293,6 +301,7 @@ pub trait StructuralEq { /// /// ``` /// # #[allow(dead_code)] +/// #[derive(Copy, Clone)] /// struct Point { /// x: i32, /// y: i32, @@ -317,6 +326,20 @@ pub trait StructuralEq { /// the trait `Copy` may not be implemented for this type; field `points` does not implement `Copy` /// ``` /// +/// Shared references (`&T`) are also `Copy`, so a type can be `Copy`, even when it holds +/// shared references of types `T` that are *not* `Copy`. Consider the following struct, +/// which can implement `Copy`, because it only holds a *shared reference* to our non-`Copy` +/// type `PointList` from above: +/// +/// ``` +/// # #![allow(dead_code)] +/// # struct PointList; +/// #[derive(Copy, Clone)] +/// struct PointListWrapper<'a> { +/// point_list_ref: &'a PointList, +/// } +/// ``` +/// /// ## When *can't* my type be `Copy`? /// /// Some types can't be copied safely. For example, copying `&mut T` would create an aliased @@ -329,7 +352,7 @@ pub trait StructuralEq { /// If you try to implement `Copy` on a struct or enum containing non-`Copy` data, you will get /// the error [E0204]. /// -/// [E0204]: ../../error-index.html#E0204 +/// [E0204]: ../../error_codes/E0204.html /// /// ## When *should* my type be `Copy`? /// @@ -345,8 +368,6 @@ pub trait StructuralEq { /// /// * Function item types (i.e., the distinct types defined for each function) /// * Function pointer types (e.g., `fn() -> i32`) -/// * Array types, for all sizes, if the item type also implements `Copy` (e.g., `[i32; 123456]`) -/// * Tuple types, if each component also implements `Copy` (e.g., `()`, `(i32, bool)`) /// * Closure types, if they capture no value from the environment /// or if all such captured values implement `Copy` themselves. /// Note that variables captured by shared reference always implement `Copy` @@ -355,14 +376,18 @@ pub trait StructuralEq { /// /// [`Vec`]: ../../std/vec/struct.Vec.html /// [`String`]: ../../std/string/struct.String.html -/// [`Drop`]: ../../std/ops/trait.Drop.html -/// [`size_of::`]: ../../std/mem/fn.size_of.html -/// [`Clone`]: ../clone/trait.Clone.html -/// [`String`]: ../../std/string/struct.String.html -/// [`i32`]: ../../std/primitive.i32.html +/// [`size_of::`]: crate::mem::size_of /// [impls]: #implementors #[stable(feature = "rust1", since = "1.0.0")] #[lang = "copy"] +// FIXME(matthewjasper) This allows copying a type that doesn't implement +// `Copy` because of unsatisfied lifetime bounds (copying `A<'_>` when only +// `A<'static>: Copy` and `A<'_>: Clone`). +// We have this attribute here for now only because there are quite a few +// existing specializations on `Copy` that already exist in the standard +// library, and there's no way to safely have this behavior right now. +#[rustc_unsafe_specialization_marker] +#[rustc_diagnostic_item = "Copy"] pub trait Copy: Clone { // Empty. } @@ -380,18 +405,18 @@ pub macro Copy($item:item) { /// This trait is automatically implemented when the compiler determines /// it's appropriate. /// -/// The precise definition is: a type `T` is `Sync` if and only if `&T` is -/// [`Send`][send]. In other words, if there is no possibility of +/// The precise definition is: a type `T` is [`Sync`] if and only if `&T` is +/// [`Send`]. In other words, if there is no possibility of /// [undefined behavior][ub] (including data races) when passing /// `&T` references between threads. /// -/// As one would expect, primitive types like [`u8`][u8] and [`f64`][f64] -/// are all `Sync`, and so are simple aggregate types containing them, -/// like tuples, structs and enums. More examples of basic `Sync` +/// As one would expect, primitive types like [`u8`] and [`f64`] +/// are all [`Sync`], and so are simple aggregate types containing them, +/// like tuples, structs and enums. More examples of basic [`Sync`] /// types include "immutable" types like `&T`, and those with simple /// inherited mutability, such as [`Box`][box], [`Vec`][vec] and -/// most other collection types. (Generic parameters need to be `Sync` -/// for their container to be `Sync`.) +/// most other collection types. (Generic parameters need to be [`Sync`] +/// for their container to be [`Sync`].) /// /// A somewhat surprising consequence of the definition is that `&mut T` /// is `Sync` (if `T` is `Sync`) even though it seems like that might @@ -401,15 +426,15 @@ pub macro Copy($item:item) { /// of a data race. /// /// Types that are not `Sync` are those that have "interior -/// mutability" in a non-thread-safe form, such as [`cell::Cell`][cell] -/// and [`cell::RefCell`][refcell]. These types allow for mutation of +/// mutability" in a non-thread-safe form, such as [`Cell`][cell] +/// and [`RefCell`][refcell]. These types allow for mutation of /// their contents even through an immutable, shared reference. For /// example the `set` method on [`Cell`][cell] takes `&self`, so it requires /// only a shared reference [`&Cell`][cell]. The method performs no /// synchronization, thus [`Cell`][cell] cannot be `Sync`. /// /// Another example of a non-`Sync` type is the reference-counting -/// pointer [`rc::Rc`][rc]. Given any reference [`&Rc`][rc], you can clone +/// pointer [`Rc`][rc]. Given any reference [`&Rc`][rc], you can clone /// a new [`Rc`][rc], modifying the reference counts in a non-atomic way. /// /// For cases when one does need thread-safe interior mutability, @@ -425,26 +450,23 @@ pub macro Copy($item:item) { /// [undefined behavior][ub]. For example, [`transmute`][transmute]-ing /// from `&T` to `&mut T` is invalid. /// -/// See [the Nomicon](../../nomicon/send-and-sync.html) for more -/// details about `Sync`. +/// See [the Nomicon][nomicon-send-and-sync] for more details about `Sync`. /// -/// [send]: trait.Send.html -/// [u8]: ../../std/primitive.u8.html -/// [f64]: ../../std/primitive.f64.html /// [box]: ../../std/boxed/struct.Box.html /// [vec]: ../../std/vec/struct.Vec.html -/// [cell]: ../cell/struct.Cell.html -/// [refcell]: ../cell/struct.RefCell.html +/// [cell]: crate::cell::Cell +/// [refcell]: crate::cell::RefCell /// [rc]: ../../std/rc/struct.Rc.html /// [arc]: ../../std/sync/struct.Arc.html -/// [atomic data types]: ../sync/atomic/index.html +/// [atomic data types]: crate::sync::atomic /// [mutex]: ../../std/sync/struct.Mutex.html /// [rwlock]: ../../std/sync/struct.RwLock.html -/// [unsafecell]: ../cell/struct.UnsafeCell.html +/// [unsafecell]: crate::cell::UnsafeCell /// [ub]: ../../reference/behavior-considered-undefined.html -/// [transmute]: ../../std/mem/fn.transmute.html +/// [transmute]: crate::mem::transmute +/// [nomicon-send-and-sync]: ../../nomicon/send-and-sync.html #[stable(feature = "rust1", since = "1.0.0")] -#[cfg_attr(not(test), rustc_diagnostic_item = "sync_trait")] +#[cfg_attr(not(test), rustc_diagnostic_item = "Sync")] #[lang = "sync"] #[rustc_on_unimplemented( message = "`{Self}` cannot be shared between threads safely", @@ -469,63 +491,6 @@ impl !Sync for *const T {} #[stable(feature = "rust1", since = "1.0.0")] impl !Sync for *mut T {} -macro_rules! impls { - ($t: ident) => { - #[stable(feature = "rust1", since = "1.0.0")] - impl Hash for $t { - #[inline] - fn hash(&self, _: &mut H) {} - } - - #[stable(feature = "rust1", since = "1.0.0")] - impl cmp::PartialEq for $t { - fn eq(&self, _other: &$t) -> bool { - true - } - } - - #[stable(feature = "rust1", since = "1.0.0")] - impl cmp::Eq for $t {} - - #[stable(feature = "rust1", since = "1.0.0")] - impl cmp::PartialOrd for $t { - fn partial_cmp(&self, _other: &$t) -> Option { - Option::Some(cmp::Ordering::Equal) - } - } - - #[stable(feature = "rust1", since = "1.0.0")] - impl cmp::Ord for $t { - fn cmp(&self, _other: &$t) -> cmp::Ordering { - cmp::Ordering::Equal - } - } - - #[stable(feature = "rust1", since = "1.0.0")] - impl Copy for $t {} - - #[stable(feature = "rust1", since = "1.0.0")] - impl Clone for $t { - fn clone(&self) -> Self { - Self - } - } - - #[stable(feature = "rust1", since = "1.0.0")] - impl Default for $t { - fn default() -> Self { - Self - } - } - - #[unstable(feature = "structural_match", issue = "31434")] - impl StructuralPartialEq for $t {} - - #[unstable(feature = "structural_match", issue = "31434")] - impl StructuralEq for $t {} - }; -} - /// Zero-sized type used to mark things that "act like" they own a `T`. /// /// Adding a `PhantomData` field to your type tells the compiler that your @@ -630,9 +595,9 @@ macro_rules! impls { /// } /// /// impl ExternalResource { -/// fn new() -> ExternalResource { +/// fn new() -> Self { /// let size_of_res = mem::size_of::(); -/// ExternalResource { +/// Self { /// resource_handle: foreign_lib::new(size_of_res), /// resource_type: PhantomData, /// } @@ -658,19 +623,90 @@ macro_rules! impls { /// (ideally) or `PhantomData<*const T>` (if no lifetime applies), so /// as not to indicate ownership. /// +/// ## Layout +/// +/// For all `T`, the following are guaranteed: +/// * `size_of::>() == 0` +/// * `align_of::>() == 1` +/// /// [drop check]: ../../nomicon/dropck.html #[lang = "phantom_data"] -#[structural_match] #[stable(feature = "rust1", since = "1.0.0")] pub struct PhantomData; -impls! { PhantomData } +#[stable(feature = "rust1", since = "1.0.0")] +impl Hash for PhantomData { + #[inline] + fn hash(&self, _: &mut H) {} +} -mod impls { - #[stable(feature = "rust1", since = "1.0.0")] - unsafe impl Send for &T {} - #[stable(feature = "rust1", since = "1.0.0")] - unsafe impl Send for &mut T {} +#[stable(feature = "rust1", since = "1.0.0")] +impl cmp::PartialEq for PhantomData { + fn eq(&self, _other: &PhantomData) -> bool { + true + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl cmp::Eq for PhantomData {} + +#[stable(feature = "rust1", since = "1.0.0")] +impl cmp::PartialOrd for PhantomData { + fn partial_cmp(&self, _other: &PhantomData) -> Option { + Option::Some(cmp::Ordering::Equal) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl cmp::Ord for PhantomData { + fn cmp(&self, _other: &PhantomData) -> cmp::Ordering { + cmp::Ordering::Equal + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Copy for PhantomData {} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Clone for PhantomData { + fn clone(&self) -> Self { + Self + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] +impl const Default for PhantomData { + fn default() -> Self { + Self + } +} + +#[unstable(feature = "structural_match", issue = "31434")] +impl StructuralPartialEq for PhantomData {} + +#[unstable(feature = "structural_match", issue = "31434")] +impl StructuralEq for PhantomData {} + +/// Compiler-internal trait used to indicate the type of enum discriminants. +/// +/// This trait is automatically implemented for every type and does not add any +/// guarantees to [`mem::Discriminant`]. It is **undefined behavior** to transmute +/// between `DiscriminantKind::Discriminant` and `mem::Discriminant`. +/// +/// [`mem::Discriminant`]: crate::mem::Discriminant +#[unstable( + feature = "discriminant_kind", + issue = "none", + reason = "this trait is unlikely to ever be stabilized, use `mem::discriminant` instead" +)] +#[lang = "discriminant_kind"] +#[rustc_deny_explicit_impl] +pub trait DiscriminantKind { + /// The type of the discriminant, which must satisfy the trait + /// bounds required by `mem::Discriminant`. + #[lang = "discriminant_type"] + type Discriminant: Clone + Copy + Debug + Eq + PartialEq + Hash + Send + Sync + Unpin; } /// Compiler-internal trait used to determine whether a type contains @@ -689,27 +725,28 @@ unsafe impl Freeze for &mut T {} /// Types that can be safely moved after being pinned. /// -/// Since Rust itself has no notion of immovable types, and considers moves -/// (e.g., through assignment or [`mem::replace`]) to always be safe, -/// this trait cannot prevent types from moving by itself. +/// Rust itself has no notion of immovable types, and considers moves (e.g., +/// through assignment or [`mem::replace`]) to always be safe. /// -/// Instead it is used to prevent moves through the type system, -/// by controlling the behavior of pointers `P` wrapped in the [`Pin

`] wrapper, -/// which "pin" the type in place by not allowing it to be moved out of them. -/// See the [`pin module`] documentation for more information on pinning. +/// The [`Pin`][Pin] type is used instead to prevent moves through the type +/// system. Pointers `P` wrapped in the [`Pin>`][Pin] wrapper can't be +/// moved out of. See the [`pin` module] documentation for more information on +/// pinning. /// -/// Implementing this trait lifts the restrictions of pinning off a type, -/// which then allows it to move out with functions such as [`mem::replace`]. +/// Implementing the `Unpin` trait for `T` lifts the restrictions of pinning off +/// the type, which then allows moving `T` out of [`Pin>`][Pin] with +/// functions such as [`mem::replace`]. /// /// `Unpin` has no consequence at all for non-pinned data. In particular, /// [`mem::replace`] happily moves `!Unpin` data (it works for any `&mut T`, not -/// just when `T: Unpin`). However, you cannot use -/// [`mem::replace`] on data wrapped inside a [`Pin

`] because you cannot get the -/// `&mut T` you need for that, and *that* is what makes this system work. +/// just when `T: Unpin`). However, you cannot use [`mem::replace`] on data +/// wrapped inside a [`Pin>`][Pin] because you cannot get the `&mut T` you +/// need for that, and *that* is what makes this system work. /// /// So this, for example, can only be done on types implementing `Unpin`: /// /// ```rust +/// # #![allow(unused_must_use)] /// use std::mem; /// use std::pin::Pin; /// @@ -724,12 +761,12 @@ unsafe impl Freeze for &mut T {} /// /// This trait is automatically implemented for almost every type. /// -/// [`mem::replace`]: ../../std/mem/fn.replace.html -/// [`Pin

`]: ../pin/struct.Pin.html -/// [`pin module`]: ../../std/pin/index.html +/// [`mem::replace`]: crate::mem::replace +/// [Pin]: crate::pin::Pin +/// [`pin` module]: crate::pin #[stable(feature = "pin", since = "1.33.0")] #[rustc_on_unimplemented( - on(_Self = "std::future::Future", note = "consider using `Box::pin`",), + note = "consider using `Box::pin`", message = "`{Self}` cannot be unpinned" )] #[lang = "unpin"] @@ -739,7 +776,7 @@ pub auto trait Unpin {} /// /// If a type contains a `PhantomPinned`, it will not implement `Unpin` by default. #[stable(feature = "pin", since = "1.33.0")] -#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] +#[derive(Debug, Default, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] pub struct PhantomPinned; #[stable(feature = "pin", since = "1.33.0")] @@ -757,10 +794,41 @@ impl Unpin for *const T {} #[stable(feature = "pin_raw", since = "1.38.0")] impl Unpin for *mut T {} +/// A marker for types that can be dropped. +/// +/// This should be used for `~const` bounds, +/// as non-const bounds will always hold for every type. +#[unstable(feature = "const_trait_impl", issue = "67792")] +#[lang = "destruct"] +#[rustc_on_unimplemented(message = "can't drop `{Self}`", append_const_msg)] +#[const_trait] +#[rustc_deny_explicit_impl] +pub trait Destruct {} + +/// A marker for tuple types. +/// +/// The implementation of this trait is built-in and cannot be implemented +/// for any user type. +#[unstable(feature = "tuple_trait", issue = "none")] +#[lang = "tuple_trait"] +#[rustc_on_unimplemented(message = "`{Self}` is not a tuple")] +#[rustc_deny_explicit_impl] +pub trait Tuple {} + +/// A marker for things +#[unstable(feature = "pointer_sized_trait", issue = "none")] +#[lang = "pointer_sized"] +#[rustc_on_unimplemented( + message = "`{Self}` needs to be a pointer-sized type", + label = "`{Self}` needs to be a pointer-sized type" +)] +pub trait PointerSized {} + /// Implementations of `Copy` for primitive types. /// /// Implementations that cannot be described in Rust -/// are implemented in `SelectionContext::copy_clone_conditions()` in librustc. +/// are implemented in `traits::SelectionContext::copy_clone_conditions()` +/// in `rustc_trait_selection`. mod copy_impls { use super::Copy; @@ -790,7 +858,7 @@ mod copy_impls { #[stable(feature = "rust1", since = "1.0.0")] impl Copy for *mut T {} - // Shared references can be copied, but mutable references *cannot*! + /// Shared references can be copied, but mutable references *cannot*! #[stable(feature = "rust1", since = "1.0.0")] impl Copy for &T {} } diff --git a/crux-mir/lib/core/src/mem/manually_drop.rs b/crux-mir/lib/core/src/mem/manually_drop.rs index 9e5c2b10d..3d719afe4 100644 --- a/crux-mir/lib/core/src/mem/manually_drop.rs +++ b/crux-mir/lib/core/src/mem/manually_drop.rs @@ -2,49 +2,47 @@ use crate::ops::{Deref, DerefMut}; use crate::ptr; /// A wrapper to inhibit compiler from automatically calling `T`’s destructor. -/// /// This wrapper is 0-cost. /// -/// `ManuallyDrop` is subject to the same layout optimizations as `T`. -/// As a consequence, it has *no effect* on the assumptions that the compiler makes -/// about all values being initialized at their type. In particular, initializing -/// a `ManuallyDrop<&mut T>` with [`mem::zeroed`] is undefined behavior. -/// If you need to handle uninitialized data, use [`MaybeUninit`] instead. +/// `ManuallyDrop` is guaranteed to have the same layout as `T`, and is subject +/// to the same layout optimizations as `T`. As a consequence, it has *no effect* +/// on the assumptions that the compiler makes about its contents. For example, +/// initializing a `ManuallyDrop<&mut T>` with [`mem::zeroed`] is undefined +/// behavior. If you need to handle uninitialized data, use [`MaybeUninit`] +/// instead. /// -/// # Examples +/// Note that accessing the value inside a `ManuallyDrop` is safe. +/// This means that a `ManuallyDrop` whose content has been dropped must not +/// be exposed through a public safe API. +/// Correspondingly, `ManuallyDrop::drop` is unsafe. /// -/// This wrapper helps with explicitly documenting the drop order dependencies between fields of -/// the type: +/// # `ManuallyDrop` and drop order. /// -/// ```rust -/// use std::mem::ManuallyDrop; -/// struct Peach; -/// struct Banana; -/// struct Melon; -/// struct FruitBox { -/// // Immediately clear there’s something non-trivial going on with these fields. -/// peach: ManuallyDrop, -/// melon: Melon, // Field that’s independent of the other two. -/// banana: ManuallyDrop, -/// } +/// Rust has a well-defined [drop order] of values. To make sure that fields or +/// locals are dropped in a specific order, reorder the declarations such that +/// the implicit drop order is the correct one. +/// +/// It is possible to use `ManuallyDrop` to control the drop order, but this +/// requires unsafe code and is hard to do correctly in the presence of +/// unwinding. +/// +/// For example, if you want to make sure that a specific field is dropped after +/// the others, make it the last field of a struct: /// -/// impl Drop for FruitBox { -/// fn drop(&mut self) { -/// unsafe { -/// // Explicit ordering in which field destructors are run specified in the intuitive -/// // location – the destructor of the structure containing the fields. -/// // Moreover, one can now reorder fields within the struct however much they want. -/// ManuallyDrop::drop(&mut self.peach); -/// ManuallyDrop::drop(&mut self.banana); -/// } -/// // After destructor for `FruitBox` runs (this function), the destructor for Melon gets -/// // invoked in the usual manner, as it is not wrapped in `ManuallyDrop`. -/// } +/// ``` +/// struct Context; +/// +/// struct Widget { +/// children: Vec, +/// // `context` will be dropped after `children`. +/// // Rust guarantees that fields are dropped in the order of declaration. +/// context: Context, /// } /// ``` /// -/// [`mem::zeroed`]: fn.zeroed.html -/// [`MaybeUninit`]: union.MaybeUninit.html +/// [drop order]: https://doc.rust-lang.org/reference/destructors.html +/// [`mem::zeroed`]: crate::mem::zeroed +/// [`MaybeUninit`]: crate::mem::MaybeUninit #[stable(feature = "manually_drop", since = "1.20.0")] #[lang = "manually_drop"] #[derive(Copy, Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -60,10 +58,14 @@ impl ManuallyDrop { /// /// ```rust /// use std::mem::ManuallyDrop; - /// ManuallyDrop::new(Box::new(())); + /// let mut x = ManuallyDrop::new(String::from("Hello World!")); + /// x.truncate(5); // You can still safely operate on the value + /// assert_eq!(*x, "Hello"); + /// // But `Drop` will not be run here /// ``` + #[must_use = "if you don't need the wrapper, you can use `mem::forget` instead"] #[stable(feature = "manually_drop", since = "1.20.0")] - #[rustc_const_stable(feature = "const_manually_drop", since = "1.36.0")] + #[rustc_const_stable(feature = "const_manually_drop", since = "1.32.0")] #[inline(always)] pub const fn new(value: T) -> ManuallyDrop { ManuallyDrop { value } @@ -81,7 +83,7 @@ impl ManuallyDrop { /// let _: Box<()> = ManuallyDrop::into_inner(x); // This drops the `Box`. /// ``` #[stable(feature = "manually_drop", since = "1.20.0")] - #[rustc_const_stable(feature = "const_manually_drop", since = "1.36.0")] + #[rustc_const_stable(feature = "const_manually_drop", since = "1.32.0")] #[inline(always)] pub const fn into_inner(slot: ManuallyDrop) -> T { slot.value @@ -102,39 +104,50 @@ impl ManuallyDrop { /// leaving the state of this container unchanged. /// It is your responsibility to ensure that this `ManuallyDrop` is not used again. /// - /// [`ManuallyDrop::drop`]: #method.drop - /// [`ManuallyDrop::into_inner`]: #method.into_inner #[must_use = "if you don't need the value, you can use `ManuallyDrop::drop` instead"] #[stable(feature = "manually_drop_take", since = "1.42.0")] #[inline] pub unsafe fn take(slot: &mut ManuallyDrop) -> T { - ptr::read(&slot.value) + // SAFETY: we are reading from a reference, which is guaranteed + // to be valid for reads. + unsafe { ptr::read(&slot.value) } } } impl ManuallyDrop { - /// Manually drops the contained value. + /// Manually drops the contained value. This is exactly equivalent to calling + /// [`ptr::drop_in_place`] with a pointer to the contained value. As such, unless + /// the contained value is a packed struct, the destructor will be called in-place + /// without moving the value, and thus can be used to safely drop [pinned] data. /// /// If you have ownership of the value, you can use [`ManuallyDrop::into_inner`] instead. /// /// # Safety /// - /// This function runs the destructor of the contained value and thus the wrapped value - /// now represents uninitialized data. It is up to the user of this method to ensure the - /// uninitialized data is not actually used. - /// In particular, this function can only be called at most once - /// for a given instance of `ManuallyDrop`. + /// This function runs the destructor of the contained value. Other than changes made by + /// the destructor itself, the memory is left unchanged, and so as far as the compiler is + /// concerned still holds a bit-pattern which is valid for the type `T`. + /// + /// However, this "zombie" value should not be exposed to safe code, and this function + /// should not be called more than once. To use a value after it's been dropped, or drop + /// a value multiple times, can cause Undefined Behavior (depending on what `drop` does). + /// This is normally prevented by the type system, but users of `ManuallyDrop` must + /// uphold those guarantees without assistance from the compiler. /// - /// [`ManuallyDrop::into_inner`]: #method.into_inner + /// [pinned]: crate::pin #[stable(feature = "manually_drop", since = "1.20.0")] #[inline] pub unsafe fn drop(slot: &mut ManuallyDrop) { - ptr::drop_in_place(&mut slot.value) + // SAFETY: we are dropping the value pointed to by a mutable reference + // which is guaranteed to be valid for writes. + // It is up to the caller to make sure that `slot` isn't dropped again. + unsafe { ptr::drop_in_place(&mut slot.value) } } } #[stable(feature = "manually_drop", since = "1.20.0")] -impl Deref for ManuallyDrop { +#[rustc_const_unstable(feature = "const_deref", issue = "88955")] +impl const Deref for ManuallyDrop { type Target = T; #[inline(always)] fn deref(&self) -> &T { @@ -143,7 +156,8 @@ impl Deref for ManuallyDrop { } #[stable(feature = "manually_drop", since = "1.20.0")] -impl DerefMut for ManuallyDrop { +#[rustc_const_unstable(feature = "const_deref", issue = "88955")] +impl const DerefMut for ManuallyDrop { #[inline(always)] fn deref_mut(&mut self) -> &mut T { &mut self.value diff --git a/crux-mir/lib/core/src/mem/maybe_uninit.rs b/crux-mir/lib/core/src/mem/maybe_uninit.rs index bf39d56fc..3f4918365 100644 --- a/crux-mir/lib/core/src/mem/maybe_uninit.rs +++ b/crux-mir/lib/core/src/mem/maybe_uninit.rs @@ -1,9 +1,9 @@ use crate::any::type_name; use crate::fmt; use crate::intrinsics; -use crate::mem::ManuallyDrop; - -// ignore-tidy-undocumented-unsafe +use crate::mem::{self, ManuallyDrop}; +use crate::ptr; +use crate::slice; /// A wrapper type to construct uninitialized instances of `T`. /// @@ -11,7 +11,7 @@ use crate::mem::ManuallyDrop; /// /// The compiler, in general, assumes that a variable is properly initialized /// according to the requirements of the variable's type. For example, a variable of -/// reference type must be aligned and non-NULL. This is an invariant that must +/// reference type must be aligned and non-null. This is an invariant that must /// *always* be upheld, even in unsafe code. As a consequence, zero-initializing a /// variable of reference type causes instantaneous [undefined behavior][ub], /// no matter whether that reference ever gets used to access memory: @@ -20,9 +20,9 @@ use crate::mem::ManuallyDrop; /// # #![allow(invalid_value)] /// use std::mem::{self, MaybeUninit}; /// -/// let x: &i32 = unsafe { mem::zeroed() }; // undefined behavior! +/// let x: &i32 = unsafe { mem::zeroed() }; // undefined behavior! ⚠️ /// // The equivalent code with `MaybeUninit<&i32>`: -/// let x: &i32 = unsafe { MaybeUninit::zeroed().assume_init() }; // undefined behavior! +/// let x: &i32 = unsafe { MaybeUninit::zeroed().assume_init() }; // undefined behavior! ⚠️ /// ``` /// /// This is exploited by the compiler for various optimizations, such as eliding @@ -35,27 +35,25 @@ use crate::mem::ManuallyDrop; /// # #![allow(invalid_value)] /// use std::mem::{self, MaybeUninit}; /// -/// let b: bool = unsafe { mem::uninitialized() }; // undefined behavior! +/// let b: bool = unsafe { mem::uninitialized() }; // undefined behavior! ⚠️ /// // The equivalent code with `MaybeUninit`: -/// let b: bool = unsafe { MaybeUninit::uninit().assume_init() }; // undefined behavior! +/// let b: bool = unsafe { MaybeUninit::uninit().assume_init() }; // undefined behavior! ⚠️ /// ``` /// -/// Moreover, uninitialized memory is special in that the compiler knows that -/// it does not have a fixed value. This makes it undefined behavior to have -/// uninitialized data in a variable even if that variable has an integer type, -/// which otherwise can hold any *fixed* bit pattern: +/// Moreover, uninitialized memory is special in that it does not have a fixed value ("fixed" +/// meaning "it won't change without being written to"). Reading the same uninitialized byte +/// multiple times can give different results. This makes it undefined behavior to have +/// uninitialized data in a variable even if that variable has an integer type, which otherwise can +/// hold any *fixed* bit pattern: /// /// ```rust,no_run /// # #![allow(invalid_value)] /// use std::mem::{self, MaybeUninit}; /// -/// let x: i32 = unsafe { mem::uninitialized() }; // undefined behavior! +/// let x: i32 = unsafe { mem::uninitialized() }; // undefined behavior! ⚠️ /// // The equivalent code with `MaybeUninit`: -/// let x: i32 = unsafe { MaybeUninit::uninit().assume_init() }; // undefined behavior! +/// let x: i32 = unsafe { MaybeUninit::uninit().assume_init() }; // undefined behavior! ⚠️ /// ``` -/// (Notice that the rules around uninitialized integers are not finalized yet, but -/// until they are, it is advisable to avoid them.) -/// /// On top of that, remember that most types have additional invariants beyond merely /// being considered initialized at the type level. For example, a `1`-initialized [`Vec`] /// is considered initialized (under the current implementation; this does not constitute @@ -79,7 +77,7 @@ use crate::mem::ManuallyDrop; /// // a `MaybeUninit` may be invalid, and hence this is not UB: /// let mut x = MaybeUninit::<&i32>::uninit(); /// // Set it to a valid value. -/// unsafe { x.as_mut_ptr().write(&0); } +/// x.write(&0); /// // Extract the initialized data -- this is only allowed *after* properly /// // initializing `x`! /// let x = unsafe { x.assume_init() }; @@ -129,13 +127,10 @@ use crate::mem::ManuallyDrop; /// MaybeUninit::uninit().assume_init() /// }; /// -/// // Dropping a `MaybeUninit` does nothing. Thus using raw pointer -/// // assignment instead of `ptr::write` does not cause the old -/// // uninitialized value to be dropped. Also if there is a panic during -/// // this loop, we have a memory leak, but there is no memory safety -/// // issue. +/// // Dropping a `MaybeUninit` does nothing, so if there is a panic during this loop, +/// // we have a memory leak, but there is no memory safety issue. /// for elem in &mut data[..] { -/// *elem = MaybeUninit::new(vec![42]); +/// elem.write(vec![42]); /// } /// /// // Everything is initialized. Transmute the array to the @@ -151,7 +146,6 @@ use crate::mem::ManuallyDrop; /// /// ``` /// use std::mem::MaybeUninit; -/// use std::ptr; /// /// // Create an uninitialized array of `MaybeUninit`. The `assume_init` is /// // safe because the type we are claiming to have initialized here is a @@ -161,23 +155,56 @@ use crate::mem::ManuallyDrop; /// let mut data_len: usize = 0; /// /// for elem in &mut data[0..500] { -/// *elem = MaybeUninit::new(String::from("hello")); +/// elem.write(String::from("hello")); /// data_len += 1; /// } /// /// // For each item in the array, drop if we allocated it. /// for elem in &mut data[0..data_len] { -/// unsafe { ptr::drop_in_place(elem.as_mut_ptr()); } +/// unsafe { elem.assume_init_drop(); } /// } /// ``` /// /// ## Initializing a struct field-by-field /// -/// There is currently no supported way to create a raw pointer or reference -/// to a field of a struct inside `MaybeUninit`. That means it is not possible -/// to create a struct by calling `MaybeUninit::uninit::()` and then writing -/// to its fields. +/// You can use `MaybeUninit`, and the [`std::ptr::addr_of_mut`] macro, to initialize structs field by field: +/// +/// ```rust +/// use std::mem::MaybeUninit; +/// use std::ptr::addr_of_mut; +/// +/// #[derive(Debug, PartialEq)] +/// pub struct Foo { +/// name: String, +/// list: Vec, +/// } +/// +/// let foo = { +/// let mut uninit: MaybeUninit = MaybeUninit::uninit(); +/// let ptr = uninit.as_mut_ptr(); /// +/// // Initializing the `name` field +/// // Using `write` instead of assignment via `=` to not call `drop` on the +/// // old, uninitialized value. +/// unsafe { addr_of_mut!((*ptr).name).write("Bob".to_string()); } +/// +/// // Initializing the `list` field +/// // If there is a panic here, then the `String` in the `name` field leaks. +/// unsafe { addr_of_mut!((*ptr).list).write(vec![0, 1, 2]); } +/// +/// // All the fields are initialized, so we call `assume_init` to get an initialized Foo. +/// unsafe { uninit.assume_init() } +/// }; +/// +/// assert_eq!( +/// foo, +/// Foo { +/// name: "Bob".to_string(), +/// list: vec![0, 1, 2] +/// } +/// ); +/// ``` +/// [`std::ptr::addr_of_mut`]: crate::ptr::addr_of_mut /// [ub]: ../../reference/behavior-considered-undefined.html /// /// # Layout @@ -214,7 +241,6 @@ use crate::mem::ManuallyDrop; /// remain `#[repr(transparent)]`. That said, `MaybeUninit` will *always* guarantee that it has /// the same size, alignment, and ABI as `T`; it's just that the way `MaybeUninit` implements that /// guarantee may evolve. -#[allow(missing_debug_implementations)] #[stable(feature = "maybe_uninit", since = "1.36.0")] // Lang item so we can wrap other types in it. This is useful for generators. #[lang = "maybe_uninit"] @@ -248,9 +274,18 @@ impl MaybeUninit { /// Note that dropping a `MaybeUninit` will never call `T`'s drop code. /// It is your responsibility to make sure `T` gets dropped if it got initialized. /// - /// [`assume_init`]: #method.assume_init + /// # Example + /// + /// ``` + /// use std::mem::MaybeUninit; + /// + /// let v: MaybeUninit> = MaybeUninit::new(vec![42]); + /// ``` + /// + /// [`assume_init`]: MaybeUninit::assume_init #[stable(feature = "maybe_uninit", since = "1.36.0")] #[rustc_const_stable(feature = "const_maybe_uninit", since = "1.36.0")] + #[must_use = "use `forget` to avoid running Drop code"] #[inline(always)] pub const fn new(val: T) -> MaybeUninit { MaybeUninit { value: ManuallyDrop::new(val) } @@ -261,11 +296,18 @@ impl MaybeUninit { /// Note that dropping a `MaybeUninit` will never call `T`'s drop code. /// It is your responsibility to make sure `T` gets dropped if it got initialized. /// - /// See the [type-level documentation][type] for some examples. + /// See the [type-level documentation][MaybeUninit] for some examples. + /// + /// # Example /// - /// [type]: union.MaybeUninit.html + /// ``` + /// use std::mem::MaybeUninit; + /// + /// let v: MaybeUninit = MaybeUninit::uninit(); + /// ``` #[stable(feature = "maybe_uninit", since = "1.36.0")] #[rustc_const_stable(feature = "const_maybe_uninit", since = "1.36.0")] + #[must_use] #[inline(always)] #[rustc_diagnostic_item = "maybe_uninit_uninit"] pub const fn uninit() -> MaybeUninit { @@ -275,14 +317,14 @@ impl MaybeUninit { /// Create a new array of `MaybeUninit` items, in an uninitialized state. /// /// Note: in a future Rust version this method may become unnecessary - /// when array literal syntax allows - /// [repeating const expressions](https://github.com/rust-lang/rust/issues/49147). - /// The example below could then use `let mut buf = [MaybeUninit::::uninit(); 32];`. + /// when Rust allows + /// [inline const expressions](https://github.com/rust-lang/rust/issues/76001). + /// The example below could then use `let mut buf = [const { MaybeUninit::::uninit() }; 32];`. /// /// # Examples /// /// ```no_run - /// #![feature(maybe_uninit_uninit_array, maybe_uninit_extra, maybe_uninit_slice_assume_init)] + /// #![feature(maybe_uninit_uninit_array, maybe_uninit_slice)] /// /// use std::mem::MaybeUninit; /// @@ -294,27 +336,22 @@ impl MaybeUninit { /// fn read(buf: &mut [MaybeUninit]) -> &[u8] { /// unsafe { /// let len = read_into_buffer(buf.as_mut_ptr() as *mut u8, buf.len()); - /// MaybeUninit::slice_get_ref(&buf[..len]) + /// MaybeUninit::slice_assume_init_ref(&buf[..len]) /// } /// } /// /// let mut buf: [MaybeUninit; 32] = MaybeUninit::uninit_array(); /// let data = read(&mut buf); /// ``` - #[unstable(feature = "maybe_uninit_uninit_array", issue = "none")] + #[unstable(feature = "maybe_uninit_uninit_array", issue = "96097")] + #[rustc_const_unstable(feature = "const_maybe_uninit_uninit_array", issue = "96097")] + #[must_use] #[inline(always)] - pub fn uninit_array() -> [Self; LEN] { - unsafe { MaybeUninit::<[MaybeUninit; LEN]>::uninit().assume_init() } + pub const fn uninit_array() -> [Self; N] { + // SAFETY: An uninitialized `[MaybeUninit<_>; LEN]` is valid. + unsafe { MaybeUninit::<[MaybeUninit; N]>::uninit().assume_init() } } - /// A promotable constant, equivalent to `uninit()`. - #[unstable( - feature = "internal_uninit_const", - issue = "none", - reason = "hack to work around promotability" - )] - pub const UNINIT: Self = Self::uninit(); - /// Creates a new `MaybeUninit` in an uninitialized state, with the memory being /// filled with `0` bytes. It depends on `T` whether that already makes for /// proper initialization. For example, `MaybeUninit::zeroed()` is initialized, @@ -337,41 +374,121 @@ impl MaybeUninit { /// assert_eq!(x, (0, false)); /// ``` /// - /// *Incorrect* usage of this function: initializing a struct with zero, where some fields - /// cannot hold 0 as a valid value. + /// *Incorrect* usage of this function: calling `x.zeroed().assume_init()` + /// when `0` is not a valid bit-pattern for the type: /// /// ```rust,no_run /// use std::mem::MaybeUninit; /// - /// enum NotZero { One = 1, Two = 2 }; + /// enum NotZero { One = 1, Two = 2 } /// /// let x = MaybeUninit::<(u8, NotZero)>::zeroed(); /// let x = unsafe { x.assume_init() }; /// // Inside a pair, we create a `NotZero` that does not have a valid discriminant. - /// // This is undefined behavior. + /// // This is undefined behavior. ⚠️ /// ``` #[stable(feature = "maybe_uninit", since = "1.36.0")] + #[rustc_const_unstable(feature = "const_maybe_uninit_zeroed", issue = "91850")] + #[must_use] #[inline] #[rustc_diagnostic_item = "maybe_uninit_zeroed"] - pub fn zeroed() -> MaybeUninit { + pub const fn zeroed() -> MaybeUninit { let mut u = MaybeUninit::::uninit(); + // SAFETY: `u.as_mut_ptr()` points to allocated memory. unsafe { u.as_mut_ptr().write_bytes(0u8, 1); } u } - /// Sets the value of the `MaybeUninit`. This overwrites any previous value - /// without dropping it, so be careful not to use this twice unless you want to - /// skip running the destructor. For your convenience, this also returns a mutable - /// reference to the (now safely initialized) contents of `self`. - #[unstable(feature = "maybe_uninit_extra", issue = "63567")] + /// Sets the value of the `MaybeUninit`. + /// + /// This overwrites any previous value without dropping it, so be careful + /// not to use this twice unless you want to skip running the destructor. + /// For your convenience, this also returns a mutable reference to the + /// (now safely initialized) contents of `self`. + /// + /// As the content is stored inside a `MaybeUninit`, the destructor is not + /// run for the inner data if the MaybeUninit leaves scope without a call to + /// [`assume_init`], [`assume_init_drop`], or similar. Code that receives + /// the mutable reference returned by this function needs to keep this in + /// mind. The safety model of Rust regards leaks as safe, but they are + /// usually still undesirable. This being said, the mutable reference + /// behaves like any other mutable reference would, so assigning a new value + /// to it will drop the old content. + /// + /// [`assume_init`]: Self::assume_init + /// [`assume_init_drop`]: Self::assume_init_drop + /// + /// # Examples + /// + /// Correct usage of this method: + /// + /// ```rust + /// use std::mem::MaybeUninit; + /// + /// let mut x = MaybeUninit::>::uninit(); + /// + /// { + /// let hello = x.write((&b"Hello, world!").to_vec()); + /// // Setting hello does not leak prior allocations, but drops them + /// *hello = (&b"Hello").to_vec(); + /// hello[0] = 'h' as u8; + /// } + /// // x is initialized now: + /// let s = unsafe { x.assume_init() }; + /// assert_eq!(b"hello", s.as_slice()); + /// ``` + /// + /// This usage of the method causes a leak: + /// + /// ```rust + /// use std::mem::MaybeUninit; + /// + /// let mut x = MaybeUninit::::uninit(); + /// + /// x.write("Hello".to_string()); + /// // This leaks the contained string: + /// x.write("hello".to_string()); + /// // x is initialized now: + /// let s = unsafe { x.assume_init() }; + /// ``` + /// + /// This method can be used to avoid unsafe in some cases. The example below + /// shows a part of an implementation of a fixed sized arena that lends out + /// pinned references. + /// With `write`, we can avoid the need to write through a raw pointer: + /// + /// ```rust + /// use core::pin::Pin; + /// use core::mem::MaybeUninit; + /// + /// struct PinArena { + /// memory: Box<[MaybeUninit]>, + /// len: usize, + /// } + /// + /// impl PinArena { + /// pub fn capacity(&self) -> usize { + /// self.memory.len() + /// } + /// pub fn push(&mut self, val: T) -> Pin<&mut T> { + /// if self.len >= self.capacity() { + /// panic!("Attempted to push to a full pin arena!"); + /// } + /// let ref_ = self.memory[self.len].write(val); + /// self.len += 1; + /// unsafe { Pin::new_unchecked(ref_) } + /// } + /// } + /// ``` + #[stable(feature = "maybe_uninit_write", since = "1.55.0")] + #[rustc_const_unstable(feature = "const_maybe_uninit_write", issue = "63567")] #[inline(always)] - pub fn write(&mut self, val: T) -> &mut T { - unsafe { - self.value = ManuallyDrop::new(val); - self.get_mut() - } + pub const fn write(&mut self, val: T) -> &mut T { + *self = MaybeUninit::new(val); + // SAFETY: We just initialized this value. + unsafe { self.assume_init_mut() } } /// Gets a pointer to the contained value. Reading from this pointer or turning it @@ -387,7 +504,7 @@ impl MaybeUninit { /// use std::mem::MaybeUninit; /// /// let mut x = MaybeUninit::>::uninit(); - /// unsafe { x.as_mut_ptr().write(vec![0,1,2]); } + /// x.write(vec![0, 1, 2]); /// // Create a reference into the `MaybeUninit`. This is okay because we initialized it. /// let x_vec = unsafe { &*x.as_ptr() }; /// assert_eq!(x_vec.len(), 3); @@ -400,15 +517,17 @@ impl MaybeUninit { /// /// let x = MaybeUninit::>::uninit(); /// let x_vec = unsafe { &*x.as_ptr() }; - /// // We have created a reference to an uninitialized vector! This is undefined behavior. + /// // We have created a reference to an uninitialized vector! This is undefined behavior. ⚠️ /// ``` /// /// (Notice that the rules around references to uninitialized data are not finalized yet, but /// until they are, it is advisable to avoid them.) #[stable(feature = "maybe_uninit", since = "1.36.0")] + #[rustc_const_stable(feature = "const_maybe_uninit_as_ptr", since = "1.59.0")] #[inline(always)] - pub fn as_ptr(&self) -> *const T { - unsafe { &*self.value as *const T } + pub const fn as_ptr(&self) -> *const T { + // `MaybeUninit` and `ManuallyDrop` are both `repr(transparent)` so we can cast the pointer. + self as *const _ as *const T } /// Gets a mutable pointer to the contained value. Reading from this pointer or turning it @@ -422,7 +541,7 @@ impl MaybeUninit { /// use std::mem::MaybeUninit; /// /// let mut x = MaybeUninit::>::uninit(); - /// unsafe { x.as_mut_ptr().write(vec![0,1,2]); } + /// x.write(vec![0, 1, 2]); /// // Create a reference into the `MaybeUninit>`. /// // This is okay because we initialized it. /// let x_vec = unsafe { &mut *x.as_mut_ptr() }; @@ -437,15 +556,17 @@ impl MaybeUninit { /// /// let mut x = MaybeUninit::>::uninit(); /// let x_vec = unsafe { &mut *x.as_mut_ptr() }; - /// // We have created a reference to an uninitialized vector! This is undefined behavior. + /// // We have created a reference to an uninitialized vector! This is undefined behavior. ⚠️ /// ``` /// /// (Notice that the rules around references to uninitialized data are not finalized yet, but /// until they are, it is advisable to avoid them.) #[stable(feature = "maybe_uninit", since = "1.36.0")] + #[rustc_const_unstable(feature = "const_maybe_uninit_as_mut_ptr", issue = "75251")] #[inline(always)] - pub fn as_mut_ptr(&mut self) -> *mut T { - unsafe { &mut *self.value as *mut T } + pub const fn as_mut_ptr(&mut self) -> *mut T { + // `MaybeUninit` and `ManuallyDrop` are both `repr(transparent)` so we can cast the pointer. + self as *mut _ as *mut T } /// Extracts the value from the `MaybeUninit` container. This is a great way @@ -469,6 +590,8 @@ impl MaybeUninit { /// *immediate* undefined behavior, but will cause undefined behavior with most /// safe operations (including dropping it). /// + /// [`Vec`]: ../../std/vec/struct.Vec.html + /// /// # Examples /// /// Correct usage of this method: @@ -477,7 +600,7 @@ impl MaybeUninit { /// use std::mem::MaybeUninit; /// /// let mut x = MaybeUninit::::uninit(); - /// unsafe { x.as_mut_ptr().write(true); } + /// x.write(true); /// let x_init = unsafe { x.assume_init() }; /// assert_eq!(x_init, true); /// ``` @@ -489,17 +612,20 @@ impl MaybeUninit { /// /// let x = MaybeUninit::>::uninit(); /// let x_init = unsafe { x.assume_init() }; - /// // `x` had not been initialized yet, so this last line caused undefined behavior. + /// // `x` had not been initialized yet, so this last line caused undefined behavior. ⚠️ /// ``` #[stable(feature = "maybe_uninit", since = "1.36.0")] + #[rustc_const_stable(feature = "const_maybe_uninit_assume_init_by_value", since = "1.59.0")] #[inline(always)] #[rustc_diagnostic_item = "assume_init"] - pub unsafe fn assume_init(self) -> T { - #[cfg(bootstrap)] - intrinsics::panic_if_uninhabited::(); - #[cfg(not(bootstrap))] - intrinsics::assert_inhabited::(); - ManuallyDrop::into_inner(self.value) + #[track_caller] + pub const unsafe fn assume_init(self) -> T { + // SAFETY: the caller must guarantee that `self` is initialized. + // This also means that `self` must be a `value` variant. + unsafe { + intrinsics::assert_inhabited::(); + ManuallyDrop::into_inner(self.value) + } } /// Reads the value from the `MaybeUninit` container. The resulting `T` is subject @@ -515,58 +641,91 @@ impl MaybeUninit { /// behavior. The [type-level documentation][inv] contains more information about /// this initialization invariant. /// - /// Moreover, this leaves a copy of the same data behind in the `MaybeUninit`. When using - /// multiple copies of the data (by calling `read` multiple times, or first - /// calling `read` and then [`assume_init`]), it is your responsibility - /// to ensure that that data may indeed be duplicated. + /// Moreover, similar to the [`ptr::read`] function, this function creates a + /// bitwise copy of the contents, regardless whether the contained type + /// implements the [`Copy`] trait or not. When using multiple copies of the + /// data (by calling `assume_init_read` multiple times, or first calling + /// `assume_init_read` and then [`assume_init`]), it is your responsibility + /// to ensure that data may indeed be duplicated. /// /// [inv]: #initialization-invariant - /// [`assume_init`]: #method.assume_init + /// [`assume_init`]: MaybeUninit::assume_init /// /// # Examples /// /// Correct usage of this method: /// /// ```rust - /// #![feature(maybe_uninit_extra)] /// use std::mem::MaybeUninit; /// /// let mut x = MaybeUninit::::uninit(); /// x.write(13); - /// let x1 = unsafe { x.read() }; + /// let x1 = unsafe { x.assume_init_read() }; /// // `u32` is `Copy`, so we may read multiple times. - /// let x2 = unsafe { x.read() }; + /// let x2 = unsafe { x.assume_init_read() }; /// assert_eq!(x1, x2); /// /// let mut x = MaybeUninit::>>::uninit(); /// x.write(None); - /// let x1 = unsafe { x.read() }; + /// let x1 = unsafe { x.assume_init_read() }; /// // Duplicating a `None` value is okay, so we may read multiple times. - /// let x2 = unsafe { x.read() }; + /// let x2 = unsafe { x.assume_init_read() }; /// assert_eq!(x1, x2); /// ``` /// /// *Incorrect* usage of this method: /// /// ```rust,no_run - /// #![feature(maybe_uninit_extra)] /// use std::mem::MaybeUninit; /// /// let mut x = MaybeUninit::>>::uninit(); - /// x.write(Some(vec![0,1,2])); - /// let x1 = unsafe { x.read() }; - /// let x2 = unsafe { x.read() }; - /// // We now created two copies of the same vector, leading to a double-free when + /// x.write(Some(vec![0, 1, 2])); + /// let x1 = unsafe { x.assume_init_read() }; + /// let x2 = unsafe { x.assume_init_read() }; + /// // We now created two copies of the same vector, leading to a double-free ⚠️ when /// // they both get dropped! /// ``` - #[unstable(feature = "maybe_uninit_extra", issue = "63567")] + #[stable(feature = "maybe_uninit_extra", since = "1.60.0")] + #[rustc_const_unstable(feature = "const_maybe_uninit_assume_init_read", issue = "63567")] #[inline(always)] - pub unsafe fn read(&self) -> T { - #[cfg(bootstrap)] - intrinsics::panic_if_uninhabited::(); - #[cfg(not(bootstrap))] - intrinsics::assert_inhabited::(); - self.as_ptr().read() + #[track_caller] + pub const unsafe fn assume_init_read(&self) -> T { + // SAFETY: the caller must guarantee that `self` is initialized. + // Reading from `self.as_ptr()` is safe since `self` should be initialized. + unsafe { + intrinsics::assert_inhabited::(); + self.as_ptr().read() + } + } + + /// Drops the contained value in place. + /// + /// If you have ownership of the `MaybeUninit`, you can also use + /// [`assume_init`] as an alternative. + /// + /// # Safety + /// + /// It is up to the caller to guarantee that the `MaybeUninit` really is + /// in an initialized state. Calling this when the content is not yet fully + /// initialized causes undefined behavior. + /// + /// On top of that, all additional invariants of the type `T` must be + /// satisfied, as the `Drop` implementation of `T` (or its members) may + /// rely on this. For example, setting a [`Vec`] to an invalid but + /// non-null address makes it initialized (under the current implementation; + /// this does not constitute a stable guarantee), because the only + /// requirement the compiler knows about it is that the data pointer must be + /// non-null. Dropping such a `Vec` however will cause undefined + /// behaviour. + /// + /// [`assume_init`]: MaybeUninit::assume_init + /// [`Vec`]: ../../std/vec/struct.Vec.html + #[stable(feature = "maybe_uninit_extra", since = "1.60.0")] + pub unsafe fn assume_init_drop(&mut self) { + // SAFETY: the caller must guarantee that `self` is initialized and + // satisfies all invariants of `T`. + // Dropping the value in place is safe if that is the case. + unsafe { ptr::drop_in_place(self.as_mut_ptr()) } } /// Gets a shared reference to the contained value. @@ -586,17 +745,16 @@ impl MaybeUninit { /// ### Correct usage of this method: /// /// ```rust - /// #![feature(maybe_uninit_ref)] /// use std::mem::MaybeUninit; /// /// let mut x = MaybeUninit::>::uninit(); /// // Initialize `x`: - /// unsafe { x.as_mut_ptr().write(vec![1, 2, 3]); } + /// x.write(vec![1, 2, 3]); /// // Now that our `MaybeUninit<_>` is known to be initialized, it is okay to /// // create a shared reference to it: /// let x: &Vec = unsafe { - /// // Safety: `x` has been initialized. - /// x.get_ref() + /// // SAFETY: `x` has been initialized. + /// x.assume_init_ref() /// }; /// assert_eq!(x, &vec![1, 2, 3]); /// ``` @@ -604,34 +762,34 @@ impl MaybeUninit { /// ### *Incorrect* usages of this method: /// /// ```rust,no_run - /// #![feature(maybe_uninit_ref)] /// use std::mem::MaybeUninit; /// /// let x = MaybeUninit::>::uninit(); - /// let x_vec: &Vec = unsafe { x.get_ref() }; - /// // We have created a reference to an uninitialized vector! This is undefined behavior. + /// let x_vec: &Vec = unsafe { x.assume_init_ref() }; + /// // We have created a reference to an uninitialized vector! This is undefined behavior. ⚠️ /// ``` /// /// ```rust,no_run - /// #![feature(maybe_uninit_ref)] /// use std::{cell::Cell, mem::MaybeUninit}; /// /// let b = MaybeUninit::>::uninit(); /// // Initialize the `MaybeUninit` using `Cell::set`: /// unsafe { - /// b.get_ref().set(true); - /// // ^^^^^^^^^^^ - /// // Reference to an uninitialized `Cell`: UB! + /// b.assume_init_ref().set(true); + /// // ^^^^^^^^^^^^^^^ + /// // Reference to an uninitialized `Cell`: UB! /// } /// ``` - #[unstable(feature = "maybe_uninit_ref", issue = "63568")] + #[stable(feature = "maybe_uninit_ref", since = "1.55.0")] + #[rustc_const_stable(feature = "const_maybe_uninit_assume_init_ref", since = "1.59.0")] #[inline(always)] - pub unsafe fn get_ref(&self) -> &T { - #[cfg(bootstrap)] - intrinsics::panic_if_uninhabited::(); - #[cfg(not(bootstrap))] - intrinsics::assert_inhabited::(); - &*self.value + pub const unsafe fn assume_init_ref(&self) -> &T { + // SAFETY: the caller must guarantee that `self` is initialized. + // This also means that `self` must be a `value` variant. + unsafe { + intrinsics::assert_inhabited::(); + &*self.as_ptr() + } } /// Gets a mutable (unique) reference to the contained value. @@ -644,7 +802,7 @@ impl MaybeUninit { /// /// Calling this when the content is not yet fully initialized causes undefined /// behavior: it is up to the caller to guarantee that the `MaybeUninit` really - /// is in an initialized state. For instance, `.get_mut()` cannot be used to + /// is in an initialized state. For instance, `.assume_init_mut()` cannot be used to /// initialize a `MaybeUninit`. /// /// # Examples @@ -652,27 +810,27 @@ impl MaybeUninit { /// ### Correct usage of this method: /// /// ```rust - /// #![feature(maybe_uninit_ref)] + /// # #![allow(unexpected_cfgs)] /// use std::mem::MaybeUninit; /// - /// # unsafe extern "C" fn initialize_buffer(buf: *mut [u8; 2048]) { *buf = [0; 2048] } + /// # unsafe extern "C" fn initialize_buffer(buf: *mut [u8; 1024]) { *buf = [0; 1024] } /// # #[cfg(FALSE)] /// extern "C" { /// /// Initializes *all* the bytes of the input buffer. - /// fn initialize_buffer(buf: *mut [u8; 2048]); + /// fn initialize_buffer(buf: *mut [u8; 1024]); /// } /// - /// let mut buf = MaybeUninit::<[u8; 2048]>::uninit(); + /// let mut buf = MaybeUninit::<[u8; 1024]>::uninit(); /// /// // Initialize `buf`: /// unsafe { initialize_buffer(buf.as_mut_ptr()); } /// // Now we know that `buf` has been initialized, so we could `.assume_init()` it. - /// // However, using `.assume_init()` may trigger a `memcpy` of the 2048 bytes. + /// // However, using `.assume_init()` may trigger a `memcpy` of the 1024 bytes. /// // To assert our buffer has been initialized without copying it, we upgrade - /// // the `&mut MaybeUninit<[u8; 2048]>` to a `&mut [u8; 2048]`: - /// let buf: &mut [u8; 2048] = unsafe { - /// // Safety: `buf` has been initialized. - /// buf.get_mut() + /// // the `&mut MaybeUninit<[u8; 1024]>` to a `&mut [u8; 1024]`: + /// let buf: &mut [u8; 1024] = unsafe { + /// // SAFETY: `buf` has been initialized. + /// buf.assume_init_mut() /// }; /// /// // Now we can use `buf` as a normal slice: @@ -685,33 +843,31 @@ impl MaybeUninit { /// /// ### *Incorrect* usages of this method: /// - /// You cannot use `.get_mut()` to initialize a value: + /// You cannot use `.assume_init_mut()` to initialize a value: /// /// ```rust,no_run - /// #![feature(maybe_uninit_ref)] /// use std::mem::MaybeUninit; /// /// let mut b = MaybeUninit::::uninit(); /// unsafe { - /// *b.get_mut() = true; + /// *b.assume_init_mut() = true; /// // We have created a (mutable) reference to an uninitialized `bool`! - /// // This is undefined behavior. + /// // This is undefined behavior. ⚠️ /// } /// ``` /// /// For instance, you cannot [`Read`] into an uninitialized buffer: /// - /// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html + /// [`Read`]: ../../std/io/trait.Read.html /// /// ```rust,no_run - /// #![feature(maybe_uninit_ref)] /// use std::{io, mem::MaybeUninit}; /// /// fn read_chunk (reader: &'_ mut dyn io::Read) -> io::Result<[u8; 64]> /// { /// let mut buffer = MaybeUninit::<[u8; 64]>::uninit(); - /// reader.read_exact(unsafe { buffer.get_mut() })?; - /// // ^^^^^^^^^^^^^^^^ + /// reader.read_exact(unsafe { buffer.assume_init_mut() })?; + /// // ^^^^^^^^^^^^^^^^^^^^^^^^ /// // (mutable) reference to uninitialized memory! /// // This is undefined behavior. /// Ok(unsafe { buffer.assume_init() }) @@ -721,7 +877,6 @@ impl MaybeUninit { /// Nor can you use direct field access to do field-by-field gradual initialization: /// /// ```rust,no_run - /// #![feature(maybe_uninit_ref)] /// use std::{mem::MaybeUninit, ptr}; /// /// struct Foo { @@ -731,28 +886,73 @@ impl MaybeUninit { /// /// let foo: Foo = unsafe { /// let mut foo = MaybeUninit::::uninit(); - /// ptr::write(&mut foo.get_mut().a as *mut u32, 1337); - /// // ^^^^^^^^^^^^^ + /// ptr::write(&mut foo.assume_init_mut().a as *mut u32, 1337); + /// // ^^^^^^^^^^^^^^^^^^^^^ /// // (mutable) reference to uninitialized memory! /// // This is undefined behavior. - /// ptr::write(&mut foo.get_mut().b as *mut u8, 42); - /// // ^^^^^^^^^^^^^ + /// ptr::write(&mut foo.assume_init_mut().b as *mut u8, 42); + /// // ^^^^^^^^^^^^^^^^^^^^^ /// // (mutable) reference to uninitialized memory! /// // This is undefined behavior. /// foo.assume_init() /// }; /// ``` - // FIXME(#53491): We currently rely on the above being incorrect, i.e., we have references - // to uninitialized data (e.g., in `libcore/fmt/float.rs`). We should make - // a final decision about the rules before stabilization. - #[unstable(feature = "maybe_uninit_ref", issue = "63568")] + #[stable(feature = "maybe_uninit_ref", since = "1.55.0")] + #[rustc_const_unstable(feature = "const_maybe_uninit_assume_init", issue = "none")] #[inline(always)] - pub unsafe fn get_mut(&mut self) -> &mut T { - #[cfg(bootstrap)] - intrinsics::panic_if_uninhabited::(); - #[cfg(not(bootstrap))] - intrinsics::assert_inhabited::(); - &mut *self.value + pub const unsafe fn assume_init_mut(&mut self) -> &mut T { + // SAFETY: the caller must guarantee that `self` is initialized. + // This also means that `self` must be a `value` variant. + unsafe { + intrinsics::assert_inhabited::(); + &mut *self.as_mut_ptr() + } + } + + /// Extracts the values from an array of `MaybeUninit` containers. + /// + /// # Safety + /// + /// It is up to the caller to guarantee that all elements of the array are + /// in an initialized state. + /// + /// # Examples + /// + /// ``` + /// #![feature(maybe_uninit_uninit_array)] + /// #![feature(maybe_uninit_array_assume_init)] + /// use std::mem::MaybeUninit; + /// + /// let mut array: [MaybeUninit; 3] = MaybeUninit::uninit_array(); + /// array[0].write(0); + /// array[1].write(1); + /// array[2].write(2); + /// + /// // SAFETY: Now safe as we initialised all elements + /// let array = unsafe { + /// MaybeUninit::array_assume_init(array) + /// }; + /// + /// assert_eq!(array, [0, 1, 2]); + /// ``` + #[unstable(feature = "maybe_uninit_array_assume_init", issue = "96097")] + #[rustc_const_unstable(feature = "const_maybe_uninit_array_assume_init", issue = "96097")] + #[inline(always)] + #[track_caller] + pub const unsafe fn array_assume_init(array: [Self; N]) -> [T; N] { + // SAFETY: + // * The caller guarantees that all elements of the array are initialized + // * `MaybeUninit` and T are guaranteed to have the same layout + // * `MaybeUninit` does not drop, so there are no double-frees + // And thus the conversion is safe + let ret = unsafe { + intrinsics::assert_inhabited::<[T; N]>(); + (&array as *const _ as *const [T; N]).read() + }; + + // FIXME: required to avoid `~const Destruct` bound + super::forget(array); + ret } /// Assuming all the elements are initialized, get a slice to them. @@ -762,10 +962,19 @@ impl MaybeUninit { /// It is up to the caller to guarantee that the `MaybeUninit` elements /// really are in an initialized state. /// Calling this when the content is not yet fully initialized causes undefined behavior. - #[unstable(feature = "maybe_uninit_slice_assume_init", issue = "none")] + /// + /// See [`assume_init_ref`] for more details and examples. + /// + /// [`assume_init_ref`]: MaybeUninit::assume_init_ref + #[unstable(feature = "maybe_uninit_slice", issue = "63569")] + #[rustc_const_unstable(feature = "maybe_uninit_slice", issue = "63569")] #[inline(always)] - pub unsafe fn slice_get_ref(slice: &[Self]) -> &[T] { - &*(slice as *const [Self] as *const [T]) + pub const unsafe fn slice_assume_init_ref(slice: &[Self]) -> &[T] { + // SAFETY: casting `slice` to a `*const [T]` is safe since the caller guarantees that + // `slice` is initialized, and `MaybeUninit` is guaranteed to have the same layout as `T`. + // The pointer obtained is valid since it refers to memory owned by `slice` which is a + // reference and thus guaranteed to be valid for reads. + unsafe { &*(slice as *const [Self] as *const [T]) } } /// Assuming all the elements are initialized, get a mutable slice to them. @@ -775,23 +984,341 @@ impl MaybeUninit { /// It is up to the caller to guarantee that the `MaybeUninit` elements /// really are in an initialized state. /// Calling this when the content is not yet fully initialized causes undefined behavior. - #[unstable(feature = "maybe_uninit_slice_assume_init", issue = "none")] + /// + /// See [`assume_init_mut`] for more details and examples. + /// + /// [`assume_init_mut`]: MaybeUninit::assume_init_mut + #[unstable(feature = "maybe_uninit_slice", issue = "63569")] + #[rustc_const_unstable(feature = "const_maybe_uninit_assume_init", issue = "none")] #[inline(always)] - pub unsafe fn slice_get_mut(slice: &mut [Self]) -> &mut [T] { - &mut *(slice as *mut [Self] as *mut [T]) + pub const unsafe fn slice_assume_init_mut(slice: &mut [Self]) -> &mut [T] { + // SAFETY: similar to safety notes for `slice_get_ref`, but we have a + // mutable reference which is also guaranteed to be valid for writes. + unsafe { &mut *(slice as *mut [Self] as *mut [T]) } } /// Gets a pointer to the first element of the array. #[unstable(feature = "maybe_uninit_slice", issue = "63569")] + #[rustc_const_unstable(feature = "maybe_uninit_slice", issue = "63569")] #[inline(always)] - pub fn first_ptr(this: &[MaybeUninit]) -> *const T { - this as *const [MaybeUninit] as *const T + pub const fn slice_as_ptr(this: &[MaybeUninit]) -> *const T { + this.as_ptr() as *const T } /// Gets a mutable pointer to the first element of the array. #[unstable(feature = "maybe_uninit_slice", issue = "63569")] + #[rustc_const_unstable(feature = "maybe_uninit_slice", issue = "63569")] #[inline(always)] - pub fn first_ptr_mut(this: &mut [MaybeUninit]) -> *mut T { - this as *mut [MaybeUninit] as *mut T + pub const fn slice_as_mut_ptr(this: &mut [MaybeUninit]) -> *mut T { + this.as_mut_ptr() as *mut T + } + + /// Copies the elements from `src` to `this`, returning a mutable reference to the now initialized contents of `this`. + /// + /// If `T` does not implement `Copy`, use [`write_slice_cloned`] + /// + /// This is similar to [`slice::copy_from_slice`]. + /// + /// # Panics + /// + /// This function will panic if the two slices have different lengths. + /// + /// # Examples + /// + /// ``` + /// #![feature(maybe_uninit_write_slice)] + /// use std::mem::MaybeUninit; + /// + /// let mut dst = [MaybeUninit::uninit(); 32]; + /// let src = [0; 32]; + /// + /// let init = MaybeUninit::write_slice(&mut dst, &src); + /// + /// assert_eq!(init, src); + /// ``` + /// + /// ``` + /// #![feature(maybe_uninit_write_slice)] + /// use std::mem::MaybeUninit; + /// + /// let mut vec = Vec::with_capacity(32); + /// let src = [0; 16]; + /// + /// MaybeUninit::write_slice(&mut vec.spare_capacity_mut()[..src.len()], &src); + /// + /// // SAFETY: we have just copied all the elements of len into the spare capacity + /// // the first src.len() elements of the vec are valid now. + /// unsafe { + /// vec.set_len(src.len()); + /// } + /// + /// assert_eq!(vec, src); + /// ``` + /// + /// [`write_slice_cloned`]: MaybeUninit::write_slice_cloned + #[unstable(feature = "maybe_uninit_write_slice", issue = "79995")] + pub fn write_slice<'a>(this: &'a mut [MaybeUninit], src: &[T]) -> &'a mut [T] + where + T: Copy, + { + // SAFETY: &[T] and &[MaybeUninit] have the same layout + let uninit_src: &[MaybeUninit] = unsafe { super::transmute(src) }; + + this.copy_from_slice(uninit_src); + + // SAFETY: Valid elements have just been copied into `this` so it is initialized + unsafe { MaybeUninit::slice_assume_init_mut(this) } + } + + /// Clones the elements from `src` to `this`, returning a mutable reference to the now initialized contents of `this`. + /// Any already initialized elements will not be dropped. + /// + /// If `T` implements `Copy`, use [`write_slice`] + /// + /// This is similar to [`slice::clone_from_slice`] but does not drop existing elements. + /// + /// # Panics + /// + /// This function will panic if the two slices have different lengths, or if the implementation of `Clone` panics. + /// + /// If there is a panic, the already cloned elements will be dropped. + /// + /// # Examples + /// + /// ``` + /// #![feature(maybe_uninit_write_slice)] + /// use std::mem::MaybeUninit; + /// + /// let mut dst = [MaybeUninit::uninit(), MaybeUninit::uninit(), MaybeUninit::uninit(), MaybeUninit::uninit(), MaybeUninit::uninit()]; + /// let src = ["wibbly".to_string(), "wobbly".to_string(), "timey".to_string(), "wimey".to_string(), "stuff".to_string()]; + /// + /// let init = MaybeUninit::write_slice_cloned(&mut dst, &src); + /// + /// assert_eq!(init, src); + /// ``` + /// + /// ``` + /// #![feature(maybe_uninit_write_slice)] + /// use std::mem::MaybeUninit; + /// + /// let mut vec = Vec::with_capacity(32); + /// let src = ["rust", "is", "a", "pretty", "cool", "language"]; + /// + /// MaybeUninit::write_slice_cloned(&mut vec.spare_capacity_mut()[..src.len()], &src); + /// + /// // SAFETY: we have just cloned all the elements of len into the spare capacity + /// // the first src.len() elements of the vec are valid now. + /// unsafe { + /// vec.set_len(src.len()); + /// } + /// + /// assert_eq!(vec, src); + /// ``` + /// + /// [`write_slice`]: MaybeUninit::write_slice + #[unstable(feature = "maybe_uninit_write_slice", issue = "79995")] + pub fn write_slice_cloned<'a>(this: &'a mut [MaybeUninit], src: &[T]) -> &'a mut [T] + where + T: Clone, + { + // unlike copy_from_slice this does not call clone_from_slice on the slice + // this is because `MaybeUninit` does not implement Clone. + + struct Guard<'a, T> { + slice: &'a mut [MaybeUninit], + initialized: usize, + } + + impl<'a, T> Drop for Guard<'a, T> { + fn drop(&mut self) { + let initialized_part = &mut self.slice[..self.initialized]; + // SAFETY: this raw slice will contain only initialized objects + // that's why, it is allowed to drop it. + unsafe { + crate::ptr::drop_in_place(MaybeUninit::slice_assume_init_mut(initialized_part)); + } + } + } + + assert_eq!(this.len(), src.len(), "destination and source slices have different lengths"); + // NOTE: We need to explicitly slice them to the same length + // for bounds checking to be elided, and the optimizer will + // generate memcpy for simple cases (for example T = u8). + let len = this.len(); + let src = &src[..len]; + + // guard is needed b/c panic might happen during a clone + let mut guard = Guard { slice: this, initialized: 0 }; + + for i in 0..len { + guard.slice[i].write(src[i].clone()); + guard.initialized += 1; + } + + super::forget(guard); + + // SAFETY: Valid elements have just been written into `this` so it is initialized + unsafe { MaybeUninit::slice_assume_init_mut(this) } + } + + /// Returns the contents of this `MaybeUninit` as a slice of potentially uninitialized bytes. + /// + /// Note that even if the contents of a `MaybeUninit` have been initialized, the value may still + /// contain padding bytes which are left uninitialized. + /// + /// # Examples + /// + /// ``` + /// #![feature(maybe_uninit_as_bytes, maybe_uninit_slice)] + /// use std::mem::MaybeUninit; + /// + /// let val = 0x12345678_i32; + /// let uninit = MaybeUninit::new(val); + /// let uninit_bytes = uninit.as_bytes(); + /// let bytes = unsafe { MaybeUninit::slice_assume_init_ref(uninit_bytes) }; + /// assert_eq!(bytes, val.to_ne_bytes()); + /// ``` + #[unstable(feature = "maybe_uninit_as_bytes", issue = "93092")] + pub fn as_bytes(&self) -> &[MaybeUninit] { + // SAFETY: MaybeUninit is always valid, even for padding bytes + unsafe { + slice::from_raw_parts(self.as_ptr() as *const MaybeUninit, mem::size_of::()) + } + } + + /// Returns the contents of this `MaybeUninit` as a mutable slice of potentially uninitialized + /// bytes. + /// + /// Note that even if the contents of a `MaybeUninit` have been initialized, the value may still + /// contain padding bytes which are left uninitialized. + /// + /// # Examples + /// + /// ``` + /// #![feature(maybe_uninit_as_bytes)] + /// use std::mem::MaybeUninit; + /// + /// let val = 0x12345678_i32; + /// let mut uninit = MaybeUninit::new(val); + /// let uninit_bytes = uninit.as_bytes_mut(); + /// if cfg!(target_endian = "little") { + /// uninit_bytes[0].write(0xcd); + /// } else { + /// uninit_bytes[3].write(0xcd); + /// } + /// let val2 = unsafe { uninit.assume_init() }; + /// assert_eq!(val2, 0x123456cd); + /// ``` + #[unstable(feature = "maybe_uninit_as_bytes", issue = "93092")] + pub fn as_bytes_mut(&mut self) -> &mut [MaybeUninit] { + // SAFETY: MaybeUninit is always valid, even for padding bytes + unsafe { + slice::from_raw_parts_mut( + self.as_mut_ptr() as *mut MaybeUninit, + mem::size_of::(), + ) + } + } + + /// Returns the contents of this slice of `MaybeUninit` as a slice of potentially uninitialized + /// bytes. + /// + /// Note that even if the contents of a `MaybeUninit` have been initialized, the value may still + /// contain padding bytes which are left uninitialized. + /// + /// # Examples + /// + /// ``` + /// #![feature(maybe_uninit_as_bytes, maybe_uninit_write_slice, maybe_uninit_slice)] + /// use std::mem::MaybeUninit; + /// + /// let uninit = [MaybeUninit::new(0x1234u16), MaybeUninit::new(0x5678u16)]; + /// let uninit_bytes = MaybeUninit::slice_as_bytes(&uninit); + /// let bytes = unsafe { MaybeUninit::slice_assume_init_ref(&uninit_bytes) }; + /// let val1 = u16::from_ne_bytes(bytes[0..2].try_into().unwrap()); + /// let val2 = u16::from_ne_bytes(bytes[2..4].try_into().unwrap()); + /// assert_eq!(&[val1, val2], &[0x1234u16, 0x5678u16]); + /// ``` + #[unstable(feature = "maybe_uninit_as_bytes", issue = "93092")] + pub fn slice_as_bytes(this: &[MaybeUninit]) -> &[MaybeUninit] { + // SAFETY: MaybeUninit is always valid, even for padding bytes + unsafe { + slice::from_raw_parts( + this.as_ptr() as *const MaybeUninit, + this.len() * mem::size_of::(), + ) + } + } + + /// Returns the contents of this mutable slice of `MaybeUninit` as a mutable slice of + /// potentially uninitialized bytes. + /// + /// Note that even if the contents of a `MaybeUninit` have been initialized, the value may still + /// contain padding bytes which are left uninitialized. + /// + /// # Examples + /// + /// ``` + /// #![feature(maybe_uninit_as_bytes, maybe_uninit_write_slice, maybe_uninit_slice)] + /// use std::mem::MaybeUninit; + /// + /// let mut uninit = [MaybeUninit::::uninit(), MaybeUninit::::uninit()]; + /// let uninit_bytes = MaybeUninit::slice_as_bytes_mut(&mut uninit); + /// MaybeUninit::write_slice(uninit_bytes, &[0x12, 0x34, 0x56, 0x78]); + /// let vals = unsafe { MaybeUninit::slice_assume_init_ref(&uninit) }; + /// if cfg!(target_endian = "little") { + /// assert_eq!(vals, &[0x3412u16, 0x7856u16]); + /// } else { + /// assert_eq!(vals, &[0x1234u16, 0x5678u16]); + /// } + /// ``` + #[unstable(feature = "maybe_uninit_as_bytes", issue = "93092")] + pub fn slice_as_bytes_mut(this: &mut [MaybeUninit]) -> &mut [MaybeUninit] { + // SAFETY: MaybeUninit is always valid, even for padding bytes + unsafe { + slice::from_raw_parts_mut( + this.as_mut_ptr() as *mut MaybeUninit, + this.len() * mem::size_of::(), + ) + } + } +} + +impl MaybeUninit<[T; N]> { + /// Transposes a `MaybeUninit<[T; N]>` into a `[MaybeUninit; N]`. + /// + /// # Examples + /// + /// ``` + /// #![feature(maybe_uninit_uninit_array_transpose)] + /// # use std::mem::MaybeUninit; + /// + /// let data: [MaybeUninit; 1000] = MaybeUninit::uninit().transpose(); + /// ``` + #[unstable(feature = "maybe_uninit_uninit_array_transpose", issue = "96097")] + #[inline] + pub const fn transpose(self) -> [MaybeUninit; N] { + // SAFETY: T and MaybeUninit have the same layout + unsafe { super::transmute_copy(&ManuallyDrop::new(self)) } + } +} + +impl [MaybeUninit; N] { + /// Transposes a `[MaybeUninit; N]` into a `MaybeUninit<[T; N]>`. + /// + /// # Examples + /// + /// ``` + /// #![feature(maybe_uninit_uninit_array_transpose)] + /// # use std::mem::MaybeUninit; + /// + /// let data = [MaybeUninit::::uninit(); 1000]; + /// let data: MaybeUninit<[u8; 1000]> = data.transpose(); + /// ``` + #[unstable(feature = "maybe_uninit_uninit_array_transpose", issue = "96097")] + #[inline] + pub const fn transpose(self) -> MaybeUninit<[T; N]> { + // SAFETY: T and MaybeUninit have the same layout + unsafe { super::transmute_copy(&ManuallyDrop::new(self)) } } } diff --git a/crux-mir/lib/core/src/mem/mod.rs b/crux-mir/lib/core/src/mem/mod.rs index 3e5d71b6f..5e01ccc07 100644 --- a/crux-mir/lib/core/src/mem/mod.rs +++ b/crux-mir/lib/core/src/mem/mod.rs @@ -10,7 +10,7 @@ use crate::cmp; use crate::fmt; use crate::hash; use crate::intrinsics; -use crate::marker::{Copy, PhantomData, Sized}; +use crate::marker::{Copy, DiscriminantKind, Sized}; use crate::ptr; mod manually_drop; @@ -21,6 +21,10 @@ mod maybe_uninit; #[stable(feature = "maybe_uninit", since = "1.36.0")] pub use maybe_uninit::MaybeUninit; +mod transmutability; +#[unstable(feature = "transmutability", issue = "99571")] +pub use transmutability::{Assume, BikeshedIntrinsicFrom}; + #[stable(feature = "rust1", since = "1.0.0")] #[doc(inline)] pub use crate::intrinsics::transmute; @@ -31,10 +35,10 @@ pub use crate::intrinsics::transmute; /// forever in an unreachable state. However, it does not guarantee that pointers /// to this memory will remain valid. /// -/// * If you want to leak memory, see [`Box::leak`][leak]. -/// * If you want to obtain a raw pointer to the memory, see [`Box::into_raw`][into_raw]. +/// * If you want to leak memory, see [`Box::leak`]. +/// * If you want to obtain a raw pointer to the memory, see [`Box::into_raw`]. /// * If you want to dispose of a value properly, running its destructor, see -/// [`mem::drop`][drop]. +/// [`mem::drop`]. /// /// # Safety /// @@ -129,37 +133,30 @@ pub use crate::intrinsics::transmute; /// erring on the side of (double-)dropping. /// /// Also, `ManuallyDrop` prevents us from having to "touch" `v` after transferring the -/// ownership to `s` - the final step of interacting with `v` to dispoe of it without +/// ownership to `s` — the final step of interacting with `v` to dispose of it without /// running its destructor is entirely avoided. /// -/// [drop]: fn.drop.html -/// [uninit]: fn.uninitialized.html -/// [clone]: ../clone/trait.Clone.html -/// [swap]: fn.swap.html -/// [box]: ../../std/boxed/struct.Box.html -/// [leak]: ../../std/boxed/struct.Box.html#method.leak -/// [into_raw]: ../../std/boxed/struct.Box.html#method.into_raw +/// [`Box`]: ../../std/boxed/struct.Box.html +/// [`Box::leak`]: ../../std/boxed/struct.Box.html#method.leak +/// [`Box::into_raw`]: ../../std/boxed/struct.Box.html#method.into_raw +/// [`mem::drop`]: drop /// [ub]: ../../reference/behavior-considered-undefined.html -/// [`ManuallyDrop`]: struct.ManuallyDrop.html #[inline] -#[rustc_const_unstable(feature = "const_forget", issue = "69616")] +#[rustc_const_stable(feature = "const_forget", since = "1.46.0")] #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "mem_forget")] pub const fn forget(t: T) { - ManuallyDrop::new(t); + let _ = ManuallyDrop::new(t); } /// Like [`forget`], but also accepts unsized values. /// /// This function is just a shim intended to be removed when the `unsized_locals` feature gets /// stabilized. -/// -/// [`forget`]: fn.forget.html #[inline] #[unstable(feature = "forget_unsized", issue = "none")] pub fn forget_unsized(t: T) { - // SAFETY: the forget intrinsic could be safe, but there's no point in making it safe since - // we'll be implementing this function soon via `ManuallyDrop` - unsafe { intrinsics::forget(t) } + intrinsics::forget(t) } /// Returns the size of a type in bytes. @@ -301,11 +298,13 @@ pub fn forget_unsized(t: T) { /// assert_eq!(2, mem::size_of::()); /// ``` /// -/// [alignment]: ./fn.align_of.html +/// [alignment]: align_of #[inline(always)] +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] #[rustc_promotable] -#[rustc_const_stable(feature = "const_size_of", since = "1.32.0")] +#[rustc_const_stable(feature = "const_mem_size_of", since = "1.24.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "mem_size_of")] pub const fn size_of() -> usize { intrinsics::size_of::() } @@ -316,7 +315,6 @@ pub const fn size_of() -> usize { /// statically-known size, e.g., a slice [`[T]`][slice] or a [trait object], /// then `size_of_val` can be used to get the dynamically-known size. /// -/// [slice]: ../../std/primitive.slice.html /// [trait object]: ../../book/ch17-02-trait-objects.html /// /// # Examples @@ -331,12 +329,65 @@ pub const fn size_of() -> usize { /// assert_eq!(13, mem::size_of_val(y)); /// ``` #[inline] +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] -pub fn size_of_val(val: &T) -> usize { - intrinsics::size_of_val(val) +#[rustc_const_unstable(feature = "const_size_of_val", issue = "46571")] +#[cfg_attr(not(test), rustc_diagnostic_item = "mem_size_of_val")] +pub const fn size_of_val(val: &T) -> usize { + // SAFETY: `val` is a reference, so it's a valid raw pointer + unsafe { intrinsics::size_of_val(val) } } -/// Returns the [ABI]-required minimum alignment of a type. +/// Returns the size of the pointed-to value in bytes. +/// +/// This is usually the same as `size_of::()`. However, when `T` *has* no +/// statically-known size, e.g., a slice [`[T]`][slice] or a [trait object], +/// then `size_of_val_raw` can be used to get the dynamically-known size. +/// +/// # Safety +/// +/// This function is only safe to call if the following conditions hold: +/// +/// - If `T` is `Sized`, this function is always safe to call. +/// - If the unsized tail of `T` is: +/// - a [slice], then the length of the slice tail must be an initialized +/// integer, and the size of the *entire value* +/// (dynamic tail length + statically sized prefix) must fit in `isize`. +/// - a [trait object], then the vtable part of the pointer must point +/// to a valid vtable acquired by an unsizing coercion, and the size +/// of the *entire value* (dynamic tail length + statically sized prefix) +/// must fit in `isize`. +/// - an (unstable) [extern type], then this function is always safe to +/// call, but may panic or otherwise return the wrong value, as the +/// extern type's layout is not known. This is the same behavior as +/// [`size_of_val`] on a reference to a type with an extern type tail. +/// - otherwise, it is conservatively not allowed to call this function. +/// +/// [trait object]: ../../book/ch17-02-trait-objects.html +/// [extern type]: ../../unstable-book/language-features/extern-types.html +/// +/// # Examples +/// +/// ``` +/// #![feature(layout_for_ptr)] +/// use std::mem; +/// +/// assert_eq!(4, mem::size_of_val(&5i32)); +/// +/// let x: [u8; 13] = [0; 13]; +/// let y: &[u8] = &x; +/// assert_eq!(13, unsafe { mem::size_of_val_raw(y) }); +/// ``` +#[inline] +#[must_use] +#[unstable(feature = "layout_for_ptr", issue = "69835")] +#[rustc_const_unstable(feature = "const_size_of_val_raw", issue = "46571")] +pub const unsafe fn size_of_val_raw(val: *const T) -> usize { + // SAFETY: the caller must provide a valid raw pointer + unsafe { intrinsics::size_of_val(val) } +} + +/// Returns the [ABI]-required minimum alignment of a type in bytes. /// /// Every reference to a value of the type `T` must be a multiple of this number. /// @@ -353,13 +404,15 @@ pub fn size_of_val(val: &T) -> usize { /// assert_eq!(4, mem::min_align_of::()); /// ``` #[inline] +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] -#[rustc_deprecated(reason = "use `align_of` instead", since = "1.2.0")] +#[deprecated(note = "use `align_of` instead", since = "1.2.0")] pub fn min_align_of() -> usize { intrinsics::min_align_of::() } -/// Returns the [ABI]-required minimum alignment of the type of the value that `val` points to. +/// Returns the [ABI]-required minimum alignment of the type of the value that `val` points to in +/// bytes. /// /// Every reference to a value of the type `T` must be a multiple of this number. /// @@ -374,13 +427,15 @@ pub fn min_align_of() -> usize { /// assert_eq!(4, mem::min_align_of_val(&5i32)); /// ``` #[inline] +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] -#[rustc_deprecated(reason = "use `align_of_val` instead", since = "1.2.0")] +#[deprecated(note = "use `align_of_val` instead", since = "1.2.0")] pub fn min_align_of_val(val: &T) -> usize { - intrinsics::min_align_of_val(val) + // SAFETY: val is a reference, so it's a valid raw pointer + unsafe { intrinsics::min_align_of_val(val) } } -/// Returns the [ABI]-required minimum alignment of a type. +/// Returns the [ABI]-required minimum alignment of a type in bytes. /// /// Every reference to a value of the type `T` must be a multiple of this number. /// @@ -396,14 +451,16 @@ pub fn min_align_of_val(val: &T) -> usize { /// assert_eq!(4, mem::align_of::()); /// ``` #[inline(always)] +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] #[rustc_promotable] -#[rustc_const_stable(feature = "const_align_of", since = "1.32.0")] +#[rustc_const_stable(feature = "const_align_of", since = "1.24.0")] pub const fn align_of() -> usize { intrinsics::min_align_of::() } -/// Returns the [ABI]-required minimum alignment of the type of the value that `val` points to. +/// Returns the [ABI]-required minimum alignment of the type of the value that `val` points to in +/// bytes. /// /// Every reference to a value of the type `T` must be a multiple of this number. /// @@ -417,10 +474,59 @@ pub const fn align_of() -> usize { /// assert_eq!(4, mem::align_of_val(&5i32)); /// ``` #[inline] +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] +#[rustc_const_unstable(feature = "const_align_of_val", issue = "46571")] #[allow(deprecated)] -pub fn align_of_val(val: &T) -> usize { - min_align_of_val(val) +pub const fn align_of_val(val: &T) -> usize { + // SAFETY: val is a reference, so it's a valid raw pointer + unsafe { intrinsics::min_align_of_val(val) } +} + +/// Returns the [ABI]-required minimum alignment of the type of the value that `val` points to in +/// bytes. +/// +/// Every reference to a value of the type `T` must be a multiple of this number. +/// +/// [ABI]: https://en.wikipedia.org/wiki/Application_binary_interface +/// +/// # Safety +/// +/// This function is only safe to call if the following conditions hold: +/// +/// - If `T` is `Sized`, this function is always safe to call. +/// - If the unsized tail of `T` is: +/// - a [slice], then the length of the slice tail must be an initialized +/// integer, and the size of the *entire value* +/// (dynamic tail length + statically sized prefix) must fit in `isize`. +/// - a [trait object], then the vtable part of the pointer must point +/// to a valid vtable acquired by an unsizing coercion, and the size +/// of the *entire value* (dynamic tail length + statically sized prefix) +/// must fit in `isize`. +/// - an (unstable) [extern type], then this function is always safe to +/// call, but may panic or otherwise return the wrong value, as the +/// extern type's layout is not known. This is the same behavior as +/// [`align_of_val`] on a reference to a type with an extern type tail. +/// - otherwise, it is conservatively not allowed to call this function. +/// +/// [trait object]: ../../book/ch17-02-trait-objects.html +/// [extern type]: ../../unstable-book/language-features/extern-types.html +/// +/// # Examples +/// +/// ``` +/// #![feature(layout_for_ptr)] +/// use std::mem; +/// +/// assert_eq!(4, unsafe { mem::align_of_val_raw(&5i32) }); +/// ``` +#[inline] +#[must_use] +#[unstable(feature = "layout_for_ptr", issue = "69835")] +#[rustc_const_unstable(feature = "const_align_of_val_raw", issue = "46571")] +pub const unsafe fn align_of_val_raw(val: *const T) -> usize { + // SAFETY: the caller must provide a valid raw pointer + unsafe { intrinsics::min_align_of_val(val) } } /// Returns `true` if dropping values of type `T` matters. @@ -446,7 +552,7 @@ pub fn align_of_val(val: &T) -> usize { /// `needs_drop` explicitly. Types like [`HashMap`], on the other hand, have to drop /// values one at a time and should use this API. /// -/// [`drop_in_place`]: ../ptr/fn.drop_in_place.html +/// [`drop_in_place`]: crate::ptr::drop_in_place /// [`HashMap`]: ../../std/collections/struct.HashMap.html /// /// # Examples @@ -480,9 +586,11 @@ pub fn align_of_val(val: &T) -> usize { /// } /// ``` #[inline] +#[must_use] #[stable(feature = "needs_drop", since = "1.21.0")] -#[rustc_const_stable(feature = "const_needs_drop", since = "1.36.0")] -pub const fn needs_drop() -> bool { +#[rustc_const_stable(feature = "const_mem_needs_drop", since = "1.36.0")] +#[rustc_diagnostic_item = "needs_drop"] +pub const fn needs_drop() -> bool { intrinsics::needs_drop::() } @@ -491,18 +599,19 @@ pub const fn needs_drop() -> bool { /// This means that, for example, the padding byte in `(u8, u16)` is not /// necessarily zeroed. /// -/// There is no guarantee that an all-zero byte-pattern represents a valid value of -/// some type `T`. For example, the all-zero byte-pattern is not a valid value -/// for reference types (`&T` and `&mut T`). Using `zeroed` on such types -/// causes immediate [undefined behavior][ub] because [the Rust compiler assumes][inv] -/// that there always is a valid value in a variable it considers initialized. +/// There is no guarantee that an all-zero byte-pattern represents a valid value +/// of some type `T`. For example, the all-zero byte-pattern is not a valid value +/// for reference types (`&T`, `&mut T`) and functions pointers. Using `zeroed` +/// on such types causes immediate [undefined behavior][ub] because [the Rust +/// compiler assumes][inv] that there always is a valid value in a variable it +/// considers initialized. /// /// This has the same effect as [`MaybeUninit::zeroed().assume_init()`][zeroed]. /// It is useful for FFI sometimes, but should generally be avoided. /// -/// [zeroed]: union.MaybeUninit.html#method.zeroed +/// [zeroed]: MaybeUninit::zeroed /// [ub]: ../../reference/behavior-considered-undefined.html -/// [inv]: union.MaybeUninit.html#initialization-invariant +/// [inv]: MaybeUninit#initialization-invariant /// /// # Examples /// @@ -522,58 +631,76 @@ pub const fn needs_drop() -> bool { /// use std::mem; /// /// let _x: &i32 = unsafe { mem::zeroed() }; // Undefined behavior! +/// let _y: fn() = unsafe { mem::zeroed() }; // And again! /// ``` #[inline(always)] +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] #[allow(deprecated_in_future)] #[allow(deprecated)] #[rustc_diagnostic_item = "mem_zeroed"] +#[track_caller] pub unsafe fn zeroed() -> T { - #[cfg(not(bootstrap))] - intrinsics::assert_zero_valid::(); - #[cfg(bootstrap)] - intrinsics::panic_if_uninhabited::(); - MaybeUninit::zeroed().assume_init() + // SAFETY: the caller must guarantee that an all-zero value is valid for `T`. + unsafe { + intrinsics::assert_zero_valid::(); + MaybeUninit::zeroed().assume_init() + } } /// Bypasses Rust's normal memory-initialization checks by pretending to /// produce a value of type `T`, while doing nothing at all. /// /// **This function is deprecated.** Use [`MaybeUninit`] instead. +/// It also might be slower than using `MaybeUninit` due to mitigations that were put in place to +/// limit the potential harm caused by incorrect use of this function in legacy code. /// /// The reason for deprecation is that the function basically cannot be used /// correctly: it has the same effect as [`MaybeUninit::uninit().assume_init()`][uninit]. /// As the [`assume_init` documentation][assume_init] explains, /// [the Rust compiler assumes][inv] that values are properly initialized. -/// As a consequence, calling e.g. `mem::uninitialized::()` causes immediate -/// undefined behavior for returning a `bool` that is not definitely either `true` -/// or `false`. Worse, truly uninitialized memory like what gets returned here +/// +/// Truly uninitialized memory like what gets returned here /// is special in that the compiler knows that it does not have a fixed value. /// This makes it undefined behavior to have uninitialized data in a variable even /// if that variable has an integer type. -/// (Notice that the rules around uninitialized integers are not finalized yet, but -/// until they are, it is advisable to avoid them.) /// -/// [`MaybeUninit`]: union.MaybeUninit.html -/// [uninit]: union.MaybeUninit.html#method.uninit -/// [assume_init]: union.MaybeUninit.html#method.assume_init -/// [inv]: union.MaybeUninit.html#initialization-invariant +/// Therefore, it is immediate undefined behavior to call this function on nearly all types, +/// including integer types and arrays of integer types, and even if the result is unused. +/// +/// [uninit]: MaybeUninit::uninit +/// [assume_init]: MaybeUninit::assume_init +/// [inv]: MaybeUninit#initialization-invariant #[inline(always)] -#[rustc_deprecated(since = "1.39.0", reason = "use `mem::MaybeUninit` instead")] +#[must_use] +#[deprecated(since = "1.39.0", note = "use `mem::MaybeUninit` instead")] #[stable(feature = "rust1", since = "1.0.0")] #[allow(deprecated_in_future)] #[allow(deprecated)] #[rustc_diagnostic_item = "mem_uninitialized"] +#[track_caller] pub unsafe fn uninitialized() -> T { - #[cfg(not(bootstrap))] - intrinsics::assert_uninit_valid::(); - #[cfg(bootstrap)] - intrinsics::panic_if_uninhabited::(); - MaybeUninit::uninit().assume_init() + // SAFETY: the caller must guarantee that an uninitialized value is valid for `T`. + unsafe { + #[cfg(not(bootstrap))] // If the compiler hits this itself then it deserves the UB. + intrinsics::assert_mem_uninitialized_valid::(); + let mut val = MaybeUninit::::uninit(); + + // Fill memory with 0x01, as an imperfect mitigation for old code that uses this function on + // bool, nonnull, and noundef types. But don't do this if we actively want to detect UB. + if !cfg!(any(miri, sanitize = "memory")) { + val.as_mut_ptr().write_bytes(0x01, 1); + } + + val.assume_init() + } } /// Swaps the values at two mutable locations, without deinitializing either one. /// +/// * If you want to swap with a default or dummy value, see [`take`]. +/// * If you want to swap with a passed value, returning the old value, see [`replace`]. +/// /// # Examples /// /// ``` @@ -589,16 +716,73 @@ pub unsafe fn uninitialized() -> T { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] -pub fn swap(x: &mut T, y: &mut T) { - // SAFETY: the raw pointers have been created from safe mutable references satisfying all the - // constraints on `ptr::swap_nonoverlapping_one` +#[rustc_const_unstable(feature = "const_swap", issue = "83163")] +pub const fn swap(x: &mut T, y: &mut T) { + // NOTE(eddyb) SPIR-V's Logical addressing model doesn't allow for arbitrary + // reinterpretation of values as (chunkable) byte arrays, and the loop in the + // block optimization in `swap_slice` is hard to rewrite back + // into the (unoptimized) direct swapping implementation, so we disable it. + // FIXME(eddyb) the block optimization also prevents MIR optimizations from + // understanding `mem::replace`, `Option::take`, etc. - a better overall + // solution might be to make `ptr::swap_nonoverlapping` into an intrinsic, which + // a backend can choose to implement using the block optimization, or not. + #[cfg(not(any(target_arch = "spirv")))] + { + // For types that are larger multiples of their alignment, the simple way + // tends to copy the whole thing to stack rather than doing it one part + // at a time, so instead treat them as one-element slices and piggy-back + // the slice optimizations that will split up the swaps. + if size_of::() / align_of::() > 4 { + // SAFETY: exclusive references always point to one non-overlapping + // element and are non-null and properly aligned. + return unsafe { ptr::swap_nonoverlapping(x, y, 1) }; + } + } + + // If a scalar consists of just a small number of alignment units, let + // the codegen just swap those pieces directly, as it's likely just a + // few instructions and anything else is probably overcomplicated. + // + // Most importantly, this covers primitives and simd types that tend to + // have size=align where doing anything else can be a pessimization. + // (This will also be used for ZSTs, though any solution works for them.) + swap_simple(x, y); +} + +/// Same as [`swap`] semantically, but always uses the simple implementation. +/// +/// Used elsewhere in `mem` and `ptr` at the bottom layer of calls. +#[rustc_const_unstable(feature = "const_swap", issue = "83163")] +#[inline] +pub(crate) const fn swap_simple(x: &mut T, y: &mut T) { + // We arrange for this to typically be called with small types, + // so this reads-and-writes approach is actually better than using + // copy_nonoverlapping as it easily puts things in LLVM registers + // directly and doesn't end up inlining allocas. + // And LLVM actually optimizes it to 3×memcpy if called with + // a type larger than it's willing to keep in a register. + // Having typed reads and writes in MIR here is also good as + // it lets MIRI and CTFE understand them better, including things + // like enforcing type validity for them. + // Importantly, read+copy_nonoverlapping+write introduces confusing + // asymmetry to the behaviour where one value went through read+write + // whereas the other was copied over by the intrinsic (see #94371). + + // SAFETY: exclusive references are always valid to read/write, + // including being aligned, and nothing here panics so it's drop-safe. unsafe { - ptr::swap_nonoverlapping_one(x, y); + let a = ptr::read(x); + let b = ptr::read(y); + ptr::write(x, b); + ptr::write(y, a); } } /// Replaces `dest` with the default value of `T`, returning the previous `dest` value. /// +/// * If you want to replace the values of two variables, see [`swap`]. +/// * If you want to replace with a passed value instead of the default value, see [`replace`]. +/// /// # Examples /// /// A simple example: @@ -649,8 +833,6 @@ pub fn swap(x: &mut T, y: &mut T) { /// assert_eq!(buffer.get_and_reset(), vec![0, 1]); /// assert_eq!(buffer.buf.len(), 0); /// ``` -/// -/// [`Clone`]: ../../std/clone/trait.Clone.html #[inline] #[stable(feature = "mem_take", since = "1.40.0")] pub fn take(dest: &mut T) -> T { @@ -661,6 +843,9 @@ pub fn take(dest: &mut T) -> T { /// /// Neither value is dropped. /// +/// * If you want to replace the values of two variables, see [`swap`]. +/// * If you want to replace with a default value, see [`take`]. +/// /// # Examples /// /// A simple example: @@ -712,18 +897,25 @@ pub fn take(dest: &mut T) -> T { /// assert_eq!(buffer.replace_index(0, 2), 0); /// assert_eq!(buffer.buf[0], 2); /// ``` -/// -/// [`Clone`]: ../../std/clone/trait.Clone.html #[inline] #[stable(feature = "rust1", since = "1.0.0")] -pub fn replace(dest: &mut T, mut src: T) -> T { - swap(dest, &mut src); - src +#[must_use = "if you don't need the old value, you can just assign the new value directly"] +#[rustc_const_unstable(feature = "const_replace", issue = "83164")] +#[cfg_attr(not(test), rustc_diagnostic_item = "mem_replace")] +pub const fn replace(dest: &mut T, src: T) -> T { + // SAFETY: We read from `dest` but directly write `src` into it afterwards, + // such that the old value is not duplicated. Nothing is dropped and + // nothing here can panic. + unsafe { + let result = ptr::read(dest); + ptr::write(dest, src); + result + } } /// Disposes of a value. /// -/// This does call the argument's implementation of [`Drop`][drop]. +/// This does so by calling the argument's implementation of [`Drop`][drop]. /// /// This effectively does nothing for types which implement `Copy`, e.g. /// integers. Such values are copied and _then_ moved into the function, so the @@ -738,7 +930,7 @@ pub fn replace(dest: &mut T, mut src: T) -> T { /// Because `_x` is moved into the function, it is automatically dropped before /// the function returns. /// -/// [drop]: ../ops/trait.Drop.html +/// [drop]: Drop /// /// # Examples /// @@ -781,27 +973,48 @@ pub fn replace(dest: &mut T, mut src: T) -> T { /// println!("x: {}, y: {}", x, y.0); // still available /// ``` /// -/// [`RefCell`]: ../../std/cell/struct.RefCell.html -/// [`Copy`]: ../../std/marker/trait.Copy.html +/// [`RefCell`]: crate::cell::RefCell #[inline] #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "mem_drop")] pub fn drop(_x: T) {} -/// Interprets `src` as having type `&U`, and then reads `src` without moving +/// Bitwise-copies a value. +/// +/// This function is not magic; it is literally defined as +/// ``` +/// pub fn copy(x: &T) -> T { *x } +/// ``` +/// +/// It is useful when you want to pass a function pointer to a combinator, rather than defining a new closure. +/// +/// Example: +/// ``` +/// #![feature(mem_copy_fn)] +/// use core::mem::copy; +/// let result_from_ffi_function: Result<(), &i32> = Err(&1); +/// let result_copied: Result<(), i32> = result_from_ffi_function.map_err(copy); +/// ``` +#[inline] +#[unstable(feature = "mem_copy_fn", issue = "98262")] +pub const fn copy(x: &T) -> T { + *x +} + +/// Interprets `src` as having type `&Dst`, and then reads `src` without moving /// the contained value. /// -/// This function will unsafely assume the pointer `src` is valid for -/// [`size_of::`][size_of] bytes by transmuting `&T` to `&U` and then reading -/// the `&U`. It will also unsafely create a copy of the contained value instead of -/// moving out of `src`. +/// This function will unsafely assume the pointer `src` is valid for [`size_of::`][size_of] +/// bytes by transmuting `&Src` to `&Dst` and then reading the `&Dst` (except that this is done +/// in a way that is correct even when `&Dst` has stricter alignment requirements than `&Src`). +/// It will also unsafely create a copy of the contained value instead of moving out of `src`. /// -/// It is not a compile-time error if `T` and `U` have different sizes, but it -/// is highly encouraged to only invoke this function where `T` and `U` have the -/// same size. This function triggers [undefined behavior][ub] if `U` is larger than -/// `T`. +/// It is not a compile-time error if `Src` and `Dst` have different sizes, but it +/// is highly encouraged to only invoke this function where `Src` and `Dst` have the +/// same size. This function triggers [undefined behavior][ub] if `Dst` is larger than +/// `Src`. /// /// [ub]: ../../reference/behavior-considered-undefined.html -/// [size_of]: fn.size_of.html /// /// # Examples /// @@ -829,18 +1042,33 @@ pub fn drop(_x: T) {} /// assert_eq!(foo_array, [10]); /// ``` #[inline] +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] -pub unsafe fn transmute_copy(src: &T) -> U { - ptr::read_unaligned(src as *const T as *const U) +#[rustc_const_unstable(feature = "const_transmute_copy", issue = "83165")] +pub const unsafe fn transmute_copy(src: &Src) -> Dst { + assert!( + size_of::() >= size_of::(), + "cannot transmute_copy if Dst is larger than Src" + ); + + // If Dst has a higher alignment requirement, src might not be suitably aligned. + if align_of::() > align_of::() { + // SAFETY: `src` is a reference which is guaranteed to be valid for reads. + // The caller must guarantee that the actual transmutation is safe. + unsafe { ptr::read_unaligned(src as *const Src as *const Dst) } + } else { + // SAFETY: `src` is a reference which is guaranteed to be valid for reads. + // We just checked that `src as *const Dst` was properly aligned. + // The caller must guarantee that the actual transmutation is safe. + unsafe { ptr::read(src as *const Src as *const Dst) } + } } /// Opaque type representing the discriminant of an enum. /// /// See the [`discriminant`] function in this module for more information. -/// -/// [`discriminant`]: fn.discriminant.html #[stable(feature = "discriminant_value", since = "1.21.0")] -pub struct Discriminant(u64, PhantomData T>); +pub struct Discriminant(::Discriminant); // N.B. These trait implementations cannot be derived because we don't want any bounds on T. @@ -886,7 +1114,10 @@ impl fmt::Debug for Discriminant { /// # Stability /// /// The discriminant of an enum variant may change if the enum definition changes. A discriminant -/// of some variant will not change between compilations with the same compiler. +/// of some variant will not change between compilations with the same compiler. See the [Reference] +/// for more information. +/// +/// [Reference]: ../../reference/items/enumerations.html#custom-discriminant-values-for-fieldless-enumerations /// /// # Examples /// @@ -902,23 +1133,143 @@ impl fmt::Debug for Discriminant { /// assert_eq!(mem::discriminant(&Foo::B(1)), mem::discriminant(&Foo::B(2))); /// assert_ne!(mem::discriminant(&Foo::B(3)), mem::discriminant(&Foo::C(3))); /// ``` +/// +/// ## Accessing the numeric value of the discriminant +/// +/// Note that it is *undefined behavior* to [`transmute`] from [`Discriminant`] to a primitive! +/// +/// If an enum has only unit variants, then the numeric value of the discriminant can be accessed +/// with an [`as`] cast: +/// +/// ``` +/// enum Enum { +/// Foo, +/// Bar, +/// Baz, +/// } +/// +/// assert_eq!(0, Enum::Foo as isize); +/// assert_eq!(1, Enum::Bar as isize); +/// assert_eq!(2, Enum::Baz as isize); +/// ``` +/// +/// If an enum has opted-in to having a [primitive representation] for its discriminant, +/// then it's possible to use pointers to read the memory location storing the discriminant. +/// That **cannot** be done for enums using the [default representation], however, as it's +/// undefined what layout the discriminant has and where it's stored — it might not even be +/// stored at all! +/// +/// [`as`]: ../../std/keyword.as.html +/// [primitive representation]: ../../reference/type-layout.html#primitive-representations +/// [default representation]: ../../reference/type-layout.html#the-default-representation +/// ``` +/// #[repr(u8)] +/// enum Enum { +/// Unit, +/// Tuple(bool), +/// Struct { a: bool }, +/// } +/// +/// impl Enum { +/// fn discriminant(&self) -> u8 { +/// // SAFETY: Because `Self` is marked `repr(u8)`, its layout is a `repr(C)` `union` +/// // between `repr(C)` structs, each of which has the `u8` discriminant as its first +/// // field, so we can read the discriminant without offsetting the pointer. +/// unsafe { *<*const _>::from(self).cast::() } +/// } +/// } +/// +/// let unit_like = Enum::Unit; +/// let tuple_like = Enum::Tuple(true); +/// let struct_like = Enum::Struct { a: false }; +/// assert_eq!(0, unit_like.discriminant()); +/// assert_eq!(1, tuple_like.discriminant()); +/// assert_eq!(2, struct_like.discriminant()); +/// +/// // ⚠️ This is undefined behavior. Don't do this. ⚠️ +/// // assert_eq!(0, unsafe { std::mem::transmute::<_, u8>(std::mem::discriminant(&unit_like)) }); +/// ``` #[stable(feature = "discriminant_value", since = "1.21.0")] #[rustc_const_unstable(feature = "const_discriminant", issue = "69821")] +#[cfg_attr(not(test), rustc_diagnostic_item = "mem_discriminant")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const fn discriminant(v: &T) -> Discriminant { - Discriminant(intrinsics::discriminant_value(v), PhantomData) + Discriminant(intrinsics::discriminant_value(v)) } +/// Returns the number of variants in the enum type `T`. +/// +/// If `T` is not an enum, calling this function will not result in undefined behavior, but the +/// return value is unspecified. Equally, if `T` is an enum with more variants than `usize::MAX` +/// the return value is unspecified. Uninhabited variants will be counted. +/// +/// Note that an enum may be expanded with additional variants in the future +/// as a non-breaking change, for example if it is marked `#[non_exhaustive]`, +/// which will change the result of this function. +/// +/// # Examples +/// +/// ``` +/// # #![feature(never_type)] +/// # #![feature(variant_count)] +/// +/// use std::mem; +/// +/// enum Void {} +/// enum Foo { A(&'static str), B(i32), C(i32) } +/// +/// assert_eq!(mem::variant_count::(), 0); +/// assert_eq!(mem::variant_count::(), 3); +/// +/// assert_eq!(mem::variant_count::>(), 2); +/// assert_eq!(mem::variant_count::>(), 2); +/// ``` +#[inline(always)] +#[must_use] +#[unstable(feature = "variant_count", issue = "73662")] +#[rustc_const_unstable(feature = "variant_count", issue = "73662")] +#[rustc_diagnostic_item = "mem_variant_count"] +pub const fn variant_count() -> usize { + intrinsics::variant_count::() +} -/// Convert `T` to `U`. Both types must have the same Crucible representation. -#[unstable(feature = "crucible_intrinsics", issue = "none")] -#[rustc_const_stable(feature = "crucible_intrinsics", since = "1.0.0")] -#[inline(never)] -#[allow(unused_attributes)] -#[allow_internal_unstable(const_fn_union)] -pub const unsafe fn crucible_identity_transmute(x: T) -> U { - union Transmute { - x: ManuallyDrop, - y: ManuallyDrop, - } - ManuallyDrop::into_inner(Transmute { x: ManuallyDrop::new(x) }.y) +/// Provides associated constants for various useful properties of types, +/// to give them a canonical form in our code and make them easier to read. +/// +/// This is here only to simplify all the ZST checks we need in the library. +/// It's not on a stabilization track right now. +#[doc(hidden)] +#[unstable(feature = "sized_type_properties", issue = "none")] +pub trait SizedTypeProperties: Sized { + /// `true` if this type requires no storage. + /// `false` if its [size](size_of) is greater than zero. + /// + /// # Examples + /// + /// ``` + /// #![feature(sized_type_properties)] + /// use core::mem::SizedTypeProperties; + /// + /// fn do_something_with() { + /// if T::IS_ZST { + /// // ... special approach ... + /// } else { + /// // ... the normal thing ... + /// } + /// } + /// + /// struct MyUnit; + /// assert!(MyUnit::IS_ZST); + /// + /// // For negative checks, consider using UFCS to emphasize the negation + /// assert!(!::IS_ZST); + /// // As it can sometimes hide in the type otherwise + /// assert!(!String::IS_ZST); + /// ``` + #[doc(hidden)] + #[unstable(feature = "sized_type_properties", issue = "none")] + const IS_ZST: bool = size_of::() == 0; } +#[doc(hidden)] +#[unstable(feature = "sized_type_properties", issue = "none")] +impl SizedTypeProperties for T {} diff --git a/crux-mir/lib/core/src/mem/transmutability.rs b/crux-mir/lib/core/src/mem/transmutability.rs new file mode 100644 index 000000000..3b98efff2 --- /dev/null +++ b/crux-mir/lib/core/src/mem/transmutability.rs @@ -0,0 +1,107 @@ +/// Are values of a type transmutable into values of another type? +/// +/// This trait is implemented on-the-fly by the compiler for types `Src` and `Self` when the bits of +/// any value of type `Self` are safely transmutable into a value of type `Dst`, in a given `Context`, +/// notwithstanding whatever safety checks you have asked the compiler to [`Assume`] are satisfied. +#[unstable(feature = "transmutability", issue = "99571")] +#[lang = "transmute_trait"] +#[rustc_on_unimplemented( + message = "`{Src}` cannot be safely transmuted into `{Self}` in the defining scope of `{Context}`.", + label = "`{Src}` cannot be safely transmuted into `{Self}` in the defining scope of `{Context}`." +)] +pub unsafe trait BikeshedIntrinsicFrom +where + Src: ?Sized, +{ +} + +/// What transmutation safety conditions shall the compiler assume that *you* are checking? +#[unstable(feature = "transmutability", issue = "99571")] +#[lang = "transmute_opts"] +#[derive(PartialEq, Eq, Clone, Copy, Debug)] +pub struct Assume { + /// When `true`, the compiler assumes that *you* are ensuring (either dynamically or statically) that + /// destination referents do not have stricter alignment requirements than source referents. + pub alignment: bool, + + /// When `true`, the compiler assume that *you* are ensuring that lifetimes are not extended in a manner + /// that violates Rust's memory model. + pub lifetimes: bool, + + /// When `true`, the compiler assumes that *you* have ensured that it is safe for you to violate the + /// type and field privacy of the destination type (and sometimes of the source type, too). + pub safety: bool, + + /// When `true`, the compiler assumes that *you* are ensuring that the source type is actually a valid + /// instance of the destination type. + pub validity: bool, +} + +impl Assume { + /// Do not assume that *you* have ensured any safety properties are met. + #[unstable(feature = "transmutability", issue = "99571")] + pub const NOTHING: Self = + Self { alignment: false, lifetimes: false, safety: false, validity: false }; + + /// Assume only that alignment conditions are met. + #[unstable(feature = "transmutability", issue = "99571")] + pub const ALIGNMENT: Self = Self { alignment: true, ..Self::NOTHING }; + + /// Assume only that lifetime conditions are met. + #[unstable(feature = "transmutability", issue = "99571")] + pub const LIFETIMES: Self = Self { lifetimes: true, ..Self::NOTHING }; + + /// Assume only that safety conditions are met. + #[unstable(feature = "transmutability", issue = "99571")] + pub const SAFETY: Self = Self { safety: true, ..Self::NOTHING }; + + /// Assume only that dynamically-satisfiable validity conditions are met. + #[unstable(feature = "transmutability", issue = "99571")] + pub const VALIDITY: Self = Self { validity: true, ..Self::NOTHING }; + + /// Assume both `self` and `other_assumptions`. + #[unstable(feature = "transmutability", issue = "99571")] + pub const fn and(self, other_assumptions: Self) -> Self { + Self { + alignment: self.alignment || other_assumptions.alignment, + lifetimes: self.lifetimes || other_assumptions.lifetimes, + safety: self.safety || other_assumptions.safety, + validity: self.validity || other_assumptions.validity, + } + } + + /// Assume `self`, excepting `other_assumptions`. + #[unstable(feature = "transmutability", issue = "99571")] + pub const fn but_not(self, other_assumptions: Self) -> Self { + Self { + alignment: self.alignment && !other_assumptions.alignment, + lifetimes: self.lifetimes && !other_assumptions.lifetimes, + safety: self.safety && !other_assumptions.safety, + validity: self.validity && !other_assumptions.validity, + } + } +} + +// FIXME(jswrenn): This const op is not actually usable. Why? +// https://github.com/rust-lang/rust/pull/100726#issuecomment-1219928926 +#[unstable(feature = "transmutability", issue = "99571")] +#[rustc_const_unstable(feature = "transmutability", issue = "99571")] +impl const core::ops::Add for Assume { + type Output = Assume; + + fn add(self, other_assumptions: Assume) -> Assume { + self.and(other_assumptions) + } +} + +// FIXME(jswrenn): This const op is not actually usable. Why? +// https://github.com/rust-lang/rust/pull/100726#issuecomment-1219928926 +#[unstable(feature = "transmutability", issue = "99571")] +#[rustc_const_unstable(feature = "transmutability", issue = "99571")] +impl const core::ops::Sub for Assume { + type Output = Assume; + + fn sub(self, other_assumptions: Assume) -> Assume { + self.but_not(other_assumptions) + } +} diff --git a/crux-mir/lib/core/src/num/bignum.rs b/crux-mir/lib/core/src/num/bignum.rs index 6f16b93d0..d2a21b6b3 100644 --- a/crux-mir/lib/core/src/num/bignum.rs +++ b/crux-mir/lib/core/src/num/bignum.rs @@ -19,19 +19,8 @@ )] #![macro_use] -use crate::intrinsics; -use crate::mem; - /// Arithmetic operations required by bignums. pub trait FullOps: Sized { - /// Returns `(carry', v')` such that `carry' * 2^W + v' = self + other + carry`, - /// where `W` is the number of bits in `Self`. - fn full_add(self, other: Self, carry: bool) -> (bool /* carry */, Self); - - /// Returns `(carry', v')` such that `carry' * 2^W + v' = self * other + carry`, - /// where `W` is the number of bits in `Self`. - fn full_mul(self, other: Self, carry: Self) -> (Self /* carry */, Self); - /// Returns `(carry', v')` such that `carry' * 2^W + v' = self * other + other2 + carry`, /// where `W` is the number of bits in `Self`. fn full_mul_add(self, other: Self, other2: Self, carry: Self) -> (Self /* carry */, Self); @@ -46,37 +35,18 @@ macro_rules! impl_full_ops { ($($ty:ty: add($addfn:path), mul/div($bigty:ident);)*) => ( $( impl FullOps for $ty { - fn full_add(self, other: $ty, carry: bool) -> (bool, $ty) { - // This cannot overflow; the output is between `0` and `2 * 2^nbits - 1`. - // FIXME: will LLVM optimize this into ADC or similar? - let (v, carry1) = intrinsics::add_with_overflow(self, other); - let (v, carry2) = intrinsics::add_with_overflow(v, if carry {1} else {0}); - (carry1 || carry2, v) - } - - fn full_mul(self, other: $ty, carry: $ty) -> ($ty, $ty) { - // This cannot overflow; - // the output is between `0` and `2^nbits * (2^nbits - 1)`. - // FIXME: will LLVM optimize this into ADC or similar? - let nbits = mem::size_of::<$ty>() * 8; - let v = (self as $bigty) * (other as $bigty) + (carry as $bigty); - ((v >> nbits) as $ty, v as $ty) - } - fn full_mul_add(self, other: $ty, other2: $ty, carry: $ty) -> ($ty, $ty) { // This cannot overflow; // the output is between `0` and `2^nbits * (2^nbits - 1)`. - let nbits = mem::size_of::<$ty>() * 8; let v = (self as $bigty) * (other as $bigty) + (other2 as $bigty) + (carry as $bigty); - ((v >> nbits) as $ty, v as $ty) + ((v >> <$ty>::BITS) as $ty, v as $ty) } fn full_div_rem(self, other: $ty, borrow: $ty) -> ($ty, $ty) { debug_assert!(borrow < other); // This cannot overflow; the output is between `0` and `other * (2^nbits - 1)`. - let nbits = mem::size_of::<$ty>() * 8; - let lhs = ((borrow as $bigty) << nbits) | (self as $bigty); + let lhs = ((borrow as $bigty) << <$ty>::BITS) | (self as $bigty); let rhs = other as $bigty; ((lhs / rhs) as $ty, (lhs % rhs) as $ty) } @@ -123,21 +93,19 @@ macro_rules! define_bignum { pub fn from_small(v: $ty) -> $name { let mut base = [0; $n]; base[0] = v; - $name { size: 1, base: base } + $name { size: 1, base } } /// Makes a bignum from `u64` value. pub fn from_u64(mut v: u64) -> $name { - use crate::mem; - let mut base = [0; $n]; let mut sz = 0; while v > 0 { base[sz] = v as $ty; - v >>= mem::size_of::<$ty>() * 8; + v >>= <$ty>::BITS; sz += 1; } - $name { size: sz, base: base } + $name { size: sz, base } } /// Returns the internal digits as a slice `[a, b, c, ...]` such that the numeric @@ -150,9 +118,7 @@ macro_rules! define_bignum { /// Returns the `i`-th bit where bit 0 is the least significant one. /// In other words, the bit with weight `2^i`. pub fn get_bit(&self, i: usize) -> u8 { - use crate::mem; - - let digitbits = mem::size_of::<$ty>() * 8; + let digitbits = <$ty>::BITS as usize; let d = i / digitbits; let b = i % digitbits; ((self.base[d] >> b) & 1) as u8 @@ -166,37 +132,26 @@ macro_rules! define_bignum { /// Returns the number of bits necessary to represent this value. Note that zero /// is considered to need 0 bits. pub fn bit_length(&self) -> usize { - use crate::mem; - - // Skip over the most significant digits which are zero. + let digitbits = <$ty>::BITS as usize; let digits = self.digits(); - let zeros = digits.iter().rev().take_while(|&&x| x == 0).count(); - let end = digits.len() - zeros; - let nonzero = &digits[..end]; - - if nonzero.is_empty() { + // Find the most significant non-zero digit. + let msd = digits.iter().rposition(|&x| x != 0); + match msd { + Some(msd) => msd * digitbits + digits[msd].ilog2() as usize + 1, // There are no non-zero digits, i.e., the number is zero. - return 0; + _ => 0, } - // This could be optimized with leading_zeros() and bit shifts, but that's - // probably not worth the hassle. - let digitbits = mem::size_of::<$ty>() * 8; - let mut i = nonzero.len() * digitbits - 1; - while self.get_bit(i) == 0 { - i -= 1; - } - i + 1 } /// Adds `other` to itself and returns its own mutable reference. pub fn add<'a>(&'a mut self, other: &$name) -> &'a mut $name { use crate::cmp; - use crate::num::bignum::FullOps; + use crate::iter; let mut sz = cmp::max(self.size, other.size); let mut carry = false; - for (a, b) in self.base[..sz].iter_mut().zip(&other.base[..sz]) { - let (c, v) = (*a).full_add(*b, carry); + for (a, b) in iter::zip(&mut self.base[..sz], &other.base[..sz]) { + let (v, c) = (*a).carrying_add(*b, carry); *a = v; carry = c; } @@ -209,13 +164,11 @@ macro_rules! define_bignum { } pub fn add_small(&mut self, other: $ty) -> &mut $name { - use crate::num::bignum::FullOps; - - let (mut carry, v) = self.base[0].full_add(other, false); + let (v, mut carry) = self.base[0].carrying_add(other, false); self.base[0] = v; let mut i = 1; while carry { - let (c, v) = self.base[i].full_add(0, carry); + let (v, c) = self.base[i].carrying_add(0, carry); self.base[i] = v; carry = c; i += 1; @@ -229,12 +182,12 @@ macro_rules! define_bignum { /// Subtracts `other` from itself and returns its own mutable reference. pub fn sub<'a>(&'a mut self, other: &$name) -> &'a mut $name { use crate::cmp; - use crate::num::bignum::FullOps; + use crate::iter; let sz = cmp::max(self.size, other.size); let mut noborrow = true; - for (a, b) in self.base[..sz].iter_mut().zip(&other.base[..sz]) { - let (c, v) = (*a).full_add(!*b, noborrow); + for (a, b) in iter::zip(&mut self.base[..sz], &other.base[..sz]) { + let (v, c) = (*a).carrying_add(!*b, noborrow); *a = v; noborrow = c; } @@ -246,12 +199,10 @@ macro_rules! define_bignum { /// Multiplies itself by a digit-sized `other` and returns its own /// mutable reference. pub fn mul_small(&mut self, other: $ty) -> &mut $name { - use crate::num::bignum::FullOps; - let mut sz = self.size; let mut carry = 0; for a in &mut self.base[..sz] { - let (c, v) = (*a).full_mul(other, carry); + let (v, c) = (*a).carrying_mul(other, carry); *a = v; carry = c; } @@ -265,9 +216,7 @@ macro_rules! define_bignum { /// Multiplies itself by `2^bits` and returns its own mutable reference. pub fn mul_pow2(&mut self, bits: usize) -> &mut $name { - use crate::mem; - - let digitbits = mem::size_of::<$ty>() * 8; + let digitbits = <$ty>::BITS as usize; let digits = bits / digitbits; let bits = bits % digitbits; @@ -393,13 +342,11 @@ macro_rules! define_bignum { /// Divide self by another bignum, overwriting `q` with the quotient and `r` with the /// remainder. pub fn div_rem(&self, d: &$name, q: &mut $name, r: &mut $name) { - use crate::mem; - // Stupid slow base-2 long division taken from // https://en.wikipedia.org/wiki/Division_algorithm // FIXME use a greater base ($ty) for the long division. assert!(!d.is_zero()); - let digitbits = mem::size_of::<$ty>() * 8; + let digitbits = <$ty>::BITS as usize; for digit in &mut q.base[..] { *digit = 0; } @@ -462,10 +409,8 @@ macro_rules! define_bignum { impl crate::fmt::Debug for $name { fn fmt(&self, f: &mut crate::fmt::Formatter<'_>) -> crate::fmt::Result { - use crate::mem; - let sz = if self.size < 1 { 1 } else { self.size }; - let digitlen = mem::size_of::<$ty>() * 2; + let digitlen = <$ty>::BITS as usize / 4; write!(f, "{:#x}", self.base[sz - 1])?; for &v in self.base[..sz - 1].iter().rev() { diff --git a/crux-mir/lib/core/src/num/dec2flt/algorithm.rs b/crux-mir/lib/core/src/num/dec2flt/algorithm.rs deleted file mode 100644 index c5f6903f3..000000000 --- a/crux-mir/lib/core/src/num/dec2flt/algorithm.rs +++ /dev/null @@ -1,417 +0,0 @@ -//! The various algorithms from the paper. - -use crate::cmp::min; -use crate::cmp::Ordering::{Equal, Greater, Less}; -use crate::num::dec2flt::num::{self, Big}; -use crate::num::dec2flt::rawfp::{self, fp_to_float, next_float, prev_float, RawFloat, Unpacked}; -use crate::num::dec2flt::table; -use crate::num::diy_float::Fp; - -/// Number of significand bits in Fp -const P: u32 = 64; - -// We simply store the best approximation for *all* exponents, so the variable "h" and the -// associated conditions can be omitted. This trades performance for a couple kilobytes of space. - -fn power_of_ten(e: i16) -> Fp { - assert!(e >= table::MIN_E); - let i = e - table::MIN_E; - let sig = table::POWERS.0[i as usize]; - let exp = table::POWERS.1[i as usize]; - Fp { f: sig, e: exp } -} - -// In most architectures, floating point operations have an explicit bit size, therefore the -// precision of the computation is determined on a per-operation basis. -#[cfg(any(not(target_arch = "x86"), target_feature = "sse2"))] -mod fpu_precision { - pub fn set_precision() {} -} - -// On x86, the x87 FPU is used for float operations if the SSE/SSE2 extensions are not available. -// The x87 FPU operates with 80 bits of precision by default, which means that operations will -// round to 80 bits causing double rounding to happen when values are eventually represented as -// 32/64 bit float values. To overcome this, the FPU control word can be set so that the -// computations are performed in the desired precision. -#[cfg(all(target_arch = "x86", not(target_feature = "sse2")))] -mod fpu_precision { - use crate::mem::size_of; - - /// A structure used to preserve the original value of the FPU control word, so that it can be - /// restored when the structure is dropped. - /// - /// The x87 FPU is a 16-bits register whose fields are as follows: - /// - /// | 12-15 | 10-11 | 8-9 | 6-7 | 5 | 4 | 3 | 2 | 1 | 0 | - /// |------:|------:|----:|----:|---:|---:|---:|---:|---:|---:| - /// | | RC | PC | | PM | UM | OM | ZM | DM | IM | - /// - /// The documentation for all of the fields is available in the IA-32 Architectures Software - /// Developer's Manual (Volume 1). - /// - /// The only field which is relevant for the following code is PC, Precision Control. This - /// field determines the precision of the operations performed by the FPU. It can be set to: - /// - 0b00, single precision i.e., 32-bits - /// - 0b10, double precision i.e., 64-bits - /// - 0b11, double extended precision i.e., 80-bits (default state) - /// The 0b01 value is reserved and should not be used. - pub struct FPUControlWord(u16); - - fn set_cw(cw: u16) { - // SAFETY: the `fldcw` instruction has been audited to be able to work correctly with - // any `u16` - unsafe { asm!("fldcw $0" :: "m" (cw) :: "volatile") } - } - - /// Sets the precision field of the FPU to `T` and returns a `FPUControlWord`. - pub fn set_precision() -> FPUControlWord { - let cw = 0u16; - - // Compute the value for the Precision Control field that is appropriate for `T`. - let cw_precision = match size_of::() { - 4 => 0x0000, // 32 bits - 8 => 0x0200, // 64 bits - _ => 0x0300, // default, 80 bits - }; - - // Get the original value of the control word to restore it later, when the - // `FPUControlWord` structure is dropped - // SAFETY: the `fnstcw` instruction has been audited to be able to work correctly with - // any `u16` - unsafe { asm!("fnstcw $0" : "=*m" (&cw) ::: "volatile") } - - // Set the control word to the desired precision. This is achieved by masking away the old - // precision (bits 8 and 9, 0x300) and replacing it with the precision flag computed above. - set_cw((cw & 0xFCFF) | cw_precision); - - FPUControlWord(cw) - } - - impl Drop for FPUControlWord { - fn drop(&mut self) { - set_cw(self.0) - } - } -} - -/// The fast path of Bellerophon using machine-sized integers and floats. -/// -/// This is extracted into a separate function so that it can be attempted before constructing -/// a bignum. -pub fn fast_path(integral: &[u8], fractional: &[u8], e: i64) -> Option { - let num_digits = integral.len() + fractional.len(); - // log_10(f64::MAX_SIG) ~ 15.95. We compare the exact value to MAX_SIG near the end, - // this is just a quick, cheap rejection (and also frees the rest of the code from - // worrying about underflow). - if num_digits > 16 { - return None; - } - if e.abs() >= T::CEIL_LOG5_OF_MAX_SIG as i64 { - return None; - } - let f = num::from_str_unchecked(integral.iter().chain(fractional.iter())); - if f > T::MAX_SIG { - return None; - } - - // The fast path crucially depends on arithmetic being rounded to the correct number of bits - // without any intermediate rounding. On x86 (without SSE or SSE2) this requires the precision - // of the x87 FPU stack to be changed so that it directly rounds to 64/32 bit. - // The `set_precision` function takes care of setting the precision on architectures which - // require setting it by changing the global state (like the control word of the x87 FPU). - let _cw = fpu_precision::set_precision::(); - - // The case e < 0 cannot be folded into the other branch. Negative powers result in - // a repeating fractional part in binary, which are rounded, which causes real - // (and occasionally quite significant!) errors in the final result. - if e >= 0 { - Some(T::from_int(f) * T::short_fast_pow10(e as usize)) - } else { - Some(T::from_int(f) / T::short_fast_pow10(e.abs() as usize)) - } -} - -/// Algorithm Bellerophon is trivial code justified by non-trivial numeric analysis. -/// -/// It rounds ``f`` to a float with 64 bit significand and multiplies it by the best approximation -/// of `10^e` (in the same floating point format). This is often enough to get the correct result. -/// However, when the result is close to halfway between two adjacent (ordinary) floats, the -/// compound rounding error from multiplying two approximation means the result may be off by a -/// few bits. When this happens, the iterative Algorithm R fixes things up. -/// -/// The hand-wavy "close to halfway" is made precise by the numeric analysis in the paper. -/// In the words of Clinger: -/// -/// > Slop, expressed in units of the least significant bit, is an inclusive bound for the error -/// > accumulated during the floating point calculation of the approximation to f * 10^e. (Slop is -/// > not a bound for the true error, but bounds the difference between the approximation z and -/// > the best possible approximation that uses p bits of significand.) -pub fn bellerophon(f: &Big, e: i16) -> T { - let slop = if f <= &Big::from_u64(T::MAX_SIG) { - // The cases abs(e) < log5(2^N) are in fast_path() - if e >= 0 { 0 } else { 3 } - } else { - if e >= 0 { 1 } else { 4 } - }; - let z = rawfp::big_to_fp(f).mul(&power_of_ten(e)).normalize(); - let exp_p_n = 1 << (P - T::SIG_BITS as u32); - let lowbits: i64 = (z.f % exp_p_n) as i64; - // Is the slop large enough to make a difference when - // rounding to n bits? - if (lowbits - exp_p_n as i64 / 2).abs() <= slop { - algorithm_r(f, e, fp_to_float(z)) - } else { - fp_to_float(z) - } -} - -/// An iterative algorithm that improves a floating point approximation of `f * 10^e`. -/// -/// Each iteration gets one unit in the last place closer, which of course takes terribly long to -/// converge if `z0` is even mildly off. Luckily, when used as fallback for Bellerophon, the -/// starting approximation is off by at most one ULP. -fn algorithm_r(f: &Big, e: i16, z0: T) -> T { - let mut z = z0; - loop { - let raw = z.unpack(); - let (m, k) = (raw.sig, raw.k); - let mut x = f.clone(); - let mut y = Big::from_u64(m); - - // Find positive integers `x`, `y` such that `x / y` is exactly `(f * 10^e) / (m * 2^k)`. - // This not only avoids dealing with the signs of `e` and `k`, we also eliminate the - // power of two common to `10^e` and `2^k` to make the numbers smaller. - make_ratio(&mut x, &mut y, e, k); - - let m_digits = [(m & 0xFF_FF_FF_FF) as u32, (m >> 32) as u32]; - // This is written a bit awkwardly because our bignums don't support - // negative numbers, so we use the absolute value + sign information. - // The multiplication with m_digits can't overflow. If `x` or `y` are large enough that - // we need to worry about overflow, then they are also large enough that `make_ratio` has - // reduced the fraction by a factor of 2^64 or more. - let (d2, d_negative) = if x >= y { - // Don't need x any more, save a clone(). - x.sub(&y).mul_pow2(1).mul_digits(&m_digits); - (x, false) - } else { - // Still need y - make a copy. - let mut y = y.clone(); - y.sub(&x).mul_pow2(1).mul_digits(&m_digits); - (y, true) - }; - - if d2 < y { - let mut d2_double = d2; - d2_double.mul_pow2(1); - if m == T::MIN_SIG && d_negative && d2_double > y { - z = prev_float(z); - } else { - return z; - } - } else if d2 == y { - if m % 2 == 0 { - if m == T::MIN_SIG && d_negative { - z = prev_float(z); - } else { - return z; - } - } else if d_negative { - z = prev_float(z); - } else { - z = next_float(z); - } - } else if d_negative { - z = prev_float(z); - } else { - z = next_float(z); - } - } -} - -/// Given `x = f` and `y = m` where `f` represent input decimal digits as usual and `m` is the -/// significand of a floating point approximation, make the ratio `x / y` equal to -/// `(f * 10^e) / (m * 2^k)`, possibly reduced by a power of two both have in common. -fn make_ratio(x: &mut Big, y: &mut Big, e: i16, k: i16) { - let (e_abs, k_abs) = (e.abs() as usize, k.abs() as usize); - if e >= 0 { - if k >= 0 { - // x = f * 10^e, y = m * 2^k, except that we reduce the fraction by some power of two. - let common = min(e_abs, k_abs); - x.mul_pow5(e_abs).mul_pow2(e_abs - common); - y.mul_pow2(k_abs - common); - } else { - // x = f * 10^e * 2^abs(k), y = m - // This can't overflow because it requires positive `e` and negative `k`, which can - // only happen for values extremely close to 1, which means that `e` and `k` will be - // comparatively tiny. - x.mul_pow5(e_abs).mul_pow2(e_abs + k_abs); - } - } else { - if k >= 0 { - // x = f, y = m * 10^abs(e) * 2^k - // This can't overflow either, see above. - y.mul_pow5(e_abs).mul_pow2(k_abs + e_abs); - } else { - // x = f * 2^abs(k), y = m * 10^abs(e), again reducing by a common power of two. - let common = min(e_abs, k_abs); - x.mul_pow2(k_abs - common); - y.mul_pow5(e_abs).mul_pow2(e_abs - common); - } - } -} - -/// Conceptually, Algorithm M is the simplest way to convert a decimal to a float. -/// -/// We form a ratio that is equal to `f * 10^e`, then throwing in powers of two until it gives -/// a valid float significand. The binary exponent `k` is the number of times we multiplied -/// numerator or denominator by two, i.e., at all times `f * 10^e` equals `(u / v) * 2^k`. -/// When we have found out significand, we only need to round by inspecting the remainder of the -/// division, which is done in helper functions further below. -/// -/// This algorithm is super slow, even with the optimization described in `quick_start()`. -/// However, it's the simplest of the algorithms to adapt for overflow, underflow, and subnormal -/// results. This implementation takes over when Bellerophon and Algorithm R are overwhelmed. -/// Detecting underflow and overflow is easy: The ratio still isn't an in-range significand, -/// yet the minimum/maximum exponent has been reached. In the case of overflow, we simply return -/// infinity. -/// -/// Handling underflow and subnormals is trickier. One big problem is that, with the minimum -/// exponent, the ratio might still be too large for a significand. See underflow() for details. -pub fn algorithm_m(f: &Big, e: i16) -> T { - let mut u; - let mut v; - let e_abs = e.abs() as usize; - let mut k = 0; - if e < 0 { - u = f.clone(); - v = Big::from_small(1); - v.mul_pow5(e_abs).mul_pow2(e_abs); - } else { - // FIXME possible optimization: generalize big_to_fp so that we can do the equivalent of - // fp_to_float(big_to_fp(u)) here, only without the double rounding. - u = f.clone(); - u.mul_pow5(e_abs).mul_pow2(e_abs); - v = Big::from_small(1); - } - quick_start::(&mut u, &mut v, &mut k); - let mut rem = Big::from_small(0); - let mut x = Big::from_small(0); - let min_sig = Big::from_u64(T::MIN_SIG); - let max_sig = Big::from_u64(T::MAX_SIG); - loop { - u.div_rem(&v, &mut x, &mut rem); - if k == T::MIN_EXP_INT { - // We have to stop at the minimum exponent, if we wait until `k < T::MIN_EXP_INT`, - // then we'd be off by a factor of two. Unfortunately this means we have to special- - // case normal numbers with the minimum exponent. - // FIXME find a more elegant formulation, but run the `tiny-pow10` test to make sure - // that it's actually correct! - if x >= min_sig && x <= max_sig { - break; - } - return underflow(x, v, rem); - } - if k > T::MAX_EXP_INT { - return T::INFINITY; - } - if x < min_sig { - u.mul_pow2(1); - k -= 1; - } else if x > max_sig { - v.mul_pow2(1); - k += 1; - } else { - break; - } - } - let q = num::to_u64(&x); - let z = rawfp::encode_normal(Unpacked::new(q, k)); - round_by_remainder(v, rem, q, z) -} - -/// Skips over most Algorithm M iterations by checking the bit length. -fn quick_start(u: &mut Big, v: &mut Big, k: &mut i16) { - // The bit length is an estimate of the base two logarithm, and log(u / v) = log(u) - log(v). - // The estimate is off by at most 1, but always an under-estimate, so the error on log(u) - // and log(v) are of the same sign and cancel out (if both are large). Therefore the error - // for log(u / v) is at most one as well. - // The target ratio is one where u/v is in an in-range significand. Thus our termination - // condition is log2(u / v) being the significand bits, plus/minus one. - // FIXME Looking at the second bit could improve the estimate and avoid some more divisions. - let target_ratio = T::SIG_BITS as i16; - let log2_u = u.bit_length() as i16; - let log2_v = v.bit_length() as i16; - let mut u_shift: i16 = 0; - let mut v_shift: i16 = 0; - assert!(*k == 0); - loop { - if *k == T::MIN_EXP_INT { - // Underflow or subnormal. Leave it to the main function. - break; - } - if *k == T::MAX_EXP_INT { - // Overflow. Leave it to the main function. - break; - } - let log2_ratio = (log2_u + u_shift) - (log2_v + v_shift); - if log2_ratio < target_ratio - 1 { - u_shift += 1; - *k -= 1; - } else if log2_ratio > target_ratio + 1 { - v_shift += 1; - *k += 1; - } else { - break; - } - } - u.mul_pow2(u_shift as usize); - v.mul_pow2(v_shift as usize); -} - -fn underflow(x: Big, v: Big, rem: Big) -> T { - if x < Big::from_u64(T::MIN_SIG) { - let q = num::to_u64(&x); - let z = rawfp::encode_subnormal(q); - return round_by_remainder(v, rem, q, z); - } - // Ratio isn't an in-range significand with the minimum exponent, so we need to round off - // excess bits and adjust the exponent accordingly. The real value now looks like this: - // - // x lsb - // /--------------\/ - // 1010101010101010.10101010101010 * 2^k - // \-----/\-------/ \------------/ - // q trunc. (represented by rem) - // - // Therefore, when the rounded-off bits are != 0.5 ULP, they decide the rounding - // on their own. When they are equal and the remainder is non-zero, the value still - // needs to be rounded up. Only when the rounded off bits are 1/2 and the remainder - // is zero, we have a half-to-even situation. - let bits = x.bit_length(); - let lsb = bits - T::SIG_BITS as usize; - let q = num::get_bits(&x, lsb, bits); - let k = T::MIN_EXP_INT + lsb as i16; - let z = rawfp::encode_normal(Unpacked::new(q, k)); - let q_even = q % 2 == 0; - match num::compare_with_half_ulp(&x, lsb) { - Greater => next_float(z), - Less => z, - Equal if rem.is_zero() && q_even => z, - Equal => next_float(z), - } -} - -/// Ordinary round-to-even, obfuscated by having to round based on the remainder of a division. -fn round_by_remainder(v: Big, r: Big, q: u64, z: T) -> T { - let mut v_minus_r = v; - v_minus_r.sub(&r); - if r < v_minus_r { - z - } else if r > v_minus_r { - next_float(z) - } else if q % 2 == 0 { - z - } else { - next_float(z) - } -} diff --git a/crux-mir/lib/core/src/num/dec2flt/common.rs b/crux-mir/lib/core/src/num/dec2flt/common.rs new file mode 100644 index 000000000..17957d7e7 --- /dev/null +++ b/crux-mir/lib/core/src/num/dec2flt/common.rs @@ -0,0 +1,198 @@ +//! Common utilities, for internal use only. + +use crate::ptr; + +/// Helper methods to process immutable bytes. +pub(crate) trait ByteSlice: AsRef<[u8]> { + unsafe fn first_unchecked(&self) -> u8 { + debug_assert!(!self.is_empty()); + // SAFETY: safe as long as self is not empty + unsafe { *self.as_ref().get_unchecked(0) } + } + + /// Get if the slice contains no elements. + fn is_empty(&self) -> bool { + self.as_ref().is_empty() + } + + /// Check if the slice at least `n` length. + fn check_len(&self, n: usize) -> bool { + n <= self.as_ref().len() + } + + /// Check if the first character in the slice is equal to c. + fn first_is(&self, c: u8) -> bool { + self.as_ref().first() == Some(&c) + } + + /// Check if the first character in the slice is equal to c1 or c2. + fn first_is2(&self, c1: u8, c2: u8) -> bool { + if let Some(&c) = self.as_ref().first() { c == c1 || c == c2 } else { false } + } + + /// Bounds-checked test if the first character in the slice is a digit. + fn first_isdigit(&self) -> bool { + if let Some(&c) = self.as_ref().first() { c.is_ascii_digit() } else { false } + } + + /// Check if self starts with u with a case-insensitive comparison. + fn starts_with_ignore_case(&self, u: &[u8]) -> bool { + debug_assert!(self.as_ref().len() >= u.len()); + let iter = self.as_ref().iter().zip(u.iter()); + let d = iter.fold(0, |i, (&x, &y)| i | (x ^ y)); + d == 0 || d == 32 + } + + /// Get the remaining slice after the first N elements. + fn advance(&self, n: usize) -> &[u8] { + &self.as_ref()[n..] + } + + /// Get the slice after skipping all leading characters equal c. + fn skip_chars(&self, c: u8) -> &[u8] { + let mut s = self.as_ref(); + while s.first_is(c) { + s = s.advance(1); + } + s + } + + /// Get the slice after skipping all leading characters equal c1 or c2. + fn skip_chars2(&self, c1: u8, c2: u8) -> &[u8] { + let mut s = self.as_ref(); + while s.first_is2(c1, c2) { + s = s.advance(1); + } + s + } + + /// Read 8 bytes as a 64-bit integer in little-endian order. + unsafe fn read_u64_unchecked(&self) -> u64 { + debug_assert!(self.check_len(8)); + let src = self.as_ref().as_ptr() as *const u64; + // SAFETY: safe as long as self is at least 8 bytes + u64::from_le(unsafe { ptr::read_unaligned(src) }) + } + + /// Try to read the next 8 bytes from the slice. + fn read_u64(&self) -> Option { + if self.check_len(8) { + // SAFETY: self must be at least 8 bytes. + Some(unsafe { self.read_u64_unchecked() }) + } else { + None + } + } + + /// Calculate the offset of slice from another. + fn offset_from(&self, other: &Self) -> isize { + other.as_ref().len() as isize - self.as_ref().len() as isize + } +} + +impl ByteSlice for [u8] {} + +/// Helper methods to process mutable bytes. +pub(crate) trait ByteSliceMut: AsMut<[u8]> { + /// Write a 64-bit integer as 8 bytes in little-endian order. + unsafe fn write_u64_unchecked(&mut self, value: u64) { + debug_assert!(self.as_mut().len() >= 8); + let dst = self.as_mut().as_mut_ptr() as *mut u64; + // NOTE: we must use `write_unaligned`, since dst is not + // guaranteed to be properly aligned. Miri will warn us + // if we use `write` instead of `write_unaligned`, as expected. + // SAFETY: safe as long as self is at least 8 bytes + unsafe { + ptr::write_unaligned(dst, u64::to_le(value)); + } + } +} + +impl ByteSliceMut for [u8] {} + +/// Bytes wrapper with specialized methods for ASCII characters. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct AsciiStr<'a> { + slc: &'a [u8], +} + +impl<'a> AsciiStr<'a> { + pub fn new(slc: &'a [u8]) -> Self { + Self { slc } + } + + /// Advance the view by n, advancing it in-place to (n..). + pub unsafe fn step_by(&mut self, n: usize) -> &mut Self { + // SAFETY: safe as long n is less than the buffer length + self.slc = unsafe { self.slc.get_unchecked(n..) }; + self + } + + /// Advance the view by n, advancing it in-place to (1..). + pub unsafe fn step(&mut self) -> &mut Self { + // SAFETY: safe as long as self is not empty + unsafe { self.step_by(1) } + } + + /// Iteratively parse and consume digits from bytes. + pub fn parse_digits(&mut self, mut func: impl FnMut(u8)) { + while let Some(&c) = self.as_ref().first() { + let c = c.wrapping_sub(b'0'); + if c < 10 { + func(c); + // SAFETY: self cannot be empty + unsafe { + self.step(); + } + } else { + break; + } + } + } +} + +impl<'a> AsRef<[u8]> for AsciiStr<'a> { + #[inline] + fn as_ref(&self) -> &[u8] { + self.slc + } +} + +impl<'a> ByteSlice for AsciiStr<'a> {} + +/// Determine if 8 bytes are all decimal digits. +/// This does not care about the order in which the bytes were loaded. +pub(crate) fn is_8digits(v: u64) -> bool { + let a = v.wrapping_add(0x4646_4646_4646_4646); + let b = v.wrapping_sub(0x3030_3030_3030_3030); + (a | b) & 0x8080_8080_8080_8080 == 0 +} + +/// Iteratively parse and consume digits from bytes. +pub(crate) fn parse_digits(s: &mut &[u8], mut f: impl FnMut(u8)) { + while let Some(&c) = s.get(0) { + let c = c.wrapping_sub(b'0'); + if c < 10 { + f(c); + *s = s.advance(1); + } else { + break; + } + } +} + +/// A custom 64-bit floating point type, representing `f * 2^e`. +/// e is biased, so it be directly shifted into the exponent bits. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Default)] +pub struct BiasedFp { + /// The significant digits. + pub f: u64, + /// The biased, binary exponent. + pub e: i32, +} + +impl BiasedFp { + pub const fn zero_pow2(e: i32) -> Self { + Self { f: 0, e } + } +} diff --git a/crux-mir/lib/core/src/num/dec2flt/decimal.rs b/crux-mir/lib/core/src/num/dec2flt/decimal.rs new file mode 100644 index 000000000..2019f71e6 --- /dev/null +++ b/crux-mir/lib/core/src/num/dec2flt/decimal.rs @@ -0,0 +1,351 @@ +//! Arbitrary-precision decimal class for fallback algorithms. +//! +//! This is only used if the fast-path (native floats) and +//! the Eisel-Lemire algorithm are unable to unambiguously +//! determine the float. +//! +//! The technique used is "Simple Decimal Conversion", developed +//! by Nigel Tao and Ken Thompson. A detailed description of the +//! algorithm can be found in "ParseNumberF64 by Simple Decimal Conversion", +//! available online: . + +use crate::num::dec2flt::common::{is_8digits, parse_digits, ByteSlice, ByteSliceMut}; + +#[derive(Clone)] +pub struct Decimal { + /// The number of significant digits in the decimal. + pub num_digits: usize, + /// The offset of the decimal point in the significant digits. + pub decimal_point: i32, + /// If the number of significant digits stored in the decimal is truncated. + pub truncated: bool, + /// Buffer of the raw digits, in the range [0, 9]. + pub digits: [u8; Self::MAX_DIGITS], +} + +impl Default for Decimal { + fn default() -> Self { + Self { num_digits: 0, decimal_point: 0, truncated: false, digits: [0; Self::MAX_DIGITS] } + } +} + +impl Decimal { + /// The maximum number of digits required to unambiguously round a float. + /// + /// For a double-precision IEEE 754 float, this required 767 digits, + /// so we store the max digits + 1. + /// + /// We can exactly represent a float in radix `b` from radix 2 if + /// `b` is divisible by 2. This function calculates the exact number of + /// digits required to exactly represent that float. + /// + /// According to the "Handbook of Floating Point Arithmetic", + /// for IEEE754, with emin being the min exponent, p2 being the + /// precision, and b being the radix, the number of digits follows as: + /// + /// `−emin + p2 + ⌊(emin + 1) log(2, b) − log(1 − 2^(−p2), b)⌋` + /// + /// For f32, this follows as: + /// emin = -126 + /// p2 = 24 + /// + /// For f64, this follows as: + /// emin = -1022 + /// p2 = 53 + /// + /// In Python: + /// `-emin + p2 + math.floor((emin+ 1)*math.log(2, b)-math.log(1-2**(-p2), b))` + pub const MAX_DIGITS: usize = 768; + /// The max digits that can be exactly represented in a 64-bit integer. + pub const MAX_DIGITS_WITHOUT_OVERFLOW: usize = 19; + pub const DECIMAL_POINT_RANGE: i32 = 2047; + + /// Append a digit to the buffer. + pub fn try_add_digit(&mut self, digit: u8) { + if self.num_digits < Self::MAX_DIGITS { + self.digits[self.num_digits] = digit; + } + self.num_digits += 1; + } + + /// Trim trailing zeros from the buffer. + pub fn trim(&mut self) { + // All of the following calls to `Decimal::trim` can't panic because: + // + // 1. `parse_decimal` sets `num_digits` to a max of `Decimal::MAX_DIGITS`. + // 2. `right_shift` sets `num_digits` to `write_index`, which is bounded by `num_digits`. + // 3. `left_shift` `num_digits` to a max of `Decimal::MAX_DIGITS`. + // + // Trim is only called in `right_shift` and `left_shift`. + debug_assert!(self.num_digits <= Self::MAX_DIGITS); + while self.num_digits != 0 && self.digits[self.num_digits - 1] == 0 { + self.num_digits -= 1; + } + } + + pub fn round(&self) -> u64 { + if self.num_digits == 0 || self.decimal_point < 0 { + return 0; + } else if self.decimal_point > 18 { + return 0xFFFF_FFFF_FFFF_FFFF_u64; + } + let dp = self.decimal_point as usize; + let mut n = 0_u64; + for i in 0..dp { + n *= 10; + if i < self.num_digits { + n += self.digits[i] as u64; + } + } + let mut round_up = false; + if dp < self.num_digits { + round_up = self.digits[dp] >= 5; + if self.digits[dp] == 5 && dp + 1 == self.num_digits { + round_up = self.truncated || ((dp != 0) && (1 & self.digits[dp - 1] != 0)) + } + } + if round_up { + n += 1; + } + n + } + + /// Computes decimal * 2^shift. + pub fn left_shift(&mut self, shift: usize) { + if self.num_digits == 0 { + return; + } + let num_new_digits = number_of_digits_decimal_left_shift(self, shift); + let mut read_index = self.num_digits; + let mut write_index = self.num_digits + num_new_digits; + let mut n = 0_u64; + while read_index != 0 { + read_index -= 1; + write_index -= 1; + n += (self.digits[read_index] as u64) << shift; + let quotient = n / 10; + let remainder = n - (10 * quotient); + if write_index < Self::MAX_DIGITS { + self.digits[write_index] = remainder as u8; + } else if remainder > 0 { + self.truncated = true; + } + n = quotient; + } + while n > 0 { + write_index -= 1; + let quotient = n / 10; + let remainder = n - (10 * quotient); + if write_index < Self::MAX_DIGITS { + self.digits[write_index] = remainder as u8; + } else if remainder > 0 { + self.truncated = true; + } + n = quotient; + } + self.num_digits += num_new_digits; + if self.num_digits > Self::MAX_DIGITS { + self.num_digits = Self::MAX_DIGITS; + } + self.decimal_point += num_new_digits as i32; + self.trim(); + } + + /// Computes decimal * 2^-shift. + pub fn right_shift(&mut self, shift: usize) { + let mut read_index = 0; + let mut write_index = 0; + let mut n = 0_u64; + while (n >> shift) == 0 { + if read_index < self.num_digits { + n = (10 * n) + self.digits[read_index] as u64; + read_index += 1; + } else if n == 0 { + return; + } else { + while (n >> shift) == 0 { + n *= 10; + read_index += 1; + } + break; + } + } + self.decimal_point -= read_index as i32 - 1; + if self.decimal_point < -Self::DECIMAL_POINT_RANGE { + // `self = Self::Default()`, but without the overhead of clearing `digits`. + self.num_digits = 0; + self.decimal_point = 0; + self.truncated = false; + return; + } + let mask = (1_u64 << shift) - 1; + while read_index < self.num_digits { + let new_digit = (n >> shift) as u8; + n = (10 * (n & mask)) + self.digits[read_index] as u64; + read_index += 1; + self.digits[write_index] = new_digit; + write_index += 1; + } + while n > 0 { + let new_digit = (n >> shift) as u8; + n = 10 * (n & mask); + if write_index < Self::MAX_DIGITS { + self.digits[write_index] = new_digit; + write_index += 1; + } else if new_digit > 0 { + self.truncated = true; + } + } + self.num_digits = write_index; + self.trim(); + } +} + +/// Parse a big integer representation of the float as a decimal. +pub fn parse_decimal(mut s: &[u8]) -> Decimal { + let mut d = Decimal::default(); + let start = s; + s = s.skip_chars(b'0'); + parse_digits(&mut s, |digit| d.try_add_digit(digit)); + if s.first_is(b'.') { + s = s.advance(1); + let first = s; + // Skip leading zeros. + if d.num_digits == 0 { + s = s.skip_chars(b'0'); + } + while s.len() >= 8 && d.num_digits + 8 < Decimal::MAX_DIGITS { + // SAFETY: s is at least 8 bytes. + let v = unsafe { s.read_u64_unchecked() }; + if !is_8digits(v) { + break; + } + // SAFETY: d.num_digits + 8 is less than d.digits.len() + unsafe { + d.digits[d.num_digits..].write_u64_unchecked(v - 0x3030_3030_3030_3030); + } + d.num_digits += 8; + s = s.advance(8); + } + parse_digits(&mut s, |digit| d.try_add_digit(digit)); + d.decimal_point = s.len() as i32 - first.len() as i32; + } + if d.num_digits != 0 { + // Ignore the trailing zeros if there are any + let mut n_trailing_zeros = 0; + for &c in start[..(start.len() - s.len())].iter().rev() { + if c == b'0' { + n_trailing_zeros += 1; + } else if c != b'.' { + break; + } + } + d.decimal_point += n_trailing_zeros as i32; + d.num_digits -= n_trailing_zeros; + d.decimal_point += d.num_digits as i32; + if d.num_digits > Decimal::MAX_DIGITS { + d.truncated = true; + d.num_digits = Decimal::MAX_DIGITS; + } + } + if s.first_is2(b'e', b'E') { + s = s.advance(1); + let mut neg_exp = false; + if s.first_is(b'-') { + neg_exp = true; + s = s.advance(1); + } else if s.first_is(b'+') { + s = s.advance(1); + } + let mut exp_num = 0_i32; + parse_digits(&mut s, |digit| { + if exp_num < 0x10000 { + exp_num = 10 * exp_num + digit as i32; + } + }); + d.decimal_point += if neg_exp { -exp_num } else { exp_num }; + } + for i in d.num_digits..Decimal::MAX_DIGITS_WITHOUT_OVERFLOW { + d.digits[i] = 0; + } + d +} + +fn number_of_digits_decimal_left_shift(d: &Decimal, mut shift: usize) -> usize { + #[rustfmt::skip] + const TABLE: [u16; 65] = [ + 0x0000, 0x0800, 0x0801, 0x0803, 0x1006, 0x1009, 0x100D, 0x1812, 0x1817, 0x181D, 0x2024, + 0x202B, 0x2033, 0x203C, 0x2846, 0x2850, 0x285B, 0x3067, 0x3073, 0x3080, 0x388E, 0x389C, + 0x38AB, 0x38BB, 0x40CC, 0x40DD, 0x40EF, 0x4902, 0x4915, 0x4929, 0x513E, 0x5153, 0x5169, + 0x5180, 0x5998, 0x59B0, 0x59C9, 0x61E3, 0x61FD, 0x6218, 0x6A34, 0x6A50, 0x6A6D, 0x6A8B, + 0x72AA, 0x72C9, 0x72E9, 0x7B0A, 0x7B2B, 0x7B4D, 0x8370, 0x8393, 0x83B7, 0x83DC, 0x8C02, + 0x8C28, 0x8C4F, 0x9477, 0x949F, 0x94C8, 0x9CF2, 0x051C, 0x051C, 0x051C, 0x051C, + ]; + #[rustfmt::skip] + const TABLE_POW5: [u8; 0x051C] = [ + 5, 2, 5, 1, 2, 5, 6, 2, 5, 3, 1, 2, 5, 1, 5, 6, 2, 5, 7, 8, 1, 2, 5, 3, 9, 0, 6, 2, 5, 1, + 9, 5, 3, 1, 2, 5, 9, 7, 6, 5, 6, 2, 5, 4, 8, 8, 2, 8, 1, 2, 5, 2, 4, 4, 1, 4, 0, 6, 2, 5, + 1, 2, 2, 0, 7, 0, 3, 1, 2, 5, 6, 1, 0, 3, 5, 1, 5, 6, 2, 5, 3, 0, 5, 1, 7, 5, 7, 8, 1, 2, + 5, 1, 5, 2, 5, 8, 7, 8, 9, 0, 6, 2, 5, 7, 6, 2, 9, 3, 9, 4, 5, 3, 1, 2, 5, 3, 8, 1, 4, 6, + 9, 7, 2, 6, 5, 6, 2, 5, 1, 9, 0, 7, 3, 4, 8, 6, 3, 2, 8, 1, 2, 5, 9, 5, 3, 6, 7, 4, 3, 1, + 6, 4, 0, 6, 2, 5, 4, 7, 6, 8, 3, 7, 1, 5, 8, 2, 0, 3, 1, 2, 5, 2, 3, 8, 4, 1, 8, 5, 7, 9, + 1, 0, 1, 5, 6, 2, 5, 1, 1, 9, 2, 0, 9, 2, 8, 9, 5, 5, 0, 7, 8, 1, 2, 5, 5, 9, 6, 0, 4, 6, + 4, 4, 7, 7, 5, 3, 9, 0, 6, 2, 5, 2, 9, 8, 0, 2, 3, 2, 2, 3, 8, 7, 6, 9, 5, 3, 1, 2, 5, 1, + 4, 9, 0, 1, 1, 6, 1, 1, 9, 3, 8, 4, 7, 6, 5, 6, 2, 5, 7, 4, 5, 0, 5, 8, 0, 5, 9, 6, 9, 2, + 3, 8, 2, 8, 1, 2, 5, 3, 7, 2, 5, 2, 9, 0, 2, 9, 8, 4, 6, 1, 9, 1, 4, 0, 6, 2, 5, 1, 8, 6, + 2, 6, 4, 5, 1, 4, 9, 2, 3, 0, 9, 5, 7, 0, 3, 1, 2, 5, 9, 3, 1, 3, 2, 2, 5, 7, 4, 6, 1, 5, + 4, 7, 8, 5, 1, 5, 6, 2, 5, 4, 6, 5, 6, 6, 1, 2, 8, 7, 3, 0, 7, 7, 3, 9, 2, 5, 7, 8, 1, 2, + 5, 2, 3, 2, 8, 3, 0, 6, 4, 3, 6, 5, 3, 8, 6, 9, 6, 2, 8, 9, 0, 6, 2, 5, 1, 1, 6, 4, 1, 5, + 3, 2, 1, 8, 2, 6, 9, 3, 4, 8, 1, 4, 4, 5, 3, 1, 2, 5, 5, 8, 2, 0, 7, 6, 6, 0, 9, 1, 3, 4, + 6, 7, 4, 0, 7, 2, 2, 6, 5, 6, 2, 5, 2, 9, 1, 0, 3, 8, 3, 0, 4, 5, 6, 7, 3, 3, 7, 0, 3, 6, + 1, 3, 2, 8, 1, 2, 5, 1, 4, 5, 5, 1, 9, 1, 5, 2, 2, 8, 3, 6, 6, 8, 5, 1, 8, 0, 6, 6, 4, 0, + 6, 2, 5, 7, 2, 7, 5, 9, 5, 7, 6, 1, 4, 1, 8, 3, 4, 2, 5, 9, 0, 3, 3, 2, 0, 3, 1, 2, 5, 3, + 6, 3, 7, 9, 7, 8, 8, 0, 7, 0, 9, 1, 7, 1, 2, 9, 5, 1, 6, 6, 0, 1, 5, 6, 2, 5, 1, 8, 1, 8, + 9, 8, 9, 4, 0, 3, 5, 4, 5, 8, 5, 6, 4, 7, 5, 8, 3, 0, 0, 7, 8, 1, 2, 5, 9, 0, 9, 4, 9, 4, + 7, 0, 1, 7, 7, 2, 9, 2, 8, 2, 3, 7, 9, 1, 5, 0, 3, 9, 0, 6, 2, 5, 4, 5, 4, 7, 4, 7, 3, 5, + 0, 8, 8, 6, 4, 6, 4, 1, 1, 8, 9, 5, 7, 5, 1, 9, 5, 3, 1, 2, 5, 2, 2, 7, 3, 7, 3, 6, 7, 5, + 4, 4, 3, 2, 3, 2, 0, 5, 9, 4, 7, 8, 7, 5, 9, 7, 6, 5, 6, 2, 5, 1, 1, 3, 6, 8, 6, 8, 3, 7, + 7, 2, 1, 6, 1, 6, 0, 2, 9, 7, 3, 9, 3, 7, 9, 8, 8, 2, 8, 1, 2, 5, 5, 6, 8, 4, 3, 4, 1, 8, + 8, 6, 0, 8, 0, 8, 0, 1, 4, 8, 6, 9, 6, 8, 9, 9, 4, 1, 4, 0, 6, 2, 5, 2, 8, 4, 2, 1, 7, 0, + 9, 4, 3, 0, 4, 0, 4, 0, 0, 7, 4, 3, 4, 8, 4, 4, 9, 7, 0, 7, 0, 3, 1, 2, 5, 1, 4, 2, 1, 0, + 8, 5, 4, 7, 1, 5, 2, 0, 2, 0, 0, 3, 7, 1, 7, 4, 2, 2, 4, 8, 5, 3, 5, 1, 5, 6, 2, 5, 7, 1, + 0, 5, 4, 2, 7, 3, 5, 7, 6, 0, 1, 0, 0, 1, 8, 5, 8, 7, 1, 1, 2, 4, 2, 6, 7, 5, 7, 8, 1, 2, + 5, 3, 5, 5, 2, 7, 1, 3, 6, 7, 8, 8, 0, 0, 5, 0, 0, 9, 2, 9, 3, 5, 5, 6, 2, 1, 3, 3, 7, 8, + 9, 0, 6, 2, 5, 1, 7, 7, 6, 3, 5, 6, 8, 3, 9, 4, 0, 0, 2, 5, 0, 4, 6, 4, 6, 7, 7, 8, 1, 0, + 6, 6, 8, 9, 4, 5, 3, 1, 2, 5, 8, 8, 8, 1, 7, 8, 4, 1, 9, 7, 0, 0, 1, 2, 5, 2, 3, 2, 3, 3, + 8, 9, 0, 5, 3, 3, 4, 4, 7, 2, 6, 5, 6, 2, 5, 4, 4, 4, 0, 8, 9, 2, 0, 9, 8, 5, 0, 0, 6, 2, + 6, 1, 6, 1, 6, 9, 4, 5, 2, 6, 6, 7, 2, 3, 6, 3, 2, 8, 1, 2, 5, 2, 2, 2, 0, 4, 4, 6, 0, 4, + 9, 2, 5, 0, 3, 1, 3, 0, 8, 0, 8, 4, 7, 2, 6, 3, 3, 3, 6, 1, 8, 1, 6, 4, 0, 6, 2, 5, 1, 1, + 1, 0, 2, 2, 3, 0, 2, 4, 6, 2, 5, 1, 5, 6, 5, 4, 0, 4, 2, 3, 6, 3, 1, 6, 6, 8, 0, 9, 0, 8, + 2, 0, 3, 1, 2, 5, 5, 5, 5, 1, 1, 1, 5, 1, 2, 3, 1, 2, 5, 7, 8, 2, 7, 0, 2, 1, 1, 8, 1, 5, + 8, 3, 4, 0, 4, 5, 4, 1, 0, 1, 5, 6, 2, 5, 2, 7, 7, 5, 5, 5, 7, 5, 6, 1, 5, 6, 2, 8, 9, 1, + 3, 5, 1, 0, 5, 9, 0, 7, 9, 1, 7, 0, 2, 2, 7, 0, 5, 0, 7, 8, 1, 2, 5, 1, 3, 8, 7, 7, 7, 8, + 7, 8, 0, 7, 8, 1, 4, 4, 5, 6, 7, 5, 5, 2, 9, 5, 3, 9, 5, 8, 5, 1, 1, 3, 5, 2, 5, 3, 9, 0, + 6, 2, 5, 6, 9, 3, 8, 8, 9, 3, 9, 0, 3, 9, 0, 7, 2, 2, 8, 3, 7, 7, 6, 4, 7, 6, 9, 7, 9, 2, + 5, 5, 6, 7, 6, 2, 6, 9, 5, 3, 1, 2, 5, 3, 4, 6, 9, 4, 4, 6, 9, 5, 1, 9, 5, 3, 6, 1, 4, 1, + 8, 8, 8, 2, 3, 8, 4, 8, 9, 6, 2, 7, 8, 3, 8, 1, 3, 4, 7, 6, 5, 6, 2, 5, 1, 7, 3, 4, 7, 2, + 3, 4, 7, 5, 9, 7, 6, 8, 0, 7, 0, 9, 4, 4, 1, 1, 9, 2, 4, 4, 8, 1, 3, 9, 1, 9, 0, 6, 7, 3, + 8, 2, 8, 1, 2, 5, 8, 6, 7, 3, 6, 1, 7, 3, 7, 9, 8, 8, 4, 0, 3, 5, 4, 7, 2, 0, 5, 9, 6, 2, + 2, 4, 0, 6, 9, 5, 9, 5, 3, 3, 6, 9, 1, 4, 0, 6, 2, 5, + ]; + + shift &= 63; + let x_a = TABLE[shift]; + let x_b = TABLE[shift + 1]; + let num_new_digits = (x_a >> 11) as _; + let pow5_a = (0x7FF & x_a) as usize; + let pow5_b = (0x7FF & x_b) as usize; + let pow5 = &TABLE_POW5[pow5_a..]; + for (i, &p5) in pow5.iter().enumerate().take(pow5_b - pow5_a) { + if i >= d.num_digits { + return num_new_digits - 1; + } else if d.digits[i] == p5 { + continue; + } else if d.digits[i] < p5 { + return num_new_digits - 1; + } else { + return num_new_digits; + } + } + num_new_digits +} diff --git a/crux-mir/lib/core/src/num/dec2flt/float.rs b/crux-mir/lib/core/src/num/dec2flt/float.rs new file mode 100644 index 000000000..5921c5ed4 --- /dev/null +++ b/crux-mir/lib/core/src/num/dec2flt/float.rs @@ -0,0 +1,207 @@ +//! Helper trait for generic float types. + +use crate::fmt::{Debug, LowerExp}; +use crate::num::FpCategory; +use crate::ops::{Add, Div, Mul, Neg}; + +/// A helper trait to avoid duplicating basically all the conversion code for `f32` and `f64`. +/// +/// See the parent module's doc comment for why this is necessary. +/// +/// Should **never ever** be implemented for other types or be used outside the dec2flt module. +#[doc(hidden)] +pub trait RawFloat: + Sized + + Div + + Neg + + Mul + + Add + + LowerExp + + PartialEq + + PartialOrd + + Default + + Clone + + Copy + + Debug +{ + const INFINITY: Self; + const NEG_INFINITY: Self; + const NAN: Self; + const NEG_NAN: Self; + + /// The number of bits in the significand, *excluding* the hidden bit. + const MANTISSA_EXPLICIT_BITS: usize; + + // Round-to-even only happens for negative values of q + // when q ≥ −4 in the 64-bit case and when q ≥ −17 in + // the 32-bitcase. + // + // When q ≥ 0,we have that 5^q ≤ 2m+1. In the 64-bit case,we + // have 5^q ≤ 2m+1 ≤ 2^54 or q ≤ 23. In the 32-bit case,we have + // 5^q ≤ 2m+1 ≤ 2^25 or q ≤ 10. + // + // When q < 0, we have w ≥ (2m+1)×5^−q. We must have that w < 2^64 + // so (2m+1)×5^−q < 2^64. We have that 2m+1 > 2^53 (64-bit case) + // or 2m+1 > 2^24 (32-bit case). Hence,we must have 2^53×5^−q < 2^64 + // (64-bit) and 2^24×5^−q < 2^64 (32-bit). Hence we have 5^−q < 2^11 + // or q ≥ −4 (64-bit case) and 5^−q < 2^40 or q ≥ −17 (32-bitcase). + // + // Thus we have that we only need to round ties to even when + // we have that q ∈ [−4,23](in the 64-bit case) or q∈[−17,10] + // (in the 32-bit case). In both cases,the power of five(5^|q|) + // fits in a 64-bit word. + const MIN_EXPONENT_ROUND_TO_EVEN: i32; + const MAX_EXPONENT_ROUND_TO_EVEN: i32; + + // Minimum exponent that for a fast path case, or `-⌊(MANTISSA_EXPLICIT_BITS+1)/log2(5)⌋` + const MIN_EXPONENT_FAST_PATH: i64; + + // Maximum exponent that for a fast path case, or `⌊(MANTISSA_EXPLICIT_BITS+1)/log2(5)⌋` + const MAX_EXPONENT_FAST_PATH: i64; + + // Maximum exponent that can be represented for a disguised-fast path case. + // This is `MAX_EXPONENT_FAST_PATH + ⌊(MANTISSA_EXPLICIT_BITS+1)/log2(10)⌋` + const MAX_EXPONENT_DISGUISED_FAST_PATH: i64; + + // Minimum exponent value `-(1 << (EXP_BITS - 1)) + 1`. + const MINIMUM_EXPONENT: i32; + + // Largest exponent value `(1 << EXP_BITS) - 1`. + const INFINITE_POWER: i32; + + // Index (in bits) of the sign. + const SIGN_INDEX: usize; + + // Smallest decimal exponent for a non-zero value. + const SMALLEST_POWER_OF_TEN: i32; + + // Largest decimal exponent for a non-infinite value. + const LARGEST_POWER_OF_TEN: i32; + + // Maximum mantissa for the fast-path (`1 << 53` for f64). + const MAX_MANTISSA_FAST_PATH: u64 = 2_u64 << Self::MANTISSA_EXPLICIT_BITS; + + /// Convert integer into float through an as cast. + /// This is only called in the fast-path algorithm, and therefore + /// will not lose precision, since the value will always have + /// only if the value is <= Self::MAX_MANTISSA_FAST_PATH. + fn from_u64(v: u64) -> Self; + + /// Performs a raw transmutation from an integer. + fn from_u64_bits(v: u64) -> Self; + + /// Get a small power-of-ten for fast-path multiplication. + fn pow10_fast_path(exponent: usize) -> Self; + + /// Returns the category that this number falls into. + fn classify(self) -> FpCategory; + + /// Returns the mantissa, exponent and sign as integers. + fn integer_decode(self) -> (u64, i16, i8); +} + +impl RawFloat for f32 { + const INFINITY: Self = f32::INFINITY; + const NEG_INFINITY: Self = f32::NEG_INFINITY; + const NAN: Self = f32::NAN; + const NEG_NAN: Self = -f32::NAN; + + const MANTISSA_EXPLICIT_BITS: usize = 23; + const MIN_EXPONENT_ROUND_TO_EVEN: i32 = -17; + const MAX_EXPONENT_ROUND_TO_EVEN: i32 = 10; + const MIN_EXPONENT_FAST_PATH: i64 = -10; // assuming FLT_EVAL_METHOD = 0 + const MAX_EXPONENT_FAST_PATH: i64 = 10; + const MAX_EXPONENT_DISGUISED_FAST_PATH: i64 = 17; + const MINIMUM_EXPONENT: i32 = -127; + const INFINITE_POWER: i32 = 0xFF; + const SIGN_INDEX: usize = 31; + const SMALLEST_POWER_OF_TEN: i32 = -65; + const LARGEST_POWER_OF_TEN: i32 = 38; + + fn from_u64(v: u64) -> Self { + debug_assert!(v <= Self::MAX_MANTISSA_FAST_PATH); + v as _ + } + + fn from_u64_bits(v: u64) -> Self { + f32::from_bits((v & 0xFFFFFFFF) as u32) + } + + fn pow10_fast_path(exponent: usize) -> Self { + #[allow(clippy::use_self)] + const TABLE: [f32; 16] = + [1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 0., 0., 0., 0., 0.]; + TABLE[exponent & 15] + } + + /// Returns the mantissa, exponent and sign as integers. + fn integer_decode(self) -> (u64, i16, i8) { + let bits = self.to_bits(); + let sign: i8 = if bits >> 31 == 0 { 1 } else { -1 }; + let mut exponent: i16 = ((bits >> 23) & 0xff) as i16; + let mantissa = + if exponent == 0 { (bits & 0x7fffff) << 1 } else { (bits & 0x7fffff) | 0x800000 }; + // Exponent bias + mantissa shift + exponent -= 127 + 23; + (mantissa as u64, exponent, sign) + } + + fn classify(self) -> FpCategory { + self.classify() + } +} + +impl RawFloat for f64 { + const INFINITY: Self = f64::INFINITY; + const NEG_INFINITY: Self = f64::NEG_INFINITY; + const NAN: Self = f64::NAN; + const NEG_NAN: Self = -f64::NAN; + + const MANTISSA_EXPLICIT_BITS: usize = 52; + const MIN_EXPONENT_ROUND_TO_EVEN: i32 = -4; + const MAX_EXPONENT_ROUND_TO_EVEN: i32 = 23; + const MIN_EXPONENT_FAST_PATH: i64 = -22; // assuming FLT_EVAL_METHOD = 0 + const MAX_EXPONENT_FAST_PATH: i64 = 22; + const MAX_EXPONENT_DISGUISED_FAST_PATH: i64 = 37; + const MINIMUM_EXPONENT: i32 = -1023; + const INFINITE_POWER: i32 = 0x7FF; + const SIGN_INDEX: usize = 63; + const SMALLEST_POWER_OF_TEN: i32 = -342; + const LARGEST_POWER_OF_TEN: i32 = 308; + + fn from_u64(v: u64) -> Self { + debug_assert!(v <= Self::MAX_MANTISSA_FAST_PATH); + v as _ + } + + fn from_u64_bits(v: u64) -> Self { + f64::from_bits(v) + } + + fn pow10_fast_path(exponent: usize) -> Self { + const TABLE: [f64; 32] = [ + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, + 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22, 0., 0., 0., 0., 0., 0., 0., 0., 0., + ]; + TABLE[exponent & 31] + } + + /// Returns the mantissa, exponent and sign as integers. + fn integer_decode(self) -> (u64, i16, i8) { + let bits = self.to_bits(); + let sign: i8 = if bits >> 63 == 0 { 1 } else { -1 }; + let mut exponent: i16 = ((bits >> 52) & 0x7ff) as i16; + let mantissa = if exponent == 0 { + (bits & 0xfffffffffffff) << 1 + } else { + (bits & 0xfffffffffffff) | 0x10000000000000 + }; + // Exponent bias + mantissa shift + exponent -= 1023 + 52; + (mantissa, exponent, sign) + } + + fn classify(self) -> FpCategory { + self.classify() + } +} diff --git a/crux-mir/lib/core/src/num/dec2flt/fpu.rs b/crux-mir/lib/core/src/num/dec2flt/fpu.rs new file mode 100644 index 000000000..3806977f7 --- /dev/null +++ b/crux-mir/lib/core/src/num/dec2flt/fpu.rs @@ -0,0 +1,90 @@ +//! Platform-specific, assembly instructions to avoid +//! intermediate rounding on architectures with FPUs. + +pub use fpu_precision::set_precision; + +// On x86, the x87 FPU is used for float operations if the SSE/SSE2 extensions are not available. +// The x87 FPU operates with 80 bits of precision by default, which means that operations will +// round to 80 bits causing double rounding to happen when values are eventually represented as +// 32/64 bit float values. To overcome this, the FPU control word can be set so that the +// computations are performed in the desired precision. +#[cfg(all(target_arch = "x86", not(target_feature = "sse2")))] +mod fpu_precision { + use core::arch::asm; + use core::mem::size_of; + + /// A structure used to preserve the original value of the FPU control word, so that it can be + /// restored when the structure is dropped. + /// + /// The x87 FPU is a 16-bits register whose fields are as follows: + /// + /// | 12-15 | 10-11 | 8-9 | 6-7 | 5 | 4 | 3 | 2 | 1 | 0 | + /// |------:|------:|----:|----:|---:|---:|---:|---:|---:|---:| + /// | | RC | PC | | PM | UM | OM | ZM | DM | IM | + /// + /// The documentation for all of the fields is available in the IA-32 Architectures Software + /// Developer's Manual (Volume 1). + /// + /// The only field which is relevant for the following code is PC, Precision Control. This + /// field determines the precision of the operations performed by the FPU. It can be set to: + /// - 0b00, single precision i.e., 32-bits + /// - 0b10, double precision i.e., 64-bits + /// - 0b11, double extended precision i.e., 80-bits (default state) + /// The 0b01 value is reserved and should not be used. + pub struct FPUControlWord(u16); + + fn set_cw(cw: u16) { + // SAFETY: the `fldcw` instruction has been audited to be able to work correctly with + // any `u16` + unsafe { + asm!( + "fldcw word ptr [{}]", + in(reg) &cw, + options(nostack), + ) + } + } + + /// Sets the precision field of the FPU to `T` and returns a `FPUControlWord`. + pub fn set_precision() -> FPUControlWord { + let mut cw = 0_u16; + + // Compute the value for the Precision Control field that is appropriate for `T`. + let cw_precision = match size_of::() { + 4 => 0x0000, // 32 bits + 8 => 0x0200, // 64 bits + _ => 0x0300, // default, 80 bits + }; + + // Get the original value of the control word to restore it later, when the + // `FPUControlWord` structure is dropped + // SAFETY: the `fnstcw` instruction has been audited to be able to work correctly with + // any `u16` + unsafe { + asm!( + "fnstcw word ptr [{}]", + in(reg) &mut cw, + options(nostack), + ) + } + + // Set the control word to the desired precision. This is achieved by masking away the old + // precision (bits 8 and 9, 0x300) and replacing it with the precision flag computed above. + set_cw((cw & 0xFCFF) | cw_precision); + + FPUControlWord(cw) + } + + impl Drop for FPUControlWord { + fn drop(&mut self) { + set_cw(self.0) + } + } +} + +// In most architectures, floating point operations have an explicit bit size, therefore the +// precision of the computation is determined on a per-operation basis. +#[cfg(any(not(target_arch = "x86"), target_feature = "sse2"))] +mod fpu_precision { + pub fn set_precision() {} +} diff --git a/crux-mir/lib/core/src/num/dec2flt/lemire.rs b/crux-mir/lib/core/src/num/dec2flt/lemire.rs new file mode 100644 index 000000000..9f7594460 --- /dev/null +++ b/crux-mir/lib/core/src/num/dec2flt/lemire.rs @@ -0,0 +1,166 @@ +//! Implementation of the Eisel-Lemire algorithm. + +use crate::num::dec2flt::common::BiasedFp; +use crate::num::dec2flt::float::RawFloat; +use crate::num::dec2flt::table::{ + LARGEST_POWER_OF_FIVE, POWER_OF_FIVE_128, SMALLEST_POWER_OF_FIVE, +}; + +/// Compute w * 10^q using an extended-precision float representation. +/// +/// Fast conversion of a the significant digits and decimal exponent +/// a float to an extended representation with a binary float. This +/// algorithm will accurately parse the vast majority of cases, +/// and uses a 128-bit representation (with a fallback 192-bit +/// representation). +/// +/// This algorithm scales the exponent by the decimal exponent +/// using pre-computed powers-of-5, and calculates if the +/// representation can be unambiguously rounded to the nearest +/// machine float. Near-halfway cases are not handled here, +/// and are represented by a negative, biased binary exponent. +/// +/// The algorithm is described in detail in "Daniel Lemire, Number Parsing +/// at a Gigabyte per Second" in section 5, "Fast Algorithm", and +/// section 6, "Exact Numbers And Ties", available online: +/// . +pub fn compute_float(q: i64, mut w: u64) -> BiasedFp { + let fp_zero = BiasedFp::zero_pow2(0); + let fp_inf = BiasedFp::zero_pow2(F::INFINITE_POWER); + let fp_error = BiasedFp::zero_pow2(-1); + + // Short-circuit if the value can only be a literal 0 or infinity. + if w == 0 || q < F::SMALLEST_POWER_OF_TEN as i64 { + return fp_zero; + } else if q > F::LARGEST_POWER_OF_TEN as i64 { + return fp_inf; + } + // Normalize our significant digits, so the most-significant bit is set. + let lz = w.leading_zeros(); + w <<= lz; + let (lo, hi) = compute_product_approx(q, w, F::MANTISSA_EXPLICIT_BITS + 3); + if lo == 0xFFFF_FFFF_FFFF_FFFF { + // If we have failed to approximate w x 5^-q with our 128-bit value. + // Since the addition of 1 could lead to an overflow which could then + // round up over the half-way point, this can lead to improper rounding + // of a float. + // + // However, this can only occur if q ∈ [-27, 55]. The upper bound of q + // is 55 because 5^55 < 2^128, however, this can only happen if 5^q > 2^64, + // since otherwise the product can be represented in 64-bits, producing + // an exact result. For negative exponents, rounding-to-even can + // only occur if 5^-q < 2^64. + // + // For detailed explanations of rounding for negative exponents, see + // . For detailed + // explanations of rounding for positive exponents, see + // . + let inside_safe_exponent = (q >= -27) && (q <= 55); + if !inside_safe_exponent { + return fp_error; + } + } + let upperbit = (hi >> 63) as i32; + let mut mantissa = hi >> (upperbit + 64 - F::MANTISSA_EXPLICIT_BITS as i32 - 3); + let mut power2 = power(q as i32) + upperbit - lz as i32 - F::MINIMUM_EXPONENT; + if power2 <= 0 { + if -power2 + 1 >= 64 { + // Have more than 64 bits below the minimum exponent, must be 0. + return fp_zero; + } + // Have a subnormal value. + mantissa >>= -power2 + 1; + mantissa += mantissa & 1; + mantissa >>= 1; + power2 = (mantissa >= (1_u64 << F::MANTISSA_EXPLICIT_BITS)) as i32; + return BiasedFp { f: mantissa, e: power2 }; + } + // Need to handle rounding ties. Normally, we need to round up, + // but if we fall right in between and we have an even basis, we + // need to round down. + // + // This will only occur if: + // 1. The lower 64 bits of the 128-bit representation is 0. + // IE, 5^q fits in single 64-bit word. + // 2. The least-significant bit prior to truncated mantissa is odd. + // 3. All the bits truncated when shifting to mantissa bits + 1 are 0. + // + // Or, we may fall between two floats: we are exactly halfway. + if lo <= 1 + && q >= F::MIN_EXPONENT_ROUND_TO_EVEN as i64 + && q <= F::MAX_EXPONENT_ROUND_TO_EVEN as i64 + && mantissa & 3 == 1 + && (mantissa << (upperbit + 64 - F::MANTISSA_EXPLICIT_BITS as i32 - 3)) == hi + { + // Zero the lowest bit, so we don't round up. + mantissa &= !1_u64; + } + // Round-to-even, then shift the significant digits into place. + mantissa += mantissa & 1; + mantissa >>= 1; + if mantissa >= (2_u64 << F::MANTISSA_EXPLICIT_BITS) { + // Rounding up overflowed, so the carry bit is set. Set the + // mantissa to 1 (only the implicit, hidden bit is set) and + // increase the exponent. + mantissa = 1_u64 << F::MANTISSA_EXPLICIT_BITS; + power2 += 1; + } + // Zero out the hidden bit. + mantissa &= !(1_u64 << F::MANTISSA_EXPLICIT_BITS); + if power2 >= F::INFINITE_POWER { + // Exponent is above largest normal value, must be infinite. + return fp_inf; + } + BiasedFp { f: mantissa, e: power2 } +} + +/// Calculate a base 2 exponent from a decimal exponent. +/// This uses a pre-computed integer approximation for +/// log2(10), where 217706 / 2^16 is accurate for the +/// entire range of non-finite decimal exponents. +fn power(q: i32) -> i32 { + (q.wrapping_mul(152_170 + 65536) >> 16) + 63 +} + +fn full_multiplication(a: u64, b: u64) -> (u64, u64) { + let r = (a as u128) * (b as u128); + (r as u64, (r >> 64) as u64) +} + +// This will compute or rather approximate w * 5**q and return a pair of 64-bit words +// approximating the result, with the "high" part corresponding to the most significant +// bits and the low part corresponding to the least significant bits. +fn compute_product_approx(q: i64, w: u64, precision: usize) -> (u64, u64) { + debug_assert!(q >= SMALLEST_POWER_OF_FIVE as i64); + debug_assert!(q <= LARGEST_POWER_OF_FIVE as i64); + debug_assert!(precision <= 64); + + let mask = if precision < 64 { + 0xFFFF_FFFF_FFFF_FFFF_u64 >> precision + } else { + 0xFFFF_FFFF_FFFF_FFFF_u64 + }; + + // 5^q < 2^64, then the multiplication always provides an exact value. + // That means whenever we need to round ties to even, we always have + // an exact value. + let index = (q - SMALLEST_POWER_OF_FIVE as i64) as usize; + let (lo5, hi5) = POWER_OF_FIVE_128[index]; + // Only need one multiplication as long as there is 1 zero but + // in the explicit mantissa bits, +1 for the hidden bit, +1 to + // determine the rounding direction, +1 for if the computed + // product has a leading zero. + let (mut first_lo, mut first_hi) = full_multiplication(w, lo5); + if first_hi & mask == mask { + // Need to do a second multiplication to get better precision + // for the lower product. This will always be exact + // where q is < 55, since 5^55 < 2^128. If this wraps, + // then we need to need to round up the hi product. + let (_, second_hi) = full_multiplication(w, hi5); + first_lo = first_lo.wrapping_add(second_hi); + if second_hi > first_lo { + first_hi += 1; + } + } + (first_lo, first_hi) +} diff --git a/crux-mir/lib/core/src/num/dec2flt/mod.rs b/crux-mir/lib/core/src/num/dec2flt/mod.rs index c83c6b0ec..a888ced49 100644 --- a/crux-mir/lib/core/src/num/dec2flt/mod.rs +++ b/crux-mir/lib/core/src/num/dec2flt/mod.rs @@ -3,7 +3,7 @@ //! # Problem statement //! //! We are given a decimal string such as `12.34e56`. This string consists of integral (`12`), -//! fractional (`45`), and exponent (`56`) parts. All parts are optional and interpreted as zero +//! fractional (`34`), and exponent (`56`) parts. All parts are optional and interpreted as zero //! when missing. //! //! We seek the IEEE 754 floating point number that is closest to the exact value of the decimal @@ -27,20 +27,12 @@ //! //! We then try a long chain of progressively more general and expensive special cases using //! machine-sized integers and small, fixed-sized floating point numbers (first `f32`/`f64`, then -//! a type with 64 bit significand, `Fp`). When all these fail, we bite the bullet and resort to a -//! simple but very slow algorithm that involved computing `f * 10^e` fully and doing an iterative -//! search for the best approximation. -//! -//! Primarily, this module and its children implement the algorithms described in: -//! "How to Read Floating Point Numbers Accurately" by William D. Clinger, -//! available online: http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.45.4152 -//! -//! In addition, there are numerous helper functions that are used in the paper but not available -//! in Rust (or at least in core). Our version is additionally complicated by the need to handle -//! overflow and underflow and the desire to handle subnormal numbers. Bellerophon and -//! Algorithm R have trouble with overflow, subnormals, and underflow. We conservatively switch to -//! Algorithm M (with the modifications described in section 8 of the paper) well before the -//! inputs get into the critical region. +//! a type with 64 bit significand). The extended-precision algorithm +//! uses the Eisel-Lemire algorithm, which uses a 128-bit (or 192-bit) +//! representation that can accurately and quickly compute the vast majority +//! of floats. When all these fail, we bite the bullet and resort to using +//! a large-decimal representation, shifting the digits into range, calculating +//! the upper significant bits and exactly round to the nearest representation. //! //! Another aspect that needs attention is the ``RawFloat`` trait by which almost all functions //! are parametrized. One might think that it's enough to parse to `f64` and cast the result to @@ -54,10 +46,9 @@ //! operations as well, if you want 0.5 ULP accuracy you need to do *everything* in full precision //! and round *exactly once, at the end*, by considering all truncated bits at once. //! -//! FIXME: Although some code duplication is necessary, perhaps parts of the code could be shuffled -//! around such that less code is duplicated. Large parts of the algorithms are independent of the -//! float type to output, or only needs access to a few constants, which could be passed in as -//! parameters. +//! Primarily, this module and its children implement the algorithms described in: +//! "Number Parsing at a Gigabyte per Second", available online: +//! . //! //! # Other //! @@ -87,16 +78,22 @@ use crate::fmt; use crate::str::FromStr; -use self::num::digits_to_big; -use self::parse::{parse_decimal, Decimal, ParseResult, Sign}; -use self::rawfp::RawFloat; +use self::common::{BiasedFp, ByteSlice}; +use self::float::RawFloat; +use self::lemire::compute_float; +use self::parse::{parse_inf_nan, parse_number}; +use self::slow::parse_long_mantissa; -mod algorithm; -mod num; +mod common; +mod decimal; +mod fpu; +mod slow; mod table; -// These two have their own tests. +// float is used in flt2dec, and all are used in unit tests. +pub mod float; +pub mod lemire; +pub mod number; pub mod parse; -pub mod rawfp; macro_rules! from_str_float_impl { ($t:ty) => { @@ -115,34 +112,29 @@ macro_rules! from_str_float_impl { /// * '2.5E-10' /// * '5.' /// * '.5', or, equivalently, '0.5' - /// * 'inf', '-inf', 'NaN' + /// * 'inf', '-inf', '+infinity', 'NaN' + /// + /// Note that alphabetical characters are not case-sensitive. /// /// Leading and trailing whitespace represent an error. /// /// # Grammar /// - /// All strings that adhere to the following [EBNF] grammar - /// will result in an [`Ok`] being returned: + /// All strings that adhere to the following [EBNF] grammar when + /// lowercased will result in an [`Ok`] being returned: /// /// ```txt - /// Float ::= Sign? ( 'inf' | 'NaN' | Number ) + /// Float ::= Sign? ( 'inf' | 'infinity' | 'nan' | Number ) /// Number ::= ( Digit+ | /// Digit+ '.' Digit* | /// Digit* '.' Digit+ ) Exp? - /// Exp ::= [eE] Sign? Digit+ + /// Exp ::= 'e' Sign? Digit+ /// Sign ::= [+-] /// Digit ::= [0-9] /// ``` /// /// [EBNF]: https://www.w3.org/TR/REC-xml/#sec-notation /// - /// # Known bugs - /// - /// In some situations, some strings that should create a valid float - /// instead return an error. See [issue #31407] for details. - /// - /// [issue #31407]: https://github.com/rust-lang/rust/issues/31407 - /// /// # Arguments /// /// * src - A string @@ -150,8 +142,10 @@ macro_rules! from_str_float_impl { /// # Return value /// /// `Err(ParseFloatError)` if the string did not represent a valid - /// number. Otherwise, `Ok(n)` where `n` is the floating-point - /// number represented by `src`. + /// number. Otherwise, `Ok(n)` where `n` is the closest + /// representable floating-point number to the number represented + /// by `src` (following the same rules for rounding as for the + /// results of primitive operations). #[inline] fn from_str(src: &str) -> Result { dec2flt(src) @@ -167,9 +161,15 @@ from_str_float_impl!(f64); /// This error is used as the error type for the [`FromStr`] implementation /// for [`f32`] and [`f64`]. /// -/// [`FromStr`]: ../str/trait.FromStr.html -/// [`f32`]: ../../std/primitive.f32.html -/// [`f64`]: ../../std/primitive.f64.html +/// # Example +/// +/// ``` +/// use std::str::FromStr; +/// +/// if let Err(e) = f64::from_str("a.12") { +/// println!("Failed conversion to f64: {e}"); +/// } +/// ``` #[derive(Debug, Clone, PartialEq, Eq)] #[stable(feature = "rust1", since = "1.0.0")] pub struct ParseFloatError { @@ -205,146 +205,65 @@ impl fmt::Display for ParseFloatError { } } -fn pfe_empty() -> ParseFloatError { +pub(super) fn pfe_empty() -> ParseFloatError { ParseFloatError { kind: FloatErrorKind::Empty } } -fn pfe_invalid() -> ParseFloatError { +// Used in unit tests, keep public. +// This is much better than making FloatErrorKind and ParseFloatError::kind public. +pub fn pfe_invalid() -> ParseFloatError { ParseFloatError { kind: FloatErrorKind::Invalid } } -/// Splits a decimal string into sign and the rest, without inspecting or validating the rest. -fn extract_sign(s: &str) -> (Sign, &str) { - match s.as_bytes()[0] { - b'+' => (Sign::Positive, &s[1..]), - b'-' => (Sign::Negative, &s[1..]), - // If the string is invalid, we never use the sign, so we don't need to validate here. - _ => (Sign::Positive, s), - } +/// Converts a `BiasedFp` to the closest machine float type. +fn biased_fp_to_float(x: BiasedFp) -> T { + let mut word = x.f; + word |= (x.e as u64) << T::MANTISSA_EXPLICIT_BITS; + T::from_u64_bits(word) } /// Converts a decimal string into a floating point number. -fn dec2flt(s: &str) -> Result { - if s.is_empty() { +pub fn dec2flt(s: &str) -> Result { + let mut s = s.as_bytes(); + let c = if let Some(&c) = s.first() { + c + } else { return Err(pfe_empty()); - } - let (sign, s) = extract_sign(s); - let flt = match parse_decimal(s) { - ParseResult::Valid(decimal) => convert(decimal)?, - ParseResult::ShortcutToInf => T::INFINITY, - ParseResult::ShortcutToZero => T::ZERO, - ParseResult::Invalid => match s { - "inf" => T::INFINITY, - "NaN" => T::NAN, - _ => { - return Err(pfe_invalid()); - } - }, }; - - match sign { - Sign::Positive => Ok(flt), - Sign::Negative => Ok(-flt), - } -} - -/// The main workhorse for the decimal-to-float conversion: Orchestrate all the preprocessing -/// and figure out which algorithm should do the actual conversion. -fn convert(mut decimal: Decimal<'_>) -> Result { - simplify(&mut decimal); - if let Some(x) = trivial_cases(&decimal) { - return Ok(x); + let negative = c == b'-'; + if c == b'-' || c == b'+' { + s = s.advance(1); } - // Remove/shift out the decimal point. - let e = decimal.exp - decimal.fractional.len() as i64; - if let Some(x) = algorithm::fast_path(decimal.integral, decimal.fractional, e) { - return Ok(x); - } - // Big32x40 is limited to 1280 bits, which translates to about 385 decimal digits. - // If we exceed this, we'll crash, so we error out before getting too close (within 10^10). - let upper_bound = bound_intermediate_digits(&decimal, e); - if upper_bound > 375 { + if s.is_empty() { return Err(pfe_invalid()); } - let f = digits_to_big(decimal.integral, decimal.fractional); - // Now the exponent certainly fits in 16 bit, which is used throughout the main algorithms. - let e = e as i16; - // FIXME These bounds are rather conservative. A more careful analysis of the failure modes - // of Bellerophon could allow using it in more cases for a massive speed up. - let exponent_in_range = table::MIN_E <= e && e <= table::MAX_E; - let value_in_range = upper_bound <= T::MAX_NORMAL_DIGITS as u64; - if exponent_in_range && value_in_range { - Ok(algorithm::bellerophon(&f, e)) - } else { - Ok(algorithm::algorithm_m(&f, e)) + let num = match parse_number(s, negative) { + Some(r) => r, + None if let Some(value) = parse_inf_nan(s, negative) => return Ok(value), + None => return Err(pfe_invalid()), + }; + if let Some(value) = num.try_fast_path::() { + return Ok(value); } -} - -// As written, this optimizes badly (see #27130, though it refers to an old version of the code). -// `inline(always)` is a workaround for that. There are only two call sites overall and it doesn't -// make code size worse. -/// Strip zeros where possible, even when this requires changing the exponent -#[inline(always)] -fn simplify(decimal: &mut Decimal<'_>) { - let is_zero = &|&&d: &&u8| -> bool { d == b'0' }; - // Trimming these zeros does not change anything but may enable the fast path (< 15 digits). - let leading_zeros = decimal.integral.iter().take_while(is_zero).count(); - decimal.integral = &decimal.integral[leading_zeros..]; - let trailing_zeros = decimal.fractional.iter().rev().take_while(is_zero).count(); - let end = decimal.fractional.len() - trailing_zeros; - decimal.fractional = &decimal.fractional[..end]; - // Simplify numbers of the form 0.0...x and x...0.0, adjusting the exponent accordingly. - // This may not always be a win (possibly pushes some numbers out of the fast path), but it - // simplifies other parts significantly (notably, approximating the magnitude of the value). - if decimal.integral.is_empty() { - let leading_zeros = decimal.fractional.iter().take_while(is_zero).count(); - decimal.fractional = &decimal.fractional[leading_zeros..]; - decimal.exp -= leading_zeros as i64; - } else if decimal.fractional.is_empty() { - let trailing_zeros = decimal.integral.iter().rev().take_while(is_zero).count(); - let end = decimal.integral.len() - trailing_zeros; - decimal.integral = &decimal.integral[..end]; - decimal.exp += trailing_zeros as i64; + // If significant digits were truncated, then we can have rounding error + // only if `mantissa + 1` produces a different result. We also avoid + // redundantly using the Eisel-Lemire algorithm if it was unable to + // correctly round on the first pass. + let mut fp = compute_float::(num.exponent, num.mantissa); + if num.many_digits && fp.e >= 0 && fp != compute_float::(num.exponent, num.mantissa + 1) { + fp.e = -1; } -} - -/// Returns a quick-an-dirty upper bound on the size (log10) of the largest value that Algorithm R -/// and Algorithm M will compute while working on the given decimal. -fn bound_intermediate_digits(decimal: &Decimal<'_>, e: i64) -> u64 { - // We don't need to worry too much about overflow here thanks to trivial_cases() and the - // parser, which filter out the most extreme inputs for us. - let f_len: u64 = decimal.integral.len() as u64 + decimal.fractional.len() as u64; - if e >= 0 { - // In the case e >= 0, both algorithms compute about `f * 10^e`. Algorithm R proceeds to - // do some complicated calculations with this but we can ignore that for the upper bound - // because it also reduces the fraction beforehand, so we have plenty of buffer there. - f_len + (e as u64) - } else { - // If e < 0, Algorithm R does roughly the same thing, but Algorithm M differs: - // It tries to find a positive number k such that `f << k / 10^e` is an in-range - // significand. This will result in about `2^53 * f * 10^e` < `10^17 * f * 10^e`. - // One input that triggers this is 0.33...33 (375 x 3). - f_len + (e.abs() as u64) + 17 + // Unable to correctly round the float using the Eisel-Lemire algorithm. + // Fallback to a slower, but always correct algorithm. + if fp.e < 0 { + fp = parse_long_mantissa::(s); } -} -/// Detects obvious overflows and underflows without even looking at the decimal digits. -fn trivial_cases(decimal: &Decimal<'_>) -> Option { - // There were zeros but they were stripped by simplify() - if decimal.integral.is_empty() && decimal.fractional.is_empty() { - return Some(T::ZERO); - } - // This is a crude approximation of ceil(log10(the real value)). We don't need to worry too - // much about overflow here because the input length is tiny (at least compared to 2^64) and - // the parser already handles exponents whose absolute value is greater than 10^18 - // (which is still 10^19 short of 2^64). - let max_place = decimal.exp + decimal.integral.len() as i64; - if max_place > T::INF_CUTOFF { - return Some(T::INFINITY); - } else if max_place < T::ZERO_CUTOFF { - return Some(T::ZERO); + let mut float = biased_fp_to_float::(fp); + if num.negative { + float = -float; } - None + Ok(float) } diff --git a/crux-mir/lib/core/src/num/dec2flt/num.rs b/crux-mir/lib/core/src/num/dec2flt/num.rs deleted file mode 100644 index 208783dd3..000000000 --- a/crux-mir/lib/core/src/num/dec2flt/num.rs +++ /dev/null @@ -1,81 +0,0 @@ -//! Utility functions for bignums that don't make too much sense to turn into methods. - -// FIXME This module's name is a bit unfortunate, since other modules also import `core::num`. - -use crate::cmp::Ordering::{self, Equal, Greater, Less}; - -pub use crate::num::bignum::Big32x40 as Big; - -/// Test whether truncating all bits less significant than `ones_place` introduces -/// a relative error less, equal, or greater than 0.5 ULP. -pub fn compare_with_half_ulp(f: &Big, ones_place: usize) -> Ordering { - if ones_place == 0 { - return Less; - } - let half_bit = ones_place - 1; - if f.get_bit(half_bit) == 0 { - // < 0.5 ULP - return Less; - } - // If all remaining bits are zero, it's = 0.5 ULP, otherwise > 0.5 - // If there are no more bits (half_bit == 0), the below also correctly returns Equal. - for i in 0..half_bit { - if f.get_bit(i) == 1 { - return Greater; - } - } - Equal -} - -/// Converts an ASCII string containing only decimal digits to a `u64`. -/// -/// Does not perform checks for overflow or invalid characters, so if the caller is not careful, -/// the result is bogus and can panic (though it won't be `unsafe`). Additionally, empty strings -/// are treated as zero. This function exists because -/// -/// 1. using `FromStr` on `&[u8]` requires `from_utf8_unchecked`, which is bad, and -/// 2. piecing together the results of `integral.parse()` and `fractional.parse()` is -/// more complicated than this entire function. -pub fn from_str_unchecked<'a, T>(bytes: T) -> u64 -where - T: IntoIterator, -{ - let mut result = 0; - for &c in bytes { - result = result * 10 + (c - b'0') as u64; - } - result -} - -/// Converts a string of ASCII digits into a bignum. -/// -/// Like `from_str_unchecked`, this function relies on the parser to weed out non-digits. -pub fn digits_to_big(integral: &[u8], fractional: &[u8]) -> Big { - let mut f = Big::from_small(0); - for &c in integral.iter().chain(fractional) { - let n = (c - b'0') as u32; - f.mul_small(10); - f.add_small(n); - } - f -} - -/// Unwraps a bignum into a 64 bit integer. Panics if the number is too large. -pub fn to_u64(x: &Big) -> u64 { - assert!(x.bit_length() < 64); - let d = x.digits(); - if d.len() < 2 { d[0] as u64 } else { (d[1] as u64) << 32 | d[0] as u64 } -} - -/// Extracts a range of bits. - -/// Index 0 is the least significant bit and the range is half-open as usual. -/// Panics if asked to extract more bits than fit into the return type. -pub fn get_bits(x: &Big, start: usize, end: usize) -> u64 { - assert!(end - start <= 64); - let mut result: u64 = 0; - for i in (start..end).rev() { - result = result << 1 | x.get_bit(i) as u64; - } - result -} diff --git a/crux-mir/lib/core/src/num/dec2flt/number.rs b/crux-mir/lib/core/src/num/dec2flt/number.rs new file mode 100644 index 000000000..405f7e7b6 --- /dev/null +++ b/crux-mir/lib/core/src/num/dec2flt/number.rs @@ -0,0 +1,86 @@ +//! Representation of a float as the significant digits and exponent. + +use crate::num::dec2flt::float::RawFloat; +use crate::num::dec2flt::fpu::set_precision; + +#[rustfmt::skip] +const INT_POW10: [u64; 16] = [ + 1, + 10, + 100, + 1000, + 10000, + 100000, + 1000000, + 10000000, + 100000000, + 1000000000, + 10000000000, + 100000000000, + 1000000000000, + 10000000000000, + 100000000000000, + 1000000000000000, +]; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Number { + pub exponent: i64, + pub mantissa: u64, + pub negative: bool, + pub many_digits: bool, +} + +impl Number { + /// Detect if the float can be accurately reconstructed from native floats. + fn is_fast_path(&self) -> bool { + F::MIN_EXPONENT_FAST_PATH <= self.exponent + && self.exponent <= F::MAX_EXPONENT_DISGUISED_FAST_PATH + && self.mantissa <= F::MAX_MANTISSA_FAST_PATH + && !self.many_digits + } + + /// The fast path algorithm using machine-sized integers and floats. + /// + /// This is extracted into a separate function so that it can be attempted before constructing + /// a Decimal. This only works if both the mantissa and the exponent + /// can be exactly represented as a machine float, since IEE-754 guarantees + /// no rounding will occur. + /// + /// There is an exception: disguised fast-path cases, where we can shift + /// powers-of-10 from the exponent to the significant digits. + pub fn try_fast_path(&self) -> Option { + // The fast path crucially depends on arithmetic being rounded to the correct number of bits + // without any intermediate rounding. On x86 (without SSE or SSE2) this requires the precision + // of the x87 FPU stack to be changed so that it directly rounds to 64/32 bit. + // The `set_precision` function takes care of setting the precision on architectures which + // require setting it by changing the global state (like the control word of the x87 FPU). + let _cw = set_precision::(); + + if self.is_fast_path::() { + let mut value = if self.exponent <= F::MAX_EXPONENT_FAST_PATH { + // normal fast path + let value = F::from_u64(self.mantissa); + if self.exponent < 0 { + value / F::pow10_fast_path((-self.exponent) as _) + } else { + value * F::pow10_fast_path(self.exponent as _) + } + } else { + // disguised fast path + let shift = self.exponent - F::MAX_EXPONENT_FAST_PATH; + let mantissa = self.mantissa.checked_mul(INT_POW10[shift as usize])?; + if mantissa > F::MAX_MANTISSA_FAST_PATH { + return None; + } + F::from_u64(mantissa) * F::pow10_fast_path(F::MAX_EXPONENT_FAST_PATH as _) + }; + if self.negative { + value = -value; + } + Some(value) + } else { + None + } + } +} diff --git a/crux-mir/lib/core/src/num/dec2flt/parse.rs b/crux-mir/lib/core/src/num/dec2flt/parse.rs index 93b08bce8..1a90e0d20 100644 --- a/crux-mir/lib/core/src/num/dec2flt/parse.rs +++ b/crux-mir/lib/core/src/num/dec2flt/parse.rs @@ -1,124 +1,233 @@ -//! Validating and decomposing a decimal string of the form: -//! -//! `(digits | digits? '.'? digits?) (('e' | 'E') ('+' | '-')? digits)?` -//! -//! In other words, standard floating-point syntax, with two exceptions: No sign, and no -//! handling of "inf" and "NaN". These are handled by the driver function (super::dec2flt). -//! -//! Although recognizing valid inputs is relatively easy, this module also has to reject the -//! countless invalid variations, never panic, and perform numerous checks that the other -//! modules rely on to not panic (or overflow) in turn. -//! To make matters worse, all that happens in a single pass over the input. -//! So, be careful when modifying anything, and double-check with the other modules. -use self::ParseResult::{Invalid, ShortcutToInf, ShortcutToZero, Valid}; -use super::num; - -#[derive(Debug)] -pub enum Sign { - Positive, - Negative, +//! Functions to parse floating-point numbers. + +use crate::num::dec2flt::common::{is_8digits, AsciiStr, ByteSlice}; +use crate::num::dec2flt::float::RawFloat; +use crate::num::dec2flt::number::Number; + +const MIN_19DIGIT_INT: u64 = 100_0000_0000_0000_0000; + +/// Parse 8 digits, loaded as bytes in little-endian order. +/// +/// This uses the trick where every digit is in [0x030, 0x39], +/// and therefore can be parsed in 3 multiplications, much +/// faster than the normal 8. +/// +/// This is based off the algorithm described in "Fast numeric string to +/// int", available here: . +fn parse_8digits(mut v: u64) -> u64 { + const MASK: u64 = 0x0000_00FF_0000_00FF; + const MUL1: u64 = 0x000F_4240_0000_0064; + const MUL2: u64 = 0x0000_2710_0000_0001; + v -= 0x3030_3030_3030_3030; + v = (v * 10) + (v >> 8); // will not overflow, fits in 63 bits + let v1 = (v & MASK).wrapping_mul(MUL1); + let v2 = ((v >> 16) & MASK).wrapping_mul(MUL2); + ((v1.wrapping_add(v2) >> 32) as u32) as u64 } -#[derive(Debug, PartialEq, Eq)] -/// The interesting parts of a decimal string. -pub struct Decimal<'a> { - pub integral: &'a [u8], - pub fractional: &'a [u8], - /// The decimal exponent, guaranteed to have fewer than 18 decimal digits. - pub exp: i64, +/// Parse digits until a non-digit character is found. +fn try_parse_digits(s: &mut AsciiStr<'_>, x: &mut u64) { + // may cause overflows, to be handled later + s.parse_digits(|digit| { + *x = x.wrapping_mul(10).wrapping_add(digit as _); + }); } -impl<'a> Decimal<'a> { - pub fn new(integral: &'a [u8], fractional: &'a [u8], exp: i64) -> Decimal<'a> { - Decimal { integral, fractional, exp } +/// Parse up to 19 digits (the max that can be stored in a 64-bit integer). +fn try_parse_19digits(s: &mut AsciiStr<'_>, x: &mut u64) { + while *x < MIN_19DIGIT_INT { + if let Some(&c) = s.as_ref().first() { + let digit = c.wrapping_sub(b'0'); + if digit < 10 { + *x = (*x * 10) + digit as u64; // no overflows here + // SAFETY: cannot be empty + unsafe { + s.step(); + } + } else { + break; + } + } else { + break; + } } } -#[derive(Debug, PartialEq, Eq)] -pub enum ParseResult<'a> { - Valid(Decimal<'a>), - ShortcutToInf, - ShortcutToZero, - Invalid, +/// Try to parse 8 digits at a time, using an optimized algorithm. +fn try_parse_8digits(s: &mut AsciiStr<'_>, x: &mut u64) { + // may cause overflows, to be handled later + if let Some(v) = s.read_u64() { + if is_8digits(v) { + *x = x.wrapping_mul(1_0000_0000).wrapping_add(parse_8digits(v)); + // SAFETY: already ensured the buffer was >= 8 bytes in read_u64. + unsafe { + s.step_by(8); + } + if let Some(v) = s.read_u64() { + if is_8digits(v) { + *x = x.wrapping_mul(1_0000_0000).wrapping_add(parse_8digits(v)); + // SAFETY: already ensured the buffer was >= 8 bytes in try_read_u64. + unsafe { + s.step_by(8); + } + } + } + } + } } -/// Checks if the input string is a valid floating point number and if so, locate the integral -/// part, the fractional part, and the exponent in it. Does not handle signs. -pub fn parse_decimal(s: &str) -> ParseResult<'_> { - if s.is_empty() { - return Invalid; +/// Parse the scientific notation component of a float. +fn parse_scientific(s: &mut AsciiStr<'_>) -> Option { + let mut exponent = 0_i64; + let mut negative = false; + if let Some(&c) = s.as_ref().get(0) { + negative = c == b'-'; + if c == b'-' || c == b'+' { + // SAFETY: s cannot be empty + unsafe { + s.step(); + } + } + } + if s.first_isdigit() { + s.parse_digits(|digit| { + // no overflows here, saturate well before overflow + if exponent < 0x10000 { + exponent = 10 * exponent + digit as i64; + } + }); + if negative { Some(-exponent) } else { Some(exponent) } + } else { + None } +} - let s = s.as_bytes(); - let (integral, s) = eat_digits(s); +/// Parse a partial, non-special floating point number. +/// +/// This creates a representation of the float as the +/// significant digits and the decimal exponent. +fn parse_partial_number(s: &[u8], negative: bool) -> Option<(Number, usize)> { + let mut s = AsciiStr::new(s); + let start = s; + debug_assert!(!s.is_empty()); - match s.first() { - None => Valid(Decimal::new(integral, b"", 0)), - Some(&b'e') | Some(&b'E') => { - if integral.is_empty() { - return Invalid; // No digits before 'e' - } + // parse initial digits before dot + let mut mantissa = 0_u64; + let digits_start = s; + try_parse_digits(&mut s, &mut mantissa); + let mut n_digits = s.offset_from(&digits_start); + + // handle dot with the following digits + let mut n_after_dot = 0; + let mut exponent = 0_i64; + let int_end = s; + if s.first_is(b'.') { + // SAFETY: s cannot be empty due to first_is + unsafe { s.step() }; + let before = s; + try_parse_8digits(&mut s, &mut mantissa); + try_parse_digits(&mut s, &mut mantissa); + n_after_dot = s.offset_from(&before); + exponent = -n_after_dot as i64; + } - parse_exp(integral, b"", &s[1..]) + n_digits += n_after_dot; + if n_digits == 0 { + return None; + } + + // handle scientific format + let mut exp_number = 0_i64; + if s.first_is2(b'e', b'E') { + // SAFETY: s cannot be empty + unsafe { + s.step(); } - Some(&b'.') => { - let (fractional, s) = eat_digits(&s[1..]); - if integral.is_empty() && fractional.is_empty() { - // We require at least a single digit before or after the point. - return Invalid; - } + // If None, we have no trailing digits after exponent, or an invalid float. + exp_number = parse_scientific(&mut s)?; + exponent += exp_number; + } - match s.first() { - None => Valid(Decimal::new(integral, fractional, 0)), - Some(&b'e') | Some(&b'E') => parse_exp(integral, fractional, &s[1..]), - _ => Invalid, // Trailing junk after fractional part - } + let len = s.offset_from(&start) as _; + + // handle uncommon case with many digits + if n_digits <= 19 { + return Some((Number { exponent, mantissa, negative, many_digits: false }, len)); + } + + n_digits -= 19; + let mut many_digits = false; + let mut p = digits_start; + while p.first_is2(b'0', b'.') { + // SAFETY: p cannot be empty due to first_is2 + unsafe { + // '0' = b'.' + 2 + n_digits -= p.first_unchecked().saturating_sub(b'0' - 1) as isize; + p.step(); } - _ => Invalid, // Trailing junk after first digit string } + if n_digits > 0 { + // at this point we have more than 19 significant digits, let's try again + many_digits = true; + mantissa = 0; + let mut s = digits_start; + try_parse_19digits(&mut s, &mut mantissa); + exponent = if mantissa >= MIN_19DIGIT_INT { + // big int + int_end.offset_from(&s) + } else { + // SAFETY: the next byte must be present and be '.' + // We know this is true because we had more than 19 + // digits previously, so we overflowed a 64-bit integer, + // but parsing only the integral digits produced less + // than 19 digits. That means we must have a decimal + // point, and at least 1 fractional digit. + unsafe { s.step() }; + let before = s; + try_parse_19digits(&mut s, &mut mantissa); + -s.offset_from(&before) + } as i64; + // add back the explicit part + exponent += exp_number; + } + + Some((Number { exponent, mantissa, negative, many_digits }, len)) } -/// Carves off decimal digits up to the first non-digit character. -fn eat_digits(s: &[u8]) -> (&[u8], &[u8]) { - let mut i = 0; - while i < s.len() && b'0' <= s[i] && s[i] <= b'9' { - i += 1; +/// Try to parse a non-special floating point number. +pub fn parse_number(s: &[u8], negative: bool) -> Option { + if let Some((float, rest)) = parse_partial_number(s, negative) { + if rest == s.len() { + return Some(float); + } } - (&s[..i], &s[i..]) + None } -/// Exponent extraction and error checking. -fn parse_exp<'a>(integral: &'a [u8], fractional: &'a [u8], rest: &'a [u8]) -> ParseResult<'a> { - let (sign, rest) = match rest.first() { - Some(&b'-') => (Sign::Negative, &rest[1..]), - Some(&b'+') => (Sign::Positive, &rest[1..]), - _ => (Sign::Positive, rest), - }; - let (mut number, trailing) = eat_digits(rest); - if !trailing.is_empty() { - return Invalid; // Trailing junk after exponent +/// Parse a partial representation of a special, non-finite float. +fn parse_partial_inf_nan(s: &[u8]) -> Option<(F, usize)> { + fn parse_inf_rest(s: &[u8]) -> usize { + if s.len() >= 8 && s[3..].as_ref().starts_with_ignore_case(b"inity") { 8 } else { 3 } } - if number.is_empty() { - return Invalid; // Empty exponent - } - // At this point, we certainly have a valid string of digits. It may be too long to put into - // an `i64`, but if it's that huge, the input is certainly zero or infinity. Since each zero - // in the decimal digits only adjusts the exponent by +/- 1, at exp = 10^18 the input would - // have to be 17 exabyte (!) of zeros to get even remotely close to being finite. - // This is not exactly a use case we need to cater to. - while number.first() == Some(&b'0') { - number = &number[1..]; + if s.len() >= 3 { + if s.starts_with_ignore_case(b"nan") { + return Some((F::NAN, 3)); + } else if s.starts_with_ignore_case(b"inf") { + return Some((F::INFINITY, parse_inf_rest(s))); + } } - if number.len() >= 18 { - return match sign { - Sign::Positive => ShortcutToInf, - Sign::Negative => ShortcutToZero, - }; + None +} + +/// Try to parse a special, non-finite float. +pub fn parse_inf_nan(s: &[u8], negative: bool) -> Option { + if let Some((mut float, rest)) = parse_partial_inf_nan::(s) { + if rest == s.len() { + if negative { + float = -float; + } + return Some(float); + } } - let abs_exp = num::from_str_unchecked(number); - let e = match sign { - Sign::Positive => abs_exp as i64, - Sign::Negative => -(abs_exp as i64), - }; - Valid(Decimal::new(integral, fractional, e)) + None } diff --git a/crux-mir/lib/core/src/num/dec2flt/rawfp.rs b/crux-mir/lib/core/src/num/dec2flt/rawfp.rs deleted file mode 100644 index 0ab15b23e..000000000 --- a/crux-mir/lib/core/src/num/dec2flt/rawfp.rs +++ /dev/null @@ -1,363 +0,0 @@ -//! Bit fiddling on positive IEEE 754 floats. Negative numbers aren't and needn't be handled. -//! Normal floating point numbers have a canonical representation as (frac, exp) such that the -//! value is 2exp * (1 + sum(frac[N-i] / 2i)) where N is the number of bits. -//! Subnormals are slightly different and weird, but the same principle applies. -//! -//! Here, however, we represent them as (sig, k) with f positive, such that the value is f * -//! 2e. Besides making the "hidden bit" explicit, this changes the exponent by the -//! so-called mantissa shift. -//! -//! Put another way, normally floats are written as (1) but here they are written as (2): -//! -//! 1. `1.101100...11 * 2^m` -//! 2. `1101100...11 * 2^n` -//! -//! We call (1) the **fractional representation** and (2) the **integral representation**. -//! -//! Many functions in this module only handle normal numbers. The dec2flt routines conservatively -//! take the universally-correct slow path (Algorithm M) for very small and very large numbers. -//! That algorithm needs only next_float() which does handle subnormals and zeros. -use crate::cmp::Ordering::{Equal, Greater, Less}; -use crate::convert::{TryFrom, TryInto}; -use crate::fmt::{Debug, LowerExp}; -use crate::num::dec2flt::num::{self, Big}; -use crate::num::dec2flt::table; -use crate::num::diy_float::Fp; -use crate::num::FpCategory; -use crate::num::FpCategory::{Infinite, Nan, Normal, Subnormal, Zero}; -use crate::ops::{Add, Div, Mul, Neg}; - -#[derive(Copy, Clone, Debug)] -pub struct Unpacked { - pub sig: u64, - pub k: i16, -} - -impl Unpacked { - pub fn new(sig: u64, k: i16) -> Self { - Unpacked { sig, k } - } -} - -/// A helper trait to avoid duplicating basically all the conversion code for `f32` and `f64`. -/// -/// See the parent module's doc comment for why this is necessary. -/// -/// Should **never ever** be implemented for other types or be used outside the dec2flt module. -pub trait RawFloat: - Copy + Debug + LowerExp + Mul + Div + Neg -{ - const INFINITY: Self; - const NAN: Self; - const ZERO: Self; - - /// Type used by `to_bits` and `from_bits`. - type Bits: Add + From + TryFrom; - - /// Performs a raw transmutation to an integer. - fn to_bits(self) -> Self::Bits; - - /// Performs a raw transmutation from an integer. - fn from_bits(v: Self::Bits) -> Self; - - /// Returns the category that this number falls into. - fn classify(self) -> FpCategory; - - /// Returns the mantissa, exponent and sign as integers. - fn integer_decode(self) -> (u64, i16, i8); - - /// Decodes the float. - fn unpack(self) -> Unpacked; - - /// Casts from a small integer that can be represented exactly. Panic if the integer can't be - /// represented, the other code in this module makes sure to never let that happen. - fn from_int(x: u64) -> Self; - - /// Gets the value 10e from a pre-computed table. - /// Panics for `e >= CEIL_LOG5_OF_MAX_SIG`. - fn short_fast_pow10(e: usize) -> Self; - - /// What the name says. It's easier to hard code than juggling intrinsics and - /// hoping LLVM constant folds it. - const CEIL_LOG5_OF_MAX_SIG: i16; - - // A conservative bound on the decimal digits of inputs that can't produce overflow or zero or - /// subnormals. Probably the decimal exponent of the maximum normal value, hence the name. - const MAX_NORMAL_DIGITS: usize; - - /// When the most significant decimal digit has a place value greater than this, the number - /// is certainly rounded to infinity. - const INF_CUTOFF: i64; - - /// When the most significant decimal digit has a place value less than this, the number - /// is certainly rounded to zero. - const ZERO_CUTOFF: i64; - - /// The number of bits in the exponent. - const EXP_BITS: u8; - - /// The number of bits in the significand, *including* the hidden bit. - const SIG_BITS: u8; - - /// The number of bits in the significand, *excluding* the hidden bit. - const EXPLICIT_SIG_BITS: u8; - - /// The maximum legal exponent in fractional representation. - const MAX_EXP: i16; - - /// The minimum legal exponent in fractional representation, excluding subnormals. - const MIN_EXP: i16; - - /// `MAX_EXP` for integral representation, i.e., with the shift applied. - const MAX_EXP_INT: i16; - - /// `MAX_EXP` encoded (i.e., with offset bias) - const MAX_ENCODED_EXP: i16; - - /// `MIN_EXP` for integral representation, i.e., with the shift applied. - const MIN_EXP_INT: i16; - - /// The maximum normalized significand in integral representation. - const MAX_SIG: u64; - - /// The minimal normalized significand in integral representation. - const MIN_SIG: u64; -} - -// Mostly a workaround for #34344. -macro_rules! other_constants { - ($type: ident) => { - const EXPLICIT_SIG_BITS: u8 = Self::SIG_BITS - 1; - const MAX_EXP: i16 = (1 << (Self::EXP_BITS - 1)) - 1; - const MIN_EXP: i16 = -::MAX_EXP + 1; - const MAX_EXP_INT: i16 = ::MAX_EXP - (Self::SIG_BITS as i16 - 1); - const MAX_ENCODED_EXP: i16 = (1 << Self::EXP_BITS) - 1; - const MIN_EXP_INT: i16 = ::MIN_EXP - (Self::SIG_BITS as i16 - 1); - const MAX_SIG: u64 = (1 << Self::SIG_BITS) - 1; - const MIN_SIG: u64 = 1 << (Self::SIG_BITS - 1); - - const INFINITY: Self = $type::INFINITY; - const NAN: Self = $type::NAN; - const ZERO: Self = 0.0; - }; -} - -impl RawFloat for f32 { - type Bits = u32; - - const SIG_BITS: u8 = 24; - const EXP_BITS: u8 = 8; - const CEIL_LOG5_OF_MAX_SIG: i16 = 11; - const MAX_NORMAL_DIGITS: usize = 35; - const INF_CUTOFF: i64 = 40; - const ZERO_CUTOFF: i64 = -48; - other_constants!(f32); - - /// Returns the mantissa, exponent and sign as integers. - fn integer_decode(self) -> (u64, i16, i8) { - let bits = self.to_bits(); - let sign: i8 = if bits >> 31 == 0 { 1 } else { -1 }; - let mut exponent: i16 = ((bits >> 23) & 0xff) as i16; - let mantissa = - if exponent == 0 { (bits & 0x7fffff) << 1 } else { (bits & 0x7fffff) | 0x800000 }; - // Exponent bias + mantissa shift - exponent -= 127 + 23; - (mantissa as u64, exponent, sign) - } - - fn unpack(self) -> Unpacked { - let (sig, exp, _sig) = self.integer_decode(); - Unpacked::new(sig, exp) - } - - fn from_int(x: u64) -> f32 { - // rkruppe is uncertain whether `as` rounds correctly on all platforms. - debug_assert!(x as f32 == fp_to_float(Fp { f: x, e: 0 })); - x as f32 - } - - fn short_fast_pow10(e: usize) -> Self { - table::F32_SHORT_POWERS[e] - } - - fn classify(self) -> FpCategory { - self.classify() - } - fn to_bits(self) -> Self::Bits { - self.to_bits() - } - fn from_bits(v: Self::Bits) -> Self { - Self::from_bits(v) - } -} - -impl RawFloat for f64 { - type Bits = u64; - - const SIG_BITS: u8 = 53; - const EXP_BITS: u8 = 11; - const CEIL_LOG5_OF_MAX_SIG: i16 = 23; - const MAX_NORMAL_DIGITS: usize = 305; - const INF_CUTOFF: i64 = 310; - const ZERO_CUTOFF: i64 = -326; - other_constants!(f64); - - /// Returns the mantissa, exponent and sign as integers. - fn integer_decode(self) -> (u64, i16, i8) { - let bits = self.to_bits(); - let sign: i8 = if bits >> 63 == 0 { 1 } else { -1 }; - let mut exponent: i16 = ((bits >> 52) & 0x7ff) as i16; - let mantissa = if exponent == 0 { - (bits & 0xfffffffffffff) << 1 - } else { - (bits & 0xfffffffffffff) | 0x10000000000000 - }; - // Exponent bias + mantissa shift - exponent -= 1023 + 52; - (mantissa, exponent, sign) - } - - fn unpack(self) -> Unpacked { - let (sig, exp, _sig) = self.integer_decode(); - Unpacked::new(sig, exp) - } - - fn from_int(x: u64) -> f64 { - // rkruppe is uncertain whether `as` rounds correctly on all platforms. - debug_assert!(x as f64 == fp_to_float(Fp { f: x, e: 0 })); - x as f64 - } - - fn short_fast_pow10(e: usize) -> Self { - table::F64_SHORT_POWERS[e] - } - - fn classify(self) -> FpCategory { - self.classify() - } - fn to_bits(self) -> Self::Bits { - self.to_bits() - } - fn from_bits(v: Self::Bits) -> Self { - Self::from_bits(v) - } -} - -/// Converts an `Fp` to the closest machine float type. -/// Does not handle subnormal results. -pub fn fp_to_float(x: Fp) -> T { - let x = x.normalize(); - // x.f is 64 bit, so x.e has a mantissa shift of 63 - let e = x.e + 63; - if e > T::MAX_EXP { - panic!("fp_to_float: exponent {} too large", e) - } else if e > T::MIN_EXP { - encode_normal(round_normal::(x)) - } else { - panic!("fp_to_float: exponent {} too small", e) - } -} - -/// Round the 64-bit significand to T::SIG_BITS bits with half-to-even. -/// Does not handle exponent overflow. -pub fn round_normal(x: Fp) -> Unpacked { - let excess = 64 - T::SIG_BITS as i16; - let half: u64 = 1 << (excess - 1); - let (q, rem) = (x.f >> excess, x.f & ((1 << excess) - 1)); - assert_eq!(q << excess | rem, x.f); - // Adjust mantissa shift - let k = x.e + excess; - if rem < half { - Unpacked::new(q, k) - } else if rem == half && (q % 2) == 0 { - Unpacked::new(q, k) - } else if q == T::MAX_SIG { - Unpacked::new(T::MIN_SIG, k + 1) - } else { - Unpacked::new(q + 1, k) - } -} - -/// Inverse of `RawFloat::unpack()` for normalized numbers. -/// Panics if the significand or exponent are not valid for normalized numbers. -pub fn encode_normal(x: Unpacked) -> T { - debug_assert!( - T::MIN_SIG <= x.sig && x.sig <= T::MAX_SIG, - "encode_normal: significand not normalized" - ); - // Remove the hidden bit - let sig_enc = x.sig & !(1 << T::EXPLICIT_SIG_BITS); - // Adjust the exponent for exponent bias and mantissa shift - let k_enc = x.k + T::MAX_EXP + T::EXPLICIT_SIG_BITS as i16; - debug_assert!(k_enc != 0 && k_enc < T::MAX_ENCODED_EXP, "encode_normal: exponent out of range"); - // Leave sign bit at 0 ("+"), our numbers are all positive - let bits = (k_enc as u64) << T::EXPLICIT_SIG_BITS | sig_enc; - T::from_bits(bits.try_into().unwrap_or_else(|_| unreachable!())) -} - -/// Construct a subnormal. A mantissa of 0 is allowed and constructs zero. -pub fn encode_subnormal(significand: u64) -> T { - assert!(significand < T::MIN_SIG, "encode_subnormal: not actually subnormal"); - // Encoded exponent is 0, the sign bit is 0, so we just have to reinterpret the bits. - T::from_bits(significand.try_into().unwrap_or_else(|_| unreachable!())) -} - -/// Approximate a bignum with an Fp. Rounds within 0.5 ULP with half-to-even. -pub fn big_to_fp(f: &Big) -> Fp { - let end = f.bit_length(); - assert!(end != 0, "big_to_fp: unexpectedly, input is zero"); - let start = end.saturating_sub(64); - let leading = num::get_bits(f, start, end); - // We cut off all bits prior to the index `start`, i.e., we effectively right-shift by - // an amount of `start`, so this is also the exponent we need. - let e = start as i16; - let rounded_down = Fp { f: leading, e }.normalize(); - // Round (half-to-even) depending on the truncated bits. - match num::compare_with_half_ulp(f, start) { - Less => rounded_down, - Equal if leading % 2 == 0 => rounded_down, - Equal | Greater => match leading.checked_add(1) { - Some(f) => Fp { f, e }.normalize(), - None => Fp { f: 1 << 63, e: e + 1 }, - }, - } -} - -/// Finds the largest floating point number strictly smaller than the argument. -/// Does not handle subnormals, zero, or exponent underflow. -pub fn prev_float(x: T) -> T { - match x.classify() { - Infinite => panic!("prev_float: argument is infinite"), - Nan => panic!("prev_float: argument is NaN"), - Subnormal => panic!("prev_float: argument is subnormal"), - Zero => panic!("prev_float: argument is zero"), - Normal => { - let Unpacked { sig, k } = x.unpack(); - if sig == T::MIN_SIG { - encode_normal(Unpacked::new(T::MAX_SIG, k - 1)) - } else { - encode_normal(Unpacked::new(sig - 1, k)) - } - } - } -} - -// Find the smallest floating point number strictly larger than the argument. -// This operation is saturating, i.e., next_float(inf) == inf. -// Unlike most code in this module, this function does handle zero, subnormals, and infinities. -// However, like all other code here, it does not deal with NaN and negative numbers. -pub fn next_float(x: T) -> T { - match x.classify() { - Nan => panic!("next_float: argument is NaN"), - Infinite => T::INFINITY, - // This seems too good to be true, but it works. - // 0.0 is encoded as the all-zero word. Subnormals are 0x000m...m where m is the mantissa. - // In particular, the smallest subnormal is 0x0...01 and the largest is 0x000F...F. - // The smallest normal number is 0x0010...0, so this corner case works as well. - // If the increment overflows the mantissa, the carry bit increments the exponent as we - // want, and the mantissa bits become zero. Because of the hidden bit convention, this - // too is exactly what we want! - // Finally, f64::MAX + 1 = 7eff...f + 1 = 7ff0...0 = f64::INFINITY. - Zero | Subnormal | Normal => T::from_bits(x.to_bits() + T::Bits::from(1u8)), - } -} diff --git a/crux-mir/lib/core/src/num/dec2flt/slow.rs b/crux-mir/lib/core/src/num/dec2flt/slow.rs new file mode 100644 index 000000000..bf1044033 --- /dev/null +++ b/crux-mir/lib/core/src/num/dec2flt/slow.rs @@ -0,0 +1,109 @@ +//! Slow, fallback algorithm for cases the Eisel-Lemire algorithm cannot round. + +use crate::num::dec2flt::common::BiasedFp; +use crate::num::dec2flt::decimal::{parse_decimal, Decimal}; +use crate::num::dec2flt::float::RawFloat; + +/// Parse the significant digits and biased, binary exponent of a float. +/// +/// This is a fallback algorithm that uses a big-integer representation +/// of the float, and therefore is considerably slower than faster +/// approximations. However, it will always determine how to round +/// the significant digits to the nearest machine float, allowing +/// use to handle near half-way cases. +/// +/// Near half-way cases are halfway between two consecutive machine floats. +/// For example, the float `16777217.0` has a bitwise representation of +/// `100000000000000000000000 1`. Rounding to a single-precision float, +/// the trailing `1` is truncated. Using round-nearest, tie-even, any +/// value above `16777217.0` must be rounded up to `16777218.0`, while +/// any value before or equal to `16777217.0` must be rounded down +/// to `16777216.0`. These near-halfway conversions therefore may require +/// a large number of digits to unambiguously determine how to round. +/// +/// The algorithms described here are based on "Processing Long Numbers Quickly", +/// available here: . +pub(crate) fn parse_long_mantissa(s: &[u8]) -> BiasedFp { + const MAX_SHIFT: usize = 60; + const NUM_POWERS: usize = 19; + const POWERS: [u8; 19] = + [0, 3, 6, 9, 13, 16, 19, 23, 26, 29, 33, 36, 39, 43, 46, 49, 53, 56, 59]; + + let get_shift = |n| { + if n < NUM_POWERS { POWERS[n] as usize } else { MAX_SHIFT } + }; + + let fp_zero = BiasedFp::zero_pow2(0); + let fp_inf = BiasedFp::zero_pow2(F::INFINITE_POWER); + + let mut d = parse_decimal(s); + + // Short-circuit if the value can only be a literal 0 or infinity. + if d.num_digits == 0 || d.decimal_point < -324 { + return fp_zero; + } else if d.decimal_point >= 310 { + return fp_inf; + } + let mut exp2 = 0_i32; + // Shift right toward (1/2 ... 1]. + while d.decimal_point > 0 { + let n = d.decimal_point as usize; + let shift = get_shift(n); + d.right_shift(shift); + if d.decimal_point < -Decimal::DECIMAL_POINT_RANGE { + return fp_zero; + } + exp2 += shift as i32; + } + // Shift left toward (1/2 ... 1]. + while d.decimal_point <= 0 { + let shift = if d.decimal_point == 0 { + match d.digits[0] { + digit if digit >= 5 => break, + 0 | 1 => 2, + _ => 1, + } + } else { + get_shift((-d.decimal_point) as _) + }; + d.left_shift(shift); + if d.decimal_point > Decimal::DECIMAL_POINT_RANGE { + return fp_inf; + } + exp2 -= shift as i32; + } + // We are now in the range [1/2 ... 1] but the binary format uses [1 ... 2]. + exp2 -= 1; + while (F::MINIMUM_EXPONENT + 1) > exp2 { + let mut n = ((F::MINIMUM_EXPONENT + 1) - exp2) as usize; + if n > MAX_SHIFT { + n = MAX_SHIFT; + } + d.right_shift(n); + exp2 += n as i32; + } + if (exp2 - F::MINIMUM_EXPONENT) >= F::INFINITE_POWER { + return fp_inf; + } + // Shift the decimal to the hidden bit, and then round the value + // to get the high mantissa+1 bits. + d.left_shift(F::MANTISSA_EXPLICIT_BITS + 1); + let mut mantissa = d.round(); + if mantissa >= (1_u64 << (F::MANTISSA_EXPLICIT_BITS + 1)) { + // Rounding up overflowed to the carry bit, need to + // shift back to the hidden bit. + d.right_shift(1); + exp2 += 1; + mantissa = d.round(); + if (exp2 - F::MINIMUM_EXPONENT) >= F::INFINITE_POWER { + return fp_inf; + } + } + let mut power2 = exp2 - F::MINIMUM_EXPONENT; + if mantissa < (1_u64 << F::MANTISSA_EXPLICIT_BITS) { + power2 -= 1; + } + // Zero out all the bits above the explicit mantissa bits. + mantissa &= (1_u64 << F::MANTISSA_EXPLICIT_BITS) - 1; + BiasedFp { f: mantissa, e: power2 } +} diff --git a/crux-mir/lib/core/src/num/dec2flt/table.rs b/crux-mir/lib/core/src/num/dec2flt/table.rs index 1bd94ffa0..4856074a6 100644 --- a/crux-mir/lib/core/src/num/dec2flt/table.rs +++ b/crux-mir/lib/core/src/num/dec2flt/table.rs @@ -1,1277 +1,670 @@ -//! Tables of approximations of powers of ten. +//! Pre-computed tables powers-of-5 for extended-precision representations. +//! +//! These tables enable fast scaling of the significant digits +//! of a float to the decimal exponent, with minimal rounding +//! errors, in a 128 or 192-bit representation. +//! //! DO NOT MODIFY: Generated by `src/etc/dec2flt_table.py` -pub const MIN_E: i16 = -305; -pub const MAX_E: i16 = 305; +pub const SMALLEST_POWER_OF_FIVE: i32 = -342; +pub const LARGEST_POWER_OF_FIVE: i32 = 308; +pub const N_POWERS_OF_FIVE: usize = (LARGEST_POWER_OF_FIVE - SMALLEST_POWER_OF_FIVE + 1) as usize; +// Use static to avoid long compile times: Rust compiler errors +// can have the entire table compiled multiple times, and then +// emit code multiple times, even if it's stripped out in +// the final binary. #[rustfmt::skip] -pub const POWERS: ([u64; 611], [i16; 611]) = ( - [ - 0xe0b62e2929aba83c, - 0x8c71dcd9ba0b4926, - 0xaf8e5410288e1b6f, - 0xdb71e91432b1a24b, - 0x892731ac9faf056f, - 0xab70fe17c79ac6ca, - 0xd64d3d9db981787d, - 0x85f0468293f0eb4e, - 0xa76c582338ed2622, - 0xd1476e2c07286faa, - 0x82cca4db847945ca, - 0xa37fce126597973d, - 0xcc5fc196fefd7d0c, - 0xff77b1fcbebcdc4f, - 0x9faacf3df73609b1, - 0xc795830d75038c1e, - 0xf97ae3d0d2446f25, - 0x9becce62836ac577, - 0xc2e801fb244576d5, - 0xf3a20279ed56d48a, - 0x9845418c345644d7, - 0xbe5691ef416bd60c, - 0xedec366b11c6cb8f, - 0x94b3a202eb1c3f39, - 0xb9e08a83a5e34f08, - 0xe858ad248f5c22ca, - 0x91376c36d99995be, - 0xb58547448ffffb2e, - 0xe2e69915b3fff9f9, - 0x8dd01fad907ffc3c, - 0xb1442798f49ffb4b, - 0xdd95317f31c7fa1d, - 0x8a7d3eef7f1cfc52, - 0xad1c8eab5ee43b67, - 0xd863b256369d4a41, - 0x873e4f75e2224e68, - 0xa90de3535aaae202, - 0xd3515c2831559a83, - 0x8412d9991ed58092, - 0xa5178fff668ae0b6, - 0xce5d73ff402d98e4, - 0x80fa687f881c7f8e, - 0xa139029f6a239f72, - 0xc987434744ac874f, - 0xfbe9141915d7a922, - 0x9d71ac8fada6c9b5, - 0xc4ce17b399107c23, - 0xf6019da07f549b2b, - 0x99c102844f94e0fb, - 0xc0314325637a193a, - 0xf03d93eebc589f88, - 0x96267c7535b763b5, - 0xbbb01b9283253ca3, - 0xea9c227723ee8bcb, - 0x92a1958a7675175f, - 0xb749faed14125d37, - 0xe51c79a85916f485, - 0x8f31cc0937ae58d3, - 0xb2fe3f0b8599ef08, - 0xdfbdcece67006ac9, - 0x8bd6a141006042be, - 0xaecc49914078536d, - 0xda7f5bf590966849, - 0x888f99797a5e012d, - 0xaab37fd7d8f58179, - 0xd5605fcdcf32e1d7, - 0x855c3be0a17fcd26, - 0xa6b34ad8c9dfc070, - 0xd0601d8efc57b08c, - 0x823c12795db6ce57, - 0xa2cb1717b52481ed, - 0xcb7ddcdda26da269, - 0xfe5d54150b090b03, - 0x9efa548d26e5a6e2, - 0xc6b8e9b0709f109a, - 0xf867241c8cc6d4c1, - 0x9b407691d7fc44f8, - 0xc21094364dfb5637, - 0xf294b943e17a2bc4, - 0x979cf3ca6cec5b5b, - 0xbd8430bd08277231, - 0xece53cec4a314ebe, - 0x940f4613ae5ed137, - 0xb913179899f68584, - 0xe757dd7ec07426e5, - 0x9096ea6f3848984f, - 0xb4bca50b065abe63, - 0xe1ebce4dc7f16dfc, - 0x8d3360f09cf6e4bd, - 0xb080392cc4349ded, - 0xdca04777f541c568, - 0x89e42caaf9491b61, - 0xac5d37d5b79b6239, - 0xd77485cb25823ac7, - 0x86a8d39ef77164bd, - 0xa8530886b54dbdec, - 0xd267caa862a12d67, - 0x8380dea93da4bc60, - 0xa46116538d0deb78, - 0xcd795be870516656, - 0x806bd9714632dff6, - 0xa086cfcd97bf97f4, - 0xc8a883c0fdaf7df0, - 0xfad2a4b13d1b5d6c, - 0x9cc3a6eec6311a64, - 0xc3f490aa77bd60fd, - 0xf4f1b4d515acb93c, - 0x991711052d8bf3c5, - 0xbf5cd54678eef0b7, - 0xef340a98172aace5, - 0x9580869f0e7aac0f, - 0xbae0a846d2195713, - 0xe998d258869facd7, - 0x91ff83775423cc06, - 0xb67f6455292cbf08, - 0xe41f3d6a7377eeca, - 0x8e938662882af53e, - 0xb23867fb2a35b28e, - 0xdec681f9f4c31f31, - 0x8b3c113c38f9f37f, - 0xae0b158b4738705f, - 0xd98ddaee19068c76, - 0x87f8a8d4cfa417ca, - 0xa9f6d30a038d1dbc, - 0xd47487cc8470652b, - 0x84c8d4dfd2c63f3b, - 0xa5fb0a17c777cf0a, - 0xcf79cc9db955c2cc, - 0x81ac1fe293d599c0, - 0xa21727db38cb0030, - 0xca9cf1d206fdc03c, - 0xfd442e4688bd304b, - 0x9e4a9cec15763e2f, - 0xc5dd44271ad3cdba, - 0xf7549530e188c129, - 0x9a94dd3e8cf578ba, - 0xc13a148e3032d6e8, - 0xf18899b1bc3f8ca2, - 0x96f5600f15a7b7e5, - 0xbcb2b812db11a5de, - 0xebdf661791d60f56, - 0x936b9fcebb25c996, - 0xb84687c269ef3bfb, - 0xe65829b3046b0afa, - 0x8ff71a0fe2c2e6dc, - 0xb3f4e093db73a093, - 0xe0f218b8d25088b8, - 0x8c974f7383725573, - 0xafbd2350644eead0, - 0xdbac6c247d62a584, - 0x894bc396ce5da772, - 0xab9eb47c81f5114f, - 0xd686619ba27255a3, - 0x8613fd0145877586, - 0xa798fc4196e952e7, - 0xd17f3b51fca3a7a1, - 0x82ef85133de648c5, - 0xa3ab66580d5fdaf6, - 0xcc963fee10b7d1b3, - 0xffbbcfe994e5c620, - 0x9fd561f1fd0f9bd4, - 0xc7caba6e7c5382c9, - 0xf9bd690a1b68637b, - 0x9c1661a651213e2d, - 0xc31bfa0fe5698db8, - 0xf3e2f893dec3f126, - 0x986ddb5c6b3a76b8, - 0xbe89523386091466, - 0xee2ba6c0678b597f, - 0x94db483840b717f0, - 0xba121a4650e4ddec, - 0xe896a0d7e51e1566, - 0x915e2486ef32cd60, - 0xb5b5ada8aaff80b8, - 0xe3231912d5bf60e6, - 0x8df5efabc5979c90, - 0xb1736b96b6fd83b4, - 0xddd0467c64bce4a1, - 0x8aa22c0dbef60ee4, - 0xad4ab7112eb3929e, - 0xd89d64d57a607745, - 0x87625f056c7c4a8b, - 0xa93af6c6c79b5d2e, - 0xd389b47879823479, - 0x843610cb4bf160cc, - 0xa54394fe1eedb8ff, - 0xce947a3da6a9273e, - 0x811ccc668829b887, - 0xa163ff802a3426a9, - 0xc9bcff6034c13053, - 0xfc2c3f3841f17c68, - 0x9d9ba7832936edc1, - 0xc5029163f384a931, - 0xf64335bcf065d37d, - 0x99ea0196163fa42e, - 0xc06481fb9bcf8d3a, - 0xf07da27a82c37088, - 0x964e858c91ba2655, - 0xbbe226efb628afeb, - 0xeadab0aba3b2dbe5, - 0x92c8ae6b464fc96f, - 0xb77ada0617e3bbcb, - 0xe55990879ddcaabe, - 0x8f57fa54c2a9eab7, - 0xb32df8e9f3546564, - 0xdff9772470297ebd, - 0x8bfbea76c619ef36, - 0xaefae51477a06b04, - 0xdab99e59958885c5, - 0x88b402f7fd75539b, - 0xaae103b5fcd2a882, - 0xd59944a37c0752a2, - 0x857fcae62d8493a5, - 0xa6dfbd9fb8e5b88f, - 0xd097ad07a71f26b2, - 0x825ecc24c8737830, - 0xa2f67f2dfa90563b, - 0xcbb41ef979346bca, - 0xfea126b7d78186bd, - 0x9f24b832e6b0f436, - 0xc6ede63fa05d3144, - 0xf8a95fcf88747d94, - 0x9b69dbe1b548ce7d, - 0xc24452da229b021c, - 0xf2d56790ab41c2a3, - 0x97c560ba6b0919a6, - 0xbdb6b8e905cb600f, - 0xed246723473e3813, - 0x9436c0760c86e30c, - 0xb94470938fa89bcf, - 0xe7958cb87392c2c3, - 0x90bd77f3483bb9ba, - 0xb4ecd5f01a4aa828, - 0xe2280b6c20dd5232, - 0x8d590723948a535f, - 0xb0af48ec79ace837, - 0xdcdb1b2798182245, - 0x8a08f0f8bf0f156b, - 0xac8b2d36eed2dac6, - 0xd7adf884aa879177, - 0x86ccbb52ea94baeb, - 0xa87fea27a539e9a5, - 0xd29fe4b18e88640f, - 0x83a3eeeef9153e89, - 0xa48ceaaab75a8e2b, - 0xcdb02555653131b6, - 0x808e17555f3ebf12, - 0xa0b19d2ab70e6ed6, - 0xc8de047564d20a8c, - 0xfb158592be068d2f, - 0x9ced737bb6c4183d, - 0xc428d05aa4751e4d, - 0xf53304714d9265e0, - 0x993fe2c6d07b7fac, - 0xbf8fdb78849a5f97, - 0xef73d256a5c0f77d, - 0x95a8637627989aae, - 0xbb127c53b17ec159, - 0xe9d71b689dde71b0, - 0x9226712162ab070e, - 0xb6b00d69bb55c8d1, - 0xe45c10c42a2b3b06, - 0x8eb98a7a9a5b04e3, - 0xb267ed1940f1c61c, - 0xdf01e85f912e37a3, - 0x8b61313bbabce2c6, - 0xae397d8aa96c1b78, - 0xd9c7dced53c72256, - 0x881cea14545c7575, - 0xaa242499697392d3, - 0xd4ad2dbfc3d07788, - 0x84ec3c97da624ab5, - 0xa6274bbdd0fadd62, - 0xcfb11ead453994ba, - 0x81ceb32c4b43fcf5, - 0xa2425ff75e14fc32, - 0xcad2f7f5359a3b3e, - 0xfd87b5f28300ca0e, - 0x9e74d1b791e07e48, - 0xc612062576589ddb, - 0xf79687aed3eec551, - 0x9abe14cd44753b53, - 0xc16d9a0095928a27, - 0xf1c90080baf72cb1, - 0x971da05074da7bef, - 0xbce5086492111aeb, - 0xec1e4a7db69561a5, - 0x9392ee8e921d5d07, - 0xb877aa3236a4b449, - 0xe69594bec44de15b, - 0x901d7cf73ab0acd9, - 0xb424dc35095cd80f, - 0xe12e13424bb40e13, - 0x8cbccc096f5088cc, - 0xafebff0bcb24aaff, - 0xdbe6fecebdedd5bf, - 0x89705f4136b4a597, - 0xabcc77118461cefd, - 0xd6bf94d5e57a42bc, - 0x8637bd05af6c69b6, - 0xa7c5ac471b478423, - 0xd1b71758e219652c, - 0x83126e978d4fdf3b, - 0xa3d70a3d70a3d70a, - 0xcccccccccccccccd, - 0x8000000000000000, - 0xa000000000000000, - 0xc800000000000000, - 0xfa00000000000000, - 0x9c40000000000000, - 0xc350000000000000, - 0xf424000000000000, - 0x9896800000000000, - 0xbebc200000000000, - 0xee6b280000000000, - 0x9502f90000000000, - 0xba43b74000000000, - 0xe8d4a51000000000, - 0x9184e72a00000000, - 0xb5e620f480000000, - 0xe35fa931a0000000, - 0x8e1bc9bf04000000, - 0xb1a2bc2ec5000000, - 0xde0b6b3a76400000, - 0x8ac7230489e80000, - 0xad78ebc5ac620000, - 0xd8d726b7177a8000, - 0x878678326eac9000, - 0xa968163f0a57b400, - 0xd3c21bcecceda100, - 0x84595161401484a0, - 0xa56fa5b99019a5c8, - 0xcecb8f27f4200f3a, - 0x813f3978f8940984, - 0xa18f07d736b90be5, - 0xc9f2c9cd04674edf, - 0xfc6f7c4045812296, - 0x9dc5ada82b70b59e, - 0xc5371912364ce305, - 0xf684df56c3e01bc7, - 0x9a130b963a6c115c, - 0xc097ce7bc90715b3, - 0xf0bdc21abb48db20, - 0x96769950b50d88f4, - 0xbc143fa4e250eb31, - 0xeb194f8e1ae525fd, - 0x92efd1b8d0cf37be, - 0xb7abc627050305ae, - 0xe596b7b0c643c719, - 0x8f7e32ce7bea5c70, - 0xb35dbf821ae4f38c, - 0xe0352f62a19e306f, - 0x8c213d9da502de45, - 0xaf298d050e4395d7, - 0xdaf3f04651d47b4c, - 0x88d8762bf324cd10, - 0xab0e93b6efee0054, - 0xd5d238a4abe98068, - 0x85a36366eb71f041, - 0xa70c3c40a64e6c52, - 0xd0cf4b50cfe20766, - 0x82818f1281ed44a0, - 0xa321f2d7226895c8, - 0xcbea6f8ceb02bb3a, - 0xfee50b7025c36a08, - 0x9f4f2726179a2245, - 0xc722f0ef9d80aad6, - 0xf8ebad2b84e0d58c, - 0x9b934c3b330c8577, - 0xc2781f49ffcfa6d5, - 0xf316271c7fc3908b, - 0x97edd871cfda3a57, - 0xbde94e8e43d0c8ec, - 0xed63a231d4c4fb27, - 0x945e455f24fb1cf9, - 0xb975d6b6ee39e437, - 0xe7d34c64a9c85d44, - 0x90e40fbeea1d3a4b, - 0xb51d13aea4a488dd, - 0xe264589a4dcdab15, - 0x8d7eb76070a08aed, - 0xb0de65388cc8ada8, - 0xdd15fe86affad912, - 0x8a2dbf142dfcc7ab, - 0xacb92ed9397bf996, - 0xd7e77a8f87daf7fc, - 0x86f0ac99b4e8dafd, - 0xa8acd7c0222311bd, - 0xd2d80db02aabd62c, - 0x83c7088e1aab65db, - 0xa4b8cab1a1563f52, - 0xcde6fd5e09abcf27, - 0x80b05e5ac60b6178, - 0xa0dc75f1778e39d6, - 0xc913936dd571c84c, - 0xfb5878494ace3a5f, - 0x9d174b2dcec0e47b, - 0xc45d1df942711d9a, - 0xf5746577930d6501, - 0x9968bf6abbe85f20, - 0xbfc2ef456ae276e9, - 0xefb3ab16c59b14a3, - 0x95d04aee3b80ece6, - 0xbb445da9ca61281f, - 0xea1575143cf97227, - 0x924d692ca61be758, - 0xb6e0c377cfa2e12e, - 0xe498f455c38b997a, - 0x8edf98b59a373fec, - 0xb2977ee300c50fe7, - 0xdf3d5e9bc0f653e1, - 0x8b865b215899f46d, - 0xae67f1e9aec07188, - 0xda01ee641a708dea, - 0x884134fe908658b2, - 0xaa51823e34a7eedf, - 0xd4e5e2cdc1d1ea96, - 0x850fadc09923329e, - 0xa6539930bf6bff46, - 0xcfe87f7cef46ff17, - 0x81f14fae158c5f6e, - 0xa26da3999aef774a, - 0xcb090c8001ab551c, - 0xfdcb4fa002162a63, - 0x9e9f11c4014dda7e, - 0xc646d63501a1511e, - 0xf7d88bc24209a565, - 0x9ae757596946075f, - 0xc1a12d2fc3978937, - 0xf209787bb47d6b85, - 0x9745eb4d50ce6333, - 0xbd176620a501fc00, - 0xec5d3fa8ce427b00, - 0x93ba47c980e98ce0, - 0xb8a8d9bbe123f018, - 0xe6d3102ad96cec1e, - 0x9043ea1ac7e41393, - 0xb454e4a179dd1877, - 0xe16a1dc9d8545e95, - 0x8ce2529e2734bb1d, - 0xb01ae745b101e9e4, - 0xdc21a1171d42645d, - 0x899504ae72497eba, - 0xabfa45da0edbde69, - 0xd6f8d7509292d603, - 0x865b86925b9bc5c2, - 0xa7f26836f282b733, - 0xd1ef0244af2364ff, - 0x8335616aed761f1f, - 0xa402b9c5a8d3a6e7, - 0xcd036837130890a1, - 0x802221226be55a65, - 0xa02aa96b06deb0fe, - 0xc83553c5c8965d3d, - 0xfa42a8b73abbf48d, - 0x9c69a97284b578d8, - 0xc38413cf25e2d70e, - 0xf46518c2ef5b8cd1, - 0x98bf2f79d5993803, - 0xbeeefb584aff8604, - 0xeeaaba2e5dbf6785, - 0x952ab45cfa97a0b3, - 0xba756174393d88e0, - 0xe912b9d1478ceb17, - 0x91abb422ccb812ef, - 0xb616a12b7fe617aa, - 0xe39c49765fdf9d95, - 0x8e41ade9fbebc27d, - 0xb1d219647ae6b31c, - 0xde469fbd99a05fe3, - 0x8aec23d680043bee, - 0xada72ccc20054aea, - 0xd910f7ff28069da4, - 0x87aa9aff79042287, - 0xa99541bf57452b28, - 0xd3fa922f2d1675f2, - 0x847c9b5d7c2e09b7, - 0xa59bc234db398c25, - 0xcf02b2c21207ef2f, - 0x8161afb94b44f57d, - 0xa1ba1ba79e1632dc, - 0xca28a291859bbf93, - 0xfcb2cb35e702af78, - 0x9defbf01b061adab, - 0xc56baec21c7a1916, - 0xf6c69a72a3989f5c, - 0x9a3c2087a63f6399, - 0xc0cb28a98fcf3c80, - 0xf0fdf2d3f3c30b9f, - 0x969eb7c47859e744, - 0xbc4665b596706115, - 0xeb57ff22fc0c795a, - 0x9316ff75dd87cbd8, - 0xb7dcbf5354e9bece, - 0xe5d3ef282a242e82, - 0x8fa475791a569d11, - 0xb38d92d760ec4455, - 0xe070f78d3927556b, - 0x8c469ab843b89563, - 0xaf58416654a6babb, - 0xdb2e51bfe9d0696a, - 0x88fcf317f22241e2, - 0xab3c2fddeeaad25b, - 0xd60b3bd56a5586f2, - 0x85c7056562757457, - 0xa738c6bebb12d16d, - 0xd106f86e69d785c8, - 0x82a45b450226b39d, - 0xa34d721642b06084, - 0xcc20ce9bd35c78a5, - 0xff290242c83396ce, - 0x9f79a169bd203e41, - 0xc75809c42c684dd1, - 0xf92e0c3537826146, - 0x9bbcc7a142b17ccc, - 0xc2abf989935ddbfe, - 0xf356f7ebf83552fe, - 0x98165af37b2153df, - 0xbe1bf1b059e9a8d6, - 0xeda2ee1c7064130c, - 0x9485d4d1c63e8be8, - 0xb9a74a0637ce2ee1, - 0xe8111c87c5c1ba9a, - 0x910ab1d4db9914a0, - 0xb54d5e4a127f59c8, - 0xe2a0b5dc971f303a, - 0x8da471a9de737e24, - 0xb10d8e1456105dad, - 0xdd50f1996b947519, - 0x8a5296ffe33cc930, - 0xace73cbfdc0bfb7b, - 0xd8210befd30efa5a, - 0x8714a775e3e95c78, - 0xa8d9d1535ce3b396, - 0xd31045a8341ca07c, - 0x83ea2b892091e44e, - 0xa4e4b66b68b65d61, - 0xce1de40642e3f4b9, - 0x80d2ae83e9ce78f4, - 0xa1075a24e4421731, - 0xc94930ae1d529cfd, - 0xfb9b7cd9a4a7443c, - 0x9d412e0806e88aa6, - 0xc491798a08a2ad4f, - 0xf5b5d7ec8acb58a3, - 0x9991a6f3d6bf1766, - 0xbff610b0cc6edd3f, - 0xeff394dcff8a948f, - 0x95f83d0a1fb69cd9, - 0xbb764c4ca7a44410, - 0xea53df5fd18d5514, - 0x92746b9be2f8552c, - 0xb7118682dbb66a77, - 0xe4d5e82392a40515, - 0x8f05b1163ba6832d, - 0xb2c71d5bca9023f8, - 0xdf78e4b2bd342cf7, - 0x8bab8eefb6409c1a, - 0xae9672aba3d0c321, - 0xda3c0f568cc4f3e9, - 0x8865899617fb1871, - 0xaa7eebfb9df9de8e, - 0xd51ea6fa85785631, - 0x8533285c936b35df, - 0xa67ff273b8460357, - 0xd01fef10a657842c, - 0x8213f56a67f6b29c, - 0xa298f2c501f45f43, - 0xcb3f2f7642717713, - 0xfe0efb53d30dd4d8, - 0x9ec95d1463e8a507, - 0xc67bb4597ce2ce49, - 0xf81aa16fdc1b81db, - 0x9b10a4e5e9913129, - 0xc1d4ce1f63f57d73, - 0xf24a01a73cf2dcd0, - 0x976e41088617ca02, - 0xbd49d14aa79dbc82, - 0xec9c459d51852ba3, - 0x93e1ab8252f33b46, - 0xb8da1662e7b00a17, - 0xe7109bfba19c0c9d, - 0x906a617d450187e2, - 0xb484f9dc9641e9db, - 0xe1a63853bbd26451, - 0x8d07e33455637eb3, - 0xb049dc016abc5e60, - 0xdc5c5301c56b75f7, - 0x89b9b3e11b6329bb, - 0xac2820d9623bf429, - 0xd732290fbacaf134, - 0x867f59a9d4bed6c0, - 0xa81f301449ee8c70, - 0xd226fc195c6a2f8c, - 0x83585d8fd9c25db8, - 0xa42e74f3d032f526, - 0xcd3a1230c43fb26f, - 0x80444b5e7aa7cf85, - 0xa0555e361951c367, - 0xc86ab5c39fa63441, - 0xfa856334878fc151, - 0x9c935e00d4b9d8d2, - 0xc3b8358109e84f07, - 0xf4a642e14c6262c9, - 0x98e7e9cccfbd7dbe, - 0xbf21e44003acdd2d, - 0xeeea5d5004981478, - 0x95527a5202df0ccb, - 0xbaa718e68396cffe, - 0xe950df20247c83fd, - 0x91d28b7416cdd27e, - ], - [ - -1077, - -1073, - -1070, - -1067, - -1063, - -1060, - -1057, - -1053, - -1050, - -1047, - -1043, - -1040, - -1037, - -1034, - -1030, - -1027, - -1024, - -1020, - -1017, - -1014, - -1010, - -1007, - -1004, - -1000, - -997, - -994, - -990, - -987, - -984, - -980, - -977, - -974, - -970, - -967, - -964, - -960, - -957, - -954, - -950, - -947, - -944, - -940, - -937, - -934, - -931, - -927, - -924, - -921, - -917, - -914, - -911, - -907, - -904, - -901, - -897, - -894, - -891, - -887, - -884, - -881, - -877, - -874, - -871, - -867, - -864, - -861, - -857, - -854, - -851, - -847, - -844, - -841, - -838, - -834, - -831, - -828, - -824, - -821, - -818, - -814, - -811, - -808, - -804, - -801, - -798, - -794, - -791, - -788, - -784, - -781, - -778, - -774, - -771, - -768, - -764, - -761, - -758, - -754, - -751, - -748, - -744, - -741, - -738, - -735, - -731, - -728, - -725, - -721, - -718, - -715, - -711, - -708, - -705, - -701, - -698, - -695, - -691, - -688, - -685, - -681, - -678, - -675, - -671, - -668, - -665, - -661, - -658, - -655, - -651, - -648, - -645, - -642, - -638, - -635, - -632, - -628, - -625, - -622, - -618, - -615, - -612, - -608, - -605, - -602, - -598, - -595, - -592, - -588, - -585, - -582, - -578, - -575, - -572, - -568, - -565, - -562, - -558, - -555, - -552, - -549, - -545, - -542, - -539, - -535, - -532, - -529, - -525, - -522, - -519, - -515, - -512, - -509, - -505, - -502, - -499, - -495, - -492, - -489, - -485, - -482, - -479, - -475, - -472, - -469, - -465, - -462, - -459, - -455, - -452, - -449, - -446, - -442, - -439, - -436, - -432, - -429, - -426, - -422, - -419, - -416, - -412, - -409, - -406, - -402, - -399, - -396, - -392, - -389, - -386, - -382, - -379, - -376, - -372, - -369, - -366, - -362, - -359, - -356, - -353, - -349, - -346, - -343, - -339, - -336, - -333, - -329, - -326, - -323, - -319, - -316, - -313, - -309, - -306, - -303, - -299, - -296, - -293, - -289, - -286, - -283, - -279, - -276, - -273, - -269, - -266, - -263, - -259, - -256, - -253, - -250, - -246, - -243, - -240, - -236, - -233, - -230, - -226, - -223, - -220, - -216, - -213, - -210, - -206, - -203, - -200, - -196, - -193, - -190, - -186, - -183, - -180, - -176, - -173, - -170, - -166, - -163, - -160, - -157, - -153, - -150, - -147, - -143, - -140, - -137, - -133, - -130, - -127, - -123, - -120, - -117, - -113, - -110, - -107, - -103, - -100, - -97, - -93, - -90, - -87, - -83, - -80, - -77, - -73, - -70, - -67, - -63, - -60, - -57, - -54, - -50, - -47, - -44, - -40, - -37, - -34, - -30, - -27, - -24, - -20, - -17, - -14, - -10, - -7, - -4, - 0, - 3, - 6, - 10, - 13, - 16, - 20, - 23, - 26, - 30, - 33, - 36, - 39, - 43, - 46, - 49, - 53, - 56, - 59, - 63, - 66, - 69, - 73, - 76, - 79, - 83, - 86, - 89, - 93, - 96, - 99, - 103, - 106, - 109, - 113, - 116, - 119, - 123, - 126, - 129, - 132, - 136, - 139, - 142, - 146, - 149, - 152, - 156, - 159, - 162, - 166, - 169, - 172, - 176, - 179, - 182, - 186, - 189, - 192, - 196, - 199, - 202, - 206, - 209, - 212, - 216, - 219, - 222, - 226, - 229, - 232, - 235, - 239, - 242, - 245, - 249, - 252, - 255, - 259, - 262, - 265, - 269, - 272, - 275, - 279, - 282, - 285, - 289, - 292, - 295, - 299, - 302, - 305, - 309, - 312, - 315, - 319, - 322, - 325, - 328, - 332, - 335, - 338, - 342, - 345, - 348, - 352, - 355, - 358, - 362, - 365, - 368, - 372, - 375, - 378, - 382, - 385, - 388, - 392, - 395, - 398, - 402, - 405, - 408, - 412, - 415, - 418, - 422, - 425, - 428, - 431, - 435, - 438, - 441, - 445, - 448, - 451, - 455, - 458, - 461, - 465, - 468, - 471, - 475, - 478, - 481, - 485, - 488, - 491, - 495, - 498, - 501, - 505, - 508, - 511, - 515, - 518, - 521, - 524, - 528, - 531, - 534, - 538, - 541, - 544, - 548, - 551, - 554, - 558, - 561, - 564, - 568, - 571, - 574, - 578, - 581, - 584, - 588, - 591, - 594, - 598, - 601, - 604, - 608, - 611, - 614, - 617, - 621, - 624, - 627, - 631, - 634, - 637, - 641, - 644, - 647, - 651, - 654, - 657, - 661, - 664, - 667, - 671, - 674, - 677, - 681, - 684, - 687, - 691, - 694, - 697, - 701, - 704, - 707, - 711, - 714, - 717, - 720, - 724, - 727, - 730, - 734, - 737, - 740, - 744, - 747, - 750, - 754, - 757, - 760, - 764, - 767, - 770, - 774, - 777, - 780, - 784, - 787, - 790, - 794, - 797, - 800, - 804, - 807, - 810, - 813, - 817, - 820, - 823, - 827, - 830, - 833, - 837, - 840, - 843, - 847, - 850, - 853, - 857, - 860, - 863, - 867, - 870, - 873, - 877, - 880, - 883, - 887, - 890, - 893, - 897, - 900, - 903, - 907, - 910, - 913, - 916, - 920, - 923, - 926, - 930, - 933, - 936, - 940, - 943, - 946, - 950, - ], -); - -#[rustfmt::skip] -pub const F32_SHORT_POWERS: [f32; 11] = [ - 1e0, - 1e1, - 1e2, - 1e3, - 1e4, - 1e5, - 1e6, - 1e7, - 1e8, - 1e9, - 1e10, -]; - -#[rustfmt::skip] -pub const F64_SHORT_POWERS: [f64; 23] = [ - 1e0, - 1e1, - 1e2, - 1e3, - 1e4, - 1e5, - 1e6, - 1e7, - 1e8, - 1e9, - 1e10, - 1e11, - 1e12, - 1e13, - 1e14, - 1e15, - 1e16, - 1e17, - 1e18, - 1e19, - 1e20, - 1e21, - 1e22, +pub static POWER_OF_FIVE_128: [(u64, u64); N_POWERS_OF_FIVE] = [ + (0xeef453d6923bd65a, 0x113faa2906a13b3f), // 5^-342 + (0x9558b4661b6565f8, 0x4ac7ca59a424c507), // 5^-341 + (0xbaaee17fa23ebf76, 0x5d79bcf00d2df649), // 5^-340 + (0xe95a99df8ace6f53, 0xf4d82c2c107973dc), // 5^-339 + (0x91d8a02bb6c10594, 0x79071b9b8a4be869), // 5^-338 + (0xb64ec836a47146f9, 0x9748e2826cdee284), // 5^-337 + (0xe3e27a444d8d98b7, 0xfd1b1b2308169b25), // 5^-336 + (0x8e6d8c6ab0787f72, 0xfe30f0f5e50e20f7), // 5^-335 + (0xb208ef855c969f4f, 0xbdbd2d335e51a935), // 5^-334 + (0xde8b2b66b3bc4723, 0xad2c788035e61382), // 5^-333 + (0x8b16fb203055ac76, 0x4c3bcb5021afcc31), // 5^-332 + (0xaddcb9e83c6b1793, 0xdf4abe242a1bbf3d), // 5^-331 + (0xd953e8624b85dd78, 0xd71d6dad34a2af0d), // 5^-330 + (0x87d4713d6f33aa6b, 0x8672648c40e5ad68), // 5^-329 + (0xa9c98d8ccb009506, 0x680efdaf511f18c2), // 5^-328 + (0xd43bf0effdc0ba48, 0x212bd1b2566def2), // 5^-327 + (0x84a57695fe98746d, 0x14bb630f7604b57), // 5^-326 + (0xa5ced43b7e3e9188, 0x419ea3bd35385e2d), // 5^-325 + (0xcf42894a5dce35ea, 0x52064cac828675b9), // 5^-324 + (0x818995ce7aa0e1b2, 0x7343efebd1940993), // 5^-323 + (0xa1ebfb4219491a1f, 0x1014ebe6c5f90bf8), // 5^-322 + (0xca66fa129f9b60a6, 0xd41a26e077774ef6), // 5^-321 + (0xfd00b897478238d0, 0x8920b098955522b4), // 5^-320 + (0x9e20735e8cb16382, 0x55b46e5f5d5535b0), // 5^-319 + (0xc5a890362fddbc62, 0xeb2189f734aa831d), // 5^-318 + (0xf712b443bbd52b7b, 0xa5e9ec7501d523e4), // 5^-317 + (0x9a6bb0aa55653b2d, 0x47b233c92125366e), // 5^-316 + (0xc1069cd4eabe89f8, 0x999ec0bb696e840a), // 5^-315 + (0xf148440a256e2c76, 0xc00670ea43ca250d), // 5^-314 + (0x96cd2a865764dbca, 0x380406926a5e5728), // 5^-313 + (0xbc807527ed3e12bc, 0xc605083704f5ecf2), // 5^-312 + (0xeba09271e88d976b, 0xf7864a44c633682e), // 5^-311 + (0x93445b8731587ea3, 0x7ab3ee6afbe0211d), // 5^-310 + (0xb8157268fdae9e4c, 0x5960ea05bad82964), // 5^-309 + (0xe61acf033d1a45df, 0x6fb92487298e33bd), // 5^-308 + (0x8fd0c16206306bab, 0xa5d3b6d479f8e056), // 5^-307 + (0xb3c4f1ba87bc8696, 0x8f48a4899877186c), // 5^-306 + (0xe0b62e2929aba83c, 0x331acdabfe94de87), // 5^-305 + (0x8c71dcd9ba0b4925, 0x9ff0c08b7f1d0b14), // 5^-304 + (0xaf8e5410288e1b6f, 0x7ecf0ae5ee44dd9), // 5^-303 + (0xdb71e91432b1a24a, 0xc9e82cd9f69d6150), // 5^-302 + (0x892731ac9faf056e, 0xbe311c083a225cd2), // 5^-301 + (0xab70fe17c79ac6ca, 0x6dbd630a48aaf406), // 5^-300 + (0xd64d3d9db981787d, 0x92cbbccdad5b108), // 5^-299 + (0x85f0468293f0eb4e, 0x25bbf56008c58ea5), // 5^-298 + (0xa76c582338ed2621, 0xaf2af2b80af6f24e), // 5^-297 + (0xd1476e2c07286faa, 0x1af5af660db4aee1), // 5^-296 + (0x82cca4db847945ca, 0x50d98d9fc890ed4d), // 5^-295 + (0xa37fce126597973c, 0xe50ff107bab528a0), // 5^-294 + (0xcc5fc196fefd7d0c, 0x1e53ed49a96272c8), // 5^-293 + (0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7a), // 5^-292 + (0x9faacf3df73609b1, 0x77b191618c54e9ac), // 5^-291 + (0xc795830d75038c1d, 0xd59df5b9ef6a2417), // 5^-290 + (0xf97ae3d0d2446f25, 0x4b0573286b44ad1d), // 5^-289 + (0x9becce62836ac577, 0x4ee367f9430aec32), // 5^-288 + (0xc2e801fb244576d5, 0x229c41f793cda73f), // 5^-287 + (0xf3a20279ed56d48a, 0x6b43527578c1110f), // 5^-286 + (0x9845418c345644d6, 0x830a13896b78aaa9), // 5^-285 + (0xbe5691ef416bd60c, 0x23cc986bc656d553), // 5^-284 + (0xedec366b11c6cb8f, 0x2cbfbe86b7ec8aa8), // 5^-283 + (0x94b3a202eb1c3f39, 0x7bf7d71432f3d6a9), // 5^-282 + (0xb9e08a83a5e34f07, 0xdaf5ccd93fb0cc53), // 5^-281 + (0xe858ad248f5c22c9, 0xd1b3400f8f9cff68), // 5^-280 + (0x91376c36d99995be, 0x23100809b9c21fa1), // 5^-279 + (0xb58547448ffffb2d, 0xabd40a0c2832a78a), // 5^-278 + (0xe2e69915b3fff9f9, 0x16c90c8f323f516c), // 5^-277 + (0x8dd01fad907ffc3b, 0xae3da7d97f6792e3), // 5^-276 + (0xb1442798f49ffb4a, 0x99cd11cfdf41779c), // 5^-275 + (0xdd95317f31c7fa1d, 0x40405643d711d583), // 5^-274 + (0x8a7d3eef7f1cfc52, 0x482835ea666b2572), // 5^-273 + (0xad1c8eab5ee43b66, 0xda3243650005eecf), // 5^-272 + (0xd863b256369d4a40, 0x90bed43e40076a82), // 5^-271 + (0x873e4f75e2224e68, 0x5a7744a6e804a291), // 5^-270 + (0xa90de3535aaae202, 0x711515d0a205cb36), // 5^-269 + (0xd3515c2831559a83, 0xd5a5b44ca873e03), // 5^-268 + (0x8412d9991ed58091, 0xe858790afe9486c2), // 5^-267 + (0xa5178fff668ae0b6, 0x626e974dbe39a872), // 5^-266 + (0xce5d73ff402d98e3, 0xfb0a3d212dc8128f), // 5^-265 + (0x80fa687f881c7f8e, 0x7ce66634bc9d0b99), // 5^-264 + (0xa139029f6a239f72, 0x1c1fffc1ebc44e80), // 5^-263 + (0xc987434744ac874e, 0xa327ffb266b56220), // 5^-262 + (0xfbe9141915d7a922, 0x4bf1ff9f0062baa8), // 5^-261 + (0x9d71ac8fada6c9b5, 0x6f773fc3603db4a9), // 5^-260 + (0xc4ce17b399107c22, 0xcb550fb4384d21d3), // 5^-259 + (0xf6019da07f549b2b, 0x7e2a53a146606a48), // 5^-258 + (0x99c102844f94e0fb, 0x2eda7444cbfc426d), // 5^-257 + (0xc0314325637a1939, 0xfa911155fefb5308), // 5^-256 + (0xf03d93eebc589f88, 0x793555ab7eba27ca), // 5^-255 + (0x96267c7535b763b5, 0x4bc1558b2f3458de), // 5^-254 + (0xbbb01b9283253ca2, 0x9eb1aaedfb016f16), // 5^-253 + (0xea9c227723ee8bcb, 0x465e15a979c1cadc), // 5^-252 + (0x92a1958a7675175f, 0xbfacd89ec191ec9), // 5^-251 + (0xb749faed14125d36, 0xcef980ec671f667b), // 5^-250 + (0xe51c79a85916f484, 0x82b7e12780e7401a), // 5^-249 + (0x8f31cc0937ae58d2, 0xd1b2ecb8b0908810), // 5^-248 + (0xb2fe3f0b8599ef07, 0x861fa7e6dcb4aa15), // 5^-247 + (0xdfbdcece67006ac9, 0x67a791e093e1d49a), // 5^-246 + (0x8bd6a141006042bd, 0xe0c8bb2c5c6d24e0), // 5^-245 + (0xaecc49914078536d, 0x58fae9f773886e18), // 5^-244 + (0xda7f5bf590966848, 0xaf39a475506a899e), // 5^-243 + (0x888f99797a5e012d, 0x6d8406c952429603), // 5^-242 + (0xaab37fd7d8f58178, 0xc8e5087ba6d33b83), // 5^-241 + (0xd5605fcdcf32e1d6, 0xfb1e4a9a90880a64), // 5^-240 + (0x855c3be0a17fcd26, 0x5cf2eea09a55067f), // 5^-239 + (0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481e), // 5^-238 + (0xd0601d8efc57b08b, 0xf13b94daf124da26), // 5^-237 + (0x823c12795db6ce57, 0x76c53d08d6b70858), // 5^-236 + (0xa2cb1717b52481ed, 0x54768c4b0c64ca6e), // 5^-235 + (0xcb7ddcdda26da268, 0xa9942f5dcf7dfd09), // 5^-234 + (0xfe5d54150b090b02, 0xd3f93b35435d7c4c), // 5^-233 + (0x9efa548d26e5a6e1, 0xc47bc5014a1a6daf), // 5^-232 + (0xc6b8e9b0709f109a, 0x359ab6419ca1091b), // 5^-231 + (0xf867241c8cc6d4c0, 0xc30163d203c94b62), // 5^-230 + (0x9b407691d7fc44f8, 0x79e0de63425dcf1d), // 5^-229 + (0xc21094364dfb5636, 0x985915fc12f542e4), // 5^-228 + (0xf294b943e17a2bc4, 0x3e6f5b7b17b2939d), // 5^-227 + (0x979cf3ca6cec5b5a, 0xa705992ceecf9c42), // 5^-226 + (0xbd8430bd08277231, 0x50c6ff782a838353), // 5^-225 + (0xece53cec4a314ebd, 0xa4f8bf5635246428), // 5^-224 + (0x940f4613ae5ed136, 0x871b7795e136be99), // 5^-223 + (0xb913179899f68584, 0x28e2557b59846e3f), // 5^-222 + (0xe757dd7ec07426e5, 0x331aeada2fe589cf), // 5^-221 + (0x9096ea6f3848984f, 0x3ff0d2c85def7621), // 5^-220 + (0xb4bca50b065abe63, 0xfed077a756b53a9), // 5^-219 + (0xe1ebce4dc7f16dfb, 0xd3e8495912c62894), // 5^-218 + (0x8d3360f09cf6e4bd, 0x64712dd7abbbd95c), // 5^-217 + (0xb080392cc4349dec, 0xbd8d794d96aacfb3), // 5^-216 + (0xdca04777f541c567, 0xecf0d7a0fc5583a0), // 5^-215 + (0x89e42caaf9491b60, 0xf41686c49db57244), // 5^-214 + (0xac5d37d5b79b6239, 0x311c2875c522ced5), // 5^-213 + (0xd77485cb25823ac7, 0x7d633293366b828b), // 5^-212 + (0x86a8d39ef77164bc, 0xae5dff9c02033197), // 5^-211 + (0xa8530886b54dbdeb, 0xd9f57f830283fdfc), // 5^-210 + (0xd267caa862a12d66, 0xd072df63c324fd7b), // 5^-209 + (0x8380dea93da4bc60, 0x4247cb9e59f71e6d), // 5^-208 + (0xa46116538d0deb78, 0x52d9be85f074e608), // 5^-207 + (0xcd795be870516656, 0x67902e276c921f8b), // 5^-206 + (0x806bd9714632dff6, 0xba1cd8a3db53b6), // 5^-205 + (0xa086cfcd97bf97f3, 0x80e8a40eccd228a4), // 5^-204 + (0xc8a883c0fdaf7df0, 0x6122cd128006b2cd), // 5^-203 + (0xfad2a4b13d1b5d6c, 0x796b805720085f81), // 5^-202 + (0x9cc3a6eec6311a63, 0xcbe3303674053bb0), // 5^-201 + (0xc3f490aa77bd60fc, 0xbedbfc4411068a9c), // 5^-200 + (0xf4f1b4d515acb93b, 0xee92fb5515482d44), // 5^-199 + (0x991711052d8bf3c5, 0x751bdd152d4d1c4a), // 5^-198 + (0xbf5cd54678eef0b6, 0xd262d45a78a0635d), // 5^-197 + (0xef340a98172aace4, 0x86fb897116c87c34), // 5^-196 + (0x9580869f0e7aac0e, 0xd45d35e6ae3d4da0), // 5^-195 + (0xbae0a846d2195712, 0x8974836059cca109), // 5^-194 + (0xe998d258869facd7, 0x2bd1a438703fc94b), // 5^-193 + (0x91ff83775423cc06, 0x7b6306a34627ddcf), // 5^-192 + (0xb67f6455292cbf08, 0x1a3bc84c17b1d542), // 5^-191 + (0xe41f3d6a7377eeca, 0x20caba5f1d9e4a93), // 5^-190 + (0x8e938662882af53e, 0x547eb47b7282ee9c), // 5^-189 + (0xb23867fb2a35b28d, 0xe99e619a4f23aa43), // 5^-188 + (0xdec681f9f4c31f31, 0x6405fa00e2ec94d4), // 5^-187 + (0x8b3c113c38f9f37e, 0xde83bc408dd3dd04), // 5^-186 + (0xae0b158b4738705e, 0x9624ab50b148d445), // 5^-185 + (0xd98ddaee19068c76, 0x3badd624dd9b0957), // 5^-184 + (0x87f8a8d4cfa417c9, 0xe54ca5d70a80e5d6), // 5^-183 + (0xa9f6d30a038d1dbc, 0x5e9fcf4ccd211f4c), // 5^-182 + (0xd47487cc8470652b, 0x7647c3200069671f), // 5^-181 + (0x84c8d4dfd2c63f3b, 0x29ecd9f40041e073), // 5^-180 + (0xa5fb0a17c777cf09, 0xf468107100525890), // 5^-179 + (0xcf79cc9db955c2cc, 0x7182148d4066eeb4), // 5^-178 + (0x81ac1fe293d599bf, 0xc6f14cd848405530), // 5^-177 + (0xa21727db38cb002f, 0xb8ada00e5a506a7c), // 5^-176 + (0xca9cf1d206fdc03b, 0xa6d90811f0e4851c), // 5^-175 + (0xfd442e4688bd304a, 0x908f4a166d1da663), // 5^-174 + (0x9e4a9cec15763e2e, 0x9a598e4e043287fe), // 5^-173 + (0xc5dd44271ad3cdba, 0x40eff1e1853f29fd), // 5^-172 + (0xf7549530e188c128, 0xd12bee59e68ef47c), // 5^-171 + (0x9a94dd3e8cf578b9, 0x82bb74f8301958ce), // 5^-170 + (0xc13a148e3032d6e7, 0xe36a52363c1faf01), // 5^-169 + (0xf18899b1bc3f8ca1, 0xdc44e6c3cb279ac1), // 5^-168 + (0x96f5600f15a7b7e5, 0x29ab103a5ef8c0b9), // 5^-167 + (0xbcb2b812db11a5de, 0x7415d448f6b6f0e7), // 5^-166 + (0xebdf661791d60f56, 0x111b495b3464ad21), // 5^-165 + (0x936b9fcebb25c995, 0xcab10dd900beec34), // 5^-164 + (0xb84687c269ef3bfb, 0x3d5d514f40eea742), // 5^-163 + (0xe65829b3046b0afa, 0xcb4a5a3112a5112), // 5^-162 + (0x8ff71a0fe2c2e6dc, 0x47f0e785eaba72ab), // 5^-161 + (0xb3f4e093db73a093, 0x59ed216765690f56), // 5^-160 + (0xe0f218b8d25088b8, 0x306869c13ec3532c), // 5^-159 + (0x8c974f7383725573, 0x1e414218c73a13fb), // 5^-158 + (0xafbd2350644eeacf, 0xe5d1929ef90898fa), // 5^-157 + (0xdbac6c247d62a583, 0xdf45f746b74abf39), // 5^-156 + (0x894bc396ce5da772, 0x6b8bba8c328eb783), // 5^-155 + (0xab9eb47c81f5114f, 0x66ea92f3f326564), // 5^-154 + (0xd686619ba27255a2, 0xc80a537b0efefebd), // 5^-153 + (0x8613fd0145877585, 0xbd06742ce95f5f36), // 5^-152 + (0xa798fc4196e952e7, 0x2c48113823b73704), // 5^-151 + (0xd17f3b51fca3a7a0, 0xf75a15862ca504c5), // 5^-150 + (0x82ef85133de648c4, 0x9a984d73dbe722fb), // 5^-149 + (0xa3ab66580d5fdaf5, 0xc13e60d0d2e0ebba), // 5^-148 + (0xcc963fee10b7d1b3, 0x318df905079926a8), // 5^-147 + (0xffbbcfe994e5c61f, 0xfdf17746497f7052), // 5^-146 + (0x9fd561f1fd0f9bd3, 0xfeb6ea8bedefa633), // 5^-145 + (0xc7caba6e7c5382c8, 0xfe64a52ee96b8fc0), // 5^-144 + (0xf9bd690a1b68637b, 0x3dfdce7aa3c673b0), // 5^-143 + (0x9c1661a651213e2d, 0x6bea10ca65c084e), // 5^-142 + (0xc31bfa0fe5698db8, 0x486e494fcff30a62), // 5^-141 + (0xf3e2f893dec3f126, 0x5a89dba3c3efccfa), // 5^-140 + (0x986ddb5c6b3a76b7, 0xf89629465a75e01c), // 5^-139 + (0xbe89523386091465, 0xf6bbb397f1135823), // 5^-138 + (0xee2ba6c0678b597f, 0x746aa07ded582e2c), // 5^-137 + (0x94db483840b717ef, 0xa8c2a44eb4571cdc), // 5^-136 + (0xba121a4650e4ddeb, 0x92f34d62616ce413), // 5^-135 + (0xe896a0d7e51e1566, 0x77b020baf9c81d17), // 5^-134 + (0x915e2486ef32cd60, 0xace1474dc1d122e), // 5^-133 + (0xb5b5ada8aaff80b8, 0xd819992132456ba), // 5^-132 + (0xe3231912d5bf60e6, 0x10e1fff697ed6c69), // 5^-131 + (0x8df5efabc5979c8f, 0xca8d3ffa1ef463c1), // 5^-130 + (0xb1736b96b6fd83b3, 0xbd308ff8a6b17cb2), // 5^-129 + (0xddd0467c64bce4a0, 0xac7cb3f6d05ddbde), // 5^-128 + (0x8aa22c0dbef60ee4, 0x6bcdf07a423aa96b), // 5^-127 + (0xad4ab7112eb3929d, 0x86c16c98d2c953c6), // 5^-126 + (0xd89d64d57a607744, 0xe871c7bf077ba8b7), // 5^-125 + (0x87625f056c7c4a8b, 0x11471cd764ad4972), // 5^-124 + (0xa93af6c6c79b5d2d, 0xd598e40d3dd89bcf), // 5^-123 + (0xd389b47879823479, 0x4aff1d108d4ec2c3), // 5^-122 + (0x843610cb4bf160cb, 0xcedf722a585139ba), // 5^-121 + (0xa54394fe1eedb8fe, 0xc2974eb4ee658828), // 5^-120 + (0xce947a3da6a9273e, 0x733d226229feea32), // 5^-119 + (0x811ccc668829b887, 0x806357d5a3f525f), // 5^-118 + (0xa163ff802a3426a8, 0xca07c2dcb0cf26f7), // 5^-117 + (0xc9bcff6034c13052, 0xfc89b393dd02f0b5), // 5^-116 + (0xfc2c3f3841f17c67, 0xbbac2078d443ace2), // 5^-115 + (0x9d9ba7832936edc0, 0xd54b944b84aa4c0d), // 5^-114 + (0xc5029163f384a931, 0xa9e795e65d4df11), // 5^-113 + (0xf64335bcf065d37d, 0x4d4617b5ff4a16d5), // 5^-112 + (0x99ea0196163fa42e, 0x504bced1bf8e4e45), // 5^-111 + (0xc06481fb9bcf8d39, 0xe45ec2862f71e1d6), // 5^-110 + (0xf07da27a82c37088, 0x5d767327bb4e5a4c), // 5^-109 + (0x964e858c91ba2655, 0x3a6a07f8d510f86f), // 5^-108 + (0xbbe226efb628afea, 0x890489f70a55368b), // 5^-107 + (0xeadab0aba3b2dbe5, 0x2b45ac74ccea842e), // 5^-106 + (0x92c8ae6b464fc96f, 0x3b0b8bc90012929d), // 5^-105 + (0xb77ada0617e3bbcb, 0x9ce6ebb40173744), // 5^-104 + (0xe55990879ddcaabd, 0xcc420a6a101d0515), // 5^-103 + (0x8f57fa54c2a9eab6, 0x9fa946824a12232d), // 5^-102 + (0xb32df8e9f3546564, 0x47939822dc96abf9), // 5^-101 + (0xdff9772470297ebd, 0x59787e2b93bc56f7), // 5^-100 + (0x8bfbea76c619ef36, 0x57eb4edb3c55b65a), // 5^-99 + (0xaefae51477a06b03, 0xede622920b6b23f1), // 5^-98 + (0xdab99e59958885c4, 0xe95fab368e45eced), // 5^-97 + (0x88b402f7fd75539b, 0x11dbcb0218ebb414), // 5^-96 + (0xaae103b5fcd2a881, 0xd652bdc29f26a119), // 5^-95 + (0xd59944a37c0752a2, 0x4be76d3346f0495f), // 5^-94 + (0x857fcae62d8493a5, 0x6f70a4400c562ddb), // 5^-93 + (0xa6dfbd9fb8e5b88e, 0xcb4ccd500f6bb952), // 5^-92 + (0xd097ad07a71f26b2, 0x7e2000a41346a7a7), // 5^-91 + (0x825ecc24c873782f, 0x8ed400668c0c28c8), // 5^-90 + (0xa2f67f2dfa90563b, 0x728900802f0f32fa), // 5^-89 + (0xcbb41ef979346bca, 0x4f2b40a03ad2ffb9), // 5^-88 + (0xfea126b7d78186bc, 0xe2f610c84987bfa8), // 5^-87 + (0x9f24b832e6b0f436, 0xdd9ca7d2df4d7c9), // 5^-86 + (0xc6ede63fa05d3143, 0x91503d1c79720dbb), // 5^-85 + (0xf8a95fcf88747d94, 0x75a44c6397ce912a), // 5^-84 + (0x9b69dbe1b548ce7c, 0xc986afbe3ee11aba), // 5^-83 + (0xc24452da229b021b, 0xfbe85badce996168), // 5^-82 + (0xf2d56790ab41c2a2, 0xfae27299423fb9c3), // 5^-81 + (0x97c560ba6b0919a5, 0xdccd879fc967d41a), // 5^-80 + (0xbdb6b8e905cb600f, 0x5400e987bbc1c920), // 5^-79 + (0xed246723473e3813, 0x290123e9aab23b68), // 5^-78 + (0x9436c0760c86e30b, 0xf9a0b6720aaf6521), // 5^-77 + (0xb94470938fa89bce, 0xf808e40e8d5b3e69), // 5^-76 + (0xe7958cb87392c2c2, 0xb60b1d1230b20e04), // 5^-75 + (0x90bd77f3483bb9b9, 0xb1c6f22b5e6f48c2), // 5^-74 + (0xb4ecd5f01a4aa828, 0x1e38aeb6360b1af3), // 5^-73 + (0xe2280b6c20dd5232, 0x25c6da63c38de1b0), // 5^-72 + (0x8d590723948a535f, 0x579c487e5a38ad0e), // 5^-71 + (0xb0af48ec79ace837, 0x2d835a9df0c6d851), // 5^-70 + (0xdcdb1b2798182244, 0xf8e431456cf88e65), // 5^-69 + (0x8a08f0f8bf0f156b, 0x1b8e9ecb641b58ff), // 5^-68 + (0xac8b2d36eed2dac5, 0xe272467e3d222f3f), // 5^-67 + (0xd7adf884aa879177, 0x5b0ed81dcc6abb0f), // 5^-66 + (0x86ccbb52ea94baea, 0x98e947129fc2b4e9), // 5^-65 + (0xa87fea27a539e9a5, 0x3f2398d747b36224), // 5^-64 + (0xd29fe4b18e88640e, 0x8eec7f0d19a03aad), // 5^-63 + (0x83a3eeeef9153e89, 0x1953cf68300424ac), // 5^-62 + (0xa48ceaaab75a8e2b, 0x5fa8c3423c052dd7), // 5^-61 + (0xcdb02555653131b6, 0x3792f412cb06794d), // 5^-60 + (0x808e17555f3ebf11, 0xe2bbd88bbee40bd0), // 5^-59 + (0xa0b19d2ab70e6ed6, 0x5b6aceaeae9d0ec4), // 5^-58 + (0xc8de047564d20a8b, 0xf245825a5a445275), // 5^-57 + (0xfb158592be068d2e, 0xeed6e2f0f0d56712), // 5^-56 + (0x9ced737bb6c4183d, 0x55464dd69685606b), // 5^-55 + (0xc428d05aa4751e4c, 0xaa97e14c3c26b886), // 5^-54 + (0xf53304714d9265df, 0xd53dd99f4b3066a8), // 5^-53 + (0x993fe2c6d07b7fab, 0xe546a8038efe4029), // 5^-52 + (0xbf8fdb78849a5f96, 0xde98520472bdd033), // 5^-51 + (0xef73d256a5c0f77c, 0x963e66858f6d4440), // 5^-50 + (0x95a8637627989aad, 0xdde7001379a44aa8), // 5^-49 + (0xbb127c53b17ec159, 0x5560c018580d5d52), // 5^-48 + (0xe9d71b689dde71af, 0xaab8f01e6e10b4a6), // 5^-47 + (0x9226712162ab070d, 0xcab3961304ca70e8), // 5^-46 + (0xb6b00d69bb55c8d1, 0x3d607b97c5fd0d22), // 5^-45 + (0xe45c10c42a2b3b05, 0x8cb89a7db77c506a), // 5^-44 + (0x8eb98a7a9a5b04e3, 0x77f3608e92adb242), // 5^-43 + (0xb267ed1940f1c61c, 0x55f038b237591ed3), // 5^-42 + (0xdf01e85f912e37a3, 0x6b6c46dec52f6688), // 5^-41 + (0x8b61313bbabce2c6, 0x2323ac4b3b3da015), // 5^-40 + (0xae397d8aa96c1b77, 0xabec975e0a0d081a), // 5^-39 + (0xd9c7dced53c72255, 0x96e7bd358c904a21), // 5^-38 + (0x881cea14545c7575, 0x7e50d64177da2e54), // 5^-37 + (0xaa242499697392d2, 0xdde50bd1d5d0b9e9), // 5^-36 + (0xd4ad2dbfc3d07787, 0x955e4ec64b44e864), // 5^-35 + (0x84ec3c97da624ab4, 0xbd5af13bef0b113e), // 5^-34 + (0xa6274bbdd0fadd61, 0xecb1ad8aeacdd58e), // 5^-33 + (0xcfb11ead453994ba, 0x67de18eda5814af2), // 5^-32 + (0x81ceb32c4b43fcf4, 0x80eacf948770ced7), // 5^-31 + (0xa2425ff75e14fc31, 0xa1258379a94d028d), // 5^-30 + (0xcad2f7f5359a3b3e, 0x96ee45813a04330), // 5^-29 + (0xfd87b5f28300ca0d, 0x8bca9d6e188853fc), // 5^-28 + (0x9e74d1b791e07e48, 0x775ea264cf55347e), // 5^-27 + (0xc612062576589dda, 0x95364afe032a819e), // 5^-26 + (0xf79687aed3eec551, 0x3a83ddbd83f52205), // 5^-25 + (0x9abe14cd44753b52, 0xc4926a9672793543), // 5^-24 + (0xc16d9a0095928a27, 0x75b7053c0f178294), // 5^-23 + (0xf1c90080baf72cb1, 0x5324c68b12dd6339), // 5^-22 + (0x971da05074da7bee, 0xd3f6fc16ebca5e04), // 5^-21 + (0xbce5086492111aea, 0x88f4bb1ca6bcf585), // 5^-20 + (0xec1e4a7db69561a5, 0x2b31e9e3d06c32e6), // 5^-19 + (0x9392ee8e921d5d07, 0x3aff322e62439fd0), // 5^-18 + (0xb877aa3236a4b449, 0x9befeb9fad487c3), // 5^-17 + (0xe69594bec44de15b, 0x4c2ebe687989a9b4), // 5^-16 + (0x901d7cf73ab0acd9, 0xf9d37014bf60a11), // 5^-15 + (0xb424dc35095cd80f, 0x538484c19ef38c95), // 5^-14 + (0xe12e13424bb40e13, 0x2865a5f206b06fba), // 5^-13 + (0x8cbccc096f5088cb, 0xf93f87b7442e45d4), // 5^-12 + (0xafebff0bcb24aafe, 0xf78f69a51539d749), // 5^-11 + (0xdbe6fecebdedd5be, 0xb573440e5a884d1c), // 5^-10 + (0x89705f4136b4a597, 0x31680a88f8953031), // 5^-9 + (0xabcc77118461cefc, 0xfdc20d2b36ba7c3e), // 5^-8 + (0xd6bf94d5e57a42bc, 0x3d32907604691b4d), // 5^-7 + (0x8637bd05af6c69b5, 0xa63f9a49c2c1b110), // 5^-6 + (0xa7c5ac471b478423, 0xfcf80dc33721d54), // 5^-5 + (0xd1b71758e219652b, 0xd3c36113404ea4a9), // 5^-4 + (0x83126e978d4fdf3b, 0x645a1cac083126ea), // 5^-3 + (0xa3d70a3d70a3d70a, 0x3d70a3d70a3d70a4), // 5^-2 + (0xcccccccccccccccc, 0xcccccccccccccccd), // 5^-1 + (0x8000000000000000, 0x0), // 5^0 + (0xa000000000000000, 0x0), // 5^1 + (0xc800000000000000, 0x0), // 5^2 + (0xfa00000000000000, 0x0), // 5^3 + (0x9c40000000000000, 0x0), // 5^4 + (0xc350000000000000, 0x0), // 5^5 + (0xf424000000000000, 0x0), // 5^6 + (0x9896800000000000, 0x0), // 5^7 + (0xbebc200000000000, 0x0), // 5^8 + (0xee6b280000000000, 0x0), // 5^9 + (0x9502f90000000000, 0x0), // 5^10 + (0xba43b74000000000, 0x0), // 5^11 + (0xe8d4a51000000000, 0x0), // 5^12 + (0x9184e72a00000000, 0x0), // 5^13 + (0xb5e620f480000000, 0x0), // 5^14 + (0xe35fa931a0000000, 0x0), // 5^15 + (0x8e1bc9bf04000000, 0x0), // 5^16 + (0xb1a2bc2ec5000000, 0x0), // 5^17 + (0xde0b6b3a76400000, 0x0), // 5^18 + (0x8ac7230489e80000, 0x0), // 5^19 + (0xad78ebc5ac620000, 0x0), // 5^20 + (0xd8d726b7177a8000, 0x0), // 5^21 + (0x878678326eac9000, 0x0), // 5^22 + (0xa968163f0a57b400, 0x0), // 5^23 + (0xd3c21bcecceda100, 0x0), // 5^24 + (0x84595161401484a0, 0x0), // 5^25 + (0xa56fa5b99019a5c8, 0x0), // 5^26 + (0xcecb8f27f4200f3a, 0x0), // 5^27 + (0x813f3978f8940984, 0x4000000000000000), // 5^28 + (0xa18f07d736b90be5, 0x5000000000000000), // 5^29 + (0xc9f2c9cd04674ede, 0xa400000000000000), // 5^30 + (0xfc6f7c4045812296, 0x4d00000000000000), // 5^31 + (0x9dc5ada82b70b59d, 0xf020000000000000), // 5^32 + (0xc5371912364ce305, 0x6c28000000000000), // 5^33 + (0xf684df56c3e01bc6, 0xc732000000000000), // 5^34 + (0x9a130b963a6c115c, 0x3c7f400000000000), // 5^35 + (0xc097ce7bc90715b3, 0x4b9f100000000000), // 5^36 + (0xf0bdc21abb48db20, 0x1e86d40000000000), // 5^37 + (0x96769950b50d88f4, 0x1314448000000000), // 5^38 + (0xbc143fa4e250eb31, 0x17d955a000000000), // 5^39 + (0xeb194f8e1ae525fd, 0x5dcfab0800000000), // 5^40 + (0x92efd1b8d0cf37be, 0x5aa1cae500000000), // 5^41 + (0xb7abc627050305ad, 0xf14a3d9e40000000), // 5^42 + (0xe596b7b0c643c719, 0x6d9ccd05d0000000), // 5^43 + (0x8f7e32ce7bea5c6f, 0xe4820023a2000000), // 5^44 + (0xb35dbf821ae4f38b, 0xdda2802c8a800000), // 5^45 + (0xe0352f62a19e306e, 0xd50b2037ad200000), // 5^46 + (0x8c213d9da502de45, 0x4526f422cc340000), // 5^47 + (0xaf298d050e4395d6, 0x9670b12b7f410000), // 5^48 + (0xdaf3f04651d47b4c, 0x3c0cdd765f114000), // 5^49 + (0x88d8762bf324cd0f, 0xa5880a69fb6ac800), // 5^50 + (0xab0e93b6efee0053, 0x8eea0d047a457a00), // 5^51 + (0xd5d238a4abe98068, 0x72a4904598d6d880), // 5^52 + (0x85a36366eb71f041, 0x47a6da2b7f864750), // 5^53 + (0xa70c3c40a64e6c51, 0x999090b65f67d924), // 5^54 + (0xd0cf4b50cfe20765, 0xfff4b4e3f741cf6d), // 5^55 + (0x82818f1281ed449f, 0xbff8f10e7a8921a4), // 5^56 + (0xa321f2d7226895c7, 0xaff72d52192b6a0d), // 5^57 + (0xcbea6f8ceb02bb39, 0x9bf4f8a69f764490), // 5^58 + (0xfee50b7025c36a08, 0x2f236d04753d5b4), // 5^59 + (0x9f4f2726179a2245, 0x1d762422c946590), // 5^60 + (0xc722f0ef9d80aad6, 0x424d3ad2b7b97ef5), // 5^61 + (0xf8ebad2b84e0d58b, 0xd2e0898765a7deb2), // 5^62 + (0x9b934c3b330c8577, 0x63cc55f49f88eb2f), // 5^63 + (0xc2781f49ffcfa6d5, 0x3cbf6b71c76b25fb), // 5^64 + (0xf316271c7fc3908a, 0x8bef464e3945ef7a), // 5^65 + (0x97edd871cfda3a56, 0x97758bf0e3cbb5ac), // 5^66 + (0xbde94e8e43d0c8ec, 0x3d52eeed1cbea317), // 5^67 + (0xed63a231d4c4fb27, 0x4ca7aaa863ee4bdd), // 5^68 + (0x945e455f24fb1cf8, 0x8fe8caa93e74ef6a), // 5^69 + (0xb975d6b6ee39e436, 0xb3e2fd538e122b44), // 5^70 + (0xe7d34c64a9c85d44, 0x60dbbca87196b616), // 5^71 + (0x90e40fbeea1d3a4a, 0xbc8955e946fe31cd), // 5^72 + (0xb51d13aea4a488dd, 0x6babab6398bdbe41), // 5^73 + (0xe264589a4dcdab14, 0xc696963c7eed2dd1), // 5^74 + (0x8d7eb76070a08aec, 0xfc1e1de5cf543ca2), // 5^75 + (0xb0de65388cc8ada8, 0x3b25a55f43294bcb), // 5^76 + (0xdd15fe86affad912, 0x49ef0eb713f39ebe), // 5^77 + (0x8a2dbf142dfcc7ab, 0x6e3569326c784337), // 5^78 + (0xacb92ed9397bf996, 0x49c2c37f07965404), // 5^79 + (0xd7e77a8f87daf7fb, 0xdc33745ec97be906), // 5^80 + (0x86f0ac99b4e8dafd, 0x69a028bb3ded71a3), // 5^81 + (0xa8acd7c0222311bc, 0xc40832ea0d68ce0c), // 5^82 + (0xd2d80db02aabd62b, 0xf50a3fa490c30190), // 5^83 + (0x83c7088e1aab65db, 0x792667c6da79e0fa), // 5^84 + (0xa4b8cab1a1563f52, 0x577001b891185938), // 5^85 + (0xcde6fd5e09abcf26, 0xed4c0226b55e6f86), // 5^86 + (0x80b05e5ac60b6178, 0x544f8158315b05b4), // 5^87 + (0xa0dc75f1778e39d6, 0x696361ae3db1c721), // 5^88 + (0xc913936dd571c84c, 0x3bc3a19cd1e38e9), // 5^89 + (0xfb5878494ace3a5f, 0x4ab48a04065c723), // 5^90 + (0x9d174b2dcec0e47b, 0x62eb0d64283f9c76), // 5^91 + (0xc45d1df942711d9a, 0x3ba5d0bd324f8394), // 5^92 + (0xf5746577930d6500, 0xca8f44ec7ee36479), // 5^93 + (0x9968bf6abbe85f20, 0x7e998b13cf4e1ecb), // 5^94 + (0xbfc2ef456ae276e8, 0x9e3fedd8c321a67e), // 5^95 + (0xefb3ab16c59b14a2, 0xc5cfe94ef3ea101e), // 5^96 + (0x95d04aee3b80ece5, 0xbba1f1d158724a12), // 5^97 + (0xbb445da9ca61281f, 0x2a8a6e45ae8edc97), // 5^98 + (0xea1575143cf97226, 0xf52d09d71a3293bd), // 5^99 + (0x924d692ca61be758, 0x593c2626705f9c56), // 5^100 + (0xb6e0c377cfa2e12e, 0x6f8b2fb00c77836c), // 5^101 + (0xe498f455c38b997a, 0xb6dfb9c0f956447), // 5^102 + (0x8edf98b59a373fec, 0x4724bd4189bd5eac), // 5^103 + (0xb2977ee300c50fe7, 0x58edec91ec2cb657), // 5^104 + (0xdf3d5e9bc0f653e1, 0x2f2967b66737e3ed), // 5^105 + (0x8b865b215899f46c, 0xbd79e0d20082ee74), // 5^106 + (0xae67f1e9aec07187, 0xecd8590680a3aa11), // 5^107 + (0xda01ee641a708de9, 0xe80e6f4820cc9495), // 5^108 + (0x884134fe908658b2, 0x3109058d147fdcdd), // 5^109 + (0xaa51823e34a7eede, 0xbd4b46f0599fd415), // 5^110 + (0xd4e5e2cdc1d1ea96, 0x6c9e18ac7007c91a), // 5^111 + (0x850fadc09923329e, 0x3e2cf6bc604ddb0), // 5^112 + (0xa6539930bf6bff45, 0x84db8346b786151c), // 5^113 + (0xcfe87f7cef46ff16, 0xe612641865679a63), // 5^114 + (0x81f14fae158c5f6e, 0x4fcb7e8f3f60c07e), // 5^115 + (0xa26da3999aef7749, 0xe3be5e330f38f09d), // 5^116 + (0xcb090c8001ab551c, 0x5cadf5bfd3072cc5), // 5^117 + (0xfdcb4fa002162a63, 0x73d9732fc7c8f7f6), // 5^118 + (0x9e9f11c4014dda7e, 0x2867e7fddcdd9afa), // 5^119 + (0xc646d63501a1511d, 0xb281e1fd541501b8), // 5^120 + (0xf7d88bc24209a565, 0x1f225a7ca91a4226), // 5^121 + (0x9ae757596946075f, 0x3375788de9b06958), // 5^122 + (0xc1a12d2fc3978937, 0x52d6b1641c83ae), // 5^123 + (0xf209787bb47d6b84, 0xc0678c5dbd23a49a), // 5^124 + (0x9745eb4d50ce6332, 0xf840b7ba963646e0), // 5^125 + (0xbd176620a501fbff, 0xb650e5a93bc3d898), // 5^126 + (0xec5d3fa8ce427aff, 0xa3e51f138ab4cebe), // 5^127 + (0x93ba47c980e98cdf, 0xc66f336c36b10137), // 5^128 + (0xb8a8d9bbe123f017, 0xb80b0047445d4184), // 5^129 + (0xe6d3102ad96cec1d, 0xa60dc059157491e5), // 5^130 + (0x9043ea1ac7e41392, 0x87c89837ad68db2f), // 5^131 + (0xb454e4a179dd1877, 0x29babe4598c311fb), // 5^132 + (0xe16a1dc9d8545e94, 0xf4296dd6fef3d67a), // 5^133 + (0x8ce2529e2734bb1d, 0x1899e4a65f58660c), // 5^134 + (0xb01ae745b101e9e4, 0x5ec05dcff72e7f8f), // 5^135 + (0xdc21a1171d42645d, 0x76707543f4fa1f73), // 5^136 + (0x899504ae72497eba, 0x6a06494a791c53a8), // 5^137 + (0xabfa45da0edbde69, 0x487db9d17636892), // 5^138 + (0xd6f8d7509292d603, 0x45a9d2845d3c42b6), // 5^139 + (0x865b86925b9bc5c2, 0xb8a2392ba45a9b2), // 5^140 + (0xa7f26836f282b732, 0x8e6cac7768d7141e), // 5^141 + (0xd1ef0244af2364ff, 0x3207d795430cd926), // 5^142 + (0x8335616aed761f1f, 0x7f44e6bd49e807b8), // 5^143 + (0xa402b9c5a8d3a6e7, 0x5f16206c9c6209a6), // 5^144 + (0xcd036837130890a1, 0x36dba887c37a8c0f), // 5^145 + (0x802221226be55a64, 0xc2494954da2c9789), // 5^146 + (0xa02aa96b06deb0fd, 0xf2db9baa10b7bd6c), // 5^147 + (0xc83553c5c8965d3d, 0x6f92829494e5acc7), // 5^148 + (0xfa42a8b73abbf48c, 0xcb772339ba1f17f9), // 5^149 + (0x9c69a97284b578d7, 0xff2a760414536efb), // 5^150 + (0xc38413cf25e2d70d, 0xfef5138519684aba), // 5^151 + (0xf46518c2ef5b8cd1, 0x7eb258665fc25d69), // 5^152 + (0x98bf2f79d5993802, 0xef2f773ffbd97a61), // 5^153 + (0xbeeefb584aff8603, 0xaafb550ffacfd8fa), // 5^154 + (0xeeaaba2e5dbf6784, 0x95ba2a53f983cf38), // 5^155 + (0x952ab45cfa97a0b2, 0xdd945a747bf26183), // 5^156 + (0xba756174393d88df, 0x94f971119aeef9e4), // 5^157 + (0xe912b9d1478ceb17, 0x7a37cd5601aab85d), // 5^158 + (0x91abb422ccb812ee, 0xac62e055c10ab33a), // 5^159 + (0xb616a12b7fe617aa, 0x577b986b314d6009), // 5^160 + (0xe39c49765fdf9d94, 0xed5a7e85fda0b80b), // 5^161 + (0x8e41ade9fbebc27d, 0x14588f13be847307), // 5^162 + (0xb1d219647ae6b31c, 0x596eb2d8ae258fc8), // 5^163 + (0xde469fbd99a05fe3, 0x6fca5f8ed9aef3bb), // 5^164 + (0x8aec23d680043bee, 0x25de7bb9480d5854), // 5^165 + (0xada72ccc20054ae9, 0xaf561aa79a10ae6a), // 5^166 + (0xd910f7ff28069da4, 0x1b2ba1518094da04), // 5^167 + (0x87aa9aff79042286, 0x90fb44d2f05d0842), // 5^168 + (0xa99541bf57452b28, 0x353a1607ac744a53), // 5^169 + (0xd3fa922f2d1675f2, 0x42889b8997915ce8), // 5^170 + (0x847c9b5d7c2e09b7, 0x69956135febada11), // 5^171 + (0xa59bc234db398c25, 0x43fab9837e699095), // 5^172 + (0xcf02b2c21207ef2e, 0x94f967e45e03f4bb), // 5^173 + (0x8161afb94b44f57d, 0x1d1be0eebac278f5), // 5^174 + (0xa1ba1ba79e1632dc, 0x6462d92a69731732), // 5^175 + (0xca28a291859bbf93, 0x7d7b8f7503cfdcfe), // 5^176 + (0xfcb2cb35e702af78, 0x5cda735244c3d43e), // 5^177 + (0x9defbf01b061adab, 0x3a0888136afa64a7), // 5^178 + (0xc56baec21c7a1916, 0x88aaa1845b8fdd0), // 5^179 + (0xf6c69a72a3989f5b, 0x8aad549e57273d45), // 5^180 + (0x9a3c2087a63f6399, 0x36ac54e2f678864b), // 5^181 + (0xc0cb28a98fcf3c7f, 0x84576a1bb416a7dd), // 5^182 + (0xf0fdf2d3f3c30b9f, 0x656d44a2a11c51d5), // 5^183 + (0x969eb7c47859e743, 0x9f644ae5a4b1b325), // 5^184 + (0xbc4665b596706114, 0x873d5d9f0dde1fee), // 5^185 + (0xeb57ff22fc0c7959, 0xa90cb506d155a7ea), // 5^186 + (0x9316ff75dd87cbd8, 0x9a7f12442d588f2), // 5^187 + (0xb7dcbf5354e9bece, 0xc11ed6d538aeb2f), // 5^188 + (0xe5d3ef282a242e81, 0x8f1668c8a86da5fa), // 5^189 + (0x8fa475791a569d10, 0xf96e017d694487bc), // 5^190 + (0xb38d92d760ec4455, 0x37c981dcc395a9ac), // 5^191 + (0xe070f78d3927556a, 0x85bbe253f47b1417), // 5^192 + (0x8c469ab843b89562, 0x93956d7478ccec8e), // 5^193 + (0xaf58416654a6babb, 0x387ac8d1970027b2), // 5^194 + (0xdb2e51bfe9d0696a, 0x6997b05fcc0319e), // 5^195 + (0x88fcf317f22241e2, 0x441fece3bdf81f03), // 5^196 + (0xab3c2fddeeaad25a, 0xd527e81cad7626c3), // 5^197 + (0xd60b3bd56a5586f1, 0x8a71e223d8d3b074), // 5^198 + (0x85c7056562757456, 0xf6872d5667844e49), // 5^199 + (0xa738c6bebb12d16c, 0xb428f8ac016561db), // 5^200 + (0xd106f86e69d785c7, 0xe13336d701beba52), // 5^201 + (0x82a45b450226b39c, 0xecc0024661173473), // 5^202 + (0xa34d721642b06084, 0x27f002d7f95d0190), // 5^203 + (0xcc20ce9bd35c78a5, 0x31ec038df7b441f4), // 5^204 + (0xff290242c83396ce, 0x7e67047175a15271), // 5^205 + (0x9f79a169bd203e41, 0xf0062c6e984d386), // 5^206 + (0xc75809c42c684dd1, 0x52c07b78a3e60868), // 5^207 + (0xf92e0c3537826145, 0xa7709a56ccdf8a82), // 5^208 + (0x9bbcc7a142b17ccb, 0x88a66076400bb691), // 5^209 + (0xc2abf989935ddbfe, 0x6acff893d00ea435), // 5^210 + (0xf356f7ebf83552fe, 0x583f6b8c4124d43), // 5^211 + (0x98165af37b2153de, 0xc3727a337a8b704a), // 5^212 + (0xbe1bf1b059e9a8d6, 0x744f18c0592e4c5c), // 5^213 + (0xeda2ee1c7064130c, 0x1162def06f79df73), // 5^214 + (0x9485d4d1c63e8be7, 0x8addcb5645ac2ba8), // 5^215 + (0xb9a74a0637ce2ee1, 0x6d953e2bd7173692), // 5^216 + (0xe8111c87c5c1ba99, 0xc8fa8db6ccdd0437), // 5^217 + (0x910ab1d4db9914a0, 0x1d9c9892400a22a2), // 5^218 + (0xb54d5e4a127f59c8, 0x2503beb6d00cab4b), // 5^219 + (0xe2a0b5dc971f303a, 0x2e44ae64840fd61d), // 5^220 + (0x8da471a9de737e24, 0x5ceaecfed289e5d2), // 5^221 + (0xb10d8e1456105dad, 0x7425a83e872c5f47), // 5^222 + (0xdd50f1996b947518, 0xd12f124e28f77719), // 5^223 + (0x8a5296ffe33cc92f, 0x82bd6b70d99aaa6f), // 5^224 + (0xace73cbfdc0bfb7b, 0x636cc64d1001550b), // 5^225 + (0xd8210befd30efa5a, 0x3c47f7e05401aa4e), // 5^226 + (0x8714a775e3e95c78, 0x65acfaec34810a71), // 5^227 + (0xa8d9d1535ce3b396, 0x7f1839a741a14d0d), // 5^228 + (0xd31045a8341ca07c, 0x1ede48111209a050), // 5^229 + (0x83ea2b892091e44d, 0x934aed0aab460432), // 5^230 + (0xa4e4b66b68b65d60, 0xf81da84d5617853f), // 5^231 + (0xce1de40642e3f4b9, 0x36251260ab9d668e), // 5^232 + (0x80d2ae83e9ce78f3, 0xc1d72b7c6b426019), // 5^233 + (0xa1075a24e4421730, 0xb24cf65b8612f81f), // 5^234 + (0xc94930ae1d529cfc, 0xdee033f26797b627), // 5^235 + (0xfb9b7cd9a4a7443c, 0x169840ef017da3b1), // 5^236 + (0x9d412e0806e88aa5, 0x8e1f289560ee864e), // 5^237 + (0xc491798a08a2ad4e, 0xf1a6f2bab92a27e2), // 5^238 + (0xf5b5d7ec8acb58a2, 0xae10af696774b1db), // 5^239 + (0x9991a6f3d6bf1765, 0xacca6da1e0a8ef29), // 5^240 + (0xbff610b0cc6edd3f, 0x17fd090a58d32af3), // 5^241 + (0xeff394dcff8a948e, 0xddfc4b4cef07f5b0), // 5^242 + (0x95f83d0a1fb69cd9, 0x4abdaf101564f98e), // 5^243 + (0xbb764c4ca7a4440f, 0x9d6d1ad41abe37f1), // 5^244 + (0xea53df5fd18d5513, 0x84c86189216dc5ed), // 5^245 + (0x92746b9be2f8552c, 0x32fd3cf5b4e49bb4), // 5^246 + (0xb7118682dbb66a77, 0x3fbc8c33221dc2a1), // 5^247 + (0xe4d5e82392a40515, 0xfabaf3feaa5334a), // 5^248 + (0x8f05b1163ba6832d, 0x29cb4d87f2a7400e), // 5^249 + (0xb2c71d5bca9023f8, 0x743e20e9ef511012), // 5^250 + (0xdf78e4b2bd342cf6, 0x914da9246b255416), // 5^251 + (0x8bab8eefb6409c1a, 0x1ad089b6c2f7548e), // 5^252 + (0xae9672aba3d0c320, 0xa184ac2473b529b1), // 5^253 + (0xda3c0f568cc4f3e8, 0xc9e5d72d90a2741e), // 5^254 + (0x8865899617fb1871, 0x7e2fa67c7a658892), // 5^255 + (0xaa7eebfb9df9de8d, 0xddbb901b98feeab7), // 5^256 + (0xd51ea6fa85785631, 0x552a74227f3ea565), // 5^257 + (0x8533285c936b35de, 0xd53a88958f87275f), // 5^258 + (0xa67ff273b8460356, 0x8a892abaf368f137), // 5^259 + (0xd01fef10a657842c, 0x2d2b7569b0432d85), // 5^260 + (0x8213f56a67f6b29b, 0x9c3b29620e29fc73), // 5^261 + (0xa298f2c501f45f42, 0x8349f3ba91b47b8f), // 5^262 + (0xcb3f2f7642717713, 0x241c70a936219a73), // 5^263 + (0xfe0efb53d30dd4d7, 0xed238cd383aa0110), // 5^264 + (0x9ec95d1463e8a506, 0xf4363804324a40aa), // 5^265 + (0xc67bb4597ce2ce48, 0xb143c6053edcd0d5), // 5^266 + (0xf81aa16fdc1b81da, 0xdd94b7868e94050a), // 5^267 + (0x9b10a4e5e9913128, 0xca7cf2b4191c8326), // 5^268 + (0xc1d4ce1f63f57d72, 0xfd1c2f611f63a3f0), // 5^269 + (0xf24a01a73cf2dccf, 0xbc633b39673c8cec), // 5^270 + (0x976e41088617ca01, 0xd5be0503e085d813), // 5^271 + (0xbd49d14aa79dbc82, 0x4b2d8644d8a74e18), // 5^272 + (0xec9c459d51852ba2, 0xddf8e7d60ed1219e), // 5^273 + (0x93e1ab8252f33b45, 0xcabb90e5c942b503), // 5^274 + (0xb8da1662e7b00a17, 0x3d6a751f3b936243), // 5^275 + (0xe7109bfba19c0c9d, 0xcc512670a783ad4), // 5^276 + (0x906a617d450187e2, 0x27fb2b80668b24c5), // 5^277 + (0xb484f9dc9641e9da, 0xb1f9f660802dedf6), // 5^278 + (0xe1a63853bbd26451, 0x5e7873f8a0396973), // 5^279 + (0x8d07e33455637eb2, 0xdb0b487b6423e1e8), // 5^280 + (0xb049dc016abc5e5f, 0x91ce1a9a3d2cda62), // 5^281 + (0xdc5c5301c56b75f7, 0x7641a140cc7810fb), // 5^282 + (0x89b9b3e11b6329ba, 0xa9e904c87fcb0a9d), // 5^283 + (0xac2820d9623bf429, 0x546345fa9fbdcd44), // 5^284 + (0xd732290fbacaf133, 0xa97c177947ad4095), // 5^285 + (0x867f59a9d4bed6c0, 0x49ed8eabcccc485d), // 5^286 + (0xa81f301449ee8c70, 0x5c68f256bfff5a74), // 5^287 + (0xd226fc195c6a2f8c, 0x73832eec6fff3111), // 5^288 + (0x83585d8fd9c25db7, 0xc831fd53c5ff7eab), // 5^289 + (0xa42e74f3d032f525, 0xba3e7ca8b77f5e55), // 5^290 + (0xcd3a1230c43fb26f, 0x28ce1bd2e55f35eb), // 5^291 + (0x80444b5e7aa7cf85, 0x7980d163cf5b81b3), // 5^292 + (0xa0555e361951c366, 0xd7e105bcc332621f), // 5^293 + (0xc86ab5c39fa63440, 0x8dd9472bf3fefaa7), // 5^294 + (0xfa856334878fc150, 0xb14f98f6f0feb951), // 5^295 + (0x9c935e00d4b9d8d2, 0x6ed1bf9a569f33d3), // 5^296 + (0xc3b8358109e84f07, 0xa862f80ec4700c8), // 5^297 + (0xf4a642e14c6262c8, 0xcd27bb612758c0fa), // 5^298 + (0x98e7e9cccfbd7dbd, 0x8038d51cb897789c), // 5^299 + (0xbf21e44003acdd2c, 0xe0470a63e6bd56c3), // 5^300 + (0xeeea5d5004981478, 0x1858ccfce06cac74), // 5^301 + (0x95527a5202df0ccb, 0xf37801e0c43ebc8), // 5^302 + (0xbaa718e68396cffd, 0xd30560258f54e6ba), // 5^303 + (0xe950df20247c83fd, 0x47c6b82ef32a2069), // 5^304 + (0x91d28b7416cdd27e, 0x4cdc331d57fa5441), // 5^305 + (0xb6472e511c81471d, 0xe0133fe4adf8e952), // 5^306 + (0xe3d8f9e563a198e5, 0x58180fddd97723a6), // 5^307 + (0x8e679c2f5e44ff8f, 0x570f09eaa7ea7648), // 5^308 ]; diff --git a/crux-mir/lib/core/src/num/diy_float.rs b/crux-mir/lib/core/src/num/diy_float.rs index 0a609417d..ce7f6475d 100644 --- a/crux-mir/lib/core/src/num/diy_float.rs +++ b/crux-mir/lib/core/src/num/diy_float.rs @@ -65,7 +65,7 @@ impl Fp { f <<= 1; e -= 1; } - debug_assert!(f >= (1 >> 63)); + debug_assert!(f >= (1 << 63)); Fp { f, e } } diff --git a/crux-mir/lib/core/src/num/error.rs b/crux-mir/lib/core/src/num/error.rs new file mode 100644 index 000000000..768dd8781 --- /dev/null +++ b/crux-mir/lib/core/src/num/error.rs @@ -0,0 +1,163 @@ +//! Error types for conversion to integral types. + +use crate::convert::Infallible; +use crate::error::Error; +use crate::fmt; + +/// The error type returned when a checked integral type conversion fails. +#[stable(feature = "try_from", since = "1.34.0")] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct TryFromIntError(pub(crate) ()); + +impl TryFromIntError { + #[unstable( + feature = "int_error_internals", + reason = "available through Error trait and this method should \ + not be exposed publicly", + issue = "none" + )] + #[doc(hidden)] + pub fn __description(&self) -> &str { + "out of range integral type conversion attempted" + } +} + +#[stable(feature = "try_from", since = "1.34.0")] +impl fmt::Display for TryFromIntError { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + self.__description().fmt(fmt) + } +} + +#[stable(feature = "try_from", since = "1.34.0")] +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const From for TryFromIntError { + fn from(x: Infallible) -> TryFromIntError { + match x {} + } +} + +#[unstable(feature = "never_type", issue = "35121")] +impl const From for TryFromIntError { + fn from(never: !) -> TryFromIntError { + // Match rather than coerce to make sure that code like + // `From for TryFromIntError` above will keep working + // when `Infallible` becomes an alias to `!`. + match never {} + } +} + +/// An error which can be returned when parsing an integer. +/// +/// This error is used as the error type for the `from_str_radix()` functions +/// on the primitive integer types, such as [`i8::from_str_radix`]. +/// +/// # Potential causes +/// +/// Among other causes, `ParseIntError` can be thrown because of leading or trailing whitespace +/// in the string e.g., when it is obtained from the standard input. +/// Using the [`str::trim()`] method ensures that no whitespace remains before parsing. +/// +/// # Example +/// +/// ``` +/// if let Err(e) = i32::from_str_radix("a12", 10) { +/// println!("Failed conversion to i32: {e}"); +/// } +/// ``` +#[derive(Debug, Clone, PartialEq, Eq)] +#[stable(feature = "rust1", since = "1.0.0")] +pub struct ParseIntError { + pub(super) kind: IntErrorKind, +} + +/// Enum to store the various types of errors that can cause parsing an integer to fail. +/// +/// # Example +/// +/// ``` +/// # fn main() { +/// if let Err(e) = i32::from_str_radix("a12", 10) { +/// println!("Failed conversion to i32: {:?}", e.kind()); +/// } +/// # } +/// ``` +#[stable(feature = "int_error_matching", since = "1.55.0")] +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum IntErrorKind { + /// Value being parsed is empty. + /// + /// This variant will be constructed when parsing an empty string. + #[stable(feature = "int_error_matching", since = "1.55.0")] + Empty, + /// Contains an invalid digit in its context. + /// + /// Among other causes, this variant will be constructed when parsing a string that + /// contains a non-ASCII char. + /// + /// This variant is also constructed when a `+` or `-` is misplaced within a string + /// either on its own or in the middle of a number. + #[stable(feature = "int_error_matching", since = "1.55.0")] + InvalidDigit, + /// Integer is too large to store in target integer type. + #[stable(feature = "int_error_matching", since = "1.55.0")] + PosOverflow, + /// Integer is too small to store in target integer type. + #[stable(feature = "int_error_matching", since = "1.55.0")] + NegOverflow, + /// Value was Zero + /// + /// This variant will be emitted when the parsing string has a value of zero, which + /// would be illegal for non-zero types. + #[stable(feature = "int_error_matching", since = "1.55.0")] + Zero, +} + +impl ParseIntError { + /// Outputs the detailed cause of parsing an integer failing. + #[must_use] + #[stable(feature = "int_error_matching", since = "1.55.0")] + pub fn kind(&self) -> &IntErrorKind { + &self.kind + } + #[unstable( + feature = "int_error_internals", + reason = "available through Error trait and this method should \ + not be exposed publicly", + issue = "none" + )] + #[doc(hidden)] + pub fn __description(&self) -> &str { + match self.kind { + IntErrorKind::Empty => "cannot parse integer from empty string", + IntErrorKind::InvalidDigit => "invalid digit found in string", + IntErrorKind::PosOverflow => "number too large to fit in target type", + IntErrorKind::NegOverflow => "number too small to fit in target type", + IntErrorKind::Zero => "number would be zero for non-zero type", + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Display for ParseIntError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.__description().fmt(f) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Error for ParseIntError { + #[allow(deprecated)] + fn description(&self) -> &str { + self.__description() + } +} + +#[stable(feature = "try_from", since = "1.34.0")] +impl Error for TryFromIntError { + #[allow(deprecated)] + fn description(&self) -> &str { + self.__description() + } +} diff --git a/crux-mir/lib/core/src/num/f32.rs b/crux-mir/lib/core/src/num/f32.rs index 3fdc2bae3..1308b0770 100644 --- a/crux-mir/lib/core/src/num/f32.rs +++ b/crux-mir/lib/core/src/num/f32.rs @@ -1,12 +1,13 @@ -//! This module provides constants which are specific to the implementation -//! of the `f32` floating point data type. +//! Constants for the `f32` single-precision floating point type. //! -//! *[See also the `f32` primitive type](../../std/primitive.f32.html).* +//! *[See also the `f32` primitive type][f32].* //! //! Mathematically significant numbers are provided in the `consts` sub-module. //! -//! Although using these constants won’t cause compilation warnings, -//! new code should use the associated constants directly on the primitive type. +//! For the constants defined directly in this module +//! (as distinct from those defined in the `consts` sub-module), +//! new code should instead use the associated constants +//! defined directly on the `f32` type. #![stable(feature = "rust1", since = "1.0.0")] @@ -17,70 +18,248 @@ use crate::mem; use crate::num::FpCategory; /// The radix or base of the internal representation of `f32`. -/// Use [`f32::RADIX`](../../std/primitive.f32.html#associatedconstant.RADIX) instead. +/// Use [`f32::RADIX`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let r = std::f32::RADIX; +/// +/// // intended way +/// let r = f32::RADIX; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `RADIX` associated constant on `f32`")] pub const RADIX: u32 = f32::RADIX; /// Number of significant digits in base 2. -/// Use [`f32::MANTISSA_DIGITS`](../../std/primitive.f32.html#associatedconstant.MANTISSA_DIGITS) instead. +/// Use [`f32::MANTISSA_DIGITS`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let d = std::f32::MANTISSA_DIGITS; +/// +/// // intended way +/// let d = f32::MANTISSA_DIGITS; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated( + since = "TBD", + note = "replaced by the `MANTISSA_DIGITS` associated constant on `f32`" +)] pub const MANTISSA_DIGITS: u32 = f32::MANTISSA_DIGITS; + /// Approximate number of significant digits in base 10. -/// Use [`f32::DIGITS`](../../std/primitive.f32.html#associatedconstant.DIGITS) instead. +/// Use [`f32::DIGITS`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let d = std::f32::DIGITS; +/// +/// // intended way +/// let d = f32::DIGITS; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `DIGITS` associated constant on `f32`")] pub const DIGITS: u32 = f32::DIGITS; /// [Machine epsilon] value for `f32`. -/// Use [`f32::EPSILON`](../../std/primitive.f32.html#associatedconstant.EPSILON) instead. +/// Use [`f32::EPSILON`] instead. /// /// This is the difference between `1.0` and the next larger representable number. /// /// [Machine epsilon]: https://en.wikipedia.org/wiki/Machine_epsilon +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let e = std::f32::EPSILON; +/// +/// // intended way +/// let e = f32::EPSILON; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `EPSILON` associated constant on `f32`")] pub const EPSILON: f32 = f32::EPSILON; /// Smallest finite `f32` value. -/// Use [`f32::MIN`](../../std/primitive.f32.html#associatedconstant.MIN) instead. +/// Use [`f32::MIN`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let min = std::f32::MIN; +/// +/// // intended way +/// let min = f32::MIN; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `MIN` associated constant on `f32`")] pub const MIN: f32 = f32::MIN; + /// Smallest positive normal `f32` value. -/// Use [`f32::MIN_POSITIVE`](../../std/primitive.f32.html#associatedconstant.MIN_POSITIVE) instead. +/// Use [`f32::MIN_POSITIVE`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let min = std::f32::MIN_POSITIVE; +/// +/// // intended way +/// let min = f32::MIN_POSITIVE; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `MIN_POSITIVE` associated constant on `f32`")] pub const MIN_POSITIVE: f32 = f32::MIN_POSITIVE; + /// Largest finite `f32` value. -/// Use [`f32::MAX`](../../std/primitive.f32.html#associatedconstant.MAX) instead. +/// Use [`f32::MAX`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let max = std::f32::MAX; +/// +/// // intended way +/// let max = f32::MAX; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `MAX` associated constant on `f32`")] pub const MAX: f32 = f32::MAX; /// One greater than the minimum possible normal power of 2 exponent. -/// Use [`f32::MIN_EXP`](../../std/primitive.f32.html#associatedconstant.MIN_EXP) instead. +/// Use [`f32::MIN_EXP`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let min = std::f32::MIN_EXP; +/// +/// // intended way +/// let min = f32::MIN_EXP; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `MIN_EXP` associated constant on `f32`")] pub const MIN_EXP: i32 = f32::MIN_EXP; + /// Maximum possible power of 2 exponent. -/// Use [`f32::MAX_EXP`](../../std/primitive.f32.html#associatedconstant.MAX_EXP) instead. +/// Use [`f32::MAX_EXP`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let max = std::f32::MAX_EXP; +/// +/// // intended way +/// let max = f32::MAX_EXP; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `MAX_EXP` associated constant on `f32`")] pub const MAX_EXP: i32 = f32::MAX_EXP; /// Minimum possible normal power of 10 exponent. -/// Use [`f32::MIN_10_EXP`](../../std/primitive.f32.html#associatedconstant.MIN_10_EXP) instead. +/// Use [`f32::MIN_10_EXP`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let min = std::f32::MIN_10_EXP; +/// +/// // intended way +/// let min = f32::MIN_10_EXP; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `MIN_10_EXP` associated constant on `f32`")] pub const MIN_10_EXP: i32 = f32::MIN_10_EXP; + /// Maximum possible power of 10 exponent. -/// Use [`f32::MAX_10_EXP`](../../std/primitive.f32.html#associatedconstant.MAX_10_EXP) instead. +/// Use [`f32::MAX_10_EXP`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let max = std::f32::MAX_10_EXP; +/// +/// // intended way +/// let max = f32::MAX_10_EXP; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `MAX_10_EXP` associated constant on `f32`")] pub const MAX_10_EXP: i32 = f32::MAX_10_EXP; /// Not a Number (NaN). -/// Use [`f32::NAN`](../../std/primitive.f32.html#associatedconstant.NAN) instead. +/// Use [`f32::NAN`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let nan = std::f32::NAN; +/// +/// // intended way +/// let nan = f32::NAN; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `NAN` associated constant on `f32`")] pub const NAN: f32 = f32::NAN; + /// Infinity (∞). -/// Use [`f32::INFINITY`](../../std/primitive.f32.html#associatedconstant.INFINITY) instead. +/// Use [`f32::INFINITY`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let inf = std::f32::INFINITY; +/// +/// // intended way +/// let inf = f32::INFINITY; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `INFINITY` associated constant on `f32`")] pub const INFINITY: f32 = f32::INFINITY; + /// Negative infinity (−∞). -/// Use [`f32::NEG_INFINITY`](../../std/primitive.f32.html#associatedconstant.NEG_INFINITY) instead. +/// Use [`f32::NEG_INFINITY`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let ninf = std::f32::NEG_INFINITY; +/// +/// // intended way +/// let ninf = f32::NEG_INFINITY; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `NEG_INFINITY` associated constant on `f32`")] pub const NEG_INFINITY: f32 = f32::NEG_INFINITY; /// Basic mathematical constants. @@ -95,7 +274,7 @@ pub mod consts { /// The full circle constant (τ) /// /// Equal to 2π. - #[unstable(feature = "tau_constant", issue = "66770")] + #[stable(feature = "tau_constant", since = "1.47.0")] pub const TAU: f32 = 6.28318530717958647692528676655900577_f32; /// π/2 @@ -167,7 +346,6 @@ pub mod consts { pub const LN_10: f32 = 2.30258509299404568401799145468436421_f32; } -#[lang = "f32"] #[cfg(not(test))] impl f32 { /// The radix or base of the internal representation of `f32`. @@ -215,16 +393,25 @@ impl f32 { pub const MAX_10_EXP: i32 = 38; /// Not a Number (NaN). + /// + /// Note that IEEE 754 doesn't define just a single NaN value; + /// a plethora of bit patterns are considered to be NaN. + /// Furthermore, the standard makes a difference + /// between a "signaling" and a "quiet" NaN, + /// and allows inspecting its "payload" (the unspecified bits in the bit pattern). + /// This constant isn't guaranteed to equal to any specific NaN bitpattern, + /// and the stability of its representation over Rust versions + /// and target platforms isn't guaranteed. #[stable(feature = "assoc_int_consts", since = "1.43.0")] pub const NAN: f32 = 0.0_f32 / 0.0_f32; /// Infinity (∞). #[stable(feature = "assoc_int_consts", since = "1.43.0")] pub const INFINITY: f32 = 1.0_f32 / 0.0_f32; - /// Negative infinity (-∞). + /// Negative infinity (−∞). #[stable(feature = "assoc_int_consts", since = "1.43.0")] pub const NEG_INFINITY: f32 = -1.0_f32 / 0.0_f32; - /// Returns `true` if this value is `NaN`. + /// Returns `true` if this value is NaN. /// /// ``` /// let nan = f32::NAN; @@ -233,18 +420,22 @@ impl f32 { /// assert!(nan.is_nan()); /// assert!(!f.is_nan()); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] #[inline] - pub fn is_nan(self) -> bool { + pub const fn is_nan(self) -> bool { self != self } - // FIXME(#50145): `abs` is publicly unavailable in libcore due to + // FIXME(#50145): `abs` is publicly unavailable in core due to // concerns about portability, so this implementation is for // private use internally. #[inline] - fn abs_private(self) -> f32 { - f32::from_bits(self.to_bits() & 0x7fff_ffff) + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] + pub(crate) const fn abs_private(self) -> f32 { + // SAFETY: This transmutation is fine. Probably. For the reasons std is using it. + unsafe { mem::transmute::(mem::transmute::(self) & 0x7fff_ffff) } } /// Returns `true` if this value is positive infinity or negative infinity, and @@ -262,13 +453,18 @@ impl f32 { /// assert!(inf.is_infinite()); /// assert!(neg_inf.is_infinite()); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] #[inline] - pub fn is_infinite(self) -> bool { - self.abs_private() == INFINITY + pub const fn is_infinite(self) -> bool { + // Getting clever with transmutation can result in incorrect answers on some FPUs + // FIXME: alter the Rust <-> Rust calling convention to prevent this problem. + // See https://github.com/rust-lang/rust/issues/72327 + (self == f32::INFINITY) | (self == f32::NEG_INFINITY) } - /// Returns `true` if this number is neither infinite nor `NaN`. + /// Returns `true` if this number is neither infinite nor NaN. /// /// ``` /// let f = 7.0f32; @@ -282,16 +478,44 @@ impl f32 { /// assert!(!inf.is_finite()); /// assert!(!neg_inf.is_finite()); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] #[inline] - pub fn is_finite(self) -> bool { + pub const fn is_finite(self) -> bool { // There's no need to handle NaN separately: if self is NaN, // the comparison is not true, exactly as desired. - self.abs_private() < INFINITY + self.abs_private() < Self::INFINITY + } + + /// Returns `true` if the number is [subnormal]. + /// + /// ``` + /// let min = f32::MIN_POSITIVE; // 1.17549435e-38f32 + /// let max = f32::MAX; + /// let lower_than_min = 1.0e-40_f32; + /// let zero = 0.0_f32; + /// + /// assert!(!min.is_subnormal()); + /// assert!(!max.is_subnormal()); + /// + /// assert!(!zero.is_subnormal()); + /// assert!(!f32::NAN.is_subnormal()); + /// assert!(!f32::INFINITY.is_subnormal()); + /// // Values between `0` and `min` are Subnormal. + /// assert!(lower_than_min.is_subnormal()); + /// ``` + /// [subnormal]: https://en.wikipedia.org/wiki/Denormal_number + #[must_use] + #[stable(feature = "is_subnormal", since = "1.53.0")] + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] + #[inline] + pub const fn is_subnormal(self) -> bool { + matches!(self.classify(), FpCategory::Subnormal) } /// Returns `true` if the number is neither zero, infinite, - /// [subnormal], or `NaN`. + /// [subnormal], or NaN. /// /// ``` /// let min = f32::MIN_POSITIVE; // 1.17549435e-38f32 @@ -309,10 +533,12 @@ impl f32 { /// assert!(!lower_than_min.is_normal()); /// ``` /// [subnormal]: https://en.wikipedia.org/wiki/Denormal_number + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] #[inline] - pub fn is_normal(self) -> bool { - self.classify() == FpCategory::Normal + pub const fn is_normal(self) -> bool { + matches!(self.classify(), FpCategory::Normal) } /// Returns the floating point category of the number. If only one property @@ -329,22 +555,88 @@ impl f32 { /// assert_eq!(inf.classify(), FpCategory::Infinite); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn classify(self) -> FpCategory { + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] + pub const fn classify(self) -> FpCategory { + // A previous implementation tried to only use bitmask-based checks, + // using f32::to_bits to transmute the float to its bit repr and match on that. + // Unfortunately, floating point numbers can be much worse than that. + // This also needs to not result in recursive evaluations of f64::to_bits. + // + // On some processors, in some cases, LLVM will "helpfully" lower floating point ops, + // in spite of a request for them using f32 and f64, to things like x87 operations. + // These have an f64's mantissa, but can have a larger than normal exponent. + // FIXME(jubilee): Using x87 operations is never necessary in order to function + // on x86 processors for Rust-to-Rust calls, so this issue should not happen. + // Code generation should be adjusted to use non-C calling conventions, avoiding this. + // + if self.is_infinite() { + // Thus, a value may compare unequal to infinity, despite having a "full" exponent mask. + FpCategory::Infinite + } else if self.is_nan() { + // And it may not be NaN, as it can simply be an "overextended" finite value. + FpCategory::Nan + } else { + // However, std can't simply compare to zero to check for zero, either, + // as correctness requires avoiding equality tests that may be Subnormal == -0.0 + // because it may be wrong under "denormals are zero" and "flush to zero" modes. + // Most of std's targets don't use those, but they are used for thumbv7neon. + // So, this does use bitpattern matching for the rest. + + // SAFETY: f32 to u32 is fine. Usually. + // If classify has gotten this far, the value is definitely in one of these categories. + unsafe { f32::partial_classify(self) } + } + } + + // This doesn't actually return a right answer for NaN on purpose, + // seeing as how it cannot correctly discern between a floating point NaN, + // and some normal floating point numbers truncated from an x87 FPU. + // FIXME(jubilee): This probably could at least answer things correctly for Infinity, + // like the f64 version does, but I need to run more checks on how things go on x86. + // I fear losing mantissa data that would have answered that differently. + // + // # Safety + // This requires making sure you call this function for values it answers correctly on, + // otherwise it returns a wrong answer. This is not important for memory safety per se, + // but getting floats correct is important for not accidentally leaking const eval + // runtime-deviating logic which may or may not be acceptable. + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] + const unsafe fn partial_classify(self) -> FpCategory { const EXP_MASK: u32 = 0x7f800000; const MAN_MASK: u32 = 0x007fffff; - let bits = self.to_bits(); - match (bits & MAN_MASK, bits & EXP_MASK) { + // SAFETY: The caller is not asking questions for which this will tell lies. + let b = unsafe { mem::transmute::(self) }; + match (b & MAN_MASK, b & EXP_MASK) { (0, 0) => FpCategory::Zero, (_, 0) => FpCategory::Subnormal, + _ => FpCategory::Normal, + } + } + + // This operates on bits, and only bits, so it can ignore concerns about weird FPUs. + // FIXME(jubilee): In a just world, this would be the entire impl for classify, + // plus a transmute. We do not live in a just world, but we can make it more so. + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] + const fn classify_bits(b: u32) -> FpCategory { + const EXP_MASK: u32 = 0x7f800000; + const MAN_MASK: u32 = 0x007fffff; + + match (b & MAN_MASK, b & EXP_MASK) { (0, EXP_MASK) => FpCategory::Infinite, (_, EXP_MASK) => FpCategory::Nan, + (0, 0) => FpCategory::Zero, + (_, 0) => FpCategory::Subnormal, _ => FpCategory::Normal, } } - /// Returns `true` if `self` has a positive sign, including `+0.0`, `NaN`s with - /// positive sign bit and positive infinity. + /// Returns `true` if `self` has a positive sign, including `+0.0`, NaNs with + /// positive sign bit and positive infinity. Note that IEEE 754 doesn't assign any + /// meaning to the sign bit in case of a NaN, and as Rust doesn't guarantee that + /// the bit pattern of NaNs are conserved over arithmetic operations, the result of + /// `is_sign_positive` on a NaN might produce an unexpected result in some cases. + /// See [explanation of NaN as a special value](f32) for more info. /// /// ``` /// let f = 7.0_f32; @@ -353,14 +645,20 @@ impl f32 { /// assert!(f.is_sign_positive()); /// assert!(!g.is_sign_positive()); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] #[inline] - pub fn is_sign_positive(self) -> bool { + pub const fn is_sign_positive(self) -> bool { !self.is_sign_negative() } - /// Returns `true` if `self` has a negative sign, including `-0.0`, `NaN`s with - /// negative sign bit and negative infinity. + /// Returns `true` if `self` has a negative sign, including `-0.0`, NaNs with + /// negative sign bit and negative infinity. Note that IEEE 754 doesn't assign any + /// meaning to the sign bit in case of a NaN, and as Rust doesn't guarantee that + /// the bit pattern of NaNs are conserved over arithmetic operations, the result of + /// `is_sign_negative` on a NaN might produce an unexpected result in some cases. + /// See [explanation of NaN as a special value](f32) for more info. /// /// ``` /// let f = 7.0f32; @@ -369,12 +667,115 @@ impl f32 { /// assert!(!f.is_sign_negative()); /// assert!(g.is_sign_negative()); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] #[inline] - pub fn is_sign_negative(self) -> bool { + pub const fn is_sign_negative(self) -> bool { // IEEE754 says: isSignMinus(x) is true if and only if x has negative sign. isSignMinus // applies to zeros and NaNs as well. - self.to_bits() & 0x8000_0000 != 0 + // SAFETY: This is just transmuting to get the sign bit, it's fine. + unsafe { mem::transmute::(self) & 0x8000_0000 != 0 } + } + + /// Returns the least number greater than `self`. + /// + /// Let `TINY` be the smallest representable positive `f32`. Then, + /// - if `self.is_nan()`, this returns `self`; + /// - if `self` is [`NEG_INFINITY`], this returns [`MIN`]; + /// - if `self` is `-TINY`, this returns -0.0; + /// - if `self` is -0.0 or +0.0, this returns `TINY`; + /// - if `self` is [`MAX`] or [`INFINITY`], this returns [`INFINITY`]; + /// - otherwise the unique least value greater than `self` is returned. + /// + /// The identity `x.next_up() == -(-x).next_down()` holds for all non-NaN `x`. When `x` + /// is finite `x == x.next_up().next_down()` also holds. + /// + /// ```rust + /// #![feature(float_next_up_down)] + /// // f32::EPSILON is the difference between 1.0 and the next number up. + /// assert_eq!(1.0f32.next_up(), 1.0 + f32::EPSILON); + /// // But not for most numbers. + /// assert!(0.1f32.next_up() < 0.1 + f32::EPSILON); + /// assert_eq!(16777216f32.next_up(), 16777218.0); + /// ``` + /// + /// [`NEG_INFINITY`]: Self::NEG_INFINITY + /// [`INFINITY`]: Self::INFINITY + /// [`MIN`]: Self::MIN + /// [`MAX`]: Self::MAX + #[unstable(feature = "float_next_up_down", issue = "91399")] + #[rustc_const_unstable(feature = "float_next_up_down", issue = "91399")] + pub const fn next_up(self) -> Self { + // We must use strictly integer arithmetic to prevent denormals from + // flushing to zero after an arithmetic operation on some platforms. + const TINY_BITS: u32 = 0x1; // Smallest positive f32. + const CLEAR_SIGN_MASK: u32 = 0x7fff_ffff; + + let bits = self.to_bits(); + if self.is_nan() || bits == Self::INFINITY.to_bits() { + return self; + } + + let abs = bits & CLEAR_SIGN_MASK; + let next_bits = if abs == 0 { + TINY_BITS + } else if bits == abs { + bits + 1 + } else { + bits - 1 + }; + Self::from_bits(next_bits) + } + + /// Returns the greatest number less than `self`. + /// + /// Let `TINY` be the smallest representable positive `f32`. Then, + /// - if `self.is_nan()`, this returns `self`; + /// - if `self` is [`INFINITY`], this returns [`MAX`]; + /// - if `self` is `TINY`, this returns 0.0; + /// - if `self` is -0.0 or +0.0, this returns `-TINY`; + /// - if `self` is [`MIN`] or [`NEG_INFINITY`], this returns [`NEG_INFINITY`]; + /// - otherwise the unique greatest value less than `self` is returned. + /// + /// The identity `x.next_down() == -(-x).next_up()` holds for all non-NaN `x`. When `x` + /// is finite `x == x.next_down().next_up()` also holds. + /// + /// ```rust + /// #![feature(float_next_up_down)] + /// let x = 1.0f32; + /// // Clamp value into range [0, 1). + /// let clamped = x.clamp(0.0, 1.0f32.next_down()); + /// assert!(clamped < 1.0); + /// assert_eq!(clamped.next_up(), 1.0); + /// ``` + /// + /// [`NEG_INFINITY`]: Self::NEG_INFINITY + /// [`INFINITY`]: Self::INFINITY + /// [`MIN`]: Self::MIN + /// [`MAX`]: Self::MAX + #[unstable(feature = "float_next_up_down", issue = "91399")] + #[rustc_const_unstable(feature = "float_next_up_down", issue = "91399")] + pub const fn next_down(self) -> Self { + // We must use strictly integer arithmetic to prevent denormals from + // flushing to zero after an arithmetic operation on some platforms. + const NEG_TINY_BITS: u32 = 0x8000_0001; // Smallest (in magnitude) negative f32. + const CLEAR_SIGN_MASK: u32 = 0x7fff_ffff; + + let bits = self.to_bits(); + if self.is_nan() || bits == Self::NEG_INFINITY.to_bits() { + return self; + } + + let abs = bits & CLEAR_SIGN_MASK; + let next_bits = if abs == 0 { + NEG_TINY_BITS + } else if bits == abs { + bits - 1 + } else { + bits + 1 + }; + Self::from_bits(next_bits) } /// Takes the reciprocal (inverse) of a number, `1/x`. @@ -385,6 +786,7 @@ impl f32 { /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[must_use = "this returns the result of the operation, without modifying the original"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn recip(self) -> f32 { @@ -394,14 +796,14 @@ impl f32 { /// Converts radians to degrees. /// /// ``` - /// use std::f32::consts; - /// - /// let angle = consts::PI; + /// let angle = std::f32::consts::PI; /// /// let abs_difference = (angle.to_degrees() - 180.0).abs(); /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[stable(feature = "f32_deg_rad_conversions", since = "1.7.0")] #[inline] pub fn to_degrees(self) -> f32 { @@ -413,14 +815,14 @@ impl f32 { /// Converts degrees to radians. /// /// ``` - /// use std::f32::consts; - /// /// let angle = 180.0f32; /// - /// let abs_difference = (angle.to_radians() - consts::PI).abs(); + /// let abs_difference = (angle.to_radians() - std::f32::consts::PI).abs(); /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[stable(feature = "f32_deg_rad_conversions", since = "1.7.0")] #[inline] pub fn to_radians(self) -> f32 { @@ -428,7 +830,12 @@ impl f32 { self * (value / 180.0f32) } - /// Returns the maximum of the two numbers. + /// Returns the maximum of the two numbers, ignoring NaN. + /// + /// If one of the arguments is NaN, then the other argument is returned. + /// This follows the IEEE 754-2008 semantics for maxNum, except for handling of signaling NaNs; + /// this function handles all NaNs the same way and avoids maxNum's problems with associativity. + /// This also matches the behavior of libm’s fmax. /// /// ``` /// let x = 1.0f32; @@ -436,15 +843,19 @@ impl f32 { /// /// assert_eq!(x.max(y), y); /// ``` - /// - /// If one of the arguments is NaN, then the other argument is returned. + #[must_use = "this returns the result of the comparison, without modifying either input"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn max(self, other: f32) -> f32 { intrinsics::maxnumf32(self, other) } - /// Returns the minimum of the two numbers. + /// Returns the minimum of the two numbers, ignoring NaN. + /// + /// If one of the arguments is NaN, then the other argument is returned. + /// This follows the IEEE 754-2008 semantics for minNum, except for handling of signaling NaNs; + /// this function handles all NaNs the same way and avoids minNum's problems with associativity. + /// This also matches the behavior of libm’s fmin. /// /// ``` /// let x = 1.0f32; @@ -452,27 +863,94 @@ impl f32 { /// /// assert_eq!(x.min(y), x); /// ``` - /// - /// If one of the arguments is NaN, then the other argument is returned. + #[must_use = "this returns the result of the comparison, without modifying either input"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn min(self, other: f32) -> f32 { intrinsics::minnumf32(self, other) } + /// Returns the maximum of the two numbers, propagating NaN. + /// + /// This returns NaN when *either* argument is NaN, as opposed to + /// [`f32::max`] which only returns NaN when *both* arguments are NaN. + /// + /// ``` + /// #![feature(float_minimum_maximum)] + /// let x = 1.0f32; + /// let y = 2.0f32; + /// + /// assert_eq!(x.maximum(y), y); + /// assert!(x.maximum(f32::NAN).is_nan()); + /// ``` + /// + /// If one of the arguments is NaN, then NaN is returned. Otherwise this returns the greater + /// of the two numbers. For this operation, -0.0 is considered to be less than +0.0. + /// Note that this follows the semantics specified in IEEE 754-2019. + /// + /// Also note that "propagation" of NaNs here doesn't necessarily mean that the bitpattern of a NaN + /// operand is conserved; see [explanation of NaN as a special value](f32) for more info. + #[must_use = "this returns the result of the comparison, without modifying either input"] + #[unstable(feature = "float_minimum_maximum", issue = "91079")] + #[inline] + pub fn maximum(self, other: f32) -> f32 { + if self > other { + self + } else if other > self { + other + } else if self == other { + if self.is_sign_positive() && other.is_sign_negative() { self } else { other } + } else { + self + other + } + } + + /// Returns the minimum of the two numbers, propagating NaN. + /// + /// This returns NaN when *either* argument is NaN, as opposed to + /// [`f32::min`] which only returns NaN when *both* arguments are NaN. + /// + /// ``` + /// #![feature(float_minimum_maximum)] + /// let x = 1.0f32; + /// let y = 2.0f32; + /// + /// assert_eq!(x.minimum(y), x); + /// assert!(x.minimum(f32::NAN).is_nan()); + /// ``` + /// + /// If one of the arguments is NaN, then NaN is returned. Otherwise this returns the lesser + /// of the two numbers. For this operation, -0.0 is considered to be less than +0.0. + /// Note that this follows the semantics specified in IEEE 754-2019. + /// + /// Also note that "propagation" of NaNs here doesn't necessarily mean that the bitpattern of a NaN + /// operand is conserved; see [explanation of NaN as a special value](f32) for more info. + #[must_use = "this returns the result of the comparison, without modifying either input"] + #[unstable(feature = "float_minimum_maximum", issue = "91079")] + #[inline] + pub fn minimum(self, other: f32) -> f32 { + if self < other { + self + } else if other < self { + other + } else if self == other { + if self.is_sign_negative() && other.is_sign_positive() { self } else { other } + } else { + self + other + } + } + /// Rounds toward zero and converts to any primitive integer type, /// assuming that the value is finite and fits in that type. /// /// ``` - /// #![feature(float_approx_unchecked_to)] - /// /// let value = 4.6_f32; - /// let rounded = unsafe { value.approx_unchecked_to::() }; + /// let rounded = unsafe { value.to_int_unchecked::() }; /// assert_eq!(rounded, 4); /// /// let value = -128.9_f32; - /// let rounded = unsafe { value.approx_unchecked_to::() }; - /// assert_eq!(rounded, std::i8::MIN); + /// let rounded = unsafe { value.to_int_unchecked::() }; + /// assert_eq!(rounded, i8::MIN); /// ``` /// /// # Safety @@ -482,21 +960,25 @@ impl f32 { /// * Not be `NaN` /// * Not be infinite /// * Be representable in the return type `Int`, after truncating off its fractional part - #[unstable(feature = "float_approx_unchecked_to", issue = "67058")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[stable(feature = "float_approx_unchecked_to", since = "1.44.0")] #[inline] - pub unsafe fn approx_unchecked_to(self) -> Int + pub unsafe fn to_int_unchecked(self) -> Int where Self: FloatToInt, { - FloatToInt::::approx_unchecked(self) + // SAFETY: the caller must uphold the safety contract for + // `FloatToInt::to_int_unchecked`. + unsafe { FloatToInt::::to_int_unchecked(self) } } /// Raw transmutation to `u32`. /// /// This is currently identical to `transmute::(self)` on all platforms. /// - /// See `from_bits` for some discussion of the portability of this operation - /// (there are almost no issues). + /// See [`from_bits`](Self::from_bits) for some discussion of the + /// portability of this operation (there are almost no issues). /// /// Note that this function is distinct from `as` casting, which attempts to /// preserve the *numeric* value, and not the bitwise value. @@ -508,11 +990,59 @@ impl f32 { /// assert_eq!((12.5f32).to_bits(), 0x41480000); /// /// ``` + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[stable(feature = "float_bits_conv", since = "1.20.0")] + #[rustc_const_unstable(feature = "const_float_bits_conv", issue = "72447")] #[inline] - pub fn to_bits(self) -> u32 { - // SAFETY: `u32` is a plain old datatype so we can always transmute to it - unsafe { mem::transmute(self) } + pub const fn to_bits(self) -> u32 { + // SAFETY: `u32` is a plain old datatype so we can always transmute to it. + // ...sorta. + // + // It turns out that at runtime, it is possible for a floating point number + // to be subject to a floating point mode that alters nonzero subnormal numbers + // to zero on reads and writes, aka "denormals are zero" and "flush to zero". + // This is not a problem per se, but at least one tier2 platform for Rust + // actually exhibits this behavior by default. + // + // In addition, on x86 targets with SSE or SSE2 disabled and the x87 FPU enabled, + // i.e. not soft-float, the way Rust does parameter passing can actually alter + // a number that is "not infinity" to have the same exponent as infinity, + // in a slightly unpredictable manner. + // + // And, of course evaluating to a NaN value is fairly nondeterministic. + // More precisely: when NaN should be returned is knowable, but which NaN? + // So far that's defined by a combination of LLVM and the CPU, not Rust. + // This function, however, allows observing the bitstring of a NaN, + // thus introspection on CTFE. + // + // In order to preserve, at least for the moment, const-to-runtime equivalence, + // we reject any of these possible situations from happening. + #[rustc_const_unstable(feature = "const_float_bits_conv", issue = "72447")] + const fn ct_f32_to_u32(ct: f32) -> u32 { + match ct.classify() { + FpCategory::Nan => { + panic!("const-eval error: cannot use f32::to_bits on a NaN") + } + FpCategory::Subnormal => { + panic!("const-eval error: cannot use f32::to_bits on a subnormal number") + } + FpCategory::Infinite | FpCategory::Normal | FpCategory::Zero => { + // SAFETY: We have a normal floating point number. Now we transmute, i.e. do a bitcopy. + unsafe { mem::transmute::(ct) } + } + } + } + + #[inline(always)] // See https://github.com/rust-lang/compiler-builtins/issues/491 + fn rt_f32_to_u32(x: f32) -> u32 { + // SAFETY: `u32` is a plain old datatype so we can always... uh... + // ...look, just pretend you forgot what you just read. + // Stability concerns. + unsafe { mem::transmute(x) } + } + // SAFETY: We use internal implementations that either always work or fail at compile time. + unsafe { intrinsics::const_eval_select((self,), ct_f32_to_u32, rt_f32_to_u32) } } /// Raw transmutation from `u32`. @@ -521,9 +1051,9 @@ impl f32 { /// It turns out this is incredibly portable, for two reasons: /// /// * Floats and Ints have the same endianness on all supported platforms. - /// * IEEE-754 very precisely specifies the bit layout of floats. + /// * IEEE 754 very precisely specifies the bit layout of floats. /// - /// However there is one caveat: prior to the 2008 version of IEEE-754, how + /// However there is one caveat: prior to the 2008 version of IEEE 754, how /// to interpret the NaN signaling bit wasn't actually specified. Most platforms /// (notably x86 and ARM) picked the interpretation that was ultimately /// standardized in 2008, but some didn't (notably MIPS). As a result, all @@ -552,40 +1082,100 @@ impl f32 { /// assert_eq!(v, 12.5); /// ``` #[stable(feature = "float_bits_conv", since = "1.20.0")] + #[rustc_const_unstable(feature = "const_float_bits_conv", issue = "72447")] + #[must_use] #[inline] - pub fn from_bits(v: u32) -> Self { - // SAFETY: `u32` is a plain old datatype so we can always transmute from it + pub const fn from_bits(v: u32) -> Self { // It turns out the safety issues with sNaN were overblown! Hooray! - unsafe { mem::transmute(v) } + // SAFETY: `u32` is a plain old datatype so we can always transmute from it + // ...sorta. + // + // It turns out that at runtime, it is possible for a floating point number + // to be subject to floating point modes that alter nonzero subnormal numbers + // to zero on reads and writes, aka "denormals are zero" and "flush to zero". + // This is not a problem usually, but at least one tier2 platform for Rust + // actually exhibits this behavior by default: thumbv7neon + // aka "the Neon FPU in AArch32 state" + // + // In addition, on x86 targets with SSE or SSE2 disabled and the x87 FPU enabled, + // i.e. not soft-float, the way Rust does parameter passing can actually alter + // a number that is "not infinity" to have the same exponent as infinity, + // in a slightly unpredictable manner. + // + // And, of course evaluating to a NaN value is fairly nondeterministic. + // More precisely: when NaN should be returned is knowable, but which NaN? + // So far that's defined by a combination of LLVM and the CPU, not Rust. + // This function, however, allows observing the bitstring of a NaN, + // thus introspection on CTFE. + // + // In order to preserve, at least for the moment, const-to-runtime equivalence, + // reject any of these possible situations from happening. + #[rustc_const_unstable(feature = "const_float_bits_conv", issue = "72447")] + const fn ct_u32_to_f32(ct: u32) -> f32 { + match f32::classify_bits(ct) { + FpCategory::Subnormal => { + panic!("const-eval error: cannot use f32::from_bits on a subnormal number") + } + FpCategory::Nan => { + panic!("const-eval error: cannot use f32::from_bits on NaN") + } + FpCategory::Infinite | FpCategory::Normal | FpCategory::Zero => { + // SAFETY: It's not a frumious number + unsafe { mem::transmute::(ct) } + } + } + } + + #[inline(always)] // See https://github.com/rust-lang/compiler-builtins/issues/491 + fn rt_u32_to_f32(x: u32) -> f32 { + // SAFETY: `u32` is a plain old datatype so we can always... uh... + // ...look, just pretend you forgot what you just read. + // Stability concerns. + unsafe { mem::transmute(x) } + } + // SAFETY: We use internal implementations that either always work or fail at compile time. + unsafe { intrinsics::const_eval_select((v,), ct_u32_to_f32, rt_u32_to_f32) } } /// Return the memory representation of this floating point number as a byte array in /// big-endian (network) byte order. /// + /// See [`from_bits`](Self::from_bits) for some discussion of the + /// portability of this operation (there are almost no issues). + /// /// # Examples /// /// ``` /// let bytes = 12.5f32.to_be_bytes(); /// assert_eq!(bytes, [0x41, 0x48, 0x00, 0x00]); /// ``` + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[stable(feature = "float_to_from_bytes", since = "1.40.0")] + #[rustc_const_unstable(feature = "const_float_bits_conv", issue = "72447")] #[inline] - pub fn to_be_bytes(self) -> [u8; 4] { + pub const fn to_be_bytes(self) -> [u8; 4] { self.to_bits().to_be_bytes() } /// Return the memory representation of this floating point number as a byte array in /// little-endian byte order. /// + /// See [`from_bits`](Self::from_bits) for some discussion of the + /// portability of this operation (there are almost no issues). + /// /// # Examples /// /// ``` /// let bytes = 12.5f32.to_le_bytes(); /// assert_eq!(bytes, [0x00, 0x00, 0x48, 0x41]); /// ``` + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[stable(feature = "float_to_from_bytes", since = "1.40.0")] + #[rustc_const_unstable(feature = "const_float_bits_conv", issue = "72447")] #[inline] - pub fn to_le_bytes(self) -> [u8; 4] { + pub const fn to_le_bytes(self) -> [u8; 4] { self.to_bits().to_le_bytes() } @@ -595,8 +1185,11 @@ impl f32 { /// As the target platform's native endianness is used, portable code /// should use [`to_be_bytes`] or [`to_le_bytes`], as appropriate, instead. /// - /// [`to_be_bytes`]: #method.to_be_bytes - /// [`to_le_bytes`]: #method.to_le_bytes + /// [`to_be_bytes`]: f32::to_be_bytes + /// [`to_le_bytes`]: f32::to_le_bytes + /// + /// See [`from_bits`](Self::from_bits) for some discussion of the + /// portability of this operation (there are almost no issues). /// /// # Examples /// @@ -611,14 +1204,20 @@ impl f32 { /// } /// ); /// ``` + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[stable(feature = "float_to_from_bytes", since = "1.40.0")] + #[rustc_const_unstable(feature = "const_float_bits_conv", issue = "72447")] #[inline] - pub fn to_ne_bytes(self) -> [u8; 4] { + pub const fn to_ne_bytes(self) -> [u8; 4] { self.to_bits().to_ne_bytes() } /// Create a floating point value from its representation as a byte array in big endian. /// + /// See [`from_bits`](Self::from_bits) for some discussion of the + /// portability of this operation (there are almost no issues). + /// /// # Examples /// /// ``` @@ -626,13 +1225,18 @@ impl f32 { /// assert_eq!(value, 12.5); /// ``` #[stable(feature = "float_to_from_bytes", since = "1.40.0")] + #[rustc_const_unstable(feature = "const_float_bits_conv", issue = "72447")] + #[must_use] #[inline] - pub fn from_be_bytes(bytes: [u8; 4]) -> Self { + pub const fn from_be_bytes(bytes: [u8; 4]) -> Self { Self::from_bits(u32::from_be_bytes(bytes)) } /// Create a floating point value from its representation as a byte array in little endian. /// + /// See [`from_bits`](Self::from_bits) for some discussion of the + /// portability of this operation (there are almost no issues). + /// /// # Examples /// /// ``` @@ -640,8 +1244,10 @@ impl f32 { /// assert_eq!(value, 12.5); /// ``` #[stable(feature = "float_to_from_bytes", since = "1.40.0")] + #[rustc_const_unstable(feature = "const_float_bits_conv", issue = "72447")] + #[must_use] #[inline] - pub fn from_le_bytes(bytes: [u8; 4]) -> Self { + pub const fn from_le_bytes(bytes: [u8; 4]) -> Self { Self::from_bits(u32::from_le_bytes(bytes)) } @@ -651,8 +1257,11 @@ impl f32 { /// likely wants to use [`from_be_bytes`] or [`from_le_bytes`], as /// appropriate instead. /// - /// [`from_be_bytes`]: #method.from_be_bytes - /// [`from_le_bytes`]: #method.from_le_bytes + /// [`from_be_bytes`]: f32::from_be_bytes + /// [`from_le_bytes`]: f32::from_le_bytes + /// + /// See [`from_bits`](Self::from_bits) for some discussion of the + /// portability of this operation (there are almost no issues). /// /// # Examples /// @@ -665,8 +1274,130 @@ impl f32 { /// assert_eq!(value, 12.5); /// ``` #[stable(feature = "float_to_from_bytes", since = "1.40.0")] + #[rustc_const_unstable(feature = "const_float_bits_conv", issue = "72447")] + #[must_use] #[inline] - pub fn from_ne_bytes(bytes: [u8; 4]) -> Self { + pub const fn from_ne_bytes(bytes: [u8; 4]) -> Self { Self::from_bits(u32::from_ne_bytes(bytes)) } + + /// Return the ordering between `self` and `other`. + /// + /// Unlike the standard partial comparison between floating point numbers, + /// this comparison always produces an ordering in accordance to + /// the `totalOrder` predicate as defined in the IEEE 754 (2008 revision) + /// floating point standard. The values are ordered in the following sequence: + /// + /// - negative quiet NaN + /// - negative signaling NaN + /// - negative infinity + /// - negative numbers + /// - negative subnormal numbers + /// - negative zero + /// - positive zero + /// - positive subnormal numbers + /// - positive numbers + /// - positive infinity + /// - positive signaling NaN + /// - positive quiet NaN. + /// + /// The ordering established by this function does not always agree with the + /// [`PartialOrd`] and [`PartialEq`] implementations of `f32`. For example, + /// they consider negative and positive zero equal, while `total_cmp` + /// doesn't. + /// + /// The interpretation of the signaling NaN bit follows the definition in + /// the IEEE 754 standard, which may not match the interpretation by some of + /// the older, non-conformant (e.g. MIPS) hardware implementations. + /// + /// # Example + /// + /// ``` + /// struct GoodBoy { + /// name: String, + /// weight: f32, + /// } + /// + /// let mut bois = vec![ + /// GoodBoy { name: "Pucci".to_owned(), weight: 0.1 }, + /// GoodBoy { name: "Woofer".to_owned(), weight: 99.0 }, + /// GoodBoy { name: "Yapper".to_owned(), weight: 10.0 }, + /// GoodBoy { name: "Chonk".to_owned(), weight: f32::INFINITY }, + /// GoodBoy { name: "Abs. Unit".to_owned(), weight: f32::NAN }, + /// GoodBoy { name: "Floaty".to_owned(), weight: -5.0 }, + /// ]; + /// + /// bois.sort_by(|a, b| a.weight.total_cmp(&b.weight)); + /// # assert!(bois.into_iter().map(|b| b.weight) + /// # .zip([-5.0, 0.1, 10.0, 99.0, f32::INFINITY, f32::NAN].iter()) + /// # .all(|(a, b)| a.to_bits() == b.to_bits())) + /// ``` + #[stable(feature = "total_cmp", since = "1.62.0")] + #[must_use] + #[inline] + pub fn total_cmp(&self, other: &Self) -> crate::cmp::Ordering { + let mut left = self.to_bits() as i32; + let mut right = other.to_bits() as i32; + + // In case of negatives, flip all the bits except the sign + // to achieve a similar layout as two's complement integers + // + // Why does this work? IEEE 754 floats consist of three fields: + // Sign bit, exponent and mantissa. The set of exponent and mantissa + // fields as a whole have the property that their bitwise order is + // equal to the numeric magnitude where the magnitude is defined. + // The magnitude is not normally defined on NaN values, but + // IEEE 754 totalOrder defines the NaN values also to follow the + // bitwise order. This leads to order explained in the doc comment. + // However, the representation of magnitude is the same for negative + // and positive numbers – only the sign bit is different. + // To easily compare the floats as signed integers, we need to + // flip the exponent and mantissa bits in case of negative numbers. + // We effectively convert the numbers to "two's complement" form. + // + // To do the flipping, we construct a mask and XOR against it. + // We branchlessly calculate an "all-ones except for the sign bit" + // mask from negative-signed values: right shifting sign-extends + // the integer, so we "fill" the mask with sign bits, and then + // convert to unsigned to push one more zero bit. + // On positive values, the mask is all zeros, so it's a no-op. + left ^= (((left >> 31) as u32) >> 1) as i32; + right ^= (((right >> 31) as u32) >> 1) as i32; + + left.cmp(&right) + } + + /// Restrict a value to a certain interval unless it is NaN. + /// + /// Returns `max` if `self` is greater than `max`, and `min` if `self` is + /// less than `min`. Otherwise this returns `self`. + /// + /// Note that this function returns NaN if the initial value was NaN as + /// well. + /// + /// # Panics + /// + /// Panics if `min > max`, `min` is NaN, or `max` is NaN. + /// + /// # Examples + /// + /// ``` + /// assert!((-3.0f32).clamp(-2.0, 1.0) == -2.0); + /// assert!((0.0f32).clamp(-2.0, 1.0) == 0.0); + /// assert!((2.0f32).clamp(-2.0, 1.0) == 1.0); + /// assert!((f32::NAN).clamp(-2.0, 1.0).is_nan()); + /// ``` + #[must_use = "method returns a new number and does not mutate the original value"] + #[stable(feature = "clamp", since = "1.50.0")] + #[inline] + pub fn clamp(mut self, min: f32, max: f32) -> f32 { + assert!(min <= max); + if self < min { + self = min; + } + if self > max { + self = max; + } + self + } } diff --git a/crux-mir/lib/core/src/num/f64.rs b/crux-mir/lib/core/src/num/f64.rs index 129df937c..2a22c4302 100644 --- a/crux-mir/lib/core/src/num/f64.rs +++ b/crux-mir/lib/core/src/num/f64.rs @@ -1,12 +1,13 @@ -//! This module provides constants which are specific to the implementation -//! of the `f64` floating point data type. +//! Constants for the `f64` double-precision floating point type. //! -//! *[See also the `f64` primitive type](../../std/primitive.f64.html).* +//! *[See also the `f64` primitive type][f64].* //! //! Mathematically significant numbers are provided in the `consts` sub-module. //! -//! Although using these constants won’t cause compilation warnings, -//! new code should use the associated constants directly on the primitive type. +//! For the constants defined directly in this module +//! (as distinct from those defined in the `consts` sub-module), +//! new code should instead use the associated constants +//! defined directly on the `f64` type. #![stable(feature = "rust1", since = "1.0.0")] @@ -17,70 +18,248 @@ use crate::mem; use crate::num::FpCategory; /// The radix or base of the internal representation of `f64`. -/// Use [`f64::RADIX`](../../std/primitive.f64.html#associatedconstant.RADIX) instead. +/// Use [`f64::RADIX`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let r = std::f64::RADIX; +/// +/// // intended way +/// let r = f64::RADIX; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `RADIX` associated constant on `f64`")] pub const RADIX: u32 = f64::RADIX; /// Number of significant digits in base 2. -/// Use [`f64::MANTISSA_DIGITS`](../../std/primitive.f64.html#associatedconstant.MANTISSA_DIGITS) instead. +/// Use [`f64::MANTISSA_DIGITS`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let d = std::f64::MANTISSA_DIGITS; +/// +/// // intended way +/// let d = f64::MANTISSA_DIGITS; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated( + since = "TBD", + note = "replaced by the `MANTISSA_DIGITS` associated constant on `f64`" +)] pub const MANTISSA_DIGITS: u32 = f64::MANTISSA_DIGITS; + /// Approximate number of significant digits in base 10. -/// Use [`f64::DIGITS`](../../std/primitive.f64.html#associatedconstant.DIGITS) instead. +/// Use [`f64::DIGITS`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let d = std::f64::DIGITS; +/// +/// // intended way +/// let d = f64::DIGITS; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `DIGITS` associated constant on `f64`")] pub const DIGITS: u32 = f64::DIGITS; /// [Machine epsilon] value for `f64`. -/// Use [`f64::EPSILON`](../../std/primitive.f64.html#associatedconstant.EPSILON) instead. +/// Use [`f64::EPSILON`] instead. /// /// This is the difference between `1.0` and the next larger representable number. /// /// [Machine epsilon]: https://en.wikipedia.org/wiki/Machine_epsilon +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let e = std::f64::EPSILON; +/// +/// // intended way +/// let e = f64::EPSILON; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `EPSILON` associated constant on `f64`")] pub const EPSILON: f64 = f64::EPSILON; /// Smallest finite `f64` value. -/// Use [`f64::MIN`](../../std/primitive.f64.html#associatedconstant.MIN) instead. +/// Use [`f64::MIN`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let min = std::f64::MIN; +/// +/// // intended way +/// let min = f64::MIN; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `MIN` associated constant on `f64`")] pub const MIN: f64 = f64::MIN; + /// Smallest positive normal `f64` value. -/// Use [`f64::MIN_POSITIVE`](../../std/primitive.f64.html#associatedconstant.MIN_POSITIVE) instead. +/// Use [`f64::MIN_POSITIVE`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let min = std::f64::MIN_POSITIVE; +/// +/// // intended way +/// let min = f64::MIN_POSITIVE; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `MIN_POSITIVE` associated constant on `f64`")] pub const MIN_POSITIVE: f64 = f64::MIN_POSITIVE; + /// Largest finite `f64` value. -/// Use [`f64::MAX`](../../std/primitive.f64.html#associatedconstant.MAX) instead. +/// Use [`f64::MAX`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let max = std::f64::MAX; +/// +/// // intended way +/// let max = f64::MAX; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `MAX` associated constant on `f64`")] pub const MAX: f64 = f64::MAX; /// One greater than the minimum possible normal power of 2 exponent. -/// Use [`f64::MIN_EXP`](../../std/primitive.f64.html#associatedconstant.MIN_EXP) instead. +/// Use [`f64::MIN_EXP`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let min = std::f64::MIN_EXP; +/// +/// // intended way +/// let min = f64::MIN_EXP; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `MIN_EXP` associated constant on `f64`")] pub const MIN_EXP: i32 = f64::MIN_EXP; + /// Maximum possible power of 2 exponent. -/// Use [`f64::MAX_EXP`](../../std/primitive.f64.html#associatedconstant.MAX_EXP) instead. +/// Use [`f64::MAX_EXP`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let max = std::f64::MAX_EXP; +/// +/// // intended way +/// let max = f64::MAX_EXP; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `MAX_EXP` associated constant on `f64`")] pub const MAX_EXP: i32 = f64::MAX_EXP; /// Minimum possible normal power of 10 exponent. -/// Use [`f64::MIN_10_EXP`](../../std/primitive.f64.html#associatedconstant.MIN_10_EXP) instead. +/// Use [`f64::MIN_10_EXP`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let min = std::f64::MIN_10_EXP; +/// +/// // intended way +/// let min = f64::MIN_10_EXP; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `MIN_10_EXP` associated constant on `f64`")] pub const MIN_10_EXP: i32 = f64::MIN_10_EXP; + /// Maximum possible power of 10 exponent. -/// Use [`f64::MAX_10_EXP`](../../std/primitive.f64.html#associatedconstant.MAX_10_EXP) instead. +/// Use [`f64::MAX_10_EXP`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let max = std::f64::MAX_10_EXP; +/// +/// // intended way +/// let max = f64::MAX_10_EXP; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `MAX_10_EXP` associated constant on `f64`")] pub const MAX_10_EXP: i32 = f64::MAX_10_EXP; /// Not a Number (NaN). -/// Use [`f64::NAN`](../../std/primitive.f64.html#associatedconstant.NAN) instead. +/// Use [`f64::NAN`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let nan = std::f64::NAN; +/// +/// // intended way +/// let nan = f64::NAN; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `NAN` associated constant on `f64`")] pub const NAN: f64 = f64::NAN; + /// Infinity (∞). -/// Use [`f64::INFINITY`](../../std/primitive.f64.html#associatedconstant.INFINITY) instead. +/// Use [`f64::INFINITY`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let inf = std::f64::INFINITY; +/// +/// // intended way +/// let inf = f64::INFINITY; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `INFINITY` associated constant on `f64`")] pub const INFINITY: f64 = f64::INFINITY; + /// Negative infinity (−∞). -/// Use [`f64::NEG_INFINITY`](../../std/primitive.f64.html#associatedconstant.NEG_INFINITY) instead. +/// Use [`f64::NEG_INFINITY`] instead. +/// +/// # Examples +/// +/// ```rust +/// // deprecated way +/// # #[allow(deprecated, deprecated_in_future)] +/// let ninf = std::f64::NEG_INFINITY; +/// +/// // intended way +/// let ninf = f64::NEG_INFINITY; +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "TBD", note = "replaced by the `NEG_INFINITY` associated constant on `f64`")] pub const NEG_INFINITY: f64 = f64::NEG_INFINITY; /// Basic mathematical constants. @@ -95,7 +274,7 @@ pub mod consts { /// The full circle constant (τ) /// /// Equal to 2π. - #[unstable(feature = "tau_constant", issue = "66770")] + #[stable(feature = "tau_constant", since = "1.47.0")] pub const TAU: f64 = 6.28318530717958647692528676655900577_f64; /// π/2 @@ -167,7 +346,6 @@ pub mod consts { pub const LN_10: f64 = 2.30258509299404568401799145468436421_f64; } -#[lang = "f64"] #[cfg(not(test))] impl f64 { /// The radix or base of the internal representation of `f64`. @@ -214,16 +392,25 @@ impl f64 { pub const MAX_10_EXP: i32 = 308; /// Not a Number (NaN). + /// + /// Note that IEEE 754 doesn't define just a single NaN value; + /// a plethora of bit patterns are considered to be NaN. + /// Furthermore, the standard makes a difference + /// between a "signaling" and a "quiet" NaN, + /// and allows inspecting its "payload" (the unspecified bits in the bit pattern). + /// This constant isn't guaranteed to equal to any specific NaN bitpattern, + /// and the stability of its representation over Rust versions + /// and target platforms isn't guaranteed. #[stable(feature = "assoc_int_consts", since = "1.43.0")] pub const NAN: f64 = 0.0_f64 / 0.0_f64; /// Infinity (∞). #[stable(feature = "assoc_int_consts", since = "1.43.0")] pub const INFINITY: f64 = 1.0_f64 / 0.0_f64; - /// Negative infinity (-∞). + /// Negative infinity (−∞). #[stable(feature = "assoc_int_consts", since = "1.43.0")] pub const NEG_INFINITY: f64 = -1.0_f64 / 0.0_f64; - /// Returns `true` if this value is `NaN`. + /// Returns `true` if this value is NaN. /// /// ``` /// let nan = f64::NAN; @@ -232,18 +419,24 @@ impl f64 { /// assert!(nan.is_nan()); /// assert!(!f.is_nan()); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] #[inline] - pub fn is_nan(self) -> bool { + pub const fn is_nan(self) -> bool { self != self } - // FIXME(#50145): `abs` is publicly unavailable in libcore due to + // FIXME(#50145): `abs` is publicly unavailable in core due to // concerns about portability, so this implementation is for // private use internally. #[inline] - fn abs_private(self) -> f64 { - f64::from_bits(self.to_bits() & 0x7fff_ffff_ffff_ffff) + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] + pub(crate) const fn abs_private(self) -> f64 { + // SAFETY: This transmutation is fine. Probably. For the reasons std is using it. + unsafe { + mem::transmute::(mem::transmute::(self) & 0x7fff_ffff_ffff_ffff) + } } /// Returns `true` if this value is positive infinity or negative infinity, and @@ -261,13 +454,18 @@ impl f64 { /// assert!(inf.is_infinite()); /// assert!(neg_inf.is_infinite()); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] #[inline] - pub fn is_infinite(self) -> bool { - self.abs_private() == INFINITY + pub const fn is_infinite(self) -> bool { + // Getting clever with transmutation can result in incorrect answers on some FPUs + // FIXME: alter the Rust <-> Rust calling convention to prevent this problem. + // See https://github.com/rust-lang/rust/issues/72327 + (self == f64::INFINITY) | (self == f64::NEG_INFINITY) } - /// Returns `true` if this number is neither infinite nor `NaN`. + /// Returns `true` if this number is neither infinite nor NaN. /// /// ``` /// let f = 7.0f64; @@ -281,16 +479,44 @@ impl f64 { /// assert!(!inf.is_finite()); /// assert!(!neg_inf.is_finite()); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] #[inline] - pub fn is_finite(self) -> bool { + pub const fn is_finite(self) -> bool { // There's no need to handle NaN separately: if self is NaN, // the comparison is not true, exactly as desired. - self.abs_private() < INFINITY + self.abs_private() < Self::INFINITY + } + + /// Returns `true` if the number is [subnormal]. + /// + /// ``` + /// let min = f64::MIN_POSITIVE; // 2.2250738585072014e-308_f64 + /// let max = f64::MAX; + /// let lower_than_min = 1.0e-308_f64; + /// let zero = 0.0_f64; + /// + /// assert!(!min.is_subnormal()); + /// assert!(!max.is_subnormal()); + /// + /// assert!(!zero.is_subnormal()); + /// assert!(!f64::NAN.is_subnormal()); + /// assert!(!f64::INFINITY.is_subnormal()); + /// // Values between `0` and `min` are Subnormal. + /// assert!(lower_than_min.is_subnormal()); + /// ``` + /// [subnormal]: https://en.wikipedia.org/wiki/Denormal_number + #[must_use] + #[stable(feature = "is_subnormal", since = "1.53.0")] + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] + #[inline] + pub const fn is_subnormal(self) -> bool { + matches!(self.classify(), FpCategory::Subnormal) } /// Returns `true` if the number is neither zero, infinite, - /// [subnormal], or `NaN`. + /// [subnormal], or NaN. /// /// ``` /// let min = f64::MIN_POSITIVE; // 2.2250738585072014e-308f64 @@ -308,10 +534,12 @@ impl f64 { /// assert!(!lower_than_min.is_normal()); /// ``` /// [subnormal]: https://en.wikipedia.org/wiki/Denormal_number + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] #[inline] - pub fn is_normal(self) -> bool { - self.classify() == FpCategory::Normal + pub const fn is_normal(self) -> bool { + matches!(self.classify(), FpCategory::Normal) } /// Returns the floating point category of the number. If only one property @@ -328,22 +556,79 @@ impl f64 { /// assert_eq!(inf.classify(), FpCategory::Infinite); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn classify(self) -> FpCategory { + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] + pub const fn classify(self) -> FpCategory { + // A previous implementation tried to only use bitmask-based checks, + // using f64::to_bits to transmute the float to its bit repr and match on that. + // Unfortunately, floating point numbers can be much worse than that. + // This also needs to not result in recursive evaluations of f64::to_bits. + // + // On some processors, in some cases, LLVM will "helpfully" lower floating point ops, + // in spite of a request for them using f32 and f64, to things like x87 operations. + // These have an f64's mantissa, but can have a larger than normal exponent. + // FIXME(jubilee): Using x87 operations is never necessary in order to function + // on x86 processors for Rust-to-Rust calls, so this issue should not happen. + // Code generation should be adjusted to use non-C calling conventions, avoiding this. + // + // Thus, a value may compare unequal to infinity, despite having a "full" exponent mask. + // And it may not be NaN, as it can simply be an "overextended" finite value. + if self.is_nan() { + FpCategory::Nan + } else { + // However, std can't simply compare to zero to check for zero, either, + // as correctness requires avoiding equality tests that may be Subnormal == -0.0 + // because it may be wrong under "denormals are zero" and "flush to zero" modes. + // Most of std's targets don't use those, but they are used for thumbv7neon. + // So, this does use bitpattern matching for the rest. + + // SAFETY: f64 to u64 is fine. Usually. + // If control flow has gotten this far, the value is definitely in one of the categories + // that f64::partial_classify can correctly analyze. + unsafe { f64::partial_classify(self) } + } + } + + // This doesn't actually return a right answer for NaN on purpose, + // seeing as how it cannot correctly discern between a floating point NaN, + // and some normal floating point numbers truncated from an x87 FPU. + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] + const unsafe fn partial_classify(self) -> FpCategory { const EXP_MASK: u64 = 0x7ff0000000000000; const MAN_MASK: u64 = 0x000fffffffffffff; - let bits = self.to_bits(); - match (bits & MAN_MASK, bits & EXP_MASK) { + // SAFETY: The caller is not asking questions for which this will tell lies. + let b = unsafe { mem::transmute::(self) }; + match (b & MAN_MASK, b & EXP_MASK) { + (0, EXP_MASK) => FpCategory::Infinite, (0, 0) => FpCategory::Zero, (_, 0) => FpCategory::Subnormal, + _ => FpCategory::Normal, + } + } + + // This operates on bits, and only bits, so it can ignore concerns about weird FPUs. + // FIXME(jubilee): In a just world, this would be the entire impl for classify, + // plus a transmute. We do not live in a just world, but we can make it more so. + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] + const fn classify_bits(b: u64) -> FpCategory { + const EXP_MASK: u64 = 0x7ff0000000000000; + const MAN_MASK: u64 = 0x000fffffffffffff; + + match (b & MAN_MASK, b & EXP_MASK) { (0, EXP_MASK) => FpCategory::Infinite, (_, EXP_MASK) => FpCategory::Nan, + (0, 0) => FpCategory::Zero, + (_, 0) => FpCategory::Subnormal, _ => FpCategory::Normal, } } - /// Returns `true` if `self` has a positive sign, including `+0.0`, `NaN`s with - /// positive sign bit and positive infinity. + /// Returns `true` if `self` has a positive sign, including `+0.0`, NaNs with + /// positive sign bit and positive infinity. Note that IEEE 754 doesn't assign any + /// meaning to the sign bit in case of a NaN, and as Rust doesn't guarantee that + /// the bit pattern of NaNs are conserved over arithmetic operations, the result of + /// `is_sign_positive` on a NaN might produce an unexpected result in some cases. + /// See [explanation of NaN as a special value](f32) for more info. /// /// ``` /// let f = 7.0_f64; @@ -352,22 +637,29 @@ impl f64 { /// assert!(f.is_sign_positive()); /// assert!(!g.is_sign_positive()); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] #[inline] - pub fn is_sign_positive(self) -> bool { + pub const fn is_sign_positive(self) -> bool { !self.is_sign_negative() } + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_deprecated(since = "1.0.0", reason = "renamed to is_sign_positive")] + #[deprecated(since = "1.0.0", note = "renamed to is_sign_positive")] #[inline] #[doc(hidden)] pub fn is_positive(self) -> bool { self.is_sign_positive() } - /// Returns `true` if `self` has a negative sign, including `-0.0`, `NaN`s with - /// negative sign bit and negative infinity. + /// Returns `true` if `self` has a negative sign, including `-0.0`, NaNs with + /// negative sign bit and negative infinity. Note that IEEE 754 doesn't assign any + /// meaning to the sign bit in case of a NaN, and as Rust doesn't guarantee that + /// the bit pattern of NaNs are conserved over arithmetic operations, the result of + /// `is_sign_negative` on a NaN might produce an unexpected result in some cases. + /// See [explanation of NaN as a special value](f32) for more info. /// /// ``` /// let f = 7.0_f64; @@ -376,20 +668,126 @@ impl f64 { /// assert!(!f.is_sign_negative()); /// assert!(g.is_sign_negative()); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")] #[inline] - pub fn is_sign_negative(self) -> bool { - self.to_bits() & 0x8000_0000_0000_0000 != 0 + pub const fn is_sign_negative(self) -> bool { + // IEEE754 says: isSignMinus(x) is true if and only if x has negative sign. isSignMinus + // applies to zeros and NaNs as well. + // SAFETY: This is just transmuting to get the sign bit, it's fine. + unsafe { mem::transmute::(self) & 0x8000_0000_0000_0000 != 0 } } + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_deprecated(since = "1.0.0", reason = "renamed to is_sign_negative")] + #[deprecated(since = "1.0.0", note = "renamed to is_sign_negative")] #[inline] #[doc(hidden)] pub fn is_negative(self) -> bool { self.is_sign_negative() } + /// Returns the least number greater than `self`. + /// + /// Let `TINY` be the smallest representable positive `f64`. Then, + /// - if `self.is_nan()`, this returns `self`; + /// - if `self` is [`NEG_INFINITY`], this returns [`MIN`]; + /// - if `self` is `-TINY`, this returns -0.0; + /// - if `self` is -0.0 or +0.0, this returns `TINY`; + /// - if `self` is [`MAX`] or [`INFINITY`], this returns [`INFINITY`]; + /// - otherwise the unique least value greater than `self` is returned. + /// + /// The identity `x.next_up() == -(-x).next_down()` holds for all non-NaN `x`. When `x` + /// is finite `x == x.next_up().next_down()` also holds. + /// + /// ```rust + /// #![feature(float_next_up_down)] + /// // f64::EPSILON is the difference between 1.0 and the next number up. + /// assert_eq!(1.0f64.next_up(), 1.0 + f64::EPSILON); + /// // But not for most numbers. + /// assert!(0.1f64.next_up() < 0.1 + f64::EPSILON); + /// assert_eq!(9007199254740992f64.next_up(), 9007199254740994.0); + /// ``` + /// + /// [`NEG_INFINITY`]: Self::NEG_INFINITY + /// [`INFINITY`]: Self::INFINITY + /// [`MIN`]: Self::MIN + /// [`MAX`]: Self::MAX + #[unstable(feature = "float_next_up_down", issue = "91399")] + #[rustc_const_unstable(feature = "float_next_up_down", issue = "91399")] + pub const fn next_up(self) -> Self { + // We must use strictly integer arithmetic to prevent denormals from + // flushing to zero after an arithmetic operation on some platforms. + const TINY_BITS: u64 = 0x1; // Smallest positive f64. + const CLEAR_SIGN_MASK: u64 = 0x7fff_ffff_ffff_ffff; + + let bits = self.to_bits(); + if self.is_nan() || bits == Self::INFINITY.to_bits() { + return self; + } + + let abs = bits & CLEAR_SIGN_MASK; + let next_bits = if abs == 0 { + TINY_BITS + } else if bits == abs { + bits + 1 + } else { + bits - 1 + }; + Self::from_bits(next_bits) + } + + /// Returns the greatest number less than `self`. + /// + /// Let `TINY` be the smallest representable positive `f64`. Then, + /// - if `self.is_nan()`, this returns `self`; + /// - if `self` is [`INFINITY`], this returns [`MAX`]; + /// - if `self` is `TINY`, this returns 0.0; + /// - if `self` is -0.0 or +0.0, this returns `-TINY`; + /// - if `self` is [`MIN`] or [`NEG_INFINITY`], this returns [`NEG_INFINITY`]; + /// - otherwise the unique greatest value less than `self` is returned. + /// + /// The identity `x.next_down() == -(-x).next_up()` holds for all non-NaN `x`. When `x` + /// is finite `x == x.next_down().next_up()` also holds. + /// + /// ```rust + /// #![feature(float_next_up_down)] + /// let x = 1.0f64; + /// // Clamp value into range [0, 1). + /// let clamped = x.clamp(0.0, 1.0f64.next_down()); + /// assert!(clamped < 1.0); + /// assert_eq!(clamped.next_up(), 1.0); + /// ``` + /// + /// [`NEG_INFINITY`]: Self::NEG_INFINITY + /// [`INFINITY`]: Self::INFINITY + /// [`MIN`]: Self::MIN + /// [`MAX`]: Self::MAX + #[unstable(feature = "float_next_up_down", issue = "91399")] + #[rustc_const_unstable(feature = "float_next_up_down", issue = "91399")] + pub const fn next_down(self) -> Self { + // We must use strictly integer arithmetic to prevent denormals from + // flushing to zero after an arithmetic operation on some platforms. + const NEG_TINY_BITS: u64 = 0x8000_0000_0000_0001; // Smallest (in magnitude) negative f64. + const CLEAR_SIGN_MASK: u64 = 0x7fff_ffff_ffff_ffff; + + let bits = self.to_bits(); + if self.is_nan() || bits == Self::NEG_INFINITY.to_bits() { + return self; + } + + let abs = bits & CLEAR_SIGN_MASK; + let next_bits = if abs == 0 { + NEG_TINY_BITS + } else if bits == abs { + bits - 1 + } else { + bits + 1 + }; + Self::from_bits(next_bits) + } + /// Takes the reciprocal (inverse) of a number, `1/x`. /// /// ``` @@ -398,6 +796,7 @@ impl f64 { /// /// assert!(abs_difference < 1e-10); /// ``` + #[must_use = "this returns the result of the operation, without modifying the original"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn recip(self) -> f64 { @@ -407,14 +806,14 @@ impl f64 { /// Converts radians to degrees. /// /// ``` - /// use std::f64::consts; - /// - /// let angle = consts::PI; + /// let angle = std::f64::consts::PI; /// /// let abs_difference = (angle.to_degrees() - 180.0).abs(); /// /// assert!(abs_difference < 1e-10); /// ``` + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn to_degrees(self) -> f64 { @@ -427,14 +826,14 @@ impl f64 { /// Converts degrees to radians. /// /// ``` - /// use std::f64::consts; - /// /// let angle = 180.0_f64; /// - /// let abs_difference = (angle.to_radians() - consts::PI).abs(); + /// let abs_difference = (angle.to_radians() - std::f64::consts::PI).abs(); /// /// assert!(abs_difference < 1e-10); /// ``` + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn to_radians(self) -> f64 { @@ -442,7 +841,12 @@ impl f64 { self * (value / 180.0) } - /// Returns the maximum of the two numbers. + /// Returns the maximum of the two numbers, ignoring NaN. + /// + /// If one of the arguments is NaN, then the other argument is returned. + /// This follows the IEEE 754-2008 semantics for maxNum, except for handling of signaling NaNs; + /// this function handles all NaNs the same way and avoids maxNum's problems with associativity. + /// This also matches the behavior of libm’s fmax. /// /// ``` /// let x = 1.0_f64; @@ -450,15 +854,19 @@ impl f64 { /// /// assert_eq!(x.max(y), y); /// ``` - /// - /// If one of the arguments is NaN, then the other argument is returned. + #[must_use = "this returns the result of the comparison, without modifying either input"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn max(self, other: f64) -> f64 { intrinsics::maxnumf64(self, other) } - /// Returns the minimum of the two numbers. + /// Returns the minimum of the two numbers, ignoring NaN. + /// + /// If one of the arguments is NaN, then the other argument is returned. + /// This follows the IEEE 754-2008 semantics for minNum, except for handling of signaling NaNs; + /// this function handles all NaNs the same way and avoids minNum's problems with associativity. + /// This also matches the behavior of libm’s fmin. /// /// ``` /// let x = 1.0_f64; @@ -466,27 +874,94 @@ impl f64 { /// /// assert_eq!(x.min(y), x); /// ``` - /// - /// If one of the arguments is NaN, then the other argument is returned. + #[must_use = "this returns the result of the comparison, without modifying either input"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn min(self, other: f64) -> f64 { intrinsics::minnumf64(self, other) } + /// Returns the maximum of the two numbers, propagating NaN. + /// + /// This returns NaN when *either* argument is NaN, as opposed to + /// [`f64::max`] which only returns NaN when *both* arguments are NaN. + /// + /// ``` + /// #![feature(float_minimum_maximum)] + /// let x = 1.0_f64; + /// let y = 2.0_f64; + /// + /// assert_eq!(x.maximum(y), y); + /// assert!(x.maximum(f64::NAN).is_nan()); + /// ``` + /// + /// If one of the arguments is NaN, then NaN is returned. Otherwise this returns the greater + /// of the two numbers. For this operation, -0.0 is considered to be less than +0.0. + /// Note that this follows the semantics specified in IEEE 754-2019. + /// + /// Also note that "propagation" of NaNs here doesn't necessarily mean that the bitpattern of a NaN + /// operand is conserved; see [explanation of NaN as a special value](f32) for more info. + #[must_use = "this returns the result of the comparison, without modifying either input"] + #[unstable(feature = "float_minimum_maximum", issue = "91079")] + #[inline] + pub fn maximum(self, other: f64) -> f64 { + if self > other { + self + } else if other > self { + other + } else if self == other { + if self.is_sign_positive() && other.is_sign_negative() { self } else { other } + } else { + self + other + } + } + + /// Returns the minimum of the two numbers, propagating NaN. + /// + /// This returns NaN when *either* argument is NaN, as opposed to + /// [`f64::min`] which only returns NaN when *both* arguments are NaN. + /// + /// ``` + /// #![feature(float_minimum_maximum)] + /// let x = 1.0_f64; + /// let y = 2.0_f64; + /// + /// assert_eq!(x.minimum(y), x); + /// assert!(x.minimum(f64::NAN).is_nan()); + /// ``` + /// + /// If one of the arguments is NaN, then NaN is returned. Otherwise this returns the lesser + /// of the two numbers. For this operation, -0.0 is considered to be less than +0.0. + /// Note that this follows the semantics specified in IEEE 754-2019. + /// + /// Also note that "propagation" of NaNs here doesn't necessarily mean that the bitpattern of a NaN + /// operand is conserved; see [explanation of NaN as a special value](f32) for more info. + #[must_use = "this returns the result of the comparison, without modifying either input"] + #[unstable(feature = "float_minimum_maximum", issue = "91079")] + #[inline] + pub fn minimum(self, other: f64) -> f64 { + if self < other { + self + } else if other < self { + other + } else if self == other { + if self.is_sign_negative() && other.is_sign_positive() { self } else { other } + } else { + self + other + } + } + /// Rounds toward zero and converts to any primitive integer type, /// assuming that the value is finite and fits in that type. /// /// ``` - /// #![feature(float_approx_unchecked_to)] - /// - /// let value = 4.6_f32; - /// let rounded = unsafe { value.approx_unchecked_to::() }; + /// let value = 4.6_f64; + /// let rounded = unsafe { value.to_int_unchecked::() }; /// assert_eq!(rounded, 4); /// - /// let value = -128.9_f32; - /// let rounded = unsafe { value.approx_unchecked_to::() }; - /// assert_eq!(rounded, std::i8::MIN); + /// let value = -128.9_f64; + /// let rounded = unsafe { value.to_int_unchecked::() }; + /// assert_eq!(rounded, i8::MIN); /// ``` /// /// # Safety @@ -496,21 +971,25 @@ impl f64 { /// * Not be `NaN` /// * Not be infinite /// * Be representable in the return type `Int`, after truncating off its fractional part - #[unstable(feature = "float_approx_unchecked_to", issue = "67058")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[stable(feature = "float_approx_unchecked_to", since = "1.44.0")] #[inline] - pub unsafe fn approx_unchecked_to(self) -> Int + pub unsafe fn to_int_unchecked(self) -> Int where Self: FloatToInt, { - FloatToInt::::approx_unchecked(self) + // SAFETY: the caller must uphold the safety contract for + // `FloatToInt::to_int_unchecked`. + unsafe { FloatToInt::::to_int_unchecked(self) } } /// Raw transmutation to `u64`. /// /// This is currently identical to `transmute::(self)` on all platforms. /// - /// See `from_bits` for some discussion of the portability of this operation - /// (there are almost no issues). + /// See [`from_bits`](Self::from_bits) for some discussion of the + /// portability of this operation (there are almost no issues). /// /// Note that this function is distinct from `as` casting, which attempts to /// preserve the *numeric* value, and not the bitwise value. @@ -522,11 +1001,41 @@ impl f64 { /// assert_eq!((12.5f64).to_bits(), 0x4029000000000000); /// /// ``` + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[stable(feature = "float_bits_conv", since = "1.20.0")] + #[rustc_const_unstable(feature = "const_float_bits_conv", issue = "72447")] #[inline] - pub fn to_bits(self) -> u64 { - // SAFETY: `u64` is a plain old datatype so we can always transmute to it - unsafe { mem::transmute(self) } + pub const fn to_bits(self) -> u64 { + // SAFETY: `u64` is a plain old datatype so we can always transmute to it. + // ...sorta. + // + // See the SAFETY comment in f64::from_bits for more. + #[rustc_const_unstable(feature = "const_float_bits_conv", issue = "72447")] + const fn ct_f64_to_u64(ct: f64) -> u64 { + match ct.classify() { + FpCategory::Nan => { + panic!("const-eval error: cannot use f64::to_bits on a NaN") + } + FpCategory::Subnormal => { + panic!("const-eval error: cannot use f64::to_bits on a subnormal number") + } + FpCategory::Infinite | FpCategory::Normal | FpCategory::Zero => { + // SAFETY: We have a normal floating point number. Now we transmute, i.e. do a bitcopy. + unsafe { mem::transmute::(ct) } + } + } + } + + #[inline(always)] // See https://github.com/rust-lang/compiler-builtins/issues/491 + fn rt_f64_to_u64(rt: f64) -> u64 { + // SAFETY: `u64` is a plain old datatype so we can always... uh... + // ...look, just pretend you forgot what you just read. + // Stability concerns. + unsafe { mem::transmute::(rt) } + } + // SAFETY: We use internal implementations that either always work or fail at compile time. + unsafe { intrinsics::const_eval_select((self,), ct_f64_to_u64, rt_f64_to_u64) } } /// Raw transmutation from `u64`. @@ -535,16 +1044,16 @@ impl f64 { /// It turns out this is incredibly portable, for two reasons: /// /// * Floats and Ints have the same endianness on all supported platforms. - /// * IEEE-754 very precisely specifies the bit layout of floats. + /// * IEEE 754 very precisely specifies the bit layout of floats. /// - /// However there is one caveat: prior to the 2008 version of IEEE-754, how + /// However there is one caveat: prior to the 2008 version of IEEE 754, how /// to interpret the NaN signaling bit wasn't actually specified. Most platforms /// (notably x86 and ARM) picked the interpretation that was ultimately /// standardized in 2008, but some didn't (notably MIPS). As a result, all /// signaling NaNs on MIPS are quiet NaNs on x86, and vice-versa. /// /// Rather than trying to preserve signaling-ness cross-platform, this - /// implementation favours preserving the exact bits. This means that + /// implementation favors preserving the exact bits. This means that /// any payloads encoded in NaNs will be preserved even if the result of /// this method is sent over the network from an x86 machine to a MIPS one. /// @@ -553,7 +1062,7 @@ impl f64 { /// /// If the input isn't NaN, then there is no portability concern. /// - /// If you don't care about signalingness (very likely), then there is no + /// If you don't care about signaling-ness (very likely), then there is no /// portability concern. /// /// Note that this function is distinct from `as` casting, which attempts to @@ -566,40 +1075,105 @@ impl f64 { /// assert_eq!(v, 12.5); /// ``` #[stable(feature = "float_bits_conv", since = "1.20.0")] + #[rustc_const_unstable(feature = "const_float_bits_conv", issue = "72447")] + #[must_use] #[inline] - pub fn from_bits(v: u64) -> Self { - // SAFETY: `u64` is a plain old datatype so we can always transmute from it + pub const fn from_bits(v: u64) -> Self { // It turns out the safety issues with sNaN were overblown! Hooray! - unsafe { mem::transmute(v) } + // SAFETY: `u64` is a plain old datatype so we can always transmute from it + // ...sorta. + // + // It turns out that at runtime, it is possible for a floating point number + // to be subject to floating point modes that alter nonzero subnormal numbers + // to zero on reads and writes, aka "denormals are zero" and "flush to zero". + // This is not a problem usually, but at least one tier2 platform for Rust + // actually exhibits an FTZ behavior by default: thumbv7neon + // aka "the Neon FPU in AArch32 state" + // + // Even with this, not all instructions exhibit the FTZ behaviors on thumbv7neon, + // so this should load the same bits if LLVM emits the "correct" instructions, + // but LLVM sometimes makes interesting choices about float optimization, + // and other FPUs may do similar. Thus, it is wise to indulge luxuriously in caution. + // + // In addition, on x86 targets with SSE or SSE2 disabled and the x87 FPU enabled, + // i.e. not soft-float, the way Rust does parameter passing can actually alter + // a number that is "not infinity" to have the same exponent as infinity, + // in a slightly unpredictable manner. + // + // And, of course evaluating to a NaN value is fairly nondeterministic. + // More precisely: when NaN should be returned is knowable, but which NaN? + // So far that's defined by a combination of LLVM and the CPU, not Rust. + // This function, however, allows observing the bitstring of a NaN, + // thus introspection on CTFE. + // + // In order to preserve, at least for the moment, const-to-runtime equivalence, + // reject any of these possible situations from happening. + #[rustc_const_unstable(feature = "const_float_bits_conv", issue = "72447")] + const fn ct_u64_to_f64(ct: u64) -> f64 { + match f64::classify_bits(ct) { + FpCategory::Subnormal => { + panic!("const-eval error: cannot use f64::from_bits on a subnormal number") + } + FpCategory::Nan => { + panic!("const-eval error: cannot use f64::from_bits on NaN") + } + FpCategory::Infinite | FpCategory::Normal | FpCategory::Zero => { + // SAFETY: It's not a frumious number + unsafe { mem::transmute::(ct) } + } + } + } + + #[inline(always)] // See https://github.com/rust-lang/compiler-builtins/issues/491 + fn rt_u64_to_f64(rt: u64) -> f64 { + // SAFETY: `u64` is a plain old datatype so we can always... uh... + // ...look, just pretend you forgot what you just read. + // Stability concerns. + unsafe { mem::transmute::(rt) } + } + // SAFETY: We use internal implementations that either always work or fail at compile time. + unsafe { intrinsics::const_eval_select((v,), ct_u64_to_f64, rt_u64_to_f64) } } /// Return the memory representation of this floating point number as a byte array in /// big-endian (network) byte order. /// + /// See [`from_bits`](Self::from_bits) for some discussion of the + /// portability of this operation (there are almost no issues). + /// /// # Examples /// /// ``` /// let bytes = 12.5f64.to_be_bytes(); /// assert_eq!(bytes, [0x40, 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]); /// ``` + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[stable(feature = "float_to_from_bytes", since = "1.40.0")] + #[rustc_const_unstable(feature = "const_float_bits_conv", issue = "72447")] #[inline] - pub fn to_be_bytes(self) -> [u8; 8] { + pub const fn to_be_bytes(self) -> [u8; 8] { self.to_bits().to_be_bytes() } /// Return the memory representation of this floating point number as a byte array in /// little-endian byte order. /// + /// See [`from_bits`](Self::from_bits) for some discussion of the + /// portability of this operation (there are almost no issues). + /// /// # Examples /// /// ``` /// let bytes = 12.5f64.to_le_bytes(); /// assert_eq!(bytes, [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x40]); /// ``` + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[stable(feature = "float_to_from_bytes", since = "1.40.0")] + #[rustc_const_unstable(feature = "const_float_bits_conv", issue = "72447")] #[inline] - pub fn to_le_bytes(self) -> [u8; 8] { + pub const fn to_le_bytes(self) -> [u8; 8] { self.to_bits().to_le_bytes() } @@ -609,8 +1183,11 @@ impl f64 { /// As the target platform's native endianness is used, portable code /// should use [`to_be_bytes`] or [`to_le_bytes`], as appropriate, instead. /// - /// [`to_be_bytes`]: #method.to_be_bytes - /// [`to_le_bytes`]: #method.to_le_bytes + /// [`to_be_bytes`]: f64::to_be_bytes + /// [`to_le_bytes`]: f64::to_le_bytes + /// + /// See [`from_bits`](Self::from_bits) for some discussion of the + /// portability of this operation (there are almost no issues). /// /// # Examples /// @@ -625,14 +1202,20 @@ impl f64 { /// } /// ); /// ``` + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[stable(feature = "float_to_from_bytes", since = "1.40.0")] + #[rustc_const_unstable(feature = "const_float_bits_conv", issue = "72447")] #[inline] - pub fn to_ne_bytes(self) -> [u8; 8] { + pub const fn to_ne_bytes(self) -> [u8; 8] { self.to_bits().to_ne_bytes() } /// Create a floating point value from its representation as a byte array in big endian. /// + /// See [`from_bits`](Self::from_bits) for some discussion of the + /// portability of this operation (there are almost no issues). + /// /// # Examples /// /// ``` @@ -640,13 +1223,18 @@ impl f64 { /// assert_eq!(value, 12.5); /// ``` #[stable(feature = "float_to_from_bytes", since = "1.40.0")] + #[rustc_const_unstable(feature = "const_float_bits_conv", issue = "72447")] + #[must_use] #[inline] - pub fn from_be_bytes(bytes: [u8; 8]) -> Self { + pub const fn from_be_bytes(bytes: [u8; 8]) -> Self { Self::from_bits(u64::from_be_bytes(bytes)) } /// Create a floating point value from its representation as a byte array in little endian. /// + /// See [`from_bits`](Self::from_bits) for some discussion of the + /// portability of this operation (there are almost no issues). + /// /// # Examples /// /// ``` @@ -654,8 +1242,10 @@ impl f64 { /// assert_eq!(value, 12.5); /// ``` #[stable(feature = "float_to_from_bytes", since = "1.40.0")] + #[rustc_const_unstable(feature = "const_float_bits_conv", issue = "72447")] + #[must_use] #[inline] - pub fn from_le_bytes(bytes: [u8; 8]) -> Self { + pub const fn from_le_bytes(bytes: [u8; 8]) -> Self { Self::from_bits(u64::from_le_bytes(bytes)) } @@ -665,8 +1255,11 @@ impl f64 { /// likely wants to use [`from_be_bytes`] or [`from_le_bytes`], as /// appropriate instead. /// - /// [`from_be_bytes`]: #method.from_be_bytes - /// [`from_le_bytes`]: #method.from_le_bytes + /// [`from_be_bytes`]: f64::from_be_bytes + /// [`from_le_bytes`]: f64::from_le_bytes + /// + /// See [`from_bits`](Self::from_bits) for some discussion of the + /// portability of this operation (there are almost no issues). /// /// # Examples /// @@ -679,8 +1272,130 @@ impl f64 { /// assert_eq!(value, 12.5); /// ``` #[stable(feature = "float_to_from_bytes", since = "1.40.0")] + #[rustc_const_unstable(feature = "const_float_bits_conv", issue = "72447")] + #[must_use] #[inline] - pub fn from_ne_bytes(bytes: [u8; 8]) -> Self { + pub const fn from_ne_bytes(bytes: [u8; 8]) -> Self { Self::from_bits(u64::from_ne_bytes(bytes)) } + + /// Return the ordering between `self` and `other`. + /// + /// Unlike the standard partial comparison between floating point numbers, + /// this comparison always produces an ordering in accordance to + /// the `totalOrder` predicate as defined in the IEEE 754 (2008 revision) + /// floating point standard. The values are ordered in the following sequence: + /// + /// - negative quiet NaN + /// - negative signaling NaN + /// - negative infinity + /// - negative numbers + /// - negative subnormal numbers + /// - negative zero + /// - positive zero + /// - positive subnormal numbers + /// - positive numbers + /// - positive infinity + /// - positive signaling NaN + /// - positive quiet NaN. + /// + /// The ordering established by this function does not always agree with the + /// [`PartialOrd`] and [`PartialEq`] implementations of `f64`. For example, + /// they consider negative and positive zero equal, while `total_cmp` + /// doesn't. + /// + /// The interpretation of the signaling NaN bit follows the definition in + /// the IEEE 754 standard, which may not match the interpretation by some of + /// the older, non-conformant (e.g. MIPS) hardware implementations. + /// + /// # Example + /// + /// ``` + /// struct GoodBoy { + /// name: String, + /// weight: f64, + /// } + /// + /// let mut bois = vec![ + /// GoodBoy { name: "Pucci".to_owned(), weight: 0.1 }, + /// GoodBoy { name: "Woofer".to_owned(), weight: 99.0 }, + /// GoodBoy { name: "Yapper".to_owned(), weight: 10.0 }, + /// GoodBoy { name: "Chonk".to_owned(), weight: f64::INFINITY }, + /// GoodBoy { name: "Abs. Unit".to_owned(), weight: f64::NAN }, + /// GoodBoy { name: "Floaty".to_owned(), weight: -5.0 }, + /// ]; + /// + /// bois.sort_by(|a, b| a.weight.total_cmp(&b.weight)); + /// # assert!(bois.into_iter().map(|b| b.weight) + /// # .zip([-5.0, 0.1, 10.0, 99.0, f64::INFINITY, f64::NAN].iter()) + /// # .all(|(a, b)| a.to_bits() == b.to_bits())) + /// ``` + #[stable(feature = "total_cmp", since = "1.62.0")] + #[must_use] + #[inline] + pub fn total_cmp(&self, other: &Self) -> crate::cmp::Ordering { + let mut left = self.to_bits() as i64; + let mut right = other.to_bits() as i64; + + // In case of negatives, flip all the bits except the sign + // to achieve a similar layout as two's complement integers + // + // Why does this work? IEEE 754 floats consist of three fields: + // Sign bit, exponent and mantissa. The set of exponent and mantissa + // fields as a whole have the property that their bitwise order is + // equal to the numeric magnitude where the magnitude is defined. + // The magnitude is not normally defined on NaN values, but + // IEEE 754 totalOrder defines the NaN values also to follow the + // bitwise order. This leads to order explained in the doc comment. + // However, the representation of magnitude is the same for negative + // and positive numbers – only the sign bit is different. + // To easily compare the floats as signed integers, we need to + // flip the exponent and mantissa bits in case of negative numbers. + // We effectively convert the numbers to "two's complement" form. + // + // To do the flipping, we construct a mask and XOR against it. + // We branchlessly calculate an "all-ones except for the sign bit" + // mask from negative-signed values: right shifting sign-extends + // the integer, so we "fill" the mask with sign bits, and then + // convert to unsigned to push one more zero bit. + // On positive values, the mask is all zeros, so it's a no-op. + left ^= (((left >> 63) as u64) >> 1) as i64; + right ^= (((right >> 63) as u64) >> 1) as i64; + + left.cmp(&right) + } + + /// Restrict a value to a certain interval unless it is NaN. + /// + /// Returns `max` if `self` is greater than `max`, and `min` if `self` is + /// less than `min`. Otherwise this returns `self`. + /// + /// Note that this function returns NaN if the initial value was NaN as + /// well. + /// + /// # Panics + /// + /// Panics if `min > max`, `min` is NaN, or `max` is NaN. + /// + /// # Examples + /// + /// ``` + /// assert!((-3.0f64).clamp(-2.0, 1.0) == -2.0); + /// assert!((0.0f64).clamp(-2.0, 1.0) == 0.0); + /// assert!((2.0f64).clamp(-2.0, 1.0) == 1.0); + /// assert!((f64::NAN).clamp(-2.0, 1.0).is_nan()); + /// ``` + #[must_use = "method returns a new number and does not mutate the original value"] + #[stable(feature = "clamp", since = "1.50.0")] + #[inline] + pub fn clamp(mut self, min: f64, max: f64) -> f64 { + assert!(min <= max); + if self < min { + self = min; + } + if self > max { + self = max; + } + self + } } diff --git a/crux-mir/lib/core/src/num/flt2dec/decoder.rs b/crux-mir/lib/core/src/num/flt2dec/decoder.rs index 2b74effbe..576386054 100644 --- a/crux-mir/lib/core/src/num/flt2dec/decoder.rs +++ b/crux-mir/lib/core/src/num/flt2dec/decoder.rs @@ -1,8 +1,7 @@ //! Decodes a floating-point value into individual parts and error ranges. -use crate::num::dec2flt::rawfp::RawFloat; +use crate::num::dec2flt::float::RawFloat; use crate::num::FpCategory; -use crate::{f32, f64}; /// Decoded unsigned finite value, such that: /// diff --git a/crux-mir/lib/core/src/num/flt2dec/mod.rs b/crux-mir/lib/core/src/num/flt2dec/mod.rs index f5cd26a18..1ff2e8c82 100644 --- a/crux-mir/lib/core/src/num/flt2dec/mod.rs +++ b/crux-mir/lib/core/src/num/flt2dec/mod.rs @@ -49,7 +49,7 @@ the supplied buffer and let the algorithm to return. # Implementation overview It is easy to get the floating point printing correct but slow (Russ Cox has -[demonstrated](http://research.swtch.com/ftoa) how it's easy), or incorrect but +[demonstrated](https://research.swtch.com/ftoa) how it's easy), or incorrect but fast (naïve division and modulo). But it is surprisingly hard to print floating point numbers correctly *and* efficiently. @@ -123,7 +123,9 @@ functions. )] pub use self::decoder::{decode, DecodableFloat, Decoded, FullDecoded}; -use crate::i16; + +use super::fmt::{Formatted, Part}; +use crate::mem::MaybeUninit; pub mod decoder; pub mod estimator; @@ -141,23 +143,23 @@ pub mod strategy { /// The exact formula is `ceil(# bits in mantissa * log_10 2 + 1)`. pub const MAX_SIG_DIGITS: usize = 17; -/// When `d[..n]` contains decimal digits, increase the last digit and propagate carry. -/// Returns a next digit when it causes the length change. +/// When `d` contains decimal digits, increase the last digit and propagate carry. +/// Returns a next digit when it causes the length to change. #[doc(hidden)] -pub fn round_up(d: &mut [u8], n: usize) -> Option { - match d[..n].iter().rposition(|&c| c != b'9') { +pub fn round_up(d: &mut [u8]) -> Option { + match d.iter().rposition(|&c| c != b'9') { Some(i) => { // d[i+1..n] is all nines d[i] += 1; - for j in i + 1..n { + for j in i + 1..d.len() { d[j] = b'0'; } None } - None if n > 0 => { + None if d.len() > 0 => { // 999..999 rounds to 1000..000 with an increased exponent d[0] = b'1'; - for j in 1..n { + for j in 1..d.len() { d[j] = b'0'; } Some(b'0') @@ -169,107 +171,6 @@ pub fn round_up(d: &mut [u8], n: usize) -> Option { } } -/// Formatted parts. -#[derive(Copy, Clone, PartialEq, Eq, Debug)] -pub enum Part<'a> { - /// Given number of zero digits. - Zero(usize), - /// A literal number up to 5 digits. - Num(u16), - /// A verbatim copy of given bytes. - Copy(&'a [u8]), -} - -impl<'a> Part<'a> { - /// Returns the exact byte length of given part. - pub fn len(&self) -> usize { - match *self { - Part::Zero(nzeroes) => nzeroes, - Part::Num(v) => { - if v < 1_000 { - if v < 10 { - 1 - } else if v < 100 { - 2 - } else { - 3 - } - } else { - if v < 10_000 { 4 } else { 5 } - } - } - Part::Copy(buf) => buf.len(), - } - } - - /// Writes a part into the supplied buffer. - /// Returns the number of written bytes, or `None` if the buffer is not enough. - /// (It may still leave partially written bytes in the buffer; do not rely on that.) - pub fn write(&self, out: &mut [u8]) -> Option { - let len = self.len(); - if out.len() >= len { - match *self { - Part::Zero(nzeroes) => { - for c in &mut out[..nzeroes] { - *c = b'0'; - } - } - Part::Num(mut v) => { - for c in out[..len].iter_mut().rev() { - *c = b'0' + (v % 10) as u8; - v /= 10; - } - } - Part::Copy(buf) => { - out[..buf.len()].copy_from_slice(buf); - } - } - Some(len) - } else { - None - } - } -} - -/// Formatted result containing one or more parts. -/// This can be written to the byte buffer or converted to the allocated string. -#[allow(missing_debug_implementations)] -#[derive(Clone)] -pub struct Formatted<'a> { - /// A byte slice representing a sign, either `""`, `"-"` or `"+"`. - pub sign: &'static str, - /// Formatted parts to be rendered after a sign and optional zero padding. - pub parts: &'a [Part<'a>], -} - -impl<'a> Formatted<'a> { - /// Returns the exact byte length of combined formatted result. - pub fn len(&self) -> usize { - let mut len = self.sign.len(); - for part in self.parts { - len += part.len(); - } - len - } - - /// Writes all formatted parts into the supplied buffer. - /// Returns the number of written bytes, or `None` if the buffer is not enough. - /// (It may still leave partially written bytes in the buffer; do not rely on that.) - pub fn write(&self, out: &mut [u8]) -> Option { - if out.len() < self.sign.len() { - return None; - } - out[..self.sign.len()].copy_from_slice(self.sign.as_bytes()); - - let mut written = self.sign.len(); - for part in self.parts { - let len = part.write(&mut out[written..])?; - written += len; - } - Some(written) - } -} - /// Formats given decimal digits `0.<...buf...> * 10^exp` into the decimal form /// with at least given number of fractional digits. The result is stored to /// the supplied parts array and a slice of written parts is returned. @@ -282,7 +183,7 @@ fn digits_to_dec_str<'a>( buf: &'a [u8], exp: i16, frac_digits: usize, - parts: &'a mut [Part<'a>], + parts: &'a mut [MaybeUninit>], ) -> &'a [Part<'a>] { assert!(!buf.is_empty()); assert!(buf[0] > b'0'); @@ -304,38 +205,44 @@ fn digits_to_dec_str<'a>( if exp <= 0 { // the decimal point is before rendered digits: [0.][000...000][1234][____] let minus_exp = -(exp as i32) as usize; - parts[0] = Part::Copy(b"0."); - parts[1] = Part::Zero(minus_exp); - parts[2] = Part::Copy(buf); + parts[0] = MaybeUninit::new(Part::Copy(b"0.")); + parts[1] = MaybeUninit::new(Part::Zero(minus_exp)); + parts[2] = MaybeUninit::new(Part::Copy(buf)); if frac_digits > buf.len() && frac_digits - buf.len() > minus_exp { - parts[3] = Part::Zero((frac_digits - buf.len()) - minus_exp); - &parts[..4] + parts[3] = MaybeUninit::new(Part::Zero((frac_digits - buf.len()) - minus_exp)); + // SAFETY: we just initialized the elements `..4`. + unsafe { MaybeUninit::slice_assume_init_ref(&parts[..4]) } } else { - &parts[..3] + // SAFETY: we just initialized the elements `..3`. + unsafe { MaybeUninit::slice_assume_init_ref(&parts[..3]) } } } else { let exp = exp as usize; if exp < buf.len() { // the decimal point is inside rendered digits: [12][.][34][____] - parts[0] = Part::Copy(&buf[..exp]); - parts[1] = Part::Copy(b"."); - parts[2] = Part::Copy(&buf[exp..]); + parts[0] = MaybeUninit::new(Part::Copy(&buf[..exp])); + parts[1] = MaybeUninit::new(Part::Copy(b".")); + parts[2] = MaybeUninit::new(Part::Copy(&buf[exp..])); if frac_digits > buf.len() - exp { - parts[3] = Part::Zero(frac_digits - (buf.len() - exp)); - &parts[..4] + parts[3] = MaybeUninit::new(Part::Zero(frac_digits - (buf.len() - exp))); + // SAFETY: we just initialized the elements `..4`. + unsafe { MaybeUninit::slice_assume_init_ref(&parts[..4]) } } else { - &parts[..3] + // SAFETY: we just initialized the elements `..3`. + unsafe { MaybeUninit::slice_assume_init_ref(&parts[..3]) } } } else { // the decimal point is after rendered digits: [1234][____0000] or [1234][__][.][__]. - parts[0] = Part::Copy(buf); - parts[1] = Part::Zero(exp - buf.len()); + parts[0] = MaybeUninit::new(Part::Copy(buf)); + parts[1] = MaybeUninit::new(Part::Zero(exp - buf.len())); if frac_digits > 0 { - parts[2] = Part::Copy(b"."); - parts[3] = Part::Zero(frac_digits); - &parts[..4] + parts[2] = MaybeUninit::new(Part::Copy(b".")); + parts[3] = MaybeUninit::new(Part::Zero(frac_digits)); + // SAFETY: we just initialized the elements `..4`. + unsafe { MaybeUninit::slice_assume_init_ref(&parts[..4]) } } else { - &parts[..2] + // SAFETY: we just initialized the elements `..2`. + unsafe { MaybeUninit::slice_assume_init_ref(&parts[..2]) } } } } @@ -355,7 +262,7 @@ fn digits_to_exp_str<'a>( exp: i16, min_ndigits: usize, upper: bool, - parts: &'a mut [Part<'a>], + parts: &'a mut [MaybeUninit>], ) -> &'a [Part<'a>] { assert!(!buf.is_empty()); assert!(buf[0] > b'0'); @@ -363,15 +270,15 @@ fn digits_to_exp_str<'a>( let mut n = 0; - parts[n] = Part::Copy(&buf[..1]); + parts[n] = MaybeUninit::new(Part::Copy(&buf[..1])); n += 1; if buf.len() > 1 || min_ndigits > 1 { - parts[n] = Part::Copy(b"."); - parts[n + 1] = Part::Copy(&buf[1..]); + parts[n] = MaybeUninit::new(Part::Copy(b".")); + parts[n + 1] = MaybeUninit::new(Part::Copy(&buf[1..])); n += 2; if min_ndigits > buf.len() { - parts[n] = Part::Zero(min_ndigits - buf.len()); + parts[n] = MaybeUninit::new(Part::Zero(min_ndigits - buf.len())); n += 1; } } @@ -379,26 +286,23 @@ fn digits_to_exp_str<'a>( // 0.1234 x 10^exp = 1.234 x 10^(exp-1) let exp = exp as i32 - 1; // avoid underflow when exp is i16::MIN if exp < 0 { - parts[n] = Part::Copy(if upper { b"E-" } else { b"e-" }); - parts[n + 1] = Part::Num(-exp as u16); + parts[n] = MaybeUninit::new(Part::Copy(if upper { b"E-" } else { b"e-" })); + parts[n + 1] = MaybeUninit::new(Part::Num(-exp as u16)); } else { - parts[n] = Part::Copy(if upper { b"E" } else { b"e" }); - parts[n + 1] = Part::Num(exp as u16); + parts[n] = MaybeUninit::new(Part::Copy(if upper { b"E" } else { b"e" })); + parts[n + 1] = MaybeUninit::new(Part::Num(exp as u16)); } - &parts[..n + 2] + // SAFETY: we just initialized the elements `..n + 2`. + unsafe { MaybeUninit::slice_assume_init_ref(&parts[..n + 2]) } } /// Sign formatting options. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum Sign { - /// Prints `-` only for the negative non-zero values. - Minus, // -inf -1 0 0 1 inf nan - /// Prints `-` only for any negative values (including the negative zero). - MinusRaw, // -inf -1 -0 0 1 inf nan - /// Prints `-` for the negative non-zero values, or `+` otherwise. - MinusPlus, // -inf -1 +0 +0 +1 +inf nan - /// Prints `-` for any negative values (including the negative zero), or `+` otherwise. - MinusPlusRaw, // -inf -1 -0 +0 +1 +inf nan + /// Prints `-` for any negative value. + Minus, // -inf -1 -0 0 1 inf nan + /// Prints `-` for any negative value, or `+` otherwise. + MinusPlus, // -inf -1 -0 +0 +1 +inf nan } /// Returns the static byte string corresponding to the sign to be formatted. @@ -406,30 +310,14 @@ pub enum Sign { fn determine_sign(sign: Sign, decoded: &FullDecoded, negative: bool) -> &'static str { match (*decoded, sign) { (FullDecoded::Nan, _) => "", - (FullDecoded::Zero, Sign::Minus) => "", - (FullDecoded::Zero, Sign::MinusRaw) => { + (_, Sign::Minus) => { if negative { "-" } else { "" } } - (FullDecoded::Zero, Sign::MinusPlus) => "+", - (FullDecoded::Zero, Sign::MinusPlusRaw) => { - if negative { - "-" - } else { - "+" - } - } - (_, Sign::Minus) | (_, Sign::MinusRaw) => { - if negative { - "-" - } else { - "" - } - } - (_, Sign::MinusPlus) | (_, Sign::MinusPlusRaw) => { + (_, Sign::MinusPlus) => { if negative { "-" } else { @@ -447,6 +335,7 @@ fn determine_sign(sign: Sign, decoded: &FullDecoded, negative: bool) -> &'static /// (which can be an empty string if no sign is rendered). /// /// `format_shortest` should be the underlying digit-generation function. +/// It should return the part of the buffer that it initialized. /// You probably would want `strategy::grisu::format_shortest` for this. /// /// `frac_digits` can be less than the number of actual fractional digits in `v`; @@ -462,12 +351,12 @@ pub fn to_shortest_str<'a, T, F>( v: T, sign: Sign, frac_digits: usize, - buf: &'a mut [u8], - parts: &'a mut [Part<'a>], + buf: &'a mut [MaybeUninit], + parts: &'a mut [MaybeUninit>], ) -> Formatted<'a> where T: DecodableFloat, - F: FnMut(&Decoded, &mut [u8]) -> (usize, i16), + F: FnMut(&Decoded, &'a mut [MaybeUninit]) -> (&'a [u8], i16), { assert!(parts.len() >= 4); assert!(buf.len() >= MAX_SIG_DIGITS); @@ -476,27 +365,37 @@ where let sign = determine_sign(sign, &full_decoded, negative); match full_decoded { FullDecoded::Nan => { - parts[0] = Part::Copy(b"NaN"); - Formatted { sign, parts: &parts[..1] } + parts[0] = MaybeUninit::new(Part::Copy(b"NaN")); + // SAFETY: we just initialized the elements `..1`. + Formatted { sign, parts: unsafe { MaybeUninit::slice_assume_init_ref(&parts[..1]) } } } FullDecoded::Infinite => { - parts[0] = Part::Copy(b"inf"); - Formatted { sign, parts: &parts[..1] } + parts[0] = MaybeUninit::new(Part::Copy(b"inf")); + // SAFETY: we just initialized the elements `..1`. + Formatted { sign, parts: unsafe { MaybeUninit::slice_assume_init_ref(&parts[..1]) } } } FullDecoded::Zero => { if frac_digits > 0 { // [0.][0000] - parts[0] = Part::Copy(b"0."); - parts[1] = Part::Zero(frac_digits); - Formatted { sign, parts: &parts[..2] } + parts[0] = MaybeUninit::new(Part::Copy(b"0.")); + parts[1] = MaybeUninit::new(Part::Zero(frac_digits)); + Formatted { + sign, + // SAFETY: we just initialized the elements `..2`. + parts: unsafe { MaybeUninit::slice_assume_init_ref(&parts[..2]) }, + } } else { - parts[0] = Part::Copy(b"0"); - Formatted { sign, parts: &parts[..1] } + parts[0] = MaybeUninit::new(Part::Copy(b"0")); + Formatted { + sign, + // SAFETY: we just initialized the elements `..1`. + parts: unsafe { MaybeUninit::slice_assume_init_ref(&parts[..1]) }, + } } } FullDecoded::Finite(ref decoded) => { - let (len, exp) = format_shortest(decoded, buf); - Formatted { sign, parts: digits_to_dec_str(&buf[..len], exp, frac_digits, parts) } + let (buf, exp) = format_shortest(decoded, buf); + Formatted { sign, parts: digits_to_dec_str(buf, exp, frac_digits, parts) } } } } @@ -510,6 +409,7 @@ where /// an empty string if no sign is rendered). /// /// `format_shortest` should be the underlying digit-generation function. +/// It should return the part of the buffer that it initialized. /// You probably would want `strategy::grisu::format_shortest` for this. /// /// The `dec_bounds` is a tuple `(lo, hi)` such that the number is formatted @@ -526,12 +426,12 @@ pub fn to_shortest_exp_str<'a, T, F>( sign: Sign, dec_bounds: (i16, i16), upper: bool, - buf: &'a mut [u8], - parts: &'a mut [Part<'a>], + buf: &'a mut [MaybeUninit], + parts: &'a mut [MaybeUninit>], ) -> Formatted<'a> where T: DecodableFloat, - F: FnMut(&Decoded, &mut [u8]) -> (usize, i16), + F: FnMut(&Decoded, &'a mut [MaybeUninit]) -> (&'a [u8], i16), { assert!(parts.len() >= 6); assert!(buf.len() >= MAX_SIG_DIGITS); @@ -541,28 +441,31 @@ where let sign = determine_sign(sign, &full_decoded, negative); match full_decoded { FullDecoded::Nan => { - parts[0] = Part::Copy(b"NaN"); - Formatted { sign, parts: &parts[..1] } + parts[0] = MaybeUninit::new(Part::Copy(b"NaN")); + // SAFETY: we just initialized the elements `..1`. + Formatted { sign, parts: unsafe { MaybeUninit::slice_assume_init_ref(&parts[..1]) } } } FullDecoded::Infinite => { - parts[0] = Part::Copy(b"inf"); - Formatted { sign, parts: &parts[..1] } + parts[0] = MaybeUninit::new(Part::Copy(b"inf")); + // SAFETY: we just initialized the elements `..1`. + Formatted { sign, parts: unsafe { MaybeUninit::slice_assume_init_ref(&parts[..1]) } } } FullDecoded::Zero => { parts[0] = if dec_bounds.0 <= 0 && 0 < dec_bounds.1 { - Part::Copy(b"0") + MaybeUninit::new(Part::Copy(b"0")) } else { - Part::Copy(if upper { b"0E0" } else { b"0e0" }) + MaybeUninit::new(Part::Copy(if upper { b"0E0" } else { b"0e0" })) }; - Formatted { sign, parts: &parts[..1] } + // SAFETY: we just initialized the elements `..1`. + Formatted { sign, parts: unsafe { MaybeUninit::slice_assume_init_ref(&parts[..1]) } } } FullDecoded::Finite(ref decoded) => { - let (len, exp) = format_shortest(decoded, buf); + let (buf, exp) = format_shortest(decoded, buf); let vis_exp = exp as i32 - 1; let parts = if dec_bounds.0 as i32 <= vis_exp && vis_exp < dec_bounds.1 as i32 { - digits_to_dec_str(&buf[..len], exp, 0, parts) + digits_to_dec_str(buf, exp, 0, parts) } else { - digits_to_exp_str(&buf[..len], exp, 0, upper, parts) + digits_to_exp_str(buf, exp, 0, upper, parts) }; Formatted { sign, parts } } @@ -601,6 +504,7 @@ fn estimate_max_buf_len(exp: i16) -> usize { /// an empty string if no sign is rendered). /// /// `format_exact` should be the underlying digit-generation function. +/// It should return the part of the buffer that it initialized. /// You probably would want `strategy::grisu::format_exact` for this. /// /// The byte buffer should be at least `ndigits` bytes long unless `ndigits` is @@ -614,12 +518,12 @@ pub fn to_exact_exp_str<'a, T, F>( sign: Sign, ndigits: usize, upper: bool, - buf: &'a mut [u8], - parts: &'a mut [Part<'a>], + buf: &'a mut [MaybeUninit], + parts: &'a mut [MaybeUninit>], ) -> Formatted<'a> where T: DecodableFloat, - F: FnMut(&Decoded, &mut [u8], i16) -> (usize, i16), + F: FnMut(&Decoded, &'a mut [MaybeUninit], i16) -> (&'a [u8], i16), { assert!(parts.len() >= 6); assert!(ndigits > 0); @@ -628,23 +532,33 @@ where let sign = determine_sign(sign, &full_decoded, negative); match full_decoded { FullDecoded::Nan => { - parts[0] = Part::Copy(b"NaN"); - Formatted { sign, parts: &parts[..1] } + parts[0] = MaybeUninit::new(Part::Copy(b"NaN")); + // SAFETY: we just initialized the elements `..1`. + Formatted { sign, parts: unsafe { MaybeUninit::slice_assume_init_ref(&parts[..1]) } } } FullDecoded::Infinite => { - parts[0] = Part::Copy(b"inf"); - Formatted { sign, parts: &parts[..1] } + parts[0] = MaybeUninit::new(Part::Copy(b"inf")); + // SAFETY: we just initialized the elements `..1`. + Formatted { sign, parts: unsafe { MaybeUninit::slice_assume_init_ref(&parts[..1]) } } } FullDecoded::Zero => { if ndigits > 1 { // [0.][0000][e0] - parts[0] = Part::Copy(b"0."); - parts[1] = Part::Zero(ndigits - 1); - parts[2] = Part::Copy(if upper { b"E0" } else { b"e0" }); - Formatted { sign, parts: &parts[..3] } + parts[0] = MaybeUninit::new(Part::Copy(b"0.")); + parts[1] = MaybeUninit::new(Part::Zero(ndigits - 1)); + parts[2] = MaybeUninit::new(Part::Copy(if upper { b"E0" } else { b"e0" })); + Formatted { + sign, + // SAFETY: we just initialized the elements `..3`. + parts: unsafe { MaybeUninit::slice_assume_init_ref(&parts[..3]) }, + } } else { - parts[0] = Part::Copy(if upper { b"0E0" } else { b"0e0" }); - Formatted { sign, parts: &parts[..1] } + parts[0] = MaybeUninit::new(Part::Copy(if upper { b"0E0" } else { b"0e0" })); + Formatted { + sign, + // SAFETY: we just initialized the elements `..1`. + parts: unsafe { MaybeUninit::slice_assume_init_ref(&parts[..1]) }, + } } } FullDecoded::Finite(ref decoded) => { @@ -652,8 +566,8 @@ where assert!(buf.len() >= ndigits || buf.len() >= maxlen); let trunc = if ndigits < maxlen { ndigits } else { maxlen }; - let (len, exp) = format_exact(decoded, &mut buf[..trunc], i16::MIN); - Formatted { sign, parts: digits_to_exp_str(&buf[..len], exp, ndigits, upper, parts) } + let (buf, exp) = format_exact(decoded, &mut buf[..trunc], i16::MIN); + Formatted { sign, parts: digits_to_exp_str(buf, exp, ndigits, upper, parts) } } } } @@ -666,6 +580,7 @@ where /// (which can be an empty string if no sign is rendered). /// /// `format_exact` should be the underlying digit-generation function. +/// It should return the part of the buffer that it initialized. /// You probably would want `strategy::grisu::format_exact` for this. /// /// The byte buffer should be enough for the output unless `frac_digits` is @@ -678,12 +593,12 @@ pub fn to_exact_fixed_str<'a, T, F>( v: T, sign: Sign, frac_digits: usize, - buf: &'a mut [u8], - parts: &'a mut [Part<'a>], + buf: &'a mut [MaybeUninit], + parts: &'a mut [MaybeUninit>], ) -> Formatted<'a> where T: DecodableFloat, - F: FnMut(&Decoded, &mut [u8], i16) -> (usize, i16), + F: FnMut(&Decoded, &'a mut [MaybeUninit], i16) -> (&'a [u8], i16), { assert!(parts.len() >= 4); @@ -691,22 +606,32 @@ where let sign = determine_sign(sign, &full_decoded, negative); match full_decoded { FullDecoded::Nan => { - parts[0] = Part::Copy(b"NaN"); - Formatted { sign, parts: &parts[..1] } + parts[0] = MaybeUninit::new(Part::Copy(b"NaN")); + // SAFETY: we just initialized the elements `..1`. + Formatted { sign, parts: unsafe { MaybeUninit::slice_assume_init_ref(&parts[..1]) } } } FullDecoded::Infinite => { - parts[0] = Part::Copy(b"inf"); - Formatted { sign, parts: &parts[..1] } + parts[0] = MaybeUninit::new(Part::Copy(b"inf")); + // SAFETY: we just initialized the elements `..1`. + Formatted { sign, parts: unsafe { MaybeUninit::slice_assume_init_ref(&parts[..1]) } } } FullDecoded::Zero => { if frac_digits > 0 { // [0.][0000] - parts[0] = Part::Copy(b"0."); - parts[1] = Part::Zero(frac_digits); - Formatted { sign, parts: &parts[..2] } + parts[0] = MaybeUninit::new(Part::Copy(b"0.")); + parts[1] = MaybeUninit::new(Part::Zero(frac_digits)); + Formatted { + sign, + // SAFETY: we just initialized the elements `..2`. + parts: unsafe { MaybeUninit::slice_assume_init_ref(&parts[..2]) }, + } } else { - parts[0] = Part::Copy(b"0"); - Formatted { sign, parts: &parts[..1] } + parts[0] = MaybeUninit::new(Part::Copy(b"0")); + Formatted { + sign, + // SAFETY: we just initialized the elements `..1`. + parts: unsafe { MaybeUninit::slice_assume_init_ref(&parts[..1]) }, + } } } FullDecoded::Finite(ref decoded) => { @@ -717,23 +642,31 @@ where // `format_exact` will end rendering digits much earlier in this case, // because we are strictly limited by `maxlen`. let limit = if frac_digits < 0x8000 { -(frac_digits as i16) } else { i16::MIN }; - let (len, exp) = format_exact(decoded, &mut buf[..maxlen], limit); + let (buf, exp) = format_exact(decoded, &mut buf[..maxlen], limit); if exp <= limit { // the restriction couldn't been met, so this should render like zero no matter // `exp` was. this does not include the case that the restriction has been met // only after the final rounding-up; it's a regular case with `exp = limit + 1`. - debug_assert_eq!(len, 0); + debug_assert_eq!(buf.len(), 0); if frac_digits > 0 { // [0.][0000] - parts[0] = Part::Copy(b"0."); - parts[1] = Part::Zero(frac_digits); - Formatted { sign, parts: &parts[..2] } + parts[0] = MaybeUninit::new(Part::Copy(b"0.")); + parts[1] = MaybeUninit::new(Part::Zero(frac_digits)); + Formatted { + sign, + // SAFETY: we just initialized the elements `..2`. + parts: unsafe { MaybeUninit::slice_assume_init_ref(&parts[..2]) }, + } } else { - parts[0] = Part::Copy(b"0"); - Formatted { sign, parts: &parts[..1] } + parts[0] = MaybeUninit::new(Part::Copy(b"0")); + Formatted { + sign, + // SAFETY: we just initialized the elements `..1`. + parts: unsafe { MaybeUninit::slice_assume_init_ref(&parts[..1]) }, + } } } else { - Formatted { sign, parts: digits_to_dec_str(&buf[..len], exp, frac_digits, parts) } + Formatted { sign, parts: digits_to_dec_str(buf, exp, frac_digits, parts) } } } } diff --git a/crux-mir/lib/core/src/num/flt2dec/strategy/dragon.rs b/crux-mir/lib/core/src/num/flt2dec/strategy/dragon.rs index c8de00043..71b14d0ae 100644 --- a/crux-mir/lib/core/src/num/flt2dec/strategy/dragon.rs +++ b/crux-mir/lib/core/src/num/flt2dec/strategy/dragon.rs @@ -5,6 +5,7 @@ //! quickly and accurately. SIGPLAN Not. 31, 5 (May. 1996), 108-116. use crate::cmp::Ordering; +use crate::mem::MaybeUninit; use crate::num::bignum::Big32x40 as Big; use crate::num::bignum::Digit32 as Digit; @@ -97,7 +98,10 @@ fn div_rem_upto_16<'a>( } /// The shortest mode implementation for Dragon. -pub fn format_shortest(d: &Decoded, buf: &mut [u8]) -> (/*#digits*/ usize, /*exp*/ i16) { +pub fn format_shortest<'a>( + d: &Decoded, + buf: &'a mut [MaybeUninit], +) -> (/*digits*/ &'a [u8], /*exp*/ i16) { // the number `v` to format is known to be: // - equal to `mant * 2^exp`; // - preceded by `(mant - 2 * minus) * 2^exp` in the original type; and @@ -186,7 +190,7 @@ pub fn format_shortest(d: &Decoded, buf: &mut [u8]) -> (/*#digits*/ usize, /*exp // generate one digit: `d[n] = floor(mant / scale) < 10`. let (d, _) = div_rem_upto_16(&mut mant, &scale, &scale2, &scale4, &scale8); debug_assert!(d < 10); - buf[i] = b'0' + d; + buf[i] = MaybeUninit::new(b'0' + d); i += 1; // this is a simplified description of the modified Dragon algorithm. @@ -241,18 +245,24 @@ pub fn format_shortest(d: &Decoded, buf: &mut [u8]) -> (/*#digits*/ usize, /*exp // if rounding up changes the length, the exponent should also change. // it seems that this condition is very hard to satisfy (possibly impossible), // but we are just being safe and consistent here. - if let Some(c) = round_up(buf, i) { - buf[i] = c; + // SAFETY: we initialized that memory above. + if let Some(c) = round_up(unsafe { MaybeUninit::slice_assume_init_mut(&mut buf[..i]) }) { + buf[i] = MaybeUninit::new(c); i += 1; k += 1; } } - (i, k) + // SAFETY: we initialized that memory above. + (unsafe { MaybeUninit::slice_assume_init_ref(&buf[..i]) }, k) } /// The exact and fixed mode implementation for Dragon. -pub fn format_exact(d: &Decoded, buf: &mut [u8], limit: i16) -> (/*#digits*/ usize, /*exp*/ i16) { +pub fn format_exact<'a>( + d: &Decoded, + buf: &'a mut [MaybeUninit], + limit: i16, +) -> (/*digits*/ &'a [u8], /*exp*/ i16) { assert!(d.mant > 0); assert!(d.minus > 0); assert!(d.plus > 0); @@ -319,9 +329,10 @@ pub fn format_exact(d: &Decoded, buf: &mut [u8], limit: i16) -> (/*#digits*/ usi // following digits are all zeroes, we stop here // do *not* try to perform rounding! rather, fill remaining digits. for c in &mut buf[i..len] { - *c = b'0'; + *c = MaybeUninit::new(b'0'); } - return (len, k); + // SAFETY: we initialized that memory above. + return (unsafe { MaybeUninit::slice_assume_init_ref(&buf[..len]) }, k); } let mut d = 0; @@ -343,7 +354,7 @@ pub fn format_exact(d: &Decoded, buf: &mut [u8], limit: i16) -> (/*#digits*/ usi } debug_assert!(mant < scale); debug_assert!(d < 10); - buf[i] = b'0' + d; + buf[i] = MaybeUninit::new(b'0' + d); mant.mul_small(10); } } @@ -353,21 +364,25 @@ pub fn format_exact(d: &Decoded, buf: &mut [u8], limit: i16) -> (/*#digits*/ usi // round to even (i.e., avoid rounding up when the prior digit is even). let order = mant.cmp(scale.mul_small(5)); if order == Ordering::Greater - || (order == Ordering::Equal && (len == 0 || buf[len - 1] & 1 == 1)) + || (order == Ordering::Equal + // SAFETY: `buf[len-1]` is initialized. + && len > 0 && unsafe { buf[len - 1].assume_init() } & 1 == 1) { // if rounding up changes the length, the exponent should also change. // but we've been requested a fixed number of digits, so do not alter the buffer... - if let Some(c) = round_up(buf, len) { + // SAFETY: we initialized that memory above. + if let Some(c) = round_up(unsafe { MaybeUninit::slice_assume_init_mut(&mut buf[..len]) }) { // ...unless we've been requested the fixed precision instead. // we also need to check that, if the original buffer was empty, // the additional digit can only be added when `k == limit` (edge case). k += 1; if k > limit && len < buf.len() { - buf[len] = c; + buf[len] = MaybeUninit::new(c); len += 1; } } } - (len, k) + // SAFETY: we initialized that memory above. + (unsafe { MaybeUninit::slice_assume_init_ref(&buf[..len]) }, k) } diff --git a/crux-mir/lib/core/src/num/flt2dec/strategy/grisu.rs b/crux-mir/lib/core/src/num/flt2dec/strategy/grisu.rs index 1e2db212d..ed3e0edaf 100644 --- a/crux-mir/lib/core/src/num/flt2dec/strategy/grisu.rs +++ b/crux-mir/lib/core/src/num/flt2dec/strategy/grisu.rs @@ -5,6 +5,7 @@ //! [^1]: Florian Loitsch. 2010. Printing floating-point numbers quickly and //! accurately with integers. SIGPLAN Not. 45, 6 (June 2010), 233-243. +use crate::mem::MaybeUninit; use crate::num::diy_float::Fp; use crate::num::flt2dec::{round_up, Decoded, MAX_SIG_DIGITS}; @@ -161,10 +162,10 @@ pub fn max_pow10_no_more_than(x: u32) -> (u8, u32) { /// The shortest mode implementation for Grisu. /// /// It returns `None` when it would return an inexact representation otherwise. -pub fn format_shortest_opt( +pub fn format_shortest_opt<'a>( d: &Decoded, - buf: &mut [u8], -) -> Option<(/*#digits*/ usize, /*exp*/ i16)> { + buf: &'a mut [MaybeUninit], +) -> Option<(/*digits*/ &'a [u8], /*exp*/ i16)> { assert!(d.mant > 0); assert!(d.minus > 0); assert!(d.plus > 0); @@ -252,7 +253,6 @@ pub fn format_shortest_opt( let delta1frac = delta1 & ((1 << e) - 1); // render integral parts, while checking for the accuracy at each step. - let mut kappa = max_kappa as i16; let mut ten_kappa = max_ten_kappa; // 10^kappa let mut remainder = plus1int; // digits yet to be rendered loop { @@ -266,26 +266,33 @@ pub fn format_shortest_opt( let q = remainder / ten_kappa; let r = remainder % ten_kappa; debug_assert!(q < 10); - buf[i] = b'0' + q as u8; + buf[i] = MaybeUninit::new(b'0' + q as u8); i += 1; let plus1rem = ((r as u64) << e) + plus1frac; // == (plus1 % 10^kappa) * 2^e if plus1rem < delta1 { // `plus1 % 10^kappa < delta1 = plus1 - minus1`; we've found the correct `kappa`. let ten_kappa = (ten_kappa as u64) << e; // scale 10^kappa back to the shared exponent - return round_and_weed(&mut buf[..i], exp, plus1rem, delta1, plus1 - v.f, ten_kappa, 1); + return round_and_weed( + // SAFETY: we initialized that memory above. + unsafe { MaybeUninit::slice_assume_init_mut(&mut buf[..i]) }, + exp, + plus1rem, + delta1, + plus1 - v.f, + ten_kappa, + 1, + ); } // break the loop when we have rendered all integral digits. // the exact number of digits is `max_kappa + 1` as `plus1 < 10^(max_kappa+1)`. if i > max_kappa as usize { debug_assert_eq!(ten_kappa, 1); - debug_assert_eq!(kappa, 0); break; } // restore invariants - kappa -= 1; ten_kappa /= 10; remainder = r; } @@ -310,13 +317,14 @@ pub fn format_shortest_opt( let q = remainder >> e; let r = remainder & ((1 << e) - 1); debug_assert!(q < 10); - buf[i] = b'0' + q as u8; + buf[i] = MaybeUninit::new(b'0' + q as u8); i += 1; if r < threshold { let ten_kappa = 1 << e; // implicit divisor return round_and_weed( - &mut buf[..i], + // SAFETY: we initialized that memory above. + unsafe { MaybeUninit::slice_assume_init_mut(&mut buf[..i]) }, exp, r, threshold, @@ -327,7 +335,6 @@ pub fn format_shortest_opt( } // restore invariants - kappa -= 1; remainder = r; } @@ -355,7 +362,7 @@ pub fn format_shortest_opt( plus1v: u64, ten_kappa: u64, ulp: u64, - ) -> Option<(usize, i16)> { + ) -> Option<(&[u8], i16)> { assert!(!buf.is_empty()); // produce two approximations to `v` (actually `plus1 - v`) within 1.5 ulps. @@ -437,20 +444,22 @@ pub fn format_shortest_opt( // this is too liberal, though, so we reject any `w(n)` not between `plus0` and `minus0`, // i.e., `plus1 - plus1w(n) <= minus0` or `plus1 - plus1w(n) >= plus0`. we utilize the facts // that `threshold = plus1 - minus1` and `plus1 - plus0 = minus0 - minus1 = 2 ulp`. - if 2 * ulp <= plus1w && plus1w <= threshold - 4 * ulp { - Some((buf.len(), exp)) - } else { - None - } + if 2 * ulp <= plus1w && plus1w <= threshold - 4 * ulp { Some((buf, exp)) } else { None } } } /// The shortest mode implementation for Grisu with Dragon fallback. /// /// This should be used for most cases. -pub fn format_shortest(d: &Decoded, buf: &mut [u8]) -> (/*#digits*/ usize, /*exp*/ i16) { +pub fn format_shortest<'a>( + d: &Decoded, + buf: &'a mut [MaybeUninit], +) -> (/*digits*/ &'a [u8], /*exp*/ i16) { use crate::num::flt2dec::strategy::dragon::format_shortest as fallback; - match format_shortest_opt(d, buf) { + // SAFETY: The borrow checker is not smart enough to let us use `buf` + // in the second branch, so we launder the lifetime here. But we only re-use + // `buf` if `format_shortest_opt` returned `None` so this is okay. + match format_shortest_opt(d, unsafe { &mut *(buf as *mut _) }) { Some(ret) => ret, None => fallback(d, buf), } @@ -459,11 +468,11 @@ pub fn format_shortest(d: &Decoded, buf: &mut [u8]) -> (/*#digits*/ usize, /*exp /// The exact and fixed mode implementation for Grisu. /// /// It returns `None` when it would return an inexact representation otherwise. -pub fn format_exact_opt( +pub fn format_exact_opt<'a>( d: &Decoded, - buf: &mut [u8], + buf: &'a mut [MaybeUninit], limit: i16, -) -> Option<(/*#digits*/ usize, /*exp*/ i16)> { +) -> Option<(/*digits*/ &'a [u8], /*exp*/ i16)> { assert!(d.mant > 0); assert!(d.mant < (1 << 61)); // we need at least three bits of additional precision assert!(!buf.is_empty()); @@ -510,7 +519,11 @@ pub fn format_exact_opt( // thus we are being sloppy here and widen the error range by a factor of 10. // this will increase the false negative rate, but only very, *very* slightly; // it can only matter noticeably when the mantissa is bigger than 60 bits. - return possibly_round(buf, 0, exp, limit, v.f / 10, (max_ten_kappa as u64) << e, err << e); + // + // SAFETY: `len=0`, so the obligation of having initialized this memory is trivial. + return unsafe { + possibly_round(buf, 0, exp, limit, v.f / 10, (max_ten_kappa as u64) << e, err << e) + }; } else if ((exp as i32 - limit as i32) as usize) < buf.len() { (exp - limit) as usize } else { @@ -534,13 +547,16 @@ pub fn format_exact_opt( let q = remainder / ten_kappa; let r = remainder % ten_kappa; debug_assert!(q < 10); - buf[i] = b'0' + q as u8; + buf[i] = MaybeUninit::new(b'0' + q as u8); i += 1; // is the buffer full? run the rounding pass with the remainder. if i == len { let vrem = ((r as u64) << e) + vfrac; // == (v % 10^kappa) * 2^e - return possibly_round(buf, len, exp, limit, vrem, (ten_kappa as u64) << e, err << e); + // SAFETY: we have initialized `len` many bytes. + return unsafe { + possibly_round(buf, len, exp, limit, vrem, (ten_kappa as u64) << e, err << e) + }; } // break the loop when we have rendered all integral digits. @@ -585,12 +601,13 @@ pub fn format_exact_opt( let q = remainder >> e; let r = remainder & ((1 << e) - 1); debug_assert!(q < 10); - buf[i] = b'0' + q as u8; + buf[i] = MaybeUninit::new(b'0' + q as u8); i += 1; // is the buffer full? run the rounding pass with the remainder. if i == len { - return possibly_round(buf, len, exp, limit, r, 1 << e, err); + // SAFETY: we have initialized `len` many bytes. + return unsafe { possibly_round(buf, len, exp, limit, r, 1 << e, err) }; } // restore invariants @@ -610,15 +627,17 @@ pub fn format_exact_opt( // - `remainder = (v % 10^kappa) * k` // - `ten_kappa = 10^kappa * k` // - `ulp = 2^-e * k` - fn possibly_round( - buf: &mut [u8], + // + // SAFETY: the first `len` bytes of `buf` must be initialized. + unsafe fn possibly_round( + buf: &mut [MaybeUninit], mut len: usize, mut exp: i16, limit: i16, remainder: u64, ten_kappa: u64, ulp: u64, - ) -> Option<(usize, i16)> { + ) -> Option<(&[u8], i16)> { debug_assert!(remainder < ten_kappa); // 10^kappa @@ -677,7 +696,8 @@ pub fn format_exact_opt( // we've already verified that `ulp < 10^kappa / 2`, so as long as // `10^kappa` did not overflow after all, the second check is fine. if ten_kappa - remainder > remainder && ten_kappa - 2 * remainder >= 2 * ulp { - return Some((len, exp)); + // SAFETY: our caller initialized that memory. + return Some((unsafe { MaybeUninit::slice_assume_init_ref(&buf[..len]) }, exp)); } // :<------- remainder ------>| : @@ -698,17 +718,21 @@ pub fn format_exact_opt( // as `10^kappa` is never zero). also note that `remainder - ulp <= 10^kappa`, // so the second check does not overflow. if remainder > ulp && ten_kappa - (remainder - ulp) <= remainder - ulp { - if let Some(c) = round_up(buf, len) { + if let Some(c) = + // SAFETY: our caller must have initialized that memory. + round_up(unsafe { MaybeUninit::slice_assume_init_mut(&mut buf[..len]) }) + { // only add an additional digit when we've been requested the fixed precision. // we also need to check that, if the original buffer was empty, // the additional digit can only be added when `exp == limit` (edge case). exp += 1; if exp > limit && len < buf.len() { - buf[len] = c; + buf[len] = MaybeUninit::new(c); len += 1; } } - return Some((len, exp)); + // SAFETY: we and our caller initialized that memory. + return Some((unsafe { MaybeUninit::slice_assume_init_ref(&buf[..len]) }, exp)); } // otherwise we are doomed (i.e., some values between `v - 1 ulp` and `v + 1 ulp` are @@ -720,9 +744,16 @@ pub fn format_exact_opt( /// The exact and fixed mode implementation for Grisu with Dragon fallback. /// /// This should be used for most cases. -pub fn format_exact(d: &Decoded, buf: &mut [u8], limit: i16) -> (/*#digits*/ usize, /*exp*/ i16) { +pub fn format_exact<'a>( + d: &Decoded, + buf: &'a mut [MaybeUninit], + limit: i16, +) -> (/*digits*/ &'a [u8], /*exp*/ i16) { use crate::num::flt2dec::strategy::dragon::format_exact as fallback; - match format_exact_opt(d, buf, limit) { + // SAFETY: The borrow checker is not smart enough to let us use `buf` + // in the second branch, so we launder the lifetime here. But we only re-use + // `buf` if `format_exact_opt` returned `None` so this is okay. + match format_exact_opt(d, unsafe { &mut *(buf as *mut _) }, limit) { Some(ret) => ret, None => fallback(d, buf, limit), } diff --git a/crux-mir/lib/core/src/num/fmt.rs b/crux-mir/lib/core/src/num/fmt.rs new file mode 100644 index 000000000..ed6119715 --- /dev/null +++ b/crux-mir/lib/core/src/num/fmt.rs @@ -0,0 +1,108 @@ +//! Shared utilities used by both float and integer formatting. +#![doc(hidden)] +#![unstable( + feature = "numfmt", + reason = "internal routines only exposed for testing", + issue = "none" +)] + +/// Formatted parts. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub enum Part<'a> { + /// Given number of zero digits. + Zero(usize), + /// A literal number up to 5 digits. + Num(u16), + /// A verbatim copy of given bytes. + Copy(&'a [u8]), +} + +impl<'a> Part<'a> { + /// Returns the exact byte length of given part. + pub fn len(&self) -> usize { + match *self { + Part::Zero(nzeroes) => nzeroes, + Part::Num(v) => { + if v < 1_000 { + if v < 10 { + 1 + } else if v < 100 { + 2 + } else { + 3 + } + } else { + if v < 10_000 { 4 } else { 5 } + } + } + Part::Copy(buf) => buf.len(), + } + } + + /// Writes a part into the supplied buffer. + /// Returns the number of written bytes, or `None` if the buffer is not enough. + /// (It may still leave partially written bytes in the buffer; do not rely on that.) + pub fn write(&self, out: &mut [u8]) -> Option { + let len = self.len(); + if out.len() >= len { + match *self { + Part::Zero(nzeroes) => { + for c in &mut out[..nzeroes] { + *c = b'0'; + } + } + Part::Num(mut v) => { + for c in out[..len].iter_mut().rev() { + *c = b'0' + (v % 10) as u8; + v /= 10; + } + } + Part::Copy(buf) => { + out[..buf.len()].copy_from_slice(buf); + } + } + Some(len) + } else { + None + } + } +} + +/// Formatted result containing one or more parts. +/// This can be written to the byte buffer or converted to the allocated string. +#[allow(missing_debug_implementations)] +#[derive(Clone)] +pub struct Formatted<'a> { + /// A byte slice representing a sign, either `""`, `"-"` or `"+"`. + pub sign: &'static str, + /// Formatted parts to be rendered after a sign and optional zero padding. + pub parts: &'a [Part<'a>], +} + +impl<'a> Formatted<'a> { + /// Returns the exact byte length of combined formatted result. + pub fn len(&self) -> usize { + let mut len = self.sign.len(); + for part in self.parts { + len += part.len(); + } + len + } + + /// Writes all formatted parts into the supplied buffer. + /// Returns the number of written bytes, or `None` if the buffer is not enough. + /// (It may still leave partially written bytes in the buffer; do not rely on that.) + pub fn write(&self, out: &mut [u8]) -> Option { + if out.len() < self.sign.len() { + return None; + } + out[..self.sign.len()].copy_from_slice(self.sign.as_bytes()); + + let mut written = self.sign.len(); + for part in self.parts { + let len = part.write(&mut out[written..])?; + written += len; + } + Some(written) + } +} diff --git a/crux-mir/lib/core/src/num/i128.rs b/crux-mir/lib/core/src/num/i128.rs deleted file mode 100644 index 08cb79594..000000000 --- a/crux-mir/lib/core/src/num/i128.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! The 128-bit signed integer type. -//! -//! *[See also the `i128` primitive type](../../std/primitive.i128.html).* -//! -//! Although using these constants won’t cause compilation warnings, -//! new code should use the associated constants directly on the primitive type. - -#![stable(feature = "i128", since = "1.26.0")] - -int_module! { i128, #[stable(feature = "i128", since="1.26.0")] } diff --git a/crux-mir/lib/core/src/num/i16.rs b/crux-mir/lib/core/src/num/i16.rs deleted file mode 100644 index 288eaceba..000000000 --- a/crux-mir/lib/core/src/num/i16.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! The 16-bit signed integer type. -//! -//! *[See also the `i16` primitive type](../../std/primitive.i16.html).* -//! -//! Although using these constants won’t cause compilation warnings, -//! new code should use the associated constants directly on the primitive type. - -#![stable(feature = "rust1", since = "1.0.0")] - -int_module! { i16 } diff --git a/crux-mir/lib/core/src/num/i32.rs b/crux-mir/lib/core/src/num/i32.rs deleted file mode 100644 index 0e1a2ec56..000000000 --- a/crux-mir/lib/core/src/num/i32.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! The 32-bit signed integer type. -//! -//! *[See also the `i32` primitive type](../../std/primitive.i32.html).* -//! -//! Although using these constants won’t cause compilation warnings, -//! new code should use the associated constants directly on the primitive type. - -#![stable(feature = "rust1", since = "1.0.0")] - -int_module! { i32 } diff --git a/crux-mir/lib/core/src/num/i64.rs b/crux-mir/lib/core/src/num/i64.rs deleted file mode 100644 index 27f709271..000000000 --- a/crux-mir/lib/core/src/num/i64.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! The 64-bit signed integer type. -//! -//! *[See also the `i64` primitive type](../../std/primitive.i64.html).* -//! -//! Although using these constants won’t cause compilation warnings, -//! new code should use the associated constants directly on the primitive type. - -#![stable(feature = "rust1", since = "1.0.0")] - -int_module! { i64 } diff --git a/crux-mir/lib/core/src/num/i8.rs b/crux-mir/lib/core/src/num/i8.rs deleted file mode 100644 index e84b421e1..000000000 --- a/crux-mir/lib/core/src/num/i8.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! The 8-bit signed integer type. -//! -//! *[See also the `i8` primitive type](../../std/primitive.i8.html).* -//! -//! Although using these constants won’t cause compilation warnings, -//! new code should use the associated constants directly on the primitive type. - -#![stable(feature = "rust1", since = "1.0.0")] - -int_module! { i8 } diff --git a/crux-mir/lib/core/src/num/int_log10.rs b/crux-mir/lib/core/src/num/int_log10.rs new file mode 100644 index 000000000..80472528f --- /dev/null +++ b/crux-mir/lib/core/src/num/int_log10.rs @@ -0,0 +1,140 @@ +/// These functions compute the integer logarithm of their type, assuming +/// that someone has already checked that the value is strictly positive. + +// 0 < val <= u8::MAX +#[inline] +pub const fn u8(val: u8) -> u32 { + let val = val as u32; + + // For better performance, avoid branches by assembling the solution + // in the bits above the low 8 bits. + + // Adding c1 to val gives 10 in the top bits for val < 10, 11 for val >= 10 + const C1: u32 = 0b11_00000000 - 10; // 758 + // Adding c2 to val gives 01 in the top bits for val < 100, 10 for val >= 100 + const C2: u32 = 0b10_00000000 - 100; // 412 + + // Value of top bits: + // +c1 +c2 1&2 + // 0..=9 10 01 00 = 0 + // 10..=99 11 01 01 = 1 + // 100..=255 11 10 10 = 2 + ((val + C1) & (val + C2)) >> 8 +} + +// 0 < val < 100_000 +#[inline] +const fn less_than_5(val: u32) -> u32 { + // Similar to u8, when adding one of these constants to val, + // we get two possible bit patterns above the low 17 bits, + // depending on whether val is below or above the threshold. + const C1: u32 = 0b011_00000000000000000 - 10; // 393206 + const C2: u32 = 0b100_00000000000000000 - 100; // 524188 + const C3: u32 = 0b111_00000000000000000 - 1000; // 916504 + const C4: u32 = 0b100_00000000000000000 - 10000; // 514288 + + // Value of top bits: + // +c1 +c2 1&2 +c3 +c4 3&4 ^ + // 0..=9 010 011 010 110 011 010 000 = 0 + // 10..=99 011 011 011 110 011 010 001 = 1 + // 100..=999 011 100 000 110 011 010 010 = 2 + // 1000..=9999 011 100 000 111 011 011 011 = 3 + // 10000..=99999 011 100 000 111 100 100 100 = 4 + (((val + C1) & (val + C2)) ^ ((val + C3) & (val + C4))) >> 17 +} + +// 0 < val <= u16::MAX +#[inline] +pub const fn u16(val: u16) -> u32 { + less_than_5(val as u32) +} + +// 0 < val <= u32::MAX +#[inline] +pub const fn u32(mut val: u32) -> u32 { + let mut log = 0; + if val >= 100_000 { + val /= 100_000; + log += 5; + } + log + less_than_5(val) +} + +// 0 < val <= u64::MAX +#[inline] +pub const fn u64(mut val: u64) -> u32 { + let mut log = 0; + if val >= 10_000_000_000 { + val /= 10_000_000_000; + log += 10; + } + if val >= 100_000 { + val /= 100_000; + log += 5; + } + log + less_than_5(val as u32) +} + +// 0 < val <= u128::MAX +#[inline] +pub const fn u128(mut val: u128) -> u32 { + let mut log = 0; + if val >= 100_000_000_000_000_000_000_000_000_000_000 { + val /= 100_000_000_000_000_000_000_000_000_000_000; + log += 32; + return log + u32(val as u32); + } + if val >= 10_000_000_000_000_000 { + val /= 10_000_000_000_000_000; + log += 16; + } + log + u64(val as u64) +} + +#[cfg(target_pointer_width = "16")] +#[inline] +pub const fn usize(val: usize) -> u32 { + u16(val as _) +} + +#[cfg(target_pointer_width = "32")] +#[inline] +pub const fn usize(val: usize) -> u32 { + u32(val as _) +} + +#[cfg(target_pointer_width = "64")] +#[inline] +pub const fn usize(val: usize) -> u32 { + u64(val as _) +} + +// 0 < val <= i8::MAX +#[inline] +pub const fn i8(val: i8) -> u32 { + u8(val as u8) +} + +// 0 < val <= i16::MAX +#[inline] +pub const fn i16(val: i16) -> u32 { + u16(val as u16) +} + +// 0 < val <= i32::MAX +#[inline] +pub const fn i32(val: i32) -> u32 { + u32(val as u32) +} + +// 0 < val <= i64::MAX +#[inline] +pub const fn i64(val: i64) -> u32 { + u64(val as u64) +} + +// 0 < val <= i128::MAX +#[inline] +pub const fn i128(val: i128) -> u32 { + u128(val as u128) +} diff --git a/crux-mir/lib/core/src/num/int_macros.rs b/crux-mir/lib/core/src/num/int_macros.rs index b68a09e11..2cae98b8e 100644 --- a/crux-mir/lib/core/src/num/int_macros.rs +++ b/crux-mir/lib/core/src/num/int_macros.rs @@ -1,27 +1,2824 @@ -#![doc(hidden)] +macro_rules! int_impl { + ($SelfT:ty, $ActualT:ident, $UnsignedT:ty, $BITS:expr, $BITS_MINUS_ONE:expr, $Min:expr, $Max:expr, + $rot:expr, $rot_op:expr, $rot_result:expr, $swap_op:expr, $swapped:expr, + $reversed:expr, $le_bytes:expr, $be_bytes:expr, + $to_xe_bytes_doc:expr, $from_xe_bytes_doc:expr, + $bound_condition:expr) => { + /// The smallest value that can be represented by this integer type + #[doc = concat!("(−2", $BITS_MINUS_ONE, "", $bound_condition, ")")] + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN, ", stringify!($Min), ");")] + /// ``` + #[stable(feature = "assoc_int_consts", since = "1.43.0")] + pub const MIN: Self = !0 ^ ((!0 as $UnsignedT) >> 1) as Self; -macro_rules! doc_comment { - ($x:expr, $($tt:tt)*) => { - #[doc = $x] - $($tt)* - }; -} + /// The largest value that can be represented by this integer type + #[doc = concat!("(2", $BITS_MINUS_ONE, " − 1", $bound_condition, ")")] + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX, ", stringify!($Max), ");")] + /// ``` + #[stable(feature = "assoc_int_consts", since = "1.43.0")] + pub const MAX: Self = !Self::MIN; + + /// The size of this integer type in bits. + /// + /// # Examples + /// + /// ``` + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::BITS, ", stringify!($BITS), ");")] + /// ``` + #[stable(feature = "int_bits_const", since = "1.53.0")] + pub const BITS: u32 = $BITS; + + /// Converts a string slice in a given base to an integer. + /// + /// The string is expected to be an optional `+` or `-` sign followed by digits. + /// Leading and trailing whitespace represent an error. Digits are a subset of these characters, + /// depending on `radix`: + /// + /// * `0-9` + /// * `a-z` + /// * `A-Z` + /// + /// # Panics + /// + /// This function panics if `radix` is not in the range from 2 to 36. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::from_str_radix(\"A\", 16), Ok(10));")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn from_str_radix(src: &str, radix: u32) -> Result { + from_str_radix(src, radix) + } + + /// Returns the number of ones in the binary representation of `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = 0b100_0000", stringify!($SelfT), ";")] + /// + /// assert_eq!(n.count_ones(), 1); + /// ``` + /// + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[doc(alias = "popcount")] + #[doc(alias = "popcnt")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn count_ones(self) -> u32 { (self as $UnsignedT).count_ones() } + + /// Returns the number of zeros in the binary representation of `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.count_zeros(), 1);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn count_zeros(self) -> u32 { + (!self).count_ones() + } + + /// Returns the number of leading zeros in the binary representation of `self`. + /// + /// Depending on what you're doing with the value, you might also be interested in the + /// [`ilog2`] function which returns a consistent number, even if the type widens. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = -1", stringify!($SelfT), ";")] + /// + /// assert_eq!(n.leading_zeros(), 0); + /// ``` + #[doc = concat!("[`ilog2`]: ", stringify!($SelfT), "::ilog2")] + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn leading_zeros(self) -> u32 { + (self as $UnsignedT).leading_zeros() + } + + /// Returns the number of trailing zeros in the binary representation of `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = -4", stringify!($SelfT), ";")] + /// + /// assert_eq!(n.trailing_zeros(), 2); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn trailing_zeros(self) -> u32 { + (self as $UnsignedT).trailing_zeros() + } + + /// Returns the number of leading ones in the binary representation of `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = -1", stringify!($SelfT), ";")] + /// + #[doc = concat!("assert_eq!(n.leading_ones(), ", stringify!($BITS), ");")] + /// ``` + #[stable(feature = "leading_trailing_ones", since = "1.46.0")] + #[rustc_const_stable(feature = "leading_trailing_ones", since = "1.46.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn leading_ones(self) -> u32 { + (self as $UnsignedT).leading_ones() + } + + /// Returns the number of trailing ones in the binary representation of `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = 3", stringify!($SelfT), ";")] + /// + /// assert_eq!(n.trailing_ones(), 2); + /// ``` + #[stable(feature = "leading_trailing_ones", since = "1.46.0")] + #[rustc_const_stable(feature = "leading_trailing_ones", since = "1.46.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn trailing_ones(self) -> u32 { + (self as $UnsignedT).trailing_ones() + } + + /// Shifts the bits to the left by a specified amount, `n`, + /// wrapping the truncated bits to the end of the resulting integer. + /// + /// Please note this isn't the same operation as the `<<` shifting operator! + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = ", $rot_op, stringify!($SelfT), ";")] + #[doc = concat!("let m = ", $rot_result, ";")] + /// + #[doc = concat!("assert_eq!(n.rotate_left(", $rot, "), m);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn rotate_left(self, n: u32) -> Self { + (self as $UnsignedT).rotate_left(n) as Self + } + + /// Shifts the bits to the right by a specified amount, `n`, + /// wrapping the truncated bits to the beginning of the resulting + /// integer. + /// + /// Please note this isn't the same operation as the `>>` shifting operator! + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = ", $rot_result, stringify!($SelfT), ";")] + #[doc = concat!("let m = ", $rot_op, ";")] + /// + #[doc = concat!("assert_eq!(n.rotate_right(", $rot, "), m);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn rotate_right(self, n: u32) -> Self { + (self as $UnsignedT).rotate_right(n) as Self + } + + /// Reverses the byte order of the integer. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = ", $swap_op, stringify!($SelfT), ";")] + /// + /// let m = n.swap_bytes(); + /// + #[doc = concat!("assert_eq!(m, ", $swapped, ");")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn swap_bytes(self) -> Self { + (self as $UnsignedT).swap_bytes() as Self + } + + /// Reverses the order of bits in the integer. The least significant bit becomes the most significant bit, + /// second least-significant bit becomes second most-significant bit, etc. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = ", $swap_op, stringify!($SelfT), ";")] + /// let m = n.reverse_bits(); + /// + #[doc = concat!("assert_eq!(m, ", $reversed, ");")] + #[doc = concat!("assert_eq!(0, 0", stringify!($SelfT), ".reverse_bits());")] + /// ``` + #[stable(feature = "reverse_bits", since = "1.37.0")] + #[rustc_const_stable(feature = "reverse_bits", since = "1.37.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn reverse_bits(self) -> Self { + (self as $UnsignedT).reverse_bits() as Self + } + + /// Converts an integer from big endian to the target's endianness. + /// + /// On big endian this is a no-op. On little endian the bytes are swapped. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = 0x1A", stringify!($SelfT), ";")] + /// + /// if cfg!(target_endian = "big") { + #[doc = concat!(" assert_eq!(", stringify!($SelfT), "::from_be(n), n)")] + /// } else { + #[doc = concat!(" assert_eq!(", stringify!($SelfT), "::from_be(n), n.swap_bytes())")] + /// } + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_conversions", since = "1.32.0")] + #[must_use] + #[inline] + pub const fn from_be(x: Self) -> Self { + #[cfg(target_endian = "big")] + { + x + } + #[cfg(not(target_endian = "big"))] + { + x.swap_bytes() + } + } + + /// Converts an integer from little endian to the target's endianness. + /// + /// On little endian this is a no-op. On big endian the bytes are swapped. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = 0x1A", stringify!($SelfT), ";")] + /// + /// if cfg!(target_endian = "little") { + #[doc = concat!(" assert_eq!(", stringify!($SelfT), "::from_le(n), n)")] + /// } else { + #[doc = concat!(" assert_eq!(", stringify!($SelfT), "::from_le(n), n.swap_bytes())")] + /// } + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_conversions", since = "1.32.0")] + #[must_use] + #[inline] + pub const fn from_le(x: Self) -> Self { + #[cfg(target_endian = "little")] + { + x + } + #[cfg(not(target_endian = "little"))] + { + x.swap_bytes() + } + } + + /// Converts `self` to big endian from the target's endianness. + /// + /// On big endian this is a no-op. On little endian the bytes are swapped. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = 0x1A", stringify!($SelfT), ";")] + /// + /// if cfg!(target_endian = "big") { + /// assert_eq!(n.to_be(), n) + /// } else { + /// assert_eq!(n.to_be(), n.swap_bytes()) + /// } + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_conversions", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn to_be(self) -> Self { // or not to be? + #[cfg(target_endian = "big")] + { + self + } + #[cfg(not(target_endian = "big"))] + { + self.swap_bytes() + } + } + + /// Converts `self` to little endian from the target's endianness. + /// + /// On little endian this is a no-op. On big endian the bytes are swapped. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = 0x1A", stringify!($SelfT), ";")] + /// + /// if cfg!(target_endian = "little") { + /// assert_eq!(n.to_le(), n) + /// } else { + /// assert_eq!(n.to_le(), n.swap_bytes()) + /// } + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_conversions", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn to_le(self) -> Self { + #[cfg(target_endian = "little")] + { + self + } + #[cfg(not(target_endian = "little"))] + { + self.swap_bytes() + } + } + + /// Checked integer addition. Computes `self + rhs`, returning `None` + /// if overflow occurred. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MAX - 2).checked_add(1), Some(", stringify!($SelfT), "::MAX - 1));")] + #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MAX - 2).checked_add(3), None);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_checked_int_methods", since = "1.47.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_add(self, rhs: Self) -> Option { + let (a, b) = self.overflowing_add(rhs); + if unlikely!(b) {None} else {Some(a)} + } + + /// Unchecked integer addition. Computes `self + rhs`, assuming overflow + /// cannot occur. + /// + /// # Safety + /// + /// This results in undefined behavior when + #[doc = concat!("`self + rhs > ", stringify!($SelfT), "::MAX` or `self + rhs < ", stringify!($SelfT), "::MIN`,")] + /// i.e. when [`checked_add`] would return `None`. + /// + #[doc = concat!("[`checked_add`]: ", stringify!($SelfT), "::checked_add")] + #[unstable( + feature = "unchecked_math", + reason = "niche optimization path", + issue = "85122", + )] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn unchecked_add(self, rhs: Self) -> Self { + // SAFETY: the caller must uphold the safety contract for + // `unchecked_add`. + unsafe { intrinsics::unchecked_add(self, rhs) } + } + + /// Checked addition with an unsigned integer. Computes `self + rhs`, + /// returning `None` if overflow occurred. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(1", stringify!($SelfT), ".checked_add_unsigned(2), Some(3));")] + #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MAX - 2).checked_add_unsigned(3), None);")] + /// ``` + #[stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[rustc_const_stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_add_unsigned(self, rhs: $UnsignedT) -> Option { + let (a, b) = self.overflowing_add_unsigned(rhs); + if unlikely!(b) {None} else {Some(a)} + } + + /// Checked integer subtraction. Computes `self - rhs`, returning `None` if + /// overflow occurred. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MIN + 2).checked_sub(1), Some(", stringify!($SelfT), "::MIN + 1));")] + #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MIN + 2).checked_sub(3), None);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_checked_int_methods", since = "1.47.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_sub(self, rhs: Self) -> Option { + let (a, b) = self.overflowing_sub(rhs); + if unlikely!(b) {None} else {Some(a)} + } + + /// Unchecked integer subtraction. Computes `self - rhs`, assuming overflow + /// cannot occur. + /// + /// # Safety + /// + /// This results in undefined behavior when + #[doc = concat!("`self - rhs > ", stringify!($SelfT), "::MAX` or `self - rhs < ", stringify!($SelfT), "::MIN`,")] + /// i.e. when [`checked_sub`] would return `None`. + /// + #[doc = concat!("[`checked_sub`]: ", stringify!($SelfT), "::checked_sub")] + #[unstable( + feature = "unchecked_math", + reason = "niche optimization path", + issue = "85122", + )] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn unchecked_sub(self, rhs: Self) -> Self { + // SAFETY: the caller must uphold the safety contract for + // `unchecked_sub`. + unsafe { intrinsics::unchecked_sub(self, rhs) } + } + + /// Checked subtraction with an unsigned integer. Computes `self - rhs`, + /// returning `None` if overflow occurred. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(1", stringify!($SelfT), ".checked_sub_unsigned(2), Some(-1));")] + #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MIN + 2).checked_sub_unsigned(3), None);")] + /// ``` + #[stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[rustc_const_stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_sub_unsigned(self, rhs: $UnsignedT) -> Option { + let (a, b) = self.overflowing_sub_unsigned(rhs); + if unlikely!(b) {None} else {Some(a)} + } + + /// Checked integer multiplication. Computes `self * rhs`, returning `None` if + /// overflow occurred. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.checked_mul(1), Some(", stringify!($SelfT), "::MAX));")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.checked_mul(2), None);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_checked_int_methods", since = "1.47.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_mul(self, rhs: Self) -> Option { + let (a, b) = self.overflowing_mul(rhs); + if unlikely!(b) {None} else {Some(a)} + } + + /// Unchecked integer multiplication. Computes `self * rhs`, assuming overflow + /// cannot occur. + /// + /// # Safety + /// + /// This results in undefined behavior when + #[doc = concat!("`self * rhs > ", stringify!($SelfT), "::MAX` or `self * rhs < ", stringify!($SelfT), "::MIN`,")] + /// i.e. when [`checked_mul`] would return `None`. + /// + #[doc = concat!("[`checked_mul`]: ", stringify!($SelfT), "::checked_mul")] + #[unstable( + feature = "unchecked_math", + reason = "niche optimization path", + issue = "85122", + )] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn unchecked_mul(self, rhs: Self) -> Self { + // SAFETY: the caller must uphold the safety contract for + // `unchecked_mul`. + unsafe { intrinsics::unchecked_mul(self, rhs) } + } + + /// Checked integer division. Computes `self / rhs`, returning `None` if `rhs == 0` + /// or the division results in overflow. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MIN + 1).checked_div(-1), Some(", stringify!($Max), "));")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.checked_div(-1), None);")] + #[doc = concat!("assert_eq!((1", stringify!($SelfT), ").checked_div(0), None);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_checked_int_div", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_div(self, rhs: Self) -> Option { + if unlikely!(rhs == 0 || ((self == Self::MIN) && (rhs == -1))) { + None + } else { + // SAFETY: div by zero and by INT_MIN have been checked above + Some(unsafe { intrinsics::unchecked_div(self, rhs) }) + } + } + + /// Checked Euclidean division. Computes `self.div_euclid(rhs)`, + /// returning `None` if `rhs == 0` or the division results in overflow. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MIN + 1).checked_div_euclid(-1), Some(", stringify!($Max), "));")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.checked_div_euclid(-1), None);")] + #[doc = concat!("assert_eq!((1", stringify!($SelfT), ").checked_div_euclid(0), None);")] + /// ``` + #[stable(feature = "euclidean_division", since = "1.38.0")] + #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_div_euclid(self, rhs: Self) -> Option { + // Using `&` helps LLVM see that it is the same check made in division. + if unlikely!(rhs == 0 || ((self == Self::MIN) & (rhs == -1))) { + None + } else { + Some(self.div_euclid(rhs)) + } + } + + /// Checked integer remainder. Computes `self % rhs`, returning `None` if + /// `rhs == 0` or the division results in overflow. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".checked_rem(2), Some(1));")] + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".checked_rem(0), None);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.checked_rem(-1), None);")] + /// ``` + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_checked_int_div", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_rem(self, rhs: Self) -> Option { + if unlikely!(rhs == 0 || ((self == Self::MIN) && (rhs == -1))) { + None + } else { + // SAFETY: div by zero and by INT_MIN have been checked above + Some(unsafe { intrinsics::unchecked_rem(self, rhs) }) + } + } + + /// Checked Euclidean remainder. Computes `self.rem_euclid(rhs)`, returning `None` + /// if `rhs == 0` or the division results in overflow. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".checked_rem_euclid(2), Some(1));")] + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".checked_rem_euclid(0), None);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.checked_rem_euclid(-1), None);")] + /// ``` + #[stable(feature = "euclidean_division", since = "1.38.0")] + #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_rem_euclid(self, rhs: Self) -> Option { + // Using `&` helps LLVM see that it is the same check made in division. + if unlikely!(rhs == 0 || ((self == Self::MIN) & (rhs == -1))) { + None + } else { + Some(self.rem_euclid(rhs)) + } + } + + /// Checked negation. Computes `-self`, returning `None` if `self == MIN`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".checked_neg(), Some(-5));")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.checked_neg(), None);")] + /// ``` + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_checked_int_methods", since = "1.47.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_neg(self) -> Option { + let (a, b) = self.overflowing_neg(); + if unlikely!(b) {None} else {Some(a)} + } + + /// Checked shift left. Computes `self << rhs`, returning `None` if `rhs` is larger + /// than or equal to the number of bits in `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(0x1", stringify!($SelfT), ".checked_shl(4), Some(0x10));")] + #[doc = concat!("assert_eq!(0x1", stringify!($SelfT), ".checked_shl(129), None);")] + /// ``` + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_checked_int_methods", since = "1.47.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_shl(self, rhs: u32) -> Option { + let (a, b) = self.overflowing_shl(rhs); + if unlikely!(b) {None} else {Some(a)} + } + + /// Unchecked shift left. Computes `self << rhs`, assuming that + /// `rhs` is less than the number of bits in `self`. + /// + /// # Safety + /// + /// This results in undefined behavior if `rhs` is larger than + /// or equal to the number of bits in `self`, + /// i.e. when [`checked_shl`] would return `None`. + /// + #[doc = concat!("[`checked_shl`]: ", stringify!($SelfT), "::checked_shl")] + #[unstable( + feature = "unchecked_math", + reason = "niche optimization path", + issue = "85122", + )] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn unchecked_shl(self, rhs: u32) -> Self { + // SAFETY: the caller must uphold the safety contract for + // `unchecked_shl`. + // Any legal shift amount is losslessly representable in the self type. + unsafe { intrinsics::unchecked_shl(self, rhs.try_into().ok().unwrap_unchecked()) } + } + + /// Checked shift right. Computes `self >> rhs`, returning `None` if `rhs` is + /// larger than or equal to the number of bits in `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(0x10", stringify!($SelfT), ".checked_shr(4), Some(0x1));")] + #[doc = concat!("assert_eq!(0x10", stringify!($SelfT), ".checked_shr(128), None);")] + /// ``` + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_checked_int_methods", since = "1.47.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_shr(self, rhs: u32) -> Option { + let (a, b) = self.overflowing_shr(rhs); + if unlikely!(b) {None} else {Some(a)} + } + + /// Unchecked shift right. Computes `self >> rhs`, assuming that + /// `rhs` is less than the number of bits in `self`. + /// + /// # Safety + /// + /// This results in undefined behavior if `rhs` is larger than + /// or equal to the number of bits in `self`, + /// i.e. when [`checked_shr`] would return `None`. + /// + #[doc = concat!("[`checked_shr`]: ", stringify!($SelfT), "::checked_shr")] + #[unstable( + feature = "unchecked_math", + reason = "niche optimization path", + issue = "85122", + )] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn unchecked_shr(self, rhs: u32) -> Self { + // SAFETY: the caller must uphold the safety contract for + // `unchecked_shr`. + // Any legal shift amount is losslessly representable in the self type. + unsafe { intrinsics::unchecked_shr(self, rhs.try_into().ok().unwrap_unchecked()) } + } + + /// Checked absolute value. Computes `self.abs()`, returning `None` if + /// `self == MIN`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!((-5", stringify!($SelfT), ").checked_abs(), Some(5));")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.checked_abs(), None);")] + /// ``` + #[stable(feature = "no_panic_abs", since = "1.13.0")] + #[rustc_const_stable(feature = "const_checked_int_methods", since = "1.47.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_abs(self) -> Option { + if self.is_negative() { + self.checked_neg() + } else { + Some(self) + } + } + + /// Checked exponentiation. Computes `self.pow(exp)`, returning `None` if + /// overflow occurred. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(8", stringify!($SelfT), ".checked_pow(2), Some(64));")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.checked_pow(2), None);")] + /// ``` + + #[stable(feature = "no_panic_pow", since = "1.34.0")] + #[rustc_const_stable(feature = "const_int_pow", since = "1.50.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_pow(self, mut exp: u32) -> Option { + if exp == 0 { + return Some(1); + } + let mut base = self; + let mut acc: Self = 1; + + while exp > 1 { + if (exp & 1) == 1 { + acc = try_opt!(acc.checked_mul(base)); + } + exp /= 2; + base = try_opt!(base.checked_mul(base)); + } + // since exp!=0, finally the exp must be 1. + // Deal with the final bit of the exponent separately, since + // squaring the base afterwards is not necessary and may cause a + // needless overflow. + acc.checked_mul(base) + } + + /// Saturating integer addition. Computes `self + rhs`, saturating at the numeric + /// bounds instead of overflowing. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".saturating_add(1), 101);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.saturating_add(100), ", stringify!($SelfT), "::MAX);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.saturating_add(-1), ", stringify!($SelfT), "::MIN);")] + /// ``` + + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_saturating_int_methods", since = "1.47.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn saturating_add(self, rhs: Self) -> Self { + intrinsics::saturating_add(self, rhs) + } + + /// Saturating addition with an unsigned integer. Computes `self + rhs`, + /// saturating at the numeric bounds instead of overflowing. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(1", stringify!($SelfT), ".saturating_add_unsigned(2), 3);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.saturating_add_unsigned(100), ", stringify!($SelfT), "::MAX);")] + /// ``` + #[stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[rustc_const_stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn saturating_add_unsigned(self, rhs: $UnsignedT) -> Self { + // Overflow can only happen at the upper bound + // We cannot use `unwrap_or` here because it is not `const` + match self.checked_add_unsigned(rhs) { + Some(x) => x, + None => Self::MAX, + } + } + + /// Saturating integer subtraction. Computes `self - rhs`, saturating at the + /// numeric bounds instead of overflowing. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".saturating_sub(127), -27);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.saturating_sub(100), ", stringify!($SelfT), "::MIN);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.saturating_sub(-1), ", stringify!($SelfT), "::MAX);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_saturating_int_methods", since = "1.47.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn saturating_sub(self, rhs: Self) -> Self { + intrinsics::saturating_sub(self, rhs) + } + + /// Saturating subtraction with an unsigned integer. Computes `self - rhs`, + /// saturating at the numeric bounds instead of overflowing. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".saturating_sub_unsigned(127), -27);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.saturating_sub_unsigned(100), ", stringify!($SelfT), "::MIN);")] + /// ``` + #[stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[rustc_const_stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn saturating_sub_unsigned(self, rhs: $UnsignedT) -> Self { + // Overflow can only happen at the lower bound + // We cannot use `unwrap_or` here because it is not `const` + match self.checked_sub_unsigned(rhs) { + Some(x) => x, + None => Self::MIN, + } + } + + /// Saturating integer negation. Computes `-self`, returning `MAX` if `self == MIN` + /// instead of overflowing. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".saturating_neg(), -100);")] + #[doc = concat!("assert_eq!((-100", stringify!($SelfT), ").saturating_neg(), 100);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.saturating_neg(), ", stringify!($SelfT), "::MAX);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.saturating_neg(), ", stringify!($SelfT), "::MIN + 1);")] + /// ``` + + #[stable(feature = "saturating_neg", since = "1.45.0")] + #[rustc_const_stable(feature = "const_saturating_int_methods", since = "1.47.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn saturating_neg(self) -> Self { + intrinsics::saturating_sub(0, self) + } + + /// Saturating absolute value. Computes `self.abs()`, returning `MAX` if `self == + /// MIN` instead of overflowing. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".saturating_abs(), 100);")] + #[doc = concat!("assert_eq!((-100", stringify!($SelfT), ").saturating_abs(), 100);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.saturating_abs(), ", stringify!($SelfT), "::MAX);")] + #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MIN + 1).saturating_abs(), ", stringify!($SelfT), "::MAX);")] + /// ``` + + #[stable(feature = "saturating_neg", since = "1.45.0")] + #[rustc_const_stable(feature = "const_saturating_int_methods", since = "1.47.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn saturating_abs(self) -> Self { + if self.is_negative() { + self.saturating_neg() + } else { + self + } + } + + /// Saturating integer multiplication. Computes `self * rhs`, saturating at the + /// numeric bounds instead of overflowing. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(10", stringify!($SelfT), ".saturating_mul(12), 120);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.saturating_mul(10), ", stringify!($SelfT), "::MAX);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.saturating_mul(10), ", stringify!($SelfT), "::MIN);")] + /// ``` + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_saturating_int_methods", since = "1.47.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn saturating_mul(self, rhs: Self) -> Self { + match self.checked_mul(rhs) { + Some(x) => x, + None => if (self < 0) == (rhs < 0) { + Self::MAX + } else { + Self::MIN + } + } + } + + /// Saturating integer division. Computes `self / rhs`, saturating at the + /// numeric bounds instead of overflowing. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".saturating_div(2), 2);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.saturating_div(-1), ", stringify!($SelfT), "::MIN + 1);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.saturating_div(-1), ", stringify!($SelfT), "::MAX);")] + /// + /// ``` + /// + /// ```should_panic + #[doc = concat!("let _ = 1", stringify!($SelfT), ".saturating_div(0);")] + /// + /// ``` + #[stable(feature = "saturating_div", since = "1.58.0")] + #[rustc_const_stable(feature = "saturating_div", since = "1.58.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn saturating_div(self, rhs: Self) -> Self { + match self.overflowing_div(rhs) { + (result, false) => result, + (_result, true) => Self::MAX, // MIN / -1 is the only possible saturating overflow + } + } + + /// Saturating integer exponentiation. Computes `self.pow(exp)`, + /// saturating at the numeric bounds instead of overflowing. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!((-4", stringify!($SelfT), ").saturating_pow(3), -64);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.saturating_pow(2), ", stringify!($SelfT), "::MAX);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.saturating_pow(3), ", stringify!($SelfT), "::MIN);")] + /// ``` + #[stable(feature = "no_panic_pow", since = "1.34.0")] + #[rustc_const_stable(feature = "const_int_pow", since = "1.50.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn saturating_pow(self, exp: u32) -> Self { + match self.checked_pow(exp) { + Some(x) => x, + None if self < 0 && exp % 2 == 1 => Self::MIN, + None => Self::MAX, + } + } + + /// Wrapping (modular) addition. Computes `self + rhs`, wrapping around at the + /// boundary of the type. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".wrapping_add(27), 127);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.wrapping_add(2), ", stringify!($SelfT), "::MIN + 1);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn wrapping_add(self, rhs: Self) -> Self { + intrinsics::wrapping_add(self, rhs) + } + + /// Wrapping (modular) addition with an unsigned integer. Computes + /// `self + rhs`, wrapping around at the boundary of the type. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".wrapping_add_unsigned(27), 127);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.wrapping_add_unsigned(2), ", stringify!($SelfT), "::MIN + 1);")] + /// ``` + #[stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[rustc_const_stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn wrapping_add_unsigned(self, rhs: $UnsignedT) -> Self { + self.wrapping_add(rhs as Self) + } + + /// Wrapping (modular) subtraction. Computes `self - rhs`, wrapping around at the + /// boundary of the type. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(0", stringify!($SelfT), ".wrapping_sub(127), -127);")] + #[doc = concat!("assert_eq!((-2", stringify!($SelfT), ").wrapping_sub(", stringify!($SelfT), "::MAX), ", stringify!($SelfT), "::MAX);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn wrapping_sub(self, rhs: Self) -> Self { + intrinsics::wrapping_sub(self, rhs) + } + + /// Wrapping (modular) subtraction with an unsigned integer. Computes + /// `self - rhs`, wrapping around at the boundary of the type. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(0", stringify!($SelfT), ".wrapping_sub_unsigned(127), -127);")] + #[doc = concat!("assert_eq!((-2", stringify!($SelfT), ").wrapping_sub_unsigned(", stringify!($UnsignedT), "::MAX), -1);")] + /// ``` + #[stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[rustc_const_stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn wrapping_sub_unsigned(self, rhs: $UnsignedT) -> Self { + self.wrapping_sub(rhs as Self) + } + + /// Wrapping (modular) multiplication. Computes `self * rhs`, wrapping around at + /// the boundary of the type. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(10", stringify!($SelfT), ".wrapping_mul(12), 120);")] + /// assert_eq!(11i8.wrapping_mul(12), -124); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn wrapping_mul(self, rhs: Self) -> Self { + intrinsics::wrapping_mul(self, rhs) + } + + /// Wrapping (modular) division. Computes `self / rhs`, wrapping around at the + /// boundary of the type. + /// + /// The only case where such wrapping can occur is when one divides `MIN / -1` on a signed type (where + /// `MIN` is the negative minimal value for the type); this is equivalent to `-MIN`, a positive value + /// that is too large to represent in the type. In such a case, this function returns `MIN` itself. + /// + /// # Panics + /// + /// This function will panic if `rhs` is 0. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".wrapping_div(10), 10);")] + /// assert_eq!((-128i8).wrapping_div(-1), -128); + /// ``` + #[stable(feature = "num_wrapping", since = "1.2.0")] + #[rustc_const_stable(feature = "const_wrapping_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn wrapping_div(self, rhs: Self) -> Self { + self.overflowing_div(rhs).0 + } + + /// Wrapping Euclidean division. Computes `self.div_euclid(rhs)`, + /// wrapping around at the boundary of the type. + /// + /// Wrapping will only occur in `MIN / -1` on a signed type (where `MIN` is the negative minimal value + /// for the type). This is equivalent to `-MIN`, a positive value that is too large to represent in the + /// type. In this case, this method returns `MIN` itself. + /// + /// # Panics + /// + /// This function will panic if `rhs` is 0. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".wrapping_div_euclid(10), 10);")] + /// assert_eq!((-128i8).wrapping_div_euclid(-1), -128); + /// ``` + #[stable(feature = "euclidean_division", since = "1.38.0")] + #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn wrapping_div_euclid(self, rhs: Self) -> Self { + self.overflowing_div_euclid(rhs).0 + } + + /// Wrapping (modular) remainder. Computes `self % rhs`, wrapping around at the + /// boundary of the type. + /// + /// Such wrap-around never actually occurs mathematically; implementation artifacts make `x % y` + /// invalid for `MIN / -1` on a signed type (where `MIN` is the negative minimal value). In such a case, + /// this function returns `0`. + /// + /// # Panics + /// + /// This function will panic if `rhs` is 0. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".wrapping_rem(10), 0);")] + /// assert_eq!((-128i8).wrapping_rem(-1), 0); + /// ``` + #[stable(feature = "num_wrapping", since = "1.2.0")] + #[rustc_const_stable(feature = "const_wrapping_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn wrapping_rem(self, rhs: Self) -> Self { + self.overflowing_rem(rhs).0 + } + + /// Wrapping Euclidean remainder. Computes `self.rem_euclid(rhs)`, wrapping around + /// at the boundary of the type. + /// + /// Wrapping will only occur in `MIN % -1` on a signed type (where `MIN` is the negative minimal value + /// for the type). In this case, this method returns 0. + /// + /// # Panics + /// + /// This function will panic if `rhs` is 0. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".wrapping_rem_euclid(10), 0);")] + /// assert_eq!((-128i8).wrapping_rem_euclid(-1), 0); + /// ``` + #[stable(feature = "euclidean_division", since = "1.38.0")] + #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn wrapping_rem_euclid(self, rhs: Self) -> Self { + self.overflowing_rem_euclid(rhs).0 + } + + /// Wrapping (modular) negation. Computes `-self`, wrapping around at the boundary + /// of the type. + /// + /// The only case where such wrapping can occur is when one negates `MIN` on a signed type (where `MIN` + /// is the negative minimal value for the type); this is a positive value that is too large to represent + /// in the type. In such a case, this function returns `MIN` itself. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".wrapping_neg(), -100);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.wrapping_neg(), ", stringify!($SelfT), "::MIN);")] + /// ``` + #[stable(feature = "num_wrapping", since = "1.2.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn wrapping_neg(self) -> Self { + (0 as $SelfT).wrapping_sub(self) + } + + /// Panic-free bitwise shift-left; yields `self << mask(rhs)`, where `mask` removes + /// any high-order bits of `rhs` that would cause the shift to exceed the bitwidth of the type. + /// + /// Note that this is *not* the same as a rotate-left; the RHS of a wrapping shift-left is restricted to + /// the range of the type, rather than the bits shifted out of the LHS being returned to the other end. + /// The primitive integer types all implement a [`rotate_left`](Self::rotate_left) function, + /// which may be what you want instead. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!((-1", stringify!($SelfT), ").wrapping_shl(7), -128);")] + #[doc = concat!("assert_eq!((-1", stringify!($SelfT), ").wrapping_shl(128), -1);")] + /// ``` + #[stable(feature = "num_wrapping", since = "1.2.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + #[rustc_allow_const_fn_unstable(const_inherent_unchecked_arith)] + pub const fn wrapping_shl(self, rhs: u32) -> Self { + // SAFETY: the masking by the bitsize of the type ensures that we do not shift + // out of bounds + unsafe { + self.unchecked_shl(rhs & ($BITS - 1)) + } + } + + /// Panic-free bitwise shift-right; yields `self >> mask(rhs)`, where `mask` + /// removes any high-order bits of `rhs` that would cause the shift to exceed the bitwidth of the type. + /// + /// Note that this is *not* the same as a rotate-right; the RHS of a wrapping shift-right is restricted + /// to the range of the type, rather than the bits shifted out of the LHS being returned to the other + /// end. The primitive integer types all implement a [`rotate_right`](Self::rotate_right) function, + /// which may be what you want instead. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!((-128", stringify!($SelfT), ").wrapping_shr(7), -1);")] + /// assert_eq!((-128i16).wrapping_shr(64), -128); + /// ``` + #[stable(feature = "num_wrapping", since = "1.2.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + #[rustc_allow_const_fn_unstable(const_inherent_unchecked_arith)] + pub const fn wrapping_shr(self, rhs: u32) -> Self { + // SAFETY: the masking by the bitsize of the type ensures that we do not shift + // out of bounds + unsafe { + self.unchecked_shr(rhs & ($BITS - 1)) + } + } + + /// Wrapping (modular) absolute value. Computes `self.abs()`, wrapping around at + /// the boundary of the type. + /// + /// The only case where such wrapping can occur is when one takes the absolute value of the negative + /// minimal value for the type; this is a positive value that is too large to represent in the type. In + /// such a case, this function returns `MIN` itself. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".wrapping_abs(), 100);")] + #[doc = concat!("assert_eq!((-100", stringify!($SelfT), ").wrapping_abs(), 100);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.wrapping_abs(), ", stringify!($SelfT), "::MIN);")] + /// assert_eq!((-128i8).wrapping_abs() as u8, 128); + /// ``` + #[stable(feature = "no_panic_abs", since = "1.13.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[allow(unused_attributes)] + #[inline] + pub const fn wrapping_abs(self) -> Self { + if self.is_negative() { + self.wrapping_neg() + } else { + self + } + } + + /// Computes the absolute value of `self` without any wrapping + /// or panicking. + /// + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".unsigned_abs(), 100", stringify!($UnsignedT), ");")] + #[doc = concat!("assert_eq!((-100", stringify!($SelfT), ").unsigned_abs(), 100", stringify!($UnsignedT), ");")] + /// assert_eq!((-128i8).unsigned_abs(), 128u8); + /// ``` + #[stable(feature = "unsigned_abs", since = "1.51.0")] + #[rustc_const_stable(feature = "unsigned_abs", since = "1.51.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn unsigned_abs(self) -> $UnsignedT { + self.wrapping_abs() as $UnsignedT + } + + /// Wrapping (modular) exponentiation. Computes `self.pow(exp)`, + /// wrapping around at the boundary of the type. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(3", stringify!($SelfT), ".wrapping_pow(4), 81);")] + /// assert_eq!(3i8.wrapping_pow(5), -13); + /// assert_eq!(3i8.wrapping_pow(6), -39); + /// ``` + #[stable(feature = "no_panic_pow", since = "1.34.0")] + #[rustc_const_stable(feature = "const_int_pow", since = "1.50.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn wrapping_pow(self, mut exp: u32) -> Self { + if exp == 0 { + return 1; + } + let mut base = self; + let mut acc: Self = 1; + + while exp > 1 { + if (exp & 1) == 1 { + acc = acc.wrapping_mul(base); + } + exp /= 2; + base = base.wrapping_mul(base); + } + + // since exp!=0, finally the exp must be 1. + // Deal with the final bit of the exponent separately, since + // squaring the base afterwards is not necessary and may cause a + // needless overflow. + acc.wrapping_mul(base) + } + + /// Calculates `self` + `rhs` + /// + /// Returns a tuple of the addition along with a boolean indicating whether an arithmetic overflow would + /// occur. If an overflow would have occurred then the wrapped value is returned. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".overflowing_add(2), (7, false));")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.overflowing_add(1), (", stringify!($SelfT), "::MIN, true));")] + /// ``` + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn overflowing_add(self, rhs: Self) -> (Self, bool) { + let (a, b) = intrinsics::add_with_overflow(self as $ActualT, rhs as $ActualT); + (a as Self, b) + } + + /// Calculates `self` + `rhs` + `carry` and checks for overflow. + /// + /// Performs "ternary addition" of two integer operands and a carry-in + /// bit, and returns a tuple of the sum along with a boolean indicating + /// whether an arithmetic overflow would occur. On overflow, the wrapped + /// value is returned. + /// + /// This allows chaining together multiple additions to create a wider + /// addition, and can be useful for bignum addition. This method should + /// only be used for the most significant word; for the less significant + /// words the unsigned method + #[doc = concat!("[`", stringify!($UnsignedT), "::carrying_add`]")] + /// should be used. + /// + /// The output boolean returned by this method is *not* a carry flag, + /// and should *not* be added to a more significant word. + /// + /// If the input carry is false, this method is equivalent to + /// [`overflowing_add`](Self::overflowing_add). + /// + /// # Examples + /// + /// ``` + /// #![feature(bigint_helper_methods)] + /// // Only the most significant word is signed. + /// // + #[doc = concat!("// 10 MAX (a = 10 × 2^", stringify!($BITS), " + 2^", stringify!($BITS), " - 1)")] + #[doc = concat!("// + -5 9 (b = -5 × 2^", stringify!($BITS), " + 9)")] + /// // --------- + #[doc = concat!("// 6 8 (sum = 6 × 2^", stringify!($BITS), " + 8)")] + /// + #[doc = concat!("let (a1, a0): (", stringify!($SelfT), ", ", stringify!($UnsignedT), ") = (10, ", stringify!($UnsignedT), "::MAX);")] + #[doc = concat!("let (b1, b0): (", stringify!($SelfT), ", ", stringify!($UnsignedT), ") = (-5, 9);")] + /// let carry0 = false; + /// + #[doc = concat!("// ", stringify!($UnsignedT), "::carrying_add for the less significant words")] + /// let (sum0, carry1) = a0.carrying_add(b0, carry0); + /// assert_eq!(carry1, true); + /// + #[doc = concat!("// ", stringify!($SelfT), "::carrying_add for the most significant word")] + /// let (sum1, overflow) = a1.carrying_add(b1, carry1); + /// assert_eq!(overflow, false); + /// + /// assert_eq!((sum1, sum0), (6, 8)); + /// ``` + #[unstable(feature = "bigint_helper_methods", issue = "85532")] + #[rustc_const_unstable(feature = "const_bigint_helper_methods", issue = "85532")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn carrying_add(self, rhs: Self, carry: bool) -> (Self, bool) { + // note: longer-term this should be done via an intrinsic. + // note: no intermediate overflow is required (https://github.com/rust-lang/rust/issues/85532#issuecomment-1032214946). + let (a, b) = self.overflowing_add(rhs); + let (c, d) = a.overflowing_add(carry as $SelfT); + (c, b != d) + } + + /// Calculates `self` + `rhs` with an unsigned `rhs` + /// + /// Returns a tuple of the addition along with a boolean indicating + /// whether an arithmetic overflow would occur. If an overflow would + /// have occurred then the wrapped value is returned. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(1", stringify!($SelfT), ".overflowing_add_unsigned(2), (3, false));")] + #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MIN).overflowing_add_unsigned(", stringify!($UnsignedT), "::MAX), (", stringify!($SelfT), "::MAX, false));")] + #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MAX - 2).overflowing_add_unsigned(3), (", stringify!($SelfT), "::MIN, true));")] + /// ``` + #[stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[rustc_const_stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn overflowing_add_unsigned(self, rhs: $UnsignedT) -> (Self, bool) { + let rhs = rhs as Self; + let (res, overflowed) = self.overflowing_add(rhs); + (res, overflowed ^ (rhs < 0)) + } + + /// Calculates `self` - `rhs` + /// + /// Returns a tuple of the subtraction along with a boolean indicating whether an arithmetic overflow + /// would occur. If an overflow would have occurred then the wrapped value is returned. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".overflowing_sub(2), (3, false));")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.overflowing_sub(1), (", stringify!($SelfT), "::MAX, true));")] + /// ``` + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn overflowing_sub(self, rhs: Self) -> (Self, bool) { + let (a, b) = intrinsics::sub_with_overflow(self as $ActualT, rhs as $ActualT); + (a as Self, b) + } + + /// Calculates `self` − `rhs` − `borrow` and checks for + /// overflow. + /// + /// Performs "ternary subtraction" by subtracting both an integer + /// operand and a borrow-in bit from `self`, and returns a tuple of the + /// difference along with a boolean indicating whether an arithmetic + /// overflow would occur. On overflow, the wrapped value is returned. + /// + /// This allows chaining together multiple subtractions to create a + /// wider subtraction, and can be useful for bignum subtraction. This + /// method should only be used for the most significant word; for the + /// less significant words the unsigned method + #[doc = concat!("[`", stringify!($UnsignedT), "::borrowing_sub`]")] + /// should be used. + /// + /// The output boolean returned by this method is *not* a borrow flag, + /// and should *not* be subtracted from a more significant word. + /// + /// If the input borrow is false, this method is equivalent to + /// [`overflowing_sub`](Self::overflowing_sub). + /// + /// # Examples + /// + /// ``` + /// #![feature(bigint_helper_methods)] + /// // Only the most significant word is signed. + /// // + #[doc = concat!("// 6 8 (a = 6 × 2^", stringify!($BITS), " + 8)")] + #[doc = concat!("// - -5 9 (b = -5 × 2^", stringify!($BITS), " + 9)")] + /// // --------- + #[doc = concat!("// 10 MAX (diff = 10 × 2^", stringify!($BITS), " + 2^", stringify!($BITS), " - 1)")] + /// + #[doc = concat!("let (a1, a0): (", stringify!($SelfT), ", ", stringify!($UnsignedT), ") = (6, 8);")] + #[doc = concat!("let (b1, b0): (", stringify!($SelfT), ", ", stringify!($UnsignedT), ") = (-5, 9);")] + /// let borrow0 = false; + /// + #[doc = concat!("// ", stringify!($UnsignedT), "::borrowing_sub for the less significant words")] + /// let (diff0, borrow1) = a0.borrowing_sub(b0, borrow0); + /// assert_eq!(borrow1, true); + /// + #[doc = concat!("// ", stringify!($SelfT), "::borrowing_sub for the most significant word")] + /// let (diff1, overflow) = a1.borrowing_sub(b1, borrow1); + /// assert_eq!(overflow, false); + /// + #[doc = concat!("assert_eq!((diff1, diff0), (10, ", stringify!($UnsignedT), "::MAX));")] + /// ``` + #[unstable(feature = "bigint_helper_methods", issue = "85532")] + #[rustc_const_unstable(feature = "const_bigint_helper_methods", issue = "85532")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn borrowing_sub(self, rhs: Self, borrow: bool) -> (Self, bool) { + // note: longer-term this should be done via an intrinsic. + // note: no intermediate overflow is required (https://github.com/rust-lang/rust/issues/85532#issuecomment-1032214946). + let (a, b) = self.overflowing_sub(rhs); + let (c, d) = a.overflowing_sub(borrow as $SelfT); + (c, b != d) + } + + /// Calculates `self` - `rhs` with an unsigned `rhs` + /// + /// Returns a tuple of the subtraction along with a boolean indicating + /// whether an arithmetic overflow would occur. If an overflow would + /// have occurred then the wrapped value is returned. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(1", stringify!($SelfT), ".overflowing_sub_unsigned(2), (-1, false));")] + #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MAX).overflowing_sub_unsigned(", stringify!($UnsignedT), "::MAX), (", stringify!($SelfT), "::MIN, false));")] + #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MIN + 2).overflowing_sub_unsigned(3), (", stringify!($SelfT), "::MAX, true));")] + /// ``` + #[stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[rustc_const_stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn overflowing_sub_unsigned(self, rhs: $UnsignedT) -> (Self, bool) { + let rhs = rhs as Self; + let (res, overflowed) = self.overflowing_sub(rhs); + (res, overflowed ^ (rhs < 0)) + } + + /// Calculates the multiplication of `self` and `rhs`. + /// + /// Returns a tuple of the multiplication along with a boolean indicating whether an arithmetic overflow + /// would occur. If an overflow would have occurred then the wrapped value is returned. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".overflowing_mul(2), (10, false));")] + /// assert_eq!(1_000_000_000i32.overflowing_mul(10), (1410065408, true)); + /// ``` + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn overflowing_mul(self, rhs: Self) -> (Self, bool) { + let (a, b) = intrinsics::mul_with_overflow(self as $ActualT, rhs as $ActualT); + (a as Self, b) + } + + /// Calculates the divisor when `self` is divided by `rhs`. + /// + /// Returns a tuple of the divisor along with a boolean indicating whether an arithmetic overflow would + /// occur. If an overflow would occur then self is returned. + /// + /// # Panics + /// + /// This function will panic if `rhs` is 0. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".overflowing_div(2), (2, false));")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.overflowing_div(-1), (", stringify!($SelfT), "::MIN, true));")] + /// ``` + #[inline] + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_overflowing_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + pub const fn overflowing_div(self, rhs: Self) -> (Self, bool) { + // Using `&` helps LLVM see that it is the same check made in division. + if unlikely!((self == Self::MIN) & (rhs == -1)) { + (self, true) + } else { + (self / rhs, false) + } + } + + /// Calculates the quotient of Euclidean division `self.div_euclid(rhs)`. + /// + /// Returns a tuple of the divisor along with a boolean indicating whether an arithmetic overflow would + /// occur. If an overflow would occur then `self` is returned. + /// + /// # Panics + /// + /// This function will panic if `rhs` is 0. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".overflowing_div_euclid(2), (2, false));")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.overflowing_div_euclid(-1), (", stringify!($SelfT), "::MIN, true));")] + /// ``` + #[inline] + #[stable(feature = "euclidean_division", since = "1.38.0")] + #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + pub const fn overflowing_div_euclid(self, rhs: Self) -> (Self, bool) { + // Using `&` helps LLVM see that it is the same check made in division. + if unlikely!((self == Self::MIN) & (rhs == -1)) { + (self, true) + } else { + (self.div_euclid(rhs), false) + } + } + + /// Calculates the remainder when `self` is divided by `rhs`. + /// + /// Returns a tuple of the remainder after dividing along with a boolean indicating whether an + /// arithmetic overflow would occur. If an overflow would occur then 0 is returned. + /// + /// # Panics + /// + /// This function will panic if `rhs` is 0. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".overflowing_rem(2), (1, false));")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.overflowing_rem(-1), (0, true));")] + /// ``` + #[inline] + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_overflowing_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + pub const fn overflowing_rem(self, rhs: Self) -> (Self, bool) { + if unlikely!(rhs == -1) { + (0, self == Self::MIN) + } else { + (self % rhs, false) + } + } + + + /// Overflowing Euclidean remainder. Calculates `self.rem_euclid(rhs)`. + /// + /// Returns a tuple of the remainder after dividing along with a boolean indicating whether an + /// arithmetic overflow would occur. If an overflow would occur then 0 is returned. + /// + /// # Panics + /// + /// This function will panic if `rhs` is 0. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".overflowing_rem_euclid(2), (1, false));")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.overflowing_rem_euclid(-1), (0, true));")] + /// ``` + #[stable(feature = "euclidean_division", since = "1.38.0")] + #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn overflowing_rem_euclid(self, rhs: Self) -> (Self, bool) { + if unlikely!(rhs == -1) { + (0, self == Self::MIN) + } else { + (self.rem_euclid(rhs), false) + } + } + + + /// Negates self, overflowing if this is equal to the minimum value. + /// + /// Returns a tuple of the negated version of self along with a boolean indicating whether an overflow + /// happened. If `self` is the minimum value (e.g., `i32::MIN` for values of type `i32`), then the + /// minimum value will be returned again and `true` will be returned for an overflow happening. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(2", stringify!($SelfT), ".overflowing_neg(), (-2, false));")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.overflowing_neg(), (", stringify!($SelfT), "::MIN, true));")] + /// ``` + #[inline] + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[allow(unused_attributes)] + pub const fn overflowing_neg(self) -> (Self, bool) { + if unlikely!(self == Self::MIN) { + (Self::MIN, true) + } else { + (-self, false) + } + } + + /// Shifts self left by `rhs` bits. + /// + /// Returns a tuple of the shifted version of self along with a boolean indicating whether the shift + /// value was larger than or equal to the number of bits. If the shift value is too large, then value is + /// masked (N-1) where N is the number of bits, and this value is then used to perform the shift. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(0x1", stringify!($SelfT),".overflowing_shl(4), (0x10, false));")] + /// assert_eq!(0x1i32.overflowing_shl(36), (0x10, true)); + /// ``` + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn overflowing_shl(self, rhs: u32) -> (Self, bool) { + (self.wrapping_shl(rhs), (rhs > ($BITS - 1))) + } + + /// Shifts self right by `rhs` bits. + /// + /// Returns a tuple of the shifted version of self along with a boolean indicating whether the shift + /// value was larger than or equal to the number of bits. If the shift value is too large, then value is + /// masked (N-1) where N is the number of bits, and this value is then used to perform the shift. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(0x10", stringify!($SelfT), ".overflowing_shr(4), (0x1, false));")] + /// assert_eq!(0x10i32.overflowing_shr(36), (0x1, true)); + /// ``` + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn overflowing_shr(self, rhs: u32) -> (Self, bool) { + (self.wrapping_shr(rhs), (rhs > ($BITS - 1))) + } + + /// Computes the absolute value of `self`. + /// + /// Returns a tuple of the absolute version of self along with a boolean indicating whether an overflow + /// happened. If self is the minimum value + #[doc = concat!("(e.g., ", stringify!($SelfT), "::MIN for values of type ", stringify!($SelfT), "),")] + /// then the minimum value will be returned again and true will be returned + /// for an overflow happening. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(10", stringify!($SelfT), ".overflowing_abs(), (10, false));")] + #[doc = concat!("assert_eq!((-10", stringify!($SelfT), ").overflowing_abs(), (10, false));")] + #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MIN).overflowing_abs(), (", stringify!($SelfT), "::MIN, true));")] + /// ``` + #[stable(feature = "no_panic_abs", since = "1.13.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn overflowing_abs(self) -> (Self, bool) { + (self.wrapping_abs(), self == Self::MIN) + } + + /// Raises self to the power of `exp`, using exponentiation by squaring. + /// + /// Returns a tuple of the exponentiation along with a bool indicating + /// whether an overflow happened. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(3", stringify!($SelfT), ".overflowing_pow(4), (81, false));")] + /// assert_eq!(3i8.overflowing_pow(5), (-13, true)); + /// ``` + #[stable(feature = "no_panic_pow", since = "1.34.0")] + #[rustc_const_stable(feature = "const_int_pow", since = "1.50.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn overflowing_pow(self, mut exp: u32) -> (Self, bool) { + if exp == 0 { + return (1,false); + } + let mut base = self; + let mut acc: Self = 1; + let mut overflown = false; + // Scratch space for storing results of overflowing_mul. + let mut r; + + while exp > 1 { + if (exp & 1) == 1 { + r = acc.overflowing_mul(base); + acc = r.0; + overflown |= r.1; + } + exp /= 2; + r = base.overflowing_mul(base); + base = r.0; + overflown |= r.1; + } + + // since exp!=0, finally the exp must be 1. + // Deal with the final bit of the exponent separately, since + // squaring the base afterwards is not necessary and may cause a + // needless overflow. + r = acc.overflowing_mul(base); + r.1 |= overflown; + r + } + + /// Raises self to the power of `exp`, using exponentiation by squaring. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let x: ", stringify!($SelfT), " = 2; // or any other integer type")] + /// + /// assert_eq!(x.pow(5), 32); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_pow", since = "1.50.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + #[rustc_inherit_overflow_checks] + pub const fn pow(self, mut exp: u32) -> Self { + if exp == 0 { + return 1; + } + let mut base = self; + let mut acc = 1; + + while exp > 1 { + if (exp & 1) == 1 { + acc = acc * base; + } + exp /= 2; + base = base * base; + } + + // since exp!=0, finally the exp must be 1. + // Deal with the final bit of the exponent separately, since + // squaring the base afterwards is not necessary and may cause a + // needless overflow. + acc * base + } + + /// Calculates the quotient of Euclidean division of `self` by `rhs`. + /// + /// This computes the integer `q` such that `self = q * rhs + r`, with + /// `r = self.rem_euclid(rhs)` and `0 <= r < abs(rhs)`. + /// + /// In other words, the result is `self / rhs` rounded to the integer `q` + /// such that `self >= q * rhs`. + /// If `self > 0`, this is equal to round towards zero (the default in Rust); + /// if `self < 0`, this is equal to round towards +/- infinity. + /// + /// # Panics + /// + /// This function will panic if `rhs` is 0 or the division results in overflow. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let a: ", stringify!($SelfT), " = 7; // or any other integer type")] + /// let b = 4; + /// + /// assert_eq!(a.div_euclid(b), 1); // 7 >= 4 * 1 + /// assert_eq!(a.div_euclid(-b), -1); // 7 >= -4 * -1 + /// assert_eq!((-a).div_euclid(b), -2); // -7 >= 4 * -2 + /// assert_eq!((-a).div_euclid(-b), 2); // -7 >= -4 * 2 + /// ``` + #[stable(feature = "euclidean_division", since = "1.38.0")] + #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + #[rustc_inherit_overflow_checks] + pub const fn div_euclid(self, rhs: Self) -> Self { + let q = self / rhs; + if self % rhs < 0 { + return if rhs > 0 { q - 1 } else { q + 1 } + } + q + } + + + /// Calculates the least nonnegative remainder of `self (mod rhs)`. + /// + /// This is done as if by the Euclidean division algorithm -- given + /// `r = self.rem_euclid(rhs)`, `self = rhs * self.div_euclid(rhs) + r`, and + /// `0 <= r < abs(rhs)`. + /// + /// # Panics + /// + /// This function will panic if `rhs` is 0 or the division results in overflow. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let a: ", stringify!($SelfT), " = 7; // or any other integer type")] + /// let b = 4; + /// + /// assert_eq!(a.rem_euclid(b), 3); + /// assert_eq!((-a).rem_euclid(b), 1); + /// assert_eq!(a.rem_euclid(-b), 3); + /// assert_eq!((-a).rem_euclid(-b), 1); + /// ``` + #[stable(feature = "euclidean_division", since = "1.38.0")] + #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + #[rustc_inherit_overflow_checks] + pub const fn rem_euclid(self, rhs: Self) -> Self { + let r = self % rhs; + if r < 0 { + // Semantically equivalent to `if rhs < 0 { r - rhs } else { r + rhs }`. + // If `rhs` is not `Self::MIN`, then `r + abs(rhs)` will not overflow + // and is clearly equivalent, because `r` is negative. + // Otherwise, `rhs` is `Self::MIN`, then we have + // `r.wrapping_add(Self::MIN.wrapping_abs())`, which evaluates + // to `r.wrapping_add(Self::MIN)`, which is equivalent to + // `r - Self::MIN`, which is what we wanted (and will not overflow + // for negative `r`). + r.wrapping_add(rhs.wrapping_abs()) + } else { + r + } + } + + /// Calculates the quotient of `self` and `rhs`, rounding the result towards negative infinity. + /// + /// # Panics + /// + /// This function will panic if `rhs` is zero. + /// + /// ## Overflow behavior + /// + /// On overflow, this function will panic if overflow checks are enabled (default in debug + /// mode) and wrap if overflow checks are disabled (default in release mode). + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(int_roundings)] + #[doc = concat!("let a: ", stringify!($SelfT)," = 8;")] + /// let b = 3; + /// + /// assert_eq!(a.div_floor(b), 2); + /// assert_eq!(a.div_floor(-b), -3); + /// assert_eq!((-a).div_floor(b), -3); + /// assert_eq!((-a).div_floor(-b), 2); + /// ``` + #[unstable(feature = "int_roundings", issue = "88581")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + #[rustc_inherit_overflow_checks] + pub const fn div_floor(self, rhs: Self) -> Self { + let d = self / rhs; + let r = self % rhs; + if (r > 0 && rhs < 0) || (r < 0 && rhs > 0) { + d - 1 + } else { + d + } + } + + /// Calculates the quotient of `self` and `rhs`, rounding the result towards positive infinity. + /// + /// # Panics + /// + /// This function will panic if `rhs` is zero. + /// + /// ## Overflow behavior + /// + /// On overflow, this function will panic if overflow checks are enabled (default in debug + /// mode) and wrap if overflow checks are disabled (default in release mode). + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(int_roundings)] + #[doc = concat!("let a: ", stringify!($SelfT)," = 8;")] + /// let b = 3; + /// + /// assert_eq!(a.div_ceil(b), 3); + /// assert_eq!(a.div_ceil(-b), -2); + /// assert_eq!((-a).div_ceil(b), -2); + /// assert_eq!((-a).div_ceil(-b), 3); + /// ``` + #[unstable(feature = "int_roundings", issue = "88581")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + #[rustc_inherit_overflow_checks] + pub const fn div_ceil(self, rhs: Self) -> Self { + let d = self / rhs; + let r = self % rhs; + if (r > 0 && rhs > 0) || (r < 0 && rhs < 0) { + d + 1 + } else { + d + } + } + + /// If `rhs` is positive, calculates the smallest value greater than or + /// equal to `self` that is a multiple of `rhs`. If `rhs` is negative, + /// calculates the largest value less than or equal to `self` that is a + /// multiple of `rhs`. + /// + /// # Panics + /// + /// This function will panic if `rhs` is zero. + /// + /// ## Overflow behavior + /// + /// On overflow, this function will panic if overflow checks are enabled (default in debug + /// mode) and wrap if overflow checks are disabled (default in release mode). + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(int_roundings)] + #[doc = concat!("assert_eq!(16_", stringify!($SelfT), ".next_multiple_of(8), 16);")] + #[doc = concat!("assert_eq!(23_", stringify!($SelfT), ".next_multiple_of(8), 24);")] + #[doc = concat!("assert_eq!(16_", stringify!($SelfT), ".next_multiple_of(-8), 16);")] + #[doc = concat!("assert_eq!(23_", stringify!($SelfT), ".next_multiple_of(-8), 16);")] + #[doc = concat!("assert_eq!((-16_", stringify!($SelfT), ").next_multiple_of(8), -16);")] + #[doc = concat!("assert_eq!((-23_", stringify!($SelfT), ").next_multiple_of(8), -16);")] + #[doc = concat!("assert_eq!((-16_", stringify!($SelfT), ").next_multiple_of(-8), -16);")] + #[doc = concat!("assert_eq!((-23_", stringify!($SelfT), ").next_multiple_of(-8), -24);")] + /// ``` + #[unstable(feature = "int_roundings", issue = "88581")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + #[rustc_inherit_overflow_checks] + pub const fn next_multiple_of(self, rhs: Self) -> Self { + // This would otherwise fail when calculating `r` when self == T::MIN. + if rhs == -1 { + return self; + } + + let r = self % rhs; + let m = if (r > 0 && rhs < 0) || (r < 0 && rhs > 0) { + r + rhs + } else { + r + }; + + if m == 0 { + self + } else { + self + (rhs - m) + } + } + + /// If `rhs` is positive, calculates the smallest value greater than or + /// equal to `self` that is a multiple of `rhs`. If `rhs` is negative, + /// calculates the largest value less than or equal to `self` that is a + /// multiple of `rhs`. Returns `None` if `rhs` is zero or the operation + /// would result in overflow. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(int_roundings)] + #[doc = concat!("assert_eq!(16_", stringify!($SelfT), ".checked_next_multiple_of(8), Some(16));")] + #[doc = concat!("assert_eq!(23_", stringify!($SelfT), ".checked_next_multiple_of(8), Some(24));")] + #[doc = concat!("assert_eq!(16_", stringify!($SelfT), ".checked_next_multiple_of(-8), Some(16));")] + #[doc = concat!("assert_eq!(23_", stringify!($SelfT), ".checked_next_multiple_of(-8), Some(16));")] + #[doc = concat!("assert_eq!((-16_", stringify!($SelfT), ").checked_next_multiple_of(8), Some(-16));")] + #[doc = concat!("assert_eq!((-23_", stringify!($SelfT), ").checked_next_multiple_of(8), Some(-16));")] + #[doc = concat!("assert_eq!((-16_", stringify!($SelfT), ").checked_next_multiple_of(-8), Some(-16));")] + #[doc = concat!("assert_eq!((-23_", stringify!($SelfT), ").checked_next_multiple_of(-8), Some(-24));")] + #[doc = concat!("assert_eq!(1_", stringify!($SelfT), ".checked_next_multiple_of(0), None);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.checked_next_multiple_of(2), None);")] + /// ``` + #[unstable(feature = "int_roundings", issue = "88581")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_next_multiple_of(self, rhs: Self) -> Option { + // This would otherwise fail when calculating `r` when self == T::MIN. + if rhs == -1 { + return Some(self); + } + + let r = try_opt!(self.checked_rem(rhs)); + let m = if (r > 0 && rhs < 0) || (r < 0 && rhs > 0) { + // r + rhs cannot overflow because they have opposite signs + r + rhs + } else { + r + }; + + if m == 0 { + Some(self) + } else { + // rhs - m cannot overflow because m has the same sign as rhs + self.checked_add(rhs - m) + } + } + + /// Returns the logarithm of the number with respect to an arbitrary base, + /// rounded down. + /// + /// This method might not be optimized owing to implementation details; + /// `ilog2` can produce results more efficiently for base 2, and `ilog10` + /// can produce results more efficiently for base 10. + /// + /// # Panics + /// + /// This function will panic if `self` is less than or equal to zero, + /// or if `base` is less than 2. + /// + /// # Examples + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".ilog(5), 1);")] + /// ``` + #[stable(feature = "int_log", since = "1.67.0")] + #[rustc_const_stable(feature = "int_log", since = "1.67.0")] + #[rustc_allow_const_fn_unstable(const_option)] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + #[track_caller] + pub const fn ilog(self, base: Self) -> u32 { + assert!(base >= 2, "base of integer logarithm must be at least 2"); + self.checked_ilog(base).expect("argument of integer logarithm must be positive") + } + + /// Returns the base 2 logarithm of the number, rounded down. + /// + /// # Panics + /// + /// This function will panic if `self` is less than or equal to zero. + /// + /// # Examples + /// + /// ``` + #[doc = concat!("assert_eq!(2", stringify!($SelfT), ".ilog2(), 1);")] + /// ``` + #[stable(feature = "int_log", since = "1.67.0")] + #[rustc_const_stable(feature = "int_log", since = "1.67.0")] + #[rustc_allow_const_fn_unstable(const_option)] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + #[track_caller] + pub const fn ilog2(self) -> u32 { + self.checked_ilog2().expect("argument of integer logarithm must be positive") + } + + /// Returns the base 10 logarithm of the number, rounded down. + /// + /// # Panics + /// + /// This function will panic if `self` is less than or equal to zero. + /// + /// # Example + /// + /// ``` + #[doc = concat!("assert_eq!(10", stringify!($SelfT), ".ilog10(), 1);")] + /// ``` + #[stable(feature = "int_log", since = "1.67.0")] + #[rustc_const_stable(feature = "int_log", since = "1.67.0")] + #[rustc_allow_const_fn_unstable(const_option)] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + #[track_caller] + pub const fn ilog10(self) -> u32 { + self.checked_ilog10().expect("argument of integer logarithm must be positive") + } + + /// Returns the logarithm of the number with respect to an arbitrary base, + /// rounded down. + /// + /// Returns `None` if the number is negative or zero, or if the base is not at least 2. + /// + /// This method might not be optimized owing to implementation details; + /// `checked_ilog2` can produce results more efficiently for base 2, and + /// `checked_ilog10` can produce results more efficiently for base 10. + /// + /// # Examples + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".checked_ilog(5), Some(1));")] + /// ``` + #[stable(feature = "int_log", since = "1.67.0")] + #[rustc_const_stable(feature = "int_log", since = "1.67.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_ilog(self, base: Self) -> Option { + if self <= 0 || base <= 1 { + None + } else { + let mut n = 0; + let mut r = self; + + // Optimization for 128 bit wide integers. + if Self::BITS == 128 { + let b = Self::ilog2(self) / (Self::ilog2(base) + 1); + n += b; + r /= base.pow(b as u32); + } + + while r >= base { + r /= base; + n += 1; + } + Some(n) + } + } + + /// Returns the base 2 logarithm of the number, rounded down. + /// + /// Returns `None` if the number is negative or zero. + /// + /// # Examples + /// + /// ``` + #[doc = concat!("assert_eq!(2", stringify!($SelfT), ".checked_ilog2(), Some(1));")] + /// ``` + #[stable(feature = "int_log", since = "1.67.0")] + #[rustc_const_stable(feature = "int_log", since = "1.67.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_ilog2(self) -> Option { + if self <= 0 { + None + } else { + // SAFETY: We just checked that this number is positive + let log = (Self::BITS - 1) - unsafe { intrinsics::ctlz_nonzero(self) as u32 }; + Some(log) + } + } + + /// Returns the base 10 logarithm of the number, rounded down. + /// + /// Returns `None` if the number is negative or zero. + /// + /// # Example + /// + /// ``` + #[doc = concat!("assert_eq!(10", stringify!($SelfT), ".checked_ilog10(), Some(1));")] + /// ``` + #[stable(feature = "int_log", since = "1.67.0")] + #[rustc_const_stable(feature = "int_log", since = "1.67.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_ilog10(self) -> Option { + if self > 0 { + Some(int_log10::$ActualT(self as $ActualT)) + } else { + None + } + } + + /// Computes the absolute value of `self`. + /// + /// # Overflow behavior + /// + /// The absolute value of + #[doc = concat!("`", stringify!($SelfT), "::MIN`")] + /// cannot be represented as an + #[doc = concat!("`", stringify!($SelfT), "`,")] + /// and attempting to calculate it will cause an overflow. This means + /// that code in debug mode will trigger a panic on this case and + /// optimized code will return + #[doc = concat!("`", stringify!($SelfT), "::MIN`")] + /// without a panic. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(10", stringify!($SelfT), ".abs(), 10);")] + #[doc = concat!("assert_eq!((-10", stringify!($SelfT), ").abs(), 10);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[allow(unused_attributes)] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + #[rustc_inherit_overflow_checks] + pub const fn abs(self) -> Self { + // Note that the #[rustc_inherit_overflow_checks] and #[inline] + // above mean that the overflow semantics of the subtraction + // depend on the crate we're being called from. + if self.is_negative() { + -self + } else { + self + } + } + + /// Computes the absolute difference between `self` and `other`. + /// + /// This function always returns the correct answer without overflow or + /// panics by returning an unsigned integer. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".abs_diff(80), 20", stringify!($UnsignedT), ");")] + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".abs_diff(110), 10", stringify!($UnsignedT), ");")] + #[doc = concat!("assert_eq!((-100", stringify!($SelfT), ").abs_diff(80), 180", stringify!($UnsignedT), ");")] + #[doc = concat!("assert_eq!((-100", stringify!($SelfT), ").abs_diff(-120), 20", stringify!($UnsignedT), ");")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.abs_diff(", stringify!($SelfT), "::MAX), ", stringify!($UnsignedT), "::MAX);")] + /// ``` + #[stable(feature = "int_abs_diff", since = "1.60.0")] + #[rustc_const_stable(feature = "int_abs_diff", since = "1.60.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn abs_diff(self, other: Self) -> $UnsignedT { + if self < other { + // Converting a non-negative x from signed to unsigned by using + // `x as U` is left unchanged, but a negative x is converted + // to value x + 2^N. Thus if `s` and `o` are binary variables + // respectively indicating whether `self` and `other` are + // negative, we are computing the mathematical value: + // + // (other + o*2^N) - (self + s*2^N) mod 2^N + // other - self + (o-s)*2^N mod 2^N + // other - self mod 2^N + // + // Finally, taking the mod 2^N of the mathematical value of + // `other - self` does not change it as it already is + // in the range [0, 2^N). + (other as $UnsignedT).wrapping_sub(self as $UnsignedT) + } else { + (self as $UnsignedT).wrapping_sub(other as $UnsignedT) + } + } + + /// Returns a number representing sign of `self`. + /// + /// - `0` if the number is zero + /// - `1` if the number is positive + /// - `-1` if the number is negative + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(10", stringify!($SelfT), ".signum(), 1);")] + #[doc = concat!("assert_eq!(0", stringify!($SelfT), ".signum(), 0);")] + #[doc = concat!("assert_eq!((-10", stringify!($SelfT), ").signum(), -1);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_sign", since = "1.47.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn signum(self) -> Self { + match self { + n if n > 0 => 1, + 0 => 0, + _ => -1, + } + } + + /// Returns `true` if `self` is positive and `false` if the number is zero or + /// negative. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert!(10", stringify!($SelfT), ".is_positive());")] + #[doc = concat!("assert!(!(-10", stringify!($SelfT), ").is_positive());")] + /// ``` + #[must_use] + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[inline(always)] + pub const fn is_positive(self) -> bool { self > 0 } + + /// Returns `true` if `self` is negative and `false` if the number is zero or + /// positive. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert!((-10", stringify!($SelfT), ").is_negative());")] + #[doc = concat!("assert!(!10", stringify!($SelfT), ".is_negative());")] + /// ``` + #[must_use] + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] + #[inline(always)] + pub const fn is_negative(self) -> bool { self < 0 } + + /// Return the memory representation of this integer as a byte array in + /// big-endian (network) byte order. + /// + #[doc = $to_xe_bytes_doc] + /// + /// # Examples + /// + /// ``` + #[doc = concat!("let bytes = ", $swap_op, stringify!($SelfT), ".to_be_bytes();")] + #[doc = concat!("assert_eq!(bytes, ", $be_bytes, ");")] + /// ``` + #[stable(feature = "int_to_from_bytes", since = "1.32.0")] + #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn to_be_bytes(self) -> [u8; mem::size_of::()] { + self.to_be().to_ne_bytes() + } + + /// Return the memory representation of this integer as a byte array in + /// little-endian byte order. + /// + #[doc = $to_xe_bytes_doc] + /// + /// # Examples + /// + /// ``` + #[doc = concat!("let bytes = ", $swap_op, stringify!($SelfT), ".to_le_bytes();")] + #[doc = concat!("assert_eq!(bytes, ", $le_bytes, ");")] + /// ``` + #[stable(feature = "int_to_from_bytes", since = "1.32.0")] + #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn to_le_bytes(self) -> [u8; mem::size_of::()] { + self.to_le().to_ne_bytes() + } + + /// Return the memory representation of this integer as a byte array in + /// native byte order. + /// + /// As the target platform's native endianness is used, portable code + /// should use [`to_be_bytes`] or [`to_le_bytes`], as appropriate, + /// instead. + /// + #[doc = $to_xe_bytes_doc] + /// + /// [`to_be_bytes`]: Self::to_be_bytes + /// [`to_le_bytes`]: Self::to_le_bytes + /// + /// # Examples + /// + /// ``` + #[doc = concat!("let bytes = ", $swap_op, stringify!($SelfT), ".to_ne_bytes();")] + /// assert_eq!( + /// bytes, + /// if cfg!(target_endian = "big") { + #[doc = concat!(" ", $be_bytes)] + /// } else { + #[doc = concat!(" ", $le_bytes)] + /// } + /// ); + /// ``` + #[stable(feature = "int_to_from_bytes", since = "1.32.0")] + #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] + // SAFETY: const sound because integers are plain old datatypes so we can always + // transmute them to arrays of bytes + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn to_ne_bytes(self) -> [u8; mem::size_of::()] { + // SAFETY: integers are plain old datatypes so we can always transmute them to + // arrays of bytes + unsafe { mem::transmute(self) } + } + + /// Create an integer value from its representation as a byte array in + /// big endian. + /// + #[doc = $from_xe_bytes_doc] + /// + /// # Examples + /// + /// ``` + #[doc = concat!("let value = ", stringify!($SelfT), "::from_be_bytes(", $be_bytes, ");")] + #[doc = concat!("assert_eq!(value, ", $swap_op, ");")] + /// ``` + /// + /// When starting from a slice rather than an array, fallible conversion APIs can be used: + /// + /// ``` + #[doc = concat!("fn read_be_", stringify!($SelfT), "(input: &mut &[u8]) -> ", stringify!($SelfT), " {")] + #[doc = concat!(" let (int_bytes, rest) = input.split_at(std::mem::size_of::<", stringify!($SelfT), ">());")] + /// *input = rest; + #[doc = concat!(" ", stringify!($SelfT), "::from_be_bytes(int_bytes.try_into().unwrap())")] + /// } + /// ``` + #[stable(feature = "int_to_from_bytes", since = "1.32.0")] + #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] + #[must_use] + #[inline] + pub const fn from_be_bytes(bytes: [u8; mem::size_of::()]) -> Self { + Self::from_be(Self::from_ne_bytes(bytes)) + } + + /// Create an integer value from its representation as a byte array in + /// little endian. + /// + #[doc = $from_xe_bytes_doc] + /// + /// # Examples + /// + /// ``` + #[doc = concat!("let value = ", stringify!($SelfT), "::from_le_bytes(", $le_bytes, ");")] + #[doc = concat!("assert_eq!(value, ", $swap_op, ");")] + /// ``` + /// + /// When starting from a slice rather than an array, fallible conversion APIs can be used: + /// + /// ``` + #[doc = concat!("fn read_le_", stringify!($SelfT), "(input: &mut &[u8]) -> ", stringify!($SelfT), " {")] + #[doc = concat!(" let (int_bytes, rest) = input.split_at(std::mem::size_of::<", stringify!($SelfT), ">());")] + /// *input = rest; + #[doc = concat!(" ", stringify!($SelfT), "::from_le_bytes(int_bytes.try_into().unwrap())")] + /// } + /// ``` + #[stable(feature = "int_to_from_bytes", since = "1.32.0")] + #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] + #[must_use] + #[inline] + pub const fn from_le_bytes(bytes: [u8; mem::size_of::()]) -> Self { + Self::from_le(Self::from_ne_bytes(bytes)) + } + + /// Create an integer value from its memory representation as a byte + /// array in native endianness. + /// + /// As the target platform's native endianness is used, portable code + /// likely wants to use [`from_be_bytes`] or [`from_le_bytes`], as + /// appropriate instead. + /// + /// [`from_be_bytes`]: Self::from_be_bytes + /// [`from_le_bytes`]: Self::from_le_bytes + /// + #[doc = $from_xe_bytes_doc] + /// + /// # Examples + /// + /// ``` + #[doc = concat!("let value = ", stringify!($SelfT), "::from_ne_bytes(if cfg!(target_endian = \"big\") {")] + #[doc = concat!(" ", $be_bytes)] + /// } else { + #[doc = concat!(" ", $le_bytes)] + /// }); + #[doc = concat!("assert_eq!(value, ", $swap_op, ");")] + /// ``` + /// + /// When starting from a slice rather than an array, fallible conversion APIs can be used: + /// + /// ``` + #[doc = concat!("fn read_ne_", stringify!($SelfT), "(input: &mut &[u8]) -> ", stringify!($SelfT), " {")] + #[doc = concat!(" let (int_bytes, rest) = input.split_at(std::mem::size_of::<", stringify!($SelfT), ">());")] + /// *input = rest; + #[doc = concat!(" ", stringify!($SelfT), "::from_ne_bytes(int_bytes.try_into().unwrap())")] + /// } + /// ``` + #[stable(feature = "int_to_from_bytes", since = "1.32.0")] + #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] + #[must_use] + // SAFETY: const sound because integers are plain old datatypes so we can always + // transmute to them + #[inline] + pub const fn from_ne_bytes(bytes: [u8; mem::size_of::()]) -> Self { + // SAFETY: integers are plain old datatypes so we can always transmute to them + unsafe { mem::transmute(bytes) } + } -macro_rules! int_module { - ($T:ident) => (int_module!($T, #[stable(feature = "rust1", since = "1.0.0")]);); - ($T:ident, #[$attr:meta]) => ( - doc_comment! { - concat!("The smallest value that can be represented by this integer type. -Use [`", stringify!($T), "::MIN", "`](../../std/primitive.", stringify!($T), ".html#associatedconstant.MIN) instead."), - #[$attr] - pub const MIN: $T = $T::min_value(); + /// New code should prefer to use + #[doc = concat!("[`", stringify!($SelfT), "::MIN", "`] instead.")] + /// + /// Returns the smallest value that can be represented by this integer type. + #[stable(feature = "rust1", since = "1.0.0")] + #[inline(always)] + #[rustc_promotable] + #[rustc_const_stable(feature = "const_min_value", since = "1.32.0")] + #[deprecated(since = "TBD", note = "replaced by the `MIN` associated constant on this type")] + pub const fn min_value() -> Self { + Self::MIN } - doc_comment! { - concat!("The largest value that can be represented by this integer type. -Use [`", stringify!($T), "::MAX", "`](../../std/primitive.", stringify!($T), ".html#associatedconstant.MAX) instead."), - #[$attr] - pub const MAX: $T = $T::max_value(); + /// New code should prefer to use + #[doc = concat!("[`", stringify!($SelfT), "::MAX", "`] instead.")] + /// + /// Returns the largest value that can be represented by this integer type. + #[stable(feature = "rust1", since = "1.0.0")] + #[inline(always)] + #[rustc_promotable] + #[rustc_const_stable(feature = "const_max_value", since = "1.32.0")] + #[deprecated(since = "TBD", note = "replaced by the `MAX` associated constant on this type")] + pub const fn max_value() -> Self { + Self::MAX } - ) + } } diff --git a/crux-mir/lib/core/src/num/isize.rs b/crux-mir/lib/core/src/num/isize.rs deleted file mode 100644 index 0dcfa4a2b..000000000 --- a/crux-mir/lib/core/src/num/isize.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! The pointer-sized signed integer type. -//! -//! *[See also the `isize` primitive type](../../std/primitive.isize.html).* -//! -//! Although using these constants won’t cause compilation warnings, -//! new code should use the associated constants directly on the primitive type. - -#![stable(feature = "rust1", since = "1.0.0")] - -int_module! { isize } diff --git a/crux-mir/lib/core/src/num/mod.rs b/crux-mir/lib/core/src/num/mod.rs index 8431b33ce..ac7f579eb 100644 --- a/crux-mir/lib/core/src/num/mod.rs +++ b/crux-mir/lib/core/src/num/mod.rs @@ -1,15 +1,17 @@ -// ignore-tidy-filelength - //! Numeric traits and functions for the built-in numeric types. #![stable(feature = "rust1", since = "1.0.0")] -use crate::convert::Infallible; -use crate::fmt; +use crate::ascii; +use crate::convert::TryInto; use crate::intrinsics; use crate::mem; +use crate::ops::{Add, Mul, Sub}; use crate::str::FromStr; +#[cfg(not(no_fp_fmt_parse))] +use crate::error::Error; + // Used because the `?` operator is not allowed in a const context. macro_rules! try_opt { ($e:expr) => { @@ -20,4427 +22,289 @@ macro_rules! try_opt { }; } -macro_rules! impl_nonzero_fmt { - ( #[$stability: meta] ( $( $Trait: ident ),+ ) for $Ty: ident ) => { - $( - #[$stability] - impl fmt::$Trait for $Ty { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.get().fmt(f) - } - } - )+ - } -} - -macro_rules! doc_comment { - ($x:expr, $($tt:tt)*) => { - #[doc = $x] - $($tt)* - }; -} - -macro_rules! nonzero_integers { - ( $( #[$stability: meta] $Ty: ident($Int: ty); )+ ) => { - $( - doc_comment! { - concat!("An integer that is known not to equal zero. - -This enables some memory layout optimization. -For example, `Option<", stringify!($Ty), ">` is the same size as `", stringify!($Int), "`: - -```rust -use std::mem::size_of; -assert_eq!(size_of::>(), size_of::<", stringify!($Int), -">()); -```"), - #[$stability] - #[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] - #[repr(transparent)] - #[rustc_layout_scalar_valid_range_start(1)] - #[rustc_nonnull_optimization_guaranteed] - pub struct $Ty($Int); - } - - impl $Ty { - /// Creates a non-zero without checking the value. - /// - /// # Safety - /// - /// The value must not be zero. - #[$stability] - #[rustc_const_stable(feature = "nonzero", since = "1.34.0")] - #[inline] - pub const unsafe fn new_unchecked(n: $Int) -> Self { - Self(n) - } - - /// Creates a non-zero if the given value is not zero. - #[$stability] - #[rustc_const_unstable(feature = "const_nonzero_int_methods", issue = "53718")] - #[inline] - pub const fn new(n: $Int) -> Option { - if n != 0 { - // SAFETY: we just checked that there's no `0` - Some(unsafe { Self(n) }) - } else { - None - } - } - - /// Returns the value as a primitive type. - #[$stability] - #[inline] - #[rustc_const_stable(feature = "nonzero", since = "1.34.0")] - pub const fn get(self) -> $Int { - self.0 - } - - } - - #[stable(feature = "from_nonzero", since = "1.31.0")] - impl From<$Ty> for $Int { - doc_comment! { - concat!( -"Converts a `", stringify!($Ty), "` into an `", stringify!($Int), "`"), - fn from(nonzero: $Ty) -> Self { - nonzero.0 - } - } - } - - impl_nonzero_fmt! { - #[$stability] (Debug, Display, Binary, Octal, LowerHex, UpperHex) for $Ty - } - )+ - } -} - -nonzero_integers! { - #[stable(feature = "nonzero", since = "1.28.0")] NonZeroU8(u8); - #[stable(feature = "nonzero", since = "1.28.0")] NonZeroU16(u16); - #[stable(feature = "nonzero", since = "1.28.0")] NonZeroU32(u32); - #[stable(feature = "nonzero", since = "1.28.0")] NonZeroU64(u64); - #[stable(feature = "nonzero", since = "1.28.0")] NonZeroU128(u128); - #[stable(feature = "nonzero", since = "1.28.0")] NonZeroUsize(usize); - #[stable(feature = "signed_nonzero", since = "1.34.0")] NonZeroI8(i8); - #[stable(feature = "signed_nonzero", since = "1.34.0")] NonZeroI16(i16); - #[stable(feature = "signed_nonzero", since = "1.34.0")] NonZeroI32(i32); - #[stable(feature = "signed_nonzero", since = "1.34.0")] NonZeroI64(i64); - #[stable(feature = "signed_nonzero", since = "1.34.0")] NonZeroI128(i128); - #[stable(feature = "signed_nonzero", since = "1.34.0")] NonZeroIsize(isize); -} - -macro_rules! from_str_radix_nzint_impl { - ($($t:ty)*) => {$( - #[stable(feature = "nonzero_parse", since = "1.35.0")] - impl FromStr for $t { - type Err = ParseIntError; - fn from_str(src: &str) -> Result { - Self::new(from_str_radix(src, 10)?) - .ok_or(ParseIntError { - kind: IntErrorKind::Zero - }) - } - } - )*} -} - -from_str_radix_nzint_impl! { NonZeroU8 NonZeroU16 NonZeroU32 NonZeroU64 NonZeroU128 NonZeroUsize -NonZeroI8 NonZeroI16 NonZeroI32 NonZeroI64 NonZeroI128 NonZeroIsize } - -/// Provides intentionally-wrapped arithmetic on `T`. -/// -/// Operations like `+` on `u32` values are intended to never overflow, -/// and in some debug configurations overflow is detected and results -/// in a panic. While most arithmetic falls into this category, some -/// code explicitly expects and relies upon modular arithmetic (e.g., -/// hashing). -/// -/// Wrapping arithmetic can be achieved either through methods like -/// `wrapping_add`, or through the `Wrapping` type, which says that -/// all standard arithmetic operations on the underlying value are -/// intended to have wrapping semantics. -/// -/// The underlying value can be retrieved through the `.0` index of the -/// `Wrapping` tuple. -/// -/// # Examples -/// -/// ``` -/// use std::num::Wrapping; -/// -/// let zero = Wrapping(0u32); -/// let one = Wrapping(1u32); -/// -/// assert_eq!(std::u32::MAX, (zero - one).0); -/// ``` -#[stable(feature = "rust1", since = "1.0.0")] -#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default, Hash)] -#[repr(transparent)] -pub struct Wrapping(#[stable(feature = "rust1", since = "1.0.0")] pub T); - -#[stable(feature = "rust1", since = "1.0.0")] -impl fmt::Debug for Wrapping { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } -} - -#[stable(feature = "wrapping_display", since = "1.10.0")] -impl fmt::Display for Wrapping { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } -} - -#[stable(feature = "wrapping_fmt", since = "1.11.0")] -impl fmt::Binary for Wrapping { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } -} - -#[stable(feature = "wrapping_fmt", since = "1.11.0")] -impl fmt::Octal for Wrapping { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } -} - -#[stable(feature = "wrapping_fmt", since = "1.11.0")] -impl fmt::LowerHex for Wrapping { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } -} - -#[stable(feature = "wrapping_fmt", since = "1.11.0")] -impl fmt::UpperHex for Wrapping { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } -} - -// All these modules are technically private and only exposed for coretests: -pub mod bignum; -pub mod dec2flt; -pub mod diy_float; -pub mod flt2dec; - -mod wrapping; - -macro_rules! usize_isize_to_xe_bytes_doc { - () => { - " - -**Note**: This function returns an array of length 2, 4 or 8 bytes -depending on the target pointer size. - -" - }; -} - -macro_rules! usize_isize_from_xe_bytes_doc { - () => { - " - -**Note**: This function takes an array of length 2, 4 or 8 bytes -depending on the target pointer size. - -" +#[allow_internal_unstable(const_likely)] +macro_rules! unlikely { + ($e: expr) => { + intrinsics::unlikely($e) }; -} - -macro_rules! int_impl { - ($SelfT:ty, $ActualT:ident, $UnsignedT:ty, $BITS:expr, $Min:expr, $Max:expr, $Feature:expr, - $EndFeature:expr, $rot:expr, $rot_op:expr, $rot_result:expr, $swap_op:expr, $swapped:expr, - $reversed:expr, $le_bytes:expr, $be_bytes:expr, - $to_xe_bytes_doc:expr, $from_xe_bytes_doc:expr) => { - doc_comment! { - concat!("The smallest value that can be represented by this integer type. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(", stringify!($SelfT), "::MIN, ", stringify!($Min), ");", -$EndFeature, " -```"), - #[stable(feature = "assoc_int_consts", since = "1.43.0")] - pub const MIN: Self = !0 ^ ((!0 as $UnsignedT) >> 1) as Self; - } - - doc_comment! { - concat!("The largest value that can be represented by this integer type. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(", stringify!($SelfT), "::MAX, ", stringify!($Max), ");", -$EndFeature, " -```"), - #[stable(feature = "assoc_int_consts", since = "1.43.0")] - pub const MAX: Self = !Self::MIN; - } - - doc_comment! { - concat!("Converts a string slice in a given base to an integer. - -The string is expected to be an optional `+` or `-` sign followed by digits. -Leading and trailing whitespace represent an error. Digits are a subset of these characters, -depending on `radix`: - - * `0-9` - * `a-z` - * `A-Z` - -# Panics - -This function panics if `radix` is not in the range from 2 to 36. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(", stringify!($SelfT), "::from_str_radix(\"A\", 16), Ok(10));", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - pub fn from_str_radix(src: &str, radix: u32) -> Result { - from_str_radix(src, radix) - } - } - - doc_comment! { - concat!("Returns the number of ones in the binary representation of `self`. - -# Examples - -Basic usage: - -``` -", $Feature, "let n = 0b100_0000", stringify!($SelfT), "; - -assert_eq!(n.count_ones(), 1);", -$EndFeature, " -``` -"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[inline] - pub const fn count_ones(self) -> u32 { (self as $UnsignedT).count_ones() } - } - - doc_comment! { - concat!("Returns the number of zeros in the binary representation of `self`. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(", stringify!($SelfT), "::MAX.count_zeros(), 1);", $EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[inline] - pub const fn count_zeros(self) -> u32 { - (!self).count_ones() - } - } - - doc_comment! { - concat!("Returns the number of leading zeros in the binary representation of `self`. - -# Examples - -Basic usage: - -``` -", $Feature, "let n = -1", stringify!($SelfT), "; - -assert_eq!(n.leading_zeros(), 0);", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[inline] - pub const fn leading_zeros(self) -> u32 { - (self as $UnsignedT).leading_zeros() - } - } - - doc_comment! { - concat!("Returns the number of trailing zeros in the binary representation of `self`. - -# Examples - -Basic usage: - -``` -", $Feature, "let n = -4", stringify!($SelfT), "; - -assert_eq!(n.trailing_zeros(), 2);", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[inline] - pub const fn trailing_zeros(self) -> u32 { - (self as $UnsignedT).trailing_zeros() - } - } - - doc_comment! { - concat!("Returns the number of leading ones in the binary representation of `self`. - -# Examples - -Basic usage: - -``` -", $Feature, "#![feature(leading_trailing_ones)] -let n = -1", stringify!($SelfT), "; - -assert_eq!(n.leading_ones(), ", stringify!($BITS), ");", -$EndFeature, " -```"), - #[unstable(feature = "leading_trailing_ones", issue = "57969")] - #[inline] - pub const fn leading_ones(self) -> u32 { - (self as $UnsignedT).leading_ones() - } - } - - doc_comment! { - concat!("Returns the number of trailing ones in the binary representation of `self`. - -# Examples - -Basic usage: - -``` -", $Feature, "#![feature(leading_trailing_ones)] -let n = 3", stringify!($SelfT), "; - -assert_eq!(n.trailing_ones(), 2);", -$EndFeature, " -```"), - #[unstable(feature = "leading_trailing_ones", issue = "57969")] - #[inline] - pub const fn trailing_ones(self) -> u32 { - (self as $UnsignedT).trailing_ones() - } - } - - doc_comment! { - concat!("Shifts the bits to the left by a specified amount, `n`, -wrapping the truncated bits to the end of the resulting integer. - -Please note this isn't the same operation as the `<<` shifting operator! - -# Examples - -Basic usage: - -``` -let n = ", $rot_op, stringify!($SelfT), "; -let m = ", $rot_result, "; - -assert_eq!(n.rotate_left(", $rot, "), m); -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn rotate_left(self, n: u32) -> Self { - (self as $UnsignedT).rotate_left(n) as Self - } - } - - doc_comment! { - concat!("Shifts the bits to the right by a specified amount, `n`, -wrapping the truncated bits to the beginning of the resulting -integer. - -Please note this isn't the same operation as the `>>` shifting operator! - -# Examples - -Basic usage: - -``` -let n = ", $rot_result, stringify!($SelfT), "; -let m = ", $rot_op, "; - -assert_eq!(n.rotate_right(", $rot, "), m); -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn rotate_right(self, n: u32) -> Self { - (self as $UnsignedT).rotate_right(n) as Self - } - } - - doc_comment! { - concat!("Reverses the byte order of the integer. - -# Examples - -Basic usage: - -``` -let n = ", $swap_op, stringify!($SelfT), "; - -let m = n.swap_bytes(); - -assert_eq!(m, ", $swapped, "); -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[inline] - pub const fn swap_bytes(self) -> Self { - (self as $UnsignedT).swap_bytes() as Self - } - } - - doc_comment! { - concat!("Reverses the bit pattern of the integer. - -# Examples - -Basic usage: - -``` -let n = ", $swap_op, stringify!($SelfT), "; -let m = n.reverse_bits(); - -assert_eq!(m, ", $reversed, "); -```"), - #[stable(feature = "reverse_bits", since = "1.37.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[inline] - #[must_use] - pub const fn reverse_bits(self) -> Self { - (self as $UnsignedT).reverse_bits() as Self - } - } - - doc_comment! { - concat!("Converts an integer from big endian to the target's endianness. - -On big endian this is a no-op. On little endian the bytes are swapped. - -# Examples - -Basic usage: - -``` -", $Feature, "let n = 0x1A", stringify!($SelfT), "; - -if cfg!(target_endian = \"big\") { - assert_eq!(", stringify!($SelfT), "::from_be(n), n) -} else { - assert_eq!(", stringify!($SelfT), "::from_be(n), n.swap_bytes()) -}", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_int_conversions", since = "1.32.0")] - #[inline] - pub const fn from_be(x: Self) -> Self { - #[cfg(target_endian = "big")] - { - x - } - #[cfg(not(target_endian = "big"))] - { - x.swap_bytes() - } - } - } - - doc_comment! { - concat!("Converts an integer from little endian to the target's endianness. - -On little endian this is a no-op. On big endian the bytes are swapped. - -# Examples - -Basic usage: - -``` -", $Feature, "let n = 0x1A", stringify!($SelfT), "; - -if cfg!(target_endian = \"little\") { - assert_eq!(", stringify!($SelfT), "::from_le(n), n) -} else { - assert_eq!(", stringify!($SelfT), "::from_le(n), n.swap_bytes()) -}", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_int_conversions", since = "1.32.0")] - #[inline] - pub const fn from_le(x: Self) -> Self { - #[cfg(target_endian = "little")] - { - x - } - #[cfg(not(target_endian = "little"))] - { - x.swap_bytes() - } - } - } - - doc_comment! { - concat!("Converts `self` to big endian from the target's endianness. - -On big endian this is a no-op. On little endian the bytes are swapped. - -# Examples - -Basic usage: - -``` -", $Feature, "let n = 0x1A", stringify!($SelfT), "; - -if cfg!(target_endian = \"big\") { - assert_eq!(n.to_be(), n) -} else { - assert_eq!(n.to_be(), n.swap_bytes()) -}", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_int_conversions", since = "1.32.0")] - #[inline] - pub const fn to_be(self) -> Self { // or not to be? - #[cfg(target_endian = "big")] - { - self - } - #[cfg(not(target_endian = "big"))] - { - self.swap_bytes() - } - } - } - - doc_comment! { - concat!("Converts `self` to little endian from the target's endianness. - -On little endian this is a no-op. On big endian the bytes are swapped. - -# Examples - -Basic usage: - -``` -", $Feature, "let n = 0x1A", stringify!($SelfT), "; - -if cfg!(target_endian = \"little\") { - assert_eq!(n.to_le(), n) -} else { - assert_eq!(n.to_le(), n.swap_bytes()) -}", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_int_conversions", since = "1.32.0")] - #[inline] - pub const fn to_le(self) -> Self { - #[cfg(target_endian = "little")] - { - self - } - #[cfg(not(target_endian = "little"))] - { - self.swap_bytes() - } - } - } - - doc_comment! { - concat!("Checked integer addition. Computes `self + rhs`, returning `None` -if overflow occurred. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!((", stringify!($SelfT), -"::MAX - 2).checked_add(1), Some(", stringify!($SelfT), "::MAX - 1)); -assert_eq!((", stringify!($SelfT), "::MAX - 2).checked_add(3), None);", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_checked_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn checked_add(self, rhs: Self) -> Option { - let (a, b) = self.overflowing_add(rhs); - if b {None} else {Some(a)} - } - } - - doc_comment! { - concat!("Checked integer subtraction. Computes `self - rhs`, returning `None` if -overflow occurred. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!((", stringify!($SelfT), -"::MIN + 2).checked_sub(1), Some(", stringify!($SelfT), "::MIN + 1)); -assert_eq!((", stringify!($SelfT), "::MIN + 2).checked_sub(3), None);", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_checked_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn checked_sub(self, rhs: Self) -> Option { - let (a, b) = self.overflowing_sub(rhs); - if b {None} else {Some(a)} - } - } - - doc_comment! { - concat!("Checked integer multiplication. Computes `self * rhs`, returning `None` if -overflow occurred. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(", stringify!($SelfT), -"::MAX.checked_mul(1), Some(", stringify!($SelfT), "::MAX)); -assert_eq!(", stringify!($SelfT), "::MAX.checked_mul(2), None);", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_checked_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn checked_mul(self, rhs: Self) -> Option { - let (a, b) = self.overflowing_mul(rhs); - if b {None} else {Some(a)} - } - } - - doc_comment! { - concat!("Checked integer division. Computes `self / rhs`, returning `None` if `rhs == 0` -or the division results in overflow. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!((", stringify!($SelfT), -"::MIN + 1).checked_div(-1), Some(", stringify!($Max), ")); -assert_eq!(", stringify!($SelfT), "::MIN.checked_div(-1), None); -assert_eq!((1", stringify!($SelfT), ").checked_div(0), None);", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_checked_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn checked_div(self, rhs: Self) -> Option { - if rhs == 0 || (self == Self::min_value() && rhs == -1) { - None - } else { - // SAFETY: div by zero and by INT_MIN have been checked above - Some(unsafe { intrinsics::unchecked_div(self, rhs) }) - } - } - } - - doc_comment! { - concat!("Checked Euclidean division. Computes `self.div_euclid(rhs)`, -returning `None` if `rhs == 0` or the division results in overflow. - -# Examples - -Basic usage: - -``` -assert_eq!((", stringify!($SelfT), -"::MIN + 1).checked_div_euclid(-1), Some(", stringify!($Max), ")); -assert_eq!(", stringify!($SelfT), "::MIN.checked_div_euclid(-1), None); -assert_eq!((1", stringify!($SelfT), ").checked_div_euclid(0), None); -```"), - #[stable(feature = "euclidean_division", since = "1.38.0")] - #[rustc_const_unstable(feature = "const_euclidean_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn checked_div_euclid(self, rhs: Self) -> Option { - if rhs == 0 || (self == Self::min_value() && rhs == -1) { - None - } else { - Some(self.div_euclid(rhs)) - } - } - } - - doc_comment! { - concat!("Checked integer remainder. Computes `self % rhs`, returning `None` if -`rhs == 0` or the division results in overflow. - -# Examples - -Basic usage: - -``` -", $Feature, " -assert_eq!(5", stringify!($SelfT), ".checked_rem(2), Some(1)); -assert_eq!(5", stringify!($SelfT), ".checked_rem(0), None); -assert_eq!(", stringify!($SelfT), "::MIN.checked_rem(-1), None);", -$EndFeature, " -```"), - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_unstable(feature = "const_checked_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn checked_rem(self, rhs: Self) -> Option { - if rhs == 0 || (self == Self::min_value() && rhs == -1) { - None - } else { - // SAFETY: div by zero and by INT_MIN have been checked above - Some(unsafe { intrinsics::unchecked_rem(self, rhs) }) - } - } - } - - doc_comment! { - concat!("Checked Euclidean remainder. Computes `self.rem_euclid(rhs)`, returning `None` -if `rhs == 0` or the division results in overflow. - -# Examples - -Basic usage: - -``` -assert_eq!(5", stringify!($SelfT), ".checked_rem_euclid(2), Some(1)); -assert_eq!(5", stringify!($SelfT), ".checked_rem_euclid(0), None); -assert_eq!(", stringify!($SelfT), "::MIN.checked_rem_euclid(-1), None); -```"), - #[stable(feature = "euclidean_division", since = "1.38.0")] - #[rustc_const_unstable(feature = "const_euclidean_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn checked_rem_euclid(self, rhs: Self) -> Option { - if rhs == 0 || (self == Self::min_value() && rhs == -1) { - None - } else { - Some(self.rem_euclid(rhs)) - } - } - } - - doc_comment! { - concat!("Checked negation. Computes `-self`, returning `None` if `self == MIN`. - -# Examples - -Basic usage: - -``` -", $Feature, " -assert_eq!(5", stringify!($SelfT), ".checked_neg(), Some(-5)); -assert_eq!(", stringify!($SelfT), "::MIN.checked_neg(), None);", -$EndFeature, " -```"), - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_unstable(feature = "const_checked_int_methods", issue = "53718")] - #[inline] - pub const fn checked_neg(self) -> Option { - let (a, b) = self.overflowing_neg(); - if b {None} else {Some(a)} - } - } - - doc_comment! { - concat!("Checked shift left. Computes `self << rhs`, returning `None` if `rhs` is larger -than or equal to the number of bits in `self`. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(0x1", stringify!($SelfT), ".checked_shl(4), Some(0x10)); -assert_eq!(0x1", stringify!($SelfT), ".checked_shl(129), None);", -$EndFeature, " -```"), - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_unstable(feature = "const_checked_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn checked_shl(self, rhs: u32) -> Option { - let (a, b) = self.overflowing_shl(rhs); - if b {None} else {Some(a)} - } - } - - doc_comment! { - concat!("Checked shift right. Computes `self >> rhs`, returning `None` if `rhs` is -larger than or equal to the number of bits in `self`. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(0x10", stringify!($SelfT), ".checked_shr(4), Some(0x1)); -assert_eq!(0x10", stringify!($SelfT), ".checked_shr(128), None);", -$EndFeature, " -```"), - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_unstable(feature = "const_checked_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn checked_shr(self, rhs: u32) -> Option { - let (a, b) = self.overflowing_shr(rhs); - if b {None} else {Some(a)} - } - } - - doc_comment! { - concat!("Checked absolute value. Computes `self.abs()`, returning `None` if -`self == MIN`. - -# Examples - -Basic usage: - -``` -", $Feature, " -assert_eq!((-5", stringify!($SelfT), ").checked_abs(), Some(5)); -assert_eq!(", stringify!($SelfT), "::MIN.checked_abs(), None);", -$EndFeature, " -```"), - #[stable(feature = "no_panic_abs", since = "1.13.0")] - #[rustc_const_unstable(feature = "const_checked_int_methods", issue = "53718")] - #[inline] - pub const fn checked_abs(self) -> Option { - if self.is_negative() { - self.checked_neg() - } else { - Some(self) - } - } - } - - doc_comment! { - concat!("Checked exponentiation. Computes `self.pow(exp)`, returning `None` if -overflow occurred. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(8", stringify!($SelfT), ".checked_pow(2), Some(64)); -assert_eq!(", stringify!($SelfT), "::MAX.checked_pow(2), None);", -$EndFeature, " -```"), - - #[stable(feature = "no_panic_pow", since = "1.34.0")] - #[rustc_const_unstable(feature = "const_int_pow", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn checked_pow(self, mut exp: u32) -> Option { - let mut base = self; - let mut acc: Self = 1; - - while exp > 1 { - if (exp & 1) == 1 { - acc = try_opt!(acc.checked_mul(base)); - } - exp /= 2; - base = try_opt!(base.checked_mul(base)); - } - - // Deal with the final bit of the exponent separately, since - // squaring the base afterwards is not necessary and may cause a - // needless overflow. - if exp == 1 { - acc = try_opt!(acc.checked_mul(base)); - } - - Some(acc) - } - } - - doc_comment! { - concat!("Saturating integer addition. Computes `self + rhs`, saturating at the numeric -bounds instead of overflowing. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(100", stringify!($SelfT), ".saturating_add(1), 101); -assert_eq!(", stringify!($SelfT), "::MAX.saturating_add(100), ", stringify!($SelfT), -"::MAX); -assert_eq!(", stringify!($SelfT), "::MIN.saturating_add(-1), ", stringify!($SelfT), -"::MIN);", -$EndFeature, " -```"), - - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_saturating_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn saturating_add(self, rhs: Self) -> Self { - intrinsics::saturating_add(self, rhs) - } - } - - doc_comment! { - concat!("Saturating integer subtraction. Computes `self - rhs`, saturating at the -numeric bounds instead of overflowing. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(100", stringify!($SelfT), ".saturating_sub(127), -27); -assert_eq!(", stringify!($SelfT), "::MIN.saturating_sub(100), ", stringify!($SelfT), -"::MIN); -assert_eq!(", stringify!($SelfT), "::MAX.saturating_sub(-1), ", stringify!($SelfT), -"::MAX);", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_saturating_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn saturating_sub(self, rhs: Self) -> Self { - intrinsics::saturating_sub(self, rhs) - } - } - - doc_comment! { - concat!("Saturating integer negation. Computes `-self`, returning `MAX` if `self == MIN` -instead of overflowing. - -# Examples - -Basic usage: - -``` -", $Feature, "#![feature(saturating_neg)] -assert_eq!(100", stringify!($SelfT), ".saturating_neg(), -100); -assert_eq!((-100", stringify!($SelfT), ").saturating_neg(), 100); -assert_eq!(", stringify!($SelfT), "::MIN.saturating_neg(), ", stringify!($SelfT), -"::MAX); -assert_eq!(", stringify!($SelfT), "::MAX.saturating_neg(), ", stringify!($SelfT), -"::MIN + 1);", -$EndFeature, " -```"), - - #[unstable(feature = "saturating_neg", issue = "59983")] - #[rustc_const_unstable(feature = "const_saturating_int_methods", issue = "53718")] - #[inline] - pub const fn saturating_neg(self) -> Self { - intrinsics::saturating_sub(0, self) - } - } - - doc_comment! { - concat!("Saturating absolute value. Computes `self.abs()`, returning `MAX` if `self == -MIN` instead of overflowing. - -# Examples - -Basic usage: - -``` -", $Feature, "#![feature(saturating_neg)] -assert_eq!(100", stringify!($SelfT), ".saturating_abs(), 100); -assert_eq!((-100", stringify!($SelfT), ").saturating_abs(), 100); -assert_eq!(", stringify!($SelfT), "::MIN.saturating_abs(), ", stringify!($SelfT), -"::MAX); -assert_eq!((", stringify!($SelfT), "::MIN + 1).saturating_abs(), ", stringify!($SelfT), -"::MAX);", -$EndFeature, " -```"), - - #[unstable(feature = "saturating_neg", issue = "59983")] - #[rustc_const_unstable(feature = "const_saturating_int_methods", issue = "53718")] - #[inline] - pub const fn saturating_abs(self) -> Self { - if self.is_negative() { - self.saturating_neg() - } else { - self - } - } - } - - doc_comment! { - concat!("Saturating integer multiplication. Computes `self * rhs`, saturating at the -numeric bounds instead of overflowing. - -# Examples - -Basic usage: - -``` -", $Feature, " -assert_eq!(10", stringify!($SelfT), ".saturating_mul(12), 120); -assert_eq!(", stringify!($SelfT), "::MAX.saturating_mul(10), ", stringify!($SelfT), "::MAX); -assert_eq!(", stringify!($SelfT), "::MIN.saturating_mul(10), ", stringify!($SelfT), "::MIN);", -$EndFeature, " -```"), - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_unstable(feature = "const_saturating_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn saturating_mul(self, rhs: Self) -> Self { - match self.checked_mul(rhs) { - Some(x) => x, - None => if (self < 0) == (rhs < 0) { - Self::max_value() - } else { - Self::min_value() - } - } - } - } - - doc_comment! { - concat!("Saturating integer exponentiation. Computes `self.pow(exp)`, -saturating at the numeric bounds instead of overflowing. - -# Examples - -Basic usage: - -``` -", $Feature, " -assert_eq!((-4", stringify!($SelfT), ").saturating_pow(3), -64); -assert_eq!(", stringify!($SelfT), "::MIN.saturating_pow(2), ", stringify!($SelfT), "::MAX); -assert_eq!(", stringify!($SelfT), "::MIN.saturating_pow(3), ", stringify!($SelfT), "::MIN);", -$EndFeature, " -```"), - #[stable(feature = "no_panic_pow", since = "1.34.0")] - #[rustc_const_unstable(feature = "const_int_pow", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn saturating_pow(self, exp: u32) -> Self { - match self.checked_pow(exp) { - Some(x) => x, - None if self < 0 && exp % 2 == 1 => Self::min_value(), - None => Self::max_value(), - } - } - } - - doc_comment! { - concat!("Wrapping (modular) addition. Computes `self + rhs`, wrapping around at the -boundary of the type. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(100", stringify!($SelfT), ".wrapping_add(27), 127); -assert_eq!(", stringify!($SelfT), "::MAX.wrapping_add(2), ", stringify!($SelfT), -"::MIN + 1);", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn wrapping_add(self, rhs: Self) -> Self { - intrinsics::wrapping_add(self, rhs) - } - } - - doc_comment! { - concat!("Wrapping (modular) subtraction. Computes `self - rhs`, wrapping around at the -boundary of the type. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(0", stringify!($SelfT), ".wrapping_sub(127), -127); -assert_eq!((-2", stringify!($SelfT), ").wrapping_sub(", stringify!($SelfT), "::MAX), ", -stringify!($SelfT), "::MAX);", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn wrapping_sub(self, rhs: Self) -> Self { - intrinsics::wrapping_sub(self, rhs) - } - } - - doc_comment! { - concat!("Wrapping (modular) multiplication. Computes `self * rhs`, wrapping around at -the boundary of the type. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(10", stringify!($SelfT), ".wrapping_mul(12), 120); -assert_eq!(11i8.wrapping_mul(12), -124);", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn wrapping_mul(self, rhs: Self) -> Self { - intrinsics::wrapping_mul(self, rhs) - } - } - - doc_comment! { - concat!("Wrapping (modular) division. Computes `self / rhs`, wrapping around at the -boundary of the type. - -The only case where such wrapping can occur is when one divides `MIN / -1` on a signed type (where -`MIN` is the negative minimal value for the type); this is equivalent to `-MIN`, a positive value -that is too large to represent in the type. In such a case, this function returns `MIN` itself. - -# Panics - -This function will panic if `rhs` is 0. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(100", stringify!($SelfT), ".wrapping_div(10), 10); -assert_eq!((-128i8).wrapping_div(-1), -128);", -$EndFeature, " -```"), - #[stable(feature = "num_wrapping", since = "1.2.0")] - #[rustc_const_unstable(feature = "const_wrapping_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn wrapping_div(self, rhs: Self) -> Self { - self.overflowing_div(rhs).0 - } - } - - doc_comment! { - concat!("Wrapping Euclidean division. Computes `self.div_euclid(rhs)`, -wrapping around at the boundary of the type. - -Wrapping will only occur in `MIN / -1` on a signed type (where `MIN` is the negative minimal value -for the type). This is equivalent to `-MIN`, a positive value that is too large to represent in the -type. In this case, this method returns `MIN` itself. - -# Panics - -This function will panic if `rhs` is 0. - -# Examples - -Basic usage: - -``` -assert_eq!(100", stringify!($SelfT), ".wrapping_div_euclid(10), 10); -assert_eq!((-128i8).wrapping_div_euclid(-1), -128); -```"), - #[stable(feature = "euclidean_division", since = "1.38.0")] - #[rustc_const_unstable(feature = "const_euclidean_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn wrapping_div_euclid(self, rhs: Self) -> Self { - self.overflowing_div_euclid(rhs).0 - } - } - - doc_comment! { - concat!("Wrapping (modular) remainder. Computes `self % rhs`, wrapping around at the -boundary of the type. - -Such wrap-around never actually occurs mathematically; implementation artifacts make `x % y` -invalid for `MIN / -1` on a signed type (where `MIN` is the negative minimal value). In such a case, -this function returns `0`. - -# Panics - -This function will panic if `rhs` is 0. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(100", stringify!($SelfT), ".wrapping_rem(10), 0); -assert_eq!((-128i8).wrapping_rem(-1), 0);", -$EndFeature, " -```"), - #[stable(feature = "num_wrapping", since = "1.2.0")] - #[rustc_const_unstable(feature = "const_wrapping_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn wrapping_rem(self, rhs: Self) -> Self { - self.overflowing_rem(rhs).0 - } - } - - doc_comment! { - concat!("Wrapping Euclidean remainder. Computes `self.rem_euclid(rhs)`, wrapping around -at the boundary of the type. - -Wrapping will only occur in `MIN % -1` on a signed type (where `MIN` is the negative minimal value -for the type). In this case, this method returns 0. - -# Panics - -This function will panic if `rhs` is 0. - -# Examples - -Basic usage: - -``` -assert_eq!(100", stringify!($SelfT), ".wrapping_rem_euclid(10), 0); -assert_eq!((-128i8).wrapping_rem_euclid(-1), 0); -```"), - #[stable(feature = "euclidean_division", since = "1.38.0")] - #[rustc_const_unstable(feature = "const_euclidean_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn wrapping_rem_euclid(self, rhs: Self) -> Self { - self.overflowing_rem_euclid(rhs).0 - } - } - - doc_comment! { - concat!("Wrapping (modular) negation. Computes `-self`, wrapping around at the boundary -of the type. - -The only case where such wrapping can occur is when one negates `MIN` on a signed type (where `MIN` -is the negative minimal value for the type); this is a positive value that is too large to represent -in the type. In such a case, this function returns `MIN` itself. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(100", stringify!($SelfT), ".wrapping_neg(), -100); -assert_eq!(", stringify!($SelfT), "::MIN.wrapping_neg(), ", stringify!($SelfT), -"::MIN);", -$EndFeature, " -```"), - #[stable(feature = "num_wrapping", since = "1.2.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[inline] - pub const fn wrapping_neg(self) -> Self { - self.overflowing_neg().0 - } - } - - doc_comment! { - concat!("Panic-free bitwise shift-left; yields `self << mask(rhs)`, where `mask` removes -any high-order bits of `rhs` that would cause the shift to exceed the bitwidth of the type. - -Note that this is *not* the same as a rotate-left; the RHS of a wrapping shift-left is restricted to -the range of the type, rather than the bits shifted out of the LHS being returned to the other end. -The primitive integer types all implement a `rotate_left` function, which may be what you want -instead. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!((-1", stringify!($SelfT), ").wrapping_shl(7), -128); -assert_eq!((-1", stringify!($SelfT), ").wrapping_shl(128), -1);", -$EndFeature, " -```"), - #[stable(feature = "num_wrapping", since = "1.2.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn wrapping_shl(self, rhs: u32) -> Self { - // SAFETY: the masking by the bitsize of the type ensures that we do not shift - // out of bounds - unsafe { - intrinsics::unchecked_shl(self, (rhs & ($BITS - 1)) as $SelfT) - } - } - } - - doc_comment! { - concat!("Panic-free bitwise shift-right; yields `self >> mask(rhs)`, where `mask` -removes any high-order bits of `rhs` that would cause the shift to exceed the bitwidth of the type. - -Note that this is *not* the same as a rotate-right; the RHS of a wrapping shift-right is restricted -to the range of the type, rather than the bits shifted out of the LHS being returned to the other -end. The primitive integer types all implement a `rotate_right` function, which may be what you want -instead. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!((-128", stringify!($SelfT), ").wrapping_shr(7), -1); -assert_eq!((-128i16).wrapping_shr(64), -128);", -$EndFeature, " -```"), - #[stable(feature = "num_wrapping", since = "1.2.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn wrapping_shr(self, rhs: u32) -> Self { - // SAFETY: the masking by the bitsize of the type ensures that we do not shift - // out of bounds - unsafe { - intrinsics::unchecked_shr(self, (rhs & ($BITS - 1)) as $SelfT) - } - } - } - - doc_comment! { - concat!("Wrapping (modular) absolute value. Computes `self.abs()`, wrapping around at -the boundary of the type. - -The only case where such wrapping can occur is when one takes the absolute value of the negative -minimal value for the type this is a positive value that is too large to represent in the type. In -such a case, this function returns `MIN` itself. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(100", stringify!($SelfT), ".wrapping_abs(), 100); -assert_eq!((-100", stringify!($SelfT), ").wrapping_abs(), 100); -assert_eq!(", stringify!($SelfT), "::MIN.wrapping_abs(), ", stringify!($SelfT), -"::MIN); -assert_eq!((-128i8).wrapping_abs() as u8, 128);", -$EndFeature, " -```"), - #[stable(feature = "no_panic_abs", since = "1.13.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[allow(unused_attributes)] - #[allow_internal_unstable(const_if_match)] - #[inline] - pub const fn wrapping_abs(self) -> Self { - if self.is_negative() { - self.wrapping_neg() - } else { - self - } - } - } - - doc_comment! { - concat!("Wrapping (modular) exponentiation. Computes `self.pow(exp)`, -wrapping around at the boundary of the type. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(3", stringify!($SelfT), ".wrapping_pow(4), 81); -assert_eq!(3i8.wrapping_pow(5), -13); -assert_eq!(3i8.wrapping_pow(6), -39);", -$EndFeature, " -```"), - #[stable(feature = "no_panic_pow", since = "1.34.0")] - #[rustc_const_unstable(feature = "const_int_pow", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn wrapping_pow(self, mut exp: u32) -> Self { - let mut base = self; - let mut acc: Self = 1; - - while exp > 1 { - if (exp & 1) == 1 { - acc = acc.wrapping_mul(base); - } - exp /= 2; - base = base.wrapping_mul(base); - } - - // Deal with the final bit of the exponent separately, since - // squaring the base afterwards is not necessary and may cause a - // needless overflow. - if exp == 1 { - acc = acc.wrapping_mul(base); - } - - acc - } - } - - doc_comment! { - concat!("Calculates `self` + `rhs` - -Returns a tuple of the addition along with a boolean indicating whether an arithmetic overflow would -occur. If an overflow would have occurred then the wrapped value is returned. - -# Examples - -Basic usage: - -``` -", $Feature, " -assert_eq!(5", stringify!($SelfT), ".overflowing_add(2), (7, false)); -assert_eq!(", stringify!($SelfT), "::MAX.overflowing_add(1), (", stringify!($SelfT), -"::MIN, true));", $EndFeature, " -```"), - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn overflowing_add(self, rhs: Self) -> (Self, bool) { - let (a, b) = intrinsics::add_with_overflow(self as $ActualT, rhs as $ActualT); - (a as Self, b) - } - } - - doc_comment! { - concat!("Calculates `self` - `rhs` - -Returns a tuple of the subtraction along with a boolean indicating whether an arithmetic overflow -would occur. If an overflow would have occurred then the wrapped value is returned. - -# Examples - -Basic usage: - -``` -", $Feature, " -assert_eq!(5", stringify!($SelfT), ".overflowing_sub(2), (3, false)); -assert_eq!(", stringify!($SelfT), "::MIN.overflowing_sub(1), (", stringify!($SelfT), -"::MAX, true));", $EndFeature, " -```"), - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn overflowing_sub(self, rhs: Self) -> (Self, bool) { - let (a, b) = intrinsics::sub_with_overflow(self as $ActualT, rhs as $ActualT); - (a as Self, b) - } - } - - doc_comment! { - concat!("Calculates the multiplication of `self` and `rhs`. - -Returns a tuple of the multiplication along with a boolean indicating whether an arithmetic overflow -would occur. If an overflow would have occurred then the wrapped value is returned. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(5", stringify!($SelfT), ".overflowing_mul(2), (10, false)); -assert_eq!(1_000_000_000i32.overflowing_mul(10), (1410065408, true));", -$EndFeature, " -```"), - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn overflowing_mul(self, rhs: Self) -> (Self, bool) { - let (a, b) = intrinsics::mul_with_overflow(self as $ActualT, rhs as $ActualT); - (a as Self, b) - } - } - - doc_comment! { - concat!("Calculates the divisor when `self` is divided by `rhs`. - -Returns a tuple of the divisor along with a boolean indicating whether an arithmetic overflow would -occur. If an overflow would occur then self is returned. - -# Panics - -This function will panic if `rhs` is 0. - -# Examples - -Basic usage: - -``` -", $Feature, " -assert_eq!(5", stringify!($SelfT), ".overflowing_div(2), (2, false)); -assert_eq!(", stringify!($SelfT), "::MIN.overflowing_div(-1), (", stringify!($SelfT), -"::MIN, true));", -$EndFeature, " -```"), - #[inline] - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_unstable(feature = "const_overflowing_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - pub const fn overflowing_div(self, rhs: Self) -> (Self, bool) { - if self == Self::min_value() && rhs == -1 { - (self, true) - } else { - (self / rhs, false) - } - } - } - - doc_comment! { - concat!("Calculates the quotient of Euclidean division `self.div_euclid(rhs)`. - -Returns a tuple of the divisor along with a boolean indicating whether an arithmetic overflow would -occur. If an overflow would occur then `self` is returned. - -# Panics - -This function will panic if `rhs` is 0. - -# Examples - -Basic usage: - -``` -assert_eq!(5", stringify!($SelfT), ".overflowing_div_euclid(2), (2, false)); -assert_eq!(", stringify!($SelfT), "::MIN.overflowing_div_euclid(-1), (", stringify!($SelfT), -"::MIN, true)); -```"), - #[inline] - #[stable(feature = "euclidean_division", since = "1.38.0")] - #[rustc_const_unstable(feature = "const_euclidean_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - pub const fn overflowing_div_euclid(self, rhs: Self) -> (Self, bool) { - if self == Self::min_value() && rhs == -1 { - (self, true) - } else { - (self.div_euclid(rhs), false) - } - } - } - - doc_comment! { - concat!("Calculates the remainder when `self` is divided by `rhs`. - -Returns a tuple of the remainder after dividing along with a boolean indicating whether an -arithmetic overflow would occur. If an overflow would occur then 0 is returned. - -# Panics - -This function will panic if `rhs` is 0. - -# Examples - -Basic usage: - -``` -", $Feature, " -assert_eq!(5", stringify!($SelfT), ".overflowing_rem(2), (1, false)); -assert_eq!(", stringify!($SelfT), "::MIN.overflowing_rem(-1), (0, true));", -$EndFeature, " -```"), - #[inline] - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_unstable(feature = "const_overflowing_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - pub const fn overflowing_rem(self, rhs: Self) -> (Self, bool) { - if self == Self::min_value() && rhs == -1 { - (0, true) - } else { - (self % rhs, false) - } - } - } - - - doc_comment! { - concat!("Overflowing Euclidean remainder. Calculates `self.rem_euclid(rhs)`. - -Returns a tuple of the remainder after dividing along with a boolean indicating whether an -arithmetic overflow would occur. If an overflow would occur then 0 is returned. - -# Panics - -This function will panic if `rhs` is 0. - -# Examples - -Basic usage: - -``` -assert_eq!(5", stringify!($SelfT), ".overflowing_rem_euclid(2), (1, false)); -assert_eq!(", stringify!($SelfT), "::MIN.overflowing_rem_euclid(-1), (0, true)); -```"), - #[stable(feature = "euclidean_division", since = "1.38.0")] - #[rustc_const_unstable(feature = "const_euclidean_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn overflowing_rem_euclid(self, rhs: Self) -> (Self, bool) { - if self == Self::min_value() && rhs == -1 { - (0, true) - } else { - (self.rem_euclid(rhs), false) - } - } - } - - - doc_comment! { - concat!("Negates self, overflowing if this is equal to the minimum value. - -Returns a tuple of the negated version of self along with a boolean indicating whether an overflow -happened. If `self` is the minimum value (e.g., `i32::MIN` for values of type `i32`), then the -minimum value will be returned again and `true` will be returned for an overflow happening. - -# Examples - -Basic usage: - -``` -assert_eq!(2", stringify!($SelfT), ".overflowing_neg(), (-2, false)); -assert_eq!(", stringify!($SelfT), "::MIN.overflowing_neg(), (", stringify!($SelfT), -"::MIN, true));", $EndFeature, " -```"), - #[inline] - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[allow(unused_attributes)] - #[allow_internal_unstable(const_if_match)] - pub const fn overflowing_neg(self) -> (Self, bool) { - if self == Self::min_value() { - (Self::min_value(), true) - } else { - (-self, false) - } - } - } - - doc_comment! { - concat!("Shifts self left by `rhs` bits. - -Returns a tuple of the shifted version of self along with a boolean indicating whether the shift -value was larger than or equal to the number of bits. If the shift value is too large, then value is -masked (N-1) where N is the number of bits, and this value is then used to perform the shift. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(0x1", stringify!($SelfT),".overflowing_shl(4), (0x10, false)); -assert_eq!(0x1i32.overflowing_shl(36), (0x10, true));", -$EndFeature, " -```"), - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn overflowing_shl(self, rhs: u32) -> (Self, bool) { - (self.wrapping_shl(rhs), (rhs > ($BITS - 1))) - } - } - - doc_comment! { - concat!("Shifts self right by `rhs` bits. - -Returns a tuple of the shifted version of self along with a boolean indicating whether the shift -value was larger than or equal to the number of bits. If the shift value is too large, then value is -masked (N-1) where N is the number of bits, and this value is then used to perform the shift. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(0x10", stringify!($SelfT), ".overflowing_shr(4), (0x1, false)); -assert_eq!(0x10i32.overflowing_shr(36), (0x1, true));", -$EndFeature, " -```"), - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn overflowing_shr(self, rhs: u32) -> (Self, bool) { - (self.wrapping_shr(rhs), (rhs > ($BITS - 1))) - } - } - - doc_comment! { - concat!("Computes the absolute value of `self`. - -Returns a tuple of the absolute version of self along with a boolean indicating whether an overflow -happened. If self is the minimum value (e.g., ", stringify!($SelfT), "::MIN for values of type - ", stringify!($SelfT), "), then the minimum value will be returned again and true will be returned -for an overflow happening. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(10", stringify!($SelfT), ".overflowing_abs(), (10, false)); -assert_eq!((-10", stringify!($SelfT), ").overflowing_abs(), (10, false)); -assert_eq!((", stringify!($SelfT), "::MIN).overflowing_abs(), (", stringify!($SelfT), -"::MIN, true));", -$EndFeature, " -```"), - #[stable(feature = "no_panic_abs", since = "1.13.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[inline] - pub const fn overflowing_abs(self) -> (Self, bool) { - (self.wrapping_abs(), self == Self::min_value()) - } - } - - doc_comment! { - concat!("Raises self to the power of `exp`, using exponentiation by squaring. - -Returns a tuple of the exponentiation along with a bool indicating -whether an overflow happened. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(3", stringify!($SelfT), ".overflowing_pow(4), (81, false)); -assert_eq!(3i8.overflowing_pow(5), (-13, true));", -$EndFeature, " -```"), - #[stable(feature = "no_panic_pow", since = "1.34.0")] - #[rustc_const_unstable(feature = "const_int_pow", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn overflowing_pow(self, mut exp: u32) -> (Self, bool) { - let mut base = self; - let mut acc: Self = 1; - let mut overflown = false; - // Scratch space for storing results of overflowing_mul. - let mut r; - - while exp > 1 { - if (exp & 1) == 1 { - r = acc.overflowing_mul(base); - acc = r.0; - overflown |= r.1; - } - exp /= 2; - r = base.overflowing_mul(base); - base = r.0; - overflown |= r.1; - } - - // Deal with the final bit of the exponent separately, since - // squaring the base afterwards is not necessary and may cause a - // needless overflow. - if exp == 1 { - r = acc.overflowing_mul(base); - acc = r.0; - overflown |= r.1; - } - - (acc, overflown) - } - } - - doc_comment! { - concat!("Raises self to the power of `exp`, using exponentiation by squaring. - -# Examples - -Basic usage: - -``` -", $Feature, "let x: ", stringify!($SelfT), " = 2; // or any other integer type - -assert_eq!(x.pow(5), 32);", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_int_pow", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - #[rustc_inherit_overflow_checks] - pub const fn pow(self, mut exp: u32) -> Self { - let mut base = self; - let mut acc = 1; - - while exp > 1 { - if (exp & 1) == 1 { - acc = acc * base; - } - exp /= 2; - base = base * base; - } - - // Deal with the final bit of the exponent separately, since - // squaring the base afterwards is not necessary and may cause a - // needless overflow. - if exp == 1 { - acc = acc * base; - } - - acc - } - } - - doc_comment! { - concat!("Calculates the quotient of Euclidean division of `self` by `rhs`. - -This computes the integer `n` such that `self = n * rhs + self.rem_euclid(rhs)`, -with `0 <= self.rem_euclid(rhs) < rhs`. - -In other words, the result is `self / rhs` rounded to the integer `n` -such that `self >= n * rhs`. -If `self > 0`, this is equal to round towards zero (the default in Rust); -if `self < 0`, this is equal to round towards +/- infinity. - -# Panics - -This function will panic if `rhs` is 0 or the division results in overflow. - -# Examples - -Basic usage: - -``` -let a: ", stringify!($SelfT), " = 7; // or any other integer type -let b = 4; - -assert_eq!(a.div_euclid(b), 1); // 7 >= 4 * 1 -assert_eq!(a.div_euclid(-b), -1); // 7 >= -4 * -1 -assert_eq!((-a).div_euclid(b), -2); // -7 >= 4 * -2 -assert_eq!((-a).div_euclid(-b), 2); // -7 >= -4 * 2 -```"), - #[stable(feature = "euclidean_division", since = "1.38.0")] - #[rustc_const_unstable(feature = "const_euclidean_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - #[rustc_inherit_overflow_checks] - pub const fn div_euclid(self, rhs: Self) -> Self { - let q = self / rhs; - if self % rhs < 0 { - return if rhs > 0 { q - 1 } else { q + 1 } - } - q - } - } - - - doc_comment! { - concat!("Calculates the least nonnegative remainder of `self (mod rhs)`. - -This is done as if by the Euclidean division algorithm -- given -`r = self.rem_euclid(rhs)`, `self = rhs * self.div_euclid(rhs) + r`, and -`0 <= r < abs(rhs)`. - -# Panics - -This function will panic if `rhs` is 0 or the division results in overflow. - -# Examples - -Basic usage: - -``` -let a: ", stringify!($SelfT), " = 7; // or any other integer type -let b = 4; - -assert_eq!(a.rem_euclid(b), 3); -assert_eq!((-a).rem_euclid(b), 1); -assert_eq!(a.rem_euclid(-b), 3); -assert_eq!((-a).rem_euclid(-b), 1); -```"), - #[stable(feature = "euclidean_division", since = "1.38.0")] - #[rustc_const_unstable(feature = "const_euclidean_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - #[rustc_inherit_overflow_checks] - pub const fn rem_euclid(self, rhs: Self) -> Self { - let r = self % rhs; - if r < 0 { - if rhs < 0 { - r - rhs - } else { - r + rhs - } - } else { - r - } - } - } - - doc_comment! { - concat!("Computes the absolute value of `self`. - -# Overflow behavior - -The absolute value of `", stringify!($SelfT), "::MIN` cannot be represented as an -`", stringify!($SelfT), "`, and attempting to calculate it will cause an overflow. This means that -code in debug mode will trigger a panic on this case and optimized code will return `", -stringify!($SelfT), "::MIN` without a panic. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(10", stringify!($SelfT), ".abs(), 10); -assert_eq!((-10", stringify!($SelfT), ").abs(), 10);", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[allow(unused_attributes)] - #[allow_internal_unstable(const_if_match)] - #[inline] - #[rustc_inherit_overflow_checks] - pub const fn abs(self) -> Self { - // Note that the #[inline] above means that the overflow - // semantics of the subtraction depend on the crate we're being - // inlined into. - if self.is_negative() { - -self - } else { - self - } - } - } - - doc_comment! { - concat!("Returns a number representing sign of `self`. - - - `0` if the number is zero - - `1` if the number is positive - - `-1` if the number is negative - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(10", stringify!($SelfT), ".signum(), 1); -assert_eq!(0", stringify!($SelfT), ".signum(), 0); -assert_eq!((-10", stringify!($SelfT), ").signum(), -1);", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_int_sign", issue = "53718")] - #[inline] - pub const fn signum(self) -> Self { - match self { - n if n > 0 => 1, - 0 => 0, - _ => -1, - } - } - } - - doc_comment! { - concat!("Returns `true` if `self` is positive and `false` if the number is zero or -negative. - -# Examples - -Basic usage: - -``` -", $Feature, "assert!(10", stringify!($SelfT), ".is_positive()); -assert!(!(-10", stringify!($SelfT), ").is_positive());", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[inline] - pub const fn is_positive(self) -> bool { self > 0 } - } - - doc_comment! { - concat!("Returns `true` if `self` is negative and `false` if the number is zero or -positive. - -# Examples - -Basic usage: - -``` -", $Feature, "assert!((-10", stringify!($SelfT), ").is_negative()); -assert!(!10", stringify!($SelfT), ".is_negative());", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")] - #[inline] - pub const fn is_negative(self) -> bool { self < 0 } - } - - doc_comment! { - concat!("Return the memory representation of this integer as a byte array in -big-endian (network) byte order. -", -$to_xe_bytes_doc, -" -# Examples - -``` -let bytes = ", $swap_op, stringify!($SelfT), ".to_be_bytes(); -assert_eq!(bytes, ", $be_bytes, "); -```"), - #[stable(feature = "int_to_from_bytes", since = "1.32.0")] - #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] - #[inline] - pub const fn to_be_bytes(self) -> [u8; mem::size_of::()] { - let mut buf = [0; $BITS / 8]; - buf[15 % ($BITS / 8)] = (self >> ( 0 % $BITS)) as u8; - buf[14 % ($BITS / 8)] = (self >> ( 8 % $BITS)) as u8; - buf[13 % ($BITS / 8)] = (self >> ( 16 % $BITS)) as u8; - buf[12 % ($BITS / 8)] = (self >> ( 24 % $BITS)) as u8; - buf[11 % ($BITS / 8)] = (self >> ( 32 % $BITS)) as u8; - buf[10 % ($BITS / 8)] = (self >> ( 40 % $BITS)) as u8; - buf[ 9 % ($BITS / 8)] = (self >> ( 48 % $BITS)) as u8; - buf[ 8 % ($BITS / 8)] = (self >> ( 56 % $BITS)) as u8; - buf[ 7 % ($BITS / 8)] = (self >> ( 64 % $BITS)) as u8; - buf[ 6 % ($BITS / 8)] = (self >> ( 72 % $BITS)) as u8; - buf[ 5 % ($BITS / 8)] = (self >> ( 80 % $BITS)) as u8; - buf[ 4 % ($BITS / 8)] = (self >> ( 88 % $BITS)) as u8; - buf[ 3 % ($BITS / 8)] = (self >> ( 96 % $BITS)) as u8; - buf[ 2 % ($BITS / 8)] = (self >> (104 % $BITS)) as u8; - buf[ 1 % ($BITS / 8)] = (self >> (112 % $BITS)) as u8; - buf[ 0 % ($BITS / 8)] = (self >> (120 % $BITS)) as u8; - buf - } - } - -doc_comment! { - concat!("Return the memory representation of this integer as a byte array in -little-endian byte order. -", -$to_xe_bytes_doc, -" -# Examples - -``` -let bytes = ", $swap_op, stringify!($SelfT), ".to_le_bytes(); -assert_eq!(bytes, ", $le_bytes, "); -```"), - #[stable(feature = "int_to_from_bytes", since = "1.32.0")] - #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] - #[inline] - pub const fn to_le_bytes(self) -> [u8; mem::size_of::()] { - let mut buf = [0; $BITS / 8]; - buf[ 0 % ($BITS / 8)] = (self >> ( 0 % $BITS)) as u8; - buf[ 1 % ($BITS / 8)] = (self >> ( 8 % $BITS)) as u8; - buf[ 2 % ($BITS / 8)] = (self >> ( 16 % $BITS)) as u8; - buf[ 3 % ($BITS / 8)] = (self >> ( 24 % $BITS)) as u8; - buf[ 4 % ($BITS / 8)] = (self >> ( 32 % $BITS)) as u8; - buf[ 5 % ($BITS / 8)] = (self >> ( 40 % $BITS)) as u8; - buf[ 6 % ($BITS / 8)] = (self >> ( 48 % $BITS)) as u8; - buf[ 7 % ($BITS / 8)] = (self >> ( 56 % $BITS)) as u8; - buf[ 8 % ($BITS / 8)] = (self >> ( 64 % $BITS)) as u8; - buf[ 9 % ($BITS / 8)] = (self >> ( 72 % $BITS)) as u8; - buf[10 % ($BITS / 8)] = (self >> ( 80 % $BITS)) as u8; - buf[11 % ($BITS / 8)] = (self >> ( 88 % $BITS)) as u8; - buf[12 % ($BITS / 8)] = (self >> ( 96 % $BITS)) as u8; - buf[13 % ($BITS / 8)] = (self >> (104 % $BITS)) as u8; - buf[14 % ($BITS / 8)] = (self >> (112 % $BITS)) as u8; - buf[15 % ($BITS / 8)] = (self >> (120 % $BITS)) as u8; - buf - } - } - - doc_comment! { - concat!(" -Return the memory representation of this integer as a byte array in -native byte order. - -As the target platform's native endianness is used, portable code -should use [`to_be_bytes`] or [`to_le_bytes`], as appropriate, -instead. -", -$to_xe_bytes_doc, -" -[`to_be_bytes`]: #method.to_be_bytes -[`to_le_bytes`]: #method.to_le_bytes - -# Examples - -``` -let bytes = ", $swap_op, stringify!($SelfT), ".to_ne_bytes(); -assert_eq!( - bytes, - if cfg!(target_endian = \"big\") { - ", $be_bytes, " - } else { - ", $le_bytes, " - } -); -```"), - #[stable(feature = "int_to_from_bytes", since = "1.32.0")] - #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] - // SAFETY: const sound because integers are plain old datatypes so we can always - // transmute them to arrays of bytes - #[allow_internal_unstable(const_fn_union)] - #[inline] - pub const fn to_ne_bytes(self) -> [u8; mem::size_of::()] { - #[repr(C)] - union Bytes { - val: $SelfT, - bytes: [u8; mem::size_of::<$SelfT>()], - } - // SAFETY: integers are plain old datatypes so we can always transmute them to - // arrays of bytes - unsafe { Bytes { val: self }.bytes } - } - } - -doc_comment! { - concat!("Create an integer value from its representation as a byte array in -big endian. -", -$from_xe_bytes_doc, -" -# Examples - -``` -let value = ", stringify!($SelfT), "::from_be_bytes(", $be_bytes, "); -assert_eq!(value, ", $swap_op, "); -``` - -When starting from a slice rather than an array, fallible conversion APIs can be used: - -``` -use std::convert::TryInto; - -fn read_be_", stringify!($SelfT), "(input: &mut &[u8]) -> ", stringify!($SelfT), " { - let (int_bytes, rest) = input.split_at(std::mem::size_of::<", stringify!($SelfT), ">()); - *input = rest; - ", stringify!($SelfT), "::from_be_bytes(int_bytes.try_into().unwrap()) -} -```"), - #[stable(feature = "int_to_from_bytes", since = "1.32.0")] - #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] - #[inline] - pub const fn from_be_bytes(bytes: [u8; mem::size_of::()]) -> Self { - Self::from_be(Self::from_ne_bytes(bytes)) - } - } - -doc_comment! { - concat!(" -Create an integer value from its representation as a byte array in -little endian. -", -$from_xe_bytes_doc, -" -# Examples - -``` -let value = ", stringify!($SelfT), "::from_le_bytes(", $le_bytes, "); -assert_eq!(value, ", $swap_op, "); -``` - -When starting from a slice rather than an array, fallible conversion APIs can be used: - -``` -use std::convert::TryInto; - -fn read_le_", stringify!($SelfT), "(input: &mut &[u8]) -> ", stringify!($SelfT), " { - let (int_bytes, rest) = input.split_at(std::mem::size_of::<", stringify!($SelfT), ">()); - *input = rest; - ", stringify!($SelfT), "::from_le_bytes(int_bytes.try_into().unwrap()) -} -```"), - #[stable(feature = "int_to_from_bytes", since = "1.32.0")] - #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] - #[inline] - pub const fn from_le_bytes(bytes: [u8; mem::size_of::()]) -> Self { - Self::from_le(Self::from_ne_bytes(bytes)) - } - } - - doc_comment! { - concat!("Create an integer value from its memory representation as a byte -array in native endianness. - -As the target platform's native endianness is used, portable code -likely wants to use [`from_be_bytes`] or [`from_le_bytes`], as -appropriate instead. - -[`from_be_bytes`]: #method.from_be_bytes -[`from_le_bytes`]: #method.from_le_bytes -", -$from_xe_bytes_doc, -" -# Examples - -``` -let value = ", stringify!($SelfT), "::from_ne_bytes(if cfg!(target_endian = \"big\") { - ", $be_bytes, " -} else { - ", $le_bytes, " -}); -assert_eq!(value, ", $swap_op, "); -``` - -When starting from a slice rather than an array, fallible conversion APIs can be used: - -``` -use std::convert::TryInto; - -fn read_ne_", stringify!($SelfT), "(input: &mut &[u8]) -> ", stringify!($SelfT), " { - let (int_bytes, rest) = input.split_at(std::mem::size_of::<", stringify!($SelfT), ">()); - *input = rest; - ", stringify!($SelfT), "::from_ne_bytes(int_bytes.try_into().unwrap()) -} -```"), - #[stable(feature = "int_to_from_bytes", since = "1.32.0")] - #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] - // SAFETY: const sound because integers are plain old datatypes so we can always - // transmute to them - #[allow_internal_unstable(const_fn_union)] - #[inline] - pub const fn from_ne_bytes(bytes: [u8; mem::size_of::()]) -> Self { - #[repr(C)] - union Bytes { - val: $SelfT, - bytes: [u8; mem::size_of::<$SelfT>()], - } - // SAFETY: integers are plain old datatypes so we can always transmute to them - unsafe { Bytes { bytes }.val } - } - } - - doc_comment! { - concat!("**This method is soft-deprecated.** - -Although using it won’t cause compilation warning, -new code should use [`", stringify!($SelfT), "::MIN", "`](#associatedconstant.MIN) instead. - -Returns the smallest value that can be represented by this integer type."), - #[stable(feature = "rust1", since = "1.0.0")] - #[inline(always)] - #[rustc_promotable] - #[rustc_const_stable(feature = "const_min_value", since = "1.32.0")] - pub const fn min_value() -> Self { - Self::MIN - } - } - - doc_comment! { - concat!("**This method is soft-deprecated.** - -Although using it won’t cause compilation warning, -new code should use [`", stringify!($SelfT), "::MAX", "`](#associatedconstant.MAX) instead. - -Returns the largest value that can be represented by this integer type."), - #[stable(feature = "rust1", since = "1.0.0")] - #[inline(always)] - #[rustc_promotable] - #[rustc_const_stable(feature = "const_max_value", since = "1.32.0")] - pub const fn max_value() -> Self { - Self::MAX - } - } - } -} - -#[lang = "i8"] -impl i8 { - int_impl! { i8, i8, u8, 8, -128, 127, "", "", 2, "-0x7e", "0xa", "0x12", "0x12", "0x48", - "[0x12]", "[0x12]", "", "" } -} - -#[lang = "i16"] -impl i16 { - int_impl! { i16, i16, u16, 16, -32768, 32767, "", "", 4, "-0x5ffd", "0x3a", "0x1234", "0x3412", - "0x2c48", "[0x34, 0x12]", "[0x12, 0x34]", "", "" } -} - -#[lang = "i32"] -impl i32 { - int_impl! { i32, i32, u32, 32, -2147483648, 2147483647, "", "", 8, "0x10000b3", "0xb301", - "0x12345678", "0x78563412", "0x1e6a2c48", "[0x78, 0x56, 0x34, 0x12]", - "[0x12, 0x34, 0x56, 0x78]", "", "" } -} - -#[lang = "i64"] -impl i64 { - int_impl! { i64, i64, u64, 64, -9223372036854775808, 9223372036854775807, "", "", 12, - "0xaa00000000006e1", "0x6e10aa", "0x1234567890123456", "0x5634129078563412", - "0x6a2c48091e6a2c48", "[0x56, 0x34, 0x12, 0x90, 0x78, 0x56, 0x34, 0x12]", - "[0x12, 0x34, 0x56, 0x78, 0x90, 0x12, 0x34, 0x56]", "", "" } -} - -#[lang = "i128"] -impl i128 { - int_impl! { i128, i128, u128, 128, -170141183460469231731687303715884105728, - 170141183460469231731687303715884105727, "", "", 16, - "0x13f40000000000000000000000004f76", "0x4f7613f4", "0x12345678901234567890123456789012", - "0x12907856341290785634129078563412", "0x48091e6a2c48091e6a2c48091e6a2c48", - "[0x12, 0x90, 0x78, 0x56, 0x34, 0x12, 0x90, 0x78, \ - 0x56, 0x34, 0x12, 0x90, 0x78, 0x56, 0x34, 0x12]", - "[0x12, 0x34, 0x56, 0x78, 0x90, 0x12, 0x34, 0x56, \ - 0x78, 0x90, 0x12, 0x34, 0x56, 0x78, 0x90, 0x12]", "", "" } -} - -#[cfg(target_pointer_width = "16")] -#[lang = "isize"] -impl isize { - int_impl! { isize, i16, u16, 16, -32768, 32767, "", "", 4, "-0x5ffd", "0x3a", "0x1234", - "0x3412", "0x2c48", "[0x34, 0x12]", "[0x12, 0x34]", - usize_isize_to_xe_bytes_doc!(), usize_isize_from_xe_bytes_doc!() } -} - -#[cfg(target_pointer_width = "32")] -#[lang = "isize"] -impl isize { - int_impl! { isize, i32, u32, 32, -2147483648, 2147483647, "", "", 8, "0x10000b3", "0xb301", - "0x12345678", "0x78563412", "0x1e6a2c48", "[0x78, 0x56, 0x34, 0x12]", - "[0x12, 0x34, 0x56, 0x78]", - usize_isize_to_xe_bytes_doc!(), usize_isize_from_xe_bytes_doc!() } -} - -#[cfg(target_pointer_width = "64")] -#[lang = "isize"] -impl isize { - int_impl! { isize, i64, u64, 64, -9223372036854775808, 9223372036854775807, "", "", - 12, "0xaa00000000006e1", "0x6e10aa", "0x1234567890123456", "0x5634129078563412", - "0x6a2c48091e6a2c48", "[0x56, 0x34, 0x12, 0x90, 0x78, 0x56, 0x34, 0x12]", - "[0x12, 0x34, 0x56, 0x78, 0x90, 0x12, 0x34, 0x56]", - usize_isize_to_xe_bytes_doc!(), usize_isize_from_xe_bytes_doc!() } -} - -macro_rules! uint_impl { - ($SelfT:ty, $ActualT:ty, $BITS:expr, $MaxV:expr, $Feature:expr, $EndFeature:expr, - $rot:expr, $rot_op:expr, $rot_result:expr, $swap_op:expr, $swapped:expr, - $reversed:expr, $le_bytes:expr, $be_bytes:expr, - $to_xe_bytes_doc:expr, $from_xe_bytes_doc:expr) => { - doc_comment! { - concat!("The smallest value that can be represented by this integer type. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(", stringify!($SelfT), "::MIN, 0);", $EndFeature, " -```"), - #[stable(feature = "assoc_int_consts", since = "1.43.0")] - pub const MIN: Self = 0; - } - - doc_comment! { - concat!("The largest value that can be represented by this integer type. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(", stringify!($SelfT), "::MAX, ", stringify!($MaxV), ");", -$EndFeature, " -```"), - #[stable(feature = "assoc_int_consts", since = "1.43.0")] - pub const MAX: Self = !0; - } - - doc_comment! { - concat!("Converts a string slice in a given base to an integer. - -The string is expected to be an optional `+` sign -followed by digits. -Leading and trailing whitespace represent an error. -Digits are a subset of these characters, depending on `radix`: - -* `0-9` -* `a-z` -* `A-Z` - -# Panics - -This function panics if `radix` is not in the range from 2 to 36. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(", stringify!($SelfT), "::from_str_radix(\"A\", 16), Ok(10));", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - pub fn from_str_radix(src: &str, radix: u32) -> Result { - from_str_radix(src, radix) - } - } - - doc_comment! { - concat!("Returns the number of ones in the binary representation of `self`. - -# Examples - -Basic usage: - -``` -", $Feature, "let n = 0b01001100", stringify!($SelfT), "; - -assert_eq!(n.count_ones(), 3);", $EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_math", since = "1.32.0")] - #[inline] - pub const fn count_ones(self) -> u32 { - intrinsics::ctpop(self as $ActualT) as u32 - } - } - - doc_comment! { - concat!("Returns the number of zeros in the binary representation of `self`. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(", stringify!($SelfT), "::MAX.count_zeros(), 0);", $EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_math", since = "1.32.0")] - #[inline] - pub const fn count_zeros(self) -> u32 { - (!self).count_ones() - } - } - - doc_comment! { - concat!("Returns the number of leading zeros in the binary representation of `self`. - -# Examples - -Basic usage: - -``` -", $Feature, "let n = ", stringify!($SelfT), "::MAX >> 2; - -assert_eq!(n.leading_zeros(), 2);", $EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_math", since = "1.32.0")] - #[inline] - pub const fn leading_zeros(self) -> u32 { - intrinsics::ctlz(self as $ActualT) as u32 - } - } - - doc_comment! { - concat!("Returns the number of trailing zeros in the binary representation -of `self`. - -# Examples - -Basic usage: - -``` -", $Feature, "let n = 0b0101000", stringify!($SelfT), "; - -assert_eq!(n.trailing_zeros(), 3);", $EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_math", since = "1.32.0")] - #[inline] - pub const fn trailing_zeros(self) -> u32 { - intrinsics::cttz(self) as u32 - } - } - - doc_comment! { - concat!("Returns the number of leading ones in the binary representation of `self`. - -# Examples - -Basic usage: - -``` -", $Feature, "#![feature(leading_trailing_ones)] -let n = !(", stringify!($SelfT), "::MAX >> 2); - -assert_eq!(n.leading_ones(), 2);", $EndFeature, " -```"), - #[unstable(feature = "leading_trailing_ones", issue = "57969")] - #[inline] - pub const fn leading_ones(self) -> u32 { - (!self).leading_zeros() - } - } - - doc_comment! { - concat!("Returns the number of trailing ones in the binary representation -of `self`. - -# Examples - -Basic usage: - -``` -", $Feature, "#![feature(leading_trailing_ones)] -let n = 0b1010111", stringify!($SelfT), "; - -assert_eq!(n.trailing_ones(), 3);", $EndFeature, " -```"), - #[unstable(feature = "leading_trailing_ones", issue = "57969")] - #[inline] - pub const fn trailing_ones(self) -> u32 { - (!self).trailing_zeros() - } - } - - doc_comment! { - concat!("Shifts the bits to the left by a specified amount, `n`, -wrapping the truncated bits to the end of the resulting integer. - -Please note this isn't the same operation as the `<<` shifting operator! - -# Examples - -Basic usage: - -``` -let n = ", $rot_op, stringify!($SelfT), "; -let m = ", $rot_result, "; - -assert_eq!(n.rotate_left(", $rot, "), m); -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_math", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn rotate_left(self, n: u32) -> Self { - intrinsics::rotate_left(self, n as $SelfT) - } - } - - doc_comment! { - concat!("Shifts the bits to the right by a specified amount, `n`, -wrapping the truncated bits to the beginning of the resulting -integer. - -Please note this isn't the same operation as the `>>` shifting operator! - -# Examples - -Basic usage: - -``` -let n = ", $rot_result, stringify!($SelfT), "; -let m = ", $rot_op, "; - -assert_eq!(n.rotate_right(", $rot, "), m); -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_math", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn rotate_right(self, n: u32) -> Self { - intrinsics::rotate_right(self, n as $SelfT) - } - } - - doc_comment! { - concat!(" -Reverses the byte order of the integer. - -# Examples - -Basic usage: - -``` -let n = ", $swap_op, stringify!($SelfT), "; -let m = n.swap_bytes(); - -assert_eq!(m, ", $swapped, "); -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_math", since = "1.32.0")] - #[inline] - pub const fn swap_bytes(self) -> Self { - intrinsics::bswap(self as $ActualT) as Self - } - } - - doc_comment! { - concat!("Reverses the bit pattern of the integer. - -# Examples - -Basic usage: - -``` -let n = ", $swap_op, stringify!($SelfT), "; -let m = n.reverse_bits(); - -assert_eq!(m, ", $reversed, "); -```"), - #[stable(feature = "reverse_bits", since = "1.37.0")] - #[rustc_const_stable(feature = "const_math", since = "1.32.0")] - #[inline] - #[must_use] - pub const fn reverse_bits(self) -> Self { - intrinsics::bitreverse(self as $ActualT) as Self - } - } - - doc_comment! { - concat!("Converts an integer from big endian to the target's endianness. - -On big endian this is a no-op. On little endian the bytes are -swapped. - -# Examples - -Basic usage: - -``` -", $Feature, "let n = 0x1A", stringify!($SelfT), "; - -if cfg!(target_endian = \"big\") { - assert_eq!(", stringify!($SelfT), "::from_be(n), n) -} else { - assert_eq!(", stringify!($SelfT), "::from_be(n), n.swap_bytes()) -}", $EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_math", since = "1.32.0")] - #[inline] - pub const fn from_be(x: Self) -> Self { - #[cfg(target_endian = "big")] - { - x - } - #[cfg(not(target_endian = "big"))] - { - x.swap_bytes() - } - } - } - - doc_comment! { - concat!("Converts an integer from little endian to the target's endianness. - -On little endian this is a no-op. On big endian the bytes are -swapped. - -# Examples - -Basic usage: - -``` -", $Feature, "let n = 0x1A", stringify!($SelfT), "; - -if cfg!(target_endian = \"little\") { - assert_eq!(", stringify!($SelfT), "::from_le(n), n) -} else { - assert_eq!(", stringify!($SelfT), "::from_le(n), n.swap_bytes()) -}", $EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_math", since = "1.32.0")] - #[inline] - pub const fn from_le(x: Self) -> Self { - #[cfg(target_endian = "little")] - { - x - } - #[cfg(not(target_endian = "little"))] - { - x.swap_bytes() - } - } - } - - doc_comment! { - concat!("Converts `self` to big endian from the target's endianness. - -On big endian this is a no-op. On little endian the bytes are -swapped. - -# Examples - -Basic usage: - -``` -", $Feature, "let n = 0x1A", stringify!($SelfT), "; - -if cfg!(target_endian = \"big\") { - assert_eq!(n.to_be(), n) -} else { - assert_eq!(n.to_be(), n.swap_bytes()) -}", $EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_math", since = "1.32.0")] - #[inline] - pub const fn to_be(self) -> Self { // or not to be? - #[cfg(target_endian = "big")] - { - self - } - #[cfg(not(target_endian = "big"))] - { - self.swap_bytes() - } - } - } - - doc_comment! { - concat!("Converts `self` to little endian from the target's endianness. - -On little endian this is a no-op. On big endian the bytes are -swapped. - -# Examples - -Basic usage: - -``` -", $Feature, "let n = 0x1A", stringify!($SelfT), "; - -if cfg!(target_endian = \"little\") { - assert_eq!(n.to_le(), n) -} else { - assert_eq!(n.to_le(), n.swap_bytes()) -}", $EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_math", since = "1.32.0")] - #[inline] - pub const fn to_le(self) -> Self { - #[cfg(target_endian = "little")] - { - self - } - #[cfg(not(target_endian = "little"))] - { - self.swap_bytes() - } - } - } - - doc_comment! { - concat!("Checked integer addition. Computes `self + rhs`, returning `None` -if overflow occurred. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!((", stringify!($SelfT), "::MAX - 2).checked_add(1), ", -"Some(", stringify!($SelfT), "::MAX - 1)); -assert_eq!((", stringify!($SelfT), "::MAX - 2).checked_add(3), None);", $EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_checked_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn checked_add(self, rhs: Self) -> Option { - let (a, b) = self.overflowing_add(rhs); - if b {None} else {Some(a)} - } - } - - doc_comment! { - concat!("Checked integer subtraction. Computes `self - rhs`, returning -`None` if overflow occurred. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(1", stringify!($SelfT), ".checked_sub(1), Some(0)); -assert_eq!(0", stringify!($SelfT), ".checked_sub(1), None);", $EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_checked_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn checked_sub(self, rhs: Self) -> Option { - let (a, b) = self.overflowing_sub(rhs); - if b {None} else {Some(a)} - } - } - - doc_comment! { - concat!("Checked integer multiplication. Computes `self * rhs`, returning -`None` if overflow occurred. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(5", stringify!($SelfT), ".checked_mul(1), Some(5)); -assert_eq!(", stringify!($SelfT), "::MAX.checked_mul(2), None);", $EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_checked_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn checked_mul(self, rhs: Self) -> Option { - let (a, b) = self.overflowing_mul(rhs); - if b {None} else {Some(a)} - } - } - - doc_comment! { - concat!("Checked integer division. Computes `self / rhs`, returning `None` -if `rhs == 0`. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(128", stringify!($SelfT), ".checked_div(2), Some(64)); -assert_eq!(1", stringify!($SelfT), ".checked_div(0), None);", $EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_checked_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn checked_div(self, rhs: Self) -> Option { - match rhs { - 0 => None, - // SAFETY: div by zero has been checked above and unsigned types have no other - // failure modes for division - rhs => Some(unsafe { intrinsics::unchecked_div(self, rhs) }), - } - } - } - - doc_comment! { - concat!("Checked Euclidean division. Computes `self.div_euclid(rhs)`, returning `None` -if `rhs == 0`. - -# Examples - -Basic usage: - -``` -assert_eq!(128", stringify!($SelfT), ".checked_div_euclid(2), Some(64)); -assert_eq!(1", stringify!($SelfT), ".checked_div_euclid(0), None); -```"), - #[stable(feature = "euclidean_division", since = "1.38.0")] - #[rustc_const_unstable(feature = "const_euclidean_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn checked_div_euclid(self, rhs: Self) -> Option { - if rhs == 0 { - None - } else { - Some(self.div_euclid(rhs)) - } - } - } - - - doc_comment! { - concat!("Checked integer remainder. Computes `self % rhs`, returning `None` -if `rhs == 0`. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(5", stringify!($SelfT), ".checked_rem(2), Some(1)); -assert_eq!(5", stringify!($SelfT), ".checked_rem(0), None);", $EndFeature, " -```"), - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_unstable(feature = "const_checked_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn checked_rem(self, rhs: Self) -> Option { - if rhs == 0 { - None - } else { - // SAFETY: div by zero has been checked above and unsigned types have no other - // failure modes for division - Some(unsafe { intrinsics::unchecked_rem(self, rhs) }) - } - } - } - - doc_comment! { - concat!("Checked Euclidean modulo. Computes `self.rem_euclid(rhs)`, returning `None` -if `rhs == 0`. - -# Examples - -Basic usage: - -``` -assert_eq!(5", stringify!($SelfT), ".checked_rem_euclid(2), Some(1)); -assert_eq!(5", stringify!($SelfT), ".checked_rem_euclid(0), None); -```"), - #[stable(feature = "euclidean_division", since = "1.38.0")] - #[rustc_const_unstable(feature = "const_euclidean_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn checked_rem_euclid(self, rhs: Self) -> Option { - if rhs == 0 { - None - } else { - Some(self.rem_euclid(rhs)) - } - } - } - - doc_comment! { - concat!("Checked negation. Computes `-self`, returning `None` unless `self == -0`. - -Note that negating any positive integer will overflow. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(0", stringify!($SelfT), ".checked_neg(), Some(0)); -assert_eq!(1", stringify!($SelfT), ".checked_neg(), None);", $EndFeature, " -```"), - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_unstable(feature = "const_checked_int_methods", issue = "53718")] - #[inline] - pub const fn checked_neg(self) -> Option { - let (a, b) = self.overflowing_neg(); - if b {None} else {Some(a)} - } - } - - doc_comment! { - concat!("Checked shift left. Computes `self << rhs`, returning `None` -if `rhs` is larger than or equal to the number of bits in `self`. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(0x1", stringify!($SelfT), ".checked_shl(4), Some(0x10)); -assert_eq!(0x10", stringify!($SelfT), ".checked_shl(129), None);", $EndFeature, " -```"), - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_unstable(feature = "const_checked_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn checked_shl(self, rhs: u32) -> Option { - let (a, b) = self.overflowing_shl(rhs); - if b {None} else {Some(a)} - } - } - - doc_comment! { - concat!("Checked shift right. Computes `self >> rhs`, returning `None` -if `rhs` is larger than or equal to the number of bits in `self`. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(0x10", stringify!($SelfT), ".checked_shr(4), Some(0x1)); -assert_eq!(0x10", stringify!($SelfT), ".checked_shr(129), None);", $EndFeature, " -```"), - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_unstable(feature = "const_checked_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn checked_shr(self, rhs: u32) -> Option { - let (a, b) = self.overflowing_shr(rhs); - if b {None} else {Some(a)} - } - } - - doc_comment! { - concat!("Checked exponentiation. Computes `self.pow(exp)`, returning `None` if -overflow occurred. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(2", stringify!($SelfT), ".checked_pow(5), Some(32)); -assert_eq!(", stringify!($SelfT), "::MAX.checked_pow(2), None);", $EndFeature, " -```"), - #[stable(feature = "no_panic_pow", since = "1.34.0")] - #[rustc_const_unstable(feature = "const_int_pow", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn checked_pow(self, mut exp: u32) -> Option { - let mut base = self; - let mut acc: Self = 1; - - while exp > 1 { - if (exp & 1) == 1 { - acc = try_opt!(acc.checked_mul(base)); - } - exp /= 2; - base = try_opt!(base.checked_mul(base)); - } - - // Deal with the final bit of the exponent separately, since - // squaring the base afterwards is not necessary and may cause a - // needless overflow. - if exp == 1 { - acc = try_opt!(acc.checked_mul(base)); - } - - Some(acc) - } - } - - doc_comment! { - concat!("Saturating integer addition. Computes `self + rhs`, saturating at -the numeric bounds instead of overflowing. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(100", stringify!($SelfT), ".saturating_add(1), 101); -assert_eq!(200u8.saturating_add(127), 255);", $EndFeature, " -```"), - - #[stable(feature = "rust1", since = "1.0.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[rustc_const_unstable(feature = "const_saturating_int_methods", issue = "53718")] - #[inline] - pub const fn saturating_add(self, rhs: Self) -> Self { - intrinsics::saturating_add(self, rhs) - } - } - - doc_comment! { - concat!("Saturating integer subtraction. Computes `self - rhs`, saturating -at the numeric bounds instead of overflowing. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(100", stringify!($SelfT), ".saturating_sub(27), 73); -assert_eq!(13", stringify!($SelfT), ".saturating_sub(127), 0);", $EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[rustc_const_unstable(feature = "const_saturating_int_methods", issue = "53718")] - #[inline] - pub const fn saturating_sub(self, rhs: Self) -> Self { - intrinsics::saturating_sub(self, rhs) - } - } - - doc_comment! { - concat!("Saturating integer multiplication. Computes `self * rhs`, -saturating at the numeric bounds instead of overflowing. - -# Examples - -Basic usage: - -``` -", $Feature, " -assert_eq!(2", stringify!($SelfT), ".saturating_mul(10), 20); -assert_eq!((", stringify!($SelfT), "::MAX).saturating_mul(10), ", stringify!($SelfT), -"::MAX);", $EndFeature, " -```"), - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_unstable(feature = "const_saturating_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn saturating_mul(self, rhs: Self) -> Self { - match self.checked_mul(rhs) { - Some(x) => x, - None => Self::max_value(), - } - } - } - - doc_comment! { - concat!("Saturating integer exponentiation. Computes `self.pow(exp)`, -saturating at the numeric bounds instead of overflowing. - -# Examples - -Basic usage: - -``` -", $Feature, " -assert_eq!(4", stringify!($SelfT), ".saturating_pow(3), 64); -assert_eq!(", stringify!($SelfT), "::MAX.saturating_pow(2), ", stringify!($SelfT), "::MAX);", -$EndFeature, " -```"), - #[stable(feature = "no_panic_pow", since = "1.34.0")] - #[rustc_const_unstable(feature = "const_int_pow", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn saturating_pow(self, exp: u32) -> Self { - match self.checked_pow(exp) { - Some(x) => x, - None => Self::max_value(), - } - } - } - - doc_comment! { - concat!("Wrapping (modular) addition. Computes `self + rhs`, -wrapping around at the boundary of the type. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(200", stringify!($SelfT), ".wrapping_add(55), 255); -assert_eq!(200", stringify!($SelfT), ".wrapping_add(", stringify!($SelfT), "::MAX), 199);", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn wrapping_add(self, rhs: Self) -> Self { - intrinsics::wrapping_add(self, rhs) - } - } - - doc_comment! { - concat!("Wrapping (modular) subtraction. Computes `self - rhs`, -wrapping around at the boundary of the type. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(100", stringify!($SelfT), ".wrapping_sub(100), 0); -assert_eq!(100", stringify!($SelfT), ".wrapping_sub(", stringify!($SelfT), "::MAX), 101);", -$EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn wrapping_sub(self, rhs: Self) -> Self { - intrinsics::wrapping_sub(self, rhs) - } - } - - /// Wrapping (modular) multiplication. Computes `self * - /// rhs`, wrapping around at the boundary of the type. - /// - /// # Examples - /// - /// Basic usage: - /// - /// Please note that this example is shared between integer types. - /// Which explains why `u8` is used here. - /// - /// ``` - /// assert_eq!(10u8.wrapping_mul(12), 120); - /// assert_eq!(25u8.wrapping_mul(12), 44); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn wrapping_mul(self, rhs: Self) -> Self { - intrinsics::wrapping_mul(self, rhs) - } - - doc_comment! { - concat!("Wrapping (modular) division. Computes `self / rhs`. -Wrapped division on unsigned types is just normal division. -There's no way wrapping could ever happen. -This function exists, so that all operations -are accounted for in the wrapping operations. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(100", stringify!($SelfT), ".wrapping_div(10), 10);", $EndFeature, " -```"), - #[stable(feature = "num_wrapping", since = "1.2.0")] - #[rustc_const_unstable(feature = "const_wrapping_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn wrapping_div(self, rhs: Self) -> Self { - self / rhs - } - } - - doc_comment! { - concat!("Wrapping Euclidean division. Computes `self.div_euclid(rhs)`. -Wrapped division on unsigned types is just normal division. -There's no way wrapping could ever happen. -This function exists, so that all operations -are accounted for in the wrapping operations. -Since, for the positive integers, all common -definitions of division are equal, this -is exactly equal to `self.wrapping_div(rhs)`. - -# Examples - -Basic usage: - -``` -assert_eq!(100", stringify!($SelfT), ".wrapping_div_euclid(10), 10); -```"), - #[stable(feature = "euclidean_division", since = "1.38.0")] - #[rustc_const_unstable(feature = "const_euclidean_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn wrapping_div_euclid(self, rhs: Self) -> Self { - self / rhs - } - } - - doc_comment! { - concat!("Wrapping (modular) remainder. Computes `self % rhs`. -Wrapped remainder calculation on unsigned types is -just the regular remainder calculation. -There's no way wrapping could ever happen. -This function exists, so that all operations -are accounted for in the wrapping operations. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(100", stringify!($SelfT), ".wrapping_rem(10), 0);", $EndFeature, " -```"), - #[stable(feature = "num_wrapping", since = "1.2.0")] - #[rustc_const_unstable(feature = "const_wrapping_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn wrapping_rem(self, rhs: Self) -> Self { - self % rhs - } - } - - doc_comment! { - concat!("Wrapping Euclidean modulo. Computes `self.rem_euclid(rhs)`. -Wrapped modulo calculation on unsigned types is -just the regular remainder calculation. -There's no way wrapping could ever happen. -This function exists, so that all operations -are accounted for in the wrapping operations. -Since, for the positive integers, all common -definitions of division are equal, this -is exactly equal to `self.wrapping_rem(rhs)`. - -# Examples - -Basic usage: - -``` -assert_eq!(100", stringify!($SelfT), ".wrapping_rem_euclid(10), 0); -```"), - #[stable(feature = "euclidean_division", since = "1.38.0")] - #[rustc_const_unstable(feature = "const_euclidean_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn wrapping_rem_euclid(self, rhs: Self) -> Self { - self % rhs - } - } - - /// Wrapping (modular) negation. Computes `-self`, - /// wrapping around at the boundary of the type. - /// - /// Since unsigned types do not have negative equivalents - /// all applications of this function will wrap (except for `-0`). - /// For values smaller than the corresponding signed type's maximum - /// the result is the same as casting the corresponding signed value. - /// Any larger values are equivalent to `MAX + 1 - (val - MAX - 1)` where - /// `MAX` is the corresponding signed type's maximum. - /// - /// # Examples - /// - /// Basic usage: - /// - /// Please note that this example is shared between integer types. - /// Which explains why `i8` is used here. - /// - /// ``` - /// assert_eq!(100i8.wrapping_neg(), -100); - /// assert_eq!((-128i8).wrapping_neg(), -128); - /// ``` - #[stable(feature = "num_wrapping", since = "1.2.0")] - #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] - #[inline] - pub const fn wrapping_neg(self) -> Self { - self.overflowing_neg().0 - } - - doc_comment! { - concat!("Panic-free bitwise shift-left; yields `self << mask(rhs)`, -where `mask` removes any high-order bits of `rhs` that -would cause the shift to exceed the bitwidth of the type. - -Note that this is *not* the same as a rotate-left; the -RHS of a wrapping shift-left is restricted to the range -of the type, rather than the bits shifted out of the LHS -being returned to the other end. The primitive integer -types all implement a `rotate_left` function, which may -be what you want instead. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(1", stringify!($SelfT), ".wrapping_shl(7), 128); -assert_eq!(1", stringify!($SelfT), ".wrapping_shl(128), 1);", $EndFeature, " -```"), - #[stable(feature = "num_wrapping", since = "1.2.0")] - #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn wrapping_shl(self, rhs: u32) -> Self { - // SAFETY: the masking by the bitsize of the type ensures that we do not shift - // out of bounds - unsafe { - intrinsics::unchecked_shl(self, (rhs & ($BITS - 1)) as $SelfT) - } - } - } - - doc_comment! { - concat!("Panic-free bitwise shift-right; yields `self >> mask(rhs)`, -where `mask` removes any high-order bits of `rhs` that -would cause the shift to exceed the bitwidth of the type. - -Note that this is *not* the same as a rotate-right; the -RHS of a wrapping shift-right is restricted to the range -of the type, rather than the bits shifted out of the LHS -being returned to the other end. The primitive integer -types all implement a `rotate_right` function, which may -be what you want instead. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(128", stringify!($SelfT), ".wrapping_shr(7), 1); -assert_eq!(128", stringify!($SelfT), ".wrapping_shr(128), 128);", $EndFeature, " -```"), - #[stable(feature = "num_wrapping", since = "1.2.0")] - #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn wrapping_shr(self, rhs: u32) -> Self { - // SAFETY: the masking by the bitsize of the type ensures that we do not shift - // out of bounds - unsafe { - intrinsics::unchecked_shr(self, (rhs & ($BITS - 1)) as $SelfT) - } - } - } - - doc_comment! { - concat!("Wrapping (modular) exponentiation. Computes `self.pow(exp)`, -wrapping around at the boundary of the type. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(3", stringify!($SelfT), ".wrapping_pow(5), 243); -assert_eq!(3u8.wrapping_pow(6), 217);", $EndFeature, " -```"), - #[stable(feature = "no_panic_pow", since = "1.34.0")] - #[rustc_const_unstable(feature = "const_int_pow", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn wrapping_pow(self, mut exp: u32) -> Self { - let mut base = self; - let mut acc: Self = 1; - - while exp > 1 { - if (exp & 1) == 1 { - acc = acc.wrapping_mul(base); - } - exp /= 2; - base = base.wrapping_mul(base); - } - - // Deal with the final bit of the exponent separately, since - // squaring the base afterwards is not necessary and may cause a - // needless overflow. - if exp == 1 { - acc = acc.wrapping_mul(base); - } - - acc - } - } - - doc_comment! { - concat!("Calculates `self` + `rhs` - -Returns a tuple of the addition along with a boolean indicating -whether an arithmetic overflow would occur. If an overflow would -have occurred then the wrapped value is returned. - -# Examples - -Basic usage - -``` -", $Feature, " -assert_eq!(5", stringify!($SelfT), ".overflowing_add(2), (7, false)); -assert_eq!(", stringify!($SelfT), "::MAX.overflowing_add(1), (0, true));", $EndFeature, " -```"), - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn overflowing_add(self, rhs: Self) -> (Self, bool) { - let (a, b) = intrinsics::add_with_overflow(self as $ActualT, rhs as $ActualT); - (a as Self, b) - } - } - - doc_comment! { - concat!("Calculates `self` - `rhs` - -Returns a tuple of the subtraction along with a boolean indicating -whether an arithmetic overflow would occur. If an overflow would -have occurred then the wrapped value is returned. - -# Examples - -Basic usage - -``` -", $Feature, " -assert_eq!(5", stringify!($SelfT), ".overflowing_sub(2), (3, false)); -assert_eq!(0", stringify!($SelfT), ".overflowing_sub(1), (", stringify!($SelfT), "::MAX, true));", -$EndFeature, " -```"), - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn overflowing_sub(self, rhs: Self) -> (Self, bool) { - let (a, b) = intrinsics::sub_with_overflow(self as $ActualT, rhs as $ActualT); - (a as Self, b) - } - } - - /// Calculates the multiplication of `self` and `rhs`. - /// - /// Returns a tuple of the multiplication along with a boolean - /// indicating whether an arithmetic overflow would occur. If an - /// overflow would have occurred then the wrapped value is returned. - /// - /// # Examples - /// - /// Basic usage: - /// - /// Please note that this example is shared between integer types. - /// Which explains why `u32` is used here. - /// - /// ``` - /// assert_eq!(5u32.overflowing_mul(2), (10, false)); - /// assert_eq!(1_000_000_000u32.overflowing_mul(10), (1410065408, true)); - /// ``` - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn overflowing_mul(self, rhs: Self) -> (Self, bool) { - let (a, b) = intrinsics::mul_with_overflow(self as $ActualT, rhs as $ActualT); - (a as Self, b) - } - - doc_comment! { - concat!("Calculates the divisor when `self` is divided by `rhs`. - -Returns a tuple of the divisor along with a boolean indicating -whether an arithmetic overflow would occur. Note that for unsigned -integers overflow never occurs, so the second value is always -`false`. - -# Panics - -This function will panic if `rhs` is 0. - -# Examples - -Basic usage - -``` -", $Feature, "assert_eq!(5", stringify!($SelfT), ".overflowing_div(2), (2, false));", $EndFeature, " -```"), - #[inline] - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_unstable(feature = "const_overflowing_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - pub const fn overflowing_div(self, rhs: Self) -> (Self, bool) { - (self / rhs, false) - } - } - - doc_comment! { - concat!("Calculates the quotient of Euclidean division `self.div_euclid(rhs)`. - -Returns a tuple of the divisor along with a boolean indicating -whether an arithmetic overflow would occur. Note that for unsigned -integers overflow never occurs, so the second value is always -`false`. -Since, for the positive integers, all common -definitions of division are equal, this -is exactly equal to `self.overflowing_div(rhs)`. - -# Panics - -This function will panic if `rhs` is 0. - -# Examples - -Basic usage - -``` -assert_eq!(5", stringify!($SelfT), ".overflowing_div_euclid(2), (2, false)); -```"), - #[inline] - #[stable(feature = "euclidean_division", since = "1.38.0")] - #[rustc_const_unstable(feature = "const_euclidean_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - pub const fn overflowing_div_euclid(self, rhs: Self) -> (Self, bool) { - (self / rhs, false) - } - } - - doc_comment! { - concat!("Calculates the remainder when `self` is divided by `rhs`. - -Returns a tuple of the remainder after dividing along with a boolean -indicating whether an arithmetic overflow would occur. Note that for -unsigned integers overflow never occurs, so the second value is -always `false`. - -# Panics - -This function will panic if `rhs` is 0. - -# Examples - -Basic usage - -``` -", $Feature, "assert_eq!(5", stringify!($SelfT), ".overflowing_rem(2), (1, false));", $EndFeature, " -```"), - #[inline] - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_unstable(feature = "const_overflowing_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - pub const fn overflowing_rem(self, rhs: Self) -> (Self, bool) { - (self % rhs, false) - } - } - - doc_comment! { - concat!("Calculates the remainder `self.rem_euclid(rhs)` as if by Euclidean division. - -Returns a tuple of the modulo after dividing along with a boolean -indicating whether an arithmetic overflow would occur. Note that for -unsigned integers overflow never occurs, so the second value is -always `false`. -Since, for the positive integers, all common -definitions of division are equal, this operation -is exactly equal to `self.overflowing_rem(rhs)`. - -# Panics - -This function will panic if `rhs` is 0. - -# Examples - -Basic usage - -``` -assert_eq!(5", stringify!($SelfT), ".overflowing_rem_euclid(2), (1, false)); -```"), - #[inline] - #[stable(feature = "euclidean_division", since = "1.38.0")] - #[rustc_const_unstable(feature = "const_euclidean_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - pub const fn overflowing_rem_euclid(self, rhs: Self) -> (Self, bool) { - (self % rhs, false) - } - } - - doc_comment! { - concat!("Negates self in an overflowing fashion. - -Returns `!self + 1` using wrapping operations to return the value -that represents the negation of this unsigned value. Note that for -positive unsigned values overflow always occurs, but negating 0 does -not overflow. - -# Examples - -Basic usage - -``` -", $Feature, "assert_eq!(0", stringify!($SelfT), ".overflowing_neg(), (0, false)); -assert_eq!(2", stringify!($SelfT), ".overflowing_neg(), (-2i32 as ", stringify!($SelfT), -", true));", $EndFeature, " -```"), - #[inline] - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] - pub const fn overflowing_neg(self) -> (Self, bool) { - ((!self).wrapping_add(1), self != 0) - } - } - - doc_comment! { - concat!("Shifts self left by `rhs` bits. - -Returns a tuple of the shifted version of self along with a boolean -indicating whether the shift value was larger than or equal to the -number of bits. If the shift value is too large, then value is -masked (N-1) where N is the number of bits, and this value is then -used to perform the shift. - -# Examples - -Basic usage - -``` -", $Feature, "assert_eq!(0x1", stringify!($SelfT), ".overflowing_shl(4), (0x10, false)); -assert_eq!(0x1", stringify!($SelfT), ".overflowing_shl(132), (0x10, true));", $EndFeature, " -```"), - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn overflowing_shl(self, rhs: u32) -> (Self, bool) { - (self.wrapping_shl(rhs), (rhs > ($BITS - 1))) - } - } - - doc_comment! { - concat!("Shifts self right by `rhs` bits. - -Returns a tuple of the shifted version of self along with a boolean -indicating whether the shift value was larger than or equal to the -number of bits. If the shift value is too large, then value is -masked (N-1) where N is the number of bits, and this value is then -used to perform the shift. - -# Examples - -Basic usage - -``` -", $Feature, "assert_eq!(0x10", stringify!($SelfT), ".overflowing_shr(4), (0x1, false)); -assert_eq!(0x10", stringify!($SelfT), ".overflowing_shr(132), (0x1, true));", $EndFeature, " -```"), - #[stable(feature = "wrapping", since = "1.7.0")] - #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn overflowing_shr(self, rhs: u32) -> (Self, bool) { - (self.wrapping_shr(rhs), (rhs > ($BITS - 1))) - } - } - - doc_comment! { - concat!("Raises self to the power of `exp`, using exponentiation by squaring. - -Returns a tuple of the exponentiation along with a bool indicating -whether an overflow happened. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(3", stringify!($SelfT), ".overflowing_pow(5), (243, false)); -assert_eq!(3u8.overflowing_pow(6), (217, true));", $EndFeature, " -```"), - #[stable(feature = "no_panic_pow", since = "1.34.0")] - #[rustc_const_unstable(feature = "const_int_pow", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - pub const fn overflowing_pow(self, mut exp: u32) -> (Self, bool) { - let mut base = self; - let mut acc: Self = 1; - let mut overflown = false; - // Scratch space for storing results of overflowing_mul. - let mut r; - - while exp > 1 { - if (exp & 1) == 1 { - r = acc.overflowing_mul(base); - acc = r.0; - overflown |= r.1; - } - exp /= 2; - r = base.overflowing_mul(base); - base = r.0; - overflown |= r.1; - } - - // Deal with the final bit of the exponent separately, since - // squaring the base afterwards is not necessary and may cause a - // needless overflow. - if exp == 1 { - r = acc.overflowing_mul(base); - acc = r.0; - overflown |= r.1; - } - - (acc, overflown) - } - } - - doc_comment! { - concat!("Raises self to the power of `exp`, using exponentiation by squaring. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(2", stringify!($SelfT), ".pow(5), 32);", $EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_int_pow", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - #[rustc_inherit_overflow_checks] - pub const fn pow(self, mut exp: u32) -> Self { - let mut base = self; - let mut acc = 1; - - while exp > 1 { - if (exp & 1) == 1 { - acc = acc * base; - } - exp /= 2; - base = base * base; - } - - // Deal with the final bit of the exponent separately, since - // squaring the base afterwards is not necessary and may cause a - // needless overflow. - if exp == 1 { - acc = acc * base; - } - - acc - } - } - - doc_comment! { - concat!("Performs Euclidean division. - -Since, for the positive integers, all common -definitions of division are equal, this -is exactly equal to `self / rhs`. - -# Panics - -This function will panic if `rhs` is 0. - -# Examples - -Basic usage: - -``` -assert_eq!(7", stringify!($SelfT), ".div_euclid(4), 1); // or any other integer type -```"), - #[stable(feature = "euclidean_division", since = "1.38.0")] - #[rustc_const_unstable(feature = "const_euclidean_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - #[rustc_inherit_overflow_checks] - pub const fn div_euclid(self, rhs: Self) -> Self { - self / rhs - } - } - - - doc_comment! { - concat!("Calculates the least remainder of `self (mod rhs)`. - -Since, for the positive integers, all common -definitions of division are equal, this -is exactly equal to `self % rhs`. - -# Panics - -This function will panic if `rhs` is 0. - -# Examples - -Basic usage: - -``` -assert_eq!(7", stringify!($SelfT), ".rem_euclid(4), 3); // or any other integer type -```"), - #[stable(feature = "euclidean_division", since = "1.38.0")] - #[rustc_const_unstable(feature = "const_euclidean_int_methods", issue = "53718")] - #[must_use = "this returns the result of the operation, \ - without modifying the original"] - #[inline] - #[rustc_inherit_overflow_checks] - pub const fn rem_euclid(self, rhs: Self) -> Self { - self % rhs - } - } - - doc_comment! { - concat!("Returns `true` if and only if `self == 2^k` for some `k`. - -# Examples - -Basic usage: - -``` -", $Feature, "assert!(16", stringify!($SelfT), ".is_power_of_two()); -assert!(!10", stringify!($SelfT), ".is_power_of_two());", $EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_is_power_of_two", since = "1.32.0")] - #[inline] - #[allow_internal_unstable(const_if_match)] - pub const fn is_power_of_two(self) -> bool { - self > 0 && (self - 1) & self == 0 - } - } - - // Returns one less than next power of two. - // (For 8u8 next power of two is 8u8 and for 6u8 it is 8u8) - // - // 8u8.one_less_than_next_power_of_two() == 7 - // 6u8.one_less_than_next_power_of_two() == 7 - // - // This method cannot overflow, as in the `next_power_of_two` - // overflow cases it instead ends up returning the maximum value - // of the type, and can return 0 for 0. - #[inline] - #[rustc_const_unstable(feature = "const_int_pow", issue = "53718")] - const fn one_less_than_next_power_of_two(self) -> Self { - if self <= 1 { return 0; } - - let p = self - 1; - // SAFETY: Because `p > 0`, it cannot consist entirely of leading zeros. - // That means the shift is always in-bounds, and some processors - // (such as intel pre-haswell) have more efficient ctlz - // intrinsics when the argument is non-zero. - let z = unsafe { intrinsics::ctlz_nonzero(p) }; - <$SelfT>::max_value() >> z - } - - doc_comment! { - concat!("Returns the smallest power of two greater than or equal to `self`. - -When return value overflows (i.e., `self > (1 << (N-1))` for type -`uN`), it panics in debug mode and return value is wrapped to 0 in -release mode (the only situation in which method can return 0). - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(2", stringify!($SelfT), ".next_power_of_two(), 2); -assert_eq!(3", stringify!($SelfT), ".next_power_of_two(), 4);", $EndFeature, " -```"), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_int_pow", issue = "53718")] - #[inline] - #[rustc_inherit_overflow_checks] - pub const fn next_power_of_two(self) -> Self { - self.one_less_than_next_power_of_two() + 1 - } - } - - doc_comment! { - concat!("Returns the smallest power of two greater than or equal to `n`. If -the next power of two is greater than the type's maximum value, -`None` is returned, otherwise the power of two is wrapped in `Some`. - -# Examples - -Basic usage: - -``` -", $Feature, "assert_eq!(2", stringify!($SelfT), -".checked_next_power_of_two(), Some(2)); -assert_eq!(3", stringify!($SelfT), ".checked_next_power_of_two(), Some(4)); -assert_eq!(", stringify!($SelfT), "::MAX.checked_next_power_of_two(), None);", -$EndFeature, " -```"), - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_int_pow", issue = "53718")] - pub const fn checked_next_power_of_two(self) -> Option { - self.one_less_than_next_power_of_two().checked_add(1) - } - } - - doc_comment! { - concat!("Returns the smallest power of two greater than or equal to `n`. If -the next power of two is greater than the type's maximum value, -the return value is wrapped to `0`. - -# Examples - -Basic usage: - -``` -#![feature(wrapping_next_power_of_two)] -", $Feature, " -assert_eq!(2", stringify!($SelfT), ".wrapping_next_power_of_two(), 2); -assert_eq!(3", stringify!($SelfT), ".wrapping_next_power_of_two(), 4); -assert_eq!(", stringify!($SelfT), "::MAX.wrapping_next_power_of_two(), 0);", -$EndFeature, " -```"), - #[unstable(feature = "wrapping_next_power_of_two", issue = "32463", - reason = "needs decision on wrapping behaviour")] - #[rustc_const_unstable(feature = "const_int_pow", issue = "53718")] - pub const fn wrapping_next_power_of_two(self) -> Self { - self.one_less_than_next_power_of_two().wrapping_add(1) - } - } - - doc_comment! { - concat!("Return the memory representation of this integer as a byte array in -big-endian (network) byte order. -", -$to_xe_bytes_doc, -" -# Examples - -``` -let bytes = ", $swap_op, stringify!($SelfT), ".to_be_bytes(); -assert_eq!(bytes, ", $be_bytes, "); -```"), - #[stable(feature = "int_to_from_bytes", since = "1.32.0")] - #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] - #[inline] - pub const fn to_be_bytes(self) -> [u8; mem::size_of::()] { - let mut buf = [0; $BITS / 8]; - buf[15 % ($BITS / 8)] = (self >> ( 0 % $BITS)) as u8; - buf[14 % ($BITS / 8)] = (self >> ( 8 % $BITS)) as u8; - buf[13 % ($BITS / 8)] = (self >> ( 16 % $BITS)) as u8; - buf[12 % ($BITS / 8)] = (self >> ( 24 % $BITS)) as u8; - buf[11 % ($BITS / 8)] = (self >> ( 32 % $BITS)) as u8; - buf[10 % ($BITS / 8)] = (self >> ( 40 % $BITS)) as u8; - buf[ 9 % ($BITS / 8)] = (self >> ( 48 % $BITS)) as u8; - buf[ 8 % ($BITS / 8)] = (self >> ( 56 % $BITS)) as u8; - buf[ 7 % ($BITS / 8)] = (self >> ( 64 % $BITS)) as u8; - buf[ 6 % ($BITS / 8)] = (self >> ( 72 % $BITS)) as u8; - buf[ 5 % ($BITS / 8)] = (self >> ( 80 % $BITS)) as u8; - buf[ 4 % ($BITS / 8)] = (self >> ( 88 % $BITS)) as u8; - buf[ 3 % ($BITS / 8)] = (self >> ( 96 % $BITS)) as u8; - buf[ 2 % ($BITS / 8)] = (self >> (104 % $BITS)) as u8; - buf[ 1 % ($BITS / 8)] = (self >> (112 % $BITS)) as u8; - buf[ 0 % ($BITS / 8)] = (self >> (120 % $BITS)) as u8; - buf - } - } - - doc_comment! { - concat!("Return the memory representation of this integer as a byte array in -little-endian byte order. -", -$to_xe_bytes_doc, -" -# Examples - -``` -let bytes = ", $swap_op, stringify!($SelfT), ".to_le_bytes(); -assert_eq!(bytes, ", $le_bytes, "); -```"), - #[stable(feature = "int_to_from_bytes", since = "1.32.0")] - #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] - #[inline] - pub const fn to_le_bytes(self) -> [u8; mem::size_of::()] { - let mut buf = [0; $BITS / 8]; - buf[ 0 % ($BITS / 8)] = (self >> ( 0 % $BITS)) as u8; - buf[ 1 % ($BITS / 8)] = (self >> ( 8 % $BITS)) as u8; - buf[ 2 % ($BITS / 8)] = (self >> ( 16 % $BITS)) as u8; - buf[ 3 % ($BITS / 8)] = (self >> ( 24 % $BITS)) as u8; - buf[ 4 % ($BITS / 8)] = (self >> ( 32 % $BITS)) as u8; - buf[ 5 % ($BITS / 8)] = (self >> ( 40 % $BITS)) as u8; - buf[ 6 % ($BITS / 8)] = (self >> ( 48 % $BITS)) as u8; - buf[ 7 % ($BITS / 8)] = (self >> ( 56 % $BITS)) as u8; - buf[ 8 % ($BITS / 8)] = (self >> ( 64 % $BITS)) as u8; - buf[ 9 % ($BITS / 8)] = (self >> ( 72 % $BITS)) as u8; - buf[10 % ($BITS / 8)] = (self >> ( 80 % $BITS)) as u8; - buf[11 % ($BITS / 8)] = (self >> ( 88 % $BITS)) as u8; - buf[12 % ($BITS / 8)] = (self >> ( 96 % $BITS)) as u8; - buf[13 % ($BITS / 8)] = (self >> (104 % $BITS)) as u8; - buf[14 % ($BITS / 8)] = (self >> (112 % $BITS)) as u8; - buf[15 % ($BITS / 8)] = (self >> (120 % $BITS)) as u8; - buf - } - } +} - doc_comment! { - concat!(" -Return the memory representation of this integer as a byte array in -native byte order. +// All these modules are technically private and only exposed for coretests: +#[cfg(not(no_fp_fmt_parse))] +pub mod bignum; +#[cfg(not(no_fp_fmt_parse))] +pub mod dec2flt; +#[cfg(not(no_fp_fmt_parse))] +pub mod diy_float; +#[cfg(not(no_fp_fmt_parse))] +pub mod flt2dec; +pub mod fmt; + +#[macro_use] +mod int_macros; // import int_impl! +#[macro_use] +mod uint_macros; // import uint_impl! + +mod error; +mod int_log10; +mod nonzero; +#[unstable(feature = "saturating_int_impl", issue = "87920")] +mod saturating; +mod wrapping; -As the target platform's native endianness is used, portable code -should use [`to_be_bytes`] or [`to_le_bytes`], as appropriate, -instead. -", -$to_xe_bytes_doc, -" -[`to_be_bytes`]: #method.to_be_bytes -[`to_le_bytes`]: #method.to_le_bytes +#[unstable(feature = "saturating_int_impl", issue = "87920")] +pub use saturating::Saturating; +#[stable(feature = "rust1", since = "1.0.0")] +pub use wrapping::Wrapping; -# Examples +#[stable(feature = "rust1", since = "1.0.0")] +#[cfg(not(no_fp_fmt_parse))] +pub use dec2flt::ParseFloatError; -``` -let bytes = ", $swap_op, stringify!($SelfT), ".to_ne_bytes(); -assert_eq!( - bytes, - if cfg!(target_endian = \"big\") { - ", $be_bytes, " - } else { - ", $le_bytes, " +#[cfg(not(no_fp_fmt_parse))] +#[stable(feature = "rust1", since = "1.0.0")] +impl Error for ParseFloatError { + #[allow(deprecated)] + fn description(&self) -> &str { + self.__description() } -); -```"), - #[stable(feature = "int_to_from_bytes", since = "1.32.0")] - #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] - // SAFETY: const sound because integers are plain old datatypes so we can always - // transmute them to arrays of bytes - #[allow_internal_unstable(const_fn_union)] - #[inline] - pub const fn to_ne_bytes(self) -> [u8; mem::size_of::()] { - #[repr(C)] - union Bytes { - val: $SelfT, - bytes: [u8; mem::size_of::<$SelfT>()], - } - // SAFETY: integers are plain old datatypes so we can always transmute them to - // arrays of bytes - unsafe { Bytes { val: self }.bytes } - } - } - - doc_comment! { - concat!("Create an integer value from its representation as a byte array in -big endian. -", -$from_xe_bytes_doc, -" -# Examples - -``` -let value = ", stringify!($SelfT), "::from_be_bytes(", $be_bytes, "); -assert_eq!(value, ", $swap_op, "); -``` +} -When starting from a slice rather than an array, fallible conversion APIs can be used: +#[stable(feature = "rust1", since = "1.0.0")] +pub use error::ParseIntError; -``` -use std::convert::TryInto; +#[stable(feature = "nonzero", since = "1.28.0")] +pub use nonzero::{NonZeroU128, NonZeroU16, NonZeroU32, NonZeroU64, NonZeroU8, NonZeroUsize}; -fn read_be_", stringify!($SelfT), "(input: &mut &[u8]) -> ", stringify!($SelfT), " { - let (int_bytes, rest) = input.split_at(std::mem::size_of::<", stringify!($SelfT), ">()); - *input = rest; - ", stringify!($SelfT), "::from_be_bytes(int_bytes.try_into().unwrap()) -} -```"), - #[stable(feature = "int_to_from_bytes", since = "1.32.0")] - #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] - #[inline] - pub const fn from_be_bytes(bytes: [u8; mem::size_of::()]) -> Self { - let mut x = 0; - x |= (bytes[15 % ($BITS / 8)] as Self) << ( 0 % $BITS); - x |= (bytes[14 % ($BITS / 8)] as Self) << ( 8 % $BITS); - x |= (bytes[13 % ($BITS / 8)] as Self) << ( 16 % $BITS); - x |= (bytes[12 % ($BITS / 8)] as Self) << ( 24 % $BITS); - x |= (bytes[11 % ($BITS / 8)] as Self) << ( 32 % $BITS); - x |= (bytes[10 % ($BITS / 8)] as Self) << ( 40 % $BITS); - x |= (bytes[ 9 % ($BITS / 8)] as Self) << ( 48 % $BITS); - x |= (bytes[ 8 % ($BITS / 8)] as Self) << ( 56 % $BITS); - x |= (bytes[ 7 % ($BITS / 8)] as Self) << ( 64 % $BITS); - x |= (bytes[ 6 % ($BITS / 8)] as Self) << ( 72 % $BITS); - x |= (bytes[ 5 % ($BITS / 8)] as Self) << ( 80 % $BITS); - x |= (bytes[ 4 % ($BITS / 8)] as Self) << ( 88 % $BITS); - x |= (bytes[ 3 % ($BITS / 8)] as Self) << ( 96 % $BITS); - x |= (bytes[ 2 % ($BITS / 8)] as Self) << (104 % $BITS); - x |= (bytes[ 1 % ($BITS / 8)] as Self) << (112 % $BITS); - x |= (bytes[ 0 % ($BITS / 8)] as Self) << (120 % $BITS); - x - } - } +#[stable(feature = "signed_nonzero", since = "1.34.0")] +pub use nonzero::{NonZeroI128, NonZeroI16, NonZeroI32, NonZeroI64, NonZeroI8, NonZeroIsize}; - doc_comment! { - concat!(" -Create an integer value from its representation as a byte array in -little endian. -", -$from_xe_bytes_doc, -" -# Examples +#[stable(feature = "try_from", since = "1.34.0")] +pub use error::TryFromIntError; -``` -let value = ", stringify!($SelfT), "::from_le_bytes(", $le_bytes, "); -assert_eq!(value, ", $swap_op, "); -``` +#[stable(feature = "int_error_matching", since = "1.55.0")] +pub use error::IntErrorKind; -When starting from a slice rather than an array, fallible conversion APIs can be used: +macro_rules! usize_isize_to_xe_bytes_doc { + () => { + " -``` -use std::convert::TryInto; +**Note**: This function returns an array of length 2, 4 or 8 bytes +depending on the target pointer size. -fn read_le_", stringify!($SelfT), "(input: &mut &[u8]) -> ", stringify!($SelfT), " { - let (int_bytes, rest) = input.split_at(std::mem::size_of::<", stringify!($SelfT), ">()); - *input = rest; - ", stringify!($SelfT), "::from_le_bytes(int_bytes.try_into().unwrap()) +" + }; } -```"), - #[stable(feature = "int_to_from_bytes", since = "1.32.0")] - #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] - #[inline] - pub const fn from_le_bytes(bytes: [u8; mem::size_of::()]) -> Self { - let mut x = 0; - x |= (bytes[ 0 % ($BITS / 8)] as Self) << ( 0 % $BITS); - x |= (bytes[ 1 % ($BITS / 8)] as Self) << ( 8 % $BITS); - x |= (bytes[ 2 % ($BITS / 8)] as Self) << ( 16 % $BITS); - x |= (bytes[ 3 % ($BITS / 8)] as Self) << ( 24 % $BITS); - x |= (bytes[ 4 % ($BITS / 8)] as Self) << ( 32 % $BITS); - x |= (bytes[ 5 % ($BITS / 8)] as Self) << ( 40 % $BITS); - x |= (bytes[ 6 % ($BITS / 8)] as Self) << ( 48 % $BITS); - x |= (bytes[ 7 % ($BITS / 8)] as Self) << ( 56 % $BITS); - x |= (bytes[ 8 % ($BITS / 8)] as Self) << ( 64 % $BITS); - x |= (bytes[ 9 % ($BITS / 8)] as Self) << ( 72 % $BITS); - x |= (bytes[10 % ($BITS / 8)] as Self) << ( 80 % $BITS); - x |= (bytes[11 % ($BITS / 8)] as Self) << ( 88 % $BITS); - x |= (bytes[12 % ($BITS / 8)] as Self) << ( 96 % $BITS); - x |= (bytes[13 % ($BITS / 8)] as Self) << (104 % $BITS); - x |= (bytes[14 % ($BITS / 8)] as Self) << (112 % $BITS); - x |= (bytes[15 % ($BITS / 8)] as Self) << (120 % $BITS); - x - } - } - doc_comment! { - concat!("Create an integer value from its memory representation as a byte -array in native endianness. +macro_rules! usize_isize_from_xe_bytes_doc { + () => { + " -As the target platform's native endianness is used, portable code -likely wants to use [`from_be_bytes`] or [`from_le_bytes`], as -appropriate instead. +**Note**: This function takes an array of length 2, 4 or 8 bytes +depending on the target pointer size. -[`from_be_bytes`]: #method.from_be_bytes -[`from_le_bytes`]: #method.from_le_bytes -", -$from_xe_bytes_doc, " -# Examples + }; +} -``` -let value = ", stringify!($SelfT), "::from_ne_bytes(if cfg!(target_endian = \"big\") { - ", $be_bytes, " -} else { - ", $le_bytes, " -}); -assert_eq!(value, ", $swap_op, "); -``` +macro_rules! widening_impl { + ($SelfT:ty, $WideT:ty, $BITS:literal, unsigned) => { + /// Calculates the complete product `self * rhs` without the possibility to overflow. + /// + /// This returns the low-order (wrapping) bits and the high-order (overflow) bits + /// of the result as two separate values, in that order. + /// + /// If you also need to add a carry to the wide result, then you want + /// [`Self::carrying_mul`] instead. + /// + /// # Examples + /// + /// Basic usage: + /// + /// Please note that this example is shared between integer types. + /// Which explains why `u32` is used here. + /// + /// ``` + /// #![feature(bigint_helper_methods)] + /// assert_eq!(5u32.widening_mul(2), (10, 0)); + /// assert_eq!(1_000_000_000u32.widening_mul(10), (1410065408, 2)); + /// ``` + #[unstable(feature = "bigint_helper_methods", issue = "85532")] + #[rustc_const_unstable(feature = "const_bigint_helper_methods", issue = "85532")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn widening_mul(self, rhs: Self) -> (Self, Self) { + // note: longer-term this should be done via an intrinsic, + // but for now we can deal without an impl for u128/i128 + // SAFETY: overflow will be contained within the wider types + let wide = unsafe { (self as $WideT).unchecked_mul(rhs as $WideT) }; + (wide as $SelfT, (wide >> $BITS) as $SelfT) + } -When starting from a slice rather than an array, fallible conversion APIs can be used: + /// Calculates the "full multiplication" `self * rhs + carry` + /// without the possibility to overflow. + /// + /// This returns the low-order (wrapping) bits and the high-order (overflow) bits + /// of the result as two separate values, in that order. + /// + /// Performs "long multiplication" which takes in an extra amount to add, and may return an + /// additional amount of overflow. This allows for chaining together multiple + /// multiplications to create "big integers" which represent larger values. + /// + /// If you don't need the `carry`, then you can use [`Self::widening_mul`] instead. + /// + /// # Examples + /// + /// Basic usage: + /// + /// Please note that this example is shared between integer types. + /// Which explains why `u32` is used here. + /// + /// ``` + /// #![feature(bigint_helper_methods)] + /// assert_eq!(5u32.carrying_mul(2, 0), (10, 0)); + /// assert_eq!(5u32.carrying_mul(2, 10), (20, 0)); + /// assert_eq!(1_000_000_000u32.carrying_mul(10, 0), (1410065408, 2)); + /// assert_eq!(1_000_000_000u32.carrying_mul(10, 10), (1410065418, 2)); + #[doc = concat!("assert_eq!(", + stringify!($SelfT), "::MAX.carrying_mul(", stringify!($SelfT), "::MAX, ", stringify!($SelfT), "::MAX), ", + "(0, ", stringify!($SelfT), "::MAX));" + )] + /// ``` + /// + /// This is the core operation needed for scalar multiplication when + /// implementing it for wider-than-native types. + /// + /// ``` + /// #![feature(bigint_helper_methods)] + /// fn scalar_mul_eq(little_endian_digits: &mut Vec, multiplicand: u16) { + /// let mut carry = 0; + /// for d in little_endian_digits.iter_mut() { + /// (*d, carry) = d.carrying_mul(multiplicand, carry); + /// } + /// if carry != 0 { + /// little_endian_digits.push(carry); + /// } + /// } + /// + /// let mut v = vec![10, 20]; + /// scalar_mul_eq(&mut v, 3); + /// assert_eq!(v, [30, 60]); + /// + /// assert_eq!(0x87654321_u64 * 0xFEED, 0x86D3D159E38D); + /// let mut v = vec![0x4321, 0x8765]; + /// scalar_mul_eq(&mut v, 0xFEED); + /// assert_eq!(v, [0xE38D, 0xD159, 0x86D3]); + /// ``` + /// + /// If `carry` is zero, this is similar to [`overflowing_mul`](Self::overflowing_mul), + /// except that it gives the value of the overflow instead of just whether one happened: + /// + /// ``` + /// #![feature(bigint_helper_methods)] + /// let r = u8::carrying_mul(7, 13, 0); + /// assert_eq!((r.0, r.1 != 0), u8::overflowing_mul(7, 13)); + /// let r = u8::carrying_mul(13, 42, 0); + /// assert_eq!((r.0, r.1 != 0), u8::overflowing_mul(13, 42)); + /// ``` + /// + /// The value of the first field in the returned tuple matches what you'd get + /// by combining the [`wrapping_mul`](Self::wrapping_mul) and + /// [`wrapping_add`](Self::wrapping_add) methods: + /// + /// ``` + /// #![feature(bigint_helper_methods)] + /// assert_eq!( + /// 789_u16.carrying_mul(456, 123).0, + /// 789_u16.wrapping_mul(456).wrapping_add(123), + /// ); + /// ``` + #[unstable(feature = "bigint_helper_methods", issue = "85532")] + #[rustc_const_unstable(feature = "bigint_helper_methods", issue = "85532")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn carrying_mul(self, rhs: Self, carry: Self) -> (Self, Self) { + // note: longer-term this should be done via an intrinsic, + // but for now we can deal without an impl for u128/i128 + // SAFETY: overflow will be contained within the wider types + let wide = unsafe { + (self as $WideT).unchecked_mul(rhs as $WideT).unchecked_add(carry as $WideT) + }; + (wide as $SelfT, (wide >> $BITS) as $SelfT) + } + }; +} -``` -use std::convert::TryInto; +impl i8 { + int_impl! { i8, i8, u8, 8, 7, -128, 127, 2, "-0x7e", "0xa", "0x12", "0x12", "0x48", + "[0x12]", "[0x12]", "", "", "" } +} -fn read_ne_", stringify!($SelfT), "(input: &mut &[u8]) -> ", stringify!($SelfT), " { - let (int_bytes, rest) = input.split_at(std::mem::size_of::<", stringify!($SelfT), ">()); - *input = rest; - ", stringify!($SelfT), "::from_ne_bytes(int_bytes.try_into().unwrap()) +impl i16 { + int_impl! { i16, i16, u16, 16, 15, -32768, 32767, 4, "-0x5ffd", "0x3a", "0x1234", "0x3412", + "0x2c48", "[0x34, 0x12]", "[0x12, 0x34]", "", "", "" } } -```"), - #[stable(feature = "int_to_from_bytes", since = "1.32.0")] - #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] - // SAFETY: const sound because integers are plain old datatypes so we can always - // transmute to them - #[allow_internal_unstable(const_fn_union)] - #[inline] - pub const fn from_ne_bytes(bytes: [u8; mem::size_of::()]) -> Self { - #[repr(C)] - union Bytes { - val: $SelfT, - bytes: [u8; mem::size_of::<$SelfT>()], - } - // SAFETY: integers are plain old datatypes so we can always transmute to them - unsafe { Bytes { bytes }.val } - } - } - doc_comment! { - concat!("**This method is soft-deprecated.** +impl i32 { + int_impl! { i32, i32, u32, 32, 31, -2147483648, 2147483647, 8, "0x10000b3", "0xb301", + "0x12345678", "0x78563412", "0x1e6a2c48", "[0x78, 0x56, 0x34, 0x12]", + "[0x12, 0x34, 0x56, 0x78]", "", "", "" } +} -Although using it won’t cause compilation warning, -new code should use [`", stringify!($SelfT), "::MIN", "`](#associatedconstant.MIN) instead. +impl i64 { + int_impl! { i64, i64, u64, 64, 63, -9223372036854775808, 9223372036854775807, 12, + "0xaa00000000006e1", "0x6e10aa", "0x1234567890123456", "0x5634129078563412", + "0x6a2c48091e6a2c48", "[0x56, 0x34, 0x12, 0x90, 0x78, 0x56, 0x34, 0x12]", + "[0x12, 0x34, 0x56, 0x78, 0x90, 0x12, 0x34, 0x56]", "", "", "" } +} -Returns the smallest value that can be represented by this integer type."), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_promotable] - #[inline(always)] - #[rustc_const_stable(feature = "const_max_value", since = "1.32.0")] - pub const fn min_value() -> Self { Self::MIN } - } +impl i128 { + int_impl! { i128, i128, u128, 128, 127, -170141183460469231731687303715884105728, + 170141183460469231731687303715884105727, 16, + "0x13f40000000000000000000000004f76", "0x4f7613f4", "0x12345678901234567890123456789012", + "0x12907856341290785634129078563412", "0x48091e6a2c48091e6a2c48091e6a2c48", + "[0x12, 0x90, 0x78, 0x56, 0x34, 0x12, 0x90, 0x78, \ + 0x56, 0x34, 0x12, 0x90, 0x78, 0x56, 0x34, 0x12]", + "[0x12, 0x34, 0x56, 0x78, 0x90, 0x12, 0x34, 0x56, \ + 0x78, 0x90, 0x12, 0x34, 0x56, 0x78, 0x90, 0x12]", "", "", "" } +} - doc_comment! { - concat!("**This method is soft-deprecated.** +#[cfg(target_pointer_width = "16")] +impl isize { + int_impl! { isize, i16, usize, 16, 15, -32768, 32767, 4, "-0x5ffd", "0x3a", "0x1234", + "0x3412", "0x2c48", "[0x34, 0x12]", "[0x12, 0x34]", + usize_isize_to_xe_bytes_doc!(), usize_isize_from_xe_bytes_doc!(), + " on 16-bit targets" } +} -Although using it won’t cause compilation warning, -new code should use [`", stringify!($SelfT), "::MAX", "`](#associatedconstant.MAX) instead. +#[cfg(target_pointer_width = "32")] +impl isize { + int_impl! { isize, i32, usize, 32, 31, -2147483648, 2147483647, 8, "0x10000b3", "0xb301", + "0x12345678", "0x78563412", "0x1e6a2c48", "[0x78, 0x56, 0x34, 0x12]", + "[0x12, 0x34, 0x56, 0x78]", + usize_isize_to_xe_bytes_doc!(), usize_isize_from_xe_bytes_doc!(), + " on 32-bit targets" } +} -Returns the largest value that can be represented by this integer type."), - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_promotable] - #[inline(always)] - #[rustc_const_stable(feature = "const_max_value", since = "1.32.0")] - pub const fn max_value() -> Self { Self::MAX } - } - } +#[cfg(target_pointer_width = "64")] +impl isize { + int_impl! { isize, i64, usize, 64, 63, -9223372036854775808, 9223372036854775807, + 12, "0xaa00000000006e1", "0x6e10aa", "0x1234567890123456", "0x5634129078563412", + "0x6a2c48091e6a2c48", "[0x56, 0x34, 0x12, 0x90, 0x78, 0x56, 0x34, 0x12]", + "[0x12, 0x34, 0x56, 0x78, 0x90, 0x12, 0x34, 0x56]", + usize_isize_to_xe_bytes_doc!(), usize_isize_from_xe_bytes_doc!(), + " on 64-bit targets" } } -#[lang = "u8"] +/// If 6th bit set ascii is upper case. +const ASCII_CASE_MASK: u8 = 0b0010_0000; + impl u8 { - uint_impl! { u8, u8, 8, 255, "", "", 2, "0x82", "0xa", "0x12", "0x12", "0x48", "[0x12]", - "[0x12]", "", "" } + uint_impl! { u8, u8, i8, NonZeroU8, 8, 255, 2, "0x82", "0xa", "0x12", "0x12", "0x48", "[0x12]", + "[0x12]", "", "", "" } + widening_impl! { u8, u16, 8, unsigned } /// Checks if the value is within the ASCII range. /// @@ -4453,8 +317,9 @@ impl u8 { /// assert!(ascii.is_ascii()); /// assert!(!non_ascii.is_ascii()); /// ``` + #[must_use] #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] - #[rustc_const_stable(feature = "const_ascii_methods_on_intrinsics", since = "1.43.0")] + #[rustc_const_stable(feature = "const_u8_is_ascii", since = "1.43.0")] #[inline] pub const fn is_ascii(&self) -> bool { *self & 128 == 0 @@ -4475,12 +340,14 @@ impl u8 { /// assert_eq!(65, lowercase_a.to_ascii_uppercase()); /// ``` /// - /// [`make_ascii_uppercase`]: #method.make_ascii_uppercase + /// [`make_ascii_uppercase`]: Self::make_ascii_uppercase + #[must_use = "to uppercase the value in-place, use `make_ascii_uppercase()`"] #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] + #[rustc_const_stable(feature = "const_ascii_methods_on_intrinsics", since = "1.52.0")] #[inline] - pub fn to_ascii_uppercase(&self) -> u8 { - // Unset the fith bit if this is a lowercase letter - *self & !((self.is_ascii_lowercase() as u8) << 5) + pub const fn to_ascii_uppercase(&self) -> u8 { + // Toggle the fifth bit if this is a lowercase letter + *self ^ ((self.is_ascii_lowercase() as u8) * ASCII_CASE_MASK) } /// Makes a copy of the value in its ASCII lower case equivalent. @@ -4498,12 +365,20 @@ impl u8 { /// assert_eq!(97, uppercase_a.to_ascii_lowercase()); /// ``` /// - /// [`make_ascii_lowercase`]: #method.make_ascii_lowercase + /// [`make_ascii_lowercase`]: Self::make_ascii_lowercase + #[must_use = "to lowercase the value in-place, use `make_ascii_lowercase()`"] #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] + #[rustc_const_stable(feature = "const_ascii_methods_on_intrinsics", since = "1.52.0")] + #[inline] + pub const fn to_ascii_lowercase(&self) -> u8 { + // Set the fifth bit if this is an uppercase letter + *self | (self.is_ascii_uppercase() as u8 * ASCII_CASE_MASK) + } + + /// Assumes self is ascii #[inline] - pub fn to_ascii_lowercase(&self) -> u8 { - // Set the fith bit if this is an uppercase letter - *self | ((self.is_ascii_uppercase() as u8) << 5) + pub(crate) const fn ascii_change_case_unchecked(&self) -> u8 { + *self ^ ASCII_CASE_MASK } /// Checks that two values are an ASCII case-insensitive match. @@ -4519,8 +394,9 @@ impl u8 { /// assert!(lowercase_a.eq_ignore_ascii_case(&uppercase_a)); /// ``` #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] + #[rustc_const_stable(feature = "const_ascii_methods_on_intrinsics", since = "1.52.0")] #[inline] - pub fn eq_ignore_ascii_case(&self, other: &u8) -> bool { + pub const fn eq_ignore_ascii_case(&self, other: &u8) -> bool { self.to_ascii_lowercase() == other.to_ascii_lowercase() } @@ -4542,7 +418,7 @@ impl u8 { /// assert_eq!(b'A', byte); /// ``` /// - /// [`to_ascii_uppercase`]: #method.to_ascii_uppercase + /// [`to_ascii_uppercase`]: Self::to_ascii_uppercase #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] #[inline] pub fn make_ascii_uppercase(&mut self) { @@ -4567,7 +443,7 @@ impl u8 { /// assert_eq!(b'a', byte); /// ``` /// - /// [`to_ascii_lowercase`]: #method.to_ascii_lowercase + /// [`to_ascii_lowercase`]: Self::to_ascii_lowercase #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] #[inline] pub fn make_ascii_lowercase(&mut self) { @@ -4590,7 +466,7 @@ impl u8 { /// let percent = b'%'; /// let space = b' '; /// let lf = b'\n'; - /// let esc = 0x1b_u8; + /// let esc = b'\x1b'; /// /// assert!(uppercase_a.is_ascii_alphabetic()); /// assert!(uppercase_g.is_ascii_alphabetic()); @@ -4602,8 +478,9 @@ impl u8 { /// assert!(!lf.is_ascii_alphabetic()); /// assert!(!esc.is_ascii_alphabetic()); /// ``` + #[must_use] #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] - #[rustc_const_unstable(feature = "const_ascii_ctype_on_intrinsics", issue = "68983")] + #[rustc_const_stable(feature = "const_ascii_ctype_on_intrinsics", since = "1.47.0")] #[inline] pub const fn is_ascii_alphabetic(&self) -> bool { matches!(*self, b'A'..=b'Z' | b'a'..=b'z') @@ -4623,7 +500,7 @@ impl u8 { /// let percent = b'%'; /// let space = b' '; /// let lf = b'\n'; - /// let esc = 0x1b_u8; + /// let esc = b'\x1b'; /// /// assert!(uppercase_a.is_ascii_uppercase()); /// assert!(uppercase_g.is_ascii_uppercase()); @@ -4635,8 +512,9 @@ impl u8 { /// assert!(!lf.is_ascii_uppercase()); /// assert!(!esc.is_ascii_uppercase()); /// ``` + #[must_use] #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] - #[rustc_const_unstable(feature = "const_ascii_ctype_on_intrinsics", issue = "68983")] + #[rustc_const_stable(feature = "const_ascii_ctype_on_intrinsics", since = "1.47.0")] #[inline] pub const fn is_ascii_uppercase(&self) -> bool { matches!(*self, b'A'..=b'Z') @@ -4656,7 +534,7 @@ impl u8 { /// let percent = b'%'; /// let space = b' '; /// let lf = b'\n'; - /// let esc = 0x1b_u8; + /// let esc = b'\x1b'; /// /// assert!(!uppercase_a.is_ascii_lowercase()); /// assert!(!uppercase_g.is_ascii_lowercase()); @@ -4668,8 +546,9 @@ impl u8 { /// assert!(!lf.is_ascii_lowercase()); /// assert!(!esc.is_ascii_lowercase()); /// ``` + #[must_use] #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] - #[rustc_const_unstable(feature = "const_ascii_ctype_on_intrinsics", issue = "68983")] + #[rustc_const_stable(feature = "const_ascii_ctype_on_intrinsics", since = "1.47.0")] #[inline] pub const fn is_ascii_lowercase(&self) -> bool { matches!(*self, b'a'..=b'z') @@ -4692,7 +571,7 @@ impl u8 { /// let percent = b'%'; /// let space = b' '; /// let lf = b'\n'; - /// let esc = 0x1b_u8; + /// let esc = b'\x1b'; /// /// assert!(uppercase_a.is_ascii_alphanumeric()); /// assert!(uppercase_g.is_ascii_alphanumeric()); @@ -4704,8 +583,9 @@ impl u8 { /// assert!(!lf.is_ascii_alphanumeric()); /// assert!(!esc.is_ascii_alphanumeric()); /// ``` + #[must_use] #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] - #[rustc_const_unstable(feature = "const_ascii_ctype_on_intrinsics", issue = "68983")] + #[rustc_const_stable(feature = "const_ascii_ctype_on_intrinsics", since = "1.47.0")] #[inline] pub const fn is_ascii_alphanumeric(&self) -> bool { matches!(*self, b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') @@ -4725,7 +605,7 @@ impl u8 { /// let percent = b'%'; /// let space = b' '; /// let lf = b'\n'; - /// let esc = 0x1b_u8; + /// let esc = b'\x1b'; /// /// assert!(!uppercase_a.is_ascii_digit()); /// assert!(!uppercase_g.is_ascii_digit()); @@ -4737,13 +617,46 @@ impl u8 { /// assert!(!lf.is_ascii_digit()); /// assert!(!esc.is_ascii_digit()); /// ``` + #[must_use] #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] - #[rustc_const_unstable(feature = "const_ascii_ctype_on_intrinsics", issue = "68983")] + #[rustc_const_stable(feature = "const_ascii_ctype_on_intrinsics", since = "1.47.0")] #[inline] pub const fn is_ascii_digit(&self) -> bool { matches!(*self, b'0'..=b'9') } + /// Checks if the value is an ASCII octal digit: + /// U+0030 '0' ..= U+0037 '7'. + /// + /// # Examples + /// + /// ``` + /// #![feature(is_ascii_octdigit)] + /// + /// let uppercase_a = b'A'; + /// let a = b'a'; + /// let zero = b'0'; + /// let seven = b'7'; + /// let nine = b'9'; + /// let percent = b'%'; + /// let lf = b'\n'; + /// + /// assert!(!uppercase_a.is_ascii_octdigit()); + /// assert!(!a.is_ascii_octdigit()); + /// assert!(zero.is_ascii_octdigit()); + /// assert!(seven.is_ascii_octdigit()); + /// assert!(!nine.is_ascii_octdigit()); + /// assert!(!percent.is_ascii_octdigit()); + /// assert!(!lf.is_ascii_octdigit()); + /// ``` + #[must_use] + #[unstable(feature = "is_ascii_octdigit", issue = "101288")] + #[rustc_const_unstable(feature = "is_ascii_octdigit", issue = "101288")] + #[inline] + pub const fn is_ascii_octdigit(&self) -> bool { + matches!(*self, b'0'..=b'7') + } + /// Checks if the value is an ASCII hexadecimal digit: /// /// - U+0030 '0' ..= U+0039 '9', or @@ -4761,7 +674,7 @@ impl u8 { /// let percent = b'%'; /// let space = b' '; /// let lf = b'\n'; - /// let esc = 0x1b_u8; + /// let esc = b'\x1b'; /// /// assert!(uppercase_a.is_ascii_hexdigit()); /// assert!(!uppercase_g.is_ascii_hexdigit()); @@ -4773,8 +686,9 @@ impl u8 { /// assert!(!lf.is_ascii_hexdigit()); /// assert!(!esc.is_ascii_hexdigit()); /// ``` + #[must_use] #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] - #[rustc_const_unstable(feature = "const_ascii_ctype_on_intrinsics", issue = "68983")] + #[rustc_const_stable(feature = "const_ascii_ctype_on_intrinsics", since = "1.47.0")] #[inline] pub const fn is_ascii_hexdigit(&self) -> bool { matches!(*self, b'0'..=b'9' | b'A'..=b'F' | b'a'..=b'f') @@ -4784,7 +698,7 @@ impl u8 { /// /// - U+0021 ..= U+002F `! " # $ % & ' ( ) * + , - . /`, or /// - U+003A ..= U+0040 `: ; < = > ? @`, or - /// - U+005B ..= U+0060 ``[ \ ] ^ _ ` ``, or + /// - U+005B ..= U+0060 `` [ \ ] ^ _ ` ``, or /// - U+007B ..= U+007E `{ | } ~` /// /// # Examples @@ -4798,7 +712,7 @@ impl u8 { /// let percent = b'%'; /// let space = b' '; /// let lf = b'\n'; - /// let esc = 0x1b_u8; + /// let esc = b'\x1b'; /// /// assert!(!uppercase_a.is_ascii_punctuation()); /// assert!(!uppercase_g.is_ascii_punctuation()); @@ -4810,8 +724,9 @@ impl u8 { /// assert!(!lf.is_ascii_punctuation()); /// assert!(!esc.is_ascii_punctuation()); /// ``` + #[must_use] #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] - #[rustc_const_unstable(feature = "const_ascii_ctype_on_intrinsics", issue = "68983")] + #[rustc_const_stable(feature = "const_ascii_ctype_on_intrinsics", since = "1.47.0")] #[inline] pub const fn is_ascii_punctuation(&self) -> bool { matches!(*self, b'!'..=b'/' | b':'..=b'@' | b'['..=b'`' | b'{'..=b'~') @@ -4831,7 +746,7 @@ impl u8 { /// let percent = b'%'; /// let space = b' '; /// let lf = b'\n'; - /// let esc = 0x1b_u8; + /// let esc = b'\x1b'; /// /// assert!(uppercase_a.is_ascii_graphic()); /// assert!(uppercase_g.is_ascii_graphic()); @@ -4843,8 +758,9 @@ impl u8 { /// assert!(!lf.is_ascii_graphic()); /// assert!(!esc.is_ascii_graphic()); /// ``` + #[must_use] #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] - #[rustc_const_unstable(feature = "const_ascii_ctype_on_intrinsics", issue = "68983")] + #[rustc_const_stable(feature = "const_ascii_ctype_on_intrinsics", since = "1.47.0")] #[inline] pub const fn is_ascii_graphic(&self) -> bool { matches!(*self, b'!'..=b'~') @@ -4867,8 +783,8 @@ impl u8 { /// before using this function. /// /// [infra-aw]: https://infra.spec.whatwg.org/#ascii-whitespace - /// [pct]: http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap07.html#tag_07_03_01 - /// [bfs]: http://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_05 + /// [pct]: https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap07.html#tag_07_03_01 + /// [bfs]: https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_05 /// /// # Examples /// @@ -4881,7 +797,7 @@ impl u8 { /// let percent = b'%'; /// let space = b' '; /// let lf = b'\n'; - /// let esc = 0x1b_u8; + /// let esc = b'\x1b'; /// /// assert!(!uppercase_a.is_ascii_whitespace()); /// assert!(!uppercase_g.is_ascii_whitespace()); @@ -4893,8 +809,9 @@ impl u8 { /// assert!(lf.is_ascii_whitespace()); /// assert!(!esc.is_ascii_whitespace()); /// ``` + #[must_use] #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] - #[rustc_const_unstable(feature = "const_ascii_ctype_on_intrinsics", issue = "68983")] + #[rustc_const_stable(feature = "const_ascii_ctype_on_intrinsics", since = "1.47.0")] #[inline] pub const fn is_ascii_whitespace(&self) -> bool { matches!(*self, b'\t' | b'\n' | b'\x0C' | b'\r' | b' ') @@ -4916,7 +833,7 @@ impl u8 { /// let percent = b'%'; /// let space = b' '; /// let lf = b'\n'; - /// let esc = 0x1b_u8; + /// let esc = b'\x1b'; /// /// assert!(!uppercase_a.is_ascii_control()); /// assert!(!uppercase_g.is_ascii_control()); @@ -4928,70 +845,151 @@ impl u8 { /// assert!(lf.is_ascii_control()); /// assert!(esc.is_ascii_control()); /// ``` + #[must_use] #[stable(feature = "ascii_ctype_on_intrinsics", since = "1.24.0")] - #[rustc_const_unstable(feature = "const_ascii_ctype_on_intrinsics", issue = "68983")] + #[rustc_const_stable(feature = "const_ascii_ctype_on_intrinsics", since = "1.47.0")] #[inline] pub const fn is_ascii_control(&self) -> bool { matches!(*self, b'\0'..=b'\x1F' | b'\x7F') } + + /// Returns an iterator that produces an escaped version of a `u8`, + /// treating it as an ASCII character. + /// + /// The behavior is identical to [`ascii::escape_default`]. + /// + /// # Examples + /// + /// ``` + /// + /// assert_eq!("0", b'0'.escape_ascii().to_string()); + /// assert_eq!("\\t", b'\t'.escape_ascii().to_string()); + /// assert_eq!("\\r", b'\r'.escape_ascii().to_string()); + /// assert_eq!("\\n", b'\n'.escape_ascii().to_string()); + /// assert_eq!("\\'", b'\''.escape_ascii().to_string()); + /// assert_eq!("\\\"", b'"'.escape_ascii().to_string()); + /// assert_eq!("\\\\", b'\\'.escape_ascii().to_string()); + /// assert_eq!("\\x9d", b'\x9d'.escape_ascii().to_string()); + /// ``` + #[must_use = "this returns the escaped byte as an iterator, \ + without modifying the original"] + #[stable(feature = "inherent_ascii_escape", since = "1.60.0")] + #[inline] + pub fn escape_ascii(self) -> ascii::EscapeDefault { + ascii::escape_default(self) + } + + #[inline] + pub(crate) const fn is_utf8_char_boundary(self) -> bool { + // This is bit magic equivalent to: b < 128 || b >= 192 + (self as i8) >= -0x40 + } } -#[lang = "u16"] impl u16 { - uint_impl! { u16, u16, 16, 65535, "", "", 4, "0xa003", "0x3a", "0x1234", "0x3412", "0x2c48", - "[0x34, 0x12]", "[0x12, 0x34]", "", "" } + uint_impl! { u16, u16, i16, NonZeroU16, 16, 65535, 4, "0xa003", "0x3a", "0x1234", "0x3412", "0x2c48", + "[0x34, 0x12]", "[0x12, 0x34]", "", "", "" } + widening_impl! { u16, u32, 16, unsigned } + + /// Checks if the value is a Unicode surrogate code point, which are disallowed values for [`char`]. + /// + /// # Examples + /// + /// ``` + /// #![feature(utf16_extra)] + /// + /// let low_non_surrogate = 0xA000u16; + /// let low_surrogate = 0xD800u16; + /// let high_surrogate = 0xDC00u16; + /// let high_non_surrogate = 0xE000u16; + /// + /// assert!(!low_non_surrogate.is_utf16_surrogate()); + /// assert!(low_surrogate.is_utf16_surrogate()); + /// assert!(high_surrogate.is_utf16_surrogate()); + /// assert!(!high_non_surrogate.is_utf16_surrogate()); + /// ``` + #[must_use] + #[unstable(feature = "utf16_extra", issue = "94919")] + #[rustc_const_unstable(feature = "utf16_extra_const", issue = "94919")] + #[inline] + pub const fn is_utf16_surrogate(self) -> bool { + matches!(self, 0xD800..=0xDFFF) + } } -#[lang = "u32"] impl u32 { - uint_impl! { u32, u32, 32, 4294967295, "", "", 8, "0x10000b3", "0xb301", "0x12345678", - "0x78563412", "0x1e6a2c48", "[0x78, 0x56, 0x34, 0x12]", "[0x12, 0x34, 0x56, 0x78]", "", "" } + uint_impl! { u32, u32, i32, NonZeroU32, 32, 4294967295, 8, "0x10000b3", "0xb301", "0x12345678", + "0x78563412", "0x1e6a2c48", "[0x78, 0x56, 0x34, 0x12]", "[0x12, 0x34, 0x56, 0x78]", "", "", "" } + widening_impl! { u32, u64, 32, unsigned } } -#[lang = "u64"] impl u64 { - uint_impl! { u64, u64, 64, 18446744073709551615, "", "", 12, "0xaa00000000006e1", "0x6e10aa", + uint_impl! { u64, u64, i64, NonZeroU64, 64, 18446744073709551615, 12, "0xaa00000000006e1", "0x6e10aa", "0x1234567890123456", "0x5634129078563412", "0x6a2c48091e6a2c48", "[0x56, 0x34, 0x12, 0x90, 0x78, 0x56, 0x34, 0x12]", "[0x12, 0x34, 0x56, 0x78, 0x90, 0x12, 0x34, 0x56]", - "", ""} + "", "", ""} + widening_impl! { u64, u128, 64, unsigned } } -#[lang = "u128"] impl u128 { - uint_impl! { u128, u128, 128, 340282366920938463463374607431768211455, "", "", 16, + uint_impl! { u128, u128, i128, NonZeroU128, 128, 340282366920938463463374607431768211455, 16, "0x13f40000000000000000000000004f76", "0x4f7613f4", "0x12345678901234567890123456789012", "0x12907856341290785634129078563412", "0x48091e6a2c48091e6a2c48091e6a2c48", "[0x12, 0x90, 0x78, 0x56, 0x34, 0x12, 0x90, 0x78, \ 0x56, 0x34, 0x12, 0x90, 0x78, 0x56, 0x34, 0x12]", "[0x12, 0x34, 0x56, 0x78, 0x90, 0x12, 0x34, 0x56, \ 0x78, 0x90, 0x12, 0x34, 0x56, 0x78, 0x90, 0x12]", - "", ""} + "", "", ""} } #[cfg(target_pointer_width = "16")] -#[lang = "usize"] impl usize { - uint_impl! { usize, u16, 16, 65535, "", "", 4, "0xa003", "0x3a", "0x1234", "0x3412", "0x2c48", + uint_impl! { usize, u16, isize, NonZeroUsize, 16, 65535, 4, "0xa003", "0x3a", "0x1234", "0x3412", "0x2c48", "[0x34, 0x12]", "[0x12, 0x34]", - usize_isize_to_xe_bytes_doc!(), usize_isize_from_xe_bytes_doc!() } + usize_isize_to_xe_bytes_doc!(), usize_isize_from_xe_bytes_doc!(), + " on 16-bit targets" } + widening_impl! { usize, u32, 16, unsigned } } #[cfg(target_pointer_width = "32")] -#[lang = "usize"] impl usize { - uint_impl! { usize, u32, 32, 4294967295, "", "", 8, "0x10000b3", "0xb301", "0x12345678", + uint_impl! { usize, u32, isize, NonZeroUsize, 32, 4294967295, 8, "0x10000b3", "0xb301", "0x12345678", "0x78563412", "0x1e6a2c48", "[0x78, 0x56, 0x34, 0x12]", "[0x12, 0x34, 0x56, 0x78]", - usize_isize_to_xe_bytes_doc!(), usize_isize_from_xe_bytes_doc!() } + usize_isize_to_xe_bytes_doc!(), usize_isize_from_xe_bytes_doc!(), + " on 32-bit targets" } + widening_impl! { usize, u64, 32, unsigned } } #[cfg(target_pointer_width = "64")] -#[lang = "usize"] impl usize { - uint_impl! { usize, u64, 64, 18446744073709551615, "", "", 12, "0xaa00000000006e1", "0x6e10aa", + uint_impl! { usize, u64, isize, NonZeroUsize, 64, 18446744073709551615, 12, "0xaa00000000006e1", "0x6e10aa", "0x1234567890123456", "0x5634129078563412", "0x6a2c48091e6a2c48", "[0x56, 0x34, 0x12, 0x90, 0x78, 0x56, 0x34, 0x12]", - "[0x12, 0x34, 0x56, 0x78, 0x90, 0x12, 0x34, 0x56]", - usize_isize_to_xe_bytes_doc!(), usize_isize_from_xe_bytes_doc!() } + "[0x12, 0x34, 0x56, 0x78, 0x90, 0x12, 0x34, 0x56]", + usize_isize_to_xe_bytes_doc!(), usize_isize_from_xe_bytes_doc!(), + " on 64-bit targets" } + widening_impl! { usize, u128, 64, unsigned } +} + +impl usize { + /// Returns an `usize` where every byte is equal to `x`. + #[inline] + pub(crate) const fn repeat_u8(x: u8) -> usize { + usize::from_ne_bytes([x; mem::size_of::()]) + } + + /// Returns an `usize` where every byte pair is equal to `x`. + #[inline] + pub(crate) const fn repeat_u16(x: u16) -> usize { + let mut r = 0usize; + let mut i = 0; + while i < mem::size_of::() { + // Use `wrapping_shl` to make it work on targets with 16-bit `usize` + r = r.wrapping_shl(16) | (x as usize); + i += 2; + } + r + } } /// A classification of floating point numbers. @@ -4999,9 +997,6 @@ impl usize { /// This `enum` is used as the return type for [`f32::classify`] and [`f64::classify`]. See /// their documentation for more. /// -/// [`f32::classify`]: ../../std/primitive.f32.html#method.classify -/// [`f64::classify`]: ../../std/primitive.f64.html#method.classify -/// /// # Examples /// /// ``` @@ -5016,33 +1011,62 @@ impl usize { /// assert_eq!(num.classify(), FpCategory::Normal); /// assert_eq!(inf.classify(), FpCategory::Infinite); /// assert_eq!(zero.classify(), FpCategory::Zero); -/// assert_eq!(nan.classify(), FpCategory::Nan); /// assert_eq!(sub.classify(), FpCategory::Subnormal); +/// assert_eq!(nan.classify(), FpCategory::Nan); /// ``` #[derive(Copy, Clone, PartialEq, Eq, Debug)] #[stable(feature = "rust1", since = "1.0.0")] pub enum FpCategory { - /// "Not a Number", often obtained by dividing by zero. + /// NaN (not a number): this value results from calculations like `(-1.0).sqrt()`. + /// + /// See [the documentation for `f32`](f32) for more information on the unusual properties + /// of NaN. #[stable(feature = "rust1", since = "1.0.0")] Nan, - /// Positive or negative infinity. + /// Positive or negative infinity, which often results from dividing a nonzero number + /// by zero. #[stable(feature = "rust1", since = "1.0.0")] Infinite, /// Positive or negative zero. + /// + /// See [the documentation for `f32`](f32) for more information on the signedness of zeroes. #[stable(feature = "rust1", since = "1.0.0")] Zero, - /// De-normalized floating point representation (less precise than `Normal`). + /// “Subnormal” or “denormal” floating point representation (less precise, relative to + /// their magnitude, than [`Normal`]). + /// + /// Subnormal numbers are larger in magnitude than [`Zero`] but smaller in magnitude than all + /// [`Normal`] numbers. + /// + /// [`Normal`]: Self::Normal + /// [`Zero`]: Self::Zero #[stable(feature = "rust1", since = "1.0.0")] Subnormal, - /// A regular floating point number. + /// A regular floating point number, not any of the exceptional categories. + /// + /// The smallest positive normal numbers are [`f32::MIN_POSITIVE`] and [`f64::MIN_POSITIVE`], + /// and the largest positive normal numbers are [`f32::MAX`] and [`f64::MAX`]. (Unlike signed + /// integers, floating point numbers are symmetric in their range, so negating any of these + /// constants will produce their negative counterpart.) #[stable(feature = "rust1", since = "1.0.0")] Normal, } +#[doc(hidden)] +trait FromStrRadixHelper: + PartialOrd + Copy + Add + Sub + Mul +{ + const MIN: Self; + fn from_u32(u: u32) -> Self; + fn checked_mul(&self, other: u32) -> Option; + fn checked_sub(&self, other: u32) -> Option; + fn checked_add(&self, other: u32) -> Option; +} + macro_rules! from_str_radix_int_impl { ($($t:ty)*) => {$( #[stable(feature = "rust1", since = "1.0.0")] @@ -5056,64 +1080,9 @@ macro_rules! from_str_radix_int_impl { } from_str_radix_int_impl! { isize i8 i16 i32 i64 i128 usize u8 u16 u32 u64 u128 } -/// The error type returned when a checked integral type conversion fails. -#[stable(feature = "try_from", since = "1.34.0")] -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub struct TryFromIntError(pub(crate) ()); - -impl TryFromIntError { - #[unstable( - feature = "int_error_internals", - reason = "available through Error trait and this method should \ - not be exposed publicly", - issue = "none" - )] - #[doc(hidden)] - pub fn __description(&self) -> &str { - "out of range integral type conversion attempted" - } -} - -#[stable(feature = "try_from", since = "1.34.0")] -impl fmt::Display for TryFromIntError { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - self.__description().fmt(fmt) - } -} - -#[stable(feature = "try_from", since = "1.34.0")] -impl From for TryFromIntError { - fn from(x: Infallible) -> TryFromIntError { - match x {} - } -} - -#[unstable(feature = "never_type", issue = "35121")] -impl From for TryFromIntError { - fn from(never: !) -> TryFromIntError { - // Match rather than coerce to make sure that code like - // `From for TryFromIntError` above will keep working - // when `Infallible` becomes an alias to `!`. - match never {} - } -} - -#[doc(hidden)] -trait FromStrRadixHelper: PartialOrd + Copy { - fn min_value() -> Self; - fn max_value() -> Self; - fn from_u32(u: u32) -> Self; - fn checked_mul(&self, other: u32) -> Option; - fn checked_sub(&self, other: u32) -> Option; - fn checked_add(&self, other: u32) -> Option; -} - -macro_rules! doit { +macro_rules! impl_helper_for { ($($t:ty)*) => ($(impl FromStrRadixHelper for $t { - #[inline] - fn min_value() -> Self { Self::min_value() } - #[inline] - fn max_value() -> Self { Self::max_value() } + const MIN: Self = Self::MIN; #[inline] fn from_u32(u: u32) -> Self { u as Self } #[inline] @@ -5130,14 +1099,25 @@ macro_rules! doit { } })*) } -doit! { i8 i16 i32 i64 i128 isize u8 u16 u32 u64 u128 usize } +impl_helper_for! { i8 i16 i32 i64 i128 isize u8 u16 u32 u64 u128 usize } + +/// Determines if a string of text of that length of that radix could be guaranteed to be +/// stored in the given type T. +/// Note that if the radix is known to the compiler, it is just the check of digits.len that +/// is done at runtime. +#[doc(hidden)] +#[inline(always)] +#[unstable(issue = "none", feature = "std_internals")] +pub fn can_not_overflow(radix: u32, is_signed_ty: bool, digits: &[u8]) -> bool { + radix <= 16 && digits.len() <= mem::size_of::() * 2 - is_signed_ty as usize +} fn from_str_radix(src: &str, radix: u32) -> Result { use self::IntErrorKind::*; use self::ParseIntError as PIE; assert!( - radix >= 2 && radix <= 36, + (2..=36).contains(&radix), "from_str_radix_int: must lie in the range `[2, 36]` - found {}", radix ); @@ -5146,7 +1126,7 @@ fn from_str_radix(src: &str, radix: u32) -> Result T::min_value(); + let is_signed_ty = T::from_u32(0) > T::MIN; // all valid digits are ascii, so we will just iterate over the utf8 bytes // and cast them to chars. .to_digit() will safely return None for anything @@ -5155,136 +1135,65 @@ fn from_str_radix(src: &str, radix: u32) -> Result { + return Err(PIE { kind: InvalidDigit }); + } b'+' => (true, &src[1..]), b'-' if is_signed_ty => (false, &src[1..]), _ => (true, src), }; - if digits.is_empty() { - return Err(PIE { kind: Empty }); - } - let mut result = T::from_u32(0); - if is_positive { - // The number is positive - for &c in digits { - let x = match (c as char).to_digit(radix) { - Some(x) => x, - None => return Err(PIE { kind: InvalidDigit }), - }; - result = match result.checked_mul(radix) { - Some(result) => result, - None => return Err(PIE { kind: Overflow }), - }; - result = match result.checked_add(x) { - Some(result) => result, - None => return Err(PIE { kind: Overflow }), + + if can_not_overflow::(radix, is_signed_ty, digits) { + // If the len of the str is short compared to the range of the type + // we are parsing into, then we can be certain that an overflow will not occur. + // This bound is when `radix.pow(digits.len()) - 1 <= T::MAX` but the condition + // above is a faster (conservative) approximation of this. + // + // Consider radix 16 as it has the highest information density per digit and will thus overflow the earliest: + // `u8::MAX` is `ff` - any str of len 2 is guaranteed to not overflow. + // `i8::MAX` is `7f` - only a str of len 1 is guaranteed to not overflow. + macro_rules! run_unchecked_loop { + ($unchecked_additive_op:expr) => { + for &c in digits { + result = result * T::from_u32(radix); + let x = (c as char).to_digit(radix).ok_or(PIE { kind: InvalidDigit })?; + result = $unchecked_additive_op(result, T::from_u32(x)); + } }; } + if is_positive { + run_unchecked_loop!(::add) + } else { + run_unchecked_loop!(::sub) + }; } else { - // The number is negative - for &c in digits { - let x = match (c as char).to_digit(radix) { - Some(x) => x, - None => return Err(PIE { kind: InvalidDigit }), - }; - result = match result.checked_mul(radix) { - Some(result) => result, - None => return Err(PIE { kind: Underflow }), - }; - result = match result.checked_sub(x) { - Some(result) => result, - None => return Err(PIE { kind: Underflow }), + macro_rules! run_checked_loop { + ($checked_additive_op:ident, $overflow_err:expr) => { + for &c in digits { + // When `radix` is passed in as a literal, rather than doing a slow `imul` + // the compiler can use shifts if `radix` can be expressed as a + // sum of powers of 2 (x*10 can be written as x*8 + x*2). + // When the compiler can't use these optimisations, + // the latency of the multiplication can be hidden by issuing it + // before the result is needed to improve performance on + // modern out-of-order CPU as multiplication here is slower + // than the other instructions, we can get the end result faster + // doing multiplication first and let the CPU spends other cycles + // doing other computation and get multiplication result later. + let mul = result.checked_mul(radix); + let x = (c as char).to_digit(radix).ok_or(PIE { kind: InvalidDigit })?; + result = mul.ok_or_else($overflow_err)?; + result = T::$checked_additive_op(&result, x).ok_or_else($overflow_err)?; + } }; } + if is_positive { + run_checked_loop!(checked_add, || PIE { kind: PosOverflow }) + } else { + run_checked_loop!(checked_sub, || PIE { kind: NegOverflow }) + }; } Ok(result) } - -/// An error which can be returned when parsing an integer. -/// -/// This error is used as the error type for the `from_str_radix()` functions -/// on the primitive integer types, such as [`i8::from_str_radix`]. -/// -/// # Potential causes -/// -/// Among other causes, `ParseIntError` can be thrown because of leading or trailing whitespace -/// in the string e.g., when it is obtained from the standard input. -/// Using the [`str.trim()`] method ensures that no whitespace remains before parsing. -/// -/// [`str.trim()`]: ../../std/primitive.str.html#method.trim -/// [`i8::from_str_radix`]: ../../std/primitive.i8.html#method.from_str_radix -#[derive(Debug, Clone, PartialEq, Eq)] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct ParseIntError { - kind: IntErrorKind, -} - -/// Enum to store the various types of errors that can cause parsing an integer to fail. -#[unstable( - feature = "int_error_matching", - reason = "it can be useful to match errors when making error messages \ - for integer parsing", - issue = "22639" -)] -#[derive(Debug, Clone, PartialEq, Eq)] -#[non_exhaustive] -pub enum IntErrorKind { - /// Value being parsed is empty. - /// - /// Among other causes, this variant will be constructed when parsing an empty string. - Empty, - /// Contains an invalid digit. - /// - /// Among other causes, this variant will be constructed when parsing a string that - /// contains a letter. - InvalidDigit, - /// Integer is too large to store in target integer type. - Overflow, - /// Integer is too small to store in target integer type. - Underflow, - /// Value was Zero - /// - /// This variant will be emitted when the parsing string has a value of zero, which - /// would be illegal for non-zero types. - Zero, -} - -impl ParseIntError { - /// Outputs the detailed cause of parsing an integer failing. - #[unstable( - feature = "int_error_matching", - reason = "it can be useful to match errors when making error messages \ - for integer parsing", - issue = "22639" - )] - pub fn kind(&self) -> &IntErrorKind { - &self.kind - } - #[unstable( - feature = "int_error_internals", - reason = "available through Error trait and this method should \ - not be exposed publicly", - issue = "none" - )] - #[doc(hidden)] - pub fn __description(&self) -> &str { - match self.kind { - IntErrorKind::Empty => "cannot parse integer from empty string", - IntErrorKind::InvalidDigit => "invalid digit found in string", - IntErrorKind::Overflow => "number too large to fit in target type", - IntErrorKind::Underflow => "number too small to fit in target type", - IntErrorKind::Zero => "number would be zero for non-zero type", - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl fmt::Display for ParseIntError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.__description().fmt(f) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -pub use crate::num::dec2flt::ParseFloatError; diff --git a/crux-mir/lib/core/src/num/nonzero.rs b/crux-mir/lib/core/src/num/nonzero.rs new file mode 100644 index 000000000..fbda8f82b --- /dev/null +++ b/crux-mir/lib/core/src/num/nonzero.rs @@ -0,0 +1,1276 @@ +//! Definitions of integer that is known not to equal zero. + +use crate::fmt; +use crate::ops::{BitOr, BitOrAssign, Div, Rem}; +use crate::str::FromStr; + +use super::from_str_radix; +use super::{IntErrorKind, ParseIntError}; +use crate::intrinsics; + +macro_rules! impl_nonzero_fmt { + ( #[$stability: meta] ( $( $Trait: ident ),+ ) for $Ty: ident ) => { + $( + #[$stability] + impl fmt::$Trait for $Ty { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } + } + )+ + } +} + +macro_rules! nonzero_integers { + ( $( #[$stability: meta] #[$const_new_unchecked_stability: meta] $Ty: ident($Int: ty); )+ ) => { + $( + /// An integer that is known not to equal zero. + /// + /// This enables some memory layout optimization. + #[doc = concat!("For example, `Option<", stringify!($Ty), ">` is the same size as `", stringify!($Int), "`:")] + /// + /// ```rust + /// use std::mem::size_of; + #[doc = concat!("assert_eq!(size_of::>(), size_of::<", stringify!($Int), ">());")] + /// ``` + #[$stability] + #[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] + #[repr(transparent)] + #[rustc_layout_scalar_valid_range_start(1)] + #[rustc_nonnull_optimization_guaranteed] + #[rustc_diagnostic_item = stringify!($Ty)] + pub struct $Ty($Int); + + impl $Ty { + /// Creates a non-zero without checking whether the value is non-zero. + /// This results in undefined behaviour if the value is zero. + /// + /// # Safety + /// + /// The value must not be zero. + #[$stability] + #[$const_new_unchecked_stability] + #[must_use] + #[inline] + pub const unsafe fn new_unchecked(n: $Int) -> Self { + // SAFETY: this is guaranteed to be safe by the caller. + unsafe { + core::intrinsics::assert_unsafe_precondition!( + concat!(stringify!($Ty), "::new_unchecked requires a non-zero argument"), + (n: $Int) => n != 0 + ); + Self(n) + } + } + + /// Creates a non-zero if the given value is not zero. + #[$stability] + #[rustc_const_stable(feature = "const_nonzero_int_methods", since = "1.47.0")] + #[must_use] + #[inline] + pub const fn new(n: $Int) -> Option { + if n != 0 { + // SAFETY: we just checked that there's no `0` + Some(unsafe { Self(n) }) + } else { + None + } + } + + /// Returns the value as a primitive type. + #[$stability] + #[inline] + #[rustc_const_stable(feature = "const_nonzero_get", since = "1.34.0")] + pub const fn get(self) -> $Int { + self.0 + } + + } + + #[stable(feature = "from_nonzero", since = "1.31.0")] + #[rustc_const_unstable(feature = "const_num_from_num", issue = "87852")] + impl const From<$Ty> for $Int { + #[doc = concat!("Converts a `", stringify!($Ty), "` into an `", stringify!($Int), "`")] + #[inline] + fn from(nonzero: $Ty) -> Self { + nonzero.0 + } + } + + #[stable(feature = "nonzero_bitor", since = "1.45.0")] + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const BitOr for $Ty { + type Output = Self; + #[inline] + fn bitor(self, rhs: Self) -> Self::Output { + // SAFETY: since `self` and `rhs` are both nonzero, the + // result of the bitwise-or will be nonzero. + unsafe { $Ty::new_unchecked(self.get() | rhs.get()) } + } + } + + #[stable(feature = "nonzero_bitor", since = "1.45.0")] + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const BitOr<$Int> for $Ty { + type Output = Self; + #[inline] + fn bitor(self, rhs: $Int) -> Self::Output { + // SAFETY: since `self` is nonzero, the result of the + // bitwise-or will be nonzero regardless of the value of + // `rhs`. + unsafe { $Ty::new_unchecked(self.get() | rhs) } + } + } + + #[stable(feature = "nonzero_bitor", since = "1.45.0")] + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const BitOr<$Ty> for $Int { + type Output = $Ty; + #[inline] + fn bitor(self, rhs: $Ty) -> Self::Output { + // SAFETY: since `rhs` is nonzero, the result of the + // bitwise-or will be nonzero regardless of the value of + // `self`. + unsafe { $Ty::new_unchecked(self | rhs.get()) } + } + } + + #[stable(feature = "nonzero_bitor", since = "1.45.0")] + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const BitOrAssign for $Ty { + #[inline] + fn bitor_assign(&mut self, rhs: Self) { + *self = *self | rhs; + } + } + + #[stable(feature = "nonzero_bitor", since = "1.45.0")] + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const BitOrAssign<$Int> for $Ty { + #[inline] + fn bitor_assign(&mut self, rhs: $Int) { + *self = *self | rhs; + } + } + + impl_nonzero_fmt! { + #[$stability] (Debug, Display, Binary, Octal, LowerHex, UpperHex) for $Ty + } + )+ + } +} + +nonzero_integers! { + #[stable(feature = "nonzero", since = "1.28.0")] #[rustc_const_stable(feature = "nonzero", since = "1.28.0")] NonZeroU8(u8); + #[stable(feature = "nonzero", since = "1.28.0")] #[rustc_const_stable(feature = "nonzero", since = "1.28.0")] NonZeroU16(u16); + #[stable(feature = "nonzero", since = "1.28.0")] #[rustc_const_stable(feature = "nonzero", since = "1.28.0")] NonZeroU32(u32); + #[stable(feature = "nonzero", since = "1.28.0")] #[rustc_const_stable(feature = "nonzero", since = "1.28.0")] NonZeroU64(u64); + #[stable(feature = "nonzero", since = "1.28.0")] #[rustc_const_stable(feature = "nonzero", since = "1.28.0")] NonZeroU128(u128); + #[stable(feature = "nonzero", since = "1.28.0")] #[rustc_const_stable(feature = "nonzero", since = "1.28.0")] NonZeroUsize(usize); + #[stable(feature = "signed_nonzero", since = "1.34.0")] #[rustc_const_stable(feature = "signed_nonzero", since = "1.34.0")] NonZeroI8(i8); + #[stable(feature = "signed_nonzero", since = "1.34.0")] #[rustc_const_stable(feature = "signed_nonzero", since = "1.34.0")] NonZeroI16(i16); + #[stable(feature = "signed_nonzero", since = "1.34.0")] #[rustc_const_stable(feature = "signed_nonzero", since = "1.34.0")] NonZeroI32(i32); + #[stable(feature = "signed_nonzero", since = "1.34.0")] #[rustc_const_stable(feature = "signed_nonzero", since = "1.34.0")] NonZeroI64(i64); + #[stable(feature = "signed_nonzero", since = "1.34.0")] #[rustc_const_stable(feature = "signed_nonzero", since = "1.34.0")] NonZeroI128(i128); + #[stable(feature = "signed_nonzero", since = "1.34.0")] #[rustc_const_stable(feature = "signed_nonzero", since = "1.34.0")] NonZeroIsize(isize); +} + +macro_rules! from_str_radix_nzint_impl { + ($($t:ty)*) => {$( + #[stable(feature = "nonzero_parse", since = "1.35.0")] + impl FromStr for $t { + type Err = ParseIntError; + fn from_str(src: &str) -> Result { + Self::new(from_str_radix(src, 10)?) + .ok_or(ParseIntError { + kind: IntErrorKind::Zero + }) + } + } + )*} +} + +from_str_radix_nzint_impl! { NonZeroU8 NonZeroU16 NonZeroU32 NonZeroU64 NonZeroU128 NonZeroUsize +NonZeroI8 NonZeroI16 NonZeroI32 NonZeroI64 NonZeroI128 NonZeroIsize } + +macro_rules! nonzero_leading_trailing_zeros { + ( $( $Ty: ident($Uint: ty) , $LeadingTestExpr:expr ;)+ ) => { + $( + impl $Ty { + /// Returns the number of leading zeros in the binary representation of `self`. + /// + /// On many architectures, this function can perform better than `leading_zeros()` on the underlying integer type, as special handling of zero can be avoided. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = std::num::", stringify!($Ty), "::new(", stringify!($LeadingTestExpr), ").unwrap();")] + /// + /// assert_eq!(n.leading_zeros(), 0); + /// ``` + #[stable(feature = "nonzero_leading_trailing_zeros", since = "1.53.0")] + #[rustc_const_stable(feature = "nonzero_leading_trailing_zeros", since = "1.53.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn leading_zeros(self) -> u32 { + // SAFETY: since `self` cannot be zero, it is safe to call `ctlz_nonzero`. + unsafe { intrinsics::ctlz_nonzero(self.0 as $Uint) as u32 } + } + + /// Returns the number of trailing zeros in the binary representation + /// of `self`. + /// + /// On many architectures, this function can perform better than `trailing_zeros()` on the underlying integer type, as special handling of zero can be avoided. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = std::num::", stringify!($Ty), "::new(0b0101000).unwrap();")] + /// + /// assert_eq!(n.trailing_zeros(), 3); + /// ``` + #[stable(feature = "nonzero_leading_trailing_zeros", since = "1.53.0")] + #[rustc_const_stable(feature = "nonzero_leading_trailing_zeros", since = "1.53.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn trailing_zeros(self) -> u32 { + // SAFETY: since `self` cannot be zero, it is safe to call `cttz_nonzero`. + unsafe { intrinsics::cttz_nonzero(self.0 as $Uint) as u32 } + } + + } + )+ + } +} + +nonzero_leading_trailing_zeros! { + NonZeroU8(u8), u8::MAX; + NonZeroU16(u16), u16::MAX; + NonZeroU32(u32), u32::MAX; + NonZeroU64(u64), u64::MAX; + NonZeroU128(u128), u128::MAX; + NonZeroUsize(usize), usize::MAX; + NonZeroI8(u8), -1i8; + NonZeroI16(u16), -1i16; + NonZeroI32(u32), -1i32; + NonZeroI64(u64), -1i64; + NonZeroI128(u128), -1i128; + NonZeroIsize(usize), -1isize; +} + +macro_rules! nonzero_integers_div { + ( $( $Ty: ident($Int: ty); )+ ) => { + $( + #[stable(feature = "nonzero_div", since = "1.51.0")] + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Div<$Ty> for $Int { + type Output = $Int; + /// This operation rounds towards zero, + /// truncating any fractional part of the exact result, and cannot panic. + #[inline] + fn div(self, other: $Ty) -> $Int { + // SAFETY: div by zero is checked because `other` is a nonzero, + // and MIN/-1 is checked because `self` is an unsigned int. + unsafe { crate::intrinsics::unchecked_div(self, other.get()) } + } + } + + #[stable(feature = "nonzero_div", since = "1.51.0")] + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Rem<$Ty> for $Int { + type Output = $Int; + /// This operation satisfies `n % d == n - (n / d) * d`, and cannot panic. + #[inline] + fn rem(self, other: $Ty) -> $Int { + // SAFETY: rem by zero is checked because `other` is a nonzero, + // and MIN/-1 is checked because `self` is an unsigned int. + unsafe { crate::intrinsics::unchecked_rem(self, other.get()) } + } + } + )+ + } +} + +nonzero_integers_div! { + NonZeroU8(u8); + NonZeroU16(u16); + NonZeroU32(u32); + NonZeroU64(u64); + NonZeroU128(u128); + NonZeroUsize(usize); +} + +// A bunch of methods for unsigned nonzero types only. +macro_rules! nonzero_unsigned_operations { + ( $( $Ty: ident($Int: ident); )+ ) => { + $( + impl $Ty { + /// Adds an unsigned integer to a non-zero value. + /// Checks for overflow and returns [`None`] on overflow. + /// As a consequence, the result cannot wrap to zero. + /// + /// + /// # Examples + /// + /// ``` + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + /// # fn main() { test().unwrap(); } + /// # fn test() -> Option<()> { + #[doc = concat!("let one = ", stringify!($Ty), "::new(1)?;")] + #[doc = concat!("let two = ", stringify!($Ty), "::new(2)?;")] + #[doc = concat!("let max = ", stringify!($Ty), "::new(", + stringify!($Int), "::MAX)?;")] + /// + /// assert_eq!(Some(two), one.checked_add(1)); + /// assert_eq!(None, max.checked_add(1)); + /// # Some(()) + /// # } + /// ``` + #[stable(feature = "nonzero_checked_ops", since = "1.64.0")] + #[rustc_const_stable(feature = "const_nonzero_checked_ops", since = "1.64.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_add(self, other: $Int) -> Option<$Ty> { + if let Some(result) = self.get().checked_add(other) { + // SAFETY: $Int::checked_add returns None on overflow + // so the result cannot be zero. + Some(unsafe { $Ty::new_unchecked(result) }) + } else { + None + } + } + + /// Adds an unsigned integer to a non-zero value. + #[doc = concat!("Return [`", stringify!($Int), "::MAX`] on overflow.")] + /// + /// # Examples + /// + /// ``` + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + /// # fn main() { test().unwrap(); } + /// # fn test() -> Option<()> { + #[doc = concat!("let one = ", stringify!($Ty), "::new(1)?;")] + #[doc = concat!("let two = ", stringify!($Ty), "::new(2)?;")] + #[doc = concat!("let max = ", stringify!($Ty), "::new(", + stringify!($Int), "::MAX)?;")] + /// + /// assert_eq!(two, one.saturating_add(1)); + /// assert_eq!(max, max.saturating_add(1)); + /// # Some(()) + /// # } + /// ``` + #[stable(feature = "nonzero_checked_ops", since = "1.64.0")] + #[rustc_const_stable(feature = "const_nonzero_checked_ops", since = "1.64.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn saturating_add(self, other: $Int) -> $Ty { + // SAFETY: $Int::saturating_add returns $Int::MAX on overflow + // so the result cannot be zero. + unsafe { $Ty::new_unchecked(self.get().saturating_add(other)) } + } + + /// Adds an unsigned integer to a non-zero value, + /// assuming overflow cannot occur. + /// Overflow is unchecked, and it is undefined behaviour to overflow + /// *even if the result would wrap to a non-zero value*. + /// The behaviour is undefined as soon as + #[doc = concat!("`self + rhs > ", stringify!($Int), "::MAX`.")] + /// + /// # Examples + /// + /// ``` + /// #![feature(nonzero_ops)] + /// + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + /// # fn main() { test().unwrap(); } + /// # fn test() -> Option<()> { + #[doc = concat!("let one = ", stringify!($Ty), "::new(1)?;")] + #[doc = concat!("let two = ", stringify!($Ty), "::new(2)?;")] + /// + /// assert_eq!(two, unsafe { one.unchecked_add(1) }); + /// # Some(()) + /// # } + /// ``` + #[unstable(feature = "nonzero_ops", issue = "84186")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const unsafe fn unchecked_add(self, other: $Int) -> $Ty { + // SAFETY: The caller ensures there is no overflow. + unsafe { $Ty::new_unchecked(self.get().unchecked_add(other)) } + } + + /// Returns the smallest power of two greater than or equal to n. + /// Checks for overflow and returns [`None`] + /// if the next power of two is greater than the type’s maximum value. + /// As a consequence, the result cannot wrap to zero. + /// + /// # Examples + /// + /// ``` + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + /// # fn main() { test().unwrap(); } + /// # fn test() -> Option<()> { + #[doc = concat!("let two = ", stringify!($Ty), "::new(2)?;")] + #[doc = concat!("let three = ", stringify!($Ty), "::new(3)?;")] + #[doc = concat!("let four = ", stringify!($Ty), "::new(4)?;")] + #[doc = concat!("let max = ", stringify!($Ty), "::new(", + stringify!($Int), "::MAX)?;")] + /// + /// assert_eq!(Some(two), two.checked_next_power_of_two() ); + /// assert_eq!(Some(four), three.checked_next_power_of_two() ); + /// assert_eq!(None, max.checked_next_power_of_two() ); + /// # Some(()) + /// # } + /// ``` + #[stable(feature = "nonzero_checked_ops", since = "1.64.0")] + #[rustc_const_stable(feature = "const_nonzero_checked_ops", since = "1.64.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_next_power_of_two(self) -> Option<$Ty> { + if let Some(nz) = self.get().checked_next_power_of_two() { + // SAFETY: The next power of two is positive + // and overflow is checked. + Some(unsafe { $Ty::new_unchecked(nz) }) + } else { + None + } + } + + /// Returns the base 2 logarithm of the number, rounded down. + /// + /// This is the same operation as + #[doc = concat!("[`", stringify!($Int), "::ilog2`],")] + /// except that it has no failure cases to worry about + /// since this value can never be zero. + /// + /// # Examples + /// + /// ``` + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + #[doc = concat!("assert_eq!(", stringify!($Ty), "::new(7).unwrap().ilog2(), 2);")] + #[doc = concat!("assert_eq!(", stringify!($Ty), "::new(8).unwrap().ilog2(), 3);")] + #[doc = concat!("assert_eq!(", stringify!($Ty), "::new(9).unwrap().ilog2(), 3);")] + /// ``` + #[stable(feature = "int_log", since = "1.67.0")] + #[rustc_const_stable(feature = "int_log", since = "1.67.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn ilog2(self) -> u32 { + Self::BITS - 1 - self.leading_zeros() + } + + /// Returns the base 10 logarithm of the number, rounded down. + /// + /// This is the same operation as + #[doc = concat!("[`", stringify!($Int), "::ilog10`],")] + /// except that it has no failure cases to worry about + /// since this value can never be zero. + /// + /// # Examples + /// + /// ``` + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + #[doc = concat!("assert_eq!(", stringify!($Ty), "::new(99).unwrap().ilog10(), 1);")] + #[doc = concat!("assert_eq!(", stringify!($Ty), "::new(100).unwrap().ilog10(), 2);")] + #[doc = concat!("assert_eq!(", stringify!($Ty), "::new(101).unwrap().ilog10(), 2);")] + /// ``` + #[stable(feature = "int_log", since = "1.67.0")] + #[rustc_const_stable(feature = "int_log", since = "1.67.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn ilog10(self) -> u32 { + super::int_log10::$Int(self.0) + } + } + )+ + } +} + +nonzero_unsigned_operations! { + NonZeroU8(u8); + NonZeroU16(u16); + NonZeroU32(u32); + NonZeroU64(u64); + NonZeroU128(u128); + NonZeroUsize(usize); +} + +// A bunch of methods for signed nonzero types only. +macro_rules! nonzero_signed_operations { + ( $( $Ty: ident($Int: ty) -> $Uty: ident($Uint: ty); )+ ) => { + $( + impl $Ty { + /// Computes the absolute value of self. + #[doc = concat!("See [`", stringify!($Int), "::abs`]")] + /// for documentation on overflow behaviour. + /// + /// # Example + /// + /// ``` + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + /// # fn main() { test().unwrap(); } + /// # fn test() -> Option<()> { + #[doc = concat!("let pos = ", stringify!($Ty), "::new(1)?;")] + #[doc = concat!("let neg = ", stringify!($Ty), "::new(-1)?;")] + /// + /// assert_eq!(pos, pos.abs()); + /// assert_eq!(pos, neg.abs()); + /// # Some(()) + /// # } + /// ``` + #[stable(feature = "nonzero_checked_ops", since = "1.64.0")] + #[rustc_const_stable(feature = "const_nonzero_checked_ops", since = "1.64.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn abs(self) -> $Ty { + // SAFETY: This cannot overflow to zero. + unsafe { $Ty::new_unchecked(self.get().abs()) } + } + + /// Checked absolute value. + /// Checks for overflow and returns [`None`] if + #[doc = concat!("`self == ", stringify!($Int), "::MIN`.")] + /// The result cannot be zero. + /// + /// # Example + /// + /// ``` + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + /// # fn main() { test().unwrap(); } + /// # fn test() -> Option<()> { + #[doc = concat!("let pos = ", stringify!($Ty), "::new(1)?;")] + #[doc = concat!("let neg = ", stringify!($Ty), "::new(-1)?;")] + #[doc = concat!("let min = ", stringify!($Ty), "::new(", + stringify!($Int), "::MIN)?;")] + /// + /// assert_eq!(Some(pos), neg.checked_abs()); + /// assert_eq!(None, min.checked_abs()); + /// # Some(()) + /// # } + /// ``` + #[stable(feature = "nonzero_checked_ops", since = "1.64.0")] + #[rustc_const_stable(feature = "const_nonzero_checked_ops", since = "1.64.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_abs(self) -> Option<$Ty> { + if let Some(nz) = self.get().checked_abs() { + // SAFETY: absolute value of nonzero cannot yield zero values. + Some(unsafe { $Ty::new_unchecked(nz) }) + } else { + None + } + } + + /// Computes the absolute value of self, + /// with overflow information, see + #[doc = concat!("[`", stringify!($Int), "::overflowing_abs`].")] + /// + /// # Example + /// + /// ``` + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + /// # fn main() { test().unwrap(); } + /// # fn test() -> Option<()> { + #[doc = concat!("let pos = ", stringify!($Ty), "::new(1)?;")] + #[doc = concat!("let neg = ", stringify!($Ty), "::new(-1)?;")] + #[doc = concat!("let min = ", stringify!($Ty), "::new(", + stringify!($Int), "::MIN)?;")] + /// + /// assert_eq!((pos, false), pos.overflowing_abs()); + /// assert_eq!((pos, false), neg.overflowing_abs()); + /// assert_eq!((min, true), min.overflowing_abs()); + /// # Some(()) + /// # } + /// ``` + #[stable(feature = "nonzero_checked_ops", since = "1.64.0")] + #[rustc_const_stable(feature = "const_nonzero_checked_ops", since = "1.64.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn overflowing_abs(self) -> ($Ty, bool) { + let (nz, flag) = self.get().overflowing_abs(); + ( + // SAFETY: absolute value of nonzero cannot yield zero values. + unsafe { $Ty::new_unchecked(nz) }, + flag, + ) + } + + /// Saturating absolute value, see + #[doc = concat!("[`", stringify!($Int), "::saturating_abs`].")] + /// + /// # Example + /// + /// ``` + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + /// # fn main() { test().unwrap(); } + /// # fn test() -> Option<()> { + #[doc = concat!("let pos = ", stringify!($Ty), "::new(1)?;")] + #[doc = concat!("let neg = ", stringify!($Ty), "::new(-1)?;")] + #[doc = concat!("let min = ", stringify!($Ty), "::new(", + stringify!($Int), "::MIN)?;")] + #[doc = concat!("let min_plus = ", stringify!($Ty), "::new(", + stringify!($Int), "::MIN + 1)?;")] + #[doc = concat!("let max = ", stringify!($Ty), "::new(", + stringify!($Int), "::MAX)?;")] + /// + /// assert_eq!(pos, pos.saturating_abs()); + /// assert_eq!(pos, neg.saturating_abs()); + /// assert_eq!(max, min.saturating_abs()); + /// assert_eq!(max, min_plus.saturating_abs()); + /// # Some(()) + /// # } + /// ``` + #[stable(feature = "nonzero_checked_ops", since = "1.64.0")] + #[rustc_const_stable(feature = "const_nonzero_checked_ops", since = "1.64.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn saturating_abs(self) -> $Ty { + // SAFETY: absolute value of nonzero cannot yield zero values. + unsafe { $Ty::new_unchecked(self.get().saturating_abs()) } + } + + /// Wrapping absolute value, see + #[doc = concat!("[`", stringify!($Int), "::wrapping_abs`].")] + /// + /// # Example + /// + /// ``` + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + /// # fn main() { test().unwrap(); } + /// # fn test() -> Option<()> { + #[doc = concat!("let pos = ", stringify!($Ty), "::new(1)?;")] + #[doc = concat!("let neg = ", stringify!($Ty), "::new(-1)?;")] + #[doc = concat!("let min = ", stringify!($Ty), "::new(", + stringify!($Int), "::MIN)?;")] + #[doc = concat!("# let max = ", stringify!($Ty), "::new(", + stringify!($Int), "::MAX)?;")] + /// + /// assert_eq!(pos, pos.wrapping_abs()); + /// assert_eq!(pos, neg.wrapping_abs()); + /// assert_eq!(min, min.wrapping_abs()); + /// # // FIXME: add once Neg is implemented? + /// # // assert_eq!(max, (-max).wrapping_abs()); + /// # Some(()) + /// # } + /// ``` + #[stable(feature = "nonzero_checked_ops", since = "1.64.0")] + #[rustc_const_stable(feature = "const_nonzero_checked_ops", since = "1.64.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn wrapping_abs(self) -> $Ty { + // SAFETY: absolute value of nonzero cannot yield zero values. + unsafe { $Ty::new_unchecked(self.get().wrapping_abs()) } + } + + /// Computes the absolute value of self + /// without any wrapping or panicking. + /// + /// # Example + /// + /// ``` + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + #[doc = concat!("# use std::num::", stringify!($Uty), ";")] + /// + /// # fn main() { test().unwrap(); } + /// # fn test() -> Option<()> { + #[doc = concat!("let u_pos = ", stringify!($Uty), "::new(1)?;")] + #[doc = concat!("let i_pos = ", stringify!($Ty), "::new(1)?;")] + #[doc = concat!("let i_neg = ", stringify!($Ty), "::new(-1)?;")] + #[doc = concat!("let i_min = ", stringify!($Ty), "::new(", + stringify!($Int), "::MIN)?;")] + #[doc = concat!("let u_max = ", stringify!($Uty), "::new(", + stringify!($Uint), "::MAX / 2 + 1)?;")] + /// + /// assert_eq!(u_pos, i_pos.unsigned_abs()); + /// assert_eq!(u_pos, i_neg.unsigned_abs()); + /// assert_eq!(u_max, i_min.unsigned_abs()); + /// # Some(()) + /// # } + /// ``` + #[stable(feature = "nonzero_checked_ops", since = "1.64.0")] + #[rustc_const_stable(feature = "const_nonzero_checked_ops", since = "1.64.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn unsigned_abs(self) -> $Uty { + // SAFETY: absolute value of nonzero cannot yield zero values. + unsafe { $Uty::new_unchecked(self.get().unsigned_abs()) } + } + + /// Returns `true` if `self` is negative and `false` if the + /// number is positive. + /// + /// # Example + /// + /// ``` + /// #![feature(nonzero_negation_ops)] + /// + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + /// # fn main() { test().unwrap(); } + /// # fn test() -> Option<()> { + #[doc = concat!("let pos_five = ", stringify!($Ty), "::new(5)?;")] + #[doc = concat!("let neg_five = ", stringify!($Ty), "::new(-5)?;")] + /// + /// assert!(neg_five.is_negative()); + /// assert!(!pos_five.is_negative()); + /// # Some(()) + /// # } + /// ``` + #[must_use] + #[inline] + #[unstable(feature = "nonzero_negation_ops", issue = "102443")] + pub const fn is_negative(self) -> bool { + self.get().is_negative() + } + + /// Checked negation. Computes `-self`, returning `None` if `self == i32::MIN`. + /// + /// # Example + /// + /// ``` + /// #![feature(nonzero_negation_ops)] + /// + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + /// # fn main() { test().unwrap(); } + /// # fn test() -> Option<()> { + #[doc = concat!("let pos_five = ", stringify!($Ty), "::new(5)?;")] + #[doc = concat!("let neg_five = ", stringify!($Ty), "::new(-5)?;")] + #[doc = concat!("let min = ", stringify!($Ty), "::new(", + stringify!($Int), "::MIN)?;")] + /// + /// assert_eq!(pos_five.checked_neg(), Some(neg_five)); + /// assert_eq!(min.checked_neg(), None); + /// # Some(()) + /// # } + /// ``` + #[inline] + #[unstable(feature = "nonzero_negation_ops", issue = "102443")] + pub const fn checked_neg(self) -> Option<$Ty> { + if let Some(result) = self.get().checked_neg() { + // SAFETY: negation of nonzero cannot yield zero values. + return Some(unsafe { $Ty::new_unchecked(result) }); + } + None + } + + /// Negates self, overflowing if this is equal to the minimum value. + /// + #[doc = concat!("See [`", stringify!($Int), "::overflowing_neg`]")] + /// for documentation on overflow behaviour. + /// + /// # Example + /// + /// ``` + /// #![feature(nonzero_negation_ops)] + /// + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + /// # fn main() { test().unwrap(); } + /// # fn test() -> Option<()> { + #[doc = concat!("let pos_five = ", stringify!($Ty), "::new(5)?;")] + #[doc = concat!("let neg_five = ", stringify!($Ty), "::new(-5)?;")] + #[doc = concat!("let min = ", stringify!($Ty), "::new(", + stringify!($Int), "::MIN)?;")] + /// + /// assert_eq!(pos_five.overflowing_neg(), (neg_five, false)); + /// assert_eq!(min.overflowing_neg(), (min, true)); + /// # Some(()) + /// # } + /// ``` + #[inline] + #[unstable(feature = "nonzero_negation_ops", issue = "102443")] + pub const fn overflowing_neg(self) -> ($Ty, bool) { + let (result, overflow) = self.get().overflowing_neg(); + // SAFETY: negation of nonzero cannot yield zero values. + ((unsafe { $Ty::new_unchecked(result) }), overflow) + } + + /// Saturating negation. Computes `-self`, returning `MAX` if + /// `self == i32::MIN` instead of overflowing. + /// + /// # Example + /// + /// ``` + /// #![feature(nonzero_negation_ops)] + /// + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + /// # fn main() { test().unwrap(); } + /// # fn test() -> Option<()> { + #[doc = concat!("let pos_five = ", stringify!($Ty), "::new(5)?;")] + #[doc = concat!("let neg_five = ", stringify!($Ty), "::new(-5)?;")] + #[doc = concat!("let min = ", stringify!($Ty), "::new(", + stringify!($Int), "::MIN)?;")] + #[doc = concat!("let min_plus_one = ", stringify!($Ty), "::new(", + stringify!($Int), "::MIN + 1)?;")] + #[doc = concat!("let max = ", stringify!($Ty), "::new(", + stringify!($Int), "::MAX)?;")] + /// + /// assert_eq!(pos_five.saturating_neg(), neg_five); + /// assert_eq!(min.saturating_neg(), max); + /// assert_eq!(max.saturating_neg(), min_plus_one); + /// # Some(()) + /// # } + /// ``` + #[inline] + #[unstable(feature = "nonzero_negation_ops", issue = "102443")] + pub const fn saturating_neg(self) -> $Ty { + if let Some(result) = self.checked_neg() { + return result; + } + $Ty::MAX + } + + /// Wrapping (modular) negation. Computes `-self`, wrapping around at the boundary + /// of the type. + /// + #[doc = concat!("See [`", stringify!($Int), "::wrapping_neg`]")] + /// for documentation on overflow behaviour. + /// + /// # Example + /// + /// ``` + /// #![feature(nonzero_negation_ops)] + /// + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + /// # fn main() { test().unwrap(); } + /// # fn test() -> Option<()> { + #[doc = concat!("let pos_five = ", stringify!($Ty), "::new(5)?;")] + #[doc = concat!("let neg_five = ", stringify!($Ty), "::new(-5)?;")] + #[doc = concat!("let min = ", stringify!($Ty), "::new(", + stringify!($Int), "::MIN)?;")] + /// + /// assert_eq!(pos_five.wrapping_neg(), neg_five); + /// assert_eq!(min.wrapping_neg(), min); + /// # Some(()) + /// # } + /// ``` + #[inline] + #[unstable(feature = "nonzero_negation_ops", issue = "102443")] + pub const fn wrapping_neg(self) -> $Ty { + let result = self.get().wrapping_neg(); + // SAFETY: negation of nonzero cannot yield zero values. + unsafe { $Ty::new_unchecked(result) } + } + } + )+ + } +} + +nonzero_signed_operations! { + NonZeroI8(i8) -> NonZeroU8(u8); + NonZeroI16(i16) -> NonZeroU16(u16); + NonZeroI32(i32) -> NonZeroU32(u32); + NonZeroI64(i64) -> NonZeroU64(u64); + NonZeroI128(i128) -> NonZeroU128(u128); + NonZeroIsize(isize) -> NonZeroUsize(usize); +} + +// A bunch of methods for both signed and unsigned nonzero types. +macro_rules! nonzero_unsigned_signed_operations { + ( $( $signedness:ident $Ty: ident($Int: ty); )+ ) => { + $( + impl $Ty { + /// Multiplies two non-zero integers together. + /// Checks for overflow and returns [`None`] on overflow. + /// As a consequence, the result cannot wrap to zero. + /// + /// # Examples + /// + /// ``` + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + /// # fn main() { test().unwrap(); } + /// # fn test() -> Option<()> { + #[doc = concat!("let two = ", stringify!($Ty), "::new(2)?;")] + #[doc = concat!("let four = ", stringify!($Ty), "::new(4)?;")] + #[doc = concat!("let max = ", stringify!($Ty), "::new(", + stringify!($Int), "::MAX)?;")] + /// + /// assert_eq!(Some(four), two.checked_mul(two)); + /// assert_eq!(None, max.checked_mul(two)); + /// # Some(()) + /// # } + /// ``` + #[stable(feature = "nonzero_checked_ops", since = "1.64.0")] + #[rustc_const_stable(feature = "const_nonzero_checked_ops", since = "1.64.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_mul(self, other: $Ty) -> Option<$Ty> { + if let Some(result) = self.get().checked_mul(other.get()) { + // SAFETY: checked_mul returns None on overflow + // and `other` is also non-null + // so the result cannot be zero. + Some(unsafe { $Ty::new_unchecked(result) }) + } else { + None + } + } + + /// Multiplies two non-zero integers together. + #[doc = concat!("Return [`", stringify!($Int), "::MAX`] on overflow.")] + /// + /// # Examples + /// + /// ``` + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + /// # fn main() { test().unwrap(); } + /// # fn test() -> Option<()> { + #[doc = concat!("let two = ", stringify!($Ty), "::new(2)?;")] + #[doc = concat!("let four = ", stringify!($Ty), "::new(4)?;")] + #[doc = concat!("let max = ", stringify!($Ty), "::new(", + stringify!($Int), "::MAX)?;")] + /// + /// assert_eq!(four, two.saturating_mul(two)); + /// assert_eq!(max, four.saturating_mul(max)); + /// # Some(()) + /// # } + /// ``` + #[stable(feature = "nonzero_checked_ops", since = "1.64.0")] + #[rustc_const_stable(feature = "const_nonzero_checked_ops", since = "1.64.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn saturating_mul(self, other: $Ty) -> $Ty { + // SAFETY: saturating_mul returns u*::MAX on overflow + // and `other` is also non-null + // so the result cannot be zero. + unsafe { $Ty::new_unchecked(self.get().saturating_mul(other.get())) } + } + + /// Multiplies two non-zero integers together, + /// assuming overflow cannot occur. + /// Overflow is unchecked, and it is undefined behaviour to overflow + /// *even if the result would wrap to a non-zero value*. + /// The behaviour is undefined as soon as + #[doc = sign_dependent_expr!{ + $signedness ? + if signed { + concat!("`self * rhs > ", stringify!($Int), "::MAX`, ", + "or `self * rhs < ", stringify!($Int), "::MIN`.") + } + if unsigned { + concat!("`self * rhs > ", stringify!($Int), "::MAX`.") + } + }] + /// + /// # Examples + /// + /// ``` + /// #![feature(nonzero_ops)] + /// + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + /// # fn main() { test().unwrap(); } + /// # fn test() -> Option<()> { + #[doc = concat!("let two = ", stringify!($Ty), "::new(2)?;")] + #[doc = concat!("let four = ", stringify!($Ty), "::new(4)?;")] + /// + /// assert_eq!(four, unsafe { two.unchecked_mul(two) }); + /// # Some(()) + /// # } + /// ``` + #[unstable(feature = "nonzero_ops", issue = "84186")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const unsafe fn unchecked_mul(self, other: $Ty) -> $Ty { + // SAFETY: The caller ensures there is no overflow. + unsafe { $Ty::new_unchecked(self.get().unchecked_mul(other.get())) } + } + + /// Raises non-zero value to an integer power. + /// Checks for overflow and returns [`None`] on overflow. + /// As a consequence, the result cannot wrap to zero. + /// + /// # Examples + /// + /// ``` + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + /// # fn main() { test().unwrap(); } + /// # fn test() -> Option<()> { + #[doc = concat!("let three = ", stringify!($Ty), "::new(3)?;")] + #[doc = concat!("let twenty_seven = ", stringify!($Ty), "::new(27)?;")] + #[doc = concat!("let half_max = ", stringify!($Ty), "::new(", + stringify!($Int), "::MAX / 2)?;")] + /// + /// assert_eq!(Some(twenty_seven), three.checked_pow(3)); + /// assert_eq!(None, half_max.checked_pow(3)); + /// # Some(()) + /// # } + /// ``` + #[stable(feature = "nonzero_checked_ops", since = "1.64.0")] + #[rustc_const_stable(feature = "const_nonzero_checked_ops", since = "1.64.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_pow(self, other: u32) -> Option<$Ty> { + if let Some(result) = self.get().checked_pow(other) { + // SAFETY: checked_pow returns None on overflow + // so the result cannot be zero. + Some(unsafe { $Ty::new_unchecked(result) }) + } else { + None + } + } + + /// Raise non-zero value to an integer power. + #[doc = sign_dependent_expr!{ + $signedness ? + if signed { + concat!("Return [`", stringify!($Int), "::MIN`] ", + "or [`", stringify!($Int), "::MAX`] on overflow.") + } + if unsigned { + concat!("Return [`", stringify!($Int), "::MAX`] on overflow.") + } + }] + /// + /// # Examples + /// + /// ``` + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + /// # fn main() { test().unwrap(); } + /// # fn test() -> Option<()> { + #[doc = concat!("let three = ", stringify!($Ty), "::new(3)?;")] + #[doc = concat!("let twenty_seven = ", stringify!($Ty), "::new(27)?;")] + #[doc = concat!("let max = ", stringify!($Ty), "::new(", + stringify!($Int), "::MAX)?;")] + /// + /// assert_eq!(twenty_seven, three.saturating_pow(3)); + /// assert_eq!(max, max.saturating_pow(3)); + /// # Some(()) + /// # } + /// ``` + #[stable(feature = "nonzero_checked_ops", since = "1.64.0")] + #[rustc_const_stable(feature = "const_nonzero_checked_ops", since = "1.64.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn saturating_pow(self, other: u32) -> $Ty { + // SAFETY: saturating_pow returns u*::MAX on overflow + // so the result cannot be zero. + unsafe { $Ty::new_unchecked(self.get().saturating_pow(other)) } + } + } + )+ + } +} + +// Use this when the generated code should differ between signed and unsigned types. +macro_rules! sign_dependent_expr { + (signed ? if signed { $signed_case:expr } if unsigned { $unsigned_case:expr } ) => { + $signed_case + }; + (unsigned ? if signed { $signed_case:expr } if unsigned { $unsigned_case:expr } ) => { + $unsigned_case + }; +} + +nonzero_unsigned_signed_operations! { + unsigned NonZeroU8(u8); + unsigned NonZeroU16(u16); + unsigned NonZeroU32(u32); + unsigned NonZeroU64(u64); + unsigned NonZeroU128(u128); + unsigned NonZeroUsize(usize); + signed NonZeroI8(i8); + signed NonZeroI16(i16); + signed NonZeroI32(i32); + signed NonZeroI64(i64); + signed NonZeroI128(i128); + signed NonZeroIsize(isize); +} + +macro_rules! nonzero_unsigned_is_power_of_two { + ( $( $Ty: ident )+ ) => { + $( + impl $Ty { + + /// Returns `true` if and only if `self == (1 << k)` for some `k`. + /// + /// On many architectures, this function can perform better than `is_power_of_two()` + /// on the underlying integer type, as special handling of zero can be avoided. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let eight = std::num::", stringify!($Ty), "::new(8).unwrap();")] + /// assert!(eight.is_power_of_two()); + #[doc = concat!("let ten = std::num::", stringify!($Ty), "::new(10).unwrap();")] + /// assert!(!ten.is_power_of_two()); + /// ``` + #[must_use] + #[stable(feature = "nonzero_is_power_of_two", since = "1.59.0")] + #[rustc_const_stable(feature = "nonzero_is_power_of_two", since = "1.59.0")] + #[inline] + pub const fn is_power_of_two(self) -> bool { + // LLVM 11 normalizes `unchecked_sub(x, 1) & x == 0` to the implementation seen here. + // On the basic x86-64 target, this saves 3 instructions for the zero check. + // On x86_64 with BMI1, being nonzero lets it codegen to `BLSR`, which saves an instruction + // compared to the `POPCNT` implementation on the underlying integer type. + + intrinsics::ctpop(self.get()) < 2 + } + + } + )+ + } +} + +nonzero_unsigned_is_power_of_two! { NonZeroU8 NonZeroU16 NonZeroU32 NonZeroU64 NonZeroU128 NonZeroUsize } + +macro_rules! nonzero_min_max_unsigned { + ( $( $Ty: ident($Int: ident); )+ ) => { + $( + impl $Ty { + /// The smallest value that can be represented by this non-zero + /// integer type, 1. + /// + /// # Examples + /// + /// ``` + /// #![feature(nonzero_min_max)] + /// + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + #[doc = concat!("assert_eq!(", stringify!($Ty), "::MIN.get(), 1", stringify!($Int), ");")] + /// ``` + #[unstable(feature = "nonzero_min_max", issue = "89065")] + pub const MIN: Self = Self::new(1).unwrap(); + + /// The largest value that can be represented by this non-zero + /// integer type, + #[doc = concat!("equal to [`", stringify!($Int), "::MAX`].")] + /// + /// # Examples + /// + /// ``` + /// #![feature(nonzero_min_max)] + /// + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + #[doc = concat!("assert_eq!(", stringify!($Ty), "::MAX.get(), ", stringify!($Int), "::MAX);")] + /// ``` + #[unstable(feature = "nonzero_min_max", issue = "89065")] + pub const MAX: Self = Self::new(<$Int>::MAX).unwrap(); + } + )+ + } +} + +macro_rules! nonzero_min_max_signed { + ( $( $Ty: ident($Int: ident); )+ ) => { + $( + impl $Ty { + /// The smallest value that can be represented by this non-zero + /// integer type, + #[doc = concat!("equal to [`", stringify!($Int), "::MIN`].")] + /// + /// Note: While most integer types are defined for every whole + /// number between `MIN` and `MAX`, signed non-zero integers are + /// a special case. They have a "gap" at 0. + /// + /// # Examples + /// + /// ``` + /// #![feature(nonzero_min_max)] + /// + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + #[doc = concat!("assert_eq!(", stringify!($Ty), "::MIN.get(), ", stringify!($Int), "::MIN);")] + /// ``` + #[unstable(feature = "nonzero_min_max", issue = "89065")] + pub const MIN: Self = Self::new(<$Int>::MIN).unwrap(); + + /// The largest value that can be represented by this non-zero + /// integer type, + #[doc = concat!("equal to [`", stringify!($Int), "::MAX`].")] + /// + /// Note: While most integer types are defined for every whole + /// number between `MIN` and `MAX`, signed non-zero integers are + /// a special case. They have a "gap" at 0. + /// + /// # Examples + /// + /// ``` + /// #![feature(nonzero_min_max)] + /// + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + #[doc = concat!("assert_eq!(", stringify!($Ty), "::MAX.get(), ", stringify!($Int), "::MAX);")] + /// ``` + #[unstable(feature = "nonzero_min_max", issue = "89065")] + pub const MAX: Self = Self::new(<$Int>::MAX).unwrap(); + } + )+ + } +} + +nonzero_min_max_unsigned! { + NonZeroU8(u8); + NonZeroU16(u16); + NonZeroU32(u32); + NonZeroU64(u64); + NonZeroU128(u128); + NonZeroUsize(usize); +} + +nonzero_min_max_signed! { + NonZeroI8(i8); + NonZeroI16(i16); + NonZeroI32(i32); + NonZeroI64(i64); + NonZeroI128(i128); + NonZeroIsize(isize); +} + +macro_rules! nonzero_bits { + ( $( $Ty: ident($Int: ty); )+ ) => { + $( + impl $Ty { + /// The size of this non-zero integer type in bits. + /// + #[doc = concat!("This value is equal to [`", stringify!($Int), "::BITS`].")] + /// + /// # Examples + /// + /// ``` + #[doc = concat!("# use std::num::", stringify!($Ty), ";")] + /// + #[doc = concat!("assert_eq!(", stringify!($Ty), "::BITS, ", stringify!($Int), "::BITS);")] + /// ``` + #[stable(feature = "nonzero_bits", since = "1.67.0")] + pub const BITS: u32 = <$Int>::BITS; + } + )+ + } +} + +nonzero_bits! { + NonZeroU8(u8); + NonZeroI8(i8); + NonZeroU16(u16); + NonZeroI16(i16); + NonZeroU32(u32); + NonZeroI32(i32); + NonZeroU64(u64); + NonZeroI64(i64); + NonZeroU128(u128); + NonZeroI128(i128); + NonZeroUsize(usize); + NonZeroIsize(isize); +} diff --git a/crux-mir/lib/core/src/num/saturating.rs b/crux-mir/lib/core/src/num/saturating.rs new file mode 100644 index 000000000..8982473b2 --- /dev/null +++ b/crux-mir/lib/core/src/num/saturating.rs @@ -0,0 +1,1081 @@ +//! Definitions of `Saturating`. + +use crate::fmt; +use crate::ops::{Add, AddAssign, BitAnd, BitAndAssign, BitOr, BitOrAssign}; +use crate::ops::{BitXor, BitXorAssign, Div, DivAssign}; +use crate::ops::{Mul, MulAssign, Neg, Not, Rem, RemAssign}; +use crate::ops::{Shl, ShlAssign, Shr, ShrAssign, Sub, SubAssign}; + +/// Provides intentionally-saturating arithmetic on `T`. +/// +/// Operations like `+` on `u32` values are intended to never overflow, +/// and in some debug configurations overflow is detected and results +/// in a panic. While most arithmetic falls into this category, some +/// code explicitly expects and relies upon saturating arithmetic. +/// +/// Saturating arithmetic can be achieved either through methods like +/// `saturating_add`, or through the `Saturating` type, which says that +/// all standard arithmetic operations on the underlying value are +/// intended to have saturating semantics. +/// +/// The underlying value can be retrieved through the `.0` index of the +/// `Saturating` tuple. +/// +/// # Examples +/// +/// ``` +/// #![feature(saturating_int_impl)] +/// use std::num::Saturating; +/// +/// let max = Saturating(u32::MAX); +/// let one = Saturating(1u32); +/// +/// assert_eq!(u32::MAX, (max + one).0); +/// ``` +#[unstable(feature = "saturating_int_impl", issue = "87920")] +#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default, Hash)] +#[repr(transparent)] +pub struct Saturating(#[unstable(feature = "saturating_int_impl", issue = "87920")] pub T); + +#[unstable(feature = "saturating_int_impl", issue = "87920")] +impl fmt::Debug for Saturating { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +#[unstable(feature = "saturating_int_impl", issue = "87920")] +impl fmt::Display for Saturating { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +#[unstable(feature = "saturating_int_impl", issue = "87920")] +impl fmt::Binary for Saturating { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +#[unstable(feature = "saturating_int_impl", issue = "87920")] +impl fmt::Octal for Saturating { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +#[unstable(feature = "saturating_int_impl", issue = "87920")] +impl fmt::LowerHex for Saturating { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +#[unstable(feature = "saturating_int_impl", issue = "87920")] +impl fmt::UpperHex for Saturating { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} +#[allow(unused_macros)] +macro_rules! sh_impl_signed { + ($t:ident, $f:ident) => { + // FIXME what is the correct implementation here? see discussion https://github.com/rust-lang/rust/pull/87921#discussion_r695870065 + // + // #[unstable(feature = "saturating_int_impl", issue = "87920")] + // impl Shl<$f> for Saturating<$t> { + // type Output = Saturating<$t>; + // + // #[inline] + // fn shl(self, other: $f) -> Saturating<$t> { + // if other < 0 { + // Saturating(self.0.shr((-other & self::shift_max::$t as $f) as u32)) + // } else { + // Saturating(self.0.shl((other & self::shift_max::$t as $f) as u32)) + // } + // } + // } + // forward_ref_binop! { impl Shl, shl for Saturating<$t>, $f, + // #[unstable(feature = "saturating_int_impl", issue = "87920")] } + // + // #[unstable(feature = "saturating_int_impl", issue = "87920")] + // impl ShlAssign<$f> for Saturating<$t> { + // #[inline] + // fn shl_assign(&mut self, other: $f) { + // *self = *self << other; + // } + // } + // forward_ref_op_assign! { impl ShlAssign, shl_assign for Saturating<$t>, $f } + + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl Shr<$f> for Saturating<$t> { + type Output = Saturating<$t>; + + #[inline] + fn shr(self, other: $f) -> Saturating<$t> { + if other < 0 { + Saturating(self.0.shl((-other & self::shift_max::$t as $f) as u32)) + } else { + Saturating(self.0.shr((other & self::shift_max::$t as $f) as u32)) + } + } + } + forward_ref_binop! { impl Shr, shr for Saturating<$t>, $f, + #[unstable(feature = "saturating_int_impl", issue = "87920")] } + + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl ShrAssign<$f> for Saturating<$t> { + #[inline] + fn shr_assign(&mut self, other: $f) { + *self = *self >> other; + } + } + forward_ref_op_assign! { impl ShrAssign, shr_assign for Saturating<$t>, $f } + }; +} + +macro_rules! sh_impl_unsigned { + ($t:ident, $f:ident) => { + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl Shl<$f> for Saturating<$t> { + type Output = Saturating<$t>; + + #[inline] + fn shl(self, other: $f) -> Saturating<$t> { + Saturating(self.0.wrapping_shl(other as u32)) + } + } + forward_ref_binop! { impl Shl, shl for Saturating<$t>, $f, + #[unstable(feature = "saturating_int_impl", issue = "87920")] } + + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl ShlAssign<$f> for Saturating<$t> { + #[inline] + fn shl_assign(&mut self, other: $f) { + *self = *self << other; + } + } + forward_ref_op_assign! { impl ShlAssign, shl_assign for Saturating<$t>, $f } + + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl Shr<$f> for Saturating<$t> { + type Output = Saturating<$t>; + + #[inline] + fn shr(self, other: $f) -> Saturating<$t> { + Saturating(self.0.wrapping_shr(other as u32)) + } + } + forward_ref_binop! { impl Shr, shr for Saturating<$t>, $f, + #[unstable(feature = "saturating_int_impl", issue = "87920")] } + + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl ShrAssign<$f> for Saturating<$t> { + #[inline] + fn shr_assign(&mut self, other: $f) { + *self = *self >> other; + } + } + forward_ref_op_assign! { impl ShrAssign, shr_assign for Saturating<$t>, $f } + }; +} + +// FIXME (#23545): uncomment the remaining impls +macro_rules! sh_impl_all { + ($($t:ident)*) => ($( + //sh_impl_unsigned! { $t, u8 } + //sh_impl_unsigned! { $t, u16 } + //sh_impl_unsigned! { $t, u32 } + //sh_impl_unsigned! { $t, u64 } + //sh_impl_unsigned! { $t, u128 } + sh_impl_unsigned! { $t, usize } + + //sh_impl_signed! { $t, i8 } + //sh_impl_signed! { $t, i16 } + //sh_impl_signed! { $t, i32 } + //sh_impl_signed! { $t, i64 } + //sh_impl_signed! { $t, i128 } + //sh_impl_signed! { $t, isize } + )*) +} + +sh_impl_all! { u8 u16 u32 u64 u128 usize i8 i16 i32 i64 i128 isize } + +// FIXME(30524): impl Op for Saturating, impl OpAssign for Saturating +macro_rules! saturating_impl { + ($($t:ty)*) => ($( + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl Add for Saturating<$t> { + type Output = Saturating<$t>; + + #[inline] + fn add(self, other: Saturating<$t>) -> Saturating<$t> { + Saturating(self.0.saturating_add(other.0)) + } + } + forward_ref_binop! { impl Add, add for Saturating<$t>, Saturating<$t>, + #[unstable(feature = "saturating_int_impl", issue = "87920")] } + + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl AddAssign for Saturating<$t> { + #[inline] + fn add_assign(&mut self, other: Saturating<$t>) { + *self = *self + other; + } + } + forward_ref_op_assign! { impl AddAssign, add_assign for Saturating<$t>, Saturating<$t> } + + #[unstable(feature = "saturating_int_assign_impl", issue = "92354")] + impl AddAssign<$t> for Saturating<$t> { + #[inline] + fn add_assign(&mut self, other: $t) { + *self = *self + Saturating(other); + } + } + forward_ref_op_assign! { impl AddAssign, add_assign for Saturating<$t>, $t } + + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl Sub for Saturating<$t> { + type Output = Saturating<$t>; + + #[inline] + fn sub(self, other: Saturating<$t>) -> Saturating<$t> { + Saturating(self.0.saturating_sub(other.0)) + } + } + forward_ref_binop! { impl Sub, sub for Saturating<$t>, Saturating<$t>, + #[unstable(feature = "saturating_int_impl", issue = "87920")] } + + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl SubAssign for Saturating<$t> { + #[inline] + fn sub_assign(&mut self, other: Saturating<$t>) { + *self = *self - other; + } + } + forward_ref_op_assign! { impl SubAssign, sub_assign for Saturating<$t>, Saturating<$t> } + + #[unstable(feature = "saturating_int_assign_impl", issue = "92354")] + impl SubAssign<$t> for Saturating<$t> { + #[inline] + fn sub_assign(&mut self, other: $t) { + *self = *self - Saturating(other); + } + } + forward_ref_op_assign! { impl SubAssign, sub_assign for Saturating<$t>, $t } + + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl Mul for Saturating<$t> { + type Output = Saturating<$t>; + + #[inline] + fn mul(self, other: Saturating<$t>) -> Saturating<$t> { + Saturating(self.0.saturating_mul(other.0)) + } + } + forward_ref_binop! { impl Mul, mul for Saturating<$t>, Saturating<$t>, + #[unstable(feature = "saturating_int_impl", issue = "87920")] } + + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl MulAssign for Saturating<$t> { + #[inline] + fn mul_assign(&mut self, other: Saturating<$t>) { + *self = *self * other; + } + } + forward_ref_op_assign! { impl MulAssign, mul_assign for Saturating<$t>, Saturating<$t> } + + #[unstable(feature = "saturating_int_assign_impl", issue = "92354")] + impl MulAssign<$t> for Saturating<$t> { + #[inline] + fn mul_assign(&mut self, other: $t) { + *self = *self * Saturating(other); + } + } + forward_ref_op_assign! { impl MulAssign, mul_assign for Saturating<$t>, $t } + + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + #[doc = concat!("assert_eq!(Saturating(2", stringify!($t), "), Saturating(5", stringify!($t), ") / Saturating(2));")] + #[doc = concat!("assert_eq!(Saturating(", stringify!($t), "::MAX), Saturating(", stringify!($t), "::MAX) / Saturating(1));")] + #[doc = concat!("assert_eq!(Saturating(", stringify!($t), "::MIN), Saturating(", stringify!($t), "::MIN) / Saturating(1));")] + /// ``` + /// + /// ```should_panic + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + #[doc = concat!("let _ = Saturating(0", stringify!($t), ") / Saturating(0);")] + /// ``` + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl Div for Saturating<$t> { + type Output = Saturating<$t>; + + #[inline] + fn div(self, other: Saturating<$t>) -> Saturating<$t> { + Saturating(self.0.saturating_div(other.0)) + } + } + forward_ref_binop! { impl Div, div for Saturating<$t>, Saturating<$t>, + #[unstable(feature = "saturating_int_impl", issue = "87920")] } + + + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl DivAssign for Saturating<$t> { + #[inline] + fn div_assign(&mut self, other: Saturating<$t>) { + *self = *self / other; + } + } + forward_ref_op_assign! { impl DivAssign, div_assign for Saturating<$t>, Saturating<$t> } + + #[unstable(feature = "saturating_int_assign_impl", issue = "92354")] + impl DivAssign<$t> for Saturating<$t> { + #[inline] + fn div_assign(&mut self, other: $t) { + *self = *self / Saturating(other); + } + } + forward_ref_op_assign! { impl DivAssign, div_assign for Saturating<$t>, $t } + + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl Rem for Saturating<$t> { + type Output = Saturating<$t>; + + #[inline] + fn rem(self, other: Saturating<$t>) -> Saturating<$t> { + Saturating(self.0.rem(other.0)) + } + } + forward_ref_binop! { impl Rem, rem for Saturating<$t>, Saturating<$t>, + #[unstable(feature = "saturating_int_impl", issue = "87920")] } + + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl RemAssign for Saturating<$t> { + #[inline] + fn rem_assign(&mut self, other: Saturating<$t>) { + *self = *self % other; + } + } + forward_ref_op_assign! { impl RemAssign, rem_assign for Saturating<$t>, Saturating<$t> } + + #[unstable(feature = "saturating_int_assign_impl", issue = "92354")] + impl RemAssign<$t> for Saturating<$t> { + #[inline] + fn rem_assign(&mut self, other: $t) { + *self = *self % Saturating(other); + } + } + forward_ref_op_assign! { impl RemAssign, rem_assign for Saturating<$t>, $t } + + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl Not for Saturating<$t> { + type Output = Saturating<$t>; + + #[inline] + fn not(self) -> Saturating<$t> { + Saturating(!self.0) + } + } + forward_ref_unop! { impl Not, not for Saturating<$t>, + #[unstable(feature = "saturating_int_impl", issue = "87920")] } + + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl BitXor for Saturating<$t> { + type Output = Saturating<$t>; + + #[inline] + fn bitxor(self, other: Saturating<$t>) -> Saturating<$t> { + Saturating(self.0 ^ other.0) + } + } + forward_ref_binop! { impl BitXor, bitxor for Saturating<$t>, Saturating<$t>, + #[unstable(feature = "saturating_int_impl", issue = "87920")] } + + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl BitXorAssign for Saturating<$t> { + #[inline] + fn bitxor_assign(&mut self, other: Saturating<$t>) { + *self = *self ^ other; + } + } + forward_ref_op_assign! { impl BitXorAssign, bitxor_assign for Saturating<$t>, Saturating<$t> } + + #[unstable(feature = "saturating_int_assign_impl", issue = "92354")] + impl BitXorAssign<$t> for Saturating<$t> { + #[inline] + fn bitxor_assign(&mut self, other: $t) { + *self = *self ^ Saturating(other); + } + } + forward_ref_op_assign! { impl BitXorAssign, bitxor_assign for Saturating<$t>, $t } + + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl BitOr for Saturating<$t> { + type Output = Saturating<$t>; + + #[inline] + fn bitor(self, other: Saturating<$t>) -> Saturating<$t> { + Saturating(self.0 | other.0) + } + } + forward_ref_binop! { impl BitOr, bitor for Saturating<$t>, Saturating<$t>, + #[unstable(feature = "saturating_int_impl", issue = "87920")] } + + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl BitOrAssign for Saturating<$t> { + #[inline] + fn bitor_assign(&mut self, other: Saturating<$t>) { + *self = *self | other; + } + } + forward_ref_op_assign! { impl BitOrAssign, bitor_assign for Saturating<$t>, Saturating<$t> } + + #[unstable(feature = "saturating_int_assign_impl", issue = "92354")] + impl BitOrAssign<$t> for Saturating<$t> { + #[inline] + fn bitor_assign(&mut self, other: $t) { + *self = *self | Saturating(other); + } + } + forward_ref_op_assign! { impl BitOrAssign, bitor_assign for Saturating<$t>, $t } + + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl BitAnd for Saturating<$t> { + type Output = Saturating<$t>; + + #[inline] + fn bitand(self, other: Saturating<$t>) -> Saturating<$t> { + Saturating(self.0 & other.0) + } + } + forward_ref_binop! { impl BitAnd, bitand for Saturating<$t>, Saturating<$t>, + #[unstable(feature = "saturating_int_impl", issue = "87920")] } + + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl BitAndAssign for Saturating<$t> { + #[inline] + fn bitand_assign(&mut self, other: Saturating<$t>) { + *self = *self & other; + } + } + forward_ref_op_assign! { impl BitAndAssign, bitand_assign for Saturating<$t>, Saturating<$t> } + + #[unstable(feature = "saturating_int_assign_impl", issue = "92354")] + impl BitAndAssign<$t> for Saturating<$t> { + #[inline] + fn bitand_assign(&mut self, other: $t) { + *self = *self & Saturating(other); + } + } + forward_ref_op_assign! { impl BitAndAssign, bitand_assign for Saturating<$t>, $t } + + )*) +} + +saturating_impl! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } + +macro_rules! saturating_int_impl { + ($($t:ty)*) => ($( + impl Saturating<$t> { + /// Returns the smallest value that can be represented by this integer type. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + #[doc = concat!("assert_eq!(>::MIN, Saturating(", stringify!($t), "::MIN));")] + /// ``` + #[unstable(feature = "saturating_int_impl", issue = "87920")] + pub const MIN: Self = Self(<$t>::MIN); + + /// Returns the largest value that can be represented by this integer type. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + #[doc = concat!("assert_eq!(>::MAX, Saturating(", stringify!($t), "::MAX));")] + /// ``` + #[unstable(feature = "saturating_int_impl", issue = "87920")] + pub const MAX: Self = Self(<$t>::MAX); + + /// Returns the size of this integer type in bits. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + #[doc = concat!("assert_eq!(>::BITS, ", stringify!($t), "::BITS);")] + /// ``` + #[unstable(feature = "saturating_int_impl", issue = "87920")] + pub const BITS: u32 = <$t>::BITS; + + /// Returns the number of ones in the binary representation of `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + #[doc = concat!("let n = Saturating(0b01001100", stringify!($t), ");")] + /// + /// assert_eq!(n.count_ones(), 3); + /// ``` + #[inline] + #[doc(alias = "popcount")] + #[doc(alias = "popcnt")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[unstable(feature = "saturating_int_impl", issue = "87920")] + pub const fn count_ones(self) -> u32 { + self.0.count_ones() + } + + /// Returns the number of zeros in the binary representation of `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + #[doc = concat!("assert_eq!(Saturating(!0", stringify!($t), ").count_zeros(), 0);")] + /// ``` + #[inline] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[unstable(feature = "saturating_int_impl", issue = "87920")] + pub const fn count_zeros(self) -> u32 { + self.0.count_zeros() + } + + /// Returns the number of trailing zeros in the binary representation of `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + #[doc = concat!("let n = Saturating(0b0101000", stringify!($t), ");")] + /// + /// assert_eq!(n.trailing_zeros(), 3); + /// ``` + #[inline] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[unstable(feature = "saturating_int_impl", issue = "87920")] + pub const fn trailing_zeros(self) -> u32 { + self.0.trailing_zeros() + } + + /// Shifts the bits to the left by a specified amount, `n`, + /// saturating the truncated bits to the end of the resulting + /// integer. + /// + /// Please note this isn't the same operation as the `<<` shifting + /// operator! + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + /// let n: Saturating = Saturating(0x0123456789ABCDEF); + /// let m: Saturating = Saturating(-0x76543210FEDCBA99); + /// + /// assert_eq!(n.rotate_left(32), m); + /// ``` + #[inline] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[unstable(feature = "saturating_int_impl", issue = "87920")] + pub const fn rotate_left(self, n: u32) -> Self { + Saturating(self.0.rotate_left(n)) + } + + /// Shifts the bits to the right by a specified amount, `n`, + /// saturating the truncated bits to the beginning of the resulting + /// integer. + /// + /// Please note this isn't the same operation as the `>>` shifting + /// operator! + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + /// let n: Saturating = Saturating(0x0123456789ABCDEF); + /// let m: Saturating = Saturating(-0xFEDCBA987654322); + /// + /// assert_eq!(n.rotate_right(4), m); + /// ``` + #[inline] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[unstable(feature = "saturating_int_impl", issue = "87920")] + pub const fn rotate_right(self, n: u32) -> Self { + Saturating(self.0.rotate_right(n)) + } + + /// Reverses the byte order of the integer. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + /// let n: Saturating = Saturating(0b0000000_01010101); + /// assert_eq!(n, Saturating(85)); + /// + /// let m = n.swap_bytes(); + /// + /// assert_eq!(m, Saturating(0b01010101_00000000)); + /// assert_eq!(m, Saturating(21760)); + /// ``` + #[inline] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[unstable(feature = "saturating_int_impl", issue = "87920")] + pub const fn swap_bytes(self) -> Self { + Saturating(self.0.swap_bytes()) + } + + /// Reverses the bit pattern of the integer. + /// + /// # Examples + /// + /// Please note that this example is shared between integer types. + /// Which explains why `i16` is used here. + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + /// let n = Saturating(0b0000000_01010101i16); + /// assert_eq!(n, Saturating(85)); + /// + /// let m = n.reverse_bits(); + /// + /// assert_eq!(m.0 as u16, 0b10101010_00000000); + /// assert_eq!(m, Saturating(-22016)); + /// ``` + #[inline] + #[unstable(feature = "saturating_int_impl", issue = "87920")] + #[rustc_const_unstable(feature = "saturating_int_impl", issue = "87920")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + pub const fn reverse_bits(self) -> Self { + Saturating(self.0.reverse_bits()) + } + + /// Converts an integer from big endian to the target's endianness. + /// + /// On big endian this is a no-op. On little endian the bytes are + /// swapped. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + #[doc = concat!("let n = Saturating(0x1A", stringify!($t), ");")] + /// + /// if cfg!(target_endian = "big") { + #[doc = concat!(" assert_eq!(>::from_be(n), n)")] + /// } else { + #[doc = concat!(" assert_eq!(>::from_be(n), n.swap_bytes())")] + /// } + /// ``` + #[inline] + #[must_use] + #[unstable(feature = "saturating_int_impl", issue = "87920")] + pub const fn from_be(x: Self) -> Self { + Saturating(<$t>::from_be(x.0)) + } + + /// Converts an integer from little endian to the target's endianness. + /// + /// On little endian this is a no-op. On big endian the bytes are + /// swapped. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + #[doc = concat!("let n = Saturating(0x1A", stringify!($t), ");")] + /// + /// if cfg!(target_endian = "little") { + #[doc = concat!(" assert_eq!(>::from_le(n), n)")] + /// } else { + #[doc = concat!(" assert_eq!(>::from_le(n), n.swap_bytes())")] + /// } + /// ``` + #[inline] + #[must_use] + #[unstable(feature = "saturating_int_impl", issue = "87920")] + pub const fn from_le(x: Self) -> Self { + Saturating(<$t>::from_le(x.0)) + } + + /// Converts `self` to big endian from the target's endianness. + /// + /// On big endian this is a no-op. On little endian the bytes are + /// swapped. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + #[doc = concat!("let n = Saturating(0x1A", stringify!($t), ");")] + /// + /// if cfg!(target_endian = "big") { + /// assert_eq!(n.to_be(), n) + /// } else { + /// assert_eq!(n.to_be(), n.swap_bytes()) + /// } + /// ``` + #[inline] + #[unstable(feature = "saturating_int_impl", issue = "87920")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + pub const fn to_be(self) -> Self { + Saturating(self.0.to_be()) + } + + /// Converts `self` to little endian from the target's endianness. + /// + /// On little endian this is a no-op. On big endian the bytes are + /// swapped. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + #[doc = concat!("let n = Saturating(0x1A", stringify!($t), ");")] + /// + /// if cfg!(target_endian = "little") { + /// assert_eq!(n.to_le(), n) + /// } else { + /// assert_eq!(n.to_le(), n.swap_bytes()) + /// } + /// ``` + #[inline] + #[unstable(feature = "saturating_int_impl", issue = "87920")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + pub const fn to_le(self) -> Self { + Saturating(self.0.to_le()) + } + + /// Raises self to the power of `exp`, using exponentiation by squaring. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + #[doc = concat!("assert_eq!(Saturating(3", stringify!($t), ").pow(4), Saturating(81));")] + /// ``` + /// + /// Results that are too large are saturated: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + /// assert_eq!(Saturating(3i8).pow(5), Saturating(127)); + /// assert_eq!(Saturating(3i8).pow(6), Saturating(127)); + /// ``` + #[inline] + #[unstable(feature = "saturating_int_impl", issue = "87920")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + pub fn pow(self, exp: u32) -> Self { + Saturating(self.0.saturating_pow(exp)) + } + } + )*) +} + +saturating_int_impl! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } + +macro_rules! saturating_int_impl_signed { + ($($t:ty)*) => ($( + impl Saturating<$t> { + /// Returns the number of leading zeros in the binary representation of `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + #[doc = concat!("let n = Saturating(", stringify!($t), "::MAX >> 2);")] + /// + /// assert_eq!(n.leading_zeros(), 3); + /// ``` + #[inline] + #[unstable(feature = "saturating_int_impl", issue = "87920")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + pub const fn leading_zeros(self) -> u32 { + self.0.leading_zeros() + } + + /// Saturating absolute value. Computes `self.abs()`, returning `MAX` if `self == MIN` + /// instead of overflowing. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + #[doc = concat!("assert_eq!(Saturating(100", stringify!($t), ").abs(), Saturating(100));")] + #[doc = concat!("assert_eq!(Saturating(-100", stringify!($t), ").abs(), Saturating(100));")] + #[doc = concat!("assert_eq!(Saturating(", stringify!($t), "::MIN).abs(), Saturating((", stringify!($t), "::MIN + 1).abs()));")] + #[doc = concat!("assert_eq!(Saturating(", stringify!($t), "::MIN).abs(), Saturating(", stringify!($t), "::MIN.saturating_abs()));")] + #[doc = concat!("assert_eq!(Saturating(", stringify!($t), "::MIN).abs(), Saturating(", stringify!($t), "::MAX));")] + /// ``` + #[inline] + #[unstable(feature = "saturating_int_impl", issue = "87920")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + pub fn abs(self) -> Saturating<$t> { + Saturating(self.0.saturating_abs()) + } + + /// Returns a number representing sign of `self`. + /// + /// - `0` if the number is zero + /// - `1` if the number is positive + /// - `-1` if the number is negative + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + #[doc = concat!("assert_eq!(Saturating(10", stringify!($t), ").signum(), Saturating(1));")] + #[doc = concat!("assert_eq!(Saturating(0", stringify!($t), ").signum(), Saturating(0));")] + #[doc = concat!("assert_eq!(Saturating(-10", stringify!($t), ").signum(), Saturating(-1));")] + /// ``` + #[inline] + #[unstable(feature = "saturating_int_impl", issue = "87920")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + pub fn signum(self) -> Saturating<$t> { + Saturating(self.0.signum()) + } + + /// Returns `true` if `self` is positive and `false` if the number is zero or + /// negative. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + #[doc = concat!("assert!(Saturating(10", stringify!($t), ").is_positive());")] + #[doc = concat!("assert!(!Saturating(-10", stringify!($t), ").is_positive());")] + /// ``` + #[must_use] + #[inline] + #[unstable(feature = "saturating_int_impl", issue = "87920")] + pub const fn is_positive(self) -> bool { + self.0.is_positive() + } + + /// Returns `true` if `self` is negative and `false` if the number is zero or + /// positive. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + #[doc = concat!("assert!(Saturating(-10", stringify!($t), ").is_negative());")] + #[doc = concat!("assert!(!Saturating(10", stringify!($t), ").is_negative());")] + /// ``` + #[must_use] + #[inline] + #[unstable(feature = "saturating_int_impl", issue = "87920")] + pub const fn is_negative(self) -> bool { + self.0.is_negative() + } + } + + #[unstable(feature = "saturating_int_impl", issue = "87920")] + impl Neg for Saturating<$t> { + type Output = Self; + #[inline] + fn neg(self) -> Self { + Saturating(self.0.saturating_neg()) + } + } + forward_ref_unop! { impl Neg, neg for Saturating<$t>, + #[unstable(feature = "saturating_int_impl", issue = "87920")] } + )*) +} + +saturating_int_impl_signed! { isize i8 i16 i32 i64 i128 } + +macro_rules! saturating_int_impl_unsigned { + ($($t:ty)*) => ($( + impl Saturating<$t> { + /// Returns the number of leading zeros in the binary representation of `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + #[doc = concat!("let n = Saturating(", stringify!($t), "::MAX >> 2);")] + /// + /// assert_eq!(n.leading_zeros(), 2); + /// ``` + #[inline] + #[unstable(feature = "saturating_int_impl", issue = "87920")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + pub const fn leading_zeros(self) -> u32 { + self.0.leading_zeros() + } + + /// Returns `true` if and only if `self == 2^k` for some `k`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(saturating_int_impl)] + /// use std::num::Saturating; + /// + #[doc = concat!("assert!(Saturating(16", stringify!($t), ").is_power_of_two());")] + #[doc = concat!("assert!(!Saturating(10", stringify!($t), ").is_power_of_two());")] + /// ``` + #[must_use] + #[inline] + #[unstable(feature = "saturating_int_impl", issue = "87920")] + pub fn is_power_of_two(self) -> bool { + self.0.is_power_of_two() + } + + } + )*) +} + +saturating_int_impl_unsigned! { usize u8 u16 u32 u64 u128 } + +// Related to potential Shl and ShlAssign implementation +// +// mod shift_max { +// #![allow(non_upper_case_globals)] +// +// #[cfg(target_pointer_width = "16")] +// mod platform { +// pub const usize: u32 = super::u16; +// pub const isize: u32 = super::i16; +// } +// +// #[cfg(target_pointer_width = "32")] +// mod platform { +// pub const usize: u32 = super::u32; +// pub const isize: u32 = super::i32; +// } +// +// #[cfg(target_pointer_width = "64")] +// mod platform { +// pub const usize: u32 = super::u64; +// pub const isize: u32 = super::i64; +// } +// +// pub const i8: u32 = (1 << 3) - 1; +// pub const i16: u32 = (1 << 4) - 1; +// pub const i32: u32 = (1 << 5) - 1; +// pub const i64: u32 = (1 << 6) - 1; +// pub const i128: u32 = (1 << 7) - 1; +// pub use self::platform::isize; +// +// pub const u8: u32 = i8; +// pub const u16: u32 = i16; +// pub const u32: u32 = i32; +// pub const u64: u32 = i64; +// pub const u128: u32 = i128; +// pub use self::platform::usize; +// } diff --git a/crux-mir/lib/core/src/num/shells/i128.rs b/crux-mir/lib/core/src/num/shells/i128.rs new file mode 100644 index 000000000..7b048dc52 --- /dev/null +++ b/crux-mir/lib/core/src/num/shells/i128.rs @@ -0,0 +1,13 @@ +//! Constants for the 128-bit signed integer type. +//! +//! *[See also the `i128` primitive type][i128].* +//! +//! New code should use the associated constants directly on the primitive type. + +#![stable(feature = "i128", since = "1.26.0")] +#![deprecated( + since = "TBD", + note = "all constants in this module replaced by associated constants on `i128`" +)] + +int_module! { i128, #[stable(feature = "i128", since="1.26.0")] } diff --git a/crux-mir/lib/core/src/num/shells/i16.rs b/crux-mir/lib/core/src/num/shells/i16.rs new file mode 100644 index 000000000..5c5812d5c --- /dev/null +++ b/crux-mir/lib/core/src/num/shells/i16.rs @@ -0,0 +1,13 @@ +//! Constants for the 16-bit signed integer type. +//! +//! *[See also the `i16` primitive type][i16].* +//! +//! New code should use the associated constants directly on the primitive type. + +#![stable(feature = "rust1", since = "1.0.0")] +#![deprecated( + since = "TBD", + note = "all constants in this module replaced by associated constants on `i16`" +)] + +int_module! { i16 } diff --git a/crux-mir/lib/core/src/num/shells/i32.rs b/crux-mir/lib/core/src/num/shells/i32.rs new file mode 100644 index 000000000..b283ac644 --- /dev/null +++ b/crux-mir/lib/core/src/num/shells/i32.rs @@ -0,0 +1,13 @@ +//! Constants for the 32-bit signed integer type. +//! +//! *[See also the `i32` primitive type][i32].* +//! +//! New code should use the associated constants directly on the primitive type. + +#![stable(feature = "rust1", since = "1.0.0")] +#![deprecated( + since = "TBD", + note = "all constants in this module replaced by associated constants on `i32`" +)] + +int_module! { i32 } diff --git a/crux-mir/lib/core/src/num/shells/i64.rs b/crux-mir/lib/core/src/num/shells/i64.rs new file mode 100644 index 000000000..a416fa7e9 --- /dev/null +++ b/crux-mir/lib/core/src/num/shells/i64.rs @@ -0,0 +1,13 @@ +//! Constants for the 64-bit signed integer type. +//! +//! *[See also the `i64` primitive type][i64].* +//! +//! New code should use the associated constants directly on the primitive type. + +#![stable(feature = "rust1", since = "1.0.0")] +#![deprecated( + since = "TBD", + note = "all constants in this module replaced by associated constants on `i64`" +)] + +int_module! { i64 } diff --git a/crux-mir/lib/core/src/num/shells/i8.rs b/crux-mir/lib/core/src/num/shells/i8.rs new file mode 100644 index 000000000..02465013a --- /dev/null +++ b/crux-mir/lib/core/src/num/shells/i8.rs @@ -0,0 +1,13 @@ +//! Constants for the 8-bit signed integer type. +//! +//! *[See also the `i8` primitive type][i8].* +//! +//! New code should use the associated constants directly on the primitive type. + +#![stable(feature = "rust1", since = "1.0.0")] +#![deprecated( + since = "TBD", + note = "all constants in this module replaced by associated constants on `i8`" +)] + +int_module! { i8 } diff --git a/crux-mir/lib/core/src/num/shells/int_macros.rs b/crux-mir/lib/core/src/num/shells/int_macros.rs new file mode 100644 index 000000000..2b1133e11 --- /dev/null +++ b/crux-mir/lib/core/src/num/shells/int_macros.rs @@ -0,0 +1,44 @@ +#![doc(hidden)] + +macro_rules! int_module { + ($T:ident) => (int_module!($T, #[stable(feature = "rust1", since = "1.0.0")]);); + ($T:ident, #[$attr:meta]) => ( + #[doc = concat!( + "The smallest value that can be represented by this integer type. Use ", + "[`", stringify!($T), "::MIN", "`] instead." + )] + /// + /// # Examples + /// + /// ```rust + /// // deprecated way + #[doc = concat!("let min = std::", stringify!($T), "::MIN;")] + /// + /// // intended way + #[doc = concat!("let min = ", stringify!($T), "::MIN;")] + /// ``` + /// + #[$attr] + #[deprecated(since = "TBD", note = "replaced by the `MIN` associated constant on this type")] + pub const MIN: $T = $T::MIN; + + #[doc = concat!( + "The largest value that can be represented by this integer type. Use ", + "[`", stringify!($T), "::MAX", "`] instead." + )] + /// + /// # Examples + /// + /// ```rust + /// // deprecated way + #[doc = concat!("let max = std::", stringify!($T), "::MAX;")] + /// + /// // intended way + #[doc = concat!("let max = ", stringify!($T), "::MAX;")] + /// ``` + /// + #[$attr] + #[deprecated(since = "TBD", note = "replaced by the `MAX` associated constant on this type")] + pub const MAX: $T = $T::MAX; + ) +} diff --git a/crux-mir/lib/core/src/num/shells/isize.rs b/crux-mir/lib/core/src/num/shells/isize.rs new file mode 100644 index 000000000..1579fbab6 --- /dev/null +++ b/crux-mir/lib/core/src/num/shells/isize.rs @@ -0,0 +1,13 @@ +//! Constants for the pointer-sized signed integer type. +//! +//! *[See also the `isize` primitive type][isize].* +//! +//! New code should use the associated constants directly on the primitive type. + +#![stable(feature = "rust1", since = "1.0.0")] +#![deprecated( + since = "TBD", + note = "all constants in this module replaced by associated constants on `isize`" +)] + +int_module! { isize } diff --git a/crux-mir/lib/core/src/num/shells/u128.rs b/crux-mir/lib/core/src/num/shells/u128.rs new file mode 100644 index 000000000..fe08cee58 --- /dev/null +++ b/crux-mir/lib/core/src/num/shells/u128.rs @@ -0,0 +1,13 @@ +//! Constants for the 128-bit unsigned integer type. +//! +//! *[See also the `u128` primitive type][u128].* +//! +//! New code should use the associated constants directly on the primitive type. + +#![stable(feature = "i128", since = "1.26.0")] +#![deprecated( + since = "TBD", + note = "all constants in this module replaced by associated constants on `u128`" +)] + +int_module! { u128, #[stable(feature = "i128", since="1.26.0")] } diff --git a/crux-mir/lib/core/src/num/shells/u16.rs b/crux-mir/lib/core/src/num/shells/u16.rs new file mode 100644 index 000000000..36f8c6978 --- /dev/null +++ b/crux-mir/lib/core/src/num/shells/u16.rs @@ -0,0 +1,13 @@ +//! Constants for the 16-bit unsigned integer type. +//! +//! *[See also the `u16` primitive type][u16].* +//! +//! New code should use the associated constants directly on the primitive type. + +#![stable(feature = "rust1", since = "1.0.0")] +#![deprecated( + since = "TBD", + note = "all constants in this module replaced by associated constants on `u16`" +)] + +int_module! { u16 } diff --git a/crux-mir/lib/core/src/num/shells/u32.rs b/crux-mir/lib/core/src/num/shells/u32.rs new file mode 100644 index 000000000..1c369097d --- /dev/null +++ b/crux-mir/lib/core/src/num/shells/u32.rs @@ -0,0 +1,13 @@ +//! Constants for the 32-bit unsigned integer type. +//! +//! *[See also the `u32` primitive type][u32].* +//! +//! New code should use the associated constants directly on the primitive type. + +#![stable(feature = "rust1", since = "1.0.0")] +#![deprecated( + since = "TBD", + note = "all constants in this module replaced by associated constants on `u32`" +)] + +int_module! { u32 } diff --git a/crux-mir/lib/core/src/num/shells/u64.rs b/crux-mir/lib/core/src/num/shells/u64.rs new file mode 100644 index 000000000..e8b691d15 --- /dev/null +++ b/crux-mir/lib/core/src/num/shells/u64.rs @@ -0,0 +1,13 @@ +//! Constants for the 64-bit unsigned integer type. +//! +//! *[See also the `u64` primitive type][u64].* +//! +//! New code should use the associated constants directly on the primitive type. + +#![stable(feature = "rust1", since = "1.0.0")] +#![deprecated( + since = "TBD", + note = "all constants in this module replaced by associated constants on `u64`" +)] + +int_module! { u64 } diff --git a/crux-mir/lib/core/src/num/shells/u8.rs b/crux-mir/lib/core/src/num/shells/u8.rs new file mode 100644 index 000000000..817c6a18a --- /dev/null +++ b/crux-mir/lib/core/src/num/shells/u8.rs @@ -0,0 +1,13 @@ +//! Constants for the 8-bit unsigned integer type. +//! +//! *[See also the `u8` primitive type][u8].* +//! +//! New code should use the associated constants directly on the primitive type. + +#![stable(feature = "rust1", since = "1.0.0")] +#![deprecated( + since = "TBD", + note = "all constants in this module replaced by associated constants on `u8`" +)] + +int_module! { u8 } diff --git a/crux-mir/lib/core/src/num/shells/usize.rs b/crux-mir/lib/core/src/num/shells/usize.rs new file mode 100644 index 000000000..3e1bec5ec --- /dev/null +++ b/crux-mir/lib/core/src/num/shells/usize.rs @@ -0,0 +1,13 @@ +//! Constants for the pointer-sized unsigned integer type. +//! +//! *[See also the `usize` primitive type][usize].* +//! +//! New code should use the associated constants directly on the primitive type. + +#![stable(feature = "rust1", since = "1.0.0")] +#![deprecated( + since = "TBD", + note = "all constants in this module replaced by associated constants on `usize`" +)] + +int_module! { usize } diff --git a/crux-mir/lib/core/src/num/u128.rs b/crux-mir/lib/core/src/num/u128.rs deleted file mode 100644 index dd45ff141..000000000 --- a/crux-mir/lib/core/src/num/u128.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! The 128-bit unsigned integer type. -//! -//! *[See also the `u128` primitive type](../../std/primitive.u128.html).* -//! -//! Although using these constants won’t cause compilation warnings, -//! new code should use the associated constants directly on the primitive type. - -#![stable(feature = "i128", since = "1.26.0")] -int_module! { u128, #[stable(feature = "i128", since="1.26.0")] } diff --git a/crux-mir/lib/core/src/num/u16.rs b/crux-mir/lib/core/src/num/u16.rs deleted file mode 100644 index 738071643..000000000 --- a/crux-mir/lib/core/src/num/u16.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! The 16-bit unsigned integer type. -//! -//! *[See also the `u16` primitive type](../../std/primitive.u16.html).* -//! -//! Although using these constants won’t cause compilation warnings, -//! new code should use the associated constants directly on the primitive type. - -#![stable(feature = "rust1", since = "1.0.0")] - -int_module! { u16 } diff --git a/crux-mir/lib/core/src/num/u32.rs b/crux-mir/lib/core/src/num/u32.rs deleted file mode 100644 index 9800c9099..000000000 --- a/crux-mir/lib/core/src/num/u32.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! The 32-bit unsigned integer type. -//! -//! *[See also the `u32` primitive type](../../std/primitive.u32.html).* -//! -//! Although using these constants won’t cause compilation warnings, -//! new code should use the associated constants directly on the primitive type. - -#![stable(feature = "rust1", since = "1.0.0")] - -int_module! { u32 } diff --git a/crux-mir/lib/core/src/num/u64.rs b/crux-mir/lib/core/src/num/u64.rs deleted file mode 100644 index fb686c396..000000000 --- a/crux-mir/lib/core/src/num/u64.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! The 64-bit unsigned integer type. -//! -//! *[See also the `u64` primitive type](../../std/primitive.u64.html).* -//! -//! Although using these constants won’t cause compilation warnings, -//! new code should use the associated constants directly on the primitive type. - -#![stable(feature = "rust1", since = "1.0.0")] - -int_module! { u64 } diff --git a/crux-mir/lib/core/src/num/u8.rs b/crux-mir/lib/core/src/num/u8.rs deleted file mode 100644 index c03cbdda2..000000000 --- a/crux-mir/lib/core/src/num/u8.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! The 8-bit unsigned integer type. -//! -//! *[See also the `u8` primitive type](../../std/primitive.u8.html).* -//! -//! Although using these constants won’t cause compilation warnings, -//! new code should use the associated constants directly on the primitive type. - -#![stable(feature = "rust1", since = "1.0.0")] - -int_module! { u8 } diff --git a/crux-mir/lib/core/src/num/uint_macros.rs b/crux-mir/lib/core/src/num/uint_macros.rs new file mode 100644 index 000000000..1c97c4686 --- /dev/null +++ b/crux-mir/lib/core/src/num/uint_macros.rs @@ -0,0 +1,2442 @@ +macro_rules! uint_impl { + ($SelfT:ty, $ActualT:ident, $SignedT:ident, $NonZeroT:ident, + $BITS:expr, $MaxV:expr, + $rot:expr, $rot_op:expr, $rot_result:expr, $swap_op:expr, $swapped:expr, + $reversed:expr, $le_bytes:expr, $be_bytes:expr, + $to_xe_bytes_doc:expr, $from_xe_bytes_doc:expr, + $bound_condition:expr) => { + /// The smallest value that can be represented by this integer type. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN, 0);")] + /// ``` + #[stable(feature = "assoc_int_consts", since = "1.43.0")] + pub const MIN: Self = 0; + + /// The largest value that can be represented by this integer type + #[doc = concat!("(2", $BITS, " − 1", $bound_condition, ")")] + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX, ", stringify!($MaxV), ");")] + /// ``` + #[stable(feature = "assoc_int_consts", since = "1.43.0")] + pub const MAX: Self = !0; + + /// The size of this integer type in bits. + /// + /// # Examples + /// + /// ``` + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::BITS, ", stringify!($BITS), ");")] + /// ``` + #[stable(feature = "int_bits_const", since = "1.53.0")] + pub const BITS: u32 = $BITS; + + /// Converts a string slice in a given base to an integer. + /// + /// The string is expected to be an optional `+` sign + /// followed by digits. + /// Leading and trailing whitespace represent an error. + /// Digits are a subset of these characters, depending on `radix`: + /// + /// * `0-9` + /// * `a-z` + /// * `A-Z` + /// + /// # Panics + /// + /// This function panics if `radix` is not in the range from 2 to 36. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::from_str_radix(\"A\", 16), Ok(10));")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn from_str_radix(src: &str, radix: u32) -> Result { + from_str_radix(src, radix) + } + + /// Returns the number of ones in the binary representation of `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = 0b01001100", stringify!($SelfT), ";")] + /// + /// assert_eq!(n.count_ones(), 3); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_math", since = "1.32.0")] + #[doc(alias = "popcount")] + #[doc(alias = "popcnt")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn count_ones(self) -> u32 { + intrinsics::ctpop(self as $ActualT) as u32 + } + + /// Returns the number of zeros in the binary representation of `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.count_zeros(), 0);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_math", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn count_zeros(self) -> u32 { + (!self).count_ones() + } + + /// Returns the number of leading zeros in the binary representation of `self`. + /// + /// Depending on what you're doing with the value, you might also be interested in the + /// [`ilog2`] function which returns a consistent number, even if the type widens. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = ", stringify!($SelfT), "::MAX >> 2;")] + /// + /// assert_eq!(n.leading_zeros(), 2); + /// ``` + #[doc = concat!("[`ilog2`]: ", stringify!($SelfT), "::ilog2")] + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_math", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn leading_zeros(self) -> u32 { + intrinsics::ctlz(self as $ActualT) as u32 + } + + /// Returns the number of trailing zeros in the binary representation + /// of `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = 0b0101000", stringify!($SelfT), ";")] + /// + /// assert_eq!(n.trailing_zeros(), 3); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_math", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn trailing_zeros(self) -> u32 { + intrinsics::cttz(self) as u32 + } + + /// Returns the number of leading ones in the binary representation of `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = !(", stringify!($SelfT), "::MAX >> 2);")] + /// + /// assert_eq!(n.leading_ones(), 2); + /// ``` + #[stable(feature = "leading_trailing_ones", since = "1.46.0")] + #[rustc_const_stable(feature = "leading_trailing_ones", since = "1.46.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn leading_ones(self) -> u32 { + (!self).leading_zeros() + } + + /// Returns the number of trailing ones in the binary representation + /// of `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = 0b1010111", stringify!($SelfT), ";")] + /// + /// assert_eq!(n.trailing_ones(), 3); + /// ``` + #[stable(feature = "leading_trailing_ones", since = "1.46.0")] + #[rustc_const_stable(feature = "leading_trailing_ones", since = "1.46.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn trailing_ones(self) -> u32 { + (!self).trailing_zeros() + } + + /// Shifts the bits to the left by a specified amount, `n`, + /// wrapping the truncated bits to the end of the resulting integer. + /// + /// Please note this isn't the same operation as the `<<` shifting operator! + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = ", $rot_op, stringify!($SelfT), ";")] + #[doc = concat!("let m = ", $rot_result, ";")] + /// + #[doc = concat!("assert_eq!(n.rotate_left(", $rot, "), m);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_math", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn rotate_left(self, n: u32) -> Self { + intrinsics::rotate_left(self, n as $SelfT) + } + + /// Shifts the bits to the right by a specified amount, `n`, + /// wrapping the truncated bits to the beginning of the resulting + /// integer. + /// + /// Please note this isn't the same operation as the `>>` shifting operator! + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = ", $rot_result, stringify!($SelfT), ";")] + #[doc = concat!("let m = ", $rot_op, ";")] + /// + #[doc = concat!("assert_eq!(n.rotate_right(", $rot, "), m);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_math", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn rotate_right(self, n: u32) -> Self { + intrinsics::rotate_right(self, n as $SelfT) + } + + /// Reverses the byte order of the integer. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = ", $swap_op, stringify!($SelfT), ";")] + /// let m = n.swap_bytes(); + /// + #[doc = concat!("assert_eq!(m, ", $swapped, ");")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_math", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn swap_bytes(self) -> Self { + intrinsics::bswap(self as $ActualT) as Self + } + + /// Reverses the order of bits in the integer. The least significant bit becomes the most significant bit, + /// second least-significant bit becomes second most-significant bit, etc. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = ", $swap_op, stringify!($SelfT), ";")] + /// let m = n.reverse_bits(); + /// + #[doc = concat!("assert_eq!(m, ", $reversed, ");")] + #[doc = concat!("assert_eq!(0, 0", stringify!($SelfT), ".reverse_bits());")] + /// ``` + #[stable(feature = "reverse_bits", since = "1.37.0")] + #[rustc_const_stable(feature = "reverse_bits", since = "1.37.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn reverse_bits(self) -> Self { + intrinsics::bitreverse(self as $ActualT) as Self + } + + /// Converts an integer from big endian to the target's endianness. + /// + /// On big endian this is a no-op. On little endian the bytes are + /// swapped. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = 0x1A", stringify!($SelfT), ";")] + /// + /// if cfg!(target_endian = "big") { + #[doc = concat!(" assert_eq!(", stringify!($SelfT), "::from_be(n), n)")] + /// } else { + #[doc = concat!(" assert_eq!(", stringify!($SelfT), "::from_be(n), n.swap_bytes())")] + /// } + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_math", since = "1.32.0")] + #[must_use] + #[inline(always)] + pub const fn from_be(x: Self) -> Self { + #[cfg(target_endian = "big")] + { + x + } + #[cfg(not(target_endian = "big"))] + { + x.swap_bytes() + } + } + + /// Converts an integer from little endian to the target's endianness. + /// + /// On little endian this is a no-op. On big endian the bytes are + /// swapped. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = 0x1A", stringify!($SelfT), ";")] + /// + /// if cfg!(target_endian = "little") { + #[doc = concat!(" assert_eq!(", stringify!($SelfT), "::from_le(n), n)")] + /// } else { + #[doc = concat!(" assert_eq!(", stringify!($SelfT), "::from_le(n), n.swap_bytes())")] + /// } + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_math", since = "1.32.0")] + #[must_use] + #[inline(always)] + pub const fn from_le(x: Self) -> Self { + #[cfg(target_endian = "little")] + { + x + } + #[cfg(not(target_endian = "little"))] + { + x.swap_bytes() + } + } + + /// Converts `self` to big endian from the target's endianness. + /// + /// On big endian this is a no-op. On little endian the bytes are + /// swapped. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = 0x1A", stringify!($SelfT), ";")] + /// + /// if cfg!(target_endian = "big") { + /// assert_eq!(n.to_be(), n) + /// } else { + /// assert_eq!(n.to_be(), n.swap_bytes()) + /// } + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_math", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn to_be(self) -> Self { // or not to be? + #[cfg(target_endian = "big")] + { + self + } + #[cfg(not(target_endian = "big"))] + { + self.swap_bytes() + } + } + + /// Converts `self` to little endian from the target's endianness. + /// + /// On little endian this is a no-op. On big endian the bytes are + /// swapped. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("let n = 0x1A", stringify!($SelfT), ";")] + /// + /// if cfg!(target_endian = "little") { + /// assert_eq!(n.to_le(), n) + /// } else { + /// assert_eq!(n.to_le(), n.swap_bytes()) + /// } + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_math", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn to_le(self) -> Self { + #[cfg(target_endian = "little")] + { + self + } + #[cfg(not(target_endian = "little"))] + { + self.swap_bytes() + } + } + + /// Checked integer addition. Computes `self + rhs`, returning `None` + /// if overflow occurred. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!( + "assert_eq!((", stringify!($SelfT), "::MAX - 2).checked_add(1), ", + "Some(", stringify!($SelfT), "::MAX - 1));" + )] + #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MAX - 2).checked_add(3), None);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_checked_int_methods", since = "1.47.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_add(self, rhs: Self) -> Option { + let (a, b) = self.overflowing_add(rhs); + if unlikely!(b) {None} else {Some(a)} + } + + /// Unchecked integer addition. Computes `self + rhs`, assuming overflow + /// cannot occur. + /// + /// # Safety + /// + /// This results in undefined behavior when + #[doc = concat!("`self + rhs > ", stringify!($SelfT), "::MAX` or `self + rhs < ", stringify!($SelfT), "::MIN`,")] + /// i.e. when [`checked_add`] would return `None`. + /// + #[doc = concat!("[`checked_add`]: ", stringify!($SelfT), "::checked_add")] + #[unstable( + feature = "unchecked_math", + reason = "niche optimization path", + issue = "85122", + )] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn unchecked_add(self, rhs: Self) -> Self { + // SAFETY: the caller must uphold the safety contract for + // `unchecked_add`. + unsafe { intrinsics::unchecked_add(self, rhs) } + } + + /// Checked addition with a signed integer. Computes `self + rhs`, + /// returning `None` if overflow occurred. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(1", stringify!($SelfT), ".checked_add_signed(2), Some(3));")] + #[doc = concat!("assert_eq!(1", stringify!($SelfT), ".checked_add_signed(-2), None);")] + #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MAX - 2).checked_add_signed(3), None);")] + /// ``` + #[stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[rustc_const_stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_add_signed(self, rhs: $SignedT) -> Option { + let (a, b) = self.overflowing_add_signed(rhs); + if unlikely!(b) {None} else {Some(a)} + } + + /// Checked integer subtraction. Computes `self - rhs`, returning + /// `None` if overflow occurred. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(1", stringify!($SelfT), ".checked_sub(1), Some(0));")] + #[doc = concat!("assert_eq!(0", stringify!($SelfT), ".checked_sub(1), None);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_checked_int_methods", since = "1.47.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_sub(self, rhs: Self) -> Option { + let (a, b) = self.overflowing_sub(rhs); + if unlikely!(b) {None} else {Some(a)} + } + + /// Unchecked integer subtraction. Computes `self - rhs`, assuming overflow + /// cannot occur. + /// + /// # Safety + /// + /// This results in undefined behavior when + #[doc = concat!("`self - rhs > ", stringify!($SelfT), "::MAX` or `self - rhs < ", stringify!($SelfT), "::MIN`,")] + /// i.e. when [`checked_sub`] would return `None`. + /// + #[doc = concat!("[`checked_sub`]: ", stringify!($SelfT), "::checked_sub")] + #[unstable( + feature = "unchecked_math", + reason = "niche optimization path", + issue = "85122", + )] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn unchecked_sub(self, rhs: Self) -> Self { + // SAFETY: the caller must uphold the safety contract for + // `unchecked_sub`. + unsafe { intrinsics::unchecked_sub(self, rhs) } + } + + /// Checked integer multiplication. Computes `self * rhs`, returning + /// `None` if overflow occurred. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".checked_mul(1), Some(5));")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.checked_mul(2), None);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_checked_int_methods", since = "1.47.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_mul(self, rhs: Self) -> Option { + let (a, b) = self.overflowing_mul(rhs); + if unlikely!(b) {None} else {Some(a)} + } + + /// Unchecked integer multiplication. Computes `self * rhs`, assuming overflow + /// cannot occur. + /// + /// # Safety + /// + /// This results in undefined behavior when + #[doc = concat!("`self * rhs > ", stringify!($SelfT), "::MAX` or `self * rhs < ", stringify!($SelfT), "::MIN`,")] + /// i.e. when [`checked_mul`] would return `None`. + /// + #[doc = concat!("[`checked_mul`]: ", stringify!($SelfT), "::checked_mul")] + #[unstable( + feature = "unchecked_math", + reason = "niche optimization path", + issue = "85122", + )] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn unchecked_mul(self, rhs: Self) -> Self { + // SAFETY: the caller must uphold the safety contract for + // `unchecked_mul`. + unsafe { intrinsics::unchecked_mul(self, rhs) } + } + + /// Checked integer division. Computes `self / rhs`, returning `None` + /// if `rhs == 0`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(128", stringify!($SelfT), ".checked_div(2), Some(64));")] + #[doc = concat!("assert_eq!(1", stringify!($SelfT), ".checked_div(0), None);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_checked_int_div", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_div(self, rhs: Self) -> Option { + if unlikely!(rhs == 0) { + None + } else { + // SAFETY: div by zero has been checked above and unsigned types have no other + // failure modes for division + Some(unsafe { intrinsics::unchecked_div(self, rhs) }) + } + } + + /// Checked Euclidean division. Computes `self.div_euclid(rhs)`, returning `None` + /// if `rhs == 0`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(128", stringify!($SelfT), ".checked_div_euclid(2), Some(64));")] + #[doc = concat!("assert_eq!(1", stringify!($SelfT), ".checked_div_euclid(0), None);")] + /// ``` + #[stable(feature = "euclidean_division", since = "1.38.0")] + #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_div_euclid(self, rhs: Self) -> Option { + if unlikely!(rhs == 0) { + None + } else { + Some(self.div_euclid(rhs)) + } + } + + + /// Checked integer remainder. Computes `self % rhs`, returning `None` + /// if `rhs == 0`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".checked_rem(2), Some(1));")] + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".checked_rem(0), None);")] + /// ``` + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_checked_int_div", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_rem(self, rhs: Self) -> Option { + if unlikely!(rhs == 0) { + None + } else { + // SAFETY: div by zero has been checked above and unsigned types have no other + // failure modes for division + Some(unsafe { intrinsics::unchecked_rem(self, rhs) }) + } + } + + /// Checked Euclidean modulo. Computes `self.rem_euclid(rhs)`, returning `None` + /// if `rhs == 0`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".checked_rem_euclid(2), Some(1));")] + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".checked_rem_euclid(0), None);")] + /// ``` + #[stable(feature = "euclidean_division", since = "1.38.0")] + #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_rem_euclid(self, rhs: Self) -> Option { + if unlikely!(rhs == 0) { + None + } else { + Some(self.rem_euclid(rhs)) + } + } + + /// Returns the logarithm of the number with respect to an arbitrary base, + /// rounded down. + /// + /// This method might not be optimized owing to implementation details; + /// `ilog2` can produce results more efficiently for base 2, and `ilog10` + /// can produce results more efficiently for base 10. + /// + /// # Panics + /// + /// This function will panic if `self` is zero, or if `base` is less than 2. + /// + /// # Examples + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".ilog(5), 1);")] + /// ``` + #[stable(feature = "int_log", since = "1.67.0")] + #[rustc_const_stable(feature = "int_log", since = "1.67.0")] + #[rustc_allow_const_fn_unstable(const_option)] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + #[track_caller] + pub const fn ilog(self, base: Self) -> u32 { + assert!(base >= 2, "base of integer logarithm must be at least 2"); + self.checked_ilog(base).expect("argument of integer logarithm must be positive") + } + + /// Returns the base 2 logarithm of the number, rounded down. + /// + /// # Panics + /// + /// This function will panic if `self` is zero. + /// + /// # Examples + /// + /// ``` + #[doc = concat!("assert_eq!(2", stringify!($SelfT), ".ilog2(), 1);")] + /// ``` + #[stable(feature = "int_log", since = "1.67.0")] + #[rustc_const_stable(feature = "int_log", since = "1.67.0")] + #[rustc_allow_const_fn_unstable(const_option)] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + #[track_caller] + pub const fn ilog2(self) -> u32 { + self.checked_ilog2().expect("argument of integer logarithm must be positive") + } + + /// Returns the base 10 logarithm of the number, rounded down. + /// + /// # Panics + /// + /// This function will panic if `self` is zero. + /// + /// # Example + /// + /// ``` + #[doc = concat!("assert_eq!(10", stringify!($SelfT), ".ilog10(), 1);")] + /// ``` + #[stable(feature = "int_log", since = "1.67.0")] + #[rustc_const_stable(feature = "int_log", since = "1.67.0")] + #[rustc_allow_const_fn_unstable(const_option)] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + #[track_caller] + pub const fn ilog10(self) -> u32 { + self.checked_ilog10().expect("argument of integer logarithm must be positive") + } + + /// Returns the logarithm of the number with respect to an arbitrary base, + /// rounded down. + /// + /// Returns `None` if the number is zero, or if the base is not at least 2. + /// + /// This method might not be optimized owing to implementation details; + /// `checked_ilog2` can produce results more efficiently for base 2, and + /// `checked_ilog10` can produce results more efficiently for base 10. + /// + /// # Examples + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".checked_ilog(5), Some(1));")] + /// ``` + #[stable(feature = "int_log", since = "1.67.0")] + #[rustc_const_stable(feature = "int_log", since = "1.67.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_ilog(self, base: Self) -> Option { + if self <= 0 || base <= 1 { + None + } else { + let mut n = 0; + let mut r = self; + + // Optimization for 128 bit wide integers. + if Self::BITS == 128 { + let b = Self::ilog2(self) / (Self::ilog2(base) + 1); + n += b; + r /= base.pow(b as u32); + } + + while r >= base { + r /= base; + n += 1; + } + Some(n) + } + } + + /// Returns the base 2 logarithm of the number, rounded down. + /// + /// Returns `None` if the number is zero. + /// + /// # Examples + /// + /// ``` + #[doc = concat!("assert_eq!(2", stringify!($SelfT), ".checked_ilog2(), Some(1));")] + /// ``` + #[stable(feature = "int_log", since = "1.67.0")] + #[rustc_const_stable(feature = "int_log", since = "1.67.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_ilog2(self) -> Option { + if let Some(x) = <$NonZeroT>::new(self) { + Some(x.ilog2()) + } else { + None + } + } + + /// Returns the base 10 logarithm of the number, rounded down. + /// + /// Returns `None` if the number is zero. + /// + /// # Examples + /// + /// ``` + #[doc = concat!("assert_eq!(10", stringify!($SelfT), ".checked_ilog10(), Some(1));")] + /// ``` + #[stable(feature = "int_log", since = "1.67.0")] + #[rustc_const_stable(feature = "int_log", since = "1.67.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_ilog10(self) -> Option { + if let Some(x) = <$NonZeroT>::new(self) { + Some(x.ilog10()) + } else { + None + } + } + + /// Checked negation. Computes `-self`, returning `None` unless `self == + /// 0`. + /// + /// Note that negating any positive integer will overflow. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(0", stringify!($SelfT), ".checked_neg(), Some(0));")] + #[doc = concat!("assert_eq!(1", stringify!($SelfT), ".checked_neg(), None);")] + /// ``` + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_checked_int_methods", since = "1.47.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_neg(self) -> Option { + let (a, b) = self.overflowing_neg(); + if unlikely!(b) {None} else {Some(a)} + } + + /// Checked shift left. Computes `self << rhs`, returning `None` + /// if `rhs` is larger than or equal to the number of bits in `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(0x1", stringify!($SelfT), ".checked_shl(4), Some(0x10));")] + #[doc = concat!("assert_eq!(0x10", stringify!($SelfT), ".checked_shl(129), None);")] + /// ``` + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_checked_int_methods", since = "1.47.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_shl(self, rhs: u32) -> Option { + let (a, b) = self.overflowing_shl(rhs); + if unlikely!(b) {None} else {Some(a)} + } + + /// Unchecked shift left. Computes `self << rhs`, assuming that + /// `rhs` is less than the number of bits in `self`. + /// + /// # Safety + /// + /// This results in undefined behavior if `rhs` is larger than + /// or equal to the number of bits in `self`, + /// i.e. when [`checked_shl`] would return `None`. + /// + #[doc = concat!("[`checked_shl`]: ", stringify!($SelfT), "::checked_shl")] + #[unstable( + feature = "unchecked_math", + reason = "niche optimization path", + issue = "85122", + )] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn unchecked_shl(self, rhs: u32) -> Self { + // SAFETY: the caller must uphold the safety contract for + // `unchecked_shl`. + // Any legal shift amount is losslessly representable in the self type. + unsafe { intrinsics::unchecked_shl(self, rhs.try_into().ok().unwrap_unchecked()) } + } + + /// Checked shift right. Computes `self >> rhs`, returning `None` + /// if `rhs` is larger than or equal to the number of bits in `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(0x10", stringify!($SelfT), ".checked_shr(4), Some(0x1));")] + #[doc = concat!("assert_eq!(0x10", stringify!($SelfT), ".checked_shr(129), None);")] + /// ``` + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_checked_int_methods", since = "1.47.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_shr(self, rhs: u32) -> Option { + let (a, b) = self.overflowing_shr(rhs); + if unlikely!(b) {None} else {Some(a)} + } + + /// Unchecked shift right. Computes `self >> rhs`, assuming that + /// `rhs` is less than the number of bits in `self`. + /// + /// # Safety + /// + /// This results in undefined behavior if `rhs` is larger than + /// or equal to the number of bits in `self`, + /// i.e. when [`checked_shr`] would return `None`. + /// + #[doc = concat!("[`checked_shr`]: ", stringify!($SelfT), "::checked_shr")] + #[unstable( + feature = "unchecked_math", + reason = "niche optimization path", + issue = "85122", + )] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn unchecked_shr(self, rhs: u32) -> Self { + // SAFETY: the caller must uphold the safety contract for + // `unchecked_shr`. + // Any legal shift amount is losslessly representable in the self type. + unsafe { intrinsics::unchecked_shr(self, rhs.try_into().ok().unwrap_unchecked()) } + } + + /// Checked exponentiation. Computes `self.pow(exp)`, returning `None` if + /// overflow occurred. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(2", stringify!($SelfT), ".checked_pow(5), Some(32));")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.checked_pow(2), None);")] + /// ``` + #[stable(feature = "no_panic_pow", since = "1.34.0")] + #[rustc_const_stable(feature = "const_int_pow", since = "1.50.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_pow(self, mut exp: u32) -> Option { + if exp == 0 { + return Some(1); + } + let mut base = self; + let mut acc: Self = 1; + + while exp > 1 { + if (exp & 1) == 1 { + acc = try_opt!(acc.checked_mul(base)); + } + exp /= 2; + base = try_opt!(base.checked_mul(base)); + } + + // since exp!=0, finally the exp must be 1. + // Deal with the final bit of the exponent separately, since + // squaring the base afterwards is not necessary and may cause a + // needless overflow. + + acc.checked_mul(base) + } + + /// Saturating integer addition. Computes `self + rhs`, saturating at + /// the numeric bounds instead of overflowing. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".saturating_add(1), 101);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.saturating_add(127), ", stringify!($SelfT), "::MAX);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[rustc_const_stable(feature = "const_saturating_int_methods", since = "1.47.0")] + #[inline(always)] + pub const fn saturating_add(self, rhs: Self) -> Self { + intrinsics::saturating_add(self, rhs) + } + + /// Saturating addition with a signed integer. Computes `self + rhs`, + /// saturating at the numeric bounds instead of overflowing. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(1", stringify!($SelfT), ".saturating_add_signed(2), 3);")] + #[doc = concat!("assert_eq!(1", stringify!($SelfT), ".saturating_add_signed(-2), 0);")] + #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MAX - 2).saturating_add_signed(4), ", stringify!($SelfT), "::MAX);")] + /// ``` + #[stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[rustc_const_stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn saturating_add_signed(self, rhs: $SignedT) -> Self { + let (res, overflow) = self.overflowing_add(rhs as Self); + if overflow == (rhs < 0) { + res + } else if overflow { + Self::MAX + } else { + 0 + } + } + + /// Saturating integer subtraction. Computes `self - rhs`, saturating + /// at the numeric bounds instead of overflowing. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".saturating_sub(27), 73);")] + #[doc = concat!("assert_eq!(13", stringify!($SelfT), ".saturating_sub(127), 0);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[rustc_const_stable(feature = "const_saturating_int_methods", since = "1.47.0")] + #[inline(always)] + pub const fn saturating_sub(self, rhs: Self) -> Self { + intrinsics::saturating_sub(self, rhs) + } + + /// Saturating integer multiplication. Computes `self * rhs`, + /// saturating at the numeric bounds instead of overflowing. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(2", stringify!($SelfT), ".saturating_mul(10), 20);")] + #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MAX).saturating_mul(10), ", stringify!($SelfT),"::MAX);")] + /// ``` + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_saturating_int_methods", since = "1.47.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn saturating_mul(self, rhs: Self) -> Self { + match self.checked_mul(rhs) { + Some(x) => x, + None => Self::MAX, + } + } + + /// Saturating integer division. Computes `self / rhs`, saturating at the + /// numeric bounds instead of overflowing. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".saturating_div(2), 2);")] + /// + /// ``` + /// + /// ```should_panic + #[doc = concat!("let _ = 1", stringify!($SelfT), ".saturating_div(0);")] + /// + /// ``` + #[stable(feature = "saturating_div", since = "1.58.0")] + #[rustc_const_stable(feature = "saturating_div", since = "1.58.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn saturating_div(self, rhs: Self) -> Self { + // on unsigned types, there is no overflow in integer division + self.wrapping_div(rhs) + } + + /// Saturating integer exponentiation. Computes `self.pow(exp)`, + /// saturating at the numeric bounds instead of overflowing. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(4", stringify!($SelfT), ".saturating_pow(3), 64);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.saturating_pow(2), ", stringify!($SelfT), "::MAX);")] + /// ``` + #[stable(feature = "no_panic_pow", since = "1.34.0")] + #[rustc_const_stable(feature = "const_int_pow", since = "1.50.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn saturating_pow(self, exp: u32) -> Self { + match self.checked_pow(exp) { + Some(x) => x, + None => Self::MAX, + } + } + + /// Wrapping (modular) addition. Computes `self + rhs`, + /// wrapping around at the boundary of the type. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(200", stringify!($SelfT), ".wrapping_add(55), 255);")] + #[doc = concat!("assert_eq!(200", stringify!($SelfT), ".wrapping_add(", stringify!($SelfT), "::MAX), 199);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn wrapping_add(self, rhs: Self) -> Self { + intrinsics::wrapping_add(self, rhs) + } + + /// Wrapping (modular) addition with a signed integer. Computes + /// `self + rhs`, wrapping around at the boundary of the type. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(1", stringify!($SelfT), ".wrapping_add_signed(2), 3);")] + #[doc = concat!("assert_eq!(1", stringify!($SelfT), ".wrapping_add_signed(-2), ", stringify!($SelfT), "::MAX);")] + #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MAX - 2).wrapping_add_signed(4), 1);")] + /// ``` + #[stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[rustc_const_stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn wrapping_add_signed(self, rhs: $SignedT) -> Self { + self.wrapping_add(rhs as Self) + } + + /// Wrapping (modular) subtraction. Computes `self - rhs`, + /// wrapping around at the boundary of the type. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".wrapping_sub(100), 0);")] + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".wrapping_sub(", stringify!($SelfT), "::MAX), 101);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn wrapping_sub(self, rhs: Self) -> Self { + intrinsics::wrapping_sub(self, rhs) + } + + /// Wrapping (modular) multiplication. Computes `self * + /// rhs`, wrapping around at the boundary of the type. + /// + /// # Examples + /// + /// Basic usage: + /// + /// Please note that this example is shared between integer types. + /// Which explains why `u8` is used here. + /// + /// ``` + /// assert_eq!(10u8.wrapping_mul(12), 120); + /// assert_eq!(25u8.wrapping_mul(12), 44); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn wrapping_mul(self, rhs: Self) -> Self { + intrinsics::wrapping_mul(self, rhs) + } + + /// Wrapping (modular) division. Computes `self / rhs`. + /// Wrapped division on unsigned types is just normal division. + /// There's no way wrapping could ever happen. + /// This function exists, so that all operations + /// are accounted for in the wrapping operations. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".wrapping_div(10), 10);")] + /// ``` + #[stable(feature = "num_wrapping", since = "1.2.0")] + #[rustc_const_stable(feature = "const_wrapping_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn wrapping_div(self, rhs: Self) -> Self { + self / rhs + } + + /// Wrapping Euclidean division. Computes `self.div_euclid(rhs)`. + /// Wrapped division on unsigned types is just normal division. + /// There's no way wrapping could ever happen. + /// This function exists, so that all operations + /// are accounted for in the wrapping operations. + /// Since, for the positive integers, all common + /// definitions of division are equal, this + /// is exactly equal to `self.wrapping_div(rhs)`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".wrapping_div_euclid(10), 10);")] + /// ``` + #[stable(feature = "euclidean_division", since = "1.38.0")] + #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn wrapping_div_euclid(self, rhs: Self) -> Self { + self / rhs + } + + /// Wrapping (modular) remainder. Computes `self % rhs`. + /// Wrapped remainder calculation on unsigned types is + /// just the regular remainder calculation. + /// There's no way wrapping could ever happen. + /// This function exists, so that all operations + /// are accounted for in the wrapping operations. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".wrapping_rem(10), 0);")] + /// ``` + #[stable(feature = "num_wrapping", since = "1.2.0")] + #[rustc_const_stable(feature = "const_wrapping_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn wrapping_rem(self, rhs: Self) -> Self { + self % rhs + } + + /// Wrapping Euclidean modulo. Computes `self.rem_euclid(rhs)`. + /// Wrapped modulo calculation on unsigned types is + /// just the regular remainder calculation. + /// There's no way wrapping could ever happen. + /// This function exists, so that all operations + /// are accounted for in the wrapping operations. + /// Since, for the positive integers, all common + /// definitions of division are equal, this + /// is exactly equal to `self.wrapping_rem(rhs)`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".wrapping_rem_euclid(10), 0);")] + /// ``` + #[stable(feature = "euclidean_division", since = "1.38.0")] + #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn wrapping_rem_euclid(self, rhs: Self) -> Self { + self % rhs + } + + /// Wrapping (modular) negation. Computes `-self`, + /// wrapping around at the boundary of the type. + /// + /// Since unsigned types do not have negative equivalents + /// all applications of this function will wrap (except for `-0`). + /// For values smaller than the corresponding signed type's maximum + /// the result is the same as casting the corresponding signed value. + /// Any larger values are equivalent to `MAX + 1 - (val - MAX - 1)` where + /// `MAX` is the corresponding signed type's maximum. + /// + /// # Examples + /// + /// Basic usage: + /// + /// Please note that this example is shared between integer types. + /// Which explains why `i8` is used here. + /// + /// ``` + /// assert_eq!(100i8.wrapping_neg(), -100); + /// assert_eq!((-128i8).wrapping_neg(), -128); + /// ``` + #[stable(feature = "num_wrapping", since = "1.2.0")] + #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn wrapping_neg(self) -> Self { + (0 as $SelfT).wrapping_sub(self) + } + + /// Panic-free bitwise shift-left; yields `self << mask(rhs)`, + /// where `mask` removes any high-order bits of `rhs` that + /// would cause the shift to exceed the bitwidth of the type. + /// + /// Note that this is *not* the same as a rotate-left; the + /// RHS of a wrapping shift-left is restricted to the range + /// of the type, rather than the bits shifted out of the LHS + /// being returned to the other end. The primitive integer + /// types all implement a [`rotate_left`](Self::rotate_left) function, + /// which may be what you want instead. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(1", stringify!($SelfT), ".wrapping_shl(7), 128);")] + #[doc = concat!("assert_eq!(1", stringify!($SelfT), ".wrapping_shl(128), 1);")] + /// ``` + #[stable(feature = "num_wrapping", since = "1.2.0")] + #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + #[rustc_allow_const_fn_unstable(const_inherent_unchecked_arith)] + pub const fn wrapping_shl(self, rhs: u32) -> Self { + // SAFETY: the masking by the bitsize of the type ensures that we do not shift + // out of bounds + unsafe { + self.unchecked_shl(rhs & ($BITS - 1)) + } + } + + /// Panic-free bitwise shift-right; yields `self >> mask(rhs)`, + /// where `mask` removes any high-order bits of `rhs` that + /// would cause the shift to exceed the bitwidth of the type. + /// + /// Note that this is *not* the same as a rotate-right; the + /// RHS of a wrapping shift-right is restricted to the range + /// of the type, rather than the bits shifted out of the LHS + /// being returned to the other end. The primitive integer + /// types all implement a [`rotate_right`](Self::rotate_right) function, + /// which may be what you want instead. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(128", stringify!($SelfT), ".wrapping_shr(7), 1);")] + #[doc = concat!("assert_eq!(128", stringify!($SelfT), ".wrapping_shr(128), 128);")] + /// ``` + #[stable(feature = "num_wrapping", since = "1.2.0")] + #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + #[rustc_allow_const_fn_unstable(const_inherent_unchecked_arith)] + pub const fn wrapping_shr(self, rhs: u32) -> Self { + // SAFETY: the masking by the bitsize of the type ensures that we do not shift + // out of bounds + unsafe { + self.unchecked_shr(rhs & ($BITS - 1)) + } + } + + /// Wrapping (modular) exponentiation. Computes `self.pow(exp)`, + /// wrapping around at the boundary of the type. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(3", stringify!($SelfT), ".wrapping_pow(5), 243);")] + /// assert_eq!(3u8.wrapping_pow(6), 217); + /// ``` + #[stable(feature = "no_panic_pow", since = "1.34.0")] + #[rustc_const_stable(feature = "const_int_pow", since = "1.50.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn wrapping_pow(self, mut exp: u32) -> Self { + if exp == 0 { + return 1; + } + let mut base = self; + let mut acc: Self = 1; + + while exp > 1 { + if (exp & 1) == 1 { + acc = acc.wrapping_mul(base); + } + exp /= 2; + base = base.wrapping_mul(base); + } + + // since exp!=0, finally the exp must be 1. + // Deal with the final bit of the exponent separately, since + // squaring the base afterwards is not necessary and may cause a + // needless overflow. + acc.wrapping_mul(base) + } + + /// Calculates `self` + `rhs` + /// + /// Returns a tuple of the addition along with a boolean indicating + /// whether an arithmetic overflow would occur. If an overflow would + /// have occurred then the wrapped value is returned. + /// + /// # Examples + /// + /// Basic usage + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".overflowing_add(2), (7, false));")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.overflowing_add(1), (0, true));")] + /// ``` + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn overflowing_add(self, rhs: Self) -> (Self, bool) { + let (a, b) = intrinsics::add_with_overflow(self as $ActualT, rhs as $ActualT); + (a as Self, b) + } + + /// Calculates `self` + `rhs` + `carry` and returns a tuple containing + /// the sum and the output carry. + /// + /// Performs "ternary addition" of two integer operands and a carry-in + /// bit, and returns an output integer and a carry-out bit. This allows + /// chaining together multiple additions to create a wider addition, and + /// can be useful for bignum addition. + /// + #[doc = concat!("This can be thought of as a ", stringify!($BITS), "-bit \"full adder\", in the electronics sense.")] + /// + /// If the input carry is false, this method is equivalent to + /// [`overflowing_add`](Self::overflowing_add), and the output carry is + /// equal to the overflow flag. Note that although carry and overflow + /// flags are similar for unsigned integers, they are different for + /// signed integers. + /// + /// # Examples + /// + /// ``` + /// #![feature(bigint_helper_methods)] + /// + #[doc = concat!("// 3 MAX (a = 3 × 2^", stringify!($BITS), " + 2^", stringify!($BITS), " - 1)")] + #[doc = concat!("// + 5 7 (b = 5 × 2^", stringify!($BITS), " + 7)")] + /// // --------- + #[doc = concat!("// 9 6 (sum = 9 × 2^", stringify!($BITS), " + 6)")] + /// + #[doc = concat!("let (a1, a0): (", stringify!($SelfT), ", ", stringify!($SelfT), ") = (3, ", stringify!($SelfT), "::MAX);")] + #[doc = concat!("let (b1, b0): (", stringify!($SelfT), ", ", stringify!($SelfT), ") = (5, 7);")] + /// let carry0 = false; + /// + /// let (sum0, carry1) = a0.carrying_add(b0, carry0); + /// assert_eq!(carry1, true); + /// let (sum1, carry2) = a1.carrying_add(b1, carry1); + /// assert_eq!(carry2, false); + /// + /// assert_eq!((sum1, sum0), (9, 6)); + /// ``` + #[unstable(feature = "bigint_helper_methods", issue = "85532")] + #[rustc_const_unstable(feature = "const_bigint_helper_methods", issue = "85532")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn carrying_add(self, rhs: Self, carry: bool) -> (Self, bool) { + // note: longer-term this should be done via an intrinsic, but this has been shown + // to generate optimal code for now, and LLVM doesn't have an equivalent intrinsic + let (a, b) = self.overflowing_add(rhs); + let (c, d) = a.overflowing_add(carry as $SelfT); + (c, b || d) + } + + /// Calculates `self` + `rhs` with a signed `rhs` + /// + /// Returns a tuple of the addition along with a boolean indicating + /// whether an arithmetic overflow would occur. If an overflow would + /// have occurred then the wrapped value is returned. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(1", stringify!($SelfT), ".overflowing_add_signed(2), (3, false));")] + #[doc = concat!("assert_eq!(1", stringify!($SelfT), ".overflowing_add_signed(-2), (", stringify!($SelfT), "::MAX, true));")] + #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MAX - 2).overflowing_add_signed(4), (1, true));")] + /// ``` + #[stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[rustc_const_stable(feature = "mixed_integer_ops", since = "1.66.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn overflowing_add_signed(self, rhs: $SignedT) -> (Self, bool) { + let (res, overflowed) = self.overflowing_add(rhs as Self); + (res, overflowed ^ (rhs < 0)) + } + + /// Calculates `self` - `rhs` + /// + /// Returns a tuple of the subtraction along with a boolean indicating + /// whether an arithmetic overflow would occur. If an overflow would + /// have occurred then the wrapped value is returned. + /// + /// # Examples + /// + /// Basic usage + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".overflowing_sub(2), (3, false));")] + #[doc = concat!("assert_eq!(0", stringify!($SelfT), ".overflowing_sub(1), (", stringify!($SelfT), "::MAX, true));")] + /// ``` + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn overflowing_sub(self, rhs: Self) -> (Self, bool) { + let (a, b) = intrinsics::sub_with_overflow(self as $ActualT, rhs as $ActualT); + (a as Self, b) + } + + /// Calculates `self` − `rhs` − `borrow` and returns a tuple + /// containing the difference and the output borrow. + /// + /// Performs "ternary subtraction" by subtracting both an integer + /// operand and a borrow-in bit from `self`, and returns an output + /// integer and a borrow-out bit. This allows chaining together multiple + /// subtractions to create a wider subtraction, and can be useful for + /// bignum subtraction. + /// + /// # Examples + /// + /// ``` + /// #![feature(bigint_helper_methods)] + /// + #[doc = concat!("// 9 6 (a = 9 × 2^", stringify!($BITS), " + 6)")] + #[doc = concat!("// - 5 7 (b = 5 × 2^", stringify!($BITS), " + 7)")] + /// // --------- + #[doc = concat!("// 3 MAX (diff = 3 × 2^", stringify!($BITS), " + 2^", stringify!($BITS), " - 1)")] + /// + #[doc = concat!("let (a1, a0): (", stringify!($SelfT), ", ", stringify!($SelfT), ") = (9, 6);")] + #[doc = concat!("let (b1, b0): (", stringify!($SelfT), ", ", stringify!($SelfT), ") = (5, 7);")] + /// let borrow0 = false; + /// + /// let (diff0, borrow1) = a0.borrowing_sub(b0, borrow0); + /// assert_eq!(borrow1, true); + /// let (diff1, borrow2) = a1.borrowing_sub(b1, borrow1); + /// assert_eq!(borrow2, false); + /// + #[doc = concat!("assert_eq!((diff1, diff0), (3, ", stringify!($SelfT), "::MAX));")] + /// ``` + #[unstable(feature = "bigint_helper_methods", issue = "85532")] + #[rustc_const_unstable(feature = "const_bigint_helper_methods", issue = "85532")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn borrowing_sub(self, rhs: Self, borrow: bool) -> (Self, bool) { + // note: longer-term this should be done via an intrinsic, but this has been shown + // to generate optimal code for now, and LLVM doesn't have an equivalent intrinsic + let (a, b) = self.overflowing_sub(rhs); + let (c, d) = a.overflowing_sub(borrow as $SelfT); + (c, b || d) + } + + /// Computes the absolute difference between `self` and `other`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".abs_diff(80), 20", stringify!($SelfT), ");")] + #[doc = concat!("assert_eq!(100", stringify!($SelfT), ".abs_diff(110), 10", stringify!($SelfT), ");")] + /// ``` + #[stable(feature = "int_abs_diff", since = "1.60.0")] + #[rustc_const_stable(feature = "int_abs_diff", since = "1.60.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn abs_diff(self, other: Self) -> Self { + if mem::size_of::() == 1 { + // Trick LLVM into generating the psadbw instruction when SSE2 + // is available and this function is autovectorized for u8's. + (self as i32).wrapping_sub(other as i32).abs() as Self + } else { + if self < other { + other - self + } else { + self - other + } + } + } + + /// Calculates the multiplication of `self` and `rhs`. + /// + /// Returns a tuple of the multiplication along with a boolean + /// indicating whether an arithmetic overflow would occur. If an + /// overflow would have occurred then the wrapped value is returned. + /// + /// # Examples + /// + /// Basic usage: + /// + /// Please note that this example is shared between integer types. + /// Which explains why `u32` is used here. + /// + /// ``` + /// assert_eq!(5u32.overflowing_mul(2), (10, false)); + /// assert_eq!(1_000_000_000u32.overflowing_mul(10), (1410065408, true)); + /// ``` + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn overflowing_mul(self, rhs: Self) -> (Self, bool) { + let (a, b) = intrinsics::mul_with_overflow(self as $ActualT, rhs as $ActualT); + (a as Self, b) + } + + /// Calculates the divisor when `self` is divided by `rhs`. + /// + /// Returns a tuple of the divisor along with a boolean indicating + /// whether an arithmetic overflow would occur. Note that for unsigned + /// integers overflow never occurs, so the second value is always + /// `false`. + /// + /// # Panics + /// + /// This function will panic if `rhs` is 0. + /// + /// # Examples + /// + /// Basic usage + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".overflowing_div(2), (2, false));")] + /// ``` + #[inline(always)] + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_overflowing_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + pub const fn overflowing_div(self, rhs: Self) -> (Self, bool) { + (self / rhs, false) + } + + /// Calculates the quotient of Euclidean division `self.div_euclid(rhs)`. + /// + /// Returns a tuple of the divisor along with a boolean indicating + /// whether an arithmetic overflow would occur. Note that for unsigned + /// integers overflow never occurs, so the second value is always + /// `false`. + /// Since, for the positive integers, all common + /// definitions of division are equal, this + /// is exactly equal to `self.overflowing_div(rhs)`. + /// + /// # Panics + /// + /// This function will panic if `rhs` is 0. + /// + /// # Examples + /// + /// Basic usage + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".overflowing_div_euclid(2), (2, false));")] + /// ``` + #[inline(always)] + #[stable(feature = "euclidean_division", since = "1.38.0")] + #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + pub const fn overflowing_div_euclid(self, rhs: Self) -> (Self, bool) { + (self / rhs, false) + } + + /// Calculates the remainder when `self` is divided by `rhs`. + /// + /// Returns a tuple of the remainder after dividing along with a boolean + /// indicating whether an arithmetic overflow would occur. Note that for + /// unsigned integers overflow never occurs, so the second value is + /// always `false`. + /// + /// # Panics + /// + /// This function will panic if `rhs` is 0. + /// + /// # Examples + /// + /// Basic usage + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".overflowing_rem(2), (1, false));")] + /// ``` + #[inline(always)] + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_overflowing_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + pub const fn overflowing_rem(self, rhs: Self) -> (Self, bool) { + (self % rhs, false) + } + + /// Calculates the remainder `self.rem_euclid(rhs)` as if by Euclidean division. + /// + /// Returns a tuple of the modulo after dividing along with a boolean + /// indicating whether an arithmetic overflow would occur. Note that for + /// unsigned integers overflow never occurs, so the second value is + /// always `false`. + /// Since, for the positive integers, all common + /// definitions of division are equal, this operation + /// is exactly equal to `self.overflowing_rem(rhs)`. + /// + /// # Panics + /// + /// This function will panic if `rhs` is 0. + /// + /// # Examples + /// + /// Basic usage + /// + /// ``` + #[doc = concat!("assert_eq!(5", stringify!($SelfT), ".overflowing_rem_euclid(2), (1, false));")] + /// ``` + #[inline(always)] + #[stable(feature = "euclidean_division", since = "1.38.0")] + #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + pub const fn overflowing_rem_euclid(self, rhs: Self) -> (Self, bool) { + (self % rhs, false) + } + + /// Negates self in an overflowing fashion. + /// + /// Returns `!self + 1` using wrapping operations to return the value + /// that represents the negation of this unsigned value. Note that for + /// positive unsigned values overflow always occurs, but negating 0 does + /// not overflow. + /// + /// # Examples + /// + /// Basic usage + /// + /// ``` + #[doc = concat!("assert_eq!(0", stringify!($SelfT), ".overflowing_neg(), (0, false));")] + #[doc = concat!("assert_eq!(2", stringify!($SelfT), ".overflowing_neg(), (-2i32 as ", stringify!($SelfT), ", true));")] + /// ``` + #[inline(always)] + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + pub const fn overflowing_neg(self) -> (Self, bool) { + ((!self).wrapping_add(1), self != 0) + } + + /// Shifts self left by `rhs` bits. + /// + /// Returns a tuple of the shifted version of self along with a boolean + /// indicating whether the shift value was larger than or equal to the + /// number of bits. If the shift value is too large, then value is + /// masked (N-1) where N is the number of bits, and this value is then + /// used to perform the shift. + /// + /// # Examples + /// + /// Basic usage + /// + /// ``` + #[doc = concat!("assert_eq!(0x1", stringify!($SelfT), ".overflowing_shl(4), (0x10, false));")] + #[doc = concat!("assert_eq!(0x1", stringify!($SelfT), ".overflowing_shl(132), (0x10, true));")] + /// ``` + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn overflowing_shl(self, rhs: u32) -> (Self, bool) { + (self.wrapping_shl(rhs), (rhs > ($BITS - 1))) + } + + /// Shifts self right by `rhs` bits. + /// + /// Returns a tuple of the shifted version of self along with a boolean + /// indicating whether the shift value was larger than or equal to the + /// number of bits. If the shift value is too large, then value is + /// masked (N-1) where N is the number of bits, and this value is then + /// used to perform the shift. + /// + /// # Examples + /// + /// Basic usage + /// + /// ``` + #[doc = concat!("assert_eq!(0x10", stringify!($SelfT), ".overflowing_shr(4), (0x1, false));")] + #[doc = concat!("assert_eq!(0x10", stringify!($SelfT), ".overflowing_shr(132), (0x1, true));")] + /// ``` + #[stable(feature = "wrapping", since = "1.7.0")] + #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn overflowing_shr(self, rhs: u32) -> (Self, bool) { + (self.wrapping_shr(rhs), (rhs > ($BITS - 1))) + } + + /// Raises self to the power of `exp`, using exponentiation by squaring. + /// + /// Returns a tuple of the exponentiation along with a bool indicating + /// whether an overflow happened. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(3", stringify!($SelfT), ".overflowing_pow(5), (243, false));")] + /// assert_eq!(3u8.overflowing_pow(6), (217, true)); + /// ``` + #[stable(feature = "no_panic_pow", since = "1.34.0")] + #[rustc_const_stable(feature = "const_int_pow", since = "1.50.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn overflowing_pow(self, mut exp: u32) -> (Self, bool) { + if exp == 0{ + return (1,false); + } + let mut base = self; + let mut acc: Self = 1; + let mut overflown = false; + // Scratch space for storing results of overflowing_mul. + let mut r; + + while exp > 1 { + if (exp & 1) == 1 { + r = acc.overflowing_mul(base); + acc = r.0; + overflown |= r.1; + } + exp /= 2; + r = base.overflowing_mul(base); + base = r.0; + overflown |= r.1; + } + + // since exp!=0, finally the exp must be 1. + // Deal with the final bit of the exponent separately, since + // squaring the base afterwards is not necessary and may cause a + // needless overflow. + r = acc.overflowing_mul(base); + r.1 |= overflown; + + r + } + + /// Raises self to the power of `exp`, using exponentiation by squaring. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(2", stringify!($SelfT), ".pow(5), 32);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_pow", since = "1.50.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + #[rustc_inherit_overflow_checks] + pub const fn pow(self, mut exp: u32) -> Self { + if exp == 0 { + return 1; + } + let mut base = self; + let mut acc = 1; + + while exp > 1 { + if (exp & 1) == 1 { + acc = acc * base; + } + exp /= 2; + base = base * base; + } + + // since exp!=0, finally the exp must be 1. + // Deal with the final bit of the exponent separately, since + // squaring the base afterwards is not necessary and may cause a + // needless overflow. + acc * base + } + + /// Performs Euclidean division. + /// + /// Since, for the positive integers, all common + /// definitions of division are equal, this + /// is exactly equal to `self / rhs`. + /// + /// # Panics + /// + /// This function will panic if `rhs` is 0. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(7", stringify!($SelfT), ".div_euclid(4), 1); // or any other integer type")] + /// ``` + #[stable(feature = "euclidean_division", since = "1.38.0")] + #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + #[rustc_inherit_overflow_checks] + pub const fn div_euclid(self, rhs: Self) -> Self { + self / rhs + } + + + /// Calculates the least remainder of `self (mod rhs)`. + /// + /// Since, for the positive integers, all common + /// definitions of division are equal, this + /// is exactly equal to `self % rhs`. + /// + /// # Panics + /// + /// This function will panic if `rhs` is 0. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(7", stringify!($SelfT), ".rem_euclid(4), 3); // or any other integer type")] + /// ``` + #[stable(feature = "euclidean_division", since = "1.38.0")] + #[rustc_const_stable(feature = "const_euclidean_int_methods", since = "1.52.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + #[rustc_inherit_overflow_checks] + pub const fn rem_euclid(self, rhs: Self) -> Self { + self % rhs + } + + /// Calculates the quotient of `self` and `rhs`, rounding the result towards negative infinity. + /// + /// This is the same as performing `self / rhs` for all unsigned integers. + /// + /// # Panics + /// + /// This function will panic if `rhs` is zero. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(int_roundings)] + #[doc = concat!("assert_eq!(7_", stringify!($SelfT), ".div_floor(4), 1);")] + /// ``` + #[unstable(feature = "int_roundings", issue = "88581")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn div_floor(self, rhs: Self) -> Self { + self / rhs + } + + /// Calculates the quotient of `self` and `rhs`, rounding the result towards positive infinity. + /// + /// # Panics + /// + /// This function will panic if `rhs` is zero. + /// + /// ## Overflow behavior + /// + /// On overflow, this function will panic if overflow checks are enabled (default in debug + /// mode) and wrap if overflow checks are disabled (default in release mode). + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(int_roundings)] + #[doc = concat!("assert_eq!(7_", stringify!($SelfT), ".div_ceil(4), 2);")] + /// ``` + #[unstable(feature = "int_roundings", issue = "88581")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + #[rustc_inherit_overflow_checks] + pub const fn div_ceil(self, rhs: Self) -> Self { + let d = self / rhs; + let r = self % rhs; + if r > 0 && rhs > 0 { + d + 1 + } else { + d + } + } + + /// Calculates the smallest value greater than or equal to `self` that + /// is a multiple of `rhs`. + /// + /// # Panics + /// + /// This function will panic if `rhs` is zero. + /// + /// ## Overflow behavior + /// + /// On overflow, this function will panic if overflow checks are enabled (default in debug + /// mode) and wrap if overflow checks are disabled (default in release mode). + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(int_roundings)] + #[doc = concat!("assert_eq!(16_", stringify!($SelfT), ".next_multiple_of(8), 16);")] + #[doc = concat!("assert_eq!(23_", stringify!($SelfT), ".next_multiple_of(8), 24);")] + /// ``` + #[unstable(feature = "int_roundings", issue = "88581")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + #[rustc_inherit_overflow_checks] + pub const fn next_multiple_of(self, rhs: Self) -> Self { + match self % rhs { + 0 => self, + r => self + (rhs - r) + } + } + + /// Calculates the smallest value greater than or equal to `self` that + /// is a multiple of `rhs`. Returns `None` if `rhs` is zero or the + /// operation would result in overflow. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(int_roundings)] + #[doc = concat!("assert_eq!(16_", stringify!($SelfT), ".checked_next_multiple_of(8), Some(16));")] + #[doc = concat!("assert_eq!(23_", stringify!($SelfT), ".checked_next_multiple_of(8), Some(24));")] + #[doc = concat!("assert_eq!(1_", stringify!($SelfT), ".checked_next_multiple_of(0), None);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.checked_next_multiple_of(2), None);")] + /// ``` + #[unstable(feature = "int_roundings", issue = "88581")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn checked_next_multiple_of(self, rhs: Self) -> Option { + match try_opt!(self.checked_rem(rhs)) { + 0 => Some(self), + // rhs - r cannot overflow because r is smaller than rhs + r => self.checked_add(rhs - r) + } + } + + /// Returns `true` if and only if `self == 2^k` for some `k`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert!(16", stringify!($SelfT), ".is_power_of_two());")] + #[doc = concat!("assert!(!10", stringify!($SelfT), ".is_power_of_two());")] + /// ``` + #[must_use] + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_is_power_of_two", since = "1.32.0")] + #[inline(always)] + pub const fn is_power_of_two(self) -> bool { + self.count_ones() == 1 + } + + // Returns one less than next power of two. + // (For 8u8 next power of two is 8u8 and for 6u8 it is 8u8) + // + // 8u8.one_less_than_next_power_of_two() == 7 + // 6u8.one_less_than_next_power_of_two() == 7 + // + // This method cannot overflow, as in the `next_power_of_two` + // overflow cases it instead ends up returning the maximum value + // of the type, and can return 0 for 0. + #[inline] + #[rustc_const_stable(feature = "const_int_pow", since = "1.50.0")] + const fn one_less_than_next_power_of_two(self) -> Self { + if self <= 1 { return 0; } + + let p = self - 1; + // SAFETY: Because `p > 0`, it cannot consist entirely of leading zeros. + // That means the shift is always in-bounds, and some processors + // (such as intel pre-haswell) have more efficient ctlz + // intrinsics when the argument is non-zero. + let z = unsafe { intrinsics::ctlz_nonzero(p) }; + <$SelfT>::MAX >> z + } + + /// Returns the smallest power of two greater than or equal to `self`. + /// + /// When return value overflows (i.e., `self > (1 << (N-1))` for type + /// `uN`), it panics in debug mode and the return value is wrapped to 0 in + /// release mode (the only situation in which method can return 0). + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(2", stringify!($SelfT), ".next_power_of_two(), 2);")] + #[doc = concat!("assert_eq!(3", stringify!($SelfT), ".next_power_of_two(), 4);")] + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_pow", since = "1.50.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + #[rustc_inherit_overflow_checks] + pub const fn next_power_of_two(self) -> Self { + self.one_less_than_next_power_of_two() + 1 + } + + /// Returns the smallest power of two greater than or equal to `n`. If + /// the next power of two is greater than the type's maximum value, + /// `None` is returned, otherwise the power of two is wrapped in `Some`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + #[doc = concat!("assert_eq!(2", stringify!($SelfT), ".checked_next_power_of_two(), Some(2));")] + #[doc = concat!("assert_eq!(3", stringify!($SelfT), ".checked_next_power_of_two(), Some(4));")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.checked_next_power_of_two(), None);")] + /// ``` + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_int_pow", since = "1.50.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + pub const fn checked_next_power_of_two(self) -> Option { + self.one_less_than_next_power_of_two().checked_add(1) + } + + /// Returns the smallest power of two greater than or equal to `n`. If + /// the next power of two is greater than the type's maximum value, + /// the return value is wrapped to `0`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(wrapping_next_power_of_two)] + /// + #[doc = concat!("assert_eq!(2", stringify!($SelfT), ".wrapping_next_power_of_two(), 2);")] + #[doc = concat!("assert_eq!(3", stringify!($SelfT), ".wrapping_next_power_of_two(), 4);")] + #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MAX.wrapping_next_power_of_two(), 0);")] + /// ``` + #[inline] + #[unstable(feature = "wrapping_next_power_of_two", issue = "32463", + reason = "needs decision on wrapping behaviour")] + #[rustc_const_unstable(feature = "wrapping_next_power_of_two", issue = "32463")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + pub const fn wrapping_next_power_of_two(self) -> Self { + self.one_less_than_next_power_of_two().wrapping_add(1) + } + + /// Return the memory representation of this integer as a byte array in + /// big-endian (network) byte order. + /// + #[doc = $to_xe_bytes_doc] + /// + /// # Examples + /// + /// ``` + #[doc = concat!("let bytes = ", $swap_op, stringify!($SelfT), ".to_be_bytes();")] + #[doc = concat!("assert_eq!(bytes, ", $be_bytes, ");")] + /// ``` + #[stable(feature = "int_to_from_bytes", since = "1.32.0")] + #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn to_be_bytes(self) -> [u8; mem::size_of::()] { + self.to_be().to_ne_bytes() + } + + /// Return the memory representation of this integer as a byte array in + /// little-endian byte order. + /// + #[doc = $to_xe_bytes_doc] + /// + /// # Examples + /// + /// ``` + #[doc = concat!("let bytes = ", $swap_op, stringify!($SelfT), ".to_le_bytes();")] + #[doc = concat!("assert_eq!(bytes, ", $le_bytes, ");")] + /// ``` + #[stable(feature = "int_to_from_bytes", since = "1.32.0")] + #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn to_le_bytes(self) -> [u8; mem::size_of::()] { + self.to_le().to_ne_bytes() + } + + /// Return the memory representation of this integer as a byte array in + /// native byte order. + /// + /// As the target platform's native endianness is used, portable code + /// should use [`to_be_bytes`] or [`to_le_bytes`], as appropriate, + /// instead. + /// + #[doc = $to_xe_bytes_doc] + /// + /// [`to_be_bytes`]: Self::to_be_bytes + /// [`to_le_bytes`]: Self::to_le_bytes + /// + /// # Examples + /// + /// ``` + #[doc = concat!("let bytes = ", $swap_op, stringify!($SelfT), ".to_ne_bytes();")] + /// assert_eq!( + /// bytes, + /// if cfg!(target_endian = "big") { + #[doc = concat!(" ", $be_bytes)] + /// } else { + #[doc = concat!(" ", $le_bytes)] + /// } + /// ); + /// ``` + #[stable(feature = "int_to_from_bytes", since = "1.32.0")] + #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + // SAFETY: const sound because integers are plain old datatypes so we can always + // transmute them to arrays of bytes + #[inline] + pub const fn to_ne_bytes(self) -> [u8; mem::size_of::()] { + // SAFETY: integers are plain old datatypes so we can always transmute them to + // arrays of bytes + unsafe { mem::transmute(self) } + } + + /// Create a native endian integer value from its representation + /// as a byte array in big endian. + /// + #[doc = $from_xe_bytes_doc] + /// + /// # Examples + /// + /// ``` + #[doc = concat!("let value = ", stringify!($SelfT), "::from_be_bytes(", $be_bytes, ");")] + #[doc = concat!("assert_eq!(value, ", $swap_op, ");")] + /// ``` + /// + /// When starting from a slice rather than an array, fallible conversion APIs can be used: + /// + /// ``` + #[doc = concat!("fn read_be_", stringify!($SelfT), "(input: &mut &[u8]) -> ", stringify!($SelfT), " {")] + #[doc = concat!(" let (int_bytes, rest) = input.split_at(std::mem::size_of::<", stringify!($SelfT), ">());")] + /// *input = rest; + #[doc = concat!(" ", stringify!($SelfT), "::from_be_bytes(int_bytes.try_into().unwrap())")] + /// } + /// ``` + #[stable(feature = "int_to_from_bytes", since = "1.32.0")] + #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] + #[must_use] + #[inline] + pub const fn from_be_bytes(bytes: [u8; mem::size_of::()]) -> Self { + Self::from_be(Self::from_ne_bytes(bytes)) + } + + /// Create a native endian integer value from its representation + /// as a byte array in little endian. + /// + #[doc = $from_xe_bytes_doc] + /// + /// # Examples + /// + /// ``` + #[doc = concat!("let value = ", stringify!($SelfT), "::from_le_bytes(", $le_bytes, ");")] + #[doc = concat!("assert_eq!(value, ", $swap_op, ");")] + /// ``` + /// + /// When starting from a slice rather than an array, fallible conversion APIs can be used: + /// + /// ``` + #[doc = concat!("fn read_le_", stringify!($SelfT), "(input: &mut &[u8]) -> ", stringify!($SelfT), " {")] + #[doc = concat!(" let (int_bytes, rest) = input.split_at(std::mem::size_of::<", stringify!($SelfT), ">());")] + /// *input = rest; + #[doc = concat!(" ", stringify!($SelfT), "::from_le_bytes(int_bytes.try_into().unwrap())")] + /// } + /// ``` + #[stable(feature = "int_to_from_bytes", since = "1.32.0")] + #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] + #[must_use] + #[inline] + pub const fn from_le_bytes(bytes: [u8; mem::size_of::()]) -> Self { + Self::from_le(Self::from_ne_bytes(bytes)) + } + + /// Create a native endian integer value from its memory representation + /// as a byte array in native endianness. + /// + /// As the target platform's native endianness is used, portable code + /// likely wants to use [`from_be_bytes`] or [`from_le_bytes`], as + /// appropriate instead. + /// + /// [`from_be_bytes`]: Self::from_be_bytes + /// [`from_le_bytes`]: Self::from_le_bytes + /// + #[doc = $from_xe_bytes_doc] + /// + /// # Examples + /// + /// ``` + #[doc = concat!("let value = ", stringify!($SelfT), "::from_ne_bytes(if cfg!(target_endian = \"big\") {")] + #[doc = concat!(" ", $be_bytes, "")] + /// } else { + #[doc = concat!(" ", $le_bytes, "")] + /// }); + #[doc = concat!("assert_eq!(value, ", $swap_op, ");")] + /// ``` + /// + /// When starting from a slice rather than an array, fallible conversion APIs can be used: + /// + /// ``` + #[doc = concat!("fn read_ne_", stringify!($SelfT), "(input: &mut &[u8]) -> ", stringify!($SelfT), " {")] + #[doc = concat!(" let (int_bytes, rest) = input.split_at(std::mem::size_of::<", stringify!($SelfT), ">());")] + /// *input = rest; + #[doc = concat!(" ", stringify!($SelfT), "::from_ne_bytes(int_bytes.try_into().unwrap())")] + /// } + /// ``` + #[stable(feature = "int_to_from_bytes", since = "1.32.0")] + #[rustc_const_stable(feature = "const_int_conversion", since = "1.44.0")] + #[must_use] + // SAFETY: const sound because integers are plain old datatypes so we can always + // transmute to them + #[inline] + pub const fn from_ne_bytes(bytes: [u8; mem::size_of::()]) -> Self { + // SAFETY: integers are plain old datatypes so we can always transmute to them + unsafe { mem::transmute(bytes) } + } + + /// New code should prefer to use + #[doc = concat!("[`", stringify!($SelfT), "::MIN", "`] instead.")] + /// + /// Returns the smallest value that can be represented by this integer type. + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_promotable] + #[inline(always)] + #[rustc_const_stable(feature = "const_max_value", since = "1.32.0")] + #[deprecated(since = "TBD", note = "replaced by the `MIN` associated constant on this type")] + pub const fn min_value() -> Self { Self::MIN } + + /// New code should prefer to use + #[doc = concat!("[`", stringify!($SelfT), "::MAX", "`] instead.")] + /// + /// Returns the largest value that can be represented by this integer type. + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_promotable] + #[inline(always)] + #[rustc_const_stable(feature = "const_max_value", since = "1.32.0")] + #[deprecated(since = "TBD", note = "replaced by the `MAX` associated constant on this type")] + pub const fn max_value() -> Self { Self::MAX } + } +} diff --git a/crux-mir/lib/core/src/num/usize.rs b/crux-mir/lib/core/src/num/usize.rs deleted file mode 100644 index a89304161..000000000 --- a/crux-mir/lib/core/src/num/usize.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! The pointer-sized unsigned integer type. -//! -//! *[See also the `usize` primitive type](../../std/primitive.usize.html).* -//! -//! Although using these constants won’t cause compilation warnings, -//! new code should use the associated constants directly on the primitive type. - -#![stable(feature = "rust1", since = "1.0.0")] - -int_module! { usize } diff --git a/crux-mir/lib/core/src/num/wrapping.rs b/crux-mir/lib/core/src/num/wrapping.rs index 82fa6acfb..5353d900e 100644 --- a/crux-mir/lib/core/src/num/wrapping.rs +++ b/crux-mir/lib/core/src/num/wrapping.rs @@ -1,12 +1,94 @@ -use super::Wrapping; +//! Definitions of `Wrapping`. + +use crate::fmt; +use crate::ops::{Add, AddAssign, BitAnd, BitAndAssign, BitOr, BitOrAssign}; +use crate::ops::{BitXor, BitXorAssign, Div, DivAssign}; +use crate::ops::{Mul, MulAssign, Neg, Not, Rem, RemAssign}; +use crate::ops::{Shl, ShlAssign, Shr, ShrAssign, Sub, SubAssign}; + +/// Provides intentionally-wrapped arithmetic on `T`. +/// +/// Operations like `+` on `u32` values are intended to never overflow, +/// and in some debug configurations overflow is detected and results +/// in a panic. While most arithmetic falls into this category, some +/// code explicitly expects and relies upon modular arithmetic (e.g., +/// hashing). +/// +/// Wrapping arithmetic can be achieved either through methods like +/// `wrapping_add`, or through the `Wrapping` type, which says that +/// all standard arithmetic operations on the underlying value are +/// intended to have wrapping semantics. +/// +/// The underlying value can be retrieved through the `.0` index of the +/// `Wrapping` tuple. +/// +/// # Examples +/// +/// ``` +/// use std::num::Wrapping; +/// +/// let zero = Wrapping(0u32); +/// let one = Wrapping(1u32); +/// +/// assert_eq!(u32::MAX, (zero - one).0); +/// ``` +/// +/// # Layout +/// +/// `Wrapping` is guaranteed to have the same layout and ABI as `T`. +#[stable(feature = "rust1", since = "1.0.0")] +#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default, Hash)] +#[repr(transparent)] +pub struct Wrapping(#[stable(feature = "rust1", since = "1.0.0")] pub T); + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Debug for Wrapping { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +#[stable(feature = "wrapping_display", since = "1.10.0")] +impl fmt::Display for Wrapping { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +#[stable(feature = "wrapping_fmt", since = "1.11.0")] +impl fmt::Binary for Wrapping { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +#[stable(feature = "wrapping_fmt", since = "1.11.0")] +impl fmt::Octal for Wrapping { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +#[stable(feature = "wrapping_fmt", since = "1.11.0")] +impl fmt::LowerHex for Wrapping { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} -use crate::ops::*; +#[stable(feature = "wrapping_fmt", since = "1.11.0")] +impl fmt::UpperHex for Wrapping { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} #[allow(unused_macros)] macro_rules! sh_impl_signed { ($t:ident, $f:ident) => { #[stable(feature = "rust1", since = "1.0.0")] - impl Shl<$f> for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Shl<$f> for Wrapping<$t> { type Output = Wrapping<$t>; #[inline] @@ -18,20 +100,22 @@ macro_rules! sh_impl_signed { } } } - forward_ref_binop! { impl Shl, shl for Wrapping<$t>, $f, + forward_ref_binop! { impl const Shl, shl for Wrapping<$t>, $f, #[stable(feature = "wrapping_ref_ops", since = "1.39.0")] } #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl ShlAssign<$f> for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const ShlAssign<$f> for Wrapping<$t> { #[inline] fn shl_assign(&mut self, other: $f) { *self = *self << other; } } - forward_ref_op_assign! { impl ShlAssign, shl_assign for Wrapping<$t>, $f } + forward_ref_op_assign! { impl const ShlAssign, shl_assign for Wrapping<$t>, $f } #[stable(feature = "rust1", since = "1.0.0")] - impl Shr<$f> for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Shr<$f> for Wrapping<$t> { type Output = Wrapping<$t>; #[inline] @@ -43,24 +127,26 @@ macro_rules! sh_impl_signed { } } } - forward_ref_binop! { impl Shr, shr for Wrapping<$t>, $f, + forward_ref_binop! { impl const Shr, shr for Wrapping<$t>, $f, #[stable(feature = "wrapping_ref_ops", since = "1.39.0")] } #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl ShrAssign<$f> for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const ShrAssign<$f> for Wrapping<$t> { #[inline] fn shr_assign(&mut self, other: $f) { *self = *self >> other; } } - forward_ref_op_assign! { impl ShrAssign, shr_assign for Wrapping<$t>, $f } + forward_ref_op_assign! { impl const ShrAssign, shr_assign for Wrapping<$t>, $f } }; } macro_rules! sh_impl_unsigned { ($t:ident, $f:ident) => { #[stable(feature = "rust1", since = "1.0.0")] - impl Shl<$f> for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Shl<$f> for Wrapping<$t> { type Output = Wrapping<$t>; #[inline] @@ -68,20 +154,22 @@ macro_rules! sh_impl_unsigned { Wrapping(self.0.wrapping_shl((other & self::shift_max::$t as $f) as u32)) } } - forward_ref_binop! { impl Shl, shl for Wrapping<$t>, $f, + forward_ref_binop! { impl const Shl, shl for Wrapping<$t>, $f, #[stable(feature = "wrapping_ref_ops", since = "1.39.0")] } #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl ShlAssign<$f> for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const ShlAssign<$f> for Wrapping<$t> { #[inline] fn shl_assign(&mut self, other: $f) { *self = *self << other; } } - forward_ref_op_assign! { impl ShlAssign, shl_assign for Wrapping<$t>, $f } + forward_ref_op_assign! { impl const ShlAssign, shl_assign for Wrapping<$t>, $f } #[stable(feature = "rust1", since = "1.0.0")] - impl Shr<$f> for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Shr<$f> for Wrapping<$t> { type Output = Wrapping<$t>; #[inline] @@ -89,17 +177,18 @@ macro_rules! sh_impl_unsigned { Wrapping(self.0.wrapping_shr((other & self::shift_max::$t as $f) as u32)) } } - forward_ref_binop! { impl Shr, shr for Wrapping<$t>, $f, + forward_ref_binop! { impl const Shr, shr for Wrapping<$t>, $f, #[stable(feature = "wrapping_ref_ops", since = "1.39.0")] } #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl ShrAssign<$f> for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const ShrAssign<$f> for Wrapping<$t> { #[inline] fn shr_assign(&mut self, other: $f) { *self = *self >> other; } } - forward_ref_op_assign! { impl ShrAssign, shr_assign for Wrapping<$t>, $f } + forward_ref_op_assign! { impl const ShrAssign, shr_assign for Wrapping<$t>, $f } }; } @@ -128,7 +217,8 @@ sh_impl_all! { u8 u16 u32 u64 u128 usize i8 i16 i32 i64 i128 isize } macro_rules! wrapping_impl { ($($t:ty)*) => ($( #[stable(feature = "rust1", since = "1.0.0")] - impl Add for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Add for Wrapping<$t> { type Output = Wrapping<$t>; #[inline] @@ -136,20 +226,32 @@ macro_rules! wrapping_impl { Wrapping(self.0.wrapping_add(other.0)) } } - forward_ref_binop! { impl Add, add for Wrapping<$t>, Wrapping<$t>, + forward_ref_binop! { impl const Add, add for Wrapping<$t>, Wrapping<$t>, #[stable(feature = "wrapping_ref", since = "1.14.0")] } #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl AddAssign for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const AddAssign for Wrapping<$t> { #[inline] fn add_assign(&mut self, other: Wrapping<$t>) { *self = *self + other; } } - forward_ref_op_assign! { impl AddAssign, add_assign for Wrapping<$t>, Wrapping<$t> } + forward_ref_op_assign! { impl const AddAssign, add_assign for Wrapping<$t>, Wrapping<$t> } + + #[stable(feature = "wrapping_int_assign_impl", since = "1.60.0")] + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const AddAssign<$t> for Wrapping<$t> { + #[inline] + fn add_assign(&mut self, other: $t) { + *self = *self + Wrapping(other); + } + } + forward_ref_op_assign! { impl const AddAssign, add_assign for Wrapping<$t>, $t } #[stable(feature = "rust1", since = "1.0.0")] - impl Sub for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Sub for Wrapping<$t> { type Output = Wrapping<$t>; #[inline] @@ -157,20 +259,32 @@ macro_rules! wrapping_impl { Wrapping(self.0.wrapping_sub(other.0)) } } - forward_ref_binop! { impl Sub, sub for Wrapping<$t>, Wrapping<$t>, + forward_ref_binop! { impl const Sub, sub for Wrapping<$t>, Wrapping<$t>, #[stable(feature = "wrapping_ref", since = "1.14.0")] } #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl SubAssign for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const SubAssign for Wrapping<$t> { #[inline] fn sub_assign(&mut self, other: Wrapping<$t>) { *self = *self - other; } } - forward_ref_op_assign! { impl SubAssign, sub_assign for Wrapping<$t>, Wrapping<$t> } + forward_ref_op_assign! { impl const SubAssign, sub_assign for Wrapping<$t>, Wrapping<$t> } + + #[stable(feature = "wrapping_int_assign_impl", since = "1.60.0")] + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const SubAssign<$t> for Wrapping<$t> { + #[inline] + fn sub_assign(&mut self, other: $t) { + *self = *self - Wrapping(other); + } + } + forward_ref_op_assign! { impl const SubAssign, sub_assign for Wrapping<$t>, $t } #[stable(feature = "rust1", since = "1.0.0")] - impl Mul for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Mul for Wrapping<$t> { type Output = Wrapping<$t>; #[inline] @@ -182,16 +296,28 @@ macro_rules! wrapping_impl { #[stable(feature = "wrapping_ref", since = "1.14.0")] } #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl MulAssign for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const MulAssign for Wrapping<$t> { #[inline] fn mul_assign(&mut self, other: Wrapping<$t>) { *self = *self * other; } } - forward_ref_op_assign! { impl MulAssign, mul_assign for Wrapping<$t>, Wrapping<$t> } + forward_ref_op_assign! { impl const MulAssign, mul_assign for Wrapping<$t>, Wrapping<$t> } + + #[stable(feature = "wrapping_int_assign_impl", since = "1.60.0")] + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const MulAssign<$t> for Wrapping<$t> { + #[inline] + fn mul_assign(&mut self, other: $t) { + *self = *self * Wrapping(other); + } + } + forward_ref_op_assign! { impl const MulAssign, mul_assign for Wrapping<$t>, $t } #[stable(feature = "wrapping_div", since = "1.3.0")] - impl Div for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Div for Wrapping<$t> { type Output = Wrapping<$t>; #[inline] @@ -199,20 +325,32 @@ macro_rules! wrapping_impl { Wrapping(self.0.wrapping_div(other.0)) } } - forward_ref_binop! { impl Div, div for Wrapping<$t>, Wrapping<$t>, + forward_ref_binop! { impl const Div, div for Wrapping<$t>, Wrapping<$t>, #[stable(feature = "wrapping_ref", since = "1.14.0")] } #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl DivAssign for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const DivAssign for Wrapping<$t> { #[inline] fn div_assign(&mut self, other: Wrapping<$t>) { *self = *self / other; } } - forward_ref_op_assign! { impl DivAssign, div_assign for Wrapping<$t>, Wrapping<$t> } + forward_ref_op_assign! { impl const DivAssign, div_assign for Wrapping<$t>, Wrapping<$t> } + + #[stable(feature = "wrapping_int_assign_impl", since = "1.60.0")] + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const DivAssign<$t> for Wrapping<$t> { + #[inline] + fn div_assign(&mut self, other: $t) { + *self = *self / Wrapping(other); + } + } + forward_ref_op_assign! { impl const DivAssign, div_assign for Wrapping<$t>, $t } #[stable(feature = "wrapping_impls", since = "1.7.0")] - impl Rem for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Rem for Wrapping<$t> { type Output = Wrapping<$t>; #[inline] @@ -220,20 +358,32 @@ macro_rules! wrapping_impl { Wrapping(self.0.wrapping_rem(other.0)) } } - forward_ref_binop! { impl Rem, rem for Wrapping<$t>, Wrapping<$t>, + forward_ref_binop! { impl const Rem, rem for Wrapping<$t>, Wrapping<$t>, #[stable(feature = "wrapping_ref", since = "1.14.0")] } #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl RemAssign for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const RemAssign for Wrapping<$t> { #[inline] fn rem_assign(&mut self, other: Wrapping<$t>) { *self = *self % other; } } - forward_ref_op_assign! { impl RemAssign, rem_assign for Wrapping<$t>, Wrapping<$t> } + forward_ref_op_assign! { impl const RemAssign, rem_assign for Wrapping<$t>, Wrapping<$t> } + + #[stable(feature = "wrapping_int_assign_impl", since = "1.60.0")] + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const RemAssign<$t> for Wrapping<$t> { + #[inline] + fn rem_assign(&mut self, other: $t) { + *self = *self % Wrapping(other); + } + } + forward_ref_op_assign! { impl const RemAssign, rem_assign for Wrapping<$t>, $t } #[stable(feature = "rust1", since = "1.0.0")] - impl Not for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Not for Wrapping<$t> { type Output = Wrapping<$t>; #[inline] @@ -241,11 +391,12 @@ macro_rules! wrapping_impl { Wrapping(!self.0) } } - forward_ref_unop! { impl Not, not for Wrapping<$t>, + forward_ref_unop! { impl const Not, not for Wrapping<$t>, #[stable(feature = "wrapping_ref", since = "1.14.0")] } #[stable(feature = "rust1", since = "1.0.0")] - impl BitXor for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const BitXor for Wrapping<$t> { type Output = Wrapping<$t>; #[inline] @@ -253,20 +404,32 @@ macro_rules! wrapping_impl { Wrapping(self.0 ^ other.0) } } - forward_ref_binop! { impl BitXor, bitxor for Wrapping<$t>, Wrapping<$t>, + forward_ref_binop! { impl const BitXor, bitxor for Wrapping<$t>, Wrapping<$t>, #[stable(feature = "wrapping_ref", since = "1.14.0")] } #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl BitXorAssign for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const BitXorAssign for Wrapping<$t> { #[inline] fn bitxor_assign(&mut self, other: Wrapping<$t>) { *self = *self ^ other; } } - forward_ref_op_assign! { impl BitXorAssign, bitxor_assign for Wrapping<$t>, Wrapping<$t> } + forward_ref_op_assign! { impl const BitXorAssign, bitxor_assign for Wrapping<$t>, Wrapping<$t> } + + #[stable(feature = "wrapping_int_assign_impl", since = "1.60.0")] + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const BitXorAssign<$t> for Wrapping<$t> { + #[inline] + fn bitxor_assign(&mut self, other: $t) { + *self = *self ^ Wrapping(other); + } + } + forward_ref_op_assign! { impl const BitXorAssign, bitxor_assign for Wrapping<$t>, $t } #[stable(feature = "rust1", since = "1.0.0")] - impl BitOr for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const BitOr for Wrapping<$t> { type Output = Wrapping<$t>; #[inline] @@ -274,20 +437,32 @@ macro_rules! wrapping_impl { Wrapping(self.0 | other.0) } } - forward_ref_binop! { impl BitOr, bitor for Wrapping<$t>, Wrapping<$t>, + forward_ref_binop! { impl const BitOr, bitor for Wrapping<$t>, Wrapping<$t>, #[stable(feature = "wrapping_ref", since = "1.14.0")] } #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl BitOrAssign for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const BitOrAssign for Wrapping<$t> { #[inline] fn bitor_assign(&mut self, other: Wrapping<$t>) { *self = *self | other; } } - forward_ref_op_assign! { impl BitOrAssign, bitor_assign for Wrapping<$t>, Wrapping<$t> } + forward_ref_op_assign! { impl const BitOrAssign, bitor_assign for Wrapping<$t>, Wrapping<$t> } + + #[stable(feature = "wrapping_int_assign_impl", since = "1.60.0")] + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const BitOrAssign<$t> for Wrapping<$t> { + #[inline] + fn bitor_assign(&mut self, other: $t) { + *self = *self | Wrapping(other); + } + } + forward_ref_op_assign! { impl const BitOrAssign, bitor_assign for Wrapping<$t>, $t } #[stable(feature = "rust1", since = "1.0.0")] - impl BitAnd for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const BitAnd for Wrapping<$t> { type Output = Wrapping<$t>; #[inline] @@ -295,27 +470,39 @@ macro_rules! wrapping_impl { Wrapping(self.0 & other.0) } } - forward_ref_binop! { impl BitAnd, bitand for Wrapping<$t>, Wrapping<$t>, + forward_ref_binop! { impl const BitAnd, bitand for Wrapping<$t>, Wrapping<$t>, #[stable(feature = "wrapping_ref", since = "1.14.0")] } #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl BitAndAssign for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const BitAndAssign for Wrapping<$t> { #[inline] fn bitand_assign(&mut self, other: Wrapping<$t>) { *self = *self & other; } } - forward_ref_op_assign! { impl BitAndAssign, bitand_assign for Wrapping<$t>, Wrapping<$t> } + forward_ref_op_assign! { impl const BitAndAssign, bitand_assign for Wrapping<$t>, Wrapping<$t> } + + #[stable(feature = "wrapping_int_assign_impl", since = "1.60.0")] + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const BitAndAssign<$t> for Wrapping<$t> { + #[inline] + fn bitand_assign(&mut self, other: $t) { + *self = *self & Wrapping(other); + } + } + forward_ref_op_assign! { impl const BitAndAssign, bitand_assign for Wrapping<$t>, $t } #[stable(feature = "wrapping_neg", since = "1.10.0")] - impl Neg for Wrapping<$t> { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Neg for Wrapping<$t> { type Output = Self; #[inline] fn neg(self) -> Self { Wrapping(0) - self } } - forward_ref_unop! { impl Neg, neg for Wrapping<$t>, + forward_ref_unop! { impl const Neg, neg for Wrapping<$t>, #[stable(feature = "wrapping_ref", since = "1.14.0")] } )*) @@ -326,111 +513,115 @@ wrapping_impl! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } macro_rules! wrapping_int_impl { ($($t:ty)*) => ($( impl Wrapping<$t> { - doc_comment! { - concat!("Returns the smallest value that can be represented by this integer type. - -# Examples - -Basic usage: - -``` -#![feature(wrapping_int_impl)] -use std::num::Wrapping; - -assert_eq!(>::min_value(), ", -"Wrapping(", stringify!($t), "::min_value())); -```"), - #[unstable(feature = "wrapping_int_impl", issue = "32463")] - #[inline] - pub const fn min_value() -> Self { - Wrapping(<$t>::min_value()) - } - } - - doc_comment! { - concat!("Returns the largest value that can be represented by this integer type. - -# Examples - -Basic usage: - -``` -#![feature(wrapping_int_impl)] -use std::num::Wrapping; - -assert_eq!(>::max_value(), ", -"Wrapping(", stringify!($t), "::max_value())); -```"), - #[unstable(feature = "wrapping_int_impl", issue = "32463")] - #[inline] - pub const fn max_value() -> Self { - Wrapping(<$t>::max_value()) - } - } - - doc_comment! { - concat!("Returns the number of ones in the binary representation of `self`. - -# Examples - -Basic usage: + /// Returns the smallest value that can be represented by this integer type. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(wrapping_int_impl)] + /// use std::num::Wrapping; + /// + #[doc = concat!("assert_eq!(>::MIN, Wrapping(", stringify!($t), "::MIN));")] + /// ``` + #[unstable(feature = "wrapping_int_impl", issue = "32463")] + pub const MIN: Self = Self(<$t>::MIN); -``` -#![feature(wrapping_int_impl)] -use std::num::Wrapping; + /// Returns the largest value that can be represented by this integer type. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(wrapping_int_impl)] + /// use std::num::Wrapping; + /// + #[doc = concat!("assert_eq!(>::MAX, Wrapping(", stringify!($t), "::MAX));")] + /// ``` + #[unstable(feature = "wrapping_int_impl", issue = "32463")] + pub const MAX: Self = Self(<$t>::MAX); -let n = Wrapping(0b01001100", stringify!($t), "); + /// Returns the size of this integer type in bits. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(wrapping_int_impl)] + /// use std::num::Wrapping; + /// + #[doc = concat!("assert_eq!(>::BITS, ", stringify!($t), "::BITS);")] + /// ``` + #[unstable(feature = "wrapping_int_impl", issue = "32463")] + pub const BITS: u32 = <$t>::BITS; -assert_eq!(n.count_ones(), 3); -```"), - #[inline] - #[unstable(feature = "wrapping_int_impl", issue = "32463")] - pub const fn count_ones(self) -> u32 { - self.0.count_ones() - } + /// Returns the number of ones in the binary representation of `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(wrapping_int_impl)] + /// use std::num::Wrapping; + /// + #[doc = concat!("let n = Wrapping(0b01001100", stringify!($t), ");")] + /// + /// assert_eq!(n.count_ones(), 3); + /// ``` + #[inline] + #[doc(alias = "popcount")] + #[doc(alias = "popcnt")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[unstable(feature = "wrapping_int_impl", issue = "32463")] + pub const fn count_ones(self) -> u32 { + self.0.count_ones() } - doc_comment! { - concat!("Returns the number of zeros in the binary representation of `self`. - -# Examples - -Basic usage: - -``` -#![feature(wrapping_int_impl)] -use std::num::Wrapping; - -assert_eq!(Wrapping(!0", stringify!($t), ").count_zeros(), 0); -```"), - #[inline] - #[unstable(feature = "wrapping_int_impl", issue = "32463")] - pub const fn count_zeros(self) -> u32 { - self.0.count_zeros() - } + /// Returns the number of zeros in the binary representation of `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(wrapping_int_impl)] + /// use std::num::Wrapping; + /// + #[doc = concat!("assert_eq!(Wrapping(!0", stringify!($t), ").count_zeros(), 0);")] + /// ``` + #[inline] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[unstable(feature = "wrapping_int_impl", issue = "32463")] + pub const fn count_zeros(self) -> u32 { + self.0.count_zeros() } - doc_comment! { - concat!("Returns the number of trailing zeros in the binary representation -of `self`. - -# Examples - -Basic usage: - -``` -#![feature(wrapping_int_impl)] -use std::num::Wrapping; - -let n = Wrapping(0b0101000", stringify!($t), "); - -assert_eq!(n.trailing_zeros(), 3); -```"), - #[inline] - #[unstable(feature = "wrapping_int_impl", issue = "32463")] - pub const fn trailing_zeros(self) -> u32 { - self.0.trailing_zeros() - } + /// Returns the number of trailing zeros in the binary representation of `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(wrapping_int_impl)] + /// use std::num::Wrapping; + /// + #[doc = concat!("let n = Wrapping(0b0101000", stringify!($t), ");")] + /// + /// assert_eq!(n.trailing_zeros(), 3); + /// ``` + #[inline] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[unstable(feature = "wrapping_int_impl", issue = "32463")] + pub const fn trailing_zeros(self) -> u32 { + self.0.trailing_zeros() } /// Shifts the bits to the left by a specified amount, `n`, @@ -454,6 +645,8 @@ assert_eq!(n.trailing_zeros(), 3); /// assert_eq!(n.rotate_left(32), m); /// ``` #[inline] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[unstable(feature = "wrapping_int_impl", issue = "32463")] pub const fn rotate_left(self, n: u32) -> Self { Wrapping(self.0.rotate_left(n)) @@ -480,6 +673,8 @@ assert_eq!(n.trailing_zeros(), 3); /// assert_eq!(n.rotate_right(4), m); /// ``` #[inline] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[unstable(feature = "wrapping_int_impl", issue = "32463")] pub const fn rotate_right(self, n: u32) -> Self { Wrapping(self.0.rotate_right(n)) @@ -504,6 +699,8 @@ assert_eq!(n.trailing_zeros(), 3); /// assert_eq!(m, Wrapping(21760)); /// ``` #[inline] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[unstable(feature = "wrapping_int_impl", issue = "32463")] pub const fn swap_bytes(self) -> Self { Wrapping(self.0.swap_bytes()) @@ -531,156 +728,155 @@ assert_eq!(n.trailing_zeros(), 3); /// ``` #[stable(feature = "reverse_bits", since = "1.37.0")] #[rustc_const_stable(feature = "const_reverse_bits", since = "1.37.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[inline] - #[must_use] pub const fn reverse_bits(self) -> Self { Wrapping(self.0.reverse_bits()) } - doc_comment! { - concat!("Converts an integer from big endian to the target's endianness. - -On big endian this is a no-op. On little endian the bytes are -swapped. - -# Examples - -Basic usage: - -``` -#![feature(wrapping_int_impl)] -use std::num::Wrapping; - -let n = Wrapping(0x1A", stringify!($t), "); - -if cfg!(target_endian = \"big\") { - assert_eq!(>::from_be(n), n) -} else { - assert_eq!(>::from_be(n), n.swap_bytes()) -} -```"), - #[inline] - #[unstable(feature = "wrapping_int_impl", issue = "32463")] - pub const fn from_be(x: Self) -> Self { - Wrapping(<$t>::from_be(x.0)) - } + /// Converts an integer from big endian to the target's endianness. + /// + /// On big endian this is a no-op. On little endian the bytes are + /// swapped. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(wrapping_int_impl)] + /// use std::num::Wrapping; + /// + #[doc = concat!("let n = Wrapping(0x1A", stringify!($t), ");")] + /// + /// if cfg!(target_endian = "big") { + #[doc = concat!(" assert_eq!(>::from_be(n), n)")] + /// } else { + #[doc = concat!(" assert_eq!(>::from_be(n), n.swap_bytes())")] + /// } + /// ``` + #[inline] + #[must_use] + #[unstable(feature = "wrapping_int_impl", issue = "32463")] + pub const fn from_be(x: Self) -> Self { + Wrapping(<$t>::from_be(x.0)) } - doc_comment! { - concat!("Converts an integer from little endian to the target's endianness. - -On little endian this is a no-op. On big endian the bytes are -swapped. - -# Examples - -Basic usage: - -``` -#![feature(wrapping_int_impl)] -use std::num::Wrapping; - -let n = Wrapping(0x1A", stringify!($t), "); - -if cfg!(target_endian = \"little\") { - assert_eq!(>::from_le(n), n) -} else { - assert_eq!(>::from_le(n), n.swap_bytes()) -} -```"), - #[inline] - #[unstable(feature = "wrapping_int_impl", issue = "32463")] - pub const fn from_le(x: Self) -> Self { - Wrapping(<$t>::from_le(x.0)) - } + /// Converts an integer from little endian to the target's endianness. + /// + /// On little endian this is a no-op. On big endian the bytes are + /// swapped. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(wrapping_int_impl)] + /// use std::num::Wrapping; + /// + #[doc = concat!("let n = Wrapping(0x1A", stringify!($t), ");")] + /// + /// if cfg!(target_endian = "little") { + #[doc = concat!(" assert_eq!(>::from_le(n), n)")] + /// } else { + #[doc = concat!(" assert_eq!(>::from_le(n), n.swap_bytes())")] + /// } + /// ``` + #[inline] + #[must_use] + #[unstable(feature = "wrapping_int_impl", issue = "32463")] + pub const fn from_le(x: Self) -> Self { + Wrapping(<$t>::from_le(x.0)) } - doc_comment! { - concat!("Converts `self` to big endian from the target's endianness. - -On big endian this is a no-op. On little endian the bytes are -swapped. - -# Examples - -Basic usage: - -``` -#![feature(wrapping_int_impl)] -use std::num::Wrapping; - -let n = Wrapping(0x1A", stringify!($t), "); - -if cfg!(target_endian = \"big\") { - assert_eq!(n.to_be(), n) -} else { - assert_eq!(n.to_be(), n.swap_bytes()) -} -```"), - #[inline] - #[unstable(feature = "wrapping_int_impl", issue = "32463")] - pub const fn to_be(self) -> Self { - Wrapping(self.0.to_be()) - } + /// Converts `self` to big endian from the target's endianness. + /// + /// On big endian this is a no-op. On little endian the bytes are + /// swapped. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(wrapping_int_impl)] + /// use std::num::Wrapping; + /// + #[doc = concat!("let n = Wrapping(0x1A", stringify!($t), ");")] + /// + /// if cfg!(target_endian = "big") { + /// assert_eq!(n.to_be(), n) + /// } else { + /// assert_eq!(n.to_be(), n.swap_bytes()) + /// } + /// ``` + #[inline] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[unstable(feature = "wrapping_int_impl", issue = "32463")] + pub const fn to_be(self) -> Self { + Wrapping(self.0.to_be()) } - doc_comment! { - concat!("Converts `self` to little endian from the target's endianness. - -On little endian this is a no-op. On big endian the bytes are -swapped. - -# Examples - -Basic usage: - -``` -#![feature(wrapping_int_impl)] -use std::num::Wrapping; - -let n = Wrapping(0x1A", stringify!($t), "); - -if cfg!(target_endian = \"little\") { - assert_eq!(n.to_le(), n) -} else { - assert_eq!(n.to_le(), n.swap_bytes()) -} -```"), - #[inline] - #[unstable(feature = "wrapping_int_impl", issue = "32463")] - pub const fn to_le(self) -> Self { - Wrapping(self.0.to_le()) - } + /// Converts `self` to little endian from the target's endianness. + /// + /// On little endian this is a no-op. On big endian the bytes are + /// swapped. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(wrapping_int_impl)] + /// use std::num::Wrapping; + /// + #[doc = concat!("let n = Wrapping(0x1A", stringify!($t), ");")] + /// + /// if cfg!(target_endian = "little") { + /// assert_eq!(n.to_le(), n) + /// } else { + /// assert_eq!(n.to_le(), n.swap_bytes()) + /// } + /// ``` + #[inline] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[unstable(feature = "wrapping_int_impl", issue = "32463")] + pub const fn to_le(self) -> Self { + Wrapping(self.0.to_le()) } - doc_comment! { - concat!("Raises self to the power of `exp`, using exponentiation by squaring. - -# Examples - -Basic usage: - -``` -#![feature(wrapping_int_impl)] -use std::num::Wrapping; - -assert_eq!(Wrapping(3", stringify!($t), ").pow(4), Wrapping(81)); -``` - -Results that are too large are wrapped: - -``` -#![feature(wrapping_int_impl)] -use std::num::Wrapping; - -assert_eq!(Wrapping(3i8).pow(5), Wrapping(-13)); -assert_eq!(Wrapping(3i8).pow(6), Wrapping(-39)); -```"), - #[inline] - #[unstable(feature = "wrapping_int_impl", issue = "32463")] - pub fn pow(self, exp: u32) -> Self { - Wrapping(self.0.wrapping_pow(exp)) - } + /// Raises self to the power of `exp`, using exponentiation by squaring. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(wrapping_int_impl)] + /// use std::num::Wrapping; + /// + #[doc = concat!("assert_eq!(Wrapping(3", stringify!($t), ").pow(4), Wrapping(81));")] + /// ``` + /// + /// Results that are too large are wrapped: + /// + /// ``` + /// #![feature(wrapping_int_impl)] + /// use std::num::Wrapping; + /// + /// assert_eq!(Wrapping(3i8).pow(5), Wrapping(-13)); + /// assert_eq!(Wrapping(3i8).pow(6), Wrapping(-39)); + /// ``` + #[inline] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[unstable(feature = "wrapping_int_impl", issue = "32463")] + pub fn pow(self, exp: u32) -> Self { + Wrapping(self.0.wrapping_pow(exp)) } } )*) @@ -691,125 +887,122 @@ wrapping_int_impl! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } macro_rules! wrapping_int_impl_signed { ($($t:ty)*) => ($( impl Wrapping<$t> { - doc_comment! { - concat!("Returns the number of leading zeros in the binary representation of `self`. - -# Examples - -Basic usage: - -``` -#![feature(wrapping_int_impl)] -use std::num::Wrapping; - -let n = Wrapping(", stringify!($t), "::max_value()) >> 2; - -assert_eq!(n.leading_zeros(), 3); -```"), - #[inline] - #[unstable(feature = "wrapping_int_impl", issue = "32463")] - pub const fn leading_zeros(self) -> u32 { - self.0.leading_zeros() - } + /// Returns the number of leading zeros in the binary representation of `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(wrapping_int_impl)] + /// use std::num::Wrapping; + /// + #[doc = concat!("let n = Wrapping(", stringify!($t), "::MAX) >> 2;")] + /// + /// assert_eq!(n.leading_zeros(), 3); + /// ``` + #[inline] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[unstable(feature = "wrapping_int_impl", issue = "32463")] + pub const fn leading_zeros(self) -> u32 { + self.0.leading_zeros() } - doc_comment! { - concat!("Computes the absolute value of `self`, wrapping around at -the boundary of the type. - -The only case where such wrapping can occur is when one takes the absolute value of the negative -minimal value for the type this is a positive value that is too large to represent in the type. In -such a case, this function returns `MIN` itself. - -# Examples - -Basic usage: - -``` -#![feature(wrapping_int_impl)] -use std::num::Wrapping; - -assert_eq!(Wrapping(100", stringify!($t), ").abs(), Wrapping(100)); -assert_eq!(Wrapping(-100", stringify!($t), ").abs(), Wrapping(100)); -assert_eq!(Wrapping(", stringify!($t), "::min_value()).abs(), Wrapping(", stringify!($t), -"::min_value())); -assert_eq!(Wrapping(-128i8).abs().0 as u8, 128u8); -```"), - #[inline] - #[unstable(feature = "wrapping_int_impl", issue = "32463")] - pub fn abs(self) -> Wrapping<$t> { - Wrapping(self.0.wrapping_abs()) - } + /// Computes the absolute value of `self`, wrapping around at + /// the boundary of the type. + /// + /// The only case where such wrapping can occur is when one takes the absolute value of the negative + /// minimal value for the type this is a positive value that is too large to represent in the type. In + /// such a case, this function returns `MIN` itself. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(wrapping_int_impl)] + /// use std::num::Wrapping; + /// + #[doc = concat!("assert_eq!(Wrapping(100", stringify!($t), ").abs(), Wrapping(100));")] + #[doc = concat!("assert_eq!(Wrapping(-100", stringify!($t), ").abs(), Wrapping(100));")] + #[doc = concat!("assert_eq!(Wrapping(", stringify!($t), "::MIN).abs(), Wrapping(", stringify!($t), "::MIN));")] + /// assert_eq!(Wrapping(-128i8).abs().0 as u8, 128u8); + /// ``` + #[inline] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[unstable(feature = "wrapping_int_impl", issue = "32463")] + pub fn abs(self) -> Wrapping<$t> { + Wrapping(self.0.wrapping_abs()) } - doc_comment! { - concat!("Returns a number representing sign of `self`. - - - `0` if the number is zero - - `1` if the number is positive - - `-1` if the number is negative - -# Examples - -Basic usage: - -``` -#![feature(wrapping_int_impl)] -use std::num::Wrapping; - -assert_eq!(Wrapping(10", stringify!($t), ").signum(), Wrapping(1)); -assert_eq!(Wrapping(0", stringify!($t), ").signum(), Wrapping(0)); -assert_eq!(Wrapping(-10", stringify!($t), ").signum(), Wrapping(-1)); -```"), - #[inline] - #[unstable(feature = "wrapping_int_impl", issue = "32463")] - pub fn signum(self) -> Wrapping<$t> { - Wrapping(self.0.signum()) - } + /// Returns a number representing sign of `self`. + /// + /// - `0` if the number is zero + /// - `1` if the number is positive + /// - `-1` if the number is negative + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(wrapping_int_impl)] + /// use std::num::Wrapping; + /// + #[doc = concat!("assert_eq!(Wrapping(10", stringify!($t), ").signum(), Wrapping(1));")] + #[doc = concat!("assert_eq!(Wrapping(0", stringify!($t), ").signum(), Wrapping(0));")] + #[doc = concat!("assert_eq!(Wrapping(-10", stringify!($t), ").signum(), Wrapping(-1));")] + /// ``` + #[inline] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[unstable(feature = "wrapping_int_impl", issue = "32463")] + pub fn signum(self) -> Wrapping<$t> { + Wrapping(self.0.signum()) } - doc_comment! { - concat!("Returns `true` if `self` is positive and `false` if the number is zero or -negative. - -# Examples - -Basic usage: - -``` -#![feature(wrapping_int_impl)] -use std::num::Wrapping; - -assert!(Wrapping(10", stringify!($t), ").is_positive()); -assert!(!Wrapping(-10", stringify!($t), ").is_positive()); -```"), - #[inline] - #[unstable(feature = "wrapping_int_impl", issue = "32463")] - pub const fn is_positive(self) -> bool { - self.0.is_positive() - } + /// Returns `true` if `self` is positive and `false` if the number is zero or + /// negative. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(wrapping_int_impl)] + /// use std::num::Wrapping; + /// + #[doc = concat!("assert!(Wrapping(10", stringify!($t), ").is_positive());")] + #[doc = concat!("assert!(!Wrapping(-10", stringify!($t), ").is_positive());")] + /// ``` + #[must_use] + #[inline] + #[unstable(feature = "wrapping_int_impl", issue = "32463")] + pub const fn is_positive(self) -> bool { + self.0.is_positive() } - doc_comment! { - concat!("Returns `true` if `self` is negative and `false` if the number is zero or -positive. - -# Examples - -Basic usage: - -``` -#![feature(wrapping_int_impl)] -use std::num::Wrapping; - -assert!(Wrapping(-10", stringify!($t), ").is_negative()); -assert!(!Wrapping(10", stringify!($t), ").is_negative()); -```"), - #[inline] - #[unstable(feature = "wrapping_int_impl", issue = "32463")] - pub const fn is_negative(self) -> bool { - self.0.is_negative() - } + /// Returns `true` if `self` is negative and `false` if the number is zero or + /// positive. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(wrapping_int_impl)] + /// use std::num::Wrapping; + /// + #[doc = concat!("assert!(Wrapping(-10", stringify!($t), ").is_negative());")] + #[doc = concat!("assert!(!Wrapping(10", stringify!($t), ").is_negative());")] + /// ``` + #[must_use] + #[inline] + #[unstable(feature = "wrapping_int_impl", issue = "32463")] + pub const fn is_negative(self) -> bool { + self.0.is_negative() } } )*) @@ -820,73 +1013,72 @@ wrapping_int_impl_signed! { isize i8 i16 i32 i64 i128 } macro_rules! wrapping_int_impl_unsigned { ($($t:ty)*) => ($( impl Wrapping<$t> { - doc_comment! { - concat!("Returns the number of leading zeros in the binary representation of `self`. - -# Examples - -Basic usage: - -``` -#![feature(wrapping_int_impl)] -use std::num::Wrapping; - -let n = Wrapping(", stringify!($t), "::max_value()) >> 2; - -assert_eq!(n.leading_zeros(), 2); -```"), - #[inline] - #[unstable(feature = "wrapping_int_impl", issue = "32463")] - pub const fn leading_zeros(self) -> u32 { - self.0.leading_zeros() - } + /// Returns the number of leading zeros in the binary representation of `self`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(wrapping_int_impl)] + /// use std::num::Wrapping; + /// + #[doc = concat!("let n = Wrapping(", stringify!($t), "::MAX) >> 2;")] + /// + /// assert_eq!(n.leading_zeros(), 2); + /// ``` + #[inline] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[unstable(feature = "wrapping_int_impl", issue = "32463")] + pub const fn leading_zeros(self) -> u32 { + self.0.leading_zeros() } - doc_comment! { - concat!("Returns `true` if and only if `self == 2^k` for some `k`. - -# Examples - -Basic usage: - -``` -#![feature(wrapping_int_impl)] -use std::num::Wrapping; - -assert!(Wrapping(16", stringify!($t), ").is_power_of_two()); -assert!(!Wrapping(10", stringify!($t), ").is_power_of_two()); -```"), - #[inline] - #[unstable(feature = "wrapping_int_impl", issue = "32463")] - pub fn is_power_of_two(self) -> bool { - self.0.is_power_of_two() - } + /// Returns `true` if and only if `self == 2^k` for some `k`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(wrapping_int_impl)] + /// use std::num::Wrapping; + /// + #[doc = concat!("assert!(Wrapping(16", stringify!($t), ").is_power_of_two());")] + #[doc = concat!("assert!(!Wrapping(10", stringify!($t), ").is_power_of_two());")] + /// ``` + #[must_use] + #[inline] + #[unstable(feature = "wrapping_int_impl", issue = "32463")] + pub fn is_power_of_two(self) -> bool { + self.0.is_power_of_two() } - doc_comment! { - concat!("Returns the smallest power of two greater than or equal to `self`. - -When return value overflows (i.e., `self > (1 << (N-1))` for type -`uN`), overflows to `2^N = 0`. - -# Examples - -Basic usage: - -``` -#![feature(wrapping_next_power_of_two)] -use std::num::Wrapping; - -assert_eq!(Wrapping(2", stringify!($t), ").next_power_of_two(), Wrapping(2)); -assert_eq!(Wrapping(3", stringify!($t), ").next_power_of_two(), Wrapping(4)); -assert_eq!(Wrapping(200_u8).next_power_of_two(), Wrapping(0)); -```"), - #[inline] - #[unstable(feature = "wrapping_next_power_of_two", issue = "32463", - reason = "needs decision on wrapping behaviour")] - pub fn next_power_of_two(self) -> Self { - Wrapping(self.0.wrapping_next_power_of_two()) - } + /// Returns the smallest power of two greater than or equal to `self`. + /// + /// When return value overflows (i.e., `self > (1 << (N-1))` for type + /// `uN`), overflows to `2^N = 0`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(wrapping_next_power_of_two)] + /// use std::num::Wrapping; + /// + #[doc = concat!("assert_eq!(Wrapping(2", stringify!($t), ").next_power_of_two(), Wrapping(2));")] + #[doc = concat!("assert_eq!(Wrapping(3", stringify!($t), ").next_power_of_two(), Wrapping(4));")] + #[doc = concat!("assert_eq!(Wrapping(200_u8).next_power_of_two(), Wrapping(0));")] + /// ``` + #[inline] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[unstable(feature = "wrapping_next_power_of_two", issue = "32463", + reason = "needs decision on wrapping behaviour")] + pub fn next_power_of_two(self) -> Self { + Wrapping(self.0.wrapping_next_power_of_two()) } } )*) diff --git a/crux-mir/lib/core/src/ops/arith.rs b/crux-mir/lib/core/src/ops/arith.rs index e9ec81394..75c52d3ec 100644 --- a/crux-mir/lib/core/src/ops/arith.rs +++ b/crux-mir/lib/core/src/ops/arith.rs @@ -69,15 +69,23 @@ on(all(_Self = "{integer}", Rhs = "{float}"), message = "cannot add a float to an integer",), on(all(_Self = "{float}", Rhs = "{integer}"), message = "cannot add an integer to a float",), message = "cannot add `{Rhs}` to `{Self}`", - label = "no implementation for `{Self} + {Rhs}`" + label = "no implementation for `{Self} + {Rhs}`", + append_const_msg )] #[doc(alias = "+")] +#[const_trait] pub trait Add { /// The resulting type after applying the `+` operator. #[stable(feature = "rust1", since = "1.0.0")] type Output; /// Performs the `+` operation. + /// + /// # Example + /// + /// ``` + /// assert_eq!(12 + 1, 13); + /// ``` #[must_use] #[stable(feature = "rust1", since = "1.0.0")] fn add(self, rhs: Rhs) -> Self::Output; @@ -86,7 +94,8 @@ pub trait Add { macro_rules! add_impl { ($($t:ty)*) => ($( #[stable(feature = "rust1", since = "1.0.0")] - impl Add for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Add for $t { type Output = $t; #[inline] @@ -94,7 +103,7 @@ macro_rules! add_impl { fn add(self, other: $t) -> $t { self + other } } - forward_ref_binop! { impl Add, add for $t, $t } + forward_ref_binop! { impl const Add, add for $t, $t } )*) } @@ -122,10 +131,10 @@ add_impl! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 f32 f64 } /// } /// /// impl Sub for Point { -/// type Output = Point; +/// type Output = Self; /// -/// fn sub(self, other: Point) -> Point { -/// Point { +/// fn sub(self, other: Self) -> Self::Output { +/// Self { /// x: self.x - other.x, /// y: self.y - other.y, /// } @@ -169,15 +178,23 @@ add_impl! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 f32 f64 } #[stable(feature = "rust1", since = "1.0.0")] #[rustc_on_unimplemented( message = "cannot subtract `{Rhs}` from `{Self}`", - label = "no implementation for `{Self} - {Rhs}`" + label = "no implementation for `{Self} - {Rhs}`", + append_const_msg )] #[doc(alias = "-")] +#[const_trait] pub trait Sub { /// The resulting type after applying the `-` operator. #[stable(feature = "rust1", since = "1.0.0")] type Output; /// Performs the `-` operation. + /// + /// # Example + /// + /// ``` + /// assert_eq!(12 - 1, 11); + /// ``` #[must_use] #[stable(feature = "rust1", since = "1.0.0")] fn sub(self, rhs: Rhs) -> Self::Output; @@ -186,7 +203,8 @@ pub trait Sub { macro_rules! sub_impl { ($($t:ty)*) => ($( #[stable(feature = "rust1", since = "1.0.0")] - impl Sub for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Sub for $t { type Output = $t; #[inline] @@ -194,7 +212,7 @@ macro_rules! sub_impl { fn sub(self, other: $t) -> $t { self - other } } - forward_ref_binop! { impl Sub, sub for $t, $t } + forward_ref_binop! { impl const Sub, sub for $t, $t } )*) } @@ -229,7 +247,7 @@ sub_impl! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 f32 f64 } /// // Reduce to lowest terms by dividing by the greatest common /// // divisor. /// let gcd = gcd(numerator, denominator); -/// Rational { +/// Self { /// numerator: numerator / gcd, /// denominator: denominator / gcd, /// } @@ -243,7 +261,7 @@ sub_impl! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 f32 f64 } /// fn mul(self, rhs: Self) -> Self { /// let numerator = self.numerator * rhs.numerator; /// let denominator = self.denominator * rhs.denominator; -/// Rational::new(numerator, denominator) +/// Self::new(numerator, denominator) /// } /// } /// @@ -279,7 +297,7 @@ sub_impl! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 f32 f64 } /// type Output = Self; /// /// fn mul(self, rhs: Scalar) -> Self::Output { -/// Vector { value: self.value.iter().map(|v| v * rhs.value).collect() } +/// Self { value: self.value.iter().map(|v| v * rhs.value).collect() } /// } /// } /// @@ -290,16 +308,23 @@ sub_impl! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 f32 f64 } #[lang = "mul"] #[stable(feature = "rust1", since = "1.0.0")] #[rustc_on_unimplemented( - message = "cannot multiply `{Rhs}` to `{Self}`", + message = "cannot multiply `{Self}` by `{Rhs}`", label = "no implementation for `{Self} * {Rhs}`" )] #[doc(alias = "*")] +#[const_trait] pub trait Mul { /// The resulting type after applying the `*` operator. #[stable(feature = "rust1", since = "1.0.0")] type Output; /// Performs the `*` operation. + /// + /// # Example + /// + /// ``` + /// assert_eq!(12 * 2, 24); + /// ``` #[must_use] #[stable(feature = "rust1", since = "1.0.0")] fn mul(self, rhs: Rhs) -> Self::Output; @@ -308,7 +333,8 @@ pub trait Mul { macro_rules! mul_impl { ($($t:ty)*) => ($( #[stable(feature = "rust1", since = "1.0.0")] - impl Mul for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Mul for $t { type Output = $t; #[inline] @@ -316,7 +342,7 @@ macro_rules! mul_impl { fn mul(self, other: $t) -> $t { self * other } } - forward_ref_binop! { impl Mul, mul for $t, $t } + forward_ref_binop! { impl const Mul, mul for $t, $t } )*) } @@ -351,7 +377,7 @@ mul_impl! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 f32 f64 } /// // Reduce to lowest terms by dividing by the greatest common /// // divisor. /// let gcd = gcd(numerator, denominator); -/// Rational { +/// Self { /// numerator: numerator / gcd, /// denominator: denominator / gcd, /// } @@ -369,7 +395,7 @@ mul_impl! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 f32 f64 } /// /// let numerator = self.numerator * rhs.denominator; /// let denominator = self.denominator * rhs.numerator; -/// Rational::new(numerator, denominator) +/// Self::new(numerator, denominator) /// } /// } /// @@ -405,7 +431,7 @@ mul_impl! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 f32 f64 } /// type Output = Self; /// /// fn div(self, rhs: Scalar) -> Self::Output { -/// Vector { value: self.value.iter().map(|v| v / rhs.value).collect() } +/// Self { value: self.value.iter().map(|v| v / rhs.value).collect() } /// } /// } /// @@ -420,46 +446,62 @@ mul_impl! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 f32 f64 } label = "no implementation for `{Self} / {Rhs}`" )] #[doc(alias = "/")] +#[const_trait] pub trait Div { /// The resulting type after applying the `/` operator. #[stable(feature = "rust1", since = "1.0.0")] type Output; /// Performs the `/` operation. + /// + /// # Example + /// + /// ``` + /// assert_eq!(12 / 2, 6); + /// ``` #[must_use] #[stable(feature = "rust1", since = "1.0.0")] fn div(self, rhs: Rhs) -> Self::Output; } macro_rules! div_impl_integer { - ($($t:ty)*) => ($( + ($(($($t:ty)*) => $panic:expr),*) => ($($( /// This operation rounds towards zero, truncating any /// fractional part of the exact result. + /// + /// # Panics + /// + #[doc = $panic] #[stable(feature = "rust1", since = "1.0.0")] - impl Div for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Div for $t { type Output = $t; #[inline] fn div(self, other: $t) -> $t { self / other } } - forward_ref_binop! { impl Div, div for $t, $t } - )*) + forward_ref_binop! { impl const Div, div for $t, $t } + )*)*) } -div_impl_integer! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } +div_impl_integer! { + (usize u8 u16 u32 u64 u128) => "This operation will panic if `other == 0`.", + (isize i8 i16 i32 i64 i128) => "This operation will panic if `other == 0` or the division results in overflow." +} macro_rules! div_impl_float { ($($t:ty)*) => ($( #[stable(feature = "rust1", since = "1.0.0")] - impl Div for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Div for $t { type Output = $t; #[inline] fn div(self, other: $t) -> $t { self / other } } - forward_ref_binop! { impl Div, div for $t, $t } + forward_ref_binop! { impl const Div, div for $t, $t } )*) } @@ -491,7 +533,7 @@ div_impl_float! { f32 f64 } /// let len = self.slice.len(); /// let rem = len % modulus; /// let start = len - rem; -/// SplitSlice {slice: &self.slice[start..]} +/// Self {slice: &self.slice[start..]} /// } /// } /// @@ -507,34 +549,49 @@ div_impl_float! { f32 f64 } label = "no implementation for `{Self} % {Rhs}`" )] #[doc(alias = "%")] +#[const_trait] pub trait Rem { /// The resulting type after applying the `%` operator. #[stable(feature = "rust1", since = "1.0.0")] type Output; /// Performs the `%` operation. + /// + /// # Example + /// + /// ``` + /// assert_eq!(12 % 10, 2); + /// ``` #[must_use] #[stable(feature = "rust1", since = "1.0.0")] fn rem(self, rhs: Rhs) -> Self::Output; } macro_rules! rem_impl_integer { - ($($t:ty)*) => ($( + ($(($($t:ty)*) => $panic:expr),*) => ($($( /// This operation satisfies `n % d == n - (n / d) * d`. The /// result has the same sign as the left operand. + /// + /// # Panics + /// + #[doc = $panic] #[stable(feature = "rust1", since = "1.0.0")] - impl Rem for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Rem for $t { type Output = $t; #[inline] fn rem(self, other: $t) -> $t { self % other } } - forward_ref_binop! { impl Rem, rem for $t, $t } - )*) + forward_ref_binop! { impl const Rem, rem for $t, $t } + )*)*) } -rem_impl_integer! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } +rem_impl_integer! { + (usize u8 u16 u32 u64 u128) => "This operation will panic if `other == 0`.", + (isize i8 i16 i32 i64 i128) => "This operation will panic if `other == 0` or if `self / other` results in overflow." +} macro_rules! rem_impl_float { ($($t:ty)*) => ($( @@ -554,14 +611,15 @@ macro_rules! rem_impl_float { /// assert_eq!(x % y, remainder); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - impl Rem for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Rem for $t { type Output = $t; #[inline] fn rem(self, other: $t) -> $t { self % other } } - forward_ref_binop! { impl Rem, rem for $t, $t } + forward_ref_binop! { impl const Rem, rem for $t, $t } )*) } @@ -585,7 +643,7 @@ rem_impl_float! { f32 f64 } /// } /// /// impl Neg for Sign { -/// type Output = Sign; +/// type Output = Self; /// /// fn neg(self) -> Self::Output { /// match self { @@ -606,46 +664,42 @@ rem_impl_float! { f32 f64 } #[lang = "neg"] #[stable(feature = "rust1", since = "1.0.0")] #[doc(alias = "-")] +#[const_trait] pub trait Neg { /// The resulting type after applying the `-` operator. #[stable(feature = "rust1", since = "1.0.0")] type Output; /// Performs the unary `-` operation. + /// + /// # Example + /// + /// ``` + /// let x: i32 = 12; + /// assert_eq!(-x, -12); + /// ``` #[must_use] #[stable(feature = "rust1", since = "1.0.0")] fn neg(self) -> Self::Output; } -macro_rules! neg_impl_core { - ($id:ident => $body:expr, $($t:ty)*) => ($( +macro_rules! neg_impl { + ($($t:ty)*) => ($( #[stable(feature = "rust1", since = "1.0.0")] - impl Neg for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Neg for $t { type Output = $t; #[inline] #[rustc_inherit_overflow_checks] - fn neg(self) -> $t { let $id = self; $body } + fn neg(self) -> $t { -self } } - forward_ref_unop! { impl Neg, neg for $t } + forward_ref_unop! { impl const Neg, neg for $t } )*) } -macro_rules! neg_impl_numeric { - ($($t:ty)*) => { neg_impl_core!{ x => -x, $($t)*} } -} - -#[allow(unused_macros)] -macro_rules! neg_impl_unsigned { - ($($t:ty)*) => { - neg_impl_core!{ x => { - !x.wrapping_add(1) - }, $($t)*} } -} - -// neg_impl_unsigned! { usize u8 u16 u32 u64 } -neg_impl_numeric! { isize i8 i16 i32 i64 i128 f32 f64 } +neg_impl! { isize i8 i16 i32 i64 i128 f32 f64 } /// The addition assignment operator `+=`. /// @@ -684,8 +738,17 @@ neg_impl_numeric! { isize i8 i16 i32 i64 i128 f32 f64 } )] #[doc(alias = "+")] #[doc(alias = "+=")] +#[const_trait] pub trait AddAssign { /// Performs the `+=` operation. + /// + /// # Example + /// + /// ``` + /// let mut x: u32 = 12; + /// x += 1; + /// assert_eq!(x, 13); + /// ``` #[stable(feature = "op_assign_traits", since = "1.8.0")] fn add_assign(&mut self, rhs: Rhs); } @@ -693,13 +756,14 @@ pub trait AddAssign { macro_rules! add_assign_impl { ($($t:ty)+) => ($( #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl AddAssign for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const AddAssign for $t { #[inline] #[rustc_inherit_overflow_checks] fn add_assign(&mut self, other: $t) { *self += other } } - forward_ref_op_assign! { impl AddAssign, add_assign for $t, $t } + forward_ref_op_assign! { impl const AddAssign, add_assign for $t, $t } )+) } @@ -742,8 +806,17 @@ add_assign_impl! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 f32 f64 } )] #[doc(alias = "-")] #[doc(alias = "-=")] +#[const_trait] pub trait SubAssign { /// Performs the `-=` operation. + /// + /// # Example + /// + /// ``` + /// let mut x: u32 = 12; + /// x -= 1; + /// assert_eq!(x, 11); + /// ``` #[stable(feature = "op_assign_traits", since = "1.8.0")] fn sub_assign(&mut self, rhs: Rhs); } @@ -751,13 +824,14 @@ pub trait SubAssign { macro_rules! sub_assign_impl { ($($t:ty)+) => ($( #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl SubAssign for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const SubAssign for $t { #[inline] #[rustc_inherit_overflow_checks] fn sub_assign(&mut self, other: $t) { *self -= other } } - forward_ref_op_assign! { impl SubAssign, sub_assign for $t, $t } + forward_ref_op_assign! { impl const SubAssign, sub_assign for $t, $t } )+) } @@ -786,13 +860,22 @@ sub_assign_impl! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 f32 f64 } #[lang = "mul_assign"] #[stable(feature = "op_assign_traits", since = "1.8.0")] #[rustc_on_unimplemented( - message = "cannot multiply-assign `{Rhs}` to `{Self}`", + message = "cannot multiply-assign `{Self}` by `{Rhs}`", label = "no implementation for `{Self} *= {Rhs}`" )] #[doc(alias = "*")] #[doc(alias = "*=")] +#[const_trait] pub trait MulAssign { /// Performs the `*=` operation. + /// + /// # Example + /// + /// ``` + /// let mut x: u32 = 12; + /// x *= 2; + /// assert_eq!(x, 24); + /// ``` #[stable(feature = "op_assign_traits", since = "1.8.0")] fn mul_assign(&mut self, rhs: Rhs); } @@ -800,13 +883,14 @@ pub trait MulAssign { macro_rules! mul_assign_impl { ($($t:ty)+) => ($( #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl MulAssign for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const MulAssign for $t { #[inline] #[rustc_inherit_overflow_checks] fn mul_assign(&mut self, other: $t) { *self *= other } } - forward_ref_op_assign! { impl MulAssign, mul_assign for $t, $t } + forward_ref_op_assign! { impl const MulAssign, mul_assign for $t, $t } )+) } @@ -840,8 +924,17 @@ mul_assign_impl! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 f32 f64 } )] #[doc(alias = "/")] #[doc(alias = "/=")] +#[const_trait] pub trait DivAssign { /// Performs the `/=` operation. + /// + /// # Example + /// + /// ``` + /// let mut x: u32 = 12; + /// x /= 2; + /// assert_eq!(x, 6); + /// ``` #[stable(feature = "op_assign_traits", since = "1.8.0")] fn div_assign(&mut self, rhs: Rhs); } @@ -849,12 +942,13 @@ pub trait DivAssign { macro_rules! div_assign_impl { ($($t:ty)+) => ($( #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl DivAssign for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const DivAssign for $t { #[inline] fn div_assign(&mut self, other: $t) { *self /= other } } - forward_ref_op_assign! { impl DivAssign, div_assign for $t, $t } + forward_ref_op_assign! { impl const DivAssign, div_assign for $t, $t } )+) } @@ -892,8 +986,17 @@ div_assign_impl! { usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 f32 f64 } )] #[doc(alias = "%")] #[doc(alias = "%=")] +#[const_trait] pub trait RemAssign { /// Performs the `%=` operation. + /// + /// # Example + /// + /// ``` + /// let mut x: u32 = 12; + /// x %= 10; + /// assert_eq!(x, 2); + /// ``` #[stable(feature = "op_assign_traits", since = "1.8.0")] fn rem_assign(&mut self, rhs: Rhs); } @@ -901,12 +1004,13 @@ pub trait RemAssign { macro_rules! rem_assign_impl { ($($t:ty)+) => ($( #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl RemAssign for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const RemAssign for $t { #[inline] fn rem_assign(&mut self, other: $t) { *self %= other } } - forward_ref_op_assign! { impl RemAssign, rem_assign for $t, $t } + forward_ref_op_assign! { impl const RemAssign, rem_assign for $t, $t } )+) } diff --git a/crux-mir/lib/core/src/ops/bit.rs b/crux-mir/lib/core/src/ops/bit.rs index bcfff4a22..327009801 100644 --- a/crux-mir/lib/core/src/ops/bit.rs +++ b/crux-mir/lib/core/src/ops/bit.rs @@ -15,7 +15,7 @@ /// } /// /// impl Not for Answer { -/// type Output = Answer; +/// type Output = Self; /// /// fn not(self) -> Self::Output { /// match self { @@ -30,12 +30,23 @@ /// ``` #[lang = "not"] #[stable(feature = "rust1", since = "1.0.0")] +#[doc(alias = "!")] +#[const_trait] pub trait Not { /// The resulting type after applying the `!` operator. #[stable(feature = "rust1", since = "1.0.0")] type Output; /// Performs the unary `!` operation. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(!true, false); + /// assert_eq!(!false, true); + /// assert_eq!(!1u8, 254); + /// assert_eq!(!0u8, 255); + /// ``` #[must_use] #[stable(feature = "rust1", since = "1.0.0")] fn not(self) -> Self::Output; @@ -44,19 +55,31 @@ pub trait Not { macro_rules! not_impl { ($($t:ty)*) => ($( #[stable(feature = "rust1", since = "1.0.0")] - impl Not for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Not for $t { type Output = $t; #[inline] fn not(self) -> $t { !self } } - forward_ref_unop! { impl Not, not for $t } + forward_ref_unop! { impl const Not, not for $t } )*) } not_impl! { bool usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } +#[stable(feature = "not_never", since = "1.60.0")] +#[rustc_const_unstable(feature = "const_ops", issue = "90080")] +impl const Not for ! { + type Output = !; + + #[inline] + fn not(self) -> ! { + match self {} + } +} + /// The bitwise AND operator `&`. /// /// Note that `Rhs` is `Self` by default, but this is not mandatory. @@ -76,7 +99,7 @@ not_impl! { bool usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } /// /// // rhs is the "right-hand side" of the expression `a & b` /// fn bitand(self, rhs: Self) -> Self::Output { -/// Scalar(self.0 & rhs.0) +/// Self(self.0 & rhs.0) /// } /// } /// @@ -97,10 +120,15 @@ not_impl! { bool usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } /// impl BitAnd for BooleanVector { /// type Output = Self; /// -/// fn bitand(self, BooleanVector(rhs): Self) -> Self::Output { -/// let BooleanVector(lhs) = self; +/// fn bitand(self, Self(rhs): Self) -> Self::Output { +/// let Self(lhs) = self; /// assert_eq!(lhs.len(), rhs.len()); -/// BooleanVector(lhs.iter().zip(rhs.iter()).map(|(x, y)| *x && *y).collect()) +/// Self( +/// lhs.iter() +/// .zip(rhs.iter()) +/// .map(|(x, y)| *x & *y) +/// .collect() +/// ) /// } /// } /// @@ -116,12 +144,22 @@ not_impl! { bool usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } message = "no implementation for `{Self} & {Rhs}`", label = "no implementation for `{Self} & {Rhs}`" )] +#[const_trait] pub trait BitAnd { /// The resulting type after applying the `&` operator. #[stable(feature = "rust1", since = "1.0.0")] type Output; /// Performs the `&` operation. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(true & false, false); + /// assert_eq!(true & true, true); + /// assert_eq!(5u8 & 1u8, 1); + /// assert_eq!(5u8 & 2u8, 0); + /// ``` #[must_use] #[stable(feature = "rust1", since = "1.0.0")] fn bitand(self, rhs: Rhs) -> Self::Output; @@ -130,14 +168,15 @@ pub trait BitAnd { macro_rules! bitand_impl { ($($t:ty)*) => ($( #[stable(feature = "rust1", since = "1.0.0")] - impl BitAnd for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const BitAnd for $t { type Output = $t; #[inline] fn bitand(self, rhs: $t) -> $t { self & rhs } } - forward_ref_binop! { impl BitAnd, bitand for $t, $t } + forward_ref_binop! { impl const BitAnd, bitand for $t, $t } )*) } @@ -161,8 +200,8 @@ bitand_impl! { bool usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } /// type Output = Self; /// /// // rhs is the "right-hand side" of the expression `a | b` -/// fn bitor(self, rhs: Self) -> Self { -/// Scalar(self.0 | rhs.0) +/// fn bitor(self, rhs: Self) -> Self::Output { +/// Self(self.0 | rhs.0) /// } /// } /// @@ -183,10 +222,15 @@ bitand_impl! { bool usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } /// impl BitOr for BooleanVector { /// type Output = Self; /// -/// fn bitor(self, BooleanVector(rhs): Self) -> Self::Output { -/// let BooleanVector(lhs) = self; +/// fn bitor(self, Self(rhs): Self) -> Self::Output { +/// let Self(lhs) = self; /// assert_eq!(lhs.len(), rhs.len()); -/// BooleanVector(lhs.iter().zip(rhs.iter()).map(|(x, y)| *x || *y).collect()) +/// Self( +/// lhs.iter() +/// .zip(rhs.iter()) +/// .map(|(x, y)| *x | *y) +/// .collect() +/// ) /// } /// } /// @@ -202,12 +246,22 @@ bitand_impl! { bool usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } message = "no implementation for `{Self} | {Rhs}`", label = "no implementation for `{Self} | {Rhs}`" )] +#[const_trait] pub trait BitOr { /// The resulting type after applying the `|` operator. #[stable(feature = "rust1", since = "1.0.0")] type Output; /// Performs the `|` operation. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(true | false, true); + /// assert_eq!(false | false, false); + /// assert_eq!(5u8 | 1u8, 5); + /// assert_eq!(5u8 | 2u8, 7); + /// ``` #[must_use] #[stable(feature = "rust1", since = "1.0.0")] fn bitor(self, rhs: Rhs) -> Self::Output; @@ -216,14 +270,15 @@ pub trait BitOr { macro_rules! bitor_impl { ($($t:ty)*) => ($( #[stable(feature = "rust1", since = "1.0.0")] - impl BitOr for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const BitOr for $t { type Output = $t; #[inline] fn bitor(self, rhs: $t) -> $t { self | rhs } } - forward_ref_binop! { impl BitOr, bitor for $t, $t } + forward_ref_binop! { impl const BitOr, bitor for $t, $t } )*) } @@ -248,7 +303,7 @@ bitor_impl! { bool usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } /// /// // rhs is the "right-hand side" of the expression `a ^ b` /// fn bitxor(self, rhs: Self) -> Self::Output { -/// Scalar(self.0 ^ rhs.0) +/// Self(self.0 ^ rhs.0) /// } /// } /// @@ -269,13 +324,15 @@ bitor_impl! { bool usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } /// impl BitXor for BooleanVector { /// type Output = Self; /// -/// fn bitxor(self, BooleanVector(rhs): Self) -> Self::Output { -/// let BooleanVector(lhs) = self; +/// fn bitxor(self, Self(rhs): Self) -> Self::Output { +/// let Self(lhs) = self; /// assert_eq!(lhs.len(), rhs.len()); -/// BooleanVector(lhs.iter() -/// .zip(rhs.iter()) -/// .map(|(x, y)| (*x || *y) && !(*x && *y)) -/// .collect()) +/// Self( +/// lhs.iter() +/// .zip(rhs.iter()) +/// .map(|(x, y)| *x ^ *y) +/// .collect() +/// ) /// } /// } /// @@ -291,12 +348,22 @@ bitor_impl! { bool usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } message = "no implementation for `{Self} ^ {Rhs}`", label = "no implementation for `{Self} ^ {Rhs}`" )] +#[const_trait] pub trait BitXor { /// The resulting type after applying the `^` operator. #[stable(feature = "rust1", since = "1.0.0")] type Output; /// Performs the `^` operation. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(true ^ false, true); + /// assert_eq!(true ^ true, false); + /// assert_eq!(5u8 ^ 1u8, 4); + /// assert_eq!(5u8 ^ 2u8, 7); + /// ``` #[must_use] #[stable(feature = "rust1", since = "1.0.0")] fn bitxor(self, rhs: Rhs) -> Self::Output; @@ -305,14 +372,15 @@ pub trait BitXor { macro_rules! bitxor_impl { ($($t:ty)*) => ($( #[stable(feature = "rust1", since = "1.0.0")] - impl BitXor for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const BitXor for $t { type Output = $t; #[inline] fn bitxor(self, other: $t) -> $t { self ^ other } } - forward_ref_binop! { impl BitXor, bitxor for $t, $t } + forward_ref_binop! { impl const BitXor, bitxor for $t, $t } )*) } @@ -339,9 +407,9 @@ bitxor_impl! { bool usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } /// impl Shl for Scalar { /// type Output = Self; /// -/// fn shl(self, Scalar(rhs): Self) -> Scalar { -/// let Scalar(lhs) = self; -/// Scalar(lhs << rhs) +/// fn shl(self, Self(rhs): Self) -> Self::Output { +/// let Self(lhs) = self; +/// Self(lhs << rhs) /// } /// } /// @@ -364,10 +432,10 @@ bitxor_impl! { bool usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } /// fn shl(self, rhs: usize) -> Self::Output { /// // Rotate the vector by `rhs` places. /// let (a, b) = self.vec.split_at(rhs); -/// let mut spun_vector: Vec = vec![]; +/// let mut spun_vector = vec![]; /// spun_vector.extend_from_slice(b); /// spun_vector.extend_from_slice(a); -/// SpinVector { vec: spun_vector } +/// Self { vec: spun_vector } /// } /// } /// @@ -381,12 +449,20 @@ bitxor_impl! { bool usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } message = "no implementation for `{Self} << {Rhs}`", label = "no implementation for `{Self} << {Rhs}`" )] +#[const_trait] pub trait Shl { /// The resulting type after applying the `<<` operator. #[stable(feature = "rust1", since = "1.0.0")] type Output; /// Performs the `<<` operation. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(5u8 << 1, 10); + /// assert_eq!(1u8 << 1, 2); + /// ``` #[must_use] #[stable(feature = "rust1", since = "1.0.0")] fn shl(self, rhs: Rhs) -> Self::Output; @@ -395,7 +471,8 @@ pub trait Shl { macro_rules! shl_impl { ($t:ty, $f:ty) => { #[stable(feature = "rust1", since = "1.0.0")] - impl Shl<$f> for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Shl<$f> for $t { type Output = $t; #[inline] @@ -405,7 +482,7 @@ macro_rules! shl_impl { } } - forward_ref_binop! { impl Shl, shl for $t, $f } + forward_ref_binop! { impl const Shl, shl for $t, $f } }; } @@ -450,9 +527,9 @@ shl_impl_all! { u8 u16 u32 u64 u128 usize i8 i16 i32 i64 isize i128 } /// impl Shr for Scalar { /// type Output = Self; /// -/// fn shr(self, Scalar(rhs): Self) -> Scalar { -/// let Scalar(lhs) = self; -/// Scalar(lhs >> rhs) +/// fn shr(self, Self(rhs): Self) -> Self::Output { +/// let Self(lhs) = self; +/// Self(lhs >> rhs) /// } /// } /// @@ -475,10 +552,10 @@ shl_impl_all! { u8 u16 u32 u64 u128 usize i8 i16 i32 i64 isize i128 } /// fn shr(self, rhs: usize) -> Self::Output { /// // Rotate the vector by `rhs` places. /// let (a, b) = self.vec.split_at(self.vec.len() - rhs); -/// let mut spun_vector: Vec = vec![]; +/// let mut spun_vector = vec![]; /// spun_vector.extend_from_slice(b); /// spun_vector.extend_from_slice(a); -/// SpinVector { vec: spun_vector } +/// Self { vec: spun_vector } /// } /// } /// @@ -492,12 +569,20 @@ shl_impl_all! { u8 u16 u32 u64 u128 usize i8 i16 i32 i64 isize i128 } message = "no implementation for `{Self} >> {Rhs}`", label = "no implementation for `{Self} >> {Rhs}`" )] +#[const_trait] pub trait Shr { /// The resulting type after applying the `>>` operator. #[stable(feature = "rust1", since = "1.0.0")] type Output; /// Performs the `>>` operation. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(5u8 >> 1, 2); + /// assert_eq!(2u8 >> 1, 1); + /// ``` #[must_use] #[stable(feature = "rust1", since = "1.0.0")] fn shr(self, rhs: Rhs) -> Self::Output; @@ -506,7 +591,8 @@ pub trait Shr { macro_rules! shr_impl { ($t:ty, $f:ty) => { #[stable(feature = "rust1", since = "1.0.0")] - impl Shr<$f> for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const Shr<$f> for $t { type Output = $t; #[inline] @@ -516,7 +602,7 @@ macro_rules! shr_impl { } } - forward_ref_binop! { impl Shr, shr for $t, $f } + forward_ref_binop! { impl const Shr, shr for $t, $f } }; } @@ -556,7 +642,7 @@ shr_impl_all! { u8 u16 u32 u64 u128 usize i8 i16 i32 i64 i128 isize } /// impl BitAndAssign for Scalar { /// // rhs is the "right-hand side" of the expression `a &= b` /// fn bitand_assign(&mut self, rhs: Self) { -/// *self = Scalar(self.0 & rhs.0) +/// *self = Self(self.0 & rhs.0) /// } /// } /// @@ -590,11 +676,13 @@ shr_impl_all! { u8 u16 u32 u64 u128 usize i8 i16 i32 i64 i128 isize } /// // `rhs` is the "right-hand side" of the expression `a &= b`. /// fn bitand_assign(&mut self, rhs: Self) { /// assert_eq!(self.0.len(), rhs.0.len()); -/// *self = BooleanVector(self.0 -/// .iter() -/// .zip(rhs.0.iter()) -/// .map(|(x, y)| *x && *y) -/// .collect()); +/// *self = Self( +/// self.0 +/// .iter() +/// .zip(rhs.0.iter()) +/// .map(|(x, y)| *x & *y) +/// .collect() +/// ); /// } /// } /// @@ -610,8 +698,29 @@ shr_impl_all! { u8 u16 u32 u64 u128 usize i8 i16 i32 i64 i128 isize } message = "no implementation for `{Self} &= {Rhs}`", label = "no implementation for `{Self} &= {Rhs}`" )] +#[const_trait] pub trait BitAndAssign { /// Performs the `&=` operation. + /// + /// # Examples + /// + /// ``` + /// let mut x = true; + /// x &= false; + /// assert_eq!(x, false); + /// + /// let mut x = true; + /// x &= true; + /// assert_eq!(x, true); + /// + /// let mut x: u8 = 5; + /// x &= 1; + /// assert_eq!(x, 1); + /// + /// let mut x: u8 = 5; + /// x &= 2; + /// assert_eq!(x, 0); + /// ``` #[stable(feature = "op_assign_traits", since = "1.8.0")] fn bitand_assign(&mut self, rhs: Rhs); } @@ -619,12 +728,13 @@ pub trait BitAndAssign { macro_rules! bitand_assign_impl { ($($t:ty)+) => ($( #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl BitAndAssign for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const BitAndAssign for $t { #[inline] fn bitand_assign(&mut self, other: $t) { *self &= other } } - forward_ref_op_assign! { impl BitAndAssign, bitand_assign for $t, $t } + forward_ref_op_assign! { impl const BitAndAssign, bitand_assign for $t, $t } )+) } @@ -661,8 +771,29 @@ bitand_assign_impl! { bool usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } message = "no implementation for `{Self} |= {Rhs}`", label = "no implementation for `{Self} |= {Rhs}`" )] +#[const_trait] pub trait BitOrAssign { /// Performs the `|=` operation. + /// + /// # Examples + /// + /// ``` + /// let mut x = true; + /// x |= false; + /// assert_eq!(x, true); + /// + /// let mut x = false; + /// x |= false; + /// assert_eq!(x, false); + /// + /// let mut x: u8 = 5; + /// x |= 1; + /// assert_eq!(x, 5); + /// + /// let mut x: u8 = 5; + /// x |= 2; + /// assert_eq!(x, 7); + /// ``` #[stable(feature = "op_assign_traits", since = "1.8.0")] fn bitor_assign(&mut self, rhs: Rhs); } @@ -670,12 +801,13 @@ pub trait BitOrAssign { macro_rules! bitor_assign_impl { ($($t:ty)+) => ($( #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl BitOrAssign for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const BitOrAssign for $t { #[inline] fn bitor_assign(&mut self, other: $t) { *self |= other } } - forward_ref_op_assign! { impl BitOrAssign, bitor_assign for $t, $t } + forward_ref_op_assign! { impl const BitOrAssign, bitor_assign for $t, $t } )+) } @@ -712,8 +844,29 @@ bitor_assign_impl! { bool usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } message = "no implementation for `{Self} ^= {Rhs}`", label = "no implementation for `{Self} ^= {Rhs}`" )] +#[const_trait] pub trait BitXorAssign { /// Performs the `^=` operation. + /// + /// # Examples + /// + /// ``` + /// let mut x = true; + /// x ^= false; + /// assert_eq!(x, true); + /// + /// let mut x = true; + /// x ^= true; + /// assert_eq!(x, false); + /// + /// let mut x: u8 = 5; + /// x ^= 1; + /// assert_eq!(x, 4); + /// + /// let mut x: u8 = 5; + /// x ^= 2; + /// assert_eq!(x, 7); + /// ``` #[stable(feature = "op_assign_traits", since = "1.8.0")] fn bitxor_assign(&mut self, rhs: Rhs); } @@ -721,12 +874,13 @@ pub trait BitXorAssign { macro_rules! bitxor_assign_impl { ($($t:ty)+) => ($( #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl BitXorAssign for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const BitXorAssign for $t { #[inline] fn bitxor_assign(&mut self, other: $t) { *self ^= other } } - forward_ref_op_assign! { impl BitXorAssign, bitxor_assign for $t, $t } + forward_ref_op_assign! { impl const BitXorAssign, bitxor_assign for $t, $t } )+) } @@ -761,8 +915,21 @@ bitxor_assign_impl! { bool usize u8 u16 u32 u64 u128 isize i8 i16 i32 i64 i128 } message = "no implementation for `{Self} <<= {Rhs}`", label = "no implementation for `{Self} <<= {Rhs}`" )] +#[const_trait] pub trait ShlAssign { /// Performs the `<<=` operation. + /// + /// # Examples + /// + /// ``` + /// let mut x: u8 = 5; + /// x <<= 1; + /// assert_eq!(x, 10); + /// + /// let mut x: u8 = 1; + /// x <<= 1; + /// assert_eq!(x, 2); + /// ``` #[stable(feature = "op_assign_traits", since = "1.8.0")] fn shl_assign(&mut self, rhs: Rhs); } @@ -770,7 +937,8 @@ pub trait ShlAssign { macro_rules! shl_assign_impl { ($t:ty, $f:ty) => { #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl ShlAssign<$f> for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const ShlAssign<$f> for $t { #[inline] #[rustc_inherit_overflow_checks] fn shl_assign(&mut self, other: $f) { @@ -778,7 +946,7 @@ macro_rules! shl_assign_impl { } } - forward_ref_op_assign! { impl ShlAssign, shl_assign for $t, $f } + forward_ref_op_assign! { impl const ShlAssign, shl_assign for $t, $f } }; } @@ -831,8 +999,21 @@ shl_assign_impl_all! { u8 u16 u32 u64 u128 usize i8 i16 i32 i64 i128 isize } message = "no implementation for `{Self} >>= {Rhs}`", label = "no implementation for `{Self} >>= {Rhs}`" )] +#[const_trait] pub trait ShrAssign { /// Performs the `>>=` operation. + /// + /// # Examples + /// + /// ``` + /// let mut x: u8 = 5; + /// x >>= 1; + /// assert_eq!(x, 2); + /// + /// let mut x: u8 = 2; + /// x >>= 1; + /// assert_eq!(x, 1); + /// ``` #[stable(feature = "op_assign_traits", since = "1.8.0")] fn shr_assign(&mut self, rhs: Rhs); } @@ -840,7 +1021,8 @@ pub trait ShrAssign { macro_rules! shr_assign_impl { ($t:ty, $f:ty) => { #[stable(feature = "op_assign_traits", since = "1.8.0")] - impl ShrAssign<$f> for $t { + #[rustc_const_unstable(feature = "const_ops", issue = "90080")] + impl const ShrAssign<$f> for $t { #[inline] #[rustc_inherit_overflow_checks] fn shr_assign(&mut self, other: $f) { @@ -848,7 +1030,7 @@ macro_rules! shr_assign_impl { } } - forward_ref_op_assign! { impl ShrAssign, shr_assign for $t, $f } + forward_ref_op_assign! { impl const ShrAssign, shr_assign for $t, $f } }; } diff --git a/crux-mir/lib/core/src/ops/control_flow.rs b/crux-mir/lib/core/src/ops/control_flow.rs new file mode 100644 index 000000000..cd183540c --- /dev/null +++ b/crux-mir/lib/core/src/ops/control_flow.rs @@ -0,0 +1,304 @@ +use crate::{convert, ops}; + +/// Used to tell an operation whether it should exit early or go on as usual. +/// +/// This is used when exposing things (like graph traversals or visitors) where +/// you want the user to be able to choose whether to exit early. +/// Having the enum makes it clearer -- no more wondering "wait, what did `false` +/// mean again?" -- and allows including a value. +/// +/// Similar to [`Option`] and [`Result`], this enum can be used with the `?` operator +/// to return immediately if the [`Break`] variant is present or otherwise continue normally +/// with the value inside the [`Continue`] variant. +/// +/// # Examples +/// +/// Early-exiting from [`Iterator::try_for_each`]: +/// ``` +/// use std::ops::ControlFlow; +/// +/// let r = (2..100).try_for_each(|x| { +/// if 403 % x == 0 { +/// return ControlFlow::Break(x) +/// } +/// +/// ControlFlow::Continue(()) +/// }); +/// assert_eq!(r, ControlFlow::Break(13)); +/// ``` +/// +/// A basic tree traversal: +/// ``` +/// use std::ops::ControlFlow; +/// +/// pub struct TreeNode { +/// value: T, +/// left: Option>>, +/// right: Option>>, +/// } +/// +/// impl TreeNode { +/// pub fn traverse_inorder(&self, f: &mut impl FnMut(&T) -> ControlFlow) -> ControlFlow { +/// if let Some(left) = &self.left { +/// left.traverse_inorder(f)?; +/// } +/// f(&self.value)?; +/// if let Some(right) = &self.right { +/// right.traverse_inorder(f)?; +/// } +/// ControlFlow::Continue(()) +/// } +/// fn leaf(value: T) -> Option>> { +/// Some(Box::new(Self { value, left: None, right: None })) +/// } +/// } +/// +/// let node = TreeNode { +/// value: 0, +/// left: TreeNode::leaf(1), +/// right: Some(Box::new(TreeNode { +/// value: -1, +/// left: TreeNode::leaf(5), +/// right: TreeNode::leaf(2), +/// })) +/// }; +/// let mut sum = 0; +/// +/// let res = node.traverse_inorder(&mut |val| { +/// if *val < 0 { +/// ControlFlow::Break(*val) +/// } else { +/// sum += *val; +/// ControlFlow::Continue(()) +/// } +/// }); +/// assert_eq!(res, ControlFlow::Break(-1)); +/// assert_eq!(sum, 6); +/// ``` +/// +/// [`Break`]: ControlFlow::Break +/// [`Continue`]: ControlFlow::Continue +#[stable(feature = "control_flow_enum_type", since = "1.55.0")] +// ControlFlow should not implement PartialOrd or Ord, per RFC 3058: +// https://rust-lang.github.io/rfcs/3058-try-trait-v2.html#traits-for-controlflow +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ControlFlow { + /// Move on to the next phase of the operation as normal. + #[stable(feature = "control_flow_enum_type", since = "1.55.0")] + #[lang = "Continue"] + Continue(C), + /// Exit the operation without running subsequent phases. + #[stable(feature = "control_flow_enum_type", since = "1.55.0")] + #[lang = "Break"] + Break(B), + // Yes, the order of the variants doesn't match the type parameters. + // They're in this order so that `ControlFlow` <-> `Result` + // is a no-op conversion in the `Try` implementation. +} + +#[unstable(feature = "try_trait_v2", issue = "84277")] +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const ops::Try for ControlFlow { + type Output = C; + type Residual = ControlFlow; + + #[inline] + fn from_output(output: Self::Output) -> Self { + ControlFlow::Continue(output) + } + + #[inline] + fn branch(self) -> ControlFlow { + match self { + ControlFlow::Continue(c) => ControlFlow::Continue(c), + ControlFlow::Break(b) => ControlFlow::Break(ControlFlow::Break(b)), + } + } +} + +#[unstable(feature = "try_trait_v2", issue = "84277")] +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const ops::FromResidual for ControlFlow { + #[inline] + fn from_residual(residual: ControlFlow) -> Self { + match residual { + ControlFlow::Break(b) => ControlFlow::Break(b), + } + } +} + +#[unstable(feature = "try_trait_v2_residual", issue = "91285")] +#[rustc_const_unstable(feature = "const_try", issue = "74935")] +impl const ops::Residual for ControlFlow { + type TryType = ControlFlow; +} + +impl ControlFlow { + /// Returns `true` if this is a `Break` variant. + /// + /// # Examples + /// + /// ``` + /// use std::ops::ControlFlow; + /// + /// assert!(ControlFlow::::Break(3).is_break()); + /// assert!(!ControlFlow::::Continue(3).is_break()); + /// ``` + #[inline] + #[stable(feature = "control_flow_enum_is", since = "1.59.0")] + pub fn is_break(&self) -> bool { + matches!(*self, ControlFlow::Break(_)) + } + + /// Returns `true` if this is a `Continue` variant. + /// + /// # Examples + /// + /// ``` + /// use std::ops::ControlFlow; + /// + /// assert!(!ControlFlow::::Break(3).is_continue()); + /// assert!(ControlFlow::::Continue(3).is_continue()); + /// ``` + #[inline] + #[stable(feature = "control_flow_enum_is", since = "1.59.0")] + pub fn is_continue(&self) -> bool { + matches!(*self, ControlFlow::Continue(_)) + } + + /// Converts the `ControlFlow` into an `Option` which is `Some` if the + /// `ControlFlow` was `Break` and `None` otherwise. + /// + /// # Examples + /// + /// ``` + /// #![feature(control_flow_enum)] + /// use std::ops::ControlFlow; + /// + /// assert_eq!(ControlFlow::::Break(3).break_value(), Some(3)); + /// assert_eq!(ControlFlow::::Continue(3).break_value(), None); + /// ``` + #[inline] + #[unstable(feature = "control_flow_enum", reason = "new API", issue = "75744")] + pub fn break_value(self) -> Option { + match self { + ControlFlow::Continue(..) => None, + ControlFlow::Break(x) => Some(x), + } + } + + /// Maps `ControlFlow` to `ControlFlow` by applying a function + /// to the break value in case it exists. + #[inline] + #[unstable(feature = "control_flow_enum", reason = "new API", issue = "75744")] + pub fn map_break(self, f: F) -> ControlFlow + where + F: FnOnce(B) -> T, + { + match self { + ControlFlow::Continue(x) => ControlFlow::Continue(x), + ControlFlow::Break(x) => ControlFlow::Break(f(x)), + } + } + + /// Converts the `ControlFlow` into an `Option` which is `Some` if the + /// `ControlFlow` was `Continue` and `None` otherwise. + /// + /// # Examples + /// + /// ``` + /// #![feature(control_flow_enum)] + /// use std::ops::ControlFlow; + /// + /// assert_eq!(ControlFlow::::Break(3).continue_value(), None); + /// assert_eq!(ControlFlow::::Continue(3).continue_value(), Some(3)); + /// ``` + #[inline] + #[unstable(feature = "control_flow_enum", reason = "new API", issue = "75744")] + pub fn continue_value(self) -> Option { + match self { + ControlFlow::Continue(x) => Some(x), + ControlFlow::Break(..) => None, + } + } + + /// Maps `ControlFlow` to `ControlFlow` by applying a function + /// to the continue value in case it exists. + #[inline] + #[unstable(feature = "control_flow_enum", reason = "new API", issue = "75744")] + pub fn map_continue(self, f: F) -> ControlFlow + where + F: FnOnce(C) -> T, + { + match self { + ControlFlow::Continue(x) => ControlFlow::Continue(f(x)), + ControlFlow::Break(x) => ControlFlow::Break(x), + } + } +} + +/// These are used only as part of implementing the iterator adapters. +/// They have mediocre names and non-obvious semantics, so aren't +/// currently on a path to potential stabilization. +impl ControlFlow { + /// Create a `ControlFlow` from any type implementing `Try`. + #[inline] + pub(crate) fn from_try(r: R) -> Self { + match R::branch(r) { + ControlFlow::Continue(v) => ControlFlow::Continue(v), + ControlFlow::Break(v) => ControlFlow::Break(R::from_residual(v)), + } + } + + /// Convert a `ControlFlow` into any type implementing `Try`; + #[inline] + pub(crate) fn into_try(self) -> R { + match self { + ControlFlow::Continue(v) => R::from_output(v), + ControlFlow::Break(v) => v, + } + } +} + +impl ControlFlow { + /// It's frequently the case that there's no value needed with `Continue`, + /// so this provides a way to avoid typing `(())`, if you prefer it. + /// + /// # Examples + /// + /// ``` + /// #![feature(control_flow_enum)] + /// use std::ops::ControlFlow; + /// + /// let mut partial_sum = 0; + /// let last_used = (1..10).chain(20..25).try_for_each(|x| { + /// partial_sum += x; + /// if partial_sum > 100 { ControlFlow::Break(x) } + /// else { ControlFlow::CONTINUE } + /// }); + /// assert_eq!(last_used.break_value(), Some(22)); + /// ``` + #[unstable(feature = "control_flow_enum", reason = "new API", issue = "75744")] + pub const CONTINUE: Self = ControlFlow::Continue(()); +} + +impl ControlFlow<(), C> { + /// APIs like `try_for_each` don't need values with `Break`, + /// so this provides a way to avoid typing `(())`, if you prefer it. + /// + /// # Examples + /// + /// ``` + /// #![feature(control_flow_enum)] + /// use std::ops::ControlFlow; + /// + /// let mut partial_sum = 0; + /// (1..10).chain(20..25).try_for_each(|x| { + /// if partial_sum > 100 { ControlFlow::BREAK } + /// else { partial_sum += x; ControlFlow::CONTINUE } + /// }); + /// assert_eq!(partial_sum, 108); + /// ``` + #[unstable(feature = "control_flow_enum", reason = "new API", issue = "75744")] + pub const BREAK: Self = ControlFlow::Break(()); +} diff --git a/crux-mir/lib/core/src/ops/deref.rs b/crux-mir/lib/core/src/ops/deref.rs index 68244fdb3..c67867f44 100644 --- a/crux-mir/lib/core/src/ops/deref.rs +++ b/crux-mir/lib/core/src/ops/deref.rs @@ -18,8 +18,8 @@ /// /// If `T` implements `Deref`, and `x` is a value of type `T`, then: /// -/// * In immutable contexts, `*x` on non-pointer types is equivalent to -/// `*Deref::deref(&x)`. +/// * In immutable contexts, `*x` (where `T` is neither a reference nor a raw pointer) +/// is equivalent to `*Deref::deref(&x)`. /// * Values of type `&T` are coerced to values of type `&U` /// * `T` implicitly implements all the (immutable) methods of the type `U`. /// @@ -28,7 +28,6 @@ /// [method resolution] and [type coercions]. /// /// [book]: ../../book/ch15-02-deref.html -/// [`DerefMut`]: trait.DerefMut.html /// [more]: #more-on-deref-coercion /// [ref-deref-op]: ../../reference/expressions/operator-expr.html#the-dereference-operator /// [method resolution]: ../../reference/expressions/method-call-expr.html @@ -61,28 +60,39 @@ #[doc(alias = "*")] #[doc(alias = "&*")] #[stable(feature = "rust1", since = "1.0.0")] +#[rustc_diagnostic_item = "Deref"] +#[const_trait] pub trait Deref { /// The resulting type after dereferencing. #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_diagnostic_item = "deref_target"] + #[lang = "deref_target"] type Target: ?Sized; /// Dereferences the value. #[must_use] #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_diagnostic_item = "deref_method"] fn deref(&self) -> &Self::Target; } #[stable(feature = "rust1", since = "1.0.0")] -impl Deref for &T { +#[rustc_const_unstable(feature = "const_deref", issue = "88955")] +impl const Deref for &T { type Target = T; + #[rustc_diagnostic_item = "noop_method_deref"] fn deref(&self) -> &T { *self } } #[stable(feature = "rust1", since = "1.0.0")] -impl Deref for &mut T { +impl !DerefMut for &T {} + +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_const_unstable(feature = "const_deref", issue = "88955")] +impl const Deref for &mut T { type Target = T; fn deref(&self) -> &T { @@ -112,8 +122,8 @@ impl Deref for &mut T { /// If `T` implements `DerefMut`, and `x` is a value of type `T`, /// then: /// -/// * In mutable contexts, `*x` on non-pointer types is equivalent to -/// `*DerefMut::deref_mut(&mut x)`. +/// * In mutable contexts, `*x` (where `T` is neither a reference nor a raw pointer) +/// is equivalent to `*DerefMut::deref_mut(&mut x)`. /// * Values of type `&mut T` are coerced to values of type `&mut U` /// * `T` implicitly implements all the (mutable) methods of the type `U`. /// @@ -122,7 +132,6 @@ impl Deref for &mut T { /// [method resolution] and [type coercions]. /// /// [book]: ../../book/ch15-02-deref.html -/// [`Deref`]: trait.Deref.html /// [more]: #more-on-deref-coercion /// [ref-deref-op]: ../../reference/expressions/operator-expr.html#the-dereference-operator /// [method resolution]: ../../reference/expressions/method-call-expr.html @@ -156,11 +165,12 @@ impl Deref for &mut T { /// /// let mut x = DerefMutExample { value: 'a' }; /// *x = 'b'; -/// assert_eq!('b', *x); +/// assert_eq!('b', x.value); /// ``` #[lang = "deref_mut"] #[doc(alias = "*")] #[stable(feature = "rust1", since = "1.0.0")] +#[const_trait] pub trait DerefMut: Deref { /// Mutably dereferences the value. #[stable(feature = "rust1", since = "1.0.0")] diff --git a/crux-mir/lib/core/src/ops/drop.rs b/crux-mir/lib/core/src/ops/drop.rs index 5233b475c..a2c3d978c 100644 --- a/crux-mir/lib/core/src/ops/drop.rs +++ b/crux-mir/lib/core/src/ops/drop.rs @@ -1,93 +1,146 @@ -/// Used to run some code when a value goes out of scope. -/// This is sometimes called a 'destructor'. +/// Custom code within the destructor. /// -/// When a value goes out of scope, it will have its `drop` method called if -/// its type implements `Drop`. Then, any fields the value contains will also -/// be dropped recursively. +/// When a value is no longer needed, Rust will run a "destructor" on that value. +/// The most common way that a value is no longer needed is when it goes out of +/// scope. Destructors may still run in other circumstances, but we're going to +/// focus on scope for the examples here. To learn about some of those other cases, +/// please see [the reference] section on destructors. /// -/// Because of this recursive dropping, you do not need to implement this trait -/// unless your type needs its own destructor logic. +/// [the reference]: https://doc.rust-lang.org/reference/destructors.html /// -/// Refer to [the chapter on `Drop` in *The Rust Programming Language*][book] -/// for some more elaboration. +/// This destructor consists of two components: +/// - A call to `Drop::drop` for that value, if this special `Drop` trait is implemented for its type. +/// - The automatically generated "drop glue" which recursively calls the destructors +/// of all the fields of this value. /// -/// [book]: ../../book/ch15-03-drop.html +/// As Rust automatically calls the destructors of all contained fields, +/// you don't have to implement `Drop` in most cases. But there are some cases where +/// it is useful, for example for types which directly manage a resource. +/// That resource may be memory, it may be a file descriptor, it may be a network socket. +/// Once a value of that type is no longer going to be used, it should "clean up" its +/// resource by freeing the memory or closing the file or socket. This is +/// the job of a destructor, and therefore the job of `Drop::drop`. /// -/// # Examples +/// ## Examples /// -/// ## Implementing `Drop` +/// To see destructors in action, let's take a look at the following program: /// -/// The `drop` method is called when `_x` goes out of scope, and therefore -/// `main` prints `Dropping!`. -/// -/// ``` +/// ```rust /// struct HasDrop; /// /// impl Drop for HasDrop { /// fn drop(&mut self) { -/// println!("Dropping!"); +/// println!("Dropping HasDrop!"); +/// } +/// } +/// +/// struct HasTwoDrops { +/// one: HasDrop, +/// two: HasDrop, +/// } +/// +/// impl Drop for HasTwoDrops { +/// fn drop(&mut self) { +/// println!("Dropping HasTwoDrops!"); /// } /// } /// /// fn main() { -/// let _x = HasDrop; +/// let _x = HasTwoDrops { one: HasDrop, two: HasDrop }; +/// println!("Running!"); /// } /// ``` /// -/// ## Dropping is done recursively +/// Rust will first call `Drop::drop` for `_x` and then for both `_x.one` and `_x.two`, +/// meaning that running this will print /// -/// When `outer` goes out of scope, the `drop` method will be called first for -/// `Outer`, then for `Inner`. Therefore, `main` prints `Dropping Outer!` and -/// then `Dropping Inner!`. +/// ```text +/// Running! +/// Dropping HasTwoDrops! +/// Dropping HasDrop! +/// Dropping HasDrop! +/// ``` +/// +/// Even if we remove the implementation of `Drop` for `HasTwoDrop`, the destructors of its fields are still called. +/// This would result in /// +/// ```test +/// Running! +/// Dropping HasDrop! +/// Dropping HasDrop! /// ``` -/// struct Inner; -/// struct Outer(Inner); /// -/// impl Drop for Inner { +/// ## You cannot call `Drop::drop` yourself +/// +/// Because `Drop::drop` is used to clean up a value, it may be dangerous to use this value after +/// the method has been called. As `Drop::drop` does not take ownership of its input, +/// Rust prevents misuse by not allowing you to call `Drop::drop` directly. +/// +/// In other words, if you tried to explicitly call `Drop::drop` in the above example, you'd get a compiler error. +/// +/// If you'd like to explicitly call the destructor of a value, [`mem::drop`] can be used instead. +/// +/// [`mem::drop`]: drop +/// +/// ## Drop order +/// +/// Which of our two `HasDrop` drops first, though? For structs, it's the same +/// order that they're declared: first `one`, then `two`. If you'd like to try +/// this yourself, you can modify `HasDrop` above to contain some data, like an +/// integer, and then use it in the `println!` inside of `Drop`. This behavior is +/// guaranteed by the language. +/// +/// Unlike for structs, local variables are dropped in reverse order: +/// +/// ```rust +/// struct Foo; +/// +/// impl Drop for Foo { /// fn drop(&mut self) { -/// println!("Dropping Inner!"); +/// println!("Dropping Foo!") /// } /// } /// -/// impl Drop for Outer { +/// struct Bar; +/// +/// impl Drop for Bar { /// fn drop(&mut self) { -/// println!("Dropping Outer!"); +/// println!("Dropping Bar!") /// } /// } /// /// fn main() { -/// let _x = Outer(Inner); +/// let _foo = Foo; +/// let _bar = Bar; /// } /// ``` /// -/// ## Variables are dropped in reverse order of declaration -/// -/// `_first` is declared first and `_second` is declared second, so `main` will -/// print `Declared second!` and then `Declared first!`. +/// This will print /// +/// ```text +/// Dropping Bar! +/// Dropping Foo! /// ``` -/// struct PrintOnDrop(&'static str); /// -/// impl Drop for PrintOnDrop { -/// fn drop(&mut self) { -/// println!("{}", self.0); -/// } -/// } +/// Please see [the reference] for the full rules. /// -/// fn main() { -/// let _first = PrintOnDrop("Declared first!"); -/// let _second = PrintOnDrop("Declared second!"); -/// } -/// ``` +/// [the reference]: https://doc.rust-lang.org/reference/destructors.html +/// +/// ## `Copy` and `Drop` are exclusive +/// +/// You cannot implement both [`Copy`] and `Drop` on the same type. Types that +/// are `Copy` get implicitly duplicated by the compiler, making it very +/// hard to predict when, and how often destructors will be executed. As such, +/// these types cannot have destructors. #[lang = "drop"] #[stable(feature = "rust1", since = "1.0.0")] +#[const_trait] pub trait Drop { /// Executes the destructor for this type. /// /// This method is called implicitly when the value goes out of scope, /// and cannot be called explicitly (this is compiler error [E0040]). - /// However, the [`std::mem::drop`] function in the prelude can be + /// However, the [`mem::drop`] function in the prelude can be /// used to call the argument's `Drop` implementation. /// /// When this method has been called, `self` has not yet been deallocated. @@ -102,12 +155,12 @@ pub trait Drop { /// Note that even if this panics, the value is considered to be dropped; /// you must not cause `drop` to be called again. This is normally automatically /// handled by the compiler, but when using unsafe code, can sometimes occur - /// unintentionally, particularly when using [`std::ptr::drop_in_place`]. + /// unintentionally, particularly when using [`ptr::drop_in_place`]. /// - /// [E0040]: ../../error-index.html#E0040 - /// [`panic!`]: ../macro.panic.html - /// [`std::mem::drop`]: ../../std/mem/fn.drop.html - /// [`std::ptr::drop_in_place`]: ../../std/ptr/fn.drop_in_place.html + /// [E0040]: ../../error_codes/E0040.html + /// [`panic!`]: crate::panic! + /// [`mem::drop`]: drop + /// [`ptr::drop_in_place`]: crate::ptr::drop_in_place #[stable(feature = "rust1", since = "1.0.0")] fn drop(&mut self); } diff --git a/crux-mir/lib/core/src/ops/function.rs b/crux-mir/lib/core/src/ops/function.rs index 04c7789fa..b7e1aee9d 100644 --- a/crux-mir/lib/core/src/ops/function.rs +++ b/crux-mir/lib/core/src/ops/function.rs @@ -1,3 +1,5 @@ +use crate::marker::Tuple; + /// The version of the call operator that takes an immutable receiver. /// /// Instances of `Fn` can be called repeatedly without mutating state. @@ -28,9 +30,7 @@ /// this can refer to [the relevant section in the *Rustonomicon*][nomicon]. /// /// [book]: ../../book/ch13-01-closures.html -/// [`FnMut`]: trait.FnMut.html -/// [`FnOnce`]: trait.FnOnce.html -/// [function pointers]: ../../std/primitive.fn.html +/// [function pointers]: fn /// [nomicon]: ../../nomicon/hrtb.html /// /// # Examples @@ -59,14 +59,21 @@ #[rustc_on_unimplemented( on( Args = "()", - note = "wrap the `{Self}` in a closure with no arguments: `|| {{ /* code */ }}" + note = "wrap the `{Self}` in a closure with no arguments: `|| {{ /* code */ }}`" + ), + on( + _Self = "unsafe fn", + note = "unsafe function cannot be called generically without an unsafe block", + // SAFETY: tidy is not smart enough to tell that the below unsafe block is a string + label = "call the function in a closure: `|| unsafe {{ /* code */ }}`" ), message = "expected a `{Fn}<{Args}>` closure, found `{Self}`", label = "expected an `Fn<{Args}>` closure, found `{Self}`" )] #[fundamental] // so that regex can rely that `&str: !FnMut` #[must_use = "closures are lazy and do nothing unless called"] -pub trait Fn: FnMut { +#[const_trait] +pub trait Fn: FnMut { /// Performs the call operation. #[unstable(feature = "fn_traits", issue = "29625")] extern "rust-call" fn call(&self, args: Args) -> Self::Output; @@ -99,9 +106,7 @@ pub trait Fn: FnMut { /// this can refer to [the relevant section in the *Rustonomicon*][nomicon]. /// /// [book]: ../../book/ch13-01-closures.html -/// [`Fn`]: trait.Fn.html -/// [`FnOnce`]: trait.FnOnce.html -/// [function pointers]: ../../std/primitive.fn.html +/// [function pointers]: fn /// [nomicon]: ../../nomicon/hrtb.html /// /// # Examples @@ -141,14 +146,21 @@ pub trait Fn: FnMut { #[rustc_on_unimplemented( on( Args = "()", - note = "wrap the `{Self}` in a closure with no arguments: `|| {{ /* code */ }}" + note = "wrap the `{Self}` in a closure with no arguments: `|| {{ /* code */ }}`" + ), + on( + _Self = "unsafe fn", + note = "unsafe function cannot be called generically without an unsafe block", + // SAFETY: tidy is not smart enough to tell that the below unsafe block is a string + label = "call the function in a closure: `|| unsafe {{ /* code */ }}`" ), message = "expected a `{FnMut}<{Args}>` closure, found `{Self}`", label = "expected an `FnMut<{Args}>` closure, found `{Self}`" )] #[fundamental] // so that regex can rely that `&str: !FnMut` #[must_use = "closures are lazy and do nothing unless called"] -pub trait FnMut: FnOnce { +#[const_trait] +pub trait FnMut: FnOnce { /// Performs the call operation. #[unstable(feature = "fn_traits", issue = "29625")] extern "rust-call" fn call_mut(&mut self, args: Args) -> Self::Output; @@ -160,7 +172,7 @@ pub trait FnMut: FnOnce { /// times. Because of this, if the only thing known about a type is that it /// implements `FnOnce`, it can only be called once. /// -/// `FnOnce` is implemented automatically by closure that might consume captured +/// `FnOnce` is implemented automatically by closures that might consume captured /// variables, as well as all types that implement [`FnMut`], e.g., (safe) /// [function pointers] (since `FnOnce` is a supertrait of [`FnMut`]). /// @@ -180,9 +192,7 @@ pub trait FnMut: FnOnce { /// this can refer to [the relevant section in the *Rustonomicon*][nomicon]. /// /// [book]: ../../book/ch13-01-closures.html -/// [`Fn`]: trait.Fn.html -/// [`FnMut`]: trait.FnMut.html -/// [function pointers]: ../../std/primitive.fn.html +/// [function pointers]: fn /// [nomicon]: ../../nomicon/hrtb.html /// /// # Examples @@ -215,15 +225,23 @@ pub trait FnMut: FnOnce { #[rustc_on_unimplemented( on( Args = "()", - note = "wrap the `{Self}` in a closure with no arguments: `|| {{ /* code */ }}" + note = "wrap the `{Self}` in a closure with no arguments: `|| {{ /* code */ }}`" + ), + on( + _Self = "unsafe fn", + note = "unsafe function cannot be called generically without an unsafe block", + // SAFETY: tidy is not smart enough to tell that the below unsafe block is a string + label = "call the function in a closure: `|| unsafe {{ /* code */ }}`" ), message = "expected a `{FnOnce}<{Args}>` closure, found `{Self}`", label = "expected an `FnOnce<{Args}>` closure, found `{Self}`" )] #[fundamental] // so that regex can rely that `&str: !FnMut` #[must_use = "closures are lazy and do nothing unless called"] -pub trait FnOnce { +#[const_trait] +pub trait FnOnce { /// The returned type after the call operator is used. + #[lang = "fn_once_output"] #[stable(feature = "fn_once_output", since = "1.12.0")] type Output; @@ -233,10 +251,13 @@ pub trait FnOnce { } mod impls { + use crate::marker::Tuple; + #[stable(feature = "rust1", since = "1.0.0")] - impl Fn for &F + #[rustc_const_unstable(feature = "const_fn_trait_ref_impls", issue = "101803")] + impl const Fn for &F where - F: Fn, + F: ~const Fn, { extern "rust-call" fn call(&self, args: A) -> F::Output { (**self).call(args) @@ -244,9 +265,10 @@ mod impls { } #[stable(feature = "rust1", since = "1.0.0")] - impl FnMut for &F + #[rustc_const_unstable(feature = "const_fn_trait_ref_impls", issue = "101803")] + impl const FnMut for &F where - F: Fn, + F: ~const Fn, { extern "rust-call" fn call_mut(&mut self, args: A) -> F::Output { (**self).call(args) @@ -254,9 +276,10 @@ mod impls { } #[stable(feature = "rust1", since = "1.0.0")] - impl FnOnce for &F + #[rustc_const_unstable(feature = "const_fn_trait_ref_impls", issue = "101803")] + impl const FnOnce for &F where - F: Fn, + F: ~const Fn, { type Output = F::Output; @@ -266,9 +289,10 @@ mod impls { } #[stable(feature = "rust1", since = "1.0.0")] - impl FnMut for &mut F + #[rustc_const_unstable(feature = "const_fn_trait_ref_impls", issue = "101803")] + impl const FnMut for &mut F where - F: FnMut, + F: ~const FnMut, { extern "rust-call" fn call_mut(&mut self, args: A) -> F::Output { (*self).call_mut(args) @@ -276,9 +300,10 @@ mod impls { } #[stable(feature = "rust1", since = "1.0.0")] - impl FnOnce for &mut F + #[rustc_const_unstable(feature = "const_fn_trait_ref_impls", issue = "101803")] + impl const FnOnce for &mut F where - F: FnMut, + F: ~const FnMut, { type Output = F::Output; extern "rust-call" fn call_once(self, args: A) -> F::Output { diff --git a/crux-mir/lib/core/src/ops/generator.rs b/crux-mir/lib/core/src/ops/generator.rs index 4f23620b9..fee4beb1e 100644 --- a/crux-mir/lib/core/src/ops/generator.rs +++ b/crux-mir/lib/core/src/ops/generator.rs @@ -47,7 +47,7 @@ pub enum GeneratorState { /// fn main() { /// let mut generator = || { /// yield 1; -/// return "foo" +/// "foo" /// }; /// /// match Pin::new(&mut generator).resume(()) { @@ -61,9 +61,10 @@ pub enum GeneratorState { /// } /// ``` /// -/// More documentation of generators can be found in the unstable book. +/// More documentation of generators can be found in the [unstable book]. /// /// [RFC 2033]: https://github.com/rust-lang/rfcs/pull/2033 +/// [unstable book]: ../../unstable-book/language-features/generators.html #[lang = "generator"] #[unstable(feature = "generator_trait", issue = "43122")] #[fundamental] diff --git a/crux-mir/lib/core/src/ops/index.rs b/crux-mir/lib/core/src/ops/index.rs index aae069112..228efb0bc 100644 --- a/crux-mir/lib/core/src/ops/index.rs +++ b/crux-mir/lib/core/src/ops/index.rs @@ -5,9 +5,6 @@ /// [`IndexMut`] is used instead. This allows nice things such as /// `let value = v[index]` if the type of `value` implements [`Copy`]. /// -/// [`IndexMut`]: ../../std/ops/trait.IndexMut.html -/// [`Copy`]: ../../std/marker/trait.Copy.html -/// /// # Examples /// /// The following example implements `Index` on a read-only `NucleotideCount` @@ -58,13 +55,19 @@ #[doc(alias = "]")] #[doc(alias = "[")] #[doc(alias = "[]")] +#[const_trait] pub trait Index { /// The returned type after indexing. #[stable(feature = "rust1", since = "1.0.0")] type Output: ?Sized; /// Performs the indexing (`container[index]`) operation. + /// + /// # Panics + /// + /// May panic if the index is out of bounds. #[stable(feature = "rust1", since = "1.0.0")] + #[track_caller] fn index(&self, index: Idx) -> &Self::Output; } @@ -75,15 +78,13 @@ pub trait Index { /// an immutable value is requested, the [`Index`] trait is used instead. This /// allows nice things such as `v[index] = value`. /// -/// [`Index`]: ../../std/ops/trait.Index.html -/// /// # Examples /// /// A very simple implementation of a `Balance` struct that has two sides, where /// each can be indexed mutably and immutably. /// /// ``` -/// use std::ops::{Index,IndexMut}; +/// use std::ops::{Index, IndexMut}; /// /// #[derive(Debug)] /// enum Side { @@ -106,7 +107,7 @@ pub trait Index { /// type Output = Weight; /// /// fn index(&self, index: Side) -> &Self::Output { -/// println!("Accessing {:?}-side of balance immutably", index); +/// println!("Accessing {index:?}-side of balance immutably"); /// match index { /// Side::Left => &self.left, /// Side::Right => &self.right, @@ -116,7 +117,7 @@ pub trait Index { /// /// impl IndexMut for Balance { /// fn index_mut(&mut self, index: Side) -> &mut Self::Output { -/// println!("Accessing {:?}-side of balance mutably", index); +/// println!("Accessing {index:?}-side of balance mutably"); /// match index { /// Side::Left => &mut self.left, /// Side::Right => &mut self.right, @@ -163,8 +164,14 @@ see chapter in The Book : Index { +#[const_trait] +pub trait IndexMut: ~const Index { /// Performs the mutable indexing (`container[index]`) operation. + /// + /// # Panics + /// + /// May panic if the index is out of bounds. #[stable(feature = "rust1", since = "1.0.0")] + #[track_caller] fn index_mut(&mut self, index: Idx) -> &mut Self::Output; } diff --git a/crux-mir/lib/core/src/ops/index_range.rs b/crux-mir/lib/core/src/ops/index_range.rs new file mode 100644 index 000000000..3e06776d2 --- /dev/null +++ b/crux-mir/lib/core/src/ops/index_range.rs @@ -0,0 +1,171 @@ +use crate::intrinsics::{assert_unsafe_precondition, unchecked_add, unchecked_sub}; +use crate::iter::{FusedIterator, TrustedLen}; + +/// Like a `Range`, but with a safety invariant that `start <= end`. +/// +/// This means that `end - start` cannot overflow, allowing some μoptimizations. +/// +/// (Normal `Range` code needs to handle degenerate ranges like `10..0`, +/// which takes extra checks compared to only handling the canonical form.) +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct IndexRange { + start: usize, + end: usize, +} + +impl IndexRange { + /// # Safety + /// - `start <= end` + #[inline] + pub const unsafe fn new_unchecked(start: usize, end: usize) -> Self { + // SAFETY: comparisons on usize are pure + unsafe { + assert_unsafe_precondition!( + "IndexRange::new_unchecked requires `start <= end`", + (start: usize, end: usize) => start <= end + ) + }; + IndexRange { start, end } + } + + #[inline] + pub const fn zero_to(end: usize) -> Self { + IndexRange { start: 0, end } + } + + #[inline] + pub const fn start(&self) -> usize { + self.start + } + + #[inline] + pub const fn end(&self) -> usize { + self.end + } + + #[inline] + pub const fn len(&self) -> usize { + // SAFETY: By invariant, this cannot wrap + unsafe { unchecked_sub(self.end, self.start) } + } + + /// # Safety + /// - Can only be called when `start < end`, aka when `len > 0`. + #[inline] + unsafe fn next_unchecked(&mut self) -> usize { + debug_assert!(self.start < self.end); + + let value = self.start; + // SAFETY: The range isn't empty, so this cannot overflow + self.start = unsafe { unchecked_add(value, 1) }; + value + } + + /// # Safety + /// - Can only be called when `start < end`, aka when `len > 0`. + #[inline] + unsafe fn next_back_unchecked(&mut self) -> usize { + debug_assert!(self.start < self.end); + + // SAFETY: The range isn't empty, so this cannot overflow + let value = unsafe { unchecked_sub(self.end, 1) }; + self.end = value; + value + } + + /// Removes the first `n` items from this range, returning them as an `IndexRange`. + /// If there are fewer than `n`, then the whole range is returned and + /// `self` is left empty. + /// + /// This is designed to help implement `Iterator::advance_by`. + #[inline] + pub fn take_prefix(&mut self, n: usize) -> Self { + let mid = if n <= self.len() { + // SAFETY: We just checked that this will be between start and end, + // and thus the addition cannot overflow. + unsafe { unchecked_add(self.start, n) } + } else { + self.end + }; + let prefix = Self { start: self.start, end: mid }; + self.start = mid; + prefix + } + + /// Removes the last `n` items from this range, returning them as an `IndexRange`. + /// If there are fewer than `n`, then the whole range is returned and + /// `self` is left empty. + /// + /// This is designed to help implement `Iterator::advance_back_by`. + #[inline] + pub fn take_suffix(&mut self, n: usize) -> Self { + let mid = if n <= self.len() { + // SAFETY: We just checked that this will be between start and end, + // and thus the addition cannot overflow. + unsafe { unchecked_sub(self.end, n) } + } else { + self.start + }; + let suffix = Self { start: mid, end: self.end }; + self.end = mid; + suffix + } +} + +impl Iterator for IndexRange { + type Item = usize; + + #[inline] + fn next(&mut self) -> Option { + if self.len() > 0 { + // SAFETY: We just checked that the range is non-empty + unsafe { Some(self.next_unchecked()) } + } else { + None + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let len = self.len(); + (len, Some(len)) + } + + #[inline] + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + let original_len = self.len(); + self.take_prefix(n); + if n > original_len { Err(original_len) } else { Ok(()) } + } +} + +impl DoubleEndedIterator for IndexRange { + #[inline] + fn next_back(&mut self) -> Option { + if self.len() > 0 { + // SAFETY: We just checked that the range is non-empty + unsafe { Some(self.next_back_unchecked()) } + } else { + None + } + } + + #[inline] + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + let original_len = self.len(); + self.take_suffix(n); + if n > original_len { Err(original_len) } else { Ok(()) } + } +} + +impl ExactSizeIterator for IndexRange { + #[inline] + fn len(&self) -> usize { + self.len() + } +} + +// SAFETY: Because we only deal in `usize`, our `len` is always perfect. +unsafe impl TrustedLen for IndexRange {} + +impl FusedIterator for IndexRange {} diff --git a/crux-mir/lib/core/src/ops/mod.rs b/crux-mir/lib/core/src/ops/mod.rs index e3e5934b4..97d9b750d 100644 --- a/crux-mir/lib/core/src/ops/mod.rs +++ b/crux-mir/lib/core/src/ops/mod.rs @@ -17,10 +17,10 @@ //! should have some resemblance to multiplication (and share expected //! properties like associativity). //! -//! Note that the `&&` and `||` operators short-circuit, i.e., they only -//! evaluate their second operand if it contributes to the result. Since this -//! behavior is not enforceable by traits, `&&` and `||` are not supported as -//! overloadable operators. +//! Note that the `&&` and `||` operators are currently not supported for +//! overloading. Due to their short circuiting nature, they require a different +//! design from traits for other operators like [`BitAnd`]. Designs for them are +//! under discussion. //! //! Many of the operators take their operands by value. In non-generic //! contexts involving built-in types, this is usually not a problem. @@ -49,18 +49,18 @@ //! } //! //! impl Add for Point { -//! type Output = Point; +//! type Output = Self; //! -//! fn add(self, other: Point) -> Point { -//! Point {x: self.x + other.x, y: self.y + other.y} +//! fn add(self, other: Self) -> Self { +//! Self {x: self.x + other.x, y: self.y + other.y} //! } //! } //! //! impl Sub for Point { -//! type Output = Point; +//! type Output = Self; //! -//! fn sub(self, other: Point) -> Point { -//! Point {x: self.x - other.x, y: self.y - other.y} +//! fn sub(self, other: Self) -> Self { +//! Self {x: self.x - other.x, y: self.y - other.y} //! } //! } //! @@ -133,26 +133,22 @@ //! // `consume_and_return_x` can no longer be invoked at this point //! ``` //! -//! [`Fn`]: trait.Fn.html -//! [`FnMut`]: trait.FnMut.html -//! [`FnOnce`]: trait.FnOnce.html -//! [`Add`]: trait.Add.html -//! [`Sub`]: trait.Sub.html -//! [`Mul`]: trait.Mul.html -//! [`clone`]: ../clone/trait.Clone.html#tymethod.clone +//! [`clone`]: Clone::clone //! [operator precedence]: ../../reference/expressions.html#expression-precedence #![stable(feature = "rust1", since = "1.0.0")] mod arith; mod bit; +mod control_flow; mod deref; mod drop; mod function; mod generator; mod index; +mod index_range; mod range; -mod r#try; +mod try_trait; mod unsize; #[stable(feature = "rust1", since = "1.0.0")] @@ -183,17 +179,33 @@ pub use self::index::{Index, IndexMut}; #[stable(feature = "rust1", since = "1.0.0")] pub use self::range::{Range, RangeFrom, RangeFull, RangeTo}; +pub(crate) use self::index_range::IndexRange; + #[stable(feature = "inclusive_range", since = "1.26.0")] pub use self::range::{Bound, RangeBounds, RangeInclusive, RangeToInclusive}; -#[unstable(feature = "try_trait", issue = "42327")] -pub use self::r#try::Try; +#[unstable(feature = "one_sided_range", issue = "69780")] +pub use self::range::OneSidedRange; + +#[unstable(feature = "try_trait_v2", issue = "84277")] +pub use self::try_trait::{FromResidual, Try}; + +#[unstable(feature = "try_trait_v2_yeet", issue = "96374")] +pub use self::try_trait::Yeet; + +#[unstable(feature = "try_trait_v2_residual", issue = "91285")] +pub use self::try_trait::Residual; + +pub(crate) use self::try_trait::{ChangeOutputType, NeverShortCircuit}; #[unstable(feature = "generator_trait", issue = "43122")] pub use self::generator::{Generator, GeneratorState}; -#[unstable(feature = "coerce_unsized", issue = "27732")] +#[unstable(feature = "coerce_unsized", issue = "18598")] pub use self::unsize::CoerceUnsized; #[unstable(feature = "dispatch_from_dyn", issue = "none")] pub use self::unsize::DispatchFromDyn; + +#[unstable(feature = "control_flow_enum", reason = "new API", issue = "75744")] +pub use self::control_flow::ControlFlow; diff --git a/crux-mir/lib/core/src/ops/range.rs b/crux-mir/lib/core/src/ops/range.rs index adee8cea4..d29ae3561 100644 --- a/crux-mir/lib/core/src/ops/range.rs +++ b/crux-mir/lib/core/src/ops/range.rs @@ -19,7 +19,7 @@ use crate::hash::Hash; /// /// ```compile_fail,E0277 /// for i in .. { -/// // ... +/// // ... /// } /// ``` /// @@ -27,19 +27,18 @@ use crate::hash::Hash; /// /// ``` /// let arr = [0, 1, 2, 3, 4]; -/// assert_eq!(arr[ .. ], [0,1,2,3,4]); // RangeFull -/// assert_eq!(arr[ .. 3], [0,1,2 ]); -/// assert_eq!(arr[ ..=3], [0,1,2,3 ]); -/// assert_eq!(arr[1.. ], [ 1,2,3,4]); -/// assert_eq!(arr[1.. 3], [ 1,2 ]); -/// assert_eq!(arr[1..=3], [ 1,2,3 ]); +/// assert_eq!(arr[ .. ], [0, 1, 2, 3, 4]); // This is the `RangeFull` +/// assert_eq!(arr[ .. 3], [0, 1, 2 ]); +/// assert_eq!(arr[ ..=3], [0, 1, 2, 3 ]); +/// assert_eq!(arr[1.. ], [ 1, 2, 3, 4]); +/// assert_eq!(arr[1.. 3], [ 1, 2 ]); +/// assert_eq!(arr[1..=3], [ 1, 2, 3 ]); /// ``` /// -/// [`IntoIterator`]: ../iter/trait.Iterator.html -/// [`Iterator`]: ../iter/trait.IntoIterator.html -/// [slicing index]: ../slice/trait.SliceIndex.html +/// [slicing index]: crate::slice::SliceIndex +#[lang = "RangeFull"] #[doc(alias = "..")] -#[derive(Copy, Clone, PartialEq, Eq, Hash)] +#[derive(Copy, Clone, Default, PartialEq, Eq, Hash)] #[stable(feature = "rust1", since = "1.0.0")] pub struct RangeFull; @@ -53,25 +52,30 @@ impl fmt::Debug for RangeFull { /// A (half-open) range bounded inclusively below and exclusively above /// (`start..end`). /// -/// The `Range` `start..end` contains all values with `x >= start` and -/// `x < end`. It is empty unless `start < end`. +/// The range `start..end` contains all values with `start <= x < end`. +/// It is empty if `start >= end`. /// /// # Examples /// +/// The `start..end` syntax is a `Range`: +/// /// ``` /// assert_eq!((3..5), std::ops::Range { start: 3, end: 5 }); /// assert_eq!(3 + 4 + 5, (3..6).sum()); +/// ``` /// +/// ``` /// let arr = [0, 1, 2, 3, 4]; -/// assert_eq!(arr[ .. ], [0,1,2,3,4]); -/// assert_eq!(arr[ .. 3], [0,1,2 ]); -/// assert_eq!(arr[ ..=3], [0,1,2,3 ]); -/// assert_eq!(arr[1.. ], [ 1,2,3,4]); -/// assert_eq!(arr[1.. 3], [ 1,2 ]); // Range -/// assert_eq!(arr[1..=3], [ 1,2,3 ]); +/// assert_eq!(arr[ .. ], [0, 1, 2, 3, 4]); +/// assert_eq!(arr[ .. 3], [0, 1, 2 ]); +/// assert_eq!(arr[ ..=3], [0, 1, 2, 3 ]); +/// assert_eq!(arr[1.. ], [ 1, 2, 3, 4]); +/// assert_eq!(arr[1.. 3], [ 1, 2 ]); // This is a `Range` +/// assert_eq!(arr[1..=3], [ 1, 2, 3 ]); /// ``` +#[lang = "Range"] #[doc(alias = "..")] -#[derive(Clone, PartialEq, Eq, Hash)] // not Copy -- see #27186 +#[derive(Clone, Default, PartialEq, Eq, Hash)] // not Copy -- see #27186 #[stable(feature = "rust1", since = "1.0.0")] pub struct Range { /// The lower bound of the range (inclusive). @@ -98,8 +102,6 @@ impl> Range { /// # Examples /// /// ``` - /// use std::f32; - /// /// assert!(!(3..5).contains(&2)); /// assert!( (3..5).contains(&3)); /// assert!( (3..5).contains(&4)); @@ -127,8 +129,6 @@ impl> Range { /// # Examples /// /// ``` - /// #![feature(range_is_empty)] - /// /// assert!(!(3..5).is_empty()); /// assert!( (3..3).is_empty()); /// assert!( (3..2).is_empty()); @@ -137,14 +137,11 @@ impl> Range { /// The range is empty if either side is incomparable: /// /// ``` - /// #![feature(range_is_empty)] - /// - /// use std::f32::NAN; /// assert!(!(3.0..5.0).is_empty()); - /// assert!( (3.0..NAN).is_empty()); - /// assert!( (NAN..5.0).is_empty()); + /// assert!( (3.0..f32::NAN).is_empty()); + /// assert!( (f32::NAN..5.0).is_empty()); /// ``` - #[unstable(feature = "range_is_empty", reason = "recently added", issue = "48111")] + #[stable(feature = "range_is_empty", since = "1.47.0")] pub fn is_empty(&self) -> bool { !(self.start < self.end) } @@ -154,27 +151,36 @@ impl> Range { /// /// The `RangeFrom` `start..` contains all values with `x >= start`. /// -/// *Note*: Currently, no overflow checking is done for the [`Iterator`] -/// implementation; if you use an integer range and the integer overflows, it -/// might panic in debug mode or create an endless loop in release mode. **This -/// overflow behavior might change in the future.** +/// *Note*: Overflow in the [`Iterator`] implementation (when the contained +/// data type reaches its numerical limit) is allowed to panic, wrap, or +/// saturate. This behavior is defined by the implementation of the [`Step`] +/// trait. For primitive integers, this follows the normal rules, and respects +/// the overflow checks profile (panic in debug, wrap in release). Note also +/// that overflow happens earlier than you might assume: the overflow happens +/// in the call to `next` that yields the maximum value, as the range must be +/// set to a state to yield the next value. +/// +/// [`Step`]: crate::iter::Step /// /// # Examples /// +/// The `start..` syntax is a `RangeFrom`: +/// /// ``` /// assert_eq!((2..), std::ops::RangeFrom { start: 2 }); /// assert_eq!(2 + 3 + 4, (2..).take(3).sum()); +/// ``` /// +/// ``` /// let arr = [0, 1, 2, 3, 4]; -/// assert_eq!(arr[ .. ], [0,1,2,3,4]); -/// assert_eq!(arr[ .. 3], [0,1,2 ]); -/// assert_eq!(arr[ ..=3], [0,1,2,3 ]); -/// assert_eq!(arr[1.. ], [ 1,2,3,4]); // RangeFrom -/// assert_eq!(arr[1.. 3], [ 1,2 ]); -/// assert_eq!(arr[1..=3], [ 1,2,3 ]); +/// assert_eq!(arr[ .. ], [0, 1, 2, 3, 4]); +/// assert_eq!(arr[ .. 3], [0, 1, 2 ]); +/// assert_eq!(arr[ ..=3], [0, 1, 2, 3 ]); +/// assert_eq!(arr[1.. ], [ 1, 2, 3, 4]); // This is a `RangeFrom` +/// assert_eq!(arr[1.. 3], [ 1, 2 ]); +/// assert_eq!(arr[1..=3], [ 1, 2, 3 ]); /// ``` -/// -/// [`Iterator`]: ../iter/trait.IntoIterator.html +#[lang = "RangeFrom"] #[doc(alias = "..")] #[derive(Clone, PartialEq, Eq, Hash)] // not Copy -- see #27186 #[stable(feature = "rust1", since = "1.0.0")] @@ -199,8 +205,6 @@ impl> RangeFrom { /// # Examples /// /// ``` - /// use std::f32; - /// /// assert!(!(3..).contains(&2)); /// assert!( (3..).contains(&3)); /// assert!( (3..).contains(&1_000_000_000)); @@ -248,17 +252,16 @@ impl> RangeFrom { /// /// ``` /// let arr = [0, 1, 2, 3, 4]; -/// assert_eq!(arr[ .. ], [0,1,2,3,4]); -/// assert_eq!(arr[ .. 3], [0,1,2 ]); // RangeTo -/// assert_eq!(arr[ ..=3], [0,1,2,3 ]); -/// assert_eq!(arr[1.. ], [ 1,2,3,4]); -/// assert_eq!(arr[1.. 3], [ 1,2 ]); -/// assert_eq!(arr[1..=3], [ 1,2,3 ]); +/// assert_eq!(arr[ .. ], [0, 1, 2, 3, 4]); +/// assert_eq!(arr[ .. 3], [0, 1, 2 ]); // This is a `RangeTo` +/// assert_eq!(arr[ ..=3], [0, 1, 2, 3 ]); +/// assert_eq!(arr[1.. ], [ 1, 2, 3, 4]); +/// assert_eq!(arr[1.. 3], [ 1, 2 ]); +/// assert_eq!(arr[1..=3], [ 1, 2, 3 ]); /// ``` /// -/// [`IntoIterator`]: ../iter/trait.Iterator.html -/// [`Iterator`]: ../iter/trait.IntoIterator.html -/// [slicing index]: ../slice/trait.SliceIndex.html +/// [slicing index]: crate::slice::SliceIndex +#[lang = "RangeTo"] #[doc(alias = "..")] #[derive(Copy, Clone, PartialEq, Eq, Hash)] #[stable(feature = "rust1", since = "1.0.0")] @@ -283,8 +286,6 @@ impl> RangeTo { /// # Examples /// /// ``` - /// use std::f32; - /// /// assert!( (..5).contains(&-1_000_000_000)); /// assert!( (..5).contains(&4)); /// assert!(!(..5).contains(&5)); @@ -312,23 +313,28 @@ impl> RangeTo { /// iteration has finished are **unspecified** other than that [`.is_empty()`] /// will return `true` once no more values will be produced. /// -/// [fused]: ../iter/trait.FusedIterator.html -/// [`.is_empty()`]: #method.is_empty +/// [fused]: crate::iter::FusedIterator +/// [`.is_empty()`]: RangeInclusive::is_empty /// /// # Examples /// +/// The `start..=end` syntax is a `RangeInclusive`: +/// /// ``` /// assert_eq!((3..=5), std::ops::RangeInclusive::new(3, 5)); /// assert_eq!(3 + 4 + 5, (3..=5).sum()); +/// ``` /// +/// ``` /// let arr = [0, 1, 2, 3, 4]; -/// assert_eq!(arr[ .. ], [0,1,2,3,4]); -/// assert_eq!(arr[ .. 3], [0,1,2 ]); -/// assert_eq!(arr[ ..=3], [0,1,2,3 ]); -/// assert_eq!(arr[1.. ], [ 1,2,3,4]); -/// assert_eq!(arr[1.. 3], [ 1,2 ]); -/// assert_eq!(arr[1..=3], [ 1,2,3 ]); // RangeInclusive +/// assert_eq!(arr[ .. ], [0, 1, 2, 3, 4]); +/// assert_eq!(arr[ .. 3], [0, 1, 2 ]); +/// assert_eq!(arr[ ..=3], [0, 1, 2, 3 ]); +/// assert_eq!(arr[1.. ], [ 1, 2, 3, 4]); +/// assert_eq!(arr[1.. 3], [ 1, 2 ]); +/// assert_eq!(arr[1..=3], [ 1, 2, 3 ]); // This is a `RangeInclusive` /// ``` +#[lang = "RangeInclusive"] #[doc(alias = "..=")] #[derive(Clone, PartialEq, Eq, Hash)] // not Copy -- see #27186 #[stable(feature = "inclusive_range", since = "1.26.0")] @@ -360,6 +366,7 @@ impl RangeInclusive { /// /// assert_eq!(3..=5, RangeInclusive::new(3, 5)); /// ``` + #[lang = "range_inclusive_new"] #[stable(feature = "inclusive_range_methods", since = "1.27.0")] #[inline] #[rustc_promotable] @@ -378,8 +385,8 @@ impl RangeInclusive { /// Note: the value returned by this method is unspecified after the range /// has been iterated to exhaustion. /// - /// [`end()`]: #method.end - /// [`is_empty()`]: #method.is_empty + /// [`end()`]: RangeInclusive::end + /// [`is_empty()`]: RangeInclusive::is_empty /// /// # Examples /// @@ -403,8 +410,8 @@ impl RangeInclusive { /// Note: the value returned by this method is unspecified after the range /// has been iterated to exhaustion. /// - /// [`start()`]: #method.start - /// [`is_empty()`]: #method.is_empty + /// [`start()`]: RangeInclusive::start + /// [`is_empty()`]: RangeInclusive::is_empty /// /// # Examples /// @@ -435,6 +442,20 @@ impl RangeInclusive { } } +impl RangeInclusive { + /// Converts to an exclusive `Range` for `SliceIndex` implementations. + /// The caller is responsible for dealing with `end == usize::MAX`. + #[inline] + pub(crate) const fn into_slice_range(self) -> Range { + // If we're not exhausted, we want to simply slice `start..end + 1`. + // If we are exhausted, then slicing with `end + 1..end + 1` gives us an + // empty range that is still subject to bounds-checks for that endpoint. + let exclusive_end = self.end + 1; + let start = if self.exhausted { exclusive_end } else { self.start }; + start..exclusive_end + } +} + #[stable(feature = "inclusive_range", since = "1.26.0")] impl fmt::Debug for RangeInclusive { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -454,8 +475,6 @@ impl> RangeInclusive { /// # Examples /// /// ``` - /// use std::f32; - /// /// assert!(!(3..=5).contains(&2)); /// assert!( (3..=5).contains(&3)); /// assert!( (3..=5).contains(&4)); @@ -470,6 +489,16 @@ impl> RangeInclusive { /// assert!(!(0.0..=f32::NAN).contains(&0.0)); /// assert!(!(f32::NAN..=1.0).contains(&1.0)); /// ``` + /// + /// This method always returns `false` after iteration has finished: + /// + /// ``` + /// let mut r = 3..=5; + /// assert!(r.contains(&3) && r.contains(&5)); + /// for _ in r.by_ref() {} + /// // Precise field values are unspecified here + /// assert!(!r.contains(&3) && !r.contains(&5)); + /// ``` #[stable(feature = "range_contains", since = "1.35.0")] pub fn contains(&self, item: &U) -> bool where @@ -484,8 +513,6 @@ impl> RangeInclusive { /// # Examples /// /// ``` - /// #![feature(range_is_empty)] - /// /// assert!(!(3..=5).is_empty()); /// assert!(!(3..=3).is_empty()); /// assert!( (3..=2).is_empty()); @@ -494,25 +521,20 @@ impl> RangeInclusive { /// The range is empty if either side is incomparable: /// /// ``` - /// #![feature(range_is_empty)] - /// - /// use std::f32::NAN; /// assert!(!(3.0..=5.0).is_empty()); - /// assert!( (3.0..=NAN).is_empty()); - /// assert!( (NAN..=5.0).is_empty()); + /// assert!( (3.0..=f32::NAN).is_empty()); + /// assert!( (f32::NAN..=5.0).is_empty()); /// ``` /// /// This method returns `true` after iteration has finished: /// /// ``` - /// #![feature(range_is_empty)] - /// /// let mut r = 3..=5; /// for _ in r.by_ref() {} /// // Precise field values are unspecified here /// assert!(r.is_empty()); /// ``` - #[unstable(feature = "range_is_empty", reason = "recently added", issue = "48111")] + #[stable(feature = "range_is_empty", since = "1.47.0")] #[inline] pub fn is_empty(&self) -> bool { self.exhausted || !(self.start <= self.end) @@ -548,17 +570,16 @@ impl> RangeInclusive { /// /// ``` /// let arr = [0, 1, 2, 3, 4]; -/// assert_eq!(arr[ .. ], [0,1,2,3,4]); -/// assert_eq!(arr[ .. 3], [0,1,2 ]); -/// assert_eq!(arr[ ..=3], [0,1,2,3 ]); // RangeToInclusive -/// assert_eq!(arr[1.. ], [ 1,2,3,4]); -/// assert_eq!(arr[1.. 3], [ 1,2 ]); -/// assert_eq!(arr[1..=3], [ 1,2,3 ]); +/// assert_eq!(arr[ .. ], [0, 1, 2, 3, 4]); +/// assert_eq!(arr[ .. 3], [0, 1, 2 ]); +/// assert_eq!(arr[ ..=3], [0, 1, 2, 3 ]); // This is a `RangeToInclusive` +/// assert_eq!(arr[1.. ], [ 1, 2, 3, 4]); +/// assert_eq!(arr[1.. 3], [ 1, 2 ]); +/// assert_eq!(arr[1..=3], [ 1, 2, 3 ]); /// ``` /// -/// [`IntoIterator`]: ../iter/trait.Iterator.html -/// [`Iterator`]: ../iter/trait.IntoIterator.html -/// [slicing index]: ../slice/trait.SliceIndex.html +/// [slicing index]: crate::slice::SliceIndex +#[lang = "RangeToInclusive"] #[doc(alias = "..=")] #[derive(Copy, Clone, PartialEq, Eq, Hash)] #[stable(feature = "inclusive_range", since = "1.26.0")] @@ -583,8 +604,6 @@ impl> RangeToInclusive { /// # Examples /// /// ``` - /// use std::f32; - /// /// assert!( (..=5).contains(&-1_000_000_000)); /// assert!( (..=5).contains(&5)); /// assert!(!(..=5).contains(&6)); @@ -634,7 +653,7 @@ impl> RangeToInclusive { /// map.insert(8, "c"); /// /// for (key, value) in map.range((Excluded(3), Included(8))) { -/// println!("{}: {}", key, value); +/// println!("{key}: {value}"); /// } /// /// assert_eq!(Some((&3, &"a")), map.range((Unbounded, Included(5))).next()); @@ -655,20 +674,77 @@ pub enum Bound { Unbounded, } +impl Bound { + /// Converts from `&Bound` to `Bound<&T>`. + #[inline] + #[stable(feature = "bound_as_ref_shared", since = "1.65.0")] + pub fn as_ref(&self) -> Bound<&T> { + match *self { + Included(ref x) => Included(x), + Excluded(ref x) => Excluded(x), + Unbounded => Unbounded, + } + } + + /// Converts from `&mut Bound` to `Bound<&mut T>`. + #[inline] + #[unstable(feature = "bound_as_ref", issue = "80996")] + pub fn as_mut(&mut self) -> Bound<&mut T> { + match *self { + Included(ref mut x) => Included(x), + Excluded(ref mut x) => Excluded(x), + Unbounded => Unbounded, + } + } + + /// Maps a `Bound` to a `Bound` by applying a function to the contained value (including + /// both `Included` and `Excluded`), returning a `Bound` of the same kind. + /// + /// # Examples + /// + /// ``` + /// #![feature(bound_map)] + /// use std::ops::Bound::*; + /// + /// let bound_string = Included("Hello, World!"); + /// + /// assert_eq!(bound_string.map(|s| s.len()), Included(13)); + /// ``` + /// + /// ``` + /// #![feature(bound_map)] + /// use std::ops::Bound; + /// use Bound::*; + /// + /// let unbounded_string: Bound = Unbounded; + /// + /// assert_eq!(unbounded_string.map(|s| s.len()), Unbounded); + /// ``` + #[inline] + #[unstable(feature = "bound_map", issue = "86026")] + pub fn map U>(self, f: F) -> Bound { + match self { + Unbounded => Unbounded, + Included(x) => Included(f(x)), + Excluded(x) => Excluded(f(x)), + } + } +} + impl Bound<&T> { /// Map a `Bound<&T>` to a `Bound` by cloning the contents of the bound. /// /// # Examples /// /// ``` - /// #![feature(bound_cloned)] /// use std::ops::Bound::*; /// use std::ops::RangeBounds; /// /// assert_eq!((1..12).start_bound(), Included(&1)); /// assert_eq!((1..12).start_bound().cloned(), Included(1)); /// ``` - #[unstable(feature = "bound_cloned", issue = "61356")] + #[must_use = "`self` will be dropped if the result is not used"] + #[stable(feature = "bound_cloned", since = "1.55.0")] pub fn cloned(self) -> Bound { match self { Bound::Unbounded => Bound::Unbounded, @@ -678,9 +754,9 @@ impl Bound<&T> { } } -#[stable(feature = "collections_range", since = "1.28.0")] /// `RangeBounds` is implemented by Rust's built-in range types, produced /// by range syntax like `..`, `a..`, `..b`, `..=c`, `d..e`, or `f..=g`. +#[stable(feature = "collections_range", since = "1.28.0")] pub trait RangeBounds { /// Start index bound. /// @@ -723,8 +799,6 @@ pub trait RangeBounds { /// # Examples /// /// ``` - /// use std::f32; - /// /// assert!( (3..5).contains(&4)); /// assert!(!(3..5).contains(&2)); /// @@ -739,12 +813,12 @@ pub trait RangeBounds { U: ?Sized + PartialOrd, { (match self.start_bound() { - Included(ref start) => *start <= item, - Excluded(ref start) => *start < item, + Included(start) => start <= item, + Excluded(start) => start < item, Unbounded => true, }) && (match self.end_bound() { - Included(ref end) => item <= *end, - Excluded(ref end) => item < *end, + Included(end) => item <= end, + Excluded(end) => item < end, Unbounded => true, }) } @@ -798,7 +872,13 @@ impl RangeBounds for RangeInclusive { Included(&self.start) } fn end_bound(&self) -> Bound<&T> { - Included(&self.end) + if self.exhausted { + // When the iterator is exhausted, we usually have start == end, + // but we want the range to appear empty, containing nothing. + Excluded(&self.end) + } else { + Included(&self.end) + } } } @@ -891,3 +971,21 @@ impl RangeBounds for RangeToInclusive<&T> { Included(self.end) } } + +/// `OneSidedRange` is implemented for built-in range types that are unbounded +/// on one side. For example, `a..`, `..b` and `..=c` implement `OneSidedRange`, +/// but `..`, `d..e`, and `f..=g` do not. +/// +/// Types that implement `OneSidedRange` must return `Bound::Unbounded` +/// from one of `RangeBounds::start_bound` or `RangeBounds::end_bound`. +#[unstable(feature = "one_sided_range", issue = "69780")] +pub trait OneSidedRange: RangeBounds {} + +#[unstable(feature = "one_sided_range", issue = "69780")] +impl OneSidedRange for RangeTo where Self: RangeBounds {} + +#[unstable(feature = "one_sided_range", issue = "69780")] +impl OneSidedRange for RangeFrom where Self: RangeBounds {} + +#[unstable(feature = "one_sided_range", issue = "69780")] +impl OneSidedRange for RangeToInclusive where Self: RangeBounds {} diff --git a/crux-mir/lib/core/src/ops/try.rs b/crux-mir/lib/core/src/ops/try.rs deleted file mode 100644 index 996a01d41..000000000 --- a/crux-mir/lib/core/src/ops/try.rs +++ /dev/null @@ -1,57 +0,0 @@ -/// A trait for customizing the behavior of the `?` operator. -/// -/// A type implementing `Try` is one that has a canonical way to view it -/// in terms of a success/failure dichotomy. This trait allows both -/// extracting those success or failure values from an existing instance and -/// creating a new instance from a success or failure value. -#[unstable(feature = "try_trait", issue = "42327")] -#[rustc_on_unimplemented( - on( - all( - any(from_method = "from_error", from_method = "from_ok"), - from_desugaring = "QuestionMark" - ), - message = "the `?` operator can only be used in {ItemContext} \ - that returns `Result` or `Option` \ - (or another type that implements `{Try}`)", - label = "cannot use the `?` operator in {ItemContext} that returns `{Self}`", - enclosing_scope = "this function should return `Result` or `Option` to accept `?`" - ), - on( - all(from_method = "into_result", from_desugaring = "QuestionMark"), - message = "the `?` operator can only be applied to values \ - that implement `{Try}`", - label = "the `?` operator cannot be applied to type `{Self}`" - ) -)] -#[doc(alias = "?")] -pub trait Try { - /// The type of this value when viewed as successful. - #[unstable(feature = "try_trait", issue = "42327")] - type Ok; - /// The type of this value when viewed as failed. - #[unstable(feature = "try_trait", issue = "42327")] - type Error; - - /// Applies the "?" operator. A return of `Ok(t)` means that the - /// execution should continue normally, and the result of `?` is the - /// value `t`. A return of `Err(e)` means that execution should branch - /// to the innermost enclosing `catch`, or return from the function. - /// - /// If an `Err(e)` result is returned, the value `e` will be "wrapped" - /// in the return type of the enclosing scope (which must itself implement - /// `Try`). Specifically, the value `X::from_error(From::from(e))` - /// is returned, where `X` is the return type of the enclosing function. - #[unstable(feature = "try_trait", issue = "42327")] - fn into_result(self) -> Result; - - /// Wrap an error value to construct the composite result. For example, - /// `Result::Err(x)` and `Result::from_error(x)` are equivalent. - #[unstable(feature = "try_trait", issue = "42327")] - fn from_error(v: Self::Error) -> Self; - - /// Wrap an OK value to construct the composite result. For example, - /// `Result::Ok(x)` and `Result::from_ok(x)` are equivalent. - #[unstable(feature = "try_trait", issue = "42327")] - fn from_ok(v: Self::Ok) -> Self; -} diff --git a/crux-mir/lib/core/src/ops/try_trait.rs b/crux-mir/lib/core/src/ops/try_trait.rs new file mode 100644 index 000000000..84a690468 --- /dev/null +++ b/crux-mir/lib/core/src/ops/try_trait.rs @@ -0,0 +1,424 @@ +use crate::ops::ControlFlow; + +/// The `?` operator and `try {}` blocks. +/// +/// `try_*` methods typically involve a type implementing this trait. For +/// example, the closures passed to [`Iterator::try_fold`] and +/// [`Iterator::try_for_each`] must return such a type. +/// +/// `Try` types are typically those containing two or more categories of values, +/// some subset of which are so commonly handled via early returns that it's +/// worth providing a terse (but still visible) syntax to make that easy. +/// +/// This is most often seen for error handling with [`Result`] and [`Option`]. +/// The quintessential implementation of this trait is on [`ControlFlow`]. +/// +/// # Using `Try` in Generic Code +/// +/// `Iterator::try_fold` was stabilized to call back in Rust 1.27, but +/// this trait is much newer. To illustrate the various associated types and +/// methods, let's implement our own version. +/// +/// As a reminder, an infallible version of a fold looks something like this: +/// ``` +/// fn simple_fold( +/// iter: impl Iterator, +/// mut accum: A, +/// mut f: impl FnMut(A, T) -> A, +/// ) -> A { +/// for x in iter { +/// accum = f(accum, x); +/// } +/// accum +/// } +/// ``` +/// +/// So instead of `f` returning just an `A`, we'll need it to return some other +/// type that produces an `A` in the "don't short circuit" path. Conveniently, +/// that's also the type we need to return from the function. +/// +/// Let's add a new generic parameter `R` for that type, and bound it to the +/// output type that we want: +/// ``` +/// # #![feature(try_trait_v2)] +/// # use std::ops::Try; +/// fn simple_try_fold_1>( +/// iter: impl Iterator, +/// mut accum: A, +/// mut f: impl FnMut(A, T) -> R, +/// ) -> R { +/// todo!() +/// } +/// ``` +/// +/// If we get through the entire iterator, we need to wrap up the accumulator +/// into the return type using [`Try::from_output`]: +/// ``` +/// # #![feature(try_trait_v2)] +/// # use std::ops::{ControlFlow, Try}; +/// fn simple_try_fold_2>( +/// iter: impl Iterator, +/// mut accum: A, +/// mut f: impl FnMut(A, T) -> R, +/// ) -> R { +/// for x in iter { +/// let cf = f(accum, x).branch(); +/// match cf { +/// ControlFlow::Continue(a) => accum = a, +/// ControlFlow::Break(_) => todo!(), +/// } +/// } +/// R::from_output(accum) +/// } +/// ``` +/// +/// We'll also need [`FromResidual::from_residual`] to turn the residual back +/// into the original type. But because it's a supertrait of `Try`, we don't +/// need to mention it in the bounds. All types which implement `Try` can be +/// recreated from their corresponding residual, so we'll just call it: +/// ``` +/// # #![feature(try_trait_v2)] +/// # use std::ops::{ControlFlow, Try}; +/// pub fn simple_try_fold_3>( +/// iter: impl Iterator, +/// mut accum: A, +/// mut f: impl FnMut(A, T) -> R, +/// ) -> R { +/// for x in iter { +/// let cf = f(accum, x).branch(); +/// match cf { +/// ControlFlow::Continue(a) => accum = a, +/// ControlFlow::Break(r) => return R::from_residual(r), +/// } +/// } +/// R::from_output(accum) +/// } +/// ``` +/// +/// But this "call `branch`, then `match` on it, and `return` if it was a +/// `Break`" is exactly what happens inside the `?` operator. So rather than +/// do all this manually, we can just use `?` instead: +/// ``` +/// # #![feature(try_trait_v2)] +/// # use std::ops::Try; +/// fn simple_try_fold>( +/// iter: impl Iterator, +/// mut accum: A, +/// mut f: impl FnMut(A, T) -> R, +/// ) -> R { +/// for x in iter { +/// accum = f(accum, x)?; +/// } +/// R::from_output(accum) +/// } +/// ``` +#[unstable(feature = "try_trait_v2", issue = "84277")] +#[rustc_on_unimplemented( + on( + all(from_desugaring = "TryBlock"), + message = "a `try` block must return `Result` or `Option` \ + (or another type that implements `{Try}`)", + label = "could not wrap the final value of the block as `{Self}` doesn't implement `Try`", + ), + on( + all(from_desugaring = "QuestionMark"), + message = "the `?` operator can only be applied to values that implement `{Try}`", + label = "the `?` operator cannot be applied to type `{Self}`" + ) +)] +#[doc(alias = "?")] +#[lang = "Try"] +#[const_trait] +pub trait Try: ~const FromResidual { + /// The type of the value produced by `?` when *not* short-circuiting. + #[unstable(feature = "try_trait_v2", issue = "84277")] + type Output; + + /// The type of the value passed to [`FromResidual::from_residual`] + /// as part of `?` when short-circuiting. + /// + /// This represents the possible values of the `Self` type which are *not* + /// represented by the `Output` type. + /// + /// # Note to Implementors + /// + /// The choice of this type is critical to interconversion. + /// Unlike the `Output` type, which will often be a raw generic type, + /// this type is typically a newtype of some sort to "color" the type + /// so that it's distinguishable from the residuals of other types. + /// + /// This is why `Result::Residual` is not `E`, but `Result`. + /// That way it's distinct from `ControlFlow::Residual`, for example, + /// and thus `?` on `ControlFlow` cannot be used in a method returning `Result`. + /// + /// If you're making a generic type `Foo` that implements `Try`, + /// then typically you can use `Foo` as its `Residual` + /// type: that type will have a "hole" in the correct place, and will maintain the + /// "foo-ness" of the residual so other types need to opt-in to interconversion. + #[unstable(feature = "try_trait_v2", issue = "84277")] + type Residual; + + /// Constructs the type from its `Output` type. + /// + /// This should be implemented consistently with the `branch` method + /// such that applying the `?` operator will get back the original value: + /// `Try::from_output(x).branch() --> ControlFlow::Continue(x)`. + /// + /// # Examples + /// + /// ``` + /// #![feature(try_trait_v2)] + /// use std::ops::Try; + /// + /// assert_eq!( as Try>::from_output(3), Ok(3)); + /// assert_eq!( as Try>::from_output(4), Some(4)); + /// assert_eq!( + /// as Try>::from_output(5), + /// std::ops::ControlFlow::Continue(5), + /// ); + /// + /// # fn make_question_mark_work() -> Option<()> { + /// assert_eq!(Option::from_output(4)?, 4); + /// # None } + /// # make_question_mark_work(); + /// + /// // This is used, for example, on the accumulator in `try_fold`: + /// let r = std::iter::empty().try_fold(4, |_, ()| -> Option<_> { unreachable!() }); + /// assert_eq!(r, Some(4)); + /// ``` + #[lang = "from_output"] + #[unstable(feature = "try_trait_v2", issue = "84277")] + fn from_output(output: Self::Output) -> Self; + + /// Used in `?` to decide whether the operator should produce a value + /// (because this returned [`ControlFlow::Continue`]) + /// or propagate a value back to the caller + /// (because this returned [`ControlFlow::Break`]). + /// + /// # Examples + /// + /// ``` + /// #![feature(try_trait_v2)] + /// use std::ops::{ControlFlow, Try}; + /// + /// assert_eq!(Ok::<_, String>(3).branch(), ControlFlow::Continue(3)); + /// assert_eq!(Err::(3).branch(), ControlFlow::Break(Err(3))); + /// + /// assert_eq!(Some(3).branch(), ControlFlow::Continue(3)); + /// assert_eq!(None::.branch(), ControlFlow::Break(None)); + /// + /// assert_eq!(ControlFlow::::Continue(3).branch(), ControlFlow::Continue(3)); + /// assert_eq!( + /// ControlFlow::<_, String>::Break(3).branch(), + /// ControlFlow::Break(ControlFlow::Break(3)), + /// ); + /// ``` + #[lang = "branch"] + #[unstable(feature = "try_trait_v2", issue = "84277")] + fn branch(self) -> ControlFlow; +} + +/// Used to specify which residuals can be converted into which [`crate::ops::Try`] types. +/// +/// Every `Try` type needs to be recreatable from its own associated +/// `Residual` type, but can also have additional `FromResidual` implementations +/// to support interconversion with other `Try` types. +#[rustc_on_unimplemented( + on( + all( + from_desugaring = "QuestionMark", + _Self = "std::result::Result", + R = "std::option::Option" + ), + message = "the `?` operator can only be used on `Result`s, not `Option`s, \ + in {ItemContext} that returns `Result`", + label = "use `.ok_or(...)?` to provide an error compatible with `{Self}`", + parent_label = "this function returns a `Result`" + ), + on( + all( + from_desugaring = "QuestionMark", + _Self = "std::result::Result", + ), + // There's a special error message in the trait selection code for + // `From` in `?`, so this is not shown for result-in-result errors, + // and thus it can be phrased more strongly than `ControlFlow`'s. + message = "the `?` operator can only be used on `Result`s \ + in {ItemContext} that returns `Result`", + label = "this `?` produces `{R}`, which is incompatible with `{Self}`", + parent_label = "this function returns a `Result`" + ), + on( + all( + from_desugaring = "QuestionMark", + _Self = "std::option::Option", + R = "std::result::Result", + ), + message = "the `?` operator can only be used on `Option`s, not `Result`s, \ + in {ItemContext} that returns `Option`", + label = "use `.ok()?` if you want to discard the `{R}` error information", + parent_label = "this function returns an `Option`" + ), + on( + all( + from_desugaring = "QuestionMark", + _Self = "std::option::Option", + ), + // `Option`-in-`Option` always works, as there's only one possible + // residual, so this can also be phrased strongly. + message = "the `?` operator can only be used on `Option`s \ + in {ItemContext} that returns `Option`", + label = "this `?` produces `{R}`, which is incompatible with `{Self}`", + parent_label = "this function returns an `Option`" + ), + on( + all( + from_desugaring = "QuestionMark", + _Self = "std::ops::ControlFlow", + R = "std::ops::ControlFlow", + ), + message = "the `?` operator in {ItemContext} that returns `ControlFlow` \ + can only be used on other `ControlFlow`s (with the same Break type)", + label = "this `?` produces `{R}`, which is incompatible with `{Self}`", + parent_label = "this function returns a `ControlFlow`", + note = "unlike `Result`, there's no `From`-conversion performed for `ControlFlow`" + ), + on( + all( + from_desugaring = "QuestionMark", + _Self = "std::ops::ControlFlow", + // `R` is not a `ControlFlow`, as that case was matched previously + ), + message = "the `?` operator can only be used on `ControlFlow`s \ + in {ItemContext} that returns `ControlFlow`", + label = "this `?` produces `{R}`, which is incompatible with `{Self}`", + parent_label = "this function returns a `ControlFlow`", + ), + on( + all(from_desugaring = "QuestionMark"), + message = "the `?` operator can only be used in {ItemContext} \ + that returns `Result` or `Option` \ + (or another type that implements `{FromResidual}`)", + label = "cannot use the `?` operator in {ItemContext} that returns `{Self}`", + parent_label = "this function should return `Result` or `Option` to accept `?`" + ), +)] +#[rustc_diagnostic_item = "FromResidual"] +#[unstable(feature = "try_trait_v2", issue = "84277")] +#[const_trait] +pub trait FromResidual::Residual> { + /// Constructs the type from a compatible `Residual` type. + /// + /// This should be implemented consistently with the `branch` method such + /// that applying the `?` operator will get back an equivalent residual: + /// `FromResidual::from_residual(r).branch() --> ControlFlow::Break(r)`. + /// (It must not be an *identical* residual when interconversion is involved.) + /// + /// # Examples + /// + /// ``` + /// #![feature(try_trait_v2)] + /// use std::ops::{ControlFlow, FromResidual}; + /// + /// assert_eq!(Result::::from_residual(Err(3_u8)), Err(3)); + /// assert_eq!(Option::::from_residual(None), None); + /// assert_eq!( + /// ControlFlow::<_, String>::from_residual(ControlFlow::Break(5)), + /// ControlFlow::Break(5), + /// ); + /// ``` + #[lang = "from_residual"] + #[unstable(feature = "try_trait_v2", issue = "84277")] + fn from_residual(residual: R) -> Self; +} + +#[unstable( + feature = "yeet_desugar_details", + issue = "none", + reason = "just here to simplify the desugaring; will never be stabilized" +)] +#[inline] +#[track_caller] // because `Result::from_residual` has it +#[lang = "from_yeet"] +pub fn from_yeet(yeeted: Y) -> T +where + T: FromResidual>, +{ + FromResidual::from_residual(Yeet(yeeted)) +} + +/// Allows retrieving the canonical type implementing [`Try`] that has this type +/// as its residual and allows it to hold an `O` as its output. +/// +/// If you think of the `Try` trait as splitting a type into its [`Try::Output`] +/// and [`Try::Residual`] components, this allows putting them back together. +/// +/// For example, +/// `Result: Try>`, +/// and in the other direction, +/// ` as Residual>::TryType = Result`. +#[unstable(feature = "try_trait_v2_residual", issue = "91285")] +#[const_trait] +pub trait Residual { + /// The "return" type of this meta-function. + #[unstable(feature = "try_trait_v2_residual", issue = "91285")] + type TryType: ~const Try; +} + +#[unstable(feature = "pub_crate_should_not_need_unstable_attr", issue = "none")] +pub(crate) type ChangeOutputType = <::Residual as Residual>::TryType; + +/// An adapter for implementing non-try methods via the `Try` implementation. +/// +/// Conceptually the same as `Result`, but requiring less work in trait +/// solving and inhabited-ness checking and such, by being an obvious newtype +/// and not having `From` bounds lying around. +/// +/// Not currently planned to be exposed publicly, so just `pub(crate)`. +#[repr(transparent)] +pub(crate) struct NeverShortCircuit(pub T); + +impl NeverShortCircuit { + /// Implementation for building `ConstFnMutClosure` for wrapping the output of a ~const FnMut in a `NeverShortCircuit`. + #[inline] + pub const fn wrap_mut_2_imp T>( + f: &mut F, + (a, b): (A, B), + ) -> NeverShortCircuit { + NeverShortCircuit(f(a, b)) + } +} + +pub(crate) enum NeverShortCircuitResidual {} + +impl const Try for NeverShortCircuit { + type Output = T; + type Residual = NeverShortCircuitResidual; + + #[inline] + fn branch(self) -> ControlFlow { + ControlFlow::Continue(self.0) + } + + #[inline] + fn from_output(x: T) -> Self { + NeverShortCircuit(x) + } +} + +impl const FromResidual for NeverShortCircuit { + #[inline] + fn from_residual(never: NeverShortCircuitResidual) -> Self { + match never {} + } +} + +impl const Residual for NeverShortCircuitResidual { + type TryType = NeverShortCircuit; +} + +/// Implement `FromResidual>` on your type to enable +/// `do yeet expr` syntax in functions returning your type. +#[unstable(feature = "try_trait_v2_yeet", issue = "96374")] +#[derive(Debug)] +pub struct Yeet(pub T); diff --git a/crux-mir/lib/core/src/ops/unsize.rs b/crux-mir/lib/core/src/ops/unsize.rs index 95a439359..b51f12580 100644 --- a/crux-mir/lib/core/src/ops/unsize.rs +++ b/crux-mir/lib/core/src/ops/unsize.rs @@ -29,46 +29,77 @@ use crate::marker::Unsize; /// pointers. It is implemented automatically by the compiler. /// /// [dst-coerce]: https://github.com/rust-lang/rfcs/blob/master/text/0982-dst-coercion.md -/// [unsize]: ../marker/trait.Unsize.html +/// [unsize]: crate::marker::Unsize /// [nomicon-coerce]: ../../nomicon/coercions.html -#[unstable(feature = "coerce_unsized", issue = "27732")] +#[unstable(feature = "coerce_unsized", issue = "18598")] #[lang = "coerce_unsized"] pub trait CoerceUnsized { // Empty. } // &mut T -> &mut U -#[unstable(feature = "coerce_unsized", issue = "27732")] +#[unstable(feature = "coerce_unsized", issue = "18598")] impl<'a, T: ?Sized + Unsize, U: ?Sized> CoerceUnsized<&'a mut U> for &'a mut T {} // &mut T -> &U -#[unstable(feature = "coerce_unsized", issue = "27732")] +#[unstable(feature = "coerce_unsized", issue = "18598")] impl<'a, 'b: 'a, T: ?Sized + Unsize, U: ?Sized> CoerceUnsized<&'a U> for &'b mut T {} // &mut T -> *mut U -#[unstable(feature = "coerce_unsized", issue = "27732")] +#[unstable(feature = "coerce_unsized", issue = "18598")] impl<'a, T: ?Sized + Unsize, U: ?Sized> CoerceUnsized<*mut U> for &'a mut T {} // &mut T -> *const U -#[unstable(feature = "coerce_unsized", issue = "27732")] +#[unstable(feature = "coerce_unsized", issue = "18598")] impl<'a, T: ?Sized + Unsize, U: ?Sized> CoerceUnsized<*const U> for &'a mut T {} // &T -> &U -#[unstable(feature = "coerce_unsized", issue = "27732")] +#[unstable(feature = "coerce_unsized", issue = "18598")] impl<'a, 'b: 'a, T: ?Sized + Unsize, U: ?Sized> CoerceUnsized<&'a U> for &'b T {} // &T -> *const U -#[unstable(feature = "coerce_unsized", issue = "27732")] +#[unstable(feature = "coerce_unsized", issue = "18598")] impl<'a, T: ?Sized + Unsize, U: ?Sized> CoerceUnsized<*const U> for &'a T {} // *mut T -> *mut U -#[unstable(feature = "coerce_unsized", issue = "27732")] +#[unstable(feature = "coerce_unsized", issue = "18598")] impl, U: ?Sized> CoerceUnsized<*mut U> for *mut T {} // *mut T -> *const U -#[unstable(feature = "coerce_unsized", issue = "27732")] +#[unstable(feature = "coerce_unsized", issue = "18598")] impl, U: ?Sized> CoerceUnsized<*const U> for *mut T {} // *const T -> *const U -#[unstable(feature = "coerce_unsized", issue = "27732")] +#[unstable(feature = "coerce_unsized", issue = "18598")] impl, U: ?Sized> CoerceUnsized<*const U> for *const T {} -/// This is used for object safety, to check that a method's receiver type can be dispatched on. +/// `DispatchFromDyn` is used in the implementation of object safety checks (specifically allowing +/// arbitrary self types), to guarantee that a method's receiver type can be dispatched on. +/// +/// Note: `DispatchFromDyn` was briefly named `CoerceSized` (and had a slightly different +/// interpretation). +/// +/// Imagine we have a trait object `t` with type `&dyn Tr`, where `Tr` is some trait with a method +/// `m` defined as `fn m(&self);`. When calling `t.m()`, the receiver `t` is a wide pointer, but an +/// implementation of `m` will expect a narrow pointer as `&self` (a reference to the concrete +/// type). The compiler must generate an implicit conversion from the trait object/wide pointer to +/// the concrete reference/narrow pointer. Implementing `DispatchFromDyn` indicates that that +/// conversion is allowed and thus that the type implementing `DispatchFromDyn` is safe to use as +/// the self type in an object-safe method. (in the above example, the compiler will require +/// `DispatchFromDyn` is implemented for `&'a U`). +/// +/// `DispatchFromDyn` does not specify the conversion from wide pointer to narrow pointer; the +/// conversion is hard-wired into the compiler. For the conversion to work, the following +/// properties must hold (i.e., it is only safe to implement `DispatchFromDyn` for types which have +/// these properties, these are also checked by the compiler): +/// +/// * EITHER `Self` and `T` are either both references or both raw pointers; in either case, with +/// the same mutability. +/// * OR, all of the following hold +/// - `Self` and `T` must have the same type constructor, and only vary in a single type parameter +/// formal (the *coerced type*, e.g., `impl DispatchFromDyn> for Rc` is ok and the +/// single type parameter (instantiated with `T` or `U`) is the coerced type, +/// `impl DispatchFromDyn> for Rc` is not ok). +/// - The definition for `Self` must be a struct. +/// - The definition for `Self` must not be `#[repr(packed)]` or `#[repr(C)]`. +/// - Other than one-aligned, zero-sized fields, the definition for `Self` must have exactly one +/// field and that field's type must be the coerced type. Furthermore, `Self`'s field type must +/// implement `DispatchFromDyn` where `F` is the type of `T`'s field type. /// /// An example implementation of the trait: /// diff --git a/crux-mir/lib/core/src/option.rs b/crux-mir/lib/core/src/option.rs index 3aab8b1b3..7cc00e3f8 100644 --- a/crux-mir/lib/core/src/option.rs +++ b/crux-mir/lib/core/src/option.rs @@ -34,7 +34,7 @@ //! // Pattern match to retrieve the value //! match result { //! // The division was valid -//! Some(x) => println!("Result: {}", x), +//! Some(x) => println!("Result: {x}"), //! // The division was invalid //! None => println!("Cannot divide by 0"), //! } @@ -47,11 +47,13 @@ //! //! Rust's pointer types must always point to a valid location; there are //! no "null" references. Instead, Rust has *optional* pointers, like -//! the optional owned box, [`Option`]`<`[`Box`]`>`. +//! the optional owned box, [Option]<[Box\]>. +//! +//! [Box\]: ../../std/boxed/struct.Box.html //! //! The following example uses [`Option`] to create an optional box of -//! [`i32`]. Notice that in order to use the inner [`i32`] value first, the -//! `check_optional` function needs to use pattern matching to +//! [`i32`]. Notice that in order to use the inner [`i32`] value, the +//! `check_optional` function first needs to use pattern matching to //! determine whether the box has a value (i.e., it is [`Some(...)`][`Some`]) or //! not ([`None`]). //! @@ -64,16 +66,431 @@ //! //! fn check_optional(optional: Option>) { //! match optional { -//! Some(p) => println!("has value {}", p), +//! Some(p) => println!("has value {p}"), //! None => println!("has no value"), //! } //! } //! ``` //! -//! This usage of [`Option`] to create safe nullable pointers is so -//! common that Rust does special optimizations to make the -//! representation of [`Option`]`<`[`Box`]`>` a single pointer. Optional pointers -//! in Rust are stored as efficiently as any other pointer type. +//! # The question mark operator, `?` +//! +//! Similar to the [`Result`] type, when writing code that calls many functions that return the +//! [`Option`] type, handling `Some`/`None` can be tedious. The question mark +//! operator, [`?`], hides some of the boilerplate of propagating values +//! up the call stack. +//! +//! It replaces this: +//! +//! ``` +//! # #![allow(dead_code)] +//! fn add_last_numbers(stack: &mut Vec) -> Option { +//! let a = stack.pop(); +//! let b = stack.pop(); +//! +//! match (a, b) { +//! (Some(x), Some(y)) => Some(x + y), +//! _ => None, +//! } +//! } +//! +//! ``` +//! +//! With this: +//! +//! ``` +//! # #![allow(dead_code)] +//! fn add_last_numbers(stack: &mut Vec) -> Option { +//! Some(stack.pop()? + stack.pop()?) +//! } +//! ``` +//! +//! *It's much nicer!* +//! +//! Ending the expression with [`?`] will result in the [`Some`]'s unwrapped value, unless the +//! result is [`None`], in which case [`None`] is returned early from the enclosing function. +//! +//! [`?`] can be used in functions that return [`Option`] because of the +//! early return of [`None`] that it provides. +//! +//! [`?`]: crate::ops::Try +//! [`Some`]: Some +//! [`None`]: None +//! +//! # Representation +//! +//! Rust guarantees to optimize the following types `T` such that +//! [`Option`] has the same size as `T`: +//! +//! * [`Box`] +//! * `&U` +//! * `&mut U` +//! * `fn`, `extern "C" fn`[^extern_fn] +//! * [`num::NonZero*`] +//! * [`ptr::NonNull`] +//! * `#[repr(transparent)]` struct around one of the types in this list. +//! +//! [^extern_fn]: this remains true for any other ABI: `extern "abi" fn` (_e.g._, `extern "system" fn`) +//! +//! [`Box`]: ../../std/boxed/struct.Box.html +//! [`num::NonZero*`]: crate::num +//! [`ptr::NonNull`]: crate::ptr::NonNull +//! +//! This is called the "null pointer optimization" or NPO. +//! +//! It is further guaranteed that, for the cases above, one can +//! [`mem::transmute`] from all valid values of `T` to `Option` and +//! from `Some::(_)` to `T` (but transmuting `None::` to `T` +//! is undefined behaviour). +//! +//! # Method overview +//! +//! In addition to working with pattern matching, [`Option`] provides a wide +//! variety of different methods. +//! +//! ## Querying the variant +//! +//! The [`is_some`] and [`is_none`] methods return [`true`] if the [`Option`] +//! is [`Some`] or [`None`], respectively. +//! +//! [`is_none`]: Option::is_none +//! [`is_some`]: Option::is_some +//! +//! ## Adapters for working with references +//! +//! * [`as_ref`] converts from [&][][Option]\ to [Option]<[&]T> +//! * [`as_mut`] converts from [&mut] [Option]\ to [Option]<[&mut] T> +//! * [`as_deref`] converts from [&][][Option]\ to +//! [Option]<[&]T::[Target]> +//! * [`as_deref_mut`] converts from [&mut] [Option]\ to +//! [Option]<[&mut] T::[Target]> +//! * [`as_pin_ref`] converts from [Pin]<[&][][Option]\> to +//! [Option]<[Pin]<[&]T>> +//! * [`as_pin_mut`] converts from [Pin]<[&mut] [Option]\> to +//! [Option]<[Pin]<[&mut] T>> +//! +//! [&]: reference "shared reference" +//! [&mut]: reference "mutable reference" +//! [Target]: Deref::Target "ops::Deref::Target" +//! [`as_deref`]: Option::as_deref +//! [`as_deref_mut`]: Option::as_deref_mut +//! [`as_mut`]: Option::as_mut +//! [`as_pin_mut`]: Option::as_pin_mut +//! [`as_pin_ref`]: Option::as_pin_ref +//! [`as_ref`]: Option::as_ref +//! +//! ## Extracting the contained value +//! +//! These methods extract the contained value in an [`Option`] when it +//! is the [`Some`] variant. If the [`Option`] is [`None`]: +//! +//! * [`expect`] panics with a provided custom message +//! * [`unwrap`] panics with a generic message +//! * [`unwrap_or`] returns the provided default value +//! * [`unwrap_or_default`] returns the default value of the type `T` +//! (which must implement the [`Default`] trait) +//! * [`unwrap_or_else`] returns the result of evaluating the provided +//! function +//! +//! [`expect`]: Option::expect +//! [`unwrap`]: Option::unwrap +//! [`unwrap_or`]: Option::unwrap_or +//! [`unwrap_or_default`]: Option::unwrap_or_default +//! [`unwrap_or_else`]: Option::unwrap_or_else +//! +//! ## Transforming contained values +//! +//! These methods transform [`Option`] to [`Result`]: +//! +//! * [`ok_or`] transforms [`Some(v)`] to [`Ok(v)`], and [`None`] to +//! [`Err(err)`] using the provided default `err` value +//! * [`ok_or_else`] transforms [`Some(v)`] to [`Ok(v)`], and [`None`] to +//! a value of [`Err`] using the provided function +//! * [`transpose`] transposes an [`Option`] of a [`Result`] into a +//! [`Result`] of an [`Option`] +//! +//! [`Err(err)`]: Err +//! [`Ok(v)`]: Ok +//! [`Some(v)`]: Some +//! [`ok_or`]: Option::ok_or +//! [`ok_or_else`]: Option::ok_or_else +//! [`transpose`]: Option::transpose +//! +//! These methods transform the [`Some`] variant: +//! +//! * [`filter`] calls the provided predicate function on the contained +//! value `t` if the [`Option`] is [`Some(t)`], and returns [`Some(t)`] +//! if the function returns `true`; otherwise, returns [`None`] +//! * [`flatten`] removes one level of nesting from an +//! [`Option>`] +//! * [`map`] transforms [`Option`] to [`Option`] by applying the +//! provided function to the contained value of [`Some`] and leaving +//! [`None`] values unchanged +//! +//! [`Some(t)`]: Some +//! [`filter`]: Option::filter +//! [`flatten`]: Option::flatten +//! [`map`]: Option::map +//! +//! These methods transform [`Option`] to a value of a possibly +//! different type `U`: +//! +//! * [`map_or`] applies the provided function to the contained value of +//! [`Some`], or returns the provided default value if the [`Option`] is +//! [`None`] +//! * [`map_or_else`] applies the provided function to the contained value +//! of [`Some`], or returns the result of evaluating the provided +//! fallback function if the [`Option`] is [`None`] +//! +//! [`map_or`]: Option::map_or +//! [`map_or_else`]: Option::map_or_else +//! +//! These methods combine the [`Some`] variants of two [`Option`] values: +//! +//! * [`zip`] returns [`Some((s, o))`] if `self` is [`Some(s)`] and the +//! provided [`Option`] value is [`Some(o)`]; otherwise, returns [`None`] +//! * [`zip_with`] calls the provided function `f` and returns +//! [`Some(f(s, o))`] if `self` is [`Some(s)`] and the provided +//! [`Option`] value is [`Some(o)`]; otherwise, returns [`None`] +//! +//! [`Some(f(s, o))`]: Some +//! [`Some(o)`]: Some +//! [`Some(s)`]: Some +//! [`Some((s, o))`]: Some +//! [`zip`]: Option::zip +//! [`zip_with`]: Option::zip_with +//! +//! ## Boolean operators +//! +//! These methods treat the [`Option`] as a boolean value, where [`Some`] +//! acts like [`true`] and [`None`] acts like [`false`]. There are two +//! categories of these methods: ones that take an [`Option`] as input, and +//! ones that take a function as input (to be lazily evaluated). +//! +//! The [`and`], [`or`], and [`xor`] methods take another [`Option`] as +//! input, and produce an [`Option`] as output. Only the [`and`] method can +//! produce an [`Option`] value having a different inner type `U` than +//! [`Option`]. +//! +//! | method | self | input | output | +//! |---------|-----------|-----------|-----------| +//! | [`and`] | `None` | (ignored) | `None` | +//! | [`and`] | `Some(x)` | `None` | `None` | +//! | [`and`] | `Some(x)` | `Some(y)` | `Some(y)` | +//! | [`or`] | `None` | `None` | `None` | +//! | [`or`] | `None` | `Some(y)` | `Some(y)` | +//! | [`or`] | `Some(x)` | (ignored) | `Some(x)` | +//! | [`xor`] | `None` | `None` | `None` | +//! | [`xor`] | `None` | `Some(y)` | `Some(y)` | +//! | [`xor`] | `Some(x)` | `None` | `Some(x)` | +//! | [`xor`] | `Some(x)` | `Some(y)` | `None` | +//! +//! [`and`]: Option::and +//! [`or`]: Option::or +//! [`xor`]: Option::xor +//! +//! The [`and_then`] and [`or_else`] methods take a function as input, and +//! only evaluate the function when they need to produce a new value. Only +//! the [`and_then`] method can produce an [`Option`] value having a +//! different inner type `U` than [`Option`]. +//! +//! | method | self | function input | function result | output | +//! |--------------|-----------|----------------|-----------------|-----------| +//! | [`and_then`] | `None` | (not provided) | (not evaluated) | `None` | +//! | [`and_then`] | `Some(x)` | `x` | `None` | `None` | +//! | [`and_then`] | `Some(x)` | `x` | `Some(y)` | `Some(y)` | +//! | [`or_else`] | `None` | (not provided) | `None` | `None` | +//! | [`or_else`] | `None` | (not provided) | `Some(y)` | `Some(y)` | +//! | [`or_else`] | `Some(x)` | (not provided) | (not evaluated) | `Some(x)` | +//! +//! [`and_then`]: Option::and_then +//! [`or_else`]: Option::or_else +//! +//! This is an example of using methods like [`and_then`] and [`or`] in a +//! pipeline of method calls. Early stages of the pipeline pass failure +//! values ([`None`]) through unchanged, and continue processing on +//! success values ([`Some`]). Toward the end, [`or`] substitutes an error +//! message if it receives [`None`]. +//! +//! ``` +//! # use std::collections::BTreeMap; +//! let mut bt = BTreeMap::new(); +//! bt.insert(20u8, "foo"); +//! bt.insert(42u8, "bar"); +//! let res = [0u8, 1, 11, 200, 22] +//! .into_iter() +//! .map(|x| { +//! // `checked_sub()` returns `None` on error +//! x.checked_sub(1) +//! // same with `checked_mul()` +//! .and_then(|x| x.checked_mul(2)) +//! // `BTreeMap::get` returns `None` on error +//! .and_then(|x| bt.get(&x)) +//! // Substitute an error message if we have `None` so far +//! .or(Some(&"error!")) +//! .copied() +//! // Won't panic because we unconditionally used `Some` above +//! .unwrap() +//! }) +//! .collect::>(); +//! assert_eq!(res, ["error!", "error!", "foo", "error!", "bar"]); +//! ``` +//! +//! ## Comparison operators +//! +//! If `T` implements [`PartialOrd`] then [`Option`] will derive its +//! [`PartialOrd`] implementation. With this order, [`None`] compares as +//! less than any [`Some`], and two [`Some`] compare the same way as their +//! contained values would in `T`. If `T` also implements +//! [`Ord`], then so does [`Option`]. +//! +//! ``` +//! assert!(None < Some(0)); +//! assert!(Some(0) < Some(1)); +//! ``` +//! +//! ## Iterating over `Option` +//! +//! An [`Option`] can be iterated over. This can be helpful if you need an +//! iterator that is conditionally empty. The iterator will either produce +//! a single value (when the [`Option`] is [`Some`]), or produce no values +//! (when the [`Option`] is [`None`]). For example, [`into_iter`] acts like +//! [`once(v)`] if the [`Option`] is [`Some(v)`], and like [`empty()`] if +//! the [`Option`] is [`None`]. +//! +//! [`Some(v)`]: Some +//! [`empty()`]: crate::iter::empty +//! [`once(v)`]: crate::iter::once +//! +//! Iterators over [`Option`] come in three types: +//! +//! * [`into_iter`] consumes the [`Option`] and produces the contained +//! value +//! * [`iter`] produces an immutable reference of type `&T` to the +//! contained value +//! * [`iter_mut`] produces a mutable reference of type `&mut T` to the +//! contained value +//! +//! [`into_iter`]: Option::into_iter +//! [`iter`]: Option::iter +//! [`iter_mut`]: Option::iter_mut +//! +//! An iterator over [`Option`] can be useful when chaining iterators, for +//! example, to conditionally insert items. (It's not always necessary to +//! explicitly call an iterator constructor: many [`Iterator`] methods that +//! accept other iterators will also accept iterable types that implement +//! [`IntoIterator`], which includes [`Option`].) +//! +//! ``` +//! let yep = Some(42); +//! let nope = None; +//! // chain() already calls into_iter(), so we don't have to do so +//! let nums: Vec = (0..4).chain(yep).chain(4..8).collect(); +//! assert_eq!(nums, [0, 1, 2, 3, 42, 4, 5, 6, 7]); +//! let nums: Vec = (0..4).chain(nope).chain(4..8).collect(); +//! assert_eq!(nums, [0, 1, 2, 3, 4, 5, 6, 7]); +//! ``` +//! +//! One reason to chain iterators in this way is that a function returning +//! `impl Iterator` must have all possible return values be of the same +//! concrete type. Chaining an iterated [`Option`] can help with that. +//! +//! ``` +//! fn make_iter(do_insert: bool) -> impl Iterator { +//! // Explicit returns to illustrate return types matching +//! match do_insert { +//! true => return (0..4).chain(Some(42)).chain(4..8), +//! false => return (0..4).chain(None).chain(4..8), +//! } +//! } +//! println!("{:?}", make_iter(true).collect::>()); +//! println!("{:?}", make_iter(false).collect::>()); +//! ``` +//! +//! If we try to do the same thing, but using [`once()`] and [`empty()`], +//! we can't return `impl Iterator` anymore because the concrete types of +//! the return values differ. +//! +//! [`empty()`]: crate::iter::empty +//! [`once()`]: crate::iter::once +//! +//! ```compile_fail,E0308 +//! # use std::iter::{empty, once}; +//! // This won't compile because all possible returns from the function +//! // must have the same concrete type. +//! fn make_iter(do_insert: bool) -> impl Iterator { +//! // Explicit returns to illustrate return types not matching +//! match do_insert { +//! true => return (0..4).chain(once(42)).chain(4..8), +//! false => return (0..4).chain(empty()).chain(4..8), +//! } +//! } +//! ``` +//! +//! ## Collecting into `Option` +//! +//! [`Option`] implements the [`FromIterator`][impl-FromIterator] trait, +//! which allows an iterator over [`Option`] values to be collected into an +//! [`Option`] of a collection of each contained value of the original +//! [`Option`] values, or [`None`] if any of the elements was [`None`]. +//! +//! [impl-FromIterator]: Option#impl-FromIterator%3COption%3CA%3E%3E-for-Option%3CV%3E +//! +//! ``` +//! let v = [Some(2), Some(4), None, Some(8)]; +//! let res: Option> = v.into_iter().collect(); +//! assert_eq!(res, None); +//! let v = [Some(2), Some(4), Some(8)]; +//! let res: Option> = v.into_iter().collect(); +//! assert_eq!(res, Some(vec![2, 4, 8])); +//! ``` +//! +//! [`Option`] also implements the [`Product`][impl-Product] and +//! [`Sum`][impl-Sum] traits, allowing an iterator over [`Option`] values +//! to provide the [`product`][Iterator::product] and +//! [`sum`][Iterator::sum] methods. +//! +//! [impl-Product]: Option#impl-Product%3COption%3CU%3E%3E-for-Option%3CT%3E +//! [impl-Sum]: Option#impl-Sum%3COption%3CU%3E%3E-for-Option%3CT%3E +//! +//! ``` +//! let v = [None, Some(1), Some(2), Some(3)]; +//! let res: Option = v.into_iter().sum(); +//! assert_eq!(res, None); +//! let v = [Some(1), Some(2), Some(21)]; +//! let res: Option = v.into_iter().product(); +//! assert_eq!(res, Some(42)); +//! ``` +//! +//! ## Modifying an [`Option`] in-place +//! +//! These methods return a mutable reference to the contained value of an +//! [`Option`]: +//! +//! * [`insert`] inserts a value, dropping any old contents +//! * [`get_or_insert`] gets the current value, inserting a provided +//! default value if it is [`None`] +//! * [`get_or_insert_default`] gets the current value, inserting the +//! default value of type `T` (which must implement [`Default`]) if it is +//! [`None`] +//! * [`get_or_insert_with`] gets the current value, inserting a default +//! computed by the provided function if it is [`None`] +//! +//! [`get_or_insert`]: Option::get_or_insert +//! [`get_or_insert_default`]: Option::get_or_insert_default +//! [`get_or_insert_with`]: Option::get_or_insert_with +//! [`insert`]: Option::insert +//! +//! These methods transfer ownership of the contained value of an +//! [`Option`]: +//! +//! * [`take`] takes ownership of the contained value of an [`Option`], if +//! any, replacing the [`Option`] with [`None`] +//! * [`replace`] takes ownership of the contained value of an [`Option`], +//! if any, replacing the [`Option`] with a [`Some`] containing the +//! provided value +//! +//! [`replace`]: Option::replace +//! [`take`]: Option::take //! //! # Examples //! @@ -122,42 +539,33 @@ //! } //! //! match name_of_biggest_animal { -//! Some(name) => println!("the biggest animal is {}", name), +//! Some(name) => println!("the biggest animal is {name}"), //! None => println!("there are no animals :("), //! } //! ``` -//! -//! [`Option`]: enum.Option.html -//! [`Some`]: enum.Option.html#variant.Some -//! [`None`]: enum.Option.html#variant.None -//! [`Box`]: ../../std/boxed/struct.Box.html -//! [`i32`]: ../../std/primitive.i32.html - -// ignore-tidy-undocumented-unsafe #![stable(feature = "rust1", since = "1.0.0")] -use crate::iter::{FromIterator, FusedIterator, TrustedLen}; +use crate::iter::{self, FromIterator, FusedIterator, TrustedLen}; +use crate::marker::Destruct; +use crate::panicking::{panic, panic_str}; use crate::pin::Pin; use crate::{ - convert, fmt, hint, mem, - ops::{self, Deref, DerefMut}, + convert, hint, mem, + ops::{self, ControlFlow, Deref, DerefMut}, }; -// Note that this is not a lang item per se, but it has a hidden dependency on -// `Iterator`, which is one. The compiler assumes that the `next` method of -// `Iterator` is an enumeration with one type parameter and two variants, -// which basically means it must be `Option`. - -/// The `Option` type. See [the module level documentation](index.html) for more. -#[derive(Copy, PartialEq, PartialOrd, Eq, Ord, Debug, Hash)] -#[rustc_diagnostic_item = "option_type"] +/// The `Option` type. See [the module level documentation](self) for more. +#[derive(Copy, PartialOrd, Eq, Ord, Debug, Hash)] +#[rustc_diagnostic_item = "Option"] #[stable(feature = "rust1", since = "1.0.0")] pub enum Option { - /// No value + /// No value. + #[lang = "None"] #[stable(feature = "rust1", since = "1.0.0")] None, - /// Some value `T` + /// Some value of type `T`. + #[lang = "Some"] #[stable(feature = "rust1", since = "1.0.0")] Some(#[stable(feature = "rust1", since = "1.0.0")] T), } @@ -182,63 +590,58 @@ impl Option { /// let x: Option = None; /// assert_eq!(x.is_some(), false); /// ``` - /// - /// [`Some`]: #variant.Some #[must_use = "if you intended to assert that this has a value, consider `.unwrap()` instead"] #[inline] #[stable(feature = "rust1", since = "1.0.0")] - pub fn is_some(&self) -> bool { + #[rustc_const_stable(feature = "const_option_basics", since = "1.48.0")] + pub const fn is_some(&self) -> bool { matches!(*self, Some(_)) } - /// Returns `true` if the option is a [`None`] value. + /// Returns `true` if the option is a [`Some`] and the value inside of it matches a predicate. /// /// # Examples /// /// ``` + /// #![feature(is_some_and)] + /// /// let x: Option = Some(2); - /// assert_eq!(x.is_none(), false); + /// assert_eq!(x.is_some_and(|x| x > 1), true); + /// + /// let x: Option = Some(0); + /// assert_eq!(x.is_some_and(|x| x > 1), false); /// /// let x: Option = None; - /// assert_eq!(x.is_none(), true); + /// assert_eq!(x.is_some_and(|x| x > 1), false); /// ``` - /// - /// [`None`]: #variant.None - #[must_use = "if you intended to assert that this doesn't have a value, consider \ - `.and_then(|| panic!(\"`Option` had a value when expected `None`\"))` instead"] + #[must_use] #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn is_none(&self) -> bool { - !self.is_some() + #[unstable(feature = "is_some_and", issue = "93050")] + pub fn is_some_and(self, f: impl FnOnce(T) -> bool) -> bool { + match self { + None => false, + Some(x) => f(x), + } } - /// Returns `true` if the option is a [`Some`] value containing the given value. + /// Returns `true` if the option is a [`None`] value. /// /// # Examples /// /// ``` - /// #![feature(option_result_contains)] - /// /// let x: Option = Some(2); - /// assert_eq!(x.contains(&2), true); - /// - /// let x: Option = Some(3); - /// assert_eq!(x.contains(&2), false); + /// assert_eq!(x.is_none(), false); /// /// let x: Option = None; - /// assert_eq!(x.contains(&2), false); + /// assert_eq!(x.is_none(), true); /// ``` - #[must_use] + #[must_use = "if you intended to assert that this doesn't have a value, consider \ + `.and_then(|_| panic!(\"`Option` had a value when expected `None`\"))` instead"] #[inline] - #[unstable(feature = "option_result_contains", issue = "62358")] - pub fn contains(&self, x: &U) -> bool - where - U: PartialEq, - { - match self { - Some(y) => x == y, - None => false, - } + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_option_basics", since = "1.48.0")] + pub const fn is_none(&self) -> bool { + !self.is_some() } ///////////////////////////////////////////////////////////////////////// @@ -249,25 +652,26 @@ impl Option { /// /// # Examples /// - /// Converts an `Option<`[`String`]`>` into an `Option<`[`usize`]`>`, preserving the original. - /// The [`map`] method takes the `self` argument by value, consuming the original, - /// so this technique uses `as_ref` to first take an `Option` to a reference - /// to the value inside the original. + /// Calculates the length of an Option<[String]> as an Option<[usize]> + /// without moving the [`String`]. The [`map`] method takes the `self` argument by value, + /// consuming the original, so this technique uses `as_ref` to first take an `Option` to a + /// reference to the value inside the original. /// - /// [`map`]: enum.Option.html#method.map - /// [`String`]: ../../std/string/struct.String.html - /// [`usize`]: ../../std/primitive.usize.html + /// [`map`]: Option::map + /// [String]: ../../std/string/struct.String.html "String" + /// [`String`]: ../../std/string/struct.String.html "String" /// /// ``` /// let text: Option = Some("Hello, world!".to_string()); /// // First, cast `Option` to `Option<&String>` with `as_ref`, /// // then consume *that* with `map`, leaving `text` on the stack. /// let text_length: Option = text.as_ref().map(|s| s.len()); - /// println!("still can print text: {:?}", text); + /// println!("still can print text: {text:?}"); /// ``` #[inline] + #[rustc_const_stable(feature = "const_option_basics", since = "1.48.0")] #[stable(feature = "rust1", since = "1.0.0")] - pub fn as_ref(&self) -> Option<&T> { + pub const fn as_ref(&self) -> Option<&T> { match *self { Some(ref x) => Some(x), None => None, @@ -288,29 +692,46 @@ impl Option { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - pub fn as_mut(&mut self) -> Option<&mut T> { + #[rustc_const_unstable(feature = "const_option", issue = "67441")] + pub const fn as_mut(&mut self) -> Option<&mut T> { match *self { Some(ref mut x) => Some(x), None => None, } } - /// Converts from [`Pin`]`<&Option>` to `Option<`[`Pin`]`<&T>>`. + /// Converts from [Pin]<[&]Option\> to Option<[Pin]<[&]T>>. /// - /// [`Pin`]: ../pin/struct.Pin.html + /// [&]: reference "shared reference" #[inline] + #[must_use] #[stable(feature = "pin", since = "1.33.0")] - pub fn as_pin_ref(self: Pin<&Self>) -> Option> { - unsafe { Pin::get_ref(self).as_ref().map(|x| Pin::new_unchecked(x)) } + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn as_pin_ref(self: Pin<&Self>) -> Option> { + match Pin::get_ref(self).as_ref() { + // SAFETY: `x` is guaranteed to be pinned because it comes from `self` + // which is pinned. + Some(x) => unsafe { Some(Pin::new_unchecked(x)) }, + None => None, + } } - /// Converts from [`Pin`]`<&mut Option>` to `Option<`[`Pin`]`<&mut T>>`. + /// Converts from [Pin]<[&mut] Option\> to Option<[Pin]<[&mut] T>>. /// - /// [`Pin`]: ../pin/struct.Pin.html + /// [&mut]: reference "mutable reference" #[inline] + #[must_use] #[stable(feature = "pin", since = "1.33.0")] - pub fn as_pin_mut(self: Pin<&mut Self>) -> Option> { - unsafe { Pin::get_unchecked_mut(self).as_mut().map(|x| Pin::new_unchecked(x)) } + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn as_pin_mut(self: Pin<&mut Self>) -> Option> { + // SAFETY: `get_unchecked_mut` is never used to move the `Option` inside `self`. + // `x` is guaranteed to be pinned because it comes from `self` which is pinned. + unsafe { + match Pin::get_unchecked_mut(self).as_mut() { + Some(x) => Some(Pin::new_unchecked(x)), + None => None, + } + } } ///////////////////////////////////////////////////////////////////////// @@ -324,9 +745,6 @@ impl Option { /// Panics if the value is a [`None`] with a custom panic message provided by /// `msg`. /// - /// [`Some`]: #variant.Some - /// [`None`]: #variant.None - /// /// # Examples /// /// ``` @@ -334,14 +752,35 @@ impl Option { /// assert_eq!(x.expect("fruits are healthy"), "value"); /// ``` /// - /// ```{.should_panic} + /// ```should_panic /// let x: Option<&str> = None; /// x.expect("fruits are healthy"); // panics with `fruits are healthy` /// ``` + /// + /// # Recommended Message Style + /// + /// We recommend that `expect` messages are used to describe the reason you + /// _expect_ the `Option` should be `Some`. + /// + /// ```should_panic + /// # let slice: &[u8] = &[]; + /// let item = slice.get(0) + /// .expect("slice should not be empty"); + /// ``` + /// + /// **Hint**: If you're having trouble remembering how to phrase expect + /// error messages remember to focus on the word "should" as in "env + /// variable should be set by blah" or "the given binary should be available + /// and executable by the current user". + /// + /// For more detail on expect message styles and the reasoning behind our + /// recommendation please refer to the section on ["Common Message + /// Styles"](../../std/error/index.html#common-message-styles) in the [`std::error`](../../std/error/index.html) module docs. #[inline] #[track_caller] #[stable(feature = "rust1", since = "1.0.0")] - pub fn expect(self, msg: &str) -> T { + #[rustc_const_unstable(feature = "const_option", issue = "67441")] + pub const fn expect(self, msg: &str) -> T { match self { Some(val) => val, None => expect_failed(msg), @@ -355,17 +794,14 @@ impl Option { /// case explicitly, or call [`unwrap_or`], [`unwrap_or_else`], or /// [`unwrap_or_default`]. /// - /// [`unwrap_or`]: #method.unwrap_or - /// [`unwrap_or_else`]: #method.unwrap_or_else - /// [`unwrap_or_default`]: #method.unwrap_or_default + /// [`unwrap_or`]: Option::unwrap_or + /// [`unwrap_or_else`]: Option::unwrap_or_else + /// [`unwrap_or_default`]: Option::unwrap_or_default /// /// # Panics /// /// Panics if the self value equals [`None`]. /// - /// [`Some`]: #variant.Some - /// [`None`]: #variant.None - /// /// # Examples /// /// ``` @@ -373,17 +809,18 @@ impl Option { /// assert_eq!(x.unwrap(), "air"); /// ``` /// - /// ```{.should_panic} + /// ```should_panic /// let x: Option<&str> = None; /// assert_eq!(x.unwrap(), "air"); // fails /// ``` #[inline] #[track_caller] #[stable(feature = "rust1", since = "1.0.0")] - pub fn unwrap(self) -> T { + #[rustc_const_unstable(feature = "const_option", issue = "67441")] + pub const fn unwrap(self) -> T { match self { Some(val) => val, - None => panic!("called `Option::unwrap()` on a `None` value"), + None => panic("called `Option::unwrap()` on a `None` value"), } } @@ -393,8 +830,7 @@ impl Option { /// the result of a function call, it is recommended to use [`unwrap_or_else`], /// which is lazily evaluated. /// - /// [`Some`]: #variant.Some - /// [`unwrap_or_else`]: #method.unwrap_or_else + /// [`unwrap_or_else`]: Option::unwrap_or_else /// /// # Examples /// @@ -404,7 +840,11 @@ impl Option { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - pub fn unwrap_or(self, default: T) -> T { + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn unwrap_or(self, default: T) -> T + where + T: ~const Destruct, + { match self { Some(x) => x, None => default, @@ -422,13 +862,83 @@ impl Option { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - pub fn unwrap_or_else T>(self, f: F) -> T { + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn unwrap_or_else(self, f: F) -> T + where + F: ~const FnOnce() -> T, + F: ~const Destruct, + { match self { Some(x) => x, None => f(), } } + /// Returns the contained [`Some`] value or a default. + /// + /// Consumes the `self` argument then, if [`Some`], returns the contained + /// value, otherwise if [`None`], returns the [default value] for that + /// type. + /// + /// # Examples + /// + /// ``` + /// let x: Option = None; + /// let y: Option = Some(12); + /// + /// assert_eq!(x.unwrap_or_default(), 0); + /// assert_eq!(y.unwrap_or_default(), 12); + /// ``` + /// + /// [default value]: Default::default + /// [`parse`]: str::parse + /// [`FromStr`]: crate::str::FromStr + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn unwrap_or_default(self) -> T + where + T: ~const Default, + { + match self { + Some(x) => x, + None => Default::default(), + } + } + + /// Returns the contained [`Some`] value, consuming the `self` value, + /// without checking that the value is not [`None`]. + /// + /// # Safety + /// + /// Calling this method on [`None`] is *[undefined behavior]*. + /// + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html + /// + /// # Examples + /// + /// ``` + /// let x = Some("air"); + /// assert_eq!(unsafe { x.unwrap_unchecked() }, "air"); + /// ``` + /// + /// ```no_run + /// let x: Option<&str> = None; + /// assert_eq!(unsafe { x.unwrap_unchecked() }, "air"); // Undefined behavior! + /// ``` + #[inline] + #[track_caller] + #[stable(feature = "option_result_unwrap_unchecked", since = "1.58.0")] + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const unsafe fn unwrap_unchecked(self) -> T { + debug_assert!(self.is_some()); + match self { + Some(val) => val, + // SAFETY: the safety contract must be upheld by the caller. + None => unsafe { hint::unreachable_unchecked() }, + } + } + ///////////////////////////////////////////////////////////////////////// // Transforming contained values ///////////////////////////////////////////////////////////////////////// @@ -437,11 +947,10 @@ impl Option { /// /// # Examples /// - /// Converts an `Option<`[`String`]`>` into an `Option<`[`usize`]`>`, consuming the original: - /// - /// [`String`]: ../../std/string/struct.String.html - /// [`usize`]: ../../std/primitive.usize.html + /// Calculates the length of an Option<[String]> as an + /// Option<[usize]>, consuming the original: /// + /// [String]: ../../std/string/struct.String.html "String" /// ``` /// let maybe_some_string = Some(String::from("Hello, World!")); /// // `Option::map` takes self *by value*, consuming `maybe_some_string` @@ -451,21 +960,56 @@ impl Option { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - pub fn map U>(self, f: F) -> Option { + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn map(self, f: F) -> Option + where + F: ~const FnOnce(T) -> U, + F: ~const Destruct, + { match self { Some(x) => Some(f(x)), None => None, } } - /// Applies a function to the contained value (if any), - /// or returns the provided default (if not). + /// Calls the provided closure with a reference to the contained value (if [`Some`]). + /// + /// # Examples + /// + /// ``` + /// #![feature(result_option_inspect)] + /// + /// let v = vec![1, 2, 3, 4, 5]; + /// + /// // prints "got: 4" + /// let x: Option<&usize> = v.get(3).inspect(|x| println!("got: {x}")); + /// + /// // prints nothing + /// let x: Option<&usize> = v.get(5).inspect(|x| println!("got: {x}")); + /// ``` + #[inline] + #[unstable(feature = "result_option_inspect", issue = "91345")] + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn inspect(self, f: F) -> Self + where + F: ~const FnOnce(&T), + F: ~const Destruct, + { + if let Some(ref x) = self { + f(x); + } + + self + } + + /// Returns the provided default result (if none), + /// or applies a function to the contained value (if any). /// /// Arguments passed to `map_or` are eagerly evaluated; if you are passing /// the result of a function call, it is recommended to use [`map_or_else`], /// which is lazily evaluated. /// - /// [`map_or_else`]: #method.map_or_else + /// [`map_or_else`]: Option::map_or_else /// /// # Examples /// @@ -478,15 +1022,21 @@ impl Option { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - pub fn map_or U>(self, default: U, f: F) -> U { + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn map_or(self, default: U, f: F) -> U + where + F: ~const FnOnce(T) -> U, + F: ~const Destruct, + U: ~const Destruct, + { match self { Some(t) => f(t), None => default, } } - /// Applies a function to the contained value (if any), - /// or computes a default (if not). + /// Computes a default function result (if none), or + /// applies a different function to the contained value (if any). /// /// # Examples /// @@ -501,7 +1051,14 @@ impl Option { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - pub fn map_or_else U, F: FnOnce(T) -> U>(self, default: D, f: F) -> U { + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn map_or_else(self, default: D, f: F) -> U + where + D: ~const FnOnce() -> U, + D: ~const Destruct, + F: ~const FnOnce(T) -> U, + F: ~const Destruct, + { match self { Some(t) => f(t), None => default(), @@ -515,12 +1072,10 @@ impl Option { /// result of a function call, it is recommended to use [`ok_or_else`], which is /// lazily evaluated. /// - /// [`Result`]: ../../std/result/enum.Result.html - /// [`Ok(v)`]: ../../std/result/enum.Result.html#variant.Ok - /// [`Err(err)`]: ../../std/result/enum.Result.html#variant.Err - /// [`None`]: #variant.None - /// [`Some(v)`]: #variant.Some - /// [`ok_or_else`]: #method.ok_or_else + /// [`Ok(v)`]: Ok + /// [`Err(err)`]: Err + /// [`Some(v)`]: Some + /// [`ok_or_else`]: Option::ok_or_else /// /// # Examples /// @@ -533,7 +1088,11 @@ impl Option { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - pub fn ok_or(self, err: E) -> Result { + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn ok_or(self, err: E) -> Result + where + E: ~const Destruct, + { match self { Some(v) => Ok(v), None => Err(err), @@ -543,11 +1102,9 @@ impl Option { /// Transforms the `Option` into a [`Result`], mapping [`Some(v)`] to /// [`Ok(v)`] and [`None`] to [`Err(err())`]. /// - /// [`Result`]: ../../std/result/enum.Result.html - /// [`Ok(v)`]: ../../std/result/enum.Result.html#variant.Ok - /// [`Err(err())`]: ../../std/result/enum.Result.html#variant.Err - /// [`None`]: #variant.None - /// [`Some(v)`]: #variant.Some + /// [`Ok(v)`]: Ok + /// [`Err(err())`]: Err + /// [`Some(v)`]: Some /// /// # Examples /// @@ -560,13 +1117,70 @@ impl Option { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - pub fn ok_or_else E>(self, err: F) -> Result { + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn ok_or_else(self, err: F) -> Result + where + F: ~const FnOnce() -> E, + F: ~const Destruct, + { match self { Some(v) => Ok(v), None => Err(err()), } } + /// Converts from `Option` (or `&Option`) to `Option<&T::Target>`. + /// + /// Leaves the original Option in-place, creating a new one with a reference + /// to the original one, additionally coercing the contents via [`Deref`]. + /// + /// # Examples + /// + /// ``` + /// let x: Option = Some("hey".to_owned()); + /// assert_eq!(x.as_deref(), Some("hey")); + /// + /// let x: Option = None; + /// assert_eq!(x.as_deref(), None); + /// ``` + #[stable(feature = "option_deref", since = "1.40.0")] + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn as_deref(&self) -> Option<&T::Target> + where + T: ~const Deref, + { + match self.as_ref() { + Some(t) => Some(t.deref()), + None => None, + } + } + + /// Converts from `Option` (or `&mut Option`) to `Option<&mut T::Target>`. + /// + /// Leaves the original `Option` in-place, creating a new one containing a mutable reference to + /// the inner type's [`Deref::Target`] type. + /// + /// # Examples + /// + /// ``` + /// let mut x: Option = Some("hey".to_owned()); + /// assert_eq!(x.as_deref_mut().map(|x| { + /// x.make_ascii_uppercase(); + /// x + /// }), Some("HEY".to_owned().as_mut_str())); + /// ``` + #[stable(feature = "option_deref", since = "1.40.0")] + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn as_deref_mut(&mut self) -> Option<&mut T::Target> + where + T: ~const DerefMut, + { + match self.as_mut() { + Some(t) => Some(t.deref_mut()), + None => None, + } + } + ///////////////////////////////////////////////////////////////////////// // Iterator constructors ///////////////////////////////////////////////////////////////////////// @@ -583,8 +1197,9 @@ impl Option { /// assert_eq!(x.iter().next(), None); /// ``` #[inline] + #[rustc_const_unstable(feature = "const_option", issue = "67441")] #[stable(feature = "rust1", since = "1.0.0")] - pub fn iter(&self) -> Iter<'_, T> { + pub const fn iter(&self) -> Iter<'_, T> { Iter { inner: Item { opt: self.as_ref() } } } @@ -615,7 +1230,11 @@ impl Option { /// Returns [`None`] if the option is [`None`], otherwise returns `optb`. /// - /// [`None`]: #variant.None + /// Arguments passed to `and` are eagerly evaluated; if you are passing the + /// result of a function call, it is recommended to use [`and_then`], which is + /// lazily evaluated. + /// + /// [`and_then`]: Option::and_then /// /// # Examples /// @@ -638,7 +1257,12 @@ impl Option { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - pub fn and(self, optb: Option) -> Option { + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn and(self, optb: Option) -> Option + where + T: ~const Destruct, + U: ~const Destruct, + { match self { Some(_) => optb, None => None, @@ -650,22 +1274,37 @@ impl Option { /// /// Some languages call this operation flatmap. /// - /// [`None`]: #variant.None - /// /// # Examples /// /// ``` - /// fn sq(x: u32) -> Option { Some(x * x) } - /// fn nope(_: u32) -> Option { None } + /// fn sq_then_to_string(x: u32) -> Option { + /// x.checked_mul(x).map(|sq| sq.to_string()) + /// } + /// + /// assert_eq!(Some(2).and_then(sq_then_to_string), Some(4.to_string())); + /// assert_eq!(Some(1_000_000).and_then(sq_then_to_string), None); // overflowed! + /// assert_eq!(None.and_then(sq_then_to_string), None); + /// ``` + /// + /// Often used to chain fallible operations that may return [`None`]. /// - /// assert_eq!(Some(2).and_then(sq).and_then(sq), Some(16)); - /// assert_eq!(Some(2).and_then(sq).and_then(nope), None); - /// assert_eq!(Some(2).and_then(nope).and_then(sq), None); - /// assert_eq!(None.and_then(sq).and_then(sq), None); + /// ``` + /// let arr_2d = [["A0", "A1"], ["B0", "B1"]]; + /// + /// let item_0_1 = arr_2d.get(0).and_then(|row| row.get(1)); + /// assert_eq!(item_0_1, Some(&"A1")); + /// + /// let item_2_0 = arr_2d.get(2).and_then(|row| row.get(0)); + /// assert_eq!(item_2_0, None); /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - pub fn and_then Option>(self, f: F) -> Option { + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn and_then(self, f: F) -> Option + where + F: ~const FnOnce(T) -> Option, + F: ~const Destruct, + { match self { Some(x) => f(x), None => None, @@ -695,12 +1334,16 @@ impl Option { /// assert_eq!(Some(4).filter(is_even), Some(4)); /// ``` /// - /// [`None`]: #variant.None - /// [`Some(t)`]: #variant.Some - /// [`Iterator::filter()`]: ../../std/iter/trait.Iterator.html#method.filter + /// [`Some(t)`]: Some #[inline] #[stable(feature = "option_filter", since = "1.27.0")] - pub fn filter bool>(self, predicate: P) -> Self { + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn filter

(self, predicate: P) -> Self + where + T: ~const Destruct, + P: ~const FnOnce(&T) -> bool, + P: ~const Destruct, + { if let Some(x) = self { if predicate(&x) { return Some(x); @@ -715,7 +1358,7 @@ impl Option { /// result of a function call, it is recommended to use [`or_else`], which is /// lazily evaluated. /// - /// [`or_else`]: #method.or_else + /// [`or_else`]: Option::or_else /// /// # Examples /// @@ -738,9 +1381,13 @@ impl Option { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - pub fn or(self, optb: Option) -> Option { + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn or(self, optb: Option) -> Option + where + T: ~const Destruct, + { match self { - Some(_) => self, + Some(x) => Some(x), None => optb, } } @@ -760,18 +1407,20 @@ impl Option { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - pub fn or_else Option>(self, f: F) -> Option { + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn or_else(self, f: F) -> Option + where + F: ~const FnOnce() -> Option, + F: ~const Destruct, + { match self { - Some(_) => self, + Some(x) => Some(x), None => f(), } } /// Returns [`Some`] if exactly one of `self`, `optb` is [`Some`], otherwise returns [`None`]. /// - /// [`Some`]: #variant.Some - /// [`None`]: #variant.None - /// /// # Examples /// /// ``` @@ -793,7 +1442,11 @@ impl Option { /// ``` #[inline] #[stable(feature = "option_xor", since = "1.37.0")] - pub fn xor(self, optb: Option) -> Option { + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn xor(self, optb: Option) -> Option + where + T: ~const Destruct, + { match (self, optb) { (Some(a), None) => Some(a), (None, Some(b)) => Some(b), @@ -802,13 +1455,47 @@ impl Option { } ///////////////////////////////////////////////////////////////////////// - // Entry-like operations to insert if None and return a reference + // Entry-like operations to insert a value and return a reference ///////////////////////////////////////////////////////////////////////// - /// Inserts `v` into the option if it is [`None`], then + /// Inserts `value` into the option, then returns a mutable reference to it. + /// + /// If the option already contains a value, the old value is dropped. + /// + /// See also [`Option::get_or_insert`], which doesn't update the value if + /// the option already contains [`Some`]. + /// + /// # Example + /// + /// ``` + /// let mut opt = None; + /// let val = opt.insert(1); + /// assert_eq!(*val, 1); + /// assert_eq!(opt.unwrap(), 1); + /// let val = opt.insert(2); + /// assert_eq!(*val, 2); + /// *val = 3; + /// assert_eq!(opt.unwrap(), 3); + /// ``` + #[must_use = "if you intended to set a value, consider assignment instead"] + #[inline] + #[stable(feature = "option_insert", since = "1.53.0")] + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn insert(&mut self, value: T) -> &mut T + where + T: ~const Destruct, + { + *self = Some(value); + + // SAFETY: the code above just filled the option + unsafe { self.as_mut().unwrap_unchecked() } + } + + /// Inserts `value` into the option if it is [`None`], then /// returns a mutable reference to the contained value. /// - /// [`None`]: #variant.None + /// See also [`Option::insert`], which updates the value even if + /// the option already contains [`Some`]. /// /// # Examples /// @@ -826,14 +1513,55 @@ impl Option { /// ``` #[inline] #[stable(feature = "option_entry", since = "1.20.0")] - pub fn get_or_insert(&mut self, v: T) -> &mut T { - self.get_or_insert_with(|| v) + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn get_or_insert(&mut self, value: T) -> &mut T + where + T: ~const Destruct, + { + if let None = *self { + *self = Some(value); + } + + // SAFETY: a `None` variant for `self` would have been replaced by a `Some` + // variant in the code above. + unsafe { self.as_mut().unwrap_unchecked() } } - /// Inserts a value computed from `f` into the option if it is [`None`], then + /// Inserts the default value into the option if it is [`None`], then /// returns a mutable reference to the contained value. /// - /// [`None`]: #variant.None + /// # Examples + /// + /// ``` + /// #![feature(option_get_or_insert_default)] + /// + /// let mut x = None; + /// + /// { + /// let y: &mut u32 = x.get_or_insert_default(); + /// assert_eq!(y, &0); + /// + /// *y = 7; + /// } + /// + /// assert_eq!(x, Some(7)); + /// ``` + #[inline] + #[unstable(feature = "option_get_or_insert_default", issue = "82901")] + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn get_or_insert_default(&mut self) -> &mut T + where + T: ~const Default, + { + const fn default() -> T { + T::default() + } + + self.get_or_insert_with(default) + } + + /// Inserts a value computed from `f` into the option if it is [`None`], + /// then returns a mutable reference to the contained value. /// /// # Examples /// @@ -851,15 +1579,21 @@ impl Option { /// ``` #[inline] #[stable(feature = "option_entry", since = "1.20.0")] - pub fn get_or_insert_with T>(&mut self, f: F) -> &mut T { - if let None = *self { - *self = Some(f()); - } - - match *self { - Some(ref mut v) => v, - None => unsafe { hint::unreachable_unchecked() }, + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn get_or_insert_with(&mut self, f: F) -> &mut T + where + F: ~const FnOnce() -> T, + F: ~const Destruct, + { + if let None = *self { + // the compiler isn't smart enough to know that we are not dropping a `T` + // here and wants us to ensure `T` can be dropped at compile time. + mem::forget(mem::replace(self, Some(f()))) } + + // SAFETY: a `None` variant for `self` would have been replaced by a `Some` + // variant in the code above. + unsafe { self.as_mut().unwrap_unchecked() } } ///////////////////////////////////////////////////////////////////////// @@ -868,8 +1602,6 @@ impl Option { /// Takes the value out of the option, leaving a [`None`] in its place. /// - /// [`None`]: #variant.None - /// /// # Examples /// /// ``` @@ -885,16 +1617,16 @@ impl Option { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - pub fn take(&mut self) -> Option { - mem::take(self) + #[rustc_const_unstable(feature = "const_option", issue = "67441")] + pub const fn take(&mut self) -> Option { + // FIXME replace `mem::replace` by `mem::take` when the latter is const ready + mem::replace(self, None) } /// Replaces the actual value in the option by the value given in parameter, /// returning the old value if present, /// leaving a [`Some`] in its place without deinitializing either one. /// - /// [`Some`]: #variant.Some - /// /// # Examples /// /// ``` @@ -909,11 +1641,42 @@ impl Option { /// assert_eq!(old, None); /// ``` #[inline] + #[rustc_const_unstable(feature = "const_option", issue = "67441")] #[stable(feature = "option_replace", since = "1.31.0")] - pub fn replace(&mut self, value: T) -> Option { + pub const fn replace(&mut self, value: T) -> Option { mem::replace(self, Some(value)) } + /// Returns `true` if the option is a [`Some`] value containing the given value. + /// + /// # Examples + /// + /// ``` + /// #![feature(option_result_contains)] + /// + /// let x: Option = Some(2); + /// assert_eq!(x.contains(&2), true); + /// + /// let x: Option = Some(3); + /// assert_eq!(x.contains(&2), false); + /// + /// let x: Option = None; + /// assert_eq!(x.contains(&2), false); + /// ``` + #[must_use] + #[inline] + #[unstable(feature = "option_result_contains", issue = "62358")] + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn contains(&self, x: &U) -> bool + where + U: ~const PartialEq, + { + match self { + Some(y) => x.eq(y), + None => false, + } + } + /// Zips `self` with another `Option`. /// /// If `self` is `Some(s)` and `other` is `Some(o)`, this method returns `Some((s, o))`. @@ -922,7 +1685,6 @@ impl Option { /// # Examples /// /// ``` - /// #![feature(option_zip)] /// let x = Some(1); /// let y = Some("hi"); /// let z = None::; @@ -930,9 +1692,17 @@ impl Option { /// assert_eq!(x.zip(y), Some((1, "hi"))); /// assert_eq!(x.zip(z), None); /// ``` - #[unstable(feature = "option_zip", issue = "70086")] - pub fn zip(self, other: Option) -> Option<(T, U)> { - self.zip_with(other, |a, b| (a, b)) + #[stable(feature = "option_zip_option", since = "1.46.0")] + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn zip(self, other: Option) -> Option<(T, U)> + where + T: ~const Destruct, + U: ~const Destruct, + { + match (self, other) { + (Some(a), Some(b)) => Some((a, b)), + _ => None, + } } /// Zips `self` and another `Option` with function `f`. @@ -964,53 +1734,79 @@ impl Option { /// assert_eq!(x.zip_with(None, Point::new), None); /// ``` #[unstable(feature = "option_zip", issue = "70086")] - pub fn zip_with(self, other: Option, f: F) -> Option + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn zip_with(self, other: Option, f: F) -> Option where - F: FnOnce(T, U) -> R, + F: ~const FnOnce(T, U) -> R, + F: ~const Destruct, + T: ~const Destruct, + U: ~const Destruct, { - Some(f(self?, other?)) + match (self, other) { + (Some(a), Some(b)) => Some(f(a, b)), + _ => None, + } } } -impl Option<&T> { - /// Maps an `Option<&T>` to an `Option` by copying the contents of the - /// option. +impl Option<(T, U)> { + /// Unzips an option containing a tuple of two options. + /// + /// If `self` is `Some((a, b))` this method returns `(Some(a), Some(b))`. + /// Otherwise, `(None, None)` is returned. /// /// # Examples /// /// ``` - /// let x = 12; - /// let opt_x = Some(&x); - /// assert_eq!(opt_x, Some(&12)); - /// let copied = opt_x.copied(); - /// assert_eq!(copied, Some(12)); + /// let x = Some((1, "hi")); + /// let y = None::<(u8, u32)>; + /// + /// assert_eq!(x.unzip(), (Some(1), Some("hi"))); + /// assert_eq!(y.unzip(), (None, None)); /// ``` - #[stable(feature = "copied", since = "1.35.0")] - pub fn copied(self) -> Option { - self.map(|&t| t) + #[inline] + #[stable(feature = "unzip_option", since = "1.66.0")] + #[rustc_const_unstable(feature = "const_option", issue = "67441")] + pub const fn unzip(self) -> (Option, Option) + where + T: ~const Destruct, + U: ~const Destruct, + { + match self { + Some((a, b)) => (Some(a), Some(b)), + None => (None, None), + } } } -impl Option<&mut T> { - /// Maps an `Option<&mut T>` to an `Option` by copying the contents of the +impl Option<&T> { + /// Maps an `Option<&T>` to an `Option` by copying the contents of the /// option. /// /// # Examples /// /// ``` - /// let mut x = 12; - /// let opt_x = Some(&mut x); - /// assert_eq!(opt_x, Some(&mut 12)); + /// let x = 12; + /// let opt_x = Some(&x); + /// assert_eq!(opt_x, Some(&12)); /// let copied = opt_x.copied(); /// assert_eq!(copied, Some(12)); /// ``` + #[must_use = "`self` will be dropped if the result is not used"] #[stable(feature = "copied", since = "1.35.0")] - pub fn copied(self) -> Option { - self.map(|&mut t| t) + #[rustc_const_unstable(feature = "const_option", issue = "67441")] + pub const fn copied(self) -> Option + where + T: Copy, + { + // FIXME: this implementation, which sidesteps using `Option::map` since it's not const + // ready yet, should be reverted when possible to avoid code repetition + match self { + Some(&v) => Some(v), + None => None, + } } -} -impl Option<&T> { /// Maps an `Option<&T>` to an `Option` by cloning the contents of the /// option. /// @@ -1023,14 +1819,22 @@ impl Option<&T> { /// let cloned = opt_x.cloned(); /// assert_eq!(cloned, Some(12)); /// ``` + #[must_use = "`self` will be dropped if the result is not used"] #[stable(feature = "rust1", since = "1.0.0")] - pub fn cloned(self) -> Option { - self.map(|t| t.clone()) + #[rustc_const_unstable(feature = "const_option_cloned", issue = "91582")] + pub const fn cloned(self) -> Option + where + T: ~const Clone, + { + match self { + Some(t) => Some(t.clone()), + None => None, + } } } -impl Option<&mut T> { - /// Maps an `Option<&mut T>` to an `Option` by cloning the contents of the +impl Option<&mut T> { + /// Maps an `Option<&mut T>` to an `Option` by copying the contents of the /// option. /// /// # Examples @@ -1039,197 +1843,54 @@ impl Option<&mut T> { /// let mut x = 12; /// let opt_x = Some(&mut x); /// assert_eq!(opt_x, Some(&mut 12)); - /// let cloned = opt_x.cloned(); - /// assert_eq!(cloned, Some(12)); - /// ``` - #[stable(since = "1.26.0", feature = "option_ref_mut_cloned")] - pub fn cloned(self) -> Option { - self.map(|t| t.clone()) - } -} - -impl Option { - /// Consumes `self` while expecting [`None`] and returning nothing. - /// - /// # Panics - /// - /// Panics if the value is a [`Some`], with a panic message including the - /// passed message, and the content of the [`Some`]. - /// - /// [`Some`]: #variant.Some - /// [`None`]: #variant.None - /// - /// # Examples - /// - /// ``` - /// #![feature(option_expect_none)] - /// - /// use std::collections::HashMap; - /// let mut squares = HashMap::new(); - /// for i in -10..=10 { - /// // This will not panic, since all keys are unique. - /// squares.insert(i, i * i).expect_none("duplicate key"); - /// } - /// ``` - /// - /// ```{.should_panic} - /// #![feature(option_expect_none)] - /// - /// use std::collections::HashMap; - /// let mut sqrts = HashMap::new(); - /// for i in -10..=10 { - /// // This will panic, since both negative and positive `i` will - /// // insert the same `i * i` key, returning the old `Some(i)`. - /// sqrts.insert(i * i, i).expect_none("duplicate key"); - /// } - /// ``` - #[inline] - #[track_caller] - #[unstable(feature = "option_expect_none", reason = "newly added", issue = "62633")] - pub fn expect_none(self, msg: &str) { - if let Some(val) = self { - expect_none_failed(msg, &val); - } - } - - /// Consumes `self` while expecting [`None`] and returning nothing. - /// - /// # Panics - /// - /// Panics if the value is a [`Some`], with a custom panic message provided - /// by the [`Some`]'s value. - /// - /// [`Some(v)`]: #variant.Some - /// [`None`]: #variant.None - /// - /// # Examples - /// - /// ``` - /// #![feature(option_unwrap_none)] - /// - /// use std::collections::HashMap; - /// let mut squares = HashMap::new(); - /// for i in -10..=10 { - /// // This will not panic, since all keys are unique. - /// squares.insert(i, i * i).unwrap_none(); - /// } - /// ``` - /// - /// ```{.should_panic} - /// #![feature(option_unwrap_none)] - /// - /// use std::collections::HashMap; - /// let mut sqrts = HashMap::new(); - /// for i in -10..=10 { - /// // This will panic, since both negative and positive `i` will - /// // insert the same `i * i` key, returning the old `Some(i)`. - /// sqrts.insert(i * i, i).unwrap_none(); - /// } + /// let copied = opt_x.copied(); + /// assert_eq!(copied, Some(12)); /// ``` - #[inline] - #[track_caller] - #[unstable(feature = "option_unwrap_none", reason = "newly added", issue = "62633")] - pub fn unwrap_none(self) { - if let Some(val) = self { - expect_none_failed("called `Option::unwrap_none()` on a `Some` value", &val); + #[must_use = "`self` will be dropped if the result is not used"] + #[stable(feature = "copied", since = "1.35.0")] + #[rustc_const_unstable(feature = "const_option_ext", issue = "91930")] + pub const fn copied(self) -> Option + where + T: Copy, + { + match self { + Some(&mut t) => Some(t), + None => None, } } -} -impl Option { - /// Returns the contained [`Some`] value or a default - /// - /// Consumes the `self` argument then, if [`Some`], returns the contained - /// value, otherwise if [`None`], returns the [default value] for that - /// type. + /// Maps an `Option<&mut T>` to an `Option` by cloning the contents of the + /// option. /// /// # Examples /// - /// Converts a string to an integer, turning poorly-formed strings - /// into 0 (the default value for integers). [`parse`] converts - /// a string to any other type that implements [`FromStr`], returning - /// [`None`] on error. - /// /// ``` - /// let good_year_from_input = "1909"; - /// let bad_year_from_input = "190blarg"; - /// let good_year = good_year_from_input.parse().ok().unwrap_or_default(); - /// let bad_year = bad_year_from_input.parse().ok().unwrap_or_default(); - /// - /// assert_eq!(1909, good_year); - /// assert_eq!(0, bad_year); + /// let mut x = 12; + /// let opt_x = Some(&mut x); + /// assert_eq!(opt_x, Some(&mut 12)); + /// let cloned = opt_x.cloned(); + /// assert_eq!(cloned, Some(12)); /// ``` - /// - /// [`Some`]: #variant.Some - /// [`None`]: #variant.None - /// [default value]: ../default/trait.Default.html#tymethod.default - /// [`parse`]: ../../std/primitive.str.html#method.parse - /// [`FromStr`]: ../../std/str/trait.FromStr.html - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn unwrap_or_default(self) -> T { + #[must_use = "`self` will be dropped if the result is not used"] + #[stable(since = "1.26.0", feature = "option_ref_mut_cloned")] + #[rustc_const_unstable(feature = "const_option_cloned", issue = "91582")] + pub const fn cloned(self) -> Option + where + T: ~const Clone, + { match self { - Some(x) => x, - None => Default::default(), + Some(t) => Some(t.clone()), + None => None, } } } -impl Option { - /// Converts from `Option` (or `&Option`) to `Option<&T::Target>`. - /// - /// Leaves the original Option in-place, creating a new one with a reference - /// to the original one, additionally coercing the contents via [`Deref`]. - /// - /// [`Deref`]: ../../std/ops/trait.Deref.html - /// - /// # Examples - /// - /// ``` - /// let x: Option = Some("hey".to_owned()); - /// assert_eq!(x.as_deref(), Some("hey")); - /// - /// let x: Option = None; - /// assert_eq!(x.as_deref(), None); - /// ``` - #[stable(feature = "option_deref", since = "1.40.0")] - pub fn as_deref(&self) -> Option<&T::Target> { - self.as_ref().map(|t| t.deref()) - } -} - -impl Option { - /// Converts from `Option` (or `&mut Option`) to `Option<&mut T::Target>`. - /// - /// Leaves the original `Option` in-place, creating a new one containing a mutable reference to - /// the inner type's `Deref::Target` type. - /// - /// # Examples - /// - /// ``` - /// let mut x: Option = Some("hey".to_owned()); - /// assert_eq!(x.as_deref_mut().map(|x| { - /// x.make_ascii_uppercase(); - /// x - /// }), Some("HEY".to_owned().as_mut_str())); - /// ``` - #[stable(feature = "option_deref", since = "1.40.0")] - pub fn as_deref_mut(&mut self) -> Option<&mut T::Target> { - self.as_mut().map(|t| t.deref_mut()) - } -} - impl Option> { /// Transposes an `Option` of a [`Result`] into a [`Result`] of an `Option`. /// - /// [`None`] will be mapped to [`Ok`]`(`[`None`]`)`. - /// [`Some`]`(`[`Ok`]`(_))` and [`Some`]`(`[`Err`]`(_))` will be mapped to - /// [`Ok`]`(`[`Some`]`(_))` and [`Err`]`(_)`. - /// - /// [`None`]: #variant.None - /// [`Ok`]: ../../std/result/enum.Result.html#variant.Ok - /// [`Some`]: #variant.Some - /// [`Err`]: ../../std/result/enum.Result.html#variant.Err + /// [`None`] will be mapped to [Ok]\([None]). + /// [Some]\([Ok]\(\_)) and [Some]\([Err]\(\_)) will be mapped to + /// [Ok]\([Some]\(\_)) and [Err]\(\_). /// /// # Examples /// @@ -1243,7 +1904,8 @@ impl Option> { /// ``` #[inline] #[stable(feature = "transpose_result", since = "1.33.0")] - pub fn transpose(self) -> Result, E> { + #[rustc_const_unstable(feature = "const_option", issue = "67441")] + pub const fn transpose(self) -> Result, E> { match self { Some(Ok(x)) => Ok(Some(x)), Some(Err(e)) => Err(e), @@ -1253,19 +1915,13 @@ impl Option> { } // This is a separate function to reduce the code size of .expect() itself. -#[inline(never)] +#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never))] +#[cfg_attr(feature = "panic_immediate_abort", inline)] #[cold] #[track_caller] -fn expect_failed(msg: &str) -> ! { - panic!("{}", msg) -} - -// This is a separate function to reduce the code size of .expect_none() itself. -#[inline(never)] -#[cold] -#[track_caller] -fn expect_none_failed(msg: &str, value: &dyn fmt::Debug) -> ! { - panic!("{}: {:?}", msg, value) +#[rustc_const_unstable(feature = "const_option", issue = "67441")] +const fn expect_failed(msg: &str) -> ! { + panic_str(msg) } ///////////////////////////////////////////////////////////////////////////// @@ -1273,7 +1929,11 @@ fn expect_none_failed(msg: &str, value: &dyn fmt::Debug) -> ! { ///////////////////////////////////////////////////////////////////////////// #[stable(feature = "rust1", since = "1.0.0")] -impl Clone for Option { +#[rustc_const_unstable(feature = "const_clone", issue = "91805")] +impl const Clone for Option +where + T: ~const Clone + ~const Destruct, +{ #[inline] fn clone(&self) -> Self { match self { @@ -1292,7 +1952,8 @@ impl Clone for Option { } #[stable(feature = "rust1", since = "1.0.0")] -impl Default for Option { +#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] +impl const Default for Option { /// Returns [`None`][Option::None]. /// /// # Examples @@ -1352,26 +2013,139 @@ impl<'a, T> IntoIterator for &'a mut Option { } #[stable(since = "1.12.0", feature = "option_from")] -impl From for Option { +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const From for Option { + /// Moves `val` into a new [`Some`]. + /// + /// # Examples + /// + /// ``` + /// let o: Option = Option::from(67); + /// + /// assert_eq!(Some(67), o); + /// ``` fn from(val: T) -> Option { Some(val) } } #[stable(feature = "option_ref_from_ref_option", since = "1.30.0")] -impl<'a, T> From<&'a Option> for Option<&'a T> { +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl<'a, T> const From<&'a Option> for Option<&'a T> { + /// Converts from `&Option` to `Option<&T>`. + /// + /// # Examples + /// + /// Converts an [Option]<[String]> into an [Option]<[usize]>, preserving + /// the original. The [`map`] method takes the `self` argument by value, consuming the original, + /// so this technique uses `from` to first take an [`Option`] to a reference + /// to the value inside the original. + /// + /// [`map`]: Option::map + /// [String]: ../../std/string/struct.String.html "String" + /// + /// ``` + /// let s: Option = Some(String::from("Hello, Rustaceans!")); + /// let o: Option = Option::from(&s).map(|ss: &String| ss.len()); + /// + /// println!("Can still print s: {s:?}"); + /// + /// assert_eq!(o, Some(18)); + /// ``` fn from(o: &'a Option) -> Option<&'a T> { o.as_ref() } } #[stable(feature = "option_ref_from_ref_option", since = "1.30.0")] -impl<'a, T> From<&'a mut Option> for Option<&'a mut T> { +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl<'a, T> const From<&'a mut Option> for Option<&'a mut T> { + /// Converts from `&mut Option` to `Option<&mut T>` + /// + /// # Examples + /// + /// ``` + /// let mut s = Some(String::from("Hello")); + /// let o: Option<&mut String> = Option::from(&mut s); + /// + /// match o { + /// Some(t) => *t = String::from("Hello, Rustaceans!"), + /// None => (), + /// } + /// + /// assert_eq!(s, Some(String::from("Hello, Rustaceans!"))); + /// ``` fn from(o: &'a mut Option) -> Option<&'a mut T> { o.as_mut() } } +#[stable(feature = "rust1", since = "1.0.0")] +impl crate::marker::StructuralPartialEq for Option {} +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialEq for Option { + #[inline] + fn eq(&self, other: &Self) -> bool { + SpecOptionPartialEq::eq(self, other) + } +} + +#[unstable(feature = "spec_option_partial_eq", issue = "none", reason = "exposed only for rustc")] +#[doc(hidden)] +pub trait SpecOptionPartialEq: Sized { + fn eq(l: &Option, other: &Option) -> bool; +} + +#[unstable(feature = "spec_option_partial_eq", issue = "none", reason = "exposed only for rustc")] +impl SpecOptionPartialEq for T { + #[inline] + default fn eq(l: &Option, r: &Option) -> bool { + match (l, r) { + (Some(l), Some(r)) => *l == *r, + (None, None) => true, + _ => false, + } + } +} + +macro_rules! non_zero_option { + ( $( #[$stability: meta] $NZ:ty; )+ ) => { + $( + #[$stability] + impl SpecOptionPartialEq for $NZ { + #[inline] + fn eq(l: &Option, r: &Option) -> bool { + l.map(Self::get).unwrap_or(0) == r.map(Self::get).unwrap_or(0) + } + } + )+ + }; +} + +non_zero_option! { + #[stable(feature = "nonzero", since = "1.28.0")] crate::num::NonZeroU8; + #[stable(feature = "nonzero", since = "1.28.0")] crate::num::NonZeroU16; + #[stable(feature = "nonzero", since = "1.28.0")] crate::num::NonZeroU32; + #[stable(feature = "nonzero", since = "1.28.0")] crate::num::NonZeroU64; + #[stable(feature = "nonzero", since = "1.28.0")] crate::num::NonZeroU128; + #[stable(feature = "nonzero", since = "1.28.0")] crate::num::NonZeroUsize; + #[stable(feature = "signed_nonzero", since = "1.34.0")] crate::num::NonZeroI8; + #[stable(feature = "signed_nonzero", since = "1.34.0")] crate::num::NonZeroI16; + #[stable(feature = "signed_nonzero", since = "1.34.0")] crate::num::NonZeroI32; + #[stable(feature = "signed_nonzero", since = "1.34.0")] crate::num::NonZeroI64; + #[stable(feature = "signed_nonzero", since = "1.34.0")] crate::num::NonZeroI128; + #[stable(feature = "signed_nonzero", since = "1.34.0")] crate::num::NonZeroIsize; +} + +#[stable(feature = "nonnull", since = "1.25.0")] +impl SpecOptionPartialEq for crate::ptr::NonNull { + #[inline] + fn eq(l: &Option, r: &Option) -> bool { + l.map(Self::as_ptr).unwrap_or_else(|| crate::ptr::null_mut()) + == r.map(Self::as_ptr).unwrap_or_else(|| crate::ptr::null_mut()) + } +} + ///////////////////////////////////////////////////////////////////////////// // The Option Iterators ///////////////////////////////////////////////////////////////////////////// @@ -1414,10 +2188,6 @@ unsafe impl TrustedLen for Item {} /// The iterator yields one value if the [`Option`] is a [`Some`], otherwise none. /// /// This `struct` is created by the [`Option::iter`] function. -/// -/// [`Option`]: enum.Option.html -/// [`Some`]: enum.Option.html#variant.Some -/// [`Option::iter`]: enum.Option.html#method.iter #[stable(feature = "rust1", since = "1.0.0")] #[derive(Debug)] pub struct Iter<'a, A: 'a> { @@ -1468,10 +2238,6 @@ impl Clone for Iter<'_, A> { /// The iterator yields one value if the [`Option`] is a [`Some`], otherwise none. /// /// This `struct` is created by the [`Option::iter_mut`] function. -/// -/// [`Option`]: enum.Option.html -/// [`Some`]: enum.Option.html#variant.Some -/// [`Option::iter_mut`]: enum.Option.html#method.iter_mut #[stable(feature = "rust1", since = "1.0.0")] #[derive(Debug)] pub struct IterMut<'a, A: 'a> { @@ -1513,10 +2279,6 @@ unsafe impl TrustedLen for IterMut<'_, A> {} /// The iterator yields one value if the [`Option`] is a [`Some`], otherwise none. /// /// This `struct` is created by the [`Option::into_iter`] function. -/// -/// [`Option`]: enum.Option.html -/// [`Some`]: enum.Option.html#variant.Some -/// [`Option::into_iter`]: enum.Option.html#method.into_iter #[derive(Clone, Debug)] #[stable(feature = "rust1", since = "1.0.0")] pub struct IntoIter { @@ -1562,8 +2324,8 @@ unsafe impl TrustedLen for IntoIter {} impl> FromIterator> for Option { /// Takes each element in the [`Iterator`]: if it is [`None`][Option::None], /// no further elements are taken, and the [`None`][Option::None] is - /// returned. Should no [`None`][Option::None] occur, a container with the - /// values of each [`Option`] is returned. + /// returned. Should no [`None`][Option::None] occur, a container of type + /// `V` containing the values of each [`Option`] is returned. /// /// # Examples /// @@ -1620,51 +2382,67 @@ impl> FromIterator> for Option { /// /// Since the third element caused an underflow, no further elements were taken, /// so the final value of `shared` is 6 (= `3 + 2 + 1`), not 16. - /// - /// [`Iterator`]: ../iter/trait.Iterator.html #[inline] fn from_iter>>(iter: I) -> Option { // FIXME(#11084): This could be replaced with Iterator::scan when this // performance bug is closed. - iter.into_iter().map(|x| x.ok_or(())).collect::>().ok() + iter::try_process(iter.into_iter(), |i| i.collect()) } } -/// The error type that results from applying the try operator (`?`) to a `None` value. If you wish -/// to allow `x?` (where `x` is an `Option`) to be converted into your error type, you can -/// implement `impl From` for `YourErrorType`. In that case, `x?` within a function that -/// returns `Result<_, YourErrorType>` will translate a `None` value into an `Err` result. -#[unstable(feature = "try_trait", issue = "42327")] -#[derive(Clone, Copy, PartialEq, PartialOrd, Eq, Ord, Debug, Hash)] -pub struct NoneError; +#[unstable(feature = "try_trait_v2", issue = "84277")] +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const ops::Try for Option { + type Output = T; + type Residual = Option; -#[unstable(feature = "try_trait", issue = "42327")] -impl ops::Try for Option { - type Ok = T; - type Error = NoneError; + #[inline] + fn from_output(output: Self::Output) -> Self { + Some(output) + } #[inline] - fn into_result(self) -> Result { - self.ok_or(NoneError) + fn branch(self) -> ControlFlow { + match self { + Some(v) => ControlFlow::Continue(v), + None => ControlFlow::Break(None), + } } +} +#[unstable(feature = "try_trait_v2", issue = "84277")] +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const ops::FromResidual for Option { #[inline] - fn from_ok(v: T) -> Self { - Some(v) + fn from_residual(residual: Option) -> Self { + match residual { + None => None, + } } +} +#[unstable(feature = "try_trait_v2_yeet", issue = "96374")] +impl ops::FromResidual> for Option { #[inline] - fn from_error(_: NoneError) -> Self { + fn from_residual(ops::Yeet(()): ops::Yeet<()>) -> Self { None } } +#[unstable(feature = "try_trait_v2_residual", issue = "91285")] +#[rustc_const_unstable(feature = "const_try", issue = "74935")] +impl const ops::Residual for Option { + type TryType = Option; +} + impl Option> { - /// Converts from `Option>` to `Option` + /// Converts from `Option>` to `Option`. /// /// # Examples + /// /// Basic usage: + /// /// ``` /// let x: Option> = Some(Some(6)); /// assert_eq!(Some(6), x.flatten()); @@ -1675,7 +2453,9 @@ impl Option> { /// let x: Option> = None; /// assert_eq!(None, x.flatten()); /// ``` - /// Flattening once only removes one level of nesting: + /// + /// Flattening only removes one level of nesting at a time: + /// /// ``` /// let x: Option>> = Some(Some(Some(6))); /// assert_eq!(Some(Some(6)), x.flatten()); @@ -1683,7 +2463,11 @@ impl Option> { /// ``` #[inline] #[stable(feature = "option_flattening", since = "1.40.0")] - pub fn flatten(self) -> Option { - self.and_then(convert::identity) + #[rustc_const_unstable(feature = "const_option", issue = "67441")] + pub const fn flatten(self) -> Option { + match self { + Some(inner) => inner, + None => None, + } } } diff --git a/crux-mir/lib/core/src/panic.rs b/crux-mir/lib/core/src/panic.rs index dbfcbca3f..8338a5d7e 100644 --- a/crux-mir/lib/core/src/panic.rs +++ b/crux-mir/lib/core/src/panic.rs @@ -2,331 +2,102 @@ #![stable(feature = "core_panic_info", since = "1.41.0")] +mod location; +mod panic_info; +mod unwind_safe; + use crate::any::Any; -use crate::fmt; -/// A struct providing information about a panic. -/// -/// `PanicInfo` structure is passed to a panic hook set by the [`set_hook`] -/// function. -/// -/// [`set_hook`]: ../../std/panic/fn.set_hook.html -/// -/// # Examples -/// -/// ```should_panic -/// use std::panic; -/// -/// panic::set_hook(Box::new(|panic_info| { -/// if let Some(s) = panic_info.payload().downcast_ref::<&str>() { -/// println!("panic occurred: {:?}", s); -/// } else { -/// println!("panic occurred"); -/// } -/// })); -/// -/// panic!("Normal panic"); -/// ``` -#[lang = "panic_info"] #[stable(feature = "panic_hooks", since = "1.10.0")] -#[derive(Debug)] -pub struct PanicInfo<'a> { - payload: &'a (dyn Any + Send), - message: Option<&'a fmt::Arguments<'a>>, - location: &'a Location<'a>, -} - -impl<'a> PanicInfo<'a> { - #[unstable( - feature = "panic_internals", - reason = "internal details of the implementation of the `panic!` \ - and related macros", - issue = "none" - )] - #[doc(hidden)] - #[inline] - pub fn internal_constructor( - message: Option<&'a fmt::Arguments<'a>>, - location: &'a Location<'a>, - ) -> Self { - struct NoPayload; - PanicInfo { location, message, payload: &NoPayload } - } - - #[unstable( - feature = "panic_internals", - reason = "internal details of the implementation of the `panic!` \ - and related macros", - issue = "none" - )] - #[doc(hidden)] - #[inline] - pub fn set_payload(&mut self, info: &'a (dyn Any + Send)) { - self.payload = info; - } - - /// Returns the payload associated with the panic. - /// - /// This will commonly, but not always, be a `&'static str` or [`String`]. - /// - /// [`String`]: ../../std/string/struct.String.html - /// - /// # Examples - /// - /// ```should_panic - /// use std::panic; - /// - /// panic::set_hook(Box::new(|panic_info| { - /// println!("panic occurred: {:?}", panic_info.payload().downcast_ref::<&str>().unwrap()); - /// })); - /// - /// panic!("Normal panic"); - /// ``` - #[stable(feature = "panic_hooks", since = "1.10.0")] - pub fn payload(&self) -> &(dyn Any + Send) { - self.payload - } - - /// If the `panic!` macro from the `core` crate (not from `std`) - /// was used with a formatting string and some additional arguments, - /// returns that message ready to be used for example with [`fmt::write`] - /// - /// [`fmt::write`]: ../fmt/fn.write.html - #[unstable(feature = "panic_info_message", issue = "66745")] - pub fn message(&self) -> Option<&fmt::Arguments<'_>> { - self.message - } - - /// Returns information about the location from which the panic originated, - /// if available. - /// - /// This method will currently always return [`Some`], but this may change - /// in future versions. - /// - /// [`Some`]: ../../std/option/enum.Option.html#variant.Some - /// - /// # Examples - /// - /// ```should_panic - /// use std::panic; - /// - /// panic::set_hook(Box::new(|panic_info| { - /// if let Some(location) = panic_info.location() { - /// println!("panic occurred in file '{}' at line {}", location.file(), - /// location.line()); - /// } else { - /// println!("panic occurred but can't get location information..."); - /// } - /// })); - /// - /// panic!("Normal panic"); - /// ``` - #[stable(feature = "panic_hooks", since = "1.10.0")] - pub fn location(&self) -> Option<&Location<'_>> { - // NOTE: If this is changed to sometimes return None, - // deal with that case in std::panicking::default_hook and std::panicking::begin_panic_fmt. - Some(&self.location) - } -} - -#[stable(feature = "panic_hook_display", since = "1.26.0")] -impl fmt::Display for PanicInfo<'_> { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("panicked at ")?; - if let Some(message) = self.message { - write!(formatter, "'{}', ", message)? - } else if let Some(payload) = self.payload.downcast_ref::<&'static str>() { - write!(formatter, "'{}', ", payload)? - } - // NOTE: we cannot use downcast_ref::() here - // since String is not available in libcore! - // The payload is a String when `std::panic!` is called with multiple arguments, - // but in that case the message is also available. - - self.location.fmt(formatter) - } -} - -/// A struct containing information about the location of a panic. -/// -/// This structure is created by the [`location`] method of [`PanicInfo`]. -/// -/// [`location`]: ../../std/panic/struct.PanicInfo.html#method.location -/// [`PanicInfo`]: ../../std/panic/struct.PanicInfo.html -/// -/// # Examples -/// -/// ```should_panic -/// use std::panic; -/// -/// panic::set_hook(Box::new(|panic_info| { -/// if let Some(location) = panic_info.location() { -/// println!("panic occurred in file '{}' at line {}", location.file(), location.line()); -/// } else { -/// println!("panic occurred but can't get location information..."); -/// } -/// })); -/// -/// panic!("Normal panic"); -/// ``` -#[lang = "panic_location"] -#[derive(Debug)] +pub use self::location::Location; #[stable(feature = "panic_hooks", since = "1.10.0")] -pub struct Location<'a> { - file: &'a str, - line: u32, - col: u32, -} +pub use self::panic_info::PanicInfo; +#[stable(feature = "catch_unwind", since = "1.9.0")] +pub use self::unwind_safe::{AssertUnwindSafe, RefUnwindSafe, UnwindSafe}; -impl<'a> Location<'a> { - /// Returns the source location of the caller of this function. If that function's caller is - /// annotated then its call location will be returned, and so on up the stack to the first call - /// within a non-tracked function body. - /// - /// # Examples - /// - /// ``` - /// #![feature(track_caller)] - /// use core::panic::Location; - /// - /// /// Returns the [`Location`] at which it is called. - /// #[track_caller] - /// fn get_caller_location() -> &'static Location<'static> { - /// Location::caller() - /// } - /// - /// /// Returns a [`Location`] from within this function's definition. - /// fn get_just_one_location() -> &'static Location<'static> { - /// get_caller_location() - /// } - /// - /// let fixed_location = get_just_one_location(); - /// assert_eq!(fixed_location.file(), file!()); - /// assert_eq!(fixed_location.line(), 15); - /// assert_eq!(fixed_location.column(), 5); - /// - /// // running the same untracked function in a different location gives us the same result - /// let second_fixed_location = get_just_one_location(); - /// assert_eq!(fixed_location.file(), second_fixed_location.file()); - /// assert_eq!(fixed_location.line(), second_fixed_location.line()); - /// assert_eq!(fixed_location.column(), second_fixed_location.column()); - /// - /// let this_location = get_caller_location(); - /// assert_eq!(this_location.file(), file!()); - /// assert_eq!(this_location.line(), 29); - /// assert_eq!(this_location.column(), 21); - /// - /// // running the tracked function in a different location produces a different value - /// let another_location = get_caller_location(); - /// assert_eq!(this_location.file(), another_location.file()); - /// assert_ne!(this_location.line(), another_location.line()); - /// assert_ne!(this_location.column(), another_location.column()); - /// ``` - #[unstable( - feature = "track_caller", - reason = "uses #[track_caller] which is not yet stable", - issue = "47809" - )] - #[track_caller] - pub const fn caller() -> &'static Location<'static> { - crate::intrinsics::caller_location() - } +#[doc(hidden)] +#[unstable(feature = "edition_panic", issue = "none", reason = "use panic!() instead")] +#[allow_internal_unstable(core_panic, const_format_args)] +#[rustc_diagnostic_item = "core_panic_2015_macro"] +#[rustc_macro_transparency = "semitransparent"] +pub macro panic_2015 { + () => ( + $crate::panicking::panic("explicit panic") + ), + ($msg:literal $(,)?) => ( + $crate::panicking::panic($msg) + ), + // Use `panic_str` instead of `panic_display::<&str>` for non_fmt_panic lint. + ($msg:expr $(,)?) => ( + $crate::panicking::panic_str($msg) + ), + // Special-case the single-argument case for const_panic. + ("{}", $arg:expr $(,)?) => ( + $crate::panicking::panic_display(&$arg) + ), + ($fmt:expr, $($arg:tt)+) => ( + $crate::panicking::panic_fmt($crate::const_format_args!($fmt, $($arg)+)) + ), } -impl<'a> Location<'a> { - #![unstable( - feature = "panic_internals", - reason = "internal details of the implementation of the `panic!` \ - and related macros", - issue = "none" - )] - #[doc(hidden)] - pub const fn internal_constructor(file: &'a str, line: u32, col: u32) -> Self { - Location { file, line, col } - } - - /// Returns the name of the source file from which the panic originated. - /// - /// # Examples - /// - /// ```should_panic - /// use std::panic; - /// - /// panic::set_hook(Box::new(|panic_info| { - /// if let Some(location) = panic_info.location() { - /// println!("panic occurred in file '{}'", location.file()); - /// } else { - /// println!("panic occurred but can't get location information..."); - /// } - /// })); - /// - /// panic!("Normal panic"); - /// ``` - #[stable(feature = "panic_hooks", since = "1.10.0")] - pub fn file(&self) -> &str { - self.file - } - - /// Returns the line number from which the panic originated. - /// - /// # Examples - /// - /// ```should_panic - /// use std::panic; - /// - /// panic::set_hook(Box::new(|panic_info| { - /// if let Some(location) = panic_info.location() { - /// println!("panic occurred at line {}", location.line()); - /// } else { - /// println!("panic occurred but can't get location information..."); - /// } - /// })); - /// - /// panic!("Normal panic"); - /// ``` - #[stable(feature = "panic_hooks", since = "1.10.0")] - pub fn line(&self) -> u32 { - self.line - } +#[doc(hidden)] +#[unstable(feature = "edition_panic", issue = "none", reason = "use panic!() instead")] +#[allow_internal_unstable(core_panic, const_format_args)] +#[rustc_diagnostic_item = "core_panic_2021_macro"] +#[rustc_macro_transparency = "semitransparent"] +pub macro panic_2021 { + () => ( + $crate::panicking::panic("explicit panic") + ), + // Special-case the single-argument case for const_panic. + ("{}", $arg:expr $(,)?) => ( + $crate::panicking::panic_display(&$arg) + ), + ($($t:tt)+) => ( + $crate::panicking::panic_fmt($crate::const_format_args!($($t)+)) + ), +} - /// Returns the column from which the panic originated. - /// - /// # Examples - /// - /// ```should_panic - /// use std::panic; - /// - /// panic::set_hook(Box::new(|panic_info| { - /// if let Some(location) = panic_info.location() { - /// println!("panic occurred at column {}", location.column()); - /// } else { - /// println!("panic occurred but can't get location information..."); - /// } - /// })); - /// - /// panic!("Normal panic"); - /// ``` - #[stable(feature = "panic_col", since = "1.25.0")] - pub fn column(&self) -> u32 { - self.col - } +#[doc(hidden)] +#[unstable(feature = "edition_panic", issue = "none", reason = "use unreachable!() instead")] +#[allow_internal_unstable(core_panic)] +#[rustc_diagnostic_item = "unreachable_2015_macro"] +#[rustc_macro_transparency = "semitransparent"] +pub macro unreachable_2015 { + () => ( + $crate::panicking::panic("internal error: entered unreachable code") + ), + // Use of `unreachable_display` for non_fmt_panic lint. + // NOTE: the message ("internal error ...") is embedded directly in unreachable_display + ($msg:expr $(,)?) => ( + $crate::panicking::unreachable_display(&$msg) + ), + ($fmt:expr, $($arg:tt)*) => ( + $crate::panic!($crate::concat!("internal error: entered unreachable code: ", $fmt), $($arg)*) + ), } -#[stable(feature = "panic_hook_display", since = "1.26.0")] -impl fmt::Display for Location<'_> { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(formatter, "{}:{}:{}", self.file, self.line, self.col) - } +#[doc(hidden)] +#[unstable(feature = "edition_panic", issue = "none", reason = "use unreachable!() instead")] +#[allow_internal_unstable(core_panic)] +#[rustc_macro_transparency = "semitransparent"] +pub macro unreachable_2021 { + () => ( + $crate::panicking::panic("internal error: entered unreachable code") + ), + ($($t:tt)+) => ( + $crate::panic!("internal error: entered unreachable code: {}", $crate::format_args!($($t)+)) + ), } -/// An internal trait used by libstd to pass data from libstd to `panic_unwind` -/// and other panic runtimes. Not intended to be stabilized any time soon, do -/// not use. +/// An internal trait used by std to pass data from std to `panic_unwind` and +/// other panic runtimes. Not intended to be stabilized any time soon, do not +/// use. #[unstable(feature = "std_internals", issue = "none")] #[doc(hidden)] pub unsafe trait BoxMeUp { /// Take full ownership of the contents. - /// The return type is actually `Box`, but we cannot use `Box` in libcore. + /// The return type is actually `Box`, but we cannot use `Box` in core. /// /// After this method got called, only some dummy default value is left in `self`. /// Calling this method twice, or calling `get` after calling this method, is an error. diff --git a/crux-mir/lib/core/src/panic/location.rs b/crux-mir/lib/core/src/panic/location.rs new file mode 100644 index 000000000..6dcf23dde --- /dev/null +++ b/crux-mir/lib/core/src/panic/location.rs @@ -0,0 +1,200 @@ +use crate::fmt; + +/// A struct containing information about the location of a panic. +/// +/// This structure is created by [`PanicInfo::location()`]. +/// +/// [`PanicInfo::location()`]: crate::panic::PanicInfo::location +/// +/// # Examples +/// +/// ```should_panic +/// use std::panic; +/// +/// panic::set_hook(Box::new(|panic_info| { +/// if let Some(location) = panic_info.location() { +/// println!("panic occurred in file '{}' at line {}", location.file(), location.line()); +/// } else { +/// println!("panic occurred but can't get location information..."); +/// } +/// })); +/// +/// panic!("Normal panic"); +/// ``` +/// +/// # Comparisons +/// +/// Comparisons for equality and ordering are made in file, line, then column priority. +/// Files are compared as strings, not `Path`, which could be unexpected. +/// See [`Location::file`]'s documentation for more discussion. +#[lang = "panic_location"] +#[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[stable(feature = "panic_hooks", since = "1.10.0")] +pub struct Location<'a> { + file: &'a str, + line: u32, + col: u32, +} + +impl<'a> Location<'a> { + /// Returns the source location of the caller of this function. If that function's caller is + /// annotated then its call location will be returned, and so on up the stack to the first call + /// within a non-tracked function body. + /// + /// # Examples + /// + /// ``` + /// use std::panic::Location; + /// + /// /// Returns the [`Location`] at which it is called. + /// #[track_caller] + /// fn get_caller_location() -> &'static Location<'static> { + /// Location::caller() + /// } + /// + /// /// Returns a [`Location`] from within this function's definition. + /// fn get_just_one_location() -> &'static Location<'static> { + /// get_caller_location() + /// } + /// + /// let fixed_location = get_just_one_location(); + /// assert_eq!(fixed_location.file(), file!()); + /// assert_eq!(fixed_location.line(), 14); + /// assert_eq!(fixed_location.column(), 5); + /// + /// // running the same untracked function in a different location gives us the same result + /// let second_fixed_location = get_just_one_location(); + /// assert_eq!(fixed_location.file(), second_fixed_location.file()); + /// assert_eq!(fixed_location.line(), second_fixed_location.line()); + /// assert_eq!(fixed_location.column(), second_fixed_location.column()); + /// + /// let this_location = get_caller_location(); + /// assert_eq!(this_location.file(), file!()); + /// assert_eq!(this_location.line(), 28); + /// assert_eq!(this_location.column(), 21); + /// + /// // running the tracked function in a different location produces a different value + /// let another_location = get_caller_location(); + /// assert_eq!(this_location.file(), another_location.file()); + /// assert_ne!(this_location.line(), another_location.line()); + /// assert_ne!(this_location.column(), another_location.column()); + /// ``` + #[must_use] + #[stable(feature = "track_caller", since = "1.46.0")] + #[rustc_const_unstable(feature = "const_caller_location", issue = "76156")] + #[track_caller] + #[inline] + pub const fn caller() -> &'static Location<'static> { + crate::intrinsics::caller_location() + } + + /// Returns the name of the source file from which the panic originated. + /// + /// # `&str`, not `&Path` + /// + /// The returned name refers to a source path on the compiling system, but it isn't valid to + /// represent this directly as a `&Path`. The compiled code may run on a different system with + /// a different `Path` implementation than the system providing the contents and this library + /// does not currently have a different "host path" type. + /// + /// The most surprising behavior occurs when "the same" file is reachable via multiple paths in + /// the module system (usually using the `#[path = "..."]` attribute or similar), which can + /// cause what appears to be identical code to return differing values from this function. + /// + /// # Cross-compilation + /// + /// This value is not suitable for passing to `Path::new` or similar constructors when the host + /// platform and target platform differ. + /// + /// # Examples + /// + /// ```should_panic + /// use std::panic; + /// + /// panic::set_hook(Box::new(|panic_info| { + /// if let Some(location) = panic_info.location() { + /// println!("panic occurred in file '{}'", location.file()); + /// } else { + /// println!("panic occurred but can't get location information..."); + /// } + /// })); + /// + /// panic!("Normal panic"); + /// ``` + #[must_use] + #[stable(feature = "panic_hooks", since = "1.10.0")] + #[rustc_const_unstable(feature = "const_location_fields", issue = "102911")] + #[inline] + pub const fn file(&self) -> &str { + self.file + } + + /// Returns the line number from which the panic originated. + /// + /// # Examples + /// + /// ```should_panic + /// use std::panic; + /// + /// panic::set_hook(Box::new(|panic_info| { + /// if let Some(location) = panic_info.location() { + /// println!("panic occurred at line {}", location.line()); + /// } else { + /// println!("panic occurred but can't get location information..."); + /// } + /// })); + /// + /// panic!("Normal panic"); + /// ``` + #[must_use] + #[stable(feature = "panic_hooks", since = "1.10.0")] + #[rustc_const_unstable(feature = "const_location_fields", issue = "102911")] + #[inline] + pub const fn line(&self) -> u32 { + self.line + } + + /// Returns the column from which the panic originated. + /// + /// # Examples + /// + /// ```should_panic + /// use std::panic; + /// + /// panic::set_hook(Box::new(|panic_info| { + /// if let Some(location) = panic_info.location() { + /// println!("panic occurred at column {}", location.column()); + /// } else { + /// println!("panic occurred but can't get location information..."); + /// } + /// })); + /// + /// panic!("Normal panic"); + /// ``` + #[must_use] + #[stable(feature = "panic_col", since = "1.25.0")] + #[rustc_const_unstable(feature = "const_location_fields", issue = "102911")] + #[inline] + pub const fn column(&self) -> u32 { + self.col + } +} + +#[unstable( + feature = "panic_internals", + reason = "internal details of the implementation of the `panic!` and related macros", + issue = "none" +)] +impl<'a> Location<'a> { + #[doc(hidden)] + pub const fn internal_constructor(file: &'a str, line: u32, col: u32) -> Self { + Location { file, line, col } + } +} + +#[stable(feature = "panic_hook_display", since = "1.26.0")] +impl fmt::Display for Location<'_> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{}:{}:{}", self.file, self.line, self.col) + } +} diff --git a/crux-mir/lib/core/src/panic/panic_info.rs b/crux-mir/lib/core/src/panic/panic_info.rs new file mode 100644 index 000000000..0d385c9d1 --- /dev/null +++ b/crux-mir/lib/core/src/panic/panic_info.rs @@ -0,0 +1,166 @@ +use crate::any::Any; +use crate::fmt; +use crate::panic::Location; + +/// A struct providing information about a panic. +/// +/// `PanicInfo` structure is passed to a panic hook set by the [`set_hook`] +/// function. +/// +/// [`set_hook`]: ../../std/panic/fn.set_hook.html +/// +/// # Examples +/// +/// ```should_panic +/// use std::panic; +/// +/// panic::set_hook(Box::new(|panic_info| { +/// if let Some(s) = panic_info.payload().downcast_ref::<&str>() { +/// println!("panic occurred: {s:?}"); +/// } else { +/// println!("panic occurred"); +/// } +/// })); +/// +/// panic!("Normal panic"); +/// ``` +#[lang = "panic_info"] +#[stable(feature = "panic_hooks", since = "1.10.0")] +#[derive(Debug)] +pub struct PanicInfo<'a> { + payload: &'a (dyn Any + Send), + message: Option<&'a fmt::Arguments<'a>>, + location: &'a Location<'a>, + can_unwind: bool, +} + +impl<'a> PanicInfo<'a> { + #[unstable( + feature = "panic_internals", + reason = "internal details of the implementation of the `panic!` and related macros", + issue = "none" + )] + #[doc(hidden)] + #[inline] + pub fn internal_constructor( + message: Option<&'a fmt::Arguments<'a>>, + location: &'a Location<'a>, + can_unwind: bool, + ) -> Self { + struct NoPayload; + PanicInfo { location, message, payload: &NoPayload, can_unwind } + } + + #[unstable( + feature = "panic_internals", + reason = "internal details of the implementation of the `panic!` and related macros", + issue = "none" + )] + #[doc(hidden)] + #[inline] + pub fn set_payload(&mut self, info: &'a (dyn Any + Send)) { + self.payload = info; + } + + /// Returns the payload associated with the panic. + /// + /// This will commonly, but not always, be a `&'static str` or [`String`]. + /// + /// [`String`]: ../../std/string/struct.String.html + /// + /// # Examples + /// + /// ```should_panic + /// use std::panic; + /// + /// panic::set_hook(Box::new(|panic_info| { + /// if let Some(s) = panic_info.payload().downcast_ref::<&str>() { + /// println!("panic occurred: {s:?}"); + /// } else { + /// println!("panic occurred"); + /// } + /// })); + /// + /// panic!("Normal panic"); + /// ``` + #[must_use] + #[stable(feature = "panic_hooks", since = "1.10.0")] + pub fn payload(&self) -> &(dyn Any + Send) { + self.payload + } + + /// If the `panic!` macro from the `core` crate (not from `std`) + /// was used with a formatting string and some additional arguments, + /// returns that message ready to be used for example with [`fmt::write`] + #[must_use] + #[unstable(feature = "panic_info_message", issue = "66745")] + pub fn message(&self) -> Option<&fmt::Arguments<'_>> { + self.message + } + + /// Returns information about the location from which the panic originated, + /// if available. + /// + /// This method will currently always return [`Some`], but this may change + /// in future versions. + /// + /// # Examples + /// + /// ```should_panic + /// use std::panic; + /// + /// panic::set_hook(Box::new(|panic_info| { + /// if let Some(location) = panic_info.location() { + /// println!("panic occurred in file '{}' at line {}", + /// location.file(), + /// location.line(), + /// ); + /// } else { + /// println!("panic occurred but can't get location information..."); + /// } + /// })); + /// + /// panic!("Normal panic"); + /// ``` + #[must_use] + #[stable(feature = "panic_hooks", since = "1.10.0")] + pub fn location(&self) -> Option<&Location<'_>> { + // NOTE: If this is changed to sometimes return None, + // deal with that case in std::panicking::default_hook and core::panicking::panic_fmt. + Some(&self.location) + } + + /// Returns whether the panic handler is allowed to unwind the stack from + /// the point where the panic occurred. + /// + /// This is true for most kinds of panics with the exception of panics + /// caused by trying to unwind out of a `Drop` implementation or a function + /// whose ABI does not support unwinding. + /// + /// It is safe for a panic handler to unwind even when this function returns + /// true, however this will simply cause the panic handler to be called + /// again. + #[must_use] + #[unstable(feature = "panic_can_unwind", issue = "92988")] + pub fn can_unwind(&self) -> bool { + self.can_unwind + } +} + +#[stable(feature = "panic_hook_display", since = "1.26.0")] +impl fmt::Display for PanicInfo<'_> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("panicked at ")?; + if let Some(message) = self.message { + write!(formatter, "'{}', ", message)? + } else if let Some(payload) = self.payload.downcast_ref::<&'static str>() { + write!(formatter, "'{}', ", payload)? + } + // NOTE: we cannot use downcast_ref::() here + // since String is not available in core! + // The payload is a String when `std::panic!` is called with multiple arguments, + // but in that case the message is also available. + + self.location.fmt(formatter) + } +} diff --git a/crux-mir/lib/core/src/panic/unwind_safe.rs b/crux-mir/lib/core/src/panic/unwind_safe.rs new file mode 100644 index 000000000..9a6153f12 --- /dev/null +++ b/crux-mir/lib/core/src/panic/unwind_safe.rs @@ -0,0 +1,312 @@ +use crate::async_iter::AsyncIterator; +use crate::cell::UnsafeCell; +use crate::fmt; +use crate::future::Future; +use crate::ops::{Deref, DerefMut}; +use crate::pin::Pin; +use crate::ptr::{NonNull, Unique}; +use crate::task::{Context, Poll}; + +/// A marker trait which represents "panic safe" types in Rust. +/// +/// This trait is implemented by default for many types and behaves similarly in +/// terms of inference of implementation to the [`Send`] and [`Sync`] traits. The +/// purpose of this trait is to encode what types are safe to cross a [`catch_unwind`] +/// boundary with no fear of unwind safety. +/// +/// [`catch_unwind`]: ../../std/panic/fn.catch_unwind.html +/// +/// ## What is unwind safety? +/// +/// In Rust a function can "return" early if it either panics or calls a +/// function which transitively panics. This sort of control flow is not always +/// anticipated, and has the possibility of causing subtle bugs through a +/// combination of two critical components: +/// +/// 1. A data structure is in a temporarily invalid state when the thread +/// panics. +/// 2. This broken invariant is then later observed. +/// +/// Typically in Rust, it is difficult to perform step (2) because catching a +/// panic involves either spawning a thread (which in turns makes it difficult +/// to later witness broken invariants) or using the `catch_unwind` function in this +/// module. Additionally, even if an invariant is witnessed, it typically isn't a +/// problem in Rust because there are no uninitialized values (like in C or C++). +/// +/// It is possible, however, for **logical** invariants to be broken in Rust, +/// which can end up causing behavioral bugs. Another key aspect of unwind safety +/// in Rust is that, in the absence of `unsafe` code, a panic cannot lead to +/// memory unsafety. +/// +/// That was a bit of a whirlwind tour of unwind safety, but for more information +/// about unwind safety and how it applies to Rust, see an [associated RFC][rfc]. +/// +/// [rfc]: https://github.com/rust-lang/rfcs/blob/master/text/1236-stabilize-catch-panic.md +/// +/// ## What is `UnwindSafe`? +/// +/// Now that we've got an idea of what unwind safety is in Rust, it's also +/// important to understand what this trait represents. As mentioned above, one +/// way to witness broken invariants is through the `catch_unwind` function in this +/// module as it allows catching a panic and then re-using the environment of +/// the closure. +/// +/// Simply put, a type `T` implements `UnwindSafe` if it cannot easily allow +/// witnessing a broken invariant through the use of `catch_unwind` (catching a +/// panic). This trait is an auto trait, so it is automatically implemented for +/// many types, and it is also structurally composed (e.g., a struct is unwind +/// safe if all of its components are unwind safe). +/// +/// Note, however, that this is not an unsafe trait, so there is not a succinct +/// contract that this trait is providing. Instead it is intended as more of a +/// "speed bump" to alert users of `catch_unwind` that broken invariants may be +/// witnessed and may need to be accounted for. +/// +/// ## Who implements `UnwindSafe`? +/// +/// Types such as `&mut T` and `&RefCell` are examples which are **not** +/// unwind safe. The general idea is that any mutable state which can be shared +/// across `catch_unwind` is not unwind safe by default. This is because it is very +/// easy to witness a broken invariant outside of `catch_unwind` as the data is +/// simply accessed as usual. +/// +/// Types like `&Mutex`, however, are unwind safe because they implement +/// poisoning by default. They still allow witnessing a broken invariant, but +/// they already provide their own "speed bumps" to do so. +/// +/// ## When should `UnwindSafe` be used? +/// +/// It is not intended that most types or functions need to worry about this trait. +/// It is only used as a bound on the `catch_unwind` function and as mentioned +/// above, the lack of `unsafe` means it is mostly an advisory. The +/// [`AssertUnwindSafe`] wrapper struct can be used to force this trait to be +/// implemented for any closed over variables passed to `catch_unwind`. +#[stable(feature = "catch_unwind", since = "1.9.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "unwind_safe_trait")] +#[rustc_on_unimplemented( + message = "the type `{Self}` may not be safely transferred across an unwind boundary", + label = "`{Self}` may not be safely transferred across an unwind boundary" +)] +pub auto trait UnwindSafe {} + +/// A marker trait representing types where a shared reference is considered +/// unwind safe. +/// +/// This trait is namely not implemented by [`UnsafeCell`], the root of all +/// interior mutability. +/// +/// This is a "helper marker trait" used to provide impl blocks for the +/// [`UnwindSafe`] trait, for more information see that documentation. +#[stable(feature = "catch_unwind", since = "1.9.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "ref_unwind_safe_trait")] +#[rustc_on_unimplemented( + message = "the type `{Self}` may contain interior mutability and a reference may not be safely \ + transferrable across a catch_unwind boundary", + label = "`{Self}` may contain interior mutability and a reference may not be safely \ + transferrable across a catch_unwind boundary" +)] +pub auto trait RefUnwindSafe {} + +/// A simple wrapper around a type to assert that it is unwind safe. +/// +/// When using [`catch_unwind`] it may be the case that some of the closed over +/// variables are not unwind safe. For example if `&mut T` is captured the +/// compiler will generate a warning indicating that it is not unwind safe. It +/// might not be the case, however, that this is actually a problem due to the +/// specific usage of [`catch_unwind`] if unwind safety is specifically taken into +/// account. This wrapper struct is useful for a quick and lightweight +/// annotation that a variable is indeed unwind safe. +/// +/// [`catch_unwind`]: ../../std/panic/fn.catch_unwind.html +/// +/// # Examples +/// +/// One way to use `AssertUnwindSafe` is to assert that the entire closure +/// itself is unwind safe, bypassing all checks for all variables: +/// +/// ``` +/// use std::panic::{self, AssertUnwindSafe}; +/// +/// let mut variable = 4; +/// +/// // This code will not compile because the closure captures `&mut variable` +/// // which is not considered unwind safe by default. +/// +/// // panic::catch_unwind(|| { +/// // variable += 3; +/// // }); +/// +/// // This, however, will compile due to the `AssertUnwindSafe` wrapper +/// let result = panic::catch_unwind(AssertUnwindSafe(|| { +/// variable += 3; +/// })); +/// // ... +/// ``` +/// +/// Wrapping the entire closure amounts to a blanket assertion that all captured +/// variables are unwind safe. This has the downside that if new captures are +/// added in the future, they will also be considered unwind safe. Therefore, +/// you may prefer to just wrap individual captures, as shown below. This is +/// more annotation, but it ensures that if a new capture is added which is not +/// unwind safe, you will get a compilation error at that time, which will +/// allow you to consider whether that new capture in fact represent a bug or +/// not. +/// +/// ``` +/// use std::panic::{self, AssertUnwindSafe}; +/// +/// let mut variable = 4; +/// let other_capture = 3; +/// +/// let result = { +/// let mut wrapper = AssertUnwindSafe(&mut variable); +/// panic::catch_unwind(move || { +/// **wrapper += other_capture; +/// }) +/// }; +/// // ... +/// ``` +#[stable(feature = "catch_unwind", since = "1.9.0")] +pub struct AssertUnwindSafe(#[stable(feature = "catch_unwind", since = "1.9.0")] pub T); + +// Implementations of the `UnwindSafe` trait: +// +// * By default everything is unwind safe +// * pointers T contains mutability of some form are not unwind safe +// * Unique, an owning pointer, lifts an implementation +// * Types like Mutex/RwLock which are explicitly poisoned are unwind safe +// * Our custom AssertUnwindSafe wrapper is indeed unwind safe + +#[stable(feature = "catch_unwind", since = "1.9.0")] +impl !UnwindSafe for &mut T {} +#[stable(feature = "catch_unwind", since = "1.9.0")] +impl UnwindSafe for &T {} +#[stable(feature = "catch_unwind", since = "1.9.0")] +impl UnwindSafe for *const T {} +#[stable(feature = "catch_unwind", since = "1.9.0")] +impl UnwindSafe for *mut T {} +#[unstable(feature = "ptr_internals", issue = "none")] +impl UnwindSafe for Unique {} +#[stable(feature = "nonnull", since = "1.25.0")] +impl UnwindSafe for NonNull {} +#[stable(feature = "catch_unwind", since = "1.9.0")] +impl UnwindSafe for AssertUnwindSafe {} + +// Pretty simple implementations for the `RefUnwindSafe` marker trait, +// basically just saying that `UnsafeCell` is the +// only thing which doesn't implement it (which then transitively applies to +// everything else). +#[stable(feature = "catch_unwind", since = "1.9.0")] +impl !RefUnwindSafe for UnsafeCell {} +#[stable(feature = "catch_unwind", since = "1.9.0")] +impl RefUnwindSafe for AssertUnwindSafe {} + +#[cfg(target_has_atomic_load_store = "ptr")] +#[stable(feature = "unwind_safe_atomic_refs", since = "1.14.0")] +impl RefUnwindSafe for crate::sync::atomic::AtomicIsize {} +#[cfg(target_has_atomic_load_store = "8")] +#[stable(feature = "integer_atomics_stable", since = "1.34.0")] +impl RefUnwindSafe for crate::sync::atomic::AtomicI8 {} +#[cfg(target_has_atomic_load_store = "16")] +#[stable(feature = "integer_atomics_stable", since = "1.34.0")] +impl RefUnwindSafe for crate::sync::atomic::AtomicI16 {} +#[cfg(target_has_atomic_load_store = "32")] +#[stable(feature = "integer_atomics_stable", since = "1.34.0")] +impl RefUnwindSafe for crate::sync::atomic::AtomicI32 {} +#[cfg(target_has_atomic_load_store = "64")] +#[stable(feature = "integer_atomics_stable", since = "1.34.0")] +impl RefUnwindSafe for crate::sync::atomic::AtomicI64 {} +#[cfg(target_has_atomic_load_store = "128")] +#[unstable(feature = "integer_atomics", issue = "99069")] +impl RefUnwindSafe for crate::sync::atomic::AtomicI128 {} + +#[cfg(target_has_atomic_load_store = "ptr")] +#[stable(feature = "unwind_safe_atomic_refs", since = "1.14.0")] +impl RefUnwindSafe for crate::sync::atomic::AtomicUsize {} +#[cfg(target_has_atomic_load_store = "8")] +#[stable(feature = "integer_atomics_stable", since = "1.34.0")] +impl RefUnwindSafe for crate::sync::atomic::AtomicU8 {} +#[cfg(target_has_atomic_load_store = "16")] +#[stable(feature = "integer_atomics_stable", since = "1.34.0")] +impl RefUnwindSafe for crate::sync::atomic::AtomicU16 {} +#[cfg(target_has_atomic_load_store = "32")] +#[stable(feature = "integer_atomics_stable", since = "1.34.0")] +impl RefUnwindSafe for crate::sync::atomic::AtomicU32 {} +#[cfg(target_has_atomic_load_store = "64")] +#[stable(feature = "integer_atomics_stable", since = "1.34.0")] +impl RefUnwindSafe for crate::sync::atomic::AtomicU64 {} +#[cfg(target_has_atomic_load_store = "128")] +#[unstable(feature = "integer_atomics", issue = "99069")] +impl RefUnwindSafe for crate::sync::atomic::AtomicU128 {} + +#[cfg(target_has_atomic_load_store = "8")] +#[stable(feature = "unwind_safe_atomic_refs", since = "1.14.0")] +impl RefUnwindSafe for crate::sync::atomic::AtomicBool {} + +#[cfg(target_has_atomic_load_store = "ptr")] +#[stable(feature = "unwind_safe_atomic_refs", since = "1.14.0")] +impl RefUnwindSafe for crate::sync::atomic::AtomicPtr {} + +#[stable(feature = "catch_unwind", since = "1.9.0")] +impl Deref for AssertUnwindSafe { + type Target = T; + + fn deref(&self) -> &T { + &self.0 + } +} + +#[stable(feature = "catch_unwind", since = "1.9.0")] +impl DerefMut for AssertUnwindSafe { + fn deref_mut(&mut self) -> &mut T { + &mut self.0 + } +} + +#[stable(feature = "catch_unwind", since = "1.9.0")] +impl R> FnOnce<()> for AssertUnwindSafe { + type Output = R; + + extern "rust-call" fn call_once(self, _args: ()) -> R { + (self.0)() + } +} + +#[stable(feature = "std_debug", since = "1.16.0")] +impl fmt::Debug for AssertUnwindSafe { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("AssertUnwindSafe").field(&self.0).finish() + } +} + +#[stable(feature = "assertunwindsafe_default", since = "1.62.0")] +impl Default for AssertUnwindSafe { + fn default() -> Self { + Self(Default::default()) + } +} + +#[stable(feature = "futures_api", since = "1.36.0")] +impl Future for AssertUnwindSafe { + type Output = F::Output; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + // SAFETY: pin projection. AssertUnwindSafe follows structural pinning. + let pinned_field = unsafe { Pin::map_unchecked_mut(self, |x| &mut x.0) }; + F::poll(pinned_field, cx) + } +} + +#[unstable(feature = "async_iterator", issue = "79024")] +impl AsyncIterator for AssertUnwindSafe { + type Item = S::Item; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + // SAFETY: pin projection. AssertUnwindSafe follows structural pinning. + unsafe { self.map_unchecked_mut(|x| &mut x.0) }.poll_next(cx) + } + + fn size_hint(&self) -> (usize, Option) { + self.0.size_hint() + } +} diff --git a/crux-mir/lib/core/src/panicking.rs b/crux-mir/lib/core/src/panicking.rs index 3587f3f0e..48e90e6d7 100644 --- a/crux-mir/lib/core/src/panicking.rs +++ b/crux-mir/lib/core/src/panicking.rs @@ -1,8 +1,8 @@ -//! Panic support for libcore +//! Panic support for core //! //! The core library cannot define panicking, but it does *declare* panicking. This -//! means that the functions inside of libcore are allowed to panic, but to be -//! useful an upstream crate must define panicking for libcore to use. The current +//! means that the functions inside of core are allowed to panic, but to be +//! useful an upstream crate must define panicking for core to use. The current //! interface for panicking is: //! //! ``` @@ -13,100 +13,253 @@ //! This definition allows for panicking with any general message, but it does not //! allow for failing with a `Box` value. (`PanicInfo` just contains a `&(dyn Any + Send)`, //! for which we fill in a dummy value in `PanicInfo::internal_constructor`.) -//! The reason for this is that libcore is not allowed to allocate. +//! The reason for this is that core is not allowed to allocate. //! //! This module contains a few other panicking functions, but these are just the //! necessary lang items for the compiler. All panics are funneled through this //! one function. The actual symbol is declared through the `#[panic_handler]` attribute. -// ignore-tidy-undocumented-unsafe - #![allow(dead_code, missing_docs)] #![unstable( feature = "core_panic", - reason = "internal details of the implementation of the `panic!` \ - and related macros", + reason = "internal details of the implementation of the `panic!` and related macros", issue = "none" )] use crate::fmt; use crate::panic::{Location, PanicInfo}; -/// The underlying implementation of libcore's `panic!` macro when no formatting is used. -#[cold] -// never inline unless panic_immediate_abort to avoid code -// bloat at the call sites as much as possible -#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never))] +// First we define the two main entry points that all panics go through. +// In the end both are just convenience wrappers around `panic_impl`. + +/// The entry point for panicking with a formatted message. +/// +/// This is designed to reduce the amount of code required at the call +/// site as much as possible (so that `panic!()` has as low an impact +/// on (e.g.) the inlining of other functions as possible), by moving +/// the actual formatting into this shared place. +// If panic_immediate_abort, inline the abort call, +// otherwise avoid inlining because of it is cold path. +#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never), cold)] +#[cfg_attr(feature = "panic_immediate_abort", inline)] #[track_caller] -#[lang = "panic"] // needed by codegen for panic on overflow and other `Assert` MIR terminators -pub fn panic(expr: &str) -> ! { +#[lang = "panic_fmt"] // needed for const-evaluated panics +#[rustc_do_not_const_check] // hooked by const-eval +#[rustc_const_unstable(feature = "core_panic", issue = "none")] +pub const fn panic_fmt(fmt: fmt::Arguments<'_>) -> ! { if cfg!(feature = "panic_immediate_abort") { - unsafe { super::intrinsics::abort() } + super::intrinsics::abort() } - // Use Arguments::new_v1 instead of format_args!("{}", expr) to potentially + // NOTE This function never crosses the FFI boundary; it's a Rust-to-Rust call + // that gets resolved to the `#[panic_handler]` function. + extern "Rust" { + #[lang = "panic_impl"] + fn panic_impl(pi: &PanicInfo<'_>) -> !; + } + + let pi = PanicInfo::internal_constructor(Some(&fmt), Location::caller(), true); + + // SAFETY: `panic_impl` is defined in safe Rust code and thus is safe to call. + unsafe { panic_impl(&pi) } +} + +/// Like `panic_fmt`, but for non-unwinding panics. +/// +/// Has to be a separate function so that it can carry the `rustc_nounwind` attribute. +#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never), cold)] +#[cfg_attr(feature = "panic_immediate_abort", inline)] +#[track_caller] +// This attribute has the key side-effect that if the panic handler ignores `can_unwind` +// and unwinds anyway, we will hit the "unwinding out of nounwind function" guard, +// which causes a "panic in a function that cannot unwind". +#[rustc_nounwind] +pub fn panic_nounwind_fmt(fmt: fmt::Arguments<'_>) -> ! { + if cfg!(feature = "panic_immediate_abort") { + super::intrinsics::abort() + } + + // NOTE This function never crosses the FFI boundary; it's a Rust-to-Rust call + // that gets resolved to the `#[panic_handler]` function. + extern "Rust" { + #[lang = "panic_impl"] + fn panic_impl(pi: &PanicInfo<'_>) -> !; + } + + // PanicInfo with the `can_unwind` flag set to false forces an abort. + let pi = PanicInfo::internal_constructor(Some(&fmt), Location::caller(), false); + + // SAFETY: `panic_impl` is defined in safe Rust code and thus is safe to call. + unsafe { panic_impl(&pi) } +} + +// Next we define a bunch of higher-level wrappers that all bottom out in the two core functions +// above. + +/// The underlying implementation of core's `panic!` macro when no formatting is used. +// never inline unless panic_immediate_abort to avoid code +// bloat at the call sites as much as possible +#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never), cold)] +#[cfg_attr(feature = "panic_immediate_abort", inline)] +#[track_caller] +#[rustc_const_unstable(feature = "core_panic", issue = "none")] +#[lang = "panic"] // needed by codegen for panic on overflow and other `Assert` MIR terminators +pub const fn panic(expr: &'static str) -> ! { + // Use Arguments::new_v1 instead of format_args!("{expr}") to potentially // reduce size overhead. The format_args! macro uses str's Display trait to // write expr, which calls Formatter::pad, which must accommodate string // truncation and padding (even though none is used here). Using // Arguments::new_v1 may allow the compiler to omit Formatter::pad from the // output binary, saving up to a few kilobytes. - #[cfg(not(bootstrap))] panic_fmt(fmt::Arguments::new_v1(&[expr], &[])); - #[cfg(bootstrap)] - panic_fmt(fmt::Arguments::new_v1(&[expr], &[]), Location::caller()); } -#[cfg(not(bootstrap))] -#[cold] -#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never))] +/// Like `panic`, but without unwinding and track_caller to reduce the impact on codesize. +#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never), cold)] +#[cfg_attr(feature = "panic_immediate_abort", inline)] +#[cfg_attr(not(bootstrap), lang = "panic_nounwind")] // needed by codegen for non-unwinding panics +#[rustc_nounwind] +pub fn panic_nounwind(expr: &'static str) -> ! { + panic_nounwind_fmt(fmt::Arguments::new_v1(&[expr], &[])); +} + +#[inline] +#[track_caller] +#[rustc_diagnostic_item = "panic_str"] +#[rustc_const_unstable(feature = "core_panic", issue = "none")] +pub const fn panic_str(expr: &str) -> ! { + panic_display(&expr); +} + +#[inline] +#[track_caller] +#[rustc_diagnostic_item = "unreachable_display"] // needed for `non-fmt-panics` lint +pub fn unreachable_display(x: &T) -> ! { + panic_fmt(format_args!("internal error: entered unreachable code: {}", *x)); +} + +#[inline] +#[track_caller] +#[lang = "panic_display"] // needed for const-evaluated panics +#[rustc_do_not_const_check] // hooked by const-eval +#[rustc_const_unstable(feature = "core_panic", issue = "none")] +pub const fn panic_display(x: &T) -> ! { + panic_fmt(format_args!("{}", *x)); +} + +#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never), cold)] +#[cfg_attr(feature = "panic_immediate_abort", inline)] #[track_caller] #[lang = "panic_bounds_check"] // needed by codegen for panic on OOB array/slice access fn panic_bounds_check(index: usize, len: usize) -> ! { if cfg!(feature = "panic_immediate_abort") { - unsafe { super::intrinsics::abort() } + super::intrinsics::abort() } - panic!("index out of bounds: the len is {} but the index is {}", len, index) + panic!("index out of bounds: the len is {len} but the index is {index}") } -// For bootstrap, we need a variant with the old argument order, and a corresponding -// `panic_fmt`. -#[cfg(bootstrap)] -#[cold] -#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never))] -#[lang = "panic_bounds_check"] // needed by codegen for panic on OOB array/slice access -fn panic_bounds_check(location: &Location<'_>, index: usize, len: usize) -> ! { - if cfg!(feature = "panic_immediate_abort") { - unsafe { super::intrinsics::abort() } +/// Panic because we cannot unwind out of a function. +/// +/// This function is called directly by the codegen backend, and must not have +/// any extra arguments (including those synthesized by track_caller). +#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never), cold)] +#[cfg_attr(feature = "panic_immediate_abort", inline)] +#[cfg_attr(bootstrap, lang = "panic_no_unwind")] // needed by codegen for panic in nounwind function +#[cfg_attr(not(bootstrap), lang = "panic_cannot_unwind")] // needed by codegen for panic in nounwind function +#[rustc_nounwind] +fn panic_cannot_unwind() -> ! { + panic_nounwind("panic in a function that cannot unwind") +} + +/// This function is used instead of panic_fmt in const eval. +#[lang = "const_panic_fmt"] +#[rustc_const_unstable(feature = "core_panic", issue = "none")] +pub const fn const_panic_fmt(fmt: fmt::Arguments<'_>) -> ! { + if let Some(msg) = fmt.as_str() { + panic_str(msg); + } else { + // SAFETY: This is only evaluated at compile time, which reliably + // handles this UB (in case this branch turns out to be reachable + // somehow). + unsafe { crate::hint::unreachable_unchecked() }; } +} - panic_fmt( - format_args!("index out of bounds: the len is {} but the index is {}", len, index), - location, - ) +#[derive(Debug)] +#[doc(hidden)] +pub enum AssertKind { + Eq, + Ne, + Match, } -/// The underlying implementation of libcore's `panic!` macro when formatting is used. -#[cold] -#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never))] +/// Internal function for `assert_eq!` and `assert_ne!` macros +#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never), cold)] #[cfg_attr(feature = "panic_immediate_abort", inline)] -#[cfg_attr(not(bootstrap), track_caller)] -pub fn panic_fmt(fmt: fmt::Arguments<'_>, #[cfg(bootstrap)] location: &Location<'_>) -> ! { - if cfg!(feature = "panic_immediate_abort") { - unsafe { super::intrinsics::abort() } - } +#[track_caller] +#[doc(hidden)] +pub fn assert_failed( + kind: AssertKind, + left: &T, + right: &U, + args: Option>, +) -> ! +where + T: fmt::Debug + ?Sized, + U: fmt::Debug + ?Sized, +{ + assert_failed_inner(kind, &left, &right, args) +} - // NOTE This function never crosses the FFI boundary; it's a Rust-to-Rust call - // that gets resolved to the `#[panic_handler]` function. - extern "Rust" { - #[lang = "panic_impl"] - fn panic_impl(pi: &PanicInfo<'_>) -> !; +/// Internal function for `assert_match!` +#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never), cold)] +#[cfg_attr(feature = "panic_immediate_abort", inline)] +#[track_caller] +#[doc(hidden)] +pub fn assert_matches_failed( + left: &T, + right: &str, + args: Option>, +) -> ! { + // The pattern is a string so it can be displayed directly. + struct Pattern<'a>(&'a str); + impl fmt::Debug for Pattern<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.0) + } } + assert_failed_inner(AssertKind::Match, &left, &Pattern(right), args); +} - #[cfg(bootstrap)] - let pi = PanicInfo::internal_constructor(Some(&fmt), location); - #[cfg(not(bootstrap))] - let pi = PanicInfo::internal_constructor(Some(&fmt), Location::caller()); +/// Non-generic version of the above functions, to avoid code bloat. +#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never), cold)] +#[cfg_attr(feature = "panic_immediate_abort", inline)] +#[track_caller] +fn assert_failed_inner( + kind: AssertKind, + left: &dyn fmt::Debug, + right: &dyn fmt::Debug, + args: Option>, +) -> ! { + let op = match kind { + AssertKind::Eq => "==", + AssertKind::Ne => "!=", + AssertKind::Match => "matches", + }; - unsafe { panic_impl(&pi) } + match args { + Some(args) => panic!( + r#"assertion failed: `(left {} right)` + left: `{:?}`, + right: `{:?}`: {}"#, + op, left, right, args + ), + None => panic!( + r#"assertion failed: `(left {} right)` + left: `{:?}`, + right: `{:?}`"#, + op, left, right, + ), + } } diff --git a/crux-mir/lib/core/src/pin.rs b/crux-mir/lib/core/src/pin.rs index 774ecd997..ec0c99848 100644 --- a/crux-mir/lib/core/src/pin.rs +++ b/crux-mir/lib/core/src/pin.rs @@ -6,18 +6,23 @@ //! as moving an object with pointers to itself will invalidate them, which could cause undefined //! behavior. //! -//! A [`Pin

`] ensures that the pointee of any pointer type `P` has a stable location in memory, -//! meaning it cannot be moved elsewhere and its memory cannot be deallocated -//! until it gets dropped. We say that the pointee is "pinned". +//! At a high level, a [Pin]\

ensures that the pointee of any pointer type +//! `P` has a stable location in memory, meaning it cannot be moved elsewhere +//! and its memory cannot be deallocated until it gets dropped. We say that the +//! pointee is "pinned". Things get more subtle when discussing types that +//! combine pinned with non-pinned data; [see below](#projections-and-structural-pinning) +//! for more details. //! //! By default, all types in Rust are movable. Rust allows passing all types by-value, -//! and common smart-pointer types such as [`Box`] and `&mut T` allow replacing and -//! moving the values they contain: you can move out of a [`Box`], or you can use [`mem::swap`]. -//! [`Pin

`] wraps a pointer type `P`, so [`Pin`]`<`[`Box`]`>` functions much like a regular -//! [`Box`]: when a [`Pin`]`<`[`Box`]`>` gets dropped, so do its contents, and the memory gets -//! deallocated. Similarly, [`Pin`]`<&mut T>` is a lot like `&mut T`. However, [`Pin

`] does -//! not let clients actually obtain a [`Box`] or `&mut T` to pinned data, which implies that you -//! cannot use operations such as [`mem::swap`]: +//! and common smart-pointer types such as [Box]\ and [&mut] T allow +//! replacing and moving the values they contain: you can move out of a [Box]\, +//! or you can use [`mem::swap`]. [Pin]\

wraps a pointer type `P`, so +//! [Pin]<[Box]\> functions much like a regular [Box]\: +//! when a [Pin]<[Box]\> gets dropped, so do its contents, and the memory gets +//! deallocated. Similarly, [Pin]<[&mut] T> is a lot like [&mut] T. +//! However, [Pin]\

does not let clients actually obtain a [Box]\ +//! or [&mut] T to pinned data, which implies that you cannot use operations such +//! as [`mem::swap`]: //! //! ``` //! use std::pin::Pin; @@ -29,18 +34,18 @@ //! } //! ``` //! -//! It is worth reiterating that [`Pin

`] does *not* change the fact that a Rust compiler -//! considers all types movable. [`mem::swap`] remains callable for any `T`. Instead, [`Pin

`] -//! prevents certain *values* (pointed to by pointers wrapped in [`Pin

`]) from being -//! moved by making it impossible to call methods that require `&mut T` on them -//! (like [`mem::swap`]). -//! -//! [`Pin

`] can be used to wrap any pointer type `P`, and as such it interacts with -//! [`Deref`] and [`DerefMut`]. A [`Pin

`] where `P: Deref` should be considered -//! as a "`P`-style pointer" to a pinned `P::Target` -- so, a [`Pin`]`<`[`Box`]`>` is -//! an owned pointer to a pinned `T`, and a [`Pin`]`<`[`Rc`]`>` is a reference-counted -//! pointer to a pinned `T`. -//! For correctness, [`Pin

`] relies on the implementations of [`Deref`] and +//! It is worth reiterating that [Pin]\

does *not* change the fact that a Rust +//! compiler considers all types movable. [`mem::swap`] remains callable for any `T`. Instead, +//! [Pin]\

prevents certain *values* (pointed to by pointers wrapped in +//! [Pin]\

) from being moved by making it impossible to call methods that require +//! [&mut] T on them (like [`mem::swap`]). +//! +//! [Pin]\

can be used to wrap any pointer type `P`, and as such it interacts with +//! [`Deref`] and [`DerefMut`]. A [Pin]\

where P: [Deref] should be +//! considered as a "`P`-style pointer" to a pinned P::[Target] – so, a +//! [Pin]<[Box]\> is an owned pointer to a pinned `T`, and a +//! [Pin]<[Rc]\> is a reference-counted pointer to a pinned `T`. +//! For correctness, [Pin]\

relies on the implementations of [`Deref`] and //! [`DerefMut`] not to move out of their `self` parameter, and only ever to //! return a pointer to pinned data when they are called on a pinned pointer. //! @@ -50,17 +55,21 @@ //! rely on having a stable address. This includes all the basic types (like //! [`bool`], [`i32`], and references) as well as types consisting solely of these //! types. Types that do not care about pinning implement the [`Unpin`] -//! auto-trait, which cancels the effect of [`Pin

`]. For `T: Unpin`, -//! [`Pin`]`<`[`Box`]`>` and [`Box`] function identically, as do [`Pin`]`<&mut T>` and -//! `&mut T`. +//! auto-trait, which cancels the effect of [Pin]\

. For T: [Unpin], +//! [Pin]<[Box]\> and [Box]\ function identically, as do +//! [Pin]<[&mut] T> and [&mut] T. //! -//! Note that pinning and [`Unpin`] only affect the pointed-to type `P::Target`, not the pointer -//! type `P` itself that got wrapped in [`Pin

`]. For example, whether or not [`Box`] is -//! [`Unpin`] has no effect on the behavior of [`Pin`]`<`[`Box`]`>` (here, `T` is the -//! pointed-to type). +//! Note that pinning and [`Unpin`] only affect the pointed-to type P::[Target], +//! not the pointer type `P` itself that got wrapped in [Pin]\

. For example, +//! whether or not [Box]\ is [`Unpin`] has no effect on the behavior of +//! [Pin]<[Box]\> (here, `T` is the pointed-to type). //! //! # Example: self-referential struct //! +//! Before we go into more details to explain the guarantees and choices +//! associated with [Pin]\

, we discuss some examples for how it might be used. +//! Feel free to [skip to where the theoretical discussion continues](#drop-guarantee). +//! //! ```rust //! use std::pin::Pin; //! use std::marker::PhantomPinned; @@ -122,13 +131,13 @@ //! //! To make this work, every element has pointers to its predecessor and successor in //! the list. Elements can only be added when they are pinned, because moving the elements -//! around would invalidate the pointers. Moreover, the [`Drop`] implementation of a linked +//! around would invalidate the pointers. Moreover, the [`Drop`][Drop] implementation of a linked //! list element will patch the pointers of its predecessor and successor to remove itself //! from the list. //! //! Crucially, we have to be able to rely on [`drop`] being called. If an element //! could be deallocated or otherwise invalidated without calling [`drop`], the pointers into it -//! from its neighbouring elements would become invalid, which would break the data structure. +//! from its neighboring elements would become invalid, which would break the data structure. //! //! Therefore, pinning also comes with a [`drop`]-related guarantee. //! @@ -139,35 +148,37 @@ //! otherwise invalidating the memory used to store the data is restricted, too. //! Concretely, for pinned data you have to maintain the invariant //! that *its memory will not get invalidated or repurposed from the moment it gets pinned until -//! when [`drop`] is called*. Memory can be invalidated by deallocation, but also by -//! replacing a [`Some(v)`] by [`None`], or calling [`Vec::set_len`] to "kill" some elements -//! off of a vector. It can be repurposed by using [`ptr::write`] to overwrite it without -//! calling the destructor first. +//! when [`drop`] is called*. Only once [`drop`] returns or panics, the memory may be reused. +//! +//! Memory can be "invalidated" by deallocation, but also by +//! replacing a [Some]\(v) by [`None`], or calling [`Vec::set_len`] to "kill" some +//! elements off of a vector. It can be repurposed by using [`ptr::write`] to overwrite it without +//! calling the destructor first. None of this is allowed for pinned data without calling [`drop`]. //! //! This is exactly the kind of guarantee that the intrusive linked list from the previous //! section needs to function correctly. //! //! Notice that this guarantee does *not* mean that memory does not leak! It is still -//! completely okay not ever to call [`drop`] on a pinned element (e.g., you can still -//! call [`mem::forget`] on a [`Pin`]`<`[`Box`]`>`). In the example of the doubly-linked -//! list, that element would just stay in the list. However you may not free or reuse the storage +//! completely okay to not ever call [`drop`] on a pinned element (e.g., you can still +//! call [`mem::forget`] on a [Pin]<[Box]\>). In the example of the doubly-linked +//! list, that element would just stay in the list. However you must not free or reuse the storage //! *without calling [`drop`]*. //! //! # `Drop` implementation //! //! If your type uses pinning (such as the two examples above), you have to be careful -//! when implementing [`Drop`]. The [`drop`] function takes `&mut self`, but this +//! when implementing [`Drop`][Drop]. The [`drop`] function takes [&mut] self, but this //! is called *even if your type was previously pinned*! It is as if the //! compiler automatically called [`Pin::get_unchecked_mut`]. //! //! This can never cause a problem in safe code because implementing a type that //! relies on pinning requires unsafe code, but be aware that deciding to make //! use of pinning in your type (for example by implementing some operation on -//! [`Pin`]`<&Self>` or [`Pin`]`<&mut Self>`) has consequences for your [`Drop`] -//! implementation as well: if an element of your type could have been pinned, -//! you must treat [`Drop`] as implicitly taking [`Pin`]`<&mut Self>`. +//! [Pin]<[&]Self> or [Pin]<[&mut] Self>) has consequences for your +//! [`Drop`][Drop] implementation as well: if an element of your type could have been pinned, +//! you must treat [`Drop`][Drop] as implicitly taking [Pin]<[&mut] Self>. //! -//! For example, you could implement `Drop` as follows: +//! For example, you could implement [`Drop`][Drop] as follows: //! //! ```rust,no_run //! # use std::pin::Pin; @@ -195,18 +206,18 @@ //! # Projections and Structural Pinning //! //! When working with pinned structs, the question arises how one can access the -//! fields of that struct in a method that takes just [`Pin`]`<&mut Struct>`. +//! fields of that struct in a method that takes just [Pin]<[&mut] Struct>. //! The usual approach is to write helper methods (so called *projections*) -//! that turn [`Pin`]`<&mut Struct>` into a reference to the field, but what -//! type should that reference have? Is it [`Pin`]`<&mut Field>` or `&mut Field`? +//! that turn [Pin]<[&mut] Struct> into a reference to the field, but what type should +//! that reference have? Is it [Pin]<[&mut] Field> or [&mut] Field? //! The same question arises with the fields of an `enum`, and also when considering -//! container/wrapper types such as [`Vec`], [`Box`], or [`RefCell`]. -//! (This question applies to both mutable and shared references, we just -//! use the more common case of mutable references here for illustration.) +//! container/wrapper types such as [Vec]\, [Box]\, +//! or [RefCell]\. (This question applies to both mutable and shared references, +//! we just use the more common case of mutable references here for illustration.) //! -//! It turns out that it is actually up to the author of the data structure -//! to decide whether the pinned projection for a particular field turns -//! [`Pin`]`<&mut Struct>` into [`Pin`]`<&mut Field>` or `&mut Field`. There are some +//! It turns out that it is actually up to the author of the data structure to decide whether +//! the pinned projection for a particular field turns [Pin]<[&mut] Struct> +//! into [Pin]<[&mut] Field> or [&mut] Field. There are some //! constraints though, and the most important constraint is *consistency*: //! every field can be *either* projected to a pinned reference, *or* have //! pinning removed as part of the projection. If both are done for the same field, @@ -221,12 +232,12 @@ //! ## Pinning *is not* structural for `field` //! //! It may seem counter-intuitive that the field of a pinned struct might not be pinned, -//! but that is actually the easiest choice: if a [`Pin`]`<&mut Field>` is never created, +//! but that is actually the easiest choice: if a [Pin]<[&mut] Field> is never created, //! nothing can go wrong! So, if you decide that some field does not have structural pinning, //! all you have to ensure is that you never create a pinned reference to that field. //! //! Fields without structural pinning may have a projection method that turns -//! [`Pin`]`<&mut Struct>` into `&mut Field`: +//! [Pin]<[&mut] Struct> into [&mut] Field: //! //! ```rust,no_run //! # use std::pin::Pin; @@ -240,16 +251,16 @@ //! } //! ``` //! -//! You may also `impl Unpin for Struct` *even if* the type of `field` +//! You may also impl [Unpin] for Struct *even if* the type of `field` //! is not [`Unpin`]. What that type thinks about pinning is not relevant -//! when no [`Pin`]`<&mut Field>` is ever created. +//! when no [Pin]<[&mut] Field> is ever created. //! //! ## Pinning *is* structural for `field` //! //! The other option is to decide that pinning is "structural" for `field`, //! meaning that if the struct is pinned then so is the field. //! -//! This allows writing a projection that creates a [`Pin`]`<&mut Field>`, thus +//! This allows writing a projection that creates a [Pin]<[&mut] Field>, thus //! witnessing that the field is pinned: //! //! ```rust,no_run @@ -269,34 +280,36 @@ //! 1. The struct must only be [`Unpin`] if all the structural fields are //! [`Unpin`]. This is the default, but [`Unpin`] is a safe trait, so as the author of //! the struct it is your responsibility *not* to add something like -//! `impl Unpin for Struct`. (Notice that adding a projection operation +//! impl\ [Unpin] for Struct\. (Notice that adding a projection operation //! requires unsafe code, so the fact that [`Unpin`] is a safe trait does not break -//! the principle that you only have to worry about any of this if you use `unsafe`.) +//! the principle that you only have to worry about any of this if you use [`unsafe`].) //! 2. The destructor of the struct must not move structural fields out of its argument. This -//! is the exact point that was raised in the [previous section][drop-impl]: `drop` takes -//! `&mut self`, but the struct (and hence its fields) might have been pinned before. -//! You have to guarantee that you do not move a field inside your [`Drop`] implementation. -//! In particular, as explained previously, this means that your struct must *not* -//! be `#[repr(packed)]`. +//! is the exact point that was raised in the [previous section][drop-impl]: [`drop`] takes +//! [&mut] self, but the struct (and hence its fields) might have been pinned +//! before. You have to guarantee that you do not move a field inside your [`Drop`][Drop] +//! implementation. In particular, as explained previously, this means that your struct +//! must *not* be `#[repr(packed)]`. //! See that section for how to write [`drop`] in a way that the compiler can help you //! not accidentally break pinning. //! 3. You must make sure that you uphold the [`Drop` guarantee][drop-guarantee]: //! once your struct is pinned, the memory that contains the //! content is not overwritten or deallocated without calling the content's destructors. -//! This can be tricky, as witnessed by [`VecDeque`]: the destructor of [`VecDeque`] -//! can fail to call [`drop`] on all elements if one of the destructors panics. This violates -//! the [`Drop`] guarantee, because it can lead to elements being deallocated without -//! their destructor being called. ([`VecDeque`] has no pinning projections, so this +//! This can be tricky, as witnessed by [VecDeque]\: the destructor of +//! [VecDeque]\ can fail to call [`drop`] on all elements if one of the +//! destructors panics. This violates the [`Drop`][Drop] guarantee, because it can lead to +//! elements being deallocated without their destructor being called. +//! ([VecDeque]\ has no pinning projections, so this //! does not cause unsoundness.) //! 4. You must not offer any other operations that could lead to data being moved out of //! the structural fields when your type is pinned. For example, if the struct contains an -//! [`Option`] and there is a `take`-like operation with type -//! `fn(Pin<&mut Struct>) -> Option`, -//! that operation can be used to move a `T` out of a pinned `Struct` -- which means +//! [Option]\ and there is a [`take`][Option::take]-like operation with type +//! fn([Pin]<[&mut] Struct\>) -> [Option]\, +//! that operation can be used to move a `T` out of a pinned `Struct` – which means //! pinning cannot be structural for the field holding this data. //! -//! For a more complex example of moving data out of a pinned type, imagine if [`RefCell`] -//! had a method `fn get_pin_mut(self: Pin<&mut Self>) -> Pin<&mut T>`. +//! For a more complex example of moving data out of a pinned type, +//! imagine if [RefCell]\ had a method +//! fn get_pin_mut(self: [Pin]<[&mut] Self>) -> [Pin]<[&mut] T>. //! Then we could do the following: //! ```compile_fail //! fn exploit_ref_cell(rc: Pin<&mut RefCell>) { @@ -306,71 +319,65 @@ //! let content = &mut *b; // And here we have `&mut T` to the same data. //! } //! ``` -//! This is catastrophic, it means we can first pin the content of the [`RefCell`] -//! (using `RefCell::get_pin_mut`) and then move that content using the mutable -//! reference we got later. +//! This is catastrophic, it means we can first pin the content of the +//! [RefCell]\ (using [RefCell]::get_pin_mut) and then move that +//! content using the mutable reference we got later. //! //! ## Examples //! -//! For a type like [`Vec`], both possibilities (structural pinning or not) make sense. -//! A [`Vec`] with structural pinning could have `get_pin`/`get_pin_mut` methods to get -//! pinned references to elements. However, it could *not* allow calling -//! [`pop`][Vec::pop] on a pinned [`Vec`] because that would move the (structurally pinned) -//! contents! Nor could it allow [`push`][Vec::push], which might reallocate and thus also move the -//! contents. +//! For a type like [Vec]\, both possibilities (structural pinning or not) make +//! sense. A [Vec]\ with structural pinning could have `get_pin`/`get_pin_mut` +//! methods to get pinned references to elements. However, it could *not* allow calling +//! [`pop`][Vec::pop] on a pinned [Vec]\ because that would move the (structurally +//! pinned) contents! Nor could it allow [`push`][Vec::push], which might reallocate and thus also +//! move the contents. //! -//! A [`Vec`] without structural pinning could `impl Unpin for Vec`, because the contents -//! are never pinned and the [`Vec`] itself is fine with being moved as well. +//! A [Vec]\ without structural pinning could +//! impl\ [Unpin] for [Vec]\, because the contents are never pinned +//! and the [Vec]\ itself is fine with being moved as well. //! At that point pinning just has no effect on the vector at all. //! //! In the standard library, pointer types generally do not have structural pinning, -//! and thus they do not offer pinning projections. This is why `Box: Unpin` holds for all `T`. -//! It makes sense to do this for pointer types, because moving the `Box` -//! does not actually move the `T`: the [`Box`] can be freely movable (aka `Unpin`) even if -//! the `T` is not. In fact, even [`Pin`]`<`[`Box`]`>` and [`Pin`]`<&mut T>` are always -//! [`Unpin`] themselves, for the same reason: their contents (the `T`) are pinned, but the -//! pointers themselves can be moved without moving the pinned data. For both [`Box`] and -//! [`Pin`]`<`[`Box`]`>`, whether the content is pinned is entirely independent of whether the +//! and thus they do not offer pinning projections. This is why [Box]\: [Unpin] +//! holds for all `T`. It makes sense to do this for pointer types, because moving the +//! [Box]\ does not actually move the `T`: the [Box]\ can be freely +//! movable (aka [`Unpin`]) even if the `T` is not. In fact, even [Pin]<[Box]\> and +//! [Pin]<[&mut] T> are always [`Unpin`] themselves, for the same reason: +//! their contents (the `T`) are pinned, but the pointers themselves can be moved without moving +//! the pinned data. For both [Box]\ and [Pin]<[Box]\>, +//! whether the content is pinned is entirely independent of whether the //! pointer is pinned, meaning pinning is *not* structural. //! //! When implementing a [`Future`] combinator, you will usually need structural pinning //! for the nested futures, as you need to get pinned references to them to call [`poll`]. //! But if your combinator contains any other data that does not need to be pinned, //! you can make those fields not structural and hence freely access them with a -//! mutable reference even when you just have [`Pin`]`<&mut Self>` (such as in your own +//! mutable reference even when you just have [Pin]<[&mut] Self> (such as in your own //! [`poll`] implementation). //! -//! [`Pin

`]: struct.Pin.html -//! [`Unpin`]: ../marker/trait.Unpin.html -//! [`Deref`]: ../ops/trait.Deref.html -//! [`DerefMut`]: ../ops/trait.DerefMut.html -//! [`mem::swap`]: ../mem/fn.swap.html -//! [`mem::forget`]: ../mem/fn.forget.html -//! [`Box`]: ../../std/boxed/struct.Box.html -//! [`Vec`]: ../../std/vec/struct.Vec.html -//! [`Vec::set_len`]: ../../std/vec/struct.Vec.html#method.set_len -//! [`Pin`]: struct.Pin.html -//! [`Box`]: ../../std/boxed/struct.Box.html -//! [Vec::pop]: ../../std/vec/struct.Vec.html#method.pop -//! [Vec::push]: ../../std/vec/struct.Vec.html#method.push -//! [`Rc`]: ../../std/rc/struct.Rc.html -//! [`RefCell`]: ../../std/cell/struct.RefCell.html -//! [`Drop`]: ../../std/ops/trait.Drop.html -//! [`drop`]: ../../std/ops/trait.Drop.html#tymethod.drop -//! [`VecDeque`]: ../../std/collections/struct.VecDeque.html -//! [`Option`]: ../../std/option/enum.Option.html -//! [`VecDeque`]: ../../std/collections/struct.VecDeque.html -//! [`RefCell`]: ../cell/struct.RefCell.html -//! [`None`]: ../option/enum.Option.html#variant.None -//! [`Some(v)`]: ../option/enum.Option.html#variant.Some -//! [`ptr::write`]: ../ptr/fn.write.html -//! [`Future`]: ../future/trait.Future.html +//! [Deref]: crate::ops::Deref "ops::Deref" +//! [`Deref`]: crate::ops::Deref "ops::Deref" +//! [Target]: crate::ops::Deref::Target "ops::Deref::Target" +//! [`DerefMut`]: crate::ops::DerefMut "ops::DerefMut" +//! [`mem::swap`]: crate::mem::swap "mem::swap" +//! [`mem::forget`]: crate::mem::forget "mem::forget" +//! [Vec]: ../../std/vec/struct.Vec.html "Vec" +//! [`Vec::set_len`]: ../../std/vec/struct.Vec.html#method.set_len "Vec::set_len" +//! [Box]: ../../std/boxed/struct.Box.html "Box" +//! [Vec::pop]: ../../std/vec/struct.Vec.html#method.pop "Vec::pop" +//! [Vec::push]: ../../std/vec/struct.Vec.html#method.push "Vec::push" +//! [Rc]: ../../std/rc/struct.Rc.html "rc::Rc" +//! [RefCell]: crate::cell::RefCell "cell::RefCell" +//! [`drop`]: Drop::drop +//! [VecDeque]: ../../std/collections/struct.VecDeque.html "collections::VecDeque" +//! [`ptr::write`]: crate::ptr::write "ptr::write" +//! [`Future`]: crate::future::Future "future::Future" //! [drop-impl]: #drop-implementation //! [drop-guarantee]: #drop-guarantee -//! [`poll`]: ../../std/future/trait.Future.html#tymethod.poll -//! [`Pin::get_unchecked_mut`]: struct.Pin.html#method.get_unchecked_mut -//! [`bool`]: ../../std/primitive.bool.html -//! [`i32`]: ../../std/primitive.i32.html +//! [`poll`]: crate::future::Future::poll "future::Future::poll" +//! [&]: reference "shared reference" +//! [&mut]: reference "mutable reference" +//! [`unsafe`]: ../../std/keyword.unsafe.html "keyword unsafe" #![stable(feature = "pin", since = "1.33.0")] @@ -388,8 +395,7 @@ use crate::ops::{CoerceUnsized, Deref, DerefMut, DispatchFromDyn, Receiver}; /// /// *See the [`pin` module] documentation for an explanation of pinning.* /// -/// [`Unpin`]: ../../std/marker/trait.Unpin.html -/// [`pin` module]: ../../std/pin/index.html +/// [`pin` module]: self // // Note: the `Clone` derive below causes unsoundness as it's possible to implement // `Clone` for mutable references. @@ -400,7 +406,14 @@ use crate::ops::{CoerceUnsized, Deref, DerefMut, DispatchFromDyn, Receiver}; #[repr(transparent)] #[derive(Copy, Clone)] pub struct Pin

{ - pointer: P, + // FIXME(#93176): this field is made `#[unstable] #[doc(hidden)] pub` to: + // - deter downstream users from accessing it (which would be unsound!), + // - let the `pin!` macro access it (such a macro requires using struct + // literal syntax in order to benefit from lifetime extension). + // Long-term, `unsafe` fields or macro hygiene are expected to offer more robust alternatives. + #[unstable(feature = "unsafe_pin_internals", issue = "none")] + #[doc(hidden)] + pub pointer: P, } // The following implementations aren't derived in order to avoid soundness @@ -473,24 +486,44 @@ impl> Pin

{ /// Unlike `Pin::new_unchecked`, this method is safe because the pointer /// `P` dereferences to an [`Unpin`] type, which cancels the pinning guarantees. /// - /// [`Unpin`]: ../../std/marker/trait.Unpin.html - #[stable(feature = "pin", since = "1.33.0")] + /// # Examples + /// + /// ``` + /// use std::pin::Pin; + /// + /// let mut val: u8 = 5; + /// // We can pin the value, since it doesn't care about being moved + /// let mut pinned: Pin<&mut u8> = Pin::new(&mut val); + /// ``` #[inline(always)] - pub fn new(pointer: P) -> Pin

{ - // Safety: the value pointed to is `Unpin`, and so has no requirements + #[rustc_const_unstable(feature = "const_pin", issue = "76654")] + #[stable(feature = "pin", since = "1.33.0")] + pub const fn new(pointer: P) -> Pin

{ + // SAFETY: the value pointed to is `Unpin`, and so has no requirements // around pinning. unsafe { Pin::new_unchecked(pointer) } } /// Unwraps this `Pin

` returning the underlying pointer. /// - /// This requires that the data inside this `Pin` is [`Unpin`] so that we + /// This requires that the data inside this `Pin` implements [`Unpin`] so that we /// can ignore the pinning invariants when unwrapping it. /// - /// [`Unpin`]: ../../std/marker/trait.Unpin.html - #[stable(feature = "pin_into_inner", since = "1.39.0")] + /// # Examples + /// + /// ``` + /// use std::pin::Pin; + /// + /// let mut val: u8 = 5; + /// let pinned: Pin<&mut u8> = Pin::new(&mut val); + /// // Unwrap the pin to get a reference to the value + /// let r = Pin::into_inner(pinned); + /// assert_eq!(*r, 5); + /// ``` #[inline(always)] - pub fn into_inner(pin: Pin

) -> P { + #[rustc_const_unstable(feature = "const_pin", issue = "76654")] + #[stable(feature = "pin_into_inner", since = "1.39.0")] + pub const fn into_inner(pin: Pin

) -> P { pin.pointer } } @@ -532,7 +565,7 @@ impl Pin

{ /// let p: Pin<&mut T> = Pin::new_unchecked(&mut a); /// // This should mean the pointee `a` can never move again. /// } - /// mem::swap(&mut a, &mut b); + /// mem::swap(&mut a, &mut b); // Potential UB down the road ⚠️ /// // The address of `a` changed to `b`'s stack slot, so `a` got moved even /// // though we have previously pinned it! We have violated the pinning API contract. /// } @@ -546,23 +579,77 @@ impl Pin

{ /// use std::pin::Pin; /// /// fn move_pinned_rc(mut x: Rc) { - /// let pinned = unsafe { Pin::new_unchecked(x.clone()) }; + /// let pinned = unsafe { Pin::new_unchecked(Rc::clone(&x)) }; /// { /// let p: Pin<&T> = pinned.as_ref(); /// // This should mean the pointee can never move again. /// } /// drop(pinned); - /// let content = Rc::get_mut(&mut x).unwrap(); + /// let content = Rc::get_mut(&mut x).unwrap(); // Potential UB down the road ⚠️ /// // Now, if `x` was the only reference, we have a mutable reference to /// // data that we pinned above, which we could use to move it as we have /// // seen in the previous example. We have violated the pinning API contract. /// } /// ``` /// - /// [`mem::swap`]: ../../std/mem/fn.swap.html - #[stable(feature = "pin", since = "1.33.0")] + /// ## Pinning of closure captures + /// + /// Particular care is required when using `Pin::new_unchecked` in a closure: + /// `Pin::new_unchecked(&mut var)` where `var` is a by-value (moved) closure capture + /// implicitly makes the promise that the closure itself is pinned, and that *all* uses + /// of this closure capture respect that pinning. + /// ``` + /// use std::pin::Pin; + /// use std::task::Context; + /// use std::future::Future; + /// + /// fn move_pinned_closure(mut x: impl Future, cx: &mut Context<'_>) { + /// // Create a closure that moves `x`, and then internally uses it in a pinned way. + /// let mut closure = move || unsafe { + /// let _ignore = Pin::new_unchecked(&mut x).poll(cx); + /// }; + /// // Call the closure, so the future can assume it has been pinned. + /// closure(); + /// // Move the closure somewhere else. This also moves `x`! + /// let mut moved = closure; + /// // Calling it again means we polled the future from two different locations, + /// // violating the pinning API contract. + /// moved(); // Potential UB ⚠️ + /// } + /// ``` + /// When passing a closure to another API, it might be moving the closure any time, so + /// `Pin::new_unchecked` on closure captures may only be used if the API explicitly documents + /// that the closure is pinned. + /// + /// The better alternative is to avoid all that trouble and do the pinning in the outer function + /// instead (here using the [`pin!`][crate::pin::pin] macro): + /// ``` + /// use std::pin::pin; + /// use std::task::Context; + /// use std::future::Future; + /// + /// fn move_pinned_closure(mut x: impl Future, cx: &mut Context<'_>) { + /// let mut x = pin!(x); + /// // Create a closure that captures `x: Pin<&mut _>`, which is safe to move. + /// let mut closure = move || { + /// let _ignore = x.as_mut().poll(cx); + /// }; + /// // Call the closure, so the future can assume it has been pinned. + /// closure(); + /// // Move the closure somewhere else. + /// let mut moved = closure; + /// // Calling it again here is fine (except that we might be polling a future that already + /// // returned `Poll::Ready`, but that is a separate problem). + /// moved(); + /// } + /// ``` + /// + /// [`mem::swap`]: crate::mem::swap + #[lang = "new_unchecked"] #[inline(always)] - pub unsafe fn new_unchecked(pointer: P) -> Pin

{ + #[rustc_const_unstable(feature = "const_pin", issue = "76654")] + #[stable(feature = "pin", since = "1.33.0")] + pub const unsafe fn new_unchecked(pointer: P) -> Pin

{ Pin { pointer } } @@ -593,12 +680,10 @@ impl Pin

{ /// /// If the underlying data is [`Unpin`], [`Pin::into_inner`] should be used /// instead. - /// - /// [`Unpin`]: ../../std/marker/trait.Unpin.html - /// [`Pin::into_inner`]: #method.into_inner - #[stable(feature = "pin_into_inner", since = "1.39.0")] #[inline(always)] - pub unsafe fn into_inner_unchecked(pin: Pin

) -> P { + #[rustc_const_unstable(feature = "const_pin", issue = "76654")] + #[stable(feature = "pin_into_inner", since = "1.39.0")] + pub const unsafe fn into_inner_unchecked(pin: Pin

) -> P { pin.pointer } } @@ -643,6 +728,18 @@ impl Pin

{ /// /// This overwrites pinned data, but that is okay: its destructor gets /// run before being overwritten, so no pinning guarantee is violated. + /// + /// # Example + /// + /// ``` + /// use std::pin::Pin; + /// + /// let mut val: u8 = 5; + /// let mut pinned: Pin<&mut u8> = Pin::new(&mut val); + /// println!("{}", pinned); // 5 + /// pinned.as_mut().set(10); + /// println!("{}", pinned); // 10 + /// ``` #[stable(feature = "pin", since = "1.33.0")] #[inline(always)] pub fn set(&mut self, value: P::Target) @@ -656,7 +753,7 @@ impl Pin

{ impl<'a, T: ?Sized> Pin<&'a T> { /// Constructs a new pin by mapping the interior value. /// - /// For example, if you wanted to get a `Pin` of a field of something, + /// For example, if you wanted to get a `Pin` of a field of something, /// you could use this to get access to that field in one line of code. /// However, there are several gotchas with these "pinning projections"; /// see the [`pin` module] documentation for further details on that topic. @@ -668,7 +765,7 @@ impl<'a, T: ?Sized> Pin<&'a T> { /// because it is one of the fields of that value), and also that you do /// not move out of the argument you receive to the interior function. /// - /// [`pin` module]: ../../std/pin/index.html#projections-and-structural-pinning + /// [`pin` module]: self#projections-and-structural-pinning #[stable(feature = "pin", since = "1.33.0")] pub unsafe fn map_unchecked(self, func: F) -> Pin<&'a U> where @@ -677,7 +774,10 @@ impl<'a, T: ?Sized> Pin<&'a T> { { let pointer = &*self.pointer; let new_pointer = func(pointer); - Pin::new_unchecked(new_pointer) + + // SAFETY: the safety contract for `new_unchecked` must be + // upheld by the caller. + unsafe { Pin::new_unchecked(new_pointer) } } /// Gets a shared reference out of a pin. @@ -696,19 +796,23 @@ impl<'a, T: ?Sized> Pin<&'a T> { /// the `Pin` itself. This method allows turning the `Pin` into a reference /// with the same lifetime as the original `Pin`. /// - /// ["pinning projections"]: ../../std/pin/index.html#projections-and-structural-pinning - #[stable(feature = "pin", since = "1.33.0")] + /// ["pinning projections"]: self#projections-and-structural-pinning #[inline(always)] - pub fn get_ref(self) -> &'a T { + #[must_use] + #[rustc_const_unstable(feature = "const_pin", issue = "76654")] + #[stable(feature = "pin", since = "1.33.0")] + pub const fn get_ref(self) -> &'a T { self.pointer } } impl<'a, T: ?Sized> Pin<&'a mut T> { /// Converts this `Pin<&mut T>` into a `Pin<&T>` with the same lifetime. - #[stable(feature = "pin", since = "1.33.0")] #[inline(always)] - pub fn into_ref(self) -> Pin<&'a T> { + #[must_use = "`self` will be dropped if the result is not used"] + #[rustc_const_unstable(feature = "const_pin", issue = "76654")] + #[stable(feature = "pin", since = "1.33.0")] + pub const fn into_ref(self) -> Pin<&'a T> { Pin { pointer: self.pointer } } @@ -721,9 +825,11 @@ impl<'a, T: ?Sized> Pin<&'a mut T> { /// that lives for as long as the borrow of the `Pin`, not the lifetime of /// the `Pin` itself. This method allows turning the `Pin` into a reference /// with the same lifetime as the original `Pin`. - #[stable(feature = "pin", since = "1.33.0")] #[inline(always)] - pub fn get_mut(self) -> &'a mut T + #[must_use = "`self` will be dropped if the result is not used"] + #[stable(feature = "pin", since = "1.33.0")] + #[rustc_const_unstable(feature = "const_pin", issue = "76654")] + pub const fn get_mut(self) -> &'a mut T where T: Unpin, { @@ -740,15 +846,17 @@ impl<'a, T: ?Sized> Pin<&'a mut T> { /// /// If the underlying data is `Unpin`, `Pin::get_mut` should be used /// instead. - #[stable(feature = "pin", since = "1.33.0")] #[inline(always)] - pub unsafe fn get_unchecked_mut(self) -> &'a mut T { + #[must_use = "`self` will be dropped if the result is not used"] + #[stable(feature = "pin", since = "1.33.0")] + #[rustc_const_unstable(feature = "const_pin", issue = "76654")] + pub const unsafe fn get_unchecked_mut(self) -> &'a mut T { self.pointer } /// Construct a new pin by mapping the interior value. /// - /// For example, if you wanted to get a `Pin` of a field of something, + /// For example, if you wanted to get a `Pin` of a field of something, /// you could use this to get access to that field in one line of code. /// However, there are several gotchas with these "pinning projections"; /// see the [`pin` module] documentation for further details on that topic. @@ -760,16 +868,88 @@ impl<'a, T: ?Sized> Pin<&'a mut T> { /// because it is one of the fields of that value), and also that you do /// not move out of the argument you receive to the interior function. /// - /// [`pin` module]: ../../std/pin/index.html#projections-and-structural-pinning + /// [`pin` module]: self#projections-and-structural-pinning + #[must_use = "`self` will be dropped if the result is not used"] #[stable(feature = "pin", since = "1.33.0")] pub unsafe fn map_unchecked_mut(self, func: F) -> Pin<&'a mut U> where U: ?Sized, F: FnOnce(&mut T) -> &mut U, { - let pointer = Pin::get_unchecked_mut(self); + // SAFETY: the caller is responsible for not moving the + // value out of this reference. + let pointer = unsafe { Pin::get_unchecked_mut(self) }; let new_pointer = func(pointer); - Pin::new_unchecked(new_pointer) + // SAFETY: as the value of `this` is guaranteed to not have + // been moved out, this call to `new_unchecked` is safe. + unsafe { Pin::new_unchecked(new_pointer) } + } +} + +impl Pin<&'static T> { + /// Get a pinned reference from a static reference. + /// + /// This is safe, because `T` is borrowed for the `'static` lifetime, which + /// never ends. + #[stable(feature = "pin_static_ref", since = "1.61.0")] + #[rustc_const_unstable(feature = "const_pin", issue = "76654")] + pub const fn static_ref(r: &'static T) -> Pin<&'static T> { + // SAFETY: The 'static borrow guarantees the data will not be + // moved/invalidated until it gets dropped (which is never). + unsafe { Pin::new_unchecked(r) } + } +} + +impl<'a, P: DerefMut> Pin<&'a mut Pin

> { + /// Gets a pinned mutable reference from this nested pinned pointer. + /// + /// This is a generic method to go from `Pin<&mut Pin>>` to `Pin<&mut T>`. It is + /// safe because the existence of a `Pin>` ensures that the pointee, `T`, cannot + /// move in the future, and this method does not enable the pointee to move. "Malicious" + /// implementations of `P::DerefMut` are likewise ruled out by the contract of + /// `Pin::new_unchecked`. + #[unstable(feature = "pin_deref_mut", issue = "86918")] + #[must_use = "`self` will be dropped if the result is not used"] + #[inline(always)] + pub fn as_deref_mut(self) -> Pin<&'a mut P::Target> { + // SAFETY: What we're asserting here is that going from + // + // Pin<&mut Pin

> + // + // to + // + // Pin<&mut P::Target> + // + // is safe. + // + // We need to ensure that two things hold for that to be the case: + // + // 1) Once we give out a `Pin<&mut P::Target>`, an `&mut P::Target` will not be given out. + // 2) By giving out a `Pin<&mut P::Target>`, we do not risk of violating `Pin<&mut Pin

>` + // + // The existence of `Pin

` is sufficient to guarantee #1: since we already have a + // `Pin

`, it must already uphold the pinning guarantees, which must mean that + // `Pin<&mut P::Target>` does as well, since `Pin::as_mut` is safe. We do not have to rely + // on the fact that P is _also_ pinned. + // + // For #2, we need to ensure that code given a `Pin<&mut P::Target>` cannot cause the + // `Pin

` to move? That is not possible, since `Pin<&mut P::Target>` no longer retains + // any access to the `P` itself, much less the `Pin

`. + unsafe { self.get_unchecked_mut() }.as_mut() + } +} + +impl Pin<&'static mut T> { + /// Get a pinned mutable reference from a static mutable reference. + /// + /// This is safe, because `T` is borrowed for the `'static` lifetime, which + /// never ends. + #[stable(feature = "pin_static_ref", since = "1.61.0")] + #[rustc_const_unstable(feature = "const_pin", issue = "76654")] + pub const fn static_mut(r: &'static mut T) -> Pin<&'static mut T> { + // SAFETY: The 'static borrow guarantees the data will not be + // moved/invalidated until it gets dropped (which is never). + unsafe { Pin::new_unchecked(r) } } } @@ -822,3 +1002,241 @@ impl CoerceUnsized> for Pin

where P: CoerceUnsized {} #[stable(feature = "pin", since = "1.33.0")] impl DispatchFromDyn> for Pin

where P: DispatchFromDyn {} + +/// Constructs a [Pin]<[&mut] T>, by pinning[^1] a `value: T` _locally_[^2]. +/// +/// Unlike [`Box::pin`], this does not involve a heap allocation. +/// +/// [^1]: If the (type `T` of the) given value does not implement [`Unpin`], then this +/// effectively pins the `value` in memory, where it will be unable to be moved. +/// Otherwise, [Pin]<[&mut] T> behaves like [&mut] T, and operations such +/// as [`mem::replace()`][crate::mem::replace] will allow extracting that value, and therefore, +/// moving it. +/// See [the `Unpin` section of the `pin` module][self#unpin] for more info. +/// +/// [^2]: This is usually dubbed "stack"-pinning. And whilst local values are almost always located +/// in the stack (_e.g._, when within the body of a non-`async` function), the truth is that inside +/// the body of an `async fn` or block —more generally, the body of a generator— any locals crossing +/// an `.await` point —a `yield` point— end up being part of the state captured by the `Future` —by +/// the `Generator`—, and thus will be stored wherever that one is. +/// +/// ## Examples +/// +/// ### Basic usage +/// +/// ```rust +/// # use core::marker::PhantomPinned as Foo; +/// use core::pin::{pin, Pin}; +/// +/// fn stuff(foo: Pin<&mut Foo>) { +/// // … +/// # let _ = foo; +/// } +/// +/// let pinned_foo = pin!(Foo { /* … */ }); +/// stuff(pinned_foo); +/// // or, directly: +/// stuff(pin!(Foo { /* … */ })); +/// ``` +/// +/// ### Manually polling a `Future` (without `Unpin` bounds) +/// +/// ```rust +/// use std::{ +/// future::Future, +/// pin::pin, +/// task::{Context, Poll}, +/// thread, +/// }; +/// # use std::{sync::Arc, task::Wake, thread::Thread}; +/// +/// # /// A waker that wakes up the current thread when called. +/// # struct ThreadWaker(Thread); +/// # +/// # impl Wake for ThreadWaker { +/// # fn wake(self: Arc) { +/// # self.0.unpark(); +/// # } +/// # } +/// # +/// /// Runs a future to completion. +/// fn block_on(fut: Fut) -> Fut::Output { +/// let waker_that_unparks_thread = // … +/// # Arc::new(ThreadWaker(thread::current())).into(); +/// let mut cx = Context::from_waker(&waker_that_unparks_thread); +/// // Pin the future so it can be polled. +/// let mut pinned_fut = pin!(fut); +/// loop { +/// match pinned_fut.as_mut().poll(&mut cx) { +/// Poll::Pending => thread::park(), +/// Poll::Ready(res) => return res, +/// } +/// } +/// } +/// # +/// # assert_eq!(42, block_on(async { 42 })); +/// ``` +/// +/// ### With `Generator`s +/// +/// ```rust +/// #![feature(generators, generator_trait)] +/// use core::{ +/// ops::{Generator, GeneratorState}, +/// pin::pin, +/// }; +/// +/// fn generator_fn() -> impl Generator /* not Unpin */ { +/// // Allow generator to be self-referential (not `Unpin`) +/// // vvvvvv so that locals can cross yield points. +/// static || { +/// let foo = String::from("foo"); +/// let foo_ref = &foo; // ------+ +/// yield 0; // | <- crosses yield point! +/// println!("{foo_ref}"); // <--+ +/// yield foo.len(); +/// } +/// } +/// +/// fn main() { +/// let mut generator = pin!(generator_fn()); +/// match generator.as_mut().resume(()) { +/// GeneratorState::Yielded(0) => {}, +/// _ => unreachable!(), +/// } +/// match generator.as_mut().resume(()) { +/// GeneratorState::Yielded(3) => {}, +/// _ => unreachable!(), +/// } +/// match generator.resume(()) { +/// GeneratorState::Yielded(_) => unreachable!(), +/// GeneratorState::Complete(()) => {}, +/// } +/// } +/// ``` +/// +/// ## Remarks +/// +/// Precisely because a value is pinned to local storage, the resulting [Pin]<[&mut] T> +/// reference ends up borrowing a local tied to that block: it can't escape it. +/// +/// The following, for instance, fails to compile: +/// +/// ```rust,compile_fail +/// use core::pin::{pin, Pin}; +/// # use core::{marker::PhantomPinned as Foo, mem::drop as stuff}; +/// +/// let x: Pin<&mut Foo> = { +/// let x: Pin<&mut Foo> = pin!(Foo { /* … */ }); +/// x +/// }; // <- Foo is dropped +/// stuff(x); // Error: use of dropped value +/// ``` +/// +///

Error message +/// +/// ```console +/// error[E0716]: temporary value dropped while borrowed +/// --> src/main.rs:9:28 +/// | +/// 8 | let x: Pin<&mut Foo> = { +/// | - borrow later stored here +/// 9 | let x: Pin<&mut Foo> = pin!(Foo { /* … */ }); +/// | ^^^^^^^^^^^^^^^^^^^^^ creates a temporary value which is freed while still in use +/// 10 | x +/// 11 | }; // <- Foo is dropped +/// | - temporary value is freed at the end of this statement +/// | +/// = note: consider using a `let` binding to create a longer lived value +/// ``` +/// +///
+/// +/// This makes [`pin!`] **unsuitable to pin values when intending to _return_ them**. Instead, the +/// value is expected to be passed around _unpinned_ until the point where it is to be consumed, +/// where it is then useful and even sensible to pin the value locally using [`pin!`]. +/// +/// If you really need to return a pinned value, consider using [`Box::pin`] instead. +/// +/// On the other hand, pinning to the stack[2](#fn2) using [`pin!`] is likely to be +/// cheaper than pinning into a fresh heap allocation using [`Box::pin`]. Moreover, by virtue of not +/// even needing an allocator, [`pin!`] is the main non-`unsafe` `#![no_std]`-compatible [`Pin`] +/// constructor. +/// +/// [`Box::pin`]: ../../std/boxed/struct.Box.html#method.pin +#[stable(feature = "pin_macro", since = "CURRENT_RUSTC_VERSION")] +#[rustc_macro_transparency = "semitransparent"] +#[allow_internal_unstable(unsafe_pin_internals)] +pub macro pin($value:expr $(,)?) { + // This is `Pin::new_unchecked(&mut { $value })`, so, for starters, let's + // review such a hypothetical macro (that any user-code could define): + // + // ```rust + // macro_rules! pin {( $value:expr ) => ( + // match &mut { $value } { at_value => unsafe { // Do not wrap `$value` in an `unsafe` block. + // $crate::pin::Pin::<&mut _>::new_unchecked(at_value) + // }} + // )} + // ``` + // + // Safety: + // - `type P = &mut _`. There are thus no pathological `Deref{,Mut}` impls + // that would break `Pin`'s invariants. + // - `{ $value }` is braced, making it a _block expression_, thus **moving** + // the given `$value`, and making it _become an **anonymous** temporary_. + // By virtue of being anonymous, it can no longer be accessed, thus + // preventing any attempts to `mem::replace` it or `mem::forget` it, _etc._ + // + // This gives us a `pin!` definition that is sound, and which works, but only + // in certain scenarios: + // - If the `pin!(value)` expression is _directly_ fed to a function call: + // `let poll = pin!(fut).poll(cx);` + // - If the `pin!(value)` expression is part of a scrutinee: + // ```rust + // match pin!(fut) { pinned_fut => { + // pinned_fut.as_mut().poll(...); + // pinned_fut.as_mut().poll(...); + // }} // <- `fut` is dropped here. + // ``` + // Alas, it doesn't work for the more straight-forward use-case: `let` bindings. + // ```rust + // let pinned_fut = pin!(fut); // <- temporary value is freed at the end of this statement + // pinned_fut.poll(...) // error[E0716]: temporary value dropped while borrowed + // // note: consider using a `let` binding to create a longer lived value + // ``` + // - Issues such as this one are the ones motivating https://github.com/rust-lang/rfcs/pull/66 + // + // This makes such a macro incredibly unergonomic in practice, and the reason most macros + // out there had to take the path of being a statement/binding macro (_e.g._, `pin!(future);`) + // instead of featuring the more intuitive ergonomics of an expression macro. + // + // Luckily, there is a way to avoid the problem. Indeed, the problem stems from the fact that a + // temporary is dropped at the end of its enclosing statement when it is part of the parameters + // given to function call, which has precisely been the case with our `Pin::new_unchecked()`! + // For instance, + // ```rust + // let p = Pin::new_unchecked(&mut ); + // ``` + // becomes: + // ```rust + // let p = { let mut anon = ; &mut anon }; + // ``` + // + // However, when using a literal braced struct to construct the value, references to temporaries + // can then be taken. This makes Rust change the lifespan of such temporaries so that they are, + // instead, dropped _at the end of the enscoping block_. + // For instance, + // ```rust + // let p = Pin { pointer: &mut }; + // ``` + // becomes: + // ```rust + // let mut anon = ; + // let p = Pin { pointer: &mut anon }; + // ``` + // which is *exactly* what we want. + // + // See https://doc.rust-lang.org/1.58.1/reference/destructors.html#temporary-lifetime-extension + // for more info. + $crate::pin::Pin::<&mut _> { pointer: &mut { $value } } +} diff --git a/crux-mir/lib/core/src/prelude/mod.rs b/crux-mir/lib/core/src/prelude/mod.rs index 51f4acf06..12f762ef1 100644 --- a/crux-mir/lib/core/src/prelude/mod.rs +++ b/crux-mir/lib/core/src/prelude/mod.rs @@ -1,5 +1,57 @@ -//! The libcore prelude +//! The core prelude +//! +//! This module is intended for users of core which do not link to std as well. +//! This module is imported by default when `#![no_std]` is used in the same +//! manner as the standard library's prelude. #![stable(feature = "core_prelude", since = "1.4.0")] pub mod v1; + +/// The 2015 version of the core prelude. +/// +/// See the [module-level documentation](self) for more. +#[stable(feature = "prelude_2015", since = "1.55.0")] +pub mod rust_2015 { + #[stable(feature = "prelude_2015", since = "1.55.0")] + #[doc(no_inline)] + pub use super::v1::*; +} + +/// The 2018 version of the core prelude. +/// +/// See the [module-level documentation](self) for more. +#[stable(feature = "prelude_2018", since = "1.55.0")] +pub mod rust_2018 { + #[stable(feature = "prelude_2018", since = "1.55.0")] + #[doc(no_inline)] + pub use super::v1::*; +} + +/// The 2021 version of the core prelude. +/// +/// See the [module-level documentation](self) for more. +#[stable(feature = "prelude_2021", since = "1.55.0")] +pub mod rust_2021 { + #[stable(feature = "prelude_2021", since = "1.55.0")] + #[doc(no_inline)] + pub use super::v1::*; + + #[stable(feature = "prelude_2021", since = "1.55.0")] + #[doc(no_inline)] + pub use crate::iter::FromIterator; + + #[stable(feature = "prelude_2021", since = "1.55.0")] + #[doc(no_inline)] + pub use crate::convert::{TryFrom, TryInto}; +} + +/// The 2024 edition of the core prelude. +/// +/// See the [module-level documentation](self) for more. +#[unstable(feature = "prelude_2024", issue = "none")] +pub mod rust_2024 { + #[unstable(feature = "prelude_2024", issue = "none")] + #[doc(no_inline)] + pub use super::rust_2021::*; +} diff --git a/crux-mir/lib/core/src/prelude/v1.rs b/crux-mir/lib/core/src/prelude/v1.rs index c91370b27..10525a16f 100644 --- a/crux-mir/lib/core/src/prelude/v1.rs +++ b/crux-mir/lib/core/src/prelude/v1.rs @@ -1,8 +1,6 @@ -//! The core prelude +//! The first version of the core prelude. //! -//! This module is intended for users of libcore which do not link to libstd as -//! well. This module is imported by default when `#![no_std]` is used in the -//! same manner as the standard library's prelude. +//! See the [module-level documentation](super) for more. #![stable(feature = "core_prelude", since = "1.4.0")] @@ -54,25 +52,54 @@ pub use crate::fmt::macros::Debug; pub use crate::hash::macros::Hash; #[stable(feature = "builtin_macro_prelude", since = "1.38.0")] +#[allow(deprecated)] #[doc(no_inline)] pub use crate::{ - asm, assert, cfg, column, compile_error, concat, concat_idents, env, file, format_args, - format_args_nl, global_asm, include, include_bytes, include_str, line, log_syntax, module_path, - option_env, stringify, trace_macros, + assert, cfg, column, compile_error, concat, concat_idents, env, file, format_args, + format_args_nl, include, include_bytes, include_str, line, log_syntax, module_path, option_env, + stringify, trace_macros, }; +#[unstable( + feature = "concat_bytes", + issue = "87555", + reason = "`concat_bytes` is not stable enough for use and is subject to change" +)] +#[doc(no_inline)] +pub use crate::concat_bytes; + +// Do not `doc(inline)` these `doc(hidden)` items. #[stable(feature = "builtin_macro_prelude", since = "1.38.0")] #[allow(deprecated)] -#[doc(no_inline)] +pub use crate::macros::builtin::{RustcDecodable, RustcEncodable}; + +// Do not `doc(no_inline)` so that they become doc items on their own +// (no public module for them to be re-exported from). +#[stable(feature = "builtin_macro_prelude", since = "1.38.0")] pub use crate::macros::builtin::{ - bench, global_allocator, test, test_case, RustcDecodable, RustcEncodable, + alloc_error_handler, bench, derive, global_allocator, test, test_case, }; -#[cfg(not(bootstrap))] +#[unstable(feature = "derive_const", issue = "none")] +pub use crate::macros::builtin::derive_const; + #[unstable( feature = "cfg_accessible", issue = "64797", reason = "`cfg_accessible` is not fully implemented" )] -#[doc(no_inline)] pub use crate::macros::builtin::cfg_accessible; + +#[unstable( + feature = "cfg_eval", + issue = "82679", + reason = "`cfg_eval` is a recently implemented feature" +)] +pub use crate::macros::builtin::cfg_eval; + +#[unstable( + feature = "type_ascription", + issue = "23416", + reason = "placeholder syntax for type ascription" +)] +pub use crate::macros::builtin::type_ascribe; diff --git a/crux-mir/lib/core/src/primitive_docs.rs b/crux-mir/lib/core/src/primitive_docs.rs new file mode 100644 index 000000000..d6e9da187 --- /dev/null +++ b/crux-mir/lib/core/src/primitive_docs.rs @@ -0,0 +1,1575 @@ +// `library/{std,core}/src/primitive_docs.rs` should have the same contents. +// These are different files so that relative links work properly without +// having to have `CARGO_PKG_NAME` set, but conceptually they should always be the same. +#[doc(primitive = "bool")] +#[doc(alias = "true")] +#[doc(alias = "false")] +/// The boolean type. +/// +/// The `bool` represents a value, which could only be either [`true`] or [`false`]. If you cast +/// a `bool` into an integer, [`true`] will be 1 and [`false`] will be 0. +/// +/// # Basic usage +/// +/// `bool` implements various traits, such as [`BitAnd`], [`BitOr`], [`Not`], etc., +/// which allow us to perform boolean operations using `&`, `|` and `!`. +/// +/// [`if`] requires a `bool` value as its conditional. [`assert!`], which is an +/// important macro in testing, checks whether an expression is [`true`] and panics +/// if it isn't. +/// +/// ``` +/// let bool_val = true & false | false; +/// assert!(!bool_val); +/// ``` +/// +/// [`true`]: ../std/keyword.true.html +/// [`false`]: ../std/keyword.false.html +/// [`BitAnd`]: ops::BitAnd +/// [`BitOr`]: ops::BitOr +/// [`Not`]: ops::Not +/// [`if`]: ../std/keyword.if.html +/// +/// # Examples +/// +/// A trivial example of the usage of `bool`: +/// +/// ``` +/// let praise_the_borrow_checker = true; +/// +/// // using the `if` conditional +/// if praise_the_borrow_checker { +/// println!("oh, yeah!"); +/// } else { +/// println!("what?!!"); +/// } +/// +/// // ... or, a match pattern +/// match praise_the_borrow_checker { +/// true => println!("keep praising!"), +/// false => println!("you should praise!"), +/// } +/// ``` +/// +/// Also, since `bool` implements the [`Copy`] trait, we don't +/// have to worry about the move semantics (just like the integer and float primitives). +/// +/// Now an example of `bool` cast to integer type: +/// +/// ``` +/// assert_eq!(true as i32, 1); +/// assert_eq!(false as i32, 0); +/// ``` +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_bool {} + +#[doc(primitive = "never")] +#[doc(alias = "!")] +// +/// The `!` type, also called "never". +/// +/// `!` represents the type of computations which never resolve to any value at all. For example, +/// the [`exit`] function `fn exit(code: i32) -> !` exits the process without ever returning, and +/// so returns `!`. +/// +/// `break`, `continue` and `return` expressions also have type `!`. For example we are allowed to +/// write: +/// +/// ``` +/// #![feature(never_type)] +/// # fn foo() -> u32 { +/// let x: ! = { +/// return 123 +/// }; +/// # } +/// ``` +/// +/// Although the `let` is pointless here, it illustrates the meaning of `!`. Since `x` is never +/// assigned a value (because `return` returns from the entire function), `x` can be given type +/// `!`. We could also replace `return 123` with a `panic!` or a never-ending `loop` and this code +/// would still be valid. +/// +/// A more realistic usage of `!` is in this code: +/// +/// ``` +/// # fn get_a_number() -> Option { None } +/// # loop { +/// let num: u32 = match get_a_number() { +/// Some(num) => num, +/// None => break, +/// }; +/// # } +/// ``` +/// +/// Both match arms must produce values of type [`u32`], but since `break` never produces a value +/// at all we know it can never produce a value which isn't a [`u32`]. This illustrates another +/// behaviour of the `!` type - expressions with type `!` will coerce into any other type. +/// +/// [`u32`]: prim@u32 +#[doc = concat!("[`exit`]: ", include_str!("../primitive_docs/process_exit.md"))] +/// +/// # `!` and generics +/// +/// ## Infallible errors +/// +/// The main place you'll see `!` used explicitly is in generic code. Consider the [`FromStr`] +/// trait: +/// +/// ``` +/// trait FromStr: Sized { +/// type Err; +/// fn from_str(s: &str) -> Result; +/// } +/// ``` +/// +/// When implementing this trait for [`String`] we need to pick a type for [`Err`]. And since +/// converting a string into a string will never result in an error, the appropriate type is `!`. +/// (Currently the type actually used is an enum with no variants, though this is only because `!` +/// was added to Rust at a later date and it may change in the future.) With an [`Err`] type of +/// `!`, if we have to call [`String::from_str`] for some reason the result will be a +/// [`Result`] which we can unpack like this: +/// +/// ``` +/// #![feature(exhaustive_patterns)] +/// use std::str::FromStr; +/// let Ok(s) = String::from_str("hello"); +/// ``` +/// +/// Since the [`Err`] variant contains a `!`, it can never occur. If the `exhaustive_patterns` +/// feature is present this means we can exhaustively match on [`Result`] by just taking the +/// [`Ok`] variant. This illustrates another behaviour of `!` - it can be used to "delete" certain +/// enum variants from generic types like `Result`. +/// +/// ## Infinite loops +/// +/// While [`Result`] is very useful for removing errors, `!` can also be used to remove +/// successes as well. If we think of [`Result`] as "if this function returns, it has not +/// errored," we get a very intuitive idea of [`Result`] as well: if the function returns, it +/// *has* errored. +/// +/// For example, consider the case of a simple web server, which can be simplified to: +/// +/// ```ignore (hypothetical-example) +/// loop { +/// let (client, request) = get_request().expect("disconnected"); +/// let response = request.process(); +/// response.send(client); +/// } +/// ``` +/// +/// Currently, this isn't ideal, because we simply panic whenever we fail to get a new connection. +/// Instead, we'd like to keep track of this error, like this: +/// +/// ```ignore (hypothetical-example) +/// loop { +/// match get_request() { +/// Err(err) => break err, +/// Ok((client, request)) => { +/// let response = request.process(); +/// response.send(client); +/// }, +/// } +/// } +/// ``` +/// +/// Now, when the server disconnects, we exit the loop with an error instead of panicking. While it +/// might be intuitive to simply return the error, we might want to wrap it in a [`Result`] +/// instead: +/// +/// ```ignore (hypothetical-example) +/// fn server_loop() -> Result { +/// loop { +/// let (client, request) = get_request()?; +/// let response = request.process(); +/// response.send(client); +/// } +/// } +/// ``` +/// +/// Now, we can use `?` instead of `match`, and the return type makes a lot more sense: if the loop +/// ever stops, it means that an error occurred. We don't even have to wrap the loop in an `Ok` +/// because `!` coerces to `Result` automatically. +/// +/// [`String::from_str`]: str::FromStr::from_str +#[doc = concat!("[`String`]: ", include_str!("../primitive_docs/string_string.md"))] +/// [`FromStr`]: str::FromStr +/// +/// # `!` and traits +/// +/// When writing your own traits, `!` should have an `impl` whenever there is an obvious `impl` +/// which doesn't `panic!`. The reason is that functions returning an `impl Trait` where `!` +/// does not have an `impl` of `Trait` cannot diverge as their only possible code path. In other +/// words, they can't return `!` from every code path. As an example, this code doesn't compile: +/// +/// ```compile_fail +/// use std::ops::Add; +/// +/// fn foo() -> impl Add { +/// unimplemented!() +/// } +/// ``` +/// +/// But this code does: +/// +/// ``` +/// use std::ops::Add; +/// +/// fn foo() -> impl Add { +/// if true { +/// unimplemented!() +/// } else { +/// 0 +/// } +/// } +/// ``` +/// +/// The reason is that, in the first example, there are many possible types that `!` could coerce +/// to, because many types implement `Add`. However, in the second example, +/// the `else` branch returns a `0`, which the compiler infers from the return type to be of type +/// `u32`. Since `u32` is a concrete type, `!` can and will be coerced to it. See issue [#36375] +/// for more information on this quirk of `!`. +/// +/// [#36375]: https://github.com/rust-lang/rust/issues/36375 +/// +/// As it turns out, though, most traits can have an `impl` for `!`. Take [`Debug`] +/// for example: +/// +/// ``` +/// #![feature(never_type)] +/// # use std::fmt; +/// # trait Debug { +/// # fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result; +/// # } +/// impl Debug for ! { +/// fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { +/// *self +/// } +/// } +/// ``` +/// +/// Once again we're using `!`'s ability to coerce into any other type, in this case +/// [`fmt::Result`]. Since this method takes a `&!` as an argument we know that it can never be +/// called (because there is no value of type `!` for it to be called with). Writing `*self` +/// essentially tells the compiler "We know that this code can never be run, so just treat the +/// entire function body as having type [`fmt::Result`]". This pattern can be used a lot when +/// implementing traits for `!`. Generally, any trait which only has methods which take a `self` +/// parameter should have such an impl. +/// +/// On the other hand, one trait which would not be appropriate to implement is [`Default`]: +/// +/// ``` +/// trait Default { +/// fn default() -> Self; +/// } +/// ``` +/// +/// Since `!` has no values, it has no default value either. It's true that we could write an +/// `impl` for this which simply panics, but the same is true for any type (we could `impl +/// Default` for (eg.) [`File`] by just making [`default()`] panic.) +/// +#[doc = concat!("[`File`]: ", include_str!("../primitive_docs/fs_file.md"))] +/// [`Debug`]: fmt::Debug +/// [`default()`]: Default::default +/// +#[unstable(feature = "never_type", issue = "35121")] +mod prim_never {} + +#[doc(primitive = "char")] +#[allow(rustdoc::invalid_rust_codeblocks)] +/// A character type. +/// +/// The `char` type represents a single character. More specifically, since +/// 'character' isn't a well-defined concept in Unicode, `char` is a '[Unicode +/// scalar value]'. +/// +/// This documentation describes a number of methods and trait implementations on the +/// `char` type. For technical reasons, there is additional, separate +/// documentation in [the `std::char` module](char/index.html) as well. +/// +/// # Validity +/// +/// A `char` is a '[Unicode scalar value]', which is any '[Unicode code point]' +/// other than a [surrogate code point]. This has a fixed numerical definition: +/// code points are in the range 0 to 0x10FFFF, inclusive. +/// Surrogate code points, used by UTF-16, are in the range 0xD800 to 0xDFFF. +/// +/// No `char` may be constructed, whether as a literal or at runtime, that is not a +/// Unicode scalar value: +/// +/// ```compile_fail +/// // Each of these is a compiler error +/// ['\u{D800}', '\u{DFFF}', '\u{110000}']; +/// ``` +/// +/// ```should_panic +/// // Panics; from_u32 returns None. +/// char::from_u32(0xDE01).unwrap(); +/// ``` +/// +/// ```no_run +/// // Undefined behaviour +/// unsafe { char::from_u32_unchecked(0x110000) }; +/// ``` +/// +/// USVs are also the exact set of values that may be encoded in UTF-8. Because +/// `char` values are USVs and `str` values are valid UTF-8, it is safe to store +/// any `char` in a `str` or read any character from a `str` as a `char`. +/// +/// The gap in valid `char` values is understood by the compiler, so in the +/// below example the two ranges are understood to cover the whole range of +/// possible `char` values and there is no error for a [non-exhaustive match]. +/// +/// ``` +/// let c: char = 'a'; +/// match c { +/// '\0' ..= '\u{D7FF}' => false, +/// '\u{E000}' ..= '\u{10FFFF}' => true, +/// }; +/// ``` +/// +/// All USVs are valid `char` values, but not all of them represent a real +/// character. Many USVs are not currently assigned to a character, but may be +/// in the future ("reserved"); some will never be a character +/// ("noncharacters"); and some may be given different meanings by different +/// users ("private use"). +/// +/// [Unicode code point]: https://www.unicode.org/glossary/#code_point +/// [Unicode scalar value]: https://www.unicode.org/glossary/#unicode_scalar_value +/// [non-exhaustive match]: ../book/ch06-02-match.html#matches-are-exhaustive +/// [surrogate code point]: https://www.unicode.org/glossary/#surrogate_code_point +/// +/// # Representation +/// +/// `char` is always four bytes in size. This is a different representation than +/// a given character would have as part of a [`String`]. For example: +/// +/// ``` +/// let v = vec!['h', 'e', 'l', 'l', 'o']; +/// +/// // five elements times four bytes for each element +/// assert_eq!(20, v.len() * std::mem::size_of::()); +/// +/// let s = String::from("hello"); +/// +/// // five elements times one byte per element +/// assert_eq!(5, s.len() * std::mem::size_of::()); +/// ``` +/// +#[doc = concat!("[`String`]: ", include_str!("../primitive_docs/string_string.md"))] +/// +/// As always, remember that a human intuition for 'character' might not map to +/// Unicode's definitions. For example, despite looking similar, the 'é' +/// character is one Unicode code point while 'é' is two Unicode code points: +/// +/// ``` +/// let mut chars = "é".chars(); +/// // U+00e9: 'latin small letter e with acute' +/// assert_eq!(Some('\u{00e9}'), chars.next()); +/// assert_eq!(None, chars.next()); +/// +/// let mut chars = "é".chars(); +/// // U+0065: 'latin small letter e' +/// assert_eq!(Some('\u{0065}'), chars.next()); +/// // U+0301: 'combining acute accent' +/// assert_eq!(Some('\u{0301}'), chars.next()); +/// assert_eq!(None, chars.next()); +/// ``` +/// +/// This means that the contents of the first string above _will_ fit into a +/// `char` while the contents of the second string _will not_. Trying to create +/// a `char` literal with the contents of the second string gives an error: +/// +/// ```text +/// error: character literal may only contain one codepoint: 'é' +/// let c = 'é'; +/// ^^^ +/// ``` +/// +/// Another implication of the 4-byte fixed size of a `char` is that +/// per-`char` processing can end up using a lot more memory: +/// +/// ``` +/// let s = String::from("love: ❤️"); +/// let v: Vec = s.chars().collect(); +/// +/// assert_eq!(12, std::mem::size_of_val(&s[..])); +/// assert_eq!(32, std::mem::size_of_val(&v[..])); +/// ``` +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_char {} + +#[doc(primitive = "unit")] +#[doc(alias = "(")] +#[doc(alias = ")")] +#[doc(alias = "()")] +// +/// The `()` type, also called "unit". +/// +/// The `()` type has exactly one value `()`, and is used when there +/// is no other meaningful value that could be returned. `()` is most +/// commonly seen implicitly: functions without a `-> ...` implicitly +/// have return type `()`, that is, these are equivalent: +/// +/// ```rust +/// fn long() -> () {} +/// +/// fn short() {} +/// ``` +/// +/// The semicolon `;` can be used to discard the result of an +/// expression at the end of a block, making the expression (and thus +/// the block) evaluate to `()`. For example, +/// +/// ```rust +/// fn returns_i64() -> i64 { +/// 1i64 +/// } +/// fn returns_unit() { +/// 1i64; +/// } +/// +/// let is_i64 = { +/// returns_i64() +/// }; +/// let is_unit = { +/// returns_i64(); +/// }; +/// ``` +/// +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_unit {} + +// Required to make auto trait impls render. +// See src/librustdoc/passes/collect_trait_impls.rs:collect_trait_impls +#[doc(hidden)] +impl () {} + +// Fake impl that's only really used for docs. +#[cfg(doc)] +#[stable(feature = "rust1", since = "1.0.0")] +impl Clone for () { + fn clone(&self) -> Self { + loop {} + } +} + +// Fake impl that's only really used for docs. +#[cfg(doc)] +#[stable(feature = "rust1", since = "1.0.0")] +impl Copy for () { + // empty +} + +#[doc(primitive = "pointer")] +#[doc(alias = "ptr")] +#[doc(alias = "*")] +#[doc(alias = "*const")] +#[doc(alias = "*mut")] +// +/// Raw, unsafe pointers, `*const T`, and `*mut T`. +/// +/// *[See also the `std::ptr` module](ptr).* +/// +/// Working with raw pointers in Rust is uncommon, typically limited to a few patterns. +/// Raw pointers can be unaligned or [`null`]. However, when a raw pointer is +/// dereferenced (using the `*` operator), it must be non-null and aligned. +/// +/// Storing through a raw pointer using `*ptr = data` calls `drop` on the old value, so +/// [`write`] must be used if the type has drop glue and memory is not already +/// initialized - otherwise `drop` would be called on the uninitialized memory. +/// +/// Use the [`null`] and [`null_mut`] functions to create null pointers, and the +/// [`is_null`] method of the `*const T` and `*mut T` types to check for null. +/// The `*const T` and `*mut T` types also define the [`offset`] method, for +/// pointer math. +/// +/// # Common ways to create raw pointers +/// +/// ## 1. Coerce a reference (`&T`) or mutable reference (`&mut T`). +/// +/// ``` +/// let my_num: i32 = 10; +/// let my_num_ptr: *const i32 = &my_num; +/// let mut my_speed: i32 = 88; +/// let my_speed_ptr: *mut i32 = &mut my_speed; +/// ``` +/// +/// To get a pointer to a boxed value, dereference the box: +/// +/// ``` +/// let my_num: Box = Box::new(10); +/// let my_num_ptr: *const i32 = &*my_num; +/// let mut my_speed: Box = Box::new(88); +/// let my_speed_ptr: *mut i32 = &mut *my_speed; +/// ``` +/// +/// This does not take ownership of the original allocation +/// and requires no resource management later, +/// but you must not use the pointer after its lifetime. +/// +/// ## 2. Consume a box (`Box`). +/// +/// The [`into_raw`] function consumes a box and returns +/// the raw pointer. It doesn't destroy `T` or deallocate any memory. +/// +/// ``` +/// let my_speed: Box = Box::new(88); +/// let my_speed: *mut i32 = Box::into_raw(my_speed); +/// +/// // By taking ownership of the original `Box` though +/// // we are obligated to put it together later to be destroyed. +/// unsafe { +/// drop(Box::from_raw(my_speed)); +/// } +/// ``` +/// +/// Note that here the call to [`drop`] is for clarity - it indicates +/// that we are done with the given value and it should be destroyed. +/// +/// ## 3. Create it using `ptr::addr_of!` +/// +/// Instead of coercing a reference to a raw pointer, you can use the macros +/// [`ptr::addr_of!`] (for `*const T`) and [`ptr::addr_of_mut!`] (for `*mut T`). +/// These macros allow you to create raw pointers to fields to which you cannot +/// create a reference (without causing undefined behaviour), such as an +/// unaligned field. This might be necessary if packed structs or uninitialized +/// memory is involved. +/// +/// ``` +/// #[derive(Debug, Default, Copy, Clone)] +/// #[repr(C, packed)] +/// struct S { +/// aligned: u8, +/// unaligned: u32, +/// } +/// let s = S::default(); +/// let p = std::ptr::addr_of!(s.unaligned); // not allowed with coercion +/// ``` +/// +/// ## 4. Get it from C. +/// +/// ``` +/// # #![feature(rustc_private)] +/// extern crate libc; +/// +/// use std::mem; +/// +/// unsafe { +/// let my_num: *mut i32 = libc::malloc(mem::size_of::()) as *mut i32; +/// if my_num.is_null() { +/// panic!("failed to allocate memory"); +/// } +/// libc::free(my_num as *mut libc::c_void); +/// } +/// ``` +/// +/// Usually you wouldn't literally use `malloc` and `free` from Rust, +/// but C APIs hand out a lot of pointers generally, so are a common source +/// of raw pointers in Rust. +/// +/// [`null`]: ptr::null +/// [`null_mut`]: ptr::null_mut +/// [`is_null`]: pointer::is_null +/// [`offset`]: pointer::offset +#[doc = concat!("[`into_raw`]: ", include_str!("../primitive_docs/box_into_raw.md"))] +/// [`drop`]: mem::drop +/// [`write`]: ptr::write +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_pointer {} + +#[doc(primitive = "array")] +#[doc(alias = "[]")] +#[doc(alias = "[T;N]")] // unfortunately, rustdoc doesn't have fuzzy search for aliases +#[doc(alias = "[T; N]")] +/// A fixed-size array, denoted `[T; N]`, for the element type, `T`, and the +/// non-negative compile-time constant size, `N`. +/// +/// There are two syntactic forms for creating an array: +/// +/// * A list with each element, i.e., `[x, y, z]`. +/// * A repeat expression `[x; N]`, which produces an array with `N` copies of `x`. +/// The type of `x` must be [`Copy`]. +/// +/// Note that `[expr; 0]` is allowed, and produces an empty array. +/// This will still evaluate `expr`, however, and immediately drop the resulting value, so +/// be mindful of side effects. +/// +/// Arrays of *any* size implement the following traits if the element type allows it: +/// +/// - [`Copy`] +/// - [`Clone`] +/// - [`Debug`] +/// - [`IntoIterator`] (implemented for `[T; N]`, `&[T; N]` and `&mut [T; N]`) +/// - [`PartialEq`], [`PartialOrd`], [`Eq`], [`Ord`] +/// - [`Hash`] +/// - [`AsRef`], [`AsMut`] +/// - [`Borrow`], [`BorrowMut`] +/// +/// Arrays of sizes from 0 to 32 (inclusive) implement the [`Default`] trait +/// if the element type allows it. As a stopgap, trait implementations are +/// statically generated up to size 32. +/// +/// Arrays coerce to [slices (`[T]`)][slice], so a slice method may be called on +/// an array. Indeed, this provides most of the API for working with arrays. +/// +/// Slices have a dynamic size and do not coerce to arrays. Instead, use +/// `slice.try_into().unwrap()` or `::try_from(slice).unwrap()`. +/// +/// Array's `try_from(slice)` implementations (and the corresponding `slice.try_into()` +/// array implementations) succeed if the input slice length is the same as the result +/// array length. They optimize especially well when the optimizer can easily determine +/// the slice length, e.g. `<[u8; 4]>::try_from(&slice[4..8]).unwrap()`. Array implements +/// [TryFrom](crate::convert::TryFrom) returning: +/// +/// - `[T; N]` copies from the slice's elements +/// - `&[T; N]` references the original slice's elements +/// - `&mut [T; N]` references the original slice's elements +/// +/// You can move elements out of an array with a [slice pattern]. If you want +/// one element, see [`mem::replace`]. +/// +/// # Examples +/// +/// ``` +/// let mut array: [i32; 3] = [0; 3]; +/// +/// array[1] = 1; +/// array[2] = 2; +/// +/// assert_eq!([1, 2], &array[1..]); +/// +/// // This loop prints: 0 1 2 +/// for x in array { +/// print!("{x} "); +/// } +/// ``` +/// +/// You can also iterate over reference to the array's elements: +/// +/// ``` +/// let array: [i32; 3] = [0; 3]; +/// +/// for x in &array { } +/// ``` +/// +/// You can use `::try_from(slice)` or `slice.try_into()` to get an array from +/// a slice: +/// +/// ``` +/// let bytes: [u8; 3] = [1, 0, 2]; +/// assert_eq!(1, u16::from_le_bytes(<[u8; 2]>::try_from(&bytes[0..2]).unwrap())); +/// assert_eq!(512, u16::from_le_bytes(bytes[1..3].try_into().unwrap())); +/// ``` +/// +/// You can use a [slice pattern] to move elements out of an array: +/// +/// ``` +/// fn move_away(_: String) { /* Do interesting things. */ } +/// +/// let [john, roa] = ["John".to_string(), "Roa".to_string()]; +/// move_away(john); +/// move_away(roa); +/// ``` +/// +/// # Editions +/// +/// Prior to Rust 1.53, arrays did not implement [`IntoIterator`] by value, so the method call +/// `array.into_iter()` auto-referenced into a [slice iterator](slice::iter). Right now, the old +/// behavior is preserved in the 2015 and 2018 editions of Rust for compatibility, ignoring +/// [`IntoIterator`] by value. In the future, the behavior on the 2015 and 2018 edition +/// might be made consistent to the behavior of later editions. +/// +/// ```rust,edition2018 +/// // Rust 2015 and 2018: +/// +/// # #![allow(array_into_iter)] // override our `deny(warnings)` +/// let array: [i32; 3] = [0; 3]; +/// +/// // This creates a slice iterator, producing references to each value. +/// for item in array.into_iter().enumerate() { +/// let (i, x): (usize, &i32) = item; +/// println!("array[{i}] = {x}"); +/// } +/// +/// // The `array_into_iter` lint suggests this change for future compatibility: +/// for item in array.iter().enumerate() { +/// let (i, x): (usize, &i32) = item; +/// println!("array[{i}] = {x}"); +/// } +/// +/// // You can explicitly iterate an array by value using `IntoIterator::into_iter` +/// for item in IntoIterator::into_iter(array).enumerate() { +/// let (i, x): (usize, i32) = item; +/// println!("array[{i}] = {x}"); +/// } +/// ``` +/// +/// Starting in the 2021 edition, `array.into_iter()` uses `IntoIterator` normally to iterate +/// by value, and `iter()` should be used to iterate by reference like previous editions. +/// +/// ```rust,edition2021 +/// // Rust 2021: +/// +/// let array: [i32; 3] = [0; 3]; +/// +/// // This iterates by reference: +/// for item in array.iter().enumerate() { +/// let (i, x): (usize, &i32) = item; +/// println!("array[{i}] = {x}"); +/// } +/// +/// // This iterates by value: +/// for item in array.into_iter().enumerate() { +/// let (i, x): (usize, i32) = item; +/// println!("array[{i}] = {x}"); +/// } +/// ``` +/// +/// Future language versions might start treating the `array.into_iter()` +/// syntax on editions 2015 and 2018 the same as on edition 2021. So code using +/// those older editions should still be written with this change in mind, to +/// prevent breakage in the future. The safest way to accomplish this is to +/// avoid the `into_iter` syntax on those editions. If an edition update is not +/// viable/desired, there are multiple alternatives: +/// * use `iter`, equivalent to the old behavior, creating references +/// * use [`IntoIterator::into_iter`], equivalent to the post-2021 behavior (Rust 1.53+) +/// * replace `for ... in array.into_iter() {` with `for ... in array {`, +/// equivalent to the post-2021 behavior (Rust 1.53+) +/// +/// ```rust,edition2018 +/// // Rust 2015 and 2018: +/// +/// let array: [i32; 3] = [0; 3]; +/// +/// // This iterates by reference: +/// for item in array.iter() { +/// let x: &i32 = item; +/// println!("{x}"); +/// } +/// +/// // This iterates by value: +/// for item in IntoIterator::into_iter(array) { +/// let x: i32 = item; +/// println!("{x}"); +/// } +/// +/// // This iterates by value: +/// for item in array { +/// let x: i32 = item; +/// println!("{x}"); +/// } +/// +/// // IntoIter can also start a chain. +/// // This iterates by value: +/// for item in IntoIterator::into_iter(array).enumerate() { +/// let (i, x): (usize, i32) = item; +/// println!("array[{i}] = {x}"); +/// } +/// ``` +/// +/// [slice]: prim@slice +/// [`Debug`]: fmt::Debug +/// [`Hash`]: hash::Hash +/// [`Borrow`]: borrow::Borrow +/// [`BorrowMut`]: borrow::BorrowMut +/// [slice pattern]: ../reference/patterns.html#slice-patterns +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_array {} + +#[doc(primitive = "slice")] +#[doc(alias = "[")] +#[doc(alias = "]")] +#[doc(alias = "[]")] +/// A dynamically-sized view into a contiguous sequence, `[T]`. Contiguous here +/// means that elements are laid out so that every element is the same +/// distance from its neighbors. +/// +/// *[See also the `std::slice` module](crate::slice).* +/// +/// Slices are a view into a block of memory represented as a pointer and a +/// length. +/// +/// ``` +/// // slicing a Vec +/// let vec = vec![1, 2, 3]; +/// let int_slice = &vec[..]; +/// // coercing an array to a slice +/// let str_slice: &[&str] = &["one", "two", "three"]; +/// ``` +/// +/// Slices are either mutable or shared. The shared slice type is `&[T]`, +/// while the mutable slice type is `&mut [T]`, where `T` represents the element +/// type. For example, you can mutate the block of memory that a mutable slice +/// points to: +/// +/// ``` +/// let mut x = [1, 2, 3]; +/// let x = &mut x[..]; // Take a full slice of `x`. +/// x[1] = 7; +/// assert_eq!(x, &[1, 7, 3]); +/// ``` +/// +/// As slices store the length of the sequence they refer to, they have twice +/// the size of pointers to [`Sized`](marker/trait.Sized.html) types. +/// Also see the reference on +/// [dynamically sized types](../reference/dynamically-sized-types.html). +/// +/// ``` +/// # use std::rc::Rc; +/// let pointer_size = std::mem::size_of::<&u8>(); +/// assert_eq!(2 * pointer_size, std::mem::size_of::<&[u8]>()); +/// assert_eq!(2 * pointer_size, std::mem::size_of::<*const [u8]>()); +/// assert_eq!(2 * pointer_size, std::mem::size_of::>()); +/// assert_eq!(2 * pointer_size, std::mem::size_of::>()); +/// ``` +/// +/// ## Trait Implementations +/// +/// Some traits are implemented for slices if the element type implements +/// that trait. This includes [`Eq`], [`Hash`] and [`Ord`]. +/// +/// ## Iteration +/// +/// The slices implement `IntoIterator`. The iterator yields references to the +/// slice elements. +/// +/// ``` +/// let numbers: &[i32] = &[0, 1, 2]; +/// for n in numbers { +/// println!("{n} is a number!"); +/// } +/// ``` +/// +/// The mutable slice yields mutable references to the elements: +/// +/// ``` +/// let mut scores: &mut [i32] = &mut [7, 8, 9]; +/// for score in scores { +/// *score += 1; +/// } +/// ``` +/// +/// This iterator yields mutable references to the slice's elements, so while +/// the element type of the slice is `i32`, the element type of the iterator is +/// `&mut i32`. +/// +/// * [`.iter`] and [`.iter_mut`] are the explicit methods to return the default +/// iterators. +/// * Further methods that return iterators are [`.split`], [`.splitn`], +/// [`.chunks`], [`.windows`] and more. +/// +/// [`Hash`]: core::hash::Hash +/// [`.iter`]: slice::iter +/// [`.iter_mut`]: slice::iter_mut +/// [`.split`]: slice::split +/// [`.splitn`]: slice::splitn +/// [`.chunks`]: slice::chunks +/// [`.windows`]: slice::windows +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_slice {} + +#[doc(primitive = "str")] +/// String slices. +/// +/// *[See also the `std::str` module](crate::str).* +/// +/// The `str` type, also called a 'string slice', is the most primitive string +/// type. It is usually seen in its borrowed form, `&str`. It is also the type +/// of string literals, `&'static str`. +/// +/// String slices are always valid UTF-8. +/// +/// # Basic Usage +/// +/// String literals are string slices: +/// +/// ``` +/// let hello_world = "Hello, World!"; +/// ``` +/// +/// Here we have declared a string slice initialized with a string literal. +/// String literals have a static lifetime, which means the string `hello_world` +/// is guaranteed to be valid for the duration of the entire program. +/// We can explicitly specify `hello_world`'s lifetime as well: +/// +/// ``` +/// let hello_world: &'static str = "Hello, world!"; +/// ``` +/// +/// # Representation +/// +/// A `&str` is made up of two components: a pointer to some bytes, and a +/// length. You can look at these with the [`as_ptr`] and [`len`] methods: +/// +/// ``` +/// use std::slice; +/// use std::str; +/// +/// let story = "Once upon a time..."; +/// +/// let ptr = story.as_ptr(); +/// let len = story.len(); +/// +/// // story has nineteen bytes +/// assert_eq!(19, len); +/// +/// // We can re-build a str out of ptr and len. This is all unsafe because +/// // we are responsible for making sure the two components are valid: +/// let s = unsafe { +/// // First, we build a &[u8]... +/// let slice = slice::from_raw_parts(ptr, len); +/// +/// // ... and then convert that slice into a string slice +/// str::from_utf8(slice) +/// }; +/// +/// assert_eq!(s, Ok(story)); +/// ``` +/// +/// [`as_ptr`]: str::as_ptr +/// [`len`]: str::len +/// +/// Note: This example shows the internals of `&str`. `unsafe` should not be +/// used to get a string slice under normal circumstances. Use `as_str` +/// instead. +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_str {} + +#[doc(primitive = "tuple")] +#[doc(alias = "(")] +#[doc(alias = ")")] +#[doc(alias = "()")] +// +/// A finite heterogeneous sequence, `(T, U, ..)`. +/// +/// Let's cover each of those in turn: +/// +/// Tuples are *finite*. In other words, a tuple has a length. Here's a tuple +/// of length `3`: +/// +/// ``` +/// ("hello", 5, 'c'); +/// ``` +/// +/// 'Length' is also sometimes called 'arity' here; each tuple of a different +/// length is a different, distinct type. +/// +/// Tuples are *heterogeneous*. This means that each element of the tuple can +/// have a different type. In that tuple above, it has the type: +/// +/// ``` +/// # let _: +/// (&'static str, i32, char) +/// # = ("hello", 5, 'c'); +/// ``` +/// +/// Tuples are a *sequence*. This means that they can be accessed by position; +/// this is called 'tuple indexing', and it looks like this: +/// +/// ```rust +/// let tuple = ("hello", 5, 'c'); +/// +/// assert_eq!(tuple.0, "hello"); +/// assert_eq!(tuple.1, 5); +/// assert_eq!(tuple.2, 'c'); +/// ``` +/// +/// The sequential nature of the tuple applies to its implementations of various +/// traits. For example, in [`PartialOrd`] and [`Ord`], the elements are compared +/// sequentially until the first non-equal set is found. +/// +/// For more about tuples, see [the book](../book/ch03-02-data-types.html#the-tuple-type). +/// +// Hardcoded anchor in src/librustdoc/html/format.rs +// linked to as `#trait-implementations-1` +/// # Trait implementations +/// +/// In this documentation the shorthand `(T₁, T₂, …, Tₙ)` is used to represent tuples of varying +/// length. When that is used, any trait bound expressed on `T` applies to each element of the +/// tuple independently. Note that this is a convenience notation to avoid repetitive +/// documentation, not valid Rust syntax. +/// +/// Due to a temporary restriction in Rust’s type system, the following traits are only +/// implemented on tuples of arity 12 or less. In the future, this may change: +/// +/// * [`PartialEq`] +/// * [`Eq`] +/// * [`PartialOrd`] +/// * [`Ord`] +/// * [`Debug`] +/// * [`Default`] +/// * [`Hash`] +/// +/// [`Debug`]: fmt::Debug +/// [`Hash`]: hash::Hash +/// +/// The following traits are implemented for tuples of any length. These traits have +/// implementations that are automatically generated by the compiler, so are not limited by +/// missing language features. +/// +/// * [`Clone`] +/// * [`Copy`] +/// * [`Send`] +/// * [`Sync`] +/// * [`Unpin`] +/// * [`UnwindSafe`] +/// * [`RefUnwindSafe`] +/// +/// [`Unpin`]: marker::Unpin +/// [`UnwindSafe`]: panic::UnwindSafe +/// [`RefUnwindSafe`]: panic::RefUnwindSafe +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// let tuple = ("hello", 5, 'c'); +/// +/// assert_eq!(tuple.0, "hello"); +/// ``` +/// +/// Tuples are often used as a return type when you want to return more than +/// one value: +/// +/// ``` +/// fn calculate_point() -> (i32, i32) { +/// // Don't do a calculation, that's not the point of the example +/// (4, 5) +/// } +/// +/// let point = calculate_point(); +/// +/// assert_eq!(point.0, 4); +/// assert_eq!(point.1, 5); +/// +/// // Combining this with patterns can be nicer. +/// +/// let (x, y) = calculate_point(); +/// +/// assert_eq!(x, 4); +/// assert_eq!(y, 5); +/// ``` +/// +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_tuple {} + +// Required to make auto trait impls render. +// See src/librustdoc/passes/collect_trait_impls.rs:collect_trait_impls +#[doc(hidden)] +impl (T,) {} + +// Fake impl that's only really used for docs. +#[cfg(doc)] +#[stable(feature = "rust1", since = "1.0.0")] +#[doc(fake_variadic)] +/// This trait is implemented on arbitrary-length tuples. +impl Clone for (T,) { + fn clone(&self) -> Self { + loop {} + } +} + +// Fake impl that's only really used for docs. +#[cfg(doc)] +#[stable(feature = "rust1", since = "1.0.0")] +#[doc(fake_variadic)] +/// This trait is implemented on arbitrary-length tuples. +impl Copy for (T,) { + // empty +} + +#[doc(primitive = "f32")] +/// A 32-bit floating point type (specifically, the "binary32" type defined in IEEE 754-2008). +/// +/// This type can represent a wide range of decimal numbers, like `3.5`, `27`, +/// `-113.75`, `0.0078125`, `34359738368`, `0`, `-1`. So unlike integer types +/// (such as `i32`), floating point types can represent non-integer numbers, +/// too. +/// +/// However, being able to represent this wide range of numbers comes at the +/// cost of precision: floats can only represent some of the real numbers and +/// calculation with floats round to a nearby representable number. For example, +/// `5.0` and `1.0` can be exactly represented as `f32`, but `1.0 / 5.0` results +/// in `0.20000000298023223876953125` since `0.2` cannot be exactly represented +/// as `f32`. Note, however, that printing floats with `println` and friends will +/// often discard insignificant digits: `println!("{}", 1.0f32 / 5.0f32)` will +/// print `0.2`. +/// +/// Additionally, `f32` can represent some special values: +/// +/// - −0.0: IEEE 754 floating point numbers have a bit that indicates their sign, so −0.0 is a +/// possible value. For comparison −0.0 = +0.0, but floating point operations can carry +/// the sign bit through arithmetic operations. This means −0.0 × +0.0 produces −0.0 and +/// a negative number rounded to a value smaller than a float can represent also produces −0.0. +/// - [∞](#associatedconstant.INFINITY) and +/// [−∞](#associatedconstant.NEG_INFINITY): these result from calculations +/// like `1.0 / 0.0`. +/// - [NaN (not a number)](#associatedconstant.NAN): this value results from +/// calculations like `(-1.0).sqrt()`. NaN has some potentially unexpected +/// behavior: +/// - It is unequal to any float, including itself! This is the reason `f32` +/// doesn't implement the `Eq` trait. +/// - It is also neither smaller nor greater than any float, making it +/// impossible to sort by the default comparison operation, which is the +/// reason `f32` doesn't implement the `Ord` trait. +/// - It is also considered *infectious* as almost all calculations where one +/// of the operands is NaN will also result in NaN. The explanations on this +/// page only explicitly document behavior on NaN operands if this default +/// is deviated from. +/// - Lastly, there are multiple bit patterns that are considered NaN. +/// Rust does not currently guarantee that the bit patterns of NaN are +/// preserved over arithmetic operations, and they are not guaranteed to be +/// portable or even fully deterministic! This means that there may be some +/// surprising results upon inspecting the bit patterns, +/// as the same calculations might produce NaNs with different bit patterns. +/// +/// When the number resulting from a primitive operation (addition, +/// subtraction, multiplication, or division) on this type is not exactly +/// representable as `f32`, it is rounded according to the roundTiesToEven +/// direction defined in IEEE 754-2008. That means: +/// +/// - The result is the representable value closest to the true value, if there +/// is a unique closest representable value. +/// - If the true value is exactly half-way between two representable values, +/// the result is the one with an even least-significant binary digit. +/// - If the true value's magnitude is ≥ `f32::MAX` + 2(`f32::MAX_EXP` − +/// `f32::MANTISSA_DIGITS` − 1), the result is ∞ or −∞ (preserving the +/// true value's sign). +/// +/// For more information on floating point numbers, see [Wikipedia][wikipedia]. +/// +/// *[See also the `std::f32::consts` module](crate::f32::consts).* +/// +/// [wikipedia]: https://en.wikipedia.org/wiki/Single-precision_floating-point_format +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_f32 {} + +#[doc(primitive = "f64")] +/// A 64-bit floating point type (specifically, the "binary64" type defined in IEEE 754-2008). +/// +/// This type is very similar to [`f32`], but has increased +/// precision by using twice as many bits. Please see [the documentation for +/// `f32`][`f32`] or [Wikipedia on double precision +/// values][wikipedia] for more information. +/// +/// *[See also the `std::f64::consts` module](crate::f64::consts).* +/// +/// [`f32`]: prim@f32 +/// [wikipedia]: https://en.wikipedia.org/wiki/Double-precision_floating-point_format +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_f64 {} + +#[doc(primitive = "i8")] +// +/// The 8-bit signed integer type. +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_i8 {} + +#[doc(primitive = "i16")] +// +/// The 16-bit signed integer type. +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_i16 {} + +#[doc(primitive = "i32")] +// +/// The 32-bit signed integer type. +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_i32 {} + +#[doc(primitive = "i64")] +// +/// The 64-bit signed integer type. +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_i64 {} + +#[doc(primitive = "i128")] +// +/// The 128-bit signed integer type. +#[stable(feature = "i128", since = "1.26.0")] +mod prim_i128 {} + +#[doc(primitive = "u8")] +// +/// The 8-bit unsigned integer type. +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_u8 {} + +#[doc(primitive = "u16")] +// +/// The 16-bit unsigned integer type. +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_u16 {} + +#[doc(primitive = "u32")] +// +/// The 32-bit unsigned integer type. +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_u32 {} + +#[doc(primitive = "u64")] +// +/// The 64-bit unsigned integer type. +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_u64 {} + +#[doc(primitive = "u128")] +// +/// The 128-bit unsigned integer type. +#[stable(feature = "i128", since = "1.26.0")] +mod prim_u128 {} + +#[doc(primitive = "isize")] +// +/// The pointer-sized signed integer type. +/// +/// The size of this primitive is how many bytes it takes to reference any +/// location in memory. For example, on a 32 bit target, this is 4 bytes +/// and on a 64 bit target, this is 8 bytes. +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_isize {} + +#[doc(primitive = "usize")] +// +/// The pointer-sized unsigned integer type. +/// +/// The size of this primitive is how many bytes it takes to reference any +/// location in memory. For example, on a 32 bit target, this is 4 bytes +/// and on a 64 bit target, this is 8 bytes. +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_usize {} + +#[doc(primitive = "reference")] +#[doc(alias = "&")] +#[doc(alias = "&mut")] +// +/// References, `&T` and `&mut T`. +/// +/// A reference represents a borrow of some owned value. You can get one by using the `&` or `&mut` +/// operators on a value, or by using a [`ref`](../std/keyword.ref.html) or +/// [ref](../std/keyword.ref.html) [mut](../std/keyword.mut.html) pattern. +/// +/// For those familiar with pointers, a reference is just a pointer that is assumed to be +/// aligned, not null, and pointing to memory containing a valid value of `T` - for example, +/// &[bool] can only point to an allocation containing the integer values `1` +/// ([`true`](../std/keyword.true.html)) or `0` ([`false`](../std/keyword.false.html)), but +/// creating a &[bool] that points to an allocation containing +/// the value `3` causes undefined behaviour. +/// In fact, [Option]\<&T> has the same memory representation as a +/// nullable but aligned pointer, and can be passed across FFI boundaries as such. +/// +/// In most cases, references can be used much like the original value. Field access, method +/// calling, and indexing work the same (save for mutability rules, of course). In addition, the +/// comparison operators transparently defer to the referent's implementation, allowing references +/// to be compared the same as owned values. +/// +/// References have a lifetime attached to them, which represents the scope for which the borrow is +/// valid. A lifetime is said to "outlive" another one if its representative scope is as long or +/// longer than the other. The `'static` lifetime is the longest lifetime, which represents the +/// total life of the program. For example, string literals have a `'static` lifetime because the +/// text data is embedded into the binary of the program, rather than in an allocation that needs +/// to be dynamically managed. +/// +/// `&mut T` references can be freely coerced into `&T` references with the same referent type, and +/// references with longer lifetimes can be freely coerced into references with shorter ones. +/// +/// Reference equality by address, instead of comparing the values pointed to, is accomplished via +/// implicit reference-pointer coercion and raw pointer equality via [`ptr::eq`], while +/// [`PartialEq`] compares values. +/// +/// ``` +/// use std::ptr; +/// +/// let five = 5; +/// let other_five = 5; +/// let five_ref = &five; +/// let same_five_ref = &five; +/// let other_five_ref = &other_five; +/// +/// assert!(five_ref == same_five_ref); +/// assert!(five_ref == other_five_ref); +/// +/// assert!(ptr::eq(five_ref, same_five_ref)); +/// assert!(!ptr::eq(five_ref, other_five_ref)); +/// ``` +/// +/// For more information on how to use references, see [the book's section on "References and +/// Borrowing"][book-refs]. +/// +/// [book-refs]: ../book/ch04-02-references-and-borrowing.html +/// +/// # Trait implementations +/// +/// The following traits are implemented for all `&T`, regardless of the type of its referent: +/// +/// * [`Copy`] +/// * [`Clone`] \(Note that this will not defer to `T`'s `Clone` implementation if it exists!) +/// * [`Deref`] +/// * [`Borrow`] +/// * [`fmt::Pointer`] +/// +/// [`Deref`]: ops::Deref +/// [`Borrow`]: borrow::Borrow +/// +/// `&mut T` references get all of the above except `Copy` and `Clone` (to prevent creating +/// multiple simultaneous mutable borrows), plus the following, regardless of the type of its +/// referent: +/// +/// * [`DerefMut`] +/// * [`BorrowMut`] +/// +/// [`DerefMut`]: ops::DerefMut +/// [`BorrowMut`]: borrow::BorrowMut +/// [bool]: prim@bool +/// +/// The following traits are implemented on `&T` references if the underlying `T` also implements +/// that trait: +/// +/// * All the traits in [`std::fmt`] except [`fmt::Pointer`] (which is implemented regardless of the type of its referent) and [`fmt::Write`] +/// * [`PartialOrd`] +/// * [`Ord`] +/// * [`PartialEq`] +/// * [`Eq`] +/// * [`AsRef`] +/// * [`Fn`] \(in addition, `&T` references get [`FnMut`] and [`FnOnce`] if `T: Fn`) +/// * [`Hash`] +/// * [`ToSocketAddrs`] +/// * [`Send`] \(`&T` references also require T: [Sync]) +/// +/// [`std::fmt`]: fmt +/// [`Hash`]: hash::Hash +#[doc = concat!("[`ToSocketAddrs`]: ", include_str!("../primitive_docs/net_tosocketaddrs.md"))] +/// +/// `&mut T` references get all of the above except `ToSocketAddrs`, plus the following, if `T` +/// implements that trait: +/// +/// * [`AsMut`] +/// * [`FnMut`] \(in addition, `&mut T` references get [`FnOnce`] if `T: FnMut`) +/// * [`fmt::Write`] +/// * [`Iterator`] +/// * [`DoubleEndedIterator`] +/// * [`ExactSizeIterator`] +/// * [`FusedIterator`] +/// * [`TrustedLen`] +/// * [`io::Write`] +/// * [`Read`] +/// * [`Seek`] +/// * [`BufRead`] +/// +/// [`FusedIterator`]: iter::FusedIterator +/// [`TrustedLen`]: iter::TrustedLen +#[doc = concat!("[`Seek`]: ", include_str!("../primitive_docs/io_seek.md"))] +#[doc = concat!("[`BufRead`]: ", include_str!("../primitive_docs/io_bufread.md"))] +#[doc = concat!("[`Read`]: ", include_str!("../primitive_docs/io_read.md"))] +#[doc = concat!("[`io::Write`]: ", include_str!("../primitive_docs/io_write.md"))] +/// +/// Note that due to method call deref coercion, simply calling a trait method will act like they +/// work on references as well as they do on owned values! The implementations described here are +/// meant for generic contexts, where the final type `T` is a type parameter or otherwise not +/// locally known. +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_ref {} + +#[doc(primitive = "fn")] +// +/// Function pointers, like `fn(usize) -> bool`. +/// +/// *See also the traits [`Fn`], [`FnMut`], and [`FnOnce`].* +/// +/// [`Fn`]: ops::Fn +/// [`FnMut`]: ops::FnMut +/// [`FnOnce`]: ops::FnOnce +/// +/// Function pointers are pointers that point to *code*, not data. They can be called +/// just like functions. Like references, function pointers are, among other things, assumed to +/// not be null, so if you want to pass a function pointer over FFI and be able to accommodate null +/// pointers, make your type [`Option`](core::option#options-and-pointers-nullable-pointers) +/// with your required signature. +/// +/// ### Safety +/// +/// Plain function pointers are obtained by casting either plain functions, or closures that don't +/// capture an environment: +/// +/// ``` +/// fn add_one(x: usize) -> usize { +/// x + 1 +/// } +/// +/// let ptr: fn(usize) -> usize = add_one; +/// assert_eq!(ptr(5), 6); +/// +/// let clos: fn(usize) -> usize = |x| x + 5; +/// assert_eq!(clos(5), 10); +/// ``` +/// +/// In addition to varying based on their signature, function pointers come in two flavors: safe +/// and unsafe. Plain `fn()` function pointers can only point to safe functions, +/// while `unsafe fn()` function pointers can point to safe or unsafe functions. +/// +/// ``` +/// fn add_one(x: usize) -> usize { +/// x + 1 +/// } +/// +/// unsafe fn add_one_unsafely(x: usize) -> usize { +/// x + 1 +/// } +/// +/// let safe_ptr: fn(usize) -> usize = add_one; +/// +/// //ERROR: mismatched types: expected normal fn, found unsafe fn +/// //let bad_ptr: fn(usize) -> usize = add_one_unsafely; +/// +/// let unsafe_ptr: unsafe fn(usize) -> usize = add_one_unsafely; +/// let really_safe_ptr: unsafe fn(usize) -> usize = add_one; +/// ``` +/// +/// ### ABI +/// +/// On top of that, function pointers can vary based on what ABI they use. This +/// is achieved by adding the `extern` keyword before the type, followed by the +/// ABI in question. The default ABI is "Rust", i.e., `fn()` is the exact same +/// type as `extern "Rust" fn()`. A pointer to a function with C ABI would have +/// type `extern "C" fn()`. +/// +/// `extern "ABI" { ... }` blocks declare functions with ABI "ABI". The default +/// here is "C", i.e., functions declared in an `extern {...}` block have "C" +/// ABI. +/// +/// For more information and a list of supported ABIs, see [the nomicon's +/// section on foreign calling conventions][nomicon-abi]. +/// +/// [nomicon-abi]: ../nomicon/ffi.html#foreign-calling-conventions +/// +/// ### Variadic functions +/// +/// Extern function declarations with the "C" or "cdecl" ABIs can also be *variadic*, allowing them +/// to be called with a variable number of arguments. Normal Rust functions, even those with an +/// `extern "ABI"`, cannot be variadic. For more information, see [the nomicon's section on +/// variadic functions][nomicon-variadic]. +/// +/// [nomicon-variadic]: ../nomicon/ffi.html#variadic-functions +/// +/// ### Creating function pointers +/// +/// When `bar` is the name of a function, then the expression `bar` is *not* a +/// function pointer. Rather, it denotes a value of an unnameable type that +/// uniquely identifies the function `bar`. The value is zero-sized because the +/// type already identifies the function. This has the advantage that "calling" +/// the value (it implements the `Fn*` traits) does not require dynamic +/// dispatch. +/// +/// This zero-sized type *coerces* to a regular function pointer. For example: +/// +/// ```rust +/// use std::mem; +/// +/// fn bar(x: i32) {} +/// +/// let not_bar_ptr = bar; // `not_bar_ptr` is zero-sized, uniquely identifying `bar` +/// assert_eq!(mem::size_of_val(¬_bar_ptr), 0); +/// +/// let bar_ptr: fn(i32) = not_bar_ptr; // force coercion to function pointer +/// assert_eq!(mem::size_of_val(&bar_ptr), mem::size_of::()); +/// +/// let footgun = &bar; // this is a shared reference to the zero-sized type identifying `bar` +/// ``` +/// +/// The last line shows that `&bar` is not a function pointer either. Rather, it +/// is a reference to the function-specific ZST. `&bar` is basically never what you +/// want when `bar` is a function. +/// +/// ### Casting to and from integers +/// +/// You cast function pointers directly to integers: +/// +/// ```rust +/// let fnptr: fn(i32) -> i32 = |x| x+2; +/// let fnptr_addr = fnptr as usize; +/// ``` +/// +/// However, a direct cast back is not possible. You need to use `transmute`: +/// +/// ```rust +/// # #[cfg(not(miri))] { // FIXME: use strict provenance APIs once they are stable, then remove this `cfg` +/// # let fnptr: fn(i32) -> i32 = |x| x+2; +/// # let fnptr_addr = fnptr as usize; +/// let fnptr = fnptr_addr as *const (); +/// let fnptr: fn(i32) -> i32 = unsafe { std::mem::transmute(fnptr) }; +/// assert_eq!(fnptr(40), 42); +/// # } +/// ``` +/// +/// Crucially, we `as`-cast to a raw pointer before `transmute`ing to a function pointer. +/// This avoids an integer-to-pointer `transmute`, which can be problematic. +/// Transmuting between raw pointers and function pointers (i.e., two pointer types) is fine. +/// +/// Note that all of this is not portable to platforms where function pointers and data pointers +/// have different sizes. +/// +/// ### Trait implementations +/// +/// In this documentation the shorthand `fn (T₁, T₂, …, Tₙ)` is used to represent non-variadic +/// function pointers of varying length. Note that this is a convenience notation to avoid +/// repetitive documentation, not valid Rust syntax. +/// +/// Due to a temporary restriction in Rust's type system, these traits are only implemented on +/// functions that take 12 arguments or less, with the `"Rust"` and `"C"` ABIs. In the future, this +/// may change: +/// +/// * [`PartialEq`] +/// * [`Eq`] +/// * [`PartialOrd`] +/// * [`Ord`] +/// * [`Hash`] +/// * [`Pointer`] +/// * [`Debug`] +/// +/// The following traits are implemented for function pointers with any number of arguments and +/// any ABI. These traits have implementations that are automatically generated by the compiler, +/// so are not limited by missing language features: +/// +/// * [`Clone`] +/// * [`Copy`] +/// * [`Send`] +/// * [`Sync`] +/// * [`Unpin`] +/// * [`UnwindSafe`] +/// * [`RefUnwindSafe`] +/// +/// [`Hash`]: hash::Hash +/// [`Pointer`]: fmt::Pointer +/// [`UnwindSafe`]: panic::UnwindSafe +/// [`RefUnwindSafe`]: panic::RefUnwindSafe +/// +/// In addition, all *safe* function pointers implement [`Fn`], [`FnMut`], and [`FnOnce`], because +/// these traits are specially known to the compiler. +#[stable(feature = "rust1", since = "1.0.0")] +mod prim_fn {} + +// Required to make auto trait impls render. +// See src/librustdoc/passes/collect_trait_impls.rs:collect_trait_impls +#[doc(hidden)] +impl fn(T) -> Ret {} + +// Fake impl that's only really used for docs. +#[cfg(doc)] +#[stable(feature = "rust1", since = "1.0.0")] +#[doc(fake_variadic)] +/// This trait is implemented on function pointers with any number of arguments. +impl Clone for fn(T) -> Ret { + fn clone(&self) -> Self { + loop {} + } +} + +// Fake impl that's only really used for docs. +#[cfg(doc)] +#[stable(feature = "rust1", since = "1.0.0")] +#[doc(fake_variadic)] +/// This trait is implemented on function pointers with any number of arguments. +impl Copy for fn(T) -> Ret { + // empty +} diff --git a/crux-mir/lib/core/src/ptr/alignment.rs b/crux-mir/lib/core/src/ptr/alignment.rs new file mode 100644 index 000000000..2123147c7 --- /dev/null +++ b/crux-mir/lib/core/src/ptr/alignment.rs @@ -0,0 +1,332 @@ +use crate::convert::{TryFrom, TryInto}; +use crate::intrinsics::assert_unsafe_precondition; +use crate::num::NonZeroUsize; +use crate::{cmp, fmt, hash, mem, num}; + +/// A type storing a `usize` which is a power of two, and thus +/// represents a possible alignment in the rust abstract machine. +/// +/// Note that particularly large alignments, while representable in this type, +/// are likely not to be supported by actual allocators and linkers. +#[unstable(feature = "ptr_alignment_type", issue = "102070")] +#[derive(Copy, Clone, Eq)] +#[derive_const(PartialEq)] +#[repr(transparent)] +pub struct Alignment(AlignmentEnum); + +// Alignment is `repr(usize)`, but via extra steps. +const _: () = assert!(mem::size_of::() == mem::size_of::()); +const _: () = assert!(mem::align_of::() == mem::align_of::()); + +fn _alignment_can_be_structurally_matched(a: Alignment) -> bool { + matches!(a, Alignment::MIN) +} + +impl Alignment { + /// The smallest possible alignment, 1. + /// + /// All addresses are always aligned at least this much. + /// + /// # Examples + /// + /// ``` + /// #![feature(ptr_alignment_type)] + /// use std::ptr::Alignment; + /// + /// assert_eq!(Alignment::MIN.as_usize(), 1); + /// ``` + #[unstable(feature = "ptr_alignment_type", issue = "102070")] + pub const MIN: Self = Self(AlignmentEnum::_Align1Shl0); + + /// Returns the alignment for a type. + /// + /// This provides the same numerical value as [`mem::align_of`], + /// but in an `Alignment` instead of a `usize. + #[unstable(feature = "ptr_alignment_type", issue = "102070")] + #[inline] + pub const fn of() -> Self { + // SAFETY: rustc ensures that type alignment is always a power of two. + unsafe { Alignment::new_unchecked(mem::align_of::()) } + } + + /// Creates an `Alignment` from a `usize`, or returns `None` if it's + /// not a power of two. + /// + /// Note that `0` is not a power of two, nor a valid alignment. + #[unstable(feature = "ptr_alignment_type", issue = "102070")] + #[inline] + pub const fn new(align: usize) -> Option { + if align.is_power_of_two() { + // SAFETY: Just checked it only has one bit set + Some(unsafe { Self::new_unchecked(align) }) + } else { + None + } + } + + /// Creates an `Alignment` from a power-of-two `usize`. + /// + /// # Safety + /// + /// `align` must be a power of two. + /// + /// Equivalently, it must be `1 << exp` for some `exp` in `0..usize::BITS`. + /// It must *not* be zero. + #[unstable(feature = "ptr_alignment_type", issue = "102070")] + #[rustc_const_unstable(feature = "ptr_alignment_type", issue = "102070")] + #[inline] + pub const unsafe fn new_unchecked(align: usize) -> Self { + // SAFETY: Precondition passed to the caller. + unsafe { + assert_unsafe_precondition!( + "Alignment::new_unchecked requires a power of two", + (align: usize) => align.is_power_of_two() + ) + }; + + // SAFETY: By precondition, this must be a power of two, and + // our variants encompass all possible powers of two. + unsafe { mem::transmute::(align) } + } + + /// Returns the alignment as a [`usize`] + #[unstable(feature = "ptr_alignment_type", issue = "102070")] + #[rustc_const_unstable(feature = "ptr_alignment_type", issue = "102070")] + #[inline] + pub const fn as_usize(self) -> usize { + self.0 as usize + } + + /// Returns the alignment as a [`NonZeroUsize`] + #[unstable(feature = "ptr_alignment_type", issue = "102070")] + #[inline] + pub const fn as_nonzero(self) -> NonZeroUsize { + // SAFETY: All the discriminants are non-zero. + unsafe { NonZeroUsize::new_unchecked(self.as_usize()) } + } + + /// Returns the base-2 logarithm of the alignment. + /// + /// This is always exact, as `self` represents a power of two. + /// + /// # Examples + /// + /// ``` + /// #![feature(ptr_alignment_type)] + /// use std::ptr::Alignment; + /// + /// assert_eq!(Alignment::of::().log2(), 0); + /// assert_eq!(Alignment::new(1024).unwrap().log2(), 10); + /// ``` + #[unstable(feature = "ptr_alignment_type", issue = "102070")] + #[inline] + pub fn log2(self) -> u32 { + self.as_nonzero().trailing_zeros() + } +} + +#[unstable(feature = "ptr_alignment_type", issue = "102070")] +impl fmt::Debug for Alignment { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?} (1 << {:?})", self.as_nonzero(), self.log2()) + } +} + +#[unstable(feature = "ptr_alignment_type", issue = "102070")] +impl TryFrom for Alignment { + type Error = num::TryFromIntError; + + #[inline] + fn try_from(align: NonZeroUsize) -> Result { + align.get().try_into() + } +} + +#[unstable(feature = "ptr_alignment_type", issue = "102070")] +impl TryFrom for Alignment { + type Error = num::TryFromIntError; + + #[inline] + fn try_from(align: usize) -> Result { + Self::new(align).ok_or(num::TryFromIntError(())) + } +} + +#[unstable(feature = "ptr_alignment_type", issue = "102070")] +impl From for NonZeroUsize { + #[inline] + fn from(align: Alignment) -> NonZeroUsize { + align.as_nonzero() + } +} + +#[unstable(feature = "ptr_alignment_type", issue = "102070")] +impl From for usize { + #[inline] + fn from(align: Alignment) -> usize { + align.as_usize() + } +} + +#[rustc_const_unstable(feature = "const_alloc_layout", issue = "67521")] +#[unstable(feature = "ptr_alignment_type", issue = "102070")] +impl const cmp::Ord for Alignment { + #[inline] + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.as_nonzero().get().cmp(&other.as_nonzero().get()) + } +} + +#[rustc_const_unstable(feature = "const_alloc_layout", issue = "67521")] +#[unstable(feature = "ptr_alignment_type", issue = "102070")] +impl const cmp::PartialOrd for Alignment { + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +#[unstable(feature = "ptr_alignment_type", issue = "102070")] +impl hash::Hash for Alignment { + #[inline] + fn hash(&self, state: &mut H) { + self.as_nonzero().hash(state) + } +} + +#[cfg(target_pointer_width = "16")] +type AlignmentEnum = AlignmentEnum16; +#[cfg(target_pointer_width = "32")] +type AlignmentEnum = AlignmentEnum32; +#[cfg(target_pointer_width = "64")] +type AlignmentEnum = AlignmentEnum64; + +#[derive(Copy, Clone, Eq)] +#[derive_const(PartialEq)] +#[repr(u16)] +enum AlignmentEnum16 { + _Align1Shl0 = 1 << 0, + _Align1Shl1 = 1 << 1, + _Align1Shl2 = 1 << 2, + _Align1Shl3 = 1 << 3, + _Align1Shl4 = 1 << 4, + _Align1Shl5 = 1 << 5, + _Align1Shl6 = 1 << 6, + _Align1Shl7 = 1 << 7, + _Align1Shl8 = 1 << 8, + _Align1Shl9 = 1 << 9, + _Align1Shl10 = 1 << 10, + _Align1Shl11 = 1 << 11, + _Align1Shl12 = 1 << 12, + _Align1Shl13 = 1 << 13, + _Align1Shl14 = 1 << 14, + _Align1Shl15 = 1 << 15, +} + +#[derive(Copy, Clone, Eq)] +#[derive_const(PartialEq)] +#[repr(u32)] +enum AlignmentEnum32 { + _Align1Shl0 = 1 << 0, + _Align1Shl1 = 1 << 1, + _Align1Shl2 = 1 << 2, + _Align1Shl3 = 1 << 3, + _Align1Shl4 = 1 << 4, + _Align1Shl5 = 1 << 5, + _Align1Shl6 = 1 << 6, + _Align1Shl7 = 1 << 7, + _Align1Shl8 = 1 << 8, + _Align1Shl9 = 1 << 9, + _Align1Shl10 = 1 << 10, + _Align1Shl11 = 1 << 11, + _Align1Shl12 = 1 << 12, + _Align1Shl13 = 1 << 13, + _Align1Shl14 = 1 << 14, + _Align1Shl15 = 1 << 15, + _Align1Shl16 = 1 << 16, + _Align1Shl17 = 1 << 17, + _Align1Shl18 = 1 << 18, + _Align1Shl19 = 1 << 19, + _Align1Shl20 = 1 << 20, + _Align1Shl21 = 1 << 21, + _Align1Shl22 = 1 << 22, + _Align1Shl23 = 1 << 23, + _Align1Shl24 = 1 << 24, + _Align1Shl25 = 1 << 25, + _Align1Shl26 = 1 << 26, + _Align1Shl27 = 1 << 27, + _Align1Shl28 = 1 << 28, + _Align1Shl29 = 1 << 29, + _Align1Shl30 = 1 << 30, + _Align1Shl31 = 1 << 31, +} + +#[derive(Copy, Clone, Eq)] +#[derive_const(PartialEq)] +#[repr(u64)] +enum AlignmentEnum64 { + _Align1Shl0 = 1 << 0, + _Align1Shl1 = 1 << 1, + _Align1Shl2 = 1 << 2, + _Align1Shl3 = 1 << 3, + _Align1Shl4 = 1 << 4, + _Align1Shl5 = 1 << 5, + _Align1Shl6 = 1 << 6, + _Align1Shl7 = 1 << 7, + _Align1Shl8 = 1 << 8, + _Align1Shl9 = 1 << 9, + _Align1Shl10 = 1 << 10, + _Align1Shl11 = 1 << 11, + _Align1Shl12 = 1 << 12, + _Align1Shl13 = 1 << 13, + _Align1Shl14 = 1 << 14, + _Align1Shl15 = 1 << 15, + _Align1Shl16 = 1 << 16, + _Align1Shl17 = 1 << 17, + _Align1Shl18 = 1 << 18, + _Align1Shl19 = 1 << 19, + _Align1Shl20 = 1 << 20, + _Align1Shl21 = 1 << 21, + _Align1Shl22 = 1 << 22, + _Align1Shl23 = 1 << 23, + _Align1Shl24 = 1 << 24, + _Align1Shl25 = 1 << 25, + _Align1Shl26 = 1 << 26, + _Align1Shl27 = 1 << 27, + _Align1Shl28 = 1 << 28, + _Align1Shl29 = 1 << 29, + _Align1Shl30 = 1 << 30, + _Align1Shl31 = 1 << 31, + _Align1Shl32 = 1 << 32, + _Align1Shl33 = 1 << 33, + _Align1Shl34 = 1 << 34, + _Align1Shl35 = 1 << 35, + _Align1Shl36 = 1 << 36, + _Align1Shl37 = 1 << 37, + _Align1Shl38 = 1 << 38, + _Align1Shl39 = 1 << 39, + _Align1Shl40 = 1 << 40, + _Align1Shl41 = 1 << 41, + _Align1Shl42 = 1 << 42, + _Align1Shl43 = 1 << 43, + _Align1Shl44 = 1 << 44, + _Align1Shl45 = 1 << 45, + _Align1Shl46 = 1 << 46, + _Align1Shl47 = 1 << 47, + _Align1Shl48 = 1 << 48, + _Align1Shl49 = 1 << 49, + _Align1Shl50 = 1 << 50, + _Align1Shl51 = 1 << 51, + _Align1Shl52 = 1 << 52, + _Align1Shl53 = 1 << 53, + _Align1Shl54 = 1 << 54, + _Align1Shl55 = 1 << 55, + _Align1Shl56 = 1 << 56, + _Align1Shl57 = 1 << 57, + _Align1Shl58 = 1 << 58, + _Align1Shl59 = 1 << 59, + _Align1Shl60 = 1 << 60, + _Align1Shl61 = 1 << 61, + _Align1Shl62 = 1 << 62, + _Align1Shl63 = 1 << 63, +} diff --git a/crux-mir/lib/core/src/ptr/const_ptr.rs b/crux-mir/lib/core/src/ptr/const_ptr.rs index 781d15b62..7b1cb5488 100644 --- a/crux-mir/lib/core/src/ptr/const_ptr.rs +++ b/crux-mir/lib/core/src/ptr/const_ptr.rs @@ -1,12 +1,9 @@ use super::*; use crate::cmp::Ordering::{self, Equal, Greater, Less}; -use crate::crucible; -use crate::intrinsics; +use crate::intrinsics::{self, const_eval_select}; use crate::mem; +use crate::slice::{self, SliceIndex}; -// ignore-tidy-undocumented-unsafe - -#[lang = "const_ptr"] impl *const T { /// Returns `true` if the pointer is null. /// @@ -15,6 +12,15 @@ impl *const T { /// Therefore, two pointers that are null may still not compare equal to /// each other. /// + /// ## Behavior during const evaluation + /// + /// When this function is used during const evaluation, it may return `false` for pointers + /// that turn out to be null at runtime. Specifically, when a pointer to some memory + /// is offset beyond its bounds in such a way that the resulting pointer is null, + /// the function will still return `false`. There is no way for CTFE to know + /// the absolute position of that memory, so we cannot tell if the pointer is + /// null or not. + /// /// # Examples /// /// Basic usage: @@ -25,45 +31,295 @@ impl *const T { /// assert!(!ptr.is_null()); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_ptr_is_null", issue = "74939")] #[inline] - pub fn is_null(self) -> bool { - crucible::ptr::compare_usize(self, 0) + pub const fn is_null(self) -> bool { + #[inline] + fn runtime_impl(ptr: *const u8) -> bool { + ptr.addr() == 0 + } + + #[inline] + const fn const_impl(ptr: *const u8) -> bool { + // Compare via a cast to a thin pointer, so fat pointers are only + // considering their "data" part for null-ness. + match (ptr).guaranteed_eq(null_mut()) { + None => false, + Some(res) => res, + } + } + + // SAFETY: The two versions are equivalent at runtime. + unsafe { const_eval_select((self as *const u8,), const_impl, runtime_impl) } } /// Casts to a pointer of another type. #[stable(feature = "ptr_cast", since = "1.38.0")] #[rustc_const_stable(feature = "const_ptr_cast", since = "1.38.0")] - #[inline] + #[inline(always)] pub const fn cast(self) -> *const U { self as _ } - /// Returns `None` if the pointer is null, or else returns a reference to - /// the value wrapped in `Some`. + /// Use the pointer value in a new pointer of another type. /// - /// # Safety + /// In case `val` is a (fat) pointer to an unsized type, this operation + /// will ignore the pointer part, whereas for (thin) pointers to sized + /// types, this has the same effect as a simple cast. + /// + /// The resulting pointer will have provenance of `self`, i.e., for a fat + /// pointer, this operation is semantically the same as creating a new + /// fat pointer with the data pointer value of `self` but the metadata of + /// `val`. /// - /// While this method and its mutable counterpart are useful for - /// null-safety, it is important to note that this is still an unsafe - /// operation because the returned value could be pointing to invalid - /// memory. + /// # Examples + /// + /// This function is primarily useful for allowing byte-wise pointer + /// arithmetic on potentially fat pointers: /// - /// When calling this method, you have to ensure that *either* the pointer is NULL *or* + /// ``` + /// #![feature(set_ptr_value)] + /// # use core::fmt::Debug; + /// let arr: [i32; 3] = [1, 2, 3]; + /// let mut ptr = arr.as_ptr() as *const dyn Debug; + /// let thin = ptr as *const u8; + /// unsafe { + /// ptr = thin.add(8).with_metadata_of(ptr); + /// # assert_eq!(*(ptr as *const i32), 3); + /// println!("{:?}", &*ptr); // will print "3" + /// } + /// ``` + #[unstable(feature = "set_ptr_value", issue = "75091")] + #[rustc_const_unstable(feature = "set_ptr_value", issue = "75091")] + #[must_use = "returns a new pointer rather than modifying its argument"] + #[inline] + pub const fn with_metadata_of(self, meta: *const U) -> *const U + where + U: ?Sized, + { + from_raw_parts::(self as *const (), metadata(meta)) + } + + /// Changes constness without changing the type. + /// + /// This is a bit safer than `as` because it wouldn't silently change the type if the code is + /// refactored. + #[stable(feature = "ptr_const_cast", since = "1.65.0")] + #[rustc_const_stable(feature = "ptr_const_cast", since = "1.65.0")] + #[inline(always)] + pub const fn cast_mut(self) -> *mut T { + self as _ + } + + /// Casts a pointer to its raw bits. + /// + /// This is equivalent to `as usize`, but is more specific to enhance readability. + /// The inverse method is [`from_bits`](#method.from_bits). + /// + /// In particular, `*p as usize` and `p as usize` will both compile for + /// pointers to numeric types but do very different things, so using this + /// helps emphasize that reading the bits was intentional. + /// + /// # Examples + /// + /// ``` + /// #![feature(ptr_to_from_bits)] + /// # #[cfg(not(miri))] { // doctest does not work with strict provenance + /// let array = [13, 42]; + /// let p0: *const i32 = &array[0]; + /// assert_eq!(<*const _>::from_bits(p0.to_bits()), p0); + /// let p1: *const i32 = &array[1]; + /// assert_eq!(p1.to_bits() - p0.to_bits(), 4); + /// # } + /// ``` + #[unstable(feature = "ptr_to_from_bits", issue = "91126")] + #[deprecated( + since = "1.67", + note = "replaced by the `exposed_addr` method, or update your code \ + to follow the strict provenance rules using its APIs" + )] + #[inline(always)] + pub fn to_bits(self) -> usize + where + T: Sized, + { + self as usize + } + + /// Creates a pointer from its raw bits. + /// + /// This is equivalent to `as *const T`, but is more specific to enhance readability. + /// The inverse method is [`to_bits`](#method.to_bits). + /// + /// # Examples + /// + /// ``` + /// #![feature(ptr_to_from_bits)] + /// # #[cfg(not(miri))] { // doctest does not work with strict provenance + /// use std::ptr::NonNull; + /// let dangling: *const u8 = NonNull::dangling().as_ptr(); + /// assert_eq!(<*const u8>::from_bits(1), dangling); + /// # } + /// ``` + #[unstable(feature = "ptr_to_from_bits", issue = "91126")] + #[deprecated( + since = "1.67", + note = "replaced by the `ptr::from_exposed_addr` function, or update \ + your code to follow the strict provenance rules using its APIs" + )] + #[allow(fuzzy_provenance_casts)] // this is an unstable and semi-deprecated cast function + #[inline(always)] + pub fn from_bits(bits: usize) -> Self + where + T: Sized, + { + bits as Self + } + + /// Gets the "address" portion of the pointer. + /// + /// This is similar to `self as usize`, which semantically discards *provenance* and + /// *address-space* information. However, unlike `self as usize`, casting the returned address + /// back to a pointer yields [`invalid`][], which is undefined behavior to dereference. To + /// properly restore the lost information and obtain a dereferenceable pointer, use + /// [`with_addr`][pointer::with_addr] or [`map_addr`][pointer::map_addr]. + /// + /// If using those APIs is not possible because there is no way to preserve a pointer with the + /// required provenance, use [`expose_addr`][pointer::expose_addr] and + /// [`from_exposed_addr`][from_exposed_addr] instead. However, note that this makes + /// your code less portable and less amenable to tools that check for compliance with the Rust + /// memory model. + /// + /// On most platforms this will produce a value with the same bytes as the original + /// pointer, because all the bytes are dedicated to describing the address. + /// Platforms which need to store additional information in the pointer may + /// perform a change of representation to produce a value containing only the address + /// portion of the pointer. What that means is up to the platform to define. + /// + /// This API and its claimed semantics are part of the Strict Provenance experiment, and as such + /// might change in the future (including possibly weakening this so it becomes wholly + /// equivalent to `self as usize`). See the [module documentation][crate::ptr] for details. + #[must_use] + #[inline(always)] + #[unstable(feature = "strict_provenance", issue = "95228")] + pub fn addr(self) -> usize { + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + // SAFETY: Pointer-to-integer transmutes are valid (if you are okay with losing the + // provenance). + unsafe { mem::transmute(self.cast::<()>()) } + } + + /// Gets the "address" portion of the pointer, and 'exposes' the "provenance" part for future + /// use in [`from_exposed_addr`][]. + /// + /// This is equivalent to `self as usize`, which semantically discards *provenance* and + /// *address-space* information. Furthermore, this (like the `as` cast) has the implicit + /// side-effect of marking the provenance as 'exposed', so on platforms that support it you can + /// later call [`from_exposed_addr`][] to reconstitute the original pointer including its + /// provenance. (Reconstructing address space information, if required, is your responsibility.) + /// + /// Using this method means that code is *not* following Strict Provenance rules. Supporting + /// [`from_exposed_addr`][] complicates specification and reasoning and may not be supported by + /// tools that help you to stay conformant with the Rust memory model, so it is recommended to + /// use [`addr`][pointer::addr] wherever possible. + /// + /// On most platforms this will produce a value with the same bytes as the original pointer, + /// because all the bytes are dedicated to describing the address. Platforms which need to store + /// additional information in the pointer may not support this operation, since the 'expose' + /// side-effect which is required for [`from_exposed_addr`][] to work is typically not + /// available. + /// + /// This API and its claimed semantics are part of the Strict Provenance experiment, see the + /// [module documentation][crate::ptr] for details. + /// + /// [`from_exposed_addr`]: from_exposed_addr + #[must_use] + #[inline(always)] + #[unstable(feature = "strict_provenance", issue = "95228")] + pub fn expose_addr(self) -> usize { + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + self.cast::<()>() as usize + } + + /// Creates a new pointer with the given address. + /// + /// This performs the same operation as an `addr as ptr` cast, but copies + /// the *address-space* and *provenance* of `self` to the new pointer. + /// This allows us to dynamically preserve and propagate this important + /// information in a way that is otherwise impossible with a unary cast. + /// + /// This is equivalent to using [`wrapping_offset`][pointer::wrapping_offset] to offset + /// `self` to the given address, and therefore has all the same capabilities and restrictions. + /// + /// This API and its claimed semantics are part of the Strict Provenance experiment, + /// see the [module documentation][crate::ptr] for details. + #[must_use] + #[inline] + #[unstable(feature = "strict_provenance", issue = "95228")] + pub fn with_addr(self, addr: usize) -> Self { + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + // + // In the mean-time, this operation is defined to be "as if" it was + // a wrapping_offset, so we can emulate it as such. This should properly + // restore pointer provenance even under today's compiler. + let self_addr = self.addr() as isize; + let dest_addr = addr as isize; + let offset = dest_addr.wrapping_sub(self_addr); + + // This is the canonical desugarring of this operation + self.wrapping_byte_offset(offset) + } + + /// Creates a new pointer by mapping `self`'s address to a new one. + /// + /// This is a convenience for [`with_addr`][pointer::with_addr], see that method for details. + /// + /// This API and its claimed semantics are part of the Strict Provenance experiment, + /// see the [module documentation][crate::ptr] for details. + #[must_use] + #[inline] + #[unstable(feature = "strict_provenance", issue = "95228")] + pub fn map_addr(self, f: impl FnOnce(usize) -> usize) -> Self { + self.with_addr(f(self.addr())) + } + + /// Decompose a (possibly wide) pointer into its address and metadata components. + /// + /// The pointer can be later reconstructed with [`from_raw_parts`]. + #[unstable(feature = "ptr_metadata", issue = "81513")] + #[rustc_const_unstable(feature = "ptr_metadata", issue = "81513")] + #[inline] + pub const fn to_raw_parts(self) -> (*const (), ::Metadata) { + (self.cast(), metadata(self)) + } + + /// Returns `None` if the pointer is null, or else returns a shared reference to + /// the value wrapped in `Some`. If the value may be uninitialized, [`as_uninit_ref`] + /// must be used instead. + /// + /// [`as_uninit_ref`]: #method.as_uninit_ref + /// + /// # Safety + /// + /// When calling this method, you have to ensure that *either* the pointer is null *or* /// all of the following is true: - /// - it is properly aligned - /// - it must point to an initialized instance of T; in particular, the pointer must be - /// "dereferenceable" in the sense defined [here]. + /// + /// * The pointer must be properly aligned. + /// + /// * It must be "dereferenceable" in the sense defined in [the module documentation]. + /// + /// * The pointer must point to an initialized instance of `T`. + /// + /// * You must enforce Rust's aliasing rules, since the returned lifetime `'a` is + /// arbitrarily chosen and does not necessarily reflect the actual lifetime of the data. + /// In particular, while this reference exists, the memory the pointer points to must + /// not get mutated (except inside `UnsafeCell`). /// /// This applies even if the result of this method is unused! /// (The part about being initialized is not yet fully decided, but until /// it is, the only safe approach is to ensure that they are indeed initialized.) /// - /// Additionally, the lifetime `'a` returned is arbitrarily chosen and does - /// not necessarily reflect the actual lifetime of the data. *You* must enforce - /// Rust's aliasing rules. In particular, for the duration of this lifetime, - /// the memory the pointer points to must not get mutated (except inside `UnsafeCell`). - /// - /// [here]: crate::ptr#safety + /// [the module documentation]: crate::ptr#safety /// /// # Examples /// @@ -74,7 +330,7 @@ impl *const T { /// /// unsafe { /// if let Some(val_back) = ptr.as_ref() { - /// println!("We got back the value: {}!", val_back); + /// println!("We got back the value: {val_back}!"); /// } /// } /// ``` @@ -90,13 +346,67 @@ impl *const T { /// /// unsafe { /// let val_back = &*ptr; - /// println!("We got back the value: {}!", val_back); + /// println!("We got back the value: {val_back}!"); /// } /// ``` #[stable(feature = "ptr_as_ref", since = "1.9.0")] + #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")] + #[inline] + pub const unsafe fn as_ref<'a>(self) -> Option<&'a T> { + // SAFETY: the caller must guarantee that `self` is valid + // for a reference if it isn't null. + if self.is_null() { None } else { unsafe { Some(&*self) } } + } + + /// Returns `None` if the pointer is null, or else returns a shared reference to + /// the value wrapped in `Some`. In contrast to [`as_ref`], this does not require + /// that the value has to be initialized. + /// + /// [`as_ref`]: #method.as_ref + /// + /// # Safety + /// + /// When calling this method, you have to ensure that *either* the pointer is null *or* + /// all of the following is true: + /// + /// * The pointer must be properly aligned. + /// + /// * It must be "dereferenceable" in the sense defined in [the module documentation]. + /// + /// * You must enforce Rust's aliasing rules, since the returned lifetime `'a` is + /// arbitrarily chosen and does not necessarily reflect the actual lifetime of the data. + /// In particular, while this reference exists, the memory the pointer points to must + /// not get mutated (except inside `UnsafeCell`). + /// + /// This applies even if the result of this method is unused! + /// + /// [the module documentation]: crate::ptr#safety + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(ptr_as_uninit)] + /// + /// let ptr: *const u8 = &10u8 as *const u8; + /// + /// unsafe { + /// if let Some(val_back) = ptr.as_uninit_ref() { + /// println!("We got back the value: {}!", val_back.assume_init()); + /// } + /// } + /// ``` #[inline] - pub unsafe fn as_ref<'a>(self) -> Option<&'a T> { - if self.is_null() { None } else { Some(&*self) } + #[unstable(feature = "ptr_as_uninit", issue = "75402")] + #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")] + pub const unsafe fn as_uninit_ref<'a>(self) -> Option<&'a MaybeUninit> + where + T: Sized, + { + // SAFETY: the caller must guarantee that `self` meets all the + // requirements for a reference. + if self.is_null() { None } else { Some(unsafe { &*(self as *const MaybeUninit) }) } } /// Calculates the offset from a pointer. @@ -110,8 +420,7 @@ impl *const T { /// Behavior: /// /// * Both the starting and resulting pointer must be either in bounds or one - /// byte past the end of the same allocated object. Note that in Rust, - /// every (stack-allocated) variable is considered a separate allocated object. + /// byte past the end of the same [allocated object]. /// /// * The computed offset, **in bytes**, cannot overflow an `isize`. /// @@ -136,6 +445,7 @@ impl *const T { /// enables more aggressive compiler optimizations. /// /// [`wrapping_offset`]: #method.wrapping_offset + /// [allocated object]: crate::ptr#allocated-object /// /// # Examples /// @@ -151,12 +461,36 @@ impl *const T { /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] - #[inline] - pub unsafe fn offset(self, count: isize) -> *const T + #[must_use = "returns a new pointer rather than modifying its argument"] + #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn offset(self, count: isize) -> *const T where T: Sized, { - intrinsics::offset(self, count) + // SAFETY: the caller must uphold the safety contract for `offset`. + unsafe { intrinsics::offset(self, count) } + } + + /// Calculates the offset from a pointer in bytes. + /// + /// `count` is in units of **bytes**. + /// + /// This is purely a convenience for casting to a `u8` pointer and + /// using [offset][pointer::offset] on it. See that method for documentation + /// and safety requirements. + /// + /// For non-`Sized` pointees this operation changes only the data pointer, + /// leaving the metadata untouched. + #[must_use] + #[inline(always)] + #[unstable(feature = "pointer_byte_offsets", issue = "96283")] + #[rustc_const_unstable(feature = "const_pointer_byte_offsets", issue = "96283")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn byte_offset(self, count: isize) -> Self { + // SAFETY: the caller must uphold the safety contract for `offset`. + unsafe { self.cast::().offset(count).with_metadata_of(self) } } /// Calculates the offset from a pointer using wrapping arithmetic. @@ -166,28 +500,29 @@ impl *const T { /// /// # Safety /// - /// The resulting pointer does not need to be in bounds, but it is - /// potentially hazardous to dereference (which requires `unsafe`). + /// This operation itself is always safe, but using the resulting pointer is not. /// - /// In particular, the resulting pointer remains attached to the same allocated - /// object that `self` points to. It may *not* be used to access a - /// different allocated object. Note that in Rust, - /// every (stack-allocated) variable is considered a separate allocated object. + /// The resulting pointer "remembers" the [allocated object] that `self` points to; it must not + /// be used to read or write other allocated objects. /// - /// In other words, `x.wrapping_offset(y.wrapping_offset_from(x))` is - /// *not* the same as `y`, and dereferencing it is undefined behavior - /// unless `x` and `y` point into the same allocated object. + /// In other words, `let z = x.wrapping_offset((y as isize) - (x as isize))` does *not* make `z` + /// the same as `y` even if we assume `T` has size `1` and there is no overflow: `z` is still + /// attached to the object `x` is attached to, and dereferencing it is Undefined Behavior unless + /// `x` and `y` point into the same allocated object. /// - /// Compared to [`offset`], this method basically delays the requirement of staying - /// within the same allocated object: [`offset`] is immediate Undefined Behavior when - /// crossing object boundaries; `wrapping_offset` produces a pointer but still leads - /// to Undefined Behavior if that pointer is dereferenced. [`offset`] can be optimized - /// better and is thus preferable in performance-sensitive code. + /// Compared to [`offset`], this method basically delays the requirement of staying within the + /// same allocated object: [`offset`] is immediate Undefined Behavior when crossing object + /// boundaries; `wrapping_offset` produces a pointer but still leads to Undefined Behavior if a + /// pointer is dereferenced when it is out-of-bounds of the object it is attached to. [`offset`] + /// can be optimized better and is thus preferable in performance-sensitive code. /// - /// If you need to cross object boundaries, cast the pointer to an integer and - /// do the arithmetic there. + /// The delayed check only considers the value of the pointer that was dereferenced, not the + /// intermediate values used during the computation of the final result. For example, + /// `x.wrapping_offset(o).wrapping_offset(o.wrapping_neg())` is always the same as `x`. In other + /// words, leaving the allocated object and then re-entering it later is permitted. /// /// [`offset`]: #method.offset + /// [allocated object]: crate::ptr#allocated-object /// /// # Examples /// @@ -209,21 +544,79 @@ impl *const T { /// } /// ``` #[stable(feature = "ptr_wrapping_offset", since = "1.16.0")] - #[inline] - pub fn wrapping_offset(self, count: isize) -> *const T + #[must_use = "returns a new pointer rather than modifying its argument"] + #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] + #[inline(always)] + pub const fn wrapping_offset(self, count: isize) -> *const T where T: Sized, { + // SAFETY: the `arith_offset` intrinsic has no prerequisites to be called. unsafe { intrinsics::arith_offset(self, count) } } + /// Calculates the offset from a pointer in bytes using wrapping arithmetic. + /// + /// `count` is in units of **bytes**. + /// + /// This is purely a convenience for casting to a `u8` pointer and + /// using [wrapping_offset][pointer::wrapping_offset] on it. See that method + /// for documentation. + /// + /// For non-`Sized` pointees this operation changes only the data pointer, + /// leaving the metadata untouched. + #[must_use] + #[inline(always)] + #[unstable(feature = "pointer_byte_offsets", issue = "96283")] + #[rustc_const_unstable(feature = "const_pointer_byte_offsets", issue = "96283")] + pub const fn wrapping_byte_offset(self, count: isize) -> Self { + self.cast::().wrapping_offset(count).with_metadata_of(self) + } + + /// Masks out bits of the pointer according to a mask. + /// + /// This is convenience for `ptr.map_addr(|a| a & mask)`. + /// + /// For non-`Sized` pointees this operation changes only the data pointer, + /// leaving the metadata untouched. + /// + /// ## Examples + /// + /// ``` + /// #![feature(ptr_mask, strict_provenance)] + /// let v = 17_u32; + /// let ptr: *const u32 = &v; + /// + /// // `u32` is 4 bytes aligned, + /// // which means that lower 2 bits are always 0. + /// let tag_mask = 0b11; + /// let ptr_mask = !tag_mask; + /// + /// // We can store something in these lower bits + /// let tagged_ptr = ptr.map_addr(|a| a | 0b10); + /// + /// // Get the "tag" back + /// let tag = tagged_ptr.addr() & tag_mask; + /// assert_eq!(tag, 0b10); + /// + /// // Note that `tagged_ptr` is unaligned, it's UB to read from it. + /// // To get original pointer `mask` can be used: + /// let masked_ptr = tagged_ptr.mask(ptr_mask); + /// assert_eq!(unsafe { *masked_ptr }, 17); + /// ``` + #[unstable(feature = "ptr_mask", issue = "98290")] + #[must_use = "returns a new pointer rather than modifying its argument"] + #[inline(always)] + pub fn mask(self, mask: usize) -> *const T { + intrinsics::ptr_mask(self.cast::<()>(), mask).with_metadata_of(self) + } + /// Calculates the distance between two pointers. The returned value is in - /// units of T: the distance in bytes is divided by `mem::size_of::()`. + /// units of T: the distance in bytes divided by `mem::size_of::()`. /// /// This function is the inverse of [`offset`]. /// /// [`offset`]: #method.offset - /// [`wrapping_offset_from`]: #method.wrapping_offset_from /// /// # Safety /// @@ -231,32 +624,37 @@ impl *const T { /// Behavior: /// /// * Both the starting and other pointer must be either in bounds or one - /// byte past the end of the same allocated object. Note that in Rust, - /// every (stack-allocated) variable is considered a separate allocated object. + /// byte past the end of the same [allocated object]. /// - /// * The distance between the pointers, **in bytes**, cannot overflow an `isize`. + /// * Both pointers must be *derived from* a pointer to the same object. + /// (See below for an example.) /// /// * The distance between the pointers, in bytes, must be an exact multiple /// of the size of `T`. /// + /// * The distance between the pointers, **in bytes**, cannot overflow an `isize`. + /// /// * The distance being in bounds cannot rely on "wrapping around" the address space. /// - /// The compiler and standard library generally try to ensure allocations - /// never reach a size where an offset is a concern. For instance, `Vec` - /// and `Box` ensure they never allocate more than `isize::MAX` bytes, so - /// `ptr_into_vec.offset_from(vec.as_ptr())` is always safe. + /// Rust types are never larger than `isize::MAX` and Rust allocations never wrap around the + /// address space, so two pointers within some value of any Rust type `T` will always satisfy + /// the last two conditions. The standard library also generally ensures that allocations + /// never reach a size where an offset is a concern. For instance, `Vec` and `Box` ensure they + /// never allocate more than `isize::MAX` bytes, so `ptr_into_vec.offset_from(vec.as_ptr())` + /// always satisfies the last two conditions. /// - /// Most platforms fundamentally can't even construct such an allocation. + /// Most platforms fundamentally can't even construct such a large allocation. /// For instance, no known 64-bit platform can ever serve a request /// for 263 bytes due to page-table limitations or splitting the address space. /// However, some 32-bit and 16-bit platforms may successfully serve a request for /// more than `isize::MAX` bytes with things like Physical Address /// Extension. As such, memory acquired directly from allocators or memory /// mapped files *may* be too large to handle with this function. + /// (Note that [`offset`] and [`add`] also have a similar limitation and hence cannot be used on + /// such large allocations either.) /// - /// Consider using [`wrapping_offset_from`] instead if these constraints are - /// difficult to satisfy. The only advantage of this method is that it - /// enables more aggressive compiler optimizations. + /// [`add`]: #method.add + /// [allocated object]: crate::ptr#allocated-object /// /// # Panics /// @@ -267,8 +665,6 @@ impl *const T { /// Basic usage: /// /// ``` - /// #![feature(ptr_offset_from)] - /// /// let a = [0; 5]; /// let ptr1: *const i32 = &a[1]; /// let ptr2: *const i32 = &a[3]; @@ -279,63 +675,197 @@ impl *const T { /// assert_eq!(ptr2.offset(-2), ptr1); /// } /// ``` - #[unstable(feature = "ptr_offset_from", issue = "41079")] - #[rustc_const_unstable(feature = "const_ptr_offset_from", issue = "41079")] + /// + /// *Incorrect* usage: + /// + /// ```rust,no_run + /// let ptr1 = Box::into_raw(Box::new(0u8)) as *const u8; + /// let ptr2 = Box::into_raw(Box::new(1u8)) as *const u8; + /// let diff = (ptr2 as isize).wrapping_sub(ptr1 as isize); + /// // Make ptr2_other an "alias" of ptr2, but derived from ptr1. + /// let ptr2_other = (ptr1 as *const u8).wrapping_offset(diff); + /// assert_eq!(ptr2 as usize, ptr2_other as usize); + /// // Since ptr2_other and ptr2 are derived from pointers to different objects, + /// // computing their offset is undefined behavior, even though + /// // they point to the same address! + /// unsafe { + /// let zero = ptr2_other.offset_from(ptr2); // Undefined Behavior + /// } + /// ``` + #[stable(feature = "ptr_offset_from", since = "1.47.0")] + #[rustc_const_stable(feature = "const_ptr_offset_from", since = "1.65.0")] #[inline] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn offset_from(self, origin: *const T) -> isize where T: Sized, { let pointee_size = mem::size_of::(); - assert!(0 < pointee_size && pointee_size <= isize::max_value() as usize); - intrinsics::ptr_offset_from(self, origin) + assert!(0 < pointee_size && pointee_size <= isize::MAX as usize); + // SAFETY: the caller must uphold the safety contract for `ptr_offset_from`. + unsafe { intrinsics::ptr_offset_from(self, origin) } } /// Calculates the distance between two pointers. The returned value is in + /// units of **bytes**. + /// + /// This is purely a convenience for casting to a `u8` pointer and + /// using [offset_from][pointer::offset_from] on it. See that method for + /// documentation and safety requirements. + /// + /// For non-`Sized` pointees this operation considers only the data pointers, + /// ignoring the metadata. + #[inline(always)] + #[unstable(feature = "pointer_byte_offsets", issue = "96283")] + #[rustc_const_unstable(feature = "const_pointer_byte_offsets", issue = "96283")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn byte_offset_from(self, origin: *const U) -> isize { + // SAFETY: the caller must uphold the safety contract for `offset_from`. + unsafe { self.cast::().offset_from(origin.cast::()) } + } + + /// Calculates the distance between two pointers, *where it's known that + /// `self` is equal to or greater than `origin`*. The returned value is in /// units of T: the distance in bytes is divided by `mem::size_of::()`. /// - /// If the address different between the two pointers is not a multiple of - /// `mem::size_of::()` then the result of the division is rounded towards - /// zero. + /// This computes the same value that [`offset_from`](#method.offset_from) + /// would compute, but with the added precondition that the offset is + /// guaranteed to be non-negative. This method is equivalent to + /// `usize::from(self.offset_from(origin)).unwrap_unchecked()`, + /// but it provides slightly more information to the optimizer, which can + /// sometimes allow it to optimize slightly better with some backends. + /// + /// This method can be though of as recovering the `count` that was passed + /// to [`add`](#method.add) (or, with the parameters in the other order, + /// to [`sub`](#method.sub)). The following are all equivalent, assuming + /// that their safety preconditions are met: + /// ```rust + /// # #![feature(ptr_sub_ptr)] + /// # unsafe fn blah(ptr: *const i32, origin: *const i32, count: usize) -> bool { + /// ptr.sub_ptr(origin) == count + /// # && + /// origin.add(count) == ptr + /// # && + /// ptr.sub(count) == origin + /// # } + /// ``` + /// + /// # Safety + /// + /// - The distance between the pointers must be non-negative (`self >= origin`) + /// + /// - *All* the safety conditions of [`offset_from`](#method.offset_from) + /// apply to this method as well; see it for the full details. /// - /// Though this method is safe for any two pointers, note that its result - /// will be mostly useless if the two pointers aren't into the same allocated - /// object, for example if they point to two different local variables. + /// Importantly, despite the return type of this method being able to represent + /// a larger offset, it's still *not permitted* to pass pointers which differ + /// by more than `isize::MAX` *bytes*. As such, the result of this method will + /// always be less than or equal to `isize::MAX as usize`. /// /// # Panics /// - /// This function panics if `T` is a zero-sized type. + /// This function panics if `T` is a Zero-Sized Type ("ZST"). /// /// # Examples /// - /// Basic usage: - /// /// ``` - /// #![feature(ptr_wrapping_offset_from)] + /// #![feature(ptr_sub_ptr)] /// /// let a = [0; 5]; /// let ptr1: *const i32 = &a[1]; /// let ptr2: *const i32 = &a[3]; - /// assert_eq!(ptr2.wrapping_offset_from(ptr1), 2); - /// assert_eq!(ptr1.wrapping_offset_from(ptr2), -2); - /// assert_eq!(ptr1.wrapping_offset(2), ptr2); - /// assert_eq!(ptr2.wrapping_offset(-2), ptr1); + /// unsafe { + /// assert_eq!(ptr2.sub_ptr(ptr1), 2); + /// assert_eq!(ptr1.add(2), ptr2); + /// assert_eq!(ptr2.sub(2), ptr1); + /// assert_eq!(ptr2.sub_ptr(ptr2), 0); + /// } /// - /// let ptr1: *const i32 = 3 as _; - /// let ptr2: *const i32 = 13 as _; - /// assert_eq!(ptr2.wrapping_offset_from(ptr1), 2); + /// // This would be incorrect, as the pointers are not correctly ordered: + /// // ptr1.sub_ptr(ptr2) /// ``` - #[unstable(feature = "ptr_wrapping_offset_from", issue = "41079")] + #[unstable(feature = "ptr_sub_ptr", issue = "95892")] + #[rustc_const_unstable(feature = "const_ptr_sub_ptr", issue = "95892")] #[inline] - pub fn wrapping_offset_from(self, origin: *const T) -> isize + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn sub_ptr(self, origin: *const T) -> usize where T: Sized, { + let this = self; + // SAFETY: The comparison has no side-effects, and the intrinsic + // does this check internally in the CTFE implementation. + unsafe { + assert_unsafe_precondition!( + "ptr::sub_ptr requires `this >= origin`", + [T](this: *const T, origin: *const T) => this >= origin + ) + }; + let pointee_size = mem::size_of::(); - assert!(0 < pointee_size && pointee_size <= isize::max_value() as usize); + assert!(0 < pointee_size && pointee_size <= isize::MAX as usize); + // SAFETY: the caller must uphold the safety contract for `ptr_offset_from_unsigned`. + unsafe { intrinsics::ptr_offset_from_unsigned(self, origin) } + } + + /// Returns whether two pointers are guaranteed to be equal. + /// + /// At runtime this function behaves like `Some(self == other)`. + /// However, in some contexts (e.g., compile-time evaluation), + /// it is not always possible to determine equality of two pointers, so this function may + /// spuriously return `None` for pointers that later actually turn out to have its equality known. + /// But when it returns `Some`, the pointers' equality is guaranteed to be known. + /// + /// The return value may change from `Some` to `None` and vice versa depending on the compiler + /// version and unsafe code must not + /// rely on the result of this function for soundness. It is suggested to only use this function + /// for performance optimizations where spurious `None` return values by this function do not + /// affect the outcome, but just the performance. + /// The consequences of using this method to make runtime and compile-time code behave + /// differently have not been explored. This method should not be used to introduce such + /// differences, and it should also not be stabilized before we have a better understanding + /// of this issue. + #[unstable(feature = "const_raw_ptr_comparison", issue = "53020")] + #[rustc_const_unstable(feature = "const_raw_ptr_comparison", issue = "53020")] + #[inline] + pub const fn guaranteed_eq(self, other: *const T) -> Option + where + T: Sized, + { + match intrinsics::ptr_guaranteed_cmp(self as _, other as _) { + 2 => None, + other => Some(other == 1), + } + } - let d = isize::wrapping_sub(self as _, origin as _); - d.wrapping_div(pointee_size as _) + /// Returns whether two pointers are guaranteed to be inequal. + /// + /// At runtime this function behaves like `Some(self != other)`. + /// However, in some contexts (e.g., compile-time evaluation), + /// it is not always possible to determine inequality of two pointers, so this function may + /// spuriously return `None` for pointers that later actually turn out to have its inequality known. + /// But when it returns `Some`, the pointers' inequality is guaranteed to be known. + /// + /// The return value may change from `Some` to `None` and vice versa depending on the compiler + /// version and unsafe code must not + /// rely on the result of this function for soundness. It is suggested to only use this function + /// for performance optimizations where spurious `None` return values by this function do not + /// affect the outcome, but just the performance. + /// The consequences of using this method to make runtime and compile-time code behave + /// differently have not been explored. This method should not be used to introduce such + /// differences, and it should also not be stabilized before we have a better understanding + /// of this issue. + #[unstable(feature = "const_raw_ptr_comparison", issue = "53020")] + #[rustc_const_unstable(feature = "const_raw_ptr_comparison", issue = "53020")] + #[inline] + pub const fn guaranteed_ne(self, other: *const T) -> Option + where + T: Sized, + { + match self.guaranteed_eq(other) { + None => None, + Some(eq) => Some(!eq), + } } /// Calculates the offset from a pointer (convenience for `.offset(count as isize)`). @@ -349,8 +879,7 @@ impl *const T { /// Behavior: /// /// * Both the starting and resulting pointer must be either in bounds or one - /// byte past the end of the same allocated object. Note that in Rust, - /// every (stack-allocated) variable is considered a separate allocated object. + /// byte past the end of the same [allocated object]. /// /// * The computed offset, **in bytes**, cannot overflow an `isize`. /// @@ -375,6 +904,7 @@ impl *const T { /// enables more aggressive compiler optimizations. /// /// [`wrapping_add`]: #method.wrapping_add + /// [allocated object]: crate::ptr#allocated-object /// /// # Examples /// @@ -390,12 +920,36 @@ impl *const T { /// } /// ``` #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] - pub unsafe fn add(self, count: usize) -> Self + #[must_use = "returns a new pointer rather than modifying its argument"] + #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn add(self, count: usize) -> Self where T: Sized, { - self.offset(count as isize) + // SAFETY: the caller must uphold the safety contract for `offset`. + unsafe { self.offset(count as isize) } + } + + /// Calculates the offset from a pointer in bytes (convenience for `.byte_offset(count as isize)`). + /// + /// `count` is in units of bytes. + /// + /// This is purely a convenience for casting to a `u8` pointer and + /// using [add][pointer::add] on it. See that method for documentation + /// and safety requirements. + /// + /// For non-`Sized` pointees this operation changes only the data pointer, + /// leaving the metadata untouched. + #[must_use] + #[inline(always)] + #[unstable(feature = "pointer_byte_offsets", issue = "96283")] + #[rustc_const_unstable(feature = "const_pointer_byte_offsets", issue = "96283")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn byte_add(self, count: usize) -> Self { + // SAFETY: the caller must uphold the safety contract for `add`. + unsafe { self.cast::().add(count).with_metadata_of(self) } } /// Calculates the offset from a pointer (convenience for @@ -410,8 +964,7 @@ impl *const T { /// Behavior: /// /// * Both the starting and resulting pointer must be either in bounds or one - /// byte past the end of the same allocated object. Note that in Rust, - /// every (stack-allocated) variable is considered a separate allocated object. + /// byte past the end of the same [allocated object]. /// /// * The computed offset cannot exceed `isize::MAX` **bytes**. /// @@ -436,6 +989,7 @@ impl *const T { /// enables more aggressive compiler optimizations. /// /// [`wrapping_sub`]: #method.wrapping_sub + /// [allocated object]: crate::ptr#allocated-object /// /// # Examples /// @@ -451,12 +1005,37 @@ impl *const T { /// } /// ``` #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] - pub unsafe fn sub(self, count: usize) -> Self + #[must_use = "returns a new pointer rather than modifying its argument"] + #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn sub(self, count: usize) -> Self where T: Sized, { - self.offset((count as isize).wrapping_neg()) + // SAFETY: the caller must uphold the safety contract for `offset`. + unsafe { self.offset((count as isize).wrapping_neg()) } + } + + /// Calculates the offset from a pointer in bytes (convenience for + /// `.byte_offset((count as isize).wrapping_neg())`). + /// + /// `count` is in units of bytes. + /// + /// This is purely a convenience for casting to a `u8` pointer and + /// using [sub][pointer::sub] on it. See that method for documentation + /// and safety requirements. + /// + /// For non-`Sized` pointees this operation changes only the data pointer, + /// leaving the metadata untouched. + #[must_use] + #[inline(always)] + #[unstable(feature = "pointer_byte_offsets", issue = "96283")] + #[rustc_const_unstable(feature = "const_pointer_byte_offsets", issue = "96283")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn byte_sub(self, count: usize) -> Self { + // SAFETY: the caller must uphold the safety contract for `sub`. + unsafe { self.cast::().sub(count).with_metadata_of(self) } } /// Calculates the offset from a pointer using wrapping arithmetic. @@ -467,24 +1046,29 @@ impl *const T { /// /// # Safety /// - /// The resulting pointer does not need to be in bounds, but it is - /// potentially hazardous to dereference (which requires `unsafe`). + /// This operation itself is always safe, but using the resulting pointer is not. /// - /// In particular, the resulting pointer remains attached to the same allocated - /// object that `self` points to. It may *not* be used to access a - /// different allocated object. Note that in Rust, - /// every (stack-allocated) variable is considered a separate allocated object. + /// The resulting pointer "remembers" the [allocated object] that `self` points to; it must not + /// be used to read or write other allocated objects. /// - /// Compared to [`add`], this method basically delays the requirement of staying - /// within the same allocated object: [`add`] is immediate Undefined Behavior when - /// crossing object boundaries; `wrapping_add` produces a pointer but still leads - /// to Undefined Behavior if that pointer is dereferenced. [`add`] can be optimized - /// better and is thus preferable in performance-sensitive code. + /// In other words, `let z = x.wrapping_add((y as usize) - (x as usize))` does *not* make `z` + /// the same as `y` even if we assume `T` has size `1` and there is no overflow: `z` is still + /// attached to the object `x` is attached to, and dereferencing it is Undefined Behavior unless + /// `x` and `y` point into the same allocated object. /// - /// If you need to cross object boundaries, cast the pointer to an integer and - /// do the arithmetic there. + /// Compared to [`add`], this method basically delays the requirement of staying within the + /// same allocated object: [`add`] is immediate Undefined Behavior when crossing object + /// boundaries; `wrapping_add` produces a pointer but still leads to Undefined Behavior if a + /// pointer is dereferenced when it is out-of-bounds of the object it is attached to. [`add`] + /// can be optimized better and is thus preferable in performance-sensitive code. + /// + /// The delayed check only considers the value of the pointer that was dereferenced, not the + /// intermediate values used during the computation of the final result. For example, + /// `x.wrapping_add(o).wrapping_sub(o)` is always the same as `x`. In other words, leaving the + /// allocated object and then re-entering it later is permitted. /// /// [`add`]: #method.add + /// [allocated object]: crate::ptr#allocated-object /// /// # Examples /// @@ -506,40 +1090,65 @@ impl *const T { /// } /// ``` #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] - pub fn wrapping_add(self, count: usize) -> Self + #[must_use = "returns a new pointer rather than modifying its argument"] + #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] + #[inline(always)] + pub const fn wrapping_add(self, count: usize) -> Self where T: Sized, { self.wrapping_offset(count as isize) } + /// Calculates the offset from a pointer in bytes using wrapping arithmetic. + /// (convenience for `.wrapping_byte_offset(count as isize)`) + /// + /// `count` is in units of bytes. + /// + /// This is purely a convenience for casting to a `u8` pointer and + /// using [wrapping_add][pointer::wrapping_add] on it. See that method for documentation. + /// + /// For non-`Sized` pointees this operation changes only the data pointer, + /// leaving the metadata untouched. + #[must_use] + #[inline(always)] + #[unstable(feature = "pointer_byte_offsets", issue = "96283")] + #[rustc_const_unstable(feature = "const_pointer_byte_offsets", issue = "96283")] + pub const fn wrapping_byte_add(self, count: usize) -> Self { + self.cast::().wrapping_add(count).with_metadata_of(self) + } + /// Calculates the offset from a pointer using wrapping arithmetic. - /// (convenience for `.wrapping_offset((count as isize).wrapping_sub())`) + /// (convenience for `.wrapping_offset((count as isize).wrapping_neg())`) /// /// `count` is in units of T; e.g., a `count` of 3 represents a pointer /// offset of `3 * size_of::()` bytes. /// /// # Safety /// - /// The resulting pointer does not need to be in bounds, but it is - /// potentially hazardous to dereference (which requires `unsafe`). + /// This operation itself is always safe, but using the resulting pointer is not. + /// + /// The resulting pointer "remembers" the [allocated object] that `self` points to; it must not + /// be used to read or write other allocated objects. /// - /// In particular, the resulting pointer remains attached to the same allocated - /// object that `self` points to. It may *not* be used to access a - /// different allocated object. Note that in Rust, - /// every (stack-allocated) variable is considered a separate allocated object. + /// In other words, `let z = x.wrapping_sub((x as usize) - (y as usize))` does *not* make `z` + /// the same as `y` even if we assume `T` has size `1` and there is no overflow: `z` is still + /// attached to the object `x` is attached to, and dereferencing it is Undefined Behavior unless + /// `x` and `y` point into the same allocated object. /// - /// Compared to [`sub`], this method basically delays the requirement of staying - /// within the same allocated object: [`sub`] is immediate Undefined Behavior when - /// crossing object boundaries; `wrapping_sub` produces a pointer but still leads - /// to Undefined Behavior if that pointer is dereferenced. [`sub`] can be optimized - /// better and is thus preferable in performance-sensitive code. + /// Compared to [`sub`], this method basically delays the requirement of staying within the + /// same allocated object: [`sub`] is immediate Undefined Behavior when crossing object + /// boundaries; `wrapping_sub` produces a pointer but still leads to Undefined Behavior if a + /// pointer is dereferenced when it is out-of-bounds of the object it is attached to. [`sub`] + /// can be optimized better and is thus preferable in performance-sensitive code. /// - /// If you need to cross object boundaries, cast the pointer to an integer and - /// do the arithmetic there. + /// The delayed check only considers the value of the pointer that was dereferenced, not the + /// intermediate values used during the computation of the final result. For example, + /// `x.wrapping_add(o).wrapping_sub(o)` is always the same as `x`. In other words, leaving the + /// allocated object and then re-entering it later is permitted. /// /// [`sub`]: #method.sub + /// [allocated object]: crate::ptr#allocated-object /// /// # Examples /// @@ -561,27 +1170,50 @@ impl *const T { /// } /// ``` #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] - pub fn wrapping_sub(self, count: usize) -> Self + #[must_use = "returns a new pointer rather than modifying its argument"] + #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] + #[inline(always)] + pub const fn wrapping_sub(self, count: usize) -> Self where T: Sized, { self.wrapping_offset((count as isize).wrapping_neg()) } + /// Calculates the offset from a pointer in bytes using wrapping arithmetic. + /// (convenience for `.wrapping_offset((count as isize).wrapping_neg())`) + /// + /// `count` is in units of bytes. + /// + /// This is purely a convenience for casting to a `u8` pointer and + /// using [wrapping_sub][pointer::wrapping_sub] on it. See that method for documentation. + /// + /// For non-`Sized` pointees this operation changes only the data pointer, + /// leaving the metadata untouched. + #[must_use] + #[inline(always)] + #[unstable(feature = "pointer_byte_offsets", issue = "96283")] + #[rustc_const_unstable(feature = "const_pointer_byte_offsets", issue = "96283")] + pub const fn wrapping_byte_sub(self, count: usize) -> Self { + self.cast::().wrapping_sub(count).with_metadata_of(self) + } + /// Reads the value from `self` without moving it. This leaves the /// memory in `self` unchanged. /// /// See [`ptr::read`] for safety concerns and examples. /// - /// [`ptr::read`]: ./ptr/fn.read.html + /// [`ptr::read`]: crate::ptr::read() #[stable(feature = "pointer_methods", since = "1.26.0")] + #[rustc_const_unstable(feature = "const_ptr_read", issue = "80377")] #[inline] - pub unsafe fn read(self) -> T + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn read(self) -> T where T: Sized, { - read(self) + // SAFETY: the caller must uphold the safety contract for `read`. + unsafe { read(self) } } /// Performs a volatile read of the value from `self` without moving it. This @@ -593,14 +1225,16 @@ impl *const T { /// /// See [`ptr::read_volatile`] for safety concerns and examples. /// - /// [`ptr::read_volatile`]: ./ptr/fn.read_volatile.html + /// [`ptr::read_volatile`]: crate::ptr::read_volatile() #[stable(feature = "pointer_methods", since = "1.26.0")] #[inline] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub unsafe fn read_volatile(self) -> T where T: Sized, { - read_volatile(self) + // SAFETY: the caller must uphold the safety contract for `read_volatile`. + unsafe { read_volatile(self) } } /// Reads the value from `self` without moving it. This leaves the @@ -610,14 +1244,17 @@ impl *const T { /// /// See [`ptr::read_unaligned`] for safety concerns and examples. /// - /// [`ptr::read_unaligned`]: ./ptr/fn.read_unaligned.html + /// [`ptr::read_unaligned`]: crate::ptr::read_unaligned() #[stable(feature = "pointer_methods", since = "1.26.0")] + #[rustc_const_unstable(feature = "const_ptr_read", issue = "80377")] #[inline] - pub unsafe fn read_unaligned(self) -> T + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn read_unaligned(self) -> T where T: Sized, { - read_unaligned(self) + // SAFETY: the caller must uphold the safety contract for `read_unaligned`. + unsafe { read_unaligned(self) } } /// Copies `count * size_of` bytes from `self` to `dest`. The source @@ -627,14 +1264,17 @@ impl *const T { /// /// See [`ptr::copy`] for safety concerns and examples. /// - /// [`ptr::copy`]: ./ptr/fn.copy.html + /// [`ptr::copy`]: crate::ptr::copy() + #[rustc_const_stable(feature = "const_intrinsic_copy", since = "1.63.0")] #[stable(feature = "pointer_methods", since = "1.26.0")] #[inline] - pub unsafe fn copy_to(self, dest: *mut T, count: usize) + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn copy_to(self, dest: *mut T, count: usize) where T: Sized, { - copy(self, dest, count) + // SAFETY: the caller must uphold the safety contract for `copy`. + unsafe { copy(self, dest, count) } } /// Copies `count * size_of` bytes from `self` to `dest`. The source @@ -644,22 +1284,25 @@ impl *const T { /// /// See [`ptr::copy_nonoverlapping`] for safety concerns and examples. /// - /// [`ptr::copy_nonoverlapping`]: ./ptr/fn.copy_nonoverlapping.html + /// [`ptr::copy_nonoverlapping`]: crate::ptr::copy_nonoverlapping() + #[rustc_const_stable(feature = "const_intrinsic_copy", since = "1.63.0")] #[stable(feature = "pointer_methods", since = "1.26.0")] #[inline] - pub unsafe fn copy_to_nonoverlapping(self, dest: *mut T, count: usize) + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn copy_to_nonoverlapping(self, dest: *mut T, count: usize) where T: Sized, { - copy_nonoverlapping(self, dest, count) + // SAFETY: the caller must uphold the safety contract for `copy_nonoverlapping`. + unsafe { copy_nonoverlapping(self, dest, count) } } /// Computes the offset that needs to be applied to the pointer in order to make it aligned to /// `align`. /// /// If it is not possible to align the pointer, the implementation returns - /// `usize::max_value()`. It is permissible for the implementation to *always* - /// return `usize::max_value()`. Only your algorithm's performance can depend + /// `usize::MAX`. It is permissible for the implementation to *always* + /// return `usize::MAX`. Only your algorithm's performance can depend /// on getting a usable offset here, not its correctness. /// /// The offset is expressed in number of `T` elements, and not bytes. The value returned can be @@ -678,30 +1321,412 @@ impl *const T { /// Accessing adjacent `u8` as `u16` /// /// ``` - /// # fn foo(n: usize) { - /// # use std::mem::align_of; + /// use std::mem::align_of; + /// /// # unsafe { - /// let x = [5u8, 6u8, 7u8, 8u8, 9u8]; - /// let ptr = &x[n] as *const u8; + /// let x = [5_u8, 6, 7, 8, 9]; + /// let ptr = x.as_ptr(); /// let offset = ptr.align_offset(align_of::()); - /// if offset < x.len() - n - 1 { - /// let u16_ptr = ptr.add(offset) as *const u16; - /// assert_ne!(*u16_ptr, 500); + /// + /// if offset < x.len() - 1 { + /// let u16_ptr = ptr.add(offset).cast::(); + /// assert!(*u16_ptr == u16::from_ne_bytes([5, 6]) || *u16_ptr == u16::from_ne_bytes([6, 7])); /// } else { /// // while the pointer can be aligned via `offset`, it would point /// // outside the allocation /// } - /// # } } + /// # } /// ``` + #[must_use] + #[inline] #[stable(feature = "align_offset", since = "1.36.0")] - pub fn align_offset(self, align: usize) -> usize + #[rustc_const_unstable(feature = "const_align_offset", issue = "90962")] + pub const fn align_offset(self, align: usize) -> usize where T: Sized, { if !align.is_power_of_two() { panic!("align_offset: align is not a power-of-two"); } - unsafe { align_offset(self, align) } + + { + // SAFETY: `align` has been checked to be a power of 2 above + unsafe { align_offset(self, align) } + } + } + + /// Returns whether the pointer is properly aligned for `T`. + /// + /// # Examples + /// + /// Basic usage: + /// ``` + /// #![feature(pointer_is_aligned)] + /// #![feature(pointer_byte_offsets)] + /// + /// // On some platforms, the alignment of i32 is less than 4. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// + /// let data = AlignedI32(42); + /// let ptr = &data as *const AlignedI32; + /// + /// assert!(ptr.is_aligned()); + /// assert!(!ptr.wrapping_byte_add(1).is_aligned()); + /// ``` + /// + /// # At compiletime + /// **Note: Alignment at compiletime is experimental and subject to change. See the + /// [tracking issue] for details.** + /// + /// At compiletime, the compiler may not know where a value will end up in memory. + /// Calling this function on a pointer created from a reference at compiletime will only + /// return `true` if the pointer is guaranteed to be aligned. This means that the pointer + /// is never aligned if cast to a type with a stricter alignment than the reference's + /// underlying allocation. + /// + /// ``` + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// // On some platforms, the alignment of primitives is less than their size. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// #[repr(align(8))] + /// struct AlignedI64(i64); + /// + /// const _: () = { + /// let data = AlignedI32(42); + /// let ptr = &data as *const AlignedI32; + /// assert!(ptr.is_aligned()); + /// + /// // At runtime either `ptr1` or `ptr2` would be aligned, but at compiletime neither is aligned. + /// let ptr1 = ptr.cast::(); + /// let ptr2 = ptr.wrapping_add(1).cast::(); + /// assert!(!ptr1.is_aligned()); + /// assert!(!ptr2.is_aligned()); + /// }; + /// ``` + /// + /// Due to this behavior, it is possible that a runtime pointer derived from a compiletime + /// pointer is aligned, even if the compiletime pointer wasn't aligned. + /// + /// ``` + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// // On some platforms, the alignment of primitives is less than their size. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// #[repr(align(8))] + /// struct AlignedI64(i64); + /// + /// // At compiletime, neither `COMPTIME_PTR` nor `COMPTIME_PTR + 1` is aligned. + /// const COMPTIME_PTR: *const AlignedI32 = &AlignedI32(42); + /// const _: () = assert!(!COMPTIME_PTR.cast::().is_aligned()); + /// const _: () = assert!(!COMPTIME_PTR.wrapping_add(1).cast::().is_aligned()); + /// + /// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned. + /// let runtime_ptr = COMPTIME_PTR; + /// assert_ne!( + /// runtime_ptr.cast::().is_aligned(), + /// runtime_ptr.wrapping_add(1).cast::().is_aligned(), + /// ); + /// ``` + /// + /// If a pointer is created from a fixed address, this function behaves the same during + /// runtime and compiletime. + /// + /// ``` + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// // On some platforms, the alignment of primitives is less than their size. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// #[repr(align(8))] + /// struct AlignedI64(i64); + /// + /// const _: () = { + /// let ptr = 40 as *const AlignedI32; + /// assert!(ptr.is_aligned()); + /// + /// // For pointers with a known address, runtime and compiletime behavior are identical. + /// let ptr1 = ptr.cast::(); + /// let ptr2 = ptr.wrapping_add(1).cast::(); + /// assert!(ptr1.is_aligned()); + /// assert!(!ptr2.is_aligned()); + /// }; + /// ``` + /// + /// [tracking issue]: https://github.com/rust-lang/rust/issues/104203 + #[must_use] + #[inline] + #[unstable(feature = "pointer_is_aligned", issue = "96284")] + #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")] + pub const fn is_aligned(self) -> bool + where + T: Sized, + { + self.is_aligned_to(mem::align_of::()) + } + + /// Returns whether the pointer is aligned to `align`. + /// + /// For non-`Sized` pointees this operation considers only the data pointer, + /// ignoring the metadata. + /// + /// # Panics + /// + /// The function panics if `align` is not a power-of-two (this includes 0). + /// + /// # Examples + /// + /// Basic usage: + /// ``` + /// #![feature(pointer_is_aligned)] + /// #![feature(pointer_byte_offsets)] + /// + /// // On some platforms, the alignment of i32 is less than 4. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// + /// let data = AlignedI32(42); + /// let ptr = &data as *const AlignedI32; + /// + /// assert!(ptr.is_aligned_to(1)); + /// assert!(ptr.is_aligned_to(2)); + /// assert!(ptr.is_aligned_to(4)); + /// + /// assert!(ptr.wrapping_byte_add(2).is_aligned_to(2)); + /// assert!(!ptr.wrapping_byte_add(2).is_aligned_to(4)); + /// + /// assert_ne!(ptr.is_aligned_to(8), ptr.wrapping_add(1).is_aligned_to(8)); + /// ``` + /// + /// # At compiletime + /// **Note: Alignment at compiletime is experimental and subject to change. See the + /// [tracking issue] for details.** + /// + /// At compiletime, the compiler may not know where a value will end up in memory. + /// Calling this function on a pointer created from a reference at compiletime will only + /// return `true` if the pointer is guaranteed to be aligned. This means that the pointer + /// cannot be stricter aligned than the reference's underlying allocation. + /// + /// ``` + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// // On some platforms, the alignment of i32 is less than 4. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// + /// const _: () = { + /// let data = AlignedI32(42); + /// let ptr = &data as *const AlignedI32; + /// + /// assert!(ptr.is_aligned_to(1)); + /// assert!(ptr.is_aligned_to(2)); + /// assert!(ptr.is_aligned_to(4)); + /// + /// // At compiletime, we know for sure that the pointer isn't aligned to 8. + /// assert!(!ptr.is_aligned_to(8)); + /// assert!(!ptr.wrapping_add(1).is_aligned_to(8)); + /// }; + /// ``` + /// + /// Due to this behavior, it is possible that a runtime pointer derived from a compiletime + /// pointer is aligned, even if the compiletime pointer wasn't aligned. + /// + /// ``` + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// // On some platforms, the alignment of i32 is less than 4. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// + /// // At compiletime, neither `COMPTIME_PTR` nor `COMPTIME_PTR + 1` is aligned. + /// const COMPTIME_PTR: *const AlignedI32 = &AlignedI32(42); + /// const _: () = assert!(!COMPTIME_PTR.is_aligned_to(8)); + /// const _: () = assert!(!COMPTIME_PTR.wrapping_add(1).is_aligned_to(8)); + /// + /// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned. + /// let runtime_ptr = COMPTIME_PTR; + /// assert_ne!( + /// runtime_ptr.is_aligned_to(8), + /// runtime_ptr.wrapping_add(1).is_aligned_to(8), + /// ); + /// ``` + /// + /// If a pointer is created from a fixed address, this function behaves the same during + /// runtime and compiletime. + /// + /// ``` + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// const _: () = { + /// let ptr = 40 as *const u8; + /// assert!(ptr.is_aligned_to(1)); + /// assert!(ptr.is_aligned_to(2)); + /// assert!(ptr.is_aligned_to(4)); + /// assert!(ptr.is_aligned_to(8)); + /// assert!(!ptr.is_aligned_to(16)); + /// }; + /// ``` + /// + /// [tracking issue]: https://github.com/rust-lang/rust/issues/104203 + #[must_use] + #[inline] + #[unstable(feature = "pointer_is_aligned", issue = "96284")] + #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")] + pub const fn is_aligned_to(self, align: usize) -> bool { + if !align.is_power_of_two() { + panic!("is_aligned_to: align is not a power-of-two"); + } + + #[inline] + fn runtime_impl(ptr: *const (), align: usize) -> bool { + ptr.addr() & (align - 1) == 0 + } + + #[inline] + const fn const_impl(ptr: *const (), align: usize) -> bool { + // We can't use the address of `self` in a `const fn`, so we use `align_offset` instead. + // The cast to `()` is used to + // 1. deal with fat pointers; and + // 2. ensure that `align_offset` doesn't actually try to compute an offset. + ptr.align_offset(align) == 0 + } + + // SAFETY: The two versions are equivalent at runtime. + unsafe { const_eval_select((self.cast::<()>(), align), const_impl, runtime_impl) } + } +} + +impl *const [T] { + /// Returns the length of a raw slice. + /// + /// The returned value is the number of **elements**, not the number of bytes. + /// + /// This function is safe, even when the raw slice cannot be cast to a slice + /// reference because the pointer is null or unaligned. + /// + /// # Examples + /// + /// ```rust + /// #![feature(slice_ptr_len)] + /// + /// use std::ptr; + /// + /// let slice: *const [i8] = ptr::slice_from_raw_parts(ptr::null(), 3); + /// assert_eq!(slice.len(), 3); + /// ``` + #[inline] + #[unstable(feature = "slice_ptr_len", issue = "71146")] + #[rustc_const_unstable(feature = "const_slice_ptr_len", issue = "71146")] + pub const fn len(self) -> usize { + metadata(self) + } + + /// Returns a raw pointer to the slice's buffer. + /// + /// This is equivalent to casting `self` to `*const T`, but more type-safe. + /// + /// # Examples + /// + /// ```rust + /// #![feature(slice_ptr_get)] + /// use std::ptr; + /// + /// let slice: *const [i8] = ptr::slice_from_raw_parts(ptr::null(), 3); + /// assert_eq!(slice.as_ptr(), ptr::null()); + /// ``` + #[inline] + #[unstable(feature = "slice_ptr_get", issue = "74265")] + #[rustc_const_unstable(feature = "slice_ptr_get", issue = "74265")] + pub const fn as_ptr(self) -> *const T { + self as *const T + } + + /// Returns a raw pointer to an element or subslice, without doing bounds + /// checking. + /// + /// Calling this method with an out-of-bounds index or when `self` is not dereferenceable + /// is *[undefined behavior]* even if the resulting pointer is not used. + /// + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_ptr_get)] + /// + /// let x = &[1, 2, 4] as *const [i32]; + /// + /// unsafe { + /// assert_eq!(x.get_unchecked(1), x.as_ptr().add(1)); + /// } + /// ``` + #[unstable(feature = "slice_ptr_get", issue = "74265")] + #[rustc_const_unstable(feature = "const_slice_index", issue = "none")] + #[inline] + pub const unsafe fn get_unchecked(self, index: I) -> *const I::Output + where + I: ~const SliceIndex<[T]>, + { + // SAFETY: the caller ensures that `self` is dereferenceable and `index` in-bounds. + unsafe { index.get_unchecked(self) } + } + + /// Returns `None` if the pointer is null, or else returns a shared slice to + /// the value wrapped in `Some`. In contrast to [`as_ref`], this does not require + /// that the value has to be initialized. + /// + /// [`as_ref`]: #method.as_ref + /// + /// # Safety + /// + /// When calling this method, you have to ensure that *either* the pointer is null *or* + /// all of the following is true: + /// + /// * The pointer must be [valid] for reads for `ptr.len() * mem::size_of::()` many bytes, + /// and it must be properly aligned. This means in particular: + /// + /// * The entire memory range of this slice must be contained within a single [allocated object]! + /// Slices can never span across multiple allocated objects. + /// + /// * The pointer must be aligned even for zero-length slices. One + /// reason for this is that enum layout optimizations may rely on references + /// (including slices of any length) being aligned and non-null to distinguish + /// them from other data. You can obtain a pointer that is usable as `data` + /// for zero-length slices using [`NonNull::dangling()`]. + /// + /// * The total size `ptr.len() * mem::size_of::()` of the slice must be no larger than `isize::MAX`. + /// See the safety documentation of [`pointer::offset`]. + /// + /// * You must enforce Rust's aliasing rules, since the returned lifetime `'a` is + /// arbitrarily chosen and does not necessarily reflect the actual lifetime of the data. + /// In particular, while this reference exists, the memory the pointer points to must + /// not get mutated (except inside `UnsafeCell`). + /// + /// This applies even if the result of this method is unused! + /// + /// See also [`slice::from_raw_parts`][]. + /// + /// [valid]: crate::ptr#safety + /// [allocated object]: crate::ptr#allocated-object + #[inline] + #[unstable(feature = "ptr_as_uninit", issue = "75402")] + #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")] + pub const unsafe fn as_uninit_slice<'a>(self) -> Option<&'a [MaybeUninit]> { + if self.is_null() { + None + } else { + // SAFETY: the caller must uphold the safety contract for `as_uninit_slice`. + Some(unsafe { slice::from_raw_parts(self as *const MaybeUninit, self.len()) }) + } } } diff --git a/crux-mir/lib/core/src/ptr/metadata.rs b/crux-mir/lib/core/src/ptr/metadata.rs new file mode 100644 index 000000000..2ea032d4a --- /dev/null +++ b/crux-mir/lib/core/src/ptr/metadata.rs @@ -0,0 +1,273 @@ +#![unstable(feature = "ptr_metadata", issue = "81513")] + +use crate::fmt; +use crate::hash::{Hash, Hasher}; + +/// Provides the pointer metadata type of any pointed-to type. +/// +/// # Pointer metadata +/// +/// Raw pointer types and reference types in Rust can be thought of as made of two parts: +/// a data pointer that contains the memory address of the value, and some metadata. +/// +/// For statically-sized types (that implement the `Sized` traits) +/// as well as for `extern` types, +/// pointers are said to be “thin”: metadata is zero-sized and its type is `()`. +/// +/// Pointers to [dynamically-sized types][dst] are said to be “wide” or “fat”, +/// they have non-zero-sized metadata: +/// +/// * For structs whose last field is a DST, metadata is the metadata for the last field +/// * For the `str` type, metadata is the length in bytes as `usize` +/// * For slice types like `[T]`, metadata is the length in items as `usize` +/// * For trait objects like `dyn SomeTrait`, metadata is [`DynMetadata`][DynMetadata] +/// (e.g. `DynMetadata`) +/// +/// In the future, the Rust language may gain new kinds of types +/// that have different pointer metadata. +/// +/// [dst]: https://doc.rust-lang.org/nomicon/exotic-sizes.html#dynamically-sized-types-dsts +/// +/// +/// # The `Pointee` trait +/// +/// The point of this trait is its `Metadata` associated type, +/// which is `()` or `usize` or `DynMetadata<_>` as described above. +/// It is automatically implemented for every type. +/// It can be assumed to be implemented in a generic context, even without a corresponding bound. +/// +/// +/// # Usage +/// +/// Raw pointers can be decomposed into the data address and metadata components +/// with their [`to_raw_parts`] method. +/// +/// Alternatively, metadata alone can be extracted with the [`metadata`] function. +/// A reference can be passed to [`metadata`] and implicitly coerced. +/// +/// A (possibly-wide) pointer can be put back together from its address and metadata +/// with [`from_raw_parts`] or [`from_raw_parts_mut`]. +/// +/// [`to_raw_parts`]: *const::to_raw_parts +#[lang = "pointee_trait"] +#[rustc_deny_explicit_impl] +pub trait Pointee { + /// The type for metadata in pointers and references to `Self`. + #[lang = "metadata_type"] + // NOTE: Keep trait bounds in `static_assert_expected_bounds_for_metadata` + // in `library/core/src/ptr/metadata.rs` + // in sync with those here: + type Metadata: Copy + Send + Sync + Ord + Hash + Unpin; +} + +/// Pointers to types implementing this trait alias are “thin”. +/// +/// This includes statically-`Sized` types and `extern` types. +/// +/// # Example +/// +/// ```rust +/// #![feature(ptr_metadata)] +/// +/// fn this_never_panics() { +/// assert_eq!(std::mem::size_of::<&T>(), std::mem::size_of::()) +/// } +/// ``` +#[unstable(feature = "ptr_metadata", issue = "81513")] +// NOTE: don’t stabilize this before trait aliases are stable in the language? +pub trait Thin = Pointee; + +/// Extract the metadata component of a pointer. +/// +/// Values of type `*mut T`, `&T`, or `&mut T` can be passed directly to this function +/// as they implicitly coerce to `*const T`. +/// +/// # Example +/// +/// ``` +/// #![feature(ptr_metadata)] +/// +/// assert_eq!(std::ptr::metadata("foo"), 3_usize); +/// ``` +#[rustc_const_unstable(feature = "ptr_metadata", issue = "81513")] +#[inline] +pub const fn metadata(ptr: *const T) -> ::Metadata { + // SAFETY: Accessing the value from the `PtrRepr` union is safe since *const T + // and PtrComponents have the same memory layouts. Only std can make this + // guarantee. + unsafe { PtrRepr { const_ptr: ptr }.components.metadata } +} + +/// Forms a (possibly-wide) raw pointer from a data address and metadata. +/// +/// This function is safe but the returned pointer is not necessarily safe to dereference. +/// For slices, see the documentation of [`slice::from_raw_parts`] for safety requirements. +/// For trait objects, the metadata must come from a pointer to the same underlying erased type. +/// +/// [`slice::from_raw_parts`]: crate::slice::from_raw_parts +#[unstable(feature = "ptr_metadata", issue = "81513")] +#[rustc_const_unstable(feature = "ptr_metadata", issue = "81513")] +#[inline] +pub const fn from_raw_parts( + data_address: *const (), + metadata: ::Metadata, +) -> *const T { + // SAFETY: Accessing the value from the `PtrRepr` union is safe since *const T + // and PtrComponents have the same memory layouts. Only std can make this + // guarantee. + unsafe { PtrRepr { components: PtrComponents { data_address, metadata } }.const_ptr } +} + +/// Performs the same functionality as [`from_raw_parts`], except that a +/// raw `*mut` pointer is returned, as opposed to a raw `*const` pointer. +/// +/// See the documentation of [`from_raw_parts`] for more details. +#[unstable(feature = "ptr_metadata", issue = "81513")] +#[rustc_const_unstable(feature = "ptr_metadata", issue = "81513")] +#[inline] +pub const fn from_raw_parts_mut( + data_address: *mut (), + metadata: ::Metadata, +) -> *mut T { + // SAFETY: Accessing the value from the `PtrRepr` union is safe since *const T + // and PtrComponents have the same memory layouts. Only std can make this + // guarantee. + unsafe { PtrRepr { components: PtrComponents { data_address, metadata } }.mut_ptr } +} + +#[repr(C)] +union PtrRepr { + const_ptr: *const T, + mut_ptr: *mut T, + components: PtrComponents, +} + +#[repr(C)] +struct PtrComponents { + data_address: *const (), + metadata: ::Metadata, +} + +// Manual impl needed to avoid `T: Copy` bound. +impl Copy for PtrComponents {} + +// Manual impl needed to avoid `T: Clone` bound. +impl Clone for PtrComponents { + fn clone(&self) -> Self { + *self + } +} + +/// The metadata for a `Dyn = dyn SomeTrait` trait object type. +/// +/// It is a pointer to a vtable (virtual call table) +/// that represents all the necessary information +/// to manipulate the concrete type stored inside a trait object. +/// The vtable notably it contains: +/// +/// * type size +/// * type alignment +/// * a pointer to the type’s `drop_in_place` impl (may be a no-op for plain-old-data) +/// * pointers to all the methods for the type’s implementation of the trait +/// +/// Note that the first three are special because they’re necessary to allocate, drop, +/// and deallocate any trait object. +/// +/// It is possible to name this struct with a type parameter that is not a `dyn` trait object +/// (for example `DynMetadata`) but not to obtain a meaningful value of that struct. +#[lang = "dyn_metadata"] +pub struct DynMetadata { + vtable_ptr: &'static VTable, + phantom: crate::marker::PhantomData, +} + +extern "C" { + /// Opaque type for accessing vtables. + /// + /// Private implementation detail of `DynMetadata::size_of` etc. + /// There is conceptually not actually any Abstract Machine memory behind this pointer. + type VTable; +} + +impl DynMetadata { + /// Returns the size of the type associated with this vtable. + #[inline] + pub fn size_of(self) -> usize { + // Note that "size stored in vtable" is *not* the same as "result of size_of_val_raw". + // Consider a reference like `&(i32, dyn Send)`: the vtable will only store the size of the + // `Send` part! + // SAFETY: DynMetadata always contains a valid vtable pointer + return unsafe { + crate::intrinsics::vtable_size(self.vtable_ptr as *const VTable as *const ()) + }; + } + + /// Returns the alignment of the type associated with this vtable. + #[inline] + pub fn align_of(self) -> usize { + // SAFETY: DynMetadata always contains a valid vtable pointer + return unsafe { + crate::intrinsics::vtable_align(self.vtable_ptr as *const VTable as *const ()) + }; + } + + /// Returns the size and alignment together as a `Layout` + #[inline] + pub fn layout(self) -> crate::alloc::Layout { + // SAFETY: the compiler emitted this vtable for a concrete Rust type which + // is known to have a valid layout. Same rationale as in `Layout::for_value`. + unsafe { crate::alloc::Layout::from_size_align_unchecked(self.size_of(), self.align_of()) } + } +} + +unsafe impl Send for DynMetadata {} +unsafe impl Sync for DynMetadata {} + +impl fmt::Debug for DynMetadata { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("DynMetadata").field(&(self.vtable_ptr as *const VTable)).finish() + } +} + +// Manual impls needed to avoid `Dyn: $Trait` bounds. + +impl Unpin for DynMetadata {} + +impl Copy for DynMetadata {} + +impl Clone for DynMetadata { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl Eq for DynMetadata {} + +impl PartialEq for DynMetadata { + #[inline] + fn eq(&self, other: &Self) -> bool { + crate::ptr::eq::(self.vtable_ptr, other.vtable_ptr) + } +} + +impl Ord for DynMetadata { + #[inline] + fn cmp(&self, other: &Self) -> crate::cmp::Ordering { + (self.vtable_ptr as *const VTable).cmp(&(other.vtable_ptr as *const VTable)) + } +} + +impl PartialOrd for DynMetadata { + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Hash for DynMetadata { + #[inline] + fn hash(&self, hasher: &mut H) { + crate::ptr::hash::(self.vtable_ptr, hasher) + } +} diff --git a/crux-mir/lib/core/src/ptr/mod.rs b/crux-mir/lib/core/src/ptr/mod.rs index 4913cd73a..1ad9af154 100644 --- a/crux-mir/lib/core/src/ptr/mod.rs +++ b/crux-mir/lib/core/src/ptr/mod.rs @@ -1,6 +1,6 @@ //! Manually manage memory through raw pointers. //! -//! *[See also the pointer primitive types](../../std/primitive.pointer.html).* +//! *[See also the pointer primitive types](pointer).* //! //! # Safety //! @@ -16,12 +16,17 @@ //! provided at this point are very minimal: //! //! * A [null] pointer is *never* valid, not even for accesses of [size zero][zst]. -//! * All pointers (except for the null pointer) are valid for all operations of -//! [size zero][zst]. //! * For a pointer to be valid, it is necessary, but not always sufficient, that the pointer //! be *dereferenceable*: the memory range of the given size starting at the pointer must all be //! within the bounds of a single allocated object. Note that in Rust, //! every (stack-allocated) variable is considered a separate allocated object. +//! * Even for operations of [size zero][zst], the pointer must not be pointing to deallocated +//! memory, i.e., deallocation makes pointers invalid even for zero-sized operations. However, +//! casting any non-zero integer *literal* to a pointer is valid for zero-sized accesses, even if +//! some memory happens to exist at that address and gets deallocated. This corresponds to writing +//! your own allocator: allocating zero-sized objects is not very hard. The canonical way to +//! obtain a pointer that is valid for zero-sized accesses is [`NonNull::dangling`]. +//FIXME: mention `ptr::invalid` above, once it is stable. //! * All accesses performed by functions in this module are *non-atomic* in the sense //! of [atomic operations] used to synchronize between threads. This means it is //! undefined behavior to perform two concurrent accesses to the same location from different @@ -30,7 +35,8 @@ //! be used for inter-thread synchronization. //! * The result of casting a reference to a pointer is valid for as long as the //! underlying object is live and no reference (just raw pointers) is used to -//! access the same memory. +//! access the same memory. That is, reference and pointer accesses cannot be +//! interleaved. //! //! These axioms, along with careful use of [`offset`] for pointer arithmetic, //! are enough to correctly implement many useful things in unsafe code. Stronger guarantees @@ -51,39 +57,346 @@ //! has size 0, i.e., even if memory is not actually touched. Consider using //! [`NonNull::dangling`] in such cases. //! +//! ## Allocated object +//! +//! For several operations, such as [`offset`] or field projections (`expr.field`), the notion of an +//! "allocated object" becomes relevant. An allocated object is a contiguous region of memory. +//! Common examples of allocated objects include stack-allocated variables (each variable is a +//! separate allocated object), heap allocations (each allocation created by the global allocator is +//! a separate allocated object), and `static` variables. +//! +//! # Strict Provenance +//! +//! **The following text is non-normative, insufficiently formal, and is an extremely strict +//! interpretation of provenance. It's ok if your code doesn't strictly conform to it.** +//! +//! [Strict Provenance][] is an experimental set of APIs that help tools that try +//! to validate the memory-safety of your program's execution. Notably this includes [Miri][] +//! and [CHERI][], which can detect when you access out of bounds memory or otherwise violate +//! Rust's memory model. +//! +//! Provenance must exist in some form for any programming +//! language compiled for modern computer architectures, but specifying a model for provenance +//! in a way that is useful to both compilers and programmers is an ongoing challenge. +//! The [Strict Provenance][] experiment seeks to explore the question: *what if we just said you +//! couldn't do all the nasty operations that make provenance so messy?* +//! +//! What APIs would have to be removed? What APIs would have to be added? How much would code +//! have to change, and is it worse or better now? Would any patterns become truly inexpressible? +//! Could we carve out special exceptions for those patterns? Should we? +//! +//! A secondary goal of this project is to see if we can disambiguate the many functions of +//! pointer<->integer casts enough for the definition of `usize` to be loosened so that it +//! isn't *pointer*-sized but address-space/offset/allocation-sized (we'll probably continue +//! to conflate these notions). This would potentially make it possible to more efficiently +//! target platforms where pointers are larger than offsets, such as CHERI and maybe some +//! segmented architectures. +//! +//! ## Provenance +//! +//! **This section is *non-normative* and is part of the [Strict Provenance][] experiment.** +//! +//! Pointers are not *simply* an "integer" or "address". For instance, it's uncontroversial +//! to say that a Use After Free is clearly Undefined Behaviour, even if you "get lucky" +//! and the freed memory gets reallocated before your read/write (in fact this is the +//! worst-case scenario, UAFs would be much less concerning if this didn't happen!). +//! To rationalize this claim, pointers need to somehow be *more* than just their addresses: +//! they must have provenance. +//! +//! When an allocation is created, that allocation has a unique Original Pointer. For alloc +//! APIs this is literally the pointer the call returns, and for local variables and statics, +//! this is the name of the variable/static. This is mildly overloading the term "pointer" +//! for the sake of brevity/exposition. +//! +//! The Original Pointer for an allocation is guaranteed to have unique access to the entire +//! allocation and *only* that allocation. In this sense, an allocation can be thought of +//! as a "sandbox" that cannot be broken into or out of. *Provenance* is the permission +//! to access an allocation's sandbox and has both a *spatial* and *temporal* component: +//! +//! * Spatial: A range of bytes that the pointer is allowed to access. +//! * Temporal: The lifetime (of the allocation) that access to these bytes is tied to. +//! +//! Spatial provenance makes sure you don't go beyond your sandbox, while temporal provenance +//! makes sure that you can't "get lucky" after your permission to access some memory +//! has been revoked (either through deallocations or borrows expiring). +//! +//! Provenance is implicitly shared with all pointers transitively derived from +//! The Original Pointer through operations like [`offset`], borrowing, and pointer casts. +//! Some operations may *shrink* the derived provenance, limiting how much memory it can +//! access or how long it's valid for (i.e. borrowing a subfield and subslicing). +//! +//! Shrinking provenance cannot be undone: even if you "know" there is a larger allocation, you +//! can't derive a pointer with a larger provenance. Similarly, you cannot "recombine" +//! two contiguous provenances back into one (i.e. with a `fn merge(&[T], &[T]) -> &[T]`). +//! +//! A reference to a value always has provenance over exactly the memory that field occupies. +//! A reference to a slice always has provenance over exactly the range that slice describes. +//! +//! If an allocation is deallocated, all pointers with provenance to that allocation become +//! invalidated, and effectively lose their provenance. +//! +//! The strict provenance experiment is mostly only interested in exploring stricter *spatial* +//! provenance. In this sense it can be thought of as a subset of the more ambitious and +//! formal [Stacked Borrows][] research project, which is what tools like [Miri][] are based on. +//! In particular, Stacked Borrows is necessary to properly describe what borrows are allowed +//! to do and when they become invalidated. This necessarily involves much more complex +//! *temporal* reasoning than simply identifying allocations. Adjusting APIs and code +//! for the strict provenance experiment will also greatly help Stacked Borrows. +//! +//! +//! ## Pointer Vs Addresses +//! +//! **This section is *non-normative* and is part of the [Strict Provenance][] experiment.** +//! +//! One of the largest historical issues with trying to define provenance is that programmers +//! freely convert between pointers and integers. Once you allow for this, it generally becomes +//! impossible to accurately track and preserve provenance information, and you need to appeal +//! to very complex and unreliable heuristics. But of course, converting between pointers and +//! integers is very useful, so what can we do? +//! +//! Also did you know WASM is actually a "Harvard Architecture"? As in function pointers are +//! handled completely differently from data pointers? And we kind of just shipped Rust on WASM +//! without really addressing the fact that we let you freely convert between function pointers +//! and data pointers, because it mostly Just Works? Let's just put that on the "pointer casts +//! are dubious" pile. +//! +//! Strict Provenance attempts to square these circles by decoupling Rust's traditional conflation +//! of pointers and `usize` (and `isize`), and defining a pointer to semantically contain the +//! following information: +//! +//! * The **address-space** it is part of (e.g. "data" vs "code" in WASM). +//! * The **address** it points to, which can be represented by a `usize`. +//! * The **provenance** it has, defining the memory it has permission to access. +//! +//! Under Strict Provenance, a usize *cannot* accurately represent a pointer, and converting from +//! a pointer to a usize is generally an operation which *only* extracts the address. It is +//! therefore *impossible* to construct a valid pointer from a usize because there is no way +//! to restore the address-space and provenance. In other words, pointer-integer-pointer +//! roundtrips are not possible (in the sense that the resulting pointer is not dereferenceable). +//! +//! The key insight to making this model *at all* viable is the [`with_addr`][] method: +//! +//! ```text +//! /// Creates a new pointer with the given address. +//! /// +//! /// This performs the same operation as an `addr as ptr` cast, but copies +//! /// the *address-space* and *provenance* of `self` to the new pointer. +//! /// This allows us to dynamically preserve and propagate this important +//! /// information in a way that is otherwise impossible with a unary cast. +//! /// +//! /// This is equivalent to using `wrapping_offset` to offset `self` to the +//! /// given address, and therefore has all the same capabilities and restrictions. +//! pub fn with_addr(self, addr: usize) -> Self; +//! ``` +//! +//! So you're still able to drop down to the address representation and do whatever +//! clever bit tricks you want *as long as* you're able to keep around a pointer +//! into the allocation you care about that can "reconstitute" the other parts of the pointer. +//! Usually this is very easy, because you only are taking a pointer, messing with the address, +//! and then immediately converting back to a pointer. To make this use case more ergonomic, +//! we provide the [`map_addr`][] method. +//! +//! To help make it clear that code is "following" Strict Provenance semantics, we also provide an +//! [`addr`][] method which promises that the returned address is not part of a +//! pointer-usize-pointer roundtrip. In the future we may provide a lint for pointer<->integer +//! casts to help you audit if your code conforms to strict provenance. +//! +//! +//! ## Using Strict Provenance +//! +//! Most code needs no changes to conform to strict provenance, as the only really concerning +//! operation that *wasn't* obviously already Undefined Behaviour is casts from usize to a +//! pointer. For code which *does* cast a usize to a pointer, the scope of the change depends +//! on exactly what you're doing. +//! +//! In general you just need to make sure that if you want to convert a usize address to a +//! pointer and then use that pointer to read/write memory, you need to keep around a pointer +//! that has sufficient provenance to perform that read/write itself. In this way all of your +//! casts from an address to a pointer are essentially just applying offsets/indexing. +//! +//! This is generally trivial to do for simple cases like tagged pointers *as long as you +//! represent the tagged pointer as an actual pointer and not a usize*. For instance: +//! +//! ``` +//! #![feature(strict_provenance)] +//! +//! unsafe { +//! // A flag we want to pack into our pointer +//! static HAS_DATA: usize = 0x1; +//! static FLAG_MASK: usize = !HAS_DATA; +//! +//! // Our value, which must have enough alignment to have spare least-significant-bits. +//! let my_precious_data: u32 = 17; +//! assert!(core::mem::align_of::() > 1); +//! +//! // Create a tagged pointer +//! let ptr = &my_precious_data as *const u32; +//! let tagged = ptr.map_addr(|addr| addr | HAS_DATA); +//! +//! // Check the flag: +//! if tagged.addr() & HAS_DATA != 0 { +//! // Untag and read the pointer +//! let data = *tagged.map_addr(|addr| addr & FLAG_MASK); +//! assert_eq!(data, 17); +//! } else { +//! unreachable!() +//! } +//! } +//! ``` +//! +//! (Yes, if you've been using AtomicUsize for pointers in concurrent datastructures, you should +//! be using AtomicPtr instead. If that messes up the way you atomically manipulate pointers, +//! we would like to know why, and what needs to be done to fix it.) +//! +//! Something more complicated and just generally *evil* like an XOR-List requires more significant +//! changes like allocating all nodes in a pre-allocated Vec or Arena and using a pointer +//! to the whole allocation to reconstitute the XORed addresses. +//! +//! Situations where a valid pointer *must* be created from just an address, such as baremetal code +//! accessing a memory-mapped interface at a fixed address, are an open question on how to support. +//! These situations *will* still be allowed, but we might require some kind of "I know what I'm +//! doing" annotation to explain the situation to the compiler. It's also possible they need no +//! special attention at all, because they're generally accessing memory outside the scope of +//! "the abstract machine", or already using "I know what I'm doing" annotations like "volatile". +//! +//! Under [Strict Provenance] it is Undefined Behaviour to: +//! +//! * Access memory through a pointer that does not have provenance over that memory. +//! +//! * [`offset`] a pointer to or from an address it doesn't have provenance over. +//! This means it's always UB to offset a pointer derived from something deallocated, +//! even if the offset is 0. Note that a pointer "one past the end" of its provenance +//! is not actually outside its provenance, it just has 0 bytes it can load/store. +//! +//! But it *is* still sound to: +//! +//! * Create an invalid pointer from just an address (see [`ptr::invalid`][]). This can +//! be used for sentinel values like `null` *or* to represent a tagged pointer that will +//! never be dereferenceable. In general, it is always sound for an integer to pretend +//! to be a pointer "for fun" as long as you don't use operations on it which require +//! it to be valid (offset, read, write, etc). +//! +//! * Forge an allocation of size zero at any sufficiently aligned non-null address. +//! i.e. the usual "ZSTs are fake, do what you want" rules apply *but* this only applies +//! for actual forgery (integers cast to pointers). If you borrow some struct's field +//! that *happens* to be zero-sized, the resulting pointer will have provenance tied to +//! that allocation and it will still get invalidated if the allocation gets deallocated. +//! In the future we may introduce an API to make such a forged allocation explicit. +//! +//! * [`wrapping_offset`][] a pointer outside its provenance. This includes invalid pointers +//! which have "no" provenance. Unfortunately there may be practical limits on this for a +//! particular platform, and it's an open question as to how to specify this (if at all). +//! Notably, [CHERI][] relies on a compression scheme that can't handle a +//! pointer getting offset "too far" out of bounds. If this happens, the address +//! returned by `addr` will be the value you expect, but the provenance will get invalidated +//! and using it to read/write will fault. The details of this are architecture-specific +//! and based on alignment, but the buffer on either side of the pointer's range is pretty +//! generous (think kilobytes, not bytes). +//! +//! * Compare arbitrary pointers by address. Addresses *are* just integers and so there is +//! always a coherent answer, even if the pointers are invalid or from different +//! address-spaces/provenances. Of course, comparing addresses from different address-spaces +//! is generally going to be *meaningless*, but so is comparing Kilograms to Meters, and Rust +//! doesn't prevent that either. Similarly, if you get "lucky" and notice that a pointer +//! one-past-the-end is the "same" address as the start of an unrelated allocation, anything +//! you do with that fact is *probably* going to be gibberish. The scope of that gibberish +//! is kept under control by the fact that the two pointers *still* aren't allowed to access +//! the other's allocation (bytes), because they still have different provenance. +//! +//! * Perform pointer tagging tricks. This falls out of [`wrapping_offset`] but is worth +//! mentioning in more detail because of the limitations of [CHERI][]. Low-bit tagging +//! is very robust, and often doesn't even go out of bounds because types ensure +//! size >= align (and over-aligning actually gives CHERI more flexibility). Anything +//! more complex than this rapidly enters "extremely platform-specific" territory as +//! certain things may or may not be allowed based on specific supported operations. +//! For instance, ARM explicitly supports high-bit tagging, and so CHERI on ARM inherits +//! that and should support it. +//! +//! ## Pointer-usize-pointer roundtrips and 'exposed' provenance +//! +//! **This section is *non-normative* and is part of the [Strict Provenance] experiment.** +//! +//! As discussed above, pointer-usize-pointer roundtrips are not possible under [Strict Provenance]. +//! However, there exists legacy Rust code that is full of such roundtrips, and legacy platform APIs +//! regularly assume that `usize` can capture all the information that makes up a pointer. There +//! also might be code that cannot be ported to Strict Provenance (which is something we would [like +//! to hear about][Strict Provenance]). +//! +//! For situations like this, there is a fallback plan, a way to 'opt out' of Strict Provenance. +//! However, note that this makes your code a lot harder to specify, and the code will not work +//! (well) with tools like [Miri] and [CHERI]. +//! +//! This fallback plan is provided by the [`expose_addr`] and [`from_exposed_addr`] methods (which +//! are equivalent to `as` casts between pointers and integers). [`expose_addr`] is a lot like +//! [`addr`], but additionally adds the provenance of the pointer to a global list of 'exposed' +//! provenances. (This list is purely conceptual, it exists for the purpose of specifying Rust but +//! is not materialized in actual executions, except in tools like [Miri].) [`from_exposed_addr`] +//! can be used to construct a pointer with one of these previously 'exposed' provenances. +//! [`from_exposed_addr`] takes only `addr: usize` as arguments, so unlike in [`with_addr`] there is +//! no indication of what the correct provenance for the returned pointer is -- and that is exactly +//! what makes pointer-usize-pointer roundtrips so tricky to rigorously specify! There is no +//! algorithm that decides which provenance will be used. You can think of this as "guessing" the +//! right provenance, and the guess will be "maximally in your favor", in the sense that if there is +//! any way to avoid undefined behavior, then that is the guess that will be taken. However, if +//! there is *no* previously 'exposed' provenance that justifies the way the returned pointer will +//! be used, the program has undefined behavior. +//! +//! Using [`expose_addr`] or [`from_exposed_addr`] (or the equivalent `as` casts) means that code is +//! *not* following Strict Provenance rules. The goal of the Strict Provenance experiment is to +//! determine whether it is possible to use Rust without [`expose_addr`] and [`from_exposed_addr`]. +//! If this is successful, it would be a major win for avoiding specification complexity and to +//! facilitate adoption of tools like [CHERI] and [Miri] that can be a big help in increasing the +//! confidence in (unsafe) Rust code. +//! //! [aliasing]: ../../nomicon/aliasing.html //! [book]: ../../book/ch19-01-unsafe-rust.html#dereferencing-a-raw-pointer //! [ub]: ../../reference/behavior-considered-undefined.html -//! [null]: ./fn.null.html //! [zst]: ../../nomicon/exotic-sizes.html#zero-sized-types-zsts -//! [atomic operations]: ../../std/sync/atomic/index.html -//! [`copy`]: ../../std/ptr/fn.copy.html -//! [`offset`]: ../../std/primitive.pointer.html#method.offset -//! [`read_unaligned`]: ./fn.read_unaligned.html -//! [`write_unaligned`]: ./fn.write_unaligned.html -//! [`read_volatile`]: ./fn.read_volatile.html -//! [`write_volatile`]: ./fn.write_volatile.html -//! [`NonNull::dangling`]: ./struct.NonNull.html#method.dangling - -// ignore-tidy-undocumented-unsafe +//! [atomic operations]: crate::sync::atomic +//! [`offset`]: pointer::offset +//! [`wrapping_offset`]: pointer::wrapping_offset +//! [`with_addr`]: pointer::with_addr +//! [`map_addr`]: pointer::map_addr +//! [`addr`]: pointer::addr +//! [`ptr::invalid`]: core::ptr::invalid +//! [`expose_addr`]: pointer::expose_addr +//! [`from_exposed_addr`]: from_exposed_addr +//! [Miri]: https://github.com/rust-lang/miri +//! [CHERI]: https://www.cl.cam.ac.uk/research/security/ctsrd/cheri/ +//! [Strict Provenance]: https://github.com/rust-lang/rust/issues/95228 +//! [Stacked Borrows]: https://plv.mpi-sws.org/rustbelt/stacked-borrows/ #![stable(feature = "rust1", since = "1.0.0")] use crate::cmp::Ordering; use crate::fmt; use crate::hash; -use crate::intrinsics::{self, is_aligned_and_not_null, is_nonoverlapping}; +use crate::intrinsics::{ + self, assert_unsafe_precondition, is_aligned_and_not_null, is_nonoverlapping, +}; + use crate::mem::{self, MaybeUninit}; +mod alignment; +#[unstable(feature = "ptr_alignment_type", issue = "102070")] +pub use alignment::Alignment; + #[stable(feature = "rust1", since = "1.0.0")] +#[doc(inline)] pub use crate::intrinsics::copy_nonoverlapping; #[stable(feature = "rust1", since = "1.0.0")] +#[doc(inline)] pub use crate::intrinsics::copy; #[stable(feature = "rust1", since = "1.0.0")] +#[doc(inline)] pub use crate::intrinsics::write_bytes; +mod metadata; +#[unstable(feature = "ptr_metadata", issue = "81513")] +pub use metadata::{from_raw_parts, from_raw_parts_mut, metadata, DynMetadata, Pointee, Thin}; + mod non_null; #[stable(feature = "nonnull", since = "1.25.0")] pub use non_null::NonNull; @@ -105,15 +418,21 @@ mod mut_ptr; /// dropped normally. /// /// * It is friendlier to the optimizer to do this over [`ptr::read`] when -/// dropping manually allocated memory (e.g., when writing Box/Rc/Vec), -/// as the compiler doesn't need to prove that it's sound to elide the -/// copy. +/// dropping manually allocated memory (e.g., in the implementations of +/// `Box`/`Rc`/`Vec`), as the compiler doesn't need to prove that it's +/// sound to elide the copy. +/// +/// * It can be used to drop [pinned] data when `T` is not `repr(packed)` +/// (pinned data must not be moved before it is dropped). /// /// Unaligned values cannot be dropped in place, they must be copied to an aligned -/// location first using [`ptr::read_unaligned`]. +/// location first using [`ptr::read_unaligned`]. For packed structs, this move is +/// done automatically by the compiler. This means the fields of packed structs +/// are not dropped in-place. /// -/// [`ptr::read`]: ../ptr/fn.read.html -/// [`ptr::read_unaligned`]: ../ptr/fn.read_unaligned.html +/// [`ptr::read`]: self::read +/// [`ptr::read_unaligned`]: self::read_unaligned +/// [pinned]: crate::pin /// /// # Safety /// @@ -129,14 +448,12 @@ mod mut_ptr; /// Additionally, if `T` is not [`Copy`], using the pointed-to value after /// calling `drop_in_place` can cause undefined behavior. Note that `*to_drop = /// foo` counts as a use because it will cause the value to be dropped -/// again. [`write`] can be used to overwrite data without causing it to be +/// again. [`write()`] can be used to overwrite data without causing it to be /// dropped. /// -/// Note that even if `T` has size `0`, the pointer must be non-NULL and properly aligned. +/// Note that even if `T` has size `0`, the pointer must be non-null and properly aligned. /// -/// [valid]: ../ptr/index.html#safety -/// [`Copy`]: ../marker/trait.Copy.html -/// [`write`]: ../ptr/fn.write.html +/// [valid]: self#safety /// /// # Examples /// @@ -167,17 +484,15 @@ mod mut_ptr; /// // Ensure that the last item was dropped. /// assert!(weak.upgrade().is_none()); /// ``` -/// -/// Notice that the compiler performs this copy automatically when dropping packed structs, -/// i.e., you do not usually have to worry about such issues unless you call `drop_in_place` -/// manually. #[stable(feature = "drop_in_place", since = "1.8.0")] #[lang = "drop_in_place"] #[allow(unconditional_recursion)] pub unsafe fn drop_in_place(to_drop: *mut T) { // Code here does not matter - this is replaced by the // real drop glue by the compiler. - drop_in_place(to_drop) + + // SAFETY: see comment above + unsafe { drop_in_place(to_drop) } } /// Creates a null raw pointer. @@ -191,11 +506,14 @@ pub unsafe fn drop_in_place(to_drop: *mut T) { /// assert!(p.is_null()); /// ``` #[inline(always)] +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] #[rustc_promotable] -#[rustc_const_stable(feature = "const_ptr_null", since = "1.32.0")] -pub const fn null() -> *const T { - 0 as *const T +#[rustc_const_stable(feature = "const_ptr_null", since = "1.24.0")] +#[rustc_allow_const_fn_unstable(ptr_metadata)] +#[rustc_diagnostic_item = "ptr_null"] +pub const fn null() -> *const T { + from_raw_parts(invalid(0), ()) } /// Creates a null mutable raw pointer. @@ -209,24 +527,183 @@ pub const fn null() -> *const T { /// assert!(p.is_null()); /// ``` #[inline(always)] +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] #[rustc_promotable] -#[rustc_const_stable(feature = "const_ptr_null", since = "1.32.0")] -pub const fn null_mut() -> *mut T { - 0 as *mut T +#[rustc_const_stable(feature = "const_ptr_null", since = "1.24.0")] +#[rustc_allow_const_fn_unstable(ptr_metadata)] +#[rustc_diagnostic_item = "ptr_null_mut"] +pub const fn null_mut() -> *mut T { + from_raw_parts_mut(invalid_mut(0), ()) } -#[repr(C)] -pub(crate) union Repr { - pub(crate) rust: *const [T], - rust_mut: *mut [T], - pub(crate) raw: FatPtr, +/// Creates an invalid pointer with the given address. +/// +/// This is different from `addr as *const T`, which creates a pointer that picks up a previously +/// exposed provenance. See [`from_exposed_addr`] for more details on that operation. +/// +/// The module's top-level documentation discusses the precise meaning of an "invalid" +/// pointer but essentially this expresses that the pointer is not associated +/// with any actual allocation and is little more than a usize address in disguise. +/// +/// This pointer will have no provenance associated with it and is therefore +/// UB to read/write/offset. This mostly exists to facilitate things +/// like `ptr::null` and `NonNull::dangling` which make invalid pointers. +/// +/// (Standard "Zero-Sized-Types get to cheat and lie" caveats apply, although it +/// may be desirable to give them their own API just to make that 100% clear.) +/// +/// This API and its claimed semantics are part of the Strict Provenance experiment, +/// see the [module documentation][crate::ptr] for details. +#[inline(always)] +#[must_use] +#[rustc_const_stable(feature = "stable_things_using_strict_provenance", since = "1.61.0")] +#[unstable(feature = "strict_provenance", issue = "95228")] +pub const fn invalid(addr: usize) -> *const T { + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + // We use transmute rather than a cast so tools like Miri can tell that this + // is *not* the same as from_exposed_addr. + // SAFETY: every valid integer is also a valid pointer (as long as you don't dereference that + // pointer). + unsafe { mem::transmute(addr) } } -#[repr(C)] -pub(crate) struct FatPtr { - data: *const T, - pub(crate) len: usize, +/// Creates an invalid mutable pointer with the given address. +/// +/// This is different from `addr as *mut T`, which creates a pointer that picks up a previously +/// exposed provenance. See [`from_exposed_addr_mut`] for more details on that operation. +/// +/// The module's top-level documentation discusses the precise meaning of an "invalid" +/// pointer but essentially this expresses that the pointer is not associated +/// with any actual allocation and is little more than a usize address in disguise. +/// +/// This pointer will have no provenance associated with it and is therefore +/// UB to read/write/offset. This mostly exists to facilitate things +/// like `ptr::null` and `NonNull::dangling` which make invalid pointers. +/// +/// (Standard "Zero-Sized-Types get to cheat and lie" caveats apply, although it +/// may be desirable to give them their own API just to make that 100% clear.) +/// +/// This API and its claimed semantics are part of the Strict Provenance experiment, +/// see the [module documentation][crate::ptr] for details. +#[inline(always)] +#[must_use] +#[rustc_const_stable(feature = "stable_things_using_strict_provenance", since = "1.61.0")] +#[unstable(feature = "strict_provenance", issue = "95228")] +pub const fn invalid_mut(addr: usize) -> *mut T { + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + // We use transmute rather than a cast so tools like Miri can tell that this + // is *not* the same as from_exposed_addr. + // SAFETY: every valid integer is also a valid pointer (as long as you don't dereference that + // pointer). + unsafe { mem::transmute(addr) } +} + +/// Convert an address back to a pointer, picking up a previously 'exposed' provenance. +/// +/// This is equivalent to `addr as *const T`. The provenance of the returned pointer is that of *any* +/// pointer that was previously exposed by passing it to [`expose_addr`][pointer::expose_addr], +/// or a `ptr as usize` cast. In addition, memory which is outside the control of the Rust abstract +/// machine (MMIO registers, for example) is always considered to be exposed, so long as this memory +/// is disjoint from memory that will be used by the abstract machine such as the stack, heap, +/// and statics. +/// +/// If there is no 'exposed' provenance that justifies the way this pointer will be used, +/// the program has undefined behavior. In particular, the aliasing rules still apply: pointers +/// and references that have been invalidated due to aliasing accesses cannot be used any more, +/// even if they have been exposed! +/// +/// Note that there is no algorithm that decides which provenance will be used. You can think of this +/// as "guessing" the right provenance, and the guess will be "maximally in your favor", in the sense +/// that if there is any way to avoid undefined behavior (while upholding all aliasing requirements), +/// then that is the guess that will be taken. +/// +/// On platforms with multiple address spaces, it is your responsibility to ensure that the +/// address makes sense in the address space that this pointer will be used with. +/// +/// Using this method means that code is *not* following strict provenance rules. "Guessing" a +/// suitable provenance complicates specification and reasoning and may not be supported by +/// tools that help you to stay conformant with the Rust memory model, so it is recommended to +/// use [`with_addr`][pointer::with_addr] wherever possible. +/// +/// On most platforms this will produce a value with the same bytes as the address. Platforms +/// which need to store additional information in a pointer may not support this operation, +/// since it is generally not possible to actually *compute* which provenance the returned +/// pointer has to pick up. +/// +/// This API and its claimed semantics are part of the Strict Provenance experiment, see the +/// [module documentation][crate::ptr] for details. +#[must_use] +#[inline(always)] +#[unstable(feature = "strict_provenance", issue = "95228")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces +#[allow(fuzzy_provenance_casts)] // this *is* the strict provenance API one should use instead +pub fn from_exposed_addr(addr: usize) -> *const T +where + T: Sized, +{ + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + addr as *const T +} + +/// Convert an address back to a mutable pointer, picking up a previously 'exposed' provenance. +/// +/// This is equivalent to `addr as *mut T`. The provenance of the returned pointer is that of *any* +/// pointer that was previously passed to [`expose_addr`][pointer::expose_addr] or a `ptr as usize` +/// cast. If there is no previously 'exposed' provenance that justifies the way this pointer will be +/// used, the program has undefined behavior. Note that there is no algorithm that decides which +/// provenance will be used. You can think of this as "guessing" the right provenance, and the guess +/// will be "maximally in your favor", in the sense that if there is any way to avoid undefined +/// behavior, then that is the guess that will be taken. +/// +/// On platforms with multiple address spaces, it is your responsibility to ensure that the +/// address makes sense in the address space that this pointer will be used with. +/// +/// Using this method means that code is *not* following strict provenance rules. "Guessing" a +/// suitable provenance complicates specification and reasoning and may not be supported by +/// tools that help you to stay conformant with the Rust memory model, so it is recommended to +/// use [`with_addr`][pointer::with_addr] wherever possible. +/// +/// On most platforms this will produce a value with the same bytes as the address. Platforms +/// which need to store additional information in a pointer may not support this operation, +/// since it is generally not possible to actually *compute* which provenance the returned +/// pointer has to pick up. +/// +/// This API and its claimed semantics are part of the Strict Provenance experiment, see the +/// [module documentation][crate::ptr] for details. +#[must_use] +#[inline(always)] +#[unstable(feature = "strict_provenance", issue = "95228")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces +#[allow(fuzzy_provenance_casts)] // this *is* the strict provenance API one should use instead +pub fn from_exposed_addr_mut(addr: usize) -> *mut T +where + T: Sized, +{ + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + addr as *mut T +} + +/// Convert a reference to a raw pointer. +/// +/// This is equivalent to `r as *const T`, but is a bit safer since it will never silently change +/// type or mutability, in particular if the code is refactored. +#[inline(always)] +#[must_use] +#[unstable(feature = "ptr_from_ref", issue = "106116")] +pub fn from_ref(r: &T) -> *const T { + r +} + +/// Convert a mutable reference to a raw pointer. +/// +/// This is equivalent to `r as *mut T`, but is a bit safer since it will never silently change +/// type or mutability, in particular if the code is refactored. +#[inline(always)] +#[must_use] +#[unstable(feature = "ptr_from_ref", issue = "106116")] +pub fn from_mut(r: &mut T) -> *mut T { + r } /// Forms a raw slice from a pointer and a length. @@ -234,9 +711,9 @@ pub(crate) struct FatPtr { /// The `len` argument is the number of **elements**, not the number of bytes. /// /// This function is safe, but actually using the return value is unsafe. -/// See the documentation of [`from_raw_parts`] for slice safety requirements. +/// See the documentation of [`slice::from_raw_parts`] for slice safety requirements. /// -/// [`from_raw_parts`]: ../../std/slice/fn.from_raw_parts.html +/// [`slice::from_raw_parts`]: crate::slice::from_raw_parts /// /// # Examples /// @@ -245,15 +722,16 @@ pub(crate) struct FatPtr { /// /// // create a slice pointer when starting out with a pointer to the first element /// let x = [5, 6, 7]; -/// let ptr = x.as_ptr(); -/// let slice = ptr::slice_from_raw_parts(ptr, 3); +/// let raw_pointer = x.as_ptr(); +/// let slice = ptr::slice_from_raw_parts(raw_pointer, 3); /// assert_eq!(unsafe { &*slice }[2], 7); /// ``` #[inline] #[stable(feature = "slice_from_raw_parts", since = "1.42.0")] -#[rustc_const_unstable(feature = "const_slice_from_raw_parts", issue = "67456")] +#[rustc_const_stable(feature = "const_slice_from_raw_parts", since = "1.64.0")] +#[rustc_allow_const_fn_unstable(ptr_metadata)] pub const fn slice_from_raw_parts(data: *const T, len: usize) -> *const [T] { - unsafe { Repr { raw: FatPtr { data, len } }.rust } + from_raw_parts(data.cast(), len) } /// Performs the same functionality as [`slice_from_raw_parts`], except that a @@ -262,21 +740,36 @@ pub const fn slice_from_raw_parts(data: *const T, len: usize) -> *const [T] { /// See the documentation of [`slice_from_raw_parts`] for more details. /// /// This function is safe, but actually using the return value is unsafe. -/// See the documentation of [`from_raw_parts_mut`] for slice safety requirements. +/// See the documentation of [`slice::from_raw_parts_mut`] for slice safety requirements. +/// +/// [`slice::from_raw_parts_mut`]: crate::slice::from_raw_parts_mut +/// +/// # Examples +/// +/// ```rust +/// use std::ptr; +/// +/// let x = &mut [5, 6, 7]; +/// let raw_pointer = x.as_mut_ptr(); +/// let slice = ptr::slice_from_raw_parts_mut(raw_pointer, 3); +/// +/// unsafe { +/// (*slice)[2] = 99; // assign a value at an index in the slice +/// }; /// -/// [`slice_from_raw_parts`]: fn.slice_from_raw_parts.html -/// [`from_raw_parts_mut`]: ../../std/slice/fn.from_raw_parts_mut.html +/// assert_eq!(unsafe { &*slice }[2], 99); +/// ``` #[inline] #[stable(feature = "slice_from_raw_parts", since = "1.42.0")] -#[rustc_const_unstable(feature = "const_slice_from_raw_parts", issue = "67456")] +#[rustc_const_unstable(feature = "const_slice_from_raw_parts_mut", issue = "67456")] pub const fn slice_from_raw_parts_mut(data: *mut T, len: usize) -> *mut [T] { - unsafe { Repr { raw: FatPtr { data, len } }.rust_mut } + from_raw_parts_mut(data.cast(), len) } /// Swaps the values at two mutable locations of the same type, without /// deinitializing either. /// -/// But for the following two exceptions, this function is semantically +/// But for the following exceptions, this function is semantically /// equivalent to [`mem::swap`]: /// /// * It operates on raw pointers instead of references. When references are @@ -286,7 +779,8 @@ pub const fn slice_from_raw_parts_mut(data: *mut T, len: usize) -> *mut [T] { /// overlapping region of memory from `x` will be used. This is demonstrated /// in the second example below. /// -/// [`mem::swap`]: ../mem/fn.swap.html +/// * The operation is "untyped" in the sense that data may be uninitialized or otherwise violate +/// the requirements of `T`. The initialization state is preserved exactly. /// /// # Safety /// @@ -296,9 +790,9 @@ pub const fn slice_from_raw_parts_mut(data: *mut T, len: usize) -> *mut [T] { /// /// * Both `x` and `y` must be properly aligned. /// -/// Note that even if `T` has size `0`, the pointers must be non-NULL and properly aligned. +/// Note that even if `T` has size `0`, the pointers must be non-null and properly aligned. /// -/// [valid]: ../ptr/index.html#safety +/// [valid]: self#safety /// /// # Examples /// @@ -309,8 +803,9 @@ pub const fn slice_from_raw_parts_mut(data: *mut T, len: usize) -> *mut [T] { /// /// let mut array = [0, 1, 2, 3]; /// -/// let x = array[0..].as_mut_ptr() as *mut [u32; 2]; // this is `array[0..2]` -/// let y = array[2..].as_mut_ptr() as *mut [u32; 2]; // this is `array[2..4]` +/// let (x, y) = array.split_at_mut(2); +/// let x = x.as_mut_ptr().cast::<[u32; 2]>(); // this is `array[0..2]` +/// let y = y.as_mut_ptr().cast::<[u32; 2]>(); // this is `array[2..4]` /// /// unsafe { /// ptr::swap(x, y); @@ -323,10 +818,12 @@ pub const fn slice_from_raw_parts_mut(data: *mut T, len: usize) -> *mut [T] { /// ``` /// use std::ptr; /// -/// let mut array = [0, 1, 2, 3]; +/// let mut array: [i32; 4] = [0, 1, 2, 3]; /// -/// let x = array[0..].as_mut_ptr() as *mut [u32; 3]; // this is `array[0..3]` -/// let y = array[1..].as_mut_ptr() as *mut [u32; 3]; // this is `array[1..4]` +/// let array_ptr: *mut i32 = array.as_mut_ptr(); +/// +/// let x = array_ptr as *mut [i32; 3]; // this is `array[0..3]` +/// let y = unsafe { array_ptr.add(1) } as *mut [i32; 3]; // this is `array[1..4]` /// /// unsafe { /// ptr::swap(x, y); @@ -340,20 +837,30 @@ pub const fn slice_from_raw_parts_mut(data: *mut T, len: usize) -> *mut [T] { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] -pub unsafe fn swap(x: *mut T, y: *mut T) { +#[rustc_const_unstable(feature = "const_swap", issue = "83163")] +pub const unsafe fn swap(x: *mut T, y: *mut T) { // Give ourselves some scratch space to work with. // We do not have to worry about drops: `MaybeUninit` does nothing when dropped. let mut tmp = MaybeUninit::::uninit(); // Perform the swap - copy_nonoverlapping(x, tmp.as_mut_ptr(), 1); - copy(y, x, 1); // `x` and `y` may overlap - copy_nonoverlapping(tmp.as_ptr(), y, 1); + // SAFETY: the caller must guarantee that `x` and `y` are + // valid for writes and properly aligned. `tmp` cannot be + // overlapping either `x` or `y` because `tmp` was just allocated + // on the stack as a separate allocated object. + unsafe { + copy_nonoverlapping(x, tmp.as_mut_ptr(), 1); + copy(y, x, 1); // `x` and `y` may overlap + copy_nonoverlapping(tmp.as_ptr(), y, 1); + } } /// Swaps `count * size_of::()` bytes between the two regions of memory /// beginning at `x` and `y`. The two regions must *not* overlap. /// +/// The operation is "untyped" in the sense that data may be uninitialized or otherwise violate the +/// requirements of `T`. The initialization state is preserved exactly. +/// /// # Safety /// /// Behavior is undefined if any of the following conditions are violated: @@ -368,9 +875,9 @@ pub unsafe fn swap(x: *mut T, y: *mut T) { /// beginning at `y` with the same size. /// /// Note that even if the effectively copied size (`count * size_of::()`) is `0`, -/// the pointers must be non-NULL and properly aligned. +/// the pointers must be non-null and properly aligned. /// -/// [valid]: ../ptr/index.html#safety +/// [valid]: self#safety /// /// # Examples /// @@ -391,76 +898,74 @@ pub unsafe fn swap(x: *mut T, y: *mut T) { /// ``` #[inline] #[stable(feature = "swap_nonoverlapping", since = "1.27.0")] -pub unsafe fn swap_nonoverlapping(x: *mut T, y: *mut T, count: usize) { - debug_assert!(is_aligned_and_not_null(x), "attempt to swap unaligned or null pointer"); - debug_assert!(is_aligned_and_not_null(y), "attempt to swap unaligned or null pointer"); - debug_assert!(is_nonoverlapping(x, y, count), "attempt to swap overlapping memory"); - - let x = x as *mut u8; - let y = y as *mut u8; - let len = mem::size_of::() * count; - swap_nonoverlapping_bytes(x, y, len) -} - -#[inline] -pub(crate) unsafe fn swap_nonoverlapping_one(x: *mut T, y: *mut T) { - // For types smaller than the block optimization below, - // just swap directly to avoid pessimizing codegen. - if mem::size_of::() < 32 { - let z = read(x); - copy_nonoverlapping(y, x, 1); - write(y, z); - } else { - swap_nonoverlapping(x, y, 1); +#[rustc_const_unstable(feature = "const_swap", issue = "83163")] +pub const unsafe fn swap_nonoverlapping(x: *mut T, y: *mut T, count: usize) { + #[allow(unused)] + macro_rules! attempt_swap_as_chunks { + ($ChunkTy:ty) => { + if mem::align_of::() >= mem::align_of::<$ChunkTy>() + && mem::size_of::() % mem::size_of::<$ChunkTy>() == 0 + { + let x: *mut $ChunkTy = x.cast(); + let y: *mut $ChunkTy = y.cast(); + let count = count * (mem::size_of::() / mem::size_of::<$ChunkTy>()); + // SAFETY: these are the same bytes that the caller promised were + // ok, just typed as `MaybeUninit`s instead of as `T`s. + // The `if` condition above ensures that we're not violating + // alignment requirements, and that the division is exact so + // that we don't lose any bytes off the end. + return unsafe { swap_nonoverlapping_simple_untyped(x, y, count) }; + } + }; } -} -#[inline] -unsafe fn swap_nonoverlapping_bytes(x: *mut u8, y: *mut u8, len: usize) { - // The approach here is to utilize simd to swap x & y efficiently. Testing reveals - // that swapping either 32 bytes or 64 bytes at a time is most efficient for Intel - // Haswell E processors. LLVM is more able to optimize if we give a struct a - // #[repr(simd)], even if we don't actually use this struct directly. - // - // FIXME repr(simd) broken on emscripten and redox - #[cfg_attr(not(any(target_os = "emscripten", target_os = "redox")), repr(simd))] - struct Block(u64, u64, u64, u64); - struct UnalignedBlock(u64, u64, u64, u64); - - let block_size = mem::size_of::(); + // SAFETY: the caller must guarantee that `x` and `y` are + // valid for writes and properly aligned. + unsafe { + assert_unsafe_precondition!( + "ptr::swap_nonoverlapping requires that both pointer arguments are aligned and non-null \ + and the specified memory ranges do not overlap", + [T](x: *mut T, y: *mut T, count: usize) => + is_aligned_and_not_null(x) + && is_aligned_and_not_null(y) + && is_nonoverlapping(x, y, count) + ); + } - // Loop through x & y, copying them `Block` at a time - // The optimizer should unroll the loop fully for most types - // N.B. We can't use a for loop as the `range` impl calls `mem::swap` recursively - let mut i = 0; - while i + block_size <= len { - // Create some uninitialized memory as scratch space - // Declaring `t` here avoids aligning the stack when this loop is unused - let mut t = mem::MaybeUninit::::uninit(); - let t = t.as_mut_ptr() as *mut u8; - let x = x.add(i); - let y = y.add(i); - - // Swap a block of bytes of x & y, using t as a temporary buffer - // This should be optimized into efficient SIMD operations where available - copy_nonoverlapping(x, t, block_size); - copy_nonoverlapping(y, x, block_size); - copy_nonoverlapping(t, y, block_size); - i += block_size; + // Split up the slice into small power-of-two-sized chunks that LLVM is able + // to vectorize (unless it's a special type with more-than-pointer alignment, + // because we don't want to pessimize things like slices of SIMD vectors.) + if mem::align_of::() <= mem::size_of::() + && (!mem::size_of::().is_power_of_two() + || mem::size_of::() > mem::size_of::() * 2) + { + attempt_swap_as_chunks!(usize); + attempt_swap_as_chunks!(u8); } - if i < len { - // Swap any remaining bytes - let mut t = mem::MaybeUninit::::uninit(); - let rem = len - i; + // SAFETY: Same preconditions as this function + unsafe { swap_nonoverlapping_simple_untyped(x, y, count) } +} - let t = t.as_mut_ptr() as *mut u8; - let x = x.add(i); - let y = y.add(i); +/// Same behaviour and safety conditions as [`swap_nonoverlapping`] +/// +/// LLVM can vectorize this (at least it can for the power-of-two-sized types +/// `swap_nonoverlapping` tries to use) so no need to manually SIMD it. +#[inline] +#[rustc_const_unstable(feature = "const_swap", issue = "83163")] +const unsafe fn swap_nonoverlapping_simple_untyped(x: *mut T, y: *mut T, count: usize) { + let x = x.cast::>(); + let y = y.cast::>(); + let mut i = 0; + while i < count { + // SAFETY: By precondition, `i` is in-bounds because it's below `n` + let x = unsafe { &mut *x.add(i) }; + // SAFETY: By precondition, `i` is in-bounds because it's below `n` + // and it's distinct from `x` since the ranges are non-overlapping + let y = unsafe { &mut *y.add(i) }; + mem::swap_simple::>(x, y); - copy_nonoverlapping(x, t, rem); - copy_nonoverlapping(y, x, rem); - copy_nonoverlapping(t, y, rem); + i += 1; } } @@ -472,8 +977,6 @@ unsafe fn swap_nonoverlapping_bytes(x: *mut u8, y: *mut u8, len: usize) { /// operates on raw pointers instead of references. When references are /// available, [`mem::replace`] should be preferred. /// -/// [`mem::replace`]: ../mem/fn.replace.html -/// /// # Safety /// /// Behavior is undefined if any of the following conditions are violated: @@ -484,9 +987,9 @@ unsafe fn swap_nonoverlapping_bytes(x: *mut u8, y: *mut u8, len: usize) { /// /// * `dst` must point to a properly initialized value of type `T`. /// -/// Note that even if `T` has size `0`, the pointer must be non-NULL and properly aligned. +/// Note that even if `T` has size `0`, the pointer must be non-null and properly aligned. /// -/// [valid]: ../ptr/index.html#safety +/// [valid]: self#safety /// /// # Examples /// @@ -506,8 +1009,19 @@ unsafe fn swap_nonoverlapping_bytes(x: *mut u8, y: *mut u8, len: usize) { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] -pub unsafe fn replace(dst: *mut T, mut src: T) -> T { - mem::swap(&mut *dst, &mut src); // cannot overlap +#[rustc_const_unstable(feature = "const_replace", issue = "83164")] +pub const unsafe fn replace(dst: *mut T, mut src: T) -> T { + // SAFETY: the caller must guarantee that `dst` is valid to be + // cast to a mutable reference (valid for writes, aligned, initialized), + // and cannot overlap `src` since `dst` must point to a distinct + // allocated object. + unsafe { + assert_unsafe_precondition!( + "ptr::replace requires that the pointer argument is aligned and non-null", + [T](dst: *mut T) => is_aligned_and_not_null(dst) + ); + mem::swap(&mut *dst, &mut src); // cannot overlap + } src } @@ -525,7 +1039,7 @@ pub unsafe fn replace(dst: *mut T, mut src: T) -> T { /// /// * `src` must point to a properly initialized value of type `T`. /// -/// Note that even if `T` has size `0`, the pointer must be non-NULL and properly aligned. +/// Note that even if `T` has size `0`, the pointer must be non-null and properly aligned. /// /// # Examples /// @@ -586,7 +1100,7 @@ pub unsafe fn replace(dst: *mut T, mut src: T) -> T { /// `*src` can violate memory safety. Note that assigning to `*src` counts as a /// use because it will attempt to drop the value at `*src`. /// -/// [`write`] can be used to overwrite data without causing it to be dropped. +/// [`write()`] can be used to overwrite data without causing it to be dropped. /// /// ``` /// use std::ptr; @@ -615,18 +1129,34 @@ pub unsafe fn replace(dst: *mut T, mut src: T) -> T { /// assert_eq!(s, "bar"); /// ``` /// -/// [`mem::swap`]: ../mem/fn.swap.html -/// [valid]: ../ptr/index.html#safety -/// [`Copy`]: ../marker/trait.Copy.html -/// [`read_unaligned`]: ./fn.read_unaligned.html -/// [`write`]: ./fn.write.html +/// [valid]: self#safety #[inline] #[stable(feature = "rust1", since = "1.0.0")] -pub unsafe fn read(src: *const T) -> T { - // `copy_nonoverlapping` takes care of debug_assert. +#[rustc_const_unstable(feature = "const_ptr_read", issue = "80377")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces +pub const unsafe fn read(src: *const T) -> T { + // We are calling the intrinsics directly to avoid function calls in the generated code + // as `intrinsics::copy_nonoverlapping` is a wrapper function. + extern "rust-intrinsic" { + #[rustc_const_stable(feature = "const_intrinsic_copy", since = "1.63.0")] + fn copy_nonoverlapping(src: *const T, dst: *mut T, count: usize); + } + let mut tmp = MaybeUninit::::uninit(); - copy_nonoverlapping(src, tmp.as_mut_ptr(), 1); - tmp.assume_init() + // SAFETY: the caller must guarantee that `src` is valid for reads. + // `src` cannot overlap `tmp` because `tmp` was just allocated on + // the stack as a separate allocated object. + // + // Also, since we just wrote a valid value into `tmp`, it is guaranteed + // to be properly initialized. + unsafe { + assert_unsafe_precondition!( + "ptr::read requires that the pointer argument is aligned and non-null", + [T](src: *const T) => is_aligned_and_not_null(src) + ); + copy_nonoverlapping(src, tmp.as_mut_ptr(), 1); + tmp.assume_init() + } } /// Reads the value from `src` without moving it. This leaves the @@ -646,19 +1176,13 @@ pub unsafe fn read(src: *const T) -> T { /// whether `T` is [`Copy`]. If `T` is not [`Copy`], using both the returned /// value and the value at `*src` can [violate memory safety][read-ownership]. /// -/// Note that even if `T` has size `0`, the pointer must be non-NULL. +/// Note that even if `T` has size `0`, the pointer must be non-null. /// -/// [`Copy`]: ../marker/trait.Copy.html -/// [`read`]: ./fn.read.html -/// [`write_unaligned`]: ./fn.write_unaligned.html -/// [read-ownership]: ./fn.read.html#ownership-of-the-returned-value -/// [valid]: ../ptr/index.html#safety +/// [read-ownership]: read#ownership-of-the-returned-value +/// [valid]: self#safety /// /// ## On `packed` structs /// -/// It is currently impossible to create raw pointers to unaligned fields -/// of a packed struct. -/// /// Attempting to create a raw pointer to an `unaligned` struct field with /// an expression such as `&packed.unaligned as *const FieldType` creates an /// intermediate unaligned reference before converting that to a raw pointer. @@ -667,9 +1191,13 @@ pub unsafe fn read(src: *const T) -> T { /// As a result, using `&packed.unaligned as *const FieldType` causes immediate /// *undefined behavior* in your program. /// +/// Instead you must use the [`ptr::addr_of!`](addr_of) macro to +/// create the pointer. You may use that returned pointer together with this +/// function. +/// /// An example of what not to do and how this relates to `read_unaligned` is: /// -/// ```no_run +/// ``` /// #[repr(packed, C)] /// struct Packed { /// _padding: u8, @@ -681,27 +1209,19 @@ pub unsafe fn read(src: *const T) -> T { /// unaligned: 0x01020304, /// }; /// -/// let v = unsafe { -/// // Here we attempt to take the address of a 32-bit integer which is not aligned. -/// let unaligned = -/// // A temporary unaligned reference is created here which results in -/// // undefined behavior regardless of whether the reference is used or not. -/// &packed.unaligned -/// // Casting to a raw pointer doesn't help; the mistake already happened. -/// as *const u32; +/// // Take the address of a 32-bit integer which is not aligned. +/// // In contrast to `&packed.unaligned as *const _`, this has no undefined behavior. +/// let unaligned = std::ptr::addr_of!(packed.unaligned); /// -/// let v = std::ptr::read_unaligned(unaligned); -/// -/// v -/// }; +/// let v = unsafe { std::ptr::read_unaligned(unaligned) }; +/// assert_eq!(v, 0x01020304); /// ``` /// /// Accessing unaligned fields directly with e.g. `packed.unaligned` is safe however. -// FIXME: Update docs based on outcome of RFC #2582 and friends. /// /// # Examples /// -/// Read an usize value from a byte buffer: +/// Read a usize value from a byte buffer: /// /// ``` /// use std::mem; @@ -716,11 +1236,20 @@ pub unsafe fn read(src: *const T) -> T { /// ``` #[inline] #[stable(feature = "ptr_unaligned", since = "1.17.0")] -pub unsafe fn read_unaligned(src: *const T) -> T { - // `copy_nonoverlapping` takes care of debug_assert. +#[rustc_const_unstable(feature = "const_ptr_read", issue = "80377")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces +pub const unsafe fn read_unaligned(src: *const T) -> T { let mut tmp = MaybeUninit::::uninit(); - copy_nonoverlapping(src as *const u8, tmp.as_mut_ptr() as *mut u8, mem::size_of::()); - tmp.assume_init() + // SAFETY: the caller must guarantee that `src` is valid for reads. + // `src` cannot overlap `tmp` because `tmp` was just allocated on + // the stack as a separate allocated object. + // + // Also, since we just wrote a valid value into `tmp`, it is guaranteed + // to be properly initialized. + unsafe { + copy_nonoverlapping(src as *const u8, tmp.as_mut_ptr() as *mut u8, mem::size_of::()); + tmp.assume_init() + } } /// Overwrites a memory location with the given value without reading or @@ -736,8 +1265,6 @@ pub unsafe fn read_unaligned(src: *const T) -> T { /// This is appropriate for initializing uninitialized memory, or overwriting /// memory that has previously been [`read`] from. /// -/// [`read`]: ./fn.read.html -/// /// # Safety /// /// Behavior is undefined if any of the following conditions are violated: @@ -747,10 +1274,9 @@ pub unsafe fn read_unaligned(src: *const T) -> T { /// * `dst` must be properly aligned. Use [`write_unaligned`] if this is not the /// case. /// -/// Note that even if `T` has size `0`, the pointer must be non-NULL and properly aligned. +/// Note that even if `T` has size `0`, the pointer must be non-null and properly aligned. /// -/// [valid]: ../ptr/index.html#safety -/// [`write_unaligned`]: ./fn.write_unaligned.html +/// [valid]: self#safety /// /// # Examples /// @@ -805,19 +1331,35 @@ pub unsafe fn read_unaligned(src: *const T) -> T { /// assert_eq!(foo, "bar"); /// assert_eq!(bar, "foo"); /// ``` -/// -/// [`mem::swap`]: ../mem/fn.swap.html #[inline] #[stable(feature = "rust1", since = "1.0.0")] -pub unsafe fn write(dst: *mut T, src: T) { - debug_assert!(is_aligned_and_not_null(dst), "attempt to write to unaligned or null pointer"); - intrinsics::move_val_init(&mut *dst, src) +#[rustc_const_unstable(feature = "const_ptr_write", issue = "86302")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces +pub const unsafe fn write(dst: *mut T, src: T) { + // We are calling the intrinsics directly to avoid function calls in the generated code + // as `intrinsics::copy_nonoverlapping` is a wrapper function. + extern "rust-intrinsic" { + #[rustc_const_stable(feature = "const_intrinsic_copy", since = "1.63.0")] + fn copy_nonoverlapping(src: *const T, dst: *mut T, count: usize); + } + + // SAFETY: the caller must guarantee that `dst` is valid for writes. + // `dst` cannot overlap `src` because the caller has mutable access + // to `dst` while `src` is owned by this function. + unsafe { + assert_unsafe_precondition!( + "ptr::write requires that the pointer argument is aligned and non-null", + [T](dst: *mut T) => is_aligned_and_not_null(dst) + ); + copy_nonoverlapping(&src as *const T, dst, 1); + intrinsics::forget(src); + } } /// Overwrites a memory location with the given value without reading or /// dropping the old value. /// -/// Unlike [`write`], the pointer may be unaligned. +/// Unlike [`write()`], the pointer may be unaligned. /// /// `write_unaligned` does not drop the contents of `dst`. This is safe, but it /// could leak allocations or resources, so care should be taken not to overwrite @@ -829,24 +1371,18 @@ pub unsafe fn write(dst: *mut T, src: T) { /// This is appropriate for initializing uninitialized memory, or overwriting /// memory that has previously been read with [`read_unaligned`]. /// -/// [`write`]: ./fn.write.html -/// [`read_unaligned`]: ./fn.read_unaligned.html -/// /// # Safety /// /// Behavior is undefined if any of the following conditions are violated: /// /// * `dst` must be [valid] for writes. /// -/// Note that even if `T` has size `0`, the pointer must be non-NULL. +/// Note that even if `T` has size `0`, the pointer must be non-null. /// -/// [valid]: ../ptr/index.html#safety +/// [valid]: self#safety /// /// ## On `packed` structs /// -/// It is currently impossible to create raw pointers to unaligned fields -/// of a packed struct. -/// /// Attempting to create a raw pointer to an `unaligned` struct field with /// an expression such as `&packed.unaligned as *const FieldType` creates an /// intermediate unaligned reference before converting that to a raw pointer. @@ -855,39 +1391,36 @@ pub unsafe fn write(dst: *mut T, src: T) { /// As a result, using `&packed.unaligned as *const FieldType` causes immediate /// *undefined behavior* in your program. /// -/// An example of what not to do and how this relates to `write_unaligned` is: +/// Instead you must use the [`ptr::addr_of_mut!`](addr_of_mut) +/// macro to create the pointer. You may use that returned pointer together with +/// this function. +/// +/// An example of how to do it and how this relates to `write_unaligned` is: /// -/// ```no_run +/// ``` /// #[repr(packed, C)] /// struct Packed { /// _padding: u8, /// unaligned: u32, /// } /// -/// let v = 0x01020304; /// let mut packed: Packed = unsafe { std::mem::zeroed() }; /// -/// let v = unsafe { -/// // Here we attempt to take the address of a 32-bit integer which is not aligned. -/// let unaligned = -/// // A temporary unaligned reference is created here which results in -/// // undefined behavior regardless of whether the reference is used or not. -/// &mut packed.unaligned -/// // Casting to a raw pointer doesn't help; the mistake already happened. -/// as *mut u32; +/// // Take the address of a 32-bit integer which is not aligned. +/// // In contrast to `&packed.unaligned as *mut _`, this has no undefined behavior. +/// let unaligned = std::ptr::addr_of_mut!(packed.unaligned); /// -/// std::ptr::write_unaligned(unaligned, v); +/// unsafe { std::ptr::write_unaligned(unaligned, 42) }; /// -/// v -/// }; +/// assert_eq!({packed.unaligned}, 42); // `{...}` forces copying the field instead of creating a reference. /// ``` /// -/// Accessing unaligned fields directly with e.g. `packed.unaligned` is safe however. -// FIXME: Update docs based on outcome of RFC #2582 and friends. +/// Accessing unaligned fields directly with e.g. `packed.unaligned` is safe however +/// (as can be seen in the `assert_eq!` above). /// /// # Examples /// -/// Write an usize value to a byte buffer: +/// Write a usize value to a byte buffer: /// /// ``` /// use std::mem; @@ -902,10 +1435,17 @@ pub unsafe fn write(dst: *mut T, src: T) { /// ``` #[inline] #[stable(feature = "ptr_unaligned", since = "1.17.0")] -pub unsafe fn write_unaligned(dst: *mut T, src: T) { - // `copy_nonoverlapping` takes care of debug_assert. - copy_nonoverlapping(&src as *const T as *const u8, dst as *mut u8, mem::size_of::()); - mem::forget(src); +#[rustc_const_unstable(feature = "const_ptr_write", issue = "86302")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces +pub const unsafe fn write_unaligned(dst: *mut T, src: T) { + // SAFETY: the caller must guarantee that `dst` is valid for writes. + // `dst` cannot overlap `src` because the caller has mutable access + // to `dst` while `src` is owned by this function. + unsafe { + copy_nonoverlapping(&src as *const T as *const u8, dst as *mut u8, mem::size_of::()); + // We are calling the intrinsic directly to avoid function calls in the generated code. + intrinsics::forget(src); + } } /// Performs a volatile read of the value from `src` without moving it. This @@ -915,8 +1455,6 @@ pub unsafe fn write_unaligned(dst: *mut T, src: T) { /// to not be elided or reordered by the compiler across other volatile /// operations. /// -/// [`write_volatile`]: ./fn.write_volatile.html -/// /// # Notes /// /// Rust does not currently have a rigorously and formally defined memory model, @@ -947,12 +1485,10 @@ pub unsafe fn write_unaligned(dst: *mut T, src: T) { /// However, storing non-[`Copy`] types in volatile memory is almost certainly /// incorrect. /// -/// Note that even if `T` has size `0`, the pointer must be non-NULL and properly aligned. +/// Note that even if `T` has size `0`, the pointer must be non-null and properly aligned. /// -/// [valid]: ../ptr/index.html#safety -/// [`Copy`]: ../marker/trait.Copy.html -/// [`read`]: ./fn.read.html -/// [read-ownership]: ./fn.read.html#ownership-of-the-returned-value +/// [valid]: self#safety +/// [read-ownership]: read#ownership-of-the-returned-value /// /// Just like in C, whether an operation is volatile has no bearing whatsoever /// on questions involving concurrent access from multiple threads. Volatile @@ -974,9 +1510,16 @@ pub unsafe fn write_unaligned(dst: *mut T, src: T) { /// ``` #[inline] #[stable(feature = "volatile", since = "1.9.0")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub unsafe fn read_volatile(src: *const T) -> T { - debug_assert!(is_aligned_and_not_null(src), "attempt to read from unaligned or null pointer"); - intrinsics::volatile_load(src) + // SAFETY: the caller must uphold the safety contract for `volatile_load`. + unsafe { + assert_unsafe_precondition!( + "ptr::read_volatile requires that the pointer argument is aligned and non-null", + [T](src: *const T) => is_aligned_and_not_null(src) + ); + intrinsics::volatile_load(src) + } } /// Performs a volatile write of a memory location with the given value without @@ -993,8 +1536,6 @@ pub unsafe fn read_volatile(src: *const T) -> T { /// Additionally, it does not drop `src`. Semantically, `src` is moved into the /// location pointed to by `dst`. /// -/// [`read_volatile`]: ./fn.read_volatile.html -/// /// # Notes /// /// Rust does not currently have a rigorously and formally defined memory model, @@ -1017,9 +1558,9 @@ pub unsafe fn read_volatile(src: *const T) -> T { /// /// * `dst` must be properly aligned. /// -/// Note that even if `T` has size `0`, the pointer must be non-NULL and properly aligned. +/// Note that even if `T` has size `0`, the pointer must be non-null and properly aligned. /// -/// [valid]: ../ptr/index.html#safety +/// [valid]: self#safety /// /// Just like in C, whether an operation is volatile has no bearing whatsoever /// on questions involving concurrent access from multiple threads. Volatile @@ -1043,17 +1584,28 @@ pub unsafe fn read_volatile(src: *const T) -> T { /// ``` #[inline] #[stable(feature = "volatile", since = "1.9.0")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub unsafe fn write_volatile(dst: *mut T, src: T) { - debug_assert!(is_aligned_and_not_null(dst), "attempt to write to unaligned or null pointer"); - intrinsics::volatile_store(dst, src); + // SAFETY: the caller must uphold the safety contract for `volatile_store`. + unsafe { + assert_unsafe_precondition!( + "ptr::write_volatile requires that the pointer argument is aligned and non-null", + [T](dst: *mut T) => is_aligned_and_not_null(dst) + ); + intrinsics::volatile_store(dst, src); + } } /// Align pointer `p`. /// -/// Calculate offset (in terms of elements of `stride` stride) that has to be applied +/// Calculate offset (in terms of elements of `size_of::()` stride) that has to be applied /// to pointer `p` so that pointer `p` would get aligned to `a`. /// -/// Note: This implementation has been carefully tailored to not panic. It is UB for this to panic. +/// # Safety +/// `a` must be a power of two. +/// +/// # Notes +/// This implementation has been carefully tailored to not panic. It is UB for this to panic. /// The only real change that can be made here is change of `INV_TABLE_MOD_16` and associated /// constants. /// @@ -1063,17 +1615,24 @@ pub unsafe fn write_volatile(dst: *mut T, src: T) { /// /// Any questions go to @nagisa. #[lang = "align_offset"] -pub(crate) unsafe fn align_offset(p: *const T, a: usize) -> usize { +pub(crate) const unsafe fn align_offset(p: *const T, a: usize) -> usize { + // FIXME(#75598): Direct use of these intrinsics improves codegen significantly at opt-level <= + // 1, where the method versions of these operations are not inlined. + use intrinsics::{ + cttz_nonzero, exact_div, mul_with_overflow, unchecked_rem, unchecked_shl, unchecked_shr, + unchecked_sub, wrapping_add, wrapping_mul, wrapping_sub, + }; + /// Calculate multiplicative modular inverse of `x` modulo `m`. /// - /// This implementation is tailored for align_offset and has following preconditions: + /// This implementation is tailored for `align_offset` and has following preconditions: /// /// * `m` is a power-of-two; /// * `x < m`; (if `x ≥ m`, pass in `x % m` instead) /// /// Implementation of this function shall not panic. Ever. #[inline] - fn mod_inv(x: usize, m: usize) -> usize { + const unsafe fn mod_inv(x: usize, m: usize) -> usize { /// Multiplicative modular inverse table modulo 2⁴ = 16. /// /// Note, that this table does not contain values where inverse does not exist (i.e., for @@ -1081,61 +1640,103 @@ pub(crate) unsafe fn align_offset(p: *const T, a: usize) -> usize { const INV_TABLE_MOD_16: [u8; 8] = [1, 11, 13, 7, 9, 3, 5, 15]; /// Modulo for which the `INV_TABLE_MOD_16` is intended. const INV_TABLE_MOD: usize = 16; - /// INV_TABLE_MOD² - const INV_TABLE_MOD_SQUARED: usize = INV_TABLE_MOD * INV_TABLE_MOD; - let table_inverse = INV_TABLE_MOD_16[(x & (INV_TABLE_MOD - 1)) >> 1] as usize; - if m <= INV_TABLE_MOD { - table_inverse & (m - 1) - } else { - // We iterate "up" using the following formula: - // - // $$ xy ≡ 1 (mod 2ⁿ) → xy (2 - xy) ≡ 1 (mod 2²ⁿ) $$ + // SAFETY: `m` is required to be a power-of-two, hence non-zero. + let m_minus_one = unsafe { unchecked_sub(m, 1) }; + let mut inverse = INV_TABLE_MOD_16[(x & (INV_TABLE_MOD - 1)) >> 1] as usize; + let mut mod_gate = INV_TABLE_MOD; + // We iterate "up" using the following formula: + // + // $$ xy ≡ 1 (mod 2ⁿ) → xy (2 - xy) ≡ 1 (mod 2²ⁿ) $$ + // + // This application needs to be applied at least until `2²ⁿ ≥ m`, at which point we can + // finally reduce the computation to our desired `m` by taking `inverse mod m`. + // + // This computation is `O(log log m)`, which is to say, that on 64-bit machines this loop + // will always finish in at most 4 iterations. + loop { + // y = y * (2 - xy) mod n // - // until 2²ⁿ ≥ m. Then we can reduce to our desired `m` by taking the result `mod m`. - let mut inverse = table_inverse; - let mut going_mod = INV_TABLE_MOD_SQUARED; - loop { - // y = y * (2 - xy) mod n - // - // Note, that we use wrapping operations here intentionally – the original formula - // uses e.g., subtraction `mod n`. It is entirely fine to do them `mod - // usize::max_value()` instead, because we take the result `mod n` at the end - // anyway. - inverse = inverse.wrapping_mul(2usize.wrapping_sub(x.wrapping_mul(inverse))); - if going_mod >= m { - return inverse & (m - 1); - } - going_mod = going_mod.wrapping_mul(going_mod); + // Note, that we use wrapping operations here intentionally – the original formula + // uses e.g., subtraction `mod n`. It is entirely fine to do them `mod + // usize::MAX` instead, because we take the result `mod n` at the end + // anyway. + if mod_gate >= m { + break; + } + inverse = wrapping_mul(inverse, wrapping_sub(2usize, wrapping_mul(x, inverse))); + let (new_gate, overflow) = mul_with_overflow(mod_gate, mod_gate); + if overflow { + break; } + mod_gate = new_gate; } + inverse & m_minus_one } let stride = mem::size_of::(); - let a_minus_one = a.wrapping_sub(1); - let pmoda = p as usize & a_minus_one; - if pmoda == 0 { - // Already aligned. Yay! - return 0; + // SAFETY: This is just an inlined `p.addr()` (which is not + // a `const fn` so we cannot call it). + // During const eval, we hook this function to ensure that the pointer never + // has provenance, making this sound. + let addr: usize = unsafe { mem::transmute(p) }; + + // SAFETY: `a` is a power-of-two, therefore non-zero. + let a_minus_one = unsafe { unchecked_sub(a, 1) }; + + if stride == 0 { + // SPECIAL_CASE: handle 0-sized types. No matter how many times we step, the address will + // stay the same, so no offset will be able to align the pointer unless it is already + // aligned. This branch _will_ be optimized out as `stride` is known at compile-time. + let p_mod_a = addr & a_minus_one; + return if p_mod_a == 0 { 0 } else { usize::MAX }; } - if stride <= 1 { - return if stride == 0 { - // If the pointer is not aligned, and the element is zero-sized, then no amount of - // elements will ever align the pointer. - !0 + // SAFETY: `stride == 0` case has been handled by the special case above. + let a_mod_stride = unsafe { unchecked_rem(a, stride) }; + if a_mod_stride == 0 { + // SPECIAL_CASE: In cases where the `a` is divisible by `stride`, byte offset to align a + // pointer can be computed more simply through `-p (mod a)`. In the off-chance the byte + // offset is not a multiple of `stride`, the input pointer was misaligned and no pointer + // offset will be able to produce a `p` aligned to the specified `a`. + // + // The naive `-p (mod a)` equation inhibits LLVM's ability to select instructions + // like `lea`. We compute `(round_up_to_next_alignment(p, a) - p)` instead. This + // redistributes operations around the load-bearing, but pessimizing `and` instruction + // sufficiently for LLVM to be able to utilize the various optimizations it knows about. + // + // LLVM handles the branch here particularly nicely. If this branch needs to be evaluated + // at runtime, it will produce a mask `if addr_mod_stride == 0 { 0 } else { usize::MAX }` + // in a branch-free way and then bitwise-OR it with whatever result the `-p mod a` + // computation produces. + + // SAFETY: `stride == 0` case has been handled by the special case above. + let addr_mod_stride = unsafe { unchecked_rem(addr, stride) }; + + return if addr_mod_stride == 0 { + let aligned_address = wrapping_add(addr, a_minus_one) & wrapping_sub(0, a); + let byte_offset = wrapping_sub(aligned_address, addr); + // SAFETY: `stride` is non-zero. This is guaranteed to divide exactly as well, because + // addr has been verified to be aligned to the original type’s alignment requirements. + unsafe { exact_div(byte_offset, stride) } } else { - a.wrapping_sub(pmoda) + usize::MAX }; } - let smoda = stride & a_minus_one; - // a is power-of-two so cannot be 0. stride = 0 is handled above. - let gcdpow = intrinsics::cttz_nonzero(stride).min(intrinsics::cttz_nonzero(a)); - let gcd = 1usize << gcdpow; + // GENERAL_CASE: From here on we’re handling the very general case where `addr` may be + // misaligned, there isn’t an obvious relationship between `stride` and `a` that we can take an + // advantage of, etc. This case produces machine code that isn’t particularly high quality, + // compared to the special cases above. The code produced here is still within the realm of + // miracles, given the situations this case has to deal with. - if p as usize & (gcd.wrapping_sub(1)) == 0 { + // SAFETY: a is power-of-two hence non-zero. stride == 0 case is handled above. + let gcdpow = unsafe { cttz_nonzero(stride).min(cttz_nonzero(a)) }; + // SAFETY: gcdpow has an upper-bound that’s at most the number of bits in a usize. + let gcd = unsafe { unchecked_shl(1usize, gcdpow) }; + // SAFETY: gcd is always greater or equal to 1. + if addr & unsafe { unchecked_sub(gcd, 1) } == 0 { // This branch solves for the following linear congruence equation: // // ` p + so = 0 mod a ` @@ -1143,29 +1744,40 @@ pub(crate) unsafe fn align_offset(p: *const T, a: usize) -> usize { // `p` here is the pointer value, `s` - stride of `T`, `o` offset in `T`s, and `a` - the // requested alignment. // - // With `g = gcd(a, s)`, and the above asserting that `p` is also divisible by `g`, we can - // denote `a' = a/g`, `s' = s/g`, `p' = p/g`, then this becomes equivalent to: + // With `g = gcd(a, s)`, and the above condition asserting that `p` is also divisible by + // `g`, we can denote `a' = a/g`, `s' = s/g`, `p' = p/g`, then this becomes equivalent to: // // ` p' + s'o = 0 mod a' ` // ` o = (a' - (p' mod a')) * (s'^-1 mod a') ` // - // The first term is "the relative alignment of `p` to `a`" (divided by the `g`), the second - // term is "how does incrementing `p` by `s` bytes change the relative alignment of `p`" (again - // divided by `g`). - // Division by `g` is necessary to make the inverse well formed if `a` and `s` are not - // co-prime. + // The first term is "the relative alignment of `p` to `a`" (divided by the `g`), the + // second term is "how does incrementing `p` by `s` bytes change the relative alignment of + // `p`" (again divided by `g`). Division by `g` is necessary to make the inverse well + // formed if `a` and `s` are not co-prime. // // Furthermore, the result produced by this solution is not "minimal", so it is necessary - // to take the result `o mod lcm(s, a)`. We can replace `lcm(s, a)` with just a `a'`. - let a2 = a >> gcdpow; - let a2minus1 = a2.wrapping_sub(1); - let s2 = smoda >> gcdpow; - let minusp2 = a2.wrapping_sub(pmoda >> gcdpow); - return (minusp2.wrapping_mul(mod_inv(s2, a2))) & a2minus1; + // to take the result `o mod lcm(s, a)`. This `lcm(s, a)` is the same as `a'`. + + // SAFETY: `gcdpow` has an upper-bound not greater than the number of trailing 0-bits in + // `a`. + let a2 = unsafe { unchecked_shr(a, gcdpow) }; + // SAFETY: `a2` is non-zero. Shifting `a` by `gcdpow` cannot shift out any of the set bits + // in `a` (of which it has exactly one). + let a2minus1 = unsafe { unchecked_sub(a2, 1) }; + // SAFETY: `gcdpow` has an upper-bound not greater than the number of trailing 0-bits in + // `a`. + let s2 = unsafe { unchecked_shr(stride & a_minus_one, gcdpow) }; + // SAFETY: `gcdpow` has an upper-bound not greater than the number of trailing 0-bits in + // `a`. Furthermore, the subtraction cannot overflow, because `a2 = a >> gcdpow` will + // always be strictly greater than `(p % a) >> gcdpow`. + let minusp2 = unsafe { unchecked_sub(a2, unchecked_shr(addr & a_minus_one, gcdpow)) }; + // SAFETY: `a2` is a power-of-two, as proven above. `s2` is strictly less than `a2` + // because `(s % a) >> gcdpow` is strictly less than `a >> gcdpow`. + return wrapping_mul(minusp2, unsafe { mod_inv(s2, a2) }) & a2minus1; } // Cannot be aligned at all. - usize::max_value() + usize::MAX } /// Compares raw pointers for equality. @@ -1178,6 +1790,12 @@ pub(crate) unsafe fn align_offset(p: *const T, a: usize) -> usize { /// by their address rather than comparing the values they point to /// (which is what the `PartialEq for &T` implementation does). /// +/// When comparing wide pointers, both the address and the metadata are tested for equality. +/// However, note that comparing trait object pointers (`*const dyn Trait`) is unreliable: pointers +/// to values of the same underlying type can compare inequal (because vtables are duplicated in +/// multiple codegen units), and pointers to values of *different* underlying type can compare equal +/// (since identical vtables can be deduplicated within a codegen unit). +/// /// # Examples /// /// ``` @@ -1204,43 +1822,8 @@ pub(crate) unsafe fn align_offset(p: *const T, a: usize) -> usize { /// assert!(!std::ptr::eq(&a[..2], &a[..3])); /// assert!(!std::ptr::eq(&a[0..2], &a[1..3])); /// ``` -/// -/// Traits are also compared by their implementation: -/// -/// ``` -/// #[repr(transparent)] -/// struct Wrapper { member: i32 } -/// -/// trait Trait {} -/// impl Trait for Wrapper {} -/// impl Trait for i32 {} -/// -/// let wrapper = Wrapper { member: 10 }; -/// -/// // Pointers have equal addresses. -/// assert!(std::ptr::eq( -/// &wrapper as *const Wrapper as *const u8, -/// &wrapper.member as *const i32 as *const u8 -/// )); -/// -/// // Objects have equal addresses, but `Trait` has different implementations. -/// assert!(!std::ptr::eq( -/// &wrapper as &dyn Trait, -/// &wrapper.member as &dyn Trait, -/// )); -/// assert!(!std::ptr::eq( -/// &wrapper as &dyn Trait as *const dyn Trait, -/// &wrapper.member as &dyn Trait as *const dyn Trait, -/// )); -/// -/// // Converting the reference to a `*const u8` compares by address. -/// assert!(std::ptr::eq( -/// &wrapper as &dyn Trait as *const dyn Trait as *const u8, -/// &wrapper.member as &dyn Trait as *const dyn Trait as *const u8, -/// )); -/// ``` #[stable(feature = "ptr_eq", since = "1.17.0")] -#[inline] +#[inline(always)] pub fn eq(a: *const T, b: *const T) -> bool { a == b } @@ -1277,54 +1860,107 @@ pub fn hash(hashee: *const T, into: &mut S) { hashee.hash(into); } +// If this is a unary fn pointer, it adds a doc comment. +// Otherwise, it hides the docs entirely. +macro_rules! maybe_fnptr_doc { + (@ #[$meta:meta] $item:item) => { + #[doc(hidden)] + #[$meta] + $item + }; + ($a:ident @ #[$meta:meta] $item:item) => { + #[doc(fake_variadic)] + #[doc = "This trait is implemented for function pointers with up to twelve arguments."] + #[$meta] + $item + }; + ($a:ident $($rest_a:ident)+ @ #[$meta:meta] $item:item) => { + #[doc(hidden)] + #[$meta] + $item + }; +} + +// FIXME(strict_provenance_magic): function pointers have buggy codegen that +// necessitates casting to a usize to get the backend to do the right thing. +// for now I will break AVR to silence *a billion* lints. We should probably +// have a proper "opaque function pointer type" to handle this kind of thing. + // Impls for function pointers macro_rules! fnptr_impls_safety_abi { ($FnTy: ty, $($Arg: ident),*) => { - #[stable(feature = "fnptr_impls", since = "1.4.0")] - impl PartialEq for $FnTy { - #[inline] - fn eq(&self, other: &Self) -> bool { - *self as usize == *other as usize + fnptr_impls_safety_abi! { #[stable(feature = "fnptr_impls", since = "1.4.0")] $FnTy, $($Arg),* } + }; + (@c_unwind $FnTy: ty, $($Arg: ident),*) => { + fnptr_impls_safety_abi! { #[unstable(feature = "c_unwind", issue = "74990")] $FnTy, $($Arg),* } + }; + (#[$meta:meta] $FnTy: ty, $($Arg: ident),*) => { + maybe_fnptr_doc! { + $($Arg)* @ + #[$meta] + impl PartialEq for $FnTy { + #[inline] + fn eq(&self, other: &Self) -> bool { + *self as usize == *other as usize + } } } - #[stable(feature = "fnptr_impls", since = "1.4.0")] - impl Eq for $FnTy {} + maybe_fnptr_doc! { + $($Arg)* @ + #[$meta] + impl Eq for $FnTy {} + } - #[stable(feature = "fnptr_impls", since = "1.4.0")] - impl PartialOrd for $FnTy { - #[inline] - fn partial_cmp(&self, other: &Self) -> Option { - (*self as usize).partial_cmp(&(*other as usize)) + maybe_fnptr_doc! { + $($Arg)* @ + #[$meta] + impl PartialOrd for $FnTy { + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + (*self as usize).partial_cmp(&(*other as usize)) + } } } - #[stable(feature = "fnptr_impls", since = "1.4.0")] - impl Ord for $FnTy { - #[inline] - fn cmp(&self, other: &Self) -> Ordering { - (*self as usize).cmp(&(*other as usize)) + maybe_fnptr_doc! { + $($Arg)* @ + #[$meta] + impl Ord for $FnTy { + #[inline] + fn cmp(&self, other: &Self) -> Ordering { + (*self as usize).cmp(&(*other as usize)) + } } } - #[stable(feature = "fnptr_impls", since = "1.4.0")] - impl hash::Hash for $FnTy { - fn hash(&self, state: &mut HH) { - state.write_usize(*self as usize) + maybe_fnptr_doc! { + $($Arg)* @ + #[$meta] + impl hash::Hash for $FnTy { + fn hash(&self, state: &mut HH) { + state.write_usize(*self as usize) + } } } - #[stable(feature = "fnptr_impls", since = "1.4.0")] - impl fmt::Pointer for $FnTy { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Pointer::fmt(&(*self as *const ()), f) + maybe_fnptr_doc! { + $($Arg)* @ + #[$meta] + impl fmt::Pointer for $FnTy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::pointer_fmt_inner(*self as usize, f) + } } } - #[stable(feature = "fnptr_impls", since = "1.4.0")] - impl fmt::Debug for $FnTy { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Pointer::fmt(&(*self as *const ()), f) + maybe_fnptr_doc! { + $($Arg)* @ + #[$meta] + impl fmt::Debug for $FnTy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::pointer_fmt_inner(*self as usize, f) + } } } } @@ -1335,21 +1971,27 @@ macro_rules! fnptr_impls_args { fnptr_impls_safety_abi! { extern "Rust" fn($($Arg),+) -> Ret, $($Arg),+ } fnptr_impls_safety_abi! { extern "C" fn($($Arg),+) -> Ret, $($Arg),+ } fnptr_impls_safety_abi! { extern "C" fn($($Arg),+ , ...) -> Ret, $($Arg),+ } + fnptr_impls_safety_abi! { @c_unwind extern "C-unwind" fn($($Arg),+) -> Ret, $($Arg),+ } + fnptr_impls_safety_abi! { @c_unwind extern "C-unwind" fn($($Arg),+ , ...) -> Ret, $($Arg),+ } fnptr_impls_safety_abi! { unsafe extern "Rust" fn($($Arg),+) -> Ret, $($Arg),+ } fnptr_impls_safety_abi! { unsafe extern "C" fn($($Arg),+) -> Ret, $($Arg),+ } fnptr_impls_safety_abi! { unsafe extern "C" fn($($Arg),+ , ...) -> Ret, $($Arg),+ } + fnptr_impls_safety_abi! { @c_unwind unsafe extern "C-unwind" fn($($Arg),+) -> Ret, $($Arg),+ } + fnptr_impls_safety_abi! { @c_unwind unsafe extern "C-unwind" fn($($Arg),+ , ...) -> Ret, $($Arg),+ } }; () => { // No variadic functions with 0 parameters fnptr_impls_safety_abi! { extern "Rust" fn() -> Ret, } fnptr_impls_safety_abi! { extern "C" fn() -> Ret, } + fnptr_impls_safety_abi! { @c_unwind extern "C-unwind" fn() -> Ret, } fnptr_impls_safety_abi! { unsafe extern "Rust" fn() -> Ret, } fnptr_impls_safety_abi! { unsafe extern "C" fn() -> Ret, } + fnptr_impls_safety_abi! { @c_unwind unsafe extern "C-unwind" fn() -> Ret, } }; } fnptr_impls_args! {} -fnptr_impls_args! { A } +fnptr_impls_args! { T } fnptr_impls_args! { A, B } fnptr_impls_args! { A, B, C } fnptr_impls_args! { A, B, C, D } @@ -1361,3 +2003,99 @@ fnptr_impls_args! { A, B, C, D, E, F, G, H, I } fnptr_impls_args! { A, B, C, D, E, F, G, H, I, J } fnptr_impls_args! { A, B, C, D, E, F, G, H, I, J, K } fnptr_impls_args! { A, B, C, D, E, F, G, H, I, J, K, L } + +/// Create a `const` raw pointer to a place, without creating an intermediate reference. +/// +/// Creating a reference with `&`/`&mut` is only allowed if the pointer is properly aligned +/// and points to initialized data. For cases where those requirements do not hold, +/// raw pointers should be used instead. However, `&expr as *const _` creates a reference +/// before casting it to a raw pointer, and that reference is subject to the same rules +/// as all other references. This macro can create a raw pointer *without* creating +/// a reference first. +/// +/// Note, however, that the `expr` in `addr_of!(expr)` is still subject to all +/// the usual rules. In particular, `addr_of!(*ptr::null())` is Undefined +/// Behavior because it dereferences a null pointer. +/// +/// # Example +/// +/// ``` +/// use std::ptr; +/// +/// #[repr(packed)] +/// struct Packed { +/// f1: u8, +/// f2: u16, +/// } +/// +/// let packed = Packed { f1: 1, f2: 2 }; +/// // `&packed.f2` would create an unaligned reference, and thus be Undefined Behavior! +/// let raw_f2 = ptr::addr_of!(packed.f2); +/// assert_eq!(unsafe { raw_f2.read_unaligned() }, 2); +/// ``` +/// +/// See [`addr_of_mut`] for how to create a pointer to unininitialized data. +/// Doing that with `addr_of` would not make much sense since one could only +/// read the data, and that would be Undefined Behavior. +#[stable(feature = "raw_ref_macros", since = "1.51.0")] +#[rustc_macro_transparency = "semitransparent"] +#[allow_internal_unstable(raw_ref_op)] +pub macro addr_of($place:expr) { + &raw const $place +} + +/// Create a `mut` raw pointer to a place, without creating an intermediate reference. +/// +/// Creating a reference with `&`/`&mut` is only allowed if the pointer is properly aligned +/// and points to initialized data. For cases where those requirements do not hold, +/// raw pointers should be used instead. However, `&mut expr as *mut _` creates a reference +/// before casting it to a raw pointer, and that reference is subject to the same rules +/// as all other references. This macro can create a raw pointer *without* creating +/// a reference first. +/// +/// Note, however, that the `expr` in `addr_of_mut!(expr)` is still subject to all +/// the usual rules. In particular, `addr_of_mut!(*ptr::null_mut())` is Undefined +/// Behavior because it dereferences a null pointer. +/// +/// # Examples +/// +/// **Creating a pointer to unaligned data:** +/// +/// ``` +/// use std::ptr; +/// +/// #[repr(packed)] +/// struct Packed { +/// f1: u8, +/// f2: u16, +/// } +/// +/// let mut packed = Packed { f1: 1, f2: 2 }; +/// // `&mut packed.f2` would create an unaligned reference, and thus be Undefined Behavior! +/// let raw_f2 = ptr::addr_of_mut!(packed.f2); +/// unsafe { raw_f2.write_unaligned(42); } +/// assert_eq!({packed.f2}, 42); // `{...}` forces copying the field instead of creating a reference. +/// ``` +/// +/// **Creating a pointer to uninitialized data:** +/// +/// ```rust +/// use std::{ptr, mem::MaybeUninit}; +/// +/// struct Demo { +/// field: bool, +/// } +/// +/// let mut uninit = MaybeUninit::::uninit(); +/// // `&uninit.as_mut().field` would create a reference to an uninitialized `bool`, +/// // and thus be Undefined Behavior! +/// let f1_ptr = unsafe { ptr::addr_of_mut!((*uninit.as_mut_ptr()).field) }; +/// unsafe { f1_ptr.write(true); } +/// let init = unsafe { uninit.assume_init() }; +/// ``` +#[stable(feature = "raw_ref_macros", since = "1.51.0")] +#[rustc_macro_transparency = "semitransparent"] +#[allow_internal_unstable(raw_ref_op)] +pub macro addr_of_mut($place:expr) { + &raw mut $place +} diff --git a/crux-mir/lib/core/src/ptr/mut_ptr.rs b/crux-mir/lib/core/src/ptr/mut_ptr.rs index 338e208d2..ed1e3bd48 100644 --- a/crux-mir/lib/core/src/ptr/mut_ptr.rs +++ b/crux-mir/lib/core/src/ptr/mut_ptr.rs @@ -1,11 +1,8 @@ use super::*; use crate::cmp::Ordering::{self, Equal, Greater, Less}; -use crate::crucible; -use crate::intrinsics; +use crate::intrinsics::{self, const_eval_select}; +use crate::slice::{self, SliceIndex}; -// ignore-tidy-undocumented-unsafe - -#[lang = "mut_ptr"] impl *mut T { /// Returns `true` if the pointer is null. /// @@ -14,6 +11,15 @@ impl *mut T { /// Therefore, two pointers that are null may still not compare equal to /// each other. /// + /// ## Behavior during const evaluation + /// + /// When this function is used during const evaluation, it may return `false` for pointers + /// that turn out to be null at runtime. Specifically, when a pointer to some memory + /// is offset beyond its bounds in such a way that the resulting pointer is null, + /// the function will still return `false`. There is no way for CTFE to know + /// the absolute position of that memory, so we cannot tell if the pointer is + /// null or not. + /// /// # Examples /// /// Basic usage: @@ -24,40 +30,305 @@ impl *mut T { /// assert!(!ptr.is_null()); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_ptr_is_null", issue = "74939")] #[inline] - pub fn is_null(self) -> bool { - crucible::ptr::compare_usize(self, 0) + pub const fn is_null(self) -> bool { + #[inline] + fn runtime_impl(ptr: *mut u8) -> bool { + ptr.addr() == 0 + } + + #[inline] + const fn const_impl(ptr: *mut u8) -> bool { + // Compare via a cast to a thin pointer, so fat pointers are only + // considering their "data" part for null-ness. + match (ptr).guaranteed_eq(null_mut()) { + None => false, + Some(res) => res, + } + } + + // SAFETY: The two versions are equivalent at runtime. + unsafe { const_eval_select((self as *mut u8,), const_impl, runtime_impl) } } /// Casts to a pointer of another type. #[stable(feature = "ptr_cast", since = "1.38.0")] #[rustc_const_stable(feature = "const_ptr_cast", since = "1.38.0")] - #[inline] + #[inline(always)] pub const fn cast(self) -> *mut U { self as _ } - /// Returns `None` if the pointer is null, or else returns a reference to - /// the value wrapped in `Some`. + /// Use the pointer value in a new pointer of another type. + /// + /// In case `val` is a (fat) pointer to an unsized type, this operation + /// will ignore the pointer part, whereas for (thin) pointers to sized + /// types, this has the same effect as a simple cast. + /// + /// The resulting pointer will have provenance of `self`, i.e., for a fat + /// pointer, this operation is semantically the same as creating a new + /// fat pointer with the data pointer value of `self` but the metadata of + /// `val`. + /// + /// # Examples + /// + /// This function is primarily useful for allowing byte-wise pointer + /// arithmetic on potentially fat pointers: + /// + /// ``` + /// #![feature(set_ptr_value)] + /// # use core::fmt::Debug; + /// let mut arr: [i32; 3] = [1, 2, 3]; + /// let mut ptr = arr.as_mut_ptr() as *mut dyn Debug; + /// let thin = ptr as *mut u8; + /// unsafe { + /// ptr = thin.add(8).with_metadata_of(ptr); + /// # assert_eq!(*(ptr as *mut i32), 3); + /// println!("{:?}", &*ptr); // will print "3" + /// } + /// ``` + #[unstable(feature = "set_ptr_value", issue = "75091")] + #[rustc_const_unstable(feature = "set_ptr_value", issue = "75091")] + #[must_use = "returns a new pointer rather than modifying its argument"] + #[inline] + pub const fn with_metadata_of(self, meta: *const U) -> *mut U + where + U: ?Sized, + { + from_raw_parts_mut::(self as *mut (), metadata(meta)) + } + + /// Changes constness without changing the type. + /// + /// This is a bit safer than `as` because it wouldn't silently change the type if the code is + /// refactored. + /// + /// While not strictly required (`*mut T` coerces to `*const T`), this is provided for symmetry + /// with [`cast_mut`] on `*const T` and may have documentation value if used instead of implicit + /// coercion. + /// + /// [`cast_mut`]: #method.cast_mut + #[stable(feature = "ptr_const_cast", since = "1.65.0")] + #[rustc_const_stable(feature = "ptr_const_cast", since = "1.65.0")] + #[inline(always)] + pub const fn cast_const(self) -> *const T { + self as _ + } + + /// Casts a pointer to its raw bits. + /// + /// This is equivalent to `as usize`, but is more specific to enhance readability. + /// The inverse method is [`from_bits`](#method.from_bits-1). + /// + /// In particular, `*p as usize` and `p as usize` will both compile for + /// pointers to numeric types but do very different things, so using this + /// helps emphasize that reading the bits was intentional. + /// + /// # Examples + /// + /// ``` + /// #![feature(ptr_to_from_bits)] + /// # #[cfg(not(miri))] { // doctest does not work with strict provenance + /// let mut array = [13, 42]; + /// let mut it = array.iter_mut(); + /// let p0: *mut i32 = it.next().unwrap(); + /// assert_eq!(<*mut _>::from_bits(p0.to_bits()), p0); + /// let p1: *mut i32 = it.next().unwrap(); + /// assert_eq!(p1.to_bits() - p0.to_bits(), 4); + /// } + /// ``` + #[unstable(feature = "ptr_to_from_bits", issue = "91126")] + #[deprecated( + since = "1.67", + note = "replaced by the `exposed_addr` method, or update your code \ + to follow the strict provenance rules using its APIs" + )] + #[inline(always)] + pub fn to_bits(self) -> usize + where + T: Sized, + { + self as usize + } + + /// Creates a pointer from its raw bits. + /// + /// This is equivalent to `as *mut T`, but is more specific to enhance readability. + /// The inverse method is [`to_bits`](#method.to_bits-1). + /// + /// # Examples + /// + /// ``` + /// #![feature(ptr_to_from_bits)] + /// # #[cfg(not(miri))] { // doctest does not work with strict provenance + /// use std::ptr::NonNull; + /// let dangling: *mut u8 = NonNull::dangling().as_ptr(); + /// assert_eq!(<*mut u8>::from_bits(1), dangling); + /// } + /// ``` + #[unstable(feature = "ptr_to_from_bits", issue = "91126")] + #[deprecated( + since = "1.67", + note = "replaced by the `ptr::from_exposed_addr_mut` function, or \ + update your code to follow the strict provenance rules using its APIs" + )] + #[allow(fuzzy_provenance_casts)] // this is an unstable and semi-deprecated cast function + #[inline(always)] + pub fn from_bits(bits: usize) -> Self + where + T: Sized, + { + bits as Self + } + + /// Gets the "address" portion of the pointer. + /// + /// This is similar to `self as usize`, which semantically discards *provenance* and + /// *address-space* information. However, unlike `self as usize`, casting the returned address + /// back to a pointer yields [`invalid`][], which is undefined behavior to dereference. To + /// properly restore the lost information and obtain a dereferenceable pointer, use + /// [`with_addr`][pointer::with_addr] or [`map_addr`][pointer::map_addr]. + /// + /// If using those APIs is not possible because there is no way to preserve a pointer with the + /// required provenance, use [`expose_addr`][pointer::expose_addr] and + /// [`from_exposed_addr_mut`][from_exposed_addr_mut] instead. However, note that this makes + /// your code less portable and less amenable to tools that check for compliance with the Rust + /// memory model. + /// + /// On most platforms this will produce a value with the same bytes as the original + /// pointer, because all the bytes are dedicated to describing the address. + /// Platforms which need to store additional information in the pointer may + /// perform a change of representation to produce a value containing only the address + /// portion of the pointer. What that means is up to the platform to define. + /// + /// This API and its claimed semantics are part of the Strict Provenance experiment, and as such + /// might change in the future (including possibly weakening this so it becomes wholly + /// equivalent to `self as usize`). See the [module documentation][crate::ptr] for details. + #[must_use] + #[inline(always)] + #[unstable(feature = "strict_provenance", issue = "95228")] + pub fn addr(self) -> usize { + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + // SAFETY: Pointer-to-integer transmutes are valid (if you are okay with losing the + // provenance). + unsafe { mem::transmute(self.cast::<()>()) } + } + + /// Gets the "address" portion of the pointer, and 'exposes' the "provenance" part for future + /// use in [`from_exposed_addr`][]. + /// + /// This is equivalent to `self as usize`, which semantically discards *provenance* and + /// *address-space* information. Furthermore, this (like the `as` cast) has the implicit + /// side-effect of marking the provenance as 'exposed', so on platforms that support it you can + /// later call [`from_exposed_addr_mut`][] to reconstitute the original pointer including its + /// provenance. (Reconstructing address space information, if required, is your responsibility.) + /// + /// Using this method means that code is *not* following Strict Provenance rules. Supporting + /// [`from_exposed_addr_mut`][] complicates specification and reasoning and may not be supported + /// by tools that help you to stay conformant with the Rust memory model, so it is recommended + /// to use [`addr`][pointer::addr] wherever possible. + /// + /// On most platforms this will produce a value with the same bytes as the original pointer, + /// because all the bytes are dedicated to describing the address. Platforms which need to store + /// additional information in the pointer may not support this operation, since the 'expose' + /// side-effect which is required for [`from_exposed_addr_mut`][] to work is typically not + /// available. + /// + /// This API and its claimed semantics are part of the Strict Provenance experiment, see the + /// [module documentation][crate::ptr] for details. + /// + /// [`from_exposed_addr_mut`]: from_exposed_addr_mut + #[must_use] + #[inline(always)] + #[unstable(feature = "strict_provenance", issue = "95228")] + pub fn expose_addr(self) -> usize { + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + self.cast::<()>() as usize + } + + /// Creates a new pointer with the given address. + /// + /// This performs the same operation as an `addr as ptr` cast, but copies + /// the *address-space* and *provenance* of `self` to the new pointer. + /// This allows us to dynamically preserve and propagate this important + /// information in a way that is otherwise impossible with a unary cast. + /// + /// This is equivalent to using [`wrapping_offset`][pointer::wrapping_offset] to offset + /// `self` to the given address, and therefore has all the same capabilities and restrictions. + /// + /// This API and its claimed semantics are part of the Strict Provenance experiment, + /// see the [module documentation][crate::ptr] for details. + #[must_use] + #[inline] + #[unstable(feature = "strict_provenance", issue = "95228")] + pub fn with_addr(self, addr: usize) -> Self { + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + // + // In the mean-time, this operation is defined to be "as if" it was + // a wrapping_offset, so we can emulate it as such. This should properly + // restore pointer provenance even under today's compiler. + let self_addr = self.addr() as isize; + let dest_addr = addr as isize; + let offset = dest_addr.wrapping_sub(self_addr); + + // This is the canonical desugarring of this operation + self.wrapping_byte_offset(offset) + } + + /// Creates a new pointer by mapping `self`'s address to a new one. + /// + /// This is a convenience for [`with_addr`][pointer::with_addr], see that method for details. + /// + /// This API and its claimed semantics are part of the Strict Provenance experiment, + /// see the [module documentation][crate::ptr] for details. + #[must_use] + #[inline] + #[unstable(feature = "strict_provenance", issue = "95228")] + pub fn map_addr(self, f: impl FnOnce(usize) -> usize) -> Self { + self.with_addr(f(self.addr())) + } + + /// Decompose a (possibly wide) pointer into its address and metadata components. + /// + /// The pointer can be later reconstructed with [`from_raw_parts_mut`]. + #[unstable(feature = "ptr_metadata", issue = "81513")] + #[rustc_const_unstable(feature = "ptr_metadata", issue = "81513")] + #[inline] + pub const fn to_raw_parts(self) -> (*mut (), ::Metadata) { + (self.cast(), super::metadata(self)) + } + + /// Returns `None` if the pointer is null, or else returns a shared reference to + /// the value wrapped in `Some`. If the value may be uninitialized, [`as_uninit_ref`] + /// must be used instead. + /// + /// For the mutable counterpart see [`as_mut`]. + /// + /// [`as_uninit_ref`]: #method.as_uninit_ref-1 + /// [`as_mut`]: #method.as_mut /// /// # Safety /// - /// While this method and its mutable counterpart are useful for - /// null-safety, it is important to note that this is still an unsafe - /// operation because the returned value could be pointing to invalid - /// memory. + /// When calling this method, you have to ensure that *either* the pointer is null *or* + /// all of the following is true: + /// + /// * The pointer must be properly aligned. + /// + /// * It must be "dereferenceable" in the sense defined in [the module documentation]. + /// + /// * The pointer must point to an initialized instance of `T`. + /// + /// * You must enforce Rust's aliasing rules, since the returned lifetime `'a` is + /// arbitrarily chosen and does not necessarily reflect the actual lifetime of the data. + /// In particular, while this reference exists, the memory the pointer points to must + /// not get mutated (except inside `UnsafeCell`). /// - /// When calling this method, you have to ensure that if the pointer is - /// non-NULL, then it is properly aligned, dereferenceable (for the whole - /// size of `T`) and points to an initialized instance of `T`. This applies - /// even if the result of this method is unused! + /// This applies even if the result of this method is unused! /// (The part about being initialized is not yet fully decided, but until /// it is, the only safe approach is to ensure that they are indeed initialized.) /// - /// Additionally, the lifetime `'a` returned is arbitrarily chosen and does - /// not necessarily reflect the actual lifetime of the data. It is up to the - /// caller to ensure that for the duration of this lifetime, the memory this - /// pointer points to does not get written to outside of `UnsafeCell`. + /// [the module documentation]: crate::ptr#safety /// /// # Examples /// @@ -68,7 +339,7 @@ impl *mut T { /// /// unsafe { /// if let Some(val_back) = ptr.as_ref() { - /// println!("We got back the value: {}!", val_back); + /// println!("We got back the value: {val_back}!"); /// } /// } /// ``` @@ -84,13 +355,70 @@ impl *mut T { /// /// unsafe { /// let val_back = &*ptr; - /// println!("We got back the value: {}!", val_back); + /// println!("We got back the value: {val_back}!"); /// } /// ``` #[stable(feature = "ptr_as_ref", since = "1.9.0")] + #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")] + #[inline] + pub const unsafe fn as_ref<'a>(self) -> Option<&'a T> { + // SAFETY: the caller must guarantee that `self` is valid for a + // reference if it isn't null. + if self.is_null() { None } else { unsafe { Some(&*self) } } + } + + /// Returns `None` if the pointer is null, or else returns a shared reference to + /// the value wrapped in `Some`. In contrast to [`as_ref`], this does not require + /// that the value has to be initialized. + /// + /// For the mutable counterpart see [`as_uninit_mut`]. + /// + /// [`as_ref`]: #method.as_ref-1 + /// [`as_uninit_mut`]: #method.as_uninit_mut + /// + /// # Safety + /// + /// When calling this method, you have to ensure that *either* the pointer is null *or* + /// all of the following is true: + /// + /// * The pointer must be properly aligned. + /// + /// * It must be "dereferenceable" in the sense defined in [the module documentation]. + /// + /// * You must enforce Rust's aliasing rules, since the returned lifetime `'a` is + /// arbitrarily chosen and does not necessarily reflect the actual lifetime of the data. + /// In particular, while this reference exists, the memory the pointer points to must + /// not get mutated (except inside `UnsafeCell`). + /// + /// This applies even if the result of this method is unused! + /// + /// [the module documentation]: crate::ptr#safety + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(ptr_as_uninit)] + /// + /// let ptr: *mut u8 = &mut 10u8 as *mut u8; + /// + /// unsafe { + /// if let Some(val_back) = ptr.as_uninit_ref() { + /// println!("We got back the value: {}!", val_back.assume_init()); + /// } + /// } + /// ``` #[inline] - pub unsafe fn as_ref<'a>(self) -> Option<&'a T> { - if self.is_null() { None } else { Some(&*self) } + #[unstable(feature = "ptr_as_uninit", issue = "75402")] + #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")] + pub const unsafe fn as_uninit_ref<'a>(self) -> Option<&'a MaybeUninit> + where + T: Sized, + { + // SAFETY: the caller must guarantee that `self` meets all the + // requirements for a reference. + if self.is_null() { None } else { Some(unsafe { &*(self as *const MaybeUninit) }) } } /// Calculates the offset from a pointer. @@ -104,8 +432,7 @@ impl *mut T { /// Behavior: /// /// * Both the starting and resulting pointer must be either in bounds or one - /// byte past the end of the same allocated object. Note that in Rust, - /// every (stack-allocated) variable is considered a separate allocated object. + /// byte past the end of the same [allocated object]. /// /// * The computed offset, **in bytes**, cannot overflow an `isize`. /// @@ -130,6 +457,7 @@ impl *mut T { /// enables more aggressive compiler optimizations. /// /// [`wrapping_offset`]: #method.wrapping_offset + /// [allocated object]: crate::ptr#allocated-object /// /// # Examples /// @@ -145,12 +473,38 @@ impl *mut T { /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] - #[inline] - pub unsafe fn offset(self, count: isize) -> *mut T + #[must_use = "returns a new pointer rather than modifying its argument"] + #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn offset(self, count: isize) -> *mut T where T: Sized, { - intrinsics::offset(self, count) as *mut T + // SAFETY: the caller must uphold the safety contract for `offset`. + // The obtained pointer is valid for writes since the caller must + // guarantee that it points to the same allocated object as `self`. + unsafe { intrinsics::offset(self, count) as *mut T } + } + + /// Calculates the offset from a pointer in bytes. + /// + /// `count` is in units of **bytes**. + /// + /// This is purely a convenience for casting to a `u8` pointer and + /// using [offset][pointer::offset] on it. See that method for documentation + /// and safety requirements. + /// + /// For non-`Sized` pointees this operation changes only the data pointer, + /// leaving the metadata untouched. + #[must_use] + #[inline(always)] + #[unstable(feature = "pointer_byte_offsets", issue = "96283")] + #[rustc_const_unstable(feature = "const_pointer_byte_offsets", issue = "96283")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn byte_offset(self, count: isize) -> Self { + // SAFETY: the caller must uphold the safety contract for `offset`. + unsafe { self.cast::().offset(count).with_metadata_of(self) } } /// Calculates the offset from a pointer using wrapping arithmetic. @@ -159,28 +513,29 @@ impl *mut T { /// /// # Safety /// - /// The resulting pointer does not need to be in bounds, but it is - /// potentially hazardous to dereference (which requires `unsafe`). + /// This operation itself is always safe, but using the resulting pointer is not. /// - /// In particular, the resulting pointer remains attached to the same allocated - /// object that `self` points to. It may *not* be used to access a - /// different allocated object. Note that in Rust, - /// every (stack-allocated) variable is considered a separate allocated object. + /// The resulting pointer "remembers" the [allocated object] that `self` points to; it must not + /// be used to read or write other allocated objects. /// - /// In other words, `x.wrapping_offset(y.wrapping_offset_from(x))` is - /// *not* the same as `y`, and dereferencing it is undefined behavior - /// unless `x` and `y` point into the same allocated object. + /// In other words, `let z = x.wrapping_offset((y as isize) - (x as isize))` does *not* make `z` + /// the same as `y` even if we assume `T` has size `1` and there is no overflow: `z` is still + /// attached to the object `x` is attached to, and dereferencing it is Undefined Behavior unless + /// `x` and `y` point into the same allocated object. /// - /// Compared to [`offset`], this method basically delays the requirement of staying - /// within the same allocated object: [`offset`] is immediate Undefined Behavior when - /// crossing object boundaries; `wrapping_offset` produces a pointer but still leads - /// to Undefined Behavior if that pointer is dereferenced. [`offset`] can be optimized - /// better and is thus preferable in performance-sensitive code. + /// Compared to [`offset`], this method basically delays the requirement of staying within the + /// same allocated object: [`offset`] is immediate Undefined Behavior when crossing object + /// boundaries; `wrapping_offset` produces a pointer but still leads to Undefined Behavior if a + /// pointer is dereferenced when it is out-of-bounds of the object it is attached to. [`offset`] + /// can be optimized better and is thus preferable in performance-sensitive code. /// - /// If you need to cross object boundaries, cast the pointer to an integer and - /// do the arithmetic there. + /// The delayed check only considers the value of the pointer that was dereferenced, not the + /// intermediate values used during the computation of the final result. For example, + /// `x.wrapping_offset(o).wrapping_offset(o.wrapping_neg())` is always the same as `x`. In other + /// words, leaving the allocated object and then re-entering it later is permitted. /// /// [`offset`]: #method.offset + /// [allocated object]: crate::ptr#allocated-object /// /// # Examples /// @@ -202,41 +557,106 @@ impl *mut T { /// assert_eq!(&data, &[0, 2, 0, 4, 0]); /// ``` #[stable(feature = "ptr_wrapping_offset", since = "1.16.0")] - #[inline] - pub fn wrapping_offset(self, count: isize) -> *mut T + #[must_use = "returns a new pointer rather than modifying its argument"] + #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] + #[inline(always)] + pub const fn wrapping_offset(self, count: isize) -> *mut T where T: Sized, { + // SAFETY: the `arith_offset` intrinsic has no prerequisites to be called. unsafe { intrinsics::arith_offset(self, count) as *mut T } } - /// Returns `None` if the pointer is null, or else returns a mutable - /// reference to the value wrapped in `Some`. + /// Calculates the offset from a pointer in bytes using wrapping arithmetic. + /// + /// `count` is in units of **bytes**. + /// + /// This is purely a convenience for casting to a `u8` pointer and + /// using [wrapping_offset][pointer::wrapping_offset] on it. See that method + /// for documentation. + /// + /// For non-`Sized` pointees this operation changes only the data pointer, + /// leaving the metadata untouched. + #[must_use] + #[inline(always)] + #[unstable(feature = "pointer_byte_offsets", issue = "96283")] + #[rustc_const_unstable(feature = "const_pointer_byte_offsets", issue = "96283")] + pub const fn wrapping_byte_offset(self, count: isize) -> Self { + self.cast::().wrapping_offset(count).with_metadata_of(self) + } + + /// Masks out bits of the pointer according to a mask. /// - /// # Safety + /// This is convenience for `ptr.map_addr(|a| a & mask)`. + /// + /// For non-`Sized` pointees this operation changes only the data pointer, + /// leaving the metadata untouched. + /// + /// ## Examples + /// + /// ``` + /// #![feature(ptr_mask, strict_provenance)] + /// let mut v = 17_u32; + /// let ptr: *mut u32 = &mut v; + /// + /// // `u32` is 4 bytes aligned, + /// // which means that lower 2 bits are always 0. + /// let tag_mask = 0b11; + /// let ptr_mask = !tag_mask; + /// + /// // We can store something in these lower bits + /// let tagged_ptr = ptr.map_addr(|a| a | 0b10); + /// + /// // Get the "tag" back + /// let tag = tagged_ptr.addr() & tag_mask; + /// assert_eq!(tag, 0b10); + /// + /// // Note that `tagged_ptr` is unaligned, it's UB to read from/write to it. + /// // To get original pointer `mask` can be used: + /// let masked_ptr = tagged_ptr.mask(ptr_mask); + /// assert_eq!(unsafe { *masked_ptr }, 17); + /// + /// unsafe { *masked_ptr = 0 }; + /// assert_eq!(v, 0); + /// ``` + #[unstable(feature = "ptr_mask", issue = "98290")] + #[must_use = "returns a new pointer rather than modifying its argument"] + #[inline(always)] + pub fn mask(self, mask: usize) -> *mut T { + intrinsics::ptr_mask(self.cast::<()>(), mask).cast_mut().with_metadata_of(self) + } + + /// Returns `None` if the pointer is null, or else returns a unique reference to + /// the value wrapped in `Some`. If the value may be uninitialized, [`as_uninit_mut`] + /// must be used instead. + /// + /// For the shared counterpart see [`as_ref`]. + /// + /// [`as_uninit_mut`]: #method.as_uninit_mut + /// [`as_ref`]: #method.as_ref-1 /// - /// As with [`as_ref`], this is unsafe because it cannot verify the validity - /// of the returned pointer, nor can it ensure that the lifetime `'a` - /// returned is indeed a valid lifetime for the contained data. + /// # Safety /// - /// When calling this method, you have to ensure that *either* the pointer is NULL *or* + /// When calling this method, you have to ensure that *either* the pointer is null *or* /// all of the following is true: - /// - it is properly aligned - /// - it must point to an initialized instance of T; in particular, the pointer must be - /// "dereferenceable" in the sense defined [here]. + /// + /// * The pointer must be properly aligned. + /// + /// * It must be "dereferenceable" in the sense defined in [the module documentation]. + /// + /// * The pointer must point to an initialized instance of `T`. + /// + /// * You must enforce Rust's aliasing rules, since the returned lifetime `'a` is + /// arbitrarily chosen and does not necessarily reflect the actual lifetime of the data. + /// In particular, while this reference exists, the memory the pointer points to must + /// not get accessed (read or written) through any other pointer. /// /// This applies even if the result of this method is unused! /// (The part about being initialized is not yet fully decided, but until - /// it is the only safe approach is to ensure that they are indeed initialized.) - /// - /// Additionally, the lifetime `'a` returned is arbitrarily chosen and does - /// not necessarily reflect the actual lifetime of the data. *You* must enforce - /// Rust's aliasing rules. In particular, for the duration of this lifetime, - /// the memory this pointer points to must not get accessed (read or written) - /// through any other pointer. + /// it is, the only safe approach is to ensure that they are indeed initialized.) /// - /// [here]: crate::ptr#safety - /// [`as_ref`]: #method.as_ref + /// [the module documentation]: crate::ptr#safety /// /// # Examples /// @@ -247,7 +667,8 @@ impl *mut T { /// let ptr: *mut u32 = s.as_mut_ptr(); /// let first_value = unsafe { ptr.as_mut().unwrap() }; /// *first_value = 4; - /// println!("{:?}", s); // It'll print: "[4, 2, 3]". + /// # assert_eq!(s, [4, 2, 3]); + /// println!("{s:?}"); // It'll print: "[4, 2, 3]". /// ``` /// /// # Null-unchecked version @@ -261,21 +682,116 @@ impl *mut T { /// let ptr: *mut u32 = s.as_mut_ptr(); /// let first_value = unsafe { &mut *ptr }; /// *first_value = 4; - /// println!("{:?}", s); // It'll print: "[4, 2, 3]". + /// # assert_eq!(s, [4, 2, 3]); + /// println!("{s:?}"); // It'll print: "[4, 2, 3]". /// ``` #[stable(feature = "ptr_as_ref", since = "1.9.0")] + #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")] #[inline] - pub unsafe fn as_mut<'a>(self) -> Option<&'a mut T> { - if self.is_null() { None } else { Some(&mut *self) } + pub const unsafe fn as_mut<'a>(self) -> Option<&'a mut T> { + // SAFETY: the caller must guarantee that `self` is be valid for + // a mutable reference if it isn't null. + if self.is_null() { None } else { unsafe { Some(&mut *self) } } + } + + /// Returns `None` if the pointer is null, or else returns a unique reference to + /// the value wrapped in `Some`. In contrast to [`as_mut`], this does not require + /// that the value has to be initialized. + /// + /// For the shared counterpart see [`as_uninit_ref`]. + /// + /// [`as_mut`]: #method.as_mut + /// [`as_uninit_ref`]: #method.as_uninit_ref-1 + /// + /// # Safety + /// + /// When calling this method, you have to ensure that *either* the pointer is null *or* + /// all of the following is true: + /// + /// * The pointer must be properly aligned. + /// + /// * It must be "dereferenceable" in the sense defined in [the module documentation]. + /// + /// * You must enforce Rust's aliasing rules, since the returned lifetime `'a` is + /// arbitrarily chosen and does not necessarily reflect the actual lifetime of the data. + /// In particular, while this reference exists, the memory the pointer points to must + /// not get accessed (read or written) through any other pointer. + /// + /// This applies even if the result of this method is unused! + /// + /// [the module documentation]: crate::ptr#safety + #[inline] + #[unstable(feature = "ptr_as_uninit", issue = "75402")] + #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")] + pub const unsafe fn as_uninit_mut<'a>(self) -> Option<&'a mut MaybeUninit> + where + T: Sized, + { + // SAFETY: the caller must guarantee that `self` meets all the + // requirements for a reference. + if self.is_null() { None } else { Some(unsafe { &mut *(self as *mut MaybeUninit) }) } + } + + /// Returns whether two pointers are guaranteed to be equal. + /// + /// At runtime this function behaves like `Some(self == other)`. + /// However, in some contexts (e.g., compile-time evaluation), + /// it is not always possible to determine equality of two pointers, so this function may + /// spuriously return `None` for pointers that later actually turn out to have its equality known. + /// But when it returns `Some`, the pointers' equality is guaranteed to be known. + /// + /// The return value may change from `Some` to `None` and vice versa depending on the compiler + /// version and unsafe code must not + /// rely on the result of this function for soundness. It is suggested to only use this function + /// for performance optimizations where spurious `None` return values by this function do not + /// affect the outcome, but just the performance. + /// The consequences of using this method to make runtime and compile-time code behave + /// differently have not been explored. This method should not be used to introduce such + /// differences, and it should also not be stabilized before we have a better understanding + /// of this issue. + #[unstable(feature = "const_raw_ptr_comparison", issue = "53020")] + #[rustc_const_unstable(feature = "const_raw_ptr_comparison", issue = "53020")] + #[inline] + pub const fn guaranteed_eq(self, other: *mut T) -> Option + where + T: Sized, + { + (self as *const T).guaranteed_eq(other as _) + } + + /// Returns whether two pointers are guaranteed to be inequal. + /// + /// At runtime this function behaves like `Some(self != other)`. + /// However, in some contexts (e.g., compile-time evaluation), + /// it is not always possible to determine inequality of two pointers, so this function may + /// spuriously return `None` for pointers that later actually turn out to have its inequality known. + /// But when it returns `Some`, the pointers' inequality is guaranteed to be known. + /// + /// The return value may change from `Some` to `None` and vice versa depending on the compiler + /// version and unsafe code must not + /// rely on the result of this function for soundness. It is suggested to only use this function + /// for performance optimizations where spurious `None` return values by this function do not + /// affect the outcome, but just the performance. + /// The consequences of using this method to make runtime and compile-time code behave + /// differently have not been explored. This method should not be used to introduce such + /// differences, and it should also not be stabilized before we have a better understanding + /// of this issue. + #[unstable(feature = "const_raw_ptr_comparison", issue = "53020")] + #[rustc_const_unstable(feature = "const_raw_ptr_comparison", issue = "53020")] + #[inline] + pub const fn guaranteed_ne(self, other: *mut T) -> Option + where + T: Sized, + { + (self as *const T).guaranteed_ne(other as _) } /// Calculates the distance between two pointers. The returned value is in - /// units of T: the distance in bytes is divided by `mem::size_of::()`. + /// units of T: the distance in bytes divided by `mem::size_of::()`. /// /// This function is the inverse of [`offset`]. /// /// [`offset`]: #method.offset-1 - /// [`wrapping_offset_from`]: #method.wrapping_offset_from-1 /// /// # Safety /// @@ -283,32 +799,37 @@ impl *mut T { /// Behavior: /// /// * Both the starting and other pointer must be either in bounds or one - /// byte past the end of the same allocated object. Note that in Rust, - /// every (stack-allocated) variable is considered a separate allocated object. + /// byte past the end of the same [allocated object]. /// - /// * The distance between the pointers, **in bytes**, cannot overflow an `isize`. + /// * Both pointers must be *derived from* a pointer to the same object. + /// (See below for an example.) /// /// * The distance between the pointers, in bytes, must be an exact multiple /// of the size of `T`. /// + /// * The distance between the pointers, **in bytes**, cannot overflow an `isize`. + /// /// * The distance being in bounds cannot rely on "wrapping around" the address space. /// - /// The compiler and standard library generally try to ensure allocations - /// never reach a size where an offset is a concern. For instance, `Vec` - /// and `Box` ensure they never allocate more than `isize::MAX` bytes, so - /// `ptr_into_vec.offset_from(vec.as_ptr())` is always safe. + /// Rust types are never larger than `isize::MAX` and Rust allocations never wrap around the + /// address space, so two pointers within some value of any Rust type `T` will always satisfy + /// the last two conditions. The standard library also generally ensures that allocations + /// never reach a size where an offset is a concern. For instance, `Vec` and `Box` ensure they + /// never allocate more than `isize::MAX` bytes, so `ptr_into_vec.offset_from(vec.as_ptr())` + /// always satisfies the last two conditions. /// - /// Most platforms fundamentally can't even construct such an allocation. + /// Most platforms fundamentally can't even construct such a large allocation. /// For instance, no known 64-bit platform can ever serve a request /// for 263 bytes due to page-table limitations or splitting the address space. /// However, some 32-bit and 16-bit platforms may successfully serve a request for /// more than `isize::MAX` bytes with things like Physical Address /// Extension. As such, memory acquired directly from allocators or memory /// mapped files *may* be too large to handle with this function. + /// (Note that [`offset`] and [`add`] also have a similar limitation and hence cannot be used on + /// such large allocations either.) /// - /// Consider using [`wrapping_offset_from`] instead if these constraints are - /// difficult to satisfy. The only advantage of this method is that it - /// enables more aggressive compiler optimizations. + /// [`add`]: #method.add + /// [allocated object]: crate::ptr#allocated-object /// /// # Panics /// @@ -319,8 +840,6 @@ impl *mut T { /// Basic usage: /// /// ``` - /// #![feature(ptr_offset_from)] - /// /// let mut a = [0; 5]; /// let ptr1: *mut i32 = &mut a[1]; /// let ptr2: *mut i32 = &mut a[3]; @@ -331,57 +850,124 @@ impl *mut T { /// assert_eq!(ptr2.offset(-2), ptr1); /// } /// ``` - #[unstable(feature = "ptr_offset_from", issue = "41079")] - #[rustc_const_unstable(feature = "const_ptr_offset_from", issue = "41079")] - #[inline] + /// + /// *Incorrect* usage: + /// + /// ```rust,no_run + /// let ptr1 = Box::into_raw(Box::new(0u8)); + /// let ptr2 = Box::into_raw(Box::new(1u8)); + /// let diff = (ptr2 as isize).wrapping_sub(ptr1 as isize); + /// // Make ptr2_other an "alias" of ptr2, but derived from ptr1. + /// let ptr2_other = (ptr1 as *mut u8).wrapping_offset(diff); + /// assert_eq!(ptr2 as usize, ptr2_other as usize); + /// // Since ptr2_other and ptr2 are derived from pointers to different objects, + /// // computing their offset is undefined behavior, even though + /// // they point to the same address! + /// unsafe { + /// let zero = ptr2_other.offset_from(ptr2); // Undefined Behavior + /// } + /// ``` + #[stable(feature = "ptr_offset_from", since = "1.47.0")] + #[rustc_const_stable(feature = "const_ptr_offset_from", since = "1.65.0")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn offset_from(self, origin: *const T) -> isize where T: Sized, { - (self as *const T).offset_from(origin) + // SAFETY: the caller must uphold the safety contract for `offset_from`. + unsafe { (self as *const T).offset_from(origin) } } /// Calculates the distance between two pointers. The returned value is in + /// units of **bytes**. + /// + /// This is purely a convenience for casting to a `u8` pointer and + /// using [offset_from][pointer::offset_from] on it. See that method for + /// documentation and safety requirements. + /// + /// For non-`Sized` pointees this operation considers only the data pointers, + /// ignoring the metadata. + #[inline(always)] + #[unstable(feature = "pointer_byte_offsets", issue = "96283")] + #[rustc_const_unstable(feature = "const_pointer_byte_offsets", issue = "96283")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn byte_offset_from(self, origin: *const U) -> isize { + // SAFETY: the caller must uphold the safety contract for `offset_from`. + unsafe { self.cast::().offset_from(origin.cast::()) } + } + + /// Calculates the distance between two pointers, *where it's known that + /// `self` is equal to or greater than `origin`*. The returned value is in /// units of T: the distance in bytes is divided by `mem::size_of::()`. /// - /// If the address different between the two pointers is not a multiple of - /// `mem::size_of::()` then the result of the division is rounded towards - /// zero. + /// This computes the same value that [`offset_from`](#method.offset_from) + /// would compute, but with the added precondition that the offset is + /// guaranteed to be non-negative. This method is equivalent to + /// `usize::from(self.offset_from(origin)).unwrap_unchecked()`, + /// but it provides slightly more information to the optimizer, which can + /// sometimes allow it to optimize slightly better with some backends. + /// + /// This method can be though of as recovering the `count` that was passed + /// to [`add`](#method.add) (or, with the parameters in the other order, + /// to [`sub`](#method.sub)). The following are all equivalent, assuming + /// that their safety preconditions are met: + /// ```rust + /// # #![feature(ptr_sub_ptr)] + /// # unsafe fn blah(ptr: *mut i32, origin: *mut i32, count: usize) -> bool { + /// ptr.sub_ptr(origin) == count + /// # && + /// origin.add(count) == ptr + /// # && + /// ptr.sub(count) == origin + /// # } + /// ``` + /// + /// # Safety /// - /// Though this method is safe for any two pointers, note that its result - /// will be mostly useless if the two pointers aren't into the same allocated - /// object, for example if they point to two different local variables. + /// - The distance between the pointers must be non-negative (`self >= origin`) + /// + /// - *All* the safety conditions of [`offset_from`](#method.offset_from) + /// apply to this method as well; see it for the full details. + /// + /// Importantly, despite the return type of this method being able to represent + /// a larger offset, it's still *not permitted* to pass pointers which differ + /// by more than `isize::MAX` *bytes*. As such, the result of this method will + /// always be less than or equal to `isize::MAX as usize`. /// /// # Panics /// - /// This function panics if `T` is a zero-sized type. + /// This function panics if `T` is a Zero-Sized Type ("ZST"). /// /// # Examples /// - /// Basic usage: - /// /// ``` - /// #![feature(ptr_wrapping_offset_from)] + /// #![feature(ptr_sub_ptr)] /// /// let mut a = [0; 5]; - /// let ptr1: *mut i32 = &mut a[1]; - /// let ptr2: *mut i32 = &mut a[3]; - /// assert_eq!(ptr2.wrapping_offset_from(ptr1), 2); - /// assert_eq!(ptr1.wrapping_offset_from(ptr2), -2); - /// assert_eq!(ptr1.wrapping_offset(2), ptr2); - /// assert_eq!(ptr2.wrapping_offset(-2), ptr1); + /// let p: *mut i32 = a.as_mut_ptr(); + /// unsafe { + /// let ptr1: *mut i32 = p.add(1); + /// let ptr2: *mut i32 = p.add(3); /// - /// let ptr1: *mut i32 = 3 as _; - /// let ptr2: *mut i32 = 13 as _; - /// assert_eq!(ptr2.wrapping_offset_from(ptr1), 2); - /// ``` - #[unstable(feature = "ptr_wrapping_offset_from", issue = "41079")] + /// assert_eq!(ptr2.sub_ptr(ptr1), 2); + /// assert_eq!(ptr1.add(2), ptr2); + /// assert_eq!(ptr2.sub(2), ptr1); + /// assert_eq!(ptr2.sub_ptr(ptr2), 0); + /// } + /// + /// // This would be incorrect, as the pointers are not correctly ordered: + /// // ptr1.offset_from(ptr2) + #[unstable(feature = "ptr_sub_ptr", issue = "95892")] + #[rustc_const_unstable(feature = "const_ptr_sub_ptr", issue = "95892")] #[inline] - pub fn wrapping_offset_from(self, origin: *const T) -> isize + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn sub_ptr(self, origin: *const T) -> usize where T: Sized, { - (self as *const T).wrapping_offset_from(origin) + // SAFETY: the caller must uphold the safety contract for `sub_ptr`. + unsafe { (self as *const T).sub_ptr(origin) } } /// Calculates the offset from a pointer (convenience for `.offset(count as isize)`). @@ -395,8 +981,7 @@ impl *mut T { /// Behavior: /// /// * Both the starting and resulting pointer must be either in bounds or one - /// byte past the end of the same allocated object. Note that in Rust, - /// every (stack-allocated) variable is considered a separate allocated object. + /// byte past the end of the same [allocated object]. /// /// * The computed offset, **in bytes**, cannot overflow an `isize`. /// @@ -421,6 +1006,7 @@ impl *mut T { /// enables more aggressive compiler optimizations. /// /// [`wrapping_add`]: #method.wrapping_add + /// [allocated object]: crate::ptr#allocated-object /// /// # Examples /// @@ -436,12 +1022,36 @@ impl *mut T { /// } /// ``` #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] - pub unsafe fn add(self, count: usize) -> Self + #[must_use = "returns a new pointer rather than modifying its argument"] + #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn add(self, count: usize) -> Self where T: Sized, { - self.offset(count as isize) + // SAFETY: the caller must uphold the safety contract for `offset`. + unsafe { self.offset(count as isize) } + } + + /// Calculates the offset from a pointer in bytes (convenience for `.byte_offset(count as isize)`). + /// + /// `count` is in units of bytes. + /// + /// This is purely a convenience for casting to a `u8` pointer and + /// using [add][pointer::add] on it. See that method for documentation + /// and safety requirements. + /// + /// For non-`Sized` pointees this operation changes only the data pointer, + /// leaving the metadata untouched. + #[must_use] + #[inline(always)] + #[unstable(feature = "pointer_byte_offsets", issue = "96283")] + #[rustc_const_unstable(feature = "const_pointer_byte_offsets", issue = "96283")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn byte_add(self, count: usize) -> Self { + // SAFETY: the caller must uphold the safety contract for `add`. + unsafe { self.cast::().add(count).with_metadata_of(self) } } /// Calculates the offset from a pointer (convenience for @@ -456,8 +1066,7 @@ impl *mut T { /// Behavior: /// /// * Both the starting and resulting pointer must be either in bounds or one - /// byte past the end of the same allocated object. Note that in Rust, - /// every (stack-allocated) variable is considered a separate allocated object. + /// byte past the end of the same [allocated object]. /// /// * The computed offset cannot exceed `isize::MAX` **bytes**. /// @@ -482,6 +1091,7 @@ impl *mut T { /// enables more aggressive compiler optimizations. /// /// [`wrapping_sub`]: #method.wrapping_sub + /// [allocated object]: crate::ptr#allocated-object /// /// # Examples /// @@ -497,12 +1107,37 @@ impl *mut T { /// } /// ``` #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] - pub unsafe fn sub(self, count: usize) -> Self + #[must_use = "returns a new pointer rather than modifying its argument"] + #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn sub(self, count: usize) -> Self where T: Sized, { - self.offset((count as isize).wrapping_neg()) + // SAFETY: the caller must uphold the safety contract for `offset`. + unsafe { self.offset((count as isize).wrapping_neg()) } + } + + /// Calculates the offset from a pointer in bytes (convenience for + /// `.byte_offset((count as isize).wrapping_neg())`). + /// + /// `count` is in units of bytes. + /// + /// This is purely a convenience for casting to a `u8` pointer and + /// using [sub][pointer::sub] on it. See that method for documentation + /// and safety requirements. + /// + /// For non-`Sized` pointees this operation changes only the data pointer, + /// leaving the metadata untouched. + #[must_use] + #[inline(always)] + #[unstable(feature = "pointer_byte_offsets", issue = "96283")] + #[rustc_const_unstable(feature = "const_pointer_byte_offsets", issue = "96283")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn byte_sub(self, count: usize) -> Self { + // SAFETY: the caller must uphold the safety contract for `sub`. + unsafe { self.cast::().sub(count).with_metadata_of(self) } } /// Calculates the offset from a pointer using wrapping arithmetic. @@ -513,24 +1148,29 @@ impl *mut T { /// /// # Safety /// - /// The resulting pointer does not need to be in bounds, but it is - /// potentially hazardous to dereference (which requires `unsafe`). + /// This operation itself is always safe, but using the resulting pointer is not. + /// + /// The resulting pointer "remembers" the [allocated object] that `self` points to; it must not + /// be used to read or write other allocated objects. /// - /// In particular, the resulting pointer remains attached to the same allocated - /// object that `self` points to. It may *not* be used to access a - /// different allocated object. Note that in Rust, - /// every (stack-allocated) variable is considered a separate allocated object. + /// In other words, `let z = x.wrapping_add((y as usize) - (x as usize))` does *not* make `z` + /// the same as `y` even if we assume `T` has size `1` and there is no overflow: `z` is still + /// attached to the object `x` is attached to, and dereferencing it is Undefined Behavior unless + /// `x` and `y` point into the same allocated object. /// - /// Compared to [`add`], this method basically delays the requirement of staying - /// within the same allocated object: [`add`] is immediate Undefined Behavior when - /// crossing object boundaries; `wrapping_add` produces a pointer but still leads - /// to Undefined Behavior if that pointer is dereferenced. [`add`] can be optimized - /// better and is thus preferable in performance-sensitive code. + /// Compared to [`add`], this method basically delays the requirement of staying within the + /// same allocated object: [`add`] is immediate Undefined Behavior when crossing object + /// boundaries; `wrapping_add` produces a pointer but still leads to Undefined Behavior if a + /// pointer is dereferenced when it is out-of-bounds of the object it is attached to. [`add`] + /// can be optimized better and is thus preferable in performance-sensitive code. /// - /// If you need to cross object boundaries, cast the pointer to an integer and - /// do the arithmetic there. + /// The delayed check only considers the value of the pointer that was dereferenced, not the + /// intermediate values used during the computation of the final result. For example, + /// `x.wrapping_add(o).wrapping_sub(o)` is always the same as `x`. In other words, leaving the + /// allocated object and then re-entering it later is permitted. /// /// [`add`]: #method.add + /// [allocated object]: crate::ptr#allocated-object /// /// # Examples /// @@ -552,40 +1192,65 @@ impl *mut T { /// } /// ``` #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] - pub fn wrapping_add(self, count: usize) -> Self + #[must_use = "returns a new pointer rather than modifying its argument"] + #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] + #[inline(always)] + pub const fn wrapping_add(self, count: usize) -> Self where T: Sized, { self.wrapping_offset(count as isize) } + /// Calculates the offset from a pointer in bytes using wrapping arithmetic. + /// (convenience for `.wrapping_byte_offset(count as isize)`) + /// + /// `count` is in units of bytes. + /// + /// This is purely a convenience for casting to a `u8` pointer and + /// using [wrapping_add][pointer::wrapping_add] on it. See that method for documentation. + /// + /// For non-`Sized` pointees this operation changes only the data pointer, + /// leaving the metadata untouched. + #[must_use] + #[inline(always)] + #[unstable(feature = "pointer_byte_offsets", issue = "96283")] + #[rustc_const_unstable(feature = "const_pointer_byte_offsets", issue = "96283")] + pub const fn wrapping_byte_add(self, count: usize) -> Self { + self.cast::().wrapping_add(count).with_metadata_of(self) + } + /// Calculates the offset from a pointer using wrapping arithmetic. - /// (convenience for `.wrapping_offset((count as isize).wrapping_sub())`) + /// (convenience for `.wrapping_offset((count as isize).wrapping_neg())`) /// /// `count` is in units of T; e.g., a `count` of 3 represents a pointer /// offset of `3 * size_of::()` bytes. /// /// # Safety /// - /// The resulting pointer does not need to be in bounds, but it is - /// potentially hazardous to dereference (which requires `unsafe`). + /// This operation itself is always safe, but using the resulting pointer is not. + /// + /// The resulting pointer "remembers" the [allocated object] that `self` points to; it must not + /// be used to read or write other allocated objects. /// - /// In particular, the resulting pointer remains attached to the same allocated - /// object that `self` points to. It may *not* be used to access a - /// different allocated object. Note that in Rust, - /// every (stack-allocated) variable is considered a separate allocated object. + /// In other words, `let z = x.wrapping_sub((x as usize) - (y as usize))` does *not* make `z` + /// the same as `y` even if we assume `T` has size `1` and there is no overflow: `z` is still + /// attached to the object `x` is attached to, and dereferencing it is Undefined Behavior unless + /// `x` and `y` point into the same allocated object. /// - /// Compared to [`sub`], this method basically delays the requirement of staying - /// within the same allocated object: [`sub`] is immediate Undefined Behavior when - /// crossing object boundaries; `wrapping_sub` produces a pointer but still leads - /// to Undefined Behavior if that pointer is dereferenced. [`sub`] can be optimized - /// better and is thus preferable in performance-sensitive code. + /// Compared to [`sub`], this method basically delays the requirement of staying within the + /// same allocated object: [`sub`] is immediate Undefined Behavior when crossing object + /// boundaries; `wrapping_sub` produces a pointer but still leads to Undefined Behavior if a + /// pointer is dereferenced when it is out-of-bounds of the object it is attached to. [`sub`] + /// can be optimized better and is thus preferable in performance-sensitive code. /// - /// If you need to cross object boundaries, cast the pointer to an integer and - /// do the arithmetic there. + /// The delayed check only considers the value of the pointer that was dereferenced, not the + /// intermediate values used during the computation of the final result. For example, + /// `x.wrapping_add(o).wrapping_sub(o)` is always the same as `x`. In other words, leaving the + /// allocated object and then re-entering it later is permitted. /// /// [`sub`]: #method.sub + /// [allocated object]: crate::ptr#allocated-object /// /// # Examples /// @@ -607,27 +1272,50 @@ impl *mut T { /// } /// ``` #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] - pub fn wrapping_sub(self, count: usize) -> Self + #[must_use = "returns a new pointer rather than modifying its argument"] + #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] + #[inline(always)] + pub const fn wrapping_sub(self, count: usize) -> Self where T: Sized, { self.wrapping_offset((count as isize).wrapping_neg()) } + /// Calculates the offset from a pointer in bytes using wrapping arithmetic. + /// (convenience for `.wrapping_offset((count as isize).wrapping_neg())`) + /// + /// `count` is in units of bytes. + /// + /// This is purely a convenience for casting to a `u8` pointer and + /// using [wrapping_sub][pointer::wrapping_sub] on it. See that method for documentation. + /// + /// For non-`Sized` pointees this operation changes only the data pointer, + /// leaving the metadata untouched. + #[must_use] + #[inline(always)] + #[unstable(feature = "pointer_byte_offsets", issue = "96283")] + #[rustc_const_unstable(feature = "const_pointer_byte_offsets", issue = "96283")] + pub const fn wrapping_byte_sub(self, count: usize) -> Self { + self.cast::().wrapping_sub(count).with_metadata_of(self) + } + /// Reads the value from `self` without moving it. This leaves the /// memory in `self` unchanged. /// /// See [`ptr::read`] for safety concerns and examples. /// - /// [`ptr::read`]: ./ptr/fn.read.html + /// [`ptr::read`]: crate::ptr::read() #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] - pub unsafe fn read(self) -> T + #[rustc_const_unstable(feature = "const_ptr_read", issue = "80377")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn read(self) -> T where T: Sized, { - read(self) + // SAFETY: the caller must uphold the safety contract for ``. + unsafe { read(self) } } /// Performs a volatile read of the value from `self` without moving it. This @@ -639,14 +1327,16 @@ impl *mut T { /// /// See [`ptr::read_volatile`] for safety concerns and examples. /// - /// [`ptr::read_volatile`]: ./ptr/fn.read_volatile.html + /// [`ptr::read_volatile`]: crate::ptr::read_volatile() #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub unsafe fn read_volatile(self) -> T where T: Sized, { - read_volatile(self) + // SAFETY: the caller must uphold the safety contract for `read_volatile`. + unsafe { read_volatile(self) } } /// Reads the value from `self` without moving it. This leaves the @@ -656,14 +1346,17 @@ impl *mut T { /// /// See [`ptr::read_unaligned`] for safety concerns and examples. /// - /// [`ptr::read_unaligned`]: ./ptr/fn.read_unaligned.html + /// [`ptr::read_unaligned`]: crate::ptr::read_unaligned() #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] - pub unsafe fn read_unaligned(self) -> T + #[rustc_const_unstable(feature = "const_ptr_read", issue = "80377")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn read_unaligned(self) -> T where T: Sized, { - read_unaligned(self) + // SAFETY: the caller must uphold the safety contract for `read_unaligned`. + unsafe { read_unaligned(self) } } /// Copies `count * size_of` bytes from `self` to `dest`. The source @@ -673,14 +1366,17 @@ impl *mut T { /// /// See [`ptr::copy`] for safety concerns and examples. /// - /// [`ptr::copy`]: ./ptr/fn.copy.html + /// [`ptr::copy`]: crate::ptr::copy() + #[rustc_const_stable(feature = "const_intrinsic_copy", since = "1.63.0")] #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] - pub unsafe fn copy_to(self, dest: *mut T, count: usize) + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn copy_to(self, dest: *mut T, count: usize) where T: Sized, { - copy(self, dest, count) + // SAFETY: the caller must uphold the safety contract for `copy`. + unsafe { copy(self, dest, count) } } /// Copies `count * size_of` bytes from `self` to `dest`. The source @@ -690,14 +1386,17 @@ impl *mut T { /// /// See [`ptr::copy_nonoverlapping`] for safety concerns and examples. /// - /// [`ptr::copy_nonoverlapping`]: ./ptr/fn.copy_nonoverlapping.html + /// [`ptr::copy_nonoverlapping`]: crate::ptr::copy_nonoverlapping() + #[rustc_const_stable(feature = "const_intrinsic_copy", since = "1.63.0")] #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] - pub unsafe fn copy_to_nonoverlapping(self, dest: *mut T, count: usize) + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn copy_to_nonoverlapping(self, dest: *mut T, count: usize) where T: Sized, { - copy_nonoverlapping(self, dest, count) + // SAFETY: the caller must uphold the safety contract for `copy_nonoverlapping`. + unsafe { copy_nonoverlapping(self, dest, count) } } /// Copies `count * size_of` bytes from `src` to `self`. The source @@ -707,14 +1406,17 @@ impl *mut T { /// /// See [`ptr::copy`] for safety concerns and examples. /// - /// [`ptr::copy`]: ./ptr/fn.copy.html + /// [`ptr::copy`]: crate::ptr::copy() + #[rustc_const_stable(feature = "const_intrinsic_copy", since = "1.63.0")] #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] - pub unsafe fn copy_from(self, src: *const T, count: usize) + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn copy_from(self, src: *const T, count: usize) where T: Sized, { - copy(src, self, count) + // SAFETY: the caller must uphold the safety contract for `copy`. + unsafe { copy(src, self, count) } } /// Copies `count * size_of` bytes from `src` to `self`. The source @@ -724,25 +1426,29 @@ impl *mut T { /// /// See [`ptr::copy_nonoverlapping`] for safety concerns and examples. /// - /// [`ptr::copy_nonoverlapping`]: ./ptr/fn.copy_nonoverlapping.html + /// [`ptr::copy_nonoverlapping`]: crate::ptr::copy_nonoverlapping() + #[rustc_const_stable(feature = "const_intrinsic_copy", since = "1.63.0")] #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] - pub unsafe fn copy_from_nonoverlapping(self, src: *const T, count: usize) + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn copy_from_nonoverlapping(self, src: *const T, count: usize) where T: Sized, { - copy_nonoverlapping(src, self, count) + // SAFETY: the caller must uphold the safety contract for `copy_nonoverlapping`. + unsafe { copy_nonoverlapping(src, self, count) } } /// Executes the destructor (if any) of the pointed-to value. /// /// See [`ptr::drop_in_place`] for safety concerns and examples. /// - /// [`ptr::drop_in_place`]: ./ptr/fn.drop_in_place.html + /// [`ptr::drop_in_place`]: crate::ptr::drop_in_place() #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] + #[inline(always)] pub unsafe fn drop_in_place(self) { - drop_in_place(self) + // SAFETY: the caller must uphold the safety contract for `drop_in_place`. + unsafe { drop_in_place(self) } } /// Overwrites a memory location with the given value without reading or @@ -750,14 +1456,17 @@ impl *mut T { /// /// See [`ptr::write`] for safety concerns and examples. /// - /// [`ptr::write`]: ./ptr/fn.write.html + /// [`ptr::write`]: crate::ptr::write() #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] - pub unsafe fn write(self, val: T) + #[rustc_const_unstable(feature = "const_ptr_write", issue = "86302")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn write(self, val: T) where T: Sized, { - write(self, val) + // SAFETY: the caller must uphold the safety contract for `write`. + unsafe { write(self, val) } } /// Invokes memset on the specified pointer, setting `count * size_of::()` @@ -765,14 +1474,18 @@ impl *mut T { /// /// See [`ptr::write_bytes`] for safety concerns and examples. /// - /// [`ptr::write_bytes`]: ./ptr/fn.write_bytes.html + /// [`ptr::write_bytes`]: crate::ptr::write_bytes() + #[doc(alias = "memset")] #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] - pub unsafe fn write_bytes(self, val: u8, count: usize) + #[rustc_const_unstable(feature = "const_ptr_write", issue = "86302")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn write_bytes(self, val: u8, count: usize) where T: Sized, { - write_bytes(self, val, count) + // SAFETY: the caller must uphold the safety contract for `write_bytes`. + unsafe { write_bytes(self, val, count) } } /// Performs a volatile write of a memory location with the given value without @@ -784,14 +1497,16 @@ impl *mut T { /// /// See [`ptr::write_volatile`] for safety concerns and examples. /// - /// [`ptr::write_volatile`]: ./ptr/fn.write_volatile.html + /// [`ptr::write_volatile`]: crate::ptr::write_volatile() #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub unsafe fn write_volatile(self, val: T) where T: Sized, { - write_volatile(self, val) + // SAFETY: the caller must uphold the safety contract for `write_volatile`. + unsafe { write_volatile(self, val) } } /// Overwrites a memory location with the given value without reading or @@ -801,14 +1516,17 @@ impl *mut T { /// /// See [`ptr::write_unaligned`] for safety concerns and examples. /// - /// [`ptr::write_unaligned`]: ./ptr/fn.write_unaligned.html + /// [`ptr::write_unaligned`]: crate::ptr::write_unaligned() #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] - pub unsafe fn write_unaligned(self, val: T) + #[rustc_const_unstable(feature = "const_ptr_write", issue = "86302")] + #[inline(always)] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn write_unaligned(self, val: T) where T: Sized, { - write_unaligned(self, val) + // SAFETY: the caller must uphold the safety contract for `write_unaligned`. + unsafe { write_unaligned(self, val) } } /// Replaces the value at `self` with `src`, returning the old @@ -816,14 +1534,15 @@ impl *mut T { /// /// See [`ptr::replace`] for safety concerns and examples. /// - /// [`ptr::replace`]: ./ptr/fn.replace.html + /// [`ptr::replace`]: crate::ptr::replace() #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] + #[inline(always)] pub unsafe fn replace(self, src: T) -> T where T: Sized, { - replace(self, src) + // SAFETY: the caller must uphold the safety contract for `replace`. + unsafe { replace(self, src) } } /// Swaps the values at two mutable locations of the same type, without @@ -832,22 +1551,24 @@ impl *mut T { /// /// See [`ptr::swap`] for safety concerns and examples. /// - /// [`ptr::swap`]: ./ptr/fn.swap.html + /// [`ptr::swap`]: crate::ptr::swap() #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] - pub unsafe fn swap(self, with: *mut T) + #[rustc_const_unstable(feature = "const_swap", issue = "83163")] + #[inline(always)] + pub const unsafe fn swap(self, with: *mut T) where T: Sized, { - swap(self, with) + // SAFETY: the caller must uphold the safety contract for `swap`. + unsafe { swap(self, with) } } /// Computes the offset that needs to be applied to the pointer in order to make it aligned to /// `align`. /// /// If it is not possible to align the pointer, the implementation returns - /// `usize::max_value()`. It is permissible for the implementation to *always* - /// return `usize::max_value()`. Only your algorithm's performance can depend + /// `usize::MAX`. It is permissible for the implementation to *always* + /// return `usize::MAX`. Only your algorithm's performance can depend /// on getting a usable offset here, not its correctness. /// /// The offset is expressed in number of `T` elements, and not bytes. The value returned can be @@ -866,37 +1587,596 @@ impl *mut T { /// Accessing adjacent `u8` as `u16` /// /// ``` - /// # fn foo(n: usize) { - /// # use std::mem::align_of; + /// use std::mem::align_of; + /// /// # unsafe { - /// let x = [5u8, 6u8, 7u8, 8u8, 9u8]; - /// let ptr = &x[n] as *const u8; + /// let mut x = [5_u8, 6, 7, 8, 9]; + /// let ptr = x.as_mut_ptr(); /// let offset = ptr.align_offset(align_of::()); - /// if offset < x.len() - n - 1 { - /// let u16_ptr = ptr.add(offset) as *const u16; - /// assert_ne!(*u16_ptr, 500); + /// + /// if offset < x.len() - 1 { + /// let u16_ptr = ptr.add(offset).cast::(); + /// *u16_ptr = 0; + /// + /// assert!(x == [0, 0, 7, 8, 9] || x == [5, 0, 0, 8, 9]); /// } else { /// // while the pointer can be aligned via `offset`, it would point /// // outside the allocation /// } - /// # } } + /// # } /// ``` + #[must_use] + #[inline] #[stable(feature = "align_offset", since = "1.36.0")] - pub fn align_offset(self, align: usize) -> usize + #[rustc_const_unstable(feature = "const_align_offset", issue = "90962")] + pub const fn align_offset(self, align: usize) -> usize where T: Sized, { if !align.is_power_of_two() { panic!("align_offset: align is not a power-of-two"); } - unsafe { align_offset(self, align) } + + { + // SAFETY: `align` has been checked to be a power of 2 above + unsafe { align_offset(self, align) } + } + } + + /// Returns whether the pointer is properly aligned for `T`. + /// + /// # Examples + /// + /// Basic usage: + /// ``` + /// #![feature(pointer_is_aligned)] + /// #![feature(pointer_byte_offsets)] + /// + /// // On some platforms, the alignment of i32 is less than 4. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// + /// let mut data = AlignedI32(42); + /// let ptr = &mut data as *mut AlignedI32; + /// + /// assert!(ptr.is_aligned()); + /// assert!(!ptr.wrapping_byte_add(1).is_aligned()); + /// ``` + /// + /// # At compiletime + /// **Note: Alignment at compiletime is experimental and subject to change. See the + /// [tracking issue] for details.** + /// + /// At compiletime, the compiler may not know where a value will end up in memory. + /// Calling this function on a pointer created from a reference at compiletime will only + /// return `true` if the pointer is guaranteed to be aligned. This means that the pointer + /// is never aligned if cast to a type with a stricter alignment than the reference's + /// underlying allocation. + /// + /// ``` + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// #![feature(const_mut_refs)] + /// + /// // On some platforms, the alignment of primitives is less than their size. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// #[repr(align(8))] + /// struct AlignedI64(i64); + /// + /// const _: () = { + /// let mut data = AlignedI32(42); + /// let ptr = &mut data as *mut AlignedI32; + /// assert!(ptr.is_aligned()); + /// + /// // At runtime either `ptr1` or `ptr2` would be aligned, but at compiletime neither is aligned. + /// let ptr1 = ptr.cast::(); + /// let ptr2 = ptr.wrapping_add(1).cast::(); + /// assert!(!ptr1.is_aligned()); + /// assert!(!ptr2.is_aligned()); + /// }; + /// ``` + /// + /// Due to this behavior, it is possible that a runtime pointer derived from a compiletime + /// pointer is aligned, even if the compiletime pointer wasn't aligned. + /// + /// ``` + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// // On some platforms, the alignment of primitives is less than their size. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// #[repr(align(8))] + /// struct AlignedI64(i64); + /// + /// // At compiletime, neither `COMPTIME_PTR` nor `COMPTIME_PTR + 1` is aligned. + /// // Also, note that mutable references are not allowed in the final value of constants. + /// const COMPTIME_PTR: *mut AlignedI32 = (&AlignedI32(42) as *const AlignedI32).cast_mut(); + /// const _: () = assert!(!COMPTIME_PTR.cast::().is_aligned()); + /// const _: () = assert!(!COMPTIME_PTR.wrapping_add(1).cast::().is_aligned()); + /// + /// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned. + /// let runtime_ptr = COMPTIME_PTR; + /// assert_ne!( + /// runtime_ptr.cast::().is_aligned(), + /// runtime_ptr.wrapping_add(1).cast::().is_aligned(), + /// ); + /// ``` + /// + /// If a pointer is created from a fixed address, this function behaves the same during + /// runtime and compiletime. + /// + /// ``` + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// // On some platforms, the alignment of primitives is less than their size. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// #[repr(align(8))] + /// struct AlignedI64(i64); + /// + /// const _: () = { + /// let ptr = 40 as *mut AlignedI32; + /// assert!(ptr.is_aligned()); + /// + /// // For pointers with a known address, runtime and compiletime behavior are identical. + /// let ptr1 = ptr.cast::(); + /// let ptr2 = ptr.wrapping_add(1).cast::(); + /// assert!(ptr1.is_aligned()); + /// assert!(!ptr2.is_aligned()); + /// }; + /// ``` + /// + /// [tracking issue]: https://github.com/rust-lang/rust/issues/104203 + #[must_use] + #[inline] + #[unstable(feature = "pointer_is_aligned", issue = "96284")] + #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")] + pub const fn is_aligned(self) -> bool + where + T: Sized, + { + self.is_aligned_to(mem::align_of::()) + } + + /// Returns whether the pointer is aligned to `align`. + /// + /// For non-`Sized` pointees this operation considers only the data pointer, + /// ignoring the metadata. + /// + /// # Panics + /// + /// The function panics if `align` is not a power-of-two (this includes 0). + /// + /// # Examples + /// + /// Basic usage: + /// ``` + /// #![feature(pointer_is_aligned)] + /// #![feature(pointer_byte_offsets)] + /// + /// // On some platforms, the alignment of i32 is less than 4. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// + /// let mut data = AlignedI32(42); + /// let ptr = &mut data as *mut AlignedI32; + /// + /// assert!(ptr.is_aligned_to(1)); + /// assert!(ptr.is_aligned_to(2)); + /// assert!(ptr.is_aligned_to(4)); + /// + /// assert!(ptr.wrapping_byte_add(2).is_aligned_to(2)); + /// assert!(!ptr.wrapping_byte_add(2).is_aligned_to(4)); + /// + /// assert_ne!(ptr.is_aligned_to(8), ptr.wrapping_add(1).is_aligned_to(8)); + /// ``` + /// + /// # At compiletime + /// **Note: Alignment at compiletime is experimental and subject to change. See the + /// [tracking issue] for details.** + /// + /// At compiletime, the compiler may not know where a value will end up in memory. + /// Calling this function on a pointer created from a reference at compiletime will only + /// return `true` if the pointer is guaranteed to be aligned. This means that the pointer + /// cannot be stricter aligned than the reference's underlying allocation. + /// + /// ``` + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// #![feature(const_mut_refs)] + /// + /// // On some platforms, the alignment of i32 is less than 4. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// + /// const _: () = { + /// let mut data = AlignedI32(42); + /// let ptr = &mut data as *mut AlignedI32; + /// + /// assert!(ptr.is_aligned_to(1)); + /// assert!(ptr.is_aligned_to(2)); + /// assert!(ptr.is_aligned_to(4)); + /// + /// // At compiletime, we know for sure that the pointer isn't aligned to 8. + /// assert!(!ptr.is_aligned_to(8)); + /// assert!(!ptr.wrapping_add(1).is_aligned_to(8)); + /// }; + /// ``` + /// + /// Due to this behavior, it is possible that a runtime pointer derived from a compiletime + /// pointer is aligned, even if the compiletime pointer wasn't aligned. + /// + /// ``` + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// // On some platforms, the alignment of i32 is less than 4. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// + /// // At compiletime, neither `COMPTIME_PTR` nor `COMPTIME_PTR + 1` is aligned. + /// // Also, note that mutable references are not allowed in the final value of constants. + /// const COMPTIME_PTR: *mut AlignedI32 = (&AlignedI32(42) as *const AlignedI32).cast_mut(); + /// const _: () = assert!(!COMPTIME_PTR.is_aligned_to(8)); + /// const _: () = assert!(!COMPTIME_PTR.wrapping_add(1).is_aligned_to(8)); + /// + /// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned. + /// let runtime_ptr = COMPTIME_PTR; + /// assert_ne!( + /// runtime_ptr.is_aligned_to(8), + /// runtime_ptr.wrapping_add(1).is_aligned_to(8), + /// ); + /// ``` + /// + /// If a pointer is created from a fixed address, this function behaves the same during + /// runtime and compiletime. + /// + /// ``` + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// const _: () = { + /// let ptr = 40 as *mut u8; + /// assert!(ptr.is_aligned_to(1)); + /// assert!(ptr.is_aligned_to(2)); + /// assert!(ptr.is_aligned_to(4)); + /// assert!(ptr.is_aligned_to(8)); + /// assert!(!ptr.is_aligned_to(16)); + /// }; + /// ``` + /// + /// [tracking issue]: https://github.com/rust-lang/rust/issues/104203 + #[must_use] + #[inline] + #[unstable(feature = "pointer_is_aligned", issue = "96284")] + #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")] + pub const fn is_aligned_to(self, align: usize) -> bool { + if !align.is_power_of_two() { + panic!("is_aligned_to: align is not a power-of-two"); + } + + #[inline] + fn runtime_impl(ptr: *mut (), align: usize) -> bool { + ptr.addr() & (align - 1) == 0 + } + + #[inline] + const fn const_impl(ptr: *mut (), align: usize) -> bool { + // We can't use the address of `self` in a `const fn`, so we use `align_offset` instead. + // The cast to `()` is used to + // 1. deal with fat pointers; and + // 2. ensure that `align_offset` doesn't actually try to compute an offset. + ptr.align_offset(align) == 0 + } + + // SAFETY: The two versions are equivalent at runtime. + unsafe { const_eval_select((self.cast::<()>(), align), const_impl, runtime_impl) } + } +} + +impl *mut [T] { + /// Returns the length of a raw slice. + /// + /// The returned value is the number of **elements**, not the number of bytes. + /// + /// This function is safe, even when the raw slice cannot be cast to a slice + /// reference because the pointer is null or unaligned. + /// + /// # Examples + /// + /// ```rust + /// #![feature(slice_ptr_len)] + /// use std::ptr; + /// + /// let slice: *mut [i8] = ptr::slice_from_raw_parts_mut(ptr::null_mut(), 3); + /// assert_eq!(slice.len(), 3); + /// ``` + #[inline(always)] + #[unstable(feature = "slice_ptr_len", issue = "71146")] + #[rustc_const_unstable(feature = "const_slice_ptr_len", issue = "71146")] + pub const fn len(self) -> usize { + metadata(self) + } + + /// Returns `true` if the raw slice has a length of 0. + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_ptr_len)] + /// + /// let mut a = [1, 2, 3]; + /// let ptr = &mut a as *mut [_]; + /// assert!(!ptr.is_empty()); + /// ``` + #[inline(always)] + #[unstable(feature = "slice_ptr_len", issue = "71146")] + #[rustc_const_unstable(feature = "const_slice_ptr_len", issue = "71146")] + pub const fn is_empty(self) -> bool { + self.len() == 0 + } + + /// Divides one mutable raw slice into two at an index. + /// + /// The first will contain all indices from `[0, mid)` (excluding + /// the index `mid` itself) and the second will contain all + /// indices from `[mid, len)` (excluding the index `len` itself). + /// + /// # Panics + /// + /// Panics if `mid > len`. + /// + /// # Safety + /// + /// `mid` must be [in-bounds] of the underlying [allocated object]. + /// Which means `self` must be dereferenceable and span a single allocation + /// that is at least `mid * size_of::()` bytes long. Not upholding these + /// requirements is *[undefined behavior]* even if the resulting pointers are not used. + /// + /// Since `len` being in-bounds it is not a safety invariant of `*mut [T]` the + /// safety requirements of this method are the same as for [`split_at_mut_unchecked`]. + /// The explicit bounds check is only as useful as `len` is correct. + /// + /// [`split_at_mut_unchecked`]: #method.split_at_mut_unchecked + /// [in-bounds]: #method.add + /// [allocated object]: crate::ptr#allocated-object + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html + /// + /// # Examples + /// + /// ``` + /// #![feature(raw_slice_split)] + /// #![feature(slice_ptr_get)] + /// + /// let mut v = [1, 0, 3, 0, 5, 6]; + /// let ptr = &mut v as *mut [_]; + /// unsafe { + /// let (left, right) = ptr.split_at_mut(2); + /// assert_eq!(&*left, [1, 0]); + /// assert_eq!(&*right, [3, 0, 5, 6]); + /// } + /// ``` + #[inline(always)] + #[track_caller] + #[unstable(feature = "raw_slice_split", issue = "95595")] + pub unsafe fn split_at_mut(self, mid: usize) -> (*mut [T], *mut [T]) { + assert!(mid <= self.len()); + // SAFETY: The assert above is only a safety-net as long as `self.len()` is correct + // The actual safety requirements of this function are the same as for `split_at_mut_unchecked` + unsafe { self.split_at_mut_unchecked(mid) } + } + + /// Divides one mutable raw slice into two at an index, without doing bounds checking. + /// + /// The first will contain all indices from `[0, mid)` (excluding + /// the index `mid` itself) and the second will contain all + /// indices from `[mid, len)` (excluding the index `len` itself). + /// + /// # Safety + /// + /// `mid` must be [in-bounds] of the underlying [allocated object]. + /// Which means `self` must be dereferenceable and span a single allocation + /// that is at least `mid * size_of::()` bytes long. Not upholding these + /// requirements is *[undefined behavior]* even if the resulting pointers are not used. + /// + /// [in-bounds]: #method.add + /// [out-of-bounds index]: #method.add + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html + /// + /// # Examples + /// + /// ``` + /// #![feature(raw_slice_split)] + /// + /// let mut v = [1, 0, 3, 0, 5, 6]; + /// // scoped to restrict the lifetime of the borrows + /// unsafe { + /// let ptr = &mut v as *mut [_]; + /// let (left, right) = ptr.split_at_mut_unchecked(2); + /// assert_eq!(&*left, [1, 0]); + /// assert_eq!(&*right, [3, 0, 5, 6]); + /// (&mut *left)[1] = 2; + /// (&mut *right)[1] = 4; + /// } + /// assert_eq!(v, [1, 2, 3, 4, 5, 6]); + /// ``` + #[inline(always)] + #[unstable(feature = "raw_slice_split", issue = "95595")] + pub unsafe fn split_at_mut_unchecked(self, mid: usize) -> (*mut [T], *mut [T]) { + let len = self.len(); + let ptr = self.as_mut_ptr(); + + // SAFETY: Caller must pass a valid pointer and an index that is in-bounds. + let tail = unsafe { ptr.add(mid) }; + ( + crate::ptr::slice_from_raw_parts_mut(ptr, mid), + crate::ptr::slice_from_raw_parts_mut(tail, len - mid), + ) + } + + /// Returns a raw pointer to the slice's buffer. + /// + /// This is equivalent to casting `self` to `*mut T`, but more type-safe. + /// + /// # Examples + /// + /// ```rust + /// #![feature(slice_ptr_get)] + /// use std::ptr; + /// + /// let slice: *mut [i8] = ptr::slice_from_raw_parts_mut(ptr::null_mut(), 3); + /// assert_eq!(slice.as_mut_ptr(), ptr::null_mut()); + /// ``` + #[inline(always)] + #[unstable(feature = "slice_ptr_get", issue = "74265")] + #[rustc_const_unstable(feature = "slice_ptr_get", issue = "74265")] + pub const fn as_mut_ptr(self) -> *mut T { + self as *mut T + } + + /// Returns a raw pointer to an element or subslice, without doing bounds + /// checking. + /// + /// Calling this method with an [out-of-bounds index] or when `self` is not dereferenceable + /// is *[undefined behavior]* even if the resulting pointer is not used. + /// + /// [out-of-bounds index]: #method.add + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_ptr_get)] + /// + /// let x = &mut [1, 2, 4] as *mut [i32]; + /// + /// unsafe { + /// assert_eq!(x.get_unchecked_mut(1), x.as_mut_ptr().add(1)); + /// } + /// ``` + #[unstable(feature = "slice_ptr_get", issue = "74265")] + #[rustc_const_unstable(feature = "const_slice_index", issue = "none")] + #[inline(always)] + pub const unsafe fn get_unchecked_mut(self, index: I) -> *mut I::Output + where + I: ~const SliceIndex<[T]>, + { + // SAFETY: the caller ensures that `self` is dereferenceable and `index` in-bounds. + unsafe { index.get_unchecked_mut(self) } + } + + /// Returns `None` if the pointer is null, or else returns a shared slice to + /// the value wrapped in `Some`. In contrast to [`as_ref`], this does not require + /// that the value has to be initialized. + /// + /// For the mutable counterpart see [`as_uninit_slice_mut`]. + /// + /// [`as_ref`]: #method.as_ref-1 + /// [`as_uninit_slice_mut`]: #method.as_uninit_slice_mut + /// + /// # Safety + /// + /// When calling this method, you have to ensure that *either* the pointer is null *or* + /// all of the following is true: + /// + /// * The pointer must be [valid] for reads for `ptr.len() * mem::size_of::()` many bytes, + /// and it must be properly aligned. This means in particular: + /// + /// * The entire memory range of this slice must be contained within a single [allocated object]! + /// Slices can never span across multiple allocated objects. + /// + /// * The pointer must be aligned even for zero-length slices. One + /// reason for this is that enum layout optimizations may rely on references + /// (including slices of any length) being aligned and non-null to distinguish + /// them from other data. You can obtain a pointer that is usable as `data` + /// for zero-length slices using [`NonNull::dangling()`]. + /// + /// * The total size `ptr.len() * mem::size_of::()` of the slice must be no larger than `isize::MAX`. + /// See the safety documentation of [`pointer::offset`]. + /// + /// * You must enforce Rust's aliasing rules, since the returned lifetime `'a` is + /// arbitrarily chosen and does not necessarily reflect the actual lifetime of the data. + /// In particular, while this reference exists, the memory the pointer points to must + /// not get mutated (except inside `UnsafeCell`). + /// + /// This applies even if the result of this method is unused! + /// + /// See also [`slice::from_raw_parts`][]. + /// + /// [valid]: crate::ptr#safety + /// [allocated object]: crate::ptr#allocated-object + #[inline] + #[unstable(feature = "ptr_as_uninit", issue = "75402")] + #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")] + pub const unsafe fn as_uninit_slice<'a>(self) -> Option<&'a [MaybeUninit]> { + if self.is_null() { + None + } else { + // SAFETY: the caller must uphold the safety contract for `as_uninit_slice`. + Some(unsafe { slice::from_raw_parts(self as *const MaybeUninit, self.len()) }) + } + } + + /// Returns `None` if the pointer is null, or else returns a unique slice to + /// the value wrapped in `Some`. In contrast to [`as_mut`], this does not require + /// that the value has to be initialized. + /// + /// For the shared counterpart see [`as_uninit_slice`]. + /// + /// [`as_mut`]: #method.as_mut + /// [`as_uninit_slice`]: #method.as_uninit_slice-1 + /// + /// # Safety + /// + /// When calling this method, you have to ensure that *either* the pointer is null *or* + /// all of the following is true: + /// + /// * The pointer must be [valid] for reads and writes for `ptr.len() * mem::size_of::()` + /// many bytes, and it must be properly aligned. This means in particular: + /// + /// * The entire memory range of this slice must be contained within a single [allocated object]! + /// Slices can never span across multiple allocated objects. + /// + /// * The pointer must be aligned even for zero-length slices. One + /// reason for this is that enum layout optimizations may rely on references + /// (including slices of any length) being aligned and non-null to distinguish + /// them from other data. You can obtain a pointer that is usable as `data` + /// for zero-length slices using [`NonNull::dangling()`]. + /// + /// * The total size `ptr.len() * mem::size_of::()` of the slice must be no larger than `isize::MAX`. + /// See the safety documentation of [`pointer::offset`]. + /// + /// * You must enforce Rust's aliasing rules, since the returned lifetime `'a` is + /// arbitrarily chosen and does not necessarily reflect the actual lifetime of the data. + /// In particular, while this reference exists, the memory the pointer points to must + /// not get accessed (read or written) through any other pointer. + /// + /// This applies even if the result of this method is unused! + /// + /// See also [`slice::from_raw_parts_mut`][]. + /// + /// [valid]: crate::ptr#safety + /// [allocated object]: crate::ptr#allocated-object + #[inline] + #[unstable(feature = "ptr_as_uninit", issue = "75402")] + #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")] + pub const unsafe fn as_uninit_slice_mut<'a>(self) -> Option<&'a mut [MaybeUninit]> { + if self.is_null() { + None + } else { + // SAFETY: the caller must uphold the safety contract for `as_uninit_slice_mut`. + Some(unsafe { slice::from_raw_parts_mut(self as *mut MaybeUninit, self.len()) }) + } } } // Equality for pointers #[stable(feature = "rust1", since = "1.0.0")] impl PartialEq for *mut T { - #[inline] + #[inline(always)] fn eq(&self, other: &*mut T) -> bool { *self == *other } @@ -921,27 +2201,27 @@ impl Ord for *mut T { #[stable(feature = "rust1", since = "1.0.0")] impl PartialOrd for *mut T { - #[inline] + #[inline(always)] fn partial_cmp(&self, other: &*mut T) -> Option { Some(self.cmp(other)) } - #[inline] + #[inline(always)] fn lt(&self, other: &*mut T) -> bool { *self < *other } - #[inline] + #[inline(always)] fn le(&self, other: &*mut T) -> bool { *self <= *other } - #[inline] + #[inline(always)] fn gt(&self, other: &*mut T) -> bool { *self > *other } - #[inline] + #[inline(always)] fn ge(&self, other: &*mut T) -> bool { *self >= *other } diff --git a/crux-mir/lib/core/src/ptr/non_null.rs b/crux-mir/lib/core/src/ptr/non_null.rs index 626e58d49..8c1a64886 100644 --- a/crux-mir/lib/core/src/ptr/non_null.rs +++ b/crux-mir/lib/core/src/ptr/non_null.rs @@ -2,14 +2,15 @@ use crate::cmp::Ordering; use crate::convert::From; use crate::fmt; use crate::hash; +use crate::intrinsics::assert_unsafe_precondition; use crate::marker::Unsize; -use crate::mem; +use crate::mem::{self, MaybeUninit}; +use crate::num::NonZeroUsize; use crate::ops::{CoerceUnsized, DispatchFromDyn}; use crate::ptr::Unique; +use crate::slice::{self, SliceIndex}; -// ignore-tidy-undocumented-unsafe - -/// `*mut T` but non-zero and covariant. +/// `*mut T` but non-zero and [covariant]. /// /// This is often the correct thing to use when building data structures using /// raw pointers, but is ultimately more dangerous to use because of its additional @@ -20,12 +21,19 @@ use crate::ptr::Unique; /// as a discriminant -- `Option>` has the same size as `*mut T`. /// However the pointer may still dangle if it isn't dereferenced. /// -/// Unlike `*mut T`, `NonNull` is covariant over `T`. If this is incorrect -/// for your use case, you should include some [`PhantomData`] in your type to -/// provide invariance, such as `PhantomData>` or `PhantomData<&'a mut T>`. -/// Usually this won't be necessary; covariance is correct for most safe abstractions, -/// such as `Box`, `Rc`, `Arc`, `Vec`, and `LinkedList`. This is the case because they -/// provide a public API that follows the normal shared XOR mutable rules of Rust. +/// Unlike `*mut T`, `NonNull` was chosen to be covariant over `T`. This makes it +/// possible to use `NonNull` when building covariant types, but introduces the +/// risk of unsoundness if used in a type that shouldn't actually be covariant. +/// (The opposite choice was made for `*mut T` even though technically the unsoundness +/// could only be caused by calling unsafe functions.) +/// +/// Covariance is correct for most safe abstractions, such as `Box`, `Rc`, `Arc`, `Vec`, +/// and `LinkedList`. This is the case because they provide a public API that follows the +/// normal shared XOR mutable rules of Rust. +/// +/// If your type cannot safely be covariant, you must ensure it contains some +/// additional field to provide invariance. Often this field will be a [`PhantomData`] +/// type like `PhantomData>` or `PhantomData<&'a mut T>`. /// /// Notice that `NonNull` has a `From` instance for `&T`. However, this does /// not change the fact that mutating through a (pointer derived from a) shared @@ -35,8 +43,9 @@ use crate::ptr::Unique; /// it is your responsibility to ensure that `as_mut` is never called, and `as_ptr` /// is never used for mutation. /// -/// [`PhantomData`]: ../marker/struct.PhantomData.html -/// [`UnsafeCell`]: ../cell/struct.UnsafeCell.html +/// [covariant]: https://doc.rust-lang.org/reference/subtyping.html +/// [`PhantomData`]: crate::marker::PhantomData +/// [`UnsafeCell`]: crate::cell::UnsafeCell #[stable(feature = "nonnull", since = "1.25.0")] #[repr(transparent)] #[rustc_layout_scalar_valid_range_start(1)] @@ -65,15 +74,97 @@ impl NonNull { /// a `T`, which means this must not be used as a "not yet initialized" /// sentinel value. Types that lazily allocate must track initialization by /// some other means. + /// + /// # Examples + /// + /// ``` + /// use std::ptr::NonNull; + /// + /// let ptr = NonNull::::dangling(); + /// // Important: don't try to access the value of `ptr` without + /// // initializing it first! The pointer is not null but isn't valid either! + /// ``` #[stable(feature = "nonnull", since = "1.25.0")] - #[rustc_const_stable(feature = "const_nonnull_dangling", since = "1.32.0")] + #[rustc_const_stable(feature = "const_nonnull_dangling", since = "1.36.0")] + #[must_use] #[inline] pub const fn dangling() -> Self { + // SAFETY: mem::align_of() returns a non-zero usize which is then casted + // to a *mut T. Therefore, `ptr` is not null and the conditions for + // calling new_unchecked() are respected. unsafe { - let ptr = mem::align_of::() as *mut T; + let ptr = crate::ptr::invalid_mut::(mem::align_of::()); NonNull::new_unchecked(ptr) } } + + /// Returns a shared references to the value. In contrast to [`as_ref`], this does not require + /// that the value has to be initialized. + /// + /// For the mutable counterpart see [`as_uninit_mut`]. + /// + /// [`as_ref`]: NonNull::as_ref + /// [`as_uninit_mut`]: NonNull::as_uninit_mut + /// + /// # Safety + /// + /// When calling this method, you have to ensure that all of the following is true: + /// + /// * The pointer must be properly aligned. + /// + /// * It must be "dereferenceable" in the sense defined in [the module documentation]. + /// + /// * You must enforce Rust's aliasing rules, since the returned lifetime `'a` is + /// arbitrarily chosen and does not necessarily reflect the actual lifetime of the data. + /// In particular, while this reference exists, the memory the pointer points to must + /// not get mutated (except inside `UnsafeCell`). + /// + /// This applies even if the result of this method is unused! + /// + /// [the module documentation]: crate::ptr#safety + #[inline] + #[must_use] + #[unstable(feature = "ptr_as_uninit", issue = "75402")] + #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")] + pub const unsafe fn as_uninit_ref<'a>(self) -> &'a MaybeUninit { + // SAFETY: the caller must guarantee that `self` meets all the + // requirements for a reference. + unsafe { &*self.cast().as_ptr() } + } + + /// Returns a unique references to the value. In contrast to [`as_mut`], this does not require + /// that the value has to be initialized. + /// + /// For the shared counterpart see [`as_uninit_ref`]. + /// + /// [`as_mut`]: NonNull::as_mut + /// [`as_uninit_ref`]: NonNull::as_uninit_ref + /// + /// # Safety + /// + /// When calling this method, you have to ensure that all of the following is true: + /// + /// * The pointer must be properly aligned. + /// + /// * It must be "dereferenceable" in the sense defined in [the module documentation]. + /// + /// * You must enforce Rust's aliasing rules, since the returned lifetime `'a` is + /// arbitrarily chosen and does not necessarily reflect the actual lifetime of the data. + /// In particular, while this reference exists, the memory the pointer points to must + /// not get accessed (read or written) through any other pointer. + /// + /// This applies even if the result of this method is unused! + /// + /// [the module documentation]: crate::ptr#safety + #[inline] + #[must_use] + #[unstable(feature = "ptr_as_uninit", issue = "75402")] + #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")] + pub const unsafe fn as_uninit_mut<'a>(self) -> &'a mut MaybeUninit { + // SAFETY: the caller must guarantee that `self` meets all the + // requirements for a reference. + unsafe { &mut *self.cast().as_ptr() } + } } impl NonNull { @@ -82,62 +173,528 @@ impl NonNull { /// # Safety /// /// `ptr` must be non-null. + /// + /// # Examples + /// + /// ``` + /// use std::ptr::NonNull; + /// + /// let mut x = 0u32; + /// let ptr = unsafe { NonNull::new_unchecked(&mut x as *mut _) }; + /// ``` + /// + /// *Incorrect* usage of this function: + /// + /// ```rust,no_run + /// use std::ptr::NonNull; + /// + /// // NEVER DO THAT!!! This is undefined behavior. ⚠️ + /// let ptr = unsafe { NonNull::::new_unchecked(std::ptr::null_mut()) }; + /// ``` #[stable(feature = "nonnull", since = "1.25.0")] - #[rustc_const_stable(feature = "const_nonnull_new_unchecked", since = "1.32.0")] + #[rustc_const_stable(feature = "const_nonnull_new_unchecked", since = "1.25.0")] #[inline] pub const unsafe fn new_unchecked(ptr: *mut T) -> Self { - NonNull { pointer: ptr as _ } + // SAFETY: the caller must guarantee that `ptr` is non-null. + unsafe { + assert_unsafe_precondition!("NonNull::new_unchecked requires that the pointer is non-null", [T: ?Sized](ptr: *mut T) => !ptr.is_null()); + NonNull { pointer: ptr as _ } + } } /// Creates a new `NonNull` if `ptr` is non-null. + /// + /// # Examples + /// + /// ``` + /// use std::ptr::NonNull; + /// + /// let mut x = 0u32; + /// let ptr = NonNull::::new(&mut x as *mut _).expect("ptr is null!"); + /// + /// if let Some(ptr) = NonNull::::new(std::ptr::null_mut()) { + /// unreachable!(); + /// } + /// ``` #[stable(feature = "nonnull", since = "1.25.0")] + #[rustc_const_unstable(feature = "const_nonnull_new", issue = "93235")] #[inline] - pub fn new(ptr: *mut T) -> Option { - if !ptr.is_null() { Some(unsafe { Self::new_unchecked(ptr) }) } else { None } + pub const fn new(ptr: *mut T) -> Option { + if !ptr.is_null() { + // SAFETY: The pointer is already checked and is not null + Some(unsafe { Self::new_unchecked(ptr) }) + } else { + None + } + } + + /// Performs the same functionality as [`std::ptr::from_raw_parts`], except that a + /// `NonNull` pointer is returned, as opposed to a raw `*const` pointer. + /// + /// See the documentation of [`std::ptr::from_raw_parts`] for more details. + /// + /// [`std::ptr::from_raw_parts`]: crate::ptr::from_raw_parts + #[unstable(feature = "ptr_metadata", issue = "81513")] + #[rustc_const_unstable(feature = "ptr_metadata", issue = "81513")] + #[inline] + pub const fn from_raw_parts( + data_address: NonNull<()>, + metadata: ::Metadata, + ) -> NonNull { + // SAFETY: The result of `ptr::from::raw_parts_mut` is non-null because `data_address` is. + unsafe { + NonNull::new_unchecked(super::from_raw_parts_mut(data_address.as_ptr(), metadata)) + } + } + + /// Decompose a (possibly wide) pointer into its address and metadata components. + /// + /// The pointer can be later reconstructed with [`NonNull::from_raw_parts`]. + #[unstable(feature = "ptr_metadata", issue = "81513")] + #[rustc_const_unstable(feature = "ptr_metadata", issue = "81513")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn to_raw_parts(self) -> (NonNull<()>, ::Metadata) { + (self.cast(), super::metadata(self.as_ptr())) + } + + /// Gets the "address" portion of the pointer. + /// + /// For more details see the equivalent method on a raw pointer, [`pointer::addr`]. + /// + /// This API and its claimed semantics are part of the Strict Provenance experiment, + /// see the [`ptr` module documentation][crate::ptr]. + #[must_use] + #[inline] + #[unstable(feature = "strict_provenance", issue = "95228")] + pub fn addr(self) -> NonZeroUsize { + // SAFETY: The pointer is guaranteed by the type to be non-null, + // meaning that the address will be non-zero. + unsafe { NonZeroUsize::new_unchecked(self.pointer.addr()) } + } + + /// Creates a new pointer with the given address. + /// + /// For more details see the equivalent method on a raw pointer, [`pointer::with_addr`]. + /// + /// This API and its claimed semantics are part of the Strict Provenance experiment, + /// see the [`ptr` module documentation][crate::ptr]. + #[must_use] + #[inline] + #[unstable(feature = "strict_provenance", issue = "95228")] + pub fn with_addr(self, addr: NonZeroUsize) -> Self { + // SAFETY: The result of `ptr::from::with_addr` is non-null because `addr` is guaranteed to be non-zero. + unsafe { NonNull::new_unchecked(self.pointer.with_addr(addr.get()) as *mut _) } + } + + /// Creates a new pointer by mapping `self`'s address to a new one. + /// + /// For more details see the equivalent method on a raw pointer, [`pointer::map_addr`]. + /// + /// This API and its claimed semantics are part of the Strict Provenance experiment, + /// see the [`ptr` module documentation][crate::ptr]. + #[must_use] + #[inline] + #[unstable(feature = "strict_provenance", issue = "95228")] + pub fn map_addr(self, f: impl FnOnce(NonZeroUsize) -> NonZeroUsize) -> Self { + self.with_addr(f(self.addr())) } /// Acquires the underlying `*mut` pointer. + /// + /// # Examples + /// + /// ``` + /// use std::ptr::NonNull; + /// + /// let mut x = 0u32; + /// let ptr = NonNull::new(&mut x).expect("ptr is null!"); + /// + /// let x_value = unsafe { *ptr.as_ptr() }; + /// assert_eq!(x_value, 0); + /// + /// unsafe { *ptr.as_ptr() += 2; } + /// let x_value = unsafe { *ptr.as_ptr() }; + /// assert_eq!(x_value, 2); + /// ``` #[stable(feature = "nonnull", since = "1.25.0")] #[rustc_const_stable(feature = "const_nonnull_as_ptr", since = "1.32.0")] - #[inline] + #[must_use] + #[inline(always)] pub const fn as_ptr(self) -> *mut T { self.pointer as *mut T } - /// Dereferences the content. + /// Returns a shared reference to the value. If the value may be uninitialized, [`as_uninit_ref`] + /// must be used instead. + /// + /// For the mutable counterpart see [`as_mut`]. + /// + /// [`as_uninit_ref`]: NonNull::as_uninit_ref + /// [`as_mut`]: NonNull::as_mut + /// + /// # Safety + /// + /// When calling this method, you have to ensure that all of the following is true: + /// + /// * The pointer must be properly aligned. + /// + /// * It must be "dereferenceable" in the sense defined in [the module documentation]. /// - /// The resulting lifetime is bound to self so this behaves "as if" - /// it were actually an instance of T that is getting borrowed. If a longer - /// (unbound) lifetime is needed, use `&*my_ptr.as_ptr()`. + /// * The pointer must point to an initialized instance of `T`. + /// + /// * You must enforce Rust's aliasing rules, since the returned lifetime `'a` is + /// arbitrarily chosen and does not necessarily reflect the actual lifetime of the data. + /// In particular, while this reference exists, the memory the pointer points to must + /// not get mutated (except inside `UnsafeCell`). + /// + /// This applies even if the result of this method is unused! + /// (The part about being initialized is not yet fully decided, but until + /// it is, the only safe approach is to ensure that they are indeed initialized.) + /// + /// # Examples + /// + /// ``` + /// use std::ptr::NonNull; + /// + /// let mut x = 0u32; + /// let ptr = NonNull::new(&mut x as *mut _).expect("ptr is null!"); + /// + /// let ref_x = unsafe { ptr.as_ref() }; + /// println!("{ref_x}"); + /// ``` + /// + /// [the module documentation]: crate::ptr#safety #[stable(feature = "nonnull", since = "1.25.0")] - #[inline] - pub unsafe fn as_ref(&self) -> &T { - &*self.as_ptr() + #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")] + #[must_use] + #[inline(always)] + pub const unsafe fn as_ref<'a>(&self) -> &'a T { + // SAFETY: the caller must guarantee that `self` meets all the + // requirements for a reference. + unsafe { &*self.as_ptr() } } - /// Mutably dereferences the content. + /// Returns a unique reference to the value. If the value may be uninitialized, [`as_uninit_mut`] + /// must be used instead. + /// + /// For the shared counterpart see [`as_ref`]. /// - /// The resulting lifetime is bound to self so this behaves "as if" - /// it were actually an instance of T that is getting borrowed. If a longer - /// (unbound) lifetime is needed, use `&mut *my_ptr.as_ptr()`. + /// [`as_uninit_mut`]: NonNull::as_uninit_mut + /// [`as_ref`]: NonNull::as_ref + /// + /// # Safety + /// + /// When calling this method, you have to ensure that all of the following is true: + /// + /// * The pointer must be properly aligned. + /// + /// * It must be "dereferenceable" in the sense defined in [the module documentation]. + /// + /// * The pointer must point to an initialized instance of `T`. + /// + /// * You must enforce Rust's aliasing rules, since the returned lifetime `'a` is + /// arbitrarily chosen and does not necessarily reflect the actual lifetime of the data. + /// In particular, while this reference exists, the memory the pointer points to must + /// not get accessed (read or written) through any other pointer. + /// + /// This applies even if the result of this method is unused! + /// (The part about being initialized is not yet fully decided, but until + /// it is, the only safe approach is to ensure that they are indeed initialized.) + /// # Examples + /// + /// ``` + /// use std::ptr::NonNull; + /// + /// let mut x = 0u32; + /// let mut ptr = NonNull::new(&mut x).expect("null pointer"); + /// + /// let x_ref = unsafe { ptr.as_mut() }; + /// assert_eq!(*x_ref, 0); + /// *x_ref += 2; + /// assert_eq!(*x_ref, 2); + /// ``` + /// + /// [the module documentation]: crate::ptr#safety #[stable(feature = "nonnull", since = "1.25.0")] - #[inline] - pub unsafe fn as_mut(&mut self) -> &mut T { - &mut *self.as_ptr() + #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")] + #[must_use] + #[inline(always)] + pub const unsafe fn as_mut<'a>(&mut self) -> &'a mut T { + // SAFETY: the caller must guarantee that `self` meets all the + // requirements for a mutable reference. + unsafe { &mut *self.as_ptr() } } /// Casts to a pointer of another type. + /// + /// # Examples + /// + /// ``` + /// use std::ptr::NonNull; + /// + /// let mut x = 0u32; + /// let ptr = NonNull::new(&mut x as *mut _).expect("null pointer"); + /// + /// let casted_ptr = ptr.cast::(); + /// let raw_ptr: *mut i8 = casted_ptr.as_ptr(); + /// ``` #[stable(feature = "nonnull_cast", since = "1.27.0")] - #[rustc_const_stable(feature = "const_nonnull_cast", since = "1.32.0")] + #[rustc_const_stable(feature = "const_nonnull_cast", since = "1.36.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[inline] pub const fn cast(self) -> NonNull { + // SAFETY: `self` is a `NonNull` pointer which is necessarily non-null unsafe { NonNull::new_unchecked(self.as_ptr() as *mut U) } } } -#[stable(feature = "nonnull", since = "1.25.0")] -impl Clone for NonNull { +impl NonNull<[T]> { + /// Creates a non-null raw slice from a thin pointer and a length. + /// + /// The `len` argument is the number of **elements**, not the number of bytes. + /// + /// This function is safe, but dereferencing the return value is unsafe. + /// See the documentation of [`slice::from_raw_parts`] for slice safety requirements. + /// + /// # Examples + /// + /// ```rust + /// #![feature(nonnull_slice_from_raw_parts)] + /// + /// use std::ptr::NonNull; + /// + /// // create a slice pointer when starting out with a pointer to the first element + /// let mut x = [5, 6, 7]; + /// let nonnull_pointer = NonNull::new(x.as_mut_ptr()).unwrap(); + /// let slice = NonNull::slice_from_raw_parts(nonnull_pointer, 3); + /// assert_eq!(unsafe { slice.as_ref()[2] }, 7); + /// ``` + /// + /// (Note that this example artificially demonstrates a use of this method, + /// but `let slice = NonNull::from(&x[..]);` would be a better way to write code like this.) + #[unstable(feature = "nonnull_slice_from_raw_parts", issue = "71941")] + #[rustc_const_unstable(feature = "const_nonnull_slice_from_raw_parts", issue = "71941")] + #[must_use] + #[inline] + pub const fn slice_from_raw_parts(data: NonNull, len: usize) -> Self { + // SAFETY: `data` is a `NonNull` pointer which is necessarily non-null + unsafe { Self::new_unchecked(super::slice_from_raw_parts_mut(data.as_ptr(), len)) } + } + + /// Returns the length of a non-null raw slice. + /// + /// The returned value is the number of **elements**, not the number of bytes. + /// + /// This function is safe, even when the non-null raw slice cannot be dereferenced to a slice + /// because the pointer does not have a valid address. + /// + /// # Examples + /// + /// ```rust + /// #![feature(nonnull_slice_from_raw_parts)] + /// use std::ptr::NonNull; + /// + /// let slice: NonNull<[i8]> = NonNull::slice_from_raw_parts(NonNull::dangling(), 3); + /// assert_eq!(slice.len(), 3); + /// ``` + #[stable(feature = "slice_ptr_len_nonnull", since = "1.63.0")] + #[rustc_const_stable(feature = "const_slice_ptr_len_nonnull", since = "1.63.0")] + #[rustc_allow_const_fn_unstable(const_slice_ptr_len)] + #[must_use] + #[inline] + pub const fn len(self) -> usize { + self.as_ptr().len() + } + + /// Returns a non-null pointer to the slice's buffer. + /// + /// # Examples + /// + /// ```rust + /// #![feature(slice_ptr_get, nonnull_slice_from_raw_parts)] + /// use std::ptr::NonNull; + /// + /// let slice: NonNull<[i8]> = NonNull::slice_from_raw_parts(NonNull::dangling(), 3); + /// assert_eq!(slice.as_non_null_ptr(), NonNull::::dangling()); + /// ``` + #[inline] + #[must_use] + #[unstable(feature = "slice_ptr_get", issue = "74265")] + #[rustc_const_unstable(feature = "slice_ptr_get", issue = "74265")] + pub const fn as_non_null_ptr(self) -> NonNull { + // SAFETY: We know `self` is non-null. + unsafe { NonNull::new_unchecked(self.as_ptr().as_mut_ptr()) } + } + + /// Returns a raw pointer to the slice's buffer. + /// + /// # Examples + /// + /// ```rust + /// #![feature(slice_ptr_get, nonnull_slice_from_raw_parts)] + /// use std::ptr::NonNull; + /// + /// let slice: NonNull<[i8]> = NonNull::slice_from_raw_parts(NonNull::dangling(), 3); + /// assert_eq!(slice.as_mut_ptr(), NonNull::::dangling().as_ptr()); + /// ``` + #[inline] + #[must_use] + #[unstable(feature = "slice_ptr_get", issue = "74265")] + #[rustc_const_unstable(feature = "slice_ptr_get", issue = "74265")] + pub const fn as_mut_ptr(self) -> *mut T { + self.as_non_null_ptr().as_ptr() + } + + /// Returns a shared reference to a slice of possibly uninitialized values. In contrast to + /// [`as_ref`], this does not require that the value has to be initialized. + /// + /// For the mutable counterpart see [`as_uninit_slice_mut`]. + /// + /// [`as_ref`]: NonNull::as_ref + /// [`as_uninit_slice_mut`]: NonNull::as_uninit_slice_mut + /// + /// # Safety + /// + /// When calling this method, you have to ensure that all of the following is true: + /// + /// * The pointer must be [valid] for reads for `ptr.len() * mem::size_of::()` many bytes, + /// and it must be properly aligned. This means in particular: + /// + /// * The entire memory range of this slice must be contained within a single allocated object! + /// Slices can never span across multiple allocated objects. + /// + /// * The pointer must be aligned even for zero-length slices. One + /// reason for this is that enum layout optimizations may rely on references + /// (including slices of any length) being aligned and non-null to distinguish + /// them from other data. You can obtain a pointer that is usable as `data` + /// for zero-length slices using [`NonNull::dangling()`]. + /// + /// * The total size `ptr.len() * mem::size_of::()` of the slice must be no larger than `isize::MAX`. + /// See the safety documentation of [`pointer::offset`]. + /// + /// * You must enforce Rust's aliasing rules, since the returned lifetime `'a` is + /// arbitrarily chosen and does not necessarily reflect the actual lifetime of the data. + /// In particular, while this reference exists, the memory the pointer points to must + /// not get mutated (except inside `UnsafeCell`). + /// + /// This applies even if the result of this method is unused! + /// + /// See also [`slice::from_raw_parts`]. + /// + /// [valid]: crate::ptr#safety #[inline] + #[must_use] + #[unstable(feature = "ptr_as_uninit", issue = "75402")] + #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")] + pub const unsafe fn as_uninit_slice<'a>(self) -> &'a [MaybeUninit] { + // SAFETY: the caller must uphold the safety contract for `as_uninit_slice`. + unsafe { slice::from_raw_parts(self.cast().as_ptr(), self.len()) } + } + + /// Returns a unique reference to a slice of possibly uninitialized values. In contrast to + /// [`as_mut`], this does not require that the value has to be initialized. + /// + /// For the shared counterpart see [`as_uninit_slice`]. + /// + /// [`as_mut`]: NonNull::as_mut + /// [`as_uninit_slice`]: NonNull::as_uninit_slice + /// + /// # Safety + /// + /// When calling this method, you have to ensure that all of the following is true: + /// + /// * The pointer must be [valid] for reads and writes for `ptr.len() * mem::size_of::()` + /// many bytes, and it must be properly aligned. This means in particular: + /// + /// * The entire memory range of this slice must be contained within a single allocated object! + /// Slices can never span across multiple allocated objects. + /// + /// * The pointer must be aligned even for zero-length slices. One + /// reason for this is that enum layout optimizations may rely on references + /// (including slices of any length) being aligned and non-null to distinguish + /// them from other data. You can obtain a pointer that is usable as `data` + /// for zero-length slices using [`NonNull::dangling()`]. + /// + /// * The total size `ptr.len() * mem::size_of::()` of the slice must be no larger than `isize::MAX`. + /// See the safety documentation of [`pointer::offset`]. + /// + /// * You must enforce Rust's aliasing rules, since the returned lifetime `'a` is + /// arbitrarily chosen and does not necessarily reflect the actual lifetime of the data. + /// In particular, while this reference exists, the memory the pointer points to must + /// not get accessed (read or written) through any other pointer. + /// + /// This applies even if the result of this method is unused! + /// + /// See also [`slice::from_raw_parts_mut`]. + /// + /// [valid]: crate::ptr#safety + /// + /// # Examples + /// + /// ```rust + /// #![feature(allocator_api, ptr_as_uninit)] + /// + /// use std::alloc::{Allocator, Layout, Global}; + /// use std::mem::MaybeUninit; + /// use std::ptr::NonNull; + /// + /// let memory: NonNull<[u8]> = Global.allocate(Layout::new::<[u8; 32]>())?; + /// // This is safe as `memory` is valid for reads and writes for `memory.len()` many bytes. + /// // Note that calling `memory.as_mut()` is not allowed here as the content may be uninitialized. + /// # #[allow(unused_variables)] + /// let slice: &mut [MaybeUninit] = unsafe { memory.as_uninit_slice_mut() }; + /// # Ok::<_, std::alloc::AllocError>(()) + /// ``` + #[inline] + #[must_use] + #[unstable(feature = "ptr_as_uninit", issue = "75402")] + #[rustc_const_unstable(feature = "const_ptr_as_ref", issue = "91822")] + pub const unsafe fn as_uninit_slice_mut<'a>(self) -> &'a mut [MaybeUninit] { + // SAFETY: the caller must uphold the safety contract for `as_uninit_slice_mut`. + unsafe { slice::from_raw_parts_mut(self.cast().as_ptr(), self.len()) } + } + + /// Returns a raw pointer to an element or subslice, without doing bounds + /// checking. + /// + /// Calling this method with an out-of-bounds index or when `self` is not dereferenceable + /// is *[undefined behavior]* even if the resulting pointer is not used. + /// + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_ptr_get, nonnull_slice_from_raw_parts)] + /// use std::ptr::NonNull; + /// + /// let x = &mut [1, 2, 4]; + /// let x = NonNull::slice_from_raw_parts(NonNull::new(x.as_mut_ptr()).unwrap(), x.len()); + /// + /// unsafe { + /// assert_eq!(x.get_unchecked_mut(1).as_ptr(), x.as_non_null_ptr().as_ptr().add(1)); + /// } + /// ``` + #[unstable(feature = "slice_ptr_get", issue = "74265")] + #[rustc_const_unstable(feature = "const_slice_index", issue = "none")] + #[inline] + pub const unsafe fn get_unchecked_mut(self, index: I) -> NonNull + where + I: ~const SliceIndex<[T]>, + { + // SAFETY: the caller ensures that `self` is dereferenceable and `index` in-bounds. + // As a consequence, the resulting pointer cannot be null. + unsafe { NonNull::new_unchecked(self.as_ptr().get_unchecked_mut(index)) } + } +} + +#[stable(feature = "nonnull", since = "1.25.0")] +#[rustc_const_unstable(feature = "const_clone", issue = "91805")] +impl const Clone for NonNull { + #[inline(always)] fn clone(&self) -> Self { *self } @@ -146,7 +703,7 @@ impl Clone for NonNull { #[stable(feature = "nonnull", since = "1.25.0")] impl Copy for NonNull {} -#[unstable(feature = "coerce_unsized", issue = "27732")] +#[unstable(feature = "coerce_unsized", issue = "18598")] impl CoerceUnsized> for NonNull where T: Unsize {} #[unstable(feature = "dispatch_from_dyn", issue = "none")] @@ -202,25 +759,39 @@ impl hash::Hash for NonNull { } #[unstable(feature = "ptr_internals", issue = "none")] -impl From> for NonNull { +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const From> for NonNull { #[inline] fn from(unique: Unique) -> Self { + // SAFETY: A Unique pointer cannot be null, so the conditions for + // new_unchecked() are respected. unsafe { NonNull::new_unchecked(unique.as_ptr()) } } } #[stable(feature = "nonnull", since = "1.25.0")] -impl From<&mut T> for NonNull { +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const From<&mut T> for NonNull { + /// Converts a `&mut T` to a `NonNull`. + /// + /// This conversion is safe and infallible since references cannot be null. #[inline] fn from(reference: &mut T) -> Self { + // SAFETY: A mutable reference cannot be null. unsafe { NonNull { pointer: reference as *mut T } } } } #[stable(feature = "nonnull", since = "1.25.0")] -impl From<&T> for NonNull { +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const From<&T> for NonNull { + /// Converts a `&T` to a `NonNull`. + /// + /// This conversion is safe and infallible since references cannot be null. #[inline] fn from(reference: &T) -> Self { + // SAFETY: A reference cannot be null, so the conditions for + // new_unchecked() are respected. unsafe { NonNull { pointer: reference as *const T } } } } diff --git a/crux-mir/lib/core/src/ptr/unique.rs b/crux-mir/lib/core/src/ptr/unique.rs index 87b56d951..64616142b 100644 --- a/crux-mir/lib/core/src/ptr/unique.rs +++ b/crux-mir/lib/core/src/ptr/unique.rs @@ -1,12 +1,9 @@ use crate::convert::From; use crate::fmt; use crate::marker::{PhantomData, Unsize}; -use crate::mem; use crate::ops::{CoerceUnsized, DispatchFromDyn}; use crate::ptr::NonNull; -// ignore-tidy-undocumented-unsafe - /// A wrapper around a raw non-null `*mut T` that indicates that the possessor /// of this wrapper owns the referent. Useful for building abstractions like /// `Box`, `Vec`, `String`, and `HashMap`. @@ -35,9 +32,8 @@ use crate::ptr::NonNull; )] #[doc(hidden)] #[repr(transparent)] -#[rustc_layout_scalar_valid_range_start(1)] pub struct Unique { - pointer: *const T, + pointer: NonNull, // NOTE: this marker has no consequences for variance, but is necessary // for dropck to understand that we logically own a `T`. // @@ -71,10 +67,10 @@ impl Unique { /// a `T`, which means this must not be used as a "not yet initialized" /// sentinel value. Types that lazily allocate must track initialization by /// some other means. - // FIXME: rename to dangling() to match NonNull? + #[must_use] #[inline] - pub const fn empty() -> Self { - unsafe { Unique::new_unchecked(mem::align_of::() as *mut T) } + pub const fn dangling() -> Self { + Self::from(NonNull::dangling()) } } @@ -87,23 +83,25 @@ impl Unique { /// `ptr` must be non-null. #[inline] pub const unsafe fn new_unchecked(ptr: *mut T) -> Self { - Unique { pointer: ptr as _, _marker: PhantomData } + // SAFETY: the caller must guarantee that `ptr` is non-null. + unsafe { Unique { pointer: NonNull::new_unchecked(ptr), _marker: PhantomData } } } /// Creates a new `Unique` if `ptr` is non-null. #[inline] - pub fn new(ptr: *mut T) -> Option { - if !ptr.is_null() { - Some(unsafe { Unique { pointer: ptr as _, _marker: PhantomData } }) + pub const fn new(ptr: *mut T) -> Option { + if let Some(pointer) = NonNull::new(ptr) { + Some(Unique { pointer, _marker: PhantomData }) } else { None } } /// Acquires the underlying `*mut` pointer. + #[must_use = "`self` will be dropped if the result is not used"] #[inline] pub const fn as_ptr(self) -> *mut T { - self.pointer as *mut T + self.pointer.as_ptr() } /// Dereferences the content. @@ -111,9 +109,12 @@ impl Unique { /// The resulting lifetime is bound to self so this behaves "as if" /// it were actually an instance of T that is getting borrowed. If a longer /// (unbound) lifetime is needed, use `&*my_ptr.as_ptr()`. + #[must_use] #[inline] - pub unsafe fn as_ref(&self) -> &T { - &*self.as_ptr() + pub const unsafe fn as_ref(&self) -> &T { + // SAFETY: the caller must guarantee that `self` meets all the + // requirements for a reference. + unsafe { self.pointer.as_ref() } } /// Mutably dereferences the content. @@ -121,20 +122,25 @@ impl Unique { /// The resulting lifetime is bound to self so this behaves "as if" /// it were actually an instance of T that is getting borrowed. If a longer /// (unbound) lifetime is needed, use `&mut *my_ptr.as_ptr()`. + #[must_use] #[inline] - pub unsafe fn as_mut(&mut self) -> &mut T { - &mut *self.as_ptr() + pub const unsafe fn as_mut(&mut self) -> &mut T { + // SAFETY: the caller must guarantee that `self` meets all the + // requirements for a mutable reference. + unsafe { self.pointer.as_mut() } } /// Casts to a pointer of another type. + #[must_use = "`self` will be dropped if the result is not used"] #[inline] pub const fn cast(self) -> Unique { - unsafe { Unique::new_unchecked(self.as_ptr() as *mut U) } + Unique::from(self.pointer.cast()) } } #[unstable(feature = "ptr_internals", issue = "none")] -impl Clone for Unique { +#[rustc_const_unstable(feature = "const_clone", issue = "91805")] +impl const Clone for Unique { #[inline] fn clone(&self) -> Self { *self @@ -165,25 +171,23 @@ impl fmt::Pointer for Unique { } #[unstable(feature = "ptr_internals", issue = "none")] -impl From<&mut T> for Unique { +impl const From<&mut T> for Unique { + /// Converts a `&mut T` to a `Unique`. + /// + /// This conversion is infallible since references cannot be null. #[inline] fn from(reference: &mut T) -> Self { - unsafe { Unique { pointer: reference as *mut T, _marker: PhantomData } } - } -} - -#[unstable(feature = "ptr_internals", issue = "none")] -impl From<&T> for Unique { - #[inline] - fn from(reference: &T) -> Self { - unsafe { Unique { pointer: reference as *const T, _marker: PhantomData } } + Self::from(NonNull::from(reference)) } } #[unstable(feature = "ptr_internals", issue = "none")] -impl From> for Unique { +impl const From> for Unique { + /// Converts a `NonNull` to a `Unique`. + /// + /// This conversion is infallible since `NonNull` cannot be null. #[inline] - fn from(p: NonNull) -> Self { - unsafe { Unique::new_unchecked(p.as_ptr()) } + fn from(pointer: NonNull) -> Self { + Unique { pointer, _marker: PhantomData } } } diff --git a/crux-mir/lib/core/src/raw.rs b/crux-mir/lib/core/src/raw.rs deleted file mode 100644 index 75c329a7d..000000000 --- a/crux-mir/lib/core/src/raw.rs +++ /dev/null @@ -1,85 +0,0 @@ -#![allow(missing_docs)] -#![unstable(feature = "raw", issue = "27751")] - -//! Contains struct definitions for the layout of compiler built-in types. -//! -//! They can be used as targets of transmutes in unsafe code for manipulating -//! the raw representations directly. -//! -//! Their definition should always match the ABI defined in `rustc::back::abi`. - -/// The representation of a trait object like `&SomeTrait`. -/// -/// This struct has the same layout as types like `&SomeTrait` and -/// `Box`. -/// -/// `TraitObject` is guaranteed to match layouts, but it is not the -/// type of trait objects (e.g., the fields are not directly accessible -/// on a `&SomeTrait`) nor does it control that layout (changing the -/// definition will not change the layout of a `&SomeTrait`). It is -/// only designed to be used by unsafe code that needs to manipulate -/// the low-level details. -/// -/// There is no way to refer to all trait objects generically, so the only -/// way to create values of this type is with functions like -/// [`std::mem::transmute`][transmute]. Similarly, the only way to create a true -/// trait object from a `TraitObject` value is with `transmute`. -/// -/// [transmute]: ../intrinsics/fn.transmute.html -/// -/// Synthesizing a trait object with mismatched types—one where the -/// vtable does not correspond to the type of the value to which the -/// data pointer points—is highly likely to lead to undefined -/// behavior. -/// -/// # Examples -/// -/// ``` -/// #![feature(raw)] -/// -/// use std::{mem, raw}; -/// -/// // an example trait -/// trait Foo { -/// fn bar(&self) -> i32; -/// } -/// -/// impl Foo for i32 { -/// fn bar(&self) -> i32 { -/// *self + 1 -/// } -/// } -/// -/// let value: i32 = 123; -/// -/// // let the compiler make a trait object -/// let object: &dyn Foo = &value; -/// -/// // look at the raw representation -/// let raw_object: raw::TraitObject = unsafe { mem::transmute(object) }; -/// -/// // the data pointer is the address of `value` -/// assert_eq!(raw_object.data as *const i32, &value as *const _); -/// -/// let other_value: i32 = 456; -/// -/// // construct a new object, pointing to a different `i32`, being -/// // careful to use the `i32` vtable from `object` -/// let synthesized: &dyn Foo = unsafe { -/// mem::transmute(raw::TraitObject { -/// data: &other_value as *const _ as *mut (), -/// vtable: raw_object.vtable, -/// }) -/// }; -/// -/// // it should work just as if we had constructed a trait object out of -/// // `other_value` directly -/// assert_eq!(synthesized.bar(), 457); -/// ``` -#[repr(C)] -#[derive(Copy, Clone)] -#[allow(missing_debug_implementations)] -pub struct TraitObject { - pub data: *mut (), - pub vtable: *mut (), -} diff --git a/crux-mir/lib/core/src/result.rs b/crux-mir/lib/core/src/result.rs index 0bc29e1bc..f00c40f35 100644 --- a/crux-mir/lib/core/src/result.rs +++ b/crux-mir/lib/core/src/result.rs @@ -35,8 +35,8 @@ //! //! let version = parse_version(&[1, 2, 3, 4]); //! match version { -//! Ok(v) => println!("working with version: {:?}", v), -//! Err(e) => println!("error parsing header: {:?}", e), +//! Ok(v) => println!("working with version: {v:?}"), +//! Err(e) => println!("error parsing header: {e:?}"), //! } //! ``` //! @@ -88,7 +88,7 @@ //! ``` //! //! *Note: The actual definition of [`Write`] uses [`io::Result`], which -//! is just a synonym for [`Result`]``.* +//! is just a synonym for [Result].* //! //! This method doesn't produce a value, but the write may //! fail. It's crucial to handle the error case, and *not* write @@ -112,7 +112,7 @@ //! assert success with [`expect`]. This will panic if the //! write fails, providing a marginally useful message indicating why: //! -//! ```{.no_run} +//! ```no_run //! use std::fs::File; //! use std::io::prelude::*; //! @@ -122,7 +122,7 @@ //! //! You might also simply assert success: //! -//! ```{.no_run} +//! ```no_run //! # use std::fs::File; //! # use std::io::prelude::*; //! # let mut file = File::create("valuable_data.txt").unwrap(); @@ -209,47 +209,305 @@ //! //! *It's much nicer!* //! -//! Ending the expression with [`?`] will result in the unwrapped -//! success ([`Ok`]) value, unless the result is [`Err`], in which case -//! [`Err`] is returned early from the enclosing function. +//! Ending the expression with [`?`] will result in the [`Ok`]'s unwrapped value, unless the result +//! is [`Err`], in which case [`Err`] is returned early from the enclosing function. //! -//! [`?`] can only be used in functions that return [`Result`] because of the +//! [`?`] can be used in functions that return [`Result`] because of the //! early return of [`Err`] that it provides. //! -//! [`expect`]: enum.Result.html#method.expect -//! [`Write`]: ../../std/io/trait.Write.html -//! [`write_all`]: ../../std/io/trait.Write.html#method.write_all -//! [`io::Result`]: ../../std/io/type.Result.html -//! [`?`]: ../../std/macro.try.html -//! [`Result`]: enum.Result.html -//! [`Ok(T)`]: enum.Result.html#variant.Ok -//! [`Err(E)`]: enum.Result.html#variant.Err -//! [`io::Error`]: ../../std/io/struct.Error.html -//! [`Ok`]: enum.Result.html#variant.Ok -//! [`Err`]: enum.Result.html#variant.Err +//! [`expect`]: Result::expect +//! [`Write`]: ../../std/io/trait.Write.html "io::Write" +//! [`write_all`]: ../../std/io/trait.Write.html#method.write_all "io::Write::write_all" +//! [`io::Result`]: ../../std/io/type.Result.html "io::Result" +//! [`?`]: crate::ops::Try +//! [`Ok(T)`]: Ok +//! [`Err(E)`]: Err +//! [io::Error]: ../../std/io/struct.Error.html "io::Error" +//! +//! # Method overview +//! +//! In addition to working with pattern matching, [`Result`] provides a +//! wide variety of different methods. +//! +//! ## Querying the variant +//! +//! The [`is_ok`] and [`is_err`] methods return [`true`] if the [`Result`] +//! is [`Ok`] or [`Err`], respectively. +//! +//! [`is_err`]: Result::is_err +//! [`is_ok`]: Result::is_ok +//! +//! ## Adapters for working with references +//! +//! * [`as_ref`] converts from `&Result` to `Result<&T, &E>` +//! * [`as_mut`] converts from `&mut Result` to `Result<&mut T, &mut E>` +//! * [`as_deref`] converts from `&Result` to `Result<&T::Target, &E>` +//! * [`as_deref_mut`] converts from `&mut Result` to +//! `Result<&mut T::Target, &mut E>` +//! +//! [`as_deref`]: Result::as_deref +//! [`as_deref_mut`]: Result::as_deref_mut +//! [`as_mut`]: Result::as_mut +//! [`as_ref`]: Result::as_ref +//! +//! ## Extracting contained values +//! +//! These methods extract the contained value in a [`Result`] when it +//! is the [`Ok`] variant. If the [`Result`] is [`Err`]: +//! +//! * [`expect`] panics with a provided custom message +//! * [`unwrap`] panics with a generic message +//! * [`unwrap_or`] returns the provided default value +//! * [`unwrap_or_default`] returns the default value of the type `T` +//! (which must implement the [`Default`] trait) +//! * [`unwrap_or_else`] returns the result of evaluating the provided +//! function +//! +//! The panicking methods [`expect`] and [`unwrap`] require `E` to +//! implement the [`Debug`] trait. +//! +//! [`Debug`]: crate::fmt::Debug +//! [`expect`]: Result::expect +//! [`unwrap`]: Result::unwrap +//! [`unwrap_or`]: Result::unwrap_or +//! [`unwrap_or_default`]: Result::unwrap_or_default +//! [`unwrap_or_else`]: Result::unwrap_or_else +//! +//! These methods extract the contained value in a [`Result`] when it +//! is the [`Err`] variant. They require `T` to implement the [`Debug`] +//! trait. If the [`Result`] is [`Ok`]: +//! +//! * [`expect_err`] panics with a provided custom message +//! * [`unwrap_err`] panics with a generic message +//! +//! [`Debug`]: crate::fmt::Debug +//! [`expect_err`]: Result::expect_err +//! [`unwrap_err`]: Result::unwrap_err +//! +//! ## Transforming contained values +//! +//! These methods transform [`Result`] to [`Option`]: +//! +//! * [`err`][Result::err] transforms [`Result`] into [`Option`], +//! mapping [`Err(e)`] to [`Some(e)`] and [`Ok(v)`] to [`None`] +//! * [`ok`][Result::ok] transforms [`Result`] into [`Option`], +//! mapping [`Ok(v)`] to [`Some(v)`] and [`Err(e)`] to [`None`] +//! * [`transpose`] transposes a [`Result`] of an [`Option`] into an +//! [`Option`] of a [`Result`] +//! +// Do NOT add link reference definitions for `err` or `ok`, because they +// will generate numerous incorrect URLs for `Err` and `Ok` elsewhere, due +// to case folding. +//! +//! [`Err(e)`]: Err +//! [`Ok(v)`]: Ok +//! [`Some(e)`]: Option::Some +//! [`Some(v)`]: Option::Some +//! [`transpose`]: Result::transpose +//! +//! This method transforms the contained value of the [`Ok`] variant: +//! +//! * [`map`] transforms [`Result`] into [`Result`] by applying +//! the provided function to the contained value of [`Ok`] and leaving +//! [`Err`] values unchanged +//! +//! [`map`]: Result::map +//! +//! This method transforms the contained value of the [`Err`] variant: +//! +//! * [`map_err`] transforms [`Result`] into [`Result`] by +//! applying the provided function to the contained value of [`Err`] and +//! leaving [`Ok`] values unchanged +//! +//! [`map_err`]: Result::map_err +//! +//! These methods transform a [`Result`] into a value of a possibly +//! different type `U`: +//! +//! * [`map_or`] applies the provided function to the contained value of +//! [`Ok`], or returns the provided default value if the [`Result`] is +//! [`Err`] +//! * [`map_or_else`] applies the provided function to the contained value +//! of [`Ok`], or applies the provided default fallback function to the +//! contained value of [`Err`] +//! +//! [`map_or`]: Result::map_or +//! [`map_or_else`]: Result::map_or_else +//! +//! ## Boolean operators +//! +//! These methods treat the [`Result`] as a boolean value, where [`Ok`] +//! acts like [`true`] and [`Err`] acts like [`false`]. There are two +//! categories of these methods: ones that take a [`Result`] as input, and +//! ones that take a function as input (to be lazily evaluated). +//! +//! The [`and`] and [`or`] methods take another [`Result`] as input, and +//! produce a [`Result`] as output. The [`and`] method can produce a +//! [`Result`] value having a different inner type `U` than +//! [`Result`]. The [`or`] method can produce a [`Result`] +//! value having a different error type `F` than [`Result`]. +//! +//! | method | self | input | output | +//! |---------|----------|-----------|----------| +//! | [`and`] | `Err(e)` | (ignored) | `Err(e)` | +//! | [`and`] | `Ok(x)` | `Err(d)` | `Err(d)` | +//! | [`and`] | `Ok(x)` | `Ok(y)` | `Ok(y)` | +//! | [`or`] | `Err(e)` | `Err(d)` | `Err(d)` | +//! | [`or`] | `Err(e)` | `Ok(y)` | `Ok(y)` | +//! | [`or`] | `Ok(x)` | (ignored) | `Ok(x)` | +//! +//! [`and`]: Result::and +//! [`or`]: Result::or +//! +//! The [`and_then`] and [`or_else`] methods take a function as input, and +//! only evaluate the function when they need to produce a new value. The +//! [`and_then`] method can produce a [`Result`] value having a +//! different inner type `U` than [`Result`]. The [`or_else`] method +//! can produce a [`Result`] value having a different error type `F` +//! than [`Result`]. +//! +//! | method | self | function input | function result | output | +//! |--------------|----------|----------------|-----------------|----------| +//! | [`and_then`] | `Err(e)` | (not provided) | (not evaluated) | `Err(e)` | +//! | [`and_then`] | `Ok(x)` | `x` | `Err(d)` | `Err(d)` | +//! | [`and_then`] | `Ok(x)` | `x` | `Ok(y)` | `Ok(y)` | +//! | [`or_else`] | `Err(e)` | `e` | `Err(d)` | `Err(d)` | +//! | [`or_else`] | `Err(e)` | `e` | `Ok(y)` | `Ok(y)` | +//! | [`or_else`] | `Ok(x)` | (not provided) | (not evaluated) | `Ok(x)` | +//! +//! [`and_then`]: Result::and_then +//! [`or_else`]: Result::or_else +//! +//! ## Comparison operators +//! +//! If `T` and `E` both implement [`PartialOrd`] then [`Result`] will +//! derive its [`PartialOrd`] implementation. With this order, an [`Ok`] +//! compares as less than any [`Err`], while two [`Ok`] or two [`Err`] +//! compare as their contained values would in `T` or `E` respectively. If `T` +//! and `E` both also implement [`Ord`], then so does [`Result`]. +//! +//! ``` +//! assert!(Ok(1) < Err(0)); +//! let x: Result = Ok(0); +//! let y = Ok(1); +//! assert!(x < y); +//! let x: Result<(), i32> = Err(0); +//! let y = Err(1); +//! assert!(x < y); +//! ``` +//! +//! ## Iterating over `Result` +//! +//! A [`Result`] can be iterated over. This can be helpful if you need an +//! iterator that is conditionally empty. The iterator will either produce +//! a single value (when the [`Result`] is [`Ok`]), or produce no values +//! (when the [`Result`] is [`Err`]). For example, [`into_iter`] acts like +//! [`once(v)`] if the [`Result`] is [`Ok(v)`], and like [`empty()`] if the +//! [`Result`] is [`Err`]. +//! +//! [`Ok(v)`]: Ok +//! [`empty()`]: crate::iter::empty +//! [`once(v)`]: crate::iter::once +//! +//! Iterators over [`Result`] come in three types: +//! +//! * [`into_iter`] consumes the [`Result`] and produces the contained +//! value +//! * [`iter`] produces an immutable reference of type `&T` to the +//! contained value +//! * [`iter_mut`] produces a mutable reference of type `&mut T` to the +//! contained value +//! +//! See [Iterating over `Option`] for examples of how this can be useful. +//! +//! [Iterating over `Option`]: crate::option#iterating-over-option +//! [`into_iter`]: Result::into_iter +//! [`iter`]: Result::iter +//! [`iter_mut`]: Result::iter_mut +//! +//! You might want to use an iterator chain to do multiple instances of an +//! operation that can fail, but would like to ignore failures while +//! continuing to process the successful results. In this example, we take +//! advantage of the iterable nature of [`Result`] to select only the +//! [`Ok`] values using [`flatten`][Iterator::flatten]. +//! +//! ``` +//! # use std::str::FromStr; +//! let mut results = vec![]; +//! let mut errs = vec![]; +//! let nums: Vec<_> = ["17", "not a number", "99", "-27", "768"] +//! .into_iter() +//! .map(u8::from_str) +//! // Save clones of the raw `Result` values to inspect +//! .inspect(|x| results.push(x.clone())) +//! // Challenge: explain how this captures only the `Err` values +//! .inspect(|x| errs.extend(x.clone().err())) +//! .flatten() +//! .collect(); +//! assert_eq!(errs.len(), 3); +//! assert_eq!(nums, [17, 99]); +//! println!("results {results:?}"); +//! println!("errs {errs:?}"); +//! println!("nums {nums:?}"); +//! ``` +//! +//! ## Collecting into `Result` +//! +//! [`Result`] implements the [`FromIterator`][impl-FromIterator] trait, +//! which allows an iterator over [`Result`] values to be collected into a +//! [`Result`] of a collection of each contained value of the original +//! [`Result`] values, or [`Err`] if any of the elements was [`Err`]. +//! +//! [impl-FromIterator]: Result#impl-FromIterator%3CResult%3CA%2C%20E%3E%3E-for-Result%3CV%2C%20E%3E +//! +//! ``` +//! let v = [Ok(2), Ok(4), Err("err!"), Ok(8)]; +//! let res: Result, &str> = v.into_iter().collect(); +//! assert_eq!(res, Err("err!")); +//! let v = [Ok(2), Ok(4), Ok(8)]; +//! let res: Result, &str> = v.into_iter().collect(); +//! assert_eq!(res, Ok(vec![2, 4, 8])); +//! ``` +//! +//! [`Result`] also implements the [`Product`][impl-Product] and +//! [`Sum`][impl-Sum] traits, allowing an iterator over [`Result`] values +//! to provide the [`product`][Iterator::product] and +//! [`sum`][Iterator::sum] methods. +//! +//! [impl-Product]: Result#impl-Product%3CResult%3CU%2C%20E%3E%3E-for-Result%3CT%2C%20E%3E +//! [impl-Sum]: Result#impl-Sum%3CResult%3CU%2C%20E%3E%3E-for-Result%3CT%2C%20E%3E +//! +//! ``` +//! let v = [Err("error!"), Ok(1), Ok(2), Ok(3), Err("foo")]; +//! let res: Result = v.into_iter().sum(); +//! assert_eq!(res, Err("error!")); +//! let v = [Ok(1), Ok(2), Ok(21)]; +//! let res: Result = v.into_iter().product(); +//! assert_eq!(res, Ok(42)); +//! ``` #![stable(feature = "rust1", since = "1.0.0")] -use crate::fmt; use crate::iter::{self, FromIterator, FusedIterator, TrustedLen}; -use crate::ops::{self, Deref, DerefMut}; +use crate::marker::Destruct; +use crate::ops::{self, ControlFlow, Deref, DerefMut}; +use crate::{convert, fmt, hint}; /// `Result` is a type that represents either success ([`Ok`]) or failure ([`Err`]). /// -/// See the [`std::result`](index.html) module documentation for details. -/// -/// [`Ok`]: enum.Result.html#variant.Ok -/// [`Err`]: enum.Result.html#variant.Err +/// See the [module documentation](self) for details. #[derive(Copy, PartialEq, PartialOrd, Eq, Ord, Debug, Hash)] #[must_use = "this `Result` may be an `Err` variant, which should be handled"] -#[rustc_diagnostic_item = "result_type"] +#[rustc_diagnostic_item = "Result"] #[stable(feature = "rust1", since = "1.0.0")] pub enum Result { /// Contains the success value + #[lang = "Ok"] #[stable(feature = "rust1", since = "1.0.0")] Ok(#[stable(feature = "rust1", since = "1.0.0")] T), /// Contains the error value + #[lang = "Err"] #[stable(feature = "rust1", since = "1.0.0")] Err(#[stable(feature = "rust1", since = "1.0.0")] E), } @@ -265,8 +523,6 @@ impl Result { /// Returns `true` if the result is [`Ok`]. /// - /// [`Ok`]: enum.Result.html#variant.Ok - /// /// # Examples /// /// Basic usage: @@ -279,16 +535,40 @@ impl Result { /// assert_eq!(x.is_ok(), false); /// ``` #[must_use = "if you intended to assert that this is ok, consider `.unwrap()` instead"] - #[rustc_const_unstable(feature = "const_result", issue = "67520")] + #[rustc_const_stable(feature = "const_result_basics", since = "1.48.0")] #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub const fn is_ok(&self) -> bool { matches!(*self, Ok(_)) } - /// Returns `true` if the result is [`Err`]. + /// Returns `true` if the result is [`Ok`] and the value inside of it matches a predicate. + /// + /// # Examples /// - /// [`Err`]: enum.Result.html#variant.Err + /// ``` + /// #![feature(is_some_and)] + /// + /// let x: Result = Ok(2); + /// assert_eq!(x.is_ok_and(|x| x > 1), true); + /// + /// let x: Result = Ok(0); + /// assert_eq!(x.is_ok_and(|x| x > 1), false); + /// + /// let x: Result = Err("hey"); + /// assert_eq!(x.is_ok_and(|x| x > 1), false); + /// ``` + #[must_use] + #[inline] + #[unstable(feature = "is_some_and", issue = "93050")] + pub fn is_ok_and(self, f: impl FnOnce(T) -> bool) -> bool { + match self { + Err(_) => false, + Ok(x) => f(x), + } + } + + /// Returns `true` if the result is [`Err`]. /// /// # Examples /// @@ -302,68 +582,37 @@ impl Result { /// assert_eq!(x.is_err(), true); /// ``` #[must_use = "if you intended to assert that this is err, consider `.unwrap_err()` instead"] - #[rustc_const_unstable(feature = "const_result", issue = "67520")] + #[rustc_const_stable(feature = "const_result_basics", since = "1.48.0")] #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub const fn is_err(&self) -> bool { !self.is_ok() } - /// Returns `true` if the result is an [`Ok`] value containing the given value. - /// - /// # Examples - /// - /// ``` - /// #![feature(option_result_contains)] - /// - /// let x: Result = Ok(2); - /// assert_eq!(x.contains(&2), true); - /// - /// let x: Result = Ok(3); - /// assert_eq!(x.contains(&2), false); - /// - /// let x: Result = Err("Some error message"); - /// assert_eq!(x.contains(&2), false); - /// ``` - #[must_use] - #[inline] - #[unstable(feature = "option_result_contains", issue = "62358")] - pub fn contains(&self, x: &U) -> bool - where - U: PartialEq, - { - match self { - Ok(y) => x == y, - Err(_) => false, - } - } - - /// Returns `true` if the result is an [`Err`] value containing the given value. + /// Returns `true` if the result is [`Err`] and the value inside of it matches a predicate. /// /// # Examples /// /// ``` - /// #![feature(result_contains_err)] + /// #![feature(is_some_and)] + /// use std::io::{Error, ErrorKind}; /// - /// let x: Result = Ok(2); - /// assert_eq!(x.contains_err(&"Some error message"), false); + /// let x: Result = Err(Error::new(ErrorKind::NotFound, "!")); + /// assert_eq!(x.is_err_and(|x| x.kind() == ErrorKind::NotFound), true); /// - /// let x: Result = Err("Some error message"); - /// assert_eq!(x.contains_err(&"Some error message"), true); + /// let x: Result = Err(Error::new(ErrorKind::PermissionDenied, "!")); + /// assert_eq!(x.is_err_and(|x| x.kind() == ErrorKind::NotFound), false); /// - /// let x: Result = Err("Some other error message"); - /// assert_eq!(x.contains_err(&"Some error message"), false); + /// let x: Result = Ok(123); + /// assert_eq!(x.is_err_and(|x| x.kind() == ErrorKind::NotFound), false); /// ``` #[must_use] #[inline] - #[unstable(feature = "result_contains_err", issue = "62358")] - pub fn contains_err(&self, f: &F) -> bool - where - F: PartialEq, - { + #[unstable(feature = "is_some_and", issue = "93050")] + pub fn is_err_and(self, f: impl FnOnce(E) -> bool) -> bool { match self { Ok(_) => false, - Err(e) => f == e, + Err(e) => f(e), } } @@ -376,8 +625,6 @@ impl Result { /// Converts `self` into an [`Option`], consuming `self`, /// and discarding the error, if any. /// - /// [`Option`]: ../../std/option/enum.Option.html - /// /// # Examples /// /// Basic usage: @@ -391,10 +638,16 @@ impl Result { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - pub fn ok(self) -> Option { + #[rustc_const_unstable(feature = "const_result_drop", issue = "92384")] + pub const fn ok(self) -> Option + where + E: ~const Destruct, + { match self { Ok(x) => Some(x), - Err(_) => None, + // FIXME: ~const Drop doesn't quite work right yet + #[allow(unused_variables)] + Err(x) => None, } } @@ -403,8 +656,6 @@ impl Result { /// Converts `self` into an [`Option`], consuming `self`, /// and discarding the success value, if any. /// - /// [`Option`]: ../../std/option/enum.Option.html - /// /// # Examples /// /// Basic usage: @@ -418,9 +669,15 @@ impl Result { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - pub fn err(self) -> Option { + #[rustc_const_unstable(feature = "const_result_drop", issue = "92384")] + pub const fn err(self) -> Option + where + T: ~const Destruct, + { match self { - Ok(_) => None, + // FIXME: ~const Drop doesn't quite work right yet + #[allow(unused_variables)] + Ok(x) => None, Err(x) => Some(x), } } @@ -446,7 +703,7 @@ impl Result { /// assert_eq!(x.as_ref(), Err(&"Error")); /// ``` #[inline] - #[rustc_const_unstable(feature = "const_result", issue = "67520")] + #[rustc_const_stable(feature = "const_result_basics", since = "1.48.0")] #[stable(feature = "rust1", since = "1.0.0")] pub const fn as_ref(&self) -> Result<&T, &E> { match *self { @@ -479,7 +736,8 @@ impl Result { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - pub fn as_mut(&mut self) -> Result<&mut T, &mut E> { + #[rustc_const_unstable(feature = "const_result", issue = "82814")] + pub const fn as_mut(&mut self) -> Result<&mut T, &mut E> { match *self { Ok(ref mut x) => Ok(x), Err(ref mut x) => Err(x), @@ -495,9 +753,6 @@ impl Result { /// /// This function can be used to compose the results of two functions. /// - /// [`Ok`]: enum.Result.html#variant.Ok - /// [`Err`]: enum.Result.html#variant.Err - /// /// # Examples /// /// Print the numbers on each line of a string multiplied by two. @@ -507,7 +762,7 @@ impl Result { /// /// for num in line.lines() { /// match num.parse::().map(|i| i * 2) { - /// Ok(n) => println!("{}", n), + /// Ok(n) => println!("{n}"), /// Err(..) => {} /// } /// } @@ -521,14 +776,14 @@ impl Result { } } - /// Applies a function to the contained value (if any), - /// or returns the provided default (if not). + /// Returns the provided default (if [`Err`]), or + /// applies a function to the contained value (if [`Ok`]), /// /// Arguments passed to `map_or` are eagerly evaluated; if you are passing /// the result of a function call, it is recommended to use [`map_or_else`], /// which is lazily evaluated. /// - /// [`map_or_else`]: #method.map_or_else + /// [`map_or_else`]: Result::map_or_else /// /// # Examples /// @@ -548,15 +803,12 @@ impl Result { } } - /// Maps a `Result` to `U` by applying a function to a - /// contained [`Ok`] value, or a fallback function to a - /// contained [`Err`] value. + /// Maps a `Result` to `U` by applying fallback function `default` to + /// a contained [`Err`] value, or function `f` to a contained [`Ok`] value. /// /// This function can be used to unpack a successful result /// while handling an error. /// - /// [`Ok`]: enum.Result.html#variant.Ok - /// [`Err`]: enum.Result.html#variant.Err /// /// # Examples /// @@ -586,15 +838,13 @@ impl Result { /// This function can be used to pass through a successful result while handling /// an error. /// - /// [`Ok`]: enum.Result.html#variant.Ok - /// [`Err`]: enum.Result.html#variant.Err /// /// # Examples /// /// Basic usage: /// /// ``` - /// fn stringify(x: u32) -> String { format!("error code: {}", x) } + /// fn stringify(x: u32) -> String { format!("error code: {x}") } /// /// let x: Result = Ok(2); /// assert_eq!(x.map_err(stringify), Ok(2)); @@ -611,6 +861,103 @@ impl Result { } } + /// Calls the provided closure with a reference to the contained value (if [`Ok`]). + /// + /// # Examples + /// + /// ``` + /// #![feature(result_option_inspect)] + /// + /// let x: u8 = "4" + /// .parse::() + /// .inspect(|x| println!("original: {x}")) + /// .map(|x| x.pow(3)) + /// .expect("failed to parse number"); + /// ``` + #[inline] + #[unstable(feature = "result_option_inspect", issue = "91345")] + pub fn inspect(self, f: F) -> Self { + if let Ok(ref t) = self { + f(t); + } + + self + } + + /// Calls the provided closure with a reference to the contained error (if [`Err`]). + /// + /// # Examples + /// + /// ``` + /// #![feature(result_option_inspect)] + /// + /// use std::{fs, io}; + /// + /// fn read() -> io::Result { + /// fs::read_to_string("address.txt") + /// .inspect_err(|e| eprintln!("failed to read file: {e}")) + /// } + /// ``` + #[inline] + #[unstable(feature = "result_option_inspect", issue = "91345")] + pub fn inspect_err(self, f: F) -> Self { + if let Err(ref e) = self { + f(e); + } + + self + } + + /// Converts from `Result` (or `&Result`) to `Result<&::Target, &E>`. + /// + /// Coerces the [`Ok`] variant of the original [`Result`] via [`Deref`](crate::ops::Deref) + /// and returns the new [`Result`]. + /// + /// # Examples + /// + /// ``` + /// let x: Result = Ok("hello".to_string()); + /// let y: Result<&str, &u32> = Ok("hello"); + /// assert_eq!(x.as_deref(), y); + /// + /// let x: Result = Err(42); + /// let y: Result<&str, &u32> = Err(&42); + /// assert_eq!(x.as_deref(), y); + /// ``` + #[stable(feature = "inner_deref", since = "1.47.0")] + pub fn as_deref(&self) -> Result<&T::Target, &E> + where + T: Deref, + { + self.as_ref().map(|t| t.deref()) + } + + /// Converts from `Result` (or `&mut Result`) to `Result<&mut ::Target, &mut E>`. + /// + /// Coerces the [`Ok`] variant of the original [`Result`] via [`DerefMut`](crate::ops::DerefMut) + /// and returns the new [`Result`]. + /// + /// # Examples + /// + /// ``` + /// let mut s = "HELLO".to_string(); + /// let mut x: Result = Ok("hello".to_string()); + /// let y: Result<&mut str, &mut u32> = Ok(&mut s); + /// assert_eq!(x.as_deref_mut().map(|x| { x.make_ascii_uppercase(); x }), y); + /// + /// let mut i = 42; + /// let mut x: Result = Err(42); + /// let y: Result<&mut str, &mut u32> = Err(&mut i); + /// assert_eq!(x.as_deref_mut().map(|x| { x.make_ascii_uppercase(); x }), y); + /// ``` + #[stable(feature = "inner_deref", since = "1.47.0")] + pub fn as_deref_mut(&mut self) -> Result<&mut T::Target, &mut E> + where + T: DerefMut, + { + self.as_mut().map(|t| t.deref_mut()) + } + ///////////////////////////////////////////////////////////////////////// // Iterator constructors ///////////////////////////////////////////////////////////////////////// @@ -661,529 +1008,714 @@ impl Result { IterMut { inner: self.as_mut().ok() } } - //////////////////////////////////////////////////////////////////////// - // Boolean operations on the values, eager and lazy + ///////////////////////////////////////////////////////////////////////// + // Extract a value ///////////////////////////////////////////////////////////////////////// - /// Returns `res` if the result is [`Ok`], otherwise returns the [`Err`] value of `self`. + /// Returns the contained [`Ok`] value, consuming the `self` value. + /// + /// Because this function may panic, its use is generally discouraged. + /// Instead, prefer to use pattern matching and handle the [`Err`] + /// case explicitly, or call [`unwrap_or`], [`unwrap_or_else`], or + /// [`unwrap_or_default`]. + /// + /// [`unwrap_or`]: Result::unwrap_or + /// [`unwrap_or_else`]: Result::unwrap_or_else + /// [`unwrap_or_default`]: Result::unwrap_or_default + /// + /// # Panics + /// + /// Panics if the value is an [`Err`], with a panic message including the + /// passed message, and the content of the [`Err`]. /// - /// [`Ok`]: enum.Result.html#variant.Ok - /// [`Err`]: enum.Result.html#variant.Err /// /// # Examples /// /// Basic usage: /// + /// ```should_panic + /// let x: Result = Err("emergency failure"); + /// x.expect("Testing expect"); // panics with `Testing expect: emergency failure` /// ``` - /// let x: Result = Ok(2); - /// let y: Result<&str, &str> = Err("late error"); - /// assert_eq!(x.and(y), Err("late error")); /// - /// let x: Result = Err("early error"); - /// let y: Result<&str, &str> = Ok("foo"); - /// assert_eq!(x.and(y), Err("early error")); + /// # Recommended Message Style /// - /// let x: Result = Err("not a 2"); - /// let y: Result<&str, &str> = Err("late error"); - /// assert_eq!(x.and(y), Err("not a 2")); + /// We recommend that `expect` messages are used to describe the reason you + /// _expect_ the `Result` should be `Ok`. /// - /// let x: Result = Ok(2); - /// let y: Result<&str, &str> = Ok("different result type"); - /// assert_eq!(x.and(y), Ok("different result type")); + /// ```should_panic + /// let path = std::env::var("IMPORTANT_PATH") + /// .expect("env variable `IMPORTANT_PATH` should be set by `wrapper_script.sh`"); /// ``` + /// + /// **Hint**: If you're having trouble remembering how to phrase expect + /// error messages remember to focus on the word "should" as in "env + /// variable should be set by blah" or "the given binary should be available + /// and executable by the current user". + /// + /// For more detail on expect message styles and the reasoning behind our recommendation please + /// refer to the section on ["Common Message + /// Styles"](../../std/error/index.html#common-message-styles) in the + /// [`std::error`](../../std/error/index.html) module docs. #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn and(self, res: Result) -> Result { + #[track_caller] + #[stable(feature = "result_expect", since = "1.4.0")] + pub fn expect(self, msg: &str) -> T + where + E: fmt::Debug, + { match self { - Ok(_) => res, - Err(e) => Err(e), + Ok(t) => t, + Err(e) => unwrap_failed(msg, &e), } } - /// Calls `op` if the result is [`Ok`], otherwise returns the [`Err`] value of `self`. + /// Returns the contained [`Ok`] value, consuming the `self` value. + /// + /// Because this function may panic, its use is generally discouraged. + /// Instead, prefer to use pattern matching and handle the [`Err`] + /// case explicitly, or call [`unwrap_or`], [`unwrap_or_else`], or + /// [`unwrap_or_default`]. /// - /// [`Ok`]: enum.Result.html#variant.Ok - /// [`Err`]: enum.Result.html#variant.Err + /// [`unwrap_or`]: Result::unwrap_or + /// [`unwrap_or_else`]: Result::unwrap_or_else + /// [`unwrap_or_default`]: Result::unwrap_or_default + /// + /// # Panics + /// + /// Panics if the value is an [`Err`], with a panic message provided by the + /// [`Err`]'s value. /// - /// This function can be used for control flow based on `Result` values. /// /// # Examples /// /// Basic usage: /// /// ``` - /// fn sq(x: u32) -> Result { Ok(x * x) } - /// fn err(x: u32) -> Result { Err(x) } + /// let x: Result = Ok(2); + /// assert_eq!(x.unwrap(), 2); + /// ``` /// - /// assert_eq!(Ok(2).and_then(sq).and_then(sq), Ok(16)); - /// assert_eq!(Ok(2).and_then(sq).and_then(err), Err(4)); - /// assert_eq!(Ok(2).and_then(err).and_then(sq), Err(2)); - /// assert_eq!(Err(3).and_then(sq).and_then(sq), Err(3)); + /// ```should_panic + /// let x: Result = Err("emergency failure"); + /// x.unwrap(); // panics with `emergency failure` /// ``` #[inline] + #[track_caller] #[stable(feature = "rust1", since = "1.0.0")] - pub fn and_then Result>(self, op: F) -> Result { + pub fn unwrap(self) -> T + where + E: fmt::Debug, + { match self { - Ok(t) => op(t), - Err(e) => Err(e), + Ok(t) => t, + Err(e) => unwrap_failed("called `Result::unwrap()` on an `Err` value", &e), } } - /// Returns `res` if the result is [`Err`], otherwise returns the [`Ok`] value of `self`. + /// Returns the contained [`Ok`] value or a default /// - /// Arguments passed to `or` are eagerly evaluated; if you are passing the - /// result of a function call, it is recommended to use [`or_else`], which is - /// lazily evaluated. + /// Consumes the `self` argument then, if [`Ok`], returns the contained + /// value, otherwise if [`Err`], returns the default value for that + /// type. + /// + /// # Examples + /// + /// Converts a string to an integer, turning poorly-formed strings + /// into 0 (the default value for integers). [`parse`] converts + /// a string to any other type that implements [`FromStr`], returning an + /// [`Err`] on error. + /// + /// ``` + /// let good_year_from_input = "1909"; + /// let bad_year_from_input = "190blarg"; + /// let good_year = good_year_from_input.parse().unwrap_or_default(); + /// let bad_year = bad_year_from_input.parse().unwrap_or_default(); + /// + /// assert_eq!(1909, good_year); + /// assert_eq!(0, bad_year); + /// ``` + /// + /// [`parse`]: str::parse + /// [`FromStr`]: crate::str::FromStr + #[inline] + #[stable(feature = "result_unwrap_or_default", since = "1.16.0")] + pub fn unwrap_or_default(self) -> T + where + T: Default, + { + match self { + Ok(x) => x, + Err(_) => Default::default(), + } + } + + /// Returns the contained [`Err`] value, consuming the `self` value. + /// + /// # Panics + /// + /// Panics if the value is an [`Ok`], with a panic message including the + /// passed message, and the content of the [`Ok`]. /// - /// [`Ok`]: enum.Result.html#variant.Ok - /// [`Err`]: enum.Result.html#variant.Err - /// [`or_else`]: #method.or_else /// /// # Examples /// /// Basic usage: /// + /// ```should_panic + /// let x: Result = Ok(10); + /// x.expect_err("Testing expect_err"); // panics with `Testing expect_err: 10` /// ``` - /// let x: Result = Ok(2); - /// let y: Result = Err("late error"); - /// assert_eq!(x.or(y), Ok(2)); + #[inline] + #[track_caller] + #[stable(feature = "result_expect_err", since = "1.17.0")] + pub fn expect_err(self, msg: &str) -> E + where + T: fmt::Debug, + { + match self { + Ok(t) => unwrap_failed(msg, &t), + Err(e) => e, + } + } + + /// Returns the contained [`Err`] value, consuming the `self` value. /// - /// let x: Result = Err("early error"); - /// let y: Result = Ok(2); - /// assert_eq!(x.or(y), Ok(2)); + /// # Panics /// - /// let x: Result = Err("not a 2"); - /// let y: Result = Err("late error"); - /// assert_eq!(x.or(y), Err("late error")); + /// Panics if the value is an [`Ok`], with a custom panic message provided + /// by the [`Ok`]'s value. /// + /// # Examples + /// + /// ```should_panic /// let x: Result = Ok(2); - /// let y: Result = Ok(100); - /// assert_eq!(x.or(y), Ok(2)); + /// x.unwrap_err(); // panics with `2` + /// ``` + /// + /// ``` + /// let x: Result = Err("emergency failure"); + /// assert_eq!(x.unwrap_err(), "emergency failure"); /// ``` #[inline] + #[track_caller] #[stable(feature = "rust1", since = "1.0.0")] - pub fn or(self, res: Result) -> Result { + pub fn unwrap_err(self) -> E + where + T: fmt::Debug, + { match self { - Ok(v) => Ok(v), - Err(_) => res, + Ok(t) => unwrap_failed("called `Result::unwrap_err()` on an `Ok` value", &t), + Err(e) => e, } } - /// Calls `op` if the result is [`Err`], otherwise returns the [`Ok`] value of `self`. + /// Returns the contained [`Ok`] value, but never panics. /// - /// This function can be used for control flow based on result values. + /// Unlike [`unwrap`], this method is known to never panic on the + /// result types it is implemented for. Therefore, it can be used + /// instead of `unwrap` as a maintainability safeguard that will fail + /// to compile if the error type of the `Result` is later changed + /// to an error that can actually occur. /// - /// [`Ok`]: enum.Result.html#variant.Ok - /// [`Err`]: enum.Result.html#variant.Err + /// [`unwrap`]: Result::unwrap /// /// # Examples /// /// Basic usage: /// /// ``` - /// fn sq(x: u32) -> Result { Ok(x * x) } - /// fn err(x: u32) -> Result { Err(x) } + /// # #![feature(never_type)] + /// # #![feature(unwrap_infallible)] /// - /// assert_eq!(Ok(2).or_else(sq).or_else(sq), Ok(2)); - /// assert_eq!(Ok(2).or_else(err).or_else(sq), Ok(2)); - /// assert_eq!(Err(3).or_else(sq).or_else(err), Ok(9)); - /// assert_eq!(Err(3).or_else(err).or_else(err), Err(3)); + /// fn only_good_news() -> Result { + /// Ok("this is fine".into()) + /// } + /// + /// let s: String = only_good_news().into_ok(); + /// println!("{s}"); /// ``` + #[unstable(feature = "unwrap_infallible", reason = "newly added", issue = "61695")] #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn or_else Result>(self, op: O) -> Result { + pub fn into_ok(self) -> T + where + E: Into, + { match self { - Ok(t) => Ok(t), - Err(e) => op(e), + Ok(x) => x, + Err(e) => e.into(), } } - /// Returns the contained [`Ok`] value or a provided default. + /// Returns the contained [`Err`] value, but never panics. /// - /// Arguments passed to `unwrap_or` are eagerly evaluated; if you are passing - /// the result of a function call, it is recommended to use [`unwrap_or_else`], - /// which is lazily evaluated. + /// Unlike [`unwrap_err`], this method is known to never panic on the + /// result types it is implemented for. Therefore, it can be used + /// instead of `unwrap_err` as a maintainability safeguard that will fail + /// to compile if the ok type of the `Result` is later changed + /// to a type that can actually occur. /// - /// [`Ok`]: enum.Result.html#variant.Ok - /// [`Err`]: enum.Result.html#variant.Err - /// [`unwrap_or_else`]: #method.unwrap_or_else + /// [`unwrap_err`]: Result::unwrap_err /// /// # Examples /// /// Basic usage: /// /// ``` - /// let default = 2; - /// let x: Result = Ok(9); - /// assert_eq!(x.unwrap_or(default), 9); + /// # #![feature(never_type)] + /// # #![feature(unwrap_infallible)] /// - /// let x: Result = Err("error"); - /// assert_eq!(x.unwrap_or(default), default); + /// fn only_bad_news() -> Result { + /// Err("Oops, it failed".into()) + /// } + /// + /// let error: String = only_bad_news().into_err(); + /// println!("{error}"); /// ``` + #[unstable(feature = "unwrap_infallible", reason = "newly added", issue = "61695")] #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn unwrap_or(self, default: T) -> T { + pub fn into_err(self) -> E + where + T: Into, + { match self { - Ok(t) => t, - Err(_) => default, + Ok(x) => x.into(), + Err(e) => e, } } - /// Returns the contained [`Ok`] value or computes it from a closure. + //////////////////////////////////////////////////////////////////////// + // Boolean operations on the values, eager and lazy + ///////////////////////////////////////////////////////////////////////// + + /// Returns `res` if the result is [`Ok`], otherwise returns the [`Err`] value of `self`. /// - /// [`Ok`]: enum.Result.html#variant.Ok + /// Arguments passed to `and` are eagerly evaluated; if you are passing the + /// result of a function call, it is recommended to use [`and_then`], which is + /// lazily evaluated. + /// + /// [`and_then`]: Result::and_then /// /// # Examples /// /// Basic usage: /// /// ``` - /// fn count(x: &str) -> usize { x.len() } + /// let x: Result = Ok(2); + /// let y: Result<&str, &str> = Err("late error"); + /// assert_eq!(x.and(y), Err("late error")); /// - /// assert_eq!(Ok(2).unwrap_or_else(count), 2); - /// assert_eq!(Err("foo").unwrap_or_else(count), 3); + /// let x: Result = Err("early error"); + /// let y: Result<&str, &str> = Ok("foo"); + /// assert_eq!(x.and(y), Err("early error")); + /// + /// let x: Result = Err("not a 2"); + /// let y: Result<&str, &str> = Err("late error"); + /// assert_eq!(x.and(y), Err("not a 2")); + /// + /// let x: Result = Ok(2); + /// let y: Result<&str, &str> = Ok("different result type"); + /// assert_eq!(x.and(y), Ok("different result type")); /// ``` #[inline] + #[rustc_const_unstable(feature = "const_result_drop", issue = "92384")] #[stable(feature = "rust1", since = "1.0.0")] - pub fn unwrap_or_else T>(self, op: F) -> T { + pub const fn and(self, res: Result) -> Result + where + T: ~const Destruct, + U: ~const Destruct, + E: ~const Destruct, + { match self { - Ok(t) => t, - Err(e) => op(e), + // FIXME: ~const Drop doesn't quite work right yet + #[allow(unused_variables)] + Ok(x) => res, + Err(e) => Err(e), } } -} -impl Result<&T, E> { - /// Maps a `Result<&T, E>` to a `Result` by copying the contents of the - /// `Ok` part. + /// Calls `op` if the result is [`Ok`], otherwise returns the [`Err`] value of `self`. + /// + /// + /// This function can be used for control flow based on `Result` values. /// /// # Examples /// /// ``` - /// #![feature(result_copied)] - /// let val = 12; - /// let x: Result<&i32, i32> = Ok(&val); - /// assert_eq!(x, Ok(&12)); - /// let copied = x.copied(); - /// assert_eq!(copied, Ok(12)); + /// fn sq_then_to_string(x: u32) -> Result { + /// x.checked_mul(x).map(|sq| sq.to_string()).ok_or("overflowed") + /// } + /// + /// assert_eq!(Ok(2).and_then(sq_then_to_string), Ok(4.to_string())); + /// assert_eq!(Ok(1_000_000).and_then(sq_then_to_string), Err("overflowed")); + /// assert_eq!(Err("not a number").and_then(sq_then_to_string), Err("not a number")); /// ``` - #[unstable(feature = "result_copied", reason = "newly added", issue = "63168")] - pub fn copied(self) -> Result { - self.map(|&t| t) - } -} - -impl Result<&mut T, E> { - /// Maps a `Result<&mut T, E>` to a `Result` by copying the contents of the - /// `Ok` part. /// - /// # Examples + /// Often used to chain fallible operations that may return [`Err`]. /// /// ``` - /// #![feature(result_copied)] - /// let mut val = 12; - /// let x: Result<&mut i32, i32> = Ok(&mut val); - /// assert_eq!(x, Ok(&mut 12)); - /// let copied = x.copied(); - /// assert_eq!(copied, Ok(12)); + /// use std::{io::ErrorKind, path::Path}; + /// + /// // Note: on Windows "/" maps to "C:\" + /// let root_modified_time = Path::new("/").metadata().and_then(|md| md.modified()); + /// assert!(root_modified_time.is_ok()); + /// + /// let should_fail = Path::new("/bad/path").metadata().and_then(|md| md.modified()); + /// assert!(should_fail.is_err()); + /// assert_eq!(should_fail.unwrap_err().kind(), ErrorKind::NotFound); /// ``` - #[unstable(feature = "result_copied", reason = "newly added", issue = "63168")] - pub fn copied(self) -> Result { - self.map(|&mut t| t) + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn and_then Result>(self, op: F) -> Result { + match self { + Ok(t) => op(t), + Err(e) => Err(e), + } } -} -impl Result<&T, E> { - /// Maps a `Result<&T, E>` to a `Result` by cloning the contents of the - /// `Ok` part. + /// Returns `res` if the result is [`Err`], otherwise returns the [`Ok`] value of `self`. + /// + /// Arguments passed to `or` are eagerly evaluated; if you are passing the + /// result of a function call, it is recommended to use [`or_else`], which is + /// lazily evaluated. + /// + /// [`or_else`]: Result::or_else /// /// # Examples /// + /// Basic usage: + /// /// ``` - /// #![feature(result_cloned)] - /// let val = 12; - /// let x: Result<&i32, i32> = Ok(&val); - /// assert_eq!(x, Ok(&12)); - /// let cloned = x.cloned(); - /// assert_eq!(cloned, Ok(12)); + /// let x: Result = Ok(2); + /// let y: Result = Err("late error"); + /// assert_eq!(x.or(y), Ok(2)); + /// + /// let x: Result = Err("early error"); + /// let y: Result = Ok(2); + /// assert_eq!(x.or(y), Ok(2)); + /// + /// let x: Result = Err("not a 2"); + /// let y: Result = Err("late error"); + /// assert_eq!(x.or(y), Err("late error")); + /// + /// let x: Result = Ok(2); + /// let y: Result = Ok(100); + /// assert_eq!(x.or(y), Ok(2)); /// ``` - #[unstable(feature = "result_cloned", reason = "newly added", issue = "63168")] - pub fn cloned(self) -> Result { - self.map(|t| t.clone()) + #[inline] + #[rustc_const_unstable(feature = "const_result_drop", issue = "92384")] + #[stable(feature = "rust1", since = "1.0.0")] + pub const fn or(self, res: Result) -> Result + where + T: ~const Destruct, + E: ~const Destruct, + F: ~const Destruct, + { + match self { + Ok(v) => Ok(v), + // FIXME: ~const Drop doesn't quite work right yet + #[allow(unused_variables)] + Err(e) => res, + } } -} -impl Result<&mut T, E> { - /// Maps a `Result<&mut T, E>` to a `Result` by cloning the contents of the - /// `Ok` part. + /// Calls `op` if the result is [`Err`], otherwise returns the [`Ok`] value of `self`. + /// + /// This function can be used for control flow based on result values. + /// /// /// # Examples /// + /// Basic usage: + /// /// ``` - /// #![feature(result_cloned)] - /// let mut val = 12; - /// let x: Result<&mut i32, i32> = Ok(&mut val); - /// assert_eq!(x, Ok(&mut 12)); - /// let cloned = x.cloned(); - /// assert_eq!(cloned, Ok(12)); + /// fn sq(x: u32) -> Result { Ok(x * x) } + /// fn err(x: u32) -> Result { Err(x) } + /// + /// assert_eq!(Ok(2).or_else(sq).or_else(sq), Ok(2)); + /// assert_eq!(Ok(2).or_else(err).or_else(sq), Ok(2)); + /// assert_eq!(Err(3).or_else(sq).or_else(err), Ok(9)); + /// assert_eq!(Err(3).or_else(err).or_else(err), Err(3)); /// ``` - #[unstable(feature = "result_cloned", reason = "newly added", issue = "63168")] - pub fn cloned(self) -> Result { - self.map(|t| t.clone()) + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn or_else Result>(self, op: O) -> Result { + match self { + Ok(t) => Ok(t), + Err(e) => op(e), + } } -} -impl Result { - /// Returns the contained [`Ok`] value, consuming the `self` value. - /// - /// # Panics + /// Returns the contained [`Ok`] value or a provided default. /// - /// Panics if the value is an [`Err`], with a panic message including the - /// passed message, and the content of the [`Err`]. + /// Arguments passed to `unwrap_or` are eagerly evaluated; if you are passing + /// the result of a function call, it is recommended to use [`unwrap_or_else`], + /// which is lazily evaluated. /// - /// [`Ok`]: enum.Result.html#variant.Ok - /// [`Err`]: enum.Result.html#variant.Err + /// [`unwrap_or_else`]: Result::unwrap_or_else /// /// # Examples /// /// Basic usage: /// - /// ```{.should_panic} - /// let x: Result = Err("emergency failure"); - /// x.expect("Testing expect"); // panics with `Testing expect: emergency failure` + /// ``` + /// let default = 2; + /// let x: Result = Ok(9); + /// assert_eq!(x.unwrap_or(default), 9); + /// + /// let x: Result = Err("error"); + /// assert_eq!(x.unwrap_or(default), default); /// ``` #[inline] - #[track_caller] - #[stable(feature = "result_expect", since = "1.4.0")] - pub fn expect(self, msg: &str) -> T { + #[rustc_const_unstable(feature = "const_result_drop", issue = "92384")] + #[stable(feature = "rust1", since = "1.0.0")] + pub const fn unwrap_or(self, default: T) -> T + where + T: ~const Destruct, + E: ~const Destruct, + { match self { Ok(t) => t, - Err(e) => unwrap_failed(msg, &e), + // FIXME: ~const Drop doesn't quite work right yet + #[allow(unused_variables)] + Err(e) => default, } } - /// Returns the contained [`Ok`] value, consuming the `self` value. - /// - /// Because this function may panic, its use is generally discouraged. - /// Instead, prefer to use pattern matching and handle the [`Err`] - /// case explicitly, or call [`unwrap_or`], [`unwrap_or_else`], or - /// [`unwrap_or_default`]. - /// - /// [`unwrap_or`]: #method.unwrap_or - /// [`unwrap_or_else`]: #method.unwrap_or_else - /// [`unwrap_or_default`]: #method.unwrap_or_default - /// - /// # Panics - /// - /// Panics if the value is an [`Err`], with a panic message provided by the - /// [`Err`]'s value. + /// Returns the contained [`Ok`] value or computes it from a closure. /// - /// [`Ok`]: enum.Result.html#variant.Ok - /// [`Err`]: enum.Result.html#variant.Err /// /// # Examples /// /// Basic usage: /// /// ``` - /// let x: Result = Ok(2); - /// assert_eq!(x.unwrap(), 2); - /// ``` + /// fn count(x: &str) -> usize { x.len() } /// - /// ```{.should_panic} - /// let x: Result = Err("emergency failure"); - /// x.unwrap(); // panics with `emergency failure` + /// assert_eq!(Ok(2).unwrap_or_else(count), 2); + /// assert_eq!(Err("foo").unwrap_or_else(count), 3); /// ``` #[inline] - #[track_caller] #[stable(feature = "rust1", since = "1.0.0")] - pub fn unwrap(self) -> T { + pub fn unwrap_or_else T>(self, op: F) -> T { match self { Ok(t) => t, - Err(e) => unwrap_failed("called `Result::unwrap()` on an `Err` value", &e), + Err(e) => op(e), } } -} -impl Result { - /// Returns the contained [`Err`] value, consuming the `self` value. + /// Returns the contained [`Ok`] value, consuming the `self` value, + /// without checking that the value is not an [`Err`]. /// - /// # Panics + /// # Safety /// - /// Panics if the value is an [`Ok`], with a panic message including the - /// passed message, and the content of the [`Ok`]. + /// Calling this method on an [`Err`] is *[undefined behavior]*. /// - /// [`Ok`]: enum.Result.html#variant.Ok - /// [`Err`]: enum.Result.html#variant.Err + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html /// /// # Examples /// - /// Basic usage: + /// ``` + /// let x: Result = Ok(2); + /// assert_eq!(unsafe { x.unwrap_unchecked() }, 2); + /// ``` /// - /// ```{.should_panic} - /// let x: Result = Ok(10); - /// x.expect_err("Testing expect_err"); // panics with `Testing expect_err: 10` + /// ```no_run + /// let x: Result = Err("emergency failure"); + /// unsafe { x.unwrap_unchecked(); } // Undefined behavior! /// ``` #[inline] #[track_caller] - #[stable(feature = "result_expect_err", since = "1.17.0")] - pub fn expect_err(self, msg: &str) -> E { + #[stable(feature = "option_result_unwrap_unchecked", since = "1.58.0")] + pub unsafe fn unwrap_unchecked(self) -> T { + debug_assert!(self.is_ok()); match self { - Ok(t) => unwrap_failed(msg, &t), - Err(e) => e, + Ok(t) => t, + // SAFETY: the safety contract must be upheld by the caller. + Err(_) => unsafe { hint::unreachable_unchecked() }, } } - /// Returns the contained [`Err`] value, consuming the `self` value. - /// - /// # Panics + /// Returns the contained [`Err`] value, consuming the `self` value, + /// without checking that the value is not an [`Ok`]. /// - /// Panics if the value is an [`Ok`], with a custom panic message provided - /// by the [`Ok`]'s value. + /// # Safety /// - /// [`Ok`]: enum.Result.html#variant.Ok - /// [`Err`]: enum.Result.html#variant.Err + /// Calling this method on an [`Ok`] is *[undefined behavior]*. /// + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html /// /// # Examples /// - /// ```{.should_panic} + /// ```no_run /// let x: Result = Ok(2); - /// x.unwrap_err(); // panics with `2` + /// unsafe { x.unwrap_err_unchecked() }; // Undefined behavior! /// ``` /// /// ``` /// let x: Result = Err("emergency failure"); - /// assert_eq!(x.unwrap_err(), "emergency failure"); + /// assert_eq!(unsafe { x.unwrap_err_unchecked() }, "emergency failure"); /// ``` #[inline] #[track_caller] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn unwrap_err(self) -> E { + #[stable(feature = "option_result_unwrap_unchecked", since = "1.58.0")] + pub unsafe fn unwrap_err_unchecked(self) -> E { + debug_assert!(self.is_err()); match self { - Ok(t) => unwrap_failed("called `Result::unwrap_err()` on an `Ok` value", &t), + // SAFETY: the safety contract must be upheld by the caller. + Ok(_) => unsafe { hint::unreachable_unchecked() }, Err(e) => e, } } -} -impl Result { - /// Returns the contained [`Ok`] value or a default - /// - /// Consumes the `self` argument then, if [`Ok`], returns the contained - /// value, otherwise if [`Err`], returns the default value for that - /// type. + ///////////////////////////////////////////////////////////////////////// + // Misc or niche + ///////////////////////////////////////////////////////////////////////// + + /// Returns `true` if the result is an [`Ok`] value containing the given value. /// /// # Examples /// - /// Converts a string to an integer, turning poorly-formed strings - /// into 0 (the default value for integers). [`parse`] converts - /// a string to any other type that implements [`FromStr`], returning an - /// [`Err`] on error. - /// /// ``` - /// let good_year_from_input = "1909"; - /// let bad_year_from_input = "190blarg"; - /// let good_year = good_year_from_input.parse().unwrap_or_default(); - /// let bad_year = bad_year_from_input.parse().unwrap_or_default(); + /// #![feature(option_result_contains)] /// - /// assert_eq!(1909, good_year); - /// assert_eq!(0, bad_year); - /// ``` + /// let x: Result = Ok(2); + /// assert_eq!(x.contains(&2), true); + /// + /// let x: Result = Ok(3); + /// assert_eq!(x.contains(&2), false); /// - /// [`parse`]: ../../std/primitive.str.html#method.parse - /// [`FromStr`]: ../../std/str/trait.FromStr.html - /// [`Ok`]: enum.Result.html#variant.Ok - /// [`Err`]: enum.Result.html#variant.Err + /// let x: Result = Err("Some error message"); + /// assert_eq!(x.contains(&2), false); + /// ``` + #[must_use] #[inline] - #[stable(feature = "result_unwrap_or_default", since = "1.16.0")] - pub fn unwrap_or_default(self) -> T { + #[unstable(feature = "option_result_contains", issue = "62358")] + pub fn contains(&self, x: &U) -> bool + where + U: PartialEq, + { match self { - Ok(x) => x, - Err(_) => Default::default(), + Ok(y) => x == y, + Err(_) => false, } } -} -#[unstable(feature = "unwrap_infallible", reason = "newly added", issue = "61695")] -impl> Result { - /// Returns the contained [`Ok`] value, but never panics. - /// - /// Unlike [`unwrap`], this method is known to never panic on the - /// result types it is implemented for. Therefore, it can be used - /// instead of `unwrap` as a maintainability safeguard that will fail - /// to compile if the error type of the `Result` is later changed - /// to an error that can actually occur. - /// - /// [`Ok`]: enum.Result.html#variant.Ok - /// [`Err`]: enum.Result.html#variant.Err - /// [`unwrap`]: enum.Result.html#method.unwrap + /// Returns `true` if the result is an [`Err`] value containing the given value. /// /// # Examples /// - /// Basic usage: - /// /// ``` - /// # #![feature(never_type)] - /// # #![feature(unwrap_infallible)] + /// #![feature(result_contains_err)] /// - /// fn only_good_news() -> Result { - /// Ok("this is fine".into()) - /// } + /// let x: Result = Ok(2); + /// assert_eq!(x.contains_err(&"Some error message"), false); /// - /// let s: String = only_good_news().into_ok(); - /// println!("{}", s); + /// let x: Result = Err("Some error message"); + /// assert_eq!(x.contains_err(&"Some error message"), true); + /// + /// let x: Result = Err("Some other error message"); + /// assert_eq!(x.contains_err(&"Some error message"), false); /// ``` + #[must_use] #[inline] - pub fn into_ok(self) -> T { + #[unstable(feature = "result_contains_err", issue = "62358")] + pub fn contains_err(&self, f: &F) -> bool + where + F: PartialEq, + { match self { - Ok(x) => x, - Err(e) => e.into(), + Ok(_) => false, + Err(e) => f == e, } } } -#[unstable(feature = "inner_deref", reason = "newly added", issue = "50264")] -impl Result { - /// Converts from `Result` (or `&Result`) to `Result<&T::Target, &E>`. +impl Result<&T, E> { + /// Maps a `Result<&T, E>` to a `Result` by copying the contents of the + /// `Ok` part. /// - /// Leaves the original `Result` in-place, creating a new one containing a reference to the - /// `Ok` type's `Deref::Target` type. - pub fn as_deref(&self) -> Result<&T::Target, &E> { - self.as_ref().map(|t| t.deref()) + /// # Examples + /// + /// ``` + /// let val = 12; + /// let x: Result<&i32, i32> = Ok(&val); + /// assert_eq!(x, Ok(&12)); + /// let copied = x.copied(); + /// assert_eq!(copied, Ok(12)); + /// ``` + #[inline] + #[stable(feature = "result_copied", since = "1.59.0")] + pub fn copied(self) -> Result + where + T: Copy, + { + self.map(|&t| t) } -} -#[unstable(feature = "inner_deref", reason = "newly added", issue = "50264")] -impl Result { - /// Converts from `Result` (or `&Result`) to `Result<&T, &E::Target>`. + /// Maps a `Result<&T, E>` to a `Result` by cloning the contents of the + /// `Ok` part. + /// + /// # Examples /// - /// Leaves the original `Result` in-place, creating a new one containing a reference to the - /// `Err` type's `Deref::Target` type. - pub fn as_deref_err(&self) -> Result<&T, &E::Target> { - self.as_ref().map_err(|e| e.deref()) + /// ``` + /// let val = 12; + /// let x: Result<&i32, i32> = Ok(&val); + /// assert_eq!(x, Ok(&12)); + /// let cloned = x.cloned(); + /// assert_eq!(cloned, Ok(12)); + /// ``` + #[inline] + #[stable(feature = "result_cloned", since = "1.59.0")] + pub fn cloned(self) -> Result + where + T: Clone, + { + self.map(|t| t.clone()) } } -#[unstable(feature = "inner_deref", reason = "newly added", issue = "50264")] -impl Result { - /// Converts from `Result` (or `&mut Result`) to `Result<&mut T::Target, &mut E>`. +impl Result<&mut T, E> { + /// Maps a `Result<&mut T, E>` to a `Result` by copying the contents of the + /// `Ok` part. /// - /// Leaves the original `Result` in-place, creating a new one containing a mutable reference to - /// the `Ok` type's `Deref::Target` type. - pub fn as_deref_mut(&mut self) -> Result<&mut T::Target, &mut E> { - self.as_mut().map(|t| t.deref_mut()) + /// # Examples + /// + /// ``` + /// let mut val = 12; + /// let x: Result<&mut i32, i32> = Ok(&mut val); + /// assert_eq!(x, Ok(&mut 12)); + /// let copied = x.copied(); + /// assert_eq!(copied, Ok(12)); + /// ``` + #[inline] + #[stable(feature = "result_copied", since = "1.59.0")] + pub fn copied(self) -> Result + where + T: Copy, + { + self.map(|&mut t| t) } -} -#[unstable(feature = "inner_deref", reason = "newly added", issue = "50264")] -impl Result { - /// Converts from `Result` (or `&mut Result`) to `Result<&mut T, &mut E::Target>`. + /// Maps a `Result<&mut T, E>` to a `Result` by cloning the contents of the + /// `Ok` part. + /// + /// # Examples /// - /// Leaves the original `Result` in-place, creating a new one containing a mutable reference to - /// the `Err` type's `Deref::Target` type. - pub fn as_deref_mut_err(&mut self) -> Result<&mut T, &mut E::Target> { - self.as_mut().map_err(|e| e.deref_mut()) + /// ``` + /// let mut val = 12; + /// let x: Result<&mut i32, i32> = Ok(&mut val); + /// assert_eq!(x, Ok(&mut 12)); + /// let cloned = x.cloned(); + /// assert_eq!(cloned, Ok(12)); + /// ``` + #[inline] + #[stable(feature = "result_cloned", since = "1.59.0")] + pub fn cloned(self) -> Result + where + T: Clone, + { + self.map(|t| t.clone()) } } @@ -1205,7 +1737,8 @@ impl Result, E> { /// ``` #[inline] #[stable(feature = "transpose_result", since = "1.33.0")] - pub fn transpose(self) -> Option> { + #[rustc_const_unstable(feature = "const_result", issue = "82814")] + pub const fn transpose(self) -> Option> { match self { Ok(Some(x)) => Some(Ok(x)), Ok(None) => None, @@ -1214,12 +1747,59 @@ impl Result, E> { } } +impl Result, E> { + /// Converts from `Result, E>` to `Result` + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(result_flattening)] + /// let x: Result, u32> = Ok(Ok("hello")); + /// assert_eq!(Ok("hello"), x.flatten()); + /// + /// let x: Result, u32> = Ok(Err(6)); + /// assert_eq!(Err(6), x.flatten()); + /// + /// let x: Result, u32> = Err(6); + /// assert_eq!(Err(6), x.flatten()); + /// ``` + /// + /// Flattening only removes one level of nesting at a time: + /// + /// ``` + /// #![feature(result_flattening)] + /// let x: Result, u32>, u32> = Ok(Ok(Ok("hello"))); + /// assert_eq!(Ok(Ok("hello")), x.flatten()); + /// assert_eq!(Ok("hello"), x.flatten().flatten()); + /// ``` + #[inline] + #[unstable(feature = "result_flattening", issue = "70142")] + pub fn flatten(self) -> Result { + self.and_then(convert::identity) + } +} + // This is a separate function to reduce the code size of the methods +#[cfg(not(feature = "panic_immediate_abort"))] #[inline(never)] #[cold] #[track_caller] fn unwrap_failed(msg: &str, error: &dyn fmt::Debug) -> ! { - panic!("{}: {:?}", msg, error) + panic!("{msg}: {error:?}") +} + +// This is a separate function to avoid constructing a `dyn Debug` +// that gets immediately thrown away, since vtables don't get cleaned up +// by dead code elimination if a trait object is constructed even if it goes +// unused +#[cfg(feature = "panic_immediate_abort")] +#[inline] +#[cold] +#[track_caller] +fn unwrap_failed(_msg: &str, _error: &T) -> ! { + panic!() } ///////////////////////////////////////////////////////////////////////////// @@ -1227,7 +1807,12 @@ fn unwrap_failed(msg: &str, error: &dyn fmt::Debug) -> ! { ///////////////////////////////////////////////////////////////////////////// #[stable(feature = "rust1", since = "1.0.0")] -impl Clone for Result { +#[rustc_const_unstable(feature = "const_clone", issue = "91805")] +impl const Clone for Result +where + T: ~const Clone + ~const Destruct, + E: ~const Clone + ~const Destruct, +{ #[inline] fn clone(&self) -> Self { match self { @@ -1303,10 +1888,6 @@ impl<'a, T, E> IntoIterator for &'a mut Result { /// The iterator yields one value if the result is [`Ok`], otherwise none. /// /// Created by [`Result::iter`]. -/// -/// [`Ok`]: enum.Result.html#variant.Ok -/// [`Result`]: enum.Result.html -/// [`Result::iter`]: enum.Result.html#method.iter #[derive(Debug)] #[stable(feature = "rust1", since = "1.0.0")] pub struct Iter<'a, T: 'a> { @@ -1356,10 +1937,6 @@ impl Clone for Iter<'_, T> { /// An iterator over a mutable reference to the [`Ok`] variant of a [`Result`]. /// /// Created by [`Result::iter_mut`]. -/// -/// [`Ok`]: enum.Result.html#variant.Ok -/// [`Result`]: enum.Result.html -/// [`Result::iter_mut`]: enum.Result.html#method.iter_mut #[derive(Debug)] #[stable(feature = "rust1", since = "1.0.0")] pub struct IterMut<'a, T: 'a> { @@ -1405,10 +1982,7 @@ unsafe impl TrustedLen for IterMut<'_, A> {} /// This struct is created by the [`into_iter`] method on /// [`Result`] (provided by the [`IntoIterator`] trait). /// -/// [`Ok`]: enum.Result.html#variant.Ok -/// [`Result`]: enum.Result.html -/// [`into_iter`]: ../iter/trait.IntoIterator.html#tymethod.into_iter -/// [`IntoIterator`]: ../iter/trait.IntoIterator.html +/// [`into_iter`]: IntoIterator::into_iter #[derive(Clone, Debug)] #[stable(feature = "rust1", since = "1.0.0")] pub struct IntoIter { @@ -1497,30 +2071,54 @@ impl> FromIterator> for Result { /// so the final value of `shared` is 6 (= `3 + 2 + 1`), not 16. #[inline] fn from_iter>>(iter: I) -> Result { - // FIXME(#11084): This could be replaced with Iterator::scan when this - // performance bug is closed. - - iter::process_results(iter.into_iter(), |i| i.collect()) + iter::try_process(iter.into_iter(), |i| i.collect()) } } -#[unstable(feature = "try_trait", issue = "42327")] -impl ops::Try for Result { - type Ok = T; - type Error = E; +#[unstable(feature = "try_trait_v2", issue = "84277")] +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const ops::Try for Result { + type Output = T; + type Residual = Result; #[inline] - fn into_result(self) -> Self { - self + fn from_output(output: Self::Output) -> Self { + Ok(output) + } + + #[inline] + fn branch(self) -> ControlFlow { + match self { + Ok(v) => ControlFlow::Continue(v), + Err(e) => ControlFlow::Break(Err(e)), + } } +} +#[unstable(feature = "try_trait_v2", issue = "84277")] +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl> const ops::FromResidual> + for Result +{ #[inline] - fn from_ok(v: T) -> Self { - Ok(v) + #[track_caller] + fn from_residual(residual: Result) -> Self { + match residual { + Err(e) => Err(From::from(e)), + } } +} +#[unstable(feature = "try_trait_v2_yeet", issue = "96374")] +impl> ops::FromResidual> for Result { #[inline] - fn from_error(v: E) -> Self { - Err(v) + fn from_residual(ops::Yeet(e): ops::Yeet) -> Self { + Err(From::from(e)) } } + +#[unstable(feature = "try_trait_v2_residual", issue = "91285")] +#[rustc_const_unstable(feature = "const_try", issue = "74935")] +impl const ops::Residual for Result { + type TryType = Result; +} diff --git a/crux-mir/lib/core/src/slice/ascii.rs b/crux-mir/lib/core/src/slice/ascii.rs new file mode 100644 index 000000000..5e5399acc --- /dev/null +++ b/crux-mir/lib/core/src/slice/ascii.rs @@ -0,0 +1,328 @@ +//! Operations on ASCII `[u8]`. + +use crate::ascii; +use crate::fmt::{self, Write}; +use crate::iter; +use crate::mem; +use crate::ops; + +#[cfg(not(test))] +impl [u8] { + /// Checks if all bytes in this slice are within the ASCII range. + #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] + #[must_use] + #[inline] + pub fn is_ascii(&self) -> bool { + is_ascii(self) + } + + /// Checks that two slices are an ASCII case-insensitive match. + /// + /// Same as `to_ascii_lowercase(a) == to_ascii_lowercase(b)`, + /// but without allocating and copying temporaries. + #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] + #[must_use] + #[inline] + pub fn eq_ignore_ascii_case(&self, other: &[u8]) -> bool { + self.len() == other.len() && iter::zip(self, other).all(|(a, b)| a.eq_ignore_ascii_case(b)) + } + + /// Converts this slice to its ASCII upper case equivalent in-place. + /// + /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', + /// but non-ASCII letters are unchanged. + /// + /// To return a new uppercased value without modifying the existing one, use + /// [`to_ascii_uppercase`]. + /// + /// [`to_ascii_uppercase`]: #method.to_ascii_uppercase + #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] + #[inline] + pub fn make_ascii_uppercase(&mut self) { + for byte in self { + byte.make_ascii_uppercase(); + } + } + + /// Converts this slice to its ASCII lower case equivalent in-place. + /// + /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', + /// but non-ASCII letters are unchanged. + /// + /// To return a new lowercased value without modifying the existing one, use + /// [`to_ascii_lowercase`]. + /// + /// [`to_ascii_lowercase`]: #method.to_ascii_lowercase + #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] + #[inline] + pub fn make_ascii_lowercase(&mut self) { + for byte in self { + byte.make_ascii_lowercase(); + } + } + + /// Returns an iterator that produces an escaped version of this slice, + /// treating it as an ASCII string. + /// + /// # Examples + /// + /// ``` + /// + /// let s = b"0\t\r\n'\"\\\x9d"; + /// let escaped = s.escape_ascii().to_string(); + /// assert_eq!(escaped, "0\\t\\r\\n\\'\\\"\\\\\\x9d"); + /// ``` + #[must_use = "this returns the escaped bytes as an iterator, \ + without modifying the original"] + #[stable(feature = "inherent_ascii_escape", since = "1.60.0")] + pub fn escape_ascii(&self) -> EscapeAscii<'_> { + EscapeAscii { inner: self.iter().flat_map(EscapeByte) } + } + + /// Returns a byte slice with leading ASCII whitespace bytes removed. + /// + /// 'Whitespace' refers to the definition used by + /// `u8::is_ascii_whitespace`. + /// + /// # Examples + /// + /// ``` + /// #![feature(byte_slice_trim_ascii)] + /// + /// assert_eq!(b" \t hello world\n".trim_ascii_start(), b"hello world\n"); + /// assert_eq!(b" ".trim_ascii_start(), b""); + /// assert_eq!(b"".trim_ascii_start(), b""); + /// ``` + #[unstable(feature = "byte_slice_trim_ascii", issue = "94035")] + pub const fn trim_ascii_start(&self) -> &[u8] { + let mut bytes = self; + // Note: A pattern matching based approach (instead of indexing) allows + // making the function const. + while let [first, rest @ ..] = bytes { + if first.is_ascii_whitespace() { + bytes = rest; + } else { + break; + } + } + bytes + } + + /// Returns a byte slice with trailing ASCII whitespace bytes removed. + /// + /// 'Whitespace' refers to the definition used by + /// `u8::is_ascii_whitespace`. + /// + /// # Examples + /// + /// ``` + /// #![feature(byte_slice_trim_ascii)] + /// + /// assert_eq!(b"\r hello world\n ".trim_ascii_end(), b"\r hello world"); + /// assert_eq!(b" ".trim_ascii_end(), b""); + /// assert_eq!(b"".trim_ascii_end(), b""); + /// ``` + #[unstable(feature = "byte_slice_trim_ascii", issue = "94035")] + pub const fn trim_ascii_end(&self) -> &[u8] { + let mut bytes = self; + // Note: A pattern matching based approach (instead of indexing) allows + // making the function const. + while let [rest @ .., last] = bytes { + if last.is_ascii_whitespace() { + bytes = rest; + } else { + break; + } + } + bytes + } + + /// Returns a byte slice with leading and trailing ASCII whitespace bytes + /// removed. + /// + /// 'Whitespace' refers to the definition used by + /// `u8::is_ascii_whitespace`. + /// + /// # Examples + /// + /// ``` + /// #![feature(byte_slice_trim_ascii)] + /// + /// assert_eq!(b"\r hello world\n ".trim_ascii(), b"hello world"); + /// assert_eq!(b" ".trim_ascii(), b""); + /// assert_eq!(b"".trim_ascii(), b""); + /// ``` + #[unstable(feature = "byte_slice_trim_ascii", issue = "94035")] + pub const fn trim_ascii(&self) -> &[u8] { + self.trim_ascii_start().trim_ascii_end() + } +} + +impl_fn_for_zst! { + #[derive(Clone)] + struct EscapeByte impl Fn = |byte: &u8| -> ascii::EscapeDefault { + ascii::escape_default(*byte) + }; +} + +/// An iterator over the escaped version of a byte slice. +/// +/// This `struct` is created by the [`slice::escape_ascii`] method. See its +/// documentation for more information. +#[stable(feature = "inherent_ascii_escape", since = "1.60.0")] +#[derive(Clone)] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct EscapeAscii<'a> { + inner: iter::FlatMap, ascii::EscapeDefault, EscapeByte>, +} + +#[stable(feature = "inherent_ascii_escape", since = "1.60.0")] +impl<'a> iter::Iterator for EscapeAscii<'a> { + type Item = u8; + #[inline] + fn next(&mut self) -> Option { + self.inner.next() + } + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } + #[inline] + fn try_fold(&mut self, init: Acc, fold: Fold) -> R + where + Fold: FnMut(Acc, Self::Item) -> R, + R: ops::Try, + { + self.inner.try_fold(init, fold) + } + #[inline] + fn fold(self, init: Acc, fold: Fold) -> Acc + where + Fold: FnMut(Acc, Self::Item) -> Acc, + { + self.inner.fold(init, fold) + } + #[inline] + fn last(mut self) -> Option { + self.next_back() + } +} + +#[stable(feature = "inherent_ascii_escape", since = "1.60.0")] +impl<'a> iter::DoubleEndedIterator for EscapeAscii<'a> { + fn next_back(&mut self) -> Option { + self.inner.next_back() + } +} +#[stable(feature = "inherent_ascii_escape", since = "1.60.0")] +impl<'a> iter::FusedIterator for EscapeAscii<'a> {} +#[stable(feature = "inherent_ascii_escape", since = "1.60.0")] +impl<'a> fmt::Display for EscapeAscii<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.clone().try_for_each(|b| f.write_char(b as char)) + } +} +#[stable(feature = "inherent_ascii_escape", since = "1.60.0")] +impl<'a> fmt::Debug for EscapeAscii<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EscapeAscii").finish_non_exhaustive() + } +} + +/// Returns `true` if any byte in the word `v` is nonascii (>= 128). Snarfed +/// from `../str/mod.rs`, which does something similar for utf8 validation. +#[inline] +fn contains_nonascii(v: usize) -> bool { + const NONASCII_MASK: usize = usize::repeat_u8(0x80); + (NONASCII_MASK & v) != 0 +} + +/// Optimized ASCII test that will use usize-at-a-time operations instead of +/// byte-at-a-time operations (when possible). +/// +/// The algorithm we use here is pretty simple. If `s` is too short, we just +/// check each byte and be done with it. Otherwise: +/// +/// - Read the first word with an unaligned load. +/// - Align the pointer, read subsequent words until end with aligned loads. +/// - Read the last `usize` from `s` with an unaligned load. +/// +/// If any of these loads produces something for which `contains_nonascii` +/// (above) returns true, then we know the answer is false. +#[inline] +fn is_ascii(s: &[u8]) -> bool { + const USIZE_SIZE: usize = mem::size_of::(); + + let len = s.len(); + let align_offset = s.as_ptr().align_offset(USIZE_SIZE); + + // If we wouldn't gain anything from the word-at-a-time implementation, fall + // back to a scalar loop. + // + // We also do this for architectures where `size_of::()` isn't + // sufficient alignment for `usize`, because it's a weird edge case. + if len < USIZE_SIZE || len < align_offset || USIZE_SIZE < mem::align_of::() { + return s.iter().all(|b| b.is_ascii()); + } + + // We always read the first word unaligned, which means `align_offset` is + // 0, we'd read the same value again for the aligned read. + let offset_to_aligned = if align_offset == 0 { USIZE_SIZE } else { align_offset }; + + let start = s.as_ptr(); + // SAFETY: We verify `len < USIZE_SIZE` above. + let first_word = unsafe { (start as *const usize).read_unaligned() }; + + if contains_nonascii(first_word) { + return false; + } + // We checked this above, somewhat implicitly. Note that `offset_to_aligned` + // is either `align_offset` or `USIZE_SIZE`, both of are explicitly checked + // above. + debug_assert!(offset_to_aligned <= len); + + // SAFETY: word_ptr is the (properly aligned) usize ptr we use to read the + // middle chunk of the slice. + let mut word_ptr = unsafe { start.add(offset_to_aligned) as *const usize }; + + // `byte_pos` is the byte index of `word_ptr`, used for loop end checks. + let mut byte_pos = offset_to_aligned; + + // Paranoia check about alignment, since we're about to do a bunch of + // unaligned loads. In practice this should be impossible barring a bug in + // `align_offset` though. + debug_assert_eq!(word_ptr.addr() % mem::align_of::(), 0); + + // Read subsequent words until the last aligned word, excluding the last + // aligned word by itself to be done in tail check later, to ensure that + // tail is always one `usize` at most to extra branch `byte_pos == len`. + while byte_pos < len - USIZE_SIZE { + debug_assert!( + // Sanity check that the read is in bounds + (word_ptr.addr() + USIZE_SIZE) <= start.addr().wrapping_add(len) && + // And that our assumptions about `byte_pos` hold. + (word_ptr.addr() - start.addr()) == byte_pos + ); + + // SAFETY: We know `word_ptr` is properly aligned (because of + // `align_offset`), and we know that we have enough bytes between `word_ptr` and the end + let word = unsafe { word_ptr.read() }; + if contains_nonascii(word) { + return false; + } + + byte_pos += USIZE_SIZE; + // SAFETY: We know that `byte_pos <= len - USIZE_SIZE`, which means that + // after this `add`, `word_ptr` will be at most one-past-the-end. + word_ptr = unsafe { word_ptr.add(1) }; + } + + // Sanity check to ensure there really is only one `usize` left. This should + // be guaranteed by our loop condition. + debug_assert!(byte_pos <= len && len - byte_pos <= USIZE_SIZE); + + // SAFETY: This relies on `len >= USIZE_SIZE`, which we check at the start. + let last_word = unsafe { (start.add(len - USIZE_SIZE) as *const usize).read_unaligned() }; + + !contains_nonascii(last_word) +} diff --git a/crux-mir/lib/core/src/slice/cmp.rs b/crux-mir/lib/core/src/slice/cmp.rs new file mode 100644 index 000000000..5e1b218e5 --- /dev/null +++ b/crux-mir/lib/core/src/slice/cmp.rs @@ -0,0 +1,260 @@ +//! Comparison traits for `[T]`. + +use crate::cmp::{self, Ordering}; +use crate::ffi; +use crate::mem; + +use super::from_raw_parts; +use super::memchr; + +extern "C" { + /// Calls implementation provided memcmp. + /// + /// Interprets the data as u8. + /// + /// Returns 0 for equal, < 0 for less than and > 0 for greater + /// than. + fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> ffi::c_int; +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialEq<[B]> for [A] +where + A: PartialEq, +{ + fn eq(&self, other: &[B]) -> bool { + SlicePartialEq::equal(self, other) + } + + fn ne(&self, other: &[B]) -> bool { + SlicePartialEq::not_equal(self, other) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Eq for [T] {} + +/// Implements comparison of vectors [lexicographically](Ord#lexicographical-comparison). +#[stable(feature = "rust1", since = "1.0.0")] +impl Ord for [T] { + fn cmp(&self, other: &[T]) -> Ordering { + SliceOrd::compare(self, other) + } +} + +/// Implements comparison of vectors [lexicographically](Ord#lexicographical-comparison). +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialOrd for [T] { + fn partial_cmp(&self, other: &[T]) -> Option { + SlicePartialOrd::partial_compare(self, other) + } +} + +#[doc(hidden)] +// intermediate trait for specialization of slice's PartialEq +trait SlicePartialEq { + fn equal(&self, other: &[B]) -> bool; + + fn not_equal(&self, other: &[B]) -> bool { + !self.equal(other) + } +} + +// Generic slice equality +impl SlicePartialEq for [A] +where + A: PartialEq, +{ + default fn equal(&self, other: &[B]) -> bool { + if self.len() != other.len() { + return false; + } + + self.iter().zip(other.iter()).all(|(x, y)| x == y) + } +} + +// Use memcmp for bytewise equality when the types allow +impl SlicePartialEq for [A] +where + A: BytewiseEquality, +{ + fn equal(&self, other: &[B]) -> bool { + if self.len() != other.len() { + return false; + } + + // SAFETY: `self` and `other` are references and are thus guaranteed to be valid. + // The two slices have been checked to have the same size above. + unsafe { + let size = mem::size_of_val(self); + memcmp(self.as_ptr() as *const u8, other.as_ptr() as *const u8, size) == 0 + } + } +} + +#[doc(hidden)] +// intermediate trait for specialization of slice's PartialOrd +trait SlicePartialOrd: Sized { + fn partial_compare(left: &[Self], right: &[Self]) -> Option; +} + +impl SlicePartialOrd for A { + default fn partial_compare(left: &[A], right: &[A]) -> Option { + let l = cmp::min(left.len(), right.len()); + + // Slice to the loop iteration range to enable bound check + // elimination in the compiler + let lhs = &left[..l]; + let rhs = &right[..l]; + + for i in 0..l { + match lhs[i].partial_cmp(&rhs[i]) { + Some(Ordering::Equal) => (), + non_eq => return non_eq, + } + } + + left.len().partial_cmp(&right.len()) + } +} + +// This is the impl that we would like to have. Unfortunately it's not sound. +// See `partial_ord_slice.rs`. +/* +impl SlicePartialOrd for A +where + A: Ord, +{ + default fn partial_compare(left: &[A], right: &[A]) -> Option { + Some(SliceOrd::compare(left, right)) + } +} +*/ + +impl SlicePartialOrd for A { + fn partial_compare(left: &[A], right: &[A]) -> Option { + Some(SliceOrd::compare(left, right)) + } +} + +#[rustc_specialization_trait] +trait AlwaysApplicableOrd: SliceOrd + Ord {} + +macro_rules! always_applicable_ord { + ($([$($p:tt)*] $t:ty,)*) => { + $(impl<$($p)*> AlwaysApplicableOrd for $t {})* + } +} + +always_applicable_ord! { + [] u8, [] u16, [] u32, [] u64, [] u128, [] usize, + [] i8, [] i16, [] i32, [] i64, [] i128, [] isize, + [] bool, [] char, + [T: ?Sized] *const T, [T: ?Sized] *mut T, + [T: AlwaysApplicableOrd] &T, + [T: AlwaysApplicableOrd] &mut T, + [T: AlwaysApplicableOrd] Option, +} + +#[doc(hidden)] +// intermediate trait for specialization of slice's Ord +trait SliceOrd: Sized { + fn compare(left: &[Self], right: &[Self]) -> Ordering; +} + +impl SliceOrd for A { + default fn compare(left: &[Self], right: &[Self]) -> Ordering { + let l = cmp::min(left.len(), right.len()); + + // Slice to the loop iteration range to enable bound check + // elimination in the compiler + let lhs = &left[..l]; + let rhs = &right[..l]; + + for i in 0..l { + match lhs[i].cmp(&rhs[i]) { + Ordering::Equal => (), + non_eq => return non_eq, + } + } + + left.len().cmp(&right.len()) + } +} + +// memcmp compares a sequence of unsigned bytes lexicographically. +// this matches the order we want for [u8], but no others (not even [i8]). +impl SliceOrd for u8 { + #[inline] + fn compare(left: &[Self], right: &[Self]) -> Ordering { + // Since the length of a slice is always less than or equal to isize::MAX, this never underflows. + let diff = left.len() as isize - right.len() as isize; + // This comparison gets optimized away (on x86_64 and ARM) because the subtraction updates flags. + let len = if left.len() < right.len() { left.len() } else { right.len() }; + // SAFETY: `left` and `right` are references and are thus guaranteed to be valid. + // We use the minimum of both lengths which guarantees that both regions are + // valid for reads in that interval. + let mut order = unsafe { memcmp(left.as_ptr(), right.as_ptr(), len) as isize }; + if order == 0 { + order = diff; + } + order.cmp(&0) + } +} + +// Hack to allow specializing on `Eq` even though `Eq` has a method. +#[rustc_unsafe_specialization_marker] +trait MarkerEq: PartialEq {} + +impl MarkerEq for T {} + +#[doc(hidden)] +/// Trait implemented for types that can be compared for equality using +/// their bytewise representation +#[rustc_specialization_trait] +trait BytewiseEquality: MarkerEq + Copy {} + +macro_rules! impl_marker_for { + ($traitname:ident, $($ty:ty)*) => { + $( + impl $traitname<$ty> for $ty { } + )* + } +} + +impl_marker_for!(BytewiseEquality, + u8 i8 u16 i16 u32 i32 u64 i64 u128 i128 usize isize char bool); + +pub(super) trait SliceContains: Sized { + fn slice_contains(&self, x: &[Self]) -> bool; +} + +impl SliceContains for T +where + T: PartialEq, +{ + default fn slice_contains(&self, x: &[Self]) -> bool { + x.iter().any(|y| *y == *self) + } +} + +impl SliceContains for u8 { + #[inline] + fn slice_contains(&self, x: &[Self]) -> bool { + memchr::memchr(*self, x).is_some() + } +} + +impl SliceContains for i8 { + #[inline] + fn slice_contains(&self, x: &[Self]) -> bool { + let byte = *self as u8; + // SAFETY: `i8` and `u8` have the same memory layout, thus casting `x.as_ptr()` + // as `*const u8` is safe. The `x.as_ptr()` comes from a reference and is thus guaranteed + // to be valid for reads for the length of the slice `x.len()`, which cannot be larger + // than `isize::MAX`. The returned slice is never mutated. + let bytes: &[u8] = unsafe { from_raw_parts(x.as_ptr() as *const u8, x.len()) }; + memchr::memchr(byte, bytes).is_some() + } +} diff --git a/crux-mir/lib/core/src/slice/index.rs b/crux-mir/lib/core/src/slice/index.rs new file mode 100644 index 000000000..c295a0e06 --- /dev/null +++ b/crux-mir/lib/core/src/slice/index.rs @@ -0,0 +1,835 @@ +//! Indexing implementations for `[T]`. + +use crate::intrinsics::assert_unsafe_precondition; +use crate::intrinsics::const_eval_select; +use crate::ops; +use crate::ptr; + +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +impl const ops::Index for [T] +where + I: ~const SliceIndex<[T]>, +{ + type Output = I::Output; + + #[inline] + fn index(&self, index: I) -> &I::Output { + index.index(self) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +impl const ops::IndexMut for [T] +where + I: ~const SliceIndex<[T]>, +{ + #[inline] + fn index_mut(&mut self, index: I) -> &mut I::Output { + index.index_mut(self) + } +} + +#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never), cold)] +#[cfg_attr(feature = "panic_immediate_abort", inline)] +#[track_caller] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +const fn slice_start_index_len_fail(index: usize, len: usize) -> ! { + // SAFETY: we are just panicking here + unsafe { + const_eval_select( + (index, len), + slice_start_index_len_fail_ct, + slice_start_index_len_fail_rt, + ) + } +} + +// FIXME const-hack +#[inline] +#[track_caller] +fn slice_start_index_len_fail_rt(index: usize, len: usize) -> ! { + panic!("range start index {index} out of range for slice of length {len}"); +} + +#[inline] +#[track_caller] +const fn slice_start_index_len_fail_ct(_: usize, _: usize) -> ! { + panic!("slice start index is out of range for slice"); +} + +#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never), cold)] +#[cfg_attr(feature = "panic_immediate_abort", inline)] +#[track_caller] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +const fn slice_end_index_len_fail(index: usize, len: usize) -> ! { + // SAFETY: we are just panicking here + unsafe { + const_eval_select((index, len), slice_end_index_len_fail_ct, slice_end_index_len_fail_rt) + } +} + +// FIXME const-hack +#[inline] +#[track_caller] +fn slice_end_index_len_fail_rt(index: usize, len: usize) -> ! { + panic!("range end index {index} out of range for slice of length {len}"); +} + +#[inline] +#[track_caller] +const fn slice_end_index_len_fail_ct(_: usize, _: usize) -> ! { + panic!("slice end index is out of range for slice"); +} + +#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never), cold)] +#[cfg_attr(feature = "panic_immediate_abort", inline)] +#[track_caller] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +const fn slice_index_order_fail(index: usize, end: usize) -> ! { + // SAFETY: we are just panicking here + unsafe { const_eval_select((index, end), slice_index_order_fail_ct, slice_index_order_fail_rt) } +} + +// FIXME const-hack +#[inline] +#[track_caller] +fn slice_index_order_fail_rt(index: usize, end: usize) -> ! { + panic!("slice index starts at {index} but ends at {end}"); +} + +#[inline] +#[track_caller] +const fn slice_index_order_fail_ct(_: usize, _: usize) -> ! { + panic!("slice index start is larger than end"); +} + +#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never), cold)] +#[cfg_attr(feature = "panic_immediate_abort", inline)] +#[track_caller] +const fn slice_start_index_overflow_fail() -> ! { + panic!("attempted to index slice from after maximum usize"); +} + +#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never), cold)] +#[cfg_attr(feature = "panic_immediate_abort", inline)] +#[track_caller] +const fn slice_end_index_overflow_fail() -> ! { + panic!("attempted to index slice up to maximum usize"); +} + +mod private_slice_index { + use super::ops; + #[stable(feature = "slice_get_slice", since = "1.28.0")] + pub trait Sealed {} + + #[stable(feature = "slice_get_slice", since = "1.28.0")] + impl Sealed for usize {} + #[stable(feature = "slice_get_slice", since = "1.28.0")] + impl Sealed for ops::Range {} + #[stable(feature = "slice_get_slice", since = "1.28.0")] + impl Sealed for ops::RangeTo {} + #[stable(feature = "slice_get_slice", since = "1.28.0")] + impl Sealed for ops::RangeFrom {} + #[stable(feature = "slice_get_slice", since = "1.28.0")] + impl Sealed for ops::RangeFull {} + #[stable(feature = "slice_get_slice", since = "1.28.0")] + impl Sealed for ops::RangeInclusive {} + #[stable(feature = "slice_get_slice", since = "1.28.0")] + impl Sealed for ops::RangeToInclusive {} + #[stable(feature = "slice_index_with_ops_bound_pair", since = "1.53.0")] + impl Sealed for (ops::Bound, ops::Bound) {} + + impl Sealed for ops::IndexRange {} +} + +/// A helper trait used for indexing operations. +/// +/// Implementations of this trait have to promise that if the argument +/// to `get_unchecked(_mut)` is a safe reference, then so is the result. +#[stable(feature = "slice_get_slice", since = "1.28.0")] +#[rustc_diagnostic_item = "SliceIndex"] +#[rustc_on_unimplemented( + on(T = "str", label = "string indices are ranges of `usize`",), + on( + all(any(T = "str", T = "&str", T = "std::string::String"), _Self = "{integer}"), + note = "you can use `.chars().nth()` or `.bytes().nth()`\n\ + for more information, see chapter 8 in The Book: \ + " + ), + message = "the type `{T}` cannot be indexed by `{Self}`", + label = "slice indices are of type `usize` or ranges of `usize`" +)] +#[const_trait] +pub unsafe trait SliceIndex: private_slice_index::Sealed { + /// The output type returned by methods. + #[stable(feature = "slice_get_slice", since = "1.28.0")] + type Output: ?Sized; + + /// Returns a shared reference to the output at this location, if in + /// bounds. + #[unstable(feature = "slice_index_methods", issue = "none")] + fn get(self, slice: &T) -> Option<&Self::Output>; + + /// Returns a mutable reference to the output at this location, if in + /// bounds. + #[unstable(feature = "slice_index_methods", issue = "none")] + fn get_mut(self, slice: &mut T) -> Option<&mut Self::Output>; + + /// Returns a shared reference to the output at this location, without + /// performing any bounds checking. + /// Calling this method with an out-of-bounds index or a dangling `slice` pointer + /// is *[undefined behavior]* even if the resulting reference is not used. + /// + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html + #[unstable(feature = "slice_index_methods", issue = "none")] + unsafe fn get_unchecked(self, slice: *const T) -> *const Self::Output; + + /// Returns a mutable reference to the output at this location, without + /// performing any bounds checking. + /// Calling this method with an out-of-bounds index or a dangling `slice` pointer + /// is *[undefined behavior]* even if the resulting reference is not used. + /// + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html + #[unstable(feature = "slice_index_methods", issue = "none")] + unsafe fn get_unchecked_mut(self, slice: *mut T) -> *mut Self::Output; + + /// Returns a shared reference to the output at this location, panicking + /// if out of bounds. + #[unstable(feature = "slice_index_methods", issue = "none")] + #[track_caller] + fn index(self, slice: &T) -> &Self::Output; + + /// Returns a mutable reference to the output at this location, panicking + /// if out of bounds. + #[unstable(feature = "slice_index_methods", issue = "none")] + #[track_caller] + fn index_mut(self, slice: &mut T) -> &mut Self::Output; +} + +#[stable(feature = "slice_get_slice_impls", since = "1.15.0")] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +unsafe impl const SliceIndex<[T]> for usize { + type Output = T; + + #[inline] + fn get(self, slice: &[T]) -> Option<&T> { + // SAFETY: `self` is checked to be in bounds. + if self < slice.len() { unsafe { Some(&*self.get_unchecked(slice)) } } else { None } + } + + #[inline] + fn get_mut(self, slice: &mut [T]) -> Option<&mut T> { + // SAFETY: `self` is checked to be in bounds. + if self < slice.len() { unsafe { Some(&mut *self.get_unchecked_mut(slice)) } } else { None } + } + + #[inline] + unsafe fn get_unchecked(self, slice: *const [T]) -> *const T { + let this = self; + // SAFETY: the caller guarantees that `slice` is not dangling, so it + // cannot be longer than `isize::MAX`. They also guarantee that + // `self` is in bounds of `slice` so `self` cannot overflow an `isize`, + // so the call to `add` is safe. + unsafe { + assert_unsafe_precondition!( + "slice::get_unchecked requires that the index is within the slice", + [T](this: usize, slice: *const [T]) => this < slice.len() + ); + slice.as_ptr().add(self) + } + } + + #[inline] + unsafe fn get_unchecked_mut(self, slice: *mut [T]) -> *mut T { + let this = self; + // SAFETY: see comments for `get_unchecked` above. + unsafe { + assert_unsafe_precondition!( + "slice::get_unchecked_mut requires that the index is within the slice", + [T](this: usize, slice: *mut [T]) => this < slice.len() + ); + slice.as_mut_ptr().add(self) + } + } + + #[inline] + fn index(self, slice: &[T]) -> &T { + // N.B., use intrinsic indexing + &(*slice)[self] + } + + #[inline] + fn index_mut(self, slice: &mut [T]) -> &mut T { + // N.B., use intrinsic indexing + &mut (*slice)[self] + } +} + +/// Because `IndexRange` guarantees `start <= end`, fewer checks are needed here +/// than there are for a general `Range` (which might be `100..3`). +#[rustc_const_unstable(feature = "const_index_range_slice_index", issue = "none")] +unsafe impl const SliceIndex<[T]> for ops::IndexRange { + type Output = [T]; + + #[inline] + fn get(self, slice: &[T]) -> Option<&[T]> { + if self.end() <= slice.len() { + // SAFETY: `self` is checked to be valid and in bounds above. + unsafe { Some(&*self.get_unchecked(slice)) } + } else { + None + } + } + + #[inline] + fn get_mut(self, slice: &mut [T]) -> Option<&mut [T]> { + if self.end() <= slice.len() { + // SAFETY: `self` is checked to be valid and in bounds above. + unsafe { Some(&mut *self.get_unchecked_mut(slice)) } + } else { + None + } + } + + #[inline] + unsafe fn get_unchecked(self, slice: *const [T]) -> *const [T] { + let end = self.end(); + // SAFETY: the caller guarantees that `slice` is not dangling, so it + // cannot be longer than `isize::MAX`. They also guarantee that + // `self` is in bounds of `slice` so `self` cannot overflow an `isize`, + // so the call to `add` is safe. + + unsafe { + assert_unsafe_precondition!( + "slice::get_unchecked requires that the index is within the slice", + [T](end: usize, slice: *const [T]) => end <= slice.len() + ); + ptr::slice_from_raw_parts(slice.as_ptr().add(self.start()), self.len()) + } + } + + #[inline] + unsafe fn get_unchecked_mut(self, slice: *mut [T]) -> *mut [T] { + let end = self.end(); + // SAFETY: see comments for `get_unchecked` above. + unsafe { + assert_unsafe_precondition!( + "slice::get_unchecked_mut requires that the index is within the slice", + [T](end: usize, slice: *mut [T]) => end <= slice.len() + ); + ptr::slice_from_raw_parts_mut(slice.as_mut_ptr().add(self.start()), self.len()) + } + } + + #[inline] + fn index(self, slice: &[T]) -> &[T] { + if self.end() <= slice.len() { + // SAFETY: `self` is checked to be valid and in bounds above. + unsafe { &*self.get_unchecked(slice) } + } else { + slice_end_index_len_fail(self.end(), slice.len()) + } + } + + #[inline] + fn index_mut(self, slice: &mut [T]) -> &mut [T] { + if self.end() <= slice.len() { + // SAFETY: `self` is checked to be valid and in bounds above. + unsafe { &mut *self.get_unchecked_mut(slice) } + } else { + slice_end_index_len_fail(self.end(), slice.len()) + } + } +} + +#[stable(feature = "slice_get_slice_impls", since = "1.15.0")] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +unsafe impl const SliceIndex<[T]> for ops::Range { + type Output = [T]; + + #[inline] + fn get(self, slice: &[T]) -> Option<&[T]> { + if self.start > self.end || self.end > slice.len() { + None + } else { + // SAFETY: `self` is checked to be valid and in bounds above. + unsafe { Some(&*self.get_unchecked(slice)) } + } + } + + #[inline] + fn get_mut(self, slice: &mut [T]) -> Option<&mut [T]> { + if self.start > self.end || self.end > slice.len() { + None + } else { + // SAFETY: `self` is checked to be valid and in bounds above. + unsafe { Some(&mut *self.get_unchecked_mut(slice)) } + } + } + + #[inline] + unsafe fn get_unchecked(self, slice: *const [T]) -> *const [T] { + let this = ops::Range { start: self.start, end: self.end }; + // SAFETY: the caller guarantees that `slice` is not dangling, so it + // cannot be longer than `isize::MAX`. They also guarantee that + // `self` is in bounds of `slice` so `self` cannot overflow an `isize`, + // so the call to `add` is safe. + + unsafe { + assert_unsafe_precondition!( + "slice::get_unchecked requires that the range is within the slice", + [T](this: ops::Range, slice: *const [T]) => + this.end >= this.start && this.end <= slice.len() + ); + ptr::slice_from_raw_parts(slice.as_ptr().add(self.start), self.end - self.start) + } + } + + #[inline] + unsafe fn get_unchecked_mut(self, slice: *mut [T]) -> *mut [T] { + let this = ops::Range { start: self.start, end: self.end }; + // SAFETY: see comments for `get_unchecked` above. + unsafe { + assert_unsafe_precondition!( + "slice::get_unchecked_mut requires that the range is within the slice", + [T](this: ops::Range, slice: *mut [T]) => + this.end >= this.start && this.end <= slice.len() + ); + ptr::slice_from_raw_parts_mut(slice.as_mut_ptr().add(self.start), self.end - self.start) + } + } + + #[inline] + fn index(self, slice: &[T]) -> &[T] { + if self.start > self.end { + slice_index_order_fail(self.start, self.end); + } else if self.end > slice.len() { + slice_end_index_len_fail(self.end, slice.len()); + } + // SAFETY: `self` is checked to be valid and in bounds above. + unsafe { &*self.get_unchecked(slice) } + } + + #[inline] + fn index_mut(self, slice: &mut [T]) -> &mut [T] { + if self.start > self.end { + slice_index_order_fail(self.start, self.end); + } else if self.end > slice.len() { + slice_end_index_len_fail(self.end, slice.len()); + } + // SAFETY: `self` is checked to be valid and in bounds above. + unsafe { &mut *self.get_unchecked_mut(slice) } + } +} + +#[stable(feature = "slice_get_slice_impls", since = "1.15.0")] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +unsafe impl const SliceIndex<[T]> for ops::RangeTo { + type Output = [T]; + + #[inline] + fn get(self, slice: &[T]) -> Option<&[T]> { + (0..self.end).get(slice) + } + + #[inline] + fn get_mut(self, slice: &mut [T]) -> Option<&mut [T]> { + (0..self.end).get_mut(slice) + } + + #[inline] + unsafe fn get_unchecked(self, slice: *const [T]) -> *const [T] { + // SAFETY: the caller has to uphold the safety contract for `get_unchecked`. + unsafe { (0..self.end).get_unchecked(slice) } + } + + #[inline] + unsafe fn get_unchecked_mut(self, slice: *mut [T]) -> *mut [T] { + // SAFETY: the caller has to uphold the safety contract for `get_unchecked_mut`. + unsafe { (0..self.end).get_unchecked_mut(slice) } + } + + #[inline] + fn index(self, slice: &[T]) -> &[T] { + (0..self.end).index(slice) + } + + #[inline] + fn index_mut(self, slice: &mut [T]) -> &mut [T] { + (0..self.end).index_mut(slice) + } +} + +#[stable(feature = "slice_get_slice_impls", since = "1.15.0")] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +unsafe impl const SliceIndex<[T]> for ops::RangeFrom { + type Output = [T]; + + #[inline] + fn get(self, slice: &[T]) -> Option<&[T]> { + (self.start..slice.len()).get(slice) + } + + #[inline] + fn get_mut(self, slice: &mut [T]) -> Option<&mut [T]> { + (self.start..slice.len()).get_mut(slice) + } + + #[inline] + unsafe fn get_unchecked(self, slice: *const [T]) -> *const [T] { + // SAFETY: the caller has to uphold the safety contract for `get_unchecked`. + unsafe { (self.start..slice.len()).get_unchecked(slice) } + } + + #[inline] + unsafe fn get_unchecked_mut(self, slice: *mut [T]) -> *mut [T] { + // SAFETY: the caller has to uphold the safety contract for `get_unchecked_mut`. + unsafe { (self.start..slice.len()).get_unchecked_mut(slice) } + } + + #[inline] + fn index(self, slice: &[T]) -> &[T] { + if self.start > slice.len() { + slice_start_index_len_fail(self.start, slice.len()); + } + // SAFETY: `self` is checked to be valid and in bounds above. + unsafe { &*self.get_unchecked(slice) } + } + + #[inline] + fn index_mut(self, slice: &mut [T]) -> &mut [T] { + if self.start > slice.len() { + slice_start_index_len_fail(self.start, slice.len()); + } + // SAFETY: `self` is checked to be valid and in bounds above. + unsafe { &mut *self.get_unchecked_mut(slice) } + } +} + +#[stable(feature = "slice_get_slice_impls", since = "1.15.0")] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +unsafe impl const SliceIndex<[T]> for ops::RangeFull { + type Output = [T]; + + #[inline] + fn get(self, slice: &[T]) -> Option<&[T]> { + Some(slice) + } + + #[inline] + fn get_mut(self, slice: &mut [T]) -> Option<&mut [T]> { + Some(slice) + } + + #[inline] + unsafe fn get_unchecked(self, slice: *const [T]) -> *const [T] { + slice + } + + #[inline] + unsafe fn get_unchecked_mut(self, slice: *mut [T]) -> *mut [T] { + slice + } + + #[inline] + fn index(self, slice: &[T]) -> &[T] { + slice + } + + #[inline] + fn index_mut(self, slice: &mut [T]) -> &mut [T] { + slice + } +} + +#[stable(feature = "inclusive_range", since = "1.26.0")] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +unsafe impl const SliceIndex<[T]> for ops::RangeInclusive { + type Output = [T]; + + #[inline] + fn get(self, slice: &[T]) -> Option<&[T]> { + if *self.end() == usize::MAX { None } else { self.into_slice_range().get(slice) } + } + + #[inline] + fn get_mut(self, slice: &mut [T]) -> Option<&mut [T]> { + if *self.end() == usize::MAX { None } else { self.into_slice_range().get_mut(slice) } + } + + #[inline] + unsafe fn get_unchecked(self, slice: *const [T]) -> *const [T] { + // SAFETY: the caller has to uphold the safety contract for `get_unchecked`. + unsafe { self.into_slice_range().get_unchecked(slice) } + } + + #[inline] + unsafe fn get_unchecked_mut(self, slice: *mut [T]) -> *mut [T] { + // SAFETY: the caller has to uphold the safety contract for `get_unchecked_mut`. + unsafe { self.into_slice_range().get_unchecked_mut(slice) } + } + + #[inline] + fn index(self, slice: &[T]) -> &[T] { + if *self.end() == usize::MAX { + slice_end_index_overflow_fail(); + } + self.into_slice_range().index(slice) + } + + #[inline] + fn index_mut(self, slice: &mut [T]) -> &mut [T] { + if *self.end() == usize::MAX { + slice_end_index_overflow_fail(); + } + self.into_slice_range().index_mut(slice) + } +} + +#[stable(feature = "inclusive_range", since = "1.26.0")] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +unsafe impl const SliceIndex<[T]> for ops::RangeToInclusive { + type Output = [T]; + + #[inline] + fn get(self, slice: &[T]) -> Option<&[T]> { + (0..=self.end).get(slice) + } + + #[inline] + fn get_mut(self, slice: &mut [T]) -> Option<&mut [T]> { + (0..=self.end).get_mut(slice) + } + + #[inline] + unsafe fn get_unchecked(self, slice: *const [T]) -> *const [T] { + // SAFETY: the caller has to uphold the safety contract for `get_unchecked`. + unsafe { (0..=self.end).get_unchecked(slice) } + } + + #[inline] + unsafe fn get_unchecked_mut(self, slice: *mut [T]) -> *mut [T] { + // SAFETY: the caller has to uphold the safety contract for `get_unchecked_mut`. + unsafe { (0..=self.end).get_unchecked_mut(slice) } + } + + #[inline] + fn index(self, slice: &[T]) -> &[T] { + (0..=self.end).index(slice) + } + + #[inline] + fn index_mut(self, slice: &mut [T]) -> &mut [T] { + (0..=self.end).index_mut(slice) + } +} + +/// Performs bounds-checking of a range. +/// +/// This method is similar to [`Index::index`] for slices, but it returns a +/// [`Range`] equivalent to `range`. You can use this method to turn any range +/// into `start` and `end` values. +/// +/// `bounds` is the range of the slice to use for bounds-checking. It should +/// be a [`RangeTo`] range that ends at the length of the slice. +/// +/// The returned [`Range`] is safe to pass to [`slice::get_unchecked`] and +/// [`slice::get_unchecked_mut`] for slices with the given range. +/// +/// [`Range`]: ops::Range +/// [`RangeTo`]: ops::RangeTo +/// [`slice::get_unchecked`]: slice::get_unchecked +/// [`slice::get_unchecked_mut`]: slice::get_unchecked_mut +/// +/// # Panics +/// +/// Panics if `range` would be out of bounds. +/// +/// # Examples +/// +/// ``` +/// #![feature(slice_range)] +/// +/// use std::slice; +/// +/// let v = [10, 40, 30]; +/// assert_eq!(1..2, slice::range(1..2, ..v.len())); +/// assert_eq!(0..2, slice::range(..2, ..v.len())); +/// assert_eq!(1..3, slice::range(1.., ..v.len())); +/// ``` +/// +/// Panics when [`Index::index`] would panic: +/// +/// ```should_panic +/// #![feature(slice_range)] +/// +/// use std::slice; +/// +/// let _ = slice::range(2..1, ..3); +/// ``` +/// +/// ```should_panic +/// #![feature(slice_range)] +/// +/// use std::slice; +/// +/// let _ = slice::range(1..4, ..3); +/// ``` +/// +/// ```should_panic +/// #![feature(slice_range)] +/// +/// use std::slice; +/// +/// let _ = slice::range(1..=usize::MAX, ..3); +/// ``` +/// +/// [`Index::index`]: ops::Index::index +#[track_caller] +#[unstable(feature = "slice_range", issue = "76393")] +#[must_use] +pub fn range(range: R, bounds: ops::RangeTo) -> ops::Range +where + R: ops::RangeBounds, +{ + let len = bounds.end; + + let start: ops::Bound<&usize> = range.start_bound(); + let start = match start { + ops::Bound::Included(&start) => start, + ops::Bound::Excluded(start) => { + start.checked_add(1).unwrap_or_else(|| slice_start_index_overflow_fail()) + } + ops::Bound::Unbounded => 0, + }; + + let end: ops::Bound<&usize> = range.end_bound(); + let end = match end { + ops::Bound::Included(end) => { + end.checked_add(1).unwrap_or_else(|| slice_end_index_overflow_fail()) + } + ops::Bound::Excluded(&end) => end, + ops::Bound::Unbounded => len, + }; + + if start > end { + slice_index_order_fail(start, end); + } + if end > len { + slice_end_index_len_fail(end, len); + } + + ops::Range { start, end } +} + +/// Convert pair of `ops::Bound`s into `ops::Range` without performing any bounds checking and (in debug) overflow checking +fn into_range_unchecked( + len: usize, + (start, end): (ops::Bound, ops::Bound), +) -> ops::Range { + use ops::Bound; + let start = match start { + Bound::Included(i) => i, + Bound::Excluded(i) => i + 1, + Bound::Unbounded => 0, + }; + let end = match end { + Bound::Included(i) => i + 1, + Bound::Excluded(i) => i, + Bound::Unbounded => len, + }; + start..end +} + +/// Convert pair of `ops::Bound`s into `ops::Range`. +/// Returns `None` on overflowing indices. +fn into_range( + len: usize, + (start, end): (ops::Bound, ops::Bound), +) -> Option> { + use ops::Bound; + let start = match start { + Bound::Included(start) => start, + Bound::Excluded(start) => start.checked_add(1)?, + Bound::Unbounded => 0, + }; + + let end = match end { + Bound::Included(end) => end.checked_add(1)?, + Bound::Excluded(end) => end, + Bound::Unbounded => len, + }; + + // Don't bother with checking `start < end` and `end <= len` + // since these checks are handled by `Range` impls + + Some(start..end) +} + +/// Convert pair of `ops::Bound`s into `ops::Range`. +/// Panics on overflowing indices. +fn into_slice_range( + len: usize, + (start, end): (ops::Bound, ops::Bound), +) -> ops::Range { + use ops::Bound; + let start = match start { + Bound::Included(start) => start, + Bound::Excluded(start) => { + start.checked_add(1).unwrap_or_else(|| slice_start_index_overflow_fail()) + } + Bound::Unbounded => 0, + }; + + let end = match end { + Bound::Included(end) => { + end.checked_add(1).unwrap_or_else(|| slice_end_index_overflow_fail()) + } + Bound::Excluded(end) => end, + Bound::Unbounded => len, + }; + + // Don't bother with checking `start < end` and `end <= len` + // since these checks are handled by `Range` impls + + start..end +} + +#[stable(feature = "slice_index_with_ops_bound_pair", since = "1.53.0")] +unsafe impl SliceIndex<[T]> for (ops::Bound, ops::Bound) { + type Output = [T]; + + #[inline] + fn get(self, slice: &[T]) -> Option<&Self::Output> { + into_range(slice.len(), self)?.get(slice) + } + + #[inline] + fn get_mut(self, slice: &mut [T]) -> Option<&mut Self::Output> { + into_range(slice.len(), self)?.get_mut(slice) + } + + #[inline] + unsafe fn get_unchecked(self, slice: *const [T]) -> *const Self::Output { + // SAFETY: the caller has to uphold the safety contract for `get_unchecked`. + unsafe { into_range_unchecked(slice.len(), self).get_unchecked(slice) } + } + + #[inline] + unsafe fn get_unchecked_mut(self, slice: *mut [T]) -> *mut Self::Output { + // SAFETY: the caller has to uphold the safety contract for `get_unchecked_mut`. + unsafe { into_range_unchecked(slice.len(), self).get_unchecked_mut(slice) } + } + + #[inline] + fn index(self, slice: &[T]) -> &Self::Output { + into_slice_range(slice.len(), self).index(slice) + } + + #[inline] + fn index_mut(self, slice: &mut [T]) -> &mut Self::Output { + into_slice_range(slice.len(), self).index_mut(slice) + } +} diff --git a/crux-mir/lib/core/src/slice/iter.rs b/crux-mir/lib/core/src/slice/iter.rs new file mode 100644 index 000000000..90ab43d12 --- /dev/null +++ b/crux-mir/lib/core/src/slice/iter.rs @@ -0,0 +1,3409 @@ +//! Definitions of a bunch of iterators for `[T]`. + +#[macro_use] // import iterator! and forward_iterator! +mod macros; + +use crate::cmp; +use crate::cmp::Ordering; +use crate::fmt; +use crate::intrinsics::assume; +use crate::iter::{FusedIterator, TrustedLen, TrustedRandomAccess, TrustedRandomAccessNoCoerce}; +use crate::marker::{PhantomData, Send, Sized, Sync}; +use crate::mem::{self, SizedTypeProperties}; +use crate::num::NonZeroUsize; +use crate::ptr::NonNull; + +use super::{from_raw_parts, from_raw_parts_mut}; + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T> IntoIterator for &'a [T] { + type Item = &'a T; + type IntoIter = Iter<'a, T>; + + fn into_iter(self) -> Iter<'a, T> { + self.iter() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T> IntoIterator for &'a mut [T] { + type Item = &'a mut T; + type IntoIter = IterMut<'a, T>; + + fn into_iter(self) -> IterMut<'a, T> { + self.iter_mut() + } +} + +/// Immutable slice iterator +/// +/// This struct is created by the [`iter`] method on [slices]. +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// // First, we declare a type which has `iter` method to get the `Iter` struct (`&[usize]` here): +/// let slice = &[1, 2, 3]; +/// +/// // Then, we iterate over it: +/// for element in slice.iter() { +/// println!("{element}"); +/// } +/// ``` +/// +/// [`iter`]: slice::iter +/// [slices]: slice +#[stable(feature = "rust1", since = "1.0.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct Iter<'a, T: 'a> { + ptr: NonNull, + end: *const T, // If T is a ZST, this is actually ptr+len. This encoding is picked so that + // ptr == end is a quick test for the Iterator being empty, that works + // for both ZST and non-ZST. + _marker: PhantomData<&'a T>, +} + +#[stable(feature = "core_impl_debug", since = "1.9.0")] +impl fmt::Debug for Iter<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("Iter").field(&self.as_slice()).finish() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +unsafe impl Sync for Iter<'_, T> {} +#[stable(feature = "rust1", since = "1.0.0")] +unsafe impl Send for Iter<'_, T> {} + +impl<'a, T> Iter<'a, T> { + #[inline] + pub(super) fn new(slice: &'a [T]) -> Self { + let ptr = slice.as_ptr(); + // SAFETY: Similar to `IterMut::new`. + unsafe { + assume(!ptr.is_null()); + + let end = + if T::IS_ZST { ptr.wrapping_byte_add(slice.len()) } else { ptr.add(slice.len()) }; + + Self { ptr: NonNull::new_unchecked(ptr as *mut T), end, _marker: PhantomData } + } + } + + /// Views the underlying data as a subslice of the original data. + /// + /// This has the same lifetime as the original slice, and so the + /// iterator can continue to be used while this exists. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// // First, we declare a type which has the `iter` method to get the `Iter` + /// // struct (`&[usize]` here): + /// let slice = &[1, 2, 3]; + /// + /// // Then, we get the iterator: + /// let mut iter = slice.iter(); + /// // So if we print what `as_slice` method returns here, we have "[1, 2, 3]": + /// println!("{:?}", iter.as_slice()); + /// + /// // Next, we move to the second element of the slice: + /// iter.next(); + /// // Now `as_slice` returns "[2, 3]": + /// println!("{:?}", iter.as_slice()); + /// ``` + #[must_use] + #[stable(feature = "iter_to_slice", since = "1.4.0")] + #[inline] + pub fn as_slice(&self) -> &'a [T] { + self.make_slice() + } +} + +iterator! {struct Iter -> *const T, &'a T, const, {/* no mut */}, { + fn is_sorted_by(self, mut compare: F) -> bool + where + Self: Sized, + F: FnMut(&Self::Item, &Self::Item) -> Option, + { + self.as_slice().windows(2).all(|w| { + compare(&&w[0], &&w[1]).map(|o| o != Ordering::Greater).unwrap_or(false) + }) + } +}} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Clone for Iter<'_, T> { + #[inline] + fn clone(&self) -> Self { + Iter { ptr: self.ptr, end: self.end, _marker: self._marker } + } +} + +#[stable(feature = "slice_iter_as_ref", since = "1.13.0")] +impl AsRef<[T]> for Iter<'_, T> { + #[inline] + fn as_ref(&self) -> &[T] { + self.as_slice() + } +} + +/// Mutable slice iterator. +/// +/// This struct is created by the [`iter_mut`] method on [slices]. +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// // First, we declare a type which has `iter_mut` method to get the `IterMut` +/// // struct (`&[usize]` here): +/// let mut slice = &mut [1, 2, 3]; +/// +/// // Then, we iterate over it and increment each element value: +/// for element in slice.iter_mut() { +/// *element += 1; +/// } +/// +/// // We now have "[2, 3, 4]": +/// println!("{slice:?}"); +/// ``` +/// +/// [`iter_mut`]: slice::iter_mut +/// [slices]: slice +#[stable(feature = "rust1", since = "1.0.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct IterMut<'a, T: 'a> { + ptr: NonNull, + end: *mut T, // If T is a ZST, this is actually ptr+len. This encoding is picked so that + // ptr == end is a quick test for the Iterator being empty, that works + // for both ZST and non-ZST. + _marker: PhantomData<&'a mut T>, +} + +#[stable(feature = "core_impl_debug", since = "1.9.0")] +impl fmt::Debug for IterMut<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("IterMut").field(&self.make_slice()).finish() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +unsafe impl Sync for IterMut<'_, T> {} +#[stable(feature = "rust1", since = "1.0.0")] +unsafe impl Send for IterMut<'_, T> {} + +impl<'a, T> IterMut<'a, T> { + #[inline] + pub(super) fn new(slice: &'a mut [T]) -> Self { + let ptr = slice.as_mut_ptr(); + // SAFETY: There are several things here: + // + // `ptr` has been obtained by `slice.as_ptr()` where `slice` is a valid + // reference thus it is non-NUL and safe to use and pass to + // `NonNull::new_unchecked` . + // + // Adding `slice.len()` to the starting pointer gives a pointer + // at the end of `slice`. `end` will never be dereferenced, only checked + // for direct pointer equality with `ptr` to check if the iterator is + // done. + // + // In the case of a ZST, the end pointer is just the start pointer plus + // the length, to also allows for the fast `ptr == end` check. + // + // See the `next_unchecked!` and `is_empty!` macros as well as the + // `post_inc_start` method for more information. + unsafe { + assume(!ptr.is_null()); + + let end = + if T::IS_ZST { ptr.wrapping_byte_add(slice.len()) } else { ptr.add(slice.len()) }; + + Self { ptr: NonNull::new_unchecked(ptr), end, _marker: PhantomData } + } + } + + /// Views the underlying data as a subslice of the original data. + /// + /// To avoid creating `&mut` references that alias, this is forced + /// to consume the iterator. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// // First, we declare a type which has `iter_mut` method to get the `IterMut` + /// // struct (`&[usize]` here): + /// let mut slice = &mut [1, 2, 3]; + /// + /// { + /// // Then, we get the iterator: + /// let mut iter = slice.iter_mut(); + /// // We move to next element: + /// iter.next(); + /// // So if we print what `into_slice` method returns here, we have "[2, 3]": + /// println!("{:?}", iter.into_slice()); + /// } + /// + /// // Now let's modify a value of the slice: + /// { + /// // First we get back the iterator: + /// let mut iter = slice.iter_mut(); + /// // We change the value of the first element of the slice returned by the `next` method: + /// *iter.next().unwrap() += 1; + /// } + /// // Now slice is "[2, 2, 3]": + /// println!("{slice:?}"); + /// ``` + #[must_use = "`self` will be dropped if the result is not used"] + #[stable(feature = "iter_to_slice", since = "1.4.0")] + pub fn into_slice(self) -> &'a mut [T] { + // SAFETY: the iterator was created from a mutable slice with pointer + // `self.ptr` and length `len!(self)`. This guarantees that all the prerequisites + // for `from_raw_parts_mut` are fulfilled. + unsafe { from_raw_parts_mut(self.ptr.as_ptr(), len!(self)) } + } + + /// Views the underlying data as a subslice of the original data. + /// + /// To avoid creating `&mut [T]` references that alias, the returned slice + /// borrows its lifetime from the iterator the method is applied on. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// let mut slice: &mut [usize] = &mut [1, 2, 3]; + /// + /// // First, we get the iterator: + /// let mut iter = slice.iter_mut(); + /// // So if we check what the `as_slice` method returns here, we have "[1, 2, 3]": + /// assert_eq!(iter.as_slice(), &[1, 2, 3]); + /// + /// // Next, we move to the second element of the slice: + /// iter.next(); + /// // Now `as_slice` returns "[2, 3]": + /// assert_eq!(iter.as_slice(), &[2, 3]); + /// ``` + #[must_use] + #[stable(feature = "slice_iter_mut_as_slice", since = "1.53.0")] + #[inline] + pub fn as_slice(&self) -> &[T] { + self.make_slice() + } + + /// Views the underlying data as a mutable subslice of the original data. + /// + /// To avoid creating `&mut [T]` references that alias, the returned slice + /// borrows its lifetime from the iterator the method is applied on. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(slice_iter_mut_as_mut_slice)] + /// + /// let mut slice: &mut [usize] = &mut [1, 2, 3]; + /// + /// // First, we get the iterator: + /// let mut iter = slice.iter_mut(); + /// // Then, we get a mutable slice from it: + /// let mut_slice = iter.as_mut_slice(); + /// // So if we check what the `as_mut_slice` method returned, we have "[1, 2, 3]": + /// assert_eq!(mut_slice, &mut [1, 2, 3]); + /// + /// // We can use it to mutate the slice: + /// mut_slice[0] = 4; + /// mut_slice[2] = 5; + /// + /// // Next, we can move to the second element of the slice, checking that + /// // it yields the value we just wrote: + /// assert_eq!(iter.next(), Some(&mut 4)); + /// // Now `as_mut_slice` returns "[2, 5]": + /// assert_eq!(iter.as_mut_slice(), &mut [2, 5]); + /// ``` + #[must_use] + // FIXME: Uncomment the `AsMut<[T]>` impl when this gets stabilized. + #[unstable(feature = "slice_iter_mut_as_mut_slice", issue = "93079")] + pub fn as_mut_slice(&mut self) -> &mut [T] { + // SAFETY: the iterator was created from a mutable slice with pointer + // `self.ptr` and length `len!(self)`. This guarantees that all the prerequisites + // for `from_raw_parts_mut` are fulfilled. + unsafe { from_raw_parts_mut(self.ptr.as_ptr(), len!(self)) } + } +} + +#[stable(feature = "slice_iter_mut_as_slice", since = "1.53.0")] +impl AsRef<[T]> for IterMut<'_, T> { + #[inline] + fn as_ref(&self) -> &[T] { + self.as_slice() + } +} + +// #[stable(feature = "slice_iter_mut_as_mut_slice", since = "FIXME")] +// impl AsMut<[T]> for IterMut<'_, T> { +// fn as_mut(&mut self) -> &mut [T] { +// self.as_mut_slice() +// } +// } + +iterator! {struct IterMut -> *mut T, &'a mut T, mut, {mut}, {}} + +/// An internal abstraction over the splitting iterators, so that +/// splitn, splitn_mut etc can be implemented once. +#[doc(hidden)] +pub(super) trait SplitIter: DoubleEndedIterator { + /// Marks the underlying iterator as complete, extracting the remaining + /// portion of the slice. + fn finish(&mut self) -> Option; +} + +/// An iterator over subslices separated by elements that match a predicate +/// function. +/// +/// This struct is created by the [`split`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// let slice = [10, 40, 33, 20]; +/// let mut iter = slice.split(|num| num % 3 == 0); +/// ``` +/// +/// [`split`]: slice::split +/// [slices]: slice +#[stable(feature = "rust1", since = "1.0.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct Split<'a, T: 'a, P> +where + P: FnMut(&T) -> bool, +{ + // Used for `SplitWhitespace` and `SplitAsciiWhitespace` `as_str` methods + pub(crate) v: &'a [T], + pred: P, + // Used for `SplitAsciiWhitespace` `as_str` method + pub(crate) finished: bool, +} + +impl<'a, T: 'a, P: FnMut(&T) -> bool> Split<'a, T, P> { + #[inline] + pub(super) fn new(slice: &'a [T], pred: P) -> Self { + Self { v: slice, pred, finished: false } + } + /// Returns a slice which contains items not yet handled by split. + /// # Example + /// + /// ``` + /// #![feature(split_as_slice)] + /// let slice = [1,2,3,4,5]; + /// let mut split = slice.split(|v| v % 2 == 0); + /// assert!(split.next().is_some()); + /// assert_eq!(split.as_slice(), &[3,4,5]); + /// ``` + #[unstable(feature = "split_as_slice", issue = "96137")] + pub fn as_slice(&self) -> &'a [T] { + if self.finished { &[] } else { &self.v } + } +} + +#[stable(feature = "core_impl_debug", since = "1.9.0")] +impl fmt::Debug for Split<'_, T, P> +where + P: FnMut(&T) -> bool, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Split").field("v", &self.v).field("finished", &self.finished).finish() + } +} + +// FIXME(#26925) Remove in favor of `#[derive(Clone)]` +#[stable(feature = "rust1", since = "1.0.0")] +impl Clone for Split<'_, T, P> +where + P: Clone + FnMut(&T) -> bool, +{ + fn clone(&self) -> Self { + Split { v: self.v, pred: self.pred.clone(), finished: self.finished } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T, P> Iterator for Split<'a, T, P> +where + P: FnMut(&T) -> bool, +{ + type Item = &'a [T]; + + #[inline] + fn next(&mut self) -> Option<&'a [T]> { + if self.finished { + return None; + } + + match self.v.iter().position(|x| (self.pred)(x)) { + None => self.finish(), + Some(idx) => { + let ret = Some(&self.v[..idx]); + self.v = &self.v[idx + 1..]; + ret + } + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + if self.finished { + (0, Some(0)) + } else { + // If the predicate doesn't match anything, we yield one slice. + // If it matches every element, we yield `len() + 1` empty slices. + (1, Some(self.v.len() + 1)) + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T, P> DoubleEndedIterator for Split<'a, T, P> +where + P: FnMut(&T) -> bool, +{ + #[inline] + fn next_back(&mut self) -> Option<&'a [T]> { + if self.finished { + return None; + } + + match self.v.iter().rposition(|x| (self.pred)(x)) { + None => self.finish(), + Some(idx) => { + let ret = Some(&self.v[idx + 1..]); + self.v = &self.v[..idx]; + ret + } + } + } +} + +impl<'a, T, P> SplitIter for Split<'a, T, P> +where + P: FnMut(&T) -> bool, +{ + #[inline] + fn finish(&mut self) -> Option<&'a [T]> { + if self.finished { + None + } else { + self.finished = true; + Some(self.v) + } + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Split<'_, T, P> where P: FnMut(&T) -> bool {} + +/// An iterator over subslices separated by elements that match a predicate +/// function. Unlike `Split`, it contains the matched part as a terminator +/// of the subslice. +/// +/// This struct is created by the [`split_inclusive`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// let slice = [10, 40, 33, 20]; +/// let mut iter = slice.split_inclusive(|num| num % 3 == 0); +/// ``` +/// +/// [`split_inclusive`]: slice::split_inclusive +/// [slices]: slice +#[stable(feature = "split_inclusive", since = "1.51.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct SplitInclusive<'a, T: 'a, P> +where + P: FnMut(&T) -> bool, +{ + v: &'a [T], + pred: P, + finished: bool, +} + +impl<'a, T: 'a, P: FnMut(&T) -> bool> SplitInclusive<'a, T, P> { + #[inline] + pub(super) fn new(slice: &'a [T], pred: P) -> Self { + let finished = slice.is_empty(); + Self { v: slice, pred, finished } + } +} + +#[stable(feature = "split_inclusive", since = "1.51.0")] +impl fmt::Debug for SplitInclusive<'_, T, P> +where + P: FnMut(&T) -> bool, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SplitInclusive") + .field("v", &self.v) + .field("finished", &self.finished) + .finish() + } +} + +// FIXME(#26925) Remove in favor of `#[derive(Clone)]` +#[stable(feature = "split_inclusive", since = "1.51.0")] +impl Clone for SplitInclusive<'_, T, P> +where + P: Clone + FnMut(&T) -> bool, +{ + fn clone(&self) -> Self { + SplitInclusive { v: self.v, pred: self.pred.clone(), finished: self.finished } + } +} + +#[stable(feature = "split_inclusive", since = "1.51.0")] +impl<'a, T, P> Iterator for SplitInclusive<'a, T, P> +where + P: FnMut(&T) -> bool, +{ + type Item = &'a [T]; + + #[inline] + fn next(&mut self) -> Option<&'a [T]> { + if self.finished { + return None; + } + + let idx = + self.v.iter().position(|x| (self.pred)(x)).map(|idx| idx + 1).unwrap_or(self.v.len()); + if idx == self.v.len() { + self.finished = true; + } + let ret = Some(&self.v[..idx]); + self.v = &self.v[idx..]; + ret + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + if self.finished { + (0, Some(0)) + } else { + // If the predicate doesn't match anything, we yield one slice. + // If it matches every element, we yield `len()` one-element slices, + // or a single empty slice. + (1, Some(cmp::max(1, self.v.len()))) + } + } +} + +#[stable(feature = "split_inclusive", since = "1.51.0")] +impl<'a, T, P> DoubleEndedIterator for SplitInclusive<'a, T, P> +where + P: FnMut(&T) -> bool, +{ + #[inline] + fn next_back(&mut self) -> Option<&'a [T]> { + if self.finished { + return None; + } + + // The last index of self.v is already checked and found to match + // by the last iteration, so we start searching a new match + // one index to the left. + let remainder = if self.v.is_empty() { &[] } else { &self.v[..(self.v.len() - 1)] }; + let idx = remainder.iter().rposition(|x| (self.pred)(x)).map(|idx| idx + 1).unwrap_or(0); + if idx == 0 { + self.finished = true; + } + let ret = Some(&self.v[idx..]); + self.v = &self.v[..idx]; + ret + } +} + +#[stable(feature = "split_inclusive", since = "1.51.0")] +impl FusedIterator for SplitInclusive<'_, T, P> where P: FnMut(&T) -> bool {} + +/// An iterator over the mutable subslices of the vector which are separated +/// by elements that match `pred`. +/// +/// This struct is created by the [`split_mut`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// let mut v = [10, 40, 30, 20, 60, 50]; +/// let iter = v.split_mut(|num| *num % 3 == 0); +/// ``` +/// +/// [`split_mut`]: slice::split_mut +/// [slices]: slice +#[stable(feature = "rust1", since = "1.0.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct SplitMut<'a, T: 'a, P> +where + P: FnMut(&T) -> bool, +{ + v: &'a mut [T], + pred: P, + finished: bool, +} + +impl<'a, T: 'a, P: FnMut(&T) -> bool> SplitMut<'a, T, P> { + #[inline] + pub(super) fn new(slice: &'a mut [T], pred: P) -> Self { + Self { v: slice, pred, finished: false } + } +} + +#[stable(feature = "core_impl_debug", since = "1.9.0")] +impl fmt::Debug for SplitMut<'_, T, P> +where + P: FnMut(&T) -> bool, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SplitMut").field("v", &self.v).field("finished", &self.finished).finish() + } +} + +impl<'a, T, P> SplitIter for SplitMut<'a, T, P> +where + P: FnMut(&T) -> bool, +{ + #[inline] + fn finish(&mut self) -> Option<&'a mut [T]> { + if self.finished { + None + } else { + self.finished = true; + Some(mem::replace(&mut self.v, &mut [])) + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T, P> Iterator for SplitMut<'a, T, P> +where + P: FnMut(&T) -> bool, +{ + type Item = &'a mut [T]; + + #[inline] + fn next(&mut self) -> Option<&'a mut [T]> { + if self.finished { + return None; + } + + match self.v.iter().position(|x| (self.pred)(x)) { + None => self.finish(), + Some(idx) => { + let tmp = mem::take(&mut self.v); + // idx is the index of the element we are splitting on. We want to set self to the + // region after idx, and return the subslice before and not including idx. + // So first we split after idx + let (head, tail) = tmp.split_at_mut(idx + 1); + self.v = tail; + // Then return the subslice up to but not including the found element + Some(&mut head[..idx]) + } + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + if self.finished { + (0, Some(0)) + } else { + // If the predicate doesn't match anything, we yield one slice. + // If it matches every element, we yield `len() + 1` empty slices. + (1, Some(self.v.len() + 1)) + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T, P> DoubleEndedIterator for SplitMut<'a, T, P> +where + P: FnMut(&T) -> bool, +{ + #[inline] + fn next_back(&mut self) -> Option<&'a mut [T]> { + if self.finished { + return None; + } + + let idx_opt = { + // work around borrowck limitations + let pred = &mut self.pred; + self.v.iter().rposition(|x| (*pred)(x)) + }; + match idx_opt { + None => self.finish(), + Some(idx) => { + let tmp = mem::replace(&mut self.v, &mut []); + let (head, tail) = tmp.split_at_mut(idx); + self.v = head; + Some(&mut tail[1..]) + } + } + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for SplitMut<'_, T, P> where P: FnMut(&T) -> bool {} + +/// An iterator over the mutable subslices of the vector which are separated +/// by elements that match `pred`. Unlike `SplitMut`, it contains the matched +/// parts in the ends of the subslices. +/// +/// This struct is created by the [`split_inclusive_mut`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// let mut v = [10, 40, 30, 20, 60, 50]; +/// let iter = v.split_inclusive_mut(|num| *num % 3 == 0); +/// ``` +/// +/// [`split_inclusive_mut`]: slice::split_inclusive_mut +/// [slices]: slice +#[stable(feature = "split_inclusive", since = "1.51.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct SplitInclusiveMut<'a, T: 'a, P> +where + P: FnMut(&T) -> bool, +{ + v: &'a mut [T], + pred: P, + finished: bool, +} + +impl<'a, T: 'a, P: FnMut(&T) -> bool> SplitInclusiveMut<'a, T, P> { + #[inline] + pub(super) fn new(slice: &'a mut [T], pred: P) -> Self { + let finished = slice.is_empty(); + Self { v: slice, pred, finished } + } +} + +#[stable(feature = "split_inclusive", since = "1.51.0")] +impl fmt::Debug for SplitInclusiveMut<'_, T, P> +where + P: FnMut(&T) -> bool, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SplitInclusiveMut") + .field("v", &self.v) + .field("finished", &self.finished) + .finish() + } +} + +#[stable(feature = "split_inclusive", since = "1.51.0")] +impl<'a, T, P> Iterator for SplitInclusiveMut<'a, T, P> +where + P: FnMut(&T) -> bool, +{ + type Item = &'a mut [T]; + + #[inline] + fn next(&mut self) -> Option<&'a mut [T]> { + if self.finished { + return None; + } + + let idx_opt = { + // work around borrowck limitations + let pred = &mut self.pred; + self.v.iter().position(|x| (*pred)(x)) + }; + let idx = idx_opt.map(|idx| idx + 1).unwrap_or(self.v.len()); + if idx == self.v.len() { + self.finished = true; + } + let tmp = mem::replace(&mut self.v, &mut []); + let (head, tail) = tmp.split_at_mut(idx); + self.v = tail; + Some(head) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + if self.finished { + (0, Some(0)) + } else { + // If the predicate doesn't match anything, we yield one slice. + // If it matches every element, we yield `len()` one-element slices, + // or a single empty slice. + (1, Some(cmp::max(1, self.v.len()))) + } + } +} + +#[stable(feature = "split_inclusive", since = "1.51.0")] +impl<'a, T, P> DoubleEndedIterator for SplitInclusiveMut<'a, T, P> +where + P: FnMut(&T) -> bool, +{ + #[inline] + fn next_back(&mut self) -> Option<&'a mut [T]> { + if self.finished { + return None; + } + + let idx_opt = if self.v.is_empty() { + None + } else { + // work around borrowck limitations + let pred = &mut self.pred; + + // The last index of self.v is already checked and found to match + // by the last iteration, so we start searching a new match + // one index to the left. + let remainder = &self.v[..(self.v.len() - 1)]; + remainder.iter().rposition(|x| (*pred)(x)) + }; + let idx = idx_opt.map(|idx| idx + 1).unwrap_or(0); + if idx == 0 { + self.finished = true; + } + let tmp = mem::replace(&mut self.v, &mut []); + let (head, tail) = tmp.split_at_mut(idx); + self.v = head; + Some(tail) + } +} + +#[stable(feature = "split_inclusive", since = "1.51.0")] +impl FusedIterator for SplitInclusiveMut<'_, T, P> where P: FnMut(&T) -> bool {} + +/// An iterator over subslices separated by elements that match a predicate +/// function, starting from the end of the slice. +/// +/// This struct is created by the [`rsplit`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// let slice = [11, 22, 33, 0, 44, 55]; +/// let iter = slice.rsplit(|num| *num == 0); +/// ``` +/// +/// [`rsplit`]: slice::rsplit +/// [slices]: slice +#[stable(feature = "slice_rsplit", since = "1.27.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct RSplit<'a, T: 'a, P> +where + P: FnMut(&T) -> bool, +{ + inner: Split<'a, T, P>, +} + +impl<'a, T: 'a, P: FnMut(&T) -> bool> RSplit<'a, T, P> { + #[inline] + pub(super) fn new(slice: &'a [T], pred: P) -> Self { + Self { inner: Split::new(slice, pred) } + } +} + +#[stable(feature = "slice_rsplit", since = "1.27.0")] +impl fmt::Debug for RSplit<'_, T, P> +where + P: FnMut(&T) -> bool, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RSplit") + .field("v", &self.inner.v) + .field("finished", &self.inner.finished) + .finish() + } +} + +// FIXME(#26925) Remove in favor of `#[derive(Clone)]` +#[stable(feature = "slice_rsplit", since = "1.27.0")] +impl Clone for RSplit<'_, T, P> +where + P: Clone + FnMut(&T) -> bool, +{ + fn clone(&self) -> Self { + RSplit { inner: self.inner.clone() } + } +} + +#[stable(feature = "slice_rsplit", since = "1.27.0")] +impl<'a, T, P> Iterator for RSplit<'a, T, P> +where + P: FnMut(&T) -> bool, +{ + type Item = &'a [T]; + + #[inline] + fn next(&mut self) -> Option<&'a [T]> { + self.inner.next_back() + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} + +#[stable(feature = "slice_rsplit", since = "1.27.0")] +impl<'a, T, P> DoubleEndedIterator for RSplit<'a, T, P> +where + P: FnMut(&T) -> bool, +{ + #[inline] + fn next_back(&mut self) -> Option<&'a [T]> { + self.inner.next() + } +} + +#[stable(feature = "slice_rsplit", since = "1.27.0")] +impl<'a, T, P> SplitIter for RSplit<'a, T, P> +where + P: FnMut(&T) -> bool, +{ + #[inline] + fn finish(&mut self) -> Option<&'a [T]> { + self.inner.finish() + } +} + +#[stable(feature = "slice_rsplit", since = "1.27.0")] +impl FusedIterator for RSplit<'_, T, P> where P: FnMut(&T) -> bool {} + +/// An iterator over the subslices of the vector which are separated +/// by elements that match `pred`, starting from the end of the slice. +/// +/// This struct is created by the [`rsplit_mut`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// let mut slice = [11, 22, 33, 0, 44, 55]; +/// let iter = slice.rsplit_mut(|num| *num == 0); +/// ``` +/// +/// [`rsplit_mut`]: slice::rsplit_mut +/// [slices]: slice +#[stable(feature = "slice_rsplit", since = "1.27.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct RSplitMut<'a, T: 'a, P> +where + P: FnMut(&T) -> bool, +{ + inner: SplitMut<'a, T, P>, +} + +impl<'a, T: 'a, P: FnMut(&T) -> bool> RSplitMut<'a, T, P> { + #[inline] + pub(super) fn new(slice: &'a mut [T], pred: P) -> Self { + Self { inner: SplitMut::new(slice, pred) } + } +} + +#[stable(feature = "slice_rsplit", since = "1.27.0")] +impl fmt::Debug for RSplitMut<'_, T, P> +where + P: FnMut(&T) -> bool, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RSplitMut") + .field("v", &self.inner.v) + .field("finished", &self.inner.finished) + .finish() + } +} + +#[stable(feature = "slice_rsplit", since = "1.27.0")] +impl<'a, T, P> SplitIter for RSplitMut<'a, T, P> +where + P: FnMut(&T) -> bool, +{ + #[inline] + fn finish(&mut self) -> Option<&'a mut [T]> { + self.inner.finish() + } +} + +#[stable(feature = "slice_rsplit", since = "1.27.0")] +impl<'a, T, P> Iterator for RSplitMut<'a, T, P> +where + P: FnMut(&T) -> bool, +{ + type Item = &'a mut [T]; + + #[inline] + fn next(&mut self) -> Option<&'a mut [T]> { + self.inner.next_back() + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} + +#[stable(feature = "slice_rsplit", since = "1.27.0")] +impl<'a, T, P> DoubleEndedIterator for RSplitMut<'a, T, P> +where + P: FnMut(&T) -> bool, +{ + #[inline] + fn next_back(&mut self) -> Option<&'a mut [T]> { + self.inner.next() + } +} + +#[stable(feature = "slice_rsplit", since = "1.27.0")] +impl FusedIterator for RSplitMut<'_, T, P> where P: FnMut(&T) -> bool {} + +/// An private iterator over subslices separated by elements that +/// match a predicate function, splitting at most a fixed number of +/// times. +#[derive(Debug)] +struct GenericSplitN { + iter: I, + count: usize, +} + +impl> Iterator for GenericSplitN { + type Item = T; + + #[inline] + fn next(&mut self) -> Option { + match self.count { + 0 => None, + 1 => { + self.count -= 1; + self.iter.finish() + } + _ => { + self.count -= 1; + self.iter.next() + } + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let (lower, upper_opt) = self.iter.size_hint(); + ( + cmp::min(self.count, lower), + Some(upper_opt.map_or(self.count, |upper| cmp::min(self.count, upper))), + ) + } +} + +/// An iterator over subslices separated by elements that match a predicate +/// function, limited to a given number of splits. +/// +/// This struct is created by the [`splitn`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// let slice = [10, 40, 30, 20, 60, 50]; +/// let iter = slice.splitn(2, |num| *num % 3 == 0); +/// ``` +/// +/// [`splitn`]: slice::splitn +/// [slices]: slice +#[stable(feature = "rust1", since = "1.0.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct SplitN<'a, T: 'a, P> +where + P: FnMut(&T) -> bool, +{ + inner: GenericSplitN>, +} + +impl<'a, T: 'a, P: FnMut(&T) -> bool> SplitN<'a, T, P> { + #[inline] + pub(super) fn new(s: Split<'a, T, P>, n: usize) -> Self { + Self { inner: GenericSplitN { iter: s, count: n } } + } +} + +#[stable(feature = "core_impl_debug", since = "1.9.0")] +impl fmt::Debug for SplitN<'_, T, P> +where + P: FnMut(&T) -> bool, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SplitN").field("inner", &self.inner).finish() + } +} + +/// An iterator over subslices separated by elements that match a +/// predicate function, limited to a given number of splits, starting +/// from the end of the slice. +/// +/// This struct is created by the [`rsplitn`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// let slice = [10, 40, 30, 20, 60, 50]; +/// let iter = slice.rsplitn(2, |num| *num % 3 == 0); +/// ``` +/// +/// [`rsplitn`]: slice::rsplitn +/// [slices]: slice +#[stable(feature = "rust1", since = "1.0.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct RSplitN<'a, T: 'a, P> +where + P: FnMut(&T) -> bool, +{ + inner: GenericSplitN>, +} + +impl<'a, T: 'a, P: FnMut(&T) -> bool> RSplitN<'a, T, P> { + #[inline] + pub(super) fn new(s: RSplit<'a, T, P>, n: usize) -> Self { + Self { inner: GenericSplitN { iter: s, count: n } } + } +} + +#[stable(feature = "core_impl_debug", since = "1.9.0")] +impl fmt::Debug for RSplitN<'_, T, P> +where + P: FnMut(&T) -> bool, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RSplitN").field("inner", &self.inner).finish() + } +} + +/// An iterator over subslices separated by elements that match a predicate +/// function, limited to a given number of splits. +/// +/// This struct is created by the [`splitn_mut`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// let mut slice = [10, 40, 30, 20, 60, 50]; +/// let iter = slice.splitn_mut(2, |num| *num % 3 == 0); +/// ``` +/// +/// [`splitn_mut`]: slice::splitn_mut +/// [slices]: slice +#[stable(feature = "rust1", since = "1.0.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct SplitNMut<'a, T: 'a, P> +where + P: FnMut(&T) -> bool, +{ + inner: GenericSplitN>, +} + +impl<'a, T: 'a, P: FnMut(&T) -> bool> SplitNMut<'a, T, P> { + #[inline] + pub(super) fn new(s: SplitMut<'a, T, P>, n: usize) -> Self { + Self { inner: GenericSplitN { iter: s, count: n } } + } +} + +#[stable(feature = "core_impl_debug", since = "1.9.0")] +impl fmt::Debug for SplitNMut<'_, T, P> +where + P: FnMut(&T) -> bool, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SplitNMut").field("inner", &self.inner).finish() + } +} + +/// An iterator over subslices separated by elements that match a +/// predicate function, limited to a given number of splits, starting +/// from the end of the slice. +/// +/// This struct is created by the [`rsplitn_mut`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// let mut slice = [10, 40, 30, 20, 60, 50]; +/// let iter = slice.rsplitn_mut(2, |num| *num % 3 == 0); +/// ``` +/// +/// [`rsplitn_mut`]: slice::rsplitn_mut +/// [slices]: slice +#[stable(feature = "rust1", since = "1.0.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct RSplitNMut<'a, T: 'a, P> +where + P: FnMut(&T) -> bool, +{ + inner: GenericSplitN>, +} + +impl<'a, T: 'a, P: FnMut(&T) -> bool> RSplitNMut<'a, T, P> { + #[inline] + pub(super) fn new(s: RSplitMut<'a, T, P>, n: usize) -> Self { + Self { inner: GenericSplitN { iter: s, count: n } } + } +} + +#[stable(feature = "core_impl_debug", since = "1.9.0")] +impl fmt::Debug for RSplitNMut<'_, T, P> +where + P: FnMut(&T) -> bool, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RSplitNMut").field("inner", &self.inner).finish() + } +} + +forward_iterator! { SplitN: T, &'a [T] } +forward_iterator! { RSplitN: T, &'a [T] } +forward_iterator! { SplitNMut: T, &'a mut [T] } +forward_iterator! { RSplitNMut: T, &'a mut [T] } + +/// An iterator over overlapping subslices of length `size`. +/// +/// This struct is created by the [`windows`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// let slice = ['r', 'u', 's', 't']; +/// let iter = slice.windows(2); +/// ``` +/// +/// [`windows`]: slice::windows +/// [slices]: slice +#[derive(Debug)] +#[stable(feature = "rust1", since = "1.0.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct Windows<'a, T: 'a> { + v: &'a [T], + size: NonZeroUsize, +} + +impl<'a, T: 'a> Windows<'a, T> { + #[inline] + pub(super) fn new(slice: &'a [T], size: NonZeroUsize) -> Self { + Self { v: slice, size } + } +} + +// FIXME(#26925) Remove in favor of `#[derive(Clone)]` +#[stable(feature = "rust1", since = "1.0.0")] +impl Clone for Windows<'_, T> { + fn clone(&self) -> Self { + Windows { v: self.v, size: self.size } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T> Iterator for Windows<'a, T> { + type Item = &'a [T]; + + #[inline] + fn next(&mut self) -> Option<&'a [T]> { + if self.size.get() > self.v.len() { + None + } else { + let ret = Some(&self.v[..self.size.get()]); + self.v = &self.v[1..]; + ret + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + if self.size.get() > self.v.len() { + (0, Some(0)) + } else { + let size = self.v.len() - self.size.get() + 1; + (size, Some(size)) + } + } + + #[inline] + fn count(self) -> usize { + self.len() + } + + #[inline] + fn nth(&mut self, n: usize) -> Option { + let (end, overflow) = self.size.get().overflowing_add(n); + if end > self.v.len() || overflow { + self.v = &[]; + None + } else { + let nth = &self.v[n..end]; + self.v = &self.v[n + 1..]; + Some(nth) + } + } + + #[inline] + fn last(self) -> Option { + if self.size.get() > self.v.len() { + None + } else { + let start = self.v.len() - self.size.get(); + Some(&self.v[start..]) + } + } + + unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> Self::Item { + // SAFETY: since the caller guarantees that `i` is in bounds, + // which means that `i` cannot overflow an `isize`, and the + // slice created by `from_raw_parts` is a subslice of `self.v` + // thus is guaranteed to be valid for the lifetime `'a` of `self.v`. + unsafe { from_raw_parts(self.v.as_ptr().add(idx), self.size.get()) } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T> DoubleEndedIterator for Windows<'a, T> { + #[inline] + fn next_back(&mut self) -> Option<&'a [T]> { + if self.size.get() > self.v.len() { + None + } else { + let ret = Some(&self.v[self.v.len() - self.size.get()..]); + self.v = &self.v[..self.v.len() - 1]; + ret + } + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option { + let (end, overflow) = self.v.len().overflowing_sub(n); + if end < self.size.get() || overflow { + self.v = &[]; + None + } else { + let ret = &self.v[end - self.size.get()..end]; + self.v = &self.v[..end - 1]; + Some(ret) + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ExactSizeIterator for Windows<'_, T> {} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for Windows<'_, T> {} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Windows<'_, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccess for Windows<'a, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccessNoCoerce for Windows<'a, T> { + const MAY_HAVE_SIDE_EFFECT: bool = false; +} + +/// An iterator over a slice in (non-overlapping) chunks (`chunk_size` elements at a +/// time), starting at the beginning of the slice. +/// +/// When the slice len is not evenly divided by the chunk size, the last slice +/// of the iteration will be the remainder. +/// +/// This struct is created by the [`chunks`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// let slice = ['l', 'o', 'r', 'e', 'm']; +/// let iter = slice.chunks(2); +/// ``` +/// +/// [`chunks`]: slice::chunks +/// [slices]: slice +#[derive(Debug)] +#[stable(feature = "rust1", since = "1.0.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct Chunks<'a, T: 'a> { + v: &'a [T], + chunk_size: usize, +} + +impl<'a, T: 'a> Chunks<'a, T> { + #[inline] + pub(super) fn new(slice: &'a [T], size: usize) -> Self { + Self { v: slice, chunk_size: size } + } +} + +// FIXME(#26925) Remove in favor of `#[derive(Clone)]` +#[stable(feature = "rust1", since = "1.0.0")] +impl Clone for Chunks<'_, T> { + fn clone(&self) -> Self { + Chunks { v: self.v, chunk_size: self.chunk_size } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T> Iterator for Chunks<'a, T> { + type Item = &'a [T]; + + #[inline] + fn next(&mut self) -> Option<&'a [T]> { + if self.v.is_empty() { + None + } else { + let chunksz = cmp::min(self.v.len(), self.chunk_size); + let (fst, snd) = self.v.split_at(chunksz); + self.v = snd; + Some(fst) + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + if self.v.is_empty() { + (0, Some(0)) + } else { + let n = self.v.len() / self.chunk_size; + let rem = self.v.len() % self.chunk_size; + let n = if rem > 0 { n + 1 } else { n }; + (n, Some(n)) + } + } + + #[inline] + fn count(self) -> usize { + self.len() + } + + #[inline] + fn nth(&mut self, n: usize) -> Option { + let (start, overflow) = n.overflowing_mul(self.chunk_size); + if start >= self.v.len() || overflow { + self.v = &[]; + None + } else { + let end = match start.checked_add(self.chunk_size) { + Some(sum) => cmp::min(self.v.len(), sum), + None => self.v.len(), + }; + let nth = &self.v[start..end]; + self.v = &self.v[end..]; + Some(nth) + } + } + + #[inline] + fn last(self) -> Option { + if self.v.is_empty() { + None + } else { + let start = (self.v.len() - 1) / self.chunk_size * self.chunk_size; + Some(&self.v[start..]) + } + } + + unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> Self::Item { + let start = idx * self.chunk_size; + // SAFETY: the caller guarantees that `i` is in bounds, + // which means that `start` must be in bounds of the + // underlying `self.v` slice, and we made sure that `len` + // is also in bounds of `self.v`. Thus, `start` cannot overflow + // an `isize`, and the slice constructed by `from_raw_parts` + // is a subslice of `self.v` which is guaranteed to be valid + // for the lifetime `'a` of `self.v`. + unsafe { + let len = cmp::min(self.v.len().unchecked_sub(start), self.chunk_size); + from_raw_parts(self.v.as_ptr().add(start), len) + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T> DoubleEndedIterator for Chunks<'a, T> { + #[inline] + fn next_back(&mut self) -> Option<&'a [T]> { + if self.v.is_empty() { + None + } else { + let remainder = self.v.len() % self.chunk_size; + let chunksz = if remainder != 0 { remainder } else { self.chunk_size }; + // SAFETY: split_at_unchecked requires the argument be less than or + // equal to the length. This is guaranteed, but subtle: `chunksz` + // will always either be `self.v.len() % self.chunk_size`, which + // will always evaluate to strictly less than `self.v.len()` (or + // panic, in the case that `self.chunk_size` is zero), or it can be + // `self.chunk_size`, in the case that the length is exactly + // divisible by the chunk size. + // + // While it seems like using `self.chunk_size` in this case could + // lead to a value greater than `self.v.len()`, it cannot: if + // `self.chunk_size` were greater than `self.v.len()`, then + // `self.v.len() % self.chunk_size` would return nonzero (note that + // in this branch of the `if`, we already know that `self.v` is + // non-empty). + let (fst, snd) = unsafe { self.v.split_at_unchecked(self.v.len() - chunksz) }; + self.v = fst; + Some(snd) + } + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option { + let len = self.len(); + if n >= len { + self.v = &[]; + None + } else { + let start = (len - 1 - n) * self.chunk_size; + let end = match start.checked_add(self.chunk_size) { + Some(res) => cmp::min(self.v.len(), res), + None => self.v.len(), + }; + let nth_back = &self.v[start..end]; + self.v = &self.v[..start]; + Some(nth_back) + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ExactSizeIterator for Chunks<'_, T> {} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for Chunks<'_, T> {} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Chunks<'_, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccess for Chunks<'a, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccessNoCoerce for Chunks<'a, T> { + const MAY_HAVE_SIDE_EFFECT: bool = false; +} + +/// An iterator over a slice in (non-overlapping) mutable chunks (`chunk_size` +/// elements at a time), starting at the beginning of the slice. +/// +/// When the slice len is not evenly divided by the chunk size, the last slice +/// of the iteration will be the remainder. +/// +/// This struct is created by the [`chunks_mut`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// let mut slice = ['l', 'o', 'r', 'e', 'm']; +/// let iter = slice.chunks_mut(2); +/// ``` +/// +/// [`chunks_mut`]: slice::chunks_mut +/// [slices]: slice +#[derive(Debug)] +#[stable(feature = "rust1", since = "1.0.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct ChunksMut<'a, T: 'a> { + /// # Safety + /// This slice pointer must point at a valid region of `T` with at least length `v.len()`. Normally, + /// those requirements would mean that we could instead use a `&mut [T]` here, but we cannot + /// because `__iterator_get_unchecked` needs to return `&mut [T]`, which guarantees certain aliasing + /// properties that we cannot uphold if we hold on to the full original `&mut [T]`. Wrapping a raw + /// slice instead lets us hand out non-overlapping `&mut [T]` subslices of the slice we wrap. + v: *mut [T], + chunk_size: usize, + _marker: PhantomData<&'a mut T>, +} + +impl<'a, T: 'a> ChunksMut<'a, T> { + #[inline] + pub(super) fn new(slice: &'a mut [T], size: usize) -> Self { + Self { v: slice, chunk_size: size, _marker: PhantomData } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T> Iterator for ChunksMut<'a, T> { + type Item = &'a mut [T]; + + #[inline] + fn next(&mut self) -> Option<&'a mut [T]> { + if self.v.is_empty() { + None + } else { + let sz = cmp::min(self.v.len(), self.chunk_size); + // SAFETY: The self.v contract ensures that any split_at_mut is valid. + let (head, tail) = unsafe { self.v.split_at_mut(sz) }; + self.v = tail; + // SAFETY: Nothing else points to or will point to the contents of this slice. + Some(unsafe { &mut *head }) + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + if self.v.is_empty() { + (0, Some(0)) + } else { + let n = self.v.len() / self.chunk_size; + let rem = self.v.len() % self.chunk_size; + let n = if rem > 0 { n + 1 } else { n }; + (n, Some(n)) + } + } + + #[inline] + fn count(self) -> usize { + self.len() + } + + #[inline] + fn nth(&mut self, n: usize) -> Option<&'a mut [T]> { + let (start, overflow) = n.overflowing_mul(self.chunk_size); + if start >= self.v.len() || overflow { + self.v = &mut []; + None + } else { + let end = match start.checked_add(self.chunk_size) { + Some(sum) => cmp::min(self.v.len(), sum), + None => self.v.len(), + }; + // SAFETY: The self.v contract ensures that any split_at_mut is valid. + let (head, tail) = unsafe { self.v.split_at_mut(end) }; + // SAFETY: The self.v contract ensures that any split_at_mut is valid. + let (_, nth) = unsafe { head.split_at_mut(start) }; + self.v = tail; + // SAFETY: Nothing else points to or will point to the contents of this slice. + Some(unsafe { &mut *nth }) + } + } + + #[inline] + fn last(self) -> Option { + if self.v.is_empty() { + None + } else { + let start = (self.v.len() - 1) / self.chunk_size * self.chunk_size; + // SAFETY: Nothing else points to or will point to the contents of this slice. + Some(unsafe { &mut *self.v.get_unchecked_mut(start..) }) + } + } + + unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> Self::Item { + let start = idx * self.chunk_size; + // SAFETY: see comments for `Chunks::__iterator_get_unchecked` and `self.v`. + // + // Also note that the caller also guarantees that we're never called + // with the same index again, and that no other methods that will + // access this subslice are called, so it is valid for the returned + // slice to be mutable. + unsafe { + let len = cmp::min(self.v.len().unchecked_sub(start), self.chunk_size); + from_raw_parts_mut(self.v.as_mut_ptr().add(start), len) + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T> DoubleEndedIterator for ChunksMut<'a, T> { + #[inline] + fn next_back(&mut self) -> Option<&'a mut [T]> { + if self.v.is_empty() { + None + } else { + let remainder = self.v.len() % self.chunk_size; + let sz = if remainder != 0 { remainder } else { self.chunk_size }; + let len = self.v.len(); + // SAFETY: Similar to `Chunks::next_back` + let (head, tail) = unsafe { self.v.split_at_mut_unchecked(len - sz) }; + self.v = head; + // SAFETY: Nothing else points to or will point to the contents of this slice. + Some(unsafe { &mut *tail }) + } + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option { + let len = self.len(); + if n >= len { + self.v = &mut []; + None + } else { + let start = (len - 1 - n) * self.chunk_size; + let end = match start.checked_add(self.chunk_size) { + Some(res) => cmp::min(self.v.len(), res), + None => self.v.len(), + }; + // SAFETY: The self.v contract ensures that any split_at_mut is valid. + let (temp, _tail) = unsafe { self.v.split_at_mut(end) }; + // SAFETY: The self.v contract ensures that any split_at_mut is valid. + let (head, nth_back) = unsafe { temp.split_at_mut(start) }; + self.v = head; + // SAFETY: Nothing else points to or will point to the contents of this slice. + Some(unsafe { &mut *nth_back }) + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ExactSizeIterator for ChunksMut<'_, T> {} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for ChunksMut<'_, T> {} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for ChunksMut<'_, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccess for ChunksMut<'a, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccessNoCoerce for ChunksMut<'a, T> { + const MAY_HAVE_SIDE_EFFECT: bool = false; +} + +#[stable(feature = "rust1", since = "1.0.0")] +unsafe impl Send for ChunksMut<'_, T> where T: Send {} + +#[stable(feature = "rust1", since = "1.0.0")] +unsafe impl Sync for ChunksMut<'_, T> where T: Sync {} + +/// An iterator over a slice in (non-overlapping) chunks (`chunk_size` elements at a +/// time), starting at the beginning of the slice. +/// +/// When the slice len is not evenly divided by the chunk size, the last +/// up to `chunk_size-1` elements will be omitted but can be retrieved from +/// the [`remainder`] function from the iterator. +/// +/// This struct is created by the [`chunks_exact`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// let slice = ['l', 'o', 'r', 'e', 'm']; +/// let iter = slice.chunks_exact(2); +/// ``` +/// +/// [`chunks_exact`]: slice::chunks_exact +/// [`remainder`]: ChunksExact::remainder +/// [slices]: slice +#[derive(Debug)] +#[stable(feature = "chunks_exact", since = "1.31.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct ChunksExact<'a, T: 'a> { + v: &'a [T], + rem: &'a [T], + chunk_size: usize, +} + +impl<'a, T> ChunksExact<'a, T> { + #[inline] + pub(super) fn new(slice: &'a [T], chunk_size: usize) -> Self { + let rem = slice.len() % chunk_size; + let fst_len = slice.len() - rem; + // SAFETY: 0 <= fst_len <= slice.len() by construction above + let (fst, snd) = unsafe { slice.split_at_unchecked(fst_len) }; + Self { v: fst, rem: snd, chunk_size } + } + + /// Returns the remainder of the original slice that is not going to be + /// returned by the iterator. The returned slice has at most `chunk_size-1` + /// elements. + /// + /// # Example + /// + /// ``` + /// let slice = ['l', 'o', 'r', 'e', 'm']; + /// let mut iter = slice.chunks_exact(2); + /// assert_eq!(iter.remainder(), &['m'][..]); + /// assert_eq!(iter.next(), Some(&['l', 'o'][..])); + /// assert_eq!(iter.remainder(), &['m'][..]); + /// assert_eq!(iter.next(), Some(&['r', 'e'][..])); + /// assert_eq!(iter.remainder(), &['m'][..]); + /// assert_eq!(iter.next(), None); + /// assert_eq!(iter.remainder(), &['m'][..]); + /// ``` + #[must_use] + #[stable(feature = "chunks_exact", since = "1.31.0")] + pub fn remainder(&self) -> &'a [T] { + self.rem + } +} + +// FIXME(#26925) Remove in favor of `#[derive(Clone)]` +#[stable(feature = "chunks_exact", since = "1.31.0")] +impl Clone for ChunksExact<'_, T> { + fn clone(&self) -> Self { + ChunksExact { v: self.v, rem: self.rem, chunk_size: self.chunk_size } + } +} + +#[stable(feature = "chunks_exact", since = "1.31.0")] +impl<'a, T> Iterator for ChunksExact<'a, T> { + type Item = &'a [T]; + + #[inline] + fn next(&mut self) -> Option<&'a [T]> { + if self.v.len() < self.chunk_size { + None + } else { + let (fst, snd) = self.v.split_at(self.chunk_size); + self.v = snd; + Some(fst) + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let n = self.v.len() / self.chunk_size; + (n, Some(n)) + } + + #[inline] + fn count(self) -> usize { + self.len() + } + + #[inline] + fn nth(&mut self, n: usize) -> Option { + let (start, overflow) = n.overflowing_mul(self.chunk_size); + if start >= self.v.len() || overflow { + self.v = &[]; + None + } else { + let (_, snd) = self.v.split_at(start); + self.v = snd; + self.next() + } + } + + #[inline] + fn last(mut self) -> Option { + self.next_back() + } + + unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> Self::Item { + let start = idx * self.chunk_size; + // SAFETY: mostly identical to `Chunks::__iterator_get_unchecked`. + unsafe { from_raw_parts(self.v.as_ptr().add(start), self.chunk_size) } + } +} + +#[stable(feature = "chunks_exact", since = "1.31.0")] +impl<'a, T> DoubleEndedIterator for ChunksExact<'a, T> { + #[inline] + fn next_back(&mut self) -> Option<&'a [T]> { + if self.v.len() < self.chunk_size { + None + } else { + let (fst, snd) = self.v.split_at(self.v.len() - self.chunk_size); + self.v = fst; + Some(snd) + } + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option { + let len = self.len(); + if n >= len { + self.v = &[]; + None + } else { + let start = (len - 1 - n) * self.chunk_size; + let end = start + self.chunk_size; + let nth_back = &self.v[start..end]; + self.v = &self.v[..start]; + Some(nth_back) + } + } +} + +#[stable(feature = "chunks_exact", since = "1.31.0")] +impl ExactSizeIterator for ChunksExact<'_, T> { + fn is_empty(&self) -> bool { + self.v.is_empty() + } +} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for ChunksExact<'_, T> {} + +#[stable(feature = "chunks_exact", since = "1.31.0")] +impl FusedIterator for ChunksExact<'_, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccess for ChunksExact<'a, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccessNoCoerce for ChunksExact<'a, T> { + const MAY_HAVE_SIDE_EFFECT: bool = false; +} + +/// An iterator over a slice in (non-overlapping) mutable chunks (`chunk_size` +/// elements at a time), starting at the beginning of the slice. +/// +/// When the slice len is not evenly divided by the chunk size, the last up to +/// `chunk_size-1` elements will be omitted but can be retrieved from the +/// [`into_remainder`] function from the iterator. +/// +/// This struct is created by the [`chunks_exact_mut`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// let mut slice = ['l', 'o', 'r', 'e', 'm']; +/// let iter = slice.chunks_exact_mut(2); +/// ``` +/// +/// [`chunks_exact_mut`]: slice::chunks_exact_mut +/// [`into_remainder`]: ChunksExactMut::into_remainder +/// [slices]: slice +#[derive(Debug)] +#[stable(feature = "chunks_exact", since = "1.31.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct ChunksExactMut<'a, T: 'a> { + /// # Safety + /// This slice pointer must point at a valid region of `T` with at least length `v.len()`. Normally, + /// those requirements would mean that we could instead use a `&mut [T]` here, but we cannot + /// because `__iterator_get_unchecked` needs to return `&mut [T]`, which guarantees certain aliasing + /// properties that we cannot uphold if we hold on to the full original `&mut [T]`. Wrapping a raw + /// slice instead lets us hand out non-overlapping `&mut [T]` subslices of the slice we wrap. + v: *mut [T], + rem: &'a mut [T], // The iterator never yields from here, so this can be unique + chunk_size: usize, + _marker: PhantomData<&'a mut T>, +} + +impl<'a, T> ChunksExactMut<'a, T> { + #[inline] + pub(super) fn new(slice: &'a mut [T], chunk_size: usize) -> Self { + let rem = slice.len() % chunk_size; + let fst_len = slice.len() - rem; + // SAFETY: 0 <= fst_len <= slice.len() by construction above + let (fst, snd) = unsafe { slice.split_at_mut_unchecked(fst_len) }; + Self { v: fst, rem: snd, chunk_size, _marker: PhantomData } + } + + /// Returns the remainder of the original slice that is not going to be + /// returned by the iterator. The returned slice has at most `chunk_size-1` + /// elements. + #[must_use = "`self` will be dropped if the result is not used"] + #[stable(feature = "chunks_exact", since = "1.31.0")] + pub fn into_remainder(self) -> &'a mut [T] { + self.rem + } +} + +#[stable(feature = "chunks_exact", since = "1.31.0")] +impl<'a, T> Iterator for ChunksExactMut<'a, T> { + type Item = &'a mut [T]; + + #[inline] + fn next(&mut self) -> Option<&'a mut [T]> { + if self.v.len() < self.chunk_size { + None + } else { + // SAFETY: self.chunk_size is inbounds because we compared above against self.v.len() + let (head, tail) = unsafe { self.v.split_at_mut(self.chunk_size) }; + self.v = tail; + // SAFETY: Nothing else points to or will point to the contents of this slice. + Some(unsafe { &mut *head }) + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let n = self.v.len() / self.chunk_size; + (n, Some(n)) + } + + #[inline] + fn count(self) -> usize { + self.len() + } + + #[inline] + fn nth(&mut self, n: usize) -> Option<&'a mut [T]> { + let (start, overflow) = n.overflowing_mul(self.chunk_size); + if start >= self.v.len() || overflow { + self.v = &mut []; + None + } else { + // SAFETY: The self.v contract ensures that any split_at_mut is valid. + let (_, snd) = unsafe { self.v.split_at_mut(start) }; + self.v = snd; + self.next() + } + } + + #[inline] + fn last(mut self) -> Option { + self.next_back() + } + + unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> Self::Item { + let start = idx * self.chunk_size; + // SAFETY: see comments for `Chunks::__iterator_get_unchecked` and `self.v`. + unsafe { from_raw_parts_mut(self.v.as_mut_ptr().add(start), self.chunk_size) } + } +} + +#[stable(feature = "chunks_exact", since = "1.31.0")] +impl<'a, T> DoubleEndedIterator for ChunksExactMut<'a, T> { + #[inline] + fn next_back(&mut self) -> Option<&'a mut [T]> { + if self.v.len() < self.chunk_size { + None + } else { + // SAFETY: This subtraction is inbounds because of the check above + let (head, tail) = unsafe { self.v.split_at_mut(self.v.len() - self.chunk_size) }; + self.v = head; + // SAFETY: Nothing else points to or will point to the contents of this slice. + Some(unsafe { &mut *tail }) + } + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option { + let len = self.len(); + if n >= len { + self.v = &mut []; + None + } else { + let start = (len - 1 - n) * self.chunk_size; + let end = start + self.chunk_size; + // SAFETY: The self.v contract ensures that any split_at_mut is valid. + let (temp, _tail) = unsafe { mem::replace(&mut self.v, &mut []).split_at_mut(end) }; + // SAFETY: The self.v contract ensures that any split_at_mut is valid. + let (head, nth_back) = unsafe { temp.split_at_mut(start) }; + self.v = head; + // SAFETY: Nothing else points to or will point to the contents of this slice. + Some(unsafe { &mut *nth_back }) + } + } +} + +#[stable(feature = "chunks_exact", since = "1.31.0")] +impl ExactSizeIterator for ChunksExactMut<'_, T> { + fn is_empty(&self) -> bool { + self.v.is_empty() + } +} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for ChunksExactMut<'_, T> {} + +#[stable(feature = "chunks_exact", since = "1.31.0")] +impl FusedIterator for ChunksExactMut<'_, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccess for ChunksExactMut<'a, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccessNoCoerce for ChunksExactMut<'a, T> { + const MAY_HAVE_SIDE_EFFECT: bool = false; +} + +#[stable(feature = "chunks_exact", since = "1.31.0")] +unsafe impl Send for ChunksExactMut<'_, T> where T: Send {} + +#[stable(feature = "chunks_exact", since = "1.31.0")] +unsafe impl Sync for ChunksExactMut<'_, T> where T: Sync {} + +/// A windowed iterator over a slice in overlapping chunks (`N` elements at a +/// time), starting at the beginning of the slice +/// +/// This struct is created by the [`array_windows`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// #![feature(array_windows)] +/// +/// let slice = [0, 1, 2, 3]; +/// let iter = slice.array_windows::<2>(); +/// ``` +/// +/// [`array_windows`]: slice::array_windows +/// [slices]: slice +#[derive(Debug, Clone, Copy)] +#[unstable(feature = "array_windows", issue = "75027")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct ArrayWindows<'a, T: 'a, const N: usize> { + slice_head: *const T, + num: usize, + marker: PhantomData<&'a [T; N]>, +} + +impl<'a, T: 'a, const N: usize> ArrayWindows<'a, T, N> { + #[inline] + pub(super) fn new(slice: &'a [T]) -> Self { + let num_windows = slice.len().saturating_sub(N - 1); + Self { slice_head: slice.as_ptr(), num: num_windows, marker: PhantomData } + } +} + +#[unstable(feature = "array_windows", issue = "75027")] +impl<'a, T, const N: usize> Iterator for ArrayWindows<'a, T, N> { + type Item = &'a [T; N]; + + #[inline] + fn next(&mut self) -> Option { + if self.num == 0 { + return None; + } + // SAFETY: + // This is safe because it's indexing into a slice guaranteed to be length > N. + let ret = unsafe { &*self.slice_head.cast::<[T; N]>() }; + // SAFETY: Guaranteed that there are at least 1 item remaining otherwise + // earlier branch would've been hit + self.slice_head = unsafe { self.slice_head.add(1) }; + + self.num -= 1; + Some(ret) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + (self.num, Some(self.num)) + } + + #[inline] + fn count(self) -> usize { + self.num + } + + #[inline] + fn nth(&mut self, n: usize) -> Option { + if self.num <= n { + self.num = 0; + return None; + } + // SAFETY: + // This is safe because it's indexing into a slice guaranteed to be length > N. + let ret = unsafe { &*self.slice_head.add(n).cast::<[T; N]>() }; + // SAFETY: Guaranteed that there are at least n items remaining + self.slice_head = unsafe { self.slice_head.add(n + 1) }; + + self.num -= n + 1; + Some(ret) + } + + #[inline] + fn last(mut self) -> Option { + self.nth(self.num.checked_sub(1)?) + } +} + +#[unstable(feature = "array_windows", issue = "75027")] +impl<'a, T, const N: usize> DoubleEndedIterator for ArrayWindows<'a, T, N> { + #[inline] + fn next_back(&mut self) -> Option<&'a [T; N]> { + if self.num == 0 { + return None; + } + // SAFETY: Guaranteed that there are n items remaining, n-1 for 0-indexing. + let ret = unsafe { &*self.slice_head.add(self.num - 1).cast::<[T; N]>() }; + self.num -= 1; + Some(ret) + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option<&'a [T; N]> { + if self.num <= n { + self.num = 0; + return None; + } + // SAFETY: Guaranteed that there are n items remaining, n-1 for 0-indexing. + let ret = unsafe { &*self.slice_head.add(self.num - (n + 1)).cast::<[T; N]>() }; + self.num -= n + 1; + Some(ret) + } +} + +#[unstable(feature = "array_windows", issue = "75027")] +impl ExactSizeIterator for ArrayWindows<'_, T, N> { + fn is_empty(&self) -> bool { + self.num == 0 + } +} + +/// An iterator over a slice in (non-overlapping) chunks (`N` elements at a +/// time), starting at the beginning of the slice. +/// +/// When the slice len is not evenly divided by the chunk size, the last +/// up to `N-1` elements will be omitted but can be retrieved from +/// the [`remainder`] function from the iterator. +/// +/// This struct is created by the [`array_chunks`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// #![feature(array_chunks)] +/// +/// let slice = ['l', 'o', 'r', 'e', 'm']; +/// let iter = slice.array_chunks::<2>(); +/// ``` +/// +/// [`array_chunks`]: slice::array_chunks +/// [`remainder`]: ArrayChunks::remainder +/// [slices]: slice +#[derive(Debug)] +#[unstable(feature = "array_chunks", issue = "74985")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct ArrayChunks<'a, T: 'a, const N: usize> { + iter: Iter<'a, [T; N]>, + rem: &'a [T], +} + +impl<'a, T, const N: usize> ArrayChunks<'a, T, N> { + #[inline] + pub(super) fn new(slice: &'a [T]) -> Self { + let (array_slice, rem) = slice.as_chunks(); + Self { iter: array_slice.iter(), rem } + } + + /// Returns the remainder of the original slice that is not going to be + /// returned by the iterator. The returned slice has at most `N-1` + /// elements. + #[must_use] + #[unstable(feature = "array_chunks", issue = "74985")] + pub fn remainder(&self) -> &'a [T] { + self.rem + } +} + +// FIXME(#26925) Remove in favor of `#[derive(Clone)]` +#[unstable(feature = "array_chunks", issue = "74985")] +impl Clone for ArrayChunks<'_, T, N> { + fn clone(&self) -> Self { + ArrayChunks { iter: self.iter.clone(), rem: self.rem } + } +} + +#[unstable(feature = "array_chunks", issue = "74985")] +impl<'a, T, const N: usize> Iterator for ArrayChunks<'a, T, N> { + type Item = &'a [T; N]; + + #[inline] + fn next(&mut self) -> Option<&'a [T; N]> { + self.iter.next() + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } + + #[inline] + fn count(self) -> usize { + self.iter.count() + } + + #[inline] + fn nth(&mut self, n: usize) -> Option { + self.iter.nth(n) + } + + #[inline] + fn last(self) -> Option { + self.iter.last() + } + + unsafe fn __iterator_get_unchecked(&mut self, i: usize) -> &'a [T; N] { + // SAFETY: The safety guarantees of `__iterator_get_unchecked` are + // transferred to the caller. + unsafe { self.iter.__iterator_get_unchecked(i) } + } +} + +#[unstable(feature = "array_chunks", issue = "74985")] +impl<'a, T, const N: usize> DoubleEndedIterator for ArrayChunks<'a, T, N> { + #[inline] + fn next_back(&mut self) -> Option<&'a [T; N]> { + self.iter.next_back() + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option { + self.iter.nth_back(n) + } +} + +#[unstable(feature = "array_chunks", issue = "74985")] +impl ExactSizeIterator for ArrayChunks<'_, T, N> { + fn is_empty(&self) -> bool { + self.iter.is_empty() + } +} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for ArrayChunks<'_, T, N> {} + +#[unstable(feature = "array_chunks", issue = "74985")] +impl FusedIterator for ArrayChunks<'_, T, N> {} + +#[doc(hidden)] +#[unstable(feature = "array_chunks", issue = "74985")] +unsafe impl<'a, T, const N: usize> TrustedRandomAccess for ArrayChunks<'a, T, N> {} + +#[doc(hidden)] +#[unstable(feature = "array_chunks", issue = "74985")] +unsafe impl<'a, T, const N: usize> TrustedRandomAccessNoCoerce for ArrayChunks<'a, T, N> { + const MAY_HAVE_SIDE_EFFECT: bool = false; +} + +/// An iterator over a slice in (non-overlapping) mutable chunks (`N` elements +/// at a time), starting at the beginning of the slice. +/// +/// When the slice len is not evenly divided by the chunk size, the last +/// up to `N-1` elements will be omitted but can be retrieved from +/// the [`into_remainder`] function from the iterator. +/// +/// This struct is created by the [`array_chunks_mut`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// #![feature(array_chunks)] +/// +/// let mut slice = ['l', 'o', 'r', 'e', 'm']; +/// let iter = slice.array_chunks_mut::<2>(); +/// ``` +/// +/// [`array_chunks_mut`]: slice::array_chunks_mut +/// [`into_remainder`]: ../../std/slice/struct.ArrayChunksMut.html#method.into_remainder +/// [slices]: slice +#[derive(Debug)] +#[unstable(feature = "array_chunks", issue = "74985")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct ArrayChunksMut<'a, T: 'a, const N: usize> { + iter: IterMut<'a, [T; N]>, + rem: &'a mut [T], +} + +impl<'a, T, const N: usize> ArrayChunksMut<'a, T, N> { + #[inline] + pub(super) fn new(slice: &'a mut [T]) -> Self { + let (array_slice, rem) = slice.as_chunks_mut(); + Self { iter: array_slice.iter_mut(), rem } + } + + /// Returns the remainder of the original slice that is not going to be + /// returned by the iterator. The returned slice has at most `N-1` + /// elements. + #[must_use = "`self` will be dropped if the result is not used"] + #[unstable(feature = "array_chunks", issue = "74985")] + pub fn into_remainder(self) -> &'a mut [T] { + self.rem + } +} + +#[unstable(feature = "array_chunks", issue = "74985")] +impl<'a, T, const N: usize> Iterator for ArrayChunksMut<'a, T, N> { + type Item = &'a mut [T; N]; + + #[inline] + fn next(&mut self) -> Option<&'a mut [T; N]> { + self.iter.next() + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } + + #[inline] + fn count(self) -> usize { + self.iter.count() + } + + #[inline] + fn nth(&mut self, n: usize) -> Option { + self.iter.nth(n) + } + + #[inline] + fn last(self) -> Option { + self.iter.last() + } + + unsafe fn __iterator_get_unchecked(&mut self, i: usize) -> &'a mut [T; N] { + // SAFETY: The safety guarantees of `__iterator_get_unchecked` are transferred to + // the caller. + unsafe { self.iter.__iterator_get_unchecked(i) } + } +} + +#[unstable(feature = "array_chunks", issue = "74985")] +impl<'a, T, const N: usize> DoubleEndedIterator for ArrayChunksMut<'a, T, N> { + #[inline] + fn next_back(&mut self) -> Option<&'a mut [T; N]> { + self.iter.next_back() + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option { + self.iter.nth_back(n) + } +} + +#[unstable(feature = "array_chunks", issue = "74985")] +impl ExactSizeIterator for ArrayChunksMut<'_, T, N> { + fn is_empty(&self) -> bool { + self.iter.is_empty() + } +} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for ArrayChunksMut<'_, T, N> {} + +#[unstable(feature = "array_chunks", issue = "74985")] +impl FusedIterator for ArrayChunksMut<'_, T, N> {} + +#[doc(hidden)] +#[unstable(feature = "array_chunks", issue = "74985")] +unsafe impl<'a, T, const N: usize> TrustedRandomAccess for ArrayChunksMut<'a, T, N> {} + +#[doc(hidden)] +#[unstable(feature = "array_chunks", issue = "74985")] +unsafe impl<'a, T, const N: usize> TrustedRandomAccessNoCoerce for ArrayChunksMut<'a, T, N> { + const MAY_HAVE_SIDE_EFFECT: bool = false; +} + +/// An iterator over a slice in (non-overlapping) chunks (`chunk_size` elements at a +/// time), starting at the end of the slice. +/// +/// When the slice len is not evenly divided by the chunk size, the last slice +/// of the iteration will be the remainder. +/// +/// This struct is created by the [`rchunks`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// let slice = ['l', 'o', 'r', 'e', 'm']; +/// let iter = slice.rchunks(2); +/// ``` +/// +/// [`rchunks`]: slice::rchunks +/// [slices]: slice +#[derive(Debug)] +#[stable(feature = "rchunks", since = "1.31.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct RChunks<'a, T: 'a> { + v: &'a [T], + chunk_size: usize, +} + +impl<'a, T: 'a> RChunks<'a, T> { + #[inline] + pub(super) fn new(slice: &'a [T], size: usize) -> Self { + Self { v: slice, chunk_size: size } + } +} + +// FIXME(#26925) Remove in favor of `#[derive(Clone)]` +#[stable(feature = "rchunks", since = "1.31.0")] +impl Clone for RChunks<'_, T> { + fn clone(&self) -> Self { + RChunks { v: self.v, chunk_size: self.chunk_size } + } +} + +#[stable(feature = "rchunks", since = "1.31.0")] +impl<'a, T> Iterator for RChunks<'a, T> { + type Item = &'a [T]; + + #[inline] + fn next(&mut self) -> Option<&'a [T]> { + if self.v.is_empty() { + None + } else { + let len = self.v.len(); + let chunksz = cmp::min(len, self.chunk_size); + // SAFETY: split_at_unchecked just requires the argument be less + // than the length. This could only happen if the expression `len - + // chunksz` overflows. This could only happen if `chunksz > len`, + // which is impossible as we initialize it as the `min` of `len` and + // `self.chunk_size`. + let (fst, snd) = unsafe { self.v.split_at_unchecked(len - chunksz) }; + self.v = fst; + Some(snd) + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + if self.v.is_empty() { + (0, Some(0)) + } else { + let n = self.v.len() / self.chunk_size; + let rem = self.v.len() % self.chunk_size; + let n = if rem > 0 { n + 1 } else { n }; + (n, Some(n)) + } + } + + #[inline] + fn count(self) -> usize { + self.len() + } + + #[inline] + fn nth(&mut self, n: usize) -> Option { + let (end, overflow) = n.overflowing_mul(self.chunk_size); + if end >= self.v.len() || overflow { + self.v = &[]; + None + } else { + // Can't underflow because of the check above + let end = self.v.len() - end; + let start = match end.checked_sub(self.chunk_size) { + Some(sum) => sum, + None => 0, + }; + let nth = &self.v[start..end]; + self.v = &self.v[0..start]; + Some(nth) + } + } + + #[inline] + fn last(self) -> Option { + if self.v.is_empty() { + None + } else { + let rem = self.v.len() % self.chunk_size; + let end = if rem == 0 { self.chunk_size } else { rem }; + Some(&self.v[0..end]) + } + } + + unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> Self::Item { + let end = self.v.len() - idx * self.chunk_size; + let start = match end.checked_sub(self.chunk_size) { + None => 0, + Some(start) => start, + }; + // SAFETY: mostly identical to `Chunks::__iterator_get_unchecked`. + unsafe { from_raw_parts(self.v.as_ptr().add(start), end - start) } + } +} + +#[stable(feature = "rchunks", since = "1.31.0")] +impl<'a, T> DoubleEndedIterator for RChunks<'a, T> { + #[inline] + fn next_back(&mut self) -> Option<&'a [T]> { + if self.v.is_empty() { + None + } else { + let remainder = self.v.len() % self.chunk_size; + let chunksz = if remainder != 0 { remainder } else { self.chunk_size }; + // SAFETY: similar to Chunks::next_back + let (fst, snd) = unsafe { self.v.split_at_unchecked(chunksz) }; + self.v = snd; + Some(fst) + } + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option { + let len = self.len(); + if n >= len { + self.v = &[]; + None + } else { + // can't underflow because `n < len` + let offset_from_end = (len - 1 - n) * self.chunk_size; + let end = self.v.len() - offset_from_end; + let start = end.saturating_sub(self.chunk_size); + let nth_back = &self.v[start..end]; + self.v = &self.v[end..]; + Some(nth_back) + } + } +} + +#[stable(feature = "rchunks", since = "1.31.0")] +impl ExactSizeIterator for RChunks<'_, T> {} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for RChunks<'_, T> {} + +#[stable(feature = "rchunks", since = "1.31.0")] +impl FusedIterator for RChunks<'_, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccess for RChunks<'a, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccessNoCoerce for RChunks<'a, T> { + const MAY_HAVE_SIDE_EFFECT: bool = false; +} + +/// An iterator over a slice in (non-overlapping) mutable chunks (`chunk_size` +/// elements at a time), starting at the end of the slice. +/// +/// When the slice len is not evenly divided by the chunk size, the last slice +/// of the iteration will be the remainder. +/// +/// This struct is created by the [`rchunks_mut`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// let mut slice = ['l', 'o', 'r', 'e', 'm']; +/// let iter = slice.rchunks_mut(2); +/// ``` +/// +/// [`rchunks_mut`]: slice::rchunks_mut +/// [slices]: slice +#[derive(Debug)] +#[stable(feature = "rchunks", since = "1.31.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct RChunksMut<'a, T: 'a> { + /// # Safety + /// This slice pointer must point at a valid region of `T` with at least length `v.len()`. Normally, + /// those requirements would mean that we could instead use a `&mut [T]` here, but we cannot + /// because `__iterator_get_unchecked` needs to return `&mut [T]`, which guarantees certain aliasing + /// properties that we cannot uphold if we hold on to the full original `&mut [T]`. Wrapping a raw + /// slice instead lets us hand out non-overlapping `&mut [T]` subslices of the slice we wrap. + v: *mut [T], + chunk_size: usize, + _marker: PhantomData<&'a mut T>, +} + +impl<'a, T: 'a> RChunksMut<'a, T> { + #[inline] + pub(super) fn new(slice: &'a mut [T], size: usize) -> Self { + Self { v: slice, chunk_size: size, _marker: PhantomData } + } +} + +#[stable(feature = "rchunks", since = "1.31.0")] +impl<'a, T> Iterator for RChunksMut<'a, T> { + type Item = &'a mut [T]; + + #[inline] + fn next(&mut self) -> Option<&'a mut [T]> { + if self.v.is_empty() { + None + } else { + let sz = cmp::min(self.v.len(), self.chunk_size); + let len = self.v.len(); + // SAFETY: split_at_mut_unchecked just requires the argument be less + // than the length. This could only happen if the expression + // `len - sz` overflows. This could only happen if `sz > + // len`, which is impossible as we initialize it as the `min` of + // `self.v.len()` (e.g. `len`) and `self.chunk_size`. + let (head, tail) = unsafe { self.v.split_at_mut_unchecked(len - sz) }; + self.v = head; + // SAFETY: Nothing else points to or will point to the contents of this slice. + Some(unsafe { &mut *tail }) + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + if self.v.is_empty() { + (0, Some(0)) + } else { + let n = self.v.len() / self.chunk_size; + let rem = self.v.len() % self.chunk_size; + let n = if rem > 0 { n + 1 } else { n }; + (n, Some(n)) + } + } + + #[inline] + fn count(self) -> usize { + self.len() + } + + #[inline] + fn nth(&mut self, n: usize) -> Option<&'a mut [T]> { + let (end, overflow) = n.overflowing_mul(self.chunk_size); + if end >= self.v.len() || overflow { + self.v = &mut []; + None + } else { + // Can't underflow because of the check above + let end = self.v.len() - end; + let start = match end.checked_sub(self.chunk_size) { + Some(sum) => sum, + None => 0, + }; + // SAFETY: This type ensures that self.v is a valid pointer with a correct len. + // Therefore the bounds check in split_at_mut guarantees the split point is inbounds. + let (head, tail) = unsafe { self.v.split_at_mut(start) }; + // SAFETY: This type ensures that self.v is a valid pointer with a correct len. + // Therefore the bounds check in split_at_mut guarantees the split point is inbounds. + let (nth, _) = unsafe { tail.split_at_mut(end - start) }; + self.v = head; + // SAFETY: Nothing else points to or will point to the contents of this slice. + Some(unsafe { &mut *nth }) + } + } + + #[inline] + fn last(self) -> Option { + if self.v.is_empty() { + None + } else { + let rem = self.v.len() % self.chunk_size; + let end = if rem == 0 { self.chunk_size } else { rem }; + // SAFETY: Nothing else points to or will point to the contents of this slice. + Some(unsafe { &mut *self.v.get_unchecked_mut(0..end) }) + } + } + + unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> Self::Item { + let end = self.v.len() - idx * self.chunk_size; + let start = match end.checked_sub(self.chunk_size) { + None => 0, + Some(start) => start, + }; + // SAFETY: see comments for `RChunks::__iterator_get_unchecked` and + // `ChunksMut::__iterator_get_unchecked`, `self.v`. + unsafe { from_raw_parts_mut(self.v.as_mut_ptr().add(start), end - start) } + } +} + +#[stable(feature = "rchunks", since = "1.31.0")] +impl<'a, T> DoubleEndedIterator for RChunksMut<'a, T> { + #[inline] + fn next_back(&mut self) -> Option<&'a mut [T]> { + if self.v.is_empty() { + None + } else { + let remainder = self.v.len() % self.chunk_size; + let sz = if remainder != 0 { remainder } else { self.chunk_size }; + // SAFETY: Similar to `Chunks::next_back` + let (head, tail) = unsafe { self.v.split_at_mut_unchecked(sz) }; + self.v = tail; + // SAFETY: Nothing else points to or will point to the contents of this slice. + Some(unsafe { &mut *head }) + } + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option { + let len = self.len(); + if n >= len { + self.v = &mut []; + None + } else { + // can't underflow because `n < len` + let offset_from_end = (len - 1 - n) * self.chunk_size; + let end = self.v.len() - offset_from_end; + let start = end.saturating_sub(self.chunk_size); + // SAFETY: The self.v contract ensures that any split_at_mut is valid. + let (tmp, tail) = unsafe { self.v.split_at_mut(end) }; + // SAFETY: The self.v contract ensures that any split_at_mut is valid. + let (_, nth_back) = unsafe { tmp.split_at_mut(start) }; + self.v = tail; + // SAFETY: Nothing else points to or will point to the contents of this slice. + Some(unsafe { &mut *nth_back }) + } + } +} + +#[stable(feature = "rchunks", since = "1.31.0")] +impl ExactSizeIterator for RChunksMut<'_, T> {} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for RChunksMut<'_, T> {} + +#[stable(feature = "rchunks", since = "1.31.0")] +impl FusedIterator for RChunksMut<'_, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccess for RChunksMut<'a, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccessNoCoerce for RChunksMut<'a, T> { + const MAY_HAVE_SIDE_EFFECT: bool = false; +} + +#[stable(feature = "rchunks", since = "1.31.0")] +unsafe impl Send for RChunksMut<'_, T> where T: Send {} + +#[stable(feature = "rchunks", since = "1.31.0")] +unsafe impl Sync for RChunksMut<'_, T> where T: Sync {} + +/// An iterator over a slice in (non-overlapping) chunks (`chunk_size` elements at a +/// time), starting at the end of the slice. +/// +/// When the slice len is not evenly divided by the chunk size, the last +/// up to `chunk_size-1` elements will be omitted but can be retrieved from +/// the [`remainder`] function from the iterator. +/// +/// This struct is created by the [`rchunks_exact`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// let slice = ['l', 'o', 'r', 'e', 'm']; +/// let iter = slice.rchunks_exact(2); +/// ``` +/// +/// [`rchunks_exact`]: slice::rchunks_exact +/// [`remainder`]: RChunksExact::remainder +/// [slices]: slice +#[derive(Debug)] +#[stable(feature = "rchunks", since = "1.31.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct RChunksExact<'a, T: 'a> { + v: &'a [T], + rem: &'a [T], + chunk_size: usize, +} + +impl<'a, T> RChunksExact<'a, T> { + #[inline] + pub(super) fn new(slice: &'a [T], chunk_size: usize) -> Self { + let rem = slice.len() % chunk_size; + // SAFETY: 0 <= rem <= slice.len() by construction above + let (fst, snd) = unsafe { slice.split_at_unchecked(rem) }; + Self { v: snd, rem: fst, chunk_size } + } + + /// Returns the remainder of the original slice that is not going to be + /// returned by the iterator. The returned slice has at most `chunk_size-1` + /// elements. + /// + /// # Example + /// + /// ``` + /// let slice = ['l', 'o', 'r', 'e', 'm']; + /// let mut iter = slice.rchunks_exact(2); + /// assert_eq!(iter.remainder(), &['l'][..]); + /// assert_eq!(iter.next(), Some(&['e', 'm'][..])); + /// assert_eq!(iter.remainder(), &['l'][..]); + /// assert_eq!(iter.next(), Some(&['o', 'r'][..])); + /// assert_eq!(iter.remainder(), &['l'][..]); + /// assert_eq!(iter.next(), None); + /// assert_eq!(iter.remainder(), &['l'][..]); + /// ``` + #[must_use] + #[stable(feature = "rchunks", since = "1.31.0")] + pub fn remainder(&self) -> &'a [T] { + self.rem + } +} + +// FIXME(#26925) Remove in favor of `#[derive(Clone)]` +#[stable(feature = "rchunks", since = "1.31.0")] +impl<'a, T> Clone for RChunksExact<'a, T> { + fn clone(&self) -> RChunksExact<'a, T> { + RChunksExact { v: self.v, rem: self.rem, chunk_size: self.chunk_size } + } +} + +#[stable(feature = "rchunks", since = "1.31.0")] +impl<'a, T> Iterator for RChunksExact<'a, T> { + type Item = &'a [T]; + + #[inline] + fn next(&mut self) -> Option<&'a [T]> { + if self.v.len() < self.chunk_size { + None + } else { + let (fst, snd) = self.v.split_at(self.v.len() - self.chunk_size); + self.v = fst; + Some(snd) + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let n = self.v.len() / self.chunk_size; + (n, Some(n)) + } + + #[inline] + fn count(self) -> usize { + self.len() + } + + #[inline] + fn nth(&mut self, n: usize) -> Option { + let (end, overflow) = n.overflowing_mul(self.chunk_size); + if end >= self.v.len() || overflow { + self.v = &[]; + None + } else { + let (fst, _) = self.v.split_at(self.v.len() - end); + self.v = fst; + self.next() + } + } + + #[inline] + fn last(mut self) -> Option { + self.next_back() + } + + unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> Self::Item { + let end = self.v.len() - idx * self.chunk_size; + let start = end - self.chunk_size; + // SAFETY: mostly identical to `Chunks::__iterator_get_unchecked`. + unsafe { from_raw_parts(self.v.as_ptr().add(start), self.chunk_size) } + } +} + +#[stable(feature = "rchunks", since = "1.31.0")] +impl<'a, T> DoubleEndedIterator for RChunksExact<'a, T> { + #[inline] + fn next_back(&mut self) -> Option<&'a [T]> { + if self.v.len() < self.chunk_size { + None + } else { + let (fst, snd) = self.v.split_at(self.chunk_size); + self.v = snd; + Some(fst) + } + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option { + let len = self.len(); + if n >= len { + self.v = &[]; + None + } else { + // now that we know that `n` corresponds to a chunk, + // none of these operations can underflow/overflow + let offset = (len - n) * self.chunk_size; + let start = self.v.len() - offset; + let end = start + self.chunk_size; + let nth_back = &self.v[start..end]; + self.v = &self.v[end..]; + Some(nth_back) + } + } +} + +#[stable(feature = "rchunks", since = "1.31.0")] +impl<'a, T> ExactSizeIterator for RChunksExact<'a, T> { + fn is_empty(&self) -> bool { + self.v.is_empty() + } +} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for RChunksExact<'_, T> {} + +#[stable(feature = "rchunks", since = "1.31.0")] +impl FusedIterator for RChunksExact<'_, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccess for RChunksExact<'a, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccessNoCoerce for RChunksExact<'a, T> { + const MAY_HAVE_SIDE_EFFECT: bool = false; +} + +/// An iterator over a slice in (non-overlapping) mutable chunks (`chunk_size` +/// elements at a time), starting at the end of the slice. +/// +/// When the slice len is not evenly divided by the chunk size, the last up to +/// `chunk_size-1` elements will be omitted but can be retrieved from the +/// [`into_remainder`] function from the iterator. +/// +/// This struct is created by the [`rchunks_exact_mut`] method on [slices]. +/// +/// # Example +/// +/// ``` +/// let mut slice = ['l', 'o', 'r', 'e', 'm']; +/// let iter = slice.rchunks_exact_mut(2); +/// ``` +/// +/// [`rchunks_exact_mut`]: slice::rchunks_exact_mut +/// [`into_remainder`]: RChunksExactMut::into_remainder +/// [slices]: slice +#[derive(Debug)] +#[stable(feature = "rchunks", since = "1.31.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct RChunksExactMut<'a, T: 'a> { + /// # Safety + /// This slice pointer must point at a valid region of `T` with at least length `v.len()`. Normally, + /// those requirements would mean that we could instead use a `&mut [T]` here, but we cannot + /// because `__iterator_get_unchecked` needs to return `&mut [T]`, which guarantees certain aliasing + /// properties that we cannot uphold if we hold on to the full original `&mut [T]`. Wrapping a raw + /// slice instead lets us hand out non-overlapping `&mut [T]` subslices of the slice we wrap. + v: *mut [T], + rem: &'a mut [T], + chunk_size: usize, +} + +impl<'a, T> RChunksExactMut<'a, T> { + #[inline] + pub(super) fn new(slice: &'a mut [T], chunk_size: usize) -> Self { + let rem = slice.len() % chunk_size; + // SAFETY: 0 <= rem <= slice.len() by construction above + let (fst, snd) = unsafe { slice.split_at_mut_unchecked(rem) }; + Self { v: snd, rem: fst, chunk_size } + } + + /// Returns the remainder of the original slice that is not going to be + /// returned by the iterator. The returned slice has at most `chunk_size-1` + /// elements. + #[must_use = "`self` will be dropped if the result is not used"] + #[stable(feature = "rchunks", since = "1.31.0")] + pub fn into_remainder(self) -> &'a mut [T] { + self.rem + } +} + +#[stable(feature = "rchunks", since = "1.31.0")] +impl<'a, T> Iterator for RChunksExactMut<'a, T> { + type Item = &'a mut [T]; + + #[inline] + fn next(&mut self) -> Option<&'a mut [T]> { + if self.v.len() < self.chunk_size { + None + } else { + let len = self.v.len(); + // SAFETY: The self.v contract ensures that any split_at_mut is valid. + let (head, tail) = unsafe { self.v.split_at_mut(len - self.chunk_size) }; + self.v = head; + // SAFETY: Nothing else points to or will point to the contents of this slice. + Some(unsafe { &mut *tail }) + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let n = self.v.len() / self.chunk_size; + (n, Some(n)) + } + + #[inline] + fn count(self) -> usize { + self.len() + } + + #[inline] + fn nth(&mut self, n: usize) -> Option<&'a mut [T]> { + let (end, overflow) = n.overflowing_mul(self.chunk_size); + if end >= self.v.len() || overflow { + self.v = &mut []; + None + } else { + let len = self.v.len(); + // SAFETY: The self.v contract ensures that any split_at_mut is valid. + let (fst, _) = unsafe { self.v.split_at_mut(len - end) }; + self.v = fst; + self.next() + } + } + + #[inline] + fn last(mut self) -> Option { + self.next_back() + } + + unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> Self::Item { + let end = self.v.len() - idx * self.chunk_size; + let start = end - self.chunk_size; + // SAFETY: see comments for `RChunksMut::__iterator_get_unchecked` and `self.v`. + unsafe { from_raw_parts_mut(self.v.as_mut_ptr().add(start), self.chunk_size) } + } +} + +#[stable(feature = "rchunks", since = "1.31.0")] +impl<'a, T> DoubleEndedIterator for RChunksExactMut<'a, T> { + #[inline] + fn next_back(&mut self) -> Option<&'a mut [T]> { + if self.v.len() < self.chunk_size { + None + } else { + // SAFETY: The self.v contract ensures that any split_at_mut is valid. + let (head, tail) = unsafe { self.v.split_at_mut(self.chunk_size) }; + self.v = tail; + // SAFETY: Nothing else points to or will point to the contents of this slice. + Some(unsafe { &mut *head }) + } + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option { + let len = self.len(); + if n >= len { + self.v = &mut []; + None + } else { + // now that we know that `n` corresponds to a chunk, + // none of these operations can underflow/overflow + let offset = (len - n) * self.chunk_size; + let start = self.v.len() - offset; + let end = start + self.chunk_size; + // SAFETY: The self.v contract ensures that any split_at_mut is valid. + let (tmp, tail) = unsafe { self.v.split_at_mut(end) }; + // SAFETY: The self.v contract ensures that any split_at_mut is valid. + let (_, nth_back) = unsafe { tmp.split_at_mut(start) }; + self.v = tail; + // SAFETY: Nothing else points to or will point to the contents of this slice. + Some(unsafe { &mut *nth_back }) + } + } +} + +#[stable(feature = "rchunks", since = "1.31.0")] +impl ExactSizeIterator for RChunksExactMut<'_, T> { + fn is_empty(&self) -> bool { + self.v.is_empty() + } +} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for RChunksExactMut<'_, T> {} + +#[stable(feature = "rchunks", since = "1.31.0")] +impl FusedIterator for RChunksExactMut<'_, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccess for RChunksExactMut<'a, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccessNoCoerce for RChunksExactMut<'a, T> { + const MAY_HAVE_SIDE_EFFECT: bool = false; +} + +#[stable(feature = "rchunks", since = "1.31.0")] +unsafe impl Send for RChunksExactMut<'_, T> where T: Send {} + +#[stable(feature = "rchunks", since = "1.31.0")] +unsafe impl Sync for RChunksExactMut<'_, T> where T: Sync {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccess for Iter<'a, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccessNoCoerce for Iter<'a, T> { + const MAY_HAVE_SIDE_EFFECT: bool = false; +} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccess for IterMut<'a, T> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl<'a, T> TrustedRandomAccessNoCoerce for IterMut<'a, T> { + const MAY_HAVE_SIDE_EFFECT: bool = false; +} + +/// An iterator over slice in (non-overlapping) chunks separated by a predicate. +/// +/// This struct is created by the [`group_by`] method on [slices]. +/// +/// [`group_by`]: slice::group_by +/// [slices]: slice +#[unstable(feature = "slice_group_by", issue = "80552")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct GroupBy<'a, T: 'a, P> { + slice: &'a [T], + predicate: P, +} + +#[unstable(feature = "slice_group_by", issue = "80552")] +impl<'a, T: 'a, P> GroupBy<'a, T, P> { + pub(super) fn new(slice: &'a [T], predicate: P) -> Self { + GroupBy { slice, predicate } + } +} + +#[unstable(feature = "slice_group_by", issue = "80552")] +impl<'a, T: 'a, P> Iterator for GroupBy<'a, T, P> +where + P: FnMut(&T, &T) -> bool, +{ + type Item = &'a [T]; + + #[inline] + fn next(&mut self) -> Option { + if self.slice.is_empty() { + None + } else { + let mut len = 1; + let mut iter = self.slice.windows(2); + while let Some([l, r]) = iter.next() { + if (self.predicate)(l, r) { len += 1 } else { break } + } + let (head, tail) = self.slice.split_at(len); + self.slice = tail; + Some(head) + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + if self.slice.is_empty() { (0, Some(0)) } else { (1, Some(self.slice.len())) } + } + + #[inline] + fn last(mut self) -> Option { + self.next_back() + } +} + +#[unstable(feature = "slice_group_by", issue = "80552")] +impl<'a, T: 'a, P> DoubleEndedIterator for GroupBy<'a, T, P> +where + P: FnMut(&T, &T) -> bool, +{ + #[inline] + fn next_back(&mut self) -> Option { + if self.slice.is_empty() { + None + } else { + let mut len = 1; + let mut iter = self.slice.windows(2); + while let Some([l, r]) = iter.next_back() { + if (self.predicate)(l, r) { len += 1 } else { break } + } + let (head, tail) = self.slice.split_at(self.slice.len() - len); + self.slice = head; + Some(tail) + } + } +} + +#[unstable(feature = "slice_group_by", issue = "80552")] +impl<'a, T: 'a, P> FusedIterator for GroupBy<'a, T, P> where P: FnMut(&T, &T) -> bool {} + +#[unstable(feature = "slice_group_by", issue = "80552")] +impl<'a, T: 'a + fmt::Debug, P> fmt::Debug for GroupBy<'a, T, P> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("GroupBy").field("slice", &self.slice).finish() + } +} + +/// An iterator over slice in (non-overlapping) mutable chunks separated +/// by a predicate. +/// +/// This struct is created by the [`group_by_mut`] method on [slices]. +/// +/// [`group_by_mut`]: slice::group_by_mut +/// [slices]: slice +#[unstable(feature = "slice_group_by", issue = "80552")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct GroupByMut<'a, T: 'a, P> { + slice: &'a mut [T], + predicate: P, +} + +#[unstable(feature = "slice_group_by", issue = "80552")] +impl<'a, T: 'a, P> GroupByMut<'a, T, P> { + pub(super) fn new(slice: &'a mut [T], predicate: P) -> Self { + GroupByMut { slice, predicate } + } +} + +#[unstable(feature = "slice_group_by", issue = "80552")] +impl<'a, T: 'a, P> Iterator for GroupByMut<'a, T, P> +where + P: FnMut(&T, &T) -> bool, +{ + type Item = &'a mut [T]; + + #[inline] + fn next(&mut self) -> Option { + if self.slice.is_empty() { + None + } else { + let mut len = 1; + let mut iter = self.slice.windows(2); + while let Some([l, r]) = iter.next() { + if (self.predicate)(l, r) { len += 1 } else { break } + } + let slice = mem::take(&mut self.slice); + let (head, tail) = slice.split_at_mut(len); + self.slice = tail; + Some(head) + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + if self.slice.is_empty() { (0, Some(0)) } else { (1, Some(self.slice.len())) } + } + + #[inline] + fn last(mut self) -> Option { + self.next_back() + } +} + +#[unstable(feature = "slice_group_by", issue = "80552")] +impl<'a, T: 'a, P> DoubleEndedIterator for GroupByMut<'a, T, P> +where + P: FnMut(&T, &T) -> bool, +{ + #[inline] + fn next_back(&mut self) -> Option { + if self.slice.is_empty() { + None + } else { + let mut len = 1; + let mut iter = self.slice.windows(2); + while let Some([l, r]) = iter.next_back() { + if (self.predicate)(l, r) { len += 1 } else { break } + } + let slice = mem::take(&mut self.slice); + let (head, tail) = slice.split_at_mut(slice.len() - len); + self.slice = head; + Some(tail) + } + } +} + +#[unstable(feature = "slice_group_by", issue = "80552")] +impl<'a, T: 'a, P> FusedIterator for GroupByMut<'a, T, P> where P: FnMut(&T, &T) -> bool {} + +#[unstable(feature = "slice_group_by", issue = "80552")] +impl<'a, T: 'a + fmt::Debug, P> fmt::Debug for GroupByMut<'a, T, P> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("GroupByMut").field("slice", &self.slice).finish() + } +} diff --git a/crux-mir/lib/core/src/slice/iter/macros.rs b/crux-mir/lib/core/src/slice/iter/macros.rs new file mode 100644 index 000000000..0fd57b197 --- /dev/null +++ b/crux-mir/lib/core/src/slice/iter/macros.rs @@ -0,0 +1,413 @@ +//! Macros used by iterators of slice. + +// Inlining is_empty and len makes a huge performance difference +macro_rules! is_empty { + // The way we encode the length of a ZST iterator, this works both for ZST + // and non-ZST. + ($self: ident) => { + $self.ptr.as_ptr() as *const T == $self.end + }; +} + +macro_rules! len { + ($self: ident) => {{ + #![allow(unused_unsafe)] // we're sometimes used within an unsafe block + + let start = $self.ptr; + if T::IS_ZST { + // This _cannot_ use `ptr_sub` because we depend on wrapping + // to represent the length of long ZST slice iterators. + $self.end.addr().wrapping_sub(start.as_ptr().addr()) + } else { + // To get rid of some bounds checks (see `position`), we use ptr_sub instead of + // offset_from (Tested by `codegen/slice-position-bounds-check`.) + // SAFETY: by the type invariant pointers are aligned and `start <= end` + unsafe { $self.end.sub_ptr(start.as_ptr()) } + } + }}; +} + +// The shared definition of the `Iter` and `IterMut` iterators +macro_rules! iterator { + ( + struct $name:ident -> $ptr:ty, + $elem:ty, + $raw_mut:tt, + {$( $mut_:tt )?}, + {$($extra:tt)*} + ) => { + // Returns the first element and moves the start of the iterator forwards by 1. + // Greatly improves performance compared to an inlined function. The iterator + // must not be empty. + macro_rules! next_unchecked { + ($self: ident) => {& $( $mut_ )? *$self.post_inc_start(1)} + } + + // Returns the last element and moves the end of the iterator backwards by 1. + // Greatly improves performance compared to an inlined function. The iterator + // must not be empty. + macro_rules! next_back_unchecked { + ($self: ident) => {& $( $mut_ )? *$self.pre_dec_end(1)} + } + + // Shrinks the iterator when T is a ZST, by moving the end of the iterator + // backwards by `n`. `n` must not exceed `self.len()`. + macro_rules! zst_shrink { + ($self: ident, $n: ident) => { + $self.end = $self.end.wrapping_byte_sub($n); + } + } + + impl<'a, T> $name<'a, T> { + // Helper function for creating a slice from the iterator. + #[inline(always)] + fn make_slice(&self) -> &'a [T] { + // SAFETY: the iterator was created from a slice with pointer + // `self.ptr` and length `len!(self)`. This guarantees that all + // the prerequisites for `from_raw_parts` are fulfilled. + unsafe { from_raw_parts(self.ptr.as_ptr(), len!(self)) } + } + + // Helper function for moving the start of the iterator forwards by `offset` elements, + // returning the old start. + // Unsafe because the offset must not exceed `self.len()`. + #[inline(always)] + unsafe fn post_inc_start(&mut self, offset: usize) -> * $raw_mut T { + if mem::size_of::() == 0 { + zst_shrink!(self, offset); + self.ptr.as_ptr() + } else { + let old = self.ptr.as_ptr(); + // SAFETY: the caller guarantees that `offset` doesn't exceed `self.len()`, + // so this new pointer is inside `self` and thus guaranteed to be non-null. + self.ptr = unsafe { NonNull::new_unchecked(self.ptr.as_ptr().add(offset)) }; + old + } + } + + // Helper function for moving the end of the iterator backwards by `offset` elements, + // returning the new end. + // Unsafe because the offset must not exceed `self.len()`. + #[inline(always)] + unsafe fn pre_dec_end(&mut self, offset: usize) -> * $raw_mut T { + if T::IS_ZST { + zst_shrink!(self, offset); + self.ptr.as_ptr() + } else { + // SAFETY: the caller guarantees that `offset` doesn't exceed `self.len()`, + // which is guaranteed to not overflow an `isize`. Also, the resulting pointer + // is in bounds of `slice`, which fulfills the other requirements for `offset`. + self.end = unsafe { self.end.sub(offset) }; + self.end + } + } + } + + #[stable(feature = "rust1", since = "1.0.0")] + impl ExactSizeIterator for $name<'_, T> { + #[inline(always)] + fn len(&self) -> usize { + len!(self) + } + + #[inline(always)] + fn is_empty(&self) -> bool { + is_empty!(self) + } + } + + #[stable(feature = "rust1", since = "1.0.0")] + impl<'a, T> Iterator for $name<'a, T> { + type Item = $elem; + + #[inline] + fn next(&mut self) -> Option<$elem> { + // could be implemented with slices, but this avoids bounds checks + + // SAFETY: `assume` calls are safe since a slice's start pointer + // must be non-null, and slices over non-ZSTs must also have a + // non-null end pointer. The call to `next_unchecked!` is safe + // since we check if the iterator is empty first. + unsafe { + assume(!self.ptr.as_ptr().is_null()); + if !::IS_ZST { + assume(!self.end.is_null()); + } + if is_empty!(self) { + None + } else { + Some(next_unchecked!(self)) + } + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let exact = len!(self); + (exact, Some(exact)) + } + + #[inline] + fn count(self) -> usize { + len!(self) + } + + #[inline] + fn nth(&mut self, n: usize) -> Option<$elem> { + if n >= len!(self) { + // This iterator is now empty. + if T::IS_ZST { + // We have to do it this way as `ptr` may never be 0, but `end` + // could be (due to wrapping). + self.end = self.ptr.as_ptr(); + } else { + // SAFETY: end can't be 0 if T isn't ZST because ptr isn't 0 and end >= ptr + unsafe { + self.ptr = NonNull::new_unchecked(self.end as *mut T); + } + } + return None; + } + // SAFETY: We are in bounds. `post_inc_start` does the right thing even for ZSTs. + unsafe { + self.post_inc_start(n); + Some(next_unchecked!(self)) + } + } + + #[inline] + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + let advance = cmp::min(len!(self), n); + // SAFETY: By construction, `advance` does not exceed `self.len()`. + unsafe { self.post_inc_start(advance) }; + if advance == n { Ok(()) } else { Err(advance) } + } + + #[inline] + fn last(mut self) -> Option<$elem> { + self.next_back() + } + + // We override the default implementation, which uses `try_fold`, + // because this simple implementation generates less LLVM IR and is + // faster to compile. + #[inline] + fn for_each(mut self, mut f: F) + where + Self: Sized, + F: FnMut(Self::Item), + { + while let Some(x) = self.next() { + f(x); + } + } + + // We override the default implementation, which uses `try_fold`, + // because this simple implementation generates less LLVM IR and is + // faster to compile. + #[inline] + fn all(&mut self, mut f: F) -> bool + where + Self: Sized, + F: FnMut(Self::Item) -> bool, + { + while let Some(x) = self.next() { + if !f(x) { + return false; + } + } + true + } + + // We override the default implementation, which uses `try_fold`, + // because this simple implementation generates less LLVM IR and is + // faster to compile. + #[inline] + fn any(&mut self, mut f: F) -> bool + where + Self: Sized, + F: FnMut(Self::Item) -> bool, + { + while let Some(x) = self.next() { + if f(x) { + return true; + } + } + false + } + + // We override the default implementation, which uses `try_fold`, + // because this simple implementation generates less LLVM IR and is + // faster to compile. + #[inline] + fn find

(&mut self, mut predicate: P) -> Option + where + Self: Sized, + P: FnMut(&Self::Item) -> bool, + { + while let Some(x) = self.next() { + if predicate(&x) { + return Some(x); + } + } + None + } + + // We override the default implementation, which uses `try_fold`, + // because this simple implementation generates less LLVM IR and is + // faster to compile. + #[inline] + fn find_map(&mut self, mut f: F) -> Option + where + Self: Sized, + F: FnMut(Self::Item) -> Option, + { + while let Some(x) = self.next() { + if let Some(y) = f(x) { + return Some(y); + } + } + None + } + + // We override the default implementation, which uses `try_fold`, + // because this simple implementation generates less LLVM IR and is + // faster to compile. Also, the `assume` avoids a bounds check. + #[inline] + #[rustc_inherit_overflow_checks] + fn position

(&mut self, mut predicate: P) -> Option where + Self: Sized, + P: FnMut(Self::Item) -> bool, + { + let n = len!(self); + let mut i = 0; + while let Some(x) = self.next() { + if predicate(x) { + // SAFETY: we are guaranteed to be in bounds by the loop invariant: + // when `i >= n`, `self.next()` returns `None` and the loop breaks. + unsafe { assume(i < n) }; + return Some(i); + } + i += 1; + } + None + } + + // We override the default implementation, which uses `try_fold`, + // because this simple implementation generates less LLVM IR and is + // faster to compile. Also, the `assume` avoids a bounds check. + #[inline] + fn rposition

(&mut self, mut predicate: P) -> Option where + P: FnMut(Self::Item) -> bool, + Self: Sized + ExactSizeIterator + DoubleEndedIterator + { + let n = len!(self); + let mut i = n; + while let Some(x) = self.next_back() { + i -= 1; + if predicate(x) { + // SAFETY: `i` must be lower than `n` since it starts at `n` + // and is only decreasing. + unsafe { assume(i < n) }; + return Some(i); + } + } + None + } + + #[inline] + unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> Self::Item { + // SAFETY: the caller must guarantee that `i` is in bounds of + // the underlying slice, so `i` cannot overflow an `isize`, and + // the returned references is guaranteed to refer to an element + // of the slice and thus guaranteed to be valid. + // + // Also note that the caller also guarantees that we're never + // called with the same index again, and that no other methods + // that will access this subslice are called, so it is valid + // for the returned reference to be mutable in the case of + // `IterMut` + unsafe { & $( $mut_ )? * self.ptr.as_ptr().add(idx) } + } + + $($extra)* + } + + #[stable(feature = "rust1", since = "1.0.0")] + impl<'a, T> DoubleEndedIterator for $name<'a, T> { + #[inline] + fn next_back(&mut self) -> Option<$elem> { + // could be implemented with slices, but this avoids bounds checks + + // SAFETY: `assume` calls are safe since a slice's start pointer must be non-null, + // and slices over non-ZSTs must also have a non-null end pointer. + // The call to `next_back_unchecked!` is safe since we check if the iterator is + // empty first. + unsafe { + assume(!self.ptr.as_ptr().is_null()); + if !::IS_ZST { + assume(!self.end.is_null()); + } + if is_empty!(self) { + None + } else { + Some(next_back_unchecked!(self)) + } + } + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option<$elem> { + if n >= len!(self) { + // This iterator is now empty. + self.end = self.ptr.as_ptr(); + return None; + } + // SAFETY: We are in bounds. `pre_dec_end` does the right thing even for ZSTs. + unsafe { + self.pre_dec_end(n); + Some(next_back_unchecked!(self)) + } + } + + #[inline] + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + let advance = cmp::min(len!(self), n); + // SAFETY: By construction, `advance` does not exceed `self.len()`. + unsafe { self.pre_dec_end(advance) }; + if advance == n { Ok(()) } else { Err(advance) } + } + } + + #[stable(feature = "fused", since = "1.26.0")] + impl FusedIterator for $name<'_, T> {} + + #[unstable(feature = "trusted_len", issue = "37572")] + unsafe impl TrustedLen for $name<'_, T> {} + } +} + +macro_rules! forward_iterator { + ($name:ident: $elem:ident, $iter_of:ty) => { + #[stable(feature = "rust1", since = "1.0.0")] + impl<'a, $elem, P> Iterator for $name<'a, $elem, P> + where + P: FnMut(&T) -> bool, + { + type Item = $iter_of; + + #[inline] + fn next(&mut self) -> Option<$iter_of> { + self.inner.next() + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } + } + + #[stable(feature = "fused", since = "1.26.0")] + impl<'a, $elem, P> FusedIterator for $name<'a, $elem, P> where P: FnMut(&T) -> bool {} + }; +} diff --git a/crux-mir/lib/core/src/slice/memchr.rs b/crux-mir/lib/core/src/slice/memchr.rs index 2a2169dd3..c848c2e18 100644 --- a/crux-mir/lib/core/src/slice/memchr.rs +++ b/crux-mir/lib/core/src/slice/memchr.rs @@ -1,17 +1,12 @@ // Original implementation taken from rust-memchr. // Copyright 2015 Andrew Gallant, bluss and Nicolas Koch -// ignore-tidy-undocumented-unsafe - use crate::cmp; use crate::mem; -const LO_U64: u64 = 0x0101010101010101; -const HI_U64: u64 = 0x8080808080808080; - -// Use truncation. -const LO_USIZE: usize = LO_U64 as usize; -const HI_USIZE: usize = HI_U64 as usize; +const LO_USIZE: usize = usize::repeat_u8(0x01); +const HI_USIZE: usize = usize::repeat_u8(0x80); +const USIZE_BYTES: usize = mem::size_of::(); /// Returns `true` if `x` contains any zero byte. /// @@ -21,68 +16,96 @@ const HI_USIZE: usize = HI_U64 as usize; /// bytes where the borrow propagated all the way to the most significant /// bit." #[inline] -fn contains_zero_byte(x: usize) -> bool { +const fn contains_zero_byte(x: usize) -> bool { x.wrapping_sub(LO_USIZE) & !x & HI_USIZE != 0 } #[cfg(target_pointer_width = "16")] #[inline] -fn repeat_byte(b: u8) -> usize { +const fn repeat_byte(b: u8) -> usize { (b as usize) << 8 | b as usize } #[cfg(not(target_pointer_width = "16"))] #[inline] -fn repeat_byte(b: u8) -> usize { - (b as usize) * (crate::usize::MAX / 255) +const fn repeat_byte(b: u8) -> usize { + (b as usize) * (usize::MAX / 255) } /// Returns the first index matching the byte `x` in `text`. -pub fn memchr(x: u8, text: &[u8]) -> Option { +#[must_use] +#[inline] +pub const fn memchr(x: u8, text: &[u8]) -> Option { + // Fast path for small slices. + if text.len() < 2 * USIZE_BYTES { + return memchr_naive(x, text); + } + + memchr_aligned(x, text) +} + +#[inline] +const fn memchr_naive(x: u8, text: &[u8]) -> Option { + let mut i = 0; + + // FIXME(const-hack): Replace with `text.iter().pos(|c| *c == x)`. + while i < text.len() { + if text[i] == x { + return Some(i); + } + + i += 1; + } + + None +} + +const fn memchr_aligned(x: u8, text: &[u8]) -> Option { // Scan for a single byte value by reading two `usize` words at a time. // // Split `text` in three parts // - unaligned initial part, before the first word aligned address in text // - body, scan by 2 words at a time // - the last remaining part, < 2 word size + + // search up to an aligned boundary let len = text.len(); let ptr = text.as_ptr(); - let usize_bytes = mem::size_of::(); + let mut offset = ptr.align_offset(USIZE_BYTES); - // search up to an aligned boundary - let mut offset = ptr.align_offset(usize_bytes); if offset > 0 { offset = cmp::min(offset, len); - if let Some(index) = text[..offset].iter().position(|elt| *elt == x) { + if let Some(index) = memchr_naive(x, &text[..offset]) { return Some(index); } } // search the body of the text let repeated_x = repeat_byte(x); + while offset <= len - 2 * USIZE_BYTES { + // SAFETY: the while's predicate guarantees a distance of at least 2 * usize_bytes + // between the offset and the end of the slice. + unsafe { + let u = *(ptr.add(offset) as *const usize); + let v = *(ptr.add(offset + USIZE_BYTES) as *const usize); - if len >= 2 * usize_bytes { - while offset <= len - 2 * usize_bytes { - unsafe { - let u = *(ptr.add(offset) as *const usize); - let v = *(ptr.add(offset + usize_bytes) as *const usize); - - // break if there is a matching byte - let zu = contains_zero_byte(u ^ repeated_x); - let zv = contains_zero_byte(v ^ repeated_x); - if zu || zv { - break; - } + // break if there is a matching byte + let zu = contains_zero_byte(u ^ repeated_x); + let zv = contains_zero_byte(v ^ repeated_x); + if zu || zv { + break; } - offset += usize_bytes * 2; } + offset += USIZE_BYTES * 2; } // Find the byte after the point the body loop stopped. - text[offset..].iter().position(|elt| *elt == x).map(|i| offset + i) + // FIXME(const-hack): Use `?` instead. + if let Some(i) = memchr_naive(x, &text[offset..]) { Some(offset + i) } else { None } } /// Returns the last index matching the byte `x` in `text`. +#[must_use] pub fn memrchr(x: u8, text: &[u8]) -> Option { // Scan for a single byte value by reading two `usize` words at a time. // @@ -97,6 +120,8 @@ pub fn memrchr(x: u8, text: &[u8]) -> Option { let (min_aligned_offset, max_aligned_offset) = { // We call this just to obtain the length of the prefix and suffix. // In the middle we always process two chunks at once. + // SAFETY: transmuting `[u8]` to `[usize]` is safe except for size differences + // which are handled by `align_to`. let (prefix, _, suffix) = unsafe { text.align_to::<(Chunk, Chunk)>() }; (prefix.len(), len - suffix.len()) }; @@ -113,9 +138,11 @@ pub fn memrchr(x: u8, text: &[u8]) -> Option { let chunk_bytes = mem::size_of::(); while offset > min_aligned_offset { + // SAFETY: offset starts at len - suffix.len(), as long as it is greater than + // min_aligned_offset (prefix.len()) the remaining distance is at least 2 * chunk_bytes. unsafe { - let u = *(ptr.offset(offset as isize - 2 * chunk_bytes as isize) as *const Chunk); - let v = *(ptr.offset(offset as isize - chunk_bytes as isize) as *const Chunk); + let u = *(ptr.add(offset - 2 * chunk_bytes) as *const Chunk); + let v = *(ptr.add(offset - chunk_bytes) as *const Chunk); // Break if there is a matching byte. let zu = contains_zero_byte(u ^ repeated_x); diff --git a/crux-mir/lib/core/src/slice/mod.rs b/crux-mir/lib/core/src/slice/mod.rs index 4013cda2a..d93a3a57e 100644 --- a/crux-mir/lib/core/src/slice/mod.rs +++ b/crux-mir/lib/core/src/slice/mod.rs @@ -1,6 +1,3 @@ -// ignore-tidy-filelength -// ignore-tidy-undocumented-unsafe - //! Slice management and manipulation. //! //! For more details see [`std::slice`]. @@ -9,34 +6,20 @@ #![stable(feature = "rust1", since = "1.0.0")] -// How this module is organized. -// -// The library infrastructure for slices is fairly messy. There's -// a lot of stuff defined here. Let's keep it clean. -// -// The layout of this file is thus: -// -// * Inherent methods. This is where most of the slice API resides. -// * Implementations of a few common traits with important slice ops. -// * Definitions of a bunch of iterators. -// * Free functions. -// * The `raw` and `bytes` submodules. -// * Boilerplate trait implementations. - -use crate::cmp; -use crate::cmp::Ordering::{self, Equal, Greater, Less}; +use crate::cmp::Ordering::{self, Greater, Less}; use crate::fmt; -use crate::intrinsics::{assume, exact_div, is_aligned_and_not_null, unchecked_sub}; -use crate::isize; -use crate::iter::*; -use crate::marker::{self, Copy, Send, Sized, Sync}; -use crate::mem; -use crate::ops::{self, FnMut, Range}; +use crate::intrinsics::{assert_unsafe_precondition, exact_div}; +use crate::marker::Copy; +use crate::mem::{self, SizedTypeProperties}; +use crate::num::NonZeroUsize; +use crate::ops::{Bound, FnMut, OneSidedRange, Range, RangeBounds}; use crate::option::Option; use crate::option::Option::{None, Some}; -use crate::ptr::{self, NonNull}; +use crate::ptr; use crate::result::Result; use crate::result::Result::{Err, Ok}; +use crate::simd::{self, Simd}; +use crate::slice; #[unstable( feature = "slice_internals", @@ -46,14 +29,94 @@ use crate::result::Result::{Err, Ok}; /// Pure rust memchr implementation, taken from rust-memchr pub mod memchr; +#[unstable( + feature = "slice_internals", + issue = "none", + reason = "exposed from core to be reused in std;" +)] +pub mod sort; + +mod ascii; +mod cmp; +mod index; +mod iter; +mod raw; mod rotate; -mod sort; +mod specialize; + +#[stable(feature = "rust1", since = "1.0.0")] +pub use iter::{Chunks, ChunksMut, Windows}; +#[stable(feature = "rust1", since = "1.0.0")] +pub use iter::{Iter, IterMut}; +#[stable(feature = "rust1", since = "1.0.0")] +pub use iter::{RSplitN, RSplitNMut, Split, SplitMut, SplitN, SplitNMut}; + +#[stable(feature = "slice_rsplit", since = "1.27.0")] +pub use iter::{RSplit, RSplitMut}; + +#[stable(feature = "chunks_exact", since = "1.31.0")] +pub use iter::{ChunksExact, ChunksExactMut}; + +#[stable(feature = "rchunks", since = "1.31.0")] +pub use iter::{RChunks, RChunksExact, RChunksExactMut, RChunksMut}; + +#[unstable(feature = "array_chunks", issue = "74985")] +pub use iter::{ArrayChunks, ArrayChunksMut}; + +#[unstable(feature = "array_windows", issue = "75027")] +pub use iter::ArrayWindows; + +#[unstable(feature = "slice_group_by", issue = "80552")] +pub use iter::{GroupBy, GroupByMut}; + +#[stable(feature = "split_inclusive", since = "1.51.0")] +pub use iter::{SplitInclusive, SplitInclusiveMut}; + +#[stable(feature = "rust1", since = "1.0.0")] +pub use raw::{from_raw_parts, from_raw_parts_mut}; + +#[stable(feature = "from_ref", since = "1.28.0")] +pub use raw::{from_mut, from_ref}; + +#[unstable(feature = "slice_from_ptr_range", issue = "89792")] +pub use raw::{from_mut_ptr_range, from_ptr_range}; + +// This function is public only because there is no other way to unit test heapsort. +#[unstable(feature = "sort_internals", reason = "internal to sort module", issue = "none")] +pub use sort::heapsort; + +#[stable(feature = "slice_get_slice", since = "1.28.0")] +pub use index::SliceIndex; + +#[unstable(feature = "slice_range", issue = "76393")] +pub use index::range; + +#[stable(feature = "inherent_ascii_escape", since = "1.60.0")] +pub use ascii::EscapeAscii; -// -// Extension traits -// +/// Calculates the direction and split point of a one-sided range. +/// +/// This is a helper function for `take` and `take_mut` that returns +/// the direction of the split (front or back) as well as the index at +/// which to split. Returns `None` if the split index would overflow. +#[inline] +fn split_point_of(range: impl OneSidedRange) -> Option<(Direction, usize)> { + use Bound::*; + + Some(match (range.start_bound(), range.end_bound()) { + (Unbounded, Excluded(i)) => (Direction::Front, *i), + (Unbounded, Included(i)) => (Direction::Front, i.checked_add(1)?), + (Excluded(i), Unbounded) => (Direction::Back, i.checked_add(1)?), + (Included(i), Unbounded) => (Direction::Back, *i), + _ => unreachable!(), + }) +} + +enum Direction { + Front, + Back, +} -#[lang = "slice"] #[cfg(not(test))] impl [T] { /// Returns the number of elements in the slice. @@ -64,18 +127,14 @@ impl [T] { /// let a = [1, 2, 3]; /// assert_eq!(a.len(), 3); /// ``` + #[lang = "slice_len_fn"] #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_slice_len", since = "1.32.0")] + #[rustc_const_stable(feature = "const_slice_len", since = "1.39.0")] + #[rustc_allow_const_fn_unstable(ptr_metadata)] #[inline] - // SAFETY: const sound because we transmute out the length field as a usize (which it must be) - #[allow(unused_attributes)] - #[allow_internal_unstable(const_fn_union)] + #[must_use] pub const fn len(&self) -> usize { - #[allow_internal_unstable(const_fn_union)] - const fn crucible_slice_len_hook(slice: &[T]) -> usize { - unsafe { crate::ptr::Repr { rust: slice }.raw.len } - } - crucible_slice_len_hook(self) + ptr::metadata(self) } /// Returns `true` if the slice has a length of 0. @@ -87,8 +146,9 @@ impl [T] { /// assert!(!a.is_empty()); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_slice_is_empty", since = "1.32.0")] + #[rustc_const_stable(feature = "const_slice_is_empty", since = "1.39.0")] #[inline] + #[must_use] pub const fn is_empty(&self) -> bool { self.len() == 0 } @@ -105,8 +165,10 @@ impl [T] { /// assert_eq!(None, w.first()); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_slice_first_last_not_mut", since = "1.56.0")] #[inline] - pub fn first(&self) -> Option<&T> { + #[must_use] + pub const fn first(&self) -> Option<&T> { if let [first, ..] = self { Some(first) } else { None } } @@ -123,8 +185,10 @@ impl [T] { /// assert_eq!(x, &[5, 1, 2]); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_slice_first_last", issue = "83570")] #[inline] - pub fn first_mut(&mut self) -> Option<&mut T> { + #[must_use] + pub const fn first_mut(&mut self) -> Option<&mut T> { if let [first, ..] = self { Some(first) } else { None } } @@ -141,8 +205,10 @@ impl [T] { /// } /// ``` #[stable(feature = "slice_splits", since = "1.5.0")] + #[rustc_const_stable(feature = "const_slice_first_last_not_mut", since = "1.56.0")] #[inline] - pub fn split_first(&self) -> Option<(&T, &[T])> { + #[must_use] + pub const fn split_first(&self) -> Option<(&T, &[T])> { if let [first, tail @ ..] = self { Some((first, tail)) } else { None } } @@ -161,8 +227,10 @@ impl [T] { /// assert_eq!(x, &[3, 4, 5]); /// ``` #[stable(feature = "slice_splits", since = "1.5.0")] + #[rustc_const_unstable(feature = "const_slice_first_last", issue = "83570")] #[inline] - pub fn split_first_mut(&mut self) -> Option<(&mut T, &mut [T])> { + #[must_use] + pub const fn split_first_mut(&mut self) -> Option<(&mut T, &mut [T])> { if let [first, tail @ ..] = self { Some((first, tail)) } else { None } } @@ -179,8 +247,10 @@ impl [T] { /// } /// ``` #[stable(feature = "slice_splits", since = "1.5.0")] + #[rustc_const_stable(feature = "const_slice_first_last_not_mut", since = "1.56.0")] #[inline] - pub fn split_last(&self) -> Option<(&T, &[T])> { + #[must_use] + pub const fn split_last(&self) -> Option<(&T, &[T])> { if let [init @ .., last] = self { Some((last, init)) } else { None } } @@ -199,8 +269,10 @@ impl [T] { /// assert_eq!(x, &[4, 5, 3]); /// ``` #[stable(feature = "slice_splits", since = "1.5.0")] + #[rustc_const_unstable(feature = "const_slice_first_last", issue = "83570")] #[inline] - pub fn split_last_mut(&mut self) -> Option<(&mut T, &mut [T])> { + #[must_use] + pub const fn split_last_mut(&mut self) -> Option<(&mut T, &mut [T])> { if let [init @ .., last] = self { Some((last, init)) } else { None } } @@ -216,8 +288,10 @@ impl [T] { /// assert_eq!(None, w.last()); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_stable(feature = "const_slice_first_last_not_mut", since = "1.56.0")] #[inline] - pub fn last(&self) -> Option<&T> { + #[must_use] + pub const fn last(&self) -> Option<&T> { if let [.., last] = self { Some(last) } else { None } } @@ -234,8 +308,10 @@ impl [T] { /// assert_eq!(x, &[0, 1, 10]); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_slice_first_last", issue = "83570")] #[inline] - pub fn last_mut(&mut self) -> Option<&mut T> { + #[must_use] + pub const fn last_mut(&mut self) -> Option<&mut T> { if let [.., last] = self { Some(last) } else { None } } @@ -257,10 +333,12 @@ impl [T] { /// assert_eq!(None, v.get(0..4)); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_slice_index", issue = "none")] #[inline] - pub fn get(&self, index: I) -> Option<&I::Output> + #[must_use] + pub const fn get(&self, index: I) -> Option<&I::Output> where - I: SliceIndex, + I: ~const SliceIndex, { index.get(self) } @@ -268,7 +346,7 @@ impl [T] { /// Returns a mutable reference to an element or subslice depending on the /// type of index (see [`get`]) or `None` if the index is out of bounds. /// - /// [`get`]: #method.get + /// [`get`]: slice::get /// /// # Examples /// @@ -281,10 +359,12 @@ impl [T] { /// assert_eq!(x, &[0, 42, 2]); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_slice_index", issue = "none")] #[inline] - pub fn get_mut(&mut self, index: I) -> Option<&mut I::Output> + #[must_use] + pub const fn get_mut(&mut self, index: I) -> Option<&mut I::Output> where - I: SliceIndex, + I: ~const SliceIndex, { index.get_mut(self) } @@ -292,12 +372,14 @@ impl [T] { /// Returns a reference to an element or subslice, without doing bounds /// checking. /// - /// This is generally not recommended, use with caution! + /// For a safe alternative see [`get`]. + /// + /// # Safety + /// /// Calling this method with an out-of-bounds index is *[undefined behavior]* /// even if the resulting reference is not used. - /// For a safe alternative see [`get`]. /// - /// [`get`]: #method.get + /// [`get`]: slice::get /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html /// /// # Examples @@ -310,23 +392,30 @@ impl [T] { /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_slice_index", issue = "none")] #[inline] - pub unsafe fn get_unchecked(&self, index: I) -> &I::Output + #[must_use] + pub const unsafe fn get_unchecked(&self, index: I) -> &I::Output where - I: SliceIndex, + I: ~const SliceIndex, { - index.get_unchecked(self) + // SAFETY: the caller must uphold most of the safety requirements for `get_unchecked`; + // the slice is dereferenceable because `self` is a safe reference. + // The returned pointer is safe because impls of `SliceIndex` have to guarantee that it is. + unsafe { &*index.get_unchecked(self) } } /// Returns a mutable reference to an element or subslice, without doing /// bounds checking. /// - /// This is generally not recommended, use with caution! + /// For a safe alternative see [`get_mut`]. + /// + /// # Safety + /// /// Calling this method with an out-of-bounds index is *[undefined behavior]* /// even if the resulting reference is not used. - /// For a safe alternative see [`get_mut`]. /// - /// [`get_mut`]: #method.get_mut + /// [`get_mut`]: slice::get_mut /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html /// /// # Examples @@ -341,12 +430,17 @@ impl [T] { /// assert_eq!(x, &[1, 13, 4]); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_slice_index", issue = "none")] #[inline] - pub unsafe fn get_unchecked_mut(&mut self, index: I) -> &mut I::Output + #[must_use] + pub const unsafe fn get_unchecked_mut(&mut self, index: I) -> &mut I::Output where - I: SliceIndex, + I: ~const SliceIndex, { - index.get_unchecked_mut(self) + // SAFETY: the caller must uphold the safety requirements for `get_unchecked_mut`; + // the slice is dereferenceable because `self` is a safe reference. + // The returned pointer is safe because impls of `SliceIndex` have to guarantee that it is. + unsafe { &mut *index.get_unchecked_mut(self) } } /// Returns a raw pointer to the slice's buffer. @@ -374,10 +468,11 @@ impl [T] { /// } /// ``` /// - /// [`as_mut_ptr`]: #method.as_mut_ptr + /// [`as_mut_ptr`]: slice::as_mut_ptr #[stable(feature = "rust1", since = "1.0.0")] #[rustc_const_stable(feature = "const_slice_as_ptr", since = "1.32.0")] - #[inline] + #[inline(always)] + #[must_use] pub const fn as_ptr(&self) -> *const T { self as *const [T] as *const T } @@ -404,8 +499,11 @@ impl [T] { /// assert_eq!(x, &[3, 4, 6]); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - #[inline] - pub fn as_mut_ptr(&mut self) -> *mut T { + #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] + #[rustc_allow_const_fn_unstable(const_mut_refs)] + #[inline(always)] + #[must_use] + pub const fn as_mut_ptr(&mut self) -> *mut T { self as *mut [T] as *mut T } @@ -414,7 +512,7 @@ impl [T] { /// The returned range is half-open, which means that the end pointer /// points *one past* the last element of the slice. This way, an empty /// slice is represented by two equal pointers, and the difference between - /// the two pointers represents the size of the size. + /// the two pointers represents the size of the slice. /// /// See [`as_ptr`] for warnings on using these pointers. The end pointer /// requires extra caution, as it does not point to a valid element in the @@ -428,8 +526,6 @@ impl [T] { /// element of this slice: /// /// ``` - /// #![feature(slice_ptr_range)] - /// /// let a = [1, 2, 3]; /// let x = &a[1] as *const _; /// let y = &5 as *const _; @@ -438,11 +534,14 @@ impl [T] { /// assert!(!a.as_ptr_range().contains(&y)); /// ``` /// - /// [`as_ptr`]: #method.as_ptr - #[unstable(feature = "slice_ptr_range", issue = "65807")] + /// [`as_ptr`]: slice::as_ptr + #[stable(feature = "slice_ptr_range", since = "1.48.0")] + #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] #[inline] - pub fn as_ptr_range(&self) -> Range<*const T> { - // The `add` here is safe, because: + #[must_use] + pub const fn as_ptr_range(&self) -> Range<*const T> { + let start = self.as_ptr(); + // SAFETY: The `add` here is safe, because: // // - Both pointers are part of the same object, as pointing directly // past the object also counts. @@ -459,7 +558,6 @@ impl [T] { // the end of the address space. // // See the documentation of pointer::add. - let start = self.as_ptr(); let end = unsafe { start.add(self.len()) }; start..end } @@ -469,7 +567,7 @@ impl [T] { /// The returned range is half-open, which means that the end pointer /// points *one past* the last element of the slice. This way, an empty /// slice is represented by two equal pointers, and the difference between - /// the two pointers represents the size of the size. + /// the two pointers represents the size of the slice. /// /// See [`as_mut_ptr`] for warnings on using these pointers. The end /// pointer requires extra caution, as it does not point to a valid element @@ -479,12 +577,15 @@ impl [T] { /// use two pointers to refer to a range of elements in memory, as is /// common in C++. /// - /// [`as_mut_ptr`]: #method.as_mut_ptr - #[unstable(feature = "slice_ptr_range", issue = "65807")] + /// [`as_mut_ptr`]: slice::as_mut_ptr + #[stable(feature = "slice_ptr_range", since = "1.48.0")] + #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] + #[rustc_allow_const_fn_unstable(const_mut_refs)] #[inline] - pub fn as_mut_ptr_range(&mut self) -> Range<*mut T> { - // See as_ptr_range() above for why `add` here is safe. + #[must_use] + pub const fn as_mut_ptr_range(&mut self) -> Range<*mut T> { let start = self.as_mut_ptr(); + // SAFETY: See as_ptr_range() above for why `add` here is safe. let end = unsafe { start.add(self.len()) }; start..end } @@ -503,22 +604,70 @@ impl [T] { /// # Examples /// /// ``` - /// let mut v = ["a", "b", "c", "d"]; - /// v.swap(1, 3); - /// assert!(v == ["a", "d", "c", "b"]); + /// let mut v = ["a", "b", "c", "d", "e"]; + /// v.swap(2, 4); + /// assert!(v == ["a", "b", "e", "d", "c"]); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - #[inline] - pub fn swap(&mut self, a: usize, b: usize) { + #[rustc_const_unstable(feature = "const_swap", issue = "83163")] + #[inline] + #[track_caller] + pub const fn swap(&mut self, a: usize, b: usize) { + // FIXME: use swap_unchecked here (https://github.com/rust-lang/rust/pull/88540#issuecomment-944344343) + // Can't take two mutable loans from one vector, so instead use raw pointers. + let pa = ptr::addr_of_mut!(self[a]); + let pb = ptr::addr_of_mut!(self[b]); + // SAFETY: `pa` and `pb` have been created from safe mutable references and refer + // to elements in the slice and therefore are guaranteed to be valid and aligned. + // Note that accessing the elements behind `a` and `b` is checked and will + // panic when out of bounds. unsafe { - // Can't take two mutable loans from one vector, so instead just cast - // them to their raw pointers to do the swap - let pa: *mut T = &mut self[a]; - let pb: *mut T = &mut self[b]; ptr::swap(pa, pb); } } + /// Swaps two elements in the slice, without doing bounds checking. + /// + /// For a safe alternative see [`swap`]. + /// + /// # Arguments + /// + /// * a - The index of the first element + /// * b - The index of the second element + /// + /// # Safety + /// + /// Calling this method with an out-of-bounds index is *[undefined behavior]*. + /// The caller has to ensure that `a < self.len()` and `b < self.len()`. + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_swap_unchecked)] + /// + /// let mut v = ["a", "b", "c", "d"]; + /// // SAFETY: we know that 1 and 3 are both indices of the slice + /// unsafe { v.swap_unchecked(1, 3) }; + /// assert!(v == ["a", "d", "c", "b"]); + /// ``` + /// + /// [`swap`]: slice::swap + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html + #[unstable(feature = "slice_swap_unchecked", issue = "88539")] + #[rustc_const_unstable(feature = "const_swap", issue = "83163")] + pub const unsafe fn swap_unchecked(&mut self, a: usize, b: usize) { + let this = self; + let ptr = this.as_mut_ptr(); + // SAFETY: caller has to guarantee that `a < self.len()` and `b < self.len()` + unsafe { + assert_unsafe_precondition!( + "slice::swap_unchecked requires that the indices are within the slice", + [T](a: usize, b: usize, this: &mut [T]) => a < this.len() && b < this.len() + ); + ptr::swap(ptr.add(a), ptr.add(b)); + } + } + /// Reverses the order of elements in the slice, in place. /// /// # Examples @@ -529,71 +678,53 @@ impl [T] { /// assert!(v == [3, 2, 1]); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - #[inline] - pub fn reverse(&mut self) { - let mut i: usize = 0; - let ln = self.len(); - - // For very small types, all the individual reads in the normal - // path perform poorly. We can do better, given efficient unaligned - // load/store, by loading a larger chunk and reversing a register. - - // Ideally LLVM would do this for us, as it knows better than we do - // whether unaligned reads are efficient (since that changes between - // different ARM versions, for example) and what the best chunk size - // would be. Unfortunately, as of LLVM 4.0 (2017-05) it only unrolls - // the loop, so we need to do this ourselves. (Hypothesis: reverse - // is troublesome because the sides can be aligned differently -- - // will be, when the length is odd -- so there's no way of emitting - // pre- and postludes to use fully-aligned SIMD in the middle.) - - let fast_unaligned = cfg!(any(target_arch = "x86", target_arch = "x86_64")); - - if fast_unaligned && mem::size_of::() == 1 { - // Use the llvm.bswap intrinsic to reverse u8s in a usize - let chunk = mem::size_of::(); - while i + chunk - 1 < ln / 2 { - unsafe { - let pa: *mut T = self.get_unchecked_mut(i); - let pb: *mut T = self.get_unchecked_mut(ln - i - chunk); - let va = ptr::read_unaligned(pa as *mut usize); - let vb = ptr::read_unaligned(pb as *mut usize); - ptr::write_unaligned(pa as *mut usize, vb.swap_bytes()); - ptr::write_unaligned(pb as *mut usize, va.swap_bytes()); - } - i += chunk; - } - } + #[rustc_const_unstable(feature = "const_reverse", issue = "100784")] + #[inline] + pub const fn reverse(&mut self) { + let half_len = self.len() / 2; + let Range { start, end } = self.as_mut_ptr_range(); + + // These slices will skip the middle item for an odd length, + // since that one doesn't need to move. + let (front_half, back_half) = + // SAFETY: Both are subparts of the original slice, so the memory + // range is valid, and they don't overlap because they're each only + // half (or less) of the original slice. + unsafe { + ( + slice::from_raw_parts_mut(start, half_len), + slice::from_raw_parts_mut(end.sub(half_len), half_len), + ) + }; - if fast_unaligned && mem::size_of::() == 2 { - // Use rotate-by-16 to reverse u16s in a u32 - let chunk = mem::size_of::() / 2; - while i + chunk - 1 < ln / 2 { - unsafe { - let pa: *mut T = self.get_unchecked_mut(i); - let pb: *mut T = self.get_unchecked_mut(ln - i - chunk); - let va = ptr::read_unaligned(pa as *mut u32); - let vb = ptr::read_unaligned(pb as *mut u32); - ptr::write_unaligned(pa as *mut u32, vb.rotate_left(16)); - ptr::write_unaligned(pb as *mut u32, va.rotate_left(16)); - } - i += chunk; - } - } + // Introducing a function boundary here means that the two halves + // get `noalias` markers, allowing better optimization as LLVM + // knows that they're disjoint, unlike in the original slice. + revswap(front_half, back_half, half_len); - while i < ln / 2 { - // Unsafe swap to avoid the bounds check in safe swap. - unsafe { - let pa: *mut T = self.get_unchecked_mut(i); - let pb: *mut T = self.get_unchecked_mut(ln - i - 1); - ptr::swap(pa, pb); + #[inline] + const fn revswap(a: &mut [T], b: &mut [T], n: usize) { + debug_assert!(a.len() == n); + debug_assert!(b.len() == n); + + // Because this function is first compiled in isolation, + // this check tells LLVM that the indexing below is + // in-bounds. Then after inlining -- once the actual + // lengths of the slices are known -- it's removed. + let (a, b) = (&mut a[..n], &mut b[..n]); + + let mut i = 0; + while i < n { + mem::swap(&mut a[i], &mut b[n - 1 - i]); + i += 1; } - i += 1; } } /// Returns an iterator over the slice. /// + /// The iterator yields all items from start to end. + /// /// # Examples /// /// ``` @@ -608,27 +739,13 @@ impl [T] { #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn iter(&self) -> Iter<'_, T> { - unsafe { - let ptr = self.as_ptr(); - assume(!ptr.is_null()); - - let end = if mem::size_of::() == 0 { - (ptr as *const u8).wrapping_add(self.len()) as *const T - } else { - ptr.add(self.len()) - }; - - Iter { - ptr: NonNull::new_unchecked(ptr as *mut T), - end, - len: self.len(), - _marker: marker::PhantomData, - } - } + Iter::new(self) } /// Returns an iterator that allows modifying each value. /// + /// The iterator yields all items from start to end. + /// /// # Examples /// /// ``` @@ -641,23 +758,7 @@ impl [T] { #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn iter_mut(&mut self) -> IterMut<'_, T> { - unsafe { - let ptr = self.as_mut_ptr(); - assume(!ptr.is_null()); - - let end = if mem::size_of::() == 0 { - (ptr as *mut u8).wrapping_add(self.len()) as *mut T - } else { - ptr.add(self.len()) - }; - - IterMut { - ptr: NonNull::new_unchecked(ptr), - end, - len: self.len(), - _marker: marker::PhantomData, - } - } + IterMut::new(self) } /// Returns an iterator over all contiguous windows of length @@ -686,11 +787,27 @@ impl [T] { /// let mut iter = slice.windows(4); /// assert!(iter.next().is_none()); /// ``` + /// + /// There's no `windows_mut`, as that existing would let safe code violate the + /// "only one `&mut` at a time to the same thing" rule. However, you can sometimes + /// use [`Cell::as_slice_of_cells`](crate::cell::Cell::as_slice_of_cells) in + /// conjunction with `windows` to accomplish something similar: + /// ``` + /// use std::cell::Cell; + /// + /// let mut array = ['R', 'u', 's', 't', ' ', '2', '0', '1', '5']; + /// let slice = &mut array[..]; + /// let slice_of_cells: &[Cell] = Cell::from_mut(slice).as_slice_of_cells(); + /// for w in slice_of_cells.windows(3) { + /// Cell::swap(&w[0], &w[2]); + /// } + /// assert_eq!(array, ['s', 't', ' ', '2', '0', '1', '5', 'u', 'R']); + /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn windows(&self, size: usize) -> Windows<'_, T> { - assert!(size != 0); - Windows { v: self, size } + let size = NonZeroUsize::new(size).expect("size is zero"); + Windows::new(self, size) } /// Returns an iterator over `chunk_size` elements of the slice at a time, starting at the @@ -718,13 +835,13 @@ impl [T] { /// assert!(iter.next().is_none()); /// ``` /// - /// [`chunks_exact`]: #method.chunks_exact - /// [`rchunks`]: #method.rchunks + /// [`chunks_exact`]: slice::chunks_exact + /// [`rchunks`]: slice::rchunks #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn chunks(&self, chunk_size: usize) -> Chunks<'_, T> { - assert!(chunk_size != 0); - Chunks { v: self, chunk_size } + assert_ne!(chunk_size, 0, "chunks cannot have a size of zero"); + Chunks::new(self, chunk_size) } /// Returns an iterator over `chunk_size` elements of the slice at a time, starting at the @@ -756,13 +873,13 @@ impl [T] { /// assert_eq!(v, &[1, 1, 2, 2, 3]); /// ``` /// - /// [`chunks_exact_mut`]: #method.chunks_exact_mut - /// [`rchunks_mut`]: #method.rchunks_mut + /// [`chunks_exact_mut`]: slice::chunks_exact_mut + /// [`rchunks_mut`]: slice::rchunks_mut #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn chunks_mut(&mut self, chunk_size: usize) -> ChunksMut<'_, T> { - assert!(chunk_size != 0); - ChunksMut { v: self, chunk_size } + assert_ne!(chunk_size, 0, "chunks cannot have a size of zero"); + ChunksMut::new(self, chunk_size) } /// Returns an iterator over `chunk_size` elements of the slice at a time, starting at the @@ -793,16 +910,13 @@ impl [T] { /// assert_eq!(iter.remainder(), &['m']); /// ``` /// - /// [`chunks`]: #method.chunks - /// [`rchunks_exact`]: #method.rchunks_exact + /// [`chunks`]: slice::chunks + /// [`rchunks_exact`]: slice::rchunks_exact #[stable(feature = "chunks_exact", since = "1.31.0")] #[inline] pub fn chunks_exact(&self, chunk_size: usize) -> ChunksExact<'_, T> { - assert!(chunk_size != 0); - let rem = self.len() % chunk_size; - let len = self.len() - rem; - let (fst, snd) = self.split_at(len); - ChunksExact { v: fst, rem: snd, chunk_size } + assert_ne!(chunk_size, 0, "chunks cannot have a size of zero"); + ChunksExact::new(self, chunk_size) } /// Returns an iterator over `chunk_size` elements of the slice at a time, starting at the @@ -838,16 +952,354 @@ impl [T] { /// assert_eq!(v, &[1, 1, 2, 2, 0]); /// ``` /// - /// [`chunks_mut`]: #method.chunks_mut - /// [`rchunks_exact_mut`]: #method.rchunks_exact_mut + /// [`chunks_mut`]: slice::chunks_mut + /// [`rchunks_exact_mut`]: slice::rchunks_exact_mut #[stable(feature = "chunks_exact", since = "1.31.0")] #[inline] pub fn chunks_exact_mut(&mut self, chunk_size: usize) -> ChunksExactMut<'_, T> { - assert!(chunk_size != 0); - let rem = self.len() % chunk_size; - let len = self.len() - rem; - let (fst, snd) = self.split_at_mut(len); - ChunksExactMut { v: fst, rem: snd, chunk_size } + assert_ne!(chunk_size, 0, "chunks cannot have a size of zero"); + ChunksExactMut::new(self, chunk_size) + } + + /// Splits the slice into a slice of `N`-element arrays, + /// assuming that there's no remainder. + /// + /// # Safety + /// + /// This may only be called when + /// - The slice splits exactly into `N`-element chunks (aka `self.len() % N == 0`). + /// - `N != 0`. + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_as_chunks)] + /// let slice: &[char] = &['l', 'o', 'r', 'e', 'm', '!']; + /// let chunks: &[[char; 1]] = + /// // SAFETY: 1-element chunks never have remainder + /// unsafe { slice.as_chunks_unchecked() }; + /// assert_eq!(chunks, &[['l'], ['o'], ['r'], ['e'], ['m'], ['!']]); + /// let chunks: &[[char; 3]] = + /// // SAFETY: The slice length (6) is a multiple of 3 + /// unsafe { slice.as_chunks_unchecked() }; + /// assert_eq!(chunks, &[['l', 'o', 'r'], ['e', 'm', '!']]); + /// + /// // These would be unsound: + /// // let chunks: &[[_; 5]] = slice.as_chunks_unchecked() // The slice length is not a multiple of 5 + /// // let chunks: &[[_; 0]] = slice.as_chunks_unchecked() // Zero-length chunks are never allowed + /// ``` + #[unstable(feature = "slice_as_chunks", issue = "74985")] + #[inline] + #[must_use] + pub unsafe fn as_chunks_unchecked(&self) -> &[[T; N]] { + let this = self; + // SAFETY: Caller must guarantee that `N` is nonzero and exactly divides the slice length + let new_len = unsafe { + assert_unsafe_precondition!( + "slice::as_chunks_unchecked requires `N != 0` and the slice to split exactly into `N`-element chunks", + [T](this: &[T], N: usize) => N != 0 && this.len() % N == 0 + ); + exact_div(self.len(), N) + }; + // SAFETY: We cast a slice of `new_len * N` elements into + // a slice of `new_len` many `N` elements chunks. + unsafe { from_raw_parts(self.as_ptr().cast(), new_len) } + } + + /// Splits the slice into a slice of `N`-element arrays, + /// starting at the beginning of the slice, + /// and a remainder slice with length strictly less than `N`. + /// + /// # Panics + /// + /// Panics if `N` is 0. This check will most probably get changed to a compile time + /// error before this method gets stabilized. + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_as_chunks)] + /// let slice = ['l', 'o', 'r', 'e', 'm']; + /// let (chunks, remainder) = slice.as_chunks(); + /// assert_eq!(chunks, &[['l', 'o'], ['r', 'e']]); + /// assert_eq!(remainder, &['m']); + /// ``` + /// + /// If you expect the slice to be an exact multiple, you can combine + /// `let`-`else` with an empty slice pattern: + /// ``` + /// #![feature(slice_as_chunks)] + /// let slice = ['R', 'u', 's', 't']; + /// let (chunks, []) = slice.as_chunks::<2>() else { + /// panic!("slice didn't have even length") + /// }; + /// assert_eq!(chunks, &[['R', 'u'], ['s', 't']]); + /// ``` + #[unstable(feature = "slice_as_chunks", issue = "74985")] + #[inline] + #[must_use] + pub fn as_chunks(&self) -> (&[[T; N]], &[T]) { + assert_ne!(N, 0, "chunks cannot have a size of zero"); + let len = self.len() / N; + let (multiple_of_n, remainder) = self.split_at(len * N); + // SAFETY: We already panicked for zero, and ensured by construction + // that the length of the subslice is a multiple of N. + let array_slice = unsafe { multiple_of_n.as_chunks_unchecked() }; + (array_slice, remainder) + } + + /// Splits the slice into a slice of `N`-element arrays, + /// starting at the end of the slice, + /// and a remainder slice with length strictly less than `N`. + /// + /// # Panics + /// + /// Panics if `N` is 0. This check will most probably get changed to a compile time + /// error before this method gets stabilized. + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_as_chunks)] + /// let slice = ['l', 'o', 'r', 'e', 'm']; + /// let (remainder, chunks) = slice.as_rchunks(); + /// assert_eq!(remainder, &['l']); + /// assert_eq!(chunks, &[['o', 'r'], ['e', 'm']]); + /// ``` + #[unstable(feature = "slice_as_chunks", issue = "74985")] + #[inline] + #[must_use] + pub fn as_rchunks(&self) -> (&[T], &[[T; N]]) { + assert_ne!(N, 0, "chunks cannot have a size of zero"); + let len = self.len() / N; + let (remainder, multiple_of_n) = self.split_at(self.len() - len * N); + // SAFETY: We already panicked for zero, and ensured by construction + // that the length of the subslice is a multiple of N. + let array_slice = unsafe { multiple_of_n.as_chunks_unchecked() }; + (remainder, array_slice) + } + + /// Returns an iterator over `N` elements of the slice at a time, starting at the + /// beginning of the slice. + /// + /// The chunks are array references and do not overlap. If `N` does not divide the + /// length of the slice, then the last up to `N-1` elements will be omitted and can be + /// retrieved from the `remainder` function of the iterator. + /// + /// This method is the const generic equivalent of [`chunks_exact`]. + /// + /// # Panics + /// + /// Panics if `N` is 0. This check will most probably get changed to a compile time + /// error before this method gets stabilized. + /// + /// # Examples + /// + /// ``` + /// #![feature(array_chunks)] + /// let slice = ['l', 'o', 'r', 'e', 'm']; + /// let mut iter = slice.array_chunks(); + /// assert_eq!(iter.next().unwrap(), &['l', 'o']); + /// assert_eq!(iter.next().unwrap(), &['r', 'e']); + /// assert!(iter.next().is_none()); + /// assert_eq!(iter.remainder(), &['m']); + /// ``` + /// + /// [`chunks_exact`]: slice::chunks_exact + #[unstable(feature = "array_chunks", issue = "74985")] + #[inline] + pub fn array_chunks(&self) -> ArrayChunks<'_, T, N> { + assert_ne!(N, 0, "chunks cannot have a size of zero"); + ArrayChunks::new(self) + } + + /// Splits the slice into a slice of `N`-element arrays, + /// assuming that there's no remainder. + /// + /// # Safety + /// + /// This may only be called when + /// - The slice splits exactly into `N`-element chunks (aka `self.len() % N == 0`). + /// - `N != 0`. + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_as_chunks)] + /// let slice: &mut [char] = &mut ['l', 'o', 'r', 'e', 'm', '!']; + /// let chunks: &mut [[char; 1]] = + /// // SAFETY: 1-element chunks never have remainder + /// unsafe { slice.as_chunks_unchecked_mut() }; + /// chunks[0] = ['L']; + /// assert_eq!(chunks, &[['L'], ['o'], ['r'], ['e'], ['m'], ['!']]); + /// let chunks: &mut [[char; 3]] = + /// // SAFETY: The slice length (6) is a multiple of 3 + /// unsafe { slice.as_chunks_unchecked_mut() }; + /// chunks[1] = ['a', 'x', '?']; + /// assert_eq!(slice, &['L', 'o', 'r', 'a', 'x', '?']); + /// + /// // These would be unsound: + /// // let chunks: &[[_; 5]] = slice.as_chunks_unchecked_mut() // The slice length is not a multiple of 5 + /// // let chunks: &[[_; 0]] = slice.as_chunks_unchecked_mut() // Zero-length chunks are never allowed + /// ``` + #[unstable(feature = "slice_as_chunks", issue = "74985")] + #[inline] + #[must_use] + pub unsafe fn as_chunks_unchecked_mut(&mut self) -> &mut [[T; N]] { + let this = &*self; + // SAFETY: Caller must guarantee that `N` is nonzero and exactly divides the slice length + let new_len = unsafe { + assert_unsafe_precondition!( + "slice::as_chunks_unchecked_mut requires `N != 0` and the slice to split exactly into `N`-element chunks", + [T](this: &[T], N: usize) => N != 0 && this.len() % N == 0 + ); + exact_div(this.len(), N) + }; + // SAFETY: We cast a slice of `new_len * N` elements into + // a slice of `new_len` many `N` elements chunks. + unsafe { from_raw_parts_mut(self.as_mut_ptr().cast(), new_len) } + } + + /// Splits the slice into a slice of `N`-element arrays, + /// starting at the beginning of the slice, + /// and a remainder slice with length strictly less than `N`. + /// + /// # Panics + /// + /// Panics if `N` is 0. This check will most probably get changed to a compile time + /// error before this method gets stabilized. + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_as_chunks)] + /// let v = &mut [0, 0, 0, 0, 0]; + /// let mut count = 1; + /// + /// let (chunks, remainder) = v.as_chunks_mut(); + /// remainder[0] = 9; + /// for chunk in chunks { + /// *chunk = [count; 2]; + /// count += 1; + /// } + /// assert_eq!(v, &[1, 1, 2, 2, 9]); + /// ``` + #[unstable(feature = "slice_as_chunks", issue = "74985")] + #[inline] + #[must_use] + pub fn as_chunks_mut(&mut self) -> (&mut [[T; N]], &mut [T]) { + assert_ne!(N, 0, "chunks cannot have a size of zero"); + let len = self.len() / N; + let (multiple_of_n, remainder) = self.split_at_mut(len * N); + // SAFETY: We already panicked for zero, and ensured by construction + // that the length of the subslice is a multiple of N. + let array_slice = unsafe { multiple_of_n.as_chunks_unchecked_mut() }; + (array_slice, remainder) + } + + /// Splits the slice into a slice of `N`-element arrays, + /// starting at the end of the slice, + /// and a remainder slice with length strictly less than `N`. + /// + /// # Panics + /// + /// Panics if `N` is 0. This check will most probably get changed to a compile time + /// error before this method gets stabilized. + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_as_chunks)] + /// let v = &mut [0, 0, 0, 0, 0]; + /// let mut count = 1; + /// + /// let (remainder, chunks) = v.as_rchunks_mut(); + /// remainder[0] = 9; + /// for chunk in chunks { + /// *chunk = [count; 2]; + /// count += 1; + /// } + /// assert_eq!(v, &[9, 1, 1, 2, 2]); + /// ``` + #[unstable(feature = "slice_as_chunks", issue = "74985")] + #[inline] + #[must_use] + pub fn as_rchunks_mut(&mut self) -> (&mut [T], &mut [[T; N]]) { + assert_ne!(N, 0, "chunks cannot have a size of zero"); + let len = self.len() / N; + let (remainder, multiple_of_n) = self.split_at_mut(self.len() - len * N); + // SAFETY: We already panicked for zero, and ensured by construction + // that the length of the subslice is a multiple of N. + let array_slice = unsafe { multiple_of_n.as_chunks_unchecked_mut() }; + (remainder, array_slice) + } + + /// Returns an iterator over `N` elements of the slice at a time, starting at the + /// beginning of the slice. + /// + /// The chunks are mutable array references and do not overlap. If `N` does not divide + /// the length of the slice, then the last up to `N-1` elements will be omitted and + /// can be retrieved from the `into_remainder` function of the iterator. + /// + /// This method is the const generic equivalent of [`chunks_exact_mut`]. + /// + /// # Panics + /// + /// Panics if `N` is 0. This check will most probably get changed to a compile time + /// error before this method gets stabilized. + /// + /// # Examples + /// + /// ``` + /// #![feature(array_chunks)] + /// let v = &mut [0, 0, 0, 0, 0]; + /// let mut count = 1; + /// + /// for chunk in v.array_chunks_mut() { + /// *chunk = [count; 2]; + /// count += 1; + /// } + /// assert_eq!(v, &[1, 1, 2, 2, 0]); + /// ``` + /// + /// [`chunks_exact_mut`]: slice::chunks_exact_mut + #[unstable(feature = "array_chunks", issue = "74985")] + #[inline] + pub fn array_chunks_mut(&mut self) -> ArrayChunksMut<'_, T, N> { + assert_ne!(N, 0, "chunks cannot have a size of zero"); + ArrayChunksMut::new(self) + } + + /// Returns an iterator over overlapping windows of `N` elements of a slice, + /// starting at the beginning of the slice. + /// + /// This is the const generic equivalent of [`windows`]. + /// + /// If `N` is greater than the size of the slice, it will return no windows. + /// + /// # Panics + /// + /// Panics if `N` is 0. This check will most probably get changed to a compile time + /// error before this method gets stabilized. + /// + /// # Examples + /// + /// ``` + /// #![feature(array_windows)] + /// let slice = [0, 1, 2, 3]; + /// let mut iter = slice.array_windows(); + /// assert_eq!(iter.next().unwrap(), &[0, 1]); + /// assert_eq!(iter.next().unwrap(), &[1, 2]); + /// assert_eq!(iter.next().unwrap(), &[2, 3]); + /// assert!(iter.next().is_none()); + /// ``` + /// + /// [`windows`]: slice::windows + #[unstable(feature = "array_windows", issue = "75027")] + #[inline] + pub fn array_windows(&self) -> ArrayWindows<'_, T, N> { + assert_ne!(N, 0, "windows cannot have a size of zero"); + ArrayWindows::new(self) } /// Returns an iterator over `chunk_size` elements of the slice at a time, starting at the end @@ -875,13 +1327,13 @@ impl [T] { /// assert!(iter.next().is_none()); /// ``` /// - /// [`rchunks_exact`]: #method.rchunks_exact - /// [`chunks`]: #method.chunks + /// [`rchunks_exact`]: slice::rchunks_exact + /// [`chunks`]: slice::chunks #[stable(feature = "rchunks", since = "1.31.0")] #[inline] pub fn rchunks(&self, chunk_size: usize) -> RChunks<'_, T> { assert!(chunk_size != 0); - RChunks { v: self, chunk_size } + RChunks::new(self, chunk_size) } /// Returns an iterator over `chunk_size` elements of the slice at a time, starting at the end @@ -913,13 +1365,13 @@ impl [T] { /// assert_eq!(v, &[3, 2, 2, 1, 1]); /// ``` /// - /// [`rchunks_exact_mut`]: #method.rchunks_exact_mut - /// [`chunks_mut`]: #method.chunks_mut + /// [`rchunks_exact_mut`]: slice::rchunks_exact_mut + /// [`chunks_mut`]: slice::chunks_mut #[stable(feature = "rchunks", since = "1.31.0")] #[inline] pub fn rchunks_mut(&mut self, chunk_size: usize) -> RChunksMut<'_, T> { assert!(chunk_size != 0); - RChunksMut { v: self, chunk_size } + RChunksMut::new(self, chunk_size) } /// Returns an iterator over `chunk_size` elements of the slice at a time, starting at the @@ -930,7 +1382,7 @@ impl [T] { /// from the `remainder` function of the iterator. /// /// Due to each chunk having exactly `chunk_size` elements, the compiler can often optimize the - /// resulting code better than in the case of [`chunks`]. + /// resulting code better than in the case of [`rchunks`]. /// /// See [`rchunks`] for a variant of this iterator that also returns the remainder as a smaller /// chunk, and [`chunks_exact`] for the same iterator but starting at the beginning of the @@ -951,16 +1403,14 @@ impl [T] { /// assert_eq!(iter.remainder(), &['l']); /// ``` /// - /// [`chunks`]: #method.chunks - /// [`rchunks`]: #method.rchunks - /// [`chunks_exact`]: #method.chunks_exact + /// [`chunks`]: slice::chunks + /// [`rchunks`]: slice::rchunks + /// [`chunks_exact`]: slice::chunks_exact #[stable(feature = "rchunks", since = "1.31.0")] #[inline] pub fn rchunks_exact(&self, chunk_size: usize) -> RChunksExact<'_, T> { assert!(chunk_size != 0); - let rem = self.len() % chunk_size; - let (fst, snd) = self.split_at(rem); - RChunksExact { v: snd, rem: fst, chunk_size } + RChunksExact::new(self, chunk_size) } /// Returns an iterator over `chunk_size` elements of the slice at a time, starting at the end @@ -996,19 +1446,107 @@ impl [T] { /// assert_eq!(v, &[0, 2, 2, 1, 1]); /// ``` /// - /// [`chunks_mut`]: #method.chunks_mut - /// [`rchunks_mut`]: #method.rchunks_mut - /// [`chunks_exact_mut`]: #method.chunks_exact_mut + /// [`chunks_mut`]: slice::chunks_mut + /// [`rchunks_mut`]: slice::rchunks_mut + /// [`chunks_exact_mut`]: slice::chunks_exact_mut #[stable(feature = "rchunks", since = "1.31.0")] #[inline] pub fn rchunks_exact_mut(&mut self, chunk_size: usize) -> RChunksExactMut<'_, T> { assert!(chunk_size != 0); - let rem = self.len() % chunk_size; - let (fst, snd) = self.split_at_mut(rem); - RChunksExactMut { v: snd, rem: fst, chunk_size } + RChunksExactMut::new(self, chunk_size) } - /// Divides one slice into two at an index. + /// Returns an iterator over the slice producing non-overlapping runs + /// of elements using the predicate to separate them. + /// + /// The predicate is called on two elements following themselves, + /// it means the predicate is called on `slice[0]` and `slice[1]` + /// then on `slice[1]` and `slice[2]` and so on. + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_group_by)] + /// + /// let slice = &[1, 1, 1, 3, 3, 2, 2, 2]; + /// + /// let mut iter = slice.group_by(|a, b| a == b); + /// + /// assert_eq!(iter.next(), Some(&[1, 1, 1][..])); + /// assert_eq!(iter.next(), Some(&[3, 3][..])); + /// assert_eq!(iter.next(), Some(&[2, 2, 2][..])); + /// assert_eq!(iter.next(), None); + /// ``` + /// + /// This method can be used to extract the sorted subslices: + /// + /// ``` + /// #![feature(slice_group_by)] + /// + /// let slice = &[1, 1, 2, 3, 2, 3, 2, 3, 4]; + /// + /// let mut iter = slice.group_by(|a, b| a <= b); + /// + /// assert_eq!(iter.next(), Some(&[1, 1, 2, 3][..])); + /// assert_eq!(iter.next(), Some(&[2, 3][..])); + /// assert_eq!(iter.next(), Some(&[2, 3, 4][..])); + /// assert_eq!(iter.next(), None); + /// ``` + #[unstable(feature = "slice_group_by", issue = "80552")] + #[inline] + pub fn group_by(&self, pred: F) -> GroupBy<'_, T, F> + where + F: FnMut(&T, &T) -> bool, + { + GroupBy::new(self, pred) + } + + /// Returns an iterator over the slice producing non-overlapping mutable + /// runs of elements using the predicate to separate them. + /// + /// The predicate is called on two elements following themselves, + /// it means the predicate is called on `slice[0]` and `slice[1]` + /// then on `slice[1]` and `slice[2]` and so on. + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_group_by)] + /// + /// let slice = &mut [1, 1, 1, 3, 3, 2, 2, 2]; + /// + /// let mut iter = slice.group_by_mut(|a, b| a == b); + /// + /// assert_eq!(iter.next(), Some(&mut [1, 1, 1][..])); + /// assert_eq!(iter.next(), Some(&mut [3, 3][..])); + /// assert_eq!(iter.next(), Some(&mut [2, 2, 2][..])); + /// assert_eq!(iter.next(), None); + /// ``` + /// + /// This method can be used to extract the sorted subslices: + /// + /// ``` + /// #![feature(slice_group_by)] + /// + /// let slice = &mut [1, 1, 2, 3, 2, 3, 2, 3, 4]; + /// + /// let mut iter = slice.group_by_mut(|a, b| a <= b); + /// + /// assert_eq!(iter.next(), Some(&mut [1, 1, 2, 3][..])); + /// assert_eq!(iter.next(), Some(&mut [2, 3][..])); + /// assert_eq!(iter.next(), Some(&mut [2, 3, 4][..])); + /// assert_eq!(iter.next(), None); + /// ``` + #[unstable(feature = "slice_group_by", issue = "80552")] + #[inline] + pub fn group_by_mut(&mut self, pred: F) -> GroupByMut<'_, T, F> + where + F: FnMut(&T, &T) -> bool, + { + GroupByMut::new(self, pred) + } + + /// Divides one slice into two at an index. /// /// The first will contain all indices from `[0, mid)` (excluding /// the index `mid` itself) and the second will contain all @@ -1025,26 +1563,32 @@ impl [T] { /// /// { /// let (left, right) = v.split_at(0); - /// assert!(left == []); - /// assert!(right == [1, 2, 3, 4, 5, 6]); + /// assert_eq!(left, []); + /// assert_eq!(right, [1, 2, 3, 4, 5, 6]); /// } /// /// { /// let (left, right) = v.split_at(2); - /// assert!(left == [1, 2]); - /// assert!(right == [3, 4, 5, 6]); + /// assert_eq!(left, [1, 2]); + /// assert_eq!(right, [3, 4, 5, 6]); /// } /// /// { /// let (left, right) = v.split_at(6); - /// assert!(left == [1, 2, 3, 4, 5, 6]); - /// assert!(right == []); + /// assert_eq!(left, [1, 2, 3, 4, 5, 6]); + /// assert_eq!(right, []); /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_slice_split_at_not_mut", issue = "101158")] #[inline] - pub fn split_at(&self, mid: usize) -> (&[T], &[T]) { - (&self[..mid], &self[mid..]) + #[track_caller] + #[must_use] + pub const fn split_at(&self, mid: usize) -> (&[T], &[T]) { + assert!(mid <= self.len()); + // SAFETY: `[ptr; mid]` and `[mid; len]` are inside `self`, which + // fulfills the requirements of `split_at_unchecked`. + unsafe { self.split_at_unchecked(mid) } } /// Divides one mutable slice into two at an index. @@ -1061,29 +1605,295 @@ impl [T] { /// /// ``` /// let mut v = [1, 0, 3, 0, 5, 6]; + /// let (left, right) = v.split_at_mut(2); + /// assert_eq!(left, [1, 0]); + /// assert_eq!(right, [3, 0, 5, 6]); + /// left[1] = 2; + /// right[1] = 4; + /// assert_eq!(v, [1, 2, 3, 4, 5, 6]); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[inline] + #[track_caller] + #[must_use] + #[rustc_const_unstable(feature = "const_slice_split_at_mut", issue = "101804")] + pub const fn split_at_mut(&mut self, mid: usize) -> (&mut [T], &mut [T]) { + assert!(mid <= self.len()); + // SAFETY: `[ptr; mid]` and `[mid; len]` are inside `self`, which + // fulfills the requirements of `from_raw_parts_mut`. + unsafe { self.split_at_mut_unchecked(mid) } + } + + /// Divides one slice into two at an index, without doing bounds checking. + /// + /// The first will contain all indices from `[0, mid)` (excluding + /// the index `mid` itself) and the second will contain all + /// indices from `[mid, len)` (excluding the index `len` itself). + /// + /// For a safe alternative see [`split_at`]. + /// + /// # Safety + /// + /// Calling this method with an out-of-bounds index is *[undefined behavior]* + /// even if the resulting reference is not used. The caller has to ensure that + /// `0 <= mid <= self.len()`. + /// + /// [`split_at`]: slice::split_at + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_split_at_unchecked)] + /// + /// let v = [1, 2, 3, 4, 5, 6]; + /// + /// unsafe { + /// let (left, right) = v.split_at_unchecked(0); + /// assert_eq!(left, []); + /// assert_eq!(right, [1, 2, 3, 4, 5, 6]); + /// } + /// + /// unsafe { + /// let (left, right) = v.split_at_unchecked(2); + /// assert_eq!(left, [1, 2]); + /// assert_eq!(right, [3, 4, 5, 6]); + /// } + /// + /// unsafe { + /// let (left, right) = v.split_at_unchecked(6); + /// assert_eq!(left, [1, 2, 3, 4, 5, 6]); + /// assert_eq!(right, []); + /// } + /// ``` + #[unstable(feature = "slice_split_at_unchecked", reason = "new API", issue = "76014")] + #[rustc_const_unstable(feature = "slice_split_at_unchecked", issue = "76014")] + #[inline] + #[must_use] + pub const unsafe fn split_at_unchecked(&self, mid: usize) -> (&[T], &[T]) { + // HACK: the const function `from_raw_parts` is used to make this + // function const; previously the implementation used + // `(self.get_unchecked(..mid), self.get_unchecked(mid..))` + + let len = self.len(); + let ptr = self.as_ptr(); + + // SAFETY: Caller has to check that `0 <= mid <= self.len()` + unsafe { (from_raw_parts(ptr, mid), from_raw_parts(ptr.add(mid), len - mid)) } + } + + /// Divides one mutable slice into two at an index, without doing bounds checking. + /// + /// The first will contain all indices from `[0, mid)` (excluding + /// the index `mid` itself) and the second will contain all + /// indices from `[mid, len)` (excluding the index `len` itself). + /// + /// For a safe alternative see [`split_at_mut`]. + /// + /// # Safety + /// + /// Calling this method with an out-of-bounds index is *[undefined behavior]* + /// even if the resulting reference is not used. The caller has to ensure that + /// `0 <= mid <= self.len()`. + /// + /// [`split_at_mut`]: slice::split_at_mut + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_split_at_unchecked)] + /// + /// let mut v = [1, 0, 3, 0, 5, 6]; /// // scoped to restrict the lifetime of the borrows - /// { - /// let (left, right) = v.split_at_mut(2); - /// assert!(left == [1, 0]); - /// assert!(right == [3, 0, 5, 6]); + /// unsafe { + /// let (left, right) = v.split_at_mut_unchecked(2); + /// assert_eq!(left, [1, 0]); + /// assert_eq!(right, [3, 0, 5, 6]); /// left[1] = 2; /// right[1] = 4; /// } - /// assert!(v == [1, 2, 3, 4, 5, 6]); + /// assert_eq!(v, [1, 2, 3, 4, 5, 6]); /// ``` - #[stable(feature = "rust1", since = "1.0.0")] + #[unstable(feature = "slice_split_at_unchecked", reason = "new API", issue = "76014")] + #[rustc_const_unstable(feature = "const_slice_split_at_mut", issue = "101804")] #[inline] - pub fn split_at_mut(&mut self, mid: usize) -> (&mut [T], &mut [T]) { + #[must_use] + pub const unsafe fn split_at_mut_unchecked(&mut self, mid: usize) -> (&mut [T], &mut [T]) { let len = self.len(); let ptr = self.as_mut_ptr(); + // SAFETY: Caller has to check that `0 <= mid <= self.len()`. + // + // `[ptr; mid]` and `[mid; len]` are not overlapping, so returning a mutable reference + // is fine. unsafe { - assert!(mid <= len); - + assert_unsafe_precondition!( + "slice::split_at_mut_unchecked requires the index to be within the slice", + (mid: usize, len: usize) => mid <= len + ); (from_raw_parts_mut(ptr, mid), from_raw_parts_mut(ptr.add(mid), len - mid)) } } + /// Divides one slice into an array and a remainder slice at an index. + /// + /// The array will contain all indices from `[0, N)` (excluding + /// the index `N` itself) and the slice will contain all + /// indices from `[N, len)` (excluding the index `len` itself). + /// + /// # Panics + /// + /// Panics if `N > len`. + /// + /// # Examples + /// + /// ``` + /// #![feature(split_array)] + /// + /// let v = &[1, 2, 3, 4, 5, 6][..]; + /// + /// { + /// let (left, right) = v.split_array_ref::<0>(); + /// assert_eq!(left, &[]); + /// assert_eq!(right, [1, 2, 3, 4, 5, 6]); + /// } + /// + /// { + /// let (left, right) = v.split_array_ref::<2>(); + /// assert_eq!(left, &[1, 2]); + /// assert_eq!(right, [3, 4, 5, 6]); + /// } + /// + /// { + /// let (left, right) = v.split_array_ref::<6>(); + /// assert_eq!(left, &[1, 2, 3, 4, 5, 6]); + /// assert_eq!(right, []); + /// } + /// ``` + #[unstable(feature = "split_array", reason = "new API", issue = "90091")] + #[inline] + #[track_caller] + #[must_use] + pub fn split_array_ref(&self) -> (&[T; N], &[T]) { + let (a, b) = self.split_at(N); + // SAFETY: a points to [T; N]? Yes it's [T] of length N (checked by split_at) + unsafe { (&*(a.as_ptr() as *const [T; N]), b) } + } + + /// Divides one mutable slice into an array and a remainder slice at an index. + /// + /// The array will contain all indices from `[0, N)` (excluding + /// the index `N` itself) and the slice will contain all + /// indices from `[N, len)` (excluding the index `len` itself). + /// + /// # Panics + /// + /// Panics if `N > len`. + /// + /// # Examples + /// + /// ``` + /// #![feature(split_array)] + /// + /// let mut v = &mut [1, 0, 3, 0, 5, 6][..]; + /// let (left, right) = v.split_array_mut::<2>(); + /// assert_eq!(left, &mut [1, 0]); + /// assert_eq!(right, [3, 0, 5, 6]); + /// left[1] = 2; + /// right[1] = 4; + /// assert_eq!(v, [1, 2, 3, 4, 5, 6]); + /// ``` + #[unstable(feature = "split_array", reason = "new API", issue = "90091")] + #[inline] + #[track_caller] + #[must_use] + pub fn split_array_mut(&mut self) -> (&mut [T; N], &mut [T]) { + let (a, b) = self.split_at_mut(N); + // SAFETY: a points to [T; N]? Yes it's [T] of length N (checked by split_at_mut) + unsafe { (&mut *(a.as_mut_ptr() as *mut [T; N]), b) } + } + + /// Divides one slice into an array and a remainder slice at an index from + /// the end. + /// + /// The slice will contain all indices from `[0, len - N)` (excluding + /// the index `len - N` itself) and the array will contain all + /// indices from `[len - N, len)` (excluding the index `len` itself). + /// + /// # Panics + /// + /// Panics if `N > len`. + /// + /// # Examples + /// + /// ``` + /// #![feature(split_array)] + /// + /// let v = &[1, 2, 3, 4, 5, 6][..]; + /// + /// { + /// let (left, right) = v.rsplit_array_ref::<0>(); + /// assert_eq!(left, [1, 2, 3, 4, 5, 6]); + /// assert_eq!(right, &[]); + /// } + /// + /// { + /// let (left, right) = v.rsplit_array_ref::<2>(); + /// assert_eq!(left, [1, 2, 3, 4]); + /// assert_eq!(right, &[5, 6]); + /// } + /// + /// { + /// let (left, right) = v.rsplit_array_ref::<6>(); + /// assert_eq!(left, []); + /// assert_eq!(right, &[1, 2, 3, 4, 5, 6]); + /// } + /// ``` + #[unstable(feature = "split_array", reason = "new API", issue = "90091")] + #[inline] + #[must_use] + pub fn rsplit_array_ref(&self) -> (&[T], &[T; N]) { + assert!(N <= self.len()); + let (a, b) = self.split_at(self.len() - N); + // SAFETY: b points to [T; N]? Yes it's [T] of length N (checked by split_at) + unsafe { (a, &*(b.as_ptr() as *const [T; N])) } + } + + /// Divides one mutable slice into an array and a remainder slice at an + /// index from the end. + /// + /// The slice will contain all indices from `[0, len - N)` (excluding + /// the index `N` itself) and the array will contain all + /// indices from `[len - N, len)` (excluding the index `len` itself). + /// + /// # Panics + /// + /// Panics if `N > len`. + /// + /// # Examples + /// + /// ``` + /// #![feature(split_array)] + /// + /// let mut v = &mut [1, 0, 3, 0, 5, 6][..]; + /// let (left, right) = v.rsplit_array_mut::<4>(); + /// assert_eq!(left, [1, 0]); + /// assert_eq!(right, &mut [3, 0, 5, 6]); + /// left[1] = 2; + /// right[1] = 4; + /// assert_eq!(v, [1, 2, 3, 4, 5, 6]); + /// ``` + #[unstable(feature = "split_array", reason = "new API", issue = "90091")] + #[inline] + #[must_use] + pub fn rsplit_array_mut(&mut self) -> (&mut [T], &mut [T; N]) { + assert!(N <= self.len()); + let (a, b) = self.split_at_mut(self.len() - N); + // SAFETY: b points to [T; N]? Yes it's [T] of length N (checked by split_at_mut) + unsafe { (a, &mut *(b.as_mut_ptr() as *mut [T; N])) } + } + /// Returns an iterator over subslices separated by elements that match /// `pred`. The matched element is not contained in the subslices. /// @@ -1130,7 +1940,7 @@ impl [T] { where F: FnMut(&T) -> bool, { - Split { v: self, pred, finished: false } + Split::new(self, pred) } /// Returns an iterator over mutable subslices separated by elements that @@ -1152,7 +1962,7 @@ impl [T] { where F: FnMut(&T) -> bool, { - SplitMut { v: self, pred, finished: false } + SplitMut::new(self, pred) } /// Returns an iterator over subslices separated by elements that match @@ -1162,7 +1972,6 @@ impl [T] { /// # Examples /// /// ``` - /// #![feature(split_inclusive)] /// let slice = [10, 40, 33, 20]; /// let mut iter = slice.split_inclusive(|num| num % 3 == 0); /// @@ -1176,7 +1985,6 @@ impl [T] { /// That slice will be the last item returned by the iterator. /// /// ``` - /// #![feature(split_inclusive)] /// let slice = [3, 10, 40, 33]; /// let mut iter = slice.split_inclusive(|num| num % 3 == 0); /// @@ -1184,13 +1992,13 @@ impl [T] { /// assert_eq!(iter.next().unwrap(), &[10, 40, 33]); /// assert!(iter.next().is_none()); /// ``` - #[unstable(feature = "split_inclusive", issue = "none")] + #[stable(feature = "split_inclusive", since = "1.51.0")] #[inline] pub fn split_inclusive(&self, pred: F) -> SplitInclusive<'_, T, F> where F: FnMut(&T) -> bool, { - SplitInclusive { v: self, pred, finished: false } + SplitInclusive::new(self, pred) } /// Returns an iterator over mutable subslices separated by elements that @@ -1200,7 +2008,6 @@ impl [T] { /// # Examples /// /// ``` - /// #![feature(split_inclusive)] /// let mut v = [10, 40, 30, 20, 60, 50]; /// /// for group in v.split_inclusive_mut(|num| *num % 3 == 0) { @@ -1209,13 +2016,13 @@ impl [T] { /// } /// assert_eq!(v, [10, 40, 1, 20, 1, 1]); /// ``` - #[unstable(feature = "split_inclusive", issue = "none")] + #[stable(feature = "split_inclusive", since = "1.51.0")] #[inline] pub fn split_inclusive_mut(&mut self, pred: F) -> SplitInclusiveMut<'_, T, F> where F: FnMut(&T) -> bool, { - SplitInclusiveMut { v: self, pred, finished: false } + SplitInclusiveMut::new(self, pred) } /// Returns an iterator over subslices separated by elements that match @@ -1251,7 +2058,7 @@ impl [T] { where F: FnMut(&T) -> bool, { - RSplit { inner: self.split(pred) } + RSplit::new(self, pred) } /// Returns an iterator over mutable subslices separated by elements that @@ -1277,7 +2084,7 @@ impl [T] { where F: FnMut(&T) -> bool, { - RSplitMut { inner: self.split_mut(pred) } + RSplitMut::new(self, pred) } /// Returns an iterator over subslices separated by elements that match @@ -1296,7 +2103,7 @@ impl [T] { /// let v = [10, 40, 30, 20, 60, 50]; /// /// for group in v.splitn(2, |num| *num % 3 == 0) { - /// println!("{:?}", group); + /// println!("{group:?}"); /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] @@ -1305,10 +2112,10 @@ impl [T] { where F: FnMut(&T) -> bool, { - SplitN { inner: GenericSplitN { iter: self.split(pred), count: n } } + SplitN::new(self.split(pred), n) } - /// Returns an iterator over subslices separated by elements that match + /// Returns an iterator over mutable subslices separated by elements that match /// `pred`, limited to returning at most `n` items. The matched element is /// not contained in the subslices. /// @@ -1331,7 +2138,7 @@ impl [T] { where F: FnMut(&T) -> bool, { - SplitNMut { inner: GenericSplitN { iter: self.split_mut(pred), count: n } } + SplitNMut::new(self.split_mut(pred), n) } /// Returns an iterator over subslices separated by elements that match @@ -1351,7 +2158,7 @@ impl [T] { /// let v = [10, 40, 30, 20, 60, 50]; /// /// for group in v.rsplitn(2, |num| *num % 3 == 0) { - /// println!("{:?}", group); + /// println!("{group:?}"); /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] @@ -1360,7 +2167,7 @@ impl [T] { where F: FnMut(&T) -> bool, { - RSplitN { inner: GenericSplitN { iter: self.rsplit(pred), count: n } } + RSplitN::new(self.rsplit(pred), n) } /// Returns an iterator over subslices separated by elements that match @@ -1387,11 +2194,17 @@ impl [T] { where F: FnMut(&T) -> bool, { - RSplitNMut { inner: GenericSplitN { iter: self.rsplit_mut(pred), count: n } } + RSplitNMut::new(self.rsplit_mut(pred), n) } /// Returns `true` if the slice contains an element with the given value. /// + /// This operation is *O*(*n*). + /// + /// Note that if you have a sorted slice, [`binary_search`] may be faster. + /// + /// [`binary_search`]: slice::binary_search + /// /// # Examples /// /// ``` @@ -1400,8 +2213,9 @@ impl [T] { /// assert!(!v.contains(&50)); /// ``` /// - /// If you do not have an `&T`, but just an `&U` such that `T: Borrow` - /// (e.g. `String: Borrow`), you can use `iter().any`: + /// If you do not have a `&T`, but some other value that you can compare + /// with one (for example, `String` implements `PartialEq`), you can + /// use `iter().any`: /// /// ``` /// let v = [String::from("hello"), String::from("world")]; // slice of `String` @@ -1409,11 +2223,13 @@ impl [T] { /// assert!(!v.iter().any(|e| e == "hi")); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[inline] + #[must_use] pub fn contains(&self, x: &T) -> bool where T: PartialEq, { - x.slice_contains(self) + cmp::SliceContains::slice_contains(x, self) } /// Returns `true` if `needle` is a prefix of the slice. @@ -1437,6 +2253,7 @@ impl [T] { /// assert!(v.starts_with(&[])); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] pub fn starts_with(&self, needle: &[T]) -> bool where T: PartialEq, @@ -1466,6 +2283,7 @@ impl [T] { /// assert!(v.ends_with(&[])); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] pub fn ends_with(&self, needle: &[T]) -> bool where T: PartialEq, @@ -1474,13 +2292,95 @@ impl [T] { m >= n && needle == &self[m - n..] } - /// Binary searches this sorted slice for a given element. + /// Returns a subslice with the prefix removed. + /// + /// If the slice starts with `prefix`, returns the subslice after the prefix, wrapped in `Some`. + /// If `prefix` is empty, simply returns the original slice. + /// + /// If the slice does not start with `prefix`, returns `None`. + /// + /// # Examples + /// + /// ``` + /// let v = &[10, 40, 30]; + /// assert_eq!(v.strip_prefix(&[10]), Some(&[40, 30][..])); + /// assert_eq!(v.strip_prefix(&[10, 40]), Some(&[30][..])); + /// assert_eq!(v.strip_prefix(&[50]), None); + /// assert_eq!(v.strip_prefix(&[10, 50]), None); + /// + /// let prefix : &str = "he"; + /// assert_eq!(b"hello".strip_prefix(prefix.as_bytes()), + /// Some(b"llo".as_ref())); + /// ``` + #[must_use = "returns the subslice without modifying the original"] + #[stable(feature = "slice_strip", since = "1.51.0")] + pub fn strip_prefix + ?Sized>(&self, prefix: &P) -> Option<&[T]> + where + T: PartialEq, + { + // This function will need rewriting if and when SlicePattern becomes more sophisticated. + let prefix = prefix.as_slice(); + let n = prefix.len(); + if n <= self.len() { + let (head, tail) = self.split_at(n); + if head == prefix { + return Some(tail); + } + } + None + } + + /// Returns a subslice with the suffix removed. + /// + /// If the slice ends with `suffix`, returns the subslice before the suffix, wrapped in `Some`. + /// If `suffix` is empty, simply returns the original slice. + /// + /// If the slice does not end with `suffix`, returns `None`. + /// + /// # Examples + /// + /// ``` + /// let v = &[10, 40, 30]; + /// assert_eq!(v.strip_suffix(&[30]), Some(&[10, 40][..])); + /// assert_eq!(v.strip_suffix(&[40, 30]), Some(&[10][..])); + /// assert_eq!(v.strip_suffix(&[50]), None); + /// assert_eq!(v.strip_suffix(&[50, 30]), None); + /// ``` + #[must_use = "returns the subslice without modifying the original"] + #[stable(feature = "slice_strip", since = "1.51.0")] + pub fn strip_suffix + ?Sized>(&self, suffix: &P) -> Option<&[T]> + where + T: PartialEq, + { + // This function will need rewriting if and when SlicePattern becomes more sophisticated. + let suffix = suffix.as_slice(); + let (len, n) = (self.len(), suffix.len()); + if n <= len { + let (head, tail) = self.split_at(len - n); + if tail == suffix { + return Some(head); + } + } + None + } + + /// Binary searches this slice for a given element. + /// This behaves similarly to [`contains`] if this slice is sorted. /// /// If the value is found then [`Result::Ok`] is returned, containing the /// index of the matching element. If there are multiple matches, then any - /// one of the matches could be returned. If the value is not found then - /// [`Result::Err`] is returned, containing the index where a matching - /// element could be inserted while maintaining sorted order. + /// one of the matches could be returned. The index is chosen + /// deterministically, but is subject to change in future versions of Rust. + /// If the value is not found then [`Result::Err`] is returned, containing + /// the index where a matching element could be inserted while maintaining + /// sorted order. + /// + /// See also [`binary_search_by`], [`binary_search_by_key`], and [`partition_point`]. + /// + /// [`contains`]: slice::contains + /// [`binary_search_by`]: slice::binary_search_by + /// [`binary_search_by_key`]: slice::binary_search_by_key + /// [`partition_point`]: slice::partition_point /// /// # Examples /// @@ -1498,13 +2398,36 @@ impl [T] { /// assert!(match r { Ok(1..=4) => true, _ => false, }); /// ``` /// + /// If you want to find that whole *range* of matching items, rather than + /// an arbitrary matching one, that can be done using [`partition_point`]: + /// ``` + /// let s = [0, 1, 1, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55]; + /// + /// let low = s.partition_point(|x| x < &1); + /// assert_eq!(low, 1); + /// let high = s.partition_point(|x| x <= &1); + /// assert_eq!(high, 5); + /// let r = s.binary_search(&1); + /// assert!((low..high).contains(&r.unwrap())); + /// + /// assert!(s[..low].iter().all(|&x| x < 1)); + /// assert!(s[low..high].iter().all(|&x| x == 1)); + /// assert!(s[high..].iter().all(|&x| x > 1)); + /// + /// // For something not found, the "range" of equal items is empty + /// assert_eq!(s.partition_point(|x| x < &11), 9); + /// assert_eq!(s.partition_point(|x| x <= &11), 9); + /// assert_eq!(s.binary_search(&11), Err(9)); + /// ``` + /// /// If you want to insert an item to a sorted vector, while maintaining - /// sort order: + /// sort order, consider using [`partition_point`]: /// /// ``` /// let mut s = vec![0, 1, 1, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55]; /// let num = 42; - /// let idx = s.binary_search(&num).unwrap_or_else(|x| x); + /// let idx = s.partition_point(|&x| x < num); + /// // The above is equivalent to `let idx = s.binary_search(&num).unwrap_or_else(|x| x);` /// s.insert(idx, num); /// assert_eq!(s, [0, 1, 1, 1, 1, 2, 3, 5, 8, 13, 21, 34, 42, 55]); /// ``` @@ -1516,7 +2439,8 @@ impl [T] { self.binary_search_by(|p| p.cmp(x)) } - /// Binary searches this sorted slice with a comparator function. + /// Binary searches this slice with a comparator function. + /// This behaves similarly to [`contains`] if this slice is sorted. /// /// The comparator function should implement an order consistent /// with the sort order of the underlying slice, returning an @@ -1525,9 +2449,18 @@ impl [T] { /// /// If the value is found then [`Result::Ok`] is returned, containing the /// index of the matching element. If there are multiple matches, then any - /// one of the matches could be returned. If the value is not found then - /// [`Result::Err`] is returned, containing the index where a matching - /// element could be inserted while maintaining sorted order. + /// one of the matches could be returned. The index is chosen + /// deterministically, but is subject to change in future versions of Rust. + /// If the value is not found then [`Result::Err`] is returned, containing + /// the index where a matching element could be inserted while maintaining + /// sorted order. + /// + /// See also [`binary_search`], [`binary_search_by_key`], and [`partition_point`]. + /// + /// [`contains`]: slice::contains + /// [`binary_search`]: slice::binary_search + /// [`binary_search_by_key`]: slice::binary_search_by_key + /// [`partition_point`]: slice::partition_point /// /// # Examples /// @@ -1554,39 +2487,65 @@ impl [T] { where F: FnMut(&'a T) -> Ordering, { - let s = self; - let mut size = s.len(); - if size == 0 { - return Err(0); - } - let mut base = 0usize; - while size > 1 { - let half = size / 2; - let mid = base + half; - // mid is always in [0, size), that means mid is >= 0 and < size. - // mid >= 0: by definition - // mid < size: mid = size / 2 + size / 4 + size / 8 ... - let cmp = f(unsafe { s.get_unchecked(mid) }); - base = if cmp == Greater { base } else { mid }; - size -= half; + // INVARIANTS: + // - 0 <= left <= left + size = right <= self.len() + // - f returns Less for everything in self[..left] + // - f returns Greater for everything in self[right..] + let mut size = self.len(); + let mut left = 0; + let mut right = size; + while left < right { + let mid = left + size / 2; + + // SAFETY: the while condition means `size` is strictly positive, so + // `size/2 < size`. Thus `left + size/2 < left + size`, which + // coupled with the `left + size <= self.len()` invariant means + // we have `left + size/2 < self.len()`, and this is in-bounds. + let cmp = f(unsafe { self.get_unchecked(mid) }); + + // The reason why we use if/else control flow rather than match + // is because match reorders comparison operations, which is perf sensitive. + // This is x86 asm for u8: https://rust.godbolt.org/z/8Y8Pra. + if cmp == Less { + left = mid + 1; + } else if cmp == Greater { + right = mid; + } else { + // SAFETY: same as the `get_unchecked` above + unsafe { crate::intrinsics::assume(mid < self.len()) }; + return Ok(mid); + } + + size = right - left; } - // base is always in [0, size) because base <= mid. - let cmp = f(unsafe { s.get_unchecked(base) }); - if cmp == Equal { Ok(base) } else { Err(base + (cmp == Less) as usize) } + + // SAFETY: directly true from the overall invariant. + // Note that this is `<=`, unlike the assume in the `Ok` path. + unsafe { crate::intrinsics::assume(left <= self.len()) }; + Err(left) } - /// Binary searches this sorted slice with a key extraction function. + /// Binary searches this slice with a key extraction function. + /// This behaves similarly to [`contains`] if this slice is sorted. /// /// Assumes that the slice is sorted by the key, for instance with /// [`sort_by_key`] using the same key extraction function. /// /// If the value is found then [`Result::Ok`] is returned, containing the /// index of the matching element. If there are multiple matches, then any - /// one of the matches could be returned. If the value is not found then - /// [`Result::Err`] is returned, containing the index where a matching - /// element could be inserted while maintaining sorted order. + /// one of the matches could be returned. The index is chosen + /// deterministically, but is subject to change in future versions of Rust. + /// If the value is not found then [`Result::Err`] is returned, containing + /// the index where a matching element could be inserted while maintaining + /// sorted order. + /// + /// See also [`binary_search`], [`binary_search_by`], and [`partition_point`]. /// - /// [`sort_by_key`]: #method.sort_by_key + /// [`contains`]: slice::contains + /// [`sort_by_key`]: slice::sort_by_key + /// [`binary_search`]: slice::binary_search + /// [`binary_search_by`]: slice::binary_search_by + /// [`partition_point`]: slice::partition_point /// /// # Examples /// @@ -1600,12 +2559,17 @@ impl [T] { /// (1, 2), (2, 3), (4, 5), (5, 8), (3, 13), /// (1, 21), (2, 34), (4, 55)]; /// - /// assert_eq!(s.binary_search_by_key(&13, |&(a,b)| b), Ok(9)); - /// assert_eq!(s.binary_search_by_key(&4, |&(a,b)| b), Err(7)); - /// assert_eq!(s.binary_search_by_key(&100, |&(a,b)| b), Err(13)); - /// let r = s.binary_search_by_key(&1, |&(a,b)| b); + /// assert_eq!(s.binary_search_by_key(&13, |&(a, b)| b), Ok(9)); + /// assert_eq!(s.binary_search_by_key(&4, |&(a, b)| b), Err(7)); + /// assert_eq!(s.binary_search_by_key(&100, |&(a, b)| b), Err(13)); + /// let r = s.binary_search_by_key(&1, |&(a, b)| b); /// assert!(match r { Ok(1..=4) => true, _ => false, }); /// ``` + // Lint rustdoc::broken_intra_doc_links is allowed as `slice::sort_by_key` is + // in crate `alloc`, and as such doesn't exists yet when building `core`: #74481. + // This breaks links when slice is displayed in core, but changing it to use relative links + // would break when the item is re-exported. So allow the core links to be broken for now. + #[allow(rustdoc::broken_intra_doc_links)] #[stable(feature = "slice_binary_search_by_key", since = "1.10.0")] #[inline] pub fn binary_search_by_key<'a, B, F>(&'a self, b: &B, mut f: F) -> Result @@ -1616,10 +2580,10 @@ impl [T] { self.binary_search_by(|k| f(k).cmp(b)) } - /// Sorts the slice, but may not preserve the order of equal elements. + /// Sorts the slice, but might not preserve the order of equal elements. /// /// This sort is unstable (i.e., may reorder equal elements), in-place - /// (i.e., does not allocate), and `O(n log n)` worst-case. + /// (i.e., does not allocate), and *O*(*n* \* log(*n*)) worst-case. /// /// # Current implementation /// @@ -1648,28 +2612,28 @@ impl [T] { where T: Ord, { - sort::quicksort(self, |a, b| a.lt(b)); + sort::quicksort(self, T::lt); } - /// Sorts the slice with a comparator function, but may not preserve the order of equal + /// Sorts the slice with a comparator function, but might not preserve the order of equal /// elements. /// /// This sort is unstable (i.e., may reorder equal elements), in-place - /// (i.e., does not allocate), and `O(n log n)` worst-case. + /// (i.e., does not allocate), and *O*(*n* \* log(*n*)) worst-case. /// /// The comparator function must define a total ordering for the elements in the slice. If /// the ordering is not total, the order of the elements is unspecified. An order is a - /// total order if it is (for all a, b and c): + /// total order if it is (for all `a`, `b` and `c`): /// - /// * total and antisymmetric: exactly one of a < b, a == b or a > b is true; and - /// * transitive, a < b and b < c implies a < c. The same must hold for both == and >. + /// * total and antisymmetric: exactly one of `a < b`, `a == b` or `a > b` is true, and + /// * transitive, `a < b` and `b < c` implies `a < c`. The same must hold for both `==` and `>`. /// /// For example, while [`f64`] doesn't implement [`Ord`] because `NaN != NaN`, we can use /// `partial_cmp` as our sort function when we know the slice doesn't contain a `NaN`. /// /// ``` /// let mut floats = [5f64, 4.0, 1.0, 3.0, 2.0]; - /// floats.sort_by(|a, b| a.partial_cmp(b).unwrap()); + /// floats.sort_unstable_by(|a, b| a.partial_cmp(b).unwrap()); /// assert_eq!(floats, [1.0, 2.0, 3.0, 4.0, 5.0]); /// ``` /// @@ -1706,12 +2670,12 @@ impl [T] { sort::quicksort(self, |a, b| compare(a, b) == Ordering::Less); } - /// Sorts the slice with a key extraction function, but may not preserve the order of equal + /// Sorts the slice with a key extraction function, but might not preserve the order of equal /// elements. /// /// This sort is unstable (i.e., may reorder equal elements), in-place - /// (i.e., does not allocate), and `O(m n log(m n))` worst-case, where the key function is - /// `O(m)`. + /// (i.e., does not allocate), and *O*(m \* *n* \* log(*n*)) worst-case, where the key function is + /// *O*(*m*). /// /// # Current implementation /// @@ -1750,17 +2714,18 @@ impl [T] { /// This reordering has the additional property that any value at position `i < index` will be /// less than or equal to any value at a position `j > index`. Additionally, this reordering is /// unstable (i.e. any number of equal elements may end up at position `index`), in-place - /// (i.e. does not allocate), and `O(n)` worst-case. This function is also/ known as "kth - /// element" in other libraries. It returns a triplet of the following values: all elements less - /// than the one at the given index, the value at the given index, and all elements greater than - /// the one at the given index. + /// (i.e. does not allocate), and *O*(*n*) worst-case. This function is also/ known as "kth + /// element" in other libraries. It returns a triplet of the following from the reordered slice: + /// the subslice prior to `index`, the element at `index`, and the subslice after `index`; + /// accordingly, the values in those two subslices will respectively all be less-than-or-equal-to + /// and greater-than-or-equal-to the value of the element at `index`. /// /// # Current implementation /// /// The current algorithm is based on the quickselect portion of the same quicksort algorithm /// used for [`sort_unstable`]. /// - /// [`sort_unstable`]: #method.sort_unstable + /// [`sort_unstable`]: slice::sort_unstable /// /// # Panics /// @@ -1769,12 +2734,10 @@ impl [T] { /// # Examples /// /// ``` - /// #![feature(slice_partition_at_index)] - /// /// let mut v = [-5i32, 4, 1, -3, 2]; /// /// // Find the median - /// v.partition_at_index(2); + /// v.select_nth_unstable(2); /// /// // We are only guaranteed the slice will be one of the following, based on the way we sort /// // about the specified index. @@ -1783,14 +2746,13 @@ impl [T] { /// v == [-3, -5, 1, 4, 2] || /// v == [-5, -3, 1, 4, 2]); /// ``` - #[unstable(feature = "slice_partition_at_index", issue = "55300")] + #[stable(feature = "slice_select_nth_unstable", since = "1.49.0")] #[inline] - pub fn partition_at_index(&mut self, index: usize) -> (&mut [T], &mut T, &mut [T]) + pub fn select_nth_unstable(&mut self, index: usize) -> (&mut [T], &mut T, &mut [T]) where T: Ord, { - let mut f = |a: &T, b: &T| a.lt(b); - sort::partition_at_index(self, index, &mut f) + sort::partition_at_index(self, index, T::lt) } /// Reorder the slice with a comparator function such that the element at `index` is at its @@ -1799,18 +2761,19 @@ impl [T] { /// This reordering has the additional property that any value at position `i < index` will be /// less than or equal to any value at a position `j > index` using the comparator function. /// Additionally, this reordering is unstable (i.e. any number of equal elements may end up at - /// position `index`), in-place (i.e. does not allocate), and `O(n)` worst-case. This function - /// is also known as "kth element" in other libraries. It returns a triplet of the following - /// values: all elements less than the one at the given index, the value at the given index, - /// and all elements greater than the one at the given index, using the provided comparator - /// function. + /// position `index`), in-place (i.e. does not allocate), and *O*(*n*) worst-case. This function + /// is also known as "kth element" in other libraries. It returns a triplet of the following from + /// the slice reordered according to the provided comparator function: the subslice prior to + /// `index`, the element at `index`, and the subslice after `index`; accordingly, the values in + /// those two subslices will respectively all be less-than-or-equal-to and greater-than-or-equal-to + /// the value of the element at `index`. /// /// # Current implementation /// /// The current algorithm is based on the quickselect portion of the same quicksort algorithm /// used for [`sort_unstable`]. /// - /// [`sort_unstable`]: #method.sort_unstable + /// [`sort_unstable`]: slice::sort_unstable /// /// # Panics /// @@ -1819,12 +2782,10 @@ impl [T] { /// # Examples /// /// ``` - /// #![feature(slice_partition_at_index)] - /// /// let mut v = [-5i32, 4, 1, -3, 2]; /// /// // Find the median as if the slice were sorted in descending order. - /// v.partition_at_index_by(2, |a, b| b.cmp(a)); + /// v.select_nth_unstable_by(2, |a, b| b.cmp(a)); /// /// // We are only guaranteed the slice will be one of the following, based on the way we sort /// // about the specified index. @@ -1833,9 +2794,9 @@ impl [T] { /// v == [4, 2, 1, -5, -3] || /// v == [4, 2, 1, -3, -5]); /// ``` - #[unstable(feature = "slice_partition_at_index", issue = "55300")] + #[stable(feature = "slice_select_nth_unstable", since = "1.49.0")] #[inline] - pub fn partition_at_index_by( + pub fn select_nth_unstable_by( &mut self, index: usize, mut compare: F, @@ -1843,8 +2804,7 @@ impl [T] { where F: FnMut(&T, &T) -> Ordering, { - let mut f = |a: &T, b: &T| compare(a, b) == Less; - sort::partition_at_index(self, index, &mut f) + sort::partition_at_index(self, index, |a: &T, b: &T| compare(a, b) == Less) } /// Reorder the slice with a key extraction function such that the element at `index` is at its @@ -1853,18 +2813,19 @@ impl [T] { /// This reordering has the additional property that any value at position `i < index` will be /// less than or equal to any value at a position `j > index` using the key extraction function. /// Additionally, this reordering is unstable (i.e. any number of equal elements may end up at - /// position `index`), in-place (i.e. does not allocate), and `O(n)` worst-case. This function - /// is also known as "kth element" in other libraries. It returns a triplet of the following - /// values: all elements less than the one at the given index, the value at the given index, and - /// all elements greater than the one at the given index, using the provided key extraction - /// function. + /// position `index`), in-place (i.e. does not allocate), and *O*(*n*) worst-case. This function + /// is also known as "kth element" in other libraries. It returns a triplet of the following from + /// the slice reordered according to the provided key extraction function: the subslice prior to + /// `index`, the element at `index`, and the subslice after `index`; accordingly, the values in + /// those two subslices will respectively all be less-than-or-equal-to and greater-than-or-equal-to + /// the value of the element at `index`. /// /// # Current implementation /// /// The current algorithm is based on the quickselect portion of the same quicksort algorithm /// used for [`sort_unstable`]. /// - /// [`sort_unstable`]: #method.sort_unstable + /// [`sort_unstable`]: slice::sort_unstable /// /// # Panics /// @@ -1873,12 +2834,10 @@ impl [T] { /// # Examples /// /// ``` - /// #![feature(slice_partition_at_index)] - /// /// let mut v = [-5i32, 4, 1, -3, 2]; /// /// // Return the median as if the array were sorted according to absolute value. - /// v.partition_at_index_by_key(2, |a| a.abs()); + /// v.select_nth_unstable_by_key(2, |a| a.abs()); /// /// // We are only guaranteed the slice will be one of the following, based on the way we sort /// // about the specified index. @@ -1887,9 +2846,9 @@ impl [T] { /// v == [2, 1, -3, 4, -5] || /// v == [2, 1, -3, -5, 4]); /// ``` - #[unstable(feature = "slice_partition_at_index", issue = "55300")] + #[stable(feature = "slice_select_nth_unstable", since = "1.49.0")] #[inline] - pub fn partition_at_index_by_key( + pub fn select_nth_unstable_by_key( &mut self, index: usize, mut f: F, @@ -1898,8 +2857,7 @@ impl [T] { F: FnMut(&T) -> K, K: Ord, { - let mut g = |a: &T, b: &T| f(a).lt(&f(b)); - sort::partition_at_index(self, index, &mut g) + sort::partition_at_index(self, index, |a: &T, b: &T| f(a).lt(&f(b))) } /// Moves all consecutive repeated elements to the end of the slice according to the @@ -1970,7 +2928,7 @@ impl [T] { // over all the elements, swapping as we go so that at the end // the elements we wish to keep are in the front, and those we // wish to reject are at the back. We can then split the slice. - // This operation is still O(n). + // This operation is still `O(n)`. // // Example: We start in this state, where `r` represents "next // read" and `w` represents "next_write`. @@ -2028,6 +2986,21 @@ impl [T] { let mut next_read: usize = 1; let mut next_write: usize = 1; + // SAFETY: the `while` condition guarantees `next_read` and `next_write` + // are less than `len`, thus are inside `self`. `prev_ptr_write` points to + // one element before `ptr_write`, but `next_write` starts at 1, so + // `prev_ptr_write` is never less than 0 and is inside the slice. + // This fulfils the requirements for dereferencing `ptr_read`, `prev_ptr_write` + // and `ptr_write`, and for using `ptr.add(next_read)`, `ptr.add(next_write - 1)` + // and `prev_ptr_write.offset(1)`. + // + // `next_write` is also incremented at most once per loop at most meaning + // no element is skipped when it may need to be swapped. + // + // `ptr_read` and `prev_ptr_write` never point to the same element. This + // is required for `&mut *ptr_read`, `&mut *prev_ptr_write` to be safe. + // The explanation is simply that `next_read >= next_write` is always true, + // thus `next_read > next_write - 1` is too. unsafe { // Avoid bounds checks by using raw pointers. while next_read < len { @@ -2035,7 +3008,7 @@ impl [T] { let prev_ptr_write = ptr.add(next_write - 1); if !same_bucket(&mut *ptr_read, &mut *prev_ptr_write) { if next_read != next_write { - let ptr_write = prev_ptr_write.offset(1); + let ptr_write = prev_ptr_write.add(1); mem::swap(&mut *ptr_read, &mut *ptr_write); } next_write += 1; @@ -2111,9 +3084,11 @@ impl [T] { pub fn rotate_left(&mut self, mid: usize) { assert!(mid <= self.len()); let k = self.len() - mid; + let p = self.as_mut_ptr(); + // SAFETY: The range `[p.add(mid) - mid, p.add(mid) + k)` is trivially + // valid for reading and writing, as required by `ptr_rotate`. unsafe { - let p = self.as_mut_ptr(); rotate::ptr_rotate(mid, p.add(mid), k); } } @@ -2152,34 +3127,77 @@ impl [T] { pub fn rotate_right(&mut self, k: usize) { assert!(k <= self.len()); let mid = self.len() - k; + let p = self.as_mut_ptr(); + // SAFETY: The range `[p.add(mid) - mid, p.add(mid) + k)` is trivially + // valid for reading and writing, as required by `ptr_rotate`. unsafe { - let p = self.as_mut_ptr(); rotate::ptr_rotate(mid, p.add(mid), k); } } - /// Copies the elements from `src` into `self`. + /// Fills `self` with elements by cloning `value`. /// - /// The length of `src` must be the same as `self`. + /// # Examples /// - /// If `src` implements `Copy`, it can be more performant to use - /// [`copy_from_slice`]. + /// ``` + /// let mut buf = vec![0; 10]; + /// buf.fill(1); + /// assert_eq!(buf, vec![1; 10]); + /// ``` + #[doc(alias = "memset")] + #[stable(feature = "slice_fill", since = "1.50.0")] + pub fn fill(&mut self, value: T) + where + T: Clone, + { + specialize::SpecFill::spec_fill(self, value); + } + + /// Fills `self` with elements returned by calling a closure repeatedly. /// - /// # Panics + /// This method uses a closure to create new values. If you'd rather + /// [`Clone`] a given value, use [`fill`]. If you want to use the [`Default`] + /// trait to generate values, you can pass [`Default::default`] as the + /// argument. /// - /// This function will panic if the two slices have different lengths. + /// [`fill`]: slice::fill /// /// # Examples /// - /// Cloning two elements from a slice into another: - /// /// ``` - /// let src = [1, 2, 3, 4]; - /// let mut dst = [0, 0]; - /// - /// // Because the slices have to be the same length, - /// // we slice the source slice from four elements + /// let mut buf = vec![1; 10]; + /// buf.fill_with(Default::default); + /// assert_eq!(buf, vec![0; 10]); + /// ``` + #[stable(feature = "slice_fill_with", since = "1.51.0")] + pub fn fill_with(&mut self, mut f: F) + where + F: FnMut() -> T, + { + for el in self { + *el = f(); + } + } + + /// Copies the elements from `src` into `self`. + /// + /// The length of `src` must be the same as `self`. + /// + /// # Panics + /// + /// This function will panic if the two slices have different lengths. + /// + /// # Examples + /// + /// Cloning two elements from a slice into another: + /// + /// ``` + /// let src = [1, 2, 3, 4]; + /// let mut dst = [0, 0]; + /// + /// // Because the slices have to be the same length, + /// // we slice the source slice from four elements /// // to two. It will panic if we don't do this. /// dst.clone_from_slice(&src[2..]); /// @@ -2212,29 +3230,22 @@ impl [T] { /// assert_eq!(slice, [4, 5, 3, 4, 5]); /// ``` /// - /// [`copy_from_slice`]: #method.copy_from_slice - /// [`split_at_mut`]: #method.split_at_mut + /// [`copy_from_slice`]: slice::copy_from_slice + /// [`split_at_mut`]: slice::split_at_mut #[stable(feature = "clone_from_slice", since = "1.7.0")] + #[track_caller] pub fn clone_from_slice(&mut self, src: &[T]) where T: Clone, { - assert!(self.len() == src.len(), "destination and source slices have different lengths"); - // NOTE: We need to explicitly slice them to the same length - // for bounds checking to be elided, and the optimizer will - // generate memcpy for simple cases (for example T = u8). - let len = self.len(); - let src = &src[..len]; - for i in 0..len { - self[i].clone_from(&src[i]); - } + self.spec_clone_from(src); } /// Copies all elements from `src` into `self`, using a memcpy. /// /// The length of `src` must be the same as `self`. /// - /// If `src` does not implement `Copy`, use [`clone_from_slice`]. + /// If `T` does not implement `Copy`, use [`clone_from_slice`]. /// /// # Panics /// @@ -2282,14 +3293,34 @@ impl [T] { /// assert_eq!(slice, [4, 5, 3, 4, 5]); /// ``` /// - /// [`clone_from_slice`]: #method.clone_from_slice - /// [`split_at_mut`]: #method.split_at_mut + /// [`clone_from_slice`]: slice::clone_from_slice + /// [`split_at_mut`]: slice::split_at_mut + #[doc(alias = "memcpy")] #[stable(feature = "copy_from_slice", since = "1.9.0")] + #[track_caller] pub fn copy_from_slice(&mut self, src: &[T]) where T: Copy, { - assert_eq!(self.len(), src.len(), "destination and source slices have different lengths"); + // The panic code path was put into a cold function to not bloat the + // call site. + #[inline(never)] + #[cold] + #[track_caller] + fn len_mismatch_fail(dst_len: usize, src_len: usize) -> ! { + panic!( + "source slice length ({}) does not match destination slice length ({})", + src_len, dst_len, + ); + } + + if self.len() != src.len() { + len_mismatch_fail(self.len(), src.len()); + } + + // SAFETY: `self` is valid for `self.len()` elements by definition, and `src` was + // checked to have the same length. The slices cannot overlap because + // mutable references are exclusive. unsafe { ptr::copy_nonoverlapping(src.as_ptr(), self.as_mut_ptr(), self.len()); } @@ -2320,30 +3351,22 @@ impl [T] { /// assert_eq!(&bytes, b"Hello, Wello!"); /// ``` #[stable(feature = "copy_within", since = "1.37.0")] - pub fn copy_within>(&mut self, src: R, dest: usize) + #[track_caller] + pub fn copy_within>(&mut self, src: R, dest: usize) where T: Copy, { - let src_start = match src.start_bound() { - ops::Bound::Included(&n) => n, - ops::Bound::Excluded(&n) => { - n.checked_add(1).unwrap_or_else(|| slice_index_overflow_fail()) - } - ops::Bound::Unbounded => 0, - }; - let src_end = match src.end_bound() { - ops::Bound::Included(&n) => { - n.checked_add(1).unwrap_or_else(|| slice_index_overflow_fail()) - } - ops::Bound::Excluded(&n) => n, - ops::Bound::Unbounded => self.len(), - }; - assert!(src_start <= src_end, "src end is before src start"); - assert!(src_end <= self.len(), "src is out of bounds"); + let Range { start: src_start, end: src_end } = slice::range(src, ..self.len()); let count = src_end - src_start; assert!(dest <= self.len() - count, "dest is out of bounds"); + // SAFETY: the conditions for `ptr::copy` have all been checked above, + // as have those for `ptr::add`. unsafe { - ptr::copy(self.as_ptr().add(src_start), self.as_mut_ptr().add(dest), count); + // Derive both `src_ptr` and `dest_ptr` from the same loan + let ptr = self.as_mut_ptr(); + let src_ptr = ptr.add(src_start); + let dest_ptr = ptr.add(dest); + ptr::copy(src_ptr, dest_ptr, count); } } @@ -2393,10 +3416,14 @@ impl [T] { /// assert_eq!(slice, [4, 5, 3, 1, 2]); /// ``` /// - /// [`split_at_mut`]: #method.split_at_mut + /// [`split_at_mut`]: slice::split_at_mut #[stable(feature = "swap_with_slice", since = "1.27.0")] + #[track_caller] pub fn swap_with_slice(&mut self, other: &mut [T]) { assert!(self.len() == other.len(), "destination and source slices have different lengths"); + // SAFETY: `self` is valid for `self.len()` elements by definition, and `src` was + // checked to have the same length. The slices cannot overlap because + // mutable references are exclusive. unsafe { ptr::swap_nonoverlapping(self.as_mut_ptr(), other.as_mut_ptr(), self.len()); } @@ -2428,6 +3455,8 @@ impl [T] { // iterative stein’s algorithm // We should still make this `const fn` (and revert to recursive algorithm if we do) // because relying on llvm to consteval all this is… well, it makes me uncomfortable. + + // SAFETY: `a` and `b` are checked to be non-zero values. let (ctz_a, mut ctz_b) = unsafe { if a == 0 { return b; @@ -2447,6 +3476,7 @@ impl [T] { mem::swap(&mut a, &mut b); } b = b - a; + // SAFETY: `b` is checked to be non-zero. unsafe { if b == 0 { break; @@ -2471,10 +3501,11 @@ impl [T] { /// maintained. /// /// This method splits the slice into three distinct slices: prefix, correctly aligned middle - /// slice of a new type, and the suffix slice. The method may make the middle slice the greatest - /// length possible for a given type and input slice, but only your algorithm's performance - /// should depend on that, not its correctness. It is permissible for all of the input data to - /// be returned as the prefix or suffix slice. + /// slice of a new type, and the suffix slice. How exactly the slice is split up is not + /// specified; the middle part may be smaller than necessary. However, if this fails to return a + /// maximal middle part, that is because code is running in a context where performance does not + /// matter, such as a sanitizer attempting to find alignment bugs. Regular code running + /// in a default (debug or release) execution *will* return a maximal middle part. /// /// This method has no purpose when either input element `T` or output element `U` are /// zero-sized and will return the original slice without splitting anything. @@ -2498,9 +3529,10 @@ impl [T] { /// } /// ``` #[stable(feature = "slice_align_to", since = "1.30.0")] + #[must_use] pub unsafe fn align_to(&self) -> (&[T], &[U], &[T]) { // Note that most of this function will be constant-evaluated, - if mem::size_of::() == 0 || mem::size_of::() == 0 { + if U::IS_ZST || T::IS_ZST { // handle ZSTs specially, which is – don't handle them at all. return (self, &[], &[]); } @@ -2508,29 +3540,34 @@ impl [T] { // First, find at what point do we split between the first and 2nd slice. Easy with // ptr.align_offset. let ptr = self.as_ptr(); - let offset = crate::ptr::align_offset(ptr, mem::align_of::()); + // SAFETY: See the `align_to_mut` method for the detailed safety comment. + let offset = unsafe { crate::ptr::align_offset(ptr, mem::align_of::()) }; if offset > self.len() { (self, &[], &[]) } else { let (left, rest) = self.split_at(offset); - // now `rest` is definitely aligned, so `from_raw_parts_mut` below is okay let (us_len, ts_len) = rest.align_to_offsets::(); - ( - left, - from_raw_parts(rest.as_ptr() as *const U, us_len), - from_raw_parts(rest.as_ptr().add(rest.len() - ts_len), ts_len), - ) + // SAFETY: now `rest` is definitely aligned, so `from_raw_parts` below is okay, + // since the caller guarantees that we can transmute `T` to `U` safely. + unsafe { + ( + left, + from_raw_parts(rest.as_ptr() as *const U, us_len), + from_raw_parts(rest.as_ptr().add(rest.len() - ts_len), ts_len), + ) + } } } - /// Transmute the slice to a slice of another type, ensuring alignment of the types is - /// maintained. + /// Transmute the mutable slice to a mutable slice of another type, ensuring alignment of the + /// types is maintained. /// /// This method splits the slice into three distinct slices: prefix, correctly aligned middle - /// slice of a new type, and the suffix slice. The method may make the middle slice the greatest - /// length possible for a given type and input slice, but only your algorithm's performance - /// should depend on that, not its correctness. It is permissible for all of the input data to - /// be returned as the prefix or suffix slice. + /// slice of a new type, and the suffix slice. How exactly the slice is split up is not + /// specified; the middle part may be smaller than necessary. However, if this fails to return a + /// maximal middle part, that is because code is running in a context where performance does not + /// matter, such as a sanitizer attempting to find alignment bugs. Regular code running + /// in a default (debug or release) execution *will* return a maximal middle part. /// /// This method has no purpose when either input element `T` or output element `U` are /// zero-sized and will return the original slice without splitting anything. @@ -2554,9 +3591,10 @@ impl [T] { /// } /// ``` #[stable(feature = "slice_align_to", since = "1.30.0")] + #[must_use] pub unsafe fn align_to_mut(&mut self) -> (&mut [T], &mut [U], &mut [T]) { // Note that most of this function will be constant-evaluated, - if mem::size_of::() == 0 || mem::size_of::() == 0 { + if U::IS_ZST || T::IS_ZST { // handle ZSTs specially, which is – don't handle them at all. return (self, &mut [], &mut []); } @@ -2564,24 +3602,152 @@ impl [T] { // First, find at what point do we split between the first and 2nd slice. Easy with // ptr.align_offset. let ptr = self.as_ptr(); - let offset = crate::ptr::align_offset(ptr, mem::align_of::()); + // SAFETY: Here we are ensuring we will use aligned pointers for U for the + // rest of the method. This is done by passing a pointer to &[T] with an + // alignment targeted for U. + // `crate::ptr::align_offset` is called with a correctly aligned and + // valid pointer `ptr` (it comes from a reference to `self`) and with + // a size that is a power of two (since it comes from the alignment for U), + // satisfying its safety constraints. + let offset = unsafe { crate::ptr::align_offset(ptr, mem::align_of::()) }; if offset > self.len() { (self, &mut [], &mut []) } else { let (left, rest) = self.split_at_mut(offset); - // now `rest` is definitely aligned, so `from_raw_parts_mut` below is okay let (us_len, ts_len) = rest.align_to_offsets::(); let rest_len = rest.len(); let mut_ptr = rest.as_mut_ptr(); // We can't use `rest` again after this, that would invalidate its alias `mut_ptr`! - ( - left, - from_raw_parts_mut(mut_ptr as *mut U, us_len), - from_raw_parts_mut(mut_ptr.add(rest_len - ts_len), ts_len), - ) + // SAFETY: see comments for `align_to`. + unsafe { + ( + left, + from_raw_parts_mut(mut_ptr as *mut U, us_len), + from_raw_parts_mut(mut_ptr.add(rest_len - ts_len), ts_len), + ) + } } } + /// Split a slice into a prefix, a middle of aligned SIMD types, and a suffix. + /// + /// This is a safe wrapper around [`slice::align_to`], so has the same weak + /// postconditions as that method. You're only assured that + /// `self.len() == prefix.len() + middle.len() * LANES + suffix.len()`. + /// + /// Notably, all of the following are possible: + /// - `prefix.len() >= LANES`. + /// - `middle.is_empty()` despite `self.len() >= 3 * LANES`. + /// - `suffix.len() >= LANES`. + /// + /// That said, this is a safe method, so if you're only writing safe code, + /// then this can at most cause incorrect logic, not unsoundness. + /// + /// # Panics + /// + /// This will panic if the size of the SIMD type is different from + /// `LANES` times that of the scalar. + /// + /// At the time of writing, the trait restrictions on `Simd` keeps + /// that from ever happening, as only power-of-two numbers of lanes are + /// supported. It's possible that, in the future, those restrictions might + /// be lifted in a way that would make it possible to see panics from this + /// method for something like `LANES == 3`. + /// + /// # Examples + /// + /// ``` + /// #![feature(portable_simd)] + /// use core::simd::SimdFloat; + /// + /// let short = &[1, 2, 3]; + /// let (prefix, middle, suffix) = short.as_simd::<4>(); + /// assert_eq!(middle, []); // Not enough elements for anything in the middle + /// + /// // They might be split in any possible way between prefix and suffix + /// let it = prefix.iter().chain(suffix).copied(); + /// assert_eq!(it.collect::>(), vec![1, 2, 3]); + /// + /// fn basic_simd_sum(x: &[f32]) -> f32 { + /// use std::ops::Add; + /// use std::simd::f32x4; + /// let (prefix, middle, suffix) = x.as_simd(); + /// let sums = f32x4::from_array([ + /// prefix.iter().copied().sum(), + /// 0.0, + /// 0.0, + /// suffix.iter().copied().sum(), + /// ]); + /// let sums = middle.iter().copied().fold(sums, f32x4::add); + /// sums.reduce_sum() + /// } + /// + /// let numbers: Vec = (1..101).map(|x| x as _).collect(); + /// assert_eq!(basic_simd_sum(&numbers[1..99]), 4949.0); + /// ``` + #[unstable(feature = "portable_simd", issue = "86656")] + #[must_use] + pub fn as_simd(&self) -> (&[T], &[Simd], &[T]) + where + Simd: AsRef<[T; LANES]>, + T: simd::SimdElement, + simd::LaneCount: simd::SupportedLaneCount, + { + // These are expected to always match, as vector types are laid out like + // arrays per , but we + // might as well double-check since it'll optimize away anyhow. + assert_eq!(mem::size_of::>(), mem::size_of::<[T; LANES]>()); + + // SAFETY: The simd types have the same layout as arrays, just with + // potentially-higher alignment, so the de-facto transmutes are sound. + unsafe { self.align_to() } + } + + /// Split a mutable slice into a mutable prefix, a middle of aligned SIMD types, + /// and a mutable suffix. + /// + /// This is a safe wrapper around [`slice::align_to_mut`], so has the same weak + /// postconditions as that method. You're only assured that + /// `self.len() == prefix.len() + middle.len() * LANES + suffix.len()`. + /// + /// Notably, all of the following are possible: + /// - `prefix.len() >= LANES`. + /// - `middle.is_empty()` despite `self.len() >= 3 * LANES`. + /// - `suffix.len() >= LANES`. + /// + /// That said, this is a safe method, so if you're only writing safe code, + /// then this can at most cause incorrect logic, not unsoundness. + /// + /// This is the mutable version of [`slice::as_simd`]; see that for examples. + /// + /// # Panics + /// + /// This will panic if the size of the SIMD type is different from + /// `LANES` times that of the scalar. + /// + /// At the time of writing, the trait restrictions on `Simd` keeps + /// that from ever happening, as only power-of-two numbers of lanes are + /// supported. It's possible that, in the future, those restrictions might + /// be lifted in a way that would make it possible to see panics from this + /// method for something like `LANES == 3`. + #[unstable(feature = "portable_simd", issue = "86656")] + #[must_use] + pub fn as_simd_mut(&mut self) -> (&mut [T], &mut [Simd], &mut [T]) + where + Simd: AsMut<[T; LANES]>, + T: simd::SimdElement, + simd::LaneCount: simd::SupportedLaneCount, + { + // These are expected to always match, as vector types are laid out like + // arrays per , but we + // might as well double-check since it'll optimize away anyhow. + assert_eq!(mem::size_of::>(), mem::size_of::<[T; LANES]>()); + + // SAFETY: The simd types have the same layout as arrays, just with + // potentially-higher alignment, so the de-facto transmutes are sound. + unsafe { self.align_to_mut() } + } + /// Checks if the elements of this slice are sorted. /// /// That is, for each element `a` and its following element `b`, `a <= b` must hold. If the @@ -2601,10 +3767,11 @@ impl [T] { /// assert!(![1, 3, 2, 4].is_sorted()); /// assert!([0].is_sorted()); /// assert!(empty.is_sorted()); - /// assert!(![0.0, 1.0, std::f32::NAN].is_sorted()); + /// assert!(![0.0, 1.0, f32::NAN].is_sorted()); /// ``` #[inline] #[unstable(feature = "is_sorted", reason = "new API", issue = "53485")] + #[must_use] pub fn is_sorted(&self) -> bool where T: PartialOrd, @@ -2618,11 +3785,12 @@ impl [T] { /// function to determine the ordering of two elements. Apart from that, it's equivalent to /// [`is_sorted`]; see its documentation for more information. /// - /// [`is_sorted`]: #method.is_sorted + /// [`is_sorted`]: slice::is_sorted #[unstable(feature = "is_sorted", reason = "new API", issue = "53485")] - pub fn is_sorted_by(&self, mut compare: F) -> bool + #[must_use] + pub fn is_sorted_by<'a, F>(&'a self, mut compare: F) -> bool where - F: FnMut(&T, &T) -> Option, + F: FnMut(&'a T, &'a T) -> Option, { self.iter().is_sorted_by(|a, b| compare(*a, *b)) } @@ -2633,7 +3801,7 @@ impl [T] { /// elements, as determined by `f`. Apart from that, it's equivalent to [`is_sorted`]; see its /// documentation for more information. /// - /// [`is_sorted`]: #method.is_sorted + /// [`is_sorted`]: slice::is_sorted /// /// # Examples /// @@ -2645,472 +3813,569 @@ impl [T] { /// ``` #[inline] #[unstable(feature = "is_sorted", reason = "new API", issue = "53485")] - pub fn is_sorted_by_key(&self, f: F) -> bool + #[must_use] + pub fn is_sorted_by_key<'a, F, K>(&'a self, f: F) -> bool where - F: FnMut(&T) -> K, + F: FnMut(&'a T) -> K, K: PartialOrd, { self.iter().is_sorted_by_key(f) } -} - -#[lang = "slice_u8"] -#[cfg(not(test))] -impl [u8] { - /// Checks if all bytes in this slice are within the ASCII range. - #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] - #[inline] - pub fn is_ascii(&self) -> bool { - self.iter().all(|b| b.is_ascii()) - } - /// Checks that two slices are an ASCII case-insensitive match. + /// Returns the index of the partition point according to the given predicate + /// (the index of the first element of the second partition). /// - /// Same as `to_ascii_lowercase(a) == to_ascii_lowercase(b)`, - /// but without allocating and copying temporaries. - #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] - #[inline] - pub fn eq_ignore_ascii_case(&self, other: &[u8]) -> bool { - self.len() == other.len() && self.iter().zip(other).all(|(a, b)| a.eq_ignore_ascii_case(b)) + /// The slice is assumed to be partitioned according to the given predicate. + /// This means that all elements for which the predicate returns true are at the start of the slice + /// and all elements for which the predicate returns false are at the end. + /// For example, `[7, 15, 3, 5, 4, 12, 6]` is partitioned under the predicate `x % 2 != 0` + /// (all odd numbers are at the start, all even at the end). + /// + /// If this slice is not partitioned, the returned result is unspecified and meaningless, + /// as this method performs a kind of binary search. + /// + /// See also [`binary_search`], [`binary_search_by`], and [`binary_search_by_key`]. + /// + /// [`binary_search`]: slice::binary_search + /// [`binary_search_by`]: slice::binary_search_by + /// [`binary_search_by_key`]: slice::binary_search_by_key + /// + /// # Examples + /// + /// ``` + /// let v = [1, 2, 3, 3, 5, 6, 7]; + /// let i = v.partition_point(|&x| x < 5); + /// + /// assert_eq!(i, 4); + /// assert!(v[..i].iter().all(|&x| x < 5)); + /// assert!(v[i..].iter().all(|&x| !(x < 5))); + /// ``` + /// + /// If all elements of the slice match the predicate, including if the slice + /// is empty, then the length of the slice will be returned: + /// + /// ``` + /// let a = [2, 4, 8]; + /// assert_eq!(a.partition_point(|x| x < &100), a.len()); + /// let a: [i32; 0] = []; + /// assert_eq!(a.partition_point(|x| x < &100), 0); + /// ``` + /// + /// If you want to insert an item to a sorted vector, while maintaining + /// sort order: + /// + /// ``` + /// let mut s = vec![0, 1, 1, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55]; + /// let num = 42; + /// let idx = s.partition_point(|&x| x < num); + /// s.insert(idx, num); + /// assert_eq!(s, [0, 1, 1, 1, 1, 2, 3, 5, 8, 13, 21, 34, 42, 55]); + /// ``` + #[stable(feature = "partition_point", since = "1.52.0")] + #[must_use] + pub fn partition_point

(&self, mut pred: P) -> usize + where + P: FnMut(&T) -> bool, + { + self.binary_search_by(|x| if pred(x) { Less } else { Greater }).unwrap_or_else(|i| i) } - /// Converts this slice to its ASCII upper case equivalent in-place. + /// Removes the subslice corresponding to the given range + /// and returns a reference to it. + /// + /// Returns `None` and does not modify the slice if the given + /// range is out of bounds. + /// + /// Note that this method only accepts one-sided ranges such as + /// `2..` or `..6`, but not `2..6`. /// - /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', - /// but non-ASCII letters are unchanged. + /// # Examples + /// + /// Taking the first three elements of a slice: + /// + /// ``` + /// #![feature(slice_take)] + /// + /// let mut slice: &[_] = &['a', 'b', 'c', 'd']; + /// let mut first_three = slice.take(..3).unwrap(); + /// + /// assert_eq!(slice, &['d']); + /// assert_eq!(first_three, &['a', 'b', 'c']); + /// ``` + /// + /// Taking the last two elements of a slice: + /// + /// ``` + /// #![feature(slice_take)] + /// + /// let mut slice: &[_] = &['a', 'b', 'c', 'd']; + /// let mut tail = slice.take(2..).unwrap(); + /// + /// assert_eq!(slice, &['a', 'b']); + /// assert_eq!(tail, &['c', 'd']); + /// ``` + /// + /// Getting `None` when `range` is out of bounds: + /// + /// ``` + /// #![feature(slice_take)] /// - /// To return a new uppercased value without modifying the existing one, use - /// [`to_ascii_uppercase`]. + /// let mut slice: &[_] = &['a', 'b', 'c', 'd']; /// - /// [`to_ascii_uppercase`]: #method.to_ascii_uppercase - #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] + /// assert_eq!(None, slice.take(5..)); + /// assert_eq!(None, slice.take(..5)); + /// assert_eq!(None, slice.take(..=4)); + /// let expected: &[char] = &['a', 'b', 'c', 'd']; + /// assert_eq!(Some(expected), slice.take(..4)); + /// ``` #[inline] - pub fn make_ascii_uppercase(&mut self) { - for byte in self { - byte.make_ascii_uppercase(); + #[must_use = "method does not modify the slice if the range is out of bounds"] + #[unstable(feature = "slice_take", issue = "62280")] + pub fn take<'a, R: OneSidedRange>(self: &mut &'a Self, range: R) -> Option<&'a Self> { + let (direction, split_index) = split_point_of(range)?; + if split_index > self.len() { + return None; + } + let (front, back) = self.split_at(split_index); + match direction { + Direction::Front => { + *self = back; + Some(front) + } + Direction::Back => { + *self = front; + Some(back) + } } } - /// Converts this slice to its ASCII lower case equivalent in-place. + /// Removes the subslice corresponding to the given range + /// and returns a mutable reference to it. + /// + /// Returns `None` and does not modify the slice if the given + /// range is out of bounds. + /// + /// Note that this method only accepts one-sided ranges such as + /// `2..` or `..6`, but not `2..6`. + /// + /// # Examples + /// + /// Taking the first three elements of a slice: + /// + /// ``` + /// #![feature(slice_take)] + /// + /// let mut slice: &mut [_] = &mut ['a', 'b', 'c', 'd']; + /// let mut first_three = slice.take_mut(..3).unwrap(); + /// + /// assert_eq!(slice, &mut ['d']); + /// assert_eq!(first_three, &mut ['a', 'b', 'c']); + /// ``` + /// + /// Taking the last two elements of a slice: /// - /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', - /// but non-ASCII letters are unchanged. + /// ``` + /// #![feature(slice_take)] + /// + /// let mut slice: &mut [_] = &mut ['a', 'b', 'c', 'd']; + /// let mut tail = slice.take_mut(2..).unwrap(); + /// + /// assert_eq!(slice, &mut ['a', 'b']); + /// assert_eq!(tail, &mut ['c', 'd']); + /// ``` /// - /// To return a new lowercased value without modifying the existing one, use - /// [`to_ascii_lowercase`]. + /// Getting `None` when `range` is out of bounds: /// - /// [`to_ascii_lowercase`]: #method.to_ascii_lowercase - #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] + /// ``` + /// #![feature(slice_take)] + /// + /// let mut slice: &mut [_] = &mut ['a', 'b', 'c', 'd']; + /// + /// assert_eq!(None, slice.take_mut(5..)); + /// assert_eq!(None, slice.take_mut(..5)); + /// assert_eq!(None, slice.take_mut(..=4)); + /// let expected: &mut [_] = &mut ['a', 'b', 'c', 'd']; + /// assert_eq!(Some(expected), slice.take_mut(..4)); + /// ``` #[inline] - pub fn make_ascii_lowercase(&mut self) { - for byte in self { - byte.make_ascii_lowercase(); + #[must_use = "method does not modify the slice if the range is out of bounds"] + #[unstable(feature = "slice_take", issue = "62280")] + pub fn take_mut<'a, R: OneSidedRange>( + self: &mut &'a mut Self, + range: R, + ) -> Option<&'a mut Self> { + let (direction, split_index) = split_point_of(range)?; + if split_index > self.len() { + return None; + } + let (front, back) = mem::take(self).split_at_mut(split_index); + match direction { + Direction::Front => { + *self = back; + Some(front) + } + Direction::Back => { + *self = front; + Some(back) + } } } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ops::Index for [T] -where - I: SliceIndex<[T]>, -{ - type Output = I::Output; - - #[inline] - fn index(&self, index: I) -> &I::Output { - index.index(self) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ops::IndexMut for [T] -where - I: SliceIndex<[T]>, -{ - #[inline] - fn index_mut(&mut self, index: I) -> &mut I::Output { - index.index_mut(self) - } -} - -#[inline(never)] -#[cold] -fn slice_index_len_fail(index: usize, len: usize) -> ! { - panic!("index {} out of range for slice of length {}", index, len); -} - -#[inline(never)] -#[cold] -fn slice_index_order_fail(index: usize, end: usize) -> ! { - panic!("slice index starts at {} but ends at {}", index, end); -} - -#[inline(never)] -#[cold] -fn slice_index_overflow_fail() -> ! { - panic!("attempted to index slice up to maximum usize"); -} - -mod private_slice_index { - use super::ops; - #[stable(feature = "slice_get_slice", since = "1.28.0")] - pub trait Sealed {} - - #[stable(feature = "slice_get_slice", since = "1.28.0")] - impl Sealed for usize {} - #[stable(feature = "slice_get_slice", since = "1.28.0")] - impl Sealed for ops::Range {} - #[stable(feature = "slice_get_slice", since = "1.28.0")] - impl Sealed for ops::RangeTo {} - #[stable(feature = "slice_get_slice", since = "1.28.0")] - impl Sealed for ops::RangeFrom {} - #[stable(feature = "slice_get_slice", since = "1.28.0")] - impl Sealed for ops::RangeFull {} - #[stable(feature = "slice_get_slice", since = "1.28.0")] - impl Sealed for ops::RangeInclusive {} - #[stable(feature = "slice_get_slice", since = "1.28.0")] - impl Sealed for ops::RangeToInclusive {} -} - -/// A helper trait used for indexing operations. -#[stable(feature = "slice_get_slice", since = "1.28.0")] -#[rustc_on_unimplemented( - on(T = "str", label = "string indices are ranges of `usize`",), - on( - all(any(T = "str", T = "&str", T = "std::string::String"), _Self = "{integer}"), - note = "you can use `.chars().nth()` or `.bytes().nth()` -see chapter in The Book " - ), - message = "the type `{T}` cannot be indexed by `{Self}`", - label = "slice indices are of type `usize` or ranges of `usize`" -)] -pub trait SliceIndex: private_slice_index::Sealed { - /// The output type returned by methods. - #[stable(feature = "slice_get_slice", since = "1.28.0")] - type Output: ?Sized; - - /// Returns a shared reference to the output at this location, if in - /// bounds. - #[unstable(feature = "slice_index_methods", issue = "none")] - fn get(self, slice: &T) -> Option<&Self::Output>; - - /// Returns a mutable reference to the output at this location, if in - /// bounds. - #[unstable(feature = "slice_index_methods", issue = "none")] - fn get_mut(self, slice: &mut T) -> Option<&mut Self::Output>; - - /// Returns a shared reference to the output at this location, without - /// performing any bounds checking. - /// Calling this method with an out-of-bounds index is *[undefined behavior]* - /// even if the resulting reference is not used. - /// [undefined behavior]: ../../reference/behavior-considered-undefined.html - #[unstable(feature = "slice_index_methods", issue = "none")] - unsafe fn get_unchecked(self, slice: &T) -> &Self::Output; - - /// Returns a mutable reference to the output at this location, without - /// performing any bounds checking. - /// Calling this method with an out-of-bounds index is *[undefined behavior]* - /// even if the resulting reference is not used. - /// [undefined behavior]: ../../reference/behavior-considered-undefined.html - #[unstable(feature = "slice_index_methods", issue = "none")] - unsafe fn get_unchecked_mut(self, slice: &mut T) -> &mut Self::Output; - - /// Returns a shared reference to the output at this location, panicking - /// if out of bounds. - #[unstable(feature = "slice_index_methods", issue = "none")] - fn index(self, slice: &T) -> &Self::Output; - - /// Returns a mutable reference to the output at this location, panicking - /// if out of bounds. - #[unstable(feature = "slice_index_methods", issue = "none")] - fn index_mut(self, slice: &mut T) -> &mut Self::Output; -} - -#[stable(feature = "slice_get_slice_impls", since = "1.15.0")] -impl SliceIndex<[T]> for usize { - type Output = T; + /// Removes the first element of the slice and returns a reference + /// to it. + /// + /// Returns `None` if the slice is empty. + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_take)] + /// + /// let mut slice: &[_] = &['a', 'b', 'c']; + /// let first = slice.take_first().unwrap(); + /// + /// assert_eq!(slice, &['b', 'c']); + /// assert_eq!(first, &'a'); + /// ``` #[inline] - fn get(self, slice: &[T]) -> Option<&T> { - if self < slice.len() { unsafe { Some(self.get_unchecked(slice)) } } else { None } + #[unstable(feature = "slice_take", issue = "62280")] + pub fn take_first<'a>(self: &mut &'a Self) -> Option<&'a T> { + let (first, rem) = self.split_first()?; + *self = rem; + Some(first) } + /// Removes the first element of the slice and returns a mutable + /// reference to it. + /// + /// Returns `None` if the slice is empty. + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_take)] + /// + /// let mut slice: &mut [_] = &mut ['a', 'b', 'c']; + /// let first = slice.take_first_mut().unwrap(); + /// *first = 'd'; + /// + /// assert_eq!(slice, &['b', 'c']); + /// assert_eq!(first, &'d'); + /// ``` #[inline] - fn get_mut(self, slice: &mut [T]) -> Option<&mut T> { - if self < slice.len() { unsafe { Some(self.get_unchecked_mut(slice)) } } else { None } + #[unstable(feature = "slice_take", issue = "62280")] + pub fn take_first_mut<'a>(self: &mut &'a mut Self) -> Option<&'a mut T> { + let (first, rem) = mem::take(self).split_first_mut()?; + *self = rem; + Some(first) } + /// Removes the last element of the slice and returns a reference + /// to it. + /// + /// Returns `None` if the slice is empty. + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_take)] + /// + /// let mut slice: &[_] = &['a', 'b', 'c']; + /// let last = slice.take_last().unwrap(); + /// + /// assert_eq!(slice, &['a', 'b']); + /// assert_eq!(last, &'c'); + /// ``` #[inline] - unsafe fn get_unchecked(self, slice: &[T]) -> &T { - &*slice.as_ptr().add(self) + #[unstable(feature = "slice_take", issue = "62280")] + pub fn take_last<'a>(self: &mut &'a Self) -> Option<&'a T> { + let (last, rem) = self.split_last()?; + *self = rem; + Some(last) } + /// Removes the last element of the slice and returns a mutable + /// reference to it. + /// + /// Returns `None` if the slice is empty. + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_take)] + /// + /// let mut slice: &mut [_] = &mut ['a', 'b', 'c']; + /// let last = slice.take_last_mut().unwrap(); + /// *last = 'd'; + /// + /// assert_eq!(slice, &['a', 'b']); + /// assert_eq!(last, &'d'); + /// ``` #[inline] - unsafe fn get_unchecked_mut(self, slice: &mut [T]) -> &mut T { - &mut *slice.as_mut_ptr().add(self) + #[unstable(feature = "slice_take", issue = "62280")] + pub fn take_last_mut<'a>(self: &mut &'a mut Self) -> Option<&'a mut T> { + let (last, rem) = mem::take(self).split_last_mut()?; + *self = rem; + Some(last) } + /// Returns mutable references to many indices at once, without doing any checks. + /// + /// For a safe alternative see [`get_many_mut`]. + /// + /// # Safety + /// + /// Calling this method with overlapping or out-of-bounds indices is *[undefined behavior]* + /// even if the resulting references are not used. + /// + /// # Examples + /// + /// ``` + /// #![feature(get_many_mut)] + /// + /// let x = &mut [1, 2, 4]; + /// + /// unsafe { + /// let [a, b] = x.get_many_unchecked_mut([0, 2]); + /// *a *= 10; + /// *b *= 100; + /// } + /// assert_eq!(x, &[10, 2, 400]); + /// ``` + /// + /// [`get_many_mut`]: slice::get_many_mut + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html + #[unstable(feature = "get_many_mut", issue = "104642")] #[inline] - fn index(self, slice: &[T]) -> &T { - // N.B., use intrinsic indexing - &(*slice)[self] + pub unsafe fn get_many_unchecked_mut( + &mut self, + indices: [usize; N], + ) -> [&mut T; N] { + // NB: This implementation is written as it is because any variation of + // `indices.map(|i| self.get_unchecked_mut(i))` would make miri unhappy, + // or generate worse code otherwise. This is also why we need to go + // through a raw pointer here. + let slice: *mut [T] = self; + let mut arr: mem::MaybeUninit<[&mut T; N]> = mem::MaybeUninit::uninit(); + let arr_ptr = arr.as_mut_ptr(); + + // SAFETY: We expect `indices` to contain disjunct values that are + // in bounds of `self`. + unsafe { + for i in 0..N { + let idx = *indices.get_unchecked(i); + *(*arr_ptr).get_unchecked_mut(i) = &mut *slice.get_unchecked_mut(idx); + } + arr.assume_init() + } } + /// Returns mutable references to many indices at once. + /// + /// Returns an error if any index is out-of-bounds, or if the same index was + /// passed more than once. + /// + /// # Examples + /// + /// ``` + /// #![feature(get_many_mut)] + /// + /// let v = &mut [1, 2, 3]; + /// if let Ok([a, b]) = v.get_many_mut([0, 2]) { + /// *a = 413; + /// *b = 612; + /// } + /// assert_eq!(v, &[413, 2, 612]); + /// ``` + #[unstable(feature = "get_many_mut", issue = "104642")] #[inline] - fn index_mut(self, slice: &mut [T]) -> &mut T { - // N.B., use intrinsic indexing - &mut (*slice)[self] + pub fn get_many_mut( + &mut self, + indices: [usize; N], + ) -> Result<[&mut T; N], GetManyMutError> { + if !get_many_check_valid(&indices, self.len()) { + return Err(GetManyMutError { _private: () }); + } + // SAFETY: The `get_many_check_valid()` call checked that all indices + // are disjunct and in bounds. + unsafe { Ok(self.get_many_unchecked_mut(indices)) } } } -#[stable(feature = "slice_get_slice_impls", since = "1.15.0")] -impl SliceIndex<[T]> for ops::Range { - type Output = [T]; - - #[inline] - fn get(self, slice: &[T]) -> Option<&[T]> { - if self.start > self.end || self.end > slice.len() { - None - } else { - unsafe { Some(self.get_unchecked(slice)) } - } - } - - #[inline] - fn get_mut(self, slice: &mut [T]) -> Option<&mut [T]> { - if self.start > self.end || self.end > slice.len() { - None +impl [[T; N]] { + /// Takes a `&[[T; N]]`, and flattens it to a `&[T]`. + /// + /// # Panics + /// + /// This panics if the length of the resulting slice would overflow a `usize`. + /// + /// This is only possible when flattening a slice of arrays of zero-sized + /// types, and thus tends to be irrelevant in practice. If + /// `size_of::() > 0`, this will never panic. + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_flatten)] + /// + /// assert_eq!([[1, 2, 3], [4, 5, 6]].flatten(), &[1, 2, 3, 4, 5, 6]); + /// + /// assert_eq!( + /// [[1, 2, 3], [4, 5, 6]].flatten(), + /// [[1, 2], [3, 4], [5, 6]].flatten(), + /// ); + /// + /// let slice_of_empty_arrays: &[[i32; 0]] = &[[], [], [], [], []]; + /// assert!(slice_of_empty_arrays.flatten().is_empty()); + /// + /// let empty_slice_of_arrays: &[[u32; 10]] = &[]; + /// assert!(empty_slice_of_arrays.flatten().is_empty()); + /// ``` + #[unstable(feature = "slice_flatten", issue = "95629")] + pub fn flatten(&self) -> &[T] { + let len = if T::IS_ZST { + self.len().checked_mul(N).expect("slice len overflow") } else { - unsafe { Some(self.get_unchecked_mut(slice)) } - } - } - - #[inline] - unsafe fn get_unchecked(self, slice: &[T]) -> &[T] { - from_raw_parts(slice.as_ptr().add(self.start), self.end - self.start) - } - - #[inline] - unsafe fn get_unchecked_mut(self, slice: &mut [T]) -> &mut [T] { - from_raw_parts_mut(slice.as_mut_ptr().add(self.start), self.end - self.start) - } - - #[inline] - fn index(self, slice: &[T]) -> &[T] { - if self.start > self.end { - slice_index_order_fail(self.start, self.end); - } else if self.end > slice.len() { - slice_index_len_fail(self.end, slice.len()); - } - unsafe { self.get_unchecked(slice) } + // SAFETY: `self.len() * N` cannot overflow because `self` is + // already in the address space. + unsafe { self.len().unchecked_mul(N) } + }; + // SAFETY: `[T]` is layout-identical to `[T; N]` + unsafe { from_raw_parts(self.as_ptr().cast(), len) } } - #[inline] - fn index_mut(self, slice: &mut [T]) -> &mut [T] { - if self.start > self.end { - slice_index_order_fail(self.start, self.end); - } else if self.end > slice.len() { - slice_index_len_fail(self.end, slice.len()); - } - unsafe { self.get_unchecked_mut(slice) } + /// Takes a `&mut [[T; N]]`, and flattens it to a `&mut [T]`. + /// + /// # Panics + /// + /// This panics if the length of the resulting slice would overflow a `usize`. + /// + /// This is only possible when flattening a slice of arrays of zero-sized + /// types, and thus tends to be irrelevant in practice. If + /// `size_of::() > 0`, this will never panic. + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_flatten)] + /// + /// fn add_5_to_all(slice: &mut [i32]) { + /// for i in slice { + /// *i += 5; + /// } + /// } + /// + /// let mut array = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]; + /// add_5_to_all(array.flatten_mut()); + /// assert_eq!(array, [[6, 7, 8], [9, 10, 11], [12, 13, 14]]); + /// ``` + #[unstable(feature = "slice_flatten", issue = "95629")] + pub fn flatten_mut(&mut self) -> &mut [T] { + let len = if T::IS_ZST { + self.len().checked_mul(N).expect("slice len overflow") + } else { + // SAFETY: `self.len() * N` cannot overflow because `self` is + // already in the address space. + unsafe { self.len().unchecked_mul(N) } + }; + // SAFETY: `[T]` is layout-identical to `[T; N]` + unsafe { from_raw_parts_mut(self.as_mut_ptr().cast(), len) } } } -#[stable(feature = "slice_get_slice_impls", since = "1.15.0")] -impl SliceIndex<[T]> for ops::RangeTo { - type Output = [T]; - - #[inline] - fn get(self, slice: &[T]) -> Option<&[T]> { - (0..self.end).get(slice) - } - - #[inline] - fn get_mut(self, slice: &mut [T]) -> Option<&mut [T]> { - (0..self.end).get_mut(slice) - } - - #[inline] - unsafe fn get_unchecked(self, slice: &[T]) -> &[T] { - (0..self.end).get_unchecked(slice) - } - - #[inline] - unsafe fn get_unchecked_mut(self, slice: &mut [T]) -> &mut [T] { - (0..self.end).get_unchecked_mut(slice) - } - - #[inline] - fn index(self, slice: &[T]) -> &[T] { - (0..self.end).index(slice) - } - +#[cfg(not(test))] +impl [f32] { + /// Sorts the slice of floats. + /// + /// This sort is in-place (i.e. does not allocate), *O*(*n* \* log(*n*)) worst-case, and uses + /// the ordering defined by [`f32::total_cmp`]. + /// + /// # Current implementation + /// + /// This uses the same sorting algorithm as [`sort_unstable_by`](slice::sort_unstable_by). + /// + /// # Examples + /// + /// ``` + /// #![feature(sort_floats)] + /// let mut v = [2.6, -5e-8, f32::NAN, 8.29, f32::INFINITY, -1.0, 0.0, -f32::INFINITY, -0.0]; + /// + /// v.sort_floats(); + /// let sorted = [-f32::INFINITY, -1.0, -5e-8, -0.0, 0.0, 2.6, 8.29, f32::INFINITY, f32::NAN]; + /// assert_eq!(&v[..8], &sorted[..8]); + /// assert!(v[8].is_nan()); + /// ``` + #[unstable(feature = "sort_floats", issue = "93396")] #[inline] - fn index_mut(self, slice: &mut [T]) -> &mut [T] { - (0..self.end).index_mut(slice) + pub fn sort_floats(&mut self) { + self.sort_unstable_by(f32::total_cmp); } } -#[stable(feature = "slice_get_slice_impls", since = "1.15.0")] -impl SliceIndex<[T]> for ops::RangeFrom { - type Output = [T]; - - #[inline] - fn get(self, slice: &[T]) -> Option<&[T]> { - (self.start..slice.len()).get(slice) - } - - #[inline] - fn get_mut(self, slice: &mut [T]) -> Option<&mut [T]> { - (self.start..slice.len()).get_mut(slice) - } - - #[inline] - unsafe fn get_unchecked(self, slice: &[T]) -> &[T] { - (self.start..slice.len()).get_unchecked(slice) - } - - #[inline] - unsafe fn get_unchecked_mut(self, slice: &mut [T]) -> &mut [T] { - (self.start..slice.len()).get_unchecked_mut(slice) - } - - #[inline] - fn index(self, slice: &[T]) -> &[T] { - (self.start..slice.len()).index(slice) - } - +#[cfg(not(test))] +impl [f64] { + /// Sorts the slice of floats. + /// + /// This sort is in-place (i.e. does not allocate), *O*(*n* \* log(*n*)) worst-case, and uses + /// the ordering defined by [`f64::total_cmp`]. + /// + /// # Current implementation + /// + /// This uses the same sorting algorithm as [`sort_unstable_by`](slice::sort_unstable_by). + /// + /// # Examples + /// + /// ``` + /// #![feature(sort_floats)] + /// let mut v = [2.6, -5e-8, f64::NAN, 8.29, f64::INFINITY, -1.0, 0.0, -f64::INFINITY, -0.0]; + /// + /// v.sort_floats(); + /// let sorted = [-f64::INFINITY, -1.0, -5e-8, -0.0, 0.0, 2.6, 8.29, f64::INFINITY, f64::NAN]; + /// assert_eq!(&v[..8], &sorted[..8]); + /// assert!(v[8].is_nan()); + /// ``` + #[unstable(feature = "sort_floats", issue = "93396")] #[inline] - fn index_mut(self, slice: &mut [T]) -> &mut [T] { - (self.start..slice.len()).index_mut(slice) + pub fn sort_floats(&mut self) { + self.sort_unstable_by(f64::total_cmp); } } -#[stable(feature = "slice_get_slice_impls", since = "1.15.0")] -impl SliceIndex<[T]> for ops::RangeFull { - type Output = [T]; - - #[inline] - fn get(self, slice: &[T]) -> Option<&[T]> { - Some(slice) - } - - #[inline] - fn get_mut(self, slice: &mut [T]) -> Option<&mut [T]> { - Some(slice) - } - - #[inline] - unsafe fn get_unchecked(self, slice: &[T]) -> &[T] { - slice - } - - #[inline] - unsafe fn get_unchecked_mut(self, slice: &mut [T]) -> &mut [T] { - slice - } - - #[inline] - fn index(self, slice: &[T]) -> &[T] { - slice - } - - #[inline] - fn index_mut(self, slice: &mut [T]) -> &mut [T] { - slice - } +trait CloneFromSpec { + fn spec_clone_from(&mut self, src: &[T]); } -#[stable(feature = "inclusive_range", since = "1.26.0")] -impl SliceIndex<[T]> for ops::RangeInclusive { - type Output = [T]; - - #[inline] - fn get(self, slice: &[T]) -> Option<&[T]> { - if *self.end() == usize::max_value() { - None - } else { - (*self.start()..self.end() + 1).get(slice) - } - } - - #[inline] - fn get_mut(self, slice: &mut [T]) -> Option<&mut [T]> { - if *self.end() == usize::max_value() { - None - } else { - (*self.start()..self.end() + 1).get_mut(slice) - } - } - - #[inline] - unsafe fn get_unchecked(self, slice: &[T]) -> &[T] { - (*self.start()..self.end() + 1).get_unchecked(slice) - } - - #[inline] - unsafe fn get_unchecked_mut(self, slice: &mut [T]) -> &mut [T] { - (*self.start()..self.end() + 1).get_unchecked_mut(slice) - } - - #[inline] - fn index(self, slice: &[T]) -> &[T] { - if *self.end() == usize::max_value() { - slice_index_overflow_fail(); - } - (*self.start()..self.end() + 1).index(slice) - } - - #[inline] - fn index_mut(self, slice: &mut [T]) -> &mut [T] { - if *self.end() == usize::max_value() { - slice_index_overflow_fail(); +impl CloneFromSpec for [T] +where + T: Clone, +{ + #[track_caller] + default fn spec_clone_from(&mut self, src: &[T]) { + assert!(self.len() == src.len(), "destination and source slices have different lengths"); + // NOTE: We need to explicitly slice them to the same length + // to make it easier for the optimizer to elide bounds checking. + // But since it can't be relied on we also have an explicit specialization for T: Copy. + let len = self.len(); + let src = &src[..len]; + for i in 0..len { + self[i].clone_from(&src[i]); } - (*self.start()..self.end() + 1).index_mut(slice) } } -#[stable(feature = "inclusive_range", since = "1.26.0")] -impl SliceIndex<[T]> for ops::RangeToInclusive { - type Output = [T]; - - #[inline] - fn get(self, slice: &[T]) -> Option<&[T]> { - (0..=self.end).get(slice) - } - - #[inline] - fn get_mut(self, slice: &mut [T]) -> Option<&mut [T]> { - (0..=self.end).get_mut(slice) - } - - #[inline] - unsafe fn get_unchecked(self, slice: &[T]) -> &[T] { - (0..=self.end).get_unchecked(slice) - } - - #[inline] - unsafe fn get_unchecked_mut(self, slice: &mut [T]) -> &mut [T] { - (0..=self.end).get_unchecked_mut(slice) - } - - #[inline] - fn index(self, slice: &[T]) -> &[T] { - (0..=self.end).index(slice) - } - - #[inline] - fn index_mut(self, slice: &mut [T]) -> &mut [T] { - (0..=self.end).index_mut(slice) +impl CloneFromSpec for [T] +where + T: Copy, +{ + #[track_caller] + fn spec_clone_from(&mut self, src: &[T]) { + self.copy_from_slice(src); } } -//////////////////////////////////////////////////////////////////////////////// -// Common traits -//////////////////////////////////////////////////////////////////////////////// - #[stable(feature = "rust1", since = "1.0.0")] -impl Default for &[T] { +#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] +impl const Default for &[T] { /// Creates an empty slice. fn default() -> Self { &[] @@ -3118,2894 +4383,95 @@ impl Default for &[T] { } #[stable(feature = "mut_slice_default", since = "1.5.0")] -impl Default for &mut [T] { +#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] +impl const Default for &mut [T] { /// Creates a mutable empty slice. fn default() -> Self { &mut [] } } -// -// Iterators -// - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T> IntoIterator for &'a [T] { - type Item = &'a T; - type IntoIter = Iter<'a, T>; +#[unstable(feature = "slice_pattern", reason = "stopgap trait for slice patterns", issue = "56345")] +/// Patterns in slices - currently, only used by `strip_prefix` and `strip_suffix`. At a future +/// point, we hope to generalise `core::str::Pattern` (which at the time of writing is limited to +/// `str`) to slices, and then this trait will be replaced or abolished. +pub trait SlicePattern { + /// The element type of the slice being matched on. + type Item; - fn into_iter(self) -> Iter<'a, T> { - self.iter() - } + /// Currently, the consumers of `SlicePattern` need a slice. + fn as_slice(&self) -> &[Self::Item]; } -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T> IntoIterator for &'a mut [T] { - type Item = &'a mut T; - type IntoIter = IterMut<'a, T>; +#[stable(feature = "slice_strip", since = "1.51.0")] +impl SlicePattern for [T] { + type Item = T; - fn into_iter(self) -> IterMut<'a, T> { - self.iter_mut() + #[inline] + fn as_slice(&self) -> &[Self::Item] { + self } } -// Macro helper functions -#[inline(always)] -fn size_from_ptr(_: *const T) -> usize { - mem::size_of::() -} +#[stable(feature = "slice_strip", since = "1.51.0")] +impl SlicePattern for [T; N] { + type Item = T; -// Inlining is_empty and len makes a huge performance difference -macro_rules! is_empty { - // The way we encode the length of a ZST iterator, this works both for ZST - // and non-ZST. - ($self: ident) => { - $self.len == 0 - }; -} -// To get rid of some bounds checks (see `position`), we compute the length in a somewhat -// unexpected way. (Tested by `codegen/slice-position-bounds-check`.) -macro_rules! len { - ($self: ident) => {{ - #![allow(unused_unsafe)] // we're sometimes used within an unsafe block - $self.len - }}; + #[inline] + fn as_slice(&self) -> &[Self::Item] { + self + } } -// The shared definition of the `Iter` and `IterMut` iterators -macro_rules! iterator { - ( - struct $name:ident -> $ptr:ty, - $elem:ty, - $raw_mut:tt, - {$( $mut_:tt )*}, - {$($extra:tt)*} - ) => { - // Returns the first element and moves the start of the iterator forwards by 1. - // Greatly improves performance compared to an inlined function. The iterator - // must not be empty. - macro_rules! next_unchecked { - ($self: ident) => {& $( $mut_ )* *$self.post_inc_start(1)} - } - - // Returns the last element and moves the end of the iterator backwards by 1. - // Greatly improves performance compared to an inlined function. The iterator - // must not be empty. - macro_rules! next_back_unchecked { - ($self: ident) => {& $( $mut_ )* *$self.pre_dec_end(1)} - } - - // Shrinks the iterator when T is a ZST, by moving the end of the iterator - // backwards by `n`. `n` must not exceed `self.len()`. - macro_rules! zst_shrink { - ($self: ident, $n: ident) => { - $self.end = ($self.end as * $raw_mut u8).wrapping_offset(-$n) as * $raw_mut T; - } - } - - impl<'a, T> $name<'a, T> { - // Helper function for creating a slice from the iterator. - #[inline(always)] - fn make_slice(&self) -> &'a [T] { - unsafe { from_raw_parts(self.ptr.as_ptr(), len!(self)) } - } - - // Helper function for moving the start of the iterator forwards by `offset` elements, - // returning the old start. - // Unsafe because the offset must not exceed `self.len()`. - #[inline(always)] - unsafe fn post_inc_start(&mut self, offset: isize) -> * $raw_mut T { - self.len -= offset as usize; - if mem::size_of::() == 0 { - zst_shrink!(self, offset); - self.ptr.as_ptr() - } else { - let old = self.ptr.as_ptr(); - self.ptr = NonNull::new_unchecked(self.ptr.as_ptr().offset(offset)); - old - } - } - - // Helper function for moving the end of the iterator backwards by `offset` elements, - // returning the new end. - // Unsafe because the offset must not exceed `self.len()`. - #[inline(always)] - unsafe fn pre_dec_end(&mut self, offset: isize) -> * $raw_mut T { - self.len -= offset as usize; - if mem::size_of::() == 0 { - zst_shrink!(self, offset); - self.ptr.as_ptr() - } else { - self.end = self.end.offset(-offset); - self.end - } - } - } - - #[stable(feature = "rust1", since = "1.0.0")] - impl ExactSizeIterator for $name<'_, T> { - #[inline(always)] - fn len(&self) -> usize { - len!(self) - } - - #[inline(always)] - fn is_empty(&self) -> bool { - is_empty!(self) - } - } - - #[stable(feature = "rust1", since = "1.0.0")] - impl<'a, T> Iterator for $name<'a, T> { - type Item = $elem; - - #[inline] - fn next(&mut self) -> Option<$elem> { - // could be implemented with slices, but this avoids bounds checks - unsafe { - assume(!self.ptr.as_ptr().is_null()); - if mem::size_of::() != 0 { - assume(!self.end.is_null()); - } - if is_empty!(self) { - None - } else { - Some(next_unchecked!(self)) - } - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let exact = len!(self); - (exact, Some(exact)) - } - - #[inline] - fn count(self) -> usize { - len!(self) - } - - #[inline] - fn nth(&mut self, n: usize) -> Option<$elem> { - if n >= len!(self) { - // This iterator is now empty. - self.len = 0; - if mem::size_of::() == 0 { - // We have to do it this way as `ptr` may never be 0, but `end` - // could be (due to wrapping). - self.end = self.ptr.as_ptr(); - } else { - unsafe { - // End can't be 0 if T isn't ZST because ptr isn't 0 and end >= ptr - self.ptr = NonNull::new_unchecked(self.end as *mut T); - } - } - return None; - } - // We are in bounds. `post_inc_start` does the right thing even for ZSTs. - unsafe { - self.post_inc_start(n as isize); - Some(next_unchecked!(self)) - } - } - - #[inline] - fn last(mut self) -> Option<$elem> { - self.next_back() - } - - #[inline] - #[rustc_inherit_overflow_checks] - fn position

(&mut self, mut predicate: P) -> Option where - Self: Sized, - P: FnMut(Self::Item) -> bool, - { - // The addition might panic on overflow. - let n = len!(self); - self.try_fold(0, move |i, x| { - if predicate(x) { Err(i) } - else { Ok(i + 1) } - }).err() - .map(|i| { - unsafe { assume(i < n) }; - i - }) - } - - #[inline] - fn rposition

(&mut self, mut predicate: P) -> Option where - P: FnMut(Self::Item) -> bool, - Self: Sized + ExactSizeIterator + DoubleEndedIterator - { - // No need for an overflow check here, because `ExactSizeIterator` - let n = len!(self); - self.try_rfold(n, move |i, x| { - let i = i - 1; - if predicate(x) { Err(i) } - else { Ok(i) } - }).err() - .map(|i| { - unsafe { assume(i < n) }; - i - }) - } - - $($extra)* - } - - #[stable(feature = "rust1", since = "1.0.0")] - impl<'a, T> DoubleEndedIterator for $name<'a, T> { - #[inline] - fn next_back(&mut self) -> Option<$elem> { - // could be implemented with slices, but this avoids bounds checks - unsafe { - assume(!self.ptr.as_ptr().is_null()); - if mem::size_of::() != 0 { - assume(!self.end.is_null()); - } - if is_empty!(self) { - None - } else { - Some(next_back_unchecked!(self)) - } - } - } - - #[inline] - fn nth_back(&mut self, n: usize) -> Option<$elem> { - if n >= len!(self) { - // This iterator is now empty. - self.len = 0; - self.end = self.ptr.as_ptr(); - return None; - } - // We are in bounds. `pre_dec_end` does the right thing even for ZSTs. - unsafe { - self.pre_dec_end(n as isize); - Some(next_back_unchecked!(self)) - } - } +/// This checks every index against each other, and against `len`. +/// +/// This will do `binomial(N + 1, 2) = N * (N + 1) / 2 = 0, 1, 3, 6, 10, ..` +/// comparison operations. +fn get_many_check_valid(indices: &[usize; N], len: usize) -> bool { + // NB: The optimzer should inline the loops into a sequence + // of instructions without additional branching. + let mut valid = true; + for (i, &idx) in indices.iter().enumerate() { + valid &= idx < len; + for &idx2 in &indices[..i] { + valid &= idx != idx2; } - - #[stable(feature = "fused", since = "1.26.0")] - impl FusedIterator for $name<'_, T> {} - - #[unstable(feature = "trusted_len", issue = "37572")] - unsafe impl TrustedLen for $name<'_, T> {} } + valid } -/// Immutable slice iterator +/// The error type returned by [`get_many_mut`][`slice::get_many_mut`]. /// -/// This struct is created by the [`iter`] method on [slices]. +/// It indicates one of two possible errors: +/// - An index is out-of-bounds. +/// - The same index appeared multiple times in the array. /// /// # Examples /// -/// Basic usage: -/// /// ``` -/// // First, we declare a type which has `iter` method to get the `Iter` struct (&[usize here]): -/// let slice = &[1, 2, 3]; +/// #![feature(get_many_mut)] /// -/// // Then, we iterate over it: -/// for element in slice.iter() { -/// println!("{}", element); -/// } +/// let v = &mut [1, 2, 3]; +/// assert!(v.get_many_mut([0, 999]).is_err()); +/// assert!(v.get_many_mut([1, 1]).is_err()); /// ``` -/// -/// [`iter`]: ../../std/primitive.slice.html#method.iter -/// [slices]: ../../std/primitive.slice.html -#[stable(feature = "rust1", since = "1.0.0")] -pub struct Iter<'a, T: 'a> { - ptr: NonNull, - end: *const T, // If T is a ZST, this is actually ptr+len. This encoding is picked so that - // ptr == end is a quick test for the Iterator being empty, that works - // for both ZST and non-ZST. - len: usize, - _marker: marker::PhantomData<&'a T>, +#[unstable(feature = "get_many_mut", issue = "104642")] +// NB: The N here is there to be forward-compatible with adding more details +// to the error type at a later point +pub struct GetManyMutError { + _private: (), } -#[stable(feature = "core_impl_debug", since = "1.9.0")] -impl fmt::Debug for Iter<'_, T> { +#[unstable(feature = "get_many_mut", issue = "104642")] +impl fmt::Debug for GetManyMutError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("Iter").field(&self.as_slice()).finish() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -unsafe impl Sync for Iter<'_, T> {} -#[stable(feature = "rust1", since = "1.0.0")] -unsafe impl Send for Iter<'_, T> {} - -impl<'a, T> Iter<'a, T> { - /// Views the underlying data as a subslice of the original data. - /// - /// This has the same lifetime as the original slice, and so the - /// iterator can continue to be used while this exists. - /// - /// # Examples - /// - /// Basic usage: - /// - /// ``` - /// // First, we declare a type which has the `iter` method to get the `Iter` - /// // struct (&[usize here]): - /// let slice = &[1, 2, 3]; - /// - /// // Then, we get the iterator: - /// let mut iter = slice.iter(); - /// // So if we print what `as_slice` method returns here, we have "[1, 2, 3]": - /// println!("{:?}", iter.as_slice()); - /// - /// // Next, we move to the second element of the slice: - /// iter.next(); - /// // Now `as_slice` returns "[2, 3]": - /// println!("{:?}", iter.as_slice()); - /// ``` - #[stable(feature = "iter_to_slice", since = "1.4.0")] - pub fn as_slice(&self) -> &'a [T] { - self.make_slice() + f.debug_struct("GetManyMutError").finish_non_exhaustive() } } -iterator! {struct Iter -> *const T, &'a T, const, {/* no mut */}, { - fn is_sorted_by(self, mut compare: F) -> bool - where - Self: Sized, - F: FnMut(&Self::Item, &Self::Item) -> Option, - { - self.as_slice().windows(2).all(|w| { - compare(&&w[0], &&w[1]).map(|o| o != Ordering::Greater).unwrap_or(false) - }) - } -}} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Clone for Iter<'_, T> { - fn clone(&self) -> Self { - Iter { ptr: self.ptr, end: self.end, len: self.len, _marker: self._marker } - } -} - -#[stable(feature = "slice_iter_as_ref", since = "1.13.0")] -impl AsRef<[T]> for Iter<'_, T> { - fn as_ref(&self) -> &[T] { - self.as_slice() - } -} - -/// Mutable slice iterator. -/// -/// This struct is created by the [`iter_mut`] method on [slices]. -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// // First, we declare a type which has `iter_mut` method to get the `IterMut` -/// // struct (&[usize here]): -/// let mut slice = &mut [1, 2, 3]; -/// -/// // Then, we iterate over it and increment each element value: -/// for element in slice.iter_mut() { -/// *element += 1; -/// } -/// -/// // We now have "[2, 3, 4]": -/// println!("{:?}", slice); -/// ``` -/// -/// [`iter_mut`]: ../../std/primitive.slice.html#method.iter_mut -/// [slices]: ../../std/primitive.slice.html -#[stable(feature = "rust1", since = "1.0.0")] -pub struct IterMut<'a, T: 'a> { - ptr: NonNull, - end: *mut T, // If T is a ZST, this is actually ptr+len. This encoding is picked so that - // ptr == end is a quick test for the Iterator being empty, that works - // for both ZST and non-ZST. - len: usize, - _marker: marker::PhantomData<&'a mut T>, -} - -#[stable(feature = "core_impl_debug", since = "1.9.0")] -impl fmt::Debug for IterMut<'_, T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("IterMut").field(&self.make_slice()).finish() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -unsafe impl Sync for IterMut<'_, T> {} -#[stable(feature = "rust1", since = "1.0.0")] -unsafe impl Send for IterMut<'_, T> {} - -impl<'a, T> IterMut<'a, T> { - /// Views the underlying data as a subslice of the original data. - /// - /// To avoid creating `&mut` references that alias, this is forced - /// to consume the iterator. - /// - /// # Examples - /// - /// Basic usage: - /// - /// ``` - /// // First, we declare a type which has `iter_mut` method to get the `IterMut` - /// // struct (&[usize here]): - /// let mut slice = &mut [1, 2, 3]; - /// - /// { - /// // Then, we get the iterator: - /// let mut iter = slice.iter_mut(); - /// // We move to next element: - /// iter.next(); - /// // So if we print what `into_slice` method returns here, we have "[2, 3]": - /// println!("{:?}", iter.into_slice()); - /// } - /// - /// // Now let's modify a value of the slice: - /// { - /// // First we get back the iterator: - /// let mut iter = slice.iter_mut(); - /// // We change the value of the first element of the slice returned by the `next` method: - /// *iter.next().unwrap() += 1; - /// } - /// // Now slice is "[2, 2, 3]": - /// println!("{:?}", slice); - /// ``` - #[stable(feature = "iter_to_slice", since = "1.4.0")] - pub fn into_slice(self) -> &'a mut [T] { - unsafe { from_raw_parts_mut(self.ptr.as_ptr(), len!(self)) } - } - - /// Views the underlying data as a subslice of the original data. - /// - /// To avoid creating `&mut [T]` references that alias, the returned slice - /// borrows its lifetime from the iterator the method is applied on. - /// - /// # Examples - /// - /// Basic usage: - /// - /// ``` - /// # #![feature(slice_iter_mut_as_slice)] - /// let mut slice: &mut [usize] = &mut [1, 2, 3]; - /// - /// // First, we get the iterator: - /// let mut iter = slice.iter_mut(); - /// // So if we check what the `as_slice` method returns here, we have "[1, 2, 3]": - /// assert_eq!(iter.as_slice(), &[1, 2, 3]); - /// - /// // Next, we move to the second element of the slice: - /// iter.next(); - /// // Now `as_slice` returns "[2, 3]": - /// assert_eq!(iter.as_slice(), &[2, 3]); - /// ``` - #[unstable(feature = "slice_iter_mut_as_slice", reason = "recently added", issue = "58957")] - pub fn as_slice(&self) -> &[T] { - self.make_slice() - } -} - -iterator! {struct IterMut -> *mut T, &'a mut T, mut, {mut}, {}} - -/// An internal abstraction over the splitting iterators, so that -/// splitn, splitn_mut etc can be implemented once. -#[doc(hidden)] -trait SplitIter: DoubleEndedIterator { - /// Marks the underlying iterator as complete, extracting the remaining - /// portion of the slice. - fn finish(&mut self) -> Option; -} - -/// An iterator over subslices separated by elements that match a predicate -/// function. -/// -/// This struct is created by the [`split`] method on [slices]. -/// -/// [`split`]: ../../std/primitive.slice.html#method.split -/// [slices]: ../../std/primitive.slice.html -#[stable(feature = "rust1", since = "1.0.0")] -pub struct Split<'a, T: 'a, P> -where - P: FnMut(&T) -> bool, -{ - v: &'a [T], - pred: P, - finished: bool, -} - -#[stable(feature = "core_impl_debug", since = "1.9.0")] -impl fmt::Debug for Split<'_, T, P> -where - P: FnMut(&T) -> bool, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Split").field("v", &self.v).field("finished", &self.finished).finish() - } -} - -// FIXME(#26925) Remove in favor of `#[derive(Clone)]` -#[stable(feature = "rust1", since = "1.0.0")] -impl Clone for Split<'_, T, P> -where - P: Clone + FnMut(&T) -> bool, -{ - fn clone(&self) -> Self { - Split { v: self.v, pred: self.pred.clone(), finished: self.finished } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T, P> Iterator for Split<'a, T, P> -where - P: FnMut(&T) -> bool, -{ - type Item = &'a [T]; - - #[inline] - fn next(&mut self) -> Option<&'a [T]> { - if self.finished { - return None; - } - - match self.v.iter().position(|x| (self.pred)(x)) { - None => self.finish(), - Some(idx) => { - let ret = Some(&self.v[..idx]); - self.v = &self.v[idx + 1..]; - ret - } - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - if self.finished { (0, Some(0)) } else { (1, Some(self.v.len() + 1)) } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T, P> DoubleEndedIterator for Split<'a, T, P> -where - P: FnMut(&T) -> bool, -{ - #[inline] - fn next_back(&mut self) -> Option<&'a [T]> { - if self.finished { - return None; - } - - match self.v.iter().rposition(|x| (self.pred)(x)) { - None => self.finish(), - Some(idx) => { - let ret = Some(&self.v[idx + 1..]); - self.v = &self.v[..idx]; - ret - } - } - } -} - -impl<'a, T, P> SplitIter for Split<'a, T, P> -where - P: FnMut(&T) -> bool, -{ - #[inline] - fn finish(&mut self) -> Option<&'a [T]> { - if self.finished { - None - } else { - self.finished = true; - Some(self.v) - } - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Split<'_, T, P> where P: FnMut(&T) -> bool {} - -/// An iterator over subslices separated by elements that match a predicate -/// function. Unlike `Split`, it contains the matched part as a terminator -/// of the subslice. -/// -/// This struct is created by the [`split_inclusive`] method on [slices]. -/// -/// [`split_inclusive`]: ../../std/primitive.slice.html#method.split_inclusive -/// [slices]: ../../std/primitive.slice.html -#[unstable(feature = "split_inclusive", issue = "none")] -pub struct SplitInclusive<'a, T: 'a, P> -where - P: FnMut(&T) -> bool, -{ - v: &'a [T], - pred: P, - finished: bool, -} - -#[unstable(feature = "split_inclusive", issue = "none")] -impl fmt::Debug for SplitInclusive<'_, T, P> -where - P: FnMut(&T) -> bool, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SplitInclusive") - .field("v", &self.v) - .field("finished", &self.finished) - .finish() - } -} - -// FIXME(#26925) Remove in favor of `#[derive(Clone)]` -#[unstable(feature = "split_inclusive", issue = "none")] -impl Clone for SplitInclusive<'_, T, P> -where - P: Clone + FnMut(&T) -> bool, -{ - fn clone(&self) -> Self { - SplitInclusive { v: self.v, pred: self.pred.clone(), finished: self.finished } - } -} - -#[unstable(feature = "split_inclusive", issue = "none")] -impl<'a, T, P> Iterator for SplitInclusive<'a, T, P> -where - P: FnMut(&T) -> bool, -{ - type Item = &'a [T]; - - #[inline] - fn next(&mut self) -> Option<&'a [T]> { - if self.finished { - return None; - } - - let idx = - self.v.iter().position(|x| (self.pred)(x)).map(|idx| idx + 1).unwrap_or(self.v.len()); - if idx == self.v.len() { - self.finished = true; - } - let ret = Some(&self.v[..idx]); - self.v = &self.v[idx..]; - ret - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - if self.finished { (0, Some(0)) } else { (1, Some(self.v.len() + 1)) } - } -} - -#[unstable(feature = "split_inclusive", issue = "none")] -impl<'a, T, P> DoubleEndedIterator for SplitInclusive<'a, T, P> -where - P: FnMut(&T) -> bool, -{ - #[inline] - fn next_back(&mut self) -> Option<&'a [T]> { - if self.finished { - return None; - } - - // The last index of self.v is already checked and found to match - // by the last iteration, so we start searching a new match - // one index to the left. - let remainder = if self.v.is_empty() { &[] } else { &self.v[..(self.v.len() - 1)] }; - let idx = remainder.iter().rposition(|x| (self.pred)(x)).map(|idx| idx + 1).unwrap_or(0); - if idx == 0 { - self.finished = true; - } - let ret = Some(&self.v[idx..]); - self.v = &self.v[..idx]; - ret - } -} - -#[unstable(feature = "split_inclusive", issue = "none")] -impl FusedIterator for SplitInclusive<'_, T, P> where P: FnMut(&T) -> bool {} - -/// An iterator over the mutable subslices of the vector which are separated -/// by elements that match `pred`. -/// -/// This struct is created by the [`split_mut`] method on [slices]. -/// -/// [`split_mut`]: ../../std/primitive.slice.html#method.split_mut -/// [slices]: ../../std/primitive.slice.html -#[stable(feature = "rust1", since = "1.0.0")] -pub struct SplitMut<'a, T: 'a, P> -where - P: FnMut(&T) -> bool, -{ - v: &'a mut [T], - pred: P, - finished: bool, -} - -#[stable(feature = "core_impl_debug", since = "1.9.0")] -impl fmt::Debug for SplitMut<'_, T, P> -where - P: FnMut(&T) -> bool, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SplitMut").field("v", &self.v).field("finished", &self.finished).finish() - } -} - -impl<'a, T, P> SplitIter for SplitMut<'a, T, P> -where - P: FnMut(&T) -> bool, -{ - #[inline] - fn finish(&mut self) -> Option<&'a mut [T]> { - if self.finished { - None - } else { - self.finished = true; - Some(mem::replace(&mut self.v, &mut [])) - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T, P> Iterator for SplitMut<'a, T, P> -where - P: FnMut(&T) -> bool, -{ - type Item = &'a mut [T]; - - #[inline] - fn next(&mut self) -> Option<&'a mut [T]> { - if self.finished { - return None; - } - - let idx_opt = { - // work around borrowck limitations - let pred = &mut self.pred; - self.v.iter().position(|x| (*pred)(x)) - }; - match idx_opt { - None => self.finish(), - Some(idx) => { - let tmp = mem::replace(&mut self.v, &mut []); - let (head, tail) = tmp.split_at_mut(idx); - self.v = &mut tail[1..]; - Some(head) - } - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - if self.finished { - (0, Some(0)) - } else { - // if the predicate doesn't match anything, we yield one slice - // if it matches every element, we yield len+1 empty slices. - (1, Some(self.v.len() + 1)) - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T, P> DoubleEndedIterator for SplitMut<'a, T, P> -where - P: FnMut(&T) -> bool, -{ - #[inline] - fn next_back(&mut self) -> Option<&'a mut [T]> { - if self.finished { - return None; - } - - let idx_opt = { - // work around borrowck limitations - let pred = &mut self.pred; - self.v.iter().rposition(|x| (*pred)(x)) - }; - match idx_opt { - None => self.finish(), - Some(idx) => { - let tmp = mem::replace(&mut self.v, &mut []); - let (head, tail) = tmp.split_at_mut(idx); - self.v = head; - Some(&mut tail[1..]) - } - } - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for SplitMut<'_, T, P> where P: FnMut(&T) -> bool {} - -/// An iterator over the mutable subslices of the vector which are separated -/// by elements that match `pred`. Unlike `SplitMut`, it contains the matched -/// parts in the ends of the subslices. -/// -/// This struct is created by the [`split_inclusive_mut`] method on [slices]. -/// -/// [`split_inclusive_mut`]: ../../std/primitive.slice.html#method.split_inclusive_mut -/// [slices]: ../../std/primitive.slice.html -#[unstable(feature = "split_inclusive", issue = "none")] -pub struct SplitInclusiveMut<'a, T: 'a, P> -where - P: FnMut(&T) -> bool, -{ - v: &'a mut [T], - pred: P, - finished: bool, -} - -#[unstable(feature = "split_inclusive", issue = "none")] -impl fmt::Debug for SplitInclusiveMut<'_, T, P> -where - P: FnMut(&T) -> bool, -{ +#[unstable(feature = "get_many_mut", issue = "104642")] +impl fmt::Display for GetManyMutError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SplitInclusiveMut") - .field("v", &self.v) - .field("finished", &self.finished) - .finish() - } -} - -#[unstable(feature = "split_inclusive", issue = "none")] -impl<'a, T, P> Iterator for SplitInclusiveMut<'a, T, P> -where - P: FnMut(&T) -> bool, -{ - type Item = &'a mut [T]; - - #[inline] - fn next(&mut self) -> Option<&'a mut [T]> { - if self.finished { - return None; - } - - let idx_opt = { - // work around borrowck limitations - let pred = &mut self.pred; - self.v.iter().position(|x| (*pred)(x)) - }; - let idx = idx_opt.map(|idx| idx + 1).unwrap_or(self.v.len()); - if idx == self.v.len() { - self.finished = true; - } - let tmp = mem::replace(&mut self.v, &mut []); - let (head, tail) = tmp.split_at_mut(idx); - self.v = tail; - Some(head) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - if self.finished { - (0, Some(0)) - } else { - // if the predicate doesn't match anything, we yield one slice - // if it matches every element, we yield len+1 empty slices. - (1, Some(self.v.len() + 1)) - } - } -} - -#[unstable(feature = "split_inclusive", issue = "none")] -impl<'a, T, P> DoubleEndedIterator for SplitInclusiveMut<'a, T, P> -where - P: FnMut(&T) -> bool, -{ - #[inline] - fn next_back(&mut self) -> Option<&'a mut [T]> { - if self.finished { - return None; - } - - let idx_opt = if self.v.is_empty() { - None - } else { - // work around borrowck limitations - let pred = &mut self.pred; - - // The last index of self.v is already checked and found to match - // by the last iteration, so we start searching a new match - // one index to the left. - let remainder = &self.v[..(self.v.len() - 1)]; - remainder.iter().rposition(|x| (*pred)(x)) - }; - let idx = idx_opt.map(|idx| idx + 1).unwrap_or(0); - if idx == 0 { - self.finished = true; - } - let tmp = mem::replace(&mut self.v, &mut []); - let (head, tail) = tmp.split_at_mut(idx); - self.v = head; - Some(tail) - } -} - -#[unstable(feature = "split_inclusive", issue = "none")] -impl FusedIterator for SplitInclusiveMut<'_, T, P> where P: FnMut(&T) -> bool {} - -/// An iterator over subslices separated by elements that match a predicate -/// function, starting from the end of the slice. -/// -/// This struct is created by the [`rsplit`] method on [slices]. -/// -/// [`rsplit`]: ../../std/primitive.slice.html#method.rsplit -/// [slices]: ../../std/primitive.slice.html -#[stable(feature = "slice_rsplit", since = "1.27.0")] -#[derive(Clone)] // Is this correct, or does it incorrectly require `T: Clone`? -pub struct RSplit<'a, T: 'a, P> -where - P: FnMut(&T) -> bool, -{ - inner: Split<'a, T, P>, -} - -#[stable(feature = "slice_rsplit", since = "1.27.0")] -impl fmt::Debug for RSplit<'_, T, P> -where - P: FnMut(&T) -> bool, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("RSplit") - .field("v", &self.inner.v) - .field("finished", &self.inner.finished) - .finish() - } -} - -#[stable(feature = "slice_rsplit", since = "1.27.0")] -impl<'a, T, P> Iterator for RSplit<'a, T, P> -where - P: FnMut(&T) -> bool, -{ - type Item = &'a [T]; - - #[inline] - fn next(&mut self) -> Option<&'a [T]> { - self.inner.next_back() - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.inner.size_hint() - } -} - -#[stable(feature = "slice_rsplit", since = "1.27.0")] -impl<'a, T, P> DoubleEndedIterator for RSplit<'a, T, P> -where - P: FnMut(&T) -> bool, -{ - #[inline] - fn next_back(&mut self) -> Option<&'a [T]> { - self.inner.next() - } -} - -#[stable(feature = "slice_rsplit", since = "1.27.0")] -impl<'a, T, P> SplitIter for RSplit<'a, T, P> -where - P: FnMut(&T) -> bool, -{ - #[inline] - fn finish(&mut self) -> Option<&'a [T]> { - self.inner.finish() - } -} - -#[stable(feature = "slice_rsplit", since = "1.27.0")] -impl FusedIterator for RSplit<'_, T, P> where P: FnMut(&T) -> bool {} - -/// An iterator over the subslices of the vector which are separated -/// by elements that match `pred`, starting from the end of the slice. -/// -/// This struct is created by the [`rsplit_mut`] method on [slices]. -/// -/// [`rsplit_mut`]: ../../std/primitive.slice.html#method.rsplit_mut -/// [slices]: ../../std/primitive.slice.html -#[stable(feature = "slice_rsplit", since = "1.27.0")] -pub struct RSplitMut<'a, T: 'a, P> -where - P: FnMut(&T) -> bool, -{ - inner: SplitMut<'a, T, P>, -} - -#[stable(feature = "slice_rsplit", since = "1.27.0")] -impl fmt::Debug for RSplitMut<'_, T, P> -where - P: FnMut(&T) -> bool, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("RSplitMut") - .field("v", &self.inner.v) - .field("finished", &self.inner.finished) - .finish() - } -} - -#[stable(feature = "slice_rsplit", since = "1.27.0")] -impl<'a, T, P> SplitIter for RSplitMut<'a, T, P> -where - P: FnMut(&T) -> bool, -{ - #[inline] - fn finish(&mut self) -> Option<&'a mut [T]> { - self.inner.finish() - } -} - -#[stable(feature = "slice_rsplit", since = "1.27.0")] -impl<'a, T, P> Iterator for RSplitMut<'a, T, P> -where - P: FnMut(&T) -> bool, -{ - type Item = &'a mut [T]; - - #[inline] - fn next(&mut self) -> Option<&'a mut [T]> { - self.inner.next_back() - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.inner.size_hint() - } -} - -#[stable(feature = "slice_rsplit", since = "1.27.0")] -impl<'a, T, P> DoubleEndedIterator for RSplitMut<'a, T, P> -where - P: FnMut(&T) -> bool, -{ - #[inline] - fn next_back(&mut self) -> Option<&'a mut [T]> { - self.inner.next() - } -} - -#[stable(feature = "slice_rsplit", since = "1.27.0")] -impl FusedIterator for RSplitMut<'_, T, P> where P: FnMut(&T) -> bool {} - -/// An private iterator over subslices separated by elements that -/// match a predicate function, splitting at most a fixed number of -/// times. -#[derive(Debug)] -struct GenericSplitN { - iter: I, - count: usize, -} - -impl> Iterator for GenericSplitN { - type Item = T; - - #[inline] - fn next(&mut self) -> Option { - match self.count { - 0 => None, - 1 => { - self.count -= 1; - self.iter.finish() - } - _ => { - self.count -= 1; - self.iter.next() - } - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let (lower, upper_opt) = self.iter.size_hint(); - (lower, upper_opt.map(|upper| cmp::min(self.count, upper))) - } -} - -/// An iterator over subslices separated by elements that match a predicate -/// function, limited to a given number of splits. -/// -/// This struct is created by the [`splitn`] method on [slices]. -/// -/// [`splitn`]: ../../std/primitive.slice.html#method.splitn -/// [slices]: ../../std/primitive.slice.html -#[stable(feature = "rust1", since = "1.0.0")] -pub struct SplitN<'a, T: 'a, P> -where - P: FnMut(&T) -> bool, -{ - inner: GenericSplitN>, -} - -#[stable(feature = "core_impl_debug", since = "1.9.0")] -impl fmt::Debug for SplitN<'_, T, P> -where - P: FnMut(&T) -> bool, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SplitN").field("inner", &self.inner).finish() - } -} - -/// An iterator over subslices separated by elements that match a -/// predicate function, limited to a given number of splits, starting -/// from the end of the slice. -/// -/// This struct is created by the [`rsplitn`] method on [slices]. -/// -/// [`rsplitn`]: ../../std/primitive.slice.html#method.rsplitn -/// [slices]: ../../std/primitive.slice.html -#[stable(feature = "rust1", since = "1.0.0")] -pub struct RSplitN<'a, T: 'a, P> -where - P: FnMut(&T) -> bool, -{ - inner: GenericSplitN>, -} - -#[stable(feature = "core_impl_debug", since = "1.9.0")] -impl fmt::Debug for RSplitN<'_, T, P> -where - P: FnMut(&T) -> bool, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("RSplitN").field("inner", &self.inner).finish() - } -} - -/// An iterator over subslices separated by elements that match a predicate -/// function, limited to a given number of splits. -/// -/// This struct is created by the [`splitn_mut`] method on [slices]. -/// -/// [`splitn_mut`]: ../../std/primitive.slice.html#method.splitn_mut -/// [slices]: ../../std/primitive.slice.html -#[stable(feature = "rust1", since = "1.0.0")] -pub struct SplitNMut<'a, T: 'a, P> -where - P: FnMut(&T) -> bool, -{ - inner: GenericSplitN>, -} - -#[stable(feature = "core_impl_debug", since = "1.9.0")] -impl fmt::Debug for SplitNMut<'_, T, P> -where - P: FnMut(&T) -> bool, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SplitNMut").field("inner", &self.inner).finish() - } -} - -/// An iterator over subslices separated by elements that match a -/// predicate function, limited to a given number of splits, starting -/// from the end of the slice. -/// -/// This struct is created by the [`rsplitn_mut`] method on [slices]. -/// -/// [`rsplitn_mut`]: ../../std/primitive.slice.html#method.rsplitn_mut -/// [slices]: ../../std/primitive.slice.html -#[stable(feature = "rust1", since = "1.0.0")] -pub struct RSplitNMut<'a, T: 'a, P> -where - P: FnMut(&T) -> bool, -{ - inner: GenericSplitN>, -} - -#[stable(feature = "core_impl_debug", since = "1.9.0")] -impl fmt::Debug for RSplitNMut<'_, T, P> -where - P: FnMut(&T) -> bool, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("RSplitNMut").field("inner", &self.inner).finish() - } -} - -macro_rules! forward_iterator { - ($name:ident: $elem:ident, $iter_of:ty) => { - #[stable(feature = "rust1", since = "1.0.0")] - impl<'a, $elem, P> Iterator for $name<'a, $elem, P> - where - P: FnMut(&T) -> bool, - { - type Item = $iter_of; - - #[inline] - fn next(&mut self) -> Option<$iter_of> { - self.inner.next() - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.inner.size_hint() - } - } - - #[stable(feature = "fused", since = "1.26.0")] - impl<'a, $elem, P> FusedIterator for $name<'a, $elem, P> where P: FnMut(&T) -> bool {} - }; -} - -forward_iterator! { SplitN: T, &'a [T] } -forward_iterator! { RSplitN: T, &'a [T] } -forward_iterator! { SplitNMut: T, &'a mut [T] } -forward_iterator! { RSplitNMut: T, &'a mut [T] } - -/// An iterator over overlapping subslices of length `size`. -/// -/// This struct is created by the [`windows`] method on [slices]. -/// -/// [`windows`]: ../../std/primitive.slice.html#method.windows -/// [slices]: ../../std/primitive.slice.html -#[derive(Debug)] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct Windows<'a, T: 'a> { - v: &'a [T], - size: usize, -} - -// FIXME(#26925) Remove in favor of `#[derive(Clone)]` -#[stable(feature = "rust1", since = "1.0.0")] -impl Clone for Windows<'_, T> { - fn clone(&self) -> Self { - Windows { v: self.v, size: self.size } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T> Iterator for Windows<'a, T> { - type Item = &'a [T]; - - #[inline] - fn next(&mut self) -> Option<&'a [T]> { - if self.size > self.v.len() { - None - } else { - let ret = Some(&self.v[..self.size]); - self.v = &self.v[1..]; - ret - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - if self.size > self.v.len() { - (0, Some(0)) - } else { - let size = self.v.len() - self.size + 1; - (size, Some(size)) - } - } - - #[inline] - fn count(self) -> usize { - self.len() - } - - #[inline] - fn nth(&mut self, n: usize) -> Option { - let (end, overflow) = self.size.overflowing_add(n); - if end > self.v.len() || overflow { - self.v = &[]; - None - } else { - let nth = &self.v[n..end]; - self.v = &self.v[n + 1..]; - Some(nth) - } - } - - #[inline] - fn last(self) -> Option { - if self.size > self.v.len() { - None - } else { - let start = self.v.len() - self.size; - Some(&self.v[start..]) - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T> DoubleEndedIterator for Windows<'a, T> { - #[inline] - fn next_back(&mut self) -> Option<&'a [T]> { - if self.size > self.v.len() { - None - } else { - let ret = Some(&self.v[self.v.len() - self.size..]); - self.v = &self.v[..self.v.len() - 1]; - ret - } - } - - #[inline] - fn nth_back(&mut self, n: usize) -> Option { - let (end, overflow) = self.v.len().overflowing_sub(n); - if end < self.size || overflow { - self.v = &[]; - None - } else { - let ret = &self.v[end - self.size..end]; - self.v = &self.v[..end - 1]; - Some(ret) - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ExactSizeIterator for Windows<'_, T> {} - -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for Windows<'_, T> {} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Windows<'_, T> {} - -#[doc(hidden)] -unsafe impl<'a, T> TrustedRandomAccess for Windows<'a, T> { - unsafe fn get_unchecked(&mut self, i: usize) -> &'a [T] { - from_raw_parts(self.v.as_ptr().add(i), self.size) - } - fn may_have_side_effect() -> bool { - false - } -} - -/// An iterator over a slice in (non-overlapping) chunks (`chunk_size` elements at a -/// time), starting at the beginning of the slice. -/// -/// When the slice len is not evenly divided by the chunk size, the last slice -/// of the iteration will be the remainder. -/// -/// This struct is created by the [`chunks`] method on [slices]. -/// -/// [`chunks`]: ../../std/primitive.slice.html#method.chunks -/// [slices]: ../../std/primitive.slice.html -#[derive(Debug)] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct Chunks<'a, T: 'a> { - v: &'a [T], - chunk_size: usize, -} - -// FIXME(#26925) Remove in favor of `#[derive(Clone)]` -#[stable(feature = "rust1", since = "1.0.0")] -impl Clone for Chunks<'_, T> { - fn clone(&self) -> Self { - Chunks { v: self.v, chunk_size: self.chunk_size } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T> Iterator for Chunks<'a, T> { - type Item = &'a [T]; - - #[inline] - fn next(&mut self) -> Option<&'a [T]> { - if self.v.is_empty() { - None - } else { - let chunksz = cmp::min(self.v.len(), self.chunk_size); - let (fst, snd) = self.v.split_at(chunksz); - self.v = snd; - Some(fst) - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - if self.v.is_empty() { - (0, Some(0)) - } else { - let n = self.v.len() / self.chunk_size; - let rem = self.v.len() % self.chunk_size; - let n = if rem > 0 { n + 1 } else { n }; - (n, Some(n)) - } - } - - #[inline] - fn count(self) -> usize { - self.len() - } - - #[inline] - fn nth(&mut self, n: usize) -> Option { - let (start, overflow) = n.overflowing_mul(self.chunk_size); - if start >= self.v.len() || overflow { - self.v = &[]; - None - } else { - let end = match start.checked_add(self.chunk_size) { - Some(sum) => cmp::min(self.v.len(), sum), - None => self.v.len(), - }; - let nth = &self.v[start..end]; - self.v = &self.v[end..]; - Some(nth) - } - } - - #[inline] - fn last(self) -> Option { - if self.v.is_empty() { - None - } else { - let start = (self.v.len() - 1) / self.chunk_size * self.chunk_size; - Some(&self.v[start..]) - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T> DoubleEndedIterator for Chunks<'a, T> { - #[inline] - fn next_back(&mut self) -> Option<&'a [T]> { - if self.v.is_empty() { - None - } else { - let remainder = self.v.len() % self.chunk_size; - let chunksz = if remainder != 0 { remainder } else { self.chunk_size }; - let (fst, snd) = self.v.split_at(self.v.len() - chunksz); - self.v = fst; - Some(snd) - } - } - - #[inline] - fn nth_back(&mut self, n: usize) -> Option { - let len = self.len(); - if n >= len { - self.v = &[]; - None - } else { - let start = (len - 1 - n) * self.chunk_size; - let end = match start.checked_add(self.chunk_size) { - Some(res) => cmp::min(res, self.v.len()), - None => self.v.len(), - }; - let nth_back = &self.v[start..end]; - self.v = &self.v[..start]; - Some(nth_back) - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ExactSizeIterator for Chunks<'_, T> {} - -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for Chunks<'_, T> {} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Chunks<'_, T> {} - -#[doc(hidden)] -unsafe impl<'a, T> TrustedRandomAccess for Chunks<'a, T> { - unsafe fn get_unchecked(&mut self, i: usize) -> &'a [T] { - let start = i * self.chunk_size; - let end = match start.checked_add(self.chunk_size) { - None => self.v.len(), - Some(end) => cmp::min(end, self.v.len()), - }; - from_raw_parts(self.v.as_ptr().add(start), end - start) - } - fn may_have_side_effect() -> bool { - false - } -} - -/// An iterator over a slice in (non-overlapping) mutable chunks (`chunk_size` -/// elements at a time), starting at the beginning of the slice. -/// -/// When the slice len is not evenly divided by the chunk size, the last slice -/// of the iteration will be the remainder. -/// -/// This struct is created by the [`chunks_mut`] method on [slices]. -/// -/// [`chunks_mut`]: ../../std/primitive.slice.html#method.chunks_mut -/// [slices]: ../../std/primitive.slice.html -#[derive(Debug)] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct ChunksMut<'a, T: 'a> { - v: &'a mut [T], - chunk_size: usize, -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T> Iterator for ChunksMut<'a, T> { - type Item = &'a mut [T]; - - #[inline] - fn next(&mut self) -> Option<&'a mut [T]> { - if self.v.is_empty() { - None - } else { - let sz = cmp::min(self.v.len(), self.chunk_size); - let tmp = mem::replace(&mut self.v, &mut []); - let (head, tail) = tmp.split_at_mut(sz); - self.v = tail; - Some(head) - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - if self.v.is_empty() { - (0, Some(0)) - } else { - let n = self.v.len() / self.chunk_size; - let rem = self.v.len() % self.chunk_size; - let n = if rem > 0 { n + 1 } else { n }; - (n, Some(n)) - } - } - - #[inline] - fn count(self) -> usize { - self.len() - } - - #[inline] - fn nth(&mut self, n: usize) -> Option<&'a mut [T]> { - let (start, overflow) = n.overflowing_mul(self.chunk_size); - if start >= self.v.len() || overflow { - self.v = &mut []; - None - } else { - let end = match start.checked_add(self.chunk_size) { - Some(sum) => cmp::min(self.v.len(), sum), - None => self.v.len(), - }; - let tmp = mem::replace(&mut self.v, &mut []); - let (head, tail) = tmp.split_at_mut(end); - let (_, nth) = head.split_at_mut(start); - self.v = tail; - Some(nth) - } - } - - #[inline] - fn last(self) -> Option { - if self.v.is_empty() { - None - } else { - let start = (self.v.len() - 1) / self.chunk_size * self.chunk_size; - Some(&mut self.v[start..]) - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T> DoubleEndedIterator for ChunksMut<'a, T> { - #[inline] - fn next_back(&mut self) -> Option<&'a mut [T]> { - if self.v.is_empty() { - None - } else { - let remainder = self.v.len() % self.chunk_size; - let sz = if remainder != 0 { remainder } else { self.chunk_size }; - let tmp = mem::replace(&mut self.v, &mut []); - let tmp_len = tmp.len(); - let (head, tail) = tmp.split_at_mut(tmp_len - sz); - self.v = head; - Some(tail) - } - } - - #[inline] - fn nth_back(&mut self, n: usize) -> Option { - let len = self.len(); - if n >= len { - self.v = &mut []; - None - } else { - let start = (len - 1 - n) * self.chunk_size; - let end = match start.checked_add(self.chunk_size) { - Some(res) => cmp::min(res, self.v.len()), - None => self.v.len(), - }; - let (temp, _tail) = mem::replace(&mut self.v, &mut []).split_at_mut(end); - let (head, nth_back) = temp.split_at_mut(start); - self.v = head; - Some(nth_back) - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ExactSizeIterator for ChunksMut<'_, T> {} - -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for ChunksMut<'_, T> {} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for ChunksMut<'_, T> {} - -#[doc(hidden)] -unsafe impl<'a, T> TrustedRandomAccess for ChunksMut<'a, T> { - unsafe fn get_unchecked(&mut self, i: usize) -> &'a mut [T] { - let start = i * self.chunk_size; - let end = match start.checked_add(self.chunk_size) { - None => self.v.len(), - Some(end) => cmp::min(end, self.v.len()), - }; - from_raw_parts_mut(self.v.as_mut_ptr().add(start), end - start) - } - fn may_have_side_effect() -> bool { - false - } -} - -/// An iterator over a slice in (non-overlapping) chunks (`chunk_size` elements at a -/// time), starting at the beginning of the slice. -/// -/// When the slice len is not evenly divided by the chunk size, the last -/// up to `chunk_size-1` elements will be omitted but can be retrieved from -/// the [`remainder`] function from the iterator. -/// -/// This struct is created by the [`chunks_exact`] method on [slices]. -/// -/// [`chunks_exact`]: ../../std/primitive.slice.html#method.chunks_exact -/// [`remainder`]: ../../std/slice/struct.ChunksExact.html#method.remainder -/// [slices]: ../../std/primitive.slice.html -#[derive(Debug)] -#[stable(feature = "chunks_exact", since = "1.31.0")] -pub struct ChunksExact<'a, T: 'a> { - v: &'a [T], - rem: &'a [T], - chunk_size: usize, -} - -impl<'a, T> ChunksExact<'a, T> { - /// Returns the remainder of the original slice that is not going to be - /// returned by the iterator. The returned slice has at most `chunk_size-1` - /// elements. - #[stable(feature = "chunks_exact", since = "1.31.0")] - pub fn remainder(&self) -> &'a [T] { - self.rem - } -} - -// FIXME(#26925) Remove in favor of `#[derive(Clone)]` -#[stable(feature = "chunks_exact", since = "1.31.0")] -impl Clone for ChunksExact<'_, T> { - fn clone(&self) -> Self { - ChunksExact { v: self.v, rem: self.rem, chunk_size: self.chunk_size } - } -} - -#[stable(feature = "chunks_exact", since = "1.31.0")] -impl<'a, T> Iterator for ChunksExact<'a, T> { - type Item = &'a [T]; - - #[inline] - fn next(&mut self) -> Option<&'a [T]> { - if self.v.len() < self.chunk_size { - None - } else { - let (fst, snd) = self.v.split_at(self.chunk_size); - self.v = snd; - Some(fst) - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let n = self.v.len() / self.chunk_size; - (n, Some(n)) - } - - #[inline] - fn count(self) -> usize { - self.len() - } - - #[inline] - fn nth(&mut self, n: usize) -> Option { - let (start, overflow) = n.overflowing_mul(self.chunk_size); - if start >= self.v.len() || overflow { - self.v = &[]; - None - } else { - let (_, snd) = self.v.split_at(start); - self.v = snd; - self.next() - } - } - - #[inline] - fn last(mut self) -> Option { - self.next_back() - } -} - -#[stable(feature = "chunks_exact", since = "1.31.0")] -impl<'a, T> DoubleEndedIterator for ChunksExact<'a, T> { - #[inline] - fn next_back(&mut self) -> Option<&'a [T]> { - if self.v.len() < self.chunk_size { - None - } else { - let (fst, snd) = self.v.split_at(self.v.len() - self.chunk_size); - self.v = fst; - Some(snd) - } - } - - #[inline] - fn nth_back(&mut self, n: usize) -> Option { - let len = self.len(); - if n >= len { - self.v = &[]; - None - } else { - let start = (len - 1 - n) * self.chunk_size; - let end = start + self.chunk_size; - let nth_back = &self.v[start..end]; - self.v = &self.v[..start]; - Some(nth_back) - } - } -} - -#[stable(feature = "chunks_exact", since = "1.31.0")] -impl ExactSizeIterator for ChunksExact<'_, T> { - fn is_empty(&self) -> bool { - self.v.is_empty() - } -} - -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for ChunksExact<'_, T> {} - -#[stable(feature = "chunks_exact", since = "1.31.0")] -impl FusedIterator for ChunksExact<'_, T> {} - -#[doc(hidden)] -#[stable(feature = "chunks_exact", since = "1.31.0")] -unsafe impl<'a, T> TrustedRandomAccess for ChunksExact<'a, T> { - unsafe fn get_unchecked(&mut self, i: usize) -> &'a [T] { - let start = i * self.chunk_size; - from_raw_parts(self.v.as_ptr().add(start), self.chunk_size) - } - fn may_have_side_effect() -> bool { - false - } -} - -/// An iterator over a slice in (non-overlapping) mutable chunks (`chunk_size` -/// elements at a time), starting at the beginning of the slice. -/// -/// When the slice len is not evenly divided by the chunk size, the last up to -/// `chunk_size-1` elements will be omitted but can be retrieved from the -/// [`into_remainder`] function from the iterator. -/// -/// This struct is created by the [`chunks_exact_mut`] method on [slices]. -/// -/// [`chunks_exact_mut`]: ../../std/primitive.slice.html#method.chunks_exact_mut -/// [`into_remainder`]: ../../std/slice/struct.ChunksExactMut.html#method.into_remainder -/// [slices]: ../../std/primitive.slice.html -#[derive(Debug)] -#[stable(feature = "chunks_exact", since = "1.31.0")] -pub struct ChunksExactMut<'a, T: 'a> { - v: &'a mut [T], - rem: &'a mut [T], - chunk_size: usize, -} - -impl<'a, T> ChunksExactMut<'a, T> { - /// Returns the remainder of the original slice that is not going to be - /// returned by the iterator. The returned slice has at most `chunk_size-1` - /// elements. - #[stable(feature = "chunks_exact", since = "1.31.0")] - pub fn into_remainder(self) -> &'a mut [T] { - self.rem - } -} - -#[stable(feature = "chunks_exact", since = "1.31.0")] -impl<'a, T> Iterator for ChunksExactMut<'a, T> { - type Item = &'a mut [T]; - - #[inline] - fn next(&mut self) -> Option<&'a mut [T]> { - if self.v.len() < self.chunk_size { - None - } else { - let tmp = mem::replace(&mut self.v, &mut []); - let (head, tail) = tmp.split_at_mut(self.chunk_size); - self.v = tail; - Some(head) - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let n = self.v.len() / self.chunk_size; - (n, Some(n)) - } - - #[inline] - fn count(self) -> usize { - self.len() - } - - #[inline] - fn nth(&mut self, n: usize) -> Option<&'a mut [T]> { - let (start, overflow) = n.overflowing_mul(self.chunk_size); - if start >= self.v.len() || overflow { - self.v = &mut []; - None - } else { - let tmp = mem::replace(&mut self.v, &mut []); - let (_, snd) = tmp.split_at_mut(start); - self.v = snd; - self.next() - } - } - - #[inline] - fn last(mut self) -> Option { - self.next_back() - } -} - -#[stable(feature = "chunks_exact", since = "1.31.0")] -impl<'a, T> DoubleEndedIterator for ChunksExactMut<'a, T> { - #[inline] - fn next_back(&mut self) -> Option<&'a mut [T]> { - if self.v.len() < self.chunk_size { - None - } else { - let tmp = mem::replace(&mut self.v, &mut []); - let tmp_len = tmp.len(); - let (head, tail) = tmp.split_at_mut(tmp_len - self.chunk_size); - self.v = head; - Some(tail) - } - } - - #[inline] - fn nth_back(&mut self, n: usize) -> Option { - let len = self.len(); - if n >= len { - self.v = &mut []; - None - } else { - let start = (len - 1 - n) * self.chunk_size; - let end = start + self.chunk_size; - let (temp, _tail) = mem::replace(&mut self.v, &mut []).split_at_mut(end); - let (head, nth_back) = temp.split_at_mut(start); - self.v = head; - Some(nth_back) - } - } -} - -#[stable(feature = "chunks_exact", since = "1.31.0")] -impl ExactSizeIterator for ChunksExactMut<'_, T> { - fn is_empty(&self) -> bool { - self.v.is_empty() - } -} - -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for ChunksExactMut<'_, T> {} - -#[stable(feature = "chunks_exact", since = "1.31.0")] -impl FusedIterator for ChunksExactMut<'_, T> {} - -#[doc(hidden)] -#[stable(feature = "chunks_exact", since = "1.31.0")] -unsafe impl<'a, T> TrustedRandomAccess for ChunksExactMut<'a, T> { - unsafe fn get_unchecked(&mut self, i: usize) -> &'a mut [T] { - let start = i * self.chunk_size; - from_raw_parts_mut(self.v.as_mut_ptr().add(start), self.chunk_size) - } - fn may_have_side_effect() -> bool { - false - } -} - -/// An iterator over a slice in (non-overlapping) chunks (`chunk_size` elements at a -/// time), starting at the end of the slice. -/// -/// When the slice len is not evenly divided by the chunk size, the last slice -/// of the iteration will be the remainder. -/// -/// This struct is created by the [`rchunks`] method on [slices]. -/// -/// [`rchunks`]: ../../std/primitive.slice.html#method.rchunks -/// [slices]: ../../std/primitive.slice.html -#[derive(Debug)] -#[stable(feature = "rchunks", since = "1.31.0")] -pub struct RChunks<'a, T: 'a> { - v: &'a [T], - chunk_size: usize, -} - -// FIXME(#26925) Remove in favor of `#[derive(Clone)]` -#[stable(feature = "rchunks", since = "1.31.0")] -impl Clone for RChunks<'_, T> { - fn clone(&self) -> Self { - RChunks { v: self.v, chunk_size: self.chunk_size } - } -} - -#[stable(feature = "rchunks", since = "1.31.0")] -impl<'a, T> Iterator for RChunks<'a, T> { - type Item = &'a [T]; - - #[inline] - fn next(&mut self) -> Option<&'a [T]> { - if self.v.is_empty() { - None - } else { - let chunksz = cmp::min(self.v.len(), self.chunk_size); - let (fst, snd) = self.v.split_at(self.v.len() - chunksz); - self.v = fst; - Some(snd) - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - if self.v.is_empty() { - (0, Some(0)) - } else { - let n = self.v.len() / self.chunk_size; - let rem = self.v.len() % self.chunk_size; - let n = if rem > 0 { n + 1 } else { n }; - (n, Some(n)) - } - } - - #[inline] - fn count(self) -> usize { - self.len() - } - - #[inline] - fn nth(&mut self, n: usize) -> Option { - let (end, overflow) = n.overflowing_mul(self.chunk_size); - if end >= self.v.len() || overflow { - self.v = &[]; - None - } else { - // Can't underflow because of the check above - let end = self.v.len() - end; - let start = match end.checked_sub(self.chunk_size) { - Some(sum) => sum, - None => 0, - }; - let nth = &self.v[start..end]; - self.v = &self.v[0..start]; - Some(nth) - } - } - - #[inline] - fn last(self) -> Option { - if self.v.is_empty() { - None - } else { - let rem = self.v.len() % self.chunk_size; - let end = if rem == 0 { self.chunk_size } else { rem }; - Some(&self.v[0..end]) - } - } -} - -#[stable(feature = "rchunks", since = "1.31.0")] -impl<'a, T> DoubleEndedIterator for RChunks<'a, T> { - #[inline] - fn next_back(&mut self) -> Option<&'a [T]> { - if self.v.is_empty() { - None - } else { - let remainder = self.v.len() % self.chunk_size; - let chunksz = if remainder != 0 { remainder } else { self.chunk_size }; - let (fst, snd) = self.v.split_at(chunksz); - self.v = snd; - Some(fst) - } - } - - #[inline] - fn nth_back(&mut self, n: usize) -> Option { - let len = self.len(); - if n >= len { - self.v = &[]; - None - } else { - // can't underflow because `n < len` - let offset_from_end = (len - 1 - n) * self.chunk_size; - let end = self.v.len() - offset_from_end; - let start = end.saturating_sub(self.chunk_size); - let nth_back = &self.v[start..end]; - self.v = &self.v[end..]; - Some(nth_back) - } - } -} - -#[stable(feature = "rchunks", since = "1.31.0")] -impl ExactSizeIterator for RChunks<'_, T> {} - -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for RChunks<'_, T> {} - -#[stable(feature = "rchunks", since = "1.31.0")] -impl FusedIterator for RChunks<'_, T> {} - -#[doc(hidden)] -#[stable(feature = "rchunks", since = "1.31.0")] -unsafe impl<'a, T> TrustedRandomAccess for RChunks<'a, T> { - unsafe fn get_unchecked(&mut self, i: usize) -> &'a [T] { - let end = self.v.len() - i * self.chunk_size; - let start = match end.checked_sub(self.chunk_size) { - None => 0, - Some(start) => start, - }; - from_raw_parts(self.v.as_ptr().add(start), end - start) - } - fn may_have_side_effect() -> bool { - false - } -} - -/// An iterator over a slice in (non-overlapping) mutable chunks (`chunk_size` -/// elements at a time), starting at the end of the slice. -/// -/// When the slice len is not evenly divided by the chunk size, the last slice -/// of the iteration will be the remainder. -/// -/// This struct is created by the [`rchunks_mut`] method on [slices]. -/// -/// [`rchunks_mut`]: ../../std/primitive.slice.html#method.rchunks_mut -/// [slices]: ../../std/primitive.slice.html -#[derive(Debug)] -#[stable(feature = "rchunks", since = "1.31.0")] -pub struct RChunksMut<'a, T: 'a> { - v: &'a mut [T], - chunk_size: usize, -} - -#[stable(feature = "rchunks", since = "1.31.0")] -impl<'a, T> Iterator for RChunksMut<'a, T> { - type Item = &'a mut [T]; - - #[inline] - fn next(&mut self) -> Option<&'a mut [T]> { - if self.v.is_empty() { - None - } else { - let sz = cmp::min(self.v.len(), self.chunk_size); - let tmp = mem::replace(&mut self.v, &mut []); - let tmp_len = tmp.len(); - let (head, tail) = tmp.split_at_mut(tmp_len - sz); - self.v = head; - Some(tail) - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - if self.v.is_empty() { - (0, Some(0)) - } else { - let n = self.v.len() / self.chunk_size; - let rem = self.v.len() % self.chunk_size; - let n = if rem > 0 { n + 1 } else { n }; - (n, Some(n)) - } - } - - #[inline] - fn count(self) -> usize { - self.len() - } - - #[inline] - fn nth(&mut self, n: usize) -> Option<&'a mut [T]> { - let (end, overflow) = n.overflowing_mul(self.chunk_size); - if end >= self.v.len() || overflow { - self.v = &mut []; - None - } else { - // Can't underflow because of the check above - let end = self.v.len() - end; - let start = match end.checked_sub(self.chunk_size) { - Some(sum) => sum, - None => 0, - }; - let tmp = mem::replace(&mut self.v, &mut []); - let (head, tail) = tmp.split_at_mut(start); - let (nth, _) = tail.split_at_mut(end - start); - self.v = head; - Some(nth) - } - } - - #[inline] - fn last(self) -> Option { - if self.v.is_empty() { - None - } else { - let rem = self.v.len() % self.chunk_size; - let end = if rem == 0 { self.chunk_size } else { rem }; - Some(&mut self.v[0..end]) - } - } -} - -#[stable(feature = "rchunks", since = "1.31.0")] -impl<'a, T> DoubleEndedIterator for RChunksMut<'a, T> { - #[inline] - fn next_back(&mut self) -> Option<&'a mut [T]> { - if self.v.is_empty() { - None - } else { - let remainder = self.v.len() % self.chunk_size; - let sz = if remainder != 0 { remainder } else { self.chunk_size }; - let tmp = mem::replace(&mut self.v, &mut []); - let (head, tail) = tmp.split_at_mut(sz); - self.v = tail; - Some(head) - } - } - - #[inline] - fn nth_back(&mut self, n: usize) -> Option { - let len = self.len(); - if n >= len { - self.v = &mut []; - None - } else { - // can't underflow because `n < len` - let offset_from_end = (len - 1 - n) * self.chunk_size; - let end = self.v.len() - offset_from_end; - let start = end.saturating_sub(self.chunk_size); - let (tmp, tail) = mem::replace(&mut self.v, &mut []).split_at_mut(end); - let (_, nth_back) = tmp.split_at_mut(start); - self.v = tail; - Some(nth_back) - } - } -} - -#[stable(feature = "rchunks", since = "1.31.0")] -impl ExactSizeIterator for RChunksMut<'_, T> {} - -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for RChunksMut<'_, T> {} - -#[stable(feature = "rchunks", since = "1.31.0")] -impl FusedIterator for RChunksMut<'_, T> {} - -#[doc(hidden)] -#[stable(feature = "rchunks", since = "1.31.0")] -unsafe impl<'a, T> TrustedRandomAccess for RChunksMut<'a, T> { - unsafe fn get_unchecked(&mut self, i: usize) -> &'a mut [T] { - let end = self.v.len() - i * self.chunk_size; - let start = match end.checked_sub(self.chunk_size) { - None => 0, - Some(start) => start, - }; - from_raw_parts_mut(self.v.as_mut_ptr().add(start), end - start) - } - fn may_have_side_effect() -> bool { - false - } -} - -/// An iterator over a slice in (non-overlapping) chunks (`chunk_size` elements at a -/// time), starting at the end of the slice. -/// -/// When the slice len is not evenly divided by the chunk size, the last -/// up to `chunk_size-1` elements will be omitted but can be retrieved from -/// the [`remainder`] function from the iterator. -/// -/// This struct is created by the [`rchunks_exact`] method on [slices]. -/// -/// [`rchunks_exact`]: ../../std/primitive.slice.html#method.rchunks_exact -/// [`remainder`]: ../../std/slice/struct.ChunksExact.html#method.remainder -/// [slices]: ../../std/primitive.slice.html -#[derive(Debug)] -#[stable(feature = "rchunks", since = "1.31.0")] -pub struct RChunksExact<'a, T: 'a> { - v: &'a [T], - rem: &'a [T], - chunk_size: usize, -} - -impl<'a, T> RChunksExact<'a, T> { - /// Returns the remainder of the original slice that is not going to be - /// returned by the iterator. The returned slice has at most `chunk_size-1` - /// elements. - #[stable(feature = "rchunks", since = "1.31.0")] - pub fn remainder(&self) -> &'a [T] { - self.rem - } -} - -// FIXME(#26925) Remove in favor of `#[derive(Clone)]` -#[stable(feature = "rchunks", since = "1.31.0")] -impl<'a, T> Clone for RChunksExact<'a, T> { - fn clone(&self) -> RChunksExact<'a, T> { - RChunksExact { v: self.v, rem: self.rem, chunk_size: self.chunk_size } - } -} - -#[stable(feature = "rchunks", since = "1.31.0")] -impl<'a, T> Iterator for RChunksExact<'a, T> { - type Item = &'a [T]; - - #[inline] - fn next(&mut self) -> Option<&'a [T]> { - if self.v.len() < self.chunk_size { - None - } else { - let (fst, snd) = self.v.split_at(self.v.len() - self.chunk_size); - self.v = fst; - Some(snd) - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let n = self.v.len() / self.chunk_size; - (n, Some(n)) - } - - #[inline] - fn count(self) -> usize { - self.len() - } - - #[inline] - fn nth(&mut self, n: usize) -> Option { - let (end, overflow) = n.overflowing_mul(self.chunk_size); - if end >= self.v.len() || overflow { - self.v = &[]; - None - } else { - let (fst, _) = self.v.split_at(self.v.len() - end); - self.v = fst; - self.next() - } - } - - #[inline] - fn last(mut self) -> Option { - self.next_back() - } -} - -#[stable(feature = "rchunks", since = "1.31.0")] -impl<'a, T> DoubleEndedIterator for RChunksExact<'a, T> { - #[inline] - fn next_back(&mut self) -> Option<&'a [T]> { - if self.v.len() < self.chunk_size { - None - } else { - let (fst, snd) = self.v.split_at(self.chunk_size); - self.v = snd; - Some(fst) - } - } - - #[inline] - fn nth_back(&mut self, n: usize) -> Option { - let len = self.len(); - if n >= len { - self.v = &[]; - None - } else { - // now that we know that `n` corresponds to a chunk, - // none of these operations can underflow/overflow - let offset = (len - n) * self.chunk_size; - let start = self.v.len() - offset; - let end = start + self.chunk_size; - let nth_back = &self.v[start..end]; - self.v = &self.v[end..]; - Some(nth_back) - } - } -} - -#[stable(feature = "rchunks", since = "1.31.0")] -impl<'a, T> ExactSizeIterator for RChunksExact<'a, T> { - fn is_empty(&self) -> bool { - self.v.is_empty() - } -} - -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for RChunksExact<'_, T> {} - -#[stable(feature = "rchunks", since = "1.31.0")] -impl FusedIterator for RChunksExact<'_, T> {} - -#[doc(hidden)] -#[stable(feature = "rchunks", since = "1.31.0")] -unsafe impl<'a, T> TrustedRandomAccess for RChunksExact<'a, T> { - unsafe fn get_unchecked(&mut self, i: usize) -> &'a [T] { - let end = self.v.len() - i * self.chunk_size; - let start = end - self.chunk_size; - from_raw_parts(self.v.as_ptr().add(start), self.chunk_size) - } - fn may_have_side_effect() -> bool { - false - } -} - -/// An iterator over a slice in (non-overlapping) mutable chunks (`chunk_size` -/// elements at a time), starting at the end of the slice. -/// -/// When the slice len is not evenly divided by the chunk size, the last up to -/// `chunk_size-1` elements will be omitted but can be retrieved from the -/// [`into_remainder`] function from the iterator. -/// -/// This struct is created by the [`rchunks_exact_mut`] method on [slices]. -/// -/// [`rchunks_exact_mut`]: ../../std/primitive.slice.html#method.rchunks_exact_mut -/// [`into_remainder`]: ../../std/slice/struct.ChunksExactMut.html#method.into_remainder -/// [slices]: ../../std/primitive.slice.html -#[derive(Debug)] -#[stable(feature = "rchunks", since = "1.31.0")] -pub struct RChunksExactMut<'a, T: 'a> { - v: &'a mut [T], - rem: &'a mut [T], - chunk_size: usize, -} - -impl<'a, T> RChunksExactMut<'a, T> { - /// Returns the remainder of the original slice that is not going to be - /// returned by the iterator. The returned slice has at most `chunk_size-1` - /// elements. - #[stable(feature = "rchunks", since = "1.31.0")] - pub fn into_remainder(self) -> &'a mut [T] { - self.rem - } -} - -#[stable(feature = "rchunks", since = "1.31.0")] -impl<'a, T> Iterator for RChunksExactMut<'a, T> { - type Item = &'a mut [T]; - - #[inline] - fn next(&mut self) -> Option<&'a mut [T]> { - if self.v.len() < self.chunk_size { - None - } else { - let tmp = mem::replace(&mut self.v, &mut []); - let tmp_len = tmp.len(); - let (head, tail) = tmp.split_at_mut(tmp_len - self.chunk_size); - self.v = head; - Some(tail) - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let n = self.v.len() / self.chunk_size; - (n, Some(n)) - } - - #[inline] - fn count(self) -> usize { - self.len() - } - - #[inline] - fn nth(&mut self, n: usize) -> Option<&'a mut [T]> { - let (end, overflow) = n.overflowing_mul(self.chunk_size); - if end >= self.v.len() || overflow { - self.v = &mut []; - None - } else { - let tmp = mem::replace(&mut self.v, &mut []); - let tmp_len = tmp.len(); - let (fst, _) = tmp.split_at_mut(tmp_len - end); - self.v = fst; - self.next() - } - } - - #[inline] - fn last(mut self) -> Option { - self.next_back() - } -} - -#[stable(feature = "rchunks", since = "1.31.0")] -impl<'a, T> DoubleEndedIterator for RChunksExactMut<'a, T> { - #[inline] - fn next_back(&mut self) -> Option<&'a mut [T]> { - if self.v.len() < self.chunk_size { - None - } else { - let tmp = mem::replace(&mut self.v, &mut []); - let (head, tail) = tmp.split_at_mut(self.chunk_size); - self.v = tail; - Some(head) - } - } - - #[inline] - fn nth_back(&mut self, n: usize) -> Option { - let len = self.len(); - if n >= len { - self.v = &mut []; - None - } else { - // now that we know that `n` corresponds to a chunk, - // none of these operations can underflow/overflow - let offset = (len - n) * self.chunk_size; - let start = self.v.len() - offset; - let end = start + self.chunk_size; - let (tmp, tail) = mem::replace(&mut self.v, &mut []).split_at_mut(end); - let (_, nth_back) = tmp.split_at_mut(start); - self.v = tail; - Some(nth_back) - } - } -} - -#[stable(feature = "rchunks", since = "1.31.0")] -impl ExactSizeIterator for RChunksExactMut<'_, T> { - fn is_empty(&self) -> bool { - self.v.is_empty() - } -} - -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for RChunksExactMut<'_, T> {} - -#[stable(feature = "rchunks", since = "1.31.0")] -impl FusedIterator for RChunksExactMut<'_, T> {} - -#[doc(hidden)] -#[stable(feature = "rchunks", since = "1.31.0")] -unsafe impl<'a, T> TrustedRandomAccess for RChunksExactMut<'a, T> { - unsafe fn get_unchecked(&mut self, i: usize) -> &'a mut [T] { - let end = self.v.len() - i * self.chunk_size; - let start = end - self.chunk_size; - from_raw_parts_mut(self.v.as_mut_ptr().add(start), self.chunk_size) - } - fn may_have_side_effect() -> bool { - false - } -} - -// -// Free functions -// - -/// Forms a slice from a pointer and a length. -/// -/// The `len` argument is the number of **elements**, not the number of bytes. -/// -/// # Safety -/// -/// Behavior is undefined if any of the following conditions are violated: -/// -/// * `data` must be [valid] for reads for `len * mem::size_of::()` many bytes, -/// and it must be properly aligned. This means in particular: -/// -/// * The entire memory range of this slice must be contained within a single allocated object! -/// Slices can never span across multiple allocated objects. -/// * `data` must be non-null and aligned even for zero-length slices. One -/// reason for this is that enum layout optimizations may rely on references -/// (including slices of any length) being aligned and non-null to distinguish -/// them from other data. You can obtain a pointer that is usable as `data` -/// for zero-length slices using [`NonNull::dangling()`]. -/// -/// * The memory referenced by the returned slice must not be mutated for the duration -/// of lifetime `'a`, except inside an `UnsafeCell`. -/// -/// * The total size `len * mem::size_of::()` of the slice must be no larger than `isize::MAX`. -/// See the safety documentation of [`pointer::offset`]. -/// -/// # Caveat -/// -/// The lifetime for the returned slice is inferred from its usage. To -/// prevent accidental misuse, it's suggested to tie the lifetime to whichever -/// source lifetime is safe in the context, such as by providing a helper -/// function taking the lifetime of a host value for the slice, or by explicit -/// annotation. -/// -/// # Examples -/// -/// ``` -/// use std::slice; -/// -/// // manifest a slice for a single element -/// let x = 42; -/// let ptr = &x as *const _; -/// let slice = unsafe { slice::from_raw_parts(ptr, 1) }; -/// assert_eq!(slice[0], 42); -/// ``` -/// -/// [valid]: ../../std/ptr/index.html#safety -/// [`NonNull::dangling()`]: ../../std/ptr/struct.NonNull.html#method.dangling -/// [`pointer::offset`]: ../../std/primitive.pointer.html#method.offset -#[inline] -#[stable(feature = "rust1", since = "1.0.0")] -pub unsafe fn from_raw_parts<'a, T>(data: *const T, len: usize) -> &'a [T] { - debug_assert!(is_aligned_and_not_null(data), "attempt to create unaligned or null slice"); - debug_assert!( - mem::size_of::().saturating_mul(len) <= isize::MAX as usize, - "attempt to create slice covering at least half the address space" - ); - &*ptr::slice_from_raw_parts(data, len) -} - -/// Performs the same functionality as [`from_raw_parts`], except that a -/// mutable slice is returned. -/// -/// # Safety -/// -/// Behavior is undefined if any of the following conditions are violated: -/// -/// * `data` must be [valid] for writes for `len * mem::size_of::()` many bytes, -/// and it must be properly aligned. This means in particular: -/// -/// * The entire memory range of this slice must be contained within a single allocated object! -/// Slices can never span across multiple allocated objects. -/// * `data` must be non-null and aligned even for zero-length slices. One -/// reason for this is that enum layout optimizations may rely on references -/// (including slices of any length) being aligned and non-null to distinguish -/// them from other data. You can obtain a pointer that is usable as `data` -/// for zero-length slices using [`NonNull::dangling()`]. -/// -/// * The memory referenced by the returned slice must not be accessed through any other pointer -/// (not derived from the return value) for the duration of lifetime `'a`. -/// Both read and write accesses are forbidden. -/// -/// * The total size `len * mem::size_of::()` of the slice must be no larger than `isize::MAX`. -/// See the safety documentation of [`pointer::offset`]. -/// -/// [valid]: ../../std/ptr/index.html#safety -/// [`NonNull::dangling()`]: ../../std/ptr/struct.NonNull.html#method.dangling -/// [`pointer::offset`]: ../../std/primitive.pointer.html#method.offset -/// [`from_raw_parts`]: ../../std/slice/fn.from_raw_parts.html -#[inline] -#[stable(feature = "rust1", since = "1.0.0")] -pub unsafe fn from_raw_parts_mut<'a, T>(data: *mut T, len: usize) -> &'a mut [T] { - debug_assert!(is_aligned_and_not_null(data), "attempt to create unaligned or null slice"); - debug_assert!( - mem::size_of::().saturating_mul(len) <= isize::MAX as usize, - "attempt to create slice covering at least half the address space" - ); - &mut *ptr::slice_from_raw_parts_mut(data, len) -} - -/// Converts a reference to T into a slice of length 1 (without copying). -#[stable(feature = "from_ref", since = "1.28.0")] -pub fn from_ref(s: &T) -> &[T] { - unsafe { from_raw_parts(s, 1) } -} - -/// Converts a reference to T into a slice of length 1 (without copying). -#[stable(feature = "from_ref", since = "1.28.0")] -pub fn from_mut(s: &mut T) -> &mut [T] { - unsafe { from_raw_parts_mut(s, 1) } -} - -// This function is public only because there is no other way to unit test heapsort. -#[unstable(feature = "sort_internals", reason = "internal to sort module", issue = "none")] -#[doc(hidden)] -pub fn heapsort(v: &mut [T], mut is_less: F) -where - F: FnMut(&T, &T) -> bool, -{ - sort::heapsort(v, &mut is_less); -} - -// -// Comparison traits -// - -extern "C" { - /// Calls implementation provided memcmp. - /// - /// Interprets the data as u8. - /// - /// Returns 0 for equal, < 0 for less than and > 0 for greater - /// than. - // FIXME(#32610): Return type should be c_int - fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32; -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl PartialEq<[B]> for [A] -where - A: PartialEq, -{ - fn eq(&self, other: &[B]) -> bool { - SlicePartialEq::equal(self, other) - } - - fn ne(&self, other: &[B]) -> bool { - SlicePartialEq::not_equal(self, other) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Eq for [T] {} - -/// Implements comparison of vectors lexicographically. -#[stable(feature = "rust1", since = "1.0.0")] -impl Ord for [T] { - fn cmp(&self, other: &[T]) -> Ordering { - SliceOrd::compare(self, other) - } -} - -/// Implements comparison of vectors lexicographically. -#[stable(feature = "rust1", since = "1.0.0")] -impl PartialOrd for [T] { - fn partial_cmp(&self, other: &[T]) -> Option { - SlicePartialOrd::partial_compare(self, other) - } -} - -#[doc(hidden)] -// intermediate trait for specialization of slice's PartialEq -trait SlicePartialEq { - fn equal(&self, other: &[B]) -> bool; - - fn not_equal(&self, other: &[B]) -> bool { - !self.equal(other) - } -} - -// Generic slice equality -impl SlicePartialEq for [A] -where - A: PartialEq, -{ - default fn equal(&self, other: &[B]) -> bool { - if self.len() != other.len() { - return false; - } - - self.iter().zip(other.iter()).all(|(x, y)| x == y) - } -} - -// Use an equal-pointer optimization when types are `Eq` -impl SlicePartialEq for [A] -where - A: PartialEq + Eq, -{ - default fn equal(&self, other: &[A]) -> bool { - if self.len() != other.len() { - return false; - } - - // Crux: pointer comparison removed - - self.iter().zip(other.iter()).all(|(x, y)| x == y) - } -} - -/* -// Use memcmp for bytewise equality when the types allow -impl SlicePartialEq for [A] -where - A: PartialEq + BytewiseEquality, -{ - fn equal(&self, other: &[A]) -> bool { - if self.len() != other.len() { - return false; - } - if self.as_ptr() == other.as_ptr() { - return true; - } - unsafe { - let size = mem::size_of_val(self); - memcmp(self.as_ptr() as *const u8, other.as_ptr() as *const u8, size) == 0 - } - } -} -*/ - -#[doc(hidden)] -// intermediate trait for specialization of slice's PartialOrd -trait SlicePartialOrd: Sized { - fn partial_compare(left: &[Self], right: &[Self]) -> Option; -} - -impl SlicePartialOrd for A { - default fn partial_compare(left: &[A], right: &[A]) -> Option { - let l = cmp::min(left.len(), right.len()); - - // Slice to the loop iteration range to enable bound check - // elimination in the compiler - let lhs = &left[..l]; - let rhs = &right[..l]; - - for i in 0..l { - match lhs[i].partial_cmp(&rhs[i]) { - Some(Ordering::Equal) => (), - non_eq => return non_eq, - } - } - - left.len().partial_cmp(&right.len()) - } -} - -// This is the impl that we would like to have. Unfortunately it's not sound. -// See `partial_ord_slice.rs`. -/* -impl SlicePartialOrd for A -where - A: Ord, -{ - default fn partial_compare(left: &[A], right: &[A]) -> Option { - Some(SliceOrd::compare(left, right)) - } -} -*/ - -impl SlicePartialOrd for A { - fn partial_compare(left: &[A], right: &[A]) -> Option { - Some(SliceOrd::compare(left, right)) - } -} - -trait AlwaysApplicableOrd: SliceOrd + Ord {} - -macro_rules! always_applicable_ord { - ($([$($p:tt)*] $t:ty,)*) => { - $(impl<$($p)*> AlwaysApplicableOrd for $t {})* - } -} - -always_applicable_ord! { - [] u8, [] u16, [] u32, [] u64, [] u128, [] usize, - [] i8, [] i16, [] i32, [] i64, [] i128, [] isize, - [] bool, [] char, - [T: ?Sized] *const T, [T: ?Sized] *mut T, - [T: AlwaysApplicableOrd] &T, - [T: AlwaysApplicableOrd] &mut T, - [T: AlwaysApplicableOrd] Option, -} - -#[doc(hidden)] -// intermediate trait for specialization of slice's Ord -trait SliceOrd: Sized { - fn compare(left: &[Self], right: &[Self]) -> Ordering; -} - -impl SliceOrd for A { - default fn compare(left: &[Self], right: &[Self]) -> Ordering { - let l = cmp::min(left.len(), right.len()); - - // Slice to the loop iteration range to enable bound check - // elimination in the compiler - let lhs = &left[..l]; - let rhs = &right[..l]; - - for i in 0..l { - match lhs[i].cmp(&rhs[i]) { - Ordering::Equal => (), - non_eq => return non_eq, - } - } - - left.len().cmp(&right.len()) - } -} - -// memcmp compares a sequence of unsigned bytes lexicographically. -// this matches the order we want for [u8], but no others (not even [i8]). -impl SliceOrd for u8 { - #[inline] - fn compare(left: &[Self], right: &[Self]) -> Ordering { - let order = - unsafe { memcmp(left.as_ptr(), right.as_ptr(), cmp::min(left.len(), right.len())) }; - if order == 0 { - left.len().cmp(&right.len()) - } else if order < 0 { - Less - } else { - Greater - } - } -} - -#[doc(hidden)] -/// Trait implemented for types that can be compared for equality using -/// their bytewise representation -trait BytewiseEquality: Eq + Copy {} - -macro_rules! impl_marker_for { - ($traitname:ident, $($ty:ty)*) => { - $( - impl $traitname for $ty { } - )* - } -} - -impl_marker_for!(BytewiseEquality, - u8 i8 u16 i16 u32 i32 u64 i64 u128 i128 usize isize char bool); - -#[doc(hidden)] -unsafe impl<'a, T> TrustedRandomAccess for Iter<'a, T> { - unsafe fn get_unchecked(&mut self, i: usize) -> &'a T { - &*self.ptr.as_ptr().add(i) - } - fn may_have_side_effect() -> bool { - false - } -} - -#[doc(hidden)] -unsafe impl<'a, T> TrustedRandomAccess for IterMut<'a, T> { - unsafe fn get_unchecked(&mut self, i: usize) -> &'a mut T { - &mut *self.ptr.as_ptr().add(i) - } - fn may_have_side_effect() -> bool { - false - } -} - -trait SliceContains: Sized { - fn slice_contains(&self, x: &[Self]) -> bool; -} - -impl SliceContains for T -where - T: PartialEq, -{ - default fn slice_contains(&self, x: &[Self]) -> bool { - x.iter().any(|y| *y == *self) - } -} - -impl SliceContains for u8 { - fn slice_contains(&self, x: &[Self]) -> bool { - memchr::memchr(*self, x).is_some() - } -} - -impl SliceContains for i8 { - fn slice_contains(&self, x: &[Self]) -> bool { - let byte = *self as u8; - let bytes: &[u8] = unsafe { from_raw_parts(x.as_ptr() as *const u8, x.len()) }; - memchr::memchr(byte, bytes).is_some() + fmt::Display::fmt("an index is out of bounds or appeared multiple times in the array", f) } } diff --git a/crux-mir/lib/core/src/slice/raw.rs b/crux-mir/lib/core/src/slice/raw.rs new file mode 100644 index 000000000..052fd34d0 --- /dev/null +++ b/crux-mir/lib/core/src/slice/raw.rs @@ -0,0 +1,297 @@ +//! Free functions to create `&[T]` and `&mut [T]`. + +use crate::array; +use crate::intrinsics::{ + assert_unsafe_precondition, is_aligned_and_not_null, is_valid_allocation_size, +}; +use crate::ops::Range; +use crate::ptr; + +/// Forms a slice from a pointer and a length. +/// +/// The `len` argument is the number of **elements**, not the number of bytes. +/// +/// # Safety +/// +/// Behavior is undefined if any of the following conditions are violated: +/// +/// * `data` must be [valid] for reads for `len * mem::size_of::()` many bytes, +/// and it must be properly aligned. This means in particular: +/// +/// * The entire memory range of this slice must be contained within a single allocated object! +/// Slices can never span across multiple allocated objects. See [below](#incorrect-usage) +/// for an example incorrectly not taking this into account. +/// * `data` must be non-null and aligned even for zero-length slices. One +/// reason for this is that enum layout optimizations may rely on references +/// (including slices of any length) being aligned and non-null to distinguish +/// them from other data. You can obtain a pointer that is usable as `data` +/// for zero-length slices using [`NonNull::dangling()`]. +/// +/// * `data` must point to `len` consecutive properly initialized values of type `T`. +/// +/// * The memory referenced by the returned slice must not be mutated for the duration +/// of lifetime `'a`, except inside an `UnsafeCell`. +/// +/// * The total size `len * mem::size_of::()` of the slice must be no larger than `isize::MAX`. +/// See the safety documentation of [`pointer::offset`]. +/// +/// # Caveat +/// +/// The lifetime for the returned slice is inferred from its usage. To +/// prevent accidental misuse, it's suggested to tie the lifetime to whichever +/// source lifetime is safe in the context, such as by providing a helper +/// function taking the lifetime of a host value for the slice, or by explicit +/// annotation. +/// +/// # Examples +/// +/// ``` +/// use std::slice; +/// +/// // manifest a slice for a single element +/// let x = 42; +/// let ptr = &x as *const _; +/// let slice = unsafe { slice::from_raw_parts(ptr, 1) }; +/// assert_eq!(slice[0], 42); +/// ``` +/// +/// ### Incorrect usage +/// +/// The following `join_slices` function is **unsound** ⚠️ +/// +/// ```rust,no_run +/// use std::slice; +/// +/// fn join_slices<'a, T>(fst: &'a [T], snd: &'a [T]) -> &'a [T] { +/// let fst_end = fst.as_ptr().wrapping_add(fst.len()); +/// let snd_start = snd.as_ptr(); +/// assert_eq!(fst_end, snd_start, "Slices must be contiguous!"); +/// unsafe { +/// // The assertion above ensures `fst` and `snd` are contiguous, but they might +/// // still be contained within _different allocated objects_, in which case +/// // creating this slice is undefined behavior. +/// slice::from_raw_parts(fst.as_ptr(), fst.len() + snd.len()) +/// } +/// } +/// +/// fn main() { +/// // `a` and `b` are different allocated objects... +/// let a = 42; +/// let b = 27; +/// // ... which may nevertheless be laid out contiguously in memory: | a | b | +/// let _ = join_slices(slice::from_ref(&a), slice::from_ref(&b)); // UB +/// } +/// ``` +/// +/// [valid]: ptr#safety +/// [`NonNull::dangling()`]: ptr::NonNull::dangling +#[inline] +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_const_stable(feature = "const_slice_from_raw_parts", since = "1.64.0")] +#[must_use] +pub const unsafe fn from_raw_parts<'a, T>(data: *const T, len: usize) -> &'a [T] { + // SAFETY: the caller must uphold the safety contract for `from_raw_parts`. + unsafe { + assert_unsafe_precondition!( + "slice::from_raw_parts requires the pointer to be aligned and non-null, and the total size of the slice not to exceed `isize::MAX`", + [T](data: *const T, len: usize) => is_aligned_and_not_null(data) + && is_valid_allocation_size::(len) + ); + &*ptr::slice_from_raw_parts(data, len) + } +} + +/// Performs the same functionality as [`from_raw_parts`], except that a +/// mutable slice is returned. +/// +/// # Safety +/// +/// Behavior is undefined if any of the following conditions are violated: +/// +/// * `data` must be [valid] for both reads and writes for `len * mem::size_of::()` many bytes, +/// and it must be properly aligned. This means in particular: +/// +/// * The entire memory range of this slice must be contained within a single allocated object! +/// Slices can never span across multiple allocated objects. +/// * `data` must be non-null and aligned even for zero-length slices. One +/// reason for this is that enum layout optimizations may rely on references +/// (including slices of any length) being aligned and non-null to distinguish +/// them from other data. You can obtain a pointer that is usable as `data` +/// for zero-length slices using [`NonNull::dangling()`]. +/// +/// * `data` must point to `len` consecutive properly initialized values of type `T`. +/// +/// * The memory referenced by the returned slice must not be accessed through any other pointer +/// (not derived from the return value) for the duration of lifetime `'a`. +/// Both read and write accesses are forbidden. +/// +/// * The total size `len * mem::size_of::()` of the slice must be no larger than `isize::MAX`. +/// See the safety documentation of [`pointer::offset`]. +/// +/// [valid]: ptr#safety +/// [`NonNull::dangling()`]: ptr::NonNull::dangling +#[inline] +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_const_unstable(feature = "const_slice_from_raw_parts_mut", issue = "67456")] +#[must_use] +pub const unsafe fn from_raw_parts_mut<'a, T>(data: *mut T, len: usize) -> &'a mut [T] { + // SAFETY: the caller must uphold the safety contract for `from_raw_parts_mut`. + unsafe { + assert_unsafe_precondition!( + "slice::from_raw_parts_mut requires the pointer to be aligned and non-null, and the total size of the slice not to exceed `isize::MAX`", + [T](data: *mut T, len: usize) => is_aligned_and_not_null(data) + && is_valid_allocation_size::(len) + ); + &mut *ptr::slice_from_raw_parts_mut(data, len) + } +} + +/// Converts a reference to T into a slice of length 1 (without copying). +#[stable(feature = "from_ref", since = "1.28.0")] +#[rustc_const_stable(feature = "const_slice_from_ref_shared", since = "1.63.0")] +#[must_use] +pub const fn from_ref(s: &T) -> &[T] { + array::from_ref(s) +} + +/// Converts a reference to T into a slice of length 1 (without copying). +#[stable(feature = "from_ref", since = "1.28.0")] +#[rustc_const_unstable(feature = "const_slice_from_ref", issue = "90206")] +#[must_use] +pub const fn from_mut(s: &mut T) -> &mut [T] { + array::from_mut(s) +} + +/// Forms a slice from a pointer range. +/// +/// This function is useful for interacting with foreign interfaces which +/// use two pointers to refer to a range of elements in memory, as is +/// common in C++. +/// +/// # Safety +/// +/// Behavior is undefined if any of the following conditions are violated: +/// +/// * The `start` pointer of the range must be a [valid] and properly aligned pointer +/// to the first element of a slice. +/// +/// * The `end` pointer must be a [valid] and properly aligned pointer to *one past* +/// the last element, such that the offset from the end to the start pointer is +/// the length of the slice. +/// +/// * The range must contain `N` consecutive properly initialized values of type `T`: +/// +/// * The entire memory range of this slice must be contained within a single allocated object! +/// Slices can never span across multiple allocated objects. +/// +/// * The memory referenced by the returned slice must not be mutated for the duration +/// of lifetime `'a`, except inside an `UnsafeCell`. +/// +/// * The total length of the range must be no larger than `isize::MAX`. +/// See the safety documentation of [`pointer::offset`]. +/// +/// Note that a range created from [`slice::as_ptr_range`] fulfills these requirements. +/// +/// # Panics +/// +/// This function panics if `T` is a Zero-Sized Type (“ZST”). +/// +/// # Caveat +/// +/// The lifetime for the returned slice is inferred from its usage. To +/// prevent accidental misuse, it's suggested to tie the lifetime to whichever +/// source lifetime is safe in the context, such as by providing a helper +/// function taking the lifetime of a host value for the slice, or by explicit +/// annotation. +/// +/// # Examples +/// +/// ``` +/// #![feature(slice_from_ptr_range)] +/// +/// use core::slice; +/// +/// let x = [1, 2, 3]; +/// let range = x.as_ptr_range(); +/// +/// unsafe { +/// assert_eq!(slice::from_ptr_range(range), &x); +/// } +/// ``` +/// +/// [valid]: ptr#safety +#[unstable(feature = "slice_from_ptr_range", issue = "89792")] +#[rustc_const_unstable(feature = "const_slice_from_ptr_range", issue = "89792")] +pub const unsafe fn from_ptr_range<'a, T>(range: Range<*const T>) -> &'a [T] { + // SAFETY: the caller must uphold the safety contract for `from_ptr_range`. + unsafe { from_raw_parts(range.start, range.end.sub_ptr(range.start)) } +} + +/// Forms a mutable slice from a pointer range. +/// +/// This is the same functionality as [`from_ptr_range`], except that a +/// mutable slice is returned. +/// +/// This function is useful for interacting with foreign interfaces which +/// use two pointers to refer to a range of elements in memory, as is +/// common in C++. +/// +/// # Safety +/// +/// Behavior is undefined if any of the following conditions are violated: +/// +/// * The `start` pointer of the range must be a [valid] and properly aligned pointer +/// to the first element of a slice. +/// +/// * The `end` pointer must be a [valid] and properly aligned pointer to *one past* +/// the last element, such that the offset from the end to the start pointer is +/// the length of the slice. +/// +/// * The range must contain `N` consecutive properly initialized values of type `T`: +/// +/// * The entire memory range of this slice must be contained within a single allocated object! +/// Slices can never span across multiple allocated objects. +/// +/// * The memory referenced by the returned slice must not be accessed through any other pointer +/// (not derived from the return value) for the duration of lifetime `'a`. +/// Both read and write accesses are forbidden. +/// +/// * The total length of the range must be no larger than `isize::MAX`. +/// See the safety documentation of [`pointer::offset`]. +/// +/// Note that a range created from [`slice::as_mut_ptr_range`] fulfills these requirements. +/// +/// # Panics +/// +/// This function panics if `T` is a Zero-Sized Type (“ZST”). +/// +/// # Caveat +/// +/// The lifetime for the returned slice is inferred from its usage. To +/// prevent accidental misuse, it's suggested to tie the lifetime to whichever +/// source lifetime is safe in the context, such as by providing a helper +/// function taking the lifetime of a host value for the slice, or by explicit +/// annotation. +/// +/// # Examples +/// +/// ``` +/// #![feature(slice_from_ptr_range)] +/// +/// use core::slice; +/// +/// let mut x = [1, 2, 3]; +/// let range = x.as_mut_ptr_range(); +/// +/// unsafe { +/// assert_eq!(slice::from_mut_ptr_range(range), &mut [1, 2, 3]); +/// } +/// ``` +/// +/// [valid]: ptr#safety +#[unstable(feature = "slice_from_ptr_range", issue = "89792")] +#[rustc_const_unstable(feature = "const_slice_from_mut_ptr_range", issue = "89792")] +pub const unsafe fn from_mut_ptr_range<'a, T>(range: Range<*mut T>) -> &'a mut [T] { + // SAFETY: the caller must uphold the safety contract for `from_mut_ptr_range`. + unsafe { from_raw_parts_mut(range.start, range.end.sub_ptr(range.start)) } +} diff --git a/crux-mir/lib/core/src/slice/rotate.rs b/crux-mir/lib/core/src/slice/rotate.rs index f73e14f27..fa8c238f8 100644 --- a/crux-mir/lib/core/src/slice/rotate.rs +++ b/crux-mir/lib/core/src/slice/rotate.rs @@ -1,5 +1,5 @@ use crate::cmp; -use crate::mem::{self, MaybeUninit}; +use crate::mem::{self, MaybeUninit, SizedTypeProperties}; use crate::ptr; /// Rotates the range `[mid-left, mid+right)` such that the element at `mid` becomes the first @@ -63,7 +63,7 @@ use crate::ptr; /// when `left < right` the swapping happens from the left instead. pub unsafe fn ptr_rotate(mut left: usize, mut mid: *mut T, mut right: usize) { type BufType = [usize; 32]; - if mem::size_of::() == 0 { + if T::IS_ZST { return; } loop { @@ -77,9 +77,11 @@ pub unsafe fn ptr_rotate(mut left: usize, mut mid: *mut T, mut right: usize) // the way until about `left + right == 32`, but the worst case performance breaks even // around 16. 24 was chosen as middle ground. If the size of `T` is larger than 4 // `usize`s, this algorithm also outperforms other algorithms. - let x = mid.sub(left); + // SAFETY: callers must ensure `mid - left` is valid for reading and writing. + let x = unsafe { mid.sub(left) }; // beginning of first round - let mut tmp: T = x.read(); + // SAFETY: see previous comment. + let mut tmp: T = unsafe { x.read() }; let mut i = right; // `gcd` can be found before hand by calculating `gcd(left + right, right)`, // but it is faster to do one loop which calculates the gcd as a side effect, then @@ -90,7 +92,22 @@ pub unsafe fn ptr_rotate(mut left: usize, mut mid: *mut T, mut right: usize) // the very end. This is possibly due to the fact that swapping or replacing temporaries // uses only one memory address in the loop instead of needing to manage two. loop { - tmp = x.add(i).replace(tmp); + // [long-safety-expl] + // SAFETY: callers must ensure `[left, left+mid+right)` are all valid for reading and + // writing. + // + // - `i` start with `right` so `mid-left <= x+i = x+right = mid-left+right < mid+right` + // - `i <= left+right-1` is always true + // - if `i < left`, `right` is added so `i < left+right` and on the next + // iteration `left` is removed from `i` so it doesn't go further + // - if `i >= left`, `left` is removed immediately and so it doesn't go further. + // - overflows cannot happen for `i` since the function's safety contract ask for + // `mid+right-1 = x+left+right` to be valid for writing + // - underflows cannot happen because `i` must be bigger or equal to `left` for + // a subtraction of `left` to happen. + // + // So `x+i` is valid for reading and writing if the caller respected the contract + tmp = unsafe { x.add(i).replace(tmp) }; // instead of incrementing `i` and then checking if it is outside the bounds, we // check if `i` will go outside the bounds on the next increment. This prevents // any wrapping of pointers or `usize`. @@ -98,7 +115,9 @@ pub unsafe fn ptr_rotate(mut left: usize, mut mid: *mut T, mut right: usize) i -= left; if i == 0 { // end of first round - x.write(tmp); + // SAFETY: tmp has been read from a valid source and x is valid for writing + // according to the caller. + unsafe { x.write(tmp) }; break; } // this conditional must be here if `left + right >= 15` @@ -111,14 +130,25 @@ pub unsafe fn ptr_rotate(mut left: usize, mut mid: *mut T, mut right: usize) } // finish the chunk with more rounds for start in 1..gcd { - tmp = x.add(start).read(); + // SAFETY: `gcd` is at most equal to `right` so all values in `1..gcd` are valid for + // reading and writing as per the function's safety contract, see [long-safety-expl] + // above + tmp = unsafe { x.add(start).read() }; + // [safety-expl-addition] + // + // Here `start < gcd` so `start < right` so `i < right+right`: `right` being the + // greatest common divisor of `(left+right, right)` means that `left = right` so + // `i < left+right` so `x+i = mid-left+i` is always valid for reading and writing + // according to the function's safety contract. i = start + right; loop { - tmp = x.add(i).replace(tmp); + // SAFETY: see [long-safety-expl] and [safety-expl-addition] + tmp = unsafe { x.add(i).replace(tmp) }; if i >= left { i -= left; if i == start { - x.add(start).write(tmp); + // SAFETY: see [long-safety-expl] and [safety-expl-addition] + unsafe { x.add(start).write(tmp) }; break; } } else { @@ -133,15 +163,35 @@ pub unsafe fn ptr_rotate(mut left: usize, mut mid: *mut T, mut right: usize) // The `[T; 0]` here is to ensure this is appropriately aligned for T let mut rawarray = MaybeUninit::<(BufType, [T; 0])>::uninit(); let buf = rawarray.as_mut_ptr() as *mut T; - let dim = mid.sub(left).add(right); + // SAFETY: `mid-left <= mid-left+right < mid+right` + let dim = unsafe { mid.sub(left).add(right) }; if left <= right { - ptr::copy_nonoverlapping(mid.sub(left), buf, left); - ptr::copy(mid, mid.sub(left), right); - ptr::copy_nonoverlapping(buf, dim, left); + // SAFETY: + // + // 1) The `else if` condition about the sizes ensures `[mid-left; left]` will fit in + // `buf` without overflow and `buf` was created just above and so cannot be + // overlapped with any value of `[mid-left; left]` + // 2) [mid-left, mid+right) are all valid for reading and writing and we don't care + // about overlaps here. + // 3) The `if` condition about `left <= right` ensures writing `left` elements to + // `dim = mid-left+right` is valid because: + // - `buf` is valid and `left` elements were written in it in 1) + // - `dim+left = mid-left+right+left = mid+right` and we write `[dim, dim+left)` + unsafe { + // 1) + ptr::copy_nonoverlapping(mid.sub(left), buf, left); + // 2) + ptr::copy(mid, mid.sub(left), right); + // 3) + ptr::copy_nonoverlapping(buf, dim, left); + } } else { - ptr::copy_nonoverlapping(mid, buf, right); - ptr::copy(mid.sub(left), dim, left); - ptr::copy_nonoverlapping(buf, mid.sub(left), right); + // SAFETY: same reasoning as above but with `left` and `right` reversed + unsafe { + ptr::copy_nonoverlapping(mid, buf, right); + ptr::copy(mid.sub(left), dim, left); + ptr::copy_nonoverlapping(buf, mid.sub(left), right); + } } return; } else if left >= right { @@ -150,8 +200,14 @@ pub unsafe fn ptr_rotate(mut left: usize, mut mid: *mut T, mut right: usize) // of this algorithm would be, and swapping using that last chunk instead of swapping // adjacent chunks like this algorithm is doing, but this way is still faster. loop { - ptr::swap_nonoverlapping(mid.sub(right), mid, right); - mid = mid.sub(right); + // SAFETY: + // `left >= right` so `[mid-right, mid+right)` is valid for reading and writing + // Subtracting `right` from `mid` each turn is counterbalanced by the addition and + // check after it. + unsafe { + ptr::swap_nonoverlapping(mid.sub(right), mid, right); + mid = mid.sub(right); + } left -= right; if left < right { break; @@ -160,8 +216,14 @@ pub unsafe fn ptr_rotate(mut left: usize, mut mid: *mut T, mut right: usize) } else { // Algorithm 3, `left < right` loop { - ptr::swap_nonoverlapping(mid.sub(left), mid, left); - mid = mid.add(left); + // SAFETY: `[mid-left, mid+left)` is valid for reading and writing because + // `left < right` so `mid+left < mid+right`. + // Adding `left` to `mid` each turn is counterbalanced by the subtraction and check + // after it. + unsafe { + ptr::swap_nonoverlapping(mid.sub(left), mid, left); + mid = mid.add(left); + } right -= left; if right < left { break; diff --git a/crux-mir/lib/core/src/slice/sort.rs b/crux-mir/lib/core/src/slice/sort.rs index 019832e16..2181f9a81 100644 --- a/crux-mir/lib/core/src/slice/sort.rs +++ b/crux-mir/lib/core/src/slice/sort.rs @@ -1,25 +1,29 @@ //! Slice sorting //! -//! This module contains an sort algorithm based on Orson Peters' pattern-defeating quicksort, -//! published at: https://github.com/orlp/pdqsort +//! This module contains a sorting algorithm based on Orson Peters' pattern-defeating quicksort, +//! published at: //! -//! Unstable sorting is compatible with libcore because it doesn't allocate memory, unlike our +//! Unstable sorting is compatible with core because it doesn't allocate memory, unlike our //! stable sorting implementation. - -// ignore-tidy-undocumented-unsafe +//! +//! In addition it also contains the core logic of the stable sort used by `slice::sort` based on +//! TimSort. use crate::cmp; -use crate::mem::{self, MaybeUninit}; +use crate::mem::{self, MaybeUninit, SizedTypeProperties}; use crate::ptr; /// When dropped, copies from `src` into `dest`. struct CopyOnDrop { - src: *mut T, + src: *const T, dest: *mut T, } impl Drop for CopyOnDrop { fn drop(&mut self) { + // SAFETY: This is a helper class. + // Please refer to its usage for correctness. + // Namely, one must be sure that `src` and `dst` does not overlap as required by `ptr::copy_nonoverlapping`. unsafe { ptr::copy_nonoverlapping(self.src, self.dest, 1); } @@ -32,24 +36,40 @@ where F: FnMut(&T, &T) -> bool, { let len = v.len(); + // SAFETY: The unsafe operations below involves indexing without a bounds check (by offsetting a + // pointer) and copying memory (`ptr::copy_nonoverlapping`). + // + // a. Indexing: + // 1. We checked the size of the array to >=2. + // 2. All the indexing that we will do is always between {0 <= index < len} at most. + // + // b. Memory copying + // 1. We are obtaining pointers to references which are guaranteed to be valid. + // 2. They cannot overlap because we obtain pointers to difference indices of the slice. + // Namely, `i` and `i-1`. + // 3. If the slice is properly aligned, the elements are properly aligned. + // It is the caller's responsibility to make sure the slice is properly aligned. + // + // See comments below for further detail. unsafe { // If the first two elements are out-of-order... if len >= 2 && is_less(v.get_unchecked(1), v.get_unchecked(0)) { // Read the first element into a stack-allocated variable. If a following comparison // operation panics, `hole` will get dropped and automatically write the element back // into the slice. - let mut tmp = mem::ManuallyDrop::new(ptr::read(v.get_unchecked(0))); - let mut hole = CopyOnDrop { src: &mut *tmp, dest: v.get_unchecked_mut(1) }; - ptr::copy_nonoverlapping(v.get_unchecked(1), v.get_unchecked_mut(0), 1); + let tmp = mem::ManuallyDrop::new(ptr::read(v.get_unchecked(0))); + let v = v.as_mut_ptr(); + let mut hole = CopyOnDrop { src: &*tmp, dest: v.add(1) }; + ptr::copy_nonoverlapping(v.add(1), v.add(0), 1); for i in 2..len { - if !is_less(v.get_unchecked(i), &*tmp) { + if !is_less(&*v.add(i), &*tmp) { break; } // Move `i`-th element one place to the left, thus shifting the hole to the right. - ptr::copy_nonoverlapping(v.get_unchecked(i), v.get_unchecked_mut(i - 1), 1); - hole.dest = v.get_unchecked_mut(i); + ptr::copy_nonoverlapping(v.add(i), v.add(i - 1), 1); + hole.dest = v.add(i); } // `hole` gets dropped and thus copies `tmp` into the remaining hole in `v`. } @@ -62,24 +82,40 @@ where F: FnMut(&T, &T) -> bool, { let len = v.len(); + // SAFETY: The unsafe operations below involves indexing without a bound check (by offsetting a + // pointer) and copying memory (`ptr::copy_nonoverlapping`). + // + // a. Indexing: + // 1. We checked the size of the array to >= 2. + // 2. All the indexing that we will do is always between `0 <= index < len-1` at most. + // + // b. Memory copying + // 1. We are obtaining pointers to references which are guaranteed to be valid. + // 2. They cannot overlap because we obtain pointers to difference indices of the slice. + // Namely, `i` and `i+1`. + // 3. If the slice is properly aligned, the elements are properly aligned. + // It is the caller's responsibility to make sure the slice is properly aligned. + // + // See comments below for further detail. unsafe { // If the last two elements are out-of-order... if len >= 2 && is_less(v.get_unchecked(len - 1), v.get_unchecked(len - 2)) { // Read the last element into a stack-allocated variable. If a following comparison // operation panics, `hole` will get dropped and automatically write the element back // into the slice. - let mut tmp = mem::ManuallyDrop::new(ptr::read(v.get_unchecked(len - 1))); - let mut hole = CopyOnDrop { src: &mut *tmp, dest: v.get_unchecked_mut(len - 2) }; - ptr::copy_nonoverlapping(v.get_unchecked(len - 2), v.get_unchecked_mut(len - 1), 1); + let tmp = mem::ManuallyDrop::new(ptr::read(v.get_unchecked(len - 1))); + let v = v.as_mut_ptr(); + let mut hole = CopyOnDrop { src: &*tmp, dest: v.add(len - 2) }; + ptr::copy_nonoverlapping(v.add(len - 2), v.add(len - 1), 1); for i in (0..len - 2).rev() { - if !is_less(&*tmp, v.get_unchecked(i)) { + if !is_less(&*tmp, &*v.add(i)) { break; } // Move `i`-th element one place to the right, thus shifting the hole to the left. - ptr::copy_nonoverlapping(v.get_unchecked(i), v.get_unchecked_mut(i + 1), 1); - hole.dest = v.get_unchecked_mut(i); + ptr::copy_nonoverlapping(v.add(i), v.add(i + 1), 1); + hole.dest = v.add(i); } // `hole` gets dropped and thus copies `tmp` into the remaining hole in `v`. } @@ -88,7 +124,7 @@ where /// Partially sorts a slice by shifting several out-of-order elements around. /// -/// Returns `true` if the slice is sorted at the end. This function is `O(n)` worst-case. +/// Returns `true` if the slice is sorted at the end. This function is *O*(*n*) worst-case. #[cold] fn partial_insertion_sort(v: &mut [T], is_less: &mut F) -> bool where @@ -103,6 +139,8 @@ where let mut i = 1; for _ in 0..MAX_STEPS { + // SAFETY: We already explicitly did the bound checking with `i < len`. + // All our subsequent indexing is only in the range `0 <= index < len` unsafe { // Find the next pair of adjacent out-of-order elements. while i < len && !is_less(v.get_unchecked(i), v.get_unchecked(i - 1)) { @@ -133,7 +171,7 @@ where false } -/// Sorts a slice using insertion sort, which is `O(n^2)` worst-case. +/// Sorts a slice using insertion sort, which is *O*(*n*^2) worst-case. fn insertion_sort(v: &mut [T], is_less: &mut F) where F: FnMut(&T, &T) -> bool, @@ -143,31 +181,35 @@ where } } -/// Sorts `v` using heapsort, which guarantees `O(n log n)` worst-case. +/// Sorts `v` using heapsort, which guarantees *O*(*n* \* log(*n*)) worst-case. #[cold] -pub fn heapsort(v: &mut [T], is_less: &mut F) +#[unstable(feature = "sort_internals", reason = "internal to sort module", issue = "none")] +pub fn heapsort(v: &mut [T], mut is_less: F) where F: FnMut(&T, &T) -> bool, { // This binary heap respects the invariant `parent >= child`. let mut sift_down = |v: &mut [T], mut node| { loop { - // Children of `node`: - let left = 2 * node + 1; - let right = 2 * node + 2; + // Children of `node`. + let mut child = 2 * node + 1; + if child >= v.len() { + break; + } // Choose the greater child. - let greater = - if right < v.len() && is_less(&v[left], &v[right]) { right } else { left }; + if child + 1 < v.len() && is_less(&v[child], &v[child + 1]) { + child += 1; + } // Stop if the invariant holds at `node`. - if greater >= v.len() || !is_less(&v[node], &v[greater]) { + if !is_less(&v[node], &v[child]) { break; } // Swap `node` with the greater child, move one step down, and continue sifting. - v.swap(node, greater); - node = greater; + v.swap(node, child); + node = child; } }; @@ -191,7 +233,7 @@ where /// Partitioning is performed block-by-block in order to minimize the cost of branching operations. /// This idea is presented in the [BlockQuicksort][pdf] paper. /// -/// [pdf]: http://drops.dagstuhl.de/opus/volltexte/2016/6389/pdf/LIPIcs-ESA-2016-38.pdf +/// [pdf]: https://drops.dagstuhl.de/opus/volltexte/2016/6389/pdf/LIPIcs-ESA-2016-38.pdf fn partition_in_blocks(v: &mut [T], pivot: &T, is_less: &mut F) -> usize where F: FnMut(&T, &T) -> bool, @@ -220,6 +262,7 @@ where let mut offsets_l = [MaybeUninit::::uninit(); BLOCK]; // The current block on the right side (from `r.sub(block_r)` to `r`). + // SAFETY: The documentation for .add() specifically mention that `vec.as_ptr().add(vec.len())` is always safe` let mut r = unsafe { l.add(v.len()) }; let mut block_r = BLOCK; let mut start_r = ptr::null_mut(); @@ -232,7 +275,9 @@ where // Returns the number of elements between pointers `l` (inclusive) and `r` (exclusive). fn width(l: *mut T, r: *mut T) -> usize { assert!(mem::size_of::() > 0); - (r as usize - l as usize) / mem::size_of::() + // FIXME: this should *likely* use `offset_from`, but more + // investigation is needed (including running tests in miri). + (r.addr() - l.addr()) / mem::size_of::() } loop { @@ -254,6 +299,9 @@ where } else if start_r < end_r { block_l = rem; } else { + // There were the same number of elements to switch on both blocks during the last + // iteration, so there are no remaining elements on either block. Cover the remaining + // items with roughly equally-sized blocks. block_l = rem / 2; block_r = rem - block_l; } @@ -263,32 +311,53 @@ where if start_l == end_l { // Trace `block_l` elements from the left side. - start_l = MaybeUninit::first_ptr_mut(&mut offsets_l); - end_l = MaybeUninit::first_ptr_mut(&mut offsets_l); + start_l = MaybeUninit::slice_as_mut_ptr(&mut offsets_l); + end_l = start_l; let mut elem = l; for i in 0..block_l { + // SAFETY: The unsafety operations below involve the usage of the `offset`. + // According to the conditions required by the function, we satisfy them because: + // 1. `offsets_l` is stack-allocated, and thus considered separate allocated object. + // 2. The function `is_less` returns a `bool`. + // Casting a `bool` will never overflow `isize`. + // 3. We have guaranteed that `block_l` will be `<= BLOCK`. + // Plus, `end_l` was initially set to the begin pointer of `offsets_` which was declared on the stack. + // Thus, we know that even in the worst case (all invocations of `is_less` returns false) we will only be at most 1 byte pass the end. + // Another unsafety operation here is dereferencing `elem`. + // However, `elem` was initially the begin pointer to the slice which is always valid. unsafe { // Branchless comparison. *end_l = i as u8; - end_l = end_l.offset(!is_less(&*elem, pivot) as isize); - elem = elem.offset(1); + end_l = end_l.add(!is_less(&*elem, pivot) as usize); + elem = elem.add(1); } } } if start_r == end_r { // Trace `block_r` elements from the right side. - start_r = MaybeUninit::first_ptr_mut(&mut offsets_r); - end_r = MaybeUninit::first_ptr_mut(&mut offsets_r); + start_r = MaybeUninit::slice_as_mut_ptr(&mut offsets_r); + end_r = start_r; let mut elem = r; for i in 0..block_r { + // SAFETY: The unsafety operations below involve the usage of the `offset`. + // According to the conditions required by the function, we satisfy them because: + // 1. `offsets_r` is stack-allocated, and thus considered separate allocated object. + // 2. The function `is_less` returns a `bool`. + // Casting a `bool` will never overflow `isize`. + // 3. We have guaranteed that `block_r` will be `<= BLOCK`. + // Plus, `end_r` was initially set to the begin pointer of `offsets_` which was declared on the stack. + // Thus, we know that even in the worst case (all invocations of `is_less` returns true) we will only be at most 1 byte pass the end. + // Another unsafety operation here is dereferencing `elem`. + // However, `elem` was initially `1 * sizeof(T)` past the end and we decrement it by `1 * sizeof(T)` before accessing it. + // Plus, `block_r` was asserted to be less than `BLOCK` and `elem` will therefore at most be pointing to the beginning of the slice. unsafe { // Branchless comparison. - elem = elem.offset(-1); + elem = elem.sub(1); *end_r = i as u8; - end_r = end_r.offset(is_less(&*elem, pivot) as isize); + end_r = end_r.add(is_less(&*elem, pivot) as usize); } } } @@ -299,44 +368,70 @@ where if count > 0 { macro_rules! left { () => { - l.offset(*start_l as isize) + l.add(usize::from(*start_l)) }; } macro_rules! right { () => { - r.offset(-(*start_r as isize) - 1) + r.sub(usize::from(*start_r) + 1) }; } // Instead of swapping one pair at the time, it is more efficient to perform a cyclic // permutation. This is not strictly equivalent to swapping, but produces a similar // result using fewer memory operations. + + // SAFETY: The use of `ptr::read` is valid because there is at least one element in + // both `offsets_l` and `offsets_r`, so `left!` is a valid pointer to read from. + // + // The uses of `left!` involve calls to `offset` on `l`, which points to the + // beginning of `v`. All the offsets pointed-to by `start_l` are at most `block_l`, so + // these `offset` calls are safe as all reads are within the block. The same argument + // applies for the uses of `right!`. + // + // The calls to `start_l.offset` are valid because there are at most `count-1` of them, + // plus the final one at the end of the unsafe block, where `count` is the minimum number + // of collected offsets in `offsets_l` and `offsets_r`, so there is no risk of there not + // being enough elements. The same reasoning applies to the calls to `start_r.offset`. + // + // The calls to `copy_nonoverlapping` are safe because `left!` and `right!` are guaranteed + // not to overlap, and are valid because of the reasoning above. unsafe { let tmp = ptr::read(left!()); ptr::copy_nonoverlapping(right!(), left!(), 1); for _ in 1..count { - start_l = start_l.offset(1); + start_l = start_l.add(1); ptr::copy_nonoverlapping(left!(), right!(), 1); - start_r = start_r.offset(1); + start_r = start_r.add(1); ptr::copy_nonoverlapping(right!(), left!(), 1); } ptr::copy_nonoverlapping(&tmp, right!(), 1); mem::forget(tmp); - start_l = start_l.offset(1); - start_r = start_r.offset(1); + start_l = start_l.add(1); + start_r = start_r.add(1); } } if start_l == end_l { // All out-of-order elements in the left block were moved. Move to the next block. - l = unsafe { l.offset(block_l as isize) }; + + // block-width-guarantee + // SAFETY: if `!is_done` then the slice width is guaranteed to be at least `2*BLOCK` wide. There + // are at most `BLOCK` elements in `offsets_l` because of its size, so the `offset` operation is + // safe. Otherwise, the debug assertions in the `is_done` case guarantee that + // `width(l, r) == block_l + block_r`, namely, that the block sizes have been adjusted to account + // for the smaller number of remaining elements. + l = unsafe { l.add(block_l) }; } if start_r == end_r { // All out-of-order elements in the right block were moved. Move to the previous block. - r = unsafe { r.offset(-(block_r as isize)) }; + + // SAFETY: Same argument as [block-width-guarantee]. Either this is a full block `2*BLOCK`-wide, + // or `block_r` has been adjusted for the last handful of elements. + r = unsafe { r.sub(block_r) }; } if is_done { @@ -353,10 +448,21 @@ where // Move its remaining out-of-order elements to the far right. debug_assert_eq!(width(l, r), block_l); while start_l < end_l { + // remaining-elements-safety + // SAFETY: while the loop condition holds there are still elements in `offsets_l`, so it + // is safe to point `end_l` to the previous element. + // + // The `ptr::swap` is safe if both its arguments are valid for reads and writes: + // - Per the debug assert above, the distance between `l` and `r` is `block_l` + // elements, so there can be at most `block_l` remaining offsets between `start_l` + // and `end_l`. This means `r` will be moved at most `block_l` steps back, which + // makes the `r.offset` calls valid (at that point `l == r`). + // - `offsets_l` contains valid offsets into `v` collected during the partitioning of + // the last block, so the `l.offset` calls are valid. unsafe { - end_l = end_l.offset(-1); - ptr::swap(l.offset(*end_l as isize), r.offset(-1)); - r = r.offset(-1); + end_l = end_l.sub(1); + ptr::swap(l.add(usize::from(*end_l)), r.sub(1)); + r = r.sub(1); } } width(v.as_mut_ptr(), r) @@ -365,10 +471,11 @@ where // Move its remaining out-of-order elements to the far left. debug_assert_eq!(width(l, r), block_r); while start_r < end_r { + // SAFETY: See the reasoning in [remaining-elements-safety]. unsafe { - end_r = end_r.offset(-1); - ptr::swap(l, r.offset(-(*end_r as isize) - 1)); - l = l.offset(1); + end_r = end_r.sub(1); + ptr::swap(l, r.sub(usize::from(*end_r) + 1)); + l = l.add(1); } } width(v.as_mut_ptr(), l) @@ -397,15 +504,22 @@ where // Read the pivot into a stack-allocated variable for efficiency. If a following comparison // operation panics, the pivot will be automatically written back into the slice. - let mut tmp = mem::ManuallyDrop::new(unsafe { ptr::read(pivot) }); - let _pivot_guard = CopyOnDrop { src: &mut *tmp, dest: pivot }; + + // SAFETY: `pivot` is a reference to the first element of `v`, so `ptr::read` is safe. + let tmp = mem::ManuallyDrop::new(unsafe { ptr::read(pivot) }); + let _pivot_guard = CopyOnDrop { src: &*tmp, dest: pivot }; let pivot = &*tmp; // Find the first pair of out-of-order elements. let mut l = 0; let mut r = v.len(); + + // SAFETY: The unsafety below involves indexing an array. + // For the first one: We already do the bounds checking here with `l < r`. + // For the second one: We initially have `l == 0` and `r == v.len()` and we checked that `l < r` at every indexing operation. + // From here we know that `r` must be at least `r == l` which was shown to be valid from the first one. unsafe { - // Find the first element greater then or equal to the pivot. + // Find the first element greater than or equal to the pivot. while l < r && is_less(v.get_unchecked(l), pivot) { l += 1; } @@ -444,16 +558,21 @@ where // Read the pivot into a stack-allocated variable for efficiency. If a following comparison // operation panics, the pivot will be automatically written back into the slice. - let mut tmp = mem::ManuallyDrop::new(unsafe { ptr::read(pivot) }); - let _pivot_guard = CopyOnDrop { src: &mut *tmp, dest: pivot }; + // SAFETY: The pointer here is valid because it is obtained from a reference to a slice. + let tmp = mem::ManuallyDrop::new(unsafe { ptr::read(pivot) }); + let _pivot_guard = CopyOnDrop { src: &*tmp, dest: pivot }; let pivot = &*tmp; // Now partition the slice. let mut l = 0; let mut r = v.len(); loop { + // SAFETY: The unsafety below involves indexing an array. + // For the first one: We already do the bounds checking here with `l < r`. + // For the second one: We initially have `l == 0` and `r == v.len()` and we checked that `l < r` at every indexing operation. + // From here we know that `r` must be at least `r == l` which was shown to be valid from the first one. unsafe { - // Find the first element greater that the pivot. + // Find the first element greater than the pivot. while l < r && !is_less(pivot, v.get_unchecked(l)) { l += 1; } @@ -470,7 +589,8 @@ where // Swap the found pair of out-of-order elements. r -= 1; - ptr::swap(v.get_unchecked_mut(l), v.get_unchecked_mut(r)); + let ptr = v.as_mut_ptr(); + ptr::swap(ptr.add(l), ptr.add(r)); l += 1; } } @@ -497,7 +617,7 @@ fn break_patterns(v: &mut [T]) { random }; let mut gen_usize = || { - if mem::size_of::() <= 4 { + if usize::BITS <= 32 { gen_u32() as usize } else { (((gen_u32() as u64) << 32) | (gen_u32() as u64)) as usize @@ -552,6 +672,12 @@ where if len >= 8 { // Swaps indices so that `v[a] <= v[b]`. + // SAFETY: `len >= 8` so there are at least two elements in the neighborhoods of + // `a`, `b` and `c`. This means the three calls to `sort_adjacent` result in + // corresponding calls to `sort3` with valid 3-item neighborhoods around each + // pointer, which in turn means the calls to `sort2` are done with valid + // references. Thus the `v.get_unchecked` calls are safe, as is the `ptr::swap` + // call. let mut sort2 = |a: &mut usize, b: &mut usize| unsafe { if is_less(v.get_unchecked(*b), v.get_unchecked(*a)) { ptr::swap(a, b); @@ -599,7 +725,7 @@ where /// /// `limit` is the number of allowed imbalanced partitions before switching to `heapsort`. If zero, /// this function will immediately switch to heapsort. -fn recurse<'a, T, F>(mut v: &'a mut [T], is_less: &mut F, mut pred: Option<&'a T>, mut limit: usize) +fn recurse<'a, T, F>(mut v: &'a mut [T], is_less: &mut F, mut pred: Option<&'a T>, mut limit: u32) where F: FnMut(&T, &T) -> bool, { @@ -621,7 +747,7 @@ where } // If too many bad pivot choices were made, simply fall back to heapsort in order to - // guarantee `O(n log n)` worst-case. + // guarantee `O(n * log(n))` worst-case. if limit == 0 { heapsort(v, is_less); return; @@ -655,7 +781,7 @@ where let mid = partition_equal(v, pivot, is_less); // Continue sorting elements greater than the pivot. - v = &mut { v }[mid..]; + v = &mut v[mid..]; continue; } } @@ -666,7 +792,7 @@ where was_partitioned = was_p; // Split the slice into `left`, `pivot`, and `right`. - let (left, right) = { v }.split_at_mut(mid); + let (left, right) = v.split_at_mut(mid); let (pivot, right) = right.split_at_mut(1); let pivot = &pivot[0]; @@ -684,18 +810,18 @@ where } } -/// Sorts `v` using pattern-defeating quicksort, which is `O(n log n)` worst-case. +/// Sorts `v` using pattern-defeating quicksort, which is *O*(*n* \* log(*n*)) worst-case. pub fn quicksort(v: &mut [T], mut is_less: F) where F: FnMut(&T, &T) -> bool, { // Sorting has no meaningful behavior on zero-sized types. - if mem::size_of::() == 0 { + if T::IS_ZST { return; } // Limit the number of imbalanced partitions to `floor(log2(len)) + 1`. - let limit = mem::size_of::() * 8 - v.len().leading_zeros() as usize; + let limit = usize::BITS - v.len().leading_zeros(); recurse(v, &mut is_less, None, limit); } @@ -708,6 +834,15 @@ fn partition_at_index_loop<'a, T, F>( ) where F: FnMut(&T, &T) -> bool, { + // Limit the amount of iterations and fall back to heapsort, similarly to `slice::sort_unstable`. + // This lowers the worst case running time from O(n^2) to O(n log n). + // FIXME: Investigate whether it would be better to use something like Median of Medians + // or Fast Deterministic Selection to guarantee O(n) worst case. + let mut limit = usize::BITS - v.len().leading_zeros(); + + // True if the last partitioning was reasonably balanced. + let mut was_balanced = true; + loop { // For slices of up to this length it's probably faster to simply sort them. const MAX_INSERTION: usize = 10; @@ -716,6 +851,18 @@ fn partition_at_index_loop<'a, T, F>( return; } + if limit == 0 { + heapsort(v, is_less); + return; + } + + // If the last partitioning was imbalanced, try breaking patterns in the slice by shuffling + // some elements around. Hopefully we'll choose a better pivot this time. + if !was_balanced { + break_patterns(v); + limit -= 1; + } + // Choose a pivot let (pivot, _) = choose_pivot(v, is_less); @@ -740,9 +887,10 @@ fn partition_at_index_loop<'a, T, F>( } let (mid, _) = partition(v, pivot, is_less); + was_balanced = cmp::min(mid, v.len() - mid) >= v.len() / 8; // Split the slice into `left`, `pivot`, and `right`. - let (left, right) = { v }.split_at_mut(mid); + let (left, right) = v.split_at_mut(mid); let (pivot, right) = right.split_at_mut(1); let pivot = &pivot[0]; @@ -760,6 +908,7 @@ fn partition_at_index_loop<'a, T, F>( } } +/// Reorder the slice such that the element at `index` is at its final sorted position. pub fn partition_at_index( v: &mut [T], index: usize, @@ -775,7 +924,7 @@ where panic!("partition_at_index index {} greater than length of slice {}", index, v.len()); } - if mem::size_of::() == 0 { + if T::IS_ZST { // Sorting has no meaningful behavior on zero-sized types. Do nothing. } else if index == v.len() - 1 { // Find max element and place it in the last position of the array. We're free to use @@ -804,3 +953,513 @@ where let pivot = &mut pivot[0]; (left, pivot, right) } + +/// Inserts `v[0]` into pre-sorted sequence `v[1..]` so that whole `v[..]` becomes sorted. +/// +/// This is the integral subroutine of insertion sort. +fn insert_head(v: &mut [T], is_less: &mut F) +where + F: FnMut(&T, &T) -> bool, +{ + if v.len() >= 2 && is_less(&v[1], &v[0]) { + // SAFETY: Copy tmp back even if panic, and ensure unique observation. + unsafe { + // There are three ways to implement insertion here: + // + // 1. Swap adjacent elements until the first one gets to its final destination. + // However, this way we copy data around more than is necessary. If elements are big + // structures (costly to copy), this method will be slow. + // + // 2. Iterate until the right place for the first element is found. Then shift the + // elements succeeding it to make room for it and finally place it into the + // remaining hole. This is a good method. + // + // 3. Copy the first element into a temporary variable. Iterate until the right place + // for it is found. As we go along, copy every traversed element into the slot + // preceding it. Finally, copy data from the temporary variable into the remaining + // hole. This method is very good. Benchmarks demonstrated slightly better + // performance than with the 2nd method. + // + // All methods were benchmarked, and the 3rd showed best results. So we chose that one. + let tmp = mem::ManuallyDrop::new(ptr::read(&v[0])); + + // Intermediate state of the insertion process is always tracked by `hole`, which + // serves two purposes: + // 1. Protects integrity of `v` from panics in `is_less`. + // 2. Fills the remaining hole in `v` in the end. + // + // Panic safety: + // + // If `is_less` panics at any point during the process, `hole` will get dropped and + // fill the hole in `v` with `tmp`, thus ensuring that `v` still holds every object it + // initially held exactly once. + let mut hole = InsertionHole { src: &*tmp, dest: &mut v[1] }; + ptr::copy_nonoverlapping(&v[1], &mut v[0], 1); + + for i in 2..v.len() { + if !is_less(&v[i], &*tmp) { + break; + } + ptr::copy_nonoverlapping(&v[i], &mut v[i - 1], 1); + hole.dest = &mut v[i]; + } + // `hole` gets dropped and thus copies `tmp` into the remaining hole in `v`. + } + } + + // When dropped, copies from `src` into `dest`. + struct InsertionHole { + src: *const T, + dest: *mut T, + } + + impl Drop for InsertionHole { + fn drop(&mut self) { + // SAFETY: The caller must ensure that src and dest are correctly set. + unsafe { + ptr::copy_nonoverlapping(self.src, self.dest, 1); + } + } + } +} + +/// Merges non-decreasing runs `v[..mid]` and `v[mid..]` using `buf` as temporary storage, and +/// stores the result into `v[..]`. +/// +/// # Safety +/// +/// The two slices must be non-empty and `mid` must be in bounds. Buffer `buf` must be long enough +/// to hold a copy of the shorter slice. Also, `T` must not be a zero-sized type. +unsafe fn merge(v: &mut [T], mid: usize, buf: *mut T, is_less: &mut F) +where + F: FnMut(&T, &T) -> bool, +{ + let len = v.len(); + let v = v.as_mut_ptr(); + + // SAFETY: mid and len must be in-bounds of v. + let (v_mid, v_end) = unsafe { (v.add(mid), v.add(len)) }; + + // The merge process first copies the shorter run into `buf`. Then it traces the newly copied + // run and the longer run forwards (or backwards), comparing their next unconsumed elements and + // copying the lesser (or greater) one into `v`. + // + // As soon as the shorter run is fully consumed, the process is done. If the longer run gets + // consumed first, then we must copy whatever is left of the shorter run into the remaining + // hole in `v`. + // + // Intermediate state of the process is always tracked by `hole`, which serves two purposes: + // 1. Protects integrity of `v` from panics in `is_less`. + // 2. Fills the remaining hole in `v` if the longer run gets consumed first. + // + // Panic safety: + // + // If `is_less` panics at any point during the process, `hole` will get dropped and fill the + // hole in `v` with the unconsumed range in `buf`, thus ensuring that `v` still holds every + // object it initially held exactly once. + let mut hole; + + if mid <= len - mid { + // The left run is shorter. + + // SAFETY: buf must have enough capacity for `v[..mid]`. + unsafe { + ptr::copy_nonoverlapping(v, buf, mid); + hole = MergeHole { start: buf, end: buf.add(mid), dest: v }; + } + + // Initially, these pointers point to the beginnings of their arrays. + let left = &mut hole.start; + let mut right = v_mid; + let out = &mut hole.dest; + + while *left < hole.end && right < v_end { + // Consume the lesser side. + // If equal, prefer the left run to maintain stability. + + // SAFETY: left and right must be valid and part of v same for out. + unsafe { + let to_copy = if is_less(&*right, &**left) { + get_and_increment(&mut right) + } else { + get_and_increment(left) + }; + ptr::copy_nonoverlapping(to_copy, get_and_increment(out), 1); + } + } + } else { + // The right run is shorter. + + // SAFETY: buf must have enough capacity for `v[mid..]`. + unsafe { + ptr::copy_nonoverlapping(v_mid, buf, len - mid); + hole = MergeHole { start: buf, end: buf.add(len - mid), dest: v_mid }; + } + + // Initially, these pointers point past the ends of their arrays. + let left = &mut hole.dest; + let right = &mut hole.end; + let mut out = v_end; + + while v < *left && buf < *right { + // Consume the greater side. + // If equal, prefer the right run to maintain stability. + + // SAFETY: left and right must be valid and part of v same for out. + unsafe { + let to_copy = if is_less(&*right.sub(1), &*left.sub(1)) { + decrement_and_get(left) + } else { + decrement_and_get(right) + }; + ptr::copy_nonoverlapping(to_copy, decrement_and_get(&mut out), 1); + } + } + } + // Finally, `hole` gets dropped. If the shorter run was not fully consumed, whatever remains of + // it will now be copied into the hole in `v`. + + unsafe fn get_and_increment(ptr: &mut *mut T) -> *mut T { + let old = *ptr; + + // SAFETY: ptr.add(1) must still be a valid pointer and part of `v`. + *ptr = unsafe { ptr.add(1) }; + old + } + + unsafe fn decrement_and_get(ptr: &mut *mut T) -> *mut T { + // SAFETY: ptr.sub(1) must still be a valid pointer and part of `v`. + *ptr = unsafe { ptr.sub(1) }; + *ptr + } + + // When dropped, copies the range `start..end` into `dest..`. + struct MergeHole { + start: *mut T, + end: *mut T, + dest: *mut T, + } + + impl Drop for MergeHole { + fn drop(&mut self) { + // SAFETY: `T` is not a zero-sized type, and these are pointers into a slice's elements. + unsafe { + let len = self.end.sub_ptr(self.start); + ptr::copy_nonoverlapping(self.start, self.dest, len); + } + } + } +} + +/// This merge sort borrows some (but not all) ideas from TimSort, which used to be described in +/// detail [here](https://github.com/python/cpython/blob/main/Objects/listsort.txt). However Python +/// has switched to a Powersort based implementation. +/// +/// The algorithm identifies strictly descending and non-descending subsequences, which are called +/// natural runs. There is a stack of pending runs yet to be merged. Each newly found run is pushed +/// onto the stack, and then some pairs of adjacent runs are merged until these two invariants are +/// satisfied: +/// +/// 1. for every `i` in `1..runs.len()`: `runs[i - 1].len > runs[i].len` +/// 2. for every `i` in `2..runs.len()`: `runs[i - 2].len > runs[i - 1].len + runs[i].len` +/// +/// The invariants ensure that the total running time is *O*(*n* \* log(*n*)) worst-case. +pub fn merge_sort( + v: &mut [T], + is_less: &mut CmpF, + elem_alloc_fn: ElemAllocF, + elem_dealloc_fn: ElemDeallocF, + run_alloc_fn: RunAllocF, + run_dealloc_fn: RunDeallocF, +) where + CmpF: FnMut(&T, &T) -> bool, + ElemAllocF: Fn(usize) -> *mut T, + ElemDeallocF: Fn(*mut T, usize), + RunAllocF: Fn(usize) -> *mut TimSortRun, + RunDeallocF: Fn(*mut TimSortRun, usize), +{ + // Slices of up to this length get sorted using insertion sort. + const MAX_INSERTION: usize = 20; + // Very short runs are extended using insertion sort to span at least this many elements. + const MIN_RUN: usize = 10; + + // The caller should have already checked that. + debug_assert!(!T::IS_ZST); + + let len = v.len(); + + // Short arrays get sorted in-place via insertion sort to avoid allocations. + if len <= MAX_INSERTION { + if len >= 2 { + for i in (0..len - 1).rev() { + insert_head(&mut v[i..], is_less); + } + } + return; + } + + // Allocate a buffer to use as scratch memory. We keep the length 0 so we can keep in it + // shallow copies of the contents of `v` without risking the dtors running on copies if + // `is_less` panics. When merging two sorted runs, this buffer holds a copy of the shorter run, + // which will always have length at most `len / 2`. + let buf = BufGuard::new(len / 2, elem_alloc_fn, elem_dealloc_fn); + let buf_ptr = buf.buf_ptr; + + let mut runs = RunVec::new(run_alloc_fn, run_dealloc_fn); + + // In order to identify natural runs in `v`, we traverse it backwards. That might seem like a + // strange decision, but consider the fact that merges more often go in the opposite direction + // (forwards). According to benchmarks, merging forwards is slightly faster than merging + // backwards. To conclude, identifying runs by traversing backwards improves performance. + let mut end = len; + while end > 0 { + // Find the next natural run, and reverse it if it's strictly descending. + let mut start = end - 1; + if start > 0 { + start -= 1; + + // SAFETY: The v.get_unchecked must be fed with correct inbound indicies. + unsafe { + if is_less(v.get_unchecked(start + 1), v.get_unchecked(start)) { + while start > 0 && is_less(v.get_unchecked(start), v.get_unchecked(start - 1)) { + start -= 1; + } + v[start..end].reverse(); + } else { + while start > 0 && !is_less(v.get_unchecked(start), v.get_unchecked(start - 1)) + { + start -= 1; + } + } + } + } + + // Insert some more elements into the run if it's too short. Insertion sort is faster than + // merge sort on short sequences, so this significantly improves performance. + while start > 0 && end - start < MIN_RUN { + start -= 1; + insert_head(&mut v[start..end], is_less); + } + + // Push this run onto the stack. + runs.push(TimSortRun { start, len: end - start }); + end = start; + + // Merge some pairs of adjacent runs to satisfy the invariants. + while let Some(r) = collapse(runs.as_slice()) { + let left = runs[r + 1]; + let right = runs[r]; + // SAFETY: `buf_ptr` must hold enough capacity for the shorter of the two sides, and + // neither side may be on length 0. + unsafe { + merge(&mut v[left.start..right.start + right.len], left.len, buf_ptr, is_less); + } + runs[r] = TimSortRun { start: left.start, len: left.len + right.len }; + runs.remove(r + 1); + } + } + + // Finally, exactly one run must remain in the stack. + debug_assert!(runs.len() == 1 && runs[0].start == 0 && runs[0].len == len); + + // Examines the stack of runs and identifies the next pair of runs to merge. More specifically, + // if `Some(r)` is returned, that means `runs[r]` and `runs[r + 1]` must be merged next. If the + // algorithm should continue building a new run instead, `None` is returned. + // + // TimSort is infamous for its buggy implementations, as described here: + // http://envisage-project.eu/timsort-specification-and-verification/ + // + // The gist of the story is: we must enforce the invariants on the top four runs on the stack. + // Enforcing them on just top three is not sufficient to ensure that the invariants will still + // hold for *all* runs in the stack. + // + // This function correctly checks invariants for the top four runs. Additionally, if the top + // run starts at index 0, it will always demand a merge operation until the stack is fully + // collapsed, in order to complete the sort. + #[inline] + fn collapse(runs: &[TimSortRun]) -> Option { + let n = runs.len(); + if n >= 2 + && (runs[n - 1].start == 0 + || runs[n - 2].len <= runs[n - 1].len + || (n >= 3 && runs[n - 3].len <= runs[n - 2].len + runs[n - 1].len) + || (n >= 4 && runs[n - 4].len <= runs[n - 3].len + runs[n - 2].len)) + { + if n >= 3 && runs[n - 3].len < runs[n - 1].len { Some(n - 3) } else { Some(n - 2) } + } else { + None + } + } + + // Extremely basic versions of Vec. + // Their use is super limited and by having the code here, it allows reuse between the sort + // implementations. + struct BufGuard + where + ElemDeallocF: Fn(*mut T, usize), + { + buf_ptr: *mut T, + capacity: usize, + elem_dealloc_fn: ElemDeallocF, + } + + impl BufGuard + where + ElemDeallocF: Fn(*mut T, usize), + { + fn new( + len: usize, + elem_alloc_fn: ElemAllocF, + elem_dealloc_fn: ElemDeallocF, + ) -> Self + where + ElemAllocF: Fn(usize) -> *mut T, + { + Self { buf_ptr: elem_alloc_fn(len), capacity: len, elem_dealloc_fn } + } + } + + impl Drop for BufGuard + where + ElemDeallocF: Fn(*mut T, usize), + { + fn drop(&mut self) { + (self.elem_dealloc_fn)(self.buf_ptr, self.capacity); + } + } + + struct RunVec + where + RunAllocF: Fn(usize) -> *mut TimSortRun, + RunDeallocF: Fn(*mut TimSortRun, usize), + { + buf_ptr: *mut TimSortRun, + capacity: usize, + len: usize, + run_alloc_fn: RunAllocF, + run_dealloc_fn: RunDeallocF, + } + + impl RunVec + where + RunAllocF: Fn(usize) -> *mut TimSortRun, + RunDeallocF: Fn(*mut TimSortRun, usize), + { + fn new(run_alloc_fn: RunAllocF, run_dealloc_fn: RunDeallocF) -> Self { + // Most slices can be sorted with at most 16 runs in-flight. + const START_RUN_CAPACITY: usize = 16; + + Self { + buf_ptr: run_alloc_fn(START_RUN_CAPACITY), + capacity: START_RUN_CAPACITY, + len: 0, + run_alloc_fn, + run_dealloc_fn, + } + } + + fn push(&mut self, val: TimSortRun) { + if self.len == self.capacity { + let old_capacity = self.capacity; + let old_buf_ptr = self.buf_ptr; + + self.capacity = self.capacity * 2; + self.buf_ptr = (self.run_alloc_fn)(self.capacity); + + // SAFETY: buf_ptr new and old were correctly allocated and old_buf_ptr has + // old_capacity valid elements. + unsafe { + ptr::copy_nonoverlapping(old_buf_ptr, self.buf_ptr, old_capacity); + } + + (self.run_dealloc_fn)(old_buf_ptr, old_capacity); + } + + // SAFETY: The invariant was just checked. + unsafe { + self.buf_ptr.add(self.len).write(val); + } + self.len += 1; + } + + fn remove(&mut self, index: usize) { + if index >= self.len { + panic!("Index out of bounds"); + } + + // SAFETY: buf_ptr needs to be valid and len invariant upheld. + unsafe { + // the place we are taking from. + let ptr = self.buf_ptr.add(index); + + // Shift everything down to fill in that spot. + ptr::copy(ptr.add(1), ptr, self.len - index - 1); + } + self.len -= 1; + } + + fn as_slice(&self) -> &[TimSortRun] { + // SAFETY: Safe as long as buf_ptr is valid and len invariant was upheld. + unsafe { &*ptr::slice_from_raw_parts(self.buf_ptr, self.len) } + } + + fn len(&self) -> usize { + self.len + } + } + + impl core::ops::Index for RunVec + where + RunAllocF: Fn(usize) -> *mut TimSortRun, + RunDeallocF: Fn(*mut TimSortRun, usize), + { + type Output = TimSortRun; + + fn index(&self, index: usize) -> &Self::Output { + if index < self.len { + // SAFETY: buf_ptr and len invariant must be upheld. + unsafe { + return &*(self.buf_ptr.add(index)); + } + } + + panic!("Index out of bounds"); + } + } + + impl core::ops::IndexMut for RunVec + where + RunAllocF: Fn(usize) -> *mut TimSortRun, + RunDeallocF: Fn(*mut TimSortRun, usize), + { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + if index < self.len { + // SAFETY: buf_ptr and len invariant must be upheld. + unsafe { + return &mut *(self.buf_ptr.add(index)); + } + } + + panic!("Index out of bounds"); + } + } + + impl Drop for RunVec + where + RunAllocF: Fn(usize) -> *mut TimSortRun, + RunDeallocF: Fn(*mut TimSortRun, usize), + { + fn drop(&mut self) { + // As long as TimSortRun is Copy we don't need to drop them individually but just the + // whole allocation. + (self.run_dealloc_fn)(self.buf_ptr, self.capacity); + } + } +} + +/// Internal type used by merge_sort. +#[derive(Clone, Copy, Debug)] +pub struct TimSortRun { + len: usize, + start: usize, +} diff --git a/crux-mir/lib/core/src/slice/specialize.rs b/crux-mir/lib/core/src/slice/specialize.rs new file mode 100644 index 000000000..80eb59058 --- /dev/null +++ b/crux-mir/lib/core/src/slice/specialize.rs @@ -0,0 +1,23 @@ +pub(super) trait SpecFill { + fn spec_fill(&mut self, value: T); +} + +impl SpecFill for [T] { + default fn spec_fill(&mut self, value: T) { + if let Some((last, elems)) = self.split_last_mut() { + for el in elems { + el.clone_from(&value); + } + + *last = value + } + } +} + +impl SpecFill for [T] { + fn spec_fill(&mut self, value: T) { + for item in self.iter_mut() { + *item = value; + } + } +} diff --git a/crux-mir/lib/core/src/str/converts.rs b/crux-mir/lib/core/src/str/converts.rs new file mode 100644 index 000000000..5f8748206 --- /dev/null +++ b/crux-mir/lib/core/src/str/converts.rs @@ -0,0 +1,203 @@ +//! Ways to create a `str` from bytes slice. + +use crate::mem; + +use super::validations::run_utf8_validation; +use super::Utf8Error; + +/// Converts a slice of bytes to a string slice. +/// +/// A string slice ([`&str`]) is made of bytes ([`u8`]), and a byte slice +/// ([`&[u8]`][byteslice]) is made of bytes, so this function converts between +/// the two. Not all byte slices are valid string slices, however: [`&str`] requires +/// that it is valid UTF-8. `from_utf8()` checks to ensure that the bytes are valid +/// UTF-8, and then does the conversion. +/// +/// [`&str`]: str +/// [byteslice]: slice +/// +/// If you are sure that the byte slice is valid UTF-8, and you don't want to +/// incur the overhead of the validity check, there is an unsafe version of +/// this function, [`from_utf8_unchecked`], which has the same +/// behavior but skips the check. +/// +/// If you need a `String` instead of a `&str`, consider +/// [`String::from_utf8`][string]. +/// +/// [string]: ../../std/string/struct.String.html#method.from_utf8 +/// +/// Because you can stack-allocate a `[u8; N]`, and you can take a +/// [`&[u8]`][byteslice] of it, this function is one way to have a +/// stack-allocated string. There is an example of this in the +/// examples section below. +/// +/// [byteslice]: slice +/// +/// # Errors +/// +/// Returns `Err` if the slice is not UTF-8 with a description as to why the +/// provided slice is not UTF-8. +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// use std::str; +/// +/// // some bytes, in a vector +/// let sparkle_heart = vec![240, 159, 146, 150]; +/// +/// // We know these bytes are valid, so just use `unwrap()`. +/// let sparkle_heart = str::from_utf8(&sparkle_heart).unwrap(); +/// +/// assert_eq!("💖", sparkle_heart); +/// ``` +/// +/// Incorrect bytes: +/// +/// ``` +/// use std::str; +/// +/// // some invalid bytes, in a vector +/// let sparkle_heart = vec![0, 159, 146, 150]; +/// +/// assert!(str::from_utf8(&sparkle_heart).is_err()); +/// ``` +/// +/// See the docs for [`Utf8Error`] for more details on the kinds of +/// errors that can be returned. +/// +/// A "stack allocated string": +/// +/// ``` +/// use std::str; +/// +/// // some bytes, in a stack-allocated array +/// let sparkle_heart = [240, 159, 146, 150]; +/// +/// // We know these bytes are valid, so just use `unwrap()`. +/// let sparkle_heart: &str = str::from_utf8(&sparkle_heart).unwrap(); +/// +/// assert_eq!("💖", sparkle_heart); +/// ``` +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_const_stable(feature = "const_str_from_utf8_shared", since = "1.63.0")] +#[rustc_allow_const_fn_unstable(str_internals)] +pub const fn from_utf8(v: &[u8]) -> Result<&str, Utf8Error> { + // FIXME: This should use `?` again, once it's `const` + match run_utf8_validation(v) { + Ok(_) => { + // SAFETY: validation succeeded. + Ok(unsafe { from_utf8_unchecked(v) }) + } + Err(err) => Err(err), + } +} + +/// Converts a mutable slice of bytes to a mutable string slice. +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// use std::str; +/// +/// // "Hello, Rust!" as a mutable vector +/// let mut hellorust = vec![72, 101, 108, 108, 111, 44, 32, 82, 117, 115, 116, 33]; +/// +/// // As we know these bytes are valid, we can use `unwrap()` +/// let outstr = str::from_utf8_mut(&mut hellorust).unwrap(); +/// +/// assert_eq!("Hello, Rust!", outstr); +/// ``` +/// +/// Incorrect bytes: +/// +/// ``` +/// use std::str; +/// +/// // Some invalid bytes in a mutable vector +/// let mut invalid = vec![128, 223]; +/// +/// assert!(str::from_utf8_mut(&mut invalid).is_err()); +/// ``` +/// See the docs for [`Utf8Error`] for more details on the kinds of +/// errors that can be returned. +#[stable(feature = "str_mut_extras", since = "1.20.0")] +#[rustc_const_unstable(feature = "const_str_from_utf8", issue = "91006")] +pub const fn from_utf8_mut(v: &mut [u8]) -> Result<&mut str, Utf8Error> { + // This should use `?` again, once it's `const` + match run_utf8_validation(v) { + Ok(_) => { + // SAFETY: validation succeeded. + Ok(unsafe { from_utf8_unchecked_mut(v) }) + } + Err(err) => Err(err), + } +} + +/// Converts a slice of bytes to a string slice without checking +/// that the string contains valid UTF-8. +/// +/// See the safe version, [`from_utf8`], for more information. +/// +/// # Safety +/// +/// The bytes passed in must be valid UTF-8. +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// use std::str; +/// +/// // some bytes, in a vector +/// let sparkle_heart = vec![240, 159, 146, 150]; +/// +/// let sparkle_heart = unsafe { +/// str::from_utf8_unchecked(&sparkle_heart) +/// }; +/// +/// assert_eq!("💖", sparkle_heart); +/// ``` +#[inline] +#[must_use] +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_const_stable(feature = "const_str_from_utf8_unchecked", since = "1.55.0")] +pub const unsafe fn from_utf8_unchecked(v: &[u8]) -> &str { + // SAFETY: the caller must guarantee that the bytes `v` are valid UTF-8. + // Also relies on `&str` and `&[u8]` having the same layout. + unsafe { mem::transmute(v) } +} + +/// Converts a slice of bytes to a string slice without checking +/// that the string contains valid UTF-8; mutable version. +/// +/// See the immutable version, [`from_utf8_unchecked()`] for more information. +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// use std::str; +/// +/// let mut heart = vec![240, 159, 146, 150]; +/// let heart = unsafe { str::from_utf8_unchecked_mut(&mut heart) }; +/// +/// assert_eq!("💖", heart); +/// ``` +#[inline] +#[must_use] +#[stable(feature = "str_mut_extras", since = "1.20.0")] +#[rustc_const_unstable(feature = "const_str_from_utf8_unchecked_mut", issue = "91005")] +pub const unsafe fn from_utf8_unchecked_mut(v: &mut [u8]) -> &mut str { + // SAFETY: the caller must guarantee that the bytes `v` + // are valid UTF-8, thus the cast to `*mut str` is safe. + // Also, the pointer dereference is safe because that pointer + // comes from a reference which is guaranteed to be valid for writes. + unsafe { &mut *(v as *mut [u8] as *mut str) } +} diff --git a/crux-mir/lib/core/src/str/count.rs b/crux-mir/lib/core/src/str/count.rs new file mode 100644 index 000000000..28567a7e7 --- /dev/null +++ b/crux-mir/lib/core/src/str/count.rs @@ -0,0 +1,136 @@ +//! Code for efficiently counting the number of `char`s in a UTF-8 encoded +//! string. +//! +//! Broadly, UTF-8 encodes `char`s as a "leading" byte which begins the `char`, +//! followed by some number (possibly 0) of continuation bytes. +//! +//! The leading byte can have a number of bit-patterns (with the specific +//! pattern indicating how many continuation bytes follow), but the continuation +//! bytes are always in the format `0b10XX_XXXX` (where the `X`s can take any +//! value). That is, the most significant bit is set, and the second most +//! significant bit is unset. +//! +//! To count the number of characters, we can just count the number of bytes in +//! the string which are not continuation bytes, which can be done many bytes at +//! a time fairly easily. +//! +//! Note: Because the term "leading byte" can sometimes be ambiguous (for +//! example, it could also refer to the first byte of a slice), we'll often use +//! the term "non-continuation byte" to refer to these bytes in the code. +use core::intrinsics::unlikely; + +const USIZE_SIZE: usize = core::mem::size_of::(); +const UNROLL_INNER: usize = 4; + +#[inline] +pub(super) fn count_chars(s: &str) -> usize { + if s.len() < USIZE_SIZE * UNROLL_INNER { + // Avoid entering the optimized implementation for strings where the + // difference is not likely to matter, or where it might even be slower. + // That said, a ton of thought was not spent on the particular threshold + // here, beyond "this value seems to make sense". + char_count_general_case(s.as_bytes()) + } else { + do_count_chars(s) + } +} + +fn do_count_chars(s: &str) -> usize { + // For correctness, `CHUNK_SIZE` must be: + // + // - Less than or equal to 255, otherwise we'll overflow bytes in `counts`. + // - A multiple of `UNROLL_INNER`, otherwise our `break` inside the + // `body.chunks(CHUNK_SIZE)` loop is incorrect. + // + // For performance, `CHUNK_SIZE` should be: + // - Relatively cheap to `/` against (so some simple sum of powers of two). + // - Large enough to avoid paying for the cost of the `sum_bytes_in_usize` + // too often. + const CHUNK_SIZE: usize = 192; + + // Check the properties of `CHUNK_SIZE` and `UNROLL_INNER` that are required + // for correctness. + const _: () = assert!(CHUNK_SIZE < 256); + const _: () = assert!(CHUNK_SIZE % UNROLL_INNER == 0); + + // SAFETY: transmuting `[u8]` to `[usize]` is safe except for size + // differences which are handled by `align_to`. + let (head, body, tail) = unsafe { s.as_bytes().align_to::() }; + + // This should be quite rare, and basically exists to handle the degenerate + // cases where align_to fails (as well as miri under symbolic alignment + // mode). + // + // The `unlikely` helps discourage LLVM from inlining the body, which is + // nice, as we would rather not mark the `char_count_general_case` function + // as cold. + if unlikely(body.is_empty() || head.len() > USIZE_SIZE || tail.len() > USIZE_SIZE) { + return char_count_general_case(s.as_bytes()); + } + + let mut total = char_count_general_case(head) + char_count_general_case(tail); + // Split `body` into `CHUNK_SIZE` chunks to reduce the frequency with which + // we call `sum_bytes_in_usize`. + for chunk in body.chunks(CHUNK_SIZE) { + // We accumulate intermediate sums in `counts`, where each byte contains + // a subset of the sum of this chunk, like a `[u8; size_of::()]`. + let mut counts = 0; + + let (unrolled_chunks, remainder) = chunk.as_chunks::(); + for unrolled in unrolled_chunks { + for &word in unrolled { + // Because `CHUNK_SIZE` is < 256, this addition can't cause the + // count in any of the bytes to overflow into a subsequent byte. + counts += contains_non_continuation_byte(word); + } + } + + // Sum the values in `counts` (which, again, is conceptually a `[u8; + // size_of::()]`), and accumulate the result into `total`. + total += sum_bytes_in_usize(counts); + + // If there's any data in `remainder`, then handle it. This will only + // happen for the last `chunk` in `body.chunks()` (because `CHUNK_SIZE` + // is divisible by `UNROLL_INNER`), so we explicitly break at the end + // (which seems to help LLVM out). + if !remainder.is_empty() { + // Accumulate all the data in the remainder. + let mut counts = 0; + for &word in remainder { + counts += contains_non_continuation_byte(word); + } + total += sum_bytes_in_usize(counts); + break; + } + } + total +} + +// Checks each byte of `w` to see if it contains the first byte in a UTF-8 +// sequence. Bytes in `w` which are continuation bytes are left as `0x00` (e.g. +// false), and bytes which are non-continuation bytes are left as `0x01` (e.g. +// true) +#[inline] +fn contains_non_continuation_byte(w: usize) -> usize { + const LSB: usize = usize::repeat_u8(0x01); + ((!w >> 7) | (w >> 6)) & LSB +} + +// Morally equivalent to `values.to_ne_bytes().into_iter().sum::()`, but +// more efficient. +#[inline] +fn sum_bytes_in_usize(values: usize) -> usize { + const LSB_SHORTS: usize = usize::repeat_u16(0x0001); + const SKIP_BYTES: usize = usize::repeat_u16(0x00ff); + + let pair_sum: usize = (values & SKIP_BYTES) + ((values >> 8) & SKIP_BYTES); + pair_sum.wrapping_mul(LSB_SHORTS) >> ((USIZE_SIZE - 2) * 8) +} + +// This is the most direct implementation of the concept of "count the number of +// bytes in the string which are not continuation bytes", and is used for the +// head and tail of the input string (the first and last item in the tuple +// returned by `slice::align_to`). +fn char_count_general_case(s: &[u8]) -> usize { + s.iter().filter(|&&byte| !super::validations::utf8_is_cont_byte(byte)).count() +} diff --git a/crux-mir/lib/core/src/str/error.rs b/crux-mir/lib/core/src/str/error.rs new file mode 100644 index 000000000..a11b5add4 --- /dev/null +++ b/crux-mir/lib/core/src/str/error.rs @@ -0,0 +1,155 @@ +//! Defines utf8 error type. + +use crate::error::Error; +use crate::fmt; + +/// Errors which can occur when attempting to interpret a sequence of [`u8`] +/// as a string. +/// +/// As such, the `from_utf8` family of functions and methods for both [`String`]s +/// and [`&str`]s make use of this error, for example. +/// +/// [`String`]: ../../std/string/struct.String.html#method.from_utf8 +/// [`&str`]: super::from_utf8 +/// +/// # Examples +/// +/// This error type’s methods can be used to create functionality +/// similar to `String::from_utf8_lossy` without allocating heap memory: +/// +/// ``` +/// fn from_utf8_lossy(mut input: &[u8], mut push: F) where F: FnMut(&str) { +/// loop { +/// match std::str::from_utf8(input) { +/// Ok(valid) => { +/// push(valid); +/// break +/// } +/// Err(error) => { +/// let (valid, after_valid) = input.split_at(error.valid_up_to()); +/// unsafe { +/// push(std::str::from_utf8_unchecked(valid)) +/// } +/// push("\u{FFFD}"); +/// +/// if let Some(invalid_sequence_length) = error.error_len() { +/// input = &after_valid[invalid_sequence_length..] +/// } else { +/// break +/// } +/// } +/// } +/// } +/// } +/// ``` +#[derive(Copy, Eq, PartialEq, Clone, Debug)] +#[stable(feature = "rust1", since = "1.0.0")] +pub struct Utf8Error { + pub(super) valid_up_to: usize, + pub(super) error_len: Option, +} + +impl Utf8Error { + /// Returns the index in the given string up to which valid UTF-8 was + /// verified. + /// + /// It is the maximum index such that `from_utf8(&input[..index])` + /// would return `Ok(_)`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use std::str; + /// + /// // some invalid bytes, in a vector + /// let sparkle_heart = vec![0, 159, 146, 150]; + /// + /// // std::str::from_utf8 returns a Utf8Error + /// let error = str::from_utf8(&sparkle_heart).unwrap_err(); + /// + /// // the second byte is invalid here + /// assert_eq!(1, error.valid_up_to()); + /// ``` + #[stable(feature = "utf8_error", since = "1.5.0")] + #[rustc_const_stable(feature = "const_str_from_utf8_shared", since = "1.63.0")] + #[must_use] + #[inline] + pub const fn valid_up_to(&self) -> usize { + self.valid_up_to + } + + /// Provides more information about the failure: + /// + /// * `None`: the end of the input was reached unexpectedly. + /// `self.valid_up_to()` is 1 to 3 bytes from the end of the input. + /// If a byte stream (such as a file or a network socket) is being decoded incrementally, + /// this could be a valid `char` whose UTF-8 byte sequence is spanning multiple chunks. + /// + /// * `Some(len)`: an unexpected byte was encountered. + /// The length provided is that of the invalid byte sequence + /// that starts at the index given by `valid_up_to()`. + /// Decoding should resume after that sequence + /// (after inserting a [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]) in case of + /// lossy decoding. + /// + /// [U+FFFD]: ../../std/char/constant.REPLACEMENT_CHARACTER.html + #[stable(feature = "utf8_error_error_len", since = "1.20.0")] + #[rustc_const_stable(feature = "const_str_from_utf8_shared", since = "1.63.0")] + #[must_use] + #[inline] + pub const fn error_len(&self) -> Option { + // FIXME: This should become `map` again, once it's `const` + match self.error_len { + Some(len) => Some(len as usize), + None => None, + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Display for Utf8Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if let Some(error_len) = self.error_len { + write!( + f, + "invalid utf-8 sequence of {} bytes from index {}", + error_len, self.valid_up_to + ) + } else { + write!(f, "incomplete utf-8 byte sequence from index {}", self.valid_up_to) + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Error for Utf8Error { + #[allow(deprecated)] + fn description(&self) -> &str { + "invalid utf-8: corrupt contents" + } +} + +/// An error returned when parsing a `bool` using [`from_str`] fails +/// +/// [`from_str`]: super::FromStr::from_str +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +#[stable(feature = "rust1", since = "1.0.0")] +pub struct ParseBoolError; + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Display for ParseBoolError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + "provided string was not `true` or `false`".fmt(f) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Error for ParseBoolError { + #[allow(deprecated)] + fn description(&self) -> &str { + "failed to parse bool" + } +} diff --git a/crux-mir/lib/core/src/str/iter.rs b/crux-mir/lib/core/src/str/iter.rs new file mode 100644 index 000000000..d969475aa --- /dev/null +++ b/crux-mir/lib/core/src/str/iter.rs @@ -0,0 +1,1516 @@ +//! Iterators for `str` methods. + +use crate::char; +use crate::fmt::{self, Write}; +use crate::iter::{Chain, FlatMap, Flatten}; +use crate::iter::{Copied, Filter, FusedIterator, Map, TrustedLen}; +use crate::iter::{TrustedRandomAccess, TrustedRandomAccessNoCoerce}; +use crate::ops::Try; +use crate::option; +use crate::slice::{self, Split as SliceSplit}; + +use super::from_utf8_unchecked; +use super::pattern::Pattern; +use super::pattern::{DoubleEndedSearcher, ReverseSearcher, Searcher}; +use super::validations::{next_code_point, next_code_point_reverse}; +use super::LinesAnyMap; +use super::{BytesIsNotEmpty, UnsafeBytesToStr}; +use super::{CharEscapeDebugContinue, CharEscapeDefault, CharEscapeUnicode}; +use super::{IsAsciiWhitespace, IsNotEmpty, IsWhitespace}; + +/// An iterator over the [`char`]s of a string slice. +/// +/// +/// This struct is created by the [`chars`] method on [`str`]. +/// See its documentation for more. +/// +/// [`char`]: prim@char +/// [`chars`]: str::chars +#[derive(Clone)] +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "rust1", since = "1.0.0")] +pub struct Chars<'a> { + pub(super) iter: slice::Iter<'a, u8>, +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a> Iterator for Chars<'a> { + type Item = char; + + #[inline] + fn next(&mut self) -> Option { + // SAFETY: `str` invariant says `self.iter` is a valid UTF-8 string and + // the resulting `ch` is a valid Unicode Scalar Value. + unsafe { next_code_point(&mut self.iter).map(|ch| char::from_u32_unchecked(ch)) } + } + + #[inline] + fn count(self) -> usize { + super::count::count_chars(self.as_str()) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let len = self.iter.len(); + // `(len + 3)` can't overflow, because we know that the `slice::Iter` + // belongs to a slice in memory which has a maximum length of + // `isize::MAX` (that's well below `usize::MAX`). + ((len + 3) / 4, Some(len)) + } + + #[inline] + fn last(mut self) -> Option { + // No need to go through the entire string. + self.next_back() + } +} + +#[stable(feature = "chars_debug_impl", since = "1.38.0")] +impl fmt::Debug for Chars<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Chars(")?; + f.debug_list().entries(self.clone()).finish()?; + write!(f, ")")?; + Ok(()) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a> DoubleEndedIterator for Chars<'a> { + #[inline] + fn next_back(&mut self) -> Option { + // SAFETY: `str` invariant says `self.iter` is a valid UTF-8 string and + // the resulting `ch` is a valid Unicode Scalar Value. + unsafe { next_code_point_reverse(&mut self.iter).map(|ch| char::from_u32_unchecked(ch)) } + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Chars<'_> {} + +impl<'a> Chars<'a> { + /// Views the underlying data as a subslice of the original data. + /// + /// This has the same lifetime as the original slice, and so the + /// iterator can continue to be used while this exists. + /// + /// # Examples + /// + /// ``` + /// let mut chars = "abc".chars(); + /// + /// assert_eq!(chars.as_str(), "abc"); + /// chars.next(); + /// assert_eq!(chars.as_str(), "bc"); + /// chars.next(); + /// chars.next(); + /// assert_eq!(chars.as_str(), ""); + /// ``` + #[stable(feature = "iter_to_slice", since = "1.4.0")] + #[must_use] + #[inline] + pub fn as_str(&self) -> &'a str { + // SAFETY: `Chars` is only made from a str, which guarantees the iter is valid UTF-8. + unsafe { from_utf8_unchecked(self.iter.as_slice()) } + } +} + +/// An iterator over the [`char`]s of a string slice, and their positions. +/// +/// This struct is created by the [`char_indices`] method on [`str`]. +/// See its documentation for more. +/// +/// [`char`]: prim@char +/// [`char_indices`]: str::char_indices +#[derive(Clone, Debug)] +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "rust1", since = "1.0.0")] +pub struct CharIndices<'a> { + pub(super) front_offset: usize, + pub(super) iter: Chars<'a>, +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a> Iterator for CharIndices<'a> { + type Item = (usize, char); + + #[inline] + fn next(&mut self) -> Option<(usize, char)> { + let pre_len = self.iter.iter.len(); + match self.iter.next() { + None => None, + Some(ch) => { + let index = self.front_offset; + let len = self.iter.iter.len(); + self.front_offset += pre_len - len; + Some((index, ch)) + } + } + } + + #[inline] + fn count(self) -> usize { + self.iter.count() + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } + + #[inline] + fn last(mut self) -> Option<(usize, char)> { + // No need to go through the entire string. + self.next_back() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a> DoubleEndedIterator for CharIndices<'a> { + #[inline] + fn next_back(&mut self) -> Option<(usize, char)> { + self.iter.next_back().map(|ch| { + let index = self.front_offset + self.iter.iter.len(); + (index, ch) + }) + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for CharIndices<'_> {} + +impl<'a> CharIndices<'a> { + /// Views the underlying data as a subslice of the original data. + /// + /// This has the same lifetime as the original slice, and so the + /// iterator can continue to be used while this exists. + #[stable(feature = "iter_to_slice", since = "1.4.0")] + #[must_use] + #[inline] + pub fn as_str(&self) -> &'a str { + self.iter.as_str() + } + + /// Returns the byte position of the next character, or the length + /// of the underlying string if there are no more characters. + /// + /// # Examples + /// + /// ``` + /// #![feature(char_indices_offset)] + /// let mut chars = "a楽".char_indices(); + /// + /// assert_eq!(chars.offset(), 0); + /// assert_eq!(chars.next(), Some((0, 'a'))); + /// + /// assert_eq!(chars.offset(), 1); + /// assert_eq!(chars.next(), Some((1, '楽'))); + /// + /// assert_eq!(chars.offset(), 4); + /// assert_eq!(chars.next(), None); + /// ``` + #[inline] + #[must_use] + #[unstable(feature = "char_indices_offset", issue = "83871")] + pub fn offset(&self) -> usize { + self.front_offset + } +} + +/// An iterator over the bytes of a string slice. +/// +/// This struct is created by the [`bytes`] method on [`str`]. +/// See its documentation for more. +/// +/// [`bytes`]: str::bytes +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "rust1", since = "1.0.0")] +#[derive(Clone, Debug)] +pub struct Bytes<'a>(pub(super) Copied>); + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for Bytes<'_> { + type Item = u8; + + #[inline] + fn next(&mut self) -> Option { + self.0.next() + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.0.size_hint() + } + + #[inline] + fn count(self) -> usize { + self.0.count() + } + + #[inline] + fn last(self) -> Option { + self.0.last() + } + + #[inline] + fn nth(&mut self, n: usize) -> Option { + self.0.nth(n) + } + + #[inline] + fn all(&mut self, f: F) -> bool + where + F: FnMut(Self::Item) -> bool, + { + self.0.all(f) + } + + #[inline] + fn any(&mut self, f: F) -> bool + where + F: FnMut(Self::Item) -> bool, + { + self.0.any(f) + } + + #[inline] + fn find

(&mut self, predicate: P) -> Option + where + P: FnMut(&Self::Item) -> bool, + { + self.0.find(predicate) + } + + #[inline] + fn position

(&mut self, predicate: P) -> Option + where + P: FnMut(Self::Item) -> bool, + { + self.0.position(predicate) + } + + #[inline] + fn rposition

(&mut self, predicate: P) -> Option + where + P: FnMut(Self::Item) -> bool, + { + self.0.rposition(predicate) + } + + #[inline] + unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> u8 { + // SAFETY: the caller must uphold the safety contract + // for `Iterator::__iterator_get_unchecked`. + unsafe { self.0.__iterator_get_unchecked(idx) } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl DoubleEndedIterator for Bytes<'_> { + #[inline] + fn next_back(&mut self) -> Option { + self.0.next_back() + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option { + self.0.nth_back(n) + } + + #[inline] + fn rfind

(&mut self, predicate: P) -> Option + where + P: FnMut(&Self::Item) -> bool, + { + self.0.rfind(predicate) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ExactSizeIterator for Bytes<'_> { + #[inline] + fn len(&self) -> usize { + self.0.len() + } + + #[inline] + fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Bytes<'_> {} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for Bytes<'_> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl TrustedRandomAccess for Bytes<'_> {} + +#[doc(hidden)] +#[unstable(feature = "trusted_random_access", issue = "none")] +unsafe impl TrustedRandomAccessNoCoerce for Bytes<'_> { + const MAY_HAVE_SIDE_EFFECT: bool = false; +} + +/// This macro generates a Clone impl for string pattern API +/// wrapper types of the form X<'a, P> +macro_rules! derive_pattern_clone { + (clone $t:ident with |$s:ident| $e:expr) => { + impl<'a, P> Clone for $t<'a, P> + where + P: Pattern<'a, Searcher: Clone>, + { + fn clone(&self) -> Self { + let $s = self; + $e + } + } + }; +} + +/// This macro generates two public iterator structs +/// wrapping a private internal one that makes use of the `Pattern` API. +/// +/// For all patterns `P: Pattern<'a>` the following items will be +/// generated (generics omitted): +/// +/// struct $forward_iterator($internal_iterator); +/// struct $reverse_iterator($internal_iterator); +/// +/// impl Iterator for $forward_iterator +/// { /* internal ends up calling Searcher::next_match() */ } +/// +/// impl DoubleEndedIterator for $forward_iterator +/// where P::Searcher: DoubleEndedSearcher +/// { /* internal ends up calling Searcher::next_match_back() */ } +/// +/// impl Iterator for $reverse_iterator +/// where P::Searcher: ReverseSearcher +/// { /* internal ends up calling Searcher::next_match_back() */ } +/// +/// impl DoubleEndedIterator for $reverse_iterator +/// where P::Searcher: DoubleEndedSearcher +/// { /* internal ends up calling Searcher::next_match() */ } +/// +/// The internal one is defined outside the macro, and has almost the same +/// semantic as a DoubleEndedIterator by delegating to `pattern::Searcher` and +/// `pattern::ReverseSearcher` for both forward and reverse iteration. +/// +/// "Almost", because a `Searcher` and a `ReverseSearcher` for a given +/// `Pattern` might not return the same elements, so actually implementing +/// `DoubleEndedIterator` for it would be incorrect. +/// (See the docs in `str::pattern` for more details) +/// +/// However, the internal struct still represents a single ended iterator from +/// either end, and depending on pattern is also a valid double ended iterator, +/// so the two wrapper structs implement `Iterator` +/// and `DoubleEndedIterator` depending on the concrete pattern type, leading +/// to the complex impls seen above. +macro_rules! generate_pattern_iterators { + { + // Forward iterator + forward: + $(#[$forward_iterator_attribute:meta])* + struct $forward_iterator:ident; + + // Reverse iterator + reverse: + $(#[$reverse_iterator_attribute:meta])* + struct $reverse_iterator:ident; + + // Stability of all generated items + stability: + $(#[$common_stability_attribute:meta])* + + // Internal almost-iterator that is being delegated to + internal: + $internal_iterator:ident yielding ($iterty:ty); + + // Kind of delegation - either single ended or double ended + delegate $($t:tt)* + } => { + $(#[$forward_iterator_attribute])* + $(#[$common_stability_attribute])* + pub struct $forward_iterator<'a, P: Pattern<'a>>(pub(super) $internal_iterator<'a, P>); + + $(#[$common_stability_attribute])* + impl<'a, P> fmt::Debug for $forward_iterator<'a, P> + where + P: Pattern<'a, Searcher: fmt::Debug>, + { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple(stringify!($forward_iterator)) + .field(&self.0) + .finish() + } + } + + $(#[$common_stability_attribute])* + impl<'a, P: Pattern<'a>> Iterator for $forward_iterator<'a, P> { + type Item = $iterty; + + #[inline] + fn next(&mut self) -> Option<$iterty> { + self.0.next() + } + } + + $(#[$common_stability_attribute])* + impl<'a, P> Clone for $forward_iterator<'a, P> + where + P: Pattern<'a, Searcher: Clone>, + { + fn clone(&self) -> Self { + $forward_iterator(self.0.clone()) + } + } + + $(#[$reverse_iterator_attribute])* + $(#[$common_stability_attribute])* + pub struct $reverse_iterator<'a, P: Pattern<'a>>(pub(super) $internal_iterator<'a, P>); + + $(#[$common_stability_attribute])* + impl<'a, P> fmt::Debug for $reverse_iterator<'a, P> + where + P: Pattern<'a, Searcher: fmt::Debug>, + { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple(stringify!($reverse_iterator)) + .field(&self.0) + .finish() + } + } + + $(#[$common_stability_attribute])* + impl<'a, P> Iterator for $reverse_iterator<'a, P> + where + P: Pattern<'a, Searcher: ReverseSearcher<'a>>, + { + type Item = $iterty; + + #[inline] + fn next(&mut self) -> Option<$iterty> { + self.0.next_back() + } + } + + $(#[$common_stability_attribute])* + impl<'a, P> Clone for $reverse_iterator<'a, P> + where + P: Pattern<'a, Searcher: Clone>, + { + fn clone(&self) -> Self { + $reverse_iterator(self.0.clone()) + } + } + + #[stable(feature = "fused", since = "1.26.0")] + impl<'a, P: Pattern<'a>> FusedIterator for $forward_iterator<'a, P> {} + + #[stable(feature = "fused", since = "1.26.0")] + impl<'a, P> FusedIterator for $reverse_iterator<'a, P> + where + P: Pattern<'a, Searcher: ReverseSearcher<'a>>, + {} + + generate_pattern_iterators!($($t)* with $(#[$common_stability_attribute])*, + $forward_iterator, + $reverse_iterator, $iterty); + }; + { + double ended; with $(#[$common_stability_attribute:meta])*, + $forward_iterator:ident, + $reverse_iterator:ident, $iterty:ty + } => { + $(#[$common_stability_attribute])* + impl<'a, P> DoubleEndedIterator for $forward_iterator<'a, P> + where + P: Pattern<'a, Searcher: DoubleEndedSearcher<'a>>, + { + #[inline] + fn next_back(&mut self) -> Option<$iterty> { + self.0.next_back() + } + } + + $(#[$common_stability_attribute])* + impl<'a, P> DoubleEndedIterator for $reverse_iterator<'a, P> + where + P: Pattern<'a, Searcher: DoubleEndedSearcher<'a>>, + { + #[inline] + fn next_back(&mut self) -> Option<$iterty> { + self.0.next() + } + } + }; + { + single ended; with $(#[$common_stability_attribute:meta])*, + $forward_iterator:ident, + $reverse_iterator:ident, $iterty:ty + } => {} +} + +derive_pattern_clone! { + clone SplitInternal + with |s| SplitInternal { matcher: s.matcher.clone(), ..*s } +} + +pub(super) struct SplitInternal<'a, P: Pattern<'a>> { + pub(super) start: usize, + pub(super) end: usize, + pub(super) matcher: P::Searcher, + pub(super) allow_trailing_empty: bool, + pub(super) finished: bool, +} + +impl<'a, P> fmt::Debug for SplitInternal<'a, P> +where + P: Pattern<'a, Searcher: fmt::Debug>, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SplitInternal") + .field("start", &self.start) + .field("end", &self.end) + .field("matcher", &self.matcher) + .field("allow_trailing_empty", &self.allow_trailing_empty) + .field("finished", &self.finished) + .finish() + } +} + +impl<'a, P: Pattern<'a>> SplitInternal<'a, P> { + #[inline] + fn get_end(&mut self) -> Option<&'a str> { + if !self.finished { + self.finished = true; + + if self.allow_trailing_empty || self.end - self.start > 0 { + // SAFETY: `self.start` and `self.end` always lie on unicode boundaries. + let string = unsafe { self.matcher.haystack().get_unchecked(self.start..self.end) }; + return Some(string); + } + } + + None + } + + #[inline] + fn next(&mut self) -> Option<&'a str> { + if self.finished { + return None; + } + + let haystack = self.matcher.haystack(); + match self.matcher.next_match() { + // SAFETY: `Searcher` guarantees that `a` and `b` lie on unicode boundaries. + Some((a, b)) => unsafe { + let elt = haystack.get_unchecked(self.start..a); + self.start = b; + Some(elt) + }, + None => self.get_end(), + } + } + + #[inline] + fn next_inclusive(&mut self) -> Option<&'a str> { + if self.finished { + return None; + } + + let haystack = self.matcher.haystack(); + match self.matcher.next_match() { + // SAFETY: `Searcher` guarantees that `b` lies on unicode boundary, + // and self.start is either the start of the original string, + // or `b` was assigned to it, so it also lies on unicode boundary. + Some((_, b)) => unsafe { + let elt = haystack.get_unchecked(self.start..b); + self.start = b; + Some(elt) + }, + None => self.get_end(), + } + } + + #[inline] + fn next_back(&mut self) -> Option<&'a str> + where + P::Searcher: ReverseSearcher<'a>, + { + if self.finished { + return None; + } + + if !self.allow_trailing_empty { + self.allow_trailing_empty = true; + match self.next_back() { + Some(elt) if !elt.is_empty() => return Some(elt), + _ => { + if self.finished { + return None; + } + } + } + } + + let haystack = self.matcher.haystack(); + match self.matcher.next_match_back() { + // SAFETY: `Searcher` guarantees that `a` and `b` lie on unicode boundaries. + Some((a, b)) => unsafe { + let elt = haystack.get_unchecked(b..self.end); + self.end = a; + Some(elt) + }, + // SAFETY: `self.start` and `self.end` always lie on unicode boundaries. + None => unsafe { + self.finished = true; + Some(haystack.get_unchecked(self.start..self.end)) + }, + } + } + + #[inline] + fn next_back_inclusive(&mut self) -> Option<&'a str> + where + P::Searcher: ReverseSearcher<'a>, + { + if self.finished { + return None; + } + + if !self.allow_trailing_empty { + self.allow_trailing_empty = true; + match self.next_back_inclusive() { + Some(elt) if !elt.is_empty() => return Some(elt), + _ => { + if self.finished { + return None; + } + } + } + } + + let haystack = self.matcher.haystack(); + match self.matcher.next_match_back() { + // SAFETY: `Searcher` guarantees that `b` lies on unicode boundary, + // and self.end is either the end of the original string, + // or `b` was assigned to it, so it also lies on unicode boundary. + Some((_, b)) => unsafe { + let elt = haystack.get_unchecked(b..self.end); + self.end = b; + Some(elt) + }, + // SAFETY: self.start is either the start of the original string, + // or start of a substring that represents the part of the string that hasn't + // iterated yet. Either way, it is guaranteed to lie on unicode boundary. + // self.end is either the end of the original string, + // or `b` was assigned to it, so it also lies on unicode boundary. + None => unsafe { + self.finished = true; + Some(haystack.get_unchecked(self.start..self.end)) + }, + } + } + + #[inline] + fn remainder(&self) -> Option<&'a str> { + // `Self::get_end` doesn't change `self.start` + if self.finished { + return None; + } + + // SAFETY: `self.start` and `self.end` always lie on unicode boundaries. + Some(unsafe { self.matcher.haystack().get_unchecked(self.start..self.end) }) + } +} + +generate_pattern_iterators! { + forward: + /// Created with the method [`split`]. + /// + /// [`split`]: str::split + struct Split; + reverse: + /// Created with the method [`rsplit`]. + /// + /// [`rsplit`]: str::rsplit + struct RSplit; + stability: + #[stable(feature = "rust1", since = "1.0.0")] + internal: + SplitInternal yielding (&'a str); + delegate double ended; +} + +impl<'a, P: Pattern<'a>> Split<'a, P> { + /// Returns remainder of the split string. + /// + /// If the iterator is empty, returns `None`. + /// + /// # Examples + /// + /// ``` + /// #![feature(str_split_remainder)] + /// let mut split = "Mary had a little lamb".split(' '); + /// assert_eq!(split.remainder(), Some("Mary had a little lamb")); + /// split.next(); + /// assert_eq!(split.remainder(), Some("had a little lamb")); + /// split.by_ref().for_each(drop); + /// assert_eq!(split.remainder(), None); + /// ``` + #[inline] + #[unstable(feature = "str_split_remainder", issue = "77998")] + pub fn remainder(&self) -> Option<&'a str> { + self.0.remainder() + } +} + +impl<'a, P: Pattern<'a>> RSplit<'a, P> { + /// Returns remainder of the split string. + /// + /// If the iterator is empty, returns `None`. + /// + /// # Examples + /// + /// ``` + /// #![feature(str_split_remainder)] + /// let mut split = "Mary had a little lamb".rsplit(' '); + /// assert_eq!(split.remainder(), Some("Mary had a little lamb")); + /// split.next(); + /// assert_eq!(split.remainder(), Some("Mary had a little")); + /// split.by_ref().for_each(drop); + /// assert_eq!(split.remainder(), None); + /// ``` + #[inline] + #[unstable(feature = "str_split_remainder", issue = "77998")] + pub fn remainder(&self) -> Option<&'a str> { + self.0.remainder() + } +} + +generate_pattern_iterators! { + forward: + /// Created with the method [`split_terminator`]. + /// + /// [`split_terminator`]: str::split_terminator + struct SplitTerminator; + reverse: + /// Created with the method [`rsplit_terminator`]. + /// + /// [`rsplit_terminator`]: str::rsplit_terminator + struct RSplitTerminator; + stability: + #[stable(feature = "rust1", since = "1.0.0")] + internal: + SplitInternal yielding (&'a str); + delegate double ended; +} + +impl<'a, P: Pattern<'a>> SplitTerminator<'a, P> { + /// Returns remainder of the split string. + /// + /// If the iterator is empty, returns `None`. + /// + /// # Examples + /// + /// ``` + /// #![feature(str_split_remainder)] + /// let mut split = "A..B..".split_terminator('.'); + /// assert_eq!(split.remainder(), Some("A..B..")); + /// split.next(); + /// assert_eq!(split.remainder(), Some(".B..")); + /// split.by_ref().for_each(drop); + /// assert_eq!(split.remainder(), None); + /// ``` + #[inline] + #[unstable(feature = "str_split_remainder", issue = "77998")] + pub fn remainder(&self) -> Option<&'a str> { + self.0.remainder() + } +} + +impl<'a, P: Pattern<'a>> RSplitTerminator<'a, P> { + /// Returns remainder of the split string. + /// + /// If the iterator is empty, returns `None`. + /// + /// # Examples + /// + /// ``` + /// #![feature(str_split_remainder)] + /// let mut split = "A..B..".rsplit_terminator('.'); + /// assert_eq!(split.remainder(), Some("A..B..")); + /// split.next(); + /// assert_eq!(split.remainder(), Some("A..B")); + /// split.by_ref().for_each(drop); + /// assert_eq!(split.remainder(), None); + /// ``` + #[inline] + #[unstable(feature = "str_split_remainder", issue = "77998")] + pub fn remainder(&self) -> Option<&'a str> { + self.0.remainder() + } +} + +derive_pattern_clone! { + clone SplitNInternal + with |s| SplitNInternal { iter: s.iter.clone(), ..*s } +} + +pub(super) struct SplitNInternal<'a, P: Pattern<'a>> { + pub(super) iter: SplitInternal<'a, P>, + /// The number of splits remaining + pub(super) count: usize, +} + +impl<'a, P> fmt::Debug for SplitNInternal<'a, P> +where + P: Pattern<'a, Searcher: fmt::Debug>, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SplitNInternal") + .field("iter", &self.iter) + .field("count", &self.count) + .finish() + } +} + +impl<'a, P: Pattern<'a>> SplitNInternal<'a, P> { + #[inline] + fn next(&mut self) -> Option<&'a str> { + match self.count { + 0 => None, + 1 => { + self.count = 0; + self.iter.get_end() + } + _ => { + self.count -= 1; + self.iter.next() + } + } + } + + #[inline] + fn next_back(&mut self) -> Option<&'a str> + where + P::Searcher: ReverseSearcher<'a>, + { + match self.count { + 0 => None, + 1 => { + self.count = 0; + self.iter.get_end() + } + _ => { + self.count -= 1; + self.iter.next_back() + } + } + } + + #[inline] + fn remainder(&self) -> Option<&'a str> { + self.iter.remainder() + } +} + +generate_pattern_iterators! { + forward: + /// Created with the method [`splitn`]. + /// + /// [`splitn`]: str::splitn + struct SplitN; + reverse: + /// Created with the method [`rsplitn`]. + /// + /// [`rsplitn`]: str::rsplitn + struct RSplitN; + stability: + #[stable(feature = "rust1", since = "1.0.0")] + internal: + SplitNInternal yielding (&'a str); + delegate single ended; +} + +impl<'a, P: Pattern<'a>> SplitN<'a, P> { + /// Returns remainder of the split string. + /// + /// If the iterator is empty, returns `None`. + /// + /// # Examples + /// + /// ``` + /// #![feature(str_split_remainder)] + /// let mut split = "Mary had a little lamb".splitn(3, ' '); + /// assert_eq!(split.remainder(), Some("Mary had a little lamb")); + /// split.next(); + /// assert_eq!(split.remainder(), Some("had a little lamb")); + /// split.by_ref().for_each(drop); + /// assert_eq!(split.remainder(), None); + /// ``` + #[inline] + #[unstable(feature = "str_split_remainder", issue = "77998")] + pub fn remainder(&self) -> Option<&'a str> { + self.0.remainder() + } +} + +impl<'a, P: Pattern<'a>> RSplitN<'a, P> { + /// Returns remainder of the split string. + /// + /// If the iterator is empty, returns `None`. + /// + /// # Examples + /// + /// ``` + /// #![feature(str_split_remainder)] + /// let mut split = "Mary had a little lamb".rsplitn(3, ' '); + /// assert_eq!(split.remainder(), Some("Mary had a little lamb")); + /// split.next(); + /// assert_eq!(split.remainder(), Some("Mary had a little")); + /// split.by_ref().for_each(drop); + /// assert_eq!(split.remainder(), None); + /// ``` + #[inline] + #[unstable(feature = "str_split_remainder", issue = "77998")] + pub fn remainder(&self) -> Option<&'a str> { + self.0.remainder() + } +} + +derive_pattern_clone! { + clone MatchIndicesInternal + with |s| MatchIndicesInternal(s.0.clone()) +} + +pub(super) struct MatchIndicesInternal<'a, P: Pattern<'a>>(pub(super) P::Searcher); + +impl<'a, P> fmt::Debug for MatchIndicesInternal<'a, P> +where + P: Pattern<'a, Searcher: fmt::Debug>, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("MatchIndicesInternal").field(&self.0).finish() + } +} + +impl<'a, P: Pattern<'a>> MatchIndicesInternal<'a, P> { + #[inline] + fn next(&mut self) -> Option<(usize, &'a str)> { + self.0 + .next_match() + // SAFETY: `Searcher` guarantees that `start` and `end` lie on unicode boundaries. + .map(|(start, end)| unsafe { (start, self.0.haystack().get_unchecked(start..end)) }) + } + + #[inline] + fn next_back(&mut self) -> Option<(usize, &'a str)> + where + P::Searcher: ReverseSearcher<'a>, + { + self.0 + .next_match_back() + // SAFETY: `Searcher` guarantees that `start` and `end` lie on unicode boundaries. + .map(|(start, end)| unsafe { (start, self.0.haystack().get_unchecked(start..end)) }) + } +} + +generate_pattern_iterators! { + forward: + /// Created with the method [`match_indices`]. + /// + /// [`match_indices`]: str::match_indices + struct MatchIndices; + reverse: + /// Created with the method [`rmatch_indices`]. + /// + /// [`rmatch_indices`]: str::rmatch_indices + struct RMatchIndices; + stability: + #[stable(feature = "str_match_indices", since = "1.5.0")] + internal: + MatchIndicesInternal yielding ((usize, &'a str)); + delegate double ended; +} + +derive_pattern_clone! { + clone MatchesInternal + with |s| MatchesInternal(s.0.clone()) +} + +pub(super) struct MatchesInternal<'a, P: Pattern<'a>>(pub(super) P::Searcher); + +impl<'a, P> fmt::Debug for MatchesInternal<'a, P> +where + P: Pattern<'a, Searcher: fmt::Debug>, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("MatchesInternal").field(&self.0).finish() + } +} + +impl<'a, P: Pattern<'a>> MatchesInternal<'a, P> { + #[inline] + fn next(&mut self) -> Option<&'a str> { + // SAFETY: `Searcher` guarantees that `start` and `end` lie on unicode boundaries. + self.0.next_match().map(|(a, b)| unsafe { + // Indices are known to be on utf8 boundaries + self.0.haystack().get_unchecked(a..b) + }) + } + + #[inline] + fn next_back(&mut self) -> Option<&'a str> + where + P::Searcher: ReverseSearcher<'a>, + { + // SAFETY: `Searcher` guarantees that `start` and `end` lie on unicode boundaries. + self.0.next_match_back().map(|(a, b)| unsafe { + // Indices are known to be on utf8 boundaries + self.0.haystack().get_unchecked(a..b) + }) + } +} + +generate_pattern_iterators! { + forward: + /// Created with the method [`matches`]. + /// + /// [`matches`]: str::matches + struct Matches; + reverse: + /// Created with the method [`rmatches`]. + /// + /// [`rmatches`]: str::rmatches + struct RMatches; + stability: + #[stable(feature = "str_matches", since = "1.2.0")] + internal: + MatchesInternal yielding (&'a str); + delegate double ended; +} + +/// An iterator over the lines of a string, as string slices. +/// +/// This struct is created with the [`lines`] method on [`str`]. +/// See its documentation for more. +/// +/// [`lines`]: str::lines +#[stable(feature = "rust1", since = "1.0.0")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[derive(Clone, Debug)] +pub struct Lines<'a>(pub(super) Map, LinesAnyMap>); + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a> Iterator for Lines<'a> { + type Item = &'a str; + + #[inline] + fn next(&mut self) -> Option<&'a str> { + self.0.next() + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.0.size_hint() + } + + #[inline] + fn last(mut self) -> Option<&'a str> { + self.next_back() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a> DoubleEndedIterator for Lines<'a> { + #[inline] + fn next_back(&mut self) -> Option<&'a str> { + self.0.next_back() + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Lines<'_> {} + +/// Created with the method [`lines_any`]. +/// +/// [`lines_any`]: str::lines_any +#[stable(feature = "rust1", since = "1.0.0")] +#[deprecated(since = "1.4.0", note = "use lines()/Lines instead now")] +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[derive(Clone, Debug)] +#[allow(deprecated)] +pub struct LinesAny<'a>(pub(super) Lines<'a>); + +#[stable(feature = "rust1", since = "1.0.0")] +#[allow(deprecated)] +impl<'a> Iterator for LinesAny<'a> { + type Item = &'a str; + + #[inline] + fn next(&mut self) -> Option<&'a str> { + self.0.next() + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.0.size_hint() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +#[allow(deprecated)] +impl<'a> DoubleEndedIterator for LinesAny<'a> { + #[inline] + fn next_back(&mut self) -> Option<&'a str> { + self.0.next_back() + } +} + +#[stable(feature = "fused", since = "1.26.0")] +#[allow(deprecated)] +impl FusedIterator for LinesAny<'_> {} + +/// An iterator over the non-whitespace substrings of a string, +/// separated by any amount of whitespace. +/// +/// This struct is created by the [`split_whitespace`] method on [`str`]. +/// See its documentation for more. +/// +/// [`split_whitespace`]: str::split_whitespace +#[stable(feature = "split_whitespace", since = "1.1.0")] +#[derive(Clone, Debug)] +pub struct SplitWhitespace<'a> { + pub(super) inner: Filter, IsNotEmpty>, +} + +/// An iterator over the non-ASCII-whitespace substrings of a string, +/// separated by any amount of ASCII whitespace. +/// +/// This struct is created by the [`split_ascii_whitespace`] method on [`str`]. +/// See its documentation for more. +/// +/// [`split_ascii_whitespace`]: str::split_ascii_whitespace +#[stable(feature = "split_ascii_whitespace", since = "1.34.0")] +#[derive(Clone, Debug)] +pub struct SplitAsciiWhitespace<'a> { + pub(super) inner: + Map, BytesIsNotEmpty>, UnsafeBytesToStr>, +} + +/// An iterator over the substrings of a string, +/// terminated by a substring matching to a predicate function +/// Unlike `Split`, it contains the matched part as a terminator +/// of the subslice. +/// +/// This struct is created by the [`split_inclusive`] method on [`str`]. +/// See its documentation for more. +/// +/// [`split_inclusive`]: str::split_inclusive +#[stable(feature = "split_inclusive", since = "1.51.0")] +pub struct SplitInclusive<'a, P: Pattern<'a>>(pub(super) SplitInternal<'a, P>); + +#[stable(feature = "split_whitespace", since = "1.1.0")] +impl<'a> Iterator for SplitWhitespace<'a> { + type Item = &'a str; + + #[inline] + fn next(&mut self) -> Option<&'a str> { + self.inner.next() + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } + + #[inline] + fn last(mut self) -> Option<&'a str> { + self.next_back() + } +} + +#[stable(feature = "split_whitespace", since = "1.1.0")] +impl<'a> DoubleEndedIterator for SplitWhitespace<'a> { + #[inline] + fn next_back(&mut self) -> Option<&'a str> { + self.inner.next_back() + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for SplitWhitespace<'_> {} + +impl<'a> SplitWhitespace<'a> { + /// Returns remainder of the split string + /// + /// # Examples + /// + /// ``` + /// #![feature(str_split_whitespace_remainder)] + /// + /// let mut split = "Mary had a little lamb".split_whitespace(); + /// assert_eq!(split.remainder(), Some("Mary had a little lamb")); + /// + /// split.next(); + /// assert_eq!(split.remainder(), Some("had a little lamb")); + /// + /// split.by_ref().for_each(drop); + /// assert_eq!(split.remainder(), None); + /// ``` + #[inline] + #[must_use] + #[unstable(feature = "str_split_whitespace_remainder", issue = "77998")] + pub fn remainder(&self) -> Option<&'a str> { + self.inner.iter.remainder() + } +} + +#[stable(feature = "split_ascii_whitespace", since = "1.34.0")] +impl<'a> Iterator for SplitAsciiWhitespace<'a> { + type Item = &'a str; + + #[inline] + fn next(&mut self) -> Option<&'a str> { + self.inner.next() + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } + + #[inline] + fn last(mut self) -> Option<&'a str> { + self.next_back() + } +} + +#[stable(feature = "split_ascii_whitespace", since = "1.34.0")] +impl<'a> DoubleEndedIterator for SplitAsciiWhitespace<'a> { + #[inline] + fn next_back(&mut self) -> Option<&'a str> { + self.inner.next_back() + } +} + +#[stable(feature = "split_ascii_whitespace", since = "1.34.0")] +impl FusedIterator for SplitAsciiWhitespace<'_> {} + +impl<'a> SplitAsciiWhitespace<'a> { + /// Returns remainder of the split string. + /// + /// If the iterator is empty, returns `None`. + /// + /// # Examples + /// + /// ``` + /// #![feature(str_split_whitespace_remainder)] + /// + /// let mut split = "Mary had a little lamb".split_ascii_whitespace(); + /// assert_eq!(split.remainder(), Some("Mary had a little lamb")); + /// + /// split.next(); + /// assert_eq!(split.remainder(), Some("had a little lamb")); + /// + /// split.by_ref().for_each(drop); + /// assert_eq!(split.remainder(), None); + /// ``` + #[inline] + #[must_use] + #[unstable(feature = "str_split_whitespace_remainder", issue = "77998")] + pub fn remainder(&self) -> Option<&'a str> { + if self.inner.iter.iter.finished { + return None; + } + + // SAFETY: Slice is created from str. + Some(unsafe { crate::str::from_utf8_unchecked(&self.inner.iter.iter.v) }) + } +} + +#[stable(feature = "split_inclusive", since = "1.51.0")] +impl<'a, P: Pattern<'a>> Iterator for SplitInclusive<'a, P> { + type Item = &'a str; + + #[inline] + fn next(&mut self) -> Option<&'a str> { + self.0.next_inclusive() + } +} + +#[stable(feature = "split_inclusive", since = "1.51.0")] +impl<'a, P: Pattern<'a, Searcher: fmt::Debug>> fmt::Debug for SplitInclusive<'a, P> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SplitInclusive").field("0", &self.0).finish() + } +} + +// FIXME(#26925) Remove in favor of `#[derive(Clone)]` +#[stable(feature = "split_inclusive", since = "1.51.0")] +impl<'a, P: Pattern<'a, Searcher: Clone>> Clone for SplitInclusive<'a, P> { + fn clone(&self) -> Self { + SplitInclusive(self.0.clone()) + } +} + +#[stable(feature = "split_inclusive", since = "1.51.0")] +impl<'a, P: Pattern<'a, Searcher: ReverseSearcher<'a>>> DoubleEndedIterator + for SplitInclusive<'a, P> +{ + #[inline] + fn next_back(&mut self) -> Option<&'a str> { + self.0.next_back_inclusive() + } +} + +#[stable(feature = "split_inclusive", since = "1.51.0")] +impl<'a, P: Pattern<'a>> FusedIterator for SplitInclusive<'a, P> {} + +impl<'a, P: Pattern<'a>> SplitInclusive<'a, P> { + /// Returns remainder of the split string. + /// + /// If the iterator is empty, returns `None`. + /// + /// # Examples + /// + /// ``` + /// #![feature(str_split_inclusive_remainder)] + /// let mut split = "Mary had a little lamb".split_inclusive(' '); + /// assert_eq!(split.remainder(), Some("Mary had a little lamb")); + /// split.next(); + /// assert_eq!(split.remainder(), Some("had a little lamb")); + /// split.by_ref().for_each(drop); + /// assert_eq!(split.remainder(), None); + /// ``` + #[inline] + #[unstable(feature = "str_split_inclusive_remainder", issue = "77998")] + pub fn remainder(&self) -> Option<&'a str> { + self.0.remainder() + } +} + +/// An iterator of [`u16`] over the string encoded as UTF-16. +/// +/// This struct is created by the [`encode_utf16`] method on [`str`]. +/// See its documentation for more. +/// +/// [`encode_utf16`]: str::encode_utf16 +#[derive(Clone)] +#[stable(feature = "encode_utf16", since = "1.8.0")] +pub struct EncodeUtf16<'a> { + pub(super) chars: Chars<'a>, + pub(super) extra: u16, +} + +#[stable(feature = "collection_debug", since = "1.17.0")] +impl fmt::Debug for EncodeUtf16<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EncodeUtf16").finish_non_exhaustive() + } +} + +#[stable(feature = "encode_utf16", since = "1.8.0")] +impl<'a> Iterator for EncodeUtf16<'a> { + type Item = u16; + + #[inline] + fn next(&mut self) -> Option { + if self.extra != 0 { + let tmp = self.extra; + self.extra = 0; + return Some(tmp); + } + + let mut buf = [0; 2]; + self.chars.next().map(|ch| { + let n = ch.encode_utf16(&mut buf).len(); + if n == 2 { + self.extra = buf[1]; + } + buf[0] + }) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let (low, high) = self.chars.size_hint(); + // every char gets either one u16 or two u16, + // so this iterator is between 1 or 2 times as + // long as the underlying iterator. + (low, high.and_then(|n| n.checked_mul(2))) + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for EncodeUtf16<'_> {} + +/// The return type of [`str::escape_debug`]. +#[stable(feature = "str_escape", since = "1.34.0")] +#[derive(Clone, Debug)] +pub struct EscapeDebug<'a> { + pub(super) inner: Chain< + Flatten>, + FlatMap, char::EscapeDebug, CharEscapeDebugContinue>, + >, +} + +/// The return type of [`str::escape_default`]. +#[stable(feature = "str_escape", since = "1.34.0")] +#[derive(Clone, Debug)] +pub struct EscapeDefault<'a> { + pub(super) inner: FlatMap, char::EscapeDefault, CharEscapeDefault>, +} + +/// The return type of [`str::escape_unicode`]. +#[stable(feature = "str_escape", since = "1.34.0")] +#[derive(Clone, Debug)] +pub struct EscapeUnicode<'a> { + pub(super) inner: FlatMap, char::EscapeUnicode, CharEscapeUnicode>, +} + +macro_rules! escape_types_impls { + ($( $Name: ident ),+) => {$( + #[stable(feature = "str_escape", since = "1.34.0")] + impl<'a> fmt::Display for $Name<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.clone().try_for_each(|c| f.write_char(c)) + } + } + + #[stable(feature = "str_escape", since = "1.34.0")] + impl<'a> Iterator for $Name<'a> { + type Item = char; + + #[inline] + fn next(&mut self) -> Option { self.inner.next() } + + #[inline] + fn size_hint(&self) -> (usize, Option) { self.inner.size_hint() } + + #[inline] + fn try_fold(&mut self, init: Acc, fold: Fold) -> R where + Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, R: Try + { + self.inner.try_fold(init, fold) + } + + #[inline] + fn fold(self, init: Acc, fold: Fold) -> Acc + where Fold: FnMut(Acc, Self::Item) -> Acc, + { + self.inner.fold(init, fold) + } + } + + #[stable(feature = "str_escape", since = "1.34.0")] + impl<'a> FusedIterator for $Name<'a> {} + )+} +} + +escape_types_impls!(EscapeDebug, EscapeDefault, EscapeUnicode); diff --git a/crux-mir/lib/core/src/str/lossy.rs b/crux-mir/lib/core/src/str/lossy.rs index 88b2bc551..59f873d12 100644 --- a/crux-mir/lib/core/src/str/lossy.rs +++ b/crux-mir/lib/core/src/str/lossy.rs @@ -1,51 +1,170 @@ -use crate::char; -use crate::fmt::{self, Write}; -use crate::mem; -use crate::str as core_str; +use crate::fmt; +use crate::fmt::Formatter; +use crate::fmt::Write; +use crate::iter::FusedIterator; -/// Lossy UTF-8 string. -#[unstable(feature = "str_internals", issue = "none")] -pub struct Utf8Lossy { - bytes: [u8], +use super::from_utf8_unchecked; +use super::validations::utf8_char_width; + +/// An item returned by the [`Utf8Chunks`] iterator. +/// +/// A `Utf8Chunk` stores a sequence of [`u8`] up to the first broken character +/// when decoding a UTF-8 string. +/// +/// # Examples +/// +/// ``` +/// #![feature(utf8_chunks)] +/// +/// use std::str::Utf8Chunks; +/// +/// // An invalid UTF-8 string +/// let bytes = b"foo\xF1\x80bar"; +/// +/// // Decode the first `Utf8Chunk` +/// let chunk = Utf8Chunks::new(bytes).next().unwrap(); +/// +/// // The first three characters are valid UTF-8 +/// assert_eq!("foo", chunk.valid()); +/// +/// // The fourth character is broken +/// assert_eq!(b"\xF1\x80", chunk.invalid()); +/// ``` +#[unstable(feature = "utf8_chunks", issue = "99543")] +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Utf8Chunk<'a> { + valid: &'a str, + invalid: &'a [u8], } -impl Utf8Lossy { - pub fn from_str(s: &str) -> &Utf8Lossy { - Utf8Lossy::from_bytes(s.as_bytes()) +impl<'a> Utf8Chunk<'a> { + /// Returns the next validated UTF-8 substring. + /// + /// This substring can be empty at the start of the string or between + /// broken UTF-8 characters. + #[must_use] + #[unstable(feature = "utf8_chunks", issue = "99543")] + pub fn valid(&self) -> &'a str { + self.valid } - pub fn from_bytes(bytes: &[u8]) -> &Utf8Lossy { - // SAFETY: Both use the same memory layout, and UTF-8 correctness isn't required. - unsafe { mem::transmute(bytes) } + /// Returns the invalid sequence that caused a failure. + /// + /// The returned slice will have a maximum length of 3 and starts after the + /// substring given by [`valid`]. Decoding will resume after this sequence. + /// + /// If empty, this is the last chunk in the string. If non-empty, an + /// unexpected byte was encountered or the end of the input was reached + /// unexpectedly. + /// + /// Lossy decoding would replace this sequence with [`U+FFFD REPLACEMENT + /// CHARACTER`]. + /// + /// [`valid`]: Self::valid + /// [`U+FFFD REPLACEMENT CHARACTER`]: crate::char::REPLACEMENT_CHARACTER + #[must_use] + #[unstable(feature = "utf8_chunks", issue = "99543")] + pub fn invalid(&self) -> &'a [u8] { + self.invalid } +} + +#[must_use] +#[unstable(feature = "str_internals", issue = "none")] +pub struct Debug<'a>(&'a [u8]); + +#[unstable(feature = "str_internals", issue = "none")] +impl fmt::Debug for Debug<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.write_char('"')?; + + for chunk in Utf8Chunks::new(self.0) { + // Valid part. + // Here we partially parse UTF-8 again which is suboptimal. + { + let valid = chunk.valid(); + let mut from = 0; + for (i, c) in valid.char_indices() { + let esc = c.escape_debug(); + // If char needs escaping, flush backlog so far and write, else skip + if esc.len() != 1 { + f.write_str(&valid[from..i])?; + for c in esc { + f.write_char(c)?; + } + from = i + c.len_utf8(); + } + } + f.write_str(&valid[from..])?; + } + + // Broken parts of string as hex escape. + for &b in chunk.invalid() { + write!(f, "\\x{:02X}", b)?; + } + } - pub fn chunks(&self) -> Utf8LossyChunksIter<'_> { - Utf8LossyChunksIter { source: &self.bytes } + f.write_char('"') } } -/// Iterator over lossy UTF-8 string -#[unstable(feature = "str_internals", issue = "none")] -#[allow(missing_debug_implementations)] -pub struct Utf8LossyChunksIter<'a> { +/// An iterator used to decode a slice of mostly UTF-8 bytes to string slices +/// ([`&str`]) and byte slices ([`&[u8]`][byteslice]). +/// +/// If you want a simple conversion from UTF-8 byte slices to string slices, +/// [`from_utf8`] is easier to use. +/// +/// [byteslice]: slice +/// [`from_utf8`]: super::from_utf8 +/// +/// # Examples +/// +/// This can be used to create functionality similar to +/// [`String::from_utf8_lossy`] without allocating heap memory: +/// +/// ``` +/// #![feature(utf8_chunks)] +/// +/// use std::str::Utf8Chunks; +/// +/// fn from_utf8_lossy(input: &[u8], mut push: F) where F: FnMut(&str) { +/// for chunk in Utf8Chunks::new(input) { +/// push(chunk.valid()); +/// +/// if !chunk.invalid().is_empty() { +/// push("\u{FFFD}"); +/// } +/// } +/// } +/// ``` +/// +/// [`String::from_utf8_lossy`]: ../../std/string/struct.String.html#method.from_utf8_lossy +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[unstable(feature = "utf8_chunks", issue = "99543")] +#[derive(Clone)] +pub struct Utf8Chunks<'a> { source: &'a [u8], } -#[unstable(feature = "str_internals", issue = "none")] -#[derive(PartialEq, Eq, Debug)] -pub struct Utf8LossyChunk<'a> { - /// Sequence of valid chars. - /// Can be empty between broken UTF-8 chars. - pub valid: &'a str, - /// Single broken char, empty if none. - /// Empty iff iterator item is last. - pub broken: &'a [u8], +impl<'a> Utf8Chunks<'a> { + /// Creates a new iterator to decode the bytes. + #[unstable(feature = "utf8_chunks", issue = "99543")] + pub fn new(bytes: &'a [u8]) -> Self { + Self { source: bytes } + } + + #[doc(hidden)] + #[unstable(feature = "str_internals", issue = "none")] + pub fn debug(&self) -> Debug<'_> { + Debug(self.source) + } } -impl<'a> Iterator for Utf8LossyChunksIter<'a> { - type Item = Utf8LossyChunk<'a>; +#[unstable(feature = "utf8_chunks", issue = "99543")] +impl<'a> Iterator for Utf8Chunks<'a> { + type Item = Utf8Chunk<'a>; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option> { if self.source.is_empty() { return None; } @@ -56,36 +175,26 @@ impl<'a> Iterator for Utf8LossyChunksIter<'a> { } let mut i = 0; + let mut valid_up_to = 0; while i < self.source.len() { - let i_ = i; - - // SAFETY: `i` starts at `0`, is less than `self.source.len()`, and - // only increases, so `0 <= i < self.source.len()`. + // SAFETY: `i < self.source.len()` per previous line. + // For some reason the following are both significantly slower: + // while let Some(&byte) = self.source.get(i) { + // while let Some(byte) = self.source.get(i).copied() { let byte = unsafe { *self.source.get_unchecked(i) }; i += 1; if byte < 128 { + // This could be a `1 => ...` case in the match below, but for + // the common case of all-ASCII inputs, we bypass loading the + // sizeable UTF8_CHAR_WIDTH table into cache. } else { - let w = core_str::utf8_char_width(byte); - - macro_rules! error { - () => {{ - // SAFETY: We have checked up to `i` that source is valid UTF-8. - unsafe { - let r = Utf8LossyChunk { - valid: core_str::from_utf8_unchecked(&self.source[0..i_]), - broken: &self.source[i_..i], - }; - self.source = &self.source[i..]; - return Some(r); - } - }}; - } + let w = utf8_char_width(byte); match w { 2 => { if safe_get(self.source, i) & 192 != TAG_CONT_U8 { - error!(); + break; } i += 1; } @@ -95,13 +204,11 @@ impl<'a> Iterator for Utf8LossyChunksIter<'a> { (0xE1..=0xEC, 0x80..=0xBF) => (), (0xED, 0x80..=0x9F) => (), (0xEE..=0xEF, 0x80..=0xBF) => (), - _ => { - error!(); - } + _ => break, } i += 1; if safe_get(self.source, i) & 192 != TAG_CONT_U8 { - error!(); + break; } i += 1; } @@ -110,92 +217,54 @@ impl<'a> Iterator for Utf8LossyChunksIter<'a> { (0xF0, 0x90..=0xBF) => (), (0xF1..=0xF3, 0x80..=0xBF) => (), (0xF4, 0x80..=0x8F) => (), - _ => { - error!(); - } + _ => break, } i += 1; if safe_get(self.source, i) & 192 != TAG_CONT_U8 { - error!(); + break; } i += 1; if safe_get(self.source, i) & 192 != TAG_CONT_U8 { - error!(); + break; } i += 1; } - _ => { - error!(); - } + _ => break, } } - } - let r = Utf8LossyChunk { - // SAFETY: We have checked that the entire source is valid UTF-8. - valid: unsafe { core_str::from_utf8_unchecked(self.source) }, - broken: &[], - }; - self.source = &[]; - Some(r) - } -} - -impl fmt::Display for Utf8Lossy { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // If we're the empty string then our iterator won't actually yield - // anything, so perform the formatting manually - if self.bytes.is_empty() { - return "".fmt(f); + valid_up_to = i; } - for Utf8LossyChunk { valid, broken } in self.chunks() { - // If we successfully decoded the whole chunk as a valid string then - // we can return a direct formatting of the string which will also - // respect various formatting flags if possible. - if valid.len() == self.bytes.len() { - assert!(broken.is_empty()); - return valid.fmt(f); - } + // SAFETY: `i <= self.source.len()` because it is only ever incremented + // via `i += 1` and in between every single one of those increments, `i` + // is compared against `self.source.len()`. That happens either + // literally by `i < self.source.len()` in the while-loop's condition, + // or indirectly by `safe_get(self.source, i) & 192 != TAG_CONT_U8`. The + // loop is terminated as soon as the latest `i += 1` has made `i` no + // longer less than `self.source.len()`, which means it'll be at most + // equal to `self.source.len()`. + let (inspected, remaining) = unsafe { self.source.split_at_unchecked(i) }; + self.source = remaining; - f.write_str(valid)?; - if !broken.is_empty() { - f.write_char(char::REPLACEMENT_CHARACTER)?; - } - } - Ok(()) + // SAFETY: `valid_up_to <= i` because it is only ever assigned via + // `valid_up_to = i` and `i` only increases. + let (valid, invalid) = unsafe { inspected.split_at_unchecked(valid_up_to) }; + + Some(Utf8Chunk { + // SAFETY: All bytes up to `valid_up_to` are valid UTF-8. + valid: unsafe { from_utf8_unchecked(valid) }, + invalid, + }) } } -impl fmt::Debug for Utf8Lossy { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_char('"')?; - - for Utf8LossyChunk { valid, broken } in self.chunks() { - // Valid part. - // Here we partially parse UTF-8 again which is suboptimal. - { - let mut from = 0; - for (i, c) in valid.char_indices() { - let esc = c.escape_debug(); - // If char needs escaping, flush backlog so far and write, else skip - if esc.len() != 1 { - f.write_str(&valid[from..i])?; - for c in esc { - f.write_char(c)?; - } - from = i + c.len_utf8(); - } - } - f.write_str(&valid[from..])?; - } - - // Broken parts of string as hex escape. - for &b in broken { - write!(f, "\\x{:02x}", b)?; - } - } +#[unstable(feature = "utf8_chunks", issue = "99543")] +impl FusedIterator for Utf8Chunks<'_> {} - f.write_char('"') +#[unstable(feature = "utf8_chunks", issue = "99543")] +impl fmt::Debug for Utf8Chunks<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("Utf8Chunks").field("source", &self.debug()).finish() } } diff --git a/crux-mir/lib/core/src/str/mod.rs b/crux-mir/lib/core/src/str/mod.rs index be27ca3da..ab2f8520e 100644 --- a/crux-mir/lib/core/src/str/mod.rs +++ b/crux-mir/lib/core/src/str/mod.rs @@ -1,2199 +1,112 @@ -// ignore-tidy-filelength - -//! String manipulation. -//! -//! For more details, see the [`std::str`] module. -//! -//! [`std::str`]: ../../std/str/index.html - -#![stable(feature = "rust1", since = "1.0.0")] - -use self::pattern::Pattern; -use self::pattern::{DoubleEndedSearcher, ReverseSearcher, SearchStep, Searcher}; - -use crate::char; -use crate::fmt::{self, Write}; -use crate::iter::{Chain, FlatMap, Flatten}; -use crate::iter::{Copied, Filter, FusedIterator, Map, TrustedLen, TrustedRandomAccess}; -use crate::mem; -use crate::ops::Try; -use crate::option; -use crate::slice::{self, SliceIndex, Split as SliceSplit}; - -pub mod pattern; - -#[unstable(feature = "str_internals", issue = "none")] -#[allow(missing_docs)] -pub mod lossy; - -/// Parse a value from a string -/// -/// `FromStr`'s [`from_str`] method is often used implicitly, through -/// [`str`]'s [`parse`] method. See [`parse`]'s documentation for examples. -/// -/// [`from_str`]: #tymethod.from_str -/// [`str`]: ../../std/primitive.str.html -/// [`parse`]: ../../std/primitive.str.html#method.parse -/// -/// `FromStr` does not have a lifetime parameter, and so you can only parse types -/// that do not contain a lifetime parameter themselves. In other words, you can -/// parse an `i32` with `FromStr`, but not a `&i32`. You can parse a struct that -/// contains an `i32`, but not one that contains an `&i32`. -/// -/// # Examples -/// -/// Basic implementation of `FromStr` on an example `Point` type: -/// -/// ``` -/// use std::str::FromStr; -/// use std::num::ParseIntError; -/// -/// #[derive(Debug, PartialEq)] -/// struct Point { -/// x: i32, -/// y: i32 -/// } -/// -/// impl FromStr for Point { -/// type Err = ParseIntError; -/// -/// fn from_str(s: &str) -> Result { -/// let coords: Vec<&str> = s.trim_matches(|p| p == '(' || p == ')' ) -/// .split(',') -/// .collect(); -/// -/// let x_fromstr = coords[0].parse::()?; -/// let y_fromstr = coords[1].parse::()?; -/// -/// Ok(Point { x: x_fromstr, y: y_fromstr }) -/// } -/// } -/// -/// let p = Point::from_str("(1,2)"); -/// assert_eq!(p.unwrap(), Point{ x: 1, y: 2} ) -/// ``` -#[stable(feature = "rust1", since = "1.0.0")] -pub trait FromStr: Sized { - /// The associated error which can be returned from parsing. - #[stable(feature = "rust1", since = "1.0.0")] - type Err; - - /// Parses a string `s` to return a value of this type. - /// - /// If parsing succeeds, return the value inside [`Ok`], otherwise - /// when the string is ill-formatted return an error specific to the - /// inside [`Err`]. The error type is specific to implementation of the trait. - /// - /// [`Ok`]: ../../std/result/enum.Result.html#variant.Ok - /// [`Err`]: ../../std/result/enum.Result.html#variant.Err - /// - /// # Examples - /// - /// Basic usage with [`i32`][ithirtytwo], a type that implements `FromStr`: - /// - /// [ithirtytwo]: ../../std/primitive.i32.html - /// - /// ``` - /// use std::str::FromStr; - /// - /// let s = "5"; - /// let x = i32::from_str(s).unwrap(); - /// - /// assert_eq!(5, x); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - fn from_str(s: &str) -> Result; -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl FromStr for bool { - type Err = ParseBoolError; - - /// Parse a `bool` from a string. - /// - /// Yields a `Result`, because `s` may or may not - /// actually be parseable. - /// - /// # Examples - /// - /// ``` - /// use std::str::FromStr; - /// - /// assert_eq!(FromStr::from_str("true"), Ok(true)); - /// assert_eq!(FromStr::from_str("false"), Ok(false)); - /// assert!(::from_str("not even a boolean").is_err()); - /// ``` - /// - /// Note, in many cases, the `.parse()` method on `str` is more proper. - /// - /// ``` - /// assert_eq!("true".parse(), Ok(true)); - /// assert_eq!("false".parse(), Ok(false)); - /// assert!("not even a boolean".parse::().is_err()); - /// ``` - #[inline] - fn from_str(s: &str) -> Result { - match s { - "true" => Ok(true), - "false" => Ok(false), - _ => Err(ParseBoolError { _priv: () }), - } - } -} - -/// An error returned when parsing a `bool` using [`from_str`] fails -/// -/// [`from_str`]: ../../std/primitive.bool.html#method.from_str -#[derive(Debug, Clone, PartialEq, Eq)] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct ParseBoolError { - _priv: (), -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl fmt::Display for ParseBoolError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - "provided string was not `true` or `false`".fmt(f) - } -} - -/* -Section: Creating a string -*/ - -/// Errors which can occur when attempting to interpret a sequence of [`u8`] -/// as a string. -/// -/// [`u8`]: ../../std/primitive.u8.html -/// -/// As such, the `from_utf8` family of functions and methods for both [`String`]s -/// and [`&str`]s make use of this error, for example. -/// -/// [`String`]: ../../std/string/struct.String.html#method.from_utf8 -/// [`&str`]: ../../std/str/fn.from_utf8.html -/// -/// # Examples -/// -/// This error type’s methods can be used to create functionality -/// similar to `String::from_utf8_lossy` without allocating heap memory: -/// -/// ``` -/// fn from_utf8_lossy(mut input: &[u8], mut push: F) where F: FnMut(&str) { -/// loop { -/// match std::str::from_utf8(input) { -/// Ok(valid) => { -/// push(valid); -/// break -/// } -/// Err(error) => { -/// let (valid, after_valid) = input.split_at(error.valid_up_to()); -/// unsafe { -/// push(std::str::from_utf8_unchecked(valid)) -/// } -/// push("\u{FFFD}"); -/// -/// if let Some(invalid_sequence_length) = error.error_len() { -/// input = &after_valid[invalid_sequence_length..] -/// } else { -/// break -/// } -/// } -/// } -/// } -/// } -/// ``` -#[derive(Copy, Eq, PartialEq, Clone, Debug)] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct Utf8Error { - valid_up_to: usize, - error_len: Option, -} - -impl Utf8Error { - /// Returns the index in the given string up to which valid UTF-8 was - /// verified. - /// - /// It is the maximum index such that `from_utf8(&input[..index])` - /// would return `Ok(_)`. - /// - /// # Examples - /// - /// Basic usage: - /// - /// ``` - /// use std::str; - /// - /// // some invalid bytes, in a vector - /// let sparkle_heart = vec![0, 159, 146, 150]; - /// - /// // std::str::from_utf8 returns a Utf8Error - /// let error = str::from_utf8(&sparkle_heart).unwrap_err(); - /// - /// // the second byte is invalid here - /// assert_eq!(1, error.valid_up_to()); - /// ``` - #[stable(feature = "utf8_error", since = "1.5.0")] - pub fn valid_up_to(&self) -> usize { - self.valid_up_to - } - - /// Provides more information about the failure: - /// - /// * `None`: the end of the input was reached unexpectedly. - /// `self.valid_up_to()` is 1 to 3 bytes from the end of the input. - /// If a byte stream (such as a file or a network socket) is being decoded incrementally, - /// this could be a valid `char` whose UTF-8 byte sequence is spanning multiple chunks. - /// - /// * `Some(len)`: an unexpected byte was encountered. - /// The length provided is that of the invalid byte sequence - /// that starts at the index given by `valid_up_to()`. - /// Decoding should resume after that sequence - /// (after inserting a [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]) in case of - /// lossy decoding. - /// - /// [U+FFFD]: ../../std/char/constant.REPLACEMENT_CHARACTER.html - #[stable(feature = "utf8_error_error_len", since = "1.20.0")] - pub fn error_len(&self) -> Option { - self.error_len.map(|len| len as usize) - } -} - -/// Converts a slice of bytes to a string slice. -/// -/// A string slice ([`&str`]) is made of bytes ([`u8`]), and a byte slice -/// ([`&[u8]`][byteslice]) is made of bytes, so this function converts between -/// the two. Not all byte slices are valid string slices, however: [`&str`] requires -/// that it is valid UTF-8. `from_utf8()` checks to ensure that the bytes are valid -/// UTF-8, and then does the conversion. -/// -/// [`&str`]: ../../std/primitive.str.html -/// [`u8`]: ../../std/primitive.u8.html -/// [byteslice]: ../../std/primitive.slice.html -/// -/// If you are sure that the byte slice is valid UTF-8, and you don't want to -/// incur the overhead of the validity check, there is an unsafe version of -/// this function, [`from_utf8_unchecked`][fromutf8u], which has the same -/// behavior but skips the check. -/// -/// [fromutf8u]: fn.from_utf8_unchecked.html -/// -/// If you need a `String` instead of a `&str`, consider -/// [`String::from_utf8`][string]. -/// -/// [string]: ../../std/string/struct.String.html#method.from_utf8 -/// -/// Because you can stack-allocate a `[u8; N]`, and you can take a -/// [`&[u8]`][byteslice] of it, this function is one way to have a -/// stack-allocated string. There is an example of this in the -/// examples section below. -/// -/// [byteslice]: ../../std/primitive.slice.html -/// -/// # Errors -/// -/// Returns `Err` if the slice is not UTF-8 with a description as to why the -/// provided slice is not UTF-8. -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// use std::str; -/// -/// // some bytes, in a vector -/// let sparkle_heart = vec![240, 159, 146, 150]; -/// -/// // We know these bytes are valid, so just use `unwrap()`. -/// let sparkle_heart = str::from_utf8(&sparkle_heart).unwrap(); -/// -/// assert_eq!("💖", sparkle_heart); -/// ``` -/// -/// Incorrect bytes: -/// -/// ``` -/// use std::str; -/// -/// // some invalid bytes, in a vector -/// let sparkle_heart = vec![0, 159, 146, 150]; -/// -/// assert!(str::from_utf8(&sparkle_heart).is_err()); -/// ``` -/// -/// See the docs for [`Utf8Error`][error] for more details on the kinds of -/// errors that can be returned. -/// -/// [error]: struct.Utf8Error.html -/// -/// A "stack allocated string": -/// -/// ``` -/// use std::str; -/// -/// // some bytes, in a stack-allocated array -/// let sparkle_heart = [240, 159, 146, 150]; -/// -/// // We know these bytes are valid, so just use `unwrap()`. -/// let sparkle_heart = str::from_utf8(&sparkle_heart).unwrap(); -/// -/// assert_eq!("💖", sparkle_heart); -/// ``` -#[stable(feature = "rust1", since = "1.0.0")] -pub fn from_utf8(v: &[u8]) -> Result<&str, Utf8Error> { - run_utf8_validation(v)?; - // SAFETY: Just ran validation. - Ok(unsafe { from_utf8_unchecked(v) }) -} - -/// Converts a mutable slice of bytes to a mutable string slice. -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// use std::str; -/// -/// // "Hello, Rust!" as a mutable vector -/// let mut hellorust = vec![72, 101, 108, 108, 111, 44, 32, 82, 117, 115, 116, 33]; -/// -/// // As we know these bytes are valid, we can use `unwrap()` -/// let outstr = str::from_utf8_mut(&mut hellorust).unwrap(); -/// -/// assert_eq!("Hello, Rust!", outstr); -/// ``` -/// -/// Incorrect bytes: -/// -/// ``` -/// use std::str; -/// -/// // Some invalid bytes in a mutable vector -/// let mut invalid = vec![128, 223]; -/// -/// assert!(str::from_utf8_mut(&mut invalid).is_err()); -/// ``` -/// See the docs for [`Utf8Error`][error] for more details on the kinds of -/// errors that can be returned. -/// -/// [error]: struct.Utf8Error.html -#[stable(feature = "str_mut_extras", since = "1.20.0")] -pub fn from_utf8_mut(v: &mut [u8]) -> Result<&mut str, Utf8Error> { - run_utf8_validation(v)?; - // SAFETY: Just ran validation. - Ok(unsafe { from_utf8_unchecked_mut(v) }) -} - -/// Converts a slice of bytes to a string slice without checking -/// that the string contains valid UTF-8. -/// -/// See the safe version, [`from_utf8`][fromutf8], for more information. -/// -/// [fromutf8]: fn.from_utf8.html -/// -/// # Safety -/// -/// This function is unsafe because it does not check that the bytes passed to -/// it are valid UTF-8. If this constraint is violated, undefined behavior -/// results, as the rest of Rust assumes that [`&str`]s are valid UTF-8. -/// -/// [`&str`]: ../../std/primitive.str.html -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// use std::str; -/// -/// // some bytes, in a vector -/// let sparkle_heart = vec![240, 159, 146, 150]; -/// -/// let sparkle_heart = unsafe { -/// str::from_utf8_unchecked(&sparkle_heart) -/// }; -/// -/// assert_eq!("💖", sparkle_heart); -/// ``` -#[inline] -#[stable(feature = "rust1", since = "1.0.0")] -pub unsafe fn from_utf8_unchecked(v: &[u8]) -> &str { - &*(v as *const [u8] as *const str) -} - -/// Converts a slice of bytes to a string slice without checking -/// that the string contains valid UTF-8; mutable version. -/// -/// See the immutable version, [`from_utf8_unchecked()`][fromutf8], for more information. -/// -/// [fromutf8]: fn.from_utf8_unchecked.html -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// use std::str; -/// -/// let mut heart = vec![240, 159, 146, 150]; -/// let heart = unsafe { str::from_utf8_unchecked_mut(&mut heart) }; -/// -/// assert_eq!("💖", heart); -/// ``` -#[inline] -#[stable(feature = "str_mut_extras", since = "1.20.0")] -pub unsafe fn from_utf8_unchecked_mut(v: &mut [u8]) -> &mut str { - &mut *(v as *mut [u8] as *mut str) -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl fmt::Display for Utf8Error { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if let Some(error_len) = self.error_len { - write!( - f, - "invalid utf-8 sequence of {} bytes from index {}", - error_len, self.valid_up_to - ) - } else { - write!(f, "incomplete utf-8 byte sequence from index {}", self.valid_up_to) - } - } -} - -/* -Section: Iterators -*/ - -/// An iterator over the [`char`]s of a string slice. -/// -/// [`char`]: ../../std/primitive.char.html -/// -/// This struct is created by the [`chars`] method on [`str`]. -/// See its documentation for more. -/// -/// [`chars`]: ../../std/primitive.str.html#method.chars -/// [`str`]: ../../std/primitive.str.html -#[derive(Clone)] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct Chars<'a> { - iter: slice::Iter<'a, u8>, -} - -/// Returns the initial codepoint accumulator for the first byte. -/// The first byte is special, only want bottom 5 bits for width 2, 4 bits -/// for width 3, and 3 bits for width 4. -#[inline] -fn utf8_first_byte(byte: u8, width: u32) -> u32 { - (byte & (0x7F >> width)) as u32 -} - -/// Returns the value of `ch` updated with continuation byte `byte`. -#[inline] -fn utf8_acc_cont_byte(ch: u32, byte: u8) -> u32 { - (ch << 6) | (byte & CONT_MASK) as u32 -} - -/// Checks whether the byte is a UTF-8 continuation byte (i.e., starts with the -/// bits `10`). -#[inline] -fn utf8_is_cont_byte(byte: u8) -> bool { - (byte & !CONT_MASK) == TAG_CONT_U8 -} - -#[inline] -fn unwrap_or_0(opt: Option<&u8>) -> u8 { - match opt { - Some(&byte) => byte, - None => 0, - } -} - -/// Reads the next code point out of a byte iterator (assuming a -/// UTF-8-like encoding). -#[unstable(feature = "str_internals", issue = "none")] -#[inline] -pub fn next_code_point<'a, I: Iterator>(bytes: &mut I) -> Option { - // Decode UTF-8 - let x = *bytes.next()?; - if x < 128 { - return Some(x as u32); - } - - // Multibyte case follows - // Decode from a byte combination out of: [[[x y] z] w] - // NOTE: Performance is sensitive to the exact formulation here - let init = utf8_first_byte(x, 2); - let y = unwrap_or_0(bytes.next()); - let mut ch = utf8_acc_cont_byte(init, y); - if x >= 0xE0 { - // [[x y z] w] case - // 5th bit in 0xE0 .. 0xEF is always clear, so `init` is still valid - let z = unwrap_or_0(bytes.next()); - let y_z = utf8_acc_cont_byte((y & CONT_MASK) as u32, z); - ch = init << 12 | y_z; - if x >= 0xF0 { - // [x y z w] case - // use only the lower 3 bits of `init` - let w = unwrap_or_0(bytes.next()); - ch = (init & 7) << 18 | utf8_acc_cont_byte(y_z, w); - } - } - - Some(ch) -} - -/// Reads the last code point out of a byte iterator (assuming a -/// UTF-8-like encoding). -#[inline] -fn next_code_point_reverse<'a, I>(bytes: &mut I) -> Option -where - I: DoubleEndedIterator, -{ - // Decode UTF-8 - let w = match *bytes.next_back()? { - next_byte if next_byte < 128 => return Some(next_byte as u32), - back_byte => back_byte, - }; - - // Multibyte case follows - // Decode from a byte combination out of: [x [y [z w]]] - let mut ch; - let z = unwrap_or_0(bytes.next_back()); - ch = utf8_first_byte(z, 2); - if utf8_is_cont_byte(z) { - let y = unwrap_or_0(bytes.next_back()); - ch = utf8_first_byte(y, 3); - if utf8_is_cont_byte(y) { - let x = unwrap_or_0(bytes.next_back()); - ch = utf8_first_byte(x, 4); - ch = utf8_acc_cont_byte(ch, y); - } - ch = utf8_acc_cont_byte(ch, z); - } - ch = utf8_acc_cont_byte(ch, w); - - Some(ch) -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a> Iterator for Chars<'a> { - type Item = char; - - #[inline] - fn next(&mut self) -> Option { - next_code_point(&mut self.iter).map(|ch| { - // SAFETY: `str` invariant says `ch` is a valid Unicode Scalar Value. - unsafe { char::from_u32_unchecked(ch) } - }) - } - - #[inline] - fn count(self) -> usize { - // length in `char` is equal to the number of non-continuation bytes - let bytes_len = self.iter.len(); - let mut cont_bytes = 0; - for &byte in self.iter { - cont_bytes += utf8_is_cont_byte(byte) as usize; - } - bytes_len - cont_bytes - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let len = self.iter.len(); - // `(len + 3)` can't overflow, because we know that the `slice::Iter` - // belongs to a slice in memory which has a maximum length of - // `isize::MAX` (that's well below `usize::MAX`). - ((len + 3) / 4, Some(len)) - } - - #[inline] - fn last(mut self) -> Option { - // No need to go through the entire string. - self.next_back() - } -} - -#[stable(feature = "chars_debug_impl", since = "1.38.0")] -impl fmt::Debug for Chars<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Chars(")?; - f.debug_list().entries(self.clone()).finish()?; - write!(f, ")")?; - Ok(()) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a> DoubleEndedIterator for Chars<'a> { - #[inline] - fn next_back(&mut self) -> Option { - next_code_point_reverse(&mut self.iter).map(|ch| { - // SAFETY: `str` invariant says `ch` is a valid Unicode Scalar Value. - unsafe { char::from_u32_unchecked(ch) } - }) - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Chars<'_> {} - -impl<'a> Chars<'a> { - /// Views the underlying data as a subslice of the original data. - /// - /// This has the same lifetime as the original slice, and so the - /// iterator can continue to be used while this exists. - /// - /// # Examples - /// - /// ``` - /// let mut chars = "abc".chars(); - /// - /// assert_eq!(chars.as_str(), "abc"); - /// chars.next(); - /// assert_eq!(chars.as_str(), "bc"); - /// chars.next(); - /// chars.next(); - /// assert_eq!(chars.as_str(), ""); - /// ``` - #[stable(feature = "iter_to_slice", since = "1.4.0")] - #[inline] - pub fn as_str(&self) -> &'a str { - // SAFETY: `Chars` is only made from a str, which guarantees the iter is valid UTF-8. - unsafe { from_utf8_unchecked(self.iter.as_slice()) } - } -} - -/// An iterator over the [`char`]s of a string slice, and their positions. -/// -/// [`char`]: ../../std/primitive.char.html -/// -/// This struct is created by the [`char_indices`] method on [`str`]. -/// See its documentation for more. -/// -/// [`char_indices`]: ../../std/primitive.str.html#method.char_indices -/// [`str`]: ../../std/primitive.str.html -#[derive(Clone, Debug)] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct CharIndices<'a> { - front_offset: usize, - iter: Chars<'a>, -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a> Iterator for CharIndices<'a> { - type Item = (usize, char); - - #[inline] - fn next(&mut self) -> Option<(usize, char)> { - let pre_len = self.iter.iter.len(); - match self.iter.next() { - None => None, - Some(ch) => { - let index = self.front_offset; - let len = self.iter.iter.len(); - self.front_offset += pre_len - len; - Some((index, ch)) - } - } - } - - #[inline] - fn count(self) -> usize { - self.iter.count() - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() - } - - #[inline] - fn last(mut self) -> Option<(usize, char)> { - // No need to go through the entire string. - self.next_back() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a> DoubleEndedIterator for CharIndices<'a> { - #[inline] - fn next_back(&mut self) -> Option<(usize, char)> { - self.iter.next_back().map(|ch| { - let index = self.front_offset + self.iter.iter.len(); - (index, ch) - }) - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for CharIndices<'_> {} - -impl<'a> CharIndices<'a> { - /// Views the underlying data as a subslice of the original data. - /// - /// This has the same lifetime as the original slice, and so the - /// iterator can continue to be used while this exists. - #[stable(feature = "iter_to_slice", since = "1.4.0")] - #[inline] - pub fn as_str(&self) -> &'a str { - self.iter.as_str() - } -} - -/// An iterator over the bytes of a string slice. -/// -/// This struct is created by the [`bytes`] method on [`str`]. -/// See its documentation for more. -/// -/// [`bytes`]: ../../std/primitive.str.html#method.bytes -/// [`str`]: ../../std/primitive.str.html -#[stable(feature = "rust1", since = "1.0.0")] -#[derive(Clone, Debug)] -pub struct Bytes<'a>(Copied>); - -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for Bytes<'_> { - type Item = u8; - - #[inline] - fn next(&mut self) -> Option { - self.0.next() - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.0.size_hint() - } - - #[inline] - fn count(self) -> usize { - self.0.count() - } - - #[inline] - fn last(self) -> Option { - self.0.last() - } - - #[inline] - fn nth(&mut self, n: usize) -> Option { - self.0.nth(n) - } - - #[inline] - fn all(&mut self, f: F) -> bool - where - F: FnMut(Self::Item) -> bool, - { - self.0.all(f) - } - - #[inline] - fn any(&mut self, f: F) -> bool - where - F: FnMut(Self::Item) -> bool, - { - self.0.any(f) - } - - #[inline] - fn find

(&mut self, predicate: P) -> Option - where - P: FnMut(&Self::Item) -> bool, - { - self.0.find(predicate) - } - - #[inline] - fn position

(&mut self, predicate: P) -> Option - where - P: FnMut(Self::Item) -> bool, - { - self.0.position(predicate) - } - - #[inline] - fn rposition

(&mut self, predicate: P) -> Option - where - P: FnMut(Self::Item) -> bool, - { - self.0.rposition(predicate) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl DoubleEndedIterator for Bytes<'_> { - #[inline] - fn next_back(&mut self) -> Option { - self.0.next_back() - } - - #[inline] - fn nth_back(&mut self, n: usize) -> Option { - self.0.nth_back(n) - } - - #[inline] - fn rfind

(&mut self, predicate: P) -> Option - where - P: FnMut(&Self::Item) -> bool, - { - self.0.rfind(predicate) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ExactSizeIterator for Bytes<'_> { - #[inline] - fn len(&self) -> usize { - self.0.len() - } - - #[inline] - fn is_empty(&self) -> bool { - self.0.is_empty() - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Bytes<'_> {} - -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for Bytes<'_> {} - -#[doc(hidden)] -unsafe impl TrustedRandomAccess for Bytes<'_> { - unsafe fn get_unchecked(&mut self, i: usize) -> u8 { - self.0.get_unchecked(i) - } - fn may_have_side_effect() -> bool { - false - } -} - -/// This macro generates a Clone impl for string pattern API -/// wrapper types of the form X<'a, P> -macro_rules! derive_pattern_clone { - (clone $t:ident with |$s:ident| $e:expr) => { - impl<'a, P> Clone for $t<'a, P> - where - P: Pattern<'a, Searcher: Clone>, - { - fn clone(&self) -> Self { - let $s = self; - $e - } - } - }; -} - -/// This macro generates two public iterator structs -/// wrapping a private internal one that makes use of the `Pattern` API. -/// -/// For all patterns `P: Pattern<'a>` the following items will be -/// generated (generics omitted): -/// -/// struct $forward_iterator($internal_iterator); -/// struct $reverse_iterator($internal_iterator); -/// -/// impl Iterator for $forward_iterator -/// { /* internal ends up calling Searcher::next_match() */ } -/// -/// impl DoubleEndedIterator for $forward_iterator -/// where P::Searcher: DoubleEndedSearcher -/// { /* internal ends up calling Searcher::next_match_back() */ } -/// -/// impl Iterator for $reverse_iterator -/// where P::Searcher: ReverseSearcher -/// { /* internal ends up calling Searcher::next_match_back() */ } -/// -/// impl DoubleEndedIterator for $reverse_iterator -/// where P::Searcher: DoubleEndedSearcher -/// { /* internal ends up calling Searcher::next_match() */ } -/// -/// The internal one is defined outside the macro, and has almost the same -/// semantic as a DoubleEndedIterator by delegating to `pattern::Searcher` and -/// `pattern::ReverseSearcher` for both forward and reverse iteration. -/// -/// "Almost", because a `Searcher` and a `ReverseSearcher` for a given -/// `Pattern` might not return the same elements, so actually implementing -/// `DoubleEndedIterator` for it would be incorrect. -/// (See the docs in `str::pattern` for more details) -/// -/// However, the internal struct still represents a single ended iterator from -/// either end, and depending on pattern is also a valid double ended iterator, -/// so the two wrapper structs implement `Iterator` -/// and `DoubleEndedIterator` depending on the concrete pattern type, leading -/// to the complex impls seen above. -macro_rules! generate_pattern_iterators { - { - // Forward iterator - forward: - $(#[$forward_iterator_attribute:meta])* - struct $forward_iterator:ident; - - // Reverse iterator - reverse: - $(#[$reverse_iterator_attribute:meta])* - struct $reverse_iterator:ident; - - // Stability of all generated items - stability: - $(#[$common_stability_attribute:meta])* - - // Internal almost-iterator that is being delegated to - internal: - $internal_iterator:ident yielding ($iterty:ty); - - // Kind of delegation - either single ended or double ended - delegate $($t:tt)* - } => { - $(#[$forward_iterator_attribute])* - $(#[$common_stability_attribute])* - pub struct $forward_iterator<'a, P: Pattern<'a>>($internal_iterator<'a, P>); - - $(#[$common_stability_attribute])* - impl<'a, P> fmt::Debug for $forward_iterator<'a, P> - where - P: Pattern<'a, Searcher: fmt::Debug>, - { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple(stringify!($forward_iterator)) - .field(&self.0) - .finish() - } - } - - $(#[$common_stability_attribute])* - impl<'a, P: Pattern<'a>> Iterator for $forward_iterator<'a, P> { - type Item = $iterty; - - #[inline] - fn next(&mut self) -> Option<$iterty> { - self.0.next() - } - } - - $(#[$common_stability_attribute])* - impl<'a, P> Clone for $forward_iterator<'a, P> - where - P: Pattern<'a, Searcher: Clone>, - { - fn clone(&self) -> Self { - $forward_iterator(self.0.clone()) - } - } - - $(#[$reverse_iterator_attribute])* - $(#[$common_stability_attribute])* - pub struct $reverse_iterator<'a, P: Pattern<'a>>($internal_iterator<'a, P>); - - $(#[$common_stability_attribute])* - impl<'a, P> fmt::Debug for $reverse_iterator<'a, P> - where - P: Pattern<'a, Searcher: fmt::Debug>, - { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple(stringify!($reverse_iterator)) - .field(&self.0) - .finish() - } - } - - $(#[$common_stability_attribute])* - impl<'a, P> Iterator for $reverse_iterator<'a, P> - where - P: Pattern<'a, Searcher: ReverseSearcher<'a>>, - { - type Item = $iterty; - - #[inline] - fn next(&mut self) -> Option<$iterty> { - self.0.next_back() - } - } - - $(#[$common_stability_attribute])* - impl<'a, P> Clone for $reverse_iterator<'a, P> - where - P: Pattern<'a, Searcher: Clone>, - { - fn clone(&self) -> Self { - $reverse_iterator(self.0.clone()) - } - } - - #[stable(feature = "fused", since = "1.26.0")] - impl<'a, P: Pattern<'a>> FusedIterator for $forward_iterator<'a, P> {} - - #[stable(feature = "fused", since = "1.26.0")] - impl<'a, P> FusedIterator for $reverse_iterator<'a, P> - where - P: Pattern<'a, Searcher: ReverseSearcher<'a>>, - {} - - generate_pattern_iterators!($($t)* with $(#[$common_stability_attribute])*, - $forward_iterator, - $reverse_iterator, $iterty); - }; - { - double ended; with $(#[$common_stability_attribute:meta])*, - $forward_iterator:ident, - $reverse_iterator:ident, $iterty:ty - } => { - $(#[$common_stability_attribute])* - impl<'a, P> DoubleEndedIterator for $forward_iterator<'a, P> - where - P: Pattern<'a, Searcher: DoubleEndedSearcher<'a>>, - { - #[inline] - fn next_back(&mut self) -> Option<$iterty> { - self.0.next_back() - } - } - - $(#[$common_stability_attribute])* - impl<'a, P> DoubleEndedIterator for $reverse_iterator<'a, P> - where - P: Pattern<'a, Searcher: DoubleEndedSearcher<'a>>, - { - #[inline] - fn next_back(&mut self) -> Option<$iterty> { - self.0.next() - } - } - }; - { - single ended; with $(#[$common_stability_attribute:meta])*, - $forward_iterator:ident, - $reverse_iterator:ident, $iterty:ty - } => {} -} - -derive_pattern_clone! { - clone SplitInternal - with |s| SplitInternal { matcher: s.matcher.clone(), ..*s } -} - -struct SplitInternal<'a, P: Pattern<'a>> { - start: usize, - end: usize, - matcher: P::Searcher, - allow_trailing_empty: bool, - finished: bool, -} - -impl<'a, P> fmt::Debug for SplitInternal<'a, P> -where - P: Pattern<'a, Searcher: fmt::Debug>, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SplitInternal") - .field("start", &self.start) - .field("end", &self.end) - .field("matcher", &self.matcher) - .field("allow_trailing_empty", &self.allow_trailing_empty) - .field("finished", &self.finished) - .finish() - } -} - -impl<'a, P: Pattern<'a>> SplitInternal<'a, P> { - #[inline] - fn get_end(&mut self) -> Option<&'a str> { - if !self.finished && (self.allow_trailing_empty || self.end - self.start > 0) { - self.finished = true; - // SAFETY: `self.start` and `self.end` always lie on unicode boundaries. - unsafe { - let string = self.matcher.haystack().get_unchecked(self.start..self.end); - Some(string) - } - } else { - None - } - } - - #[inline] - fn next(&mut self) -> Option<&'a str> { - if self.finished { - return None; - } - - let haystack = self.matcher.haystack(); - match self.matcher.next_match() { - // SAFETY: `Searcher` guarantees that `a` and `b` lie on unicode boundaries. - Some((a, b)) => unsafe { - let elt = haystack.get_unchecked(self.start..a); - self.start = b; - Some(elt) - }, - None => self.get_end(), - } - } - - #[inline] - fn next_inclusive(&mut self) -> Option<&'a str> { - if self.finished { - return None; - } - - let haystack = self.matcher.haystack(); - match self.matcher.next_match() { - // SAFETY: `Searcher` guarantees that `b` lies on unicode boundary, - // and self.start is either the start of the original string, - // or `b` was assigned to it, so it also lies on unicode boundary. - Some((_, b)) => unsafe { - let elt = haystack.get_unchecked(self.start..b); - self.start = b; - Some(elt) - }, - None => self.get_end(), - } - } - - #[inline] - fn next_back(&mut self) -> Option<&'a str> - where - P::Searcher: ReverseSearcher<'a>, - { - if self.finished { - return None; - } - - if !self.allow_trailing_empty { - self.allow_trailing_empty = true; - match self.next_back() { - Some(elt) if !elt.is_empty() => return Some(elt), - _ => { - if self.finished { - return None; - } - } - } - } - - let haystack = self.matcher.haystack(); - match self.matcher.next_match_back() { - // SAFETY: `Searcher` guarantees that `a` and `b` lie on unicode boundaries. - Some((a, b)) => unsafe { - let elt = haystack.get_unchecked(b..self.end); - self.end = a; - Some(elt) - }, - // SAFETY: `self.start` and `self.end` always lie on unicode boundaries. - None => unsafe { - self.finished = true; - Some(haystack.get_unchecked(self.start..self.end)) - }, - } - } - - #[inline] - fn next_back_inclusive(&mut self) -> Option<&'a str> - where - P::Searcher: ReverseSearcher<'a>, - { - if self.finished { - return None; - } - - if !self.allow_trailing_empty { - self.allow_trailing_empty = true; - match self.next_back_inclusive() { - Some(elt) if !elt.is_empty() => return Some(elt), - _ => { - if self.finished { - return None; - } - } - } - } - - let haystack = self.matcher.haystack(); - match self.matcher.next_match_back() { - // SAFETY: `Searcher` guarantees that `b` lies on unicode boundary, - // and self.end is either the end of the original string, - // or `b` was assigned to it, so it also lies on unicode boundary. - Some((_, b)) => unsafe { - let elt = haystack.get_unchecked(b..self.end); - self.end = b; - Some(elt) - }, - // SAFETY: self.start is either the start of the original string, - // or start of a substring that represents the part of the string that hasn't - // iterated yet. Either way, it is guaranteed to lie on unicode boundary. - // self.end is either the end of the original string, - // or `b` was assigned to it, so it also lies on unicode boundary. - None => unsafe { - self.finished = true; - Some(haystack.get_unchecked(self.start..self.end)) - }, - } - } -} - -generate_pattern_iterators! { - forward: - /// Created with the method [`split`]. - /// - /// [`split`]: ../../std/primitive.str.html#method.split - struct Split; - reverse: - /// Created with the method [`rsplit`]. - /// - /// [`rsplit`]: ../../std/primitive.str.html#method.rsplit - struct RSplit; - stability: - #[stable(feature = "rust1", since = "1.0.0")] - internal: - SplitInternal yielding (&'a str); - delegate double ended; -} - -generate_pattern_iterators! { - forward: - /// Created with the method [`split_terminator`]. - /// - /// [`split_terminator`]: ../../std/primitive.str.html#method.split_terminator - struct SplitTerminator; - reverse: - /// Created with the method [`rsplit_terminator`]. - /// - /// [`rsplit_terminator`]: ../../std/primitive.str.html#method.rsplit_terminator - struct RSplitTerminator; - stability: - #[stable(feature = "rust1", since = "1.0.0")] - internal: - SplitInternal yielding (&'a str); - delegate double ended; -} - -derive_pattern_clone! { - clone SplitNInternal - with |s| SplitNInternal { iter: s.iter.clone(), ..*s } -} - -struct SplitNInternal<'a, P: Pattern<'a>> { - iter: SplitInternal<'a, P>, - /// The number of splits remaining - count: usize, -} - -impl<'a, P> fmt::Debug for SplitNInternal<'a, P> -where - P: Pattern<'a, Searcher: fmt::Debug>, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SplitNInternal") - .field("iter", &self.iter) - .field("count", &self.count) - .finish() - } -} - -impl<'a, P: Pattern<'a>> SplitNInternal<'a, P> { - #[inline] - fn next(&mut self) -> Option<&'a str> { - match self.count { - 0 => None, - 1 => { - self.count = 0; - self.iter.get_end() - } - _ => { - self.count -= 1; - self.iter.next() - } - } - } - - #[inline] - fn next_back(&mut self) -> Option<&'a str> - where - P::Searcher: ReverseSearcher<'a>, - { - match self.count { - 0 => None, - 1 => { - self.count = 0; - self.iter.get_end() - } - _ => { - self.count -= 1; - self.iter.next_back() - } - } - } -} - -generate_pattern_iterators! { - forward: - /// Created with the method [`splitn`]. - /// - /// [`splitn`]: ../../std/primitive.str.html#method.splitn - struct SplitN; - reverse: - /// Created with the method [`rsplitn`]. - /// - /// [`rsplitn`]: ../../std/primitive.str.html#method.rsplitn - struct RSplitN; - stability: - #[stable(feature = "rust1", since = "1.0.0")] - internal: - SplitNInternal yielding (&'a str); - delegate single ended; -} - -derive_pattern_clone! { - clone MatchIndicesInternal - with |s| MatchIndicesInternal(s.0.clone()) -} - -struct MatchIndicesInternal<'a, P: Pattern<'a>>(P::Searcher); - -impl<'a, P> fmt::Debug for MatchIndicesInternal<'a, P> -where - P: Pattern<'a, Searcher: fmt::Debug>, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("MatchIndicesInternal").field(&self.0).finish() - } -} - -impl<'a, P: Pattern<'a>> MatchIndicesInternal<'a, P> { - #[inline] - fn next(&mut self) -> Option<(usize, &'a str)> { - self.0 - .next_match() - // SAFETY: `Searcher` guarantees that `start` and `end` lie on unicode boundaries. - .map(|(start, end)| unsafe { (start, self.0.haystack().get_unchecked(start..end)) }) - } - - #[inline] - fn next_back(&mut self) -> Option<(usize, &'a str)> - where - P::Searcher: ReverseSearcher<'a>, - { - self.0 - .next_match_back() - // SAFETY: `Searcher` guarantees that `start` and `end` lie on unicode boundaries. - .map(|(start, end)| unsafe { (start, self.0.haystack().get_unchecked(start..end)) }) - } -} - -generate_pattern_iterators! { - forward: - /// Created with the method [`match_indices`]. - /// - /// [`match_indices`]: ../../std/primitive.str.html#method.match_indices - struct MatchIndices; - reverse: - /// Created with the method [`rmatch_indices`]. - /// - /// [`rmatch_indices`]: ../../std/primitive.str.html#method.rmatch_indices - struct RMatchIndices; - stability: - #[stable(feature = "str_match_indices", since = "1.5.0")] - internal: - MatchIndicesInternal yielding ((usize, &'a str)); - delegate double ended; -} - -derive_pattern_clone! { - clone MatchesInternal - with |s| MatchesInternal(s.0.clone()) -} - -struct MatchesInternal<'a, P: Pattern<'a>>(P::Searcher); - -impl<'a, P> fmt::Debug for MatchesInternal<'a, P> -where - P: Pattern<'a, Searcher: fmt::Debug>, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("MatchesInternal").field(&self.0).finish() - } -} - -impl<'a, P: Pattern<'a>> MatchesInternal<'a, P> { - #[inline] - fn next(&mut self) -> Option<&'a str> { - // SAFETY: `Searcher` guarantees that `start` and `end` lie on unicode boundaries. - self.0.next_match().map(|(a, b)| unsafe { - // Indices are known to be on utf8 boundaries - self.0.haystack().get_unchecked(a..b) - }) - } - - #[inline] - fn next_back(&mut self) -> Option<&'a str> - where - P::Searcher: ReverseSearcher<'a>, - { - // SAFETY: `Searcher` guarantees that `start` and `end` lie on unicode boundaries. - self.0.next_match_back().map(|(a, b)| unsafe { - // Indices are known to be on utf8 boundaries - self.0.haystack().get_unchecked(a..b) - }) - } -} +//! String manipulation. +//! +//! For more details, see the [`std::str`] module. +//! +//! [`std::str`]: ../../std/str/index.html -generate_pattern_iterators! { - forward: - /// Created with the method [`matches`]. - /// - /// [`matches`]: ../../std/primitive.str.html#method.matches - struct Matches; - reverse: - /// Created with the method [`rmatches`]. - /// - /// [`rmatches`]: ../../std/primitive.str.html#method.rmatches - struct RMatches; - stability: - #[stable(feature = "str_matches", since = "1.2.0")] - internal: - MatchesInternal yielding (&'a str); - delegate double ended; -} +#![stable(feature = "rust1", since = "1.0.0")] -/// An iterator over the lines of a string, as string slices. -/// -/// This struct is created with the [`lines`] method on [`str`]. -/// See its documentation for more. -/// -/// [`lines`]: ../../std/primitive.str.html#method.lines -/// [`str`]: ../../std/primitive.str.html -#[stable(feature = "rust1", since = "1.0.0")] -#[derive(Clone, Debug)] -pub struct Lines<'a>(Map, LinesAnyMap>); +mod converts; +mod count; +mod error; +mod iter; +mod traits; +mod validations; -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a> Iterator for Lines<'a> { - type Item = &'a str; +use self::pattern::Pattern; +use self::pattern::{DoubleEndedSearcher, ReverseSearcher, Searcher}; - #[inline] - fn next(&mut self) -> Option<&'a str> { - self.0.next() - } +use crate::char::{self, EscapeDebugExtArgs}; +use crate::mem; +use crate::slice::{self, SliceIndex}; - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.0.size_hint() - } +pub mod pattern; - #[inline] - fn last(mut self) -> Option<&'a str> { - self.next_back() - } -} +mod lossy; +#[unstable(feature = "utf8_chunks", issue = "99543")] +pub use lossy::{Utf8Chunk, Utf8Chunks}; #[stable(feature = "rust1", since = "1.0.0")] -impl<'a> DoubleEndedIterator for Lines<'a> { - #[inline] - fn next_back(&mut self) -> Option<&'a str> { - self.0.next_back() - } -} +pub use converts::{from_utf8, from_utf8_unchecked}; -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Lines<'_> {} +#[stable(feature = "str_mut_extras", since = "1.20.0")] +pub use converts::{from_utf8_mut, from_utf8_unchecked_mut}; -/// Created with the method [`lines_any`]. -/// -/// [`lines_any`]: ../../std/primitive.str.html#method.lines_any #[stable(feature = "rust1", since = "1.0.0")] -#[rustc_deprecated(since = "1.4.0", reason = "use lines()/Lines instead now")] -#[derive(Clone, Debug)] -#[allow(deprecated)] -pub struct LinesAny<'a>(Lines<'a>); - -impl_fn_for_zst! { - /// A nameable, cloneable fn type - #[derive(Clone)] - struct LinesAnyMap impl<'a> Fn = |line: &'a str| -> &'a str { - let l = line.len(); - if l > 0 && line.as_bytes()[l - 1] == b'\r' { &line[0 .. l - 1] } - else { line } - }; -} +pub use error::{ParseBoolError, Utf8Error}; #[stable(feature = "rust1", since = "1.0.0")] -#[allow(deprecated)] -impl<'a> Iterator for LinesAny<'a> { - type Item = &'a str; - - #[inline] - fn next(&mut self) -> Option<&'a str> { - self.0.next() - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.0.size_hint() - } -} +pub use traits::FromStr; #[stable(feature = "rust1", since = "1.0.0")] -#[allow(deprecated)] -impl<'a> DoubleEndedIterator for LinesAny<'a> { - #[inline] - fn next_back(&mut self) -> Option<&'a str> { - self.0.next_back() - } -} +pub use iter::{Bytes, CharIndices, Chars, Lines, SplitWhitespace}; -#[stable(feature = "fused", since = "1.26.0")] +#[stable(feature = "rust1", since = "1.0.0")] #[allow(deprecated)] -impl FusedIterator for LinesAny<'_> {} - -/* -Section: UTF-8 validation -*/ - -// use truncation to fit u64 into usize -const NONASCII_MASK: usize = 0x80808080_80808080u64 as usize; +pub use iter::LinesAny; -/// Returns `true` if any byte in the word `x` is nonascii (>= 128). -#[inline] -fn contains_nonascii(x: usize) -> bool { - (x & NONASCII_MASK) != 0 -} +#[stable(feature = "rust1", since = "1.0.0")] +pub use iter::{RSplit, RSplitTerminator, Split, SplitTerminator}; -/// Walks through `v` checking that it's a valid UTF-8 sequence, -/// returning `Ok(())` in that case, or, if it is invalid, `Err(err)`. -#[inline(always)] -fn run_utf8_validation(v: &[u8]) -> Result<(), Utf8Error> { - let mut index = 0; - let len = v.len(); +#[stable(feature = "rust1", since = "1.0.0")] +pub use iter::{RSplitN, SplitN}; - let usize_bytes = mem::size_of::(); - let ascii_block_size = 2 * usize_bytes; - let blocks_end = if len >= ascii_block_size { len - ascii_block_size + 1 } else { 0 }; - let align = v.as_ptr().align_offset(usize_bytes); +#[stable(feature = "str_matches", since = "1.2.0")] +pub use iter::{Matches, RMatches}; - while index < len { - let old_offset = index; - macro_rules! err { - ($error_len: expr) => { - return Err(Utf8Error { valid_up_to: old_offset, error_len: $error_len }); - }; - } +#[stable(feature = "str_match_indices", since = "1.5.0")] +pub use iter::{MatchIndices, RMatchIndices}; - macro_rules! next { - () => {{ - index += 1; - // we needed data, but there was none: error! - if index >= len { - err!(None) - } - v[index] - }}; - } +#[stable(feature = "encode_utf16", since = "1.8.0")] +pub use iter::EncodeUtf16; - let first = v[index]; - if first >= 128 { - let w = UTF8_CHAR_WIDTH[first as usize]; - // 2-byte encoding is for codepoints \u{0080} to \u{07ff} - // first C2 80 last DF BF - // 3-byte encoding is for codepoints \u{0800} to \u{ffff} - // first E0 A0 80 last EF BF BF - // excluding surrogates codepoints \u{d800} to \u{dfff} - // ED A0 80 to ED BF BF - // 4-byte encoding is for codepoints \u{1000}0 to \u{10ff}ff - // first F0 90 80 80 last F4 8F BF BF - // - // Use the UTF-8 syntax from the RFC - // - // https://tools.ietf.org/html/rfc3629 - // UTF8-1 = %x00-7F - // UTF8-2 = %xC2-DF UTF8-tail - // UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / - // %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) - // UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / - // %xF4 %x80-8F 2( UTF8-tail ) - match w { - 2 => { - if next!() & !CONT_MASK != TAG_CONT_U8 { - err!(Some(1)) - } - } - 3 => { - match (first, next!()) { - (0xE0, 0xA0..=0xBF) - | (0xE1..=0xEC, 0x80..=0xBF) - | (0xED, 0x80..=0x9F) - | (0xEE..=0xEF, 0x80..=0xBF) => {} - _ => err!(Some(1)), - } - if next!() & !CONT_MASK != TAG_CONT_U8 { - err!(Some(2)) - } - } - 4 => { - match (first, next!()) { - (0xF0, 0x90..=0xBF) | (0xF1..=0xF3, 0x80..=0xBF) | (0xF4, 0x80..=0x8F) => {} - _ => err!(Some(1)), - } - if next!() & !CONT_MASK != TAG_CONT_U8 { - err!(Some(2)) - } - if next!() & !CONT_MASK != TAG_CONT_U8 { - err!(Some(3)) - } - } - _ => err!(Some(1)), - } - index += 1; - } else { - // Ascii case, try to skip forward quickly. - // When the pointer is aligned, read 2 words of data per iteration - // until we find a word containing a non-ascii byte. - if align != usize::max_value() && align.wrapping_sub(index) % usize_bytes == 0 { - let ptr = v.as_ptr(); - while index < blocks_end { - // SAFETY: since `align - index` and `ascii_block_size` are - // multiples of `usize_bytes`, `block = ptr.add(index)` is - // always aligned with a `usize` so it's safe to dereference - // both `block` and `block.offset(1)`. - unsafe { - let block = ptr.add(index) as *const usize; - // break if there is a nonascii byte - let zu = contains_nonascii(*block); - let zv = contains_nonascii(*block.offset(1)); - if zu | zv { - break; - } - } - index += ascii_block_size; - } - // step from the point where the wordwise loop stopped - while index < len && v[index] < 128 { - index += 1; - } - } else { - index += 1; - } - } - } +#[stable(feature = "str_escape", since = "1.34.0")] +pub use iter::{EscapeDebug, EscapeDefault, EscapeUnicode}; - Ok(()) -} +#[stable(feature = "split_ascii_whitespace", since = "1.34.0")] +pub use iter::SplitAsciiWhitespace; -// https://tools.ietf.org/html/rfc3629 -static UTF8_CHAR_WIDTH: [u8; 256] = [ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, // 0x1F - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, // 0x3F - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, // 0x5F - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, // 0x7F - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, // 0x9F - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, // 0xBF - 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, // 0xDF - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // 0xEF - 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0xFF -]; +#[stable(feature = "split_inclusive", since = "1.51.0")] +pub use iter::SplitInclusive; -/// Given a first byte, determines how many bytes are in this UTF-8 character. #[unstable(feature = "str_internals", issue = "none")] -#[inline] -pub fn utf8_char_width(b: u8) -> usize { - UTF8_CHAR_WIDTH[b as usize] as usize -} - -/// Mask of the value bits of a continuation byte. -const CONT_MASK: u8 = 0b0011_1111; -/// Value of the tag bits (tag mask is !CONT_MASK) of a continuation byte. -const TAG_CONT_U8: u8 = 0b1000_0000; - -/* -Section: Trait implementations -*/ - -mod traits { - use crate::cmp::Ordering; - use crate::ops; - use crate::slice::{self, SliceIndex}; - - /// Implements ordering of strings. - /// - /// Strings are ordered lexicographically by their byte values. This orders Unicode code - /// points based on their positions in the code charts. This is not necessarily the same as - /// "alphabetical" order, which varies by language and locale. Sorting strings according to - /// culturally-accepted standards requires locale-specific data that is outside the scope of - /// the `str` type. - #[stable(feature = "rust1", since = "1.0.0")] - impl Ord for str { - #[inline] - fn cmp(&self, other: &str) -> Ordering { - self.as_bytes().cmp(other.as_bytes()) - } - } - - #[stable(feature = "rust1", since = "1.0.0")] - impl PartialEq for str { - #[inline] - fn eq(&self, other: &str) -> bool { - self.as_bytes() == other.as_bytes() - } - #[inline] - fn ne(&self, other: &str) -> bool { - !(*self).eq(other) - } - } - - #[stable(feature = "rust1", since = "1.0.0")] - impl Eq for str {} - - /// Implements comparison operations on strings. - /// - /// Strings are compared lexicographically by their byte values. This compares Unicode code - /// points based on their positions in the code charts. This is not necessarily the same as - /// "alphabetical" order, which varies by language and locale. Comparing strings according to - /// culturally-accepted standards requires locale-specific data that is outside the scope of - /// the `str` type. - #[stable(feature = "rust1", since = "1.0.0")] - impl PartialOrd for str { - #[inline] - fn partial_cmp(&self, other: &str) -> Option { - Some(self.cmp(other)) - } - } - - #[stable(feature = "rust1", since = "1.0.0")] - impl ops::Index for str - where - I: SliceIndex, - { - type Output = I::Output; - - #[inline] - fn index(&self, index: I) -> &I::Output { - index.index(self) - } - } - - #[stable(feature = "rust1", since = "1.0.0")] - impl ops::IndexMut for str - where - I: SliceIndex, - { - #[inline] - fn index_mut(&mut self, index: I) -> &mut I::Output { - index.index_mut(self) - } - } - - #[inline(never)] - #[cold] - fn str_index_overflow_fail() -> ! { - panic!("attempted to index str up to maximum usize"); - } - - /// Implements substring slicing with syntax `&self[..]` or `&mut self[..]`. - /// - /// Returns a slice of the whole string, i.e., returns `&self` or `&mut - /// self`. Equivalent to `&self[0 .. len]` or `&mut self[0 .. len]`. Unlike - /// other indexing operations, this can never panic. - /// - /// This operation is `O(1)`. - /// - /// Prior to 1.20.0, these indexing operations were still supported by - /// direct implementation of `Index` and `IndexMut`. - /// - /// Equivalent to `&self[0 .. len]` or `&mut self[0 .. len]`. - #[stable(feature = "str_checked_slicing", since = "1.20.0")] - impl SliceIndex for ops::RangeFull { - type Output = str; - #[inline] - fn get(self, slice: &str) -> Option<&Self::Output> { - Some(slice) - } - #[inline] - fn get_mut(self, slice: &mut str) -> Option<&mut Self::Output> { - Some(slice) - } - #[inline] - unsafe fn get_unchecked(self, slice: &str) -> &Self::Output { - slice - } - #[inline] - unsafe fn get_unchecked_mut(self, slice: &mut str) -> &mut Self::Output { - slice - } - #[inline] - fn index(self, slice: &str) -> &Self::Output { - slice - } - #[inline] - fn index_mut(self, slice: &mut str) -> &mut Self::Output { - slice - } - } - - /// Implements substring slicing with syntax `&self[begin .. end]` or `&mut - /// self[begin .. end]`. - /// - /// Returns a slice of the given string from the byte range - /// [`begin`, `end`). - /// - /// This operation is `O(1)`. - /// - /// Prior to 1.20.0, these indexing operations were still supported by - /// direct implementation of `Index` and `IndexMut`. - /// - /// # Panics - /// - /// Panics if `begin` or `end` does not point to the starting byte offset of - /// a character (as defined by `is_char_boundary`), if `begin > end`, or if - /// `end > len`. - /// - /// # Examples - /// - /// ``` - /// let s = "Löwe 老虎 Léopard"; - /// assert_eq!(&s[0 .. 1], "L"); - /// - /// assert_eq!(&s[1 .. 9], "öwe 老"); - /// - /// // these will panic: - /// // byte 2 lies within `ö`: - /// // &s[2 ..3]; - /// - /// // byte 8 lies within `老` - /// // &s[1 .. 8]; - /// - /// // byte 100 is outside the string - /// // &s[3 .. 100]; - /// ``` - #[stable(feature = "str_checked_slicing", since = "1.20.0")] - impl SliceIndex for ops::Range { - type Output = str; - #[inline] - fn get(self, slice: &str) -> Option<&Self::Output> { - if self.start <= self.end - && slice.is_char_boundary(self.start) - && slice.is_char_boundary(self.end) - { - // SAFETY: just checked that `start` and `end` are on a char boundary. - Some(unsafe { self.get_unchecked(slice) }) - } else { - None - } - } - #[inline] - fn get_mut(self, slice: &mut str) -> Option<&mut Self::Output> { - if self.start <= self.end - && slice.is_char_boundary(self.start) - && slice.is_char_boundary(self.end) - { - // SAFETY: just checked that `start` and `end` are on a char boundary. - Some(unsafe { self.get_unchecked_mut(slice) }) - } else { - None - } - } - #[inline] - unsafe fn get_unchecked(self, slice: &str) -> &Self::Output { - let ptr = slice.as_ptr().add(self.start); - let len = self.end - self.start; - super::from_utf8_unchecked(slice::from_raw_parts(ptr, len)) - } - #[inline] - unsafe fn get_unchecked_mut(self, slice: &mut str) -> &mut Self::Output { - let ptr = slice.as_mut_ptr().add(self.start); - let len = self.end - self.start; - super::from_utf8_unchecked_mut(slice::from_raw_parts_mut(ptr, len)) - } - #[inline] - fn index(self, slice: &str) -> &Self::Output { - let (start, end) = (self.start, self.end); - self.get(slice).unwrap_or_else(|| super::slice_error_fail(slice, start, end)) - } - #[inline] - fn index_mut(self, slice: &mut str) -> &mut Self::Output { - // is_char_boundary checks that the index is in [0, .len()] - // cannot reuse `get` as above, because of NLL trouble - if self.start <= self.end - && slice.is_char_boundary(self.start) - && slice.is_char_boundary(self.end) - { - // SAFETY: just checked that `start` and `end` are on a char boundary. - unsafe { self.get_unchecked_mut(slice) } - } else { - super::slice_error_fail(slice, self.start, self.end) - } - } - } - - /// Implements substring slicing with syntax `&self[.. end]` or `&mut - /// self[.. end]`. - /// - /// Returns a slice of the given string from the byte range [`0`, `end`). - /// Equivalent to `&self[0 .. end]` or `&mut self[0 .. end]`. - /// - /// This operation is `O(1)`. - /// - /// Prior to 1.20.0, these indexing operations were still supported by - /// direct implementation of `Index` and `IndexMut`. - /// - /// # Panics - /// - /// Panics if `end` does not point to the starting byte offset of a - /// character (as defined by `is_char_boundary`), or if `end > len`. - #[stable(feature = "str_checked_slicing", since = "1.20.0")] - impl SliceIndex for ops::RangeTo { - type Output = str; - #[inline] - fn get(self, slice: &str) -> Option<&Self::Output> { - if slice.is_char_boundary(self.end) { - // SAFETY: just checked that `end` is on a char boundary. - Some(unsafe { self.get_unchecked(slice) }) - } else { - None - } - } - #[inline] - fn get_mut(self, slice: &mut str) -> Option<&mut Self::Output> { - if slice.is_char_boundary(self.end) { - // SAFETY: just checked that `end` is on a char boundary. - Some(unsafe { self.get_unchecked_mut(slice) }) - } else { - None - } - } - #[inline] - unsafe fn get_unchecked(self, slice: &str) -> &Self::Output { - let ptr = slice.as_ptr(); - super::from_utf8_unchecked(slice::from_raw_parts(ptr, self.end)) - } - #[inline] - unsafe fn get_unchecked_mut(self, slice: &mut str) -> &mut Self::Output { - let ptr = slice.as_mut_ptr(); - super::from_utf8_unchecked_mut(slice::from_raw_parts_mut(ptr, self.end)) - } - #[inline] - fn index(self, slice: &str) -> &Self::Output { - let end = self.end; - self.get(slice).unwrap_or_else(|| super::slice_error_fail(slice, 0, end)) - } - #[inline] - fn index_mut(self, slice: &mut str) -> &mut Self::Output { - if slice.is_char_boundary(self.end) { - // SAFETY: just checked that `end` is on a char boundary. - unsafe { self.get_unchecked_mut(slice) } - } else { - super::slice_error_fail(slice, 0, self.end) - } - } - } - - /// Implements substring slicing with syntax `&self[begin ..]` or `&mut - /// self[begin ..]`. - /// - /// Returns a slice of the given string from the byte range [`begin`, - /// `len`). Equivalent to `&self[begin .. len]` or `&mut self[begin .. - /// len]`. - /// - /// This operation is `O(1)`. - /// - /// Prior to 1.20.0, these indexing operations were still supported by - /// direct implementation of `Index` and `IndexMut`. - /// - /// # Panics - /// - /// Panics if `begin` does not point to the starting byte offset of - /// a character (as defined by `is_char_boundary`), or if `begin >= len`. - #[stable(feature = "str_checked_slicing", since = "1.20.0")] - impl SliceIndex for ops::RangeFrom { - type Output = str; - #[inline] - fn get(self, slice: &str) -> Option<&Self::Output> { - if slice.is_char_boundary(self.start) { - // SAFETY: just checked that `start` is on a char boundary. - Some(unsafe { self.get_unchecked(slice) }) - } else { - None - } - } - #[inline] - fn get_mut(self, slice: &mut str) -> Option<&mut Self::Output> { - if slice.is_char_boundary(self.start) { - // SAFETY: just checked that `start` is on a char boundary. - Some(unsafe { self.get_unchecked_mut(slice) }) - } else { - None - } - } - #[inline] - unsafe fn get_unchecked(self, slice: &str) -> &Self::Output { - let ptr = slice.as_ptr().add(self.start); - let len = slice.len() - self.start; - super::from_utf8_unchecked(slice::from_raw_parts(ptr, len)) - } - #[inline] - unsafe fn get_unchecked_mut(self, slice: &mut str) -> &mut Self::Output { - let ptr = slice.as_mut_ptr().add(self.start); - let len = slice.len() - self.start; - super::from_utf8_unchecked_mut(slice::from_raw_parts_mut(ptr, len)) - } - #[inline] - fn index(self, slice: &str) -> &Self::Output { - let (start, end) = (self.start, slice.len()); - self.get(slice).unwrap_or_else(|| super::slice_error_fail(slice, start, end)) - } - #[inline] - fn index_mut(self, slice: &mut str) -> &mut Self::Output { - if slice.is_char_boundary(self.start) { - // SAFETY: just checked that `start` is on a char boundary. - unsafe { self.get_unchecked_mut(slice) } - } else { - super::slice_error_fail(slice, self.start, slice.len()) - } - } - } +pub use validations::{next_code_point, utf8_char_width}; - /// Implements substring slicing with syntax `&self[begin ..= end]` or `&mut - /// self[begin ..= end]`. - /// - /// Returns a slice of the given string from the byte range - /// [`begin`, `end`]. Equivalent to `&self [begin .. end + 1]` or `&mut - /// self[begin .. end + 1]`, except if `end` has the maximum value for - /// `usize`. - /// - /// This operation is `O(1)`. - /// - /// # Panics - /// - /// Panics if `begin` does not point to the starting byte offset of - /// a character (as defined by `is_char_boundary`), if `end` does not point - /// to the ending byte offset of a character (`end + 1` is either a starting - /// byte offset or equal to `len`), if `begin > end`, or if `end >= len`. - #[stable(feature = "inclusive_range", since = "1.26.0")] - impl SliceIndex for ops::RangeInclusive { - type Output = str; - #[inline] - fn get(self, slice: &str) -> Option<&Self::Output> { - if *self.end() == usize::max_value() { - None - } else { - (*self.start()..self.end() + 1).get(slice) - } - } - #[inline] - fn get_mut(self, slice: &mut str) -> Option<&mut Self::Output> { - if *self.end() == usize::max_value() { - None - } else { - (*self.start()..self.end() + 1).get_mut(slice) - } - } - #[inline] - unsafe fn get_unchecked(self, slice: &str) -> &Self::Output { - (*self.start()..self.end() + 1).get_unchecked(slice) - } - #[inline] - unsafe fn get_unchecked_mut(self, slice: &mut str) -> &mut Self::Output { - (*self.start()..self.end() + 1).get_unchecked_mut(slice) - } - #[inline] - fn index(self, slice: &str) -> &Self::Output { - if *self.end() == usize::max_value() { - str_index_overflow_fail(); - } - (*self.start()..self.end() + 1).index(slice) - } - #[inline] - fn index_mut(self, slice: &mut str) -> &mut Self::Output { - if *self.end() == usize::max_value() { - str_index_overflow_fail(); - } - (*self.start()..self.end() + 1).index_mut(slice) - } - } +use iter::MatchIndicesInternal; +use iter::SplitInternal; +use iter::{MatchesInternal, SplitNInternal}; - /// Implements substring slicing with syntax `&self[..= end]` or `&mut - /// self[..= end]`. - /// - /// Returns a slice of the given string from the byte range [0, `end`]. - /// Equivalent to `&self [0 .. end + 1]`, except if `end` has the maximum - /// value for `usize`. - /// - /// This operation is `O(1)`. - /// - /// # Panics - /// - /// Panics if `end` does not point to the ending byte offset of a character - /// (`end + 1` is either a starting byte offset as defined by - /// `is_char_boundary`, or equal to `len`), or if `end >= len`. - #[stable(feature = "inclusive_range", since = "1.26.0")] - impl SliceIndex for ops::RangeToInclusive { - type Output = str; - #[inline] - fn get(self, slice: &str) -> Option<&Self::Output> { - if self.end == usize::max_value() { None } else { (..self.end + 1).get(slice) } - } - #[inline] - fn get_mut(self, slice: &mut str) -> Option<&mut Self::Output> { - if self.end == usize::max_value() { None } else { (..self.end + 1).get_mut(slice) } - } - #[inline] - unsafe fn get_unchecked(self, slice: &str) -> &Self::Output { - (..self.end + 1).get_unchecked(slice) - } - #[inline] - unsafe fn get_unchecked_mut(self, slice: &mut str) -> &mut Self::Output { - (..self.end + 1).get_unchecked_mut(slice) - } - #[inline] - fn index(self, slice: &str) -> &Self::Output { - if self.end == usize::max_value() { - str_index_overflow_fail(); - } - (..self.end + 1).index(slice) - } - #[inline] - fn index_mut(self, slice: &mut str) -> &mut Self::Output { - if self.end == usize::max_value() { - str_index_overflow_fail(); - } - (..self.end + 1).index_mut(slice) - } +#[inline(never)] +#[cold] +#[track_caller] +#[rustc_allow_const_fn_unstable(const_eval_select)] +const fn slice_error_fail(s: &str, begin: usize, end: usize) -> ! { + // SAFETY: panics for both branches + unsafe { + crate::intrinsics::const_eval_select( + (s, begin, end), + slice_error_fail_ct, + slice_error_fail_rt, + ) } } -// truncate `&str` to length at most equal to `max` -// return `true` if it were truncated, and the new str. -fn truncate_to_char_boundary(s: &str, mut max: usize) -> (bool, &str) { - if max >= s.len() { - (false, s) - } else { - while !s.is_char_boundary(max) { - max -= 1; - } - (true, &s[..max]) - } +#[track_caller] +const fn slice_error_fail_ct(_: &str, _: usize, _: usize) -> ! { + panic!("failed to slice string"); } -#[inline(never)] -#[cold] -fn slice_error_fail(s: &str, begin: usize, end: usize) -> ! { +#[track_caller] +fn slice_error_fail_rt(s: &str, begin: usize, end: usize) -> ! { const MAX_DISPLAY_LENGTH: usize = 256; - let (truncated, s_trunc) = truncate_to_char_boundary(s, MAX_DISPLAY_LENGTH); - let ellipsis = if truncated { "[...]" } else { "" }; + let trunc_len = s.floor_char_boundary(MAX_DISPLAY_LENGTH); + let s_trunc = &s[..trunc_len]; + let ellipsis = if trunc_len < s.len() { "[...]" } else { "" }; // 1. out of bounds if begin > s.len() || end > s.len() { let oob_index = if begin > s.len() { begin } else { end }; - panic!("byte index {} is out of bounds of `{}`{}", oob_index, s_trunc, ellipsis); + panic!("byte index {oob_index} is out of bounds of `{s_trunc}`{ellipsis}"); } // 2. begin <= end @@ -2209,10 +122,7 @@ fn slice_error_fail(s: &str, begin: usize, end: usize) -> ! { // 3. character boundary let index = if !s.is_char_boundary(begin) { begin } else { end }; // find the character - let mut char_start = index; - while !s.is_char_boundary(char_start) { - char_start -= 1; - } + let char_start = s.floor_char_boundary(index); // `char_start` must be less than len and a char boundary let ch = s[char_start..].chars().next().unwrap(); let char_range = char_start..char_start + ch.len_utf8(); @@ -2222,13 +132,14 @@ fn slice_error_fail(s: &str, begin: usize, end: usize) -> ! { ); } -#[lang = "str"] #[cfg(not(test))] impl str { /// Returns the length of `self`. /// /// This length is in bytes, not [`char`]s or graphemes. In other words, - /// it may not be what a human considers the length of the string. + /// it might not be what a human considers the length of the string. + /// + /// [`char`]: prim@char /// /// # Examples /// @@ -2242,7 +153,8 @@ impl str { /// assert_eq!("ƒoo".chars().count(), 3); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_str_len", since = "1.32.0")] + #[rustc_const_stable(feature = "const_str_len", since = "1.39.0")] + #[must_use] #[inline] pub const fn len(&self) -> usize { self.as_bytes().len() @@ -2261,19 +173,19 @@ impl str { /// let s = "not empty"; /// assert!(!s.is_empty()); /// ``` - #[inline] #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_str_is_empty", since = "1.32.0")] + #[rustc_const_stable(feature = "const_str_is_empty", since = "1.39.0")] + #[must_use] + #[inline] pub const fn is_empty(&self) -> bool { self.len() == 0 } - /// Checks that `index`-th byte lies at the start and/or end of a - /// UTF-8 code point sequence. + /// Checks that `index`-th byte is the first byte in a UTF-8 code point + /// sequence or the end of the string. /// /// The start and end of the string (when `index == self.len()`) are - /// considered to be - /// boundaries. + /// considered to be boundaries. /// /// Returns `false` if `index` is greater than `self.len()`. /// @@ -2292,26 +204,110 @@ impl str { /// // third byte of `老` /// assert!(!s.is_char_boundary(8)); /// ``` + #[must_use] #[stable(feature = "is_char_boundary", since = "1.9.0")] + #[rustc_const_unstable(feature = "const_is_char_boundary", issue = "none")] #[inline] - pub fn is_char_boundary(&self, index: usize) -> bool { - // 0 and len are always ok. + pub const fn is_char_boundary(&self, index: usize) -> bool { + // 0 is always ok. // Test for 0 explicitly so that it can optimize out the check // easily and skip reading string data for that case. - if index == 0 || index == self.len() { + // Note that optimizing `self.get(..index)` relies on this. + if index == 0 { return true; } + match self.as_bytes().get(index) { - None => false, - // This is bit magic equivalent to: b < 128 || b >= 192 - Some(&b) => (b as i8) >= -0x40, + // For `None` we have two options: + // + // - index == self.len() + // Empty strings are valid, so return true + // - index > self.len() + // In this case return false + // + // The check is placed exactly here, because it improves generated + // code on higher opt-levels. See PR #84751 for more details. + None => index == self.len(), + + Some(&b) => b.is_utf8_char_boundary(), + } + } + + /// Finds the closest `x` not exceeding `index` where `is_char_boundary(x)` is `true`. + /// + /// This method can help you truncate a string so that it's still valid UTF-8, but doesn't + /// exceed a given number of bytes. Note that this is done purely at the character level + /// and can still visually split graphemes, even though the underlying characters aren't + /// split. For example, the emoji 🧑‍🔬 (scientist) could be split so that the string only + /// includes 🧑 (person) instead. + /// + /// # Examples + /// + /// ``` + /// #![feature(round_char_boundary)] + /// let s = "❤️🧡💛💚💙💜"; + /// assert_eq!(s.len(), 26); + /// assert!(!s.is_char_boundary(13)); + /// + /// let closest = s.floor_char_boundary(13); + /// assert_eq!(closest, 10); + /// assert_eq!(&s[..closest], "❤️🧡"); + /// ``` + #[unstable(feature = "round_char_boundary", issue = "93743")] + #[inline] + pub fn floor_char_boundary(&self, index: usize) -> usize { + if index >= self.len() { + self.len() + } else { + let lower_bound = index.saturating_sub(3); + let new_index = self.as_bytes()[lower_bound..=index] + .iter() + .rposition(|b| b.is_utf8_char_boundary()); + + // SAFETY: we know that the character boundary will be within four bytes + unsafe { lower_bound + new_index.unwrap_unchecked() } + } + } + + /// Finds the closest `x` not below `index` where `is_char_boundary(x)` is `true`. + /// + /// This method is the natural complement to [`floor_char_boundary`]. See that method + /// for more details. + /// + /// [`floor_char_boundary`]: str::floor_char_boundary + /// + /// # Panics + /// + /// Panics if `index > self.len()`. + /// + /// # Examples + /// + /// ``` + /// #![feature(round_char_boundary)] + /// let s = "❤️🧡💛💚💙💜"; + /// assert_eq!(s.len(), 26); + /// assert!(!s.is_char_boundary(13)); + /// + /// let closest = s.ceil_char_boundary(13); + /// assert_eq!(closest, 14); + /// assert_eq!(&s[..closest], "❤️🧡💛"); + /// ``` + #[unstable(feature = "round_char_boundary", issue = "93743")] + #[inline] + pub fn ceil_char_boundary(&self, index: usize) -> usize { + if index > self.len() { + slice_error_fail(self, index, index) + } else { + let upper_bound = Ord::min(index + 4, self.len()); + self.as_bytes()[index..upper_bound] + .iter() + .position(|b| b.is_utf8_char_boundary()) + .map_or(upper_bound, |pos| pos + index) } } /// Converts a string slice to a byte slice. To convert the byte slice back - /// into a string slice, use the [`str::from_utf8`] function. - /// - /// [`str::from_utf8`]: ./str/fn.from_utf8.html + /// into a string slice, use the [`from_utf8`] function. /// /// # Examples /// @@ -2322,19 +318,23 @@ impl str { /// assert_eq!(b"bors", bytes); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "str_as_bytes", since = "1.32.0")] + #[rustc_const_stable(feature = "str_as_bytes", since = "1.39.0")] + #[must_use] #[inline(always)] #[allow(unused_attributes)] - #[allow_internal_unstable(const_fn_union)] pub const fn as_bytes(&self) -> &[u8] { - unsafe { mem::crucible_identity_transmute(self) } + // SAFETY: const sound because we transmute two types with the same layout + unsafe { mem::transmute(self) } } - /// Converts a mutable string slice to a mutable byte slice. To convert the - /// mutable byte slice back into a mutable string slice, use the - /// [`str::from_utf8_mut`] function. + /// Converts a mutable string slice to a mutable byte slice. + /// + /// # Safety /// - /// [`str::from_utf8_mut`]: ./str/fn.from_utf8_mut.html + /// The caller must ensure that the content of the slice is valid UTF-8 + /// before the borrow ends and the underlying `str` is used. + /// + /// Use of a `str` whose contents are not valid UTF-8 is undefined behavior. /// /// # Examples /// @@ -2364,9 +364,14 @@ impl str { /// assert_eq!("🍔∈🌏", s); /// ``` #[stable(feature = "str_mut_extras", since = "1.20.0")] + #[must_use] #[inline(always)] pub unsafe fn as_bytes_mut(&mut self) -> &mut [u8] { - mem::crucible_identity_transmute(self) + // SAFETY: the cast from `&str` to `&[u8]` is safe since `str` + // has the same layout as `&[u8]` (only std can make this guarantee). + // The pointer dereference is safe since it comes from a mutable reference which + // is guaranteed to be valid for writes. + unsafe { &mut *(self as *mut str as *mut [u8]) } } /// Converts a string slice to a raw pointer. @@ -2378,8 +383,7 @@ impl str { /// The caller must ensure that the returned pointer is never written to. /// If you need to mutate the contents of the string slice, use [`as_mut_ptr`]. /// - /// [`u8`]: primitive.u8.html - /// [`as_mut_ptr`]: #method.as_mut_ptr + /// [`as_mut_ptr`]: str::as_mut_ptr /// /// # Examples /// @@ -2391,7 +395,8 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[rustc_const_stable(feature = "rustc_str_as_ptr", since = "1.32.0")] - #[inline] + #[must_use] + #[inline(always)] pub const fn as_ptr(&self) -> *const u8 { self as *const str as *const u8 } @@ -2404,10 +409,9 @@ impl str { /// /// It is your responsibility to make sure that the string slice only gets /// modified in a way that it remains valid UTF-8. - /// - /// [`u8`]: primitive.u8.html #[stable(feature = "str_as_mut_ptr", since = "1.36.0")] - #[inline] + #[must_use] + #[inline(always)] pub fn as_mut_ptr(&mut self) -> *mut u8 { self as *mut str as *mut u8 } @@ -2417,8 +421,6 @@ impl str { /// This is the non-panicking alternative to indexing the `str`. Returns /// [`None`] whenever equivalent indexing operation would panic. /// - /// [`None`]: option/enum.Option.html#variant.None - /// /// # Examples /// /// ``` @@ -2434,8 +436,9 @@ impl str { /// assert!(v.get(..42).is_none()); /// ``` #[stable(feature = "str_checked_slicing", since = "1.20.0")] + #[rustc_const_unstable(feature = "const_slice_index", issue = "none")] #[inline] - pub fn get>(&self, i: I) -> Option<&I::Output> { + pub const fn get>(&self, i: I) -> Option<&I::Output> { i.get(self) } @@ -2444,8 +447,6 @@ impl str { /// This is the non-panicking alternative to indexing the `str`. Returns /// [`None`] whenever equivalent indexing operation would panic. /// - /// [`None`]: option/enum.Option.html#variant.None - /// /// # Examples /// /// ``` @@ -2468,8 +469,9 @@ impl str { /// assert_eq!("HEllo", v); /// ``` #[stable(feature = "str_checked_slicing", since = "1.20.0")] + #[rustc_const_unstable(feature = "const_slice_index", issue = "none")] #[inline] - pub fn get_mut>(&mut self, i: I) -> Option<&mut I::Output> { + pub const fn get_mut>(&mut self, i: I) -> Option<&mut I::Output> { i.get_mut(self) } @@ -2500,9 +502,13 @@ impl str { /// } /// ``` #[stable(feature = "str_checked_slicing", since = "1.20.0")] + #[rustc_const_unstable(feature = "const_slice_index", issue = "none")] #[inline] - pub unsafe fn get_unchecked>(&self, i: I) -> &I::Output { - i.get_unchecked(self) + pub const unsafe fn get_unchecked>(&self, i: I) -> &I::Output { + // SAFETY: the caller must uphold the safety contract for `get_unchecked`; + // the slice is dereferenceable because `self` is a safe reference. + // The returned pointer is safe because impls of `SliceIndex` have to guarantee that it is. + unsafe { &*i.get_unchecked(self) } } /// Returns a mutable, unchecked subslice of `str`. @@ -2532,9 +538,16 @@ impl str { /// } /// ``` #[stable(feature = "str_checked_slicing", since = "1.20.0")] + #[rustc_const_unstable(feature = "const_slice_index", issue = "none")] #[inline] - pub unsafe fn get_unchecked_mut>(&mut self, i: I) -> &mut I::Output { - i.get_unchecked_mut(self) + pub const unsafe fn get_unchecked_mut>( + &mut self, + i: I, + ) -> &mut I::Output { + // SAFETY: the caller must uphold the safety contract for `get_unchecked_mut`; + // the slice is dereferenceable because `self` is a safe reference. + // The returned pointer is safe because impls of `SliceIndex` have to guarantee that it is. + unsafe { &mut *i.get_unchecked_mut(self) } } /// Creates a string slice from another string slice, bypassing safety @@ -2543,8 +556,7 @@ impl str { /// This is generally not recommended, use with caution! For a safe /// alternative see [`str`] and [`Index`]. /// - /// [`str`]: primitive.str.html - /// [`Index`]: ops/trait.Index.html + /// [`Index`]: crate::ops::Index /// /// This new slice goes from `begin` to `end`, including `begin` but /// excluding `end`. @@ -2552,7 +564,7 @@ impl str { /// To get a mutable string slice instead, see the /// [`slice_mut_unchecked`] method. /// - /// [`slice_mut_unchecked`]: #method.slice_mut_unchecked + /// [`slice_mut_unchecked`]: str::slice_mut_unchecked /// /// # Safety /// @@ -2581,10 +593,14 @@ impl str { /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_deprecated(since = "1.29.0", reason = "use `get_unchecked(begin..end)` instead")] + #[deprecated(since = "1.29.0", note = "use `get_unchecked(begin..end)` instead")] + #[must_use] #[inline] pub unsafe fn slice_unchecked(&self, begin: usize, end: usize) -> &str { - (begin..end).get_unchecked(self) + // SAFETY: the caller must uphold the safety contract for `get_unchecked`; + // the slice is dereferenceable because `self` is a safe reference. + // The returned pointer is safe because impls of `SliceIndex` have to guarantee that it is. + unsafe { &*(begin..end).get_unchecked(self) } } /// Creates a string slice from another string slice, bypassing safety @@ -2592,8 +608,7 @@ impl str { /// This is generally not recommended, use with caution! For a safe /// alternative see [`str`] and [`IndexMut`]. /// - /// [`str`]: primitive.str.html - /// [`IndexMut`]: ops/trait.IndexMut.html + /// [`IndexMut`]: crate::ops::IndexMut /// /// This new slice goes from `begin` to `end`, including `begin` but /// excluding `end`. @@ -2601,7 +616,7 @@ impl str { /// To get an immutable string slice instead, see the /// [`slice_unchecked`] method. /// - /// [`slice_unchecked`]: #method.slice_unchecked + /// [`slice_unchecked`]: str::slice_unchecked /// /// # Safety /// @@ -2612,10 +627,13 @@ impl str { /// * `begin` and `end` must be byte positions within the string slice. /// * `begin` and `end` must lie on UTF-8 sequence boundaries. #[stable(feature = "str_slice_mut", since = "1.5.0")] - #[rustc_deprecated(since = "1.29.0", reason = "use `get_unchecked_mut(begin..end)` instead")] + #[deprecated(since = "1.29.0", note = "use `get_unchecked_mut(begin..end)` instead")] #[inline] pub unsafe fn slice_mut_unchecked(&mut self, begin: usize, end: usize) -> &mut str { - (begin..end).get_unchecked_mut(self) + // SAFETY: the caller must uphold the safety contract for `get_unchecked_mut`; + // the slice is dereferenceable because `self` is a safe reference. + // The returned pointer is safe because impls of `SliceIndex` have to guarantee that it is. + unsafe { &mut *(begin..end).get_unchecked_mut(self) } } /// Divide one string slice into two at an index. @@ -2629,12 +647,12 @@ impl str { /// To get mutable string slices instead, see the [`split_at_mut`] /// method. /// - /// [`split_at_mut`]: #method.split_at_mut + /// [`split_at_mut`]: str::split_at_mut /// /// # Panics /// /// Panics if `mid` is not on a UTF-8 code point boundary, or if it is - /// beyond the last code point of the string slice. + /// past the end of the last code point of the string slice. /// /// # Examples /// @@ -2649,6 +667,7 @@ impl str { /// assert_eq!(" Martin-Löf", last); /// ``` #[inline] + #[must_use] #[stable(feature = "str_split_at", since = "1.4.0")] pub fn split_at(&self, mid: usize) -> (&str, &str) { // is_char_boundary checks that the index is in [0, .len()] @@ -2670,12 +689,12 @@ impl str { /// /// To get immutable string slices instead, see the [`split_at`] method. /// - /// [`split_at`]: #method.split_at + /// [`split_at`]: str::split_at /// /// # Panics /// /// Panics if `mid` is not on a UTF-8 code point boundary, or if it is - /// beyond the last code point of the string slice. + /// past the end of the last code point of the string slice. /// /// # Examples /// @@ -2692,6 +711,7 @@ impl str { /// assert_eq!("PER Martin-Löf", s); /// ``` #[inline] + #[must_use] #[stable(feature = "str_split_at", since = "1.4.0")] pub fn split_at_mut(&mut self, mid: usize) -> (&mut str, &mut str) { // is_char_boundary checks that the index is in [0, .len()] @@ -2716,7 +736,7 @@ impl str { /// string slice by [`char`]. This method returns such an iterator. /// /// It's important to remember that [`char`] represents a Unicode Scalar - /// Value, and may not match your idea of what a 'character' is. Iteration + /// Value, and might not match your idea of what a 'character' is. Iteration /// over grapheme clusters may be what you actually want. This functionality /// is not provided by Rust's standard library, check crates.io instead. /// @@ -2743,7 +763,9 @@ impl str { /// assert_eq!(None, chars.next()); /// ``` /// - /// Remember, [`char`]s may not match your human intuition about characters: + /// Remember, [`char`]s might not match your intuition about characters: + /// + /// [`char`]: prim@char /// /// ``` /// let y = "y̆"; @@ -2794,7 +816,9 @@ impl str { /// assert_eq!(None, char_indices.next()); /// ``` /// - /// Remember, [`char`]s may not match your human intuition about characters: + /// Remember, [`char`]s might not match your intuition about characters: + /// + /// [`char`]: prim@char /// /// ``` /// let yes = "y̆es"; @@ -2850,7 +874,7 @@ impl str { /// Core Property `White_Space`. If you only want to split on ASCII whitespace /// instead, use [`split_ascii_whitespace`]. /// - /// [`split_ascii_whitespace`]: #method.split_ascii_whitespace + /// [`split_ascii_whitespace`]: str::split_ascii_whitespace /// /// # Examples /// @@ -2878,7 +902,16 @@ impl str { /// /// assert_eq!(None, iter.next()); /// ``` + /// + /// If the string is empty or all whitespace, the iterator yields no string slices: + /// ``` + /// assert_eq!("".split_whitespace().next(), None); + /// assert_eq!(" ".split_whitespace().next(), None); + /// ``` + #[must_use = "this returns the split string as an iterator, \ + without modifying the original"] #[stable(feature = "split_whitespace", since = "1.1.0")] + #[cfg_attr(not(test), rustc_diagnostic_item = "str_split_whitespace")] #[inline] pub fn split_whitespace(&self) -> SplitWhitespace<'_> { SplitWhitespace { inner: self.split(IsWhitespace).filter(IsNotEmpty) } @@ -2891,7 +924,7 @@ impl str { /// /// To split by Unicode `Whitespace` instead, use [`split_whitespace`]. /// - /// [`split_whitespace`]: #method.split_whitespace + /// [`split_whitespace`]: str::split_whitespace /// /// # Examples /// @@ -2919,6 +952,14 @@ impl str { /// /// assert_eq!(None, iter.next()); /// ``` + /// + /// If the string is empty or all ASCII whitespace, the iterator yields no string slices: + /// ``` + /// assert_eq!("".split_ascii_whitespace().next(), None); + /// assert_eq!(" ".split_ascii_whitespace().next(), None); + /// ``` + #[must_use = "this returns the split string as an iterator, \ + without modifying the original"] #[stable(feature = "split_ascii_whitespace", since = "1.34.0")] #[inline] pub fn split_ascii_whitespace(&self) -> SplitAsciiWhitespace<'_> { @@ -2929,10 +970,14 @@ impl str { /// An iterator over the lines of a string, as string slices. /// - /// Lines are ended with either a newline (`\n`) or a carriage return with - /// a line feed (`\r\n`). + /// Lines are split at line endings that are either newlines (`\n`) or + /// sequences of a carriage return followed by a line feed (`\r\n`). /// - /// The final line ending is optional. + /// Line terminators are not included in the lines returned by the iterator. + /// + /// The final line ending is optional. A string that ends with a final line + /// ending will return the same lines as an otherwise identical string + /// without a final line ending. /// /// # Examples /// @@ -2971,7 +1016,7 @@ impl str { /// An iterator over the lines of a string. #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_deprecated(since = "1.4.0", reason = "use lines() instead now")] + #[deprecated(since = "1.4.0", note = "use lines() instead now")] #[inline] #[allow(deprecated)] pub fn lines_any(&self) -> LinesAny<'_> { @@ -2992,6 +1037,8 @@ impl str { /// /// assert!(utf16_len <= utf8_len); /// ``` + #[must_use = "this returns the encoded string as an iterator, \ + without modifying the original"] #[stable(feature = "encode_utf16", since = "1.8.0")] pub fn encode_utf16(&self) -> EncodeUtf16<'_> { EncodeUtf16 { chars: self.chars(), extra: 0 } @@ -3002,6 +1049,12 @@ impl str { /// /// Returns `false` if it does not. /// + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. + /// + /// [`char`]: prim@char + /// [pattern]: self::pattern + /// /// # Examples /// /// Basic usage: @@ -3023,6 +1076,12 @@ impl str { /// /// Returns `false` if it does not. /// + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. + /// + /// [`char`]: prim@char + /// [pattern]: self::pattern + /// /// # Examples /// /// Basic usage: @@ -3043,6 +1102,12 @@ impl str { /// /// Returns `false` if it does not. /// + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. + /// + /// [`char`]: prim@char + /// [pattern]: self::pattern + /// /// # Examples /// /// Basic usage: @@ -3066,21 +1131,22 @@ impl str { /// /// Returns [`None`] if the pattern doesn't match. /// - /// The pattern can be a `&str`, [`char`], or a closure that determines if - /// a character matches. + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. /// - /// [`None`]: option/enum.Option.html#variant.None + /// [`char`]: prim@char + /// [pattern]: self::pattern /// /// # Examples /// /// Simple patterns: /// /// ``` - /// let s = "Löwe 老虎 Léopard"; + /// let s = "Löwe 老虎 Léopard Gepardi"; /// /// assert_eq!(s.find('L'), Some(0)); /// assert_eq!(s.find('é'), Some(14)); - /// assert_eq!(s.find("Léopard"), Some(13)); + /// assert_eq!(s.find("pard"), Some(17)); /// ``` /// /// More complex patterns using point-free style and closures: @@ -3108,25 +1174,27 @@ impl str { pat.into_searcher(self).next_match().map(|(i, _)| i) } - /// Returns the byte index of the last character of this string slice that - /// matches the pattern. + /// Returns the byte index for the first character of the last match of the pattern in + /// this string slice. /// /// Returns [`None`] if the pattern doesn't match. /// - /// The pattern can be a `&str`, [`char`], or a closure that determines if - /// a character matches. + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. /// - /// [`None`]: option/enum.Option.html#variant.None + /// [`char`]: prim@char + /// [pattern]: self::pattern /// /// # Examples /// /// Simple patterns: /// /// ``` - /// let s = "Löwe 老虎 Léopard"; + /// let s = "Löwe 老虎 Léopard Gepardi"; /// /// assert_eq!(s.rfind('L'), Some(13)); /// assert_eq!(s.rfind('é'), Some(14)); + /// assert_eq!(s.rfind("pard"), Some(24)); /// ``` /// /// More complex patterns with closures: @@ -3158,8 +1226,11 @@ impl str { /// An iterator over substrings of this string slice, separated by /// characters matched by a pattern. /// - /// The pattern can be any type that implements the Pattern trait. Notable - /// examples are `&str`, [`char`], and closures that determines the split. + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. + /// + /// [`char`]: prim@char + /// [pattern]: self::pattern /// /// # Iterator behavior /// @@ -3167,12 +1238,10 @@ impl str { /// allows a reverse search and forward/reverse search yields the same /// elements. This is true for, e.g., [`char`], but not for `&str`. /// - /// [`DoubleEndedIterator`]: iter/trait.DoubleEndedIterator.html - /// /// If the pattern allows a reverse search but its results might differ /// from a forward search, the [`rsplit`] method can be used. /// - /// [`rsplit`]: #method.rsplit + /// [`rsplit`]: str::rsplit /// /// # Examples /// @@ -3198,6 +1267,13 @@ impl str { /// assert_eq!(v, ["lion", "tiger", "leopard"]); /// ``` /// + /// If the pattern is a slice of chars, split on each occurrence of any of the characters: + /// + /// ``` + /// let v: Vec<&str> = "2020-11-03 23:59".split(&['-', ' ', ':', '@'][..]).collect(); + /// assert_eq!(v, ["2020", "11", "03", "23", "59"]); + /// ``` + /// /// A more complex pattern, using a closure: /// /// ``` @@ -3259,7 +1335,7 @@ impl str { /// /// Use [`split_whitespace`] for this behavior. /// - /// [`split_whitespace`]: #method.split_whitespace + /// [`split_whitespace`]: str::split_whitespace #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn split<'a, P: Pattern<'a>>(&'a self, pat: P) -> Split<'a, P> { @@ -3277,10 +1353,15 @@ impl str { /// `split` in that `split_inclusive` leaves the matched part as the /// terminator of the substring. /// + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. + /// + /// [`char`]: prim@char + /// [pattern]: self::pattern + /// /// # Examples /// /// ``` - /// #![feature(split_inclusive)] /// let v: Vec<&str> = "Mary had a little lamb\nlittle lamb\nlittle lamb." /// .split_inclusive('\n').collect(); /// assert_eq!(v, ["Mary had a little lamb\n", "little lamb\n", "little lamb."]); @@ -3291,12 +1372,11 @@ impl str { /// That substring will be the last item returned by the iterator. /// /// ``` - /// #![feature(split_inclusive)] /// let v: Vec<&str> = "Mary had a little lamb\nlittle lamb\nlittle lamb.\n" /// .split_inclusive('\n').collect(); /// assert_eq!(v, ["Mary had a little lamb\n", "little lamb\n", "little lamb.\n"]); /// ``` - #[unstable(feature = "split_inclusive", issue = "none")] + #[stable(feature = "split_inclusive", since = "1.51.0")] #[inline] pub fn split_inclusive<'a, P: Pattern<'a>>(&'a self, pat: P) -> SplitInclusive<'a, P> { SplitInclusive(SplitInternal { @@ -3311,8 +1391,11 @@ impl str { /// An iterator over substrings of the given string slice, separated by /// characters matched by a pattern and yielded in reverse order. /// - /// The pattern can be any type that implements the Pattern trait. Notable - /// examples are `&str`, [`char`], and closures that determines the split. + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. + /// + /// [`char`]: prim@char + /// [pattern]: self::pattern /// /// # Iterator behavior /// @@ -3320,11 +1403,9 @@ impl str { /// search, and it will be a [`DoubleEndedIterator`] if a forward/reverse /// search yields the same elements. /// - /// [`DoubleEndedIterator`]: iter/trait.DoubleEndedIterator.html - /// /// For iterating from the front, the [`split`] method can be used. /// - /// [`split`]: #method.split + /// [`split`]: str::split /// /// # Examples /// @@ -3362,13 +1443,16 @@ impl str { /// An iterator over substrings of the given string slice, separated by /// characters matched by a pattern. /// - /// The pattern can be any type that implements the Pattern trait. Notable - /// examples are `&str`, [`char`], and closures that determines the split. + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. + /// + /// [`char`]: prim@char + /// [pattern]: self::pattern /// /// Equivalent to [`split`], except that the trailing substring /// is skipped if empty. /// - /// [`split`]: #method.split + /// [`split`]: str::split /// /// This method can be used for string data that is _terminated_, /// rather than _separated_ by a pattern. @@ -3379,12 +1463,10 @@ impl str { /// allows a reverse search and forward/reverse search yields the same /// elements. This is true for, e.g., [`char`], but not for `&str`. /// - /// [`DoubleEndedIterator`]: iter/trait.DoubleEndedIterator.html - /// /// If the pattern allows a reverse search but its results might differ /// from a forward search, the [`rsplit_terminator`] method can be used. /// - /// [`rsplit_terminator`]: #method.rsplit_terminator + /// [`rsplit_terminator`]: str::rsplit_terminator /// /// # Examples /// @@ -3396,6 +1478,9 @@ impl str { /// /// let v: Vec<&str> = "A..B..".split_terminator(".").collect(); /// assert_eq!(v, ["A", "", "B", ""]); + /// + /// let v: Vec<&str> = "A.B:C.D".split_terminator(&['.', ':'][..]).collect(); + /// assert_eq!(v, ["A", "B", "C", "D"]); /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -3406,15 +1491,16 @@ impl str { /// An iterator over substrings of `self`, separated by characters /// matched by a pattern and yielded in reverse order. /// - /// The pattern can be any type that implements the Pattern trait. Notable - /// examples are `&str`, [`char`], and closures that determines the split. - /// Additional libraries might provide more complex patterns like - /// regular expressions. + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. + /// + /// [`char`]: prim@char + /// [pattern]: self::pattern /// /// Equivalent to [`split`], except that the trailing substring is /// skipped if empty. /// - /// [`split`]: #method.split + /// [`split`]: str::split /// /// This method can be used for string data that is _terminated_, /// rather than _separated_ by a pattern. @@ -3428,7 +1514,7 @@ impl str { /// For iterating from the front, the [`split_terminator`] method can be /// used. /// - /// [`split_terminator`]: #method.split_terminator + /// [`split_terminator`]: str::split_terminator /// /// # Examples /// @@ -3438,6 +1524,9 @@ impl str { /// /// let v: Vec<&str> = "A..B..".rsplit_terminator(".").collect(); /// assert_eq!(v, ["", "B", "", "A"]); + /// + /// let v: Vec<&str> = "A.B:C.D".rsplit_terminator(&['.', ':'][..]).collect(); + /// assert_eq!(v, ["D", "C", "B", "A"]); /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -3454,8 +1543,11 @@ impl str { /// If `n` substrings are returned, the last substring (the `n`th substring) /// will contain the remainder of the string. /// - /// The pattern can be any type that implements the Pattern trait. Notable - /// examples are `&str`, [`char`], and closures that determines the split. + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. + /// + /// [`char`]: prim@char + /// [pattern]: self::pattern /// /// # Iterator behavior /// @@ -3465,7 +1557,7 @@ impl str { /// If the pattern allows a reverse search, the [`rsplitn`] method can be /// used. /// - /// [`rsplitn`]: #method.rsplitn + /// [`rsplitn`]: str::rsplitn /// /// # Examples /// @@ -3504,8 +1596,11 @@ impl str { /// If `n` substrings are returned, the last substring (the `n`th substring) /// will contain the remainder of the string. /// - /// The pattern can be any type that implements the Pattern trait. Notable - /// examples are `&str`, [`char`], and closures that determines the split. + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. + /// + /// [`char`]: prim@char + /// [pattern]: self::pattern /// /// # Iterator behavior /// @@ -3514,7 +1609,7 @@ impl str { /// /// For splitting from the front, the [`splitn`] method can be used. /// - /// [`splitn`]: #method.splitn + /// [`splitn`]: str::splitn /// /// # Examples /// @@ -3546,11 +1641,54 @@ impl str { RSplitN(self.splitn(n, pat).0) } + /// Splits the string on the first occurrence of the specified delimiter and + /// returns prefix before delimiter and suffix after delimiter. + /// + /// # Examples + /// + /// ``` + /// assert_eq!("cfg".split_once('='), None); + /// assert_eq!("cfg=".split_once('='), Some(("cfg", ""))); + /// assert_eq!("cfg=foo".split_once('='), Some(("cfg", "foo"))); + /// assert_eq!("cfg=foo=bar".split_once('='), Some(("cfg", "foo=bar"))); + /// ``` + #[stable(feature = "str_split_once", since = "1.52.0")] + #[inline] + pub fn split_once<'a, P: Pattern<'a>>(&'a self, delimiter: P) -> Option<(&'a str, &'a str)> { + let (start, end) = delimiter.into_searcher(self).next_match()?; + // SAFETY: `Searcher` is known to return valid indices. + unsafe { Some((self.get_unchecked(..start), self.get_unchecked(end..))) } + } + + /// Splits the string on the last occurrence of the specified delimiter and + /// returns prefix before delimiter and suffix after delimiter. + /// + /// # Examples + /// + /// ``` + /// assert_eq!("cfg".rsplit_once('='), None); + /// assert_eq!("cfg=foo".rsplit_once('='), Some(("cfg", "foo"))); + /// assert_eq!("cfg=foo=bar".rsplit_once('='), Some(("cfg=foo", "bar"))); + /// ``` + #[stable(feature = "str_split_once", since = "1.52.0")] + #[inline] + pub fn rsplit_once<'a, P>(&'a self, delimiter: P) -> Option<(&'a str, &'a str)> + where + P: Pattern<'a, Searcher: ReverseSearcher<'a>>, + { + let (start, end) = delimiter.into_searcher(self).next_match_back()?; + // SAFETY: `Searcher` is known to return valid indices. + unsafe { Some((self.get_unchecked(..start), self.get_unchecked(end..))) } + } + /// An iterator over the disjoint matches of a pattern within the given string /// slice. /// - /// The pattern can be a `&str`, [`char`], or a closure that determines if - /// a character matches. + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. + /// + /// [`char`]: prim@char + /// [pattern]: self::pattern /// /// # Iterator behavior /// @@ -3558,12 +1696,10 @@ impl str { /// allows a reverse search and forward/reverse search yields the same /// elements. This is true for, e.g., [`char`], but not for `&str`. /// - /// [`DoubleEndedIterator`]: iter/trait.DoubleEndedIterator.html - /// /// If the pattern allows a reverse search but its results might differ /// from a forward search, the [`rmatches`] method can be used. /// - /// [`rmatches`]: #method.rmatches + /// [`rmatches`]: str::matches /// /// # Examples /// @@ -3585,8 +1721,11 @@ impl str { /// An iterator over the disjoint matches of a pattern within this string slice, /// yielded in reverse order. /// - /// The pattern can be a `&str`, [`char`], or a closure that determines if - /// a character matches. + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. + /// + /// [`char`]: prim@char + /// [pattern]: self::pattern /// /// # Iterator behavior /// @@ -3594,11 +1733,9 @@ impl str { /// search, and it will be a [`DoubleEndedIterator`] if a forward/reverse /// search yields the same elements. /// - /// [`DoubleEndedIterator`]: iter/trait.DoubleEndedIterator.html - /// /// For iterating from the front, the [`matches`] method can be used. /// - /// [`matches`]: #method.matches + /// [`matches`]: str::matches /// /// # Examples /// @@ -3626,8 +1763,11 @@ impl str { /// For matches of `pat` within `self` that overlap, only the indices /// corresponding to the first match are returned. /// - /// The pattern can be a `&str`, [`char`], or a closure that determines - /// if a character matches. + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. + /// + /// [`char`]: prim@char + /// [pattern]: self::pattern /// /// # Iterator behavior /// @@ -3635,12 +1775,10 @@ impl str { /// allows a reverse search and forward/reverse search yields the same /// elements. This is true for, e.g., [`char`], but not for `&str`. /// - /// [`DoubleEndedIterator`]: iter/trait.DoubleEndedIterator.html - /// /// If the pattern allows a reverse search but its results might differ /// from a forward search, the [`rmatch_indices`] method can be used. /// - /// [`rmatch_indices`]: #method.rmatch_indices + /// [`rmatch_indices`]: str::rmatch_indices /// /// # Examples /// @@ -3668,8 +1806,11 @@ impl str { /// For matches of `pat` within `self` that overlap, only the indices /// corresponding to the last match are returned. /// - /// The pattern can be a `&str`, [`char`], or a closure that determines if a - /// character matches. + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. + /// + /// [`char`]: prim@char + /// [pattern]: self::pattern /// /// # Iterator behavior /// @@ -3677,11 +1818,9 @@ impl str { /// search, and it will be a [`DoubleEndedIterator`] if a forward/reverse /// search yields the same elements. /// - /// [`DoubleEndedIterator`]: iter/trait.DoubleEndedIterator.html - /// /// For iterating from the front, the [`match_indices`] method can be used. /// - /// [`match_indices`]: #method.match_indices + /// [`match_indices`]: str::match_indices /// /// # Examples /// @@ -3709,20 +1848,22 @@ impl str { /// Returns a string slice with leading and trailing whitespace removed. /// /// 'Whitespace' is defined according to the terms of the Unicode Derived - /// Core Property `White_Space`. + /// Core Property `White_Space`, which includes newlines. /// /// # Examples /// /// Basic usage: /// /// ``` - /// let s = " Hello\tworld\t"; + /// let s = "\n Hello\tworld\t\n"; /// /// assert_eq!("Hello\tworld", s.trim()); /// ``` + #[inline] #[must_use = "this returns the trimmed string as a slice, \ without modifying the original"] #[stable(feature = "rust1", since = "1.0.0")] + #[cfg_attr(not(test), rustc_diagnostic_item = "str_trim")] pub fn trim(&self) -> &str { self.trim_matches(|c: char| c.is_whitespace()) } @@ -3730,7 +1871,7 @@ impl str { /// Returns a string slice with leading whitespace removed. /// /// 'Whitespace' is defined according to the terms of the Unicode Derived - /// Core Property `White_Space`. + /// Core Property `White_Space`, which includes newlines. /// /// # Text directionality /// @@ -3744,8 +1885,8 @@ impl str { /// Basic usage: /// /// ``` - /// let s = " Hello\tworld\t"; - /// assert_eq!("Hello\tworld\t", s.trim_start()); + /// let s = "\n Hello\tworld\t\n"; + /// assert_eq!("Hello\tworld\t\n", s.trim_start()); /// ``` /// /// Directionality: @@ -3757,9 +1898,11 @@ impl str { /// let s = " עברית "; /// assert!(Some('ע') == s.trim_start().chars().next()); /// ``` + #[inline] #[must_use = "this returns the trimmed string as a new slice, \ without modifying the original"] #[stable(feature = "trim_direction", since = "1.30.0")] + #[cfg_attr(not(test), rustc_diagnostic_item = "str_trim_start")] pub fn trim_start(&self) -> &str { self.trim_start_matches(|c: char| c.is_whitespace()) } @@ -3767,7 +1910,7 @@ impl str { /// Returns a string slice with trailing whitespace removed. /// /// 'Whitespace' is defined according to the terms of the Unicode Derived - /// Core Property `White_Space`. + /// Core Property `White_Space`, which includes newlines. /// /// # Text directionality /// @@ -3781,8 +1924,8 @@ impl str { /// Basic usage: /// /// ``` - /// let s = " Hello\tworld\t"; - /// assert_eq!(" Hello\tworld", s.trim_end()); + /// let s = "\n Hello\tworld\t\n"; + /// assert_eq!("\n Hello\tworld", s.trim_end()); /// ``` /// /// Directionality: @@ -3794,9 +1937,11 @@ impl str { /// let s = " עברית "; /// assert!(Some('ת') == s.trim_end().chars().rev().next()); /// ``` + #[inline] #[must_use = "this returns the trimmed string as a new slice, \ without modifying the original"] #[stable(feature = "trim_direction", since = "1.30.0")] + #[cfg_attr(not(test), rustc_diagnostic_item = "str_trim_end")] pub fn trim_end(&self) -> &str { self.trim_end_matches(|c: char| c.is_whitespace()) } @@ -3832,12 +1977,11 @@ impl str { /// let s = " עברית"; /// assert!(Some('ע') == s.trim_left().chars().next()); /// ``` + #[must_use = "this returns the trimmed string as a new slice, \ + without modifying the original"] + #[inline] #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_deprecated( - since = "1.33.0", - reason = "superseded by `trim_start`", - suggestion = "trim_start" - )] + #[deprecated(since = "1.33.0", note = "superseded by `trim_start`", suggestion = "trim_start")] pub fn trim_left(&self) -> &str { self.trim_start() } @@ -3873,12 +2017,11 @@ impl str { /// let s = "עברית "; /// assert!(Some('ת') == s.trim_right().chars().rev().next()); /// ``` + #[must_use = "this returns the trimmed string as a new slice, \ + without modifying the original"] + #[inline] #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_deprecated( - since = "1.33.0", - reason = "superseded by `trim_end`", - suggestion = "trim_end" - )] + #[deprecated(since = "1.33.0", note = "superseded by `trim_end`", suggestion = "trim_end")] pub fn trim_right(&self) -> &str { self.trim_end() } @@ -3886,8 +2029,11 @@ impl str { /// Returns a string slice with all prefixes and suffixes that match a /// pattern repeatedly removed. /// - /// The pattern can be a [`char`] or a closure that determines if a - /// character matches. + /// The [pattern] can be a [`char`], a slice of [`char`]s, or a function + /// or closure that determines if a character matches. + /// + /// [`char`]: prim@char + /// [pattern]: self::pattern /// /// # Examples /// @@ -3931,8 +2077,11 @@ impl str { /// Returns a string slice with all prefixes that match a pattern /// repeatedly removed. /// - /// The pattern can be a `&str`, [`char`], or a closure that determines if - /// a character matches. + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. + /// + /// [`char`]: prim@char + /// [pattern]: self::pattern /// /// # Text directionality /// @@ -3967,83 +2116,70 @@ impl str { /// Returns a string slice with the prefix removed. /// - /// If the string starts with the pattern `prefix`, `Some` is returned with the substring where - /// the prefix is removed. Unlike `trim_start_matches`, this method removes the prefix exactly - /// once. + /// If the string starts with the pattern `prefix`, returns substring after the prefix, wrapped + /// in `Some`. Unlike `trim_start_matches`, this method removes the prefix exactly once. + /// + /// If the string does not start with `prefix`, returns `None`. /// - /// If the string does not start with `prefix`, `None` is returned. + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. + /// + /// [`char`]: prim@char + /// [pattern]: self::pattern /// /// # Examples /// /// ``` - /// #![feature(str_strip)] - /// - /// assert_eq!("foobar".strip_prefix("foo"), Some("bar")); - /// assert_eq!("foobar".strip_prefix("bar"), None); + /// assert_eq!("foo:bar".strip_prefix("foo:"), Some("bar")); + /// assert_eq!("foo:bar".strip_prefix("bar"), None); /// assert_eq!("foofoo".strip_prefix("foo"), Some("foo")); /// ``` #[must_use = "this returns the remaining substring as a new slice, \ without modifying the original"] - #[unstable(feature = "str_strip", reason = "newly added", issue = "67302")] + #[stable(feature = "str_strip", since = "1.45.0")] pub fn strip_prefix<'a, P: Pattern<'a>>(&'a self, prefix: P) -> Option<&'a str> { - let mut matcher = prefix.into_searcher(self); - if let SearchStep::Match(start, len) = matcher.next() { - debug_assert_eq!( - start, 0, - "The first search step from Searcher \ - must include the first character" - ); - // SAFETY: `Searcher` is known to return valid indices. - unsafe { Some(self.get_unchecked(len..)) } - } else { - None - } + prefix.strip_prefix_of(self) } /// Returns a string slice with the suffix removed. /// - /// If the string ends with the pattern `suffix`, `Some` is returned with the substring where - /// the suffix is removed. Unlike `trim_end_matches`, this method removes the suffix exactly - /// once. + /// If the string ends with the pattern `suffix`, returns the substring before the suffix, + /// wrapped in `Some`. Unlike `trim_end_matches`, this method removes the suffix exactly once. + /// + /// If the string does not end with `suffix`, returns `None`. + /// + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. /// - /// If the string does not end with `suffix`, `None` is returned. + /// [`char`]: prim@char + /// [pattern]: self::pattern /// /// # Examples /// /// ``` - /// #![feature(str_strip)] - /// assert_eq!("barfoo".strip_suffix("foo"), Some("bar")); - /// assert_eq!("barfoo".strip_suffix("bar"), None); + /// assert_eq!("bar:foo".strip_suffix(":foo"), Some("bar")); + /// assert_eq!("bar:foo".strip_suffix("bar"), None); /// assert_eq!("foofoo".strip_suffix("foo"), Some("foo")); /// ``` #[must_use = "this returns the remaining substring as a new slice, \ without modifying the original"] - #[unstable(feature = "str_strip", reason = "newly added", issue = "67302")] + #[stable(feature = "str_strip", since = "1.45.0")] pub fn strip_suffix<'a, P>(&'a self, suffix: P) -> Option<&'a str> where P: Pattern<'a>,

>::Searcher: ReverseSearcher<'a>, { - let mut matcher = suffix.into_searcher(self); - if let SearchStep::Match(start, end) = matcher.next_back() { - debug_assert_eq!( - end, - self.len(), - "The first search step from ReverseSearcher \ - must include the last character" - ); - // SAFETY: `Searcher` is known to return valid indices. - unsafe { Some(self.get_unchecked(..start)) } - } else { - None - } + suffix.strip_suffix_of(self) } /// Returns a string slice with all suffixes that match a pattern /// repeatedly removed. /// - /// The pattern can be a `&str`, [`char`], or a closure that - /// determines if a character matches. + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. + /// + /// [`char`]: prim@char + /// [pattern]: self::pattern /// /// # Text directionality /// @@ -4088,10 +2224,11 @@ impl str { /// Returns a string slice with all prefixes that match a pattern /// repeatedly removed. /// - /// The pattern can be a `&str`, [`char`], or a closure that determines if - /// a character matches. + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. /// - /// [`char`]: primitive.char.html + /// [`char`]: prim@char + /// [pattern]: self::pattern /// /// # Text directionality /// @@ -4112,9 +2249,9 @@ impl str { /// assert_eq!("12foo1bar12".trim_left_matches(x), "foo1bar12"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_deprecated( + #[deprecated( since = "1.33.0", - reason = "superseded by `trim_start_matches`", + note = "superseded by `trim_start_matches`", suggestion = "trim_start_matches" )] pub fn trim_left_matches<'a, P: Pattern<'a>>(&'a self, pat: P) -> &'a str { @@ -4124,10 +2261,11 @@ impl str { /// Returns a string slice with all suffixes that match a pattern /// repeatedly removed. /// - /// The pattern can be a `&str`, [`char`], or a closure that - /// determines if a character matches. + /// The [pattern] can be a `&str`, [`char`], a slice of [`char`]s, or a + /// function or closure that determines if a character matches. /// - /// [`char`]: primitive.char.html + /// [`char`]: prim@char + /// [pattern]: self::pattern /// /// # Text directionality /// @@ -4154,9 +2292,9 @@ impl str { /// assert_eq!("1fooX".trim_right_matches(|c| c == '1' || c == 'X'), "1foo"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_deprecated( + #[deprecated( since = "1.33.0", - reason = "superseded by `trim_end_matches`", + note = "superseded by `trim_end_matches`", suggestion = "trim_end_matches" )] pub fn trim_right_matches<'a, P>(&'a self, pat: P) -> &'a str @@ -4174,16 +2312,15 @@ impl str { /// helps the inference algorithm understand specifically which type /// you're trying to parse into. /// - /// `parse` can parse any type that implements the [`FromStr`] trait. - /// - /// [`FromStr`]: str/trait.FromStr.html + /// `parse` can parse into any type that implements the [`FromStr`] trait. + /// /// # Errors /// /// Will return [`Err`] if it's not possible to parse this string slice into /// the desired type. /// - /// [`Err`]: str/trait.FromStr.html#associatedtype.Err + /// [`Err`]: FromStr::Err /// /// # Examples /// @@ -4228,12 +2365,13 @@ impl str { /// assert!(!non_ascii.is_ascii()); /// ``` #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] + #[must_use] #[inline] pub fn is_ascii(&self) -> bool { // We can treat each byte as character here: all multibyte characters - // start with a byte that is not in the ascii range, so we will stop + // start with a byte that is not in the ASCII range, so we will stop // there already. - self.bytes().all(|b| b.is_ascii()) + self.as_bytes().is_ascii() } /// Checks that two strings are an ASCII case-insensitive match. @@ -4249,6 +2387,7 @@ impl str { /// assert!(!"Ferrös".eq_ignore_ascii_case("FERRÖS")); /// ``` #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] + #[must_use] #[inline] pub fn eq_ignore_ascii_case(&self, other: &str) -> bool { self.as_bytes().eq_ignore_ascii_case(other.as_bytes()) @@ -4260,9 +2399,9 @@ impl str { /// but non-ASCII letters are unchanged. /// /// To return a new uppercased value without modifying the existing one, use - /// [`to_ascii_uppercase`]. + /// [`to_ascii_uppercase()`]. /// - /// [`to_ascii_uppercase`]: #method.to_ascii_uppercase + /// [`to_ascii_uppercase()`]: #method.to_ascii_uppercase /// /// # Examples /// @@ -4274,8 +2413,9 @@ impl str { /// assert_eq!("GRüßE, JüRGEN ❤", s); /// ``` #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] + #[inline] pub fn make_ascii_uppercase(&mut self) { - // SAFETY: safe because we transmute two types with the same layout. + // SAFETY: changing ASCII letters only does not invalidate UTF-8. let me = unsafe { self.as_bytes_mut() }; me.make_ascii_uppercase() } @@ -4286,9 +2426,9 @@ impl str { /// but non-ASCII letters are unchanged. /// /// To return a new lowercased value without modifying the existing one, use - /// [`to_ascii_lowercase`]. + /// [`to_ascii_lowercase()`]. /// - /// [`to_ascii_lowercase`]: #method.to_ascii_lowercase + /// [`to_ascii_lowercase()`]: #method.to_ascii_lowercase /// /// # Examples /// @@ -4300,8 +2440,9 @@ impl str { /// assert_eq!("grÜße, jÜrgen ❤", s); /// ``` #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] + #[inline] pub fn make_ascii_lowercase(&mut self) { - // SAFETY: safe because we transmute two types with the same layout. + // SAFETY: changing ASCII letters only does not invalidate UTF-8. let me = unsafe { self.as_bytes_mut() }; me.make_ascii_lowercase() } @@ -4311,15 +2452,13 @@ impl str { /// Note: only extended grapheme codepoints that begin the string will be /// escaped. /// - /// [`char::escape_debug`]: ../std/primitive.char.html#method.escape_debug - /// /// # Examples /// /// As an iterator: /// /// ``` /// for c in "❤\n!".escape_debug() { - /// print!("{}", c); + /// print!("{c}"); /// } /// println!(); /// ``` @@ -4342,13 +2481,15 @@ impl str { /// ``` /// assert_eq!("❤\n!".escape_debug().to_string(), "❤\\n!"); /// ``` + #[must_use = "this returns the escaped string as an iterator, \ + without modifying the original"] #[stable(feature = "str_escape", since = "1.34.0")] pub fn escape_debug(&self) -> EscapeDebug<'_> { let mut chars = self.chars(); EscapeDebug { inner: chars .next() - .map(|first| first.escape_debug_ext(true)) + .map(|first| first.escape_debug_ext(EscapeDebugExtArgs::ESCAPE_ALL)) .into_iter() .flatten() .chain(chars.flat_map(CharEscapeDebugContinue)), @@ -4357,15 +2498,13 @@ impl str { /// Return an iterator that escapes each char in `self` with [`char::escape_default`]. /// - /// [`char::escape_default`]: ../std/primitive.char.html#method.escape_default - /// /// # Examples /// /// As an iterator: /// /// ``` /// for c in "❤\n!".escape_default() { - /// print!("{}", c); + /// print!("{c}"); /// } /// println!(); /// ``` @@ -4388,6 +2527,8 @@ impl str { /// ``` /// assert_eq!("❤\n!".escape_default().to_string(), "\\u{2764}\\n!"); /// ``` + #[must_use = "this returns the escaped string as an iterator, \ + without modifying the original"] #[stable(feature = "str_escape", since = "1.34.0")] pub fn escape_default(&self) -> EscapeDefault<'_> { EscapeDefault { inner: self.chars().flat_map(CharEscapeDefault) } @@ -4395,15 +2536,13 @@ impl str { /// Return an iterator that escapes each char in `self` with [`char::escape_unicode`]. /// - /// [`char::escape_unicode`]: ../std/primitive.char.html#method.escape_unicode - /// /// # Examples /// /// As an iterator: /// /// ``` /// for c in "❤\n!".escape_unicode() { - /// print!("{}", c); + /// print!("{c}"); /// } /// println!(); /// ``` @@ -4426,28 +2565,14 @@ impl str { /// ``` /// assert_eq!("❤\n!".escape_unicode().to_string(), "\\u{2764}\\u{a}\\u{21}"); /// ``` + #[must_use = "this returns the escaped string as an iterator, \ + without modifying the original"] #[stable(feature = "str_escape", since = "1.34.0")] pub fn escape_unicode(&self) -> EscapeUnicode<'_> { EscapeUnicode { inner: self.chars().flat_map(CharEscapeUnicode) } } } -impl_fn_for_zst! { - #[derive(Clone)] - struct CharEscapeDebugContinue impl Fn = |c: char| -> char::EscapeDebug { - c.escape_debug_ext(false) - }; - - #[derive(Clone)] - struct CharEscapeUnicode impl Fn = |c: char| -> char::EscapeUnicode { - c.escape_unicode() - }; - #[derive(Clone)] - struct CharEscapeDefault impl Fn = |c: char| -> char::EscapeDefault { - c.escape_default() - }; -} - #[stable(feature = "rust1", since = "1.0.0")] impl AsRef<[u8]> for str { #[inline] @@ -4457,8 +2582,10 @@ impl AsRef<[u8]> for str { } #[stable(feature = "rust1", since = "1.0.0")] -impl Default for &str { +#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] +impl const Default for &str { /// Creates an empty str + #[inline] fn default() -> Self { "" } @@ -4467,54 +2594,40 @@ impl Default for &str { #[stable(feature = "default_mut_str", since = "1.28.0")] impl Default for &mut str { /// Creates an empty mutable str + #[inline] fn default() -> Self { // SAFETY: The empty string is valid UTF-8. unsafe { from_utf8_unchecked_mut(&mut []) } } } -/// An iterator over the non-whitespace substrings of a string, -/// separated by any amount of whitespace. -/// -/// This struct is created by the [`split_whitespace`] method on [`str`]. -/// See its documentation for more. -/// -/// [`split_whitespace`]: ../../std/primitive.str.html#method.split_whitespace -/// [`str`]: ../../std/primitive.str.html -#[stable(feature = "split_whitespace", since = "1.1.0")] -#[derive(Clone, Debug)] -pub struct SplitWhitespace<'a> { - inner: Filter, IsNotEmpty>, -} +impl_fn_for_zst! { + /// A nameable, cloneable fn type + #[derive(Clone)] + struct LinesAnyMap impl<'a> Fn = |line: &'a str| -> &'a str { + let l = line.len(); + if l > 0 && line.as_bytes()[l - 1] == b'\r' { &line[0 .. l - 1] } + else { line } + }; -/// An iterator over the non-ASCII-whitespace substrings of a string, -/// separated by any amount of ASCII whitespace. -/// -/// This struct is created by the [`split_ascii_whitespace`] method on [`str`]. -/// See its documentation for more. -/// -/// [`split_ascii_whitespace`]: ../../std/primitive.str.html#method.split_ascii_whitespace -/// [`str`]: ../../std/primitive.str.html -#[stable(feature = "split_ascii_whitespace", since = "1.34.0")] -#[derive(Clone, Debug)] -pub struct SplitAsciiWhitespace<'a> { - inner: Map, BytesIsNotEmpty>, UnsafeBytesToStr>, -} + #[derive(Clone)] + struct CharEscapeDebugContinue impl Fn = |c: char| -> char::EscapeDebug { + c.escape_debug_ext(EscapeDebugExtArgs { + escape_grapheme_extended: false, + escape_single_quote: true, + escape_double_quote: true + }) + }; -/// An iterator over the substrings of a string, -/// terminated by a substring matching to a predicate function -/// Unlike `Split`, it contains the matched part as a terminator -/// of the subslice. -/// -/// This struct is created by the [`split_inclusive`] method on [`str`]. -/// See its documentation for more. -/// -/// [`split_inclusive`]: ../../std/primitive.str.html#method.split_inclusive -/// [`str`]: ../../std/primitive.str.html -#[unstable(feature = "split_inclusive", issue = "none")] -pub struct SplitInclusive<'a, P: Pattern<'a>>(SplitInternal<'a, P>); + #[derive(Clone)] + struct CharEscapeUnicode impl Fn = |c: char| -> char::EscapeUnicode { + c.escape_unicode() + }; + #[derive(Clone)] + struct CharEscapeDefault impl Fn = |c: char| -> char::EscapeDefault { + c.escape_default() + }; -impl_fn_for_zst! { #[derive(Clone)] struct IsWhitespace impl Fn = |c: char| -> bool { c.is_whitespace() @@ -4542,231 +2655,5 @@ impl_fn_for_zst! { }; } -#[stable(feature = "split_whitespace", since = "1.1.0")] -impl<'a> Iterator for SplitWhitespace<'a> { - type Item = &'a str; - - #[inline] - fn next(&mut self) -> Option<&'a str> { - self.inner.next() - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.inner.size_hint() - } - - #[inline] - fn last(mut self) -> Option<&'a str> { - self.next_back() - } -} - -#[stable(feature = "split_whitespace", since = "1.1.0")] -impl<'a> DoubleEndedIterator for SplitWhitespace<'a> { - #[inline] - fn next_back(&mut self) -> Option<&'a str> { - self.inner.next_back() - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for SplitWhitespace<'_> {} - -#[stable(feature = "split_ascii_whitespace", since = "1.34.0")] -impl<'a> Iterator for SplitAsciiWhitespace<'a> { - type Item = &'a str; - - #[inline] - fn next(&mut self) -> Option<&'a str> { - self.inner.next() - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.inner.size_hint() - } - - #[inline] - fn last(mut self) -> Option<&'a str> { - self.next_back() - } -} - -#[stable(feature = "split_ascii_whitespace", since = "1.34.0")] -impl<'a> DoubleEndedIterator for SplitAsciiWhitespace<'a> { - #[inline] - fn next_back(&mut self) -> Option<&'a str> { - self.inner.next_back() - } -} - -#[stable(feature = "split_ascii_whitespace", since = "1.34.0")] -impl FusedIterator for SplitAsciiWhitespace<'_> {} - -#[unstable(feature = "split_inclusive", issue = "none")] -impl<'a, P: Pattern<'a>> Iterator for SplitInclusive<'a, P> { - type Item = &'a str; - - #[inline] - fn next(&mut self) -> Option<&'a str> { - self.0.next_inclusive() - } -} - -#[unstable(feature = "split_inclusive", issue = "none")] -impl<'a, P: Pattern<'a, Searcher: fmt::Debug>> fmt::Debug for SplitInclusive<'a, P> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SplitInclusive").field("0", &self.0).finish() - } -} - -// FIXME(#26925) Remove in favor of `#[derive(Clone)]` -#[unstable(feature = "split_inclusive", issue = "none")] -impl<'a, P: Pattern<'a, Searcher: Clone>> Clone for SplitInclusive<'a, P> { - fn clone(&self) -> Self { - SplitInclusive(self.0.clone()) - } -} - -#[unstable(feature = "split_inclusive", issue = "none")] -impl<'a, P: Pattern<'a, Searcher: ReverseSearcher<'a>>> DoubleEndedIterator - for SplitInclusive<'a, P> -{ - #[inline] - fn next_back(&mut self) -> Option<&'a str> { - self.0.next_back_inclusive() - } -} - -#[unstable(feature = "split_inclusive", issue = "none")] -impl<'a, P: Pattern<'a>> FusedIterator for SplitInclusive<'a, P> {} - -/// An iterator of [`u16`] over the string encoded as UTF-16. -/// -/// [`u16`]: ../../std/primitive.u16.html -/// -/// This struct is created by the [`encode_utf16`] method on [`str`]. -/// See its documentation for more. -/// -/// [`encode_utf16`]: ../../std/primitive.str.html#method.encode_utf16 -/// [`str`]: ../../std/primitive.str.html -#[derive(Clone)] -#[stable(feature = "encode_utf16", since = "1.8.0")] -pub struct EncodeUtf16<'a> { - chars: Chars<'a>, - extra: u16, -} - -#[stable(feature = "collection_debug", since = "1.17.0")] -impl fmt::Debug for EncodeUtf16<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("EncodeUtf16 { .. }") - } -} - -#[stable(feature = "encode_utf16", since = "1.8.0")] -impl<'a> Iterator for EncodeUtf16<'a> { - type Item = u16; - - #[inline] - fn next(&mut self) -> Option { - if self.extra != 0 { - let tmp = self.extra; - self.extra = 0; - return Some(tmp); - } - - let mut buf = [0; 2]; - self.chars.next().map(|ch| { - let n = ch.encode_utf16(&mut buf).len(); - if n == 2 { - self.extra = buf[1]; - } - buf[0] - }) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let (low, high) = self.chars.size_hint(); - // every char gets either one u16 or two u16, - // so this iterator is between 1 or 2 times as - // long as the underlying iterator. - (low, high.and_then(|n| n.checked_mul(2))) - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for EncodeUtf16<'_> {} - -/// The return type of [`str::escape_debug`]. -/// -/// [`str::escape_debug`]: ../../std/primitive.str.html#method.escape_debug -#[stable(feature = "str_escape", since = "1.34.0")] -#[derive(Clone, Debug)] -pub struct EscapeDebug<'a> { - inner: Chain< - Flatten>, - FlatMap, char::EscapeDebug, CharEscapeDebugContinue>, - >, -} - -/// The return type of [`str::escape_default`]. -/// -/// [`str::escape_default`]: ../../std/primitive.str.html#method.escape_default -#[stable(feature = "str_escape", since = "1.34.0")] -#[derive(Clone, Debug)] -pub struct EscapeDefault<'a> { - inner: FlatMap, char::EscapeDefault, CharEscapeDefault>, -} - -/// The return type of [`str::escape_unicode`]. -/// -/// [`str::escape_unicode`]: ../../std/primitive.str.html#method.escape_unicode -#[stable(feature = "str_escape", since = "1.34.0")] -#[derive(Clone, Debug)] -pub struct EscapeUnicode<'a> { - inner: FlatMap, char::EscapeUnicode, CharEscapeUnicode>, -} - -macro_rules! escape_types_impls { - ($( $Name: ident ),+) => {$( - #[stable(feature = "str_escape", since = "1.34.0")] - impl<'a> fmt::Display for $Name<'a> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.clone().try_for_each(|c| f.write_char(c)) - } - } - - #[stable(feature = "str_escape", since = "1.34.0")] - impl<'a> Iterator for $Name<'a> { - type Item = char; - - #[inline] - fn next(&mut self) -> Option { self.inner.next() } - - #[inline] - fn size_hint(&self) -> (usize, Option) { self.inner.size_hint() } - - #[inline] - fn try_fold(&mut self, init: Acc, fold: Fold) -> R where - Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, R: Try - { - self.inner.try_fold(init, fold) - } - - #[inline] - fn fold(self, init: Acc, fold: Fold) -> Acc - where Fold: FnMut(Acc, Self::Item) -> Acc, - { - self.inner.fold(init, fold) - } - } - - #[stable(feature = "str_escape", since = "1.34.0")] - impl<'a> FusedIterator for $Name<'a> {} - )+} -} - -escape_types_impls!(EscapeDebug, EscapeDefault, EscapeUnicode); +#[stable(feature = "rust1", since = "1.0.0")] +impl !crate::error::Error for &str {} diff --git a/crux-mir/lib/core/src/str/pattern.rs b/crux-mir/lib/core/src/str/pattern.rs index ffa418cba..19da6d2fb 100644 --- a/crux-mir/lib/core/src/str/pattern.rs +++ b/crux-mir/lib/core/src/str/pattern.rs @@ -1,7 +1,36 @@ //! The string Pattern API. //! +//! The Pattern API provides a generic mechanism for using different pattern +//! types when searching through a string. +//! //! For more details, see the traits [`Pattern`], [`Searcher`], //! [`ReverseSearcher`], and [`DoubleEndedSearcher`]. +//! +//! Although this API is unstable, it is exposed via stable APIs on the +//! [`str`] type. +//! +//! # Examples +//! +//! [`Pattern`] is [implemented][pattern-impls] in the stable API for +//! [`&str`][`str`], [`char`], slices of [`char`], and functions and closures +//! implementing `FnMut(char) -> bool`. +//! +//! ``` +//! let s = "Can you find a needle in a haystack?"; +//! +//! // &str pattern +//! assert_eq!(s.find("you"), Some(4)); +//! // char pattern +//! assert_eq!(s.find('n'), Some(2)); +//! // array of chars pattern +//! assert_eq!(s.find(&['a', 'e', 'i', 'o', 'u']), Some(1)); +//! // slice of chars pattern +//! assert_eq!(s.find(&['a', 'e', 'i', 'o', 'u'][..]), Some(1)); +//! // closure pattern +//! assert_eq!(s.find(|c: char| c.is_ascii_punctuation()), Some(35)); +//! ``` +//! +//! [pattern-impls]: Pattern#implementors #![unstable( feature = "pattern", @@ -10,23 +39,63 @@ )] use crate::cmp; +use crate::cmp::Ordering; use crate::fmt; use crate::slice::memchr; -use crate::usize; // Pattern /// A string pattern. /// /// A `Pattern<'a>` expresses that the implementing type -/// can be used as a string pattern for searching in a `&'a str`. +/// can be used as a string pattern for searching in a [`&'a str`][str]. /// /// For example, both `'a'` and `"aa"` are patterns that /// would match at index `1` in the string `"baaaab"`. /// /// The trait itself acts as a builder for an associated -/// `Searcher` type, which does the actual work of finding +/// [`Searcher`] type, which does the actual work of finding /// occurrences of the pattern in a string. +/// +/// Depending on the type of the pattern, the behaviour of methods like +/// [`str::find`] and [`str::contains`] can change. The table below describes +/// some of those behaviours. +/// +/// | Pattern type | Match condition | +/// |--------------------------|-------------------------------------------| +/// | `&str` | is substring | +/// | `char` | is contained in string | +/// | `&[char]` | any char in slice is contained in string | +/// | `F: FnMut(char) -> bool` | `F` returns `true` for a char in string | +/// | `&&str` | is substring | +/// | `&String` | is substring | +/// +/// # Examples +/// +/// ``` +/// // &str +/// assert_eq!("abaaa".find("ba"), Some(1)); +/// assert_eq!("abaaa".find("bac"), None); +/// +/// // char +/// assert_eq!("abaaa".find('a'), Some(0)); +/// assert_eq!("abaaa".find('b'), Some(1)); +/// assert_eq!("abaaa".find('c'), None); +/// +/// // &[char; N] +/// assert_eq!("ab".find(&['b', 'a']), Some(0)); +/// assert_eq!("abaaa".find(&['a', 'z']), Some(0)); +/// assert_eq!("abaaa".find(&['c', 'd']), None); +/// +/// // &[char] +/// assert_eq!("ab".find(&['b', 'a'][..]), Some(0)); +/// assert_eq!("abaaa".find(&['a', 'z'][..]), Some(0)); +/// assert_eq!("abaaa".find(&['c', 'd'][..]), None); +/// +/// // FnMut(char) -> bool +/// assert_eq!("abcdef_z".find(|ch| ch > 'd' && ch < 'y'), Some(4)); +/// assert_eq!("abcddd_z".find(|ch| ch > 'd' && ch < 'y'), None); +/// ``` pub trait Pattern<'a>: Sized { /// Associated searcher for this pattern type Searcher: Searcher<'a>; @@ -55,11 +124,47 @@ pub trait Pattern<'a>: Sized { { matches!(self.into_searcher(haystack).next_back(), SearchStep::Match(_, j) if haystack.len() == j) } + + /// Removes the pattern from the front of haystack, if it matches. + #[inline] + fn strip_prefix_of(self, haystack: &'a str) -> Option<&'a str> { + if let SearchStep::Match(start, len) = self.into_searcher(haystack).next() { + debug_assert_eq!( + start, 0, + "The first search step from Searcher \ + must include the first character" + ); + // SAFETY: `Searcher` is known to return valid indices. + unsafe { Some(haystack.get_unchecked(len..)) } + } else { + None + } + } + + /// Removes the pattern from the back of haystack, if it matches. + #[inline] + fn strip_suffix_of(self, haystack: &'a str) -> Option<&'a str> + where + Self::Searcher: ReverseSearcher<'a>, + { + if let SearchStep::Match(start, end) = self.into_searcher(haystack).next_back() { + debug_assert_eq!( + end, + haystack.len(), + "The first search step from ReverseSearcher \ + must include the last character" + ); + // SAFETY: `Searcher` is known to return valid indices. + unsafe { Some(haystack.get_unchecked(..start)) } + } else { + None + } + } } // Searcher -/// Result of calling `Searcher::next()` or `ReverseSearcher::next_back()`. +/// Result of calling [`Searcher::next()`] or [`ReverseSearcher::next_back()`]. #[derive(Copy, Clone, Eq, PartialEq, Debug)] pub enum SearchStep { /// Expresses that a match of the pattern has been found at @@ -82,44 +187,47 @@ pub enum SearchStep { /// matches of a pattern starting from the front (left) of a string. /// /// It will be implemented by associated `Searcher` -/// types of the `Pattern` trait. +/// types of the [`Pattern`] trait. /// /// The trait is marked unsafe because the indices returned by the -/// `next()` methods are required to lie on valid utf8 boundaries in -/// the haystack. This enables consumers of this trait to +/// [`next()`][Searcher::next] methods are required to lie on valid utf8 +/// boundaries in the haystack. This enables consumers of this trait to /// slice the haystack without additional runtime checks. pub unsafe trait Searcher<'a> { /// Getter for the underlying string to be searched in /// - /// Will always return the same `&str` + /// Will always return the same [`&str`][str]. fn haystack(&self) -> &'a str; /// Performs the next search step starting from the front. /// - /// - Returns `Match(a, b)` if `haystack[a..b]` matches the pattern. - /// - Returns `Reject(a, b)` if `haystack[a..b]` can not match the - /// pattern, even partially. - /// - Returns `Done` if every byte of the haystack has been visited + /// - Returns [`Match(a, b)`][SearchStep::Match] if `haystack[a..b]` matches + /// the pattern. + /// - Returns [`Reject(a, b)`][SearchStep::Reject] if `haystack[a..b]` can + /// not match the pattern, even partially. + /// - Returns [`Done`][SearchStep::Done] if every byte of the haystack has + /// been visited. /// - /// The stream of `Match` and `Reject` values up to a `Done` + /// The stream of [`Match`][SearchStep::Match] and + /// [`Reject`][SearchStep::Reject] values up to a [`Done`][SearchStep::Done] /// will contain index ranges that are adjacent, non-overlapping, /// covering the whole haystack, and laying on utf8 boundaries. /// - /// A `Match` result needs to contain the whole matched pattern, - /// however `Reject` results may be split up into arbitrary - /// many adjacent fragments. Both ranges may have zero length. + /// A [`Match`][SearchStep::Match] result needs to contain the whole matched + /// pattern, however [`Reject`][SearchStep::Reject] results may be split up + /// into arbitrary many adjacent fragments. Both ranges may have zero length. /// /// As an example, the pattern `"aaa"` and the haystack `"cbaaaaab"` /// might produce the stream /// `[Reject(0, 1), Reject(1, 2), Match(2, 5), Reject(5, 8)]` fn next(&mut self) -> SearchStep; - /// Finds the next `Match` result. See `next()` + /// Finds the next [`Match`][SearchStep::Match] result. See [`next()`][Searcher::next]. /// - /// Unlike next(), there is no guarantee that the returned ranges - /// of this and next_reject will overlap. This will return (start_match, end_match), - /// where start_match is the index of where the match begins, and end_match is - /// the index after the end of the match. + /// Unlike [`next()`][Searcher::next], there is no guarantee that the returned ranges + /// of this and [`next_reject`][Searcher::next_reject] will overlap. This will return + /// `(start_match, end_match)`, where start_match is the index of where + /// the match begins, and end_match is the index after the end of the match. #[inline] fn next_match(&mut self) -> Option<(usize, usize)> { loop { @@ -131,10 +239,11 @@ pub unsafe trait Searcher<'a> { } } - /// Finds the next `Reject` result. See `next()` and `next_match()` + /// Finds the next [`Reject`][SearchStep::Reject] result. See [`next()`][Searcher::next] + /// and [`next_match()`][Searcher::next_match]. /// - /// Unlike next(), there is no guarantee that the returned ranges - /// of this and next_match will overlap. + /// Unlike [`next()`][Searcher::next], there is no guarantee that the returned ranges + /// of this and [`next_match`][Searcher::next_match] will overlap. #[inline] fn next_reject(&mut self) -> Option<(usize, usize)> { loop { @@ -152,37 +261,41 @@ pub unsafe trait Searcher<'a> { /// This trait provides methods for searching for non-overlapping /// matches of a pattern starting from the back (right) of a string. /// -/// It will be implemented by associated `Searcher` -/// types of the `Pattern` trait if the pattern supports searching +/// It will be implemented by associated [`Searcher`] +/// types of the [`Pattern`] trait if the pattern supports searching /// for it from the back. /// /// The index ranges returned by this trait are not required /// to exactly match those of the forward search in reverse. /// -/// For the reason why this trait is marked unsafe, see them -/// parent trait `Searcher`. +/// For the reason why this trait is marked unsafe, see the +/// parent trait [`Searcher`]. pub unsafe trait ReverseSearcher<'a>: Searcher<'a> { /// Performs the next search step starting from the back. /// - /// - Returns `Match(a, b)` if `haystack[a..b]` matches the pattern. - /// - Returns `Reject(a, b)` if `haystack[a..b]` can not match the - /// pattern, even partially. - /// - Returns `Done` if every byte of the haystack has been visited + /// - Returns [`Match(a, b)`][SearchStep::Match] if `haystack[a..b]` + /// matches the pattern. + /// - Returns [`Reject(a, b)`][SearchStep::Reject] if `haystack[a..b]` + /// can not match the pattern, even partially. + /// - Returns [`Done`][SearchStep::Done] if every byte of the haystack + /// has been visited /// - /// The stream of `Match` and `Reject` values up to a `Done` + /// The stream of [`Match`][SearchStep::Match] and + /// [`Reject`][SearchStep::Reject] values up to a [`Done`][SearchStep::Done] /// will contain index ranges that are adjacent, non-overlapping, /// covering the whole haystack, and laying on utf8 boundaries. /// - /// A `Match` result needs to contain the whole matched pattern, - /// however `Reject` results may be split up into arbitrary - /// many adjacent fragments. Both ranges may have zero length. + /// A [`Match`][SearchStep::Match] result needs to contain the whole matched + /// pattern, however [`Reject`][SearchStep::Reject] results may be split up + /// into arbitrary many adjacent fragments. Both ranges may have zero length. /// /// As an example, the pattern `"aaa"` and the haystack `"cbaaaaab"` /// might produce the stream - /// `[Reject(7, 8), Match(4, 7), Reject(1, 4), Reject(0, 1)]` + /// `[Reject(7, 8), Match(4, 7), Reject(1, 4), Reject(0, 1)]`. fn next_back(&mut self) -> SearchStep; - /// Finds the next `Match` result. See `next_back()` + /// Finds the next [`Match`][SearchStep::Match] result. + /// See [`next_back()`][ReverseSearcher::next_back]. #[inline] fn next_match_back(&mut self) -> Option<(usize, usize)> { loop { @@ -194,7 +307,8 @@ pub unsafe trait ReverseSearcher<'a>: Searcher<'a> { } } - /// Finds the next `Reject` result. See `next_back()` + /// Finds the next [`Reject`][SearchStep::Reject] result. + /// See [`next_back()`][ReverseSearcher::next_back]. #[inline] fn next_reject_back(&mut self) -> Option<(usize, usize)> { loop { @@ -207,10 +321,10 @@ pub unsafe trait ReverseSearcher<'a>: Searcher<'a> { } } -/// A marker trait to express that a `ReverseSearcher` -/// can be used for a `DoubleEndedIterator` implementation. +/// A marker trait to express that a [`ReverseSearcher`] +/// can be used for a [`DoubleEndedIterator`] implementation. /// -/// For this, the impl of `Searcher` and `ReverseSearcher` need +/// For this, the impl of [`Searcher`] and [`ReverseSearcher`] need /// to follow these conditions: /// /// - All results of `next()` need to be identical @@ -222,7 +336,7 @@ pub unsafe trait ReverseSearcher<'a>: Searcher<'a> { /// # Examples /// /// `char::Searcher` is a `DoubleEndedSearcher` because searching for a -/// `char` only requires looking at one at a time, which behaves the same +/// [`char`] only requires looking at one at a time, which behaves the same /// from both ends. /// /// `(&str)::Searcher` is not a `DoubleEndedSearcher` because @@ -249,13 +363,13 @@ pub struct CharSearcher<'a> { /// `finger_back` is the current byte index of the reverse search. /// Imagine that it exists after the byte at its index, i.e. /// haystack[finger_back - 1] is the last byte of the slice we must inspect during - /// forward searching (and thus the first byte to be inspected when calling next_back()) + /// forward searching (and thus the first byte to be inspected when calling next_back()). finger_back: usize, /// The character being searched for needle: char, // safety invariant: `utf8_size` must be less than 5 - /// The number of bytes `needle` takes up when encoded in utf8 + /// The number of bytes `needle` takes up when encoded in utf8. utf8_size: usize, /// A utf8 encoded copy of the `needle` utf8_encoded: [u8; 4], @@ -415,7 +529,13 @@ unsafe impl<'a> ReverseSearcher<'a> for CharSearcher<'a> { impl<'a> DoubleEndedSearcher<'a> for CharSearcher<'a> {} -/// Searches for chars that are equal to a given char +/// Searches for chars that are equal to a given [`char`]. +/// +/// # Examples +/// +/// ``` +/// assert_eq!("Hello world".find('o'), Some(4)); +/// ``` impl<'a> Pattern<'a> for char { type Searcher = CharSearcher<'a>; @@ -448,6 +568,11 @@ impl<'a> Pattern<'a> for char { self.encode_utf8(&mut [0u8; 4]).is_prefix_of(haystack) } + #[inline] + fn strip_prefix_of(self, haystack: &'a str) -> Option<&'a str> { + self.encode_utf8(&mut [0u8; 4]).strip_prefix_of(haystack) + } + #[inline] fn is_suffix_of(self, haystack: &'a str) -> bool where @@ -455,6 +580,14 @@ impl<'a> Pattern<'a> for char { { self.encode_utf8(&mut [0u8; 4]).is_suffix_of(haystack) } + + #[inline] + fn strip_suffix_of(self, haystack: &'a str) -> Option<&'a str> + where + Self::Searcher: ReverseSearcher<'a>, + { + self.encode_utf8(&mut [0u8; 4]).strip_suffix_of(haystack) + } } ///////////////////////////////////////////////////////////////////////////// @@ -476,6 +609,20 @@ where } } +impl MultiCharEq for [char; N] { + #[inline] + fn matches(&mut self, c: char) -> bool { + self.iter().any(|&m| m == c) + } +} + +impl MultiCharEq for &[char; N] { + #[inline] + fn matches(&mut self, c: char) -> bool { + self.iter().any(|&m| m == c) + } +} + impl MultiCharEq for &[char] { #[inline] fn matches(&mut self, c: char) -> bool { @@ -569,6 +716,11 @@ macro_rules! pattern_methods { ($pmap)(self).is_prefix_of(haystack) } + #[inline] + fn strip_prefix_of(self, haystack: &'a str) -> Option<&'a str> { + ($pmap)(self).strip_prefix_of(haystack) + } + #[inline] fn is_suffix_of(self, haystack: &'a str) -> bool where @@ -576,6 +728,14 @@ macro_rules! pattern_methods { { ($pmap)(self).is_suffix_of(haystack) } + + #[inline] + fn strip_suffix_of(self, haystack: &'a str) -> Option<&'a str> + where + $t: ReverseSearcher<'a>, + { + ($pmap)(self).strip_suffix_of(haystack) + } }; } @@ -614,6 +774,58 @@ macro_rules! searcher_methods { }; } +/// Associated type for `<[char; N] as Pattern<'a>>::Searcher`. +#[derive(Clone, Debug)] +pub struct CharArraySearcher<'a, const N: usize>( + as Pattern<'a>>::Searcher, +); + +/// Associated type for `<&[char; N] as Pattern<'a>>::Searcher`. +#[derive(Clone, Debug)] +pub struct CharArrayRefSearcher<'a, 'b, const N: usize>( + as Pattern<'a>>::Searcher, +); + +/// Searches for chars that are equal to any of the [`char`]s in the array. +/// +/// # Examples +/// +/// ``` +/// assert_eq!("Hello world".find(['l', 'l']), Some(2)); +/// assert_eq!("Hello world".find(['l', 'l']), Some(2)); +/// ``` +impl<'a, const N: usize> Pattern<'a> for [char; N] { + pattern_methods!(CharArraySearcher<'a, N>, MultiCharEqPattern, CharArraySearcher); +} + +unsafe impl<'a, const N: usize> Searcher<'a> for CharArraySearcher<'a, N> { + searcher_methods!(forward); +} + +unsafe impl<'a, const N: usize> ReverseSearcher<'a> for CharArraySearcher<'a, N> { + searcher_methods!(reverse); +} + +/// Searches for chars that are equal to any of the [`char`]s in the array. +/// +/// # Examples +/// +/// ``` +/// assert_eq!("Hello world".find(&['l', 'l']), Some(2)); +/// assert_eq!("Hello world".find(&['l', 'l']), Some(2)); +/// ``` +impl<'a, 'b, const N: usize> Pattern<'a> for &'b [char; N] { + pattern_methods!(CharArrayRefSearcher<'a, 'b, N>, MultiCharEqPattern, CharArrayRefSearcher); +} + +unsafe impl<'a, 'b, const N: usize> Searcher<'a> for CharArrayRefSearcher<'a, 'b, N> { + searcher_methods!(forward); +} + +unsafe impl<'a, 'b, const N: usize> ReverseSearcher<'a> for CharArrayRefSearcher<'a, 'b, N> { + searcher_methods!(reverse); +} + ///////////////////////////////////////////////////////////////////////////// // Impl for &[char] ///////////////////////////////////////////////////////////////////////////// @@ -634,7 +846,14 @@ unsafe impl<'a, 'b> ReverseSearcher<'a> for CharSliceSearcher<'a, 'b> { impl<'a, 'b> DoubleEndedSearcher<'a> for CharSliceSearcher<'a, 'b> {} -/// Searches for chars that are equal to any of the chars in the array +/// Searches for chars that are equal to any of the [`char`]s in the slice. +/// +/// # Examples +/// +/// ``` +/// assert_eq!("Hello world".find(&['l', 'l'] as &[_]), Some(2)); +/// assert_eq!("Hello world".find(&['l', 'l'][..]), Some(2)); +/// ``` impl<'a, 'b> Pattern<'a> for &'b [char] { pattern_methods!(CharSliceSearcher<'a, 'b>, MultiCharEqPattern, CharSliceSearcher); } @@ -676,7 +895,14 @@ where impl<'a, F> DoubleEndedSearcher<'a> for CharPredicateSearcher<'a, F> where F: FnMut(char) -> bool {} -/// Searches for chars that match the given predicate +/// Searches for [`char`]s that match the given predicate. +/// +/// # Examples +/// +/// ``` +/// assert_eq!("Hello world".find(char::is_uppercase), Some(0)); +/// assert_eq!("Hello world".find(|c| "aeiou".contains(c)), Some(1)); +/// ``` impl<'a, F> Pattern<'a> for F where F: FnMut(char) -> bool, @@ -701,6 +927,12 @@ impl<'a, 'b, 'c> Pattern<'a> for &'c &'b str { /// /// Will handle the pattern `""` as returning empty matches at each character /// boundary. +/// +/// # Examples +/// +/// ``` +/// assert_eq!("Hello world".find("world"), Some(6)); +/// ``` impl<'a, 'b> Pattern<'a> for &'b str { type Searcher = StrSearcher<'a, 'b>; @@ -709,17 +941,66 @@ impl<'a, 'b> Pattern<'a> for &'b str { StrSearcher::new(haystack, self) } - /// Checks whether the pattern matches at the front of the haystack + /// Checks whether the pattern matches at the front of the haystack. #[inline] fn is_prefix_of(self, haystack: &'a str) -> bool { haystack.as_bytes().starts_with(self.as_bytes()) } - /// Checks whether the pattern matches at the back of the haystack + /// Checks whether the pattern matches anywhere in the haystack + #[inline] + fn is_contained_in(self, haystack: &'a str) -> bool { + if self.len() == 0 { + return true; + } + + match self.len().cmp(&haystack.len()) { + Ordering::Less => { + if self.len() == 1 { + return haystack.as_bytes().contains(&self.as_bytes()[0]); + } + + #[cfg(all(target_arch = "x86_64", target_feature = "sse2"))] + if self.len() <= 32 { + if let Some(result) = simd_contains(self, haystack) { + return result; + } + } + + self.into_searcher(haystack).next_match().is_some() + } + _ => self == haystack, + } + } + + /// Removes the pattern from the front of haystack, if it matches. + #[inline] + fn strip_prefix_of(self, haystack: &'a str) -> Option<&'a str> { + if self.is_prefix_of(haystack) { + // SAFETY: prefix was just verified to exist. + unsafe { Some(haystack.get_unchecked(self.as_bytes().len()..)) } + } else { + None + } + } + + /// Checks whether the pattern matches at the back of the haystack. #[inline] fn is_suffix_of(self, haystack: &'a str) -> bool { haystack.as_bytes().ends_with(self.as_bytes()) } + + /// Removes the pattern from the back of haystack, if it matches. + #[inline] + fn strip_suffix_of(self, haystack: &'a str) -> Option<&'a str> { + if self.is_suffix_of(haystack) { + let i = haystack.len() - self.as_bytes().len(); + // SAFETY: suffix was just verified to exist. + unsafe { Some(haystack.get_unchecked(..i)) } + } else { + None + } + } } ///////////////////////////////////////////////////////////////////////////// @@ -747,6 +1028,8 @@ struct EmptyNeedle { end: usize, is_match_fw: bool, is_match_bw: bool, + // Needed in case of an empty haystack, see #85462 + is_finished: bool, } impl<'a, 'b> StrSearcher<'a, 'b> { @@ -760,6 +1043,7 @@ impl<'a, 'b> StrSearcher<'a, 'b> { end: haystack.len(), is_match_fw: true, is_match_bw: true, + is_finished: false, }), } } else { @@ -785,13 +1069,19 @@ unsafe impl<'a, 'b> Searcher<'a> for StrSearcher<'a, 'b> { fn next(&mut self) -> SearchStep { match self.searcher { StrSearcherImpl::Empty(ref mut searcher) => { + if searcher.is_finished { + return SearchStep::Done; + } // empty needle rejects every char and matches every empty string between them let is_match = searcher.is_match_fw; searcher.is_match_fw = !searcher.is_match_fw; let pos = searcher.position; match self.haystack[pos..].chars().next() { _ if is_match => SearchStep::Match(pos, pos), - None => SearchStep::Done, + None => { + searcher.is_finished = true; + SearchStep::Done + } Some(ch) => { searcher.position += ch.len_utf8(); SearchStep::Reject(pos, searcher.position) @@ -864,12 +1154,18 @@ unsafe impl<'a, 'b> ReverseSearcher<'a> for StrSearcher<'a, 'b> { fn next_back(&mut self) -> SearchStep { match self.searcher { StrSearcherImpl::Empty(ref mut searcher) => { + if searcher.is_finished { + return SearchStep::Done; + } let is_match = searcher.is_match_bw; searcher.is_match_bw = !searcher.is_match_bw; let end = searcher.end; match self.haystack[..end].chars().next_back() { _ if is_match => SearchStep::Match(end, end), - None => SearchStep::Done, + None => { + searcher.is_finished = true; + SearchStep::Done + } Some(ch) => { searcher.end -= ch.len_utf8(); SearchStep::Reject(searcher.end, end) @@ -1415,3 +1711,210 @@ impl TwoWayStrategy for RejectAndMatch { SearchStep::Match(a, b) } } + +/// SIMD search for short needles based on +/// Wojciech Muła's "SIMD-friendly algorithms for substring searching"[0] +/// +/// It skips ahead by the vector width on each iteration (rather than the needle length as two-way +/// does) by probing the first and last byte of the needle for the whole vector width +/// and only doing full needle comparisons when the vectorized probe indicated potential matches. +/// +/// Since the x86_64 baseline only offers SSE2 we only use u8x16 here. +/// If we ever ship std with for x86-64-v3 or adapt this for other platforms then wider vectors +/// should be evaluated. +/// +/// For haystacks smaller than vector-size + needle length it falls back to +/// a naive O(n*m) search so this implementation should not be called on larger needles. +/// +/// [0]: http://0x80.pl/articles/simd-strfind.html#sse-avx2 +#[cfg(all(target_arch = "x86_64", target_feature = "sse2"))] +#[inline] +fn simd_contains(needle: &str, haystack: &str) -> Option { + let needle = needle.as_bytes(); + let haystack = haystack.as_bytes(); + + debug_assert!(needle.len() > 1); + + use crate::ops::BitAnd; + use crate::simd::mask8x16 as Mask; + use crate::simd::u8x16 as Block; + use crate::simd::{SimdPartialEq, ToBitMask}; + + let first_probe = needle[0]; + let last_byte_offset = needle.len() - 1; + + // the offset used for the 2nd vector + let second_probe_offset = if needle.len() == 2 { + // never bail out on len=2 needles because the probes will fully cover them and have + // no degenerate cases. + 1 + } else { + // try a few bytes in case first and last byte of the needle are the same + let Some(second_probe_offset) = (needle.len().saturating_sub(4)..needle.len()).rfind(|&idx| needle[idx] != first_probe) else { + // fall back to other search methods if we can't find any different bytes + // since we could otherwise hit some degenerate cases + return None; + }; + second_probe_offset + }; + + // do a naive search if the haystack is too small to fit + if haystack.len() < Block::LANES + last_byte_offset { + return Some(haystack.windows(needle.len()).any(|c| c == needle)); + } + + let first_probe: Block = Block::splat(first_probe); + let second_probe: Block = Block::splat(needle[second_probe_offset]); + // first byte are already checked by the outer loop. to verify a match only the + // remainder has to be compared. + let trimmed_needle = &needle[1..]; + + // this #[cold] is load-bearing, benchmark before removing it... + let check_mask = #[cold] + |idx, mask: u16, skip: bool| -> bool { + if skip { + return false; + } + + // and so is this. optimizations are weird. + let mut mask = mask; + + while mask != 0 { + let trailing = mask.trailing_zeros(); + let offset = idx + trailing as usize + 1; + // SAFETY: mask is between 0 and 15 trailing zeroes, we skip one additional byte that was already compared + // and then take trimmed_needle.len() bytes. This is within the bounds defined by the outer loop + unsafe { + let sub = haystack.get_unchecked(offset..).get_unchecked(..trimmed_needle.len()); + if small_slice_eq(sub, trimmed_needle) { + return true; + } + } + mask &= !(1 << trailing); + } + return false; + }; + + let test_chunk = |idx| -> u16 { + // SAFETY: this requires at least LANES bytes being readable at idx + // that is ensured by the loop ranges (see comments below) + let a: Block = unsafe { haystack.as_ptr().add(idx).cast::().read_unaligned() }; + // SAFETY: this requires LANES + block_offset bytes being readable at idx + let b: Block = unsafe { + haystack.as_ptr().add(idx).add(second_probe_offset).cast::().read_unaligned() + }; + let eq_first: Mask = a.simd_eq(first_probe); + let eq_last: Mask = b.simd_eq(second_probe); + let both = eq_first.bitand(eq_last); + let mask = both.to_bitmask(); + + return mask; + }; + + let mut i = 0; + let mut result = false; + // The loop condition must ensure that there's enough headroom to read LANE bytes, + // and not only at the current index but also at the index shifted by block_offset + const UNROLL: usize = 4; + while i + last_byte_offset + UNROLL * Block::LANES < haystack.len() && !result { + let mut masks = [0u16; UNROLL]; + for j in 0..UNROLL { + masks[j] = test_chunk(i + j * Block::LANES); + } + for j in 0..UNROLL { + let mask = masks[j]; + if mask != 0 { + result |= check_mask(i + j * Block::LANES, mask, result); + } + } + i += UNROLL * Block::LANES; + } + while i + last_byte_offset + Block::LANES < haystack.len() && !result { + let mask = test_chunk(i); + if mask != 0 { + result |= check_mask(i, mask, result); + } + i += Block::LANES; + } + + // Process the tail that didn't fit into LANES-sized steps. + // This simply repeats the same procedure but as right-aligned chunk instead + // of a left-aligned one. The last byte must be exactly flush with the string end so + // we don't miss a single byte or read out of bounds. + let i = haystack.len() - last_byte_offset - Block::LANES; + let mask = test_chunk(i); + if mask != 0 { + result |= check_mask(i, mask, result); + } + + Some(result) +} + +/// Compares short slices for equality. +/// +/// It avoids a call to libc's memcmp which is faster on long slices +/// due to SIMD optimizations but it incurs a function call overhead. +/// +/// # Safety +/// +/// Both slices must have the same length. +#[cfg(all(target_arch = "x86_64", target_feature = "sse2"))] // only called on x86 +#[inline] +unsafe fn small_slice_eq(x: &[u8], y: &[u8]) -> bool { + debug_assert_eq!(x.len(), y.len()); + // This function is adapted from + // https://github.com/BurntSushi/memchr/blob/8037d11b4357b0f07be2bb66dc2659d9cf28ad32/src/memmem/util.rs#L32 + + // If we don't have enough bytes to do 4-byte at a time loads, then + // fall back to the naive slow version. + // + // Potential alternative: We could do a copy_nonoverlapping combined with a mask instead + // of a loop. Benchmark it. + if x.len() < 4 { + for (&b1, &b2) in x.iter().zip(y) { + if b1 != b2 { + return false; + } + } + return true; + } + // When we have 4 or more bytes to compare, then proceed in chunks of 4 at + // a time using unaligned loads. + // + // Also, why do 4 byte loads instead of, say, 8 byte loads? The reason is + // that this particular version of memcmp is likely to be called with tiny + // needles. That means that if we do 8 byte loads, then a higher proportion + // of memcmp calls will use the slower variant above. With that said, this + // is a hypothesis and is only loosely supported by benchmarks. There's + // likely some improvement that could be made here. The main thing here + // though is to optimize for latency, not throughput. + + // SAFETY: Via the conditional above, we know that both `px` and `py` + // have the same length, so `px < pxend` implies that `py < pyend`. + // Thus, derefencing both `px` and `py` in the loop below is safe. + // + // Moreover, we set `pxend` and `pyend` to be 4 bytes before the actual + // end of `px` and `py`. Thus, the final dereference outside of the + // loop is guaranteed to be valid. (The final comparison will overlap with + // the last comparison done in the loop for lengths that aren't multiples + // of four.) + // + // Finally, we needn't worry about alignment here, since we do unaligned + // loads. + unsafe { + let (mut px, mut py) = (x.as_ptr(), y.as_ptr()); + let (pxend, pyend) = (px.add(x.len() - 4), py.add(y.len() - 4)); + while px < pxend { + let vx = (px as *const u32).read_unaligned(); + let vy = (py as *const u32).read_unaligned(); + if vx != vy { + return false; + } + px = px.add(4); + py = py.add(4); + } + let vx = (pxend as *const u32).read_unaligned(); + let vy = (pyend as *const u32).read_unaligned(); + vx == vy + } +} diff --git a/crux-mir/lib/core/src/str/traits.rs b/crux-mir/lib/core/src/str/traits.rs new file mode 100644 index 000000000..d3ed811b1 --- /dev/null +++ b/crux-mir/lib/core/src/str/traits.rs @@ -0,0 +1,608 @@ +//! Trait implementations for `str`. + +use crate::cmp::Ordering; +use crate::ops; +use crate::ptr; +use crate::slice::SliceIndex; + +use super::ParseBoolError; + +/// Implements ordering of strings. +/// +/// Strings are ordered [lexicographically](Ord#lexicographical-comparison) by their byte values. This orders Unicode code +/// points based on their positions in the code charts. This is not necessarily the same as +/// "alphabetical" order, which varies by language and locale. Sorting strings according to +/// culturally-accepted standards requires locale-specific data that is outside the scope of +/// the `str` type. +#[stable(feature = "rust1", since = "1.0.0")] +impl Ord for str { + #[inline] + fn cmp(&self, other: &str) -> Ordering { + self.as_bytes().cmp(other.as_bytes()) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialEq for str { + #[inline] + fn eq(&self, other: &str) -> bool { + self.as_bytes() == other.as_bytes() + } + #[inline] + fn ne(&self, other: &str) -> bool { + !(*self).eq(other) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Eq for str {} + +/// Implements comparison operations on strings. +/// +/// Strings are compared [lexicographically](Ord#lexicographical-comparison) by their byte values. This compares Unicode code +/// points based on their positions in the code charts. This is not necessarily the same as +/// "alphabetical" order, which varies by language and locale. Comparing strings according to +/// culturally-accepted standards requires locale-specific data that is outside the scope of +/// the `str` type. +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialOrd for str { + #[inline] + fn partial_cmp(&self, other: &str) -> Option { + Some(self.cmp(other)) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +impl const ops::Index for str +where + I: ~const SliceIndex, +{ + type Output = I::Output; + + #[inline] + fn index(&self, index: I) -> &I::Output { + index.index(self) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +impl const ops::IndexMut for str +where + I: ~const SliceIndex, +{ + #[inline] + fn index_mut(&mut self, index: I) -> &mut I::Output { + index.index_mut(self) + } +} + +#[inline(never)] +#[cold] +#[track_caller] +const fn str_index_overflow_fail() -> ! { + panic!("attempted to index str up to maximum usize"); +} + +/// Implements substring slicing with syntax `&self[..]` or `&mut self[..]`. +/// +/// Returns a slice of the whole string, i.e., returns `&self` or `&mut +/// self`. Equivalent to `&self[0 .. len]` or `&mut self[0 .. len]`. Unlike +/// other indexing operations, this can never panic. +/// +/// This operation is *O*(1). +/// +/// Prior to 1.20.0, these indexing operations were still supported by +/// direct implementation of `Index` and `IndexMut`. +/// +/// Equivalent to `&self[0 .. len]` or `&mut self[0 .. len]`. +#[stable(feature = "str_checked_slicing", since = "1.20.0")] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +unsafe impl const SliceIndex for ops::RangeFull { + type Output = str; + #[inline] + fn get(self, slice: &str) -> Option<&Self::Output> { + Some(slice) + } + #[inline] + fn get_mut(self, slice: &mut str) -> Option<&mut Self::Output> { + Some(slice) + } + #[inline] + unsafe fn get_unchecked(self, slice: *const str) -> *const Self::Output { + slice + } + #[inline] + unsafe fn get_unchecked_mut(self, slice: *mut str) -> *mut Self::Output { + slice + } + #[inline] + fn index(self, slice: &str) -> &Self::Output { + slice + } + #[inline] + fn index_mut(self, slice: &mut str) -> &mut Self::Output { + slice + } +} + +/// Implements substring slicing with syntax `&self[begin .. end]` or `&mut +/// self[begin .. end]`. +/// +/// Returns a slice of the given string from the byte range +/// [`begin`, `end`). +/// +/// This operation is *O*(1). +/// +/// Prior to 1.20.0, these indexing operations were still supported by +/// direct implementation of `Index` and `IndexMut`. +/// +/// # Panics +/// +/// Panics if `begin` or `end` does not point to the starting byte offset of +/// a character (as defined by `is_char_boundary`), if `begin > end`, or if +/// `end > len`. +/// +/// # Examples +/// +/// ``` +/// let s = "Löwe 老虎 Léopard"; +/// assert_eq!(&s[0 .. 1], "L"); +/// +/// assert_eq!(&s[1 .. 9], "öwe 老"); +/// +/// // these will panic: +/// // byte 2 lies within `ö`: +/// // &s[2 ..3]; +/// +/// // byte 8 lies within `老` +/// // &s[1 .. 8]; +/// +/// // byte 100 is outside the string +/// // &s[3 .. 100]; +/// ``` +#[stable(feature = "str_checked_slicing", since = "1.20.0")] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +unsafe impl const SliceIndex for ops::Range { + type Output = str; + #[inline] + fn get(self, slice: &str) -> Option<&Self::Output> { + if self.start <= self.end + && slice.is_char_boundary(self.start) + && slice.is_char_boundary(self.end) + { + // SAFETY: just checked that `start` and `end` are on a char boundary, + // and we are passing in a safe reference, so the return value will also be one. + // We also checked char boundaries, so this is valid UTF-8. + Some(unsafe { &*self.get_unchecked(slice) }) + } else { + None + } + } + #[inline] + fn get_mut(self, slice: &mut str) -> Option<&mut Self::Output> { + if self.start <= self.end + && slice.is_char_boundary(self.start) + && slice.is_char_boundary(self.end) + { + // SAFETY: just checked that `start` and `end` are on a char boundary. + // We know the pointer is unique because we got it from `slice`. + Some(unsafe { &mut *self.get_unchecked_mut(slice) }) + } else { + None + } + } + #[inline] + unsafe fn get_unchecked(self, slice: *const str) -> *const Self::Output { + let slice = slice as *const [u8]; + // SAFETY: the caller guarantees that `self` is in bounds of `slice` + // which satisfies all the conditions for `add`. + let ptr = unsafe { slice.as_ptr().add(self.start) }; + let len = self.end - self.start; + ptr::slice_from_raw_parts(ptr, len) as *const str + } + #[inline] + unsafe fn get_unchecked_mut(self, slice: *mut str) -> *mut Self::Output { + let slice = slice as *mut [u8]; + // SAFETY: see comments for `get_unchecked`. + let ptr = unsafe { slice.as_mut_ptr().add(self.start) }; + let len = self.end - self.start; + ptr::slice_from_raw_parts_mut(ptr, len) as *mut str + } + #[inline] + fn index(self, slice: &str) -> &Self::Output { + let (start, end) = (self.start, self.end); + match self.get(slice) { + Some(s) => s, + None => super::slice_error_fail(slice, start, end), + } + } + #[inline] + fn index_mut(self, slice: &mut str) -> &mut Self::Output { + // is_char_boundary checks that the index is in [0, .len()] + // cannot reuse `get` as above, because of NLL trouble + if self.start <= self.end + && slice.is_char_boundary(self.start) + && slice.is_char_boundary(self.end) + { + // SAFETY: just checked that `start` and `end` are on a char boundary, + // and we are passing in a safe reference, so the return value will also be one. + unsafe { &mut *self.get_unchecked_mut(slice) } + } else { + super::slice_error_fail(slice, self.start, self.end) + } + } +} + +/// Implements substring slicing with syntax `&self[.. end]` or `&mut +/// self[.. end]`. +/// +/// Returns a slice of the given string from the byte range \[0, `end`). +/// Equivalent to `&self[0 .. end]` or `&mut self[0 .. end]`. +/// +/// This operation is *O*(1). +/// +/// Prior to 1.20.0, these indexing operations were still supported by +/// direct implementation of `Index` and `IndexMut`. +/// +/// # Panics +/// +/// Panics if `end` does not point to the starting byte offset of a +/// character (as defined by `is_char_boundary`), or if `end > len`. +#[stable(feature = "str_checked_slicing", since = "1.20.0")] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +unsafe impl const SliceIndex for ops::RangeTo { + type Output = str; + #[inline] + fn get(self, slice: &str) -> Option<&Self::Output> { + if slice.is_char_boundary(self.end) { + // SAFETY: just checked that `end` is on a char boundary, + // and we are passing in a safe reference, so the return value will also be one. + Some(unsafe { &*self.get_unchecked(slice) }) + } else { + None + } + } + #[inline] + fn get_mut(self, slice: &mut str) -> Option<&mut Self::Output> { + if slice.is_char_boundary(self.end) { + // SAFETY: just checked that `end` is on a char boundary, + // and we are passing in a safe reference, so the return value will also be one. + Some(unsafe { &mut *self.get_unchecked_mut(slice) }) + } else { + None + } + } + #[inline] + unsafe fn get_unchecked(self, slice: *const str) -> *const Self::Output { + let slice = slice as *const [u8]; + let ptr = slice.as_ptr(); + ptr::slice_from_raw_parts(ptr, self.end) as *const str + } + #[inline] + unsafe fn get_unchecked_mut(self, slice: *mut str) -> *mut Self::Output { + let slice = slice as *mut [u8]; + let ptr = slice.as_mut_ptr(); + ptr::slice_from_raw_parts_mut(ptr, self.end) as *mut str + } + #[inline] + fn index(self, slice: &str) -> &Self::Output { + let end = self.end; + match self.get(slice) { + Some(s) => s, + None => super::slice_error_fail(slice, 0, end), + } + } + #[inline] + fn index_mut(self, slice: &mut str) -> &mut Self::Output { + if slice.is_char_boundary(self.end) { + // SAFETY: just checked that `end` is on a char boundary, + // and we are passing in a safe reference, so the return value will also be one. + unsafe { &mut *self.get_unchecked_mut(slice) } + } else { + super::slice_error_fail(slice, 0, self.end) + } + } +} + +/// Implements substring slicing with syntax `&self[begin ..]` or `&mut +/// self[begin ..]`. +/// +/// Returns a slice of the given string from the byte range \[`begin`, `len`). +/// Equivalent to `&self[begin .. len]` or `&mut self[begin .. len]`. +/// +/// This operation is *O*(1). +/// +/// Prior to 1.20.0, these indexing operations were still supported by +/// direct implementation of `Index` and `IndexMut`. +/// +/// # Panics +/// +/// Panics if `begin` does not point to the starting byte offset of +/// a character (as defined by `is_char_boundary`), or if `begin > len`. +#[stable(feature = "str_checked_slicing", since = "1.20.0")] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +unsafe impl const SliceIndex for ops::RangeFrom { + type Output = str; + #[inline] + fn get(self, slice: &str) -> Option<&Self::Output> { + if slice.is_char_boundary(self.start) { + // SAFETY: just checked that `start` is on a char boundary, + // and we are passing in a safe reference, so the return value will also be one. + Some(unsafe { &*self.get_unchecked(slice) }) + } else { + None + } + } + #[inline] + fn get_mut(self, slice: &mut str) -> Option<&mut Self::Output> { + if slice.is_char_boundary(self.start) { + // SAFETY: just checked that `start` is on a char boundary, + // and we are passing in a safe reference, so the return value will also be one. + Some(unsafe { &mut *self.get_unchecked_mut(slice) }) + } else { + None + } + } + #[inline] + unsafe fn get_unchecked(self, slice: *const str) -> *const Self::Output { + let slice = slice as *const [u8]; + // SAFETY: the caller guarantees that `self` is in bounds of `slice` + // which satisfies all the conditions for `add`. + let ptr = unsafe { slice.as_ptr().add(self.start) }; + let len = slice.len() - self.start; + ptr::slice_from_raw_parts(ptr, len) as *const str + } + #[inline] + unsafe fn get_unchecked_mut(self, slice: *mut str) -> *mut Self::Output { + let slice = slice as *mut [u8]; + // SAFETY: identical to `get_unchecked`. + let ptr = unsafe { slice.as_mut_ptr().add(self.start) }; + let len = slice.len() - self.start; + ptr::slice_from_raw_parts_mut(ptr, len) as *mut str + } + #[inline] + fn index(self, slice: &str) -> &Self::Output { + let (start, end) = (self.start, slice.len()); + match self.get(slice) { + Some(s) => s, + None => super::slice_error_fail(slice, start, end), + } + } + #[inline] + fn index_mut(self, slice: &mut str) -> &mut Self::Output { + if slice.is_char_boundary(self.start) { + // SAFETY: just checked that `start` is on a char boundary, + // and we are passing in a safe reference, so the return value will also be one. + unsafe { &mut *self.get_unchecked_mut(slice) } + } else { + super::slice_error_fail(slice, self.start, slice.len()) + } + } +} + +/// Implements substring slicing with syntax `&self[begin ..= end]` or `&mut +/// self[begin ..= end]`. +/// +/// Returns a slice of the given string from the byte range +/// [`begin`, `end`]. Equivalent to `&self [begin .. end + 1]` or `&mut +/// self[begin .. end + 1]`, except if `end` has the maximum value for +/// `usize`. +/// +/// This operation is *O*(1). +/// +/// # Panics +/// +/// Panics if `begin` does not point to the starting byte offset of +/// a character (as defined by `is_char_boundary`), if `end` does not point +/// to the ending byte offset of a character (`end + 1` is either a starting +/// byte offset or equal to `len`), if `begin > end`, or if `end >= len`. +#[stable(feature = "inclusive_range", since = "1.26.0")] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +unsafe impl const SliceIndex for ops::RangeInclusive { + type Output = str; + #[inline] + fn get(self, slice: &str) -> Option<&Self::Output> { + if *self.end() == usize::MAX { None } else { self.into_slice_range().get(slice) } + } + #[inline] + fn get_mut(self, slice: &mut str) -> Option<&mut Self::Output> { + if *self.end() == usize::MAX { None } else { self.into_slice_range().get_mut(slice) } + } + #[inline] + unsafe fn get_unchecked(self, slice: *const str) -> *const Self::Output { + // SAFETY: the caller must uphold the safety contract for `get_unchecked`. + unsafe { self.into_slice_range().get_unchecked(slice) } + } + #[inline] + unsafe fn get_unchecked_mut(self, slice: *mut str) -> *mut Self::Output { + // SAFETY: the caller must uphold the safety contract for `get_unchecked_mut`. + unsafe { self.into_slice_range().get_unchecked_mut(slice) } + } + #[inline] + fn index(self, slice: &str) -> &Self::Output { + if *self.end() == usize::MAX { + str_index_overflow_fail(); + } + self.into_slice_range().index(slice) + } + #[inline] + fn index_mut(self, slice: &mut str) -> &mut Self::Output { + if *self.end() == usize::MAX { + str_index_overflow_fail(); + } + self.into_slice_range().index_mut(slice) + } +} + +/// Implements substring slicing with syntax `&self[..= end]` or `&mut +/// self[..= end]`. +/// +/// Returns a slice of the given string from the byte range \[0, `end`\]. +/// Equivalent to `&self [0 .. end + 1]`, except if `end` has the maximum +/// value for `usize`. +/// +/// This operation is *O*(1). +/// +/// # Panics +/// +/// Panics if `end` does not point to the ending byte offset of a character +/// (`end + 1` is either a starting byte offset as defined by +/// `is_char_boundary`, or equal to `len`), or if `end >= len`. +#[stable(feature = "inclusive_range", since = "1.26.0")] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +unsafe impl const SliceIndex for ops::RangeToInclusive { + type Output = str; + #[inline] + fn get(self, slice: &str) -> Option<&Self::Output> { + if self.end == usize::MAX { None } else { (..self.end + 1).get(slice) } + } + #[inline] + fn get_mut(self, slice: &mut str) -> Option<&mut Self::Output> { + if self.end == usize::MAX { None } else { (..self.end + 1).get_mut(slice) } + } + #[inline] + unsafe fn get_unchecked(self, slice: *const str) -> *const Self::Output { + // SAFETY: the caller must uphold the safety contract for `get_unchecked`. + unsafe { (..self.end + 1).get_unchecked(slice) } + } + #[inline] + unsafe fn get_unchecked_mut(self, slice: *mut str) -> *mut Self::Output { + // SAFETY: the caller must uphold the safety contract for `get_unchecked_mut`. + unsafe { (..self.end + 1).get_unchecked_mut(slice) } + } + #[inline] + fn index(self, slice: &str) -> &Self::Output { + if self.end == usize::MAX { + str_index_overflow_fail(); + } + (..self.end + 1).index(slice) + } + #[inline] + fn index_mut(self, slice: &mut str) -> &mut Self::Output { + if self.end == usize::MAX { + str_index_overflow_fail(); + } + (..self.end + 1).index_mut(slice) + } +} + +/// Parse a value from a string +/// +/// `FromStr`'s [`from_str`] method is often used implicitly, through +/// [`str`]'s [`parse`] method. See [`parse`]'s documentation for examples. +/// +/// [`from_str`]: FromStr::from_str +/// [`parse`]: str::parse +/// +/// `FromStr` does not have a lifetime parameter, and so you can only parse types +/// that do not contain a lifetime parameter themselves. In other words, you can +/// parse an `i32` with `FromStr`, but not a `&i32`. You can parse a struct that +/// contains an `i32`, but not one that contains an `&i32`. +/// +/// # Examples +/// +/// Basic implementation of `FromStr` on an example `Point` type: +/// +/// ``` +/// use std::str::FromStr; +/// +/// #[derive(Debug, PartialEq)] +/// struct Point { +/// x: i32, +/// y: i32 +/// } +/// +/// #[derive(Debug, PartialEq, Eq)] +/// struct ParsePointError; +/// +/// impl FromStr for Point { +/// type Err = ParsePointError; +/// +/// fn from_str(s: &str) -> Result { +/// let (x, y) = s +/// .strip_prefix('(') +/// .and_then(|s| s.strip_suffix(')')) +/// .and_then(|s| s.split_once(',')) +/// .ok_or(ParsePointError)?; +/// +/// let x_fromstr = x.parse::().map_err(|_| ParsePointError)?; +/// let y_fromstr = y.parse::().map_err(|_| ParsePointError)?; +/// +/// Ok(Point { x: x_fromstr, y: y_fromstr }) +/// } +/// } +/// +/// let expected = Ok(Point { x: 1, y: 2 }); +/// // Explicit call +/// assert_eq!(Point::from_str("(1,2)"), expected); +/// // Implicit calls, through parse +/// assert_eq!("(1,2)".parse(), expected); +/// assert_eq!("(1,2)".parse::(), expected); +/// // Invalid input string +/// assert!(Point::from_str("(1 2)").is_err()); +/// ``` +#[stable(feature = "rust1", since = "1.0.0")] +pub trait FromStr: Sized { + /// The associated error which can be returned from parsing. + #[stable(feature = "rust1", since = "1.0.0")] + type Err; + + /// Parses a string `s` to return a value of this type. + /// + /// If parsing succeeds, return the value inside [`Ok`], otherwise + /// when the string is ill-formatted return an error specific to the + /// inside [`Err`]. The error type is specific to the implementation of the trait. + /// + /// # Examples + /// + /// Basic usage with [`i32`], a type that implements `FromStr`: + /// + /// ``` + /// use std::str::FromStr; + /// + /// let s = "5"; + /// let x = i32::from_str(s).unwrap(); + /// + /// assert_eq!(5, x); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + fn from_str(s: &str) -> Result; +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl FromStr for bool { + type Err = ParseBoolError; + + /// Parse a `bool` from a string. + /// + /// The only accepted values are `"true"` and `"false"`. Any other input + /// will return an error. + /// + /// # Examples + /// + /// ``` + /// use std::str::FromStr; + /// + /// assert_eq!(FromStr::from_str("true"), Ok(true)); + /// assert_eq!(FromStr::from_str("false"), Ok(false)); + /// assert!(::from_str("not even a boolean").is_err()); + /// ``` + /// + /// Note, in many cases, the `.parse()` method on `str` is more proper. + /// + /// ``` + /// assert_eq!("true".parse(), Ok(true)); + /// assert_eq!("false".parse(), Ok(false)); + /// assert!("not even a boolean".parse::().is_err()); + /// ``` + #[inline] + fn from_str(s: &str) -> Result { + match s { + "true" => Ok(true), + "false" => Ok(false), + _ => Err(ParseBoolError), + } + } +} diff --git a/crux-mir/lib/core/src/str/validations.rs b/crux-mir/lib/core/src/str/validations.rs new file mode 100644 index 000000000..2acef432f --- /dev/null +++ b/crux-mir/lib/core/src/str/validations.rs @@ -0,0 +1,274 @@ +//! Operations related to UTF-8 validation. + +use crate::mem; + +use super::Utf8Error; + +/// Returns the initial codepoint accumulator for the first byte. +/// The first byte is special, only want bottom 5 bits for width 2, 4 bits +/// for width 3, and 3 bits for width 4. +#[inline] +const fn utf8_first_byte(byte: u8, width: u32) -> u32 { + (byte & (0x7F >> width)) as u32 +} + +/// Returns the value of `ch` updated with continuation byte `byte`. +#[inline] +const fn utf8_acc_cont_byte(ch: u32, byte: u8) -> u32 { + (ch << 6) | (byte & CONT_MASK) as u32 +} + +/// Checks whether the byte is a UTF-8 continuation byte (i.e., starts with the +/// bits `10`). +#[inline] +pub(super) const fn utf8_is_cont_byte(byte: u8) -> bool { + (byte as i8) < -64 +} + +/// Reads the next code point out of a byte iterator (assuming a +/// UTF-8-like encoding). +/// +/// # Safety +/// +/// `bytes` must produce a valid UTF-8-like (UTF-8 or WTF-8) string +#[unstable(feature = "str_internals", issue = "none")] +#[inline] +pub unsafe fn next_code_point<'a, I: Iterator>(bytes: &mut I) -> Option { + // Decode UTF-8 + let x = *bytes.next()?; + if x < 128 { + return Some(x as u32); + } + + // Multibyte case follows + // Decode from a byte combination out of: [[[x y] z] w] + // NOTE: Performance is sensitive to the exact formulation here + let init = utf8_first_byte(x, 2); + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + let y = unsafe { *bytes.next().unwrap_unchecked() }; + let mut ch = utf8_acc_cont_byte(init, y); + if x >= 0xE0 { + // [[x y z] w] case + // 5th bit in 0xE0 .. 0xEF is always clear, so `init` is still valid + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + let z = unsafe { *bytes.next().unwrap_unchecked() }; + let y_z = utf8_acc_cont_byte((y & CONT_MASK) as u32, z); + ch = init << 12 | y_z; + if x >= 0xF0 { + // [x y z w] case + // use only the lower 3 bits of `init` + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + let w = unsafe { *bytes.next().unwrap_unchecked() }; + ch = (init & 7) << 18 | utf8_acc_cont_byte(y_z, w); + } + } + + Some(ch) +} + +/// Reads the last code point out of a byte iterator (assuming a +/// UTF-8-like encoding). +/// +/// # Safety +/// +/// `bytes` must produce a valid UTF-8-like (UTF-8 or WTF-8) string +#[inline] +pub(super) unsafe fn next_code_point_reverse<'a, I>(bytes: &mut I) -> Option +where + I: DoubleEndedIterator, +{ + // Decode UTF-8 + let w = match *bytes.next_back()? { + next_byte if next_byte < 128 => return Some(next_byte as u32), + back_byte => back_byte, + }; + + // Multibyte case follows + // Decode from a byte combination out of: [x [y [z w]]] + let mut ch; + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + let z = unsafe { *bytes.next_back().unwrap_unchecked() }; + ch = utf8_first_byte(z, 2); + if utf8_is_cont_byte(z) { + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + let y = unsafe { *bytes.next_back().unwrap_unchecked() }; + ch = utf8_first_byte(y, 3); + if utf8_is_cont_byte(y) { + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + let x = unsafe { *bytes.next_back().unwrap_unchecked() }; + ch = utf8_first_byte(x, 4); + ch = utf8_acc_cont_byte(ch, y); + } + ch = utf8_acc_cont_byte(ch, z); + } + ch = utf8_acc_cont_byte(ch, w); + + Some(ch) +} + +const NONASCII_MASK: usize = usize::repeat_u8(0x80); + +/// Returns `true` if any byte in the word `x` is nonascii (>= 128). +#[inline] +const fn contains_nonascii(x: usize) -> bool { + (x & NONASCII_MASK) != 0 +} + +/// Walks through `v` checking that it's a valid UTF-8 sequence, +/// returning `Ok(())` in that case, or, if it is invalid, `Err(err)`. +#[inline(always)] +#[rustc_const_unstable(feature = "str_internals", issue = "none")] +pub(super) const fn run_utf8_validation(v: &[u8]) -> Result<(), Utf8Error> { + let mut index = 0; + let len = v.len(); + + let usize_bytes = mem::size_of::(); + let ascii_block_size = 2 * usize_bytes; + let blocks_end = if len >= ascii_block_size { len - ascii_block_size + 1 } else { 0 }; + let align = v.as_ptr().align_offset(usize_bytes); + + while index < len { + let old_offset = index; + macro_rules! err { + ($error_len: expr) => { + return Err(Utf8Error { valid_up_to: old_offset, error_len: $error_len }) + }; + } + + macro_rules! next { + () => {{ + index += 1; + // we needed data, but there was none: error! + if index >= len { + err!(None) + } + v[index] + }}; + } + + let first = v[index]; + if first >= 128 { + let w = utf8_char_width(first); + // 2-byte encoding is for codepoints \u{0080} to \u{07ff} + // first C2 80 last DF BF + // 3-byte encoding is for codepoints \u{0800} to \u{ffff} + // first E0 A0 80 last EF BF BF + // excluding surrogates codepoints \u{d800} to \u{dfff} + // ED A0 80 to ED BF BF + // 4-byte encoding is for codepoints \u{1000}0 to \u{10ff}ff + // first F0 90 80 80 last F4 8F BF BF + // + // Use the UTF-8 syntax from the RFC + // + // https://tools.ietf.org/html/rfc3629 + // UTF8-1 = %x00-7F + // UTF8-2 = %xC2-DF UTF8-tail + // UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / + // %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) + // UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / + // %xF4 %x80-8F 2( UTF8-tail ) + match w { + 2 => { + if next!() as i8 >= -64 { + err!(Some(1)) + } + } + 3 => { + match (first, next!()) { + (0xE0, 0xA0..=0xBF) + | (0xE1..=0xEC, 0x80..=0xBF) + | (0xED, 0x80..=0x9F) + | (0xEE..=0xEF, 0x80..=0xBF) => {} + _ => err!(Some(1)), + } + if next!() as i8 >= -64 { + err!(Some(2)) + } + } + 4 => { + match (first, next!()) { + (0xF0, 0x90..=0xBF) | (0xF1..=0xF3, 0x80..=0xBF) | (0xF4, 0x80..=0x8F) => {} + _ => err!(Some(1)), + } + if next!() as i8 >= -64 { + err!(Some(2)) + } + if next!() as i8 >= -64 { + err!(Some(3)) + } + } + _ => err!(Some(1)), + } + index += 1; + } else { + // Ascii case, try to skip forward quickly. + // When the pointer is aligned, read 2 words of data per iteration + // until we find a word containing a non-ascii byte. + if align != usize::MAX && align.wrapping_sub(index) % usize_bytes == 0 { + let ptr = v.as_ptr(); + while index < blocks_end { + // SAFETY: since `align - index` and `ascii_block_size` are + // multiples of `usize_bytes`, `block = ptr.add(index)` is + // always aligned with a `usize` so it's safe to dereference + // both `block` and `block.add(1)`. + unsafe { + let block = ptr.add(index) as *const usize; + // break if there is a nonascii byte + let zu = contains_nonascii(*block); + let zv = contains_nonascii(*block.add(1)); + if zu || zv { + break; + } + } + index += ascii_block_size; + } + // step from the point where the wordwise loop stopped + while index < len && v[index] < 128 { + index += 1; + } + } else { + index += 1; + } + } + } + + Ok(()) +} + +// https://tools.ietf.org/html/rfc3629 +const UTF8_CHAR_WIDTH: &[u8; 256] = &[ + // 1 2 3 4 5 6 7 8 9 A B C D E F + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 1 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 2 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 3 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 5 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 8 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 9 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // A + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // B + 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // D + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // E + 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // F +]; + +/// Given a first byte, determines how many bytes are in this UTF-8 character. +#[unstable(feature = "str_internals", issue = "none")] +#[must_use] +#[inline] +pub const fn utf8_char_width(b: u8) -> usize { + UTF8_CHAR_WIDTH[b as usize] as usize +} + +/// Mask of the value bits of a continuation byte. +const CONT_MASK: u8 = 0b0011_1111; diff --git a/crux-mir/lib/core/src/sync/atomic.rs b/crux-mir/lib/core/src/sync/atomic.rs index 69c21f49d..14367eb09 100644 --- a/crux-mir/lib/core/src/sync/atomic.rs +++ b/crux-mir/lib/core/src/sync/atomic.rs @@ -4,24 +4,23 @@ //! threads, and are the building blocks of other concurrent //! types. //! +//! Rust atomics currently follow the same rules as [C++20 atomics][cpp], specifically `atomic_ref`. +//! Basically, creating a *shared reference* to one of the Rust atomic types corresponds to creating +//! an `atomic_ref` in C++; the `atomic_ref` is destroyed when the lifetime of the shared reference +//! ends. (A Rust atomic type that is exclusively owned or behind a mutable reference does *not* +//! correspond to an "atomic object" in C++, since it can be accessed via non-atomic operations.) +//! //! This module defines atomic versions of a select number of primitive //! types, including [`AtomicBool`], [`AtomicIsize`], [`AtomicUsize`], //! [`AtomicI8`], [`AtomicU16`], etc. //! Atomic types present operations that, when used correctly, synchronize //! updates between threads. //! -//! [`AtomicBool`]: struct.AtomicBool.html -//! [`AtomicIsize`]: struct.AtomicIsize.html -//! [`AtomicUsize`]: struct.AtomicUsize.html -//! [`AtomicI8`]: struct.AtomicI8.html -//! [`AtomicU16`]: struct.AtomicU16.html -//! //! Each method takes an [`Ordering`] which represents the strength of //! the memory barrier for that operation. These orderings are the //! same as the [C++20 atomic orderings][1]. For more information see the [nomicon][2]. //! -//! [`Ordering`]: enum.Ordering.html -//! +//! [cpp]: https://en.cppreference.com/w/cpp/atomic //! [1]: https://en.cppreference.com/w/cpp/atomic/memory_order //! [2]: ../../../nomicon/atomics.html //! @@ -31,15 +30,12 @@ //! The most common way to share an atomic variable is to put it into an [`Arc`][arc] (an //! atomically-reference-counted shared pointer). //! -//! [`Sync`]: ../../marker/trait.Sync.html //! [arc]: ../../../std/sync/struct.Arc.html //! //! Atomic types may be stored in static variables, initialized using //! the constant initializers like [`AtomicBool::new`]. Atomic statics //! are often used for lazy global initialization. //! -//! [`AtomicBool::new`]: struct.AtomicBool.html#method.new -//! //! # Portability //! //! All atomic types in this module are guaranteed to be [lock-free] if they're @@ -52,26 +48,34 @@ //! instructions to implement `AtomicI8`. Note that this emulation should not //! have an impact on correctness of code, it's just something to be aware of. //! -//! The atomic types in this module may not be available on all platforms. The +//! The atomic types in this module might not be available on all platforms. The //! atomic types here are all widely available, however, and can generally be //! relied upon existing. Some notable exceptions are: //! //! * PowerPC and MIPS platforms with 32-bit pointers do not have `AtomicU64` or //! `AtomicI64` types. -//! * ARM platforms like `armv5te` that aren't for Linux do not have any atomics -//! at all. -//! * ARM targets with `thumbv6m` do not have atomic operations at all. +//! * ARM platforms like `armv5te` that aren't for Linux only provide `load` +//! and `store` operations, and do not support Compare and Swap (CAS) +//! operations, such as `swap`, `fetch_add`, etc. Additionally on Linux, +//! these CAS operations are implemented via [operating system support], which +//! may come with a performance penalty. +//! * ARM targets with `thumbv6m` only provide `load` and `store` operations, +//! and do not support Compare and Swap (CAS) operations, such as `swap`, +//! `fetch_add`, etc. +//! +//! [operating system support]: https://www.kernel.org/doc/Documentation/arm/kernel_user_helpers.txt //! //! Note that future platforms may be added that also do not have support for //! some atomic operations. Maximally portable code will want to be careful //! about which atomic types are used. `AtomicUsize` and `AtomicIsize` are //! generally the most portable, but even then they're not available everywhere. -//! For reference, the `std` library requires pointer-sized atomics, although +//! For reference, the `std` library requires `AtomicBool`s and pointer-sized atomics, although //! `core` does not. //! -//! Currently you'll need to use `#[cfg(target_arch)]` primarily to -//! conditionally compile in code with atomics. There is an unstable -//! `#[cfg(target_has_atomic)]` as well which may be stabilized in the future. +//! The `#[cfg(target_has_atomic)]` attribute can be used to conditionally +//! compile based on the target's supported bit widths. It is a key-value +//! option set for each supported size, with values "8", "16", "32", "64", +//! "128", and "ptr" for pointer-sized atomics. //! //! [lock-free]: https://en.wikipedia.org/wiki/Non-blocking_algorithm //! @@ -82,21 +86,23 @@ //! ``` //! use std::sync::Arc; //! use std::sync::atomic::{AtomicUsize, Ordering}; -//! use std::thread; +//! use std::{hint, thread}; //! //! fn main() { //! let spinlock = Arc::new(AtomicUsize::new(1)); //! -//! let spinlock_clone = spinlock.clone(); +//! let spinlock_clone = Arc::clone(&spinlock); //! let thread = thread::spawn(move|| { //! spinlock_clone.store(0, Ordering::SeqCst); //! }); //! //! // Wait for the other thread to release the lock -//! while spinlock.load(Ordering::SeqCst) != 0 {} +//! while spinlock.load(Ordering::SeqCst) != 0 { +//! hint::spin_loop(); +//! } //! //! if let Err(panic) = thread.join() { -//! println!("Thread had an error: {:?}", panic); +//! println!("Thread had an error: {panic:?}"); //! } //! } //! ``` @@ -115,6 +121,7 @@ #![stable(feature = "rust1", since = "1.0.0")] #![cfg_attr(not(target_has_atomic_load_store = "8"), allow(dead_code))] #![cfg_attr(not(target_has_atomic_load_store = "8"), allow(unused_imports))] +#![rustc_diagnostic_item = "atomic_mod"] use self::Ordering::*; @@ -124,50 +131,26 @@ use crate::intrinsics; use crate::hint::spin_loop; -use crate::crucible::concurrency; - -/// Signals the processor that it is inside a busy-wait spin-loop ("spin lock"). -/// -/// Upon receiving spin-loop signal the processor can optimize its behavior by, for example, saving -/// power or switching hyper-threads. -/// -/// This function is different from [`std::thread::yield_now`] which directly yields to the -/// system's scheduler, whereas `spin_loop_hint` does not interact with the operating system. -/// -/// A common use case for `spin_loop_hint` is implementing bounded optimistic spinning in a CAS -/// loop in synchronization primitives. To avoid problems like priority inversion, it is strongly -/// recommended that the spin loop is terminated after a finite amount of iterations and an -/// appropriate blocking syscall is made. -/// -/// **Note**: On platforms that do not support receiving spin-loop hints this function does not -/// do anything at all. -/// -/// [`std::thread::yield_now`]: ../../../std/thread/fn.yield_now.html -/// [`std::thread::sleep`]: ../../../std/thread/fn.sleep.html -/// [`std::sync::Mutex`]: ../../../std/sync/struct.Mutex.html -#[inline] -#[stable(feature = "spin_loop_hint", since = "1.24.0")] -pub fn spin_loop_hint() { - spin_loop() -} - /// A boolean type which can be safely shared between threads. /// /// This type has the same in-memory representation as a [`bool`]. /// -/// [`bool`]: ../../../std/primitive.bool.html +/// **Note**: This type is only available on platforms that support atomic +/// loads and stores of `u8`. #[cfg(target_has_atomic_load_store = "8")] #[stable(feature = "rust1", since = "1.0.0")] +#[rustc_diagnostic_item = "AtomicBool"] #[repr(C, align(1))] pub struct AtomicBool { v: UnsafeCell, - model: bool, } #[cfg(target_has_atomic_load_store = "8")] #[stable(feature = "rust1", since = "1.0.0")] -impl Default for AtomicBool { +#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] +impl const Default for AtomicBool { /// Creates an `AtomicBool` initialized to `false`. + #[inline] fn default() -> Self { Self::new(false) } @@ -181,8 +164,12 @@ unsafe impl Sync for AtomicBool {} /// A raw pointer type which can be safely shared between threads. /// /// This type has the same in-memory representation as a `*mut T`. +/// +/// **Note**: This type is only available on platforms that support atomic +/// loads and stores of pointers. Its size depends on the target pointer's size. #[cfg(target_has_atomic_load_store = "ptr")] #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "AtomicPtr")] #[cfg_attr(target_pointer_width = "16", repr(C, align(2)))] #[cfg_attr(target_pointer_width = "32", repr(C, align(4)))] #[cfg_attr(target_pointer_width = "64", repr(C, align(8)))] @@ -192,7 +179,8 @@ pub struct AtomicPtr { #[cfg(target_has_atomic_load_store = "ptr")] #[stable(feature = "rust1", since = "1.0.0")] -impl Default for AtomicPtr { +#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] +impl const Default for AtomicPtr { /// Creates a null `AtomicPtr`. fn default() -> AtomicPtr { AtomicPtr::new(crate::ptr::null_mut()) @@ -209,8 +197,8 @@ unsafe impl Sync for AtomicPtr {} /// Atomic memory orderings /// /// Memory orderings specify the way atomic operations synchronize memory. -/// In its weakest [`Relaxed`][Ordering::Relaxed], only the memory directly touched by the -/// operation is synchronized. On the other hand, a store-load pair of [`SeqCst`][Ordering::SeqCst] +/// In its weakest [`Ordering::Relaxed`], only the memory directly touched by the +/// operation is synchronized. On the other hand, a store-load pair of [`Ordering::SeqCst`] /// operations synchronize other memory while additionally preserving a total order of such /// operations across all threads. /// @@ -220,11 +208,10 @@ unsafe impl Sync for AtomicPtr {} /// For more information see the [nomicon]. /// /// [nomicon]: ../../../nomicon/atomics.html -/// [Ordering::Relaxed]: #variant.Relaxed -/// [Ordering::SeqCst]: #variant.SeqCst #[stable(feature = "rust1", since = "1.0.0")] #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] #[non_exhaustive] +#[rustc_diagnostic_item = "Ordering"] pub enum Ordering { /// No ordering constraints, only atomic operations. /// @@ -245,9 +232,6 @@ pub enum Ordering { /// /// Corresponds to [`memory_order_release`] in C++20. /// - /// [`Release`]: #variant.Release - /// [`Acquire`]: #variant.Acquire - /// [`Relaxed`]: #variant.Relaxed /// [`memory_order_release`]: https://en.cppreference.com/w/cpp/atomic/memory_order#Release-Acquire_ordering #[stable(feature = "rust1", since = "1.0.0")] Release, @@ -263,9 +247,6 @@ pub enum Ordering { /// /// Corresponds to [`memory_order_acquire`] in C++20. /// - /// [`Acquire`]: #variant.Acquire - /// [`Release`]: #variant.Release - /// [`Relaxed`]: #variant.Relaxed /// [`memory_order_acquire`]: https://en.cppreference.com/w/cpp/atomic/memory_order#Release-Acquire_ordering #[stable(feature = "rust1", since = "1.0.0")] Acquire, @@ -281,9 +262,6 @@ pub enum Ordering { /// Corresponds to [`memory_order_acq_rel`] in C++20. /// /// [`memory_order_acq_rel`]: https://en.cppreference.com/w/cpp/atomic/memory_order#Release-Acquire_ordering - /// [`Acquire`]: #variant.Acquire - /// [`Release`]: #variant.Release - /// [`Relaxed`]: #variant.Relaxed #[stable(feature = "rust1", since = "1.0.0")] AcqRel, /// Like [`Acquire`]/[`Release`]/[`AcqRel`] (for load, store, and load-with-store @@ -293,21 +271,16 @@ pub enum Ordering { /// Corresponds to [`memory_order_seq_cst`] in C++20. /// /// [`memory_order_seq_cst`]: https://en.cppreference.com/w/cpp/atomic/memory_order#Sequentially-consistent_ordering - /// [`Acquire`]: #variant.Acquire - /// [`Release`]: #variant.Release - /// [`AcqRel`]: #variant.AcqRel #[stable(feature = "rust1", since = "1.0.0")] SeqCst, } /// An [`AtomicBool`] initialized to `false`. -/// -/// [`AtomicBool`]: struct.AtomicBool.html #[cfg(target_has_atomic_load_store = "8")] #[stable(feature = "rust1", since = "1.0.0")] -#[rustc_deprecated( +#[deprecated( since = "1.34.0", - reason = "the `new` function is now preferred", + note = "the `new` function is now preferred", suggestion = "AtomicBool::new(false)" )] pub const ATOMIC_BOOL_INIT: AtomicBool = AtomicBool::new(false); @@ -321,21 +294,15 @@ impl AtomicBool { /// ``` /// use std::sync::atomic::AtomicBool; /// - /// let atomic_true = AtomicBool::new(true); + /// let atomic_true = AtomicBool::new(true); /// let atomic_false = AtomicBool::new(false); /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_atomic_new", since = "1.32.0")] + #[rustc_const_stable(feature = "const_atomic_new", since = "1.24.0")] + #[must_use] pub const fn new(v: bool) -> AtomicBool { - AtomicBool { v: UnsafeCell::new(v as u8), model: true } - } - - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_atomic_new", since = "1.32.0")] - pub const fn new_unmodeled(v: bool) -> AtomicBool { - AtomicBool { v: UnsafeCell::new(v as u8), model: false } + AtomicBool { v: UnsafeCell::new(v as u8) } } /// Returns a mutable reference to the underlying [`bool`]. @@ -343,8 +310,6 @@ impl AtomicBool { /// This is safe because the mutable reference guarantees that no other threads are /// concurrently accessing the atomic data. /// - /// [`bool`]: ../../../std/primitive.bool.html - /// /// # Examples /// /// ``` @@ -362,6 +327,88 @@ impl AtomicBool { unsafe { &mut *(self.v.get() as *mut bool) } } + /// Get atomic access to a `&mut bool`. + /// + /// # Examples + /// + /// ``` + /// #![feature(atomic_from_mut)] + /// use std::sync::atomic::{AtomicBool, Ordering}; + /// + /// let mut some_bool = true; + /// let a = AtomicBool::from_mut(&mut some_bool); + /// a.store(false, Ordering::Relaxed); + /// assert_eq!(some_bool, false); + /// ``` + #[inline] + #[cfg(target_has_atomic_equal_alignment = "8")] + #[unstable(feature = "atomic_from_mut", issue = "76314")] + pub fn from_mut(v: &mut bool) -> &mut Self { + // SAFETY: the mutable reference guarantees unique ownership, and + // alignment of both `bool` and `Self` is 1. + unsafe { &mut *(v as *mut bool as *mut Self) } + } + + /// Get non-atomic access to a `&mut [AtomicBool]` slice. + /// + /// This is safe because the mutable reference guarantees that no other threads are + /// concurrently accessing the atomic data. + /// + /// # Examples + /// + /// ``` + /// #![feature(atomic_from_mut, inline_const)] + /// use std::sync::atomic::{AtomicBool, Ordering}; + /// + /// let mut some_bools = [const { AtomicBool::new(false) }; 10]; + /// + /// let view: &mut [bool] = AtomicBool::get_mut_slice(&mut some_bools); + /// assert_eq!(view, [false; 10]); + /// view[..5].copy_from_slice(&[true; 5]); + /// + /// std::thread::scope(|s| { + /// for t in &some_bools[..5] { + /// s.spawn(move || assert_eq!(t.load(Ordering::Relaxed), true)); + /// } + /// + /// for f in &some_bools[5..] { + /// s.spawn(move || assert_eq!(f.load(Ordering::Relaxed), false)); + /// } + /// }); + /// ``` + #[inline] + #[unstable(feature = "atomic_from_mut", issue = "76314")] + pub fn get_mut_slice(this: &mut [Self]) -> &mut [bool] { + // SAFETY: the mutable reference guarantees unique ownership. + unsafe { &mut *(this as *mut [Self] as *mut [bool]) } + } + + /// Get atomic access to a `&mut [bool]` slice. + /// + /// # Examples + /// + /// ``` + /// #![feature(atomic_from_mut)] + /// use std::sync::atomic::{AtomicBool, Ordering}; + /// + /// let mut some_bools = [false; 10]; + /// let a = &*AtomicBool::from_mut_slice(&mut some_bools); + /// std::thread::scope(|s| { + /// for i in 0..a.len() { + /// s.spawn(move || a[i].store(true, Ordering::Relaxed)); + /// } + /// }); + /// assert_eq!(some_bools, [true; 10]); + /// ``` + #[inline] + #[cfg(target_has_atomic_equal_alignment = "8")] + #[unstable(feature = "atomic_from_mut", issue = "76314")] + pub fn from_mut_slice(v: &mut [bool]) -> &mut [Self] { + // SAFETY: the mutable reference guarantees unique ownership, and + // alignment of both `bool` and `Self` is 1. + unsafe { &mut *(v as *mut [bool] as *mut [Self]) } + } + /// Consumes the atomic and returns the contained value. /// /// This is safe because passing `self` by value guarantees that no other threads are @@ -377,7 +424,8 @@ impl AtomicBool { /// ``` #[inline] #[stable(feature = "atomic_access", since = "1.15.0")] - pub fn into_inner(self) -> bool { + #[rustc_const_unstable(feature = "const_cell_into_inner", issue = "78729")] + pub const fn into_inner(self) -> bool { self.v.into_inner() != 0 } @@ -390,13 +438,6 @@ impl AtomicBool { /// /// Panics if `order` is [`Release`] or [`AcqRel`]. /// - /// [`Ordering`]: enum.Ordering.html - /// [`Relaxed`]: enum.Ordering.html#variant.Relaxed - /// [`Release`]: enum.Ordering.html#variant.Release - /// [`Acquire`]: enum.Ordering.html#variant.Acquire - /// [`AcqRel`]: enum.Ordering.html#variant.AcqRel - /// [`SeqCst`]: enum.Ordering.html#variant.SeqCst - /// /// # Examples /// /// ``` @@ -408,15 +449,11 @@ impl AtomicBool { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn load(&self, order: Ordering) -> bool { // SAFETY: any data races are prevented by atomic intrinsics and the raw // pointer passed in is valid because we got it from a reference. - if self.model { - concurrency::sched_yield(true, self.v.get()); - } - unsafe { - atomic_load(self.v.get(), order) != 0 - } + unsafe { atomic_load(self.v.get(), order) != 0 } } /// Stores a value into the bool. @@ -428,13 +465,6 @@ impl AtomicBool { /// /// Panics if `order` is [`Acquire`] or [`AcqRel`]. /// - /// [`Ordering`]: enum.Ordering.html - /// [`Relaxed`]: enum.Ordering.html#variant.Relaxed - /// [`Release`]: enum.Ordering.html#variant.Release - /// [`Acquire`]: enum.Ordering.html#variant.Acquire - /// [`AcqRel`]: enum.Ordering.html#variant.AcqRel - /// [`SeqCst`]: enum.Ordering.html#variant.SeqCst - /// /// # Examples /// /// ``` @@ -447,12 +477,10 @@ impl AtomicBool { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn store(&self, val: bool, order: Ordering) { // SAFETY: any data races are prevented by atomic intrinsics and the raw // pointer passed in is valid because we got it from a reference. - if self.model { - concurrency::sched_yield(false, self.v.get()); - } unsafe { atomic_store(self.v.get(), val as u8, order); } @@ -465,10 +493,8 @@ impl AtomicBool { /// [`Acquire`] makes the store part of this operation [`Relaxed`], and /// using [`Release`] makes the load part [`Relaxed`]. /// - /// [`Ordering`]: enum.Ordering.html - /// [`Relaxed`]: enum.Ordering.html#variant.Relaxed - /// [`Release`]: enum.Ordering.html#variant.Release - /// [`Acquire`]: enum.Ordering.html#variant.Acquire + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. /// /// # Examples /// @@ -483,11 +509,9 @@ impl AtomicBool { #[inline] #[stable(feature = "rust1", since = "1.0.0")] #[cfg(target_has_atomic = "8")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn swap(&self, val: bool, order: Ordering) -> bool { // SAFETY: data races are prevented by atomic intrinsics. - if self.model { - concurrency::sched_yield(false, self.v.get()); - } unsafe { atomic_swap(self.v.get(), val as u8, order) != 0 } } @@ -502,12 +526,25 @@ impl AtomicBool { /// Using [`Acquire`] makes the store part of this operation [`Relaxed`] if it /// happens, and using [`Release`] makes the load part [`Relaxed`]. /// - /// [`Ordering`]: enum.Ordering.html - /// [`Relaxed`]: enum.Ordering.html#variant.Relaxed - /// [`Release`]: enum.Ordering.html#variant.Release - /// [`Acquire`]: enum.Ordering.html#variant.Acquire - /// [`AcqRel`]: enum.Ordering.html#variant.AcqRel - /// [`bool`]: ../../../std/primitive.bool.html + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. + /// + /// # Migrating to `compare_exchange` and `compare_exchange_weak` + /// + /// `compare_and_swap` is equivalent to `compare_exchange` with the following mapping for + /// memory orderings: + /// + /// Original | Success | Failure + /// -------- | ------- | ------- + /// Relaxed | Relaxed | Relaxed + /// Acquire | Acquire | Acquire + /// Release | Release | Relaxed + /// AcqRel | AcqRel | Acquire + /// SeqCst | SeqCst | SeqCst + /// + /// `compare_exchange_weak` is allowed to fail spuriously even when the comparison succeeds, + /// which allows the compiler to generate better assembly code when the compare and swap + /// is used in a loop. /// /// # Examples /// @@ -524,7 +561,12 @@ impl AtomicBool { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] + #[deprecated( + since = "1.50.0", + note = "Use `compare_exchange` or `compare_exchange_weak` instead" + )] #[cfg(target_has_atomic = "8")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn compare_and_swap(&self, current: bool, new: bool, order: Ordering) -> bool { match self.compare_exchange(current, new, order, strongest_failure_ordering(order)) { Ok(x) => x, @@ -538,20 +580,15 @@ impl AtomicBool { /// the previous value. On success this value is guaranteed to be equal to `current`. /// /// `compare_exchange` takes two [`Ordering`] arguments to describe the memory - /// ordering of this operation. The first describes the required ordering if the - /// operation succeeds while the second describes the required ordering when the - /// operation fails. Using [`Acquire`] as success ordering makes the store part + /// ordering of this operation. `success` describes the required ordering for the + /// read-modify-write operation that takes place if the comparison with `current` succeeds. + /// `failure` describes the required ordering for the load operation that takes place when + /// the comparison fails. Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] - /// and must be equivalent to or weaker than the success ordering. + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// - /// - /// [`bool`]: ../../../std/primitive.bool.html - /// [`Ordering`]: enum.Ordering.html - /// [`Relaxed`]: enum.Ordering.html#variant.Relaxed - /// [`Release`]: enum.Ordering.html#variant.Release - /// [`Acquire`]: enum.Ordering.html#variant.Acquire - /// [`SeqCst`]: enum.Ordering.html#variant.SeqCst + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. /// /// # Examples /// @@ -575,7 +612,9 @@ impl AtomicBool { /// ``` #[inline] #[stable(feature = "extended_compare_and_swap", since = "1.10.0")] + #[doc(alias = "compare_and_swap")] #[cfg(target_has_atomic = "8")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn compare_exchange( &self, current: bool, @@ -584,10 +623,6 @@ impl AtomicBool { failure: Ordering, ) -> Result { // SAFETY: data races are prevented by atomic intrinsics. - // - if self.model { - concurrency::sched_yield(false, self.v.get()); - } match unsafe { atomic_compare_exchange(self.v.get(), current as u8, new as u8, success, failure) } { @@ -598,26 +633,21 @@ impl AtomicBool { /// Stores a value into the [`bool`] if the current value is the same as the `current` value. /// - /// Unlike [`compare_exchange`], this function is allowed to spuriously fail even when the + /// Unlike [`AtomicBool::compare_exchange`], this function is allowed to spuriously fail even when the /// comparison succeeds, which can result in more efficient code on some platforms. The /// return value is a result indicating whether the new value was written and containing the /// previous value. /// /// `compare_exchange_weak` takes two [`Ordering`] arguments to describe the memory - /// ordering of this operation. The first describes the required ordering if the - /// operation succeeds while the second describes the required ordering when the - /// operation fails. Using [`Acquire`] as success ordering makes the store part + /// ordering of this operation. `success` describes the required ordering for the + /// read-modify-write operation that takes place if the comparison with `current` succeeds. + /// `failure` describes the required ordering for the load operation that takes place when + /// the comparison fails. Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] - /// and must be equivalent to or weaker than the success ordering. + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// - /// [`bool`]: ../../../std/primitive.bool.html - /// [`compare_exchange`]: #method.compare_exchange - /// [`Ordering`]: enum.Ordering.html - /// [`Relaxed`]: enum.Ordering.html#variant.Relaxed - /// [`Release`]: enum.Ordering.html#variant.Release - /// [`Acquire`]: enum.Ordering.html#variant.Acquire - /// [`SeqCst`]: enum.Ordering.html#variant.SeqCst + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. /// /// # Examples /// @@ -637,7 +667,9 @@ impl AtomicBool { /// ``` #[inline] #[stable(feature = "extended_compare_and_swap", since = "1.10.0")] + #[doc(alias = "compare_and_swap")] #[cfg(target_has_atomic = "8")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn compare_exchange_weak( &self, current: bool, @@ -646,9 +678,6 @@ impl AtomicBool { failure: Ordering, ) -> Result { // SAFETY: data races are prevented by atomic intrinsics. - if self.model { - concurrency::sched_yield(false, self.v.get()); - } match unsafe { atomic_compare_exchange_weak(self.v.get(), current as u8, new as u8, success, failure) } { @@ -669,10 +698,8 @@ impl AtomicBool { /// [`Acquire`] makes the store part of this operation [`Relaxed`], and /// using [`Release`] makes the load part [`Relaxed`]. /// - /// [`Ordering`]: enum.Ordering.html - /// [`Relaxed`]: enum.Ordering.html#variant.Relaxed - /// [`Release`]: enum.Ordering.html#variant.Release - /// [`Acquire`]: enum.Ordering.html#variant.Acquire + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. /// /// # Examples /// @@ -694,11 +721,9 @@ impl AtomicBool { #[inline] #[stable(feature = "rust1", since = "1.0.0")] #[cfg(target_has_atomic = "8")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn fetch_and(&self, val: bool, order: Ordering) -> bool { // SAFETY: data races are prevented by atomic intrinsics. - if self.model { - concurrency::sched_yield(false, self.v.get()); - } unsafe { atomic_and(self.v.get(), val as u8, order) != 0 } } @@ -714,10 +739,8 @@ impl AtomicBool { /// [`Acquire`] makes the store part of this operation [`Relaxed`], and /// using [`Release`] makes the load part [`Relaxed`]. /// - /// [`Ordering`]: enum.Ordering.html - /// [`Relaxed`]: enum.Ordering.html#variant.Relaxed - /// [`Release`]: enum.Ordering.html#variant.Release - /// [`Acquire`]: enum.Ordering.html#variant.Acquire + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. /// /// # Examples /// @@ -740,6 +763,7 @@ impl AtomicBool { #[inline] #[stable(feature = "rust1", since = "1.0.0")] #[cfg(target_has_atomic = "8")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn fetch_nand(&self, val: bool, order: Ordering) -> bool { // We can't use atomic_nand here because it can result in a bool with // an invalid value. This happens because the atomic operation is done @@ -768,10 +792,8 @@ impl AtomicBool { /// [`Acquire`] makes the store part of this operation [`Relaxed`], and /// using [`Release`] makes the load part [`Relaxed`]. /// - /// [`Ordering`]: enum.Ordering.html - /// [`Relaxed`]: enum.Ordering.html#variant.Relaxed - /// [`Release`]: enum.Ordering.html#variant.Release - /// [`Acquire`]: enum.Ordering.html#variant.Acquire + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. /// /// # Examples /// @@ -793,11 +815,9 @@ impl AtomicBool { #[inline] #[stable(feature = "rust1", since = "1.0.0")] #[cfg(target_has_atomic = "8")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn fetch_or(&self, val: bool, order: Ordering) -> bool { // SAFETY: data races are prevented by atomic intrinsics. - if self.model { - concurrency::sched_yield(false, self.v.get()); - } unsafe { atomic_or(self.v.get(), val as u8, order) != 0 } } @@ -813,10 +833,8 @@ impl AtomicBool { /// [`Acquire`] makes the store part of this operation [`Relaxed`], and /// using [`Release`] makes the load part [`Relaxed`]. /// - /// [`Ordering`]: enum.Ordering.html - /// [`Relaxed`]: enum.Ordering.html#variant.Relaxed - /// [`Release`]: enum.Ordering.html#variant.Release - /// [`Acquire`]: enum.Ordering.html#variant.Acquire + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. /// /// # Examples /// @@ -838,14 +856,49 @@ impl AtomicBool { #[inline] #[stable(feature = "rust1", since = "1.0.0")] #[cfg(target_has_atomic = "8")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn fetch_xor(&self, val: bool, order: Ordering) -> bool { // SAFETY: data races are prevented by atomic intrinsics. - if self.model { - concurrency::sched_yield(false, self.v.get()); - } unsafe { atomic_xor(self.v.get(), val as u8, order) != 0 } } + /// Logical "not" with a boolean value. + /// + /// Performs a logical "not" operation on the current value, and sets + /// the new value to the result. + /// + /// Returns the previous value. + /// + /// `fetch_not` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. + /// + /// # Examples + /// + /// ``` + /// #![feature(atomic_bool_fetch_not)] + /// use std::sync::atomic::{AtomicBool, Ordering}; + /// + /// let foo = AtomicBool::new(true); + /// assert_eq!(foo.fetch_not(Ordering::SeqCst), true); + /// assert_eq!(foo.load(Ordering::SeqCst), false); + /// + /// let foo = AtomicBool::new(false); + /// assert_eq!(foo.fetch_not(Ordering::SeqCst), false); + /// assert_eq!(foo.load(Ordering::SeqCst), true); + /// ``` + #[inline] + #[unstable(feature = "atomic_bool_fetch_not", issue = "98485")] + #[cfg(target_has_atomic = "8")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn fetch_not(&self, order: Ordering) -> bool { + self.fetch_xor(true, order) + } + /// Returns a mutable pointer to the underlying [`bool`]. /// /// Doing non-atomic reads and writes on the resulting integer can be a data race. @@ -858,14 +911,12 @@ impl AtomicBool { /// use of the returned raw pointer requires an `unsafe` block and still has to uphold the same /// restriction: operations on it must be atomic. /// - /// [`bool`]: ../../../std/primitive.bool.html - /// /// # Examples /// /// ```ignore (extern-declaration) /// # fn main() { /// use std::sync::atomic::AtomicBool; - /// extern { + /// extern "C" { /// fn my_atomic_op(arg: *mut bool); /// } /// @@ -880,6 +931,71 @@ impl AtomicBool { pub fn as_mut_ptr(&self) -> *mut bool { self.v.get() as *mut bool } + + /// Fetches the value, and applies a function to it that returns an optional + /// new value. Returns a `Result` of `Ok(previous_value)` if the function + /// returned `Some(_)`, else `Err(previous_value)`. + /// + /// Note: This may call the function multiple times if the value has been + /// changed from other threads in the meantime, as long as the function + /// returns `Some(_)`, but the function will have been applied only once to + /// the stored value. + /// + /// `fetch_update` takes two [`Ordering`] arguments to describe the memory + /// ordering of this operation. The first describes the required ordering for + /// when the operation finally succeeds while the second describes the + /// required ordering for loads. These correspond to the success and failure + /// orderings of [`AtomicBool::compare_exchange`] respectively. + /// + /// Using [`Acquire`] as success ordering makes the store part of this + /// operation [`Relaxed`], and using [`Release`] makes the final successful + /// load [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], + /// [`Acquire`] or [`Relaxed`]. + /// + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. + /// + /// # Considerations + /// + /// This method is not magic; it is not provided by the hardware. + /// It is implemented in terms of [`AtomicBool::compare_exchange_weak`], and suffers from the same drawbacks. + /// In particular, this method will not circumvent the [ABA Problem]. + /// + /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem + /// + /// # Examples + /// + /// ```rust + /// use std::sync::atomic::{AtomicBool, Ordering}; + /// + /// let x = AtomicBool::new(false); + /// assert_eq!(x.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |_| None), Err(false)); + /// assert_eq!(x.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |x| Some(!x)), Ok(false)); + /// assert_eq!(x.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |x| Some(!x)), Ok(true)); + /// assert_eq!(x.load(Ordering::SeqCst), false); + /// ``` + #[inline] + #[stable(feature = "atomic_fetch_update", since = "1.53.0")] + #[cfg(target_has_atomic = "8")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn fetch_update( + &self, + set_order: Ordering, + fetch_order: Ordering, + mut f: F, + ) -> Result + where + F: FnMut(bool) -> Option, + { + let mut prev = self.load(fetch_order); + while let Some(next) = f(prev) { + match self.compare_exchange_weak(prev, next, set_order, fetch_order) { + x @ Ok(_) => return x, + Err(next_prev) => prev = next_prev, + } + } + Err(prev) + } } #[cfg(target_has_atomic_load_store = "ptr")] @@ -892,11 +1008,11 @@ impl AtomicPtr { /// use std::sync::atomic::AtomicPtr; /// /// let ptr = &mut 5; - /// let atomic_ptr = AtomicPtr::new(ptr); + /// let atomic_ptr = AtomicPtr::new(ptr); /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_atomic_new", since = "1.32.0")] + #[rustc_const_stable(feature = "const_atomic_new", since = "1.24.0")] pub const fn new(p: *mut T) -> AtomicPtr { AtomicPtr { p: UnsafeCell::new(p) } } @@ -911,15 +1027,120 @@ impl AtomicPtr { /// ``` /// use std::sync::atomic::{AtomicPtr, Ordering}; /// - /// let mut atomic_ptr = AtomicPtr::new(&mut 10); - /// *atomic_ptr.get_mut() = &mut 5; + /// let mut data = 10; + /// let mut atomic_ptr = AtomicPtr::new(&mut data); + /// let mut other_data = 5; + /// *atomic_ptr.get_mut() = &mut other_data; /// assert_eq!(unsafe { *atomic_ptr.load(Ordering::SeqCst) }, 5); /// ``` #[inline] #[stable(feature = "atomic_access", since = "1.15.0")] pub fn get_mut(&mut self) -> &mut *mut T { + self.p.get_mut() + } + + /// Get atomic access to a pointer. + /// + /// # Examples + /// + /// ``` + /// #![feature(atomic_from_mut)] + /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// + /// let mut data = 123; + /// let mut some_ptr = &mut data as *mut i32; + /// let a = AtomicPtr::from_mut(&mut some_ptr); + /// let mut other_data = 456; + /// a.store(&mut other_data, Ordering::Relaxed); + /// assert_eq!(unsafe { *some_ptr }, 456); + /// ``` + #[inline] + #[cfg(target_has_atomic_equal_alignment = "ptr")] + #[unstable(feature = "atomic_from_mut", issue = "76314")] + pub fn from_mut(v: &mut *mut T) -> &mut Self { + use crate::mem::align_of; + let [] = [(); align_of::>() - align_of::<*mut ()>()]; + // SAFETY: + // - the mutable reference guarantees unique ownership. + // - the alignment of `*mut T` and `Self` is the same on all platforms + // supported by rust, as verified above. + unsafe { &mut *(v as *mut *mut T as *mut Self) } + } + + /// Get non-atomic access to a `&mut [AtomicPtr]` slice. + /// + /// This is safe because the mutable reference guarantees that no other threads are + /// concurrently accessing the atomic data. + /// + /// # Examples + /// + /// ``` + /// #![feature(atomic_from_mut, inline_const)] + /// use std::ptr::null_mut; + /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// + /// let mut some_ptrs = [const { AtomicPtr::new(null_mut::()) }; 10]; + /// + /// let view: &mut [*mut String] = AtomicPtr::get_mut_slice(&mut some_ptrs); + /// assert_eq!(view, [null_mut::(); 10]); + /// view + /// .iter_mut() + /// .enumerate() + /// .for_each(|(i, ptr)| *ptr = Box::into_raw(Box::new(format!("iteration#{i}")))); + /// + /// std::thread::scope(|s| { + /// for ptr in &some_ptrs { + /// s.spawn(move || { + /// let ptr = ptr.load(Ordering::Relaxed); + /// assert!(!ptr.is_null()); + /// + /// let name = unsafe { Box::from_raw(ptr) }; + /// println!("Hello, {name}!"); + /// }); + /// } + /// }); + /// ``` + #[inline] + #[unstable(feature = "atomic_from_mut", issue = "76314")] + pub fn get_mut_slice(this: &mut [Self]) -> &mut [*mut T] { // SAFETY: the mutable reference guarantees unique ownership. - unsafe { &mut *self.p.get() } + unsafe { &mut *(this as *mut [Self] as *mut [*mut T]) } + } + + /// Get atomic access to a slice of pointers. + /// + /// # Examples + /// + /// ``` + /// #![feature(atomic_from_mut)] + /// use std::ptr::null_mut; + /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// + /// let mut some_ptrs = [null_mut::(); 10]; + /// let a = &*AtomicPtr::from_mut_slice(&mut some_ptrs); + /// std::thread::scope(|s| { + /// for i in 0..a.len() { + /// s.spawn(move || { + /// let name = Box::new(format!("thread{i}")); + /// a[i].store(Box::into_raw(name), Ordering::Relaxed); + /// }); + /// } + /// }); + /// for p in some_ptrs { + /// assert!(!p.is_null()); + /// let name = unsafe { Box::from_raw(p) }; + /// println!("Hello, {name}!"); + /// } + /// ``` + #[inline] + #[cfg(target_has_atomic_equal_alignment = "ptr")] + #[unstable(feature = "atomic_from_mut", issue = "76314")] + pub fn from_mut_slice(v: &mut [*mut T]) -> &mut [Self] { + // SAFETY: + // - the mutable reference guarantees unique ownership. + // - the alignment of `*mut T` and `Self` is the same on all platforms + // supported by rust, as verified above. + unsafe { &mut *(v as *mut [*mut T] as *mut [Self]) } } /// Consumes the atomic and returns the contained value. @@ -932,12 +1153,14 @@ impl AtomicPtr { /// ``` /// use std::sync::atomic::AtomicPtr; /// - /// let atomic_ptr = AtomicPtr::new(&mut 5); + /// let mut data = 5; + /// let atomic_ptr = AtomicPtr::new(&mut data); /// assert_eq!(unsafe { *atomic_ptr.into_inner() }, 5); /// ``` #[inline] #[stable(feature = "atomic_access", since = "1.15.0")] - pub fn into_inner(self) -> *mut T { + #[rustc_const_unstable(feature = "const_cell_into_inner", issue = "78729")] + pub const fn into_inner(self) -> *mut T { self.p.into_inner() } @@ -950,29 +1173,22 @@ impl AtomicPtr { /// /// Panics if `order` is [`Release`] or [`AcqRel`]. /// - /// [`Ordering`]: enum.Ordering.html - /// [`Relaxed`]: enum.Ordering.html#variant.Relaxed - /// [`Release`]: enum.Ordering.html#variant.Release - /// [`Acquire`]: enum.Ordering.html#variant.Acquire - /// [`AcqRel`]: enum.Ordering.html#variant.AcqRel - /// [`SeqCst`]: enum.Ordering.html#variant.SeqCst - /// /// # Examples /// /// ``` /// use std::sync::atomic::{AtomicPtr, Ordering}; /// /// let ptr = &mut 5; - /// let some_ptr = AtomicPtr::new(ptr); + /// let some_ptr = AtomicPtr::new(ptr); /// /// let value = some_ptr.load(Ordering::Relaxed); /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn load(&self, order: Ordering) -> *mut T { // SAFETY: data races are prevented by atomic intrinsics. - concurrency::sched_yield(true, self.p.get()); - unsafe { atomic_load(self.p.get() as *mut usize, order) as *mut T } + unsafe { atomic_load(self.p.get(), order) } } /// Stores a value into the pointer. @@ -984,20 +1200,13 @@ impl AtomicPtr { /// /// Panics if `order` is [`Acquire`] or [`AcqRel`]. /// - /// [`Ordering`]: enum.Ordering.html - /// [`Relaxed`]: enum.Ordering.html#variant.Relaxed - /// [`Release`]: enum.Ordering.html#variant.Release - /// [`Acquire`]: enum.Ordering.html#variant.Acquire - /// [`AcqRel`]: enum.Ordering.html#variant.AcqRel - /// [`SeqCst`]: enum.Ordering.html#variant.SeqCst - /// /// # Examples /// /// ``` /// use std::sync::atomic::{AtomicPtr, Ordering}; /// /// let ptr = &mut 5; - /// let some_ptr = AtomicPtr::new(ptr); + /// let some_ptr = AtomicPtr::new(ptr); /// /// let other_ptr = &mut 10; /// @@ -1005,12 +1214,11 @@ impl AtomicPtr { /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn store(&self, ptr: *mut T, order: Ordering) { // SAFETY: data races are prevented by atomic intrinsics. - // - concurrency::sched_yield(false, self.p.get()); unsafe { - atomic_store(self.p.get() as *mut usize, ptr as usize, order); + atomic_store(self.p.get(), ptr, order); } } @@ -1021,10 +1229,8 @@ impl AtomicPtr { /// [`Acquire`] makes the store part of this operation [`Relaxed`], and /// using [`Release`] makes the load part [`Relaxed`]. /// - /// [`Ordering`]: enum.Ordering.html - /// [`Relaxed`]: enum.Ordering.html#variant.Relaxed - /// [`Release`]: enum.Ordering.html#variant.Release - /// [`Acquire`]: enum.Ordering.html#variant.Acquire + /// **Note:** This method is only available on platforms that support atomic + /// operations on pointers. /// /// # Examples /// @@ -1032,7 +1238,7 @@ impl AtomicPtr { /// use std::sync::atomic::{AtomicPtr, Ordering}; /// /// let ptr = &mut 5; - /// let some_ptr = AtomicPtr::new(ptr); + /// let some_ptr = AtomicPtr::new(ptr); /// /// let other_ptr = &mut 10; /// @@ -1041,10 +1247,10 @@ impl AtomicPtr { #[inline] #[stable(feature = "rust1", since = "1.0.0")] #[cfg(target_has_atomic = "ptr")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn swap(&self, ptr: *mut T, order: Ordering) -> *mut T { // SAFETY: data races are prevented by atomic intrinsics. - concurrency::sched_yield(false, self.p.get()); - unsafe { atomic_swap(self.p.get() as *mut usize, ptr as usize, order) as *mut T } + unsafe { atomic_swap(self.p.get(), ptr, order) } } /// Stores a value into the pointer if the current value is the same as the `current` value. @@ -1058,11 +1264,25 @@ impl AtomicPtr { /// Using [`Acquire`] makes the store part of this operation [`Relaxed`] if it /// happens, and using [`Release`] makes the load part [`Relaxed`]. /// - /// [`Ordering`]: enum.Ordering.html - /// [`Relaxed`]: enum.Ordering.html#variant.Relaxed - /// [`Release`]: enum.Ordering.html#variant.Release - /// [`Acquire`]: enum.Ordering.html#variant.Acquire - /// [`AcqRel`]: enum.Ordering.html#variant.AcqRel + /// **Note:** This method is only available on platforms that support atomic + /// operations on pointers. + /// + /// # Migrating to `compare_exchange` and `compare_exchange_weak` + /// + /// `compare_and_swap` is equivalent to `compare_exchange` with the following mapping for + /// memory orderings: + /// + /// Original | Success | Failure + /// -------- | ------- | ------- + /// Relaxed | Relaxed | Relaxed + /// Acquire | Acquire | Acquire + /// Release | Release | Relaxed + /// AcqRel | AcqRel | Acquire + /// SeqCst | SeqCst | SeqCst + /// + /// `compare_exchange_weak` is allowed to fail spuriously even when the comparison succeeds, + /// which allows the compiler to generate better assembly code when the compare and swap + /// is used in a loop. /// /// # Examples /// @@ -1070,15 +1290,20 @@ impl AtomicPtr { /// use std::sync::atomic::{AtomicPtr, Ordering}; /// /// let ptr = &mut 5; - /// let some_ptr = AtomicPtr::new(ptr); + /// let some_ptr = AtomicPtr::new(ptr); /// - /// let other_ptr = &mut 10; + /// let other_ptr = &mut 10; /// /// let value = some_ptr.compare_and_swap(ptr, other_ptr, Ordering::Relaxed); /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] + #[deprecated( + since = "1.50.0", + note = "Use `compare_exchange` or `compare_exchange_weak` instead" + )] #[cfg(target_has_atomic = "ptr")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn compare_and_swap(&self, current: *mut T, new: *mut T, order: Ordering) -> *mut T { match self.compare_exchange(current, new, order, strongest_failure_ordering(order)) { Ok(x) => x, @@ -1092,18 +1317,15 @@ impl AtomicPtr { /// the previous value. On success this value is guaranteed to be equal to `current`. /// /// `compare_exchange` takes two [`Ordering`] arguments to describe the memory - /// ordering of this operation. The first describes the required ordering if the - /// operation succeeds while the second describes the required ordering when the - /// operation fails. Using [`Acquire`] as success ordering makes the store part + /// ordering of this operation. `success` describes the required ordering for the + /// read-modify-write operation that takes place if the comparison with `current` succeeds. + /// `failure` describes the required ordering for the load operation that takes place when + /// the comparison fails. Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] - /// and must be equivalent to or weaker than the success ordering. + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// - /// [`Ordering`]: enum.Ordering.html - /// [`Relaxed`]: enum.Ordering.html#variant.Relaxed - /// [`Release`]: enum.Ordering.html#variant.Release - /// [`Acquire`]: enum.Ordering.html#variant.Acquire - /// [`SeqCst`]: enum.Ordering.html#variant.SeqCst + /// **Note:** This method is only available on platforms that support atomic + /// operations on pointers. /// /// # Examples /// @@ -1111,9 +1333,9 @@ impl AtomicPtr { /// use std::sync::atomic::{AtomicPtr, Ordering}; /// /// let ptr = &mut 5; - /// let some_ptr = AtomicPtr::new(ptr); + /// let some_ptr = AtomicPtr::new(ptr); /// - /// let other_ptr = &mut 10; + /// let other_ptr = &mut 10; /// /// let value = some_ptr.compare_exchange(ptr, other_ptr, /// Ordering::SeqCst, Ordering::Relaxed); @@ -1121,6 +1343,7 @@ impl AtomicPtr { #[inline] #[stable(feature = "extended_compare_and_swap", since = "1.10.0")] #[cfg(target_has_atomic = "ptr")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn compare_exchange( &self, current: *mut T, @@ -1129,43 +1352,26 @@ impl AtomicPtr { failure: Ordering, ) -> Result<*mut T, *mut T> { // SAFETY: data races are prevented by atomic intrinsics. - concurrency::sched_yield(false, self.p.get()); - unsafe { - let res = atomic_compare_exchange( - self.p.get() as *mut usize, - current as usize, - new as usize, - success, - failure, - ); - match res { - Ok(x) => Ok(x as *mut T), - Err(x) => Err(x as *mut T), - } - } + unsafe { atomic_compare_exchange(self.p.get(), current, new, success, failure) } } /// Stores a value into the pointer if the current value is the same as the `current` value. /// - /// Unlike [`compare_exchange`], this function is allowed to spuriously fail even when the + /// Unlike [`AtomicPtr::compare_exchange`], this function is allowed to spuriously fail even when the /// comparison succeeds, which can result in more efficient code on some platforms. The /// return value is a result indicating whether the new value was written and containing the /// previous value. /// /// `compare_exchange_weak` takes two [`Ordering`] arguments to describe the memory - /// ordering of this operation. The first describes the required ordering if the - /// operation succeeds while the second describes the required ordering when the - /// operation fails. Using [`Acquire`] as success ordering makes the store part + /// ordering of this operation. `success` describes the required ordering for the + /// read-modify-write operation that takes place if the comparison with `current` succeeds. + /// `failure` describes the required ordering for the load operation that takes place when + /// the comparison fails. Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] - /// and must be equivalent to or weaker than the success ordering. + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// - /// [`compare_exchange`]: #method.compare_exchange - /// [`Ordering`]: enum.Ordering.html - /// [`Relaxed`]: enum.Ordering.html#variant.Relaxed - /// [`Release`]: enum.Ordering.html#variant.Release - /// [`Acquire`]: enum.Ordering.html#variant.Acquire - /// [`SeqCst`]: enum.Ordering.html#variant.SeqCst + /// **Note:** This method is only available on platforms that support atomic + /// operations on pointers. /// /// # Examples /// @@ -1186,6 +1392,7 @@ impl AtomicPtr { #[inline] #[stable(feature = "extended_compare_and_swap", since = "1.10.0")] #[cfg(target_has_atomic = "ptr")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn compare_exchange_weak( &self, current: *mut T, @@ -1193,118 +1400,538 @@ impl AtomicPtr { success: Ordering, failure: Ordering, ) -> Result<*mut T, *mut T> { - // SAFETY: data races are prevented by atomic intrinsics. - concurrency::sched_yield(false, self.p.get()); - unsafe { - let res = atomic_compare_exchange_weak( - self.p.get() as *mut usize, - current as usize, - new as usize, - success, - failure, - ); - match res { - Ok(x) => Ok(x as *mut T), - Err(x) => Err(x as *mut T), + // SAFETY: This intrinsic is unsafe because it operates on a raw pointer + // but we know for sure that the pointer is valid (we just got it from + // an `UnsafeCell` that we have by reference) and the atomic operation + // itself allows us to safely mutate the `UnsafeCell` contents. + unsafe { atomic_compare_exchange_weak(self.p.get(), current, new, success, failure) } + } + + /// Fetches the value, and applies a function to it that returns an optional + /// new value. Returns a `Result` of `Ok(previous_value)` if the function + /// returned `Some(_)`, else `Err(previous_value)`. + /// + /// Note: This may call the function multiple times if the value has been + /// changed from other threads in the meantime, as long as the function + /// returns `Some(_)`, but the function will have been applied only once to + /// the stored value. + /// + /// `fetch_update` takes two [`Ordering`] arguments to describe the memory + /// ordering of this operation. The first describes the required ordering for + /// when the operation finally succeeds while the second describes the + /// required ordering for loads. These correspond to the success and failure + /// orderings of [`AtomicPtr::compare_exchange`] respectively. + /// + /// Using [`Acquire`] as success ordering makes the store part of this + /// operation [`Relaxed`], and using [`Release`] makes the final successful + /// load [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], + /// [`Acquire`] or [`Relaxed`]. + /// + /// **Note:** This method is only available on platforms that support atomic + /// operations on pointers. + /// + /// # Considerations + /// + /// This method is not magic; it is not provided by the hardware. + /// It is implemented in terms of [`AtomicPtr::compare_exchange_weak`], and suffers from the same drawbacks. + /// In particular, this method will not circumvent the [ABA Problem]. + /// + /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem + /// + /// # Examples + /// + /// ```rust + /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// + /// let ptr: *mut _ = &mut 5; + /// let some_ptr = AtomicPtr::new(ptr); + /// + /// let new: *mut _ = &mut 10; + /// assert_eq!(some_ptr.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |_| None), Err(ptr)); + /// let result = some_ptr.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |x| { + /// if x == ptr { + /// Some(new) + /// } else { + /// None + /// } + /// }); + /// assert_eq!(result, Ok(ptr)); + /// assert_eq!(some_ptr.load(Ordering::SeqCst), new); + /// ``` + #[inline] + #[stable(feature = "atomic_fetch_update", since = "1.53.0")] + #[cfg(target_has_atomic = "ptr")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn fetch_update( + &self, + set_order: Ordering, + fetch_order: Ordering, + mut f: F, + ) -> Result<*mut T, *mut T> + where + F: FnMut(*mut T) -> Option<*mut T>, + { + let mut prev = self.load(fetch_order); + while let Some(next) = f(prev) { + match self.compare_exchange_weak(prev, next, set_order, fetch_order) { + x @ Ok(_) => return x, + Err(next_prev) => prev = next_prev, } } + Err(prev) } -} -#[cfg(target_has_atomic_load_store = "8")] -#[stable(feature = "atomic_bool_from", since = "1.24.0")] -impl From for AtomicBool { - /// Converts a `bool` into an `AtomicBool`. + /// Offsets the pointer's address by adding `val` (in units of `T`), + /// returning the previous pointer. + /// + /// This is equivalent to using [`wrapping_add`] to atomically perform the + /// equivalent of `ptr = ptr.wrapping_add(val);`. + /// + /// This method operates in units of `T`, which means that it cannot be used + /// to offset the pointer by an amount which is not a multiple of + /// `size_of::()`. This can sometimes be inconvenient, as you may want to + /// work with a deliberately misaligned pointer. In such cases, you may use + /// the [`fetch_byte_add`](Self::fetch_byte_add) method instead. + /// + /// `fetch_ptr_add` takes an [`Ordering`] argument which describes the + /// memory ordering of this operation. All ordering modes are possible. Note + /// that using [`Acquire`] makes the store part of this operation + /// [`Relaxed`], and using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic + /// operations on [`AtomicPtr`]. + /// + /// [`wrapping_add`]: pointer::wrapping_add /// /// # Examples /// /// ``` - /// use std::sync::atomic::AtomicBool; - /// let atomic_bool = AtomicBool::from(true); - /// assert_eq!(format!("{:?}", atomic_bool), "true") + /// #![feature(strict_provenance_atomic_ptr, strict_provenance)] + /// use core::sync::atomic::{AtomicPtr, Ordering}; + /// + /// let atom = AtomicPtr::::new(core::ptr::null_mut()); + /// assert_eq!(atom.fetch_ptr_add(1, Ordering::Relaxed).addr(), 0); + /// // Note: units of `size_of::()`. + /// assert_eq!(atom.load(Ordering::Relaxed).addr(), 8); /// ``` #[inline] - fn from(b: bool) -> Self { - Self::new(b) + #[cfg(target_has_atomic = "ptr")] + #[unstable(feature = "strict_provenance_atomic_ptr", issue = "99108")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn fetch_ptr_add(&self, val: usize, order: Ordering) -> *mut T { + self.fetch_byte_add(val.wrapping_mul(core::mem::size_of::()), order) } -} -#[cfg(target_has_atomic_load_store = "ptr")] -#[stable(feature = "atomic_from", since = "1.23.0")] -impl From<*mut T> for AtomicPtr { + /// Offsets the pointer's address by subtracting `val` (in units of `T`), + /// returning the previous pointer. + /// + /// This is equivalent to using [`wrapping_sub`] to atomically perform the + /// equivalent of `ptr = ptr.wrapping_sub(val);`. + /// + /// This method operates in units of `T`, which means that it cannot be used + /// to offset the pointer by an amount which is not a multiple of + /// `size_of::()`. This can sometimes be inconvenient, as you may want to + /// work with a deliberately misaligned pointer. In such cases, you may use + /// the [`fetch_byte_sub`](Self::fetch_byte_sub) method instead. + /// + /// `fetch_ptr_sub` takes an [`Ordering`] argument which describes the memory + /// ordering of this operation. All ordering modes are possible. Note that + /// using [`Acquire`] makes the store part of this operation [`Relaxed`], + /// and using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic + /// operations on [`AtomicPtr`]. + /// + /// [`wrapping_sub`]: pointer::wrapping_sub + /// + /// # Examples + /// + /// ``` + /// #![feature(strict_provenance_atomic_ptr)] + /// use core::sync::atomic::{AtomicPtr, Ordering}; + /// + /// let array = [1i32, 2i32]; + /// let atom = AtomicPtr::new(array.as_ptr().wrapping_add(1) as *mut _); + /// + /// assert!(core::ptr::eq( + /// atom.fetch_ptr_sub(1, Ordering::Relaxed), + /// &array[1], + /// )); + /// assert!(core::ptr::eq(atom.load(Ordering::Relaxed), &array[0])); + /// ``` #[inline] - fn from(p: *mut T) -> Self { - Self::new(p) + #[cfg(target_has_atomic = "ptr")] + #[unstable(feature = "strict_provenance_atomic_ptr", issue = "99108")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn fetch_ptr_sub(&self, val: usize, order: Ordering) -> *mut T { + self.fetch_byte_sub(val.wrapping_mul(core::mem::size_of::()), order) } -} -#[cfg(target_has_atomic_load_store = "8")] -macro_rules! atomic_int { - ($cfg_cas:meta, - $stable:meta, - $stable_cxchg:meta, - $stable_debug:meta, - $stable_access:meta, - $stable_from:meta, - $stable_nand:meta, - $const_stable:meta, - $stable_init_const:meta, - $s_int_type:expr, $int_ref:expr, - $extra_feature:expr, - $min_fn:ident, $max_fn:ident, - $align:expr, - $atomic_new:expr, - $int_type:ident $atomic_type:ident $atomic_init:ident) => { - /// An integer type which can be safely shared between threads. - /// - /// This type has the same in-memory representation as the underlying - /// integer type, [` + /// Offsets the pointer's address by adding `val` *bytes*, returning the + /// previous pointer. + /// + /// This is equivalent to using [`wrapping_byte_add`] to atomically + /// perform `ptr = ptr.wrapping_byte_add(val)`. + /// + /// `fetch_byte_add` takes an [`Ordering`] argument which describes the + /// memory ordering of this operation. All ordering modes are possible. Note + /// that using [`Acquire`] makes the store part of this operation + /// [`Relaxed`], and using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic + /// operations on [`AtomicPtr`]. + /// + /// [`wrapping_byte_add`]: pointer::wrapping_byte_add + /// + /// # Examples + /// + /// ``` + /// #![feature(strict_provenance_atomic_ptr, strict_provenance)] + /// use core::sync::atomic::{AtomicPtr, Ordering}; + /// + /// let atom = AtomicPtr::::new(core::ptr::null_mut()); + /// assert_eq!(atom.fetch_byte_add(1, Ordering::Relaxed).addr(), 0); + /// // Note: in units of bytes, not `size_of::()`. + /// assert_eq!(atom.load(Ordering::Relaxed).addr(), 1); + /// ``` + #[inline] + #[cfg(target_has_atomic = "ptr")] + #[unstable(feature = "strict_provenance_atomic_ptr", issue = "99108")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn fetch_byte_add(&self, val: usize, order: Ordering) -> *mut T { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_add(self.p.get(), core::ptr::invalid_mut(val), order).cast() } + } + + /// Offsets the pointer's address by subtracting `val` *bytes*, returning the + /// previous pointer. + /// + /// This is equivalent to using [`wrapping_byte_sub`] to atomically + /// perform `ptr = ptr.wrapping_byte_sub(val)`. + /// + /// `fetch_byte_sub` takes an [`Ordering`] argument which describes the + /// memory ordering of this operation. All ordering modes are possible. Note + /// that using [`Acquire`] makes the store part of this operation + /// [`Relaxed`], and using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic + /// operations on [`AtomicPtr`]. + /// + /// [`wrapping_byte_sub`]: pointer::wrapping_byte_sub + /// + /// # Examples + /// + /// ``` + /// #![feature(strict_provenance_atomic_ptr, strict_provenance)] + /// use core::sync::atomic::{AtomicPtr, Ordering}; + /// + /// let atom = AtomicPtr::::new(core::ptr::invalid_mut(1)); + /// assert_eq!(atom.fetch_byte_sub(1, Ordering::Relaxed).addr(), 1); + /// assert_eq!(atom.load(Ordering::Relaxed).addr(), 0); + /// ``` + #[inline] + #[cfg(target_has_atomic = "ptr")] + #[unstable(feature = "strict_provenance_atomic_ptr", issue = "99108")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn fetch_byte_sub(&self, val: usize, order: Ordering) -> *mut T { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_sub(self.p.get(), core::ptr::invalid_mut(val), order).cast() } + } + + /// Performs a bitwise "or" operation on the address of the current pointer, + /// and the argument `val`, and stores a pointer with provenance of the + /// current pointer and the resulting address. + /// + /// This is equivalent to using [`map_addr`] to atomically perform + /// `ptr = ptr.map_addr(|a| a | val)`. This can be used in tagged + /// pointer schemes to atomically set tag bits. + /// + /// **Caveat**: This operation returns the previous value. To compute the + /// stored value without losing provenance, you may use [`map_addr`]. For + /// example: `a.fetch_or(val).map_addr(|a| a | val)`. + /// + /// `fetch_or` takes an [`Ordering`] argument which describes the memory + /// ordering of this operation. All ordering modes are possible. Note that + /// using [`Acquire`] makes the store part of this operation [`Relaxed`], + /// and using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic + /// operations on [`AtomicPtr`]. + /// + /// This API and its claimed semantics are part of the Strict Provenance + /// experiment, see the [module documentation for `ptr`][crate::ptr] for + /// details. + /// + /// [`map_addr`]: pointer::map_addr + /// + /// # Examples + /// + /// ``` + /// #![feature(strict_provenance_atomic_ptr, strict_provenance)] + /// use core::sync::atomic::{AtomicPtr, Ordering}; + /// + /// let pointer = &mut 3i64 as *mut i64; + /// + /// let atom = AtomicPtr::::new(pointer); + /// // Tag the bottom bit of the pointer. + /// assert_eq!(atom.fetch_or(1, Ordering::Relaxed).addr() & 1, 0); + /// // Extract and untag. + /// let tagged = atom.load(Ordering::Relaxed); + /// assert_eq!(tagged.addr() & 1, 1); + /// assert_eq!(tagged.map_addr(|p| p & !1), pointer); + /// ``` + #[inline] + #[cfg(target_has_atomic = "ptr")] + #[unstable(feature = "strict_provenance_atomic_ptr", issue = "99108")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn fetch_or(&self, val: usize, order: Ordering) -> *mut T { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_or(self.p.get(), core::ptr::invalid_mut(val), order).cast() } + } + + /// Performs a bitwise "and" operation on the address of the current + /// pointer, and the argument `val`, and stores a pointer with provenance of + /// the current pointer and the resulting address. + /// + /// This is equivalent to using [`map_addr`] to atomically perform + /// `ptr = ptr.map_addr(|a| a & val)`. This can be used in tagged + /// pointer schemes to atomically unset tag bits. + /// + /// **Caveat**: This operation returns the previous value. To compute the + /// stored value without losing provenance, you may use [`map_addr`]. For + /// example: `a.fetch_and(val).map_addr(|a| a & val)`. + /// + /// `fetch_and` takes an [`Ordering`] argument which describes the memory + /// ordering of this operation. All ordering modes are possible. Note that + /// using [`Acquire`] makes the store part of this operation [`Relaxed`], + /// and using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic + /// operations on [`AtomicPtr`]. + /// + /// This API and its claimed semantics are part of the Strict Provenance + /// experiment, see the [module documentation for `ptr`][crate::ptr] for + /// details. + /// + /// [`map_addr`]: pointer::map_addr + /// + /// # Examples + /// + /// ``` + /// #![feature(strict_provenance_atomic_ptr, strict_provenance)] + /// use core::sync::atomic::{AtomicPtr, Ordering}; + /// + /// let pointer = &mut 3i64 as *mut i64; + /// // A tagged pointer + /// let atom = AtomicPtr::::new(pointer.map_addr(|a| a | 1)); + /// assert_eq!(atom.fetch_or(1, Ordering::Relaxed).addr() & 1, 1); + /// // Untag, and extract the previously tagged pointer. + /// let untagged = atom.fetch_and(!1, Ordering::Relaxed) + /// .map_addr(|a| a & !1); + /// assert_eq!(untagged, pointer); + /// ``` + #[inline] + #[cfg(target_has_atomic = "ptr")] + #[unstable(feature = "strict_provenance_atomic_ptr", issue = "99108")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn fetch_and(&self, val: usize, order: Ordering) -> *mut T { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_and(self.p.get(), core::ptr::invalid_mut(val), order).cast() } + } + + /// Performs a bitwise "xor" operation on the address of the current + /// pointer, and the argument `val`, and stores a pointer with provenance of + /// the current pointer and the resulting address. + /// + /// This is equivalent to using [`map_addr`] to atomically perform + /// `ptr = ptr.map_addr(|a| a ^ val)`. This can be used in tagged + /// pointer schemes to atomically toggle tag bits. + /// + /// **Caveat**: This operation returns the previous value. To compute the + /// stored value without losing provenance, you may use [`map_addr`]. For + /// example: `a.fetch_xor(val).map_addr(|a| a ^ val)`. + /// + /// `fetch_xor` takes an [`Ordering`] argument which describes the memory + /// ordering of this operation. All ordering modes are possible. Note that + /// using [`Acquire`] makes the store part of this operation [`Relaxed`], + /// and using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic + /// operations on [`AtomicPtr`]. + /// + /// This API and its claimed semantics are part of the Strict Provenance + /// experiment, see the [module documentation for `ptr`][crate::ptr] for + /// details. + /// + /// [`map_addr`]: pointer::map_addr + /// + /// # Examples + /// + /// ``` + /// #![feature(strict_provenance_atomic_ptr, strict_provenance)] + /// use core::sync::atomic::{AtomicPtr, Ordering}; + /// + /// let pointer = &mut 3i64 as *mut i64; + /// let atom = AtomicPtr::::new(pointer); + /// + /// // Toggle a tag bit on the pointer. + /// atom.fetch_xor(1, Ordering::Relaxed); + /// assert_eq!(atom.load(Ordering::Relaxed).addr() & 1, 1); + /// ``` + #[inline] + #[cfg(target_has_atomic = "ptr")] + #[unstable(feature = "strict_provenance_atomic_ptr", issue = "99108")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn fetch_xor(&self, val: usize, order: Ordering) -> *mut T { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_xor(self.p.get(), core::ptr::invalid_mut(val), order).cast() } + } + + /// Returns a mutable pointer to the underlying pointer. + /// + /// Doing non-atomic reads and writes on the resulting integer can be a data race. + /// This method is mostly useful for FFI, where the function signature may use + /// `*mut *mut T` instead of `&AtomicPtr`. + /// + /// Returning an `*mut` pointer from a shared reference to this atomic is safe because the + /// atomic types work with interior mutability. All modifications of an atomic change the value + /// through a shared reference, and can do so safely as long as they use atomic operations. Any + /// use of the returned raw pointer requires an `unsafe` block and still has to uphold the same + /// restriction: operations on it must be atomic. + /// + /// # Examples + /// + /// ```ignore (extern-declaration) + /// #![feature(atomic_mut_ptr)] + //// use std::sync::atomic::AtomicPtr; + /// + /// extern "C" { + /// fn my_atomic_op(arg: *mut *mut u32); + /// } + /// + /// let mut value = 17; + /// let atomic = AtomicPtr::new(&mut value); + /// + /// // SAFETY: Safe as long as `my_atomic_op` is atomic. + /// unsafe { + /// my_atomic_op(atomic.as_mut_ptr()); + /// } + /// ``` + #[inline] + #[unstable(feature = "atomic_mut_ptr", reason = "recently added", issue = "66893")] + pub fn as_mut_ptr(&self) -> *mut *mut T { + self.p.get() + } +} + +#[cfg(target_has_atomic_load_store = "8")] +#[stable(feature = "atomic_bool_from", since = "1.24.0")] +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const From for AtomicBool { + /// Converts a `bool` into an `AtomicBool`. + /// + /// # Examples + /// + /// ``` + /// use std::sync::atomic::AtomicBool; + /// let atomic_bool = AtomicBool::from(true); + /// assert_eq!(format!("{atomic_bool:?}"), "true") + /// ``` + #[inline] + fn from(b: bool) -> Self { + Self::new(b) + } +} + +#[cfg(target_has_atomic_load_store = "ptr")] +#[stable(feature = "atomic_from", since = "1.23.0")] +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const From<*mut T> for AtomicPtr { + /// Converts a `*mut T` into an `AtomicPtr`. + #[inline] + fn from(p: *mut T) -> Self { + Self::new(p) + } +} + +#[allow(unused_macros)] // This macro ends up being unused on some architectures. +macro_rules! if_not_8_bit { + (u8, $($tt:tt)*) => { "" }; + (i8, $($tt:tt)*) => { "" }; + ($_:ident, $($tt:tt)*) => { $($tt)* }; +} + +#[cfg(target_has_atomic_load_store = "8")] +macro_rules! atomic_int { + ($cfg_cas:meta, + $cfg_align:meta, + $stable:meta, + $stable_cxchg:meta, + $stable_debug:meta, + $stable_access:meta, + $stable_from:meta, + $stable_nand:meta, + $const_stable:meta, + $stable_init_const:meta, + $diagnostic_item:meta, + $s_int_type:literal, + $extra_feature:expr, + $min_fn:ident, $max_fn:ident, + $align:expr, + $atomic_new:expr, + $int_type:ident $atomic_type:ident $atomic_init:ident) => { + /// An integer type which can be safely shared between threads. + /// + /// This type has the same in-memory representation as the underlying + /// integer type, [` #[doc = $s_int_type] - /// `]( - #[doc = $int_ref] - /// ). For more about the differences between atomic types and + /// `]. For more about the differences between atomic types and /// non-atomic types as well as information about the portability of /// this type, please see the [module-level documentation]. /// - /// [module-level documentation]: index.html + /// **Note:** This type is only available on platforms that support + /// atomic loads and stores of [` + #[doc = $s_int_type] + /// `]. + /// + /// [module-level documentation]: crate::sync::atomic #[$stable] + #[$diagnostic_item] #[repr(C, align($align))] pub struct $atomic_type { v: UnsafeCell<$int_type>, - model: bool } /// An atomic integer initialized to `0`. #[$stable_init_const] - #[rustc_deprecated( + #[deprecated( since = "1.34.0", - reason = "the `new` function is now preferred", + note = "the `new` function is now preferred", suggestion = $atomic_new, )] pub const $atomic_init: $atomic_type = $atomic_type::new(0); #[$stable] - impl Default for $atomic_type { + #[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] + impl const Default for $atomic_type { + #[inline] fn default() -> Self { Self::new(Default::default()) } } - #[$stable_from] - impl From<$int_type> for $atomic_type { - doc_comment! { - concat!( -"Converts an `", stringify!($int_type), "` into an `", stringify!($atomic_type), "`."), - #[inline] - fn from(v: $int_type) -> Self { Self::new(v) } - } + #[rustc_const_unstable(feature = "const_num_from_num", issue = "87852")] + impl const From<$int_type> for $atomic_type { + #[doc = concat!("Converts an `", stringify!($int_type), "` into an `", stringify!($atomic_type), "`.")] + #[inline] + fn from(v: $int_type) -> Self { Self::new(v) } } #[$stable_debug] impl fmt::Debug for $atomic_type { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&self.load(Ordering::SeqCst), f) + fmt::Debug::fmt(&self.load(Ordering::Relaxed), f) } } @@ -1313,801 +1940,794 @@ macro_rules! atomic_int { unsafe impl Sync for $atomic_type {} impl $atomic_type { - doc_comment! { - concat!("Creates a new atomic integer. - -# Examples - -``` -", $extra_feature, "use std::sync::atomic::", stringify!($atomic_type), "; - -let atomic_forty_two = ", stringify!($atomic_type), "::new(42); -```"), - #[inline] - #[$stable] - #[$const_stable] - pub const fn new(v: $int_type) -> Self { - Self { - v: UnsafeCell::new(v), - model: true, - } - } + /// Creates a new atomic integer. + /// + /// # Examples + /// + /// ``` + #[doc = concat!($extra_feature, "use std::sync::atomic::", stringify!($atomic_type), ";")] + /// + #[doc = concat!("let atomic_forty_two = ", stringify!($atomic_type), "::new(42);")] + /// ``` + #[inline] + #[$stable] + #[$const_stable] + #[must_use] + pub const fn new(v: $int_type) -> Self { + Self {v: UnsafeCell::new(v)} } - doc_comment! { - concat!("Creates a new atomic integer. - -# Examples - -``` -", $extra_feature, "use std::sync::atomic::", stringify!($atomic_type), "; - -let atomic_forty_two = ", stringify!($atomic_type), "::new(42); -```"), - #[inline] - #[$stable] - #[$const_stable] - pub const fn new_unmodeled(v: $int_type) -> Self { - let me = $atomic_type::new(v); - Self { model:false,..me } - } - } - - doc_comment! { - concat!("Returns a mutable reference to the underlying integer. - -This is safe because the mutable reference guarantees that no other threads are -concurrently accessing the atomic data. - -# Examples -``` -", $extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering}; - -let mut some_var = ", stringify!($atomic_type), "::new(10); -assert_eq!(*some_var.get_mut(), 10); -*some_var.get_mut() = 5; -assert_eq!(some_var.load(Ordering::SeqCst), 5); -```"), - #[inline] - #[$stable_access] - pub fn get_mut(&mut self) -> &mut $int_type { - // SAFETY: the mutable reference guarantees unique ownership. - unsafe { &mut *self.v.get() } - } + /// Returns a mutable reference to the underlying integer. + /// + /// This is safe because the mutable reference guarantees that no other threads are + /// concurrently accessing the atomic data. + /// + /// # Examples + /// + /// ``` + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let mut some_var = ", stringify!($atomic_type), "::new(10);")] + /// assert_eq!(*some_var.get_mut(), 10); + /// *some_var.get_mut() = 5; + /// assert_eq!(some_var.load(Ordering::SeqCst), 5); + /// ``` + #[inline] + #[$stable_access] + pub fn get_mut(&mut self) -> &mut $int_type { + self.v.get_mut() } - doc_comment! { - concat!("Consumes the atomic and returns the contained value. - -This is safe because passing `self` by value guarantees that no other threads are -concurrently accessing the atomic data. - -# Examples - -``` -", $extra_feature, "use std::sync::atomic::", stringify!($atomic_type), "; - -let some_var = ", stringify!($atomic_type), "::new(5); -assert_eq!(some_var.into_inner(), 5); -```"), - #[inline] - #[$stable_access] - pub fn into_inner(self) -> $int_type { - self.v.into_inner() - } + #[doc = concat!("Get atomic access to a `&mut ", stringify!($int_type), "`.")] + /// + #[doc = if_not_8_bit! { + $int_type, + concat!( + "**Note:** This function is only available on targets where `", + stringify!($int_type), "` has an alignment of ", $align, " bytes." + ) + }] + /// + /// # Examples + /// + /// ``` + /// #![feature(atomic_from_mut)] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + /// let mut some_int = 123; + #[doc = concat!("let a = ", stringify!($atomic_type), "::from_mut(&mut some_int);")] + /// a.store(100, Ordering::Relaxed); + /// assert_eq!(some_int, 100); + /// ``` + /// + #[inline] + #[$cfg_align] + #[unstable(feature = "atomic_from_mut", issue = "76314")] + pub fn from_mut(v: &mut $int_type) -> &mut Self { + use crate::mem::align_of; + let [] = [(); align_of::() - align_of::<$int_type>()]; + // SAFETY: + // - the mutable reference guarantees unique ownership. + // - the alignment of `$int_type` and `Self` is the + // same, as promised by $cfg_align and verified above. + unsafe { &mut *(v as *mut $int_type as *mut Self) } } - doc_comment! { - concat!("Loads a value from the atomic integer. - -`load` takes an [`Ordering`] argument which describes the memory ordering of this operation. -Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`]. - -# Panics - -Panics if `order` is [`Release`] or [`AcqRel`]. - -[`Ordering`]: enum.Ordering.html -[`Relaxed`]: enum.Ordering.html#variant.Relaxed -[`Release`]: enum.Ordering.html#variant.Release -[`Acquire`]: enum.Ordering.html#variant.Acquire -[`AcqRel`]: enum.Ordering.html#variant.AcqRel -[`SeqCst`]: enum.Ordering.html#variant.SeqCst - -# Examples - -``` -", $extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering}; - -let some_var = ", stringify!($atomic_type), "::new(5); - -assert_eq!(some_var.load(Ordering::Relaxed), 5); -```"), - #[inline] - #[$stable] - pub fn load(&self, order: Ordering) -> $int_type { - // SAFETY: data races are prevented by atomic intrinsics. - if self.model { - concurrency::sched_yield(true, self.v.get()); - } - unsafe { atomic_load(self.v.get(), order) } - } + #[doc = concat!("Get non-atomic access to a `&mut [", stringify!($atomic_type), "]` slice")] + /// + /// This is safe because the mutable reference guarantees that no other threads are + /// concurrently accessing the atomic data. + /// + /// # Examples + /// + /// ``` + /// #![feature(atomic_from_mut, inline_const)] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let mut some_ints = [const { ", stringify!($atomic_type), "::new(0) }; 10];")] + /// + #[doc = concat!("let view: &mut [", stringify!($int_type), "] = ", stringify!($atomic_type), "::get_mut_slice(&mut some_ints);")] + /// assert_eq!(view, [0; 10]); + /// view + /// .iter_mut() + /// .enumerate() + /// .for_each(|(idx, int)| *int = idx as _); + /// + /// std::thread::scope(|s| { + /// some_ints + /// .iter() + /// .enumerate() + /// .for_each(|(idx, int)| { + /// s.spawn(move || assert_eq!(int.load(Ordering::Relaxed), idx as _)); + /// }) + /// }); + /// ``` + #[inline] + #[unstable(feature = "atomic_from_mut", issue = "76314")] + pub fn get_mut_slice(this: &mut [Self]) -> &mut [$int_type] { + // SAFETY: the mutable reference guarantees unique ownership. + unsafe { &mut *(this as *mut [Self] as *mut [$int_type]) } } - doc_comment! { - concat!("Stores a value into the atomic integer. - -`store` takes an [`Ordering`] argument which describes the memory ordering of this operation. - Possible values are [`SeqCst`], [`Release`] and [`Relaxed`]. - -# Panics - -Panics if `order` is [`Acquire`] or [`AcqRel`]. - -[`Ordering`]: enum.Ordering.html -[`Relaxed`]: enum.Ordering.html#variant.Relaxed -[`Release`]: enum.Ordering.html#variant.Release -[`Acquire`]: enum.Ordering.html#variant.Acquire -[`AcqRel`]: enum.Ordering.html#variant.AcqRel -[`SeqCst`]: enum.Ordering.html#variant.SeqCst - -# Examples - -``` -", $extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering}; - -let some_var = ", stringify!($atomic_type), "::new(5); - -some_var.store(10, Ordering::Relaxed); -assert_eq!(some_var.load(Ordering::Relaxed), 10); -```"), - #[inline] - #[$stable] - pub fn store(&self, val: $int_type, order: Ordering) { - // SAFETY: data races are prevented by atomic intrinsics. - if self.model { - concurrency::sched_yield(false, self.v.get()); - } - unsafe { atomic_store(self.v.get(), val, order); } - } + #[doc = concat!("Get atomic access to a `&mut [", stringify!($int_type), "]` slice.")] + /// + /// # Examples + /// + /// ``` + /// #![feature(atomic_from_mut)] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + /// let mut some_ints = [0; 10]; + #[doc = concat!("let a = &*", stringify!($atomic_type), "::from_mut_slice(&mut some_ints);")] + /// std::thread::scope(|s| { + /// for i in 0..a.len() { + /// s.spawn(move || a[i].store(i as _, Ordering::Relaxed)); + /// } + /// }); + /// for (i, n) in some_ints.into_iter().enumerate() { + /// assert_eq!(i, n as usize); + /// } + /// ``` + #[inline] + #[$cfg_align] + #[unstable(feature = "atomic_from_mut", issue = "76314")] + pub fn from_mut_slice(v: &mut [$int_type]) -> &mut [Self] { + use crate::mem::align_of; + let [] = [(); align_of::() - align_of::<$int_type>()]; + // SAFETY: + // - the mutable reference guarantees unique ownership. + // - the alignment of `$int_type` and `Self` is the + // same, as promised by $cfg_align and verified above. + unsafe { &mut *(v as *mut [$int_type] as *mut [Self]) } } - doc_comment! { - concat!("Stores a value into the atomic integer, returning the previous value. - -`swap` takes an [`Ordering`] argument which describes the memory ordering -of this operation. All ordering modes are possible. Note that using -[`Acquire`] makes the store part of this operation [`Relaxed`], and -using [`Release`] makes the load part [`Relaxed`]. - -[`Ordering`]: enum.Ordering.html -[`Relaxed`]: enum.Ordering.html#variant.Relaxed -[`Release`]: enum.Ordering.html#variant.Release -[`Acquire`]: enum.Ordering.html#variant.Acquire - -# Examples - -``` -", $extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering}; - -let some_var = ", stringify!($atomic_type), "::new(5); - -assert_eq!(some_var.swap(10, Ordering::Relaxed), 5); -```"), - #[inline] - #[$stable] - #[$cfg_cas] - pub fn swap(&self, val: $int_type, order: Ordering) -> $int_type { - // SAFETY: data races are prevented by atomic intrinsics. - if self.model { - concurrency::sched_yield(false, self.v.get()); - } - unsafe { atomic_swap(self.v.get(), val, order) } - } + /// Consumes the atomic and returns the contained value. + /// + /// This is safe because passing `self` by value guarantees that no other threads are + /// concurrently accessing the atomic data. + /// + /// # Examples + /// + /// ``` + #[doc = concat!($extra_feature, "use std::sync::atomic::", stringify!($atomic_type), ";")] + /// + #[doc = concat!("let some_var = ", stringify!($atomic_type), "::new(5);")] + /// assert_eq!(some_var.into_inner(), 5); + /// ``` + #[inline] + #[$stable_access] + #[rustc_const_unstable(feature = "const_cell_into_inner", issue = "78729")] + pub const fn into_inner(self) -> $int_type { + self.v.into_inner() } - doc_comment! { - concat!("Stores a value into the atomic integer if the current value is the same as -the `current` value. - -The return value is always the previous value. If it is equal to `current`, then the -value was updated. - -`compare_and_swap` also takes an [`Ordering`] argument which describes the memory -ordering of this operation. Notice that even when using [`AcqRel`], the operation -might fail and hence just perform an `Acquire` load, but not have `Release` semantics. -Using [`Acquire`] makes the store part of this operation [`Relaxed`] if it -happens, and using [`Release`] makes the load part [`Relaxed`]. - -[`Ordering`]: enum.Ordering.html -[`Relaxed`]: enum.Ordering.html#variant.Relaxed -[`Release`]: enum.Ordering.html#variant.Release -[`Acquire`]: enum.Ordering.html#variant.Acquire -[`AcqRel`]: enum.Ordering.html#variant.AcqRel - -# Examples - -``` -", $extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering}; - -let some_var = ", stringify!($atomic_type), "::new(5); - -assert_eq!(some_var.compare_and_swap(5, 10, Ordering::Relaxed), 5); -assert_eq!(some_var.load(Ordering::Relaxed), 10); - -assert_eq!(some_var.compare_and_swap(6, 12, Ordering::Relaxed), 10); -assert_eq!(some_var.load(Ordering::Relaxed), 10); -```"), - #[inline] - #[$stable] - #[$cfg_cas] - pub fn compare_and_swap(&self, - current: $int_type, - new: $int_type, - order: Ordering) -> $int_type { - match self.compare_exchange(current, - new, - order, - strongest_failure_ordering(order)) { - Ok(x) => x, - Err(x) => x, - } - } + /// Loads a value from the atomic integer. + /// + /// `load` takes an [`Ordering`] argument which describes the memory ordering of this operation. + /// Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`]. + /// + /// # Panics + /// + /// Panics if `order` is [`Release`] or [`AcqRel`]. + /// + /// # Examples + /// + /// ``` + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let some_var = ", stringify!($atomic_type), "::new(5);")] + /// + /// assert_eq!(some_var.load(Ordering::Relaxed), 5); + /// ``` + #[inline] + #[$stable] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn load(&self, order: Ordering) -> $int_type { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_load(self.v.get(), order) } } - doc_comment! { - concat!("Stores a value into the atomic integer if the current value is the same as -the `current` value. - -The return value is a result indicating whether the new value was written and -containing the previous value. On success this value is guaranteed to be equal to -`current`. - -`compare_exchange` takes two [`Ordering`] arguments to describe the memory -ordering of this operation. The first describes the required ordering if the -operation succeeds while the second describes the required ordering when the -operation fails. Using [`Acquire`] as success ordering makes the store part -of this operation [`Relaxed`], and using [`Release`] makes the successful load -[`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] -and must be equivalent to or weaker than the success ordering. - -[`Ordering`]: enum.Ordering.html -[`Relaxed`]: enum.Ordering.html#variant.Relaxed -[`Release`]: enum.Ordering.html#variant.Release -[`Acquire`]: enum.Ordering.html#variant.Acquire -[`SeqCst`]: enum.Ordering.html#variant.SeqCst - -# Examples - -``` -", $extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering}; - -let some_var = ", stringify!($atomic_type), "::new(5); - -assert_eq!(some_var.compare_exchange(5, 10, - Ordering::Acquire, - Ordering::Relaxed), - Ok(5)); -assert_eq!(some_var.load(Ordering::Relaxed), 10); - -assert_eq!(some_var.compare_exchange(6, 12, - Ordering::SeqCst, - Ordering::Acquire), - Err(10)); -assert_eq!(some_var.load(Ordering::Relaxed), 10); -```"), - #[inline] - #[$stable_cxchg] - #[$cfg_cas] - pub fn compare_exchange(&self, - current: $int_type, - new: $int_type, - success: Ordering, - failure: Ordering) -> Result<$int_type, $int_type> { - // SAFETY: data races are prevented by atomic intrinsics. - if self.model { - concurrency::sched_yield(false, self.v.get()); - } - unsafe { atomic_compare_exchange(self.v.get(), current, new, success, failure) } - } + /// Stores a value into the atomic integer. + /// + /// `store` takes an [`Ordering`] argument which describes the memory ordering of this operation. + /// Possible values are [`SeqCst`], [`Release`] and [`Relaxed`]. + /// + /// # Panics + /// + /// Panics if `order` is [`Acquire`] or [`AcqRel`]. + /// + /// # Examples + /// + /// ``` + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let some_var = ", stringify!($atomic_type), "::new(5);")] + /// + /// some_var.store(10, Ordering::Relaxed); + /// assert_eq!(some_var.load(Ordering::Relaxed), 10); + /// ``` + #[inline] + #[$stable] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn store(&self, val: $int_type, order: Ordering) { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_store(self.v.get(), val, order); } } - doc_comment! { - concat!("Stores a value into the atomic integer if the current value is the same as -the `current` value. - -Unlike [`compare_exchange`], this function is allowed to spuriously fail even -when the comparison succeeds, which can result in more efficient code on some -platforms. The return value is a result indicating whether the new value was -written and containing the previous value. - -`compare_exchange_weak` takes two [`Ordering`] arguments to describe the memory -ordering of this operation. The first describes the required ordering if the -operation succeeds while the second describes the required ordering when the -operation fails. Using [`Acquire`] as success ordering makes the store part -of this operation [`Relaxed`], and using [`Release`] makes the successful load -[`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] -and must be equivalent to or weaker than the success ordering. - -[`compare_exchange`]: #method.compare_exchange -[`Ordering`]: enum.Ordering.html -[`Relaxed`]: enum.Ordering.html#variant.Relaxed -[`Release`]: enum.Ordering.html#variant.Release -[`Acquire`]: enum.Ordering.html#variant.Acquire -[`SeqCst`]: enum.Ordering.html#variant.SeqCst - -# Examples - -``` -", $extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering}; - -let val = ", stringify!($atomic_type), "::new(4); - -let mut old = val.load(Ordering::Relaxed); -loop { - let new = old * 2; - match val.compare_exchange_weak(old, new, Ordering::SeqCst, Ordering::Relaxed) { - Ok(_) => break, - Err(x) => old = x, - } -} -```"), - #[inline] - #[$stable_cxchg] - #[$cfg_cas] - pub fn compare_exchange_weak(&self, - current: $int_type, - new: $int_type, - success: Ordering, - failure: Ordering) -> Result<$int_type, $int_type> { - // SAFETY: data races are prevented by atomic intrinsics. - if self.model { - concurrency::sched_yield(false, self.v.get()); - } - unsafe { - atomic_compare_exchange_weak(self.v.get(), current, new, success, failure) - } - } + /// Stores a value into the atomic integer, returning the previous value. + /// + /// `swap` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + /// ``` + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let some_var = ", stringify!($atomic_type), "::new(5);")] + /// + /// assert_eq!(some_var.swap(10, Ordering::Relaxed), 5); + /// ``` + #[inline] + #[$stable] + #[$cfg_cas] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn swap(&self, val: $int_type, order: Ordering) -> $int_type { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_swap(self.v.get(), val, order) } } - doc_comment! { - concat!("Adds to the current value, returning the previous value. - -This operation wraps around on overflow. - -`fetch_add` takes an [`Ordering`] argument which describes the memory ordering -of this operation. All ordering modes are possible. Note that using -[`Acquire`] makes the store part of this operation [`Relaxed`], and -using [`Release`] makes the load part [`Relaxed`]. - -[`Ordering`]: enum.Ordering.html -[`Relaxed`]: enum.Ordering.html#variant.Relaxed -[`Release`]: enum.Ordering.html#variant.Release -[`Acquire`]: enum.Ordering.html#variant.Acquire - -# Examples - -``` -", $extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering}; - -let foo = ", stringify!($atomic_type), "::new(0); -assert_eq!(foo.fetch_add(10, Ordering::SeqCst), 0); -assert_eq!(foo.load(Ordering::SeqCst), 10); -```"), - #[inline] - #[$stable] - #[$cfg_cas] - pub fn fetch_add(&self, val: $int_type, order: Ordering) -> $int_type { - // SAFETY: data races are prevented by atomic intrinsics. - if self.model { - concurrency::sched_yield(false, self.v.get()); - } - unsafe { atomic_add(self.v.get(), val, order) } + /// Stores a value into the atomic integer if the current value is the same as + /// the `current` value. + /// + /// The return value is always the previous value. If it is equal to `current`, then the + /// value was updated. + /// + /// `compare_and_swap` also takes an [`Ordering`] argument which describes the memory + /// ordering of this operation. Notice that even when using [`AcqRel`], the operation + /// might fail and hence just perform an `Acquire` load, but not have `Release` semantics. + /// Using [`Acquire`] makes the store part of this operation [`Relaxed`] if it + /// happens, and using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Migrating to `compare_exchange` and `compare_exchange_weak` + /// + /// `compare_and_swap` is equivalent to `compare_exchange` with the following mapping for + /// memory orderings: + /// + /// Original | Success | Failure + /// -------- | ------- | ------- + /// Relaxed | Relaxed | Relaxed + /// Acquire | Acquire | Acquire + /// Release | Release | Relaxed + /// AcqRel | AcqRel | Acquire + /// SeqCst | SeqCst | SeqCst + /// + /// `compare_exchange_weak` is allowed to fail spuriously even when the comparison succeeds, + /// which allows the compiler to generate better assembly code when the compare and swap + /// is used in a loop. + /// + /// # Examples + /// + /// ``` + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let some_var = ", stringify!($atomic_type), "::new(5);")] + /// + /// assert_eq!(some_var.compare_and_swap(5, 10, Ordering::Relaxed), 5); + /// assert_eq!(some_var.load(Ordering::Relaxed), 10); + /// + /// assert_eq!(some_var.compare_and_swap(6, 12, Ordering::Relaxed), 10); + /// assert_eq!(some_var.load(Ordering::Relaxed), 10); + /// ``` + #[inline] + #[$stable] + #[deprecated( + since = "1.50.0", + note = "Use `compare_exchange` or `compare_exchange_weak` instead") + ] + #[$cfg_cas] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn compare_and_swap(&self, + current: $int_type, + new: $int_type, + order: Ordering) -> $int_type { + match self.compare_exchange(current, + new, + order, + strongest_failure_ordering(order)) { + Ok(x) => x, + Err(x) => x, } } - doc_comment! { - concat!("Subtracts from the current value, returning the previous value. - -This operation wraps around on overflow. - -`fetch_sub` takes an [`Ordering`] argument which describes the memory ordering -of this operation. All ordering modes are possible. Note that using -[`Acquire`] makes the store part of this operation [`Relaxed`], and -using [`Release`] makes the load part [`Relaxed`]. - -[`Ordering`]: enum.Ordering.html -[`Relaxed`]: enum.Ordering.html#variant.Relaxed -[`Release`]: enum.Ordering.html#variant.Release -[`Acquire`]: enum.Ordering.html#variant.Acquire - -# Examples - -``` -", $extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering}; - -let foo = ", stringify!($atomic_type), "::new(20); -assert_eq!(foo.fetch_sub(10, Ordering::SeqCst), 20); -assert_eq!(foo.load(Ordering::SeqCst), 10); -```"), - #[inline] - #[$stable] - #[$cfg_cas] - pub fn fetch_sub(&self, val: $int_type, order: Ordering) -> $int_type { - // SAFETY: data races are prevented by atomic intrinsics. - if self.model { - concurrency::sched_yield(false, self.v.get()); - } - unsafe { atomic_sub(self.v.get(), val, order) } - } + /// Stores a value into the atomic integer if the current value is the same as + /// the `current` value. + /// + /// The return value is a result indicating whether the new value was written and + /// containing the previous value. On success this value is guaranteed to be equal to + /// `current`. + /// + /// `compare_exchange` takes two [`Ordering`] arguments to describe the memory + /// ordering of this operation. `success` describes the required ordering for the + /// read-modify-write operation that takes place if the comparison with `current` succeeds. + /// `failure` describes the required ordering for the load operation that takes place when + /// the comparison fails. Using [`Acquire`] as success ordering makes the store part + /// of this operation [`Relaxed`], and using [`Release`] makes the successful load + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + /// ``` + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let some_var = ", stringify!($atomic_type), "::new(5);")] + /// + /// assert_eq!(some_var.compare_exchange(5, 10, + /// Ordering::Acquire, + /// Ordering::Relaxed), + /// Ok(5)); + /// assert_eq!(some_var.load(Ordering::Relaxed), 10); + /// + /// assert_eq!(some_var.compare_exchange(6, 12, + /// Ordering::SeqCst, + /// Ordering::Acquire), + /// Err(10)); + /// assert_eq!(some_var.load(Ordering::Relaxed), 10); + /// ``` + #[inline] + #[$stable_cxchg] + #[$cfg_cas] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn compare_exchange(&self, + current: $int_type, + new: $int_type, + success: Ordering, + failure: Ordering) -> Result<$int_type, $int_type> { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_compare_exchange(self.v.get(), current, new, success, failure) } } - doc_comment! { - concat!("Bitwise \"and\" with the current value. - -Performs a bitwise \"and\" operation on the current value and the argument `val`, and -sets the new value to the result. - -Returns the previous value. - -`fetch_and` takes an [`Ordering`] argument which describes the memory ordering -of this operation. All ordering modes are possible. Note that using -[`Acquire`] makes the store part of this operation [`Relaxed`], and -using [`Release`] makes the load part [`Relaxed`]. - -[`Ordering`]: enum.Ordering.html -[`Relaxed`]: enum.Ordering.html#variant.Relaxed -[`Release`]: enum.Ordering.html#variant.Release -[`Acquire`]: enum.Ordering.html#variant.Acquire - -# Examples - -``` -", $extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering}; - -let foo = ", stringify!($atomic_type), "::new(0b101101); -assert_eq!(foo.fetch_and(0b110011, Ordering::SeqCst), 0b101101); -assert_eq!(foo.load(Ordering::SeqCst), 0b100001); -```"), - #[inline] - #[$stable] - #[$cfg_cas] - pub fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type { - // SAFETY: data races are prevented by atomic intrinsics. - if self.model { - concurrency::sched_yield(false, self.v.get()); - } - unsafe { atomic_and(self.v.get(), val, order) } + /// Stores a value into the atomic integer if the current value is the same as + /// the `current` value. + /// + #[doc = concat!("Unlike [`", stringify!($atomic_type), "::compare_exchange`],")] + /// this function is allowed to spuriously fail even + /// when the comparison succeeds, which can result in more efficient code on some + /// platforms. The return value is a result indicating whether the new value was + /// written and containing the previous value. + /// + /// `compare_exchange_weak` takes two [`Ordering`] arguments to describe the memory + /// ordering of this operation. `success` describes the required ordering for the + /// read-modify-write operation that takes place if the comparison with `current` succeeds. + /// `failure` describes the required ordering for the load operation that takes place when + /// the comparison fails. Using [`Acquire`] as success ordering makes the store part + /// of this operation [`Relaxed`], and using [`Release`] makes the successful load + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + /// ``` + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let val = ", stringify!($atomic_type), "::new(4);")] + /// + /// let mut old = val.load(Ordering::Relaxed); + /// loop { + /// let new = old * 2; + /// match val.compare_exchange_weak(old, new, Ordering::SeqCst, Ordering::Relaxed) { + /// Ok(_) => break, + /// Err(x) => old = x, + /// } + /// } + /// ``` + #[inline] + #[$stable_cxchg] + #[$cfg_cas] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn compare_exchange_weak(&self, + current: $int_type, + new: $int_type, + success: Ordering, + failure: Ordering) -> Result<$int_type, $int_type> { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { + atomic_compare_exchange_weak(self.v.get(), current, new, success, failure) } } - doc_comment! { - concat!("Bitwise \"nand\" with the current value. - -Performs a bitwise \"nand\" operation on the current value and the argument `val`, and -sets the new value to the result. - -Returns the previous value. - -`fetch_nand` takes an [`Ordering`] argument which describes the memory ordering -of this operation. All ordering modes are possible. Note that using -[`Acquire`] makes the store part of this operation [`Relaxed`], and -using [`Release`] makes the load part [`Relaxed`]. - -[`Ordering`]: enum.Ordering.html -[`Relaxed`]: enum.Ordering.html#variant.Relaxed -[`Release`]: enum.Ordering.html#variant.Release -[`Acquire`]: enum.Ordering.html#variant.Acquire - -# Examples - -``` -", $extra_feature, " -use std::sync::atomic::{", stringify!($atomic_type), ", Ordering}; - -let foo = ", stringify!($atomic_type), "::new(0x13); -assert_eq!(foo.fetch_nand(0x31, Ordering::SeqCst), 0x13); -assert_eq!(foo.load(Ordering::SeqCst), !(0x13 & 0x31)); -```"), - #[inline] - #[$stable_nand] - #[$cfg_cas] - pub fn fetch_nand(&self, val: $int_type, order: Ordering) -> $int_type { - // SAFETY: data races are prevented by atomic intrinsics. - if self.model { - concurrency::sched_yield(false, self.v.get()); - } - unsafe { atomic_nand(self.v.get(), val, order) } - } + /// Adds to the current value, returning the previous value. + /// + /// This operation wraps around on overflow. + /// + /// `fetch_add` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + /// ``` + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(0);")] + /// assert_eq!(foo.fetch_add(10, Ordering::SeqCst), 0); + /// assert_eq!(foo.load(Ordering::SeqCst), 10); + /// ``` + #[inline] + #[$stable] + #[$cfg_cas] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn fetch_add(&self, val: $int_type, order: Ordering) -> $int_type { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_add(self.v.get(), val, order) } } - doc_comment! { - concat!("Bitwise \"or\" with the current value. - -Performs a bitwise \"or\" operation on the current value and the argument `val`, and -sets the new value to the result. - -Returns the previous value. - -`fetch_or` takes an [`Ordering`] argument which describes the memory ordering -of this operation. All ordering modes are possible. Note that using -[`Acquire`] makes the store part of this operation [`Relaxed`], and -using [`Release`] makes the load part [`Relaxed`]. - -[`Ordering`]: enum.Ordering.html -[`Relaxed`]: enum.Ordering.html#variant.Relaxed -[`Release`]: enum.Ordering.html#variant.Release -[`Acquire`]: enum.Ordering.html#variant.Acquire - -# Examples - -``` -", $extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering}; - -let foo = ", stringify!($atomic_type), "::new(0b101101); -assert_eq!(foo.fetch_or(0b110011, Ordering::SeqCst), 0b101101); -assert_eq!(foo.load(Ordering::SeqCst), 0b111111); -```"), - #[inline] - #[$stable] - #[$cfg_cas] - pub fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type { - // SAFETY: data races are prevented by atomic intrinsics. - if self.model { - concurrency::sched_yield(false, self.v.get()); - } - unsafe { atomic_or(self.v.get(), val, order) } - } + /// Subtracts from the current value, returning the previous value. + /// + /// This operation wraps around on overflow. + /// + /// `fetch_sub` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + /// ``` + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(20);")] + /// assert_eq!(foo.fetch_sub(10, Ordering::SeqCst), 20); + /// assert_eq!(foo.load(Ordering::SeqCst), 10); + /// ``` + #[inline] + #[$stable] + #[$cfg_cas] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn fetch_sub(&self, val: $int_type, order: Ordering) -> $int_type { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_sub(self.v.get(), val, order) } } - doc_comment! { - concat!("Bitwise \"xor\" with the current value. - -Performs a bitwise \"xor\" operation on the current value and the argument `val`, and -sets the new value to the result. - -Returns the previous value. - -`fetch_xor` takes an [`Ordering`] argument which describes the memory ordering -of this operation. All ordering modes are possible. Note that using -[`Acquire`] makes the store part of this operation [`Relaxed`], and -using [`Release`] makes the load part [`Relaxed`]. - -[`Ordering`]: enum.Ordering.html -[`Relaxed`]: enum.Ordering.html#variant.Relaxed -[`Release`]: enum.Ordering.html#variant.Release -[`Acquire`]: enum.Ordering.html#variant.Acquire - -# Examples - -``` -", $extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering}; + /// Bitwise "and" with the current value. + /// + /// Performs a bitwise "and" operation on the current value and the argument `val`, and + /// sets the new value to the result. + /// + /// Returns the previous value. + /// + /// `fetch_and` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + /// ``` + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(0b101101);")] + /// assert_eq!(foo.fetch_and(0b110011, Ordering::SeqCst), 0b101101); + /// assert_eq!(foo.load(Ordering::SeqCst), 0b100001); + /// ``` + #[inline] + #[$stable] + #[$cfg_cas] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_and(self.v.get(), val, order) } + } -let foo = ", stringify!($atomic_type), "::new(0b101101); -assert_eq!(foo.fetch_xor(0b110011, Ordering::SeqCst), 0b101101); -assert_eq!(foo.load(Ordering::SeqCst), 0b011110); -```"), - #[inline] - #[$stable] - #[$cfg_cas] - pub fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type { - // SAFETY: data races are prevented by atomic intrinsics. - if self.model { - concurrency::sched_yield(false, self.v.get()); - } - unsafe { atomic_xor(self.v.get(), val, order) } - } + /// Bitwise "nand" with the current value. + /// + /// Performs a bitwise "nand" operation on the current value and the argument `val`, and + /// sets the new value to the result. + /// + /// Returns the previous value. + /// + /// `fetch_nand` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + /// ``` + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(0x13);")] + /// assert_eq!(foo.fetch_nand(0x31, Ordering::SeqCst), 0x13); + /// assert_eq!(foo.load(Ordering::SeqCst), !(0x13 & 0x31)); + /// ``` + #[inline] + #[$stable_nand] + #[$cfg_cas] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn fetch_nand(&self, val: $int_type, order: Ordering) -> $int_type { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_nand(self.v.get(), val, order) } } - doc_comment! { - concat!("Fetches the value, and applies a function to it that returns an optional -new value. Returns a `Result` of `Ok(previous_value)` if the function returned `Some(_)`, else -`Err(previous_value)`. - -Note: This may call the function multiple times if the value has been changed from other threads in -the meantime, as long as the function returns `Some(_)`, but the function will have been applied -but once to the stored value. - -`fetch_update` takes two [`Ordering`] arguments to describe the memory -ordering of this operation. The first describes the required ordering for loads -and failed updates while the second describes the required ordering when the -operation finally succeeds. Beware that this is different from the two -modes in [`compare_exchange`]! - -Using [`Acquire`] as success ordering makes the store part -of this operation [`Relaxed`], and using [`Release`] makes the final successful load -[`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] -and must be equivalent to or weaker than the success ordering. - -[`bool`]: ../../../std/primitive.bool.html -[`compare_exchange`]: #method.compare_exchange -[`Ordering`]: enum.Ordering.html -[`Relaxed`]: enum.Ordering.html#variant.Relaxed -[`Release`]: enum.Ordering.html#variant.Release -[`Acquire`]: enum.Ordering.html#variant.Acquire -[`SeqCst`]: enum.Ordering.html#variant.SeqCst - -# Examples - -```rust -#![feature(no_more_cas)] -", $extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering}; - -let x = ", stringify!($atomic_type), "::new(7); -assert_eq!(x.fetch_update(|_| None, Ordering::SeqCst, Ordering::SeqCst), Err(7)); -assert_eq!(x.fetch_update(|x| Some(x + 1), Ordering::SeqCst, Ordering::SeqCst), Ok(7)); -assert_eq!(x.fetch_update(|x| Some(x + 1), Ordering::SeqCst, Ordering::SeqCst), Ok(8)); -assert_eq!(x.load(Ordering::SeqCst), 9); -```"), - #[inline] - #[unstable(feature = "no_more_cas", - reason = "no more CAS loops in user code", - issue = "48655")] - #[$cfg_cas] - pub fn fetch_update(&self, - mut f: F, - fetch_order: Ordering, - set_order: Ordering) -> Result<$int_type, $int_type> - where F: FnMut($int_type) -> Option<$int_type> { - let mut prev = self.load(fetch_order); - while let Some(next) = f(prev) { - match self.compare_exchange_weak(prev, next, set_order, fetch_order) { - x @ Ok(_) => return x, - Err(next_prev) => prev = next_prev - } - } - Err(prev) - } + /// Bitwise "or" with the current value. + /// + /// Performs a bitwise "or" operation on the current value and the argument `val`, and + /// sets the new value to the result. + /// + /// Returns the previous value. + /// + /// `fetch_or` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + /// ``` + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(0b101101);")] + /// assert_eq!(foo.fetch_or(0b110011, Ordering::SeqCst), 0b101101); + /// assert_eq!(foo.load(Ordering::SeqCst), 0b111111); + /// ``` + #[inline] + #[$stable] + #[$cfg_cas] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_or(self.v.get(), val, order) } } - doc_comment! { - concat!("Maximum with the current value. - -Finds the maximum of the current value and the argument `val`, and -sets the new value to the result. - -Returns the previous value. - -`fetch_max` takes an [`Ordering`] argument which describes the memory ordering -of this operation. All ordering modes are possible. Note that using -[`Acquire`] makes the store part of this operation [`Relaxed`], and -using [`Release`] makes the load part [`Relaxed`]. - -[`Ordering`]: enum.Ordering.html -[`Relaxed`]: enum.Ordering.html#variant.Relaxed -[`Release`]: enum.Ordering.html#variant.Release -[`Acquire`]: enum.Ordering.html#variant.Acquire - -# Examples - -``` -#![feature(atomic_min_max)] -", $extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering}; - -let foo = ", stringify!($atomic_type), "::new(23); -assert_eq!(foo.fetch_max(42, Ordering::SeqCst), 23); -assert_eq!(foo.load(Ordering::SeqCst), 42); -``` - -If you want to obtain the maximum value in one step, you can use the following: - -``` -#![feature(atomic_min_max)] -", $extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering}; - -let foo = ", stringify!($atomic_type), "::new(23); -let bar = 42; -let max_foo = foo.fetch_max(bar, Ordering::SeqCst).max(bar); -assert!(max_foo == 42); -```"), - #[inline] - #[unstable(feature = "atomic_min_max", - reason = "easier and faster min/max than writing manual CAS loop", - issue = "48655")] - #[$cfg_cas] - pub fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type { - // SAFETY: data races are prevented by atomic intrinsics. - if self.model { - concurrency::sched_yield(false, self.v.get()); - } - unsafe { $max_fn(self.v.get(), val, order) } - } + /// Bitwise "xor" with the current value. + /// + /// Performs a bitwise "xor" operation on the current value and the argument `val`, and + /// sets the new value to the result. + /// + /// Returns the previous value. + /// + /// `fetch_xor` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + /// ``` + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(0b101101);")] + /// assert_eq!(foo.fetch_xor(0b110011, Ordering::SeqCst), 0b101101); + /// assert_eq!(foo.load(Ordering::SeqCst), 0b011110); + /// ``` + #[inline] + #[$stable] + #[$cfg_cas] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_xor(self.v.get(), val, order) } } - doc_comment! { - concat!("Minimum with the current value. - -Finds the minimum of the current value and the argument `val`, and -sets the new value to the result. - -Returns the previous value. - -`fetch_min` takes an [`Ordering`] argument which describes the memory ordering -of this operation. All ordering modes are possible. Note that using -[`Acquire`] makes the store part of this operation [`Relaxed`], and -using [`Release`] makes the load part [`Relaxed`]. - -[`Ordering`]: enum.Ordering.html -[`Relaxed`]: enum.Ordering.html#variant.Relaxed -[`Release`]: enum.Ordering.html#variant.Release -[`Acquire`]: enum.Ordering.html#variant.Acquire - -# Examples - -``` -#![feature(atomic_min_max)] -", $extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering}; - -let foo = ", stringify!($atomic_type), "::new(23); -assert_eq!(foo.fetch_min(42, Ordering::Relaxed), 23); -assert_eq!(foo.load(Ordering::Relaxed), 23); -assert_eq!(foo.fetch_min(22, Ordering::Relaxed), 23); -assert_eq!(foo.load(Ordering::Relaxed), 22); -``` - -If you want to obtain the minimum value in one step, you can use the following: - -``` -#![feature(atomic_min_max)] -", $extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering}; - -let foo = ", stringify!($atomic_type), "::new(23); -let bar = 12; -let min_foo = foo.fetch_min(bar, Ordering::SeqCst).min(bar); -assert_eq!(min_foo, 12); -```"), - #[inline] - #[unstable(feature = "atomic_min_max", - reason = "easier and faster min/max than writing manual CAS loop", - issue = "48655")] - #[$cfg_cas] - pub fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type { - // SAFETY: data races are prevented by atomic intrinsics. - if self.model { - concurrency::sched_yield(false, self.v.get()); + /// Fetches the value, and applies a function to it that returns an optional + /// new value. Returns a `Result` of `Ok(previous_value)` if the function returned `Some(_)`, else + /// `Err(previous_value)`. + /// + /// Note: This may call the function multiple times if the value has been changed from other threads in + /// the meantime, as long as the function returns `Some(_)`, but the function will have been applied + /// only once to the stored value. + /// + /// `fetch_update` takes two [`Ordering`] arguments to describe the memory ordering of this operation. + /// The first describes the required ordering for when the operation finally succeeds while the second + /// describes the required ordering for loads. These correspond to the success and failure orderings of + #[doc = concat!("[`", stringify!($atomic_type), "::compare_exchange`]")] + /// respectively. + /// + /// Using [`Acquire`] as success ordering makes the store part + /// of this operation [`Relaxed`], and using [`Release`] makes the final successful load + /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Considerations + /// + /// This method is not magic; it is not provided by the hardware. + /// It is implemented in terms of + #[doc = concat!("[`", stringify!($atomic_type), "::compare_exchange_weak`],")] + /// and suffers from the same drawbacks. + /// In particular, this method will not circumvent the [ABA Problem]. + /// + /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem + /// + /// # Examples + /// + /// ```rust + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let x = ", stringify!($atomic_type), "::new(7);")] + /// assert_eq!(x.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |_| None), Err(7)); + /// assert_eq!(x.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |x| Some(x + 1)), Ok(7)); + /// assert_eq!(x.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |x| Some(x + 1)), Ok(8)); + /// assert_eq!(x.load(Ordering::SeqCst), 9); + /// ``` + #[inline] + #[stable(feature = "no_more_cas", since = "1.45.0")] + #[$cfg_cas] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn fetch_update(&self, + set_order: Ordering, + fetch_order: Ordering, + mut f: F) -> Result<$int_type, $int_type> + where F: FnMut($int_type) -> Option<$int_type> { + let mut prev = self.load(fetch_order); + while let Some(next) = f(prev) { + match self.compare_exchange_weak(prev, next, set_order, fetch_order) { + x @ Ok(_) => return x, + Err(next_prev) => prev = next_prev } - unsafe { $min_fn(self.v.get(), val, order) } } + Err(prev) } - doc_comment! { - concat!("Returns a mutable pointer to the underlying integer. - -Doing non-atomic reads and writes on the resulting integer can be a data race. -This method is mostly useful for FFI, where the function signature may use -`*mut ", stringify!($int_type), "` instead of `&", stringify!($atomic_type), "`. - -Returning an `*mut` pointer from a shared reference to this atomic is safe because the -atomic types work with interior mutability. All modifications of an atomic change the value -through a shared reference, and can do so safely as long as they use atomic operations. Any -use of the returned raw pointer requires an `unsafe` block and still has to uphold the same -restriction: operations on it must be atomic. - -# Examples - -```ignore (extern-declaration) -# fn main() { -", $extra_feature, "use std::sync::atomic::", stringify!($atomic_type), "; + /// Maximum with the current value. + /// + /// Finds the maximum of the current value and the argument `val`, and + /// sets the new value to the result. + /// + /// Returns the previous value. + /// + /// `fetch_max` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + /// ``` + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(23);")] + /// assert_eq!(foo.fetch_max(42, Ordering::SeqCst), 23); + /// assert_eq!(foo.load(Ordering::SeqCst), 42); + /// ``` + /// + /// If you want to obtain the maximum value in one step, you can use the following: + /// + /// ``` + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(23);")] + /// let bar = 42; + /// let max_foo = foo.fetch_max(bar, Ordering::SeqCst).max(bar); + /// assert!(max_foo == 42); + /// ``` + #[inline] + #[stable(feature = "atomic_min_max", since = "1.45.0")] + #[$cfg_cas] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { $max_fn(self.v.get(), val, order) } + } -extern { - fn my_atomic_op(arg: *mut ", stringify!($int_type), "); -} + /// Minimum with the current value. + /// + /// Finds the minimum of the current value and the argument `val`, and + /// sets the new value to the result. + /// + /// Returns the previous value. + /// + /// `fetch_min` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + /// ``` + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(23);")] + /// assert_eq!(foo.fetch_min(42, Ordering::Relaxed), 23); + /// assert_eq!(foo.load(Ordering::Relaxed), 23); + /// assert_eq!(foo.fetch_min(22, Ordering::Relaxed), 23); + /// assert_eq!(foo.load(Ordering::Relaxed), 22); + /// ``` + /// + /// If you want to obtain the minimum value in one step, you can use the following: + /// + /// ``` + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(23);")] + /// let bar = 12; + /// let min_foo = foo.fetch_min(bar, Ordering::SeqCst).min(bar); + /// assert_eq!(min_foo, 12); + /// ``` + #[inline] + #[stable(feature = "atomic_min_max", since = "1.45.0")] + #[$cfg_cas] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { $min_fn(self.v.get(), val, order) } + } -let mut atomic = ", stringify!($atomic_type), "::new(1); -", -// SAFETY: Safe as long as `my_atomic_op` is atomic. -"unsafe { - my_atomic_op(atomic.as_mut_ptr()); -} -# } -```"), - #[inline] - #[unstable(feature = "atomic_mut_ptr", - reason = "recently added", - issue = "66893")] - pub fn as_mut_ptr(&self) -> *mut $int_type { - self.v.get() - } + /// Returns a mutable pointer to the underlying integer. + /// + /// Doing non-atomic reads and writes on the resulting integer can be a data race. + /// This method is mostly useful for FFI, where the function signature may use + #[doc = concat!("`*mut ", stringify!($int_type), "` instead of `&", stringify!($atomic_type), "`.")] + /// + /// Returning an `*mut` pointer from a shared reference to this atomic is safe because the + /// atomic types work with interior mutability. All modifications of an atomic change the value + /// through a shared reference, and can do so safely as long as they use atomic operations. Any + /// use of the returned raw pointer requires an `unsafe` block and still has to uphold the same + /// restriction: operations on it must be atomic. + /// + /// # Examples + /// + /// ```ignore (extern-declaration) + /// # fn main() { + #[doc = concat!($extra_feature, "use std::sync::atomic::", stringify!($atomic_type), ";")] + /// + /// extern "C" { + #[doc = concat!(" fn my_atomic_op(arg: *mut ", stringify!($int_type), ");")] + /// } + /// + #[doc = concat!("let atomic = ", stringify!($atomic_type), "::new(1);")] + /// + /// // SAFETY: Safe as long as `my_atomic_op` is atomic. + /// unsafe { + /// my_atomic_op(atomic.as_mut_ptr()); + /// } + /// # } + /// ``` + #[inline] + #[unstable(feature = "atomic_mut_ptr", + reason = "recently added", + issue = "66893")] + pub fn as_mut_ptr(&self) -> *mut $int_type { + self.v.get() } } } @@ -2116,6 +2736,7 @@ let mut atomic = ", stringify!($atomic_type), "::new(1); #[cfg(target_has_atomic_load_store = "8")] atomic_int! { cfg(target_has_atomic = "8"), + cfg(target_has_atomic_equal_alignment = "8"), stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), @@ -2123,8 +2744,9 @@ atomic_int! { stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - unstable(feature = "integer_atomics", issue = "32976"), - "i8", "../../../std/primitive.i8.html", + unstable(feature = "integer_atomics", issue = "99069"), + cfg_attr(not(test), rustc_diagnostic_item = "AtomicI8"), + "i8", "", atomic_min, atomic_max, 1, @@ -2134,6 +2756,7 @@ atomic_int! { #[cfg(target_has_atomic_load_store = "8")] atomic_int! { cfg(target_has_atomic = "8"), + cfg(target_has_atomic_equal_alignment = "8"), stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), @@ -2141,8 +2764,9 @@ atomic_int! { stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - unstable(feature = "integer_atomics", issue = "32976"), - "u8", "../../../std/primitive.u8.html", + unstable(feature = "integer_atomics", issue = "99069"), + cfg_attr(not(test), rustc_diagnostic_item = "AtomicU8"), + "u8", "", atomic_umin, atomic_umax, 1, @@ -2152,6 +2776,7 @@ atomic_int! { #[cfg(target_has_atomic_load_store = "16")] atomic_int! { cfg(target_has_atomic = "16"), + cfg(target_has_atomic_equal_alignment = "16"), stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), @@ -2159,8 +2784,9 @@ atomic_int! { stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - unstable(feature = "integer_atomics", issue = "32976"), - "i16", "../../../std/primitive.i16.html", + unstable(feature = "integer_atomics", issue = "99069"), + cfg_attr(not(test), rustc_diagnostic_item = "AtomicI16"), + "i16", "", atomic_min, atomic_max, 2, @@ -2170,6 +2796,7 @@ atomic_int! { #[cfg(target_has_atomic_load_store = "16")] atomic_int! { cfg(target_has_atomic = "16"), + cfg(target_has_atomic_equal_alignment = "16"), stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), @@ -2177,8 +2804,9 @@ atomic_int! { stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - unstable(feature = "integer_atomics", issue = "32976"), - "u16", "../../../std/primitive.u16.html", + unstable(feature = "integer_atomics", issue = "99069"), + cfg_attr(not(test), rustc_diagnostic_item = "AtomicU16"), + "u16", "", atomic_umin, atomic_umax, 2, @@ -2188,6 +2816,7 @@ atomic_int! { #[cfg(target_has_atomic_load_store = "32")] atomic_int! { cfg(target_has_atomic = "32"), + cfg(target_has_atomic_equal_alignment = "32"), stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), @@ -2195,8 +2824,9 @@ atomic_int! { stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - unstable(feature = "integer_atomics", issue = "32976"), - "i32", "../../../std/primitive.i32.html", + unstable(feature = "integer_atomics", issue = "99069"), + cfg_attr(not(test), rustc_diagnostic_item = "AtomicI32"), + "i32", "", atomic_min, atomic_max, 4, @@ -2206,6 +2836,7 @@ atomic_int! { #[cfg(target_has_atomic_load_store = "32")] atomic_int! { cfg(target_has_atomic = "32"), + cfg(target_has_atomic_equal_alignment = "32"), stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), @@ -2213,8 +2844,9 @@ atomic_int! { stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - unstable(feature = "integer_atomics", issue = "32976"), - "u32", "../../../std/primitive.u32.html", + unstable(feature = "integer_atomics", issue = "99069"), + cfg_attr(not(test), rustc_diagnostic_item = "AtomicU32"), + "u32", "", atomic_umin, atomic_umax, 4, @@ -2224,6 +2856,7 @@ atomic_int! { #[cfg(target_has_atomic_load_store = "64")] atomic_int! { cfg(target_has_atomic = "64"), + cfg(target_has_atomic_equal_alignment = "64"), stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), @@ -2231,8 +2864,9 @@ atomic_int! { stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - unstable(feature = "integer_atomics", issue = "32976"), - "i64", "../../../std/primitive.i64.html", + unstable(feature = "integer_atomics", issue = "99069"), + cfg_attr(not(test), rustc_diagnostic_item = "AtomicI64"), + "i64", "", atomic_min, atomic_max, 8, @@ -2242,6 +2876,7 @@ atomic_int! { #[cfg(target_has_atomic_load_store = "64")] atomic_int! { cfg(target_has_atomic = "64"), + cfg(target_has_atomic_equal_alignment = "64"), stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), @@ -2249,8 +2884,9 @@ atomic_int! { stable(feature = "integer_atomics_stable", since = "1.34.0"), stable(feature = "integer_atomics_stable", since = "1.34.0"), rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - unstable(feature = "integer_atomics", issue = "32976"), - "u64", "../../../std/primitive.u64.html", + unstable(feature = "integer_atomics", issue = "99069"), + cfg_attr(not(test), rustc_diagnostic_item = "AtomicU64"), + "u64", "", atomic_umin, atomic_umax, 8, @@ -2260,15 +2896,17 @@ atomic_int! { #[cfg(target_has_atomic_load_store = "128")] atomic_int! { cfg(target_has_atomic = "128"), - unstable(feature = "integer_atomics", issue = "32976"), - unstable(feature = "integer_atomics", issue = "32976"), - unstable(feature = "integer_atomics", issue = "32976"), - unstable(feature = "integer_atomics", issue = "32976"), - unstable(feature = "integer_atomics", issue = "32976"), - unstable(feature = "integer_atomics", issue = "32976"), + cfg(target_has_atomic_equal_alignment = "128"), + unstable(feature = "integer_atomics", issue = "99069"), + unstable(feature = "integer_atomics", issue = "99069"), + unstable(feature = "integer_atomics", issue = "99069"), + unstable(feature = "integer_atomics", issue = "99069"), + unstable(feature = "integer_atomics", issue = "99069"), + unstable(feature = "integer_atomics", issue = "99069"), rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - unstable(feature = "integer_atomics", issue = "32976"), - "i128", "../../../std/primitive.i128.html", + unstable(feature = "integer_atomics", issue = "99069"), + cfg_attr(not(test), rustc_diagnostic_item = "AtomicI128"), + "i128", "#![feature(integer_atomics)]\n\n", atomic_min, atomic_max, 16, @@ -2278,77 +2916,75 @@ atomic_int! { #[cfg(target_has_atomic_load_store = "128")] atomic_int! { cfg(target_has_atomic = "128"), - unstable(feature = "integer_atomics", issue = "32976"), - unstable(feature = "integer_atomics", issue = "32976"), - unstable(feature = "integer_atomics", issue = "32976"), - unstable(feature = "integer_atomics", issue = "32976"), - unstable(feature = "integer_atomics", issue = "32976"), - unstable(feature = "integer_atomics", issue = "32976"), + cfg(target_has_atomic_equal_alignment = "128"), + unstable(feature = "integer_atomics", issue = "99069"), + unstable(feature = "integer_atomics", issue = "99069"), + unstable(feature = "integer_atomics", issue = "99069"), + unstable(feature = "integer_atomics", issue = "99069"), + unstable(feature = "integer_atomics", issue = "99069"), + unstable(feature = "integer_atomics", issue = "99069"), rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - unstable(feature = "integer_atomics", issue = "32976"), - "u128", "../../../std/primitive.u128.html", + unstable(feature = "integer_atomics", issue = "99069"), + cfg_attr(not(test), rustc_diagnostic_item = "AtomicU128"), + "u128", "#![feature(integer_atomics)]\n\n", atomic_umin, atomic_umax, 16, "AtomicU128::new(0)", u128 AtomicU128 ATOMIC_U128_INIT } -#[cfg(target_has_atomic_load_store = "ptr")] -#[cfg(target_pointer_width = "16")] -macro_rules! ptr_width { - () => { - 2 - }; -} -#[cfg(target_has_atomic_load_store = "ptr")] -#[cfg(target_pointer_width = "32")] -macro_rules! ptr_width { - () => { - 4 - }; -} -#[cfg(target_has_atomic_load_store = "ptr")] -#[cfg(target_pointer_width = "64")] -macro_rules! ptr_width { - () => { - 8 - }; -} -#[cfg(target_has_atomic_load_store = "ptr")] -atomic_int! { - cfg(target_has_atomic = "ptr"), - stable(feature = "rust1", since = "1.0.0"), - stable(feature = "extended_compare_and_swap", since = "1.10.0"), - stable(feature = "atomic_debug", since = "1.3.0"), - stable(feature = "atomic_access", since = "1.15.0"), - stable(feature = "atomic_from", since = "1.23.0"), - stable(feature = "atomic_nand", since = "1.27.0"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - stable(feature = "rust1", since = "1.0.0"), - "isize", "../../../std/primitive.isize.html", - "", - atomic_min, atomic_max, - ptr_width!(), - "AtomicIsize::new(0)", - isize AtomicIsize ATOMIC_ISIZE_INIT + +macro_rules! atomic_int_ptr_sized { + ( $($target_pointer_width:literal $align:literal)* ) => { $( + #[cfg(target_has_atomic_load_store = "ptr")] + #[cfg(target_pointer_width = $target_pointer_width)] + atomic_int! { + cfg(target_has_atomic = "ptr"), + cfg(target_has_atomic_equal_alignment = "ptr"), + stable(feature = "rust1", since = "1.0.0"), + stable(feature = "extended_compare_and_swap", since = "1.10.0"), + stable(feature = "atomic_debug", since = "1.3.0"), + stable(feature = "atomic_access", since = "1.15.0"), + stable(feature = "atomic_from", since = "1.23.0"), + stable(feature = "atomic_nand", since = "1.27.0"), + rustc_const_stable(feature = "const_ptr_sized_atomics", since = "1.24.0"), + stable(feature = "rust1", since = "1.0.0"), + cfg_attr(not(test), rustc_diagnostic_item = "AtomicIsize"), + "isize", + "", + atomic_min, atomic_max, + $align, + "AtomicIsize::new(0)", + isize AtomicIsize ATOMIC_ISIZE_INIT + } + #[cfg(target_has_atomic_load_store = "ptr")] + #[cfg(target_pointer_width = $target_pointer_width)] + atomic_int! { + cfg(target_has_atomic = "ptr"), + cfg(target_has_atomic_equal_alignment = "ptr"), + stable(feature = "rust1", since = "1.0.0"), + stable(feature = "extended_compare_and_swap", since = "1.10.0"), + stable(feature = "atomic_debug", since = "1.3.0"), + stable(feature = "atomic_access", since = "1.15.0"), + stable(feature = "atomic_from", since = "1.23.0"), + stable(feature = "atomic_nand", since = "1.27.0"), + rustc_const_stable(feature = "const_ptr_sized_atomics", since = "1.24.0"), + stable(feature = "rust1", since = "1.0.0"), + cfg_attr(not(test), rustc_diagnostic_item = "AtomicUsize"), + "usize", + "", + atomic_umin, atomic_umax, + $align, + "AtomicUsize::new(0)", + usize AtomicUsize ATOMIC_USIZE_INIT + } + )* }; } -#[cfg(target_has_atomic_load_store = "ptr")] -atomic_int! { - cfg(target_has_atomic = "ptr"), - stable(feature = "rust1", since = "1.0.0"), - stable(feature = "extended_compare_and_swap", since = "1.10.0"), - stable(feature = "atomic_debug", since = "1.3.0"), - stable(feature = "atomic_access", since = "1.15.0"), - stable(feature = "atomic_from", since = "1.23.0"), - stable(feature = "atomic_nand", since = "1.27.0"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - stable(feature = "rust1", since = "1.0.0"), - "usize", "../../../std/primitive.usize.html", - "", - atomic_umin, atomic_umax, - ptr_width!(), - "AtomicUsize::new(0)", - usize AtomicUsize ATOMIC_USIZE_INIT + +atomic_int_ptr_sized! { + "16" 2 + "32" 4 + "64" 8 } #[inline] @@ -2364,67 +3000,88 @@ fn strongest_failure_ordering(order: Ordering) -> Ordering { } #[inline] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces unsafe fn atomic_store(dst: *mut T, val: T, order: Ordering) { - match order { - Release => intrinsics::atomic_store_rel(dst, val), - Relaxed => intrinsics::atomic_store_relaxed(dst, val), - SeqCst => intrinsics::atomic_store(dst, val), - Acquire => panic!("there is no such thing as an acquire store"), - AcqRel => panic!("there is no such thing as an acquire/release store"), + // SAFETY: the caller must uphold the safety contract for `atomic_store`. + unsafe { + match order { + Relaxed => intrinsics::atomic_store_relaxed(dst, val), + Release => intrinsics::atomic_store_release(dst, val), + SeqCst => intrinsics::atomic_store_seqcst(dst, val), + Acquire => panic!("there is no such thing as an acquire store"), + AcqRel => panic!("there is no such thing as an acquire-release store"), + } } } #[inline] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces unsafe fn atomic_load(dst: *const T, order: Ordering) -> T { - match order { - Acquire => intrinsics::atomic_load_acq(dst), - Relaxed => intrinsics::atomic_load_relaxed(dst), - SeqCst => intrinsics::atomic_load(dst), - Release => panic!("there is no such thing as a release load"), - AcqRel => panic!("there is no such thing as an acquire/release load"), + // SAFETY: the caller must uphold the safety contract for `atomic_load`. + unsafe { + match order { + Relaxed => intrinsics::atomic_load_relaxed(dst), + Acquire => intrinsics::atomic_load_acquire(dst), + SeqCst => intrinsics::atomic_load_seqcst(dst), + Release => panic!("there is no such thing as a release load"), + AcqRel => panic!("there is no such thing as an acquire-release load"), + } } } #[inline] #[cfg(target_has_atomic = "8")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces unsafe fn atomic_swap(dst: *mut T, val: T, order: Ordering) -> T { - match order { - Acquire => intrinsics::atomic_xchg_acq(dst, val), - Release => intrinsics::atomic_xchg_rel(dst, val), - AcqRel => intrinsics::atomic_xchg_acqrel(dst, val), - Relaxed => intrinsics::atomic_xchg_relaxed(dst, val), - SeqCst => intrinsics::atomic_xchg(dst, val), + // SAFETY: the caller must uphold the safety contract for `atomic_swap`. + unsafe { + match order { + Relaxed => intrinsics::atomic_xchg_relaxed(dst, val), + Acquire => intrinsics::atomic_xchg_acquire(dst, val), + Release => intrinsics::atomic_xchg_release(dst, val), + AcqRel => intrinsics::atomic_xchg_acqrel(dst, val), + SeqCst => intrinsics::atomic_xchg_seqcst(dst, val), + } } } /// Returns the previous value (like __sync_fetch_and_add). #[inline] #[cfg(target_has_atomic = "8")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces unsafe fn atomic_add(dst: *mut T, val: T, order: Ordering) -> T { - match order { - Acquire => intrinsics::atomic_xadd_acq(dst, val), - Release => intrinsics::atomic_xadd_rel(dst, val), - AcqRel => intrinsics::atomic_xadd_acqrel(dst, val), - Relaxed => intrinsics::atomic_xadd_relaxed(dst, val), - SeqCst => intrinsics::atomic_xadd(dst, val), + // SAFETY: the caller must uphold the safety contract for `atomic_add`. + unsafe { + match order { + Relaxed => intrinsics::atomic_xadd_relaxed(dst, val), + Acquire => intrinsics::atomic_xadd_acquire(dst, val), + Release => intrinsics::atomic_xadd_release(dst, val), + AcqRel => intrinsics::atomic_xadd_acqrel(dst, val), + SeqCst => intrinsics::atomic_xadd_seqcst(dst, val), + } } } /// Returns the previous value (like __sync_fetch_and_sub). #[inline] #[cfg(target_has_atomic = "8")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces unsafe fn atomic_sub(dst: *mut T, val: T, order: Ordering) -> T { - match order { - Acquire => intrinsics::atomic_xsub_acq(dst, val), - Release => intrinsics::atomic_xsub_rel(dst, val), - AcqRel => intrinsics::atomic_xsub_acqrel(dst, val), - Relaxed => intrinsics::atomic_xsub_relaxed(dst, val), - SeqCst => intrinsics::atomic_xsub(dst, val), + // SAFETY: the caller must uphold the safety contract for `atomic_sub`. + unsafe { + match order { + Relaxed => intrinsics::atomic_xsub_relaxed(dst, val), + Acquire => intrinsics::atomic_xsub_acquire(dst, val), + Release => intrinsics::atomic_xsub_release(dst, val), + AcqRel => intrinsics::atomic_xsub_acqrel(dst, val), + SeqCst => intrinsics::atomic_xsub_seqcst(dst, val), + } } } #[inline] #[cfg(target_has_atomic = "8")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces unsafe fn atomic_compare_exchange( dst: *mut T, old: T, @@ -2432,25 +3089,34 @@ unsafe fn atomic_compare_exchange( success: Ordering, failure: Ordering, ) -> Result { - let (val, ok) = match (success, failure) { - (Acquire, Acquire) => intrinsics::atomic_cxchg_acq(dst, old, new), - (Release, Relaxed) => intrinsics::atomic_cxchg_rel(dst, old, new), - (AcqRel, Acquire) => intrinsics::atomic_cxchg_acqrel(dst, old, new), - (Relaxed, Relaxed) => intrinsics::atomic_cxchg_relaxed(dst, old, new), - (SeqCst, SeqCst) => intrinsics::atomic_cxchg(dst, old, new), - (Acquire, Relaxed) => intrinsics::atomic_cxchg_acq_failrelaxed(dst, old, new), - (AcqRel, Relaxed) => intrinsics::atomic_cxchg_acqrel_failrelaxed(dst, old, new), - (SeqCst, Relaxed) => intrinsics::atomic_cxchg_failrelaxed(dst, old, new), - (SeqCst, Acquire) => intrinsics::atomic_cxchg_failacq(dst, old, new), - (_, AcqRel) => panic!("there is no such thing as an acquire/release failure ordering"), - (_, Release) => panic!("there is no such thing as a release failure ordering"), - _ => panic!("a failure ordering can't be stronger than a success ordering"), + // SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange`. + let (val, ok) = unsafe { + match (success, failure) { + (Relaxed, Relaxed) => intrinsics::atomic_cxchg_relaxed_relaxed(dst, old, new), + (Relaxed, Acquire) => intrinsics::atomic_cxchg_relaxed_acquire(dst, old, new), + (Relaxed, SeqCst) => intrinsics::atomic_cxchg_relaxed_seqcst(dst, old, new), + (Acquire, Relaxed) => intrinsics::atomic_cxchg_acquire_relaxed(dst, old, new), + (Acquire, Acquire) => intrinsics::atomic_cxchg_acquire_acquire(dst, old, new), + (Acquire, SeqCst) => intrinsics::atomic_cxchg_acquire_seqcst(dst, old, new), + (Release, Relaxed) => intrinsics::atomic_cxchg_release_relaxed(dst, old, new), + (Release, Acquire) => intrinsics::atomic_cxchg_release_acquire(dst, old, new), + (Release, SeqCst) => intrinsics::atomic_cxchg_release_seqcst(dst, old, new), + (AcqRel, Relaxed) => intrinsics::atomic_cxchg_acqrel_relaxed(dst, old, new), + (AcqRel, Acquire) => intrinsics::atomic_cxchg_acqrel_acquire(dst, old, new), + (AcqRel, SeqCst) => intrinsics::atomic_cxchg_acqrel_seqcst(dst, old, new), + (SeqCst, Relaxed) => intrinsics::atomic_cxchg_seqcst_relaxed(dst, old, new), + (SeqCst, Acquire) => intrinsics::atomic_cxchg_seqcst_acquire(dst, old, new), + (SeqCst, SeqCst) => intrinsics::atomic_cxchg_seqcst_seqcst(dst, old, new), + (_, AcqRel) => panic!("there is no such thing as an acquire-release failure ordering"), + (_, Release) => panic!("there is no such thing as a release failure ordering"), + } }; if ok { Ok(val) } else { Err(val) } } #[inline] #[cfg(target_has_atomic = "8")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces unsafe fn atomic_compare_exchange_weak( dst: *mut T, old: T, @@ -2458,120 +3124,160 @@ unsafe fn atomic_compare_exchange_weak( success: Ordering, failure: Ordering, ) -> Result { - let (val, ok) = match (success, failure) { - (Acquire, Acquire) => intrinsics::atomic_cxchgweak_acq(dst, old, new), - (Release, Relaxed) => intrinsics::atomic_cxchgweak_rel(dst, old, new), - (AcqRel, Acquire) => intrinsics::atomic_cxchgweak_acqrel(dst, old, new), - (Relaxed, Relaxed) => intrinsics::atomic_cxchgweak_relaxed(dst, old, new), - (SeqCst, SeqCst) => intrinsics::atomic_cxchgweak(dst, old, new), - (Acquire, Relaxed) => intrinsics::atomic_cxchgweak_acq_failrelaxed(dst, old, new), - (AcqRel, Relaxed) => intrinsics::atomic_cxchgweak_acqrel_failrelaxed(dst, old, new), - (SeqCst, Relaxed) => intrinsics::atomic_cxchgweak_failrelaxed(dst, old, new), - (SeqCst, Acquire) => intrinsics::atomic_cxchgweak_failacq(dst, old, new), - (_, AcqRel) => panic!("there is no such thing as an acquire/release failure ordering"), - (_, Release) => panic!("there is no such thing as a release failure ordering"), - _ => panic!("a failure ordering can't be stronger than a success ordering"), + // SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange_weak`. + let (val, ok) = unsafe { + match (success, failure) { + (Relaxed, Relaxed) => intrinsics::atomic_cxchgweak_relaxed_relaxed(dst, old, new), + (Relaxed, Acquire) => intrinsics::atomic_cxchgweak_relaxed_acquire(dst, old, new), + (Relaxed, SeqCst) => intrinsics::atomic_cxchgweak_relaxed_seqcst(dst, old, new), + (Acquire, Relaxed) => intrinsics::atomic_cxchgweak_acquire_relaxed(dst, old, new), + (Acquire, Acquire) => intrinsics::atomic_cxchgweak_acquire_acquire(dst, old, new), + (Acquire, SeqCst) => intrinsics::atomic_cxchgweak_acquire_seqcst(dst, old, new), + (Release, Relaxed) => intrinsics::atomic_cxchgweak_release_relaxed(dst, old, new), + (Release, Acquire) => intrinsics::atomic_cxchgweak_release_acquire(dst, old, new), + (Release, SeqCst) => intrinsics::atomic_cxchgweak_release_seqcst(dst, old, new), + (AcqRel, Relaxed) => intrinsics::atomic_cxchgweak_acqrel_relaxed(dst, old, new), + (AcqRel, Acquire) => intrinsics::atomic_cxchgweak_acqrel_acquire(dst, old, new), + (AcqRel, SeqCst) => intrinsics::atomic_cxchgweak_acqrel_seqcst(dst, old, new), + (SeqCst, Relaxed) => intrinsics::atomic_cxchgweak_seqcst_relaxed(dst, old, new), + (SeqCst, Acquire) => intrinsics::atomic_cxchgweak_seqcst_acquire(dst, old, new), + (SeqCst, SeqCst) => intrinsics::atomic_cxchgweak_seqcst_seqcst(dst, old, new), + (_, AcqRel) => panic!("there is no such thing as an acquire-release failure ordering"), + (_, Release) => panic!("there is no such thing as a release failure ordering"), + } }; if ok { Ok(val) } else { Err(val) } } #[inline] #[cfg(target_has_atomic = "8")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces unsafe fn atomic_and(dst: *mut T, val: T, order: Ordering) -> T { - match order { - Acquire => intrinsics::atomic_and_acq(dst, val), - Release => intrinsics::atomic_and_rel(dst, val), - AcqRel => intrinsics::atomic_and_acqrel(dst, val), - Relaxed => intrinsics::atomic_and_relaxed(dst, val), - SeqCst => intrinsics::atomic_and(dst, val), + // SAFETY: the caller must uphold the safety contract for `atomic_and` + unsafe { + match order { + Relaxed => intrinsics::atomic_and_relaxed(dst, val), + Acquire => intrinsics::atomic_and_acquire(dst, val), + Release => intrinsics::atomic_and_release(dst, val), + AcqRel => intrinsics::atomic_and_acqrel(dst, val), + SeqCst => intrinsics::atomic_and_seqcst(dst, val), + } } } #[inline] #[cfg(target_has_atomic = "8")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces unsafe fn atomic_nand(dst: *mut T, val: T, order: Ordering) -> T { - match order { - Acquire => intrinsics::atomic_nand_acq(dst, val), - Release => intrinsics::atomic_nand_rel(dst, val), - AcqRel => intrinsics::atomic_nand_acqrel(dst, val), - Relaxed => intrinsics::atomic_nand_relaxed(dst, val), - SeqCst => intrinsics::atomic_nand(dst, val), + // SAFETY: the caller must uphold the safety contract for `atomic_nand` + unsafe { + match order { + Relaxed => intrinsics::atomic_nand_relaxed(dst, val), + Acquire => intrinsics::atomic_nand_acquire(dst, val), + Release => intrinsics::atomic_nand_release(dst, val), + AcqRel => intrinsics::atomic_nand_acqrel(dst, val), + SeqCst => intrinsics::atomic_nand_seqcst(dst, val), + } } } #[inline] #[cfg(target_has_atomic = "8")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces unsafe fn atomic_or(dst: *mut T, val: T, order: Ordering) -> T { - match order { - Acquire => intrinsics::atomic_or_acq(dst, val), - Release => intrinsics::atomic_or_rel(dst, val), - AcqRel => intrinsics::atomic_or_acqrel(dst, val), - Relaxed => intrinsics::atomic_or_relaxed(dst, val), - SeqCst => intrinsics::atomic_or(dst, val), + // SAFETY: the caller must uphold the safety contract for `atomic_or` + unsafe { + match order { + SeqCst => intrinsics::atomic_or_seqcst(dst, val), + Acquire => intrinsics::atomic_or_acquire(dst, val), + Release => intrinsics::atomic_or_release(dst, val), + AcqRel => intrinsics::atomic_or_acqrel(dst, val), + Relaxed => intrinsics::atomic_or_relaxed(dst, val), + } } } #[inline] #[cfg(target_has_atomic = "8")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces unsafe fn atomic_xor(dst: *mut T, val: T, order: Ordering) -> T { - match order { - Acquire => intrinsics::atomic_xor_acq(dst, val), - Release => intrinsics::atomic_xor_rel(dst, val), - AcqRel => intrinsics::atomic_xor_acqrel(dst, val), - Relaxed => intrinsics::atomic_xor_relaxed(dst, val), - SeqCst => intrinsics::atomic_xor(dst, val), + // SAFETY: the caller must uphold the safety contract for `atomic_xor` + unsafe { + match order { + SeqCst => intrinsics::atomic_xor_seqcst(dst, val), + Acquire => intrinsics::atomic_xor_acquire(dst, val), + Release => intrinsics::atomic_xor_release(dst, val), + AcqRel => intrinsics::atomic_xor_acqrel(dst, val), + Relaxed => intrinsics::atomic_xor_relaxed(dst, val), + } } } /// returns the max value (signed comparison) #[inline] #[cfg(target_has_atomic = "8")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces unsafe fn atomic_max(dst: *mut T, val: T, order: Ordering) -> T { - match order { - Acquire => intrinsics::atomic_max_acq(dst, val), - Release => intrinsics::atomic_max_rel(dst, val), - AcqRel => intrinsics::atomic_max_acqrel(dst, val), - Relaxed => intrinsics::atomic_max_relaxed(dst, val), - SeqCst => intrinsics::atomic_max(dst, val), + // SAFETY: the caller must uphold the safety contract for `atomic_max` + unsafe { + match order { + Relaxed => intrinsics::atomic_max_relaxed(dst, val), + Acquire => intrinsics::atomic_max_acquire(dst, val), + Release => intrinsics::atomic_max_release(dst, val), + AcqRel => intrinsics::atomic_max_acqrel(dst, val), + SeqCst => intrinsics::atomic_max_seqcst(dst, val), + } } } /// returns the min value (signed comparison) #[inline] #[cfg(target_has_atomic = "8")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces unsafe fn atomic_min(dst: *mut T, val: T, order: Ordering) -> T { - match order { - Acquire => intrinsics::atomic_min_acq(dst, val), - Release => intrinsics::atomic_min_rel(dst, val), - AcqRel => intrinsics::atomic_min_acqrel(dst, val), - Relaxed => intrinsics::atomic_min_relaxed(dst, val), - SeqCst => intrinsics::atomic_min(dst, val), + // SAFETY: the caller must uphold the safety contract for `atomic_min` + unsafe { + match order { + Relaxed => intrinsics::atomic_min_relaxed(dst, val), + Acquire => intrinsics::atomic_min_acquire(dst, val), + Release => intrinsics::atomic_min_release(dst, val), + AcqRel => intrinsics::atomic_min_acqrel(dst, val), + SeqCst => intrinsics::atomic_min_seqcst(dst, val), + } } } /// returns the max value (unsigned comparison) #[inline] #[cfg(target_has_atomic = "8")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces unsafe fn atomic_umax(dst: *mut T, val: T, order: Ordering) -> T { - match order { - Acquire => intrinsics::atomic_umax_acq(dst, val), - Release => intrinsics::atomic_umax_rel(dst, val), - AcqRel => intrinsics::atomic_umax_acqrel(dst, val), - Relaxed => intrinsics::atomic_umax_relaxed(dst, val), - SeqCst => intrinsics::atomic_umax(dst, val), + // SAFETY: the caller must uphold the safety contract for `atomic_umax` + unsafe { + match order { + Relaxed => intrinsics::atomic_umax_relaxed(dst, val), + Acquire => intrinsics::atomic_umax_acquire(dst, val), + Release => intrinsics::atomic_umax_release(dst, val), + AcqRel => intrinsics::atomic_umax_acqrel(dst, val), + SeqCst => intrinsics::atomic_umax_seqcst(dst, val), + } } } /// returns the min value (unsigned comparison) #[inline] #[cfg(target_has_atomic = "8")] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces unsafe fn atomic_umin(dst: *mut T, val: T, order: Ordering) -> T { - match order { - Acquire => intrinsics::atomic_umin_acq(dst, val), - Release => intrinsics::atomic_umin_rel(dst, val), - AcqRel => intrinsics::atomic_umin_acqrel(dst, val), - Relaxed => intrinsics::atomic_umin_relaxed(dst, val), - SeqCst => intrinsics::atomic_umin(dst, val), + // SAFETY: the caller must uphold the safety contract for `atomic_umin` + unsafe { + match order { + Relaxed => intrinsics::atomic_umin_relaxed(dst, val), + Acquire => intrinsics::atomic_umin_acquire(dst, val), + Release => intrinsics::atomic_umin_release(dst, val), + AcqRel => intrinsics::atomic_umin_acqrel(dst, val), + SeqCst => intrinsics::atomic_umin_seqcst(dst, val), + } } } @@ -2585,7 +3291,7 @@ unsafe fn atomic_umin(dst: *mut T, val: T, order: Ordering) -> T { /// A fence 'A' which has (at least) [`Release`] ordering semantics, synchronizes /// with a fence 'B' with (at least) [`Acquire`] semantics, if and only if there /// exist operations X and Y, both operating on some atomic object 'M' such -/// that A is sequenced before X, Y is synchronized before B and Y observes +/// that A is sequenced before X, Y is sequenced before B and Y observes /// the change to M. This provides a happens-before dependence between A and B. /// /// ```text @@ -2634,7 +3340,12 @@ unsafe fn atomic_umin(dst: *mut T, val: T, order: Ordering) -> T { /// } /// /// pub fn lock(&self) { -/// while !self.flag.compare_and_swap(false, true, Ordering::Relaxed) {} +/// // Wait until the old value is `false`. +/// while self +/// .flag +/// .compare_exchange_weak(false, true, Ordering::Relaxed, Ordering::Relaxed) +/// .is_err() +/// {} /// // This fence synchronizes-with store in `unlock`. /// fence(Ordering::Acquire); /// } @@ -2644,31 +3355,18 @@ unsafe fn atomic_umin(dst: *mut T, val: T, order: Ordering) -> T { /// } /// } /// ``` -/// -/// [`Ordering`]: enum.Ordering.html -/// [`Acquire`]: enum.Ordering.html#variant.Acquire -/// [`SeqCst`]: enum.Ordering.html#variant.SeqCst -/// [`Release`]: enum.Ordering.html#variant.Release -/// [`AcqRel`]: enum.Ordering.html#variant.AcqRel -/// [`Relaxed`]: enum.Ordering.html#variant.Relaxed #[inline] #[stable(feature = "rust1", since = "1.0.0")] -#[cfg_attr(target_arch = "wasm32", allow(unused_variables))] +#[rustc_diagnostic_item = "fence"] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn fence(order: Ordering) { - // On wasm32 it looks like fences aren't implemented in LLVM yet in that - // they will cause LLVM to abort. The wasm instruction set doesn't have - // fences right now. There's discussion online about the best way for tools - // to conventionally implement fences at - // https://github.com/WebAssembly/tool-conventions/issues/59. We should - // follow that discussion and implement a solution when one comes about! - #[cfg(not(target_arch = "wasm32"))] // SAFETY: using an atomic fence is safe. unsafe { match order { - Acquire => intrinsics::atomic_fence_acq(), - Release => intrinsics::atomic_fence_rel(), + Acquire => intrinsics::atomic_fence_acquire(), + Release => intrinsics::atomic_fence_release(), AcqRel => intrinsics::atomic_fence_acqrel(), - SeqCst => intrinsics::atomic_fence(), + SeqCst => intrinsics::atomic_fence_seqcst(), Relaxed => panic!("there is no such thing as a relaxed fence"), } } @@ -2711,7 +3409,7 @@ pub fn fence(order: Ordering) { /// Without `compiler_fence`, the `assert_eq!` in following code /// is *not* guaranteed to succeed, despite everything happening in a single thread. /// To see why, remember that the compiler is free to swap the stores to -/// `IMPORTANT_VARIABLE` and `IS_READ` since they are both +/// `IMPORTANT_VARIABLE` and `IS_READY` since they are both /// `Ordering::Relaxed`. If it does, and the signal handler is invoked right /// after `IS_READY` is updated, then the signal handler will see /// `IS_READY=1`, but `IMPORTANT_VARIABLE=0`. @@ -2739,24 +3437,19 @@ pub fn fence(order: Ordering) { /// } /// ``` /// -/// [`fence`]: fn.fence.html -/// [`Ordering`]: enum.Ordering.html -/// [`Acquire`]: enum.Ordering.html#variant.Acquire -/// [`SeqCst`]: enum.Ordering.html#variant.SeqCst -/// [`Release`]: enum.Ordering.html#variant.Release -/// [`AcqRel`]: enum.Ordering.html#variant.AcqRel -/// [`Relaxed`]: enum.Ordering.html#variant.Relaxed /// [memory barriers]: https://www.kernel.org/doc/Documentation/memory-barriers.txt #[inline] #[stable(feature = "compiler_fences", since = "1.21.0")] +#[rustc_diagnostic_item = "compiler_fence"] +#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn compiler_fence(order: Ordering) { // SAFETY: using an atomic fence is safe. unsafe { match order { - Acquire => intrinsics::atomic_singlethreadfence_acq(), - Release => intrinsics::atomic_singlethreadfence_rel(), + Acquire => intrinsics::atomic_singlethreadfence_acquire(), + Release => intrinsics::atomic_singlethreadfence_release(), AcqRel => intrinsics::atomic_singlethreadfence_acqrel(), - SeqCst => intrinsics::atomic_singlethreadfence(), + SeqCst => intrinsics::atomic_singlethreadfence_seqcst(), Relaxed => panic!("there is no such thing as a relaxed compiler fence"), } } @@ -2766,7 +3459,7 @@ pub fn compiler_fence(order: Ordering) { #[stable(feature = "atomic_debug", since = "1.3.0")] impl fmt::Debug for AtomicBool { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&self.load(Ordering::SeqCst), f) + fmt::Debug::fmt(&self.load(Ordering::Relaxed), f) } } @@ -2774,7 +3467,7 @@ impl fmt::Debug for AtomicBool { #[stable(feature = "atomic_debug", since = "1.3.0")] impl fmt::Debug for AtomicPtr { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&self.load(Ordering::SeqCst), f) + fmt::Debug::fmt(&self.load(Ordering::Relaxed), f) } } @@ -2785,3 +3478,15 @@ impl fmt::Pointer for AtomicPtr { fmt::Pointer::fmt(&self.load(Ordering::SeqCst), f) } } + +/// Signals the processor that it is inside a busy-wait spin-loop ("spin lock"). +/// +/// This function is deprecated in favor of [`hint::spin_loop`]. +/// +/// [`hint::spin_loop`]: crate::hint::spin_loop +#[inline] +#[stable(feature = "spin_loop_hint", since = "1.24.0")] +#[deprecated(since = "1.51.0", note = "use hint::spin_loop instead")] +pub fn spin_loop_hint() { + spin_loop() +} diff --git a/crux-mir/lib/core/src/sync/exclusive.rs b/crux-mir/lib/core/src/sync/exclusive.rs new file mode 100644 index 000000000..301ad41c9 --- /dev/null +++ b/crux-mir/lib/core/src/sync/exclusive.rs @@ -0,0 +1,180 @@ +//! Defines [`Exclusive`]. + +use core::fmt; +use core::future::Future; +use core::pin::Pin; +use core::task::{Context, Poll}; + +/// `Exclusive` provides only _mutable_ access, also referred to as _exclusive_ +/// access to the underlying value. It provides no _immutable_, or _shared_ +/// access to the underlying value. +/// +/// While this may seem not very useful, it allows `Exclusive` to _unconditionally_ +/// implement [`Sync`]. Indeed, the safety requirements of `Sync` state that for `Exclusive` +/// to be `Sync`, it must be sound to _share_ across threads, that is, it must be sound +/// for `&Exclusive` to cross thread boundaries. By design, a `&Exclusive` has no API +/// whatsoever, making it useless, thus harmless, thus memory safe. +/// +/// Certain constructs like [`Future`]s can only be used with _exclusive_ access, +/// and are often `Send` but not `Sync`, so `Exclusive` can be used as hint to the +/// rust compiler that something is `Sync` in practice. +/// +/// ## Examples +/// Using a non-`Sync` future prevents the wrapping struct from being `Sync` +/// ```compile_fail +/// use core::cell::Cell; +/// +/// async fn other() {} +/// fn assert_sync(t: T) {} +/// struct State { +/// future: F +/// } +/// +/// assert_sync(State { +/// future: async { +/// let cell = Cell::new(1); +/// let cell_ref = &cell; +/// other().await; +/// let value = cell_ref.get(); +/// } +/// }); +/// ``` +/// +/// `Exclusive` ensures the struct is `Sync` without stripping the future of its +/// functionality. +/// ``` +/// #![feature(exclusive_wrapper)] +/// use core::cell::Cell; +/// use core::sync::Exclusive; +/// +/// async fn other() {} +/// fn assert_sync(t: T) {} +/// struct State { +/// future: Exclusive +/// } +/// +/// assert_sync(State { +/// future: Exclusive::new(async { +/// let cell = Cell::new(1); +/// let cell_ref = &cell; +/// other().await; +/// let value = cell_ref.get(); +/// }) +/// }); +/// ``` +/// +/// ## Parallels with a mutex +/// In some sense, `Exclusive` can be thought of as a _compile-time_ version of +/// a mutex, as the borrow-checker guarantees that only one `&mut` can exist +/// for any value. This is a parallel with the fact that +/// `&` and `&mut` references together can be thought of as a _compile-time_ +/// version of a read-write lock. +/// +/// +/// [`Sync`]: core::marker::Sync +#[unstable(feature = "exclusive_wrapper", issue = "98407")] +#[doc(alias = "SyncWrapper")] +#[doc(alias = "SyncCell")] +#[doc(alias = "Unique")] +// `Exclusive` can't have `PartialOrd`, `Clone`, etc. impls as they would +// use `&` access to the inner value, violating the `Sync` impl's safety +// requirements. +#[derive(Default)] +#[repr(transparent)] +pub struct Exclusive { + inner: T, +} + +// See `Exclusive`'s docs for justification. +#[unstable(feature = "exclusive_wrapper", issue = "98407")] +unsafe impl Sync for Exclusive {} + +#[unstable(feature = "exclusive_wrapper", issue = "98407")] +impl fmt::Debug for Exclusive { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + f.debug_struct("Exclusive").finish_non_exhaustive() + } +} + +impl Exclusive { + /// Wrap a value in an `Exclusive` + #[unstable(feature = "exclusive_wrapper", issue = "98407")] + #[must_use] + #[inline] + pub const fn new(t: T) -> Self { + Self { inner: t } + } + + /// Unwrap the value contained in the `Exclusive` + #[unstable(feature = "exclusive_wrapper", issue = "98407")] + #[must_use] + #[inline] + pub const fn into_inner(self) -> T { + self.inner + } +} + +impl Exclusive { + /// Get exclusive access to the underlying value. + #[unstable(feature = "exclusive_wrapper", issue = "98407")] + #[must_use] + #[inline] + pub const fn get_mut(&mut self) -> &mut T { + &mut self.inner + } + + /// Get pinned exclusive access to the underlying value. + /// + /// `Exclusive` is considered to _structurally pin_ the underlying + /// value, which means _unpinned_ `Exclusive`s can produce _unpinned_ + /// access to the underlying value, but _pinned_ `Exclusive`s only + /// produce _pinned_ access to the underlying value. + #[unstable(feature = "exclusive_wrapper", issue = "98407")] + #[must_use] + #[inline] + pub const fn get_pin_mut(self: Pin<&mut Self>) -> Pin<&mut T> { + // SAFETY: `Exclusive` can only produce `&mut T` if itself is unpinned + // `Pin::map_unchecked_mut` is not const, so we do this conversion manually + unsafe { Pin::new_unchecked(&mut self.get_unchecked_mut().inner) } + } + + /// Build a _mutable_ reference to an `Exclusive` from + /// a _mutable_ reference to a `T`. This allows you to skip + /// building an `Exclusive` with [`Exclusive::new`]. + #[unstable(feature = "exclusive_wrapper", issue = "98407")] + #[must_use] + #[inline] + pub const fn from_mut(r: &'_ mut T) -> &'_ mut Exclusive { + // SAFETY: repr is ≥ C, so refs have the same layout; and `Exclusive` properties are `&mut`-agnostic + unsafe { &mut *(r as *mut T as *mut Exclusive) } + } + + /// Build a _pinned mutable_ reference to an `Exclusive` from + /// a _pinned mutable_ reference to a `T`. This allows you to skip + /// building an `Exclusive` with [`Exclusive::new`]. + #[unstable(feature = "exclusive_wrapper", issue = "98407")] + #[must_use] + #[inline] + pub const fn from_pin_mut(r: Pin<&'_ mut T>) -> Pin<&'_ mut Exclusive> { + // SAFETY: `Exclusive` can only produce `&mut T` if itself is unpinned + // `Pin::map_unchecked_mut` is not const, so we do this conversion manually + unsafe { Pin::new_unchecked(Self::from_mut(r.get_unchecked_mut())) } + } +} + +#[unstable(feature = "exclusive_wrapper", issue = "98407")] +impl From for Exclusive { + #[inline] + fn from(t: T) -> Self { + Self::new(t) + } +} + +#[unstable(feature = "exclusive_wrapper", issue = "98407")] +impl Future for Exclusive { + type Output = T::Output; + #[inline] + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.get_pin_mut().poll(cx) + } +} diff --git a/crux-mir/lib/core/src/sync/mod.rs b/crux-mir/lib/core/src/sync/mod.rs index b635bae0a..4365e4cb2 100644 --- a/crux-mir/lib/core/src/sync/mod.rs +++ b/crux-mir/lib/core/src/sync/mod.rs @@ -3,3 +3,6 @@ #![stable(feature = "rust1", since = "1.0.0")] pub mod atomic; +mod exclusive; +#[unstable(feature = "exclusive_wrapper", issue = "98407")] +pub use exclusive::Exclusive; diff --git a/crux-mir/lib/core/src/task/mod.rs b/crux-mir/lib/core/src/task/mod.rs index 27760749c..c5f89b9a2 100644 --- a/crux-mir/lib/core/src/task/mod.rs +++ b/crux-mir/lib/core/src/task/mod.rs @@ -9,3 +9,9 @@ pub use self::poll::Poll; mod wake; #[stable(feature = "futures_api", since = "1.36.0")] pub use self::wake::{Context, RawWaker, RawWakerVTable, Waker}; + +mod ready; +#[stable(feature = "ready_macro", since = "1.64.0")] +pub use ready::ready; +#[unstable(feature = "poll_ready", issue = "89780")] +pub use ready::Ready; diff --git a/crux-mir/lib/core/src/task/poll.rs b/crux-mir/lib/core/src/task/poll.rs index b3a4bd20b..25b61c0e6 100644 --- a/crux-mir/lib/core/src/task/poll.rs +++ b/crux-mir/lib/core/src/task/poll.rs @@ -1,15 +1,19 @@ #![stable(feature = "futures_api", since = "1.36.0")] -use crate::ops::Try; +use crate::convert; +use crate::ops::{self, ControlFlow}; use crate::result::Result; +use crate::task::Ready; /// Indicates whether a value is available or if the current task has been /// scheduled to receive a wakeup instead. #[must_use = "this `Poll` may be a `Pending` variant, which should be handled"] #[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +#[lang = "Poll"] #[stable(feature = "futures_api", since = "1.36.0")] pub enum Poll { /// Represents that a value is immediately ready. + #[lang = "Ready"] #[stable(feature = "futures_api", since = "1.36.0")] Ready(#[stable(feature = "futures_api", since = "1.36.0")] T), @@ -18,12 +22,28 @@ pub enum Poll { /// When a function returns `Pending`, the function *must* also /// ensure that the current task is scheduled to be awoken when /// progress can be made. + #[lang = "Pending"] #[stable(feature = "futures_api", since = "1.36.0")] Pending, } impl Poll { - /// Changes the ready value of this `Poll` with the closure provided. + /// Maps a `Poll` to `Poll` by applying a function to a contained value. + /// + /// # Examples + /// + /// Converts a Poll<[String]> into a Poll<[usize]>, consuming + /// the original: + /// + /// [String]: ../../std/string/struct.String.html "String" + /// ``` + /// # use core::task::Poll; + /// let poll_some_string = Poll::Ready(String::from("Hello, World!")); + /// // `Poll::map` takes self *by value*, consuming `poll_some_string` + /// let poll_some_len = poll_some_string.map(|s| s.len()); + /// + /// assert_eq!(poll_some_len, Poll::Ready(13)); + /// ``` #[stable(feature = "futures_api", since = "1.36.0")] pub fn map(self, f: F) -> Poll where @@ -35,23 +55,94 @@ impl Poll { } } - /// Returns `true` if this is `Poll::Ready` + /// Returns `true` if the poll is a [`Poll::Ready`] value. + /// + /// # Examples + /// + /// ``` + /// # use core::task::Poll; + /// let x: Poll = Poll::Ready(2); + /// assert_eq!(x.is_ready(), true); + /// + /// let x: Poll = Poll::Pending; + /// assert_eq!(x.is_ready(), false); + /// ``` #[inline] + #[rustc_const_stable(feature = "const_poll", since = "1.49.0")] #[stable(feature = "futures_api", since = "1.36.0")] - pub fn is_ready(&self) -> bool { + pub const fn is_ready(&self) -> bool { matches!(*self, Poll::Ready(_)) } - /// Returns `true` if this is `Poll::Pending` + /// Returns `true` if the poll is a [`Pending`] value. + /// + /// [`Pending`]: Poll::Pending + /// + /// # Examples + /// + /// ``` + /// # use core::task::Poll; + /// let x: Poll = Poll::Ready(2); + /// assert_eq!(x.is_pending(), false); + /// + /// let x: Poll = Poll::Pending; + /// assert_eq!(x.is_pending(), true); + /// ``` #[inline] + #[rustc_const_stable(feature = "const_poll", since = "1.49.0")] #[stable(feature = "futures_api", since = "1.36.0")] - pub fn is_pending(&self) -> bool { + pub const fn is_pending(&self) -> bool { !self.is_ready() } + + /// Extracts the successful type of a [`Poll`]. + /// + /// When combined with the `?` operator, this function will + /// propagate any [`Poll::Pending`] values to the caller, and + /// extract the `T` from [`Poll::Ready`]. + /// + /// # Examples + /// + /// ```rust + /// #![feature(poll_ready)] + /// + /// use std::task::{Context, Poll}; + /// use std::future::{self, Future}; + /// use std::pin::Pin; + /// + /// pub fn do_poll(cx: &mut Context<'_>) -> Poll<()> { + /// let mut fut = future::ready(42); + /// let fut = Pin::new(&mut fut); + /// + /// let num = fut.poll(cx).ready()?; + /// # drop(num); + /// // ... use num + /// + /// Poll::Ready(()) + /// } + /// ``` + #[inline] + #[unstable(feature = "poll_ready", issue = "89780")] + pub fn ready(self) -> Ready { + Ready(self) + } } impl Poll> { - /// Changes the success value of this `Poll` with the closure provided. + /// Maps a `Poll>` to `Poll>` by applying a + /// function to a contained `Poll::Ready(Ok)` value, leaving all other + /// variants untouched. + /// + /// This function can be used to compose the results of two functions. + /// + /// # Examples + /// + /// ``` + /// # use core::task::Poll; + /// let res: Poll> = Poll::Ready("12".parse()); + /// let squared = res.map_ok(|n| n * n); + /// assert_eq!(squared, Poll::Ready(Ok(144))); + /// ``` #[stable(feature = "futures_api", since = "1.36.0")] pub fn map_ok(self, f: F) -> Poll> where @@ -64,7 +155,21 @@ impl Poll> { } } - /// Changes the error value of this `Poll` with the closure provided. + /// Maps a `Poll::Ready>` to `Poll::Ready>` by + /// applying a function to a contained `Poll::Ready(Err)` value, leaving all other + /// variants untouched. + /// + /// This function can be used to pass through a successful result while handling + /// an error. + /// + /// # Examples + /// + /// ``` + /// # use core::task::Poll; + /// let res: Poll> = Poll::Ready("oops".parse()); + /// let res = res.map_err(|_| 0_u8); + /// assert_eq!(res, Poll::Ready(Err(0))); + /// ``` #[stable(feature = "futures_api", since = "1.36.0")] pub fn map_err(self, f: F) -> Poll> where @@ -79,8 +184,21 @@ impl Poll> { } impl Poll>> { - /// Changes the success value of this `Poll` with the closure provided. - #[unstable(feature = "poll_map", issue = "63514")] + /// Maps a `Poll>>` to `Poll>>` by + /// applying a function to a contained `Poll::Ready(Some(Ok))` value, + /// leaving all other variants untouched. + /// + /// This function can be used to compose the results of two functions. + /// + /// # Examples + /// + /// ``` + /// # use core::task::Poll; + /// let res: Poll>> = Poll::Ready(Some("12".parse())); + /// let squared = res.map_ok(|n| n * n); + /// assert_eq!(squared, Poll::Ready(Some(Ok(144)))); + /// ``` + #[stable(feature = "poll_map", since = "1.51.0")] pub fn map_ok(self, f: F) -> Poll>> where F: FnOnce(T) -> U, @@ -93,8 +211,23 @@ impl Poll>> { } } - /// Changes the error value of this `Poll` with the closure provided. - #[unstable(feature = "poll_map", issue = "63514")] + /// Maps a `Poll::Ready>>` to + /// `Poll::Ready>>` by applying a function to a + /// contained `Poll::Ready(Some(Err))` value, leaving all other variants + /// untouched. + /// + /// This function can be used to pass through a successful result while handling + /// an error. + /// + /// # Examples + /// + /// ``` + /// # use core::task::Poll; + /// let res: Poll>> = Poll::Ready(Some("oops".parse())); + /// let res = res.map_err(|_| 0_u8); + /// assert_eq!(res, Poll::Ready(Some(Err(0)))); + /// ``` + #[stable(feature = "poll_map", since = "1.51.0")] pub fn map_err(self, f: F) -> Poll>> where F: FnOnce(E) -> U, @@ -109,59 +242,80 @@ impl Poll>> { } #[stable(feature = "futures_api", since = "1.36.0")] -impl From for Poll { +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const From for Poll { + /// Moves the value into a [`Poll::Ready`] to make a `Poll`. + /// + /// # Example + /// + /// ``` + /// # use core::task::Poll; + /// assert_eq!(Poll::from(true), Poll::Ready(true)); + /// ``` fn from(t: T) -> Poll { Poll::Ready(t) } } -#[stable(feature = "futures_api", since = "1.36.0")] -impl Try for Poll> { - type Ok = Poll; - type Error = E; +#[unstable(feature = "try_trait_v2", issue = "84277")] +impl ops::Try for Poll> { + type Output = Poll; + type Residual = Result; #[inline] - fn into_result(self) -> Result { - match self { - Poll::Ready(Ok(x)) => Ok(Poll::Ready(x)), - Poll::Ready(Err(e)) => Err(e), - Poll::Pending => Ok(Poll::Pending), - } + fn from_output(c: Self::Output) -> Self { + c.map(Ok) } #[inline] - fn from_error(e: Self::Error) -> Self { - Poll::Ready(Err(e)) + fn branch(self) -> ControlFlow { + match self { + Poll::Ready(Ok(x)) => ControlFlow::Continue(Poll::Ready(x)), + Poll::Ready(Err(e)) => ControlFlow::Break(Err(e)), + Poll::Pending => ControlFlow::Continue(Poll::Pending), + } } +} +#[unstable(feature = "try_trait_v2", issue = "84277")] +impl> ops::FromResidual> for Poll> { #[inline] - fn from_ok(x: Self::Ok) -> Self { - x.map(Ok) + fn from_residual(x: Result) -> Self { + match x { + Err(e) => Poll::Ready(Err(From::from(e))), + } } } -#[stable(feature = "futures_api", since = "1.36.0")] -impl Try for Poll>> { - type Ok = Poll>; - type Error = E; +#[unstable(feature = "try_trait_v2", issue = "84277")] +impl ops::Try for Poll>> { + type Output = Poll>; + type Residual = Result; #[inline] - fn into_result(self) -> Result { - match self { - Poll::Ready(Some(Ok(x))) => Ok(Poll::Ready(Some(x))), - Poll::Ready(Some(Err(e))) => Err(e), - Poll::Ready(None) => Ok(Poll::Ready(None)), - Poll::Pending => Ok(Poll::Pending), - } + fn from_output(c: Self::Output) -> Self { + c.map(|x| x.map(Ok)) } #[inline] - fn from_error(e: Self::Error) -> Self { - Poll::Ready(Some(Err(e))) + fn branch(self) -> ControlFlow { + match self { + Poll::Ready(Some(Ok(x))) => ControlFlow::Continue(Poll::Ready(Some(x))), + Poll::Ready(Some(Err(e))) => ControlFlow::Break(Err(e)), + Poll::Ready(None) => ControlFlow::Continue(Poll::Ready(None)), + Poll::Pending => ControlFlow::Continue(Poll::Pending), + } } +} +#[unstable(feature = "try_trait_v2", issue = "84277")] +impl> ops::FromResidual> + for Poll>> +{ #[inline] - fn from_ok(x: Self::Ok) -> Self { - x.map(|x| x.map(Ok)) + fn from_residual(x: Result) -> Self { + match x { + Err(e) => Poll::Ready(Some(Err(From::from(e)))), + } } } diff --git a/crux-mir/lib/core/src/task/ready.rs b/crux-mir/lib/core/src/task/ready.rs new file mode 100644 index 000000000..b1daf545f --- /dev/null +++ b/crux-mir/lib/core/src/task/ready.rs @@ -0,0 +1,114 @@ +use core::convert; +use core::fmt; +use core::ops::{ControlFlow, FromResidual, Try}; +use core::task::Poll; + +/// Extracts the successful type of a [`Poll`]. +/// +/// This macro bakes in propagation of [`Pending`] signals by returning early. +/// +/// [`Poll`]: crate::task::Poll +/// [`Pending`]: crate::task::Poll::Pending +/// +/// # Examples +/// +/// ``` +/// use std::task::{ready, Context, Poll}; +/// use std::future::{self, Future}; +/// use std::pin::Pin; +/// +/// pub fn do_poll(cx: &mut Context<'_>) -> Poll<()> { +/// let mut fut = future::ready(42); +/// let fut = Pin::new(&mut fut); +/// +/// let num = ready!(fut.poll(cx)); +/// # drop(num); +/// // ... use num +/// +/// Poll::Ready(()) +/// } +/// ``` +/// +/// The `ready!` call expands to: +/// +/// ``` +/// # use std::task::{Context, Poll}; +/// # use std::future::{self, Future}; +/// # use std::pin::Pin; +/// # +/// # pub fn do_poll(cx: &mut Context<'_>) -> Poll<()> { +/// # let mut fut = future::ready(42); +/// # let fut = Pin::new(&mut fut); +/// # +/// let num = match fut.poll(cx) { +/// Poll::Ready(t) => t, +/// Poll::Pending => return Poll::Pending, +/// }; +/// # drop(num); +/// # // ... use num +/// # +/// # Poll::Ready(()) +/// # } +/// ``` +#[stable(feature = "ready_macro", since = "1.64.0")] +#[rustc_macro_transparency = "semitransparent"] +pub macro ready($e:expr) { + match $e { + $crate::task::Poll::Ready(t) => t, + $crate::task::Poll::Pending => { + return $crate::task::Poll::Pending; + } + } +} + +/// Extracts the successful type of a [`Poll`]. +/// +/// See [`Poll::ready`] for details. +#[unstable(feature = "poll_ready", issue = "89780")] +pub struct Ready(pub(crate) Poll); + +#[unstable(feature = "poll_ready", issue = "89780")] +impl Try for Ready { + type Output = T; + type Residual = Ready; + + #[inline] + fn from_output(output: Self::Output) -> Self { + Ready(Poll::Ready(output)) + } + + #[inline] + fn branch(self) -> ControlFlow { + match self.0 { + Poll::Ready(v) => ControlFlow::Continue(v), + Poll::Pending => ControlFlow::Break(Ready(Poll::Pending)), + } + } +} + +#[unstable(feature = "poll_ready", issue = "89780")] +impl FromResidual for Ready { + #[inline] + fn from_residual(residual: Ready) -> Self { + match residual.0 { + Poll::Pending => Ready(Poll::Pending), + } + } +} + +#[unstable(feature = "poll_ready", issue = "89780")] +impl FromResidual> for Poll { + #[inline] + fn from_residual(residual: Ready) -> Self { + match residual.0 { + Poll::Pending => Poll::Pending, + } + } +} + +#[unstable(feature = "poll_ready", issue = "89780")] +impl fmt::Debug for Ready { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("Ready").finish() + } +} diff --git a/crux-mir/lib/core/src/task/wake.rs b/crux-mir/lib/core/src/task/wake.rs index b070b665b..89adfccd9 100644 --- a/crux-mir/lib/core/src/task/wake.rs +++ b/crux-mir/lib/core/src/task/wake.rs @@ -8,10 +8,8 @@ use crate::marker::{PhantomData, Unpin}; /// /// [vtable]: https://en.wikipedia.org/wiki/Virtual_method_table /// -/// It consists of a data pointer and a [virtual function pointer table (vtable)][vtable] that -/// customizes the behavior of the `RawWaker`. -/// -/// [`Waker`]: struct.Waker.html +/// It consists of a data pointer and a [virtual function pointer table (vtable)][vtable] +/// that customizes the behavior of the `RawWaker`. #[derive(PartialEq, Debug)] #[stable(feature = "futures_api", since = "1.36.0")] pub struct RawWaker { @@ -37,12 +35,30 @@ impl RawWaker { /// The `vtable` customizes the behavior of a `Waker` which gets created /// from a `RawWaker`. For each operation on the `Waker`, the associated /// function in the `vtable` of the underlying `RawWaker` will be called. + #[inline] #[rustc_promotable] #[stable(feature = "futures_api", since = "1.36.0")] #[rustc_const_stable(feature = "futures_api", since = "1.36.0")] + #[must_use] pub const fn new(data: *const (), vtable: &'static RawWakerVTable) -> RawWaker { RawWaker { data, vtable } } + + /// Get the `data` pointer used to create this `RawWaker`. + #[inline] + #[must_use] + #[unstable(feature = "waker_getters", issue = "87021")] + pub fn data(&self) -> *const () { + self.data + } + + /// Get the `vtable` pointer used to create this `RawWaker`. + #[inline] + #[must_use] + #[unstable(feature = "waker_getters", issue = "87021")] + pub fn vtable(&self) -> &'static RawWakerVTable { + self.vtable + } } /// A virtual function pointer table (vtable) that specifies the behavior @@ -51,12 +67,16 @@ impl RawWaker { /// The pointer passed to all functions inside the vtable is the `data` pointer /// from the enclosing [`RawWaker`] object. /// -/// The functions inside this struct are only intended be called on the `data` +/// The functions inside this struct are only intended to be called on the `data` /// pointer of a properly constructed [`RawWaker`] object from inside the /// [`RawWaker`] implementation. Calling one of the contained functions using /// any other `data` pointer will cause undefined behavior. /// -/// [`RawWaker`]: struct.RawWaker.html +/// These functions must all be thread-safe (even though [`RawWaker`] is +/// \![Send] + \![Sync]) +/// because [`Waker`] is [Send] + [Sync], and thus wakers may be moved to +/// arbitrary threads or invoked by `&` reference. For example, this means that if the +/// `clone` and `drop` functions manage a reference count, they must do so atomically. #[stable(feature = "futures_api", since = "1.36.0")] #[derive(PartialEq, Copy, Clone, Debug)] pub struct RawWakerVTable { @@ -67,9 +87,6 @@ pub struct RawWakerVTable { /// required for this additional instance of a [`RawWaker`] and associated /// task. Calling `wake` on the resulting [`RawWaker`] should result in a wakeup /// of the same task that would have been awoken by the original [`RawWaker`]. - /// - /// [`Waker`]: struct.Waker.html - /// [`RawWaker`]: struct.RawWaker.html clone: unsafe fn(*const ()) -> RawWaker, /// This function will be called when `wake` is called on the [`Waker`]. @@ -78,9 +95,6 @@ pub struct RawWakerVTable { /// The implementation of this function must make sure to release any /// resources that are associated with this instance of a [`RawWaker`] and /// associated task. - /// - /// [`Waker`]: struct.Waker.html - /// [`RawWaker`]: struct.RawWaker.html wake: unsafe fn(*const ()), /// This function will be called when `wake_by_ref` is called on the [`Waker`]. @@ -88,18 +102,13 @@ pub struct RawWakerVTable { /// /// This function is similar to `wake`, but must not consume the provided data /// pointer. - /// - /// [`Waker`]: struct.Waker.html - /// [`RawWaker`]: struct.RawWaker.html wake_by_ref: unsafe fn(*const ()), - /// This function gets called when a [`RawWaker`] gets dropped. + /// This function gets called when a [`Waker`] gets dropped. /// /// The implementation of this function must make sure to release any /// resources that are associated with this instance of a [`RawWaker`] and /// associated task. - /// - /// [`RawWaker`]: struct.RawWaker.html drop: unsafe fn(*const ()), } @@ -107,6 +116,12 @@ impl RawWakerVTable { /// Creates a new `RawWakerVTable` from the provided `clone`, `wake`, /// `wake_by_ref`, and `drop` functions. /// + /// These functions must all be thread-safe (even though [`RawWaker`] is + /// \![Send] + \![Sync]) + /// because [`Waker`] is [Send] + [Sync], and thus wakers may be moved to + /// arbitrary threads or invoked by `&` reference. For example, this means that if the + /// `clone` and `drop` functions manage a reference count, they must do so atomically. + /// /// # `clone` /// /// This function will be called when the [`RawWaker`] gets cloned, e.g. when @@ -136,22 +151,13 @@ impl RawWakerVTable { /// /// # `drop` /// - /// This function gets called when a [`RawWaker`] gets dropped. + /// This function gets called when a [`Waker`] gets dropped. /// /// The implementation of this function must make sure to release any /// resources that are associated with this instance of a [`RawWaker`] and /// associated task. - /// - /// [`Waker`]: struct.Waker.html - /// [`RawWaker`]: struct.RawWaker.html #[rustc_promotable] #[stable(feature = "futures_api", since = "1.36.0")] - // `rustc_allow_const_fn_ptr` is a hack that should not be used anywhere else - // without first consulting with T-Lang. - // - // FIXME: remove whenever we have a stable way to accept fn pointers from const fn - // (see https://github.com/rust-rfcs/const-eval/issues/19#issuecomment-472799062) - #[rustc_allow_const_fn_ptr] #[rustc_const_stable(feature = "futures_api", since = "1.36.0")] pub const fn new( clone: unsafe fn(*const ()) -> RawWaker, @@ -163,11 +169,12 @@ impl RawWakerVTable { } } -/// The `Context` of an asynchronous task. +/// The context of an asynchronous task. /// -/// Currently, `Context` only serves to provide access to a `&Waker` +/// Currently, `Context` only serves to provide access to a [`&Waker`](Waker) /// which can be used to wake the current task. #[stable(feature = "futures_api", since = "1.36.0")] +#[cfg_attr(not(bootstrap), lang = "Context")] pub struct Context<'a> { waker: &'a Waker, // Ensure we future-proof against variance changes by forcing @@ -175,20 +182,27 @@ pub struct Context<'a> { // are contravariant while return-position lifetimes are // covariant). _marker: PhantomData &'a ()>, + // Ensure `Context` is `!Send` and `!Sync` in order to allow + // for future `!Send` and / or `!Sync` fields. + _marker2: PhantomData<*mut ()>, } impl<'a> Context<'a> { - /// Create a new `Context` from a `&Waker`. + /// Create a new `Context` from a [`&Waker`](Waker). #[stable(feature = "futures_api", since = "1.36.0")] + #[rustc_const_unstable(feature = "const_waker", issue = "102012")] + #[must_use] #[inline] - pub fn from_waker(waker: &'a Waker) -> Self { - Context { waker, _marker: PhantomData } + pub const fn from_waker(waker: &'a Waker) -> Self { + Context { waker, _marker: PhantomData, _marker2: PhantomData } } - /// Returns a reference to the `Waker` for the current task. + /// Returns a reference to the [`Waker`] for the current task. #[stable(feature = "futures_api", since = "1.36.0")] + #[rustc_const_unstable(feature = "const_waker", issue = "102012")] + #[must_use] #[inline] - pub fn waker(&self) -> &'a Waker { + pub const fn waker(&self) -> &'a Waker { &self.waker } } @@ -206,9 +220,18 @@ impl fmt::Debug for Context<'_> { /// This handle encapsulates a [`RawWaker`] instance, which defines the /// executor-specific wakeup behavior. /// -/// Implements [`Clone`], [`Send`], and [`Sync`]. +/// The typical life of a `Waker` is that it is constructed by an executor, wrapped in a +/// [`Context`], then passed to [`Future::poll()`]. Then, if the future chooses to return +/// [`Poll::Pending`], it must also store the waker somehow and call [`Waker::wake()`] when +/// the future should be polled again. +/// +/// Implements [`Clone`], [`Send`], and [`Sync`]; therefore, a waker may be invoked +/// from any thread, including ones not in any way managed by the executor. For example, +/// this might be done to wake a future when a blocking function call completes on another +/// thread. /// -/// [`RawWaker`]: struct.RawWaker.html +/// [`Future::poll()`]: core::future::Future::poll +/// [`Poll::Pending`]: core::task::Poll::Pending #[repr(transparent)] #[stable(feature = "futures_api", since = "1.36.0")] pub struct Waker { @@ -224,6 +247,22 @@ unsafe impl Sync for Waker {} impl Waker { /// Wake up the task associated with this `Waker`. + /// + /// As long as the executor keeps running and the task is not finished, it is + /// guaranteed that each invocation of [`wake()`](Self::wake) (or + /// [`wake_by_ref()`](Self::wake_by_ref)) will be followed by at least one + /// [`poll()`] of the task to which this `Waker` belongs. This makes + /// it possible to temporarily yield to other tasks while running potentially + /// unbounded processing loops. + /// + /// Note that the above implies that multiple wake-ups may be coalesced into a + /// single [`poll()`] invocation by the runtime. + /// + /// Also note that yielding to competing tasks is not guaranteed: it is the + /// executor’s choice which task to run and the executor may choose to run the + /// current task again. + /// + /// [`poll()`]: crate::future::Future::poll #[inline] #[stable(feature = "futures_api", since = "1.36.0")] pub fn wake(self) { @@ -243,8 +282,8 @@ impl Waker { /// Wake up the task associated with this `Waker` without consuming the `Waker`. /// - /// This is similar to `wake`, but may be slightly less efficient in the case - /// where an owned `Waker` is available. This method should be preferred to + /// This is similar to [`wake()`](Self::wake), but may be slightly less efficient in + /// the case where an owned `Waker` is available. This method should be preferred to /// calling `waker.clone().wake()`. #[inline] #[stable(feature = "futures_api", since = "1.36.0")] @@ -256,7 +295,7 @@ impl Waker { unsafe { (self.waker.vtable.wake_by_ref)(self.waker.data) } } - /// Returns `true` if this `Waker` and another `Waker` have awoken the same task. + /// Returns `true` if this `Waker` and another `Waker` would awake the same task. /// /// This function works on a best-effort basis, and may return false even /// when the `Waker`s would awaken the same task. However, if this function @@ -264,6 +303,7 @@ impl Waker { /// /// This function is primarily used for optimization purposes. #[inline] + #[must_use] #[stable(feature = "futures_api", since = "1.36.0")] pub fn will_wake(&self, other: &Waker) -> bool { self.waker == other.waker @@ -274,14 +314,21 @@ impl Waker { /// The behavior of the returned `Waker` is undefined if the contract defined /// in [`RawWaker`]'s and [`RawWakerVTable`]'s documentation is not upheld. /// Therefore this method is unsafe. - /// - /// [`RawWaker`]: struct.RawWaker.html - /// [`RawWakerVTable`]: struct.RawWakerVTable.html #[inline] + #[must_use] #[stable(feature = "futures_api", since = "1.36.0")] - pub unsafe fn from_raw(waker: RawWaker) -> Waker { + #[rustc_const_unstable(feature = "const_waker", issue = "102012")] + pub const unsafe fn from_raw(waker: RawWaker) -> Waker { Waker { waker } } + + /// Get a reference to the underlying [`RawWaker`]. + #[inline] + #[must_use] + #[unstable(feature = "waker_getters", issue = "87021")] + pub fn as_raw(&self) -> &RawWaker { + &self.waker + } } #[stable(feature = "futures_api", since = "1.36.0")] diff --git a/crux-mir/lib/core/src/time.rs b/crux-mir/lib/core/src/time.rs index 2ece2150e..ba1cb6efa 100644 --- a/crux-mir/lib/core/src/time.rs +++ b/crux-mir/lib/core/src/time.rs @@ -2,19 +2,26 @@ //! Temporal quantification. //! -//! Example: +//! # Examples: +//! +//! There are multiple ways to create a new [`Duration`]: //! //! ``` -//! use std::time::Duration; +//! # use std::time::Duration; +//! let five_seconds = Duration::from_secs(5); +//! assert_eq!(five_seconds, Duration::from_millis(5_000)); +//! assert_eq!(five_seconds, Duration::from_micros(5_000_000)); +//! assert_eq!(five_seconds, Duration::from_nanos(5_000_000_000)); //! -//! let five_seconds = Duration::new(5, 0); -//! // both declarations are equivalent -//! assert_eq!(Duration::new(5, 0), Duration::from_secs(5)); +//! let ten_seconds = Duration::from_secs(10); +//! let seven_nanos = Duration::from_nanos(7); +//! let total = ten_seconds + seven_nanos; +//! assert_eq!(total, Duration::new(10, 7)); //! ``` +use crate::fmt; use crate::iter::Sum; use crate::ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Sub, SubAssign}; -use crate::{fmt, u64}; const NANOS_PER_SEC: u32 = 1_000_000_000; const NANOS_PER_MILLI: u32 = 1_000_000; @@ -22,6 +29,20 @@ const NANOS_PER_MICRO: u32 = 1_000; const MILLIS_PER_SEC: u64 = 1_000; const MICROS_PER_SEC: u64 = 1_000_000; +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[repr(transparent)] +#[rustc_layout_scalar_valid_range_start(0)] +#[rustc_layout_scalar_valid_range_end(999_999_999)] +struct Nanoseconds(u32); + +impl Default for Nanoseconds { + #[inline] + fn default() -> Self { + // SAFETY: 0 is within the valid range + unsafe { Nanoseconds(0) } + } +} + /// A `Duration` type to represent a span of time, typically used for system /// timeouts. /// @@ -30,12 +51,10 @@ const MICROS_PER_SEC: u64 = 1_000_000; /// nanosecond-level precision, APIs binding a system timeout will typically round up /// the number of nanoseconds. /// -/// `Duration`s implement many common traits, including [`Add`], [`Sub`], and other -/// [`ops`] traits. +/// [`Duration`]s implement many common traits, including [`Add`], [`Sub`], and other +/// [`ops`] traits. It implements [`Default`] by returning a zero-length `Duration`. /// -/// [`Add`]: ../../std/ops/trait.Add.html -/// [`Sub`]: ../../std/ops/trait.Sub.html -/// [`ops`]: ../../std/ops/index.html +/// [`ops`]: crate::ops /// /// # Examples /// @@ -50,11 +69,23 @@ const MICROS_PER_SEC: u64 = 1_000_000; /// /// let ten_millis = Duration::from_millis(10); /// ``` +/// +/// # Formatting `Duration` values +/// +/// `Duration` intentionally does not have a `Display` impl, as there are a +/// variety of ways to format spans of time for human readability. `Duration` +/// provides a `Debug` impl that shows the full precision of the value. +/// +/// The `Debug` output uses the non-ASCII "µs" suffix for microseconds. If your +/// program output may appear in contexts that cannot rely on full Unicode +/// compatibility, you may wish to format `Duration` objects yourself or use a +/// crate to do so. #[stable(feature = "duration", since = "1.3.0")] #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] +#[cfg_attr(not(test), rustc_diagnostic_item = "Duration")] pub struct Duration { secs: u64, - nanos: u32, // Always 0 <= nanos < NANOS_PER_SEC + nanos: Nanoseconds, // Always 0 <= nanos < NANOS_PER_SEC } impl Duration { @@ -110,6 +141,39 @@ impl Duration { #[unstable(feature = "duration_constants", issue = "57391")] pub const NANOSECOND: Duration = Duration::from_nanos(1); + /// A duration of zero time. + /// + /// # Examples + /// + /// ``` + /// use std::time::Duration; + /// + /// let duration = Duration::ZERO; + /// assert!(duration.is_zero()); + /// assert_eq!(duration.as_nanos(), 0); + /// ``` + #[stable(feature = "duration_zero", since = "1.53.0")] + pub const ZERO: Duration = Duration::from_nanos(0); + + /// The maximum duration. + /// + /// May vary by platform as necessary. Must be able to contain the difference between + /// two instances of [`Instant`] or two instances of [`SystemTime`]. + /// This constraint gives it a value of about 584,942,417,355 years in practice, + /// which is currently used on all platforms. + /// + /// # Examples + /// + /// ``` + /// use std::time::Duration; + /// + /// assert_eq!(Duration::MAX, Duration::new(u64::MAX, 1_000_000_000 - 1)); + /// ``` + /// [`Instant`]: ../../std/time/struct.Instant.html + /// [`SystemTime`]: ../../std/time/struct.SystemTime.html + #[stable(feature = "duration_saturating_ops", since = "1.53.0")] + pub const MAX: Duration = Duration::new(u64::MAX, NANOS_PER_SEC - 1); + /// Creates a new `Duration` from the specified number of whole seconds and /// additional nanoseconds. /// @@ -130,12 +194,16 @@ impl Duration { /// ``` #[stable(feature = "duration", since = "1.3.0")] #[inline] - #[rustc_const_stable(feature = "duration_consts", since = "1.32.0")] - pub fn new(secs: u64, nanos: u32) -> Duration { - let secs = - secs.checked_add((nanos / NANOS_PER_SEC) as u64).expect("overflow in Duration::new"); + #[must_use] + #[rustc_const_stable(feature = "duration_consts_2", since = "1.58.0")] + pub const fn new(secs: u64, nanos: u32) -> Duration { + let secs = match secs.checked_add((nanos / NANOS_PER_SEC) as u64) { + Some(secs) => secs, + None => panic!("overflow in Duration::new"), + }; let nanos = nanos % NANOS_PER_SEC; - Duration { secs, nanos } + // SAFETY: nanos % NANOS_PER_SEC < NANOS_PER_SEC, therefore nanos is within the valid range + Duration { secs, nanos: unsafe { Nanoseconds(nanos) } } } /// Creates a new `Duration` from the specified number of whole seconds. @@ -151,11 +219,11 @@ impl Duration { /// assert_eq!(0, duration.subsec_nanos()); /// ``` #[stable(feature = "duration", since = "1.3.0")] + #[must_use] #[inline] - #[rustc_promotable] #[rustc_const_stable(feature = "duration_consts", since = "1.32.0")] pub const fn from_secs(secs: u64) -> Duration { - Duration { secs, nanos: 0 } + Duration::new(secs, 0) } /// Creates a new `Duration` from the specified number of milliseconds. @@ -171,13 +239,11 @@ impl Duration { /// assert_eq!(569_000_000, duration.subsec_nanos()); /// ``` #[stable(feature = "duration", since = "1.3.0")] + #[must_use] #[inline] #[rustc_const_stable(feature = "duration_consts", since = "1.32.0")] pub const fn from_millis(millis: u64) -> Duration { - Duration { - secs: millis / MILLIS_PER_SEC, - nanos: ((millis % MILLIS_PER_SEC) as u32) * NANOS_PER_MILLI, - } + Duration::new(millis / MILLIS_PER_SEC, ((millis % MILLIS_PER_SEC) as u32) * NANOS_PER_MILLI) } /// Creates a new `Duration` from the specified number of microseconds. @@ -193,13 +259,11 @@ impl Duration { /// assert_eq!(2000, duration.subsec_nanos()); /// ``` #[stable(feature = "duration_from_micros", since = "1.27.0")] + #[must_use] #[inline] #[rustc_const_stable(feature = "duration_consts", since = "1.32.0")] pub const fn from_micros(micros: u64) -> Duration { - Duration { - secs: micros / MICROS_PER_SEC, - nanos: ((micros % MICROS_PER_SEC) as u32) * NANOS_PER_MICRO, - } + Duration::new(micros / MICROS_PER_SEC, ((micros % MICROS_PER_SEC) as u32) * NANOS_PER_MICRO) } /// Creates a new `Duration` from the specified number of nanoseconds. @@ -215,45 +279,60 @@ impl Duration { /// assert_eq!(123, duration.subsec_nanos()); /// ``` #[stable(feature = "duration_extras", since = "1.27.0")] + #[must_use] #[inline] #[rustc_const_stable(feature = "duration_consts", since = "1.32.0")] pub const fn from_nanos(nanos: u64) -> Duration { - Duration { - secs: nanos / (NANOS_PER_SEC as u64), - nanos: (nanos % (NANOS_PER_SEC as u64)) as u32, - } + Duration::new(nanos / (NANOS_PER_SEC as u64), (nanos % (NANOS_PER_SEC as u64)) as u32) } - /// Returns the number of _whole_ seconds contained by this `Duration`. - /// - /// The returned value does not include the fractional (nanosecond) part of the - /// duration, which can be obtained using [`subsec_nanos`]. + /// Returns true if this `Duration` spans no time. /// /// # Examples /// /// ``` /// use std::time::Duration; /// - /// let duration = Duration::new(5, 730023852); - /// assert_eq!(duration.as_secs(), 5); + /// assert!(Duration::ZERO.is_zero()); + /// assert!(Duration::new(0, 0).is_zero()); + /// assert!(Duration::from_nanos(0).is_zero()); + /// assert!(Duration::from_secs(0).is_zero()); + /// + /// assert!(!Duration::new(1, 1).is_zero()); + /// assert!(!Duration::from_nanos(1).is_zero()); + /// assert!(!Duration::from_secs(1).is_zero()); /// ``` + #[must_use] + #[stable(feature = "duration_zero", since = "1.53.0")] + #[rustc_const_stable(feature = "duration_zero", since = "1.53.0")] + #[inline] + pub const fn is_zero(&self) -> bool { + self.secs == 0 && self.nanos.0 == 0 + } + + /// Returns the number of _whole_ seconds contained by this `Duration`. /// - /// To determine the total number of seconds represented by the `Duration`, - /// use `as_secs` in combination with [`subsec_nanos`]: + /// The returned value does not include the fractional (nanosecond) part of the + /// duration, which can be obtained using [`subsec_nanos`]. + /// + /// # Examples /// /// ``` /// use std::time::Duration; /// /// let duration = Duration::new(5, 730023852); - /// - /// assert_eq!(5.730023852, - /// duration.as_secs() as f64 - /// + duration.subsec_nanos() as f64 * 1e-9); + /// assert_eq!(duration.as_secs(), 5); /// ``` /// - /// [`subsec_nanos`]: #method.subsec_nanos + /// To determine the total number of seconds represented by the `Duration` + /// including the fractional part, use [`as_secs_f64`] or [`as_secs_f32`] + /// + /// [`as_secs_f64`]: Duration::as_secs_f64 + /// [`as_secs_f32`]: Duration::as_secs_f32 + /// [`subsec_nanos`]: Duration::subsec_nanos #[stable(feature = "duration", since = "1.3.0")] - #[rustc_const_stable(feature = "duration", since = "1.32.0")] + #[rustc_const_stable(feature = "duration_consts", since = "1.32.0")] + #[must_use] #[inline] pub const fn as_secs(&self) -> u64 { self.secs @@ -275,10 +354,11 @@ impl Duration { /// assert_eq!(duration.subsec_millis(), 432); /// ``` #[stable(feature = "duration_extras", since = "1.27.0")] - #[rustc_const_stable(feature = "duration_extras", since = "1.32.0")] + #[rustc_const_stable(feature = "duration_consts", since = "1.32.0")] + #[must_use] #[inline] pub const fn subsec_millis(&self) -> u32 { - self.nanos / NANOS_PER_MILLI + self.nanos.0 / NANOS_PER_MILLI } /// Returns the fractional part of this `Duration`, in whole microseconds. @@ -297,10 +377,11 @@ impl Duration { /// assert_eq!(duration.subsec_micros(), 234_567); /// ``` #[stable(feature = "duration_extras", since = "1.27.0")] - #[rustc_const_stable(feature = "duration_extras", since = "1.32.0")] + #[rustc_const_stable(feature = "duration_consts", since = "1.32.0")] + #[must_use] #[inline] pub const fn subsec_micros(&self) -> u32 { - self.nanos / NANOS_PER_MICRO + self.nanos.0 / NANOS_PER_MICRO } /// Returns the fractional part of this `Duration`, in nanoseconds. @@ -319,10 +400,11 @@ impl Duration { /// assert_eq!(duration.subsec_nanos(), 10_000_000); /// ``` #[stable(feature = "duration", since = "1.3.0")] - #[rustc_const_stable(feature = "duration", since = "1.32.0")] + #[rustc_const_stable(feature = "duration_consts", since = "1.32.0")] + #[must_use] #[inline] pub const fn subsec_nanos(&self) -> u32 { - self.nanos + self.nanos.0 } /// Returns the total number of whole milliseconds contained by this `Duration`. @@ -337,9 +419,10 @@ impl Duration { /// ``` #[stable(feature = "duration_as_u128", since = "1.33.0")] #[rustc_const_stable(feature = "duration_as_u128", since = "1.33.0")] + #[must_use] #[inline] pub const fn as_millis(&self) -> u128 { - self.secs as u128 * MILLIS_PER_SEC as u128 + (self.nanos / NANOS_PER_MILLI) as u128 + self.secs as u128 * MILLIS_PER_SEC as u128 + (self.nanos.0 / NANOS_PER_MILLI) as u128 } /// Returns the total number of whole microseconds contained by this `Duration`. @@ -354,9 +437,10 @@ impl Duration { /// ``` #[stable(feature = "duration_as_u128", since = "1.33.0")] #[rustc_const_stable(feature = "duration_as_u128", since = "1.33.0")] + #[must_use] #[inline] pub const fn as_micros(&self) -> u128 { - self.secs as u128 * MICROS_PER_SEC as u128 + (self.nanos / NANOS_PER_MICRO) as u128 + self.secs as u128 * MICROS_PER_SEC as u128 + (self.nanos.0 / NANOS_PER_MICRO) as u128 } /// Returns the total number of nanoseconds contained by this `Duration`. @@ -371,16 +455,15 @@ impl Duration { /// ``` #[stable(feature = "duration_as_u128", since = "1.33.0")] #[rustc_const_stable(feature = "duration_as_u128", since = "1.33.0")] + #[must_use] #[inline] pub const fn as_nanos(&self) -> u128 { - self.secs as u128 * NANOS_PER_SEC as u128 + self.nanos as u128 + self.secs as u128 * NANOS_PER_SEC as u128 + self.nanos.0 as u128 } /// Checked `Duration` addition. Computes `self + other`, returning [`None`] /// if overflow occurred. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// /// # Examples /// /// Basic usage: @@ -389,13 +472,16 @@ impl Duration { /// use std::time::Duration; /// /// assert_eq!(Duration::new(0, 0).checked_add(Duration::new(0, 1)), Some(Duration::new(0, 1))); - /// assert_eq!(Duration::new(1, 0).checked_add(Duration::new(std::u64::MAX, 0)), None); + /// assert_eq!(Duration::new(1, 0).checked_add(Duration::new(u64::MAX, 0)), None); /// ``` #[stable(feature = "duration_checked_ops", since = "1.16.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[inline] - pub fn checked_add(self, rhs: Duration) -> Option { + #[rustc_const_stable(feature = "duration_consts_2", since = "1.58.0")] + pub const fn checked_add(self, rhs: Duration) -> Option { if let Some(mut secs) = self.secs.checked_add(rhs.secs) { - let mut nanos = self.nanos + rhs.nanos; + let mut nanos = self.nanos.0 + rhs.nanos.0; if nanos >= NANOS_PER_SEC { nanos -= NANOS_PER_SEC; if let Some(new_secs) = secs.checked_add(1) { @@ -405,17 +491,39 @@ impl Duration { } } debug_assert!(nanos < NANOS_PER_SEC); - Some(Duration { secs, nanos }) + Some(Duration::new(secs, nanos)) } else { None } } + /// Saturating `Duration` addition. Computes `self + other`, returning [`Duration::MAX`] + /// if overflow occurred. + /// + /// # Examples + /// + /// ``` + /// #![feature(duration_constants)] + /// use std::time::Duration; + /// + /// assert_eq!(Duration::new(0, 0).saturating_add(Duration::new(0, 1)), Duration::new(0, 1)); + /// assert_eq!(Duration::new(1, 0).saturating_add(Duration::new(u64::MAX, 0)), Duration::MAX); + /// ``` + #[stable(feature = "duration_saturating_ops", since = "1.53.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + #[rustc_const_stable(feature = "duration_consts_2", since = "1.58.0")] + pub const fn saturating_add(self, rhs: Duration) -> Duration { + match self.checked_add(rhs) { + Some(res) => res, + None => Duration::MAX, + } + } + /// Checked `Duration` subtraction. Computes `self - other`, returning [`None`] /// if the result would be negative or if overflow occurred. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// /// # Examples /// /// Basic usage: @@ -427,31 +535,53 @@ impl Duration { /// assert_eq!(Duration::new(0, 0).checked_sub(Duration::new(0, 1)), None); /// ``` #[stable(feature = "duration_checked_ops", since = "1.16.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[inline] - pub fn checked_sub(self, rhs: Duration) -> Option { + #[rustc_const_stable(feature = "duration_consts_2", since = "1.58.0")] + pub const fn checked_sub(self, rhs: Duration) -> Option { if let Some(mut secs) = self.secs.checked_sub(rhs.secs) { - let nanos = if self.nanos >= rhs.nanos { - self.nanos - rhs.nanos + let nanos = if self.nanos.0 >= rhs.nanos.0 { + self.nanos.0 - rhs.nanos.0 + } else if let Some(sub_secs) = secs.checked_sub(1) { + secs = sub_secs; + self.nanos.0 + NANOS_PER_SEC - rhs.nanos.0 } else { - if let Some(sub_secs) = secs.checked_sub(1) { - secs = sub_secs; - self.nanos + NANOS_PER_SEC - rhs.nanos - } else { - return None; - } + return None; }; debug_assert!(nanos < NANOS_PER_SEC); - Some(Duration { secs, nanos }) + Some(Duration::new(secs, nanos)) } else { None } } + /// Saturating `Duration` subtraction. Computes `self - other`, returning [`Duration::ZERO`] + /// if the result would be negative or if overflow occurred. + /// + /// # Examples + /// + /// ``` + /// use std::time::Duration; + /// + /// assert_eq!(Duration::new(0, 1).saturating_sub(Duration::new(0, 0)), Duration::new(0, 1)); + /// assert_eq!(Duration::new(0, 0).saturating_sub(Duration::new(0, 1)), Duration::ZERO); + /// ``` + #[stable(feature = "duration_saturating_ops", since = "1.53.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + #[rustc_const_stable(feature = "duration_consts_2", since = "1.58.0")] + pub const fn saturating_sub(self, rhs: Duration) -> Duration { + match self.checked_sub(rhs) { + Some(res) => res, + None => Duration::ZERO, + } + } + /// Checked `Duration` multiplication. Computes `self * other`, returning /// [`None`] if overflow occurred. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// /// # Examples /// /// Basic usage: @@ -460,30 +590,54 @@ impl Duration { /// use std::time::Duration; /// /// assert_eq!(Duration::new(0, 500_000_001).checked_mul(2), Some(Duration::new(1, 2))); - /// assert_eq!(Duration::new(std::u64::MAX - 1, 0).checked_mul(2), None); + /// assert_eq!(Duration::new(u64::MAX - 1, 0).checked_mul(2), None); /// ``` #[stable(feature = "duration_checked_ops", since = "1.16.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[inline] - pub fn checked_mul(self, rhs: u32) -> Option { + #[rustc_const_stable(feature = "duration_consts_2", since = "1.58.0")] + pub const fn checked_mul(self, rhs: u32) -> Option { // Multiply nanoseconds as u64, because it cannot overflow that way. - let total_nanos = self.nanos as u64 * rhs as u64; + let total_nanos = self.nanos.0 as u64 * rhs as u64; let extra_secs = total_nanos / (NANOS_PER_SEC as u64); let nanos = (total_nanos % (NANOS_PER_SEC as u64)) as u32; - if let Some(secs) = - self.secs.checked_mul(rhs as u64).and_then(|s| s.checked_add(extra_secs)) - { - debug_assert!(nanos < NANOS_PER_SEC); - Some(Duration { secs, nanos }) - } else { - None + if let Some(s) = self.secs.checked_mul(rhs as u64) { + if let Some(secs) = s.checked_add(extra_secs) { + debug_assert!(nanos < NANOS_PER_SEC); + return Some(Duration::new(secs, nanos)); + } + } + None + } + + /// Saturating `Duration` multiplication. Computes `self * other`, returning + /// [`Duration::MAX`] if overflow occurred. + /// + /// # Examples + /// + /// ``` + /// #![feature(duration_constants)] + /// use std::time::Duration; + /// + /// assert_eq!(Duration::new(0, 500_000_001).saturating_mul(2), Duration::new(1, 2)); + /// assert_eq!(Duration::new(u64::MAX - 1, 0).saturating_mul(2), Duration::MAX); + /// ``` + #[stable(feature = "duration_saturating_ops", since = "1.53.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + #[rustc_const_stable(feature = "duration_consts_2", since = "1.58.0")] + pub const fn saturating_mul(self, rhs: u32) -> Duration { + match self.checked_mul(rhs) { + Some(res) => res, + None => Duration::MAX, } } /// Checked `Duration` division. Computes `self / other`, returning [`None`] /// if `other == 0`. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// /// # Examples /// /// Basic usage: @@ -496,15 +650,18 @@ impl Duration { /// assert_eq!(Duration::new(2, 0).checked_div(0), None); /// ``` #[stable(feature = "duration_checked_ops", since = "1.16.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[inline] - pub fn checked_div(self, rhs: u32) -> Option { + #[rustc_const_stable(feature = "duration_consts_2", since = "1.58.0")] + pub const fn checked_div(self, rhs: u32) -> Option { if rhs != 0 { let secs = self.secs / (rhs as u64); let carry = self.secs - secs * (rhs as u64); let extra_nanos = carry * (NANOS_PER_SEC as u64) / (rhs as u64); - let nanos = self.nanos / rhs + (extra_nanos as u32); + let nanos = self.nanos.0 / rhs + (extra_nanos as u32); debug_assert!(nanos < NANOS_PER_SEC); - Some(Duration { secs, nanos }) + Some(Duration::new(secs, nanos)) } else { None } @@ -522,9 +679,11 @@ impl Duration { /// assert_eq!(dur.as_secs_f64(), 2.7); /// ``` #[stable(feature = "duration_float", since = "1.38.0")] + #[must_use] #[inline] - pub fn as_secs_f64(&self) -> f64 { - (self.secs as f64) + (self.nanos as f64) / (NANOS_PER_SEC as f64) + #[rustc_const_unstable(feature = "duration_consts_float", issue = "72440")] + pub const fn as_secs_f64(&self) -> f64 { + (self.secs as f64) + (self.nanos.0 as f64) / (NANOS_PER_SEC as f64) } /// Returns the number of seconds contained by this `Duration` as `f32`. @@ -539,42 +698,48 @@ impl Duration { /// assert_eq!(dur.as_secs_f32(), 2.7); /// ``` #[stable(feature = "duration_float", since = "1.38.0")] + #[must_use] #[inline] - pub fn as_secs_f32(&self) -> f32 { - (self.secs as f32) + (self.nanos as f32) / (NANOS_PER_SEC as f32) + #[rustc_const_unstable(feature = "duration_consts_float", issue = "72440")] + pub const fn as_secs_f32(&self) -> f32 { + (self.secs as f32) + (self.nanos.0 as f32) / (NANOS_PER_SEC as f32) } /// Creates a new `Duration` from the specified number of seconds represented /// as `f64`. /// /// # Panics - /// This constructor will panic if `secs` is not finite, negative or overflows `Duration`. + /// This constructor will panic if `secs` is negative, overflows `Duration` or not finite. /// /// # Examples /// ``` /// use std::time::Duration; /// - /// let dur = Duration::from_secs_f64(2.7); - /// assert_eq!(dur, Duration::new(2, 700_000_000)); + /// let res = Duration::from_secs_f64(0.0); + /// assert_eq!(res, Duration::new(0, 0)); + /// let res = Duration::from_secs_f64(1e-20); + /// assert_eq!(res, Duration::new(0, 0)); + /// let res = Duration::from_secs_f64(4.2e-7); + /// assert_eq!(res, Duration::new(0, 420)); + /// let res = Duration::from_secs_f64(2.7); + /// assert_eq!(res, Duration::new(2, 700_000_000)); + /// let res = Duration::from_secs_f64(3e10); + /// assert_eq!(res, Duration::new(30_000_000_000, 0)); + /// // subnormal float + /// let res = Duration::from_secs_f64(f64::from_bits(1)); + /// assert_eq!(res, Duration::new(0, 0)); + /// // conversion uses rounding + /// let res = Duration::from_secs_f64(0.999e-9); + /// assert_eq!(res, Duration::new(0, 1)); /// ``` #[stable(feature = "duration_float", since = "1.38.0")] + #[must_use] #[inline] - pub fn from_secs_f64(secs: f64) -> Duration { - const MAX_NANOS_F64: f64 = ((u64::MAX as u128 + 1) * (NANOS_PER_SEC as u128)) as f64; - let nanos = secs * (NANOS_PER_SEC as f64); - if !nanos.is_finite() { - panic!("got non-finite value when converting float to duration"); - } - if nanos >= MAX_NANOS_F64 { - panic!("overflow when converting float to duration"); - } - if nanos < 0.0 { - panic!("underflow when converting float to duration"); - } - let nanos = nanos as u128; - Duration { - secs: (nanos / (NANOS_PER_SEC as u128)) as u64, - nanos: (nanos % (NANOS_PER_SEC as u128)) as u32, + #[rustc_const_unstable(feature = "duration_consts_float", issue = "72440")] + pub const fn from_secs_f64(secs: f64) -> Duration { + match Duration::try_from_secs_f64(secs) { + Ok(v) => v, + Err(e) => panic!("{}", e.description()), } } @@ -582,40 +747,44 @@ impl Duration { /// as `f32`. /// /// # Panics - /// This constructor will panic if `secs` is not finite, negative or overflows `Duration`. + /// This constructor will panic if `secs` is negative, overflows `Duration` or not finite. /// /// # Examples /// ``` /// use std::time::Duration; /// - /// let dur = Duration::from_secs_f32(2.7); - /// assert_eq!(dur, Duration::new(2, 700_000_000)); + /// let res = Duration::from_secs_f32(0.0); + /// assert_eq!(res, Duration::new(0, 0)); + /// let res = Duration::from_secs_f32(1e-20); + /// assert_eq!(res, Duration::new(0, 0)); + /// let res = Duration::from_secs_f32(4.2e-7); + /// assert_eq!(res, Duration::new(0, 420)); + /// let res = Duration::from_secs_f32(2.7); + /// assert_eq!(res, Duration::new(2, 700_000_048)); + /// let res = Duration::from_secs_f32(3e10); + /// assert_eq!(res, Duration::new(30_000_001_024, 0)); + /// // subnormal float + /// let res = Duration::from_secs_f32(f32::from_bits(1)); + /// assert_eq!(res, Duration::new(0, 0)); + /// // conversion uses rounding + /// let res = Duration::from_secs_f32(0.999e-9); + /// assert_eq!(res, Duration::new(0, 1)); /// ``` #[stable(feature = "duration_float", since = "1.38.0")] + #[must_use] #[inline] - pub fn from_secs_f32(secs: f32) -> Duration { - const MAX_NANOS_F32: f32 = ((u64::MAX as u128 + 1) * (NANOS_PER_SEC as u128)) as f32; - let nanos = secs * (NANOS_PER_SEC as f32); - if !nanos.is_finite() { - panic!("got non-finite value when converting float to duration"); - } - if nanos >= MAX_NANOS_F32 { - panic!("overflow when converting float to duration"); - } - if nanos < 0.0 { - panic!("underflow when converting float to duration"); - } - let nanos = nanos as u128; - Duration { - secs: (nanos / (NANOS_PER_SEC as u128)) as u64, - nanos: (nanos % (NANOS_PER_SEC as u128)) as u32, + #[rustc_const_unstable(feature = "duration_consts_float", issue = "72440")] + pub const fn from_secs_f32(secs: f32) -> Duration { + match Duration::try_from_secs_f32(secs) { + Ok(v) => v, + Err(e) => panic!("{}", e.description()), } } /// Multiplies `Duration` by `f64`. /// /// # Panics - /// This method will panic if result is not finite, negative or overflows `Duration`. + /// This method will panic if result is negative, overflows `Duration` or not finite. /// /// # Examples /// ``` @@ -626,36 +795,40 @@ impl Duration { /// assert_eq!(dur.mul_f64(3.14e5), Duration::new(847_800, 0)); /// ``` #[stable(feature = "duration_float", since = "1.38.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[inline] - pub fn mul_f64(self, rhs: f64) -> Duration { + #[rustc_const_unstable(feature = "duration_consts_float", issue = "72440")] + pub const fn mul_f64(self, rhs: f64) -> Duration { Duration::from_secs_f64(rhs * self.as_secs_f64()) } /// Multiplies `Duration` by `f32`. /// /// # Panics - /// This method will panic if result is not finite, negative or overflows `Duration`. + /// This method will panic if result is negative, overflows `Duration` or not finite. /// /// # Examples /// ``` /// use std::time::Duration; /// /// let dur = Duration::new(2, 700_000_000); - /// // note that due to rounding errors result is slightly different - /// // from 8.478 and 847800.0 - /// assert_eq!(dur.mul_f32(3.14), Duration::new(8, 478_000_640)); - /// assert_eq!(dur.mul_f32(3.14e5), Duration::new(847799, 969_120_256)); + /// assert_eq!(dur.mul_f32(3.14), Duration::new(8, 478_000_641)); + /// assert_eq!(dur.mul_f32(3.14e5), Duration::new(847800, 0)); /// ``` #[stable(feature = "duration_float", since = "1.38.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[inline] - pub fn mul_f32(self, rhs: f32) -> Duration { + #[rustc_const_unstable(feature = "duration_consts_float", issue = "72440")] + pub const fn mul_f32(self, rhs: f32) -> Duration { Duration::from_secs_f32(rhs * self.as_secs_f32()) } /// Divide `Duration` by `f64`. /// /// # Panics - /// This method will panic if result is not finite, negative or overflows `Duration`. + /// This method will panic if result is negative, overflows `Duration` or not finite. /// /// # Examples /// ``` @@ -663,19 +836,21 @@ impl Duration { /// /// let dur = Duration::new(2, 700_000_000); /// assert_eq!(dur.div_f64(3.14), Duration::new(0, 859_872_611)); - /// // note that truncation is used, not rounding - /// assert_eq!(dur.div_f64(3.14e5), Duration::new(0, 8_598)); + /// assert_eq!(dur.div_f64(3.14e5), Duration::new(0, 8_599)); /// ``` #[stable(feature = "duration_float", since = "1.38.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[inline] - pub fn div_f64(self, rhs: f64) -> Duration { + #[rustc_const_unstable(feature = "duration_consts_float", issue = "72440")] + pub const fn div_f64(self, rhs: f64) -> Duration { Duration::from_secs_f64(self.as_secs_f64() / rhs) } /// Divide `Duration` by `f32`. /// /// # Panics - /// This method will panic if result is not finite, negative or overflows `Duration`. + /// This method will panic if result is negative, overflows `Duration` or not finite. /// /// # Examples /// ``` @@ -684,13 +859,15 @@ impl Duration { /// let dur = Duration::new(2, 700_000_000); /// // note that due to rounding errors result is slightly /// // different from 0.859_872_611 - /// assert_eq!(dur.div_f32(3.14), Duration::new(0, 859_872_576)); - /// // note that truncation is used, not rounding - /// assert_eq!(dur.div_f32(3.14e5), Duration::new(0, 8_598)); + /// assert_eq!(dur.div_f32(3.14), Duration::new(0, 859_872_580)); + /// assert_eq!(dur.div_f32(3.14e5), Duration::new(0, 8_599)); /// ``` #[stable(feature = "duration_float", since = "1.38.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[inline] - pub fn div_f32(self, rhs: f32) -> Duration { + #[rustc_const_unstable(feature = "duration_consts_float", issue = "72440")] + pub const fn div_f32(self, rhs: f32) -> Duration { Duration::from_secs_f32(self.as_secs_f32() / rhs) } @@ -706,8 +883,11 @@ impl Duration { /// assert_eq!(dur1.div_duration_f64(dur2), 0.5); /// ``` #[unstable(feature = "div_duration", issue = "63139")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[inline] - pub fn div_duration_f64(self, rhs: Duration) -> f64 { + #[rustc_const_unstable(feature = "duration_consts_float", issue = "72440")] + pub const fn div_duration_f64(self, rhs: Duration) -> f64 { self.as_secs_f64() / rhs.as_secs_f64() } @@ -723,8 +903,11 @@ impl Duration { /// assert_eq!(dur1.div_duration_f32(dur2), 0.5); /// ``` #[unstable(feature = "div_duration", issue = "63139")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[inline] - pub fn div_duration_f32(self, rhs: Duration) -> f32 { + #[rustc_const_unstable(feature = "duration_consts_float", issue = "72440")] + pub const fn div_duration_f32(self, rhs: Duration) -> f32 { self.as_secs_f32() / rhs.as_secs_f32() } } @@ -810,13 +993,13 @@ macro_rules! sum_durations { for entry in $iter { total_secs = total_secs.checked_add(entry.secs).expect("overflow in iter::sum over durations"); - total_nanos = match total_nanos.checked_add(entry.nanos as u64) { + total_nanos = match total_nanos.checked_add(entry.nanos.0 as u64) { Some(n) => n, None => { total_secs = total_secs .checked_add(total_nanos / NANOS_PER_SEC as u64) .expect("overflow in iter::sum over durations"); - (total_nanos % NANOS_PER_SEC as u64) + entry.nanos as u64 + (total_nanos % NANOS_PER_SEC as u64) + entry.nanos.0 as u64 } }; } @@ -824,7 +1007,7 @@ macro_rules! sum_durations { .checked_add(total_nanos / NANOS_PER_SEC as u64) .expect("overflow in iter::sum over durations"); total_nanos = total_nanos % NANOS_PER_SEC as u64; - Duration { secs: total_secs, nanos: total_nanos as u32 } + Duration::new(total_secs, total_nanos as u32) }}; } @@ -855,11 +1038,16 @@ impl fmt::Debug for Duration { /// `divisor` must not be above 100_000_000. It also should be a power /// of 10, everything else doesn't make sense. `fractional_part` has /// to be less than `10 * divisor`! + /// + /// A prefix and postfix may be added. The whole thing is padded + /// to the formatter's `width`, if specified. fn fmt_decimal( f: &mut fmt::Formatter<'_>, - mut integer_part: u64, + integer_part: u64, mut fractional_part: u32, mut divisor: u32, + prefix: &str, + postfix: &str, ) -> fmt::Result { // Encode the fractional part into a temporary buffer. The buffer // only need to hold 9 elements, because `fractional_part` has to @@ -887,7 +1075,7 @@ impl fmt::Debug for Duration { // normal floating point numbers. However, we only need to do work // when rounding up. This happens if the first digit of the // remaining ones is >= 5. - if fractional_part > 0 && fractional_part >= divisor * 5 { + let integer_part = if fractional_part > 0 && fractional_part >= divisor * 5 { // Round up the number contained in the buffer. We go through // the buffer backwards and keep track of the carry. let mut rev_pos = pos; @@ -911,47 +1099,396 @@ impl fmt::Debug for Duration { // the whole buffer to '0's and need to increment the integer // part. if carry { - integer_part += 1; + // If `integer_part == u64::MAX` and precision < 9, any + // carry of the overflow during rounding of the + // `fractional_part` into the `integer_part` will cause the + // `integer_part` itself to overflow. Avoid this by using an + // `Option`, with `None` representing `u64::MAX + 1`. + integer_part.checked_add(1) + } else { + Some(integer_part) } - } + } else { + Some(integer_part) + }; // Determine the end of the buffer: if precision is set, we just // use as many digits from the buffer (capped to 9). If it isn't // set, we only use all digits up to the last non-zero one. let end = f.precision().map(|p| crate::cmp::min(p, 9)).unwrap_or(pos); - // If we haven't emitted a single fractional digit and the precision - // wasn't set to a non-zero value, we don't print the decimal point. - if end == 0 { - write!(f, "{}", integer_part) - } else { - // SAFETY: We are only writing ASCII digits into the buffer and it was - // initialized with '0's, so it contains valid UTF8. - let s = unsafe { crate::str::from_utf8_unchecked(&buf[..end]) }; + // This closure emits the formatted duration without emitting any + // padding (padding is calculated below). + let emit_without_padding = |f: &mut fmt::Formatter<'_>| { + if let Some(integer_part) = integer_part { + write!(f, "{}{}", prefix, integer_part)?; + } else { + // u64::MAX + 1 == 18446744073709551616 + write!(f, "{}18446744073709551616", prefix)?; + } + + // Write the decimal point and the fractional part (if any). + if end > 0 { + // SAFETY: We are only writing ASCII digits into the buffer and + // it was initialized with '0's, so it contains valid UTF8. + let s = unsafe { crate::str::from_utf8_unchecked(&buf[..end]) }; - // If the user request a precision > 9, we pad '0's at the end. - let w = f.precision().unwrap_or(pos); - write!(f, "{}.{:0 9, we pad '0's at the end. + let w = f.precision().unwrap_or(pos); + write!(f, ".{:0 { + // No `width` specified. There's no need to calculate the + // length of the output in this case, just emit it. + emit_without_padding(f) + } + Some(requested_w) => { + // A `width` was specified. Calculate the actual width of + // the output in order to calculate the required padding. + // It consists of 4 parts: + // 1. The prefix: is either "+" or "", so we can just use len(). + // 2. The postfix: can be "µs" so we have to count UTF8 characters. + let mut actual_w = prefix.len() + postfix.chars().count(); + // 3. The integer part: + if let Some(integer_part) = integer_part { + if let Some(log) = integer_part.checked_ilog10() { + // integer_part is > 0, so has length log10(x)+1 + actual_w += 1 + log as usize; + } else { + // integer_part is 0, so has length 1. + actual_w += 1; + } + } else { + // integer_part is u64::MAX + 1, so has length 20 + actual_w += 20; + } + // 4. The fractional part (if any): + if end > 0 { + let frac_part_w = f.precision().unwrap_or(pos); + actual_w += 1 + frac_part_w; + } + + if requested_w <= actual_w { + // Output is already longer than `width`, so don't pad. + emit_without_padding(f) + } else { + // We need to add padding. Use the `Formatter::padding` helper function. + let default_align = crate::fmt::rt::v1::Alignment::Left; + let post_padding = f.padding(requested_w - actual_w, default_align)?; + emit_without_padding(f)?; + post_padding.write(f) + } + } } } // Print leading '+' sign if requested - if f.sign_plus() { - write!(f, "+")?; - } + let prefix = if f.sign_plus() { "+" } else { "" }; if self.secs > 0 { - fmt_decimal(f, self.secs, self.nanos, 100_000_000)?; - f.write_str("s") - } else if self.nanos >= 1_000_000 { - fmt_decimal(f, self.nanos as u64 / 1_000_000, self.nanos % 1_000_000, 100_000)?; - f.write_str("ms") - } else if self.nanos >= 1_000 { - fmt_decimal(f, self.nanos as u64 / 1_000, self.nanos % 1_000, 100)?; - f.write_str("µs") + fmt_decimal(f, self.secs, self.nanos.0, NANOS_PER_SEC / 10, prefix, "s") + } else if self.nanos.0 >= NANOS_PER_MILLI { + fmt_decimal( + f, + (self.nanos.0 / NANOS_PER_MILLI) as u64, + self.nanos.0 % NANOS_PER_MILLI, + NANOS_PER_MILLI / 10, + prefix, + "ms", + ) + } else if self.nanos.0 >= NANOS_PER_MICRO { + fmt_decimal( + f, + (self.nanos.0 / NANOS_PER_MICRO) as u64, + self.nanos.0 % NANOS_PER_MICRO, + NANOS_PER_MICRO / 10, + prefix, + "µs", + ) } else { - fmt_decimal(f, self.nanos as u64, 0, 1)?; - f.write_str("ns") + fmt_decimal(f, self.nanos.0 as u64, 0, 1, prefix, "ns") } } } + +/// An error which can be returned when converting a floating-point value of seconds +/// into a [`Duration`]. +/// +/// This error is used as the error type for [`Duration::try_from_secs_f32`] and +/// [`Duration::try_from_secs_f64`]. +/// +/// # Example +/// +/// ``` +/// use std::time::Duration; +/// +/// if let Err(e) = Duration::try_from_secs_f32(-1.0) { +/// println!("Failed conversion to Duration: {e}"); +/// } +/// ``` +#[derive(Debug, Clone, PartialEq, Eq)] +#[stable(feature = "duration_checked_float", since = "1.66.0")] +pub struct TryFromFloatSecsError { + kind: TryFromFloatSecsErrorKind, +} + +impl TryFromFloatSecsError { + const fn description(&self) -> &'static str { + match self.kind { + TryFromFloatSecsErrorKind::Negative => { + "can not convert float seconds to Duration: value is negative" + } + TryFromFloatSecsErrorKind::OverflowOrNan => { + "can not convert float seconds to Duration: value is either too big or NaN" + } + } + } +} + +#[stable(feature = "duration_checked_float", since = "1.66.0")] +impl fmt::Display for TryFromFloatSecsError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.description().fmt(f) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum TryFromFloatSecsErrorKind { + // Value is negative. + Negative, + // Value is either too big to be represented as `Duration` or `NaN`. + OverflowOrNan, +} + +macro_rules! try_from_secs { + ( + secs = $secs: expr, + mantissa_bits = $mant_bits: literal, + exponent_bits = $exp_bits: literal, + offset = $offset: literal, + bits_ty = $bits_ty:ty, + double_ty = $double_ty:ty, + ) => {{ + const MIN_EXP: i16 = 1 - (1i16 << $exp_bits) / 2; + const MANT_MASK: $bits_ty = (1 << $mant_bits) - 1; + const EXP_MASK: $bits_ty = (1 << $exp_bits) - 1; + + if $secs < 0.0 { + return Err(TryFromFloatSecsError { kind: TryFromFloatSecsErrorKind::Negative }); + } + + let bits = $secs.to_bits(); + let mant = (bits & MANT_MASK) | (MANT_MASK + 1); + let exp = ((bits >> $mant_bits) & EXP_MASK) as i16 + MIN_EXP; + + let (secs, nanos) = if exp < -31 { + // the input represents less than 1ns and can not be rounded to it + (0u64, 0u32) + } else if exp < 0 { + // the input is less than 1 second + let t = <$double_ty>::from(mant) << ($offset + exp); + let nanos_offset = $mant_bits + $offset; + let nanos_tmp = u128::from(NANOS_PER_SEC) * u128::from(t); + let nanos = (nanos_tmp >> nanos_offset) as u32; + + let rem_mask = (1 << nanos_offset) - 1; + let rem_msb_mask = 1 << (nanos_offset - 1); + let rem = nanos_tmp & rem_mask; + let is_tie = rem == rem_msb_mask; + let is_even = (nanos & 1) == 0; + let rem_msb = nanos_tmp & rem_msb_mask == 0; + let add_ns = !(rem_msb || (is_even && is_tie)); + + // f32 does not have enough precision to trigger the second branch + // since it can not represent numbers between 0.999_999_940_395 and 1.0. + let nanos = nanos + add_ns as u32; + if ($mant_bits == 23) || (nanos != NANOS_PER_SEC) { (0, nanos) } else { (1, 0) } + } else if exp < $mant_bits { + let secs = u64::from(mant >> ($mant_bits - exp)); + let t = <$double_ty>::from((mant << exp) & MANT_MASK); + let nanos_offset = $mant_bits; + let nanos_tmp = <$double_ty>::from(NANOS_PER_SEC) * t; + let nanos = (nanos_tmp >> nanos_offset) as u32; + + let rem_mask = (1 << nanos_offset) - 1; + let rem_msb_mask = 1 << (nanos_offset - 1); + let rem = nanos_tmp & rem_mask; + let is_tie = rem == rem_msb_mask; + let is_even = (nanos & 1) == 0; + let rem_msb = nanos_tmp & rem_msb_mask == 0; + let add_ns = !(rem_msb || (is_even && is_tie)); + + // f32 does not have enough precision to trigger the second branch. + // For example, it can not represent numbers between 1.999_999_880... + // and 2.0. Bigger values result in even smaller precision of the + // fractional part. + let nanos = nanos + add_ns as u32; + if ($mant_bits == 23) || (nanos != NANOS_PER_SEC) { + (secs, nanos) + } else { + (secs + 1, 0) + } + } else if exp < 64 { + // the input has no fractional part + let secs = u64::from(mant) << (exp - $mant_bits); + (secs, 0) + } else { + return Err(TryFromFloatSecsError { kind: TryFromFloatSecsErrorKind::OverflowOrNan }); + }; + + Ok(Duration::new(secs, nanos)) + }}; +} + +impl Duration { + /// The checked version of [`from_secs_f32`]. + /// + /// [`from_secs_f32`]: Duration::from_secs_f32 + /// + /// This constructor will return an `Err` if `secs` is negative, overflows `Duration` or not finite. + /// + /// # Examples + /// ``` + /// use std::time::Duration; + /// + /// let res = Duration::try_from_secs_f32(0.0); + /// assert_eq!(res, Ok(Duration::new(0, 0))); + /// let res = Duration::try_from_secs_f32(1e-20); + /// assert_eq!(res, Ok(Duration::new(0, 0))); + /// let res = Duration::try_from_secs_f32(4.2e-7); + /// assert_eq!(res, Ok(Duration::new(0, 420))); + /// let res = Duration::try_from_secs_f32(2.7); + /// assert_eq!(res, Ok(Duration::new(2, 700_000_048))); + /// let res = Duration::try_from_secs_f32(3e10); + /// assert_eq!(res, Ok(Duration::new(30_000_001_024, 0))); + /// // subnormal float: + /// let res = Duration::try_from_secs_f32(f32::from_bits(1)); + /// assert_eq!(res, Ok(Duration::new(0, 0))); + /// + /// let res = Duration::try_from_secs_f32(-5.0); + /// assert!(res.is_err()); + /// let res = Duration::try_from_secs_f32(f32::NAN); + /// assert!(res.is_err()); + /// let res = Duration::try_from_secs_f32(2e19); + /// assert!(res.is_err()); + /// + /// // the conversion uses rounding with tie resolution to even + /// let res = Duration::try_from_secs_f32(0.999e-9); + /// assert_eq!(res, Ok(Duration::new(0, 1))); + /// + /// // this float represents exactly 976562.5e-9 + /// let val = f32::from_bits(0x3A80_0000); + /// let res = Duration::try_from_secs_f32(val); + /// assert_eq!(res, Ok(Duration::new(0, 976_562))); + /// + /// // this float represents exactly 2929687.5e-9 + /// let val = f32::from_bits(0x3B40_0000); + /// let res = Duration::try_from_secs_f32(val); + /// assert_eq!(res, Ok(Duration::new(0, 2_929_688))); + /// + /// // this float represents exactly 1.000_976_562_5 + /// let val = f32::from_bits(0x3F802000); + /// let res = Duration::try_from_secs_f32(val); + /// assert_eq!(res, Ok(Duration::new(1, 976_562))); + /// + /// // this float represents exactly 1.002_929_687_5 + /// let val = f32::from_bits(0x3F806000); + /// let res = Duration::try_from_secs_f32(val); + /// assert_eq!(res, Ok(Duration::new(1, 2_929_688))); + /// ``` + #[stable(feature = "duration_checked_float", since = "1.66.0")] + #[rustc_const_unstable(feature = "duration_consts_float", issue = "72440")] + #[inline] + pub const fn try_from_secs_f32(secs: f32) -> Result { + try_from_secs!( + secs = secs, + mantissa_bits = 23, + exponent_bits = 8, + offset = 41, + bits_ty = u32, + double_ty = u64, + ) + } + + /// The checked version of [`from_secs_f64`]. + /// + /// [`from_secs_f64`]: Duration::from_secs_f64 + /// + /// This constructor will return an `Err` if `secs` is negative, overflows `Duration` or not finite. + /// + /// # Examples + /// ``` + /// use std::time::Duration; + /// + /// let res = Duration::try_from_secs_f64(0.0); + /// assert_eq!(res, Ok(Duration::new(0, 0))); + /// let res = Duration::try_from_secs_f64(1e-20); + /// assert_eq!(res, Ok(Duration::new(0, 0))); + /// let res = Duration::try_from_secs_f64(4.2e-7); + /// assert_eq!(res, Ok(Duration::new(0, 420))); + /// let res = Duration::try_from_secs_f64(2.7); + /// assert_eq!(res, Ok(Duration::new(2, 700_000_000))); + /// let res = Duration::try_from_secs_f64(3e10); + /// assert_eq!(res, Ok(Duration::new(30_000_000_000, 0))); + /// // subnormal float + /// let res = Duration::try_from_secs_f64(f64::from_bits(1)); + /// assert_eq!(res, Ok(Duration::new(0, 0))); + /// + /// let res = Duration::try_from_secs_f64(-5.0); + /// assert!(res.is_err()); + /// let res = Duration::try_from_secs_f64(f64::NAN); + /// assert!(res.is_err()); + /// let res = Duration::try_from_secs_f64(2e19); + /// assert!(res.is_err()); + /// + /// // the conversion uses rounding with tie resolution to even + /// let res = Duration::try_from_secs_f64(0.999e-9); + /// assert_eq!(res, Ok(Duration::new(0, 1))); + /// let res = Duration::try_from_secs_f64(0.999_999_999_499); + /// assert_eq!(res, Ok(Duration::new(0, 999_999_999))); + /// let res = Duration::try_from_secs_f64(0.999_999_999_501); + /// assert_eq!(res, Ok(Duration::new(1, 0))); + /// let res = Duration::try_from_secs_f64(42.999_999_999_499); + /// assert_eq!(res, Ok(Duration::new(42, 999_999_999))); + /// let res = Duration::try_from_secs_f64(42.999_999_999_501); + /// assert_eq!(res, Ok(Duration::new(43, 0))); + /// + /// // this float represents exactly 976562.5e-9 + /// let val = f64::from_bits(0x3F50_0000_0000_0000); + /// let res = Duration::try_from_secs_f64(val); + /// assert_eq!(res, Ok(Duration::new(0, 976_562))); + /// + /// // this float represents exactly 2929687.5e-9 + /// let val = f64::from_bits(0x3F68_0000_0000_0000); + /// let res = Duration::try_from_secs_f64(val); + /// assert_eq!(res, Ok(Duration::new(0, 2_929_688))); + /// + /// // this float represents exactly 1.000_976_562_5 + /// let val = f64::from_bits(0x3FF0_0400_0000_0000); + /// let res = Duration::try_from_secs_f64(val); + /// assert_eq!(res, Ok(Duration::new(1, 976_562))); + /// + /// // this float represents exactly 1.002_929_687_5 + /// let val = f64::from_bits(0x3_FF00_C000_0000_000); + /// let res = Duration::try_from_secs_f64(val); + /// assert_eq!(res, Ok(Duration::new(1, 2_929_688))); + /// ``` + #[stable(feature = "duration_checked_float", since = "1.66.0")] + #[rustc_const_unstable(feature = "duration_consts_float", issue = "72440")] + #[inline] + pub const fn try_from_secs_f64(secs: f64) -> Result { + try_from_secs!( + secs = secs, + mantissa_bits = 52, + exponent_bits = 11, + offset = 44, + bits_ty = u64, + double_ty = u128, + ) + } +} diff --git a/crux-mir/lib/core/src/tuple.rs b/crux-mir/lib/core/src/tuple.rs index 9f8a3a1de..28275798f 100644 --- a/crux-mir/lib/core/src/tuple.rs +++ b/crux-mir/lib/core/src/tuple.rs @@ -3,73 +3,126 @@ use crate::cmp::Ordering::*; use crate::cmp::*; -// macro for implementing n-ary tuple functions and operations +// Recursive macro for implementing n-ary tuple functions and operations +// +// Also provides implementations for tuples with lesser arity. For example, tuple_impls!(A B C) +// will implement everything for (A, B, C), (A, B) and (A,). macro_rules! tuple_impls { - ($( - $Tuple:ident { - $(($idx:tt) -> $T:ident)+ - } - )+) => { - $( + // Stopping criteria (1-ary tuple) + ($T:ident) => { + tuple_impls!(@impl $T); + }; + // Running criteria (n-ary tuple, with n >= 2) + ($T:ident $( $U:ident )+) => { + tuple_impls!($( $U )+); + tuple_impls!(@impl $T $( $U )+); + }; + // "Private" internal implementation + (@impl $( $T:ident )+) => { + maybe_tuple_doc! { + $($T)+ @ #[stable(feature = "rust1", since = "1.0.0")] - impl<$($T:PartialEq),+> PartialEq for ($($T,)+) where last_type!($($T,)+): ?Sized { + #[rustc_const_unstable(feature = "const_cmp", issue = "92391")] + impl<$($T: ~const PartialEq),+> const PartialEq for ($($T,)+) + where + last_type!($($T,)+): ?Sized + { #[inline] fn eq(&self, other: &($($T,)+)) -> bool { - $(self.$idx == other.$idx)&&+ + $( ${ignore(T)} self.${index()} == other.${index()} )&&+ } #[inline] fn ne(&self, other: &($($T,)+)) -> bool { - $(self.$idx != other.$idx)||+ + $( ${ignore(T)} self.${index()} != other.${index()} )||+ } } + } + maybe_tuple_doc! { + $($T)+ @ #[stable(feature = "rust1", since = "1.0.0")] - impl<$($T:Eq),+> Eq for ($($T,)+) where last_type!($($T,)+): ?Sized {} + impl<$($T: Eq),+> Eq for ($($T,)+) + where + last_type!($($T,)+): ?Sized + {} + } + maybe_tuple_doc! { + $($T)+ @ #[stable(feature = "rust1", since = "1.0.0")] - impl<$($T:PartialOrd + PartialEq),+> PartialOrd for ($($T,)+) - where last_type!($($T,)+): ?Sized { + #[rustc_const_unstable(feature = "const_cmp", issue = "92391")] + impl<$($T: ~const PartialOrd + ~const PartialEq),+> const PartialOrd for ($($T,)+) + where + last_type!($($T,)+): ?Sized + { #[inline] fn partial_cmp(&self, other: &($($T,)+)) -> Option { - lexical_partial_cmp!($(self.$idx, other.$idx),+) + lexical_partial_cmp!($( ${ignore(T)} self.${index()}, other.${index()} ),+) } #[inline] fn lt(&self, other: &($($T,)+)) -> bool { - lexical_ord!(lt, $(self.$idx, other.$idx),+) + lexical_ord!(lt, $( ${ignore(T)} self.${index()}, other.${index()} ),+) } #[inline] fn le(&self, other: &($($T,)+)) -> bool { - lexical_ord!(le, $(self.$idx, other.$idx),+) + lexical_ord!(le, $( ${ignore(T)} self.${index()}, other.${index()} ),+) } #[inline] fn ge(&self, other: &($($T,)+)) -> bool { - lexical_ord!(ge, $(self.$idx, other.$idx),+) + lexical_ord!(ge, $( ${ignore(T)} self.${index()}, other.${index()} ),+) } #[inline] fn gt(&self, other: &($($T,)+)) -> bool { - lexical_ord!(gt, $(self.$idx, other.$idx),+) + lexical_ord!(gt, $( ${ignore(T)} self.${index()}, other.${index()} ),+) } } + } + maybe_tuple_doc! { + $($T)+ @ #[stable(feature = "rust1", since = "1.0.0")] - impl<$($T:Ord),+> Ord for ($($T,)+) where last_type!($($T,)+): ?Sized { + #[rustc_const_unstable(feature = "const_cmp", issue = "92391")] + impl<$($T: ~const Ord),+> const Ord for ($($T,)+) + where + last_type!($($T,)+): ?Sized + { #[inline] fn cmp(&self, other: &($($T,)+)) -> Ordering { - lexical_cmp!($(self.$idx, other.$idx),+) + lexical_cmp!($( ${ignore(T)} self.${index()}, other.${index()} ),+) } } + } + maybe_tuple_doc! { + $($T)+ @ #[stable(feature = "rust1", since = "1.0.0")] - impl<$($T:Default),+> Default for ($($T,)+) { + #[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] + impl<$($T: ~const Default),+> const Default for ($($T,)+) { #[inline] fn default() -> ($($T,)+) { ($({ let x: $T = Default::default(); x},)+) } } - )+ + } } } +// If this is a unary tuple, it adds a doc comment. +// Otherwise, it hides the docs entirely. +macro_rules! maybe_tuple_doc { + ($a:ident @ #[$meta:meta] $item:item) => { + #[doc(fake_variadic)] + #[doc = "This trait is implemented for tuples up to twelve items long."] + #[$meta] + $item + }; + ($a:ident $($rest_a:ident)+ @ #[$meta:meta] $item:item) => { + #[doc(hidden)] + #[$meta] + $item + }; +} + // Constructs an expression that performs a lexical ordering using method $rel. // The values are interleaved, so the macro invocation for // `(a1, a2, a3) < (b1, b2, b3)` would be `lexical_ord!(lt, a1, b1, a2, b2, @@ -107,107 +160,4 @@ macro_rules! last_type { ($a:ident, $($rest_a:ident,)+) => { last_type!($($rest_a,)+) }; } -tuple_impls! { - Tuple1 { - (0) -> A - } - Tuple2 { - (0) -> A - (1) -> B - } - Tuple3 { - (0) -> A - (1) -> B - (2) -> C - } - Tuple4 { - (0) -> A - (1) -> B - (2) -> C - (3) -> D - } - Tuple5 { - (0) -> A - (1) -> B - (2) -> C - (3) -> D - (4) -> E - } - Tuple6 { - (0) -> A - (1) -> B - (2) -> C - (3) -> D - (4) -> E - (5) -> F - } - Tuple7 { - (0) -> A - (1) -> B - (2) -> C - (3) -> D - (4) -> E - (5) -> F - (6) -> G - } - Tuple8 { - (0) -> A - (1) -> B - (2) -> C - (3) -> D - (4) -> E - (5) -> F - (6) -> G - (7) -> H - } - Tuple9 { - (0) -> A - (1) -> B - (2) -> C - (3) -> D - (4) -> E - (5) -> F - (6) -> G - (7) -> H - (8) -> I - } - Tuple10 { - (0) -> A - (1) -> B - (2) -> C - (3) -> D - (4) -> E - (5) -> F - (6) -> G - (7) -> H - (8) -> I - (9) -> J - } - Tuple11 { - (0) -> A - (1) -> B - (2) -> C - (3) -> D - (4) -> E - (5) -> F - (6) -> G - (7) -> H - (8) -> I - (9) -> J - (10) -> K - } - Tuple12 { - (0) -> A - (1) -> B - (2) -> C - (3) -> D - (4) -> E - (5) -> F - (6) -> G - (7) -> H - (8) -> I - (9) -> J - (10) -> K - (11) -> L - } -} +tuple_impls!(E D C B A Z Y X W V U T); diff --git a/crux-mir/lib/core/src/unicode/mod.rs b/crux-mir/lib/core/src/unicode/mod.rs index b6eaf06aa..e1faa407d 100644 --- a/crux-mir/lib/core/src/unicode/mod.rs +++ b/crux-mir/lib/core/src/unicode/mod.rs @@ -3,57 +3,29 @@ pub(crate) mod printable; mod unicode_data; -pub(crate) mod version; -use version::UnicodeVersion; - -/// The version of [Unicode](http://www.unicode.org/) that the Unicode parts of +/// The version of [Unicode](https://www.unicode.org/) that the Unicode parts of /// `char` and `str` methods are based on. -#[unstable(feature = "unicode_version", issue = "49726")] -pub const UNICODE_VERSION: UnicodeVersion = UnicodeVersion { - major: unicode_data::UNICODE_VERSION.0, - minor: unicode_data::UNICODE_VERSION.1, - micro: unicode_data::UNICODE_VERSION.2, - _priv: (), -}; +/// +/// New versions of Unicode are released regularly and subsequently all methods +/// in the standard library depending on Unicode are updated. Therefore the +/// behavior of some `char` and `str` methods and the value of this constant +/// changes over time. This is *not* considered to be a breaking change. +/// +/// The version numbering scheme is explained in +/// [Unicode 11.0 or later, Section 3.1 Versions of the Unicode Standard](https://www.unicode.org/versions/Unicode11.0.0/ch03.pdf#page=4). +#[stable(feature = "unicode_version", since = "1.45.0")] +pub const UNICODE_VERSION: (u8, u8, u8) = unicode_data::UNICODE_VERSION; -// For use in liballoc, not re-exported in libstd. -pub mod derived_property { - pub use super::{Case_Ignorable, Cased}; -} - -pub use unicode_data::alphabetic::lookup as Alphabetic; -pub use unicode_data::case_ignorable::lookup as Case_Ignorable; -pub use unicode_data::cased::lookup as Cased; -pub use unicode_data::cc::lookup as Cc; -pub use unicode_data::conversions; -pub use unicode_data::grapheme_extend::lookup as Grapheme_Extend; -pub use unicode_data::lowercase::lookup as Lowercase; -pub use unicode_data::n::lookup as N; -pub use unicode_data::uppercase::lookup as Uppercase; -pub use unicode_data::white_space::lookup as White_Space; +// For use in alloc, not re-exported in std. +pub use unicode_data::{ + case_ignorable::lookup as Case_Ignorable, cased::lookup as Cased, conversions, +}; -#[inline(always)] -fn range_search( - needle: u32, - chunk_idx_map: &[u8; N], - (last_chunk_idx, last_chunk_mapping): (u16, u8), - bitset_chunk_idx: &[[u8; 16]; N1], - bitset: &[u64; N2], -) -> bool { - let bucket_idx = (needle / 64) as usize; - let chunk_map_idx = bucket_idx / 16; - let chunk_piece = bucket_idx % 16; - let chunk_idx = if chunk_map_idx >= N { - if chunk_map_idx == last_chunk_idx as usize { - last_chunk_mapping - } else { - return false; - } - } else { - chunk_idx_map[chunk_map_idx] - }; - let idx = bitset_chunk_idx[(chunk_idx as usize)][chunk_piece]; - let word = bitset[(idx as usize)]; - (word & (1 << (needle % 64) as u64)) != 0 -} +pub(crate) use unicode_data::alphabetic::lookup as Alphabetic; +pub(crate) use unicode_data::cc::lookup as Cc; +pub(crate) use unicode_data::grapheme_extend::lookup as Grapheme_Extend; +pub(crate) use unicode_data::lowercase::lookup as Lowercase; +pub(crate) use unicode_data::n::lookup as N; +pub(crate) use unicode_data::uppercase::lookup as Uppercase; +pub(crate) use unicode_data::white_space::lookup as White_Space; diff --git a/crux-mir/lib/core/src/unicode/printable.py b/crux-mir/lib/core/src/unicode/printable.py index 91db6381c..7c37f5f09 100755 --- a/crux-mir/lib/core/src/unicode/printable.py +++ b/crux-mir/lib/core/src/unicode/printable.py @@ -130,7 +130,7 @@ def print_normal(normal, normalname): print("];") def main(): - file = get_file("http://www.unicode.org/Public/UNIDATA/UnicodeData.txt") + file = get_file("https://www.unicode.org/Public/UNIDATA/UnicodeData.txt") codepoints = get_codepoints(file) @@ -170,7 +170,7 @@ def main(): normal1 = compress_normal(normal1) print("""\ -// NOTE: The following code was generated by "src/libcore/unicode/printable.py", +// NOTE: The following code was generated by "library/core/src/unicode/printable.py", // do not edit directly! fn check(x: u16, singletonuppers: &[(u8, u8)], singletonlowers: &[u8], normal: &[u8]) -> bool { @@ -211,7 +211,14 @@ def main(): pub(crate) fn is_printable(x: char) -> bool { let x = x as u32; let lower = x as u16; - if x < 0x10000 { + + if x < 32 { + // ASCII fast path + false + } else if x < 127 { + // ASCII fast path + true + } else if x < 0x10000 { check(lower, SINGLETONS0U, SINGLETONS0L, NORMAL0) } else if x < 0x20000 { check(lower, SINGLETONS1U, SINGLETONS1L, NORMAL1) diff --git a/crux-mir/lib/core/src/unicode/printable.rs b/crux-mir/lib/core/src/unicode/printable.rs index 9680aa14d..ffb18a5ba 100644 --- a/crux-mir/lib/core/src/unicode/printable.rs +++ b/crux-mir/lib/core/src/unicode/printable.rs @@ -1,4 +1,4 @@ -// NOTE: The following code was generated by "src/libcore/unicode/printable.py", +// NOTE: The following code was generated by "library/core/src/unicode/printable.py", // do not edit directly! fn check(x: u16, singletonuppers: &[(u8, u8)], singletonlowers: &[u8], normal: &[u8]) -> bool { @@ -39,15 +39,22 @@ fn check(x: u16, singletonuppers: &[(u8, u8)], singletonlowers: &[u8], normal: & pub(crate) fn is_printable(x: char) -> bool { let x = x as u32; let lower = x as u16; - if x < 0x10000 { + + if x < 32 { + // ASCII fast path + false + } else if x < 127 { + // ASCII fast path + true + } else if x < 0x10000 { check(lower, SINGLETONS0U, SINGLETONS0L, NORMAL0) } else if x < 0x20000 { check(lower, SINGLETONS1U, SINGLETONS1L, NORMAL1) } else { - if 0x2a6de <= x && x < 0x2a700 { + if 0x2a6e0 <= x && x < 0x2a700 { return false; } - if 0x2b735 <= x && x < 0x2b740 { + if 0x2b73a <= x && x < 0x2b740 { return false; } if 0x2b81e <= x && x < 0x2b820 { @@ -62,7 +69,10 @@ pub(crate) fn is_printable(x: char) -> bool { if 0x2fa1e <= x && x < 0x30000 { return false; } - if 0x3134b <= x && x < 0xe0100 { + if 0x3134b <= x && x < 0x31350 { + return false; + } + if 0x323b0 <= x && x < 0xe0100 { return false; } if 0xe01f0 <= x && x < 0x110000 { @@ -77,30 +87,29 @@ const SINGLETONS0U: &[(u8, u8)] = &[ (0x00, 1), (0x03, 5), (0x05, 6), - (0x06, 3), + (0x06, 2), (0x07, 6), - (0x08, 8), + (0x08, 7), (0x09, 17), (0x0a, 28), (0x0b, 25), - (0x0c, 20), + (0x0c, 26), (0x0d, 16), - (0x0e, 13), + (0x0e, 12), (0x0f, 4), (0x10, 3), (0x12, 18), (0x13, 9), (0x16, 1), - (0x17, 5), - (0x18, 2), + (0x17, 4), + (0x18, 1), (0x19, 3), (0x1a, 7), + (0x1b, 1), (0x1c, 2), - (0x1d, 1), (0x1f, 22), (0x20, 3), (0x2b, 3), - (0x2c, 2), (0x2d, 11), (0x2e, 1), (0x30, 3), @@ -112,49 +121,48 @@ const SINGLETONS0U: &[(u8, u8)] = &[ (0xab, 8), (0xfa, 2), (0xfb, 5), - (0xfd, 4), + (0xfd, 2), (0xfe, 3), (0xff, 9), ]; #[rustfmt::skip] const SINGLETONS0L: &[u8] = &[ 0xad, 0x78, 0x79, 0x8b, 0x8d, 0xa2, 0x30, 0x57, - 0x58, 0x8b, 0x8c, 0x90, 0x1c, 0x1d, 0xdd, 0x0e, - 0x0f, 0x4b, 0x4c, 0xfb, 0xfc, 0x2e, 0x2f, 0x3f, - 0x5c, 0x5d, 0x5f, 0xb5, 0xe2, 0x84, 0x8d, 0x8e, - 0x91, 0x92, 0xa9, 0xb1, 0xba, 0xbb, 0xc5, 0xc6, - 0xc9, 0xca, 0xde, 0xe4, 0xe5, 0xff, 0x00, 0x04, - 0x11, 0x12, 0x29, 0x31, 0x34, 0x37, 0x3a, 0x3b, - 0x3d, 0x49, 0x4a, 0x5d, 0x84, 0x8e, 0x92, 0xa9, - 0xb1, 0xb4, 0xba, 0xbb, 0xc6, 0xca, 0xce, 0xcf, - 0xe4, 0xe5, 0x00, 0x04, 0x0d, 0x0e, 0x11, 0x12, - 0x29, 0x31, 0x34, 0x3a, 0x3b, 0x45, 0x46, 0x49, - 0x4a, 0x5e, 0x64, 0x65, 0x84, 0x91, 0x9b, 0x9d, - 0xc9, 0xce, 0xcf, 0x0d, 0x11, 0x29, 0x45, 0x49, - 0x57, 0x64, 0x65, 0x8d, 0x91, 0xa9, 0xb4, 0xba, - 0xbb, 0xc5, 0xc9, 0xdf, 0xe4, 0xe5, 0xf0, 0x0d, - 0x11, 0x45, 0x49, 0x64, 0x65, 0x80, 0x84, 0xb2, - 0xbc, 0xbe, 0xbf, 0xd5, 0xd7, 0xf0, 0xf1, 0x83, - 0x85, 0x8b, 0xa4, 0xa6, 0xbe, 0xbf, 0xc5, 0xc7, - 0xce, 0xcf, 0xda, 0xdb, 0x48, 0x98, 0xbd, 0xcd, - 0xc6, 0xce, 0xcf, 0x49, 0x4e, 0x4f, 0x57, 0x59, - 0x5e, 0x5f, 0x89, 0x8e, 0x8f, 0xb1, 0xb6, 0xb7, - 0xbf, 0xc1, 0xc6, 0xc7, 0xd7, 0x11, 0x16, 0x17, - 0x5b, 0x5c, 0xf6, 0xf7, 0xfe, 0xff, 0x80, 0x0d, - 0x6d, 0x71, 0xde, 0xdf, 0x0e, 0x0f, 0x1f, 0x6e, - 0x6f, 0x1c, 0x1d, 0x5f, 0x7d, 0x7e, 0xae, 0xaf, - 0xbb, 0xbc, 0xfa, 0x16, 0x17, 0x1e, 0x1f, 0x46, - 0x47, 0x4e, 0x4f, 0x58, 0x5a, 0x5c, 0x5e, 0x7e, - 0x7f, 0xb5, 0xc5, 0xd4, 0xd5, 0xdc, 0xf0, 0xf1, - 0xf5, 0x72, 0x73, 0x8f, 0x74, 0x75, 0x96, 0x2f, - 0x5f, 0x26, 0x2e, 0x2f, 0xa7, 0xaf, 0xb7, 0xbf, - 0xc7, 0xcf, 0xd7, 0xdf, 0x9a, 0x40, 0x97, 0x98, - 0x30, 0x8f, 0x1f, 0xc0, 0xc1, 0xce, 0xff, 0x4e, - 0x4f, 0x5a, 0x5b, 0x07, 0x08, 0x0f, 0x10, 0x27, - 0x2f, 0xee, 0xef, 0x6e, 0x6f, 0x37, 0x3d, 0x3f, - 0x42, 0x45, 0x90, 0x91, 0xfe, 0xff, 0x53, 0x67, - 0x75, 0xc8, 0xc9, 0xd0, 0xd1, 0xd8, 0xd9, 0xe7, - 0xfe, 0xff, + 0x58, 0x8b, 0x8c, 0x90, 0x1c, 0xdd, 0x0e, 0x0f, + 0x4b, 0x4c, 0xfb, 0xfc, 0x2e, 0x2f, 0x3f, 0x5c, + 0x5d, 0x5f, 0xe2, 0x84, 0x8d, 0x8e, 0x91, 0x92, + 0xa9, 0xb1, 0xba, 0xbb, 0xc5, 0xc6, 0xc9, 0xca, + 0xde, 0xe4, 0xe5, 0xff, 0x00, 0x04, 0x11, 0x12, + 0x29, 0x31, 0x34, 0x37, 0x3a, 0x3b, 0x3d, 0x49, + 0x4a, 0x5d, 0x84, 0x8e, 0x92, 0xa9, 0xb1, 0xb4, + 0xba, 0xbb, 0xc6, 0xca, 0xce, 0xcf, 0xe4, 0xe5, + 0x00, 0x04, 0x0d, 0x0e, 0x11, 0x12, 0x29, 0x31, + 0x34, 0x3a, 0x3b, 0x45, 0x46, 0x49, 0x4a, 0x5e, + 0x64, 0x65, 0x84, 0x91, 0x9b, 0x9d, 0xc9, 0xce, + 0xcf, 0x0d, 0x11, 0x29, 0x3a, 0x3b, 0x45, 0x49, + 0x57, 0x5b, 0x5c, 0x5e, 0x5f, 0x64, 0x65, 0x8d, + 0x91, 0xa9, 0xb4, 0xba, 0xbb, 0xc5, 0xc9, 0xdf, + 0xe4, 0xe5, 0xf0, 0x0d, 0x11, 0x45, 0x49, 0x64, + 0x65, 0x80, 0x84, 0xb2, 0xbc, 0xbe, 0xbf, 0xd5, + 0xd7, 0xf0, 0xf1, 0x83, 0x85, 0x8b, 0xa4, 0xa6, + 0xbe, 0xbf, 0xc5, 0xc7, 0xcf, 0xda, 0xdb, 0x48, + 0x98, 0xbd, 0xcd, 0xc6, 0xce, 0xcf, 0x49, 0x4e, + 0x4f, 0x57, 0x59, 0x5e, 0x5f, 0x89, 0x8e, 0x8f, + 0xb1, 0xb6, 0xb7, 0xbf, 0xc1, 0xc6, 0xc7, 0xd7, + 0x11, 0x16, 0x17, 0x5b, 0x5c, 0xf6, 0xf7, 0xfe, + 0xff, 0x80, 0x6d, 0x71, 0xde, 0xdf, 0x0e, 0x1f, + 0x6e, 0x6f, 0x1c, 0x1d, 0x5f, 0x7d, 0x7e, 0xae, + 0xaf, 0x7f, 0xbb, 0xbc, 0x16, 0x17, 0x1e, 0x1f, + 0x46, 0x47, 0x4e, 0x4f, 0x58, 0x5a, 0x5c, 0x5e, + 0x7e, 0x7f, 0xb5, 0xc5, 0xd4, 0xd5, 0xdc, 0xf0, + 0xf1, 0xf5, 0x72, 0x73, 0x8f, 0x74, 0x75, 0x96, + 0x26, 0x2e, 0x2f, 0xa7, 0xaf, 0xb7, 0xbf, 0xc7, + 0xcf, 0xd7, 0xdf, 0x9a, 0x40, 0x97, 0x98, 0x30, + 0x8f, 0x1f, 0xd2, 0xd4, 0xce, 0xff, 0x4e, 0x4f, + 0x5a, 0x5b, 0x07, 0x08, 0x0f, 0x10, 0x27, 0x2f, + 0xee, 0xef, 0x6e, 0x6f, 0x37, 0x3d, 0x3f, 0x42, + 0x45, 0x90, 0x91, 0x53, 0x67, 0x75, 0xc8, 0xc9, + 0xd0, 0xd1, 0xd8, 0xd9, 0xe7, 0xfe, 0xff, ]; #[rustfmt::skip] const SINGLETONS1U: &[(u8, u8)] = &[ @@ -162,6 +170,8 @@ const SINGLETONS1U: &[(u8, u8)] = &[ (0x01, 1), (0x03, 1), (0x04, 2), + (0x05, 7), + (0x07, 2), (0x08, 8), (0x09, 2), (0x0a, 5), @@ -177,10 +187,14 @@ const SINGLETONS1U: &[(u8, u8)] = &[ (0x19, 13), (0x1c, 5), (0x1d, 8), + (0x1f, 1), (0x24, 1), - (0x6a, 3), + (0x6a, 4), (0x6b, 2), + (0xaf, 3), + (0xb1, 2), (0xbc, 2), + (0xcf, 2), (0xd1, 2), (0xd4, 12), (0xd5, 9), @@ -189,38 +203,41 @@ const SINGLETONS1U: &[(u8, u8)] = &[ (0xda, 1), (0xe0, 5), (0xe1, 2), + (0xe7, 4), (0xe8, 2), (0xee, 32), (0xf0, 4), (0xf8, 2), - (0xf9, 2), - (0xfa, 2), + (0xfa, 3), (0xfb, 1), ]; #[rustfmt::skip] const SINGLETONS1L: &[u8] = &[ 0x0c, 0x27, 0x3b, 0x3e, 0x4e, 0x4f, 0x8f, 0x9e, - 0x9e, 0x9f, 0x06, 0x07, 0x09, 0x36, 0x3d, 0x3e, - 0x56, 0xf3, 0xd0, 0xd1, 0x04, 0x14, 0x18, 0x36, - 0x37, 0x56, 0x57, 0x7f, 0xaa, 0xae, 0xaf, 0xbd, - 0x35, 0xe0, 0x12, 0x87, 0x89, 0x8e, 0x9e, 0x04, - 0x0d, 0x0e, 0x11, 0x12, 0x29, 0x31, 0x34, 0x3a, - 0x45, 0x46, 0x49, 0x4a, 0x4e, 0x4f, 0x64, 0x65, - 0x5c, 0xb6, 0xb7, 0x1b, 0x1c, 0x07, 0x08, 0x0a, - 0x0b, 0x14, 0x17, 0x36, 0x39, 0x3a, 0xa8, 0xa9, - 0xd8, 0xd9, 0x09, 0x37, 0x90, 0x91, 0xa8, 0x07, - 0x0a, 0x3b, 0x3e, 0x66, 0x69, 0x8f, 0x92, 0x6f, - 0x5f, 0xee, 0xef, 0x5a, 0x62, 0x9a, 0x9b, 0x27, - 0x28, 0x55, 0x9d, 0xa0, 0xa1, 0xa3, 0xa4, 0xa7, - 0xa8, 0xad, 0xba, 0xbc, 0xc4, 0x06, 0x0b, 0x0c, - 0x15, 0x1d, 0x3a, 0x3f, 0x45, 0x51, 0xa6, 0xa7, - 0xcc, 0xcd, 0xa0, 0x07, 0x19, 0x1a, 0x22, 0x25, - 0x3e, 0x3f, 0xc5, 0xc6, 0x04, 0x20, 0x23, 0x25, - 0x26, 0x28, 0x33, 0x38, 0x3a, 0x48, 0x4a, 0x4c, - 0x50, 0x53, 0x55, 0x56, 0x58, 0x5a, 0x5c, 0x5e, - 0x60, 0x63, 0x65, 0x66, 0x6b, 0x73, 0x78, 0x7d, - 0x7f, 0x8a, 0xa4, 0xaa, 0xaf, 0xb0, 0xc0, 0xd0, - 0xae, 0xaf, 0x79, 0xcc, 0x6e, 0x6f, 0x93, + 0x9e, 0x9f, 0x7b, 0x8b, 0x93, 0x96, 0xa2, 0xb2, + 0xba, 0x86, 0xb1, 0x06, 0x07, 0x09, 0x36, 0x3d, + 0x3e, 0x56, 0xf3, 0xd0, 0xd1, 0x04, 0x14, 0x18, + 0x36, 0x37, 0x56, 0x57, 0x7f, 0xaa, 0xae, 0xaf, + 0xbd, 0x35, 0xe0, 0x12, 0x87, 0x89, 0x8e, 0x9e, + 0x04, 0x0d, 0x0e, 0x11, 0x12, 0x29, 0x31, 0x34, + 0x3a, 0x45, 0x46, 0x49, 0x4a, 0x4e, 0x4f, 0x64, + 0x65, 0x5c, 0xb6, 0xb7, 0x1b, 0x1c, 0x07, 0x08, + 0x0a, 0x0b, 0x14, 0x17, 0x36, 0x39, 0x3a, 0xa8, + 0xa9, 0xd8, 0xd9, 0x09, 0x37, 0x90, 0x91, 0xa8, + 0x07, 0x0a, 0x3b, 0x3e, 0x66, 0x69, 0x8f, 0x92, + 0x11, 0x6f, 0x5f, 0xbf, 0xee, 0xef, 0x5a, 0x62, + 0xf4, 0xfc, 0xff, 0x53, 0x54, 0x9a, 0x9b, 0x2e, + 0x2f, 0x27, 0x28, 0x55, 0x9d, 0xa0, 0xa1, 0xa3, + 0xa4, 0xa7, 0xa8, 0xad, 0xba, 0xbc, 0xc4, 0x06, + 0x0b, 0x0c, 0x15, 0x1d, 0x3a, 0x3f, 0x45, 0x51, + 0xa6, 0xa7, 0xcc, 0xcd, 0xa0, 0x07, 0x19, 0x1a, + 0x22, 0x25, 0x3e, 0x3f, 0xe7, 0xec, 0xef, 0xff, + 0xc5, 0xc6, 0x04, 0x20, 0x23, 0x25, 0x26, 0x28, + 0x33, 0x38, 0x3a, 0x48, 0x4a, 0x4c, 0x50, 0x53, + 0x55, 0x56, 0x58, 0x5a, 0x5c, 0x5e, 0x60, 0x63, + 0x65, 0x66, 0x6b, 0x73, 0x78, 0x7d, 0x7f, 0x8a, + 0xa4, 0xaa, 0xaf, 0xb0, 0xc0, 0xd0, 0xae, 0xaf, + 0x6e, 0x6f, 0xbe, 0x93, ]; #[rustfmt::skip] const NORMAL0: &[u8] = &[ @@ -231,9 +248,9 @@ const NORMAL0: &[u8] = &[ 0x1b, 0x04, 0x06, 0x11, 0x81, 0xac, 0x0e, - 0x80, 0xab, 0x35, - 0x28, 0x0b, - 0x80, 0xe0, 0x03, + 0x80, 0xab, 0x05, + 0x1f, 0x09, + 0x81, 0x1b, 0x03, 0x19, 0x08, 0x01, 0x04, 0x2f, 0x04, @@ -257,13 +274,11 @@ const NORMAL0: &[u8] = &[ 0x0b, 0x06, 0x01, 0x0e, 0x15, 0x05, - 0x3a, 0x03, - 0x11, 0x07, - 0x06, 0x05, - 0x10, 0x07, + 0x4e, 0x07, + 0x1b, 0x07, 0x57, 0x07, - 0x02, 0x07, - 0x15, 0x0d, + 0x02, 0x06, + 0x17, 0x0c, 0x50, 0x04, 0x43, 0x03, 0x2d, 0x03, @@ -280,8 +295,8 @@ const NORMAL0: &[u8] = &[ 0x1a, 0x06, 0x82, 0xfd, 0x03, 0x59, 0x07, - 0x15, 0x0b, - 0x17, 0x09, + 0x16, 0x09, + 0x18, 0x09, 0x14, 0x0c, 0x14, 0x0c, 0x6a, 0x06, @@ -299,10 +314,9 @@ const NORMAL0: &[u8] = &[ 0x0b, 0x03, 0x80, 0xac, 0x06, 0x0a, 0x06, - 0x21, 0x3f, - 0x4c, 0x04, - 0x2d, 0x03, - 0x74, 0x08, + 0x2f, 0x31, + 0x4d, 0x03, + 0x80, 0xa4, 0x08, 0x3c, 0x03, 0x0f, 0x03, 0x3c, 0x07, @@ -312,7 +326,7 @@ const NORMAL0: &[u8] = &[ 0x18, 0x08, 0x2f, 0x11, 0x2d, 0x03, - 0x20, 0x10, + 0x21, 0x0f, 0x21, 0x0f, 0x80, 0x8c, 0x04, 0x82, 0x97, 0x19, @@ -322,19 +336,19 @@ const NORMAL0: &[u8] = &[ 0x3b, 0x07, 0x02, 0x0e, 0x18, 0x09, - 0x80, 0xb3, 0x2d, + 0x80, 0xbe, 0x22, 0x74, 0x0c, 0x80, 0xd6, 0x1a, 0x0c, 0x05, 0x80, 0xff, 0x05, 0x80, 0xdf, 0x0c, - 0xee, 0x0d, 0x03, - 0x84, 0x8d, 0x03, + 0xf2, 0x9d, 0x03, 0x37, 0x09, 0x81, 0x5c, 0x14, 0x80, 0xb8, 0x08, - 0x80, 0xcb, 0x2a, - 0x38, 0x03, + 0x80, 0xcb, 0x05, + 0x0a, 0x18, + 0x3b, 0x03, 0x0a, 0x06, 0x38, 0x08, 0x46, 0x08, @@ -354,9 +368,9 @@ const NORMAL0: &[u8] = &[ 0x81, 0xda, 0x26, 0x07, 0x0c, 0x05, 0x05, - 0x80, 0xa5, 0x11, - 0x81, 0x6d, 0x10, - 0x78, 0x28, + 0x80, 0xa6, 0x10, + 0x81, 0xf5, 0x07, + 0x01, 0x20, 0x2a, 0x06, 0x4c, 0x04, 0x80, 0x8d, 0x04, @@ -386,10 +400,11 @@ const NORMAL1: &[u8] = &[ 0x24, 0x04, 0x28, 0x08, 0x34, 0x0b, - 0x01, 0x80, 0x90, + 0x4e, 0x43, 0x81, 0x37, 0x09, 0x16, 0x0a, - 0x08, 0x80, 0x98, + 0x08, 0x18, + 0x3b, 0x45, 0x39, 0x03, 0x63, 0x08, 0x09, 0x30, @@ -415,20 +430,21 @@ const NORMAL1: &[u8] = &[ 0x33, 0x07, 0x2e, 0x08, 0x0a, 0x81, 0x26, - 0x52, 0x4e, - 0x28, 0x08, - 0x2a, 0x56, + 0x52, 0x4b, + 0x2b, 0x08, + 0x2a, 0x16, + 0x1a, 0x26, 0x1c, 0x14, 0x17, 0x09, 0x4e, 0x04, - 0x1e, 0x0f, - 0x43, 0x0e, + 0x24, 0x09, + 0x44, 0x0d, 0x19, 0x07, 0x0a, 0x06, 0x48, 0x08, 0x27, 0x09, 0x75, 0x0b, - 0x3f, 0x41, + 0x42, 0x3e, 0x2a, 0x06, 0x3b, 0x05, 0x0a, 0x06, @@ -443,18 +459,19 @@ const NORMAL1: &[u8] = &[ 0x45, 0x0b, 0x0a, 0x06, 0x0d, 0x13, - 0x39, 0x07, + 0x3a, 0x06, 0x0a, 0x36, 0x2c, 0x04, - 0x10, 0x80, 0xc0, + 0x17, 0x80, 0xb9, 0x3c, 0x64, 0x53, 0x0c, 0x48, 0x09, 0x0a, 0x46, 0x45, 0x1b, 0x48, 0x08, - 0x53, 0x1d, - 0x39, 0x81, 0x07, + 0x53, 0x0d, + 0x49, 0x07, + 0x0a, 0x80, 0xf6, 0x46, 0x0a, 0x1d, 0x03, 0x47, 0x49, @@ -463,17 +480,21 @@ const NORMAL1: &[u8] = &[ 0x0a, 0x06, 0x39, 0x07, 0x0a, 0x81, 0x36, - 0x19, 0x80, 0xb7, + 0x19, 0x07, + 0x3b, 0x03, + 0x1c, 0x56, 0x01, 0x0f, 0x32, 0x0d, 0x83, 0x9b, 0x66, 0x75, 0x0b, - 0x80, 0xc4, 0x8a, 0xbc, - 0x84, 0x2f, 0x8f, 0xd1, + 0x80, 0xc4, 0x8a, 0x4c, + 0x63, 0x0d, + 0x84, 0x30, 0x10, + 0x16, 0x8f, 0xaa, 0x82, 0x47, 0xa1, 0xb9, 0x82, 0x39, 0x07, 0x2a, 0x04, - 0x02, 0x60, + 0x5c, 0x06, 0x26, 0x0a, 0x46, 0x0a, 0x28, 0x05, @@ -486,32 +507,42 @@ const NORMAL1: &[u8] = &[ 0x02, 0x0e, 0x97, 0xf8, 0x08, 0x84, 0xd6, 0x2a, - 0x09, 0xa2, 0xf7, - 0x81, 0x1f, 0x31, - 0x03, 0x11, + 0x09, 0xa2, 0xe7, + 0x81, 0x33, 0x0f, + 0x01, 0x1d, + 0x06, 0x0e, 0x04, 0x08, 0x81, 0x8c, 0x89, 0x04, 0x6b, 0x05, 0x0d, 0x03, 0x09, 0x07, - 0x10, 0x93, 0x60, + 0x10, 0x92, 0x60, + 0x47, 0x09, + 0x74, 0x3c, 0x80, 0xf6, 0x0a, 0x73, 0x08, - 0x6e, 0x17, - 0x46, 0x80, 0x9a, + 0x70, 0x15, + 0x46, 0x7a, + 0x14, 0x0c, 0x14, 0x0c, 0x57, 0x09, 0x19, 0x80, 0x87, 0x81, 0x47, 0x03, 0x85, 0x42, 0x0f, - 0x15, 0x85, 0x50, - 0x2b, 0x80, 0xd5, + 0x15, 0x84, 0x50, + 0x1f, 0x06, + 0x06, 0x80, 0xd5, + 0x2b, 0x05, + 0x3e, 0x21, + 0x01, 0x70, 0x2d, 0x03, 0x1a, 0x04, - 0x02, 0x81, 0x70, + 0x02, 0x81, 0x40, + 0x1f, 0x11, 0x3a, 0x05, - 0x01, 0x85, 0x00, - 0x80, 0xd7, 0x29, + 0x01, 0x81, 0xd0, + 0x2a, 0x82, 0xe6, + 0x80, 0xf7, 0x29, 0x4c, 0x04, 0x0a, 0x04, 0x02, 0x83, 0x11, @@ -531,25 +562,25 @@ const NORMAL1: &[u8] = &[ 0x09, 0x07, 0x02, 0x0e, 0x06, 0x80, 0x9a, - 0x83, 0xd8, 0x08, + 0x83, 0xd8, 0x04, + 0x11, 0x03, 0x0d, 0x03, - 0x0d, 0x03, - 0x74, 0x0c, - 0x59, 0x07, - 0x0c, 0x14, + 0x77, 0x04, + 0x5f, 0x06, + 0x0c, 0x04, + 0x01, 0x0f, 0x0c, 0x04, 0x38, 0x08, 0x0a, 0x06, 0x28, 0x08, 0x22, 0x4e, 0x81, 0x54, 0x0c, - 0x15, 0x03, - 0x03, 0x05, - 0x07, 0x09, - 0x19, 0x07, - 0x07, 0x09, - 0x03, 0x0d, - 0x07, 0x29, + 0x1d, 0x03, + 0x09, 0x07, + 0x36, 0x08, + 0x0e, 0x04, + 0x09, 0x07, + 0x09, 0x07, 0x80, 0xcb, 0x25, 0x0a, 0x84, 0x06, ]; diff --git a/crux-mir/lib/core/src/unicode/unicode_data.rs b/crux-mir/lib/core/src/unicode/unicode_data.rs index 3e9002861..bd69ca520 100644 --- a/crux-mir/lib/core/src/unicode/unicode_data.rs +++ b/crux-mir/lib/core/src/unicode/unicode_data.rs @@ -1,635 +1,604 @@ ///! This file is generated by src/tools/unicode-table-generator; do not edit manually! -use super::range_search; -pub const UNICODE_VERSION: (u32, u32, u32) = (13, 0, 0); +#[rustc_const_unstable(feature = "const_unicode_case_lookup", issue = "101400")] +#[inline(always)] +const fn bitset_search< + const N: usize, + const CHUNK_SIZE: usize, + const N1: usize, + const CANONICAL: usize, + const CANONICALIZED: usize, +>( + needle: u32, + chunk_idx_map: &[u8; N], + bitset_chunk_idx: &[[u8; CHUNK_SIZE]; N1], + bitset_canonical: &[u64; CANONICAL], + bitset_canonicalized: &[(u8, u8); CANONICALIZED], +) -> bool { + let bucket_idx = (needle / 64) as usize; + let chunk_map_idx = bucket_idx / CHUNK_SIZE; + let chunk_piece = bucket_idx % CHUNK_SIZE; + // FIXME: const-hack: Revert to `slice::get` after `const_slice_index` + // feature stabilizes. + let chunk_idx = if chunk_map_idx < chunk_idx_map.len() { + chunk_idx_map[chunk_map_idx] + } else { + return false; + }; + let idx = bitset_chunk_idx[chunk_idx as usize][chunk_piece] as usize; + // FIXME: const-hack: Revert to `slice::get` after `const_slice_index` + // feature stabilizes. + let word = if idx < bitset_canonical.len() { + bitset_canonical[idx] + } else { + let (real_idx, mapping) = bitset_canonicalized[idx - bitset_canonical.len()]; + let mut word = bitset_canonical[real_idx as usize]; + let should_invert = mapping & (1 << 6) != 0; + if should_invert { + word = !word; + } + // Lower 6 bits + let quantity = mapping & ((1 << 6) - 1); + if mapping & (1 << 7) != 0 { + // shift + word >>= quantity as u64; + } else { + word = word.rotate_left(quantity as u32); + } + word + }; + (word & (1 << (needle % 64) as u64)) != 0 +} + +fn decode_prefix_sum(short_offset_run_header: u32) -> u32 { + short_offset_run_header & ((1 << 21) - 1) +} + +fn decode_length(short_offset_run_header: u32) -> usize { + (short_offset_run_header >> 21) as usize +} + +#[inline(always)] +fn skip_search( + needle: u32, + short_offset_runs: &[u32; SOR], + offsets: &[u8; OFFSETS], +) -> bool { + // Note that this *cannot* be past the end of the array, as the last + // element is greater than std::char::MAX (the largest possible needle). + // + // So, we cannot have found it (i.e. Ok(idx) + 1 != length) and the correct + // location cannot be past it, so Err(idx) != length either. + // + // This means that we can avoid bounds checking for the accesses below, too. + let last_idx = + match short_offset_runs.binary_search_by_key(&(needle << 11), |header| header << 11) { + Ok(idx) => idx + 1, + Err(idx) => idx, + }; + + let mut offset_idx = decode_length(short_offset_runs[last_idx]); + let length = if let Some(next) = short_offset_runs.get(last_idx + 1) { + decode_length(*next) - offset_idx + } else { + offsets.len() - offset_idx + }; + let prev = + last_idx.checked_sub(1).map(|prev| decode_prefix_sum(short_offset_runs[prev])).unwrap_or(0); + + let total = needle - prev; + let mut prefix_sum = 0; + for _ in 0..(length - 1) { + let offset = offsets[offset_idx]; + prefix_sum += offset as u32; + if prefix_sum > total { + break; + } + offset_idx += 1; + } + offset_idx % 2 == 1 +} + +pub const UNICODE_VERSION: (u8, u8, u8) = (15, 0, 0); #[rustfmt::skip] pub mod alphabetic { - static BITSET_LAST_CHUNK_MAP: (u16, u8) = (196, 44); - static BITSET_CHUNKS_MAP: [u8; 196] = [ - 6, 32, 10, 18, 19, 23, 21, 12, 7, 5, 0, 20, 14, 50, 50, 50, 50, 50, 50, 37, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 49, 50, 30, 8, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 46, 0, 0, 0, 0, 0, 0, 0, 0, 4, 36, 17, 31, 16, 25, 24, 26, 13, 15, - 45, 27, 0, 0, 50, 11, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1, 50, 50, 50, 50, 50, 48, - 50, 34, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 28, 0, 0, 0, 0, 0, 29, 0, 0, 9, 0, 33, 2, 3, 0, 0, - 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 42, 50, 50, 50, - 43, 22, 50, 50, 50, 50, 41, 50, 50, 50, 50, 50, 50, 47, 0, 0, 0, 38, 0, 50, 50, 50, 50, + static SHORT_OFFSET_RUNS: [u32; 53] = [ + 706, 33559113, 872420973, 952114966, 1161831606, 1310731264, 1314926597, 1394619392, + 1444957632, 1447077005, 1451271693, 1459672996, 1648425216, 1658911342, 1661009214, + 1707147904, 1793132343, 1887506048, 2040601600, 2392923872, 2481005466, 2504077200, + 2514564144, 2520859648, 2527151687, 2529257472, 2531355193, 2533453376, 2564917240, + 2596375766, 2600579056, 2606870819, 2621551356, 2642525184, 2644628480, 2665600678, + 2743197440, 2791432848, 2841765072, 2850154464, 2854350336, 2887905584, 3026321408, + 3038947040, 3041048378, 3045248674, 3053644769, 3057842176, 3059939870, 3062038528, + 3064140619, 3066241968, 3071550384, ]; - static BITSET_INDEX_CHUNKS: [[u8; 16]; 51] = [ - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 254, 247, 39, 68], - [0, 0, 0, 0, 0, 0, 0, 0, 111, 135, 113, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 195, 205, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 254, 254, 254, 254, 254, 210, 254, 25, 136, 251, 71, 243], - [0, 0, 182, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 107, 103, 180, 254, 254, 254, 254, 254, 254, 254, 61, 0, 155, 222, 181], - [0, 148, 30, 0, 172, 226, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [48, 80, 254, 169, 206, 123, 189, 139, 95, 179, 145, 86, 211, 204, 254, 56], - [53, 0, 0, 0, 129, 17, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0], - [59, 54, 185, 203, 171, 191, 161, 117, 158, 87, 164, 118, 162, 67, 159, 23], - [62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [95, 131, 168, 105, 254, 254, 254, 82, 254, 254, 254, 254, 236, 130, 137, 120], - [101, 0, 225, 146, 151, 2, 217, 45, 144, 246, 32, 101, 0, 0, 0, 0], - [119, 253, 224, 175, 193, 254, 227, 195, 0, 0, 0, 0, 0, 0, 0, 0], - [143, 190, 91, 0, 153, 218, 24, 0, 0, 0, 0, 92, 0, 0, 66, 0], - [150, 94, 37, 85, 102, 0, 157, 0, 88, 122, 31, 46, 89, 74, 20, 0], - [154, 34, 254, 110, 0, 84, 0, 0, 0, 0, 233, 19, 216, 108, 237, 21], - [166, 42, 165, 72, 167, 177, 126, 76, 109, 16, 127, 38, 1, 192, 124, 0], - [176, 246, 234, 174, 254, 254, 254, 254, 254, 235, 140, 241, 240, 26, 228, 128], - [213, 239, 254, 77, 209, 64, 142, 238, 63, 0, 0, 0, 0, 0, 0, 0], - [225, 101, 207, 89, 98, 81, 208, 10, 232, 83, 147, 1, 188, 13, 178, 70], - [237, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254], - [253, 254, 254, 254, 254, 254, 254, 254, 254, 214, 231, 99, 79, 78, 183, 27], - [254, 6, 100, 50, 75, 90, 254, 28, 134, 0, 202, 51, 163, 43, 0, 0], - [254, 9, 75, 75, 49, 0, 0, 0, 0, 0, 69, 0, 199, 6, 195, 93], - [254, 41, 254, 8, 0, 0, 141, 33, 145, 4, 97, 0, 55, 0, 0, 0], - [254, 62, 254, 254, 254, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [254, 121, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [254, 242, 170, 252, 138, 245, 254, 254, 254, 254, 220, 173, 186, 212, 219, 14], - [254, 254, 15, 132, 254, 254, 254, 254, 57, 149, 254, 65, 223, 254, 249, 187], - [254, 254, 196, 114, 201, 44, 0, 0, 254, 254, 254, 254, 95, 47, 0, 0], - [254, 254, 250, 254, 194, 229, 156, 73, 230, 215, 254, 152, 246, 248, 71, 104], - [254, 254, 254, 5, 254, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [254, 254, 254, 22, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [254, 254, 254, 254, 37, 200, 254, 254, 254, 254, 254, 116, 0, 0, 0, 0], - [254, 254, 254, 254, 133, 246, 244, 112, 0, 184, 254, 125, 106, 221, 145, 29], - [254, 254, 254, 254, 254, 254, 254, 0, 254, 254, 254, 254, 254, 254, 254, 254], - [254, 254, 254, 254, 254, 254, 254, 254, 35, 0, 0, 0, 0, 0, 0, 0], - [254, 254, 254, 254, 254, 254, 254, 254, 101, 37, 0, 60, 65, 160, 18, 0], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 7, 0, 0, 0, 0, 0, 0], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 197, 254, 254, 254, 254, 254], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 35, 254, 254, 254, 254], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 84, 254, 254, 254], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 11, 0, 0], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 25, 0], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 198, 115], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 40], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 96], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 125], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254], + static OFFSETS: [u8; 1465] = [ + 65, 26, 6, 26, 47, 1, 10, 1, 4, 1, 5, 23, 1, 31, 1, 0, 4, 12, 14, 5, 7, 1, 1, 1, 86, 1, 42, + 5, 1, 2, 2, 4, 1, 1, 6, 1, 1, 3, 1, 1, 1, 20, 1, 83, 1, 139, 8, 166, 1, 38, 2, 1, 6, 41, 39, + 14, 1, 1, 1, 2, 1, 2, 1, 1, 8, 27, 4, 4, 29, 11, 5, 56, 1, 7, 14, 102, 1, 8, 4, 8, 4, 3, 10, + 3, 2, 1, 16, 48, 13, 101, 24, 33, 9, 2, 4, 1, 5, 24, 2, 19, 19, 25, 7, 11, 5, 24, 1, 6, 17, + 42, 10, 12, 3, 7, 6, 76, 1, 16, 1, 3, 4, 15, 13, 19, 1, 8, 2, 2, 2, 22, 1, 7, 1, 1, 3, 4, 3, + 8, 2, 2, 2, 2, 1, 1, 8, 1, 4, 2, 1, 5, 12, 2, 10, 1, 4, 3, 1, 6, 4, 2, 2, 22, 1, 7, 1, 2, 1, + 2, 1, 2, 4, 5, 4, 2, 2, 2, 4, 1, 7, 4, 1, 1, 17, 6, 11, 3, 1, 9, 1, 3, 1, 22, 1, 7, 1, 2, 1, + 5, 3, 9, 1, 3, 1, 2, 3, 1, 15, 4, 21, 4, 4, 3, 1, 8, 2, 2, 2, 22, 1, 7, 1, 2, 1, 5, 3, 8, 2, + 2, 2, 2, 9, 2, 4, 2, 1, 5, 13, 1, 16, 2, 1, 6, 3, 3, 1, 4, 3, 2, 1, 1, 1, 2, 3, 2, 3, 3, 3, + 12, 4, 5, 3, 3, 1, 3, 3, 1, 6, 1, 40, 13, 1, 3, 1, 23, 1, 16, 3, 8, 1, 3, 1, 3, 8, 2, 1, 3, + 2, 1, 2, 4, 28, 4, 1, 8, 1, 3, 1, 23, 1, 10, 1, 5, 3, 8, 1, 3, 1, 3, 8, 2, 6, 2, 1, 4, 13, + 3, 12, 13, 1, 3, 1, 41, 2, 8, 1, 3, 1, 3, 1, 1, 5, 4, 7, 5, 22, 6, 1, 3, 1, 18, 3, 24, 1, 9, + 1, 1, 2, 7, 8, 6, 1, 1, 1, 8, 18, 2, 13, 58, 5, 7, 6, 1, 51, 2, 1, 1, 1, 5, 1, 24, 1, 1, 1, + 19, 1, 3, 2, 5, 1, 1, 6, 1, 14, 4, 32, 1, 63, 8, 1, 36, 4, 19, 4, 16, 1, 36, 67, 55, 1, 1, + 2, 5, 16, 64, 10, 4, 2, 38, 1, 1, 5, 1, 2, 43, 1, 0, 1, 4, 2, 7, 1, 1, 1, 4, 2, 41, 1, 4, 2, + 33, 1, 4, 2, 7, 1, 1, 1, 4, 2, 15, 1, 57, 1, 4, 2, 67, 37, 16, 16, 86, 2, 6, 3, 0, 2, 17, 1, + 26, 5, 75, 3, 11, 7, 20, 11, 21, 12, 20, 12, 13, 1, 3, 1, 2, 12, 52, 2, 19, 14, 1, 4, 1, 67, + 89, 7, 43, 5, 70, 10, 31, 1, 12, 4, 9, 23, 30, 2, 5, 11, 44, 4, 26, 54, 28, 4, 63, 2, 20, + 50, 1, 23, 2, 11, 3, 49, 52, 1, 15, 1, 8, 51, 42, 2, 4, 10, 44, 1, 11, 14, 55, 22, 3, 10, + 36, 2, 9, 7, 43, 2, 3, 41, 4, 1, 6, 1, 2, 3, 1, 5, 192, 39, 14, 11, 0, 2, 6, 2, 38, 2, 6, 2, + 8, 1, 1, 1, 1, 1, 1, 1, 31, 2, 53, 1, 7, 1, 1, 3, 3, 1, 7, 3, 4, 2, 6, 4, 13, 5, 3, 1, 7, + 116, 1, 13, 1, 16, 13, 101, 1, 4, 1, 2, 10, 1, 1, 3, 5, 6, 1, 1, 1, 1, 1, 1, 4, 1, 11, 2, 4, + 5, 5, 4, 1, 17, 41, 0, 52, 0, 229, 6, 4, 3, 2, 12, 38, 1, 1, 5, 1, 2, 56, 7, 1, 16, 23, 9, + 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 32, 47, 1, 0, 3, 25, 9, 7, 5, 2, 5, 4, 86, + 6, 3, 1, 90, 1, 4, 5, 43, 1, 94, 17, 32, 48, 16, 0, 0, 64, 0, 67, 46, 2, 0, 3, 16, 10, 2, + 20, 47, 5, 8, 3, 113, 39, 9, 2, 103, 2, 64, 5, 2, 1, 1, 1, 5, 24, 20, 1, 33, 24, 52, 12, 68, + 1, 1, 44, 6, 3, 1, 1, 3, 10, 33, 5, 35, 13, 29, 3, 51, 1, 12, 15, 1, 16, 16, 10, 5, 1, 55, + 9, 14, 18, 23, 3, 69, 1, 1, 1, 1, 24, 3, 2, 16, 2, 4, 11, 6, 2, 6, 2, 6, 9, 7, 1, 7, 1, 43, + 1, 14, 6, 123, 21, 0, 12, 23, 4, 49, 0, 0, 2, 106, 38, 7, 12, 5, 5, 12, 1, 13, 1, 5, 1, 1, + 1, 2, 1, 2, 1, 108, 33, 0, 18, 64, 2, 54, 40, 12, 116, 5, 1, 135, 36, 26, 6, 26, 11, 89, 3, + 6, 2, 6, 2, 6, 2, 3, 35, 12, 1, 26, 1, 19, 1, 2, 1, 15, 2, 14, 34, 123, 69, 53, 0, 29, 3, + 49, 47, 32, 13, 30, 5, 43, 5, 30, 2, 36, 4, 8, 1, 5, 42, 158, 18, 36, 4, 36, 4, 40, 8, 52, + 12, 11, 1, 15, 1, 7, 1, 2, 1, 11, 1, 15, 1, 7, 1, 2, 67, 0, 9, 22, 10, 8, 24, 6, 1, 42, 1, + 9, 69, 6, 2, 1, 1, 44, 1, 2, 3, 1, 2, 23, 10, 23, 9, 31, 65, 19, 1, 2, 10, 22, 10, 26, 70, + 56, 6, 2, 64, 4, 1, 2, 5, 8, 1, 3, 1, 29, 42, 29, 3, 29, 35, 8, 1, 28, 27, 54, 10, 22, 10, + 19, 13, 18, 110, 73, 55, 51, 13, 51, 13, 40, 0, 42, 1, 2, 3, 2, 78, 29, 10, 1, 8, 22, 42, + 18, 46, 21, 27, 23, 9, 70, 43, 5, 10, 57, 9, 1, 13, 25, 23, 51, 17, 4, 8, 35, 3, 1, 9, 64, + 1, 4, 9, 2, 10, 1, 1, 1, 35, 18, 1, 34, 2, 1, 6, 4, 62, 7, 1, 1, 1, 4, 1, 15, 1, 10, 7, 57, + 23, 4, 1, 8, 2, 2, 2, 22, 1, 7, 1, 2, 1, 5, 3, 8, 2, 2, 2, 2, 3, 1, 6, 1, 5, 7, 156, 66, 1, + 3, 1, 4, 20, 3, 30, 66, 2, 2, 1, 1, 184, 54, 2, 7, 25, 6, 34, 63, 1, 1, 3, 1, 59, 54, 2, 1, + 71, 27, 2, 14, 21, 7, 185, 57, 103, 64, 31, 8, 2, 1, 2, 8, 1, 2, 1, 30, 1, 2, 2, 2, 2, 4, + 93, 8, 2, 46, 2, 6, 1, 1, 1, 2, 27, 51, 2, 10, 17, 72, 5, 1, 18, 73, 0, 9, 1, 45, 1, 7, 1, + 1, 49, 30, 2, 22, 1, 14, 73, 7, 1, 2, 1, 44, 3, 1, 1, 2, 1, 3, 1, 1, 2, 2, 24, 6, 1, 2, 1, + 37, 1, 2, 1, 4, 1, 1, 0, 23, 9, 17, 1, 41, 3, 3, 111, 1, 79, 0, 102, 111, 17, 196, 0, 97, + 15, 0, 17, 6, 0, 0, 0, 0, 7, 31, 17, 79, 17, 30, 18, 48, 16, 4, 31, 21, 5, 19, 0, 64, 128, + 75, 4, 57, 7, 17, 64, 2, 1, 1, 12, 2, 14, 0, 8, 0, 42, 9, 0, 4, 1, 7, 1, 2, 1, 0, 15, 1, 29, + 3, 2, 1, 14, 4, 8, 0, 0, 107, 5, 13, 3, 9, 7, 10, 4, 1, 0, 85, 1, 71, 1, 2, 2, 1, 2, 2, 2, + 4, 1, 12, 1, 1, 1, 7, 1, 65, 1, 4, 2, 8, 1, 7, 1, 28, 1, 4, 1, 5, 1, 1, 3, 7, 1, 0, 2, 25, + 1, 25, 1, 31, 1, 25, 1, 31, 1, 25, 1, 31, 1, 25, 1, 31, 1, 25, 1, 8, 0, 31, 6, 6, 213, 7, 1, + 17, 2, 7, 1, 2, 1, 5, 5, 62, 33, 1, 112, 45, 10, 7, 16, 1, 0, 30, 18, 44, 0, 28, 0, 7, 1, 4, + 1, 2, 1, 15, 1, 197, 59, 68, 3, 1, 3, 1, 0, 4, 1, 27, 1, 2, 1, 1, 2, 1, 1, 10, 1, 4, 1, 1, + 1, 1, 6, 1, 4, 1, 1, 1, 1, 1, 1, 3, 1, 2, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 2, + 4, 1, 7, 1, 4, 1, 4, 1, 1, 1, 10, 1, 17, 5, 3, 1, 5, 1, 17, 0, 26, 6, 26, 6, 26, 0, 0, 32, + 0, 6, 222, 2, 0, 14, 0, 0, 0, 0, 0, 5, 0, 0, ]; - static BITSET: [u64; 255] = [ - 0, 1, 7, 15, 17, 31, 63, 127, 179, 511, 1023, 2047, 2191, 4079, 4087, 8191, 8319, 16384, - 65535, 131071, 262143, 4128527, 4194303, 8461767, 24870911, 67108863, 134217727, 276824575, - 335593502, 486341884, 536805376, 536870911, 553648127, 1056964608, 1073692671, 1073741823, - 1140785663, 2147483647, 4026540127, 4294934783, 8589934591, 15032387515, 64548249055, - 68191066527, 68719476735, 115913785343, 137438953215, 1095220854783, 1099511627711, - 1099511627775, 2199023190016, 2199023255551, 4398046511103, 8641373536127, 8791831609343, - 8795690369023, 8796093022207, 13198434443263, 17592186044415, 35184321757183, - 70368744112128, 88094074470339, 140737488355327, 140737488355328, 141836999983103, - 281474976710655, 281474976710656, 563017343310239, 844472174772224, 875211255709695, - 1125625028935679, 1125899906842623, 1688915364814303, 2119858418286774, 2251795522912255, - 2251799813685247, 3377704004976767, 3509778554814463, 3905461007941631, 4503595333443583, - 4503599627370495, 8796093022142464, 9006649498927104, 9007192812290047, 9007199254740991, - 15762594400829440, 17169970223906821, 17732925109967239, 18014398491652207, - 18014398509481983, 20266198323101936, 36027697507139583, 36028792723996672, - 36028792723996703, 36028792728190975, 36028797018963967, 72057594037927935, - 90071992547409919, 143851303137705983, 144053615424700415, 144115188075855868, - 144115188075855871, 288230371860938751, 297241973452963840, 301749971126844416, - 319718190147960832, 576460743713488896, 576460743847706622, 576460752303359999, - 576460752303423486, 576460752303423487, 790380184120328175, 1152640029630136575, - 1152917029519358975, 1152921504591118335, 1152921504606845055, 1152921504606846975, - 1153765996922689951, 2161727885562420159, 2251241253188403424, 2295745090394464220, - 2305570330330005503, 2305843004918726656, 2305843004919250943, 2305843009196916483, - 2305843009213693951, 3457638613854978030, 4323455298678290390, 4557642822898941951, - 4575692405780512767, 4611686017001275199, 4611686018360336384, 4611686018427322368, - 4611686018427387903, 4656722014700830719, 6843210385291930244, 6881498031078244479, - 6908521828386340863, 8935141660164089791, 8935423131384840192, 9168765891372858879, - 9169328841326329855, 9187201948305063935, 9187343239835811327, 9216616637413720063, - 9223372036854775807, 9223372041149743103, 9223372586610589696, 9223934986808197120, - 10371930679322607615, 10502394331027995967, 11078855083321979519, 11241233151490523135, - 13006395723845991295, 13258596753222922239, 13609596598936928288, 13834776580305453567, - 13907115649320091647, 14082190885810440174, 14123225865944680428, 16212958624174047247, - 16412803692974677999, 16424062692043104238, 16424062692043104239, 16424062692043243502, - 16424625641996804079, 16429129241624174575, 16717361816799141887, 16717361816799216127, - 16788293510930366511, 17005555242810474495, 17293822569102704639, 17581979622616071300, - 17870283321271910397, 17870283321406070975, 17870283321406128127, 17978369712463020031, - 18158513764145585631, 18158781978395017215, 18194542490281852927, 18410715276682199039, - 18428729675200069631, 18428729675200069632, 18433233274827440127, 18437455399478099968, - 18437736870159843328, 18437736874452713471, 18437736874454812668, 18442240474082181119, - 18444492273895866367, 18445618173802708993, 18446181192473632767, 18446216308128218879, - 18446462598732840928, 18446462598732840959, 18446462598732840960, 18446462599806582783, - 18446462615912710143, 18446462667452317695, 18446463149025525759, 18446463629525450752, - 18446463698244468735, 18446464796682337663, 18446466966713532671, 18446466996645134335, - 18446466996779287551, 18446471394825862144, 18446471394825863167, 18446480190918885375, - 18446498607738650623, 18446532967477018623, 18446602782178705022, 18446603336221163519, - 18446603336221196287, 18446638520593285119, 18446673709243564031, 18446708893632430079, - 18446740770879700992, 18446741595513422027, 18446741874686295551, 18446743249075830783, - 18446743798965862398, 18446744056529672000, 18446744060816261120, 18446744068886102015, - 18446744069414584320, 18446744069414601696, 18446744069414617087, 18446744069414649855, - 18446744069456527359, 18446744069548736512, 18446744069548802046, 18446744069683019775, - 18446744069951455231, 18446744070421282815, 18446744070446333439, 18446744070475743231, - 18446744070488326143, 18446744071553646463, 18446744071562067967, 18446744073696837631, - 18446744073701162813, 18446744073707454463, 18446744073709027328, 18446744073709355007, - 18446744073709419615, 18446744073709486080, 18446744073709520895, 18446744073709543424, - 18446744073709550079, 18446744073709550595, 18446744073709551579, 18446744073709551599, - 18446744073709551614, 18446744073709551615, - ]; - pub fn lookup(c: char) -> bool { - super::range_search( + super::skip_search( c as u32, - &BITSET_CHUNKS_MAP, - BITSET_LAST_CHUNK_MAP, - &BITSET_INDEX_CHUNKS, - &BITSET, + &SHORT_OFFSET_RUNS, + &OFFSETS, ) } } #[rustfmt::skip] pub mod case_ignorable { - static BITSET_LAST_CHUNK_MAP: (u16, u8) = (896, 33); - static BITSET_CHUNKS_MAP: [u8; 125] = [ - 25, 14, 21, 30, 28, 4, 17, 23, 22, 0, 0, 16, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 13, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3, 6, 9, 0, 7, 11, 32, 31, 26, 29, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, - 10, 0, 8, 0, 19, 0, 12, 0, 1, + static SHORT_OFFSET_RUNS: [u32; 35] = [ + 688, 44045149, 572528402, 576724925, 807414908, 878718981, 903913493, 929080568, 933275148, + 937491230, 1138818560, 1147208189, 1210124160, 1222707713, 1235291428, 1260457643, + 1264654383, 1499535675, 1507925040, 1566646003, 1629566000, 1650551536, 1658941263, + 1671540720, 1688321181, 1700908800, 1709298023, 1717688832, 1738661888, 1763828398, + 1797383403, 1805773008, 1809970171, 1819148289, 1824457200, ]; - static BITSET_INDEX_CHUNKS: [[u8; 16]; 34] = [ - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 47, 57], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 173, 3], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 90, 136, 38], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 104, 7, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 78, 27, 0, 148, 138, 81, 44, 119], - [0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 58, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 167, 99, 77, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 48, 0, 116, 0, 0], - [0, 0, 0, 0, 0, 172, 70, 0, 0, 8, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 67, 0, 0, 24, 0, 0], - [0, 0, 0, 29, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 135, 0, 0, 0, 0, 16, 162, 46, 86, 51, 80, 13, 111], - [0, 0, 12, 0, 0, 43, 163, 92, 35, 82, 0, 71, 175, 14, 83, 131], - [0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 133, 0, 87, 0, 150, 0, 178, 75, 0, 0, 0, 0, 0, 0, 0], - [20, 5, 61, 0, 120, 0, 0, 0, 32, 156, 176, 1, 126, 91, 69, 88], - [26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [62, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0], - [66, 0, 0, 152, 72, 25, 134, 59, 102, 124, 165, 101, 0, 64, 0, 68], - [73, 33, 0, 181, 125, 85, 122, 139, 123, 100, 123, 169, 155, 54, 4, 18], - [74, 151, 36, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [106, 135, 0, 112, 177, 107, 180, 168, 0, 0, 0, 0, 0, 0, 157, 142], - [109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [113, 50, 108, 0, 0, 0, 0, 0, 0, 0, 174, 182, 182, 114, 10, 0], - [115, 0, 0, 0, 141, 5, 0, 49, 145, 34, 31, 0, 0, 0, 0, 0], - [118, 0, 42, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [143, 95, 37, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0], - [161, 0, 103, 0, 160, 11, 30, 0, 0, 0, 0, 93, 0, 0, 0, 0], - [164, 55, 155, 53, 127, 52, 2, 28, 117, 21, 128, 19, 110, 147, 129, 9], - [170, 41, 153, 6, 0, 0, 159, 39, 158, 1, 105, 0, 65, 0, 0, 0], - [171, 149, 132, 17, 98, 89, 146, 23, 140, 0, 0, 63, 127, 97, 0, 0], - [179, 182, 0, 0, 182, 182, 182, 79, 0, 0, 0, 0, 0, 0, 0, 0], - ]; - static BITSET: [u64; 183] = [ - 0, 1, 2, 3, 4, 8, 13, 15, 28, 64, 176, 191, 1016, 1792, 2047, 4080, 4096, 8192, 8193, - 16192, 30720, 32704, 32768, 40448, 131008, 262016, 2097152, 2359296, 6030336, 8323072, - 10682368, 58719232, 159383552, 234881024, 243138688, 402587711, 536805376, 536879204, - 546307648, 805306369, 1073741824, 1073741916, 2113929216, 2181038080, 3221225472, - 3758096384, 4026531840, 4294934528, 4294967296, 4512022528, 5368709120, 17179869183, - 51539615774, 51539619904, 51545907230, 51545914817, 66035122176, 115964116992, 412316860416, - 412316893184, 1030792151040, 2199023255648, 8641373536127, 8763880767488, 15397323538432, - 17303886364672, 18004502906948, 26388279066624, 36421322670080, 65128884076547, - 65970697670631, 68168642985984, 70093866270720, 70368739983360, 136957967529984, - 140737488355328, 263882790666240, 281470547525648, 281470682333183, 281474976710655, - 281474976710656, 281474976710657, 281479271675905, 562675075514368, 562949953355776, - 563001509683710, 844424930131968, 985162418487296, 1023920203366400, 2251799813685248, - 3377699721314304, 4494803534348292, 4503599627370678, 6755399441055744, 7881299349733376, - 8444256867844096, 8725724278030336, 8760633772212225, 8989057312882695, 9042383626829823, - 9851624185018758, 24822575045541890, 28848986089586688, 30958948903026688, - 35747322042253312, 53805701016846336, 58529202969772032, 72066390130950143, - 112767012056334336, 143833713099145216, 189151184399892480, 216172782113783808, - 220713756545974272, 288301294651703296, 302022650010533887, 504262420777140224, - 558446353793941504, 572520102629474304, 593978171557150752, 1008806350890729472, - 1009933895770046464, 1152921504606846976, 1152921504606846978, 1152921504606846982, - 1153202979583561736, 1441151880758558727, 1715871458028158991, 1729382256910270467, - 2301902359539744768, 2305843009196908767, 2305843009213693952, 2612078987781865472, - 2771965570646540291, 3458764513820540928, 3731232291276455943, 4539628424389459968, - 4589168020290535424, 4611404543450677248, 4611686018494513280, 4611686069967003678, - 4671217976001691648, 6341068275337658368, 6917775322003857411, 7421334051581067264, - 8070450532247928832, 8788774672813524990, 9205357638345293827, 9222809086901354496, - 9223372036854775808, 9223372036854775935, 9223512774343131136, 9224216320050987008, - 9224497932466651184, 9653465801268658176, 9727775195120332910, 10376293541461622786, - 11526998316797657088, 11529215046068469760, 12103423998558208000, 12699025049277956096, - 13005832773892571136, 13798747783286489088, 13832665517980123136, 13835058055282032640, - 13835058055282163729, 13951307220663664640, 17870283321406128128, 17906312118425092095, - 18158513697557839871, 18158513749097456062, 18374686479671623680, 18374686479671623682, - 18444496122186563584, 18445618173802708992, 18446462598732840960, 18446462598733004800, - 18446463148488654848, 18446726481523507200, 18446744069414584320, 18446744069414584322, - 18446744073575333888, 18446744073709027328, 18446744073709551615, + static OFFSETS: [u8; 875] = [ + 39, 1, 6, 1, 11, 1, 35, 1, 1, 1, 71, 1, 4, 1, 1, 1, 4, 1, 2, 2, 0, 192, 4, 2, 4, 1, 9, 2, + 1, 1, 251, 7, 207, 1, 5, 1, 49, 45, 1, 1, 1, 2, 1, 2, 1, 1, 44, 1, 11, 6, 10, 11, 1, 1, 35, + 1, 10, 21, 16, 1, 101, 8, 1, 10, 1, 4, 33, 1, 1, 1, 30, 27, 91, 11, 58, 11, 4, 1, 2, 1, 24, + 24, 43, 3, 44, 1, 7, 2, 6, 8, 41, 58, 55, 1, 1, 1, 4, 8, 4, 1, 3, 7, 10, 2, 13, 1, 15, 1, + 58, 1, 4, 4, 8, 1, 20, 2, 26, 1, 2, 2, 57, 1, 4, 2, 4, 2, 2, 3, 3, 1, 30, 2, 3, 1, 11, 2, + 57, 1, 4, 5, 1, 2, 4, 1, 20, 2, 22, 6, 1, 1, 58, 1, 2, 1, 1, 4, 8, 1, 7, 2, 11, 2, 30, 1, + 61, 1, 12, 1, 50, 1, 3, 1, 55, 1, 1, 3, 5, 3, 1, 4, 7, 2, 11, 2, 29, 1, 58, 1, 2, 1, 6, 1, + 5, 2, 20, 2, 28, 2, 57, 2, 4, 4, 8, 1, 20, 2, 29, 1, 72, 1, 7, 3, 1, 1, 90, 1, 2, 7, 11, 9, + 98, 1, 2, 9, 9, 1, 1, 7, 73, 2, 27, 1, 1, 1, 1, 1, 55, 14, 1, 5, 1, 2, 5, 11, 1, 36, 9, 1, + 102, 4, 1, 6, 1, 2, 2, 2, 25, 2, 4, 3, 16, 4, 13, 1, 2, 2, 6, 1, 15, 1, 94, 1, 0, 3, 0, 3, + 29, 2, 30, 2, 30, 2, 64, 2, 1, 7, 8, 1, 2, 11, 3, 1, 5, 1, 45, 5, 51, 1, 65, 2, 34, 1, 118, + 3, 4, 2, 9, 1, 6, 3, 219, 2, 2, 1, 58, 1, 1, 7, 1, 1, 1, 1, 2, 8, 6, 10, 2, 1, 39, 1, 8, 31, + 49, 4, 48, 1, 1, 5, 1, 1, 5, 1, 40, 9, 12, 2, 32, 4, 2, 2, 1, 3, 56, 1, 1, 2, 3, 1, 1, 3, + 58, 8, 2, 2, 64, 6, 82, 3, 1, 13, 1, 7, 4, 1, 6, 1, 3, 2, 50, 63, 13, 1, 34, 101, 0, 1, 1, + 3, 11, 3, 13, 3, 13, 3, 13, 2, 12, 5, 8, 2, 10, 1, 2, 1, 2, 5, 49, 5, 1, 10, 1, 1, 13, 1, + 16, 13, 51, 33, 0, 2, 113, 3, 125, 1, 15, 1, 96, 32, 47, 1, 0, 1, 36, 4, 3, 5, 5, 1, 93, 6, + 93, 3, 0, 1, 0, 6, 0, 1, 98, 4, 1, 10, 1, 1, 28, 4, 80, 2, 14, 34, 78, 1, 23, 3, 103, 3, 3, + 2, 8, 1, 3, 1, 4, 1, 25, 2, 5, 1, 151, 2, 26, 18, 13, 1, 38, 8, 25, 11, 46, 3, 48, 1, 2, 4, + 2, 2, 17, 1, 21, 2, 66, 6, 2, 2, 2, 2, 12, 1, 8, 1, 35, 1, 11, 1, 51, 1, 1, 3, 2, 2, 5, 2, + 1, 1, 27, 1, 14, 2, 5, 2, 1, 1, 100, 5, 9, 3, 121, 1, 2, 1, 4, 1, 0, 1, 147, 17, 0, 16, 3, + 1, 12, 16, 34, 1, 2, 1, 169, 1, 7, 1, 6, 1, 11, 1, 35, 1, 1, 1, 47, 1, 45, 2, 67, 1, 21, 3, + 0, 1, 226, 1, 149, 5, 0, 6, 1, 42, 1, 9, 0, 3, 1, 2, 5, 4, 40, 3, 4, 1, 165, 2, 0, 4, 0, 2, + 80, 3, 70, 11, 49, 4, 123, 1, 54, 15, 41, 1, 2, 2, 10, 3, 49, 4, 2, 2, 2, 1, 4, 1, 10, 1, + 50, 3, 36, 5, 1, 8, 62, 1, 12, 2, 52, 9, 10, 4, 2, 1, 95, 3, 2, 1, 1, 2, 6, 1, 2, 1, 157, 1, + 3, 8, 21, 2, 57, 2, 3, 1, 37, 7, 3, 5, 195, 8, 2, 3, 1, 1, 23, 1, 84, 6, 1, 1, 4, 2, 1, 2, + 238, 4, 6, 2, 1, 2, 27, 2, 85, 8, 2, 1, 1, 2, 106, 1, 1, 1, 2, 6, 1, 1, 101, 3, 2, 4, 1, 5, + 0, 9, 1, 2, 0, 2, 1, 1, 4, 1, 144, 4, 2, 2, 4, 1, 32, 10, 40, 6, 2, 4, 8, 1, 9, 6, 2, 3, 46, + 13, 1, 2, 0, 7, 1, 6, 1, 1, 82, 22, 2, 7, 1, 2, 1, 2, 122, 6, 3, 1, 1, 2, 1, 7, 1, 1, 72, 2, + 3, 1, 1, 1, 0, 2, 11, 2, 52, 5, 5, 1, 1, 1, 0, 17, 6, 15, 0, 5, 59, 7, 9, 4, 0, 1, 63, 17, + 64, 2, 1, 2, 0, 4, 1, 7, 1, 2, 0, 2, 1, 4, 0, 46, 2, 23, 0, 3, 9, 16, 2, 7, 30, 4, 148, 3, + 0, 55, 4, 50, 8, 1, 14, 1, 22, 5, 1, 15, 0, 7, 1, 17, 2, 7, 1, 2, 1, 5, 5, 62, 33, 1, 160, + 14, 0, 1, 61, 4, 0, 5, 0, 7, 109, 8, 0, 5, 0, 1, 30, 96, 128, 240, 0, ]; - pub fn lookup(c: char) -> bool { - super::range_search( + super::skip_search( c as u32, - &BITSET_CHUNKS_MAP, - BITSET_LAST_CHUNK_MAP, - &BITSET_INDEX_CHUNKS, - &BITSET, + &SHORT_OFFSET_RUNS, + &OFFSETS, ) } } #[rustfmt::skip] pub mod cased { - static BITSET_LAST_CHUNK_MAP: (u16, u8) = (124, 6); - static BITSET_CHUNKS_MAP: [u8; 123] = [ - 13, 18, 0, 0, 12, 0, 0, 9, 14, 10, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 2, 0, 16, 0, 8, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, - 0, 0, 0, 7, - ]; - static BITSET_INDEX_CHUNKS: [[u8; 16]; 19] = [ - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 43, 62, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 10, 0, 50, 62, 58, 20], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 42, 44, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 62, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 16, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 31, 0, 62, 62, 62, 0, 62, 62, 62, 62, 54, 26, 27, 24], - [0, 0, 39, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 51, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 51, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 25], - [0, 22, 19, 37, 62, 62, 36, 61, 62, 62, 18, 12, 0, 30, 49, 38], - [0, 29, 9, 0, 34, 52, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [46, 55, 62, 17, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [62, 6, 42, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [62, 56, 33, 60, 28, 57, 62, 62, 62, 62, 48, 35, 40, 45, 47, 5], - [62, 62, 59, 62, 41, 53, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0], + static SHORT_OFFSET_RUNS: [u32; 22] = [ + 4256, 115348384, 136322176, 144711446, 163587254, 320875520, 325101120, 350268208, + 392231680, 404815649, 413205504, 421595008, 467733632, 484513952, 492924480, 497144832, + 501339814, 578936576, 627171376, 639756544, 643952944, 649261450, ]; - static BITSET: [u64; 63] = [ - 0, 15, 24, 511, 1023, 4087, 65535, 16253055, 134217726, 536805376, 1073741823, 4294967295, - 133143986179, 4398046511103, 36009005809663, 70368744177663, 2251799813685247, - 3509778554814463, 144115188074807295, 297241973452963840, 531424756029720572, - 576460743713488896, 576460743847706622, 1152921504591118335, 2295745090394464220, - 4557642822898941951, 4611686017001275199, 6908521828386340863, 8935141660164089791, - 9223934986808197120, 13605092999309557792, 16717361816799216127, 16717361816799223999, - 17005555242810474495, 17446871633794956420, 17870283321271910397, 17870283321406128127, - 18410715276682199039, 18428729675200069631, 18428729675200069632, 18437736874452713471, - 18446462598732840959, 18446462598732840960, 18446464797621878783, 18446466996779287551, - 18446603336221163519, 18446603336221196287, 18446741874686295551, 18446743249075830783, - 18446744056529672000, 18446744056529682432, 18446744069414584320, 18446744069414601696, - 18446744069422972927, 18446744070475743231, 18446744071562067967, 18446744073707454463, - 18446744073709419615, 18446744073709517055, 18446744073709550595, 18446744073709551599, - 18446744073709551600, 18446744073709551615, + static OFFSETS: [u8; 315] = [ + 65, 26, 6, 26, 47, 1, 10, 1, 4, 1, 5, 23, 1, 31, 1, 195, 1, 4, 4, 208, 1, 36, 7, 2, 30, 5, + 96, 1, 42, 4, 2, 2, 2, 4, 1, 1, 6, 1, 1, 3, 1, 1, 1, 20, 1, 83, 1, 139, 8, 166, 1, 38, 9, + 41, 0, 38, 1, 1, 5, 1, 2, 43, 1, 4, 0, 86, 2, 6, 0, 9, 7, 43, 2, 3, 64, 192, 64, 0, 2, 6, 2, + 38, 2, 6, 2, 8, 1, 1, 1, 1, 1, 1, 1, 31, 2, 53, 1, 7, 1, 1, 3, 3, 1, 7, 3, 4, 2, 6, 4, 13, + 5, 3, 1, 7, 116, 1, 13, 1, 16, 13, 101, 1, 4, 1, 2, 10, 1, 1, 3, 5, 6, 1, 1, 1, 1, 1, 1, 4, + 1, 6, 4, 1, 2, 4, 5, 5, 4, 1, 17, 32, 3, 2, 0, 52, 0, 229, 6, 4, 3, 2, 12, 38, 1, 1, 5, 1, + 0, 46, 18, 30, 132, 102, 3, 4, 1, 59, 5, 2, 1, 1, 1, 5, 24, 5, 1, 3, 0, 43, 1, 14, 6, 80, 0, + 7, 12, 5, 0, 26, 6, 26, 0, 80, 96, 36, 4, 36, 116, 11, 1, 15, 1, 7, 1, 2, 1, 11, 1, 15, 1, + 7, 1, 2, 0, 1, 2, 3, 1, 42, 1, 9, 0, 51, 13, 51, 0, 64, 0, 64, 0, 85, 1, 71, 1, 2, 2, 1, 2, + 2, 2, 4, 1, 12, 1, 1, 1, 7, 1, 65, 1, 4, 2, 8, 1, 7, 1, 28, 1, 4, 1, 5, 1, 1, 3, 7, 1, 0, 2, + 25, 1, 25, 1, 31, 1, 25, 1, 31, 1, 25, 1, 31, 1, 25, 1, 31, 1, 25, 1, 8, 0, 10, 1, 20, 6, 6, + 0, 62, 0, 68, 0, 26, 6, 26, 6, 26, 0, ]; - pub fn lookup(c: char) -> bool { - super::range_search( + super::skip_search( c as u32, - &BITSET_CHUNKS_MAP, - BITSET_LAST_CHUNK_MAP, - &BITSET_INDEX_CHUNKS, - &BITSET, + &SHORT_OFFSET_RUNS, + &OFFSETS, ) } } #[rustfmt::skip] pub mod cc { - static BITSET_LAST_CHUNK_MAP: (u16, u8) = (0, 0); - static BITSET_CHUNKS_MAP: [u8; 0] = [ + static SHORT_OFFSET_RUNS: [u32; 1] = [ + 1114272, ]; - static BITSET_INDEX_CHUNKS: [[u8; 16]; 1] = [ - [1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + static OFFSETS: [u8; 5] = [ + 0, 32, 95, 33, 0, ]; - static BITSET: [u64; 3] = [ - 0, 4294967295, 9223372036854775808, - ]; - pub fn lookup(c: char) -> bool { - super::range_search( + super::skip_search( c as u32, - &BITSET_CHUNKS_MAP, - BITSET_LAST_CHUNK_MAP, - &BITSET_INDEX_CHUNKS, - &BITSET, + &SHORT_OFFSET_RUNS, + &OFFSETS, ) } } #[rustfmt::skip] pub mod grapheme_extend { - static BITSET_LAST_CHUNK_MAP: (u16, u8) = (896, 30); - static BITSET_CHUNKS_MAP: [u8; 123] = [ - 4, 15, 21, 27, 25, 3, 18, 23, 17, 0, 0, 14, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 2, 7, 10, 0, 8, 12, 29, 28, 24, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, - 11, 0, 9, 0, 19, 0, 13, + static SHORT_OFFSET_RUNS: [u32; 33] = [ + 768, 2098307, 6292881, 10490717, 522196754, 526393356, 731917551, 740306986, 752920175, + 761309186, 778107678, 908131840, 912326558, 920715773, 924912129, 937495844, 962662059, + 966858799, 1214323760, 1285627635, 1348547648, 1369533168, 1377922895, 1386331293, + 1398918912, 1403113829, 1411504640, 1440866304, 1466032814, 1495393516, 1503783120, + 1508769824, 1518273008, ]; - static BITSET_INDEX_CHUNKS: [[u8; 16]; 31] = [ - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 20, 46], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 74, 106, 31], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 66, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 87, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 37, 70, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 37, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 48, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 134, 82, 64, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 39, 0, 94, 0, 0], - [0, 0, 0, 0, 0, 133, 58, 0, 0, 5, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 55, 0, 0, 18, 0, 0], - [0, 0, 0, 21, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 71, 0, 118, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 9, 0, 0, 0, 129, 7, 26, 67, 0, 59, 140, 11, 68, 104], - [0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [12, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [13, 0, 50, 0, 96, 0, 0, 0, 27, 123, 139, 1, 100, 75, 57, 72], - [51, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0], - [54, 0, 0, 120, 61, 19, 105, 47, 85, 98, 131, 84, 0, 0, 0, 56], - [60, 28, 0, 141, 99, 45, 111, 109, 97, 83, 97, 136, 132, 44, 108, 22], - [63, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [89, 0, 0, 91, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0], - [93, 0, 0, 0, 113, 3, 0, 40, 115, 29, 24, 0, 0, 0, 0, 0], - [114, 78, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0], - [128, 0, 86, 0, 127, 8, 23, 0, 0, 0, 0, 76, 0, 0, 0, 0], - [130, 42, 122, 41, 112, 43, 2, 36, 95, 15, 101, 14, 90, 117, 102, 6], - [137, 34, 124, 4, 0, 0, 126, 32, 125, 1, 88, 0, 53, 0, 0, 0], - [138, 119, 92, 0, 81, 73, 116, 17, 110, 0, 0, 52, 112, 80, 0, 0], - [142, 143, 0, 0, 143, 143, 143, 66, 0, 0, 0, 0, 0, 0, 0, 0], - ]; - static BITSET: [u64; 144] = [ - 0, 1, 2, 8, 13, 28, 64, 182, 191, 1016, 2032, 2047, 4096, 14336, 16128, 32640, 32768, - 40448, 131008, 262016, 491520, 8323072, 8396801, 10682368, 58719232, 100663296, 134152192, - 159383552, 234881024, 243138688, 536879204, 537919040, 805306369, 1073741824, 1073741916, - 1610612736, 2153546752, 3221225472, 3758096384, 4294967296, 4512022528, 51545911364, - 51545914817, 51548004382, 51554295838, 51556262398, 68719476736, 137438953472, 412316860416, - 1030792151040, 2199023255648, 8641373536127, 8763880767488, 17303886364672, 18004502906948, - 26388279066624, 36421322670080, 65128884076547, 65970697670631, 67755789254656, - 69200441769984, 70093866270720, 263882790666240, 277076930199552, 281470547525648, - 281470681808895, 281474976710655, 281479271675904, 562675075514368, 562949953355776, - 844424930131968, 985162418487296, 1023920203366400, 2251799813685248, 3377699721314304, - 4494803534348292, 6755399441055744, 7881299349733376, 8444256867844096, 8725724278030336, - 8760633780600833, 8989057312882695, 9042383626829823, 9851624185018758, 18067175067615234, - 28848986089586688, 30958948903026688, 35747322042253312, 53805701016846336, - 58529202969772032, 189151184399892480, 220713756545974272, 466122561432846339, - 504262420777140224, 558446353793941504, 572520102629474304, 1009933895770046464, - 1152921504606846982, 1152921504606851080, 1441151880758558727, 1724878657282899983, - 2301902359539744768, 2305843009196908767, 2305843009213693952, 2310337812748042240, - 3731232291276455943, 4589168020290535424, 4609293481125347328, 4611686018427387908, - 4611686069975392286, 4671217976001691648, 5764607523034234882, 6341068275337658371, - 6341349750314369024, 7421334051581067264, 8788774672813524990, 9205357638345293827, - 9222809086901354496, 9223372036854775808, 9223372036854775935, 9224497932466651184, - 9727775195120332910, 10376293541461622786, 11526998316797657088, 11959590285459062784, - 12103423998558208000, 12699165786766311424, 13005832773892571136, 13798747783286489088, - 13835058055282032640, 13835058055282163729, 13951307220663664640, 14987979559889010690, - 17872468738205286400, 17906312118425092095, 18158513697557839871, 18158513749097456062, - 18374686479671623680, 18374686479671623682, 18446462598732840960, 18446462598732972032, - 18446744056529158144, 18446744069414584320, 18446744073709551615, + static OFFSETS: [u8; 727] = [ + 0, 112, 0, 7, 0, 45, 1, 1, 1, 2, 1, 2, 1, 1, 72, 11, 48, 21, 16, 1, 101, 7, 2, 6, 2, 2, 1, + 4, 35, 1, 30, 27, 91, 11, 58, 9, 9, 1, 24, 4, 1, 9, 1, 3, 1, 5, 43, 3, 60, 8, 42, 24, 1, 32, + 55, 1, 1, 1, 4, 8, 4, 1, 3, 7, 10, 2, 29, 1, 58, 1, 1, 1, 2, 4, 8, 1, 9, 1, 10, 2, 26, 1, 2, + 2, 57, 1, 4, 2, 4, 2, 2, 3, 3, 1, 30, 2, 3, 1, 11, 2, 57, 1, 4, 5, 1, 2, 4, 1, 20, 2, 22, 6, + 1, 1, 58, 1, 1, 2, 1, 4, 8, 1, 7, 3, 10, 2, 30, 1, 59, 1, 1, 1, 12, 1, 9, 1, 40, 1, 3, 1, + 55, 1, 1, 3, 5, 3, 1, 4, 7, 2, 11, 2, 29, 1, 58, 1, 2, 1, 2, 1, 3, 1, 5, 2, 7, 2, 11, 2, 28, + 2, 57, 2, 1, 1, 2, 4, 8, 1, 9, 1, 10, 2, 29, 1, 72, 1, 4, 1, 2, 3, 1, 1, 8, 1, 81, 1, 2, 7, + 12, 8, 98, 1, 2, 9, 11, 7, 73, 2, 27, 1, 1, 1, 1, 1, 55, 14, 1, 5, 1, 2, 5, 11, 1, 36, 9, 1, + 102, 4, 1, 6, 1, 2, 2, 2, 25, 2, 4, 3, 16, 4, 13, 1, 2, 2, 6, 1, 15, 1, 0, 3, 0, 3, 29, 2, + 30, 2, 30, 2, 64, 2, 1, 7, 8, 1, 2, 11, 9, 1, 45, 3, 1, 1, 117, 2, 34, 1, 118, 3, 4, 2, 9, + 1, 6, 3, 219, 2, 2, 1, 58, 1, 1, 7, 1, 1, 1, 1, 2, 8, 6, 10, 2, 1, 48, 31, 49, 4, 48, 7, 1, + 1, 5, 1, 40, 9, 12, 2, 32, 4, 2, 2, 1, 3, 56, 1, 1, 2, 3, 1, 1, 3, 58, 8, 2, 2, 152, 3, 1, + 13, 1, 7, 4, 1, 6, 1, 3, 2, 198, 64, 0, 1, 195, 33, 0, 3, 141, 1, 96, 32, 0, 6, 105, 2, 0, + 4, 1, 10, 32, 2, 80, 2, 0, 1, 3, 1, 4, 1, 25, 2, 5, 1, 151, 2, 26, 18, 13, 1, 38, 8, 25, 11, + 46, 3, 48, 1, 2, 4, 2, 2, 39, 1, 67, 6, 2, 2, 2, 2, 12, 1, 8, 1, 47, 1, 51, 1, 1, 3, 2, 2, + 5, 2, 1, 1, 42, 2, 8, 1, 238, 1, 2, 1, 4, 1, 0, 1, 0, 16, 16, 16, 0, 2, 0, 1, 226, 1, 149, + 5, 0, 3, 1, 2, 5, 4, 40, 3, 4, 1, 165, 2, 0, 4, 0, 2, 80, 3, 70, 11, 49, 4, 123, 1, 54, 15, + 41, 1, 2, 2, 10, 3, 49, 4, 2, 2, 7, 1, 61, 3, 36, 5, 1, 8, 62, 1, 12, 2, 52, 9, 10, 4, 2, 1, + 95, 3, 2, 1, 1, 2, 6, 1, 2, 1, 157, 1, 3, 8, 21, 2, 57, 2, 1, 1, 1, 1, 22, 1, 14, 7, 3, 5, + 195, 8, 2, 3, 1, 1, 23, 1, 81, 1, 2, 6, 1, 1, 2, 1, 1, 2, 1, 2, 235, 1, 2, 4, 6, 2, 1, 2, + 27, 2, 85, 8, 2, 1, 1, 2, 106, 1, 1, 1, 2, 6, 1, 1, 101, 3, 2, 4, 1, 5, 0, 9, 1, 2, 245, 1, + 10, 2, 1, 1, 4, 1, 144, 4, 2, 2, 4, 1, 32, 10, 40, 6, 2, 4, 8, 1, 9, 6, 2, 3, 46, 13, 1, 2, + 0, 7, 1, 6, 1, 1, 82, 22, 2, 7, 1, 2, 1, 2, 122, 6, 3, 1, 1, 2, 1, 7, 1, 1, 72, 2, 3, 1, 1, + 1, 0, 2, 11, 2, 52, 5, 5, 1, 1, 1, 0, 1, 6, 15, 0, 5, 59, 7, 0, 1, 63, 4, 81, 1, 0, 2, 0, + 46, 2, 23, 0, 1, 1, 3, 4, 5, 8, 8, 2, 7, 30, 4, 148, 3, 0, 55, 4, 50, 8, 1, 14, 1, 22, 5, 1, + 15, 0, 7, 1, 17, 2, 7, 1, 2, 1, 5, 100, 1, 160, 7, 0, 1, 61, 4, 0, 4, 0, 7, 109, 7, 0, 96, + 128, 240, 0, ]; - pub fn lookup(c: char) -> bool { - super::range_search( + super::skip_search( c as u32, - &BITSET_CHUNKS_MAP, - BITSET_LAST_CHUNK_MAP, - &BITSET_INDEX_CHUNKS, - &BITSET, + &SHORT_OFFSET_RUNS, + &OFFSETS, ) } } #[rustfmt::skip] pub mod lowercase { - static BITSET_LAST_CHUNK_MAP: (u16, u8) = (122, 6); - static BITSET_CHUNKS_MAP: [u8; 118] = [ - 12, 16, 0, 0, 10, 0, 0, 11, 13, 8, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 2, 1, 0, 17, 0, 9, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, + const BITSET_CHUNKS_MAP: &'static [u8; 123] = &[ + 14, 17, 0, 0, 9, 0, 0, 12, 13, 10, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 4, 1, 0, 15, 0, 8, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, + 3, 18, 0, 7, ]; - static BITSET_INDEX_CHUNKS: [[u8; 16]; 18] = [ + const BITSET_INDEX_CHUNKS: &'static [[u8; 16]; 20] = &[ [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 62, 71, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 9, 0, 50, 42, 44, 28], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 68, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35], - [0, 0, 3, 0, 71, 71, 71, 0, 46, 46, 48, 46, 24, 37, 38, 23], - [0, 29, 27, 57, 39, 51, 52, 43, 41, 70, 26, 11, 0, 34, 64, 32], - [0, 40, 8, 0, 33, 60, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [22, 13, 54, 66, 25, 15, 56, 63, 30, 19, 12, 55, 58, 61, 65, 4], - [59, 36, 46, 21, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [59, 49, 45, 47, 18, 69, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [67, 5, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 14, 55, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 43, 0, 51, 47, 49, 33], + [0, 0, 0, 0, 10, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27], + [0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 57, 0, 55, 55, 55, 0, 22, 22, 67, 22, 36, 25, 24, 37], + [0, 5, 68, 0, 29, 15, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 64, 34, 17, 23, 52, 53, 48, 46, 8, 35, 42, 0, 28, 13, 31], + [11, 58, 0, 6, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 32, 0], + [16, 26, 22, 38, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [16, 50, 2, 21, 66, 9, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [16, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [63, 41, 54, 12, 75, 61, 18, 1, 7, 62, 74, 20, 71, 72, 4, 45], ]; - static BITSET: [u64; 72] = [ - 0, 15, 16, 511, 3063, 65535, 16253055, 134217726, 536805376, 984263338, 4294967295, - 133143986179, 274877905920, 1099509514240, 4398046445568, 17592185782272, 36009005809663, - 46912496118442, 187649984473770, 281474972516352, 2251799813685247, 2339875276368554, - 4503599560261632, 61925590106570972, 71777214282006783, 72057592964186127, - 144115188074807295, 297241973452963840, 522417556774978824, 576460743713488896, - 1152921487426978047, 1152921504590069760, 1814856824841797631, 3607524039012697088, - 4362299189061746720, 4539628424389459968, 4601013482110844927, 4611405638684049471, - 4674456033467236607, 6172933889249159850, 9223934986808197120, 10663022717737544362, - 10808545280696953514, 12261519110656315968, 12294970652241842346, 12297829382473033730, - 12297829382473034410, 12297829382473045332, 12297829382829550250, 12297829383904690175, - 12298110845996498944, 15324248332066007893, 16596095761559859497, 16717361816799215616, - 16987577794709946364, 17293822586148356092, 18158513701852807104, 18410715274543104000, - 18428729675466407935, 18446462598732840960, 18446462598732858304, 18446462598737002495, - 18446464797621878783, 18446673704966422527, 18446726481523572736, 18446739675663105535, - 18446739675663106031, 18446742974197923840, 18446744056529682432, 18446744069414584320, - 18446744073709529733, 18446744073709551615, + const BITSET_CANONICAL: &'static [u64; 55] = &[ + 0b0000000000000000000000000000000000000000000000000000000000000000, + 0b1111111111111111110000000000000000000000000011111111111111111111, + 0b1010101010101010101010101010101010101010101010101010100000000010, + 0b0000000000000111111111111111111111111111111111111111111111111111, + 0b1111111111111111111111000000000000000000000000001111110111111111, + 0b1000000000000010000000000000000000000000000000000000000000000000, + 0b0000111111111111111111111111111111111111000000000000000000000000, + 0b0000111111111111111111111111110000000000000000000000000011111111, + 0b1111111111111111111111111111111111111111111111111010101010000101, + 0b1111111111111111111111111111111100000000000000000000000000000000, + 0b1111111111111111111111111111110000000000000000000000000000000000, + 0b1111111111111111111111110000000000000000000000000000000000000000, + 0b1111111111111111111111000000000000000000000000001111111111101111, + 0b1111111111111111111100000000000000000000000000010000000000000000, + 0b1111111111111111000000111111111111110111111111111111111111111111, + 0b1111111111111111000000000000000000000000000000000100001111000000, + 0b1111111111111111000000000000000000000000000000000000000000000000, + 0b1111111101111111111111111111111110000000000000000000000000000000, + 0b1111110000000000000000000000000011111111111111111111111111000000, + 0b1111011111111111111111111111111111111111111111110000000000000000, + 0b1111000000000000000000000000001111110111111111111111111111111100, + 0b1010101010101010101010101010101010101010101010101101010101010100, + 0b1010101010101010101010101010101010101010101010101010101010101010, + 0b0101010110101010101010101010101010101010101010101010101010101010, + 0b0100000011011111000000001111111100000000111111110000000011111111, + 0b0011111111111111000000001111111100000000111111110000000000111111, + 0b0011111111011010000101010110001011111111111111111111111111111111, + 0b0011111100000000000000000000000000000000000000000000000000000000, + 0b0011110010001010000000000000000000000000000000000000000000100000, + 0b0011001000010000100000000000000000000000000010001100010000000000, + 0b0001101111111011111111111111101111111111100000000000000000000000, + 0b0001100100101111101010101010101010101010111000110111111111111111, + 0b0000011111111101111111111111111111111111111111111111111110111001, + 0b0000011101011100000000000000000000000010101010100000010100001010, + 0b0000010000100000000001000000000000000000000000000000000000000000, + 0b0000000111111111111111111111111111111111111011111111111111111111, + 0b0000000011111111000000001111111100000000001111110000000011111111, + 0b0000000011011100000000001111111100000000110011110000000011011100, + 0b0000000000001000010100000001101010101010101010101010101010101010, + 0b0000000000000000001000001011111111111111111111111111111111111111, + 0b0000000000000000000001111110000001111111111111111111101111111111, + 0b0000000000000000000000001111111111111111110111111100000000000000, + 0b0000000000000000000000000001111100000000000000000000000000000011, + 0b0000000000000000000000000000000000111010101010101010101010101010, + 0b0000000000000000000000000000000000000000111110000000000001111111, + 0b0000000000000000000000000000000000000000000000000000101111110111, + 0b1001001111111010101010101010101010101010101010101010101010101010, + 0b1001010111111111101010101010101010101010101010101010101010101010, + 0b1010101000101001101010101010101010110101010101010101001001000000, + 0b1010101010100000100000101010101010101010101110100101000010101010, + 0b1010101010101010101010101010101011111111111111111111111111111111, + 0b1010101010101011101010101010100000000000000000000000000000000000, + 0b1101010010101010101010101010101010101010101010101010101101010101, + 0b1110011001010001001011010010101001001110001001000011000100101001, + 0b1110101111000000000000000000000000001111111111111111111111111100, + ]; + const BITSET_MAPPING: &'static [(u8, u8); 21] = &[ + (0, 64), (1, 188), (1, 183), (1, 176), (1, 109), (1, 124), (1, 126), (1, 66), (1, 70), + (1, 77), (2, 146), (2, 144), (2, 83), (3, 93), (3, 147), (3, 133), (4, 12), (4, 6), + (5, 187), (6, 78), (7, 132), ]; - pub fn lookup(c: char) -> bool { - super::range_search( + #[rustc_const_unstable(feature = "const_unicode_case_lookup", issue = "101400")] + pub const fn lookup(c: char) -> bool { + super::bitset_search( c as u32, &BITSET_CHUNKS_MAP, - BITSET_LAST_CHUNK_MAP, &BITSET_INDEX_CHUNKS, - &BITSET, + &BITSET_CANONICAL, + &BITSET_MAPPING, ) } } #[rustfmt::skip] pub mod n { - static BITSET_LAST_CHUNK_MAP: (u16, u8) = (127, 0); - static BITSET_CHUNKS_MAP: [u8; 127] = [ - 31, 8, 11, 25, 19, 4, 29, 21, 24, 28, 0, 16, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 3, 13, 18, 26, 17, 23, 20, 15, 22, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 5, 2, 0, 0, 10, 0, 14, 27, 12, 0, 1, + static SHORT_OFFSET_RUNS: [u32; 39] = [ + 1632, 18876774, 31461440, 102765417, 111154926, 115349830, 132128880, 165684320, 186656630, + 195046653, 199241735, 203436434, 216049184, 241215536, 249605104, 274792208, 278987015, + 283181793, 295766104, 320933114, 383848032, 392238160, 434181712, 442570976, 455154768, + 463544144, 476128256, 484534880, 488730240, 505533120, 509728718, 522314048, 526508784, + 530703600, 534898887, 539094129, 547483904, 568458224, 573766650, ]; - static BITSET_INDEX_CHUNKS: [[u8; 16]; 34] = [ - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 49], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 43, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 22, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 47, 0, 0, 0, 2], - [0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 31, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 31, 0, 45, 0, 31, 0, 31, 0, 41, 0, 34], - [0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 37, 44, 4, 0, 0, 0, 0, 52, 23, 3, 0, 13], - [0, 0, 0, 7, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 35, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 62, 47, 0, 0, 0, 0, 60, 0, 0, 24, 10, 0, 5], - [0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 2, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0], - [0, 15, 0, 15, 0, 0, 0, 0, 0, 15, 0, 2, 51, 0, 0, 0], - [0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 26, 0, 0, 0, 15, 25, 0, 0, 0, 0, 0, 0, 0, 0, 11], - [0, 32, 0, 47, 65, 0, 0, 39, 0, 0, 0, 47, 0, 0, 0, 0], - [0, 46, 2, 0, 0, 71, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 59, 0, 31, 0, 42, 0, 31, 0, 15, 0, 15, 36, 0, 0, 0], - [0, 63, 30, 61, 18, 0, 55, 70, 0, 57, 20, 28, 0, 64, 29, 0], - [0, 66, 38, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 69, 19, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 9, 0], - [15, 0, 0, 0, 0, 8, 0, 17, 0, 0, 16, 0, 0, 15, 47, 0], - [40, 0, 0, 15, 2, 0, 0, 48, 0, 15, 0, 0, 0, 0, 0, 47], - [47, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [50, 0, 0, 0, 0, 0, 12, 0, 25, 21, 67, 0, 0, 0, 0, 0], - [73, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - ]; - static BITSET: [u64; 74] = [ - 0, 999, 1023, 1026, 3072, 4064, 8191, 65408, 65472, 1048575, 1966080, 2097151, 3932160, - 4063232, 8388607, 67043328, 67044351, 134152192, 264241152, 268435455, 3758096384, - 4294901504, 17112694784, 64424509440, 549218942976, 4393751543808, 35184372023296, - 140737488355327, 272678883688448, 279275953455104, 280925220896768, 281200098803712, - 281474976448512, 492581209243648, 2251524935778304, 2251795518717952, 4503595332403200, - 4503599627370368, 8708132091985919, 9007190731849728, 17732923532771328, 71212894229889024, - 144114915328655360, 144115183780888576, 144115188075855871, 284007976623144960, - 284008251501051904, 287948901175001088, 287948901242044416, 287953294926544896, - 504407547722072192, 1152640029630136320, 1152921496016912384, 2305840810190438400, - 2305843009213693952, 3458764513820540928, 4611615649683210238, 6917529027641082367, - 8217943420044312576, 9151595642915651584, 9223372032559808512, 17870283321406128128, - 18158513697557839872, 18302628889911885824, 18374686483949813760, 18428729675200069632, - 18446181123756130304, 18446181123756131327, 18446739675663040512, 18446744069414584320, - 18446744073709355007, 18446744073709486080, 18446744073709535232, 18446744073709551615, + static OFFSETS: [u8; 275] = [ + 48, 10, 120, 2, 5, 1, 2, 3, 0, 10, 134, 10, 198, 10, 0, 10, 118, 10, 4, 6, 108, 10, 118, + 10, 118, 10, 2, 6, 110, 13, 115, 10, 8, 7, 103, 10, 104, 7, 7, 19, 109, 10, 96, 10, 118, 10, + 70, 20, 0, 10, 70, 10, 0, 20, 0, 3, 239, 10, 6, 10, 22, 10, 0, 10, 128, 11, 165, 10, 6, 10, + 182, 10, 86, 10, 134, 10, 6, 10, 0, 1, 3, 6, 6, 10, 198, 51, 2, 5, 0, 60, 78, 22, 0, 30, 0, + 1, 0, 1, 25, 9, 14, 3, 0, 4, 138, 10, 30, 8, 1, 15, 32, 10, 39, 15, 0, 10, 188, 10, 0, 6, + 154, 10, 38, 10, 198, 10, 22, 10, 86, 10, 0, 10, 0, 10, 0, 45, 12, 57, 17, 2, 0, 27, 36, 4, + 29, 1, 8, 1, 134, 5, 202, 10, 0, 8, 25, 7, 39, 9, 75, 5, 22, 6, 160, 2, 2, 16, 2, 46, 64, 9, + 52, 2, 30, 3, 75, 5, 104, 8, 24, 8, 41, 7, 0, 6, 48, 10, 0, 31, 158, 10, 42, 4, 112, 7, 134, + 30, 128, 10, 60, 10, 144, 10, 7, 20, 251, 10, 0, 10, 118, 10, 0, 10, 102, 10, 102, 12, 0, + 19, 93, 10, 0, 29, 227, 10, 70, 10, 0, 10, 102, 21, 0, 111, 0, 10, 86, 10, 134, 10, 1, 7, 0, + 23, 0, 20, 12, 20, 108, 25, 0, 50, 0, 10, 0, 10, 0, 10, 0, 9, 128, 10, 0, 59, 1, 3, 1, 4, + 76, 45, 1, 15, 0, 13, 0, 10, 0, ]; - pub fn lookup(c: char) -> bool { - super::range_search( + super::skip_search( c as u32, - &BITSET_CHUNKS_MAP, - BITSET_LAST_CHUNK_MAP, - &BITSET_INDEX_CHUNKS, - &BITSET, + &SHORT_OFFSET_RUNS, + &OFFSETS, ) } } #[rustfmt::skip] pub mod uppercase { - static BITSET_LAST_CHUNK_MAP: (u16, u8) = (124, 6); - static BITSET_CHUNKS_MAP: [u8; 123] = [ - 12, 15, 0, 0, 11, 0, 0, 8, 5, 9, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 0, 13, 0, 7, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, - 0, 0, 4, + const BITSET_CHUNKS_MAP: &'static [u8; 125] = &[ + 12, 15, 6, 6, 0, 6, 6, 2, 4, 11, 6, 16, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 8, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 5, 6, 14, 6, 10, 6, 6, 1, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 13, 6, 6, + 6, 6, 9, 6, 3, ]; - static BITSET_INDEX_CHUNKS: [[u8; 16]; 17] = [ - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 9, 0, 38, 46, 44, 28], - [0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 51, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 60, 62, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 54, 0, 0, 0, 0, 0, 43, 43, 40, 43, 56, 22, 34, 35], - [0, 0, 57, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 66, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 30], - [0, 10, 0, 11, 50, 37, 36, 45, 47, 5, 0, 0, 0, 49, 17, 53], - [14, 0, 60, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [21, 52, 43, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [24, 39, 42, 41, 59, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [58, 65, 29, 16, 48, 63, 31, 19, 55, 61, 64, 32, 27, 20, 15, 3], + const BITSET_INDEX_CHUNKS: &'static [[u8; 16]; 17] = &[ + [43, 43, 5, 34, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 5, 1], + [43, 43, 5, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43], + [43, 43, 39, 43, 43, 43, 43, 43, 17, 17, 62, 17, 42, 29, 24, 23], + [43, 43, 43, 43, 9, 8, 44, 43, 43, 43, 43, 43, 43, 43, 43, 43], + [43, 43, 43, 43, 36, 28, 66, 43, 43, 43, 43, 43, 43, 43, 43, 43], + [43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 0, 43, 43, 43], + [43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43], + [43, 43, 43, 43, 43, 43, 43, 43, 43, 54, 43, 43, 43, 43, 43, 43], + [43, 43, 43, 43, 43, 43, 43, 43, 43, 61, 60, 43, 20, 14, 16, 4], + [43, 43, 43, 43, 55, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43], + [43, 43, 58, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43], + [43, 43, 59, 45, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43], + [43, 48, 43, 31, 35, 21, 22, 15, 13, 33, 43, 43, 43, 11, 30, 38], + [51, 53, 26, 49, 12, 7, 25, 50, 40, 52, 6, 3, 65, 64, 63, 67], + [56, 43, 9, 46, 43, 41, 32, 43, 43, 43, 43, 43, 43, 43, 43, 43], + [57, 19, 2, 18, 10, 47, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43], + [57, 37, 17, 27, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43], + ]; + const BITSET_CANONICAL: &'static [u64; 43] = &[ + 0b0000011111111111111111111111111000000000000000000000000000000000, + 0b0000000000111111111111111111111111111111111111111111111111111111, + 0b0101010101010101010101010101010101010101010101010101010000000001, + 0b0000011111111111111111111111110000000000000000000000000000000001, + 0b0000000000100000000000000000000000000001010000010000001011110101, + 0b1111111111111111111111111111111100000000000000000000000000000000, + 0b1111111111111111111111110000000000000000000000000000001111111111, + 0b1111111111111111111100000000000000000000000000011111110001011111, + 0b1111111111111111000000111111111111111111111111110000001111111111, + 0b1111111111111111000000000000000000000000000000000000000000000000, + 0b1111111111111110010101010101010101010101010101010101010101010101, + 0b1000000001000101000000000000000000000000000000000000000000000000, + 0b0111101100000000000000000000000000011111110111111110011110110000, + 0b0110110000000101010101010101010101010101010101010101010101010101, + 0b0110101000000000010101010101010101010101010101010101010101010101, + 0b0101010111010010010101010101010101001010101010101010010010010000, + 0b0101010101011111011111010101010101010101010001010010100001010101, + 0b0101010101010101010101010101010101010101010101010101010101010101, + 0b0101010101010101010101010101010101010101010101010010101010101011, + 0b0101010101010101010101010101010100000000000000000000000000000000, + 0b0101010101010100010101010101010000000000000000000000000000000000, + 0b0010101101010101010101010101010101010101010101010101010010101010, + 0b0001000110101110110100101101010110110001110110111100111011010110, + 0b0000111100000000000111110000000000001111000000000000111100000000, + 0b0000111100000000000000000000000000000000000000000000000000000000, + 0b0000001111111111111111111111111100000000000000000000000000111111, + 0b0000000000111111110111100110010011010000000000000000000000000011, + 0b0000000000000100001010000000010101010101010101010101010101010101, + 0b0000000000000000111111111111111100000000000000000000000000100000, + 0b0000000000000000111111110000000010101010000000000011111100000000, + 0b0000000000000000000011111111101111111111111111101101011101000000, + 0b0000000000000000000000000000000001111111011111111111111111111111, + 0b0000000000000000000000000000000000000000001101111111011111111111, + 0b0000000000000000000000000000000000000000000000000101010101111010, + 0b0000000000000000000000000000000000000000000000000010000010111111, + 0b1010101001010101010101010101010101010101010101010101010101010101, + 0b1100000000001111001111010101000000111110001001110011100010000100, + 0b1100000000100101111010101001110100000000000000000000000000000000, + 0b1110011010010000010101010101010101010101000111001000000000000000, + 0b1110011111111111111111111111111111111111111111110000000000000000, + 0b1111000000000000000000000000001111111111111111111111111100000000, + 0b1111011111111111000000000000000000000000000000000000000000000000, + 0b1111111100000000111111110000000000111111000000001111111100000000, ]; - static BITSET: [u64; 67] = [ - 0, 8, 1023, 1024, 8383, 21882, 65535, 1048575, 8388607, 89478485, 134217726, 2139095039, - 4294967295, 17179869183, 1099511627775, 2199023190016, 4398046445568, 17575006099264, - 23456248059221, 70368743129088, 140737484161024, 140737488355327, 280378317225728, - 281470681743392, 281474976710655, 1169903278445909, 2251799813685247, 9007198986305536, - 9007199254741748, 17977448100528131, 18014398509481983, 288230371856744511, - 576460735123554305, 576460743713488896, 1080863910568919040, 1080897995681042176, - 1274187559846268630, 3122495741643543722, 6148633210533183488, 6148914689804861440, - 6148914690880001365, 6148914691236506283, 6148914691236516865, 6148914691236517205, - 6151773421467674709, 6184099063146390672, 7638198793012598101, 7783721355972007253, - 8863084067199903664, 9242793810247811072, 12273810184460391765, 13839347594782259332, - 13845730589451223040, 16613872850358272000, 16717361816799215616, 17293822586282573568, - 18374966856193736448, 18428729675200069632, 18442240474149289983, 18446274948748367189, - 18446462598732840960, 18446462598737035263, 18446466996779287551, 18446726481523637343, - 18446742974197924863, 18446742974197940223, 18446744069414584320, + const BITSET_MAPPING: &'static [(u8, u8); 25] = &[ + (0, 187), (0, 177), (0, 171), (0, 167), (0, 164), (0, 32), (0, 47), (0, 51), (0, 121), + (0, 117), (0, 109), (1, 150), (1, 148), (1, 142), (1, 134), (1, 131), (1, 64), (2, 164), + (2, 146), (2, 20), (3, 146), (3, 140), (3, 134), (4, 178), (4, 171), ]; - pub fn lookup(c: char) -> bool { - super::range_search( + #[rustc_const_unstable(feature = "const_unicode_case_lookup", issue = "101400")] + pub const fn lookup(c: char) -> bool { + super::bitset_search( c as u32, &BITSET_CHUNKS_MAP, - BITSET_LAST_CHUNK_MAP, &BITSET_INDEX_CHUNKS, - &BITSET, + &BITSET_CANONICAL, + &BITSET_MAPPING, ) } } #[rustfmt::skip] pub mod white_space { - static BITSET_LAST_CHUNK_MAP: (u16, u8) = (12, 2); - static BITSET_CHUNKS_MAP: [u8; 9] = [ - 3, 0, 0, 0, 0, 1, 0, 0, 4, - ]; - static BITSET_INDEX_CHUNKS: [[u8; 16]; 5] = [ - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0], - [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + static WHITESPACE_MAP: [u8; 256] = [ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, ]; - static BITSET: [u64; 6] = [ - 0, 1, 2147483648, 4294967328, 4294983168, 144036023240703, - ]; - + #[inline] pub fn lookup(c: char) -> bool { - super::range_search( - c as u32, - &BITSET_CHUNKS_MAP, - BITSET_LAST_CHUNK_MAP, - &BITSET_INDEX_CHUNKS, - &BITSET, - ) + match c as u32 >> 8 { + 0 => WHITESPACE_MAP[c as usize & 0xff] & 1 != 0, + 22 => c as u32 == 0x1680, + 32 => WHITESPACE_MAP[c as usize & 0xff] & 2 != 0, + 48 => c as u32 == 0x3000, + _ => false, + } } } #[rustfmt::skip] pub mod conversions { pub fn to_lower(c: char) -> [char; 3] { - match bsearch_case_table(c, LOWERCASE_TABLE) { - None => [c, '\0', '\0'], - Some(index) => LOWERCASE_TABLE[index].1, + if c.is_ascii() { + [(c as u8).to_ascii_lowercase() as char, '\0', '\0'] + } else { + match bsearch_case_table(c, LOWERCASE_TABLE) { + None => [c, '\0', '\0'], + Some(index) => LOWERCASE_TABLE[index].1, + } } } pub fn to_upper(c: char) -> [char; 3] { - match bsearch_case_table(c, UPPERCASE_TABLE) { - None => [c, '\0', '\0'], - Some(index) => UPPERCASE_TABLE[index].1, + if c.is_ascii() { + [(c as u8).to_ascii_uppercase() as char, '\0', '\0'] + } else { + match bsearch_case_table(c, UPPERCASE_TABLE) { + None => [c, '\0', '\0'], + Some(index) => UPPERCASE_TABLE[index].1, + } } } @@ -1114,113 +1083,116 @@ pub mod conversions { ('\u{2c28}', ['\u{2c58}', '\u{0}', '\u{0}']), ('\u{2c29}', ['\u{2c59}', '\u{0}', '\u{0}']), ('\u{2c2a}', ['\u{2c5a}', '\u{0}', '\u{0}']), ('\u{2c2b}', ['\u{2c5b}', '\u{0}', '\u{0}']), ('\u{2c2c}', ['\u{2c5c}', '\u{0}', '\u{0}']), ('\u{2c2d}', ['\u{2c5d}', '\u{0}', '\u{0}']), - ('\u{2c2e}', ['\u{2c5e}', '\u{0}', '\u{0}']), ('\u{2c60}', ['\u{2c61}', '\u{0}', '\u{0}']), - ('\u{2c62}', ['\u{26b}', '\u{0}', '\u{0}']), ('\u{2c63}', ['\u{1d7d}', '\u{0}', '\u{0}']), - ('\u{2c64}', ['\u{27d}', '\u{0}', '\u{0}']), ('\u{2c67}', ['\u{2c68}', '\u{0}', '\u{0}']), - ('\u{2c69}', ['\u{2c6a}', '\u{0}', '\u{0}']), ('\u{2c6b}', ['\u{2c6c}', '\u{0}', '\u{0}']), - ('\u{2c6d}', ['\u{251}', '\u{0}', '\u{0}']), ('\u{2c6e}', ['\u{271}', '\u{0}', '\u{0}']), - ('\u{2c6f}', ['\u{250}', '\u{0}', '\u{0}']), ('\u{2c70}', ['\u{252}', '\u{0}', '\u{0}']), - ('\u{2c72}', ['\u{2c73}', '\u{0}', '\u{0}']), ('\u{2c75}', ['\u{2c76}', '\u{0}', '\u{0}']), - ('\u{2c7e}', ['\u{23f}', '\u{0}', '\u{0}']), ('\u{2c7f}', ['\u{240}', '\u{0}', '\u{0}']), - ('\u{2c80}', ['\u{2c81}', '\u{0}', '\u{0}']), ('\u{2c82}', ['\u{2c83}', '\u{0}', '\u{0}']), - ('\u{2c84}', ['\u{2c85}', '\u{0}', '\u{0}']), ('\u{2c86}', ['\u{2c87}', '\u{0}', '\u{0}']), - ('\u{2c88}', ['\u{2c89}', '\u{0}', '\u{0}']), ('\u{2c8a}', ['\u{2c8b}', '\u{0}', '\u{0}']), - ('\u{2c8c}', ['\u{2c8d}', '\u{0}', '\u{0}']), ('\u{2c8e}', ['\u{2c8f}', '\u{0}', '\u{0}']), - ('\u{2c90}', ['\u{2c91}', '\u{0}', '\u{0}']), ('\u{2c92}', ['\u{2c93}', '\u{0}', '\u{0}']), - ('\u{2c94}', ['\u{2c95}', '\u{0}', '\u{0}']), ('\u{2c96}', ['\u{2c97}', '\u{0}', '\u{0}']), - ('\u{2c98}', ['\u{2c99}', '\u{0}', '\u{0}']), ('\u{2c9a}', ['\u{2c9b}', '\u{0}', '\u{0}']), - ('\u{2c9c}', ['\u{2c9d}', '\u{0}', '\u{0}']), ('\u{2c9e}', ['\u{2c9f}', '\u{0}', '\u{0}']), - ('\u{2ca0}', ['\u{2ca1}', '\u{0}', '\u{0}']), ('\u{2ca2}', ['\u{2ca3}', '\u{0}', '\u{0}']), - ('\u{2ca4}', ['\u{2ca5}', '\u{0}', '\u{0}']), ('\u{2ca6}', ['\u{2ca7}', '\u{0}', '\u{0}']), - ('\u{2ca8}', ['\u{2ca9}', '\u{0}', '\u{0}']), ('\u{2caa}', ['\u{2cab}', '\u{0}', '\u{0}']), - ('\u{2cac}', ['\u{2cad}', '\u{0}', '\u{0}']), ('\u{2cae}', ['\u{2caf}', '\u{0}', '\u{0}']), - ('\u{2cb0}', ['\u{2cb1}', '\u{0}', '\u{0}']), ('\u{2cb2}', ['\u{2cb3}', '\u{0}', '\u{0}']), - ('\u{2cb4}', ['\u{2cb5}', '\u{0}', '\u{0}']), ('\u{2cb6}', ['\u{2cb7}', '\u{0}', '\u{0}']), - ('\u{2cb8}', ['\u{2cb9}', '\u{0}', '\u{0}']), ('\u{2cba}', ['\u{2cbb}', '\u{0}', '\u{0}']), - ('\u{2cbc}', ['\u{2cbd}', '\u{0}', '\u{0}']), ('\u{2cbe}', ['\u{2cbf}', '\u{0}', '\u{0}']), - ('\u{2cc0}', ['\u{2cc1}', '\u{0}', '\u{0}']), ('\u{2cc2}', ['\u{2cc3}', '\u{0}', '\u{0}']), - ('\u{2cc4}', ['\u{2cc5}', '\u{0}', '\u{0}']), ('\u{2cc6}', ['\u{2cc7}', '\u{0}', '\u{0}']), - ('\u{2cc8}', ['\u{2cc9}', '\u{0}', '\u{0}']), ('\u{2cca}', ['\u{2ccb}', '\u{0}', '\u{0}']), - ('\u{2ccc}', ['\u{2ccd}', '\u{0}', '\u{0}']), ('\u{2cce}', ['\u{2ccf}', '\u{0}', '\u{0}']), - ('\u{2cd0}', ['\u{2cd1}', '\u{0}', '\u{0}']), ('\u{2cd2}', ['\u{2cd3}', '\u{0}', '\u{0}']), - ('\u{2cd4}', ['\u{2cd5}', '\u{0}', '\u{0}']), ('\u{2cd6}', ['\u{2cd7}', '\u{0}', '\u{0}']), - ('\u{2cd8}', ['\u{2cd9}', '\u{0}', '\u{0}']), ('\u{2cda}', ['\u{2cdb}', '\u{0}', '\u{0}']), - ('\u{2cdc}', ['\u{2cdd}', '\u{0}', '\u{0}']), ('\u{2cde}', ['\u{2cdf}', '\u{0}', '\u{0}']), - ('\u{2ce0}', ['\u{2ce1}', '\u{0}', '\u{0}']), ('\u{2ce2}', ['\u{2ce3}', '\u{0}', '\u{0}']), - ('\u{2ceb}', ['\u{2cec}', '\u{0}', '\u{0}']), ('\u{2ced}', ['\u{2cee}', '\u{0}', '\u{0}']), - ('\u{2cf2}', ['\u{2cf3}', '\u{0}', '\u{0}']), ('\u{a640}', ['\u{a641}', '\u{0}', '\u{0}']), - ('\u{a642}', ['\u{a643}', '\u{0}', '\u{0}']), ('\u{a644}', ['\u{a645}', '\u{0}', '\u{0}']), - ('\u{a646}', ['\u{a647}', '\u{0}', '\u{0}']), ('\u{a648}', ['\u{a649}', '\u{0}', '\u{0}']), - ('\u{a64a}', ['\u{a64b}', '\u{0}', '\u{0}']), ('\u{a64c}', ['\u{a64d}', '\u{0}', '\u{0}']), - ('\u{a64e}', ['\u{a64f}', '\u{0}', '\u{0}']), ('\u{a650}', ['\u{a651}', '\u{0}', '\u{0}']), - ('\u{a652}', ['\u{a653}', '\u{0}', '\u{0}']), ('\u{a654}', ['\u{a655}', '\u{0}', '\u{0}']), - ('\u{a656}', ['\u{a657}', '\u{0}', '\u{0}']), ('\u{a658}', ['\u{a659}', '\u{0}', '\u{0}']), - ('\u{a65a}', ['\u{a65b}', '\u{0}', '\u{0}']), ('\u{a65c}', ['\u{a65d}', '\u{0}', '\u{0}']), - ('\u{a65e}', ['\u{a65f}', '\u{0}', '\u{0}']), ('\u{a660}', ['\u{a661}', '\u{0}', '\u{0}']), - ('\u{a662}', ['\u{a663}', '\u{0}', '\u{0}']), ('\u{a664}', ['\u{a665}', '\u{0}', '\u{0}']), - ('\u{a666}', ['\u{a667}', '\u{0}', '\u{0}']), ('\u{a668}', ['\u{a669}', '\u{0}', '\u{0}']), - ('\u{a66a}', ['\u{a66b}', '\u{0}', '\u{0}']), ('\u{a66c}', ['\u{a66d}', '\u{0}', '\u{0}']), - ('\u{a680}', ['\u{a681}', '\u{0}', '\u{0}']), ('\u{a682}', ['\u{a683}', '\u{0}', '\u{0}']), - ('\u{a684}', ['\u{a685}', '\u{0}', '\u{0}']), ('\u{a686}', ['\u{a687}', '\u{0}', '\u{0}']), - ('\u{a688}', ['\u{a689}', '\u{0}', '\u{0}']), ('\u{a68a}', ['\u{a68b}', '\u{0}', '\u{0}']), - ('\u{a68c}', ['\u{a68d}', '\u{0}', '\u{0}']), ('\u{a68e}', ['\u{a68f}', '\u{0}', '\u{0}']), - ('\u{a690}', ['\u{a691}', '\u{0}', '\u{0}']), ('\u{a692}', ['\u{a693}', '\u{0}', '\u{0}']), - ('\u{a694}', ['\u{a695}', '\u{0}', '\u{0}']), ('\u{a696}', ['\u{a697}', '\u{0}', '\u{0}']), - ('\u{a698}', ['\u{a699}', '\u{0}', '\u{0}']), ('\u{a69a}', ['\u{a69b}', '\u{0}', '\u{0}']), - ('\u{a722}', ['\u{a723}', '\u{0}', '\u{0}']), ('\u{a724}', ['\u{a725}', '\u{0}', '\u{0}']), - ('\u{a726}', ['\u{a727}', '\u{0}', '\u{0}']), ('\u{a728}', ['\u{a729}', '\u{0}', '\u{0}']), - ('\u{a72a}', ['\u{a72b}', '\u{0}', '\u{0}']), ('\u{a72c}', ['\u{a72d}', '\u{0}', '\u{0}']), - ('\u{a72e}', ['\u{a72f}', '\u{0}', '\u{0}']), ('\u{a732}', ['\u{a733}', '\u{0}', '\u{0}']), - ('\u{a734}', ['\u{a735}', '\u{0}', '\u{0}']), ('\u{a736}', ['\u{a737}', '\u{0}', '\u{0}']), - ('\u{a738}', ['\u{a739}', '\u{0}', '\u{0}']), ('\u{a73a}', ['\u{a73b}', '\u{0}', '\u{0}']), - ('\u{a73c}', ['\u{a73d}', '\u{0}', '\u{0}']), ('\u{a73e}', ['\u{a73f}', '\u{0}', '\u{0}']), - ('\u{a740}', ['\u{a741}', '\u{0}', '\u{0}']), ('\u{a742}', ['\u{a743}', '\u{0}', '\u{0}']), - ('\u{a744}', ['\u{a745}', '\u{0}', '\u{0}']), ('\u{a746}', ['\u{a747}', '\u{0}', '\u{0}']), - ('\u{a748}', ['\u{a749}', '\u{0}', '\u{0}']), ('\u{a74a}', ['\u{a74b}', '\u{0}', '\u{0}']), - ('\u{a74c}', ['\u{a74d}', '\u{0}', '\u{0}']), ('\u{a74e}', ['\u{a74f}', '\u{0}', '\u{0}']), - ('\u{a750}', ['\u{a751}', '\u{0}', '\u{0}']), ('\u{a752}', ['\u{a753}', '\u{0}', '\u{0}']), - ('\u{a754}', ['\u{a755}', '\u{0}', '\u{0}']), ('\u{a756}', ['\u{a757}', '\u{0}', '\u{0}']), - ('\u{a758}', ['\u{a759}', '\u{0}', '\u{0}']), ('\u{a75a}', ['\u{a75b}', '\u{0}', '\u{0}']), - ('\u{a75c}', ['\u{a75d}', '\u{0}', '\u{0}']), ('\u{a75e}', ['\u{a75f}', '\u{0}', '\u{0}']), - ('\u{a760}', ['\u{a761}', '\u{0}', '\u{0}']), ('\u{a762}', ['\u{a763}', '\u{0}', '\u{0}']), - ('\u{a764}', ['\u{a765}', '\u{0}', '\u{0}']), ('\u{a766}', ['\u{a767}', '\u{0}', '\u{0}']), - ('\u{a768}', ['\u{a769}', '\u{0}', '\u{0}']), ('\u{a76a}', ['\u{a76b}', '\u{0}', '\u{0}']), - ('\u{a76c}', ['\u{a76d}', '\u{0}', '\u{0}']), ('\u{a76e}', ['\u{a76f}', '\u{0}', '\u{0}']), - ('\u{a779}', ['\u{a77a}', '\u{0}', '\u{0}']), ('\u{a77b}', ['\u{a77c}', '\u{0}', '\u{0}']), - ('\u{a77d}', ['\u{1d79}', '\u{0}', '\u{0}']), ('\u{a77e}', ['\u{a77f}', '\u{0}', '\u{0}']), - ('\u{a780}', ['\u{a781}', '\u{0}', '\u{0}']), ('\u{a782}', ['\u{a783}', '\u{0}', '\u{0}']), - ('\u{a784}', ['\u{a785}', '\u{0}', '\u{0}']), ('\u{a786}', ['\u{a787}', '\u{0}', '\u{0}']), - ('\u{a78b}', ['\u{a78c}', '\u{0}', '\u{0}']), ('\u{a78d}', ['\u{265}', '\u{0}', '\u{0}']), - ('\u{a790}', ['\u{a791}', '\u{0}', '\u{0}']), ('\u{a792}', ['\u{a793}', '\u{0}', '\u{0}']), - ('\u{a796}', ['\u{a797}', '\u{0}', '\u{0}']), ('\u{a798}', ['\u{a799}', '\u{0}', '\u{0}']), - ('\u{a79a}', ['\u{a79b}', '\u{0}', '\u{0}']), ('\u{a79c}', ['\u{a79d}', '\u{0}', '\u{0}']), - ('\u{a79e}', ['\u{a79f}', '\u{0}', '\u{0}']), ('\u{a7a0}', ['\u{a7a1}', '\u{0}', '\u{0}']), - ('\u{a7a2}', ['\u{a7a3}', '\u{0}', '\u{0}']), ('\u{a7a4}', ['\u{a7a5}', '\u{0}', '\u{0}']), - ('\u{a7a6}', ['\u{a7a7}', '\u{0}', '\u{0}']), ('\u{a7a8}', ['\u{a7a9}', '\u{0}', '\u{0}']), - ('\u{a7aa}', ['\u{266}', '\u{0}', '\u{0}']), ('\u{a7ab}', ['\u{25c}', '\u{0}', '\u{0}']), - ('\u{a7ac}', ['\u{261}', '\u{0}', '\u{0}']), ('\u{a7ad}', ['\u{26c}', '\u{0}', '\u{0}']), - ('\u{a7ae}', ['\u{26a}', '\u{0}', '\u{0}']), ('\u{a7b0}', ['\u{29e}', '\u{0}', '\u{0}']), - ('\u{a7b1}', ['\u{287}', '\u{0}', '\u{0}']), ('\u{a7b2}', ['\u{29d}', '\u{0}', '\u{0}']), - ('\u{a7b3}', ['\u{ab53}', '\u{0}', '\u{0}']), ('\u{a7b4}', ['\u{a7b5}', '\u{0}', '\u{0}']), - ('\u{a7b6}', ['\u{a7b7}', '\u{0}', '\u{0}']), ('\u{a7b8}', ['\u{a7b9}', '\u{0}', '\u{0}']), - ('\u{a7ba}', ['\u{a7bb}', '\u{0}', '\u{0}']), ('\u{a7bc}', ['\u{a7bd}', '\u{0}', '\u{0}']), - ('\u{a7be}', ['\u{a7bf}', '\u{0}', '\u{0}']), ('\u{a7c2}', ['\u{a7c3}', '\u{0}', '\u{0}']), + ('\u{2c2e}', ['\u{2c5e}', '\u{0}', '\u{0}']), ('\u{2c2f}', ['\u{2c5f}', '\u{0}', '\u{0}']), + ('\u{2c60}', ['\u{2c61}', '\u{0}', '\u{0}']), ('\u{2c62}', ['\u{26b}', '\u{0}', '\u{0}']), + ('\u{2c63}', ['\u{1d7d}', '\u{0}', '\u{0}']), ('\u{2c64}', ['\u{27d}', '\u{0}', '\u{0}']), + ('\u{2c67}', ['\u{2c68}', '\u{0}', '\u{0}']), ('\u{2c69}', ['\u{2c6a}', '\u{0}', '\u{0}']), + ('\u{2c6b}', ['\u{2c6c}', '\u{0}', '\u{0}']), ('\u{2c6d}', ['\u{251}', '\u{0}', '\u{0}']), + ('\u{2c6e}', ['\u{271}', '\u{0}', '\u{0}']), ('\u{2c6f}', ['\u{250}', '\u{0}', '\u{0}']), + ('\u{2c70}', ['\u{252}', '\u{0}', '\u{0}']), ('\u{2c72}', ['\u{2c73}', '\u{0}', '\u{0}']), + ('\u{2c75}', ['\u{2c76}', '\u{0}', '\u{0}']), ('\u{2c7e}', ['\u{23f}', '\u{0}', '\u{0}']), + ('\u{2c7f}', ['\u{240}', '\u{0}', '\u{0}']), ('\u{2c80}', ['\u{2c81}', '\u{0}', '\u{0}']), + ('\u{2c82}', ['\u{2c83}', '\u{0}', '\u{0}']), ('\u{2c84}', ['\u{2c85}', '\u{0}', '\u{0}']), + ('\u{2c86}', ['\u{2c87}', '\u{0}', '\u{0}']), ('\u{2c88}', ['\u{2c89}', '\u{0}', '\u{0}']), + ('\u{2c8a}', ['\u{2c8b}', '\u{0}', '\u{0}']), ('\u{2c8c}', ['\u{2c8d}', '\u{0}', '\u{0}']), + ('\u{2c8e}', ['\u{2c8f}', '\u{0}', '\u{0}']), ('\u{2c90}', ['\u{2c91}', '\u{0}', '\u{0}']), + ('\u{2c92}', ['\u{2c93}', '\u{0}', '\u{0}']), ('\u{2c94}', ['\u{2c95}', '\u{0}', '\u{0}']), + ('\u{2c96}', ['\u{2c97}', '\u{0}', '\u{0}']), ('\u{2c98}', ['\u{2c99}', '\u{0}', '\u{0}']), + ('\u{2c9a}', ['\u{2c9b}', '\u{0}', '\u{0}']), ('\u{2c9c}', ['\u{2c9d}', '\u{0}', '\u{0}']), + ('\u{2c9e}', ['\u{2c9f}', '\u{0}', '\u{0}']), ('\u{2ca0}', ['\u{2ca1}', '\u{0}', '\u{0}']), + ('\u{2ca2}', ['\u{2ca3}', '\u{0}', '\u{0}']), ('\u{2ca4}', ['\u{2ca5}', '\u{0}', '\u{0}']), + ('\u{2ca6}', ['\u{2ca7}', '\u{0}', '\u{0}']), ('\u{2ca8}', ['\u{2ca9}', '\u{0}', '\u{0}']), + ('\u{2caa}', ['\u{2cab}', '\u{0}', '\u{0}']), ('\u{2cac}', ['\u{2cad}', '\u{0}', '\u{0}']), + ('\u{2cae}', ['\u{2caf}', '\u{0}', '\u{0}']), ('\u{2cb0}', ['\u{2cb1}', '\u{0}', '\u{0}']), + ('\u{2cb2}', ['\u{2cb3}', '\u{0}', '\u{0}']), ('\u{2cb4}', ['\u{2cb5}', '\u{0}', '\u{0}']), + ('\u{2cb6}', ['\u{2cb7}', '\u{0}', '\u{0}']), ('\u{2cb8}', ['\u{2cb9}', '\u{0}', '\u{0}']), + ('\u{2cba}', ['\u{2cbb}', '\u{0}', '\u{0}']), ('\u{2cbc}', ['\u{2cbd}', '\u{0}', '\u{0}']), + ('\u{2cbe}', ['\u{2cbf}', '\u{0}', '\u{0}']), ('\u{2cc0}', ['\u{2cc1}', '\u{0}', '\u{0}']), + ('\u{2cc2}', ['\u{2cc3}', '\u{0}', '\u{0}']), ('\u{2cc4}', ['\u{2cc5}', '\u{0}', '\u{0}']), + ('\u{2cc6}', ['\u{2cc7}', '\u{0}', '\u{0}']), ('\u{2cc8}', ['\u{2cc9}', '\u{0}', '\u{0}']), + ('\u{2cca}', ['\u{2ccb}', '\u{0}', '\u{0}']), ('\u{2ccc}', ['\u{2ccd}', '\u{0}', '\u{0}']), + ('\u{2cce}', ['\u{2ccf}', '\u{0}', '\u{0}']), ('\u{2cd0}', ['\u{2cd1}', '\u{0}', '\u{0}']), + ('\u{2cd2}', ['\u{2cd3}', '\u{0}', '\u{0}']), ('\u{2cd4}', ['\u{2cd5}', '\u{0}', '\u{0}']), + ('\u{2cd6}', ['\u{2cd7}', '\u{0}', '\u{0}']), ('\u{2cd8}', ['\u{2cd9}', '\u{0}', '\u{0}']), + ('\u{2cda}', ['\u{2cdb}', '\u{0}', '\u{0}']), ('\u{2cdc}', ['\u{2cdd}', '\u{0}', '\u{0}']), + ('\u{2cde}', ['\u{2cdf}', '\u{0}', '\u{0}']), ('\u{2ce0}', ['\u{2ce1}', '\u{0}', '\u{0}']), + ('\u{2ce2}', ['\u{2ce3}', '\u{0}', '\u{0}']), ('\u{2ceb}', ['\u{2cec}', '\u{0}', '\u{0}']), + ('\u{2ced}', ['\u{2cee}', '\u{0}', '\u{0}']), ('\u{2cf2}', ['\u{2cf3}', '\u{0}', '\u{0}']), + ('\u{a640}', ['\u{a641}', '\u{0}', '\u{0}']), ('\u{a642}', ['\u{a643}', '\u{0}', '\u{0}']), + ('\u{a644}', ['\u{a645}', '\u{0}', '\u{0}']), ('\u{a646}', ['\u{a647}', '\u{0}', '\u{0}']), + ('\u{a648}', ['\u{a649}', '\u{0}', '\u{0}']), ('\u{a64a}', ['\u{a64b}', '\u{0}', '\u{0}']), + ('\u{a64c}', ['\u{a64d}', '\u{0}', '\u{0}']), ('\u{a64e}', ['\u{a64f}', '\u{0}', '\u{0}']), + ('\u{a650}', ['\u{a651}', '\u{0}', '\u{0}']), ('\u{a652}', ['\u{a653}', '\u{0}', '\u{0}']), + ('\u{a654}', ['\u{a655}', '\u{0}', '\u{0}']), ('\u{a656}', ['\u{a657}', '\u{0}', '\u{0}']), + ('\u{a658}', ['\u{a659}', '\u{0}', '\u{0}']), ('\u{a65a}', ['\u{a65b}', '\u{0}', '\u{0}']), + ('\u{a65c}', ['\u{a65d}', '\u{0}', '\u{0}']), ('\u{a65e}', ['\u{a65f}', '\u{0}', '\u{0}']), + ('\u{a660}', ['\u{a661}', '\u{0}', '\u{0}']), ('\u{a662}', ['\u{a663}', '\u{0}', '\u{0}']), + ('\u{a664}', ['\u{a665}', '\u{0}', '\u{0}']), ('\u{a666}', ['\u{a667}', '\u{0}', '\u{0}']), + ('\u{a668}', ['\u{a669}', '\u{0}', '\u{0}']), ('\u{a66a}', ['\u{a66b}', '\u{0}', '\u{0}']), + ('\u{a66c}', ['\u{a66d}', '\u{0}', '\u{0}']), ('\u{a680}', ['\u{a681}', '\u{0}', '\u{0}']), + ('\u{a682}', ['\u{a683}', '\u{0}', '\u{0}']), ('\u{a684}', ['\u{a685}', '\u{0}', '\u{0}']), + ('\u{a686}', ['\u{a687}', '\u{0}', '\u{0}']), ('\u{a688}', ['\u{a689}', '\u{0}', '\u{0}']), + ('\u{a68a}', ['\u{a68b}', '\u{0}', '\u{0}']), ('\u{a68c}', ['\u{a68d}', '\u{0}', '\u{0}']), + ('\u{a68e}', ['\u{a68f}', '\u{0}', '\u{0}']), ('\u{a690}', ['\u{a691}', '\u{0}', '\u{0}']), + ('\u{a692}', ['\u{a693}', '\u{0}', '\u{0}']), ('\u{a694}', ['\u{a695}', '\u{0}', '\u{0}']), + ('\u{a696}', ['\u{a697}', '\u{0}', '\u{0}']), ('\u{a698}', ['\u{a699}', '\u{0}', '\u{0}']), + ('\u{a69a}', ['\u{a69b}', '\u{0}', '\u{0}']), ('\u{a722}', ['\u{a723}', '\u{0}', '\u{0}']), + ('\u{a724}', ['\u{a725}', '\u{0}', '\u{0}']), ('\u{a726}', ['\u{a727}', '\u{0}', '\u{0}']), + ('\u{a728}', ['\u{a729}', '\u{0}', '\u{0}']), ('\u{a72a}', ['\u{a72b}', '\u{0}', '\u{0}']), + ('\u{a72c}', ['\u{a72d}', '\u{0}', '\u{0}']), ('\u{a72e}', ['\u{a72f}', '\u{0}', '\u{0}']), + ('\u{a732}', ['\u{a733}', '\u{0}', '\u{0}']), ('\u{a734}', ['\u{a735}', '\u{0}', '\u{0}']), + ('\u{a736}', ['\u{a737}', '\u{0}', '\u{0}']), ('\u{a738}', ['\u{a739}', '\u{0}', '\u{0}']), + ('\u{a73a}', ['\u{a73b}', '\u{0}', '\u{0}']), ('\u{a73c}', ['\u{a73d}', '\u{0}', '\u{0}']), + ('\u{a73e}', ['\u{a73f}', '\u{0}', '\u{0}']), ('\u{a740}', ['\u{a741}', '\u{0}', '\u{0}']), + ('\u{a742}', ['\u{a743}', '\u{0}', '\u{0}']), ('\u{a744}', ['\u{a745}', '\u{0}', '\u{0}']), + ('\u{a746}', ['\u{a747}', '\u{0}', '\u{0}']), ('\u{a748}', ['\u{a749}', '\u{0}', '\u{0}']), + ('\u{a74a}', ['\u{a74b}', '\u{0}', '\u{0}']), ('\u{a74c}', ['\u{a74d}', '\u{0}', '\u{0}']), + ('\u{a74e}', ['\u{a74f}', '\u{0}', '\u{0}']), ('\u{a750}', ['\u{a751}', '\u{0}', '\u{0}']), + ('\u{a752}', ['\u{a753}', '\u{0}', '\u{0}']), ('\u{a754}', ['\u{a755}', '\u{0}', '\u{0}']), + ('\u{a756}', ['\u{a757}', '\u{0}', '\u{0}']), ('\u{a758}', ['\u{a759}', '\u{0}', '\u{0}']), + ('\u{a75a}', ['\u{a75b}', '\u{0}', '\u{0}']), ('\u{a75c}', ['\u{a75d}', '\u{0}', '\u{0}']), + ('\u{a75e}', ['\u{a75f}', '\u{0}', '\u{0}']), ('\u{a760}', ['\u{a761}', '\u{0}', '\u{0}']), + ('\u{a762}', ['\u{a763}', '\u{0}', '\u{0}']), ('\u{a764}', ['\u{a765}', '\u{0}', '\u{0}']), + ('\u{a766}', ['\u{a767}', '\u{0}', '\u{0}']), ('\u{a768}', ['\u{a769}', '\u{0}', '\u{0}']), + ('\u{a76a}', ['\u{a76b}', '\u{0}', '\u{0}']), ('\u{a76c}', ['\u{a76d}', '\u{0}', '\u{0}']), + ('\u{a76e}', ['\u{a76f}', '\u{0}', '\u{0}']), ('\u{a779}', ['\u{a77a}', '\u{0}', '\u{0}']), + ('\u{a77b}', ['\u{a77c}', '\u{0}', '\u{0}']), ('\u{a77d}', ['\u{1d79}', '\u{0}', '\u{0}']), + ('\u{a77e}', ['\u{a77f}', '\u{0}', '\u{0}']), ('\u{a780}', ['\u{a781}', '\u{0}', '\u{0}']), + ('\u{a782}', ['\u{a783}', '\u{0}', '\u{0}']), ('\u{a784}', ['\u{a785}', '\u{0}', '\u{0}']), + ('\u{a786}', ['\u{a787}', '\u{0}', '\u{0}']), ('\u{a78b}', ['\u{a78c}', '\u{0}', '\u{0}']), + ('\u{a78d}', ['\u{265}', '\u{0}', '\u{0}']), ('\u{a790}', ['\u{a791}', '\u{0}', '\u{0}']), + ('\u{a792}', ['\u{a793}', '\u{0}', '\u{0}']), ('\u{a796}', ['\u{a797}', '\u{0}', '\u{0}']), + ('\u{a798}', ['\u{a799}', '\u{0}', '\u{0}']), ('\u{a79a}', ['\u{a79b}', '\u{0}', '\u{0}']), + ('\u{a79c}', ['\u{a79d}', '\u{0}', '\u{0}']), ('\u{a79e}', ['\u{a79f}', '\u{0}', '\u{0}']), + ('\u{a7a0}', ['\u{a7a1}', '\u{0}', '\u{0}']), ('\u{a7a2}', ['\u{a7a3}', '\u{0}', '\u{0}']), + ('\u{a7a4}', ['\u{a7a5}', '\u{0}', '\u{0}']), ('\u{a7a6}', ['\u{a7a7}', '\u{0}', '\u{0}']), + ('\u{a7a8}', ['\u{a7a9}', '\u{0}', '\u{0}']), ('\u{a7aa}', ['\u{266}', '\u{0}', '\u{0}']), + ('\u{a7ab}', ['\u{25c}', '\u{0}', '\u{0}']), ('\u{a7ac}', ['\u{261}', '\u{0}', '\u{0}']), + ('\u{a7ad}', ['\u{26c}', '\u{0}', '\u{0}']), ('\u{a7ae}', ['\u{26a}', '\u{0}', '\u{0}']), + ('\u{a7b0}', ['\u{29e}', '\u{0}', '\u{0}']), ('\u{a7b1}', ['\u{287}', '\u{0}', '\u{0}']), + ('\u{a7b2}', ['\u{29d}', '\u{0}', '\u{0}']), ('\u{a7b3}', ['\u{ab53}', '\u{0}', '\u{0}']), + ('\u{a7b4}', ['\u{a7b5}', '\u{0}', '\u{0}']), ('\u{a7b6}', ['\u{a7b7}', '\u{0}', '\u{0}']), + ('\u{a7b8}', ['\u{a7b9}', '\u{0}', '\u{0}']), ('\u{a7ba}', ['\u{a7bb}', '\u{0}', '\u{0}']), + ('\u{a7bc}', ['\u{a7bd}', '\u{0}', '\u{0}']), ('\u{a7be}', ['\u{a7bf}', '\u{0}', '\u{0}']), + ('\u{a7c0}', ['\u{a7c1}', '\u{0}', '\u{0}']), ('\u{a7c2}', ['\u{a7c3}', '\u{0}', '\u{0}']), ('\u{a7c4}', ['\u{a794}', '\u{0}', '\u{0}']), ('\u{a7c5}', ['\u{282}', '\u{0}', '\u{0}']), ('\u{a7c6}', ['\u{1d8e}', '\u{0}', '\u{0}']), ('\u{a7c7}', ['\u{a7c8}', '\u{0}', '\u{0}']), - ('\u{a7c9}', ['\u{a7ca}', '\u{0}', '\u{0}']), ('\u{a7f5}', ['\u{a7f6}', '\u{0}', '\u{0}']), - ('\u{ff21}', ['\u{ff41}', '\u{0}', '\u{0}']), ('\u{ff22}', ['\u{ff42}', '\u{0}', '\u{0}']), - ('\u{ff23}', ['\u{ff43}', '\u{0}', '\u{0}']), ('\u{ff24}', ['\u{ff44}', '\u{0}', '\u{0}']), - ('\u{ff25}', ['\u{ff45}', '\u{0}', '\u{0}']), ('\u{ff26}', ['\u{ff46}', '\u{0}', '\u{0}']), - ('\u{ff27}', ['\u{ff47}', '\u{0}', '\u{0}']), ('\u{ff28}', ['\u{ff48}', '\u{0}', '\u{0}']), - ('\u{ff29}', ['\u{ff49}', '\u{0}', '\u{0}']), ('\u{ff2a}', ['\u{ff4a}', '\u{0}', '\u{0}']), - ('\u{ff2b}', ['\u{ff4b}', '\u{0}', '\u{0}']), ('\u{ff2c}', ['\u{ff4c}', '\u{0}', '\u{0}']), - ('\u{ff2d}', ['\u{ff4d}', '\u{0}', '\u{0}']), ('\u{ff2e}', ['\u{ff4e}', '\u{0}', '\u{0}']), - ('\u{ff2f}', ['\u{ff4f}', '\u{0}', '\u{0}']), ('\u{ff30}', ['\u{ff50}', '\u{0}', '\u{0}']), - ('\u{ff31}', ['\u{ff51}', '\u{0}', '\u{0}']), ('\u{ff32}', ['\u{ff52}', '\u{0}', '\u{0}']), - ('\u{ff33}', ['\u{ff53}', '\u{0}', '\u{0}']), ('\u{ff34}', ['\u{ff54}', '\u{0}', '\u{0}']), - ('\u{ff35}', ['\u{ff55}', '\u{0}', '\u{0}']), ('\u{ff36}', ['\u{ff56}', '\u{0}', '\u{0}']), - ('\u{ff37}', ['\u{ff57}', '\u{0}', '\u{0}']), ('\u{ff38}', ['\u{ff58}', '\u{0}', '\u{0}']), - ('\u{ff39}', ['\u{ff59}', '\u{0}', '\u{0}']), ('\u{ff3a}', ['\u{ff5a}', '\u{0}', '\u{0}']), + ('\u{a7c9}', ['\u{a7ca}', '\u{0}', '\u{0}']), ('\u{a7d0}', ['\u{a7d1}', '\u{0}', '\u{0}']), + ('\u{a7d6}', ['\u{a7d7}', '\u{0}', '\u{0}']), ('\u{a7d8}', ['\u{a7d9}', '\u{0}', '\u{0}']), + ('\u{a7f5}', ['\u{a7f6}', '\u{0}', '\u{0}']), ('\u{ff21}', ['\u{ff41}', '\u{0}', '\u{0}']), + ('\u{ff22}', ['\u{ff42}', '\u{0}', '\u{0}']), ('\u{ff23}', ['\u{ff43}', '\u{0}', '\u{0}']), + ('\u{ff24}', ['\u{ff44}', '\u{0}', '\u{0}']), ('\u{ff25}', ['\u{ff45}', '\u{0}', '\u{0}']), + ('\u{ff26}', ['\u{ff46}', '\u{0}', '\u{0}']), ('\u{ff27}', ['\u{ff47}', '\u{0}', '\u{0}']), + ('\u{ff28}', ['\u{ff48}', '\u{0}', '\u{0}']), ('\u{ff29}', ['\u{ff49}', '\u{0}', '\u{0}']), + ('\u{ff2a}', ['\u{ff4a}', '\u{0}', '\u{0}']), ('\u{ff2b}', ['\u{ff4b}', '\u{0}', '\u{0}']), + ('\u{ff2c}', ['\u{ff4c}', '\u{0}', '\u{0}']), ('\u{ff2d}', ['\u{ff4d}', '\u{0}', '\u{0}']), + ('\u{ff2e}', ['\u{ff4e}', '\u{0}', '\u{0}']), ('\u{ff2f}', ['\u{ff4f}', '\u{0}', '\u{0}']), + ('\u{ff30}', ['\u{ff50}', '\u{0}', '\u{0}']), ('\u{ff31}', ['\u{ff51}', '\u{0}', '\u{0}']), + ('\u{ff32}', ['\u{ff52}', '\u{0}', '\u{0}']), ('\u{ff33}', ['\u{ff53}', '\u{0}', '\u{0}']), + ('\u{ff34}', ['\u{ff54}', '\u{0}', '\u{0}']), ('\u{ff35}', ['\u{ff55}', '\u{0}', '\u{0}']), + ('\u{ff36}', ['\u{ff56}', '\u{0}', '\u{0}']), ('\u{ff37}', ['\u{ff57}', '\u{0}', '\u{0}']), + ('\u{ff38}', ['\u{ff58}', '\u{0}', '\u{0}']), ('\u{ff39}', ['\u{ff59}', '\u{0}', '\u{0}']), + ('\u{ff3a}', ['\u{ff5a}', '\u{0}', '\u{0}']), ('\u{10400}', ['\u{10428}', '\u{0}', '\u{0}']), ('\u{10401}', ['\u{10429}', '\u{0}', '\u{0}']), ('\u{10402}', ['\u{1042a}', '\u{0}', '\u{0}']), @@ -1297,6 +1269,41 @@ pub mod conversions { ('\u{104d1}', ['\u{104f9}', '\u{0}', '\u{0}']), ('\u{104d2}', ['\u{104fa}', '\u{0}', '\u{0}']), ('\u{104d3}', ['\u{104fb}', '\u{0}', '\u{0}']), + ('\u{10570}', ['\u{10597}', '\u{0}', '\u{0}']), + ('\u{10571}', ['\u{10598}', '\u{0}', '\u{0}']), + ('\u{10572}', ['\u{10599}', '\u{0}', '\u{0}']), + ('\u{10573}', ['\u{1059a}', '\u{0}', '\u{0}']), + ('\u{10574}', ['\u{1059b}', '\u{0}', '\u{0}']), + ('\u{10575}', ['\u{1059c}', '\u{0}', '\u{0}']), + ('\u{10576}', ['\u{1059d}', '\u{0}', '\u{0}']), + ('\u{10577}', ['\u{1059e}', '\u{0}', '\u{0}']), + ('\u{10578}', ['\u{1059f}', '\u{0}', '\u{0}']), + ('\u{10579}', ['\u{105a0}', '\u{0}', '\u{0}']), + ('\u{1057a}', ['\u{105a1}', '\u{0}', '\u{0}']), + ('\u{1057c}', ['\u{105a3}', '\u{0}', '\u{0}']), + ('\u{1057d}', ['\u{105a4}', '\u{0}', '\u{0}']), + ('\u{1057e}', ['\u{105a5}', '\u{0}', '\u{0}']), + ('\u{1057f}', ['\u{105a6}', '\u{0}', '\u{0}']), + ('\u{10580}', ['\u{105a7}', '\u{0}', '\u{0}']), + ('\u{10581}', ['\u{105a8}', '\u{0}', '\u{0}']), + ('\u{10582}', ['\u{105a9}', '\u{0}', '\u{0}']), + ('\u{10583}', ['\u{105aa}', '\u{0}', '\u{0}']), + ('\u{10584}', ['\u{105ab}', '\u{0}', '\u{0}']), + ('\u{10585}', ['\u{105ac}', '\u{0}', '\u{0}']), + ('\u{10586}', ['\u{105ad}', '\u{0}', '\u{0}']), + ('\u{10587}', ['\u{105ae}', '\u{0}', '\u{0}']), + ('\u{10588}', ['\u{105af}', '\u{0}', '\u{0}']), + ('\u{10589}', ['\u{105b0}', '\u{0}', '\u{0}']), + ('\u{1058a}', ['\u{105b1}', '\u{0}', '\u{0}']), + ('\u{1058c}', ['\u{105b3}', '\u{0}', '\u{0}']), + ('\u{1058d}', ['\u{105b4}', '\u{0}', '\u{0}']), + ('\u{1058e}', ['\u{105b5}', '\u{0}', '\u{0}']), + ('\u{1058f}', ['\u{105b6}', '\u{0}', '\u{0}']), + ('\u{10590}', ['\u{105b7}', '\u{0}', '\u{0}']), + ('\u{10591}', ['\u{105b8}', '\u{0}', '\u{0}']), + ('\u{10592}', ['\u{105b9}', '\u{0}', '\u{0}']), + ('\u{10594}', ['\u{105bb}', '\u{0}', '\u{0}']), + ('\u{10595}', ['\u{105bc}', '\u{0}', '\u{0}']), ('\u{10c80}', ['\u{10cc0}', '\u{0}', '\u{0}']), ('\u{10c81}', ['\u{10cc1}', '\u{0}', '\u{0}']), ('\u{10c82}', ['\u{10cc2}', '\u{0}', '\u{0}']), @@ -1955,154 +1962,157 @@ pub mod conversions { ('\u{2c59}', ['\u{2c29}', '\u{0}', '\u{0}']), ('\u{2c5a}', ['\u{2c2a}', '\u{0}', '\u{0}']), ('\u{2c5b}', ['\u{2c2b}', '\u{0}', '\u{0}']), ('\u{2c5c}', ['\u{2c2c}', '\u{0}', '\u{0}']), ('\u{2c5d}', ['\u{2c2d}', '\u{0}', '\u{0}']), ('\u{2c5e}', ['\u{2c2e}', '\u{0}', '\u{0}']), - ('\u{2c61}', ['\u{2c60}', '\u{0}', '\u{0}']), ('\u{2c65}', ['\u{23a}', '\u{0}', '\u{0}']), - ('\u{2c66}', ['\u{23e}', '\u{0}', '\u{0}']), ('\u{2c68}', ['\u{2c67}', '\u{0}', '\u{0}']), - ('\u{2c6a}', ['\u{2c69}', '\u{0}', '\u{0}']), ('\u{2c6c}', ['\u{2c6b}', '\u{0}', '\u{0}']), - ('\u{2c73}', ['\u{2c72}', '\u{0}', '\u{0}']), ('\u{2c76}', ['\u{2c75}', '\u{0}', '\u{0}']), - ('\u{2c81}', ['\u{2c80}', '\u{0}', '\u{0}']), ('\u{2c83}', ['\u{2c82}', '\u{0}', '\u{0}']), - ('\u{2c85}', ['\u{2c84}', '\u{0}', '\u{0}']), ('\u{2c87}', ['\u{2c86}', '\u{0}', '\u{0}']), - ('\u{2c89}', ['\u{2c88}', '\u{0}', '\u{0}']), ('\u{2c8b}', ['\u{2c8a}', '\u{0}', '\u{0}']), - ('\u{2c8d}', ['\u{2c8c}', '\u{0}', '\u{0}']), ('\u{2c8f}', ['\u{2c8e}', '\u{0}', '\u{0}']), - ('\u{2c91}', ['\u{2c90}', '\u{0}', '\u{0}']), ('\u{2c93}', ['\u{2c92}', '\u{0}', '\u{0}']), - ('\u{2c95}', ['\u{2c94}', '\u{0}', '\u{0}']), ('\u{2c97}', ['\u{2c96}', '\u{0}', '\u{0}']), - ('\u{2c99}', ['\u{2c98}', '\u{0}', '\u{0}']), ('\u{2c9b}', ['\u{2c9a}', '\u{0}', '\u{0}']), - ('\u{2c9d}', ['\u{2c9c}', '\u{0}', '\u{0}']), ('\u{2c9f}', ['\u{2c9e}', '\u{0}', '\u{0}']), - ('\u{2ca1}', ['\u{2ca0}', '\u{0}', '\u{0}']), ('\u{2ca3}', ['\u{2ca2}', '\u{0}', '\u{0}']), - ('\u{2ca5}', ['\u{2ca4}', '\u{0}', '\u{0}']), ('\u{2ca7}', ['\u{2ca6}', '\u{0}', '\u{0}']), - ('\u{2ca9}', ['\u{2ca8}', '\u{0}', '\u{0}']), ('\u{2cab}', ['\u{2caa}', '\u{0}', '\u{0}']), - ('\u{2cad}', ['\u{2cac}', '\u{0}', '\u{0}']), ('\u{2caf}', ['\u{2cae}', '\u{0}', '\u{0}']), - ('\u{2cb1}', ['\u{2cb0}', '\u{0}', '\u{0}']), ('\u{2cb3}', ['\u{2cb2}', '\u{0}', '\u{0}']), - ('\u{2cb5}', ['\u{2cb4}', '\u{0}', '\u{0}']), ('\u{2cb7}', ['\u{2cb6}', '\u{0}', '\u{0}']), - ('\u{2cb9}', ['\u{2cb8}', '\u{0}', '\u{0}']), ('\u{2cbb}', ['\u{2cba}', '\u{0}', '\u{0}']), - ('\u{2cbd}', ['\u{2cbc}', '\u{0}', '\u{0}']), ('\u{2cbf}', ['\u{2cbe}', '\u{0}', '\u{0}']), - ('\u{2cc1}', ['\u{2cc0}', '\u{0}', '\u{0}']), ('\u{2cc3}', ['\u{2cc2}', '\u{0}', '\u{0}']), - ('\u{2cc5}', ['\u{2cc4}', '\u{0}', '\u{0}']), ('\u{2cc7}', ['\u{2cc6}', '\u{0}', '\u{0}']), - ('\u{2cc9}', ['\u{2cc8}', '\u{0}', '\u{0}']), ('\u{2ccb}', ['\u{2cca}', '\u{0}', '\u{0}']), - ('\u{2ccd}', ['\u{2ccc}', '\u{0}', '\u{0}']), ('\u{2ccf}', ['\u{2cce}', '\u{0}', '\u{0}']), - ('\u{2cd1}', ['\u{2cd0}', '\u{0}', '\u{0}']), ('\u{2cd3}', ['\u{2cd2}', '\u{0}', '\u{0}']), - ('\u{2cd5}', ['\u{2cd4}', '\u{0}', '\u{0}']), ('\u{2cd7}', ['\u{2cd6}', '\u{0}', '\u{0}']), - ('\u{2cd9}', ['\u{2cd8}', '\u{0}', '\u{0}']), ('\u{2cdb}', ['\u{2cda}', '\u{0}', '\u{0}']), - ('\u{2cdd}', ['\u{2cdc}', '\u{0}', '\u{0}']), ('\u{2cdf}', ['\u{2cde}', '\u{0}', '\u{0}']), - ('\u{2ce1}', ['\u{2ce0}', '\u{0}', '\u{0}']), ('\u{2ce3}', ['\u{2ce2}', '\u{0}', '\u{0}']), - ('\u{2cec}', ['\u{2ceb}', '\u{0}', '\u{0}']), ('\u{2cee}', ['\u{2ced}', '\u{0}', '\u{0}']), - ('\u{2cf3}', ['\u{2cf2}', '\u{0}', '\u{0}']), ('\u{2d00}', ['\u{10a0}', '\u{0}', '\u{0}']), - ('\u{2d01}', ['\u{10a1}', '\u{0}', '\u{0}']), ('\u{2d02}', ['\u{10a2}', '\u{0}', '\u{0}']), - ('\u{2d03}', ['\u{10a3}', '\u{0}', '\u{0}']), ('\u{2d04}', ['\u{10a4}', '\u{0}', '\u{0}']), - ('\u{2d05}', ['\u{10a5}', '\u{0}', '\u{0}']), ('\u{2d06}', ['\u{10a6}', '\u{0}', '\u{0}']), - ('\u{2d07}', ['\u{10a7}', '\u{0}', '\u{0}']), ('\u{2d08}', ['\u{10a8}', '\u{0}', '\u{0}']), - ('\u{2d09}', ['\u{10a9}', '\u{0}', '\u{0}']), ('\u{2d0a}', ['\u{10aa}', '\u{0}', '\u{0}']), - ('\u{2d0b}', ['\u{10ab}', '\u{0}', '\u{0}']), ('\u{2d0c}', ['\u{10ac}', '\u{0}', '\u{0}']), - ('\u{2d0d}', ['\u{10ad}', '\u{0}', '\u{0}']), ('\u{2d0e}', ['\u{10ae}', '\u{0}', '\u{0}']), - ('\u{2d0f}', ['\u{10af}', '\u{0}', '\u{0}']), ('\u{2d10}', ['\u{10b0}', '\u{0}', '\u{0}']), - ('\u{2d11}', ['\u{10b1}', '\u{0}', '\u{0}']), ('\u{2d12}', ['\u{10b2}', '\u{0}', '\u{0}']), - ('\u{2d13}', ['\u{10b3}', '\u{0}', '\u{0}']), ('\u{2d14}', ['\u{10b4}', '\u{0}', '\u{0}']), - ('\u{2d15}', ['\u{10b5}', '\u{0}', '\u{0}']), ('\u{2d16}', ['\u{10b6}', '\u{0}', '\u{0}']), - ('\u{2d17}', ['\u{10b7}', '\u{0}', '\u{0}']), ('\u{2d18}', ['\u{10b8}', '\u{0}', '\u{0}']), - ('\u{2d19}', ['\u{10b9}', '\u{0}', '\u{0}']), ('\u{2d1a}', ['\u{10ba}', '\u{0}', '\u{0}']), - ('\u{2d1b}', ['\u{10bb}', '\u{0}', '\u{0}']), ('\u{2d1c}', ['\u{10bc}', '\u{0}', '\u{0}']), - ('\u{2d1d}', ['\u{10bd}', '\u{0}', '\u{0}']), ('\u{2d1e}', ['\u{10be}', '\u{0}', '\u{0}']), - ('\u{2d1f}', ['\u{10bf}', '\u{0}', '\u{0}']), ('\u{2d20}', ['\u{10c0}', '\u{0}', '\u{0}']), - ('\u{2d21}', ['\u{10c1}', '\u{0}', '\u{0}']), ('\u{2d22}', ['\u{10c2}', '\u{0}', '\u{0}']), - ('\u{2d23}', ['\u{10c3}', '\u{0}', '\u{0}']), ('\u{2d24}', ['\u{10c4}', '\u{0}', '\u{0}']), - ('\u{2d25}', ['\u{10c5}', '\u{0}', '\u{0}']), ('\u{2d27}', ['\u{10c7}', '\u{0}', '\u{0}']), - ('\u{2d2d}', ['\u{10cd}', '\u{0}', '\u{0}']), ('\u{a641}', ['\u{a640}', '\u{0}', '\u{0}']), - ('\u{a643}', ['\u{a642}', '\u{0}', '\u{0}']), ('\u{a645}', ['\u{a644}', '\u{0}', '\u{0}']), - ('\u{a647}', ['\u{a646}', '\u{0}', '\u{0}']), ('\u{a649}', ['\u{a648}', '\u{0}', '\u{0}']), - ('\u{a64b}', ['\u{a64a}', '\u{0}', '\u{0}']), ('\u{a64d}', ['\u{a64c}', '\u{0}', '\u{0}']), - ('\u{a64f}', ['\u{a64e}', '\u{0}', '\u{0}']), ('\u{a651}', ['\u{a650}', '\u{0}', '\u{0}']), - ('\u{a653}', ['\u{a652}', '\u{0}', '\u{0}']), ('\u{a655}', ['\u{a654}', '\u{0}', '\u{0}']), - ('\u{a657}', ['\u{a656}', '\u{0}', '\u{0}']), ('\u{a659}', ['\u{a658}', '\u{0}', '\u{0}']), - ('\u{a65b}', ['\u{a65a}', '\u{0}', '\u{0}']), ('\u{a65d}', ['\u{a65c}', '\u{0}', '\u{0}']), - ('\u{a65f}', ['\u{a65e}', '\u{0}', '\u{0}']), ('\u{a661}', ['\u{a660}', '\u{0}', '\u{0}']), - ('\u{a663}', ['\u{a662}', '\u{0}', '\u{0}']), ('\u{a665}', ['\u{a664}', '\u{0}', '\u{0}']), - ('\u{a667}', ['\u{a666}', '\u{0}', '\u{0}']), ('\u{a669}', ['\u{a668}', '\u{0}', '\u{0}']), - ('\u{a66b}', ['\u{a66a}', '\u{0}', '\u{0}']), ('\u{a66d}', ['\u{a66c}', '\u{0}', '\u{0}']), - ('\u{a681}', ['\u{a680}', '\u{0}', '\u{0}']), ('\u{a683}', ['\u{a682}', '\u{0}', '\u{0}']), - ('\u{a685}', ['\u{a684}', '\u{0}', '\u{0}']), ('\u{a687}', ['\u{a686}', '\u{0}', '\u{0}']), - ('\u{a689}', ['\u{a688}', '\u{0}', '\u{0}']), ('\u{a68b}', ['\u{a68a}', '\u{0}', '\u{0}']), - ('\u{a68d}', ['\u{a68c}', '\u{0}', '\u{0}']), ('\u{a68f}', ['\u{a68e}', '\u{0}', '\u{0}']), - ('\u{a691}', ['\u{a690}', '\u{0}', '\u{0}']), ('\u{a693}', ['\u{a692}', '\u{0}', '\u{0}']), - ('\u{a695}', ['\u{a694}', '\u{0}', '\u{0}']), ('\u{a697}', ['\u{a696}', '\u{0}', '\u{0}']), - ('\u{a699}', ['\u{a698}', '\u{0}', '\u{0}']), ('\u{a69b}', ['\u{a69a}', '\u{0}', '\u{0}']), - ('\u{a723}', ['\u{a722}', '\u{0}', '\u{0}']), ('\u{a725}', ['\u{a724}', '\u{0}', '\u{0}']), - ('\u{a727}', ['\u{a726}', '\u{0}', '\u{0}']), ('\u{a729}', ['\u{a728}', '\u{0}', '\u{0}']), - ('\u{a72b}', ['\u{a72a}', '\u{0}', '\u{0}']), ('\u{a72d}', ['\u{a72c}', '\u{0}', '\u{0}']), - ('\u{a72f}', ['\u{a72e}', '\u{0}', '\u{0}']), ('\u{a733}', ['\u{a732}', '\u{0}', '\u{0}']), - ('\u{a735}', ['\u{a734}', '\u{0}', '\u{0}']), ('\u{a737}', ['\u{a736}', '\u{0}', '\u{0}']), - ('\u{a739}', ['\u{a738}', '\u{0}', '\u{0}']), ('\u{a73b}', ['\u{a73a}', '\u{0}', '\u{0}']), - ('\u{a73d}', ['\u{a73c}', '\u{0}', '\u{0}']), ('\u{a73f}', ['\u{a73e}', '\u{0}', '\u{0}']), - ('\u{a741}', ['\u{a740}', '\u{0}', '\u{0}']), ('\u{a743}', ['\u{a742}', '\u{0}', '\u{0}']), - ('\u{a745}', ['\u{a744}', '\u{0}', '\u{0}']), ('\u{a747}', ['\u{a746}', '\u{0}', '\u{0}']), - ('\u{a749}', ['\u{a748}', '\u{0}', '\u{0}']), ('\u{a74b}', ['\u{a74a}', '\u{0}', '\u{0}']), - ('\u{a74d}', ['\u{a74c}', '\u{0}', '\u{0}']), ('\u{a74f}', ['\u{a74e}', '\u{0}', '\u{0}']), - ('\u{a751}', ['\u{a750}', '\u{0}', '\u{0}']), ('\u{a753}', ['\u{a752}', '\u{0}', '\u{0}']), - ('\u{a755}', ['\u{a754}', '\u{0}', '\u{0}']), ('\u{a757}', ['\u{a756}', '\u{0}', '\u{0}']), - ('\u{a759}', ['\u{a758}', '\u{0}', '\u{0}']), ('\u{a75b}', ['\u{a75a}', '\u{0}', '\u{0}']), - ('\u{a75d}', ['\u{a75c}', '\u{0}', '\u{0}']), ('\u{a75f}', ['\u{a75e}', '\u{0}', '\u{0}']), - ('\u{a761}', ['\u{a760}', '\u{0}', '\u{0}']), ('\u{a763}', ['\u{a762}', '\u{0}', '\u{0}']), - ('\u{a765}', ['\u{a764}', '\u{0}', '\u{0}']), ('\u{a767}', ['\u{a766}', '\u{0}', '\u{0}']), - ('\u{a769}', ['\u{a768}', '\u{0}', '\u{0}']), ('\u{a76b}', ['\u{a76a}', '\u{0}', '\u{0}']), - ('\u{a76d}', ['\u{a76c}', '\u{0}', '\u{0}']), ('\u{a76f}', ['\u{a76e}', '\u{0}', '\u{0}']), - ('\u{a77a}', ['\u{a779}', '\u{0}', '\u{0}']), ('\u{a77c}', ['\u{a77b}', '\u{0}', '\u{0}']), - ('\u{a77f}', ['\u{a77e}', '\u{0}', '\u{0}']), ('\u{a781}', ['\u{a780}', '\u{0}', '\u{0}']), - ('\u{a783}', ['\u{a782}', '\u{0}', '\u{0}']), ('\u{a785}', ['\u{a784}', '\u{0}', '\u{0}']), - ('\u{a787}', ['\u{a786}', '\u{0}', '\u{0}']), ('\u{a78c}', ['\u{a78b}', '\u{0}', '\u{0}']), - ('\u{a791}', ['\u{a790}', '\u{0}', '\u{0}']), ('\u{a793}', ['\u{a792}', '\u{0}', '\u{0}']), - ('\u{a794}', ['\u{a7c4}', '\u{0}', '\u{0}']), ('\u{a797}', ['\u{a796}', '\u{0}', '\u{0}']), - ('\u{a799}', ['\u{a798}', '\u{0}', '\u{0}']), ('\u{a79b}', ['\u{a79a}', '\u{0}', '\u{0}']), - ('\u{a79d}', ['\u{a79c}', '\u{0}', '\u{0}']), ('\u{a79f}', ['\u{a79e}', '\u{0}', '\u{0}']), - ('\u{a7a1}', ['\u{a7a0}', '\u{0}', '\u{0}']), ('\u{a7a3}', ['\u{a7a2}', '\u{0}', '\u{0}']), - ('\u{a7a5}', ['\u{a7a4}', '\u{0}', '\u{0}']), ('\u{a7a7}', ['\u{a7a6}', '\u{0}', '\u{0}']), - ('\u{a7a9}', ['\u{a7a8}', '\u{0}', '\u{0}']), ('\u{a7b5}', ['\u{a7b4}', '\u{0}', '\u{0}']), - ('\u{a7b7}', ['\u{a7b6}', '\u{0}', '\u{0}']), ('\u{a7b9}', ['\u{a7b8}', '\u{0}', '\u{0}']), - ('\u{a7bb}', ['\u{a7ba}', '\u{0}', '\u{0}']), ('\u{a7bd}', ['\u{a7bc}', '\u{0}', '\u{0}']), - ('\u{a7bf}', ['\u{a7be}', '\u{0}', '\u{0}']), ('\u{a7c3}', ['\u{a7c2}', '\u{0}', '\u{0}']), + ('\u{2c5f}', ['\u{2c2f}', '\u{0}', '\u{0}']), ('\u{2c61}', ['\u{2c60}', '\u{0}', '\u{0}']), + ('\u{2c65}', ['\u{23a}', '\u{0}', '\u{0}']), ('\u{2c66}', ['\u{23e}', '\u{0}', '\u{0}']), + ('\u{2c68}', ['\u{2c67}', '\u{0}', '\u{0}']), ('\u{2c6a}', ['\u{2c69}', '\u{0}', '\u{0}']), + ('\u{2c6c}', ['\u{2c6b}', '\u{0}', '\u{0}']), ('\u{2c73}', ['\u{2c72}', '\u{0}', '\u{0}']), + ('\u{2c76}', ['\u{2c75}', '\u{0}', '\u{0}']), ('\u{2c81}', ['\u{2c80}', '\u{0}', '\u{0}']), + ('\u{2c83}', ['\u{2c82}', '\u{0}', '\u{0}']), ('\u{2c85}', ['\u{2c84}', '\u{0}', '\u{0}']), + ('\u{2c87}', ['\u{2c86}', '\u{0}', '\u{0}']), ('\u{2c89}', ['\u{2c88}', '\u{0}', '\u{0}']), + ('\u{2c8b}', ['\u{2c8a}', '\u{0}', '\u{0}']), ('\u{2c8d}', ['\u{2c8c}', '\u{0}', '\u{0}']), + ('\u{2c8f}', ['\u{2c8e}', '\u{0}', '\u{0}']), ('\u{2c91}', ['\u{2c90}', '\u{0}', '\u{0}']), + ('\u{2c93}', ['\u{2c92}', '\u{0}', '\u{0}']), ('\u{2c95}', ['\u{2c94}', '\u{0}', '\u{0}']), + ('\u{2c97}', ['\u{2c96}', '\u{0}', '\u{0}']), ('\u{2c99}', ['\u{2c98}', '\u{0}', '\u{0}']), + ('\u{2c9b}', ['\u{2c9a}', '\u{0}', '\u{0}']), ('\u{2c9d}', ['\u{2c9c}', '\u{0}', '\u{0}']), + ('\u{2c9f}', ['\u{2c9e}', '\u{0}', '\u{0}']), ('\u{2ca1}', ['\u{2ca0}', '\u{0}', '\u{0}']), + ('\u{2ca3}', ['\u{2ca2}', '\u{0}', '\u{0}']), ('\u{2ca5}', ['\u{2ca4}', '\u{0}', '\u{0}']), + ('\u{2ca7}', ['\u{2ca6}', '\u{0}', '\u{0}']), ('\u{2ca9}', ['\u{2ca8}', '\u{0}', '\u{0}']), + ('\u{2cab}', ['\u{2caa}', '\u{0}', '\u{0}']), ('\u{2cad}', ['\u{2cac}', '\u{0}', '\u{0}']), + ('\u{2caf}', ['\u{2cae}', '\u{0}', '\u{0}']), ('\u{2cb1}', ['\u{2cb0}', '\u{0}', '\u{0}']), + ('\u{2cb3}', ['\u{2cb2}', '\u{0}', '\u{0}']), ('\u{2cb5}', ['\u{2cb4}', '\u{0}', '\u{0}']), + ('\u{2cb7}', ['\u{2cb6}', '\u{0}', '\u{0}']), ('\u{2cb9}', ['\u{2cb8}', '\u{0}', '\u{0}']), + ('\u{2cbb}', ['\u{2cba}', '\u{0}', '\u{0}']), ('\u{2cbd}', ['\u{2cbc}', '\u{0}', '\u{0}']), + ('\u{2cbf}', ['\u{2cbe}', '\u{0}', '\u{0}']), ('\u{2cc1}', ['\u{2cc0}', '\u{0}', '\u{0}']), + ('\u{2cc3}', ['\u{2cc2}', '\u{0}', '\u{0}']), ('\u{2cc5}', ['\u{2cc4}', '\u{0}', '\u{0}']), + ('\u{2cc7}', ['\u{2cc6}', '\u{0}', '\u{0}']), ('\u{2cc9}', ['\u{2cc8}', '\u{0}', '\u{0}']), + ('\u{2ccb}', ['\u{2cca}', '\u{0}', '\u{0}']), ('\u{2ccd}', ['\u{2ccc}', '\u{0}', '\u{0}']), + ('\u{2ccf}', ['\u{2cce}', '\u{0}', '\u{0}']), ('\u{2cd1}', ['\u{2cd0}', '\u{0}', '\u{0}']), + ('\u{2cd3}', ['\u{2cd2}', '\u{0}', '\u{0}']), ('\u{2cd5}', ['\u{2cd4}', '\u{0}', '\u{0}']), + ('\u{2cd7}', ['\u{2cd6}', '\u{0}', '\u{0}']), ('\u{2cd9}', ['\u{2cd8}', '\u{0}', '\u{0}']), + ('\u{2cdb}', ['\u{2cda}', '\u{0}', '\u{0}']), ('\u{2cdd}', ['\u{2cdc}', '\u{0}', '\u{0}']), + ('\u{2cdf}', ['\u{2cde}', '\u{0}', '\u{0}']), ('\u{2ce1}', ['\u{2ce0}', '\u{0}', '\u{0}']), + ('\u{2ce3}', ['\u{2ce2}', '\u{0}', '\u{0}']), ('\u{2cec}', ['\u{2ceb}', '\u{0}', '\u{0}']), + ('\u{2cee}', ['\u{2ced}', '\u{0}', '\u{0}']), ('\u{2cf3}', ['\u{2cf2}', '\u{0}', '\u{0}']), + ('\u{2d00}', ['\u{10a0}', '\u{0}', '\u{0}']), ('\u{2d01}', ['\u{10a1}', '\u{0}', '\u{0}']), + ('\u{2d02}', ['\u{10a2}', '\u{0}', '\u{0}']), ('\u{2d03}', ['\u{10a3}', '\u{0}', '\u{0}']), + ('\u{2d04}', ['\u{10a4}', '\u{0}', '\u{0}']), ('\u{2d05}', ['\u{10a5}', '\u{0}', '\u{0}']), + ('\u{2d06}', ['\u{10a6}', '\u{0}', '\u{0}']), ('\u{2d07}', ['\u{10a7}', '\u{0}', '\u{0}']), + ('\u{2d08}', ['\u{10a8}', '\u{0}', '\u{0}']), ('\u{2d09}', ['\u{10a9}', '\u{0}', '\u{0}']), + ('\u{2d0a}', ['\u{10aa}', '\u{0}', '\u{0}']), ('\u{2d0b}', ['\u{10ab}', '\u{0}', '\u{0}']), + ('\u{2d0c}', ['\u{10ac}', '\u{0}', '\u{0}']), ('\u{2d0d}', ['\u{10ad}', '\u{0}', '\u{0}']), + ('\u{2d0e}', ['\u{10ae}', '\u{0}', '\u{0}']), ('\u{2d0f}', ['\u{10af}', '\u{0}', '\u{0}']), + ('\u{2d10}', ['\u{10b0}', '\u{0}', '\u{0}']), ('\u{2d11}', ['\u{10b1}', '\u{0}', '\u{0}']), + ('\u{2d12}', ['\u{10b2}', '\u{0}', '\u{0}']), ('\u{2d13}', ['\u{10b3}', '\u{0}', '\u{0}']), + ('\u{2d14}', ['\u{10b4}', '\u{0}', '\u{0}']), ('\u{2d15}', ['\u{10b5}', '\u{0}', '\u{0}']), + ('\u{2d16}', ['\u{10b6}', '\u{0}', '\u{0}']), ('\u{2d17}', ['\u{10b7}', '\u{0}', '\u{0}']), + ('\u{2d18}', ['\u{10b8}', '\u{0}', '\u{0}']), ('\u{2d19}', ['\u{10b9}', '\u{0}', '\u{0}']), + ('\u{2d1a}', ['\u{10ba}', '\u{0}', '\u{0}']), ('\u{2d1b}', ['\u{10bb}', '\u{0}', '\u{0}']), + ('\u{2d1c}', ['\u{10bc}', '\u{0}', '\u{0}']), ('\u{2d1d}', ['\u{10bd}', '\u{0}', '\u{0}']), + ('\u{2d1e}', ['\u{10be}', '\u{0}', '\u{0}']), ('\u{2d1f}', ['\u{10bf}', '\u{0}', '\u{0}']), + ('\u{2d20}', ['\u{10c0}', '\u{0}', '\u{0}']), ('\u{2d21}', ['\u{10c1}', '\u{0}', '\u{0}']), + ('\u{2d22}', ['\u{10c2}', '\u{0}', '\u{0}']), ('\u{2d23}', ['\u{10c3}', '\u{0}', '\u{0}']), + ('\u{2d24}', ['\u{10c4}', '\u{0}', '\u{0}']), ('\u{2d25}', ['\u{10c5}', '\u{0}', '\u{0}']), + ('\u{2d27}', ['\u{10c7}', '\u{0}', '\u{0}']), ('\u{2d2d}', ['\u{10cd}', '\u{0}', '\u{0}']), + ('\u{a641}', ['\u{a640}', '\u{0}', '\u{0}']), ('\u{a643}', ['\u{a642}', '\u{0}', '\u{0}']), + ('\u{a645}', ['\u{a644}', '\u{0}', '\u{0}']), ('\u{a647}', ['\u{a646}', '\u{0}', '\u{0}']), + ('\u{a649}', ['\u{a648}', '\u{0}', '\u{0}']), ('\u{a64b}', ['\u{a64a}', '\u{0}', '\u{0}']), + ('\u{a64d}', ['\u{a64c}', '\u{0}', '\u{0}']), ('\u{a64f}', ['\u{a64e}', '\u{0}', '\u{0}']), + ('\u{a651}', ['\u{a650}', '\u{0}', '\u{0}']), ('\u{a653}', ['\u{a652}', '\u{0}', '\u{0}']), + ('\u{a655}', ['\u{a654}', '\u{0}', '\u{0}']), ('\u{a657}', ['\u{a656}', '\u{0}', '\u{0}']), + ('\u{a659}', ['\u{a658}', '\u{0}', '\u{0}']), ('\u{a65b}', ['\u{a65a}', '\u{0}', '\u{0}']), + ('\u{a65d}', ['\u{a65c}', '\u{0}', '\u{0}']), ('\u{a65f}', ['\u{a65e}', '\u{0}', '\u{0}']), + ('\u{a661}', ['\u{a660}', '\u{0}', '\u{0}']), ('\u{a663}', ['\u{a662}', '\u{0}', '\u{0}']), + ('\u{a665}', ['\u{a664}', '\u{0}', '\u{0}']), ('\u{a667}', ['\u{a666}', '\u{0}', '\u{0}']), + ('\u{a669}', ['\u{a668}', '\u{0}', '\u{0}']), ('\u{a66b}', ['\u{a66a}', '\u{0}', '\u{0}']), + ('\u{a66d}', ['\u{a66c}', '\u{0}', '\u{0}']), ('\u{a681}', ['\u{a680}', '\u{0}', '\u{0}']), + ('\u{a683}', ['\u{a682}', '\u{0}', '\u{0}']), ('\u{a685}', ['\u{a684}', '\u{0}', '\u{0}']), + ('\u{a687}', ['\u{a686}', '\u{0}', '\u{0}']), ('\u{a689}', ['\u{a688}', '\u{0}', '\u{0}']), + ('\u{a68b}', ['\u{a68a}', '\u{0}', '\u{0}']), ('\u{a68d}', ['\u{a68c}', '\u{0}', '\u{0}']), + ('\u{a68f}', ['\u{a68e}', '\u{0}', '\u{0}']), ('\u{a691}', ['\u{a690}', '\u{0}', '\u{0}']), + ('\u{a693}', ['\u{a692}', '\u{0}', '\u{0}']), ('\u{a695}', ['\u{a694}', '\u{0}', '\u{0}']), + ('\u{a697}', ['\u{a696}', '\u{0}', '\u{0}']), ('\u{a699}', ['\u{a698}', '\u{0}', '\u{0}']), + ('\u{a69b}', ['\u{a69a}', '\u{0}', '\u{0}']), ('\u{a723}', ['\u{a722}', '\u{0}', '\u{0}']), + ('\u{a725}', ['\u{a724}', '\u{0}', '\u{0}']), ('\u{a727}', ['\u{a726}', '\u{0}', '\u{0}']), + ('\u{a729}', ['\u{a728}', '\u{0}', '\u{0}']), ('\u{a72b}', ['\u{a72a}', '\u{0}', '\u{0}']), + ('\u{a72d}', ['\u{a72c}', '\u{0}', '\u{0}']), ('\u{a72f}', ['\u{a72e}', '\u{0}', '\u{0}']), + ('\u{a733}', ['\u{a732}', '\u{0}', '\u{0}']), ('\u{a735}', ['\u{a734}', '\u{0}', '\u{0}']), + ('\u{a737}', ['\u{a736}', '\u{0}', '\u{0}']), ('\u{a739}', ['\u{a738}', '\u{0}', '\u{0}']), + ('\u{a73b}', ['\u{a73a}', '\u{0}', '\u{0}']), ('\u{a73d}', ['\u{a73c}', '\u{0}', '\u{0}']), + ('\u{a73f}', ['\u{a73e}', '\u{0}', '\u{0}']), ('\u{a741}', ['\u{a740}', '\u{0}', '\u{0}']), + ('\u{a743}', ['\u{a742}', '\u{0}', '\u{0}']), ('\u{a745}', ['\u{a744}', '\u{0}', '\u{0}']), + ('\u{a747}', ['\u{a746}', '\u{0}', '\u{0}']), ('\u{a749}', ['\u{a748}', '\u{0}', '\u{0}']), + ('\u{a74b}', ['\u{a74a}', '\u{0}', '\u{0}']), ('\u{a74d}', ['\u{a74c}', '\u{0}', '\u{0}']), + ('\u{a74f}', ['\u{a74e}', '\u{0}', '\u{0}']), ('\u{a751}', ['\u{a750}', '\u{0}', '\u{0}']), + ('\u{a753}', ['\u{a752}', '\u{0}', '\u{0}']), ('\u{a755}', ['\u{a754}', '\u{0}', '\u{0}']), + ('\u{a757}', ['\u{a756}', '\u{0}', '\u{0}']), ('\u{a759}', ['\u{a758}', '\u{0}', '\u{0}']), + ('\u{a75b}', ['\u{a75a}', '\u{0}', '\u{0}']), ('\u{a75d}', ['\u{a75c}', '\u{0}', '\u{0}']), + ('\u{a75f}', ['\u{a75e}', '\u{0}', '\u{0}']), ('\u{a761}', ['\u{a760}', '\u{0}', '\u{0}']), + ('\u{a763}', ['\u{a762}', '\u{0}', '\u{0}']), ('\u{a765}', ['\u{a764}', '\u{0}', '\u{0}']), + ('\u{a767}', ['\u{a766}', '\u{0}', '\u{0}']), ('\u{a769}', ['\u{a768}', '\u{0}', '\u{0}']), + ('\u{a76b}', ['\u{a76a}', '\u{0}', '\u{0}']), ('\u{a76d}', ['\u{a76c}', '\u{0}', '\u{0}']), + ('\u{a76f}', ['\u{a76e}', '\u{0}', '\u{0}']), ('\u{a77a}', ['\u{a779}', '\u{0}', '\u{0}']), + ('\u{a77c}', ['\u{a77b}', '\u{0}', '\u{0}']), ('\u{a77f}', ['\u{a77e}', '\u{0}', '\u{0}']), + ('\u{a781}', ['\u{a780}', '\u{0}', '\u{0}']), ('\u{a783}', ['\u{a782}', '\u{0}', '\u{0}']), + ('\u{a785}', ['\u{a784}', '\u{0}', '\u{0}']), ('\u{a787}', ['\u{a786}', '\u{0}', '\u{0}']), + ('\u{a78c}', ['\u{a78b}', '\u{0}', '\u{0}']), ('\u{a791}', ['\u{a790}', '\u{0}', '\u{0}']), + ('\u{a793}', ['\u{a792}', '\u{0}', '\u{0}']), ('\u{a794}', ['\u{a7c4}', '\u{0}', '\u{0}']), + ('\u{a797}', ['\u{a796}', '\u{0}', '\u{0}']), ('\u{a799}', ['\u{a798}', '\u{0}', '\u{0}']), + ('\u{a79b}', ['\u{a79a}', '\u{0}', '\u{0}']), ('\u{a79d}', ['\u{a79c}', '\u{0}', '\u{0}']), + ('\u{a79f}', ['\u{a79e}', '\u{0}', '\u{0}']), ('\u{a7a1}', ['\u{a7a0}', '\u{0}', '\u{0}']), + ('\u{a7a3}', ['\u{a7a2}', '\u{0}', '\u{0}']), ('\u{a7a5}', ['\u{a7a4}', '\u{0}', '\u{0}']), + ('\u{a7a7}', ['\u{a7a6}', '\u{0}', '\u{0}']), ('\u{a7a9}', ['\u{a7a8}', '\u{0}', '\u{0}']), + ('\u{a7b5}', ['\u{a7b4}', '\u{0}', '\u{0}']), ('\u{a7b7}', ['\u{a7b6}', '\u{0}', '\u{0}']), + ('\u{a7b9}', ['\u{a7b8}', '\u{0}', '\u{0}']), ('\u{a7bb}', ['\u{a7ba}', '\u{0}', '\u{0}']), + ('\u{a7bd}', ['\u{a7bc}', '\u{0}', '\u{0}']), ('\u{a7bf}', ['\u{a7be}', '\u{0}', '\u{0}']), + ('\u{a7c1}', ['\u{a7c0}', '\u{0}', '\u{0}']), ('\u{a7c3}', ['\u{a7c2}', '\u{0}', '\u{0}']), ('\u{a7c8}', ['\u{a7c7}', '\u{0}', '\u{0}']), ('\u{a7ca}', ['\u{a7c9}', '\u{0}', '\u{0}']), - ('\u{a7f6}', ['\u{a7f5}', '\u{0}', '\u{0}']), ('\u{ab53}', ['\u{a7b3}', '\u{0}', '\u{0}']), - ('\u{ab70}', ['\u{13a0}', '\u{0}', '\u{0}']), ('\u{ab71}', ['\u{13a1}', '\u{0}', '\u{0}']), - ('\u{ab72}', ['\u{13a2}', '\u{0}', '\u{0}']), ('\u{ab73}', ['\u{13a3}', '\u{0}', '\u{0}']), - ('\u{ab74}', ['\u{13a4}', '\u{0}', '\u{0}']), ('\u{ab75}', ['\u{13a5}', '\u{0}', '\u{0}']), - ('\u{ab76}', ['\u{13a6}', '\u{0}', '\u{0}']), ('\u{ab77}', ['\u{13a7}', '\u{0}', '\u{0}']), - ('\u{ab78}', ['\u{13a8}', '\u{0}', '\u{0}']), ('\u{ab79}', ['\u{13a9}', '\u{0}', '\u{0}']), - ('\u{ab7a}', ['\u{13aa}', '\u{0}', '\u{0}']), ('\u{ab7b}', ['\u{13ab}', '\u{0}', '\u{0}']), - ('\u{ab7c}', ['\u{13ac}', '\u{0}', '\u{0}']), ('\u{ab7d}', ['\u{13ad}', '\u{0}', '\u{0}']), - ('\u{ab7e}', ['\u{13ae}', '\u{0}', '\u{0}']), ('\u{ab7f}', ['\u{13af}', '\u{0}', '\u{0}']), - ('\u{ab80}', ['\u{13b0}', '\u{0}', '\u{0}']), ('\u{ab81}', ['\u{13b1}', '\u{0}', '\u{0}']), - ('\u{ab82}', ['\u{13b2}', '\u{0}', '\u{0}']), ('\u{ab83}', ['\u{13b3}', '\u{0}', '\u{0}']), - ('\u{ab84}', ['\u{13b4}', '\u{0}', '\u{0}']), ('\u{ab85}', ['\u{13b5}', '\u{0}', '\u{0}']), - ('\u{ab86}', ['\u{13b6}', '\u{0}', '\u{0}']), ('\u{ab87}', ['\u{13b7}', '\u{0}', '\u{0}']), - ('\u{ab88}', ['\u{13b8}', '\u{0}', '\u{0}']), ('\u{ab89}', ['\u{13b9}', '\u{0}', '\u{0}']), - ('\u{ab8a}', ['\u{13ba}', '\u{0}', '\u{0}']), ('\u{ab8b}', ['\u{13bb}', '\u{0}', '\u{0}']), - ('\u{ab8c}', ['\u{13bc}', '\u{0}', '\u{0}']), ('\u{ab8d}', ['\u{13bd}', '\u{0}', '\u{0}']), - ('\u{ab8e}', ['\u{13be}', '\u{0}', '\u{0}']), ('\u{ab8f}', ['\u{13bf}', '\u{0}', '\u{0}']), - ('\u{ab90}', ['\u{13c0}', '\u{0}', '\u{0}']), ('\u{ab91}', ['\u{13c1}', '\u{0}', '\u{0}']), - ('\u{ab92}', ['\u{13c2}', '\u{0}', '\u{0}']), ('\u{ab93}', ['\u{13c3}', '\u{0}', '\u{0}']), - ('\u{ab94}', ['\u{13c4}', '\u{0}', '\u{0}']), ('\u{ab95}', ['\u{13c5}', '\u{0}', '\u{0}']), - ('\u{ab96}', ['\u{13c6}', '\u{0}', '\u{0}']), ('\u{ab97}', ['\u{13c7}', '\u{0}', '\u{0}']), - ('\u{ab98}', ['\u{13c8}', '\u{0}', '\u{0}']), ('\u{ab99}', ['\u{13c9}', '\u{0}', '\u{0}']), - ('\u{ab9a}', ['\u{13ca}', '\u{0}', '\u{0}']), ('\u{ab9b}', ['\u{13cb}', '\u{0}', '\u{0}']), - ('\u{ab9c}', ['\u{13cc}', '\u{0}', '\u{0}']), ('\u{ab9d}', ['\u{13cd}', '\u{0}', '\u{0}']), - ('\u{ab9e}', ['\u{13ce}', '\u{0}', '\u{0}']), ('\u{ab9f}', ['\u{13cf}', '\u{0}', '\u{0}']), - ('\u{aba0}', ['\u{13d0}', '\u{0}', '\u{0}']), ('\u{aba1}', ['\u{13d1}', '\u{0}', '\u{0}']), - ('\u{aba2}', ['\u{13d2}', '\u{0}', '\u{0}']), ('\u{aba3}', ['\u{13d3}', '\u{0}', '\u{0}']), - ('\u{aba4}', ['\u{13d4}', '\u{0}', '\u{0}']), ('\u{aba5}', ['\u{13d5}', '\u{0}', '\u{0}']), - ('\u{aba6}', ['\u{13d6}', '\u{0}', '\u{0}']), ('\u{aba7}', ['\u{13d7}', '\u{0}', '\u{0}']), - ('\u{aba8}', ['\u{13d8}', '\u{0}', '\u{0}']), ('\u{aba9}', ['\u{13d9}', '\u{0}', '\u{0}']), - ('\u{abaa}', ['\u{13da}', '\u{0}', '\u{0}']), ('\u{abab}', ['\u{13db}', '\u{0}', '\u{0}']), - ('\u{abac}', ['\u{13dc}', '\u{0}', '\u{0}']), ('\u{abad}', ['\u{13dd}', '\u{0}', '\u{0}']), - ('\u{abae}', ['\u{13de}', '\u{0}', '\u{0}']), ('\u{abaf}', ['\u{13df}', '\u{0}', '\u{0}']), - ('\u{abb0}', ['\u{13e0}', '\u{0}', '\u{0}']), ('\u{abb1}', ['\u{13e1}', '\u{0}', '\u{0}']), - ('\u{abb2}', ['\u{13e2}', '\u{0}', '\u{0}']), ('\u{abb3}', ['\u{13e3}', '\u{0}', '\u{0}']), - ('\u{abb4}', ['\u{13e4}', '\u{0}', '\u{0}']), ('\u{abb5}', ['\u{13e5}', '\u{0}', '\u{0}']), - ('\u{abb6}', ['\u{13e6}', '\u{0}', '\u{0}']), ('\u{abb7}', ['\u{13e7}', '\u{0}', '\u{0}']), - ('\u{abb8}', ['\u{13e8}', '\u{0}', '\u{0}']), ('\u{abb9}', ['\u{13e9}', '\u{0}', '\u{0}']), - ('\u{abba}', ['\u{13ea}', '\u{0}', '\u{0}']), ('\u{abbb}', ['\u{13eb}', '\u{0}', '\u{0}']), - ('\u{abbc}', ['\u{13ec}', '\u{0}', '\u{0}']), ('\u{abbd}', ['\u{13ed}', '\u{0}', '\u{0}']), - ('\u{abbe}', ['\u{13ee}', '\u{0}', '\u{0}']), ('\u{abbf}', ['\u{13ef}', '\u{0}', '\u{0}']), - ('\u{fb00}', ['F', 'F', '\u{0}']), ('\u{fb01}', ['F', 'I', '\u{0}']), - ('\u{fb02}', ['F', 'L', '\u{0}']), ('\u{fb03}', ['F', 'F', 'I']), - ('\u{fb04}', ['F', 'F', 'L']), ('\u{fb05}', ['S', 'T', '\u{0}']), - ('\u{fb06}', ['S', 'T', '\u{0}']), ('\u{fb13}', ['\u{544}', '\u{546}', '\u{0}']), + ('\u{a7d1}', ['\u{a7d0}', '\u{0}', '\u{0}']), ('\u{a7d7}', ['\u{a7d6}', '\u{0}', '\u{0}']), + ('\u{a7d9}', ['\u{a7d8}', '\u{0}', '\u{0}']), ('\u{a7f6}', ['\u{a7f5}', '\u{0}', '\u{0}']), + ('\u{ab53}', ['\u{a7b3}', '\u{0}', '\u{0}']), ('\u{ab70}', ['\u{13a0}', '\u{0}', '\u{0}']), + ('\u{ab71}', ['\u{13a1}', '\u{0}', '\u{0}']), ('\u{ab72}', ['\u{13a2}', '\u{0}', '\u{0}']), + ('\u{ab73}', ['\u{13a3}', '\u{0}', '\u{0}']), ('\u{ab74}', ['\u{13a4}', '\u{0}', '\u{0}']), + ('\u{ab75}', ['\u{13a5}', '\u{0}', '\u{0}']), ('\u{ab76}', ['\u{13a6}', '\u{0}', '\u{0}']), + ('\u{ab77}', ['\u{13a7}', '\u{0}', '\u{0}']), ('\u{ab78}', ['\u{13a8}', '\u{0}', '\u{0}']), + ('\u{ab79}', ['\u{13a9}', '\u{0}', '\u{0}']), ('\u{ab7a}', ['\u{13aa}', '\u{0}', '\u{0}']), + ('\u{ab7b}', ['\u{13ab}', '\u{0}', '\u{0}']), ('\u{ab7c}', ['\u{13ac}', '\u{0}', '\u{0}']), + ('\u{ab7d}', ['\u{13ad}', '\u{0}', '\u{0}']), ('\u{ab7e}', ['\u{13ae}', '\u{0}', '\u{0}']), + ('\u{ab7f}', ['\u{13af}', '\u{0}', '\u{0}']), ('\u{ab80}', ['\u{13b0}', '\u{0}', '\u{0}']), + ('\u{ab81}', ['\u{13b1}', '\u{0}', '\u{0}']), ('\u{ab82}', ['\u{13b2}', '\u{0}', '\u{0}']), + ('\u{ab83}', ['\u{13b3}', '\u{0}', '\u{0}']), ('\u{ab84}', ['\u{13b4}', '\u{0}', '\u{0}']), + ('\u{ab85}', ['\u{13b5}', '\u{0}', '\u{0}']), ('\u{ab86}', ['\u{13b6}', '\u{0}', '\u{0}']), + ('\u{ab87}', ['\u{13b7}', '\u{0}', '\u{0}']), ('\u{ab88}', ['\u{13b8}', '\u{0}', '\u{0}']), + ('\u{ab89}', ['\u{13b9}', '\u{0}', '\u{0}']), ('\u{ab8a}', ['\u{13ba}', '\u{0}', '\u{0}']), + ('\u{ab8b}', ['\u{13bb}', '\u{0}', '\u{0}']), ('\u{ab8c}', ['\u{13bc}', '\u{0}', '\u{0}']), + ('\u{ab8d}', ['\u{13bd}', '\u{0}', '\u{0}']), ('\u{ab8e}', ['\u{13be}', '\u{0}', '\u{0}']), + ('\u{ab8f}', ['\u{13bf}', '\u{0}', '\u{0}']), ('\u{ab90}', ['\u{13c0}', '\u{0}', '\u{0}']), + ('\u{ab91}', ['\u{13c1}', '\u{0}', '\u{0}']), ('\u{ab92}', ['\u{13c2}', '\u{0}', '\u{0}']), + ('\u{ab93}', ['\u{13c3}', '\u{0}', '\u{0}']), ('\u{ab94}', ['\u{13c4}', '\u{0}', '\u{0}']), + ('\u{ab95}', ['\u{13c5}', '\u{0}', '\u{0}']), ('\u{ab96}', ['\u{13c6}', '\u{0}', '\u{0}']), + ('\u{ab97}', ['\u{13c7}', '\u{0}', '\u{0}']), ('\u{ab98}', ['\u{13c8}', '\u{0}', '\u{0}']), + ('\u{ab99}', ['\u{13c9}', '\u{0}', '\u{0}']), ('\u{ab9a}', ['\u{13ca}', '\u{0}', '\u{0}']), + ('\u{ab9b}', ['\u{13cb}', '\u{0}', '\u{0}']), ('\u{ab9c}', ['\u{13cc}', '\u{0}', '\u{0}']), + ('\u{ab9d}', ['\u{13cd}', '\u{0}', '\u{0}']), ('\u{ab9e}', ['\u{13ce}', '\u{0}', '\u{0}']), + ('\u{ab9f}', ['\u{13cf}', '\u{0}', '\u{0}']), ('\u{aba0}', ['\u{13d0}', '\u{0}', '\u{0}']), + ('\u{aba1}', ['\u{13d1}', '\u{0}', '\u{0}']), ('\u{aba2}', ['\u{13d2}', '\u{0}', '\u{0}']), + ('\u{aba3}', ['\u{13d3}', '\u{0}', '\u{0}']), ('\u{aba4}', ['\u{13d4}', '\u{0}', '\u{0}']), + ('\u{aba5}', ['\u{13d5}', '\u{0}', '\u{0}']), ('\u{aba6}', ['\u{13d6}', '\u{0}', '\u{0}']), + ('\u{aba7}', ['\u{13d7}', '\u{0}', '\u{0}']), ('\u{aba8}', ['\u{13d8}', '\u{0}', '\u{0}']), + ('\u{aba9}', ['\u{13d9}', '\u{0}', '\u{0}']), ('\u{abaa}', ['\u{13da}', '\u{0}', '\u{0}']), + ('\u{abab}', ['\u{13db}', '\u{0}', '\u{0}']), ('\u{abac}', ['\u{13dc}', '\u{0}', '\u{0}']), + ('\u{abad}', ['\u{13dd}', '\u{0}', '\u{0}']), ('\u{abae}', ['\u{13de}', '\u{0}', '\u{0}']), + ('\u{abaf}', ['\u{13df}', '\u{0}', '\u{0}']), ('\u{abb0}', ['\u{13e0}', '\u{0}', '\u{0}']), + ('\u{abb1}', ['\u{13e1}', '\u{0}', '\u{0}']), ('\u{abb2}', ['\u{13e2}', '\u{0}', '\u{0}']), + ('\u{abb3}', ['\u{13e3}', '\u{0}', '\u{0}']), ('\u{abb4}', ['\u{13e4}', '\u{0}', '\u{0}']), + ('\u{abb5}', ['\u{13e5}', '\u{0}', '\u{0}']), ('\u{abb6}', ['\u{13e6}', '\u{0}', '\u{0}']), + ('\u{abb7}', ['\u{13e7}', '\u{0}', '\u{0}']), ('\u{abb8}', ['\u{13e8}', '\u{0}', '\u{0}']), + ('\u{abb9}', ['\u{13e9}', '\u{0}', '\u{0}']), ('\u{abba}', ['\u{13ea}', '\u{0}', '\u{0}']), + ('\u{abbb}', ['\u{13eb}', '\u{0}', '\u{0}']), ('\u{abbc}', ['\u{13ec}', '\u{0}', '\u{0}']), + ('\u{abbd}', ['\u{13ed}', '\u{0}', '\u{0}']), ('\u{abbe}', ['\u{13ee}', '\u{0}', '\u{0}']), + ('\u{abbf}', ['\u{13ef}', '\u{0}', '\u{0}']), ('\u{fb00}', ['F', 'F', '\u{0}']), + ('\u{fb01}', ['F', 'I', '\u{0}']), ('\u{fb02}', ['F', 'L', '\u{0}']), + ('\u{fb03}', ['F', 'F', 'I']), ('\u{fb04}', ['F', 'F', 'L']), + ('\u{fb05}', ['S', 'T', '\u{0}']), ('\u{fb06}', ['S', 'T', '\u{0}']), + ('\u{fb13}', ['\u{544}', '\u{546}', '\u{0}']), ('\u{fb14}', ['\u{544}', '\u{535}', '\u{0}']), ('\u{fb15}', ['\u{544}', '\u{53b}', '\u{0}']), ('\u{fb16}', ['\u{54e}', '\u{546}', '\u{0}']), @@ -2196,6 +2206,41 @@ pub mod conversions { ('\u{104f9}', ['\u{104d1}', '\u{0}', '\u{0}']), ('\u{104fa}', ['\u{104d2}', '\u{0}', '\u{0}']), ('\u{104fb}', ['\u{104d3}', '\u{0}', '\u{0}']), + ('\u{10597}', ['\u{10570}', '\u{0}', '\u{0}']), + ('\u{10598}', ['\u{10571}', '\u{0}', '\u{0}']), + ('\u{10599}', ['\u{10572}', '\u{0}', '\u{0}']), + ('\u{1059a}', ['\u{10573}', '\u{0}', '\u{0}']), + ('\u{1059b}', ['\u{10574}', '\u{0}', '\u{0}']), + ('\u{1059c}', ['\u{10575}', '\u{0}', '\u{0}']), + ('\u{1059d}', ['\u{10576}', '\u{0}', '\u{0}']), + ('\u{1059e}', ['\u{10577}', '\u{0}', '\u{0}']), + ('\u{1059f}', ['\u{10578}', '\u{0}', '\u{0}']), + ('\u{105a0}', ['\u{10579}', '\u{0}', '\u{0}']), + ('\u{105a1}', ['\u{1057a}', '\u{0}', '\u{0}']), + ('\u{105a3}', ['\u{1057c}', '\u{0}', '\u{0}']), + ('\u{105a4}', ['\u{1057d}', '\u{0}', '\u{0}']), + ('\u{105a5}', ['\u{1057e}', '\u{0}', '\u{0}']), + ('\u{105a6}', ['\u{1057f}', '\u{0}', '\u{0}']), + ('\u{105a7}', ['\u{10580}', '\u{0}', '\u{0}']), + ('\u{105a8}', ['\u{10581}', '\u{0}', '\u{0}']), + ('\u{105a9}', ['\u{10582}', '\u{0}', '\u{0}']), + ('\u{105aa}', ['\u{10583}', '\u{0}', '\u{0}']), + ('\u{105ab}', ['\u{10584}', '\u{0}', '\u{0}']), + ('\u{105ac}', ['\u{10585}', '\u{0}', '\u{0}']), + ('\u{105ad}', ['\u{10586}', '\u{0}', '\u{0}']), + ('\u{105ae}', ['\u{10587}', '\u{0}', '\u{0}']), + ('\u{105af}', ['\u{10588}', '\u{0}', '\u{0}']), + ('\u{105b0}', ['\u{10589}', '\u{0}', '\u{0}']), + ('\u{105b1}', ['\u{1058a}', '\u{0}', '\u{0}']), + ('\u{105b3}', ['\u{1058c}', '\u{0}', '\u{0}']), + ('\u{105b4}', ['\u{1058d}', '\u{0}', '\u{0}']), + ('\u{105b5}', ['\u{1058e}', '\u{0}', '\u{0}']), + ('\u{105b6}', ['\u{1058f}', '\u{0}', '\u{0}']), + ('\u{105b7}', ['\u{10590}', '\u{0}', '\u{0}']), + ('\u{105b8}', ['\u{10591}', '\u{0}', '\u{0}']), + ('\u{105b9}', ['\u{10592}', '\u{0}', '\u{0}']), + ('\u{105bb}', ['\u{10594}', '\u{0}', '\u{0}']), + ('\u{105bc}', ['\u{10595}', '\u{0}', '\u{0}']), ('\u{10cc0}', ['\u{10c80}', '\u{0}', '\u{0}']), ('\u{10cc1}', ['\u{10c81}', '\u{0}', '\u{0}']), ('\u{10cc2}', ['\u{10c82}', '\u{0}', '\u{0}']), diff --git a/crux-mir/lib/core/src/unicode/version.rs b/crux-mir/lib/core/src/unicode/version.rs deleted file mode 100644 index 4d68d2e8c..000000000 --- a/crux-mir/lib/core/src/unicode/version.rs +++ /dev/null @@ -1,18 +0,0 @@ -/// Represents a Unicode Version. -/// -/// See also: -#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] -#[unstable(feature = "unicode_version", issue = "49726")] -pub struct UnicodeVersion { - /// Major version. - pub major: u32, - - /// Minor version. - pub minor: u32, - - /// Micro (or Update) version. - pub micro: u32, - - // Private field to keep struct expandable. - pub(crate) _priv: (), -} diff --git a/crux-mir/lib/core/src/unit.rs b/crux-mir/lib/core/src/unit.rs index f41f4a5e9..6656dd5c4 100644 --- a/crux-mir/lib/core/src/unit.rs +++ b/crux-mir/lib/core/src/unit.rs @@ -9,7 +9,7 @@ use crate::iter::FromIterator; /// use std::io::*; /// let data = vec![1, 2, 3, 4, 5]; /// let res: Result<()> = data.iter() -/// .map(|x| writeln!(stdout(), "{}", x)) +/// .map(|x| writeln!(stdout(), "{x}")) /// .collect(); /// assert!(res.is_ok()); /// ``` diff --git a/crux-mir/lib/core/tests/alloc.rs b/crux-mir/lib/core/tests/alloc.rs index c8592e40a..3ceaeadce 100644 --- a/crux-mir/lib/core/tests/alloc.rs +++ b/crux-mir/lib/core/tests/alloc.rs @@ -1,5 +1,6 @@ use core::alloc::Layout; -use core::ptr::NonNull; +use core::mem::size_of; +use core::ptr::{self, NonNull}; #[test] fn const_unchecked_layout() { @@ -9,5 +10,66 @@ fn const_unchecked_layout() { const DANGLING: NonNull = LAYOUT.dangling(); assert_eq!(LAYOUT.size(), SIZE); assert_eq!(LAYOUT.align(), ALIGN); - assert_eq!(Some(DANGLING), NonNull::new(ALIGN as *mut u8)); + assert_eq!(Some(DANGLING), NonNull::new(ptr::invalid_mut(ALIGN))); +} + +#[test] +fn layout_round_up_to_align_edge_cases() { + const MAX_SIZE: usize = isize::MAX as usize; + + for shift in 0..usize::BITS { + let align = 1_usize << shift; + let edge = (MAX_SIZE + 1) - align; + let low = edge.saturating_sub(10); + let high = edge.saturating_add(10); + assert!(Layout::from_size_align(low, align).is_ok()); + assert!(Layout::from_size_align(high, align).is_err()); + for size in low..=high { + assert_eq!( + Layout::from_size_align(size, align).is_ok(), + size.next_multiple_of(align) <= MAX_SIZE, + ); + } + } +} + +#[test] +fn layout_array_edge_cases() { + for_type::(); + for_type::<[i32; 0b10101]>(); + for_type::<[u8; 0b1010101]>(); + + // Make sure ZSTs don't lead to divide-by-zero + assert_eq!(Layout::array::<()>(usize::MAX).unwrap(), Layout::from_size_align(0, 1).unwrap()); + + fn for_type() { + const MAX_SIZE: usize = isize::MAX as usize; + + let edge = (MAX_SIZE + 1) / size_of::(); + let low = edge.saturating_sub(10); + let high = edge.saturating_add(10); + assert!(Layout::array::(low).is_ok()); + assert!(Layout::array::(high).is_err()); + for n in low..=high { + assert_eq!(Layout::array::(n).is_ok(), n * size_of::() <= MAX_SIZE); + } + } +} + +#[test] +fn layout_debug_shows_log2_of_alignment() { + // `Debug` is not stable, but here's what it does right now + let layout = Layout::from_size_align(24576, 8192).unwrap(); + let s = format!("{:?}", layout); + assert_eq!(s, "Layout { size: 24576, align: 8192 (1 << 13) }"); +} + +// Running this normally doesn't do much, but it's also run in Miri, which +// will double-check that these are allowed by the validity invariants. +#[test] +fn layout_accepts_all_valid_alignments() { + for align in 0..usize::BITS { + let layout = Layout::from_size_align(0, 1_usize << align).unwrap(); + assert_eq!(layout.align(), 1_usize << align); + } } diff --git a/crux-mir/lib/core/tests/any.rs b/crux-mir/lib/core/tests/any.rs index b0dc99034..a8f6b7ebb 100644 --- a/crux-mir/lib/core/tests/any.rs +++ b/crux-mir/lib/core/tests/any.rs @@ -24,8 +24,11 @@ fn any_referenced() { #[test] fn any_owning() { - let (a, b, c) = - (box 5_usize as Box, box TEST as Box, box Test as Box); + let (a, b, c) = ( + Box::new(5_usize) as Box, + Box::new(TEST) as Box, + Box::new(Test) as Box, + ); assert!(a.is::()); assert!(!b.is::()); @@ -46,19 +49,19 @@ fn any_downcast_ref() { match a.downcast_ref::() { Some(&5) => {} - x => panic!("Unexpected value {:?}", x), + x => panic!("Unexpected value {x:?}"), } match a.downcast_ref::() { None => {} - x => panic!("Unexpected value {:?}", x), + x => panic!("Unexpected value {x:?}"), } } #[test] fn any_downcast_mut() { let mut a = 5_usize; - let mut b: Box<_> = box 7_usize; + let mut b: Box<_> = Box::new(7_usize); let a_r = &mut a as &mut dyn Any; let tmp: &mut usize = &mut *b; @@ -69,7 +72,7 @@ fn any_downcast_mut() { assert_eq!(*x, 5); *x = 612; } - x => panic!("Unexpected value {:?}", x), + x => panic!("Unexpected value {x:?}"), } match b_r.downcast_mut::() { @@ -77,27 +80,27 @@ fn any_downcast_mut() { assert_eq!(*x, 7); *x = 413; } - x => panic!("Unexpected value {:?}", x), + x => panic!("Unexpected value {x:?}"), } match a_r.downcast_mut::() { None => (), - x => panic!("Unexpected value {:?}", x), + x => panic!("Unexpected value {x:?}"), } match b_r.downcast_mut::() { None => (), - x => panic!("Unexpected value {:?}", x), + x => panic!("Unexpected value {x:?}"), } match a_r.downcast_mut::() { Some(&mut 612) => {} - x => panic!("Unexpected value {:?}", x), + x => panic!("Unexpected value {x:?}"), } match b_r.downcast_mut::() { Some(&mut 413) => {} - x => panic!("Unexpected value {:?}", x), + x => panic!("Unexpected value {x:?}"), } } @@ -114,3 +117,95 @@ fn any_unsized() { fn is_any() {} is_any::<[i32]>(); } + +#[test] +fn distinct_type_names() { + // https://github.com/rust-lang/rust/issues/84666 + + struct Velocity(f32, f32); + + fn type_name_of_val(_: T) -> &'static str { + type_name::() + } + + assert_ne!(type_name_of_val(Velocity), type_name_of_val(Velocity(0.0, -9.8)),); +} + +#[test] +fn dyn_type_name() { + trait Foo { + type Bar; + } + + assert_eq!( + "dyn core::ops::function::Fn(i32, i32) -> i32", + std::any::type_name:: i32>() + ); + assert_eq!( + "dyn coretests::any::dyn_type_name::Foo \ + + core::marker::Send + core::marker::Sync", + std::any::type_name:: + Send + Sync>() + ); +} + +// Test the `Provider` API. + +struct SomeConcreteType { + some_string: String, +} + +impl Provider for SomeConcreteType { + fn provide<'a>(&'a self, demand: &mut Demand<'a>) { + demand + .provide_ref::(&self.some_string) + .provide_ref::(&self.some_string) + .provide_value_with::(|| "bye".to_owned()); + } +} + +// Test the provide and request mechanisms with a by-reference trait object. +#[test] +fn test_provider() { + let obj: &dyn Provider = &SomeConcreteType { some_string: "hello".to_owned() }; + + assert_eq!(&**request_ref::(obj).unwrap(), "hello"); + assert_eq!(&*request_value::(obj).unwrap(), "bye"); + assert_eq!(request_value::(obj), None); +} + +// Test the provide and request mechanisms with a boxed trait object. +#[test] +fn test_provider_boxed() { + let obj: Box = Box::new(SomeConcreteType { some_string: "hello".to_owned() }); + + assert_eq!(&**request_ref::(&*obj).unwrap(), "hello"); + assert_eq!(&*request_value::(&*obj).unwrap(), "bye"); + assert_eq!(request_value::(&*obj), None); +} + +// Test the provide and request mechanisms with a concrete object. +#[test] +fn test_provider_concrete() { + let obj = SomeConcreteType { some_string: "hello".to_owned() }; + + assert_eq!(&**request_ref::(&obj).unwrap(), "hello"); + assert_eq!(&*request_value::(&obj).unwrap(), "bye"); + assert_eq!(request_value::(&obj), None); +} + +trait OtherTrait: Provider {} + +impl OtherTrait for SomeConcreteType {} + +impl dyn OtherTrait { + fn get_ref(&self) -> Option<&T> { + request_ref::(self) + } +} + +// Test the provide and request mechanisms via an intermediate trait. +#[test] +fn test_provider_intermediate() { + let obj: &dyn OtherTrait = &SomeConcreteType { some_string: "hello".to_owned() }; + assert_eq!(obj.get_ref::().unwrap(), "hello"); +} diff --git a/crux-mir/lib/core/tests/array.rs b/crux-mir/lib/core/tests/array.rs index c2a816f0a..f268fe3ae 100644 --- a/crux-mir/lib/core/tests/array.rs +++ b/crux-mir/lib/core/tests/array.rs @@ -1,22 +1,25 @@ -use core::array::{FixedSizeArray, IntoIter}; +use core::array; use core::convert::TryFrom; +use core::sync::atomic::{AtomicUsize, Ordering}; #[test] -fn fixed_size_array() { - let mut array = [0; 64]; - let mut zero_sized = [(); 64]; - let mut empty_array = [0; 0]; - let mut empty_zero_sized = [(); 0]; - - assert_eq!(FixedSizeArray::as_slice(&array).len(), 64); - assert_eq!(FixedSizeArray::as_slice(&zero_sized).len(), 64); - assert_eq!(FixedSizeArray::as_slice(&empty_array).len(), 0); - assert_eq!(FixedSizeArray::as_slice(&empty_zero_sized).len(), 0); +fn array_from_ref() { + let value: String = "Hello World!".into(); + let arr: &[String; 1] = array::from_ref(&value); + assert_eq!(&[value.clone()], arr); + + const VALUE: &&str = &"Hello World!"; + const ARR: &[&str; 1] = array::from_ref(VALUE); + assert_eq!(&[*VALUE], ARR); + assert!(core::ptr::eq(VALUE, &ARR[0])); +} - assert_eq!(FixedSizeArray::as_mut_slice(&mut array).len(), 64); - assert_eq!(FixedSizeArray::as_mut_slice(&mut zero_sized).len(), 64); - assert_eq!(FixedSizeArray::as_mut_slice(&mut empty_array).len(), 0); - assert_eq!(FixedSizeArray::as_mut_slice(&mut empty_zero_sized).len(), 0); +#[test] +fn array_from_mut() { + let mut value: String = "Hello World".into(); + let arr: &mut [String; 1] = array::from_mut(&mut value); + arr[0].push_str("!"); + assert_eq!(&value, "Hello World!"); } #[test] @@ -25,11 +28,22 @@ fn array_try_from() { ($($N:expr)+) => { $({ type Array = [u8; $N]; - let array: Array = [0; $N]; + let mut array: Array = [0; $N]; let slice: &[u8] = &array[..]; let result = <&Array>::try_from(slice); assert_eq!(&array, result.unwrap()); + + let result = ::try_from(slice); + assert_eq!(&array, &result.unwrap()); + + let mut_slice: &mut [u8] = &mut array[..]; + let result = <&mut Array>::try_from(mut_slice); + assert_eq!(&[0; $N], result.unwrap()); + + let mut_slice: &mut [u8] = &mut array[..]; + let result = ::try_from(mut_slice); + assert_eq!(&array, &result.unwrap()); })+ } } @@ -44,14 +58,14 @@ fn array_try_from() { #[test] fn iterator_collect() { let arr = [0, 1, 2, 5, 9]; - let v: Vec<_> = IntoIter::new(arr.clone()).collect(); + let v: Vec<_> = IntoIterator::into_iter(arr.clone()).collect(); assert_eq!(&arr[..], &v[..]); } #[test] fn iterator_rev_collect() { let arr = [0, 1, 2, 5, 9]; - let v: Vec<_> = IntoIter::new(arr.clone()).rev().collect(); + let v: Vec<_> = IntoIterator::into_iter(arr.clone()).rev().collect(); assert_eq!(&v[..], &[9, 5, 2, 1, 0]); } @@ -59,11 +73,11 @@ fn iterator_rev_collect() { fn iterator_nth() { let v = [0, 1, 2, 3, 4]; for i in 0..v.len() { - assert_eq!(IntoIter::new(v.clone()).nth(i).unwrap(), v[i]); + assert_eq!(IntoIterator::into_iter(v.clone()).nth(i).unwrap(), v[i]); } - assert_eq!(IntoIter::new(v.clone()).nth(v.len()), None); + assert_eq!(IntoIterator::into_iter(v.clone()).nth(v.len()), None); - let mut iter = IntoIter::new(v); + let mut iter = IntoIterator::into_iter(v); assert_eq!(iter.nth(2).unwrap(), v[2]); assert_eq!(iter.nth(1).unwrap(), v[4]); } @@ -71,17 +85,17 @@ fn iterator_nth() { #[test] fn iterator_last() { let v = [0, 1, 2, 3, 4]; - assert_eq!(IntoIter::new(v).last().unwrap(), 4); - assert_eq!(IntoIter::new([0]).last().unwrap(), 0); + assert_eq!(IntoIterator::into_iter(v).last().unwrap(), 4); + assert_eq!(IntoIterator::into_iter([0]).last().unwrap(), 0); - let mut it = IntoIter::new([0, 9, 2, 4]); + let mut it = IntoIterator::into_iter([0, 9, 2, 4]); assert_eq!(it.next_back(), Some(4)); assert_eq!(it.last(), Some(2)); } #[test] fn iterator_clone() { - let mut it = IntoIter::new([0, 2, 4, 6, 8]); + let mut it = IntoIterator::into_iter([0, 2, 4, 6, 8]); assert_eq!(it.next(), Some(0)); assert_eq!(it.next_back(), Some(8)); let mut clone = it.clone(); @@ -95,7 +109,7 @@ fn iterator_clone() { #[test] fn iterator_fused() { - let mut it = IntoIter::new([0, 9, 2]); + let mut it = IntoIterator::into_iter([0, 9, 2]); assert_eq!(it.next(), Some(0)); assert_eq!(it.next(), Some(9)); assert_eq!(it.next(), Some(2)); @@ -108,7 +122,7 @@ fn iterator_fused() { #[test] fn iterator_len() { - let mut it = IntoIter::new([0, 1, 2, 5, 9]); + let mut it = IntoIterator::into_iter([0, 1, 2, 5, 9]); assert_eq!(it.size_hint(), (5, Some(5))); assert_eq!(it.len(), 5); assert_eq!(it.is_empty(), false); @@ -124,7 +138,7 @@ fn iterator_len() { assert_eq!(it.is_empty(), false); // Empty - let it = IntoIter::new([] as [String; 0]); + let it = IntoIterator::into_iter([] as [String; 0]); assert_eq!(it.size_hint(), (0, Some(0))); assert_eq!(it.len(), 0); assert_eq!(it.is_empty(), true); @@ -133,9 +147,9 @@ fn iterator_len() { #[test] fn iterator_count() { let v = [0, 1, 2, 3, 4]; - assert_eq!(IntoIter::new(v.clone()).count(), 5); + assert_eq!(IntoIterator::into_iter(v.clone()).count(), 5); - let mut iter2 = IntoIter::new(v); + let mut iter2 = IntoIterator::into_iter(v); iter2.next(); iter2.next(); assert_eq!(iter2.count(), 3); @@ -143,13 +157,13 @@ fn iterator_count() { #[test] fn iterator_flat_map() { - assert!((0..5).flat_map(|i| IntoIter::new([2 * i, 2 * i + 1])).eq(0..10)); + assert!((0..5).flat_map(|i| IntoIterator::into_iter([2 * i, 2 * i + 1])).eq(0..10)); } #[test] fn iterator_debug() { let arr = [0, 1, 2, 5, 9]; - assert_eq!(format!("{:?}", IntoIter::new(arr)), "IntoIter([0, 1, 2, 5, 9])",); + assert_eq!(format!("{:?}", IntoIterator::into_iter(arr)), "IntoIter([0, 1, 2, 5, 9])",); } #[test] @@ -179,14 +193,14 @@ fn iterator_drops() { // Simple: drop new iterator. let i = Cell::new(0); { - IntoIter::new(five(&i)); + IntoIterator::into_iter(five(&i)); } assert_eq!(i.get(), 5); // Call `next()` once. let i = Cell::new(0); { - let mut iter = IntoIter::new(five(&i)); + let mut iter = IntoIterator::into_iter(five(&i)); let _x = iter.next(); assert_eq!(i.get(), 0); assert_eq!(iter.count(), 4); @@ -197,7 +211,7 @@ fn iterator_drops() { // Check `clone` and calling `next`/`next_back`. let i = Cell::new(0); { - let mut iter = IntoIter::new(five(&i)); + let mut iter = IntoIterator::into_iter(five(&i)); iter.next(); assert_eq!(i.get(), 1); iter.next_back(); @@ -220,7 +234,7 @@ fn iterator_drops() { // Check via `nth`. let i = Cell::new(0); { - let mut iter = IntoIter::new(five(&i)); + let mut iter = IntoIterator::into_iter(five(&i)); let _x = iter.nth(2); assert_eq!(i.get(), 2); let _y = iter.last(); @@ -230,14 +244,459 @@ fn iterator_drops() { // Check every element. let i = Cell::new(0); - for (index, _x) in IntoIter::new(five(&i)).enumerate() { + for (index, _x) in IntoIterator::into_iter(five(&i)).enumerate() { assert_eq!(i.get(), index); } assert_eq!(i.get(), 5); let i = Cell::new(0); - for (index, _x) in IntoIter::new(five(&i)).rev().enumerate() { + for (index, _x) in IntoIterator::into_iter(five(&i)).rev().enumerate() { assert_eq!(i.get(), index); } assert_eq!(i.get(), 5); } + +// This test does not work on targets without panic=unwind support. +// To work around this problem, test is marked is should_panic, so it will +// be automagically skipped on unsuitable targets, such as +// wasm32-unknown-unknown. +// +// It means that we use panic for indicating success. +#[test] +#[should_panic(expected = "test succeeded")] +fn array_default_impl_avoids_leaks_on_panic() { + use core::sync::atomic::{AtomicUsize, Ordering::Relaxed}; + static COUNTER: AtomicUsize = AtomicUsize::new(0); + #[derive(Debug)] + struct Bomb(usize); + + impl Default for Bomb { + fn default() -> Bomb { + if COUNTER.load(Relaxed) == 3 { + panic!("bomb limit exceeded"); + } + + COUNTER.fetch_add(1, Relaxed); + Bomb(COUNTER.load(Relaxed)) + } + } + + impl Drop for Bomb { + fn drop(&mut self) { + COUNTER.fetch_sub(1, Relaxed); + } + } + + let res = std::panic::catch_unwind(|| <[Bomb; 5]>::default()); + let panic_msg = match res { + Ok(_) => unreachable!(), + Err(p) => p.downcast::<&'static str>().unwrap(), + }; + assert_eq!(*panic_msg, "bomb limit exceeded"); + // check that all bombs are successfully dropped + assert_eq!(COUNTER.load(Relaxed), 0); + panic!("test succeeded") +} + +#[test] +fn empty_array_is_always_default() { + struct DoesNotImplDefault; + + let _arr = <[DoesNotImplDefault; 0]>::default(); +} + +#[test] +fn array_map() { + let a = [1, 2, 3]; + let b = a.map(|v| v + 1); + assert_eq!(b, [2, 3, 4]); + + let a = [1u8, 2, 3]; + let b = a.map(|v| v as u64); + assert_eq!(b, [1, 2, 3]); +} + +// See note on above test for why `should_panic` is used. +#[test] +#[should_panic(expected = "test succeeded")] +fn array_map_drop_safety() { + static DROPPED: AtomicUsize = AtomicUsize::new(0); + struct DropCounter; + impl Drop for DropCounter { + fn drop(&mut self) { + DROPPED.fetch_add(1, Ordering::SeqCst); + } + } + + let num_to_create = 5; + let success = std::panic::catch_unwind(|| { + let items = [0; 10]; + let mut nth = 0; + items.map(|_| { + assert!(nth < num_to_create); + nth += 1; + DropCounter + }); + }); + assert!(success.is_err()); + assert_eq!(DROPPED.load(Ordering::SeqCst), num_to_create); + panic!("test succeeded") +} + +#[test] +fn cell_allows_array_cycle() { + use core::cell::Cell; + + #[derive(Debug)] + struct B<'a> { + a: [Cell>>; 2], + } + + impl<'a> B<'a> { + fn new() -> B<'a> { + B { a: [Cell::new(None), Cell::new(None)] } + } + } + + let b1 = B::new(); + let b2 = B::new(); + let b3 = B::new(); + + b1.a[0].set(Some(&b2)); + b1.a[1].set(Some(&b3)); + + b2.a[0].set(Some(&b2)); + b2.a[1].set(Some(&b3)); + + b3.a[0].set(Some(&b1)); + b3.a[1].set(Some(&b2)); +} + +#[test] +fn array_from_fn() { + let array = core::array::from_fn(|idx| idx); + assert_eq!(array, [0, 1, 2, 3, 4]); +} + +#[test] +fn array_try_from_fn() { + #[derive(Debug, PartialEq)] + enum SomeError { + Foo, + } + + let array = core::array::try_from_fn(|i| Ok::<_, SomeError>(i)); + assert_eq!(array, Ok([0, 1, 2, 3, 4])); + + let another_array = core::array::try_from_fn::, 2, _>(|_| Err(SomeError::Foo)); + assert_eq!(another_array, Err(SomeError::Foo)); +} + +#[cfg(not(panic = "abort"))] +#[test] +fn array_try_from_fn_drops_inserted_elements_on_err() { + static DROP_COUNTER: AtomicUsize = AtomicUsize::new(0); + + struct CountDrop; + impl Drop for CountDrop { + fn drop(&mut self) { + DROP_COUNTER.fetch_add(1, Ordering::SeqCst); + } + } + + let _ = catch_unwind_silent(move || { + let _: Result<[CountDrop; 4], ()> = core::array::try_from_fn(|idx| { + if idx == 2 { + return Err(()); + } + Ok(CountDrop) + }); + }); + + assert_eq!(DROP_COUNTER.load(Ordering::SeqCst), 2); +} + +#[cfg(not(panic = "abort"))] +#[test] +fn array_try_from_fn_drops_inserted_elements_on_panic() { + static DROP_COUNTER: AtomicUsize = AtomicUsize::new(0); + + struct CountDrop; + impl Drop for CountDrop { + fn drop(&mut self) { + DROP_COUNTER.fetch_add(1, Ordering::SeqCst); + } + } + + let _ = catch_unwind_silent(move || { + let _: Result<[CountDrop; 4], ()> = core::array::try_from_fn(|idx| { + if idx == 2 { + panic!("peek a boo"); + } + Ok(CountDrop) + }); + }); + + assert_eq!(DROP_COUNTER.load(Ordering::SeqCst), 2); +} + +#[cfg(not(panic = "abort"))] +// https://stackoverflow.com/a/59211505 +fn catch_unwind_silent(f: F) -> std::thread::Result +where + F: FnOnce() -> R + core::panic::UnwindSafe, +{ + let prev_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let result = std::panic::catch_unwind(f); + std::panic::set_hook(prev_hook); + result +} + +#[test] +fn array_split_array_mut() { + let mut v = [1, 2, 3, 4, 5, 6]; + + { + let (left, right) = v.split_array_mut::<0>(); + assert_eq!(left, &mut []); + assert_eq!(right, &mut [1, 2, 3, 4, 5, 6]); + } + + { + let (left, right) = v.split_array_mut::<6>(); + assert_eq!(left, &mut [1, 2, 3, 4, 5, 6]); + assert_eq!(right, &mut []); + } +} + +#[test] +fn array_rsplit_array_mut() { + let mut v = [1, 2, 3, 4, 5, 6]; + + { + let (left, right) = v.rsplit_array_mut::<0>(); + assert_eq!(left, &mut [1, 2, 3, 4, 5, 6]); + assert_eq!(right, &mut []); + } + + { + let (left, right) = v.rsplit_array_mut::<6>(); + assert_eq!(left, &mut []); + assert_eq!(right, &mut [1, 2, 3, 4, 5, 6]); + } +} + +#[should_panic] +#[test] +fn array_split_array_ref_out_of_bounds() { + let v = [1, 2, 3, 4, 5, 6]; + + v.split_array_ref::<7>(); +} + +#[should_panic] +#[test] +fn array_split_array_mut_out_of_bounds() { + let mut v = [1, 2, 3, 4, 5, 6]; + + v.split_array_mut::<7>(); +} + +#[should_panic] +#[test] +fn array_rsplit_array_ref_out_of_bounds() { + let v = [1, 2, 3, 4, 5, 6]; + + v.rsplit_array_ref::<7>(); +} + +#[should_panic] +#[test] +fn array_rsplit_array_mut_out_of_bounds() { + let mut v = [1, 2, 3, 4, 5, 6]; + + v.rsplit_array_mut::<7>(); +} + +#[test] +fn array_intoiter_advance_by() { + use std::cell::Cell; + struct DropCounter<'a>(usize, &'a Cell); + impl Drop for DropCounter<'_> { + fn drop(&mut self) { + let x = self.1.get(); + self.1.set(x + 1); + } + } + + let counter = Cell::new(0); + let a: [_; 100] = std::array::from_fn(|i| DropCounter(i, &counter)); + let mut it = IntoIterator::into_iter(a); + + let r = it.advance_by(1); + assert_eq!(r, Ok(())); + assert_eq!(it.len(), 99); + assert_eq!(counter.get(), 1); + + let r = it.advance_by(0); + assert_eq!(r, Ok(())); + assert_eq!(it.len(), 99); + assert_eq!(counter.get(), 1); + + let r = it.advance_by(11); + assert_eq!(r, Ok(())); + assert_eq!(it.len(), 88); + assert_eq!(counter.get(), 12); + + let x = it.next(); + assert_eq!(x.as_ref().map(|x| x.0), Some(12)); + assert_eq!(it.len(), 87); + assert_eq!(counter.get(), 12); + drop(x); + assert_eq!(counter.get(), 13); + + let r = it.advance_by(123456); + assert_eq!(r, Err(87)); + assert_eq!(it.len(), 0); + assert_eq!(counter.get(), 100); + + let r = it.advance_by(0); + assert_eq!(r, Ok(())); + assert_eq!(it.len(), 0); + assert_eq!(counter.get(), 100); + + let r = it.advance_by(10); + assert_eq!(r, Err(0)); + assert_eq!(it.len(), 0); + assert_eq!(counter.get(), 100); +} + +#[test] +fn array_intoiter_advance_back_by() { + use std::cell::Cell; + struct DropCounter<'a>(usize, &'a Cell); + impl Drop for DropCounter<'_> { + fn drop(&mut self) { + let x = self.1.get(); + self.1.set(x + 1); + } + } + + let counter = Cell::new(0); + let a: [_; 100] = std::array::from_fn(|i| DropCounter(i, &counter)); + let mut it = IntoIterator::into_iter(a); + + let r = it.advance_back_by(1); + assert_eq!(r, Ok(())); + assert_eq!(it.len(), 99); + assert_eq!(counter.get(), 1); + + let r = it.advance_back_by(0); + assert_eq!(r, Ok(())); + assert_eq!(it.len(), 99); + assert_eq!(counter.get(), 1); + + let r = it.advance_back_by(11); + assert_eq!(r, Ok(())); + assert_eq!(it.len(), 88); + assert_eq!(counter.get(), 12); + + let x = it.next_back(); + assert_eq!(x.as_ref().map(|x| x.0), Some(87)); + assert_eq!(it.len(), 87); + assert_eq!(counter.get(), 12); + drop(x); + assert_eq!(counter.get(), 13); + + let r = it.advance_back_by(123456); + assert_eq!(r, Err(87)); + assert_eq!(it.len(), 0); + assert_eq!(counter.get(), 100); + + let r = it.advance_back_by(0); + assert_eq!(r, Ok(())); + assert_eq!(it.len(), 0); + assert_eq!(counter.get(), 100); + + let r = it.advance_back_by(10); + assert_eq!(r, Err(0)); + assert_eq!(it.len(), 0); + assert_eq!(counter.get(), 100); +} + +#[test] +fn array_mixed_equality_integers() { + let array3: [i32; 3] = [1, 2, 3]; + let array3b: [i32; 3] = [3, 2, 1]; + let array4: [i32; 4] = [1, 2, 3, 4]; + + let slice3: &[i32] = &{ array3 }; + let slice3b: &[i32] = &{ array3b }; + let slice4: &[i32] = &{ array4 }; + assert!(array3 == slice3); + assert!(array3 != slice3b); + assert!(array3 != slice4); + assert!(slice3 == array3); + assert!(slice3b != array3); + assert!(slice4 != array3); + + let mut3: &mut [i32] = &mut { array3 }; + let mut3b: &mut [i32] = &mut { array3b }; + let mut4: &mut [i32] = &mut { array4 }; + assert!(array3 == mut3); + assert!(array3 != mut3b); + assert!(array3 != mut4); + assert!(mut3 == array3); + assert!(mut3b != array3); + assert!(mut4 != array3); +} + +#[test] +fn array_mixed_equality_nans() { + let array3: [f32; 3] = [1.0, std::f32::NAN, 3.0]; + + let slice3: &[f32] = &{ array3 }; + assert!(!(array3 == slice3)); + assert!(array3 != slice3); + assert!(!(slice3 == array3)); + assert!(slice3 != array3); + + let mut3: &mut [f32] = &mut { array3 }; + assert!(!(array3 == mut3)); + assert!(array3 != mut3); + assert!(!(mut3 == array3)); + assert!(mut3 != array3); +} + +#[test] +fn array_into_iter_fold() { + // Strings to help MIRI catch if we double-free or something + let a = ["Aa".to_string(), "Bb".to_string(), "Cc".to_string()]; + let mut s = "s".to_string(); + a.into_iter().for_each(|b| s += &b); + assert_eq!(s, "sAaBbCc"); + + let a = [1, 2, 3, 4, 5, 6]; + let mut it = a.into_iter(); + it.advance_by(1).unwrap(); + it.advance_back_by(2).unwrap(); + let s = it.fold(10, |a, b| 10 * a + b); + assert_eq!(s, 10234); +} + +#[test] +fn array_into_iter_rfold() { + // Strings to help MIRI catch if we double-free or something + let a = ["Aa".to_string(), "Bb".to_string(), "Cc".to_string()]; + let mut s = "s".to_string(); + a.into_iter().rev().for_each(|b| s += &b); + assert_eq!(s, "sCcBbAa"); + + let a = [1, 2, 3, 4, 5, 6]; + let mut it = a.into_iter(); + it.advance_by(1).unwrap(); + it.advance_back_by(2).unwrap(); + let s = it.rfold(10, |a, b| 10 * a + b); + assert_eq!(s, 10432); +} diff --git a/crux-mir/lib/core/tests/ascii.rs b/crux-mir/lib/core/tests/ascii.rs index 71275d40c..f5f2dd047 100644 --- a/crux-mir/lib/core/tests/ascii.rs +++ b/crux-mir/lib/core/tests/ascii.rs @@ -115,7 +115,7 @@ fn test_eq_ignore_ascii_case() { #[test] fn inference_works() { let x = "a".to_string(); - x.eq_ignore_ascii_case("A"); + let _ = x.eq_ignore_ascii_case("A"); } // Shorthands used by the is_ascii_* tests. @@ -251,6 +251,23 @@ fn test_is_ascii_digit() { ); } +#[test] +fn test_is_ascii_octdigit() { + assert_all!(is_ascii_octdigit, "", "01234567"); + assert_none!( + is_ascii_octdigit, + "abcdefghijklmnopqrstuvwxyz", + "ABCDEFGHIJKLMNOQPRSTUVWXYZ", + "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~", + " \t\n\x0c\r", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x10\x11\x12\x13\x14\x15\x16\x17", + "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x7f", + ); +} + #[test] fn test_is_ascii_hexdigit() { assert_all!(is_ascii_hexdigit, "", "0123456789", "abcdefABCDEF",); @@ -343,3 +360,122 @@ fn test_is_ascii_control() { " ", ); } + +// `is_ascii` does a good amount of pointer manipulation and has +// alignment-dependent computation. This is all sanity-checked via +// `debug_assert!`s, so we test various sizes/alignments thoroughly versus an +// "obviously correct" baseline function. +#[test] +fn test_is_ascii_align_size_thoroughly() { + // The "obviously-correct" baseline mentioned above. + fn is_ascii_baseline(s: &[u8]) -> bool { + s.iter().all(|b| b.is_ascii()) + } + + // Helper to repeat `l` copies of `b0` followed by `l` copies of `b1`. + fn repeat_concat(b0: u8, b1: u8, l: usize) -> Vec { + use core::iter::repeat; + repeat(b0).take(l).chain(repeat(b1).take(l)).collect() + } + + // Miri is too slow + let iter = if cfg!(miri) { 0..20 } else { 0..100 }; + + for i in iter { + #[cfg(not(miri))] + let cases = &[ + b"a".repeat(i), + b"\0".repeat(i), + b"\x7f".repeat(i), + b"\x80".repeat(i), + b"\xff".repeat(i), + repeat_concat(b'a', 0x80u8, i), + repeat_concat(0x80u8, b'a', i), + ]; + + #[cfg(miri)] + let cases = &[b"a".repeat(i), b"\x80".repeat(i), repeat_concat(b'a', 0x80u8, i)]; + + for case in cases { + for pos in 0..=case.len() { + // Potentially misaligned head + let prefix = &case[pos..]; + assert_eq!(is_ascii_baseline(prefix), prefix.is_ascii(),); + + // Potentially misaligned tail + let suffix = &case[..case.len() - pos]; + + assert_eq!(is_ascii_baseline(suffix), suffix.is_ascii(),); + + // Both head and tail are potentially misaligned + let mid = &case[(pos / 2)..(case.len() - (pos / 2))]; + assert_eq!(is_ascii_baseline(mid), mid.is_ascii(),); + } + } + } +} + +#[test] +fn ascii_const() { + // test that the `is_ascii` methods of `char` and `u8` are usable in a const context + + const CHAR_IS_ASCII: bool = 'a'.is_ascii(); + assert!(CHAR_IS_ASCII); + + const BYTE_IS_ASCII: bool = 97u8.is_ascii(); + assert!(BYTE_IS_ASCII); +} + +#[test] +fn ascii_ctype_const() { + macro_rules! suite { + ( $( $fn:ident => [$a:ident, $A:ident, $nine:ident, $dot:ident, $space:ident]; )* ) => { + $( + mod $fn { + const CHAR_A_LOWER: bool = 'a'.$fn(); + const CHAR_A_UPPER: bool = 'A'.$fn(); + const CHAR_NINE: bool = '9'.$fn(); + const CHAR_DOT: bool = '.'.$fn(); + const CHAR_SPACE: bool = ' '.$fn(); + + const U8_A_LOWER: bool = b'a'.$fn(); + const U8_A_UPPER: bool = b'A'.$fn(); + const U8_NINE: bool = b'9'.$fn(); + const U8_DOT: bool = b'.'.$fn(); + const U8_SPACE: bool = b' '.$fn(); + + pub fn run() { + assert_eq!(CHAR_A_LOWER, $a); + assert_eq!(CHAR_A_UPPER, $A); + assert_eq!(CHAR_NINE, $nine); + assert_eq!(CHAR_DOT, $dot); + assert_eq!(CHAR_SPACE, $space); + + assert_eq!(U8_A_LOWER, $a); + assert_eq!(U8_A_UPPER, $A); + assert_eq!(U8_NINE, $nine); + assert_eq!(U8_DOT, $dot); + assert_eq!(U8_SPACE, $space); + } + } + )* + + $( $fn::run(); )* + } + } + + suite! { + // 'a' 'A' '9' '.' ' ' + is_ascii_alphabetic => [true, true, false, false, false]; + is_ascii_uppercase => [false, true, false, false, false]; + is_ascii_lowercase => [true, false, false, false, false]; + is_ascii_alphanumeric => [true, true, true, false, false]; + is_ascii_digit => [false, false, true, false, false]; + is_ascii_octdigit => [false, false, false, false, false]; + is_ascii_hexdigit => [true, true, true, false, false]; + is_ascii_punctuation => [false, false, false, true, false]; + is_ascii_graphic => [true, true, true, true, false]; + is_ascii_whitespace => [false, false, false, false, true]; + is_ascii_control => [false, false, false, false, false]; + } +} diff --git a/crux-mir/lib/core/tests/asserting.rs b/crux-mir/lib/core/tests/asserting.rs new file mode 100644 index 000000000..4b626ba6f --- /dev/null +++ b/crux-mir/lib/core/tests/asserting.rs @@ -0,0 +1,37 @@ +use core::asserting::{Capture, TryCaptureGeneric, TryCapturePrintable, Wrapper}; + +macro_rules! test { + ($test_name:ident, $elem:expr, $captured_elem:expr, $output:literal) => { + #[test] + fn $test_name() { + let elem = $elem; + let mut capture = Capture::new(); + assert!(capture.elem == None); + (&Wrapper(&elem)).try_capture(&mut capture); + assert!(capture.elem == $captured_elem); + assert_eq!(format!("{:?}", capture), $output); + } + }; +} + +#[derive(Debug, PartialEq)] +struct NoCopy; + +#[derive(PartialEq)] +struct NoCopyNoDebug; + +#[derive(Clone, Copy, PartialEq)] +struct NoDebug; + +test!( + capture_with_non_copyable_and_non_debugabble_elem_has_correct_params, + NoCopyNoDebug, + None, + "N/A" +); + +test!(capture_with_non_copyable_elem_has_correct_params, NoCopy, None, "N/A"); + +test!(capture_with_non_debugabble_elem_has_correct_params, NoDebug, None, "N/A"); + +test!(capture_with_copyable_and_debugabble_elem_has_correct_params, 1i32, Some(1i32), "1"); diff --git a/crux-mir/lib/core/tests/atomic.rs b/crux-mir/lib/core/tests/atomic.rs index acbd91398..94b031060 100644 --- a/crux-mir/lib/core/tests/atomic.rs +++ b/crux-mir/lib/core/tests/atomic.rs @@ -4,11 +4,11 @@ use core::sync::atomic::*; #[test] fn bool_() { let a = AtomicBool::new(false); - assert_eq!(a.compare_and_swap(false, true, SeqCst), false); - assert_eq!(a.compare_and_swap(false, true, SeqCst), true); + assert_eq!(a.compare_exchange(false, true, SeqCst, SeqCst), Ok(false)); + assert_eq!(a.compare_exchange(false, true, SeqCst, SeqCst), Err(true)); a.store(false, SeqCst); - assert_eq!(a.compare_and_swap(false, true, SeqCst), false); + assert_eq!(a.compare_exchange(false, true, SeqCst, SeqCst), Ok(false)); } #[test] @@ -59,6 +59,26 @@ fn uint_xor() { assert_eq!(x.load(SeqCst), 0xf731 ^ 0x137f); } +#[test] +#[cfg(any(not(target_arch = "arm"), target_os = "linux"))] // Missing intrinsic in compiler-builtins +fn uint_min() { + let x = AtomicUsize::new(0xf731); + assert_eq!(x.fetch_min(0x137f, SeqCst), 0xf731); + assert_eq!(x.load(SeqCst), 0x137f); + assert_eq!(x.fetch_min(0xf731, SeqCst), 0x137f); + assert_eq!(x.load(SeqCst), 0x137f); +} + +#[test] +#[cfg(any(not(target_arch = "arm"), target_os = "linux"))] // Missing intrinsic in compiler-builtins +fn uint_max() { + let x = AtomicUsize::new(0x137f); + assert_eq!(x.fetch_max(0xf731, SeqCst), 0x137f); + assert_eq!(x.load(SeqCst), 0xf731); + assert_eq!(x.fetch_max(0x137f, SeqCst), 0xf731); + assert_eq!(x.load(SeqCst), 0xf731); +} + #[test] fn int_and() { let x = AtomicIsize::new(0xf731); @@ -87,6 +107,111 @@ fn int_xor() { assert_eq!(x.load(SeqCst), 0xf731 ^ 0x137f); } +#[test] +#[cfg(any(not(target_arch = "arm"), target_os = "linux"))] // Missing intrinsic in compiler-builtins +fn int_min() { + let x = AtomicIsize::new(0xf731); + assert_eq!(x.fetch_min(0x137f, SeqCst), 0xf731); + assert_eq!(x.load(SeqCst), 0x137f); + assert_eq!(x.fetch_min(0xf731, SeqCst), 0x137f); + assert_eq!(x.load(SeqCst), 0x137f); +} + +#[test] +#[cfg(any(not(target_arch = "arm"), target_os = "linux"))] // Missing intrinsic in compiler-builtins +fn int_max() { + let x = AtomicIsize::new(0x137f); + assert_eq!(x.fetch_max(0xf731, SeqCst), 0x137f); + assert_eq!(x.load(SeqCst), 0xf731); + assert_eq!(x.fetch_max(0x137f, SeqCst), 0xf731); + assert_eq!(x.load(SeqCst), 0xf731); +} + +#[test] +#[cfg(any(not(target_arch = "arm"), target_os = "linux"))] // Missing intrinsic in compiler-builtins +fn ptr_add_null() { + let atom = AtomicPtr::::new(core::ptr::null_mut()); + assert_eq!(atom.fetch_ptr_add(1, SeqCst).addr(), 0); + assert_eq!(atom.load(SeqCst).addr(), 8); + + assert_eq!(atom.fetch_byte_add(1, SeqCst).addr(), 8); + assert_eq!(atom.load(SeqCst).addr(), 9); + + assert_eq!(atom.fetch_ptr_sub(1, SeqCst).addr(), 9); + assert_eq!(atom.load(SeqCst).addr(), 1); + + assert_eq!(atom.fetch_byte_sub(1, SeqCst).addr(), 1); + assert_eq!(atom.load(SeqCst).addr(), 0); +} + +#[test] +#[cfg(any(not(target_arch = "arm"), target_os = "linux"))] // Missing intrinsic in compiler-builtins +fn ptr_add_data() { + let num = 0i64; + let n = &num as *const i64 as *mut _; + let atom = AtomicPtr::::new(n); + assert_eq!(atom.fetch_ptr_add(1, SeqCst), n); + assert_eq!(atom.load(SeqCst), n.wrapping_add(1)); + + assert_eq!(atom.fetch_ptr_sub(1, SeqCst), n.wrapping_add(1)); + assert_eq!(atom.load(SeqCst), n); + let bytes_from_n = |b| n.wrapping_byte_add(b); + + assert_eq!(atom.fetch_byte_add(1, SeqCst), n); + assert_eq!(atom.load(SeqCst), bytes_from_n(1)); + + assert_eq!(atom.fetch_byte_add(5, SeqCst), bytes_from_n(1)); + assert_eq!(atom.load(SeqCst), bytes_from_n(6)); + + assert_eq!(atom.fetch_byte_sub(1, SeqCst), bytes_from_n(6)); + assert_eq!(atom.load(SeqCst), bytes_from_n(5)); + + assert_eq!(atom.fetch_byte_sub(5, SeqCst), bytes_from_n(5)); + assert_eq!(atom.load(SeqCst), n); +} + +#[test] +#[cfg(any(not(target_arch = "arm"), target_os = "linux"))] // Missing intrinsic in compiler-builtins +fn ptr_bitops() { + let atom = AtomicPtr::::new(core::ptr::null_mut()); + assert_eq!(atom.fetch_or(0b0111, SeqCst).addr(), 0); + assert_eq!(atom.load(SeqCst).addr(), 0b0111); + + assert_eq!(atom.fetch_and(0b1101, SeqCst).addr(), 0b0111); + assert_eq!(atom.load(SeqCst).addr(), 0b0101); + + assert_eq!(atom.fetch_xor(0b1111, SeqCst).addr(), 0b0101); + assert_eq!(atom.load(SeqCst).addr(), 0b1010); +} + +#[test] +#[cfg(any(not(target_arch = "arm"), target_os = "linux"))] // Missing intrinsic in compiler-builtins +fn ptr_bitops_tagging() { + #[repr(align(16))] + struct Tagme(u128); + + let tagme = Tagme(1000); + let ptr = &tagme as *const Tagme as *mut Tagme; + let atom: AtomicPtr = AtomicPtr::new(ptr); + + const MASK_TAG: usize = 0b1111; + const MASK_PTR: usize = !MASK_TAG; + + assert_eq!(ptr.addr() & MASK_TAG, 0); + + assert_eq!(atom.fetch_or(0b0111, SeqCst), ptr); + assert_eq!(atom.load(SeqCst), ptr.map_addr(|a| a | 0b111)); + + assert_eq!(atom.fetch_and(MASK_PTR | 0b0010, SeqCst), ptr.map_addr(|a| a | 0b111)); + assert_eq!(atom.load(SeqCst), ptr.map_addr(|a| a | 0b0010)); + + assert_eq!(atom.fetch_xor(0b1011, SeqCst), ptr.map_addr(|a| a | 0b0010)); + assert_eq!(atom.load(SeqCst), ptr.map_addr(|a| a | 0b1001)); + + assert_eq!(atom.fetch_and(MASK_PTR, SeqCst), ptr.map_addr(|a| a | 0b1001)); + assert_eq!(atom.load(SeqCst), ptr); +} + static S_FALSE: AtomicBool = AtomicBool::new(false); static S_TRUE: AtomicBool = AtomicBool::new(true); static S_INT: AtomicIsize = AtomicIsize::new(0); @@ -101,3 +226,89 @@ fn static_init() { assert!(S_INT.fetch_add(1, SeqCst) == 0); assert!(S_UINT.fetch_add(1, SeqCst) == 0); } + +#[test] +fn atomic_access_bool() { + static mut ATOMIC: AtomicBool = AtomicBool::new(false); + + unsafe { + assert_eq!(*ATOMIC.get_mut(), false); + ATOMIC.store(true, SeqCst); + assert_eq!(*ATOMIC.get_mut(), true); + ATOMIC.fetch_or(false, SeqCst); + assert_eq!(*ATOMIC.get_mut(), true); + ATOMIC.fetch_and(false, SeqCst); + assert_eq!(*ATOMIC.get_mut(), false); + ATOMIC.fetch_nand(true, SeqCst); + assert_eq!(*ATOMIC.get_mut(), true); + ATOMIC.fetch_xor(true, SeqCst); + assert_eq!(*ATOMIC.get_mut(), false); + } +} + +#[test] +fn atomic_alignment() { + use std::mem::{align_of, size_of}; + + #[cfg(target_has_atomic = "8")] + assert_eq!(align_of::(), size_of::()); + #[cfg(target_has_atomic = "ptr")] + assert_eq!(align_of::>(), size_of::>()); + #[cfg(target_has_atomic = "8")] + assert_eq!(align_of::(), size_of::()); + #[cfg(target_has_atomic = "8")] + assert_eq!(align_of::(), size_of::()); + #[cfg(target_has_atomic = "16")] + assert_eq!(align_of::(), size_of::()); + #[cfg(target_has_atomic = "16")] + assert_eq!(align_of::(), size_of::()); + #[cfg(target_has_atomic = "32")] + assert_eq!(align_of::(), size_of::()); + #[cfg(target_has_atomic = "32")] + assert_eq!(align_of::(), size_of::()); + #[cfg(target_has_atomic = "64")] + assert_eq!(align_of::(), size_of::()); + #[cfg(target_has_atomic = "64")] + assert_eq!(align_of::(), size_of::()); + #[cfg(target_has_atomic = "128")] + assert_eq!(align_of::(), size_of::()); + #[cfg(target_has_atomic = "128")] + assert_eq!(align_of::(), size_of::()); + #[cfg(target_has_atomic = "ptr")] + assert_eq!(align_of::(), size_of::()); + #[cfg(target_has_atomic = "ptr")] + assert_eq!(align_of::(), size_of::()); +} + +#[test] +fn atomic_compare_exchange() { + use Ordering::*; + + static ATOMIC: AtomicIsize = AtomicIsize::new(0); + + ATOMIC.compare_exchange(0, 1, Relaxed, Relaxed).ok(); + ATOMIC.compare_exchange(0, 1, Acquire, Relaxed).ok(); + ATOMIC.compare_exchange(0, 1, Release, Relaxed).ok(); + ATOMIC.compare_exchange(0, 1, AcqRel, Relaxed).ok(); + ATOMIC.compare_exchange(0, 1, SeqCst, Relaxed).ok(); + ATOMIC.compare_exchange(0, 1, Acquire, Acquire).ok(); + ATOMIC.compare_exchange(0, 1, AcqRel, Acquire).ok(); + ATOMIC.compare_exchange(0, 1, SeqCst, Acquire).ok(); + ATOMIC.compare_exchange(0, 1, SeqCst, SeqCst).ok(); + ATOMIC.compare_exchange_weak(0, 1, Relaxed, Relaxed).ok(); + ATOMIC.compare_exchange_weak(0, 1, Acquire, Relaxed).ok(); + ATOMIC.compare_exchange_weak(0, 1, Release, Relaxed).ok(); + ATOMIC.compare_exchange_weak(0, 1, AcqRel, Relaxed).ok(); + ATOMIC.compare_exchange_weak(0, 1, SeqCst, Relaxed).ok(); + ATOMIC.compare_exchange_weak(0, 1, Acquire, Acquire).ok(); + ATOMIC.compare_exchange_weak(0, 1, AcqRel, Acquire).ok(); + ATOMIC.compare_exchange_weak(0, 1, SeqCst, Acquire).ok(); + ATOMIC.compare_exchange_weak(0, 1, SeqCst, SeqCst).ok(); +} + +#[test] +fn atomic_const_from() { + const _ATOMIC_U8: AtomicU8 = AtomicU8::from(1); + const _ATOMIC_BOOL: AtomicBool = AtomicBool::from(true); + const _ATOMIC_PTR: AtomicPtr = AtomicPtr::from(core::ptr::null_mut()); +} diff --git a/crux-mir/lib/core/tests/bool.rs b/crux-mir/lib/core/tests/bool.rs index e89eb2c7f..4819ce911 100644 --- a/crux-mir/lib/core/tests/bool.rs +++ b/crux-mir/lib/core/tests/bool.rs @@ -1,7 +1,105 @@ +use core::cmp::Ordering::{Equal, Greater, Less}; +use core::ops::{BitAnd, BitOr, BitXor}; + +#[test] +fn test_bool() { + assert_eq!(false.eq(&true), false); + assert_eq!(false == false, true); + assert_eq!(false != true, true); + assert_eq!(false.ne(&false), false); + + assert_eq!(false.bitand(false), false); + assert_eq!(true.bitand(false), false); + assert_eq!(false.bitand(true), false); + assert_eq!(true.bitand(true), true); + + assert_eq!(false & false, false); + assert_eq!(true & false, false); + assert_eq!(false & true, false); + assert_eq!(true & true, true); + + assert_eq!(false.bitor(false), false); + assert_eq!(true.bitor(false), true); + assert_eq!(false.bitor(true), true); + assert_eq!(true.bitor(true), true); + + assert_eq!(false | false, false); + assert_eq!(true | false, true); + assert_eq!(false | true, true); + assert_eq!(true | true, true); + + assert_eq!(false.bitxor(false), false); + assert_eq!(true.bitxor(false), true); + assert_eq!(false.bitxor(true), true); + assert_eq!(true.bitxor(true), false); + + assert_eq!(false ^ false, false); + assert_eq!(true ^ false, true); + assert_eq!(false ^ true, true); + assert_eq!(true ^ true, false); + + assert_eq!(!true, false); + assert_eq!(!false, true); + + let s = false.to_string(); + assert_eq!(s, "false"); + let s = true.to_string(); + assert_eq!(s, "true"); + + assert!(true > false); + assert!(!(false > true)); + + assert!(false < true); + assert!(!(true < false)); + + assert!(false <= false); + assert!(false >= false); + assert!(true <= true); + assert!(true >= true); + + assert!(false <= true); + assert!(!(false >= true)); + assert!(true >= false); + assert!(!(true <= false)); + + assert_eq!(true.cmp(&true), Equal); + assert_eq!(false.cmp(&false), Equal); + assert_eq!(true.cmp(&false), Greater); + assert_eq!(false.cmp(&true), Less); +} + +#[test] +pub fn test_bool_not() { + if !false { + assert!((true)); + } else { + assert!((false)); + } + if !true { + assert!((false)); + } else { + assert!((true)); + } +} + #[test] fn test_bool_to_option() { assert_eq!(false.then_some(0), None); assert_eq!(true.then_some(0), Some(0)); assert_eq!(false.then(|| 0), None); assert_eq!(true.then(|| 0), Some(0)); + + const fn zero() -> i32 { + 0 + } + + const A: Option = false.then_some(0); + const B: Option = true.then_some(0); + const C: Option = false.then(zero); + const D: Option = true.then(zero); + + assert_eq!(A, None); + assert_eq!(B, Some(0)); + assert_eq!(C, None); + assert_eq!(D, Some(0)); } diff --git a/crux-mir/lib/core/tests/cell.rs b/crux-mir/lib/core/tests/cell.rs index 801b60be0..7b77b2134 100644 --- a/crux-mir/lib/core/tests/cell.rs +++ b/crux-mir/lib/core/tests/cell.rs @@ -2,6 +2,38 @@ use core::cell::*; use core::default::Default; use std::mem::drop; +#[test] +fn smoketest_unsafe_cell() { + let mut x = UnsafeCell::new(10); + let ref_mut = &mut x; + unsafe { + // The asserts are repeated in order to ensure that `get()` + // is non-mutating. + assert_eq!(*ref_mut.get(), 10); + assert_eq!(*ref_mut.get(), 10); + *ref_mut.get_mut() += 5; + assert_eq!(*ref_mut.get(), 15); + assert_eq!(*ref_mut.get(), 15); + assert_eq!(x.into_inner(), 15); + } +} + +#[test] +fn unsafe_cell_raw_get() { + let x = UnsafeCell::new(10); + let ptr = &x as *const UnsafeCell; + unsafe { + // The asserts are repeated in order to ensure that `raw_get()` + // is non-mutating. + assert_eq!(*UnsafeCell::raw_get(ptr), 10); + assert_eq!(*UnsafeCell::raw_get(ptr), 10); + *UnsafeCell::raw_get(ptr) += 5; + assert_eq!(*UnsafeCell::raw_get(ptr), 15); + assert_eq!(*UnsafeCell::raw_get(ptr), 15); + assert_eq!(x.into_inner(), 15); + } +} + #[test] fn smoketest_cell() { let x = Cell::new(10); @@ -30,10 +62,10 @@ fn cell_update() { #[test] fn cell_has_sensible_show() { let x = Cell::new("foo bar"); - assert!(format!("{:?}", x).contains(x.get())); + assert!(format!("{x:?}").contains(x.get())); x.set("baz qux"); - assert!(format!("{:?}", x).contains(x.get())); + assert!(format!("{x:?}").contains(x.get())); } #[test] @@ -41,11 +73,13 @@ fn ref_and_refmut_have_sensible_show() { let refcell = RefCell::new("foo"); let refcell_refmut = refcell.borrow_mut(); - assert!(format!("{:?}", refcell_refmut).contains("foo")); + assert_eq!(format!("{refcell_refmut}"), "foo"); // Display + assert!(format!("{refcell_refmut:?}").contains("foo")); // Debug drop(refcell_refmut); let refcell_ref = refcell.borrow(); - assert!(format!("{:?}", refcell_ref).contains("foo")); + assert_eq!(format!("{refcell_ref}"), "foo"); // Display + assert!(format!("{refcell_ref:?}").contains("foo")); // Debug drop(refcell_ref); } @@ -303,6 +337,53 @@ fn cell_into_inner() { assert_eq!("Hello world".to_owned(), cell.into_inner()); } +#[test] +fn cell_exterior() { + #[derive(Copy, Clone)] + #[allow(dead_code)] + struct Point { + x: isize, + y: isize, + z: isize, + } + + fn f(p: &Cell) { + assert_eq!(p.get().z, 12); + p.set(Point { x: 10, y: 11, z: 13 }); + assert_eq!(p.get().z, 13); + } + + let a = Point { x: 10, y: 11, z: 12 }; + let b = &Cell::new(a); + assert_eq!(b.get().z, 12); + f(b); + assert_eq!(a.z, 12); + assert_eq!(b.get().z, 13); +} + +#[test] +fn cell_does_not_clone() { + #[derive(Copy)] + #[allow(dead_code)] + struct Foo { + x: isize, + } + + impl Clone for Foo { + fn clone(&self) -> Foo { + // Using Cell in any way should never cause clone() to be + // invoked -- after all, that would permit evil user code to + // abuse `Cell` and trigger crashes. + + panic!(); + } + } + + let x = Cell::new(Foo { x: 22 }); + let _y = x.get(); + let _z = x.clone(); +} + #[test] fn refcell_default() { let cell: RefCell = Default::default(); @@ -367,3 +448,32 @@ fn refcell_replace_borrows() { let _b = x.borrow(); x.replace(1); } + +#[test] +fn refcell_format() { + let name = RefCell::new("rust"); + let what = RefCell::new("rocks"); + let msg = format!("{name} {}", &*what.borrow(), name = &*name.borrow()); + assert_eq!(msg, "rust rocks".to_string()); +} + +#[allow(dead_code)] +fn const_cells() { + const UNSAFE_CELL: UnsafeCell = UnsafeCell::new(3); + const _: i32 = UNSAFE_CELL.into_inner(); + + const REF_CELL: RefCell = RefCell::new(3); + const _: i32 = REF_CELL.into_inner(); + + const CELL: Cell = Cell::new(3); + const _: i32 = CELL.into_inner(); + + const UNSAFE_CELL_FROM: UnsafeCell = UnsafeCell::from(3); + const _: i32 = UNSAFE_CELL.into_inner(); + + const REF_CELL_FROM: RefCell = RefCell::from(3); + const _: i32 = REF_CELL.into_inner(); + + const CELL_FROM: Cell = Cell::from(3); + const _: i32 = CELL.into_inner(); +} diff --git a/crux-mir/lib/core/tests/char.rs b/crux-mir/lib/core/tests/char.rs index c16f54081..ac0b2ca16 100644 --- a/crux-mir/lib/core/tests/char.rs +++ b/crux-mir/lib/core/tests/char.rs @@ -5,6 +5,8 @@ use std::{char, str}; #[test] fn test_convert() { assert_eq!(u32::from('a'), 0x61); + assert_eq!(u64::from('b'), 0x62); + assert_eq!(u128::from('c'), 0x63); assert_eq!(char::from(b'\0'), '\0'); assert_eq!(char::from(b'a'), 'a'); assert_eq!(char::from(b'\xFF'), '\u{FF}'); @@ -19,6 +21,16 @@ fn test_convert() { assert!(char::try_from(0xFFFF_FFFF_u32).is_err()); } +#[test] +const fn test_convert_const() { + assert!(u32::from('a') == 0x61); + assert!(u64::from('b') == 0x62); + assert!(u128::from('c') == 0x63); + assert!(char::from(b'\0') == '\0'); + assert!(char::from(b'a') == 'a'); + assert!(char::from(b'\xFF') == '\u{FF}'); +} + #[test] fn test_from_str() { assert_eq!(char::from_str("a").unwrap(), 'a'); @@ -67,10 +79,20 @@ fn test_to_digit() { assert_eq!('A'.to_digit(16), Some(10)); assert_eq!('b'.to_digit(16), Some(11)); assert_eq!('B'.to_digit(16), Some(11)); + assert_eq!('A'.to_digit(36), Some(10)); assert_eq!('z'.to_digit(36), Some(35)); assert_eq!('Z'.to_digit(36), Some(35)); - assert_eq!(' '.to_digit(10), None); + assert_eq!('['.to_digit(36), None); + assert_eq!('`'.to_digit(36), None); + assert_eq!('{'.to_digit(36), None); assert_eq!('$'.to_digit(36), None); + assert_eq!('@'.to_digit(16), None); + assert_eq!('G'.to_digit(16), None); + assert_eq!('g'.to_digit(16), None); + assert_eq!(' '.to_digit(10), None); + assert_eq!('/'.to_digit(10), None); + assert_eq!(':'.to_digit(10), None); + assert_eq!(':'.to_digit(11), None); } #[test] @@ -81,6 +103,9 @@ fn test_to_lowercase() { let iter: String = c.to_lowercase().collect(); let disp: String = c.to_lowercase().to_string(); assert_eq!(iter, disp); + let iter_rev: String = c.to_lowercase().rev().collect(); + let disp_rev: String = disp.chars().rev().collect(); + assert_eq!(iter_rev, disp_rev); iter } assert_eq!(lower('A'), "a"); @@ -108,6 +133,9 @@ fn test_to_uppercase() { let iter: String = c.to_uppercase().collect(); let disp: String = c.to_uppercase().to_string(); assert_eq!(iter, disp); + let iter_rev: String = c.to_uppercase().rev().collect(); + let disp_rev: String = disp.chars().rev().collect(); + assert_eq!(iter_rev, disp_rev); iter } assert_eq!(upper('a'), "A"); @@ -169,7 +197,7 @@ fn test_escape_debug() { assert_eq!(string('~'), "~"); assert_eq!(string('é'), "é"); assert_eq!(string('文'), "文"); - assert_eq!(string('\x00'), "\\u{0}"); + assert_eq!(string('\x00'), "\\0"); assert_eq!(string('\x1f'), "\\u{1f}"); assert_eq!(string('\x7f'), "\\u{7f}"); assert_eq!(string('\u{80}'), "\\u{80}"); @@ -278,6 +306,37 @@ fn test_decode_utf16() { } check(&[0xD800, 0x41, 0x42], &[Err(0xD800), Ok('A'), Ok('B')]); check(&[0xD800, 0], &[Err(0xD800), Ok('\0')]); + check(&[0xD800], &[Err(0xD800)]); + check(&[0xD840, 0xDC00], &[Ok('\u{20000}')]); + check(&[0xD840, 0xD840, 0xDC00], &[Err(0xD840), Ok('\u{20000}')]); + check(&[0xDC00, 0xD840], &[Err(0xDC00), Err(0xD840)]); +} + +#[test] +fn test_decode_utf16_size_hint() { + fn check(s: &[u16]) { + let mut iter = char::decode_utf16(s.iter().cloned()); + + loop { + let count = iter.clone().count(); + let (lower, upper) = iter.size_hint(); + + assert!( + lower <= count && count <= upper.unwrap(), + "lower = {lower}, count = {count}, upper = {upper:?}" + ); + + if let None = iter.next() { + break; + } + } + } + + check(&[0xD800, 0xD800, 0xDC00]); + check(&[0xD800, 0xD800, 0x0]); + check(&[0xD800, 0x41, 0x42]); + check(&[0xD800, 0]); + check(&[0xD834, 0x006d]); } #[test] diff --git a/crux-mir/lib/core/tests/clone.rs b/crux-mir/lib/core/tests/clone.rs index c97a87aeb..33ca9f2c6 100644 --- a/crux-mir/lib/core/tests/clone.rs +++ b/crux-mir/lib/core/tests/clone.rs @@ -8,8 +8,8 @@ fn test_borrowed_clone() { #[test] fn test_clone_from() { - let a = box 5; - let mut b = box 10; + let a = Box::new(5); + let mut b = Box::new(10); b.clone_from(&a); assert_eq!(*b, 5); } diff --git a/crux-mir/lib/core/tests/cmp.rs b/crux-mir/lib/core/tests/cmp.rs index 408691778..8d0e59d5a 100644 --- a/crux-mir/lib/core/tests/cmp.rs +++ b/crux-mir/lib/core/tests/cmp.rs @@ -1,4 +1,7 @@ -use core::cmp::{self, Ordering::*}; +use core::cmp::{ + self, + Ordering::{self, *}, +}; #[test] fn test_int_totalord() { @@ -116,3 +119,132 @@ fn test_user_defined_eq() { assert!(SketchyNum { num: 37 } == SketchyNum { num: 34 }); assert!(SketchyNum { num: 25 } != SketchyNum { num: 57 }); } + +#[test] +fn ordering_const() { + // test that the methods of `Ordering` are usable in a const context + + const ORDERING: Ordering = Greater; + + const REVERSE: Ordering = ORDERING.reverse(); + assert_eq!(REVERSE, Less); + + const THEN: Ordering = Equal.then(ORDERING); + assert_eq!(THEN, Greater); +} + +#[test] +fn ordering_structural_eq() { + // test that consts of type `Ordering` are usable in patterns + + const ORDERING: Ordering = Greater; + + const REVERSE: Ordering = ORDERING.reverse(); + match Ordering::Less { + REVERSE => {} + _ => unreachable!(), + }; +} + +#[test] +fn cmp_default() { + // Test default methods in PartialOrd and PartialEq + + #[derive(Debug)] + struct Fool(bool); + + impl PartialEq for Fool { + fn eq(&self, other: &Fool) -> bool { + let Fool(this) = *self; + let Fool(other) = *other; + this != other + } + } + + struct Int(isize); + + impl PartialEq for Int { + fn eq(&self, other: &Int) -> bool { + let Int(this) = *self; + let Int(other) = *other; + this == other + } + } + + impl PartialOrd for Int { + fn partial_cmp(&self, other: &Int) -> Option { + let Int(this) = *self; + let Int(other) = *other; + this.partial_cmp(&other) + } + } + + struct RevInt(isize); + + impl PartialEq for RevInt { + fn eq(&self, other: &RevInt) -> bool { + let RevInt(this) = *self; + let RevInt(other) = *other; + this == other + } + } + + impl PartialOrd for RevInt { + fn partial_cmp(&self, other: &RevInt) -> Option { + let RevInt(this) = *self; + let RevInt(other) = *other; + other.partial_cmp(&this) + } + } + + assert!(Int(2) > Int(1)); + assert!(Int(2) >= Int(1)); + assert!(Int(1) >= Int(1)); + assert!(Int(1) < Int(2)); + assert!(Int(1) <= Int(2)); + assert!(Int(1) <= Int(1)); + + assert!(RevInt(2) < RevInt(1)); + assert!(RevInt(2) <= RevInt(1)); + assert!(RevInt(1) <= RevInt(1)); + assert!(RevInt(1) > RevInt(2)); + assert!(RevInt(1) >= RevInt(2)); + assert!(RevInt(1) >= RevInt(1)); + + assert_eq!(Fool(true), Fool(false)); + assert!(Fool(true) != Fool(true)); + assert!(Fool(false) != Fool(false)); + assert_eq!(Fool(false), Fool(true)); +} + +mod const_cmp { + use super::*; + + struct S(i32); + + impl const PartialEq for S { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } + } + + impl const PartialOrd for S { + fn partial_cmp(&self, other: &Self) -> Option { + let ret = match (self.0, other.0) { + (a, b) if a > b => Ordering::Greater, + (a, b) if a < b => Ordering::Less, + _ => Ordering::Equal, + }; + + Some(ret) + } + } + + const _: () = assert!(S(1) == S(1)); + const _: () = assert!(S(0) != S(1)); + + const _: () = assert!(S(1) <= S(1)); + const _: () = assert!(S(1) >= S(1)); + const _: () = assert!(S(0) < S(1)); + const _: () = assert!(S(1) > S(0)); +} diff --git a/crux-mir/lib/core/tests/const_ptr.rs b/crux-mir/lib/core/tests/const_ptr.rs new file mode 100644 index 000000000..d874f0831 --- /dev/null +++ b/crux-mir/lib/core/tests/const_ptr.rs @@ -0,0 +1,101 @@ +// Aligned to two bytes +const DATA: [u16; 2] = [u16::from_ne_bytes([0x01, 0x23]), u16::from_ne_bytes([0x45, 0x67])]; + +const fn unaligned_ptr() -> *const u16 { + // Since DATA.as_ptr() is aligned to two bytes, adding 1 byte to that produces an unaligned *const u16 + unsafe { DATA.as_ptr().byte_add(1) } +} + +#[test] +fn read() { + use core::ptr; + + const FOO: i32 = unsafe { ptr::read(&42 as *const i32) }; + assert_eq!(FOO, 42); + + const ALIGNED: i32 = unsafe { ptr::read_unaligned(&42 as *const i32) }; + assert_eq!(ALIGNED, 42); + + const UNALIGNED_PTR: *const u16 = unaligned_ptr(); + + const UNALIGNED: u16 = unsafe { ptr::read_unaligned(UNALIGNED_PTR) }; + assert_eq!(UNALIGNED, u16::from_ne_bytes([0x23, 0x45])); +} + +#[test] +fn const_ptr_read() { + const FOO: i32 = unsafe { (&42 as *const i32).read() }; + assert_eq!(FOO, 42); + + const ALIGNED: i32 = unsafe { (&42 as *const i32).read_unaligned() }; + assert_eq!(ALIGNED, 42); + + const UNALIGNED_PTR: *const u16 = unaligned_ptr(); + + const UNALIGNED: u16 = unsafe { UNALIGNED_PTR.read_unaligned() }; + assert_eq!(UNALIGNED, u16::from_ne_bytes([0x23, 0x45])); +} + +#[test] +fn mut_ptr_read() { + const FOO: i32 = unsafe { (&42 as *const i32 as *mut i32).read() }; + assert_eq!(FOO, 42); + + const ALIGNED: i32 = unsafe { (&42 as *const i32 as *mut i32).read_unaligned() }; + assert_eq!(ALIGNED, 42); + + const UNALIGNED_PTR: *mut u16 = unaligned_ptr() as *mut u16; + + const UNALIGNED: u16 = unsafe { UNALIGNED_PTR.read_unaligned() }; + assert_eq!(UNALIGNED, u16::from_ne_bytes([0x23, 0x45])); +} + +#[test] +fn write() { + use core::ptr; + + const fn write_aligned() -> i32 { + let mut res = 0; + unsafe { + ptr::write(&mut res as *mut _, 42); + } + res + } + const ALIGNED: i32 = write_aligned(); + assert_eq!(ALIGNED, 42); + + const fn write_unaligned() -> [u16; 2] { + let mut two_aligned = [0u16; 2]; + unsafe { + let unaligned_ptr = two_aligned.as_mut_ptr().byte_add(1); + ptr::write_unaligned(unaligned_ptr, u16::from_ne_bytes([0x23, 0x45])); + } + two_aligned + } + const UNALIGNED: [u16; 2] = write_unaligned(); + assert_eq!(UNALIGNED, [u16::from_ne_bytes([0x00, 0x23]), u16::from_ne_bytes([0x45, 0x00])]); +} + +#[test] +fn mut_ptr_write() { + const fn aligned() -> i32 { + let mut res = 0; + unsafe { + (&mut res as *mut i32).write(42); + } + res + } + const ALIGNED: i32 = aligned(); + assert_eq!(ALIGNED, 42); + + const fn write_unaligned() -> [u16; 2] { + let mut two_aligned = [0u16; 2]; + unsafe { + let unaligned_ptr = two_aligned.as_mut_ptr().byte_add(1); + unaligned_ptr.write_unaligned(u16::from_ne_bytes([0x23, 0x45])); + } + two_aligned + } + const UNALIGNED: [u16; 2] = write_unaligned(); + assert_eq!(UNALIGNED, [u16::from_ne_bytes([0x00, 0x23]), u16::from_ne_bytes([0x45, 0x00])]); +} diff --git a/crux-mir/lib/core/tests/convert.rs b/crux-mir/lib/core/tests/convert.rs new file mode 100644 index 000000000..f1048f4cf --- /dev/null +++ b/crux-mir/lib/core/tests/convert.rs @@ -0,0 +1,16 @@ +#[test] +fn convert() { + const fn from(x: i32) -> i32 { + i32::from(x) + } + + const FOO: i32 = from(42); + assert_eq!(FOO, 42); + + const fn into(x: Vec) -> Vec { + x.into() + } + + const BAR: Vec = into(Vec::new()); + assert_eq!(BAR, Vec::::new()); +} diff --git a/crux-mir/lib/core/tests/fmt/builders.rs b/crux-mir/lib/core/tests/fmt/builders.rs index 129c121e8..487ce46be 100644 --- a/crux-mir/lib/core/tests/fmt/builders.rs +++ b/crux-mir/lib/core/tests/fmt/builders.rs @@ -11,8 +11,8 @@ mod debug_struct { } } - assert_eq!("Foo", format!("{:?}", Foo)); - assert_eq!("Foo", format!("{:#?}", Foo)); + assert_eq!("Foo", format!("{Foo:?}")); + assert_eq!("Foo", format!("{Foo:#?}")); } #[test] @@ -25,12 +25,12 @@ mod debug_struct { } } - assert_eq!("Foo { bar: true }", format!("{:?}", Foo)); + assert_eq!("Foo { bar: true }", format!("{Foo:?}")); assert_eq!( "Foo { bar: true, }", - format!("{:#?}", Foo) + format!("{Foo:#?}") ); } @@ -47,13 +47,13 @@ mod debug_struct { } } - assert_eq!("Foo { bar: true, baz: 10/20 }", format!("{:?}", Foo)); + assert_eq!("Foo { bar: true, baz: 10/20 }", format!("{Foo:?}")); assert_eq!( "Foo { bar: true, baz: 10/20, }", - format!("{:#?}", Foo) + format!("{Foo:#?}") ); } @@ -80,7 +80,7 @@ mod debug_struct { assert_eq!( "Bar { foo: Foo { bar: true, baz: 10/20 }, hello: \"world\" }", - format!("{:?}", Bar) + format!("{Bar:?}") ); assert_eq!( "Bar { @@ -90,7 +90,7 @@ mod debug_struct { }, hello: \"world\", }", - format!("{:#?}", Bar) + format!("{Bar:#?}") ); } @@ -104,13 +104,8 @@ mod debug_struct { } } - assert_eq!("Foo { .. }", format!("{:?}", Foo)); - assert_eq!( - "Foo { - .. -}", - format!("{:#?}", Foo) - ); + assert_eq!("Foo { .. }", format!("{Foo:?}")); + assert_eq!("Foo { .. }", format!("{Foo:#?}")); } #[test] @@ -126,14 +121,14 @@ mod debug_struct { } } - assert_eq!("Foo { bar: true, baz: 10/20, .. }", format!("{:?}", Foo)); + assert_eq!("Foo { bar: true, baz: 10/20, .. }", format!("{Foo:?}")); assert_eq!( "Foo { bar: true, baz: 10/20, .. }", - format!("{:#?}", Foo) + format!("{Foo:#?}") ); } @@ -163,7 +158,7 @@ mod debug_struct { assert_eq!( "Bar { foo: Foo { bar: true, baz: 10/20, .. }, hello: \"world\", .. }", - format!("{:?}", Bar) + format!("{Bar:?}") ); assert_eq!( "Bar { @@ -175,7 +170,7 @@ mod debug_struct { hello: \"world\", .. }", - format!("{:#?}", Bar) + format!("{Bar:#?}") ); } } @@ -193,8 +188,8 @@ mod debug_tuple { } } - assert_eq!("Foo", format!("{:?}", Foo)); - assert_eq!("Foo", format!("{:#?}", Foo)); + assert_eq!("Foo", format!("{Foo:?}")); + assert_eq!("Foo", format!("{Foo:#?}")); } #[test] @@ -207,12 +202,12 @@ mod debug_tuple { } } - assert_eq!("Foo(true)", format!("{:?}", Foo)); + assert_eq!("Foo(true)", format!("{Foo:?}")); assert_eq!( "Foo( true, )", - format!("{:#?}", Foo) + format!("{Foo:#?}") ); } @@ -226,13 +221,13 @@ mod debug_tuple { } } - assert_eq!("Foo(true, 10/20)", format!("{:?}", Foo)); + assert_eq!("Foo(true, 10/20)", format!("{Foo:?}")); assert_eq!( "Foo( true, 10/20, )", - format!("{:#?}", Foo) + format!("{Foo:#?}") ); } @@ -254,7 +249,7 @@ mod debug_tuple { } } - assert_eq!("Bar(Foo(true, 10/20), \"world\")", format!("{:?}", Bar)); + assert_eq!("Bar(Foo(true, 10/20), \"world\")", format!("{Bar:?}")); assert_eq!( "Bar( Foo( @@ -263,7 +258,7 @@ mod debug_tuple { ), \"world\", )", - format!("{:#?}", Bar) + format!("{Bar:#?}") ); } } @@ -281,8 +276,8 @@ mod debug_map { } } - assert_eq!("{}", format!("{:?}", Foo)); - assert_eq!("{}", format!("{:#?}", Foo)); + assert_eq!("{}", format!("{Foo:?}")); + assert_eq!("{}", format!("{Foo:#?}")); } #[test] @@ -303,15 +298,15 @@ mod debug_map { } } - assert_eq!(format!("{:?}", Entry), format!("{:?}", KeyValue)); - assert_eq!(format!("{:#?}", Entry), format!("{:#?}", KeyValue)); + assert_eq!(format!("{Entry:?}"), format!("{KeyValue:?}")); + assert_eq!(format!("{Entry:#?}"), format!("{KeyValue:#?}")); - assert_eq!("{\"bar\": true}", format!("{:?}", Entry)); + assert_eq!("{\"bar\": true}", format!("{Entry:?}")); assert_eq!( "{ \"bar\": true, }", - format!("{:#?}", Entry) + format!("{Entry:#?}") ); } @@ -341,16 +336,16 @@ mod debug_map { } } - assert_eq!(format!("{:?}", Entry), format!("{:?}", KeyValue)); - assert_eq!(format!("{:#?}", Entry), format!("{:#?}", KeyValue)); + assert_eq!(format!("{Entry:?}"), format!("{KeyValue:?}")); + assert_eq!(format!("{Entry:#?}"), format!("{KeyValue:#?}")); - assert_eq!("{\"bar\": true, 10: 10/20}", format!("{:?}", Entry)); + assert_eq!("{\"bar\": true, 10: 10/20}", format!("{Entry:?}")); assert_eq!( "{ \"bar\": true, 10: 10/20, }", - format!("{:#?}", Entry) + format!("{Entry:#?}") ); } @@ -378,7 +373,7 @@ mod debug_map { assert_eq!( "{\"foo\": {\"bar\": true, 10: 10/20}, \ {\"bar\": true, 10: 10/20}: \"world\"}", - format!("{:?}", Bar) + format!("{Bar:?}") ); assert_eq!( "{ @@ -391,7 +386,7 @@ mod debug_map { 10: 10/20, }: \"world\", }", - format!("{:#?}", Bar) + format!("{Bar:#?}") ); } @@ -446,7 +441,7 @@ mod debug_map { } } - format!("{:?}", Foo); + format!("{Foo:?}"); } #[test] @@ -460,7 +455,7 @@ mod debug_map { } } - format!("{:?}", Foo); + format!("{Foo:?}"); } #[test] @@ -474,7 +469,7 @@ mod debug_map { } } - format!("{:?}", Foo); + format!("{Foo:?}"); } } @@ -491,8 +486,8 @@ mod debug_set { } } - assert_eq!("{}", format!("{:?}", Foo)); - assert_eq!("{}", format!("{:#?}", Foo)); + assert_eq!("{}", format!("{Foo:?}")); + assert_eq!("{}", format!("{Foo:#?}")); } #[test] @@ -505,12 +500,12 @@ mod debug_set { } } - assert_eq!("{true}", format!("{:?}", Foo)); + assert_eq!("{true}", format!("{Foo:?}")); assert_eq!( "{ true, }", - format!("{:#?}", Foo) + format!("{Foo:#?}") ); } @@ -524,13 +519,13 @@ mod debug_set { } } - assert_eq!("{true, 10/20}", format!("{:?}", Foo)); + assert_eq!("{true, 10/20}", format!("{Foo:?}")); assert_eq!( "{ true, 10/20, }", - format!("{:#?}", Foo) + format!("{Foo:#?}") ); } @@ -552,7 +547,7 @@ mod debug_set { } } - assert_eq!("{{true, 10/20}, \"world\"}", format!("{:?}", Bar)); + assert_eq!("{{true, 10/20}, \"world\"}", format!("{Bar:?}")); assert_eq!( "{ { @@ -561,7 +556,7 @@ mod debug_set { }, \"world\", }", - format!("{:#?}", Bar) + format!("{Bar:#?}") ); } } @@ -579,8 +574,8 @@ mod debug_list { } } - assert_eq!("[]", format!("{:?}", Foo)); - assert_eq!("[]", format!("{:#?}", Foo)); + assert_eq!("[]", format!("{Foo:?}")); + assert_eq!("[]", format!("{Foo:#?}")); } #[test] @@ -593,12 +588,12 @@ mod debug_list { } } - assert_eq!("[true]", format!("{:?}", Foo)); + assert_eq!("[true]", format!("{Foo:?}")); assert_eq!( "[ true, ]", - format!("{:#?}", Foo) + format!("{Foo:#?}") ); } @@ -612,13 +607,13 @@ mod debug_list { } } - assert_eq!("[true, 10/20]", format!("{:?}", Foo)); + assert_eq!("[true, 10/20]", format!("{Foo:?}")); assert_eq!( "[ true, 10/20, ]", - format!("{:#?}", Foo) + format!("{Foo:#?}") ); } @@ -640,7 +635,7 @@ mod debug_list { } } - assert_eq!("[[true, 10/20], \"world\"]", format!("{:?}", Bar)); + assert_eq!("[[true, 10/20], \"world\"]", format!("{Bar:?}")); assert_eq!( "[ [ @@ -649,7 +644,7 @@ mod debug_list { ], \"world\", ]", - format!("{:#?}", Bar) + format!("{Bar:#?}") ); } } @@ -658,6 +653,7 @@ mod debug_list { fn test_formatting_parameters_are_forwarded() { use std::collections::{BTreeMap, BTreeSet}; #[derive(Debug)] + #[allow(dead_code)] struct Foo { bar: u32, baz: u32, @@ -672,13 +668,13 @@ fn test_formatting_parameters_are_forwarded() { set.insert(1024); set.insert(7); - assert_eq!(format!("{:03?}", struct_), "Foo { bar: 1024, baz: 007 }"); - assert_eq!(format!("{:03?}", tuple), "(1024, 007)"); - assert_eq!(format!("{:03?}", list), "[1024, 007]"); - assert_eq!(format!("{:03?}", map), r#"{"bar": 1024, "baz": 007}"#); - assert_eq!(format!("{:03?}", set), "{007, 1024}"); + assert_eq!(format!("{struct_:03?}"), "Foo { bar: 1024, baz: 007 }"); + assert_eq!(format!("{tuple:03?}"), "(1024, 007)"); + assert_eq!(format!("{list:03?}"), "[1024, 007]"); + assert_eq!(format!("{map:03?}"), r#"{"bar": 1024, "baz": 007}"#); + assert_eq!(format!("{set:03?}"), "{007, 1024}"); assert_eq!( - format!("{:#03?}", struct_), + format!("{struct_:#03?}"), " Foo { bar: 1024, @@ -688,7 +684,7 @@ Foo { .trim() ); assert_eq!( - format!("{:#03?}", tuple), + format!("{tuple:#03?}"), " ( 1024, @@ -698,7 +694,7 @@ Foo { .trim() ); assert_eq!( - format!("{:#03?}", list), + format!("{list:#03?}"), " [ 1024, @@ -708,7 +704,7 @@ Foo { .trim() ); assert_eq!( - format!("{:#03?}", map), + format!("{map:#03?}"), r#" { "bar": 1024, @@ -718,7 +714,7 @@ Foo { .trim() ); assert_eq!( - format!("{:#03?}", set), + format!("{set:#03?}"), " { 007, diff --git a/crux-mir/lib/core/tests/fmt/float.rs b/crux-mir/lib/core/tests/fmt/float.rs index bd0daf7a8..003782f34 100644 --- a/crux-mir/lib/core/tests/fmt/float.rs +++ b/crux-mir/lib/core/tests/fmt/float.rs @@ -5,13 +5,83 @@ fn test_format_f64() { assert_eq!("10", format!("{:.0}", 9.9f64)); assert_eq!("9.8", format!("{:.1}", 9.849f64)); assert_eq!("9.9", format!("{:.1}", 9.851f64)); - assert_eq!("1", format!("{:.0}", 0.5f64)); + assert_eq!("0", format!("{:.0}", 0.5f64)); assert_eq!("1.23456789e6", format!("{:e}", 1234567.89f64)); assert_eq!("1.23456789e3", format!("{:e}", 1234.56789f64)); assert_eq!("1.23456789E6", format!("{:E}", 1234567.89f64)); assert_eq!("1.23456789E3", format!("{:E}", 1234.56789f64)); assert_eq!("0.0", format!("{:?}", 0.0f64)); assert_eq!("1.01", format!("{:?}", 1.01f64)); + + let high_cutoff = 1e16_f64; + assert_eq!("1e16", format!("{:?}", high_cutoff)); + assert_eq!("-1e16", format!("{:?}", -high_cutoff)); + assert!(!is_exponential(&format!("{:?}", high_cutoff * (1.0 - 2.0 * f64::EPSILON)))); + assert_eq!("-3.0", format!("{:?}", -3f64)); + assert_eq!("0.0001", format!("{:?}", 0.0001f64)); + assert_eq!("9e-5", format!("{:?}", 0.00009f64)); + assert_eq!("1234567.9", format!("{:.1?}", 1234567.89f64)); + assert_eq!("1234.6", format!("{:.1?}", 1234.56789f64)); +} + +#[test] +fn test_format_f64_rounds_ties_to_even() { + assert_eq!("0", format!("{:.0}", 0.5f64)); + assert_eq!("2", format!("{:.0}", 1.5f64)); + assert_eq!("2", format!("{:.0}", 2.5f64)); + assert_eq!("4", format!("{:.0}", 3.5f64)); + assert_eq!("4", format!("{:.0}", 4.5f64)); + assert_eq!("6", format!("{:.0}", 5.5f64)); + assert_eq!("128", format!("{:.0}", 127.5f64)); + assert_eq!("128", format!("{:.0}", 128.5f64)); + assert_eq!("0.2", format!("{:.1}", 0.25f64)); + assert_eq!("0.8", format!("{:.1}", 0.75f64)); + assert_eq!("0.12", format!("{:.2}", 0.125f64)); + assert_eq!("0.88", format!("{:.2}", 0.875f64)); + assert_eq!("0.062", format!("{:.3}", 0.062f64)); + assert_eq!("-0", format!("{:.0}", -0.5f64)); + assert_eq!("-2", format!("{:.0}", -1.5f64)); + assert_eq!("-2", format!("{:.0}", -2.5f64)); + assert_eq!("-4", format!("{:.0}", -3.5f64)); + assert_eq!("-4", format!("{:.0}", -4.5f64)); + assert_eq!("-6", format!("{:.0}", -5.5f64)); + assert_eq!("-128", format!("{:.0}", -127.5f64)); + assert_eq!("-128", format!("{:.0}", -128.5f64)); + assert_eq!("-0.2", format!("{:.1}", -0.25f64)); + assert_eq!("-0.8", format!("{:.1}", -0.75f64)); + assert_eq!("-0.12", format!("{:.2}", -0.125f64)); + assert_eq!("-0.88", format!("{:.2}", -0.875f64)); + assert_eq!("-0.062", format!("{:.3}", -0.062f64)); + + assert_eq!("2e0", format!("{:.0e}", 1.5f64)); + assert_eq!("2e0", format!("{:.0e}", 2.5f64)); + assert_eq!("4e0", format!("{:.0e}", 3.5f64)); + assert_eq!("4e0", format!("{:.0e}", 4.5f64)); + assert_eq!("6e0", format!("{:.0e}", 5.5f64)); + assert_eq!("1.28e2", format!("{:.2e}", 127.5f64)); + assert_eq!("1.28e2", format!("{:.2e}", 128.5f64)); + assert_eq!("-2e0", format!("{:.0e}", -1.5f64)); + assert_eq!("-2e0", format!("{:.0e}", -2.5f64)); + assert_eq!("-4e0", format!("{:.0e}", -3.5f64)); + assert_eq!("-4e0", format!("{:.0e}", -4.5f64)); + assert_eq!("-6e0", format!("{:.0e}", -5.5f64)); + assert_eq!("-1.28e2", format!("{:.2e}", -127.5f64)); + assert_eq!("-1.28e2", format!("{:.2e}", -128.5f64)); + + assert_eq!("2E0", format!("{:.0E}", 1.5f64)); + assert_eq!("2E0", format!("{:.0E}", 2.5f64)); + assert_eq!("4E0", format!("{:.0E}", 3.5f64)); + assert_eq!("4E0", format!("{:.0E}", 4.5f64)); + assert_eq!("6E0", format!("{:.0E}", 5.5f64)); + assert_eq!("1.28E2", format!("{:.2E}", 127.5f64)); + assert_eq!("1.28E2", format!("{:.2E}", 128.5f64)); + assert_eq!("-2E0", format!("{:.0E}", -1.5f64)); + assert_eq!("-2E0", format!("{:.0E}", -2.5f64)); + assert_eq!("-4E0", format!("{:.0E}", -3.5f64)); + assert_eq!("-4E0", format!("{:.0E}", -4.5f64)); + assert_eq!("-6E0", format!("{:.0E}", -5.5f64)); + assert_eq!("-1.28E2", format!("{:.2E}", -127.5f64)); + assert_eq!("-1.28E2", format!("{:.2E}", -128.5f64)); } #[test] @@ -21,11 +91,85 @@ fn test_format_f32() { assert_eq!("10", format!("{:.0}", 9.9f32)); assert_eq!("9.8", format!("{:.1}", 9.849f32)); assert_eq!("9.9", format!("{:.1}", 9.851f32)); - assert_eq!("1", format!("{:.0}", 0.5f32)); + assert_eq!("0", format!("{:.0}", 0.5f32)); assert_eq!("1.2345679e6", format!("{:e}", 1234567.89f32)); assert_eq!("1.2345679e3", format!("{:e}", 1234.56789f32)); assert_eq!("1.2345679E6", format!("{:E}", 1234567.89f32)); assert_eq!("1.2345679E3", format!("{:E}", 1234.56789f32)); assert_eq!("0.0", format!("{:?}", 0.0f32)); assert_eq!("1.01", format!("{:?}", 1.01f32)); + + let high_cutoff = 1e16_f32; + assert_eq!("1e16", format!("{:?}", high_cutoff)); + assert_eq!("-1e16", format!("{:?}", -high_cutoff)); + assert!(!is_exponential(&format!("{:?}", high_cutoff * (1.0 - 2.0 * f32::EPSILON)))); + assert_eq!("-3.0", format!("{:?}", -3f32)); + assert_eq!("0.0001", format!("{:?}", 0.0001f32)); + assert_eq!("9e-5", format!("{:?}", 0.00009f32)); + assert_eq!("1234567.9", format!("{:.1?}", 1234567.89f32)); + assert_eq!("1234.6", format!("{:.1?}", 1234.56789f32)); +} + +#[test] +fn test_format_f32_rounds_ties_to_even() { + assert_eq!("0", format!("{:.0}", 0.5f32)); + assert_eq!("2", format!("{:.0}", 1.5f32)); + assert_eq!("2", format!("{:.0}", 2.5f32)); + assert_eq!("4", format!("{:.0}", 3.5f32)); + assert_eq!("4", format!("{:.0}", 4.5f32)); + assert_eq!("6", format!("{:.0}", 5.5f32)); + assert_eq!("128", format!("{:.0}", 127.5f32)); + assert_eq!("128", format!("{:.0}", 128.5f32)); + assert_eq!("0.2", format!("{:.1}", 0.25f32)); + assert_eq!("0.8", format!("{:.1}", 0.75f32)); + assert_eq!("0.12", format!("{:.2}", 0.125f32)); + assert_eq!("0.88", format!("{:.2}", 0.875f32)); + assert_eq!("0.062", format!("{:.3}", 0.062f32)); + assert_eq!("-0", format!("{:.0}", -0.5f32)); + assert_eq!("-2", format!("{:.0}", -1.5f32)); + assert_eq!("-2", format!("{:.0}", -2.5f32)); + assert_eq!("-4", format!("{:.0}", -3.5f32)); + assert_eq!("-4", format!("{:.0}", -4.5f32)); + assert_eq!("-6", format!("{:.0}", -5.5f32)); + assert_eq!("-128", format!("{:.0}", -127.5f32)); + assert_eq!("-128", format!("{:.0}", -128.5f32)); + assert_eq!("-0.2", format!("{:.1}", -0.25f32)); + assert_eq!("-0.8", format!("{:.1}", -0.75f32)); + assert_eq!("-0.12", format!("{:.2}", -0.125f32)); + assert_eq!("-0.88", format!("{:.2}", -0.875f32)); + assert_eq!("-0.062", format!("{:.3}", -0.062f32)); + + assert_eq!("2e0", format!("{:.0e}", 1.5f32)); + assert_eq!("2e0", format!("{:.0e}", 2.5f32)); + assert_eq!("4e0", format!("{:.0e}", 3.5f32)); + assert_eq!("4e0", format!("{:.0e}", 4.5f32)); + assert_eq!("6e0", format!("{:.0e}", 5.5f32)); + assert_eq!("1.28e2", format!("{:.2e}", 127.5f32)); + assert_eq!("1.28e2", format!("{:.2e}", 128.5f32)); + assert_eq!("-2e0", format!("{:.0e}", -1.5f32)); + assert_eq!("-2e0", format!("{:.0e}", -2.5f32)); + assert_eq!("-4e0", format!("{:.0e}", -3.5f32)); + assert_eq!("-4e0", format!("{:.0e}", -4.5f32)); + assert_eq!("-6e0", format!("{:.0e}", -5.5f32)); + assert_eq!("-1.28e2", format!("{:.2e}", -127.5f32)); + assert_eq!("-1.28e2", format!("{:.2e}", -128.5f32)); + + assert_eq!("2E0", format!("{:.0E}", 1.5f32)); + assert_eq!("2E0", format!("{:.0E}", 2.5f32)); + assert_eq!("4E0", format!("{:.0E}", 3.5f32)); + assert_eq!("4E0", format!("{:.0E}", 4.5f32)); + assert_eq!("6E0", format!("{:.0E}", 5.5f32)); + assert_eq!("1.28E2", format!("{:.2E}", 127.5f32)); + assert_eq!("1.28E2", format!("{:.2E}", 128.5f32)); + assert_eq!("-2E0", format!("{:.0E}", -1.5f32)); + assert_eq!("-2E0", format!("{:.0E}", -2.5f32)); + assert_eq!("-4E0", format!("{:.0E}", -3.5f32)); + assert_eq!("-4E0", format!("{:.0E}", -4.5f32)); + assert_eq!("-6E0", format!("{:.0E}", -5.5f32)); + assert_eq!("-1.28E2", format!("{:.2E}", -127.5f32)); + assert_eq!("-1.28E2", format!("{:.2E}", -128.5f32)); +} + +fn is_exponential(s: &str) -> bool { + s.contains("e") || s.contains("E") } diff --git a/crux-mir/lib/core/tests/fmt/mod.rs b/crux-mir/lib/core/tests/fmt/mod.rs index 7b281ce48..618076358 100644 --- a/crux-mir/lib/core/tests/fmt/mod.rs +++ b/crux-mir/lib/core/tests/fmt/mod.rs @@ -6,7 +6,7 @@ mod num; fn test_format_flags() { // No residual flags left by pointer formatting let p = "".as_ptr(); - assert_eq!(format!("{:p} {:x}", p, 16), format!("{:p} 10", p)); + assert_eq!(format!("{:p} {:x}", p, 16), format!("{p:p} 10")); assert_eq!(format!("{: >3}", 'a'), " a"); } @@ -15,8 +15,8 @@ fn test_format_flags() { fn test_pointer_formats_data_pointer() { let b: &[u8] = b""; let s: &str = ""; - assert_eq!(format!("{:p}", s), format!("{:p}", s.as_ptr())); - assert_eq!(format!("{:p}", b), format!("{:p}", b.as_ptr())); + assert_eq!(format!("{s:p}"), format!("{:p}", s.as_ptr())); + assert_eq!(format!("{b:p}"), format!("{:p}", b.as_ptr())); } #[test] @@ -41,5 +41,5 @@ fn pad_integral_resets() { } } - assert_eq!(format!("{:<03}", Bar), "1 0051 "); + assert_eq!(format!("{Bar:<03}"), "1 0051 "); } diff --git a/crux-mir/lib/core/tests/fmt/num.rs b/crux-mir/lib/core/tests/fmt/num.rs index a50c2b46a..b9ede65c9 100644 --- a/crux-mir/lib/core/tests/fmt/num.rs +++ b/crux-mir/lib/core/tests/fmt/num.rs @@ -104,7 +104,6 @@ fn test_format_int() { #[test] fn test_format_int_exp_limits() { - use core::{i128, i16, i32, i64, i8, u128, u16, u32, u64, u8}; assert_eq!(format!("{:e}", i8::MIN), "-1.28e2"); assert_eq!(format!("{:e}", i8::MAX), "1.27e2"); assert_eq!(format!("{:e}", i16::MIN), "-3.2768e4"); @@ -125,11 +124,9 @@ fn test_format_int_exp_limits() { #[test] fn test_format_int_exp_precision() { - use core::{i128, i16, i32, i64, i8}; - //test that float and integer match let big_int: u32 = 314_159_265; - assert_eq!(format!("{:.1e}", big_int), format!("{:.1e}", f64::from(big_int))); + assert_eq!(format!("{big_int:.1e}"), format!("{:.1e}", f64::from(big_int))); //test adding precision assert_eq!(format!("{:.10e}", i8::MIN), "-1.2800000000e2"); @@ -149,6 +146,7 @@ fn test_format_int_exp_precision() { assert_eq!(format!("{:.1000e}", 1), format!("1.{}e0", "0".repeat(1000))); //test zero precision assert_eq!(format!("{:.0e}", 1), format!("1e0",)); + assert_eq!(format!("{:.0e}", 35), format!("4e1",)); //test padding with precision (and sign) assert_eq!(format!("{:+10.3e}", 1), " +1.000e0"); @@ -214,7 +212,6 @@ fn test_format_int_sign_padding() { #[test] fn test_format_int_twos_complement() { - use core::{i16, i32, i64, i8}; assert_eq!(format!("{}", i8::MIN), "-128"); assert_eq!(format!("{}", i16::MIN), "-32768"); assert_eq!(format!("{}", i32::MIN), "-2147483648"); diff --git a/crux-mir/lib/core/tests/future.rs b/crux-mir/lib/core/tests/future.rs new file mode 100644 index 000000000..74b6f74e4 --- /dev/null +++ b/crux-mir/lib/core/tests/future.rs @@ -0,0 +1,128 @@ +use std::future::{join, Future}; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll, Wake}; +use std::thread; + +struct PollN { + val: usize, + polled: usize, + num: usize, +} + +impl Future for PollN { + type Output = usize; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.polled += 1; + + if self.polled == self.num { + return Poll::Ready(self.val); + } + + cx.waker().wake_by_ref(); + Poll::Pending + } +} + +fn poll_n(val: usize, num: usize) -> PollN { + PollN { val, num, polled: 0 } +} + +#[test] +#[cfg_attr(miri, ignore)] // self-referential generators do not work with Miri's aliasing checks +fn test_join() { + block_on(async move { + let x = join!(async { 0 }).await; + assert_eq!(x, 0); + + let x = join!(async { 0 }, async { 1 }).await; + assert_eq!(x, (0, 1)); + + let x = join!(async { 0 }, async { 1 }, async { 2 }).await; + assert_eq!(x, (0, 1, 2)); + + let x = join!( + poll_n(0, 1), + poll_n(1, 5), + poll_n(2, 2), + poll_n(3, 1), + poll_n(4, 2), + poll_n(5, 3), + poll_n(6, 4), + poll_n(7, 1) + ) + .await; + assert_eq!(x, (0, 1, 2, 3, 4, 5, 6, 7)); + + let y = String::new(); + let x = join!(async { + println!("{}", &y); + 1 + }) + .await; + assert_eq!(x, 1); + }); +} + +/// Tests that `join!(…)` behaves "like a function": evaluating its arguments +/// before applying any of its own logic. +/// +/// _e.g._, `join!(async_fn(&borrowed), …)` does not consume `borrowed`; +/// and `join!(opt_fut?, …)` does let that `?` refer to the callsite scope. +mod test_join_function_like_value_arg_semantics { + use super::*; + + async fn async_fn(_: impl Sized) {} + + // no need to _run_ this test, just to compile it. + fn _join_does_not_unnecessarily_move_mentioned_bindings() { + let not_copy = vec![()]; + let _ = join!(async_fn(¬_copy)); // should not move `not_copy` + let _ = ¬_copy; // OK + } + + #[test] + fn join_lets_control_flow_effects_such_as_try_flow_through() { + let maybe_fut = None; + if false { + *&mut { maybe_fut } = Some(async {}); + loop {} + } + assert!(Option::is_none(&try { join!(maybe_fut?, async { unreachable!() }) })); + } + + #[test] + fn join_is_able_to_handle_temporaries() { + let _ = join!(async_fn(&String::from("temporary"))); + let () = block_on(join!(async_fn(&String::from("temporary")))); + } +} + +fn block_on(fut: impl Future) { + struct Waker; + impl Wake for Waker { + fn wake(self: Arc) { + thread::current().unpark() + } + } + + let waker = Arc::new(Waker).into(); + let mut cx = Context::from_waker(&waker); + let mut fut = Box::pin(fut); + + loop { + match fut.as_mut().poll(&mut cx) { + Poll::Ready(_) => break, + Poll::Pending => thread::park(), + } + } +} + +// just tests by whether or not this compiles +fn _pending_impl_all_auto_traits() { + use std::panic::{RefUnwindSafe, UnwindSafe}; + fn all_auto_traits() {} + + all_auto_traits::>(); +} diff --git a/crux-mir/lib/core/tests/hash/mod.rs b/crux-mir/lib/core/tests/hash/mod.rs index 1566d3574..267245f05 100644 --- a/crux-mir/lib/core/tests/hash/mod.rs +++ b/crux-mir/lib/core/tests/hash/mod.rs @@ -1,25 +1,33 @@ mod sip; use std::default::Default; -use std::hash::{Hash, Hasher}; +use std::hash::{BuildHasher, Hash, Hasher}; +use std::ptr; use std::rc::Rc; struct MyHasher { hash: u64, } -impl Default for MyHasher { +impl const Default for MyHasher { fn default() -> MyHasher { MyHasher { hash: 0 } } } -impl Hasher for MyHasher { +impl const Hasher for MyHasher { fn write(&mut self, buf: &[u8]) { - for byte in buf { - self.hash += *byte as u64; + // FIXME(const_trait_impl): change to for loop + let mut i = 0; + while i < buf.len() { + self.hash += buf[i] as u64; + i += 1; } } + fn write_str(&mut self, s: &str) { + self.write(s.as_bytes()); + self.write_u8(0xFF); + } fn finish(&self) -> u64 { self.hash } @@ -27,12 +35,25 @@ impl Hasher for MyHasher { #[test] fn test_writer_hasher() { - fn hash(t: &T) -> u64 { + const fn hash(t: &T) -> u64 { let mut s = MyHasher { hash: 0 }; t.hash(&mut s); s.finish() } + const { + // FIXME(fee1-dead): assert_eq + assert!(hash(&()) == 0); + assert!(hash(&5_u8) == 5); + assert!(hash(&5_u16) == 5); + assert!(hash(&5_u32) == 5); + + assert!(hash(&'a') == 97); + + let s: &str = "a"; + assert!(hash(&s) == 97 + 0xFF); + }; + assert_eq!(hash(&()), 0); assert_eq!(hash(&5_u8), 5); @@ -65,10 +86,10 @@ fn test_writer_hasher() { let cs: Rc<[u8]> = Rc::new([1, 2, 3]); assert_eq!(hash(&cs), 9); - let ptr = 5_usize as *const i32; + let ptr = ptr::invalid::(5_usize); assert_eq!(hash(&ptr), 5); - let ptr = 5_usize as *mut i32; + let ptr = ptr::invalid_mut::(5_usize); assert_eq!(hash(&ptr), 5); if cfg!(miri) { @@ -92,7 +113,7 @@ struct CustomHasher { output: u64, } -impl Hasher for CustomHasher { +impl const Hasher for CustomHasher { fn finish(&self) -> u64 { self.output } @@ -104,27 +125,29 @@ impl Hasher for CustomHasher { } } -impl Default for CustomHasher { +impl const Default for CustomHasher { fn default() -> CustomHasher { CustomHasher { output: 0 } } } -impl Hash for Custom { - fn hash(&self, state: &mut H) { +impl const Hash for Custom { + fn hash(&self, state: &mut H) { state.write_u64(self.hash); } } #[test] fn test_custom_state() { - fn hash(t: &T) -> u64 { + const fn hash(t: &T) -> u64 { let mut c = CustomHasher { output: 0 }; t.hash(&mut c); c.finish() } assert_eq!(hash(&Custom { hash: 5 }), 5); + + const { assert!(hash(&Custom { hash: 6 }) == 6) }; } // FIXME: Instantiated functions with i128 in the signature is not supported in Emscripten. @@ -139,3 +162,18 @@ fn test_indirect_hasher() { } assert_eq!(hasher.hash, 5); } + +#[test] +fn test_build_hasher_object_safe() { + use std::collections::hash_map::{DefaultHasher, RandomState}; + + let _: &dyn BuildHasher = &RandomState::new(); +} + +// just tests by whether or not this compiles +fn _build_hasher_default_impl_all_auto_traits() { + use std::panic::{RefUnwindSafe, UnwindSafe}; + fn all_auto_traits() {} + + all_auto_traits::>(); +} diff --git a/crux-mir/lib/core/tests/hash/sip.rs b/crux-mir/lib/core/tests/hash/sip.rs index 5c0e114e9..3abf6efcf 100644 --- a/crux-mir/lib/core/tests/hash/sip.rs +++ b/crux-mir/lib/core/tests/hash/sip.rs @@ -8,35 +8,12 @@ use core::{mem, slice}; struct Bytes<'a>(&'a [u8]); impl<'a> Hash for Bytes<'a> { - #[allow(unused_must_use)] fn hash(&self, state: &mut H) { let Bytes(v) = *self; state.write(v); } } -macro_rules! u8to64_le { - ($buf:expr, $i:expr) => { - $buf[0 + $i] as u64 - | ($buf[1 + $i] as u64) << 8 - | ($buf[2 + $i] as u64) << 16 - | ($buf[3 + $i] as u64) << 24 - | ($buf[4 + $i] as u64) << 32 - | ($buf[5 + $i] as u64) << 40 - | ($buf[6 + $i] as u64) << 48 - | ($buf[7 + $i] as u64) << 56 - }; - ($buf:expr, $i:expr, $len:expr) => {{ - let mut t = 0; - let mut out = 0; - while t < $len { - out |= ($buf[t + $i] as u64) << t * 8; - t += 1; - } - out - }}; -} - fn hash_with(mut st: H, x: &T) -> u64 { x.hash(&mut st); st.finish() @@ -46,6 +23,20 @@ fn hash(x: &T) -> u64 { hash_with(SipHasher::new(), x) } +#[test] +const fn test_const_sip() { + let val1 = 0x45; + let val2 = 0xfeed; + + const fn const_hash(x: &T) -> u64 { + let mut st = SipHasher::new(); + x.hash(&mut st); + st.finish() + } + + assert!(const_hash(&(val1)) != const_hash(&(val2))); +} + #[test] #[allow(unused_must_use)] fn test_siphash_1_3() { @@ -123,7 +114,7 @@ fn test_siphash_1_3() { let mut state_inc = SipHasher13::new_with_keys(k0, k1); while t < 64 { - let vec = u8to64_le!(vecs[t], 0); + let vec = u64::from_le_bytes(vecs[t]); let out = hash_with(SipHasher13::new_with_keys(k0, k1), &Bytes(&buf)); assert_eq!(vec, out); @@ -217,7 +208,7 @@ fn test_siphash_2_4() { let mut state_inc = SipHasher::new_with_keys(k0, k1); while t < 64 { - let vec = u8to64_le!(vecs[t], 0); + let vec = u64::from_le_bytes(vecs[t]); let out = hash_with(SipHasher::new_with_keys(k0, k1), &Bytes(&buf)); assert_eq!(vec, out); diff --git a/crux-mir/lib/core/tests/intrinsics.rs b/crux-mir/lib/core/tests/intrinsics.rs index fed7c4a5b..06870c6d0 100644 --- a/crux-mir/lib/core/tests/intrinsics.rs +++ b/crux-mir/lib/core/tests/intrinsics.rs @@ -1,4 +1,5 @@ use core::any::TypeId; +use core::intrinsics::assume; #[test] fn test_typeid_sized_types() { @@ -20,3 +21,81 @@ fn test_typeid_unsized_types() { assert_eq!(TypeId::of::(), TypeId::of::()); assert!(TypeId::of::() != TypeId::of::()); } + +// Check that `const_assume` feature allow `assume` intrinsic +// to be used in const contexts. +#[test] +fn test_assume_can_be_in_const_contexts() { + const unsafe fn foo(x: usize, y: usize) -> usize { + // SAFETY: the entire function is not safe, + // but it is just an example not used elsewhere. + unsafe { assume(y != 0) }; + x / y + } + let rs = unsafe { foo(42, 97) }; + assert_eq!(rs, 0); +} + +#[test] +const fn test_write_bytes_in_const_contexts() { + use core::intrinsics::write_bytes; + + const TEST: [u32; 3] = { + let mut arr = [1u32, 2, 3]; + unsafe { + write_bytes(arr.as_mut_ptr(), 0, 2); + } + arr + }; + + assert!(TEST[0] == 0); + assert!(TEST[1] == 0); + assert!(TEST[2] == 3); + + const TEST2: [u32; 3] = { + let mut arr = [1u32, 2, 3]; + unsafe { + write_bytes(arr.as_mut_ptr(), 1, 2); + } + arr + }; + + assert!(TEST2[0] == 16843009); + assert!(TEST2[1] == 16843009); + assert!(TEST2[2] == 3); +} + +#[test] +fn test_hints_in_const_contexts() { + use core::intrinsics::{likely, unlikely}; + + // In const contexts, they just return their argument. + const { + assert!(true == likely(true)); + assert!(false == likely(false)); + assert!(true == unlikely(true)); + assert!(false == unlikely(false)); + assert!(42u32 == core::intrinsics::black_box(42u32)); + assert!(42u32 == core::hint::black_box(42u32)); + } +} + +#[test] +fn test_const_allocate_at_runtime() { + use core::intrinsics::const_allocate; + unsafe { + assert!(const_allocate(4, 4).is_null()); + } +} + +#[test] +fn test_const_deallocate_at_runtime() { + use core::intrinsics::const_deallocate; + const X: &u32 = &42u32; + let x = &0u32; + unsafe { + const_deallocate(X as *const _ as *mut u8, 4, 4); // nop + const_deallocate(x as *const _ as *mut u8, 4, 4); // nop + const_deallocate(core::ptr::null_mut(), 1, 1); // nop + } +} diff --git a/crux-mir/lib/core/tests/iter.rs b/crux-mir/lib/core/tests/iter.rs deleted file mode 100644 index 98e3eeb98..000000000 --- a/crux-mir/lib/core/tests/iter.rs +++ /dev/null @@ -1,3014 +0,0 @@ -// ignore-tidy-filelength - -use core::cell::Cell; -use core::convert::TryFrom; -use core::iter::*; -use core::usize; -use core::{i16, i8, isize}; - -#[test] -fn test_lt() { - let empty: [isize; 0] = []; - let xs = [1, 2, 3]; - let ys = [1, 2, 0]; - - assert!(!xs.iter().lt(ys.iter())); - assert!(!xs.iter().le(ys.iter())); - assert!(xs.iter().gt(ys.iter())); - assert!(xs.iter().ge(ys.iter())); - - assert!(ys.iter().lt(xs.iter())); - assert!(ys.iter().le(xs.iter())); - assert!(!ys.iter().gt(xs.iter())); - assert!(!ys.iter().ge(xs.iter())); - - assert!(empty.iter().lt(xs.iter())); - assert!(empty.iter().le(xs.iter())); - assert!(!empty.iter().gt(xs.iter())); - assert!(!empty.iter().ge(xs.iter())); - - // Sequence with NaN - let u = [1.0f64, 2.0]; - let v = [0.0f64 / 0.0, 3.0]; - - assert!(!u.iter().lt(v.iter())); - assert!(!u.iter().le(v.iter())); - assert!(!u.iter().gt(v.iter())); - assert!(!u.iter().ge(v.iter())); - - let a = [0.0f64 / 0.0]; - let b = [1.0f64]; - let c = [2.0f64]; - - assert!(a.iter().lt(b.iter()) == (a[0] < b[0])); - assert!(a.iter().le(b.iter()) == (a[0] <= b[0])); - assert!(a.iter().gt(b.iter()) == (a[0] > b[0])); - assert!(a.iter().ge(b.iter()) == (a[0] >= b[0])); - - assert!(c.iter().lt(b.iter()) == (c[0] < b[0])); - assert!(c.iter().le(b.iter()) == (c[0] <= b[0])); - assert!(c.iter().gt(b.iter()) == (c[0] > b[0])); - assert!(c.iter().ge(b.iter()) == (c[0] >= b[0])); -} - -#[test] -fn test_multi_iter() { - let xs = [1, 2, 3, 4]; - let ys = [4, 3, 2, 1]; - assert!(xs.iter().eq(ys.iter().rev())); - assert!(xs.iter().lt(xs.iter().skip(2))); -} - -#[test] -fn test_cmp_by() { - use core::cmp::Ordering; - - let f = |x: i32, y: i32| (x * x).cmp(&y); - let xs = || [1, 2, 3, 4].iter().copied(); - let ys = || [1, 4, 16].iter().copied(); - - assert_eq!(xs().cmp_by(ys(), f), Ordering::Less); - assert_eq!(ys().cmp_by(xs(), f), Ordering::Greater); - assert_eq!(xs().cmp_by(xs().map(|x| x * x), f), Ordering::Equal); - assert_eq!(xs().rev().cmp_by(ys().rev(), f), Ordering::Greater); - assert_eq!(xs().cmp_by(ys().rev(), f), Ordering::Less); - assert_eq!(xs().cmp_by(ys().take(2), f), Ordering::Greater); -} - -#[test] -fn test_partial_cmp_by() { - use core::cmp::Ordering; - use core::f64; - - let f = |x: i32, y: i32| (x * x).partial_cmp(&y); - let xs = || [1, 2, 3, 4].iter().copied(); - let ys = || [1, 4, 16].iter().copied(); - - assert_eq!(xs().partial_cmp_by(ys(), f), Some(Ordering::Less)); - assert_eq!(ys().partial_cmp_by(xs(), f), Some(Ordering::Greater)); - assert_eq!(xs().partial_cmp_by(xs().map(|x| x * x), f), Some(Ordering::Equal)); - assert_eq!(xs().rev().partial_cmp_by(ys().rev(), f), Some(Ordering::Greater)); - assert_eq!(xs().partial_cmp_by(xs().rev(), f), Some(Ordering::Less)); - assert_eq!(xs().partial_cmp_by(ys().take(2), f), Some(Ordering::Greater)); - - let f = |x: f64, y: f64| (x * x).partial_cmp(&y); - let xs = || [1.0, 2.0, 3.0, 4.0].iter().copied(); - let ys = || [1.0, 4.0, f64::NAN, 16.0].iter().copied(); - - assert_eq!(xs().partial_cmp_by(ys(), f), None); - assert_eq!(ys().partial_cmp_by(xs(), f), Some(Ordering::Greater)); -} - -#[test] -fn test_eq_by() { - let f = |x: i32, y: i32| x * x == y; - let xs = || [1, 2, 3, 4].iter().copied(); - let ys = || [1, 4, 9, 16].iter().copied(); - - assert!(xs().eq_by(ys(), f)); - assert!(!ys().eq_by(xs(), f)); - assert!(!xs().eq_by(xs(), f)); - assert!(!ys().eq_by(ys(), f)); - - assert!(!xs().take(3).eq_by(ys(), f)); - assert!(!xs().eq_by(ys().take(3), f)); - assert!(xs().take(3).eq_by(ys().take(3), f)); -} - -#[test] -fn test_counter_from_iter() { - let it = (0..).step_by(5).take(10); - let xs: Vec = FromIterator::from_iter(it); - assert_eq!(xs, [0, 5, 10, 15, 20, 25, 30, 35, 40, 45]); -} - -#[test] -fn test_iterator_chain() { - let xs = [0, 1, 2, 3, 4, 5]; - let ys = [30, 40, 50, 60]; - let expected = [0, 1, 2, 3, 4, 5, 30, 40, 50, 60]; - let it = xs.iter().chain(&ys); - let mut i = 0; - for &x in it { - assert_eq!(x, expected[i]); - i += 1; - } - assert_eq!(i, expected.len()); - - let ys = (30..).step_by(10).take(4); - let it = xs.iter().cloned().chain(ys); - let mut i = 0; - for x in it { - assert_eq!(x, expected[i]); - i += 1; - } - assert_eq!(i, expected.len()); -} - -#[test] -fn test_iterator_chain_nth() { - let xs = [0, 1, 2, 3, 4, 5]; - let ys = [30, 40, 50, 60]; - let zs = []; - let expected = [0, 1, 2, 3, 4, 5, 30, 40, 50, 60]; - for (i, x) in expected.iter().enumerate() { - assert_eq!(Some(x), xs.iter().chain(&ys).nth(i)); - } - assert_eq!(zs.iter().chain(&xs).nth(0), Some(&0)); - - let mut it = xs.iter().chain(&zs); - assert_eq!(it.nth(5), Some(&5)); - assert_eq!(it.next(), None); -} - -#[test] -fn test_iterator_chain_nth_back() { - let xs = [0, 1, 2, 3, 4, 5]; - let ys = [30, 40, 50, 60]; - let zs = []; - let expected = [0, 1, 2, 3, 4, 5, 30, 40, 50, 60]; - for (i, x) in expected.iter().rev().enumerate() { - assert_eq!(Some(x), xs.iter().chain(&ys).nth_back(i)); - } - assert_eq!(zs.iter().chain(&xs).nth_back(0), Some(&5)); - - let mut it = xs.iter().chain(&zs); - assert_eq!(it.nth_back(5), Some(&0)); - assert_eq!(it.next(), None); -} - -#[test] -fn test_iterator_chain_last() { - let xs = [0, 1, 2, 3, 4, 5]; - let ys = [30, 40, 50, 60]; - let zs = []; - assert_eq!(xs.iter().chain(&ys).last(), Some(&60)); - assert_eq!(zs.iter().chain(&ys).last(), Some(&60)); - assert_eq!(ys.iter().chain(&zs).last(), Some(&60)); - assert_eq!(zs.iter().chain(&zs).last(), None); -} - -#[test] -fn test_iterator_chain_count() { - let xs = [0, 1, 2, 3, 4, 5]; - let ys = [30, 40, 50, 60]; - let zs = []; - assert_eq!(xs.iter().chain(&ys).count(), 10); - assert_eq!(zs.iter().chain(&ys).count(), 4); -} - -#[test] -fn test_iterator_chain_find() { - let xs = [0, 1, 2, 3, 4, 5]; - let ys = [30, 40, 50, 60]; - let mut iter = xs.iter().chain(&ys); - assert_eq!(iter.find(|&&i| i == 4), Some(&4)); - assert_eq!(iter.next(), Some(&5)); - assert_eq!(iter.find(|&&i| i == 40), Some(&40)); - assert_eq!(iter.next(), Some(&50)); - assert_eq!(iter.find(|&&i| i == 100), None); - assert_eq!(iter.next(), None); -} - -#[test] -fn test_iterator_chain_size_hint() { - struct Iter { - is_empty: bool, - } - - impl Iterator for Iter { - type Item = (); - - // alternates between `None` and `Some(())` - fn next(&mut self) -> Option { - if self.is_empty { - self.is_empty = false; - None - } else { - self.is_empty = true; - Some(()) - } - } - - fn size_hint(&self) -> (usize, Option) { - if self.is_empty { (0, Some(0)) } else { (1, Some(1)) } - } - } - - impl DoubleEndedIterator for Iter { - fn next_back(&mut self) -> Option { - self.next() - } - } - - // this chains an iterator of length 0 with an iterator of length 1, - // so after calling `.next()` once, the iterator is empty and the - // state is `ChainState::Back`. `.size_hint()` should now disregard - // the size hint of the left iterator - let mut iter = Iter { is_empty: true }.chain(once(())); - assert_eq!(iter.next(), Some(())); - assert_eq!(iter.size_hint(), (0, Some(0))); - - let mut iter = once(()).chain(Iter { is_empty: true }); - assert_eq!(iter.next_back(), Some(())); - assert_eq!(iter.size_hint(), (0, Some(0))); -} - -#[test] -fn test_zip_nth() { - let xs = [0, 1, 2, 4, 5]; - let ys = [10, 11, 12]; - - let mut it = xs.iter().zip(&ys); - assert_eq!(it.nth(0), Some((&0, &10))); - assert_eq!(it.nth(1), Some((&2, &12))); - assert_eq!(it.nth(0), None); - - let mut it = xs.iter().zip(&ys); - assert_eq!(it.nth(3), None); - - let mut it = ys.iter().zip(&xs); - assert_eq!(it.nth(3), None); -} - -#[test] -fn test_zip_nth_side_effects() { - let mut a = Vec::new(); - let mut b = Vec::new(); - let value = [1, 2, 3, 4, 5, 6] - .iter() - .cloned() - .map(|n| { - a.push(n); - n * 10 - }) - .zip([2, 3, 4, 5, 6, 7, 8].iter().cloned().map(|n| { - b.push(n * 100); - n * 1000 - })) - .skip(1) - .nth(3); - assert_eq!(value, Some((50, 6000))); - assert_eq!(a, vec![1, 2, 3, 4, 5]); - assert_eq!(b, vec![200, 300, 400, 500, 600]); -} - -#[test] -fn test_iterator_step_by() { - // Identity - let mut it = (0..).step_by(1).take(3); - assert_eq!(it.next(), Some(0)); - assert_eq!(it.next(), Some(1)); - assert_eq!(it.next(), Some(2)); - assert_eq!(it.next(), None); - - let mut it = (0..).step_by(3).take(4); - assert_eq!(it.next(), Some(0)); - assert_eq!(it.next(), Some(3)); - assert_eq!(it.next(), Some(6)); - assert_eq!(it.next(), Some(9)); - assert_eq!(it.next(), None); - - let mut it = (0..3).step_by(1); - assert_eq!(it.next_back(), Some(2)); - assert_eq!(it.next_back(), Some(1)); - assert_eq!(it.next_back(), Some(0)); - assert_eq!(it.next_back(), None); - - let mut it = (0..11).step_by(3); - assert_eq!(it.next_back(), Some(9)); - assert_eq!(it.next_back(), Some(6)); - assert_eq!(it.next_back(), Some(3)); - assert_eq!(it.next_back(), Some(0)); - assert_eq!(it.next_back(), None); -} - -#[test] -fn test_iterator_step_by_nth() { - let mut it = (0..16).step_by(5); - assert_eq!(it.nth(0), Some(0)); - assert_eq!(it.nth(0), Some(5)); - assert_eq!(it.nth(0), Some(10)); - assert_eq!(it.nth(0), Some(15)); - assert_eq!(it.nth(0), None); - - let it = (0..18).step_by(5); - assert_eq!(it.clone().nth(0), Some(0)); - assert_eq!(it.clone().nth(1), Some(5)); - assert_eq!(it.clone().nth(2), Some(10)); - assert_eq!(it.clone().nth(3), Some(15)); - assert_eq!(it.clone().nth(4), None); - assert_eq!(it.clone().nth(42), None); -} - -#[test] -fn test_iterator_step_by_nth_overflow() { - #[cfg(target_pointer_width = "8")] - type Bigger = u16; - #[cfg(target_pointer_width = "16")] - type Bigger = u32; - #[cfg(target_pointer_width = "32")] - type Bigger = u64; - #[cfg(target_pointer_width = "64")] - type Bigger = u128; - - #[derive(Clone)] - struct Test(Bigger); - impl Iterator for &mut Test { - type Item = i32; - fn next(&mut self) -> Option { - Some(21) - } - fn nth(&mut self, n: usize) -> Option { - self.0 += n as Bigger + 1; - Some(42) - } - } - - let mut it = Test(0); - let root = usize::MAX >> (::std::mem::size_of::() * 8 / 2); - let n = root + 20; - (&mut it).step_by(n).nth(n); - assert_eq!(it.0, n as Bigger * n as Bigger); - - // large step - let mut it = Test(0); - (&mut it).step_by(usize::MAX).nth(5); - assert_eq!(it.0, (usize::MAX as Bigger) * 5); - - // n + 1 overflows - let mut it = Test(0); - (&mut it).step_by(2).nth(usize::MAX); - assert_eq!(it.0, (usize::MAX as Bigger) * 2); - - // n + 1 overflows - let mut it = Test(0); - (&mut it).step_by(1).nth(usize::MAX); - assert_eq!(it.0, (usize::MAX as Bigger) * 1); -} - -#[test] -fn test_iterator_step_by_nth_try_fold() { - let mut it = (0..).step_by(10); - assert_eq!(it.try_fold(0, i8::checked_add), None); - assert_eq!(it.next(), Some(60)); - assert_eq!(it.try_fold(0, i8::checked_add), None); - assert_eq!(it.next(), Some(90)); - - let mut it = (100..).step_by(10); - assert_eq!(it.try_fold(50, i8::checked_add), None); - assert_eq!(it.next(), Some(110)); - - let mut it = (100..=100).step_by(10); - assert_eq!(it.next(), Some(100)); - assert_eq!(it.try_fold(0, i8::checked_add), Some(0)); -} - -#[test] -fn test_iterator_step_by_nth_back() { - let mut it = (0..16).step_by(5); - assert_eq!(it.nth_back(0), Some(15)); - assert_eq!(it.nth_back(0), Some(10)); - assert_eq!(it.nth_back(0), Some(5)); - assert_eq!(it.nth_back(0), Some(0)); - assert_eq!(it.nth_back(0), None); - - let mut it = (0..16).step_by(5); - assert_eq!(it.next(), Some(0)); // to set `first_take` to `false` - assert_eq!(it.nth_back(0), Some(15)); - assert_eq!(it.nth_back(0), Some(10)); - assert_eq!(it.nth_back(0), Some(5)); - assert_eq!(it.nth_back(0), None); - - let it = || (0..18).step_by(5); - assert_eq!(it().nth_back(0), Some(15)); - assert_eq!(it().nth_back(1), Some(10)); - assert_eq!(it().nth_back(2), Some(5)); - assert_eq!(it().nth_back(3), Some(0)); - assert_eq!(it().nth_back(4), None); - assert_eq!(it().nth_back(42), None); -} - -#[test] -fn test_iterator_step_by_nth_try_rfold() { - let mut it = (0..100).step_by(10); - assert_eq!(it.try_rfold(0, i8::checked_add), None); - assert_eq!(it.next_back(), Some(70)); - assert_eq!(it.next(), Some(0)); - assert_eq!(it.try_rfold(0, i8::checked_add), None); - assert_eq!(it.next_back(), Some(30)); - - let mut it = (0..100).step_by(10); - assert_eq!(it.try_rfold(50, i8::checked_add), None); - assert_eq!(it.next_back(), Some(80)); - - let mut it = (100..=100).step_by(10); - assert_eq!(it.next_back(), Some(100)); - assert_eq!(it.try_fold(0, i8::checked_add), Some(0)); -} - -#[test] -#[should_panic] -fn test_iterator_step_by_zero() { - let mut it = (0..).step_by(0); - it.next(); -} - -#[test] -fn test_iterator_step_by_size_hint() { - struct StubSizeHint(usize, Option); - impl Iterator for StubSizeHint { - type Item = (); - fn next(&mut self) -> Option<()> { - self.0 -= 1; - if let Some(ref mut upper) = self.1 { - *upper -= 1; - } - Some(()) - } - fn size_hint(&self) -> (usize, Option) { - (self.0, self.1) - } - } - - // The two checks in each case are needed because the logic - // is different before the first call to `next()`. - - let mut it = StubSizeHint(10, Some(10)).step_by(1); - assert_eq!(it.size_hint(), (10, Some(10))); - it.next(); - assert_eq!(it.size_hint(), (9, Some(9))); - - // exact multiple - let mut it = StubSizeHint(10, Some(10)).step_by(3); - assert_eq!(it.size_hint(), (4, Some(4))); - it.next(); - assert_eq!(it.size_hint(), (3, Some(3))); - - // larger base range, but not enough to get another element - let mut it = StubSizeHint(12, Some(12)).step_by(3); - assert_eq!(it.size_hint(), (4, Some(4))); - it.next(); - assert_eq!(it.size_hint(), (3, Some(3))); - - // smaller base range, so fewer resulting elements - let mut it = StubSizeHint(9, Some(9)).step_by(3); - assert_eq!(it.size_hint(), (3, Some(3))); - it.next(); - assert_eq!(it.size_hint(), (2, Some(2))); - - // infinite upper bound - let mut it = StubSizeHint(usize::MAX, None).step_by(1); - assert_eq!(it.size_hint(), (usize::MAX, None)); - it.next(); - assert_eq!(it.size_hint(), (usize::MAX - 1, None)); - - // still infinite with larger step - let mut it = StubSizeHint(7, None).step_by(3); - assert_eq!(it.size_hint(), (3, None)); - it.next(); - assert_eq!(it.size_hint(), (2, None)); - - // propagates ExactSizeIterator - let a = [1, 2, 3, 4, 5]; - let it = a.iter().step_by(2); - assert_eq!(it.len(), 3); - - // Cannot be TrustedLen as a step greater than one makes an iterator - // with (usize::MAX, None) no longer meet the safety requirements - trait TrustedLenCheck { - fn test(self) -> bool; - } - impl TrustedLenCheck for T { - default fn test(self) -> bool { - false - } - } - impl TrustedLenCheck for T { - fn test(self) -> bool { - true - } - } - assert!(TrustedLenCheck::test(a.iter())); - assert!(!TrustedLenCheck::test(a.iter().step_by(1))); -} - -#[test] -fn test_filter_map() { - let it = (0..).step_by(1).take(10).filter_map(|x| if x % 2 == 0 { Some(x * x) } else { None }); - assert_eq!(it.collect::>(), [0 * 0, 2 * 2, 4 * 4, 6 * 6, 8 * 8]); -} - -#[test] -fn test_filter_map_fold() { - let xs = [0, 1, 2, 3, 4, 5, 6, 7, 8]; - let ys = [0 * 0, 2 * 2, 4 * 4, 6 * 6, 8 * 8]; - let it = xs.iter().filter_map(|&x| if x % 2 == 0 { Some(x * x) } else { None }); - let i = it.fold(0, |i, x| { - assert_eq!(x, ys[i]); - i + 1 - }); - assert_eq!(i, ys.len()); - - let it = xs.iter().filter_map(|&x| if x % 2 == 0 { Some(x * x) } else { None }); - let i = it.rfold(ys.len(), |i, x| { - assert_eq!(x, ys[i - 1]); - i - 1 - }); - assert_eq!(i, 0); -} - -#[test] -fn test_iterator_enumerate() { - let xs = [0, 1, 2, 3, 4, 5]; - let it = xs.iter().enumerate(); - for (i, &x) in it { - assert_eq!(i, x); - } -} - -#[test] -fn test_iterator_enumerate_nth() { - let xs = [0, 1, 2, 3, 4, 5]; - for (i, &x) in xs.iter().enumerate() { - assert_eq!(i, x); - } - - let mut it = xs.iter().enumerate(); - while let Some((i, &x)) = it.nth(0) { - assert_eq!(i, x); - } - - let mut it = xs.iter().enumerate(); - while let Some((i, &x)) = it.nth(1) { - assert_eq!(i, x); - } - - let (i, &x) = xs.iter().enumerate().nth(3).unwrap(); - assert_eq!(i, x); - assert_eq!(i, 3); -} - -#[test] -fn test_iterator_enumerate_nth_back() { - let xs = [0, 1, 2, 3, 4, 5]; - let mut it = xs.iter().enumerate(); - while let Some((i, &x)) = it.nth_back(0) { - assert_eq!(i, x); - } - - let mut it = xs.iter().enumerate(); - while let Some((i, &x)) = it.nth_back(1) { - assert_eq!(i, x); - } - - let (i, &x) = xs.iter().enumerate().nth_back(3).unwrap(); - assert_eq!(i, x); - assert_eq!(i, 2); -} - -#[test] -fn test_iterator_enumerate_count() { - let xs = [0, 1, 2, 3, 4, 5]; - assert_eq!(xs.iter().enumerate().count(), 6); -} - -#[test] -fn test_iterator_enumerate_fold() { - let xs = [0, 1, 2, 3, 4, 5]; - let mut it = xs.iter().enumerate(); - // steal a couple to get an interesting offset - assert_eq!(it.next(), Some((0, &0))); - assert_eq!(it.next(), Some((1, &1))); - let i = it.fold(2, |i, (j, &x)| { - assert_eq!(i, j); - assert_eq!(x, xs[j]); - i + 1 - }); - assert_eq!(i, xs.len()); - - let mut it = xs.iter().enumerate(); - assert_eq!(it.next(), Some((0, &0))); - let i = it.rfold(xs.len() - 1, |i, (j, &x)| { - assert_eq!(i, j); - assert_eq!(x, xs[j]); - i - 1 - }); - assert_eq!(i, 0); -} - -#[test] -fn test_iterator_filter_count() { - let xs = [0, 1, 2, 3, 4, 5, 6, 7, 8]; - assert_eq!(xs.iter().filter(|&&x| x % 2 == 0).count(), 5); -} - -#[test] -fn test_iterator_filter_fold() { - let xs = [0, 1, 2, 3, 4, 5, 6, 7, 8]; - let ys = [0, 2, 4, 6, 8]; - let it = xs.iter().filter(|&&x| x % 2 == 0); - let i = it.fold(0, |i, &x| { - assert_eq!(x, ys[i]); - i + 1 - }); - assert_eq!(i, ys.len()); - - let it = xs.iter().filter(|&&x| x % 2 == 0); - let i = it.rfold(ys.len(), |i, &x| { - assert_eq!(x, ys[i - 1]); - i - 1 - }); - assert_eq!(i, 0); -} - -#[test] -fn test_iterator_peekable() { - let xs = vec![0, 1, 2, 3, 4, 5]; - - let mut it = xs.iter().cloned().peekable(); - assert_eq!(it.len(), 6); - assert_eq!(it.peek().unwrap(), &0); - assert_eq!(it.len(), 6); - assert_eq!(it.next().unwrap(), 0); - assert_eq!(it.len(), 5); - assert_eq!(it.next().unwrap(), 1); - assert_eq!(it.len(), 4); - assert_eq!(it.next().unwrap(), 2); - assert_eq!(it.len(), 3); - assert_eq!(it.peek().unwrap(), &3); - assert_eq!(it.len(), 3); - assert_eq!(it.peek().unwrap(), &3); - assert_eq!(it.len(), 3); - assert_eq!(it.next().unwrap(), 3); - assert_eq!(it.len(), 2); - assert_eq!(it.next().unwrap(), 4); - assert_eq!(it.len(), 1); - assert_eq!(it.peek().unwrap(), &5); - assert_eq!(it.len(), 1); - assert_eq!(it.next().unwrap(), 5); - assert_eq!(it.len(), 0); - assert!(it.peek().is_none()); - assert_eq!(it.len(), 0); - assert!(it.next().is_none()); - assert_eq!(it.len(), 0); - - let mut it = xs.iter().cloned().peekable(); - assert_eq!(it.len(), 6); - assert_eq!(it.peek().unwrap(), &0); - assert_eq!(it.len(), 6); - assert_eq!(it.next_back().unwrap(), 5); - assert_eq!(it.len(), 5); - assert_eq!(it.next_back().unwrap(), 4); - assert_eq!(it.len(), 4); - assert_eq!(it.next_back().unwrap(), 3); - assert_eq!(it.len(), 3); - assert_eq!(it.peek().unwrap(), &0); - assert_eq!(it.len(), 3); - assert_eq!(it.peek().unwrap(), &0); - assert_eq!(it.len(), 3); - assert_eq!(it.next_back().unwrap(), 2); - assert_eq!(it.len(), 2); - assert_eq!(it.next_back().unwrap(), 1); - assert_eq!(it.len(), 1); - assert_eq!(it.peek().unwrap(), &0); - assert_eq!(it.len(), 1); - assert_eq!(it.next_back().unwrap(), 0); - assert_eq!(it.len(), 0); - assert!(it.peek().is_none()); - assert_eq!(it.len(), 0); - assert!(it.next_back().is_none()); - assert_eq!(it.len(), 0); -} - -#[test] -fn test_iterator_peekable_count() { - let xs = [0, 1, 2, 3, 4, 5]; - let ys = [10]; - let zs: [i32; 0] = []; - - assert_eq!(xs.iter().peekable().count(), 6); - - let mut it = xs.iter().peekable(); - assert_eq!(it.peek(), Some(&&0)); - assert_eq!(it.count(), 6); - - assert_eq!(ys.iter().peekable().count(), 1); - - let mut it = ys.iter().peekable(); - assert_eq!(it.peek(), Some(&&10)); - assert_eq!(it.count(), 1); - - assert_eq!(zs.iter().peekable().count(), 0); - - let mut it = zs.iter().peekable(); - assert_eq!(it.peek(), None); -} - -#[test] -fn test_iterator_peekable_nth() { - let xs = [0, 1, 2, 3, 4, 5]; - let mut it = xs.iter().peekable(); - - assert_eq!(it.peek(), Some(&&0)); - assert_eq!(it.nth(0), Some(&0)); - assert_eq!(it.peek(), Some(&&1)); - assert_eq!(it.nth(1), Some(&2)); - assert_eq!(it.peek(), Some(&&3)); - assert_eq!(it.nth(2), Some(&5)); - assert_eq!(it.next(), None); -} - -#[test] -fn test_iterator_peekable_last() { - let xs = [0, 1, 2, 3, 4, 5]; - let ys = [0]; - - let mut it = xs.iter().peekable(); - assert_eq!(it.peek(), Some(&&0)); - assert_eq!(it.last(), Some(&5)); - - let mut it = ys.iter().peekable(); - assert_eq!(it.peek(), Some(&&0)); - assert_eq!(it.last(), Some(&0)); - - let mut it = ys.iter().peekable(); - assert_eq!(it.next(), Some(&0)); - assert_eq!(it.peek(), None); - assert_eq!(it.last(), None); -} - -#[test] -fn test_iterator_peekable_fold() { - let xs = [0, 1, 2, 3, 4, 5]; - let mut it = xs.iter().peekable(); - assert_eq!(it.peek(), Some(&&0)); - let i = it.fold(0, |i, &x| { - assert_eq!(x, xs[i]); - i + 1 - }); - assert_eq!(i, xs.len()); -} - -#[test] -fn test_iterator_peekable_rfold() { - let xs = [0, 1, 2, 3, 4, 5]; - let mut it = xs.iter().peekable(); - assert_eq!(it.peek(), Some(&&0)); - let i = it.rfold(0, |i, &x| { - assert_eq!(x, xs[xs.len() - 1 - i]); - i + 1 - }); - assert_eq!(i, xs.len()); -} - -/// This is an iterator that follows the Iterator contract, -/// but it is not fused. After having returned None once, it will start -/// producing elements if .next() is called again. -pub struct CycleIter<'a, T> { - index: usize, - data: &'a [T], -} - -pub fn cycle(data: &[T]) -> CycleIter<'_, T> { - CycleIter { index: 0, data } -} - -impl<'a, T> Iterator for CycleIter<'a, T> { - type Item = &'a T; - fn next(&mut self) -> Option { - let elt = self.data.get(self.index); - self.index += 1; - self.index %= 1 + self.data.len(); - elt - } -} - -#[test] -fn test_iterator_peekable_remember_peek_none_1() { - // Check that the loop using .peek() terminates - let data = [1, 2, 3]; - let mut iter = cycle(&data).peekable(); - - let mut n = 0; - while let Some(_) = iter.next() { - let is_the_last = iter.peek().is_none(); - assert_eq!(is_the_last, n == data.len() - 1); - n += 1; - if n > data.len() { - break; - } - } - assert_eq!(n, data.len()); -} - -#[test] -fn test_iterator_peekable_remember_peek_none_2() { - let data = [0]; - let mut iter = cycle(&data).peekable(); - iter.next(); - assert_eq!(iter.peek(), None); - assert_eq!(iter.last(), None); -} - -#[test] -fn test_iterator_peekable_remember_peek_none_3() { - let data = [0]; - let mut iter = cycle(&data).peekable(); - iter.peek(); - assert_eq!(iter.nth(0), Some(&0)); - - let mut iter = cycle(&data).peekable(); - iter.next(); - assert_eq!(iter.peek(), None); - assert_eq!(iter.nth(0), None); -} - -#[test] -fn test_iterator_take_while() { - let xs = [0, 1, 2, 3, 5, 13, 15, 16, 17, 19]; - let ys = [0, 1, 2, 3, 5, 13]; - let it = xs.iter().take_while(|&x| *x < 15); - let mut i = 0; - for x in it { - assert_eq!(*x, ys[i]); - i += 1; - } - assert_eq!(i, ys.len()); -} - -#[test] -fn test_iterator_skip_while() { - let xs = [0, 1, 2, 3, 5, 13, 15, 16, 17, 19]; - let ys = [15, 16, 17, 19]; - let it = xs.iter().skip_while(|&x| *x < 15); - let mut i = 0; - for x in it { - assert_eq!(*x, ys[i]); - i += 1; - } - assert_eq!(i, ys.len()); -} - -#[test] -fn test_iterator_skip_while_fold() { - let xs = [0, 1, 2, 3, 5, 13, 15, 16, 17, 19]; - let ys = [15, 16, 17, 19]; - let it = xs.iter().skip_while(|&x| *x < 15); - let i = it.fold(0, |i, &x| { - assert_eq!(x, ys[i]); - i + 1 - }); - assert_eq!(i, ys.len()); - - let mut it = xs.iter().skip_while(|&x| *x < 15); - assert_eq!(it.next(), Some(&ys[0])); // process skips before folding - let i = it.fold(1, |i, &x| { - assert_eq!(x, ys[i]); - i + 1 - }); - assert_eq!(i, ys.len()); -} - -#[test] -fn test_iterator_skip() { - let xs = [0, 1, 2, 3, 5, 13, 15, 16, 17, 19, 20, 30]; - let ys = [13, 15, 16, 17, 19, 20, 30]; - let mut it = xs.iter().skip(5); - let mut i = 0; - while let Some(&x) = it.next() { - assert_eq!(x, ys[i]); - i += 1; - assert_eq!(it.len(), xs.len() - 5 - i); - } - assert_eq!(i, ys.len()); - assert_eq!(it.len(), 0); -} - -#[test] -fn test_iterator_skip_doubleended() { - let xs = [0, 1, 2, 3, 5, 13, 15, 16, 17, 19, 20, 30]; - let mut it = xs.iter().rev().skip(5); - assert_eq!(it.next(), Some(&15)); - assert_eq!(it.by_ref().rev().next(), Some(&0)); - assert_eq!(it.next(), Some(&13)); - assert_eq!(it.by_ref().rev().next(), Some(&1)); - assert_eq!(it.next(), Some(&5)); - assert_eq!(it.by_ref().rev().next(), Some(&2)); - assert_eq!(it.next(), Some(&3)); - assert_eq!(it.next(), None); - let mut it = xs.iter().rev().skip(5).rev(); - assert_eq!(it.next(), Some(&0)); - assert_eq!(it.rev().next(), Some(&15)); - let mut it_base = xs.iter(); - { - let mut it = it_base.by_ref().skip(5).rev(); - assert_eq!(it.next(), Some(&30)); - assert_eq!(it.next(), Some(&20)); - assert_eq!(it.next(), Some(&19)); - assert_eq!(it.next(), Some(&17)); - assert_eq!(it.next(), Some(&16)); - assert_eq!(it.next(), Some(&15)); - assert_eq!(it.next(), Some(&13)); - assert_eq!(it.next(), None); - } - // make sure the skipped parts have not been consumed - assert_eq!(it_base.next(), Some(&0)); - assert_eq!(it_base.next(), Some(&1)); - assert_eq!(it_base.next(), Some(&2)); - assert_eq!(it_base.next(), Some(&3)); - assert_eq!(it_base.next(), Some(&5)); - assert_eq!(it_base.next(), None); - let it = xs.iter().skip(5).rev(); - assert_eq!(it.last(), Some(&13)); -} - -#[test] -fn test_iterator_skip_nth() { - let xs = [0, 1, 2, 3, 5, 13, 15, 16, 17, 19, 20, 30]; - - let mut it = xs.iter().skip(0); - assert_eq!(it.nth(0), Some(&0)); - assert_eq!(it.nth(1), Some(&2)); - - let mut it = xs.iter().skip(5); - assert_eq!(it.nth(0), Some(&13)); - assert_eq!(it.nth(1), Some(&16)); - - let mut it = xs.iter().skip(12); - assert_eq!(it.nth(0), None); -} - -#[test] -fn test_iterator_skip_count() { - let xs = [0, 1, 2, 3, 5, 13, 15, 16, 17, 19, 20, 30]; - - assert_eq!(xs.iter().skip(0).count(), 12); - assert_eq!(xs.iter().skip(1).count(), 11); - assert_eq!(xs.iter().skip(11).count(), 1); - assert_eq!(xs.iter().skip(12).count(), 0); - assert_eq!(xs.iter().skip(13).count(), 0); -} - -#[test] -fn test_iterator_skip_last() { - let xs = [0, 1, 2, 3, 5, 13, 15, 16, 17, 19, 20, 30]; - - assert_eq!(xs.iter().skip(0).last(), Some(&30)); - assert_eq!(xs.iter().skip(1).last(), Some(&30)); - assert_eq!(xs.iter().skip(11).last(), Some(&30)); - assert_eq!(xs.iter().skip(12).last(), None); - assert_eq!(xs.iter().skip(13).last(), None); - - let mut it = xs.iter().skip(5); - assert_eq!(it.next(), Some(&13)); - assert_eq!(it.last(), Some(&30)); -} - -#[test] -fn test_iterator_skip_fold() { - let xs = [0, 1, 2, 3, 5, 13, 15, 16, 17, 19, 20, 30]; - let ys = [13, 15, 16, 17, 19, 20, 30]; - - let it = xs.iter().skip(5); - let i = it.fold(0, |i, &x| { - assert_eq!(x, ys[i]); - i + 1 - }); - assert_eq!(i, ys.len()); - - let mut it = xs.iter().skip(5); - assert_eq!(it.next(), Some(&ys[0])); // process skips before folding - let i = it.fold(1, |i, &x| { - assert_eq!(x, ys[i]); - i + 1 - }); - assert_eq!(i, ys.len()); - - let it = xs.iter().skip(5); - let i = it.rfold(ys.len(), |i, &x| { - let i = i - 1; - assert_eq!(x, ys[i]); - i - }); - assert_eq!(i, 0); - - let mut it = xs.iter().skip(5); - assert_eq!(it.next(), Some(&ys[0])); // process skips before folding - let i = it.rfold(ys.len(), |i, &x| { - let i = i - 1; - assert_eq!(x, ys[i]); - i - }); - assert_eq!(i, 1); -} - -#[test] -fn test_iterator_take() { - let xs = [0, 1, 2, 3, 5, 13, 15, 16, 17, 19]; - let ys = [0, 1, 2, 3, 5]; - - let mut it = xs.iter().take(ys.len()); - let mut i = 0; - assert_eq!(it.len(), ys.len()); - while let Some(&x) = it.next() { - assert_eq!(x, ys[i]); - i += 1; - assert_eq!(it.len(), ys.len() - i); - } - assert_eq!(i, ys.len()); - assert_eq!(it.len(), 0); - - let mut it = xs.iter().take(ys.len()); - let mut i = 0; - assert_eq!(it.len(), ys.len()); - while let Some(&x) = it.next_back() { - i += 1; - assert_eq!(x, ys[ys.len() - i]); - assert_eq!(it.len(), ys.len() - i); - } - assert_eq!(i, ys.len()); - assert_eq!(it.len(), 0); -} - -#[test] -fn test_iterator_take_nth() { - let xs = [0, 1, 2, 4, 5]; - let mut it = xs.iter(); - { - let mut take = it.by_ref().take(3); - let mut i = 0; - while let Some(&x) = take.nth(0) { - assert_eq!(x, i); - i += 1; - } - } - assert_eq!(it.nth(1), Some(&5)); - assert_eq!(it.nth(0), None); - - let xs = [0, 1, 2, 3, 4]; - let mut it = xs.iter().take(7); - let mut i = 1; - while let Some(&x) = it.nth(1) { - assert_eq!(x, i); - i += 2; - } -} - -#[test] -fn test_iterator_take_nth_back() { - let xs = [0, 1, 2, 4, 5]; - let mut it = xs.iter(); - { - let mut take = it.by_ref().take(3); - let mut i = 0; - while let Some(&x) = take.nth_back(0) { - i += 1; - assert_eq!(x, 3 - i); - } - } - assert_eq!(it.nth_back(0), None); - - let xs = [0, 1, 2, 3, 4]; - let mut it = xs.iter().take(7); - assert_eq!(it.nth_back(1), Some(&3)); - assert_eq!(it.nth_back(1), Some(&1)); - assert_eq!(it.nth_back(1), None); -} - -#[test] -fn test_iterator_take_short() { - let xs = [0, 1, 2, 3]; - - let mut it = xs.iter().take(5); - let mut i = 0; - assert_eq!(it.len(), xs.len()); - while let Some(&x) = it.next() { - assert_eq!(x, xs[i]); - i += 1; - assert_eq!(it.len(), xs.len() - i); - } - assert_eq!(i, xs.len()); - assert_eq!(it.len(), 0); - - let mut it = xs.iter().take(5); - let mut i = 0; - assert_eq!(it.len(), xs.len()); - while let Some(&x) = it.next_back() { - i += 1; - assert_eq!(x, xs[xs.len() - i]); - assert_eq!(it.len(), xs.len() - i); - } - assert_eq!(i, xs.len()); - assert_eq!(it.len(), 0); -} - -#[test] -fn test_iterator_scan() { - // test the type inference - fn add(old: &mut isize, new: &usize) -> Option { - *old += *new as isize; - Some(*old as f64) - } - let xs = [0, 1, 2, 3, 4]; - let ys = [0f64, 1.0, 3.0, 6.0, 10.0]; - - let it = xs.iter().scan(0, add); - let mut i = 0; - for x in it { - assert_eq!(x, ys[i]); - i += 1; - } - assert_eq!(i, ys.len()); -} - -#[test] -fn test_iterator_flat_map() { - let xs = [0, 3, 6]; - let ys = [0, 1, 2, 3, 4, 5, 6, 7, 8]; - let it = xs.iter().flat_map(|&x| (x..).step_by(1).take(3)); - let mut i = 0; - for x in it { - assert_eq!(x, ys[i]); - i += 1; - } - assert_eq!(i, ys.len()); -} - -/// Tests `FlatMap::fold` with items already picked off the front and back, -/// to make sure all parts of the `FlatMap` are folded correctly. -#[test] -fn test_iterator_flat_map_fold() { - let xs = [0, 3, 6]; - let ys = [1, 2, 3, 4, 5, 6, 7]; - let mut it = xs.iter().flat_map(|&x| x..x + 3); - assert_eq!(it.next(), Some(0)); - assert_eq!(it.next_back(), Some(8)); - let i = it.fold(0, |i, x| { - assert_eq!(x, ys[i]); - i + 1 - }); - assert_eq!(i, ys.len()); - - let mut it = xs.iter().flat_map(|&x| x..x + 3); - assert_eq!(it.next(), Some(0)); - assert_eq!(it.next_back(), Some(8)); - let i = it.rfold(ys.len(), |i, x| { - assert_eq!(x, ys[i - 1]); - i - 1 - }); - assert_eq!(i, 0); -} - -#[test] -fn test_iterator_flatten() { - let xs = [0, 3, 6]; - let ys = [0, 1, 2, 3, 4, 5, 6, 7, 8]; - let it = xs.iter().map(|&x| (x..).step_by(1).take(3)).flatten(); - let mut i = 0; - for x in it { - assert_eq!(x, ys[i]); - i += 1; - } - assert_eq!(i, ys.len()); -} - -/// Tests `Flatten::fold` with items already picked off the front and back, -/// to make sure all parts of the `Flatten` are folded correctly. -#[test] -fn test_iterator_flatten_fold() { - let xs = [0, 3, 6]; - let ys = [1, 2, 3, 4, 5, 6, 7]; - let mut it = xs.iter().map(|&x| x..x + 3).flatten(); - assert_eq!(it.next(), Some(0)); - assert_eq!(it.next_back(), Some(8)); - let i = it.fold(0, |i, x| { - assert_eq!(x, ys[i]); - i + 1 - }); - assert_eq!(i, ys.len()); - - let mut it = xs.iter().map(|&x| x..x + 3).flatten(); - assert_eq!(it.next(), Some(0)); - assert_eq!(it.next_back(), Some(8)); - let i = it.rfold(ys.len(), |i, x| { - assert_eq!(x, ys[i - 1]); - i - 1 - }); - assert_eq!(i, 0); -} - -#[test] -fn test_inspect() { - let xs = [1, 2, 3, 4]; - let mut n = 0; - - let ys = xs.iter().cloned().inspect(|_| n += 1).collect::>(); - - assert_eq!(n, xs.len()); - assert_eq!(&xs[..], &ys[..]); -} - -#[test] -fn test_inspect_fold() { - let xs = [1, 2, 3, 4]; - let mut n = 0; - { - let it = xs.iter().inspect(|_| n += 1); - let i = it.fold(0, |i, &x| { - assert_eq!(x, xs[i]); - i + 1 - }); - assert_eq!(i, xs.len()); - } - assert_eq!(n, xs.len()); - - let mut n = 0; - { - let it = xs.iter().inspect(|_| n += 1); - let i = it.rfold(xs.len(), |i, &x| { - assert_eq!(x, xs[i - 1]); - i - 1 - }); - assert_eq!(i, 0); - } - assert_eq!(n, xs.len()); -} - -#[test] -fn test_cycle() { - let cycle_len = 3; - let it = (0..).step_by(1).take(cycle_len).cycle(); - assert_eq!(it.size_hint(), (usize::MAX, None)); - for (i, x) in it.take(100).enumerate() { - assert_eq!(i % cycle_len, x); - } - - let mut it = (0..).step_by(1).take(0).cycle(); - assert_eq!(it.size_hint(), (0, Some(0))); - assert_eq!(it.next(), None); - - assert_eq!(empty::().cycle().fold(0, |acc, x| acc + x), 0); - - assert_eq!(once(1).cycle().skip(1).take(4).fold(0, |acc, x| acc + x), 4); - - assert_eq!((0..10).cycle().take(5).sum::(), 10); - assert_eq!((0..10).cycle().take(15).sum::(), 55); - assert_eq!((0..10).cycle().take(25).sum::(), 100); - - let mut iter = (0..10).cycle(); - iter.nth(14); - assert_eq!(iter.take(8).sum::(), 38); - - let mut iter = (0..10).cycle(); - iter.nth(9); - assert_eq!(iter.take(3).sum::(), 3); -} - -#[test] -fn test_iterator_nth() { - let v: &[_] = &[0, 1, 2, 3, 4]; - for i in 0..v.len() { - assert_eq!(v.iter().nth(i).unwrap(), &v[i]); - } - assert_eq!(v.iter().nth(v.len()), None); -} - -#[test] -fn test_iterator_nth_back() { - let v: &[_] = &[0, 1, 2, 3, 4]; - for i in 0..v.len() { - assert_eq!(v.iter().nth_back(i).unwrap(), &v[v.len() - 1 - i]); - } - assert_eq!(v.iter().nth_back(v.len()), None); -} - -#[test] -fn test_iterator_rev_nth_back() { - let v: &[_] = &[0, 1, 2, 3, 4]; - for i in 0..v.len() { - assert_eq!(v.iter().rev().nth_back(i).unwrap(), &v[i]); - } - assert_eq!(v.iter().rev().nth_back(v.len()), None); -} - -#[test] -fn test_iterator_rev_nth() { - let v: &[_] = &[0, 1, 2, 3, 4]; - for i in 0..v.len() { - assert_eq!(v.iter().rev().nth(i).unwrap(), &v[v.len() - 1 - i]); - } - assert_eq!(v.iter().rev().nth(v.len()), None); -} - -#[test] -fn test_iterator_last() { - let v: &[_] = &[0, 1, 2, 3, 4]; - assert_eq!(v.iter().last().unwrap(), &4); - assert_eq!(v[..1].iter().last().unwrap(), &0); -} - -#[test] -fn test_iterator_len() { - let v: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - assert_eq!(v[..4].iter().count(), 4); - assert_eq!(v[..10].iter().count(), 10); - assert_eq!(v[..0].iter().count(), 0); -} - -#[test] -fn test_iterator_sum() { - let v: &[i32] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - assert_eq!(v[..4].iter().cloned().sum::(), 6); - assert_eq!(v.iter().cloned().sum::(), 55); - assert_eq!(v[..0].iter().cloned().sum::(), 0); -} - -#[test] -fn test_iterator_sum_result() { - let v: &[Result] = &[Ok(1), Ok(2), Ok(3), Ok(4)]; - assert_eq!(v.iter().cloned().sum::>(), Ok(10)); - let v: &[Result] = &[Ok(1), Err(()), Ok(3), Ok(4)]; - assert_eq!(v.iter().cloned().sum::>(), Err(())); - - #[derive(PartialEq, Debug)] - struct S(Result); - - impl Sum> for S { - fn sum>>(mut iter: I) -> Self { - // takes the sum by repeatedly calling `next` on `iter`, - // thus testing that repeated calls to `ResultShunt::try_fold` - // produce the expected results - Self(iter.by_ref().sum()) - } - } - - let v: &[Result] = &[Ok(1), Ok(2), Ok(3), Ok(4)]; - assert_eq!(v.iter().cloned().sum::(), S(Ok(10))); - let v: &[Result] = &[Ok(1), Err(()), Ok(3), Ok(4)]; - assert_eq!(v.iter().cloned().sum::(), S(Err(()))); -} - -#[test] -fn test_iterator_sum_option() { - let v: &[Option] = &[Some(1), Some(2), Some(3), Some(4)]; - assert_eq!(v.iter().cloned().sum::>(), Some(10)); - let v: &[Option] = &[Some(1), None, Some(3), Some(4)]; - assert_eq!(v.iter().cloned().sum::>(), None); -} - -#[test] -fn test_iterator_product() { - let v: &[i32] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - assert_eq!(v[..4].iter().cloned().product::(), 0); - assert_eq!(v[1..5].iter().cloned().product::(), 24); - assert_eq!(v[..0].iter().cloned().product::(), 1); -} - -#[test] -fn test_iterator_product_result() { - let v: &[Result] = &[Ok(1), Ok(2), Ok(3), Ok(4)]; - assert_eq!(v.iter().cloned().product::>(), Ok(24)); - let v: &[Result] = &[Ok(1), Err(()), Ok(3), Ok(4)]; - assert_eq!(v.iter().cloned().product::>(), Err(())); -} - -/// A wrapper struct that implements `Eq` and `Ord` based on the wrapped -/// integer modulo 3. Used to test that `Iterator::max` and `Iterator::min` -/// return the correct element if some of them are equal. -#[derive(Debug)] -struct Mod3(i32); - -impl PartialEq for Mod3 { - fn eq(&self, other: &Self) -> bool { - self.0 % 3 == other.0 % 3 - } -} - -impl Eq for Mod3 {} - -impl PartialOrd for Mod3 { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for Mod3 { - fn cmp(&self, other: &Self) -> core::cmp::Ordering { - (self.0 % 3).cmp(&(other.0 % 3)) - } -} - -#[test] -fn test_iterator_product_option() { - let v: &[Option] = &[Some(1), Some(2), Some(3), Some(4)]; - assert_eq!(v.iter().cloned().product::>(), Some(24)); - let v: &[Option] = &[Some(1), None, Some(3), Some(4)]; - assert_eq!(v.iter().cloned().product::>(), None); -} - -#[test] -fn test_iterator_max() { - let v: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - assert_eq!(v[..4].iter().cloned().max(), Some(3)); - assert_eq!(v.iter().cloned().max(), Some(10)); - assert_eq!(v[..0].iter().cloned().max(), None); - assert_eq!(v.iter().cloned().map(Mod3).max().map(|x| x.0), Some(8)); -} - -#[test] -fn test_iterator_min() { - let v: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - assert_eq!(v[..4].iter().cloned().min(), Some(0)); - assert_eq!(v.iter().cloned().min(), Some(0)); - assert_eq!(v[..0].iter().cloned().min(), None); - assert_eq!(v.iter().cloned().map(Mod3).min().map(|x| x.0), Some(0)); -} - -#[test] -fn test_iterator_size_hint() { - let c = (0..).step_by(1); - let v: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; - let v2 = &[10, 11, 12]; - let vi = v.iter(); - - assert_eq!((0..).size_hint(), (usize::MAX, None)); - assert_eq!(c.size_hint(), (usize::MAX, None)); - assert_eq!(vi.clone().size_hint(), (10, Some(10))); - - assert_eq!(c.clone().take(5).size_hint(), (5, Some(5))); - assert_eq!(c.clone().skip(5).size_hint().1, None); - assert_eq!(c.clone().take_while(|_| false).size_hint(), (0, None)); - assert_eq!(c.clone().map_while(|_| None::<()>).size_hint(), (0, None)); - assert_eq!(c.clone().skip_while(|_| false).size_hint(), (0, None)); - assert_eq!(c.clone().enumerate().size_hint(), (usize::MAX, None)); - assert_eq!(c.clone().chain(vi.clone().cloned()).size_hint(), (usize::MAX, None)); - assert_eq!(c.clone().zip(vi.clone()).size_hint(), (10, Some(10))); - assert_eq!(c.clone().scan(0, |_, _| Some(0)).size_hint(), (0, None)); - assert_eq!(c.clone().filter(|_| false).size_hint(), (0, None)); - assert_eq!(c.clone().map(|_| 0).size_hint(), (usize::MAX, None)); - assert_eq!(c.filter_map(|_| Some(0)).size_hint(), (0, None)); - - assert_eq!(vi.clone().take(5).size_hint(), (5, Some(5))); - assert_eq!(vi.clone().take(12).size_hint(), (10, Some(10))); - assert_eq!(vi.clone().skip(3).size_hint(), (7, Some(7))); - assert_eq!(vi.clone().skip(12).size_hint(), (0, Some(0))); - assert_eq!(vi.clone().take_while(|_| false).size_hint(), (0, Some(10))); - assert_eq!(vi.clone().map_while(|_| None::<()>).size_hint(), (0, Some(10))); - assert_eq!(vi.clone().skip_while(|_| false).size_hint(), (0, Some(10))); - assert_eq!(vi.clone().enumerate().size_hint(), (10, Some(10))); - assert_eq!(vi.clone().chain(v2).size_hint(), (13, Some(13))); - assert_eq!(vi.clone().zip(v2).size_hint(), (3, Some(3))); - assert_eq!(vi.clone().scan(0, |_, _| Some(0)).size_hint(), (0, Some(10))); - assert_eq!(vi.clone().filter(|_| false).size_hint(), (0, Some(10))); - assert_eq!(vi.clone().map(|&i| i + 1).size_hint(), (10, Some(10))); - assert_eq!(vi.filter_map(|_| Some(0)).size_hint(), (0, Some(10))); -} - -#[test] -fn test_collect() { - let a = vec![1, 2, 3, 4, 5]; - let b: Vec = a.iter().cloned().collect(); - assert!(a == b); -} - -#[test] -fn test_all() { - let v: Box<[isize]> = Box::new([1, 2, 3, 4, 5]); - assert!(v.iter().all(|&x| x < 10)); - assert!(!v.iter().all(|&x| x % 2 == 0)); - assert!(!v.iter().all(|&x| x > 100)); - assert!(v[..0].iter().all(|_| panic!())); -} - -#[test] -fn test_any() { - let v: Box<[isize]> = Box::new([1, 2, 3, 4, 5]); - assert!(v.iter().any(|&x| x < 10)); - assert!(v.iter().any(|&x| x % 2 == 0)); - assert!(!v.iter().any(|&x| x > 100)); - assert!(!v[..0].iter().any(|_| panic!())); -} - -#[test] -fn test_find() { - let v: &[isize] = &[1, 3, 9, 27, 103, 14, 11]; - assert_eq!(*v.iter().find(|&&x| x & 1 == 0).unwrap(), 14); - assert_eq!(*v.iter().find(|&&x| x % 3 == 0).unwrap(), 3); - assert!(v.iter().find(|&&x| x % 12 == 0).is_none()); -} - -#[test] -fn test_find_map() { - let xs: &[isize] = &[]; - assert_eq!(xs.iter().find_map(half_if_even), None); - let xs: &[isize] = &[3, 5]; - assert_eq!(xs.iter().find_map(half_if_even), None); - let xs: &[isize] = &[4, 5]; - assert_eq!(xs.iter().find_map(half_if_even), Some(2)); - let xs: &[isize] = &[3, 6]; - assert_eq!(xs.iter().find_map(half_if_even), Some(3)); - - let xs: &[isize] = &[1, 2, 3, 4, 5, 6, 7]; - let mut iter = xs.iter(); - assert_eq!(iter.find_map(half_if_even), Some(1)); - assert_eq!(iter.find_map(half_if_even), Some(2)); - assert_eq!(iter.find_map(half_if_even), Some(3)); - assert_eq!(iter.next(), Some(&7)); - - fn half_if_even(x: &isize) -> Option { - if x % 2 == 0 { Some(x / 2) } else { None } - } -} - -#[test] -fn test_try_find() { - let xs: &[isize] = &[]; - assert_eq!(xs.iter().try_find(testfn), Ok(None)); - let xs: &[isize] = &[1, 2, 3, 4]; - assert_eq!(xs.iter().try_find(testfn), Ok(Some(&2))); - let xs: &[isize] = &[1, 3, 4]; - assert_eq!(xs.iter().try_find(testfn), Err(())); - - let xs: &[isize] = &[1, 2, 3, 4, 5, 6, 7]; - let mut iter = xs.iter(); - assert_eq!(iter.try_find(testfn), Ok(Some(&2))); - assert_eq!(iter.try_find(testfn), Err(())); - assert_eq!(iter.next(), Some(&5)); - - fn testfn(x: &&isize) -> Result { - if **x == 2 { - return Ok(true); - } - if **x == 4 { - return Err(()); - } - Ok(false) - } -} - -#[test] -fn test_try_find_api_usability() -> Result<(), Box> { - let a = ["1", "2"]; - - let is_my_num = |s: &str, search: i32| -> Result { - Ok(s.parse::()? == search) - }; - - let val = a.iter().try_find(|&&s| is_my_num(s, 2))?; - assert_eq!(val, Some(&"2")); - - Ok(()) -} - -#[test] -fn test_position() { - let v = &[1, 3, 9, 27, 103, 14, 11]; - assert_eq!(v.iter().position(|x| *x & 1 == 0).unwrap(), 5); - assert_eq!(v.iter().position(|x| *x % 3 == 0).unwrap(), 1); - assert!(v.iter().position(|x| *x % 12 == 0).is_none()); -} - -#[test] -fn test_count() { - let xs = &[1, 2, 2, 1, 5, 9, 0, 2]; - assert_eq!(xs.iter().filter(|x| **x == 2).count(), 3); - assert_eq!(xs.iter().filter(|x| **x == 5).count(), 1); - assert_eq!(xs.iter().filter(|x| **x == 95).count(), 0); -} - -#[test] -fn test_max_by_key() { - let xs: &[isize] = &[-3, 0, 1, 5, -10]; - assert_eq!(*xs.iter().max_by_key(|x| x.abs()).unwrap(), -10); -} - -#[test] -fn test_max_by() { - let xs: &[isize] = &[-3, 0, 1, 5, -10]; - assert_eq!(*xs.iter().max_by(|x, y| x.abs().cmp(&y.abs())).unwrap(), -10); -} - -#[test] -fn test_min_by_key() { - let xs: &[isize] = &[-3, 0, 1, 5, -10]; - assert_eq!(*xs.iter().min_by_key(|x| x.abs()).unwrap(), 0); -} - -#[test] -fn test_min_by() { - let xs: &[isize] = &[-3, 0, 1, 5, -10]; - assert_eq!(*xs.iter().min_by(|x, y| x.abs().cmp(&y.abs())).unwrap(), 0); -} - -#[test] -fn test_by_ref() { - let mut xs = 0..10; - // sum the first five values - let partial_sum = xs.by_ref().take(5).fold(0, |a, b| a + b); - assert_eq!(partial_sum, 10); - assert_eq!(xs.next(), Some(5)); -} - -#[test] -fn test_rev() { - let xs = [2, 4, 6, 8, 10, 12, 14, 16]; - let mut it = xs.iter(); - it.next(); - it.next(); - assert!(it.rev().cloned().collect::>() == vec![16, 14, 12, 10, 8, 6]); -} - -#[test] -fn test_copied() { - let xs = [2, 4, 6, 8]; - - let mut it = xs.iter().copied(); - assert_eq!(it.len(), 4); - assert_eq!(it.next(), Some(2)); - assert_eq!(it.len(), 3); - assert_eq!(it.next(), Some(4)); - assert_eq!(it.len(), 2); - assert_eq!(it.next_back(), Some(8)); - assert_eq!(it.len(), 1); - assert_eq!(it.next_back(), Some(6)); - assert_eq!(it.len(), 0); - assert_eq!(it.next_back(), None); -} - -#[test] -fn test_cloned() { - let xs = [2, 4, 6, 8]; - - let mut it = xs.iter().cloned(); - assert_eq!(it.len(), 4); - assert_eq!(it.next(), Some(2)); - assert_eq!(it.len(), 3); - assert_eq!(it.next(), Some(4)); - assert_eq!(it.len(), 2); - assert_eq!(it.next_back(), Some(8)); - assert_eq!(it.len(), 1); - assert_eq!(it.next_back(), Some(6)); - assert_eq!(it.len(), 0); - assert_eq!(it.next_back(), None); -} - -#[test] -fn test_cloned_side_effects() { - let mut count = 0; - { - let iter = [1, 2, 3] - .iter() - .map(|x| { - count += 1; - x - }) - .cloned() - .zip(&[1]); - for _ in iter {} - } - assert_eq!(count, 2); -} - -#[test] -fn test_double_ended_map() { - let xs = [1, 2, 3, 4, 5, 6]; - let mut it = xs.iter().map(|&x| x * -1); - assert_eq!(it.next(), Some(-1)); - assert_eq!(it.next(), Some(-2)); - assert_eq!(it.next_back(), Some(-6)); - assert_eq!(it.next_back(), Some(-5)); - assert_eq!(it.next(), Some(-3)); - assert_eq!(it.next_back(), Some(-4)); - assert_eq!(it.next(), None); -} - -#[test] -fn test_double_ended_enumerate() { - let xs = [1, 2, 3, 4, 5, 6]; - let mut it = xs.iter().cloned().enumerate(); - assert_eq!(it.next(), Some((0, 1))); - assert_eq!(it.next(), Some((1, 2))); - assert_eq!(it.next_back(), Some((5, 6))); - assert_eq!(it.next_back(), Some((4, 5))); - assert_eq!(it.next_back(), Some((3, 4))); - assert_eq!(it.next_back(), Some((2, 3))); - assert_eq!(it.next(), None); -} - -#[test] -fn test_double_ended_zip() { - let xs = [1, 2, 3, 4, 5, 6]; - let ys = [1, 2, 3, 7]; - let a = xs.iter().cloned(); - let b = ys.iter().cloned(); - let mut it = a.zip(b); - assert_eq!(it.next(), Some((1, 1))); - assert_eq!(it.next(), Some((2, 2))); - assert_eq!(it.next_back(), Some((4, 7))); - assert_eq!(it.next_back(), Some((3, 3))); - assert_eq!(it.next(), None); -} - -#[test] -fn test_double_ended_filter() { - let xs = [1, 2, 3, 4, 5, 6]; - let mut it = xs.iter().filter(|&x| *x & 1 == 0); - assert_eq!(it.next_back().unwrap(), &6); - assert_eq!(it.next_back().unwrap(), &4); - assert_eq!(it.next().unwrap(), &2); - assert_eq!(it.next_back(), None); -} - -#[test] -fn test_double_ended_filter_map() { - let xs = [1, 2, 3, 4, 5, 6]; - let mut it = xs.iter().filter_map(|&x| if x & 1 == 0 { Some(x * 2) } else { None }); - assert_eq!(it.next_back().unwrap(), 12); - assert_eq!(it.next_back().unwrap(), 8); - assert_eq!(it.next().unwrap(), 4); - assert_eq!(it.next_back(), None); -} - -#[test] -fn test_double_ended_chain() { - let xs = [1, 2, 3, 4, 5]; - let ys = [7, 9, 11]; - let mut it = xs.iter().chain(&ys).rev(); - assert_eq!(it.next().unwrap(), &11); - assert_eq!(it.next().unwrap(), &9); - assert_eq!(it.next_back().unwrap(), &1); - assert_eq!(it.next_back().unwrap(), &2); - assert_eq!(it.next_back().unwrap(), &3); - assert_eq!(it.next_back().unwrap(), &4); - assert_eq!(it.next_back().unwrap(), &5); - assert_eq!(it.next_back().unwrap(), &7); - assert_eq!(it.next_back(), None); - - // test that .chain() is well behaved with an unfused iterator - struct CrazyIterator(bool); - impl CrazyIterator { - fn new() -> CrazyIterator { - CrazyIterator(false) - } - } - impl Iterator for CrazyIterator { - type Item = i32; - fn next(&mut self) -> Option { - if self.0 { - Some(99) - } else { - self.0 = true; - None - } - } - } - - impl DoubleEndedIterator for CrazyIterator { - fn next_back(&mut self) -> Option { - self.next() - } - } - - assert_eq!(CrazyIterator::new().chain(0..10).rev().last(), Some(0)); - assert!((0..10).chain(CrazyIterator::new()).rev().any(|i| i == 0)); -} - -#[test] -fn test_rposition() { - fn f(xy: &(isize, char)) -> bool { - let (_x, y) = *xy; - y == 'b' - } - fn g(xy: &(isize, char)) -> bool { - let (_x, y) = *xy; - y == 'd' - } - let v = [(0, 'a'), (1, 'b'), (2, 'c'), (3, 'b')]; - - assert_eq!(v.iter().rposition(f), Some(3)); - assert!(v.iter().rposition(g).is_none()); -} - -#[test] -fn test_rev_rposition() { - let v = [0, 0, 1, 1]; - assert_eq!(v.iter().rev().rposition(|&x| x == 1), Some(1)); -} - -#[test] -#[should_panic] -fn test_rposition_panic() { - let v: [(Box<_>, Box<_>); 4] = [(box 0, box 0), (box 0, box 0), (box 0, box 0), (box 0, box 0)]; - let mut i = 0; - v.iter().rposition(|_elt| { - if i == 2 { - panic!() - } - i += 1; - false - }); -} - -#[test] -fn test_double_ended_flat_map() { - let u = [0, 1]; - let v = [5, 6, 7, 8]; - let mut it = u.iter().flat_map(|x| &v[*x..v.len()]); - assert_eq!(it.next_back().unwrap(), &8); - assert_eq!(it.next().unwrap(), &5); - assert_eq!(it.next_back().unwrap(), &7); - assert_eq!(it.next_back().unwrap(), &6); - assert_eq!(it.next_back().unwrap(), &8); - assert_eq!(it.next().unwrap(), &6); - assert_eq!(it.next_back().unwrap(), &7); - assert_eq!(it.next_back(), None); - assert_eq!(it.next(), None); - assert_eq!(it.next_back(), None); -} - -#[test] -fn test_double_ended_flatten() { - let u = [0, 1]; - let v = [5, 6, 7, 8]; - let mut it = u.iter().map(|x| &v[*x..v.len()]).flatten(); - assert_eq!(it.next_back().unwrap(), &8); - assert_eq!(it.next().unwrap(), &5); - assert_eq!(it.next_back().unwrap(), &7); - assert_eq!(it.next_back().unwrap(), &6); - assert_eq!(it.next_back().unwrap(), &8); - assert_eq!(it.next().unwrap(), &6); - assert_eq!(it.next_back().unwrap(), &7); - assert_eq!(it.next_back(), None); - assert_eq!(it.next(), None); - assert_eq!(it.next_back(), None); -} - -#[test] -fn test_double_ended_range() { - assert_eq!((11..14).rev().collect::>(), [13, 12, 11]); - for _ in (10..0).rev() { - panic!("unreachable"); - } - - assert_eq!((11..14).rev().collect::>(), [13, 12, 11]); - for _ in (10..0).rev() { - panic!("unreachable"); - } -} - -#[test] -fn test_range() { - assert_eq!((0..5).collect::>(), [0, 1, 2, 3, 4]); - assert_eq!((-10..-1).collect::>(), [-10, -9, -8, -7, -6, -5, -4, -3, -2]); - assert_eq!((0..5).rev().collect::>(), [4, 3, 2, 1, 0]); - assert_eq!((200..-5).count(), 0); - assert_eq!((200..-5).rev().count(), 0); - assert_eq!((200..200).count(), 0); - assert_eq!((200..200).rev().count(), 0); - - assert_eq!((0..100).size_hint(), (100, Some(100))); - // this test is only meaningful when sizeof usize < sizeof u64 - assert_eq!((usize::MAX - 1..usize::MAX).size_hint(), (1, Some(1))); - assert_eq!((-10..-1).size_hint(), (9, Some(9))); - assert_eq!((-1..-10).size_hint(), (0, Some(0))); - - assert_eq!((-70..58).size_hint(), (128, Some(128))); - assert_eq!((-128..127).size_hint(), (255, Some(255))); - assert_eq!( - (-2..isize::MAX).size_hint(), - (isize::MAX as usize + 2, Some(isize::MAX as usize + 2)) - ); -} - -#[test] -fn test_range_exhaustion() { - let mut r = 10..10; - assert!(r.is_empty()); - assert_eq!(r.next(), None); - assert_eq!(r.next_back(), None); - assert_eq!(r, 10..10); - - let mut r = 10..12; - assert_eq!(r.next(), Some(10)); - assert_eq!(r.next(), Some(11)); - assert!(r.is_empty()); - assert_eq!(r, 12..12); - assert_eq!(r.next(), None); - - let mut r = 10..12; - assert_eq!(r.next_back(), Some(11)); - assert_eq!(r.next_back(), Some(10)); - assert!(r.is_empty()); - assert_eq!(r, 10..10); - assert_eq!(r.next_back(), None); - - let mut r = 100..10; - assert!(r.is_empty()); - assert_eq!(r.next(), None); - assert_eq!(r.next_back(), None); - assert_eq!(r, 100..10); -} - -#[test] -fn test_range_inclusive_exhaustion() { - let mut r = 10..=10; - assert_eq!(r.next(), Some(10)); - assert!(r.is_empty()); - assert_eq!(r.next(), None); - assert_eq!(r.next(), None); - - assert_eq!(*r.start(), 10); - assert_eq!(*r.end(), 10); - assert_ne!(r, 10..=10); - - let mut r = 10..=10; - assert_eq!(r.next_back(), Some(10)); - assert!(r.is_empty()); - assert_eq!(r.next_back(), None); - - assert_eq!(*r.start(), 10); - assert_eq!(*r.end(), 10); - assert_ne!(r, 10..=10); - - let mut r = 10..=12; - assert_eq!(r.next(), Some(10)); - assert_eq!(r.next(), Some(11)); - assert_eq!(r.next(), Some(12)); - assert!(r.is_empty()); - assert_eq!(r.next(), None); - - let mut r = 10..=12; - assert_eq!(r.next_back(), Some(12)); - assert_eq!(r.next_back(), Some(11)); - assert_eq!(r.next_back(), Some(10)); - assert!(r.is_empty()); - assert_eq!(r.next_back(), None); - - let mut r = 10..=12; - assert_eq!(r.nth(2), Some(12)); - assert!(r.is_empty()); - assert_eq!(r.next(), None); - - let mut r = 10..=12; - assert_eq!(r.nth(5), None); - assert!(r.is_empty()); - assert_eq!(r.next(), None); - - let mut r = 100..=10; - assert_eq!(r.next(), None); - assert!(r.is_empty()); - assert_eq!(r.next(), None); - assert_eq!(r.next(), None); - assert_eq!(r, 100..=10); - - let mut r = 100..=10; - assert_eq!(r.next_back(), None); - assert!(r.is_empty()); - assert_eq!(r.next_back(), None); - assert_eq!(r.next_back(), None); - assert_eq!(r, 100..=10); -} - -#[test] -fn test_range_nth() { - assert_eq!((10..15).nth(0), Some(10)); - assert_eq!((10..15).nth(1), Some(11)); - assert_eq!((10..15).nth(4), Some(14)); - assert_eq!((10..15).nth(5), None); - - let mut r = 10..20; - assert_eq!(r.nth(2), Some(12)); - assert_eq!(r, 13..20); - assert_eq!(r.nth(2), Some(15)); - assert_eq!(r, 16..20); - assert_eq!(r.nth(10), None); - assert_eq!(r, 20..20); -} - -#[test] -fn test_range_nth_back() { - assert_eq!((10..15).nth_back(0), Some(14)); - assert_eq!((10..15).nth_back(1), Some(13)); - assert_eq!((10..15).nth_back(4), Some(10)); - assert_eq!((10..15).nth_back(5), None); - assert_eq!((-120..80_i8).nth_back(199), Some(-120)); - - let mut r = 10..20; - assert_eq!(r.nth_back(2), Some(17)); - assert_eq!(r, 10..17); - assert_eq!(r.nth_back(2), Some(14)); - assert_eq!(r, 10..14); - assert_eq!(r.nth_back(10), None); - assert_eq!(r, 10..10); -} - -#[test] -fn test_range_from_nth() { - assert_eq!((10..).nth(0), Some(10)); - assert_eq!((10..).nth(1), Some(11)); - assert_eq!((10..).nth(4), Some(14)); - - let mut r = 10..; - assert_eq!(r.nth(2), Some(12)); - assert_eq!(r, 13..); - assert_eq!(r.nth(2), Some(15)); - assert_eq!(r, 16..); - assert_eq!(r.nth(10), Some(26)); - assert_eq!(r, 27..); - - assert_eq!((0..).size_hint(), (usize::MAX, None)); -} - -fn is_trusted_len(_: I) {} - -#[test] -fn test_range_from_take() { - let mut it = (0..).take(3); - assert_eq!(it.next(), Some(0)); - assert_eq!(it.next(), Some(1)); - assert_eq!(it.next(), Some(2)); - assert_eq!(it.next(), None); - is_trusted_len((0..).take(3)); - assert_eq!((0..).take(3).size_hint(), (3, Some(3))); - assert_eq!((0..).take(0).size_hint(), (0, Some(0))); - assert_eq!((0..).take(usize::MAX).size_hint(), (usize::MAX, Some(usize::MAX))); -} - -#[test] -fn test_range_from_take_collect() { - let v: Vec<_> = (0..).take(3).collect(); - assert_eq!(v, vec![0, 1, 2]); -} - -#[test] -fn test_range_inclusive_nth() { - assert_eq!((10..=15).nth(0), Some(10)); - assert_eq!((10..=15).nth(1), Some(11)); - assert_eq!((10..=15).nth(5), Some(15)); - assert_eq!((10..=15).nth(6), None); - - let mut exhausted_via_next = 10_u8..=20; - while exhausted_via_next.next().is_some() {} - - let mut r = 10_u8..=20; - assert_eq!(r.nth(2), Some(12)); - assert_eq!(r, 13..=20); - assert_eq!(r.nth(2), Some(15)); - assert_eq!(r, 16..=20); - assert_eq!(r.is_empty(), false); - assert_eq!(ExactSizeIterator::is_empty(&r), false); - assert_eq!(r.nth(10), None); - assert_eq!(r.is_empty(), true); - assert_eq!(r, exhausted_via_next); - assert_eq!(ExactSizeIterator::is_empty(&r), true); -} - -#[test] -fn test_range_inclusive_nth_back() { - assert_eq!((10..=15).nth_back(0), Some(15)); - assert_eq!((10..=15).nth_back(1), Some(14)); - assert_eq!((10..=15).nth_back(5), Some(10)); - assert_eq!((10..=15).nth_back(6), None); - assert_eq!((-120..=80_i8).nth_back(200), Some(-120)); - - let mut exhausted_via_next_back = 10_u8..=20; - while exhausted_via_next_back.next_back().is_some() {} - - let mut r = 10_u8..=20; - assert_eq!(r.nth_back(2), Some(18)); - assert_eq!(r, 10..=17); - assert_eq!(r.nth_back(2), Some(15)); - assert_eq!(r, 10..=14); - assert_eq!(r.is_empty(), false); - assert_eq!(ExactSizeIterator::is_empty(&r), false); - assert_eq!(r.nth_back(10), None); - assert_eq!(r.is_empty(), true); - assert_eq!(r, exhausted_via_next_back); - assert_eq!(ExactSizeIterator::is_empty(&r), true); -} - -#[test] -fn test_range_step() { - #![allow(deprecated)] - - assert_eq!((0..20).step_by(5).collect::>(), [0, 5, 10, 15]); - assert_eq!((1..21).rev().step_by(5).collect::>(), [20, 15, 10, 5]); - assert_eq!((1..21).rev().step_by(6).collect::>(), [20, 14, 8, 2]); - assert_eq!((200..255).step_by(50).collect::>(), [200, 250]); - assert_eq!((200..-5).step_by(1).collect::>(), []); - assert_eq!((200..200).step_by(1).collect::>(), []); - - assert_eq!((0..20).step_by(1).size_hint(), (20, Some(20))); - assert_eq!((0..20).step_by(21).size_hint(), (1, Some(1))); - assert_eq!((0..20).step_by(5).size_hint(), (4, Some(4))); - assert_eq!((1..21).rev().step_by(5).size_hint(), (4, Some(4))); - assert_eq!((1..21).rev().step_by(6).size_hint(), (4, Some(4))); - assert_eq!((20..-5).step_by(1).size_hint(), (0, Some(0))); - assert_eq!((20..20).step_by(1).size_hint(), (0, Some(0))); - assert_eq!((i8::MIN..i8::MAX).step_by(-(i8::MIN as i32) as usize).size_hint(), (2, Some(2))); - assert_eq!((i16::MIN..i16::MAX).step_by(i16::MAX as usize).size_hint(), (3, Some(3))); - assert_eq!((isize::MIN..isize::MAX).step_by(1).size_hint(), (usize::MAX, Some(usize::MAX))); -} - -#[test] -fn test_step_by_skip() { - assert_eq!((0..640).step_by(128).skip(1).collect::>(), [128, 256, 384, 512]); - assert_eq!((0..=50).step_by(10).nth(3), Some(30)); - assert_eq!((200..=255u8).step_by(10).nth(3), Some(230)); -} - -#[test] -fn test_range_inclusive_step() { - assert_eq!((0..=50).step_by(10).collect::>(), [0, 10, 20, 30, 40, 50]); - assert_eq!((0..=5).step_by(1).collect::>(), [0, 1, 2, 3, 4, 5]); - assert_eq!((200..=255u8).step_by(10).collect::>(), [200, 210, 220, 230, 240, 250]); - assert_eq!((250..=255u8).step_by(1).collect::>(), [250, 251, 252, 253, 254, 255]); -} - -#[test] -fn test_range_last_max() { - assert_eq!((0..20).last(), Some(19)); - assert_eq!((-20..0).last(), Some(-1)); - assert_eq!((5..5).last(), None); - - assert_eq!((0..20).max(), Some(19)); - assert_eq!((-20..0).max(), Some(-1)); - assert_eq!((5..5).max(), None); -} - -#[test] -fn test_range_inclusive_last_max() { - assert_eq!((0..=20).last(), Some(20)); - assert_eq!((-20..=0).last(), Some(0)); - assert_eq!((5..=5).last(), Some(5)); - let mut r = 10..=10; - r.next(); - assert_eq!(r.last(), None); - - assert_eq!((0..=20).max(), Some(20)); - assert_eq!((-20..=0).max(), Some(0)); - assert_eq!((5..=5).max(), Some(5)); - let mut r = 10..=10; - r.next(); - assert_eq!(r.max(), None); -} - -#[test] -fn test_range_min() { - assert_eq!((0..20).min(), Some(0)); - assert_eq!((-20..0).min(), Some(-20)); - assert_eq!((5..5).min(), None); -} - -#[test] -fn test_range_inclusive_min() { - assert_eq!((0..=20).min(), Some(0)); - assert_eq!((-20..=0).min(), Some(-20)); - assert_eq!((5..=5).min(), Some(5)); - let mut r = 10..=10; - r.next(); - assert_eq!(r.min(), None); -} - -#[test] -fn test_range_inclusive_folds() { - assert_eq!((1..=10).sum::(), 55); - assert_eq!((1..=10).rev().sum::(), 55); - - let mut it = 44..=50; - assert_eq!(it.try_fold(0, i8::checked_add), None); - assert_eq!(it, 47..=50); - assert_eq!(it.try_fold(0, i8::checked_add), None); - assert_eq!(it, 50..=50); - assert_eq!(it.try_fold(0, i8::checked_add), Some(50)); - assert!(it.is_empty()); - assert_eq!(it.try_fold(0, i8::checked_add), Some(0)); - assert!(it.is_empty()); - - let mut it = 40..=47; - assert_eq!(it.try_rfold(0, i8::checked_add), None); - assert_eq!(it, 40..=44); - assert_eq!(it.try_rfold(0, i8::checked_add), None); - assert_eq!(it, 40..=41); - assert_eq!(it.try_rfold(0, i8::checked_add), Some(81)); - assert!(it.is_empty()); - assert_eq!(it.try_rfold(0, i8::checked_add), Some(0)); - assert!(it.is_empty()); - - let mut it = 10..=20; - assert_eq!(it.try_fold(0, |a, b| Some(a + b)), Some(165)); - assert!(it.is_empty()); - assert_eq!(it.try_fold(0, |a, b| Some(a + b)), Some(0)); - assert!(it.is_empty()); - - let mut it = 10..=20; - assert_eq!(it.try_rfold(0, |a, b| Some(a + b)), Some(165)); - assert!(it.is_empty()); - assert_eq!(it.try_rfold(0, |a, b| Some(a + b)), Some(0)); - assert!(it.is_empty()); -} - -#[test] -fn test_range_size_hint() { - use core::usize::MAX as UMAX; - assert_eq!((0..0usize).size_hint(), (0, Some(0))); - assert_eq!((0..100usize).size_hint(), (100, Some(100))); - assert_eq!((0..UMAX).size_hint(), (UMAX, Some(UMAX))); - - let umax = u128::try_from(UMAX).unwrap(); - assert_eq!((0..0u128).size_hint(), (0, Some(0))); - assert_eq!((0..100u128).size_hint(), (100, Some(100))); - assert_eq!((0..umax).size_hint(), (UMAX, Some(UMAX))); - assert_eq!((0..umax + 1).size_hint(), (UMAX, None)); - - use core::isize::{MAX as IMAX, MIN as IMIN}; - assert_eq!((0..0isize).size_hint(), (0, Some(0))); - assert_eq!((-100..100isize).size_hint(), (200, Some(200))); - assert_eq!((IMIN..IMAX).size_hint(), (UMAX, Some(UMAX))); - - let imin = i128::try_from(IMIN).unwrap(); - let imax = i128::try_from(IMAX).unwrap(); - assert_eq!((0..0i128).size_hint(), (0, Some(0))); - assert_eq!((-100..100i128).size_hint(), (200, Some(200))); - assert_eq!((imin..imax).size_hint(), (UMAX, Some(UMAX))); - assert_eq!((imin..imax + 1).size_hint(), (UMAX, None)); -} - -#[test] -fn test_range_inclusive_size_hint() { - use core::usize::MAX as UMAX; - assert_eq!((1..=0usize).size_hint(), (0, Some(0))); - assert_eq!((0..=0usize).size_hint(), (1, Some(1))); - assert_eq!((0..=100usize).size_hint(), (101, Some(101))); - assert_eq!((0..=UMAX - 1).size_hint(), (UMAX, Some(UMAX))); - assert_eq!((0..=UMAX).size_hint(), (UMAX, None)); - - let umax = u128::try_from(UMAX).unwrap(); - assert_eq!((1..=0u128).size_hint(), (0, Some(0))); - assert_eq!((0..=0u128).size_hint(), (1, Some(1))); - assert_eq!((0..=100u128).size_hint(), (101, Some(101))); - assert_eq!((0..=umax - 1).size_hint(), (UMAX, Some(UMAX))); - assert_eq!((0..=umax).size_hint(), (UMAX, None)); - assert_eq!((0..=umax + 1).size_hint(), (UMAX, None)); - - use core::isize::{MAX as IMAX, MIN as IMIN}; - assert_eq!((0..=-1isize).size_hint(), (0, Some(0))); - assert_eq!((0..=0isize).size_hint(), (1, Some(1))); - assert_eq!((-100..=100isize).size_hint(), (201, Some(201))); - assert_eq!((IMIN..=IMAX - 1).size_hint(), (UMAX, Some(UMAX))); - assert_eq!((IMIN..=IMAX).size_hint(), (UMAX, None)); - - let imin = i128::try_from(IMIN).unwrap(); - let imax = i128::try_from(IMAX).unwrap(); - assert_eq!((0..=-1i128).size_hint(), (0, Some(0))); - assert_eq!((0..=0i128).size_hint(), (1, Some(1))); - assert_eq!((-100..=100i128).size_hint(), (201, Some(201))); - assert_eq!((imin..=imax - 1).size_hint(), (UMAX, Some(UMAX))); - assert_eq!((imin..=imax).size_hint(), (UMAX, None)); - assert_eq!((imin..=imax + 1).size_hint(), (UMAX, None)); -} - -#[test] -fn test_repeat() { - let mut it = repeat(42); - assert_eq!(it.next(), Some(42)); - assert_eq!(it.next(), Some(42)); - assert_eq!(it.next(), Some(42)); - assert_eq!(repeat(42).size_hint(), (usize::MAX, None)); -} - -#[test] -fn test_repeat_take() { - let mut it = repeat(42).take(3); - assert_eq!(it.next(), Some(42)); - assert_eq!(it.next(), Some(42)); - assert_eq!(it.next(), Some(42)); - assert_eq!(it.next(), None); - is_trusted_len(repeat(42).take(3)); - assert_eq!(repeat(42).take(3).size_hint(), (3, Some(3))); - assert_eq!(repeat(42).take(0).size_hint(), (0, Some(0))); - assert_eq!(repeat(42).take(usize::MAX).size_hint(), (usize::MAX, Some(usize::MAX))); -} - -#[test] -fn test_repeat_take_collect() { - let v: Vec<_> = repeat(42).take(3).collect(); - assert_eq!(v, vec![42, 42, 42]); -} - -#[test] -fn test_repeat_with() { - #[derive(PartialEq, Debug)] - struct NotClone(usize); - let mut it = repeat_with(|| NotClone(42)); - assert_eq!(it.next(), Some(NotClone(42))); - assert_eq!(it.next(), Some(NotClone(42))); - assert_eq!(it.next(), Some(NotClone(42))); - assert_eq!(repeat_with(|| NotClone(42)).size_hint(), (usize::MAX, None)); -} - -#[test] -fn test_repeat_with_take() { - let mut it = repeat_with(|| 42).take(3); - assert_eq!(it.next(), Some(42)); - assert_eq!(it.next(), Some(42)); - assert_eq!(it.next(), Some(42)); - assert_eq!(it.next(), None); - is_trusted_len(repeat_with(|| 42).take(3)); - assert_eq!(repeat_with(|| 42).take(3).size_hint(), (3, Some(3))); - assert_eq!(repeat_with(|| 42).take(0).size_hint(), (0, Some(0))); - assert_eq!(repeat_with(|| 42).take(usize::MAX).size_hint(), (usize::MAX, Some(usize::MAX))); -} - -#[test] -fn test_repeat_with_take_collect() { - let mut curr = 1; - let v: Vec<_> = repeat_with(|| { - let tmp = curr; - curr *= 2; - tmp - }) - .take(5) - .collect(); - assert_eq!(v, vec![1, 2, 4, 8, 16]); -} - -#[test] -fn test_successors() { - let mut powers_of_10 = successors(Some(1_u16), |n| n.checked_mul(10)); - assert_eq!(powers_of_10.by_ref().collect::>(), &[1, 10, 100, 1_000, 10_000]); - assert_eq!(powers_of_10.next(), None); - - let mut empty = successors(None::, |_| unimplemented!()); - assert_eq!(empty.next(), None); - assert_eq!(empty.next(), None); -} - -#[test] -fn test_fuse() { - let mut it = 0..3; - assert_eq!(it.len(), 3); - assert_eq!(it.next(), Some(0)); - assert_eq!(it.len(), 2); - assert_eq!(it.next(), Some(1)); - assert_eq!(it.len(), 1); - assert_eq!(it.next(), Some(2)); - assert_eq!(it.len(), 0); - assert_eq!(it.next(), None); - assert_eq!(it.len(), 0); - assert_eq!(it.next(), None); - assert_eq!(it.len(), 0); - assert_eq!(it.next(), None); - assert_eq!(it.len(), 0); -} - -#[test] -fn test_fuse_nth() { - let xs = [0, 1, 2]; - let mut it = xs.iter(); - - assert_eq!(it.len(), 3); - assert_eq!(it.nth(2), Some(&2)); - assert_eq!(it.len(), 0); - assert_eq!(it.nth(2), None); - assert_eq!(it.len(), 0); -} - -#[test] -fn test_fuse_last() { - let xs = [0, 1, 2]; - let it = xs.iter(); - - assert_eq!(it.len(), 3); - assert_eq!(it.last(), Some(&2)); -} - -#[test] -fn test_fuse_count() { - let xs = [0, 1, 2]; - let it = xs.iter(); - - assert_eq!(it.len(), 3); - assert_eq!(it.count(), 3); - // Can't check len now because count consumes. -} - -#[test] -fn test_fuse_fold() { - let xs = [0, 1, 2]; - let it = xs.iter(); // `FusedIterator` - let i = it.fuse().fold(0, |i, &x| { - assert_eq!(x, xs[i]); - i + 1 - }); - assert_eq!(i, xs.len()); - - let it = xs.iter(); // `FusedIterator` - let i = it.fuse().rfold(xs.len(), |i, &x| { - assert_eq!(x, xs[i - 1]); - i - 1 - }); - assert_eq!(i, 0); - - let it = xs.iter().scan((), |_, &x| Some(x)); // `!FusedIterator` - let i = it.fuse().fold(0, |i, x| { - assert_eq!(x, xs[i]); - i + 1 - }); - assert_eq!(i, xs.len()); -} - -#[test] -fn test_once() { - let mut it = once(42); - assert_eq!(it.next(), Some(42)); - assert_eq!(it.next(), None); -} - -#[test] -fn test_once_with() { - let count = Cell::new(0); - let mut it = once_with(|| { - count.set(count.get() + 1); - 42 - }); - - assert_eq!(count.get(), 0); - assert_eq!(it.next(), Some(42)); - assert_eq!(count.get(), 1); - assert_eq!(it.next(), None); - assert_eq!(count.get(), 1); - assert_eq!(it.next(), None); - assert_eq!(count.get(), 1); -} - -#[test] -fn test_empty() { - let mut it = empty::(); - assert_eq!(it.next(), None); -} - -#[test] -fn test_chain_fold() { - let xs = [1, 2, 3]; - let ys = [1, 2, 0]; - - let mut iter = xs.iter().chain(&ys); - iter.next(); - let mut result = Vec::new(); - iter.fold((), |(), &elt| result.push(elt)); - assert_eq!(&[2, 3, 1, 2, 0], &result[..]); -} - -#[test] -fn test_step_replace_unsigned() { - let mut x = 4u32; - let y = x.replace_zero(); - assert_eq!(x, 0); - assert_eq!(y, 4); - - x = 5; - let y = x.replace_one(); - assert_eq!(x, 1); - assert_eq!(y, 5); -} - -#[test] -fn test_step_replace_signed() { - let mut x = 4i32; - let y = x.replace_zero(); - assert_eq!(x, 0); - assert_eq!(y, 4); - - x = 5; - let y = x.replace_one(); - assert_eq!(x, 1); - assert_eq!(y, 5); -} - -#[test] -fn test_step_replace_no_between() { - let mut x = 4u128; - let y = x.replace_zero(); - assert_eq!(x, 0); - assert_eq!(y, 4); - - x = 5; - let y = x.replace_one(); - assert_eq!(x, 1); - assert_eq!(y, 5); -} - -#[test] -fn test_rev_try_folds() { - let f = &|acc, x| i32::checked_add(2 * acc, x); - assert_eq!((1..10).rev().try_fold(7, f), (1..10).try_rfold(7, f)); - assert_eq!((1..10).rev().try_rfold(7, f), (1..10).try_fold(7, f)); - - let a = [10, 20, 30, 40, 100, 60, 70, 80, 90]; - let mut iter = a.iter().rev(); - assert_eq!(iter.try_fold(0_i8, |acc, &x| acc.checked_add(x)), None); - assert_eq!(iter.next(), Some(&70)); - let mut iter = a.iter().rev(); - assert_eq!(iter.try_rfold(0_i8, |acc, &x| acc.checked_add(x)), None); - assert_eq!(iter.next_back(), Some(&60)); -} - -#[test] -fn test_cloned_try_folds() { - let a = [1, 2, 3, 4, 5, 6, 7, 8, 9]; - let f = &|acc, x| i32::checked_add(2 * acc, x); - let f_ref = &|acc, &x| i32::checked_add(2 * acc, x); - assert_eq!(a.iter().cloned().try_fold(7, f), a.iter().try_fold(7, f_ref)); - assert_eq!(a.iter().cloned().try_rfold(7, f), a.iter().try_rfold(7, f_ref)); - - let a = [10, 20, 30, 40, 100, 60, 70, 80, 90]; - let mut iter = a.iter().cloned(); - assert_eq!(iter.try_fold(0_i8, |acc, x| acc.checked_add(x)), None); - assert_eq!(iter.next(), Some(60)); - let mut iter = a.iter().cloned(); - assert_eq!(iter.try_rfold(0_i8, |acc, x| acc.checked_add(x)), None); - assert_eq!(iter.next_back(), Some(70)); -} - -#[test] -fn test_chain_try_folds() { - let c = || (0..10).chain(10..20); - - let f = &|acc, x| i32::checked_add(2 * acc, x); - assert_eq!(c().try_fold(7, f), (0..20).try_fold(7, f)); - assert_eq!(c().try_rfold(7, f), (0..20).rev().try_fold(7, f)); - - let mut iter = c(); - assert_eq!(iter.position(|x| x == 5), Some(5)); - assert_eq!(iter.next(), Some(6), "stopped in front, state Both"); - assert_eq!(iter.position(|x| x == 13), Some(6)); - assert_eq!(iter.next(), Some(14), "stopped in back, state Back"); - assert_eq!(iter.try_fold(0, |acc, x| Some(acc + x)), Some((15..20).sum())); - - let mut iter = c().rev(); // use rev to access try_rfold - assert_eq!(iter.position(|x| x == 15), Some(4)); - assert_eq!(iter.next(), Some(14), "stopped in back, state Both"); - assert_eq!(iter.position(|x| x == 5), Some(8)); - assert_eq!(iter.next(), Some(4), "stopped in front, state Front"); - assert_eq!(iter.try_fold(0, |acc, x| Some(acc + x)), Some((0..4).sum())); - - let mut iter = c(); - iter.by_ref().rev().nth(14); // skip the last 15, ending in state Front - assert_eq!(iter.try_fold(7, f), (0..5).try_fold(7, f)); - - let mut iter = c(); - iter.nth(14); // skip the first 15, ending in state Back - assert_eq!(iter.try_rfold(7, f), (15..20).try_rfold(7, f)); -} - -#[test] -fn test_map_try_folds() { - let f = &|acc, x| i32::checked_add(2 * acc, x); - assert_eq!((0..10).map(|x| x + 3).try_fold(7, f), (3..13).try_fold(7, f)); - assert_eq!((0..10).map(|x| x + 3).try_rfold(7, f), (3..13).try_rfold(7, f)); - - let mut iter = (0..40).map(|x| x + 10); - assert_eq!(iter.try_fold(0, i8::checked_add), None); - assert_eq!(iter.next(), Some(20)); - assert_eq!(iter.try_rfold(0, i8::checked_add), None); - assert_eq!(iter.next_back(), Some(46)); -} - -#[test] -fn test_filter_try_folds() { - fn p(&x: &i32) -> bool { - 0 <= x && x < 10 - } - let f = &|acc, x| i32::checked_add(2 * acc, x); - assert_eq!((-10..20).filter(p).try_fold(7, f), (0..10).try_fold(7, f)); - assert_eq!((-10..20).filter(p).try_rfold(7, f), (0..10).try_rfold(7, f)); - - let mut iter = (0..40).filter(|&x| x % 2 == 1); - assert_eq!(iter.try_fold(0, i8::checked_add), None); - assert_eq!(iter.next(), Some(25)); - assert_eq!(iter.try_rfold(0, i8::checked_add), None); - assert_eq!(iter.next_back(), Some(31)); -} - -#[test] -fn test_filter_map_try_folds() { - let mp = &|x| if 0 <= x && x < 10 { Some(x * 2) } else { None }; - let f = &|acc, x| i32::checked_add(2 * acc, x); - assert_eq!((-9..20).filter_map(mp).try_fold(7, f), (0..10).map(|x| 2 * x).try_fold(7, f)); - assert_eq!((-9..20).filter_map(mp).try_rfold(7, f), (0..10).map(|x| 2 * x).try_rfold(7, f)); - - let mut iter = (0..40).filter_map(|x| if x % 2 == 1 { None } else { Some(x * 2 + 10) }); - assert_eq!(iter.try_fold(0, i8::checked_add), None); - assert_eq!(iter.next(), Some(38)); - assert_eq!(iter.try_rfold(0, i8::checked_add), None); - assert_eq!(iter.next_back(), Some(78)); -} - -#[test] -fn test_enumerate_try_folds() { - let f = &|acc, (i, x)| usize::checked_add(2 * acc, x / (i + 1) + i); - assert_eq!((9..18).enumerate().try_fold(7, f), (0..9).map(|i| (i, i + 9)).try_fold(7, f)); - assert_eq!((9..18).enumerate().try_rfold(7, f), (0..9).map(|i| (i, i + 9)).try_rfold(7, f)); - - let mut iter = (100..200).enumerate(); - let f = &|acc, (i, x)| u8::checked_add(acc, u8::checked_div(x, i as u8 + 1)?); - assert_eq!(iter.try_fold(0, f), None); - assert_eq!(iter.next(), Some((7, 107))); - assert_eq!(iter.try_rfold(0, f), None); - assert_eq!(iter.next_back(), Some((11, 111))); -} - -#[test] -fn test_peek_try_folds() { - let f = &|acc, x| i32::checked_add(2 * acc, x); - - assert_eq!((1..20).peekable().try_fold(7, f), (1..20).try_fold(7, f)); - assert_eq!((1..20).peekable().try_rfold(7, f), (1..20).try_rfold(7, f)); - - let mut iter = (1..20).peekable(); - assert_eq!(iter.peek(), Some(&1)); - assert_eq!(iter.try_fold(7, f), (1..20).try_fold(7, f)); - - let mut iter = (1..20).peekable(); - assert_eq!(iter.peek(), Some(&1)); - assert_eq!(iter.try_rfold(7, f), (1..20).try_rfold(7, f)); - - let mut iter = [100, 20, 30, 40, 50, 60, 70].iter().cloned().peekable(); - assert_eq!(iter.peek(), Some(&100)); - assert_eq!(iter.try_fold(0, i8::checked_add), None); - assert_eq!(iter.peek(), Some(&40)); - - let mut iter = [100, 20, 30, 40, 50, 60, 70].iter().cloned().peekable(); - assert_eq!(iter.peek(), Some(&100)); - assert_eq!(iter.try_rfold(0, i8::checked_add), None); - assert_eq!(iter.peek(), Some(&100)); - assert_eq!(iter.next_back(), Some(50)); - - let mut iter = (2..5).peekable(); - assert_eq!(iter.peek(), Some(&2)); - assert_eq!(iter.try_for_each(Err), Err(2)); - assert_eq!(iter.peek(), Some(&3)); - assert_eq!(iter.try_for_each(Err), Err(3)); - assert_eq!(iter.peek(), Some(&4)); - assert_eq!(iter.try_for_each(Err), Err(4)); - assert_eq!(iter.peek(), None); - assert_eq!(iter.try_for_each(Err), Ok(())); - - let mut iter = (2..5).peekable(); - assert_eq!(iter.peek(), Some(&2)); - assert_eq!(iter.try_rfold((), |(), x| Err(x)), Err(4)); - assert_eq!(iter.peek(), Some(&2)); - assert_eq!(iter.try_rfold((), |(), x| Err(x)), Err(3)); - assert_eq!(iter.peek(), Some(&2)); - assert_eq!(iter.try_rfold((), |(), x| Err(x)), Err(2)); - assert_eq!(iter.peek(), None); - assert_eq!(iter.try_rfold((), |(), x| Err(x)), Ok(())); -} - -#[test] -fn test_skip_while_try_fold() { - let f = &|acc, x| i32::checked_add(2 * acc, x); - fn p(&x: &i32) -> bool { - (x % 10) <= 5 - } - assert_eq!((1..20).skip_while(p).try_fold(7, f), (6..20).try_fold(7, f)); - let mut iter = (1..20).skip_while(p); - assert_eq!(iter.nth(5), Some(11)); - assert_eq!(iter.try_fold(7, f), (12..20).try_fold(7, f)); - - let mut iter = (0..50).skip_while(|&x| (x % 20) < 15); - assert_eq!(iter.try_fold(0, i8::checked_add), None); - assert_eq!(iter.next(), Some(23)); -} - -#[test] -fn test_take_while_folds() { - let f = &|acc, x| i32::checked_add(2 * acc, x); - assert_eq!((1..20).take_while(|&x| x != 10).try_fold(7, f), (1..10).try_fold(7, f)); - let mut iter = (1..20).take_while(|&x| x != 10); - assert_eq!(iter.try_fold(0, |x, y| Some(x + y)), Some((1..10).sum())); - assert_eq!(iter.next(), None, "flag should be set"); - let iter = (1..20).take_while(|&x| x != 10); - assert_eq!(iter.fold(0, |x, y| x + y), (1..10).sum()); - - let mut iter = (10..50).take_while(|&x| x != 40); - assert_eq!(iter.try_fold(0, i8::checked_add), None); - assert_eq!(iter.next(), Some(20)); -} - -#[test] -fn test_skip_try_folds() { - let f = &|acc, x| i32::checked_add(2 * acc, x); - assert_eq!((1..20).skip(9).try_fold(7, f), (10..20).try_fold(7, f)); - assert_eq!((1..20).skip(9).try_rfold(7, f), (10..20).try_rfold(7, f)); - - let mut iter = (0..30).skip(10); - assert_eq!(iter.try_fold(0, i8::checked_add), None); - assert_eq!(iter.next(), Some(20)); - assert_eq!(iter.try_rfold(0, i8::checked_add), None); - assert_eq!(iter.next_back(), Some(24)); -} - -#[test] -fn test_skip_nth_back() { - let xs = [0, 1, 2, 3, 4, 5]; - let mut it = xs.iter().skip(2); - assert_eq!(it.nth_back(0), Some(&5)); - assert_eq!(it.nth_back(1), Some(&3)); - assert_eq!(it.nth_back(0), Some(&2)); - assert_eq!(it.nth_back(0), None); - - let ys = [2, 3, 4, 5]; - let mut ity = ys.iter(); - let mut it = xs.iter().skip(2); - assert_eq!(it.nth_back(1), ity.nth_back(1)); - assert_eq!(it.clone().nth(0), ity.clone().nth(0)); - assert_eq!(it.nth_back(0), ity.nth_back(0)); - assert_eq!(it.clone().nth(0), ity.clone().nth(0)); - assert_eq!(it.nth_back(0), ity.nth_back(0)); - assert_eq!(it.clone().nth(0), ity.clone().nth(0)); - assert_eq!(it.nth_back(0), ity.nth_back(0)); - assert_eq!(it.clone().nth(0), ity.clone().nth(0)); - - let mut it = xs.iter().skip(2); - assert_eq!(it.nth_back(4), None); - assert_eq!(it.nth_back(0), None); - - let mut it = xs.iter(); - it.by_ref().skip(2).nth_back(3); - assert_eq!(it.next_back(), Some(&1)); - - let mut it = xs.iter(); - it.by_ref().skip(2).nth_back(10); - assert_eq!(it.next_back(), Some(&1)); -} - -#[test] -fn test_take_try_folds() { - let f = &|acc, x| i32::checked_add(2 * acc, x); - assert_eq!((10..30).take(10).try_fold(7, f), (10..20).try_fold(7, f)); - assert_eq!((10..30).take(10).try_rfold(7, f), (10..20).try_rfold(7, f)); - - let mut iter = (10..30).take(20); - assert_eq!(iter.try_fold(0, i8::checked_add), None); - assert_eq!(iter.next(), Some(20)); - assert_eq!(iter.try_rfold(0, i8::checked_add), None); - assert_eq!(iter.next_back(), Some(24)); - - let mut iter = (2..20).take(3); - assert_eq!(iter.try_for_each(Err), Err(2)); - assert_eq!(iter.try_for_each(Err), Err(3)); - assert_eq!(iter.try_for_each(Err), Err(4)); - assert_eq!(iter.try_for_each(Err), Ok(())); - - let mut iter = (2..20).take(3).rev(); - assert_eq!(iter.try_for_each(Err), Err(4)); - assert_eq!(iter.try_for_each(Err), Err(3)); - assert_eq!(iter.try_for_each(Err), Err(2)); - assert_eq!(iter.try_for_each(Err), Ok(())); -} - -#[test] -fn test_flat_map_try_folds() { - let f = &|acc, x| i32::checked_add(acc * 2 / 3, x); - let mr = &|x| (5 * x)..(5 * x + 5); - assert_eq!((0..10).flat_map(mr).try_fold(7, f), (0..50).try_fold(7, f)); - assert_eq!((0..10).flat_map(mr).try_rfold(7, f), (0..50).try_rfold(7, f)); - let mut iter = (0..10).flat_map(mr); - iter.next(); - iter.next_back(); // have front and back iters in progress - assert_eq!(iter.try_rfold(7, f), (1..49).try_rfold(7, f)); - - let mut iter = (0..10).flat_map(|x| (4 * x)..(4 * x + 4)); - assert_eq!(iter.try_fold(0, i8::checked_add), None); - assert_eq!(iter.next(), Some(17)); - assert_eq!(iter.try_rfold(0, i8::checked_add), None); - assert_eq!(iter.next_back(), Some(35)); -} - -#[test] -fn test_flatten_try_folds() { - let f = &|acc, x| i32::checked_add(acc * 2 / 3, x); - let mr = &|x| (5 * x)..(5 * x + 5); - assert_eq!((0..10).map(mr).flatten().try_fold(7, f), (0..50).try_fold(7, f)); - assert_eq!((0..10).map(mr).flatten().try_rfold(7, f), (0..50).try_rfold(7, f)); - let mut iter = (0..10).map(mr).flatten(); - iter.next(); - iter.next_back(); // have front and back iters in progress - assert_eq!(iter.try_rfold(7, f), (1..49).try_rfold(7, f)); - - let mut iter = (0..10).map(|x| (4 * x)..(4 * x + 4)).flatten(); - assert_eq!(iter.try_fold(0, i8::checked_add), None); - assert_eq!(iter.next(), Some(17)); - assert_eq!(iter.try_rfold(0, i8::checked_add), None); - assert_eq!(iter.next_back(), Some(35)); -} - -#[test] -fn test_functor_laws() { - // identity: - fn identity(x: T) -> T { - x - } - assert_eq!((0..10).map(identity).sum::(), (0..10).sum()); - - // composition: - fn f(x: usize) -> usize { - x + 3 - } - fn g(x: usize) -> usize { - x * 2 - } - fn h(x: usize) -> usize { - g(f(x)) - } - assert_eq!((0..10).map(f).map(g).sum::(), (0..10).map(h).sum()); -} - -#[test] -fn test_monad_laws_left_identity() { - fn f(x: usize) -> impl Iterator { - (0..10).map(move |y| x * y) - } - assert_eq!(once(42).flat_map(f.clone()).sum::(), f(42).sum()); -} - -#[test] -fn test_monad_laws_right_identity() { - assert_eq!((0..10).flat_map(|x| once(x)).sum::(), (0..10).sum()); -} - -#[test] -fn test_monad_laws_associativity() { - fn f(x: usize) -> impl Iterator { - 0..x - } - fn g(x: usize) -> impl Iterator { - (0..x).rev() - } - assert_eq!( - (0..10).flat_map(f).flat_map(g).sum::(), - (0..10).flat_map(|x| f(x).flat_map(g)).sum::() - ); -} - -#[test] -fn test_is_sorted() { - assert!([1, 2, 2, 9].iter().is_sorted()); - assert!(![1, 3, 2].iter().is_sorted()); - assert!([0].iter().is_sorted()); - assert!(std::iter::empty::().is_sorted()); - assert!(![0.0, 1.0, std::f32::NAN].iter().is_sorted()); - assert!([-2, -1, 0, 3].iter().is_sorted()); - assert!(![-2i32, -1, 0, 3].iter().is_sorted_by_key(|n| n.abs())); - assert!(!["c", "bb", "aaa"].iter().is_sorted()); - assert!(["c", "bb", "aaa"].iter().is_sorted_by_key(|s| s.len())); -} - -#[test] -fn test_partition() { - fn check(xs: &mut [i32], ref p: impl Fn(&i32) -> bool, expected: usize) { - let i = xs.iter_mut().partition_in_place(p); - assert_eq!(expected, i); - assert!(xs[..i].iter().all(p)); - assert!(!xs[i..].iter().any(p)); - assert!(xs.iter().is_partitioned(p)); - if i == 0 || i == xs.len() { - assert!(xs.iter().rev().is_partitioned(p)); - } else { - assert!(!xs.iter().rev().is_partitioned(p)); - } - } - - check(&mut [], |_| true, 0); - check(&mut [], |_| false, 0); - - check(&mut [0], |_| true, 1); - check(&mut [0], |_| false, 0); - - check(&mut [-1, 1], |&x| x > 0, 1); - check(&mut [-1, 1], |&x| x < 0, 1); - - let ref mut xs = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; - check(xs, |_| true, 10); - check(xs, |_| false, 0); - check(xs, |&x| x % 2 == 0, 5); // evens - check(xs, |&x| x % 2 == 1, 5); // odds - check(xs, |&x| x % 3 == 0, 4); // multiple of 3 - check(xs, |&x| x % 4 == 0, 3); // multiple of 4 - check(xs, |&x| x % 5 == 0, 2); // multiple of 5 - check(xs, |&x| x < 3, 3); // small - check(xs, |&x| x > 6, 3); // large -} - -/// An iterator that panics whenever `next` or next_back` is called -/// after `None` has already been returned. This does not violate -/// `Iterator`'s contract. Used to test that iterator adaptors don't -/// poll their inner iterators after exhausting them. -struct NonFused { - iter: I, - done: bool, -} - -impl NonFused { - fn new(iter: I) -> Self { - Self { iter, done: false } - } -} - -impl Iterator for NonFused -where - I: Iterator, -{ - type Item = I::Item; - - fn next(&mut self) -> Option { - assert!(!self.done, "this iterator has already returned None"); - self.iter.next().or_else(|| { - self.done = true; - None - }) - } -} - -impl DoubleEndedIterator for NonFused -where - I: DoubleEndedIterator, -{ - fn next_back(&mut self) -> Option { - assert!(!self.done, "this iterator has already returned None"); - self.iter.next_back().or_else(|| { - self.done = true; - None - }) - } -} - -#[test] -fn test_peekable_non_fused() { - let mut iter = NonFused::new(empty::()).peekable(); - - assert_eq!(iter.peek(), None); - assert_eq!(iter.next_back(), None); -} - -#[test] -fn test_flatten_non_fused_outer() { - let mut iter = NonFused::new(once(0..2)).flatten(); - - assert_eq!(iter.next_back(), Some(1)); - assert_eq!(iter.next(), Some(0)); - assert_eq!(iter.next(), None); -} - -#[test] -fn test_flatten_non_fused_inner() { - let mut iter = once(0..1).chain(once(1..3)).flat_map(NonFused::new); - - assert_eq!(iter.next_back(), Some(2)); - assert_eq!(iter.next(), Some(0)); - assert_eq!(iter.next(), Some(1)); - assert_eq!(iter.next(), None); -} diff --git a/crux-mir/lib/core/tests/iter/adapters/array_chunks.rs b/crux-mir/lib/core/tests/iter/adapters/array_chunks.rs new file mode 100644 index 000000000..ef4a7e53b --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/array_chunks.rs @@ -0,0 +1,180 @@ +use core::cell::Cell; +use core::iter::{self, Iterator}; + +use super::*; + +#[test] +fn test_iterator_array_chunks_infer() { + let xs = [1, 1, 2, -2, 6, 0, 3, 1]; + for [a, b, c] in xs.iter().copied().array_chunks() { + assert_eq!(a + b + c, 4); + } +} + +#[test] +fn test_iterator_array_chunks_clone_and_drop() { + let count = Cell::new(0); + let mut it = (0..5).map(|_| CountDrop::new(&count)).array_chunks::<3>(); + assert_eq!(it.by_ref().count(), 1); + assert_eq!(count.get(), 3); + let mut it2 = it.clone(); + assert_eq!(count.get(), 3); + assert_eq!(it.into_remainder().unwrap().len(), 2); + assert_eq!(count.get(), 5); + assert!(it2.next().is_none()); + assert_eq!(it2.into_remainder().unwrap().len(), 2); + assert_eq!(count.get(), 7); +} + +#[test] +fn test_iterator_array_chunks_remainder() { + let mut it = (0..11).array_chunks::<4>(); + assert_eq!(it.next(), Some([0, 1, 2, 3])); + assert_eq!(it.next(), Some([4, 5, 6, 7])); + assert_eq!(it.next(), None); + assert_eq!(it.into_remainder().unwrap().as_slice(), &[8, 9, 10]); +} + +#[test] +fn test_iterator_array_chunks_size_hint() { + let it = (0..6).array_chunks::<1>(); + assert_eq!(it.size_hint(), (6, Some(6))); + + let it = (0..6).array_chunks::<3>(); + assert_eq!(it.size_hint(), (2, Some(2))); + + let it = (0..6).array_chunks::<5>(); + assert_eq!(it.size_hint(), (1, Some(1))); + + let it = (0..6).array_chunks::<7>(); + assert_eq!(it.size_hint(), (0, Some(0))); + + let it = (1..).array_chunks::<2>(); + assert_eq!(it.size_hint(), (usize::MAX / 2, None)); + + let it = (1..).filter(|x| x % 2 != 0).array_chunks::<2>(); + assert_eq!(it.size_hint(), (0, None)); +} + +#[test] +fn test_iterator_array_chunks_count() { + let it = (0..6).array_chunks::<1>(); + assert_eq!(it.count(), 6); + + let it = (0..6).array_chunks::<3>(); + assert_eq!(it.count(), 2); + + let it = (0..6).array_chunks::<5>(); + assert_eq!(it.count(), 1); + + let it = (0..6).array_chunks::<7>(); + assert_eq!(it.count(), 0); + + let it = (0..6).filter(|x| x % 2 == 0).array_chunks::<2>(); + assert_eq!(it.count(), 1); + + let it = iter::empty::().array_chunks::<2>(); + assert_eq!(it.count(), 0); + + let it = [(); usize::MAX].iter().array_chunks::<2>(); + assert_eq!(it.count(), usize::MAX / 2); +} + +#[test] +fn test_iterator_array_chunks_next_and_next_back() { + let mut it = (0..11).array_chunks::<3>(); + assert_eq!(it.next(), Some([0, 1, 2])); + assert_eq!(it.next_back(), Some([6, 7, 8])); + assert_eq!(it.next(), Some([3, 4, 5])); + assert_eq!(it.next_back(), None); + assert_eq!(it.next(), None); + assert_eq!(it.next_back(), None); + assert_eq!(it.next(), None); + assert_eq!(it.into_remainder().unwrap().as_slice(), &[9, 10]); +} + +#[test] +fn test_iterator_array_chunks_rev_remainder() { + let mut it = (0..11).array_chunks::<4>(); + { + let mut it = it.by_ref().rev(); + assert_eq!(it.next(), Some([4, 5, 6, 7])); + assert_eq!(it.next(), Some([0, 1, 2, 3])); + assert_eq!(it.next(), None); + assert_eq!(it.next(), None); + } + assert_eq!(it.into_remainder().unwrap().as_slice(), &[8, 9, 10]); +} + +#[test] +fn test_iterator_array_chunks_try_fold() { + let count = Cell::new(0); + let mut it = (0..10).map(|_| CountDrop::new(&count)).array_chunks::<3>(); + let result: Result<_, ()> = it.by_ref().try_fold(0, |acc, _item| Ok(acc + 1)); + assert_eq!(result, Ok(3)); + assert_eq!(count.get(), 9); + drop(it); + assert_eq!(count.get(), 10); + + let count = Cell::new(0); + let mut it = (0..10).map(|_| CountDrop::new(&count)).array_chunks::<3>(); + let result = it.by_ref().try_fold(0, |acc, _item| if acc < 2 { Ok(acc + 1) } else { Err(acc) }); + assert_eq!(result, Err(2)); + assert_eq!(count.get(), 9); + drop(it); + assert_eq!(count.get(), 9); +} + +#[test] +fn test_iterator_array_chunks_fold() { + let result = (1..11).array_chunks::<3>().fold(0, |acc, [a, b, c]| { + assert_eq!(acc + 1, a); + assert_eq!(acc + 2, b); + assert_eq!(acc + 3, c); + acc + 3 + }); + assert_eq!(result, 9); + + let count = Cell::new(0); + let result = + (0..10).map(|_| CountDrop::new(&count)).array_chunks::<3>().fold(0, |acc, _item| acc + 1); + assert_eq!(result, 3); + // fold impls may or may not process the remainder + assert!(count.get() <= 10 && count.get() >= 9); +} + +#[test] +fn test_iterator_array_chunks_try_rfold() { + let count = Cell::new(0); + let mut it = (0..10).map(|_| CountDrop::new(&count)).array_chunks::<3>(); + let result: Result<_, ()> = it.try_rfold(0, |acc, _item| Ok(acc + 1)); + assert_eq!(result, Ok(3)); + assert_eq!(count.get(), 9); + drop(it); + assert_eq!(count.get(), 10); + + let count = Cell::new(0); + let mut it = (0..10).map(|_| CountDrop::new(&count)).array_chunks::<3>(); + let result = it.try_rfold(0, |acc, _item| if acc < 2 { Ok(acc + 1) } else { Err(acc) }); + assert_eq!(result, Err(2)); + assert_eq!(count.get(), 9); + drop(it); + assert_eq!(count.get(), 10); +} + +#[test] +fn test_iterator_array_chunks_rfold() { + let result = (1..11).array_chunks::<3>().rfold(0, |acc, [a, b, c]| { + assert_eq!(10 - (acc + 1), c); + assert_eq!(10 - (acc + 2), b); + assert_eq!(10 - (acc + 3), a); + acc + 3 + }); + assert_eq!(result, 9); + + let count = Cell::new(0); + let result = + (0..10).map(|_| CountDrop::new(&count)).array_chunks::<3>().rfold(0, |acc, _item| acc + 1); + assert_eq!(result, 3); + assert_eq!(count.get(), 10); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/by_ref_sized.rs b/crux-mir/lib/core/tests/iter/adapters/by_ref_sized.rs new file mode 100644 index 000000000..a9c066f0e --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/by_ref_sized.rs @@ -0,0 +1,20 @@ +use core::iter::*; + +#[test] +fn test_iterator_by_ref_sized() { + let a = ['a', 'b', 'c', 'd']; + + let mut s = String::from("Z"); + let mut it = a.iter().copied(); + ByRefSized(&mut it).take(2).for_each(|x| s.push(x)); + assert_eq!(s, "Zab"); + ByRefSized(&mut it).fold((), |(), x| s.push(x)); + assert_eq!(s, "Zabcd"); + + let mut s = String::from("Z"); + let mut it = a.iter().copied(); + ByRefSized(&mut it).rev().take(2).for_each(|x| s.push(x)); + assert_eq!(s, "Zdc"); + ByRefSized(&mut it).rfold((), |(), x| s.push(x)); + assert_eq!(s, "Zdcba"); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/chain.rs b/crux-mir/lib/core/tests/iter/adapters/chain.rs new file mode 100644 index 000000000..f419f9cec --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/chain.rs @@ -0,0 +1,280 @@ +use super::*; +use core::iter::*; + +#[test] +fn test_iterator_chain() { + let xs = [0, 1, 2, 3, 4, 5]; + let ys = [30, 40, 50, 60]; + let expected = [0, 1, 2, 3, 4, 5, 30, 40, 50, 60]; + let it = xs.iter().chain(&ys); + let mut i = 0; + for &x in it { + assert_eq!(x, expected[i]); + i += 1; + } + assert_eq!(i, expected.len()); + + let ys = (30..).step_by(10).take(4); + let it = xs.iter().cloned().chain(ys); + let mut i = 0; + for x in it { + assert_eq!(x, expected[i]); + i += 1; + } + assert_eq!(i, expected.len()); +} + +#[test] +fn test_iterator_chain_advance_by() { + fn test_chain(xs: &[i32], ys: &[i32]) { + let len = xs.len() + ys.len(); + + for i in 0..xs.len() { + let mut iter = Unfuse::new(xs).chain(Unfuse::new(ys)); + iter.advance_by(i).unwrap(); + assert_eq!(iter.next(), Some(&xs[i])); + assert_eq!(iter.advance_by(100), Err(len - i - 1)); + iter.advance_by(0).unwrap(); + } + + for i in 0..ys.len() { + let mut iter = Unfuse::new(xs).chain(Unfuse::new(ys)); + iter.advance_by(xs.len() + i).unwrap(); + assert_eq!(iter.next(), Some(&ys[i])); + assert_eq!(iter.advance_by(100), Err(ys.len() - i - 1)); + iter.advance_by(0).unwrap(); + } + + let mut iter = xs.iter().chain(ys); + iter.advance_by(len).unwrap(); + assert_eq!(iter.next(), None); + iter.advance_by(0).unwrap(); + + let mut iter = xs.iter().chain(ys); + assert_eq!(iter.advance_by(len + 1), Err(len)); + iter.advance_by(0).unwrap(); + } + + test_chain(&[], &[]); + test_chain(&[], &[0, 1, 2, 3, 4, 5]); + test_chain(&[0, 1, 2, 3, 4, 5], &[]); + test_chain(&[0, 1, 2, 3, 4, 5], &[30, 40, 50, 60]); +} + +#[test] +fn test_iterator_chain_advance_back_by() { + fn test_chain(xs: &[i32], ys: &[i32]) { + let len = xs.len() + ys.len(); + + for i in 0..ys.len() { + let mut iter = Unfuse::new(xs).chain(Unfuse::new(ys)); + iter.advance_back_by(i).unwrap(); + assert_eq!(iter.next_back(), Some(&ys[ys.len() - i - 1])); + assert_eq!(iter.advance_back_by(100), Err(len - i - 1)); + iter.advance_back_by(0).unwrap(); + } + + for i in 0..xs.len() { + let mut iter = Unfuse::new(xs).chain(Unfuse::new(ys)); + iter.advance_back_by(ys.len() + i).unwrap(); + assert_eq!(iter.next_back(), Some(&xs[xs.len() - i - 1])); + assert_eq!(iter.advance_back_by(100), Err(xs.len() - i - 1)); + iter.advance_back_by(0).unwrap(); + } + + let mut iter = xs.iter().chain(ys); + iter.advance_back_by(len).unwrap(); + assert_eq!(iter.next_back(), None); + iter.advance_back_by(0).unwrap(); + + let mut iter = xs.iter().chain(ys); + assert_eq!(iter.advance_back_by(len + 1), Err(len)); + iter.advance_back_by(0).unwrap(); + } + + test_chain(&[], &[]); + test_chain(&[], &[0, 1, 2, 3, 4, 5]); + test_chain(&[0, 1, 2, 3, 4, 5], &[]); + test_chain(&[0, 1, 2, 3, 4, 5], &[30, 40, 50, 60]); +} + +#[test] +fn test_iterator_chain_nth() { + let xs = [0, 1, 2, 3, 4, 5]; + let ys = [30, 40, 50, 60]; + let zs = []; + let expected = [0, 1, 2, 3, 4, 5, 30, 40, 50, 60]; + for (i, x) in expected.iter().enumerate() { + assert_eq!(Some(x), xs.iter().chain(&ys).nth(i)); + } + assert_eq!(zs.iter().chain(&xs).nth(0), Some(&0)); + + let mut it = xs.iter().chain(&zs); + assert_eq!(it.nth(5), Some(&5)); + assert_eq!(it.next(), None); +} + +#[test] +fn test_iterator_chain_nth_back() { + let xs = [0, 1, 2, 3, 4, 5]; + let ys = [30, 40, 50, 60]; + let zs = []; + let expected = [0, 1, 2, 3, 4, 5, 30, 40, 50, 60]; + for (i, x) in expected.iter().rev().enumerate() { + assert_eq!(Some(x), xs.iter().chain(&ys).nth_back(i)); + } + assert_eq!(zs.iter().chain(&xs).nth_back(0), Some(&5)); + + let mut it = xs.iter().chain(&zs); + assert_eq!(it.nth_back(5), Some(&0)); + assert_eq!(it.next(), None); +} + +#[test] +fn test_iterator_chain_last() { + let xs = [0, 1, 2, 3, 4, 5]; + let ys = [30, 40, 50, 60]; + let zs = []; + assert_eq!(xs.iter().chain(&ys).last(), Some(&60)); + assert_eq!(zs.iter().chain(&ys).last(), Some(&60)); + assert_eq!(ys.iter().chain(&zs).last(), Some(&60)); + assert_eq!(zs.iter().chain(&zs).last(), None); +} + +#[test] +fn test_iterator_chain_count() { + let xs = [0, 1, 2, 3, 4, 5]; + let ys = [30, 40, 50, 60]; + let zs = []; + assert_eq!(xs.iter().chain(&ys).count(), 10); + assert_eq!(zs.iter().chain(&ys).count(), 4); +} + +#[test] +fn test_iterator_chain_find() { + let xs = [0, 1, 2, 3, 4, 5]; + let ys = [30, 40, 50, 60]; + let mut iter = xs.iter().chain(&ys); + assert_eq!(iter.find(|&&i| i == 4), Some(&4)); + assert_eq!(iter.next(), Some(&5)); + assert_eq!(iter.find(|&&i| i == 40), Some(&40)); + assert_eq!(iter.next(), Some(&50)); + assert_eq!(iter.find(|&&i| i == 100), None); + assert_eq!(iter.next(), None); +} + +#[test] +fn test_iterator_chain_size_hint() { + // this chains an iterator of length 0 with an iterator of length 1, + // so after calling `.next()` once, the iterator is empty and the + // state is `ChainState::Back`. `.size_hint()` should now disregard + // the size hint of the left iterator + let mut iter = Toggle { is_empty: true }.chain(once(())); + assert_eq!(iter.next(), Some(())); + assert_eq!(iter.size_hint(), (0, Some(0))); + + let mut iter = once(()).chain(Toggle { is_empty: true }); + assert_eq!(iter.next_back(), Some(())); + assert_eq!(iter.size_hint(), (0, Some(0))); +} + +#[test] +fn test_iterator_chain_unfused() { + // Chain shouldn't be fused in its second iterator, depending on direction + let mut iter = NonFused::new(empty()).chain(Toggle { is_empty: true }); + assert!(iter.next().is_none()); + assert!(iter.next().is_some()); + assert!(iter.next().is_none()); + + let mut iter = Toggle { is_empty: true }.chain(NonFused::new(empty())); + assert!(iter.next_back().is_none()); + assert!(iter.next_back().is_some()); + assert!(iter.next_back().is_none()); +} + +#[test] +fn test_chain_fold() { + let xs = [1, 2, 3]; + let ys = [1, 2, 0]; + + let mut iter = xs.iter().chain(&ys); + iter.next(); + let mut result = Vec::new(); + iter.fold((), |(), &elt| result.push(elt)); + assert_eq!(&[2, 3, 1, 2, 0], &result[..]); +} + +#[test] +fn test_chain_try_folds() { + let c = || (0..10).chain(10..20); + + let f = &|acc, x| i32::checked_add(2 * acc, x); + assert_eq!(c().try_fold(7, f), (0..20).try_fold(7, f)); + assert_eq!(c().try_rfold(7, f), (0..20).rev().try_fold(7, f)); + + let mut iter = c(); + assert_eq!(iter.position(|x| x == 5), Some(5)); + assert_eq!(iter.next(), Some(6), "stopped in front, state Both"); + assert_eq!(iter.position(|x| x == 13), Some(6)); + assert_eq!(iter.next(), Some(14), "stopped in back, state Back"); + assert_eq!(iter.try_fold(0, |acc, x| Some(acc + x)), Some((15..20).sum())); + + let mut iter = c().rev(); // use rev to access try_rfold + assert_eq!(iter.position(|x| x == 15), Some(4)); + assert_eq!(iter.next(), Some(14), "stopped in back, state Both"); + assert_eq!(iter.position(|x| x == 5), Some(8)); + assert_eq!(iter.next(), Some(4), "stopped in front, state Front"); + assert_eq!(iter.try_fold(0, |acc, x| Some(acc + x)), Some((0..4).sum())); + + let mut iter = c(); + iter.by_ref().rev().nth(14); // skip the last 15, ending in state Front + assert_eq!(iter.try_fold(7, f), (0..5).try_fold(7, f)); + + let mut iter = c(); + iter.nth(14); // skip the first 15, ending in state Back + assert_eq!(iter.try_rfold(7, f), (15..20).try_rfold(7, f)); +} + +#[test] +fn test_double_ended_chain() { + let xs = [1, 2, 3, 4, 5]; + let ys = [7, 9, 11]; + let mut it = xs.iter().chain(&ys).rev(); + assert_eq!(it.next().unwrap(), &11); + assert_eq!(it.next().unwrap(), &9); + assert_eq!(it.next_back().unwrap(), &1); + assert_eq!(it.next_back().unwrap(), &2); + assert_eq!(it.next_back().unwrap(), &3); + assert_eq!(it.next_back().unwrap(), &4); + assert_eq!(it.next_back().unwrap(), &5); + assert_eq!(it.next_back().unwrap(), &7); + assert_eq!(it.next_back(), None); + + // test that .chain() is well behaved with an unfused iterator + struct CrazyIterator(bool); + impl CrazyIterator { + fn new() -> CrazyIterator { + CrazyIterator(false) + } + } + impl Iterator for CrazyIterator { + type Item = i32; + fn next(&mut self) -> Option { + if self.0 { + Some(99) + } else { + self.0 = true; + None + } + } + } + + impl DoubleEndedIterator for CrazyIterator { + fn next_back(&mut self) -> Option { + self.next() + } + } + + assert_eq!(CrazyIterator::new().chain(0..10).rev().last(), Some(0)); + assert!((0..10).chain(CrazyIterator::new()).rev().any(|i| i == 0)); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/cloned.rs b/crux-mir/lib/core/tests/iter/adapters/cloned.rs new file mode 100644 index 000000000..78babb7fe --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/cloned.rs @@ -0,0 +1,52 @@ +use core::iter::*; + +#[test] +fn test_cloned() { + let xs = [2, 4, 6, 8]; + + let mut it = xs.iter().cloned(); + assert_eq!(it.len(), 4); + assert_eq!(it.next(), Some(2)); + assert_eq!(it.len(), 3); + assert_eq!(it.next(), Some(4)); + assert_eq!(it.len(), 2); + assert_eq!(it.next_back(), Some(8)); + assert_eq!(it.len(), 1); + assert_eq!(it.next_back(), Some(6)); + assert_eq!(it.len(), 0); + assert_eq!(it.next_back(), None); +} + +#[test] +fn test_cloned_side_effects() { + let mut count = 0; + { + let iter = [1, 2, 3] + .iter() + .map(|x| { + count += 1; + x + }) + .cloned() + .zip(&[1]); + for _ in iter {} + } + assert_eq!(count, 2); +} + +#[test] +fn test_cloned_try_folds() { + let a = [1, 2, 3, 4, 5, 6, 7, 8, 9]; + let f = &|acc, x| i32::checked_add(2 * acc, x); + let f_ref = &|acc, &x| i32::checked_add(2 * acc, x); + assert_eq!(a.iter().cloned().try_fold(7, f), a.iter().try_fold(7, f_ref)); + assert_eq!(a.iter().cloned().try_rfold(7, f), a.iter().try_rfold(7, f_ref)); + + let a = [10, 20, 30, 40, 100, 60, 70, 80, 90]; + let mut iter = a.iter().cloned(); + assert_eq!(iter.try_fold(0_i8, |acc, x| acc.checked_add(x)), None); + assert_eq!(iter.next(), Some(60)); + let mut iter = a.iter().cloned(); + assert_eq!(iter.try_rfold(0_i8, |acc, x| acc.checked_add(x)), None); + assert_eq!(iter.next_back(), Some(70)); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/copied.rs b/crux-mir/lib/core/tests/iter/adapters/copied.rs new file mode 100644 index 000000000..b12f2035d --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/copied.rs @@ -0,0 +1,18 @@ +use core::iter::*; + +#[test] +fn test_copied() { + let xs = [2, 4, 6, 8]; + + let mut it = xs.iter().copied(); + assert_eq!(it.len(), 4); + assert_eq!(it.next(), Some(2)); + assert_eq!(it.len(), 3); + assert_eq!(it.next(), Some(4)); + assert_eq!(it.len(), 2); + assert_eq!(it.next_back(), Some(8)); + assert_eq!(it.len(), 1); + assert_eq!(it.next_back(), Some(6)); + assert_eq!(it.len(), 0); + assert_eq!(it.next_back(), None); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/cycle.rs b/crux-mir/lib/core/tests/iter/adapters/cycle.rs new file mode 100644 index 000000000..8831c09b4 --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/cycle.rs @@ -0,0 +1,31 @@ +use core::iter::*; + +#[test] +fn test_cycle() { + let cycle_len = 3; + let it = (0..).step_by(1).take(cycle_len).cycle(); + assert_eq!(it.size_hint(), (usize::MAX, None)); + for (i, x) in it.take(100).enumerate() { + assert_eq!(i % cycle_len, x); + } + + let mut it = (0..).step_by(1).take(0).cycle(); + assert_eq!(it.size_hint(), (0, Some(0))); + assert_eq!(it.next(), None); + + assert_eq!(empty::().cycle().fold(0, |acc, x| acc + x), 0); + + assert_eq!(once(1).cycle().skip(1).take(4).fold(0, |acc, x| acc + x), 4); + + assert_eq!((0..10).cycle().take(5).sum::(), 10); + assert_eq!((0..10).cycle().take(15).sum::(), 55); + assert_eq!((0..10).cycle().take(25).sum::(), 100); + + let mut iter = (0..10).cycle(); + iter.nth(14); + assert_eq!(iter.take(8).sum::(), 38); + + let mut iter = (0..10).cycle(); + iter.nth(9); + assert_eq!(iter.take(3).sum::(), 3); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/enumerate.rs b/crux-mir/lib/core/tests/iter/adapters/enumerate.rs new file mode 100644 index 000000000..0e6033878 --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/enumerate.rs @@ -0,0 +1,107 @@ +use core::iter::*; + +#[test] +fn test_iterator_enumerate() { + let xs = [0, 1, 2, 3, 4, 5]; + let it = xs.iter().enumerate(); + for (i, &x) in it { + assert_eq!(i, x); + } +} + +#[test] +fn test_iterator_enumerate_nth() { + let xs = [0, 1, 2, 3, 4, 5]; + for (i, &x) in xs.iter().enumerate() { + assert_eq!(i, x); + } + + let mut it = xs.iter().enumerate(); + while let Some((i, &x)) = it.nth(0) { + assert_eq!(i, x); + } + + let mut it = xs.iter().enumerate(); + while let Some((i, &x)) = it.nth(1) { + assert_eq!(i, x); + } + + let (i, &x) = xs.iter().enumerate().nth(3).unwrap(); + assert_eq!(i, x); + assert_eq!(i, 3); +} + +#[test] +fn test_iterator_enumerate_nth_back() { + let xs = [0, 1, 2, 3, 4, 5]; + let mut it = xs.iter().enumerate(); + while let Some((i, &x)) = it.nth_back(0) { + assert_eq!(i, x); + } + + let mut it = xs.iter().enumerate(); + while let Some((i, &x)) = it.nth_back(1) { + assert_eq!(i, x); + } + + let (i, &x) = xs.iter().enumerate().nth_back(3).unwrap(); + assert_eq!(i, x); + assert_eq!(i, 2); +} + +#[test] +fn test_iterator_enumerate_count() { + let xs = [0, 1, 2, 3, 4, 5]; + assert_eq!(xs.iter().enumerate().count(), 6); +} + +#[test] +fn test_iterator_enumerate_fold() { + let xs = [0, 1, 2, 3, 4, 5]; + let mut it = xs.iter().enumerate(); + // steal a couple to get an interesting offset + assert_eq!(it.next(), Some((0, &0))); + assert_eq!(it.next(), Some((1, &1))); + let i = it.fold(2, |i, (j, &x)| { + assert_eq!(i, j); + assert_eq!(x, xs[j]); + i + 1 + }); + assert_eq!(i, xs.len()); + + let mut it = xs.iter().enumerate(); + assert_eq!(it.next(), Some((0, &0))); + let i = it.rfold(xs.len() - 1, |i, (j, &x)| { + assert_eq!(i, j); + assert_eq!(x, xs[j]); + i - 1 + }); + assert_eq!(i, 0); +} + +#[test] +fn test_enumerate_try_folds() { + let f = &|acc, (i, x)| usize::checked_add(2 * acc, x / (i + 1) + i); + assert_eq!((9..18).enumerate().try_fold(7, f), (0..9).map(|i| (i, i + 9)).try_fold(7, f)); + assert_eq!((9..18).enumerate().try_rfold(7, f), (0..9).map(|i| (i, i + 9)).try_rfold(7, f)); + + let mut iter = (100..200).enumerate(); + let f = &|acc, (i, x)| u8::checked_add(acc, u8::checked_div(x, i as u8 + 1)?); + assert_eq!(iter.try_fold(0, f), None); + assert_eq!(iter.next(), Some((7, 107))); + assert_eq!(iter.try_rfold(0, f), None); + assert_eq!(iter.next_back(), Some((11, 111))); +} + +#[test] +fn test_double_ended_enumerate() { + let xs = [1, 2, 3, 4, 5, 6]; + let mut it = xs.iter().cloned().enumerate(); + assert_eq!(it.next(), Some((0, 1))); + assert_eq!(it.next(), Some((1, 2))); + assert_eq!(it.next_back(), Some((5, 6))); + assert_eq!(it.next_back(), Some((4, 5))); + assert_eq!(it.next_back(), Some((3, 4))); + assert_eq!(it.next_back(), Some((2, 3))); + assert_eq!(it.next(), None); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/filter.rs b/crux-mir/lib/core/tests/iter/adapters/filter.rs new file mode 100644 index 000000000..a2050d89d --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/filter.rs @@ -0,0 +1,52 @@ +use core::iter::*; + +#[test] +fn test_iterator_filter_count() { + let xs = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + assert_eq!(xs.iter().filter(|&&x| x % 2 == 0).count(), 5); +} + +#[test] +fn test_iterator_filter_fold() { + let xs = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let ys = [0, 2, 4, 6, 8]; + let it = xs.iter().filter(|&&x| x % 2 == 0); + let i = it.fold(0, |i, &x| { + assert_eq!(x, ys[i]); + i + 1 + }); + assert_eq!(i, ys.len()); + + let it = xs.iter().filter(|&&x| x % 2 == 0); + let i = it.rfold(ys.len(), |i, &x| { + assert_eq!(x, ys[i - 1]); + i - 1 + }); + assert_eq!(i, 0); +} + +#[test] +fn test_filter_try_folds() { + fn p(&x: &i32) -> bool { + 0 <= x && x < 10 + } + let f = &|acc, x| i32::checked_add(2 * acc, x); + assert_eq!((-10..20).filter(p).try_fold(7, f), (0..10).try_fold(7, f)); + assert_eq!((-10..20).filter(p).try_rfold(7, f), (0..10).try_rfold(7, f)); + + let mut iter = (0..40).filter(|&x| x % 2 == 1); + assert_eq!(iter.try_fold(0, i8::checked_add), None); + assert_eq!(iter.next(), Some(25)); + assert_eq!(iter.try_rfold(0, i8::checked_add), None); + assert_eq!(iter.next_back(), Some(31)); +} + +#[test] +fn test_double_ended_filter() { + let xs = [1, 2, 3, 4, 5, 6]; + let mut it = xs.iter().filter(|&x| *x & 1 == 0); + assert_eq!(it.next_back().unwrap(), &6); + assert_eq!(it.next_back().unwrap(), &4); + assert_eq!(it.next().unwrap(), &2); + assert_eq!(it.next_back(), None); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/filter_map.rs b/crux-mir/lib/core/tests/iter/adapters/filter_map.rs new file mode 100644 index 000000000..46738eda6 --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/filter_map.rs @@ -0,0 +1,50 @@ +use core::iter::*; + +#[test] +fn test_filter_map() { + let it = (0..).step_by(1).take(10).filter_map(|x| if x % 2 == 0 { Some(x * x) } else { None }); + assert_eq!(it.collect::>(), [0 * 0, 2 * 2, 4 * 4, 6 * 6, 8 * 8]); +} + +#[test] +fn test_filter_map_fold() { + let xs = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let ys = [0 * 0, 2 * 2, 4 * 4, 6 * 6, 8 * 8]; + let it = xs.iter().filter_map(|&x| if x % 2 == 0 { Some(x * x) } else { None }); + let i = it.fold(0, |i, x| { + assert_eq!(x, ys[i]); + i + 1 + }); + assert_eq!(i, ys.len()); + + let it = xs.iter().filter_map(|&x| if x % 2 == 0 { Some(x * x) } else { None }); + let i = it.rfold(ys.len(), |i, x| { + assert_eq!(x, ys[i - 1]); + i - 1 + }); + assert_eq!(i, 0); +} + +#[test] +fn test_filter_map_try_folds() { + let mp = &|x| if 0 <= x && x < 10 { Some(x * 2) } else { None }; + let f = &|acc, x| i32::checked_add(2 * acc, x); + assert_eq!((-9..20).filter_map(mp).try_fold(7, f), (0..10).map(|x| 2 * x).try_fold(7, f)); + assert_eq!((-9..20).filter_map(mp).try_rfold(7, f), (0..10).map(|x| 2 * x).try_rfold(7, f)); + + let mut iter = (0..40).filter_map(|x| if x % 2 == 1 { None } else { Some(x * 2 + 10) }); + assert_eq!(iter.try_fold(0, i8::checked_add), None); + assert_eq!(iter.next(), Some(38)); + assert_eq!(iter.try_rfold(0, i8::checked_add), None); + assert_eq!(iter.next_back(), Some(78)); +} + +#[test] +fn test_double_ended_filter_map() { + let xs = [1, 2, 3, 4, 5, 6]; + let mut it = xs.iter().filter_map(|&x| if x & 1 == 0 { Some(x * 2) } else { None }); + assert_eq!(it.next_back().unwrap(), 12); + assert_eq!(it.next_back().unwrap(), 8); + assert_eq!(it.next().unwrap(), 4); + assert_eq!(it.next_back(), None); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/flat_map.rs b/crux-mir/lib/core/tests/iter/adapters/flat_map.rs new file mode 100644 index 000000000..ee945e698 --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/flat_map.rs @@ -0,0 +1,74 @@ +use core::iter::*; + +#[test] +fn test_iterator_flat_map() { + let xs = [0, 3, 6]; + let ys = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let it = xs.iter().flat_map(|&x| (x..).step_by(1).take(3)); + let mut i = 0; + for x in it { + assert_eq!(x, ys[i]); + i += 1; + } + assert_eq!(i, ys.len()); +} + +/// Tests `FlatMap::fold` with items already picked off the front and back, +/// to make sure all parts of the `FlatMap` are folded correctly. +#[test] +fn test_iterator_flat_map_fold() { + let xs = [0, 3, 6]; + let ys = [1, 2, 3, 4, 5, 6, 7]; + let mut it = xs.iter().flat_map(|&x| x..x + 3); + assert_eq!(it.next(), Some(0)); + assert_eq!(it.next_back(), Some(8)); + let i = it.fold(0, |i, x| { + assert_eq!(x, ys[i]); + i + 1 + }); + assert_eq!(i, ys.len()); + + let mut it = xs.iter().flat_map(|&x| x..x + 3); + assert_eq!(it.next(), Some(0)); + assert_eq!(it.next_back(), Some(8)); + let i = it.rfold(ys.len(), |i, x| { + assert_eq!(x, ys[i - 1]); + i - 1 + }); + assert_eq!(i, 0); +} + +#[test] +fn test_flat_map_try_folds() { + let f = &|acc, x| i32::checked_add(acc * 2 / 3, x); + let mr = &|x| (5 * x)..(5 * x + 5); + assert_eq!((0..10).flat_map(mr).try_fold(7, f), (0..50).try_fold(7, f)); + assert_eq!((0..10).flat_map(mr).try_rfold(7, f), (0..50).try_rfold(7, f)); + let mut iter = (0..10).flat_map(mr); + iter.next(); + iter.next_back(); // have front and back iters in progress + assert_eq!(iter.try_rfold(7, f), (1..49).try_rfold(7, f)); + + let mut iter = (0..10).flat_map(|x| (4 * x)..(4 * x + 4)); + assert_eq!(iter.try_fold(0, i8::checked_add), None); + assert_eq!(iter.next(), Some(17)); + assert_eq!(iter.try_rfold(0, i8::checked_add), None); + assert_eq!(iter.next_back(), Some(35)); +} + +#[test] +fn test_double_ended_flat_map() { + let u = [0, 1]; + let v = [5, 6, 7, 8]; + let mut it = u.iter().flat_map(|x| &v[*x..v.len()]); + assert_eq!(it.next_back().unwrap(), &8); + assert_eq!(it.next().unwrap(), &5); + assert_eq!(it.next_back().unwrap(), &7); + assert_eq!(it.next_back().unwrap(), &6); + assert_eq!(it.next_back().unwrap(), &8); + assert_eq!(it.next().unwrap(), &6); + assert_eq!(it.next_back().unwrap(), &7); + assert_eq!(it.next_back(), None); + assert_eq!(it.next(), None); + assert_eq!(it.next_back(), None); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/flatten.rs b/crux-mir/lib/core/tests/iter/adapters/flatten.rs new file mode 100644 index 000000000..690fd0c21 --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/flatten.rs @@ -0,0 +1,212 @@ +use super::*; +use core::iter::*; + +#[test] +fn test_iterator_flatten() { + let xs = [0, 3, 6]; + let ys = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let it = xs.iter().map(|&x| (x..).step_by(1).take(3)).flatten(); + let mut i = 0; + for x in it { + assert_eq!(x, ys[i]); + i += 1; + } + assert_eq!(i, ys.len()); +} + +/// Tests `Flatten::fold` with items already picked off the front and back, +/// to make sure all parts of the `Flatten` are folded correctly. +#[test] +fn test_iterator_flatten_fold() { + let xs = [0, 3, 6]; + let ys = [1, 2, 3, 4, 5, 6, 7]; + let mut it = xs.iter().map(|&x| x..x + 3).flatten(); + assert_eq!(it.next(), Some(0)); + assert_eq!(it.next_back(), Some(8)); + let i = it.fold(0, |i, x| { + assert_eq!(x, ys[i]); + i + 1 + }); + assert_eq!(i, ys.len()); + + let mut it = xs.iter().map(|&x| x..x + 3).flatten(); + assert_eq!(it.next(), Some(0)); + assert_eq!(it.next_back(), Some(8)); + let i = it.rfold(ys.len(), |i, x| { + assert_eq!(x, ys[i - 1]); + i - 1 + }); + assert_eq!(i, 0); +} + +#[test] +fn test_flatten_try_folds() { + let f = &|acc, x| i32::checked_add(acc * 2 / 3, x); + let mr = &|x| (5 * x)..(5 * x + 5); + assert_eq!((0..10).map(mr).flatten().try_fold(7, f), (0..50).try_fold(7, f)); + assert_eq!((0..10).map(mr).flatten().try_rfold(7, f), (0..50).try_rfold(7, f)); + let mut iter = (0..10).map(mr).flatten(); + iter.next(); + iter.next_back(); // have front and back iters in progress + assert_eq!(iter.try_rfold(7, f), (1..49).try_rfold(7, f)); + + let mut iter = (0..10).map(|x| (4 * x)..(4 * x + 4)).flatten(); + assert_eq!(iter.try_fold(0, i8::checked_add), None); + assert_eq!(iter.next(), Some(17)); + assert_eq!(iter.try_rfold(0, i8::checked_add), None); + assert_eq!(iter.next_back(), Some(35)); +} + +#[test] +fn test_flatten_advance_by() { + let mut it = once(0..10).chain(once(10..30)).chain(once(30..40)).flatten(); + + it.advance_by(5).unwrap(); + assert_eq!(it.next(), Some(5)); + it.advance_by(9).unwrap(); + assert_eq!(it.next(), Some(15)); + it.advance_back_by(4).unwrap(); + assert_eq!(it.next_back(), Some(35)); + it.advance_back_by(9).unwrap(); + assert_eq!(it.next_back(), Some(25)); + + assert_eq!(it.advance_by(usize::MAX), Err(9)); + assert_eq!(it.advance_back_by(usize::MAX), Err(0)); + it.advance_by(0).unwrap(); + it.advance_back_by(0).unwrap(); + assert_eq!(it.size_hint(), (0, Some(0))); +} + +#[test] +fn test_flatten_non_fused_outer() { + let mut iter = NonFused::new(once(0..2)).flatten(); + + assert_eq!(iter.next_back(), Some(1)); + assert_eq!(iter.next(), Some(0)); + assert_eq!(iter.next(), None); + assert_eq!(iter.next(), None); + + let mut iter = NonFused::new(once(0..2)).flatten(); + + assert_eq!(iter.next(), Some(0)); + assert_eq!(iter.next_back(), Some(1)); + assert_eq!(iter.next_back(), None); + assert_eq!(iter.next_back(), None); +} + +#[test] +fn test_flatten_non_fused_inner() { + let mut iter = once(0..1).chain(once(1..3)).flat_map(NonFused::new); + + assert_eq!(iter.next_back(), Some(2)); + assert_eq!(iter.next(), Some(0)); + assert_eq!(iter.next(), Some(1)); + assert_eq!(iter.next(), None); + assert_eq!(iter.next(), None); + + let mut iter = once(0..1).chain(once(1..3)).flat_map(NonFused::new); + + assert_eq!(iter.next(), Some(0)); + assert_eq!(iter.next_back(), Some(2)); + assert_eq!(iter.next_back(), Some(1)); + assert_eq!(iter.next_back(), None); + assert_eq!(iter.next_back(), None); +} + +#[test] +fn test_double_ended_flatten() { + let u = [0, 1]; + let v = [5, 6, 7, 8]; + let mut it = u.iter().map(|x| &v[*x..v.len()]).flatten(); + assert_eq!(it.next_back().unwrap(), &8); + assert_eq!(it.next().unwrap(), &5); + assert_eq!(it.next_back().unwrap(), &7); + assert_eq!(it.next_back().unwrap(), &6); + assert_eq!(it.next_back().unwrap(), &8); + assert_eq!(it.next().unwrap(), &6); + assert_eq!(it.next_back().unwrap(), &7); + assert_eq!(it.next_back(), None); + assert_eq!(it.next(), None); + assert_eq!(it.next_back(), None); +} + +#[test] +fn test_trusted_len_flatten() { + fn assert_trusted_len(_: &T) {} + let mut iter = IntoIterator::into_iter([[0; 3]; 4]).flatten(); + assert_trusted_len(&iter); + + assert_eq!(iter.size_hint(), (12, Some(12))); + iter.next(); + assert_eq!(iter.size_hint(), (11, Some(11))); + iter.next_back(); + assert_eq!(iter.size_hint(), (10, Some(10))); + + let iter = IntoIterator::into_iter([[(); usize::MAX]; 1]).flatten(); + assert_eq!(iter.size_hint(), (usize::MAX, Some(usize::MAX))); + + let iter = IntoIterator::into_iter([[(); usize::MAX]; 2]).flatten(); + assert_eq!(iter.size_hint(), (usize::MAX, None)); + + let mut a = [(); 10]; + let mut b = [(); 10]; + + let iter = IntoIterator::into_iter([&mut a, &mut b]).flatten(); + assert_trusted_len(&iter); + assert_eq!(iter.size_hint(), (20, Some(20))); + core::mem::drop(iter); + + let iter = IntoIterator::into_iter([&a, &b]).flatten(); + assert_trusted_len(&iter); + assert_eq!(iter.size_hint(), (20, Some(20))); + + let iter = [(), (), ()].iter().flat_map(|_| [(); 1000]); + assert_trusted_len(&iter); + assert_eq!(iter.size_hint(), (3000, Some(3000))); + + let iter = [(), ()].iter().flat_map(|_| &a); + assert_trusted_len(&iter); + assert_eq!(iter.size_hint(), (20, Some(20))); +} + +#[test] +fn test_flatten_count() { + let mut it = once(0..10).chain(once(10..30)).chain(once(30..40)).flatten(); + + assert_eq!(it.clone().count(), 40); + it.advance_by(5).unwrap(); + assert_eq!(it.clone().count(), 35); + it.advance_back_by(5).unwrap(); + assert_eq!(it.clone().count(), 30); + it.advance_by(10).unwrap(); + assert_eq!(it.clone().count(), 20); + it.advance_back_by(8).unwrap(); + assert_eq!(it.clone().count(), 12); + it.advance_by(4).unwrap(); + assert_eq!(it.clone().count(), 8); + it.advance_back_by(5).unwrap(); + assert_eq!(it.clone().count(), 3); + it.advance_by(3).unwrap(); + assert_eq!(it.clone().count(), 0); +} + +#[test] +fn test_flatten_last() { + let mut it = once(0..10).chain(once(10..30)).chain(once(30..40)).flatten(); + + assert_eq!(it.clone().last(), Some(39)); + it.advance_by(5).unwrap(); // 5..40 + assert_eq!(it.clone().last(), Some(39)); + it.advance_back_by(5).unwrap(); // 5..35 + assert_eq!(it.clone().last(), Some(34)); + it.advance_by(10).unwrap(); // 15..35 + assert_eq!(it.clone().last(), Some(34)); + it.advance_back_by(8).unwrap(); // 15..27 + assert_eq!(it.clone().last(), Some(26)); + it.advance_by(4).unwrap(); // 19..27 + assert_eq!(it.clone().last(), Some(26)); + it.advance_back_by(5).unwrap(); // 19..22 + assert_eq!(it.clone().last(), Some(21)); + it.advance_by(3).unwrap(); // 22..22 + assert_eq!(it.clone().last(), None); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/fuse.rs b/crux-mir/lib/core/tests/iter/adapters/fuse.rs new file mode 100644 index 000000000..f41b379b3 --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/fuse.rs @@ -0,0 +1,75 @@ +use core::iter::*; + +#[test] +fn test_fuse_nth() { + let xs = [0, 1, 2]; + let mut it = xs.iter(); + + assert_eq!(it.len(), 3); + assert_eq!(it.nth(2), Some(&2)); + assert_eq!(it.len(), 0); + assert_eq!(it.nth(2), None); + assert_eq!(it.len(), 0); +} + +#[test] +fn test_fuse_last() { + let xs = [0, 1, 2]; + let it = xs.iter(); + + assert_eq!(it.len(), 3); + assert_eq!(it.last(), Some(&2)); +} + +#[test] +fn test_fuse_count() { + let xs = [0, 1, 2]; + let it = xs.iter(); + + assert_eq!(it.len(), 3); + assert_eq!(it.count(), 3); + // Can't check len now because count consumes. +} + +#[test] +fn test_fuse_fold() { + let xs = [0, 1, 2]; + let it = xs.iter(); // `FusedIterator` + let i = it.fuse().fold(0, |i, &x| { + assert_eq!(x, xs[i]); + i + 1 + }); + assert_eq!(i, xs.len()); + + let it = xs.iter(); // `FusedIterator` + let i = it.fuse().rfold(xs.len(), |i, &x| { + assert_eq!(x, xs[i - 1]); + i - 1 + }); + assert_eq!(i, 0); + + let it = xs.iter().scan((), |_, &x| Some(x)); // `!FusedIterator` + let i = it.fuse().fold(0, |i, x| { + assert_eq!(x, xs[i]); + i + 1 + }); + assert_eq!(i, xs.len()); +} + +#[test] +fn test_fuse() { + let mut it = 0..3; + assert_eq!(it.len(), 3); + assert_eq!(it.next(), Some(0)); + assert_eq!(it.len(), 2); + assert_eq!(it.next(), Some(1)); + assert_eq!(it.len(), 1); + assert_eq!(it.next(), Some(2)); + assert_eq!(it.len(), 0); + assert_eq!(it.next(), None); + assert_eq!(it.len(), 0); + assert_eq!(it.next(), None); + assert_eq!(it.len(), 0); + assert_eq!(it.next(), None); + assert_eq!(it.len(), 0); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/inspect.rs b/crux-mir/lib/core/tests/iter/adapters/inspect.rs new file mode 100644 index 000000000..939e3a28a --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/inspect.rs @@ -0,0 +1,38 @@ +use core::iter::*; + +#[test] +fn test_inspect() { + let xs = [1, 2, 3, 4]; + let mut n = 0; + + let ys = xs.iter().cloned().inspect(|_| n += 1).collect::>(); + + assert_eq!(n, xs.len()); + assert_eq!(&xs[..], &ys[..]); +} + +#[test] +fn test_inspect_fold() { + let xs = [1, 2, 3, 4]; + let mut n = 0; + { + let it = xs.iter().inspect(|_| n += 1); + let i = it.fold(0, |i, &x| { + assert_eq!(x, xs[i]); + i + 1 + }); + assert_eq!(i, xs.len()); + } + assert_eq!(n, xs.len()); + + let mut n = 0; + { + let it = xs.iter().inspect(|_| n += 1); + let i = it.rfold(xs.len(), |i, &x| { + assert_eq!(x, xs[i - 1]); + i - 1 + }); + assert_eq!(i, 0); + } + assert_eq!(n, xs.len()); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/intersperse.rs b/crux-mir/lib/core/tests/iter/adapters/intersperse.rs new file mode 100644 index 000000000..72ae59b6b --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/intersperse.rs @@ -0,0 +1,154 @@ +use core::iter::*; + +#[test] +fn test_intersperse() { + let v = std::iter::empty().intersperse(0u32).collect::>(); + assert_eq!(v, vec![]); + + let v = std::iter::once(1).intersperse(0).collect::>(); + assert_eq!(v, vec![1]); + + let xs = ["a", "", "b", "c"]; + let v: Vec<&str> = xs.iter().map(|x| *x).intersperse(", ").collect(); + let text: String = v.concat(); + assert_eq!(text, "a, , b, c".to_string()); + + let ys = [0, 1, 2, 3]; + let mut it = ys[..0].iter().map(|x| *x).intersperse(1); + assert!(it.next() == None); +} + +#[test] +fn test_intersperse_size_hint() { + let iter = std::iter::empty::().intersperse(0); + assert_eq!(iter.size_hint(), (0, Some(0))); + + let xs = ["a", "", "b", "c"]; + let mut iter = xs.iter().map(|x| *x).intersperse(", "); + assert_eq!(iter.size_hint(), (7, Some(7))); + + assert_eq!(iter.next(), Some("a")); + assert_eq!(iter.size_hint(), (6, Some(6))); + assert_eq!(iter.next(), Some(", ")); + assert_eq!(iter.size_hint(), (5, Some(5))); + + assert_eq!([].iter().intersperse(&()).size_hint(), (0, Some(0))); +} + +#[test] +fn test_fold_specialization_intersperse() { + let mut iter = (1..2).intersperse(0); + iter.clone().for_each(|x| assert_eq!(Some(x), iter.next())); + + let mut iter = (1..3).intersperse(0); + iter.clone().for_each(|x| assert_eq!(Some(x), iter.next())); + + let mut iter = (1..4).intersperse(0); + iter.clone().for_each(|x| assert_eq!(Some(x), iter.next())); +} + +#[test] +fn test_try_fold_specialization_intersperse_ok() { + let mut iter = (1..2).intersperse(0); + iter.clone().try_for_each(|x| { + assert_eq!(Some(x), iter.next()); + Some(()) + }); + + let mut iter = (1..3).intersperse(0); + iter.clone().try_for_each(|x| { + assert_eq!(Some(x), iter.next()); + Some(()) + }); + + let mut iter = (1..4).intersperse(0); + iter.clone().try_for_each(|x| { + assert_eq!(Some(x), iter.next()); + Some(()) + }); +} + +#[test] +fn test_intersperse_with() { + #[derive(PartialEq, Debug)] + struct NotClone { + u: u32, + } + let r = [NotClone { u: 0 }, NotClone { u: 1 }] + .into_iter() + .intersperse_with(|| NotClone { u: 2 }) + .collect::>(); + assert_eq!(r, vec![NotClone { u: 0 }, NotClone { u: 2 }, NotClone { u: 1 }]); + + let mut ctr = 100; + let separator = || { + ctr *= 2; + ctr + }; + let r = (0..3).intersperse_with(separator).collect::>(); + assert_eq!(r, vec![0, 200, 1, 400, 2]); +} + +#[test] +fn test_intersperse_fold() { + let v = (1..4).intersperse(9).fold(Vec::new(), |mut acc, x| { + acc.push(x); + acc + }); + assert_eq!(v.as_slice(), [1, 9, 2, 9, 3]); + + let mut iter = (1..4).intersperse(9); + assert_eq!(iter.next(), Some(1)); + let v = iter.fold(Vec::new(), |mut acc, x| { + acc.push(x); + acc + }); + assert_eq!(v.as_slice(), [9, 2, 9, 3]); + + struct NoneAtStart(i32); // Produces: None, Some(2), Some(3), None, ... + impl Iterator for NoneAtStart { + type Item = i32; + fn next(&mut self) -> Option { + self.0 += 1; + Some(self.0).filter(|i| i % 3 != 1) + } + } + + let v = NoneAtStart(0).intersperse(1000).fold(0, |a, b| a + b); + assert_eq!(v, 0); +} + +#[test] +fn test_intersperse_collect_string() { + let contents = [1, 2, 3]; + + let contents_string = contents + .into_iter() + .map(|id| id.to_string()) + .intersperse(", ".to_owned()) + .collect::(); + assert_eq!(contents_string, "1, 2, 3"); +} + +#[test] +fn test_try_fold_specialization_intersperse_err() { + let orig_iter = ["a", "b"].iter().copied().intersperse("-"); + + // Abort after the first item. + let mut iter = orig_iter.clone(); + iter.try_for_each(|_| None::<()>); + assert_eq!(iter.next(), Some("-")); + assert_eq!(iter.next(), Some("b")); + assert_eq!(iter.next(), None); + + // Abort after the second item. + let mut iter = orig_iter.clone(); + iter.try_for_each(|item| if item == "-" { None } else { Some(()) }); + assert_eq!(iter.next(), Some("b")); + assert_eq!(iter.next(), None); + + // Abort after the third item. + let mut iter = orig_iter.clone(); + iter.try_for_each(|item| if item == "b" { None } else { Some(()) }); + assert_eq!(iter.next(), None); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/map.rs b/crux-mir/lib/core/tests/iter/adapters/map.rs new file mode 100644 index 000000000..77ce3819b --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/map.rs @@ -0,0 +1,27 @@ +use core::iter::*; + +#[test] +fn test_map_try_folds() { + let f = &|acc, x| i32::checked_add(2 * acc, x); + assert_eq!((0..10).map(|x| x + 3).try_fold(7, f), (3..13).try_fold(7, f)); + assert_eq!((0..10).map(|x| x + 3).try_rfold(7, f), (3..13).try_rfold(7, f)); + + let mut iter = (0..40).map(|x| x + 10); + assert_eq!(iter.try_fold(0, i8::checked_add), None); + assert_eq!(iter.next(), Some(20)); + assert_eq!(iter.try_rfold(0, i8::checked_add), None); + assert_eq!(iter.next_back(), Some(46)); +} + +#[test] +fn test_double_ended_map() { + let xs = [1, 2, 3, 4, 5, 6]; + let mut it = xs.iter().map(|&x| x * -1); + assert_eq!(it.next(), Some(-1)); + assert_eq!(it.next(), Some(-2)); + assert_eq!(it.next_back(), Some(-6)); + assert_eq!(it.next_back(), Some(-5)); + assert_eq!(it.next(), Some(-3)); + assert_eq!(it.next_back(), Some(-4)); + assert_eq!(it.next(), None); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/mod.rs b/crux-mir/lib/core/tests/iter/adapters/mod.rs new file mode 100644 index 000000000..ffd5f3857 --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/mod.rs @@ -0,0 +1,209 @@ +mod array_chunks; +mod by_ref_sized; +mod chain; +mod cloned; +mod copied; +mod cycle; +mod enumerate; +mod filter; +mod filter_map; +mod flat_map; +mod flatten; +mod fuse; +mod inspect; +mod intersperse; +mod map; +mod peekable; +mod scan; +mod skip; +mod skip_while; +mod step_by; +mod take; +mod take_while; +mod zip; + +use core::cell::Cell; + +/// An iterator that panics whenever `next` or next_back` is called +/// after `None` has already been returned. This does not violate +/// `Iterator`'s contract. Used to test that iterator adapters don't +/// poll their inner iterators after exhausting them. +pub struct NonFused { + iter: I, + done: bool, +} + +impl NonFused { + pub fn new(iter: I) -> Self { + Self { iter, done: false } + } +} + +impl Iterator for NonFused +where + I: Iterator, +{ + type Item = I::Item; + + fn next(&mut self) -> Option { + assert!(!self.done, "this iterator has already returned None"); + self.iter.next().or_else(|| { + self.done = true; + None + }) + } +} + +impl DoubleEndedIterator for NonFused +where + I: DoubleEndedIterator, +{ + fn next_back(&mut self) -> Option { + assert!(!self.done, "this iterator has already returned None"); + self.iter.next_back().or_else(|| { + self.done = true; + None + }) + } +} + +/// An iterator wrapper that panics whenever `next` or `next_back` is called +/// after `None` has been returned. +pub struct Unfuse { + iter: I, + exhausted: bool, +} + +impl Unfuse { + pub fn new(iter: T) -> Self + where + T: IntoIterator, + { + Self { iter: iter.into_iter(), exhausted: false } + } +} + +impl Iterator for Unfuse +where + I: Iterator, +{ + type Item = I::Item; + + fn next(&mut self) -> Option { + assert!(!self.exhausted); + let next = self.iter.next(); + self.exhausted = next.is_none(); + next + } +} + +impl DoubleEndedIterator for Unfuse +where + I: DoubleEndedIterator, +{ + fn next_back(&mut self) -> Option { + assert!(!self.exhausted); + let next = self.iter.next_back(); + self.exhausted = next.is_none(); + next + } +} + +pub struct Toggle { + is_empty: bool, +} + +impl Iterator for Toggle { + type Item = (); + + // alternates between `None` and `Some(())` + fn next(&mut self) -> Option { + if self.is_empty { + self.is_empty = false; + None + } else { + self.is_empty = true; + Some(()) + } + } + + fn size_hint(&self) -> (usize, Option) { + if self.is_empty { (0, Some(0)) } else { (1, Some(1)) } + } +} + +impl DoubleEndedIterator for Toggle { + fn next_back(&mut self) -> Option { + self.next() + } +} + +/// This is an iterator that follows the Iterator contract, +/// but it is not fused. After having returned None once, it will start +/// producing elements if .next() is called again. +pub struct CycleIter<'a, T> { + index: usize, + data: &'a [T], +} + +impl<'a, T> CycleIter<'a, T> { + pub fn new(data: &'a [T]) -> Self { + Self { index: 0, data } + } +} + +impl<'a, T> Iterator for CycleIter<'a, T> { + type Item = &'a T; + fn next(&mut self) -> Option { + let elt = self.data.get(self.index); + self.index += 1; + self.index %= 1 + self.data.len(); + elt + } +} + +#[derive(Debug)] +struct CountClone(Cell); + +impl CountClone { + pub fn new() -> Self { + Self(Cell::new(0)) + } +} + +impl PartialEq for CountClone { + fn eq(&self, rhs: &i32) -> bool { + self.0.get() == *rhs + } +} + +impl Clone for CountClone { + fn clone(&self) -> Self { + let ret = CountClone(self.0.clone()); + let n = self.0.get(); + self.0.set(n + 1); + ret + } +} + +#[derive(Debug, Clone)] +struct CountDrop<'a> { + dropped: bool, + count: &'a Cell, +} + +impl<'a> CountDrop<'a> { + pub fn new(count: &'a Cell) -> Self { + Self { dropped: false, count } + } +} + +impl Drop for CountDrop<'_> { + fn drop(&mut self) { + if self.dropped { + panic!("double drop"); + } + self.dropped = true; + self.count.set(self.count.get() + 1); + } +} diff --git a/crux-mir/lib/core/tests/iter/adapters/peekable.rs b/crux-mir/lib/core/tests/iter/adapters/peekable.rs new file mode 100644 index 000000000..c1a1c29b6 --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/peekable.rs @@ -0,0 +1,272 @@ +use super::*; +use core::iter::*; + +#[test] +fn test_iterator_peekable() { + let xs = vec![0, 1, 2, 3, 4, 5]; + + let mut it = xs.iter().cloned().peekable(); + assert_eq!(it.len(), 6); + assert_eq!(it.peek().unwrap(), &0); + assert_eq!(it.len(), 6); + assert_eq!(it.next().unwrap(), 0); + assert_eq!(it.len(), 5); + assert_eq!(it.next().unwrap(), 1); + assert_eq!(it.len(), 4); + assert_eq!(it.next().unwrap(), 2); + assert_eq!(it.len(), 3); + assert_eq!(it.peek().unwrap(), &3); + assert_eq!(it.len(), 3); + assert_eq!(it.peek().unwrap(), &3); + assert_eq!(it.len(), 3); + assert_eq!(it.next().unwrap(), 3); + assert_eq!(it.len(), 2); + assert_eq!(it.next().unwrap(), 4); + assert_eq!(it.len(), 1); + assert_eq!(it.peek().unwrap(), &5); + assert_eq!(it.len(), 1); + assert_eq!(it.next().unwrap(), 5); + assert_eq!(it.len(), 0); + assert!(it.peek().is_none()); + assert_eq!(it.len(), 0); + assert!(it.next().is_none()); + assert_eq!(it.len(), 0); + + let mut it = xs.iter().cloned().peekable(); + assert_eq!(it.len(), 6); + assert_eq!(it.peek().unwrap(), &0); + assert_eq!(it.len(), 6); + assert_eq!(it.next_back().unwrap(), 5); + assert_eq!(it.len(), 5); + assert_eq!(it.next_back().unwrap(), 4); + assert_eq!(it.len(), 4); + assert_eq!(it.next_back().unwrap(), 3); + assert_eq!(it.len(), 3); + assert_eq!(it.peek().unwrap(), &0); + assert_eq!(it.len(), 3); + assert_eq!(it.peek().unwrap(), &0); + assert_eq!(it.len(), 3); + assert_eq!(it.next_back().unwrap(), 2); + assert_eq!(it.len(), 2); + assert_eq!(it.next_back().unwrap(), 1); + assert_eq!(it.len(), 1); + assert_eq!(it.peek().unwrap(), &0); + assert_eq!(it.len(), 1); + assert_eq!(it.next_back().unwrap(), 0); + assert_eq!(it.len(), 0); + assert!(it.peek().is_none()); + assert_eq!(it.len(), 0); + assert!(it.next_back().is_none()); + assert_eq!(it.len(), 0); +} + +#[test] +fn test_iterator_peekable_count() { + let xs = [0, 1, 2, 3, 4, 5]; + let ys = [10]; + let zs: [i32; 0] = []; + + assert_eq!(xs.iter().peekable().count(), 6); + + let mut it = xs.iter().peekable(); + assert_eq!(it.peek(), Some(&&0)); + assert_eq!(it.count(), 6); + + assert_eq!(ys.iter().peekable().count(), 1); + + let mut it = ys.iter().peekable(); + assert_eq!(it.peek(), Some(&&10)); + assert_eq!(it.count(), 1); + + assert_eq!(zs.iter().peekable().count(), 0); + + let mut it = zs.iter().peekable(); + assert_eq!(it.peek(), None); +} + +#[test] +fn test_iterator_peekable_nth() { + let xs = [0, 1, 2, 3, 4, 5]; + let mut it = xs.iter().peekable(); + + assert_eq!(it.peek(), Some(&&0)); + assert_eq!(it.nth(0), Some(&0)); + assert_eq!(it.peek(), Some(&&1)); + assert_eq!(it.nth(1), Some(&2)); + assert_eq!(it.peek(), Some(&&3)); + assert_eq!(it.nth(2), Some(&5)); + assert_eq!(it.next(), None); +} + +#[test] +fn test_iterator_peekable_last() { + let xs = [0, 1, 2, 3, 4, 5]; + let ys = [0]; + + let mut it = xs.iter().peekable(); + assert_eq!(it.peek(), Some(&&0)); + assert_eq!(it.last(), Some(&5)); + + let mut it = ys.iter().peekable(); + assert_eq!(it.peek(), Some(&&0)); + assert_eq!(it.last(), Some(&0)); + + let mut it = ys.iter().peekable(); + assert_eq!(it.next(), Some(&0)); + assert_eq!(it.peek(), None); + assert_eq!(it.last(), None); +} + +#[test] +fn test_iterator_peekable_fold() { + let xs = [0, 1, 2, 3, 4, 5]; + let mut it = xs.iter().peekable(); + assert_eq!(it.peek(), Some(&&0)); + let i = it.fold(0, |i, &x| { + assert_eq!(x, xs[i]); + i + 1 + }); + assert_eq!(i, xs.len()); +} + +#[test] +fn test_iterator_peekable_rfold() { + let xs = [0, 1, 2, 3, 4, 5]; + let mut it = xs.iter().peekable(); + assert_eq!(it.peek(), Some(&&0)); + let i = it.rfold(0, |i, &x| { + assert_eq!(x, xs[xs.len() - 1 - i]); + i + 1 + }); + assert_eq!(i, xs.len()); +} + +#[test] +fn test_iterator_peekable_next_if_eq() { + // first, try on references + let xs = ["Heart", "of", "Gold"]; + let mut it = xs.into_iter().peekable(); + // try before `peek()` + assert_eq!(it.next_if_eq(&"trillian"), None); + assert_eq!(it.next_if_eq(&"Heart"), Some("Heart")); + // try after peek() + assert_eq!(it.peek(), Some(&"of")); + assert_eq!(it.next_if_eq(&"of"), Some("of")); + assert_eq!(it.next_if_eq(&"zaphod"), None); + // make sure `next()` still behaves + assert_eq!(it.next(), Some("Gold")); + + // make sure comparison works for owned values + let xs = [String::from("Ludicrous"), "speed".into()]; + let mut it = xs.into_iter().peekable(); + // make sure basic functionality works + assert_eq!(it.next_if_eq("Ludicrous"), Some("Ludicrous".into())); + assert_eq!(it.next_if_eq("speed"), Some("speed".into())); + assert_eq!(it.next_if_eq(""), None); +} + +#[test] +fn test_iterator_peekable_mut() { + let mut it = [1, 2, 3].into_iter().peekable(); + if let Some(p) = it.peek_mut() { + if *p == 1 { + *p = 5; + } + } + assert_eq!(it.collect::>(), vec![5, 2, 3]); +} + +#[test] +fn test_iterator_peekable_remember_peek_none_1() { + // Check that the loop using .peek() terminates + let data = [1, 2, 3]; + let mut iter = CycleIter::new(&data).peekable(); + + let mut n = 0; + while let Some(_) = iter.next() { + let is_the_last = iter.peek().is_none(); + assert_eq!(is_the_last, n == data.len() - 1); + n += 1; + if n > data.len() { + break; + } + } + assert_eq!(n, data.len()); +} + +#[test] +fn test_iterator_peekable_remember_peek_none_2() { + let data = [0]; + let mut iter = CycleIter::new(&data).peekable(); + iter.next(); + assert_eq!(iter.peek(), None); + assert_eq!(iter.last(), None); +} + +#[test] +fn test_iterator_peekable_remember_peek_none_3() { + let data = [0]; + let mut iter = CycleIter::new(&data).peekable(); + iter.peek(); + assert_eq!(iter.nth(0), Some(&0)); + + let mut iter = CycleIter::new(&data).peekable(); + iter.next(); + assert_eq!(iter.peek(), None); + assert_eq!(iter.nth(0), None); +} + +#[test] +fn test_peek_try_folds() { + let f = &|acc, x| i32::checked_add(2 * acc, x); + + assert_eq!((1..20).peekable().try_fold(7, f), (1..20).try_fold(7, f)); + assert_eq!((1..20).peekable().try_rfold(7, f), (1..20).try_rfold(7, f)); + + let mut iter = (1..20).peekable(); + assert_eq!(iter.peek(), Some(&1)); + assert_eq!(iter.try_fold(7, f), (1..20).try_fold(7, f)); + + let mut iter = (1..20).peekable(); + assert_eq!(iter.peek(), Some(&1)); + assert_eq!(iter.try_rfold(7, f), (1..20).try_rfold(7, f)); + + let mut iter = [100, 20, 30, 40, 50, 60, 70].iter().cloned().peekable(); + assert_eq!(iter.peek(), Some(&100)); + assert_eq!(iter.try_fold(0, i8::checked_add), None); + assert_eq!(iter.peek(), Some(&40)); + + let mut iter = [100, 20, 30, 40, 50, 60, 70].iter().cloned().peekable(); + assert_eq!(iter.peek(), Some(&100)); + assert_eq!(iter.try_rfold(0, i8::checked_add), None); + assert_eq!(iter.peek(), Some(&100)); + assert_eq!(iter.next_back(), Some(50)); + + let mut iter = (2..5).peekable(); + assert_eq!(iter.peek(), Some(&2)); + assert_eq!(iter.try_for_each(Err), Err(2)); + assert_eq!(iter.peek(), Some(&3)); + assert_eq!(iter.try_for_each(Err), Err(3)); + assert_eq!(iter.peek(), Some(&4)); + assert_eq!(iter.try_for_each(Err), Err(4)); + assert_eq!(iter.peek(), None); + assert_eq!(iter.try_for_each(Err), Ok(())); + + let mut iter = (2..5).peekable(); + assert_eq!(iter.peek(), Some(&2)); + assert_eq!(iter.try_rfold((), |(), x| Err(x)), Err(4)); + assert_eq!(iter.peek(), Some(&2)); + assert_eq!(iter.try_rfold((), |(), x| Err(x)), Err(3)); + assert_eq!(iter.peek(), Some(&2)); + assert_eq!(iter.try_rfold((), |(), x| Err(x)), Err(2)); + assert_eq!(iter.peek(), None); + assert_eq!(iter.try_rfold((), |(), x| Err(x)), Ok(())); +} + +#[test] +fn test_peekable_non_fused() { + let mut iter = NonFused::new(empty::()).peekable(); + + assert_eq!(iter.peek(), None); + assert_eq!(iter.next_back(), None); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/scan.rs b/crux-mir/lib/core/tests/iter/adapters/scan.rs new file mode 100644 index 000000000..1d28ca6b7 --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/scan.rs @@ -0,0 +1,20 @@ +use core::iter::*; + +#[test] +fn test_iterator_scan() { + // test the type inference + fn add(old: &mut isize, new: &usize) -> Option { + *old += *new as isize; + Some(*old as f64) + } + let xs = [0, 1, 2, 3, 4]; + let ys = [0f64, 1.0, 3.0, 6.0, 10.0]; + + let it = xs.iter().scan(0, add); + let mut i = 0; + for x in it { + assert_eq!(x, ys[i]); + i += 1; + } + assert_eq!(i, ys.len()); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/skip.rs b/crux-mir/lib/core/tests/iter/adapters/skip.rs new file mode 100644 index 000000000..754641834 --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/skip.rs @@ -0,0 +1,234 @@ +use core::iter::*; + +use super::Unfuse; + +#[test] +fn test_iterator_skip() { + let xs = [0, 1, 2, 3, 5, 13, 15, 16, 17, 19, 20, 30]; + let ys = [13, 15, 16, 17, 19, 20, 30]; + let mut it = xs.iter().skip(5); + let mut i = 0; + while let Some(&x) = it.next() { + assert_eq!(x, ys[i]); + i += 1; + assert_eq!(it.len(), xs.len() - 5 - i); + } + assert_eq!(i, ys.len()); + assert_eq!(it.len(), 0); +} + +#[test] +fn test_iterator_skip_doubleended() { + let xs = [0, 1, 2, 3, 5, 13, 15, 16, 17, 19, 20, 30]; + let mut it = xs.iter().rev().skip(5); + assert_eq!(it.next(), Some(&15)); + assert_eq!(it.by_ref().rev().next(), Some(&0)); + assert_eq!(it.next(), Some(&13)); + assert_eq!(it.by_ref().rev().next(), Some(&1)); + assert_eq!(it.next(), Some(&5)); + assert_eq!(it.by_ref().rev().next(), Some(&2)); + assert_eq!(it.next(), Some(&3)); + assert_eq!(it.next(), None); + let mut it = xs.iter().rev().skip(5).rev(); + assert_eq!(it.next(), Some(&0)); + assert_eq!(it.rev().next(), Some(&15)); + let mut it_base = xs.iter(); + { + let mut it = it_base.by_ref().skip(5).rev(); + assert_eq!(it.next(), Some(&30)); + assert_eq!(it.next(), Some(&20)); + assert_eq!(it.next(), Some(&19)); + assert_eq!(it.next(), Some(&17)); + assert_eq!(it.next(), Some(&16)); + assert_eq!(it.next(), Some(&15)); + assert_eq!(it.next(), Some(&13)); + assert_eq!(it.next(), None); + } + // make sure the skipped parts have not been consumed + assert_eq!(it_base.next(), Some(&0)); + assert_eq!(it_base.next(), Some(&1)); + assert_eq!(it_base.next(), Some(&2)); + assert_eq!(it_base.next(), Some(&3)); + assert_eq!(it_base.next(), Some(&5)); + assert_eq!(it_base.next(), None); + let it = xs.iter().skip(5).rev(); + assert_eq!(it.last(), Some(&13)); +} + +#[test] +fn test_iterator_skip_nth() { + let xs = [0, 1, 2, 3, 5, 13, 15, 16, 17, 19, 20, 30]; + + let mut it = xs.iter().skip(0); + assert_eq!(it.nth(0), Some(&0)); + assert_eq!(it.nth(1), Some(&2)); + + let mut it = xs.iter().skip(5); + assert_eq!(it.nth(0), Some(&13)); + assert_eq!(it.nth(1), Some(&16)); + + let mut it = xs.iter().skip(12); + assert_eq!(it.nth(0), None); +} + +#[test] +fn test_skip_advance_by() { + assert_eq!((0..0).skip(10).advance_by(0), Ok(())); + assert_eq!((0..0).skip(10).advance_by(1), Err(0)); + assert_eq!((0u128..(usize::MAX as u128) + 1).skip(usize::MAX).advance_by(usize::MAX), Err(1)); + assert_eq!((0u128..u128::MAX).skip(usize::MAX).advance_by(1), Ok(())); + + assert_eq!((0..2).skip(1).advance_back_by(10), Err(1)); + assert_eq!((0..0).skip(1).advance_back_by(0), Ok(())); +} + +#[test] +fn test_iterator_skip_count() { + let xs = [0, 1, 2, 3, 5, 13, 15, 16, 17, 19, 20, 30]; + + assert_eq!(xs.iter().skip(0).count(), 12); + assert_eq!(xs.iter().skip(1).count(), 11); + assert_eq!(xs.iter().skip(11).count(), 1); + assert_eq!(xs.iter().skip(12).count(), 0); + assert_eq!(xs.iter().skip(13).count(), 0); +} + +#[test] +fn test_iterator_skip_last() { + let xs = [0, 1, 2, 3, 5, 13, 15, 16, 17, 19, 20, 30]; + + assert_eq!(xs.iter().skip(0).last(), Some(&30)); + assert_eq!(xs.iter().skip(1).last(), Some(&30)); + assert_eq!(xs.iter().skip(11).last(), Some(&30)); + assert_eq!(xs.iter().skip(12).last(), None); + assert_eq!(xs.iter().skip(13).last(), None); + + let mut it = xs.iter().skip(5); + assert_eq!(it.next(), Some(&13)); + assert_eq!(it.last(), Some(&30)); +} + +#[test] +fn test_iterator_skip_fold() { + let xs = [0, 1, 2, 3, 5, 13, 15, 16, 17, 19, 20, 30]; + let ys = [13, 15, 16, 17, 19, 20, 30]; + + let it = xs.iter().skip(5); + let i = it.fold(0, |i, &x| { + assert_eq!(x, ys[i]); + i + 1 + }); + assert_eq!(i, ys.len()); + + let mut it = xs.iter().skip(5); + assert_eq!(it.next(), Some(&ys[0])); // process skips before folding + let i = it.fold(1, |i, &x| { + assert_eq!(x, ys[i]); + i + 1 + }); + assert_eq!(i, ys.len()); + + let it = xs.iter().skip(5); + let i = it.rfold(ys.len(), |i, &x| { + let i = i - 1; + assert_eq!(x, ys[i]); + i + }); + assert_eq!(i, 0); + + let mut it = xs.iter().skip(5); + assert_eq!(it.next(), Some(&ys[0])); // process skips before folding + let i = it.rfold(ys.len(), |i, &x| { + let i = i - 1; + assert_eq!(x, ys[i]); + i + }); + assert_eq!(i, 1); +} + +#[test] +fn test_skip_try_folds() { + let f = &|acc, x| i32::checked_add(2 * acc, x); + assert_eq!((1..20).skip(9).try_fold(7, f), (10..20).try_fold(7, f)); + assert_eq!((1..20).skip(9).try_rfold(7, f), (10..20).try_rfold(7, f)); + + let mut iter = (0..30).skip(10); + assert_eq!(iter.try_fold(0, i8::checked_add), None); + assert_eq!(iter.next(), Some(20)); + assert_eq!(iter.try_rfold(0, i8::checked_add), None); + assert_eq!(iter.next_back(), Some(24)); +} + +#[test] +fn test_skip_nth_back() { + let xs = [0, 1, 2, 3, 4, 5]; + let mut it = xs.iter().skip(2); + assert_eq!(it.nth_back(0), Some(&5)); + assert_eq!(it.nth_back(1), Some(&3)); + assert_eq!(it.nth_back(0), Some(&2)); + assert_eq!(it.nth_back(0), None); + + let ys = [2, 3, 4, 5]; + let mut ity = ys.iter(); + let mut it = xs.iter().skip(2); + assert_eq!(it.nth_back(1), ity.nth_back(1)); + assert_eq!(it.clone().nth(0), ity.clone().nth(0)); + assert_eq!(it.nth_back(0), ity.nth_back(0)); + assert_eq!(it.clone().nth(0), ity.clone().nth(0)); + assert_eq!(it.nth_back(0), ity.nth_back(0)); + assert_eq!(it.clone().nth(0), ity.clone().nth(0)); + assert_eq!(it.nth_back(0), ity.nth_back(0)); + assert_eq!(it.clone().nth(0), ity.clone().nth(0)); + + let mut it = xs.iter().skip(2); + assert_eq!(it.nth_back(4), None); + assert_eq!(it.nth_back(0), None); + + let mut it = xs.iter(); + it.by_ref().skip(2).nth_back(3); + assert_eq!(it.next_back(), Some(&1)); + + let mut it = xs.iter(); + it.by_ref().skip(2).nth_back(10); + assert_eq!(it.next_back(), Some(&1)); +} + +#[test] +fn test_skip_non_fused() { + let non_fused = Unfuse::new(0..10); + + // `Skip` would previously exhaust the iterator in this `next` call and then erroneously try to + // advance it further. `Unfuse` tests that this doesn't happen by panicking in that scenario. + let _ = non_fused.skip(20).next(); +} + +#[test] +fn test_skip_non_fused_nth_overflow() { + let non_fused = Unfuse::new(0..10); + + // Ensures that calling skip and `nth` where the sum would overflow does not fail for non-fused + // iterators. + let _ = non_fused.skip(20).nth(usize::MAX); +} + +#[test] +fn test_skip_overflow_wrapping() { + // Test to ensure even on overflowing on `skip+nth` the correct amount of elements are yielded. + struct WrappingIterator(usize); + + impl Iterator for WrappingIterator { + type Item = usize; + + fn next(&mut self) -> core::option::Option { + ::nth(self, 0) + } + + fn nth(&mut self, nth: usize) -> core::option::Option { + self.0 = self.0.wrapping_add(nth.wrapping_add(1)); + Some(self.0) + } + } + + let wrap = WrappingIterator(0); + assert_eq!(wrap.skip(20).nth(usize::MAX), Some(20)); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/skip_while.rs b/crux-mir/lib/core/tests/iter/adapters/skip_while.rs new file mode 100644 index 000000000..929d4f6e6 --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/skip_while.rs @@ -0,0 +1,50 @@ +use core::iter::*; + +#[test] +fn test_iterator_skip_while() { + let xs = [0, 1, 2, 3, 5, 13, 15, 16, 17, 19]; + let ys = [15, 16, 17, 19]; + let it = xs.iter().skip_while(|&x| *x < 15); + let mut i = 0; + for x in it { + assert_eq!(*x, ys[i]); + i += 1; + } + assert_eq!(i, ys.len()); +} + +#[test] +fn test_iterator_skip_while_fold() { + let xs = [0, 1, 2, 3, 5, 13, 15, 16, 17, 19]; + let ys = [15, 16, 17, 19]; + let it = xs.iter().skip_while(|&x| *x < 15); + let i = it.fold(0, |i, &x| { + assert_eq!(x, ys[i]); + i + 1 + }); + assert_eq!(i, ys.len()); + + let mut it = xs.iter().skip_while(|&x| *x < 15); + assert_eq!(it.next(), Some(&ys[0])); // process skips before folding + let i = it.fold(1, |i, &x| { + assert_eq!(x, ys[i]); + i + 1 + }); + assert_eq!(i, ys.len()); +} + +#[test] +fn test_skip_while_try_fold() { + let f = &|acc, x| i32::checked_add(2 * acc, x); + fn p(&x: &i32) -> bool { + (x % 10) <= 5 + } + assert_eq!((1..20).skip_while(p).try_fold(7, f), (6..20).try_fold(7, f)); + let mut iter = (1..20).skip_while(p); + assert_eq!(iter.nth(5), Some(11)); + assert_eq!(iter.try_fold(7, f), (12..20).try_fold(7, f)); + + let mut iter = (0..50).skip_while(|&x| (x % 20) < 15); + assert_eq!(iter.try_fold(0, i8::checked_add), None); + assert_eq!(iter.next(), Some(23)); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/step_by.rs b/crux-mir/lib/core/tests/iter/adapters/step_by.rs new file mode 100644 index 000000000..94f2fa8c2 --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/step_by.rs @@ -0,0 +1,246 @@ +use core::iter::*; + +#[test] +fn test_iterator_step_by() { + // Identity + let mut it = (0..).step_by(1).take(3); + assert_eq!(it.next(), Some(0)); + assert_eq!(it.next(), Some(1)); + assert_eq!(it.next(), Some(2)); + assert_eq!(it.next(), None); + + let mut it = (0..).step_by(3).take(4); + assert_eq!(it.next(), Some(0)); + assert_eq!(it.next(), Some(3)); + assert_eq!(it.next(), Some(6)); + assert_eq!(it.next(), Some(9)); + assert_eq!(it.next(), None); + + let mut it = (0..3).step_by(1); + assert_eq!(it.next_back(), Some(2)); + assert_eq!(it.next_back(), Some(1)); + assert_eq!(it.next_back(), Some(0)); + assert_eq!(it.next_back(), None); + + let mut it = (0..11).step_by(3); + assert_eq!(it.next_back(), Some(9)); + assert_eq!(it.next_back(), Some(6)); + assert_eq!(it.next_back(), Some(3)); + assert_eq!(it.next_back(), Some(0)); + assert_eq!(it.next_back(), None); +} + +#[test] +fn test_iterator_step_by_nth() { + let mut it = (0..16).step_by(5); + assert_eq!(it.nth(0), Some(0)); + assert_eq!(it.nth(0), Some(5)); + assert_eq!(it.nth(0), Some(10)); + assert_eq!(it.nth(0), Some(15)); + assert_eq!(it.nth(0), None); + + let it = (0..18).step_by(5); + assert_eq!(it.clone().nth(0), Some(0)); + assert_eq!(it.clone().nth(1), Some(5)); + assert_eq!(it.clone().nth(2), Some(10)); + assert_eq!(it.clone().nth(3), Some(15)); + assert_eq!(it.clone().nth(4), None); + assert_eq!(it.clone().nth(42), None); +} + +#[test] +fn test_iterator_step_by_nth_overflow() { + #[cfg(target_pointer_width = "16")] + type Bigger = u32; + #[cfg(target_pointer_width = "32")] + type Bigger = u64; + #[cfg(target_pointer_width = "64")] + type Bigger = u128; + + #[derive(Clone)] + struct Test(Bigger); + impl Iterator for &mut Test { + type Item = i32; + fn next(&mut self) -> Option { + Some(21) + } + fn nth(&mut self, n: usize) -> Option { + self.0 += n as Bigger + 1; + Some(42) + } + } + + let mut it = Test(0); + let root = usize::MAX >> (usize::BITS / 2); + let n = root + 20; + (&mut it).step_by(n).nth(n); + assert_eq!(it.0, n as Bigger * n as Bigger); + + // large step + let mut it = Test(0); + (&mut it).step_by(usize::MAX).nth(5); + assert_eq!(it.0, (usize::MAX as Bigger) * 5); + + // n + 1 overflows + let mut it = Test(0); + (&mut it).step_by(2).nth(usize::MAX); + assert_eq!(it.0, (usize::MAX as Bigger) * 2); + + // n + 1 overflows + let mut it = Test(0); + (&mut it).step_by(1).nth(usize::MAX); + assert_eq!(it.0, (usize::MAX as Bigger) * 1); +} + +#[test] +fn test_iterator_step_by_nth_try_fold() { + let mut it = (0..).step_by(10); + assert_eq!(it.try_fold(0, i8::checked_add), None); + assert_eq!(it.next(), Some(60)); + assert_eq!(it.try_fold(0, i8::checked_add), None); + assert_eq!(it.next(), Some(90)); + + let mut it = (100..).step_by(10); + assert_eq!(it.try_fold(50, i8::checked_add), None); + assert_eq!(it.next(), Some(110)); + + let mut it = (100..=100).step_by(10); + assert_eq!(it.next(), Some(100)); + assert_eq!(it.try_fold(0, i8::checked_add), Some(0)); +} + +#[test] +fn test_iterator_step_by_nth_back() { + let mut it = (0..16).step_by(5); + assert_eq!(it.nth_back(0), Some(15)); + assert_eq!(it.nth_back(0), Some(10)); + assert_eq!(it.nth_back(0), Some(5)); + assert_eq!(it.nth_back(0), Some(0)); + assert_eq!(it.nth_back(0), None); + + let mut it = (0..16).step_by(5); + assert_eq!(it.next(), Some(0)); // to set `first_take` to `false` + assert_eq!(it.nth_back(0), Some(15)); + assert_eq!(it.nth_back(0), Some(10)); + assert_eq!(it.nth_back(0), Some(5)); + assert_eq!(it.nth_back(0), None); + + let it = || (0..18).step_by(5); + assert_eq!(it().nth_back(0), Some(15)); + assert_eq!(it().nth_back(1), Some(10)); + assert_eq!(it().nth_back(2), Some(5)); + assert_eq!(it().nth_back(3), Some(0)); + assert_eq!(it().nth_back(4), None); + assert_eq!(it().nth_back(42), None); +} + +#[test] +fn test_iterator_step_by_nth_try_rfold() { + let mut it = (0..100).step_by(10); + assert_eq!(it.try_rfold(0, i8::checked_add), None); + assert_eq!(it.next_back(), Some(70)); + assert_eq!(it.next(), Some(0)); + assert_eq!(it.try_rfold(0, i8::checked_add), None); + assert_eq!(it.next_back(), Some(30)); + + let mut it = (0..100).step_by(10); + assert_eq!(it.try_rfold(50, i8::checked_add), None); + assert_eq!(it.next_back(), Some(80)); + + let mut it = (100..=100).step_by(10); + assert_eq!(it.next_back(), Some(100)); + assert_eq!(it.try_fold(0, i8::checked_add), Some(0)); +} + +#[test] +#[should_panic] +fn test_iterator_step_by_zero() { + let mut it = (0..).step_by(0); + it.next(); +} + +#[test] +fn test_iterator_step_by_size_hint() { + struct StubSizeHint(usize, Option); + impl Iterator for StubSizeHint { + type Item = (); + fn next(&mut self) -> Option<()> { + self.0 -= 1; + if let Some(ref mut upper) = self.1 { + *upper -= 1; + } + Some(()) + } + fn size_hint(&self) -> (usize, Option) { + (self.0, self.1) + } + } + + // The two checks in each case are needed because the logic + // is different before the first call to `next()`. + + let mut it = StubSizeHint(10, Some(10)).step_by(1); + assert_eq!(it.size_hint(), (10, Some(10))); + it.next(); + assert_eq!(it.size_hint(), (9, Some(9))); + + // exact multiple + let mut it = StubSizeHint(10, Some(10)).step_by(3); + assert_eq!(it.size_hint(), (4, Some(4))); + it.next(); + assert_eq!(it.size_hint(), (3, Some(3))); + + // larger base range, but not enough to get another element + let mut it = StubSizeHint(12, Some(12)).step_by(3); + assert_eq!(it.size_hint(), (4, Some(4))); + it.next(); + assert_eq!(it.size_hint(), (3, Some(3))); + + // smaller base range, so fewer resulting elements + let mut it = StubSizeHint(9, Some(9)).step_by(3); + assert_eq!(it.size_hint(), (3, Some(3))); + it.next(); + assert_eq!(it.size_hint(), (2, Some(2))); + + // infinite upper bound + let mut it = StubSizeHint(usize::MAX, None).step_by(1); + assert_eq!(it.size_hint(), (usize::MAX, None)); + it.next(); + assert_eq!(it.size_hint(), (usize::MAX - 1, None)); + + // still infinite with larger step + let mut it = StubSizeHint(7, None).step_by(3); + assert_eq!(it.size_hint(), (3, None)); + it.next(); + assert_eq!(it.size_hint(), (2, None)); + + // propagates ExactSizeIterator + let a = [1, 2, 3, 4, 5]; + let it = a.iter().step_by(2); + assert_eq!(it.len(), 3); + + // Cannot be TrustedLen as a step greater than one makes an iterator + // with (usize::MAX, None) no longer meet the safety requirements + trait TrustedLenCheck { + fn test(self) -> bool; + } + impl TrustedLenCheck for T { + default fn test(self) -> bool { + false + } + } + impl TrustedLenCheck for T { + fn test(self) -> bool { + true + } + } + assert!(TrustedLenCheck::test(a.iter())); + assert!(!TrustedLenCheck::test(a.iter().step_by(1))); +} + +#[test] +fn test_step_by_skip() { + assert_eq!((0..640).step_by(128).skip(1).collect::>(), [128, 256, 384, 512]); + assert_eq!((0..=50).step_by(10).nth(3), Some(30)); + assert_eq!((200..=255u8).step_by(10).nth(3), Some(230)); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/take.rs b/crux-mir/lib/core/tests/iter/adapters/take.rs new file mode 100644 index 000000000..3e26b43a2 --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/take.rs @@ -0,0 +1,168 @@ +use core::iter::*; + +#[test] +fn test_iterator_take() { + let xs = [0, 1, 2, 3, 5, 13, 15, 16, 17, 19]; + let ys = [0, 1, 2, 3, 5]; + + let mut it = xs.iter().take(ys.len()); + let mut i = 0; + assert_eq!(it.len(), ys.len()); + while let Some(&x) = it.next() { + assert_eq!(x, ys[i]); + i += 1; + assert_eq!(it.len(), ys.len() - i); + } + assert_eq!(i, ys.len()); + assert_eq!(it.len(), 0); + + let mut it = xs.iter().take(ys.len()); + let mut i = 0; + assert_eq!(it.len(), ys.len()); + while let Some(&x) = it.next_back() { + i += 1; + assert_eq!(x, ys[ys.len() - i]); + assert_eq!(it.len(), ys.len() - i); + } + assert_eq!(i, ys.len()); + assert_eq!(it.len(), 0); +} + +#[test] +fn test_iterator_take_nth() { + let xs = [0, 1, 2, 4, 5]; + let mut it = xs.iter(); + { + let mut take = it.by_ref().take(3); + let mut i = 0; + while let Some(&x) = take.nth(0) { + assert_eq!(x, i); + i += 1; + } + } + assert_eq!(it.nth(1), Some(&5)); + assert_eq!(it.nth(0), None); + + let xs = [0, 1, 2, 3, 4]; + let mut it = xs.iter().take(7); + let mut i = 1; + while let Some(&x) = it.nth(1) { + assert_eq!(x, i); + i += 2; + } +} + +#[test] +fn test_iterator_take_nth_back() { + let xs = [0, 1, 2, 4, 5]; + let mut it = xs.iter(); + { + let mut take = it.by_ref().take(3); + let mut i = 0; + while let Some(&x) = take.nth_back(0) { + i += 1; + assert_eq!(x, 3 - i); + } + } + assert_eq!(it.nth_back(0), None); + + let xs = [0, 1, 2, 3, 4]; + let mut it = xs.iter().take(7); + assert_eq!(it.nth_back(1), Some(&3)); + assert_eq!(it.nth_back(1), Some(&1)); + assert_eq!(it.nth_back(1), None); +} + +#[test] +fn test_take_advance_by() { + let mut take = (0..10).take(3); + assert_eq!(take.advance_by(2), Ok(())); + assert_eq!(take.next(), Some(2)); + assert_eq!(take.advance_by(1), Err(0)); + + assert_eq!((0..0).take(10).advance_by(0), Ok(())); + assert_eq!((0..0).take(10).advance_by(1), Err(0)); + assert_eq!((0..10).take(4).advance_by(5), Err(4)); + + let mut take = (0..10).take(3); + assert_eq!(take.advance_back_by(2), Ok(())); + assert_eq!(take.next(), Some(0)); + assert_eq!(take.advance_back_by(1), Err(0)); + + assert_eq!((0..2).take(1).advance_back_by(10), Err(1)); + assert_eq!((0..0).take(1).advance_back_by(1), Err(0)); + assert_eq!((0..0).take(1).advance_back_by(0), Ok(())); + assert_eq!((0..usize::MAX).take(100).advance_back_by(usize::MAX), Err(100)); +} + +#[test] +fn test_iterator_take_short() { + let xs = [0, 1, 2, 3]; + + let mut it = xs.iter().take(5); + let mut i = 0; + assert_eq!(it.len(), xs.len()); + while let Some(&x) = it.next() { + assert_eq!(x, xs[i]); + i += 1; + assert_eq!(it.len(), xs.len() - i); + } + assert_eq!(i, xs.len()); + assert_eq!(it.len(), 0); + + let mut it = xs.iter().take(5); + let mut i = 0; + assert_eq!(it.len(), xs.len()); + while let Some(&x) = it.next_back() { + i += 1; + assert_eq!(x, xs[xs.len() - i]); + assert_eq!(it.len(), xs.len() - i); + } + assert_eq!(i, xs.len()); + assert_eq!(it.len(), 0); +} + +#[test] +fn test_take_try_folds() { + let f = &|acc, x| i32::checked_add(2 * acc, x); + assert_eq!((10..30).take(10).try_fold(7, f), (10..20).try_fold(7, f)); + assert_eq!((10..30).take(10).try_rfold(7, f), (10..20).try_rfold(7, f)); + + let mut iter = (10..30).take(20); + assert_eq!(iter.try_fold(0, i8::checked_add), None); + assert_eq!(iter.next(), Some(20)); + assert_eq!(iter.try_rfold(0, i8::checked_add), None); + assert_eq!(iter.next_back(), Some(24)); + + let mut iter = (2..20).take(3); + assert_eq!(iter.try_for_each(Err), Err(2)); + assert_eq!(iter.try_for_each(Err), Err(3)); + assert_eq!(iter.try_for_each(Err), Err(4)); + assert_eq!(iter.try_for_each(Err), Ok(())); + + let mut iter = (2..20).take(3).rev(); + assert_eq!(iter.try_for_each(Err), Err(4)); + assert_eq!(iter.try_for_each(Err), Err(3)); + assert_eq!(iter.try_for_each(Err), Err(2)); + assert_eq!(iter.try_for_each(Err), Ok(())); +} + +#[test] +fn test_byref_take_consumed_items() { + let mut inner = 10..90; + + let mut count = 0; + inner.by_ref().take(0).for_each(|_| count += 1); + assert_eq!(count, 0); + assert_eq!(inner, 10..90); + + let mut count = 0; + inner.by_ref().take(10).for_each(|_| count += 1); + assert_eq!(count, 10); + assert_eq!(inner, 20..90); + + let mut count = 0; + inner.by_ref().take(100).for_each(|_| count += 1); + assert_eq!(count, 70); + assert_eq!(inner, 90..90); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/take_while.rs b/crux-mir/lib/core/tests/iter/adapters/take_while.rs new file mode 100644 index 000000000..6f1ebab29 --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/take_while.rs @@ -0,0 +1,29 @@ +use core::iter::*; + +#[test] +fn test_iterator_take_while() { + let xs = [0, 1, 2, 3, 5, 13, 15, 16, 17, 19]; + let ys = [0, 1, 2, 3, 5, 13]; + let it = xs.iter().take_while(|&x| *x < 15); + let mut i = 0; + for x in it { + assert_eq!(*x, ys[i]); + i += 1; + } + assert_eq!(i, ys.len()); +} + +#[test] +fn test_take_while_folds() { + let f = &|acc, x| i32::checked_add(2 * acc, x); + assert_eq!((1..20).take_while(|&x| x != 10).try_fold(7, f), (1..10).try_fold(7, f)); + let mut iter = (1..20).take_while(|&x| x != 10); + assert_eq!(iter.try_fold(0, |x, y| Some(x + y)), Some((1..10).sum())); + assert_eq!(iter.next(), None, "flag should be set"); + let iter = (1..20).take_while(|&x| x != 10); + assert_eq!(iter.fold(0, |x, y| x + y), (1..10).sum()); + + let mut iter = (10..50).take_while(|&x| x != 40); + assert_eq!(iter.try_fold(0, i8::checked_add), None); + assert_eq!(iter.next(), Some(20)); +} diff --git a/crux-mir/lib/core/tests/iter/adapters/zip.rs b/crux-mir/lib/core/tests/iter/adapters/zip.rs new file mode 100644 index 000000000..585cfbb90 --- /dev/null +++ b/crux-mir/lib/core/tests/iter/adapters/zip.rs @@ -0,0 +1,315 @@ +use super::*; +use core::iter::*; + +#[test] +fn test_zip_nth() { + let xs = [0, 1, 2, 4, 5]; + let ys = [10, 11, 12]; + + let mut it = xs.iter().zip(&ys); + assert_eq!(it.nth(0), Some((&0, &10))); + assert_eq!(it.nth(1), Some((&2, &12))); + assert_eq!(it.nth(0), None); + + let mut it = xs.iter().zip(&ys); + assert_eq!(it.nth(3), None); + + let mut it = ys.iter().zip(&xs); + assert_eq!(it.nth(3), None); +} + +#[test] +fn test_zip_nth_side_effects() { + let mut a = Vec::new(); + let mut b = Vec::new(); + let value = [1, 2, 3, 4, 5, 6] + .iter() + .cloned() + .map(|n| { + a.push(n); + n * 10 + }) + .zip([2, 3, 4, 5, 6, 7, 8].iter().cloned().map(|n| { + b.push(n * 100); + n * 1000 + })) + .skip(1) + .nth(3); + assert_eq!(value, Some((50, 6000))); + assert_eq!(a, vec![1, 2, 3, 4, 5]); + assert_eq!(b, vec![200, 300, 400, 500, 600]); +} + +#[test] +fn test_zip_next_back_side_effects() { + let mut a = Vec::new(); + let mut b = Vec::new(); + let mut iter = [1, 2, 3, 4, 5, 6] + .iter() + .cloned() + .map(|n| { + a.push(n); + n * 10 + }) + .zip([2, 3, 4, 5, 6, 7, 8].iter().cloned().map(|n| { + b.push(n * 100); + n * 1000 + })); + + // The second iterator is one item longer, so `next_back` is called on it + // one more time. + assert_eq!(iter.next_back(), Some((60, 7000))); + assert_eq!(iter.next_back(), Some((50, 6000))); + assert_eq!(iter.next_back(), Some((40, 5000))); + assert_eq!(iter.next_back(), Some((30, 4000))); + assert_eq!(a, vec![6, 5, 4, 3]); + assert_eq!(b, vec![800, 700, 600, 500, 400]); +} + +#[test] +fn test_zip_nth_back_side_effects() { + let mut a = Vec::new(); + let mut b = Vec::new(); + let value = [1, 2, 3, 4, 5, 6] + .iter() + .cloned() + .map(|n| { + a.push(n); + n * 10 + }) + .zip([2, 3, 4, 5, 6, 7, 8].iter().cloned().map(|n| { + b.push(n * 100); + n * 1000 + })) + .nth_back(3); + assert_eq!(value, Some((30, 4000))); + assert_eq!(a, vec![6, 5, 4, 3]); + assert_eq!(b, vec![800, 700, 600, 500, 400]); +} + +#[test] +fn test_zip_next_back_side_effects_exhausted() { + let mut a = Vec::new(); + let mut b = Vec::new(); + let mut iter = [1, 2, 3, 4, 5, 6] + .iter() + .cloned() + .map(|n| { + a.push(n); + n * 10 + }) + .zip([2, 3, 4].iter().cloned().map(|n| { + b.push(n * 100); + n * 1000 + })); + + iter.next(); + iter.next(); + iter.next(); + iter.next(); + assert_eq!(iter.next_back(), None); + assert_eq!(a, vec![1, 2, 3, 4, 6, 5]); + assert_eq!(b, vec![200, 300, 400]); +} + +#[test] +fn test_zip_cloned_sideffectful() { + let xs = [CountClone::new(), CountClone::new(), CountClone::new(), CountClone::new()]; + let ys = [CountClone::new(), CountClone::new()]; + + for _ in xs.iter().cloned().zip(ys.iter().cloned()) {} + + assert_eq!(&xs, &[1, 1, 1, 0][..]); + assert_eq!(&ys, &[1, 1][..]); + + let xs = [CountClone::new(), CountClone::new()]; + let ys = [CountClone::new(), CountClone::new(), CountClone::new(), CountClone::new()]; + + for _ in xs.iter().cloned().zip(ys.iter().cloned()) {} + + assert_eq!(&xs, &[1, 1][..]); + assert_eq!(&ys, &[1, 1, 0, 0][..]); +} + +#[test] +fn test_zip_map_sideffectful() { + let mut xs = [0; 6]; + let mut ys = [0; 4]; + + for _ in xs.iter_mut().map(|x| *x += 1).zip(ys.iter_mut().map(|y| *y += 1)) {} + + assert_eq!(&xs, &[1, 1, 1, 1, 1, 0]); + assert_eq!(&ys, &[1, 1, 1, 1]); + + let mut xs = [0; 4]; + let mut ys = [0; 6]; + + for _ in xs.iter_mut().map(|x| *x += 1).zip(ys.iter_mut().map(|y| *y += 1)) {} + + assert_eq!(&xs, &[1, 1, 1, 1]); + assert_eq!(&ys, &[1, 1, 1, 1, 0, 0]); +} + +#[test] +fn test_zip_map_rev_sideffectful() { + let mut xs = [0; 6]; + let mut ys = [0; 4]; + + { + let mut it = xs.iter_mut().map(|x| *x += 1).zip(ys.iter_mut().map(|y| *y += 1)); + it.next_back(); + } + assert_eq!(&xs, &[0, 0, 0, 1, 1, 1]); + assert_eq!(&ys, &[0, 0, 0, 1]); + + let mut xs = [0; 6]; + let mut ys = [0; 4]; + + { + let mut it = xs.iter_mut().map(|x| *x += 1).zip(ys.iter_mut().map(|y| *y += 1)); + (&mut it).take(5).count(); + it.next_back(); + } + assert_eq!(&xs, &[1, 1, 1, 1, 1, 1]); + assert_eq!(&ys, &[1, 1, 1, 1]); +} + +#[test] +fn test_zip_nested_sideffectful() { + let mut xs = [0; 6]; + let ys = [0; 4]; + + { + // test that it has the side effect nested inside enumerate + let it = xs.iter_mut().map(|x| *x = 1).enumerate().zip(&ys); + it.count(); + } + assert_eq!(&xs, &[1, 1, 1, 1, 1, 0]); +} + +#[test] +fn test_zip_nth_back_side_effects_exhausted() { + let mut a = Vec::new(); + let mut b = Vec::new(); + let mut iter = [1, 2, 3, 4, 5, 6] + .iter() + .cloned() + .map(|n| { + a.push(n); + n * 10 + }) + .zip([2, 3, 4].iter().cloned().map(|n| { + b.push(n * 100); + n * 1000 + })); + + iter.next(); + iter.next(); + iter.next(); + iter.next(); + assert_eq!(iter.nth_back(0), None); + assert_eq!(a, vec![1, 2, 3, 4, 6, 5]); + assert_eq!(b, vec![200, 300, 400]); +} + +#[test] +fn test_zip_trusted_random_access_composition() { + let a = [0, 1, 2, 3, 4]; + let b = a; + let c = a; + + let a = a.iter().copied(); + let b = b.iter().copied(); + let mut c = c.iter().copied(); + c.next(); + + let mut z1 = a.zip(b); + assert_eq!(z1.next().unwrap(), (0, 0)); + + let mut z2 = z1.zip(c); + fn assert_trusted_random_access(_a: &T) {} + assert_trusted_random_access(&z2); + assert_eq!(z2.next().unwrap(), ((1, 1), 1)); +} + +#[test] +#[cfg(panic = "unwind")] +fn test_zip_trusted_random_access_next_back_drop() { + use std::panic::catch_unwind; + use std::panic::AssertUnwindSafe; + + let mut counter = 0; + + let it = [42].iter().map(|e| { + let c = counter; + counter += 1; + if c == 0 { + panic!("bomb"); + } + + e + }); + let it2 = [(); 0].iter(); + let mut zip = it.zip(it2); + catch_unwind(AssertUnwindSafe(|| { + zip.next_back(); + })) + .unwrap_err(); + assert!(zip.next().is_none()); + assert_eq!(counter, 1); +} + +#[test] +fn test_double_ended_zip() { + let xs = [1, 2, 3, 4, 5, 6]; + let ys = [1, 2, 3, 7]; + let mut it = xs.iter().cloned().zip(ys); + assert_eq!(it.next(), Some((1, 1))); + assert_eq!(it.next(), Some((2, 2))); + assert_eq!(it.next_back(), Some((4, 7))); + assert_eq!(it.next_back(), Some((3, 3))); + assert_eq!(it.next(), None); +} + +#[test] +fn test_issue_82282() { + fn overflowed_zip(arr: &[i32]) -> impl Iterator { + static UNIT_EMPTY_ARR: [(); 0] = []; + + let mapped = arr.into_iter().map(|i| *i); + let mut zipped = mapped.zip(UNIT_EMPTY_ARR.iter()); + zipped.next(); + zipped + } + + let arr = [1, 2, 3]; + let zip = overflowed_zip(&arr).zip(overflowed_zip(&arr)); + + assert_eq!(zip.size_hint(), (0, Some(0))); + for _ in zip { + panic!(); + } +} + +#[test] +fn test_issue_82291() { + use std::cell::Cell; + + let mut v1 = [()]; + let v2 = [()]; + + let called = Cell::new(0); + + let mut zip = v1 + .iter_mut() + .map(|r| { + called.set(called.get() + 1); + r + }) + .zip(&v2); + + zip.next_back(); + assert_eq!(called.get(), 1); + zip.next(); + assert_eq!(called.get(), 1); +} diff --git a/crux-mir/lib/core/tests/iter/mod.rs b/crux-mir/lib/core/tests/iter/mod.rs new file mode 100644 index 000000000..770b6f760 --- /dev/null +++ b/crux-mir/lib/core/tests/iter/mod.rs @@ -0,0 +1,102 @@ +//! Note +//! ---- +//! You're probably viewing this file because you're adding a test (or you might +//! just be browsing, in that case, hey there!). +//! +//! The iter test suite is split into two big modules, and some miscellaneous +//! smaller modules. The two big modules are `adapters` and `traits`. +//! +//! `adapters` are for methods on `Iterator` that adapt the data inside the +//! iterator, whether it be by emitting another iterator or returning an item +//! from inside the iterator after executing a closure on each item. +//! +//! `traits` are for trait's that extend an `Iterator` (and the `Iterator` +//! trait itself, mostly containing miscellaneous methods). For the most part, +//! if a test in `traits` uses a specific adapter, then it should be moved to +//! that adapter's test file in `adapters`. + +mod adapters; +mod range; +mod sources; +mod traits; + +use core::cell::Cell; +use core::convert::TryFrom; +use core::iter::*; + +pub fn is_trusted_len(_: I) {} + +#[test] +fn test_multi_iter() { + let xs = [1, 2, 3, 4]; + let ys = [4, 3, 2, 1]; + assert!(xs.iter().eq(ys.iter().rev())); + assert!(xs.iter().lt(xs.iter().skip(2))); +} + +#[test] +fn test_counter_from_iter() { + let it = (0..).step_by(5).take(10); + let xs: Vec = FromIterator::from_iter(it); + assert_eq!(xs, [0, 5, 10, 15, 20, 25, 30, 35, 40, 45]); +} + +#[test] +fn test_functor_laws() { + // identity: + fn identity(x: T) -> T { + x + } + assert_eq!((0..10).map(identity).sum::(), (0..10).sum()); + + // composition: + fn f(x: usize) -> usize { + x + 3 + } + fn g(x: usize) -> usize { + x * 2 + } + fn h(x: usize) -> usize { + g(f(x)) + } + assert_eq!((0..10).map(f).map(g).sum::(), (0..10).map(h).sum()); +} + +#[test] +fn test_monad_laws_left_identity() { + fn f(x: usize) -> impl Iterator { + (0..10).map(move |y| x * y) + } + assert_eq!(once(42).flat_map(f.clone()).sum::(), f(42).sum()); +} + +#[test] +fn test_monad_laws_right_identity() { + assert_eq!((0..10).flat_map(|x| once(x)).sum::(), (0..10).sum()); +} + +#[test] +fn test_monad_laws_associativity() { + fn f(x: usize) -> impl Iterator { + 0..x + } + fn g(x: usize) -> impl Iterator { + (0..x).rev() + } + assert_eq!( + (0..10).flat_map(f).flat_map(g).sum::(), + (0..10).flat_map(|x| f(x).flat_map(g)).sum::() + ); +} + +#[test] +pub fn extend_for_unit() { + let mut x = 0; + { + let iter = (0..5).map(|_| { + x += 1; + }); + ().extend(iter); + } + assert_eq!(x, 5); +} diff --git a/crux-mir/lib/core/tests/iter/range.rs b/crux-mir/lib/core/tests/iter/range.rs new file mode 100644 index 000000000..84498a8ea --- /dev/null +++ b/crux-mir/lib/core/tests/iter/range.rs @@ -0,0 +1,472 @@ +use super::*; + +#[test] +fn test_range() { + assert_eq!((0..5).collect::>(), [0, 1, 2, 3, 4]); + assert_eq!((-10..-1).collect::>(), [-10, -9, -8, -7, -6, -5, -4, -3, -2]); + assert_eq!((0..5).rev().collect::>(), [4, 3, 2, 1, 0]); + assert_eq!((200..-5).count(), 0); + assert_eq!((200..-5).rev().count(), 0); + assert_eq!((200..200).count(), 0); + assert_eq!((200..200).rev().count(), 0); + + assert_eq!((0..100).size_hint(), (100, Some(100))); + // this test is only meaningful when sizeof usize < sizeof u64 + assert_eq!((usize::MAX - 1..usize::MAX).size_hint(), (1, Some(1))); + assert_eq!((-10..-1).size_hint(), (9, Some(9))); + assert_eq!((-1..-10).size_hint(), (0, Some(0))); + + assert_eq!((-70..58).size_hint(), (128, Some(128))); + assert_eq!((-128..127).size_hint(), (255, Some(255))); + assert_eq!( + (-2..isize::MAX).size_hint(), + (isize::MAX as usize + 2, Some(isize::MAX as usize + 2)) + ); +} + +#[test] +fn test_char_range() { + use std::char; + // Miri is too slow + let from = if cfg!(miri) { char::from_u32(0xD800 - 10).unwrap() } else { '\0' }; + let to = if cfg!(miri) { char::from_u32(0xDFFF + 10).unwrap() } else { char::MAX }; + assert!((from..=to).eq((from as u32..=to as u32).filter_map(char::from_u32))); + assert!((from..=to).rev().eq((from as u32..=to as u32).filter_map(char::from_u32).rev())); + + assert_eq!(('\u{D7FF}'..='\u{E000}').count(), 2); + assert_eq!(('\u{D7FF}'..='\u{E000}').size_hint(), (2, Some(2))); + assert_eq!(('\u{D7FF}'..'\u{E000}').count(), 1); + assert_eq!(('\u{D7FF}'..'\u{E000}').size_hint(), (1, Some(1))); +} + +#[test] +fn test_range_exhaustion() { + let mut r = 10..10; + assert!(r.is_empty()); + assert_eq!(r.next(), None); + assert_eq!(r.next_back(), None); + assert_eq!(r, 10..10); + + let mut r = 10..12; + assert_eq!(r.next(), Some(10)); + assert_eq!(r.next(), Some(11)); + assert!(r.is_empty()); + assert_eq!(r, 12..12); + assert_eq!(r.next(), None); + + let mut r = 10..12; + assert_eq!(r.next_back(), Some(11)); + assert_eq!(r.next_back(), Some(10)); + assert!(r.is_empty()); + assert_eq!(r, 10..10); + assert_eq!(r.next_back(), None); + + let mut r = 100..10; + assert!(r.is_empty()); + assert_eq!(r.next(), None); + assert_eq!(r.next_back(), None); + assert_eq!(r, 100..10); +} + +#[test] +fn test_range_inclusive_exhaustion() { + let mut r = 10..=10; + assert_eq!(r.next(), Some(10)); + assert!(r.is_empty()); + assert_eq!(r.next(), None); + assert_eq!(r.next(), None); + + assert_eq!(*r.start(), 10); + assert_eq!(*r.end(), 10); + assert_ne!(r, 10..=10); + + let mut r = 10..=10; + assert_eq!(r.next_back(), Some(10)); + assert!(r.is_empty()); + assert_eq!(r.next_back(), None); + + assert_eq!(*r.start(), 10); + assert_eq!(*r.end(), 10); + assert_ne!(r, 10..=10); + + let mut r = 10..=12; + assert_eq!(r.next(), Some(10)); + assert_eq!(r.next(), Some(11)); + assert_eq!(r.next(), Some(12)); + assert!(r.is_empty()); + assert_eq!(r.next(), None); + + let mut r = 10..=12; + assert_eq!(r.next_back(), Some(12)); + assert_eq!(r.next_back(), Some(11)); + assert_eq!(r.next_back(), Some(10)); + assert!(r.is_empty()); + assert_eq!(r.next_back(), None); + + let mut r = 10..=12; + assert_eq!(r.nth(2), Some(12)); + assert!(r.is_empty()); + assert_eq!(r.next(), None); + + let mut r = 10..=12; + assert_eq!(r.nth(5), None); + assert!(r.is_empty()); + assert_eq!(r.next(), None); + + let mut r = 100..=10; + assert_eq!(r.next(), None); + assert!(r.is_empty()); + assert_eq!(r.next(), None); + assert_eq!(r.next(), None); + assert_eq!(r, 100..=10); + + let mut r = 100..=10; + assert_eq!(r.next_back(), None); + assert!(r.is_empty()); + assert_eq!(r.next_back(), None); + assert_eq!(r.next_back(), None); + assert_eq!(r, 100..=10); +} + +#[test] +fn test_range_nth() { + assert_eq!((10..15).nth(0), Some(10)); + assert_eq!((10..15).nth(1), Some(11)); + assert_eq!((10..15).nth(4), Some(14)); + assert_eq!((10..15).nth(5), None); + + let mut r = 10..20; + assert_eq!(r.nth(2), Some(12)); + assert_eq!(r, 13..20); + assert_eq!(r.nth(2), Some(15)); + assert_eq!(r, 16..20); + assert_eq!(r.nth(10), None); + assert_eq!(r, 20..20); +} + +#[test] +fn test_range_nth_back() { + assert_eq!((10..15).nth_back(0), Some(14)); + assert_eq!((10..15).nth_back(1), Some(13)); + assert_eq!((10..15).nth_back(4), Some(10)); + assert_eq!((10..15).nth_back(5), None); + assert_eq!((-120..80_i8).nth_back(199), Some(-120)); + + let mut r = 10..20; + assert_eq!(r.nth_back(2), Some(17)); + assert_eq!(r, 10..17); + assert_eq!(r.nth_back(2), Some(14)); + assert_eq!(r, 10..14); + assert_eq!(r.nth_back(10), None); + assert_eq!(r, 10..10); +} + +#[test] +fn test_range_from_nth() { + assert_eq!((10..).nth(0), Some(10)); + assert_eq!((10..).nth(1), Some(11)); + assert_eq!((10..).nth(4), Some(14)); + + let mut r = 10..; + assert_eq!(r.nth(2), Some(12)); + assert_eq!(r, 13..); + assert_eq!(r.nth(2), Some(15)); + assert_eq!(r, 16..); + assert_eq!(r.nth(10), Some(26)); + assert_eq!(r, 27..); + + assert_eq!((0..).size_hint(), (usize::MAX, None)); +} + +#[test] +fn test_range_from_take() { + let mut it = (0..).take(3); + assert_eq!(it.next(), Some(0)); + assert_eq!(it.next(), Some(1)); + assert_eq!(it.next(), Some(2)); + assert_eq!(it.next(), None); + is_trusted_len((0..).take(3)); + assert_eq!((0..).take(3).size_hint(), (3, Some(3))); + assert_eq!((0..).take(0).size_hint(), (0, Some(0))); + assert_eq!((0..).take(usize::MAX).size_hint(), (usize::MAX, Some(usize::MAX))); +} + +#[test] +fn test_range_from_take_collect() { + let v: Vec<_> = (0..).take(3).collect(); + assert_eq!(v, vec![0, 1, 2]); +} + +#[test] +fn test_range_inclusive_nth() { + assert_eq!((10..=15).nth(0), Some(10)); + assert_eq!((10..=15).nth(1), Some(11)); + assert_eq!((10..=15).nth(5), Some(15)); + assert_eq!((10..=15).nth(6), None); + + let mut exhausted_via_next = 10_u8..=20; + while exhausted_via_next.next().is_some() {} + + let mut r = 10_u8..=20; + assert_eq!(r.nth(2), Some(12)); + assert_eq!(r, 13..=20); + assert_eq!(r.nth(2), Some(15)); + assert_eq!(r, 16..=20); + assert_eq!(r.is_empty(), false); + assert_eq!(ExactSizeIterator::is_empty(&r), false); + assert_eq!(r.nth(10), None); + assert_eq!(r.is_empty(), true); + assert_eq!(r, exhausted_via_next); + assert_eq!(ExactSizeIterator::is_empty(&r), true); +} + +#[test] +fn test_range_inclusive_nth_back() { + assert_eq!((10..=15).nth_back(0), Some(15)); + assert_eq!((10..=15).nth_back(1), Some(14)); + assert_eq!((10..=15).nth_back(5), Some(10)); + assert_eq!((10..=15).nth_back(6), None); + assert_eq!((-120..=80_i8).nth_back(200), Some(-120)); + + let mut exhausted_via_next_back = 10_u8..=20; + while exhausted_via_next_back.next_back().is_some() {} + + let mut r = 10_u8..=20; + assert_eq!(r.nth_back(2), Some(18)); + assert_eq!(r, 10..=17); + assert_eq!(r.nth_back(2), Some(15)); + assert_eq!(r, 10..=14); + assert_eq!(r.is_empty(), false); + assert_eq!(ExactSizeIterator::is_empty(&r), false); + assert_eq!(r.nth_back(10), None); + assert_eq!(r.is_empty(), true); + assert_eq!(r, exhausted_via_next_back); + assert_eq!(ExactSizeIterator::is_empty(&r), true); +} + +#[test] +fn test_range_len() { + assert_eq!((0..10_u8).len(), 10); + assert_eq!((9..10_u8).len(), 1); + assert_eq!((10..10_u8).len(), 0); + assert_eq!((11..10_u8).len(), 0); + assert_eq!((100..10_u8).len(), 0); +} + +#[test] +fn test_range_inclusive_len() { + assert_eq!((0..=10_u8).len(), 11); + assert_eq!((9..=10_u8).len(), 2); + assert_eq!((10..=10_u8).len(), 1); + assert_eq!((11..=10_u8).len(), 0); + assert_eq!((100..=10_u8).len(), 0); +} + +#[test] +fn test_range_step() { + #![allow(deprecated)] + + assert_eq!((0..20).step_by(5).collect::>(), [0, 5, 10, 15]); + assert_eq!((1..21).rev().step_by(5).collect::>(), [20, 15, 10, 5]); + assert_eq!((1..21).rev().step_by(6).collect::>(), [20, 14, 8, 2]); + assert_eq!((200..255).step_by(50).collect::>(), [200, 250]); + assert_eq!((200..-5).step_by(1).collect::>(), []); + assert_eq!((200..200).step_by(1).collect::>(), []); + + assert_eq!((0..20).step_by(1).size_hint(), (20, Some(20))); + assert_eq!((0..20).step_by(21).size_hint(), (1, Some(1))); + assert_eq!((0..20).step_by(5).size_hint(), (4, Some(4))); + assert_eq!((1..21).rev().step_by(5).size_hint(), (4, Some(4))); + assert_eq!((1..21).rev().step_by(6).size_hint(), (4, Some(4))); + assert_eq!((20..-5).step_by(1).size_hint(), (0, Some(0))); + assert_eq!((20..20).step_by(1).size_hint(), (0, Some(0))); + assert_eq!((i8::MIN..i8::MAX).step_by(-(i8::MIN as i32) as usize).size_hint(), (2, Some(2))); + assert_eq!((i16::MIN..i16::MAX).step_by(i16::MAX as usize).size_hint(), (3, Some(3))); + assert_eq!((isize::MIN..isize::MAX).step_by(1).size_hint(), (usize::MAX, Some(usize::MAX))); +} + +#[test] +fn test_range_advance_by() { + let mut r = 0..usize::MAX; + r.advance_by(0).unwrap(); + r.advance_back_by(0).unwrap(); + + assert_eq!(r.len(), usize::MAX); + + r.advance_by(1).unwrap(); + r.advance_back_by(1).unwrap(); + + assert_eq!((r.start, r.end), (1, usize::MAX - 1)); + + assert_eq!(r.advance_by(usize::MAX), Err(usize::MAX - 2)); + + r.advance_by(0).unwrap(); + r.advance_back_by(0).unwrap(); + + let mut r = 0u128..u128::MAX; + + r.advance_by(usize::MAX).unwrap(); + r.advance_back_by(usize::MAX).unwrap(); + + assert_eq!((r.start, r.end), (0u128 + usize::MAX as u128, u128::MAX - usize::MAX as u128)); +} + +#[test] +fn test_range_inclusive_step() { + assert_eq!((0..=50).step_by(10).collect::>(), [0, 10, 20, 30, 40, 50]); + assert_eq!((0..=5).step_by(1).collect::>(), [0, 1, 2, 3, 4, 5]); + assert_eq!((200..=255u8).step_by(10).collect::>(), [200, 210, 220, 230, 240, 250]); + assert_eq!((250..=255u8).step_by(1).collect::>(), [250, 251, 252, 253, 254, 255]); +} + +#[test] +fn test_range_last_max() { + assert_eq!((0..20).last(), Some(19)); + assert_eq!((-20..0).last(), Some(-1)); + assert_eq!((5..5).last(), None); + + assert_eq!((0..20).max(), Some(19)); + assert_eq!((-20..0).max(), Some(-1)); + assert_eq!((5..5).max(), None); +} + +#[test] +fn test_range_inclusive_last_max() { + assert_eq!((0..=20).last(), Some(20)); + assert_eq!((-20..=0).last(), Some(0)); + assert_eq!((5..=5).last(), Some(5)); + let mut r = 10..=10; + r.next(); + assert_eq!(r.last(), None); + + assert_eq!((0..=20).max(), Some(20)); + assert_eq!((-20..=0).max(), Some(0)); + assert_eq!((5..=5).max(), Some(5)); + let mut r = 10..=10; + r.next(); + assert_eq!(r.max(), None); +} + +#[test] +fn test_range_min() { + assert_eq!((0..20).min(), Some(0)); + assert_eq!((-20..0).min(), Some(-20)); + assert_eq!((5..5).min(), None); +} + +#[test] +fn test_range_inclusive_min() { + assert_eq!((0..=20).min(), Some(0)); + assert_eq!((-20..=0).min(), Some(-20)); + assert_eq!((5..=5).min(), Some(5)); + let mut r = 10..=10; + r.next(); + assert_eq!(r.min(), None); +} + +#[test] +fn test_range_inclusive_folds() { + assert_eq!((1..=10).sum::(), 55); + assert_eq!((1..=10).rev().sum::(), 55); + + let mut it = 44..=50; + assert_eq!(it.try_fold(0, i8::checked_add), None); + assert_eq!(it, 47..=50); + assert_eq!(it.try_fold(0, i8::checked_add), None); + assert_eq!(it, 50..=50); + assert_eq!(it.try_fold(0, i8::checked_add), Some(50)); + assert!(it.is_empty()); + assert_eq!(it.try_fold(0, i8::checked_add), Some(0)); + assert!(it.is_empty()); + + let mut it = 40..=47; + assert_eq!(it.try_rfold(0, i8::checked_add), None); + assert_eq!(it, 40..=44); + assert_eq!(it.try_rfold(0, i8::checked_add), None); + assert_eq!(it, 40..=41); + assert_eq!(it.try_rfold(0, i8::checked_add), Some(81)); + assert!(it.is_empty()); + assert_eq!(it.try_rfold(0, i8::checked_add), Some(0)); + assert!(it.is_empty()); + + let mut it = 10..=20; + assert_eq!(it.try_fold(0, |a, b| Some(a + b)), Some(165)); + assert!(it.is_empty()); + assert_eq!(it.try_fold(0, |a, b| Some(a + b)), Some(0)); + assert!(it.is_empty()); + + let mut it = 10..=20; + assert_eq!(it.try_rfold(0, |a, b| Some(a + b)), Some(165)); + assert!(it.is_empty()); + assert_eq!(it.try_rfold(0, |a, b| Some(a + b)), Some(0)); + assert!(it.is_empty()); +} + +#[test] +fn test_range_size_hint() { + assert_eq!((0..0usize).size_hint(), (0, Some(0))); + assert_eq!((0..100usize).size_hint(), (100, Some(100))); + assert_eq!((0..usize::MAX).size_hint(), (usize::MAX, Some(usize::MAX))); + + let umax = u128::try_from(usize::MAX).unwrap(); + assert_eq!((0..0u128).size_hint(), (0, Some(0))); + assert_eq!((0..100u128).size_hint(), (100, Some(100))); + assert_eq!((0..umax).size_hint(), (usize::MAX, Some(usize::MAX))); + assert_eq!((0..umax + 1).size_hint(), (usize::MAX, None)); + + assert_eq!((0..0isize).size_hint(), (0, Some(0))); + assert_eq!((-100..100isize).size_hint(), (200, Some(200))); + assert_eq!((isize::MIN..isize::MAX).size_hint(), (usize::MAX, Some(usize::MAX))); + + let imin = i128::try_from(isize::MIN).unwrap(); + let imax = i128::try_from(isize::MAX).unwrap(); + assert_eq!((0..0i128).size_hint(), (0, Some(0))); + assert_eq!((-100..100i128).size_hint(), (200, Some(200))); + assert_eq!((imin..imax).size_hint(), (usize::MAX, Some(usize::MAX))); + assert_eq!((imin..imax + 1).size_hint(), (usize::MAX, None)); +} + +#[test] +fn test_range_inclusive_size_hint() { + assert_eq!((1..=0usize).size_hint(), (0, Some(0))); + assert_eq!((0..=0usize).size_hint(), (1, Some(1))); + assert_eq!((0..=100usize).size_hint(), (101, Some(101))); + assert_eq!((0..=usize::MAX - 1).size_hint(), (usize::MAX, Some(usize::MAX))); + assert_eq!((0..=usize::MAX).size_hint(), (usize::MAX, None)); + + let umax = u128::try_from(usize::MAX).unwrap(); + assert_eq!((1..=0u128).size_hint(), (0, Some(0))); + assert_eq!((0..=0u128).size_hint(), (1, Some(1))); + assert_eq!((0..=100u128).size_hint(), (101, Some(101))); + assert_eq!((0..=umax - 1).size_hint(), (usize::MAX, Some(usize::MAX))); + assert_eq!((0..=umax).size_hint(), (usize::MAX, None)); + assert_eq!((0..=umax + 1).size_hint(), (usize::MAX, None)); + + assert_eq!((0..=-1isize).size_hint(), (0, Some(0))); + assert_eq!((0..=0isize).size_hint(), (1, Some(1))); + assert_eq!((-100..=100isize).size_hint(), (201, Some(201))); + assert_eq!((isize::MIN..=isize::MAX - 1).size_hint(), (usize::MAX, Some(usize::MAX))); + assert_eq!((isize::MIN..=isize::MAX).size_hint(), (usize::MAX, None)); + + let imin = i128::try_from(isize::MIN).unwrap(); + let imax = i128::try_from(isize::MAX).unwrap(); + assert_eq!((0..=-1i128).size_hint(), (0, Some(0))); + assert_eq!((0..=0i128).size_hint(), (1, Some(1))); + assert_eq!((-100..=100i128).size_hint(), (201, Some(201))); + assert_eq!((imin..=imax - 1).size_hint(), (usize::MAX, Some(usize::MAX))); + assert_eq!((imin..=imax).size_hint(), (usize::MAX, None)); + assert_eq!((imin..=imax + 1).size_hint(), (usize::MAX, None)); +} + +#[test] +fn test_double_ended_range() { + assert_eq!((11..14).rev().collect::>(), [13, 12, 11]); + for _ in (10..0).rev() { + panic!("unreachable"); + } + + assert_eq!((11..14).rev().collect::>(), [13, 12, 11]); + for _ in (10..0).rev() { + panic!("unreachable"); + } +} diff --git a/crux-mir/lib/core/tests/iter/sources.rs b/crux-mir/lib/core/tests/iter/sources.rs new file mode 100644 index 000000000..a15f3a514 --- /dev/null +++ b/crux-mir/lib/core/tests/iter/sources.rs @@ -0,0 +1,157 @@ +use super::*; +use core::iter::*; + +#[test] +fn test_repeat() { + let mut it = repeat(42); + assert_eq!(it.next(), Some(42)); + assert_eq!(it.next(), Some(42)); + assert_eq!(it.next(), Some(42)); + assert_eq!(repeat(42).size_hint(), (usize::MAX, None)); +} + +#[test] +fn test_repeat_take() { + let mut it = repeat(42).take(3); + assert_eq!(it.next(), Some(42)); + assert_eq!(it.next(), Some(42)); + assert_eq!(it.next(), Some(42)); + assert_eq!(it.next(), None); + is_trusted_len(repeat(42).take(3)); + assert_eq!(repeat(42).take(3).size_hint(), (3, Some(3))); + assert_eq!(repeat(42).take(0).size_hint(), (0, Some(0))); + assert_eq!(repeat(42).take(usize::MAX).size_hint(), (usize::MAX, Some(usize::MAX))); +} + +#[test] +fn test_repeat_take_collect() { + let v: Vec<_> = repeat(42).take(3).collect(); + assert_eq!(v, vec![42, 42, 42]); +} + +#[test] +fn test_repeat_with() { + #[derive(PartialEq, Debug)] + struct NotClone(usize); + let mut it = repeat_with(|| NotClone(42)); + assert_eq!(it.next(), Some(NotClone(42))); + assert_eq!(it.next(), Some(NotClone(42))); + assert_eq!(it.next(), Some(NotClone(42))); + assert_eq!(repeat_with(|| NotClone(42)).size_hint(), (usize::MAX, None)); +} + +#[test] +fn test_repeat_with_take() { + let mut it = repeat_with(|| 42).take(3); + assert_eq!(it.next(), Some(42)); + assert_eq!(it.next(), Some(42)); + assert_eq!(it.next(), Some(42)); + assert_eq!(it.next(), None); + is_trusted_len(repeat_with(|| 42).take(3)); + assert_eq!(repeat_with(|| 42).take(3).size_hint(), (3, Some(3))); + assert_eq!(repeat_with(|| 42).take(0).size_hint(), (0, Some(0))); + assert_eq!(repeat_with(|| 42).take(usize::MAX).size_hint(), (usize::MAX, Some(usize::MAX))); +} + +#[test] +fn test_repeat_with_take_collect() { + let mut curr = 1; + let v: Vec<_> = repeat_with(|| { + let tmp = curr; + curr *= 2; + tmp + }) + .take(5) + .collect(); + assert_eq!(v, vec![1, 2, 4, 8, 16]); +} + +#[test] +fn test_successors() { + let mut powers_of_10 = successors(Some(1_u16), |n| n.checked_mul(10)); + assert_eq!(powers_of_10.by_ref().collect::>(), &[1, 10, 100, 1_000, 10_000]); + assert_eq!(powers_of_10.next(), None); + + let mut empty = successors(None::, |_| unimplemented!()); + assert_eq!(empty.next(), None); + assert_eq!(empty.next(), None); +} + +#[test] +fn test_once() { + let mut it = once(42); + assert_eq!(it.next(), Some(42)); + assert_eq!(it.next(), None); +} + +#[test] +fn test_once_with() { + let count = Cell::new(0); + let mut it = once_with(|| { + count.set(count.get() + 1); + 42 + }); + + assert_eq!(count.get(), 0); + assert_eq!(it.next(), Some(42)); + assert_eq!(count.get(), 1); + assert_eq!(it.next(), None); + assert_eq!(count.get(), 1); + assert_eq!(it.next(), None); + assert_eq!(count.get(), 1); +} + +#[test] +fn test_empty() { + let mut it = empty::(); + assert_eq!(it.next(), None); +} + +#[test] +fn test_repeat_n_drop() { + #[derive(Clone, Debug)] + struct DropCounter<'a>(&'a Cell); + impl Drop for DropCounter<'_> { + fn drop(&mut self) { + self.0.set(self.0.get() + 1); + } + } + + // `repeat_n(x, 0)` drops `x` immediately + let count = Cell::new(0); + let item = DropCounter(&count); + let mut it = repeat_n(item, 0); + assert_eq!(count.get(), 1); + assert!(it.next().is_none()); + assert_eq!(count.get(), 1); + drop(it); + assert_eq!(count.get(), 1); + + // Dropping the iterator needs to drop the item if it's non-empty + let count = Cell::new(0); + let item = DropCounter(&count); + let it = repeat_n(item, 3); + assert_eq!(count.get(), 0); + drop(it); + assert_eq!(count.get(), 1); + + // Dropping the iterator doesn't drop the item if it was exhausted + let count = Cell::new(0); + let item = DropCounter(&count); + let mut it = repeat_n(item, 3); + assert_eq!(count.get(), 0); + let x0 = it.next().unwrap(); + assert_eq!(count.get(), 0); + let x1 = it.next().unwrap(); + assert_eq!(count.get(), 0); + let x2 = it.next().unwrap(); + assert_eq!(count.get(), 0); + assert!(it.next().is_none()); + assert_eq!(count.get(), 0); + assert!(it.next().is_none()); + assert_eq!(count.get(), 0); + drop(it); + assert_eq!(count.get(), 0); + drop((x0, x1, x2)); + assert_eq!(count.get(), 3); +} diff --git a/crux-mir/lib/core/tests/iter/traits/accum.rs b/crux-mir/lib/core/tests/iter/traits/accum.rs new file mode 100644 index 000000000..f3eeb31fe --- /dev/null +++ b/crux-mir/lib/core/tests/iter/traits/accum.rs @@ -0,0 +1,66 @@ +use core::iter::*; + +#[test] +fn test_iterator_sum() { + let v: &[i32] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + assert_eq!(v[..4].iter().cloned().sum::(), 6); + assert_eq!(v.iter().cloned().sum::(), 55); + assert_eq!(v[..0].iter().cloned().sum::(), 0); +} + +#[test] +fn test_iterator_sum_result() { + let v: &[Result] = &[Ok(1), Ok(2), Ok(3), Ok(4)]; + assert_eq!(v.iter().cloned().sum::>(), Ok(10)); + let v: &[Result] = &[Ok(1), Err(()), Ok(3), Ok(4)]; + assert_eq!(v.iter().cloned().sum::>(), Err(())); + + #[derive(PartialEq, Debug)] + struct S(Result); + + impl Sum> for S { + fn sum>>(mut iter: I) -> Self { + // takes the sum by repeatedly calling `next` on `iter`, + // thus testing that repeated calls to `ResultShunt::try_fold` + // produce the expected results + Self(iter.by_ref().sum()) + } + } + + let v: &[Result] = &[Ok(1), Ok(2), Ok(3), Ok(4)]; + assert_eq!(v.iter().cloned().sum::(), S(Ok(10))); + let v: &[Result] = &[Ok(1), Err(()), Ok(3), Ok(4)]; + assert_eq!(v.iter().cloned().sum::(), S(Err(()))); +} + +#[test] +fn test_iterator_sum_option() { + let v: &[Option] = &[Some(1), Some(2), Some(3), Some(4)]; + assert_eq!(v.iter().cloned().sum::>(), Some(10)); + let v: &[Option] = &[Some(1), None, Some(3), Some(4)]; + assert_eq!(v.iter().cloned().sum::>(), None); +} + +#[test] +fn test_iterator_product() { + let v: &[i32] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + assert_eq!(v[..4].iter().cloned().product::(), 0); + assert_eq!(v[1..5].iter().cloned().product::(), 24); + assert_eq!(v[..0].iter().cloned().product::(), 1); +} + +#[test] +fn test_iterator_product_result() { + let v: &[Result] = &[Ok(1), Ok(2), Ok(3), Ok(4)]; + assert_eq!(v.iter().cloned().product::>(), Ok(24)); + let v: &[Result] = &[Ok(1), Err(()), Ok(3), Ok(4)]; + assert_eq!(v.iter().cloned().product::>(), Err(())); +} + +#[test] +fn test_iterator_product_option() { + let v: &[Option] = &[Some(1), Some(2), Some(3), Some(4)]; + assert_eq!(v.iter().cloned().product::>(), Some(24)); + let v: &[Option] = &[Some(1), None, Some(3), Some(4)]; + assert_eq!(v.iter().cloned().product::>(), None); +} diff --git a/crux-mir/lib/core/tests/iter/traits/double_ended.rs b/crux-mir/lib/core/tests/iter/traits/double_ended.rs new file mode 100644 index 000000000..00ef4a6e6 --- /dev/null +++ b/crux-mir/lib/core/tests/iter/traits/double_ended.rs @@ -0,0 +1,91 @@ +//! Note +//! ---- +//! You're probably viewing this file because you're adding a test (or you might +//! just be browsing, in that case, hey there!). +//! +//! If you've made a test that happens to use one of DoubleEnded's methods, but +//! it tests another adapter or trait, you should *add it to the adapter or +//! trait's test file*. +//! +//! Some examples would be `adapters::cloned::test_cloned_try_folds` or +//! `adapters::flat_map::test_double_ended_flat_map`, which use `try_fold` and +//! `next_back`, but test their own adapter. + +#[test] +fn test_iterator_rev_nth_back() { + let v: &[_] = &[0, 1, 2, 3, 4]; + for i in 0..v.len() { + assert_eq!(v.iter().rev().nth_back(i).unwrap(), &v[i]); + } + assert_eq!(v.iter().rev().nth_back(v.len()), None); +} + +#[test] +fn test_iterator_rev_nth() { + let v: &[_] = &[0, 1, 2, 3, 4]; + for i in 0..v.len() { + assert_eq!(v.iter().rev().nth(i).unwrap(), &v[v.len() - 1 - i]); + } + assert_eq!(v.iter().rev().nth(v.len()), None); +} + +#[test] +fn test_rev() { + let xs = [2, 4, 6, 8, 10, 12, 14, 16]; + let mut it = xs.iter(); + it.next(); + it.next(); + assert!(it.rev().cloned().collect::>() == vec![16, 14, 12, 10, 8, 6]); +} + +#[test] +fn test_rev_try_folds() { + let f = &|acc, x| i32::checked_add(2 * acc, x); + assert_eq!((1..10).rev().try_fold(7, f), (1..10).try_rfold(7, f)); + assert_eq!((1..10).rev().try_rfold(7, f), (1..10).try_fold(7, f)); + + let a = [10, 20, 30, 40, 100, 60, 70, 80, 90]; + let mut iter = a.iter().rev(); + assert_eq!(iter.try_fold(0_i8, |acc, &x| acc.checked_add(x)), None); + assert_eq!(iter.next(), Some(&70)); + let mut iter = a.iter().rev(); + assert_eq!(iter.try_rfold(0_i8, |acc, &x| acc.checked_add(x)), None); + assert_eq!(iter.next_back(), Some(&60)); +} + +#[test] +fn test_rposition() { + fn f(xy: &(isize, char)) -> bool { + let (_x, y) = *xy; + y == 'b' + } + fn g(xy: &(isize, char)) -> bool { + let (_x, y) = *xy; + y == 'd' + } + let v = [(0, 'a'), (1, 'b'), (2, 'c'), (3, 'b')]; + + assert_eq!(v.iter().rposition(f), Some(3)); + assert!(v.iter().rposition(g).is_none()); +} + +#[test] +fn test_rev_rposition() { + let v = [0, 0, 1, 1]; + assert_eq!(v.iter().rev().rposition(|&x| x == 1), Some(1)); +} + +#[test] +#[should_panic] +fn test_rposition_panic() { + let u = (Box::new(0), Box::new(0)); + let v: [(Box<_>, Box<_>); 4] = [u.clone(), u.clone(), u.clone(), u]; + let mut i = 0; + v.iter().rposition(|_elt| { + if i == 2 { + panic!() + } + i += 1; + false + }); +} diff --git a/crux-mir/lib/core/tests/iter/traits/iterator.rs b/crux-mir/lib/core/tests/iter/traits/iterator.rs new file mode 100644 index 000000000..37345c1d3 --- /dev/null +++ b/crux-mir/lib/core/tests/iter/traits/iterator.rs @@ -0,0 +1,593 @@ +/// A wrapper struct that implements `Eq` and `Ord` based on the wrapped +/// integer modulo 3. Used to test that `Iterator::max` and `Iterator::min` +/// return the correct element if some of them are equal. +#[derive(Debug)] +struct Mod3(i32); + +impl PartialEq for Mod3 { + fn eq(&self, other: &Self) -> bool { + self.0 % 3 == other.0 % 3 + } +} + +impl Eq for Mod3 {} + +impl PartialOrd for Mod3 { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Mod3 { + fn cmp(&self, other: &Self) -> core::cmp::Ordering { + (self.0 % 3).cmp(&(other.0 % 3)) + } +} + +#[test] +fn test_lt() { + let empty: [isize; 0] = []; + let xs = [1, 2, 3]; + let ys = [1, 2, 0]; + + assert!(!xs.iter().lt(ys.iter())); + assert!(!xs.iter().le(ys.iter())); + assert!(xs.iter().gt(ys.iter())); + assert!(xs.iter().ge(ys.iter())); + + assert!(ys.iter().lt(xs.iter())); + assert!(ys.iter().le(xs.iter())); + assert!(!ys.iter().gt(xs.iter())); + assert!(!ys.iter().ge(xs.iter())); + + assert!(empty.iter().lt(xs.iter())); + assert!(empty.iter().le(xs.iter())); + assert!(!empty.iter().gt(xs.iter())); + assert!(!empty.iter().ge(xs.iter())); + + // Sequence with NaN + let u = [1.0f64, 2.0]; + let v = [0.0f64 / 0.0, 3.0]; + + assert!(!u.iter().lt(v.iter())); + assert!(!u.iter().le(v.iter())); + assert!(!u.iter().gt(v.iter())); + assert!(!u.iter().ge(v.iter())); + + let a = [0.0f64 / 0.0]; + let b = [1.0f64]; + let c = [2.0f64]; + + assert!(a.iter().lt(b.iter()) == (a[0] < b[0])); + assert!(a.iter().le(b.iter()) == (a[0] <= b[0])); + assert!(a.iter().gt(b.iter()) == (a[0] > b[0])); + assert!(a.iter().ge(b.iter()) == (a[0] >= b[0])); + + assert!(c.iter().lt(b.iter()) == (c[0] < b[0])); + assert!(c.iter().le(b.iter()) == (c[0] <= b[0])); + assert!(c.iter().gt(b.iter()) == (c[0] > b[0])); + assert!(c.iter().ge(b.iter()) == (c[0] >= b[0])); +} + +#[test] +fn test_cmp_by() { + use core::cmp::Ordering; + + let f = |x: i32, y: i32| (x * x).cmp(&y); + let xs = || [1, 2, 3, 4].iter().copied(); + let ys = || [1, 4, 16].iter().copied(); + + assert_eq!(xs().cmp_by(ys(), f), Ordering::Less); + assert_eq!(ys().cmp_by(xs(), f), Ordering::Greater); + assert_eq!(xs().cmp_by(xs().map(|x| x * x), f), Ordering::Equal); + assert_eq!(xs().rev().cmp_by(ys().rev(), f), Ordering::Greater); + assert_eq!(xs().cmp_by(ys().rev(), f), Ordering::Less); + assert_eq!(xs().cmp_by(ys().take(2), f), Ordering::Greater); +} + +#[test] +fn test_partial_cmp_by() { + use core::cmp::Ordering; + + let f = |x: i32, y: i32| (x * x).partial_cmp(&y); + let xs = || [1, 2, 3, 4].iter().copied(); + let ys = || [1, 4, 16].iter().copied(); + + assert_eq!(xs().partial_cmp_by(ys(), f), Some(Ordering::Less)); + assert_eq!(ys().partial_cmp_by(xs(), f), Some(Ordering::Greater)); + assert_eq!(xs().partial_cmp_by(xs().map(|x| x * x), f), Some(Ordering::Equal)); + assert_eq!(xs().rev().partial_cmp_by(ys().rev(), f), Some(Ordering::Greater)); + assert_eq!(xs().partial_cmp_by(xs().rev(), f), Some(Ordering::Less)); + assert_eq!(xs().partial_cmp_by(ys().take(2), f), Some(Ordering::Greater)); + + let f = |x: f64, y: f64| (x * x).partial_cmp(&y); + let xs = || [1.0, 2.0, 3.0, 4.0].iter().copied(); + let ys = || [1.0, 4.0, f64::NAN, 16.0].iter().copied(); + + assert_eq!(xs().partial_cmp_by(ys(), f), None); + assert_eq!(ys().partial_cmp_by(xs(), f), Some(Ordering::Greater)); +} + +#[test] +fn test_eq_by() { + let f = |x: i32, y: i32| x * x == y; + let xs = || [1, 2, 3, 4].iter().copied(); + let ys = || [1, 4, 9, 16].iter().copied(); + + assert!(xs().eq_by(ys(), f)); + assert!(!ys().eq_by(xs(), f)); + assert!(!xs().eq_by(xs(), f)); + assert!(!ys().eq_by(ys(), f)); + + assert!(!xs().take(3).eq_by(ys(), f)); + assert!(!xs().eq_by(ys().take(3), f)); + assert!(xs().take(3).eq_by(ys().take(3), f)); +} + +#[test] +fn test_iterator_nth() { + let v: &[_] = &[0, 1, 2, 3, 4]; + for i in 0..v.len() { + assert_eq!(v.iter().nth(i).unwrap(), &v[i]); + } + assert_eq!(v.iter().nth(v.len()), None); +} + +#[test] +fn test_iterator_nth_back() { + let v: &[_] = &[0, 1, 2, 3, 4]; + for i in 0..v.len() { + assert_eq!(v.iter().nth_back(i).unwrap(), &v[v.len() - 1 - i]); + } + assert_eq!(v.iter().nth_back(v.len()), None); +} + +#[test] +fn test_iterator_advance_by() { + let v: &[_] = &[0, 1, 2, 3, 4]; + + for i in 0..v.len() { + let mut iter = v.iter(); + assert_eq!(iter.advance_by(i), Ok(())); + assert_eq!(iter.next().unwrap(), &v[i]); + assert_eq!(iter.advance_by(100), Err(v.len() - 1 - i)); + } + + assert_eq!(v.iter().advance_by(v.len()), Ok(())); + assert_eq!(v.iter().advance_by(100), Err(v.len())); +} + +#[test] +fn test_iterator_advance_back_by() { + let v: &[_] = &[0, 1, 2, 3, 4]; + + for i in 0..v.len() { + let mut iter = v.iter(); + assert_eq!(iter.advance_back_by(i), Ok(())); + assert_eq!(iter.next_back().unwrap(), &v[v.len() - 1 - i]); + assert_eq!(iter.advance_back_by(100), Err(v.len() - 1 - i)); + } + + assert_eq!(v.iter().advance_back_by(v.len()), Ok(())); + assert_eq!(v.iter().advance_back_by(100), Err(v.len())); +} + +#[test] +fn test_iterator_rev_advance_back_by() { + let v: &[_] = &[0, 1, 2, 3, 4]; + + for i in 0..v.len() { + let mut iter = v.iter().rev(); + assert_eq!(iter.advance_back_by(i), Ok(())); + assert_eq!(iter.next_back().unwrap(), &v[i]); + assert_eq!(iter.advance_back_by(100), Err(v.len() - 1 - i)); + } + + assert_eq!(v.iter().rev().advance_back_by(v.len()), Ok(())); + assert_eq!(v.iter().rev().advance_back_by(100), Err(v.len())); +} + +#[test] +fn test_iterator_last() { + let v: &[_] = &[0, 1, 2, 3, 4]; + assert_eq!(v.iter().last().unwrap(), &4); + assert_eq!(v[..1].iter().last().unwrap(), &0); +} + +#[test] +fn test_iterator_max() { + let v: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + assert_eq!(v[..4].iter().cloned().max(), Some(3)); + assert_eq!(v.iter().cloned().max(), Some(10)); + assert_eq!(v[..0].iter().cloned().max(), None); + assert_eq!(v.iter().cloned().map(Mod3).max().map(|x| x.0), Some(8)); +} + +#[test] +fn test_iterator_min() { + let v: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + assert_eq!(v[..4].iter().cloned().min(), Some(0)); + assert_eq!(v.iter().cloned().min(), Some(0)); + assert_eq!(v[..0].iter().cloned().min(), None); + assert_eq!(v.iter().cloned().map(Mod3).min().map(|x| x.0), Some(0)); +} + +#[test] +fn test_iterator_size_hint() { + let c = (0..).step_by(1); + let v: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + let v2 = &[10, 11, 12]; + let vi = v.iter(); + + assert_eq!((0..).size_hint(), (usize::MAX, None)); + assert_eq!(c.size_hint(), (usize::MAX, None)); + assert_eq!(vi.clone().size_hint(), (10, Some(10))); + + assert_eq!(c.clone().take(5).size_hint(), (5, Some(5))); + assert_eq!(c.clone().skip(5).size_hint().1, None); + assert_eq!(c.clone().take_while(|_| false).size_hint(), (0, None)); + assert_eq!(c.clone().map_while(|_| None::<()>).size_hint(), (0, None)); + assert_eq!(c.clone().skip_while(|_| false).size_hint(), (0, None)); + assert_eq!(c.clone().enumerate().size_hint(), (usize::MAX, None)); + assert_eq!(c.clone().chain(vi.clone().cloned()).size_hint(), (usize::MAX, None)); + assert_eq!(c.clone().zip(vi.clone()).size_hint(), (10, Some(10))); + assert_eq!(c.clone().scan(0, |_, _| Some(0)).size_hint(), (0, None)); + assert_eq!(c.clone().filter(|_| false).size_hint(), (0, None)); + assert_eq!(c.clone().map(|_| 0).size_hint(), (usize::MAX, None)); + assert_eq!(c.filter_map(|_| Some(0)).size_hint(), (0, None)); + + assert_eq!(vi.clone().take(5).size_hint(), (5, Some(5))); + assert_eq!(vi.clone().take(12).size_hint(), (10, Some(10))); + assert_eq!(vi.clone().skip(3).size_hint(), (7, Some(7))); + assert_eq!(vi.clone().skip(12).size_hint(), (0, Some(0))); + assert_eq!(vi.clone().take_while(|_| false).size_hint(), (0, Some(10))); + assert_eq!(vi.clone().map_while(|_| None::<()>).size_hint(), (0, Some(10))); + assert_eq!(vi.clone().skip_while(|_| false).size_hint(), (0, Some(10))); + assert_eq!(vi.clone().enumerate().size_hint(), (10, Some(10))); + assert_eq!(vi.clone().chain(v2).size_hint(), (13, Some(13))); + assert_eq!(vi.clone().zip(v2).size_hint(), (3, Some(3))); + assert_eq!(vi.clone().scan(0, |_, _| Some(0)).size_hint(), (0, Some(10))); + assert_eq!(vi.clone().filter(|_| false).size_hint(), (0, Some(10))); + assert_eq!(vi.clone().map(|&i| i + 1).size_hint(), (10, Some(10))); + assert_eq!(vi.filter_map(|_| Some(0)).size_hint(), (0, Some(10))); +} + +#[test] +fn test_all() { + let v: Box<[isize]> = Box::new([1, 2, 3, 4, 5]); + assert!(v.iter().all(|&x| x < 10)); + assert!(!v.iter().all(|&x| x % 2 == 0)); + assert!(!v.iter().all(|&x| x > 100)); + assert!(v[..0].iter().all(|_| panic!())); +} + +#[test] +fn test_any() { + let v: Box<[isize]> = Box::new([1, 2, 3, 4, 5]); + assert!(v.iter().any(|&x| x < 10)); + assert!(v.iter().any(|&x| x % 2 == 0)); + assert!(!v.iter().any(|&x| x > 100)); + assert!(!v[..0].iter().any(|_| panic!())); +} + +#[test] +fn test_find() { + let v: &[isize] = &[1, 3, 9, 27, 103, 14, 11]; + assert_eq!(*v.iter().find(|&&x| x & 1 == 0).unwrap(), 14); + assert_eq!(*v.iter().find(|&&x| x % 3 == 0).unwrap(), 3); + assert!(v.iter().find(|&&x| x % 12 == 0).is_none()); +} + +#[test] +fn test_try_find() { + let xs: &[isize] = &[]; + assert_eq!(xs.iter().try_find(testfn), Ok(None)); + let xs: &[isize] = &[1, 2, 3, 4]; + assert_eq!(xs.iter().try_find(testfn), Ok(Some(&2))); + let xs: &[isize] = &[1, 3, 4]; + assert_eq!(xs.iter().try_find(testfn), Err(())); + + let xs: &[isize] = &[1, 2, 3, 4, 5, 6, 7]; + let mut iter = xs.iter(); + assert_eq!(iter.try_find(testfn), Ok(Some(&2))); + assert_eq!(iter.try_find(testfn), Err(())); + assert_eq!(iter.next(), Some(&5)); + + fn testfn(x: &&isize) -> Result { + if **x == 2 { + return Ok(true); + } + if **x == 4 { + return Err(()); + } + Ok(false) + } +} + +#[test] +fn test_try_find_api_usability() -> Result<(), Box> { + let a = ["1", "2"]; + + let is_my_num = |s: &str, search: i32| -> Result { + Ok(s.parse::()? == search) + }; + + let val = a.iter().try_find(|&&s| is_my_num(s, 2))?; + assert_eq!(val, Some(&"2")); + + Ok(()) +} + +#[test] +fn test_position() { + let v = &[1, 3, 9, 27, 103, 14, 11]; + assert_eq!(v.iter().position(|x| *x & 1 == 0).unwrap(), 5); + assert_eq!(v.iter().position(|x| *x % 3 == 0).unwrap(), 1); + assert!(v.iter().position(|x| *x % 12 == 0).is_none()); +} + +#[test] +fn test_count() { + let xs = &[1, 2, 2, 1, 5, 9, 0, 2]; + assert_eq!(xs.iter().filter(|x| **x == 2).count(), 3); + assert_eq!(xs.iter().filter(|x| **x == 5).count(), 1); + assert_eq!(xs.iter().filter(|x| **x == 95).count(), 0); +} + +#[test] +fn test_max_by_key() { + let xs: &[isize] = &[-3, 0, 1, 5, -10]; + assert_eq!(*xs.iter().max_by_key(|x| x.abs()).unwrap(), -10); +} + +#[test] +fn test_max_by() { + let xs: &[isize] = &[-3, 0, 1, 5, -10]; + assert_eq!(*xs.iter().max_by(|x, y| x.abs().cmp(&y.abs())).unwrap(), -10); +} + +#[test] +fn test_min_by_key() { + let xs: &[isize] = &[-3, 0, 1, 5, -10]; + assert_eq!(*xs.iter().min_by_key(|x| x.abs()).unwrap(), 0); +} + +#[test] +fn test_min_by() { + let xs: &[isize] = &[-3, 0, 1, 5, -10]; + assert_eq!(*xs.iter().min_by(|x, y| x.abs().cmp(&y.abs())).unwrap(), 0); +} + +#[test] +fn test_by_ref() { + let mut xs = 0..10; + // sum the first five values + let partial_sum = xs.by_ref().take(5).fold(0, |a, b| a + b); + assert_eq!(partial_sum, 10); + assert_eq!(xs.next(), Some(5)); +} + +#[test] +fn test_is_sorted() { + assert!([1, 2, 2, 9].iter().is_sorted()); + assert!(![1, 3, 2].iter().is_sorted()); + assert!([0].iter().is_sorted()); + assert!(std::iter::empty::().is_sorted()); + assert!(![0.0, 1.0, f32::NAN].iter().is_sorted()); + assert!([-2, -1, 0, 3].iter().is_sorted()); + assert!(![-2i32, -1, 0, 3].iter().is_sorted_by_key(|n| n.abs())); + assert!(!["c", "bb", "aaa"].iter().is_sorted()); + assert!(["c", "bb", "aaa"].iter().is_sorted_by_key(|s| s.len())); +} + +#[test] +fn test_partition() { + fn check(xs: &mut [i32], ref p: impl Fn(&i32) -> bool, expected: usize) { + let i = xs.iter_mut().partition_in_place(p); + assert_eq!(expected, i); + assert!(xs[..i].iter().all(p)); + assert!(!xs[i..].iter().any(p)); + assert!(xs.iter().is_partitioned(p)); + if i == 0 || i == xs.len() { + assert!(xs.iter().rev().is_partitioned(p)); + } else { + assert!(!xs.iter().rev().is_partitioned(p)); + } + } + + check(&mut [], |_| true, 0); + check(&mut [], |_| false, 0); + + check(&mut [0], |_| true, 1); + check(&mut [0], |_| false, 0); + + check(&mut [-1, 1], |&x| x > 0, 1); + check(&mut [-1, 1], |&x| x < 0, 1); + + let ref mut xs = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + check(xs, |_| true, 10); + check(xs, |_| false, 0); + check(xs, |&x| x % 2 == 0, 5); // evens + check(xs, |&x| x % 2 == 1, 5); // odds + check(xs, |&x| x % 3 == 0, 4); // multiple of 3 + check(xs, |&x| x % 4 == 0, 3); // multiple of 4 + check(xs, |&x| x % 5 == 0, 2); // multiple of 5 + check(xs, |&x| x < 3, 3); // small + check(xs, |&x| x > 6, 3); // large +} + +#[test] +fn test_iterator_rev_advance_by() { + let v: &[_] = &[0, 1, 2, 3, 4]; + + for i in 0..v.len() { + let mut iter = v.iter().rev(); + assert_eq!(iter.advance_by(i), Ok(())); + assert_eq!(iter.next().unwrap(), &v[v.len() - 1 - i]); + assert_eq!(iter.advance_by(100), Err(v.len() - 1 - i)); + } + + assert_eq!(v.iter().rev().advance_by(v.len()), Ok(())); + assert_eq!(v.iter().rev().advance_by(100), Err(v.len())); +} + +#[test] +fn test_find_map() { + let xs: &[isize] = &[]; + assert_eq!(xs.iter().find_map(half_if_even), None); + let xs: &[isize] = &[3, 5]; + assert_eq!(xs.iter().find_map(half_if_even), None); + let xs: &[isize] = &[4, 5]; + assert_eq!(xs.iter().find_map(half_if_even), Some(2)); + let xs: &[isize] = &[3, 6]; + assert_eq!(xs.iter().find_map(half_if_even), Some(3)); + + let xs: &[isize] = &[1, 2, 3, 4, 5, 6, 7]; + let mut iter = xs.iter(); + assert_eq!(iter.find_map(half_if_even), Some(1)); + assert_eq!(iter.find_map(half_if_even), Some(2)); + assert_eq!(iter.find_map(half_if_even), Some(3)); + assert_eq!(iter.next(), Some(&7)); + + fn half_if_even(x: &isize) -> Option { + if x % 2 == 0 { Some(x / 2) } else { None } + } +} + +#[test] +fn test_try_reduce() { + let v = [1usize, 2, 3, 4, 5]; + let sum = v.into_iter().try_reduce(|x, y| x.checked_add(y)); + assert_eq!(sum, Some(Some(15))); + + let v = [1, 2, 3, 4, 5, usize::MAX]; + let sum = v.into_iter().try_reduce(|x, y| x.checked_add(y)); + assert_eq!(sum, None); + + let v: [usize; 0] = []; + let sum = v.into_iter().try_reduce(|x, y| x.checked_add(y)); + assert_eq!(sum, Some(None)); + + let v = ["1", "2", "3", "4", "5"]; + let max = v.into_iter().try_reduce(|x, y| { + if x.parse::().ok()? > y.parse::().ok()? { Some(x) } else { Some(y) } + }); + assert_eq!(max, Some(Some("5"))); + + let v = ["1", "2", "3", "4", "5"]; + let max: Result, ::Err> = + v.into_iter().try_reduce(|x, y| { + if x.parse::()? > y.parse::()? { Ok(x) } else { Ok(y) } + }); + assert_eq!(max, Ok(Some("5"))); +} + +#[test] +fn test_iterator_len() { + let v: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + assert_eq!(v[..4].iter().count(), 4); + assert_eq!(v[..10].iter().count(), 10); + assert_eq!(v[..0].iter().count(), 0); +} + +#[test] +fn test_collect() { + let a = vec![1, 2, 3, 4, 5]; + let b: Vec = a.iter().cloned().collect(); + assert!(a == b); +} + +#[test] +fn test_try_collect() { + use core::ops::ControlFlow::{Break, Continue}; + + let u = vec![Some(1), Some(2), Some(3)]; + let v = u.into_iter().try_collect::>(); + assert_eq!(v, Some(vec![1, 2, 3])); + + let u = vec![Some(1), Some(2), None, Some(3)]; + let mut it = u.into_iter(); + let v = it.try_collect::>(); + assert_eq!(v, None); + let v = it.try_collect::>(); + assert_eq!(v, Some(vec![3])); + + let u: Vec> = vec![Ok(1), Ok(2), Ok(3)]; + let v = u.into_iter().try_collect::>(); + assert_eq!(v, Ok(vec![1, 2, 3])); + + let u = vec![Ok(1), Ok(2), Err(()), Ok(3)]; + let v = u.into_iter().try_collect::>(); + assert_eq!(v, Err(())); + + let numbers = vec![1, 2, 3, 4, 5]; + let all_positive = numbers + .iter() + .cloned() + .map(|n| if n > 0 { Some(n) } else { None }) + .try_collect::>(); + assert_eq!(all_positive, Some(numbers)); + + let numbers = vec![-2, -1, 0, 1, 2]; + let all_positive = + numbers.into_iter().map(|n| if n > 0 { Some(n) } else { None }).try_collect::>(); + assert_eq!(all_positive, None); + + let u = [Continue(1), Continue(2), Break(3), Continue(4), Continue(5)]; + let mut it = u.into_iter(); + + let v = it.try_collect::>(); + assert_eq!(v, Break(3)); + + let v = it.try_collect::>(); + assert_eq!(v, Continue(vec![4, 5])); +} + +#[test] +fn test_collect_into() { + let a = vec![1, 2, 3, 4, 5]; + let mut b = Vec::new(); + a.iter().cloned().collect_into(&mut b); + assert!(a == b); +} + +#[test] +fn iter_try_collect_uses_try_fold_not_next() { + // This makes sure it picks up optimizations, and doesn't use the `&mut I` impl. + struct PanicOnNext(I); + impl Iterator for PanicOnNext { + type Item = I::Item; + fn next(&mut self) -> Option { + panic!("Iterator::next should not be called!") + } + fn try_fold(&mut self, init: B, f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: std::ops::Try, + { + self.0.try_fold(init, f) + } + } + + let it = (0..10).map(Some); + let _ = PanicOnNext(it).try_collect::>(); + // validation is just that it didn't panic. +} + +#[test] +fn test_next_chunk() { + let mut it = 0..12; + assert_eq!(it.next_chunk().unwrap(), [0, 1, 2, 3]); + assert_eq!(it.next_chunk().unwrap(), []); + assert_eq!(it.next_chunk().unwrap(), [4, 5, 6, 7, 8, 9]); + assert_eq!(it.next_chunk::<4>().unwrap_err().as_slice(), &[10, 11]); +} + +// just tests by whether or not this compiles +fn _empty_impl_all_auto_traits() { + use std::panic::{RefUnwindSafe, UnwindSafe}; + fn all_auto_traits() {} + + all_auto_traits::>(); +} diff --git a/crux-mir/lib/core/tests/iter/traits/mod.rs b/crux-mir/lib/core/tests/iter/traits/mod.rs new file mode 100644 index 000000000..80619f53f --- /dev/null +++ b/crux-mir/lib/core/tests/iter/traits/mod.rs @@ -0,0 +1,4 @@ +mod accum; +mod double_ended; +mod iterator; +mod step; diff --git a/crux-mir/lib/core/tests/iter/traits/step.rs b/crux-mir/lib/core/tests/iter/traits/step.rs new file mode 100644 index 000000000..3d82a40cd --- /dev/null +++ b/crux-mir/lib/core/tests/iter/traits/step.rs @@ -0,0 +1,89 @@ +use core::iter::*; + +#[test] +fn test_steps_between() { + assert_eq!(Step::steps_between(&20_u8, &200_u8), Some(180_usize)); + assert_eq!(Step::steps_between(&-20_i8, &80_i8), Some(100_usize)); + assert_eq!(Step::steps_between(&-120_i8, &80_i8), Some(200_usize)); + assert_eq!(Step::steps_between(&20_u32, &4_000_100_u32), Some(4_000_080_usize)); + assert_eq!(Step::steps_between(&-20_i32, &80_i32), Some(100_usize)); + assert_eq!(Step::steps_between(&-2_000_030_i32, &2_000_050_i32), Some(4_000_080_usize)); + + // Skip u64/i64 to avoid differences with 32-bit vs 64-bit platforms + + assert_eq!(Step::steps_between(&20_u128, &200_u128), Some(180_usize)); + assert_eq!(Step::steps_between(&-20_i128, &80_i128), Some(100_usize)); + if cfg!(target_pointer_width = "64") { + assert_eq!(Step::steps_between(&10_u128, &0x1_0000_0000_0000_0009_u128), Some(usize::MAX)); + } + assert_eq!(Step::steps_between(&10_u128, &0x1_0000_0000_0000_000a_u128), None); + assert_eq!(Step::steps_between(&10_i128, &0x1_0000_0000_0000_000a_i128), None); + assert_eq!( + Step::steps_between(&-0x1_0000_0000_0000_0000_i128, &0x1_0000_0000_0000_0000_i128,), + None, + ); +} + +#[test] +fn test_step_forward() { + assert_eq!(Step::forward_checked(55_u8, 200_usize), Some(255_u8)); + assert_eq!(Step::forward_checked(252_u8, 200_usize), None); + assert_eq!(Step::forward_checked(0_u8, 256_usize), None); + assert_eq!(Step::forward_checked(-110_i8, 200_usize), Some(90_i8)); + assert_eq!(Step::forward_checked(-110_i8, 248_usize), None); + assert_eq!(Step::forward_checked(-126_i8, 256_usize), None); + + assert_eq!(Step::forward_checked(35_u16, 100_usize), Some(135_u16)); + assert_eq!(Step::forward_checked(35_u16, 65500_usize), Some(u16::MAX)); + assert_eq!(Step::forward_checked(36_u16, 65500_usize), None); + assert_eq!(Step::forward_checked(-110_i16, 200_usize), Some(90_i16)); + assert_eq!(Step::forward_checked(-20_030_i16, 50_050_usize), Some(30_020_i16)); + assert_eq!(Step::forward_checked(-10_i16, 40_000_usize), None); + assert_eq!(Step::forward_checked(-10_i16, 70_000_usize), None); + + assert_eq!(Step::forward_checked(10_u128, 70_000_usize), Some(70_010_u128)); + assert_eq!(Step::forward_checked(10_i128, 70_030_usize), Some(70_040_i128)); + assert_eq!( + Step::forward_checked(0xffff_ffff_ffff_ffff__ffff_ffff_ffff_ff00_u128, 0xff_usize), + Some(u128::MAX), + ); + assert_eq!( + Step::forward_checked(0xffff_ffff_ffff_ffff__ffff_ffff_ffff_ff00_u128, 0x100_usize), + None + ); + assert_eq!( + Step::forward_checked(0x7fff_ffff_ffff_ffff__ffff_ffff_ffff_ff00_i128, 0xff_usize), + Some(i128::MAX), + ); + assert_eq!( + Step::forward_checked(0x7fff_ffff_ffff_ffff__ffff_ffff_ffff_ff00_i128, 0x100_usize), + None + ); +} + +#[test] +fn test_step_backward() { + assert_eq!(Step::backward_checked(255_u8, 200_usize), Some(55_u8)); + assert_eq!(Step::backward_checked(100_u8, 200_usize), None); + assert_eq!(Step::backward_checked(255_u8, 256_usize), None); + assert_eq!(Step::backward_checked(90_i8, 200_usize), Some(-110_i8)); + assert_eq!(Step::backward_checked(110_i8, 248_usize), None); + assert_eq!(Step::backward_checked(127_i8, 256_usize), None); + + assert_eq!(Step::backward_checked(135_u16, 100_usize), Some(35_u16)); + assert_eq!(Step::backward_checked(u16::MAX, 65500_usize), Some(35_u16)); + assert_eq!(Step::backward_checked(10_u16, 11_usize), None); + assert_eq!(Step::backward_checked(90_i16, 200_usize), Some(-110_i16)); + assert_eq!(Step::backward_checked(30_020_i16, 50_050_usize), Some(-20_030_i16)); + assert_eq!(Step::backward_checked(-10_i16, 40_000_usize), None); + assert_eq!(Step::backward_checked(-10_i16, 70_000_usize), None); + + assert_eq!(Step::backward_checked(70_010_u128, 70_000_usize), Some(10_u128)); + assert_eq!(Step::backward_checked(70_020_i128, 70_030_usize), Some(-10_i128)); + assert_eq!(Step::backward_checked(10_u128, 7_usize), Some(3_u128)); + assert_eq!(Step::backward_checked(10_u128, 11_usize), None); + assert_eq!( + Step::backward_checked(-0x7fff_ffff_ffff_ffff__ffff_ffff_ffff_ff00_i128, 0x100_usize), + Some(i128::MIN) + ); +} diff --git a/crux-mir/lib/core/tests/lazy.rs b/crux-mir/lib/core/tests/lazy.rs new file mode 100644 index 000000000..c7c3c479b --- /dev/null +++ b/crux-mir/lib/core/tests/lazy.rs @@ -0,0 +1,144 @@ +use core::{ + cell::{Cell, LazyCell, OnceCell}, + sync::atomic::{AtomicUsize, Ordering::SeqCst}, +}; + +#[test] +fn once_cell() { + let c = OnceCell::new(); + assert!(c.get().is_none()); + c.get_or_init(|| 92); + assert_eq!(c.get(), Some(&92)); + + c.get_or_init(|| panic!("Kabom!")); + assert_eq!(c.get(), Some(&92)); +} + +#[test] +fn once_cell_get_mut() { + let mut c = OnceCell::new(); + assert!(c.get_mut().is_none()); + c.set(90).unwrap(); + *c.get_mut().unwrap() += 2; + assert_eq!(c.get_mut(), Some(&mut 92)); +} + +#[test] +fn once_cell_drop() { + static DROP_CNT: AtomicUsize = AtomicUsize::new(0); + struct Dropper; + impl Drop for Dropper { + fn drop(&mut self) { + DROP_CNT.fetch_add(1, SeqCst); + } + } + + let x = OnceCell::new(); + x.get_or_init(|| Dropper); + assert_eq!(DROP_CNT.load(SeqCst), 0); + drop(x); + assert_eq!(DROP_CNT.load(SeqCst), 1); +} + +#[test] +fn unsync_once_cell_drop_empty() { + let x = OnceCell::<&'static str>::new(); + drop(x); +} + +#[test] +const fn once_cell_const() { + let _once_cell: OnceCell = OnceCell::new(); + let _once_cell: OnceCell = OnceCell::from(32); +} + +#[test] +fn clone() { + let s = OnceCell::new(); + let c = s.clone(); + assert!(c.get().is_none()); + + s.set("hello").unwrap(); + let c = s.clone(); + assert_eq!(c.get().map(|c| *c), Some("hello")); +} + +#[test] +fn from_impl() { + assert_eq!(OnceCell::from("value").get(), Some(&"value")); + assert_ne!(OnceCell::from("foo").get(), Some(&"bar")); +} + +#[test] +fn partialeq_impl() { + assert!(OnceCell::from("value") == OnceCell::from("value")); + assert!(OnceCell::from("foo") != OnceCell::from("bar")); + + assert!(OnceCell::<&'static str>::new() == OnceCell::new()); + assert!(OnceCell::<&'static str>::new() != OnceCell::from("value")); +} + +#[test] +fn into_inner() { + let cell: OnceCell<&'static str> = OnceCell::new(); + assert_eq!(cell.into_inner(), None); + let cell = OnceCell::new(); + cell.set("hello").unwrap(); + assert_eq!(cell.into_inner(), Some("hello")); +} + +#[test] +fn lazy_new() { + let called = Cell::new(0); + let x = LazyCell::new(|| { + called.set(called.get() + 1); + 92 + }); + + assert_eq!(called.get(), 0); + + let y = *x - 30; + assert_eq!(y, 62); + assert_eq!(called.get(), 1); + + let y = *x - 30; + assert_eq!(y, 62); + assert_eq!(called.get(), 1); +} + +// Check that we can infer `T` from closure's type. +#[test] +fn lazy_type_inference() { + let _ = LazyCell::new(|| ()); +} + +#[test] +fn aliasing_in_get() { + let x = OnceCell::new(); + x.set(42).unwrap(); + let at_x = x.get().unwrap(); // --- (shared) borrow of inner `Option` --+ + let _ = x.set(27); // <-- temporary (unique) borrow of inner `Option` | + println!("{at_x}"); // <------- up until here ---------------------------+ +} + +#[test] +#[should_panic(expected = "reentrant init")] +fn reentrant_init() { + let x: OnceCell> = OnceCell::new(); + let dangling_ref: Cell> = Cell::new(None); + x.get_or_init(|| { + let r = x.get_or_init(|| Box::new(92)); + dangling_ref.set(Some(r)); + Box::new(62) + }); + eprintln!("use after free: {:?}", dangling_ref.get().unwrap()); +} + +#[test] +fn dropck() { + let cell = OnceCell::new(); + { + let s = String::new(); + cell.set(&s).unwrap(); + } +} diff --git a/crux-mir/lib/core/tests/lib.rs b/crux-mir/lib/core/tests/lib.rs index 05f958cbe..42a26ae16 100644 --- a/crux-mir/lib/core/tests/lib.rs +++ b/crux-mir/lib/core/tests/lib.rs @@ -1,47 +1,115 @@ #![feature(alloc_layout_extra)] -#![feature(bool_to_option)] -#![feature(bound_cloned)] -#![feature(box_syntax)] +#![feature(array_chunks)] +#![feature(array_methods)] +#![feature(array_windows)] +#![feature(bigint_helper_methods)] #![feature(cell_update)] +#![feature(const_align_offset)] +#![feature(const_assume)] +#![feature(const_align_of_val_raw)] +#![feature(const_black_box)] +#![feature(const_bool_to_option)] +#![feature(const_caller_location)] +#![feature(const_cell_into_inner)] +#![feature(const_convert)] +#![feature(const_hash)] +#![feature(const_heap)] +#![feature(const_maybe_uninit_as_mut_ptr)] +#![feature(const_maybe_uninit_assume_init_read)] +#![feature(const_nonnull_new)] +#![feature(const_num_from_num)] +#![feature(const_pointer_byte_offsets)] +#![feature(const_pointer_is_aligned)] +#![feature(const_ptr_as_ref)] +#![feature(const_ptr_read)] +#![feature(const_ptr_write)] +#![feature(const_trait_impl)] +#![feature(const_likely)] +#![feature(const_location_fields)] +#![feature(core_intrinsics)] #![feature(core_private_bignum)] #![feature(core_private_diy_float)] -#![feature(debug_non_exhaustive)] #![feature(dec2flt)] +#![feature(div_duration)] +#![feature(duration_consts_float)] +#![feature(duration_constants)] #![feature(exact_size_is_empty)] -#![feature(fixed_size_array)] +#![feature(extern_types)] #![feature(flt2dec)] #![feature(fmt_internals)] +#![feature(float_minimum_maximum)] +#![feature(future_join)] +#![feature(generic_assert_internals)] +#![feature(array_try_from_fn)] +#![feature(hasher_prefixfree_extras)] #![feature(hashmap_internals)] #![feature(try_find)] +#![feature(inline_const)] #![feature(is_sorted)] +#![feature(layout_for_ptr)] #![feature(pattern)] -#![feature(range_is_empty)] -#![feature(raw)] -#![feature(saturating_neg)] #![feature(sort_internals)] -#![feature(slice_partition_at_index)] -#![feature(specialization)] +#![feature(slice_take)] +#![feature(slice_from_ptr_range)] +#![feature(split_as_slice)] +#![feature(maybe_uninit_uninit_array)] +#![feature(maybe_uninit_write_slice)] +#![feature(maybe_uninit_uninit_array_transpose)] +#![feature(min_specialization)] +#![feature(numfmt)] #![feature(step_trait)] #![feature(str_internals)] +#![feature(std_internals)] #![feature(test)] #![feature(trusted_len)] -#![feature(try_trait)] -#![feature(inner_deref)] +#![feature(try_blocks)] +#![feature(try_trait_v2)] #![feature(slice_internals)] #![feature(slice_partition_dedup)] -#![feature(int_error_matching)] -#![feature(array_value_iter)] +#![feature(iter_advance_by)] +#![feature(iter_array_chunks)] +#![feature(iter_collect_into)] #![feature(iter_partition_in_place)] +#![feature(iter_intersperse)] #![feature(iter_is_partitioned)] +#![feature(iter_next_chunk)] #![feature(iter_order_by)] -#![feature(cmp_min_max_by)] -#![feature(iter_map_while)] -#![feature(const_slice_from_raw_parts)] -#![feature(const_raw_ptr_deref)] +#![feature(iter_repeat_n)] +#![feature(iterator_try_collect)] +#![feature(iterator_try_reduce)] +#![feature(const_mut_refs)] +#![feature(const_pin)] +#![feature(const_waker)] #![feature(never_type)] #![feature(unwrap_infallible)] -#![feature(leading_trailing_ones)] -#![feature(const_forget)] +#![feature(pointer_byte_offsets)] +#![feature(pointer_is_aligned)] +#![feature(portable_simd)] +#![feature(ptr_metadata)] +#![feature(once_cell)] +#![feature(option_result_contains)] +#![feature(unsized_tuple_coercion)] +#![feature(const_option)] +#![feature(const_option_ext)] +#![feature(const_result)] +#![feature(integer_atomics)] +#![feature(int_roundings)] +#![feature(slice_group_by)] +#![feature(split_array)] +#![feature(strict_provenance)] +#![feature(strict_provenance_atomic_ptr)] +#![feature(trusted_random_access)] +#![feature(unsize)] +#![feature(const_array_from_ref)] +#![feature(const_slice_from_ref)] +#![feature(waker_getters)] +#![feature(slice_flatten)] +#![feature(provide_any)] +#![feature(utf8_chunks)] +#![feature(is_ascii_octdigit)] +#![feature(get_many_mut)] +#![deny(unsafe_op_in_unsafe_fn)] +#![deny(fuzzy_provenance_casts)] extern crate test; @@ -49,27 +117,53 @@ mod alloc; mod any; mod array; mod ascii; +mod asserting; mod atomic; mod bool; mod cell; mod char; mod clone; mod cmp; +mod const_ptr; +mod convert; mod fmt; +mod future; mod hash; mod intrinsics; mod iter; +mod lazy; +mod macros; mod manually_drop; mod mem; mod nonzero; mod num; mod ops; mod option; +mod panic; mod pattern; +mod pin; +mod pin_macro; mod ptr; mod result; +mod simd; mod slice; mod str; mod str_lossy; +mod task; mod time; mod tuple; +mod unicode; +mod waker; + +/// Copied from `std::test_helpers::test_rng`, see that function for rationale. +#[track_caller] +#[allow(dead_code)] // Not used in all configurations. +pub(crate) fn test_rng() -> rand_xorshift::XorShiftRng { + use core::hash::{BuildHasher, Hash, Hasher}; + let mut hasher = std::collections::hash_map::RandomState::new().build_hasher(); + core::panic::Location::caller().hash(&mut hasher); + let hc64 = hasher.finish(); + let seed_vec = hc64.to_le_bytes().into_iter().chain(0u8..8).collect::>(); + let seed: [u8; 16] = seed_vec.as_slice().try_into().unwrap(); + rand::SeedableRng::from_seed(seed) +} diff --git a/crux-mir/lib/core/tests/macros.rs b/crux-mir/lib/core/tests/macros.rs new file mode 100644 index 000000000..ff3632e35 --- /dev/null +++ b/crux-mir/lib/core/tests/macros.rs @@ -0,0 +1,20 @@ +#[test] +fn assert_eq_trailing_comma() { + assert_eq!(1, 1,); +} + +#[test] +fn assert_escape() { + assert!(r#"☃\backslash"#.contains("\\")); +} + +#[test] +fn assert_ne_trailing_comma() { + assert_ne!(1, 2,); +} + +#[rustfmt::skip] +#[test] +fn matches_leading_pipe() { + matches!(1, | 1 | 2 | 3); +} diff --git a/crux-mir/lib/core/tests/manually_drop.rs b/crux-mir/lib/core/tests/manually_drop.rs index 77a338daf..9eac27973 100644 --- a/crux-mir/lib/core/tests/manually_drop.rs +++ b/crux-mir/lib/core/tests/manually_drop.rs @@ -2,6 +2,7 @@ use core::mem::ManuallyDrop; #[test] fn smoke() { + #[derive(Clone)] struct TypeWithDrop; impl Drop for TypeWithDrop { fn drop(&mut self) { @@ -16,4 +17,11 @@ fn smoke() { let x: Box> = Box::new(ManuallyDrop::new([TypeWithDrop, TypeWithDrop])); drop(x); + + // test clone and clone_from implementations + let mut x = ManuallyDrop::new(TypeWithDrop); + let y = x.clone(); + x.clone_from(&y); + drop(x); + drop(y); } diff --git a/crux-mir/lib/core/tests/mem.rs b/crux-mir/lib/core/tests/mem.rs index 59588d977..f7740a114 100644 --- a/crux-mir/lib/core/tests/mem.rs +++ b/crux-mir/lib/core/tests/mem.rs @@ -1,4 +1,8 @@ use core::mem::*; +use core::ptr; + +#[cfg(panic = "unwind")] +use std::rc::Rc; #[test] fn size_of_basic() { @@ -72,6 +76,24 @@ fn align_of_val_basic() { assert_eq!(align_of_val(&1u32), 4); } +#[test] +fn align_of_val_raw_packed() { + #[repr(C, packed)] + struct B { + f: [u32], + } + let storage = [0u8; 4]; + let b: *const B = ptr::from_raw_parts(storage.as_ptr().cast(), 1); + assert_eq!(unsafe { align_of_val_raw(b) }, 1); + + const ALIGN_OF_VAL_RAW: usize = { + let storage = [0u8; 4]; + let b: *const B = ptr::from_raw_parts(storage.as_ptr().cast(), 1); + unsafe { align_of_val_raw(b) } + }; + assert_eq!(ALIGN_OF_VAL_RAW, 1); +} + #[test] fn test_swap() { let mut x = 31337; @@ -95,21 +117,46 @@ fn test_transmute_copy() { } #[test] -fn test_transmute() { - trait Foo { - fn dummy(&self) {} - } - impl Foo for isize {} +fn test_transmute_copy_shrink() { + assert_eq!(0_u8, unsafe { transmute_copy(&0_u64) }); +} - let a = box 100isize as Box; - unsafe { - let x: ::core::raw::TraitObject = transmute(a); - assert!(*(x.data as *const isize) == 100); - let _x: Box = transmute(x); +#[test] +fn test_transmute_copy_unaligned() { + #[repr(C)] + #[derive(Default)] + struct Unaligned { + a: u8, + b: [u8; 8], } - unsafe { - assert_eq!(transmute::<_, Vec>("L".to_string()), [76]); + let u = Unaligned::default(); + assert_eq!(0_u64, unsafe { transmute_copy(&u.b) }); +} + +#[test] +#[cfg(panic = "unwind")] +fn test_transmute_copy_grow_panics() { + use std::panic; + + let err = panic::catch_unwind(panic::AssertUnwindSafe(|| unsafe { + let _unused: u64 = transmute_copy(&1_u8); + })); + + match err { + Ok(_) => unreachable!(), + Err(payload) => { + payload + .downcast::<&'static str>() + .and_then(|s| { + if *s == "cannot transmute_copy if Dst is larger than Src" { + Ok(s) + } else { + Err(s) + } + }) + .unwrap_or_else(|p| panic::resume_unwind(p)); + } } } @@ -129,3 +176,191 @@ fn test_discriminant_send_sync() { is_send_sync::>(); is_send_sync::>(); } + +#[test] +fn assume_init_good() { + const TRUE: bool = unsafe { MaybeUninit::::new(true).assume_init() }; + + assert!(TRUE); +} + +#[test] +fn uninit_array_assume_init() { + let mut array = [MaybeUninit::::uninit(); 5]; + array[0].write(3); + array[1].write(1); + array[2].write(4); + array[3].write(1); + array[4].write(5); + + let array = unsafe { array.transpose().assume_init() }; + + assert_eq!(array, [3, 1, 4, 1, 5]); + + let [] = unsafe { [MaybeUninit::::uninit(); 0].transpose().assume_init() }; +} + +#[test] +fn uninit_write_slice() { + let mut dst = [MaybeUninit::new(255); 64]; + let src = [0; 64]; + + assert_eq!(MaybeUninit::write_slice(&mut dst, &src), &src); +} + +#[test] +#[should_panic(expected = "source slice length (32) does not match destination slice length (64)")] +fn uninit_write_slice_panic_lt() { + let mut dst = [MaybeUninit::uninit(); 64]; + let src = [0; 32]; + + MaybeUninit::write_slice(&mut dst, &src); +} + +#[test] +#[should_panic(expected = "source slice length (128) does not match destination slice length (64)")] +fn uninit_write_slice_panic_gt() { + let mut dst = [MaybeUninit::uninit(); 64]; + let src = [0; 128]; + + MaybeUninit::write_slice(&mut dst, &src); +} + +#[test] +fn uninit_clone_from_slice() { + let mut dst = [MaybeUninit::new(255); 64]; + let src = [0; 64]; + + assert_eq!(MaybeUninit::write_slice_cloned(&mut dst, &src), &src); +} + +#[test] +#[should_panic(expected = "destination and source slices have different lengths")] +fn uninit_write_slice_cloned_panic_lt() { + let mut dst = [MaybeUninit::uninit(); 64]; + let src = [0; 32]; + + MaybeUninit::write_slice_cloned(&mut dst, &src); +} + +#[test] +#[should_panic(expected = "destination and source slices have different lengths")] +fn uninit_write_slice_cloned_panic_gt() { + let mut dst = [MaybeUninit::uninit(); 64]; + let src = [0; 128]; + + MaybeUninit::write_slice_cloned(&mut dst, &src); +} + +#[test] +#[cfg(panic = "unwind")] +fn uninit_write_slice_cloned_mid_panic() { + use std::panic; + + enum IncrementOrPanic { + Increment(Rc<()>), + ExpectedPanic, + UnexpectedPanic, + } + + impl Clone for IncrementOrPanic { + fn clone(&self) -> Self { + match self { + Self::Increment(rc) => Self::Increment(rc.clone()), + Self::ExpectedPanic => panic!("expected panic on clone"), + Self::UnexpectedPanic => panic!("unexpected panic on clone"), + } + } + } + + let rc = Rc::new(()); + + let mut dst = [ + MaybeUninit::uninit(), + MaybeUninit::uninit(), + MaybeUninit::uninit(), + MaybeUninit::uninit(), + ]; + + let src = [ + IncrementOrPanic::Increment(rc.clone()), + IncrementOrPanic::Increment(rc.clone()), + IncrementOrPanic::ExpectedPanic, + IncrementOrPanic::UnexpectedPanic, + ]; + + let err = panic::catch_unwind(panic::AssertUnwindSafe(|| { + MaybeUninit::write_slice_cloned(&mut dst, &src); + })); + + drop(src); + + match err { + Ok(_) => unreachable!(), + Err(payload) => { + payload + .downcast::<&'static str>() + .and_then(|s| if *s == "expected panic on clone" { Ok(s) } else { Err(s) }) + .unwrap_or_else(|p| panic::resume_unwind(p)); + + assert_eq!(Rc::strong_count(&rc), 1) + } + } +} + +#[test] +fn uninit_write_slice_cloned_no_drop() { + #[derive(Clone)] + struct Bomb; + + impl Drop for Bomb { + fn drop(&mut self) { + panic!("dropped a bomb! kaboom") + } + } + + let mut dst = [MaybeUninit::uninit()]; + let src = [Bomb]; + + MaybeUninit::write_slice_cloned(&mut dst, &src); + + forget(src); +} + +#[test] +fn uninit_const_assume_init_read() { + const FOO: u32 = unsafe { MaybeUninit::new(42).assume_init_read() }; + assert_eq!(FOO, 42); +} + +#[test] +fn const_maybe_uninit() { + use std::ptr; + + #[derive(Debug, PartialEq)] + struct Foo { + x: u8, + y: u8, + } + + const FIELD_BY_FIELD: Foo = unsafe { + let mut val = MaybeUninit::uninit(); + init_y(&mut val); // order shouldn't matter + init_x(&mut val); + val.assume_init() + }; + + const fn init_x(foo: &mut MaybeUninit) { + unsafe { + *ptr::addr_of_mut!((*foo.as_mut_ptr()).x) = 1; + } + } + + const fn init_y(foo: &mut MaybeUninit) { + unsafe { + *ptr::addr_of_mut!((*foo.as_mut_ptr()).y) = 2; + } + } + + assert_eq!(FIELD_BY_FIELD, Foo { x: 1, y: 2 }); +} diff --git a/crux-mir/lib/core/tests/nonzero.rs b/crux-mir/lib/core/tests/nonzero.rs index 6c5d19845..a0ca919a8 100644 --- a/crux-mir/lib/core/tests/nonzero.rs +++ b/crux-mir/lib/core/tests/nonzero.rs @@ -1,4 +1,8 @@ -use core::num::{IntErrorKind, NonZeroI32, NonZeroI8, NonZeroU32, NonZeroU8}; +use core::convert::TryFrom; +use core::num::{ + IntErrorKind, NonZeroI128, NonZeroI16, NonZeroI32, NonZeroI64, NonZeroI8, NonZeroIsize, + NonZeroU128, NonZeroU16, NonZeroU32, NonZeroU64, NonZeroU8, NonZeroUsize, +}; use core::option::Option::{self, None, Some}; use std::mem::size_of; @@ -81,7 +85,7 @@ fn test_match_option_string() { let five = "Five".to_string(); match Some(five) { Some(s) => assert_eq!(s, "Five"), - None => panic!("unexpected None while matching on Some(String { ... })"), + None => panic!("{}", "unexpected None while matching on Some(String { ... })"), } } @@ -134,10 +138,199 @@ fn test_from_str() { ); assert_eq!( "-129".parse::().err().map(|e| e.kind().clone()), - Some(IntErrorKind::Underflow) + Some(IntErrorKind::NegOverflow) ); assert_eq!( "257".parse::().err().map(|e| e.kind().clone()), - Some(IntErrorKind::Overflow) + Some(IntErrorKind::PosOverflow) ); } + +#[test] +fn test_nonzero_bitor() { + let nz_alt = NonZeroU8::new(0b1010_1010).unwrap(); + let nz_low = NonZeroU8::new(0b0000_1111).unwrap(); + + let both_nz: NonZeroU8 = nz_alt | nz_low; + assert_eq!(both_nz.get(), 0b1010_1111); + + let rhs_int: NonZeroU8 = nz_low | 0b1100_0000u8; + assert_eq!(rhs_int.get(), 0b1100_1111); + + let rhs_zero: NonZeroU8 = nz_alt | 0u8; + assert_eq!(rhs_zero.get(), 0b1010_1010); + + let lhs_int: NonZeroU8 = 0b0110_0110u8 | nz_alt; + assert_eq!(lhs_int.get(), 0b1110_1110); + + let lhs_zero: NonZeroU8 = 0u8 | nz_low; + assert_eq!(lhs_zero.get(), 0b0000_1111); +} + +#[test] +fn test_nonzero_bitor_assign() { + let mut target = NonZeroU8::new(0b1010_1010).unwrap(); + + target |= NonZeroU8::new(0b0000_1111).unwrap(); + assert_eq!(target.get(), 0b1010_1111); + + target |= 0b0001_0000; + assert_eq!(target.get(), 0b1011_1111); + + target |= 0; + assert_eq!(target.get(), 0b1011_1111); +} + +#[test] +fn test_nonzero_from_int_on_success() { + assert_eq!(NonZeroU8::try_from(5), Ok(NonZeroU8::new(5).unwrap())); + assert_eq!(NonZeroU32::try_from(5), Ok(NonZeroU32::new(5).unwrap())); + + assert_eq!(NonZeroI8::try_from(-5), Ok(NonZeroI8::new(-5).unwrap())); + assert_eq!(NonZeroI32::try_from(-5), Ok(NonZeroI32::new(-5).unwrap())); +} + +#[test] +fn test_nonzero_from_int_on_err() { + assert!(NonZeroU8::try_from(0).is_err()); + assert!(NonZeroU32::try_from(0).is_err()); + + assert!(NonZeroI8::try_from(0).is_err()); + assert!(NonZeroI32::try_from(0).is_err()); +} + +#[test] +fn nonzero_const() { + // test that the methods of `NonZeroX>` are usable in a const context + // Note: only tests NonZero8 + + const NONZERO_U8: NonZeroU8 = unsafe { NonZeroU8::new_unchecked(5) }; + + const GET: u8 = NONZERO_U8.get(); + assert_eq!(GET, 5); + + const ZERO: Option = NonZeroU8::new(0); + assert!(ZERO.is_none()); + + const ONE: Option = NonZeroU8::new(1); + assert!(ONE.is_some()); + + const FROM_NONZERO_U8: u8 = u8::from(NONZERO_U8); + assert_eq!(FROM_NONZERO_U8, 5); + + const NONZERO_CONVERT: NonZeroU32 = NonZeroU32::from(NONZERO_U8); + assert_eq!(NONZERO_CONVERT.get(), 5); +} + +#[test] +fn nonzero_leading_zeros() { + assert_eq!(NonZeroU8::new(1).unwrap().leading_zeros(), 7); + assert_eq!(NonZeroI8::new(1).unwrap().leading_zeros(), 7); + assert_eq!(NonZeroU16::new(1).unwrap().leading_zeros(), 15); + assert_eq!(NonZeroI16::new(1).unwrap().leading_zeros(), 15); + assert_eq!(NonZeroU32::new(1).unwrap().leading_zeros(), 31); + assert_eq!(NonZeroI32::new(1).unwrap().leading_zeros(), 31); + assert_eq!(NonZeroU64::new(1).unwrap().leading_zeros(), 63); + assert_eq!(NonZeroI64::new(1).unwrap().leading_zeros(), 63); + assert_eq!(NonZeroU128::new(1).unwrap().leading_zeros(), 127); + assert_eq!(NonZeroI128::new(1).unwrap().leading_zeros(), 127); + assert_eq!(NonZeroUsize::new(1).unwrap().leading_zeros(), usize::BITS - 1); + assert_eq!(NonZeroIsize::new(1).unwrap().leading_zeros(), usize::BITS - 1); + + assert_eq!(NonZeroU8::new(u8::MAX >> 2).unwrap().leading_zeros(), 2); + assert_eq!(NonZeroI8::new((u8::MAX >> 2) as i8).unwrap().leading_zeros(), 2); + assert_eq!(NonZeroU16::new(u16::MAX >> 2).unwrap().leading_zeros(), 2); + assert_eq!(NonZeroI16::new((u16::MAX >> 2) as i16).unwrap().leading_zeros(), 2); + assert_eq!(NonZeroU32::new(u32::MAX >> 2).unwrap().leading_zeros(), 2); + assert_eq!(NonZeroI32::new((u32::MAX >> 2) as i32).unwrap().leading_zeros(), 2); + assert_eq!(NonZeroU64::new(u64::MAX >> 2).unwrap().leading_zeros(), 2); + assert_eq!(NonZeroI64::new((u64::MAX >> 2) as i64).unwrap().leading_zeros(), 2); + assert_eq!(NonZeroU128::new(u128::MAX >> 2).unwrap().leading_zeros(), 2); + assert_eq!(NonZeroI128::new((u128::MAX >> 2) as i128).unwrap().leading_zeros(), 2); + assert_eq!(NonZeroUsize::new(usize::MAX >> 2).unwrap().leading_zeros(), 2); + assert_eq!(NonZeroIsize::new((usize::MAX >> 2) as isize).unwrap().leading_zeros(), 2); + + assert_eq!(NonZeroU8::new(u8::MAX).unwrap().leading_zeros(), 0); + assert_eq!(NonZeroI8::new(-1i8).unwrap().leading_zeros(), 0); + assert_eq!(NonZeroU16::new(u16::MAX).unwrap().leading_zeros(), 0); + assert_eq!(NonZeroI16::new(-1i16).unwrap().leading_zeros(), 0); + assert_eq!(NonZeroU32::new(u32::MAX).unwrap().leading_zeros(), 0); + assert_eq!(NonZeroI32::new(-1i32).unwrap().leading_zeros(), 0); + assert_eq!(NonZeroU64::new(u64::MAX).unwrap().leading_zeros(), 0); + assert_eq!(NonZeroI64::new(-1i64).unwrap().leading_zeros(), 0); + assert_eq!(NonZeroU128::new(u128::MAX).unwrap().leading_zeros(), 0); + assert_eq!(NonZeroI128::new(-1i128).unwrap().leading_zeros(), 0); + assert_eq!(NonZeroUsize::new(usize::MAX).unwrap().leading_zeros(), 0); + assert_eq!(NonZeroIsize::new(-1isize).unwrap().leading_zeros(), 0); + + const LEADING_ZEROS: u32 = NonZeroU16::new(1).unwrap().leading_zeros(); + assert_eq!(LEADING_ZEROS, 15); +} + +#[test] +fn nonzero_trailing_zeros() { + assert_eq!(NonZeroU8::new(1).unwrap().trailing_zeros(), 0); + assert_eq!(NonZeroI8::new(1).unwrap().trailing_zeros(), 0); + assert_eq!(NonZeroU16::new(1).unwrap().trailing_zeros(), 0); + assert_eq!(NonZeroI16::new(1).unwrap().trailing_zeros(), 0); + assert_eq!(NonZeroU32::new(1).unwrap().trailing_zeros(), 0); + assert_eq!(NonZeroI32::new(1).unwrap().trailing_zeros(), 0); + assert_eq!(NonZeroU64::new(1).unwrap().trailing_zeros(), 0); + assert_eq!(NonZeroI64::new(1).unwrap().trailing_zeros(), 0); + assert_eq!(NonZeroU128::new(1).unwrap().trailing_zeros(), 0); + assert_eq!(NonZeroI128::new(1).unwrap().trailing_zeros(), 0); + assert_eq!(NonZeroUsize::new(1).unwrap().trailing_zeros(), 0); + assert_eq!(NonZeroIsize::new(1).unwrap().trailing_zeros(), 0); + + assert_eq!(NonZeroU8::new(1 << 2).unwrap().trailing_zeros(), 2); + assert_eq!(NonZeroI8::new(1 << 2).unwrap().trailing_zeros(), 2); + assert_eq!(NonZeroU16::new(1 << 2).unwrap().trailing_zeros(), 2); + assert_eq!(NonZeroI16::new(1 << 2).unwrap().trailing_zeros(), 2); + assert_eq!(NonZeroU32::new(1 << 2).unwrap().trailing_zeros(), 2); + assert_eq!(NonZeroI32::new(1 << 2).unwrap().trailing_zeros(), 2); + assert_eq!(NonZeroU64::new(1 << 2).unwrap().trailing_zeros(), 2); + assert_eq!(NonZeroI64::new(1 << 2).unwrap().trailing_zeros(), 2); + assert_eq!(NonZeroU128::new(1 << 2).unwrap().trailing_zeros(), 2); + assert_eq!(NonZeroI128::new(1 << 2).unwrap().trailing_zeros(), 2); + assert_eq!(NonZeroUsize::new(1 << 2).unwrap().trailing_zeros(), 2); + assert_eq!(NonZeroIsize::new(1 << 2).unwrap().trailing_zeros(), 2); + + assert_eq!(NonZeroU8::new(1 << 7).unwrap().trailing_zeros(), 7); + assert_eq!(NonZeroI8::new(1 << 7).unwrap().trailing_zeros(), 7); + assert_eq!(NonZeroU16::new(1 << 15).unwrap().trailing_zeros(), 15); + assert_eq!(NonZeroI16::new(1 << 15).unwrap().trailing_zeros(), 15); + assert_eq!(NonZeroU32::new(1 << 31).unwrap().trailing_zeros(), 31); + assert_eq!(NonZeroI32::new(1 << 31).unwrap().trailing_zeros(), 31); + assert_eq!(NonZeroU64::new(1 << 63).unwrap().trailing_zeros(), 63); + assert_eq!(NonZeroI64::new(1 << 63).unwrap().trailing_zeros(), 63); + assert_eq!(NonZeroU128::new(1 << 127).unwrap().trailing_zeros(), 127); + assert_eq!(NonZeroI128::new(1 << 127).unwrap().trailing_zeros(), 127); + + assert_eq!( + NonZeroUsize::new(1 << (usize::BITS - 1)).unwrap().trailing_zeros(), + usize::BITS - 1 + ); + assert_eq!( + NonZeroIsize::new(1 << (usize::BITS - 1)).unwrap().trailing_zeros(), + usize::BITS - 1 + ); + + const TRAILING_ZEROS: u32 = NonZeroU16::new(1 << 2).unwrap().trailing_zeros(); + assert_eq!(TRAILING_ZEROS, 2); +} + +#[test] +fn test_nonzero_uint_div() { + let nz = NonZeroU32::new(1).unwrap(); + + let x: u32 = 42u32 / nz; + assert_eq!(x, 42u32); +} + +#[test] +fn test_nonzero_uint_rem() { + let nz = NonZeroU32::new(10).unwrap(); + + let x: u32 = 42u32 % nz; + assert_eq!(x, 2u32); +} diff --git a/crux-mir/lib/core/tests/num/bignum.rs b/crux-mir/lib/core/tests/num/bignum.rs index 1457064cc..416e7cea7 100644 --- a/crux-mir/lib/core/tests/num/bignum.rs +++ b/crux-mir/lib/core/tests/num/bignum.rs @@ -1,4 +1,5 @@ use core::num::bignum::tests::Big8x3 as Big; +use core::num::bignum::Big32x40; #[test] #[should_panic] @@ -215,6 +216,16 @@ fn test_get_bit_out_of_range() { #[test] fn test_bit_length() { + for i in 0..8 * 3 { + // 010000...000 + assert_eq!(Big::from_small(1).mul_pow2(i).bit_length(), i + 1); + } + for i in 1..8 * 3 - 1 { + // 010000...001 + assert_eq!(Big::from_small(1).mul_pow2(i).add(&Big::from_small(1)).bit_length(), i + 1); + // 110000...000 + assert_eq!(Big::from_small(3).mul_pow2(i).bit_length(), i + 2); + } assert_eq!(Big::from_small(0).bit_length(), 0); assert_eq!(Big::from_small(1).bit_length(), 1); assert_eq!(Big::from_small(5).bit_length(), 3); @@ -223,6 +234,30 @@ fn test_bit_length() { assert_eq!(Big::from_u64(0xffffff).bit_length(), 24); } +#[test] +fn test_bit_length_32x40() { + for i in 0..32 * 40 { + // 010000...000 + assert_eq!(Big32x40::from_small(1).mul_pow2(i).bit_length(), i + 1); + } + for i in 1..32 * 40 - 1 { + // 010000...001 + assert_eq!( + Big32x40::from_small(1).mul_pow2(i).add(&Big32x40::from_small(1)).bit_length(), + i + 1 + ); + // 110000...000 + assert_eq!(Big32x40::from_small(3).mul_pow2(i).bit_length(), i + 2); + } + assert_eq!(Big32x40::from_small(0).bit_length(), 0); + assert_eq!(Big32x40::from_small(1).bit_length(), 1); + assert_eq!(Big32x40::from_small(5).bit_length(), 3); + assert_eq!(Big32x40::from_small(0x18).bit_length(), 5); + assert_eq!(Big32x40::from_u64(0x4073).bit_length(), 15); + assert_eq!(Big32x40::from_u64(0xffffff).bit_length(), 24); + assert_eq!(Big32x40::from_u64(0xffffffffffffffff).bit_length(), 64); +} + #[test] fn test_ord() { assert!(Big::from_u64(0) < Big::from_u64(0xffffff)); diff --git a/crux-mir/lib/core/tests/num/const_from.rs b/crux-mir/lib/core/tests/num/const_from.rs new file mode 100644 index 000000000..aca18ef39 --- /dev/null +++ b/crux-mir/lib/core/tests/num/const_from.rs @@ -0,0 +1,25 @@ +#[test] +fn from() { + use core::convert::TryFrom; + use core::num::TryFromIntError; + + // From + const FROM: i64 = i64::from(1i32); + assert_eq!(FROM, 1i64); + + // From int to float + const FROM_F64: f64 = f64::from(42u8); + assert_eq!(FROM_F64, 42f64); + + // Upper bounded + const U8_FROM_U16: Result = u8::try_from(1u16); + assert_eq!(U8_FROM_U16, Ok(1u8)); + + // Both bounded + const I8_FROM_I16: Result = i8::try_from(1i16); + assert_eq!(I8_FROM_I16, Ok(1i8)); + + // Lower bounded + const I16_FROM_U16: Result = i16::try_from(1u16); + assert_eq!(I16_FROM_U16, Ok(1i16)); +} diff --git a/crux-mir/lib/core/tests/num/dec2flt/float.rs b/crux-mir/lib/core/tests/num/dec2flt/float.rs new file mode 100644 index 000000000..7a9587a18 --- /dev/null +++ b/crux-mir/lib/core/tests/num/dec2flt/float.rs @@ -0,0 +1,33 @@ +use core::num::dec2flt::float::RawFloat; + +#[test] +fn test_f32_integer_decode() { + assert_eq!(3.14159265359f32.integer_decode(), (13176795, -22, 1)); + assert_eq!((-8573.5918555f32).integer_decode(), (8779358, -10, -1)); + assert_eq!(2f32.powf(100.0).integer_decode(), (8388608, 77, 1)); + assert_eq!(0f32.integer_decode(), (0, -150, 1)); + assert_eq!((-0f32).integer_decode(), (0, -150, -1)); + assert_eq!(f32::INFINITY.integer_decode(), (8388608, 105, 1)); + assert_eq!(f32::NEG_INFINITY.integer_decode(), (8388608, 105, -1)); + + // Ignore the "sign" (quiet / signalling flag) of NAN. + // It can vary between runtime operations and LLVM folding. + let (nan_m, nan_e, _nan_s) = f32::NAN.integer_decode(); + assert_eq!((nan_m, nan_e), (12582912, 105)); +} + +#[test] +fn test_f64_integer_decode() { + assert_eq!(3.14159265359f64.integer_decode(), (7074237752028906, -51, 1)); + assert_eq!((-8573.5918555f64).integer_decode(), (4713381968463931, -39, -1)); + assert_eq!(2f64.powf(100.0).integer_decode(), (4503599627370496, 48, 1)); + assert_eq!(0f64.integer_decode(), (0, -1075, 1)); + assert_eq!((-0f64).integer_decode(), (0, -1075, -1)); + assert_eq!(f64::INFINITY.integer_decode(), (4503599627370496, 972, 1)); + assert_eq!(f64::NEG_INFINITY.integer_decode(), (4503599627370496, 972, -1)); + + // Ignore the "sign" (quiet / signalling flag) of NAN. + // It can vary between runtime operations and LLVM folding. + let (nan_m, nan_e, _nan_s) = f64::NAN.integer_decode(); + assert_eq!((nan_m, nan_e), (6755399441055744, 972)); +} diff --git a/crux-mir/lib/core/tests/num/dec2flt/lemire.rs b/crux-mir/lib/core/tests/num/dec2flt/lemire.rs new file mode 100644 index 000000000..f71bbb7c7 --- /dev/null +++ b/crux-mir/lib/core/tests/num/dec2flt/lemire.rs @@ -0,0 +1,53 @@ +use core::num::dec2flt::lemire::compute_float; + +fn compute_float32(q: i64, w: u64) -> (i32, u64) { + let fp = compute_float::(q, w); + (fp.e, fp.f) +} + +fn compute_float64(q: i64, w: u64) -> (i32, u64) { + let fp = compute_float::(q, w); + (fp.e, fp.f) +} + +#[test] +fn compute_float_f32_rounding() { + // These test near-halfway cases for single-precision floats. + assert_eq!(compute_float32(0, 16777216), (151, 0)); + assert_eq!(compute_float32(0, 16777217), (151, 0)); + assert_eq!(compute_float32(0, 16777218), (151, 1)); + assert_eq!(compute_float32(0, 16777219), (151, 2)); + assert_eq!(compute_float32(0, 16777220), (151, 2)); + + // These are examples of the above tests, with + // digits from the exponent shifted to the mantissa. + assert_eq!(compute_float32(-10, 167772160000000000), (151, 0)); + assert_eq!(compute_float32(-10, 167772170000000000), (151, 0)); + assert_eq!(compute_float32(-10, 167772180000000000), (151, 1)); + // Let's check the lines to see if anything is different in table... + assert_eq!(compute_float32(-10, 167772190000000000), (151, 2)); + assert_eq!(compute_float32(-10, 167772200000000000), (151, 2)); +} + +#[test] +fn compute_float_f64_rounding() { + // These test near-halfway cases for double-precision floats. + assert_eq!(compute_float64(0, 9007199254740992), (1076, 0)); + assert_eq!(compute_float64(0, 9007199254740993), (1076, 0)); + assert_eq!(compute_float64(0, 9007199254740994), (1076, 1)); + assert_eq!(compute_float64(0, 9007199254740995), (1076, 2)); + assert_eq!(compute_float64(0, 9007199254740996), (1076, 2)); + assert_eq!(compute_float64(0, 18014398509481984), (1077, 0)); + assert_eq!(compute_float64(0, 18014398509481986), (1077, 0)); + assert_eq!(compute_float64(0, 18014398509481988), (1077, 1)); + assert_eq!(compute_float64(0, 18014398509481990), (1077, 2)); + assert_eq!(compute_float64(0, 18014398509481992), (1077, 2)); + + // These are examples of the above tests, with + // digits from the exponent shifted to the mantissa. + assert_eq!(compute_float64(-3, 9007199254740992000), (1076, 0)); + assert_eq!(compute_float64(-3, 9007199254740993000), (1076, 0)); + assert_eq!(compute_float64(-3, 9007199254740994000), (1076, 1)); + assert_eq!(compute_float64(-3, 9007199254740995000), (1076, 2)); + assert_eq!(compute_float64(-3, 9007199254740996000), (1076, 2)); +} diff --git a/crux-mir/lib/core/tests/num/dec2flt/mod.rs b/crux-mir/lib/core/tests/num/dec2flt/mod.rs index a1fa5556a..a2b9bb551 100644 --- a/crux-mir/lib/core/tests/num/dec2flt/mod.rs +++ b/crux-mir/lib/core/tests/num/dec2flt/mod.rs @@ -1,9 +1,8 @@ #![allow(overflowing_literals)] -use std::{f32, f64, i64}; - +mod float; +mod lemire; mod parse; -mod rawfp; // Take a float literal, turn it into a string in various ways (that are all trusted // to be correct) and see if those strings are parsed back to the value of the literal. @@ -16,14 +15,13 @@ macro_rules! test_literal { for input in inputs { assert_eq!(input.parse(), Ok(x64)); assert_eq!(input.parse(), Ok(x32)); - let neg_input = &format!("-{}", input); + let neg_input = format!("-{input}"); assert_eq!(neg_input.parse(), Ok(-x64)); assert_eq!(neg_input.parse(), Ok(-x32)); } }}; } -#[cfg_attr(all(target_arch = "wasm32", target_os = "emscripten"), ignore)] // issue 42630 #[test] fn ordinary() { test_literal!(1.0); @@ -31,16 +29,9 @@ fn ordinary() { test_literal!(0.1); test_literal!(12345.); test_literal!(0.9999999); - - if cfg!(miri) { - // Miri is too slow - return; - } - test_literal!(2.2250738585072014e-308); } -#[cfg_attr(all(target_arch = "wasm32", target_os = "emscripten"), ignore)] // issue 42630 #[test] fn special_code_paths() { test_literal!(36893488147419103229.0); // 2^65 - 3, triggers half-to-even with even significand @@ -58,7 +49,6 @@ fn large() { } #[test] -#[cfg_attr(miri, ignore)] // Miri is too slow fn subnormals() { test_literal!(5e-324); test_literal!(91e-324); @@ -70,7 +60,6 @@ fn subnormals() { } #[test] -#[cfg_attr(miri, ignore)] // Miri is too slow fn infinity() { test_literal!(1e400); test_literal!(1e309); @@ -82,12 +71,6 @@ fn infinity() { fn zero() { test_literal!(0.0); test_literal!(1e-325); - - if cfg!(miri) { - // Miri is too slow - return; - } - test_literal!(1e-326); test_literal!(1e-500); } @@ -140,9 +123,9 @@ fn inf() { #[test] fn massive_exponent() { let max = i64::MAX; - assert_eq!(format!("1e{}000", max).parse(), Ok(f64::INFINITY)); - assert_eq!(format!("1e-{}000", max).parse(), Ok(0.0)); - assert_eq!(format!("1e{}000", max).parse(), Ok(f64::INFINITY)); + assert_eq!(format!("1e{max}000").parse(), Ok(f64::INFINITY)); + assert_eq!(format!("1e-{max}000").parse(), Ok(0.0)); + assert_eq!(format!("1e{max}000").parse(), Ok(f64::INFINITY)); } #[test] diff --git a/crux-mir/lib/core/tests/num/dec2flt/parse.rs b/crux-mir/lib/core/tests/num/dec2flt/parse.rs index bb7e51d30..edc77377d 100644 --- a/crux-mir/lib/core/tests/num/dec2flt/parse.rs +++ b/crux-mir/lib/core/tests/num/dec2flt/parse.rs @@ -1,17 +1,23 @@ -use core::num::dec2flt::parse::ParseResult::{Invalid, Valid}; -use core::num::dec2flt::parse::{parse_decimal, Decimal}; +use core::num::dec2flt::number::Number; +use core::num::dec2flt::parse::parse_number; +use core::num::dec2flt::{dec2flt, pfe_invalid}; + +fn new_number(e: i64, m: u64) -> Number { + Number { exponent: e, mantissa: m, negative: false, many_digits: false } +} #[test] fn missing_pieces() { let permutations = &[".e", "1e", "e4", "e", ".12e", "321.e", "32.12e+", "12.32e-"]; for &s in permutations { - assert_eq!(parse_decimal(s), Invalid); + assert_eq!(dec2flt::(s), Err(pfe_invalid())); } } #[test] fn invalid_chars() { let invalid = "r,?(&input) == error, "did not reject invalid {:?}", input); } } } } +fn parse_positive(s: &[u8]) -> Option { + parse_number(s, false) +} + #[test] fn valid() { - assert_eq!(parse_decimal("123.456e789"), Valid(Decimal::new(b"123", b"456", 789))); - assert_eq!(parse_decimal("123.456e+789"), Valid(Decimal::new(b"123", b"456", 789))); - assert_eq!(parse_decimal("123.456e-789"), Valid(Decimal::new(b"123", b"456", -789))); - assert_eq!(parse_decimal(".050"), Valid(Decimal::new(b"", b"050", 0))); - assert_eq!(parse_decimal("999"), Valid(Decimal::new(b"999", b"", 0))); - assert_eq!(parse_decimal("1.e300"), Valid(Decimal::new(b"1", b"", 300))); - assert_eq!(parse_decimal(".1e300"), Valid(Decimal::new(b"", b"1", 300))); - assert_eq!(parse_decimal("101e-33"), Valid(Decimal::new(b"101", b"", -33))); + assert_eq!(parse_positive(b"123.456e789"), Some(new_number(786, 123456))); + assert_eq!(parse_positive(b"123.456e+789"), Some(new_number(786, 123456))); + assert_eq!(parse_positive(b"123.456e-789"), Some(new_number(-792, 123456))); + assert_eq!(parse_positive(b".050"), Some(new_number(-3, 50))); + assert_eq!(parse_positive(b"999"), Some(new_number(0, 999))); + assert_eq!(parse_positive(b"1.e300"), Some(new_number(300, 1))); + assert_eq!(parse_positive(b".1e300"), Some(new_number(299, 1))); + assert_eq!(parse_positive(b"101e-33"), Some(new_number(-33, 101))); let zeros = "0".repeat(25); - let s = format!("1.5e{}", zeros); - assert_eq!(parse_decimal(&s), Valid(Decimal::new(b"1", b"5", 0))); + let s = format!("1.5e{zeros}"); + assert_eq!(parse_positive(s.as_bytes()), Some(new_number(-1, 15))); +} + +macro_rules! assert_float_result_bits_eq { + ($bits:literal, $ty:ty, $str:literal) => {{ + let p = dec2flt::<$ty>($str); + assert_eq!(p.map(|x| x.to_bits()), Ok($bits)); + }}; +} + +#[test] +fn issue31109() { + // Regression test for #31109. + // Ensure the test produces a valid float with the expected bit pattern. + assert_float_result_bits_eq!( + 0x3fd5555555555555, + f64, + "0.3333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333" + ); +} + +#[test] +fn issue31407() { + // Regression test for #31407. + // Ensure the test produces a valid float with the expected bit pattern. + assert_float_result_bits_eq!( + 0x1752a64e34ba0d3, + f64, + "1234567890123456789012345678901234567890e-340" + ); + assert_float_result_bits_eq!( + 0xfffffffffffff, + f64, + "2.225073858507201136057409796709131975934819546351645648023426109724822222021076945516529523908135087914149158913039621106870086438694594645527657207407820621743379988141063267329253552286881372149012981122451451889849057222307285255133155755015914397476397983411801999323962548289017107081850690630666655994938275772572015763062690663332647565300009245888316433037779791869612049497390377829704905051080609940730262937128958950003583799967207254304360284078895771796150945516748243471030702609144621572289880258182545180325707018860872113128079512233426288368622321503775666622503982534335974568884423900265498198385487948292206894721689831099698365846814022854243330660339850886445804001034933970427567186443383770486037861622771738545623065874679014086723327636718749999999999999999999999999999999999999e-308" + ); + assert_float_result_bits_eq!( + 0x10000000000000, + f64, + "2.22507385850720113605740979670913197593481954635164564802342610972482222202107694551652952390813508791414915891303962110687008643869459464552765720740782062174337998814106326732925355228688137214901298112245145188984905722230728525513315575501591439747639798341180199932396254828901710708185069063066665599493827577257201576306269066333264756530000924588831643303777979186961204949739037782970490505108060994073026293712895895000358379996720725430436028407889577179615094551674824347103070260914462157228988025818254518032570701886087211312807951223342628836862232150377566662250398253433597456888442390026549819838548794829220689472168983109969836584681402285424333066033985088644580400103493397042756718644338377048603786162277173854562306587467901408672332763671875e-308" + ); + assert_float_result_bits_eq!( + 0x10000000000000, + f64, + "0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000222507385850720138309023271733240406421921598046233183055332741688720443481391819585428315901251102056406733973103581100515243416155346010885601238537771882113077799353200233047961014744258363607192156504694250373420837525080665061665815894872049117996859163964850063590877011830487479978088775374994945158045160505091539985658247081864511353793580499211598108576605199243335211435239014879569960959128889160299264151106346631339366347758651302937176204732563178148566435087212282863764204484681140761391147706280168985324411002416144742161856716615054015428508471675290190316132277889672970737312333408698898317506783884692609277397797285865965494109136909540613646756870239867831529068098461721092462539672851562500000000000000001" + ); + assert_float_result_bits_eq!( + 0x7fefffffffffffff, + f64, + "179769313486231580793728971405303415079934132710037826936173778980444968292764750946649017977587207096330286416692887910946555547851940402630657488671505820681908902000708383676273854845817711531764475730270069855571366959622842914819860834936475292719074168444365510704342711559699508093042880177904174497791.9999999999999999999999999999999999999999999999999999999999999999999999" + ); + assert_float_result_bits_eq!(0x0, f64, "2.47032822920623272e-324"); + assert_float_result_bits_eq!( + 0x8000000, + f64, + "6.631236871469758276785396630275967243399099947355303144249971758736286630139265439618068200788048744105960420552601852889715006376325666595539603330361800519107591783233358492337208057849499360899425128640718856616503093444922854759159988160304439909868291973931426625698663157749836252274523485312442358651207051292453083278116143932569727918709786004497872322193856150225415211997283078496319412124640111777216148110752815101775295719811974338451936095907419622417538473679495148632480391435931767981122396703443803335529756003353209830071832230689201383015598792184172909927924176339315507402234836120730914783168400715462440053817592702766213559042115986763819482654128770595766806872783349146967171293949598850675682115696218943412532098591327667236328125E-316" + ); + assert_float_result_bits_eq!( + 0x10000, + f64, + "3.237883913302901289588352412501532174863037669423108059901297049552301970670676565786835742587799557860615776559838283435514391084153169252689190564396459577394618038928365305143463955100356696665629202017331344031730044369360205258345803431471660032699580731300954848363975548690010751530018881758184174569652173110473696022749934638425380623369774736560008997404060967498028389191878963968575439222206416981462690113342524002724385941651051293552601421155333430225237291523843322331326138431477823591142408800030775170625915670728657003151953664260769822494937951845801530895238439819708403389937873241463484205608000027270531106827387907791444918534771598750162812548862768493201518991668028251730299953143924168545708663913273994694463908672332763671875E-319" + ); + assert_float_result_bits_eq!( + 0x800000000100, + f64, + "6.953355807847677105972805215521891690222119817145950754416205607980030131549636688806115726399441880065386399864028691275539539414652831584795668560082999889551357784961446896042113198284213107935110217162654939802416034676213829409720583759540476786936413816541621287843248433202369209916612249676005573022703244799714622116542188837770376022371172079559125853382801396219552418839469770514904192657627060319372847562301074140442660237844114174497210955449896389180395827191602886654488182452409583981389442783377001505462015745017848754574668342161759496661766020028752888783387074850773192997102997936619876226688096314989645766000479009083731736585750335262099860150896718774401964796827166283225641992040747894382698751809812609536720628966577351093292236328125E-310" + ); + assert_float_result_bits_eq!( + 0x10800, + f64, + "3.339068557571188581835713701280943911923401916998521771655656997328440314559615318168849149074662609099998113009465566426808170378434065722991659642619467706034884424989741080790766778456332168200464651593995817371782125010668346652995912233993254584461125868481633343674905074271064409763090708017856584019776878812425312008812326260363035474811532236853359905334625575404216060622858633280744301892470300555678734689978476870369853549413277156622170245846166991655321535529623870646888786637528995592800436177901746286272273374471701452991433047257863864601424252024791567368195056077320885329384322332391564645264143400798619665040608077549162173963649264049738362290606875883456826586710961041737908872035803481241600376705491726170293986797332763671875E-319" + ); + assert_float_result_bits_eq!( + 0x0, + f64, + "2.4703282292062327208828439643411068618252990130716238221279284125033775363510437593264991818081799618989828234772285886546332835517796989819938739800539093906315035659515570226392290858392449105184435931802849936536152500319370457678249219365623669863658480757001585769269903706311928279558551332927834338409351978015531246597263579574622766465272827220056374006485499977096599470454020828166226237857393450736339007967761930577506740176324673600968951340535537458516661134223766678604162159680461914467291840300530057530849048765391711386591646239524912623653881879636239373280423891018672348497668235089863388587925628302755995657524455507255189313690836254779186948667994968324049705821028513185451396213837722826145437693412532098591327667236328124999e-324" + ); + assert_float_result_bits_eq!( + 0x0, + f64, + "2.4703282292062327208828439643411068618252990130716238221279284125033775363510437593264991818081799618989828234772285886546332835517796989819938739800539093906315035659515570226392290858392449105184435931802849936536152500319370457678249219365623669863658480757001585769269903706311928279558551332927834338409351978015531246597263579574622766465272827220056374006485499977096599470454020828166226237857393450736339007967761930577506740176324673600968951340535537458516661134223766678604162159680461914467291840300530057530849048765391711386591646239524912623653881879636239373280423891018672348497668235089863388587925628302755995657524455507255189313690836254779186948667994968324049705821028513185451396213837722826145437693412532098591327667236328125e-324" + ); + assert_float_result_bits_eq!( + 0x1, + f64, + "2.4703282292062327208828439643411068618252990130716238221279284125033775363510437593264991818081799618989828234772285886546332835517796989819938739800539093906315035659515570226392290858392449105184435931802849936536152500319370457678249219365623669863658480757001585769269903706311928279558551332927834338409351978015531246597263579574622766465272827220056374006485499977096599470454020828166226237857393450736339007967761930577506740176324673600968951340535537458516661134223766678604162159680461914467291840300530057530849048765391711386591646239524912623653881879636239373280423891018672348497668235089863388587925628302755995657524455507255189313690836254779186948667994968324049705821028513185451396213837722826145437693412532098591327667236328125001e-324" + ); + assert_float_result_bits_eq!( + 0x1, + f64, + "7.4109846876186981626485318930233205854758970392148714663837852375101326090531312779794975454245398856969484704316857659638998506553390969459816219401617281718945106978546710679176872575177347315553307795408549809608457500958111373034747658096871009590975442271004757307809711118935784838675653998783503015228055934046593739791790738723868299395818481660169122019456499931289798411362062484498678713572180352209017023903285791732520220528974020802906854021606612375549983402671300035812486479041385743401875520901590172592547146296175134159774938718574737870961645638908718119841271673056017045493004705269590165763776884908267986972573366521765567941072508764337560846003984904972149117463085539556354188641513168478436313080237596295773983001708984374999e-324" + ); + assert_float_result_bits_eq!( + 0x2, + f64, + "7.4109846876186981626485318930233205854758970392148714663837852375101326090531312779794975454245398856969484704316857659638998506553390969459816219401617281718945106978546710679176872575177347315553307795408549809608457500958111373034747658096871009590975442271004757307809711118935784838675653998783503015228055934046593739791790738723868299395818481660169122019456499931289798411362062484498678713572180352209017023903285791732520220528974020802906854021606612375549983402671300035812486479041385743401875520901590172592547146296175134159774938718574737870961645638908718119841271673056017045493004705269590165763776884908267986972573366521765567941072508764337560846003984904972149117463085539556354188641513168478436313080237596295773983001708984375e-324" + ); + assert_float_result_bits_eq!( + 0x2, + f64, + "7.4109846876186981626485318930233205854758970392148714663837852375101326090531312779794975454245398856969484704316857659638998506553390969459816219401617281718945106978546710679176872575177347315553307795408549809608457500958111373034747658096871009590975442271004757307809711118935784838675653998783503015228055934046593739791790738723868299395818481660169122019456499931289798411362062484498678713572180352209017023903285791732520220528974020802906854021606612375549983402671300035812486479041385743401875520901590172592547146296175134159774938718574737870961645638908718119841271673056017045493004705269590165763776884908267986972573366521765567941072508764337560846003984904972149117463085539556354188641513168478436313080237596295773983001708984375001e-324" + ); + assert_float_result_bits_eq!( + 0x6c9a143590c14, + f64, + "94393431193180696942841837085033647913224148539854e-358" + ); + assert_float_result_bits_eq!( + 0x7802665fd9600, + f64, + "104308485241983990666713401708072175773165034278685682646111762292409330928739751702404658197872319129036519947435319418387839758990478549477777586673075945844895981012024387992135617064532141489278815239849108105951619997829153633535314849999674266169258928940692239684771590065027025835804863585454872499320500023126142553932654370362024104462255244034053203998964360882487378334860197725139151265590832887433736189468858614521708567646743455601905935595381852723723645799866672558576993978025033590728687206296379801363024094048327273913079612469982585674824156000783167963081616214710691759864332339239688734656548790656486646106983450809073750535624894296242072010195710276073042036425579852459556183541199012652571123898996574563824424330960027873516082763671875e-1075" + ); +} + +#[test] +fn many_digits() { + // Check large numbers of digits to ensure we have cases where significant + // digits (above Decimal::MAX_DIGITS) occurs. + assert_float_result_bits_eq!( + 0x7ffffe, + f32, + "1.175494140627517859246175898662808184331245864732796240031385942718174675986064769972472277004271745681762695312500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e-38" + ); + assert_float_result_bits_eq!( + 0x7ffffe, + f32, + "1.175494140627517859246175898662808184331245864732796240031385942718174675986064769972472277004271745681762695312500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e-38" + ); } diff --git a/crux-mir/lib/core/tests/num/dec2flt/rawfp.rs b/crux-mir/lib/core/tests/num/dec2flt/rawfp.rs deleted file mode 100644 index 665fb6b9e..000000000 --- a/crux-mir/lib/core/tests/num/dec2flt/rawfp.rs +++ /dev/null @@ -1,177 +0,0 @@ -use core::num::dec2flt::rawfp::RawFloat; -use core::num::dec2flt::rawfp::{fp_to_float, next_float, prev_float, round_normal}; -use core::num::diy_float::Fp; -use std::f32; -use std::f64; - -fn integer_decode(f: f64) -> (u64, i16, i8) { - RawFloat::integer_decode(f) -} - -#[test] -fn fp_to_float_half_to_even() { - fn is_normalized(sig: u64) -> bool { - // intentionally written without {min,max}_sig() as a sanity check - sig >> 52 == 1 && sig >> 53 == 0 - } - - fn conv(sig: u64) -> u64 { - // The significands are perfectly in range, so the exponent should not matter - let (m1, e1, _) = integer_decode(fp_to_float::(Fp { f: sig, e: 0 })); - assert_eq!(e1, 0 + 64 - 53); - let (m2, e2, _) = integer_decode(fp_to_float::(Fp { f: sig, e: 55 })); - assert_eq!(e2, 55 + 64 - 53); - assert_eq!(m2, m1); - let (m3, e3, _) = integer_decode(fp_to_float::(Fp { f: sig, e: -78 })); - assert_eq!(e3, -78 + 64 - 53); - assert_eq!(m3, m2); - m3 - } - - let odd = 0x1F_EDCB_A012_345F; - let even = odd - 1; - assert!(is_normalized(odd)); - assert!(is_normalized(even)); - assert_eq!(conv(odd << 11), odd); - assert_eq!(conv(even << 11), even); - assert_eq!(conv(odd << 11 | 1 << 10), odd + 1); - assert_eq!(conv(even << 11 | 1 << 10), even); - assert_eq!(conv(even << 11 | 1 << 10 | 1), even + 1); - assert_eq!(conv(odd << 11 | 1 << 9), odd); - assert_eq!(conv(even << 11 | 1 << 9), even); - assert_eq!(conv(odd << 11 | 0x7FF), odd + 1); - assert_eq!(conv(even << 11 | 0x7FF), even + 1); - assert_eq!(conv(odd << 11 | 0x3FF), odd); - assert_eq!(conv(even << 11 | 0x3FF), even); -} - -#[test] -fn integers_to_f64() { - assert_eq!(fp_to_float::(Fp { f: 1, e: 0 }), 1.0); - assert_eq!(fp_to_float::(Fp { f: 42, e: 7 }), (42 << 7) as f64); - assert_eq!(fp_to_float::(Fp { f: 1 << 20, e: 30 }), (1u64 << 50) as f64); - assert_eq!(fp_to_float::(Fp { f: 4, e: -3 }), 0.5); -} - -const SOME_FLOATS: [f64; 9] = [ - 0.1f64, - 33.568, - 42.1e-5, - 777.0e9, - 1.1111, - 0.347997, - 9843579834.35892, - 12456.0e-150, - 54389573.0e-150, -]; - -#[test] -fn human_f64_roundtrip() { - for &x in &SOME_FLOATS { - let (f, e, _) = integer_decode(x); - let fp = Fp { f: f, e: e }; - assert_eq!(fp_to_float::(fp), x); - } -} - -#[test] -fn rounding_overflow() { - let x = Fp { f: 0xFF_FF_FF_FF_FF_FF_FF_00u64, e: 42 }; - let rounded = round_normal::(x); - let adjusted_k = x.e + 64 - 53; - assert_eq!(rounded.sig, 1 << 52); - assert_eq!(rounded.k, adjusted_k + 1); -} - -#[cfg_attr(all(target_arch = "wasm32", target_os = "emscripten"), ignore)] // issue 42630 -#[test] -fn prev_float_monotonic() { - let mut x = 1.0; - for _ in 0..100 { - let x1 = prev_float(x); - assert!(x1 < x); - assert!(x - x1 < 1e-15); - x = x1; - } -} - -const MIN_SUBNORMAL: f64 = 5e-324; - -#[test] -fn next_float_zero() { - let tiny = next_float(0.0); - assert_eq!(tiny, MIN_SUBNORMAL); - assert!(tiny != 0.0); -} - -#[test] -fn next_float_subnormal() { - let second = next_float(MIN_SUBNORMAL); - // For subnormals, MIN_SUBNORMAL is the ULP - assert!(second != MIN_SUBNORMAL); - assert!(second > 0.0); - assert_eq!(second - MIN_SUBNORMAL, MIN_SUBNORMAL); -} - -#[test] -fn next_float_inf() { - assert_eq!(next_float(f64::MAX), f64::INFINITY); - assert_eq!(next_float(f64::INFINITY), f64::INFINITY); -} - -#[cfg_attr(all(target_arch = "wasm32", target_os = "emscripten"), ignore)] // issue 42630 -#[test] -fn next_prev_identity() { - for &x in &SOME_FLOATS { - assert_eq!(prev_float(next_float(x)), x); - assert_eq!(prev_float(prev_float(next_float(next_float(x)))), x); - assert_eq!(next_float(prev_float(x)), x); - assert_eq!(next_float(next_float(prev_float(prev_float(x)))), x); - } -} - -#[cfg_attr(all(target_arch = "wasm32", target_os = "emscripten"), ignore)] // issue 42630 -#[test] -fn next_float_monotonic() { - let mut x = 0.49999999999999; - assert!(x < 0.5); - for _ in 0..200 { - let x1 = next_float(x); - assert!(x1 > x); - assert!(x1 - x < 1e-15, "next_float_monotonic: delta = {:?}", x1 - x); - x = x1; - } - assert!(x > 0.5); -} - -#[test] -fn test_f32_integer_decode() { - assert_eq!(3.14159265359f32.integer_decode(), (13176795, -22, 1)); - assert_eq!((-8573.5918555f32).integer_decode(), (8779358, -10, -1)); - assert_eq!(2f32.powf(100.0).integer_decode(), (8388608, 77, 1)); - assert_eq!(0f32.integer_decode(), (0, -150, 1)); - assert_eq!((-0f32).integer_decode(), (0, -150, -1)); - assert_eq!(f32::INFINITY.integer_decode(), (8388608, 105, 1)); - assert_eq!(f32::NEG_INFINITY.integer_decode(), (8388608, 105, -1)); - - // Ignore the "sign" (quiet / signalling flag) of NAN. - // It can vary between runtime operations and LLVM folding. - let (nan_m, nan_e, _nan_s) = f32::NAN.integer_decode(); - assert_eq!((nan_m, nan_e), (12582912, 105)); -} - -#[test] -fn test_f64_integer_decode() { - assert_eq!(3.14159265359f64.integer_decode(), (7074237752028906, -51, 1)); - assert_eq!((-8573.5918555f64).integer_decode(), (4713381968463931, -39, -1)); - assert_eq!(2f64.powf(100.0).integer_decode(), (4503599627370496, 48, 1)); - assert_eq!(0f64.integer_decode(), (0, -1075, 1)); - assert_eq!((-0f64).integer_decode(), (0, -1075, -1)); - assert_eq!(f64::INFINITY.integer_decode(), (4503599627370496, 972, 1)); - assert_eq!(f64::NEG_INFINITY.integer_decode(), (4503599627370496, 972, -1)); - - // Ignore the "sign" (quiet / signalling flag) of NAN. - // It can vary between runtime operations and LLVM folding. - let (nan_m, nan_e, _nan_s) = f64::NAN.integer_decode(); - assert_eq!((nan_m, nan_e), (6755399441055744, 972)); -} diff --git a/crux-mir/lib/core/tests/num/flt2dec/estimator.rs b/crux-mir/lib/core/tests/num/flt2dec/estimator.rs index 8ee06d895..da203b5f3 100644 --- a/crux-mir/lib/core/tests/num/flt2dec/estimator.rs +++ b/crux-mir/lib/core/tests/num/flt2dec/estimator.rs @@ -52,12 +52,10 @@ fn test_estimate_scaling_factor() { assert_almost_eq!(estimate_scaling_factor(1, -1074), -323); assert_almost_eq!(estimate_scaling_factor(0x1fffffffffffff, 971), 309); - #[cfg(not(miri))] // Miri is too slow - let iter = -1074..972; - #[cfg(miri)] - let iter = (-1074..972).step_by(37); + // Miri is too slow + let step = if cfg!(miri) { 37 } else { 1 }; - for i in iter { + for i in (-1074..972).step_by(step) { let expected = super::ldexp_f64(1.0, i).log10().ceil(); assert_almost_eq!(estimate_scaling_factor(1, i as i16), expected as i16); } diff --git a/crux-mir/lib/core/tests/num/flt2dec/mod.rs b/crux-mir/lib/core/tests/num/flt2dec/mod.rs index e945d9c4a..30843cc3d 100644 --- a/crux-mir/lib/core/tests/num/flt2dec/mod.rs +++ b/crux-mir/lib/core/tests/num/flt2dec/mod.rs @@ -1,10 +1,12 @@ -use std::{f32, f64, fmt, i16, str}; +use std::mem::MaybeUninit; +use std::{fmt, str}; use core::num::flt2dec::{decode, DecodableFloat, Decoded, FullDecoded}; -use core::num::flt2dec::{round_up, Formatted, Part, Sign, MAX_SIG_DIGITS}; +use core::num::flt2dec::{round_up, Sign, MAX_SIG_DIGITS}; use core::num::flt2dec::{ to_exact_exp_str, to_exact_fixed_str, to_shortest_exp_str, to_shortest_str, }; +use core::num::fmt::{Formatted, Part}; pub use test::Bencher; @@ -18,7 +20,7 @@ mod random; pub fn decode_finite(v: T) -> Decoded { match decode(v).1 { FullDecoded::Finite(decoded) => decoded, - full_decoded => panic!("expected finite, got {:?} instead", full_decoded), + full_decoded => panic!("expected finite, got {full_decoded:?} instead"), } } @@ -36,20 +38,20 @@ macro_rules! check_shortest { ); ($f:ident($v:expr) => $buf:expr, $exp:expr; $fmt:expr, $($key:ident = $val:expr),*) => ({ - let mut buf = [b'_'; MAX_SIG_DIGITS]; - let (len, k) = $f(&decode_finite($v), &mut buf); - assert!((&buf[..len], k) == ($buf, $exp), - $fmt, actual = (str::from_utf8(&buf[..len]).unwrap(), k), + let mut buf = [MaybeUninit::new(b'_'); MAX_SIG_DIGITS]; + let (buf, k) = $f(&decode_finite($v), &mut buf); + assert!((buf, k) == ($buf, $exp), + $fmt, actual = (str::from_utf8(buf).unwrap(), k), expected = (str::from_utf8($buf).unwrap(), $exp), $($key = $val),*); }); ($f:ident{$($k:ident: $v:expr),+} => $buf:expr, $exp:expr; $fmt:expr, $($key:ident = $val:expr),*) => ({ - let mut buf = [b'_'; MAX_SIG_DIGITS]; - let (len, k) = $f(&Decoded { $($k: $v),+ }, &mut buf); - assert!((&buf[..len], k) == ($buf, $exp), - $fmt, actual = (str::from_utf8(&buf[..len]).unwrap(), k), + let mut buf = [MaybeUninit::new(b'_'); MAX_SIG_DIGITS]; + let (buf, k) = $f(&Decoded { $($k: $v),+ }, &mut buf); + assert!((buf, k) == ($buf, $exp), + $fmt, actual = (str::from_utf8(buf).unwrap(), k), expected = (str::from_utf8($buf).unwrap(), $exp), $($key = $val),*); }) @@ -58,9 +60,9 @@ macro_rules! check_shortest { macro_rules! try_exact { ($f:ident($decoded:expr) => $buf:expr, $expected:expr, $expectedk:expr; $fmt:expr, $($key:ident = $val:expr),*) => ({ - let (len, k) = $f($decoded, &mut $buf[..$expected.len()], i16::MIN); - assert!((&$buf[..len], k) == ($expected, $expectedk), - $fmt, actual = (str::from_utf8(&$buf[..len]).unwrap(), k), + let (buf, k) = $f($decoded, &mut $buf[..$expected.len()], i16::MIN); + assert!((buf, k) == ($expected, $expectedk), + $fmt, actual = (str::from_utf8(buf).unwrap(), k), expected = (str::from_utf8($expected).unwrap(), $expectedk), $($key = $val),*); }) @@ -69,9 +71,9 @@ macro_rules! try_exact { macro_rules! try_fixed { ($f:ident($decoded:expr) => $buf:expr, $request:expr, $expected:expr, $expectedk:expr; $fmt:expr, $($key:ident = $val:expr),*) => ({ - let (len, k) = $f($decoded, &mut $buf[..], $request); - assert!((&$buf[..len], k) == ($expected, $expectedk), - $fmt, actual = (str::from_utf8(&$buf[..len]).unwrap(), k), + let (buf, k) = $f($decoded, &mut $buf[..], $request); + assert!((buf, k) == ($expected, $expectedk), + $fmt, actual = (str::from_utf8(buf).unwrap(), k), expected = (str::from_utf8($expected).unwrap(), $expectedk), $($key = $val),*); }) @@ -93,10 +95,10 @@ fn ldexp_f64(a: f64, b: i32) -> f64 { fn check_exact(mut f: F, v: T, vstr: &str, expected: &[u8], expectedk: i16) where T: DecodableFloat, - F: FnMut(&Decoded, &mut [u8], i16) -> (usize, i16), + F: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit], i16) -> (&'a [u8], i16), { // use a large enough buffer - let mut buf = [b'_'; 1024]; + let mut buf = [MaybeUninit::new(b'_'); 1024]; let mut expected_ = [b'_'; 1024]; let decoded = decode_finite(v); @@ -118,7 +120,7 @@ where // we should always return `100..00` (`i` digits) instead, since that's // what we can came up with `i` digits anyway. `round_up` assumes that // the adjustment to the length is done by caller, which we simply ignore. - if let Some(_) = round_up(&mut expected_, i) { + if let Some(_) = round_up(&mut expected_[..i]) { expectedk_ += 1; } } @@ -136,7 +138,7 @@ where // check exact rounding for zero- and negative-width cases let start; - if expected[0] >= b'5' { + if expected[0] > b'5' { try_fixed!(f(&decoded) => &mut buf, expectedk, b"1", expectedk + 1; "zero-width rounding-up mismatch for v={v}: \ actual {actual:?}, expected {expected:?}", @@ -193,10 +195,10 @@ impl TestableFloat for f64 { fn check_exact_one(mut f: F, x: i64, e: isize, tstr: &str, expected: &[u8], expectedk: i16) where T: TestableFloat, - F: FnMut(&Decoded, &mut [u8], i16) -> (usize, i16), + F: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit], i16) -> (&'a [u8], i16), { // use a large enough buffer - let mut buf = [b'_'; 1024]; + let mut buf = [MaybeUninit::new(b'_'); 1024]; let v: T = TestableFloat::ldexpi(x, e); let decoded = decode_finite(v); @@ -230,7 +232,7 @@ macro_rules! check_exact_one { pub fn f32_shortest_sanity_test(mut f: F) where - F: FnMut(&Decoded, &mut [u8]) -> (usize, i16), + F: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit]) -> (&'a [u8], i16), { // 0.0999999940395355224609375 // 0.100000001490116119384765625 @@ -277,7 +279,7 @@ where pub fn f32_exact_sanity_test(mut f: F) where - F: FnMut(&Decoded, &mut [u8], i16) -> (usize, i16), + F: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit], i16) -> (&'a [u8], i16), { let minf32 = ldexp_f32(1.0, -149); @@ -321,7 +323,7 @@ where pub fn f64_shortest_sanity_test(mut f: F) where - F: FnMut(&Decoded, &mut [u8]) -> (usize, i16), + F: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit]) -> (&'a [u8], i16), { // 0.0999999999999999777955395074968691915273... // 0.1000000000000000055511151231257827021181... @@ -387,7 +389,7 @@ where pub fn f64_exact_sanity_test(mut f: F) where - F: FnMut(&Decoded, &mut [u8], i16) -> (usize, i16), + F: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit], i16) -> (&'a [u8], i16), { let minf64 = ldexp_f64(1.0, -1074); @@ -474,7 +476,7 @@ where pub fn more_shortest_sanity_test(mut f: F) where - F: FnMut(&Decoded, &mut [u8]) -> (usize, i16), + F: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit]) -> (&'a [u8], i16), { check_shortest!(f{mant: 99_999_999_999_999_999, minus: 1, plus: 1, exp: 0, inclusive: true} => b"1", 18); @@ -484,10 +486,10 @@ where fn to_string_with_parts(mut f: F) -> String where - F: for<'a> FnMut(&'a mut [u8], &'a mut [Part<'a>]) -> Formatted<'a>, + F: for<'a> FnMut(&'a mut [MaybeUninit], &'a mut [MaybeUninit>]) -> Formatted<'a>, { - let mut buf = [0; 1024]; - let mut parts = [Part::Zero(0); 16]; + let mut buf = [MaybeUninit::new(0); 1024]; + let mut parts = [MaybeUninit::new(Part::Zero(0)); 16]; let formatted = f(&mut buf, &mut parts); let mut ret = vec![0; formatted.len()]; assert_eq!(formatted.write(&mut ret), Some(ret.len())); @@ -496,14 +498,14 @@ where pub fn to_shortest_str_test(mut f_: F) where - F: FnMut(&Decoded, &mut [u8]) -> (usize, i16), + F: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit]) -> (&'a [u8], i16), { use core::num::flt2dec::Sign::*; fn to_string(f: &mut F, v: T, sign: Sign, frac_digits: usize) -> String where T: DecodableFloat, - F: FnMut(&Decoded, &mut [u8]) -> (usize, i16), + F: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit]) -> (&'a [u8], i16), { to_string_with_parts(|buf, parts| { to_shortest_str(|d, b| f(d, b), v, sign, frac_digits, buf, parts) @@ -513,51 +515,38 @@ where let f = &mut f_; assert_eq!(to_string(f, 0.0, Minus, 0), "0"); - assert_eq!(to_string(f, 0.0, MinusRaw, 0), "0"); + assert_eq!(to_string(f, 0.0, Minus, 0), "0"); assert_eq!(to_string(f, 0.0, MinusPlus, 0), "+0"); - assert_eq!(to_string(f, 0.0, MinusPlusRaw, 0), "+0"); - assert_eq!(to_string(f, -0.0, Minus, 0), "0"); - assert_eq!(to_string(f, -0.0, MinusRaw, 0), "-0"); - assert_eq!(to_string(f, -0.0, MinusPlus, 0), "+0"); - assert_eq!(to_string(f, -0.0, MinusPlusRaw, 0), "-0"); + assert_eq!(to_string(f, -0.0, Minus, 0), "-0"); + assert_eq!(to_string(f, -0.0, MinusPlus, 0), "-0"); + assert_eq!(to_string(f, 0.0, Minus, 1), "0.0"); assert_eq!(to_string(f, 0.0, Minus, 1), "0.0"); - assert_eq!(to_string(f, 0.0, MinusRaw, 1), "0.0"); assert_eq!(to_string(f, 0.0, MinusPlus, 1), "+0.0"); - assert_eq!(to_string(f, 0.0, MinusPlusRaw, 1), "+0.0"); - assert_eq!(to_string(f, -0.0, Minus, 8), "0.00000000"); - assert_eq!(to_string(f, -0.0, MinusRaw, 8), "-0.00000000"); - assert_eq!(to_string(f, -0.0, MinusPlus, 8), "+0.00000000"); - assert_eq!(to_string(f, -0.0, MinusPlusRaw, 8), "-0.00000000"); + assert_eq!(to_string(f, -0.0, Minus, 8), "-0.00000000"); + assert_eq!(to_string(f, -0.0, MinusPlus, 8), "-0.00000000"); assert_eq!(to_string(f, 1.0 / 0.0, Minus, 0), "inf"); - assert_eq!(to_string(f, 1.0 / 0.0, MinusRaw, 0), "inf"); + assert_eq!(to_string(f, 1.0 / 0.0, Minus, 0), "inf"); assert_eq!(to_string(f, 1.0 / 0.0, MinusPlus, 0), "+inf"); - assert_eq!(to_string(f, 1.0 / 0.0, MinusPlusRaw, 0), "+inf"); assert_eq!(to_string(f, 0.0 / 0.0, Minus, 0), "NaN"); - assert_eq!(to_string(f, 0.0 / 0.0, MinusRaw, 1), "NaN"); - assert_eq!(to_string(f, 0.0 / 0.0, MinusPlus, 8), "NaN"); - assert_eq!(to_string(f, 0.0 / 0.0, MinusPlusRaw, 64), "NaN"); + assert_eq!(to_string(f, 0.0 / 0.0, Minus, 1), "NaN"); + assert_eq!(to_string(f, 0.0 / 0.0, MinusPlus, 64), "NaN"); assert_eq!(to_string(f, -1.0 / 0.0, Minus, 0), "-inf"); - assert_eq!(to_string(f, -1.0 / 0.0, MinusRaw, 1), "-inf"); - assert_eq!(to_string(f, -1.0 / 0.0, MinusPlus, 8), "-inf"); - assert_eq!(to_string(f, -1.0 / 0.0, MinusPlusRaw, 64), "-inf"); + assert_eq!(to_string(f, -1.0 / 0.0, Minus, 1), "-inf"); + assert_eq!(to_string(f, -1.0 / 0.0, MinusPlus, 64), "-inf"); assert_eq!(to_string(f, 3.14, Minus, 0), "3.14"); - assert_eq!(to_string(f, 3.14, MinusRaw, 0), "3.14"); + assert_eq!(to_string(f, 3.14, Minus, 0), "3.14"); assert_eq!(to_string(f, 3.14, MinusPlus, 0), "+3.14"); - assert_eq!(to_string(f, 3.14, MinusPlusRaw, 0), "+3.14"); assert_eq!(to_string(f, -3.14, Minus, 0), "-3.14"); - assert_eq!(to_string(f, -3.14, MinusRaw, 0), "-3.14"); + assert_eq!(to_string(f, -3.14, Minus, 0), "-3.14"); assert_eq!(to_string(f, -3.14, MinusPlus, 0), "-3.14"); - assert_eq!(to_string(f, -3.14, MinusPlusRaw, 0), "-3.14"); assert_eq!(to_string(f, 3.14, Minus, 1), "3.14"); - assert_eq!(to_string(f, 3.14, MinusRaw, 2), "3.14"); - assert_eq!(to_string(f, 3.14, MinusPlus, 3), "+3.140"); - assert_eq!(to_string(f, 3.14, MinusPlusRaw, 4), "+3.1400"); + assert_eq!(to_string(f, 3.14, Minus, 2), "3.14"); + assert_eq!(to_string(f, 3.14, MinusPlus, 4), "+3.1400"); + assert_eq!(to_string(f, -3.14, Minus, 8), "-3.14000000"); assert_eq!(to_string(f, -3.14, Minus, 8), "-3.14000000"); - assert_eq!(to_string(f, -3.14, MinusRaw, 8), "-3.14000000"); assert_eq!(to_string(f, -3.14, MinusPlus, 8), "-3.14000000"); - assert_eq!(to_string(f, -3.14, MinusPlusRaw, 8), "-3.14000000"); assert_eq!(to_string(f, 7.5e-11, Minus, 0), "0.000000000075"); assert_eq!(to_string(f, 7.5e-11, Minus, 3), "0.000000000075"); @@ -597,14 +586,14 @@ where pub fn to_shortest_exp_str_test(mut f_: F) where - F: FnMut(&Decoded, &mut [u8]) -> (usize, i16), + F: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit]) -> (&'a [u8], i16), { use core::num::flt2dec::Sign::*; fn to_string(f: &mut F, v: T, sign: Sign, exp_bounds: (i16, i16), upper: bool) -> String where T: DecodableFloat, - F: FnMut(&Decoded, &mut [u8]) -> (usize, i16), + F: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit]) -> (&'a [u8], i16), { to_string_with_parts(|buf, parts| { to_shortest_exp_str(|d, b| f(d, b), v, sign, exp_bounds, upper, buf, parts) @@ -614,68 +603,48 @@ where let f = &mut f_; assert_eq!(to_string(f, 0.0, Minus, (-4, 16), false), "0"); - assert_eq!(to_string(f, 0.0, MinusRaw, (-4, 16), false), "0"); + assert_eq!(to_string(f, 0.0, Minus, (-4, 16), false), "0"); assert_eq!(to_string(f, 0.0, MinusPlus, (-4, 16), false), "+0"); - assert_eq!(to_string(f, 0.0, MinusPlusRaw, (-4, 16), false), "+0"); - assert_eq!(to_string(f, -0.0, Minus, (-4, 16), false), "0"); - assert_eq!(to_string(f, -0.0, MinusRaw, (-4, 16), false), "-0"); - assert_eq!(to_string(f, -0.0, MinusPlus, (-4, 16), false), "+0"); - assert_eq!(to_string(f, -0.0, MinusPlusRaw, (-4, 16), false), "-0"); + assert_eq!(to_string(f, -0.0, Minus, (-4, 16), false), "-0"); + assert_eq!(to_string(f, -0.0, MinusPlus, (-4, 16), false), "-0"); assert_eq!(to_string(f, 0.0, Minus, (0, 0), true), "0E0"); - assert_eq!(to_string(f, 0.0, MinusRaw, (0, 0), false), "0e0"); - assert_eq!(to_string(f, 0.0, MinusPlus, (-9, -5), true), "+0E0"); - assert_eq!(to_string(f, 0.0, MinusPlusRaw, (5, 9), false), "+0e0"); - assert_eq!(to_string(f, -0.0, Minus, (0, 0), true), "0E0"); - assert_eq!(to_string(f, -0.0, MinusRaw, (0, 0), false), "-0e0"); - assert_eq!(to_string(f, -0.0, MinusPlus, (-9, -5), true), "+0E0"); - assert_eq!(to_string(f, -0.0, MinusPlusRaw, (5, 9), false), "-0e0"); + assert_eq!(to_string(f, 0.0, Minus, (0, 0), false), "0e0"); + assert_eq!(to_string(f, 0.0, MinusPlus, (5, 9), false), "+0e0"); + assert_eq!(to_string(f, -0.0, Minus, (0, 0), true), "-0E0"); + assert_eq!(to_string(f, -0.0, MinusPlus, (5, 9), false), "-0e0"); assert_eq!(to_string(f, 1.0 / 0.0, Minus, (-4, 16), false), "inf"); - assert_eq!(to_string(f, 1.0 / 0.0, MinusRaw, (-4, 16), true), "inf"); - assert_eq!(to_string(f, 1.0 / 0.0, MinusPlus, (-4, 16), false), "+inf"); - assert_eq!(to_string(f, 1.0 / 0.0, MinusPlusRaw, (-4, 16), true), "+inf"); + assert_eq!(to_string(f, 1.0 / 0.0, Minus, (-4, 16), true), "inf"); + assert_eq!(to_string(f, 1.0 / 0.0, MinusPlus, (-4, 16), true), "+inf"); assert_eq!(to_string(f, 0.0 / 0.0, Minus, (0, 0), false), "NaN"); - assert_eq!(to_string(f, 0.0 / 0.0, MinusRaw, (0, 0), true), "NaN"); - assert_eq!(to_string(f, 0.0 / 0.0, MinusPlus, (-9, -5), false), "NaN"); - assert_eq!(to_string(f, 0.0 / 0.0, MinusPlusRaw, (5, 9), true), "NaN"); + assert_eq!(to_string(f, 0.0 / 0.0, Minus, (0, 0), true), "NaN"); + assert_eq!(to_string(f, 0.0 / 0.0, MinusPlus, (5, 9), true), "NaN"); assert_eq!(to_string(f, -1.0 / 0.0, Minus, (0, 0), false), "-inf"); - assert_eq!(to_string(f, -1.0 / 0.0, MinusRaw, (0, 0), true), "-inf"); - assert_eq!(to_string(f, -1.0 / 0.0, MinusPlus, (-9, -5), false), "-inf"); - assert_eq!(to_string(f, -1.0 / 0.0, MinusPlusRaw, (5, 9), true), "-inf"); + assert_eq!(to_string(f, -1.0 / 0.0, Minus, (0, 0), true), "-inf"); + assert_eq!(to_string(f, -1.0 / 0.0, MinusPlus, (5, 9), true), "-inf"); assert_eq!(to_string(f, 3.14, Minus, (-4, 16), false), "3.14"); - assert_eq!(to_string(f, 3.14, MinusRaw, (-4, 16), false), "3.14"); assert_eq!(to_string(f, 3.14, MinusPlus, (-4, 16), false), "+3.14"); - assert_eq!(to_string(f, 3.14, MinusPlusRaw, (-4, 16), false), "+3.14"); assert_eq!(to_string(f, -3.14, Minus, (-4, 16), false), "-3.14"); - assert_eq!(to_string(f, -3.14, MinusRaw, (-4, 16), false), "-3.14"); assert_eq!(to_string(f, -3.14, MinusPlus, (-4, 16), false), "-3.14"); - assert_eq!(to_string(f, -3.14, MinusPlusRaw, (-4, 16), false), "-3.14"); assert_eq!(to_string(f, 3.14, Minus, (0, 0), true), "3.14E0"); - assert_eq!(to_string(f, 3.14, MinusRaw, (0, 0), false), "3.14e0"); - assert_eq!(to_string(f, 3.14, MinusPlus, (-9, -5), true), "+3.14E0"); - assert_eq!(to_string(f, 3.14, MinusPlusRaw, (5, 9), false), "+3.14e0"); + assert_eq!(to_string(f, 3.14, Minus, (0, 0), false), "3.14e0"); + assert_eq!(to_string(f, 3.14, MinusPlus, (5, 9), false), "+3.14e0"); assert_eq!(to_string(f, -3.14, Minus, (0, 0), true), "-3.14E0"); - assert_eq!(to_string(f, -3.14, MinusRaw, (0, 0), false), "-3.14e0"); - assert_eq!(to_string(f, -3.14, MinusPlus, (-9, -5), true), "-3.14E0"); - assert_eq!(to_string(f, -3.14, MinusPlusRaw, (5, 9), false), "-3.14e0"); + assert_eq!(to_string(f, -3.14, Minus, (0, 0), false), "-3.14e0"); + assert_eq!(to_string(f, -3.14, MinusPlus, (5, 9), false), "-3.14e0"); assert_eq!(to_string(f, 0.1, Minus, (-4, 16), false), "0.1"); - assert_eq!(to_string(f, 0.1, MinusRaw, (-4, 16), false), "0.1"); + assert_eq!(to_string(f, 0.1, Minus, (-4, 16), false), "0.1"); assert_eq!(to_string(f, 0.1, MinusPlus, (-4, 16), false), "+0.1"); - assert_eq!(to_string(f, 0.1, MinusPlusRaw, (-4, 16), false), "+0.1"); assert_eq!(to_string(f, -0.1, Minus, (-4, 16), false), "-0.1"); - assert_eq!(to_string(f, -0.1, MinusRaw, (-4, 16), false), "-0.1"); assert_eq!(to_string(f, -0.1, MinusPlus, (-4, 16), false), "-0.1"); - assert_eq!(to_string(f, -0.1, MinusPlusRaw, (-4, 16), false), "-0.1"); assert_eq!(to_string(f, 0.1, Minus, (0, 0), true), "1E-1"); - assert_eq!(to_string(f, 0.1, MinusRaw, (0, 0), false), "1e-1"); - assert_eq!(to_string(f, 0.1, MinusPlus, (-9, -5), true), "+1E-1"); - assert_eq!(to_string(f, 0.1, MinusPlusRaw, (5, 9), false), "+1e-1"); + assert_eq!(to_string(f, 0.1, Minus, (0, 0), false), "1e-1"); + assert_eq!(to_string(f, 0.1, MinusPlus, (5, 9), false), "+1e-1"); assert_eq!(to_string(f, -0.1, Minus, (0, 0), true), "-1E-1"); - assert_eq!(to_string(f, -0.1, MinusRaw, (0, 0), false), "-1e-1"); - assert_eq!(to_string(f, -0.1, MinusPlus, (-9, -5), true), "-1E-1"); - assert_eq!(to_string(f, -0.1, MinusPlusRaw, (5, 9), false), "-1e-1"); + assert_eq!(to_string(f, -0.1, Minus, (0, 0), false), "-1e-1"); + assert_eq!(to_string(f, -0.1, MinusPlus, (5, 9), false), "-1e-1"); assert_eq!(to_string(f, 7.5e-11, Minus, (-4, 16), false), "7.5e-11"); assert_eq!(to_string(f, 7.5e-11, Minus, (-11, 10), false), "0.000000000075"); @@ -716,14 +685,14 @@ where pub fn to_exact_exp_str_test(mut f_: F) where - F: FnMut(&Decoded, &mut [u8], i16) -> (usize, i16), + F: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit], i16) -> (&'a [u8], i16), { use core::num::flt2dec::Sign::*; fn to_string(f: &mut F, v: T, sign: Sign, ndigits: usize, upper: bool) -> String where T: DecodableFloat, - F: FnMut(&Decoded, &mut [u8], i16) -> (usize, i16), + F: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit], i16) -> (&'a [u8], i16), { to_string_with_parts(|buf, parts| { to_exact_exp_str(|d, b, l| f(d, b, l), v, sign, ndigits, upper, buf, parts) @@ -733,68 +702,51 @@ where let f = &mut f_; assert_eq!(to_string(f, 0.0, Minus, 1, true), "0E0"); - assert_eq!(to_string(f, 0.0, MinusRaw, 1, false), "0e0"); - assert_eq!(to_string(f, 0.0, MinusPlus, 1, true), "+0E0"); - assert_eq!(to_string(f, 0.0, MinusPlusRaw, 1, false), "+0e0"); - assert_eq!(to_string(f, -0.0, Minus, 1, true), "0E0"); - assert_eq!(to_string(f, -0.0, MinusRaw, 1, false), "-0e0"); - assert_eq!(to_string(f, -0.0, MinusPlus, 1, true), "+0E0"); - assert_eq!(to_string(f, -0.0, MinusPlusRaw, 1, false), "-0e0"); + assert_eq!(to_string(f, 0.0, Minus, 1, false), "0e0"); + assert_eq!(to_string(f, 0.0, MinusPlus, 1, false), "+0e0"); + assert_eq!(to_string(f, -0.0, Minus, 1, true), "-0E0"); + assert_eq!(to_string(f, -0.0, MinusPlus, 1, false), "-0e0"); assert_eq!(to_string(f, 0.0, Minus, 2, true), "0.0E0"); - assert_eq!(to_string(f, 0.0, MinusRaw, 2, false), "0.0e0"); - assert_eq!(to_string(f, 0.0, MinusPlus, 2, true), "+0.0E0"); - assert_eq!(to_string(f, 0.0, MinusPlusRaw, 2, false), "+0.0e0"); - assert_eq!(to_string(f, -0.0, Minus, 8, true), "0.0000000E0"); - assert_eq!(to_string(f, -0.0, MinusRaw, 8, false), "-0.0000000e0"); - assert_eq!(to_string(f, -0.0, MinusPlus, 8, true), "+0.0000000E0"); - assert_eq!(to_string(f, -0.0, MinusPlusRaw, 8, false), "-0.0000000e0"); + assert_eq!(to_string(f, 0.0, Minus, 2, false), "0.0e0"); + assert_eq!(to_string(f, 0.0, MinusPlus, 2, false), "+0.0e0"); + assert_eq!(to_string(f, -0.0, Minus, 8, false), "-0.0000000e0"); + assert_eq!(to_string(f, -0.0, MinusPlus, 8, false), "-0.0000000e0"); assert_eq!(to_string(f, 1.0 / 0.0, Minus, 1, false), "inf"); - assert_eq!(to_string(f, 1.0 / 0.0, MinusRaw, 1, true), "inf"); - assert_eq!(to_string(f, 1.0 / 0.0, MinusPlus, 1, false), "+inf"); - assert_eq!(to_string(f, 1.0 / 0.0, MinusPlusRaw, 1, true), "+inf"); + assert_eq!(to_string(f, 1.0 / 0.0, Minus, 1, true), "inf"); + assert_eq!(to_string(f, 1.0 / 0.0, MinusPlus, 1, true), "+inf"); assert_eq!(to_string(f, 0.0 / 0.0, Minus, 8, false), "NaN"); - assert_eq!(to_string(f, 0.0 / 0.0, MinusRaw, 8, true), "NaN"); - assert_eq!(to_string(f, 0.0 / 0.0, MinusPlus, 8, false), "NaN"); - assert_eq!(to_string(f, 0.0 / 0.0, MinusPlusRaw, 8, true), "NaN"); + assert_eq!(to_string(f, 0.0 / 0.0, Minus, 8, true), "NaN"); + assert_eq!(to_string(f, 0.0 / 0.0, MinusPlus, 8, true), "NaN"); assert_eq!(to_string(f, -1.0 / 0.0, Minus, 64, false), "-inf"); - assert_eq!(to_string(f, -1.0 / 0.0, MinusRaw, 64, true), "-inf"); - assert_eq!(to_string(f, -1.0 / 0.0, MinusPlus, 64, false), "-inf"); - assert_eq!(to_string(f, -1.0 / 0.0, MinusPlusRaw, 64, true), "-inf"); + assert_eq!(to_string(f, -1.0 / 0.0, Minus, 64, true), "-inf"); + assert_eq!(to_string(f, -1.0 / 0.0, MinusPlus, 64, true), "-inf"); assert_eq!(to_string(f, 3.14, Minus, 1, true), "3E0"); - assert_eq!(to_string(f, 3.14, MinusRaw, 1, false), "3e0"); - assert_eq!(to_string(f, 3.14, MinusPlus, 1, true), "+3E0"); - assert_eq!(to_string(f, 3.14, MinusPlusRaw, 1, false), "+3e0"); + assert_eq!(to_string(f, 3.14, Minus, 1, false), "3e0"); + assert_eq!(to_string(f, 3.14, MinusPlus, 1, false), "+3e0"); assert_eq!(to_string(f, -3.14, Minus, 2, true), "-3.1E0"); - assert_eq!(to_string(f, -3.14, MinusRaw, 2, false), "-3.1e0"); - assert_eq!(to_string(f, -3.14, MinusPlus, 2, true), "-3.1E0"); - assert_eq!(to_string(f, -3.14, MinusPlusRaw, 2, false), "-3.1e0"); + assert_eq!(to_string(f, -3.14, Minus, 2, false), "-3.1e0"); + assert_eq!(to_string(f, -3.14, MinusPlus, 2, false), "-3.1e0"); assert_eq!(to_string(f, 3.14, Minus, 3, true), "3.14E0"); - assert_eq!(to_string(f, 3.14, MinusRaw, 3, false), "3.14e0"); - assert_eq!(to_string(f, 3.14, MinusPlus, 3, true), "+3.14E0"); - assert_eq!(to_string(f, 3.14, MinusPlusRaw, 3, false), "+3.14e0"); + assert_eq!(to_string(f, 3.14, Minus, 3, false), "3.14e0"); + assert_eq!(to_string(f, 3.14, MinusPlus, 3, false), "+3.14e0"); assert_eq!(to_string(f, -3.14, Minus, 4, true), "-3.140E0"); - assert_eq!(to_string(f, -3.14, MinusRaw, 4, false), "-3.140e0"); - assert_eq!(to_string(f, -3.14, MinusPlus, 4, true), "-3.140E0"); - assert_eq!(to_string(f, -3.14, MinusPlusRaw, 4, false), "-3.140e0"); + assert_eq!(to_string(f, -3.14, Minus, 4, false), "-3.140e0"); + assert_eq!(to_string(f, -3.14, MinusPlus, 4, false), "-3.140e0"); assert_eq!(to_string(f, 0.195, Minus, 1, false), "2e-1"); - assert_eq!(to_string(f, 0.195, MinusRaw, 1, true), "2E-1"); - assert_eq!(to_string(f, 0.195, MinusPlus, 1, false), "+2e-1"); - assert_eq!(to_string(f, 0.195, MinusPlusRaw, 1, true), "+2E-1"); + assert_eq!(to_string(f, 0.195, Minus, 1, true), "2E-1"); + assert_eq!(to_string(f, 0.195, MinusPlus, 1, true), "+2E-1"); assert_eq!(to_string(f, -0.195, Minus, 2, false), "-2.0e-1"); - assert_eq!(to_string(f, -0.195, MinusRaw, 2, true), "-2.0E-1"); - assert_eq!(to_string(f, -0.195, MinusPlus, 2, false), "-2.0e-1"); - assert_eq!(to_string(f, -0.195, MinusPlusRaw, 2, true), "-2.0E-1"); + assert_eq!(to_string(f, -0.195, Minus, 2, true), "-2.0E-1"); + assert_eq!(to_string(f, -0.195, MinusPlus, 2, true), "-2.0E-1"); assert_eq!(to_string(f, 0.195, Minus, 3, false), "1.95e-1"); - assert_eq!(to_string(f, 0.195, MinusRaw, 3, true), "1.95E-1"); - assert_eq!(to_string(f, 0.195, MinusPlus, 3, false), "+1.95e-1"); - assert_eq!(to_string(f, 0.195, MinusPlusRaw, 3, true), "+1.95E-1"); + assert_eq!(to_string(f, 0.195, Minus, 3, true), "1.95E-1"); + assert_eq!(to_string(f, 0.195, MinusPlus, 3, true), "+1.95E-1"); assert_eq!(to_string(f, -0.195, Minus, 4, false), "-1.950e-1"); - assert_eq!(to_string(f, -0.195, MinusRaw, 4, true), "-1.950E-1"); - assert_eq!(to_string(f, -0.195, MinusPlus, 4, false), "-1.950e-1"); - assert_eq!(to_string(f, -0.195, MinusPlusRaw, 4, true), "-1.950E-1"); + assert_eq!(to_string(f, -0.195, Minus, 4, true), "-1.950E-1"); + assert_eq!(to_string(f, -0.195, MinusPlus, 4, true), "-1.950E-1"); assert_eq!(to_string(f, 9.5, Minus, 1, false), "1e1"); assert_eq!(to_string(f, 9.5, Minus, 2, false), "9.5e0"); @@ -989,14 +941,14 @@ where pub fn to_exact_fixed_str_test(mut f_: F) where - F: FnMut(&Decoded, &mut [u8], i16) -> (usize, i16), + F: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit], i16) -> (&'a [u8], i16), { use core::num::flt2dec::Sign::*; fn to_string(f: &mut F, v: T, sign: Sign, frac_digits: usize) -> String where T: DecodableFloat, - F: FnMut(&Decoded, &mut [u8], i16) -> (usize, i16), + F: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit], i16) -> (&'a [u8], i16), { to_string_with_parts(|buf, parts| { to_exact_fixed_str(|d, b, l| f(d, b, l), v, sign, frac_digits, buf, parts) @@ -1006,68 +958,48 @@ where let f = &mut f_; assert_eq!(to_string(f, 0.0, Minus, 0), "0"); - assert_eq!(to_string(f, 0.0, MinusRaw, 0), "0"); assert_eq!(to_string(f, 0.0, MinusPlus, 0), "+0"); - assert_eq!(to_string(f, 0.0, MinusPlusRaw, 0), "+0"); - assert_eq!(to_string(f, -0.0, Minus, 0), "0"); - assert_eq!(to_string(f, -0.0, MinusRaw, 0), "-0"); - assert_eq!(to_string(f, -0.0, MinusPlus, 0), "+0"); - assert_eq!(to_string(f, -0.0, MinusPlusRaw, 0), "-0"); + assert_eq!(to_string(f, -0.0, Minus, 0), "-0"); + assert_eq!(to_string(f, -0.0, MinusPlus, 0), "-0"); assert_eq!(to_string(f, 0.0, Minus, 1), "0.0"); - assert_eq!(to_string(f, 0.0, MinusRaw, 1), "0.0"); assert_eq!(to_string(f, 0.0, MinusPlus, 1), "+0.0"); - assert_eq!(to_string(f, 0.0, MinusPlusRaw, 1), "+0.0"); - assert_eq!(to_string(f, -0.0, Minus, 8), "0.00000000"); - assert_eq!(to_string(f, -0.0, MinusRaw, 8), "-0.00000000"); - assert_eq!(to_string(f, -0.0, MinusPlus, 8), "+0.00000000"); - assert_eq!(to_string(f, -0.0, MinusPlusRaw, 8), "-0.00000000"); + assert_eq!(to_string(f, -0.0, Minus, 8), "-0.00000000"); + assert_eq!(to_string(f, -0.0, MinusPlus, 8), "-0.00000000"); assert_eq!(to_string(f, 1.0 / 0.0, Minus, 0), "inf"); - assert_eq!(to_string(f, 1.0 / 0.0, MinusRaw, 1), "inf"); - assert_eq!(to_string(f, 1.0 / 0.0, MinusPlus, 8), "+inf"); - assert_eq!(to_string(f, 1.0 / 0.0, MinusPlusRaw, 64), "+inf"); + assert_eq!(to_string(f, 1.0 / 0.0, Minus, 1), "inf"); + assert_eq!(to_string(f, 1.0 / 0.0, MinusPlus, 64), "+inf"); assert_eq!(to_string(f, 0.0 / 0.0, Minus, 0), "NaN"); - assert_eq!(to_string(f, 0.0 / 0.0, MinusRaw, 1), "NaN"); - assert_eq!(to_string(f, 0.0 / 0.0, MinusPlus, 8), "NaN"); - assert_eq!(to_string(f, 0.0 / 0.0, MinusPlusRaw, 64), "NaN"); + assert_eq!(to_string(f, 0.0 / 0.0, Minus, 1), "NaN"); + assert_eq!(to_string(f, 0.0 / 0.0, MinusPlus, 64), "NaN"); assert_eq!(to_string(f, -1.0 / 0.0, Minus, 0), "-inf"); - assert_eq!(to_string(f, -1.0 / 0.0, MinusRaw, 1), "-inf"); - assert_eq!(to_string(f, -1.0 / 0.0, MinusPlus, 8), "-inf"); - assert_eq!(to_string(f, -1.0 / 0.0, MinusPlusRaw, 64), "-inf"); + assert_eq!(to_string(f, -1.0 / 0.0, Minus, 1), "-inf"); + assert_eq!(to_string(f, -1.0 / 0.0, MinusPlus, 64), "-inf"); assert_eq!(to_string(f, 3.14, Minus, 0), "3"); - assert_eq!(to_string(f, 3.14, MinusRaw, 0), "3"); + assert_eq!(to_string(f, 3.14, Minus, 0), "3"); assert_eq!(to_string(f, 3.14, MinusPlus, 0), "+3"); - assert_eq!(to_string(f, 3.14, MinusPlusRaw, 0), "+3"); assert_eq!(to_string(f, -3.14, Minus, 0), "-3"); - assert_eq!(to_string(f, -3.14, MinusRaw, 0), "-3"); + assert_eq!(to_string(f, -3.14, Minus, 0), "-3"); assert_eq!(to_string(f, -3.14, MinusPlus, 0), "-3"); - assert_eq!(to_string(f, -3.14, MinusPlusRaw, 0), "-3"); assert_eq!(to_string(f, 3.14, Minus, 1), "3.1"); - assert_eq!(to_string(f, 3.14, MinusRaw, 2), "3.14"); - assert_eq!(to_string(f, 3.14, MinusPlus, 3), "+3.140"); - assert_eq!(to_string(f, 3.14, MinusPlusRaw, 4), "+3.1400"); + assert_eq!(to_string(f, 3.14, Minus, 2), "3.14"); + assert_eq!(to_string(f, 3.14, MinusPlus, 4), "+3.1400"); + assert_eq!(to_string(f, -3.14, Minus, 8), "-3.14000000"); assert_eq!(to_string(f, -3.14, Minus, 8), "-3.14000000"); - assert_eq!(to_string(f, -3.14, MinusRaw, 8), "-3.14000000"); assert_eq!(to_string(f, -3.14, MinusPlus, 8), "-3.14000000"); - assert_eq!(to_string(f, -3.14, MinusPlusRaw, 8), "-3.14000000"); assert_eq!(to_string(f, 0.195, Minus, 0), "0"); - assert_eq!(to_string(f, 0.195, MinusRaw, 0), "0"); assert_eq!(to_string(f, 0.195, MinusPlus, 0), "+0"); - assert_eq!(to_string(f, 0.195, MinusPlusRaw, 0), "+0"); assert_eq!(to_string(f, -0.195, Minus, 0), "-0"); - assert_eq!(to_string(f, -0.195, MinusRaw, 0), "-0"); + assert_eq!(to_string(f, -0.195, Minus, 0), "-0"); assert_eq!(to_string(f, -0.195, MinusPlus, 0), "-0"); - assert_eq!(to_string(f, -0.195, MinusPlusRaw, 0), "-0"); assert_eq!(to_string(f, 0.195, Minus, 1), "0.2"); - assert_eq!(to_string(f, 0.195, MinusRaw, 2), "0.20"); - assert_eq!(to_string(f, 0.195, MinusPlus, 3), "+0.195"); - assert_eq!(to_string(f, 0.195, MinusPlusRaw, 4), "+0.1950"); + assert_eq!(to_string(f, 0.195, Minus, 2), "0.20"); + assert_eq!(to_string(f, 0.195, MinusPlus, 4), "+0.1950"); assert_eq!(to_string(f, -0.195, Minus, 5), "-0.19500"); - assert_eq!(to_string(f, -0.195, MinusRaw, 6), "-0.195000"); - assert_eq!(to_string(f, -0.195, MinusPlus, 7), "-0.1950000"); - assert_eq!(to_string(f, -0.195, MinusPlusRaw, 8), "-0.19500000"); + assert_eq!(to_string(f, -0.195, Minus, 6), "-0.195000"); + assert_eq!(to_string(f, -0.195, MinusPlus, 8), "-0.19500000"); assert_eq!(to_string(f, 999.5, Minus, 0), "1000"); assert_eq!(to_string(f, 999.5, Minus, 1), "999.5"); @@ -1075,7 +1007,7 @@ where assert_eq!(to_string(f, 999.5, Minus, 3), "999.500"); assert_eq!(to_string(f, 999.5, Minus, 30), "999.500000000000000000000000000000"); - assert_eq!(to_string(f, 0.5, Minus, 0), "1"); + assert_eq!(to_string(f, 0.5, Minus, 0), "0"); assert_eq!(to_string(f, 0.5, Minus, 1), "0.5"); assert_eq!(to_string(f, 0.5, Minus, 2), "0.50"); assert_eq!(to_string(f, 0.5, Minus, 3), "0.500"); diff --git a/crux-mir/lib/core/tests/num/flt2dec/random.rs b/crux-mir/lib/core/tests/num/flt2dec/random.rs index ecdfc4b30..0084c1c81 100644 --- a/crux-mir/lib/core/tests/num/flt2dec/random.rs +++ b/crux-mir/lib/core/tests/num/flt2dec/random.rs @@ -1,6 +1,6 @@ #![cfg(not(target_arch = "wasm32"))] -use std::i16; +use std::mem::MaybeUninit; use std::str; use core::num::flt2dec::strategy::grisu::format_exact_opt; @@ -9,20 +9,18 @@ use core::num::flt2dec::MAX_SIG_DIGITS; use core::num::flt2dec::{decode, DecodableFloat, Decoded, FullDecoded}; use rand::distributions::{Distribution, Uniform}; -use rand::rngs::StdRng; -use rand::SeedableRng; pub fn decode_finite(v: T) -> Decoded { match decode(v).1 { FullDecoded::Finite(decoded) => decoded, - full_decoded => panic!("expected finite, got {:?} instead", full_decoded), + full_decoded => panic!("expected finite, got {full_decoded:?} instead"), } } fn iterate(func: &str, k: usize, n: usize, mut f: F, mut g: G, mut v: V) -> (usize, usize) where - F: FnMut(&Decoded, &mut [u8]) -> Option<(usize, i16)>, - G: FnMut(&Decoded, &mut [u8]) -> (usize, i16), + F: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit]) -> Option<(&'a [u8], i16)>, + G: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit]) -> (&'a [u8], i16), V: FnMut(usize) -> Decoded, { assert!(k <= 1024); @@ -43,11 +41,11 @@ where } let decoded = v(i); - let mut buf1 = [0; 1024]; - if let Some((len1, e1)) = f(&decoded, &mut buf1[..k]) { - let mut buf2 = [0; 1024]; - let (len2, e2) = g(&decoded, &mut buf2[..k]); - if e1 == e2 && &buf1[..len1] == &buf2[..len2] { + let mut buf1 = [MaybeUninit::new(0); 1024]; + if let Some((buf1, e1)) = f(&decoded, &mut buf1[..k]) { + let mut buf2 = [MaybeUninit::new(0); 1024]; + let (buf2, e2) = g(&decoded, &mut buf2[..k]); + if e1 == e2 && buf1 == buf2 { npassed += 1; } else { println!( @@ -55,9 +53,9 @@ where i, n, decoded, - str::from_utf8(&buf1[..len1]).unwrap(), + str::from_utf8(buf1).unwrap(), e1, - str::from_utf8(&buf2[..len2]).unwrap(), + str::from_utf8(buf2).unwrap(), e2 ); } @@ -86,13 +84,13 @@ where pub fn f32_random_equivalence_test(f: F, g: G, k: usize, n: usize) where - F: FnMut(&Decoded, &mut [u8]) -> Option<(usize, i16)>, - G: FnMut(&Decoded, &mut [u8]) -> (usize, i16), + F: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit]) -> Option<(&'a [u8], i16)>, + G: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit]) -> (&'a [u8], i16), { if cfg!(target_os = "emscripten") { return; // using rng pulls in i128 support, which doesn't work } - let mut rng = StdRng::from_entropy(); + let mut rng = crate::test_rng(); let f32_range = Uniform::new(0x0000_0001u32, 0x7f80_0000); iterate("f32_random_equivalence_test", k, n, f, g, |_| { let x = f32::from_bits(f32_range.sample(&mut rng)); @@ -102,13 +100,13 @@ where pub fn f64_random_equivalence_test(f: F, g: G, k: usize, n: usize) where - F: FnMut(&Decoded, &mut [u8]) -> Option<(usize, i16)>, - G: FnMut(&Decoded, &mut [u8]) -> (usize, i16), + F: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit]) -> Option<(&'a [u8], i16)>, + G: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit]) -> (&'a [u8], i16), { if cfg!(target_os = "emscripten") { return; // using rng pulls in i128 support, which doesn't work } - let mut rng = StdRng::from_entropy(); + let mut rng = crate::test_rng(); let f64_range = Uniform::new(0x0000_0000_0000_0001u64, 0x7ff0_0000_0000_0000); iterate("f64_random_equivalence_test", k, n, f, g, |_| { let x = f64::from_bits(f64_range.sample(&mut rng)); @@ -118,8 +116,8 @@ where pub fn f32_exhaustive_equivalence_test(f: F, g: G, k: usize) where - F: FnMut(&Decoded, &mut [u8]) -> Option<(usize, i16)>, - G: FnMut(&Decoded, &mut [u8]) -> (usize, i16), + F: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit]) -> Option<(&'a [u8], i16)>, + G: for<'a> FnMut(&Decoded, &'a mut [MaybeUninit]) -> (&'a [u8], i16), { // we have only 2^23 * (2^8 - 1) - 1 = 2,139,095,039 positive finite f32 values, // so why not simply testing all of them? @@ -139,13 +137,11 @@ where #[test] fn shortest_random_equivalence_test() { use core::num::flt2dec::strategy::dragon::format_shortest as fallback; - #[cfg(not(miri))] // Miri is too slow - const N: usize = 10_000; - #[cfg(miri)] - const N: usize = 10; + // Miri is too slow + let n = if cfg!(miri) { 10 } else { 10_000 }; - f64_random_equivalence_test(format_shortest_opt, fallback, MAX_SIG_DIGITS, N); - f32_random_equivalence_test(format_shortest_opt, fallback, MAX_SIG_DIGITS, N); + f64_random_equivalence_test(format_shortest_opt, fallback, MAX_SIG_DIGITS, n); + f32_random_equivalence_test(format_shortest_opt, fallback, MAX_SIG_DIGITS, n); } #[test] @@ -174,17 +170,15 @@ fn shortest_f64_hard_random_equivalence_test() { #[test] fn exact_f32_random_equivalence_test() { use core::num::flt2dec::strategy::dragon::format_exact as fallback; - #[cfg(not(miri))] // Miri is too slow - const N: usize = 1_000; - #[cfg(miri)] - const N: usize = 3; + // Miri is too slow + let n = if cfg!(miri) { 3 } else { 1_000 }; for k in 1..21 { f32_random_equivalence_test( |d, buf| format_exact_opt(d, buf, i16::MIN), |d, buf| fallback(d, buf, i16::MIN), k, - N, + n, ); } } @@ -192,17 +186,15 @@ fn exact_f32_random_equivalence_test() { #[test] fn exact_f64_random_equivalence_test() { use core::num::flt2dec::strategy::dragon::format_exact as fallback; - #[cfg(not(miri))] // Miri is too slow - const N: usize = 1_000; - #[cfg(miri)] - const N: usize = 3; + // Miri is too slow + let n = if cfg!(miri) { 2 } else { 1_000 }; for k in 1..21 { f64_random_equivalence_test( |d, buf| format_exact_opt(d, buf, i16::MIN), |d, buf| fallback(d, buf, i16::MIN), k, - N, + n, ); } } diff --git a/crux-mir/lib/core/tests/num/flt2dec/strategy/dragon.rs b/crux-mir/lib/core/tests/num/flt2dec/strategy/dragon.rs index 3d985c679..fc2e724a2 100644 --- a/crux-mir/lib/core/tests/num/flt2dec/strategy/dragon.rs +++ b/crux-mir/lib/core/tests/num/flt2dec/strategy/dragon.rs @@ -13,7 +13,6 @@ fn test_mul_pow10() { } } -#[cfg_attr(all(target_arch = "wasm32", target_os = "emscripten"), ignore)] // issue 42630 #[test] fn shortest_sanity_test() { f64_shortest_sanity_test(format_shortest); diff --git a/crux-mir/lib/core/tests/num/flt2dec/strategy/grisu.rs b/crux-mir/lib/core/tests/num/flt2dec/strategy/grisu.rs index ff8373c64..b59a3b9b7 100644 --- a/crux-mir/lib/core/tests/num/flt2dec/strategy/grisu.rs +++ b/crux-mir/lib/core/tests/num/flt2dec/strategy/grisu.rs @@ -2,6 +2,7 @@ use super::super::*; use core::num::flt2dec::strategy::grisu::*; #[test] +#[cfg_attr(miri, ignore)] // Miri is too slow fn test_cached_power() { assert_eq!(CACHED_POW10.first().unwrap().1, CACHED_POW10_FIRST_E); assert_eq!(CACHED_POW10.last().unwrap().1, CACHED_POW10_LAST_E); @@ -32,7 +33,6 @@ fn test_max_pow10_no_more_than() { } } -#[cfg_attr(all(target_arch = "wasm32", target_os = "emscripten"), ignore)] // issue 42630 #[test] fn shortest_sanity_test() { f64_shortest_sanity_test(format_shortest); diff --git a/crux-mir/lib/core/tests/num/i128.rs b/crux-mir/lib/core/tests/num/i128.rs new file mode 100644 index 000000000..1ddd20f33 --- /dev/null +++ b/crux-mir/lib/core/tests/num/i128.rs @@ -0,0 +1 @@ +int_module!(i128); diff --git a/crux-mir/lib/core/tests/num/i16.rs b/crux-mir/lib/core/tests/num/i16.rs index f5544b914..c7aa9fff9 100644 --- a/crux-mir/lib/core/tests/num/i16.rs +++ b/crux-mir/lib/core/tests/num/i16.rs @@ -1 +1 @@ -int_module!(i16, i16); +int_module!(i16); diff --git a/crux-mir/lib/core/tests/num/i32.rs b/crux-mir/lib/core/tests/num/i32.rs index 39250ee84..efd5b1596 100644 --- a/crux-mir/lib/core/tests/num/i32.rs +++ b/crux-mir/lib/core/tests/num/i32.rs @@ -1 +1,30 @@ -int_module!(i32, i32); +int_module!(i32); + +#[test] +fn test_arith_operation() { + let a: isize = 10; + assert_eq!(a * (a - 1), 90); + let i32_a: isize = 10; + assert_eq!(i32_a, 10); + assert_eq!(i32_a - 10, 0); + assert_eq!(i32_a / 10, 1); + assert_eq!(i32_a - 20, -10); + assert_eq!(i32_a << 10, 10240); + assert_eq!(i32_a << 16, 655360); + assert_eq!(i32_a * 16, 160); + assert_eq!(i32_a * i32_a * i32_a, 1000); + assert_eq!(i32_a * i32_a * i32_a * i32_a, 10000); + assert_eq!(i32_a * i32_a / i32_a * i32_a, 100); + assert_eq!(i32_a * (i32_a - 1) << (2 + i32_a as usize), 368640); + let i32_b: isize = 0x10101010; + assert_eq!(i32_b + 1 - 1, i32_b); + assert_eq!(i32_b << 1, i32_b << 1); + assert_eq!(i32_b >> 1, i32_b >> 1); + assert_eq!(i32_b & i32_b << 1, 0); + assert_eq!(i32_b | i32_b << 1, 0x30303030); + let i32_c: isize = 0x10101010; + assert_eq!( + i32_c + i32_c * 2 / 3 * 2 + (i32_c - 7 % 3), + i32_c + i32_c * 2 / 3 * 2 + (i32_c - 7 % 3) + ); +} diff --git a/crux-mir/lib/core/tests/num/i64.rs b/crux-mir/lib/core/tests/num/i64.rs index fa4d2ab66..93d23c10a 100644 --- a/crux-mir/lib/core/tests/num/i64.rs +++ b/crux-mir/lib/core/tests/num/i64.rs @@ -1 +1 @@ -int_module!(i64, i64); +int_module!(i64); diff --git a/crux-mir/lib/core/tests/num/i8.rs b/crux-mir/lib/core/tests/num/i8.rs index ccec6915f..887d4f17d 100644 --- a/crux-mir/lib/core/tests/num/i8.rs +++ b/crux-mir/lib/core/tests/num/i8.rs @@ -1 +1 @@ -int_module!(i8, i8); +int_module!(i8); diff --git a/crux-mir/lib/core/tests/num/ieee754.rs b/crux-mir/lib/core/tests/num/ieee754.rs new file mode 100644 index 000000000..f6e5dfc98 --- /dev/null +++ b/crux-mir/lib/core/tests/num/ieee754.rs @@ -0,0 +1,158 @@ +//! IEEE 754 floating point compliance tests +//! +//! To understand IEEE 754's requirements on a programming language, one must understand that the +//! requirements of IEEE 754 rest on the total programming environment, and not entirely on any +//! one component. That means the hardware, language, and even libraries are considered part of +//! conforming floating point support in a programming environment. +//! +//! A programming language's duty, accordingly, is: +//! 1. offer access to the hardware where the hardware offers support +//! 2. provide operations that fulfill the remaining requirements of the standard +//! 3. provide the ability to write additional software that can fulfill those requirements +//! +//! This may be fulfilled in any combination that the language sees fit. However, to claim that +//! a language supports IEEE 754 is to suggest that it has fulfilled requirements 1 and 2, without +//! deferring minimum requirements to libraries. This is because support for IEEE 754 is defined +//! as complete support for at least one specified floating point type as an "arithmetic" and +//! "interchange" format, plus specified type conversions to "external character sequences" and +//! integer types. +//! +//! For our purposes, +//! "interchange format" => f32, f64 +//! "arithmetic format" => f32, f64, and any "soft floats" +//! "external character sequence" => str from any float +//! "integer format" => {i,u}{8,16,32,64,128} +//! +//! None of these tests are against Rust's own implementation. They are only tests against the +//! standard. That is why they accept wildly diverse inputs or may seem to duplicate other tests. +//! Please consider this carefully when adding, removing, or reorganizing these tests. They are +//! here so that it is clear what tests are required by the standard and what can be changed. +use ::core::str::FromStr; + +// IEEE 754 for many tests is applied to specific bit patterns. +// These generally are not applicable to NaN, however. +macro_rules! assert_biteq { + ($lhs:expr, $rhs:expr) => { + assert_eq!($lhs.to_bits(), $rhs.to_bits()) + }; +} + +// ToString uses the default fmt::Display impl without special concerns, and bypasses other parts +// of the formatting infrastructure, which makes it ideal for testing here. +#[allow(unused_macros)] +macro_rules! roundtrip { + ($f:expr => $t:ty) => { + ($f).to_string().parse::<$t>().unwrap() + }; +} + +macro_rules! assert_floats_roundtrip { + ($f:ident) => { + assert_biteq!(f32::$f, roundtrip!(f32::$f => f32)); + assert_biteq!(f64::$f, roundtrip!(f64::$f => f64)); + }; + ($f:expr) => { + assert_biteq!($f as f32, roundtrip!($f => f32)); + assert_biteq!($f as f64, roundtrip!($f => f64)); + } +} + +macro_rules! assert_floats_bitne { + ($lhs:ident, $rhs:ident) => { + assert_ne!(f32::$lhs.to_bits(), f32::$rhs.to_bits()); + assert_ne!(f64::$lhs.to_bits(), f64::$rhs.to_bits()); + }; + ($lhs:expr, $rhs:expr) => { + assert_ne!(f32::to_bits($lhs), f32::to_bits($rhs)); + assert_ne!(f64::to_bits($lhs), f64::to_bits($rhs)); + }; +} + +// We must preserve signs on all numbers. That includes zero. +// -0 and 0 are == normally, so test bit equality. +#[test] +fn preserve_signed_zero() { + assert_floats_roundtrip!(-0.0); + assert_floats_roundtrip!(0.0); + assert_floats_bitne!(0.0, -0.0); +} + +#[test] +fn preserve_signed_infinity() { + assert_floats_roundtrip!(INFINITY); + assert_floats_roundtrip!(NEG_INFINITY); + assert_floats_bitne!(INFINITY, NEG_INFINITY); +} + +#[test] +fn infinity_to_str() { + assert!(match f32::INFINITY.to_string().to_lowercase().as_str() { + "+infinity" | "infinity" => true, + "+inf" | "inf" => true, + _ => false, + }); + assert!( + match f64::INFINITY.to_string().to_lowercase().as_str() { + "+infinity" | "infinity" => true, + "+inf" | "inf" => true, + _ => false, + }, + "Infinity must write to a string as some casing of inf or infinity, with an optional +." + ); +} + +#[test] +fn neg_infinity_to_str() { + assert!(match f32::NEG_INFINITY.to_string().to_lowercase().as_str() { + "-infinity" | "-inf" => true, + _ => false, + }); + assert!( + match f64::NEG_INFINITY.to_string().to_lowercase().as_str() { + "-infinity" | "-inf" => true, + _ => false, + }, + "Negative Infinity must write to a string as some casing of -inf or -infinity" + ) +} + +#[test] +fn nan_to_str() { + assert!( + match f32::NAN.to_string().to_lowercase().as_str() { + "nan" | "+nan" | "-nan" => true, + _ => false, + }, + "NaNs must write to a string as some casing of nan." + ) +} + +// "+"?("inf"|"infinity") in any case => Infinity +#[test] +fn infinity_from_str() { + assert_biteq!(f32::INFINITY, f32::from_str("infinity").unwrap()); + assert_biteq!(f32::INFINITY, f32::from_str("inf").unwrap()); + assert_biteq!(f32::INFINITY, f32::from_str("+infinity").unwrap()); + assert_biteq!(f32::INFINITY, f32::from_str("+inf").unwrap()); + // yes! this means you are weLcOmE tO mY iNfInItElY tWiStEd MiNd + assert_biteq!(f32::INFINITY, f32::from_str("+iNfInItY").unwrap()); +} + +// "-inf"|"-infinity" in any case => Negative Infinity +#[test] +fn neg_infinity_from_str() { + assert_biteq!(f32::NEG_INFINITY, f32::from_str("-infinity").unwrap()); + assert_biteq!(f32::NEG_INFINITY, f32::from_str("-inf").unwrap()); + assert_biteq!(f32::NEG_INFINITY, f32::from_str("-INF").unwrap()); + assert_biteq!(f32::NEG_INFINITY, f32::from_str("-INFinity").unwrap()); +} + +// ("+"|"-"")?"s"?"nan" in any case => qNaN +#[test] +fn qnan_from_str() { + assert!("nan".parse::().unwrap().is_nan()); + assert!("-nan".parse::().unwrap().is_nan()); + assert!("+nan".parse::().unwrap().is_nan()); + assert!("+NAN".parse::().unwrap().is_nan()); + assert!("-NaN".parse::().unwrap().is_nan()); +} diff --git a/crux-mir/lib/core/tests/num/int_log.rs b/crux-mir/lib/core/tests/num/int_log.rs new file mode 100644 index 000000000..a1edb1a51 --- /dev/null +++ b/crux-mir/lib/core/tests/num/int_log.rs @@ -0,0 +1,196 @@ +//! This tests the `Integer::{ilog,log2,log10}` methods. These tests are in a +//! separate file because there's both a large number of them, and not all tests +//! can be run on Android. This is because in Android `ilog2` uses an imprecise +//! approximation:https://github.com/rust-lang/rust/blob/4825e12fc9c79954aa0fe18f5521efa6c19c7539/src/libstd/sys/unix/android.rs#L27-L53 + +#[test] +fn checked_ilog() { + assert_eq!(999u32.checked_ilog(10), Some(2)); + assert_eq!(1000u32.checked_ilog(10), Some(3)); + assert_eq!(555u32.checked_ilog(13), Some(2)); + assert_eq!(63u32.checked_ilog(4), Some(2)); + assert_eq!(64u32.checked_ilog(4), Some(3)); + assert_eq!(10460353203u64.checked_ilog(3), Some(21)); + assert_eq!(10460353202u64.checked_ilog(3), Some(20)); + assert_eq!(147808829414345923316083210206383297601u128.checked_ilog(3), Some(80)); + assert_eq!(147808829414345923316083210206383297600u128.checked_ilog(3), Some(79)); + assert_eq!(22528399544939174411840147874772641u128.checked_ilog(19683), Some(8)); + assert_eq!(22528399544939174411840147874772631i128.checked_ilog(19683), Some(7)); + + assert_eq!(0u8.checked_ilog(4), None); + assert_eq!(0u16.checked_ilog(4), None); + assert_eq!(0i8.checked_ilog(4), None); + assert_eq!(0i16.checked_ilog(4), None); + + #[cfg(not(miri))] // Miri is too slow + for i in i16::MIN..=0 { + assert_eq!(i.checked_ilog(4), None); + } + #[cfg(not(miri))] // Miri is too slow + for i in 1..=i16::MAX { + assert_eq!(i.checked_ilog(13), Some((i as f32).log(13.0) as u32)); + } + #[cfg(not(miri))] // Miri is too slow + for i in 1..=u16::MAX { + assert_eq!(i.checked_ilog(13), Some((i as f32).log(13.0) as u32)); + } +} + +#[test] +fn checked_ilog2() { + assert_eq!(5u32.checked_ilog2(), Some(2)); + assert_eq!(0u64.checked_ilog2(), None); + assert_eq!(128i32.checked_ilog2(), Some(7)); + assert_eq!((-55i16).checked_ilog2(), None); + + assert_eq!(0u8.checked_ilog2(), None); + assert_eq!(0u16.checked_ilog2(), None); + assert_eq!(0i8.checked_ilog2(), None); + assert_eq!(0i16.checked_ilog2(), None); + + for i in 1..=u8::MAX { + assert_eq!(i.checked_ilog2(), Some((i as f32).log2() as u32)); + } + #[cfg(not(miri))] // Miri is too slow + for i in 1..=u16::MAX { + // Guard against Android's imprecise f32::ilog2 implementation. + if i != 8192 && i != 32768 { + assert_eq!(i.checked_ilog2(), Some((i as f32).log2() as u32)); + } + } + for i in i8::MIN..=0 { + assert_eq!(i.checked_ilog2(), None); + } + for i in 1..=i8::MAX { + assert_eq!(i.checked_ilog2(), Some((i as f32).log2() as u32)); + } + #[cfg(not(miri))] // Miri is too slow + for i in i16::MIN..=0 { + assert_eq!(i.checked_ilog2(), None); + } + #[cfg(not(miri))] // Miri is too slow + for i in 1..=i16::MAX { + // Guard against Android's imprecise f32::ilog2 implementation. + if i != 8192 { + assert_eq!(i.checked_ilog2(), Some((i as f32).log2() as u32)); + } + } +} + +// Validate cases that fail on Android's imprecise float ilog2 implementation. +#[test] +#[cfg(not(target_os = "android"))] +fn checked_ilog2_not_android() { + assert_eq!(8192u16.checked_ilog2(), Some((8192f32).log2() as u32)); + assert_eq!(32768u16.checked_ilog2(), Some((32768f32).log2() as u32)); + assert_eq!(8192i16.checked_ilog2(), Some((8192f32).log2() as u32)); +} + +#[test] +fn checked_ilog10() { + assert_eq!(0u8.checked_ilog10(), None); + assert_eq!(0u16.checked_ilog10(), None); + assert_eq!(0i8.checked_ilog10(), None); + assert_eq!(0i16.checked_ilog10(), None); + + #[cfg(not(miri))] // Miri is too slow + for i in i16::MIN..=0 { + assert_eq!(i.checked_ilog10(), None); + } + #[cfg(not(miri))] // Miri is too slow + for i in 1..=i16::MAX { + assert_eq!(i.checked_ilog10(), Some((i as f32).log10() as u32)); + } + #[cfg(not(miri))] // Miri is too slow + for i in 1..=u16::MAX { + assert_eq!(i.checked_ilog10(), Some((i as f32).log10() as u32)); + } + #[cfg(not(miri))] // Miri is too slow + for i in 1..=100_000u32 { + assert_eq!(i.checked_ilog10(), Some((i as f32).log10() as u32)); + } +} + +macro_rules! ilog10_loop { + ($T:ty, $ilog10_max:expr) => { + assert_eq!(<$T>::MAX.ilog10(), $ilog10_max); + for i in 0..=$ilog10_max { + let p = (10 as $T).pow(i as u32); + if p >= 10 { + assert_eq!((p - 9).ilog10(), i - 1); + assert_eq!((p - 1).ilog10(), i - 1); + } + assert_eq!(p.ilog10(), i); + assert_eq!((p + 1).ilog10(), i); + if p >= 10 { + assert_eq!((p + 9).ilog10(), i); + } + + // also check `x.ilog(10)` + if p >= 10 { + assert_eq!((p - 9).ilog(10), i - 1); + assert_eq!((p - 1).ilog(10), i - 1); + } + assert_eq!(p.ilog(10), i); + assert_eq!((p + 1).ilog(10), i); + if p >= 10 { + assert_eq!((p + 9).ilog(10), i); + } + } + }; +} + +#[test] +fn ilog10_u8() { + ilog10_loop! { u8, 2 } +} + +#[test] +fn ilog10_u16() { + ilog10_loop! { u16, 4 } +} + +#[test] +fn ilog10_u32() { + ilog10_loop! { u32, 9 } +} + +#[test] +fn ilog10_u64() { + ilog10_loop! { u64, 19 } +} + +#[test] +fn ilog10_u128() { + ilog10_loop! { u128, 38 } +} + +#[test] +#[should_panic(expected = "argument of integer logarithm must be positive")] +fn ilog2_of_0_panic() { + let _ = 0u32.ilog2(); +} + +#[test] +#[should_panic(expected = "argument of integer logarithm must be positive")] +fn ilog10_of_0_panic() { + let _ = 0u32.ilog10(); +} + +#[test] +#[should_panic(expected = "argument of integer logarithm must be positive")] +fn ilog3_of_0_panic() { + let _ = 0u32.ilog(3); +} + +#[test] +#[should_panic(expected = "base of integer logarithm must be at least 2")] +fn ilog0_of_1_panic() { + let _ = 1u32.ilog(0); +} + +#[test] +#[should_panic(expected = "base of integer logarithm must be at least 2")] +fn ilog1_of_1_panic() { + let _ = 1u32.ilog(1); +} diff --git a/crux-mir/lib/core/tests/num/int_macros.rs b/crux-mir/lib/core/tests/num/int_macros.rs index 48a49073b..18c55e43a 100644 --- a/crux-mir/lib/core/tests/num/int_macros.rs +++ b/crux-mir/lib/core/tests/num/int_macros.rs @@ -1,11 +1,9 @@ macro_rules! int_module { - ($T:ident, $T_i:ident) => { + ($T:ident) => { #[cfg(test)] mod tests { - use core::isize; - use core::mem; use core::ops::{BitAnd, BitOr, BitXor, Not, Shl, Shr}; - use core::$T_i::*; + use core::$T::*; use crate::num; @@ -83,30 +81,27 @@ macro_rules! int_module { #[test] fn test_count_zeros() { - let bits = mem::size_of::<$T>() * 8; - assert_eq!(A.count_zeros(), bits as u32 - 3); - assert_eq!(B.count_zeros(), bits as u32 - 2); - assert_eq!(C.count_zeros(), bits as u32 - 5); + assert_eq!(A.count_zeros(), $T::BITS - 3); + assert_eq!(B.count_zeros(), $T::BITS - 2); + assert_eq!(C.count_zeros(), $T::BITS - 5); } #[test] fn test_leading_trailing_ones() { - let bits = (mem::size_of::<$T>() * 8) as u32; - let a: $T = 0b0101_1111; assert_eq!(a.trailing_ones(), 5); - assert_eq!((!a).leading_ones(), bits - 7); + assert_eq!((!a).leading_ones(), $T::BITS - 7); assert_eq!(a.reverse_bits().leading_ones(), 5); - assert_eq!(_1.leading_ones(), bits); - assert_eq!(_1.trailing_ones(), bits); + assert_eq!(_1.leading_ones(), $T::BITS); + assert_eq!(_1.trailing_ones(), $T::BITS); assert_eq!((_1 << 1).trailing_ones(), 0); assert_eq!(MAX.leading_ones(), 0); - assert_eq!((_1 << 1).leading_ones(), bits - 1); - assert_eq!(MAX.trailing_ones(), bits - 1); + assert_eq!((_1 << 1).leading_ones(), $T::BITS - 1); + assert_eq!(MAX.trailing_ones(), $T::BITS - 1); assert_eq!(_0.leading_ones(), 0); assert_eq!(_0.trailing_ones(), 0); @@ -136,9 +131,9 @@ macro_rules! int_module { assert_eq!(B.rotate_left(0), B); assert_eq!(C.rotate_left(0), C); // Rotating by a multiple of word size should also have no effect - assert_eq!(A.rotate_left(64), A); - assert_eq!(B.rotate_left(64), B); - assert_eq!(C.rotate_left(64), C); + assert_eq!(A.rotate_left(128), A); + assert_eq!(B.rotate_left(128), B); + assert_eq!(C.rotate_left(128), C); } #[test] @@ -209,8 +204,8 @@ macro_rules! int_module { #[test] fn test_from_str() { - fn from_str(t: &str) -> Option { - ::std::str::FromStr::from_str(t).ok() + fn from_str(t: &str) -> Option { + std::str::FromStr::from_str(t).ok() } assert_eq!(from_str::<$T>("0"), Some(0 as $T)); assert_eq!(from_str::<$T>("3"), Some(3 as $T)); @@ -256,12 +251,118 @@ macro_rules! int_module { #[test] fn test_pow() { let mut r = 2 as $T; - assert_eq!(r.pow(2), 4 as $T); assert_eq!(r.pow(0), 1 as $T); + assert_eq!(r.wrapping_pow(2), 4 as $T); + assert_eq!(r.wrapping_pow(0), 1 as $T); + assert_eq!(r.checked_pow(2), Some(4 as $T)); + assert_eq!(r.checked_pow(0), Some(1 as $T)); + assert_eq!(r.overflowing_pow(2), (4 as $T, false)); + assert_eq!(r.overflowing_pow(0), (1 as $T, false)); + assert_eq!(r.saturating_pow(2), 4 as $T); + assert_eq!(r.saturating_pow(0), 1 as $T); + + r = MAX; + // use `^` to represent .pow() with no overflow. + // if itest::MAX == 2^j-1, then itest is a `j` bit int, + // so that `itest::MAX*itest::MAX == 2^(2*j)-2^(j+1)+1`, + // thussaturating_pow the overflowing result is exactly 1. + assert_eq!(r.wrapping_pow(2), 1 as $T); + assert_eq!(r.checked_pow(2), None); + assert_eq!(r.overflowing_pow(2), (1 as $T, true)); + assert_eq!(r.saturating_pow(2), MAX); + //test for negative exponent. r = -2 as $T; assert_eq!(r.pow(2), 4 as $T); assert_eq!(r.pow(3), -8 as $T); + assert_eq!(r.pow(0), 1 as $T); + assert_eq!(r.wrapping_pow(2), 4 as $T); + assert_eq!(r.wrapping_pow(3), -8 as $T); + assert_eq!(r.wrapping_pow(0), 1 as $T); + assert_eq!(r.checked_pow(2), Some(4 as $T)); + assert_eq!(r.checked_pow(3), Some(-8 as $T)); + assert_eq!(r.checked_pow(0), Some(1 as $T)); + assert_eq!(r.overflowing_pow(2), (4 as $T, false)); + assert_eq!(r.overflowing_pow(3), (-8 as $T, false)); + assert_eq!(r.overflowing_pow(0), (1 as $T, false)); + assert_eq!(r.saturating_pow(2), 4 as $T); + assert_eq!(r.saturating_pow(3), -8 as $T); + assert_eq!(r.saturating_pow(0), 1 as $T); + } + + #[test] + fn test_div_floor() { + let a: $T = 8; + let b = 3; + assert_eq!(a.div_floor(b), 2); + assert_eq!(a.div_floor(-b), -3); + assert_eq!((-a).div_floor(b), -3); + assert_eq!((-a).div_floor(-b), 2); + } + + #[test] + fn test_div_ceil() { + let a: $T = 8; + let b = 3; + assert_eq!(a.div_ceil(b), 3); + assert_eq!(a.div_ceil(-b), -2); + assert_eq!((-a).div_ceil(b), -2); + assert_eq!((-a).div_ceil(-b), 3); + } + + #[test] + fn test_next_multiple_of() { + assert_eq!((16 as $T).next_multiple_of(8), 16); + assert_eq!((23 as $T).next_multiple_of(8), 24); + assert_eq!((16 as $T).next_multiple_of(-8), 16); + assert_eq!((23 as $T).next_multiple_of(-8), 16); + assert_eq!((-16 as $T).next_multiple_of(8), -16); + assert_eq!((-23 as $T).next_multiple_of(8), -16); + assert_eq!((-16 as $T).next_multiple_of(-8), -16); + assert_eq!((-23 as $T).next_multiple_of(-8), -24); + assert_eq!(MIN.next_multiple_of(-1), MIN); + } + + #[test] + fn test_checked_next_multiple_of() { + assert_eq!((16 as $T).checked_next_multiple_of(8), Some(16)); + assert_eq!((23 as $T).checked_next_multiple_of(8), Some(24)); + assert_eq!((16 as $T).checked_next_multiple_of(-8), Some(16)); + assert_eq!((23 as $T).checked_next_multiple_of(-8), Some(16)); + assert_eq!((-16 as $T).checked_next_multiple_of(8), Some(-16)); + assert_eq!((-23 as $T).checked_next_multiple_of(8), Some(-16)); + assert_eq!((-16 as $T).checked_next_multiple_of(-8), Some(-16)); + assert_eq!((-23 as $T).checked_next_multiple_of(-8), Some(-24)); + assert_eq!((1 as $T).checked_next_multiple_of(0), None); + assert_eq!(MAX.checked_next_multiple_of(2), None); + assert_eq!(MIN.checked_next_multiple_of(-3), None); + assert_eq!(MIN.checked_next_multiple_of(-1), Some(MIN)); + } + + #[test] + fn test_carrying_add() { + assert_eq!($T::MAX.carrying_add(1, false), ($T::MIN, true)); + assert_eq!($T::MAX.carrying_add(0, true), ($T::MIN, true)); + assert_eq!($T::MAX.carrying_add(1, true), ($T::MIN + 1, true)); + assert_eq!($T::MAX.carrying_add(-1, false), ($T::MAX - 1, false)); + assert_eq!($T::MAX.carrying_add(-1, true), ($T::MAX, false)); // no intermediate overflow + assert_eq!($T::MIN.carrying_add(-1, false), ($T::MAX, true)); + assert_eq!($T::MIN.carrying_add(-1, true), ($T::MIN, false)); // no intermediate overflow + assert_eq!((0 as $T).carrying_add($T::MAX, true), ($T::MIN, true)); + assert_eq!((0 as $T).carrying_add($T::MIN, true), ($T::MIN + 1, false)); + } + + #[test] + fn test_borrowing_sub() { + assert_eq!($T::MIN.borrowing_sub(1, false), ($T::MAX, true)); + assert_eq!($T::MIN.borrowing_sub(0, true), ($T::MAX, true)); + assert_eq!($T::MIN.borrowing_sub(1, true), ($T::MAX - 1, true)); + assert_eq!($T::MIN.borrowing_sub(-1, false), ($T::MIN + 1, false)); + assert_eq!($T::MIN.borrowing_sub(-1, true), ($T::MIN, false)); // no intermediate overflow + assert_eq!($T::MAX.borrowing_sub(-1, false), ($T::MIN, true)); + assert_eq!($T::MAX.borrowing_sub(-1, true), ($T::MAX, false)); // no intermediate overflow + assert_eq!((0 as $T).borrowing_sub($T::MIN, false), ($T::MIN, true)); + assert_eq!((0 as $T).borrowing_sub($T::MIN, true), ($T::MAX, false)); } } }; diff --git a/crux-mir/lib/core/tests/num/mod.rs b/crux-mir/lib/core/tests/num/mod.rs index f61793a3b..c79e909e4 100644 --- a/crux-mir/lib/core/tests/num/mod.rs +++ b/crux-mir/lib/core/tests/num/mod.rs @@ -2,14 +2,16 @@ use core::cmp::PartialEq; use core::convert::{TryFrom, TryInto}; use core::fmt::Debug; use core::marker::Copy; -use core::num::TryFromIntError; +use core::num::{can_not_overflow, IntErrorKind, ParseIntError, TryFromIntError}; use core::ops::{Add, Div, Mul, Rem, Sub}; use core::option::Option; -use core::option::Option::{None, Some}; +use core::option::Option::None; +use core::str::FromStr; #[macro_use] mod int_macros; +mod i128; mod i16; mod i32; mod i64; @@ -18,14 +20,23 @@ mod i8; #[macro_use] mod uint_macros; +mod u128; mod u16; mod u32; mod u64; mod u8; mod bignum; + +mod const_from; mod dec2flt; mod flt2dec; +mod int_log; +mod ops; +mod wrapping; + +mod ieee754; +mod nan; /// Adds the attribute to all items in the block. macro_rules! cfg_block { @@ -65,6 +76,15 @@ where assert_eq!(ten.rem(two), ten % two); } +/// Helper function for asserting number parsing returns a specific error +fn test_parse(num_str: &str, expected: Result) +where + T: FromStr, + Result: PartialEq + Debug, +{ + assert_eq!(num_str.parse::().map_err(|e| e.kind().clone()), expected) +} + #[test] fn from_str_issue7588() { let u: Option = u8::from_str_radix("1000", 10).ok(); @@ -75,49 +95,121 @@ fn from_str_issue7588() { #[test] fn test_int_from_str_overflow() { - assert_eq!("127".parse::().ok(), Some(127i8)); - assert_eq!("128".parse::().ok(), None); + test_parse::("127", Ok(127)); + test_parse::("128", Err(IntErrorKind::PosOverflow)); - assert_eq!("-128".parse::().ok(), Some(-128i8)); - assert_eq!("-129".parse::().ok(), None); + test_parse::("-128", Ok(-128)); + test_parse::("-129", Err(IntErrorKind::NegOverflow)); - assert_eq!("32767".parse::().ok(), Some(32_767i16)); - assert_eq!("32768".parse::().ok(), None); + test_parse::("32767", Ok(32_767)); + test_parse::("32768", Err(IntErrorKind::PosOverflow)); - assert_eq!("-32768".parse::().ok(), Some(-32_768i16)); - assert_eq!("-32769".parse::().ok(), None); + test_parse::("-32768", Ok(-32_768)); + test_parse::("-32769", Err(IntErrorKind::NegOverflow)); - assert_eq!("2147483647".parse::().ok(), Some(2_147_483_647i32)); - assert_eq!("2147483648".parse::().ok(), None); + test_parse::("2147483647", Ok(2_147_483_647)); + test_parse::("2147483648", Err(IntErrorKind::PosOverflow)); - assert_eq!("-2147483648".parse::().ok(), Some(-2_147_483_648i32)); - assert_eq!("-2147483649".parse::().ok(), None); + test_parse::("-2147483648", Ok(-2_147_483_648)); + test_parse::("-2147483649", Err(IntErrorKind::NegOverflow)); - assert_eq!("9223372036854775807".parse::().ok(), Some(9_223_372_036_854_775_807i64)); - assert_eq!("9223372036854775808".parse::().ok(), None); + test_parse::("9223372036854775807", Ok(9_223_372_036_854_775_807)); + test_parse::("9223372036854775808", Err(IntErrorKind::PosOverflow)); - assert_eq!("-9223372036854775808".parse::().ok(), Some(-9_223_372_036_854_775_808i64)); - assert_eq!("-9223372036854775809".parse::().ok(), None); + test_parse::("-9223372036854775808", Ok(-9_223_372_036_854_775_808)); + test_parse::("-9223372036854775809", Err(IntErrorKind::NegOverflow)); +} + +#[test] +fn test_can_not_overflow() { + fn can_overflow(radix: u32, input: &str) -> bool + where + T: std::convert::TryFrom, + { + !can_not_overflow::(radix, T::try_from(-1_i8).is_ok(), input.as_bytes()) + } + + // Positive tests: + assert!(!can_overflow::(16, "F")); + assert!(!can_overflow::(16, "FF")); + + assert!(!can_overflow::(10, "9")); + assert!(!can_overflow::(10, "99")); + + // Negative tests: + + // Not currently in std lib (issue: #27728) + fn format_radix(mut x: T, radix: T) -> String + where + T: std::ops::Rem, + T: std::ops::Div, + T: std::cmp::PartialEq, + T: std::default::Default, + T: Copy, + T: Default, + u32: TryFrom, + { + let mut result = vec![]; + + loop { + let m = x % radix; + x = x / radix; + result.push( + std::char::from_digit(m.try_into().ok().unwrap(), radix.try_into().ok().unwrap()) + .unwrap(), + ); + if x == T::default() { + break; + } + } + result.into_iter().rev().collect() + } + + macro_rules! check { + ($($t:ty)*) => ($( + for base in 2..=36 { + let num = (<$t>::MAX as u128) + 1; + + // Calcutate the string length for the smallest overflowing number: + let max_len_string = format_radix(num, base as u128); + // Ensure that string length is deemed to potentially overflow: + assert!(can_overflow::<$t>(base, &max_len_string)); + } + )*) + } + + check! { i8 i16 i32 i64 i128 isize usize u8 u16 u32 u64 } + + // Check u128 separately: + for base in 2..=36 { + let num = u128::MAX as u128; + let max_len_string = format_radix(num, base as u128); + // base 16 fits perfectly for u128 and won't overflow: + assert_eq!(can_overflow::(base, &max_len_string), base != 16); + } } #[test] fn test_leading_plus() { - assert_eq!("+127".parse::().ok(), Some(127)); - assert_eq!("+9223372036854775807".parse::().ok(), Some(9223372036854775807)); + test_parse::("+127", Ok(127)); + test_parse::("+9223372036854775807", Ok(9223372036854775807)); } #[test] fn test_invalid() { - assert_eq!("--129".parse::().ok(), None); - assert_eq!("++129".parse::().ok(), None); - assert_eq!("Съешь".parse::().ok(), None); + test_parse::("--129", Err(IntErrorKind::InvalidDigit)); + test_parse::("++129", Err(IntErrorKind::InvalidDigit)); + test_parse::("Съешь", Err(IntErrorKind::InvalidDigit)); + test_parse::("123Hello", Err(IntErrorKind::InvalidDigit)); + test_parse::("--", Err(IntErrorKind::InvalidDigit)); + test_parse::("-", Err(IntErrorKind::InvalidDigit)); + test_parse::("+", Err(IntErrorKind::InvalidDigit)); + test_parse::("-1", Err(IntErrorKind::InvalidDigit)); } #[test] fn test_empty() { - assert_eq!("-".parse::().ok(), None); - assert_eq!("+".parse::().ok(), None); - assert_eq!("".parse::().ok(), None); + test_parse::("", Err(IntErrorKind::Empty)); } #[test] @@ -140,8 +232,8 @@ macro_rules! test_impl_from { ($fn_name: ident, $Small: ty, $Large: ty) => { #[test] fn $fn_name() { - let small_max = <$Small>::max_value(); - let small_min = <$Small>::min_value(); + let small_max = <$Small>::MAX; + let small_min = <$Small>::MIN; let large_max: $Large = small_max.into(); let large_min: $Large = small_min.into(); assert_eq!(large_max as $Small, small_max); @@ -205,8 +297,6 @@ test_impl_from! { test_u32f64, u32, f64 } // Float -> Float #[test] fn test_f32f64() { - use core::f32; - let max: f64 = f32::MAX.into(); assert_eq!(max as f32, f32::MAX); assert!(max.is_normal()); @@ -250,8 +340,8 @@ macro_rules! test_impl_try_from_always_ok { ($fn_name:ident, $source:ty, $target: ty) => { #[test] fn $fn_name() { - let max = <$source>::max_value(); - let min = <$source>::min_value(); + let max = <$source>::MAX; + let min = <$source>::MIN; let zero: $source = 0; assert_eq!(<$target as TryFrom<$source>>::try_from(max).unwrap(), max as $target); assert_eq!(<$target as TryFrom<$source>>::try_from(min).unwrap(), min as $target); @@ -363,8 +453,8 @@ macro_rules! test_impl_try_from_signed_to_unsigned_upper_ok { ($fn_name:ident, $source:ty, $target:ty) => { #[test] fn $fn_name() { - let max = <$source>::max_value(); - let min = <$source>::min_value(); + let max = <$source>::MAX; + let min = <$source>::MIN; let zero: $source = 0; let neg_one: $source = -1; assert_eq!(<$target as TryFrom<$source>>::try_from(max).unwrap(), max as $target); @@ -428,8 +518,8 @@ macro_rules! test_impl_try_from_unsigned_to_signed_upper_err { ($fn_name:ident, $source:ty, $target:ty) => { #[test] fn $fn_name() { - let max = <$source>::max_value(); - let min = <$source>::min_value(); + let max = <$source>::MAX; + let min = <$source>::MIN; let zero: $source = 0; assert!(<$target as TryFrom<$source>>::try_from(max).is_err()); assert_eq!(<$target as TryFrom<$source>>::try_from(min).unwrap(), min as $target); @@ -489,11 +579,11 @@ macro_rules! test_impl_try_from_same_sign_err { ($fn_name:ident, $source:ty, $target:ty) => { #[test] fn $fn_name() { - let max = <$source>::max_value(); - let min = <$source>::min_value(); + let max = <$source>::MAX; + let min = <$source>::MIN; let zero: $source = 0; - let t_max = <$target>::max_value(); - let t_min = <$target>::min_value(); + let t_max = <$target>::MAX; + let t_min = <$target>::MIN; assert!(<$target as TryFrom<$source>>::try_from(max).is_err()); if min != 0 { assert!(<$target as TryFrom<$source>>::try_from(min).is_err()); @@ -578,11 +668,11 @@ macro_rules! test_impl_try_from_signed_to_unsigned_err { ($fn_name:ident, $source:ty, $target:ty) => { #[test] fn $fn_name() { - let max = <$source>::max_value(); - let min = <$source>::min_value(); + let max = <$source>::MAX; + let min = <$source>::MIN; let zero: $source = 0; - let t_max = <$target>::max_value(); - let t_min = <$target>::min_value(); + let t_max = <$target>::MAX; + let t_min = <$target>::MIN; assert!(<$target as TryFrom<$source>>::try_from(max).is_err()); assert!(<$target as TryFrom<$source>>::try_from(min).is_err()); assert_eq!(<$target as TryFrom<$source>>::try_from(zero).unwrap(), zero as $target); @@ -636,14 +726,18 @@ assume_usize_width! { macro_rules! test_float { ($modname: ident, $fty: ty, $inf: expr, $neginf: expr, $nan: expr) => { mod $modname { - // FIXME(nagisa): these tests should test for sign of -0.0 #[test] fn min() { assert_eq!((0.0 as $fty).min(0.0), 0.0); + assert!((0.0 as $fty).min(0.0).is_sign_positive()); assert_eq!((-0.0 as $fty).min(-0.0), -0.0); + assert!((-0.0 as $fty).min(-0.0).is_sign_negative()); assert_eq!((9.0 as $fty).min(9.0), 9.0); assert_eq!((-9.0 as $fty).min(0.0), -9.0); assert_eq!((0.0 as $fty).min(9.0), 0.0); + assert!((0.0 as $fty).min(9.0).is_sign_positive()); + assert_eq!((-0.0 as $fty).min(9.0), -0.0); + assert!((-0.0 as $fty).min(9.0).is_sign_negative()); assert_eq!((-0.0 as $fty).min(-9.0), -9.0); assert_eq!(($inf as $fty).min(9.0), 9.0); assert_eq!((9.0 as $fty).min($inf), 9.0); @@ -662,11 +756,19 @@ macro_rules! test_float { #[test] fn max() { assert_eq!((0.0 as $fty).max(0.0), 0.0); + assert!((0.0 as $fty).max(0.0).is_sign_positive()); assert_eq!((-0.0 as $fty).max(-0.0), -0.0); + assert!((-0.0 as $fty).max(-0.0).is_sign_negative()); assert_eq!((9.0 as $fty).max(9.0), 9.0); assert_eq!((-9.0 as $fty).max(0.0), 0.0); + assert!((-9.0 as $fty).max(0.0).is_sign_positive()); + assert_eq!((-9.0 as $fty).max(-0.0), -0.0); + assert!((-9.0 as $fty).max(-0.0).is_sign_negative()); assert_eq!((0.0 as $fty).max(9.0), 9.0); + assert_eq!((0.0 as $fty).max(-9.0), 0.0); + assert!((0.0 as $fty).max(-9.0).is_sign_positive()); assert_eq!((-0.0 as $fty).max(-9.0), -0.0); + assert!((-0.0 as $fty).max(-9.0).is_sign_negative()); assert_eq!(($inf as $fty).max(9.0), $inf); assert_eq!((9.0 as $fty).max($inf), $inf); assert_eq!(($inf as $fty).max(-9.0), $inf); @@ -682,6 +784,67 @@ macro_rules! test_float { assert!(($nan as $fty).max($nan).is_nan()); } #[test] + fn minimum() { + assert_eq!((0.0 as $fty).minimum(0.0), 0.0); + assert!((0.0 as $fty).minimum(0.0).is_sign_positive()); + assert_eq!((-0.0 as $fty).minimum(0.0), -0.0); + assert!((-0.0 as $fty).minimum(0.0).is_sign_negative()); + assert_eq!((-0.0 as $fty).minimum(-0.0), -0.0); + assert!((-0.0 as $fty).minimum(-0.0).is_sign_negative()); + assert_eq!((9.0 as $fty).minimum(9.0), 9.0); + assert_eq!((-9.0 as $fty).minimum(0.0), -9.0); + assert_eq!((0.0 as $fty).minimum(9.0), 0.0); + assert!((0.0 as $fty).minimum(9.0).is_sign_positive()); + assert_eq!((-0.0 as $fty).minimum(9.0), -0.0); + assert!((-0.0 as $fty).minimum(9.0).is_sign_negative()); + assert_eq!((-0.0 as $fty).minimum(-9.0), -9.0); + assert_eq!(($inf as $fty).minimum(9.0), 9.0); + assert_eq!((9.0 as $fty).minimum($inf), 9.0); + assert_eq!(($inf as $fty).minimum(-9.0), -9.0); + assert_eq!((-9.0 as $fty).minimum($inf), -9.0); + assert_eq!(($neginf as $fty).minimum(9.0), $neginf); + assert_eq!((9.0 as $fty).minimum($neginf), $neginf); + assert_eq!(($neginf as $fty).minimum(-9.0), $neginf); + assert_eq!((-9.0 as $fty).minimum($neginf), $neginf); + assert!(($nan as $fty).minimum(9.0).is_nan()); + assert!(($nan as $fty).minimum(-9.0).is_nan()); + assert!((9.0 as $fty).minimum($nan).is_nan()); + assert!((-9.0 as $fty).minimum($nan).is_nan()); + assert!(($nan as $fty).minimum($nan).is_nan()); + } + #[test] + fn maximum() { + assert_eq!((0.0 as $fty).maximum(0.0), 0.0); + assert!((0.0 as $fty).maximum(0.0).is_sign_positive()); + assert_eq!((-0.0 as $fty).maximum(0.0), 0.0); + assert!((-0.0 as $fty).maximum(0.0).is_sign_positive()); + assert_eq!((-0.0 as $fty).maximum(-0.0), -0.0); + assert!((-0.0 as $fty).maximum(-0.0).is_sign_negative()); + assert_eq!((9.0 as $fty).maximum(9.0), 9.0); + assert_eq!((-9.0 as $fty).maximum(0.0), 0.0); + assert!((-9.0 as $fty).maximum(0.0).is_sign_positive()); + assert_eq!((-9.0 as $fty).maximum(-0.0), -0.0); + assert!((-9.0 as $fty).maximum(-0.0).is_sign_negative()); + assert_eq!((0.0 as $fty).maximum(9.0), 9.0); + assert_eq!((0.0 as $fty).maximum(-9.0), 0.0); + assert!((0.0 as $fty).maximum(-9.0).is_sign_positive()); + assert_eq!((-0.0 as $fty).maximum(-9.0), -0.0); + assert!((-0.0 as $fty).maximum(-9.0).is_sign_negative()); + assert_eq!(($inf as $fty).maximum(9.0), $inf); + assert_eq!((9.0 as $fty).maximum($inf), $inf); + assert_eq!(($inf as $fty).maximum(-9.0), $inf); + assert_eq!((-9.0 as $fty).maximum($inf), $inf); + assert_eq!(($neginf as $fty).maximum(9.0), 9.0); + assert_eq!((9.0 as $fty).maximum($neginf), 9.0); + assert_eq!(($neginf as $fty).maximum(-9.0), -9.0); + assert_eq!((-9.0 as $fty).maximum($neginf), -9.0); + assert!(($nan as $fty).maximum(9.0).is_nan()); + assert!(($nan as $fty).maximum(-9.0).is_nan()); + assert!((9.0 as $fty).maximum($nan).is_nan()); + assert!((-9.0 as $fty).maximum($nan).is_nan()); + assert!(($nan as $fty).maximum($nan).is_nan()); + } + #[test] fn rem_euclid() { let a: $fty = 42.0; assert!($inf.rem_euclid(a).is_nan()); @@ -704,5 +867,5 @@ macro_rules! test_float { }; } -test_float!(f32, f32, ::core::f32::INFINITY, ::core::f32::NEG_INFINITY, ::core::f32::NAN); -test_float!(f64, f64, ::core::f64::INFINITY, ::core::f64::NEG_INFINITY, ::core::f64::NAN); +test_float!(f32, f32, f32::INFINITY, f32::NEG_INFINITY, f32::NAN); +test_float!(f64, f64, f64::INFINITY, f64::NEG_INFINITY, f64::NAN); diff --git a/crux-mir/lib/core/tests/num/nan.rs b/crux-mir/lib/core/tests/num/nan.rs new file mode 100644 index 000000000..ef81988c9 --- /dev/null +++ b/crux-mir/lib/core/tests/num/nan.rs @@ -0,0 +1,7 @@ +#[test] +fn test_nan() { + let x = "NaN".to_string(); + assert_eq!(format!("{}", f64::NAN), x); + assert_eq!(format!("{:e}", f64::NAN), x); + assert_eq!(format!("{:E}", f64::NAN), x); +} diff --git a/crux-mir/lib/core/tests/num/ops.rs b/crux-mir/lib/core/tests/num/ops.rs new file mode 100644 index 000000000..ae8b93825 --- /dev/null +++ b/crux-mir/lib/core/tests/num/ops.rs @@ -0,0 +1,232 @@ +use core::ops::*; + +// For types L and R, checks that a trait implementation exists for +// * binary ops: L op R, L op &R, &L op R and &L op &R +// * assign ops: &mut L op R, &mut L op &R +macro_rules! impl_defined { + ($op:ident, $method:ident($lhs:literal, $rhs:literal), $result:literal, $lt:ty, $rt:ty) => { + let lhs = $lhs as $lt; + let rhs = $rhs as $rt; + assert_eq!($result as $lt, $op::$method(lhs, rhs)); + assert_eq!($result as $lt, $op::$method(lhs, &rhs)); + assert_eq!($result as $lt, $op::$method(&lhs, rhs)); + assert_eq!($result as $lt, $op::$method(&lhs, &rhs)); + }; + ($op:ident, $method:ident(&mut $lhs:literal, $rhs:literal), $result:literal, $lt:ty, $rt:ty) => { + let rhs = $rhs as $rt; + let mut lhs = $lhs as $lt; + $op::$method(&mut lhs, rhs); + assert_eq!($result as $lt, lhs); + + let mut lhs = $lhs as $lt; + $op::$method(&mut lhs, &rhs); + assert_eq!($result as $lt, lhs); + }; +} + +// For all specified types T, checks that a trait implementation exists for +// * binary ops: T op T, T op &T, &T op T and &T op &T +// * assign ops: &mut T op T, &mut T op &T +// * unary ops: op T and op &T +macro_rules! impls_defined { + ($op:ident, $method:ident($lhs:literal, $rhs:literal), $result:literal, $($t:ty),+) => {$( + impl_defined!($op, $method($lhs, $rhs), $result, $t, $t); + )+}; + ($op:ident, $method:ident(&mut $lhs:literal, $rhs:literal), $result:literal, $($t:ty),+) => {$( + impl_defined!($op, $method(&mut $lhs, $rhs), $result, $t, $t); + )+}; + ($op:ident, $method:ident($operand:literal), $result:literal, $($t:ty),+) => {$( + let operand = $operand as $t; + assert_eq!($result as $t, $op::$method(operand)); + assert_eq!($result as $t, $op::$method(&operand)); + )+}; +} + +macro_rules! test_op { + ($fn_name:ident, $op:ident::$method:ident($lhs:literal), $result:literal, $($t:ty),+) => { + #[test] + fn $fn_name() { + impls_defined!($op, $method($lhs), $result, $($t),+); + } + }; +} + +test_op!(test_neg_defined, Neg::neg(0), 0, i8, i16, i32, i64, f32, f64); +#[cfg(not(target_os = "emscripten"))] +test_op!(test_neg_defined_128, Neg::neg(0), 0, i128); + +test_op!(test_not_defined_bool, Not::not(true), false, bool); + +macro_rules! test_arith_op { + ($fn_name:ident, $op:ident::$method:ident($lhs:literal, $rhs:literal)) => { + #[test] + fn $fn_name() { + impls_defined!( + $op, + $method($lhs, $rhs), + 0, + i8, + i16, + i32, + i64, + isize, + u8, + u16, + u32, + u64, + usize, + f32, + f64 + ); + #[cfg(not(target_os = "emscripten"))] + impls_defined!($op, $method($lhs, $rhs), 0, i128, u128); + } + }; + ($fn_name:ident, $op:ident::$method:ident(&mut $lhs:literal, $rhs:literal)) => { + #[test] + fn $fn_name() { + impls_defined!( + $op, + $method(&mut $lhs, $rhs), + 0, + i8, + i16, + i32, + i64, + isize, + u8, + u16, + u32, + u64, + usize, + f32, + f64 + ); + #[cfg(not(target_os = "emscripten"))] + impls_defined!($op, $method(&mut $lhs, $rhs), 0, i128, u128); + } + }; +} + +test_arith_op!(test_add_defined, Add::add(0, 0)); +test_arith_op!(test_add_assign_defined, AddAssign::add_assign(&mut 0, 0)); +test_arith_op!(test_sub_defined, Sub::sub(0, 0)); +test_arith_op!(test_sub_assign_defined, SubAssign::sub_assign(&mut 0, 0)); +test_arith_op!(test_mul_defined, Mul::mul(0, 0)); +test_arith_op!(test_mul_assign_defined, MulAssign::mul_assign(&mut 0, 0)); +test_arith_op!(test_div_defined, Div::div(0, 1)); +test_arith_op!(test_div_assign_defined, DivAssign::div_assign(&mut 0, 1)); +test_arith_op!(test_rem_defined, Rem::rem(0, 1)); +test_arith_op!(test_rem_assign_defined, RemAssign::rem_assign(&mut 0, 1)); + +macro_rules! test_bitop { + ($test_name:ident, $op:ident::$method:ident) => { + #[test] + fn $test_name() { + impls_defined!( + $op, + $method(0, 0), + 0, + i8, + i16, + i32, + i64, + isize, + u8, + u16, + u32, + u64, + usize + ); + #[cfg(not(target_os = "emscripten"))] + impls_defined!($op, $method(0, 0), 0, i128, u128); + impls_defined!($op, $method(false, false), false, bool); + } + }; +} +macro_rules! test_bitop_assign { + ($test_name:ident, $op:ident::$method:ident) => { + #[test] + fn $test_name() { + impls_defined!( + $op, + $method(&mut 0, 0), + 0, + i8, + i16, + i32, + i64, + isize, + u8, + u16, + u32, + u64, + usize + ); + #[cfg(not(target_os = "emscripten"))] + impls_defined!($op, $method(&mut 0, 0), 0, i128, u128); + impls_defined!($op, $method(&mut false, false), false, bool); + } + }; +} + +test_bitop!(test_bitand_defined, BitAnd::bitand); +test_bitop_assign!(test_bitand_assign_defined, BitAndAssign::bitand_assign); +test_bitop!(test_bitor_defined, BitOr::bitor); +test_bitop_assign!(test_bitor_assign_defined, BitOrAssign::bitor_assign); +test_bitop!(test_bitxor_defined, BitXor::bitxor); +test_bitop_assign!(test_bitxor_assign_defined, BitXorAssign::bitxor_assign); + +macro_rules! test_shift_inner { + ($op:ident::$method:ident, $lt:ty, $($rt:ty),+) => { + $(impl_defined!($op, $method(0,0), 0, $lt, $rt);)+ + }; + ($op:ident::$method:ident, $lt:ty) => { + test_shift_inner!($op::$method, $lt, i8, i16, i32, i64, isize, u8, u16, u32, u64, usize); + #[cfg(not(target_os = "emscripten"))] + test_shift_inner!($op::$method, $lt, i128, u128); + }; +} + +macro_rules! test_shift { + ($op:ident::$method:ident, $($lt:ty),+) => { + $(test_shift_inner!($op::$method, $lt);)+ + }; + ($test_name:ident, $op:ident::$method:ident) => { + #[test] + fn $test_name() { + test_shift!($op::$method, i8, i16, i32, i64, isize, u8, u16, u32, u64, usize); + #[cfg(not(target_os = "emscripten"))] + test_shift!($op::$method, i128, u128); + } + }; +} + +macro_rules! test_shift_assign_inner { + ($op:ident::$method:ident, $lt:ty, $($rt:ty),+) => { + $(impl_defined!($op, $method(&mut 0,0), 0, $lt, $rt);)+ + }; + ($op:ident::$method:ident, $lt:ty) => { + test_shift_assign_inner!($op::$method, $lt, i8, i16, i32, i64, isize, u8, u16, u32, u64, usize); + #[cfg(not(target_os = "emscripten"))] + test_shift_assign_inner!($op::$method, $lt, i128, u128); + }; +} + +macro_rules! test_shift_assign { + ($op:ident::$method:ident, $($lt:ty),+) => { + $(test_shift_assign_inner!($op::$method, $lt);)+ + }; + ($test_name:ident, $op:ident::$method:ident) => { + #[test] + fn $test_name() { + test_shift_assign!($op::$method, i8, i16, i32, i64, isize, u8, u16, u32, u64, usize); + #[cfg(not(target_os = "emscripten"))] + test_shift_assign!($op::$method, i128, u128); + } + }; +} +test_shift!(test_shl_defined, Shl::shl); +test_shift_assign!(test_shl_assign_defined, ShlAssign::shl_assign); +test_shift!(test_shr_defined, Shr::shr); +test_shift_assign!(test_shr_assign_defined, ShrAssign::shr_assign); diff --git a/crux-mir/lib/core/tests/num/u128.rs b/crux-mir/lib/core/tests/num/u128.rs new file mode 100644 index 000000000..a7b0f9eff --- /dev/null +++ b/crux-mir/lib/core/tests/num/u128.rs @@ -0,0 +1 @@ +uint_module!(u128); diff --git a/crux-mir/lib/core/tests/num/u16.rs b/crux-mir/lib/core/tests/num/u16.rs index 435b91422..010596a34 100644 --- a/crux-mir/lib/core/tests/num/u16.rs +++ b/crux-mir/lib/core/tests/num/u16.rs @@ -1 +1 @@ -uint_module!(u16, u16); +uint_module!(u16); diff --git a/crux-mir/lib/core/tests/num/u32.rs b/crux-mir/lib/core/tests/num/u32.rs index 71dc005de..687d3bbaa 100644 --- a/crux-mir/lib/core/tests/num/u32.rs +++ b/crux-mir/lib/core/tests/num/u32.rs @@ -1 +1 @@ -uint_module!(u32, u32); +uint_module!(u32); diff --git a/crux-mir/lib/core/tests/num/u64.rs b/crux-mir/lib/core/tests/num/u64.rs index b498ebc52..ee55071e9 100644 --- a/crux-mir/lib/core/tests/num/u64.rs +++ b/crux-mir/lib/core/tests/num/u64.rs @@ -1 +1 @@ -uint_module!(u64, u64); +uint_module!(u64); diff --git a/crux-mir/lib/core/tests/num/u8.rs b/crux-mir/lib/core/tests/num/u8.rs index 68e938be7..12b038ce0 100644 --- a/crux-mir/lib/core/tests/num/u8.rs +++ b/crux-mir/lib/core/tests/num/u8.rs @@ -1 +1 @@ -uint_module!(u8, u8); +uint_module!(u8); diff --git a/crux-mir/lib/core/tests/num/uint_macros.rs b/crux-mir/lib/core/tests/num/uint_macros.rs index 8f1ca8e6f..15ae9f232 100644 --- a/crux-mir/lib/core/tests/num/uint_macros.rs +++ b/crux-mir/lib/core/tests/num/uint_macros.rs @@ -1,10 +1,9 @@ macro_rules! uint_module { - ($T:ident, $T_i:ident) => { + ($T:ident) => { #[cfg(test)] mod tests { use core::ops::{BitAnd, BitOr, BitXor, Not, Shl, Shr}; - use core::$T_i::*; - use std::mem; + use core::$T::*; use std::str::FromStr; use crate::num; @@ -47,30 +46,27 @@ macro_rules! uint_module { #[test] fn test_count_zeros() { - let bits = mem::size_of::<$T>() * 8; - assert!(A.count_zeros() == bits as u32 - 3); - assert!(B.count_zeros() == bits as u32 - 2); - assert!(C.count_zeros() == bits as u32 - 5); + assert!(A.count_zeros() == $T::BITS - 3); + assert!(B.count_zeros() == $T::BITS - 2); + assert!(C.count_zeros() == $T::BITS - 5); } #[test] fn test_leading_trailing_ones() { - let bits = (mem::size_of::<$T>() * 8) as u32; - let a: $T = 0b0101_1111; assert_eq!(a.trailing_ones(), 5); - assert_eq!((!a).leading_ones(), bits - 7); + assert_eq!((!a).leading_ones(), $T::BITS - 7); assert_eq!(a.reverse_bits().leading_ones(), 5); - assert_eq!(_1.leading_ones(), bits); - assert_eq!(_1.trailing_ones(), bits); + assert_eq!(_1.leading_ones(), $T::BITS); + assert_eq!(_1.trailing_ones(), $T::BITS); assert_eq!((_1 << 1).trailing_ones(), 0); assert_eq!((_1 >> 1).leading_ones(), 0); - assert_eq!((_1 << 1).leading_ones(), bits - 1); - assert_eq!((_1 >> 1).trailing_ones(), bits - 1); + assert_eq!((_1 << 1).leading_ones(), $T::BITS - 1); + assert_eq!((_1 >> 1).trailing_ones(), $T::BITS - 1); assert_eq!(_0.leading_ones(), 0); assert_eq!(_0.trailing_ones(), 0); @@ -100,9 +96,9 @@ macro_rules! uint_module { assert_eq!(B.rotate_left(0), B); assert_eq!(C.rotate_left(0), C); // Rotating by a multiple of word size should also have no effect - assert_eq!(A.rotate_left(64), A); - assert_eq!(B.rotate_left(64), B); - assert_eq!(C.rotate_left(64), C); + assert_eq!(A.rotate_left(128), A); + assert_eq!(B.rotate_left(128), B); + assert_eq!(C.rotate_left(128), C); } #[test] @@ -184,6 +180,78 @@ macro_rules! uint_module { assert_eq!($T::from_str_radix("Z", 10).ok(), None::<$T>); assert_eq!($T::from_str_radix("_", 2).ok(), None::<$T>); } + + #[test] + fn test_pow() { + let mut r = 2 as $T; + assert_eq!(r.pow(2), 4 as $T); + assert_eq!(r.pow(0), 1 as $T); + assert_eq!(r.wrapping_pow(2), 4 as $T); + assert_eq!(r.wrapping_pow(0), 1 as $T); + assert_eq!(r.checked_pow(2), Some(4 as $T)); + assert_eq!(r.checked_pow(0), Some(1 as $T)); + assert_eq!(r.overflowing_pow(2), (4 as $T, false)); + assert_eq!(r.overflowing_pow(0), (1 as $T, false)); + assert_eq!(r.saturating_pow(2), 4 as $T); + assert_eq!(r.saturating_pow(0), 1 as $T); + + r = MAX; + // use `^` to represent .pow() with no overflow. + // if itest::MAX == 2^j-1, then itest is a `j` bit int, + // so that `itest::MAX*itest::MAX == 2^(2*j)-2^(j+1)+1`, + // thussaturating_pow the overflowing result is exactly 1. + assert_eq!(r.wrapping_pow(2), 1 as $T); + assert_eq!(r.checked_pow(2), None); + assert_eq!(r.overflowing_pow(2), (1 as $T, true)); + assert_eq!(r.saturating_pow(2), MAX); + } + + #[test] + fn test_div_floor() { + assert_eq!((8 as $T).div_floor(3), 2); + } + + #[test] + fn test_div_ceil() { + assert_eq!((8 as $T).div_ceil(3), 3); + } + + #[test] + fn test_next_multiple_of() { + assert_eq!((16 as $T).next_multiple_of(8), 16); + assert_eq!((23 as $T).next_multiple_of(8), 24); + assert_eq!(MAX.next_multiple_of(1), MAX); + } + + #[test] + fn test_checked_next_multiple_of() { + assert_eq!((16 as $T).checked_next_multiple_of(8), Some(16)); + assert_eq!((23 as $T).checked_next_multiple_of(8), Some(24)); + assert_eq!((1 as $T).checked_next_multiple_of(0), None); + assert_eq!(MAX.checked_next_multiple_of(2), None); + } + + #[test] + fn test_carrying_add() { + assert_eq!($T::MAX.carrying_add(1, false), (0, true)); + assert_eq!($T::MAX.carrying_add(0, true), (0, true)); + assert_eq!($T::MAX.carrying_add(1, true), (1, true)); + + assert_eq!($T::MIN.carrying_add($T::MAX, false), ($T::MAX, false)); + assert_eq!($T::MIN.carrying_add(0, true), (1, false)); + assert_eq!($T::MIN.carrying_add($T::MAX, true), (0, true)); + } + + #[test] + fn test_borrowing_sub() { + assert_eq!($T::MIN.borrowing_sub(1, false), ($T::MAX, true)); + assert_eq!($T::MIN.borrowing_sub(0, true), ($T::MAX, true)); + assert_eq!($T::MIN.borrowing_sub(1, true), ($T::MAX - 1, true)); + + assert_eq!($T::MAX.borrowing_sub($T::MAX, false), (0, false)); + assert_eq!($T::MAX.borrowing_sub(0, true), ($T::MAX - 1, false)); + assert_eq!($T::MAX.borrowing_sub($T::MAX, true), ($T::MAX, true)); + } } }; } diff --git a/crux-mir/lib/core/tests/num/wrapping.rs b/crux-mir/lib/core/tests/num/wrapping.rs new file mode 100644 index 000000000..c5a719883 --- /dev/null +++ b/crux-mir/lib/core/tests/num/wrapping.rs @@ -0,0 +1,318 @@ +use core::num::Wrapping; + +macro_rules! wrapping_operation { + ($result:expr, $lhs:ident $op:tt $rhs:expr) => { + assert_eq!($result, $lhs $op $rhs); + assert_eq!($result, &$lhs $op $rhs); + assert_eq!($result, $lhs $op &$rhs); + assert_eq!($result, &$lhs $op &$rhs); + }; + ($result:expr, $op:tt $expr:expr) => { + assert_eq!($result, $op $expr); + assert_eq!($result, $op &$expr); + }; +} + +macro_rules! wrapping_assignment { + ($result:expr, $lhs:ident $op:tt $rhs:expr) => { + let mut lhs1 = $lhs; + lhs1 $op $rhs; + assert_eq!($result, lhs1); + + let mut lhs2 = $lhs; + lhs2 $op &$rhs; + assert_eq!($result, lhs2); + }; +} + +macro_rules! wrapping_test { + ($fn_name:ident, $type:ty, $min:expr, $max:expr) => { + #[test] + fn $fn_name() { + let zero: Wrapping<$type> = Wrapping(0); + let one: Wrapping<$type> = Wrapping(1); + let min: Wrapping<$type> = Wrapping($min); + let max: Wrapping<$type> = Wrapping($max); + + wrapping_operation!(min, max + one); + wrapping_assignment!(min, max += one); + wrapping_operation!(max, min - one); + wrapping_assignment!(max, min -= one); + wrapping_operation!(max, max * one); + wrapping_assignment!(max, max *= one); + wrapping_operation!(max, max / one); + wrapping_assignment!(max, max /= one); + wrapping_operation!(zero, max % one); + wrapping_assignment!(zero, max %= one); + wrapping_operation!(zero, zero & max); + wrapping_assignment!(zero, zero &= max); + wrapping_operation!(max, zero | max); + wrapping_assignment!(max, zero |= max); + wrapping_operation!(zero, max ^ max); + wrapping_assignment!(zero, max ^= max); + wrapping_operation!(zero, zero << 1usize); + wrapping_assignment!(zero, zero <<= 1usize); + wrapping_operation!(zero, zero >> 1usize); + wrapping_assignment!(zero, zero >>= 1usize); + wrapping_operation!(zero, -zero); + wrapping_operation!(max, !min); + } + }; +} + +wrapping_test!(test_wrapping_i8, i8, i8::MIN, i8::MAX); +wrapping_test!(test_wrapping_i16, i16, i16::MIN, i16::MAX); +wrapping_test!(test_wrapping_i32, i32, i32::MIN, i32::MAX); +wrapping_test!(test_wrapping_i64, i64, i64::MIN, i64::MAX); +#[cfg(not(target_os = "emscripten"))] +wrapping_test!(test_wrapping_i128, i128, i128::MIN, i128::MAX); +wrapping_test!(test_wrapping_isize, isize, isize::MIN, isize::MAX); +wrapping_test!(test_wrapping_u8, u8, u8::MIN, u8::MAX); +wrapping_test!(test_wrapping_u16, u16, u16::MIN, u16::MAX); +wrapping_test!(test_wrapping_u32, u32, u32::MIN, u32::MAX); +wrapping_test!(test_wrapping_u64, u64, u64::MIN, u64::MAX); +#[cfg(not(target_os = "emscripten"))] +wrapping_test!(test_wrapping_u128, u128, u128::MIN, u128::MAX); +wrapping_test!(test_wrapping_usize, usize, usize::MIN, usize::MAX); + +#[test] +fn wrapping_int_api() { + assert_eq!(i8::MAX.wrapping_add(1), i8::MIN); + assert_eq!(i16::MAX.wrapping_add(1), i16::MIN); + assert_eq!(i32::MAX.wrapping_add(1), i32::MIN); + assert_eq!(i64::MAX.wrapping_add(1), i64::MIN); + assert_eq!(isize::MAX.wrapping_add(1), isize::MIN); + + assert_eq!(i8::MIN.wrapping_sub(1), i8::MAX); + assert_eq!(i16::MIN.wrapping_sub(1), i16::MAX); + assert_eq!(i32::MIN.wrapping_sub(1), i32::MAX); + assert_eq!(i64::MIN.wrapping_sub(1), i64::MAX); + assert_eq!(isize::MIN.wrapping_sub(1), isize::MAX); + + assert_eq!(u8::MAX.wrapping_add(1), u8::MIN); + assert_eq!(u16::MAX.wrapping_add(1), u16::MIN); + assert_eq!(u32::MAX.wrapping_add(1), u32::MIN); + assert_eq!(u64::MAX.wrapping_add(1), u64::MIN); + assert_eq!(usize::MAX.wrapping_add(1), usize::MIN); + + assert_eq!(u8::MIN.wrapping_sub(1), u8::MAX); + assert_eq!(u16::MIN.wrapping_sub(1), u16::MAX); + assert_eq!(u32::MIN.wrapping_sub(1), u32::MAX); + assert_eq!(u64::MIN.wrapping_sub(1), u64::MAX); + assert_eq!(usize::MIN.wrapping_sub(1), usize::MAX); + + assert_eq!((0xfe_u8 as i8).wrapping_mul(16), (0xe0_u8 as i8)); + assert_eq!((0xfedc_u16 as i16).wrapping_mul(16), (0xedc0_u16 as i16)); + assert_eq!((0xfedc_ba98_u32 as i32).wrapping_mul(16), (0xedcb_a980_u32 as i32)); + assert_eq!( + (0xfedc_ba98_7654_3217_u64 as i64).wrapping_mul(16), + (0xedcb_a987_6543_2170_u64 as i64) + ); + + match () { + #[cfg(target_pointer_width = "32")] + () => { + assert_eq!((0xfedc_ba98_u32 as isize).wrapping_mul(16), (0xedcb_a980_u32 as isize)); + } + #[cfg(target_pointer_width = "64")] + () => { + assert_eq!( + (0xfedc_ba98_7654_3217_u64 as isize).wrapping_mul(16), + (0xedcb_a987_6543_2170_u64 as isize) + ); + } + } + + assert_eq!((0xfe as u8).wrapping_mul(16), (0xe0 as u8)); + assert_eq!((0xfedc as u16).wrapping_mul(16), (0xedc0 as u16)); + assert_eq!((0xfedc_ba98 as u32).wrapping_mul(16), (0xedcb_a980 as u32)); + assert_eq!((0xfedc_ba98_7654_3217 as u64).wrapping_mul(16), (0xedcb_a987_6543_2170 as u64)); + + match () { + #[cfg(target_pointer_width = "32")] + () => { + assert_eq!((0xfedc_ba98 as usize).wrapping_mul(16), (0xedcb_a980 as usize)); + } + #[cfg(target_pointer_width = "64")] + () => { + assert_eq!( + (0xfedc_ba98_7654_3217 as usize).wrapping_mul(16), + (0xedcb_a987_6543_2170 as usize) + ); + } + } + + macro_rules! check_mul_no_wrap { + ($e:expr, $f:expr) => { + assert_eq!(($e).wrapping_mul($f), ($e) * $f); + }; + } + macro_rules! check_mul_wraps { + ($e:expr, $f:expr) => { + assert_eq!(($e).wrapping_mul($f), $e); + }; + } + + check_mul_no_wrap!(0xfe_u8 as i8, -1); + check_mul_no_wrap!(0xfedc_u16 as i16, -1); + check_mul_no_wrap!(0xfedc_ba98_u32 as i32, -1); + check_mul_no_wrap!(0xfedc_ba98_7654_3217_u64 as i64, -1); + check_mul_no_wrap!(0xfedc_ba98_7654_3217_u64 as u64 as isize, -1); + + check_mul_no_wrap!(0xfe_u8 as i8, -2); + check_mul_no_wrap!(0xfedc_u16 as i16, -2); + check_mul_no_wrap!(0xfedc_ba98_u32 as i32, -2); + check_mul_no_wrap!(0xfedc_ba98_7654_3217_u64 as i64, -2); + check_mul_no_wrap!(0xfedc_ba98_fedc_ba98_u64 as u64 as isize, -2); + + check_mul_no_wrap!(0xfe_u8 as i8, 2); + check_mul_no_wrap!(0xfedc_u16 as i16, 2); + check_mul_no_wrap!(0xfedc_ba98_u32 as i32, 2); + check_mul_no_wrap!(0xfedc_ba98_7654_3217_u64 as i64, 2); + check_mul_no_wrap!(0xfedc_ba98_fedc_ba98_u64 as u64 as isize, 2); + + check_mul_wraps!(0x80_u8 as i8, -1); + check_mul_wraps!(0x8000_u16 as i16, -1); + check_mul_wraps!(0x8000_0000_u32 as i32, -1); + check_mul_wraps!(0x8000_0000_0000_0000_u64 as i64, -1); + match () { + #[cfg(target_pointer_width = "32")] + () => { + check_mul_wraps!(0x8000_0000_u32 as isize, -1); + } + #[cfg(target_pointer_width = "64")] + () => { + check_mul_wraps!(0x8000_0000_0000_0000_u64 as isize, -1); + } + } + + macro_rules! check_div_no_wrap { + ($e:expr, $f:expr) => { + assert_eq!(($e).wrapping_div($f), ($e) / $f); + }; + } + macro_rules! check_div_wraps { + ($e:expr, $f:expr) => { + assert_eq!(($e).wrapping_div($f), $e); + }; + } + + check_div_no_wrap!(0xfe_u8 as i8, -1); + check_div_no_wrap!(0xfedc_u16 as i16, -1); + check_div_no_wrap!(0xfedc_ba98_u32 as i32, -1); + check_div_no_wrap!(0xfedc_ba98_7654_3217_u64 as i64, -1); + check_div_no_wrap!(0xfedc_ba98_7654_3217_u64 as u64 as isize, -1); + + check_div_no_wrap!(0xfe_u8 as i8, -2); + check_div_no_wrap!(0xfedc_u16 as i16, -2); + check_div_no_wrap!(0xfedc_ba98_u32 as i32, -2); + check_div_no_wrap!(0xfedc_ba98_7654_3217_u64 as i64, -2); + check_div_no_wrap!(0xfedc_ba98_7654_3217_u64 as u64 as isize, -2); + + check_div_no_wrap!(0xfe_u8 as i8, 2); + check_div_no_wrap!(0xfedc_u16 as i16, 2); + check_div_no_wrap!(0xfedc_ba98_u32 as i32, 2); + check_div_no_wrap!(0xfedc_ba98_7654_3217_u64 as i64, 2); + check_div_no_wrap!(0xfedc_ba98_7654_3217_u64 as u64 as isize, 2); + + check_div_wraps!(-128 as i8, -1); + check_div_wraps!(0x8000_u16 as i16, -1); + check_div_wraps!(0x8000_0000_u32 as i32, -1); + check_div_wraps!(0x8000_0000_0000_0000_u64 as i64, -1); + match () { + #[cfg(target_pointer_width = "32")] + () => { + check_div_wraps!(0x8000_0000_u32 as isize, -1); + } + #[cfg(target_pointer_width = "64")] + () => { + check_div_wraps!(0x8000_0000_0000_0000_u64 as isize, -1); + } + } + + macro_rules! check_rem_no_wrap { + ($e:expr, $f:expr) => { + assert_eq!(($e).wrapping_rem($f), ($e) % $f); + }; + } + macro_rules! check_rem_wraps { + ($e:expr, $f:expr) => { + assert_eq!(($e).wrapping_rem($f), 0); + }; + } + + check_rem_no_wrap!(0xfe_u8 as i8, -1); + check_rem_no_wrap!(0xfedc_u16 as i16, -1); + check_rem_no_wrap!(0xfedc_ba98_u32 as i32, -1); + check_rem_no_wrap!(0xfedc_ba98_7654_3217_u64 as i64, -1); + check_rem_no_wrap!(0xfedc_ba98_7654_3217_u64 as u64 as isize, -1); + + check_rem_no_wrap!(0xfe_u8 as i8, -2); + check_rem_no_wrap!(0xfedc_u16 as i16, -2); + check_rem_no_wrap!(0xfedc_ba98_u32 as i32, -2); + check_rem_no_wrap!(0xfedc_ba98_7654_3217_u64 as i64, -2); + check_rem_no_wrap!(0xfedc_ba98_7654_3217_u64 as u64 as isize, -2); + + check_rem_no_wrap!(0xfe_u8 as i8, 2); + check_rem_no_wrap!(0xfedc_u16 as i16, 2); + check_rem_no_wrap!(0xfedc_ba98_u32 as i32, 2); + check_rem_no_wrap!(0xfedc_ba98_7654_3217_u64 as i64, 2); + check_rem_no_wrap!(0xfedc_ba98_7654_3217_u64 as u64 as isize, 2); + + check_rem_wraps!(0x80_u8 as i8, -1); + check_rem_wraps!(0x8000_u16 as i16, -1); + check_rem_wraps!(0x8000_0000_u32 as i32, -1); + check_rem_wraps!(0x8000_0000_0000_0000_u64 as i64, -1); + match () { + #[cfg(target_pointer_width = "32")] + () => { + check_rem_wraps!(0x8000_0000_u32 as isize, -1); + } + #[cfg(target_pointer_width = "64")] + () => { + check_rem_wraps!(0x8000_0000_0000_0000_u64 as isize, -1); + } + } + + macro_rules! check_neg_no_wrap { + ($e:expr) => { + assert_eq!(($e).wrapping_neg(), -($e)); + }; + } + macro_rules! check_neg_wraps { + ($e:expr) => { + assert_eq!(($e).wrapping_neg(), ($e)); + }; + } + + check_neg_no_wrap!(0xfe_u8 as i8); + check_neg_no_wrap!(0xfedc_u16 as i16); + check_neg_no_wrap!(0xfedc_ba98_u32 as i32); + check_neg_no_wrap!(0xfedc_ba98_7654_3217_u64 as i64); + check_neg_no_wrap!(0xfedc_ba98_7654_3217_u64 as u64 as isize); + + check_neg_wraps!(0x80_u8 as i8); + check_neg_wraps!(0x8000_u16 as i16); + check_neg_wraps!(0x8000_0000_u32 as i32); + check_neg_wraps!(0x8000_0000_0000_0000_u64 as i64); + match () { + #[cfg(target_pointer_width = "32")] + () => { + check_neg_wraps!(0x8000_0000_u32 as isize); + } + #[cfg(target_pointer_width = "64")] + () => { + check_neg_wraps!(0x8000_0000_0000_0000_u64 as isize); + } + } +} + +#[test] +fn wrapping_const() { + // Specifically the wrapping behavior of division and remainder is subtle, + // see https://github.com/rust-lang/rust/pull/94512. + const _: () = { + assert!(i32::MIN.wrapping_div(-1) == i32::MIN); + assert!(i32::MIN.wrapping_rem(-1) == 0); + }; +} diff --git a/crux-mir/lib/core/tests/ops.rs b/crux-mir/lib/core/tests/ops.rs index 43eb498d2..0c81cba35 100644 --- a/crux-mir/lib/core/tests/ops.rs +++ b/crux-mir/lib/core/tests/ops.rs @@ -1,6 +1,9 @@ -use core::ops::{Bound, Range, RangeFrom, RangeFull, RangeInclusive, RangeTo}; +mod control_flow; -// Test the Range structs without the syntactic sugar. +use core::ops::{Bound, Range, RangeFrom, RangeFull, RangeInclusive, RangeTo, RangeToInclusive}; +use core::ops::{Deref, DerefMut}; + +// Test the Range structs and syntax. #[test] fn test_range() { @@ -60,26 +63,30 @@ fn test_range_inclusive() { } #[test] -fn test_range_is_empty() { - use core::f32::*; +fn test_range_to_inclusive() { + // Not much to test. + let _ = RangeToInclusive { end: 42 }; +} +#[test] +fn test_range_is_empty() { assert!(!(0.0..10.0).is_empty()); assert!((-0.0..0.0).is_empty()); assert!((10.0..0.0).is_empty()); - assert!(!(NEG_INFINITY..INFINITY).is_empty()); - assert!((EPSILON..NAN).is_empty()); - assert!((NAN..EPSILON).is_empty()); - assert!((NAN..NAN).is_empty()); + assert!(!(f32::NEG_INFINITY..f32::INFINITY).is_empty()); + assert!((f32::EPSILON..f32::NAN).is_empty()); + assert!((f32::NAN..f32::EPSILON).is_empty()); + assert!((f32::NAN..f32::NAN).is_empty()); assert!(!(0.0..=10.0).is_empty()); assert!(!(-0.0..=0.0).is_empty()); assert!((10.0..=0.0).is_empty()); - assert!(!(NEG_INFINITY..=INFINITY).is_empty()); - assert!((EPSILON..=NAN).is_empty()); - assert!((NAN..=EPSILON).is_empty()); - assert!((NAN..=NAN).is_empty()); + assert!(!(f32::NEG_INFINITY..=f32::INFINITY).is_empty()); + assert!((f32::EPSILON..=f32::NAN).is_empty()); + assert!((f32::NAN..=f32::EPSILON).is_empty()); + assert!((f32::NAN..=f32::NAN).is_empty()); } #[test] @@ -96,3 +103,138 @@ fn test_bound_cloned_included() { fn test_bound_cloned_excluded() { assert_eq!(Bound::Excluded(&3).cloned(), Bound::Excluded(3)); } + +#[test] +#[allow(unused_comparisons)] +#[allow(unused_mut)] +fn test_range_syntax() { + let mut count = 0; + for i in 0_usize..10 { + assert!(i >= 0 && i < 10); + count += i; + } + assert_eq!(count, 45); + + let mut count = 0; + let mut range = 0_usize..10; + for i in range { + assert!(i >= 0 && i < 10); + count += i; + } + assert_eq!(count, 45); + + let mut count = 0; + let mut rf = 3_usize..; + for i in rf.take(10) { + assert!(i >= 3 && i < 13); + count += i; + } + assert_eq!(count, 75); + + let _ = 0_usize..4 + 4 - 3; + + fn foo() -> isize { + 42 + } + let _ = 0..foo(); + + let _ = { &42..&100 }; // references to literals are OK + let _ = ..42_usize; + + // Test we can use two different types with a common supertype. + let x = &42; + { + let y = 42; + let _ = x..&y; + } +} + +#[test] +#[allow(dead_code)] +fn test_range_syntax_in_return_statement() { + fn return_range_to() -> RangeTo { + return ..1; + } + fn return_full_range() -> RangeFull { + return ..; + } + // Not much to test. +} + +#[test] +fn range_structural_match() { + // test that all range types can be structurally matched upon + + const RANGE: Range = 0..1000; + match RANGE { + RANGE => {} + _ => unreachable!(), + } + + const RANGE_FROM: RangeFrom = 0..; + match RANGE_FROM { + RANGE_FROM => {} + _ => unreachable!(), + } + + const RANGE_FULL: RangeFull = ..; + match RANGE_FULL { + RANGE_FULL => {} + } + + const RANGE_INCLUSIVE: RangeInclusive = 0..=999; + match RANGE_INCLUSIVE { + RANGE_INCLUSIVE => {} + _ => unreachable!(), + } + + const RANGE_TO: RangeTo = ..1000; + match RANGE_TO { + RANGE_TO => {} + _ => unreachable!(), + } + + const RANGE_TO_INCLUSIVE: RangeToInclusive = ..=999; + match RANGE_TO_INCLUSIVE { + RANGE_TO_INCLUSIVE => {} + _ => unreachable!(), + } +} + +// Test Deref implementations + +#[test] +fn deref_mut_on_ref() { + // Test that `&mut T` implements `DerefMut` + + fn inc + DerefMut>(mut t: T) { + *t += 1; + } + + let mut x: isize = 5; + inc(&mut x); + assert_eq!(x, 6); +} + +#[test] +fn deref_on_ref() { + // Test that `&T` and `&mut T` implement `Deref` + + fn deref>(t: T) -> U { + *t + } + + let x: isize = 3; + let y = deref(&x); + assert_eq!(y, 3); + + let mut x: isize = 4; + let y = deref(&mut x); + assert_eq!(y, 4); +} + +#[test] +#[allow(unreachable_code)] +fn test_not_never() { + if !return () {} +} diff --git a/crux-mir/lib/core/tests/ops/control_flow.rs b/crux-mir/lib/core/tests/ops/control_flow.rs new file mode 100644 index 000000000..eacfd63a6 --- /dev/null +++ b/crux-mir/lib/core/tests/ops/control_flow.rs @@ -0,0 +1,18 @@ +use core::intrinsics::discriminant_value; +use core::ops::ControlFlow; + +#[test] +fn control_flow_discriminants_match_result() { + // This isn't stable surface area, but helps keep `?` cheap between them, + // even if LLVM can't always take advantage of it right now. + // (Sadly Result and Option are inconsistent, so ControlFlow can't match both.) + + assert_eq!( + discriminant_value(&ControlFlow::::Break(3)), + discriminant_value(&Result::::Err(3)), + ); + assert_eq!( + discriminant_value(&ControlFlow::::Continue(3)), + discriminant_value(&Result::::Ok(3)), + ); +} diff --git a/crux-mir/lib/core/tests/option.rs b/crux-mir/lib/core/tests/option.rs index fa308160f..dca6321cf 100644 --- a/crux-mir/lib/core/tests/option.rs +++ b/crux-mir/lib/core/tests/option.rs @@ -1,4 +1,4 @@ -use core::array::FixedSizeArray; +use core::cell::Cell; use core::clone::Clone; use core::mem; use core::ops::DerefMut; @@ -7,7 +7,7 @@ use core::option::*; #[test] fn test_get_ptr() { unsafe { - let x: Box<_> = box 0; + let x: Box<_> = Box::new(0); let addr_x: *const isize = mem::transmute(&*x); let opt = Some(x); let y = opt.unwrap(); @@ -57,6 +57,7 @@ fn test_get_resource() { } #[test] +#[allow(for_loops_over_fallibles)] fn test_option_dance() { let x = Some(()); let mut y = Some(5); @@ -86,17 +87,49 @@ fn test_and() { let x: Option = None; assert_eq!(x.and(Some(2)), None); assert_eq!(x.and(None::), None); + + const FOO: Option = Some(1); + const A: Option = FOO.and(Some(2)); + const B: Option = FOO.and(None); + assert_eq!(A, Some(2)); + assert_eq!(B, None); + + const BAR: Option = None; + const C: Option = BAR.and(Some(2)); + const D: Option = BAR.and(None); + assert_eq!(C, None); + assert_eq!(D, None); } #[test] fn test_and_then() { + const fn plus_one(x: isize) -> Option { + Some(x + 1) + } + + const fn none(_: isize) -> Option { + None + } + let x: Option = Some(1); - assert_eq!(x.and_then(|x| Some(x + 1)), Some(2)); - assert_eq!(x.and_then(|_| None::), None); + assert_eq!(x.and_then(plus_one), Some(2)); + assert_eq!(x.and_then(none), None); let x: Option = None; - assert_eq!(x.and_then(|x| Some(x + 1)), None); - assert_eq!(x.and_then(|_| None::), None); + assert_eq!(x.and_then(plus_one), None); + assert_eq!(x.and_then(none), None); + + const FOO: Option = Some(1); + const A: Option = FOO.and_then(plus_one); + const B: Option = FOO.and_then(none); + assert_eq!(A, Some(2)); + assert_eq!(B, None); + + const BAR: Option = None; + const C: Option = BAR.and_then(plus_one); + const D: Option = BAR.and_then(none); + assert_eq!(C, None); + assert_eq!(D, None); } #[test] @@ -108,17 +141,49 @@ fn test_or() { let x: Option = None; assert_eq!(x.or(Some(2)), Some(2)); assert_eq!(x.or(None), None); + + const FOO: Option = Some(1); + const A: Option = FOO.or(Some(2)); + const B: Option = FOO.or(None); + assert_eq!(A, Some(1)); + assert_eq!(B, Some(1)); + + const BAR: Option = None; + const C: Option = BAR.or(Some(2)); + const D: Option = BAR.or(None); + assert_eq!(C, Some(2)); + assert_eq!(D, None); } #[test] fn test_or_else() { + const fn two() -> Option { + Some(2) + } + + const fn none() -> Option { + None + } + let x: Option = Some(1); - assert_eq!(x.or_else(|| Some(2)), Some(1)); - assert_eq!(x.or_else(|| None), Some(1)); + assert_eq!(x.or_else(two), Some(1)); + assert_eq!(x.or_else(none), Some(1)); let x: Option = None; - assert_eq!(x.or_else(|| Some(2)), Some(2)); - assert_eq!(x.or_else(|| None), None); + assert_eq!(x.or_else(two), Some(2)); + assert_eq!(x.or_else(none), None); + + const FOO: Option = Some(1); + const A: Option = FOO.or_else(two); + const B: Option = FOO.or_else(none); + assert_eq!(A, Some(1)); + assert_eq!(B, Some(1)); + + const BAR: Option = None; + const C: Option = BAR.or_else(two); + const D: Option = BAR.or_else(none); + assert_eq!(C, Some(2)); + assert_eq!(D, None); } #[test] @@ -149,15 +214,36 @@ fn test_unwrap_or() { let x: Option = None; assert_eq!(x.unwrap_or(2), 2); + + const A: isize = Some(1).unwrap_or(2); + const B: isize = None.unwrap_or(2); + assert_eq!(A, 1); + assert_eq!(B, 2); } #[test] fn test_unwrap_or_else() { + const fn two() -> isize { + 2 + } + let x: Option = Some(1); - assert_eq!(x.unwrap_or_else(|| 2), 1); + assert_eq!(x.unwrap_or_else(two), 1); let x: Option = None; - assert_eq!(x.unwrap_or_else(|| 2), 2); + assert_eq!(x.unwrap_or_else(two), 2); + + const A: isize = Some(1).unwrap_or_else(two); + const B: isize = None.unwrap_or_else(two); + assert_eq!(A, 1); + assert_eq!(B, 2); +} + +#[test] +fn test_unwrap_unchecked() { + assert_eq!(unsafe { Some(1).unwrap_unchecked() }, 1); + let s = unsafe { Some("hello".to_string()).unwrap_unchecked() }; + assert_eq!(s, "hello"); } #[test] @@ -230,7 +316,7 @@ fn test_collect() { // test that it does not take more elements than it needs let mut functions: [Box Option<()>>; 3] = - [box || Some(()), box || None, box || panic!()]; + [Box::new(|| Some(())), Box::new(|| None), Box::new(|| panic!())]; let v: Option> = functions.iter_mut().map(|f| (*f)()).collect(); @@ -294,18 +380,6 @@ fn test_try() { Some(val) } assert_eq!(try_option_none(), None); - - fn try_option_ok() -> Result { - let val = Some(1)?; - Ok(val) - } - assert_eq!(try_option_ok(), Ok(1)); - - fn try_option_err() -> Result { - let val = None?; - Ok(val) - } - assert_eq!(try_option_err(), Err(NoneError)); } #[test] @@ -357,3 +431,126 @@ fn test_replace() { assert_eq!(x, Some(3)); assert_eq!(old, None); } + +#[test] +fn option_const() { + // test that the methods of `Option` are usable in a const context + + const OPTION: Option = Some(32); + assert_eq!(OPTION, Some(32)); + + const OPTION_FROM: Option = Option::from(32); + assert_eq!(OPTION_FROM, Some(32)); + + const REF: Option<&usize> = OPTION.as_ref(); + assert_eq!(REF, Some(&32)); + + const REF_FROM: Option<&usize> = Option::from(&OPTION); + assert_eq!(REF_FROM, Some(&32)); + + const IS_SOME: bool = OPTION.is_some(); + assert!(IS_SOME); + + const IS_NONE: bool = OPTION.is_none(); + assert!(!IS_NONE); + + const COPIED: Option = OPTION.as_ref().copied(); + assert_eq!(COPIED, OPTION); +} + +#[test] +const fn option_const_mut() { + // test that the methods of `Option` that take mutable references are usable in a const context + + let mut option: Option = Some(32); + + let _take = option.take(); + let _replace = option.replace(42); + + { + let as_mut = option.as_mut(); + match as_mut { + Some(v) => *v = 32, + None => unreachable!(), + } + } + + { + let as_mut: Option<&mut usize> = Option::from(&mut option); + match as_mut { + Some(v) => *v = 42, + None => unreachable!(), + } + } +} + +#[test] +fn test_unwrap_drop() { + struct Dtor<'a> { + x: &'a Cell, + } + + impl<'a> std::ops::Drop for Dtor<'a> { + fn drop(&mut self) { + self.x.set(self.x.get() - 1); + } + } + + fn unwrap(o: Option) -> T { + match o { + Some(v) => v, + None => panic!(), + } + } + + let x = &Cell::new(1); + + { + let b = Some(Dtor { x }); + let _c = unwrap(b); + } + + assert_eq!(x.get(), 0); +} + +#[test] +fn option_ext() { + let thing = "{{ f }}"; + let f = thing.find("{{"); + + if f.is_none() { + println!("None!"); + } +} + +#[test] +fn zip_options() { + let x = Some(10); + let y = Some("foo"); + let z: Option = None; + + assert_eq!(x.zip(y), Some((10, "foo"))); + assert_eq!(x.zip(z), None); + assert_eq!(z.zip(x), None); +} + +#[test] +fn unzip_options() { + let x = Some((10, "foo")); + let y = None::<(bool, i32)>; + + assert_eq!(x.unzip(), (Some(10), Some("foo"))); + assert_eq!(y.unzip(), (None, None)); +} + +#[test] +fn zip_unzip_roundtrip() { + let x = Some(10); + let y = Some("foo"); + + let z = x.zip(y); + assert_eq!(z, Some((10, "foo"))); + + let a = z.unzip(); + assert_eq!(a, (x, y)); +} diff --git a/crux-mir/lib/core/tests/panic.rs b/crux-mir/lib/core/tests/panic.rs new file mode 100644 index 000000000..24b6c56b3 --- /dev/null +++ b/crux-mir/lib/core/tests/panic.rs @@ -0,0 +1 @@ +mod location; diff --git a/crux-mir/lib/core/tests/panic/location.rs b/crux-mir/lib/core/tests/panic/location.rs new file mode 100644 index 000000000..d20241d83 --- /dev/null +++ b/crux-mir/lib/core/tests/panic/location.rs @@ -0,0 +1,31 @@ +use core::panic::Location; + +// Note: Some of the following tests depend on the source location, +// so please be careful when editing this file. + +#[test] +fn location_const_caller() { + const _CALLER_REFERENCE: &Location<'static> = Location::caller(); + const _CALLER: Location<'static> = *Location::caller(); +} + +#[test] +fn location_const_file() { + const CALLER: &Location<'static> = Location::caller(); + const FILE: &str = CALLER.file(); + assert_eq!(FILE, file!()); +} + +#[test] +fn location_const_line() { + const CALLER: &Location<'static> = Location::caller(); + const LINE: u32 = CALLER.line(); + assert_eq!(LINE, 21); +} + +#[test] +fn location_const_column() { + const CALLER: &Location<'static> = Location::caller(); + const COLUMN: u32 = CALLER.column(); + assert_eq!(COLUMN, 40); +} diff --git a/crux-mir/lib/core/tests/pin.rs b/crux-mir/lib/core/tests/pin.rs new file mode 100644 index 000000000..6f617c8d0 --- /dev/null +++ b/crux-mir/lib/core/tests/pin.rs @@ -0,0 +1,31 @@ +use core::pin::Pin; + +#[test] +fn pin_const() { + // test that the methods of `Pin` are usable in a const context + + const POINTER: &'static usize = &2; + + const PINNED: Pin<&'static usize> = Pin::new(POINTER); + const PINNED_UNCHECKED: Pin<&'static usize> = unsafe { Pin::new_unchecked(POINTER) }; + assert_eq!(PINNED_UNCHECKED, PINNED); + + const INNER: &'static usize = Pin::into_inner(PINNED); + assert_eq!(INNER, POINTER); + + const INNER_UNCHECKED: &'static usize = unsafe { Pin::into_inner_unchecked(PINNED) }; + assert_eq!(INNER_UNCHECKED, POINTER); + + const REF: &'static usize = PINNED.get_ref(); + assert_eq!(REF, POINTER); + + // Note: `pin_mut_const` tests that the methods of `Pin<&mut T>` are usable in a const context. + // A const fn is used because `&mut` is not (yet) usable in constants. + const fn pin_mut_const() { + let _ = Pin::new(&mut 2).into_ref(); + let _ = Pin::new(&mut 2).get_mut(); + let _ = unsafe { Pin::new(&mut 2).get_unchecked_mut() }; + } + + pin_mut_const(); +} diff --git a/crux-mir/lib/core/tests/pin_macro.rs b/crux-mir/lib/core/tests/pin_macro.rs new file mode 100644 index 000000000..79c8c166c --- /dev/null +++ b/crux-mir/lib/core/tests/pin_macro.rs @@ -0,0 +1,33 @@ +// edition:2021 +use core::{ + marker::PhantomPinned, + mem::{drop as stuff, transmute}, + pin::{pin, Pin}, +}; + +#[test] +fn basic() { + let it: Pin<&mut PhantomPinned> = pin!(PhantomPinned); + stuff(it); +} + +#[test] +fn extension_works_through_block() { + let it: Pin<&mut PhantomPinned> = { pin!(PhantomPinned) }; + stuff(it); +} + +#[test] +fn extension_works_through_unsafe_block() { + // "retro-type-inference" works as well. + let it: Pin<&mut PhantomPinned> = unsafe { pin!(transmute(())) }; + stuff(it); +} + +#[test] +fn unsize_coercion() { + let slice: Pin<&mut [PhantomPinned]> = pin!([PhantomPinned; 2]); + stuff(slice); + let dyn_obj: Pin<&mut dyn Send> = pin!([PhantomPinned; 2]); + stuff(dyn_obj); +} diff --git a/crux-mir/lib/core/tests/ptr.rs b/crux-mir/lib/core/tests/ptr.rs index a008b3319..80d30f14c 100644 --- a/crux-mir/lib/core/tests/ptr.rs +++ b/crux-mir/lib/core/tests/ptr.rs @@ -1,5 +1,9 @@ use core::cell::RefCell; +use core::mem::{self, MaybeUninit}; +use core::num::NonZeroUsize; +use core::ptr; use core::ptr::*; +use std::fmt::{Debug, Display}; #[test] fn test_const_from_raw_parts() { @@ -15,10 +19,11 @@ fn test_const_from_raw_parts() { #[test] fn test() { unsafe { + #[repr(C)] struct Pair { fst: isize, snd: isize, - }; + } let mut p = Pair { fst: 10, snd: 20 }; let pptr: *mut Pair = &mut p; let iptr: *mut isize = pptr as *mut isize; @@ -90,6 +95,15 @@ fn test_is_null() { let nmi: *mut dyn ToString = null_mut::(); assert!(nmi.is_null()); + + extern "C" { + type Extern; + } + let ec: *const Extern = null::(); + assert!(ec.is_null()); + + let em: *mut Extern = null_mut::(); + assert!(em.is_null()); } #[test] @@ -248,6 +262,20 @@ fn test_set_memory() { assert!(xs == [5u8; 20]); } +#[test] +fn test_set_memory_const() { + const XS: [u8; 20] = { + let mut xs = [0u8; 20]; + let ptr = xs.as_mut_ptr(); + unsafe { + ptr.write_bytes(5u8, xs.len()); + } + xs + }; + + assert!(XS == [5u8; 20]); +} + #[test] fn test_unsized_nonnull() { let xs: &[i32] = &[1, 2, 3]; @@ -258,16 +286,33 @@ fn test_unsized_nonnull() { } #[test] -#[allow(warnings)] -// Have a symbol for the test below. It doesn’t need to be an actual variadic function, match the -// ABI, or even point to an actual executable code, because the function itself is never invoked. -#[no_mangle] +fn test_const_nonnull_new() { + const { + assert!(NonNull::new(core::ptr::null_mut::<()>()).is_none()); + + let value = &mut 0u32; + let mut ptr = NonNull::new(value).unwrap(); + unsafe { *ptr.as_mut() = 42 }; + + let reference = unsafe { &*ptr.as_ref() }; + assert!(*reference == *value); + assert!(*reference == 42); + }; +} + +#[test] +#[cfg(unix)] // printf may not be available on other platforms +#[allow(deprecated)] // For SipHasher pub fn test_variadic_fnptr() { + use core::ffi; use core::hash::{Hash, SipHasher}; extern "C" { - fn test_variadic_fnptr(_: u64, ...) -> f64; + // This needs to use the correct function signature even though it isn't called as some + // codegen backends make it UB to declare a function with multiple conflicting signatures + // (like LLVM) while others straight up return an error (like Cranelift). + fn printf(_: *const ffi::c_char, ...) -> ffi::c_int; } - let p: unsafe extern "C" fn(u64, ...) -> f64 = test_variadic_fnptr; + let p: unsafe extern "C" fn(*const ffi::c_char, ...) -> ffi::c_int = printf; let q = p.clone(); assert_eq!(p, q); assert!(!(p < q)); @@ -300,23 +345,37 @@ fn write_unaligned_drop() { } #[test] -#[cfg_attr(miri, ignore)] // Miri does not compute a maximal `mid` for `align_offset` fn align_offset_zst() { // For pointers of stride = 0, the pointer is already aligned or it cannot be aligned at // all, because no amount of elements will align the pointer. let mut p = 1; while p < 1024 { - assert_eq!((p as *const ()).align_offset(p), 0); + assert_eq!(ptr::invalid::<()>(p).align_offset(p), 0); if p != 1 { - assert_eq!(((p + 1) as *const ()).align_offset(p), !0); + assert_eq!(ptr::invalid::<()>(p + 1).align_offset(p), !0); } p = (p + 1).next_power_of_two(); } } #[test] -#[cfg_attr(miri, ignore)] // Miri does not compute a maximal `mid` for `align_offset` -fn align_offset_stride1() { +fn align_offset_zst_const() { + const { + // For pointers of stride = 0, the pointer is already aligned or it cannot be aligned at + // all, because no amount of elements will align the pointer. + let mut p = 1; + while p < 1024 { + assert!(ptr::invalid::<()>(p).align_offset(p) == 0); + if p != 1 { + assert!(ptr::invalid::<()>(p + 1).align_offset(p) == !0); + } + p = (p + 1).next_power_of_two(); + } + } +} + +#[test] +fn align_offset_stride_one() { // For pointers of stride = 1, the pointer can always be aligned. The offset is equal to // number of bytes. let mut align = 1; @@ -325,7 +384,7 @@ fn align_offset_stride1() { let expected = ptr % align; let offset = if expected == 0 { 0 } else { align - expected }; assert_eq!( - (ptr as *const u8).align_offset(align), + ptr::invalid::(ptr).align_offset(align), offset, "ptr = {}, align = {}, size = 1", ptr, @@ -337,27 +396,29 @@ fn align_offset_stride1() { } #[test] -#[cfg_attr(miri, ignore)] // Miri is too slow -fn align_offset_weird_strides() { - #[repr(packed)] - struct A3(u16, u8); - struct A4(u32); - #[repr(packed)] - struct A5(u32, u8); - #[repr(packed)] - struct A6(u32, u16); - #[repr(packed)] - struct A7(u32, u16, u8); - #[repr(packed)] - struct A8(u32, u32); - #[repr(packed)] - struct A9(u32, u32, u8); - #[repr(packed)] - struct A10(u32, u32, u16); +fn align_offset_stride_one_const() { + const { + // For pointers of stride = 1, the pointer can always be aligned. The offset is equal to + // number of bytes. + let mut align = 1; + while align < 1024 { + let mut ptr = 1; + while ptr < 2 * align { + let expected = ptr % align; + let offset = if expected == 0 { 0 } else { align - expected }; + assert!(ptr::invalid::(ptr).align_offset(align) == offset); + ptr += 1; + } + align = (align + 1).next_power_of_two(); + } + } +} - unsafe fn test_weird_stride(ptr: *const T, align: usize) -> bool { +#[test] +fn align_offset_various_strides() { + unsafe fn test_stride(ptr: *const T, align: usize) -> bool { let numptr = ptr as usize; - let mut expected = usize::max_value(); + let mut expected = usize::MAX; // Naive but definitely correct way to find the *first* aligned element of stride::. for el in 0..align { if (numptr + el * ::std::mem::size_of::()) % align == 0 { @@ -384,20 +445,684 @@ fn align_offset_weird_strides() { // implementation let mut align = 1; let mut x = false; - while align < 1024 { + // Miri is too slow + let limit = if cfg!(miri) { 32 } else { 1024 }; + while align < limit { for ptr in 1usize..4 * align { unsafe { - x |= test_weird_stride::(ptr as *const A3, align); - x |= test_weird_stride::(ptr as *const A4, align); - x |= test_weird_stride::(ptr as *const A5, align); - x |= test_weird_stride::(ptr as *const A6, align); - x |= test_weird_stride::(ptr as *const A7, align); - x |= test_weird_stride::(ptr as *const A8, align); - x |= test_weird_stride::(ptr as *const A9, align); - x |= test_weird_stride::(ptr as *const A10, align); + #[repr(packed)] + struct A3(u16, u8); + x |= test_stride::(ptr::invalid::(ptr), align); + + struct A4(u32); + x |= test_stride::(ptr::invalid::(ptr), align); + + #[repr(packed)] + struct A5(u32, u8); + x |= test_stride::(ptr::invalid::(ptr), align); + + #[repr(packed)] + struct A6(u32, u16); + x |= test_stride::(ptr::invalid::(ptr), align); + + #[repr(packed)] + struct A7(u32, u16, u8); + x |= test_stride::(ptr::invalid::(ptr), align); + + #[repr(packed)] + struct A8(u32, u32); + x |= test_stride::(ptr::invalid::(ptr), align); + + #[repr(packed)] + struct A9(u32, u32, u8); + x |= test_stride::(ptr::invalid::(ptr), align); + + #[repr(packed)] + struct A10(u32, u32, u16); + x |= test_stride::(ptr::invalid::(ptr), align); + + x |= test_stride::(ptr::invalid::(ptr), align); + x |= test_stride::(ptr::invalid::(ptr), align); } } align = (align + 1).next_power_of_two(); } assert!(!x); } + +#[test] +fn align_offset_various_strides_const() { + const unsafe fn test_stride(ptr: *const T, numptr: usize, align: usize) { + let mut expected = usize::MAX; + // Naive but definitely correct way to find the *first* aligned element of stride::. + let mut el = 0; + while el < align { + if (numptr + el * ::std::mem::size_of::()) % align == 0 { + expected = el; + break; + } + el += 1; + } + let got = ptr.align_offset(align); + assert!(got == expected); + } + + const { + // For pointers of stride != 1, we verify the algorithm against the naivest possible + // implementation + let mut align = 1; + let limit = 32; + while align < limit { + let mut ptr = 1; + while ptr < 4 * align { + unsafe { + #[repr(packed)] + struct A3(u16, u8); + test_stride::(ptr::invalid::(ptr), ptr, align); + + struct A4(u32); + test_stride::(ptr::invalid::(ptr), ptr, align); + + #[repr(packed)] + struct A5(u32, u8); + test_stride::(ptr::invalid::(ptr), ptr, align); + + #[repr(packed)] + struct A6(u32, u16); + test_stride::(ptr::invalid::(ptr), ptr, align); + + #[repr(packed)] + struct A7(u32, u16, u8); + test_stride::(ptr::invalid::(ptr), ptr, align); + + #[repr(packed)] + struct A8(u32, u32); + test_stride::(ptr::invalid::(ptr), ptr, align); + + #[repr(packed)] + struct A9(u32, u32, u8); + test_stride::(ptr::invalid::(ptr), ptr, align); + + #[repr(packed)] + struct A10(u32, u32, u16); + test_stride::(ptr::invalid::(ptr), ptr, align); + + test_stride::(ptr::invalid::(ptr), ptr, align); + test_stride::(ptr::invalid::(ptr), ptr, align); + } + ptr += 1; + } + align = (align + 1).next_power_of_two(); + } + } +} + +#[test] +fn align_offset_with_provenance_const() { + const { + // On some platforms (e.g. msp430-none-elf), the alignment of `i32` is less than 4. + #[repr(align(4))] + struct AlignedI32(i32); + + let data = AlignedI32(42); + + // `stride % align == 0` (usual case) + + let ptr: *const i32 = &data.0; + assert!(ptr.align_offset(1) == 0); + assert!(ptr.align_offset(2) == 0); + assert!(ptr.align_offset(4) == 0); + assert!(ptr.align_offset(8) == usize::MAX); + assert!(ptr.wrapping_byte_add(1).align_offset(1) == 0); + assert!(ptr.wrapping_byte_add(1).align_offset(2) == usize::MAX); + assert!(ptr.wrapping_byte_add(2).align_offset(1) == 0); + assert!(ptr.wrapping_byte_add(2).align_offset(2) == 0); + assert!(ptr.wrapping_byte_add(2).align_offset(4) == usize::MAX); + assert!(ptr.wrapping_byte_add(3).align_offset(1) == 0); + assert!(ptr.wrapping_byte_add(3).align_offset(2) == usize::MAX); + + assert!(ptr.wrapping_add(42).align_offset(4) == 0); + assert!(ptr.wrapping_add(42).align_offset(8) == usize::MAX); + + let ptr1: *const i8 = ptr.cast(); + assert!(ptr1.align_offset(1) == 0); + assert!(ptr1.align_offset(2) == 0); + assert!(ptr1.align_offset(4) == 0); + assert!(ptr1.align_offset(8) == usize::MAX); + assert!(ptr1.wrapping_byte_add(1).align_offset(1) == 0); + assert!(ptr1.wrapping_byte_add(1).align_offset(2) == 1); + assert!(ptr1.wrapping_byte_add(1).align_offset(4) == 3); + assert!(ptr1.wrapping_byte_add(1).align_offset(8) == usize::MAX); + assert!(ptr1.wrapping_byte_add(2).align_offset(1) == 0); + assert!(ptr1.wrapping_byte_add(2).align_offset(2) == 0); + assert!(ptr1.wrapping_byte_add(2).align_offset(4) == 2); + assert!(ptr1.wrapping_byte_add(2).align_offset(8) == usize::MAX); + assert!(ptr1.wrapping_byte_add(3).align_offset(1) == 0); + assert!(ptr1.wrapping_byte_add(3).align_offset(2) == 1); + assert!(ptr1.wrapping_byte_add(3).align_offset(4) == 1); + assert!(ptr1.wrapping_byte_add(3).align_offset(8) == usize::MAX); + + let ptr2: *const i16 = ptr.cast(); + assert!(ptr2.align_offset(1) == 0); + assert!(ptr2.align_offset(2) == 0); + assert!(ptr2.align_offset(4) == 0); + assert!(ptr2.align_offset(8) == usize::MAX); + assert!(ptr2.wrapping_byte_add(1).align_offset(1) == 0); + assert!(ptr2.wrapping_byte_add(1).align_offset(2) == usize::MAX); + assert!(ptr2.wrapping_byte_add(2).align_offset(1) == 0); + assert!(ptr2.wrapping_byte_add(2).align_offset(2) == 0); + assert!(ptr2.wrapping_byte_add(2).align_offset(4) == 1); + assert!(ptr2.wrapping_byte_add(2).align_offset(8) == usize::MAX); + assert!(ptr2.wrapping_byte_add(3).align_offset(1) == 0); + assert!(ptr2.wrapping_byte_add(3).align_offset(2) == usize::MAX); + + let ptr3: *const i64 = ptr.cast(); + assert!(ptr3.align_offset(1) == 0); + assert!(ptr3.align_offset(2) == 0); + assert!(ptr3.align_offset(4) == 0); + assert!(ptr3.align_offset(8) == usize::MAX); + assert!(ptr3.wrapping_byte_add(1).align_offset(1) == 0); + assert!(ptr3.wrapping_byte_add(1).align_offset(2) == usize::MAX); + + // `stride % align != 0` (edge case) + + let ptr4: *const [u8; 3] = ptr.cast(); + assert!(ptr4.align_offset(1) == 0); + assert!(ptr4.align_offset(2) == 0); + assert!(ptr4.align_offset(4) == 0); + assert!(ptr4.align_offset(8) == usize::MAX); + assert!(ptr4.wrapping_byte_add(1).align_offset(1) == 0); + assert!(ptr4.wrapping_byte_add(1).align_offset(2) == 1); + assert!(ptr4.wrapping_byte_add(1).align_offset(4) == 1); + assert!(ptr4.wrapping_byte_add(1).align_offset(8) == usize::MAX); + assert!(ptr4.wrapping_byte_add(2).align_offset(1) == 0); + assert!(ptr4.wrapping_byte_add(2).align_offset(2) == 0); + assert!(ptr4.wrapping_byte_add(2).align_offset(4) == 2); + assert!(ptr4.wrapping_byte_add(2).align_offset(8) == usize::MAX); + assert!(ptr4.wrapping_byte_add(3).align_offset(1) == 0); + assert!(ptr4.wrapping_byte_add(3).align_offset(2) == 1); + assert!(ptr4.wrapping_byte_add(3).align_offset(4) == 3); + assert!(ptr4.wrapping_byte_add(3).align_offset(8) == usize::MAX); + + let ptr5: *const [u8; 5] = ptr.cast(); + assert!(ptr5.align_offset(1) == 0); + assert!(ptr5.align_offset(2) == 0); + assert!(ptr5.align_offset(4) == 0); + assert!(ptr5.align_offset(8) == usize::MAX); + assert!(ptr5.wrapping_byte_add(1).align_offset(1) == 0); + assert!(ptr5.wrapping_byte_add(1).align_offset(2) == 1); + assert!(ptr5.wrapping_byte_add(1).align_offset(4) == 3); + assert!(ptr5.wrapping_byte_add(1).align_offset(8) == usize::MAX); + assert!(ptr5.wrapping_byte_add(2).align_offset(1) == 0); + assert!(ptr5.wrapping_byte_add(2).align_offset(2) == 0); + assert!(ptr5.wrapping_byte_add(2).align_offset(4) == 2); + assert!(ptr5.wrapping_byte_add(2).align_offset(8) == usize::MAX); + assert!(ptr5.wrapping_byte_add(3).align_offset(1) == 0); + assert!(ptr5.wrapping_byte_add(3).align_offset(2) == 1); + assert!(ptr5.wrapping_byte_add(3).align_offset(4) == 1); + assert!(ptr5.wrapping_byte_add(3).align_offset(8) == usize::MAX); + } +} + +#[test] +fn align_offset_issue_103361() { + #[cfg(target_pointer_width = "64")] + const SIZE: usize = 1 << 47; + #[cfg(target_pointer_width = "32")] + const SIZE: usize = 1 << 30; + #[cfg(target_pointer_width = "16")] + const SIZE: usize = 1 << 13; + struct HugeSize([u8; SIZE - 1]); + let _ = ptr::invalid::(SIZE).align_offset(SIZE); +} + +#[test] +fn align_offset_issue_103361_const() { + #[cfg(target_pointer_width = "64")] + const SIZE: usize = 1 << 47; + #[cfg(target_pointer_width = "32")] + const SIZE: usize = 1 << 30; + #[cfg(target_pointer_width = "16")] + const SIZE: usize = 1 << 13; + struct HugeSize([u8; SIZE - 1]); + + const { + assert!(ptr::invalid::(SIZE - 1).align_offset(SIZE) == SIZE - 1); + assert!(ptr::invalid::(SIZE).align_offset(SIZE) == 0); + assert!(ptr::invalid::(SIZE + 1).align_offset(SIZE) == 1); + } +} + +#[test] +fn is_aligned() { + let data = 42; + let ptr: *const i32 = &data; + assert!(ptr.is_aligned()); + assert!(ptr.is_aligned_to(1)); + assert!(ptr.is_aligned_to(2)); + assert!(ptr.is_aligned_to(4)); + assert!(ptr.wrapping_byte_add(2).is_aligned_to(1)); + assert!(ptr.wrapping_byte_add(2).is_aligned_to(2)); + assert!(!ptr.wrapping_byte_add(2).is_aligned_to(4)); + + // At runtime either `ptr` or `ptr+1` is aligned to 8. + assert_ne!(ptr.is_aligned_to(8), ptr.wrapping_add(1).is_aligned_to(8)); +} + +#[test] +fn is_aligned_const() { + const { + let data = 42; + let ptr: *const i32 = &data; + assert!(ptr.is_aligned()); + assert!(ptr.is_aligned_to(1)); + assert!(ptr.is_aligned_to(2)); + assert!(ptr.is_aligned_to(4)); + assert!(ptr.wrapping_byte_add(2).is_aligned_to(1)); + assert!(ptr.wrapping_byte_add(2).is_aligned_to(2)); + assert!(!ptr.wrapping_byte_add(2).is_aligned_to(4)); + + // At comptime neither `ptr` nor `ptr+1` is aligned to 8. + assert!(!ptr.is_aligned_to(8)); + assert!(!ptr.wrapping_add(1).is_aligned_to(8)); + } +} + +#[test] +fn offset_from() { + let mut a = [0; 5]; + let ptr1: *mut i32 = &mut a[1]; + let ptr2: *mut i32 = &mut a[3]; + unsafe { + assert_eq!(ptr2.offset_from(ptr1), 2); + assert_eq!(ptr1.offset_from(ptr2), -2); + assert_eq!(ptr1.offset(2), ptr2); + assert_eq!(ptr2.offset(-2), ptr1); + } +} + +#[test] +fn ptr_metadata() { + struct Unit; + struct Pair(A, B); + extern "C" { + type Extern; + } + let () = metadata(&()); + let () = metadata(&Unit); + let () = metadata(&4_u32); + let () = metadata(&String::new()); + let () = metadata(&Some(4_u32)); + let () = metadata(&ptr_metadata); + let () = metadata(&|| {}); + let () = metadata(&[4, 7]); + let () = metadata(&(4, String::new())); + let () = metadata(&Pair(4, String::new())); + let () = metadata(ptr::null::<()>() as *const Extern); + let () = metadata(ptr::null::<()>() as *const <&u32 as std::ops::Deref>::Target); + + assert_eq!(metadata("foo"), 3_usize); + assert_eq!(metadata(&[4, 7][..]), 2_usize); + + let dst_tuple: &(bool, [u8]) = &(true, [0x66, 0x6F, 0x6F]); + let dst_struct: &Pair = &Pair(true, [0x66, 0x6F, 0x6F]); + assert_eq!(metadata(dst_tuple), 3_usize); + assert_eq!(metadata(dst_struct), 3_usize); + unsafe { + let dst_tuple: &(bool, str) = std::mem::transmute(dst_tuple); + let dst_struct: &Pair = std::mem::transmute(dst_struct); + assert_eq!(&dst_tuple.1, "foo"); + assert_eq!(&dst_struct.1, "foo"); + assert_eq!(metadata(dst_tuple), 3_usize); + assert_eq!(metadata(dst_struct), 3_usize); + } + + let vtable_1: DynMetadata = metadata(&4_u16 as &dyn Debug); + let vtable_2: DynMetadata = metadata(&4_u16 as &dyn Display); + let vtable_3: DynMetadata = metadata(&4_u32 as &dyn Display); + let vtable_4: DynMetadata = metadata(&(true, 7_u32) as &(bool, dyn Display)); + let vtable_5: DynMetadata = + metadata(&Pair(true, 7_u32) as &Pair); + unsafe { + let address_1: *const () = std::mem::transmute(vtable_1); + let address_2: *const () = std::mem::transmute(vtable_2); + let address_3: *const () = std::mem::transmute(vtable_3); + let address_4: *const () = std::mem::transmute(vtable_4); + let address_5: *const () = std::mem::transmute(vtable_5); + // Different trait => different vtable pointer + assert_ne!(address_1, address_2); + // Different erased type => different vtable pointer + assert_ne!(address_2, address_3); + // Same erased type and same trait => same vtable pointer + assert_eq!(address_3, address_4); + assert_eq!(address_3, address_5); + } +} + +#[test] +fn ptr_metadata_bounds() { + fn metadata_eq_method_address() -> usize { + // The `Metadata` associated type has an `Ord` bound, so this is valid: + <::Metadata as PartialEq>::eq as usize + } + // "Synthetic" trait impls generated by the compiler like those of `Pointee` + // are not checked for bounds of associated type. + // So with a buggy core we could have both: + // * `::Metadata == DynMetadata` + // * `DynMetadata: !PartialEq` + // … and cause an ICE here: + metadata_eq_method_address::(); + + // For this reason, let’s check here that bounds are satisfied: + + let _ = static_assert_expected_bounds_for_metadata::<()>; + let _ = static_assert_expected_bounds_for_metadata::; + let _ = static_assert_expected_bounds_for_metadata::>; + fn _static_assert_associated_type() { + let _ = static_assert_expected_bounds_for_metadata::<::Metadata>; + } + + fn static_assert_expected_bounds_for_metadata() + where + // Keep this in sync with the associated type in `library/core/src/ptr/metadata.rs` + Meta: Copy + Send + Sync + Ord + std::hash::Hash + Unpin, + { + } +} + +#[test] +fn dyn_metadata() { + #[derive(Debug)] + #[repr(align(32))] + struct Something([u8; 47]); + + let value = Something([0; 47]); + let trait_object: &dyn Debug = &value; + let meta = metadata(trait_object); + + assert_eq!(meta.size_of(), 64); + assert_eq!(meta.size_of(), std::mem::size_of::()); + assert_eq!(meta.align_of(), 32); + assert_eq!(meta.align_of(), std::mem::align_of::()); + assert_eq!(meta.layout(), std::alloc::Layout::new::()); + + assert!(format!("{meta:?}").starts_with("DynMetadata(0x")); +} + +#[test] +fn from_raw_parts() { + let mut value = 5_u32; + let address = &mut value as *mut _ as *mut (); + let trait_object: &dyn Display = &mut value; + let vtable = metadata(trait_object); + let trait_object = NonNull::from(trait_object); + + assert_eq!(ptr::from_raw_parts(address, vtable), trait_object.as_ptr()); + assert_eq!(ptr::from_raw_parts_mut(address, vtable), trait_object.as_ptr()); + assert_eq!(NonNull::from_raw_parts(NonNull::new(address).unwrap(), vtable), trait_object); + + let mut array = [5_u32, 5, 5, 5, 5]; + let address = &mut array as *mut _ as *mut (); + let array_ptr = NonNull::from(&mut array); + let slice_ptr = NonNull::from(&mut array[..]); + + assert_eq!(ptr::from_raw_parts(address, ()), array_ptr.as_ptr()); + assert_eq!(ptr::from_raw_parts_mut(address, ()), array_ptr.as_ptr()); + assert_eq!(NonNull::from_raw_parts(NonNull::new(address).unwrap(), ()), array_ptr); + + assert_eq!(ptr::from_raw_parts(address, 5), slice_ptr.as_ptr()); + assert_eq!(ptr::from_raw_parts_mut(address, 5), slice_ptr.as_ptr()); + assert_eq!(NonNull::from_raw_parts(NonNull::new(address).unwrap(), 5), slice_ptr); +} + +#[test] +fn thin_box() { + let foo = ThinBox::::new(4); + assert_eq!(foo.to_string(), "4"); + drop(foo); + let bar = ThinBox::::new(7); + assert_eq!(bar.to_string(), "7"); + + // A slightly more interesting library that could be built on top of metadata APIs. + // + // * It could be generalized to any `T: ?Sized` (not just trait object) + // if `{size,align}_of_for_meta(T::Metadata)` are added. + // * Constructing a `ThinBox` without consuming and deallocating a `Box` + // requires either the unstable `Unsize` marker trait, + // or the unstable `unsized_locals` language feature, + // or taking `&dyn T` and restricting to `T: Copy`. + + use std::alloc::*; + use std::marker::PhantomData; + + struct ThinBox + where + T: ?Sized + Pointee>, + { + ptr: NonNull>, + phantom: PhantomData, + } + + impl ThinBox + where + T: ?Sized + Pointee>, + { + pub fn new>(value: Value) -> Self { + let unsized_: &T = &value; + let meta = metadata(unsized_); + let meta_layout = Layout::for_value(&meta); + let value_layout = Layout::for_value(&value); + let (layout, offset) = meta_layout.extend(value_layout).unwrap(); + // `DynMetadata` is pointer-sized: + assert!(layout.size() > 0); + // If `ThinBox` is generalized to any `T: ?Sized`, + // handle ZSTs with a dangling pointer without going through `alloc()`, + // like `Box` does. + unsafe { + let ptr = NonNull::new(alloc(layout)) + .unwrap_or_else(|| handle_alloc_error(layout)) + .cast::>(); + ptr.as_ptr().write(meta); + ptr.as_ptr().byte_add(offset).cast::().write(value); + Self { ptr, phantom: PhantomData } + } + } + + fn meta(&self) -> DynMetadata { + unsafe { *self.ptr.as_ref() } + } + + fn layout(&self) -> (Layout, usize) { + let meta = self.meta(); + Layout::for_value(&meta).extend(meta.layout()).unwrap() + } + + fn value_ptr(&self) -> *const T { + let (_, offset) = self.layout(); + let data_ptr = unsafe { self.ptr.cast::().as_ptr().add(offset) }; + ptr::from_raw_parts(data_ptr.cast(), self.meta()) + } + + fn value_mut_ptr(&mut self) -> *mut T { + let (_, offset) = self.layout(); + // FIXME: can this line be shared with the same in `value_ptr()` + // without upsetting Stacked Borrows? + let data_ptr = unsafe { self.ptr.cast::().as_ptr().add(offset) }; + from_raw_parts_mut(data_ptr.cast(), self.meta()) + } + } + + impl std::ops::Deref for ThinBox + where + T: ?Sized + Pointee>, + { + type Target = T; + + fn deref(&self) -> &T { + unsafe { &*self.value_ptr() } + } + } + + impl std::ops::DerefMut for ThinBox + where + T: ?Sized + Pointee>, + { + fn deref_mut(&mut self) -> &mut T { + unsafe { &mut *self.value_mut_ptr() } + } + } + + impl std::ops::Drop for ThinBox + where + T: ?Sized + Pointee>, + { + fn drop(&mut self) { + let (layout, _) = self.layout(); + unsafe { + drop_in_place::(&mut **self); + dealloc(self.ptr.cast().as_ptr(), layout); + } + } + } +} + +#[test] +fn nonnull_tagged_pointer_with_provenance() { + let raw_pointer = Box::into_raw(Box::new(10)); + + let mut p = TaggedPointer::new(raw_pointer).unwrap(); + assert_eq!(p.tag(), 0); + + p.set_tag(1); + assert_eq!(p.tag(), 1); + assert_eq!(unsafe { *p.pointer().as_ptr() }, 10); + + p.set_tag(3); + assert_eq!(p.tag(), 3); + assert_eq!(unsafe { *p.pointer().as_ptr() }, 10); + + unsafe { Box::from_raw(p.pointer().as_ptr()) }; + + /// A non-null pointer type which carries several bits of metadata and maintains provenance. + #[repr(transparent)] + pub struct TaggedPointer(NonNull); + + impl Clone for TaggedPointer { + fn clone(&self) -> Self { + Self(self.0) + } + } + + impl Copy for TaggedPointer {} + + impl TaggedPointer { + /// The ABI-required minimum alignment of the `P` type. + pub const ALIGNMENT: usize = core::mem::align_of::(); + /// A mask for data-carrying bits of the address. + pub const DATA_MASK: usize = !Self::ADDRESS_MASK; + /// Number of available bits of storage in the address. + pub const NUM_BITS: u32 = Self::ALIGNMENT.trailing_zeros(); + /// A mask for the non-data-carrying bits of the address. + pub const ADDRESS_MASK: usize = usize::MAX << Self::NUM_BITS; + + /// Create a new tagged pointer from a possibly null pointer. + pub fn new(pointer: *mut T) -> Option> { + Some(TaggedPointer(NonNull::new(pointer)?)) + } + + /// Consume this tagged pointer and produce a raw mutable pointer to the + /// memory location. + pub fn pointer(self) -> NonNull { + // SAFETY: The `addr` guaranteed to have bits set in the Self::ADDRESS_MASK, so the result will be non-null. + self.0.map_addr(|addr| unsafe { + NonZeroUsize::new_unchecked(addr.get() & Self::ADDRESS_MASK) + }) + } + + /// Consume this tagged pointer and produce the data it carries. + pub fn tag(&self) -> usize { + self.0.addr().get() & Self::DATA_MASK + } + + /// Update the data this tagged pointer carries to a new value. + pub fn set_tag(&mut self, data: usize) { + assert_eq!( + data & Self::ADDRESS_MASK, + 0, + "cannot set more data beyond the lowest NUM_BITS" + ); + let data = data & Self::DATA_MASK; + + // SAFETY: This value will always be non-zero because the upper bits (from + // ADDRESS_MASK) will always be non-zero. This a property of the type and its + // construction. + self.0 = self.0.map_addr(|addr| unsafe { + NonZeroUsize::new_unchecked((addr.get() & Self::ADDRESS_MASK) | data) + }) + } + } +} + +#[test] +fn swap_copy_untyped() { + // We call `{swap,copy}{,_nonoverlapping}` at `bool` type on data that is not a valid bool. + // These should all do untyped copies, so this should work fine. + let mut x = 5u8; + let mut y = 6u8; + + let ptr1 = &mut x as *mut u8 as *mut bool; + let ptr2 = &mut y as *mut u8 as *mut bool; + + unsafe { + ptr::swap(ptr1, ptr2); + ptr::swap_nonoverlapping(ptr1, ptr2, 1); + } + assert_eq!(x, 5); + assert_eq!(y, 6); + + unsafe { + ptr::copy(ptr1, ptr2, 1); + ptr::copy_nonoverlapping(ptr1, ptr2, 1); + } + assert_eq!(x, 5); + assert_eq!(y, 5); +} + +#[test] +fn test_const_copy() { + const { + let ptr1 = &1; + let mut ptr2 = &666; + + // Copy ptr1 to ptr2, bytewise. + unsafe { + ptr::copy( + &ptr1 as *const _ as *const MaybeUninit, + &mut ptr2 as *mut _ as *mut MaybeUninit, + mem::size_of::<&i32>(), + ); + } + + // Make sure they still work. + assert!(*ptr1 == 1); + assert!(*ptr2 == 1); + }; + + const { + let ptr1 = &1; + let mut ptr2 = &666; + + // Copy ptr1 to ptr2, bytewise. + unsafe { + ptr::copy_nonoverlapping( + &ptr1 as *const _ as *const MaybeUninit, + &mut ptr2 as *mut _ as *mut MaybeUninit, + mem::size_of::<&i32>(), + ); + } + + // Make sure they still work. + assert!(*ptr1 == 1); + assert!(*ptr2 == 1); + }; +} diff --git a/crux-mir/lib/core/tests/result.rs b/crux-mir/lib/core/tests/result.rs index c835313aa..50926da3c 100644 --- a/crux-mir/lib/core/tests/result.rs +++ b/crux-mir/lib/core/tests/result.rs @@ -1,4 +1,3 @@ -use core::array::FixedSizeArray; use core::ops::DerefMut; use core::option::*; @@ -70,7 +69,7 @@ fn test_collect() { // test that it does not take more elements than it needs let mut functions: [Box Result<(), isize>>; 3] = - [box || Ok(()), box || Err(1), box || panic!()]; + [Box::new(|| Ok(())), Box::new(|| Err(1)), Box::new(|| panic!())]; let v: Result, isize> = functions.iter_mut().map(|f| (*f)()).collect(); assert!(v == Err(1)); @@ -81,9 +80,9 @@ fn test_fmt_default() { let ok: Result = Ok(100); let err: Result = Err("Err"); - let s = format!("{:?}", ok); + let s = format!("{ok:?}"); assert_eq!(s, "Ok(100)"); - let s = format!("{:?}", err); + let s = format!("{err:?}"); assert_eq!(s, "Err(\"Err\")"); } @@ -120,6 +119,18 @@ pub fn test_unwrap_or_else_panic() { let _: isize = bad_err.unwrap_or_else(handler); } +#[test] +fn test_unwrap_unchecked() { + let ok: Result = Ok(100); + assert_eq!(unsafe { ok.unwrap_unchecked() }, 100); +} + +#[test] +fn test_unwrap_err_unchecked() { + let ok_err: Result = Err("Err"); + assert_eq!(unsafe { ok_err.unwrap_err_unchecked() }, "Err"); +} + #[test] pub fn test_expect_ok() { let ok: Result = Ok(100); @@ -206,27 +217,37 @@ pub fn test_into_ok() { } #[test] -fn test_try() { - fn try_result_some() -> Option { - let val = Ok(1)?; - Some(val) +pub fn test_into_err() { + fn until_error_op() -> Result { + Err(666) + } + + assert_eq!(until_error_op().into_err(), 666); + + enum MyNeverToken {} + impl From for ! { + fn from(never: MyNeverToken) -> ! { + match never {} + } } - assert_eq!(try_result_some(), Some(1)); - fn try_result_none() -> Option { - let val = Err(NoneError)?; - Some(val) + fn until_error_op2() -> Result { + Err(667) } - assert_eq!(try_result_none(), None); - fn try_result_ok() -> Result { + assert_eq!(until_error_op2().into_err(), 667); +} + +#[test] +fn test_try() { + fn try_result_ok() -> Result { let result: Result = Ok(1); let val = result?; Ok(val) } assert_eq!(try_result_ok(), Ok(1)); - fn try_result_err() -> Result { + fn try_result_err() -> Result { let result: Result = Err(1); let val = result?; Ok(val) @@ -250,24 +271,11 @@ fn test_result_as_deref() { let expected_result = Result::Ok::<&[i32], &u32>([1, 2, 3, 4, 5].as_slice()); assert_eq!(ref_ok.as_deref(), expected_result); - // &Result::Err(T).as_deref_err() -> - // Result<&T, &E::Deref::Target>::Err(&*E) - let ref_err = &Result::Err::(&41); - let expected_result = Result::Err::<&u8, &i32>(&41); - assert_eq!(ref_err.as_deref_err(), expected_result); - - let ref_err = &Result::Err::(String::from("an error")); - let expected_result = Result::Err::<&u32, &str>("an error"); - assert_eq!(ref_err.as_deref_err(), expected_result); - - let ref_err = &Result::Err::>(vec![5, 4, 3, 2, 1]); - let expected_result = Result::Err::<&u32, &[i32]>([5, 4, 3, 2, 1].as_slice()); - assert_eq!(ref_err.as_deref_err(), expected_result); - - // &Result::Err(T).as_deref_err() -> - // Result<&T, &E::Deref::Target>::Err(&*E) - let ref_err = &Result::Err::<&u8, &i32>(&41); - let expected_result = Result::Err::<&u8, &&i32>(&&41); + // &Result::Err(T).as_deref() -> + // Result<&T::Deref::Target, &E>::Err(&*E) + let val = 41; + let ref_err = &Result::Err::<&u8, i32>(val); + let expected_result = Result::Err::<&u8, &i32>(&val); assert_eq!(ref_err.as_deref(), expected_result); let s = String::from("an error"); @@ -279,46 +287,12 @@ fn test_result_as_deref() { let ref_err = &Result::Err::<&u32, Vec>(v.clone()); let expected_result = Result::Err::<&u32, &Vec>(&v); assert_eq!(ref_err.as_deref(), expected_result); - - // The following cases test calling `as_deref_*` with the wrong variant (i.e. - // `as_deref()` with a `Result::Err()`, or `as_deref_err()` with a `Result::Ok()`. - // While uncommon, these cases are supported to ensure that an `as_deref_*` - // call can still be made even when one of the Result types does not implement - // `Deref` (for example, std::io::Error). - - // &Result::Ok(T).as_deref_err() -> - // Result<&T, &E::Deref::Target>::Ok(&T) - let ref_ok = &Result::Ok::(42); - let expected_result = Result::Ok::<&i32, &u8>(&42); - assert_eq!(ref_ok.as_deref_err(), expected_result); - - let ref_ok = &Result::Ok::<&str, &u32>("a result"); - let expected_result = Result::Ok::<&&str, &u32>(&"a result"); - assert_eq!(ref_ok.as_deref_err(), expected_result); - - let ref_ok = &Result::Ok::<[i32; 5], &u32>([1, 2, 3, 4, 5]); - let expected_result = Result::Ok::<&[i32; 5], &u32>(&[1, 2, 3, 4, 5]); - assert_eq!(ref_ok.as_deref_err(), expected_result); - - // &Result::Err(E).as_deref() -> - // Result<&T::Deref::Target, &E>::Err(&E) - let ref_err = &Result::Err::<&u8, i32>(41); - let expected_result = Result::Err::<&u8, &i32>(&41); - assert_eq!(ref_err.as_deref(), expected_result); - - let ref_err = &Result::Err::<&u32, &str>("an error"); - let expected_result = Result::Err::<&u32, &&str>(&"an error"); - assert_eq!(ref_err.as_deref(), expected_result); - - let ref_err = &Result::Err::<&u32, [i32; 5]>([5, 4, 3, 2, 1]); - let expected_result = Result::Err::<&u32, &[i32; 5]>(&[5, 4, 3, 2, 1]); - assert_eq!(ref_err.as_deref(), expected_result); } #[test] fn test_result_as_deref_mut() { - // &mut Result::Ok(T).as_deref_mut() -> - // Result<&mut T::Deref::Target, &mut E>::Ok(&mut *T) + // &mut Result::Ok(T).as_deref_mut() -> + // Result<&mut T::DerefMut::Target, &mut E>::Ok(&mut *T) let mut val = 42; let mut expected_val = 42; let mut_ok = &mut Result::Ok::<&mut i32, u8>(&mut val); @@ -335,26 +309,8 @@ fn test_result_as_deref_mut() { let expected_result = Result::Ok::<&mut [i32], &mut u32>(expected_vec.as_mut_slice()); assert_eq!(mut_ok.as_deref_mut(), expected_result); - // &mut Result::Err(T).as_deref_mut_err() -> - // Result<&mut T, &mut E::Deref::Target>::Err(&mut *E) - let mut val = 41; - let mut expected_val = 41; - let mut_err = &mut Result::Err::(&mut val); - let expected_result = Result::Err::<&mut u8, &mut i32>(&mut expected_val); - assert_eq!(mut_err.as_deref_mut_err(), expected_result); - - let mut expected_string = String::from("an error"); - let mut_err = &mut Result::Err::(expected_string.clone()); - let expected_result = Result::Err::<&mut u32, &mut str>(expected_string.deref_mut()); - assert_eq!(mut_err.as_deref_mut_err(), expected_result); - - let mut expected_vec = vec![5, 4, 3, 2, 1]; - let mut_err = &mut Result::Err::>(expected_vec.clone()); - let expected_result = Result::Err::<&mut u32, &mut [i32]>(expected_vec.as_mut_slice()); - assert_eq!(mut_err.as_deref_mut_err(), expected_result); - - // &mut Result::Err(T).as_deref_mut_err() -> - // Result<&mut T, &mut E::Deref::Target>::Err(&mut *E) + // &mut Result::Err(T).as_deref_mut() -> + // Result<&mut T, &mut E>::Err(&mut *E) let mut val = 41; let mut_err = &mut Result::Err::<&mut u8, i32>(val); let expected_result = Result::Err::<&mut u8, &mut i32>(&mut val); @@ -369,48 +325,94 @@ fn test_result_as_deref_mut() { let mut_err = &mut Result::Err::<&mut u32, Vec>(expected_vec.clone()); let expected_result = Result::Err::<&mut u32, &mut Vec>(&mut expected_vec); assert_eq!(mut_err.as_deref_mut(), expected_result); +} - // The following cases test calling `as_deref_mut_*` with the wrong variant (i.e. - // `as_deref_mut()` with a `Result::Err()`, or `as_deref_mut_err()` with a `Result::Ok()`. - // While uncommon, these cases are supported to ensure that an `as_deref_mut_*` - // call can still be made even when one of the Result types does not implement - // `Deref` (for example, std::io::Error). +#[test] +fn result_const() { + // test that the methods of `Result` are usable in a const context - // &mut Result::Ok(T).as_deref_mut_err() -> - // Result<&mut T, &mut E::Deref::Target>::Ok(&mut T) - let mut expected_val = 42; - let mut_ok = &mut Result::Ok::(expected_val.clone()); - let expected_result = Result::Ok::<&mut i32, &mut u8>(&mut expected_val); - assert_eq!(mut_ok.as_deref_mut_err(), expected_result); - - let string = String::from("a result"); - let expected_string = string.clone(); - let mut ref_str = expected_string.as_ref(); - let mut_ok = &mut Result::Ok::<&str, &mut u32>(string.as_str()); - let expected_result = Result::Ok::<&mut &str, &mut u32>(&mut ref_str); - assert_eq!(mut_ok.as_deref_mut_err(), expected_result); - - let mut expected_arr = [1, 2, 3, 4, 5]; - let mut_ok = &mut Result::Ok::<[i32; 5], &mut u32>(expected_arr.clone()); - let expected_result = Result::Ok::<&mut [i32; 5], &mut u32>(&mut expected_arr); - assert_eq!(mut_ok.as_deref_mut_err(), expected_result); - - // &mut Result::Err(E).as_deref_mut() -> - // Result<&mut T::Deref::Target, &mut E>::Err(&mut E) - let mut expected_val = 41; - let mut_err = &mut Result::Err::<&mut u8, i32>(expected_val.clone()); - let expected_result = Result::Err::<&mut u8, &mut i32>(&mut expected_val); - assert_eq!(mut_err.as_deref_mut(), expected_result); + const RESULT: Result = Ok(32); - let string = String::from("an error"); - let expected_string = string.clone(); - let mut ref_str = expected_string.as_ref(); - let mut_err = &mut Result::Err::<&mut u32, &str>(string.as_str()); - let expected_result = Result::Err::<&mut u32, &mut &str>(&mut ref_str); - assert_eq!(mut_err.as_deref_mut(), expected_result); + const REF: Result<&usize, &bool> = RESULT.as_ref(); + assert_eq!(REF, Ok(&32)); - let mut expected_arr = [5, 4, 3, 2, 1]; - let mut_err = &mut Result::Err::<&mut u32, [i32; 5]>(expected_arr.clone()); - let expected_result = Result::Err::<&mut u32, &mut [i32; 5]>(&mut expected_arr); - assert_eq!(mut_err.as_deref_mut(), expected_result); + const IS_OK: bool = RESULT.is_ok(); + assert!(IS_OK); + + const IS_ERR: bool = RESULT.is_err(); + assert!(!IS_ERR) +} + +#[test] +const fn result_const_mut() { + let mut result: Result = Ok(32); + + { + let as_mut = result.as_mut(); + match as_mut { + Ok(v) => *v = 42, + Err(_) => unreachable!(), + } + } + + let mut result_err: Result = Err(false); + + { + let as_mut = result_err.as_mut(); + match as_mut { + Ok(_) => unreachable!(), + Err(v) => *v = true, + } + } +} + +#[test] +fn result_opt_conversions() { + #[derive(Copy, Clone, Debug, PartialEq)] + struct BadNumErr; + + fn try_num(x: i32) -> Result { + if x <= 5 { Ok(x + 1) } else { Err(BadNumErr) } + } + + type ResOpt = Result, BadNumErr>; + type OptRes = Option>; + + let mut x: ResOpt = Ok(Some(5)); + let mut y: OptRes = Some(Ok(5)); + assert_eq!(x, y.transpose()); + assert_eq!(x.transpose(), y); + + x = Ok(None); + y = None; + assert_eq!(x, y.transpose()); + assert_eq!(x.transpose(), y); + + x = Err(BadNumErr); + y = Some(Err(BadNumErr)); + assert_eq!(x, y.transpose()); + assert_eq!(x.transpose(), y); + + let res: Result, BadNumErr> = (0..10) + .map(|x| { + let y = try_num(x)?; + Ok(if y % 2 == 0 { Some(y - 1) } else { None }) + }) + .filter_map(Result::transpose) + .collect(); + + assert_eq!(res, Err(BadNumErr)) +} + +#[test] +fn result_try_trait_v2_branch() { + use core::num::NonZeroU32; + use core::ops::{ControlFlow::*, Try}; + assert_eq!(Ok::(4).branch(), Continue(4)); + assert_eq!(Err::(4).branch(), Break(Err(4))); + let one = NonZeroU32::new(1).unwrap(); + assert_eq!(Ok::<(), NonZeroU32>(()).branch(), Continue(())); + assert_eq!(Err::<(), NonZeroU32>(one).branch(), Break(Err(one))); + assert_eq!(Ok::(one).branch(), Continue(one)); + assert_eq!(Err::(()).branch(), Break(Err(()))); } diff --git a/crux-mir/lib/core/tests/simd.rs b/crux-mir/lib/core/tests/simd.rs new file mode 100644 index 000000000..565c8975e --- /dev/null +++ b/crux-mir/lib/core/tests/simd.rs @@ -0,0 +1,14 @@ +use core::simd::f32x4; +use core::simd::SimdFloat; + +#[test] +fn testing() { + let x = f32x4::from_array([1.0, 1.0, 1.0, 1.0]); + let y = -x; + + let h = x * f32x4::splat(0.5); + + let r = y.abs(); + assert_eq!(x, r); + assert_eq!(h, f32x4::splat(0.5)); +} diff --git a/crux-mir/lib/core/tests/slice.rs b/crux-mir/lib/core/tests/slice.rs index dbab433e3..39559cdbb 100644 --- a/crux-mir/lib/core/tests/slice.rs +++ b/crux-mir/lib/core/tests/slice.rs @@ -1,4 +1,8 @@ +use core::cell::Cell; +use core::cmp::Ordering; +use core::mem::MaybeUninit; use core::result::Result::{Err, Ok}; +use core::slice; #[test] fn test_position() { @@ -63,6 +67,17 @@ fn test_binary_search() { assert_eq!(b.binary_search(&6), Err(4)); assert_eq!(b.binary_search(&7), Ok(4)); assert_eq!(b.binary_search(&8), Err(5)); + + let b = [(); usize::MAX]; + assert_eq!(b.binary_search(&()), Ok(usize::MAX / 2)); +} + +#[test] +fn test_binary_search_by_overflow() { + let b = [(); usize::MAX]; + assert_eq!(b.binary_search_by(|_| Ordering::Equal), Ok(usize::MAX / 2)); + assert_eq!(b.binary_search_by(|_| Ordering::Greater), Err(0)); + assert_eq!(b.binary_search_by(|_| Ordering::Less), Err(usize::MAX)); } #[test] @@ -72,13 +87,97 @@ fn test_binary_search_implementation_details() { let b = [1, 1, 2, 2, 3, 3, 3]; assert_eq!(b.binary_search(&1), Ok(1)); assert_eq!(b.binary_search(&2), Ok(3)); - assert_eq!(b.binary_search(&3), Ok(6)); + assert_eq!(b.binary_search(&3), Ok(5)); let b = [1, 1, 1, 1, 1, 3, 3, 3, 3]; assert_eq!(b.binary_search(&1), Ok(4)); - assert_eq!(b.binary_search(&3), Ok(8)); + assert_eq!(b.binary_search(&3), Ok(7)); let b = [1, 1, 1, 1, 3, 3, 3, 3, 3]; - assert_eq!(b.binary_search(&1), Ok(3)); - assert_eq!(b.binary_search(&3), Ok(8)); + assert_eq!(b.binary_search(&1), Ok(2)); + assert_eq!(b.binary_search(&3), Ok(4)); +} + +#[test] +fn test_partition_point() { + let b: [i32; 0] = []; + assert_eq!(b.partition_point(|&x| x < 5), 0); + + let b = [4]; + assert_eq!(b.partition_point(|&x| x < 3), 0); + assert_eq!(b.partition_point(|&x| x < 4), 0); + assert_eq!(b.partition_point(|&x| x < 5), 1); + + let b = [1, 2, 4, 6, 8, 9]; + assert_eq!(b.partition_point(|&x| x < 5), 3); + assert_eq!(b.partition_point(|&x| x < 6), 3); + assert_eq!(b.partition_point(|&x| x < 7), 4); + assert_eq!(b.partition_point(|&x| x < 8), 4); + + let b = [1, 2, 4, 5, 6, 8]; + assert_eq!(b.partition_point(|&x| x < 9), 6); + + let b = [1, 2, 4, 6, 7, 8, 9]; + assert_eq!(b.partition_point(|&x| x < 6), 3); + assert_eq!(b.partition_point(|&x| x < 5), 3); + assert_eq!(b.partition_point(|&x| x < 8), 5); + + let b = [1, 2, 4, 5, 6, 8, 9]; + assert_eq!(b.partition_point(|&x| x < 7), 5); + assert_eq!(b.partition_point(|&x| x < 0), 0); + + let b = [1, 3, 3, 3, 7]; + assert_eq!(b.partition_point(|&x| x < 0), 0); + assert_eq!(b.partition_point(|&x| x < 1), 0); + assert_eq!(b.partition_point(|&x| x < 2), 1); + assert_eq!(b.partition_point(|&x| x < 3), 1); + assert_eq!(b.partition_point(|&x| x < 4), 4); + assert_eq!(b.partition_point(|&x| x < 5), 4); + assert_eq!(b.partition_point(|&x| x < 6), 4); + assert_eq!(b.partition_point(|&x| x < 7), 4); + assert_eq!(b.partition_point(|&x| x < 8), 5); +} + +#[test] +fn test_iterator_advance_by() { + let v = &[0, 1, 2, 3, 4]; + + for i in 0..=v.len() { + let mut iter = v.iter(); + iter.advance_by(i).unwrap(); + assert_eq!(iter.as_slice(), &v[i..]); + } + + let mut iter = v.iter(); + assert_eq!(iter.advance_by(v.len() + 1), Err(v.len())); + assert_eq!(iter.as_slice(), &[]); + + let mut iter = v.iter(); + iter.advance_by(3).unwrap(); + assert_eq!(iter.as_slice(), &v[3..]); + iter.advance_by(2).unwrap(); + assert_eq!(iter.as_slice(), &[]); + iter.advance_by(0).unwrap(); +} + +#[test] +fn test_iterator_advance_back_by() { + let v = &[0, 1, 2, 3, 4]; + + for i in 0..=v.len() { + let mut iter = v.iter(); + iter.advance_back_by(i).unwrap(); + assert_eq!(iter.as_slice(), &v[..v.len() - i]); + } + + let mut iter = v.iter(); + assert_eq!(iter.advance_back_by(v.len() + 1), Err(v.len())); + assert_eq!(iter.as_slice(), &[]); + + let mut iter = v.iter(); + iter.advance_back_by(3).unwrap(); + assert_eq!(iter.as_slice(), &v[..v.len() - 3]); + iter.advance_back_by(2).unwrap(); + assert_eq!(iter.as_slice(), &[]); + iter.advance_back_by(0).unwrap(); } #[test] @@ -153,6 +252,40 @@ fn test_chunks_nth() { assert_eq!(c2.next(), None); } +#[test] +fn test_chunks_next() { + let v = [0, 1, 2, 3, 4, 5]; + let mut c = v.chunks(2); + assert_eq!(c.next().unwrap(), &[0, 1]); + assert_eq!(c.next().unwrap(), &[2, 3]); + assert_eq!(c.next().unwrap(), &[4, 5]); + assert_eq!(c.next(), None); + + let v = [0, 1, 2, 3, 4, 5, 6, 7]; + let mut c = v.chunks(3); + assert_eq!(c.next().unwrap(), &[0, 1, 2]); + assert_eq!(c.next().unwrap(), &[3, 4, 5]); + assert_eq!(c.next().unwrap(), &[6, 7]); + assert_eq!(c.next(), None); +} + +#[test] +fn test_chunks_next_back() { + let v = [0, 1, 2, 3, 4, 5]; + let mut c = v.chunks(2); + assert_eq!(c.next_back().unwrap(), &[4, 5]); + assert_eq!(c.next_back().unwrap(), &[2, 3]); + assert_eq!(c.next_back().unwrap(), &[0, 1]); + assert_eq!(c.next_back(), None); + + let v = [0, 1, 2, 3, 4, 5, 6, 7]; + let mut c = v.chunks(3); + assert_eq!(c.next_back().unwrap(), &[6, 7]); + assert_eq!(c.next_back().unwrap(), &[3, 4, 5]); + assert_eq!(c.next_back().unwrap(), &[0, 1, 2]); + assert_eq!(c.next_back(), None); +} + #[test] fn test_chunks_nth_back() { let v: &[i32] = &[0, 1, 2, 3, 4, 5]; @@ -276,6 +409,50 @@ fn test_chunks_mut_zip() { assert_eq!(v1, [13, 14, 19, 20, 14]); } +#[test] +fn test_chunks_mut_zip_aliasing() { + let v1: &mut [i32] = &mut [0, 1, 2, 3, 4]; + let v2: &[i32] = &[6, 7, 8, 9, 10]; + + let mut it = v1.chunks_mut(2).zip(v2.chunks(2)); + let first = it.next().unwrap(); + let _ = it.next().unwrap(); + assert_eq!(first, (&mut [0, 1][..], &[6, 7][..])); +} + +#[test] +fn test_chunks_exact_mut_zip_aliasing() { + let v1: &mut [i32] = &mut [0, 1, 2, 3, 4]; + let v2: &[i32] = &[6, 7, 8, 9, 10]; + + let mut it = v1.chunks_exact_mut(2).zip(v2.chunks(2)); + let first = it.next().unwrap(); + let _ = it.next().unwrap(); + assert_eq!(first, (&mut [0, 1][..], &[6, 7][..])); +} + +#[test] +fn test_rchunks_mut_zip_aliasing() { + let v1: &mut [i32] = &mut [0, 1, 2, 3, 4]; + let v2: &[i32] = &[6, 7, 8, 9, 10]; + + let mut it = v1.rchunks_mut(2).zip(v2.chunks(2)); + let first = it.next().unwrap(); + let _ = it.next().unwrap(); + assert_eq!(first, (&mut [3, 4][..], &[6, 7][..])); +} + +#[test] +fn test_rchunks_exact_mut_zip_aliasing() { + let v1: &mut [i32] = &mut [0, 1, 2, 3, 4]; + let v2: &[i32] = &[6, 7, 8, 9, 10]; + + let mut it = v1.rchunks_exact_mut(2).zip(v2.chunks(2)); + let first = it.next().unwrap(); + let _ = it.next().unwrap(); + assert_eq!(first, (&mut [3, 4][..], &[6, 7][..])); +} + #[test] fn test_chunks_exact_count() { let v: &[i32] = &[0, 1, 2, 3, 4, 5]; @@ -433,6 +610,243 @@ fn test_chunks_exact_mut_zip() { assert_eq!(v1, [13, 14, 19, 20, 4]); } +#[test] +fn test_array_chunks_infer() { + let v: &[i32] = &[0, 1, 2, 3, 4, -4]; + let c = v.array_chunks(); + for &[a, b, c] in c { + assert_eq!(a + b + c, 3); + } + + let v2: &[i32] = &[0, 1, 2, 3, 4, 5, 6]; + let total = v2.array_chunks().map(|&[a, b]| a * b).sum::(); + assert_eq!(total, 2 * 3 + 4 * 5); +} + +#[test] +fn test_array_chunks_count() { + let v: &[i32] = &[0, 1, 2, 3, 4, 5]; + let c = v.array_chunks::<3>(); + assert_eq!(c.count(), 2); + + let v2: &[i32] = &[0, 1, 2, 3, 4]; + let c2 = v2.array_chunks::<2>(); + assert_eq!(c2.count(), 2); + + let v3: &[i32] = &[]; + let c3 = v3.array_chunks::<2>(); + assert_eq!(c3.count(), 0); +} + +#[test] +fn test_array_chunks_nth() { + let v: &[i32] = &[0, 1, 2, 3, 4, 5]; + let mut c = v.array_chunks::<2>(); + assert_eq!(c.nth(1).unwrap(), &[2, 3]); + assert_eq!(c.next().unwrap(), &[4, 5]); + + let v2: &[i32] = &[0, 1, 2, 3, 4, 5, 6]; + let mut c2 = v2.array_chunks::<3>(); + assert_eq!(c2.nth(1).unwrap(), &[3, 4, 5]); + assert_eq!(c2.next(), None); +} + +#[test] +fn test_array_chunks_nth_back() { + let v: &[i32] = &[0, 1, 2, 3, 4, 5]; + let mut c = v.array_chunks::<2>(); + assert_eq!(c.nth_back(1).unwrap(), &[2, 3]); + assert_eq!(c.next().unwrap(), &[0, 1]); + assert_eq!(c.next(), None); + + let v2: &[i32] = &[0, 1, 2, 3, 4]; + let mut c2 = v2.array_chunks::<3>(); + assert_eq!(c2.nth_back(0).unwrap(), &[0, 1, 2]); + assert_eq!(c2.next(), None); + assert_eq!(c2.next_back(), None); + + let v3: &[i32] = &[0, 1, 2, 3, 4]; + let mut c3 = v3.array_chunks::<10>(); + assert_eq!(c3.nth_back(0), None); +} + +#[test] +fn test_array_chunks_last() { + let v: &[i32] = &[0, 1, 2, 3, 4, 5]; + let c = v.array_chunks::<2>(); + assert_eq!(c.last().unwrap(), &[4, 5]); + + let v2: &[i32] = &[0, 1, 2, 3, 4]; + let c2 = v2.array_chunks::<2>(); + assert_eq!(c2.last().unwrap(), &[2, 3]); +} + +#[test] +fn test_array_chunks_remainder() { + let v: &[i32] = &[0, 1, 2, 3, 4]; + let c = v.array_chunks::<2>(); + assert_eq!(c.remainder(), &[4]); +} + +#[test] +fn test_array_chunks_zip() { + let v1: &[i32] = &[0, 1, 2, 3, 4]; + let v2: &[i32] = &[6, 7, 8, 9, 10]; + + let res = v1 + .array_chunks::<2>() + .zip(v2.array_chunks::<2>()) + .map(|(a, b)| a.iter().sum::() + b.iter().sum::()) + .collect::>(); + assert_eq!(res, vec![14, 22]); +} + +#[test] +fn test_array_chunks_mut_infer() { + let v: &mut [i32] = &mut [0, 1, 2, 3, 4, 5, 6]; + for a in v.array_chunks_mut() { + let sum = a.iter().sum::(); + *a = [sum; 3]; + } + assert_eq!(v, &[3, 3, 3, 12, 12, 12, 6]); + + let v2: &mut [i32] = &mut [0, 1, 2, 3, 4, 5, 6]; + v2.array_chunks_mut().for_each(|[a, b]| core::mem::swap(a, b)); + assert_eq!(v2, &[1, 0, 3, 2, 5, 4, 6]); +} + +#[test] +fn test_array_chunks_mut_count() { + let v: &mut [i32] = &mut [0, 1, 2, 3, 4, 5]; + let c = v.array_chunks_mut::<3>(); + assert_eq!(c.count(), 2); + + let v2: &mut [i32] = &mut [0, 1, 2, 3, 4]; + let c2 = v2.array_chunks_mut::<2>(); + assert_eq!(c2.count(), 2); + + let v3: &mut [i32] = &mut []; + let c3 = v3.array_chunks_mut::<2>(); + assert_eq!(c3.count(), 0); +} + +#[test] +fn test_array_chunks_mut_nth() { + let v: &mut [i32] = &mut [0, 1, 2, 3, 4, 5]; + let mut c = v.array_chunks_mut::<2>(); + assert_eq!(c.nth(1).unwrap(), &[2, 3]); + assert_eq!(c.next().unwrap(), &[4, 5]); + + let v2: &mut [i32] = &mut [0, 1, 2, 3, 4, 5, 6]; + let mut c2 = v2.array_chunks_mut::<3>(); + assert_eq!(c2.nth(1).unwrap(), &[3, 4, 5]); + assert_eq!(c2.next(), None); +} + +#[test] +fn test_array_chunks_mut_nth_back() { + let v: &mut [i32] = &mut [0, 1, 2, 3, 4, 5]; + let mut c = v.array_chunks_mut::<2>(); + assert_eq!(c.nth_back(1).unwrap(), &[2, 3]); + assert_eq!(c.next().unwrap(), &[0, 1]); + assert_eq!(c.next(), None); + + let v2: &mut [i32] = &mut [0, 1, 2, 3, 4]; + let mut c2 = v2.array_chunks_mut::<3>(); + assert_eq!(c2.nth_back(0).unwrap(), &[0, 1, 2]); + assert_eq!(c2.next(), None); + assert_eq!(c2.next_back(), None); + + let v3: &mut [i32] = &mut [0, 1, 2, 3, 4]; + let mut c3 = v3.array_chunks_mut::<10>(); + assert_eq!(c3.nth_back(0), None); +} + +#[test] +fn test_array_chunks_mut_last() { + let v: &mut [i32] = &mut [0, 1, 2, 3, 4, 5]; + let c = v.array_chunks_mut::<2>(); + assert_eq!(c.last().unwrap(), &[4, 5]); + + let v2: &mut [i32] = &mut [0, 1, 2, 3, 4]; + let c2 = v2.array_chunks_mut::<2>(); + assert_eq!(c2.last().unwrap(), &[2, 3]); +} + +#[test] +fn test_array_chunks_mut_remainder() { + let v: &mut [i32] = &mut [0, 1, 2, 3, 4]; + let c = v.array_chunks_mut::<2>(); + assert_eq!(c.into_remainder(), &[4]); +} + +#[test] +fn test_array_chunks_mut_zip() { + let v1: &mut [i32] = &mut [0, 1, 2, 3, 4]; + let v2: &[i32] = &[6, 7, 8, 9, 10]; + + for (a, b) in v1.array_chunks_mut::<2>().zip(v2.array_chunks::<2>()) { + let sum = b.iter().sum::(); + for v in a { + *v += sum; + } + } + assert_eq!(v1, [13, 14, 19, 20, 4]); +} + +#[test] +fn test_array_windows_infer() { + let v: &[i32] = &[0, 1, 0, 1]; + assert_eq!(v.array_windows::<2>().count(), 3); + let c = v.array_windows(); + for &[a, b] in c { + assert_eq!(a + b, 1); + } + + let v2: &[i32] = &[0, 1, 2, 3, 4, 5, 6]; + let total = v2.array_windows().map(|&[a, b, c]| a + b + c).sum::(); + assert_eq!(total, 3 + 6 + 9 + 12 + 15); +} + +#[test] +fn test_array_windows_count() { + let v: &[i32] = &[0, 1, 2, 3, 4, 5]; + let c = v.array_windows::<3>(); + assert_eq!(c.count(), 4); + + let v2: &[i32] = &[0, 1, 2, 3, 4]; + let c2 = v2.array_windows::<6>(); + assert_eq!(c2.count(), 0); + + let v3: &[i32] = &[]; + let c3 = v3.array_windows::<2>(); + assert_eq!(c3.count(), 0); + + let v4: &[()] = &[(); usize::MAX]; + let c4 = v4.array_windows::<1>(); + assert_eq!(c4.count(), usize::MAX); +} + +#[test] +fn test_array_windows_nth() { + let v: &[i32] = &[0, 1, 2, 3, 4, 5]; + let snd = v.array_windows::<4>().nth(1); + assert_eq!(snd, Some(&[1, 2, 3, 4])); + let mut arr_windows = v.array_windows::<2>(); + assert_ne!(arr_windows.nth(0), arr_windows.nth(0)); + let last = v.array_windows::<3>().last(); + assert_eq!(last, Some(&[3, 4, 5])); +} + +#[test] +fn test_array_windows_nth_back() { + let v: &[i32] = &[0, 1, 2, 3, 4, 5]; + let snd = v.array_windows::<4>().nth_back(1); + assert_eq!(snd, Some(&[1, 2, 3, 4])); + let mut arr_windows = v.array_windows::<2>(); + assert_ne!(arr_windows.nth_back(0), arr_windows.nth_back(0)); +} + #[test] fn test_rchunks_count() { let v: &[i32] = &[0, 1, 2, 3, 4, 5]; @@ -474,6 +888,40 @@ fn test_rchunks_nth_back() { assert_eq!(c2.next_back(), None); } +#[test] +fn test_rchunks_next() { + let v = [0, 1, 2, 3, 4, 5]; + let mut c = v.rchunks(2); + assert_eq!(c.next().unwrap(), &[4, 5]); + assert_eq!(c.next().unwrap(), &[2, 3]); + assert_eq!(c.next().unwrap(), &[0, 1]); + assert_eq!(c.next(), None); + + let v = [0, 1, 2, 3, 4, 5, 6, 7]; + let mut c = v.rchunks(3); + assert_eq!(c.next().unwrap(), &[5, 6, 7]); + assert_eq!(c.next().unwrap(), &[2, 3, 4]); + assert_eq!(c.next().unwrap(), &[0, 1]); + assert_eq!(c.next(), None); +} + +#[test] +fn test_rchunks_next_back() { + let v = [0, 1, 2, 3, 4, 5]; + let mut c = v.rchunks(2); + assert_eq!(c.next_back().unwrap(), &[0, 1]); + assert_eq!(c.next_back().unwrap(), &[2, 3]); + assert_eq!(c.next_back().unwrap(), &[4, 5]); + assert_eq!(c.next_back(), None); + + let v = [0, 1, 2, 3, 4, 5, 6, 7]; + let mut c = v.rchunks(3); + assert_eq!(c.next_back().unwrap(), &[0, 1]); + assert_eq!(c.next_back().unwrap(), &[2, 3, 4]); + assert_eq!(c.next_back().unwrap(), &[5, 6, 7]); + assert_eq!(c.next_back(), None); +} + #[test] fn test_rchunks_last() { let v: &[i32] = &[0, 1, 2, 3, 4, 5]; @@ -539,6 +987,40 @@ fn test_rchunks_mut_nth_back() { assert_eq!(c2.next_back(), None); } +#[test] +fn test_rchunks_mut_next() { + let mut v = [0, 1, 2, 3, 4, 5]; + let mut c = v.rchunks_mut(2); + assert_eq!(c.next().unwrap(), &mut [4, 5]); + assert_eq!(c.next().unwrap(), &mut [2, 3]); + assert_eq!(c.next().unwrap(), &mut [0, 1]); + assert_eq!(c.next(), None); + + let mut v = [0, 1, 2, 3, 4, 5, 6, 7]; + let mut c = v.rchunks_mut(3); + assert_eq!(c.next().unwrap(), &mut [5, 6, 7]); + assert_eq!(c.next().unwrap(), &mut [2, 3, 4]); + assert_eq!(c.next().unwrap(), &mut [0, 1]); + assert_eq!(c.next(), None); +} + +#[test] +fn test_rchunks_mut_next_back() { + let mut v = [0, 1, 2, 3, 4, 5]; + let mut c = v.rchunks_mut(2); + assert_eq!(c.next_back().unwrap(), &mut [0, 1]); + assert_eq!(c.next_back().unwrap(), &mut [2, 3]); + assert_eq!(c.next_back().unwrap(), &mut [4, 5]); + assert_eq!(c.next_back(), None); + + let mut v = [0, 1, 2, 3, 4, 5, 6, 7]; + let mut c = v.rchunks_mut(3); + assert_eq!(c.next_back().unwrap(), &mut [0, 1]); + assert_eq!(c.next_back().unwrap(), &mut [2, 3, 4]); + assert_eq!(c.next_back().unwrap(), &mut [5, 6, 7]); + assert_eq!(c.next_back(), None); +} + #[test] fn test_rchunks_mut_last() { let v: &mut [i32] = &mut [0, 1, 2, 3, 4, 5]; @@ -709,6 +1191,28 @@ fn test_rchunks_exact_mut_zip() { assert_eq!(v1, [0, 16, 17, 22, 23]); } +#[test] +fn chunks_mut_are_send_and_sync() { + use std::cell::Cell; + use std::slice::{ChunksExactMut, ChunksMut, RChunksExactMut, RChunksMut}; + use std::sync::MutexGuard; + + fn assert_send_and_sync() + where + ChunksMut<'static, Cell>: Send, + ChunksMut<'static, MutexGuard<'static, u32>>: Sync, + ChunksExactMut<'static, Cell>: Send, + ChunksExactMut<'static, MutexGuard<'static, u32>>: Sync, + RChunksMut<'static, Cell>: Send, + RChunksMut<'static, MutexGuard<'static, u32>>: Sync, + RChunksExactMut<'static, Cell>: Send, + RChunksExactMut<'static, MutexGuard<'static, u32>>: Sync, + { + } + + assert_send_and_sync(); +} + #[test] fn test_windows_count() { let v: &[i32] = &[0, 1, 2, 3, 4, 5]; @@ -722,6 +1226,10 @@ fn test_windows_count() { let v3: &[i32] = &[]; let c3 = v3.windows(2); assert_eq!(c3.count(), 0); + + let v4 = &[(); usize::MAX]; + let c4 = v4.windows(1); + assert_eq!(c4.count(), usize::MAX); } #[test] @@ -776,7 +1284,6 @@ fn test_windows_zip() { } #[test] -#[allow(const_err)] fn test_iter_ref_consistency() { use std::fmt::Debug; @@ -981,7 +1488,7 @@ mod slice_index { // optional: // // one or more similar inputs for which data[input] succeeds, - // and the corresponding output as an array. This helps validate + // and the corresponding output as an array. This helps validate // "critical points" where an input range straddles the boundary // between valid and invalid. // (such as the input `len..len`, which is just barely valid) @@ -994,6 +1501,9 @@ mod slice_index { } )*) => {$( mod $case_name { + #[allow(unused_imports)] + use core::ops::Bound; + #[test] fn pass() { let mut v = $data; @@ -1048,7 +1558,7 @@ mod slice_index { good: data[6..] == []; bad: data[7..]; - message: "but ends at"; // perhaps not ideal + message: "out of range"; } in mod rangeto_len { @@ -1067,6 +1577,14 @@ mod slice_index { message: "out of range"; } + in mod rangeinclusive_len { + data: [0, 1, 2, 3, 4, 5]; + + good: data[0..=5] == [0, 1, 2, 3, 4, 5]; + bad: data[0..=6]; + message: "out of range"; + } + in mod range_len_len { data: [0, 1, 2, 3, 4, 5]; @@ -1082,6 +1600,46 @@ mod slice_index { bad: data[7..=6]; message: "out of range"; } + + in mod boundpair_len { + data: [0, 1, 2, 3, 4, 5]; + + good: data[(Bound::Included(6), Bound::Unbounded)] == []; + good: data[(Bound::Unbounded, Bound::Included(5))] == [0, 1, 2, 3, 4, 5]; + good: data[(Bound::Unbounded, Bound::Excluded(6))] == [0, 1, 2, 3, 4, 5]; + good: data[(Bound::Included(0), Bound::Included(5))] == [0, 1, 2, 3, 4, 5]; + good: data[(Bound::Included(0), Bound::Excluded(6))] == [0, 1, 2, 3, 4, 5]; + good: data[(Bound::Included(2), Bound::Excluded(4))] == [2, 3]; + good: data[(Bound::Excluded(1), Bound::Included(4))] == [2, 3, 4]; + good: data[(Bound::Excluded(5), Bound::Excluded(6))] == []; + good: data[(Bound::Included(6), Bound::Excluded(6))] == []; + good: data[(Bound::Excluded(5), Bound::Included(5))] == []; + good: data[(Bound::Included(6), Bound::Included(5))] == []; + bad: data[(Bound::Unbounded, Bound::Included(6))]; + message: "out of range"; + } + } + + panic_cases! { + in mod rangeinclusive_exhausted { + data: [0, 1, 2, 3, 4, 5]; + + good: data[0..=5] == [0, 1, 2, 3, 4, 5]; + good: data[{ + let mut iter = 0..=5; + iter.by_ref().count(); // exhaust it + iter + }] == []; + + // 0..=6 is out of range before exhaustion, so it + // stands to reason that it still would be after. + bad: data[{ + let mut iter = 0..=6; + iter.by_ref().count(); // exhaust it + iter + }]; + message: "out of range"; + } } panic_cases! { @@ -1100,6 +1658,14 @@ mod slice_index { bad: data[4..=2]; message: "but ends at"; } + + in mod boundpair_neg_width { + data: [0, 1, 2, 3, 4, 5]; + + good: data[(Bound::Included(4), Bound::Excluded(4))] == []; + bad: data[(Bound::Included(4), Bound::Excluded(3))]; + message: "but ends at"; + } } panic_cases! { @@ -1108,14 +1674,28 @@ mod slice_index { // note: using 0 specifically ensures that the result of overflowing is 0..0, // so that `get` doesn't simply return None for the wrong reason. - bad: data[0 ..= ::std::usize::MAX]; + bad: data[0 ..= usize::MAX]; message: "maximum usize"; } in mod rangetoinclusive_overflow { data: [0, 1]; - bad: data[..= ::std::usize::MAX]; + bad: data[..= usize::MAX]; + message: "maximum usize"; + } + + in mod boundpair_overflow_end { + data: [0; 1]; + + bad: data[(Bound::Unbounded, Bound::Included(usize::MAX))]; + message: "maximum usize"; + } + + in mod boundpair_overflow_start { + data: [0; 1]; + + bad: data[(Bound::Excluded(usize::MAX), Bound::Unbounded)]; message: "maximum usize"; } } // panic_cases! @@ -1225,23 +1805,17 @@ fn brute_force_rotate_test_1() { fn sort_unstable() { use core::cmp::Ordering::{Equal, Greater, Less}; use core::slice::heapsort; - use rand::{rngs::StdRng, seq::SliceRandom, Rng, SeedableRng}; - - #[cfg(not(miri))] // Miri is too slow - let large_range = 500..510; - #[cfg(not(miri))] // Miri is too slow - let rounds = 100; + use rand::{seq::SliceRandom, Rng}; - #[cfg(miri)] - let large_range = 0..0; // empty range - #[cfg(miri)] - let rounds = 1; + // Miri is too slow (but still need to `chain` to make the types match) + let lens = if cfg!(miri) { (2..20).chain(0..0) } else { (2..25).chain(500..510) }; + let rounds = if cfg!(miri) { 1 } else { 100 }; let mut v = [0; 600]; let mut tmp = [0; 600]; - let mut rng = StdRng::from_entropy(); + let mut rng = crate::test_rng(); - for len in (2..25).chain(large_range) { + for len in lens { let v = &mut v[0..len]; let tmp = &mut tmp[0..len]; @@ -1303,13 +1877,12 @@ fn sort_unstable() { #[test] #[cfg(not(target_arch = "wasm32"))] #[cfg_attr(miri, ignore)] // Miri is too slow -fn partition_at_index() { +fn select_nth_unstable() { use core::cmp::Ordering::{Equal, Greater, Less}; - use rand::rngs::StdRng; use rand::seq::SliceRandom; - use rand::{Rng, SeedableRng}; + use rand::Rng; - let mut rng = StdRng::from_entropy(); + let mut rng = crate::test_rng(); for len in (2..21).chain(500..501) { let mut orig = vec![0; len]; @@ -1329,7 +1902,7 @@ fn partition_at_index() { // Sort in default order. for pivot in 0..len { let mut v = orig.clone(); - v.partition_at_index(pivot); + v.select_nth_unstable(pivot); assert_eq!(v_sorted[pivot], v[pivot]); for i in 0..pivot { @@ -1342,7 +1915,7 @@ fn partition_at_index() { // Sort in ascending order. for pivot in 0..len { let mut v = orig.clone(); - let (left, pivot, right) = v.partition_at_index_by(pivot, |a, b| a.cmp(b)); + let (left, pivot, right) = v.select_nth_unstable_by(pivot, |a, b| a.cmp(b)); assert_eq!(left.len() + right.len(), len - 1); @@ -1365,7 +1938,7 @@ fn partition_at_index() { for pivot in 0..len { let mut v = orig.clone(); - v.partition_at_index_by(pivot, sort_descending_comparator); + v.select_nth_unstable_by(pivot, sort_descending_comparator); assert_eq!(v_sorted_descending[pivot], v[pivot]); for i in 0..pivot { @@ -1386,7 +1959,7 @@ fn partition_at_index() { } for pivot in 0..v.len() { - v.partition_at_index_by(pivot, |_, _| *[Less, Equal, Greater].choose(&mut rng).unwrap()); + v.select_nth_unstable_by(pivot, |_, _| *[Less, Equal, Greater].choose(&mut rng).unwrap()); v.sort(); for i in 0..v.len() { assert_eq!(v[i], i as i32); @@ -1394,28 +1967,28 @@ fn partition_at_index() { } // Should not panic. - [(); 10].partition_at_index(0); - [(); 10].partition_at_index(5); - [(); 10].partition_at_index(9); - [(); 100].partition_at_index(0); - [(); 100].partition_at_index(50); - [(); 100].partition_at_index(99); + [(); 10].select_nth_unstable(0); + [(); 10].select_nth_unstable(5); + [(); 10].select_nth_unstable(9); + [(); 100].select_nth_unstable(0); + [(); 100].select_nth_unstable(50); + [(); 100].select_nth_unstable(99); let mut v = [0xDEADBEEFu64]; - v.partition_at_index(0); + v.select_nth_unstable(0); assert!(v == [0xDEADBEEF]); } #[test] #[should_panic(expected = "index 0 greater than length of slice")] -fn partition_at_index_zero_length() { - [0i32; 0].partition_at_index(0); +fn select_nth_unstable_zero_length() { + [0i32; 0].select_nth_unstable(0); } #[test] #[should_panic(expected = "index 20 greater than length of slice")] -fn partition_at_index_past_length() { - [0i32; 10].partition_at_index(20); +fn select_nth_unstable_past_length() { + [0i32; 10].select_nth_unstable(20); } pub mod memchr { @@ -1505,7 +2078,6 @@ pub mod memchr { } #[test] -#[cfg_attr(miri, ignore)] // Miri does not compute a maximal `mid` for `align_offset` fn test_align_to_simple() { let bytes = [1u8, 2, 3, 4, 5, 6, 7]; let (prefix, aligned, suffix) = unsafe { bytes.align_to::() }; @@ -1535,7 +2107,6 @@ fn test_align_to_zst() { } #[test] -#[cfg_attr(miri, ignore)] // Miri does not compute a maximal `mid` for `align_offset` fn test_align_to_non_trivial() { #[repr(align(8))] struct U64(u64, u64); @@ -1672,7 +2243,7 @@ fn test_copy_within() { } #[test] -#[should_panic(expected = "src is out of bounds")] +#[should_panic(expected = "range end index 14 out of range for slice of length 13")] fn test_copy_within_panics_src_too_long() { let mut bytes = *b"Hello, World!"; // The length is only 13, so 14 is out of bounds. @@ -1686,8 +2257,9 @@ fn test_copy_within_panics_dest_too_long() { // The length is only 13, so a slice of length 4 starting at index 10 is out of bounds. bytes.copy_within(0..4, 10); } + #[test] -#[should_panic(expected = "src end is before src start")] +#[should_panic(expected = "slice index starts at 2 but ends at 1")] fn test_copy_within_panics_src_inverted() { let mut bytes = *b"Hello, World!"; // 2 is greater than 1, so this range is invalid. @@ -1697,8 +2269,8 @@ fn test_copy_within_panics_src_inverted() { #[should_panic(expected = "attempted to index slice up to maximum usize")] fn test_copy_within_panics_src_out_of_bounds() { let mut bytes = *b"Hello, World!"; - // an inclusive range ending at usize::max_value() would make src_end overflow - bytes.copy_within(usize::max_value()..=usize::max_value(), 0); + // an inclusive range ending at usize::MAX would make src_end overflow + bytes.copy_within(usize::MAX..=usize::MAX, 0); } #[test] @@ -1709,9 +2281,376 @@ fn test_is_sorted() { assert!(![1, 3, 2].is_sorted()); assert!([0].is_sorted()); assert!(empty.is_sorted()); - assert!(![0.0, 1.0, std::f32::NAN].is_sorted()); + assert!(![0.0, 1.0, f32::NAN].is_sorted()); assert!([-2, -1, 0, 3].is_sorted()); assert!(![-2i32, -1, 0, 3].is_sorted_by_key(|n| n.abs())); assert!(!["c", "bb", "aaa"].is_sorted()); assert!(["c", "bb", "aaa"].is_sorted_by_key(|s| s.len())); } + +#[test] +fn test_slice_run_destructors() { + // Make sure that destructors get run on slice literals + struct Foo<'a> { + x: &'a Cell, + } + + impl<'a> Drop for Foo<'a> { + fn drop(&mut self) { + self.x.set(self.x.get() + 1); + } + } + + fn foo(x: &Cell) -> Foo<'_> { + Foo { x } + } + + let x = &Cell::new(0); + + { + let l = &[foo(x)]; + assert_eq!(l[0].x.get(), 0); + } + + assert_eq!(x.get(), 1); +} + +#[test] +fn test_const_from_ref() { + const VALUE: &i32 = &1; + const SLICE: &[i32] = core::slice::from_ref(VALUE); + + assert!(core::ptr::eq(VALUE, &SLICE[0])) +} + +#[test] +fn test_slice_fill_with_uninit() { + // This should not UB. See #87891 + let mut a = [MaybeUninit::::uninit(); 10]; + a.fill(MaybeUninit::uninit()); +} + +#[test] +fn test_swap() { + let mut x = ["a", "b", "c", "d"]; + x.swap(1, 3); + assert_eq!(x, ["a", "d", "c", "b"]); + x.swap(0, 3); + assert_eq!(x, ["b", "d", "c", "a"]); +} + +mod swap_panics { + #[test] + #[should_panic(expected = "index out of bounds: the len is 4 but the index is 4")] + fn index_a_equals_len() { + let mut x = ["a", "b", "c", "d"]; + x.swap(4, 2); + } + + #[test] + #[should_panic(expected = "index out of bounds: the len is 4 but the index is 4")] + fn index_b_equals_len() { + let mut x = ["a", "b", "c", "d"]; + x.swap(2, 4); + } + + #[test] + #[should_panic(expected = "index out of bounds: the len is 4 but the index is 5")] + fn index_a_greater_than_len() { + let mut x = ["a", "b", "c", "d"]; + x.swap(5, 2); + } + + #[test] + #[should_panic(expected = "index out of bounds: the len is 4 but the index is 5")] + fn index_b_greater_than_len() { + let mut x = ["a", "b", "c", "d"]; + x.swap(2, 5); + } +} + +#[test] +fn slice_split_array_mut() { + let v = &mut [1, 2, 3, 4, 5, 6][..]; + + { + let (left, right) = v.split_array_mut::<0>(); + assert_eq!(left, &mut []); + assert_eq!(right, [1, 2, 3, 4, 5, 6]); + } + + { + let (left, right) = v.split_array_mut::<6>(); + assert_eq!(left, &mut [1, 2, 3, 4, 5, 6]); + assert_eq!(right, []); + } +} + +#[test] +fn slice_rsplit_array_mut() { + let v = &mut [1, 2, 3, 4, 5, 6][..]; + + { + let (left, right) = v.rsplit_array_mut::<0>(); + assert_eq!(left, [1, 2, 3, 4, 5, 6]); + assert_eq!(right, &mut []); + } + + { + let (left, right) = v.rsplit_array_mut::<6>(); + assert_eq!(left, []); + assert_eq!(right, &mut [1, 2, 3, 4, 5, 6]); + } +} + +#[test] +fn split_as_slice() { + let arr = [1, 2, 3, 4, 5, 6]; + let mut split = arr.split(|v| v % 2 == 0); + assert_eq!(split.as_slice(), &[1, 2, 3, 4, 5, 6]); + assert!(split.next().is_some()); + assert_eq!(split.as_slice(), &[3, 4, 5, 6]); + assert!(split.next().is_some()); + assert!(split.next().is_some()); + assert_eq!(split.as_slice(), &[]); +} + +#[should_panic] +#[test] +fn slice_split_array_ref_out_of_bounds() { + let v = &[1, 2, 3, 4, 5, 6][..]; + + let _ = v.split_array_ref::<7>(); +} + +#[should_panic] +#[test] +fn slice_split_array_mut_out_of_bounds() { + let v = &mut [1, 2, 3, 4, 5, 6][..]; + + let _ = v.split_array_mut::<7>(); +} + +#[should_panic] +#[test] +fn slice_rsplit_array_ref_out_of_bounds() { + let v = &[1, 2, 3, 4, 5, 6][..]; + + let _ = v.rsplit_array_ref::<7>(); +} + +#[should_panic] +#[test] +fn slice_rsplit_array_mut_out_of_bounds() { + let v = &mut [1, 2, 3, 4, 5, 6][..]; + + let _ = v.rsplit_array_mut::<7>(); +} + +macro_rules! take_tests { + (slice: &[], $($tts:tt)*) => { + take_tests!(ty: &[()], slice: &[], $($tts)*); + }; + (slice: &mut [], $($tts:tt)*) => { + take_tests!(ty: &mut [()], slice: &mut [], $($tts)*); + }; + (slice: &$slice:expr, $($tts:tt)*) => { + take_tests!(ty: &[_], slice: &$slice, $($tts)*); + }; + (slice: &mut $slice:expr, $($tts:tt)*) => { + take_tests!(ty: &mut [_], slice: &mut $slice, $($tts)*); + }; + (ty: $ty:ty, slice: $slice:expr, method: $method:ident, $(($test_name:ident, ($($args:expr),*), $output:expr, $remaining:expr),)*) => { + $( + #[test] + fn $test_name() { + let mut slice: $ty = $slice; + assert_eq!($output, slice.$method($($args)*)); + let remaining: $ty = $remaining; + assert_eq!(remaining, slice); + } + )* + }; +} + +take_tests! { + slice: &[0, 1, 2, 3], method: take, + (take_in_bounds_range_to, (..1), Some(&[0] as _), &[1, 2, 3]), + (take_in_bounds_range_to_inclusive, (..=0), Some(&[0] as _), &[1, 2, 3]), + (take_in_bounds_range_from, (2..), Some(&[2, 3] as _), &[0, 1]), + (take_oob_range_to, (..5), None, &[0, 1, 2, 3]), + (take_oob_range_to_inclusive, (..=4), None, &[0, 1, 2, 3]), + (take_oob_range_from, (5..), None, &[0, 1, 2, 3]), +} + +take_tests! { + slice: &mut [0, 1, 2, 3], method: take_mut, + (take_mut_in_bounds_range_to, (..1), Some(&mut [0] as _), &mut [1, 2, 3]), + (take_mut_in_bounds_range_to_inclusive, (..=0), Some(&mut [0] as _), &mut [1, 2, 3]), + (take_mut_in_bounds_range_from, (2..), Some(&mut [2, 3] as _), &mut [0, 1]), + (take_mut_oob_range_to, (..5), None, &mut [0, 1, 2, 3]), + (take_mut_oob_range_to_inclusive, (..=4), None, &mut [0, 1, 2, 3]), + (take_mut_oob_range_from, (5..), None, &mut [0, 1, 2, 3]), +} + +take_tests! { + slice: &[1, 2], method: take_first, + (take_first_nonempty, (), Some(&1), &[2]), +} + +take_tests! { + slice: &mut [1, 2], method: take_first_mut, + (take_first_mut_nonempty, (), Some(&mut 1), &mut [2]), +} + +take_tests! { + slice: &[1, 2], method: take_last, + (take_last_nonempty, (), Some(&2), &[1]), +} + +take_tests! { + slice: &mut [1, 2], method: take_last_mut, + (take_last_mut_nonempty, (), Some(&mut 2), &mut [1]), +} + +take_tests! { + slice: &[], method: take_first, + (take_first_empty, (), None, &[]), +} + +take_tests! { + slice: &mut [], method: take_first_mut, + (take_first_mut_empty, (), None, &mut []), +} + +take_tests! { + slice: &[], method: take_last, + (take_last_empty, (), None, &[]), +} + +take_tests! { + slice: &mut [], method: take_last_mut, + (take_last_mut_empty, (), None, &mut []), +} + +#[cfg(not(miri))] // unused in Miri +const EMPTY_MAX: &'static [()] = &[(); usize::MAX]; + +// can't be a constant due to const mutability rules +#[cfg(not(miri))] // unused in Miri +macro_rules! empty_max_mut { + () => { + &mut [(); usize::MAX] as _ + }; +} + +#[cfg(not(miri))] // Comparing usize::MAX many elements takes forever in Miri (and in rustc without optimizations) +take_tests! { + slice: &[(); usize::MAX], method: take, + (take_in_bounds_max_range_to, (..usize::MAX), Some(EMPTY_MAX), &[(); 0]), + (take_oob_max_range_to_inclusive, (..=usize::MAX), None, EMPTY_MAX), + (take_in_bounds_max_range_from, (usize::MAX..), Some(&[] as _), EMPTY_MAX), +} + +#[cfg(not(miri))] // Comparing usize::MAX many elements takes forever in Miri (and in rustc without optimizations) +take_tests! { + slice: &mut [(); usize::MAX], method: take_mut, + (take_mut_in_bounds_max_range_to, (..usize::MAX), Some(empty_max_mut!()), &mut [(); 0]), + (take_mut_oob_max_range_to_inclusive, (..=usize::MAX), None, empty_max_mut!()), + (take_mut_in_bounds_max_range_from, (usize::MAX..), Some(&mut [] as _), empty_max_mut!()), +} + +#[test] +fn test_slice_from_ptr_range() { + let arr = ["foo".to_owned(), "bar".to_owned()]; + let range = arr.as_ptr_range(); + unsafe { + assert_eq!(slice::from_ptr_range(range), &arr); + } + + let mut arr = [1, 2, 3]; + let range = arr.as_mut_ptr_range(); + unsafe { + assert_eq!(slice::from_mut_ptr_range(range), &mut [1, 2, 3]); + } + + let arr: [Vec; 0] = []; + let range = arr.as_ptr_range(); + unsafe { + assert_eq!(slice::from_ptr_range(range), &arr); + } +} + +#[test] +#[should_panic = "slice len overflow"] +fn test_flatten_size_overflow() { + let x = &[[(); usize::MAX]; 2][..]; + let _ = x.flatten(); +} + +#[test] +#[should_panic = "slice len overflow"] +fn test_flatten_mut_size_overflow() { + let x = &mut [[(); usize::MAX]; 2][..]; + let _ = x.flatten_mut(); +} + +#[test] +fn test_get_many_mut_normal_2() { + let mut v = vec![1, 2, 3, 4, 5]; + let [a, b] = v.get_many_mut([3, 0]).unwrap(); + *a += 10; + *b += 100; + assert_eq!(v, vec![101, 2, 3, 14, 5]); +} + +#[test] +fn test_get_many_mut_normal_3() { + let mut v = vec![1, 2, 3, 4, 5]; + let [a, b, c] = v.get_many_mut([0, 4, 2]).unwrap(); + *a += 10; + *b += 100; + *c += 1000; + assert_eq!(v, vec![11, 2, 1003, 4, 105]); +} + +#[test] +fn test_get_many_mut_empty() { + let mut v = vec![1, 2, 3, 4, 5]; + let [] = v.get_many_mut([]).unwrap(); + assert_eq!(v, vec![1, 2, 3, 4, 5]); +} + +#[test] +fn test_get_many_mut_single_first() { + let mut v = vec![1, 2, 3, 4, 5]; + let [a] = v.get_many_mut([0]).unwrap(); + *a += 10; + assert_eq!(v, vec![11, 2, 3, 4, 5]); +} + +#[test] +fn test_get_many_mut_single_last() { + let mut v = vec![1, 2, 3, 4, 5]; + let [a] = v.get_many_mut([4]).unwrap(); + *a += 10; + assert_eq!(v, vec![1, 2, 3, 4, 15]); +} + +#[test] +fn test_get_many_mut_oob_nonempty() { + let mut v = vec![1, 2, 3, 4, 5]; + assert!(v.get_many_mut([5]).is_err()); +} + +#[test] +fn test_get_many_mut_oob_empty() { + let mut v: Vec = vec![]; + assert!(v.get_many_mut([0]).is_err()); +} + +#[test] +fn test_get_many_mut_duplicate() { + let mut v = vec![1, 2, 3, 4, 5]; + assert!(v.get_many_mut([1, 3, 3, 4]).is_err()); +} diff --git a/crux-mir/lib/core/tests/str.rs b/crux-mir/lib/core/tests/str.rs index ed939ca71..f5066343a 100644 --- a/crux-mir/lib/core/tests/str.rs +++ b/crux-mir/lib/core/tests/str.rs @@ -1 +1 @@ -// All `str` tests live in liballoc/tests +// All `str` tests live in library/alloc/tests/str.rs diff --git a/crux-mir/lib/core/tests/str_lossy.rs b/crux-mir/lib/core/tests/str_lossy.rs index d4b47a470..9d3f0b65f 100644 --- a/crux-mir/lib/core/tests/str_lossy.rs +++ b/crux-mir/lib/core/tests/str_lossy.rs @@ -1,85 +1,85 @@ -use core::str::lossy::*; +use core::str::Utf8Chunks; #[test] fn chunks() { - let mut iter = Utf8Lossy::from_bytes(b"hello").chunks(); - assert_eq!(Some(Utf8LossyChunk { valid: "hello", broken: b"" }), iter.next()); - assert_eq!(None, iter.next()); + macro_rules! assert_chunks { + ( $string:expr, $(($valid:expr, $invalid:expr)),* $(,)? ) => {{ + let mut iter = Utf8Chunks::new($string); + $( + let chunk = iter.next().expect("missing chunk"); + assert_eq!($valid, chunk.valid()); + assert_eq!($invalid, chunk.invalid()); + )* + assert_eq!(None, iter.next()); + }}; + } - let mut iter = Utf8Lossy::from_bytes("ศไทย中华Việt Nam".as_bytes()).chunks(); - assert_eq!(Some(Utf8LossyChunk { valid: "ศไทย中华Việt Nam", broken: b"" }), iter.next()); - assert_eq!(None, iter.next()); - - let mut iter = Utf8Lossy::from_bytes(b"Hello\xC2 There\xFF Goodbye").chunks(); - assert_eq!(Some(Utf8LossyChunk { valid: "Hello", broken: b"\xC2" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: " There", broken: b"\xFF" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: " Goodbye", broken: b"" }), iter.next()); - assert_eq!(None, iter.next()); - - let mut iter = Utf8Lossy::from_bytes(b"Hello\xC0\x80 There\xE6\x83 Goodbye").chunks(); - assert_eq!(Some(Utf8LossyChunk { valid: "Hello", broken: b"\xC0" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\x80" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: " There", broken: b"\xE6\x83" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: " Goodbye", broken: b"" }), iter.next()); - assert_eq!(None, iter.next()); - - let mut iter = Utf8Lossy::from_bytes(b"\xF5foo\xF5\x80bar").chunks(); - assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\xF5" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: "foo", broken: b"\xF5" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\x80" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: "bar", broken: b"" }), iter.next()); - assert_eq!(None, iter.next()); - - let mut iter = Utf8Lossy::from_bytes(b"\xF1foo\xF1\x80bar\xF1\x80\x80baz").chunks(); - assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\xF1" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: "foo", broken: b"\xF1\x80" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: "bar", broken: b"\xF1\x80\x80" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: "baz", broken: b"" }), iter.next()); - assert_eq!(None, iter.next()); - - let mut iter = Utf8Lossy::from_bytes(b"\xF4foo\xF4\x80bar\xF4\xBFbaz").chunks(); - assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\xF4" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: "foo", broken: b"\xF4\x80" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: "bar", broken: b"\xF4" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\xBF" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: "baz", broken: b"" }), iter.next()); - assert_eq!(None, iter.next()); - - let mut iter = Utf8Lossy::from_bytes(b"\xF0\x80\x80\x80foo\xF0\x90\x80\x80bar").chunks(); - assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\xF0" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\x80" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\x80" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\x80" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: "foo\u{10000}bar", broken: b"" }), iter.next()); - assert_eq!(None, iter.next()); + assert_chunks!(b"hello", ("hello", b"")); + assert_chunks!("ศไทย中华Việt Nam".as_bytes(), ("ศไทย中华Việt Nam", b"")); + assert_chunks!( + b"Hello\xC2 There\xFF Goodbye", + ("Hello", b"\xC2"), + (" There", b"\xFF"), + (" Goodbye", b""), + ); + assert_chunks!( + b"Hello\xC0\x80 There\xE6\x83 Goodbye", + ("Hello", b"\xC0"), + ("", b"\x80"), + (" There", b"\xE6\x83"), + (" Goodbye", b""), + ); + assert_chunks!( + b"\xF5foo\xF5\x80bar", + ("", b"\xF5"), + ("foo", b"\xF5"), + ("", b"\x80"), + ("bar", b""), + ); + assert_chunks!( + b"\xF1foo\xF1\x80bar\xF1\x80\x80baz", + ("", b"\xF1"), + ("foo", b"\xF1\x80"), + ("bar", b"\xF1\x80\x80"), + ("baz", b""), + ); + assert_chunks!( + b"\xF4foo\xF4\x80bar\xF4\xBFbaz", + ("", b"\xF4"), + ("foo", b"\xF4\x80"), + ("bar", b"\xF4"), + ("", b"\xBF"), + ("baz", b""), + ); + assert_chunks!( + b"\xF0\x80\x80\x80foo\xF0\x90\x80\x80bar", + ("", b"\xF0"), + ("", b"\x80"), + ("", b"\x80"), + ("", b"\x80"), + ("foo\u{10000}bar", b""), + ); // surrogates - let mut iter = Utf8Lossy::from_bytes(b"\xED\xA0\x80foo\xED\xBF\xBFbar").chunks(); - assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\xED" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\xA0" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\x80" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: "foo", broken: b"\xED" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\xBF" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\xBF" }), iter.next()); - assert_eq!(Some(Utf8LossyChunk { valid: "bar", broken: b"" }), iter.next()); - assert_eq!(None, iter.next()); -} - -#[test] -fn display() { - assert_eq!( - "Hello\u{FFFD}\u{FFFD} There\u{FFFD} Goodbye", - &Utf8Lossy::from_bytes(b"Hello\xC0\x80 There\xE6\x83 Goodbye").to_string() + assert_chunks!( + b"\xED\xA0\x80foo\xED\xBF\xBFbar", + ("", b"\xED"), + ("", b"\xA0"), + ("", b"\x80"), + ("foo", b"\xED"), + ("", b"\xBF"), + ("", b"\xBF"), + ("bar", b""), ); } #[test] fn debug() { assert_eq!( - "\"Hello\\xc0\\x80 There\\xe6\\x83 Goodbye\\u{10d4ea}\"", + "\"Hello\\xC0\\x80 There\\xE6\\x83 Goodbye\\u{10d4ea}\"", &format!( "{:?}", - Utf8Lossy::from_bytes(b"Hello\xC0\x80 There\xE6\x83 Goodbye\xf4\x8d\x93\xaa") - ) + Utf8Chunks::new(b"Hello\xC0\x80 There\xE6\x83 Goodbye\xf4\x8d\x93\xaa").debug(), + ), ); } diff --git a/crux-mir/lib/core/tests/task.rs b/crux-mir/lib/core/tests/task.rs new file mode 100644 index 000000000..163b34c96 --- /dev/null +++ b/crux-mir/lib/core/tests/task.rs @@ -0,0 +1,25 @@ +use core::task::{Poll, RawWaker, RawWakerVTable, Waker}; + +#[test] +fn poll_const() { + // test that the methods of `Poll` are usable in a const context + + const POLL: Poll = Poll::Pending; + + const IS_READY: bool = POLL.is_ready(); + assert!(!IS_READY); + + const IS_PENDING: bool = POLL.is_pending(); + assert!(IS_PENDING); +} + +#[test] +fn waker_const() { + const VOID_TABLE: RawWakerVTable = RawWakerVTable::new(|_| VOID_WAKER, |_| {}, |_| {}, |_| {}); + + const VOID_WAKER: RawWaker = RawWaker::new(&(), &VOID_TABLE); + + static WAKER: Waker = unsafe { Waker::from_raw(VOID_WAKER) }; + + WAKER.wake_by_ref(); +} diff --git a/crux-mir/lib/core/tests/time.rs b/crux-mir/lib/core/tests/time.rs index c1fbdf7df..2975c81f8 100644 --- a/crux-mir/lib/core/tests/time.rs +++ b/crux-mir/lib/core/tests/time.rs @@ -14,7 +14,7 @@ fn creation() { #[test] #[should_panic] fn new_overflow() { - let _ = Duration::new(::core::u64::MAX, 1_000_000_000); + let _ = Duration::new(u64::MAX, 1_000_000_000); } #[test] @@ -86,7 +86,17 @@ fn checked_add() { Duration::new(0, 500_000_000).checked_add(Duration::new(0, 500_000_001)), Some(Duration::new(1, 1)) ); - assert_eq!(Duration::new(1, 0).checked_add(Duration::new(::core::u64::MAX, 0)), None); + assert_eq!(Duration::new(1, 0).checked_add(Duration::new(u64::MAX, 0)), None); +} + +#[test] +fn saturating_add() { + assert_eq!(Duration::new(0, 0).saturating_add(Duration::new(0, 1)), Duration::new(0, 1)); + assert_eq!( + Duration::new(0, 500_000_000).saturating_add(Duration::new(0, 500_000_001)), + Duration::new(1, 1) + ); + assert_eq!(Duration::new(1, 0).saturating_add(Duration::new(u64::MAX, 0)), Duration::MAX); } #[test] @@ -98,13 +108,24 @@ fn sub() { #[test] fn checked_sub() { - let zero = Duration::new(0, 0); - let one_nano = Duration::new(0, 1); - let one_sec = Duration::new(1, 0); - assert_eq!(one_nano.checked_sub(zero), Some(Duration::new(0, 1))); - assert_eq!(one_sec.checked_sub(one_nano), Some(Duration::new(0, 999_999_999))); - assert_eq!(zero.checked_sub(one_nano), None); - assert_eq!(zero.checked_sub(one_sec), None); + assert_eq!(Duration::NANOSECOND.checked_sub(Duration::ZERO), Some(Duration::NANOSECOND)); + assert_eq!( + Duration::SECOND.checked_sub(Duration::NANOSECOND), + Some(Duration::new(0, 999_999_999)) + ); + assert_eq!(Duration::ZERO.checked_sub(Duration::NANOSECOND), None); + assert_eq!(Duration::ZERO.checked_sub(Duration::SECOND), None); +} + +#[test] +fn saturating_sub() { + assert_eq!(Duration::NANOSECOND.saturating_sub(Duration::ZERO), Duration::NANOSECOND); + assert_eq!( + Duration::SECOND.saturating_sub(Duration::NANOSECOND), + Duration::new(0, 999_999_999) + ); + assert_eq!(Duration::ZERO.saturating_sub(Duration::NANOSECOND), Duration::ZERO); + assert_eq!(Duration::ZERO.saturating_sub(Duration::SECOND), Duration::ZERO); } #[test] @@ -133,7 +154,16 @@ fn checked_mul() { assert_eq!(Duration::new(1, 1).checked_mul(3), Some(Duration::new(3, 3))); assert_eq!(Duration::new(0, 500_000_001).checked_mul(4), Some(Duration::new(2, 4))); assert_eq!(Duration::new(0, 500_000_001).checked_mul(4000), Some(Duration::new(2000, 4000))); - assert_eq!(Duration::new(::core::u64::MAX - 1, 0).checked_mul(2), None); + assert_eq!(Duration::new(u64::MAX - 1, 0).checked_mul(2), None); +} + +#[test] +fn saturating_mul() { + assert_eq!(Duration::new(0, 1).saturating_mul(2), Duration::new(0, 2)); + assert_eq!(Duration::new(1, 1).saturating_mul(3), Duration::new(3, 3)); + assert_eq!(Duration::new(0, 500_000_001).saturating_mul(4), Duration::new(2, 4)); + assert_eq!(Duration::new(0, 500_000_001).saturating_mul(4000), Duration::new(2000, 4000)); + assert_eq!(Duration::new(u64::MAX - 1, 0).saturating_mul(2), Duration::MAX); } #[test] @@ -143,6 +173,32 @@ fn div() { assert_eq!(Duration::new(99, 999_999_000) / 100, Duration::new(0, 999_999_990)); } +#[test] +fn div_duration_f32() { + assert_eq!(Duration::ZERO.div_duration_f32(Duration::MAX), 0.0); + assert_eq!(Duration::MAX.div_duration_f32(Duration::ZERO), f32::INFINITY); + assert_eq!((Duration::SECOND * 2).div_duration_f32(Duration::SECOND), 2.0); + assert!(Duration::ZERO.div_duration_f32(Duration::ZERO).is_nan()); + // These tests demonstrate it doesn't panic with extreme values. + // Accuracy of the computed value is not a huge concern, we know floats don't work well + // at these extremes. + assert!((Duration::MAX).div_duration_f32(Duration::NANOSECOND) > 10.0f32.powf(28.0)); + assert!((Duration::NANOSECOND).div_duration_f32(Duration::MAX) < 0.1); +} + +#[test] +fn div_duration_f64() { + assert_eq!(Duration::ZERO.div_duration_f64(Duration::MAX), 0.0); + assert_eq!(Duration::MAX.div_duration_f64(Duration::ZERO), f64::INFINITY); + assert_eq!((Duration::SECOND * 2).div_duration_f64(Duration::SECOND), 2.0); + assert!(Duration::ZERO.div_duration_f64(Duration::ZERO).is_nan()); + // These tests demonstrate it doesn't panic with extreme values. + // Accuracy of the computed value is not a huge concern, we know floats don't work well + // at these extremes. + assert!((Duration::MAX).div_duration_f64(Duration::NANOSECOND) > 10.0f64.powf(28.0)); + assert!((Duration::NANOSECOND).div_duration_f64(Duration::MAX) < 0.1); +} + #[test] fn checked_div() { assert_eq!(Duration::new(2, 0).checked_div(2), Some(Duration::new(1, 0))); @@ -167,9 +223,31 @@ fn correct_sum() { #[test] fn debug_formatting_extreme_values() { assert_eq!( - format!("{:?}", Duration::new(18_446_744_073_709_551_615, 123_456_789)), + format!("{:?}", Duration::new(u64::MAX, 123_456_789)), "18446744073709551615.123456789s" ); + assert_eq!(format!("{:.0?}", Duration::MAX), "18446744073709551616s"); + assert_eq!(format!("{:.0?}", Duration::new(u64::MAX, 500_000_000)), "18446744073709551616s"); + assert_eq!(format!("{:.0?}", Duration::new(u64::MAX, 499_999_999)), "18446744073709551615s"); + assert_eq!( + format!("{:.3?}", Duration::new(u64::MAX, 999_500_000)), + "18446744073709551616.000s" + ); + assert_eq!( + format!("{:.3?}", Duration::new(u64::MAX, 999_499_999)), + "18446744073709551615.999s" + ); + assert_eq!( + format!("{:.8?}", Duration::new(u64::MAX, 999_999_995)), + "18446744073709551616.00000000s" + ); + assert_eq!( + format!("{:.8?}", Duration::new(u64::MAX, 999_999_994)), + "18446744073709551615.99999999s" + ); + assert_eq!(format!("{:21.0?}", Duration::MAX), "18446744073709551616s"); + assert_eq!(format!("{:22.0?}", Duration::MAX), "18446744073709551616s "); + assert_eq!(format!("{:24.0?}", Duration::MAX), "18446744073709551616s "); } #[test] @@ -283,6 +361,34 @@ fn debug_formatting_precision_two() { assert_eq!(format!("{:.2?}", Duration::new(8, 999_999_999)), "9.00s"); } +#[test] +fn debug_formatting_padding() { + assert_eq!("0ns ", format!("{:<9?}", Duration::new(0, 0))); + assert_eq!(" 0ns", format!("{:>9?}", Duration::new(0, 0))); + assert_eq!(" 0ns ", format!("{:^9?}", Duration::new(0, 0))); + assert_eq!("123ns ", format!("{:<9.0?}", Duration::new(0, 123))); + assert_eq!(" 123ns", format!("{:>9.0?}", Duration::new(0, 123))); + assert_eq!(" 123ns ", format!("{:^9.0?}", Duration::new(0, 123))); + assert_eq!("123.0ns ", format!("{:<9.1?}", Duration::new(0, 123))); + assert_eq!(" 123.0ns", format!("{:>9.1?}", Duration::new(0, 123))); + assert_eq!(" 123.0ns ", format!("{:^9.1?}", Duration::new(0, 123))); + assert_eq!("7.1µs ", format!("{:<9?}", Duration::new(0, 7_100))); + assert_eq!(" 7.1µs", format!("{:>9?}", Duration::new(0, 7_100))); + assert_eq!(" 7.1µs ", format!("{:^9?}", Duration::new(0, 7_100))); + assert_eq!("999.123456ms", format!("{:<9?}", Duration::new(0, 999_123_456))); + assert_eq!("999.123456ms", format!("{:>9?}", Duration::new(0, 999_123_456))); + assert_eq!("999.123456ms", format!("{:^9?}", Duration::new(0, 999_123_456))); + assert_eq!("5s ", format!("{:<9?}", Duration::new(5, 0))); + assert_eq!(" 5s", format!("{:>9?}", Duration::new(5, 0))); + assert_eq!(" 5s ", format!("{:^9?}", Duration::new(5, 0))); + assert_eq!("5.000000000000s", format!("{:<9.12?}", Duration::new(5, 0))); + assert_eq!("5.000000000000s", format!("{:>9.12?}", Duration::new(5, 0))); + assert_eq!("5.000000000000s", format!("{:^9.12?}", Duration::new(5, 0))); + + // default alignment is left: + assert_eq!("5s ", format!("{:9?}", Duration::new(5, 0))); +} + #[test] fn debug_formatting_precision_high() { assert_eq!(format!("{:.5?}", Duration::new(0, 23_678)), "23.67800µs"); @@ -291,3 +397,107 @@ fn debug_formatting_precision_high() { assert_eq!(format!("{:.10?}", Duration::new(4, 001_000_000)), "4.0010000000s"); assert_eq!(format!("{:.20?}", Duration::new(4, 001_000_000)), "4.00100000000000000000s"); } + +#[test] +fn duration_const() { + // test that the methods of `Duration` are usable in a const context + + const DURATION: Duration = Duration::new(0, 123_456_789); + + const SUB_SEC_MILLIS: u32 = DURATION.subsec_millis(); + assert_eq!(SUB_SEC_MILLIS, 123); + + const SUB_SEC_MICROS: u32 = DURATION.subsec_micros(); + assert_eq!(SUB_SEC_MICROS, 123_456); + + const SUB_SEC_NANOS: u32 = DURATION.subsec_nanos(); + assert_eq!(SUB_SEC_NANOS, 123_456_789); + + const IS_ZERO: bool = Duration::ZERO.is_zero(); + assert!(IS_ZERO); + + const SECONDS: u64 = Duration::SECOND.as_secs(); + assert_eq!(SECONDS, 1); + + const FROM_SECONDS: Duration = Duration::from_secs(1); + assert_eq!(FROM_SECONDS, Duration::SECOND); + + const SECONDS_F32: f32 = Duration::SECOND.as_secs_f32(); + assert_eq!(SECONDS_F32, 1.0); + + const FROM_SECONDS_F32: Duration = Duration::from_secs_f32(1.0); + assert_eq!(FROM_SECONDS_F32, Duration::SECOND); + + const SECONDS_F64: f64 = Duration::SECOND.as_secs_f64(); + assert_eq!(SECONDS_F64, 1.0); + + const FROM_SECONDS_F64: Duration = Duration::from_secs_f64(1.0); + assert_eq!(FROM_SECONDS_F64, Duration::SECOND); + + const MILLIS: u128 = Duration::SECOND.as_millis(); + assert_eq!(MILLIS, 1_000); + + const FROM_MILLIS: Duration = Duration::from_millis(1_000); + assert_eq!(FROM_MILLIS, Duration::SECOND); + + const MICROS: u128 = Duration::SECOND.as_micros(); + assert_eq!(MICROS, 1_000_000); + + const FROM_MICROS: Duration = Duration::from_micros(1_000_000); + assert_eq!(FROM_MICROS, Duration::SECOND); + + const NANOS: u128 = Duration::SECOND.as_nanos(); + assert_eq!(NANOS, 1_000_000_000); + + const FROM_NANOS: Duration = Duration::from_nanos(1_000_000_000); + assert_eq!(FROM_NANOS, Duration::SECOND); + + const MAX: Duration = Duration::new(u64::MAX, 999_999_999); + + const CHECKED_ADD: Option = MAX.checked_add(Duration::SECOND); + assert_eq!(CHECKED_ADD, None); + + const CHECKED_SUB: Option = Duration::ZERO.checked_sub(Duration::SECOND); + assert_eq!(CHECKED_SUB, None); + + const CHECKED_MUL: Option = Duration::SECOND.checked_mul(1); + assert_eq!(CHECKED_MUL, Some(Duration::SECOND)); + + const MUL_F32: Duration = Duration::SECOND.mul_f32(1.0); + assert_eq!(MUL_F32, Duration::SECOND); + + const MUL_F64: Duration = Duration::SECOND.mul_f64(1.0); + assert_eq!(MUL_F64, Duration::SECOND); + + const CHECKED_DIV: Option = Duration::SECOND.checked_div(1); + assert_eq!(CHECKED_DIV, Some(Duration::SECOND)); + + const DIV_F32: Duration = Duration::SECOND.div_f32(1.0); + assert_eq!(DIV_F32, Duration::SECOND); + + const DIV_F64: Duration = Duration::SECOND.div_f64(1.0); + assert_eq!(DIV_F64, Duration::SECOND); + + const DIV_DURATION_F32: f32 = Duration::SECOND.div_duration_f32(Duration::SECOND); + assert_eq!(DIV_DURATION_F32, 1.0); + + const DIV_DURATION_F64: f64 = Duration::SECOND.div_duration_f64(Duration::SECOND); + assert_eq!(DIV_DURATION_F64, 1.0); + + const SATURATING_ADD: Duration = MAX.saturating_add(Duration::SECOND); + assert_eq!(SATURATING_ADD, MAX); + + const SATURATING_SUB: Duration = Duration::ZERO.saturating_sub(Duration::SECOND); + assert_eq!(SATURATING_SUB, Duration::ZERO); + + const SATURATING_MUL: Duration = MAX.saturating_mul(2); + assert_eq!(SATURATING_MUL, MAX); +} + +#[test] +fn from_neg_zero() { + assert_eq!(Duration::try_from_secs_f32(-0.0), Ok(Duration::ZERO)); + assert_eq!(Duration::try_from_secs_f64(-0.0), Ok(Duration::ZERO)); + assert_eq!(Duration::from_secs_f32(-0.0), Duration::ZERO); + assert_eq!(Duration::from_secs_f64(-0.0), Duration::ZERO); +} diff --git a/crux-mir/lib/core/tests/tuple.rs b/crux-mir/lib/core/tests/tuple.rs index 3a2914698..ea1e28142 100644 --- a/crux-mir/lib/core/tests/tuple.rs +++ b/crux-mir/lib/core/tests/tuple.rs @@ -1,5 +1,4 @@ use std::cmp::Ordering::{Equal, Greater, Less}; -use std::f64::NAN; #[test] fn test_clone() { @@ -34,12 +33,12 @@ fn test_partial_ord() { assert!(big >= small); assert!(big >= big); - assert!(!((1.0f64, 2.0f64) < (NAN, 3.0))); - assert!(!((1.0f64, 2.0f64) <= (NAN, 3.0))); - assert!(!((1.0f64, 2.0f64) > (NAN, 3.0))); - assert!(!((1.0f64, 2.0f64) >= (NAN, 3.0))); - assert!(((1.0f64, 2.0f64) < (2.0, NAN))); - assert!(!((2.0f64, 2.0f64) < (2.0, NAN))); + assert!(!((1.0f64, 2.0f64) < (f64::NAN, 3.0))); + assert!(!((1.0f64, 2.0f64) <= (f64::NAN, 3.0))); + assert!(!((1.0f64, 2.0f64) > (f64::NAN, 3.0))); + assert!(!((1.0f64, 2.0f64) >= (f64::NAN, 3.0))); + assert!(((1.0f64, 2.0f64) < (2.0, f64::NAN))); + assert!(!((2.0f64, 2.0f64) < (2.0, f64::NAN))); } #[test] diff --git a/crux-mir/lib/core/tests/unicode.rs b/crux-mir/lib/core/tests/unicode.rs new file mode 100644 index 000000000..bbace0ef6 --- /dev/null +++ b/crux-mir/lib/core/tests/unicode.rs @@ -0,0 +1,5 @@ +#[test] +pub fn version() { + let (major, _minor, _update) = core::char::UNICODE_VERSION; + assert!(major >= 10); +} diff --git a/crux-mir/lib/core/tests/waker.rs b/crux-mir/lib/core/tests/waker.rs new file mode 100644 index 000000000..38a3a0ada --- /dev/null +++ b/crux-mir/lib/core/tests/waker.rs @@ -0,0 +1,22 @@ +use std::ptr; +use std::task::{RawWaker, RawWakerVTable, Waker}; + +#[test] +fn test_waker_getters() { + let raw_waker = RawWaker::new(ptr::invalid_mut(42usize), &WAKER_VTABLE); + assert_eq!(raw_waker.data() as usize, 42); + assert!(ptr::eq(raw_waker.vtable(), &WAKER_VTABLE)); + + let waker = unsafe { Waker::from_raw(raw_waker) }; + let waker2 = waker.clone(); + let raw_waker2 = waker2.as_raw(); + assert_eq!(raw_waker2.data() as usize, 43); + assert!(ptr::eq(raw_waker2.vtable(), &WAKER_VTABLE)); +} + +static WAKER_VTABLE: RawWakerVTable = RawWakerVTable::new( + |data| RawWaker::new(ptr::invalid_mut(data as usize + 1), &WAKER_VTABLE), + |_| {}, + |_| {}, + |_| {}, +); diff --git a/crux-mir/lib/crucible/lib.rs b/crux-mir/lib/crucible/lib.rs index fd734f7ec..499bc76e2 100644 --- a/crux-mir/lib/crucible/lib.rs +++ b/crux-mir/lib/crucible/lib.rs @@ -2,6 +2,7 @@ #![feature(core_intrinsics)] #![feature(crucible_intrinsics)] #![feature(unboxed_closures)] +#![feature(tuple_trait)] pub mod bitvector; pub mod cryptol; diff --git a/crux-mir/lib/crucible/method_spec/mod.rs b/crux-mir/lib/crucible/method_spec/mod.rs index 7518af18b..09e74b6a0 100644 --- a/crux-mir/lib/crucible/method_spec/mod.rs +++ b/crux-mir/lib/crucible/method_spec/mod.rs @@ -53,7 +53,7 @@ impl fmt::Debug for MethodSpecBuilder { } impl MethodSpecBuilder { - pub fn new>(f: F) -> MethodSpecBuilder { + pub fn new>(f: F) -> MethodSpecBuilder { MethodSpecBuilder { raw: raw::builder_new::(), } diff --git a/crux-mir/lib/getopts/Cargo.toml b/crux-mir/lib/getopts/Cargo.toml deleted file mode 100644 index b55900245..000000000 --- a/crux-mir/lib/getopts/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] - -name = "getopts" -version = "0.2.21" # don't forget to update html_root_url -authors = ["The Rust Project Developers"] -license = "MIT/Apache-2.0" -readme = "README.md" -repository = "https://github.com/rust-lang/getopts" -documentation = "https://doc.rust-lang.org/getopts" -homepage = "https://github.com/rust-lang/getopts" -description = """ -getopts-like option parsing. -""" -categories = ["command-line-interface"] - -[dependencies] -unicode-width = "0.1.5" -std = { version = "1.0", package = "rustc-std-workspace-std", optional = true } -core = { version = "1.0", package = "rustc-std-workspace-core", optional = true } - -[dev-dependencies] -log = "0.4" - -[features] -rustc-dep-of-std = ['unicode-width/rustc-dep-of-std', 'std', 'core'] diff --git a/crux-mir/lib/getopts/README.md b/crux-mir/lib/getopts/README.md deleted file mode 100644 index 065a05638..000000000 --- a/crux-mir/lib/getopts/README.md +++ /dev/null @@ -1,15 +0,0 @@ -getopts -=== - -A Rust library for option parsing for CLI utilities. - -[Documentation](https://docs.rs/getopts) - -## Usage - -Add this to your `Cargo.toml`: - -```toml -[dependencies] -getopts = "0.2" -``` diff --git a/crux-mir/lib/getopts/src/lib.rs b/crux-mir/lib/getopts/src/lib.rs deleted file mode 100644 index 51ba7787f..000000000 --- a/crux-mir/lib/getopts/src/lib.rs +++ /dev/null @@ -1,1142 +0,0 @@ -// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. -// -// ignore-lexer-test FIXME #15677 - -//! Simple getopt alternative. -//! -//! Construct a vector of options, either by using `reqopt`, `optopt`, and -//! `optflag` or by building them from components yourself, and pass them to -//! `getopts`, along with a vector of actual arguments (not including -//! `argv[0]`). You'll either get a failure code back, or a match. You'll have -//! to verify whether the amount of 'free' arguments in the match is what you -//! expect. Use `opt_*` accessors to get argument values out of the matches -//! object. -//! -//! Single-character options are expected to appear on the command line with a -//! single preceding dash; multiple-character options are expected to be -//! proceeded by two dashes. Options that expect an argument accept their -//! argument following either a space or an equals sign. Single-character -//! options don't require the space. -//! -//! # Usage -//! -//! This crate is [on crates.io](https://crates.io/crates/getopts) and can be -//! used by adding `getopts` to the dependencies in your project's `Cargo.toml`. -//! -//! ```toml -//! [dependencies] -//! getopts = "0.2" -//! ``` -//! -//! and this to your crate root: -//! -//! ```rust -//! extern crate getopts; -//! ``` -//! -//! # Example -//! -//! The following example shows simple command line parsing for an application -//! that requires an input file to be specified, accepts an optional output file -//! name following `-o`, and accepts both `-h` and `--help` as optional flags. -//! -//! ```{.rust} -//! extern crate getopts; -//! use getopts::Options; -//! use std::env; -//! -//! fn do_work(inp: &str, out: Option) { -//! println!("{}", inp); -//! match out { -//! Some(x) => println!("{}", x), -//! None => println!("No Output"), -//! } -//! } -//! -//! fn print_usage(program: &str, opts: Options) { -//! let brief = format!("Usage: {} FILE [options]", program); -//! print!("{}", opts.usage(&brief)); -//! } -//! -//! fn main() { -//! let args: Vec = env::args().collect(); -//! let program = args[0].clone(); -//! -//! let mut opts = Options::new(); -//! opts.optopt("o", "", "set output file name", "NAME"); -//! opts.optflag("h", "help", "print this help menu"); -//! let matches = match opts.parse(&args[1..]) { -//! Ok(m) => { m } -//! Err(f) => { panic!(f.to_string()) } -//! }; -//! if matches.opt_present("h") { -//! print_usage(&program, opts); -//! return; -//! } -//! let output = matches.opt_str("o"); -//! let input = if !matches.free.is_empty() { -//! matches.free[0].clone() -//! } else { -//! print_usage(&program, opts); -//! return; -//! }; -//! do_work(&input, output); -//! } -//! ``` - -#![doc( - html_logo_url = "https://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png", - html_favicon_url = "https://www.rust-lang.org/favicon.ico", - html_root_url = "https://docs.rs/getopts/0.2.20" -)] -#![deny(missing_docs)] -#![cfg_attr(test, deny(warnings))] - -#[cfg(test)] -#[macro_use] -extern crate log; -extern crate unicode_width; - -use self::Fail::*; -use self::HasArg::*; -use self::Name::*; -use self::Occur::*; -use self::Optval::*; - -use std::error::Error; -use std::ffi::OsStr; -use std::fmt; -use std::iter::{repeat, IntoIterator}; -use std::result; -use std::str::FromStr; - -use unicode_width::UnicodeWidthStr; - -#[cfg(test)] -mod tests; - -/// A description of the options that a program can handle. -pub struct Options { - grps: Vec, - parsing_style: ParsingStyle, - long_only: bool, -} - -impl Default for Options { - fn default() -> Self { - Self::new() - } -} - -impl Options { - /// Create a blank set of options. - pub fn new() -> Options { - Options { - grps: Vec::new(), - parsing_style: ParsingStyle::FloatingFrees, - long_only: false, - } - } - - /// Set the parsing style. - pub fn parsing_style(&mut self, style: ParsingStyle) -> &mut Options { - self.parsing_style = style; - self - } - - /// Set or clear "long options only" mode. - /// - /// In "long options only" mode, short options cannot be clustered - /// together, and long options can be given with either a single - /// "-" or the customary "--". This mode also changes the meaning - /// of "-a=b"; in the ordinary mode this will parse a short option - /// "-a" with argument "=b"; whereas in long-options-only mode the - /// argument will be simply "b". - pub fn long_only(&mut self, long_only: bool) -> &mut Options { - self.long_only = long_only; - self - } - - /// Create a generic option group, stating all parameters explicitly. - pub fn opt( - &mut self, - short_name: &str, - long_name: &str, - desc: &str, - hint: &str, - hasarg: HasArg, - occur: Occur, - ) -> &mut Options { - validate_names(short_name, long_name); - self.grps.push(OptGroup { - short_name: short_name.to_string(), - long_name: long_name.to_string(), - hint: hint.to_string(), - desc: desc.to_string(), - hasarg, - occur, - }); - self - } - - /// Create a long option that is optional and does not take an argument. - /// - /// * `short_name` - e.g. `"h"` for a `-h` option, or `""` for none - /// * `long_name` - e.g. `"help"` for a `--help` option, or `""` for none - /// * `desc` - Description for usage help - pub fn optflag(&mut self, short_name: &str, long_name: &str, desc: &str) -> &mut Options { - validate_names(short_name, long_name); - self.grps.push(OptGroup { - short_name: short_name.to_string(), - long_name: long_name.to_string(), - hint: "".to_string(), - desc: desc.to_string(), - hasarg: No, - occur: Optional, - }); - self - } - - /// Create a long option that can occur more than once and does not - /// take an argument. - /// - /// * `short_name` - e.g. `"h"` for a `-h` option, or `""` for none - /// * `long_name` - e.g. `"help"` for a `--help` option, or `""` for none - /// * `desc` - Description for usage help - pub fn optflagmulti(&mut self, short_name: &str, long_name: &str, desc: &str) -> &mut Options { - validate_names(short_name, long_name); - self.grps.push(OptGroup { - short_name: short_name.to_string(), - long_name: long_name.to_string(), - hint: "".to_string(), - desc: desc.to_string(), - hasarg: No, - occur: Multi, - }); - self - } - - /// Create a long option that is optional and takes an optional argument. - /// - /// * `short_name` - e.g. `"h"` for a `-h` option, or `""` for none - /// * `long_name` - e.g. `"help"` for a `--help` option, or `""` for none - /// * `desc` - Description for usage help - /// * `hint` - Hint that is used in place of the argument in the usage help, - /// e.g. `"FILE"` for a `-o FILE` option - pub fn optflagopt( - &mut self, - short_name: &str, - long_name: &str, - desc: &str, - hint: &str, - ) -> &mut Options { - validate_names(short_name, long_name); - self.grps.push(OptGroup { - short_name: short_name.to_string(), - long_name: long_name.to_string(), - hint: hint.to_string(), - desc: desc.to_string(), - hasarg: Maybe, - occur: Optional, - }); - self - } - - /// Create a long option that is optional, takes an argument, and may occur - /// multiple times. - /// - /// * `short_name` - e.g. `"h"` for a `-h` option, or `""` for none - /// * `long_name` - e.g. `"help"` for a `--help` option, or `""` for none - /// * `desc` - Description for usage help - /// * `hint` - Hint that is used in place of the argument in the usage help, - /// e.g. `"FILE"` for a `-o FILE` option - pub fn optmulti( - &mut self, - short_name: &str, - long_name: &str, - desc: &str, - hint: &str, - ) -> &mut Options { - validate_names(short_name, long_name); - self.grps.push(OptGroup { - short_name: short_name.to_string(), - long_name: long_name.to_string(), - hint: hint.to_string(), - desc: desc.to_string(), - hasarg: Yes, - occur: Multi, - }); - self - } - - /// Create a long option that is optional and takes an argument. - /// - /// * `short_name` - e.g. `"h"` for a `-h` option, or `""` for none - /// * `long_name` - e.g. `"help"` for a `--help` option, or `""` for none - /// * `desc` - Description for usage help - /// * `hint` - Hint that is used in place of the argument in the usage help, - /// e.g. `"FILE"` for a `-o FILE` option - pub fn optopt( - &mut self, - short_name: &str, - long_name: &str, - desc: &str, - hint: &str, - ) -> &mut Options { - validate_names(short_name, long_name); - self.grps.push(OptGroup { - short_name: short_name.to_string(), - long_name: long_name.to_string(), - hint: hint.to_string(), - desc: desc.to_string(), - hasarg: Yes, - occur: Optional, - }); - self - } - - /// Create a long option that is required and takes an argument. - /// - /// * `short_name` - e.g. `"h"` for a `-h` option, or `""` for none - /// * `long_name` - e.g. `"help"` for a `--help` option, or `""` for none - /// * `desc` - Description for usage help - /// * `hint` - Hint that is used in place of the argument in the usage help, - /// e.g. `"FILE"` for a `-o FILE` option - pub fn reqopt( - &mut self, - short_name: &str, - long_name: &str, - desc: &str, - hint: &str, - ) -> &mut Options { - validate_names(short_name, long_name); - self.grps.push(OptGroup { - short_name: short_name.to_string(), - long_name: long_name.to_string(), - hint: hint.to_string(), - desc: desc.to_string(), - hasarg: Yes, - occur: Req, - }); - self - } - - /// Parse command line arguments according to the provided options. - /// - /// On success returns `Ok(Matches)`. Use methods such as `opt_present` - /// `opt_str`, etc. to interrogate results. - /// # Panics - /// - /// Returns `Err(Fail)` on failure: use the `Debug` implementation of `Fail` - /// to display information about it. - pub fn parse(&self, args: C) -> Result - where - C::Item: AsRef, - { - let opts: Vec = self.grps.iter().map(|x| x.long_to_short()).collect(); - - let mut vals = (0..opts.len()) - .map(|_| Vec::new()) - .collect::>>(); - let mut free: Vec = Vec::new(); - let args = args - .into_iter() - .map(|i| { - i.as_ref() - .to_str() - .ok_or_else(|| Fail::UnrecognizedOption(format!("{:?}", i.as_ref()))) - .map(|s| s.to_owned()) - }) - .collect::<::std::result::Result, _>>()?; - let mut args = args.into_iter().peekable(); - let mut arg_pos = 0; - while let Some(cur) = args.next() { - if !is_arg(&cur) { - free.push(cur); - match self.parsing_style { - ParsingStyle::FloatingFrees => {} - ParsingStyle::StopAtFirstFree => { - free.extend(args); - break; - } - } - } else if cur == "--" { - free.extend(args); - break; - } else { - let mut name = None; - let mut i_arg = None; - let mut was_long = true; - if cur.as_bytes()[1] == b'-' || self.long_only { - let tail = if cur.as_bytes()[1] == b'-' { - &cur[2..] - } else { - assert!(self.long_only); - &cur[1..] - }; - let mut parts = tail.splitn(2, '='); - name = Some(Name::from_str(parts.next().unwrap())); - if let Some(rest) = parts.next() { - i_arg = Some(rest.to_string()); - } - } else { - was_long = false; - for (j, ch) in cur.char_indices().skip(1) { - let opt = Short(ch); - - let opt_id = match find_opt(&opts, &opt) { - Some(id) => id, - None => return Err(UnrecognizedOption(opt.to_string())), - }; - - // In a series of potential options (eg. -aheJ), if we - // see one which takes an argument, we assume all - // subsequent characters make up the argument. This - // allows options such as -L/usr/local/lib/foo to be - // interpreted correctly - let arg_follows = match opts[opt_id].hasarg { - Yes | Maybe => true, - No => false, - }; - - if arg_follows { - name = Some(opt); - let next = j + ch.len_utf8(); - if next < cur.len() { - i_arg = Some(cur[next..].to_string()); - break; - } - } else { - vals[opt_id].push((arg_pos, Given)); - } - } - } - if let Some(nm) = name { - let opt_id = match find_opt(&opts, &nm) { - Some(id) => id, - None => return Err(UnrecognizedOption(nm.to_string())), - }; - match opts[opt_id].hasarg { - No => { - if i_arg.is_some() { - return Err(UnexpectedArgument(nm.to_string())); - } - vals[opt_id].push((arg_pos, Given)); - } - Maybe => { - // Note that here we do not handle `--arg value`. - // This matches GNU getopt behavior; but also - // makes sense, because if this were accepted, - // then users could only write a "Maybe" long - // option at the end of the arguments when - // FloatingFrees is in use. - if let Some(i_arg) = i_arg.take() { - vals[opt_id].push((arg_pos, Val(i_arg))); - } else if was_long - || args.peek().map_or(true, |n| is_arg(&n)) - { - vals[opt_id].push((arg_pos, Given)); - } else { - vals[opt_id].push((arg_pos, Val(args.next().unwrap()))); - } - } - Yes => { - if let Some(i_arg) = i_arg.take() { - vals[opt_id].push((arg_pos, Val(i_arg))); - } else if let Some(n) = args.next() { - vals[opt_id].push((arg_pos, Val(n))); - } else { - return Err(ArgumentMissing(nm.to_string())); - } - } - } - } - } - arg_pos += 1; - } - debug_assert_eq!(vals.len(), opts.len()); - for (vals, opt) in vals.iter().zip(opts.iter()) { - if opt.occur == Req && vals.is_empty() { - return Err(OptionMissing(opt.name.to_string())); - } - if opt.occur != Multi && vals.len() > 1 { - return Err(OptionDuplicated(opt.name.to_string())); - } - } - Ok(Matches { opts, vals, free }) - } - - /// Derive a short one-line usage summary from a set of long options. - pub fn short_usage(&self, program_name: &str) -> String { - let mut line = format!("Usage: {} ", program_name); - line.push_str( - &self - .grps - .iter() - .map(format_option) - .collect::>() - .join(" "), - ); - line - } - - /// Derive a formatted message from a set of options. - pub fn usage(&self, brief: &str) -> String { - self.usage_with_format(|opts| { - format!( - "{}\n\nOptions:\n{}\n", - brief, - opts.collect::>().join("\n") - ) - }) - } - - /// Derive a custom formatted message from a set of options. The formatted options provided to - /// a closure as an iterator. - pub fn usage_with_format) -> String>( - &self, - mut formatter: F, - ) -> String { - formatter(&mut self.usage_items()) - } - - /// Derive usage items from a set of options. - fn usage_items<'a>(&'a self) -> Box + 'a> { - let desc_sep = format!("\n{}", repeat(" ").take(24).collect::()); - - let any_short = self.grps.iter().any(|optref| !optref.short_name.is_empty()); - - let rows = self.grps.iter().map(move |optref| { - let OptGroup { - short_name, - long_name, - hint, - desc, - hasarg, - .. - } = (*optref).clone(); - - let mut row = " ".to_string(); - - // short option - match short_name.width() { - 0 => { - if any_short { - row.push_str(" "); - } - } - 1 => { - row.push('-'); - row.push_str(&short_name); - if long_name.width() > 0 { - row.push_str(", "); - } else { - // Only a single space here, so that any - // argument is printed in the correct spot. - row.push(' '); - } - } - // FIXME: refer issue #7. - _ => panic!("the short name should only be 1 ascii char long"), - } - - // long option - match long_name.width() { - 0 => {} - _ => { - row.push_str(if self.long_only { "-" } else { "--" }); - row.push_str(&long_name); - row.push(' '); - } - } - - // arg - match hasarg { - No => {} - Yes => row.push_str(&hint), - Maybe => { - row.push('['); - row.push_str(&hint); - row.push(']'); - } - } - - let rowlen = row.width(); - if rowlen < 24 { - for _ in 0..24 - rowlen { - row.push(' '); - } - } else { - row.push_str(&desc_sep) - } - - let desc_rows = each_split_within(&desc, 54); - row.push_str(&desc_rows.join(&desc_sep)); - - row - }); - - Box::new(rows) - } -} - -fn validate_names(short_name: &str, long_name: &str) { - let len = short_name.len(); - assert!( - len == 1 || len == 0, - "the short_name (first argument) should be a single character, \ - or an empty string for none" - ); - let len = long_name.len(); - assert!( - len == 0 || len > 1, - "the long_name (second argument) should be longer than a single \ - character, or an empty string for none" - ); -} - -/// What parsing style to use when parsing arguments. -#[derive(Clone, Copy, PartialEq, Eq)] -pub enum ParsingStyle { - /// Flags and "free" arguments can be freely inter-mixed. - FloatingFrees, - /// As soon as a "free" argument (i.e. non-flag) is encountered, stop - /// considering any remaining arguments as flags. - StopAtFirstFree, -} - -/// Name of an option. Either a string or a single char. -#[derive(Clone, Debug, PartialEq, Eq)] -enum Name { - /// A string representing the long name of an option. - /// For example: "help" - Long(String), - /// A char representing the short name of an option. - /// For example: 'h' - Short(char), -} - -/// Describes whether an option has an argument. -#[derive(Clone, Debug, Copy, PartialEq, Eq)] -pub enum HasArg { - /// The option requires an argument. - Yes, - /// The option takes no argument. - No, - /// The option argument is optional. - Maybe, -} - -/// Describes how often an option may occur. -#[derive(Clone, Debug, Copy, PartialEq, Eq)] -pub enum Occur { - /// The option occurs once. - Req, - /// The option occurs at most once. - Optional, - /// The option occurs zero or more times. - Multi, -} - -/// A description of a possible option. -#[derive(Clone, Debug, PartialEq, Eq)] -struct Opt { - /// Name of the option - name: Name, - /// Whether it has an argument - hasarg: HasArg, - /// How often it can occur - occur: Occur, - /// Which options it aliases - aliases: Vec, -} - -/// One group of options, e.g., both `-h` and `--help`, along with -/// their shared description and properties. -#[derive(Clone, PartialEq, Eq)] -struct OptGroup { - /// Short name of the option, e.g. `h` for a `-h` option - short_name: String, - /// Long name of the option, e.g. `help` for a `--help` option - long_name: String, - /// Hint for argument, e.g. `FILE` for a `-o FILE` option - hint: String, - /// Description for usage help text - desc: String, - /// Whether option has an argument - hasarg: HasArg, - /// How often it can occur - occur: Occur, -} - -/// Describes whether an option is given at all or has a value. -#[derive(Clone, Debug, PartialEq, Eq)] -enum Optval { - Val(String), - Given, -} - -/// The result of checking command line arguments. Contains a vector -/// of matches and a vector of free strings. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Matches { - /// Options that matched - opts: Vec, - /// Values of the Options that matched and their positions - vals: Vec>, - /// Free string fragments - pub free: Vec, -} - -/// The type returned when the command line does not conform to the -/// expected format. Use the `Debug` implementation to output detailed -/// information. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum Fail { - /// The option requires an argument but none was passed. - ArgumentMissing(String), - /// The passed option is not declared among the possible options. - UnrecognizedOption(String), - /// A required option is not present. - OptionMissing(String), - /// A single occurrence option is being used multiple times. - OptionDuplicated(String), - /// There's an argument being passed to a non-argument option. - UnexpectedArgument(String), -} - -impl Error for Fail {} - -/// The result of parsing a command line with a set of options. -pub type Result = result::Result; - -impl Name { - fn from_str(nm: &str) -> Name { - if nm.len() == 1 { - Short(nm.as_bytes()[0] as char) - } else { - Long(nm.to_string()) - } - } - - fn to_string(&self) -> String { - match *self { - Short(ch) => ch.to_string(), - Long(ref s) => s.to_string(), - } - } -} - -impl OptGroup { - /// Translate OptGroup into Opt. - /// (Both short and long names correspond to different Opts). - fn long_to_short(&self) -> Opt { - let OptGroup { - short_name, - long_name, - hasarg, - occur, - .. - } = (*self).clone(); - - match (short_name.len(), long_name.len()) { - (0, 0) => panic!("this long-format option was given no name"), - (0, _) => Opt { - name: Long(long_name), - hasarg, - occur, - aliases: Vec::new(), - }, - (1, 0) => Opt { - name: Short(short_name.as_bytes()[0] as char), - hasarg, - occur, - aliases: Vec::new(), - }, - (1, _) => Opt { - name: Long(long_name), - hasarg, - occur, - aliases: vec![Opt { - name: Short(short_name.as_bytes()[0] as char), - hasarg: hasarg, - occur: occur, - aliases: Vec::new(), - }], - }, - (_, _) => panic!("something is wrong with the long-form opt"), - } - } -} - -impl Matches { - fn opt_vals(&self, nm: &str) -> Vec<(usize, Optval)> { - match find_opt(&self.opts, &Name::from_str(nm)) { - Some(id) => self.vals[id].clone(), - None => panic!("No option '{}' defined", nm), - } - } - - fn opt_val(&self, nm: &str) -> Option { - self.opt_vals(nm).into_iter().map(|(_, o)| o).next() - } - /// Returns true if an option was defined - pub fn opt_defined(&self, name: &str) -> bool { - find_opt(&self.opts, &Name::from_str(name)).is_some() - } - - /// Returns true if an option was matched. - /// - /// # Panics - /// - /// This function will panic if the option name is not defined. - pub fn opt_present(&self, name: &str) -> bool { - !self.opt_vals(name).is_empty() - } - - /// Returns the number of times an option was matched. - /// - /// # Panics - /// - /// This function will panic if the option name is not defined. - pub fn opt_count(&self, name: &str) -> usize { - self.opt_vals(name).len() - } - - /// Returns a vector of all the positions in which an option was matched. - /// - /// # Panics - /// - /// This function will panic if the option name is not defined. - pub fn opt_positions(&self, name: &str) -> Vec { - self.opt_vals(name).into_iter().map(|(pos, _)| pos).collect() - } - - /// Returns true if any of several options were matched. - pub fn opts_present(&self, names: &[String]) -> bool { - names - .iter() - .any(|nm| match find_opt(&self.opts, &Name::from_str(&nm)) { - Some(id) if !self.vals[id].is_empty() => true, - _ => false, - }) - } - - /// Returns true if any of several options were matched. - /// - /// Similar to `opts_present` but accepts any argument that can be converted - /// into an iterator over string references. - /// - /// # Panics - /// - /// This function might panic if some option name is not defined. - /// - /// # Example - /// - /// ``` - /// # use getopts::Options; - /// let mut opts = Options::new(); - /// opts.optopt("a", "alpha", "first option", "STR"); - /// opts.optopt("b", "beta", "second option", "STR"); - /// - /// let args = vec!["-a", "foo"]; - /// let matches = &match opts.parse(&args) { - /// Ok(m) => m, - /// _ => panic!(), - /// }; - /// - /// assert!(matches.opts_present_any(&["alpha"])); - /// assert!(!matches.opts_present_any(&["beta"])); - /// ``` - pub fn opts_present_any(&self, names: C) -> bool - where - C::Item: AsRef, - { - names - .into_iter() - .any(|nm| !self.opt_vals(nm.as_ref()).is_empty()) - } - - /// Returns the string argument supplied to one of several matching options or `None`. - pub fn opts_str(&self, names: &[String]) -> Option { - names - .iter() - .filter_map(|nm| match self.opt_val(&nm) { - Some(Val(s)) => Some(s), - _ => None, - }) - .next() - } - - /// Returns the string argument supplied to the first matching option of - /// several options or `None`. - /// - /// Similar to `opts_str` but accepts any argument that can be converted - /// into an iterator over string references. - /// - /// # Panics - /// - /// This function might panic if some option name is not defined. - /// - /// # Example - /// - /// ``` - /// # use getopts::Options; - /// let mut opts = Options::new(); - /// opts.optopt("a", "alpha", "first option", "STR"); - /// opts.optopt("b", "beta", "second option", "STR"); - /// - /// let args = vec!["-a", "foo", "--beta", "bar"]; - /// let matches = &match opts.parse(&args) { - /// Ok(m) => m, - /// _ => panic!(), - /// }; - /// - /// assert_eq!(Some("foo".to_string()), matches.opts_str_first(&["alpha", "beta"])); - /// assert_eq!(Some("bar".to_string()), matches.opts_str_first(&["beta", "alpha"])); - /// ``` - pub fn opts_str_first(&self, names: C) -> Option - where - C::Item: AsRef, - { - names - .into_iter() - .filter_map(|nm| match self.opt_val(nm.as_ref()) { - Some(Val(s)) => Some(s), - _ => None, - }) - .next() - } - - /// Returns a vector of the arguments provided to all matches of the given - /// option. - /// - /// Used when an option accepts multiple values. - /// - /// # Panics - /// - /// This function will panic if the option name is not defined. - pub fn opt_strs(&self, name: &str) -> Vec { - self.opt_vals(name) - .into_iter() - .filter_map(|(_, v)| match v { - Val(s) => Some(s), - _ => None, - }) - .collect() - } - - /// Returns a vector of the arguments provided to all matches of the given - /// option, together with their positions. - /// - /// Used when an option accepts multiple values. - /// - /// # Panics - /// - /// This function will panic if the option name is not defined. - pub fn opt_strs_pos(&self, name: &str) -> Vec<(usize, String)> { - self.opt_vals(name) - .into_iter() - .filter_map(|(p, v)| match v { - Val(s) => Some((p, s)), - _ => None, - }) - .collect() - } - - /// Returns the string argument supplied to a matching option or `None`. - /// - /// # Panics - /// - /// This function will panic if the option name is not defined. - pub fn opt_str(&self, name: &str) -> Option { - match self.opt_val(name) { - Some(Val(s)) => Some(s), - _ => None, - } - } - - /// Returns the matching string, a default, or `None`. - /// - /// Returns `None` if the option was not present, `def` if the option was - /// present but no argument was provided, and the argument if the option was - /// present and an argument was provided. - /// - /// # Panics - /// - /// This function will panic if the option name is not defined. - pub fn opt_default(&self, name: &str, def: &str) -> Option { - match self.opt_val(name) { - Some(Val(s)) => Some(s), - Some(_) => Some(def.to_string()), - None => None, - } - } - - /// Returns some matching value or `None`. - /// - /// Similar to opt_str, also converts matching argument using FromStr. - /// - /// # Panics - /// - /// This function will panic if the option name is not defined. - pub fn opt_get(&self, name: &str) -> result::Result, T::Err> - where - T: FromStr, - { - match self.opt_val(name) { - Some(Val(s)) => Ok(Some(s.parse()?)), - Some(Given) => Ok(None), - None => Ok(None), - } - } - - /// Returns a matching value or default. - /// - /// Similar to opt_default, except the two differences. - /// Instead of returning None when argument was not present, return `def`. - /// Instead of returning &str return type T, parsed using str::parse(). - /// - /// # Panics - /// - /// This function will panic if the option name is not defined. - pub fn opt_get_default(&self, name: &str, def: T) -> result::Result - where - T: FromStr, - { - match self.opt_val(name) { - Some(Val(s)) => s.parse(), - Some(Given) => Ok(def), - None => Ok(def), - } - } -} - -fn is_arg(arg: &str) -> bool { - arg.as_bytes().get(0) == Some(&b'-') && arg.len() > 1 -} - -fn find_opt(opts: &[Opt], nm: &Name) -> Option { - // Search main options. - let pos = opts.iter().position(|opt| &opt.name == nm); - if pos.is_some() { - return pos; - } - - // Search in aliases. - for candidate in opts.iter() { - if candidate.aliases.iter().any(|opt| &opt.name == nm) { - return opts.iter().position(|opt| opt.name == candidate.name); - } - } - - None -} - -impl fmt::Display for Fail { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match *self { - ArgumentMissing(ref nm) => write!(f, "Argument to option '{}' missing", *nm), - UnrecognizedOption(ref nm) => write!(f, "Unrecognized option: '{}'", *nm), - OptionMissing(ref nm) => write!(f, "Required option '{}' missing", *nm), - OptionDuplicated(ref nm) => write!(f, "Option '{}' given more than once", *nm), - UnexpectedArgument(ref nm) => write!(f, "Option '{}' does not take an argument", *nm), - } - } -} - -fn format_option(opt: &OptGroup) -> String { - let mut line = String::new(); - - if opt.occur != Req { - line.push('['); - } - - // Use short_name if possible, but fall back to long_name. - if !opt.short_name.is_empty() { - line.push('-'); - line.push_str(&opt.short_name); - } else { - line.push_str("--"); - line.push_str(&opt.long_name); - } - - if opt.hasarg != No { - line.push(' '); - if opt.hasarg == Maybe { - line.push('['); - } - line.push_str(&opt.hint); - if opt.hasarg == Maybe { - line.push(']'); - } - } - - if opt.occur != Req { - line.push(']'); - } - if opt.occur == Multi { - line.push_str(".."); - } - - line -} - -/// Splits a string into substrings with possibly internal whitespace, -/// each of them at most `lim` bytes long, if possible. The substrings -/// have leading and trailing whitespace removed, and are only cut at -/// whitespace boundaries. -fn each_split_within(desc: &str, lim: usize) -> Vec { - let mut rows = Vec::new(); - for line in desc.trim().lines() { - let line_chars = line.chars().chain(Some(' ')); - let words = line_chars - .fold((Vec::new(), 0, 0), |(mut words, a, z), c| { - let idx = z + c.len_utf8(); // Get the current byte offset - - // If the char is whitespace, advance the word start and maybe push a word - if c.is_whitespace() { - if a != z { - words.push(&line[a..z]); - } - (words, idx, idx) - } - // If the char is not whitespace, continue, retaining the current - else { - (words, a, idx) - } - }) - .0; - - let mut row = String::new(); - for word in words.iter() { - let sep = if !row.is_empty() { Some(" ") } else { None }; - let width = row.width() + word.width() + sep.map(UnicodeWidthStr::width).unwrap_or(0); - - if width <= lim { - if let Some(sep) = sep { - row.push_str(sep) - } - row.push_str(word); - continue; - } - if !row.is_empty() { - rows.push(row.clone()); - row.clear(); - } - row.push_str(word); - } - if !row.is_empty() { - rows.push(row); - } - } - rows -} diff --git a/crux-mir/lib/getopts/src/tests/mod.rs b/crux-mir/lib/getopts/src/tests/mod.rs deleted file mode 100644 index d578e8432..000000000 --- a/crux-mir/lib/getopts/src/tests/mod.rs +++ /dev/null @@ -1,1296 +0,0 @@ -use super::each_split_within; -use super::Fail::*; -use super::{HasArg, Name, Occur, Opt, Options, ParsingStyle}; - -#[test] -fn test_split_within() { - fn t(s: &str, i: usize, u: &[String]) { - let v = each_split_within(&(s.to_string()), i); - assert!(v.iter().zip(u.iter()).all(|(a, b)| a == b)); - } - t("", 0, &[]); - t("", 15, &[]); - t("hello", 15, &["hello".to_string()]); - t( - "\nMary had a little lamb\nLittle lamb\n", - 15, - &[ - "Mary had a".to_string(), - "little lamb".to_string(), - "Little lamb".to_string(), - ], - ); - t( - "\nMary had a little lamb\nLittle lamb\n", - ::std::usize::MAX, - &[ - "Mary had a little lamb".to_string(), - "Little lamb".to_string(), - ], - ); -} - -// Tests for reqopt -#[test] -fn test_reqopt() { - let long_args = vec!["--test=20".to_string()]; - let mut opts = Options::new(); - opts.reqopt("t", "test", "testing", "TEST"); - match opts.parse(&long_args) { - Ok(ref m) => { - assert!(m.opt_present("test")); - assert_eq!(m.opt_str("test").unwrap(), "20"); - assert!(m.opt_present("t")); - assert_eq!(m.opt_str("t").unwrap(), "20"); - } - _ => { - panic!("test_reqopt failed (long arg)"); - } - } - let short_args = vec!["-t".to_string(), "20".to_string()]; - match opts.parse(&short_args) { - Ok(ref m) => { - assert!((m.opt_present("test"))); - assert_eq!(m.opt_str("test").unwrap(), "20"); - assert!((m.opt_present("t"))); - assert_eq!(m.opt_str("t").unwrap(), "20"); - } - _ => { - panic!("test_reqopt failed (short arg)"); - } - } -} - -#[test] -fn test_reqopt_missing() { - let args = vec!["blah".to_string()]; - match Options::new() - .reqopt("t", "test", "testing", "TEST") - .parse(&args) - { - Err(OptionMissing(_)) => {} - _ => panic!(), - } -} - -#[test] -fn test_reqopt_no_arg() { - let long_args = vec!["--test".to_string()]; - let mut opts = Options::new(); - opts.reqopt("t", "test", "testing", "TEST"); - match opts.parse(&long_args) { - Err(ArgumentMissing(_)) => {} - _ => panic!(), - } - let short_args = vec!["-t".to_string()]; - match opts.parse(&short_args) { - Err(ArgumentMissing(_)) => {} - _ => panic!(), - } -} - -#[test] -fn test_reqopt_multi() { - let args = vec!["--test=20".to_string(), "-t".to_string(), "30".to_string()]; - match Options::new() - .reqopt("t", "test", "testing", "TEST") - .parse(&args) - { - Err(OptionDuplicated(_)) => {} - _ => panic!(), - } -} - -// Tests for optopt -#[test] -fn test_optopt() { - let long_args = vec!["--test=20".to_string()]; - let mut opts = Options::new(); - opts.optopt("t", "test", "testing", "TEST"); - match opts.parse(&long_args) { - Ok(ref m) => { - assert!(m.opt_present("test")); - assert_eq!(m.opt_str("test").unwrap(), "20"); - assert!((m.opt_present("t"))); - assert_eq!(m.opt_str("t").unwrap(), "20"); - } - _ => panic!(), - } - let short_args = vec!["-t".to_string(), "20".to_string()]; - match opts.parse(&short_args) { - Ok(ref m) => { - assert!((m.opt_present("test"))); - assert_eq!(m.opt_str("test").unwrap(), "20"); - assert!((m.opt_present("t"))); - assert_eq!(m.opt_str("t").unwrap(), "20"); - } - _ => panic!(), - } -} - -#[test] -fn test_optopt_missing() { - let args = vec!["blah".to_string()]; - match Options::new() - .optopt("t", "test", "testing", "TEST") - .parse(&args) - { - Ok(ref m) => { - assert!(!m.opt_present("test")); - assert!(!m.opt_present("t")); - } - _ => panic!(), - } -} - -#[test] -fn test_optopt_no_arg() { - let long_args = vec!["--test".to_string()]; - let mut opts = Options::new(); - opts.optopt("t", "test", "testing", "TEST"); - match opts.parse(&long_args) { - Err(ArgumentMissing(_)) => {} - _ => panic!(), - } - let short_args = vec!["-t".to_string()]; - match opts.parse(&short_args) { - Err(ArgumentMissing(_)) => {} - _ => panic!(), - } -} - -#[test] -fn test_optopt_multi() { - let args = vec!["--test=20".to_string(), "-t".to_string(), "30".to_string()]; - match Options::new() - .optopt("t", "test", "testing", "TEST") - .parse(&args) - { - Err(OptionDuplicated(_)) => {} - _ => panic!(), - } -} - -// Tests for optflag -#[test] -fn test_optflag() { - let long_args = vec!["--test".to_string()]; - let mut opts = Options::new(); - opts.optflag("t", "test", "testing"); - match opts.parse(&long_args) { - Ok(ref m) => { - assert!(m.opt_present("test")); - assert!(m.opt_present("t")); - } - _ => panic!(), - } - let short_args = vec!["-t".to_string()]; - match opts.parse(&short_args) { - Ok(ref m) => { - assert!(m.opt_present("test")); - assert!(m.opt_present("t")); - } - _ => panic!(), - } -} - -#[test] -fn test_optflag_missing() { - let args = vec!["blah".to_string()]; - match Options::new().optflag("t", "test", "testing").parse(&args) { - Ok(ref m) => { - assert!(!m.opt_present("test")); - assert!(!m.opt_present("t")); - } - _ => panic!(), - } -} - -#[test] -fn test_opt_end() { - let args = vec!["--".to_owned(), "-t".to_owned()]; - match Options::new().optflag("t", "test", "testing").parse(&args) { - Ok(ref m) => { - assert!(!m.opt_present("test")); - assert!(!m.opt_present("t")); - assert_eq!(m.free.len(), 1); - assert_eq!(m.free[0], "-t"); - } - _ => panic!(), - } -} - -#[test] -fn test_opt_only_end() { - let args = vec!["--".to_owned()]; - match Options::new().optflag("t", "test", "testing").parse(&args) { - Ok(ref m) => { - assert!(!m.opt_present("test")); - assert!(!m.opt_present("t")); - assert_eq!(m.free.len(), 0); - } - _ => panic!(), - } -} - -#[test] -fn test_optflag_long_arg() { - let args = vec!["--test=20".to_string()]; - match Options::new().optflag("t", "test", "testing").parse(&args) { - Err(UnexpectedArgument(_)) => {} - _ => panic!(), - } -} - -#[test] -fn test_optflag_multi() { - let args = vec!["--test".to_string(), "-t".to_string()]; - match Options::new().optflag("t", "test", "testing").parse(&args) { - Err(OptionDuplicated(_)) => {} - _ => panic!(), - } -} - -#[test] -fn test_optflag_short_arg() { - let args = vec!["-t".to_string(), "20".to_string()]; - match Options::new().optflag("t", "test", "testing").parse(&args) { - Ok(ref m) => { - // The next variable after the flag is just a free argument - - assert!(m.free[0] == "20"); - } - _ => panic!(), - } -} - -// Tests for optflagmulti -#[test] -fn test_optflagmulti_short1() { - let args = vec!["-v".to_string()]; - match Options::new() - .optflagmulti("v", "verbose", "verbosity") - .parse(&args) - { - Ok(ref m) => { - assert_eq!(m.opt_count("v"), 1); - } - _ => panic!(), - } -} - -#[test] -fn test_optflagmulti_short2a() { - let args = vec!["-v".to_string(), "-v".to_string()]; - match Options::new() - .optflagmulti("v", "verbose", "verbosity") - .parse(&args) - { - Ok(ref m) => { - assert_eq!(m.opt_count("v"), 2); - } - _ => panic!(), - } -} - -#[test] -fn test_optflagmulti_short2b() { - let args = vec!["-vv".to_string()]; - match Options::new() - .optflagmulti("v", "verbose", "verbosity") - .parse(&args) - { - Ok(ref m) => { - assert_eq!(m.opt_count("v"), 2); - } - _ => panic!(), - } -} - -#[test] -fn test_optflagmulti_long1() { - let args = vec!["--verbose".to_string()]; - match Options::new() - .optflagmulti("v", "verbose", "verbosity") - .parse(&args) - { - Ok(ref m) => { - assert_eq!(m.opt_count("verbose"), 1); - } - _ => panic!(), - } -} - -#[test] -fn test_optflagmulti_long2() { - let args = vec!["--verbose".to_string(), "--verbose".to_string()]; - match Options::new() - .optflagmulti("v", "verbose", "verbosity") - .parse(&args) - { - Ok(ref m) => { - assert_eq!(m.opt_count("verbose"), 2); - } - _ => panic!(), - } -} - -#[test] -fn test_optflagmulti_mix() { - let args = vec![ - "--verbose".to_string(), - "-v".to_string(), - "-vv".to_string(), - "verbose".to_string(), - ]; - match Options::new() - .optflagmulti("v", "verbose", "verbosity") - .parse(&args) - { - Ok(ref m) => { - assert_eq!(m.opt_count("verbose"), 4); - assert_eq!(m.opt_count("v"), 4); - } - _ => panic!(), - } -} - -// Tests for optflagopt -#[test] -fn test_optflagopt() { - let long_args = vec!["--test".to_string()]; - let mut opts = Options::new(); - opts.optflagopt("t", "test", "testing", "ARG"); - match opts.parse(&long_args) { - Ok(ref m) => { - assert!(m.opt_present("test")); - assert!(m.opt_present("t")); - } - _ => panic!(), - } - let short_args = vec!["-t".to_string()]; - match opts.parse(&short_args) { - Ok(ref m) => { - assert!(m.opt_present("test")); - assert!(m.opt_present("t")); - } - _ => panic!(), - } - let short_args = vec!["-t".to_string(), "x".to_string()]; - match opts.parse(&short_args) { - Ok(ref m) => { - assert_eq!(m.opt_str("t").unwrap(), "x"); - assert_eq!(m.opt_str("test").unwrap(), "x"); - } - _ => panic!(), - } - let long_args = vec!["--test=x".to_string()]; - match opts.parse(&long_args) { - Ok(ref m) => { - assert_eq!(m.opt_str("t").unwrap(), "x"); - assert_eq!(m.opt_str("test").unwrap(), "x"); - } - _ => panic!(), - } - let long_args = vec!["--test".to_string(), "x".to_string()]; - match opts.parse(&long_args) { - Ok(ref m) => { - assert_eq!(m.opt_str("t"), None); - assert_eq!(m.opt_str("test"), None); - } - _ => panic!(), - } - let no_args: Vec = vec![]; - match opts.parse(&no_args) { - Ok(ref m) => { - assert!(!m.opt_present("test")); - assert!(!m.opt_present("t")); - } - _ => panic!(), - } -} - -// Tests for optmulti -#[test] -fn test_optmulti() { - let long_args = vec!["--test=20".to_string()]; - let mut opts = Options::new(); - opts.optmulti("t", "test", "testing", "TEST"); - match opts.parse(&long_args) { - Ok(ref m) => { - assert!((m.opt_present("test"))); - assert_eq!(m.opt_str("test").unwrap(), "20"); - assert!((m.opt_present("t"))); - assert_eq!(m.opt_str("t").unwrap(), "20"); - } - _ => panic!(), - } - let short_args = vec!["-t".to_string(), "20".to_string()]; - match opts.parse(&short_args) { - Ok(ref m) => { - assert!((m.opt_present("test"))); - assert_eq!(m.opt_str("test").unwrap(), "20"); - assert!((m.opt_present("t"))); - assert_eq!(m.opt_str("t").unwrap(), "20"); - } - _ => panic!(), - } -} - -#[test] -fn test_optmulti_missing() { - let args = vec!["blah".to_string()]; - match Options::new() - .optmulti("t", "test", "testing", "TEST") - .parse(&args) - { - Ok(ref m) => { - assert!(!m.opt_present("test")); - assert!(!m.opt_present("t")); - } - _ => panic!(), - } -} - -#[test] -fn test_optmulti_no_arg() { - let long_args = vec!["--test".to_string()]; - let mut opts = Options::new(); - opts.optmulti("t", "test", "testing", "TEST"); - match opts.parse(&long_args) { - Err(ArgumentMissing(_)) => {} - _ => panic!(), - } - let short_args = vec!["-t".to_string()]; - match opts.parse(&short_args) { - Err(ArgumentMissing(_)) => {} - _ => panic!(), - } -} - -#[test] -fn test_optmulti_multi() { - let args = vec!["--test=20".to_string(), "-t".to_string(), "30".to_string()]; - match Options::new() - .optmulti("t", "test", "testing", "TEST") - .parse(&args) - { - Ok(ref m) => { - assert!(m.opt_present("test")); - assert_eq!(m.opt_str("test").unwrap(), "20"); - assert!(m.opt_present("t")); - assert_eq!(m.opt_str("t").unwrap(), "20"); - let pair = m.opt_strs("test"); - assert!(pair[0] == "20"); - assert!(pair[1] == "30"); - } - _ => panic!(), - } -} - -#[test] -fn test_free_argument_is_hyphen() { - let args = vec!["-".to_string()]; - match Options::new().parse(&args) { - Ok(ref m) => { - assert_eq!(m.free.len(), 1); - assert_eq!(m.free[0], "-"); - } - _ => panic!(), - } -} - -#[test] -fn test_unrecognized_option() { - let long_args = vec!["--untest".to_string()]; - let mut opts = Options::new(); - opts.optmulti("t", "test", "testing", "TEST"); - match opts.parse(&long_args) { - Err(UnrecognizedOption(_)) => {} - _ => panic!(), - } - let short_args = vec!["-u".to_string()]; - match opts.parse(&short_args) { - Err(UnrecognizedOption(_)) => {} - _ => panic!(), - } -} - -#[test] -fn test_combined() { - let args = vec![ - "prog".to_string(), - "free1".to_string(), - "-s".to_string(), - "20".to_string(), - "free2".to_string(), - "--flag".to_string(), - "--long=30".to_string(), - "-f".to_string(), - "-m".to_string(), - "40".to_string(), - "-m".to_string(), - "50".to_string(), - "-n".to_string(), - "-A B".to_string(), - "-n".to_string(), - "-60 70".to_string(), - ]; - match Options::new() - .optopt("s", "something", "something", "SOMETHING") - .optflag("", "flag", "a flag") - .reqopt("", "long", "hi", "LONG") - .optflag("f", "", "another flag") - .optmulti("m", "", "mmmmmm", "YUM") - .optmulti("n", "", "nothing", "NOTHING") - .optopt("", "notpresent", "nothing to see here", "NOPE") - .parse(&args) - { - Ok(ref m) => { - assert!(m.free[0] == "prog"); - assert!(m.free[1] == "free1"); - assert_eq!(m.opt_str("s").unwrap(), "20"); - assert!(m.free[2] == "free2"); - assert!((m.opt_present("flag"))); - assert_eq!(m.opt_str("long").unwrap(), "30"); - assert!((m.opt_present("f"))); - let pair = m.opt_strs("m"); - assert!(pair[0] == "40"); - assert!(pair[1] == "50"); - let pair = m.opt_strs("n"); - assert!(pair[0] == "-A B"); - assert!(pair[1] == "-60 70"); - assert!((!m.opt_present("notpresent"))); - } - _ => panic!(), - } -} - -#[test] -fn test_mixed_stop() { - let args = vec![ - "-a".to_string(), - "b".to_string(), - "-c".to_string(), - "d".to_string(), - ]; - match Options::new() - .parsing_style(ParsingStyle::StopAtFirstFree) - .optflag("a", "", "") - .optopt("c", "", "", "") - .parse(&args) - { - Ok(ref m) => { - println!("{}", m.opt_present("c")); - assert!(m.opt_present("a")); - assert!(!m.opt_present("c")); - assert_eq!(m.free.len(), 3); - assert_eq!(m.free[0], "b"); - assert_eq!(m.free[1], "-c"); - assert_eq!(m.free[2], "d"); - } - _ => panic!(), - } -} - -#[test] -fn test_mixed_stop_hyphen() { - let args = vec![ - "-a".to_string(), - "-".to_string(), - "-c".to_string(), - "d".to_string(), - ]; - match Options::new() - .parsing_style(ParsingStyle::StopAtFirstFree) - .optflag("a", "", "") - .optopt("c", "", "", "") - .parse(&args) - { - Ok(ref m) => { - println!("{}", m.opt_present("c")); - assert!(m.opt_present("a")); - assert!(!m.opt_present("c")); - assert_eq!(m.free.len(), 3); - assert_eq!(m.free[0], "-"); - assert_eq!(m.free[1], "-c"); - assert_eq!(m.free[2], "d"); - } - _ => panic!(), - } -} - -#[test] -fn test_multi() { - let mut opts = Options::new(); - opts.optopt("e", "", "encrypt", "ENCRYPT"); - opts.optopt("", "encrypt", "encrypt", "ENCRYPT"); - opts.optopt("f", "", "flag", "FLAG"); - let no_opts: &[&str] = &[]; - - let args_single = vec!["-e".to_string(), "foo".to_string()]; - let matches_single = &match opts.parse(&args_single) { - Ok(m) => m, - _ => panic!(), - }; - assert!(matches_single.opts_present(&["e".to_string()])); - assert!(matches_single.opts_present(&["encrypt".to_string(), "e".to_string()])); - assert!(matches_single.opts_present(&["e".to_string(), "encrypt".to_string()])); - assert!(!matches_single.opts_present(&["encrypt".to_string()])); - assert!(!matches_single.opts_present(&["thing".to_string()])); - assert!(!matches_single.opts_present(&[])); - - assert!(matches_single.opts_present_any(&["e"])); - assert!(matches_single.opts_present_any(&["encrypt", "e"])); - assert!(matches_single.opts_present_any(&["e", "encrypt"])); - assert!(!matches_single.opts_present_any(&["encrypt"])); - assert!(!matches_single.opts_present_any(no_opts)); - - assert_eq!(matches_single.opts_str(&["e".to_string()]).unwrap(), "foo"); - assert_eq!( - matches_single - .opts_str(&["e".to_string(), "encrypt".to_string()]) - .unwrap(), - "foo" - ); - assert_eq!( - matches_single - .opts_str(&["encrypt".to_string(), "e".to_string()]) - .unwrap(), - "foo" - ); - - assert_eq!(matches_single.opts_str_first(&["e"]).unwrap(), "foo"); - assert_eq!( - matches_single.opts_str_first(&["e", "encrypt"]).unwrap(), - "foo" - ); - assert_eq!( - matches_single.opts_str_first(&["encrypt", "e"]).unwrap(), - "foo" - ); - assert_eq!(matches_single.opts_str_first(&["encrypt"]), None); - assert_eq!(matches_single.opts_str_first(no_opts), None); - - let args_both = vec![ - "-e".to_string(), - "foo".to_string(), - "--encrypt".to_string(), - "bar".to_string(), - ]; - let matches_both = &match opts.parse(&args_both) { - Ok(m) => m, - _ => panic!(), - }; - assert!(matches_both.opts_present(&["e".to_string()])); - assert!(matches_both.opts_present(&["encrypt".to_string()])); - assert!(matches_both.opts_present(&["encrypt".to_string(), "e".to_string()])); - assert!(matches_both.opts_present(&["e".to_string(), "encrypt".to_string()])); - assert!(!matches_both.opts_present(&["f".to_string()])); - assert!(!matches_both.opts_present(&["thing".to_string()])); - assert!(!matches_both.opts_present(&[])); - - assert!(matches_both.opts_present_any(&["e"])); - assert!(matches_both.opts_present_any(&["encrypt"])); - assert!(matches_both.opts_present_any(&["encrypt", "e"])); - assert!(matches_both.opts_present_any(&["e", "encrypt"])); - assert!(!matches_both.opts_present_any(&["f"])); - assert!(!matches_both.opts_present_any(no_opts)); - - assert_eq!(matches_both.opts_str(&["e".to_string()]).unwrap(), "foo"); - assert_eq!( - matches_both.opts_str(&["encrypt".to_string()]).unwrap(), - "bar" - ); - assert_eq!( - matches_both - .opts_str(&["e".to_string(), "encrypt".to_string()]) - .unwrap(), - "foo" - ); - assert_eq!( - matches_both - .opts_str(&["encrypt".to_string(), "e".to_string()]) - .unwrap(), - "bar" - ); - - assert_eq!(matches_both.opts_str_first(&["e"]).unwrap(), "foo"); - assert_eq!( - matches_both.opts_str_first(&["encrypt"]).unwrap(), - "bar" - ); - assert_eq!( - matches_both.opts_str_first(&["e", "encrypt"]).unwrap(), - "foo" - ); - assert_eq!( - matches_both.opts_str_first(&["encrypt", "e"]).unwrap(), - "bar" - ); - assert_eq!(matches_both.opts_str_first(&["f"]), None); - assert_eq!(matches_both.opts_str_first(no_opts), None); -} - -#[test] -fn test_nospace() { - let args = vec!["-Lfoo".to_string(), "-M.".to_string()]; - let matches = &match Options::new() - .optmulti("L", "", "library directory", "LIB") - .optmulti("M", "", "something", "MMMM") - .parse(&args) - { - Ok(m) => m, - _ => panic!(), - }; - assert!(matches.opts_present(&["L".to_string()])); - assert_eq!(matches.opts_str(&["L".to_string()]).unwrap(), "foo"); - assert!(matches.opts_present(&["M".to_string()])); - assert_eq!(matches.opts_str(&["M".to_string()]).unwrap(), "."); -} - -#[test] -fn test_nospace_conflict() { - let args = vec!["-vvLverbose".to_string(), "-v".to_string()]; - let matches = &match Options::new() - .optmulti("L", "", "library directory", "LIB") - .optflagmulti("v", "verbose", "Verbose") - .parse(&args) - { - Ok(m) => m, - Err(e) => panic!("{}", e), - }; - assert!(matches.opts_present(&["L".to_string()])); - assert_eq!(matches.opts_str(&["L".to_string()]).unwrap(), "verbose"); - assert!(matches.opts_present(&["v".to_string()])); - assert_eq!(3, matches.opt_count("v")); -} - -#[test] -fn test_long_to_short() { - let mut short = Opt { - name: Name::Long("banana".to_string()), - hasarg: HasArg::Yes, - occur: Occur::Req, - aliases: Vec::new(), - }; - short.aliases = vec![Opt { - name: Name::Short('b'), - hasarg: HasArg::Yes, - occur: Occur::Req, - aliases: Vec::new(), - }]; - let mut opts = Options::new(); - opts.reqopt("b", "banana", "some bananas", "VAL"); - let verbose = &opts.grps[0]; - assert!(verbose.long_to_short() == short); -} - -#[test] -fn test_aliases_long_and_short() { - let args = vec!["-a".to_string(), "--apple".to_string(), "-a".to_string()]; - - let matches = Options::new() - .optflagmulti("a", "apple", "Desc") - .parse(&args) - .unwrap(); - assert_eq!(3, matches.opt_count("a")); - assert_eq!(3, matches.opt_count("apple")); -} - -#[test] -fn test_usage() { - let mut opts = Options::new(); - opts.reqopt("b", "banana", "Desc", "VAL"); - opts.optopt("a", "012345678901234567890123456789", "Desc", "VAL"); - opts.optflag("k", "kiwi", "Desc"); - opts.optflagopt("p", "", "Desc", "VAL"); - opts.optmulti("l", "", "Desc", "VAL"); - opts.optflag("", "starfruit", "Starfruit"); - - let expected = "Usage: fruits - -Options: - -b, --banana VAL Desc - -a, --012345678901234567890123456789 VAL - Desc - -k, --kiwi Desc - -p [VAL] Desc - -l VAL Desc - --starfruit Starfruit -"; - - let generated_usage = opts.usage("Usage: fruits"); - - debug!("expected: <<{}>>", expected); - debug!("generated: <<{}>>", generated_usage); - assert_eq!(generated_usage, expected); -} - -#[test] -fn test_usage_description_wrapping() { - // indentation should be 24 spaces - // lines wrap after 78: or rather descriptions wrap after 54 - - let mut opts = Options::new(); - opts.optflag( - "k", - "kiwi", - "This is a long description which won't be wrapped..+..", - ); // 54 - opts.optflag( - "a", - "apple", - "This is a long description which _will_ be wrapped..+..", - ); - opts.optflag( - "b", - "banana", - "HereWeNeedOneSingleWordThatIsLongerThanTheWrappingLengthAndThisIsIt", - ); - - let expected = "Usage: fruits - -Options: - -k, --kiwi This is a long description which won't be wrapped..+.. - -a, --apple This is a long description which _will_ be - wrapped..+.. - -b, --banana HereWeNeedOneSingleWordThatIsLongerThanTheWrappingLengthAndThisIsIt -"; - - let usage = opts.usage("Usage: fruits"); - - debug!("expected: <<{}>>", expected); - debug!("generated: <<{}>>", usage); - assert!(usage == expected) -} - -#[test] -fn test_usage_description_multibyte_handling() { - let mut opts = Options::new(); - opts.optflag( - "k", - "k\u{2013}w\u{2013}", - "The word kiwi is normally spelled with two i's", - ); - opts.optflag( - "a", - "apple", - "This \u{201C}description\u{201D} has some characters that could \ - confuse the line wrapping; an apple costs 0.51€ in some parts of Europe.", - ); - - let expected = "Usage: fruits - -Options: - -k, --k–w– The word kiwi is normally spelled with two i's - -a, --apple This “description” has some characters that could - confuse the line wrapping; an apple costs 0.51€ in - some parts of Europe. -"; - - let usage = opts.usage("Usage: fruits"); - - debug!("expected: <<{}>>", expected); - debug!("generated: <<{}>>", usage); - assert!(usage == expected) -} - -#[test] -fn test_usage_description_newline_handling() { - let mut opts = Options::new(); - opts.optflag( - "k", - "k\u{2013}w\u{2013}", - "The word kiwi is normally spelled with two i's", - ); - opts.optflag( - "a", - "apple", - "This description forces a new line.\n Here is a premature\n\ - newline", - ); - - let expected = "Usage: fruits - -Options: - -k, --k–w– The word kiwi is normally spelled with two i's - -a, --apple This description forces a new line. - Here is a premature - newline -"; - - let usage = opts.usage("Usage: fruits"); - - debug!("expected: <<{}>>", expected); - debug!("generated: <<{}>>", usage); - assert!(usage == expected) -} - -#[test] -fn test_usage_multiwidth() { - let mut opts = Options::new(); - opts.optflag("a", "apple", "apple description"); - opts.optflag("b", "banana\u{00AB}", "banana description"); - opts.optflag("c", "brûlée", "brûlée quite long description"); - opts.optflag("k", "kiwi\u{20AC}", "kiwi description"); - opts.optflag("o", "orange\u{2039}", "orange description"); - opts.optflag( - "r", - "raspberry-but-making-this-option-way-too-long", - "raspberry description is also quite long indeed longer than \ - every other piece of text we might encounter here and thus will \ - be automatically broken up", - ); - - let expected = "Usage: fruits - -Options: - -a, --apple apple description - -b, --banana« banana description - -c, --brûlée brûlée quite long description - -k, --kiwi€ kiwi description - -o, --orange‹ orange description - -r, --raspberry-but-making-this-option-way-too-long\u{0020} - raspberry description is also quite long indeed longer - than every other piece of text we might encounter here - and thus will be automatically broken up -"; - - let usage = opts.usage("Usage: fruits"); - - debug!("expected: <<{}>>", expected); - debug!("generated: <<{}>>", usage); - assert!(usage == expected) -} - -#[test] -fn test_usage_short_only() { - let mut opts = Options::new(); - opts.optopt("k", "", "Kiwi", "VAL"); - opts.optflag("s", "", "Starfruit"); - opts.optflagopt("a", "", "Apple", "TYPE"); - - let expected = "Usage: fruits - -Options: - -k VAL Kiwi - -s Starfruit - -a [TYPE] Apple -"; - - let usage = opts.usage("Usage: fruits"); - debug!("expected: <<{}>>", expected); - debug!("generated: <<{}>>", usage); - assert!(usage == expected) -} - -#[test] -fn test_usage_long_only() { - let mut opts = Options::new(); - opts.optopt("", "kiwi", "Kiwi", "VAL"); - opts.optflag("", "starfruit", "Starfruit"); - opts.optflagopt("", "apple", "Apple", "TYPE"); - - let expected = "Usage: fruits - -Options: - --kiwi VAL Kiwi - --starfruit Starfruit - --apple [TYPE] Apple -"; - - let usage = opts.usage("Usage: fruits"); - debug!("expected: <<{}>>", expected); - debug!("generated: <<{}>>", usage); - assert!(usage == expected) -} - -#[test] -fn test_short_usage() { - let mut opts = Options::new(); - opts.reqopt("b", "banana", "Desc", "VAL"); - opts.optopt("a", "012345678901234567890123456789", "Desc", "VAL"); - opts.optflag("k", "kiwi", "Desc"); - opts.optflagopt("p", "", "Desc", "VAL"); - opts.optmulti("l", "", "Desc", "VAL"); - - let expected = "Usage: fruits -b VAL [-a VAL] [-k] [-p [VAL]] [-l VAL]..".to_string(); - let generated_usage = opts.short_usage("fruits"); - - debug!("expected: <<{}>>", expected); - debug!("generated: <<{}>>", generated_usage); - assert_eq!(generated_usage, expected); -} -#[test] -fn test_nonexistant_opt() { - let mut opts = Options::new(); - opts.optflag("b", "bar", "Desc"); - let args: Vec = Vec::new(); - let matches = opts.parse(&args).unwrap(); - assert_eq!(matches.opt_defined("foo"), false); - assert_eq!(matches.opt_defined("bar"), true); -} -#[test] -fn test_args_with_equals() { - let mut opts = Options::new(); - opts.optopt("o", "one", "One", "INFO"); - opts.optopt("t", "two", "Two", "INFO"); - - let args = vec![ - "--one".to_string(), - "A=B".to_string(), - "--two=C=D".to_string(), - ]; - let matches = &match opts.parse(&args) { - Ok(m) => m, - Err(e) => panic!("{}", e), - }; - assert_eq!(matches.opts_str(&["o".to_string()]).unwrap(), "A=B"); - assert_eq!(matches.opts_str(&["t".to_string()]).unwrap(), "C=D"); -} - -#[test] -fn test_long_only_usage() { - let mut opts = Options::new(); - opts.long_only(true); - opts.optflag("k", "kiwi", "Description"); - opts.optflag("a", "apple", "Description"); - - let expected = "Usage: fruits - -Options: - -k, -kiwi Description - -a, -apple Description -"; - - let usage = opts.usage("Usage: fruits"); - - debug!("expected: <<{}>>", expected); - debug!("generated: <<{}>>", usage); - assert!(usage == expected) -} - -#[test] -fn test_long_only_mode() { - let mut opts = Options::new(); - opts.long_only(true); - opts.optopt("a", "apple", "Description", "X"); - opts.optopt("b", "banana", "Description", "X"); - opts.optopt("c", "currant", "Description", "X"); - opts.optopt("", "durian", "Description", "X"); - opts.optopt("e", "", "Description", "X"); - opts.optopt("", "fruit", "Description", "X"); - - let args = vec![ - "-a", - "A", - "-b=B", - "--c=C", - "-durian", - "D", - "--e", - "E", - "-fruit=any", - ]; - let matches = &match opts.parse(&args) { - Ok(m) => m, - Err(e) => panic!("{}", e), - }; - assert_eq!(matches.opts_str(&["a".to_string()]).unwrap(), "A"); - assert_eq!(matches.opts_str(&["b".to_string()]).unwrap(), "B"); - assert_eq!(matches.opts_str(&["c".to_string()]).unwrap(), "C"); - assert_eq!(matches.opts_str(&["durian".to_string()]).unwrap(), "D"); - assert_eq!(matches.opts_str(&["e".to_string()]).unwrap(), "E"); - assert_eq!(matches.opts_str(&["fruit".to_string()]).unwrap(), "any"); -} - -#[test] -fn test_long_only_mode_no_short_parse() { - let mut opts = Options::new(); - opts.long_only(true); - opts.optflag("h", "help", "Description"); - opts.optflag("i", "ignore", "Description"); - opts.optflag("", "hi", "Description"); - - let args = vec!["-hi"]; - let matches = &match opts.parse(&args) { - Ok(m) => m, - Err(e) => panic!("{}", e), - }; - assert!(matches.opt_present("hi")); - assert!(!matches.opt_present("h")); - assert!(!matches.opt_present("i")); -} - -#[test] -fn test_normal_mode_no_long_parse() { - // Like test_long_only_mode_no_short_parse, but we make sure - // that long_only can be disabled, and the right thing - // happens. - let mut opts = Options::new(); - opts.long_only(true); - opts.optflag("h", "help", "Description"); - opts.optflag("i", "ignore", "Description"); - opts.optflag("", "hi", "Description"); - opts.long_only(false); - - let args = vec!["-hi"]; - let matches = &match opts.parse(&args) { - Ok(m) => m, - Err(e) => panic!("{}", e), - }; - assert!(!matches.opt_present("hi")); - assert!(matches.opt_present("h")); - assert!(matches.opt_present("i")); -} - -#[test] -#[should_panic] -fn test_long_name_too_short() { - let mut opts = Options::new(); - opts.optflag("", "a", "Oops, long option too short"); -} - -#[test] -#[should_panic] -fn test_undefined_opt_present() { - let mut opts = Options::new(); - opts.optflag("h", "help", "Description"); - let args = vec!["-h"]; - match opts.parse(args) { - Ok(matches) => assert!(!matches.opt_present("undefined")), - Err(e) => panic!("{}", e), - } -} - -#[test] -fn test_opt_default() { - let mut opts = Options::new(); - opts.optflag("h", "help", "Description"); - opts.optflag("i", "ignore", "Description"); - opts.optflag("r", "run", "Description"); - opts.long_only(false); - - let args: Vec = ["-i", "-r", "10"].iter().map(|x| x.to_string()).collect(); - let matches = &match opts.parse(&args) { - Ok(m) => m, - Err(e) => panic!("{}", e), - }; - assert_eq!(matches.opt_default("help", ""), None); - assert_eq!(matches.opt_default("i", "def"), Some("def".to_string())); -} - -#[test] -fn test_opt_get() { - let mut opts = Options::new(); - opts.optflag("h", "help", "Description"); - opts.optflagopt("i", "ignore", "Description", "true | false"); - opts.optflagopt("r", "run", "Description", "0 .. 10"); - opts.optflagopt("p", "percent", "Description", "0.0 .. 10.0"); - opts.long_only(false); - - let args: Vec = ["-i", "true", "-p", "1.1"] - .iter() - .map(|x| x.to_string()) - .collect(); - let matches = &match opts.parse(&args) { - Ok(m) => m, - Err(e) => panic!("{}", e), - }; - let h_arg = matches.opt_get::("help"); - assert_eq!(h_arg, Ok(None)); - let i_arg = matches.opt_get("i"); - assert_eq!(i_arg, Ok(Some(true))); - let p_arg = matches.opt_get("p"); - assert_eq!(p_arg, Ok(Some(1.1))); -} - -#[test] -fn test_opt_get_default() { - let mut opts = Options::new(); - opts.optflag("h", "help", "Description"); - opts.optflagopt("i", "ignore", "Description", "true | false"); - opts.optflagopt("r", "run", "Description", "0 .. 10"); - opts.optflagopt("p", "percent", "Description", "0.0 .. 10.0"); - opts.long_only(false); - - let args: Vec = ["-i", "true", "-p", "1.1"] - .iter() - .map(|x| x.to_string()) - .collect(); - let matches = &match opts.parse(&args) { - Ok(m) => m, - Err(e) => panic!("{}", e), - }; - let h_arg = matches.opt_get_default("help", 10); - assert_eq!(h_arg, Ok(10)); - let i_arg = matches.opt_get_default("i", false); - assert_eq!(i_arg, Ok(true)); - let p_arg = matches.opt_get_default("p", 10.2); - assert_eq!(p_arg, Ok(1.1)); -} - -#[test] -fn test_opt_positions() { - let mut opts = Options::new(); - opts.optflagmulti("a", "act", "Description"); - opts.optflagmulti("e", "enact", "Description"); - opts.optflagmulti("r", "react", "Description"); - - let args: Vec = ["-a", "-a", "-r", "-a", "-r", "-r"] - .iter() - .map(|x| x.to_string()) - .collect(); - - let matches = &match opts.parse(&args) { - Ok(m) => m, - Err(e) => panic!("{}", e), - }; - - let a_pos = matches.opt_positions("a"); - assert_eq!(a_pos, vec![0, 1, 3]); - let e_pos = matches.opt_positions("e"); - assert_eq!(e_pos, vec![]); - let r_pos = matches.opt_positions("r"); - assert_eq!(r_pos, vec![2, 4, 5]); -} - -#[test] -fn test_opt_strs_pos() { - let mut opts = Options::new(); - opts.optmulti("a", "act", "Description", "NUM"); - opts.optmulti("e", "enact", "Description", "NUM"); - opts.optmulti("r", "react", "Description", "NUM"); - - let args: Vec = ["-a1", "-a2", "-r3", "-a4", "-r5", "-r6"] - .iter() - .map(|x| x.to_string()) - .collect(); - - let matches = &match opts.parse(&args) { - Ok(m) => m, - Err(e) => panic!("{}", e), - }; - - let a_pos = matches.opt_strs_pos("a"); - assert_eq!( - a_pos, - vec![ - (0, "1".to_string()), - (1, "2".to_string()), - (3, "4".to_string()) - ] - ); - let e_pos = matches.opt_strs_pos("e"); - assert_eq!(e_pos, vec![]); - let r_pos = matches.opt_strs_pos("r"); - assert_eq!( - r_pos, - vec![ - (2, "3".to_string()), - (4, "5".to_string()), - (5, "6".to_string()) - ] - ); -} diff --git a/crux-mir/lib/getopts/tests/smoke.rs b/crux-mir/lib/getopts/tests/smoke.rs deleted file mode 100644 index a46f9c016..000000000 --- a/crux-mir/lib/getopts/tests/smoke.rs +++ /dev/null @@ -1,8 +0,0 @@ -extern crate getopts; - -use std::env; - -#[test] -fn main() { - getopts::Options::new().parse(env::args()).unwrap(); -} diff --git a/crux-mir/lib/gimli/.cargo-ok b/crux-mir/lib/gimli/.cargo-ok new file mode 100644 index 000000000..b5754e203 --- /dev/null +++ b/crux-mir/lib/gimli/.cargo-ok @@ -0,0 +1 @@ +ok \ No newline at end of file diff --git a/crux-mir/lib/gimli/.cargo_vcs_info.json b/crux-mir/lib/gimli/.cargo_vcs_info.json new file mode 100644 index 000000000..e31844222 --- /dev/null +++ b/crux-mir/lib/gimli/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "24a25b9fede9fb4364222100415987105ec053b3" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/crux-mir/lib/gimli/.gitignore b/crux-mir/lib/gimli/.gitignore new file mode 100644 index 000000000..326a1caa5 --- /dev/null +++ b/crux-mir/lib/gimli/.gitignore @@ -0,0 +1,4 @@ +target +Cargo.lock +*.bk +*.swp diff --git a/crux-mir/lib/gimli/CHANGELOG.md b/crux-mir/lib/gimli/CHANGELOG.md new file mode 100644 index 000000000..9ca6d70a8 --- /dev/null +++ b/crux-mir/lib/gimli/CHANGELOG.md @@ -0,0 +1,873 @@ +# `gimli` Change Log + +-------------------------------------------------------------------------------- + +## 0.26.2 + +Released 2022/07/16. + +### Changed + +* Fixed CFI personality encoding when writing. + [#609](https://github.com/gimli-rs/gimli/pull/609) + +* Fixed use of raw pointer for mutation, detected by Miri. + [#614](https://github.com/gimli-rs/gimli/pull/614) + +* Fixed `DW_OP_GNU_implicit_pointer` handling for DWARF version 2. + [#618](https://github.com/gimli-rs/gimli/pull/618) + +### Added + +* Added `read::EhHdrTable::iter`. + [#619](https://github.com/gimli-rs/gimli/pull/619) + +-------------------------------------------------------------------------------- + +## 0.26.1 + +Released 2021/11/02. + +### Changed + +* Fixed segmentation fault in `ArrayVec>::into_vec`, which may be used by + `read::Evaluation::result`. This regression was introduced in 0.26.0. + [#601](https://github.com/gimli-rs/gimli/pull/601) + +-------------------------------------------------------------------------------- + +## 0.26.0 + +Released 2021/10/24. + +### Breaking changes + +* Removed `read::UninitializedUnwindContext`. Use `Box` instead. + [#593](https://github.com/gimli-rs/gimli/pull/593) + +* Renamed `read::Error::CfiStackFull` to `StackFull`. + [#595](https://github.com/gimli-rs/gimli/pull/595) + +* Added `UnwindContextStorage` type parameter to `read::UnwindContext`, `read::UnwindTable`, + `read::UnwindTableRow`, and `read::RegisterRuleMap`. + [#595](https://github.com/gimli-rs/gimli/pull/595) + +* Added `EvaluationStorage` type parameter to `read::Evaluation`. + [#595](https://github.com/gimli-rs/gimli/pull/595) + +* Added `read::SectionId::DebugCuIndex` and `read::SectionId::DebugTuIndex`. + [#588](https://github.com/gimli-rs/gimli/pull/588) + +### Changed + +* Fixed `DW_EH_PE_pcrel` handling in default `write::Writer::write_eh_pointer` implementation. + [#576](https://github.com/gimli-rs/gimli/pull/576) + +* Fixed `read::AttributeSpecification::size` for some forms. + [#597](https://github.com/gimli-rs/gimli/pull/597) + +* Display more unit details in dwarfdump. + [#584](https://github.com/gimli-rs/gimli/pull/584) + +### Added + +* Added `write::DebuggingInformationEntry::delete_child`. + [#570](https://github.com/gimli-rs/gimli/pull/570) + +* Added ARM and AArch64 register definitions. + [#574](https://github.com/gimli-rs/gimli/pull/574) + [#577](https://github.com/gimli-rs/gimli/pull/577) + +* Added RISC-V register definitions. + [#579](https://github.com/gimli-rs/gimli/pull/579) + +* Added `read::DwarfPackage`, `read::DebugCuIndex`, and `read::DebugTuIndex`. + [#588](https://github.com/gimli-rs/gimli/pull/588) + +* Added `read-core` feature to allow building without `liballoc`. + [#596](https://github.com/gimli-rs/gimli/pull/596) + +* Added `read::EntriesRaw::skip_attributes`. + [#597](https://github.com/gimli-rs/gimli/pull/597) + +-------------------------------------------------------------------------------- + +## 0.25.0 + +Released 2021/07/26. + +### Breaking changes + +* `read::FrameDescriptionEntry::unwind_info_for_address` now returns a reference + instead of cloning. + [#557](https://github.com/gimli-rs/gimli/pull/557) + +* `read::AttributeValue::RangeListsRef` now contains a `RawRangeListsOffset` + to allow handling of GNU split DWARF extensions. + Use `read::Dwarf::ranges_offset_from_raw` to handle it. + [#568](https://github.com/gimli-rs/gimli/pull/568) + [#569](https://github.com/gimli-rs/gimli/pull/569) + +* Added `read::Unit::dwo_id`. + [#569](https://github.com/gimli-rs/gimli/pull/569) + +### Changed + +* `.debug_aranges` parsing now accepts version 3. + [#560](https://github.com/gimli-rs/gimli/pull/560) + +* `read::Dwarf::attr_ranges_offset` and its callers now handle GNU split DWARF extensions. + [#568](https://github.com/gimli-rs/gimli/pull/568) + [#569](https://github.com/gimli-rs/gimli/pull/569) + +### Added + +* Added `read::DebugLineStr::new`. + [#556](https://github.com/gimli-rs/gimli/pull/556) + +* Added `read::UnwindTable::into_current_row`. + [#557](https://github.com/gimli-rs/gimli/pull/557) + +* Added more `DW_LANG` constants. + [#565](https://github.com/gimli-rs/gimli/pull/565) + +* dwarfdump: added DWO parent support. + [#568](https://github.com/gimli-rs/gimli/pull/568) + +* Added `read::Dwarf` methods: `ranges_offset_from_raw`, `raw_ranges`, and `raw_locations`. + [#568](https://github.com/gimli-rs/gimli/pull/568) + [#569](https://github.com/gimli-rs/gimli/pull/569) + +-------------------------------------------------------------------------------- + +## 0.24.0 + +Released 2021/05/01. + +### Breaking changes + +* Minimum Rust version increased to 1.42.0. + +* Added `read::Dwarf::debug_aranges`. + [#539](https://github.com/gimli-rs/gimli/pull/539) + +* Replaced `read::DebugAranges::items` with `read::DebugAranges::headers`. + [#539](https://github.com/gimli-rs/gimli/pull/539) + +* Added `read::Operation::Wasm*`. + [#546](https://github.com/gimli-rs/gimli/pull/546) + +* `read::LineRow::line` now returns `Option`. + The `read::ColumnType::Column` variant now contains a `NonZeroU64`. + [#551](https://github.com/gimli-rs/gimli/pull/551) + +* Replaced `read::Dwarf::debug_str_sup` with `read::Dwarf::sup`. + Deleted `sup` parameter of `read::Dwarf::load`. + Added `read::Dwarf::load_sup`. + [#554](https://github.com/gimli-rs/gimli/pull/554) + +### Added + +* dwarfdump: Supplementary object file support. + [#552](https://github.com/gimli-rs/gimli/pull/552) + +### Changed + +* Support `DW_FORM_addrx*` for `DW_AT_low_pc`/`DW_AT_high_pc` in `read::Dwarf`. + [#541](https://github.com/gimli-rs/gimli/pull/541) + +* Performance improvement in `EndianReader`. + [#549](https://github.com/gimli-rs/gimli/pull/549) + +-------------------------------------------------------------------------------- + +## 0.23.0 + +Released 2020/10/27. + +### Breaking changes + +* Added more variants to `read::UnitType`. + Added `read::AttributeValue::DwoId` + [#521](https://github.com/gimli-rs/gimli/pull/521) + +* Replaced `CompilationUnitHeader` and `TypeUnitHeader` with `UnitHeader`. + Replaced `CompilationUnitHeadersIter` with `DebugInfoUnitHeadersIter`. + Replaced `TypeUnitHeadersIter` with `DebugTypesUnitHeadersIter`. + [#523](https://github.com/gimli-rs/gimli/pull/523) + + +### Added + +* Added read support for split DWARF. + [#527](https://github.com/gimli-rs/gimli/pull/527) + [#529](https://github.com/gimli-rs/gimli/pull/529) + +* Added `read::Dwarf::attr_address`. + [#524](https://github.com/gimli-rs/gimli/pull/524) + +* Added read support for `DW_AT_GNU_addr_base` and `DW_AT_GNU_ranges_base`. + [#525](https://github.com/gimli-rs/gimli/pull/525) + +* dwarfdump: Display index values for attributes. + [#526](https://github.com/gimli-rs/gimli/pull/526) + +* Added `name_to_register`. + [#532](https://github.com/gimli-rs/gimli/pull/532) + +-------------------------------------------------------------------------------- + +## 0.22.0 + +Released 2020/07/03. + +### Breaking changes + +* Fixed `UnitHeader::size_of_header` for DWARF 5 units. + [#518](https://github.com/gimli-rs/gimli/pull/518) + +### Added + +* Added fuzz targets in CI. + [#512](https://github.com/gimli-rs/gimli/pull/512) + +* Added read support for `DW_OP_GNU_addr_index` and `DW_OP_GNU_const_index`. + [#516](https://github.com/gimli-rs/gimli/pull/516) + +* Added `.dwo` support to dwarfdump. + [#516](https://github.com/gimli-rs/gimli/pull/516) + +* Added `SectionId::dwo_name` and `Section::dwo_section_name`. + [#517](https://github.com/gimli-rs/gimli/pull/517) + +### Fixed + +* Fixed panic when reading `DW_FORM_indirect` combined with `DW_FORM_implicit_const`. + [#502](https://github.com/gimli-rs/gimli/pull/502) + +* Fixed panic for `read::Abbreviations::get(0)`. + [#505](https://github.com/gimli-rs/gimli/pull/505) + +* Fixed arithmetic overflow when reading `.debug_line`. + [#508](https://github.com/gimli-rs/gimli/pull/508) + +* Fixed arithmetic overflow when reading CFI. + [#509](https://github.com/gimli-rs/gimli/pull/509) + +* Fixed arithmetic overflow and division by zero when reading `.debug_aranges`. + [#510](https://github.com/gimli-rs/gimli/pull/510) + +* Don't return error from `read::Unit::new` when `DW_AT_name` or `DW_AT_comp_dir` is missing. + [#515](https://github.com/gimli-rs/gimli/pull/515) + +-------------------------------------------------------------------------------- + +## 0.21.0 + +Released 2020/05/12. + +### Breaking changes + +* Minimum Rust version increased to 1.38.0. + +* Replaced `read::Operation::Literal` with `Operation::UnsignedConstant` and `Operation::SignedConstant`. + Changed `read::Operation::Bra` and `read::Operation::Skip` to contain the target offset instead of the bytecode. + [#479](https://github.com/gimli-rs/gimli/pull/479) + +* Changed `write::Expression` to support references. Existing users can convert to use `Expression::raw`. + [#479](https://github.com/gimli-rs/gimli/pull/479) + +* Replaced `write::AttributeValue::AnyUnitEntryRef` with `DebugInfoRef`. + Renamed `write::AttributeValue::ThisUnitEntryRef` to `UnitRef`. + [#479](https://github.com/gimli-rs/gimli/pull/479) + +* Added more optional features: `endian-reader` and `fallible-iterator`. + [#495](https://github.com/gimli-rs/gimli/pull/495) + [#498](https://github.com/gimli-rs/gimli/pull/498) + +### Added + +* Added `read::Expression::operations` + [#479](https://github.com/gimli-rs/gimli/pull/479) + +### Fixed + +* Fixed newlines in `dwarfdump` example. + [#470](https://github.com/gimli-rs/gimli/pull/470) + +* Ignore zero terminators when reading `.debug_frame` sections. + [#486](https://github.com/gimli-rs/gimli/pull/486) + +* Increase the number of CFI register rules supported by `read::UnwindContext`. + [#487](https://github.com/gimli-rs/gimli/pull/487) + +* Fixed version handling and return register encoding when reading `.eh_frame` sections. + [#493](https://github.com/gimli-rs/gimli/pull/493) + +### Changed + +* Added `EhFrame` and `DebugFrame` to `write::Sections`. + [#492](https://github.com/gimli-rs/gimli/pull/492) + +* Improved performance of `write::LineProgram::generate_row`. + [#476](https://github.com/gimli-rs/gimli/pull/476) + +* Removed use of the `byteorder`, `arrayvec` and `smallvec` crates. + [#494](https://github.com/gimli-rs/gimli/pull/494) + [#496](https://github.com/gimli-rs/gimli/pull/496) + [#497](https://github.com/gimli-rs/gimli/pull/497) + +-------------------------------------------------------------------------------- + +## 0.20.0 + +Released 2020/01/11. + +### Breaking changes + +* Changed type of `DwTag`, `DwAt`, and `DwForm` constants. + [#451](https://github.com/gimli-rs/gimli/pull/451) + +* Added `read/write::AttributeValue::DebugMacroRef`, and returned where + required in `read::Attribute::value`. Added `SectionId::DebugMacro`. + [#454](https://github.com/gimli-rs/gimli/pull/454) + +* Deleted `alloc` feature, and fixed `no-std` builds with stable rust. + [#459](https://github.com/gimli-rs/gimli/pull/459) + +* Deleted `read::Error::description`, and changed `` + to display what was previously the description. + [#462](https://github.com/gimli-rs/gimli/pull/462) + +### Added + +* Added GNU view constants. + [#434](https://github.com/gimli-rs/gimli/pull/434) + +* Added `read::EntriesRaw` for low level DIE parsing. + [#455](https://github.com/gimli-rs/gimli/pull/455) + +* Added `examples/simple-line.rs`. + [#460](https://github.com/gimli-rs/gimli/pull/460) + +### Fixed + +* Fixed handling of CFI augmentations without data. + [#438](https://github.com/gimli-rs/gimli/pull/438) + +* dwarfdump: fix panic for malformed expressions. + [#447](https://github.com/gimli-rs/gimli/pull/447) + +* dwarfdump: fix handling of Mach-O relocations. + [#449](https://github.com/gimli-rs/gimli/pull/449) + +### Changed + +* Improved abbreviation parsing performance. + [#451](https://github.com/gimli-rs/gimli/pull/451) + +-------------------------------------------------------------------------------- + +## 0.19.0 + +Released 2019/07/08. + +### Breaking changes + +* Small API changes related to `.debug_loc` and `.debug_loclists`: + added `read::RawLocListEntry::AddressOrOffsetPair` enum variant, + added `write::Sections::debug_loc/debug_loclists` public members, + and replaced `write::AttributeValue::LocationListsRef` with `LocationListRef`. + [#425](https://github.com/gimli-rs/gimli/pull/425) + +### Added + +* Added `read::Attribute::exprloc_value` and `read::AttributeValue::exprloc_value`. + [#422](https://github.com/gimli-rs/gimli/pull/422) + +* Added support for writing `.debug_loc` and `.debug_loclists` sections. + [#425](https://github.com/gimli-rs/gimli/pull/425) + +* Added `-G` flag to `dwarfdump` example to display global offsets. + [#427](https://github.com/gimli-rs/gimli/pull/427) + +* Added `examples/simple.rs`. + [#429](https://github.com/gimli-rs/gimli/pull/429) + +### Fixed + +* `write::LineProgram::from` no longer requires `DW_AT_name` or `DW_AT_comp_dir` + attributes to be present in the unit DIE. + [#430](https://github.com/gimli-rs/gimli/pull/430) + +-------------------------------------------------------------------------------- + +## 0.18.0 + +Released 2019/04/25. + +The focus of this release has been on improving support for reading CFI, +and adding support for writing CFI. + +### Breaking changes + +* For types which have an `Offset` type parameter, the default `Offset` + has changed from `usize` to `R::Offset`. + [#392](https://github.com/gimli-rs/gimli/pull/392) + +* Added an `Offset` type parameter to the `read::Unit` type to allow variance. + [#393](https://github.com/gimli-rs/gimli/pull/393) + +* Changed the `UninitializedUnwindContext::initialize` method to borrow `self`, + and return `&mut UnwindContext`. Deleted the `InitializedUnwindContext` type. + [#395](https://github.com/gimli-rs/gimli/pull/395) + +* Deleted the `UnwindSection` type parameters from the `CommonInformationEntry`, + `FrameDescriptionEntry`, `UninitializedUnwindContext`, + `UnwindContext`, and `UnwindTable` types. + [#399](https://github.com/gimli-rs/gimli/pull/399) + +* Changed the signature of the `get_cie` callback parameter for various functions. + The signature now matches the `UnwindSection::cie_from_offset` method, so + that method can be used as the parameter. + [#400](https://github.com/gimli-rs/gimli/pull/400) + +* Reduced the number of lifetime parameters for the `UnwindTable` type. + [#400](https://github.com/gimli-rs/gimli/pull/400) + +* Updated `fallible-iterator` to version 0.2.0. + [#407](https://github.com/gimli-rs/gimli/pull/407) + +* Added a parameter to the `Error::UnexpectedEof` enum variant. + [#408](https://github.com/gimli-rs/gimli/pull/408) + +### Added + +* Update to 2018 edition. + [#391](https://github.com/gimli-rs/gimli/pull/391) + +* Added the `FrameDescriptionEntry::unwind_info_for_address` method. + [#396](https://github.com/gimli-rs/gimli/pull/396) + +* Added the `FrameDescriptionEntry::rows` method. + [#396](https://github.com/gimli-rs/gimli/pull/396) + +* Added the `EhHdrTable::unwind_info_for_address` method. + [#400](https://github.com/gimli-rs/gimli/pull/400) + +* Added the `EhHdrTable::fde_for_address` method and deprecated the + `EhHdrTable::lookup_and_parse` method. + [#400](https://github.com/gimli-rs/gimli/pull/400) + +* Added the `EhHdrTable::pointer_to_offset` method. + [#400](https://github.com/gimli-rs/gimli/pull/400) + +* Added the `UnwindSection::fde_for_address` method. + [#396](https://github.com/gimli-rs/gimli/pull/396) + +* Added the `UnwindSection::fde_from_offset` method. + [#400](https://github.com/gimli-rs/gimli/pull/400) + +* Added the `UnwindSection::partial_fde_from_offset` method. + [#400](https://github.com/gimli-rs/gimli/pull/400) + +* Added the `Section::id` method. + [#406](https://github.com/gimli-rs/gimli/pull/406) + +* Added the `Dwarf::load` method, and corresponding methods for individual sections. + [#406](https://github.com/gimli-rs/gimli/pull/406) + +* Added the `Dwarf::borrow` method, and corresponding methods for individual sections. + [#406](https://github.com/gimli-rs/gimli/pull/406) + +* Added the `Dwarf::format_error` method. + [#408](https://github.com/gimli-rs/gimli/pull/408) + +* Added the `Dwarf::die_ranges` method. + [#417](https://github.com/gimli-rs/gimli/pull/417) + +* Added the `Dwarf::unit_ranges` method. + [#417](https://github.com/gimli-rs/gimli/pull/417) + +* Added support for writing `.debug_frame` and `.eh_frame` sections. + [#412](https://github.com/gimli-rs/gimli/pull/412) + [#419](https://github.com/gimli-rs/gimli/pull/419) + +### Fixed + +* The `code_alignment_factor` is now used when evaluting CFI instructions + that advance the location. + [#401](https://github.com/gimli-rs/gimli/pull/401) + +* Fixed parsing of pointers encoded with `DW_EH_PE_funcrel`. + [#402](https://github.com/gimli-rs/gimli/pull/402) + +* Use the FDE address encoding from the augmentation when parsing `DW_CFA_set_loc`. + [#403](https://github.com/gimli-rs/gimli/pull/403) + +* Fixed setting of `.eh_frame` base addresses in dwarfdump. + [#410](https://github.com/gimli-rs/gimli/pull/410) + +## 0.17.0 + +Released 2019/02/21. + +The focus of this release has been on improving DWARF 5 support, and +adding support for writing DWARF. + +### Breaking changes + +* Changed register values to a `Register` type instead of `u8`/`u64`. + [#328](https://github.com/gimli-rs/gimli/pull/328) + +* Replaced `BaseAddresses::set_cfi` with `set_eh_frame_hdr` and `set_eh_frame`. + Replaced `BaseAddresses::set_data` with `set_got`. + You should now use the same `BaseAddresses` value for parsing both + `.eh_frame` and `.eh_frame_hdr`. + [#351](https://github.com/gimli-rs/gimli/pull/351) + +* Renamed many types and functions related to `.debug_line`. + Renamed `LineNumberProgram` to `LineProgram`. + Renamed `IncompleteLineNumberProgram` to `IncompleteLineProgram`. + Renamed `CompleteLineNumberProgram` to `CompleteLineProgram`. + Renamed `LineNumberProgramHeader` to `LineProgramHeader`. + Renamed `LineNumberRow` to `LineRow`. + Renamed `StateMachine` to `LineRows`. + Renamed `Opcode` to `LineInstruction`. + Renamed `OpcodesIter` to `LineInstructions`. + Renamed `LineNumberSequence` to `LineSequence`. + [#359](https://github.com/gimli-rs/gimli/pull/359) + +* Added `Offset` type parameter to `AttributeValue`, `LineProgram`, + `IncompleteLineProgram`, `CompleteLineProgram`, `LineRows`, `LineInstruction`, + and `FileEntry`. + [#324](https://github.com/gimli-rs/gimli/pull/324) + +* Changed `FileEntry::path_name`, `FileEntry::directory`, and + `LineProgramHeader::directory` to return an `AttributeValue` instead + of a `Reader`. + [#366](https://github.com/gimli-rs/gimli/pull/366) + +* Renamed `FileEntry::last_modification` to `FileEntry::timestamp` + and renamed `FileEntry::length` to `FileEntry::size`. + [#366](https://github.com/gimli-rs/gimli/pull/366) + +* Added an `Encoding` type. Changed many functions that previously accepted + `Format`, version or address size parameters to accept an `Encoding` + parameter instead. + Notable changes are `LocationLists::locations`, `RangeLists::ranges`, + and `Expression::evaluation`. + [#364](https://github.com/gimli-rs/gimli/pull/364) + +* Changed return type of `LocationLists::new` and `RangeLists::new`. + [#370](https://github.com/gimli-rs/gimli/pull/370) + +* Added parameters to `LocationsLists::locations` and `RangeLists::ranges` + to support `.debug_addr`. + [#358](https://github.com/gimli-rs/gimli/pull/358) + +* Added more `AttributeValue` variants: `DebugAddrBase`, `DebugAddrIndex`, + `DebugLocListsBase`, `DebugLocListsIndex`, `DebugRngListsBase`, `DebugRngListsIndex`, + `DebugStrOffsetsBase`, `DebugStrOffsetsIndex`, `DebugLineStrRef`. + [#358](https://github.com/gimli-rs/gimli/pull/358) + +* Changed `AttributeValue::Data*` attributes to native endian integers instead + of byte arrays. + [#365](https://github.com/gimli-rs/gimli/pull/365) + +* Replaced `EvaluationResult::TextBase` with + `EvaluationResult::RequiresRelocatedAddress`. The handling of `TextBase` + was incorrect. + [#335](https://github.com/gimli-rs/gimli/pull/335) + +* Added `EvaluationResult::IndexedAddress` for operations that require an + address from `.debug_addr`. + [#358](https://github.com/gimli-rs/gimli/pull/358) + +* Added `Reader::read_slice`. Added a default implementation of + `Reader::read_u8_array` which uses this. + [#358](https://github.com/gimli-rs/gimli/pull/358) + +### Added + +* Added initial support for writing DWARF. This is targeted at supporting + line number information only. + [#340](https://github.com/gimli-rs/gimli/pull/340) + [#344](https://github.com/gimli-rs/gimli/pull/344) + [#346](https://github.com/gimli-rs/gimli/pull/346) + [#361](https://github.com/gimli-rs/gimli/pull/361) + [#362](https://github.com/gimli-rs/gimli/pull/362) + [#365](https://github.com/gimli-rs/gimli/pull/365) + [#368](https://github.com/gimli-rs/gimli/pull/368) + [#382](https://github.com/gimli-rs/gimli/pull/382) + +* Added `read` and `write` Cargo features. Both are enabled by default. + [#343](https://github.com/gimli-rs/gimli/pull/343) + +* Added support for reading DWARF 5 `.debug_line` and `.debug_line_str` sections. + [#366](https://github.com/gimli-rs/gimli/pull/366) + +* Added support for reading DWARF 5 `.debug_str_offsets` sections, including + parsing `DW_FORM_strx*` attributes. + [#358](https://github.com/gimli-rs/gimli/pull/358) + +* Added support for reading DWARF 5 `.debug_addr` sections, including parsing + `DW_FORM_addrx*` attributes and evaluating `DW_OP_addrx` and `DW_OP_constx` + operations. + [#358](https://github.com/gimli-rs/gimli/pull/358) + +* Added support for reading DWARF 5 indexed addresses and offsets in + `.debug_loclists` and `.debug_rnglists`, including parsing `DW_FORM_rnglistx` + and `DW_FORM_loclistx` attributes. + [#358](https://github.com/gimli-rs/gimli/pull/358) + +* Added high level `Dwarf` and `Unit` types. Existing code does not need to + switch to using these types, but doing so will make DWARF 5 support simpler. + [#352](https://github.com/gimli-rs/gimli/pull/352) + [#380](https://github.com/gimli-rs/gimli/pull/380) + [#381](https://github.com/gimli-rs/gimli/pull/381) + +* Added `EhFrame::set_address_size` and `DebugFrame::set_address_size` methods + to allow parsing non-native CFI sections. The default address size is still + the native size. + [#325](https://github.com/gimli-rs/gimli/pull/325) + +* Added architecture specific definitions for `Register` values and names. + Changed dwarfdump to print them. + [#328](https://github.com/gimli-rs/gimli/pull/328) + +* Added support for reading relocatable DWARF sections. + [#337](https://github.com/gimli-rs/gimli/pull/337) + +* Added parsing of `DW_FORM_data16`. + [#366](https://github.com/gimli-rs/gimli/pull/366) + +### Fixed + +* Fixed parsing DWARF 5 ranges with `start == end == 0`. + [#323](https://github.com/gimli-rs/gimli/pull/323) + +* Changed `LineRows` to be covariant in its `Reader` type parameter. + [#324](https://github.com/gimli-rs/gimli/pull/324) + +* Fixed handling of empty units in dwarfdump. + [#330](https://github.com/gimli-rs/gimli/pull/330) + +* Fixed `UnitHeader::length_including_self` for `Dwarf64`. + [#342](https://github.com/gimli-rs/gimli/pull/342) + +* Fixed parsing of `DW_CFA_set_loc`. + [#355](https://github.com/gimli-rs/gimli/pull/355) + +* Fixed handling of multiple headers in `.debug_loclists` and `.debug_rnglists`. + [#370](https://github.com/gimli-rs/gimli/pull/370) + +-------------------------------------------------------------------------------- + +## 0.16.1 + +Released 2018/08/28. + +### Added + +* Added `EhFrameHdr::lookup_and_parse`. [#316][] +* Added support for `DW_CFA_GNU_args_size`. [#319][] + +### Fixed + +* Implement `Send`/`Sync` for `SubRange`. [#305][] +* Fixed `alloc` support on nightly. [#306][] [#310][] + +[#305]: https://github.com/gimli-rs/gimli/pull/305 +[#306]: https://github.com/gimli-rs/gimli/pull/306 +[#310]: https://github.com/gimli-rs/gimli/pull/310 +[#316]: https://github.com/gimli-rs/gimli/pull/316 +[#319]: https://github.com/gimli-rs/gimli/pull/319 + +-------------------------------------------------------------------------------- + +## 0.16.0 + +Released 2018/06/01. + +### Added + +* Added support for building in `#![no_std]` environments, when the `alloc` + crate is available. Disable the "std" feature and enable the "alloc" + feature. [#138][] [#271][] + +* Added support for DWARF 5 `.debug_rnglists` and `.debug_loclists` + sections. [#272][] + +* Added support for DWARF 5 `DW_FORM_ref_sup` and `DW_FORM_strp_sup` attribute + forms. [#288][] + +* Added support for DWARF 5 operations on typed values. [#293][] + +* A `dwarf-validate` example program that checks the integrity of the given + DWARF and its references between sections. [#290][] + +* Added the `EndianReader` type, an easy way to define a custom `Reader` + implementation with a reference to a generic buffer of bytes and an associated + endianity. [#298][] [#302][] + +### Changed + +* Various speed improvements for evaluating `.debug_line` line number + programs. [#276][] + +* The example `dwarfdump` clone is a [whole lot faster + now][dwarfdump-faster]. [#282][] [#284][] [#285][] + +### Deprecated + +* `EndianBuf` has been renamed to `EndianSlice`, use that name instead. [#295][] + +### Fixed + +* Evaluating the `DW_CFA_restore_state` opcode properly maintains the current + location. Previously it would incorrectly restore the old location when + popping from evaluation stack. [#274][] + +[#271]: https://github.com/gimli-rs/gimli/issues/271 +[#138]: https://github.com/gimli-rs/gimli/issues/138 +[#274]: https://github.com/gimli-rs/gimli/issues/274 +[#272]: https://github.com/gimli-rs/gimli/issues/272 +[#276]: https://github.com/gimli-rs/gimli/issues/276 +[#282]: https://github.com/gimli-rs/gimli/issues/282 +[#285]: https://github.com/gimli-rs/gimli/issues/285 +[#284]: https://github.com/gimli-rs/gimli/issues/284 +[#288]: https://github.com/gimli-rs/gimli/issues/288 +[#290]: https://github.com/gimli-rs/gimli/issues/290 +[#293]: https://github.com/gimli-rs/gimli/issues/293 +[#295]: https://github.com/gimli-rs/gimli/issues/295 +[#298]: https://github.com/gimli-rs/gimli/issues/298 +[#302]: https://github.com/gimli-rs/gimli/issues/302 +[dwarfdump-faster]: https://robert.ocallahan.org/2018/03/speeding-up-dwarfdump-with-rust.html + +-------------------------------------------------------------------------------- + +## 0.15.0 + +Released 2017/12/01. + +### Added + +* Added the `EndianBuf::to_string()` method. [#233][] + +* Added more robust error handling in our example `dwarfdump` clone. [#234][] + +* Added `FrameDescriptionEntry::initial_address` method. [#237][] + +* Added `FrameDescriptionEntry::len` method. [#237][] + +* Added the `FrameDescriptionEntry::entry_len` method. [#241][] + +* Added the `CommonInformationEntry::offset` method. [#241][] + +* Added the `CommonInformationEntry::entry_len` method. [#241][] + +* Added the `CommonInformationEntry::version` method. [#241][] + +* Added the `CommonInformationEntry::augmentation` method. [#241][] + +* Added the `CommonInformationEntry::code_alignment_factor` method. [#241][] + +* Added the `CommonInformationEntry::data_alignment_factor` method. [#241][] + +* Added the `CommonInformationEntry::return_address_register` method. [#241][] + +* Added support for printing `.eh_frame` sections to our example `dwarfdump` + clone. [#241][] + +* Added support for parsing the `.eh_frame_hdr` section. On Linux, the + `.eh_frame_hdr` section provides a pointer to the already-mapped-in-memory + `.eh_frame` data, so that it doesn't need to be duplicated, and a binary + search table of its entries for faster unwinding information lookups. [#250][] + +* Added support for parsing DWARF 5 compilation unit headers. [#257][] + +* Added support for DWARF 5's `DW_FORM_implicit_const`. [#257][] + +### Changed + +* Unwinding methods now give ownership of the unwinding context back to the + caller if errors are encountered, not just on the success path. This allows + recovering from errors in signal-safe code, where constructing a new unwinding + context is not an option because it requires allocation. This is a **breaking + change** affecting `UnwindSection::unwind_info_for_address` and + `UninitializedUnwindContext::initialize`. [#241][] + +* `CfaRule` and `RegisterRule` now expose their `DW_OP` expressions as + `Expression`. This is a minor **breaking change**. [#241][] + +* The `Error::UnknownVersion` variant now contains the unknown version + number. This is a minor **breaking change**. [#245][] + +* `EvaluationResult::RequiresEntryValue` requires an `Expression` instead of a + `Reader` now. This is a minor **breaking change**. [#256][] + + +[#233]: https://github.com/gimli-rs/gimli/pull/233 +[#234]: https://github.com/gimli-rs/gimli/pull/234 +[#237]: https://github.com/gimli-rs/gimli/pull/237 +[#241]: https://github.com/gimli-rs/gimli/pull/241 +[#245]: https://github.com/gimli-rs/gimli/pull/245 +[#250]: https://github.com/gimli-rs/gimli/pull/250 +[#256]: https://github.com/gimli-rs/gimli/pull/256 +[#257]: https://github.com/gimli-rs/gimli/pull/257 + +-------------------------------------------------------------------------------- + +## 0.14.0 + +Released 2017/08/08. + +### Added + +* All `pub` types now `derive(Hash)`. [#192][] + +* All the constants from DWARF 5 are now defined. [#193][] + +* Added support for the `DW_OP_GNU_parameter_ref` GNU extension to parsing and + evaluation DWARF opcodes. [#208][] + +* Improved LEB128 parsing performance. [#216][] + +* Improved `.debug_{aranges,pubnames,pubtypes}` parsing performance. [#218][] + +* Added the ability to choose endianity dynamically at run time, rather than + only statically at compile time. [#219][] + +### Changed + +* The biggest change of this release is that `gimli` no longer requires the + object file's section be fully loaded into memory. This enables using `gimli` + on 32 bit platforms where there often isn't enough contiguous virtual memory + address space to load debugging information into. The default behavior is + still geared for 64 bit platforms, where address space overfloweth, and you + can still load the whole sections of the object file (or the entire object + file) into memory. This is abstracted over with the `gimli::Reader` + trait. This manifests as small (but many) breaking changes to much of the + public API. [#182][] + +### Fixed + +* The `DW_END_*` constants for defining endianity of a compilation unit were + previously incorrect. [#193][] + +* The `DW_OP_addr` opcode is relative to the base address of the `.text` section + of the binary, but we were incorrectly treating it as an absolute value. [#210][] + +[GitHub]: https://github.com/gimli-rs/gimli +[crates.io]: https://crates.io/crates/gimli +[contributing]: https://github.com/gimli-rs/gimli/blob/master/CONTRIBUTING.md +[easy]: https://github.com/gimli-rs/gimli/issues?q=is%3Aopen+is%3Aissue+label%3Aeasy +[#192]: https://github.com/gimli-rs/gimli/pull/192 +[#193]: https://github.com/gimli-rs/gimli/pull/193 +[#182]: https://github.com/gimli-rs/gimli/issues/182 +[#208]: https://github.com/gimli-rs/gimli/pull/208 +[#210]: https://github.com/gimli-rs/gimli/pull/210 +[#216]: https://github.com/gimli-rs/gimli/pull/216 +[#218]: https://github.com/gimli-rs/gimli/pull/218 +[#219]: https://github.com/gimli-rs/gimli/pull/219 diff --git a/crux-mir/lib/gimli/CONTRIBUTING.md b/crux-mir/lib/gimli/CONTRIBUTING.md new file mode 100644 index 000000000..4f9e574ce --- /dev/null +++ b/crux-mir/lib/gimli/CONTRIBUTING.md @@ -0,0 +1,137 @@ +# Contributing to `gimli` + +Hi! We'd love to have your contributions! If you want help or mentorship, reach +out to us in a GitHub issue, or ping `fitzgen` in `#rust` on `irc.mozilla.org`. + +* [Code of Conduct](#coc) +* [Filing an Issue](#issues) +* [Building `gimli`](#building) +* [Testing `gimli`](#testing) + * [Test Coverage](#coverage) + * [Using `test-assembler`](#test-assembler) + * [Fuzzing](#fuzzing) +* [Benchmarking](#benchmarking) +* [Style](#style) + +## Code of Conduct + +We abide by the +[Rust Code of Conduct](https://www.rust-lang.org/en-US/conduct.html) and ask +that you do as well. + +## Filing an Issue + +Think you've found a bug? File an issue! To help us understand and reproduce the +issue, provide us with: + +* The (preferably minimal) test case +* Steps to reproduce the issue using the test case +* The expected result of following those steps +* The actual result of following those steps + +Definitely file an issue if you see an unexpected panic originating from within +`gimli`! `gimli` should never panic unless it is explicitly documented to panic +in the specific circumstances provided. + +## Building `gimli` + +`gimli` should always build on stable `rustc`, but we recommend using +[`rustup`](https://www.rustup.rs/) so you can switch to nightly `rustc` and run +benchmarks. + +To build `gimli`: + +``` +$ cargo build +``` + +## Testing `gimli` + +Run the tests with `cargo`: + +``` +$ cargo test +``` + +### Test Coverage + +If you have `kcov` installed under linux, then you can generate code coverage +results using the `coverage` script in the root of the repository, and view them +at `target/kcov/index.html`. Otherwise you can create a pull request and view +the coverage results on coveralls.io. + +``` +$ ./coverage +``` + +The ideal we aim to reach is having our unit tests exercise every branch in +`gimli`. We allow an exception for branches which propagate errors inside a +`try!(..)` invocation, but we *do* want to exercise the original error paths. + +Pull requests adding new code should ensure that this ideal is met. + +At the time of writing we have 94% test coverage according to our coveralls.io +continuous integration. That number should generally stay the same or go up ;) +This is a bit subjective, because -.001% is just noise and doesn't matter. + +### Using `test-assembler` + +We use the awesome +[`test-assembler`](https://github.com/luser/rust-test-assembler) crate to +construct binary test data. It makes building complex test cases readable. + +[Here is an example usage in `gimli`](https://github.com/gimli-rs/gimli/blob/156451f3fe6eeb2fa62b84b362c33fcb176e1171/src/loc.rs#L263) + +### Fuzzing + +First, install `cargo fuzz`: + +``` +$ cargo install cargo-fuzz +``` + +Optionally, [set up the corpora for our fuzz targets by following these +instructions](https://github.com/gimli-rs/gimli-libfuzzer-corpora/blob/master/README.md#using-these-corpora). + +Finally, run a fuzz target! In this case, we are running the `eh_frame` fuzz +target: + +``` +$ cargo fuzz run eh_frame +``` + +The fuzz target definitions live in `fuzz/fuzz_targets/*`. You can add new ones +via `cargo fuzz add `. + +## Benchmarking + +The benchmarks require nightly `rustc`, so use `rustup`: + +``` +$ rustup run nightly cargo bench +``` + +We aim to be the fastest DWARF library. Period. + +Please provide before and after benchmark results with your pull requests. You +may also find [`cargo benchcmp`](https://github.com/BurntSushi/cargo-benchcmp) +handy for comparing results. + +Pull requests adding `#[bench]` micro-benchmarks that exercise a new edge case +are very welcome! + +## Style + +We use `rustfmt` to automatically format and style all of our code. + +To install `rustfmt`: + +``` +$ rustup component add rustfmt-preview +``` + +To run `rustfmt` on `gimli`: + +``` +$ cargo fmt +``` diff --git a/crux-mir/lib/gimli/Cargo.toml b/crux-mir/lib/gimli/Cargo.toml new file mode 100644 index 000000000..f36ccd936 --- /dev/null +++ b/crux-mir/lib/gimli/Cargo.toml @@ -0,0 +1,146 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2018" +name = "gimli" +version = "0.26.2" +exclude = [ + "/releases/*", + "/.github", +] +description = "A library for reading and writing the DWARF debugging format." +documentation = "https://docs.rs/gimli" +readme = "./README.md" +keywords = [ + "DWARF", + "debug", + "ELF", + "eh_frame", +] +categories = [ + "development-tools::debugging", + "development-tools::profiling", + "parser-implementations", +] +license = "MIT OR Apache-2.0" +repository = "https://github.com/gimli-rs/gimli" + +[profile.bench] +codegen-units = 1 +debug = true +split-debuginfo = "packed" + +[profile.test] +split-debuginfo = "packed" + +[[example]] +name = "simple" +required-features = ["read"] + +[[example]] +name = "simple_line" +required-features = ["read"] + +[[example]] +name = "dwarfdump" +required-features = [ + "read", + "std", +] + +[[example]] +name = "dwarf-validate" +required-features = [ + "read", + "std", +] + +[dependencies.alloc] +version = "1.0.0" +optional = true +package = "rustc-std-workspace-alloc" + +[dependencies.compiler_builtins] +version = "0.1.2" +optional = true + +[dependencies.core] +version = "1.0.0" +optional = true +package = "rustc-std-workspace-core" + +[dependencies.fallible-iterator] +version = "0.2.0" +optional = true +default-features = false + +[dependencies.indexmap] +version = "1.0.2" +optional = true + +[dependencies.stable_deref_trait] +version = "1.1.0" +optional = true +default-features = false + +[dev-dependencies.crossbeam] +version = "0.8" + +[dev-dependencies.getopts] +version = "0.2" + +[dev-dependencies.memmap2] +version = "0.5.5" + +[dev-dependencies.num_cpus] +version = "1" + +[dev-dependencies.object] +version = "0.29.0" +features = ["wasm"] + +[dev-dependencies.rayon] +version = "1.0" + +[dev-dependencies.regex] +version = "1" + +[dev-dependencies.test-assembler] +version = "0.1.3" + +[dev-dependencies.typed-arena] +version = "2" + +[features] +default = [ + "read", + "write", + "std", + "fallible-iterator", + "endian-reader", +] +endian-reader = [ + "read", + "stable_deref_trait", +] +read = ["read-core"] +read-core = [] +rustc-dep-of-std = [ + "core", + "alloc", + "compiler_builtins", +] +std = [ + "fallible-iterator/std", + "stable_deref_trait/std", +] +write = ["indexmap"] diff --git a/crux-mir/lib/gimli/Cargo.toml.orig b/crux-mir/lib/gimli/Cargo.toml.orig new file mode 100644 index 000000000..bd03528e5 --- /dev/null +++ b/crux-mir/lib/gimli/Cargo.toml.orig @@ -0,0 +1,70 @@ +[package] +name = "gimli" +version = "0.26.2" +categories = ["development-tools::debugging", "development-tools::profiling", "parser-implementations"] +description = "A library for reading and writing the DWARF debugging format." +documentation = "https://docs.rs/gimli" +edition = "2018" +exclude = ["/releases/*", "/.github"] +keywords = ["DWARF", "debug", "ELF", "eh_frame"] +license = "MIT OR Apache-2.0" +readme = "./README.md" +repository = "https://github.com/gimli-rs/gimli" + +[dependencies] +fallible-iterator = { version = "0.2.0", default-features = false, optional = true } +indexmap = { version = "1.0.2", optional = true } +stable_deref_trait = { version = "1.1.0", default-features = false, optional = true } + +# Internal feature, only used when building as part of libstd, not part of the +# stable interface of this crate. +core = { version = '1.0.0', optional = true, package = 'rustc-std-workspace-core' } +alloc = { version = '1.0.0', optional = true, package = 'rustc-std-workspace-alloc' } +compiler_builtins = { version = '0.1.2', optional = true } + +[dev-dependencies] +crossbeam = "0.8" +getopts = "0.2" +memmap2 = "0.5.5" +num_cpus = "1" +object = { version = "0.29.0", features = ["wasm"] } +rayon = "1.0" +regex = "1" +test-assembler = "0.1.3" +typed-arena = "2" + +[features] +read-core = [] +read = ["read-core"] +endian-reader = ["read", "stable_deref_trait"] +write = ["indexmap"] +std = ["fallible-iterator/std", "stable_deref_trait/std"] +default = ["read", "write", "std", "fallible-iterator", "endian-reader"] + +# Internal feature, only used when building as part of libstd, not part of the +# stable interface of this crate. +rustc-dep-of-std = ['core', 'alloc', 'compiler_builtins'] + +[profile.test] +split-debuginfo = 'packed' + +[profile.bench] +debug = true +codegen-units = 1 +split-debuginfo = 'packed' + +[[example]] +name = "simple" +required-features = ["read"] + +[[example]] +name = "simple_line" +required-features = ["read"] + +[[example]] +name = "dwarfdump" +required-features = ["read", "std"] + +[[example]] +name = "dwarf-validate" +required-features = ["read", "std"] diff --git a/crux-mir/lib/getopts/LICENSE-APACHE b/crux-mir/lib/gimli/LICENSE-APACHE similarity index 100% rename from crux-mir/lib/getopts/LICENSE-APACHE rename to crux-mir/lib/gimli/LICENSE-APACHE diff --git a/crux-mir/lib/unicode-width/LICENSE-MIT b/crux-mir/lib/gimli/LICENSE-MIT similarity index 100% rename from crux-mir/lib/unicode-width/LICENSE-MIT rename to crux-mir/lib/gimli/LICENSE-MIT diff --git a/crux-mir/lib/gimli/README.md b/crux-mir/lib/gimli/README.md new file mode 100644 index 000000000..19e7bbd0e --- /dev/null +++ b/crux-mir/lib/gimli/README.md @@ -0,0 +1,78 @@ +# `gimli` + +[![](https://img.shields.io/crates/v/gimli.svg) ![](https://img.shields.io/crates/d/gimli.svg)](https://crates.io/crates/gimli) +[![](https://docs.rs/gimli/badge.svg)](https://docs.rs/gimli/) +[![Build Status](https://github.com/gimli-rs/gimli/workflows/Rust/badge.svg)](https://github.com/gimli-rs/gimli/actions) +[![Coverage Status](https://coveralls.io/repos/github/gimli-rs/gimli/badge.svg?branch=master)](https://coveralls.io/github/gimli-rs/gimli?branch=master) + +`gimli` is a blazing fast library for consuming the +[DWARF debugging format](https://dwarfstd.org/). + +* **Zero copy:** everything is just a reference to the original input buffer. No + copies of the input data get made. + +* **Lazy:** you can iterate compilation units without parsing their + contents. Parse only as many debugging information entry (DIE) trees as you + iterate over. `gimli` also uses `DW_AT_sibling` references to avoid parsing a + DIE's children to find its next sibling, when possible. + +* **Cross-platform:** `gimli` makes no assumptions about what kind of object + file you're working with. The flipside to that is that it's up to you to + provide an ELF loader on Linux or Mach-O loader on macOS. + + * Unsure which object file parser to use? Try the cross-platform + [`object`](https://github.com/gimli-rs/object) crate. See the + [`examples/`](./examples) directory for usage with `gimli`. + +## Install + +Add this to your `Cargo.toml`: + +```toml +[dependencies] +gimli = "0.26.2" +``` + +The minimum supported Rust version is 1.42.0. + +## Documentation + +* [Documentation on docs.rs](https://docs.rs/gimli/) + +* Example programs: + + * [A simple `.debug_info` parser](./examples/simple.rs) + + * [A simple `.debug_line` parser](./examples/simple_line.rs) + + * [A `dwarfdump` clone](./examples/dwarfdump.rs) + + * [An `addr2line` clone](https://github.com/gimli-rs/addr2line) + + * [`ddbug`](https://github.com/gimli-rs/ddbug), a utility giving insight into + code generation by making debugging information readable. + + * [`dwprod`](https://github.com/fitzgen/dwprod), a tiny utility to list the + compilers used to create each compilation unit within a shared library or + executable (via `DW_AT_producer`). + + * [`dwarf-validate`](./examples/dwarf-validate.rs), a program to validate the + integrity of some DWARF and its references between sections and compilation + units. + +## License + +Licensed under either of + + * Apache License, Version 2.0 ([`LICENSE-APACHE`](./LICENSE-APACHE) or https://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([`LICENSE-MIT`](./LICENSE-MIT) or https://opensource.org/licenses/MIT) + +at your option. + +## Contribution + +See [CONTRIBUTING.md](./CONTRIBUTING.md) for hacking. + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in the work by you, as defined in the Apache-2.0 license, shall be +dual licensed as above, without any additional terms or conditions. diff --git a/crux-mir/lib/gimli/benches/bench.rs b/crux-mir/lib/gimli/benches/bench.rs new file mode 100644 index 000000000..fb29df77c --- /dev/null +++ b/crux-mir/lib/gimli/benches/bench.rs @@ -0,0 +1,807 @@ +#![feature(test)] + +extern crate test; + +use gimli::{ + AttributeValue, DebugAbbrev, DebugAddr, DebugAddrBase, DebugAranges, DebugInfo, DebugLine, + DebugLineOffset, DebugLoc, DebugLocLists, DebugPubNames, DebugPubTypes, DebugRanges, + DebugRngLists, Encoding, EndianSlice, EntriesTreeNode, Expression, LittleEndian, LocationLists, + Operation, RangeLists, RangeListsOffset, Reader, ReaderOffset, +}; +use std::env; +use std::fs::File; +use std::io::Read; +use std::path::PathBuf; +use std::rc::Rc; + +pub fn read_section(section: &str) -> Vec { + let mut path = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".into())); + path.push("./fixtures/self/"); + path.push(section); + + assert!(path.is_file()); + let mut file = File::open(path).unwrap(); + + let mut buf = Vec::new(); + file.read_to_end(&mut buf).unwrap(); + buf +} + +#[bench] +fn bench_parsing_debug_abbrev(b: &mut test::Bencher) { + let debug_info = read_section("debug_info"); + let debug_info = DebugInfo::new(&debug_info, LittleEndian); + let unit = debug_info + .units() + .next() + .expect("Should have at least one compilation unit") + .expect("And it should parse OK"); + + let debug_abbrev = read_section("debug_abbrev"); + + b.iter(|| { + let debug_abbrev = DebugAbbrev::new(&debug_abbrev, LittleEndian); + test::black_box( + unit.abbreviations(&debug_abbrev) + .expect("Should parse abbreviations"), + ); + }); +} + +#[inline] +fn impl_bench_parsing_debug_info( + debug_info: DebugInfo, + debug_abbrev: DebugAbbrev, +) { + let mut iter = debug_info.units(); + while let Some(unit) = iter.next().expect("Should parse compilation unit") { + let abbrevs = unit + .abbreviations(&debug_abbrev) + .expect("Should parse abbreviations"); + + let mut cursor = unit.entries(&abbrevs); + while let Some((_, entry)) = cursor.next_dfs().expect("Should parse next dfs") { + let mut attrs = entry.attrs(); + loop { + match attrs.next() { + Ok(Some(ref attr)) => { + test::black_box(attr); + } + Ok(None) => break, + e @ Err(_) => { + e.expect("Should parse entry's attribute"); + } + } + } + } + } +} + +#[bench] +fn bench_parsing_debug_info(b: &mut test::Bencher) { + let debug_info = read_section("debug_info"); + let debug_info = DebugInfo::new(&debug_info, LittleEndian); + + let debug_abbrev = read_section("debug_abbrev"); + let debug_abbrev = DebugAbbrev::new(&debug_abbrev, LittleEndian); + + b.iter(|| impl_bench_parsing_debug_info(debug_info, debug_abbrev)); +} + +#[bench] +fn bench_parsing_debug_info_with_endian_rc_slice(b: &mut test::Bencher) { + let debug_info = read_section("debug_info"); + let debug_info = Rc::from(&debug_info[..]); + let debug_info = gimli::EndianRcSlice::new(debug_info, LittleEndian); + let debug_info = DebugInfo::from(debug_info); + + let debug_abbrev = read_section("debug_abbrev"); + let debug_abbrev = Rc::from(&debug_abbrev[..]); + let debug_abbrev = gimli::EndianRcSlice::new(debug_abbrev, LittleEndian); + let debug_abbrev = DebugAbbrev::from(debug_abbrev); + + b.iter(|| impl_bench_parsing_debug_info(debug_info.clone(), debug_abbrev.clone())); +} + +#[bench] +fn bench_parsing_debug_info_tree(b: &mut test::Bencher) { + let debug_abbrev = read_section("debug_abbrev"); + let debug_abbrev = DebugAbbrev::new(&debug_abbrev, LittleEndian); + + let debug_info = read_section("debug_info"); + + b.iter(|| { + let debug_info = DebugInfo::new(&debug_info, LittleEndian); + + let mut iter = debug_info.units(); + while let Some(unit) = iter.next().expect("Should parse compilation unit") { + let abbrevs = unit + .abbreviations(&debug_abbrev) + .expect("Should parse abbreviations"); + + let mut tree = unit + .entries_tree(&abbrevs, None) + .expect("Should have entries tree"); + let root = tree.root().expect("Should parse root entry"); + parse_debug_info_tree(root); + } + }); +} + +fn parse_debug_info_tree(node: EntriesTreeNode) { + { + let mut attrs = node.entry().attrs(); + loop { + match attrs.next() { + Ok(Some(ref attr)) => { + test::black_box(attr); + } + Ok(None) => break, + e @ Err(_) => { + e.expect("Should parse entry's attribute"); + } + } + } + } + let mut children = node.children(); + while let Some(child) = children.next().expect("Should parse child entry") { + parse_debug_info_tree(child); + } +} + +#[bench] +fn bench_parsing_debug_info_raw(b: &mut test::Bencher) { + let debug_abbrev = read_section("debug_abbrev"); + let debug_abbrev = DebugAbbrev::new(&debug_abbrev, LittleEndian); + + let debug_info = read_section("debug_info"); + + b.iter(|| { + let debug_info = DebugInfo::new(&debug_info, LittleEndian); + + let mut iter = debug_info.units(); + while let Some(unit) = iter.next().expect("Should parse compilation unit") { + let abbrevs = unit + .abbreviations(&debug_abbrev) + .expect("Should parse abbreviations"); + + let mut raw = unit + .entries_raw(&abbrevs, None) + .expect("Should have entries"); + while !raw.is_empty() { + if let Some(abbrev) = raw + .read_abbreviation() + .expect("Should parse abbreviation code") + { + for spec in abbrev.attributes().iter().cloned() { + match raw.read_attribute(spec) { + Ok(ref attr) => { + test::black_box(attr); + } + e @ Err(_) => { + e.expect("Should parse attribute"); + } + } + } + } + } + } + }); +} + +#[bench] +fn bench_parsing_debug_aranges(b: &mut test::Bencher) { + let debug_aranges = read_section("debug_aranges"); + let debug_aranges = DebugAranges::new(&debug_aranges, LittleEndian); + + b.iter(|| { + let mut headers = debug_aranges.headers(); + while let Some(header) = headers.next().expect("Should parse arange header OK") { + let mut entries = header.entries(); + while let Some(arange) = entries.next().expect("Should parse arange entry OK") { + test::black_box(arange); + } + } + }); +} + +#[bench] +fn bench_parsing_debug_pubnames(b: &mut test::Bencher) { + let debug_pubnames = read_section("debug_pubnames"); + let debug_pubnames = DebugPubNames::new(&debug_pubnames, LittleEndian); + + b.iter(|| { + let mut pubnames = debug_pubnames.items(); + while let Some(pubname) = pubnames.next().expect("Should parse pubname OK") { + test::black_box(pubname); + } + }); +} + +#[bench] +fn bench_parsing_debug_pubtypes(b: &mut test::Bencher) { + let debug_pubtypes = read_section("debug_pubtypes"); + let debug_pubtypes = DebugPubTypes::new(&debug_pubtypes, LittleEndian); + + b.iter(|| { + let mut pubtypes = debug_pubtypes.items(); + while let Some(pubtype) = pubtypes.next().expect("Should parse pubtype OK") { + test::black_box(pubtype); + } + }); +} + +// We happen to know that there is a line number program and header at +// offset 0 and that address size is 8 bytes. No need to parse DIEs to grab +// this info off of the compilation units. +const OFFSET: DebugLineOffset = DebugLineOffset(0); +const ADDRESS_SIZE: u8 = 8; + +#[bench] +fn bench_parsing_line_number_program_opcodes(b: &mut test::Bencher) { + let debug_line = read_section("debug_line"); + let debug_line = DebugLine::new(&debug_line, LittleEndian); + + b.iter(|| { + let program = debug_line + .program(OFFSET, ADDRESS_SIZE, None, None) + .expect("Should parse line number program header"); + let header = program.header(); + + let mut instructions = header.instructions(); + while let Some(instruction) = instructions + .next_instruction(header) + .expect("Should parse instruction") + { + test::black_box(instruction); + } + }); +} + +#[bench] +fn bench_executing_line_number_programs(b: &mut test::Bencher) { + let debug_line = read_section("debug_line"); + let debug_line = DebugLine::new(&debug_line, LittleEndian); + + b.iter(|| { + let program = debug_line + .program(OFFSET, ADDRESS_SIZE, None, None) + .expect("Should parse line number program header"); + + let mut rows = program.rows(); + while let Some(row) = rows + .next_row() + .expect("Should parse and execute all rows in the line number program") + { + test::black_box(row); + } + }); +} + +#[bench] +fn bench_parsing_debug_loc(b: &mut test::Bencher) { + let debug_info = read_section("debug_info"); + let debug_info = DebugInfo::new(&debug_info, LittleEndian); + + let debug_abbrev = read_section("debug_abbrev"); + let debug_abbrev = DebugAbbrev::new(&debug_abbrev, LittleEndian); + + let debug_addr = DebugAddr::from(EndianSlice::new(&[], LittleEndian)); + let debug_addr_base = DebugAddrBase(0); + + let debug_loc = read_section("debug_loc"); + let debug_loc = DebugLoc::new(&debug_loc, LittleEndian); + let debug_loclists = DebugLocLists::new(&[], LittleEndian); + let loclists = LocationLists::new(debug_loc, debug_loclists); + + let mut offsets = Vec::new(); + + let mut iter = debug_info.units(); + while let Some(unit) = iter.next().expect("Should parse compilation unit") { + let abbrevs = unit + .abbreviations(&debug_abbrev) + .expect("Should parse abbreviations"); + + let mut cursor = unit.entries(&abbrevs); + cursor.next_dfs().expect("Should parse next dfs"); + + let mut low_pc = 0; + + { + let unit_entry = cursor.current().expect("Should have a root entry"); + let low_pc_attr = unit_entry + .attr_value(gimli::DW_AT_low_pc) + .expect("Should parse low_pc"); + if let Some(gimli::AttributeValue::Addr(address)) = low_pc_attr { + low_pc = address; + } + } + + while cursor.next_dfs().expect("Should parse next dfs").is_some() { + let entry = cursor.current().expect("Should have a current entry"); + let mut attrs = entry.attrs(); + while let Some(attr) = attrs.next().expect("Should parse entry's attribute") { + if let gimli::AttributeValue::LocationListsRef(offset) = attr.value() { + offsets.push((offset, unit.encoding(), low_pc)); + } + } + } + } + + b.iter(|| { + for &(offset, encoding, base_address) in &*offsets { + let mut locs = loclists + .locations(offset, encoding, base_address, &debug_addr, debug_addr_base) + .expect("Should parse locations OK"); + while let Some(loc) = locs.next().expect("Should parse next location") { + test::black_box(loc); + } + } + }); +} + +#[bench] +fn bench_parsing_debug_ranges(b: &mut test::Bencher) { + let debug_info = read_section("debug_info"); + let debug_info = DebugInfo::new(&debug_info, LittleEndian); + + let debug_abbrev = read_section("debug_abbrev"); + let debug_abbrev = DebugAbbrev::new(&debug_abbrev, LittleEndian); + + let debug_addr = DebugAddr::from(EndianSlice::new(&[], LittleEndian)); + let debug_addr_base = DebugAddrBase(0); + + let debug_ranges = read_section("debug_ranges"); + let debug_ranges = DebugRanges::new(&debug_ranges, LittleEndian); + let debug_rnglists = DebugRngLists::new(&[], LittleEndian); + let rnglists = RangeLists::new(debug_ranges, debug_rnglists); + + let mut offsets = Vec::new(); + + let mut iter = debug_info.units(); + while let Some(unit) = iter.next().expect("Should parse compilation unit") { + let abbrevs = unit + .abbreviations(&debug_abbrev) + .expect("Should parse abbreviations"); + + let mut cursor = unit.entries(&abbrevs); + cursor.next_dfs().expect("Should parse next dfs"); + + let mut low_pc = 0; + + { + let unit_entry = cursor.current().expect("Should have a root entry"); + let low_pc_attr = unit_entry + .attr_value(gimli::DW_AT_low_pc) + .expect("Should parse low_pc"); + if let Some(gimli::AttributeValue::Addr(address)) = low_pc_attr { + low_pc = address; + } + } + + while cursor.next_dfs().expect("Should parse next dfs").is_some() { + let entry = cursor.current().expect("Should have a current entry"); + let mut attrs = entry.attrs(); + while let Some(attr) = attrs.next().expect("Should parse entry's attribute") { + if let gimli::AttributeValue::RangeListsRef(offset) = attr.value() { + offsets.push((RangeListsOffset(offset.0), unit.encoding(), low_pc)); + } + } + } + } + + b.iter(|| { + for &(offset, encoding, base_address) in &*offsets { + let mut ranges = rnglists + .ranges(offset, encoding, base_address, &debug_addr, debug_addr_base) + .expect("Should parse ranges OK"); + while let Some(range) = ranges.next().expect("Should parse next range") { + test::black_box(range); + } + } + }); +} + +fn debug_info_expressions( + debug_info: &DebugInfo, + debug_abbrev: &DebugAbbrev, +) -> Vec<(Expression, Encoding)> { + let mut expressions = Vec::new(); + + let mut iter = debug_info.units(); + while let Some(unit) = iter.next().expect("Should parse compilation unit") { + let abbrevs = unit + .abbreviations(debug_abbrev) + .expect("Should parse abbreviations"); + + let mut cursor = unit.entries(&abbrevs); + while let Some((_, entry)) = cursor.next_dfs().expect("Should parse next dfs") { + let mut attrs = entry.attrs(); + while let Some(attr) = attrs.next().expect("Should parse entry's attribute") { + if let AttributeValue::Exprloc(expression) = attr.value() { + expressions.push((expression, unit.encoding())); + } + } + } + } + + expressions +} + +#[bench] +fn bench_parsing_debug_info_expressions(b: &mut test::Bencher) { + let debug_abbrev = read_section("debug_abbrev"); + let debug_abbrev = DebugAbbrev::new(&debug_abbrev, LittleEndian); + + let debug_info = read_section("debug_info"); + let debug_info = DebugInfo::new(&debug_info, LittleEndian); + + let expressions = debug_info_expressions(&debug_info, &debug_abbrev); + + b.iter(|| { + for &(expression, encoding) in &*expressions { + let mut pc = expression.0; + while !pc.is_empty() { + Operation::parse(&mut pc, encoding).expect("Should parse operation"); + } + } + }); +} + +#[bench] +fn bench_evaluating_debug_info_expressions(b: &mut test::Bencher) { + let debug_abbrev = read_section("debug_abbrev"); + let debug_abbrev = DebugAbbrev::new(&debug_abbrev, LittleEndian); + + let debug_info = read_section("debug_info"); + let debug_info = DebugInfo::new(&debug_info, LittleEndian); + + let expressions = debug_info_expressions(&debug_info, &debug_abbrev); + + b.iter(|| { + for &(expression, encoding) in &*expressions { + let mut eval = expression.evaluation(encoding); + eval.set_initial_value(0); + let result = eval.evaluate().expect("Should evaluate expression"); + test::black_box(result); + } + }); +} + +fn debug_loc_expressions( + debug_info: &DebugInfo, + debug_abbrev: &DebugAbbrev, + debug_addr: &DebugAddr, + loclists: &LocationLists, +) -> Vec<(Expression, Encoding)> { + let debug_addr_base = DebugAddrBase(R::Offset::from_u8(0)); + + let mut expressions = Vec::new(); + + let mut iter = debug_info.units(); + while let Some(unit) = iter.next().expect("Should parse compilation unit") { + let abbrevs = unit + .abbreviations(debug_abbrev) + .expect("Should parse abbreviations"); + + let mut cursor = unit.entries(&abbrevs); + cursor.next_dfs().expect("Should parse next dfs"); + + let mut low_pc = 0; + + { + let unit_entry = cursor.current().expect("Should have a root entry"); + let low_pc_attr = unit_entry + .attr_value(gimli::DW_AT_low_pc) + .expect("Should parse low_pc"); + if let Some(gimli::AttributeValue::Addr(address)) = low_pc_attr { + low_pc = address; + } + } + + while cursor.next_dfs().expect("Should parse next dfs").is_some() { + let entry = cursor.current().expect("Should have a current entry"); + let mut attrs = entry.attrs(); + while let Some(attr) = attrs.next().expect("Should parse entry's attribute") { + if let gimli::AttributeValue::LocationListsRef(offset) = attr.value() { + let mut locs = loclists + .locations(offset, unit.encoding(), low_pc, debug_addr, debug_addr_base) + .expect("Should parse locations OK"); + while let Some(loc) = locs.next().expect("Should parse next location") { + expressions.push((loc.data, unit.encoding())); + } + } + } + } + } + + expressions +} + +#[bench] +fn bench_parsing_debug_loc_expressions(b: &mut test::Bencher) { + let debug_info = read_section("debug_info"); + let debug_info = DebugInfo::new(&debug_info, LittleEndian); + + let debug_abbrev = read_section("debug_abbrev"); + let debug_abbrev = DebugAbbrev::new(&debug_abbrev, LittleEndian); + + let debug_addr = DebugAddr::from(EndianSlice::new(&[], LittleEndian)); + + let debug_loc = read_section("debug_loc"); + let debug_loc = DebugLoc::new(&debug_loc, LittleEndian); + let debug_loclists = DebugLocLists::new(&[], LittleEndian); + let loclists = LocationLists::new(debug_loc, debug_loclists); + + let expressions = debug_loc_expressions(&debug_info, &debug_abbrev, &debug_addr, &loclists); + + b.iter(|| { + for &(expression, encoding) in &*expressions { + let mut pc = expression.0; + while !pc.is_empty() { + Operation::parse(&mut pc, encoding).expect("Should parse operation"); + } + } + }); +} + +#[bench] +fn bench_evaluating_debug_loc_expressions(b: &mut test::Bencher) { + let debug_info = read_section("debug_info"); + let debug_info = DebugInfo::new(&debug_info, LittleEndian); + + let debug_abbrev = read_section("debug_abbrev"); + let debug_abbrev = DebugAbbrev::new(&debug_abbrev, LittleEndian); + + let debug_addr = DebugAddr::from(EndianSlice::new(&[], LittleEndian)); + + let debug_loc = read_section("debug_loc"); + let debug_loc = DebugLoc::new(&debug_loc, LittleEndian); + let debug_loclists = DebugLocLists::new(&[], LittleEndian); + let loclists = LocationLists::new(debug_loc, debug_loclists); + + let expressions = debug_loc_expressions(&debug_info, &debug_abbrev, &debug_addr, &loclists); + + b.iter(|| { + for &(expression, encoding) in &*expressions { + let mut eval = expression.evaluation(encoding); + eval.set_initial_value(0); + let result = eval.evaluate().expect("Should evaluate expression"); + test::black_box(result); + } + }); +} + +// See comment above `test_parse_self_eh_frame`. +#[cfg(target_pointer_width = "64")] +mod cfi { + use super::*; + use fallible_iterator::FallibleIterator; + + use gimli::{ + BaseAddresses, CieOrFde, EhFrame, FrameDescriptionEntry, LittleEndian, UnwindContext, + UnwindSection, + }; + + #[bench] + fn iterate_entries_and_do_not_parse_any_fde(b: &mut test::Bencher) { + let eh_frame = read_section("eh_frame"); + let eh_frame = EhFrame::new(&eh_frame, LittleEndian); + + let bases = BaseAddresses::default() + .set_eh_frame(0) + .set_got(0) + .set_text(0); + + b.iter(|| { + let mut entries = eh_frame.entries(&bases); + while let Some(entry) = entries.next().expect("Should parse CFI entry OK") { + test::black_box(entry); + } + }); + } + + #[bench] + fn iterate_entries_and_parse_every_fde(b: &mut test::Bencher) { + let eh_frame = read_section("eh_frame"); + let eh_frame = EhFrame::new(&eh_frame, LittleEndian); + + let bases = BaseAddresses::default() + .set_eh_frame(0) + .set_got(0) + .set_text(0); + + b.iter(|| { + let mut entries = eh_frame.entries(&bases); + while let Some(entry) = entries.next().expect("Should parse CFI entry OK") { + match entry { + CieOrFde::Cie(cie) => { + test::black_box(cie); + } + CieOrFde::Fde(partial) => { + let fde = partial + .parse(EhFrame::cie_from_offset) + .expect("Should be able to get CIE for FED"); + test::black_box(fde); + } + }; + } + }); + } + + #[bench] + fn iterate_entries_and_parse_every_fde_and_instructions(b: &mut test::Bencher) { + let eh_frame = read_section("eh_frame"); + let eh_frame = EhFrame::new(&eh_frame, LittleEndian); + + let bases = BaseAddresses::default() + .set_eh_frame(0) + .set_got(0) + .set_text(0); + + b.iter(|| { + let mut entries = eh_frame.entries(&bases); + while let Some(entry) = entries.next().expect("Should parse CFI entry OK") { + match entry { + CieOrFde::Cie(cie) => { + let mut instrs = cie.instructions(&eh_frame, &bases); + while let Some(i) = + instrs.next().expect("Can parse next CFI instruction OK") + { + test::black_box(i); + } + } + CieOrFde::Fde(partial) => { + let fde = partial + .parse(EhFrame::cie_from_offset) + .expect("Should be able to get CIE for FED"); + let mut instrs = fde.instructions(&eh_frame, &bases); + while let Some(i) = + instrs.next().expect("Can parse next CFI instruction OK") + { + test::black_box(i); + } + } + }; + } + }); + } + + #[bench] + fn iterate_entries_evaluate_every_fde(b: &mut test::Bencher) { + let eh_frame = read_section("eh_frame"); + let eh_frame = EhFrame::new(&eh_frame, LittleEndian); + + let bases = BaseAddresses::default() + .set_eh_frame(0) + .set_got(0) + .set_text(0); + + let mut ctx = Box::new(UnwindContext::new()); + + b.iter(|| { + let mut entries = eh_frame.entries(&bases); + while let Some(entry) = entries.next().expect("Should parse CFI entry OK") { + match entry { + CieOrFde::Cie(_) => {} + CieOrFde::Fde(partial) => { + let fde = partial + .parse(EhFrame::cie_from_offset) + .expect("Should be able to get CIE for FED"); + let mut table = fde + .rows(&eh_frame, &bases, &mut ctx) + .expect("Should be able to initialize ctx"); + while let Some(row) = + table.next_row().expect("Should get next unwind table row") + { + test::black_box(row); + } + } + }; + } + }); + } + + fn instrs_len( + eh_frame: &EhFrame, + bases: &BaseAddresses, + fde: &FrameDescriptionEntry, + ) -> usize { + fde.instructions(eh_frame, bases) + .fold(0, |count, _| Ok(count + 1)) + .expect("fold over instructions OK") + } + + fn get_fde_with_longest_cfi_instructions( + eh_frame: &EhFrame, + bases: &BaseAddresses, + ) -> FrameDescriptionEntry { + let mut longest: Option<(usize, FrameDescriptionEntry<_>)> = None; + + let mut entries = eh_frame.entries(bases); + while let Some(entry) = entries.next().expect("Should parse CFI entry OK") { + match entry { + CieOrFde::Cie(_) => {} + CieOrFde::Fde(partial) => { + let fde = partial + .parse(EhFrame::cie_from_offset) + .expect("Should be able to get CIE for FED"); + + let this_len = instrs_len(eh_frame, bases, &fde); + + let found_new_longest = match longest { + None => true, + Some((longest_len, ref _fde)) => this_len > longest_len, + }; + + if found_new_longest { + longest = Some((this_len, fde)); + } + } + }; + } + + longest.expect("At least one FDE in .eh_frame").1 + } + + #[bench] + fn parse_longest_fde_instructions(b: &mut test::Bencher) { + let eh_frame = read_section("eh_frame"); + let eh_frame = EhFrame::new(&eh_frame, LittleEndian); + let bases = BaseAddresses::default() + .set_eh_frame(0) + .set_got(0) + .set_text(0); + let fde = get_fde_with_longest_cfi_instructions(&eh_frame, &bases); + + b.iter(|| { + let mut instrs = fde.instructions(&eh_frame, &bases); + while let Some(i) = instrs.next().expect("Should parse instruction OK") { + test::black_box(i); + } + }); + } + + #[bench] + fn eval_longest_fde_instructions_new_ctx_everytime(b: &mut test::Bencher) { + let eh_frame = read_section("eh_frame"); + let eh_frame = EhFrame::new(&eh_frame, LittleEndian); + let bases = BaseAddresses::default() + .set_eh_frame(0) + .set_got(0) + .set_text(0); + let fde = get_fde_with_longest_cfi_instructions(&eh_frame, &bases); + + b.iter(|| { + let mut ctx = Box::new(UnwindContext::new()); + let mut table = fde + .rows(&eh_frame, &bases, &mut ctx) + .expect("Should initialize the ctx OK"); + while let Some(row) = table.next_row().expect("Should get next unwind table row") { + test::black_box(row); + } + }); + } + + #[bench] + fn eval_longest_fde_instructions_same_ctx(b: &mut test::Bencher) { + let eh_frame = read_section("eh_frame"); + let eh_frame = EhFrame::new(&eh_frame, LittleEndian); + let bases = BaseAddresses::default() + .set_eh_frame(0) + .set_got(0) + .set_text(0); + let fde = get_fde_with_longest_cfi_instructions(&eh_frame, &bases); + + let mut ctx = Box::new(UnwindContext::new()); + + b.iter(|| { + let mut table = fde + .rows(&eh_frame, &bases, &mut ctx) + .expect("Should initialize the ctx OK"); + while let Some(row) = table.next_row().expect("Should get next unwind table row") { + test::black_box(row); + } + }); + } +} diff --git a/crux-mir/lib/gimli/examples/dwarf-validate.rs b/crux-mir/lib/gimli/examples/dwarf-validate.rs new file mode 100644 index 000000000..54d8f3a1d --- /dev/null +++ b/crux-mir/lib/gimli/examples/dwarf-validate.rs @@ -0,0 +1,267 @@ +// Allow clippy lints when building without clippy. +#![allow(unknown_lints)] + +use gimli::{AttributeValue, UnitHeader}; +use object::{Object, ObjectSection}; +use rayon::prelude::*; +use std::borrow::{Borrow, Cow}; +use std::env; +use std::fs; +use std::io::{self, BufWriter, Write}; +use std::iter::Iterator; +use std::path::{Path, PathBuf}; +use std::process; +use std::sync::Mutex; +use typed_arena::Arena; + +trait Reader: gimli::Reader + Send + Sync { + type SyncSendEndian: gimli::Endianity + Send + Sync; +} + +impl<'input, Endian> Reader for gimli::EndianSlice<'input, Endian> +where + Endian: gimli::Endianity + Send + Sync, +{ + type SyncSendEndian = Endian; +} + +struct ErrorWriter { + inner: Mutex<(W, usize)>, + path: PathBuf, +} + +impl ErrorWriter { + #[allow(clippy::needless_pass_by_value)] + fn error(&self, s: String) { + let mut lock = self.inner.lock().unwrap(); + writeln!(&mut lock.0, "DWARF error in {}: {}", self.path.display(), s).unwrap(); + lock.1 += 1; + } +} + +fn main() { + let mut w = BufWriter::new(io::stdout()); + let mut errors = 0; + for arg in env::args_os().skip(1) { + let path = Path::new(&arg); + let file = match fs::File::open(&path) { + Ok(file) => file, + Err(err) => { + eprintln!("Failed to open file '{}': {}", path.display(), err); + errors += 1; + continue; + } + }; + let file = match unsafe { memmap2::Mmap::map(&file) } { + Ok(mmap) => mmap, + Err(err) => { + eprintln!("Failed to map file '{}': {}", path.display(), &err); + errors += 1; + continue; + } + }; + let file = match object::File::parse(&*file) { + Ok(file) => file, + Err(err) => { + eprintln!("Failed to parse file '{}': {}", path.display(), err); + errors += 1; + continue; + } + }; + + let endian = if file.is_little_endian() { + gimli::RunTimeEndian::Little + } else { + gimli::RunTimeEndian::Big + }; + let mut error_writer = ErrorWriter { + inner: Mutex::new((&mut w, 0)), + path: path.to_owned(), + }; + validate_file(&mut error_writer, &file, endian); + errors += error_writer.inner.into_inner().unwrap().1; + } + // Flush any errors. + drop(w); + if errors > 0 { + process::exit(1); + } +} + +fn validate_file(w: &mut ErrorWriter, file: &object::File, endian: Endian) +where + W: Write + Send, + Endian: gimli::Endianity + Send + Sync, +{ + let arena = Arena::new(); + + fn load_section<'a, 'file, 'input, S, Endian>( + arena: &'a Arena>, + file: &'file object::File<'input>, + endian: Endian, + ) -> S + where + S: gimli::Section>, + Endian: gimli::Endianity + Send + Sync, + 'file: 'input, + 'a: 'file, + { + let data = match file.section_by_name(S::section_name()) { + Some(ref section) => section + .uncompressed_data() + .unwrap_or(Cow::Borrowed(&[][..])), + None => Cow::Borrowed(&[][..]), + }; + let data_ref = (*arena.alloc(data)).borrow(); + S::from(gimli::EndianSlice::new(data_ref, endian)) + } + + // Variables representing sections of the file. The type of each is inferred from its use in the + // validate_info function below. + let debug_abbrev = &load_section(&arena, file, endian); + let debug_info = &load_section(&arena, file, endian); + + validate_info(w, debug_info, debug_abbrev); +} + +struct UnitSummary { + // True if we successfully parsed all the DIEs and attributes in the compilation unit + internally_valid: bool, + offset: gimli::DebugInfoOffset, + die_offsets: Vec, + global_die_references: Vec<(gimli::UnitOffset, gimli::DebugInfoOffset)>, +} + +fn validate_info( + w: &mut ErrorWriter, + debug_info: &gimli::DebugInfo, + debug_abbrev: &gimli::DebugAbbrev, +) where + W: Write + Send, + R: Reader, +{ + let mut units = Vec::new(); + let mut units_iter = debug_info.units(); + let mut last_offset = 0; + loop { + let u = match units_iter.next() { + Err(err) => { + w.error(format!( + "Can't read unit header at offset {:#x}, stopping reading units: {}", + last_offset, err + )); + break; + } + Ok(None) => break, + Ok(Some(u)) => u, + }; + last_offset = u.offset().as_debug_info_offset().unwrap().0 + u.length_including_self(); + units.push(u); + } + let process_unit = |unit: UnitHeader| -> UnitSummary { + let unit_offset = unit.offset().as_debug_info_offset().unwrap(); + let mut ret = UnitSummary { + internally_valid: false, + offset: unit_offset, + die_offsets: Vec::new(), + global_die_references: Vec::new(), + }; + let abbrevs = match unit.abbreviations(debug_abbrev) { + Ok(abbrevs) => abbrevs, + Err(err) => { + w.error(format!( + "Invalid abbrevs for unit {:#x}: {}", + unit_offset.0, &err + )); + return ret; + } + }; + let mut entries = unit.entries(&abbrevs); + let mut unit_refs = Vec::new(); + loop { + let (_, entry) = match entries.next_dfs() { + Err(err) => { + w.error(format!( + "Invalid DIE for unit {:#x}: {}", + unit_offset.0, &err + )); + return ret; + } + Ok(None) => break, + Ok(Some(entry)) => entry, + }; + ret.die_offsets.push(entry.offset()); + + let mut attrs = entry.attrs(); + loop { + let attr = match attrs.next() { + Err(err) => { + w.error(format!( + "Invalid attribute for unit {:#x} at DIE {:#x}: {}", + unit_offset.0, + entry.offset().0, + &err + )); + return ret; + } + Ok(None) => break, + Ok(Some(attr)) => attr, + }; + match attr.value() { + AttributeValue::UnitRef(offset) => { + unit_refs.push((entry.offset(), offset)); + } + AttributeValue::DebugInfoRef(offset) => { + ret.global_die_references.push((entry.offset(), offset)); + } + _ => (), + } + } + } + ret.internally_valid = true; + ret.die_offsets.shrink_to_fit(); + ret.global_die_references.shrink_to_fit(); + + // Check intra-unit references + for (from, to) in unit_refs { + if ret.die_offsets.binary_search(&to).is_err() { + w.error(format!( + "Invalid intra-unit reference in unit {:#x} from DIE {:#x} to {:#x}", + unit_offset.0, from.0, to.0 + )); + } + } + + ret + }; + let processed_units = units.into_par_iter().map(process_unit).collect::>(); + + let check_unit = |summary: &UnitSummary| { + if !summary.internally_valid { + return; + } + for &(from, to) in summary.global_die_references.iter() { + let u = match processed_units.binary_search_by_key(&to, |v| v.offset) { + Ok(i) => &processed_units[i], + Err(i) => { + if i > 0 { + &processed_units[i - 1] + } else { + w.error(format!("Invalid cross-unit reference in unit {:#x} from DIE {:#x} to global DIE {:#x}: no unit found", + summary.offset.0, from.0, to.0)); + continue; + } + } + }; + if !u.internally_valid { + continue; + } + let to_offset = gimli::UnitOffset(to.0 - u.offset.0); + if u.die_offsets.binary_search(&to_offset).is_err() { + w.error(format!("Invalid cross-unit reference in unit {:#x} from DIE {:#x} to global DIE {:#x}: unit at {:#x} contains no DIE {:#x}", + summary.offset.0, from.0, to.0, u.offset.0, to_offset.0)); + } + } + }; + processed_units.par_iter().for_each(check_unit); +} diff --git a/crux-mir/lib/gimli/examples/dwarfdump.rs b/crux-mir/lib/gimli/examples/dwarfdump.rs new file mode 100644 index 000000000..4b61fd572 --- /dev/null +++ b/crux-mir/lib/gimli/examples/dwarfdump.rs @@ -0,0 +1,2417 @@ +// Allow clippy lints when building without clippy. +#![allow(unknown_lints)] + +use fallible_iterator::FallibleIterator; +use gimli::{Section, UnitHeader, UnitOffset, UnitSectionOffset, UnitType, UnwindSection}; +use object::{Object, ObjectSection, ObjectSymbol}; +use regex::bytes::Regex; +use std::borrow::{Borrow, Cow}; +use std::cmp::min; +use std::collections::HashMap; +use std::env; +use std::fmt::{self, Debug}; +use std::fs; +use std::io; +use std::io::{BufWriter, Write}; +use std::iter::Iterator; +use std::mem; +use std::process; +use std::result; +use std::sync::{Condvar, Mutex}; +use typed_arena::Arena; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Error { + GimliError(gimli::Error), + ObjectError(object::read::Error), + IoError, +} + +impl fmt::Display for Error { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter) -> ::std::result::Result<(), fmt::Error> { + Debug::fmt(self, f) + } +} + +fn writeln_error( + w: &mut W, + dwarf: &gimli::Dwarf, + err: Error, + msg: &str, +) -> io::Result<()> { + writeln!( + w, + "{}: {}", + msg, + match err { + Error::GimliError(err) => dwarf.format_error(err), + Error::ObjectError(err) => + format!("{}:{:?}", "An object error occurred while reading", err), + Error::IoError => "An I/O error occurred while writing.".to_string(), + } + ) +} + +impl From for Error { + fn from(err: gimli::Error) -> Self { + Error::GimliError(err) + } +} + +impl From for Error { + fn from(_: io::Error) -> Self { + Error::IoError + } +} + +impl From for Error { + fn from(err: object::read::Error) -> Self { + Error::ObjectError(err) + } +} + +pub type Result = result::Result; + +fn parallel_output(w: &mut W, max_workers: usize, iter: II, f: F) -> Result<()> +where + W: Write + Send, + F: Sync + Fn(II::Item, &mut Vec) -> Result<()>, + II: IntoIterator, + II::IntoIter: Send, +{ + struct ParallelOutputState { + iterator: I, + current_worker: usize, + result: Result<()>, + w: W, + } + + let state = Mutex::new(ParallelOutputState { + iterator: iter.into_iter().fuse(), + current_worker: 0, + result: Ok(()), + w, + }); + let workers = min(max_workers, num_cpus::get()); + let mut condvars = Vec::new(); + for _ in 0..workers { + condvars.push(Condvar::new()); + } + { + let state_ref = &state; + let f_ref = &f; + let condvars_ref = &condvars; + crossbeam::scope(|scope| { + for i in 0..workers { + scope.spawn(move |_| { + let mut v = Vec::new(); + let mut lock = state_ref.lock().unwrap(); + while lock.current_worker != i { + lock = condvars_ref[i].wait(lock).unwrap(); + } + loop { + let item = if lock.result.is_ok() { + lock.iterator.next() + } else { + None + }; + lock.current_worker = (i + 1) % workers; + condvars_ref[lock.current_worker].notify_one(); + mem::drop(lock); + + let ret = if let Some(item) = item { + v.clear(); + f_ref(item, &mut v) + } else { + return; + }; + + lock = state_ref.lock().unwrap(); + while lock.current_worker != i { + lock = condvars_ref[i].wait(lock).unwrap(); + } + if lock.result.is_ok() { + let ret2 = lock.w.write_all(&v); + if ret.is_err() { + lock.result = ret; + } else { + lock.result = ret2.map_err(Error::from); + } + } + } + }); + } + }) + .unwrap(); + } + state.into_inner().unwrap().result +} + +trait Reader: gimli::Reader + Send + Sync {} + +impl<'input, Endian> Reader for gimli::EndianSlice<'input, Endian> where + Endian: gimli::Endianity + Send + Sync +{ +} + +type RelocationMap = HashMap; + +fn add_relocations( + relocations: &mut RelocationMap, + file: &object::File, + section: &object::Section, +) { + for (offset64, mut relocation) in section.relocations() { + let offset = offset64 as usize; + if offset as u64 != offset64 { + continue; + } + let offset = offset as usize; + match relocation.kind() { + object::RelocationKind::Absolute => { + match relocation.target() { + object::RelocationTarget::Symbol(symbol_idx) => { + match file.symbol_by_index(symbol_idx) { + Ok(symbol) => { + let addend = + symbol.address().wrapping_add(relocation.addend() as u64); + relocation.set_addend(addend as i64); + } + Err(_) => { + eprintln!( + "Relocation with invalid symbol for section {} at offset 0x{:08x}", + section.name().unwrap(), + offset + ); + } + } + } + _ => {} + } + if relocations.insert(offset, relocation).is_some() { + eprintln!( + "Multiple relocations for section {} at offset 0x{:08x}", + section.name().unwrap(), + offset + ); + } + } + _ => { + eprintln!( + "Unsupported relocation for section {} at offset 0x{:08x}", + section.name().unwrap(), + offset + ); + } + } + } +} + +/// Apply relocations to addresses and offsets during parsing, +/// instead of requiring the data to be fully relocated prior +/// to parsing. +/// +/// Pros +/// - allows readonly buffers, we don't need to implement writing of values back to buffers +/// - potentially allows us to handle addresses and offsets differently +/// - potentially allows us to add metadata from the relocation (eg symbol names) +/// Cons +/// - maybe incomplete +#[derive(Debug, Clone)] +struct Relocate<'a, R: gimli::Reader> { + relocations: &'a RelocationMap, + section: R, + reader: R, +} + +impl<'a, R: gimli::Reader> Relocate<'a, R> { + fn relocate(&self, offset: usize, value: u64) -> u64 { + if let Some(relocation) = self.relocations.get(&offset) { + match relocation.kind() { + object::RelocationKind::Absolute => { + if relocation.has_implicit_addend() { + // Use the explicit addend too, because it may have the symbol value. + return value.wrapping_add(relocation.addend() as u64); + } else { + return relocation.addend() as u64; + } + } + _ => {} + } + }; + value + } +} + +impl<'a, R: gimli::Reader> gimli::Reader for Relocate<'a, R> { + type Endian = R::Endian; + type Offset = R::Offset; + + fn read_address(&mut self, address_size: u8) -> gimli::Result { + let offset = self.reader.offset_from(&self.section); + let value = self.reader.read_address(address_size)?; + Ok(self.relocate(offset, value)) + } + + fn read_length(&mut self, format: gimli::Format) -> gimli::Result { + let offset = self.reader.offset_from(&self.section); + let value = self.reader.read_length(format)?; + ::from_u64(self.relocate(offset, value as u64)) + } + + fn read_offset(&mut self, format: gimli::Format) -> gimli::Result { + let offset = self.reader.offset_from(&self.section); + let value = self.reader.read_offset(format)?; + ::from_u64(self.relocate(offset, value as u64)) + } + + fn read_sized_offset(&mut self, size: u8) -> gimli::Result { + let offset = self.reader.offset_from(&self.section); + let value = self.reader.read_sized_offset(size)?; + ::from_u64(self.relocate(offset, value as u64)) + } + + #[inline] + fn split(&mut self, len: Self::Offset) -> gimli::Result { + let mut other = self.clone(); + other.reader.truncate(len)?; + self.reader.skip(len)?; + Ok(other) + } + + // All remaining methods simply delegate to `self.reader`. + + #[inline] + fn endian(&self) -> Self::Endian { + self.reader.endian() + } + + #[inline] + fn len(&self) -> Self::Offset { + self.reader.len() + } + + #[inline] + fn empty(&mut self) { + self.reader.empty() + } + + #[inline] + fn truncate(&mut self, len: Self::Offset) -> gimli::Result<()> { + self.reader.truncate(len) + } + + #[inline] + fn offset_from(&self, base: &Self) -> Self::Offset { + self.reader.offset_from(&base.reader) + } + + #[inline] + fn offset_id(&self) -> gimli::ReaderOffsetId { + self.reader.offset_id() + } + + #[inline] + fn lookup_offset_id(&self, id: gimli::ReaderOffsetId) -> Option { + self.reader.lookup_offset_id(id) + } + + #[inline] + fn find(&self, byte: u8) -> gimli::Result { + self.reader.find(byte) + } + + #[inline] + fn skip(&mut self, len: Self::Offset) -> gimli::Result<()> { + self.reader.skip(len) + } + + #[inline] + fn to_slice(&self) -> gimli::Result> { + self.reader.to_slice() + } + + #[inline] + fn to_string(&self) -> gimli::Result> { + self.reader.to_string() + } + + #[inline] + fn to_string_lossy(&self) -> gimli::Result> { + self.reader.to_string_lossy() + } + + #[inline] + fn read_slice(&mut self, buf: &mut [u8]) -> gimli::Result<()> { + self.reader.read_slice(buf) + } +} + +impl<'a, R: Reader> Reader for Relocate<'a, R> {} + +#[derive(Default)] +struct Flags<'a> { + eh_frame: bool, + goff: bool, + info: bool, + line: bool, + pubnames: bool, + pubtypes: bool, + aranges: bool, + dwo: bool, + dwp: bool, + dwo_parent: Option>, + sup: Option>, + raw: bool, + match_units: Option, +} + +fn print_usage(opts: &getopts::Options) -> ! { + let brief = format!("Usage: {} ", env::args().next().unwrap()); + write!(&mut io::stderr(), "{}", opts.usage(&brief)).ok(); + process::exit(1); +} + +fn main() { + let mut opts = getopts::Options::new(); + opts.optflag( + "", + "eh-frame", + "print .eh-frame exception handling frame information", + ); + opts.optflag("G", "", "show global die offsets"); + opts.optflag("i", "", "print .debug_info and .debug_types sections"); + opts.optflag("l", "", "print .debug_line section"); + opts.optflag("p", "", "print .debug_pubnames section"); + opts.optflag("r", "", "print .debug_aranges section"); + opts.optflag("y", "", "print .debug_pubtypes section"); + opts.optflag( + "", + "dwo", + "print the .dwo versions of the selected sections", + ); + opts.optflag( + "", + "dwp", + "print the .dwp versions of the selected sections", + ); + opts.optopt( + "", + "dwo-parent", + "use the specified file as the parent of the dwo or dwp (e.g. for .debug_addr)", + "library path", + ); + opts.optflag("", "raw", "print raw data values"); + opts.optopt( + "u", + "match-units", + "print compilation units whose output matches a regex", + "REGEX", + ); + opts.optopt("", "sup", "path to supplementary object file", "PATH"); + + let matches = match opts.parse(env::args().skip(1)) { + Ok(m) => m, + Err(e) => { + writeln!(&mut io::stderr(), "{:?}\n", e).ok(); + print_usage(&opts); + } + }; + if matches.free.is_empty() { + print_usage(&opts); + } + + let mut all = true; + let mut flags = Flags::default(); + if matches.opt_present("eh-frame") { + flags.eh_frame = true; + all = false; + } + if matches.opt_present("G") { + flags.goff = true; + } + if matches.opt_present("i") { + flags.info = true; + all = false; + } + if matches.opt_present("l") { + flags.line = true; + all = false; + } + if matches.opt_present("p") { + flags.pubnames = true; + all = false; + } + if matches.opt_present("y") { + flags.pubtypes = true; + all = false; + } + if matches.opt_present("r") { + flags.aranges = true; + all = false; + } + if matches.opt_present("dwo") { + flags.dwo = true; + } + if matches.opt_present("dwp") { + flags.dwp = true; + } + if matches.opt_present("raw") { + flags.raw = true; + } + if all { + // .eh_frame is excluded even when printing all information. + // cosmetic flags like -G must be set explicitly too. + flags.info = true; + flags.line = true; + flags.pubnames = true; + flags.pubtypes = true; + flags.aranges = true; + } + flags.match_units = if let Some(r) = matches.opt_str("u") { + match Regex::new(&r) { + Ok(r) => Some(r), + Err(e) => { + eprintln!("Invalid regular expression {}: {}", r, e); + process::exit(1); + } + } + } else { + None + }; + + let arena_mmap = Arena::new(); + let load_file = |path| { + let file = match fs::File::open(&path) { + Ok(file) => file, + Err(err) => { + eprintln!("Failed to open file '{}': {}", path, err); + process::exit(1); + } + }; + let mmap = match unsafe { memmap2::Mmap::map(&file) } { + Ok(mmap) => mmap, + Err(err) => { + eprintln!("Failed to map file '{}': {}", path, err); + process::exit(1); + } + }; + let mmap_ref = (*arena_mmap.alloc(mmap)).borrow(); + match object::File::parse(&**mmap_ref) { + Ok(file) => Some(file), + Err(err) => { + eprintln!("Failed to parse file '{}': {}", path, err); + process::exit(1); + } + } + }; + + flags.sup = matches.opt_str("sup").and_then(load_file); + flags.dwo_parent = matches.opt_str("dwo-parent").and_then(load_file); + if flags.dwo_parent.is_some() && !flags.dwo && !flags.dwp { + eprintln!("--dwo-parent also requires --dwo or --dwp"); + process::exit(1); + } + if flags.dwo_parent.is_none() && flags.dwp { + eprintln!("--dwp also requires --dwo-parent"); + process::exit(1); + } + + for file_path in &matches.free { + if matches.free.len() != 1 { + println!("{}", file_path); + println!(); + } + + let file = match fs::File::open(&file_path) { + Ok(file) => file, + Err(err) => { + eprintln!("Failed to open file '{}': {}", file_path, err); + continue; + } + }; + let file = match unsafe { memmap2::Mmap::map(&file) } { + Ok(mmap) => mmap, + Err(err) => { + eprintln!("Failed to map file '{}': {}", file_path, err); + continue; + } + }; + let file = match object::File::parse(&*file) { + Ok(file) => file, + Err(err) => { + eprintln!("Failed to parse file '{}': {}", file_path, err); + continue; + } + }; + + let endian = if file.is_little_endian() { + gimli::RunTimeEndian::Little + } else { + gimli::RunTimeEndian::Big + }; + let ret = dump_file(&file, endian, &flags); + match ret { + Ok(_) => (), + Err(err) => eprintln!("Failed to dump '{}': {}", file_path, err,), + } + } +} + +fn empty_file_section<'input, 'arena, Endian: gimli::Endianity>( + endian: Endian, + arena_relocations: &'arena Arena, +) -> Relocate<'arena, gimli::EndianSlice<'arena, Endian>> { + let reader = gimli::EndianSlice::new(&[], endian); + let section = reader; + let relocations = RelocationMap::default(); + let relocations = (*arena_relocations.alloc(relocations)).borrow(); + Relocate { + relocations, + section, + reader, + } +} + +fn load_file_section<'input, 'arena, Endian: gimli::Endianity>( + id: gimli::SectionId, + file: &object::File<'input>, + endian: Endian, + is_dwo: bool, + arena_data: &'arena Arena>, + arena_relocations: &'arena Arena, +) -> Result>> { + let mut relocations = RelocationMap::default(); + let name = if is_dwo { + id.dwo_name() + } else { + Some(id.name()) + }; + + let data = match name.and_then(|name| file.section_by_name(&name)) { + Some(ref section) => { + // DWO sections never have relocations, so don't bother. + if !is_dwo { + add_relocations(&mut relocations, file, section); + } + section.uncompressed_data()? + } + // Use a non-zero capacity so that `ReaderOffsetId`s are unique. + None => Cow::Owned(Vec::with_capacity(1)), + }; + let data_ref = (*arena_data.alloc(data)).borrow(); + let reader = gimli::EndianSlice::new(data_ref, endian); + let section = reader; + let relocations = (*arena_relocations.alloc(relocations)).borrow(); + Ok(Relocate { + relocations, + section, + reader, + }) +} + +fn dump_file(file: &object::File, endian: Endian, flags: &Flags) -> Result<()> +where + Endian: gimli::Endianity + Send + Sync, +{ + let arena_data = Arena::new(); + let arena_relocations = Arena::new(); + + let dwo_parent = if let Some(dwo_parent_file) = flags.dwo_parent.as_ref() { + let mut load_dwo_parent_section = |id: gimli::SectionId| -> Result<_> { + load_file_section( + id, + dwo_parent_file, + endian, + false, + &arena_data, + &arena_relocations, + ) + }; + Some(gimli::Dwarf::load(&mut load_dwo_parent_section)?) + } else { + None + }; + let dwo_parent = dwo_parent.as_ref(); + + let dwo_parent_units = if let Some(dwo_parent) = dwo_parent { + Some( + match dwo_parent + .units() + .map(|unit_header| dwo_parent.unit(unit_header)) + .filter_map(|unit| Ok(unit.dwo_id.map(|dwo_id| (dwo_id, unit)))) + .collect() + { + Ok(units) => units, + Err(err) => { + eprintln!("Failed to process --dwo-parent units: {}", err); + return Ok(()); + } + }, + ) + } else { + None + }; + let dwo_parent_units = dwo_parent_units.as_ref(); + + let mut load_section = |id: gimli::SectionId| -> Result<_> { + load_file_section( + id, + file, + endian, + flags.dwo || flags.dwp, + &arena_data, + &arena_relocations, + ) + }; + + let w = &mut BufWriter::new(io::stdout()); + if flags.dwp { + let empty = empty_file_section(endian, &arena_relocations); + let dwp = gimli::DwarfPackage::load(&mut load_section, empty)?; + dump_dwp(w, &dwp, dwo_parent.unwrap(), dwo_parent_units, flags)?; + w.flush()?; + return Ok(()); + } + + let mut dwarf = gimli::Dwarf::load(&mut load_section)?; + if flags.dwo { + dwarf.file_type = gimli::DwarfFileType::Dwo; + if let Some(dwo_parent) = dwo_parent { + dwarf.debug_addr = dwo_parent.debug_addr.clone(); + dwarf + .ranges + .set_debug_ranges(dwo_parent.ranges.debug_ranges().clone()); + } + } + + if let Some(sup_file) = flags.sup.as_ref() { + let mut load_sup_section = |id: gimli::SectionId| -> Result<_> { + // Note: we really only need the `.debug_str` section, + // but for now we load them all. + load_file_section(id, sup_file, endian, false, &arena_data, &arena_relocations) + }; + dwarf.load_sup(&mut load_sup_section)?; + } + + if flags.eh_frame { + let eh_frame = gimli::EhFrame::load(&mut load_section).unwrap(); + dump_eh_frame(w, file, eh_frame)?; + } + if flags.info { + dump_info(w, &dwarf, dwo_parent_units, flags)?; + dump_types(w, &dwarf, dwo_parent_units, flags)?; + } + if flags.line { + dump_line(w, &dwarf)?; + } + if flags.pubnames { + let debug_pubnames = &gimli::Section::load(&mut load_section).unwrap(); + dump_pubnames(w, debug_pubnames, &dwarf.debug_info)?; + } + if flags.aranges { + let debug_aranges = &gimli::Section::load(&mut load_section).unwrap(); + dump_aranges(w, debug_aranges)?; + } + if flags.pubtypes { + let debug_pubtypes = &gimli::Section::load(&mut load_section).unwrap(); + dump_pubtypes(w, debug_pubtypes, &dwarf.debug_info)?; + } + w.flush()?; + Ok(()) +} + +fn dump_eh_frame( + w: &mut W, + file: &object::File, + mut eh_frame: gimli::EhFrame, +) -> Result<()> { + // TODO: this might be better based on the file format. + let address_size = file + .architecture() + .address_size() + .map(|w| w.bytes()) + .unwrap_or(mem::size_of::() as u8); + eh_frame.set_address_size(address_size); + + fn register_name_none(_: gimli::Register) -> Option<&'static str> { + None + } + let arch_register_name = match file.architecture() { + object::Architecture::Arm | object::Architecture::Aarch64 => gimli::Arm::register_name, + object::Architecture::I386 => gimli::X86::register_name, + object::Architecture::X86_64 => gimli::X86_64::register_name, + _ => register_name_none, + }; + let register_name = &|register| match arch_register_name(register) { + Some(name) => Cow::Borrowed(name), + None => Cow::Owned(format!("{}", register.0)), + }; + + let mut bases = gimli::BaseAddresses::default(); + if let Some(section) = file.section_by_name(".eh_frame_hdr") { + bases = bases.set_eh_frame_hdr(section.address()); + } + if let Some(section) = file.section_by_name(".eh_frame") { + bases = bases.set_eh_frame(section.address()); + } + if let Some(section) = file.section_by_name(".text") { + bases = bases.set_text(section.address()); + } + if let Some(section) = file.section_by_name(".got") { + bases = bases.set_got(section.address()); + } + + // TODO: Print "__eh_frame" here on macOS, and more generally use the + // section that we're actually looking at, which is what the canonical + // dwarfdump does. + writeln!( + w, + "Exception handling frame information for section .eh_frame" + )?; + + let mut cies = HashMap::new(); + + let mut entries = eh_frame.entries(&bases); + loop { + match entries.next()? { + None => return Ok(()), + Some(gimli::CieOrFde::Cie(cie)) => { + writeln!(w)?; + writeln!(w, "{:#010x}: CIE", cie.offset())?; + writeln!(w, " length: {:#010x}", cie.entry_len())?; + // TODO: CIE_id + writeln!(w, " version: {:#04x}", cie.version())?; + // TODO: augmentation + writeln!(w, " code_align: {}", cie.code_alignment_factor())?; + writeln!(w, " data_align: {}", cie.data_alignment_factor())?; + writeln!( + w, + " ra_register: {}", + register_name(cie.return_address_register()) + )?; + if let Some(encoding) = cie.lsda_encoding() { + writeln!( + w, + " lsda_encoding: {}/{}", + encoding.application(), + encoding.format() + )?; + } + if let Some((encoding, personality)) = cie.personality_with_encoding() { + write!( + w, + " personality: {}/{} ", + encoding.application(), + encoding.format() + )?; + dump_pointer(w, personality)?; + writeln!(w)?; + } + if let Some(encoding) = cie.fde_address_encoding() { + writeln!( + w, + " fde_encoding: {}/{}", + encoding.application(), + encoding.format() + )?; + } + let instructions = cie.instructions(&eh_frame, &bases); + dump_cfi_instructions(w, instructions, true, register_name)?; + writeln!(w)?; + } + Some(gimli::CieOrFde::Fde(partial)) => { + let mut offset = None; + let fde = partial.parse(|_, bases, o| { + offset = Some(o); + cies.entry(o) + .or_insert_with(|| eh_frame.cie_from_offset(bases, o)) + .clone() + })?; + + writeln!(w)?; + writeln!(w, "{:#010x}: FDE", fde.offset())?; + writeln!(w, " length: {:#010x}", fde.entry_len())?; + writeln!(w, " CIE_pointer: {:#010x}", offset.unwrap().0)?; + // TODO: symbolicate the start address like the canonical dwarfdump does. + writeln!(w, " start_addr: {:#018x}", fde.initial_address())?; + writeln!( + w, + " range_size: {:#018x} (end_addr = {:#018x})", + fde.len(), + fde.initial_address() + fde.len() + )?; + if let Some(lsda) = fde.lsda() { + write!(w, " lsda: ")?; + dump_pointer(w, lsda)?; + writeln!(w)?; + } + let instructions = fde.instructions(&eh_frame, &bases); + dump_cfi_instructions(w, instructions, false, register_name)?; + writeln!(w)?; + } + } + } +} + +fn dump_pointer(w: &mut W, p: gimli::Pointer) -> Result<()> { + match p { + gimli::Pointer::Direct(p) => { + write!(w, "{:#018x}", p)?; + } + gimli::Pointer::Indirect(p) => { + write!(w, "({:#018x})", p)?; + } + } + Ok(()) +} + +#[allow(clippy::unneeded_field_pattern)] +fn dump_cfi_instructions( + w: &mut W, + mut insns: gimli::CallFrameInstructionIter, + is_initial: bool, + register_name: &dyn Fn(gimli::Register) -> Cow<'static, str>, +) -> Result<()> { + use gimli::CallFrameInstruction::*; + + // TODO: we need to actually evaluate these instructions as we iterate them + // so we can print the initialized state for CIEs, and each unwind row's + // registers for FDEs. + // + // TODO: We should print DWARF expressions for the CFI instructions that + // embed DWARF expressions within themselves. + + if !is_initial { + writeln!(w, " Instructions:")?; + } + + loop { + match insns.next() { + Err(e) => { + writeln!(w, "Failed to decode CFI instruction: {}", e)?; + return Ok(()); + } + Ok(None) => { + if is_initial { + writeln!(w, " Instructions: Init State:")?; + } + return Ok(()); + } + Ok(Some(op)) => match op { + SetLoc { address } => { + writeln!(w, " DW_CFA_set_loc ({:#x})", address)?; + } + AdvanceLoc { delta } => { + writeln!(w, " DW_CFA_advance_loc ({})", delta)?; + } + DefCfa { register, offset } => { + writeln!( + w, + " DW_CFA_def_cfa ({}, {})", + register_name(register), + offset + )?; + } + DefCfaSf { + register, + factored_offset, + } => { + writeln!( + w, + " DW_CFA_def_cfa_sf ({}, {})", + register_name(register), + factored_offset + )?; + } + DefCfaRegister { register } => { + writeln!( + w, + " DW_CFA_def_cfa_register ({})", + register_name(register) + )?; + } + DefCfaOffset { offset } => { + writeln!(w, " DW_CFA_def_cfa_offset ({})", offset)?; + } + DefCfaOffsetSf { factored_offset } => { + writeln!( + w, + " DW_CFA_def_cfa_offset_sf ({})", + factored_offset + )?; + } + DefCfaExpression { expression: _ } => { + writeln!(w, " DW_CFA_def_cfa_expression (...)")?; + } + Undefined { register } => { + writeln!( + w, + " DW_CFA_undefined ({})", + register_name(register) + )?; + } + SameValue { register } => { + writeln!( + w, + " DW_CFA_same_value ({})", + register_name(register) + )?; + } + Offset { + register, + factored_offset, + } => { + writeln!( + w, + " DW_CFA_offset ({}, {})", + register_name(register), + factored_offset + )?; + } + OffsetExtendedSf { + register, + factored_offset, + } => { + writeln!( + w, + " DW_CFA_offset_extended_sf ({}, {})", + register_name(register), + factored_offset + )?; + } + ValOffset { + register, + factored_offset, + } => { + writeln!( + w, + " DW_CFA_val_offset ({}, {})", + register_name(register), + factored_offset + )?; + } + ValOffsetSf { + register, + factored_offset, + } => { + writeln!( + w, + " DW_CFA_val_offset_sf ({}, {})", + register_name(register), + factored_offset + )?; + } + Register { + dest_register, + src_register, + } => { + writeln!( + w, + " DW_CFA_register ({}, {})", + register_name(dest_register), + register_name(src_register) + )?; + } + Expression { + register, + expression: _, + } => { + writeln!( + w, + " DW_CFA_expression ({}, ...)", + register_name(register) + )?; + } + ValExpression { + register, + expression: _, + } => { + writeln!( + w, + " DW_CFA_val_expression ({}, ...)", + register_name(register) + )?; + } + Restore { register } => { + writeln!( + w, + " DW_CFA_restore ({})", + register_name(register) + )?; + } + RememberState => { + writeln!(w, " DW_CFA_remember_state")?; + } + RestoreState => { + writeln!(w, " DW_CFA_restore_state")?; + } + ArgsSize { size } => { + writeln!(w, " DW_CFA_GNU_args_size ({})", size)?; + } + Nop => { + writeln!(w, " DW_CFA_nop")?; + } + }, + } + } +} + +fn dump_dwp( + w: &mut W, + dwp: &gimli::DwarfPackage, + dwo_parent: &gimli::Dwarf, + dwo_parent_units: Option<&HashMap>>, + flags: &Flags, +) -> Result<()> +where + R::Endian: Send + Sync, +{ + if dwp.cu_index.unit_count() != 0 { + writeln!( + w, + "\n.debug_cu_index: version = {}, sections = {}, units = {}, slots = {}", + dwp.cu_index.version(), + dwp.cu_index.section_count(), + dwp.cu_index.unit_count(), + dwp.cu_index.slot_count(), + )?; + for i in 1..=dwp.cu_index.unit_count() { + writeln!(w, "\nCU index {}", i)?; + dump_dwp_sections( + w, + &dwp, + dwo_parent, + dwo_parent_units, + flags, + dwp.cu_index.sections(i)?, + )?; + } + } + + if dwp.tu_index.unit_count() != 0 { + writeln!( + w, + "\n.debug_tu_index: version = {}, sections = {}, units = {}, slots = {}", + dwp.tu_index.version(), + dwp.tu_index.section_count(), + dwp.tu_index.unit_count(), + dwp.tu_index.slot_count(), + )?; + for i in 1..=dwp.tu_index.unit_count() { + writeln!(w, "\nTU index {}", i)?; + dump_dwp_sections( + w, + &dwp, + dwo_parent, + dwo_parent_units, + flags, + dwp.tu_index.sections(i)?, + )?; + } + } + + Ok(()) +} + +fn dump_dwp_sections( + w: &mut W, + dwp: &gimli::DwarfPackage, + dwo_parent: &gimli::Dwarf, + dwo_parent_units: Option<&HashMap>>, + flags: &Flags, + sections: gimli::UnitIndexSectionIterator, +) -> Result<()> +where + R::Endian: Send + Sync, +{ + for section in sections.clone() { + writeln!( + w, + " {}: offset = 0x{:x}, size = 0x{:x}", + section.section.dwo_name().unwrap(), + section.offset, + section.size + )?; + } + let dwarf = dwp.sections(sections, dwo_parent)?; + if flags.info { + dump_info(w, &dwarf, dwo_parent_units, flags)?; + dump_types(w, &dwarf, dwo_parent_units, flags)?; + } + if flags.line { + dump_line(w, &dwarf)?; + } + Ok(()) +} + +fn dump_info( + w: &mut W, + dwarf: &gimli::Dwarf, + dwo_parent_units: Option<&HashMap>>, + flags: &Flags, +) -> Result<()> +where + R::Endian: Send + Sync, +{ + writeln!(w, "\n.debug_info")?; + + let units = match dwarf.units().collect::>() { + Ok(units) => units, + Err(err) => { + writeln_error( + w, + dwarf, + Error::GimliError(err), + "Failed to read unit headers", + )?; + return Ok(()); + } + }; + let process_unit = |header: UnitHeader, buf: &mut Vec| -> Result<()> { + dump_unit(buf, header, dwarf, dwo_parent_units, flags)?; + if !flags + .match_units + .as_ref() + .map(|r| r.is_match(&buf)) + .unwrap_or(true) + { + buf.clear(); + } + Ok(()) + }; + // Don't use more than 16 cores even if available. No point in soaking hundreds + // of cores if you happen to have them. + parallel_output(w, 16, units, process_unit) +} + +fn dump_types( + w: &mut W, + dwarf: &gimli::Dwarf, + dwo_parent_units: Option<&HashMap>>, + flags: &Flags, +) -> Result<()> { + writeln!(w, "\n.debug_types")?; + + let mut iter = dwarf.type_units(); + while let Some(header) = iter.next()? { + dump_unit(w, header, dwarf, dwo_parent_units, flags)?; + } + Ok(()) +} + +fn dump_unit( + w: &mut W, + header: UnitHeader, + dwarf: &gimli::Dwarf, + dwo_parent_units: Option<&HashMap>>, + flags: &Flags, +) -> Result<()> { + write!(w, "\nUNIT<")?; + match header.offset() { + UnitSectionOffset::DebugInfoOffset(o) => { + write!(w, ".debug_info+0x{:08x}", o.0)?; + } + UnitSectionOffset::DebugTypesOffset(o) => { + write!(w, ".debug_types+0x{:08x}", o.0)?; + } + } + writeln!(w, ">: length = 0x{:x}, format = {:?}, version = {}, address_size = {}, abbrev_offset = 0x{:x}", + header.unit_length(), + header.format(), + header.version(), + header.address_size(), + header.debug_abbrev_offset().0, + )?; + + match header.type_() { + UnitType::Compilation | UnitType::Partial => (), + UnitType::Type { + type_signature, + type_offset, + } + | UnitType::SplitType { + type_signature, + type_offset, + } => { + write!(w, " signature = ")?; + dump_type_signature(w, type_signature)?; + writeln!(w)?; + writeln!(w, " type_offset = 0x{:x}", type_offset.0,)?; + } + UnitType::Skeleton(dwo_id) | UnitType::SplitCompilation(dwo_id) => { + write!(w, " dwo_id = ")?; + writeln!(w, "0x{:016x}", dwo_id.0)?; + } + } + + let mut unit = match dwarf.unit(header) { + Ok(unit) => unit, + Err(err) => { + writeln_error(w, dwarf, err.into(), "Failed to parse unit root entry")?; + return Ok(()); + } + }; + + if let Some(dwo_parent_units) = dwo_parent_units { + if let Some(dwo_id) = unit.dwo_id { + if let Some(parent_unit) = dwo_parent_units.get(&dwo_id) { + unit.copy_relocated_attributes(parent_unit); + } + } + } + + let entries_result = dump_entries(w, unit, dwarf, flags); + if let Err(err) = entries_result { + writeln_error(w, dwarf, err, "Failed to dump entries")?; + } + Ok(()) +} + +fn spaces(buf: &mut String, len: usize) -> &str { + while buf.len() < len { + buf.push(' '); + } + &buf[..len] +} + +// " GOFF=0x{:08x}" adds exactly 16 spaces. +const GOFF_SPACES: usize = 16; + +fn write_offset( + w: &mut W, + unit: &gimli::Unit, + offset: gimli::UnitOffset, + flags: &Flags, +) -> Result<()> { + write!(w, "<0x{:08x}", offset.0)?; + if flags.goff { + let goff = match offset.to_unit_section_offset(unit) { + UnitSectionOffset::DebugInfoOffset(o) => o.0, + UnitSectionOffset::DebugTypesOffset(o) => o.0, + }; + write!(w, " GOFF=0x{:08x}", goff)?; + } + write!(w, ">")?; + Ok(()) +} + +fn dump_entries( + w: &mut W, + unit: gimli::Unit, + dwarf: &gimli::Dwarf, + flags: &Flags, +) -> Result<()> { + let mut spaces_buf = String::new(); + + let mut entries = unit.entries_raw(None)?; + while !entries.is_empty() { + let offset = entries.next_offset(); + let depth = entries.next_depth(); + let abbrev = entries.read_abbreviation()?; + + let mut indent = if depth >= 0 { + depth as usize * 2 + 2 + } else { + 2 + }; + write!(w, "<{}{}>", if depth < 10 { " " } else { "" }, depth)?; + write_offset(w, &unit, offset, flags)?; + writeln!( + w, + "{}{}", + spaces(&mut spaces_buf, indent), + abbrev.map(|x| x.tag()).unwrap_or(gimli::DW_TAG_null) + )?; + + indent += 18; + if flags.goff { + indent += GOFF_SPACES; + } + + for spec in abbrev.map(|x| x.attributes()).unwrap_or(&[]) { + let attr = entries.read_attribute(*spec)?; + w.write_all(spaces(&mut spaces_buf, indent).as_bytes())?; + if let Some(n) = attr.name().static_string() { + let right_padding = 27 - std::cmp::min(27, n.len()); + write!(w, "{}{} ", n, spaces(&mut spaces_buf, right_padding))?; + } else { + write!(w, "{:27} ", attr.name())?; + } + if flags.raw { + writeln!(w, "{:?}", attr.raw_value())?; + } else { + match dump_attr_value(w, &attr, &unit, dwarf) { + Ok(_) => (), + Err(err) => writeln_error(w, dwarf, err, "Failed to dump attribute value")?, + }; + } + } + } + Ok(()) +} + +fn dump_attr_value( + w: &mut W, + attr: &gimli::Attribute, + unit: &gimli::Unit, + dwarf: &gimli::Dwarf, +) -> Result<()> { + let value = attr.value(); + match value { + gimli::AttributeValue::Addr(address) => { + writeln!(w, "0x{:08x}", address)?; + } + gimli::AttributeValue::Block(data) => { + for byte in data.to_slice()?.iter() { + write!(w, "{:02x}", byte)?; + } + writeln!(w)?; + } + gimli::AttributeValue::Data1(_) + | gimli::AttributeValue::Data2(_) + | gimli::AttributeValue::Data4(_) + | gimli::AttributeValue::Data8(_) => { + if let (Some(udata), Some(sdata)) = (attr.udata_value(), attr.sdata_value()) { + if sdata >= 0 { + writeln!(w, "{}", udata)?; + } else { + writeln!(w, "{} ({})", udata, sdata)?; + } + } else { + writeln!(w, "{:?}", value)?; + } + } + gimli::AttributeValue::Sdata(data) => { + match attr.name() { + gimli::DW_AT_data_member_location => { + writeln!(w, "{}", data)?; + } + _ => { + if data >= 0 { + writeln!(w, "0x{:08x}", data)?; + } else { + writeln!(w, "0x{:08x} ({})", data, data)?; + } + } + }; + } + gimli::AttributeValue::Udata(data) => { + match attr.name() { + gimli::DW_AT_high_pc => { + writeln!(w, "{}", data)?; + } + gimli::DW_AT_data_member_location => { + if let Some(sdata) = attr.sdata_value() { + // This is a DW_FORM_data* value. + // libdwarf-dwarfdump displays this as signed too. + if sdata >= 0 { + writeln!(w, "{}", data)?; + } else { + writeln!(w, "{} ({})", data, sdata)?; + } + } else { + writeln!(w, "{}", data)?; + } + } + gimli::DW_AT_lower_bound | gimli::DW_AT_upper_bound => { + writeln!(w, "{}", data)?; + } + _ => { + writeln!(w, "0x{:08x}", data)?; + } + }; + } + gimli::AttributeValue::Exprloc(ref data) => { + if let gimli::AttributeValue::Exprloc(_) = attr.raw_value() { + write!(w, "len 0x{:04x}: ", data.0.len())?; + for byte in data.0.to_slice()?.iter() { + write!(w, "{:02x}", byte)?; + } + write!(w, ": ")?; + } + dump_exprloc(w, unit.encoding(), data)?; + writeln!(w)?; + } + gimli::AttributeValue::Flag(true) => { + writeln!(w, "yes")?; + } + gimli::AttributeValue::Flag(false) => { + writeln!(w, "no")?; + } + gimli::AttributeValue::SecOffset(offset) => { + writeln!(w, "0x{:08x}", offset)?; + } + gimli::AttributeValue::DebugAddrBase(base) => { + writeln!(w, "<.debug_addr+0x{:08x}>", base.0)?; + } + gimli::AttributeValue::DebugAddrIndex(index) => { + write!(w, "(indirect address, index {:#x}): ", index.0)?; + let address = dwarf.address(unit, index)?; + writeln!(w, "0x{:08x}", address)?; + } + gimli::AttributeValue::UnitRef(offset) => { + write!(w, "0x{:08x}", offset.0)?; + match offset.to_unit_section_offset(unit) { + UnitSectionOffset::DebugInfoOffset(goff) => { + write!(w, "<.debug_info+0x{:08x}>", goff.0)?; + } + UnitSectionOffset::DebugTypesOffset(goff) => { + write!(w, "<.debug_types+0x{:08x}>", goff.0)?; + } + } + writeln!(w)?; + } + gimli::AttributeValue::DebugInfoRef(offset) => { + writeln!(w, "<.debug_info+0x{:08x}>", offset.0)?; + } + gimli::AttributeValue::DebugInfoRefSup(offset) => { + writeln!(w, "<.debug_info(sup)+0x{:08x}>", offset.0)?; + } + gimli::AttributeValue::DebugLineRef(offset) => { + writeln!(w, "<.debug_line+0x{:08x}>", offset.0)?; + } + gimli::AttributeValue::LocationListsRef(offset) => { + dump_loc_list(w, offset, unit, dwarf)?; + } + gimli::AttributeValue::DebugLocListsBase(base) => { + writeln!(w, "<.debug_loclists+0x{:08x}>", base.0)?; + } + gimli::AttributeValue::DebugLocListsIndex(index) => { + write!(w, "(indirect location list, index {:#x}): ", index.0)?; + let offset = dwarf.locations_offset(unit, index)?; + dump_loc_list(w, offset, unit, dwarf)?; + } + gimli::AttributeValue::DebugMacinfoRef(offset) => { + writeln!(w, "<.debug_macinfo+0x{:08x}>", offset.0)?; + } + gimli::AttributeValue::DebugMacroRef(offset) => { + writeln!(w, "<.debug_macro+0x{:08x}>", offset.0)?; + } + gimli::AttributeValue::RangeListsRef(offset) => { + let offset = dwarf.ranges_offset_from_raw(unit, offset); + dump_range_list(w, offset, unit, dwarf)?; + } + gimli::AttributeValue::DebugRngListsBase(base) => { + writeln!(w, "<.debug_rnglists+0x{:08x}>", base.0)?; + } + gimli::AttributeValue::DebugRngListsIndex(index) => { + write!(w, "(indirect range list, index {:#x}): ", index.0)?; + let offset = dwarf.ranges_offset(unit, index)?; + dump_range_list(w, offset, unit, dwarf)?; + } + gimli::AttributeValue::DebugTypesRef(signature) => { + dump_type_signature(w, signature)?; + writeln!(w, " ")?; + } + gimli::AttributeValue::DebugStrRef(offset) => { + if let Ok(s) = dwarf.debug_str.get_str(offset) { + writeln!(w, "{}", s.to_string_lossy()?)?; + } else { + writeln!(w, "<.debug_str+0x{:08x}>", offset.0)?; + } + } + gimli::AttributeValue::DebugStrRefSup(offset) => { + if let Some(s) = dwarf + .sup() + .and_then(|sup| sup.debug_str.get_str(offset).ok()) + { + writeln!(w, "{}", s.to_string_lossy()?)?; + } else { + writeln!(w, "<.debug_str(sup)+0x{:08x}>", offset.0)?; + } + } + gimli::AttributeValue::DebugStrOffsetsBase(base) => { + writeln!(w, "<.debug_str_offsets+0x{:08x}>", base.0)?; + } + gimli::AttributeValue::DebugStrOffsetsIndex(index) => { + write!(w, "(indirect string, index {:#x}): ", index.0)?; + let offset = dwarf.debug_str_offsets.get_str_offset( + unit.encoding().format, + unit.str_offsets_base, + index, + )?; + if let Ok(s) = dwarf.debug_str.get_str(offset) { + writeln!(w, "{}", s.to_string_lossy()?)?; + } else { + writeln!(w, "<.debug_str+0x{:08x}>", offset.0)?; + } + } + gimli::AttributeValue::DebugLineStrRef(offset) => { + if let Ok(s) = dwarf.debug_line_str.get_str(offset) { + writeln!(w, "{}", s.to_string_lossy()?)?; + } else { + writeln!(w, "<.debug_line_str=0x{:08x}>", offset.0)?; + } + } + gimli::AttributeValue::String(s) => { + writeln!(w, "{}", s.to_string_lossy()?)?; + } + gimli::AttributeValue::Encoding(value) => { + writeln!(w, "{}", value)?; + } + gimli::AttributeValue::DecimalSign(value) => { + writeln!(w, "{}", value)?; + } + gimli::AttributeValue::Endianity(value) => { + writeln!(w, "{}", value)?; + } + gimli::AttributeValue::Accessibility(value) => { + writeln!(w, "{}", value)?; + } + gimli::AttributeValue::Visibility(value) => { + writeln!(w, "{}", value)?; + } + gimli::AttributeValue::Virtuality(value) => { + writeln!(w, "{}", value)?; + } + gimli::AttributeValue::Language(value) => { + writeln!(w, "{}", value)?; + } + gimli::AttributeValue::AddressClass(value) => { + writeln!(w, "{}", value)?; + } + gimli::AttributeValue::IdentifierCase(value) => { + writeln!(w, "{}", value)?; + } + gimli::AttributeValue::CallingConvention(value) => { + writeln!(w, "{}", value)?; + } + gimli::AttributeValue::Inline(value) => { + writeln!(w, "{}", value)?; + } + gimli::AttributeValue::Ordering(value) => { + writeln!(w, "{}", value)?; + } + gimli::AttributeValue::FileIndex(value) => { + write!(w, "0x{:08x}", value)?; + dump_file_index(w, value, unit, dwarf)?; + writeln!(w)?; + } + gimli::AttributeValue::DwoId(value) => { + writeln!(w, "0x{:016x}", value.0)?; + } + } + + Ok(()) +} + +fn dump_type_signature(w: &mut W, signature: gimli::DebugTypeSignature) -> Result<()> { + write!(w, "0x{:016x}", signature.0)?; + Ok(()) +} + +fn dump_file_index( + w: &mut W, + file_index: u64, + unit: &gimli::Unit, + dwarf: &gimli::Dwarf, +) -> Result<()> { + if file_index == 0 && unit.header.version() <= 4 { + return Ok(()); + } + let header = match unit.line_program { + Some(ref program) => program.header(), + None => return Ok(()), + }; + let file = match header.file(file_index) { + Some(file) => file, + None => { + writeln!(w, "Unable to get header for file {}", file_index)?; + return Ok(()); + } + }; + write!(w, " ")?; + if let Some(directory) = file.directory(header) { + let directory = dwarf.attr_string(unit, directory)?; + let directory = directory.to_string_lossy()?; + if file.directory_index() != 0 && !directory.starts_with('/') { + if let Some(ref comp_dir) = unit.comp_dir { + write!(w, "{}/", comp_dir.to_string_lossy()?,)?; + } + } + write!(w, "{}/", directory)?; + } + write!( + w, + "{}", + dwarf + .attr_string(unit, file.path_name())? + .to_string_lossy()? + )?; + Ok(()) +} + +fn dump_exprloc( + w: &mut W, + encoding: gimli::Encoding, + data: &gimli::Expression, +) -> Result<()> { + let mut pc = data.0.clone(); + let mut space = false; + while pc.len() != 0 { + let pc_clone = pc.clone(); + match gimli::Operation::parse(&mut pc, encoding) { + Ok(op) => { + if space { + write!(w, " ")?; + } else { + space = true; + } + dump_op(w, encoding, pc_clone, op)?; + } + Err(gimli::Error::InvalidExpression(op)) => { + writeln!(w, "WARNING: unsupported operation 0x{:02x}", op.0)?; + return Ok(()); + } + Err(gimli::Error::UnsupportedRegister(register)) => { + writeln!(w, "WARNING: unsupported register {}", register)?; + return Ok(()); + } + Err(gimli::Error::UnexpectedEof(_)) => { + writeln!(w, "WARNING: truncated or malformed expression")?; + return Ok(()); + } + Err(e) => { + writeln!(w, "WARNING: unexpected operation parse error: {}", e)?; + return Ok(()); + } + } + } + Ok(()) +} + +fn dump_op( + w: &mut W, + encoding: gimli::Encoding, + mut pc: R, + op: gimli::Operation, +) -> Result<()> { + let dwop = gimli::DwOp(pc.read_u8()?); + write!(w, "{}", dwop)?; + match op { + gimli::Operation::Deref { + base_type, size, .. + } => { + if dwop == gimli::DW_OP_deref_size || dwop == gimli::DW_OP_xderef_size { + write!(w, " {}", size)?; + } + if base_type != UnitOffset(0) { + write!(w, " type 0x{:08x}", base_type.0)?; + } + } + gimli::Operation::Pick { index } => { + if dwop == gimli::DW_OP_pick { + write!(w, " {}", index)?; + } + } + gimli::Operation::PlusConstant { value } => { + write!(w, " {}", value as i64)?; + } + gimli::Operation::Bra { target } => { + write!(w, " {}", target)?; + } + gimli::Operation::Skip { target } => { + write!(w, " {}", target)?; + } + gimli::Operation::SignedConstant { value } => match dwop { + gimli::DW_OP_const1s + | gimli::DW_OP_const2s + | gimli::DW_OP_const4s + | gimli::DW_OP_const8s + | gimli::DW_OP_consts => { + write!(w, " {}", value)?; + } + _ => {} + }, + gimli::Operation::UnsignedConstant { value } => match dwop { + gimli::DW_OP_const1u + | gimli::DW_OP_const2u + | gimli::DW_OP_const4u + | gimli::DW_OP_const8u + | gimli::DW_OP_constu => { + write!(w, " {}", value)?; + } + _ => { + // These have the value encoded in the operation, eg DW_OP_lit0. + } + }, + gimli::Operation::Register { register } => { + if dwop == gimli::DW_OP_regx { + write!(w, " {}", register.0)?; + } + } + gimli::Operation::RegisterOffset { + register, + offset, + base_type, + } => { + if dwop >= gimli::DW_OP_breg0 && dwop <= gimli::DW_OP_breg31 { + write!(w, "{:+}", offset)?; + } else { + write!(w, " {}", register.0)?; + if offset != 0 { + write!(w, "{:+}", offset)?; + } + if base_type != UnitOffset(0) { + write!(w, " type 0x{:08x}", base_type.0)?; + } + } + } + gimli::Operation::FrameOffset { offset } => { + write!(w, " {}", offset)?; + } + gimli::Operation::Call { offset } => match offset { + gimli::DieReference::UnitRef(gimli::UnitOffset(offset)) => { + write!(w, " 0x{:08x}", offset)?; + } + gimli::DieReference::DebugInfoRef(gimli::DebugInfoOffset(offset)) => { + write!(w, " 0x{:08x}", offset)?; + } + }, + gimli::Operation::Piece { + size_in_bits, + bit_offset: None, + } => { + write!(w, " {}", size_in_bits / 8)?; + } + gimli::Operation::Piece { + size_in_bits, + bit_offset: Some(bit_offset), + } => { + write!(w, " 0x{:08x} offset 0x{:08x}", size_in_bits, bit_offset)?; + } + gimli::Operation::ImplicitValue { data } => { + let data = data.to_slice()?; + write!(w, " 0x{:08x} contents 0x", data.len())?; + for byte in data.iter() { + write!(w, "{:02x}", byte)?; + } + } + gimli::Operation::ImplicitPointer { value, byte_offset } => { + write!(w, " 0x{:08x} {}", value.0, byte_offset)?; + } + gimli::Operation::EntryValue { expression } => { + write!(w, "(")?; + dump_exprloc(w, encoding, &gimli::Expression(expression))?; + write!(w, ")")?; + } + gimli::Operation::ParameterRef { offset } => { + write!(w, " 0x{:08x}", offset.0)?; + } + gimli::Operation::Address { address } => { + write!(w, " 0x{:08x}", address)?; + } + gimli::Operation::AddressIndex { index } => { + write!(w, " 0x{:08x}", index.0)?; + } + gimli::Operation::ConstantIndex { index } => { + write!(w, " 0x{:08x}", index.0)?; + } + gimli::Operation::TypedLiteral { base_type, value } => { + write!(w, " type 0x{:08x} contents 0x", base_type.0)?; + for byte in value.to_slice()?.iter() { + write!(w, "{:02x}", byte)?; + } + } + gimli::Operation::Convert { base_type } => { + write!(w, " type 0x{:08x}", base_type.0)?; + } + gimli::Operation::Reinterpret { base_type } => { + write!(w, " type 0x{:08x}", base_type.0)?; + } + gimli::Operation::WasmLocal { index } + | gimli::Operation::WasmGlobal { index } + | gimli::Operation::WasmStack { index } => { + let wasmop = pc.read_u8()?; + write!(w, " 0x{:x} 0x{:x}", wasmop, index)?; + } + gimli::Operation::Drop + | gimli::Operation::Swap + | gimli::Operation::Rot + | gimli::Operation::Abs + | gimli::Operation::And + | gimli::Operation::Div + | gimli::Operation::Minus + | gimli::Operation::Mod + | gimli::Operation::Mul + | gimli::Operation::Neg + | gimli::Operation::Not + | gimli::Operation::Or + | gimli::Operation::Plus + | gimli::Operation::Shl + | gimli::Operation::Shr + | gimli::Operation::Shra + | gimli::Operation::Xor + | gimli::Operation::Eq + | gimli::Operation::Ge + | gimli::Operation::Gt + | gimli::Operation::Le + | gimli::Operation::Lt + | gimli::Operation::Ne + | gimli::Operation::Nop + | gimli::Operation::PushObjectAddress + | gimli::Operation::TLS + | gimli::Operation::CallFrameCFA + | gimli::Operation::StackValue => {} + }; + Ok(()) +} + +fn dump_loc_list( + w: &mut W, + offset: gimli::LocationListsOffset, + unit: &gimli::Unit, + dwarf: &gimli::Dwarf, +) -> Result<()> { + let raw_locations = dwarf.raw_locations(unit, offset)?; + let raw_locations: Vec<_> = raw_locations.collect()?; + let mut locations = dwarf.locations(unit, offset)?; + writeln!( + w, + "", + if unit.encoding().version < 5 { + ".debug_loc" + } else { + ".debug_loclists" + }, + offset.0, + raw_locations.len() + )?; + for (i, raw) in raw_locations.iter().enumerate() { + write!(w, "\t\t\t[{:2}]", i)?; + match *raw { + gimli::RawLocListEntry::BaseAddress { addr } => { + writeln!(w, "", addr)?; + } + gimli::RawLocListEntry::BaseAddressx { addr } => { + let addr_val = dwarf.address(unit, addr)?; + writeln!(w, "", addr.0, addr_val)?; + } + gimli::RawLocListEntry::StartxEndx { + begin, + end, + ref data, + } => { + let begin_val = dwarf.address(unit, begin)?; + let end_val = dwarf.address(unit, end)?; + let location = locations.next()?.unwrap(); + write!( + w, + "", + begin.0, begin_val, location.range.begin, end.0, end_val, location.range.end + )?; + dump_exprloc(w, unit.encoding(), data)?; + writeln!(w)?; + } + gimli::RawLocListEntry::StartxLength { + begin, + length, + ref data, + } => { + let begin_val = dwarf.address(unit, begin)?; + let location = locations.next()?.unwrap(); + write!( + w, + "", + begin.0, begin_val, location.range.begin, length, location.range.end + )?; + dump_exprloc(w, unit.encoding(), data)?; + writeln!(w)?; + } + gimli::RawLocListEntry::AddressOrOffsetPair { + begin, + end, + ref data, + } + | gimli::RawLocListEntry::OffsetPair { + begin, + end, + ref data, + } => { + let location = locations.next()?.unwrap(); + write!( + w, + "", + begin, location.range.begin, end, location.range.end + )?; + dump_exprloc(w, unit.encoding(), data)?; + writeln!(w)?; + } + gimli::RawLocListEntry::DefaultLocation { ref data } => { + write!(w, "")?; + dump_exprloc(w, unit.encoding(), data)?; + writeln!(w)?; + } + gimli::RawLocListEntry::StartEnd { + begin, + end, + ref data, + } => { + let location = locations.next()?.unwrap(); + write!( + w, + "", + begin, location.range.begin, end, location.range.end + )?; + dump_exprloc(w, unit.encoding(), data)?; + writeln!(w)?; + } + gimli::RawLocListEntry::StartLength { + begin, + length, + ref data, + } => { + let location = locations.next()?.unwrap(); + write!( + w, + "", + begin, location.range.begin, length, location.range.end + )?; + dump_exprloc(w, unit.encoding(), data)?; + writeln!(w)?; + } + }; + } + Ok(()) +} + +fn dump_range_list( + w: &mut W, + offset: gimli::RangeListsOffset, + unit: &gimli::Unit, + dwarf: &gimli::Dwarf, +) -> Result<()> { + let raw_ranges = dwarf.raw_ranges(unit, offset)?; + let raw_ranges: Vec<_> = raw_ranges.collect()?; + let mut ranges = dwarf.ranges(unit, offset)?; + writeln!( + w, + "", + if unit.encoding().version < 5 { + ".debug_ranges" + } else { + ".debug_rnglists" + }, + offset.0, + raw_ranges.len() + )?; + for (i, raw) in raw_ranges.iter().enumerate() { + write!(w, "\t\t\t[{:2}] ", i)?; + match *raw { + gimli::RawRngListEntry::AddressOrOffsetPair { begin, end } => { + let range = ranges.next()?.unwrap(); + writeln!( + w, + "

", + begin, range.begin, end, range.end + )?; + } + gimli::RawRngListEntry::BaseAddress { addr } => { + writeln!(w, "", addr)?; + } + gimli::RawRngListEntry::BaseAddressx { addr } => { + let addr_val = dwarf.address(unit, addr)?; + writeln!(w, "", addr.0, addr_val)?; + } + gimli::RawRngListEntry::StartxEndx { begin, end } => { + let begin_val = dwarf.address(unit, begin)?; + let end_val = dwarf.address(unit, end)?; + let range = if begin_val == end_val { + gimli::Range { + begin: begin_val, + end: end_val, + } + } else { + ranges.next()?.unwrap() + }; + writeln!( + w, + "", + begin.0, begin_val, range.begin, end.0, end_val, range.end + )?; + } + gimli::RawRngListEntry::StartxLength { begin, length } => { + let begin_val = dwarf.address(unit, begin)?; + let range = ranges.next()?.unwrap(); + writeln!( + w, + "", + begin.0, begin_val, range.begin, length, range.end + )?; + } + gimli::RawRngListEntry::OffsetPair { begin, end } => { + let range = ranges.next()?.unwrap(); + writeln!( + w, + "", + begin, range.begin, end, range.end + )?; + } + gimli::RawRngListEntry::StartEnd { begin, end } => { + let range = if begin == end { + gimli::Range { begin, end } + } else { + ranges.next()?.unwrap() + }; + writeln!( + w, + "", + begin, range.begin, end, range.end + )?; + } + gimli::RawRngListEntry::StartLength { begin, length } => { + let range = ranges.next()?.unwrap(); + writeln!( + w, + "", + begin, range.begin, length, range.end + )?; + } + }; + } + Ok(()) +} + +fn dump_line(w: &mut W, dwarf: &gimli::Dwarf) -> Result<()> { + let mut iter = dwarf.units(); + while let Some(header) = iter.next()? { + writeln!( + w, + "\n.debug_line: line number info for unit at .debug_info offset 0x{:08x}", + header.offset().as_debug_info_offset().unwrap().0 + )?; + let unit = match dwarf.unit(header) { + Ok(unit) => unit, + Err(err) => { + writeln_error( + w, + dwarf, + err.into(), + "Failed to parse unit root entry for dump_line", + )?; + continue; + } + }; + match dump_line_program(w, &unit, dwarf) { + Ok(_) => (), + Err(Error::IoError) => return Err(Error::IoError), + Err(err) => writeln_error(w, dwarf, err, "Failed to dump line program")?, + } + } + Ok(()) +} + +fn dump_line_program( + w: &mut W, + unit: &gimli::Unit, + dwarf: &gimli::Dwarf, +) -> Result<()> { + if let Some(program) = unit.line_program.clone() { + { + let header = program.header(); + writeln!(w)?; + writeln!( + w, + "Offset: 0x{:x}", + header.offset().0 + )?; + writeln!( + w, + "Length: {}", + header.unit_length() + )?; + writeln!( + w, + "DWARF version: {}", + header.version() + )?; + writeln!( + w, + "Address size: {}", + header.address_size() + )?; + writeln!( + w, + "Prologue length: {}", + header.header_length() + )?; + writeln!( + w, + "Minimum instruction length: {}", + header.minimum_instruction_length() + )?; + writeln!( + w, + "Maximum operations per instruction: {}", + header.maximum_operations_per_instruction() + )?; + writeln!( + w, + "Default is_stmt: {}", + header.default_is_stmt() + )?; + writeln!( + w, + "Line base: {}", + header.line_base() + )?; + writeln!( + w, + "Line range: {}", + header.line_range() + )?; + writeln!( + w, + "Opcode base: {}", + header.opcode_base() + )?; + + writeln!(w)?; + writeln!(w, "Opcodes:")?; + for (i, length) in header + .standard_opcode_lengths() + .to_slice()? + .iter() + .enumerate() + { + writeln!(w, " Opcode {} has {} args", i + 1, length)?; + } + + let base = if header.version() >= 5 { 0 } else { 1 }; + writeln!(w)?; + writeln!(w, "The Directory Table:")?; + for (i, dir) in header.include_directories().iter().enumerate() { + writeln!( + w, + " {} {}", + base + i, + dwarf.attr_string(unit, dir.clone())?.to_string_lossy()? + )?; + } + + writeln!(w)?; + writeln!(w, "The File Name Table")?; + write!(w, " Entry\tDir\tTime\tSize")?; + if header.file_has_md5() { + write!(w, "\tMD5\t\t\t\t")?; + } + writeln!(w, "\tName")?; + for (i, file) in header.file_names().iter().enumerate() { + write!( + w, + " {}\t{}\t{}\t{}", + base + i, + file.directory_index(), + file.timestamp(), + file.size(), + )?; + if header.file_has_md5() { + let md5 = file.md5(); + write!(w, "\t")?; + for i in 0..16 { + write!(w, "{:02X}", md5[i])?; + } + } + writeln!( + w, + "\t{}", + dwarf + .attr_string(unit, file.path_name())? + .to_string_lossy()? + )?; + } + + writeln!(w)?; + writeln!(w, "Line Number Instructions:")?; + let mut instructions = header.instructions(); + while let Some(instruction) = instructions.next_instruction(header)? { + writeln!(w, " {}", instruction)?; + } + + writeln!(w)?; + writeln!(w, "Line Number Rows:")?; + writeln!(w, " [lno,col]")?; + } + let mut rows = program.rows(); + let mut file_index = std::u64::MAX; + while let Some((header, row)) = rows.next_row()? { + let line = match row.line() { + Some(line) => line.get(), + None => 0, + }; + let column = match row.column() { + gimli::ColumnType::Column(column) => column.get(), + gimli::ColumnType::LeftEdge => 0, + }; + write!(w, "0x{:08x} [{:4},{:2}]", row.address(), line, column)?; + if row.is_stmt() { + write!(w, " NS")?; + } + if row.basic_block() { + write!(w, " BB")?; + } + if row.end_sequence() { + write!(w, " ET")?; + } + if row.prologue_end() { + write!(w, " PE")?; + } + if row.epilogue_begin() { + write!(w, " EB")?; + } + if row.isa() != 0 { + write!(w, " IS={}", row.isa())?; + } + if row.discriminator() != 0 { + write!(w, " DI={}", row.discriminator())?; + } + if file_index != row.file_index() { + file_index = row.file_index(); + if let Some(file) = row.file(header) { + if let Some(directory) = file.directory(header) { + write!( + w, + " uri: \"{}/{}\"", + dwarf.attr_string(unit, directory)?.to_string_lossy()?, + dwarf + .attr_string(unit, file.path_name())? + .to_string_lossy()? + )?; + } else { + write!( + w, + " uri: \"{}\"", + dwarf + .attr_string(unit, file.path_name())? + .to_string_lossy()? + )?; + } + } + } + writeln!(w)?; + } + } + Ok(()) +} + +fn dump_pubnames( + w: &mut W, + debug_pubnames: &gimli::DebugPubNames, + debug_info: &gimli::DebugInfo, +) -> Result<()> { + writeln!(w, "\n.debug_pubnames")?; + + let mut cu_offset; + let mut cu_die_offset = gimli::DebugInfoOffset(0); + let mut prev_cu_offset = None; + let mut pubnames = debug_pubnames.items(); + while let Some(pubname) = pubnames.next()? { + cu_offset = pubname.unit_header_offset(); + if Some(cu_offset) != prev_cu_offset { + let cu = debug_info.header_from_offset(cu_offset)?; + cu_die_offset = gimli::DebugInfoOffset(cu_offset.0 + cu.header_size()); + prev_cu_offset = Some(cu_offset); + } + let die_in_cu = pubname.die_offset(); + let die_in_sect = cu_offset.0 + die_in_cu.0; + writeln!(w, + "global die-in-sect 0x{:08x}, cu-in-sect 0x{:08x}, die-in-cu 0x{:08x}, cu-header-in-sect 0x{:08x} '{}'", + die_in_sect, + cu_die_offset.0, + die_in_cu.0, + cu_offset.0, + pubname.name().to_string_lossy()? + )?; + } + Ok(()) +} + +fn dump_pubtypes( + w: &mut W, + debug_pubtypes: &gimli::DebugPubTypes, + debug_info: &gimli::DebugInfo, +) -> Result<()> { + writeln!(w, "\n.debug_pubtypes")?; + + let mut cu_offset; + let mut cu_die_offset = gimli::DebugInfoOffset(0); + let mut prev_cu_offset = None; + let mut pubtypes = debug_pubtypes.items(); + while let Some(pubtype) = pubtypes.next()? { + cu_offset = pubtype.unit_header_offset(); + if Some(cu_offset) != prev_cu_offset { + let cu = debug_info.header_from_offset(cu_offset)?; + cu_die_offset = gimli::DebugInfoOffset(cu_offset.0 + cu.header_size()); + prev_cu_offset = Some(cu_offset); + } + let die_in_cu = pubtype.die_offset(); + let die_in_sect = cu_offset.0 + die_in_cu.0; + writeln!(w, + "pubtype die-in-sect 0x{:08x}, cu-in-sect 0x{:08x}, die-in-cu 0x{:08x}, cu-header-in-sect 0x{:08x} '{}'", + die_in_sect, + cu_die_offset.0, + die_in_cu.0, + cu_offset.0, + pubtype.name().to_string_lossy()? + )?; + } + Ok(()) +} + +fn dump_aranges( + w: &mut W, + debug_aranges: &gimli::DebugAranges, +) -> Result<()> { + writeln!(w, "\n.debug_aranges")?; + + let mut headers = debug_aranges.headers(); + while let Some(header) = headers.next()? { + writeln!( + w, + "Address Range Header: length = 0x{:08x}, version = 0x{:04x}, cu_offset = 0x{:08x}, addr_size = 0x{:02x}, seg_size = 0x{:02x}", + header.length(), + header.encoding().version, + header.debug_info_offset().0, + header.encoding().address_size, + header.segment_size(), + )?; + let mut aranges = header.entries(); + while let Some(arange) = aranges.next()? { + let range = arange.range(); + if let Some(segment) = arange.segment() { + writeln!( + w, + "[0x{:016x}, 0x{:016x}) segment 0x{:x}", + range.begin, range.end, segment + )?; + } else { + writeln!(w, "[0x{:016x}, 0x{:016x})", range.begin, range.end)?; + } + } + } + Ok(()) +} diff --git a/crux-mir/lib/gimli/examples/simple.rs b/crux-mir/lib/gimli/examples/simple.rs new file mode 100644 index 000000000..7c958d45c --- /dev/null +++ b/crux-mir/lib/gimli/examples/simple.rs @@ -0,0 +1,67 @@ +//! A simple example of parsing `.debug_info`. + +use object::{Object, ObjectSection}; +use std::{borrow, env, fs}; + +fn main() { + for path in env::args().skip(1) { + let file = fs::File::open(&path).unwrap(); + let mmap = unsafe { memmap2::Mmap::map(&file).unwrap() }; + let object = object::File::parse(&*mmap).unwrap(); + let endian = if object.is_little_endian() { + gimli::RunTimeEndian::Little + } else { + gimli::RunTimeEndian::Big + }; + dump_file(&object, endian).unwrap(); + } +} + +fn dump_file(object: &object::File, endian: gimli::RunTimeEndian) -> Result<(), gimli::Error> { + // Load a section and return as `Cow<[u8]>`. + let load_section = |id: gimli::SectionId| -> Result, gimli::Error> { + match object.section_by_name(id.name()) { + Some(ref section) => Ok(section + .uncompressed_data() + .unwrap_or(borrow::Cow::Borrowed(&[][..]))), + None => Ok(borrow::Cow::Borrowed(&[][..])), + } + }; + + // Load all of the sections. + let dwarf_cow = gimli::Dwarf::load(&load_section)?; + + // Borrow a `Cow<[u8]>` to create an `EndianSlice`. + let borrow_section: &dyn for<'a> Fn( + &'a borrow::Cow<[u8]>, + ) -> gimli::EndianSlice<'a, gimli::RunTimeEndian> = + &|section| gimli::EndianSlice::new(&*section, endian); + + // Create `EndianSlice`s for all of the sections. + let dwarf = dwarf_cow.borrow(&borrow_section); + + // Iterate over the compilation units. + let mut iter = dwarf.units(); + while let Some(header) = iter.next()? { + println!( + "Unit at <.debug_info+0x{:x}>", + header.offset().as_debug_info_offset().unwrap().0 + ); + let unit = dwarf.unit(header)?; + + // Iterate over the Debugging Information Entries (DIEs) in the unit. + let mut depth = 0; + let mut entries = unit.entries(); + while let Some((delta_depth, entry)) = entries.next_dfs()? { + depth += delta_depth; + println!("<{}><{:x}> {}", depth, entry.offset().0, entry.tag()); + + // Iterate over the attributes in the DIE. + let mut attrs = entry.attrs(); + while let Some(attr) = attrs.next()? { + println!(" {}: {:?}", attr.name(), attr.value()); + } + } + } + Ok(()) +} diff --git a/crux-mir/lib/gimli/examples/simple_line.rs b/crux-mir/lib/gimli/examples/simple_line.rs new file mode 100644 index 000000000..87b224cda --- /dev/null +++ b/crux-mir/lib/gimli/examples/simple_line.rs @@ -0,0 +1,106 @@ +//! A simple example of parsing `.debug_line`. + +use object::{Object, ObjectSection}; +use std::{borrow, env, fs, path}; + +fn main() { + for path in env::args().skip(1) { + let file = fs::File::open(&path).unwrap(); + let mmap = unsafe { memmap2::Mmap::map(&file).unwrap() }; + let object = object::File::parse(&*mmap).unwrap(); + let endian = if object.is_little_endian() { + gimli::RunTimeEndian::Little + } else { + gimli::RunTimeEndian::Big + }; + dump_file(&object, endian).unwrap(); + } +} + +fn dump_file(object: &object::File, endian: gimli::RunTimeEndian) -> Result<(), gimli::Error> { + // Load a section and return as `Cow<[u8]>`. + let load_section = |id: gimli::SectionId| -> Result, gimli::Error> { + match object.section_by_name(id.name()) { + Some(ref section) => Ok(section + .uncompressed_data() + .unwrap_or(borrow::Cow::Borrowed(&[][..]))), + None => Ok(borrow::Cow::Borrowed(&[][..])), + } + }; + + // Load all of the sections. + let dwarf_cow = gimli::Dwarf::load(&load_section)?; + + // Borrow a `Cow<[u8]>` to create an `EndianSlice`. + let borrow_section: &dyn for<'a> Fn( + &'a borrow::Cow<[u8]>, + ) -> gimli::EndianSlice<'a, gimli::RunTimeEndian> = + &|section| gimli::EndianSlice::new(&*section, endian); + + // Create `EndianSlice`s for all of the sections. + let dwarf = dwarf_cow.borrow(&borrow_section); + + // Iterate over the compilation units. + let mut iter = dwarf.units(); + while let Some(header) = iter.next()? { + println!( + "Line number info for unit at <.debug_info+0x{:x}>", + header.offset().as_debug_info_offset().unwrap().0 + ); + let unit = dwarf.unit(header)?; + + // Get the line program for the compilation unit. + if let Some(program) = unit.line_program.clone() { + let comp_dir = if let Some(ref dir) = unit.comp_dir { + path::PathBuf::from(dir.to_string_lossy().into_owned()) + } else { + path::PathBuf::new() + }; + + // Iterate over the line program rows. + let mut rows = program.rows(); + while let Some((header, row)) = rows.next_row()? { + if row.end_sequence() { + // End of sequence indicates a possible gap in addresses. + println!("{:x} end-sequence", row.address()); + } else { + // Determine the path. Real applications should cache this for performance. + let mut path = path::PathBuf::new(); + if let Some(file) = row.file(header) { + path = comp_dir.clone(); + + // The directory index 0 is defined to correspond to the compilation unit directory. + if file.directory_index() != 0 { + if let Some(dir) = file.directory(header) { + path.push( + dwarf.attr_string(&unit, dir)?.to_string_lossy().as_ref(), + ); + } + } + + path.push( + dwarf + .attr_string(&unit, file.path_name())? + .to_string_lossy() + .as_ref(), + ); + } + + // Determine line/column. DWARF line/column is never 0, so we use that + // but other applications may want to display this differently. + let line = match row.line() { + Some(line) => line.get(), + None => 0, + }; + let column = match row.column() { + gimli::ColumnType::LeftEdge => 0, + gimli::ColumnType::Column(column) => column.get(), + }; + + println!("{:x} {}:{}:{}", row.address(), path.display(), line, column); + } + } + } + } + Ok(()) +} diff --git a/crux-mir/lib/gimli/fixtures/self/README.md b/crux-mir/lib/gimli/fixtures/self/README.md new file mode 100644 index 000000000..91053d9b4 --- /dev/null +++ b/crux-mir/lib/gimli/fixtures/self/README.md @@ -0,0 +1,147 @@ +# What are these files? + +These files are the DWARF data generated for (an early version of) this +library. Each file corresponds is a section from the built library's object +file. By splitting the sections out to their own files, we don't need to worry +about cross platform and cross object file format issues when running examples. + +# Updating and adding new sections + +## macOS + +Use `otool` to list the sections of a binary: + +``` +$ otool -l path/to/binary +``` + +You should see output similar to this: + +``` +Load command 0 + cmd LC_SEGMENT_64 + cmdsize 72 + segname __PAGEZERO + vmaddr 0x0000000000000000 + vmsize 0x0000000100000000 + fileoff 0 + filesize 0 + maxprot 0x00000000 + initprot 0x00000000 + nsects 0 + flags 0x0 +Load command 1 + cmd LC_SEGMENT_64 + cmdsize 712 + segname __TEXT + vmaddr 0x0000000100000000 + vmsize 0x00000000001b7000 + fileoff 0 + filesize 1798144 + maxprot 0x00000007 + initprot 0x00000005 + nsects 8 + flags 0x0 +Section + sectname __text + segname __TEXT + addr 0x0000000100000a50 + size 0x0000000000170716 + offset 2640 + align 2^4 (16) + reloff 0 + nreloc 0 + flags 0x80000400 + reserved1 0 + reserved2 0 +``` + +Etc. + +Find the `Section` entry of the section you'd like to isolate. For example, if +you're looking for `eh_frame`, find an entry like this: + +``` +Section + sectname __eh_frame + segname __TEXT + addr 0x0000000100192f38 + size 0x00000000000240c8 + offset 1650488 + align 2^3 (8) + reloff 0 + nreloc 0 + flags 0x00000000 + reserved1 0 + reserved2 0 +``` + +Then use `dd` to copy `size` bytes starting from `offset`: + +``` +$ dd bs=1 skip=1650488 count=$(printf "%d" 0x00000000000240c8) if=path/to/binary of=fixtures/self/eh_frame +``` + +Finally, use `otool` and `hexdump` to verify that the isolated section has the +same data as the section within the binary: + +``` +$ otool -s __TEXT __eh_frame path/to/binary | head +path/to/binary: +Contents of (__TEXT,__eh_frame) section +0000000100192f38 14 00 00 00 00 00 00 00 01 7a 52 00 01 78 10 01 +0000000100192f48 10 0c 07 08 90 01 00 00 24 00 00 00 1c 00 00 00 +0000000100192f58 f8 da e6 ff ff ff ff ff 66 00 00 00 00 00 00 00 +0000000100192f68 00 41 0e 10 86 02 43 0d 06 00 00 00 00 00 00 00 +0000000100192f78 1c 00 00 00 00 00 00 00 01 7a 50 4c 52 00 01 78 +0000000100192f88 10 07 9b 9d 40 02 00 10 10 0c 07 08 90 01 00 00 +0000000100192f98 2c 00 00 00 24 00 00 00 20 db e6 ff ff ff ff ff +0000000100192fa8 8d 00 00 00 00 00 00 00 08 37 e7 fd ff ff ff ff + +$ otool -s __TEXT __eh_frame path/to/binary | tail +00000001001b6f68 9a 0a 00 00 00 00 00 00 00 41 0e 10 86 02 43 0d +00000001001b6f78 06 50 83 07 8c 06 8d 05 8e 04 8f 03 00 00 00 00 +00000001001b6f88 24 00 00 00 7c 0e 00 00 30 a0 fb ff ff ff ff ff +00000001001b6f98 15 00 00 00 00 00 00 00 00 41 0e 10 86 02 43 0d +00000001001b6fa8 06 00 00 00 00 00 00 00 24 00 00 00 a4 0e 00 00 +00000001001b6fb8 28 a0 fb ff ff ff ff ff 1c 00 00 00 00 00 00 00 +00000001001b6fc8 00 41 0e 10 86 02 43 0d 06 00 00 00 00 00 00 00 +00000001001b6fd8 24 00 00 00 cc 0e 00 00 20 a0 fb ff ff ff ff ff +00000001001b6fe8 66 01 00 00 00 00 00 00 00 41 0e 10 86 02 43 0d +00000001001b6ff8 06 00 00 00 00 00 00 00 +``` + +This should be the same, ignoring the leading offsets: + +``` +$ hexdump fixtures/self/eh_frame | head +0000000 14 00 00 00 00 00 00 00 01 7a 52 00 01 78 10 01 +0000010 10 0c 07 08 90 01 00 00 24 00 00 00 1c 00 00 00 +0000020 f8 da e6 ff ff ff ff ff 66 00 00 00 00 00 00 00 +0000030 00 41 0e 10 86 02 43 0d 06 00 00 00 00 00 00 00 +0000040 1c 00 00 00 00 00 00 00 01 7a 50 4c 52 00 01 78 +0000050 10 07 9b 9d 40 02 00 10 10 0c 07 08 90 01 00 00 +0000060 2c 00 00 00 24 00 00 00 20 db e6 ff ff ff ff ff +0000070 8d 00 00 00 00 00 00 00 08 37 e7 fd ff ff ff ff +0000080 ff 41 0e 10 86 02 43 0d 06 00 00 00 00 00 00 00 +0000090 24 00 00 00 94 00 00 00 80 db e6 ff ff ff ff ff + +$ hexdump fixtures/self/eh_frame | tail +0024040 06 50 83 07 8c 06 8d 05 8e 04 8f 03 00 00 00 00 +0024050 24 00 00 00 7c 0e 00 00 30 a0 fb ff ff ff ff ff +0024060 15 00 00 00 00 00 00 00 00 41 0e 10 86 02 43 0d +0024070 06 00 00 00 00 00 00 00 24 00 00 00 a4 0e 00 00 +0024080 28 a0 fb ff ff ff ff ff 1c 00 00 00 00 00 00 00 +0024090 00 41 0e 10 86 02 43 0d 06 00 00 00 00 00 00 00 +00240a0 24 00 00 00 cc 0e 00 00 20 a0 fb ff ff ff ff ff +00240b0 66 01 00 00 00 00 00 00 00 41 0e 10 86 02 43 0d +00240c0 06 00 00 00 00 00 00 00 +``` + +## Linux + +Something like this: + +``` +objcopy --dump-section .eh_frame=fixtures/self/eh_frame path/to/binary +``` diff --git a/crux-mir/lib/gimli/fixtures/self/debug_abbrev b/crux-mir/lib/gimli/fixtures/self/debug_abbrev new file mode 100644 index 0000000000000000000000000000000000000000..809e61152ce70f2b54aaa304cdd9e2365bbd32fc GIT binary patch literal 1865 zcmZuxcV81h5Ph3)QL`z8GXy0lAk7v;qckPhjlD&|-aGc*!1nWV`c0g-d$*Sy{B=9C zQ{HJd;b6IH9fzR-n1A z6aZSE6$XUU?FgE-%V^%+eS zHVsI3dmM-XCW%2mkZnjSV0al4#*Fm9VUdcPE2?RXwM?)0w!{-P6Sqc@MsP-M^Q0kH zgMx48u7pN~lp85ac$5;6@Qyg)K>e|za0PaH!X4PvLXEwvAtDyp&1)X^G{+fCzW6Y+PyYlT>Sf*pU=O=4Ce*Rv_1f!#`yrVL*}!;Q$Ch8 zaDc@|rPM)lkVzkM4@#Hwz~RVD$mSfe%T39Eqjte*aI22F!cwMg0zJ7CFbi^uFW>c;s_A zGRbGIVi&+0&eVOb$9%>9cbwx269|?PUJY(_7=A4ZXZH2zp5~2k2W$l<0XO9-rJ|Cz zXmT}f%f^|*9S`L^J$F6Xd*EI(M!W9!BNzKTi1*-=6#{3k|xBNcG-IVW3`0j0S&W`S{(S$=8zs1uzmuWDVG$V9q>arl|+`GDr#Z}^GmrA7$U!En;HC3U9$T44^51|jQ{`u literal 0 HcmV?d00001 diff --git a/crux-mir/lib/gimli/fixtures/self/debug_aranges b/crux-mir/lib/gimli/fixtures/self/debug_aranges new file mode 100644 index 0000000000000000000000000000000000000000..b2d983d78ab02ee12746fad33850c81aa35c1b54 GIT binary patch literal 16304 zcmZYG3E16KoyYO`mKKUo(z2AT5L&jflq;)%NT95xK-z%;hb)e1q7tp$Pz z9cYoT$R?DF3Od6U1gR)6VH~TtOrtC#DiG9X9t4#+H^1-yYBG5qxcBq^{^#UmJ2$x( zmTv8NBi>+Ht@*b<-t#`<-c4!bc_DrW_a-R+sO9(qT;LmUyte8$z!~1k>hb!x!nS^Y z9kriK9%9QA>{i<>8Fs^NmVT$c!f%n6>$bP2!fTM%*wz=Ur}}rvgNg0-Jv@#)#MVB+ z!|~!xsPALjUnSmla8TS5x?u43&!ZZ8=_Gkq(JSzLtKU@lO7b9(x4;peh5L9sF7a=1 zv6lHzH&kKM4ErN-OiV20Ye)iz6lSK#On<+i^vd^>qK zQ~6!E#@>Gjhsg^>FFstp8GDgz`#Z#Ykf(Tr{xWyq2XF0q=wsWTC3dTA zmSDE(-Al8ic&*WLc9e3vUP|mvlV%Wg} zJT*R%{^Da*zYXoPE( zf0pWZ!Wn)Mdx`RGaj)YTk0%e@Kn6pz^FPP7J@IFi|C;T|&zH|+d;AOK9es5eT_xX2 zUSBO+|H(r6;mNA+Et6-hFK5rmE62#S+ps}bn=_vUKb9Y%|7wM7$3OT<+cDl;-FX>4 zkNM00nd+C};urF6Y_C_zHh=Z-k=7r!^Eb!Ok|)1Xd#ewAE!+8%V0T;7EIHl{R~-kh zs=Zwg5uQSQir2$Aw(F_D?qjN1qSw^lavZ!a+xCRmy);Yj4dw1KZk7z&{Z{%v%0ES3 z;Zb%z{6_hjBNY5jJ_JVC)*s{Pd*S` z;mOt?-U?@UAr9VF`{D7zJ6k=Ti7UK1t~>VJjvnmF`Q$0S1pBq}**L+rKXN=gAF#EL z-c|bpX`f^FG2ARQw(SZ2pnALh;*N7Xf#YBNQT6uu@BK-BXpH&~K9FZqpZ-NYbQR^L z+xkKDQPHb9t~>TVqIx&2HA{-8;XWRZ1NT8b=zq#6{UI7H&$fDeF7Ay{{yO6gR--+6 zgzb7v@J;0YSk=2}s99?K&p2`)bc3$lG}|mCw)4^ZsPetYL%hPqv!-%8pL%#K^*Off z^Tw%uxIa1$@dVn(_$Az1OYLpGjj>&C8MgDUkLS`p@YQ}A5N?y_i>c#`sM$uqo|3$Qm;x$UnC z+xc8diU6()Py=${*)?_I8pF zrhSBM{NB#W?feSxy?kDkyDGQqqsHsfpTC=OyPgufJ?-<|m7i+$d&qV_7h${p`}kwj z*VyK-U{AHboxH?X;`kHFZNAR%b>zWb%I)|@9jEvktExe`zv@q+eIHM<`T1bwc74~_ zeT~s9{tV@IzQuSW^0ea|zrKbFvZK`gG}h-GD-Vwso<)Bpw)x0APW5k)*VwLS?|9|E z=Xlkzdb~`#!e$D?3a5ZOwR!vt|1{4ifn?`tM_#ud8{=-N%B)dyf2` z?ay;%n}7WC7#EaUU8l5Kn$zMuZGh05*zJ6t5c zPaa+?PsjCj@)caa>0)^t$0NI0-jVf}*tR#lMY*}RL|#IFH*l4uJBP=)Ziq-QIUA9z@o@ zz*A^nuTXyHYU;0CDL>5zaQSAZzu&eUb)0ovcI^MI(;jce=A`&$9QcV+whivBFGTu&YJ{gGyUHh*~I<$3gTXOy76XoAhpRF(7M}4@VJk{E7B!7u|Z)4e>UqgJU zjSmmUw~6X)eg3BMz4TuM@;kKeZzkLNtI4vhKbaz1|HbCAZC~x)47xT4Q`?TVZ98>0 zN`v|j8Be-{{5azYd-A5_@yFVEu%m3_Np_O0zhoEL=G$;rc_RJC*!oX*Q~o@8^l{np zWOw;F8{Z!7Jl<1Y$#}!PB?zk>1d)}$BU4Qz=$Ln!^7fY3&fqP$-?Rh4{0sGUtP5B1wkKlIs5nSCVyW6m4iN7Y#pnV^2i7UJ- z4(?X{>)5|X-U5esZ=7QLKBC6M=L7d3I2elA$8)Sbw*3|LRev1Y)BlG2r0vgtm-qG6 zLGc55FI&MY@}oR&7q7`1(7yVe{5JLF`|bAr*ip@RCsE(?<%g{OMA_CCY$@Mm_Yc$L z89eX=`^gK~-WddP`)pDf)B#MfyxiF98cu_B|Aj9ZGMjTp}xlU{YW%Z^()DH zhsy3Trde`#V>Wne*JFj3k!O+e?PyN9Mghrw*sKOU%{I!9j0`qIzI zlX3rCnQ|{UPj=I2vm_VD6REE+l&9d}^YU2STOcn|bFaX5zm$JL`Ca7EMec!b z4)J_kVC%2K50Gb>+W(a2k>o4#d*sQD@*6n5Np_Es%~CIx?Ra|sDo>@phkt?no0Xr6 zQ*8Uc#CE?~+@g9rKkAMH_n-4%58p+@k``A_nF^?Bsj3-{diTlJUxII;-oIb@#jEHK*)rL# z_qyZgA?0@cq}aBn#9P<_eN6Q>|Ht?w=C86)Zs&9SU3nbiOR$~KIlkDo_Zih6%<&Ju zC)@E(@OP;XpHpt*$?>G$Mj~2 zek0p>GHl}s-&DSo^;fn0OWGHIkpE<#Pw&Ya;_9#R^{hYrn{4|d@>Xf>j|>mj{}JWx zHn&--QS!_5SF9r2`5TOu?fRu`_%hSI8}Y`tua9&Cl~6yQif~o6x9dIWc-Rqd2IujJ z_cOOU)L(YmYR&f8{NKm!YoBJx&QQJ2moR>_0D`&-P|7%G=Yv!0u;{W(od7`7|7r z@`D_2??>{_IbhW*vb*hQmgJZ6c=GI3c~AN)Uz4wAJjt8#OSZjl$*-)*cs`Kr{O#f8 zHlDvJpH2Vqhw|oX?q%58=XgKrYj13GKDeKGo5dfIC)xQmPJWHgn_xrPuGa|L$5D*! zdQ7p6r{Bp-Y{$dfNd3FV=4Qz!$?OiVkB?^m`ID9ZFXyZOG1>hL-7LKwGE>=FZP$+&lb%R9Vov*eK13IUxPPGeTZzIfBNg+|E>A* zzSBo>J=HT+zcc;y4wq-qB*XT5>ECICs~k_BHjdQdpg)_BYdC zdWQT3&Oh%g`84w6Y}wAI9NYJe-e;8C{cw(NWPM?x++1~>&THqvIc*mm$DdVh->((; z>#Q&NobqKjIJfPxii%u5P=)c0MEvm79Y_ZI>OV*R=Z!uWj4AuI-}ZG;84zPIi4>+&hAuf`|g_yOhS65H|U zJ*a%R{n+aLW$pGU9niw)4B_xa`!254XqD$HVp=mmOCf*ByJ` zQ2*AS-*M1!uj8=e2p_}o=y&XYQ|q%l>A1whA6yI;x>yMOgVcWMtp2_n>{}=7{1-^zgM!!`4P3-@-`~>}% zE9I^EzGpa4-Yj*mBi`tHY`osd1N>vVVXu{c=nTZKT6#Hi|oGU zY?jzty?K5-hw)US9 z;n#R=n)<7%f6D{B26+#!i=$2+cbwqi@yFw7pJUseKDPT&%7!ls+`AF)i3zqn{%RWU zvs6@g9*)N<|Fd~@+4fh`af&BU-+kH6{b9trbUj;tPCchv1 zG@f(mAlyKn$N5rhEPwg`^{hY4Z2Eu4eSNJC<4LNw=bK~`c@FKXj=SCJd|Q3K^%rfb z_9yavTD_C}&7j*--q>9SBi+#QnL9u#`bwv z;`t-$FgjlKyK((xC&OF;6Jgw!9~i?=Ksfw7RyWMug0Tn|J_`TL1t6 literal 0 HcmV?d00001 diff --git a/crux-mir/lib/gimli/fixtures/self/debug_info b/crux-mir/lib/gimli/fixtures/self/debug_info new file mode 100644 index 0000000000000000000000000000000000000000..aa430a5cea05f6d06984fc11be24d79ce198c77a GIT binary patch literal 392832 zcmcG1d0bT0`~SVe0K*l$GpM*xQl?p%VP$GrVx?&-ZKI|ZpWJ9u%yMTbTQJM8MQNv1 z+Of>AtjIR9jWQGaSYlaGnNgog+JEooJkMRu%z)p2e!O0sGiRRdeV*s+cbULe(+C*! z7cxzx*~TDMGK}Mqha-kT-{$u=jKBnZNcs;FGb31tEbzFQT#Ed>&+vU}T2jN*48?pi zzDwSSB(wtrD=n$KkO`adU5XX^o+)WbhEU>+l=(!A;7FJX0=g zcmo7aq(4Ne+Y)K;i5S7m6qp<6Gj|kZn}y{V5H)kHLMR8p2sV)HXx)|JQuf%-v7%K% z7?SEodOED-v?WsO6ETAGAs9H@hjct-8-(Q%5H+MtLf8WW29pqE&)1Q*3ddB?G|_Hl zT!qI$@O(N2)OMF{7Rjg4p$YQY8E zxjt1!@OF>`hNXK?I96+-v{n9A(yNKZ8Chz$r{3zxLVY8+RQLpl;GJ;*ObPr6|QPc`YE@)nZ0g6^3%p`(~0z*-K zTN2@N_O~tp+>L$-KkjBBJgON7?tVU1M(|t0y+p^ICmg}d7~Ew-IDrU$-1%)uR3?-F z?gGDrA9s-uiZ$cFUE@=Q(E;SZ<2vqo;b_!E;qTBA_FmUcF}NGslE_IY0o=`g2|w;u zA)HAz{G-DVpDH7`9OS@zI_^5*ctR5eZsP{V{e3L%hPEWSC6oZ}CclIqcZ(4EbdJHD z>r-U}r-B^#MaNwv9A%m)aMuXo6(acCNm*MG^$8__yTUKw$6Y0a-!&$pY@< zblhdaaU^IO?iwM?C4!5ayyutjqo@+XbDAoksQ0Ndg1-}r3v?7s!qL5J42tlL?7c}u z@S|w)OZZV3B5}8-3Ml&dR2jh?AP35H6nVmt)GY=@nGnVh!9{_wHNP#1%7hZ|UV&f2 zkGn_+H)zIzyT+%=2!051;2|A%y>R@ZiP9K#6MOHN?lHI<+mgshC;{BfehEMBRv{FV z4Q-6_sWO6ZfE;*J$DJ=6&6+51R|w&-9x=EJ+LEYFC;{9>ehEMBG9k<&8-Cn%K2=6= z9ms)Z9e1N}Y|=!@-^lIWID0=S#}5`Nq*LikQIj$W4QQ)L7X+6TB#({Ya$juSxhLIvFALb!?ue%#~Q zlBh~30o;Xt2|w<6LU=(l4&1dqRTx_d_e>pkgK%`+HwJgOg8e<72!7m6ZAtV?C;{9p zehEKrLnKyc#(_J}r^*O!1v#)n$30Fse%D0d?@A#Y+baflVOtV42_=Ago?pU`yIcs1 z$cDdP*85Z$!B;^J)atmKgky&$3fwu%+24I~VsN*#B{3wS1aKQdP&WLy!$O!sHvG8r zeX5M$qaX)9)o~XJ$D5ica90Z7~2*>@JC~)WA!v5Y7i`x)FoW$sa62KkyOZai;2qAm_7~BOuRYq_U$bnuNS}&U? z91Ap2;I0+Ii$o|jysIz;lrANPH5oGp8U~K^ z45LF}`REGz?(vLspM-jA_Za4W+LLN7zKuAZ&v@u4Fq|L+A46TSQu};HFED`RWnuAI ze!ln)((z)((7wcyQFn-8+=Xv&Ci(yoUe{w+@3;>%Eq3`UICftX!Hpdv(dd`(N21^D zEYSsqJynQApyalm;=DV~Cm~(t=#IgY z5zoMJ9rZ3#B&=tFs8h;)6eD;Q314c5^4{RqK235nx(t*ZL@ly!MTbDAD#|)g74Q1Q z^tIbik%%1rrqdr?(OJ6aEL#^vo9{s?AGW;~jMh~p2nT}ZHB`j0MhKUJ5X!A&bLn?D zW*@-U3h|S!S=Ak%j5QnljjWCdYgW76!2#^hFUIP)JDG4I5&Ur)(w0O~TN2~ik|=LW zVqRMkRc%RBwk1*9mPBn^5)ExhG`1zt+?GUZTN1`yZFskvsEUh6H@}3xr{%Uq80{k% z!Fh;d;7&dG1;SAk8~h=6GrV0I0sTF{EkdD>U<8ji2*x()#>#|a5@>q+7Q*8i0mkav zA~g93M(`Ihwk)DG(pKS!92{e8$UR)Mvq11J;oTa+W?m?9BKbZMBe)ELfy#&{dke><+=Myo4pHhup)@xKG9D8DGR3n5T{bOoW)|NzlTM`v*Ni_K+yc)fQ8dbF=(&`g2 zf>!`s;9XyQKZ0!dK90o-LcS59bncVN5yCbQjNm?pLU!LwFIt#vhX}`T(31U=ZLJWl zC4xV`^?nJzu}~GGco!00-G*mk_0yI}j!(qsc=+dZC)fZab|<8U-A8>Mgfw8Ej5NtU z7irL*f;8DKLYiVPLYitXK^n3jM4D!=Lu%ReNITd&kfz(eBFzx}Nr#bs3#nme(r2UN zf+tMF?19e#`(Q;qQhh!SX_7q#Y0w^zG}*ogX~-@{YT1jCcCaguX4or{+V+D;!}e;V z5&I>inRY$WPWC5AyVzeL&9Z+$+ST5Jw40rgYZ%?_E=YUW`yuUV4@8=6AB%J!`wXP} z+EJvv>}g1I>={V+vlk-W-@XQEZ+ivO1MK^d_OTyB+Sgu#^g#P1r2XuAqzBm_B0bn{ zM0$w518IMIH_}7x-;oZmgIH-FX7@mvYY#*^&>n{LaC;=uBkVJg9%-M4bdWtA=~4C^ zq=W6nNQcy$R`X`!l3t%C0i)9?1XwH%a2NA4;Y@6schkqt6nbJbVt=rz>hyeZBx`l05}!ntd744)z?R z>2?{?47(g@*j|n_Vy{G+Y2S;qlf4RQ7yC)1S#~YbuJ&t4yV)C&cDJ`6?O}g|w5R>>Nmu(u)m$gp=J+ho|g zkbP{}zaiUf*ncDY#IQqy3}Xu-jjYkIvyp8@uSWK%VGl(1U&B5Y**3#I8QEusosaBu z!=8w2yJ1g9)@0aMBHLlu#mK%e?4`)QH0;}u?KJEMkbPy?Payl+u-78{#;{*R)@;~s zAp6#^KS1`KVSkS7d*DR23pkPeVAuw>guA6F!js$-4QYz$AfkDyl=hBEc3-4H`%t9G z_7O-^?4d|g?cqp6_9&!j_NhoM`)s5g?D0s`?emdl*i(>pv@b$x+cS}d?YT%J_9CR2 z_Ektb+1DZMY%fLH#l8(`mVFP>uJ$UV-R!54cDG+dnj;8b!)HclDC?&E5xxm1i%H5> zP{}4MBq>T;s=Wm~A$vR0Gl zH8}pOG4WrGkN;{4z6xh9Mfw!YXuXY46*?N7Ak;~2M(+jnzxWSaV)g+OZFWciMLG=d zHt8WpWVNb~aq-0bWeYdrtS2d05iQLmvyyfoi$rc1>stfxkDU*(=8JtWwh1fSHa z)~V%G7ChxN&dun7V7rJ|U(^A-r~!EKUJZZ|sf0*4b0bo5xDjbUAvg1Hq!QxHw4)7! zX2U@5P-X<5chUN(cQ~^TKGWCBq z#b=yi!8|HH>lDW=r{eRf=r`_!WT1H37n<8r9_7YUKu!wWmO_3)HO?nL1#V9vSD~0s zt_rM3A#b605_v1IGKCz5;wS~YQUOm(T3%C1P7BVRrn8VS1?GHPv5r3%H|!#vSTGTqmM)Gdy?I$ zFne})#>j?p7x6BCz2s)pJ{Gt3<3Es|Iv6f<&8DXkWU4_iQ8QBStJi8!CC3zF-*V0k zmV)nN;%!c$lMW9>vqAw?g^}YFIkP_o*#d46<4z%mee1+LzZADZsvbPJK1k=RsOTIL4qP83c@(cCc@#;7(bbHm4#%8K zU)}6*a5KC)xaxHb*x)!297^amB?KMJaT{|Ok(0pCFY_Eqc{0fykI$>sC=$w?gwI12 zb}RE@d_KHJ4ZfkOpBeiz|Kr?;Jtdh|k7S6H;u#9*nKMXlAT^m(gL)aM4uq0PJBsz$63iwbn%%vEBUUER59;jrjjeI=I$)lN8~#Mmp7KA%p#6+lhYm` z<5?{Tp87D?u`w6)6~r?s`I@8gCF;U1oRmyG97PV$B;{Z%UuebBt;da69-^i8cwF+0 zl7A~Xts9AcpyZ<@pDOt^l0PW<2FZ6wPRlyT8xdN(b<4aH2&bv3%<#?hAtOR7wsy$0 zm}-Yi>$-59^3hML`O}EdQZJs&9RRk~nl4V77a<_i(lHQ6rY05WZ5k0;7q+9EmU!`G z8YwZg8xdOj#gqS4E&`brh4JJ!9Wt#JjWA=X|d~=r}35sTM91?Y9tZp zonu4>f;F6Z9#Uxup{C1uQDDq=76J3Fka!$$}eGy>lZsJ{g9q>!LoQ=+Ecb34+Am|h3H__Crz z%}LRY)Fjed_{;j@a>6%O)rTaD6`Tt?J_Ee?^HxJy;l~)+%}+V*xEZ1!lKA$HuMhXT zW4m%kdJa@zuB$3dQJ3s?bPOWG=~&|toXTQjBt&<0d=0Kgxp5KsKJdG;Y<(-nf{%+Q zla@D^P&*HNp(<%u9fb0P$I(=hPAb3Z_$M|G*0cl+#hsaf8o6G*P{E6AX+xb;z3$+> zfcP)9f5_trS1nsFwJF%?Wn8;Tsoe}))sB`ws7-Up5|W&0)Af0j%NN~FG5}0joIRFe(UW#LAS)|tFSJ*Vuz>Q5}XrXdrxl)Ypy?_~!yI?Xe^Hrp@ z*TSToS&z>=OXg;7!DpU#hJ>2_W_z>#;oP@T+Zh>v4Ga2ZJ}C<9jL5!iH0cU_6_M3p z9;pj_9U+ZA3`y#F+@4*+5sWH;cZB3-^h@xkj)F`!ZS~wudNws-qf1R^=HDc_84ZK? zNaF8x;11uWz8CYibYcEZS)7~EW5N3$;@_VTy88E5{08GXuS-K$=Kr&sBX==mmXh3g z)ZIu<1^v968VFZjcjg6^P^O_M$rzX>8e_UKaspAb^rB5H#hBq zbF5E=5bD9&!BC+w*g_cPF*KL>HJm1dYdnTJjbW(}Hh2t`8pGQ{FnYQeT8jN9QiX7w z$55{^j1q19t(&=~#|!co~S$ZCz@SRq{DF_dWx3x)8s z$I!UYk72D4=>Ab^xrjxLp8;ozb?7kqBauPUgQ*J)sxy&|?N4uk0QMDs3T2n;scFF00CXJ-S<`@j_R%txF zptDCeu%~GdmKXMOu)7V}HmN0(P1W*U2;7uyQ?=#OMD8ic0l{y=ilz4qEYI_~7ZBJR zoy?H2KSMZEa-TOk>E7sMuJC(E3W5 zus1rfH$D^j<3!%|hCqm=x8;qlXd9y9eiw?{HoEHG=xT1qBHe8pT(qW$-0dC|ZX4aC zZN$^r8%!r})CkL+2gG@!tL_aK0(+w?d*e`%TPC^d4S^6#Z_67!%}|x7_{oLhdZVZA zjh^Ox{g>`mh(iliZBDJ*B7a4Hp7?qbGZ#SmZvJ z-1UY)h^4pXjYG^(i>MgU*9na4jYD*A9AZ}cy}?B*C~|`&cY8{I_C~LGI(viZ&Y%2shyyV21BGlW3 zipAH8Qne|3!dwlXpj=iu1_FVP)TIxUn@Y~1@+NgXBRw486+)jTfE(>GSzk~ZY}$EmmE5ofXoQV&FD(#{DkzrNAoM`86$ZvAa z(X7E|vT=bjxnC;96ehPUP#2%}OQnlX0ETUDki$}_Q(JF{g)$Mq%3!qWsZDQgMUpm~$-VQ@R!xC4|O z82ADwrclp^jXc$hPgJhLHR9S*rSn8}-M2%^^1_tfMvCeeCn^|Th>ff=QTvpuz6I*D zPKEj_Qq%^I7T-$v3Uz5y_B1P!Lp36@+Z80;OTI?s);RI3<)XH#U5Dx}#wk0Bj@>i`A81fZ&;Pv2!Jsx0zhD z9uWDLMc!R5UY|nSOce9KBV{+F+zemSO72%GmZ!Xjd5^+pKtW(>%IjPl*MW*lQfSu# zYF-u@U3y)T=cKwv6w&ROa)L!_1=dj!Ykw=gG{|erUHHaQw zCg()f59klRz9Ow*L2{@`qB!vkr%@(qJz*7{w_p_Gj_K!C0s8|e+2<)naanagOnRdZQShxv}>p|CoIEO!V{ovEj&DtBPRYenGqc_N6# z98b~diUGB2;Y%{<0rG*>RTu|Kj;38HQ*@FxfI4EBN!WKgL zec)w3%I_KFG!fV;0`Q&qa+ia04=;Ci1!yI1y(I!WB=@1@mGpN)2#7{7#GvG($Ug$1 z7Gb$`se{%XBmW44taSO*AUg2&Lbz;~=} zT_}V*Jce40p-Ko_JcbI5;d3E$xzokaJl}63TL@=+40RgAc|ut3F;r>{l|uNyV`!P@ zH}Q!OGVXFQ)N2f#g>Z_;P^B@PC4?m&L+f0>i3%aSO#j9z%`B@TL&{_87`FhTuI6?ik6@d7G~En>bzw^F4-IjiF2k z&wC6N8pF#%_|;=*zQS+fFCiRpuj{Bfjp1k^%<>p2HHLXYc-muVne8{RRtP_M4D}kr zFG4uvJ{Lok#&Eb0F7g;!FZY`$62dBvp+RGKN(f(h4AmOLE+O=;axs)?38LN)MyNk2;no2pK4zlnSyT<45Nim>M_)44A%+a zC6A$8V|ZN%e|ii}m-tNt9%gVuC8za+#*in3xgJA>#!w=J=RAhyi~T0n3*mnrL!HL( zhY$un;yS8QV;CZYnI1#SMSc@=gz%)tP_Hqp5yJN#LzTwxvk(qiNSSjgz%2XP^B?^B!slbT@0<0{U*XfILTvZ&=^h^!nGblwZ?Fx z5MJ{b$~1jK4~wz6uIctMid%B~WDS0%>?qHkq43gHWnA>sC`PcT#=C}+Os zpmldceemN3j8+{{^$Aw{w_kmNYB#h;EMZt@uP-Oz19c*kSVcS9cuA? zLEjCXE`)1627NblqYz&681&uHMj@CRT@3neC?tg89)rFc8YP5ck3ruJT_c1SJqCR@ zv_S~Jdkp$+2v6mzv-QD}qw(syq2q*brN^M}h879oS&u>A4ZR?QpF9SAH`FSG!`^ir zrSFE062fI3gT5QOLI{t04Ek>986kY_F=)FXoVK+{m+QQlQJB%wFi2cU1I3BKLJSn% zIp2vvHBzL!&zv89%(*vGj8qKXNHH>~MvCboaFyh7BgG9uc)(*wI8u}bLpcI-^Cu2m zccduQM~YJQpp<)1Q_3SnvdFzHISy+0!1QW0u+X`UI;XuVNXI-@EIZCuNApO`=j|A2 zE4DC_lusQ*H^6x>k^|ofzT_Wi-(mkfA_Bi{jrHHVj{kluQ4fc`s~!%sQX83ZKgoUm zyNQlBt%HSdyvLyV&sr=5ze&E9|DI!VPqBL1>^=3g8HKQTD^=lr^+22Tizvtu%mhKM z0Y9N2943U5J%*dLON;PSSUmLdHpf$+!$jwNnAsFuAMwpeo7hu*Mc^qB@LJl(4(t~8 z)W_V?t`~v(CHHxXk5?ZP!Um5)_ten;GB&?S_S9C_BztNrn_MLVnER%5MZkrZ;0a*)Do79CrRULibU94j z+20(jg$q^EFG{B)LyQ>gd11`z=camBw#_p|WpSdaLAS^exBUw~oEh${tI?d|szBf#zEW ze8~8PRXqI%=Vr7E7>*&PLzAD!v!FEasE8e^F8ij}a3EUv^1uM~CA`wimj?#$Z4Qe? z>phaguNB(mfk%Y!vd3_}|MGwp_?7WY7hEvW!m&QgF}b@~J$-qYdiv6uBLWXdP9`PR zXQ-aNS_toY4A=X6wsr7t@h181j$Fs&kBijfo4M+tQR`k2*eE%fl$$wbD3e=-@QcTA zy}x;-H?cDt7pfo)bj;K(RF8lTR1c$~=9LRC&O<;BR}Z#YFYIBQe@fmnh`aVuXi1xt zl!gvuSb43?@r>jE_U}UN;tN`BDCbY+y8JKB&FJX}`+N%h;ACwW8?1m?M~Ogz2mqN} z*%;y=mtkxOH-nWTuv&89(3jRP2;n1-K^w*}uh(3KsT*+nm2-}}SAnpWP)<%Zvv^+T z7CKp>NDn8gF(ogPd^SZr0h^vj-_b>8HGE7@{tbfZW&C9&k12)fOE5Ka&@Xc8RJTZZ z|DxnGj=`96+BJ-0ndIdCV)6dPxM|X`HVNf~YnhY0@4r_oy@~7wm)9;zQHp1%u%e0> zNu*vOV+qSnm)wlLiyEtbKRubuxFgAQ1ryWvQkM8Za-W3M%s-M8vJ{Ubg~_43Wh}k; zX3ov1fqM#+y+XD}_baGY%A&XvrL{zjC4~=qH8yV6lI0!mJRPw*>sR;LR7c7{+f z>@QNcwZ4>G)_`b)Q&fm5*-D*z4_E0dC<)Z57KP~iTcU0ctV>q62mZR8l}^5ubNITo zm~a1l@dI^BYIzCY{z*k^-7SL5V{cx>w|_o~=blM=5#RnvML6IoVLae=SAUhq9e-cMJ!`s)@je!4>i0u2iozj`hr zSJ>ux>^1XQ|JTf3r*5Q8GO38a^gQ-zeTlkz^$m3=on=S}(j|wP>Un(kYT+g&XJqHz z#faX)A|ef zgIOnx1H>tCYpov4dVeslxu0RYD>+3$4`yvlFssD3U#nh0b!mRpMFW6hVjWda1~3rbQi+k4B>5I*erP>2#)EsxZQ($RTL292N$i0 zV){v;x?bp|>J>qfB0i(8rhZMT44aIutMO3s_ zKE`a@B?rB_n1_@g4_URvJS0*PL)L?$x4VFZK!fxmy7vghCh0}$#<-cdrJCfPmmKt3 z=|%lmf4%e~D&jBQNL3R-Gq>`d>Yh?-xL7;Ivs$x&o7hjzkX47p52={2hI2D|1Dq?r z7HNGc)U$*N=a#7W?w2x~;yYtCOO-ssIp&%QZsmO#LRbjdlA|V7(#jcv<%qf_l9pe` zHu|xR+-F(0kD-zujp)xdb_zp_oue%B8#YJc==_5{73bkE+b{$~eh}1} zDAce}VT#-3IovM)j-}J;PL1&6*-a0;!fKzA99YVw?K^JD&JpD=VWUf)Goyys@?>VotluLL0J(D5y6u|=}Pvq|^q3y1B zAp8Uf^6RM9dZB(LR6yWiV1XJ2(p$LZukhSatX>6Xef1`*I#YyTwn_X=cSxhyB69~7 ztyS2FP^5p99P|e1gIU}Mo1_m?k@{c`T7x&RmZ+Q1t+`@tv1hej`rt#-2eTXAVN<`O z?{nkobuOOzve3AWNwszfwMD3?EH6igrTzpB>G^C{o~Rm~N{_@@U%$tyCW;WySITlz zo_?z?=CwB!t!Lh6_CF*CdzmaZ<>|KyS#DC1DqF@@%eW_rvH#BZf&4u6TxJr$=T(pw6u-vsG4`=aB-`i8Q8ydN2 zeYcI-GqBO2s-msdl&*yH17D=l&C)1WmsWXY!`eqU^E`&h{iK zSmFmqqMRkTjMuUV=Y5_;8B4UJ4o5J6sKk3P_eos!_P zVSO%~sh_(wZ(@l(4ys0$;4OK{n+z7fu_(?n?ZqI{JW zqL&vyqMRkTjF=1OC{Ln{CDKFmfIr2fSb3Mr*uMzpB2S{FSlI}Nq+K<$1eekDLJh*Z z{9BHQ@44;msoKTlxTdd`8?8+&ckK>|C=Q=}P1%a2qdPbF-=tw!H;QEUFI)|yl?HCg zada9IbqzTp`J|^IPif$$ZN<`|p~nBJ4Z~U^k|Vwp473VTD!8r3F+fF=|4kW&HCjYB zdn%+;bK{R=fQkxT#WoSWXs3&!no)3XuwogYqRxNkt6>$1=--|S87R1W#4$ici~r76 z!wP=I)|X38ZJAMUe~Dv&iYi@2rHJT}7RUp7xE4LIyPMVpc2zRMhzIk2S1wM0BU8qSfCA;~1c#$$xvRVSO*63!7aO zGDvbKjAMX`3QYx0Y&&3@2y_XB%D!hM-FI>BdkILFkoHQHF6u>dn5g(!H7;5YihP>L z)2yC$5_ESB(G^^px75RF-9j|YS?|QN^J6+p2RlEf!_qXI#H=oo)_BF6<@+h z^WNk#J?~DFySf&29={$hD_gsQq&u3+FmQ^MvwrWRy?;d9RT5>3CoX2Nna+cMsPF|Bt=Pav;YgxwY z-)GWV_65utVr_wEwP7xwvE89?p||FXmk-0+JP4{8or+jIjQ_x*5NSbQ7^nPMtlqzv zUPYDy%g3s_AFc|~3wqNV*m^ZL@N1oxv#R6^(sQk1vihg7g)<~arRud?BF-1We2?K3 z|AkYW*kK!n*fn4BBk7%ycFvL5hUD^>wU?-OBNwGn2Tm{GFRL$68`Rb*iTi${f zshO%4xRiC$`hMhC)Pl>}1J*YwJJz+Lv_*2%yqtfX?#2+cT3=;CMP1l%epVcS+hPFR z<^edF0B~6Yuz~?>7NvQj)B|vP9Du5>Z1{$5aRBa#0dP-Lm97 ztcwGXa}XOg4vquxY7Br^V*qej1MnIHxK@<5NbUjH5CgH)Ui7fdv0q!Nn!q+SziakCXZxE)=V77~_A7ZD`Yg_V!9cc~eR!PxuXOug#n}HU zM2{Y8_P>z;h)!E{diKALvtM)s+s!{R&i?M00PJ=E$S}lZ4M2-HK$PZ*QZE2M#se6{ zhL3cA%D|lo_UI1SD50BXQ-LX!?ANmW zqvPX0jP1`CU7twq**_-Ee)t5o|E}ad`zPr3PjKQd_D_hnKa%ZF6G|slW8U)8qYrCfgq)x}KLD_G`G$o{>iH%STy$wm3f?&nU+6`blwk z#_4#*Ie4T$j*Iul1i>S^)=2K*86Sh^ym&mL1<#l`JX3T$Q)2K;iN`Zt@QAK8l6!cj zrqOGbJ^xIL$8##={hTm6SxPG`}#&WNi@iC&cwrz#Ta zlK4Wd z@Z1oK2e%5!vVCX|2hhcE7kLm)ZUU0O{S|Ax%Gw!93%p;CU5Y%L@}WJHJ?UY z5>l_w>)D+(50V^vO8pI9BZM%~V`%FQUcs#lEL7B!mHsNZ8T}3vwVAs#^d4>IoGJ8D zX9|~l$hqt_5jnLxBNQ^SS?6HM&8WM{JcoB9&L;C5HRtZ`~yV&q?lc<1WqN|)J=Bp=b4hpF=p;&3esH)AoG>ma< z>h%j8sI%b0ujqvBg)}Q$s4of?ruc+yeOfEdg$T$!bi(#>%n92YF(+&r)fkap!DlV4 z@_>x`5=~A{*xpm)f|WCYk*t`=IpCMc-CT0QR^dNk+a>HrdhE^e3RpQ|y zGl}(XJm1ydte>#8&gYBl%9}aDR_L)e=qGGVF(+(wTuV7&+sr3yEh7Gxr=~vUge|*4 zynNeqMrKB<5Q~>G;NZ{FtX)D~DpWTX+x1xNh>1mWOe}ujShQ%dXo-nM3&*16LPj!e zitDTPVzGWQvjsi&_G59tR2H~%nybIPShNWH5RZK?vB>G5Wr+`-aMp2L{ z17c!v1jnLAi^Wkfu{erjaq;Dhq-nP6tM+1Xl(3a}?Cr`rlNO8PI2K2XwNakcxLAb6%SSI_lV-I4d3bMg7_k^-S^b4NP^fS|-{N(WMR$6- zu^1B*i~N{aoXfEYV;Vv~Xyq#~Kkz-*sO4&PMCEkZIs z)X3ewJU`$%5L>P0DEX0%RQ@=>JFhK)44*DT_m;M_Zr7H=fO8!BJj<0%eqmh zU4-hk(?Y$SUL50_Suwtu!@enUe50-v%#UfO^VwGmN*PH@nd__e+UY1^TjsI1-%g*p ziuE3_*wx=&J6&)!vu%)k@9ngTYboRRRUC_x#ah&}8rM#9#LFKpWs`V}5wZA|VzI=s zh6%M^sE7rxYi`iT?;B%cacfL0DmfMnS}g90iN#$Ui$kwxBoCImzG^QPzYF^akG=g^ z{AUU4-ExDgzr9!-Dr{GJ?0Xr%4J<^^8zmMGaxDH9@ig%yLK+u~CZSc{%E-*GHwzW9;5#Lsz%$}%%*az~%%_}K)bLf@=bTtn@KxLwI2O5#QXb8G$%%zL zBl{A^;+JKNTYJXkjLKsx~AW97P$8oSATo!nqP!{n8&`Cbxi@+vUL$Z zM8&aqUBnwaHSy~j@$!tj7?~N}iC8S}gjl?Xb&XJ`3e}Cp`%WwhGbezx!J=!YXbdeS zDmI&)*ff=>E9~2x*wmG%E5bWe7_6^E{Ta8hai|}cL;60nuAs}V=@ne9D&8RMbb{Bq zXlWmC?d0ISxq_vKRl3e?UC5WnWrI-ep8#%I$!woWjuzUaZxHxaPj`dxkLbPPPFH`U zzCmdAZxA-#!EBixd!4>PXo%e)a4oAjc)K`wcZ&EUo|@X&4MJHpn>xLQbDRypds92Z zH@{fc-9jBHR5y6P>cQI+6TH7-f|t}m7O4`vPzPG1T5pK@!yaYhZtz+qcr9A+Iw&cc zy*YRtI;d6bBagFm_LH0=8tnz|@+X+>1(oK5Vi$pZJ*!K-`r zywmR0LElmIb%G`LvK*q^o}SMrWk)fB0vE_23 zySk6)(wCTRjpWo1^i^z2%qq4?+9|JM$FgsRiM7$5)yCLWtmBiK*BsxhgKvH(-(1ka z>MuT7B~;fp1-fr8jPcDyF}|6>z6oO!M`Hx9@Mp1a@?K@jYd8F#e3LG0=XmUU@l76E zl@?1FE0bcJuq(1*n*j|$c8!&m8jcyWwxZiw;C&Fq^Z z$2V$)e@jdszJ-0&>rF;-`&+KBoM#_pGIXAORPE?{VL!rScba?t*uthy~=LurDAI*7!*ts9Aifbu-_->BH=OX@{r-q*(_U*Rn z#LG<|GBPu|-$eX)YB$87c>V!^A*hdXGi%2iG*WL5c;6AM{YKFhIaV3e|| zT`l?><(~bUxNl4i8J@y*CPuFRE+A8{h^?u#p>aUTfxg{2L z{(A$53EMRud!;#O!+ z(MQ-W^Vr*u#Z8~E-Xy6LywYAQ#&2Qv`y~haUSd(8#iEI0agbOW} z8NCUySl0uwXzpNT3w5PX-SPWdef<93iG|!V`?C`ZS!e9wSX2l~9>4!!-{=09EkCl2 zb5ypyx$Q4uKha~~%lKW#R%QG))72X6T@i2e)WnV7j!)L^bbRvye6yq{e3O=Ltrwp> zDO7j-w$ine#PoE14b~}LKh^D)u1?-`jH{RXiZpN&`?l5TAiYJ*S_3qGOgpRS#-4~U7ypqN-3&9TU5 zl(HT^HYOIwax4aY!$=-)c74@eEKFe=I}(T}dL+8e+730slJ-v0Pq(ZYIDq)vMozbF30 z><>x4_wl<)S{!fkr*ka&i?t!1)%fvSV(|VSY|@O5KrDWzSjK)GBV1-ypOW)A`l+%MrY-`yzNN(yd)W{YI#6@K!p({ z>aqANCKfwmV(~4i`$6~x#o9bDOi$$(@d3#qjX+}fm8^*Q!Ar^n9Tc-;36QLp& zyzu%bU3;j`$j}$TsTukrI6Xrxf-3|ipXl4_OK%ZuXQ)N+F`XF6>dvmO%Jq%2_E24_ zu$|$tw|{ADW*64`RhFy2y-Q=q2-{+heJ_{B>bRD&2=1DpHY6RygIy&@NaHV!3C-xm z$jso1}1*_WwqzsSkZ7GC>1v5@)VASV_wUku<_v}pZepc4z3F9vcf zCiY-kAN6#7)n32oCu~I?d;9&OBAfN5NS)x7_WH%deVF|r$@ktba&S4s8>5DDEc%PJ zA)eK^e&NJmpT2C;j7~=^hW17*Mr2so5~0pQbz^b79*YxWVsT1LEb=)P`HWJ=sIy~Y zaW=#dh((tJ5R1tf)&`-zC{%>|Za$%(l5rYcGp5_z)rHDw8O}1e z_#V1Z@iHemP5ct>E1c-m6{`!C^Eo;-jJvE$~G=Ukm!$hQ}&1KDO_&-K`wWWy~7vMsXVmIK*y`m^4Rhr0S3=V=cJ zH_!9mz9?*k9(&zf?EztDOmYqZ!?+>^2Uagshk(~`%vwbJFHcSF9Q^^|>;{QO54=K= zR)S`<9x;1>uH3$!Vf`c2h)~gg_=?sWPDmQdU;8Z-qF1)V=>CXS>7@ns7RK$Pw)HF*Fr=uzWTO~f8r&>-@M>Pw!3}%qChq_?v=4#iq8|M02PVEq8 z>*%pp&*8{Q%W)#aEe8)@i;L!}$XXp0*rg)=pr@wlN{+06HE`(%4Wenb0^5nf-am}3 zA8?!tw)JuiZ24>-*o%&3wpS#_(9twY1KTo-jXGd+*=qG{1-1u+9VXV!@vJr$`N3K< zk7q+E(#&9}@iNAxep(jSEH7hRRJ2Yxf!Xhu9PD+MGOmHRVGYHGOIbe^=`TC)EEd^z zIp=2ddo=Uk4n$KN6SgXax=E-sO?0{dw3EXh@ZBs0y{LFf_$hA1)uowvD4!NS{s`sf zmXiKN&kr|pt8Xe9L+{);Gu)@YqF3-87iWiM`Bk}S5b=)<4<5>4TBv%-_%QVntL0n< z`Jb_zQ~l&#hM~BuO!bp{8K`JI5@oi(BnQ1l?q%2wI3z7cM^uFV$)b0oQ(eS$t^#lY z1L!fHH7-5R18^nB!vSz5$3p>V5ca+vd(9Ob4+p>%91j6-lQ8-L%vJ!VGJq2&vf<|@ zc>reV0A~3B{2}c59(zrZ4nP{50C-*){Qzbv0GBX;85gkO?FAkHX_1Zxq(v$Z93X59 zJoXxCk&XwXMGAl&!srKZi2^W-0aRYdhSR2a04~x!aFNdgr%z_Kha{&qP^f#LPzyl^ z(d!2=T>+TO05(izjYmxP08G^ZO!WbnH;viemz)4h(E&`+01OhnegKmdfMN!)tB^HL zy4VBoA05Ddd;lJ}h}l{tCjbRHfC3G`1)|pv-~t6;F$2iDlr>&|nFnB!4q%cGz`Me} zzsFuPQ3o(l1F%FG{QxE?081D^-b^;Uy2t}CUI#GV2jDkhKhI470HLOhGbKS(|&bp>AQX#FVyXNUks>iqpq1AA$zJXKSm zp1CRJ7s25mbTCm%Qg6ZA4#8Zg9>H0ZTExS8k$MDYp?Wi&@7X^iyP|>t72f6m`x&5y z7C_g-HmX)rs{Hh@tw7SVI-eg0+N1W>pJ9OgL!>VdX-vdr{7C&1Hmw>^ZZ5uzl&!Xx z;WfbfoPgKPlrCpecSsJ}zL z@)-uuX)PP}G1OnF+QtjaFy;9;h8mvh>X_lV+Bk;FSxUp3ih;s|6--k-vmLXEb2Iv7 zfngkR5W=!4Y^A=<-ySYh3QNv^K>Ii>hYq}g`8g&mzc^v3>XoewX2meV=oLsa#*tRQgvm zcfZGMKTA%{TlPOPotEn>n#Vq1frFgtV=u{3n9TtGY=~oMy<9nYzG6@=&J*dEK4E4v znu|L9LM~nwu_g+2jZooYK4ZH%qTW1QAZuTK6Yeb$^**gXt8KxH4MAYP}mVCdzicdv`+PjY5AY-N$V{_BYDg4nWyAzB%+b_w+sp}Ky$ zBQl^7;I96ZrCd7g`_+Eo9!7tF2tmwwJ{ zYa|DI`@actq3CUFa`jiqke=u_K?Vrh9FM*I-vsHsgZ19=g{$AW35;Gx3VhB@U{thz z6!yU$yK@uRl{{rQH-S+Rf9WmkMfpvTb&)mwpym}3f5%hf{EWnCxSsw(c@ma6I~kc7 zJqN*Cf&ah*N(-7Ezo73y4#iXvm?i><=V5DihYHT^sHb*~`_> zh>#8SMDX%t3b!`_8o@UhNvPo~#y9b62j>=0)EeiB$d6c#Ku4asPvi;JMX)ZAC+?q6 zyQ-?G|mEspxWlMUljYQrx_tjR*%BvjOy?}d9Mayzx*9NfQ%ak_=u z@CK(1mvI~3z>Yipdse-E7v~6idq0JeDQxF??Ct*)O6UjHTe92L-`-E5d@StQ9($QA zOXPQLVtxwcZDAZPP5^%A<_l@V&W$jt4d4DF!}v^cusb(jNE>!;gmKz%K6_Dq3gsgov^o%$0>VCr0>ZwT)^|dET&RFBPQ5L!SLOp~ zKWa&;`cQ6W+Ck{JNx!fbTXNM@c)Rd4b=>qac2#-tt0Q6I$jkx5Fi7728%uQ+E0A&y zPEX-Mrd$qAsc2ochuNQ#9PG})X)TP9O*wO;BL32A*s5IAAI?~h6>DQWtBqX@uqbQVGGnyHo({}X{;f3hGk4wKaj z*(42thP6Tnt38JMVtysg8})d6-VzDrnL%~lyeWxuGwQx=VM|0_Bg=1F*bXnns)fg(d8J&!;7vSGSG$W<>5YD_F>3r;D1HFw*`rS0T9uw#t zGID5pTLEI^ZkO-=1bq9^ySDBJOBVHC=r!J!+>AbsuiwXi+BB0y)ysKlB9=0B%dJeN zH|uARrgRP|A+Jy~P_u3$?wd-tF$lvVa4t1??8 zkLG}IBK|XarO<0fT~#Pk|j=2;-a-h6bu#xF6O@!yX&RD zlEq8;!no44_#3S4I}H{~V!TwMdx?wI1tNHRti>|jOJ$CiTGIGkyrV46&FBDFi;~3& z>V4aumnMX$jz(5~HnX0&k0UY->=h((sbcX&xSZZZlNh}F0P$R=cu*`@Ma3Dc)yOIk zbxU2fkAwap@eD{la0fzw0I9<;lob*qvK`cJnFkCu4CbJ*y<0fcam z9!%;nl5->S4!#a$-kE0@hsswSse72j94Rt}9>pcy1x`w(SYcO!|OBnh}jtAi(y_Gcl8XTl) z_*H43uCWv30|hv!q2GU%sM8Bf_t)>9?98Qm8; z&Pt?*_Jy9FS$H>02Az(u9!I{;)iVP+Zce0!&dWSKH;^7$7>a*HzRA^d2XqX*G(o-S zhfO^_50ajJapWd?5I2ffljLT!4mx%u(nG(3=;`@`^w94pvK~j?z?z=);c7K>48JS^ z9=ZX<)6?l==%HsqSdSy0s}j3o?{p{ z;yxf^9h0%1I@y#rNp42JhK`{N z66m3GEKkoa(nH4?N{pLm}Lz*yAp1nh{y6SnH53+D*$YY&r1k9*tW!Oyb;(z6q+e5G#-&=4=cn z?&3(ww7k0VSVjI6%Qb7OD7Z`%(5@zZBF}t2LI6pQ|rT>C7BT!x)jg#L+-+ zpP!YdRM0?;KQ&+<3Q!p&aTq?(3Tq(p_5UZN$)il8Zw#`&9sEd`!)JLFx;R;;Tv;X# zW!Z@IS@eJQc<%p|74;bk-i`S0Tx>q7d7jJge7^F0v3d3VT%Xa((^o4`Q+--W*t1s) z2hJ-?h89EP)#h1P>fmIrq2w4Y)6$3PNBOb%DuWn?(+5({6*VFw@I`LsDM-)zOf?wQ z&XylVO)uNy=#&FURgu>h6|IKSABQT;5r8C?yP zAClTJ>W6lzc^A>X7iC%?qFAUF&|}mpJ%e_Rs;6c-EAY$R3VsSTN0cN~MQia`Rdiy^ zsv2Y`5*X)nx+|dOw#2GvnHZ~zUX9GE!lL4MSJgdG^WVg(Xk`|wYS$qSs(ev#wX4c3 z#ejtW1iqq0RjexSoUTGtY;aZegPIA6RnhuBR@EKO#F$kvo_(}Iax*#yYTiz)ik8B$ zs_0~&eN{ApRXrfN84Z?U3y=Q<6{J}|Ru%om5UZ*Y6<@om4u_hv6RV=tLaeIaH#(|X zM8%mCoyOq0eRv8D{?tU(b;^#zQwRZy1*(Y-Poa6jJL1gBD_|9uOKwId0nzHjsE$rh z!AWO0sLDjeI#<;XP;>if@YSg0#)vUE34@fR+YJhbNnn8)ZCX?)#?PQ z&g|#3s~k~rmaFPjsQELosuvQdqMz8}c2)2*qq#+LGrG@WEdTMJpn?Hxsbkv}Jm2CJ&LgH>#h+>EY+n*P@$MD+%?k+G^8-Hy{BDoXEk zP+bo-dq~xxG41Nml=Xg9hpKjEeIp9yhyu*&qu0~GKOlupgHX&n<60CCOQB6LX1v6#R91ugs9$+GhkyR$D4!g_2G#1}4BdK^yq{XeCU|J~(`ig?KF{vw2K1_6w zVW>$)VWMG=F*Q8RSOPT;rYC_X^*UgRpSVuPj?%>>6W1VTi<=b|6|cCe#z0LKsfw@O zNWUtn-hm>xR`Ry17wd95&+xgNrY){s&5K;EFC@o>9N-yueZuP9<;Nt|JIAeFi>SD7 zz2m8MP;*#$!s^}PS0&X;6~Psfw_UyKVAS8D=vc$ZplhZodNh@C^_nocgJfjB<6yc8 zc-9i8_~nb@_ENMc{3nRsnSM+XJ^Gz1sh+4f-c>aO zYQ`s4bzTBhw98{v`J&=#SJh;wxihh<$q7_tx(lHSQL({QwGL{2O|0tD1ghvaX&6<5 zs0e)T_^SJjxOj&D1iqpjer!{qI}F5EqT+a0)k3IQn^;vz0#)?$DXgmL2S$^*+d;Jz zYC7MP5EY%L#G=~gNJo`(5Vya(F&PXsmnBw3M{=>M=x%RDRd5cUrQIUA8C?oB|0Y)T zbploN>t3v?N>rS2E~_-76P6jqyZBGgZ9em>l5X?7C>SLQ+U_>5!>E5$pfg58JwvHR z1*#r*o4T=#!8I(Lk9`f&g$ zo;sZmd_IyKzx4t&r&c6H^$?E4VpVN9z)|I#sU7d0t*wBXb%|Bc=~}ERx|=~7Xd|P_ z+{!tQi=k%UV^LImHGyzQnCJA$e+ z1aCz}du474`ORFdt0gz1zX8vTTM}09Mn5KL3Upr)SFb`;440cB&FK43bHc3&tGC#% zN~*U?6dWZA+OA&g^iHRbzUiGtDyg0kRP)}|lH&(IfT#a$39I*|ACpwCBd#_`^+d&R zxf#lg-Ul`PZckXfPyMQ#>WPA*L_yoti;W%~LjFJI-a9_3s`($kcauP}BxaMv1p>m7 z0D*uIQHm&#ppqC6qw=hXEjGl4QB;bUpr|Oe=wksaL_tL!(AWSA7zGu35POZC2Me)2 zcE9iUnYm?mL;3Lf{qft^Yww)7XJ*dKoN`aQ=lY(1+{`EDIdhu1?*5?p&}R#EO76XI zb~CYXQj{?dT?9^H$|`t|o6%Fq?u)CH-JfAg)~tCyJ??&|fsD@k34>qEQRaoQ$^vNR zkk>ZL^(!RGtblWxF;IZjd?Pd|de�xU#KIhk`qrcRL1?6#AsVi1QuEvj;gX=?u{2 z-Ga{BdYkJY9DbW^Xu!GN19ZY-iGC1GWANL3gt4azKwH^Rdk+bov*o+q{WYekl?R+t24*kz2NKVPsNkZR!uhm9*gsox>RThg3N~rX=N`rXYD`Xj7m%ECCh^>y z(qcI@$sAJdjJ3_susp9>t^G%!xUf7)C_BzOBXsHlG$MI}unfbpK3<>*?VTi)qw}2+ z`W1vm^NNEa^iYA3x-wQ}tOb_~v}2oqdZKbMPC8A-s(p}W_b`|iZ!emW9(wG*HVplY6(l1tS9leX1o~4X; zD@@k(evN7LD0WHHGSP5r`L1fJjCU(+MN^Gw^eA>o(?eC$%3ak|8Shruil!RV=uzy7 zrVaXP&1?Faj2%1XcFP)ghpACqblp+iKSUWXQ<$vjwHj0JQS4At_8#Nw{H7vIMe_!rHDh}^PGH3O1o6n>PDYh2Lep2@YJ3rkU5gJqJQ6%`3bnL)x&!>?VZJe_Y z*hNjR86}L*Doob&RgG!(D0V^9ZRI|#+?kBJ4bG3|U7{Ivr$7-Zn@OMb^~N~}bcISE z&>!7-nnZV61EC%B-d3T%_=U=TQ$Lq<@v$S9J;#ow1LKe8_8_L&>0_hmTe}gp@_rT= zDJd2|AHSEAQNhJnfWp6(F8=OdN-Ri~K^9yZDFtg>FaF1xDzbD)guA z3!So)LccOXi$Y@l!2-AC(`$B8Xy3c((^?gJ$M%IT-$|io8=*BhqUr4dBXwU&54!d% zrzia+=%(Cb%IWHG{L^#|H1UkEE8_@-D5tBPq3BCc%?g%jOw6NrAbu!Hy7+Rs zWt~1vnTF{}Z_B6Tb&J$?azFK6*8QHah}HB!fsy)VG5h3C9P|5wZVmpZrjGe$AsqOV z@EOdn(vi3^e|Kfz|6Zo^(61K2m%IvH~NXpA_PdfTfo8 zDE?6Fmx$!k_7uHa8CNPyQuHZ}c-0};2}MtLW(*RtNt#Rlgb9t@nQZEMA2$Ew&D3mK zAW-tYIB5`>(XewuFM-f#-i-E=_wxltO1_Xh@2OS5j(q{Tiu1cB;`4IcgE-`%aOZz>(uB^1&SyC&(nF%Y+t8~ zAhaZJHL4pMk+%g#sxOc^p3|p0vqVpWZqy3n$s2k}-E=%@gDJ_AC5fIKgNkJZ>!yjo zyA>vTl63K&aD*`MKR`3apO4UT*fJM@HI`xNi@ebHl{YiEJd3VME z=?`!u@(|`_nd=(`Mw)V@XBP=OJ-k7ndl!GyA7^G~g4Un^!E0zfD*cnVb8?08WrZ-V z-g$Bc&9mc8jrq!>xF!DCLArSJlLumvXnRR$((b#c=_VD>MFk{l+EZhOcoe&$X-S@F zcv@#i(-Ru7@c@7L^q_81leXbg^Ehs&jNKGM)1~n~?X5AxJPIvR&ct}KPhUYniLX4nO8ox)^I=WEO|k78Fetu7D^SN3o;{Xdq|0%bf?VRG)DuQ78yie1V5 z`ktcUm%VmT(<7AeHigNW-mNjKJc?b>w5d=ujP%(>P3QC$#+MZ)Yx<_feC1K>il$bP zBs9A3E^7L)3iwlDvZiS&x`RSc?24wv{Y1kH_S{8HKT`qyR6w$(gEXehqu3QqqXR|5 zNqg_2rkhki7Zs4KX-|zA;!*5^rqbO>UVLE-w=)mEV;|%)QCqaTeDL)WC_^ z6bU{h-436oOx$BOa*4t+O`7aePNCm)rckTKclTjlyrqZKG z7(kG2hZz+bAsP-Cxr>^vPyt^kOxE-}jj=QdQ0$7PO*)&q^}t=!^ekn(TVb-M_iId} zN3lzq9wHiU9k;8RD&yS>ThUZw8a;|#&{PV4@?;@P&dwD6J09h9(SjaQ_*V;z)ZZ#* zue{AMe;w!oD^d8r4sh=}*Pdh+=Ig-Ms4*=3{_OxisQ_CAxU(?d8g?s~rV;x)1S!{- zw9(xFoN~tU0Fp6fpD6cj==?zPw79&vx8oV?UeLdH-gpdD+00KB7^%Kd2A`8}az?es zg6@8?l{@fgtMtXc4sgR7&~m@6mNNEKh^*G}&9?#axQUInTIF#<<{E zr94NWS8(4{Yi4T9^&W*5+_!YHCs~ngwb7*9nPU1EG%n5?RVZ0^g20G1RjitIo@3q6 z)s}S){wSuLAfR=Xx^0CM6hKS-SvOY&yr)oJTQsvg-x>i{FjHk8tPuXtSzRaZm{43J ztpzoC9*>keW9{>>pd{~t-eT=d0wayH#L8PQa;)w97|z?n zRFj2B2}z#3Amz^Z>}#afJMSg+*%pD@Dt27N-Wj1MKY$t3$$Y`Nb7_r;j-V&i)nnN)0EubkJ)J6ai09hXhxdj*}hfo zqfQcTG_RGn{0q^RZ4O{djOVFx<^Z-_n=kU9NvcxLEuMsOBO%r1yYxx%JPNOd?*cgz z%oj@H%R$&V;Nx2NU8Yqq=zuk?7^Me&_>DStzRZ)opwMR&Usehx#wQ!qq?{8qeLi*k zMkU548|A!A<+a0M=X8%(F_>VE*|CRVP|vlk7^SCt_>H>#e&NbcZl6(n9VnO>pKMe} zITv~o;*3g+Pc|xP7lRdcjtIG5U+}SGvtZC}Pqku{9u?v@>Y~TRsQ)VT8O4`?f{F3T zMztvCOie#ArxN3njdEU6^5v9s&d5hEIyrSI40^UTqx8HHzfmhteVM*^i2u8&hWHlJ%oQ!>!&BE_>J202>l%cegHk@yH#_Xu94PR!Q6k}CsRyR` zJ-b4WS?Q$QKF{)%jbLJYvS*{pxy+LgR|gW~lRcZXht>)^r>%_J>=c)=>v19p{z=6} zPhIgFwZ~V&b)`a|QG8J&m>8dIRGo6>Y5Ix9B{4qPDCb2IU&VHgei`S!y>a-n_z)ES zNk-`rFmXoxqedz88O7H#f_o&yw_=o1u6Yid z67bIrr<{q}$)0z$>1l^kxE&s7)n@7wYB`5v;(`+>OoovLsvl()|y=}MOgj96Qx13Ocp z9h(BWTkwzM+s;IT=?4Sn(T4Oa9SzjHcW|&W9-)v$hQ}vIdKh+Z8o`S21YS7TI!B=$ zyBq1e%Ctr$q&3Q<6)aWCF$y8P$_PI&QMi>u=g8)|OsI+Bf|Wy0$!vw*iaS)Y(+7wy z$vb|qc;sAx;VK;vT3-S)QypP zS1{J8xKAMVC;TI|_WApXVdl_R;Hw~-^_b^>0NopptKd5-;0J{$0TXy4WaMI7-r#ml zyUS`1abGY#xjMv)OAa0?E{X0gwcv7vcIz2Q?k;jy_myxVE z?d*NUX$>7jy$n~qX3%863VlY|Opz5t%F!sB`UXE#j;|ELOOv(f5Dj@v@)BbZDfK>qWCj`JSG$Nddy~wepDJWzv2`(2y%8@h;dMA~5x2s1+TDNB9d--`O%?u04~!LW;ep{E zv=nYsj=*6~-{Nd=Eg)y&10$X@Ql}id>q(C|xe{C($(e660Xo?%jsd?F=FsQImf#Ef%H;U=R#(6&*HWv1I-jj6b)#>=5wdEV%el2JzqjMVGw@_{vu9lhShd4u@F zD`r@U_GY!y{)|EPn)b+Ff-;mfPa)c$dbjiX4sO()6IKHvTpo=tF47WdIg!K>OJVy;$b z$F7EmhbbnpH~%?(;BnR|#~Uu^Q{eiNoQXb&=M*2v-rpYn&Scyz^Hg#6`^u6;bCrxA z_SD4%JN5%oTYfQ8TogCU5?Q3jn47VCWRY2tm~et{U#<`XOZnzYM3x+a^E4t|#iPq+ z)+{3Ta)Bc`njbjjf`jt=Mi61=fa2TTb4M=xK+YXW(o_#B_G`M~5XqBY6eelidyx?NL7*Kw3fwa&;r0Xxx5rBm z6s7%GxCzR9sStftaJMOggo6_#92__p9Ya?KLGh8CyJhtEmO?vr8zekQ9d1pKaBI8{ zg3=T9YR49ZcI+weze^cI-5n9t{IZ7P&|>mtFWTMCfMbDGjb*MnUxO~~W3cv05GA-P zN>ERf6=)BxoIG4Yxr1>HNdG9e4kINAr{@mC{UZ-$>lU(dM!BTUcZEnQVIe)qAsyqA zp0>`kjYg`3h4gGCmGT#AFF8C=p&gqB3G?yKEQ_CB0nUz`?{Lj(<+;S=tq0$;hODfG6^HkaRCmXvf|LpVbWBP4;9i#jrqdC>hy1Zww*Y^|NUdc#%e7w#vHXFvrrFkas&}_A=l7 zp-i(m2sniRonT}N8q8J3a}=Tz+EBZK{Qb>WhoodMLDqgrWc_hYVh@4v4HEZF;fxIv zGb0V)?bw7ym2$O0^iaw_B>kWimh)g=y+2DKn*Y_ctUQ^^7~gyRINKVbab*fK|6>K` zXxNbpzIx$%)aCoP(%k0oVaJYF%)`aUixI=B9AhG%;5ZiiGhgE2+k6^`<6ffQf}@pw zPo?M6pp?|F7)#F3=o=LBIc1`o7>%z7CbO3$82X@+tW^j@{R%gnDT&o+^qy7de%tW*ADr#Nyh1=&L+{loO8q<;Qk>L)YwEs1iApy^FT zO1=^H!xh@G_dwU{Q;6AQfA)&Mx^A+w*Q0*=j%@u_c@<)g6MVOmH@SJLi03WQhu7im zLu>>1LZ5-RpUoE0t400nL|*1Aa^r4=zRH{%;IyE}n`(x-fT~Qw-ii+MNTBY6VOxJFNm_I4%2zvr5w`brHPEnBp zcZ4x8v$~@>rMBuH?1jl3Z|-q=(iHlCnTLU|R7P6HNT;k~tkXHm`y6|4@JH*7 z-1X<_0n_?>rK?g!4A8D4{OO1qnIKAVh^jKVtgXK%s_8D#CbZbm>RO0eORZkD3!D5} zz3Mu_hnYQ|)56K7&3LK}8x5*uocEOZhMiYp0{y@g zZXP7^I%3-%3pGAchTF$c-uZD$44LPfs>?3r1&Fn*KA$vORw`<1X15eJ4z}&djx-{N zDeen}cI-khJx8%GwP8K>)O@K8>$$7uOQNRdJxi;1Pq&(fzr+Fi_`_l+PaTU{bzM0&F~7?zB9N1+`%4>UL8AN=Gi^!9A;t23%nol>rHIqE_E zA~{Yq98`jxui&X%+Vt^Tfz6=G`39Ht+~N_=2RLx+FWd_GZ<(GaOAQC?Il=#ugZ);y z1wVGeEquhyL{JouW92-dj*NDL<|I&P$I3zZApQkI+jhr@fyK-C4fsQ^LCf(qI)INv zm1nPH-n|ld_i9^;r3sH0$+O|f32ZSI;@b2t=}5Cnewyh#r$-vT7MFjd(2flS<%RfX ztP6}gOq8plAmA+fdwdEhjY|EF!i@ESu5k39i1I!3@KHL%!@HCg^U^+F)1GaMWA0aI z#|DD_aQucou1QhLk#`UaC;j9)u`4Zd(dj&Mj`*qkL&!qiL7iF3sB%g?8*N z&_69{k3Kx!Zq}|n#zRV_QcqT>X-_d}`|6t+5m#z+GSzKXQqXZI$3Nq}z%1mmoA1D1 z-Wb;=v3^%Vv^5c^1hO*V2&WELq*wuPf-iku3CcI+WAy=oYZAHg|YiGuM)mytz=j7Ot_&c%O^Ju_m4 zkr6&dqF`L=GO|}e#uvc!lVN0QfD{?wV>Ck+eW=XV3VX9#ai4&^b}dMD@MWxf^dKK9 zdklVaBgRQrDBX<;F%YUYb_^jkR;1N9lohScliX@O5>n4Hl5dHZJOE`bI6>)7R_IA4 zwUy;qj};`%rI?#rwUD~pNbbry@9BICs(i3q>BcMcB$L{5-2-n1)<8mI#Lar(2q*|Vi6*?#7 z_k`@-hYwg+B`aqf#FZQS=th>7N)>aQEACU!T=_p}c}_}-|9K63IK{acVC6)9b*d#N z@FC9RzAV}g-DBMObV8^#pg)M4q{xy!#uI`(Cx^E6lK$^inmAZ>k!|h_n ze;qp-HD;qL&$T0%5^qN^<%D=U8g*r8qe45@?GJoq0RNu3Da=IA2e~Py$MGU7PWuy= zDY)dEi%WTK4X%G>7z3yJlMHE)owFYI+g%xj|i$UTB zAmzE=;>u{2=oUAeo_o&W&bTX-`y=lEdK_sw=Kki=49)!;VX3PnX8B?6p`|XS5%tqngtF5=_{iAqmj;RrZw4T%Ve>|3WqeMqg`XJ1nqsKpW+x( z&CkL2n!>Vz`aap@^Au)Tu!nN}OCj>Gs`5w>SDQRU!FHFJ3#+s`UG7$^N0CYzib-LC zS40tZmSJfy&_7wvj$L9~gH4ID0z(0d>QqWMx2}xD?W-y6VP=4telC2N%)lm4=fa1n zlo2JByCRCv~%xeD#rTJYKU z!U1Tx+Q;JJ@S4>>Ud)FIBy2P`15#E*xnK3HlY92w20Dcb$r5g8bXOQh*w$(5~|JD7jO?Hlm#^J5xcYH&3J&H~KB!BPp-Y)So6 z1w5nzs3^Zq+vnFbZs7?=^u8OSIez_xXE=VwCDA;tW>~h0)D0h zvtng2*~g(EmJqZk?e7+;CviWuEwno+;1nq{7za+}vv@JJGg75n8ozOi)IQ)_NZ!ZN zob8Ij`&b%FlWX{6V)#8OV5tgl4PPUMKc5(%a($&3zQ)MQob5a~jNxn2%!W!WehG#) zHA*==#w~}RLDAFFTUC}tFj3ja(;}dw3P@CTNMgKCS<1_7*oLxA$if6=OP-Uo7Amx3 zuRzg#GoWmj6w8ldoxvxxI3#l(t^1$>}gZ+hd zVESEeLX`X2=!i96wCbbKj%^0t=%C?^&+mchQ`x8Wnt^Kd+5*@8mw?a0m;>o1@_q<+ zWzkHh8Q{<)QldrlO1G#UgZnq*Uow1d$0RcOang6?%6EuW5N@h>2CuSy|BGMyJ85=7#IE8{L&U;cPli zHBxBDx`Jz8a;o<3Y<0McZpvH;eSay)jTUC5nADK zg_~e3&3#`XMi1Z^nC+SE%*ta@MrpowX^#Zmd8Ac$dWE4yY5s6&F9F>uAFV4l5=Dkk zha!b`>_gD~L)wYb)AgqBBgIO@(jobj8?vpj_=DKYb$j#KIQmS8Rw~2>F%UM8_+gtf zlP0q`!47=3406my5RLX`FAE!0#E~7Gbl-!RP!2@J=ede;@|~lR)9|<=#eKwtj}_Xnwz+uh z;19F84(azw$LCBDcD%x9Z@sawSw%cvl*yyJdvP&-5El?Ss|J%aCE`sc0#5eg7KBF zA`=bPE7PY6eHEE2JoxAru2PX(l;n3eoeHE;$8;vzX2HlKF0#%PrE(S8vFAb921O{Y zB9qPIDy&rM+gz@};21})`0}~$^KqsS4OS`B%?f>{m?4>0|BC9BWSN`J1f(%FQ9Cne za+?F}#uA%WPjRxR790)9+Kr0Wj%a46vneNL)DIW1lKU3piW*gB$=QzV*C`iOLj0vQ zjcmTAp+vV>)Vfs5?X35W*kl<7-_aDA0F4NSqxa?^>lD9PV$I@chLN14tS`vvjr6^h^eUT{&?U>!swhqESj%ylwBfMzw*2&0Yu624cGr`fh6Sy9> z&7O1k(yU_w=R_l3t!b!LXENTK-7c(B8GYSJ%H@z#kq24v&NOnoXDiX*G-b|LW(+vw zjACfdbbJX#ij`(xH=%csNNr~%G*OiHv??l5qQ_n0NEh2$i9h|~e3v0@YP>PwV%;;a zpS$B=^d7c#BL2=3)9u(;fvZOAxH8-jl|IPR3hmh0U}1bnme(;kVa70_xt#XESn+Eq z`sXOLW7mS=5&T0n>}&D)Z+3ZxYPb#GY_~UA7mn&`&R*`)QzN+kGMs$n(H&)mYHq-j z%&S1*66H9|<&1VksmDK5!xN!=oNJrF>3nJr+Afx2Ql-#u9{u5Q&gG~Pzd$3znmLki zGe+OM_p?GfwgGYf;GgpY4-z1=nb#B4oT(tDR-P%GXDh_?cQ@NQ4F8-hOO}bkW`ZPV z3*Ym1_YGw+lCPy<5>F2%U!>5EoeK7wj0nfiD#CPed12z_4M-7DudL6wBJKtIYZUQ= zBf>NQg8JT_3~5ciz^&2mgDV^*Cy#uf<)&5(n#Fr!d7~bi_nksJHUM1bkaNqm55-v? zuQnr%+Q!_1!p+Ob<8YAyb~C$K!a`6sCN@PL39K_!!2);&@R@)e?i zLfoP%?Eg5)?BQkuY$oJ@Geh_*;IXG|(<8+?9vPt!>(G-h`-|!FtSxDL?3gYLXRH*r zdf2gCL=Paz>j@-+lVlL0Cy-=b0*T<9SP|R*s}OH2L8A`cQNUkm6Q8i)>^0P#0*QiJ zR(=lExk0Z6k0{Qv@c<08jiWcA0&WA;$sR(1)l_|PeDb<7($4RkmW?l6H?4r%0# z@)Wsc-mhY&n=NZ(mct=58pprDMa;Lcrwg=W2ZQJg{5w>ik)`=|sX=FN#B;X<&t2{j z;5T18Or>$a&b*ui=dD7;41SUz5YcCF*0;m7E5VOxcue<`x`w>LQ{h)8zf@so^HG9j z7!anUQf>T9X}<|9R=8@aP>-7CD0>U;ucElC+GA*73Yk&%Dk>YiROtsQy{q`O{4$cO z_;pkq*yI(>(6dglQ(vas>)H4O-%vrfUmyvCc#<*dMm6f!8e`OLlpCCQo>1MY5Jush z&}|q6tEz?k9EEmlBU0YV)c0|kzFMPl==)7f|@y zoLfiSDJHKzF`8fcpy=?d!tH8~mQ=k_>ALwsNS;sW!<@{S&IybT!%Tthr*svb7pqO) z?Z;^R0!zUgSk4R2$B_eU9sp`*KZ2{3y^3MIzt-H}VD4`;;ZJc5WdDR~N_Hx;IW;?i zYg%>zu5GgS#IY ze@2s%lUC4JP3?QQ)#<6fX3Fvo z?d$ZE``nb{d^*wAP~Nv9HZOA)+a{z;Et>~*r^=gA#@RjMS(kzZ`zzN? z3Q?@8RvyA=zQPNvveCs%KD+Bx;Yw{=H>eBk*k4GvM-QlcccatB^10OUZZ=y?m<6`u z-K;^u)0KO*a-#{Ys|_E9_+@-Bn--Nzc!ILFY0-Ogu~XXyAgS?8?l~tpV`sX* zwvOiAnan2EHn+j*_`GR_gTcE*RtJSf2^RZ~&$B`z)n-lYIQMnJiI&O?y zqp42j`d-I1G$Q!Ba*tDPdbgGuucdfnqcQm@x&&xTjWOAvOx_C20k!N$xDFwCaNdMa zdiI^TTG{vFioVTqfR4GGf2Uv;6gK)2PS0(|eQr3$ONzPND}Ujk40nvo#sVU1?<=?x z+Oe-dc~DQ=>V9&-ogJZ6;1m|TK{?ha2ZnIf^GIK9;yI$bndmtVT3Ju@TtT@=zKr-R z2cKR0L>&HW4fXco|5x+8#Iq!0h|^T_GV#kD{&y~VpLkt>`!o^1;Ue1F{iXo-kjhrr z;TIvm5%EX_@Vcrkwmz*83u2Hkq!+l>8mVq~3OKi~n?7BlkTem}vyJ1p6ChwMh5Qk4 zRuRpx1vn>wg6ou{vvSZ=b|L8h4CG$24ONSkcK38A6}*XLokp%jhRaiRkx@0%>SCr3 zBunbWNlupB>7bRyWs3vc6AG8juU$@wOB|Qh)vk`iud6j#u*AH}+p0HaV;+pf=LUs# zY&|6G-WyWaC%AO|HeFhR$u0`%gjp#n!kX}}f?*DsHUvJ2W7@DSCM%}_^u2RG!IhcHYWM}h zEQ{R4(}J*6$*8G@7n^F>+sgR^5gl{6>qtng+`=LZl-83CuaKZU;s$ZnUcTF)wKrTD z6i>aN5GsSK2LJ9WnLN8nW+W^quqv+GcVEety_HgJ<=g|_c=fxX!4{P}=CT1|Sq>CL zBlTI5_74g%!0c;VPvhS`VlQOk^f5VrlPNj9EZ!d84|!b}#q=`sd|Zw|J2n#coAK{H zSQZ&fZVk3ic^y@0uu>nS)NF&eP~Cl~Y_)-%|JoC*!_1Kz+f6xA0*#sSZb(Z?-9)hd zLH2xmcpYxSjtM69d}G)4?1)%UGh(;r3D_HC(DhJW1D*CFM}#YjEFO&#ND(PQhb*Z-O+LPcb`JeZO?aBPP>6SDtxS~D14}! zvq9SxW}4hfZw2c?9C)CK@}PCg7;H z?YECi0!7ryVZSU{QtB0qhNRbbR#K!`i`ICzh>aO$TbuFk{-9li`iV0cPPy|Ry1=?KJs-oFHgve%Z>Ab)0Lw^p}*#{ue;s657(b6 z={>5@jxB@cV@E^#bq}FFd7Y$nJJh{VWel1uGVRzikaKbwWZh!l5Z7J3Wyj?x#Hx+_ zJ3=9j&w`|R<&gH9o%cK<(d0_0{B4KQ@UG6mJ6o4av;g0lTph+GWWhN5oojbc9ir%R z(Toj)9jDv~P4=mP8B>kg-f^qAL2+1Tg zO-VlbVoau3VfXV{H!647FAukC8p!A>0a328J^q6P1Bw*dYG-YsL{y5v@P;3jQ_FI4I| z3U|;7GcnUwkF+R5=4p=W9y!Fe?iz>Wl3QUW*GenwNvl#Rqt0-ooqVWm{i_nvHrZR_ zx>B3${UD7!j=aN6G>2zEtMpHu2LmHPiaD)Vx`S!H~8u9Nbo(7Nd;7@TjWfPbby}ligEo zCIJ0sCp5$@HSrDchNOnL-FYIulfoS|#I8SOP?l^dHDfbqmQxC&0g-+{nq$OUFre@_ z7!gYK##*6N_UQLS@3)ZZ>*{B2>}HT{_91bVP&{`4+n3# z$V_{?_|}WeoNJF%PJ0J0Q%f$uuVa>^5iP-N*xdcF6t^GQ06d`|d0wVSv{bT? zPdEL@#JKJEBbx?`q)!xLZkhTWkN;NYfY*E4x&aEdVrDq1-Ntj=?&HZ|=K@*xdGxO* zVjjmgm?y2~0$F#11@}>|QxsweXL1I4ujc-b;GxOURqfqqI6fTGr_S;2GwgaKzjqZZ z_>|*OFZoY*)c*^&XwAxYo51@ZCw2m4{S)tve~w3Ml-*s9({hZ!_@m5TpUAxiB8v(m7? zi3eJ;H_3+!nsWV9vd}QxEHt1JHONB5NE45hn~>_c#npEt_|K(`o`%NgJIBMR?=>EZ zB}B}cur}PI(2m`K=wC^)kPmofMfI?gi0*;H#QAe>9qhi)WcA6)Rj&~1b@k>A+3t7n z?K|=I^il+Vuh&;)t-vDi%c|W$kei-823IS4Jg&}a-qGg%ndbg^zWW!N`#$>YD{&`a z<=hO=J9i7NOoyXy&M0%ajw=NJ!~Om5WVm;vIalo6CoIs8rJamHEdJ55K*>Vfq>CsC zIEA%RU|d@;?fNA+k2?j_^HMHB5^ie)Zmg%Wf*sMa|>nw?&ZGqsC7-L1} zdoQdb>At82RP+n^-w5MvM zS=s+_Rec7$`%VV`Mmg~fhEL~{75I+wqOficF}SdgWB6Q1vQLAwWj@1~xrQ4>mx(3e zuENw+nNd^c*B(XQ`^_30@Ly|ABL0AQ0yWumK%=Vy94%)y^O=baQ5Us2lT_;id_r5n zy}FBBZqx!!lKP9x$sEyF102a%!Mh42Dy&JP1}>d;5?b4?seCUG=;~=_xoB_#3Ku2a zuMqL`FW|?Sy4ovHRLFe+zY8TOI`E;cCfSymZP;Oseql7rEa6>r+h=kp5dAX1mV(XG z46fBjzcRW3Upj+h1K=!oz-Q0k*Z{cM#EqU|foSuQLfE%?rr7tL*th-CS~5WR{~RdL zj-3Pd{Z1FkDW8)4f7niG&lEfTdc zW{du44_b_%ThxDaZ9PSzO*{Rt3iwMQYo_tVFUA+!OWiaVr$QEP)5*ohC+3QfS8x#Qmr6kLIU3-&2u{%}>=- zs!ch|Pu-bR#Fmm7lFGdb?O4uPwsjr;(XZ|5C4QHF?buRnT>7=KC3u7~-mcJf>39DJ zms&YrBgN3R&b}xvX)juyqR@{05A;B+==E**7{Wf6f+;}5Ru`chqtK2G!Tq!Fj}D4k zcZd#ZkDTFCeWn}f&R_Ekt7qn!C#cowa>Q5WEx*&Cd0PZpIeh%em$QCYV4}=|&`Ty7 zW25`QHPkHDU$U=6e}p@1&&+w?e9X&u!_S>@7o)-b8EDJRv^(o4ALM*;v4SExmy+Hv zq@jp~J}(TnzNr$348$=1MEs*iwBd^*?5Q`Hkri$*0pM5EataGLt9WpUikObhCq?`xy3l@~!Rg*jFZ zdo@GN&@#ACEaZBc_$NL00lYcO>6hy&2To_KaE-1N+STLuuO@i`RWv}(4bVX(8yOzoM<*) zQbOWNg_Xk&p=>TxQY)tu<_yYBA?0B-%!E#byp?C)Ev}0yTrM!uFkS2(IKy$>V$hE} z7j}0^;}@KKuIrM<#v-^`S$-~!-9uKV&!Ohf zyc5-=YXokq*l7YgQ|#Oo-((NVj{f>w@zz@k?bu0E@irp<(MM8E&pNr9c-^_;s5WsU z+@@~mc_ykRt7ra9$?KeyHe6t%#}nNoDV2~3lU6gt!|To5U}m+7nr@u)oMFX- ztotN!WPgQhg_W^MY2fDM%)!>r8N%5SLuJW|Ae2}^MJBr4Ag8o)YGwzewO$n5W@+Wt z71$nMTE|=%uLKC((#jSJczkKSQW@S<7*|@^P+37I>SK?@E3KwMZp|TXlkAmNIL>U} z?>PWXd&mk50|=Djl2gVTx$H6>4acyrat!WVZ^G%}>h@w(x7{7jo`BeO`1jaqdT!`~ znOAnisXbv5yxM0L)9^;wD`0FJh}5;0 zMDJ7>_ePod!Z1>);m`(elo7Wu_v3q`ES}_bm&Er*Sv<)+A+ho|QUNRa0*=Xl1HtG9 z9KQTL1AE{R6qspSd+}3P1KR-`$nRcR(MxdXY!xDMX5tdcor?{u&S!YMv+~eu`nMac zN8Ms&c=Sqjk6be{M~WFco4mG8rt?))Q&E-}&f_v0;z=F6TYx({k! zG%5OjMiM>_REuc&+@$pIg?m9Sazn%05G`r@g$pUT-)%LTZZ>I0W;to6?d7TVO<#sP z;uu3j%Mrzw-){`FqNJNiE}Hi+0y(R2Nw(h6pkO_s(^4ZX2TM^;b-M{SVjAQz{Ii}2 z@VypxG-jE?BRJ6M=B+?Mk*9qjX#-RI92fDJ-`^63%|9YKSs}hWMVs@ji=a??PAWh$ z)q}>b5Yn&J=zV)@rXAC2^kGi)C9OuUFH-KMtw!^t6_YD(BU-ZDZ=ZY@4yt{IX!)3_ zKYCz;*e9=k^ue<#XHQ%@=JMs4n^&9c%jHpnGIYY=uvw~ee}(w=3g~&pedk^_rPI$93y56pdbAgwix`FQTrS+OHjZY#H9W5^qpQzA|&4CDhccXKS=Tv^i0QhIGn4GI-`v{bx;L6Fy<$`(& zp<#g_DO>eVCYP<~ZIWlL5sLF@a*@>$-|LUh6s~^gnNnZPmkDTGy`yiWwog5B)2x>wM|T zr?<0$j+%h`)~O2Z*a=8-1(R(~^J;Z-8q2&_6eIOY|F+BbE_jMA1>ZNR*{Jg5H7~RP z*ZNX55B&q6=Br8W30IM37105)R0Z8Ch*?fiyQMjzq;c8Js1f`|IdYT(R#uz(6iQ=# z^6NPpofZmRr4VO0K`=jT)Ok&+`w<+|a;-^a{fFA3=bA)48&$;rTv0_BGw>T4om0{r zQKDx`8bt+PR?6=cQqR&)K%bVRXNw+z(Dha)`=>$pBK$jdY~$43>ZO9S(s?B?;(Q*d z%q@Y-K=%%5cT3~Tie5hMmd4@}?4p#V3VjYtC?cp3Ar=-Pc(=i`~p3kxN{QOYSGTBhR)h+8!x#rKH{rz*tJY7oVbh`{mmg=b$%8f#~8f2BM@ zq0jLN_Tk~r4@U;3Kw`N5KuIXaZMmo7_S5*cyTUx1ITqqVQoAd7$V_G^A1dI<_B$!& zk-}H>!j+npVYfN{3v|6O2yAz~$x%vUj<&m=Mr2m1w4tQ9krY0Eq6h~@A|h}QE;)zb zBA>bBXK@1P+;T054-a%t!^O=~<}Xu-BcDOfK1P>e?338R{WRu6g)m||C3G2189pN- z)mqi&E3{+lAgdR~zFqpne=rpId9V80h;US^=V*m?>?-hWBJXvqr|xjA%XRUbeg6i{ z=qo{Y3-!SlGMv`JY;Esy3(2iP_$mWBH5@&e&P`E^c9Z^srHDsEat>rXNjWcgX}$0f ztL8sS%P)VB79Obno|I;H8nDUS^wVV{)5qFleeEBNGzao?Wzu}@u*)VErr?496%i{G zg1ORAze(ocG0D;8Y^=JxXNsZ~bolP|R4Fn1A8X>PYg6VS@n)i#vsp^GeqdyLDl&XI zHW;HnP_qx$Zz*&e~&5n-OOi)j=V_eN9QP^<~ zS~8XB367$xMbVQSyg?K_+2FoR3Ri0jv)XNGY9NSx#4a=4Ofxw#lQo4cg~`I1%m}wO zV0~gE*rX-khoTZ&f+}r#`fCTwj(q@n_6WNi#U7-o6<*m0o~`sflz!*SO7I0`P4!rP zWheN0a&$sja@+g9?XJdAr1-nF--*)Ne-h_s3ahls_gTI3#2f-1cKx&jn!ypmm*I(k ze4Y?r0xNwbFgQ(Ry{EEJ1k2+~WJ!F9oS#r4>k>-j6;eH=M4Ikd1L|#;dNbE!x@W$& zJ<_7dzH_HzN)=4GkuFFswzPix#Wu4AX}Oi^|C^%gWPufKRwdiGedV=K^iwK(9oq%3 zrM-@-`nm*_+7+1Yb_uqCtLPeVCO=E!d5fajE#A}Z0~`Xr^U3RP+nLwF0ZQ9LY28x% ze{b8*Q&Ar(geZU8esywmeB18uy}eX&7W~_3+oJ`ow(YL(wa;FO-YcJ39lj8VS8U-2 z_Sw7a<&9GjHcz@^lsZSLQ8465H&C_+J(wI{WeWQ~vLgsJ=LNcXutEK;Phs?KvFdY$ zc5D+U**8p3#AHi9rTR!=q9PY3$8TQ|&o716*W!vX3h~8vXwvaoXi_0AF%P*4wIR?0 zG2up?f{lh9KlZ`x3jEVAb4qgcie?;qWlsA=BWyYaf(1WU?uc?DYt6R_)321LW7rXU zge!g`L{FvoeI4=UxTo}e*{ValF;_6F$kL;j`npFl%>wJKWWUkWF+LvBZ=@tX9=M4s z-JECa>S-{ms8|IY?FzaNLY}3dYEMvgoSv0&>^ltMk|BT zAfweGDr|!BrGr(nkfj?8Ww|>a+W&*7Nc0EE4d+Y4IQ0KI+d3Wpj7jbnhRrivaW;lL zD*>oEEW!WypAWW=$=cI4yN5w5tUZk=|C4mLy4I#9j~>FOYRKxzSy?OR2oTA{Hb11s z$8Friwl{Ul;|vjPMcK8V4X&KI+h=Vs(cE!CZ&5O*v#GS1abDYMw2-D-QQG_pQ~3#( zp-HpiT3VpXr-`!-i7JGbal0IUxRHlhxMM2C6TM7HX6i_Uj;GM3pa}h0 zp-<@JHc!j&gbMBoeO$86CzL85OQDX2!(B~hfwdJ&MZ& zqps}25p{*Dx3NrAFZQo5d)6Xh za@XFUL1OIvo3gvzzYPqkf@<>oqRlP=F z^af+673B&0KQVEBC8GEi$T$@jQH66Mjx!_-Rca8u?RF-bicI+rC5(dieqewHyoTl@ zz>n=MV1RcXLLh(S0<-kusl`tQm~8(NfrAhm<@W>bMj*#p2+R~C|I*ZRedz)$O&yKs zN)zq(Q2+hKLxC3%En99%a-97&%R6v=p$9yTbSYY8a~FeEd`TK88Z`}VKdEm*Vnu2o_bd8Z@4>YGtd+CN721#bVrw|{& z0q@`Vw_EAFMI@{%6H=`{Q#kKYXvf;!hTaFt3O4ixACRYwu)c0Ih6^XpmT3CIQ=+i) zED<$Fp&c6rQ4f)JhFLHZF*8zF>mjD0PjK70-?dMRV$S!R1t5M{tQO2AuN>sx+oB686&QN*a?Z_WmRA93S z_ADxJ^T&uPNDyTWn*`4Ewx;KKHiD6QNU<%fk1MoeZuc>{>Tz1e>pn)BwNIR;Q~kbP z>*HpT&tLV(n^^VSUaJ-51kge#G?e!uA(gndEBRf-^}>8C>VcU?r6}Y018pvscv^+4 zTg0M&E3{*K-XW`>Ni~-X7#?RNs#D>&DYRoV!D*(#uSzj<;clu|rI=&aeFm`a@G{7U z{($DN&$KGb9G~3RG8tmq`%zh7a(Hr-6Hz~dIzMNT*>E!GTeLX{Mrqs-2H4j=i2?Q@ zuV;Wg)LR+gCnWA=;NI`h17?8R`dS!Zbu6hzfZI087~q5FI0pFiI*9>px>(2n`wvet zaO-|JZmLHr@mOBUIJrtKE`2U28H~|6K*2A;Wbz9FXui4gTr1%(t z|1!XemU%Y<>v1QCfwvIo!vF_9Cou3S1f0ge*o`;^00EKvyeAh6%hr^8KyeHx-e%w$ zM3&x*08jj!zyMp4vk~yQ2zPnF>P$v+@5fCHaF@r!3@n1IjSMVBpoIaJ*6gJSaINGp z2Kb@xs~BKS+sFWE+TVu&_aYq50JHyW2JVGHGa2Ac;ky{%8++}SA;8^oqZ#0Noz)Cv z#BA#d2KXlLat6#>qyIsG8w?L&fP4HtWq>{M3sEGaeADVS21v6nERH@3*4G(e`To-h z`CykT2Q5fDXEJw>`8;tw~Xw-+%8 z5wjn6IKKB38H79;@&sfa1d8(+IIA5#gTTOPxHB9Lprn%J3A+RlE8K`9fvHvFA#BbX zu-<@s%?u0-K;^Xvti+vZ2#BzVC+syuyx>MG@q{geLk@iktRLgvF$nnTSv&0gOU1L5 z(V;hy@r{n;zdZquLBJn~h(kxPZ0*^yLtgi#69(x7B z=iG<|Uc?R0<9rKL@92lPx0M05dgaf?S<%Mtlb{vt{FZIa1&JhcEEqJIeGu`U8!^rE zKKpkgSpWExkY@g5KN7N{{Hog_Ac^v`&G#|D&#S+~z*P{N1w|#jQ-NuE+#|FS5z$+5 z?@trh&$bSLJEBkFP89;evzMpJB&ho+DLw(kS_FJLm7&o?^ae&-Z)5YAD}c?iCVD@f zfqTCIiXqtGb|wR}3vKH^3=BI8i?J^vz>lRr$pAn0{yqcz2KcO(5a5^HZ()Gn~BS=*)Mp*#H5N{5!;{fXMw=Rfn?C zE}(b}0iS@jek2Wrvp3#dC+$Sg@`P=XcD|?QR77lbBg%nkGFy=k>t6@!9Nfb$e81!j zKT?S2Pak~h1SHWhNUI~NY4k-f9g2XY_kbsMH{{t6Bp-bb_wHffX9ObfkC3i*Nv*d6 z7~rF0jQLud9po-nzwThZ-m*pkt4>R`c(-{Bs2GvL&qmm#yEWscz0D6tU_{GVh`dV$ z`ac|jz8Is-M8nk(ZKhR9AXMI;(r7CMji_Z;h8+dAs3LZ3A*{&j1|<#~M;$mLz5snr z`!Gu5FTVqlZ)@l05v1pT7e&uAJ@Hul*Fk7MkPk8O$Ksp&|35euf01zwUK*p0p}duX z<{>NTJ^yj`zrYg5q4~_r_~Y#T(S1xb_BeaT+3zL-&2jd58Vwmd-Co~4u<~WYar7sU za<{PTRpBa2QAt}V9%m@DW6nB8inorDa$^$&a~^>!1}Qhb3x%0$Tq6+7Dymiix4MFy zb&M2m9V4YWPEd86AcH}2lsU-R7V8(92E;4boWpwH-%_$o1TN9BpkhR-DRHyyRL*4C zm)LDzL@dTu_?~fEMoRI+S*9X9hSC=wQ|^ngQaNK~&rM*iQj#x|n5X+2osp3e8oir|%|8Zr1NMj$0ZufLp^TL310`}K%C1#%CQa^vwyTXjVu~xH zLq8N*08*yhbVp6)6zJ%a11Ovz9KMv3~eZr#2fUZ9xCuXF~&d)S{8H!45jhceXW~+o&=v>Q)fuO$1P@j~MVihU%eB3#J)c+u% z+W||L;&LEpQ@iDl1xU9KoCxp)?xqd0N@(PWNw^_CPe;P)^AYFcuIO5vkupJbJ*WnE zZPasn?~IhID@6IYJ}T# z&2I!qOC7Nu;7HuH?fo-SDmAH*$KcM$6L2X#6PH&(ldQ^(0u5Ds5A+$v4z*JXuCT|XBT(@NZcb@qZc^up zwD*uy8t`Etsc+#ol@})a6CY)j0WJ1*>EC^q!SBB)WBS^D!vDi$%ltPv?06Vci?w2 z{t4hs*%evuEB++#cd{$GI`|sk&Dj-uJNR1Qo3krM12^&-fwyE=90vSqD)9{PU#Y}1 zJ|&(JC5~~J)`4htsNxLZPf>~IfL{@+mb4 z(?{O~FgC>UB6_&`&b6(}ev3pvO~s;$yZh&U3r+I7pX0%)5)@?+d7-Ga+B z;K4SduLg()OUv#AH0GAm`1>5}U}L<4Rnqt;fEm?dT=p7SjMPf^!(|=r2iuN*8=zzG zC{N&VaRQI0z^@$i}@+R-@)MjP+2{&2gwJIceF|uB6tsO1ye^q29O@i zE_)C@GWpdWlC6;|L2*dwGq}DCg584SQg=(uej7n6`@gu_*}vl&$j;lI~-VV)~RPvvaHf>XsAZxR(9&>@c@&v+m)RN$TmGY zi}YszqaN8Inl}{~i%E8v23-j3AYz@E8<#m)G*Hl~>;?zx6)4CnyA9Y$8KCULoVpJf zTg>c!tWC>--3>Le_eA{}8_M3(6bLp>yknUH;b7_3p-m(mfa+8k${tYm2$(~amlp4MX#s=OvJXLDO8TLwqYyvT6OTJS@xVYo zax!R3r{eMj7(;DFe-AJ_G@h<)2Cg4 zts#ciYiL<_hZ1d4Nm&uF50O-82AnZA6q;dlqnCL{baOD6z7NQmSs^q&6Uu)46SK$2 zg3u*_f@8{tBhDmx1sWe~WPfN{x-TwexF1SCY&^iy)KG)zEu7j8To1;X@kpE*kJF43 zz;Y%jQRq6BodwLK{S>qR0tagh6m&1E0k(v$c?N|6W;|oeFwVjqF~h-N#+4vv6$!mS zkK6+69451o9$DnXy@7VJY?*_-87P=uwg#9{;T=3$?6RjF^xc$#i|w+P9Q3`Ef=leO zHy!lTl!7^S*@q6gIi=uIyX*rQ&QHa2GyE*^c}ks)<{0~MzRxJIt!X@1#wR5 z=$`?KbE2+_IelpxH*v8#<@Bcq{{Ri!jvN%eP+3N^VuJz&?aMj<`wHT6P|!M-bpw`- zBy;wqKMR2s6WgB+-^g!3R{B3&`r>|0`r-D#R74I8;$n@=!1aicOmk#=1WUW&GMIE3 zoPX`_z?h=vX> zn+#~OYz*nE9c(OZp5b7J(B?S~HV#%=L)q?_cDU9W`3&TizJklOpb51dy%3-(bhTF` zuch#NaJK|53@xPa6~K;SA+M+Khk<1gTMXeSXp2ojGqnqMq@Xz%3YweC_ppLkRD zzK;)bX8FCumZL6Nr6(iU2!hbOvFV|CWg8JR6>lEITBVmD_C3a?j{ck=w4W3i7#AAO zg!y`036&l}4|-OEcP)6oBU5Z#YUr?R>l3#9d*Etk55hH&eF&~8R_T|ZI~Bo>?rcRc+;;S@0O{dSnfbsre_Z`el#ulFjAJkgxFlfwjX4V8QOkNbkyhK$(xO7 zlm0dcDx!!xpXdOf=@o@Q?Rc6h-s*%TisC_3L_EMKsCb}JQ30>u0V*mH6cq&G0V-ZW5%ECd!TTzB ze7~x$-knVRe|esCS5;S4cURZcbocbkjv>!9`%&7DI-rtDF!bYuM$^cl3o2(mi7rBN zE{Ud?a+xJWiL0Lyc$9dbLE;c^YIEL1T}t3XEii`!+L<{AHKB?K5i{oxA_UNtr9?RH z|EJQ`kiHNo$VPG>NuAhS>ff3_=mH4vP-Yl;^8l}faInwV^~ju{&rGcKLa?e)xEb_U zz!nSxO(}dEXnP!q3U&Y*=9S?4gmwms8R=$l=IubTleG5C-;1muI3RieIn4os1EDi| z2pE|r2$%6ke+7mUUPcg?87?dvAh$2s;Jt(+C~h0d>khK-D=2+tWJ>TE_PPNi^bC1j z&glU2h9fzaSWd`45!txlJ<*}a#}nH_V8iA05JwXhBnnSFG=^ErB4S;IjujOISH&kl zLBtm0oQJ3kam6S~39gA2LNS>)xR%t614gkV_&6{lIteIQBKS0+M&9QToeEkgkbgO{ zf>8U!6uk?{$LIWAVS>c`5|grU^cF~vM};sAON%ZC#&b{1tNrH=gpSy8(ztB~e0SL4K1yOS++x$qazX*toZ zKhTuIKY&ud7!4PO_Pfxu!U#~BU2ti7VK1OG_hqzKVJ6T=Ni(U+D$E8-{b6)a;Rv8~ z(J(r=a01X>nmM=d0-!%@w7|%_5%7G_+O-?yZ`V3?5kMpLexznb)=ZaKMuaU z^+-Me9?Ts3EwYs0`)1+)0MObe_yMj6@azST|7)yFP9NpBPfcwL&`9l#)Jz?M)G~7Z zOR7wqL>mjdeIS1lvaI&U@GY1IzWuS}B6NFZtS!07z&?VtBNv$i2`)<`7nu*N2nTQr&2BS+<`UGI>H$KAedI7!DeyLSEF6y zSrEzk=8&Log(j{CaXYk|(V*{2+4RcW1HxMm-U@wCwnWY1{U^|OSjYZ~%osP5 zly)~wmMfVLEHKtGDF~U_Pc}gBSc`gUlq{yA5lxCla8Zv^`c=rlD2s%^xRMx8^_krH zY}zpcMxK_vR+UgyN@kFcn2eH_P!RmhZ<2-0>}?98gPPgo!zdv(ZVq5-4yy%BRc7y0 z{G>LUu9x6pKV34}?-Tv7pXf#oU2r9zffA_kx#|JN^>9~fd~w~N$8`ga>juoazavc# zDmt|hFt?HF6uRo-!#?ViQSx7EOpAb6h1>}kO*~7JzClUL0PW%#B{q=24cd6b28?I0 zCGCkXEnrd`&1}|?#u~DFsuH#IASQRj{I7EK34@7kyyhRT_(y{nKyzsL<+@f)=utYIigLI;rB@Q1tE>}6 zBLc$|Ol=MfQ1GH|;2EUgD&jv~!3zl<%&^g;N9htm`>LWl2+mSP_^8=RpdF5wd^2Su zb~r*^J&D3dhd;U3C2PQwcEpq`mF;)Lr2Qskzav~=l)MPG;P=|h>;vDcuCNUc$>8oG zwt)6=cA*l{SS#Uj(G2wj$t?aal$hD7N#$s-K~;c24K6Q3LU9NO8->|sW}b)yT{T*e_R{;EEq9la0Faak3=nNpT#U(pi%clFBQDAtNWG@8p z8Gy)r|Hk=^WIwk}Yso^L53qxR$o-;IRMfD=)cHT)UG>pmvjt{;YWg zAAop($UF=+l`v-@-S2Ed$v;%dmjGI%130lInghV)&lx~tqd@OVz=Z&^0oXOrVb_yu z(QP9~8Zc7ZX>`C@4pzQxYyc2 z5i{RJ7@=4Oh(iZY*YbJ#W|T~#nAOWPZ^wmfW^>HyWv&FG=AXT|pp>Wmy%|=|T<1Zo zW7iy#DybqBrIwoKDix*H+i?}87EOSS97_Awi!B-?H;W67hq)dC_Z}gAu8Zz9Bjx`> zPjf7ud|i(+{FOXHovblk%Wz;+St%_w%!jzZm|RV4D@@f&bIJxnZ-Z&d-X`ceG7>G} z0g)^(Qvma_eZu=-NxKwF4YtpviAB++yb>z=hRP!rR4_xtN=)D*7B6c1>kiMoKnK&C zhQha|V#w|W5PKfLBLH|x{G*uQPi=}#rt=XPcuJhv4HgUk2g6loI!8h63p_R;6^#ZE z3P=TAC6ADSHL<0x{V|hm-i$6b_tIhW*i6(QHjkVKo$WmpnX8q~X7uH19KCeXS)I_S zrpCF{ucu7sc@(&2$3Fq(*>NSxLq$?^vF;+xj=e<8j(K|ErDI|&^Wo#et{4!Bp-WRm zVSJF*n_YYpm-%O+tcy8%9=4+;N zJDAjJrp*5xb{m(IFVS2C5a0UAl>xImPOTEj54DEkwh%JkgAbQK4^1Qpj0DU72Hr!J zl3xdX=PvV~l)x)8TH~rD`~&v&ihST_Bye7@)X<>pF`elsa}CN;pgbt`G$@;;F>guE zOBOFt4~mzb7?jtcG!d9-wPcJ&?H^(w&~jJpM|JI4C=c!On1pNh5_RofdZPA|+qi0v zH1@;?8Ab6%$<4i{GYwovP=*Mb4^uTCE!u$wl{TS`vhn}QLc7vnp)oP0mD7NoEt>#p z+XCqGRKtV|A>Q%?#GWb!(tBe=hCXWA@`U378aY2={+-hsU@k!uZXm91n!Ovd)0vr; z-RKOBG5sG}+>e2$$UR|<+F+Q`ApN^_wvJDCLX&yWG^rP6`OWAv`_)0^SFZu*czKZH zWySg6qHdetvp~fYz@H)w-zoesaO$&pNb%o^z2Hq${)56d0)LnA9~qx%-1axBmBPa) zq{A4ZOog;rs-dkUhoA~+Ub+rx9y*1eG`N?9w2;=5C48YY0W=|U{|T5}3Nc-B^AFQG z16o$2%!s^(RF`Nm^N8s<;Mk6`Nydl72Y=P2PwU!F7eEtA8^{=zMNcL}1yXpc=AP~$ zL};t~S|Q#fxtVM^*P&+K|3>Sm{VSxbM)QO%=g}tZ*Co0kW^2oNTiWmZ06=#^GvtcL z*v@FoeyS;{Ly~Ebdyc^%cD0<=2t~&dj%@&tGW!Cqq_XbT1qYLdaS54an&_G*d4!f} z7kP+gU(4wag}ipd1$LM$wUs=Y9ds)@sJfYNWoJ?ofw$Dnd`7jX+g7PP-S{2~ux_L1 zYL=e|)vRa$hu(@4FqNYnJ98k1U%LkFw(F5pj6_LWl0Ds`>v$&p@|8v^E<_>wuqx#H z`cqQ%E0UW-Eaz45@qOJ6k0tX^Qi()GUI1pEmKN+nulMs)*rXj{~P9O^HX4wjrN^%8ZZUV5TZA0`xWTaepg#G#(651)tX$63zl z;Bk$~#XxyXa#0?7Oq}Z($+;a~qHc$mo@mEfcRNCxq*HygQkn^BB7S<+pqq&~$%^fR z7*7VbiNtR}Ca5>1-G(?&1RoNPHgJLzd71r7L}3L4~~6ZQ2` z2p;y!dh*LnqFM((HjkI6&Eusf%rgs0jj zcGhfjgu|R}IaflVnx5O$XeW25jyg0GR(AE~0o|KM>divYaHHhh8!u7!#!E+UoZV38 z+Q0wZ-7PkGdI{Qaf#vj#D3ebz29e?a1C$IO`b+LJsd7v!VUKQGaZkF}&c!+@ zXpZIE^<2JP8&eyE`bQZF(^uBO~m5sih5!o+|~@HAC?N z>hI0e-{Zi@X1`hWw-h+f|CjKxCv=X;R7f5g<00^LR1;&mhUJ!X6*Q^oG)?|=iPmG@ zVL3J6kjbAhwNgYc)hdebqDJ218t!|4pT3{G-GJ{bZxZMYS z{}14AF}5wT^mZYuklwNQ1N3$+NaUv0Onr|T3V;mqPHP`i^}M*XY^iRRDCBLF-wU)2#mGhfHUNF zfDNabsoWTJ0TOZY*43Q_*BaVbrNL!9*EofOnSs(_My7%eN;!E{YUhVJ-jAl2j>CJu$`?fxqQI)*<^80}~q$19{_NyhA+0 z+&sgG$CxmOD*Me86aU60W)u9zMjN*PeEl;2*9Z4yzdN{9D&sZS~F~n`;NL=6i@~nca*Tl7}7oQ0S|H`84-xxQRih77t*(_jb#dL$r z+lS4MV%L*_V)Znj>>6UZiVV*zSGjinud--aY?M`#&YMWfLkeQ@l(mEm$8*rKIzSzA ziD8F==+!KvsR`RnT-2kKPY2zpWI{iI?Op}b;09JIm`?}Yt>7^6+{bYIq|mLZh))Vt zt0MHY6LIcr5~+47e_-7^Rixrx3ruHlAi)8Eu9mt)7u{)wB6HqfvGS6xqvY zcsb-PQx_m#JR-F{JS>vwIF4Ag3qNynaX~ z64jqyiOg^3Mz2Lqo@VFa=46yDrP3$jt$4!KiYHvHctW*;It*^^Jh$RW)e3Ii)7(0B ziPySXv`)3C4{=tFWP(QlFe?N_}#o zDfP*Trqn0%89r{xy9(wi-%Y0I&&MImcavYum52qHUNJ}Y?^1mSlKBe9*ifTNJ|W9* zGug=E1KX7Ksj(;_?$Ymw>mBpI&}rlxMVgnChxcfRiuX{ErP!0sNdw6~ALCh+VxJ$S zo>L7rUb{1*hw(s=r0pp*-bRVlh^F|PjDlZ{0$ba!MuEF1Fn7_CaM}-A)vt(0tDu3 zufY`p_PNyV6SXIh+D%Q=ZgRD7ldFZBTrJ$>Qng8`n)Y9MP`VM}8Y7KsnGLbgwM=?N zEt6(c%k+BSBE3u+KqE0lMxuO6S2lg>AKiRQR~ z^Cz02wjA(VI%sle#I>re6C~%&3NO)|7;w5CAD0Y`ny-YOmOS$`bfK0kFexzGhyr0X z5ygv$)pFIDWI}-B4Z$YqwwFO*%FyI1g}hL5uFFd_ z&ks1S>bh|8hRZObbD$s>fO$b6vIX#6*QkCxB)ORraP~oH2g)Kf|HiPyj9m3E^t9_N zC#eU%pN=y8@y_0BqV#pGJQ|o}&?8^xL~zFU_nNWE{+^dk`+K7xxP&xZ2w(*O{LR{n zI(JGPr^qOpg#nzt6C}qyv|ge)3#Uh*O`HNoie%#yg{xa;&j8K=18T(i19kK|wJ!*{%d^*He<^ zAgM-@nRXg>Cjhj&K&fja80ykv;3z^8`$^QzB6SlkRzp`Qxu}L$6Mr2N)`(|gM!Pvm zV=lqaScTlKLlW!RQ#Ft^?yOQ8_erib8u7nO>U|>w%{eLD{oM1RQMzA6Ftpbp58WcI z3S6Re#>(9_eI})iP1|laLe$QkjHH^91|-~x@D!*Pv*Z$tYATV3{#l^ZUn)(G-Q}*P zK&TTv9kOd6Yd0Wa?demYQB33z4DB_@?FJ;VU2cudivcnb7_*@QxCIIu| z6eqbCUUmiEg?_{80DxF8JS+T#fK34Y24FXW*RGxh(w87*v^JK@VV=iYrwY0L$nV$ z;XZg8gd+OmNIGup? z0P+cFmx(PQ0GSsfsf4(Foau}N@fzUH9RO}2wtWEZ0$^nBLt0CGMTmIF=ez|%cs2+} z=Bxsk3$SJW1?J1djX%4R%4(3r+V_P@kTZ89Ce;(Uml?|j@hEWhXB%@!1WD!)5E}pl zjev6@n8pF0WK<%JEeBddz;6Ja0Kly;s)Xo7(<#c5M&Ad38*NupQ6rMrd!TKF1fuDM z(h>)NJ;jW*I62-D?qxO6mzs7Ra_(g#5mYbJFKXIL-F`3>p(rya7d-@EmZms)&^rY< zU)dJ|aL(>$7*_$Xi{>I}K$3Zs?i&|F;NxUxKZ=8AfN%6Ec(@H9b`U@}0Codd?DR_@ zkVy;x0v}k;$^CJ?0uSX&2(6SnG6K}m9%A5m%b5>}^H3IB3E+GH93nJlOe6x2SWX=n zI7IM8s7rJO;D~(%4!H<3E1{}}TD2bc_yO2m2X5Dc5IYCh%~az>0PZ5-4ge1W;I}uD zN1irMm>RN(P_1_TSsvAIv^*>drJ`Vs_ zJs&{4>KCQAujpQ*>Xl&Ns_Ou_st(Vis*!Nj)c^uVE$1sRvHg2W3>+;v+rPhu2>ZwS z;7%VmyJk33M^Kpk955uzu1mCDGW$i~aGSlJ8iT&y0Xp~HxD0*Y4vfwIBLMDuYuuOP zeV--MoeL!=v-bxB_no$?6MaX=v!IbXUJ1b5lkCg~2Y39FdFmz*yiDcX@q->Bb$kmX z;vLrv{}o2^w*z2EbX=Ecz0~m_LYlkdwP*}=ybtJV&_Rez0>&M`9sqZIHGp`>cgSqu zUCF8A+rhvc{}Di<<97IRxrOwV-jzyj?oW2b@Y^93YmNBEw~&Dz-q0By0^dT$c!;=# zM4>_7LhK?`ZC4}7+(Horfca&zGXpH2QTzzcM+rsKA^`Z>ZR|yqxT0wlk?WAeenP?1 z5aCB%#ud=>O|nBFmLHuaZbbZOe+lUP@GBRLb`=tS;MGrTHAve`j(2+Li3eU=ph!LN zvTLCwlg`^V00`vzoFgcsW4C-NXQZesmE6Q>+pY+Syv|XlZLdU|>r7`TsC?SC0stSd zm?Sb+NX|#oG;xMjC(dB@JmrjQvR-y*?Y1_G;iu3`CW^_4V;iZ$#%?X`qEOM_z zzi`_28x*)6W7`OfJOq^}54E{oT~=w5^Kf{Hc>d@Ng;@N3G%rJ9T&936@gOpGJf4$^ z=7Ex%I9hQZxOffr8i4p3>_ji8tiil=T7&I}pe()AD)(kF;S+jTUNfDJLtHB=gZ?s4 z&%Wp{t?6Y>^UrNiN6_C7(qvEWYU>V(N@I*Z~(Lq%dK&!&jp;2T$xDT z2p~R@3||9f$Kkv%*mxqj38g%dY=J@IIa#P$>~OE-%?P|2@N(IMOrD5%=~xZK@}OI8 z3~HWQM9siFwF)`U;A)AWW^fGvo6J+g*P<*JNo;Z+oD99>$XZEc%u^SG=!&ehL~caF z^VF>C)I9Z7FuCTb`+)L1HE*bEo>~cJnx|4QWSi^(pysKyC<z$Vl3n#;aOnaoQklMRDvZ8G!+)pMuQ7ov%n5>G|3begBc4$(0su13xVts{am zXd^(kK@I(vxe{hbzaFlfs$97NaNt>46aC*FT(d}d@V~BroEuPacrlVXO0sT5%V`Bv z1ArX&KD6Am0@?(WS3sdNWd(E#0A2z803hMGH$fRzK;4JS3TP$(9;6Kb5-Xr0;v>&O zt~`qf;#mOQhOomoqbwIm>=2YBJd4Qitk(H)BZm%Ot;XJv_}GlVBkbkU7loHjBOHbH z_z0uITB_B!MI5&X04=a)fXn5$OMtTD9s&?wVAX(`9QV|y*yLIO%5fX0C~FD2_yTCO zzzPk#N_AzZKx05V1n zK=&AFU!}Yxm-g$=Zt`gEY}cc?vt5to&Q_1(o@{`t3Ot6SN4Xx;3CxcQM!6o-jdDGv z8^w?5)N|BxS+hS7|BfRCJ^4d_soB-;^EZYT7W>G;Yo?)dYUnF^sD+%H}{c zcuD-7)=OOPv|gg#X(dr`JP$jl@0(o0@3*RVTIcaQt?qYMm#TMHd#KNU+|F_krN6Q& zYUkj1I|sYkIoQ?C!LD`=R_&aTD~~NQmMI@Hi}Xm*Xg$N)itVO9y&XHIe03Ou$DRidebv}htlF8w`-Bac8M0=7ryQ`HDF65 zZ?fqL-D+1DV&*4mSGed-=#yRH8iLW`2IQgSTa>1&C66Va_aVQB5`S(2Lconk;E>EXQ4qCDb%eckDXI+Q!|YJr{sBP!++@UWeQ^ z?ty{kxi$7!s_HvUa%ni59it3>yAnyo(-7k4xSyxclUZK2)u6!qBE|U@W&gHp%P5jH zLp`Z6?v-WRBVwb+CD-Q^z4XNLU^W!VvMmc*c-iIvpk>=5D2p%Ko)eXuC8uTEHZUZX zZPgH?W!oW8dD+(HTv@hl7n!dm=VhBFzONO^vaK0imTg`-EwKtvgL~QbnWrMXY->iB zWt*2yI%g$xsZ%f+4hTQHg%xHvMmZOUbbBTAiiun?B$eYo0m?@wp$_CWZCw+n6SkX)ggLc@(nP> zAB4ew?ezQLaJ18&cZ1@ZCb{{&dqe(qaKsKmn72trcz8qScnG{r;&_PICh0QTwN2un zYP%WN?_O~xdJ7L$vl7VqzFakp_EC9&z;nQlK8B_ zI*9OR4L$(CpF8;<0RF5&=P}p?YY)xg`;p`#;m;Z@l}0}zIepe(3RwBG28#jkXAK?) zz@IhP2_XJi11-Bk%clFBU%|ldt+yNNdT+fMC*E5>4GjF=dKrNDd+W_O@!t9ZFz{y$ z9s;1>TemCGg;1?jy-{+Wu6c>{S%Wtq8vm?;mjUxO=Q}X)XAM$|;}f{h0CA;rCFjo? zXyR}o(&tY4L5M$VFctuR*5EP#qdUUN^tqE}BDT+Y3Jm;NgZ%*9pF6Q@XoRvJfIEyC zf-4n(c?8ei&NGbJz?}!i;dB83b4qZtA>eiZYXM*>TeDz206VJ&Nez7ZMI_hYpEvlch z0BD0htxWb=TP{|Y)~S;7K5GXLkv8~akT~xQG+-fs_=c?sno;rbXD5*x0%L{~r? zuG_%j-f(T8R$;^S5a_(&8h#iRojO4_TuT7(hU=98;v25d%MI^E$!Wv&0Wk3P$7TR} z`vd*Yg+99)2^*mffZpJ5Bp6kH4FRq?eHE(Cn_P=Nr{Hu>Jq`5NV(Lv~ri(HA730BD0r)Az^l1 zqVIGi|cjkfPX8^9p1%CnMO_lKD=*nu- z&PC3936(@pdkHlFo32ysdXQrGL0<-JgFoAN0!a}iHApf?}5y~PxRTqd~@3~Z7o0BDnN zGZTqj3*~&zZ2(~YuGY&?GGUSt(azsH)Xoqb58;w zQ6w}`ZmXBO@41>d+jY+^f)L+x=K|n+?wtVQ_uOV8w$FJ741CYEW;DI$(!iHIN$cb- zLDw;IV9%o@n@=5X!Dpv&{FF}|zQJ^(Y(0q_pDZVq0mTnT1EvGG>cH;t>hvrhz&$>h zPNJ5+LXth1ZcNRRnepZKD$|<9uT$}^AYPz49!=$Kgk3>WSiWhdY$NRAHk6OKK(-Nf zar-LlSyBh1d42c_yF421F#_}~dn%r1w zana$XnleYfrw1w`YA67<)D!@5OWkz0YWY&h z$x@5Kz?OO(fVLF+-w1tn)-z%R2Y{aWR1=Ir+6e(wH3%{PT&cPz0Is?aK)mWVrBB;+ zuTk|ZFmTn?0NhnWpPlurR9y*xWK=xH+8tC}V?CmxVm!Nb^3FJa(<9=KZuaLfs^}-Y&8o$l5hXSbGC$lw*nMFDUH} za=Qje>?OCx6!F4Nl533=rZJ-;q40}Ha*^3ZVBYP!#* zO;*#FAkrBxL1=%_nS`Q;fzLLcMyjTTRMan-PebvvumJ>~78Y&9Fpu!_Y_OJK%m%NA z0-g;HohP%wtd~)1EhSm65JZXbDv}~3>RB!Tnhnlft~_A5GIP_x1Gx@NP%B9LNtL!X`vR)JvGQ_M}^A^L&2HXHl` z^!RMB3e0*oSPPg;(uh3P;c_v_U;u2A7=XA*9u||V)As|IWDyuR)T{xZO=7tJW>m>- z*;k`1X%^Sh?fFO$A9R7Al)nSX&xZgy62xM`D9@gc&)zByzGbfk-k@-LTc*6j75FTn zt9&u=LxdmYTeO|E8AhR(6&wKO-$A>`Tpqmlghp z8hF1mq^ZZ;r~1M#D`?^mS_CgEcxha}muT|K3IWj-*GVrcc!~6~LdPrP+i`aK76g{< z52#k`lblcfdx`Y2!Wk~fB1rNJ3RM7_zMzo3Qt69IPA@1pT9jWbl^cjnqtfK_TmPR4FegbdZi7Cpo>K z@DGIIFDNu)kQWrZbb3MIxT~7JpfF67=4<8g7ZgT-Grr5(j7@e~y>!}Ty#Rvzg2Jr; zWV@ALP$-f*p4WBJ3kn)}O^9qKFOgnQcm~?y`?Bf<1u_y|P^d%B51+zsfFLg@6akip zPt^eREib%P-9@UA+YLx!yP-FJ%S(Tgi)jSyBEWVXCFyS=sY0S28v%&l^4?bJa*^9r zNMbFocB!jV#iR~{OFTu}L`y9IeQPteD=kIH?HVMplSK>PEbH%A9=ccZCbu@$ZOjNu zoVjh4;x&tZqv#L_NxYd3g&c1Jp-B)a{|I=>&&T)&s7;>nF7 z7S~Df#7m@j@-dnrZe-V^$)Rr_Ru=n5a&|y3k>bgp3CYm72bIu9$vL(dJ79q(FF#F^ z;@Of@Z0TAh4dB?4MKXvjLqSc%mSw`fO3NX(Tm}w_EsKP_RB~>zmq@YYUR@Xb%0W#W zTdJu_i7f-A(?cbv*zzib;<2R}gTxjuonp&Z5Olx%6ceSBweomuNx8-qTbi*+Z1K`5 zwwwq-jxD1AxMRy~spEZJ7sVEhd?7@(lb1-bycDaQb$SnU5N_00Ca4rBp9>v8ss7E zopa>+j=dwQIYOK%)bk|Q5yGyAtQ~$&v=;%;F{hefXs<_Zhu=qI{nx6-vc?}DQVt#v z(`k*=dik4CBee}&qhn56iXf>v<_w0y;}s=(J7UhOz)w&(Rue{f?sYQfNe%IwXESh~ z^Q4A2CY`xZ<~(6lz7;sndBP!*w{y{Ij!~5W%ws}!4RVeVb<(o8CD$>kkr*XL9g(u6 zM|A50&W+dW7^SHLq?BWnCWf^LVw9K0^?QjbMx7vHah(*SyhMsoPoNn(MrD0KqrY0U zv8`B|9ob8y7`4kK=|Ga>QZ<04acRz@O5YWdQ(XE*i*j75CmF=0wu{_xsYdwMX*tBD zv%n#7X_=7km7JUDB~o0P4H?}*V>g<~FJ%JIap`R7@HojSF5L#9cwB16AaTh{r?|8p zg6_C9Pn6Er%HwhA6L8Ku9(%6zCEjLi5|_Mmic5b(kmFLf8=A(Yg;K}2x-N=K8aXUP zwv(4gap_EGYZ{lT$ViAw^<*`QOX+(+kU5V7SmII*KpmITKa@a%+-^h?tAO5kT*~^0 zi|MMi9l&-yC0QRMsYarf`2ciW3h!0wijdpYNMaAT)zzzFQilcXCrBJ5YUTt$$Di=0 zN{fTsu0;~tB3d~9_{B7>B>$KAb5X8b4Tqo5S8x_Zmk}Y8qe3H;nj=GY`ezaq&J_M} zI$p`lr4kG}D$J3xD>M&gE=SQYHFMF_C#004f+k+CMGzIdG>!^hqKXRli&$JIMFlUB zqC%I&ac8h=(d5uZ>G%7Rvm1Gd6czFklA+VrDr4O%IY)>FvOtrW%b8MKBsoQh$yyCZ zi16o-L4>#g)Wpo?65(H@dqN~%(3E)jKA&drf zG3Fw-tB}M#hTeFDs8z)V{_F<8c6c8R3ni6E_}Z=opd&;B!L+aVCA@Urt+EqwY@xELWa4~UAz_D(?r(FdefyWblB7fwfG}}dY5_EN>v`1+*ZYSnc zKmGo7=@crS>R*m(=#N-b`2tlwjmiPH(&ehOqf5G@NNdRfuCf81vO#9wu_%OruaK6Ty~nPY*3<%ds5)i30SohN-Ig}B405GrI!)B*vHSm77~0TTM$16 zvxKk*6-G5CzSw5?)$g$00ggy|$pggLXqm&oWMY1nhqmPwHBXfJgKgToQ?XvW&VO{wx~~TW%{qj``%xo zwrzgAk5V_U74TRI+_xhEw*%-w!1Dn55pdlS!x#d<&ix8WH6@Ld6z!+vYf26vsiCBS zlJswIS)Tcns=P?@NOu_4OEjnZ&+h}evwtZH2}r&b`>q7^22ch-{n*D( z5#uz=>%vMGoADzGjL=?>^sMPPjXzesxfPYupv<(jiEa1A}FvMC@J~@NgXBOLt>yJ0A_13(23Q!I#81{ z3IJQ`LI7<3s0@COAblZzbA$8QlcQ;_7*ennD6Nj(yF z3*$F5Y=AUu2Sm8Dbnu%Sb_`aJ@y>GF7C*1rwnTCWtF}!P!bORKMBB1Zja`Lg?ibXy zv!M2S0PH~p;t6B5M<7@#53*|^!L`nmvdRPx4QK+cBB`XL zmXby!+~4&7p})sTfA2&yxW6v~;QoH z{5wP|k<2#Rpa$O!TB(07utckr7{ID>0yNcR+kF>P_m9Wib0^n{h1)#c}`2cFK zBkqkpXDi500(ZUz@CE_>?!eI(0I@3od<$T-ExU);k;Bky)=@jmr5vd>sJu5DtFwPX{dn%OgWkGkvIO zh19r8*Nju~>%qXYe}0*{k%a5;tzkYA|1rw=O=c~yQ3Ux&ye2-WwaJnAW^_3c@1@i6 z_0+qX9*KWObm@xpk@#kGITG)slg|DLo$5$@`cZU>J`e9Y4_^W%J`Z1n!cb6dv1?r6azTYG(ISU>R;Q+1U!F{kv?QD0kxL z3r0>1rSeVQel*K6F3NgEIqn~lo5g36@M`?V@tHzukh~c|ezWBXi)Q2%i_tLt+^52u zakGDEooDOnp11f%T$<6vzr1wvh0Ul)UbzUBik^=oulNRq{QlH>{tUPJ*fyDNJtmL* zy@ANx;w25_C0cSY`a0A_|)f$3G{07KTJBo8vCl^pc-%5PMuwwV~Fop)3}`FQi8r}`Em9^xaAr{5o*AZgE_I0i9!xuff1WH$dOK?gx+sQik-2i=+m9wIWjd&Q~T) zRt)n~vnri#pGKY^?R@wn$aKj|1-KTVudMq^$dXfr4-qX^E7Yrh!G4S;hM`%ALn=Tl zhP-LD3Hlr{%AiT0aOX0Nyjg%R2bXzb{!Peo&2OXkBBw@}-{ZTK(N!Azkv@{WPGdjg zBiYecHTD}m-yMAu7`~5bnE$gf(xW?o;hW-y`InW^tMFr8hNff;GV<_>ph3UMm(;cU zF_202=lvIyufdfZ%>NZxIC)~!{7p3^*`bCcYit%8Vhl+Y)U=Kb`YO*+<2|HPoZ2?% zyG&CvqSg6A<19cnR z*6JGcy(}Za9WKGw&{JL>zF3#12YoN-;kysSe@Lif(D#uLbNskRcQ3=BW4*c-3^XcM zhoG-sOKkvg0GO=KLEl?iinxaF&?TLMzB(y6vlOPL-%d^$VC0hSyqQSO`xNiFcJ1yU zOHLW(5BXC+L_3YtZ;_g*N03^nE%9SkK4X->#kr{lJlRN%AT?75AhlA@MoNWt>(oVL zqtyG6nyIx&t<=|%Qen$eQ`>$`g}srQsY8%j#)SQ7Wvlm1v(F$@(CYoeXpWI{gjyEx zFi0vPqLRR{9R>ot~c zWps$%uCX3gM#t!V!049>Tb)2Yy9U?*EZAH1fzL*t0CqE)(ds0)?8tDdlcMx>duk(5 zj3|BFUSlc7IqQkFua!|y6mHcw`W%GcBFf2Nh49JIS3zNRqD0?xVJSv*H_-nPdw<2= zKT#J^5_JJf!7<~{z~7T>G04go2layz>Opa-2L}F}QQ*xhL9!DrV4gH$53;w~nqAF8 z`YbAqn%NCkrcwB%MtcJF7p?}n3-@7c{y|aU_drvj&+G>@T=+B4r=ipwWco4|+U=~J!bGSMWc%4~zj>gM@zhxE{1E)bXuL53g7RG>+ ziIq<89{|4f>3KwhqR^T65h0B3+e{LgrtMD;hemC%gR^fW! z+@2cXjaK0njXw(fcdPJo;17`gCxKV_3V#6pTN3bE;B~1mB05P%BxeE4M*JB}JPyx* z>8;elBfuXgB^!a)rxv#SMDUk^f0SC-MdPmkZ%8dXMdPmmKbTsW51jSY0dGt#JQw)o zG|roW|4xnA>}tej)rc}Jv;|C6;li1~-=;=v1->|3I1l)gmcZWzUKcLB5_sJ)z;_XT zqLH^3HQa=rwf5)Vjm+P=QS!iWUixvUP#}(7&1}mcy@ec)g0pRmMZ=vbJYdZ)ioL$ z!Clo@5p{JVFm6aO|E$JY(3Mw=WDDqRlJh@Ame!^up>Sdoh2^C1d#z#;6dE~8NyXhr z{tJ4`r2K!76|@{3rQZN~8swH^pkidWi9{I>w)WCbE=cLeaC4le-+#f?7|%8OfN9uw@Jy z4t}2EpxH%$V<>JJ$e)cYz2%wFE08}x+J}?uwZO>PTIN&FZvjRDv1QcCu%jz92BU|g zHNajYIhZ-k$Xg5WQIOjB^It@k(k3;!4u;`DIu@FZ9J)iDop%vj@*OaQLOCCTkoP5$ z&WPY<@BH4#D$Na1x&d-jdWBka2C!thvfhAy7=w-hsC&}|@B_Hxn!WPZBTF&!q7O~Q z8_~E1O!y?U=YpexIC|tSKo&M{i{6F2nDpHaj*(&Wb{?k7G=+yrV=zPx#ml_KNPdFg zvG|-hvXo=PQT*l)_wLx%RMQ};l-u>Dk+&Y)eiXMlePl|j)1&V;$lZ>9X~ihmjnXzK zZ9Bj`Yf#&h5XEmWjWcb$*N{oEr7#_3^9k(>G^Ma7&_@ZS+fKMJ6X=J8o(wdta3Iiw zgwkCoz3>d6e-L^K&|ZbFVXwU}qEOrZK(maT5hyAci)0x1+FIrq8WXc0rTwS_DxtYo zKTc@ujhqD}bu*H4Nh-x0!?zrl6yg08@8iU~0f|Grsm*y4)hK~awZI$_XlLde(u67^ zM9iGOi4a0BmJ;E(|DQ@%L;6FYAQ#DfBz0nQsYIs}QiJH{!1Z$AO2BI&Z1x$u9+_kM z%)}fZ1cMrdn?c_UHZu$~rSNT_oiT$lI{*#y3?Y+ZerKTgNuM{tnfD%&ousvA{$6AS z=71>uE&*Nb_?1fd`ypUt8a&L%@JD|I79f>ppaNr1Yk=IoWP=w&M^Ky`%IgWT4=X40 znUN{xGwg7ulh8Bda5*^u^9qq1ODrekpNMRnc~5jGa);O+0voP`hj`+|f<)nYnZ_`w zyO~&5p<_h_=BoJo(0mlE=R8JbYXB9cm}{biP|TjSmeh;`M)OMZabQMt5>T>)`81(M z&SxmfJBZ|KT<9&5@_$BF)nZ(fel+&}RQ!&6DJk)Np{}+TSpTF%TWBmsk*Gbe-L1ii zpAtxmb_RA*8(_E#4vKcuoVd5;MtcMM59z!d_o$KK7MF96sc}S6J=PdPbbqjubF{!s zY+UqLYz1UsI9pt6Wt2tFKp9uM*vfE>oQI(`?jSmt&Gc}uK?pFe1s{E8GTKoANw+{F{9fx zy4RO+tr^{|(Jy@&*O}4JG#`*^Yn@a;0mf=9ta7fZYTWWgW;En@?$jANtiftA56 z+F{U^FA=YJj72781*? zNKPe|pu|9Cqr}L0gcxd(oJkCU{IigS+jWZe?2YLp$n82)^XX^47>m&LoY#PY|A4VY zAb$d~^cEwc(~$FMokjeW8XHO7yXq z-$Se|Zb^9KQd0gXsAR7e_mJ`@fxS(GQbWq01$Gr-4?{V8>|ypXzOX=1KBh7FSbTsH z4thmjf$%pZjF3>kqurh^oF{M@8F?R|pbiWzZW@`=;wBDOe9heiwMO1ADBVk?{`~#O z!Y#5{o@N>?5Ce_qKVuEm=pkgmM6YRhFkU~BY;oRcsBbDFO`Aae<;V)!v`zN;n+71rl_hYJ;_JT68%up2lE7n;yMaWBsV3t273W%Zjem7=A=;P-5nZ zhUOZB=*GaqgYW7m9meF~6TQ~|gjP3wXl>T&Q<{17=}yI;61B%kk766S)fY6j8fjp} zDCr9>E7dfqrDirwnt=5=mVEJKq`5SOnyU8%r^?=-SzJJ3?Xb^`A77;HRlu~95H!cI zigqT|TV8*+Va!mR6aqqik)JGiD8)mFG=NPLkCj5JBM!!l`4%NcAQs}iPLwT5{<*1XxH#G3CjO6DKH z%zLb5@?433UJn>~zftxXS*9zUBVmhlWsC5O-k=goX|w^2@8%;}qhv8@8qq}42-eht ze^4xCl*N2T*`K8MGB}XZdl|b=b}MYqf=YUEDkqJwD;Wb}{9E7Pciak02c|Ro^@o@yoq5Y1s^NF)h-12=~mha=Te5SH|*$u?LC~nY2%Aj{E zsu=WP!fw(AElI=Pc>4iOj5EMxoB?8-ry63?RT)v%lF== zzA69?Qc%8drTSi>#v6e17~jcbTrv25L>|(35A#=C51jqvZXbQq|>Np9Wevuh9^?C@WBz8!&|R{^kUe9n*u45KG-=Y0UD0EnGj19u0IX(4GOwr)PB z7{oB}kk&X}G)G2)I$RJCL#A@)14@xWr8 z9z;(F$OSMLKx7A!@tajQrc0hO5HE>?VJ~qm2Jkfi`OCZnmDNJ2^{~%*9AyiDJ6{2~ z8Gzk@5j8UV59SBdh;~+&^R9I>8Ll&L$sEd7ybb`FTKr1b_XxH{b`o-1gau;aCQNx zBft0sfY&buNJY7lTT6Y)ZOT5UOa8QYz&_0tuupRZ>}(aVr{+rZ`RR2Xh5jN1>!6^# z?PC&w)~N_|A@C&J%1}yoi1H4rB_gd;5#|!$9C4mg`2T=&#Cet@4o8^lrM_j7TN~g| zWYk6^nFU=8BRzl`Ui3Mg)?n=f+?fnuB7oSF0A>JShr;iz8dkQ?c@IQ(D6R2D(aa82 zBZwxFL#3>3;!sskL=JT#$m~$HAjC!iBL~u+H5WK}7RQ%GVyxwoQ+!zg297Tq0k|A0 zLJsw(G-I3ySTFmW&%wveWj?N)E2O!yg{wKCx0GlOcDlYEqINnjU7XHKCZ{_NO<<>s z0$`_`0l@2Y=SxL1CAT*DjH!)+9C|}LP}=`6GF!-jmOu&L!nP;}+5(&%Xp3^7u20}Y zefarEaQ^}KEguWK;}nGA*BNJr@r&NDWXs zisvECRViG}3FT6vIU*G2CwquG&U@(+=e=Z#^Y5Yw9Or)n;4&>c$PJ=j_sFXEIqjbm z2RR)8MW2x7dPg_D8DXQ8XbyIeIuB7hh?g!7;w6)VOh6OZL8<_-gFFnt)eE&^(9PG* zV7==zE^3tO7rd!GB%3^Bv@gCy9j)ABD^#*OjOO7Fn&+}JD2!u616}mRJ}31l2?^r> z#6v=Q5=s#gt^$!m!f#UZv0GKm91=`HG?7BWR!F$+F%A?_Nca|H4hc0N#JWB$la8SP zc+yb;AbyuwCwl87r=V~P7&s_A4ImyAcp}w6jXlrT#T-l1sMvd8h|iwVlTp?UW#@7? z3Pj^H$+;V2Jw$W^{O(Rb2|59y>H^TdW)a^e|A)%dZE}q3Hu<$^{7rJ|K<8TNKn{R- z2aKszGL|jh)?->qW4fEJ`bl8rtA4i{*2jVKRli$Z^J$K6R87wD2nvl-~{8v0t_D(}qZS z=5yj#E&MzkxdJ#VZ{g>=$vMx9U)c)(3pks?_R}J`;+pj`HmQDI5WNVTP12hAD?SCz zW@)4FXExx05t*ZvpBKw)$}rc>K6#C_|8~i(WBt6is3te-mHbkwLsIm`JGxwcLnbJ0pLUcbpCq5K%zum#N$Q+asiAd;8Fl{3Ah`;0syg>0o)A0 zu7N%~Y|G-JY`dy|rsUQiKHEW|U5h01eFzvWQU6h2<|@EN1eg1ql$WrbM_Nw=uoVDL z2WzRcz~>Z!$U##py;GEP(7a3#O{CcW6eL_>(`cn)|GOY_;4A_mmb_77e{TRB`_BOo zkNuCmqk8hZXrRDgXmrz1$rkL+#*HWp0N;7>t^-a!1;{;_3(n zhi5Kbg>$y#);~UH02p~{I}QNPTr}4N;R-b)jFb}1!84b0Jw!co@zQ1H;w4iQzXDC* zD1JWxj^eKY@LmeK^_S{i;9WZYn^7&%=#_X>iu{i8dBkrb92sW$erV&U^aw|#ijJ>h z9ljSFkNJs#Bf~u+`>5pB8b4ovPYV9Lra$gCN(Pbol@j!_S=CBbWlaAM6|D=zx7XGM zC=eQD6-2)XbhRmX5tDnAE+BGM;1AW2I;o^`rw*?-vN|1J&p^YuBkNRnovS;tj%yFq zi+qFRkzgN;vLJgTx1RGmFGH|B%B1s}{|C*o!5=ve#lH$ZUfN^5=y$$=XkV0t{`3?^ z27{U^oC%45V{O9WNs>6$Qy4oQC8JTo4WYZN7Mr3ge94dPY(W|&H;bul&UgrOL#`A0 z_mXo%R(OckiveeqZpbQ6VeCy;LuTlDBD?e`tfEmkCm`+=u9HzXhX=7DA2=P`v(62~ z-RX$bkR&>+(E)a+mV&p{^sxb>(I`%OB9q9nT6vCyRWcg$C#Nkml0K)tr_D z(V@(eF2rzrK%N?~=<#gj(t%XcQ(f3)XA*LReou#4j_}p1v>)-4_R$mf1{ZJ%>P`Tb_=pnty!kvoX%fD*r(;BS=pMMcu-qW83; zJVnJz17`V4Fcdd!sdDKPHX8&hQSv2?*1jg5xlaWTUYhI+@PjBG(=T~UvwtIrF9Y=W zJ3HkF;I9Jo5VN!k!QU`XX-|T`W}ecMT<{=*4=_KTx(p@k5K92#<&T|2hp0I8)co4|>}~(BzK(1PGWH$`|Knhdmit> zNr~}3BZc7_gQMhWW3ANc#|QlQl*Q>SYY~exTb$k z^$DFmN}=IDh(11v$C~|SBKr-zfNW(IvsQE$9o|D`4fv$PJQ4vP4VM~;fLi#nxs%mO zwj_^+WtnCEI)Fx=MV7e+Zh5&4 z;7+xc=0O&izn*@VYFRo--t8m(M%f93l#&`Imohn<8(*p$Pb@LrU)CB^nwaf;#a1?v z*p|c_w8Yh*CCs8e;jZ9PuI*N7#FY@WRwU^=)(V&VtZ=!{-7fdJ$K@LLF};*sW~HKY zxKMQdAP8T+kEpWY=V3q}K~k@LGo90%T!pohzt3>k29_sG!2m z_q@`?&a_@hlgmthi;fjs4BN;(`C_&6!KXJaRtGFf9wPglq0P*LdWOq}XShtBqfCC0 z@xHpZe)6%t$8Mo^znx^3+i%D{_H8z1d9STvMY&?{zE$&e!OUBsu_H zu^KqX<()}%uBhT2;Pi%p^^WQyU$+Y8lR$aWl$nxSpC>s#L59x)Wj@glRtQ77A;~%B zO?>qkxN`@9tpvOX;Bx{#1MnSynE4h?+Y>MdfOQIh@c>#8un#~-0CpYp*;TEv7H|JH z`bwm%HXzj_iJcA7sl;9hU?>600E{AFJ%CaGk;jp2k>bB< zKY%g-;GK?al<<^DZuLlVlDFec2$VVWCDZE&mb2D59RQXB z;L}$JMAPxRRV%EdB**s--pf43FwSqFgwI`(8Y6Qx)HPCP_9x9Y4gi^L{DL&r0|I{o z;0481qG`F-7)`P(kA=}2k;F!TH4PEYu53#rB(ZBi>H$)06@Y#OSUd1_6#|Y2P)I-l zfMNn>0+9Tb!iWLm>Wu9;xv3C=#%hjOlGA3V zVLXO1X8?d_sj683HUZ#K%JztuHh#Vi3_MDo0gzGZda7aU5#EDZ{oEwS*oDsyqHL0p zhcxm#^lG+`g$?o6ITdV4#PH6)F?@^<{eodH%D^%eeE*NVHxG}Z=)y<4JKJQ^fhZ^- z4ur)8fgvc$5+Dg+0ud!Dpb21VPzWwa01-YE42l~fMg{jJhzl-AKtWugNF%H(` zLOWnWg|2Ub@+jOQ`v4XI=wuxiiqV;pMPJm^$qE{Axg;rgfq=Wk3|AtkN~#?*6V-X} z_zvu9D8NSmVaoj)xY`fIssBO9>3RylOILRZsHd1V()B(Fl=W&GfKFE_l|gzFg!B-- zDXrl1!hc=NNW#}xyXQQRVf!|5IVh%cf1(xsmDEEp&hA8YHhdg}{c5`_9;tXlW+PpBaFy%u-}JJBWnR3 zC9nrz0RVTVgCvN{#dO|GR4Ly|XBq~GcBU1i(y~N#DWu$)hH4SYP2zEy31E4mx(SRE z6xE{ur*@-~;J<4((yQ*P0KHS`w^;ybK%V)B7ON~XPW>uCgE|Zx={C3XU4TZg_{$oJ z>1*L}Vt2IV1RaleJK8+n<#0OM6i!jYzro7j0iQk**NHvo2}Wlg_7|N>oO4-j4GY1N z^r;>pcAzUAuw-K@*q-HyfE{V@nY4lC&>cIIbUo?S@qB`x)2$CHi=J62;AYcZ@?l-! z)bt5CLPlXWeFD={9I(hCKQG$0j_Ac`L@!cA9XRwCIrKrq)W--16MUJtI?h)(&cZ7o z#)&zZ$xwd4#*-$VuB?}t%s8Sgb0pi?X&F25e9B(IV^khC*>CBY%bm7kJ0&5`uD69%a&(^g3AcFL)P4h#r(wzx*P0uGqNDH=}ogaqv8w#0%%E-@z z7601WEq?$oRVsGvnLh&7e+)T0{{~nCP@G-+TRE?RITbSZ)T~Wlt7Z);7TVe4km%yMl|>p4f0hb%pI5N^dOqb>31^xekMS*s(2FYji3kI6Gy`S73_2k%y~9I=TnpMJ<=c8 zMm4nLFkIb#V1F)W>Een6c8N_*ISKY!O3R<{)HZUr8Z{$N}Rk^J#s_-@JAaxtCtf$&09tZE8%LqJ^* zKTBa(^8r=?gulp!|I^W2YU4-R2Ve=G+7}ZGlI)CEzk=2l_V8!q3Bo{roKK?zkVabl zU|IhFK)DJq!UMpsCjWclZ?GeIbj1(l|1@%xGu_ObW2fXmFv_ za$aVP71A02xD!i54(oCEjPnwR0SE5c1Fn8F4L|Y+)ibXXT@~k+DcyxI05VQ(kBuujrwk#pfpo$bTVCHf8c zP?z3Lxgp54ffckSd`yJdeDVd0xwWUdw!5%Y0sEh}U7{^)ye( z(|k&v=2P-?t>jgu(xNi9n|ITA3Sex2hiJvy(Y`p-#_HEm)@F1dl%r!g+>+N}=TaQU zb-LuNE%KHa@2l6PT(3h-khHrcx);TCCI-|9_&Eb^stlkrKx6?xFUpAU&GZZ*zMBJT z9i%gf*Y!tyzm#|{0z5^aca;st*Vwi0}&@%VG%`0W}JQBXA3^ z#-o5@JUI}#2f|ITzH+4P0nE+NP6>SCl zI6dMikl-^yUn*vBL5v`DF8mqcA`|$Wctzj#&89G=;HoLq$K%x-kaKzc0>I_faEGtF zGSWdqd0hx07t^f(UNN2gvrc_$F{vEpgTTeK5g=Mjal552un7NB>TTM9ww9^}y{;w@#XqQc_fou*KH&*0vW&Jl?D*Z7Us=jy4D$Zkz5C7nN|(I`vekxh(@9_R!Vjtp)wU^!>;PPIilAT5T!*0Ct{Cv`D7OL`mvzIQ>HFm+8qz z5mvbNGRzy1gHG#Yl>xUz#4yr28PCX6SX%(NWpH<^D4X{~)PdYi+J`IS1cm|hBa(F{ z^#XWArbBcYiAqUSfk${2tTg~mT(X)E=ee+}4*@CwMq00tbvLXl0Qk>MKMQ^f=ut+G zN<27}S^#`1R0a1-C>;Sflz+{i&A}Bw?gd9-j;@yMD%)kUZ3|^vn6hmm*qm))oo(;I z&gBy3Y%2K!b{^MeB-0c@&(At@mhvItjI$9^*JRb?cXC%?~f7E3;1IA z@Qsrc0-_f%eIK4*z~vA`9)^v_Ny`CvoU|Fh8z;4t6m=An#z|JKyn@>Rm{%|tcNOW{ zJz3?#;TVeFw=}@zel6r&?rH6)4Dg8j2oV?eSqG#7htt7uS^hj%uM>ZZDN*Mr^`%x3h>3{rpxo1J{1xMvx z_`+^v=~Q!iC9eh@@Kk-bnkLYJTlH+N@ExMvUWazOzIHwAIO<9B4stV}){2{V;G6mG zza-yb%iOtnSfSlog{MI545~9*9psy>4$`w#KKn9AANc2q-(@^Q)i2RYdFE14GZzpoSEglLxM?5i^`k3pj z913nKTS|7okt_Ypy3!B+3tuH7_sPlMV6Ig%8FtomN-|AR^$x})ZZLYj|G#9K?}yI1 z0$+ias#qzeGc8$F!T=~%aR?oJ+=SVX~H1I8#0&I*l(c|gLsYPZf+=e}h11UR?BO|1mD zk9-{jSU{ltAh;7Jp>uySGICNj)>0+%iAJ^ZWPXtw`!XYc zFgfx&Vp|SBydHXoIJ_<9u541Yf+u`8s$)LB_vXsQh6c($_GKKX@TK6%F-H1+{dzP)p~=7&wWs zRXBuNZ7=C0c%K9|>LB#eQ!nlu^m1D-E+>5wuNU_NMZBPsoy_Z>+#lp{f1npA`}o>^ z7Po!tx>8ZO_de50jUfe=m|p5zB$j)rCAyb-0YeY&rIzSkYBTIS!dk)+6?Z*`b3C-1 zxJ36_m9TTKwUiYsE;+8hqV6nBrfs@#)i4XC0sgXN^&Q-PfIV^)U>7;K*>DQ~30O{? zcal{nbm7g0gEJVQGl8c8dJ*^sU;u&hFh(2#z`fbWlEAM_+2kgxYeDdOvseQ4W^+K` z-s}wkUvFjtsWE}|PYxQq^B0|iJl2mB%U5DL`A8nZ;+f-703P(8e!@5Cub~uT(4Pe% z5Bg^Vcr(Ye3?B3^hAfngoZ*?{5DEQSG0n{J2Cb6Pwq90%VNX8@IUac zuh%FOhlOHNuhAX^GyHe6p-^z?z$@_~^XBAGP{(nuK7tk8t3uujg0rAUa4fpx_gHnRIum3>5 zjINo5kUsLgfIikQAl_Q}ga@2ODc*8Z-=y2d@%PgW9H?dI2|X*+eb+LYIw8Hs&fSQEZ-5_vwTNz z+vy{?!?#dO?5U%ZrYXyh`JE ztXDXzHEu|FjmGJOKhT(SbX5{>*%jV^*^s@%%!Wt}*jNp|C9p##P3OVCeQuqEp6g5K zxxR#Uk%V4KQEvAVy4{!1?Y@L=_a$@(E11a%70iXPBZ=CHPV{b_2u}1KjXBYKHReSB zqA@4>SB*!JHv2g)J%`!F8Bla1$@fWS+(MSrIu{#KeHR;2eHR;2Ib-y2Dpf8vd`OLD zNS)qch_81T;_DrT_m4-Ky+Dz#7r2t!co8@8LhdP0(YX&9%b}u; zUY9N1X|v;I)K*UEY#P-4L-BNUr95hD8l?7Jd{w-yM{P}c`>}XR0@k+BEx`xX!{W=5 zY-`JY)H$*5hMik-vmg~yal4>xQ6J$n3sNUhl96KT&xAD((o$7$gy@qb`nYLb@S34- zew6|^=|S}@eDLhYmFg43oS0SwWW3s`>xN?M(T41XgV2q%weI8&8fLjHZahX81L^(E& zj?$#}z>70iPSQ|r6hPpX;Su=({ti=t(X6EslGL+pbppZ-u&bhzP_Y0a(*St-!nw&g zT1#ORWJ;r`25{O1)f*skqnIcKI#*0?6lF0&Y7~FNhp$nj_0)}`<;l`03LuCKfsGr* z6aa1%PXl<3Vv6MCU1CzBSOWq#ik$#;8bvumgs#4_qpB^WEn4?iadn%-;E*z;h<5pa31W> zZQ?pGxlJfIaGNM6hHev8fOXnLEd*|QA5^vq_&aRcL;)md6BH;ni4Wn(O=34dv`M(- z;JLMUglFI_RL~c7#!673Q*G->*hgA#&@-Az};pg{H9)ItGy6% z#W=r-R1CKkPHuKT=y*M3$)nMM_Bo{U?P4+O3AnDrvQj75JZXuQQx#c;piRGtsU%|K z&Y=Pzb(F0(!kKfTOH;{-D*z-X!qhp{5Oa>L?uVQUq7r}$BJDych-?5Uh*Cg*K}Zkd zRzU8i=i*U;NBAZLY~`Whb_%Lp2yYJT>g>~O>j?ran&a0c03w3{RuPyC@F{_B0lp)U ziUWn)05~;e6x0UnGC|0xnGL|@x)^{{!~4H?NfUfbOy}F=FfH0vk^k?K)lRT!eRibv zD4vl3&dBg8?WF*`O8bU*tTAnQMT~$~X^n7=i3Y2*Hz5eUO8Z5u7h0u_71Ap0qwvCu zv!4U_?^7Y~+*+s{JY9wj9mM2Cj98(w5hEhr3O=n?*RB>~I$M$>wG?(XLJU8JXL^5R z#E;3UQ48C81@_1V0JKue*PTk?iggxM2q9gFh};UG7hHl5NSK!$&~elgM4kmX{c5N} zCtV&9`U^&t-N|YV2m|3}gI8b07nSC0uX-< zz~=yI1Zn|J2MA4sy9su+iFkFAYLsqU_mkfN0FMJ$eAB~F)*0%WBy|HA!ET^F2f#Ny zLcJfH3YA^`it)+T| zO^%ucg1)RI2{|G0HF8#;HAGF0qOr0ggWVOA@O%HdTdTo z4cbWhI|F#>H_SG|q(qf~z!%n@28gD=Q%s;ltpY=*A9cvBLgRI<0qA@mKzU{ZIJL=Y zJ2*>VR~_2g)=C0{06qa2X?=@lWCE;P3Csi7M_>iOQ39I*66l3@0^lS9t(5WIR>kAQl48whpkwo0lG2?2E}#2nX60A5_DN&wA#aXktG z$Mp_Cv<-9?N_QXiM=*SOeY!Zc7Sl-#DC=xrUdIwBuiJp&<#j#6Zb#*U;N^8a0`mG= z5WKvm*Pl@_%jl6DP_see*B`ak8e~eS549;_++vu2Cac9@j)5CSRQdQS-K1=6FN^a` zLpzvUN|#L(2+qM|-NeLgmEjypj_gEG(U!GFTs|_4!^x3AJ84inxY#5vyA9)S3@&)+ zP);%Zo2=S{&qIf+0lZD#ggg=b;+_cpmyEfHx0) zSSByeiAnR&rtM`Onh9X$p|X4QqUiCum`+NNw_L1&NJXQdx(Z&rEtdia>ux|-kOW&U zb3yaAT&f}bk1dx^!TF~TY5-OuE=HG5>jDJ;F&ofjB&xIJ@`a@32Qg{OrIJFR4d@yG z+Hy&|M7CVU$Q*K+n6%|`Q;aOK54zOdaxwHZ#+%u4A%(5c>9Ppd$y*$(u&_9|9Cr}R z&%Z3^WkCG=3olYH=X(hFvMwK^TFy&)#cev^XU@pqTS59_iBz4&BVFaC}E=g@^ZK6{d*Z`E=vIeL|o;uT&vKwp}*S`1(GwX^supVqK8@Z5Bv|5ENk$|Gw;M>r(ywdGRoUOO8*EqwcT zZz4gyl)IOx9r;r3uMGK8?r+4^mvZ;frQD9aD5(9k7K|8@0XZ=h1bwqdd z4@&w8PJ>RtQ=Ed(?5Wt?7qxE`(|I060ouB|yS0=*?+q^%^Of;_0 z3kWD%>=zsPrSWPJye@^?_%uWkOKAGKs)l;!GP=Y^55Gm3|`0|qx?la*J52|tSMt7d9U~P<- z;NLNV5M51(6*{AX>LG+d*A&z;fN=nkjR4+#1$``kifHn-n9i&En7xsg8Mz$GH}*%2 zT#n_>7yHXbF30kXeTk9dSUy%nDC6jGe)^@zxJ5x%!3=g6&S)_`L7@ovIDM8_jF1lD zA4kIY5Pl1sK;T38IRJDBpBGb%m%ic^=jzopAl#9SOt~9?S5=VP9Kl9m?KW(j*M9{h zj^J|uj-VDmPWi**wGU92v2bd)8Nn$#if+`Pwu7#Vr4qy_2`vGPU zs06qlAo4Z93jo~bnm{_5K-LD;ArQFFZFYh5xjg~AKG%3@D_(5$xy2xG=X5)OH|np) zL`U@k2;5Pv1MvFXdQ5a$zkR)elyr=^#{jIzCq(jNs?XOOu69Sp$zFRxu*D3oGD zU8ft>Sh0JV_i0d{^r@g`@Iz2dMlgH;auI-c0J0jcb;qC9WsqR}xg9ib{Fwpae~dqy zb@RrbrC{**vx5A42O#;1_d3zdt5mq+-4kA8u6P^THzsMg;%#tGsgHccJ61@iC5sWB z{yM7o_I%J4?*lPHIyreeeDKN1e_|XXFTkblP}T%yQedulcTu9(Os~CMdneAe+e2=G zp*AC+G5}nt5cg+29=@J|@584=2fHaLbRNSDc2iPluxSRnZVK&RTi3o#p0;sUz%6T= zZt}cGzKvUzHK9VH*kL7GHayuwkC3n0pk3UyyR_I^-D1bW&Mmf9w^-VNaK*(cl;IPBib~Ucvn>UN=)akpgIqJxLywh;Ck&= zqsq8xmt*qvKgc*SM>qohN|y1w+XQkXsMdhTU&-1Dz+cG8 z4Qj--wRjBdT!R{MeJx%BJJ+B_Tw}S*3fel{reZp$rtl)dnPLwcdD9f0C}$&qPO}ts z3}JDPmC@VRW0k_SZjV(#67*Q3LGya7Y6$AE2NIls;87W z9E84um*Z2Vk53iBtcqY(rH@ucPv)7gzs`H1#y3#Rxs0I}Px`J%;VUV8nC$nb@J+Bh3SkaoWD2t7DLC4f z)>-1X)VIWOsc(rx|Nam@9!6z6t4hvMf^*W4`?U;CCPH_gcX2yDG@KD8_`AdvM!y=ZI=|BmP( zqH{@yd?g+7m2}8g(k*l_oW~mLqk8E&SNVO^Duv!hx(r)$8A5B$->@%#JDL0~Y=Si% zyQ%p=G<9aM(Naw2S=z;`!*=roZ$-#HU4ICgRx#{Q0v8$ z5N~&iaVNW%41}HAp01Fn`depTiW&#EC9nr`Md?y8L#rUYEJheu=0skAkBxBS z^x9oK}S^>`_tJ2x+jl9zJ{r z<0rK00%pRNyqz`8C2FuI~JD-fQGWoRqB z6ZJ;{I9O}i2-2TqzkZ-?{Yw=qaxq3JvjC=HwJsCDtp?Z4E`rZ1VXXvk)~BdhSTh?9 zyV?UVfxrr^zf3343scKk0Fm7Q?+`c_Gybj6?0q z25>G;QP04~G}zT@fO`QVI{@YZ@S0*NX!ejem4v}EPutG{plLg=DOQl{Zd`%iVE+@+{$E|`HX^ghamAoqeSDfJhUd1^e73R3F;>-YnE6(!(rs9O@zHKc+ zBn3W(106>U(;eINs}S8?j`|Hkx)g(skS2c5C+m9urkKvvDe5G&?`*iKE&zJsR{-Dq zLFtvk+idvPq^K)EtQ;~SM&)6ykk=SSIPT0tD7+Gk~`yRgVe%PJqDr zwL+uy)}-n&QR(}Ez`B+Ic)HSl{BY6nW~1w!Xzd`;*Q3=U0N$kE3E*wg8!y+07uw|y zprhbT`t|_PP5SXMp%e8IFnpWzCgT4Y1@28z<3YKOwnF9tcqevZnY3H~76_bKKLB`{ zRga0x`UeEgtaNk)US`!}ZgNy#5ID0&1Juc^>7wKPM%Q~%)V&~aX1xYLndLjDbx#bR zlK%y0I{9ukYLZ)qN8}F(FQe+5gD&DK0PYK_;Dp(M8Vw=8Z%n+eOSt^Lc|t5kNbj4) z@Zo#kSW3TdzJi$FH`x$G0%+;nZL|mAZleIe>o)Fob>o{SCM_>a0)e}Y#{j%;gKk!o zAwuB>?AZT#V=q zOZfBIxNdPZvh=CAk=AZlA~z!XKM6bt@GpTk0a8GYYyxNk;AX>zTZ%{MB3K^~t)3Or zJ|jW3MtK*)OpkBNrR69jGu#Q>%uxD`l3@D+)AnV5W1IaX+AWBC}Us>|*qncwgVp%EsSNl!+C`W>Elh45sQ9Ip_b58!=Azn&L2PF)WIznD3%nBm7kCAHxHWiifwz$)ohK$0 z_!LmNz@G%*0$&Bd1-=!)D{$lYOyf5_P9;KBF7S2$UV+zRqQD1&zy&@Iz$@^2OceM- zAaH>%1MmtwZ4?T;w?sWaOfK+PAr<&Wc=ZZAmOusm4+vb~t&ugRz)eA$2`v}65w`Lb z_=WJq1wICV3;aF+ufXehL4mIZfeZXFK-~hTLJv1Vd(B1~oh>QqG?=YmS62gc1hB%n zc$Sj$k16Udm<8ngGQe;G>jB09M9vv)TeAW9z-1Nj+o!0DA$*)(WTgNv5O^4X4_rp? z6Y$B)<&g8q%S`}$^73y0K6%-)L{47z0pOFDqX6jSC7-ZN1CLHv-UDIugyoO~-GU-w zIyhnZDi}Nl*#N+Uq5}ZY17U_c#`wWe%f@4D{@A_h3czPBivXf$E^iU;9mdbmBsC2T zdVOP*XVI1NhM#a zU(LQZ@&hq6dLwrL%{L%1Av|S5!U@empzwfX0)RIliDgpK=7GS?as`0bEbB3m#?2sb zvpfOdHOqQTB(~Lc(k%M{m}beTepYmR+vti@oFyRffMgZ`kKPsmc%wJtqUgj0v5nr(RztvrNQmi~=}f*&P(ltl=PT1vwCT2ZU%I zeIFAj<N!N#MuwE#S7`vt%owKbP9Ry#3i6xnbbiU=z>k#hmesEu{4 zCS5z>B&BF`iI_Z!%#0CwqsTOL1~eHifEgKrFj=1|0Ibi`0G>YAh#nJs`m6zg_1Otv z^pUaR0*SOzOs5w{k@({+ta3!?B)Te4DqC2+0lZE38o1Wo!pcBPr!A~1(7Y`y1>t{e zVI2ZTwy-L|;7#`$^6zcB^H+z`&|-1uQ#Bi3Z=*dV^Zj;W(&^7^(E0RdDFE$L^@KnE z{7?l1{N;^F0bM=b7nAm;ioxQ&sTnbjk=x-?cN5+C`@;D5&Vv>P;;>}g(AC$%_1A>aBo=E`O?kw^$mwv@oqd=8&c_FF>ccw1A$;*T2);dy_#b{ANFL5I=TEK5YJ3 zK-fI*g89;Vpo-;IVe{YNWkZ}#6WCZ2-VfQpYwjF7m4(!Dq_#%#C&-{htUuZdP zXaG|x^rqUE{Hym?VW<`kHFct^@kLkTi%u{{SL2IrtB#JN*s8zIH2i7`wy*i23m{Gb zZ#wklfcbA%v&P# zLw49@;jM`|ycuGhsefR(#GSaiBr)N)}o;`hf_Vnr5Q}pbGf=Ydu4n>ik zK&&U}1Iz_L$pvPeNfTiH-U@&8WG?Sx>dzNEqCa#pm_mHi{`y5|mbDSjob7mAMbh}( zQW){6*?G6aL}6N~IZhiSU=F#y=4tVoPm9-lTD+#USX2X#uW@zcue+o!(ps=~i}i6~ z{-8poPopJTqw6wcu-o`F-3;xat*F(6q}3F!K!fA0oe;O2;@@G{8)@8niU&=Qt)7re zSyJ3`GG1&@A0bdt7Qb?fZ1sr!PuTfm3oW$!J-5nMPYZ5o#lOMMt*r&Owvr2HVAPJW zr_+l0C5>*Ek1wQie7xmB;|ZsYtNwR~YQfN-HL(xCye8aIgyvS^QId8iR6|X7&gPs6=RA*>FNOlnd9FGSZY8j*p#T#B z+;X_^J1-kAYp1ar9GfOliplT1sWC$DomU25yZ{U{vKE2zJ8us~(+qFc)VqA|yjhKO z$d8Fh@8EMm;KU38Fo_9U5mXVpzM+rr!t+o_-3o}^f^m3QrFfL$!B^~S0O-3DQxMKH z5dZtT6Hmf*EnIQMzEWSYzwvIlV!r`kFP!)SwL)#ySL`eG75n&mZ} zB%Q2atbpgHu|m3He;2~==BBX(x+(#|m-9p1)1--&gEQ5sv?g zed4`xv%3Nge8s++82XC6byMA&-EIa1uEHa7HT)f>@}kd9ltbdbVt+Rr`6~AN0KO~s z)#N>U5{i(n*jEBzPRq*vpRd>-L=b$%e#k7}75ns?5hJeHH@;6Qh63P<;VL+})p$f6 zfsC)%H*Bo)?KCliu>!6uu|jjjK2}auByo1Mir^48dRNYf0svgGH{TlIq;W>H1K)qc zzMP12qF*|=7w-D&A8^oL{~(5Z{bNeqVvs5lehfWG0o_4vSGFqAJi)c z=JlbnZ&EiK500@6AaL9XZWf=3(ra;U?}-u8tM4E}qfF+0-w+O)cwcb%tb?(e z&?sx0me(ezoN9xqr{ zPf5loF`Z^E)%pH#Qon$&03gVoLMDAEWhOxAGgQl1pMf1%JwFAmk<6|G6}S%pz(XU91?p@f_ef%-i`S} z-aq@rZq4N;DTBYU|aCVk~llqp;yV;e% zlq2&%;FX112qK@s#yPSdfODkLqrM!uP)0MC`$je0K;Rr124Hf;&A@n#&-_klrfd5g zF;m~M)pRiV%;1PI9`eQO8 zzQ`p4FWj6XelHNyx!C2yu|37!&&UV4)^!D9878JZG>;-2?CL|A=Gx?7jxITF@Wb(~ ze?W^F%+Xn0lz1EOdB5%*w81S{3mPq=abh}GyCbbba0o5@57#DPXQWjHhscFU(q{np z8}apC>~Gv`+Z}SYS^(ce+~Kop2`bFYG1-HzO)hpYV`IX_4*jfqCNnN}XbNI=DU;}* z_cSbg56$yd#4-jTbmp6u^|-h_XH>n>4JH1Eyve1GdKMl}gByiiDg4!5*mF$kmy76q z6XebQ)W^C;Q@HwP-Ys{{}8w1CY~8u^;kKne){8{>V+dIXV13n zO*XA#aPFOLKfKK3-r4L>^x~t)>vI@eSwt-?BY#DXzVoU56*=#F{)(I(U|Cu89P4(}{t-0fy9hfbZT&fV?yBYdL|=Mlcg z)^a{;S={6a+hR?{ehKWX?R?fY*?BOH6@JcGn<|R? zId=#YExr@%5^XOc`$E_UY(ZKVxcGFLRs0$31=^mQ?EOUM3od0l_W|uBY2CzhUUow@ zC?#W`W#q58YCi(|1#ar3CvEExfi3_J@>*R1&;WqmvZraY|CunaJ{OPBhrR%Nn*i@~ z)l~TU8g8M-{)YHlyVohe!Pr)QCwaX)2Wk@v98gK%y`Pn$kCPJ zx5|sz zs#<7b*s@HxCQ^j2W`lDt+;C)XW-G1BQ87KiUJ-C|9umuQVmgn{ncO64yJ6mBQ+7pXe9GqqcdcKiQr z&i_Yp!YVvW)d1hQOp5^DviiR_rIoG902bI{_bX1ecS7HOLXvtzUD*kot zKm$@X%tPauP&e2>?S`8-LhddBWsB+D=&Il|w$&3OVIC^S5@@{H5d;}3hiT+mP9di> zPz8{C1K3d#;teL`8Lpb>4>^`VAwLogIcPktlK|=oiteN|H*dgMq9=q^q{&$g)Y}M$ z=hsKYbCWjG&Cx071hE()&99wj>&&k$EP~T)@(hUeX-5bmH^RoV$p-;=Hn|+Yn_rLU zpd-0nOj;w~33RUY7N%#yP|xYy zK+S=e6Mnvgdbyaqu#giYq=bJDAHIc!DoOY;h&kcbT}XJ}`I7L_0G#mq0K9}hBME-Z zm+(puIN@IdcnPP&1Xqd{&x;AYN0yb6)_f-;))~bBXVN7Y>l8VJu(?2*Jtung2JrMU z{LvB<3S{KJKijO<)gT(R#y*Hz{JxM%+v!qtx-+0tk5~X0oLr0P2t6> zEUOH_>DnN27s6i&yITc;n}!XF$g2>222uD2JhCBi{z+9EAlwamzf;o23%& z@KwfFLEuu~03f9v%<81`Y?zqbL@tUEQeGc`C!N>S(c_aXRDJ`)+V{!g!I`ZZzbHl7 z6~HUXDoU7AMUZ)Q+<1OT0-;2fgTTcV>lpb6E}Y=K0G!|kFZmKYR01hBfuXq01Hq*8 z8<8Inlhe61Mo8%#3QzuY>OQrW^zVTtnojDw>DWl&npgk;UyzF*-4Q|RWHtzHm25fg z&)mH+LW=Z%UZh;)RTL`D;aJ#xF+cVD6eyJJxIY_6bK9%+CO?fzzYgf7p)7X8VL{`x{FeU!}3XKWR=BPEKGd@Cxj@o5_J4$R*9 zf^(7nJq-Rql41I{D|o~+f-%NMoT(*A@>Q=tu6A2axQn=1BUY331g4+7_;i94 z@4(r5g7ogdm&;F(N*($fLuUU=Q2z#qZS#+y4Bv@qJ8%Y1oQ=F7twzI?lb^R0+W zc_z!P5hqBIeMXUeQAPGe71EFcVRdUj+a=u0Ex5LggLhJ`&=UOE8%U`oCu1VUi%WWe6UJHNusZ^47 zlbFurB((&7?#M>w>JMe3YjCC{MZSSlAHaU{W6OF$_@5eowG7(@7fEBD1-U>1K%zUiSxjIQb0xx`Kemwx6ng1QYJ2M~a zg+8Nl^6PSD{(J!M%zWshrk3@kMAzgzUDnRDBy}ZN)o@d{0BivW8djpP>I)7^IEE2Q zZ-$?djS)Kc+vATSyjc0k@A8gCthh;cM?C`-mf>xuRs-jOhmp}L8EyW(Ty+d zohgLD3<95IE(P#TGS_47#{E+e_$2dk0PiGoJ?17y zZ32OBWBd(ZPBLS6AJxrGn~m2!^|j6%uJ++Ws1n%_Iu9hNmX&yutwRp<0dN8Moc%ya z*kxj(XI7&?U}f$B@RX^?gfg#!z{+d@(8`4Fy}+{ONobFoP;ZHg8~`PHLi$BQq)G8gae6~cr#wk@1%(q)1mGnimPv`21_CFd0>Dc|Jth+I0SKIkT>vIW-BM&5 zf7gAuL_f-?F&B3TmdauF4ghqRJu(R3b^t3xsvGVU0K9rzkBNF)4Fc!#p8#Gi*JC1=PkmEzxjTT#Wv-wVR6*w@so`+o47v?~ zGw3-0&Y;oAAX5!i7)6o%wIJx^qu1?)tgNQ69>+)R;gCmpoU#ljkqLwWt|xE}z+?hX z0o*}gC%^*$k??Z-xCeo20A3>SJ-{0PuDT!9fi9*0AkptMSsX}Gt>3~aH@K+_0qBA{ z4zyKIp9jEMY`oMM&4em3m0lhm^y^@E#Q15iLhH5U}VUGo+|w2}W1qq7a_4`4_mr)2FA zO3L~2s%@8~{sD!PmHw_THHNv{#EaDQ1%av)U!tl3rNWXdc-O$&aX(k(MiQ;uL^e#xzl=+@$5)B-IB5ZW*Hh z_@fW^0C+7U)(d^~;Z+b~TSg_dj1MH-zs2O15i4~1C#ekxfm_A_0B#wLSLv3ax&nAD z!}wiq6h_M^0)bn`G=MrSqsaxj>hu!R8I+_dK;kO&0RXp*qX1sZFkV99g<3}E_oV;^ z19$~ckBI`91p*hqA^=~@Frl?Kp`vze2ZdWk^D2MK=qwbYGgZXiVE9@_wonShbPAGG z1QbrzOaL!6hS}G|i`2Xag4Z(Y5s-+VLGZPVi(+CUg`Cw!A+#U2hML9JKmvVEnlQqRa;kvsPUXcPd5-G3^1=f&x6kU%*E>fb&|Cnghlr*wqq% zod9k%@zTmszrTU#PlzIi0sbYo0*KvmJi^o+R{=OD9Myb{RJ7gzRMEJ@?bTIRlx#6E zFH{i_L?8VlIWxrZPD6^1Q!_z2(G}6Y1i%+VbpmQBUi#2{`h$q~tt9m+-1(|e0R&+bG4`S#Eo$=A0s+5a{=$Nn}t$NrZ5-fZz~ z|2qznca}HOy%R3mjS}iVVmg~tE>cB#sjda!COZD+jrd`CVYHL@osDWX2zS6KvKU|v zfv*4_2N+L3@{lUbw#G+ybrA0K6HQ=Lf@Z?%ZWtRQO+U1)b#T(_wKbIQ8&xKRv@RGl z#LLBl-f$3$5z>O+c=*sOhP>dH_K;rVoeME^p*j*yoCZ?;9swoXW#R=sfq3b|O!uThk+T}tR zEv7S2bph)oIH@ZDmXi7wx_9_@tMKT!k7J=lGu#}ZP^W@6`vir`tB|-#G8bG$j|zI! z;L$NkaI@!;3q8_s;&w_Uy#K*rl6O}@=2qg-@wg|XsA`D&bzN@yqZs*I)LZA{Y%!-C z0;!H)yrDpx((E(5kOGv|6w#DHu%<+<6JU+{aw{n4UwY}FyGhU;n|XZ76ucF|ax)&o zqYRHJ3+sf@v8BiEc!md&i$whi)Tq!$b*;w{bGgJkU(6}?dcS(uKm%(^+qy#53ENI- z&cHmXIFw9hAZ`}sQK3_jlMTisQ}=$%F?FyLD`+>ihxlq>5yG!i18b!QmQt0Q23K9> zZiO|NFZH=X%-|3KPl`FE0v=+jpW#dt4yxZ;u(o=NQicr%DOwYQjl}U%6KW_%Fl81zL@Q@7J;ps4juA{*1P{?_q-#k9 zMO_VtXoYaAA#<%K;E=e%ualbrS}O91^1PF@~wCw?7Z=_l7H}~_$SzT z!)GOjS3KzptddXwtF-@(u+!~hv6q}&jluG6*sEBtlK!x>Zfn?o$=k5=g1Xq_zQmUS zDLk>C346K)`v=U|3+$_TW28v`f;`K4vHhezI!UL9>3n3X0q}oEHgb~}+Z9T|`Ph!! z04XoF^Jm{GNV?8e4?xOaI9Ut8UpQ&_m3-mEtp>-no}_+(9tw}(VTmVAvNCk#R?CVN zIzQPe7r~5#o4OX@Mgp?|rUIa|DMwHxi(%(uJ3BZzZWTpROON!YFx;izWGn!1cG>DH zQ0W61{3*IJ2KqCm&<{X!2v3l>9S5;N{bSBh_`&$>9uH;^7 zxwvm%{4QwRjXbQok#oM6Zs1|=28thtox6dDxxN=4gq^#Ax!Qi&M%h;rdnN4LEogfQ zZMRoYWREbvq%Z8RMeV#cb80`WR(mm>Cu}tuJYJi53V_#Uz5?L28T#2F@ei#ZM7Dh+i34=55RpiwKJ@* zg;ig$MqCUd6xYWv&l;b88`E?#h~phqhumuiNS#`0;wMSOPWSS3&R!ydD7sz5xWUoa+%# z&Idr?a&G)%w48b8=PXgAyHV&hTXh9VZ(0HH#>O;&rt46?qc6aXf*3EfuTuenugmH! zlpGW4Qd@lh3U^t%0KD|ZGAX^u7+i8q?*QN>ydDz?9|Qu|^a%i74%A~J2j+pmIj{o2 z>#}I?D_3+JVsx#v)n<@52TlNR4z&8&mjeYcUML6pfuM6BXhM2kXu%=@M!;)qu|j8= ztx6CGud&Sns6c~KivV5)-~}1uWvo%PR#n5D8`hryX3Z2Y&a0|~(?8F2ip!~A>KD)B zCvzvGUp%cF6rWlnqp5!U@si>Ru=D7tUp$S*O1^`gM_;-8`+s~jIPGbS19IclQMfI+ z7mf5&iSeMAd`I~E7$Mye?z7c*5!(=U36U;hj{<>b?|fU>xR`M1w(vb*_%C7?KpDD- z{VL>q5xWe6$X3|++Hm|fxi)+jfOl>9S=?|!Ia>?Fq-(?ZAn+7_EP$EfhtKbi?OmPT zNI4va!5%sZ52I6p1mSdxSIa#ae<9j4b!g9M8dvQ3 zeCnr&>VqSgrw0}Kl+droAb-xm#M9Eld%-+5*4Qevk`cv>I#JBf$B#IQ8P300NzLRa zG}ecYX6nO79NitFZF35$z~Q-zB&xu7lBs}cnH+3^KCd()16~?njMIUp&T@3P7r~AqXRF#8g-Eel;3dpC11hLCHWou4_+hQ4-U}YLsk2*tSiDYF&(^+4#MLG zxJ8=oL^}kqg2vhNyBp$Ip|jbEWWtNRAK$pKQt^!34Qn%iP(B{_NgUlo0cRmjJ%Df} z+*BfF#Mh9Qa{z7ya1|P^TaHI?gLrJwUo@Ww_b)|w$Y_4A6WRm$w_@LIk%;+AIH@}Uwg6b;yMS&OTMY9PwGf0}a8jQE^arrU5606lY7FyJ zwI75L#5`#?Zk90fCOi#eyJ3E&x_~f^33Kr@4ki^1*838%kZ4xuO&G@gR|&{Lr+fuM zS_wCwk9F(=p4=+R)ml8l-$0gLfzISXM@@y7ZLq5k0QM8u4RDM=${tjsS`xTK6Y70WNXpDnN#s8^jjQcwh5EV@x zLi+Eb;rp{ENy}(vTF9;Frcu%~7%URmEynE)M_q^Dc<@Wdo(uuqD~}y$=yyK|JZ-H6 zh)!E4$Am>x`s{!nCA#G($l(3L`9aKjf^a1wt)dE4g^ymt_eZdo(lERhLi?dOJ_fQx zyniI7{Xm@R@QWT2Q`&UmYQ)v_FV5!}L4(Bu+z9}j$|E>SVVK)Zc$4r=SP730V$>s` zM_U1c9$E7Q-}v2S!a(vjgTgs=0w9`GwK0lOPPO{g_aR8*@)z zTmd_efg3yY+gion!Okm?jd_?*?EZ%2({SPx=TlUul3dvN&CrD33?;Y1&Pxr=m|wCF zc3x@_d)hu(S`hnru=8+3+x3cLQ{00vLlQ0L6~_We%3v{_7LJ+@|94~~H+aP{eLgg6 z=|tv38eMTLAZfazK7^E49FGC;ietb1X2r1#9GC7=l}v?<2Syz^1>xlfQ41-APL8UC zTNUi;Yk-Xe=;yg=0m4*Sv!6qOyc?&oA=LGs*AmMhb~-xhW{7ES7s;FYW0i{IY58I2BL7AR@(|=8=F7% z(X^VdHEaYNnrDi`pMPU7ER#P6(!UQu;&i0F_U649y8i<9#>*Q%ucsj2^8ZTtTO8zs zTG8Xzfu?HR460KVZx?riT3oHFcY;R{InD#_@pDK;X?Xc zOlL#9dI-FBaEmMh=mIeQ$l)k+nlPL9RU6@+0C#KLMR;Z-Lm@0XfH4BBXs&l!Zr ztWCBR7oYn$jQF_bc?)5VhB&SbeeJUU1i(cQIgdLz(|}h1+bvpG!AOX|@B{*V9Mbqe za@-yh=<_5%6K>pNV4!i2yf15P>n(^H_n-i?-UH0RJHpDvfV**g?qC>W8mHuqgqeZ_ zHBP0yu{Dn zb6Vb0uziZcZrsjklb-i7Y}w?mgVW~hyw_m67@5$xqtm9_z_i94^HP5lMIo{BPW9OW z)}Sdw?c}uSUy#0P36kt7-y zY)~-oNAU5IvKn`D+FXg?yG4VC#1}klRAthTZ#m8J*g(b7alm&l)}4~j($1%|C=bTt zs%$G89`or7DQrCp*7$rXYpN3o8L$TOe}(liBA*LuTK)l8$0N{$ZgyOod+dt>0Qtuu z%t>S+0<`iIe!$^b6iq@8kjm}+Q;aCG3@K?{GqT@p(ad&z1q&F zRa>fJ3DaS(vGeI00UZ9F#CP)l2m3w5znl0@{yoP20PGu`{JF4mdLAVIPX0^A{c+g$ zIQh$A=lCmNuXXZ2F!mQ=KjP&71p6x#{vz1R;_`okeK|V6gi6?}8X^;sNXkU2EWn(J z2SDO-SPIg*hWW=}r)HJ#F6`9}^HZ^c%l7wR-`Fs}g|V-My{2LQdB(m9_B{>rb7AN3 zs$j2en13nknN+ZAU_U}itnn$aMk_(9&+O>~kjm2XY0kHtQt}b(_oU@dhW!dmbQ3;- zy(%sLR@gVE!Ttr=+gq(0AyrB7ISUZc-Ea#8a_7OA5*U@Y048NxpoD7gE3i=(2X3Uc zy~NnYQQLmU*h;BwuQvW}ruO@#u}!A-yTjO~I&CuY{xbe9Dabw>fPJ@;Vu}GPJb3p2jxAX`}M!gFcH$&zVk}F?sYkpA%%e+i6p3<=hLn z0NWTzEpp}pz5FSxfH~ucpQQ}n3(%7jQ+vP^8)DkN+1pnMi2wjf`hrYqDIdW?yZasotxB zv$Ari<0^#6eXm;pJxHHfcpwFhU_>AeCgk1-BR$wJ?-rONAr4+h!Q2iTy}IzDlgNbG zuu(|CJbaZn?=fRT{-@>9`uj}sha7Ke<%|V<5u!kR?%QMp8s;rQ9&vd!M#xqcy`nDe zG5TpLf}E#dSr1-n@RIn{_=evDS`GK&X*WEEr_(SM-FlprwFs{n0(0Vv3sbl9{1cwrmoSAzYjI@LY^B#p6fjHqIgfcKK;UO;h zJB)-&-q;|_qK^0fBJH~at0=brcW-Xm<&s9I8i=A4302TgB)JG&igZY*$p(?7Lwrh! z2oX?}hZsauP!Ld>f`Z*AJYxqD#q!jLE!Jm4!Sd|-`})xixsi1^e1n4N$^C!ADJ4B4YUA!tETw0XH~BPd+4twI0Z&0Y3)van?7{gN?J?o6 z#Kv5ySccw#rlCQ($eTs5BVfv))_}JXq&rl`pbmhC33di-GU$B3;{@r3l{@GHz%L1& z3)p7Rlf5nL3`|%VT>(2;1^C}uYXE}$)`if^2-u^k_v|io>_RTe!G%H>KF}z!3g~83 zP)d6<>F&nCA>H)4x?WvU;3ZRFE){5G7rbu-l_bcq3r>>Y zHyqZG;H>}4WSk?d(DngcaJZdHwX0joYq?h{|6&y2&T45z<{*7H3Mb}`c^D@rF~Rnp z8A4#y8uS$84?-<512ARKVZbyzaV0hd%-|=Afpm{J9T2nDW6+E|gTqT?wRQjFI1Nbb z7Ns{nnkN#^ht22-AfhG_PmICncR)W<4HuvSD?b6&*}r6qXWn0sof(QWg*f3O7`^|Z zl*Ie_p)-&CxSvK~L1!c*y>Q4TmA3uc;WRdJW3&&>BS~!=)G*+;@q-o@Bndyq8H9&G zdT~g+3k@4SAn~r)^JL-KRl<6IWxP*YHZ4 zHHBu~(}CKh1L1a?AElTZU5r_{i55oB0qRRO7vW}jQAXAxZZSDW5^1qPnB2QUod!o1 z?v`Vt^jdcYx-)B8pykBq1<2z{uMMndBsv?2>wbc69`g;V4zxTYx&){mX+DK39nE-(o55s3BF!)e z&A1Ng%)wz|r{b==sacN_GtK+A>EgFx)UTX-b0qfZ+6?}V0D+0i2g z9#3ew%#Qxczz-8zF1Mra8u(d4%N2I?V?bu>OL~}&wq1h%G^0NHnjV9#g1$5m-hxwS zo^=K$Ydi7<(%+y=gJAz3aO%_`H|A-BcGSdwPmw9VK?fr8SLo0+)1WI6nTmHVM%@A} zv!jiH#$a?b!0SSjXbYe#33Vr0TLWz-)Qj%P1>=#7T!KS8$Qz`Lw7Xmi&}AVGR>1-s zM-|ZYRj>@n$n`j!M|!C`AF?hTtb#j9;a(j2l0stt!8m0!I4#=x9Ngg{Zg4uaVL0A> z8RgKm9eEEKBcYg;*nbjExmg3F({SdV8%+B145ELLj4m>02(@{+K|^V($)~H2=Lx() zcLLFoo;a+5PF8aNjW{`3tG$uAhK%oklt|0kNXGX9Jx?99g^WJ{bOoV1VI1S_P9AT3 zkwGHk%^-}oSZ8BuXcIk%!kVQ6xp2Wh&`6JAYx*VKzPN@h^|Hn8WAaY zscsUz35a?t1#il^(c28_LM+^65XNMu=-meOq$@J^aEn$HTKRMb;l8u1erfn9K<`Sy z)YOEO5%S%lUsDLF)?g?#;iP_d=-0*w>!l|AWQ6A#VK_Cx(&FPj#n-j)zJN_86tU2IQvb4R(nuhi|UOn(w>KvZ?B@kgcu+$>{&KHG+GlQjK8N_dl` zSHNa?gQPP^=XOq8*k0p$%DO?a)>FDlvcSP6z=prrG&#vf*M`4vYw3e(!`~=rEixbc z&e9KlcF{G|s9`oSU@!O_UmD5V;4o5O*y39Rl!&X545tCo%p4p|FCYt712Pk%h>3;w z0a7DWx$k$fr*&`ynV#D-6aUS+6J;klE^-C}hQdSStK<24&RGg+Q=c&WP{7%Q64xWI ztQv_8NaPg7X|~Z4;qBNs9ST#BXXWg`(d3v4pi!3ey=_&K26X1*BkJ2j#R|w=v56Afk=Tty&X+j+M~!`{=kToo=MWTrBF!(6NXX`zt@0Ca zp@%v?LeIfTZH~r?BPz~5T;4qkGqu8gM*M=4_JeKpL6%iM4o8zQ&?p>nxzVp4B$sXtIrS*+(Z^dnwl{e+(xho}^ZSB~AsbJVhGeN_2F1 z9}aW>16)Jheu=XQ+1~?}RU`2yB~mf8Lc~lS5)COa42fJM%3nhMO=^0NsiJe)rO1C9 zS@;*}T`2pk(rmh^j-bs&Fy_|^=2Ii61Db02Zh8*(E6+m#BhFQE_915%DlSBpRZcx* zGR`szhn*u(cmr9^*GPPTgvFg^G};)=7o5x=@kyE5z@26?Iv9l`aY1hBH0lkLZ9MgQ z1BOC9wD4>_hYBO@b0UTrJ%@RS53=2y*+ukZmI}^=Myt!YddAN#QUY^nIxJQ&5}5YKS#;>aPK; z1&}Cx=zWG- zPtf}e^&CO(Gt`>|z0Xj%VOv4(GgNboxS)Q9Ity6?I^jT{p@R4fMFiSi=-7oCD2F0~ ze3*=sHKbqyPWtIizqA+3foMPl4)j$jh_6zl6K{!d7cO70k#u+9KwqVT_$uYs^{SQv zM@#|wDiy?6DSY`93-nbgh_6xza$5s182GDH5MQPKx5+q1T7rrJ9dV$qQbBx`@|W_6 z@JjXm99=mBg={-5HR0A_+WVJ7@-uQ0>-D$m2drs{Bck*<@fnB{N77Z1i#oo-l$|(= z-ds|lGa3FSj;1%444@z;;KXtCj(j>$1|fWF!F%vwn25ZNLrcgLn~$zR(&nDA3O=E{ zlQ^75d8z%QIHe?>JuHxb;&>^dFF>uLV3_!n-c3U7^LgynL z>4O8sMDeP}G0`lEiC#l*r1hy3#YD+sRxBn;eU=rAi4t8|P3bj^Vxo+4O-z&#e3O{y z8)0aoZi$J$90J1Yo5Vy9!Hq0LKoJwol9(uYN)m}v8idbb-%*CdM6)C&nn=!41QdlO zCd%q0;csaM;T4%;qFqTHA1p{+Vxp`-68_d~5GHPpiB2JXe8fOZG)rQltN}@!c_8Q) z{2jWHn{l9+Xp)JECTUERhCz}S6J_YdM8kNlO`=O(Vxkuj^kSl4!Dy0+i6X(_ythy% z((puLqV(B4DJ~|;I4&mI8l)%1#YBe@j*E%jPdF|nN*$OK7ZatfPl}6)GLDOhQl}=x z#YAa{CB?-=*?wG1v@u?}lj35cjN@XW?0;NLl=I_aqFjGmOq6k4Otd@MkBfMAG zp}3gnUBrA`O!Q)MDlR5kgZU;&W1^A97&`P17ODCV7OC6bq&@ zs-SN(6inrEv0y5rSTL2*l_r?VJ%Mfk%M8;!+F84aZAojy!4ip5LWHHnvQB!k;=pm=FW zaa+Mm7M6G^O$aRE znam(ed=xLGI>NDd=~zT|!m)ViwS;2v(shK!(mg41F_IK7#ZS|6ytKZ=OKAYrkHt$F zg*jeIlN!ZK(+=d{fuenKF(se{cOy;l(y+!$!xAt3nJS9KOKBvA`x6NiFC|OisEL;{ zip5J$gVnIaOY!i>@zRjSOG6SbWkR#*qUL8s6E3C56opIE5iU*tKfp%*E|fx@Lp z8ZJ$ea4FAf2o^}VbOn@>BwSjBq-pSNNK?2pNyDW{5-z2lO_FfwL1<98G^2Us5Rw!w z4VZ9g4wszr9rx9p&2=b0|n`lG)R{uK{^_oNwFXujf5n4Btg0aGKr7U z_zaznB>R_aMV6se^lH!`6geAW3eqJZNS7o*x_l~hKQ*f00wf~?aG)Svk_PFLBuH06 zYTKZ8QAUyk>9`E1?G&r7(tHZ@7n&hKIwlJeX@)^)1_kMugTp3B_c0ZI1t9?reR1CFCP>F97Np}! zy&xS!FG$DG3(_(4f^-bMARV*i1?kSBKGGl^T~Y<~QX2UNo>6qJp+@H#N_38#Y8Z>o z&4OmbSagmqmWEx4!Bhlg7{#J<<1s86;=xs-bIS;ICq7$4^G-q%ohwB$G6M&S&NZay z+ya0uSK?q5EXHwE0o7Ts63NJoI8b!1VXD4ju`V5~f?cHW01gzLYpBt=h7z44jfS!4 z9HSh%s3RXCgQ9cU8lB6Q=p1)MEIP;NA{sRmonthF+Dy?oMiQN)%Z?`qMCZDJ;YcA4 zYsk1p=dvX_$3rp}og>n+H%IvxSUPbgq(+MCY<4I>%R6EIMZp1{y`@xR)^} zNObNCDvWtSqI1pY>e<85xi^tP(Yb6y=dvX_#}}SN=YBvgMd$EutQ?)omgpSa60A{uw<6`6k zY@hIpO(U6qt`li;(JC>Y?-rU5fWtkP|bEEw-*5 zS%3zZRZ^ukx5R9BBHNdX?b5g5DO`cZ95w6@>&n%g60@@eD(oG@x?ROK_HT68rrv$HlcJDC|XB6k{IHi#;ZPkZE+ zI7B_jqji^#)^R$1cIo&zsrCI7t9K_A-yf^@ae;aza83J7y-&yINxhmEtM`6iz4uGK zljy$tgG{HYLslu5?v48oU5F0qxh@;MHD7z0?R+|Gjnv4ZTVSxO!awGH8 z%8llG^Y?t7v?u-t3q%^_U{@Iakzb^|xjHAnPkD=~)!fS?#eci#2Wt0FY4?Kv!Uy^v z5_>4MeL>!D*ed;Pf{!7B(>^5~s>6vPZrWRF>p#ZDk)f_?wzr+7c%As)-pTgDr2^f%1ypKh0XfKWcKPfvqpV%Ewh(w zGiHaV+~nJruvx#D%lc&R%-_8(GwPMK%ua7*%q~*7$+xGo zS-+UfcJ<8irD2L(t0wH3!&B^wIn47<3{(4Jdd%*ArDf6ZXN2cSdkRN>o+bsnY{w}O zB85w$^t^G16fUWyK!_BkMms>^C@D;>r9g-j+$jC``KzSBA7W!Z5F&+XQTl)MH%Vby zEenK5VS1GQL;W36m|jbP5GJW+xSSG~@e`1p*O9@396ZI*Kk6hdTa09TieV+Lm`slY z1r#3Rk1e!?X_MyTx>1Q_n;$gno?JfJCIda$K(b91dQvC~TK`3QJX;INBNqj%66@LY z@I9XX709IfkkwQ!%Ri%;DEd~xudq-^-{bhdl^Xw|e$ziiHU5Qe7`*(m@o$v0CJZDq zud{h=+w0u6$tv zlJ|1SNk^&VQT#su_svnS)KM?6 z4**i1*=wkDcpjcT=B`9)3q6voD@*?a|9%cwHV}ztDKQI)S154<67M3>VE_)tpq^OY zcJ7C84CVd{iJ3^0Tg~ws!|Kv0JttmQ_6zcxAPeP3(dllKPq(qB&U~Qqvv5dK|Ig8L z;>S+TpIHBatg=ERqDX{0pN)2&cwvO|Ou+`W^Bqd=A`ipWC>K77Lr$9( zmbKMq04pT;E?Gx@jORwbnC8YtR=^sDyQI?{3B6t7-_yQEZA;aKE)Mc8$bsFJ%Q|EqX3<`0b;F$g@YBcYa(dOd#mM)In_+OWW(B%tbH(wY} z0`i5in=Xv@cVq6e@0ozfhA)rYdvJN|Ez1c6B>3{!ghcG}7!Gy1JgOn& z%i|*?VwcB%)u~^MsOSo~tIcV0kobFqX>DdR2IX@uQPou4>1erQ>H zC%g~7g^D%rAwBbjV}zBaATFGO#^60xhsXl^!|`<#8q=d0#5B>1XXtC>cLIWodqOvxQiYGQo9 zZ19!RtHv+(`Xvv=WBp?4GELw*jqom0=YHp6Qzwl-%bHQ6Sq;aSizW%H$nEL+0jTsE z==1ygbpCkbz6{JCY-aI2$4$o15Ar7)!S(Fn{E=p+d?uQat7b2anQunsm#TT)I1iX! zJ@A^zFCI(nX|3@oq5!7(E7DRnR(dTkyHd%nl->i(Uy>BR1kA2g$|tgpR8yt6c?x;# z=5ofy&1K>yF9b-#LI8Rwk^lAw;wKbd2#|(_02J}ktbv>*3jxxG1}H}t0;Jtp%f_9o zC|?SkcNc-QdP7D+j>>72~k@tYLj}B^W}Ib;5V~!q9>2ZDJO37N-6tNnzF0{ zdN-RiRcRIdrJjBz>07+NS(f?F_pEgkLuA#gb)+f_3$o0@f~;q}{Abt{{S$rGGxSc* z3k$NIA@_M@iyO=f53&HV+C9?}HzD}#NMyl77Y!9GS7+w*rOPR_ljm1Py; zPGKK+D)GMX7+~n1%F}=JobwH&FZhIM4+WgVDDgY;%DzG(6*T2M7pM7_mPo_mgLHg1 zc^!FHP9vz998&=_%DO_d!s$8R!q7vL}%UwMF7% zBpM-+L(kNC_)ict(CVb;v|9sCGM;7XBd_cnB;@fyKgl#M676&JY`<4N8y*An_(uWqruU6F8P%kAo?+M+;>gat1^9W@I^Yk=TlaRsJ)M zM&kjadDz(rg#ZY4jv!GV39Gy*jz;4Vqlw+_pl~K>W@QGf_N>_*N2Bq$(LCaGgF+u< zO}2)x+Bh6d_7g_!MQ0q;CbQaXR$GLl$u!5N8-I-+&Ci(EhaSO4xlB zS9}UxNAds_`#9*Ae72MT3pj2Gz2YZRbbR)@BW~t*AX?FCJG5=`fEDTX2|7Uv&qM zlS`#o&vpY`+c|Vy+^Lf1O-@VOxd0Zq8yzHGgp#uy3F=S2Dol|bS|sZbek1YF67OO# z8Fw3n!_E#Es2}S(lX0(6c)>Xg1@5sn-JxbO4jP3cZN@=t;8RBYV)-J74K!_t?~zdA zcxn0wJ%_sZ1Wb?-_SQ_h9SGu_l`2_ha`6B1lb*#DYp*nqMl1Q&&9}If{HU||LrkT1 zANjbg_xC=e)Si|4(&Lgf?qf3YhzWD@4f7F+#In{S{0J&h*+=-sDpA=-`BpDcSxxhe z@6ouZtfZ|O=KlQnL?OX0`k5+y&OBOw1=hLZ=eXk1Gqd$p{hU1ijs@o1?{o6}y9Su= zzt6K1rOyEK4OsDS!0gQPan4BE9p5lu@q_tcwv{s$p^9_#`JubScKmScAq>TCT8S^<~+IVsW+cVQ+kqMJ- zmM|#~=gsVigiJR}$do;}gFTS&sirx|DQWA{3~lUfWv0>PRMR_X^G)!Y+x(7<-WPzm zu6JbgUeZLLG~c!9QM2>}Fh6Q4p4(KPHQ!~gOaBSXpJ?8b(LSr0vhfLs$Z^5rms7fiZOx{5-C%8IiRs(V?3~%? zI)e`z`RBHCGi=RAp7R(IS0TZdN*&GRPSex%VX7ArfB7)Q_kj1+c6y0Y{Dtk@juJHW zPyXxsxGD0f7R8g%6DYy=@sE)3ZU#oY=g8^OzB? zHWgLy^ku~V)iSnBUj|-iw4+WbYW1cslQGyR$n-_CjXWOAS9-S@nll2@24xh#{;8rgVDqy%(Qv`4R~${o2BYwzGa3r?JZJH9gUK+Bi+gSenUOP6r=1*n%1=?y#LPv% z*v@?Hqrv<`Q$YfWqR^b|x*(TuBsH;vwu3mSuo1#Lavd*Zs4X}d-Z#MFFA~)DvWOiO) zHaR|89Uf;KZ-yJ$7diBYvy;GlHyI%JvsPyWES{|g@cpcG6)?}%g87<#TI+vxverL5;QSkU zv1_)D!Yb!C2RM`+Z%8{6AIjMHH|3j+T31ij;`5$S!vw8%dX?a;v zVn|n8TiCF`G*@>FSP27}c$=3U1Jv#bY#InntM8Y{?Mb#r*oB2StDOlyLmo)B@1rdK znjT2rhjbpj%W0Bc;{wS?NlG76CE1?7o(4js%MA;pl8ILB)A1e|$fR_8`hptBA`QHC zMIOhI7R3jfN1j8H7R4vT_7o_DWYID8`AzlNE^0`|T8eWEdSLX?8)%>R5$dDfFV}OK-8Nv|LMmq=u?-3S zTU4X@ribjJw$$3eb|La<8wZ+u1B1yyY~#Q!9V}cv_fLP<_eRN!8fw>{RKpj)B3<9_ zHstn&nIN{H%GTYc{(m>w*5OlpCv0ye+uz%%2^q~~34H!uNc!xgac33Yk3-}j4i7*z zF|q$MIJHU4;*CYaP)Wp6aVz>Bx}YT?EXhlW{tG%?2{oei0`CL$B-EJJ3w#DNoKTZM z%Ur8JeYda^)1p7(yaKf(rqll^6`*si!uB}mP7P*vfpfRttpTs1qI0qUjnB9WMp;6UFH6V=9j2Bibf;*1Y5z@q@{(W%D zNIc!68Ovp?XtV|!@<1sOi!@TKf}u!`o`OSs!=st+seBr0Y>A z_%x))j5D!5&#;P@LxbIYf;8HTyTx}HYmY-(W28?*`Vm=s2-1uW{>CR*#UB`J`=L>7 z%Kd7r?S*uomMhM|HyO6Jlj<NII#g?2is-U*hl5;K>TXN1Qw&a}Am1fB~m&0O5S#r*Km1fB~qu7#jA|Z8!*}t1O zNL|TG&Ob&5EjbUhFiXx)LK68E2U>ETs!PsOBfR9CjSiq;KzkuF;w4D5+KN$Z$vL}5 zYq{~e4X7=05e~HEJXM#Rr^=FZE>u#>U@5UlOU~KEWQ-&$|L-WxvwKS`ax>Dj5G+*} zf~CqrFt!w12*xP35R4HT*3R0dUipZLg-eaVrAgwgg zYGYv-q@9fP6{9;C(tSqyg^><`w8lt(Hx~Lrnro!>@iCKqD1e?Kn?ZwUN8tNKWXm;YPPrcLJFGc?Y)psD7N>)g@k19g$79e#oh~- zqM_d23yi|N_W}{3doR$=F8YKP8ISZ|?7gs?D(XkGf!V9x+k1g5d3!G~!bFr2rM>Ep zP&wIqf%f30z3Q>i2=7&&Dtj;RwTlNmJW$b7R>8lZ7OBC3)_dTmQ|Kdh5R#dh5R#dh5R#dh5R#dh5TF zG1Ak$_1~ii#&6P0kGc^oIOX-<8M^*EL)L$jQyH=K-z%V*(SaDG-I=NDGP+XNrPe5l zt^ZyFa~ZMq-}?#0)_=cBNY;O^Lo%`%2U`D~LF>Qo0pLext6(3JqY54-l_znarQaE; zqp5MKM!I;HCxs7jprzj#y7W6kmVT2)Mr`RfBRtDu>30@LprzmGy7W6;mVR>|#Fl)fOTQV#mVPtBI$dncOyiKR2Q2-b0=tp9IM60dnP!uwOx>i37|ZlFX=3PY($p8N z$@Dg9qUWegZ(kQuj06XUo|nl5jE{J2e;`jGv&O`6z#+$K#YNk49r zCdP4_G_n72n>2BL+$K$2{}!}Qf4GKm+$K#~*uyk4Zj+`Ewcl*ggang#kerCyq>1|> zZj+|wWF&5rCdP4_G%=3bq=|9dCQWRwnw*p$uDOAnFq z)4L6U<2Gp;j(bq1*`$d!*`}Rnval0P*8l$KRFH-4iR9owJJDq7PBfXa6AgDAW<=SE z<|*RnS^D9Q$Pnn$wr<$Rows#sAltgp{j@=S(x+|R=$dGdL6;8w)D?A0gV?riG^iRh zrMZ^2bz_ukwsm8K2Rzx>qHUV|GAu!$SCb!#Box=}xp zM7FCj2pg#UL8^E%#7|u{kZs*E$Z%(0jXoABX!xx;PGk44*fuOY$73y+{8mD77N&T-+Q>F6TrRc^OFhcNrYqQnC0({*A(lxZz8Qq3 z(>5&BA?bKCk!@Ipfa3IJW*Zi+^x8m6$10czs}UE6H8iQ}mKW)=-;QDvafS?WOGs(Bz^hQNLMGzsh z2Yy?TKVX%%yhz8E7wNL)1vgT*yf__-wB<#*Zh4U|TVBwUNcsijG;K)FPU9T`I|0RJ zdo*|~%6?5{vE@bj|FPvoCG=>=rA*y%DN}Y_x*6ik8&Sta8JV)<(xXPg!^$8UCN%g9 z=$g!b2`TgV^>3tUC#X!_2`W=|g5qfqubOc2S7^{CP1uHnf6OLZHff?Uk)25z`J~MD zp0^@xG0Oz;owflsX=)&wH1WvoL7D>wAU6xS4QY?jhPub-BM>(9_89$^ptr|pb4*hW z>7FJ(bu)~hx5ubM(A#6QoS?VI=uU#V$LIl=9dHN-+GDgK_86sZu;oJ=H9)?*oq$3Y zK2V2Q1vGdIex#xautl4SHpHf)B&}5xxQi5M3AF(oad1c%n~K)ajnyRuMwtxVlTD>Lq=ZZf@Hw0d9>eWtgI)=dPxU9{+RJyUnlDnM3b5Dv79R;KQvl_|Su z(Q8I#Y!@w}CKE$OcG0?$OyWwSA9FyG{Y$p^%O~xk)hHCX17g}mD-*kDWy&sE_fw(! zY19=wf@I_=9B3DR{A+O+UdP<3|Iww$kZb^ zi~(+(*ncukofiM@s7&WB3!uffc$6vnZq6FtPp}Rr_P!j$@s4?!l zxmJD_lInFN2kE6$OR#?xP8p5UqUXJj<$Y93I@Qwb5=$)6{oE>^gO~ZSD1-fZ@mZsV z|8q!FtKR}>@CTFs;(Ng9qR_W5KCE1gVrf_4D>#_zyBu09xXkLi&ZK8seYcwQRIBeE zlPLc zSbdwB^mTT>&Pelzps(=d^)zWgzY8GbFR%Z>Cz`G*9BPCg;ycN?DlCJLX4qhTFf>Ah z6J|mf%*OI<%s%q74B0)wu{Fa4U0$0eK^MEIf>f_dusNDzFf(LVaJV=iu!LP5}S8 zv6A;d;5{qiej$UN2&{2y$OkK6^VHUpJ{<#nlz=S%FIBL9NsZcGG{f+;i7b307G~P) z6UlvqDBXQLTkiRLuy&m5AVlt8s7#pcL(X9!yQ3 zxh-t%Lrx(b=HW2!?=ZLpiD12iP@tojAH!S@)$(7?4iqp7)3iXaWde3MxsQCa zChVd@r^Szz(}?Q4Xbqe=je4;?MYe)DB()|yPU3TGdhgTaJUPpl66oqXV4yD8M@F|dnN1Fs5Ubg|iNBM0TyItD{ zELs~L%;WOzmB61+zTyWcKMsEm%w)Eg;nlT6!1|i<70&=>(iCq2=Hb?Xha0ah3%28- z7(PaQVC~0O?@zRm!?ck+DnD1@w}#_h3C`0n$D=ab3FXFrM>Su-mctGN;VK;5PoYFB z+k+9#nYRPX9z#a34-L=IL44&SCbP6MH??EHiq@cSMkef}OQ-<&XR7G47x+PXmu8RG@LCHv_jNV~=t_mA(kf)6SzZ?F2jP zwDYJ`-V>OooyR2qQed8T9+Q6F4b1&`fH`$v1migDJM@G~Kh<*%_3EoWVesXIvaBuv>oE69=v&P;^}5}V$36Rs z*Rw|Hzp5O3J;50ZJ+Eh%%u~PD#mLVg^1n!s*{@`a%Kwbzx261}LAgfPz#8|>QH1@_ zH_rfb-y99nWKdfFTz-++T>&0l7XG&;p$h+{XS>KH^4chQpTa1Qx;8>y$McoEHbUMd zz)W5nA@4R|Ca;Z<_X;qRr}@eGN}iN=F92rp1S@%MJo3h?UsH|qZBiTug@n8nDt$!H zp*n&!K8mGbWIz)YTCC9kbV-Y@E3lKKbFoZKFU zyt7nVtmjZ2K?k2kxSGg|_@wTIFv;YJA0Bze^2J&L^?T%1s<7duhP>5HLEbq^-c%T6 z^3D&JprH0p^i+j*xd+PbKdhA#VgQlc)JBftfrh?;ZqZ@&qe+=Xm68RR4Au z=g&!TYEVc<-ZLuwQO}_|f|q<6;gdw(2R^Br&`Ze^KRoh`y0d3O+b z?UcM$Fv{e$6Y}N)GkNWVyzRhDUOOT0ZD1aG?SwphpAhmiKMKs`NqP55U?xwnlGn~7 zuZQ~8-#Fh6a~=u_c~ew+i=IPu1ao{E;oJv6-c>%S`!r1Q$P+(2@{Hv>wFH>)$g5W2 zn7^5kcR!JrXXZQ@W%BZL&g-q@<>{OU%;e?ioCnP0<>{OU%p*_plM9qQDeqnY%;X7H z^71_Lj;nv48t3yc=b@007gVp$*K??jAkC){9!}();gh<{VUo!cKRoh`vil0>fZ_D{JAO4vnV7T+f2P4 zqUTT@K^vb&xRN@yhfnH$29w;e;+NO4#_~8V0cO08ou|U)mmBiFC-OQddG-1zc^!nj z2r!e^LC9MG%;a?t@~VNEybeNM=t3n=^9zBQJSp$a2WIj*q|np0lGni_Z>{>b$vEF3 z#o2&DLf(riwbc7i9l>#*MtC2Qcfu!i55OdoCw_V48Ovd<0nB*h<*M+d77*RZdW*HOqD49w(p6!LBYX7V};dG7)9l%o0B{ggZ@?+yZH@;ataNLk72 z=#dvuzlIp+JEl0ZQAo&JtkQjY4%HE?@@a%?h`dccsapk;OrH4Vk!LI)&=O$ABk!0B zm&`XKFOSIUtmHiaqfB0BA@6fw9(kRGycYeHyv{=2G+-vLvygW`Fq5bGp8@m8lk#r! zsFK$?p1jT;d8gFB-;MK~Q=AA233++ywWH@y9YGJDMmXa^kT=99b*I22lP7+8&($g5OghXsbbK}23xC2u8+GI?Evyo10@URNRSJ76ApU4^_}1C+e3Lf$pNOrGW+ z0A})}y!$OMk37LT^16EDZB_sF8t1#FI1L9H^4?KthI${WBlyCn5iTV1e)mb;e3)eN z#1D@=WBGKg0nB*hbyDGbG4d`U^13N`6JV6d>n7x_1!nTP33<;0GkM*Fyo^CgUN<3c zJTQ}|`BlJ7o|Jc=17`9BD|y{K@6GNJidjl~(IHR7bGUrxBh< zd?D{qU?wkL$O{Zr@-#mmn8}m!?i^qyFF%Do@+f)v9(nuKzk|m4{1j&`3JG~%sWex; z57iO;;?oEpBl7C2zA5KE2$M{n_~nsjH1f0rnDNLfRN>cpwu>Gh@_HzFZ^9^#ydFYc z{b5R84)uDFoirR@Ad&^@_NLR*TW;PO#PZ|obQq1 zT!KPE-t8(qrsq%{!QDQMu=NngJLr?T*TN)|Cw_V48O!fz2{7Z4cT$C$E6m7ynaJy< zjV@_NOS*UKX> zQ~k=(Gdy#;qmYnykxDD|9I7K2@6!l(BJyVYr0ztRWb(u>k33^}wU$8r9(h|-_=KMA zqR)xE_44+E^_EQCdZy02`>bc;@aq^H#SvtcRI7!>SD0={ISrOpX*Vo|du(f!bi+fy z+zqRw8@>eQF}zB;q3bB^hE>uHOMtl>H2*3C!ea{j7MIf3V+kH zT{M!&TchN)hEXPOjgU79n8{ls?*Q`zutvyhRjlM`{zPCVPs+PD0yBAQ z;>la%kyoaE%{I=jNpT)RAtCQ>mAD(X9(m%IN1n0#g_Zy_ z9(mT4GU5*E*)Cc{^Bmd90GRR>*UKnY^_^-acR^Z>^B`3ows7&F?f$ z$&>Q#RA44gu->=VdgPsEYR{g_m^w3iE@R@%>?sI8a-tS)Ut%VJf0Ctz+6^zm zJ?@5u(hd00XBoo_r5kzzb2luMZdd`#-LO!);U!@12F(vn&~A|OZVzDYhK2ECc%jz~ zw)$0H&+yEdghJ8{gH*ai&!IYku|AD(?!)MYnLepo36tCn;+NMA#_}pHf%?5}*rLKo zOAUD+oCfkLl)O7(l*y|Q^8O9X6F`NKmp)O+s}S~@+v&?o>c#i8s{rgoX#jDBy zfEkZGM};|+hP>eEAaAjfw*W?&yv0J^USK9~v5@x>Fps>&LS9~plDAmMn-9$7Y5pEy zCQr({9|H5pTO3c`VvoG5)W7SE^NUlQ#8N}v5taU_=TIHNzkM3vN+RzopVVywlT4oY z<&kGB*HeDLj7MIs3g3;9*OtgzqT~&RQ6_JRkara@lea|3djy!tTO#CHj*_=T$QuOA zfIKXJ*fNOq`iL1z9E5RWf!T)w5kR zjVw*nZfH@a-7rzQVGJ;L!$j$ZzXNkOOq6bT9hfJ8iP8Y{z>V}m*se242xf{eUuN#cz9a;kQ zd)-j2!iCGs1h9(8D^c>kgHayCB|=`?iVHugB{-R7cR&rxDJ01ms11Qui*HWb(u> zk33^}s+IsV9(k22%wKNE+e73zO5XD@%H%mh-tWLX@*E-W+)I=^N61?Y%;Y&j-gCfA zp634s%p*_AyIrO#c}_fejz`{B^>43n-br!BppcOFsY;uv_n|t1?|mBKLLx6&^-Vc< zAxtuP;+IFB(P*V5z>G&;p$dP}vt4wI$eXO>-3Fsf-ee)~6<{WBvXGbTDtVKIyy3u1 z-ee(fBQTSv`Nx2nJSp!cO;hp&>+Ct%Bd<*Tnr)n)oZ_@aAt7&vN{{O~R7dcjPa`~! z$a~%=buWTRCQtnE$TOBd)e>OFBhOl)bDo~?>=BAZ>o^@6EKgwsX|_#nM$7KF9c@tq`Z4CFq1bmp1i3Zc^{~MCyet`Q=C^& zNasBDdc2-Pt^9(nK0){x&v`zn`#ntZ$P>Rj@{HxFS^~^?(IFs(*Wp z^V3qC?I@&ko=Tgk_n}Yyg71BTu=Oa-d8%*9xrbnq$rHak@{C3+Edgdc@(NY>lb-FO zA|h{wlJ_Ny^2nPZ^&Hya7d+?_ggX&=&-;y`JHjGX#Zn&Qs||J%^_H1(*2*VTZ_D>65ziV3NrbKRoh` ztK}08ztl&24?a`33-14^T-<|8u=90>yaPnu7$whzQ6_JUkhck#$r~f&y#~zWjS=!rzf8#+ zBjim5X7V)uc3>t?%Db-uGkIg;FWWI5d0W-Ly~g=5Db5clBqQ&Al{QfCLv;ivd>Y|u zBF|ENQ_gL2xsoS-dE^<57Fq(#c;pqT@RXkIqIZbAVkNI9j52w}LS8vAlUFR{?F44> ziiNzBzyA{Ap-ne-3#(CtOp?TdX7UCLc^?AvWvls3DwI4a?+yiK@&xOgH`pWZ1NHBO zaei=$Q;tGH-s$S~a6O0W2+r|ogohJ(7x<*^O)$yii60($#_~ih0cJe%=BcpxYC~Rx z$Qz>MJp!Xl-Vh=01Tc@hAwpiuE0nw;Lf$N3CU1z4_b@P%r}*}aehdO(+`D&ycbn!srR8eg5y4oa3zsetF~>%VDhn%y{JG zs_>;4d6S8}VM^ZLVU)=mCgeQ{%;XId@_q#7kvB}pE4)(48z$tf0%r0wzZ#gylk)C= zfqCQ&izjcGM_xqz8e*Iumf|#BV#r&p(tUak)e)@nX@vI?d7FGvw--z@dE%Exp0RvD zOMn@Vykja{a=jUO%Za=ZN?s|9GI=9}y!F6L-UuP@2r!d3LdeTrs^pCj@=Ac2Jk7re zn8}m!?hC+7-iUbeMtJ0%QvZH8&W}iOPN0yCygc>V(Q~MdpodQ*Ttnmy@k!l`N+nPH z^2jrmXK4vA_9c{_-_3zfWcVU)?cP{^AO%;a4tK zm#dUK&7TI$!w zrTV6v`!P)N$P+(2@{C3cEdgdc@(NXWO3!xDVIr@el9#+p$?GTNT>#AF^%L^`2F&F3 z6Y>rKGkN`lyrktyp62%gX7Z%GdnquJ*DrqL_4CLpQ@>^#=li8N>rqI^yF;a~={ZzK zu-B&%E+q1*eNy*Pm}K(AFONK9`F$+`W<2t&8)U@o*Rx&p1(6q3^4^9~9(hqAFLQ;G z7ZviNz)W6L$g2Wo@}ffCabUh|H9zBOB~Qw`{ehXhXgqmQkGwO~ul9O|XU=pK67ot_ zx=zobI)d3gjqp4ouhJ)VSHmQeCw_V48OwKQ3DobASFOVGDs$P^YXR~GDtY(8D3do( z$omACN8Ugo@3d=_yn#aA6ksNAppbVjFq5bG9|QBqlk#q}l}g^gc=85%n-Gc1I#0@w~&{At&-PU$Xf=?Kg+ku-z)W7DkoPn&lUFF@g;yzgg+g8tFq5bGtAUw3Depc7%;X7H z@(MljMyOwtjPr#l&L=1&BX6}ztMweJBiQKE2wRVXyuCiDoAP%hPyF!6GnQY}5@5z7 zuSSJm>e()$pLz-IO6JA7<{#*GC0Ae#hzVluN~VRn2x2mAyOL>{ZmKUe0j3AFHhF;<;hyUJXy<^Cu{lgR4reg ziu2`}IA5MsU#uJDO01`6yXa2x<#=shj@R<#cr9O!*Yf3fEnnWQ<;&Y~zPu~G?5E1! zOQw}KR!N5Xa9Yfle^BLVT&2@e<+-&5V~2UuI^!g!m#e8vxYPJQ*+bxSWia`g}c&e~T(h6Zs3+ z0VkNAGNY0>>ZJbkjQRB;sh(NeuQO};b!IKU&aCCvnYH{fdE%Fp6Ti+Bzjy*TlW#p% zNul~MIOfauH8?uwBtUMBOXlI4Z!)O^xIvZf7^?#r?p%2gmu`)Nm)< ze!>n8?&rwik>7(g!&Q`1O^1_o$UT7jl&elJ*0UR~()(0TE`*Dh|4OP9S6aCom|N*eD|P^LD_w4-nMWnpY@{M(XJ&9C1nA&(xvimyI#Z$N zx`M$zjqp5b^<Im-fX@s{>qo49g-Occa8!feH zqfKM`+-!#Pb=2tT+UO@yh#Ng!8vP?Mcl2~=bm2|f=;=CKfw|GsYc*POO{1l(dku8C z*@Cs%)1}$T>Q65{hw2D2eH!6o)a5$CD$}t%FaBC+dPA(U~TgZY4bdFW|y8rbp#bY zjj;7kw7JSBb(g>$ZnIRWZQiG9>9?4!-bZboscqhhLfqz=(&jgTxvOVNn{#j0HqVqc zUkJ=?o>{BSl55&5W!?&0Z+Y{ur2>y-=Fn{uXWaLTUDJU~cxpTFsVR(`+g0{tdd^ z?1k~oUMS7(rv6Mf&8{OT@M(k{YW4`9)V&q8aI>XWZT37>ds)wR(d*Rg<=X7WQHYzp zT$+6fn47&^n%(18ZT50$_9S3#_VQZImR!?pDeG23mz%vjzS+yA+3VGx!+H+Y5!~U^ z2v(;6PWMgYq`Ap zE-*iH-W1Q#I_8KU#!jk#zv(%6bFve-O&`X>)u5!qhMAUi5{clg$>H25@I$1ugW}p- zr9EM7SMYA^l7U}?Kz_KA4*Td(gG1;^RXMI__iWf_N;k2SOesI#8jVkl(XH{%eH*35 z7J9ae63zgVE0cL^5A&|Il6f@mS}VB%yle5c9)hfK+wpsAX%*-^JcZjmqczy(v?4&8 z;_@ST+TsB3dp7PLq_(V<+A>zNN^;eQbunL>k}oT?R|lYh+^Z|3S7!qAs9zzy`XDg( z>I&(#H-UN7uc$TZCD)94DeL|MT^{v5X2%+#8E(^@ik{-jK3xDRj9v1Z!_B@_OT4_3bk~!z=CtzZ{)VwhYFU`>$2Kr8X&4 zNAQzRBV0(_*He8{&fN~1%)L0M+;>v7XpFH*#Mr#r+|R4U{k&S-3&I<=2T{kQDvxNI%VCH^SE$$`Pa4%)u3g|NTf|dJOGXGbqKM(0S zR7bGNrxBh<1a9(4-A$;436xrtz-m=%d%GEO*HE)(YqP6Sh?_lIn*9+lH+!};JMRu{ z_H1ePFko)>>{`v1T+?hR>s|_7Zuact`>}heHhZ=-`y2JErTP`BBly{;5#GY%M)gfO z_ZHN`&6ZltxKXv+W6j<~&7Pyp{s#(iv*$>&-v#Do&yi-g+M>;#Bh4NN%*~!ttJ#uk znk{ABSAEA$g7%j z+-Urse#Mf1 z4KP3Y6$>>9XCB8h+oUA=6(RQkGJmGpCkpOzSHPklkqnMcqOa=iG~icgTj&W%#Ft)O zkKqaS+wHr85*X%LFZa6WeHe{(QFu5khbwV#8|~7&!|m*Pcpv4|&>`b# z913v=U943v)U(?ig}7TbdEJ7(DcPb5C0h*bxkURurF|lbG41<=_NBl~`#z!l5HMf3 z`-IlBb}Q}sh4$INO#6Pxe;AlA-2FoPpTJD}LqhwRc(r2MABdy%UwqxHkn>IXpF(>l2*VB?s_0OSL#U-z zov&y23lw77pQuf{qY5pz8QRY$+Fw=L>))fazbdqM0%qD@71~z=GwrVmt-k{EXn##; zFW#fHzb5$)05k2c3GJDCbqaigsddi-=5BbCyFu>T|MF(UYt`MG^c?(mk~0ovct#AL zMAP}A$bA;Q^LCQ67+G->*j7O9rs2pBHP9wE)iZaKUue3?FZa6XO&E=JQ|O2~_SIJ5 zvyLEKiMrjidjr-mBKqAV`q{GaDe19vxEc!KlXS>^4u|15gx*(k#;n@~2Dm>zi0jXA zC3M{}$YOgRv2B|8c-}juHh$N0@T(+e5p-qVyR?(iu0nq6-J$tAO<}r2`Gux~{c^8^kHToI zgRKtq%M_I`A3P(;!6zrBFT9W1(gXN%YD*jMW?DnKc}dPB01O|`AVUYQqY^3q~H-mNzz%P;FH@Fz4SdpR74}Jyx zdSe>mlv^(p7?grfbW!kA=2%FJ*^OBxBHUc(xS} zO7-dk;>VWB#mHkgv_c)tnn#{NvJ;X`=NBACa>Rv5Te(f$=olpEN3WYsqaW*;0Q5WZ zn@*>n1eyudrxIues;~+INcO)BvL;QtN7v%qjq>Qny`r}PQT0vnyT~cgtw3xAKhueF zJ)nl)=A?WppMEa%!ksvL)503J3yQ=w{gXf~uB`&1Y}^BsKgHzJe-zZ48sPsZm{_ev z?Vdz1jF33KyR3=(Pl$E9XIw8@B~?0(iguWbc`*&k1Bvo_a`t|m)(<4o%TwtW_v^HN zAaMoZ;Db7?A4sGYm2Lp$Y5hRrHo`vx^WF5I_v#16xd-hwom-{GSr`20!fvOu`gVN4bCa47o|X8%OayuG9f z39>Exg^!U+#tS$U(qSGB9afPJ67^<@xnq=uIqL|nw`aEtFI|$hv}K0O=j)DqeA_@cYSyRmu> z9!hj}!h}5cFHz|udJfeQtoCVyb6^#`4ozBKT^ea{|ix zgzP9SAM7%NYXkN5n>x5M9?`+|{}}u7z$mID?(UfhnG6jXxvxMZ%4HHF0-~VFcw|C2 z3|Hi)hgIW+1b2~upb_t5H6q?uJQiKgc&~Rn(N#3MD2l7b6;MGSipr^czdByeOb5U3 zn?K&Gs#o=1)qC}h?yl}xC(d=pztp+biE};sygJu9ai0Ca*}2wu3AuZ zT$~1CX`v+d17b$P!(R))I9CI~sID1#l;XPmwAQ12we#R2zD>ys9yUYw5DneWI&>$YDn4cZ zEJOFj%Q|#F%h3J$iVoe+GBn4&szbL!hVDGzye{mJ^6W+(x*alf_XFp3VQ1>l{hBg# zza@vxYQoU%lc6gDY=*9uVDv^Ka>x1zL$?XldP7$PiuS4F0(R;tX`;mXSUa}}fEhZM zU{qI!Jo5EJ+PT_4iQccJbnYQaJyLUR24`-_IxR?e=yLyq8g*T$AQ-jRB9EM=wVPHF zm8rEqtEK)MAr&ipie{&|n=P%X*{-s^uDM;gj?CVr zW+3ek7_RzP~ z?A>Da$-vp{-Dze^sWDq*tCoP5&E6eKIj`&%vwu~+2C80>R)qaN4yT07K1lhET-DuZ zh0T^$)$Af=yTdbk7n!|B&3+w1Z1x^8dlzsvdykkMdt1%kBW9lhoXy^oX10_XvqiS* za`3X*ds3ZzkC>fMeQwe`(u#1FkHe`ZvoG*bs~$otY__zjX4flQ;4w4WZSr9DUNw6S zgxKu8Vs>z~n!Q)d9t)grJnR*-=L2W6_okUGrN(TLty%$IHhVArw91>?d&TV4s@Fcv zBdrJ@`8b?Ps~12^*LMfNGn3j$Kf=Q*^_ip{ebH8wdRplgeQC)j`a!5{?JF2%}ZKTlUtN!;^W4_k0+D=P?LXwT5R$k zV)CGWtI2jviH<%TTYnQ181{sIYB-R zoXxiRglUc`QfkZ=*{XHmWwU`NN80AoW#n?z=T*&H6R!7hI4+s}xQ{9$EiJ0aP0BK) z!Ho1QGI_s_bo6~SdB2QwDR5rF_sd9M2b@jbFC+aVa5i~=+DJ>OFT|c|kyeBSJ`Sgm%)ZV? zm64Vf)#L_cc~Nt_vX)E^h{+}#D!>MtaHs&AYo7Q9*jV%GNg&NsQzlP*MNaO32jmKg zO$ME`ncb*5yyxk19qAHG*Cm*yOE68BV45z$G+j)Y=pu5WOE5(j&{>xz)nV8ZW(M3% zx`fmf@;^{l2#G5!1kSDy5?6QyIJ-hfT;Utw>h{=Bi&L(Gw z$u9wClQYERuYt448EGa)NDwU7Iv>O__L|$cfjrN%1<+c?cR+ zhxa^PJ|bN*({;&A(GShT1WulA7i7uHbx@2b1F5>C^_l!-1PC%S}FbOD|FrCD{@;_0%RbjeEBB`ZyrtTbJ+(sap6)5VmD zE+QwoWTof=I_uJ+I_&jy>3#@wX`8M~+caI;rs>i)O_#Q5x|lN2MdU=6wkf&@9pAWp zQdXz7n%k8{q)WSWUD~DT(k@Mxc4@k_OVh=ai7p~1y0lBtr5#UMyiln+^z?KYO}eyC z*QI@$F74BFX`iM``!rolndl;NqD%V}T|nnC%2gc>^K_X>x@4#8lAWeYcA75PX}V;m z>0-)67m*WPvQu;copmWv9Y%V(%qLx<>AFPIbcv?v5>3-3nx=~>6J113bcv?u0y^tb zqB@N6bh(Ihanf~h(sXgsbaB#janf`#WulA7i7rlxE}*k66{^F-o-Qj%mkoNxEW={W z+sy_!V_pE9x0?-e##{}Yx0?-e#@q~?x0?-VXG|$I+l|Opwfji78{p7Ici0Vb#+;}6 z{6q6dE5aflhf_h@&5b^)?6A@z8H^>hL0Q^9WeydOlgS&^J|8%nyirVk z6F8f^QB3|0IGemN&15MxCW~xUt_TK`l0UlbBosoK4;&Ccg%pP2MCXZvf6FZ%Q*+N{z`PTb21A zH5qu)V*{aZ5H5qu)llfPG!M?oz%`Fk<>T;Oc-_hRykz}e*Q#pJJmv&rA5nJlHo zWRX=z{+>;gf3p9cZxKX(QJn^=POS-l_&A*CI+%Nq@*BA-`x8AjN{ec8k+NOwnY@oo z{y|MX1VU``4`T9k;B4{_V)A{!+2kL@o-ETzU|k*)dxynNvVJUPlg$S99f zeJ<5J(u#1hkHaY;vlsfPGRo4Tnp~$WTQs*T+aC&(H>=49f2t;L7L!i~&L(dblk0)A z$(zOGH-NLro6}5|Qe(2nR&4?=n+!Z@@@6slQPpRo=8;x}=X@MaHJSX0k18fhi)wO< zvW$7gICgI`d5fCdd!3rRMNFOyoK4;$Cf^C1P2M6VHv(sqx1^aYrN(5Dt@;+cY%=hq z$y>zacB;>bnnzj@y81YrdNO&Kk18fhi)wO(vOKK0T{(Hu*;}`DNg2@{egIOQ|tgWUHFN%O(R)n*5`fJWuuchvt!1ghf6Mr-@9y(MJ`N zrA0NlL0Q^9YexAbGI^_--0gEUd8?Q_9ypu4RZPAaIGemxOnw14o4hs6WGOW!i)_`G z;ANA6Cr#ceCa+PwGE}cfE5bS-hhu#PlYjM5#bjwwO^#}nXL%;OWb!sOx$}B8d7GF# z1~{9%O-x=1oK4;)CO-$9P2QGfvXmN=MYd`^c-ds&Nt3sU$pxyUN#wc(&X)8^50dTw=|EmBHZlba9lF^86QZa5lL`Onw45o7|FSvXmN=MYd`!c-ds&Nt0W|J>aJ1f7D6O*MyHMvMxuJ%m6flU5MO?JLilYbJEj|0vo|0E_~4V+E>NlbnWIGg-a zn#odXOcvRykHE_&15cX#lbAeC^|@5@NGrn0J`SgmOkU`tipkQVnp~$WTQs*TA0U%| zR+F>7Qj>oclZ%0~$v=z9R{&>|e-@J;2F@n`oMy6=8k0q~>I3ky$-t8)|12gys`_ly zJkpBroR7n4A(KDxQN?6wQB0n;ek)$e$u_U(Q2ZS*ioY94vDSd;%@4lzY=4lvqBFZi zYn}X}nF}vaFLjn{-#Kl*)?Vt&SHE*Y-)Jv&=Ih@%C;U%)sWV>z&-oHK_fluR2A*@w zw;CwaMau64&M{tHSiWjE@P3;S|I?ELg{pdQ2v`Lv_&nf43Gb0Xy6d^Nn_M_^fMNG! zAW<&fNebmiAW=176G+rRFk*9BkUK?qaVK#fs`UbiTu?Ly$EgGCG*J@$0!alDi2(y( z0*M+3Ms>}|Bd@<8{We7VFY$#|J&ytm+bcCkb8QB-{};8FAdwg_049*Ao?z78ggnwq zYrjGBM8}OOVM=Ey^#aXJ?d%POA3unkUYGBkkr)H9U4(N8_f!88onO^F)HlPu2pkez zUj!M)rKFCMMkJBS=SAXl&8zMQJKr=ul*f@_d0tKO> z40kh?AwYZXe)F^Wt)uKs8Yo>0#>h=-(<7R5ptK?0p+plX?WZRLrKh4&FHkyijH*9Q z^Hzka>%jl>w@`FM#uZs?mBiE=!~dp|q2n^#C&6nXC3So!zHDaEdxv5cmFg_|1e|=? zQ!2A)`*%8vN@W%`eXp~qRAx#34?2rV)9xfmsab18w(1J-@>)}xLGRD$3@J^SA*IO~ z(m-QSDl?=Ru$du6UyIY#B6k{*Bpyb!-VAAxV(Jekw;3!hCH0gvBVivd`UX{K2yQjO z@ZkpJP79L6w^|*~3P*dXr{>0Qo!tLX-AK5`dH|ReITRyoSc{S(5kIXJA(6RRjF5q( zG_qeu>hPCjq*@cQw8U^YP0%CK%SWv`40ZDika3VErZr>O+pY}Q^qQkRl7qZgN|5(# zS5=?*s_~IMzO#Tpi?T2(g2di-UZyO&z3M9OUus>LBIfAZ=UJLCVt{L`sc=h-_6Ac-cY9 zQx}o)6bC6!I!Fas>IUklhcNE5*r(cJGbLN1cTRjAfmMCw2>4$??0)(`H)ZBBBM&h@&XpJQRJO@$MqNGT~h#OifLSmD) znskuJejTa98;v9Q30YcVIGk$e5$WZlRz-eNN04#QMWjE5{m&K=RejnP6Z`(lr$k>AIaT@Ds-c9 z35JiuK5oIk(ibRaKp|v6;#%ilcADMUS zlUPJ_w03Kb9*OK%`0&?_@AwH>T4FeyM(7vm<)c z(6;O}(shwF^G@*4Tua-mM-nrr#BI%bj;d+0?ff+iP<2wp$5e$>{2ykWU8`1qwAxJ1 zPaqSj(CK*thK#3Yg-p-jPMw|=Y{{H^cj@%3U_0mhZMRO(3ET^F_5kPCE+$BMNzMyp{#fOx2>F zWNtMgccMRIZjFGHH&rV@(fQ=m0(Pt&NNOpue$k0l0l-YvI)X7(n~*!vUr}APR>xEI zV(qb;H20=z30R#vBs^7{0br`~dtT!-sR=iEm2j z$(l#XeS`rS&sx@GAJKUivQ85<7~Kh#27spmKxj?|cV!)5XB~HCBS6!YEd)X2-!;>9KTX#&bh^6G zou}&=GF_MauG94luGuV@XUJqM`$H$|nQ2SbnQ2SbnJlTFcR5=o=GNyes}5#_{w7Y; z2-ta^ov4NSU$@SZy*j$cokk>yv(Tb9A6ulD%yn}2fyJeyj*>f(R8zp!$Olyhb2Zk?*~C;z>|9GJ*CNUYEbL{%er~Or{Fjf6Q!S(l z8UuFK7TCwHz`qfoSKzCn`}ONZud{6R120-u33@T~ZXlBPo`|2JIp0I6r^5FG?zs?5 zdB^-_p!|+`6UrmUs>BJJ^Q{v<(cC)m)04MO)}T`F)(IB%{p3?frqes&h{mLS&}jsr z@D-?CWWmP&1l*7nwB7;kjsmb202@_DrPl{?c7t&08ZoTo1Ia@x0&W7#*)LhvIaW`k z5kE_7!n?joB8G5;)|x|K>x(MV$uxCTfWHstr_=Z0ZiNc+eK_{J!U0gEiPT;hIL9gn z!oSE9O``P(9o<%hs+Ukv_qGsR7q|p&Sk(-CKnfnRgVyjQj%r7Y1mAHFAjz;skc`5` zM33Eu>rTY*?W)%vfF)SL(}o0z=UiIw`{EJ9He&kFrwSO z&$qupOThuQ-N~M^@OSVG__i`pAdfcVGXbLmYk>5V@&?2o0oI4wD+uI`7#rxc6awfQ}9qjQ$kaJDtOQ>~i`B8Tn86U>kzv|1)TR zprP^yf&K+|M8icu!{u9m-a+UPpwaT5fF6%!KRm(?=3QZr2?HqK3*vxGCZa;CJY&6W zEpG$-XfV~;<(&;)0{j-coIV0J4?2dA172^J9|pXJ@bSPOw95|%{srL^fH&CX#lUHB z!;^vk%Px-tr=bs@47|}UKgr;8fWKpxR|2O&4KD-UWS1v^lZ%Bf0N!kup97rhzm)g` z<%@yGG05S|i9b+&g~6`_{#l@WC2+RqdXf*6KVZsl1^#`Y{8`|vPd)IKK=~U6Uj=+u zp!_S~XHfl*0%aIUWrcuRKq1mNvxU|$Emi~EnfuusJ6i01>P7fUkbb)6FmGO;fGnVbI`(7u&iyU z^j2h{wmswIV3&g2HjnHqyB^@9AVqEs#Ax~W7vQYku?IlD9JM*&(i@T0IH$(%Lr&f7 zR6#KQ5HK1P=NwuMo-o*aS`A(>ST(H%uNk@XXvTkJuthZEHyCV5Ag?I?i;+7Y^FJQ= zuePu>kXI6?M^D_|GEDe*E-=>R!a!a{e6T62#oUaKGT23dJU2cX*hr2pCw_~;(CN{5JusK# z(9PYf*l2*OKypH*&m#*v+3_dPF+50}P_tD`o7K3ZPrDO^nX$pB^?mT!k>X{b$F4xq z3u}6KQ0X9K)#2yjBarht^)e}T46sMR6n+biV~uzopfT!Az<1+l5gt(bEM?(X{NB?n zS%cuLsC^SS)}ScbJN6U6PXM-A)OQoIjBwwXI1~izSOgQfip2YsUP!Fr>*F^g&!zdj z0=0~dhF9>29%m>#q6R~?#k9)CCZc#d1P{tA-GwZCP&97i*$;KZK?jjGBgh?D_tjQ0 z@Aa2LBm|xgM~@A6I6D3jo`&&8Ngooj%I-&L2b6XkW{(-su>c#uQf%Pl_!=o&(g06L)jwPSIZ!2;HC zD9|FSxEqSf4ncA}_&Nsc$ux%c5K4!T6_Shw!w^o$SFB>%oQfxr=o};yNHlCuCTglE z;X_2=LgKv|2|cyznBAH;*_H@27=igD(8Vr(%@AsdkYg8rOoXqHTuX#Q{%=*LC=!P4 zWu1`RK~jgcmP%Tllp1j&1b962tXKu$haeozpYkj+HypH+XNU-T)hd4-^qasIjsgvr zzYDa5(C$E^e8Ol?dqyvy_-xhfD2lB@@&VP_zjQsavhc9@Cgik4gpWj>@vXq9Yr=4u zP<#il&#B5|(10~!BS6+K)8fPM9u#+q#P)(5#ACSdlVii-C)w-T<0KevE2(c%!zyX;KhPO;d7nAa44KZtoOsPin8$i-uW`T3alrc zPi6lER1psUGhU90se8i@Q8iP6(YZAIFfc1V7bx{a_;Et5;wMlPdkM)#EDM1)r9UC7 z4NQ&yj{MwqxR9Anl?3s`ndV)-tczlXC>H7fy}u*oRKWa0ZqK@@B@5K^ZrC zMIg_$iWi~Q*m5Ls&;#KKlaW0b3N-McprJjao=7v}=`=H5ATttFBXMGp>=vH~jN5*d z_Koul))>e;IDP@Jtz^dQnCUR%bvA=L3luTKU@+rKuyfzNOFeQku-B-WkEloPHf3wE zZO0!n*e8L!n)u7WScT7UM6%$x3!AMkd_bPkpNima$}uBU0| z!^p(XPmx`rb0Kx)UhvSI={%G=G7F0?W5WV@o#I`A9SQH~jLU~^@jk%j5Ox%GYky$1 zgyCCm1FYgv0Aq0^hk@QXjL*7F1~Ozi603L`(n-Z@2s{N~Y(A3V#FwS(A@fpV6<{%sMKkJ z+oW~48V1B)hVW_<#_~|cBfpcd zM=+(V*wX+T!4SA=Y&dWgFI+tTu0pj|>|>O!r_xaA24vB|NS3FgN&xGj72iL_5~Yu& z9!y#d*|D%?8_8D2x?|FwhE>POC|!)K%;}muBlf0hyhCRnvfVrD;Ho+rzXlRC=nk%< zyTw-mqXBVnahw~!-C#q=(EAMrw=0T2WH5X*a76Ox3Pnq-5wuD0@IcXTQTa=m0jo=I z{;P}-{wjmolfTLUMmvMtBUvgpHu$eHLd(M}(J_4=9wR+=d(0}A!lwCQ3Zl(;d>C|K|8ej*lzn5Pm;L1XzVMSZ zDE*GnUw}sbs+DGZB9%riQ^LNrLhw6LE&3ujz->V_dHx z?48}OLG%16n!$rNAXELx19aR6!lRDv4e;%9>H`BOD$iv znx&%LtQ3FBMR$|_l_0YIE(nQhfU*9M17Q8v0`Tjfnj8X<3UJq0-@?d(*Ok zn#HuN1d*rZ`2bR}7m{8Gopn1!@OAnneEpwu(e?16^Nx)eD^8m2^ zw*&C>|5^3km!f|o7+C*r0hnd}jde0wpKH$3vdKrJUTt%b=~eOUW@zE$IvC;>q+4TStq0X)jGXpx`Si%CPjki6z!U`msPG9>&3=r%hf&77KpIs~n@NpvXCq?D5zMb+18)n98~ z(B0#gUa0_=(@06~L1^L3v^olsi$KAlNkPXY)_Slei7Bj3J&GfTYl|mnUh_s;)v$;v z*_2My72^M6foRZ$;bm`uxM~1gaaeMJ81^le0Bc4)k@HiZ%H=1Y$}uR9+zRPAn>=zG zOrW*6QL7&Jr3{N*`M5Meoo_Gd)ltuo+|Z(lKzSt^#$$5@P(C&?8hKOe8l!bZw)+S* zA3=aePXzY(%QHI#Ek1V+qjmBk`{`bI?%aV)rcPU2p%vGA%D$bdY(Av7bWeU;vv-8t3Nnj?WGTX#xe zlwXBLZTiEc3f7XSw!P+Sqr+~|!8^B!vC~n1a%6CBYmsx?un27m%TUVawqadpQy8O_ zUaxs%wgT5^UQ-W&B{Lt_$IsmMdBUZg<@E0s|57HZ;SfDHN5TE0}vooe(% zP5UmoHVIQm3r{e$w)YWgPJjTfMk3I`S02gr5o)R+kn}wACai1J^i?R}6@Y7Unm}`K zkJqYcOw-9J0xe<`GTPUCn^vtXy2E_OfMLmF0IqIRjsd{v7|^7;nvep!5>KIdmM?14 zHO-^f0!#EZJ8j-=y)<}@IY zeMRd;EyO!1oj12DhUAcM#hxn7Yt92x`baw!V63?nblz9E&YBN_^1dSI&uLOlJyqTK zKdJg4ty=aKqY^A_0+&gZ6)8_~K&v}io~tb{ z)V$`Rw5qZ5f~)5KbRs*og`3gA|A)sFFTvVAqO6bgF#+ zTie3zC@b_|yEeCEI-uC8NGNh8*gIAYkRn$CCURwbDC3~)%Qm3O?7NWK+4Oo$Ap1?E z6m-+({}z#x`R5Ti<3r)$*`4t!jg>tJshxc+QoJ&0%;>G=*p-EoEvvUZWfW(VC#lJFg1gpB;Jln^|<2Mo3wnQl$nLFFpzD6fu?PxTJKA%^*$EmTV(b>v<3RpSsQoK zMBjKb+F8Pt3fod;+e@`wW!N4iwyRVTx)o;MBvnTcC2ij>Fxz;iz;r9jzExm)gTTIn zVRKu|zD|nhwwPTfMS5E-vd1Rd+CB){xPgNnqWE-Rxb0$% zKTg(>_1*kJldpG*l)p_{<;NFR}m zR@i%_PYxRP9_impW%5XW8bAtsIH->XKE%!fg&y*^*N6P=(x?8#Bf%XjMM4j}GR*_8OnKn- zH{2r6^d5Lo4Vh1qHuT(%G4Fxb1H|b)@G60Yncf30#=Hk!PoS;L*ZF~0rabT>-y(`U zfH0Uo@Zx5?2VPH6Gu{I)E?a9Jcro^gdEmv3CLee)ntb5JX!3y<%m^>aU1QTno;9)5-EJt#H$795id_2aOJRW-z1@w3CkFDdB|Neo8nY<0;{U*?8s`N(m>#f1+3+6HW+#JMm6ZqCq;5!U=gCQo;#) zQY9(jgp8+z6EdC>PRMvlI3d@U5>7an2G)cV0!S*65>CiMDdB`ikrFB4gzIs$B9szN z_%$}akO?QOLkkoY*ioYbJ4#d_^-f0wGDuV)^=-#<$h#>jkTEYRkTEYRkmV5aAW?yg zEuv?X6cxzW5)&23a$Zy*V@m@X70B2!6BWpE7n-O*E~^b_R3Kv)1@a^+@GSUjM=vVy zCBiPJsKDZCfU)zDPz+#4jREW!YIleyj%)3SSFp_*zK9*LYrF zqmuBod&vPgd@WWEev0l!v>ZowI}+VZCmp99@l$kn5VPKiQtK4mO|3g#bax41-RbEv zMRzlnYofau!*ghf?!LAiI0w?GQgk=XTzV=k(cMi@!$B-JqPrc5?xsmX6p6St7y^0r z5-T2HBf8s>=Futaw=J5e|a$Y9u8DY|mB^-!)nJA*0!C*Rt1JQtFViHI=P&v9d zbGZoz;zqB)BtWnTYK@(O3vpx4()Uf?_V(lWtzj#j%7v0==!+Uero47nz8;$ds51Hd10Px`UBo zE;2RdB2!{6=$x2&40Q^{T(Hs-a}k7Ro3rT?=$vgNTN!gv4TWHaGz2pwA(-cZjQ4xU z(Lxe}d5fX2=NODUf&9Cerrn6g0GT$P2TT#7A&n3XNrWg(sSp+}bo&?Jpa{_}y){mBOgxEQV=5!>QlRfun;w(E6 zr4-HCjiNcbX*4HAv~){Gb8aUz8O_-fdq6ijFi14#SVEK0oU;f`MsuD+Xfm4fenK^x z^LbQT_7)O~=In-O&LD=t9zy95GC-2eU>L#)4VYD2Ldh>AYN0NQ=In-OPNJrY51S3GvoZS%3IoA+siGXO%zY^g>Boxir4bhx`US*0R|0Jnr zkWe&dH$-##rIMB>rABOm01suJ75fP=MRSHUnlq%)oV;r%qdAX6kVPnp0g`CW>j_Op zbB?1eTcbHgp(qwdLeZQdjphtVG$&mh;=vwf3q^BM*C2XEqdAvQm6%D0SE8^F)-TiI z8$=Y%**y}w17wQk3?Z5`B+;CYlhBi>&?S6wMjZXwHyCbACc>_ks=6OQJbh zkSM&48w{I(MW#qJXGo$s`6P(t!#b%Km0{^XG-pVnIfHPlkQdEK%SGs6luI-x^+f1# zLao?Th*GRoM~$`WD6v+&`+BifjP;?Fn_{gP!yYHGR^(9~v8zd}6_;VFl2|LohR^~> zu~v-X8b)HR81rJSxE6`Eq9N49qqqgy#r7ik0yhgng;Rb&77qQ#F5d}&F7QKZG4$a1 z09u4_=PZ15@gca+5F3nS3h*8orE`%L_2|gac+^afPLSY8J;pkdSx4r}Cf}nAnKd60 zT-J@uS_13>RM!Jrey%m*DuAk9>_)&BQY)d-I%Lrv?c>A0hA*J=dt_59bn}O6qN9X0 z?YB*^0$#8kU%|PTr}ey$h0l9 z$sO(d*f)3^sBBpmyj)246Wf*%xnR}J0LuxqiYcA;CCMOaPpV^~K>;#!BufnvrDtv> zbsB@_4d+I_ktgp`HnNfyfAG)72miz)_`YGb+&AQVf7vPb{tjZ;npK402ImIb&Znm` z&JDD`1Ww<##NdH_3?n>-VC9w@L`S#RIKb zv#74yOkKBeU0lI!RDm^XA(h`_%J1QFx-0A4L$zA7{!Zl&n(_y^oSS;Egv%eG^2bg2 z<6O?|KQ863QTel`{8=vlf@xenE7QZT(EJ}QLw z6c6nI`eds8340J=tN1a@)8fGZV@D&IO+}&7YGk2sNqjl-Rah9pu|OXBf`V{uOlMjU z^znvF;eOL82-hx_j-eo2yF{K!#GAo-yukeAVVJ-aglmrwn1XQaqXp(C52G0No;D1Y zB7WLXBt@7q?I6{z3@+c70lN6GW{o0vh9S;i;>@e|!J4m)V*}#eq@@5b&GnJ=HX2|u zbvo^sp>IOR_oUvTt#jtxB#)0l%ftYbl;6to_{(#!*f~L)?)&Tf$&uxWUfo3rRXy@; z;VMDR)9ERJof{;9>GVd3-G^ao+ULYn8nj6Pt*SJos#2~>M9M^D27NoK{gFKAc783s z3fBHelFlcy+8;^xTCEw_XRYkz`fz42Bct#fF`t4kvX`&S!SWqNP*L_Jq^VULLs@Jr zlHG_w%5Ga~qwMtT32}O_>voW{C&DA*x!5POAeudi4hLD_A?fVNbU2{e4yLkcbT_sa zumZxSc~d)9zy=hEQ{8(Y}`^HW7?zPSa&3Ism4Bd{egI`5u_|1QG(Wp zqfo4H>;%B)5o@IM5@gZrf|Dvi%02qAn1bSa_MWi{YjXA$n-sP4c|^7(M`R0)Nd8mIQUYBI=zAa8+o-03<7o7@ zCWlO@w9930Cp`+NHw3h1yo@TMQ?(UZHKJ2_<)^BnQ>kie#u~s`nJGM3nZ#2-Rc2-J z?r{1S#B+r$8?cBtW)~2@AlC^BfT&AJ|#dD;zHEkc*LxGSkgw(6Rs4qtH1`!BN z2bb(7p%5RRW*$Q;D^4Rar2!?E^czx^^c!SJr;P|aBq~e#O71RM#&1el#&4Eoe1=O) z!13CryaXK2E7+|0wDA9e6_Gae@GsPAK5rzy&^}@<&eiAUziyXs0K2lsAdGar48o#v zJSiv1tbCGO^czrGIJ5F;-CXoVub@Ro_pE#}c+tfKQ}dihL9O)>?Pmh+ ztB`mRWr_a*SPdZO+!W4hRRWLa-Hf~-zEc9-bJij?C8hyzBnE&3y+gmky~rC#t7il5 z7|^o$)2{;Xn*L6$-D0dp)BghlH=S{4 zTGP3`sA(kJbUgrjZ@?V@Chq?Uo5YHxnsfip_7Tzli3O1Gy4^6$Q%36cm0(DAyD2gC zQnx<`j?`{%CS_pzdeGVSXbx)EioU9&~?2 z@pj;aFCeKS;x$3{*#7wT5b(t10M-I4B^g4P-WMgaH>z=C#IAYb`G zdT6cY*+04Gpu7i)7Lenb^4CrImG1Q@=MG#AKswN|as$?aGoXN}XP4pM;cf=oc_>P> z8;Gzr08R;U-vVfZa&0!;>)qqQ^bZtS3!XswlNNZ@eUURBm#=*<9;1 zq=_qmE&xD3r7LSIAvph{Z-?mD_z#Y8S+OQrQo17x>BO0=0Y;#Es>w z86}02a8siR2Ese%TJwMu?t=$L4+iB9P`ZGU_$ws)5^xiMLIR!#a5Mp50VoCFR6xS1 zLz43g(1X7d_Zgvi=+F$e|6xHZfSz~D0ko&FmkZ z2k3PCdFHtV@>@U<=&cV%AYdjSGs{BT~iMQpld4qwGXeUrk)=R{|>hdY`mtP1%TJo zdg7)v^%Ld#!f>y5?*UVCP5qx1c-6_8>L=5h`X&TZ*3|ESnw7TM7=mYaztYibb;gmNGI$nz|H#x2Cq@#F~0H7KatkdjS%(LR6hf)sga?& zrVa*R*3?QJx&@l^nreurDUsIHi4fv7bs+#=Q*Qy_t*Na%vj98=Am_XTaEAUZYwB4lX=-SDRKTr;ARk!n1K_z>VI!)kKzG`? zp_J8e8>or5!MKNj?BPMH4Q&A@0_Xt1X@-PTJphhG8|>e;x!W`kP0Dc32PI78-<>I{Pbs53Y%(88(46}MAQNfVO9IFt{8o;lNT9C}i< zdckNn727NrPDfedApo@itir!jiYv^}^o*2_SPg1nMlKG{C<;Gl(+1QYsQV!Vy901s z6wLn-Y<0x3XZ{bkwblr*ed*b@jnl39IV~Wv6P>5dbD43XM26dKggVhk0PIAw0eDW- zN(4@H5g5pcphLxBu)G0D;R_(N0Km6^#})WrSZw|c;wyMYzuL)qtS_bU(|1s-`z zuUvl6oG)ZQ_7QEows`bWdPQXwPC#ugDzaw;+*u&c0q)d+FxZ}OHsC}ph&2EP2WQ_9 zxUqu!wR5^gs46!0qOKfuXn{dWM!zGOO%^96mnTGnlP$ar4!tZ^we&H=-K{Wu92 zuDg`Wd1t{{C^P7NhWxI(1#Ml3B5T3bNKL_mhT-3C`=f)_-NZ29VWg(uQN!@wg7-i+ z=u?LLeu`FKlk|cuNKLU(-dQu(+E1l8BBIL{w8cy_Wp8R#Stq%NLxpZA!kbM6dbBA4 zkToHtkpmuWz~s~;k)sU&T1F}jy5QfqYVH$+PsVs?{cvb zL^|3ugUCml;KyoCmdZaf>ACPts*+wXvuiZ<8c-ZTy_(XcJdSQv)4s`o{EVb0PrG#d7FqW7xhh4Yoe!oDLyA=3EVc zk2#M4pkq#?Ql*S^WR8!($DcElc%J5T{P_q~u*3WUz;l>ZoN$<~$Em{{4Zu78wBm%r z%mM>DOf3L)n8^7)jUw0j2t2rkc$ZNTM~?eY3lHww0Qku90|0MuTZv$Bo#V}sBVuau zy5<>wyL1%s6I&lka_BVTXQPwJV6;UilN>r_yzZ-l(1LQ;XzTi;Jv!=~2!vU`XvotH=-hv7z0xb+9sRIrFojTVZ zM3iTdG!bQE$o&9pZxZn=B-RitZRNp~?fZIA6QL7=)}2(0yO!W6q+?_?0Q^gO$~kh@ z(KtkAEyME~da9JQEQljr{xCc{%33xZqWL8ODrT^pzzZ1Urz{tgfT5t0}`!6Qje~H^)z7wjv%(`-;uTaIJ6@#Yz548O=trl7J_wSb6uDq2r z>S~KhbNZu?`Ds;Ge!MoP!-+V+P(OC%$60fB0O!u^%Fz;Y)|Lh>?oq{a;&^;S-P%pc zw*cq9?Zyo_^%yTF_Xs+RQ(_I(Q&b6nto}CNYp*33ho}3>^zigK0Pns~Bb6TT@MH}; z(BWwxg!%9^e6$>%=F`3Q3h?5?;%Zpj;13wGib4 z({?TH{EM(>?{MD)8y}dy1Hku!8i|_@Otx}m8t(OOPI>BSp}iJ()yaX$Po@J?2?SFP zOn(7t)^w+Z8l(f$p<3T~)8J0`HZaP8DR(&b(4|^Zt2rN-{KP*!FilmdvrYZD7xXe3 z-~-c4Um!6%Ar;0Vr};!&3Q$Fh_@m&LF51{(of)#6)E!3E^&e{HP4~d$xlpQso?%c zVh*@S_)4N4Kvtp5fo6g+2Re<_IZzJ3n*+Hc{%j7oC~5t?X&!`m4m<#WE+BXgG=LX# z;NPI}9M}rLn*+b;La|44%z+N$ygASeQJw?YKZ)7HG{+nm1~#4pQvvWC$UREuz`@GZ z-*B&YuLV=m0Sf)q$sF*LX%0LA!IU|$7N|D|DyTu411D;I=a>d}x^2gsIZz9(NMBtD z`)ST|z)$?sIj~ggtuysw4iusRo&&YMLY@PLc$tw{<(>k8WEWiFlhQfhCnx8?2qM9cEDK8X%6I{lrjgRM;~wwoCF@31Jbxt0zRi1 zN#Qd`VS^qCHIA~~bHVm5aQ8+4Ujs^bp*s1zt#j2Mquy*dR1PgAMuvSdD^03N~n_QatG{2a%t27t*7n z=rK%9%mY=A{W*I<&-+=ec zhNs@gfq}2WPX*xJ$eB-%nM}OAD9`%)wD?`aw!_^3HlFpFlXceD5;yI9 z&nVZ6hI_qx449HG|GK|Ane~1$&3YGtDYO1!pvL!|25OLI{U=)AAEv>b?h9bth$46< zxFRp;LjQ{9JnQ|$Kb`fCZv46Ce1KX1B^uyaAN3V-7Z_p}BeBZuI3=|Uy7{DZ*89oH zSw9@&#&Vu`PP7>Ft^`S99?DL@#KBEDI=2`8CJud~=2>_1%{+sSHspKV`yjXyH{9H9 z0Qj!n0=lbbI3^m-``v>oaY;tE=J*o86ihb^|BxN68ro!ylpU>sVC-l!A<4eI0)Xe+ zEmV3y-!3XSkZ(T&VcyZU1Mqyi7QEQeI-IP&Jq&>7+nKsEx78fJJssuh+YJz9-_F(I zQp2{xT?RJx?Ro&bqqPt>`SuXy8ezECyPtw7>D$Hr>cqGGWb*CZ5KQsyUQ?4J?BvGO zx2I@*7nlZjx|6^tzFh&X$Rk?PpgH@tpZKS~eT7QhVd{r(pN9t6x3BRPau*ol3L~+~ zeFOr@E?DW4Qs4HIlfM07iskIva;mF^b?_#)%{1N7CIIkuv__PAI~reGTE~HcUFP@%1VIiAw?063_(T zY62=}1g)C@$eMlv1RLEYpt0Y-0l@S7S}HuC-#4P9^_spH!tD3MW~TUk^hEaivq0lD z{aOH?-w)BHWw=>;u1q|IaL5;7B4VtJKRsf#(uvW0Q-F{ag*QAQm(%k?)C2A zS*e45roTGzdq0`{ej)@@{C*)&v$8o2)F7?tmuh{Fm_56#){ z{lq`@`{%XZHKu;}{fB6P{r+WNA$NfxzF{O*xf!!lyWlOKl={7&ob>wv5I2^y-}CbQ zZ(UTsGmGngdHFWz7DN8ZodbDZzHbJQT)qv*FNU+(eE|%~<=YhOGYsG0%vGvaB^3a& zjuvXs3Rg4^8s_zsb>T5oVtlh59rj<_<@}IA_%inzX-rv%w6zeG5-oQc52&A zaq1pBWw3i|j>Wtm%GIgsAj(diuf<~w+YWa$*x0Gh2Eb0;MBL=mM=IAbhI_qxFPM@m z*$MvY#Hsyca_YArnBvsm12seISY_(eQ?$ONroo+VpM-JhB5+0isU?qU&Q9$o{;5-6 zs#3R_`r*{ap#gU4D}9CB1%`N?kyz!{Kp@!#*ZZW@sr}@nQ{N78V|mJA?!r29tB+C4 zmnV_ADT{eMO1;ILFVP#c-F2GN!afNi-ooAtkpnF3=YdBr(f8{<)IqOs_!8X^!{)Ll z?Pek=yv*7ozP4$vy*4xTv2=fd)>a>&@*Z^M4z?q=&47?D4x2@!8uNv7aF!5?rN~{4E!Ac&%kK8&cHI|8gIDQyGK-|y5Bke>SPA`$ut8i zA(%1)7XvkZ7`ardS*y8hF^W8Fc zJR4u~74mE}#Mg~RtK6*9Q@i6$pOnr3DA zydm|`dC*_;to=Dx<5Iw&`G#!U?qARx??{&c;2mimJu)yXBMoZ+e>c>k$f|v2no%@1 zzC-BkI@K5h>^dN^$G?O+J&&)avIBa26G~cp{MQg>kMDj)ipNLCu*Z)AjXi!20MFxp zR|RcV5FURC%GKjbAj%%!MT?IyY&+Zsz{Vc`ZvgD^)x=GE&H&{qG~DZ5>&(<49`3JB zJl;*1S6=&(w}rC$hET%q zOlS3_H9epAg1%H|KG)N?gg}0&Cy#5H`Ii>Fy?DHNR}v6a!Ef*brY(kF>jun%R0_uX&O8T6vMTpS;??LSFNQ z!uZXsB|S%ZdX7rbb5x3+qf+!7C3>FOpkp}Ak`Jfam2D0~*G<+DnNxI@K7^Vq5231n z^T%0q3`>rlgGE6uSgw%;|QHzEd@Y zN3rUx`TE)vK8%LCtF{844pXzHvTV+}NtX4q>OcH@@`>0QTrm;p3pz5nLRihMp(xx551u0T-Q%e*z$&1;Cd85*_~A^RlIVE4?4_guNn$?O_JWOL;#xonU7O*q1d(InZdT)-*PLVG9`F$th1NX@J08EdtnwreQhaj!2p5_U8Pkx(~kD)zxP6bc# zKBOh|d8N}_ouMCRn3;Mg$)1b1Fz4Kb>Ur%vS7zxei*#0=E3@(!;JmP%D>LgaHG0_n z8^0x5^(t_l)eCu6^P}H7*qpVm3+?gI^L3Bk=PTsyFvNh7Sml;NAlV%uE%2(8*R1^H zk^v}s>>C>(a@LL zh4ku73;Hc~vSa@9Ch+3Y?hep+61)n)n*@15@t^*h zXJ6@VKshfenajM{V4bY9VYu>5G5kB+L15z(?>GQF8%l_qW606*45J+~x{XQu@<@(9V8L$-M#&TWeu?_NzhA=Qa zax^*>6fbi0DM)hU=%)ZUa&$KUjvVcMp?T2|5>6cwjvW1uPKM2zQ{-qBSox)xs{nB1 z=%WBQabVFMUAa&!}bR*|E|pokhqk)v75HFC5VfEPLH z=QNR{A(b>WP~_-b2=YrYmjduyj9-drAxCN*E&3oBIa>55030pa|6+Y9#+?R7YC^(M zq5ZYFp_)@v=q#|ZGt>g`qC#77!Wr%Z13Sap06b@C#R+Hl0SxR6&Ly5RI7QRRrzvTq zqyjzpMAO3yJ6A!%Oa&U|=u$I{@~g1^}KH zwGx3B{Ra%}MZWc<9^gK6@CtLE*=Ym^ z2dB1>eA@lwGn-Xo^PeGh-ftsRWpI}(R_s*nlA)S7x7uw zv%qvdK;Mu&K$-9ri+)&kn<`OgTF=f){6p zsi5)M;d}tz*`Y-Z{6%vdpzBet2j~`v@&P(a|1)K<=7<}618jU`*aU!&47siz8G0($ z!G?Ri+v}>-{c?c6Iyo}<$#i5WhG5E(VJ=X!szrurHJ576#|J~4Z5rR{-U=Z(KvzJ; z$Wp%;9iaWhKRrI2uF^(;4$v>50X{yQ?JMNtgCU-4G+O2U41whF;cq@EJwEu!$>T%k ztCPK8(T`8XrW;mjby+vdso3Cm82YUUn`(j--}}cO(wD5W5!`q@Jo~np`e!+f%KMq- zLk-~rCD@fWpa$IAy@%Q_#%&fWHX8~34FWz5bStt@K>h|{A;^KeKptNIzX%}x4MOPF zK|1c~Xy2>FA`?Xp&kq7S8-Smq`ECD*yGK+!`vKl$4vKt$odOmRek| z+>d!u-%;tz^e&xAszFbdWW-u-PbbEKxJQ!D-9C_N&)>yFwxk%_s}-w4Lf@MX>i4FD z^1W$lJ;>jij$H|m{xBxcNB=OU_(o7lNP6iRR9%t%DqytJaRRa;syx3EPa6fa`Zi)V zWP}zss0qeT?aKWmeTF?3{%tRskN#ps&mcv61{PXTdKdEIGf-O4k-)R4$pz#kuV_`P zO|Q&z{{eA!naDRv+@tvd@)$$hZX~XB-+=_Xk2euZd!ddrs-(dm0yzirE2~}s+0z$)CKpVD}bJI73?h~^B27Wv~ItL{sNRvM^egv zn`M`goKpmCY6iixZYAz%Bpg|$Eq|l=0y5`%MLyNsp6QN*72le^S$p%V{LJpg;QdxL4`S#8H@LV;ub#Xm#=?Wzu@%XEkqyGl6S6jW}ZzP!YK z1v7%S0q)65T-bm$i+k!4>I!RdwMx`@l6y(=>U7DgQzfq!$y${#v&pV(R|v@!>5?l_ zC0CH-#kPI%L10`-x)nTx4^GFqJ>95F&vU1~hBd5hA z>o8jHK0gH!rGypIhcEY{EN%nCzh^|_S%JAP9-Qs;vb{=6z1@Jxe{=jY~ zjQ^XdZ%0xpR6Gfk7)~zy7fo4Ve8TfAAVU@)!N0E{om5Os7oQI>b{P`-i>9os$%lcc zyd+>1-$4uyAfdl#%E~AmgDjfWGfv+@C626KWJ(3*EMqyebQga_Lg-)qPq2vjPq2vm zC)oR_Hu3~X(VurvcSN414xm5pVC)6zfCR7zd?oTSnLG~|sTO&aOg_(Gje)#_;}-x! zOc!2z#HUinMkDf{VDF=KqR98|4Te9+xDxE#%)9hIAU6a1h${Xlfd9HN;}cnH{u7Kj zKQaFa#wsNL6O7U1e}XZZ{7*1Olm7|EX!1Y7vdOmOe}c^*RR0r<_MKP{5a?g7BKj{^ z5&4%Z*2nvoE5^oB=h44hF*c4m?_z`DBdXo35ml(`xJyy%h1ksSWRCybudV*MUt9fi zzrOhFZQJCZ`;`-#{Bu95n^F05KM$2Z_oMFRQu%Yg6&Z@kpZnc@kfQSEe&poLEPw7t z{lcjHx!;?HS^nIQ{<(om@#lU8P>KHBuWgs{p|*eipP9ucfiE^43H|e*qyPNp$Upy8 zgY4jYyYvS)^3Q){C{y@$yhI4>P(?7rGNhWH?h+H4aEln+e6s@q3pfmqpY(3 z@#mSDkSR}+2?+s_KoG?SF-8#tWiqRhfGAM`sWO8IHUt%=s}_>fd*}OppL6f?JelC<_xjBrcg{KYoZI_z&pp27zar$>o5aU@v;Tkx z?fk)<+rIdVm#CdT4jCmWwn)A+ESMJ=06Tvy5E{2EeRNnkLXnz64*CG5fj`d}`1AH{ zTnh9UMkMd(o&oGB1nS`(T{ZB>ki14boZEOc`~>q2{9Q;nm`>{DWKRZ21Ak@4z+agf z_+zD(RU&~-hAk#@71**j!=XX6v^9v9HU`lcdO9$OCTL&~?E-|ArhOANh}J6(G%$$v z0|HH3gJ=N6j24R?Fyj}4XiQEzWDrg81~)q3YIn>wsGCb+5G?%%;H~bMoq<10JY*1U z9OXj>(b&+^A%kcf|I#6YXo80fqHP1?7U_^dG(67Yq(cVLB>a#;wCl+qGKeO4$RL`e zKV%S1{6hxOB>q<9XH#U%lgN+YA%kd-GyIT2w7qe|ByA0%0gysGnoMn4h%|^M^$;?M z_B{PV2GIl$8AKC2WDrg8kU=yFubqi38*|XVTxSClu>trGW&6hg1c?Y4L=&NqL9|bq zh>$_F6=lGmLm=(xG0PC>#pv{DYY+|di5NsHGX~Mh)F2v5rwpc?)F2wmw(K}|0~$mV zY#w{mf~pXywTRW97eZhB)&$qxvfxT1!hCYiVhyUeQRm#&fQ@;MXjX?hF#=VjhBK` zFg$&(*3w3%z_}x`|irF*3yK56q$Q0hBhMB()uvnSnIm^$cGkOIhLuv;L5Z8&{~=-N@lT4 z`v8l1Sd*ZOskJmVlO#G$L*q{Hdk|427o#w+;8JU8EM-zOZ5Bh;X)SGM3ZsXpwX~lY z@EO)xnnbGB(h~5HZ6D<5MmEveShUSJRxG5l5V>N;N%TZ?q#^?iV}mX?IIw4_=~ zlT50$wCCVUYiUViEiI|m(m3fR_h!VjmL`c-Yia(H3>hv$i~EwY6Az3ff8U~ab};zn zU`vxWrqt4EN^KvI)0ZNSvFWs$Qai#@q*+;v&4i6V!#+7|DImMjI0rCIsilo6wX~X2 zV}DCy$U_ou2M15DU|I18{KYHu^y*k}RFsor7?~x!XPoS7;Ev(QvpcM%bysU?(sGBB zv*}y-(pp+2t)*2OYiTr*R~cAKyOC&ME$w-tTurF8v|ots3bfalp$Mo1(??}sEv+xn zz*^dPqQ+XxWzCDw2}2zG{oknv|3A(Wg>HNko? zw=|+ASQT?iBWi;6Vs2?fZNKi2!%6@~)D8k@Qk|U!m`2p9j1jdeHKN8ksVZYEyqzr} zafRXRI|aL(3n5cF#A#q}nD_!ps}%$+_*!Zj$eB43!n5r;)GGh}p2` zoa^o%2Ov9=BQOsGW@nSV<>P3yVPo5ZhxV5DA(3HY+e~!W*mi%;aG||r1M>9$$Jq8N z5B#CM<+lt!w6~P-Lwn1Sj4!me6g;%Il=O%8mf|1UTe6kJLVHWWLwm~{!w>B(A3#Ix zVr-j>p|G)SsfWu7mUB@BH>@CeSV}>A+@p+TWtGiqifTRj_#+&$vYu%-^E2a?<`5RWN^xD%cXc zMHM-Jiz=AEMHS57qKcfqMHN4Piz?XZ5wk^ond$VmsN109q-Bem5e0TCsS%38)0>Z(0TOH?4yCn^sAhZd$iTY}tA^+_a|6rZugbRuS?yt%CWR zRtaJzO4G?4%^*&K!^6M0IsA+3;otcn$H!w37@Lmk;UC8Zq3BB4VCL|Tam4+@zYpoj zWXj=R4|LeLfB08JY?2)QZ2}t)|5E1gFQtcnY(Odh@J}#3{Nvch!$0@YVS^CLAMS*M z91O)D5C7uk@Gq{1f9EhF|L~73F+Pe@4ky^>bBGQ0`R5=Ro%i`1NP>O7#jwO% zPgX3Rf_?szq;%Tn3%kG1x0u=Ie?@lg>alOveZDY|BF)TVNUrSjw_&=mj}P|w!U2jd zKP?8f%xE~-@o=_a6+MZ4UGWr3@xux$G_Z;;Sf1kz_xT54C0gw7^KT>O@AHKdqmS54kiaVXbtvNEGweQJBGr9<8B&_<4W|)_DYoB-iuLpNxRUbb-_S4iR>W;XbJ=m2t}(dqXgiTiObdii}wu=a@gK6EyVW&=7XjJxaiAt@|S zw1_Q+Z1DS#MDv{eJ~Wxpyv!2e_aX6n)qWom>>u{~kVG1M9}+b9J|t-HeMr#Y`;ef) z_aP~*;QP=utRwS%h*MtXU?j?Xwam@}%rA`<=1XIReraUdDt2ZT`K6J)s-l(!$uEr@ z9xBix^-H5**b`JX_QW~FO9-w4txvx+eot&q7VGx#>xFbw?8T8T^C~>D@4(p^^a_4r zzQ(e{c`UFwV>`0JFuA<0r<7#CT@Z@KV6ZRKqX30~~&0guQ6_#8{!97|Bu5 zhk4*9M!~ANTF*`bKfjL^o9|=A`h865V}J6`ho@j;nP+|<6U_fUCfI=-5{Gg4k%9L# zC$ol$xepGHpo+~ARIwgGNz3$)pjc?d*D`P(L1o$2{UfMv5kRpXK^5x}lyoxx2+Cq; zZ#;sM%Elg2kDz8SDE|oRVq$s(H5MR`po(z>Rjfx)(*N`b>S*}#2&&i|K^5x}6o>xe zy;&kWf|5k*5tN^6WOzO5fk#lq6SM0P3{RP2=9DR>r%ca)9Me;#n4U6yYAG_PSq!5? zhC^?LGs3XVfb5i(#LA4POfhrH6w^~Cwvm{gG7SO;zu8yz%?<_F`J4S{PUyHV5!9VTgCnTNi3Ued-x4)PP)Wc|Rd9F&RfQucR)bqlSL}JJGXf0t;;_P; zOaq<$D0&ba9zj*%2#VCODDip~IG((xz~K>86^@_^d0AK*mkQi$1$YEig(IkYEP+Q* zRXBqB3kf{h;t^C8j-Y-&NcxzB{6wj4ScLNkstQL?g;GJz1F2y{As|(WBd9+B<`GoP z96`m*5tOX_gCnSOxzG=epxz@I96{ZH z#L#5)2=M~dIaS#Vm*Q?0+>z1;Sp5K z96`nO2x=hNZbsuAn~v!bln9a{M^F~S`s5(8>Jd~-kDz2XiQ&T;HC$b zKZ4?L5etr>SQ4?j*d?6oYKZa(Ds7IS(s~5N{+0HRpak=epajG6OOK$~qSF2ml=xv4 zrAJVL`A1NKVY<;HD8c+AC<#T6pjd@w@bFF72geF9T`$u|S3BG{V2h_J^i?rt-`e5k zV*hvTaA$6TRQ66dAE6M6%;kt8b2(x;j-IlBlBkLzOQ zxGrXn>$r`Hnd7>@L59b5@!^@9D6rcV=O$ElaSQly*@EF>Dfh01izVGujfa6^ zNh^Z^EO}s+0jB7d3>WWLg(?>N^H;fJ06Q5T5@f~!%ua!`1^e?HLDc@d*f~!PbsHe( zikK6k0=OR*W@d=ERxy4NkC&E{_;1&447c z$8sFHJi_70nfws+Fk+wERwTvtxwM=q>o%e|zcB6aHL?}i?kJt2DAd}ELan`6Th-(x zJZYa0F&%`V0n;{l5v@nB#TD-nJdOtxRKnU&C9Ks-peT4Ks+F)#ib4!h$M2Hx)sl3Z z8&t@8t&qv*G8LmcrDC+CVxWvS?t$8yoXLkEhwe_D7i3GEe&` z%)a6dQTUwx;2EZH3;BmDd=}xo1jaS?DHjsmMZK;dT&G^B@#}$lKe=oyu5?51CuUzR zQG35gAuh3BBpOQX4e;bH&?v*r`Gs%b&a%7)F_;Cr==6*6ZUrhmu?y?mYd#!Toezaa z3ji4P4J`Ox0;1Ojs<#PF?Os6ZX5rfj6y-le^f|&%l#i~AI-7w*sre#|yDODtrgJ-x z+M#edNTE{mK|t~Y!9Jb9XaGkMI2OQ31TFz^Ish+y1R`yQQ@u+icFrly(w^p@O3iPA zb#M6Peg!ZZK)&XZs561UJ^=Cns^f61>>w*!6U`k7hAQ~w&Ihm!Ip0O3T{u6E@oIO!XF9G1sYe5#0`jr&_$3eav~=d0|LW?V*lZhQrunB*^PeEJGyHOY1uzsq{tE#6 z0I2>B<3|?}FPa;0S=5P&v6g0s;pCXaV4*XQTbh zha+v^WE0Do=AXLE%fTvbU=0Aj4Rql|8+Z{6(grpG@Y_HaPPBoFD@+^M1Au7*UOSV5 z4-0tzna+GT`N{CV6zQzK2hI&9RjX|3pv!z27~X)NQ~MFzR>*YLWv)Se)zV092fV1u z{2&A#1)lpCfM)=B&XMTW*8-{o09|G(Qsw}rVA$_G!1m{@1X0?|38t-_VcQH;n14fX z(q{T!Y1+)b0Q@%7MFeeT4j9;GkcJKfo!p~D+ov6vs*izZA+fVY^KEQ$D-h0l0JY6< z(nqOA^It$b47eZaL=$@TYoKNfVu{pfuDL4ed;mWLM0<732#6ZZ`+!{13u#;l0yLU? z1J2I|aUg(QqSG#m)dQ%$0M0rS>Lcc#PaB;N&TA-gCxBZ4bow3t3JN%Cim}-ICx+)< z2J_pjhi?FUs+BPHwf|twdspT;caThg@5;>mfzTTe@L*Ywyswk^=OA~T#K{4wuuOg! zlU(eZm`s;Crvj5Xl`6@AgEikb`UKd&@r4X@S=jLHGr){k-y!=p#$aWxjeRS0d;{S; z+8PZ-0ZgKhnY~pe{SgcnbU#?Yez1Tg*v}Z!O-g6-088??Ml^Jtns1464dzV#mF3dg z4Mh#i&mCY9mm=$}s|Q&z^wt%FEaA4+B^+}a-&$4=x{^?M3b!=z8sXQKL||O$=p8;VBaHlu*(qN$NCvlO=n|hGcxMWSk}aCX(frMJ0A^ zN5K4&2F#My$|)BZ?Sgv;EStC{aYtm|MNi!+vEs6`~hWlnPzE$`R9Og9T;Rl`7;1JpolvD1tt~uoBu@r+`HhRI(}%ab~@1f zPS>k;e)94HndA)x1kqK$tAH5)B9}u5HzTOrY5;cv;BQ`E=*+J~#u7C%oo2wHcxi?< zgz^E7+D9O_kp=T7toeQf5@IL^gI9M9XF@t{bT+}My#^k2$6})UC6;>uk>A6Bo#(}U zpKh=&{V&k`f5G!Jf#Ekqo#Gk*s{vFHNZbeweE_w4!D&I5@on+80EnZ3=gtK%o@}oI zm;pd%(suGSySZemnMnr&kePH7d}JoAnd{G_%>ctQsn-I6*9Irw47tGwwl>-y#XylA z-24I%cLa`K^8U0|{}blF3qcl$g&>il5-h0=SUQ{H;LHe*U}`#3rTO1|WWfn_mZS zQ_8YB+DQ6EZti{1xjORF^G#JY05FT9`2fRJ>B!G(1=;I>Bl{lbcr@qLQQ7y@5yrl! z+bx~;Jvn6w?R!>|;_rKQ1iS2e#sZLi&pZIK@3|a+?0ecF;cbE=`=06LW}Gnp-1oc< zR@wJBxCbNqo&f-4-!lq;zwfcKbF6GlGk=BDqB>#+Dx&F8O5UefpRgCSi+F!@mQc4O(L-*)H_;1 z2ztkBYfSI>5rFiLK6pStdIw(;klwKg5?;-Trgy|DG>u!EfA)?QV3ppn4uJHI4gk_S zz6Rj;4l7$~Wzjo&tu?)41OUHxbm2trI1CKZJ5C4S_l_={=pDC$L3+nC0L+dMqa?DB z+}=c8Z~mos6cX7xK7**=I|>=tJIdCX-myCX+dIyy($cxs{L2o{5-&DH_Krg#B)ww^ z0O=jq1MquC7ZLQ1SHK{r0*O02_YSWGIrG}#$br%(0Ew2+%)$dD+0oZ5K;3uU(GNm+ zvZJ2~fIE6G2S%?Ij%?&V?V*|a&ir#De>PZUP`e#~Uu|7DQEktIL8|Rb0DiS~;Y78S z-({+8cL289(2$X!jYvy!hN-hF&Hti9!6zN^T(j7Lr zpdCOf0ES$55{hjtkTw7?H1nWqUIO6Nos3e+!O5Sv9?vJ!?@9n4 z13+)%S>8%`CZ-h>z1Kmx^g=VIH3P6`c`bx7r#*yVHZv4G%WG4XP_NxYia)2l1$LR! zHUp4eyUpFE*ERx>UR$>ak^V!_0Ix>RR3}Se)V_ZMD?!$gY?=L0odyIT9Jk1M<(k3nSbfE zg+%t+O%U~aZ6O1DZI63RuN@A+_S)@wYUwnXf9bWBIK&XyYo|j9w~2@Oy0+ z5%k(Oz#zT0?7q&uR?hNlhJG>`<5^xk809Q)0s#LkuaMK8<$Yr$Z3sNeTLeLWPLrPB zfl}N;&%X|g((_*iAZK~S_nWi4hQ(+}(}pbID3z?oo> zHn0kS-v+vHq76I;25AHT0btsImtMlAOsADj2b_G_1EI6Lj7e3aO&!kiMuQjVtr}h{S+5v3g$gcW9b7aS($xYyiKkeoY1yS;| z1c2nH?i7=s`2b9QS^ zPyPBlWVh)c*i*j@h0>S!WJ`1@06TrW-CJA959VLS0!#eD5IKF^2O${?{s}j3!EMMLD1(O2rgAWN-70NC-8<&xn=G^Z!l8=W;VZ$;t!vdCy+=1l5F_0y($ zV9fo$lq4g+uOvA&S{HGmMec=&)EyKTgzz9dx&%5w$0(sKwz1CXYv<)sGL^XJx)4US z?NUbc4otESv&?@N!aQ6+aoSOp(V@9_5$5SY`J1`}0*}qsbjt0n*zAju z=0ST4{c?i0pN+3Gvk4B)kb2}QfL5b5Lwfc`IGx^P%dQ7y-#75CTEA_^;xFz_ZlrAY zh(;yff`EMvwhFkQC%J7UZCrEYoRHC3A#j zJxXjapJ0w2MrMbpb+RMi?17>xPK;{AU%Ys`aR-1@jJvN;_AVaSI0;n7RJ=VXjnjZp zx_Ad-tD0a{kl2pE5F&w zC5z$O@kp|&T2!Szf6`3Ku3&|*K7S(Yq{y9Riy5`3)5s2$6SAvXlrWGY8Uz-L8?`7L zRydVVLxEaUr9OWm9H4}P2L}GkYv9ek3y12&ar;yizO0Vn2V>T6JWwYVG*Bl#6sL`G z-t5<>s;(j$s1pyxT||5+QFUSfqE1|kH~}Lis!q%X91lHJC3pjb%u`j167UZ_Rdpuu z&{I_(5f7;ovjXFxr>a=>@z7IMf`^`}Vx`7I>cnhf@sK*Pgdb8T-kco=% zkUFvWht!EB{#NAAJXIxlNS$~t!w;zwFT>aVxP7V$fD|HY6R;E_pQ@622&oeJhVaOQD~qPhGKYN_{Jf{auSYDH zuSYD{DSQxzPhE+euSYC?EqvBzFZc;|dc^1vU&~aVWuLnGoY?ZnfcZ}LT!4J6prmj1 zGJt%nz_X7PWNri{y9N$bic5`3ajB{lOZdJ@v0$S)IPx_N!Jy!(?_fzPsjG|ep8@_1 zhbqOTMy0q^Rf67QVwR0Z-Hc6_ zsY)@&2BFBHW-*L`d{2;ZluckV`JUjNWMwi%rTE{(e3jx)iA|!SUG^b>R4InWuDmCh zR+VD5rL?b7ESM+n337a+N^$w4!@h;+ACAZ9f*d&bqe?N}7qHD#(&ERT@+D#dJ+ zWusWCR4HaiWsMQ@o}gg9N-?M1GL$$}il-oCsuY(RmEuxWDV74I-hgA3iFZMGR3>(l znR6kNy$lYOiR0$cu(&=NCd1cbZ18)-^Fc7hMVWZmUjW+dyabr)WO1WT7FTsL)>~ZF z$^Hips*`o^n}rEKQ77{j+P9=}Q9;z);zrFau4-=EvcIdETU^!LMp=r~r^QgN8IHQy ziEy^?G7|G%p=xPNktF&VCwmsSsg_oWmys&<-yFC({#NO#|RiEv*OC z(zy0;>*>}r1L`z^ps4PBQ-2ayj}%PCGQG2Ir2g+t&lf} zO9k$-0#r-u0kyPtOK2ehYH2T%@HQN(rS*VX+V2M$N=PyC-c$jHYH2;7mR2YgXbY>4}6X)Kz!ua@>t2B~Uk9{`l}6{ByV)~J?NmCW+JHL9h>p_Ud`wX~8F z2&r0HcYxVyI8;lE8@05!s-^8lwwq;mi>q3i2$CX8Jd0s{co#5`;l(TDA*qYD7Drk>Z0F&0GlB_KAGQ-tOp~hcz8%9{EDzg7^kF+D zjug$Y#W0}rVLLXC21Ghh4+Y6c#o*U`(4m;i>&XI zje9#~<6&ll3h(!l69c0vyi44VbAG47yI}1RqwxM8W@7`UkFn{9D!fZ!fuh;47_vcy zcPWGCtipRcqj{MnLWOtndlgf6;~Iq=CY_pn<}>pn<}>pn<}>pn<}>lvbed z?y!!G!aKWE<{>1?D!fw@lM3%Wjlz3RRd{Dgd+y9EQsJFLNl#zl{X#URp1ZRk`LwoR z&_Gm$cRncA6FUi2ct4ETo-Eew!Fd|7y*LkNMgYu?g0nN|J*n`12$1^8aGcB}xQApI zXXbE#+2i0);k{?V3@yS-$H|;UhKu1);k~C(c<-qS@8sy&hk2mFyI|F53{LiOcu?Uz zVievZs_-tgvp@N%@GjU`HXbUx3w8kW%m<+bQ-yc-J{c#V@LrCB&h~*rh4+Y2c#o*U zyR=MS;hlvRxt4*i1MBUqpjHOXHy5W7gL*sOT#Tr~yL2*8w7ppj?TrfWQrQ?9RN?(; z3S)TCHy8OjSY$1_l#{&&AQj#tP< z2Q~7%ho?Qo2Q#NN80dM}Q+x=M$WtBWyLdd4mGZ#V-$ zFYY^DPDo_3$GeC0+XK!_`t?J5mm`s33|f*Gb%srZw}G>Hz$cS6IqEF@rHcoTI}xPf zeUU<(lI+`fDX6T3Vp(q908CCvu-wGJ$nz{ow5;h@!Ba3aSPx0TUUB*>pW+bdWL#g$ zzKd1V$#TX-t*F?^EJ&cLCR7a%ei6UaG$tAxDrYQ)fiDXekjE#{H zC$IhlA^zmLOtcC0Cs)p!B-Ee81LhRWk=bkh(cQ5;Oj@AlO$)m{6?NpyX#vli7VZU9 z&Yc!8gY*7}us88kZDB4CpWlGy&H|7?mh-Ct^aJ44EXQ}Z2jO?ViQ@|MAI(+MGC(0Y zRu!4)%!bEI_jtJZcM#Ar0JuYWjgfr7{2%`sDrtQIF*>*U{w9(^2Pf_~o_- z@K*o~^id=KlKJmKIDQ9w)A@_>ea`$x=T!fQ01C;m-`tr_Irxur`oJ~6#T$*3J(qPS zE@v?>HN`v~$&q4SD#d&m(9fU-7+uQ7QyD$zvY3?mQoq!jQKr#T^Ur}%%DBar@z*Az zpKUU=j0?$yWxN>ynKIsNBnOxyWGUlngIJ;|<3cjacv~oEN*Ol-uw}f1k89N_#IT~+884o(u9E~@dKyMkZhtwlC!*@Sakn8qKM2C$kb=bzt zONAr5?;3Gtif8{sw8400=YqohKg@P40@&%X!pvEqG@J*=O!d-ZyFqmrB<-=?nqCGh zI*Z+!NR}SkZ45y?Lk)+g$i@BaBW0wE5vyvU z@dJ<#bR$^7J_XfL{}nc{$GeT#QqFkn*(g+61fNWNkH1EpiRjVp3GQhK>lnA5q9;l6 z=Nt>q|KJe7UFNdL<~#^|82y(j9(z9OEX28uyF~G3;3Mh3Sn)psZzR4*@lSvsNPMB< zyT5?QiJvU_oYMkaEUheXA4Ih#+f3yRuF;GoA1WYB;g zML1;{;d>=bQ%)m%pZh#=<25j0eP4|_XOj45NfX?6+d=D_x^m(9OX{p;AY6Ge7S*mH}-h5JPabzwfw};_v z1=Z`IEe{&l22ZK+r*d-voTFE>8d#&)HRkm)li#B{mxRS8EOlufB2_K z>vThHBlRLnJbh_-D=@`J3R6O-+&GZ z02EsK0u)X&oGCYdi3}bUVV6tJGyz_2{^v}GgjCWoCQ&OPGUp0Vf50llof~-^Jz>tn zz^`Wc9VcAci|0v@Q~pT0!i={A^2x1t)%5RCum75cvosCX^8mv%=ifFB=V}^~*2jv^ zse03-XN9I`1aL{u`64lA25=c!E|4gs1f1!snX-DU=43pr=WsemhSGYDq=Q6YHx|p_ zwLQVP2Yb!nb_yxDsu#ea!te~KLpXR3eu1D+2M(5!+fzn%gNz6xO^OU0d@C0JuFM|; z2XDv0du*Q3?kJ#yDj|)v{C{U06Bm%@b%F;8&vjozQo%n5-~|j@sqY57wAJ7qEMm`) z_mCC_#%D9#@iL{Y4nXcV4E1sbrVQ5w;2MGxC}xsBJxKoaAo&7Xmcs&;!vYooEz696 zWk$dvpk>JiEct*%K+AG?z;bxNBA{hCB49ZpU=h%=%n4ZL1S|qV`M_lR=z!_yfJs0b z&anZ@u>p&KLGjHCnC1mc0@`qn4_J;5SOi2k_+`eTLa||DQ?BPqz&XHt=O0G?Kg5l} zp0I|=Kz5{M-=WArZGr287O0@KKn0~`-ziy^HY}hi|F)n|+Jr;G7SNX4-2uzp0gHf^ z<=%kh-hf5GpsF7Vm>vq41hnBi9I!kbun5SUV>{Wc9%b^9E#8-c;_V2Ew<9RtSAydG zdr-UzO7SXK!%zh@h5Jv&ZcF-&F!H7%rr6&Kid{h|b_I82pl?eyXro0zQ|#{q8GT1M zq}TE@Xn~4PWvxzqYmj$yc%g7lkfXjl~;C}%w z3vR`yyoax?E}ocy?3C?eh%}~4%w4%CFOsF(l;c9M%q!ho=3`2qD)1Y4+Yl@Z6hr4n zS6K2C{+C1W8p3he1WciQA`A1At%Vy|2nm%vlyFk{j|jmhhTt;^r-@2h2T#D*mEw#rQFZ(L8=9SJJ-$qhQjiJI8nM^Owxd>Ml#{r!!XdO_mg-$yh7}jh( zZ7n?V*MZ{!0v`j|Agr4#!%L+1+X$sn0>=ROP#9`1HX(dP`YoXUO5nc$;y7W$0t^gZ z3!OI0+7ll6?QxT28v-W+7%Z$CEyHlq?*M&20^b8@CWHRG^h&JHc6raN+R{?ohzHaZ zvm&dD?k@YtoTcEBB}RAQn)8g6QXE$&J4y9WsfzcFR1cM!0bDw~;wZLGQoU5_4lC75 zrM|aPisK4)C#eA{wa*78t^q1_F>r}XaU4or63*AoD)lcbwX;fX|Dlmm9Hr4oYLH5u z2rem&K`QmQl~Nql)JbZvN~JzBQiE0MP~eh6#Zj-Fq-s^_7AsY&QvbD5isKS}C#hXj zYQ)DTu3c1WIdF+fajfYt4VT7Hm3qZW4OOXrpBO2{v5oB{wWmrQ11>3zJyq%fE2TIt z`gW2Uu2RuYjnr_J8V6ibs5n$iI!TRCsq3uN2$lNSN-2&#Lno=MO6~TUi7Ts8rvaC^ z6i2_lELFr8xRmC#jrD zoewUFE2mPgTPelSzdA|HRH*@*jMPk(ng?7;Lvi#kNYx&Nk9xT)Ft;4#<{kmZ3B>;m z;1mFP_kU66G62UpHxS(p=xP8-FRPQx5c8iO1E1&NgBEusW|A|2{!`F8qJIPWp`baS zUI(3;tIV{p86NrGUqxYQ48UvvJpiBp!Qib_uHQ}>K+hZCSqs4Hu)=>P`3aDpBT({n z)Oi&EQgO8j{aw=c1N}b)E&=epFw|Lw7{2M}p98&&K#y;r-wXh)*)nWT`T?NV6SxMz z-omibGK?YpWzZ)OsQxzUb~mfaHy%wYfZR{__jq^B#QA46Z}_SPPWq(2!Z6UMroAa3se=u*k{(0FDL%`~C#O z_W+REKN^Nfq%Q;ga01T)I8GQEEW=XLouBbG27$2vE)jrG@&k{BK;#M$NhYGbnl-%4wM_SIaQLugEH-Ms{53S5f&WKAj%6eGR%o`;`(21INOe<* z=%j-*h>_P=qFa)PrcJ;jZvi!iZxgOtnuyjIYPJj%eu^mSo9zv-)2E1$_d{f;h@=zI z9c&8s0wvoB=SwCHZ;|6K9-BFCGDxw*k-Fx1?C?gu6}6NU+&OU?zunf6kYYzL^-iXR zVVn<#7YbwMLSalV6y8KBe7EI8(0ZXz!XZU2pjpgZDEyi1{)NJl2r&Ob;W%Rcg~BPs z^g>}bfc!{;i}gAD700u^K;acb7(ACNhH<@OxHrgg|B9hto?J0(f(P%e7W;QsSr5E) zn0R#9Y=f9dc&a_XhS z=K=E4AuKLF#FQuX(jf~ZS;=zFd<+h2$7H`JY_q6HM2|FWWxv1KDy~XIbA~8(x3-Hy zdlGiH*qy}S)-VuKVsMMC=+M_tzlVV24T~hA&AWT@*{fj^DA~i|@Cj+pJR$9wC#1XM zBDohlAKHPpE?G6Vfa(LG=k~WA??Aww)gus!!jB0WofOMmzm1!Wu`K{%(SQo*FycvOw32w zy{uW@$r8-J%yb7i{mV>DW|@DPNihF1(|w4mOfNH)>18Gs2q}8G#A4<$lSJcRW_p0p z_?MZ)?^S!5NihF1lSCR^W)d{G%p_=VnMu&#GLxXeWu`l@v?vQMGks6gTxL3@2&H!} z9NrYijaj)VUaB|6Zv?q?02^-;I~#9`ms}Icw1I4|K5YQZt53KL{4M_K(t7oYIZ69h zpV&sz{?#Woi1hBf3X<5U!)V&S`ZO3ROZ!)!CJ^(lKAlWVuRhfS%D&2v?3MK8rhk6NNZ#!6ByDOVPU62S|hBHR7h(?@Q~KXWw;}l3Tchx7=B1= zM8Xegjg;cnT`HtCB6vt^MA9G98WI1H)`-O48fc9O9?}}AWB4Jh5nhi^S*;NOQi%Uz zqC;9EQV$`mk-Zp5NNYs!kk*LcA*~U?Ls}yeUVET5vKuSbYK;I05)slG5uuRQ$P^|b zq&4z0?(U^RS|bid-IUcD;R2RgA4#M2kyNb@mJY2itJVk0HhCOdHxCd6^R+$%o6q}a zJb)BATxYFm@m9$l;c&8aNlf7F^I1 ztq;LYj|@-)=ku^8O`c`7KH7;br`AVi0l@5OaH!Y;1JxJcuO^w5t9B9u=6I)(-@OEr zOJFMBU5!7Qe_3LDuvMOPtv4EVQT_>6ORVEBxqQDFFt zNec|0u{#Hb&jhQn2F?Y;1w_^Gy9+0f-FIY_)4(~qbzu1He@I4W!)NS4f#EZY;Rbgf z27{Z6f#I`SQg%h=d;{mgPKsRrwiqT}8aSU$c1$#6R}DX5AVt=$7DK-l!)F(gACpgD z_)IuJ2?Y-f{F!^fn|&M(?}0-J=YRNXDbsu4GXLW~quv8&b(iH>-Ts=xzXvY!vVRZ! zq!MuY_rOKYzX!e}{c!6W2F}a$9ym*x6iu7OkagYzXN8oZhv+@HH-w#;0%Ez`@kvU0kJ;je-xr<<~CnTvJ6${AoQdjT)oKE<@*YAr6?mg!|% z$q6W$Hj5!`ylgA+udt$Pio<)Xv`<<6<%V6NV?Dw^$vp0Zm4mOM_qmz?T zot%3?PF;#P#->xMlk=LTNVBpSn+Y3#hJA7vZ@}AK$oGJ$E0i+2LaD4=H)VfIVaP)g zx59|M1vMVFiHdS^3?s9I_YC%b;Ev(SZOFj6wA|t3Y?=*Us_}F(aQ+VvLk7;p3{arP zvlk|akb!fW@d+6?Kb2^p#>1QAAp_@65jAQ&Zz7hakKj;^hX&4B4Q@T%dS*bK6eQ}! zVJm}SDLOqsZ0Zk(YCJS>PHI?`c)beHICRqiaB}2@8c!i_5SI#^U3$vbqD;N#+gV2OCqIeJlVw%&G$ApZPJz=_Px% zsN^6na(D|yIQ>iZg87&11@kZ23+7+4mr(SQJ*&`+51DeT$FhCk&|*@`SWHS8i%C3G zNxdq|_G|-qVt5rg1_S4@E!n@$Ok+@U;B3LbIb#nDoD0^2xuqc`!K#>BzB(;fFXonp zlz0={7&zYm(4;#1BH%3;IA@&%2F@jlplDFq8j z+&CKE*FwCnDX&;aO2I;srj*8DfCWK*VAyCODFq8jOp^(1>uA8XbyN~glQXGmh54{~ zs&AA6Y9T2B3rQMF<3>I%8u9a&SY~3Og`@;5B!$?{5H_`KlY(s<^^%A_7pW=h5L)UB z69R49q+r`d*(5tfp>!67(pfC9v!u4h=dbR>w{TYF_IK?AE^hx?y!Ep10-OW$ zu+i>Zm0z^W02eIITW@l@kUceF+mShWVmrx)PEVQ$3J$|WtV z_5;vsq7<0&E0%GIegZp_-yqYdsQWaMVuBhK3Tl)Dr6Nr#l40kZwgPggcVfoc4j5B>N=60&h_EgFC zk`_}jui_&B+qXOMf2^QMct-69R07}FaVJR?E&J6}(Ii&U!ZE*@Dw@P9TKEu9siw)S zriJyJqfTmf1T>jxU33S~GNOmEMdIf#yyG@CAU{%iyGMz^Ba*W4U!8 z-b7*(oCXjRW1^u4hmHd^{{wiI0KhqB3pi@K;bTS{Cy@-hkMaY&6s#r`RSnl;yq} zP3jONN}AMaX;OayD&2gww4susn6{zS(uR%)Ds5;DTS=umNd|~DGC+I=KWR{RNrT#> zn`uyYN`smTR2t1%sq0lhrG1I7ZC}dzHTsxob|Qf({keY6vs z;Wakn3k+Zh5~3xxSkW&e#G4Cv-LFspVgMwz*=1q7VgMwznRPD)Kw_J@n3n+?gCJR) z3_rpwcBX0t%rO7%#KTzFqcs@w6qq}jZK~28FEu}!6|t~55z}d9w2WEJK&8Enk@j{C zP#LqvNPGJPD6N6GW0;IZyC!1}ZJD@ZrDfcORIWssx(DaR!{JD1(NtuT%vq|I@T zx{K+?gA+9k<@n9_O|YE+g=GIx!*L6|8(>36XqL&gk`xV8KQfK7G=)hM_T$i7NwN(?j~b? z2DhWv?#+h0e1^?8)0FGw#hgG9GxN1PMe8R83Y@SFS+lsUY7#J#^CI`@?K6eEHrFy>Iq zZ28FN!0^jFg8!yx;Cx1|;gS6(!5Kqs5Dt@ovHKds7st3@6BMfB-+?~{8%JJhLJ$1a zc*)3^I95cZHQ-56WdAh&$CQ9}kWU*^6-MLEnBj|DXCl2lh@{6*Kb<|?D*=iFStc=? zhyOst(W`IvSb%i^d!C#*9^gUy0CsA6&S{(gfSP|jk3`IkQ-R$E|DH#2D|8I7&xjq3 zC}52p;HU+l_3F82<0bfin|?fIX}k^?qwk5$UA*xoU=j+xS3&MQu;C+>428q^`D`D4 zkA{_($+v@%Wz8|KQ(0prORrhhi_@()J3>BZ>hgiOKJaH^X>gElf}wlXn$(HUe<#+?h@* zkh5^~i$5vv^q)}PuSA=$E{& zfVui3usv2lg#K-AErR%kk=kD^H*6b`XLmO@{}Oy<$L36a721P|KidNc9y$QyPG`hh z>O)wJuqE2fVZ=xDXzL`>X1r+1HZeM4&Qm0`(c-11Y!e6Sv4=DH3x;t^r!a02Mm;pR z&4l+e*;aI7TVcZE`w59FRVIm}iNtjZi;0)(6{a=Y#5D@1$@52nIV;k#ZQ^3}qGj8} zO7+4}P0O}h{Uj?WwQRf9Pb>s6SF_bG1f9v(GoyRh0}wf<*h9}L)~FAMUCqldJph@! zH52f+sIHvFsxUAaz9q?jlXDv_bpz8DAb1#*EcX&G;i&?^pOOBO(z)mYEK5JGnG-KZ zKj!c@`8zPBQbjV9nzOo;oYm?1T#1~|Ig>q}O)a+>;eHI5+_Vt;=7}rpp_6!BAzpIY z2OlQP5&in!P43S~nd;Z~ZrWo9tiV}8y_-lD{rcWx2s#cX@IdD->9VzXs_Nnb~NU#R-qug9O{N4ec-@QEMd=9{ChL6_* zr}hV+Z2%IlMslaXyW2nv6T@+Plw~{1{8#SupzA*oZt0TF8gW*GRx)m{J69D~r8&Mp|t4!M8wMD>! zy3rtivl)92?_xlhg(hk4!hke4fhUjSXpj1q_nr z?*N3-oW27U+6+f3v;{!o=SXg=N>iafm>{;9rDZA=8Z(F`vO@QQgjDDZ0Dgs9&I%*J z3Oxf1L4{f$KP*(}ZQux3sI5rbC0L=)f=3Fc&@=xze57znduZY8;vVc5O|@wg{cYPo z(d-I7DVp&Bf}(M1e?jfEp+s=VFcT)J@Bocd1vkGCeA@uHGww_A9LN8%Uwa3J>Ru`R+Y&d_rIdfNcp(0#HZb6aaex@a99p zYlTz&B+&a!%XrcJr?xA~Jq^~E;g|aqz}pn|dc~Yi3G4#kTL9I6gkxpft!zy+Hy#XY z;Fntn;C^zRv?b1uBe`qAa3(n)18^}pd4$>}%r_#rjbM=T)aX{hd8*feEF}MFqJH1} zSMP{{sF0XyD9R0l=ur5{C|1ajI46?Z9}HX~SCHbY;SjIpU43C3gL zmpdQ8A^_DRw!oQ!6+gD{!Vr4Hir<|O^qaI(%R?;ZPPFL*+}z9Xl7p*n0Px_-tAmf1 zgCi#uHX%Qn1i9vbs@9Z3gZYQ@D*~-P8!51)eXZ9x7dBw7fe+7C%m}d9{JRsm2F{J# zI_A6xAin^BEPbaNh7-(xa%KTxE|hFoFnE9LoNS0Z$f~kseC7{cx zs@PJbT9;E*vHg%fF}a-RDX=rS7lP?Mq|+*8NNANZ#H>Fy1e~Pd~3tlFzFU9H<`< z^ZJPu1m}bnQ&`U(Y1cRdxMG=!>sfe9v%O&*Y~C1U<~0Cx@BR zI4#5fEaQAV^83-^CK|k@!~#fG8T4Les!}Qv7M)bFj-aP}_Ai(Q;=4MI&CqWqW2p697L48D zmzxEkFM#TI(TWN=Q-9TI;0GgVL+HI{B)1%bvcy>fz@HClcScxsD9%I;K3T!%0N|mv zIvHqw6X-_}7`P1io=0yqEH@lw$HB2= zeMH}QaA#~izzl|u=wLVt_+iSi2|90c)&ZBj++#X8b^zx)fp|iU+Fr7N-Gq{HpXwbIyz&+aft#nHB4 zmovyOLqUG2L7r`c{8ECPvj(`NZj+?0no)dYLi^c9@jo~BBKX$B4@U@x@X!WF2;*d9 zFGo#!Y%p-jQS{2_+0HV;4SFf3aU)JQ_EA`lkA^6GPJi$WQ@Dlv!xfg}qrC+7kBoLv zFF7)*Q!gZD31;lXgieW>psRzUYfNESJxF)rwJenHO>3L830ab<^hcdaQ)pt%XdrI4 zNbRY^9H#|vVrnF}_jWPo0^nXd2>BeaYk=j?1aKRH+X1WxAp7UZCek_PKLMLiFM>ff zT$=!x4OjA7!|;IlpUVS*TMCE?YW_JHR=#uGHh3GCfD>QP@H)=@oz=+BNmm-J&${O# zPKVX_?6yb5$%4}@hKA-QtJFCbl%DW$y$RdnFYT5^#ta28i=1ismK4Xos7ajX)0{E4 zqY2)D{^gc~C+=h}f?xI;IQ%f>nh#U1`7pJfg66{%{^i4z1ST~3Fy*7>!_+qv2|i3+ z%&7FkRCY7G_)Q9Ws7a9dO)7G7rVOC@CKb_dQu}}s@xMupA?AORT1ia4Np15HE)o~T zGvh$fPg4>7G<619{7+L%K*awvC4uRuspas%rz!e72MoI&fBW1D2j8ZQikCH(a6hQj z5$k4j#JZ`D*oz={QynpQijG(jtclNqwXSEXELR48%--D0?KPb(kHXM6(ha?;Aj zT4K~$5i1{W5%ZOD@hXE8pTw7#v$p}HQf{K3Rmyz;6e{IH!&Q`Wld6=9o@(N<6ue3lnv-E;&$NIodp;bhjh0xo(Gpb~eW=KB zO4jqcE}!zJXvw`y230r&dzcaP4g5co6ACx_20q(C$>R~IzVHov!P+AO`Zt~m>^sC& zvH^uVHeI4>qik`c=&fgqq1N~YzC`n!eFOhVM)PvSyn!!%uR@VY-oO{^ANCD=i8N3f z6*N#A6*N#A6*N#A6*N#AeYG1R4b(7o_#^`-$X)l8ruoe=0Fw1N25>7`D(n_XS8*`E=hb z^*yO7U*=ownF>mFHXN#lmQp?RL?HDveB@*n!966y1ZB>Ilf4KI)k8}Y$VxC)XiIX&+V|ZypF%&8Z~&4m_wHT4K~gOH@5nYROj*70g!; z70g!;6-?Db*;1v)Ks|I<6n}OvI8+ZUx9XwgMm?0JSRSZ{3L2<~vbxFx_0YQ$Km+yA zOA%qs>(y^p@N6hLkHphWqC+FbUg8pdgxH7fs}{TLs{76A@xw! zNO?#-RPc~`=%cu>T^>>oJ&)ms)I%lwkb39=j4z}fDtJgeRMH<(4;BBAdZ@%7QV$h8 zq#int;fK^iUylQ~>Y)In5MyKtsfS8Egw#Wq(m$jgDtJgeRPc~`sNf;>Pzf)j9y*N` zYt=&m1c?Z#hl!>Yv_0V!v50y@X*-F(z6Rdbu54{}xR1d8%1|Tcc z03;Vd6(!`S0Z5KU6;$331CUHwg>L|o-ML~bj@xI0Q?MGV9x51AXH-3u27)SfWR+7r zl-;@liZp5fvN!z(W6~R&u22Jz>_Mcc%BIDj8N>(gDxh-%_0S460C^}WyMh(MY5-E$ zNfEUoi(!JLdgwW1$1Fs4RSy*gQe+inG4yLu551rKn3CYg#(mO@W8;Ic^kah zPvB5Jw8W@~mZ*BD%=1$y<*zMrEK`4N;j4$r{5y+fN;5$>K+UoQT};gcv6+;hY*j79x5v&UpYe&=RPJmi!;;q0fMi>Y?REJ+xfaL*E9uT-8I%RXudGrAV`~7@G+j ze};W>SPwLFyELu=O!d%mqaIqW>Y?m!;x~6>9rU+VgO7Hr)naDvegr_FI*) z{Z^TVDpz3pZ5+{EpotbAvpras66G30ZNIH08rXh&jA)=V`ZuD1?Y9`FoGPOd zwu3{Z(JCm7vT}5XTF(rqGX@Ox;;@Ofw zX|xV~X|xhxwjUfSjg}jw(Q;K99YVI7(KyGZ%T;Mq1WA$6-C~#;a%8;~ z4Y9eYT$M&;@rXgi8Mlmn7*L=zTCPf?EZ%ZoX_SLVd7w1Pk|@6m>`2&y5T(+nXOu=g zRT`BE+gBPDtRF{0e?5SiPnAa5qCCuXsx&H`n84IBlVHBms9@L=sbwa?e5FwdMU_Tb zg=W;qG-H*PoeGCaqvb|vwA?6-a?@XKltxbj50ysaP#TSIQEBu#2DJ(fl}39NjhczS zx?UAx(u=Y8s$`)bi7`^J9?UJ3Mg^;4ZmBdXSTE+5N~7N*xL#Nv)HuUje4{t1&hp9u zl}3BTMysomc61FJ8s-#=+-o2k`c92v+U~a5bq;CJRTDBVnjUl_R?~ak^I3 z!ERP{u-jB2F1w0hm)V%sZA3t2tUJAmt!d>|Y(UHxbYxzn#G^Qf*K|D^r0Y>h!IDj; ztnP+72x=1Ah#M)=2-UHQfidU5f-q}xW4(AZ%fNH)i8%AVLJV~Pif7ku05~5Y;`Q3_ z?`?up+Z(Jk_X;&P4PosDJl_IfB!T+@j0Uj76gVFrtFeD={)^XoEs*p&=s5S`vq>M5 zzCq@Hhj|dQ#2pN=_%7tY5wFANYKZg9fAIrOHCw|1gD*4xNb<}{(mv+i0PmX(`M$ur z_CC<-kOr{k-ax9K2QI0$Oivi*;>RWG{Y+{m+EhKkRB3_b5Ki#~DX>-z(%XoCujYPJ zV9fv+(XY0^`fqOv>`(xHftBFBNLJTY<{t&t21(C(08Sm8q>X)cxWFuNrcGKmS)h47t}5UxRNZ;H(+z$sO(@r(JFhGmIA7-Fco z?{>WMXUxBC?)w^itocXc!p(hwAukWSr6xO&2IgZVQtj8IWxC8To95o*M2&i&`7hR* z^qfbKnFcy7blTz6a?IESz{T}`mSWAL5|7u6|76Ytu-;UflxT}`|~ zT%t!B*#$NcYrTz-^y(gilY^67XhL0X{-s1M@k~PumFSx$q>pTfraydc@UP8(L5Xf{ z6L0?Et$hP&V2QpHF40=UG~7n7eWTT&wjfN2q70cE=g+1f8vwvUj}&AJVT``*`1hR0 zO>?XRz(}Jzn!?*2fE3vZZd7J6fctrF?KX> zKsc>*I^ZNr7b!--}GWfCPKgg*}m6sdx?SZ#cmh+@m zR{xzsm1UXkHO!{6hMLqg*;JXaqYhzs^XatF*#t+%j+(z9Y%_K=6J{ymKY4};;1yGU zQlf3jk=qX`k+I__0Mh(#1mHLS6-M?-n~1eu`YFVkqq7oD@){HBYV$88YKgZQVyHxa zF(JjRmudbb2Jd113rch^Lv9Ydr9|722C(KfgiF*iO*71}Y=@+meilvxoTRPW zNv2Sw@GNn>E%RXPIKhN;stwT;-kAnJ-~1O8-g-lRD)5%Vn~yXwABQ8={@7ueUN+38 z@QydBv8~A#o{k;6J4-aL>&(A9kxSg%a)i7CfyV$yJNp`dw6m%~p`z+z#C9jdN&SsMn*;S6^O2gE9FB{*^xS8-F|oKRy}m;2F;F(d4Z=MfY44=8=|h6mhg zXB#ZVjmQ~R6tz@27%x!34?n&rh9w|=@xArNNgg4ebF;Z|veHOJuho2xd>p>mS~;$} z4-AUhSGUK2g0G6$i8=L?;5Y|lro(NR3CCQUnEo+Q4u=l_*;^)XphVf>uSJEXhoq=t>$REAW} z6oH%y?*_b>KxGPIifF!9O|KuM*fMcVa3&^;2ueh=ImwM`)5bq-;U7u0TFcKqtZWpx zw$^l&>2g{b{hsGM=Y3!BpL?G3JkR-^=lsscd+&SCy_bTi)_cLaPD^_7^$>r|vzqdF zTf-#mG}6`EaS@)TD{r#vQDktzi>L0KV)StMjFG5-l3gL&GXy@Z!A3PiWovZ<3hQnE zA}IXl4h6M)TXi-zrE>Pf7-YMUoyb|jtRFW$q#~D4YzsNp&&UZW`u|h1JCNn~Kx`uD z`nkDy#$^qhH@8u?9oZ~`V(YXiB-VAp;`PuR+AU#rNxFr;Vmwl*pRc|51!^36D7wS6 zr&-ea-%vp|p?*l!yU&cUyIV&nxQ3BR{Yh=6)S%q`Qldvv_s|FhLyT1Fv#}u6vWF?4 z0ZqMT9!}H)6x~!Nb%2POI*VP12{U>J$a!jnwTB^2_=y z!M_qTHzLh&>%fN$lQJ?-;FkTU6xkXgpXe>3ArpBI2YC%X`P04q(DJ1V2s%rfVEK7F z=fo8vouAimM$l!++oFY6N=|*F3znLnW1tMpmGJuCn&t{gK0C7 zxVja$7Q#;(3pLe5;wBV#xp)l^Esv{H5t{$&Q7AcMLx#+N*8O%SJWFVW+RV6UY92O{ zW?(oMGG?@QraTl_sEM`cYxyCSSAdQz)I@poedPBTggLm)5Ik3iry?-rbqU*pbw#pO zFv8`7>m@Qjin&5>5aZrU{Q`#y9D?*YA$2Q2@q=s*79!tF@CP`*aSPp1(QoKKFfljk zRGvDCo8(TGy*8>xox-bblG=AR@>-JOoCH+Rf_mr$26~i3ot@o_ zLmeEk56{008@|7#{Lzo#*+#OC#$k}G4{HB&D90hcDurI3ze?rc=wE@d6vC$BN*pl} zGg}Sa$6hsQ&^;GCP-vb*yt)PAa8^>U_%!I~hZld_(0IYm+9wN8Op#?3KL~@QnvzuZgYPcm91gB}J$*aR*I6AGUP{#Xtobw$; zf5b+ApXukD)IDd>4dbz)z6|@3F~9sW2pIOJpWDbfL;Fs-^m8ipvwuIb@gDXv&LNB* z999m(u#r0*ZR~0?ZOeN}x;^PpA!>r@_Fi#VxW_nn#pgk1KD@Xyj2D2c{h0X06z^EY z3K6U#SPl>-%YrD^mbT$|b*Rk{7jr_lLm4M-mLaZ@LZS&WGl94-K}~_i|f1BA)7Aj-9+A8L`VO#RA| z8*{>Vg)&a=-Oec*mE{x?k(+ABrS+u`jylcpyJ6$?#Z6dW`k{;yq}=>AQ|E@%Dt%P(SZ86G@zk_>l;#i67_I#yi(ccT#rb7#E3owe^3 zznG%QD&~t|6+sR_m^%xiTwB_P_>Ge=`ixW1P|<0Exgx1v ze3~Gt9dp)0PutKrPqjlQTX>%u-JU{@ewpvmm3+Qa?a=v76HW-)EBJ(GnRA2Y?%0Ls zgvlseSEr0f1NmG3Az57zw-Zd|$vxlmaea#RD^Qpz4k$fc^;biO#J3xGpo zlx(+OdsPUze!vi<7tcT;-H7m`1!x(!ppjb;twWw$(a5c^#O$=hB(?WMxi!0(neqQ% zM^1;}{$%l+u%ZDc8DaSS5blA1?~a|1&V+y)-|CH<`mmu6x6C)nPD8$gJ@^pxG)QDxyZ0b}7MqB>}j={0_ literal 0 HcmV?d00001 diff --git a/crux-mir/lib/gimli/fixtures/self/debug_inlined b/crux-mir/lib/gimli/fixtures/self/debug_inlined new file mode 100644 index 0000000000000000000000000000000000000000..949d18c9326f4742118c0abc3812ec7903be9b89 GIT binary patch literal 25062 zcmYkF1$0)&)5hn$_rAeHfE0I!;!bJt5WK-%f(C2Q;_j}6pn>38v^2OBDA3|gaS0YQ zxP1@*dG7aa&f)ZT=Ipz7c6N4mc4zPH?NAX*;(~^EYmG|Veq+TE8;`uBo_N#J%{FXfEt9+rb zB}e)xM^>~X%{b+hl`Z)~o)u)t=kY2JuWm_)naXo(fPYgCsAWmj*~(ErTQY2xa;k}z z1npDKJ=v0l`<44owPf)@-JTtW|UHw%)JB=*^0+jP)v?WZWU0@T*8)zlqai>@}*TCSHqUj zACxyYv*lut^8V(wB(I`8Y=|vsYAJu-V9T)v%3~rNxqL?X!aPS-om1|%$dOYQm7D$T z$gq3LXO243^1gERi;kRms$A-eBP%~ES4!xW(@w(wIrewKUMZJExm^vf^!8Uy)XXb^ zsg*N_dgb?Y%G-B(CAh8fxZPfH+A05i+$&2vD=*{#ahfHzBz?Gc`e}bl?vl$Vw`6>G zm9MSn$n+>xY==7@8-DOeqE1?%LW~a5AJ>2MNIMD-ef7wr&RXEE$C7egloyn@WXf3O zfU1^Uo1pAjXUT;N%FDnH$b;5fQu?CGd)>0+%5&u+w=H?)@*PW3yij?fdzM@x9|ON6 zmxw`2Uuv2Cu^37pm5aZ!B<&aFqh4Fe(Ouv8+48IFx(RHVnn*41JdrKgk|@vfw`HJL zd2CW!W|6lB*b+@1@{=t&i)xufWo@ZRJ_X)LE>Yf=!Q{CWY`H>CTM6|P)8F0z-z9IU zjNwyU<;AMllB$IAdvJMjz52HNR!QY`8`!dioUV~ADJ!eoZUUDMQua2rr7StDy)8GZ zsr+q6TXNM&clnYJJN}~($6brJ?eg2<9j8S zr97yTSF$Ej9v0-4U&*~gywW0>%4gJr8>djN*1#(dzf-Q-$SVf}m1{QfN=nMdw(!cf zG%BCi$}0)eDsOM&l{e{?f7s`hKR7fh9QMlJ9aUcYZ?B{cQ*KDdE;G__?D>Xc?=T#@ zr{UQ5gME&DbFI&@2N;gs%y8yphBGHKocYyVpEIvEocTM$ndcbJoXBwINW+=`F`W6l z;mn5(XYOw}a|^?nCm7CL-*DzzVAq*f7|xv7aOQBsnV(nmIrARFna3N>oX~LQONKM& zF`PNK;mj`@`keW8W1ll$g=4$U{I;Dfo9N8JhGPdB&Kzbqb05Q*D;dt*+i>Q=hBHqv zoVl*y%nc1^u5LJUnBmNs3}?=0ICECRnOhjn9BMf8WW$*Q3};SdICFNxnY+@N3pX*G zxrgD*(+p>>YB=*Q!zg6V8ctQiaH=$hQ&l&ds-EFgtqrHjZ8%jv!>KYGPF2=$s@L^>Gh|#t-wgS# zv2TVfYB*I%!>QUDPIZ5;Z-$IL}wTTZv9ZEtxh*V~hpGEXg)fc}NLM zUJp^OR??D^gOxoMEZN8yX+O3exwzdZg{?;tZa4ZMc9Mx&emb@uy}9++iLFO(Zas?B zvZN8W9?^eW5_Mn8kNp>0n}^DuPGRJ7JJA0U>g0A{%2i9=k~dwmq~aSbzXRKWP=@-) zu^pH~ZkN!OHH?uOB*yg17-<{A`+*GaU!=9=3&Z=s4A>3?YWz?mqb->j-mgHY|CU@E zA$=N#^yd(k-)DRhHN=+O48=43Zp$Tx;;o805t5~%$QUWw)xDAd;s(cwE zbRc7xw{XZ@jA7Pes=v>;{lXqeCYd9eUj8&>=?0jW#;&Z+2XZ&p4n+z+KW?%r?_G+a$$WoR+gq zO0Oj$U3K)W^|RyvIZr%GPLgvZwj>$%Zo9yF$%T{PaDyBTUdcUOk))WND6bP>$zIBz zfwz)(C$nS}c_sFP^T|%2CHZ@(KCcT{Qe>#|*1{_P?_MdF>UpOdvm5X-k3nEWg^8x#X5VIs`D)=Ra_x$ch4-l3cn&R_Ppa*lr-dGSy=H{tJaBQcaW!V+AVhZWneVW#D*IQotw{&B4Tt|RTp zqxnG$}OKdGK*aAxg(ufrr`@m4v}-e za-{NGE&qKS&M3%h-Z=8h2bD*^b>uJdg7?_5ep30`4~~o?5A$HKi_X|nE$lQo!%uO% zQrlMfbgx&IkzZgHaJYNjpV%w8{j|)fBsiQQAFk|`dR!g$)bYv~cPEP&ERHk(>6Tap zIP(wa=as)QYCYHcd!<+=<=~-SDPBN1YL8bIwO1as-z#^xB1}X)_Kqt;)1zLQ(@D!* z<0RvM!25gN3d0C_ zj7D`e8WKx|oo$A~Rg6jo8I@EtewgHe9zf0aY=7sG)tp0***Ns%92%9#k`2G;yig>I zCBO7iE>+Nynw-C$Vb0phIqOCpY})5*nSu>5Pc2qH+tiW|%ak9sv?P4BvbU=x5j&I* zVqP1`d9Cpn90cr9`8)8iy~?X5SdyFbT{Pyt;XM3JIMtHTJp3(#?r}KB)%nemLr3(t z_vWBH=ev|R2yT8{n}IMR~pCzUfrSIl^{ivUO55d7kMw#`FTe9|nmf7;JC1)Ng zhn_-I`h@jdw`9yS<#P|Qp1oD>^c=SOtUMRInmh#u96>y->lA0n-{cDKEJ^6mv7P+A zB}2%qEn6nqDqoHR=nszanH08Uh^O2V?w*%xLz`T-~Y!sns3 z)M4~FWP~l1f7UXk7TS`cj&h1ch)T&T5TT}OsPc|~*ix&R@`FEdNZmp?=T2Lmwo<;c z2a#^5a{hg`EN!dYc)u-4+AF`lfz!Y+mG8Z=#nVqY>pL9EkXyaS;d_6TFZcjE4^Ynb3C|_B{){>Ys(j%Wlpmyg z<*O}O1}lep9PtcMUSv74pM2GJWcW~(r}knk9Ht!UhrRl6<^SS2a*LcMfg?*ssJvl9 zM?yy`&rF2!qm)l2MtO1)f0Q4s^6E)ZevIr`0VCaSzh8t6YsIXo@ac=Eb*sB^N)AErm0QvDVH|zLCq3N93-B9eF-S-v{zJ$7GBlzV`CH&rh3yI1-&Q!cy_yEXD)a8PrV=h@_y zZY`D1Vx@`YTHF0EoUL?FdBxlKZCB-Y;BDQMd)@I$l3$cF+{LLkx$r%&?C!4e81UF0 z$|LW4C9Icn=@_pR=&igDe1m0veSqXmPnCCj=#?vdl(RkZ%Bo+L_kt(%Q||NFDy`8aRK5eekDQyUsVCA{On;>Cfafx#`?$oe#MY!lSDmJ>C$%IU zw+K~|S<;pqjpcSBm)lq1Y9qA#4_Pf~JV1E`co6wSHcN`KOy5!;A&T%J4B zwZywXf4dlbJW}~8xXeQ3;Chy{B8S(vWE;!;f#rNWm-FY~hfB45$wrp6S)#lYJcfL{ zu_b3&rfD-v@+{Xfv%p_hDW3#~uTU=2+>)8(wk=RU%XDvL$s&deYrs3#Y55o6%4?Kc zw6-LSJUSFxD3-a3kmB%eEi(pTL@|aDe}Eg31IA)&NG^?#q98+xI@nGX=XUBawo`4n zoqB*R)HQ||nP%f4>Tj)oHX^|)$CQ`MN2v6Vaw>!r>lspvUT8@(h7=tTUL;{iaS!1| znNwPR2f~ZT3@^5?v1IUhmG|F(zP_yd*EUPWGQ7yL-I6BPR6cm8C8x+c5NaG{c<~7Q zk{oypzh!u_0vt#_hOi@;p~j^X*up;6I$wg*kPjmS@#CS__8T~sj?*%M4=lO-Qn?Vq zkrm8`mwp8Qd9U(12uJ#TRvr#sLSBK;q#MH#=d~q2e$_Hf5Srv2&rOd_RG5DugCW{gkT(+Ojvk@~;R}eq@-k2%*X2^eX=Z?#?jfL>pT&R#kZ*LX*ha z%8}q6KPz`fm=ey=Wa465hSgR1vCX!;YouId3qq47${n}bQlzQ!aPU}iBtn_h3}sp( ze5ur0%N*K`Blb|`bbD>7*haZMLYc1&Wu_sN>BLZG=1p6gch)i+5y}i=_!5gyCKW@L zya->mlRG}Lr4hrIWe8>7l5ZiDnZxiUBf^)G3|V%7qsVU&%G6@`QVijXFl6cR63JNd z281%L8NNJ5C=ogz#k^!bTABq-1DWimpUvbT)$#I0h@MSnc znG_6P_8^owOa6#ZW+uazQV3r{7{2sL?T9dB*^E$TBl#IZnbr(n@*#Xl!SJP12GmJj zfly{9!(>;}E{QXZZ32+?U4&{Sm&T57$s&7Pu`#nWPA1f+$}DZb&`^9?Gz$ z7($$(l!qa#dBQMdB)BNUmoMP&7~o zQ@#vgkUztnz2JHbgML65RFd*H;JFNgk|7NGMENA}0m_RaB&tezZE#P9M@PXuC~t}I zXa>Wh{@^j>(_k5>cB{t-$-7Bx3E|!{$he_4pr|da^3!2(DXC@w<@LC^c7Sp4LRQjl z;J2G=jIv(kv}7{VN~OQY$~9b9maX7urk*zD!|KDRX?1=}{vE1i(iO4f{Xpfkm9UCU zQT`1)Vx01Hq?M9SQ@$L8gWjpiPpaS`nf$1_C57gwJg}xE6Xq&U1s`F$X<{uz5%W|Y z^D}PAvHXV+OO}$uCn9oSG%{o|(qiP3IQSUD$mH%!OEw+WdX~?^0R;Ib4m(yd8d;57 zZ+FSpc3IN;ww52U$C4Rj?_ozKVBj(|@ z9XWiVBYQ5Wyx<~7UXm{$#kcB_%IhG-7jsX!)-fEGKUJQ39KYr4edm-Tzdup=fQwjV zo+ zxAR|OwSBAd3^=sTz?CBDdq+NVrI_{w8}Bb#Ch9A6Bfk=?6g;$kht(t-52xoQ#BAo( z8L3!euRJGr3HHi*-Xh#p!z&$btPf<*M-oyo_ndG&qFL zz!hY|aj!Jtjl$KOxGLT>GFo38U*>A2pD%Nj%*b3NN$ks9Ed#rmt29Y`nX4sWH*=L1 z+e|leRXV_zxw;5;Ggr&8{d6-|Q1M8q zcJpPfnqWKTX09Ga_%cFWjf~JLIJ}z?Drsbd&cpfLj8HctBUHu62tEACml0}R&X*C2 zhp@!W2tB|Fi<=ReSqb6OQgxDDIAL)!LT`urGD80U_%cGtjEqp5)4nWFdm{_<(-mLl zr@WE*`Q?r;+tb6y_VhHeJ$+()*`6>X+tbC!_Ea{qJ;6q{=f~&1Y)?6aQf{^0D$>YoR^s{`~p1zJR+cUuH%l53nAaXN2(Mfz6 zp0-BD=7^E88E#~0DjM0CXbzB!Baz%F%9QVXfzvWY(X#^f>63L;x{rK&OLEL=OR}C* z`Q)|OUtdt(^2U;j++&u&>T}&z2RM(s`s*ahhmluL=CAyr4E8uhl#^AoC3`XD?}M;k zE3O=fWJ?duoMGo2X~@}c{RKyUC09b~uOVmA%$OCYa8?`~&nwqB3mr`0mCxk5mAz6o zvCc4Ksvy21S4T4E3e#d?xUw`prIyKv6j_c`%5|FJIFH-{>9T8SRX)8n>L&*vzCFsB zD%BCM4B+L>3p9Az5X0bMhQZ4j2H#>ByeW)akVYN`BcGv>qYT5&Gz@#hFzl@pJ_8mv z3^>3rV1Qx3*y}z6RyPbd%`jjl!+`Y+1Ew+zSllq+V_ZgX4Vcj|U|z$3Ee!)+!zBgR zfOil34EU7>v@RvIq$iVS=L2#5hspm$5BYM&;4dCoPnS+jVUo8nbt?KcZVCN=O25xE zUrK*A&Y;{hehAK*+|>OdBmMpt%oxUWbRgBOz1&^2xfuqSS^V)KhJdBMB z*lRSZsL`mgMx*?UMwK@jHOpw!5~ESC!EU1(8;x3LG^&fysJ%v`jvI}7lFQeq;zpxZ z8jYI9hQv!}DE~CRuNQI~3XeCGEN>`Q&roKIp}=z2AKoCO*~{+RPN6s4Fwr2x6xWRA z&oY|4(rD&UR+6wH&g47Ld0HmLrA3~@m%_RFe(oHj;Uu|7Ye#oSkn@t)b1vh|yi%mf zi{sool;__6;!L~mDwRKLjq~|+%EJ&8W#Dz5vs56>2q@5*3e155NvS|zLxGP_;Cm{t z84A3n0;zFO?<&ySP#|#&p8^GNFzza_7Ybyd0;{s)eob2S*tWRu^OAhFpe?1dsysio zMeWHgkfSYGSLIv570HPraY#sR18z)?2Dc>#EVktraz}7~^2p`b7S+=_cdf$qu)cC2 zqJ>So-_Z}zLI&RNh@sKG>ueZpsA05LhSBaAMw@3CO$?(oH;fi%^jb>8Xe$k)RWXdP z&@e(A*fl~8!w6A^5#EDcBZL@6=xP{YxnYFQeSJmhcu*hgY_@QNHkm2AQABk}W;;q1W?c-Q@d--g1^y;057F*v(Dy*B?IS z5B*~C!wdXjp#m@hf7l2=Y|S55u8Eh%_`|;2k(+F-mGngOJ9CQ{604e66|m1e%{}bh zIGjS0hq6FI|18*}(bQAG1DNvLSqB#8O`0-wam$392&uQcOv6nCcOd_c8#eoR!=@n8 zYJN<&9cqcSiRrVTR+bDTZ=wR<&(3a1i{5Ih2jCOKlDxGLp|aH{rCe=Q?r zaAjw@au~SbB;}J}i}Ebk80O^ZSx9YMN#RLX6gWOl!v;4%-fEHlHnIs6W;PWz-Em`*!qkRv1Mv=71Q4r@J;gB_Vc_Jd<5 zVws|F>=JbBFI&B`gN~gEj$MR~JqEmzeEc@fw0X|F1LwoNc~+b_#w+P~4qOfAz0-L1 z`vzycvv|f^73a4<^89u*&S^LC%=ROl!Q0;Oz~qM4wJ|)+&+xAFhDTL>sy^@SnH8Dh z0orRRurhXCs2q;$@Jwch<6(|n%xrHStc-n_z3s+I`~oq7&*8GW4V9$f@|p}2bxAI- zt5HcUy2No*l7P$W{w_!dGd3IUR`>~8xfL!nDtr&E+zO`~6>c#qTy0c1&8_gNQQ;t? z!b?Vl-Hi%o7!^)5DqQ2*Z5f(;f=v#>+7!YjFF})kVUtT^K_A5?H^bUgo=x7zN)j*b zfVJRvHF*AxmaO?xIY}7O(3_N3gIjJ>z74L*@})Xk@^Q1u%R<9>>y@LPdu7xho#mVt zxCJs;IT$>jJmjNS3Jz8IA@FqaeJbXehSRV}o`%Iz7&t4h?}hTfe7+aTlVH+zUnuWW z5xqZ7HF;gh_Y!&S>b@7p!)y3n9Di2R_u_cpFTR(shkmvG|0V36seLa`f6d@~VY*u; z-wV^zoB3X7ZaBpE67$&&zL%H_I6hFVbf+?IH;z{h z&5ZPSfL1ugE!-aG5AI9G}(6~?s3l)gdriK8&#DLCpCI$j_g^>|1_iRRK>IbQfk#XQ-2!Pw<30upoCR+ucXtbHfv_KiGd94_FX-5<&{OFkXc$7R78!dO1# zDzZYE{!w`zoFSYpqEGytKlDDB>^q2PflAzW3x=Q)_aLIweBVLD7~>$K6fQpf?^_g? z7ocbV)u+^@&@XTy-iD;GccdlfPildxQMmeYM)~(YEP27}E;HgRIrl~7WpFKIj7L@N z$O;oxL4`?AXn~X92=WS4{(_YcK;?5;`7KoLe@)9zIF6UM9w<*eWyzmUl&wctPq_3-h68DYU9V*pR}^`H<7eq`&&){;I?hoHLx# zM`uJ{;D6RT3*kNrE4#u_2)|NzmO{G~TI60nt_W@2%f4~%j0?W_4#BrSd)#VE4${d> zt;LS=ygv6lJo9IIW?vSL-w5}PR$bKsZTI0i@=fJM2jJ(olsg_nOcd3TyUM;p*I5mHFEK~+hu#zzlC?P` z4^q5L@VeJDSbN~VCqTgfGHW&9$!Nm(3%6;Ix8*sx0u7k-6pZ$iMjLMQ>~N!J)4(8ZZ;l41q;c}XIBw5AFnaJi7{Tqqk}$$= zY;0#3p*^_@8h)6KZH>leU<20}y&MYVn^NJ4Q2q`%ipu+?{2i~Za;OS=d_Q|U;;too z*t0KVkVd4!HhL|Ty_2IVa?>T%GM89k{JbYEDfL+0U?n($yyp_$pm?Y9GMBLg^D@BM ztGHCa=`rL#d?SPx14^Vrx2Dp{w}2Nh;r@{ivZMCmj_3`o@cDj(f1E@9`_~ei^F~Re zA-?@trJq?+{*xB&ToM=m3Tw?3Sit$T)skO1JH^NBl!LQVtpj*Dgf&;hjMRWNH{uWd z12L`5;tw<9hn4ul!%(RtFGaLRB}-XJ0Zeh>oLHKo!mh0FG$xs}G=4c4zo$FgVEmyp zzMvHe*@}!WaJk~MwkS2SA>Fy`j;@P{w4%1|Z3nn(m>>@5L>BdBmiC>%_M2Sdq~ zRI&<`>_sI9LCNk^@-i#2A0i7mGEQrbMdmTjd*w+urr5~zddkl<{@xvk7Pw08LKCjA z33=QC*>>Xc2@Ck6TPLzx2eE*sG74;Cf$0?5#dcxArSR%*WRv-;?zkH9nuGTx3wysq zf!f#ATs!t4uxH`laKEA+SCf9YU(t)J$D_6AMH-ny57T z?oTcd&2L&VoBh}feOH=&*Y*_xGdK2lgR972RheQ~P?jZ7>ovuw|Br)fD$<2rIJ#E{ z*>aHS!XB8)CUT?;2H;N|fVHqJUMi*a6vDFjj*S|HAg~&P!0VXSn$j}q*(i@ZM_k7q zDMFzY1;=nN0GCDq_wsG9`>S+t?{swUW)wOdAk0qT0>l3wqXySkYr=0Yeb%;rhmRI~ z`|}isF<X$~nae>8WB}Xvker72jR%-WiN-b8y+Rq5HY44G2`+&t-k#53OKhPFritQ5E8;1+2c9+&*)1 zYl_J&183>RZh_+{kcTtg1{A2q0<+u#(`sR!FQe8@8iI+UymF2Bd_+QwySH37mxG%z zEIi8sp4qL@U%~pTN3QS;gu^J51ne&>6jL;X=_s^9AUsXs<2JZ9UsbCr!~zMXqQH3; z$lKPI`7CgG7zRF1cm{7oAkaZ%I8Gea-)Uv;+>e3S}I@0ExUO! z%GLPLgki=*ZR4R8hTOa4kZ`SN`0tgz#S<0AC(*eC6`gja41=UR>}^=vQx1u;9&A;D7l4-rQM7R zt7I>f{H?J*^gMVC6^?_F6R6~$P%<@@Yzrl)P|0MM99c>wuj77K&>O91A?|L)kPG1I zdoEsRoXm#!CxPSLWJ6Y=A*5eyl~g!>0VsTj3Lk~S8>sMhC|r#S zXGPcVq+(s5wd(_uG=bz4C2V=nY0C?wi<2 zQHlHZ^+8nPzI~kpmAG$TmqI1(+t=Ujpnm(a!;kxJ5g#;e5ic@s4p(F0fF{dq@vEn5 zj9G~f+tgRyi%QNtR8E20n1^^3`6((n7OV11xOMsLm2x)Rv`qR!ISp=CF6A-%YdCOb zre$Ms&A0~3AHX%^eB1|BeuE_42mNgc+>lK9NjWWUMb74R;@l1n(s_BW8E!u&@Y4h5 zFgC=~-cWuBh1tiVb*b69XW+c#lq+l*L7o7fO5Tb}vd{v#SJ_gOTyr(vFemQ=FCxdn zN_T0c=eWyS`jKj00Tw$?i&e(`)jYIVdfZ>V;#v>)SMxm8^0i<|Pn>cHc8<--bzs#j zv}%3qB8QQKVc~7GaCOXY0k8D8*>Hcg2(6tH_g5#8)8YPV3zo@ODCb+-4fIJ>O zn2tRdf%~gn$unHH%)J~71Un@Z9npveZwVuNY2?i?@@pD-G7S5ghTQ{W=AZ!&z<@zC z+Mh7mc^YjN4Ap?fnFQmEra?ksga$OiGZmv1PzI%d_Nj8nhp6GJ@_XZQUSfzj}6Izo_fcIjt90Oct04UPNxn!k%u1 zZe3Dyo)5i&Xe~^i`}P*TA>COy<_=cpZpu?w*t4uKzSj_;!d0#??Oj{Wu;fk(tx>me zyxc`gwnTxKEO0Z2KG&10560SVEs!fGgo_V8VpOJl&8K5juI8xBiBb6vM`db^%1IoRpD?`y@HNaYm|jkC zRQ`lfIf$e36BYJ8#3wvA^B(e-_+F{WyT`|Z@L49_J)YeJ-|FT^h|jmey%*k5f0~!} zNe%nFf6VUzWrHC8sLSX6#A?J&Arl!SEh3pVu9Vbxi^v57qj5z z-YasjTVRDzAfZv9e=#a~e}nI%8#i5HaX92gI%K=zeAZ*UG_jSRu&R9wpU5K5DWL@t zthCafzrhFA>Gw_sV0wD4G|Sh}!THQ7rfkFBe_Jg`#%6P?QH3 zijwd`Q3qTo8p#Vq#Yjx3x$l z{&`NF5E(ih-@<|ESNHeG(OwYNTIZ2o$-n)N4n%I<6mf1Ly!>3%BV}8vuy{3(%p~6j zZ)&CT6^}&X@XxcT4!+CrScT2--?85~ZC#H{^LYMO?NJEgz6obT7=n2E+p{{xibUX_ zHD{7XMi$VLH)eaJcVgx71wFD8|Kg<^<&j)PRd{ZUM}mhaR~_$>fkTzUCVHesKIJV_ XJ(9J!^3!Q36Q=y*Z}@V2Z{`03;~8Sz literal 0 HcmV?d00001 diff --git a/crux-mir/lib/gimli/fixtures/self/debug_line b/crux-mir/lib/gimli/fixtures/self/debug_line new file mode 100644 index 0000000000000000000000000000000000000000..896a0736427e8fd95e3fc731f4924cd66ab4c757 GIT binary patch literal 109251 zcmeFa33!x65Lvb!nGjE<#M~f8yb(l-EJ3t;QyR6tBNbDa!N`A^QRTh%&Dp@%9&O= zqpXylTz!DqGpfr1rIpoHft)~jd0El)$)z)^a%PrJn;s~ems2#ka$0##W%0DqszBvD zg()z-dP;Ut`HXRg6&Dp9efV+5O`URFanZ4dACWU`|*>?52Fr|p`7WHtET^K4F#rG7Ek_p zW|)-IpN)8l9V*b}g7(R-tYS4=F}V_IR{fX&K4w8(RhC!%5?WXn6E4?`;u&0qxWdfR zqKisrPUGJmmupJ-+~TSHY`I+1izip8$BKYNiVBlw&Qn#Z%1Voh`Pu7onP-%kQ7#c+ z5l~OjE?3bEi3!l6vhta-7C)EIoLyWQQ1~i~tDyGPj~SCIFS0v=a#G!5^s;Jm&)S3muJniJGqRp#-s1W zW%3cPt4bwvh8l+9X-YYWo}(yWg)uo`_kr8A8H^{4zUqK_m%!s3BNCJ3qU-Vq8lD`x zdxP*|DaQL;{oX?tQ*tadFYcvHi$t2+Ln0@2vByKTRDid2^m^FaRic1ica_j1i|f|U zqZeI4jc|OjYRIA`?&R2@5UL^l-b>GjMBO>KZ`} zh^OsCl4F|NI*(p;)#>hZq(mN} zF8$u7toj5^a*qZG8W~DiR;ohep(~j(jAkF%xdN2k6_K)cTy=Dxl$|#m*~v^Ky$;es*slRG^b`Yj=}xvdfNeZr7Yoalr49`cv<0#xQr%v0 zw7XOO*JS@Ej&o1YcycpTA5`&?DE@o5INU9ca*JQ?_5TyxVh#|+*zMNtrw|RLw`0(5 znop!8iC8sDG!U&oR)xAX81574hQ1bEAdxNs zeRbY|=1+%)*j2?Ww-?j%0IrOG+`9K|BTetyW0PZhDYl7bizG9PU859S(DH~B6owZV zsGa=L2G0!<@w9{&7zj_}7=!1Qh3taYj6}{CVkE6XuE>Hw_WzL+yLTVQw$oy-)<*ZdSb~`NhG0 z6v)S~VO~)Tc^>KCggS4MIK)j4EDUg+Bh2f!N%-92!W8kxG}QW4C|Zy3iksXCUNH`( zf2SK4hbDQ5ZeEC)wb%AOlI`8vx4rG$9xYoaM(~<-Tq=|HITh%c(s;OG#HZlbwr!M{ z9osCDeOe&9oC3?3Q-J)|C(``lHZTeE>huH}(k+hh*(}KQd-Fsl$Ung^Mhp{)_H=AH zN;NWs{?1FsRx!i|bZI~w?Sb;>^`fqbvte0B8_HfnrFx#>YO&0BKsOBqO-H4HL)&JF zGPg&FUw~boh>Rrk^Om7vF?vniv&6X!Vooa1twTj2gSO5R_qZ8$yaT&!s91@2L?6x) zmr7tr#fG5}1MaxbFHXVAn*eNkO4Qt;Pn;^cJAhC~>ZFge-C zcnE8o_*E*}*c1>Ka{HrY`(v2wrXk|5?q%Y5AJ~c#uLi^g4)wA`%Meg6e(l==VRLKr zwI><$bu9Xari$@CahzXF^6SyA{k19bIO|2Q6s;(w*`3+I67P|B(?#cI6AR9dyhMqia55Uqmk>^G)#5kWgGDQ>G@;ig>gTClG7~R-3G0_M0 zD}Lu+?MbE|XZOa6+aYepCM_3-B^We+Flg4rfvOU)9^#{Vllw;ly^NVX%qKF80M$Cc z-CPyJiK(n=wm86>xaJYQN!m8(aX@b{<*1i#kY?qX7VpJ^uo5bNrAYQ`p{;Ph4+Fe; zL%qoKt$cHdnC$z}EF2RRia6TBD-f?yb*38aHDK92(zhIcN_m7+Ky#8!uxcoHx@yRx z6`;kjYg#nGNB;((Qxo5z-w&l;$+VPhBGtE3k8;+Z6~3YLGQgkISJ1c7+5H#jT3t?% zWN(WHEcgxn9Fr;z94bys69>kN6VqwFIM6Ll%%FMl-(k?Q2f8Unoaja`4n*-|-6Gj$ zS@0~=@-zJmq7DZb(OyU_FqI?1^9O+k2Z48JxFD7bXe}>RN35#&-5x1+OJuw0ApUoRoAzVl zhQrwGI~N-45oM~@fo}wRcVh|EJ!x}5Y8H7?DIc0H2c!K*b{D+jAbT>f`(Pw!{FJ>n zS)t*{^bPmx7~naX1&AkFIv$RKZHM*E4LRW1b%BFAYwF%n7ixGaZ361m?2RZZC+L8l z0jOS#^-X(T>ys`G6tsu!D`?g~uv+EyvMhnk(BycLmgX(cY;7!2grw2u>~JNg)X(1e z)kHW5%jk;5V2un-h+idnphFVqYnPbm77x39Vqyv-KRIcG$broYhbBd2+ar~Kp+cEX zcQaeCRxXtA=~89-*tmllQr29@MXGcLJFN7!R z8AgAGP{Ql`3|8;yNjr4~SB1hBEj2%4C_ie@cn2g|90DrTpfoxHYy4Sy9%s!mOnDWf z5_#ck)P)57xKCqUSf#M0(G|>>F)8)cVpIwxXH&V@&utI-j+%ahe!W}Nx!tS90ZGhf zjP)@oY)<9Ds7ji&908hhL~^Pa;TQQSRK$*j8rsG=sy#thu>FzB_UBg%#AI#PT_;l6 zLcPp`wQ8j}l6@atH*ub+`VhCLUPf)#J_LBmc55!{$B~b+T&>Z z@v98$-epxIC&JzH`i;-q-v7@D^ZyMKS52s$&V&lSU){&qhnxI2@Dz=lYWa+|UbCjb6z5r=jf!rp=hhP}S*);;Qv>2QbZdqYxCaWg&HpC5_US0{PoNAk{EPgFEUK>>519>_|zT)9fx)1@!5#h zg-2;N#E5g;+`~hBW|?`*^ze0q_2K?lI~eOqacBxKKR6pUP&np)#vp>SQXG`RZi8C= z+TJm!KV;PV`(gM}oJe-VBtsaM0=@Y48#h&p-!Z|r%odf5HU|L@k&!Ac;HB~Nm=d^V zzeRK=g-!X>lt0_0P!FejV1lpO@Psk7H6)XcOOAaQcFW4kHdCNZ!-E>$$Ea`A?~`dA z+Y;6OAD4?m5_GkS4@_S^=DuWOK8$tip~?Etkn=u59IHioir8jo}hfrtHzJ}J$3l;)`Dz2-|K0`kgX6$ z!-VJP%y4ULh}pnA`K^y&OMPYm5)!D};V3=y5E}pU@WXFfZnJUfj%qiL$UVG%BKFH zrdYX!Mz2@fdmbn8!udO1ICDTQD#9>nM5N*x{3!t|)7`w~sT66}cZRgbnKWrmZ0t|k zW3eI^D?8lXgHrX%Ui)uVcO-4KS2QTPZPgD@Yys~y{1Hk{oSHNVyQmYP+a^M9P81{J z#a}b1Zb1|6@NfWO6s=u=c)}*nNP5dtz@h(Z`P@X9?BPNV*oL}T7}`C$ioP8R~(&zfE!}ORLRnWolB+QvZ7V( zzNSaM2J|&ODl}juJhoU<%#?W;)%I&6i7TA(84qLm6Awb_D z6m?CCEEQwD5+T_*b_EC)?nk^Q|KQSZfNB9H!*gRsSbO&x6FVmB@>%jawAHNQ@5$W4I-B^-b~MsaYG9`LM0)7Iaa38^xBCCiU+?m0a4 zhx@)neEL_$iOWy=&&t;(oAT>0Z5)m?YCsnK4TjA&v7d>(b~&IISoDYaw_!ik3{0-i zpnOcuRP2WPcz--Q;ac|Jl*N(dSH-gERoFny*Xn`IbpSyCC*rLs~Kc4c842MYuCs+yD?8akGGAnQ(+RF?*fq3?Kp`BPTXP#*e` zek#Lq?y=|zcS$UpSJ39ejWN=es+zM6B-C5W8V6J1h%~Ay&?Z1(nkmL}z=(Q<^PcS_ zq=*>d&=955$Mwu#nz;^DggERL$3YXPQuq8O zERW-`i?i61MSHM3f>|SdG`n3ht82JH+yTU`UITGu8px5!v0sUU;$PNEBVU8!=`*O$ z)#4!EHj$TjwkSvtNh!Ki;eHB1JT*(Z2g|pfrW%w*53uysyM45IzR2Rmd>rz9nA%#a zfI8hQjP0XB;mP-hj0-(!aQMRKAT7tij|bZnTEP<#VYh9Jfxv#C%_bc+%{ z=a6Anmjei=yT$KQFq6f}DR9v;+?2=$)&7(vhPi=smMC?LiLyd0SC}WRN4nAHu#0ZN zpQDG05vf?wc8F7XXE@V+E#eYf9T8$$HXqRmo*=n?&nS=uq;tiu-E=t2Df%mxKsq?! z8D*y=jlsTGCz|`p!~3WO+M64HkYbfZk8*4Q`>PQC0~2`x-akoC<}RlcI|z{fbCmoe zwJFej7B3}g-nSiW5Qfcx4(Gj`182Ck4$M77)fj1e3wu@0iHDlPcQL)gQt7ir2mxXZ zsS~FSeMyTzn^Uf@rtWe;SII7e_r7{Zzl}33yv%*Y8XO94biix5_EF$|$|^3h1Pf%_ zqB^XLZGzX%XBWw}^BK{LAI~hR3tc&%5Us2=m3fkzqngaJH?)2&*JkX7jZB)u!#au{ zUWi!3CJf&j)Gkg-5u+2tI(HgNuQ<^!aucb#N4oY;*J&ReM;n~1=N;p-#{s;?0F!>2c#NtsNWMjdE~nOpN@?2c zmjLc9iSCU>%3mtHH*xUL(_ta5UM;dyz_whom)&@rY9@<%d4PR#KkX2+79<{T;2#3M zn^#U9;nJXybhBJuQg^?MNejm|PZw#?MX>H-_1hMbDcYk!2iRM)KK~C|iD#Xl+B|@G zTN!lM*JG>~B1Bh3sP*AM{x%T;gZVvdeLKR~&sI`VO8s>FU zhdWSDlUiv@O>hIfBfpOxS}4Zw_VZb6`^RC& zx8MiO(H5ulz1(%o`U8?Y2hfC)*ll!MO*eJJQE}4>JneAj;omy^3$+dy_QU}V;9Wq$ z_+N3V8*_~7V3x_6y`eQHNhoT52=1%|)(;}#zrYO`oFQ)5ijC0rVN5zzQdI@sjL|%FvKr0s9#;L+%voSGG)quFLQRMg6!^OHv^Q};W zvuHgKt~S{PwHk1M%*t{AjSk=u7HSTJjKBn6v-xZe=O4`szYz@CF1vaSo_fLXYrybc zH})O?)irgTCe!W3F9;t>a3?-RBUhsnb_xEM zJ8_fj#3!nLudeSvQS%ctd^2#{T@7hPl>#GVR|Duyuj=a6cv7@*sdA^QbxdC|f!zAjUXW9uuOI6=&mdjOYusGv?_1~DVMGd+c-!UNpy{B*2e z>_m@C;uMuL{2WxO_8$r}7S{BTplA}lW^o2Gq!4upim@5k*ljjWGktx|W}vca+8n?QOw3_y=&cq$6KctwZffq; z*6eZ0^|dtF&1oCJ&2nn8$UIxLX{s$((0eUqm9+LOWRH@PJw+2#dOUw2xUwOL?Vwb6PC>( z98=njn35AF?5Q+`rJ)=Yp@uTg05+8&xT2bE<=qrON5oiTE%#KL`aeL3SDS-OiL(vh zYnCDSZ$1;rDRh7@pcbFzaDW{SFkG?pXn>%1I52?G*MpxmN41tkcW}-@S_Xw48lvwLE?J}>r6Kvc+}436VOI4(x5L*R7Kvw;AgltJv~ zq=?0^uS;!10w{-l1TGdC8`@txgb$znBhcsY*)u~-jHmWS)eqx1kK@w{@-LjZIRwep z|3P+i@MeQ8VjfY418-8>yJLM+-HRm~Mpu>n??$*@$!;XrEA3Ewi$S{YmB!y{#^*&f zK1XKQ&<-ye7w|rSJG`a>l3QPScNf zxgVJ9c;AZtIKbV+T{k zy#`aS!Za>}E|r+R>xb#8zL>rmQb4{Sy5i?4^H)rn4fI)60OGnLj;G6^IL2omuiRcG zrtEDvmT^)R>OxlyJX>tMm*LHcXlqto`bnli$HN9&#A%S&V&MBO(QUet zpp`s#pmE0f>HR8}guCJpkq(oD?!KLEL}!oj)0%uL4acy^SH9l`}eQl%@ z9XQNKmsI6JKE8?Lj0?od!4xDw;;oXXVXT}e91*#Z__~4bJ2%>NB|)8v8}WR6?0Vb+K{%L= z0hcHHLm6`YEI!}8H?8xrWXLUj8M57BNKdCCR?vG$Q$xsEo`Z!f^3nKfhop!j@c%IW z`_L5JBZFi4{C7?YKOC3B=RA&1rPIK7TnwQOs&>2|j&m~~emot2fZ;&obf3hJ_IXkH z16$ZQ##V_;RQqQV|138@{rt1c{7mPc)6LIJ{yEkB%;KMg=I3bsd7k-sApe|Xejdy} z|7w1Yp{Zfe*N_%*Sgi@0Tk0ej?xN zl4Y{&D|$@5FHOBG``6o($op?d5KtL&!Cj`_jXt!O>3dV0pMl&v@n^p{oBglje6w*U z0+d`0?#`gM%2>?_YC%v5>W*G3!|{TFCpO)W*pQpvC=;s?s2yfxNAwH28 zUD&r!$jdV>PC~lgGTbtS2reUJB-LW7o1S-J^AFYeMSx$7<(pHG|M5FkKYVuyF2PuX z5%;n;Cyq}zhi|UI>Gp&bKyyZDRqg4uBtimR@@9+;%&ilgD;Hcz2J{GnaB3APQ1Fsf zh^SV>`yM!LbO3yv1u~x@#!ApTG)x5}{)TA4AHyz}fV^sou1G?&#mFV{`^Bj~dSzj6 zY<;!Hgsd#P+<=%WQVgaQVVGbtJXqNqEBZIym}aWn##PGD_bxU1vM3OhlP{75gqr3b-ZG@LCwiD;?EelgG|DaE7L$XG4udr+Te{m z?(Kl&Kw%NXhB^YZdgnpI2jo=N{s#Fy5xbo@!W7tPx*hD_?avcA--h!b6u;WCOvQh3 z%+p+8l5f;m%Scs3k=LIyU84+=-TgUJF*>|{|50rlW9qN!D+t#{7KEMyOqD14R=Ff{ zm8^qImA1ZBzMs__N!;=eF;zD7t+FR_mD+Kp%Ikfre8yG!U)NHbW2(H(RnGLUT*JX< zeOthI1y=m;kD%}{gP=PCf~>>C5%eE`iX#ky9es)Jh)i_FQKrh5s!dh{A!HwI3T(xW zG>*#PH&2H`4H&RLwa1vMS0sZWlQ7G%-*t?U+WccxfdNre9A{9h&?pdHaZuzQKNyO_ z6AX$5MuB13!{M@IS~;sd(m3XxXe!*t70v5Jp{gW~vu!LfA!9Dh_L4k8)2OHK14V!*peC%C=MNVO3eew>9w{58XzQ0U21 zb^4z?%9?1i z^=OEo9q1lIR9a!;sjAKp+QpkoKI9$gog)nr^eoVY-X2!rfSzPM4l+(~Rw)e}c+~%- z8Wr^Y0Hc1IDlssQe1qdswq+;z`i=VOs)PwMM7~F8{GUxZxkoiRb=z)pJSpe^|7*v* z@Gqu;h@M%a1K)SYJpWA7|ND3d9ulxJM&XYg_(F6T$)?J&J<8k&*LK$4NpTeztWQY~ z!T%yW1fup(&!9u-S1FIxa9I zzq8LIU#Cd^QjvV+ASCaUBwsTa$-5-Umq#S|TZiPvX@=w<2O-&U#&%s~J7YH^@Wu|u zCl~GA8C!;=4+Qux8pIj9&d<)+M;AqM#%}i;XRNT?kbI{f;+E-e97r%6l3!6IuTdmF zKM2W>Ns`+KBl!tQ^16s5w>cyy&M_oEFG=>D5>7Jw-O2vh@Xxis+ob4ySJ8X-MeyzR z;>f-!3Ar;OAs;w|qi@9rO4=MkY(iR~DbhxO{$qzO)c0L%Q%4 z^ZA(K5%v@t-=MJcGXuh}stfijr~MGuN`)a!-o1(&AnSX+SiO4o(rL(k#{MaIhS0OV zE$lfr-xfB~-*b*mtH`>;)W`i4tG9jVNvEs$IMn`#?<_=M&kak`D}E_+C{~IExDgha z6+?N$MSI^}cPSLsSwVSlz40mK$sQVM;vs8uFv%X`9}7X`d;GM8fr zNyGGUhs66#-M+U= zuKQK9b~c_#)=$*$?aNvbRhO4e_*9!3DrmPe{f?H&X#J+&SEW4tE={$k-#4YUvRjsF zE4!Z3y(_!RQ~OT8-KBe1$X&s#1${zbH|z$ z(>M#gulORZU5f9AMtoZyRakHva4N85V$B>tcRDs9xzIczVfVS7xG{7*%4Q|aDU{|-sZgAr-jl3K8Wm&QmI z2Q8D$?oiFHP|bd+n!R$6W_QVEFOS&lx2oB`1EiOUp6M*7*QD8U+B@Br(_7MPIsJJ0 z-sN<88jsPZ(-E24+uCq_8Vl#P>5+u9Ai(&1@Hftlw;boHKFBQpGD3X6vB-c&fyZC=AX{n7!OMSY{ zuD-^sX4c2kSPpMYx8-pAG-Vv7E8}q5-sSM-be6-d(*_yO+tQ_RI4zPK-jnXgVYgw{ zgXvNZ``WXb>gOfZPbsR`E2`His-GT&>eZ6!X9uJDX-V~(h*Yn4sP6jAQ2mlk^}w^| zE=l*>itc+9-P;u1w+%w~yOQpegVEh3>Aoc*-P@Ho@Lgd87_k2?VTq{AuqEP$5?dm! z%CIHki;}%d#FC7@E9loHdtX5>&0yi!T@pz+f*D3QI=(e*zCL3RCH0oLI5AVU3sr2DmqbU&Ign38%@lK6rmu~m`Sp-6mW5E9!ZiH{9N;zmj0 z!x2e*!(nY=4C(E`Ee>lVE7uz&-JdABZ&GycR&-x82;HAax|a<`_fARoRT1g_%AtFF zqM`fWHr;PxNf>w&{flB-maiXX%ksa9_b$s<4`V_2ym$~n2o7Tb{F0e2yNUx?o2UzWODy4_~W3T&DU^H{2GFAE)kJJig^Vg!$@q#d}wE-^=cNKQ)r1 zTr%89N=LTo&N_U+ixu_x!|FHCV^h0n*TXqymgw@S`pYl4ooDK|CdePE}Q%?tjU#`4b)YH zTP#e!8)d2Yol4`5=vsX3OPAAViJMnU|mBdLMzS% z7FoYl;xL4+;QL9A^M5Fgv|Zyy2Xv`)EB&G{k#F3;NFH}QIAIrWLg@QyPF8q^)Aur9 zfMBH0T-{lrp_%jzpTEOpMWcO-^$t|l?^RvfbLabd)P+tJ8Y1X!7FOK4bQn_KaoGxW zPQ{R+KTX41A!~0=MGDSh{9H90%;3})?6Trgon<&n`O6e>Cf|`KauYt~h)O|sPtrA-iP z-){47{FA8{aqFo|!~gT!{Db)lb^qJ?+Z>R6TYpx_ZT*R-4_GhP0rkJ-Kj;AExxiC! zTMVwgNzg%}tUsHs>YX}4-|mk?I0f>sFh0P5bA&kPnC7Q-E{-0qT^tZ)xQ`POnaI}j zRyPvoajPBOa{=zpbE|Vi`hZZlj)FvO&f-f!KGv01?o*|A!zo%;g!BSe_vRB@yIZif3Z)MF4dLZq@Cv@-`D8S=NR5R1y&LGhtaK3Ngv^&S{+8tTA)g(8(&(v1WsC;0^=8&7Ov#%@X@LD zNCa4IzM|Bo;i?I64mbCXGxz>BE(&j;R9z{fGn(>i4F0v5z<-21*FGJe%9=%Mijb8r z4oH0fQ}G|X5g9dz@^@@_Q^mloT7I(<;onr4S#08;PHtxom33oFm zBjXmU{S&GIdT*h?^;csdGu;p>l~j?q)ZlrY7X}>DM-DS~>yN2 zz3w^j5`!W961FS{zzfozkN|qV0}8K`c&S15K5tE~5x+>-sN?mu4zM1Z1oc%~9$PdZ znHnIE^$?0_=4&jj%M?nrq~NPl$0LN0iu(l)Gf9vwx{#nRAUNG(LK-$Hma&17#1}3j zzgID=@Nz@p#Y*;Z>zD#II>1%D250-9)Sw;*w4OJQLOvLjb%jEyGtJGXBXb9Y6l%y= z+7>$gO=|q#=s*6e)cC&^I{qzJn(p*~r#$`#_|fn1+b?^rGNnGY_s#rI=yfjlY6a`R z?p8RU{_AI(29Bl6`SLz9nmrnz_Fiy>z%bv3nvaFosOHoaeVALrWE2Mm6E35uGWS=H zzt$Aw{TFjdU#|0BUMMijT%^^eA!BJ-7)4zgkV*|atouQ4@rS)K_v)O5Y=g(L#Sg)FLXl7e%Knr|{qTRWy~HHm ztf~rngD-#Hj6KyQx^}GwMw-qIaB%@#Qk>kxQJtW-k?(rZF2-#N$#7ia#@9U)K~e!uste=Bsc;q(^w2iv1L zer(brClD%)bNTapuZT+C8im80G7QH74l(m#{7TagzR6&B$fubKH7Ju-$URyAGKvY( z*Nf`tGJGfd>qYpe*`=b6zFU+{OK}I@Xy3!wf?nPXhiU^Z1i{7gS-$fztblZ!cv4Anb}JNqoA~5)m2h@M5?$AI%H;+U|;j|*466(Mn!v5=-5#85RdVQbVq6f8G^cH`9 z7jEpEeg1um{(iog?zl@E;VwNE<}SSy>Mprfo6g?BbMqK{55+j=g&HX6GkklVZC)H& z&&D@}VPxZ|<3_mK#XfOH3UnnO^~C9bn{oE!7i@!L4Ab4sl$6cSGdnqeGdu~y^`;n+ zxJne^z9D=jCWV3VIAiB+&?llg+7vzP6 zk+|CyD&9YgYM_hH3ylW+I4UF> zkh{sS>gv&86&y-jDox@~KH7CIhn}w)jW}?~N%>vps$=0gfRqm#01m(NdGjkr^A^sR zsEv57^Lg{DM;p7dt3mdYM{2;lnJtXu)lLEutkFgJPXmW%~XDf8wmf5 zci~40Y9!^qyuc&>6N4zCxC9;ezKculc0>DGHOROy@MJZ}ZEBF8?lZ`1&&AZ;%OJ0j zgZ#`s4Dxdr^2p2-ttQw>C#qu$OUONXHdt-UCN1mgmzl{+uyWMm01|fF0_jJ?{yBIc71PJ+{%od zl!j$$^%CuO*8ZTtM!NrVJ(%yp-MM^rdVsG41s(W*AlPn&6gOuu-S_JQI=I;_eW*B6 zexCdoecc2;LLO)P=v@?64=eBfSon|N%j&TP!-U1v%IZRkHjklOZFxktL$fTqoI5s8 ze!L1fE$)DDxrA#!{OxvsH|>XhS(i;g|~h@7(GDTf_??6~pShi4zggM zIaOtoXBX=R$4x~;yRwQ0ct-hDe!|dCFP>b%kK}R{S5}r+^0UX~nmltJKUt6mKBJmm6}s(+^;U zx)V4CQmFrjI*wKAg~c=EfZ6hUQbh$wn=DzJQ&3bswK#inMMchU%4Y_OX9lWrep@`d zxU9UQxH2cdY;vHaymCfW&Z(1&PCKI@yF$GPDRaheCu;CvDLvxYqg^@w<3_9IRpnd|m{(B@A^IOTt(w){H@q(edn;#7F8iMo1QmV}TKHOjg0{DTY?Sg8I z*s<~#W~Z)nfgVN_!b-3hro{PsQXobq+nUn3us=(wx zmHjANIQmH4M4)JL(ez>)6oA$#uC$+vrdQ9r=z^(Ls#ft_o~AaaqO!b16}o7;{cu55 zXeVY?1xjZqoVp7r7wvLQDGkiPGA_vrR86%XW>rtFoIEp7I#X4dth<9gR0m4SR1?e@ z)oZZk0>xRw$(Yc0sKURw%8bc#bzj*+2u{bhjiCeyX$Dqo>?*xg*`nX$yQr2omP+~X zo<)y|c8jRG-4gf|8Lkwvyc;|*RxG_pL62o&!+a6$MYg;Jc!~IxD6wF)`T&Wge=o!b zfbjBqnqYaSS+oq#U(Ro^;^b{&3u@nO3L)Js6gCM#ySCj%%pESi8` zL31<|MQKzWo|+f5gS<9Ruf(x*ev?IO=i<$X38+7{;lwR^uPS(uO%&s1v7VBGzo9k&qs&pQsP#k=;8x^*>L6vuMXEnHQ8IHVmR2k-0YhWyb61_^!k4?4%dvFh zqE5-?*F~pfx=d0>74ir}(z(Tb70LLt}+d5Aq9 zuTSUk-MuJJhkI+w6eYOx9~=ZF+eHKSR-6S{k(h2>R4~bt3UOHN#-P*snmpO|u7v@# z-D8yiVQ_vxf2g}lBm6{nZCwpJ+AJ;Y?w~}}=x(sZXGcx1MU7U}o0bO~4c3@R;J^m{ z9ERn=$B*l*sIz7H%iTdPA4Qih?nYGB6OWEuvyl54MZF}Bbn7=XmjyQpCouCSI`0t5 zRBs^@6ay*4NQ%rf%R3v09;j)cE%*>mIxG;Ll`|DX{_f$~hZwnJ>n#i0fg_rBP&c!Q z{*BkG0E|GPBnZ`P;|7s%4DFUUh} zH(lX@?12b;eH5v*EF>s8O*Pxv+HQjN9Tke%yTNRC?5TRcuuH%HHvD~$et)sWgY%N4 z?Qv%7ZZs_D-bDqHphw&g4t(KDHu;YQK`orQvs8OmqCUmWwY*EY{3Ka<-J*8T-X=;k z6&1QP4*P~uSF7LO5&k}?-)~D~Ek*aug66hTD6w4<^$#e-WR{uJEzgT+X-=Rs*bPCC z@5GNP{sA&@MtevrSaxL~2(?npLKj7Kah#;g`+)9c6 ze!>_*EqUo9Pr!;=h9ZLQpNp3?qZMzf^6)#p6RU#dJsV?uNQ0HI5sk#PW2&ItMS!&ea5TF=zu@@|O6< z^4>vf7S}ng^#WYWJD_NK9#5kr@P0WaXLC)f#v8h-rKC#bzs|BpVaeQHW^5iRRuzP@ zI0m1RYO;pDXn9_h9DW2d#^!JwZNStXLSuOHdub<>Kpcp^VPU%!h3|_3eH^_KRf3lm zt(#ki+C=|ega|x7=hf!MYJ{s4p(LTlEXuZdgU_Wv*c=W@Zoe-#a??_*_(!F{EUnCw z0y9ku3@i&~;487*6+xd?0XGy5-4xdiI|0POB$y2)v>ERjV}sPRI1j?v0V!0%>7{3e z;ERIb;7-0nUt+m`ZadAu;Pqh#B(76QT(6QiXiX_`EL8V;@Od36Eg4dJcxFJF~KHx?9F)GWJpZ)`iS#}6%Tm%{npY}Ti^k3ObV{$4*jNIsrWLuiQwnK~O<`W~ny zHfNrj&8w)FI%8TlT2Xgdo@Ps2XL&CK>Xld+d?}JUcg}a{0kgr21o{gT6mFw@jG458 z&@pTWM>ScVWtb|%Q(6?eU_D88F}wc} z(XUY*UPonDY} z9!4RKkr4e*!346@Uh4_s%@Dd?t`RK7d9oQ>inFv7H^^$2&1HH2z{R$zmmX0?H!Z|W zM+OOA#!xe{5Ju7Rh0>U!)J`q;2PnDs;42h0tVzr)fcY@V@>a>eG25Uy!G`ZRZ3w6= zdI3E2(z7mBT)ZG{Sj0~j8rcoIg=OKo`8>8P4UUb)3uEZY37Tyit6ElhU_NsOS~-C~ zvu9!Ytg7zRU&J7egP8hagpB_){G zN%Z6*W+(#i$m8H{cnC7WimVY<*sb;%=D7AQ4&kRTvr2eMG7=tjYeT7s9Yb+jkuMo!S2MfFr+>q;%TuPez77srfn zaeQ?lO9*59CyZ&v2z^NDw&bzQ?U9^X0gFDZ?j_1#+L**9jTo5@duC%ri4 z8wG2mNzh~)ICWBqYXj%sak$ZWm=*U5Xz|hy<+crDmLzGz2)f#0rv|plAnqzqd`(UC z+Cro+LBk}l^c!QM6>JSLCZOlO5v(IGT@;Yd>}OuPh^r_gVozivVs@wz;Y){r3;LS} zo&Z)P;QLj$J<*Mkhdlw23$rJ@^pa@DYEieC^&6}iq*!s01c-Mv*m^;?#QQeB^ZBL= zSeF4!QnwZTq7;OeV3}aeWvPj?MzN*uL*+RZ{Q*lBR(LNjUBlC+$?0?5BA&KFYIlwR ztc$h{$seMT2rOH@#$x-YEi9R=y}>UFXF|*qJN4Ba`0cC zhs1Vm6{bk5u%synz1IrUB@nWlN@I}d`#9)4^bT4g0AFUa@(|d=3f0i|LgeUY{vE2H zjSsB#Gm{-f-zKuMkHO^II=7RlUBG;o@dmu}v1~X22*&tS6suQOT3cta<7JK6AX(oo z*gUh9El-gZZqAV&z*a$rwXIUZwhH7ox)1(P>m^63C;R$g4GiydAUqi` z$>|n+W1NSy3q#5az*qD3MmBTfx4nuLk|!9@uywAN&3_nl4cx7vbQGS^o0W13gfAN# z@xHQ81Zdh8DePa?V4|+J#F7(jF;7jKh*aSHpwgLcnB$CklRFQ?YU^HJ>;KMaix31a zIT{uYA%Y*auzo_VA&PGN3Z>{?^~sxdtu+Q{vm;J@$=IM*c{Z}3r zGr6nP03ZVEAs<;EvW1>WQ}($Y5o_qHn>}y?9oQ zr)Z@J15}Ix%Wv%GSt(7(28rWYHA*IKscBVmGo0rVI7IXj4ivE}#%hF_I>2mcJVnly zUY;wg4Ox8*Fk9Y?Fjrth{qGT8RCv_*tL;mj;@!SU#<2TrhUpFI<1)P^lH?<_& zfyztD*xKEzLb2r=f!HEKVhVD>OdBOwoKAI0NNc7Vy56 zhc+!qjc>55LSTsP1o~!nd9aB8}VGw70qwwJQR+YX9i>9YB} zK6B--tpeO=D#`r!JT#1#*lkk9{x#1oX3=#xv6#*d!%VbUjUq_Ya<2v@0oHdOI<*_o z^h>JII7d^hsOsu!ye(t@!Hybd(k|@(5ItSf4!}@;tek)+eA(4K(aNM2H$rIiRZR(= zF9&ytKEQJ{#kS)0+&r`xgNit04UCOcf|p$jC*VWy4Or^X6=rkOd>lcSo#G8n?&g!G zo9*c4fmjcjOVMD^G2Hib9*Ah1lEpI)-P>4xn8(?!48|AkA!3XNx4tTNeF;1VGo!2c zXu*giGAtt+~TNdOIMNJ~<>6+-DY-ES%o-Bj*nMJBUBdegKpB>>_a#6D@^qn7bPnRdk~OzGn+t z(?j0yr{!)I9!$@lPCv9rNQUd2u}qJcxPNrm{+ z+ek(euD|?`a>EA9*!ejUhhz@#|2Q%0|4&4-lK^!F)BijTODCcYC~2Gejyp2AYL@LTrBPFq8FDk_n>I$h?uF=vpc8k&LSdfL`!I2>|=%(XhjGZgw~?c%(8 zkATAZhbvFM_-!Y&72d?s;k?D?ry+Tkd$vgvaIoANg5}qFIBz8@9n8*wpHf3BbvTu} zy~&;=&i2_kuN~zq$W8^SDAZR)R}B*ze$U z1cBVBO>)T;&tTo%hPU5ftrMrE6>@hFj*0#97f%Sg)*^f)YoXu8V- zPeo#Xb{-aAAoS96*kcrQHbOc}urOr$CHUoeu(nj=j4`xvHbKR?6DMxNYTAkB2{9ec zZU}lfiJ(WeJ;E>BZj)`VLpBum2zDo0XSTI_n}9q4TLIl1>)@q}_xO38;>i@ck5s0#2HQci*^Jz_7(l^a|Jzg$rH6yG{>#c!gyK+&|Bjnzy`l zvin~wjM)7&)qPs~QW(YR?ued$7>fT9rKI;7Nh!X4?8WN7*}`tyDme~^_mu%q>A}!a ztEBbFKD0W+q!0P^JZ~D?1gVBN#?sOn_-P@`fL$(LnmCbxbW;r*hz1I2W#~mxTHQ3B zQGW@EP;YY${xDG`j!w~llj|?F#=~o_sDSX9E4Lk-}mdB z68$h2GZE6iG%Bbs1x(yvXuK*q+1fu=91k169(i`tu92j20*avLqcC$e+fw$ye4cFA zS*#QOH9z7+kD*&Fp1&+}8|I3W?TOwR#r9oA9m1%7%nn@Lf}f|#vqfa1t>L(OXd1$H1BF8QS;mgIziBL#3+*K$r{jzS)l~s%5Xt|ufl?$)J8k9#)z(# zGKeF|tfmH16)!~DbLDcm6U@r=ZETxH!xgy?zGq*}k-0A-#L^U^30R0&TZBoL7NX$G zW`ORAQIoBCP6Ngfesr|0*VfFzVQTzdA6=)bwK>%uk^Vi+al zE314ViGcSHzXZ96`1UiL2(c7iQGIp&=4Oz;%Y_PtEOf;t>!wIuxjfcfdXV2?8op+T zO#Af16MO~In1&5#EK0Q)t+mJo;~dsHx@s}z6gr#a!-U-K>P0U;tAPmCTk%LG5~-=p z%@132iRHc6qAxKm_lFlg4S*EBHnARk{|gRu16MY002BQgDgZ%x8ka{`Ah`gF9GHk^ zvADYE4bg5hu&a_+axdK#4|Z4M;ps&U{kw|O@_jn1S9;eb)5#6-a-xp{_a346R-EAc zrm7By^MOB#Rv=VE__C`N^-FXw(~hM*#;Y-1n0F#F)825|=7B6QH18Dm_7dInlS}RI>2uDW|(uhMSEMWQQ6L{$e zcKPw*hQ-~8HSWZVIQr7Xlz8Y`Bw%yhC+8N>%m!;DU4o2oK~FDkKrF0)UPi9oNIFiv z0{R%b7c8^H(VVr5jQqJ1=pXJ009bUN{74VyF#J>QZ3=t=xdFfajm%$u=2H91>M_i* z4_q7x^U$7}ygx!%4(Di_=a=a1+Qk#(WVtc`aiO(w7=wv;oe+?3|5n92g3OiI;&^UR z6|)xSk86T-kK5+WwalA5$(tM898(eJphh}4gK);F5EIlc!CEBJSZhAVMl#g~KmZ2? zd1_-a$VS-eY;6v)I#C9gJk%b?iKUf@&Y<*0#6`rZka;brU0aNxuC&umvW7{#50&%a zGVz-egBXB-nm`xjQJ@#8 z9QcwH!h(weoD>_2Tlxy1L9OUVEsG7cc*M~7`qhK|^;pbqE4LNk$BA}=!wpzxqe0$F z3lWl`;W(>71$pPpeb%>L3ZoQal8$yXm?lJ|qxJTLX69@MBcqfwH21La!!K zIGVuA%&HT?j@8JW z#cd-MdB{k6A)yJ+trfP&4MSQZYz63mZxc8?y1N3x2O_<6GtwA&FsilmeTJpNZH=Ji zoJ?YgBZ2o01jpR;78acoXh#$nmu-#Rj5o{Ya?os7R1jl$40H!=iQ+o3^bBIBU<^;B z8<5Gx{@G{9e}iyxu6$jzJ^p9cS<#Cvx|3OcT}3C&U`64fOA_lQqT)vtK}Dc+a99OI z0ZSNh@AM-Mq-0BpY@VBEBTS)p6Ku8wBwIRdwxEUg@SZTA@CXmkL$Mf5T8Bx)qWBTV zEF*{_47Ps4$h7@u!gz=+j7@N<(5rPr!3l~BO2XeR@3zMZo9BFZMZWy}q9*y09`M&$ zW9jKC?uEmCY|%lJBaJYM)>rT$kkAPPO^@CCpMq*Ng_=0l?56jJa$@yPIU)#`LTAEc zg9nQUr{@$o_G^)jhMW0fIdqvo*9TawVUO63bcXtLNdsACd{Bw)ez=(-nT(w$uGMzg zTy#~tZS2yyCwOy_ZEG)C%Dy2B7E4GJ!%Z5m(GUher3^B=)!>k zT>XwMhiFP;-CkN}jiM9;_^oJODMr`PACUG%BTMk(g1Wp;EFmKh;b*9q;K;O-Z=v(Sx2<3IdiC`;j0#jXB zkJ&|o=DDnrS8!fXHX~zWIQm`4IhLtx9YoXSC+(gf?2CU}|}=>7!_ zw3S_7bh#1Nox3HH2c!nTy&%baT1C)5D)Wr`<{5+b-i;5O`6qmq02?-Q+wnUFMl-!M zgO?4~s%w)mlU}sq_nXo$}wcQY^w(CA- zwQbR7QQg=JppH3EPaRfNvxSolY?dQ4QfqHEs}avguG8yh#daeIkNT7#bX7`!6Aj-P z&+PTm2bFnX`Cajxf)0ZgTMW_kL?tV1o8ufO@Y0t_ol;R~2GyfAsbs{jP zz>={KsNpoCVY^v1VJXK@FtOe8uCnMAR@ulKLEfRj(q0R$duetMkJvCon5QPNo3ABb z?uK_j-_77IuwZ*n;bTh+b~ z;D6!>-Q7%D~-W2J>d@dE-RdemIdvYbrS36$`2@MNrEaI6fEmz}#Sj z!xs|!cF}|)_Q?V-%?VlwA}I|x;(E~^bpS}_l?j;XANx2H+)GbXaBxXRQ&`y15L_P| zF5-dvIR(Bo@$H)>vubDX zi)dOZ7kGr4aIi7r8(_!nGq^S4C6AkSv127gZvqT9Y>mQ99om3{eph3W zj9~+XwZiIIY@HFUAIifYN6%KGhbt%*14}Pqv=+BtG39ZzZZZs7>1;_U+bd7wa1BN|h7Z$yV87=6eTz7$unwXflK7EyGl;i+o)>C(j(i+j!tUSF z07t5@#hAc)lPTXai&f6GLm8i>8`NS8U3(Zw)Qt>Q6lf1%{18QBFXrOtrOR4+Y-!(+VKzB*qXLWw|_K+eV&IAa&P8;xLdm^1M}EnLMYw)}V|zlSqy ze2R#N7mIM;Q$+TB=C0w`3r-QmEVqVYFWK^ZBQ=LS*rw((jyhm2dwwO^@pg5q>=)Pp z-zzdZ_;j0QN4g)?m>mtW8rXqxV|L)cm`rJA%l(mLM(U-HajG$lD+#ovlC@I|UtJx} zhi)+}6^Jkz=MaP!+5<rCka3;k}L-edU1Pkevq?weA+aq=zMMS}UYSI|*IsMZm(lZja@$m!@Sny3WIh zQ&w_sCxNaKXl(7%pjMl>8$r(-NFcY%*`dyz#EVB1=jI|ig2&+nWG@Sf3uv<=>pQbj zJ(0q8uQ{ea-htd8A!2w2RKR!Tdut~$@4f94TJKeB4b+ip7z7gVfb&4WS!=+t4gqm!YvY90+JIE4)&N=+H8|7> zgLBn>zyI2Mt$j{NP;M>s>8+)(&t7Z2Ywb0@>mB}YCr4>ERL1`nA3E54OxM_hqB3{6 zn+v60N^1@w!O55gSWdA%XB^qQ#tl9!XcvI6`-Kpdl)?wTKD4FB@>MYlC_wrwYg+s zaaAOlSTi}{oc>zL#KC{@U-J}GsH$XYn7R$=l>_!{OVZD%&YOITlayDZ$tRb2v&>Gu zXA11;VP~>^?tObNQg{>rPk9q2-W}GBuVWeM|Vt0e16Eh@!4qoiT@^cC2 z*@vIYQXBl7=R993k<>9T>!&t`IgIJO;#(~TXUFD#*|F12R_$DE)`K7&o#WRXEh&tB zU16RaYRB!{H+*Bl_qm>r8>ii?<8bw~7{K@NKGVi{8H}GeiEHiHz;KNraiEK`?zF$J zQGpV3)zOYra$D}nJ;?Fnjt}y8r+39izlutikF!oS+Vs7xNO-rKsxfE5k?6w~AFXrCm?wscUB3jMwt@6dMNFq<;NB?r`NK3T z#KN6&6gSGvTgY9|wXkpFyXE~Pk>wS`qAm)|ua6e1jGNogUUL{#n)`-E zdVuWVxK0Fp0oqLpQ ziv9Mjojqf&1{EDGMiu>!Brjto$)yynAA6|WZ#fG4b-$%7a*bg^Ysi>;j?(s5Vs1kO zr&Wq*4Y{OuH`4-@<59H5ayo1VIV=yJ>BSltzb$TH$RBceP z7ixdIP(0!Lle^oyG5QC4H~xUTrt9XMyXIo)t!jP(1nEozXVCZHce9ey z@>hd;mz)cQC*(j7QLw!gdlvTBTjW|F8LJSwHS);A6YDxltw!0S=T zSriYUGPz%JkpO$c-Y;##+5#P8x|qMv0@J{|GIMiEdW-yh>UNuFd22MIunNHEAV0@C za*Q@|`VF2sC2qFPB{;sj;s6sH zOOC4Z8{P^wJj;BaI!X6O5JYMs8 zqmP^@0u|=`;dD>L^v_AMJqv+M$s}uhG=CDalf0nr)%d*2lDwN7$+a=bYZH>IlWgzg zkbHNdxM7hzEy-Ij!hiPVwee?PKT&H(%KUYBY6J}26V)i6_C~X^Xw4e4L$?A=Ae*T% z7f;l=J<2>YoOH!I8m>Pqs^L}W0tg>4h99py7CnE8=R@rk<_hWJiRdf|y?JMNR%Oyr znc-EUJk-pesM}|ud3t!32FrEE8K^DZ!MFG^T5S9EemgKF=H{A4*RRWxY&|a9Zzt*2 zmNK`YXDj%0RCIa#?$`F6)N?~?!|J2ZBmFz1&qkvM2Z0Swa9XNj-fN%%g=us#W=LmP zc~Tgxq!ZekVCE55^we@~m<#Niz9ePNZ=x6J!xxBpKAe+4-=#iX*hHV27mIPpz^rG2 zOp1aNF$0)8ckZTJ&D#-T1uP4W-YZtD08&nmm0?D&cBrCnkEwzByNAj5_0&4qa2+3f zQ~qwHy{?PONBb^1X{6?qd3u%vxQo%xFtFy8D7r6ebKY=HCvz6UoEr1;kF;X#vKoOD zr|Oz@{O@aFfEBYq?+`6ciY+Ondfw5jt6>yLKnpTy3~x9#YmmCmuz9bW1V?C)_v!Gy zG2Q_j3M3dtl9~Zs_HqqCMVqjt=?LraRl1>+(^uvhfhlS0?Ck0PKbQ)ifDr{})UMA^ zY=_Elm$Uho!=X@DNo*e`sil?X#Tp5%K=+xX>q}W=3eqn#M&dwxTAbAPQqUGFi*+f; zNB-&3Mi5U47r=o_a zA8XAuGz3pEOu^0r_aiKv8Wybx!@Ze;@7A@2V+|tbz~Jyyh#1g*oCOYy&C=Wn!@gX2Pj?ZHjsIF7UT0Oqk@s;VfyQCfsXyOuXf{*VcK+duYuv}p32lV z|I;v$3)ZD6qDa_pJg+E{Rh%_Vhleq57G?LMpDVekn6vOXD}-mTJBv41rDNVKbrI`;DQeZZs#g;q)+fm6AKG_iNY^tFPx@(tb{N6fQ-2bYo5R?khzpuGZi;H zTVHz-bA|4wQ6x^&o?2*LFN!Xmr%!7))5Ur`GmC8%7Rg*Rs2dbIvTmAMk6~RG?HR4< z_uZ(IR>gI1wDOEs>}~bCnsvnbD5T18q|R5nKO;|ZZ5xR(k=phGz+|><)`k(!4=$If z{Y)uS`;WFz&(th}8A*6)QCrlay9RadZ4*~bl@up3_RDS4rozbgZiUsIC2cad6*b~a z%2ttCYYCjhCt1^Gw$@oqYlsX>v#$pj5^eQVn-9L4saS_=1}8#fak&&D!|77;Ju^#& zA@7&=kbPcr^OrfTHEV$I)F6CK7v6HSZv*m8rg=K}7Tg_ie0kcT8Y9}E3VacBF^a;! z%xt_gR~U#gl+VK208c<6J~BM-%xb=ZljnwXo0Iyl{-U4R-Kkj2yb69&hB{b>(5suQ zoMVugKT}^qz%Fawkul5Aq{TZ!p!=AlMdm>)tKq0YTh%IG2U^o+QW*(!NW(BC|KfhT z?aK89^=_?i{E~0lQ?6;t|JtVgCT`mA{;H;xf5kWLz5;=Z!l`v}-q#J0lV){eXInED zkd7|cpx%ZyAtyEU^fYxeHp{xHzOkdNp@U;E+u7FEn`IScyBavaQ}tbAy1Sbinj5;i zI_m4&da|9(OxbKlM|Ve71>3WY4c#3L9a;W}0&g0+yT>$ibY#07$@Xk>GjHk}p*S?v zTyOtrE^Q<@QCfvzci)2 z-ED~LzwSHW4r4dO9fvdgCj-Bz4N+#sXof!%ZHSj4GTJG!wR|!N#%U4$6t|tsO`(BZ z3q}Kgbff0Jbq~hiqt+t2mYPl4Qc3ua21FC!Wa`#?rmhv{9bC0XVH<)xP(Q1&QRP;M zfw9bZ$v4t$K$V>ecP0+C7^c{u*QPwzDzO$;=s_Flv2m|9MKCNe{49%Q`}_J2JO?CX z3Y3_6fn1RXnP&fr9hl$j@X?_t`_7YG(X5{RjR5lwZtl*m&u=)=smbpm&`4Z#=nVfJcldxX%eBjjzWH`u153 z`yOz>z6bEBSXzNWTWH3hpG6){*8qo%;ynmQm^s!18t44NXV`|ZE;nX$2jb8}rH7al z6FT6agAO_>Ce%b?wXEnzje+DLjbVB!J2#dum(0#}y6UxNh)iM3eodTo zlJ`bLbwT*fo4Rh6`~{ z(H~7js}C8S<8M&e{~3?|4~*}!W%u?RH*qw{Id0+#4HQQVVM+Y(kmC*`Knb4YkrRC6 z`0kwGSviT@J7kpUq2$j1HRjV5g#fDeR6;8YR1T*f@2-P~!;d(xHW z(pl{4GY>f6fV8m*4e2!e44yO@ZBE=z5!lRz(F%igydTa6_|-MoLLqL-;FWf+JmyNw zC!~CvP>;*cSEr;UcEOpeN>t5|Cg^3%(%H@bBeN={?bUDq?~K_`=LNKfrf~72MDrJp zCl29Ja&}3xZ`Mige}&+C7SZw{Am0I-zA&xxQU|l3r!5?NbTCVg?)KTS-|?S67Y?28 zDly+t2c#+c`JP3K?5F;q-PyFXCHaYN&#Am0OlW20t%ww?81UOnqM+a{#%l8|H44x7 z&uCKrix5`1$HPb8jRWo%9>3yH3*g`uf8Rhw9^`_}LThSuJV|gY(~26ei61x0u6=!qgP&SJ6?b8MPCn|2d;R!k}n5Git!$w~?~8 z<#gNDk;m8-b$10+*I#4QYg6e4J_X(wD#HG9mfE#BRIj?4cfm!#5?{lk$X zoG(|KOL~K(5{|ioyP|(g72et+eIZ8N*w^>w`~4^kJo4|hA?M)&?k z`~E2RzTkj(N4&dW7?`yj!{yx7?Dgvi!IB%ZTd;rh`$y>K6wK5`({&~9k4px_WeOkk zK!Fm~rW$F9If51z^7 zh$qnY5b>@=I}q<`4MNq1GV?e#M3?4p(ZSy5EgB~6v%$dCIKiS zD0<-_OX|gfMp{{kYS5MjMH678l)0C(Y3!?5h-&(St-Ya z#PS2<8SyK9L&*!l*h;qvl>c{}^mru8{hUH17_982aq@$V(jPr|5PUCnGCkSR(lOHv z(aJ71cM$}gj7t~U-dRXclN4UmFC$>BlqboNx_a>;aesC}gn?W-OXK(92GXoGb{HkFEJHA<;1G34?}Op8J_j#?q0K)qzn;T7Z#RbZADJuPE-;xN z$dYgImXyQyMpxy8?dGzaBV&V6buuof&$IK~AI}bp?^$uOL`=6xCY(EUdConDN*}?S%H#k%Qb?kJWp0buQ|^~-4MrlGphqcAb3Y)cE><-|Z^b_GprSw8 zWms5LLQG!JwJ^COq8=6w<6r~8>jG|$;~sE<3&PS@LTq)An-;Gn&(Mk=SUIWrIKICN z!od?vqasS|-s{fG^%J9W3HlN&VfYya5eDeA-86L`p&gsch_><7KBrUvZyXLE{Ymss zMb6R9haG$LAx9PrxbJQM81^}WCH9G&6OLWnop2m>hc+J>DUtNBWffA+i33G1Jca^WKZ5^z4v~c^=@H>qjl*j#&!j9>x6hXClx3@XMjy=kOyX4+fdvUTVejRtKne zu+XvSj-OUn{u7n zM!fy{N%0<5L-6tOFDw$Oh2#O!|Dek$wGtYo=9d{^3sNQws)mPY z!I5=zcD#IF&cE`6Wxspt)d5fgBefa*q=UlH@znl7#?kdbzg%z%iE z)E111zNFx&+~g=M@)VxTr*NaEaDPH!>@-i|W=EmjQ+O($!VR9n0||xo35B0K3YT~a zPv=uu<|(XBD6~!YZTO|5aIvSbA)mrhPvOCY!am1)3cqp`F7y-)?lz^3 z358o7h4Vdy=kqDBMq>)BQAc6%4Bv)x!6TK`0m5dixMGj41W&@k8w3z!K{xr&N(xE4W~N_zx5P;B?@Cag-1Mvs}c(P%<>e@bQFH$DcqXhhP9r; z)d_`y7EhtgQMlVvxFf#}4|@vNB@~t?6teyG)A}U+O*ayc`WmjxZ{y3phJ{HDi(7pg z-*yz%c?v(vZ{v%e!Z`_rgJ*jRe|8kscna6#Q}~moa9%=T)ErOYJx77=h)0|5$Q`Ta zJcSDq3hNUJ8_PAHCdrODnNI~L_!=H{HN5F-I8`-#!q0}&{EU4gnX$_g3XeGouX_ro z=TkVvQ+O+(FemHV@PwnV$x}EppTbF=0=TbkPVY0}5bZaioC z++OrA?o2$}_rps`PtG~Tx8VI!hBER z)r3O9X`aGbM}g0cDe$?u6wdS%HYF66ClppX3QYf)0@FX2H{hU(&jJoAH>)R~?%S}+ zQDFMV6qx?G6ga423LI39!l*Mmh5H=^rhiO<>7Pr1gDR%LLFFi{PAIH)6qx=o1*U&4 zg>yZH7ZM6{&h%}#+fiuq6gG-NG9NDY6dp?`>~of? z&5uTD!+g2&B(1k^xlu{l>6|`!^^Hx^F12Z%;yy{rW%()Nlav#O#H{}Bj&(P-TL@NQ z?NftuFPUhW_J@zHw@m-C!M+n$CnPR*B!HrgFMao05_n-6?(<>c`JtK1jkO++3LH@% z<}KcGggzN+soyFDKudpRLcib~-&H@y`;vR_h^nvxTh;~mWbX$i`r%x!kGH0?D8KPM z9~|9h+mdn*Y)!dQ=lc4dpVpgfb4JXt$obT#ioXLXWiL0LS|cl|_4V_m-Q!9gCW&s2Tl zD9{!eb?N)hQSb|7BNb*{Yc`hL1RvwoIXSW$W3EWG5%|SMmkd5ub?X%KRB_LOb}%-~ zS@?4Xrnk7q4)2Pe2xdj~-8Q7t87355pu>giNL%C0I05Rr<8Ow z<#+z#q|Be`w9RNPPtxD=ooZ>$ohn1>lzFeXFU_$R`*z9+cr(6le3Jfxr^)x`w`_Tm z>%}D3aB}4i`q-cPx?W0h)%BsZILY6smK%^ zRt-}`jEPri3}~aWEDFqu<5{xMBV(btC4<2(2vYc`FV+KiM8OXbN;yvENsGxSl2;4@ zn84h99E;nQyk=0RLk?Wu?wdP*u)YN^TRge0l?Bn7blN5GZoP z&En%Ei&CGh9Ms{+E?(rx-jGA~6i0Rr!aHq21CxDU*zK0=iwW6-7kjb~MP%^>N4#a; z12Q*_i|@$_=R(__vOz?AS1lrZLV96)SRH)3BHSft#Qk?p``N)hWw!8%$05z6cwmbU zH%|!A)K)!zaFhF@v6uO(=QD7_@l=TR+x`yDPGPPS( zs;26TZ?+J)X>*s}u;7LW&XWM7@>{hT>a-Ax%zdSu-hbe7{|UF34ojEn^t)Wa^+X|a z7eo-uW(W(tkjH85**3{kIerKc-L4SPse_be6O{@!ezC(yRFM6UN9w^VK`zE z<_P+RuxK&~bb+0bsOHCc=BJA?E}y;JDB}lTha#hdtkTwQtZS*PxfCtNT8D zgD&Z(L2LiF4VrVEZ_w*rqTJiw-P`Nr$s3a#Z{+02J7^E?@KoL$D9`F7&s#oEp96C2 z5>MxCp9d}`fB#;bF>T~p$FZGoEJW!rFNuGCe^5lMdlHBXj`6;WDl)3%4ZJ!%_z4LN%xhmgMN^KS%jZ{wp^0pJQ^8vQzc0FAxb z*Yljulh@X^B**jp=h&F!_*4Hm#{b;c@_12OFFcKx`p+>r$?^yO*E?4nGN=gyFFH$t}gCe)YU$v8-{EzVIJwcKN}SDnCBoasR5)Dd`@}CAE9fR zj?cTM>iiai{CF`Zq#^#hKOaZvAb}1xu{tKnl`*G)9-?#f?imCHx2GtF7P)lPzc7;n zll+BdQ`i)iOrSuadDQ?ba;`0N-Hfa))5s>=qjQ&;ph_lyg5nQ@7=tgKSNlCZrVLUL|G&_&vD9 zY8tKrd20|;?-m>d(rCudEC$4w_ z#IxfjPvZHwCM|z`>AG25EX*CiY%2frM2@aV!3&1} zYz%k(7PbicL#Vcbo7tMseB~(`yrLC^pbGGCD!xCTG|-)c?HT%W0r`v+^-qv&ywSn zyTkO9;FCNZ8t!UV@udX71K5SVYeF!#ns6IH;r{IKFoug= zILYiy+hsfMP1~_^cXoNolUUM|kYi_vO`NkcB%3oAREZp8_*@58lDw6mvLy!Y5cW2H zqiYVIuFkf4*kg;xCTTt$QIl!_@GaY;;mtNT5ERLwvem@0+3wA{NWx&vkltA@sIT(g|EYC zVEdLebTou88AyD1Gmg^lV`?xI((|u#fcGuvG4dblDMUgHry8G;V1`ERbS* zHW?F858Z$?H4+z)r7@{WY}UW_$7cPLMcFGpv;PL5T3hJckDj~T9*!Tb>$WkM<&Z}P z>|e`fIqoADWJ4UsKtMc>Z(^A=TQ3P7avdA^nfMrb_X*xknle#fUvMumcg8#Kz`(O- ziFkHg7muEw;@RPk@Mx=qW8yKF8Iw~(!p*@6I;M?$IgFh^z>b&{7f+Zp@_QzY0Kk0; ztuS~y%v{Q4H5Y)4x&UOgY@$@KhTwl@A@zV$Z0jvDi!j0fnOztU;@Wd@nGFE5c!$a=-GBtX>LbKV<$JtUK|H>Qv%p+i>z&2j%^>Q>ZHd^*58D!ys<#7IFX1K} zu$c!kJ3OZSfZxt&2lb;f>$@6cRMCyA1_O0Ni>6~yI2w@ZtOP1?h@FYZe=^2~*f~kP zSpw`LlK9g2+~uHFGv!=T=12x*T`(KVt-F*ya}E^f?%y8eKFLBlCAj{+wA z!6VYWRU;yb8WB~k^|qcdd4R7;B0Tl_sR*LBg`KbpbQ-NuIcJ(a* zRu}FZ?1Up6P(--0?M!$~`WG}DMqjlG_b0Evb8v}rSEklPe{0i57g;j9J6W4rIWMHL zfb!HJ#s=-itZJod`)&6b#+Nr->Wjm@4e%Z|A!C-tivjzJZm)@aI>Gun?D`k`kHCcZ{)@}Cf=z9`%ZfUUcPNL2G^8^G{`remdE4JL9 z#TXC$*7^#ng%i5_Z63|`P+@sXl1;Y185PgkEe?7u3qNY)YyCFTp5KjOfDvkR;+zpE zJHey0g8h8ZLG$obSQqwAF<{c0=1r}a;#y7^U}}BelhTB|bec66j17)7k<##(G0C40 z_s_xX6ujfNzvlrmqw`A4PiL!T{f%$Zf!yXTzo8i+e&YjnwVWS{%3cup1b`q{_m7XY?gybXtzc%s(8Gy{;pG*lWWfODP8l$Ecu{Dn(iW7FXXj51- zqQ40xQLGBy_-FrtrvTh%Kk&Vlh7b6>h?#O9*Zv`Ahx;_~s4Fd-W(bHm#x-TxyS^z8 zbJP@;1UuL9Rk9}!OP(=nb%>j*11u2=3>RZ&#I9G2$}O+c{-Yf8j*M^Od!TTunY3Ozba0iBIm8%iPmSO6p(M;&uUnH4Kg zIy1!z6w~Wt1!|h)9=L=99AYD}TZYNzjU(l~X8XKP-XbF9HRm%_DRUA0zgni~c&h9? zS=qsmUz;Jt#{8?kLQ)j>Lzi58SZF^`K4SRm2#IJ@#$G>-@sS{&i}uOd646%0S@xHH*JDpV$H|A^Xril)Zn>;L_64(gs=KB}+;khQaq&_I;-;G| zSNni0$z!gTyRStV)C9eq)ev0~6li*Rs+$V}qI6lh+j%xjV3~X>kw{>36*8+|Hdo#7 z>lPiVn(a%8SK2QpOBL^Hv8;@U4;Aq;(+k#R{?f>aLB2lp(|f+#{oe4qpRRp57s&xm z$Lv<_e%jdEOhq(-e;#NPHReTtQMDs~$pSIGi=b(9G@ZYZ(C?!sJD^~#U|YlT{2l0! zW0{?WCUfFD2033Emm`8hY9=Y!@=P={O_xRKCK#Kk&N{WmYL|Eh(4)ZNmNwax2>$Oc zP~T>gXB&%doy7PbdCvY{Gqzxjdhv1u;Fn`Wz+Co{XXM9T`YzdW^N)`@Z0aF`HbQpp zQ)0ZH_3;wHQWC_11pR)aTRPT%gp?I-QI!|QE3Q&&YCi`?J<7fQgnb>RCb|M+UCP*~ zK+I8(cdx&o*Wb=5aIi~RZr5~f(qxyk!X|x32>dQ&u1k8zm0IGG7Q3WBMoG?DWtn^b zynS!}wHx!Twh4r{5TBPZ-38rNWok<5tz**ubZt=Z`ae&dy~f7@B97bZuK@Sk-Yd9Z z@*xusI}V)^0~+g;bmBP8mDZ8B_=%63@}g++*9!f7gl0(*mx5iHEC+_AlbAAIN?&bIdqcZj_BEeF!Jp5^SP!A zX!f1TpB&CufyZ1M1%7&K1D-wS#j~T?N9RvUp3a}&-gbfK`1IlF!@w|3I^;On9+3M` zbt#V}6jZmxSd8j6{ruuV%_PR182wOpPd((IBY%9HlK7=d)MB?mziPd}O>txx_Vys{ z&Z8;6+3q~x1mEUY*X_Q+Zqqx~M_<|YFlh3-^)@~w;=^gc$-8Y0nsPP-Yu4m91Qwj% zrg%=wod(*$@0?M(F`IGg5NeZZ{b1BFo}k1Z5=5ENTf9 zgk{f$nH$0#uSZlHn1_l{Q6|PdTP1xi*-ZM3xVJw8o1)jYAbl={*R_Tl%Orgc5ht%p zqZ2aM;I$}6t?uF;N)JJBYDQa$uS}@^GDC{`0cE^67fOb^$|@O%fY^drvV9}Nk5X7| z3_o}yxo#6$m`DeKstc16!Jb$FZ`}tAc!$@>j-kxFsz^l=>`Jkfs|7)h8w+;VDn7en zVGJzTMORQ@9+p`PVRwnCO~2flR%B+Z>Wi^fdT6F}7e&18LT@~JUN~rx?XIg8;+?O{ zm}js8C|p1mbPm&bX^!gQ?Qw|e&0xIMFmU)jccVQz!m`&PJ4pVS1W;)qu&$N->r>t# zs0wBseLsYD%bn}jsH;T%@#(#kylS?h6Axz4^Xs7L1Y8a}$XW~za7Q@gwJ`m*t?UK` z&bE~mnH5AhE;YMI(}1U*X|UPPn}tc1Kq%7a9`7%d!Xh-ko}RU%_HGMTDm!YCB2Ahj zjR1CWNb8mJ-WC~Ll_`KY23`5G^L5ZHG53VJgzaiBwU$I$0?!6gz<7H%p;ZI~0E?{M zYw9i|;Bi*Az=9nE{M=i-vqsWL28Evl!!^E#zOl&sa<(k~Giz*|QXAU3_N$XX1R}?n9-4h89vK;o6ZHJ=ov3j^8QS4?+r_^r zCT2U6#~JrU7Ha1jv`rsmX4r{NsUOW(zk5$Gw34>>ult%}CuOqxXyl*4W=|UFXQmQv zjNRzWboBfEnbKSmQN9>Mi=-wj=uS3jdZGDwP#=ajQ`neaOr=|bp*8<`_JRpOx8O*a zGG#K9D<+b34^(76rR__2*LM|k;jvOXn36^2CY21s+sIpLdco(mGb!75_hafOyI(XE zk?~?*i!l|yxkU|>POIKdz&o*9<-OnPMpWA=b{^rI>=dIRp}AUiS9XT}LXO!3=(uXx z0=0|0!AuFJYX^^Z5gNRu?a-=g;|)=>`>k2BW7776AS6#2oTKsDr`AvB4x|>?b5$Yp zVTyHVO73XxDTt=fx+0xv|2vK9=JSdf+OI}+YtJ+^@#acCQ{(zbk>)^jrU@%+OmwF8 z=8S0{t>tDX8d}SEM5FU8(o#0XTFR|B#598H`j6%i^Kj53{a8J11-Xm%L(TfU3}D!L zw5y_@?z?toG4TR(`qaj`dH-1~(@3yIW$U(Vfv zd`^&}B6)I({slV9=A=auig@`D%hBC#$m0b&dUk>xp548rk6weS1rq242&DgEL|am2DrPO}y^6EJ^Nf z;j`4;?!%GpH!q%ZH!tlnITBR6>G)kI7L$(ODO!v&@zeEYSg=P5P>{K-{2eMxX*Mi6 z8!V46S77x2$)T*pTJZSA99FV$CRj^Mv#Tr+Gr|7c`;P)iWbV?AU>8Lswg2p1X*U82 z2z+Mu3jIC2A{b@=cF!zzP;S#3g1u4r1v{Y>T%bo?o8nTr`8}P8+CYC(OO!Jz`)`1a zR0R6lw#27KZRkt04rmkG{u)7@L?0CWvtwlkZQO+pnG%L;g!hWEa!vd$y@Iq)2>pbx zKtz${OM*S4j`)}A4*HOC6xyjqg{m#tPtvJGy>evJ$1u5mdLrs?alu-1S5umPKh~t zW~m@&(!suj@BML@z9FWwV?;+WeMik2?+ycLZgso*^$m{BS9}ENk2Sn&FPq%PeAY7G9)kzT) zk16S9epM^Xa<%=E9r2If!#1kVldW_|V8-^?(i3xEQKDC<;dz;lt)s*;i>QwW_=4mp5$xt zdt?N7AEw5brJ9bsc|Z`L^w90C-HdCN{&2m*3XYmdcLclnukue(7p2!lnUZeli9b?4 zi~x(UGni{r!FN;P?&dBWhG++u?G*Q644idqy^_}au@x%3xjb&o^R#9Jt@(IGUW))- zvK^P-BH!biwa7Jk^A^2Z5FC^uzSzsIMGI|<&cG|mnAZx%m@|<$rFe?iw<~5#)L~3# ztK^|Q7PJXKriw~K1TvPo9$C)ss8ijl*E{#8jtA@*qg1SoZ5g(UIkV$O?B^wGG`ENX zt^xLSRZvj{N|iE0Aa^?vJ@;mK{&U5qO{T2(_1D0`>NRb%d(EO$ulanm!OR>YwcnlF z=;pgUE~DF^> zIgDzge<&WaFa%=UY>-WC3{5lVyD_nK_9~7;EgB+;?{C7Z3N?&gH_kfn#m!+7K4bS9_LF3CbB%sn1hI=WP72bij>F z&0WIpZ^;KNFkcbVWn(Ynd^e76D>kpDz2n{b#<`iQ90pGrraz8LLBNE~PM zlIrp2((25Cdi()=WrvuHXS)u6N$QZJm#z$po{PS9qMVk~W<=cIrS$hyE_P-{)W7F} zbs*aB6?M^eW9CkeJYrYmx>Ih}*Xo8s^XCFkK+UJy>_ENHit^B0oa$Wi8V!0KCoXlx z{1lu_pT4++zKC^f1wV-UVx<*xbMPbQzT2wlj+1(HP~$iNEyK(tOYE*zsTYk|mTH`0 z%Cg>i(-Aq%-eReqJol0}Mb6GC#9IMe(>&D*fLkPkTI~uf#Iude=z^^0Qp&7Ob#`_% za@U~1GV5)LLLE;XhPp))u`7U0r0Zx}u`a#_k4k(iO`90FId~0so3m2#A{|<}A|U$5 z%dI_11FP!eU+M1CH_WF1xo`5SQqs*YMW{(s^=xfI7F+d6O? z2se_ng!|y;EeYzX{++ARl=)wU$>-8$*1Q(iQU!`9+~<8^>f0_FMcYAc4$=dIgdY0~ zMVf`@Vnqo8_#C*v;{bD&y0_5$u0F* z7`%=`bG&d=Q|871)YN>MJ%WVQIIfr_8s+Ba*b&5JPoT3?c-jLuYpx9vU|1Sw*U>L5 z=z-SYC9}G01xo-kh!B#&QO6mrS>~cFjB3T zX`4A%HWL@&VxEsXBdzg^G+tZfIR@?QJG}S>D@iud&&`rkZZ66Uy>VXwcT`hOOT%Dx z1T%wo8l+{vjIP(h%rKz>b3FR=2xf*KpXI)=g2j4aiuQ-Q&I^mqvRmb^7*MOE(+2VG ztTy}7Y261SK!h=ok1SM01s|R*OB!l3lCCC$Njb#RliryCCXZS`stPDXu$xzRQysuDGR^6 za6cEvVti9PVyBy7VR^~@`I9Ewnsvh9IWFT<3~fmXZmo8xuD263Fb^_8P4F4wr!2L5 z#fBEm!?bDFN@rAF*IMU;=ds$^gP1n<^Tl0wgfnrz(*3$cd%G1)9_J!ae0Hfh0|S-J+#iJIwncldTr_U)Fm;lh9KEx;+|Vrkaa_~0P?U!bB7fSsa6*_m z&28wbU7nY1o^j-vES82`rmT69Tx6usMCw24@R-`0wtWXMtaTN(AHJXiQ4>uTMKUUg zKu>HlzrjRa!OM(UgFlRv^Js7M@V>!1~RZec6bnIb={Ain7 z(f&Q?N&hiyTR+Igbej+Dmi58bz*SHJKlAXZl0@;i1ylap&^`H367F6 z6TdM@_-(|-Bbqg)PKRiUV@P1dP220K+39Ho$Eg9#X>rqW@b?)iZ3YW`@tiz{;6UE6QS`kyu*@sJhLsdpD> zaJR`c?7v^P;+y)NW9ys!9D6&GEL|`gHqkA?Hr>-kq;xMlC`_+qW=-R&X@kx+*=eTG zGn!^U)ieY7%k2I^(~K*}N3klC?#Nx0+jeJ2rde(qwqi&=%+9clr`cb<+x0um3TF6e zb``R;3PLkh^j$Wv>sS{AhXuoG7ffi>W~+6zY<6>JynAMuakz)Y647M|$=iVaB5#Q~ zZ??`KyF-Iw&PCjkxN0`%T6_zc8MVW#PwV#7)<`^>Y9vXv)Gafc!`d@#`|nFP+AGeN zINXDR3$~?Y(MQjU4yb(C=KSDxvc%lpI=9G;%4%40eEr3b9(Blc%eYCA;*1XAWfOif zR;lFXOI#>P6uZ~vCbdMfnUaOvjfYl*_2!t4kk~#c$Na5#*}%f6FENAK@>0uZ#VccN zbPNn>0Q~ePqufP*luLfm))Be%4;H3?`7-|dm^r6{PwChI`nWZM8u*FCV`WVt(XcTp zs9MagI?Ta8L-P);p0Y1&;*17c^h7W;C7_)r(7rHp*fjh0oq`BS`78^FQHa{)!0&~@ zK5#hSXv0EgkJ`m9ZpNtAxaoHwLxPFyOa&LmYpQ(Cmz&-yGF! ztN7V$psT5RhA#rZBUTSKkSCPnib`f^ia4z0o2wtk9embeV4jUr3q5(AKXBi@F<>o{m||*)kfZgM-LuH(h zHr*}37b6N?{6hZt9sN2j;FGaBleoxS*Me-;ln?_L*vO8vQNrP z2|mWR)-ti7bxTX|^@y@)?Iv2afMQTOJ<6b3az z2_|koXcs={KT`FLZS+}hcT-zKdp*&+pjk`bci1d-Wcg3~%~iasx8uCsO(S|)J2E=`19SC#$k`z;LvI|q8(u| zu6I1dMweqw6aenWnC}dQUK8IRn*V{?n3}n89Yec)DEA9vnrB2Mj;#)=hVoelt3(j& OQL}>VI}n17Cj5Wy!xmWp literal 0 HcmV?d00001 diff --git a/crux-mir/lib/gimli/fixtures/self/debug_loc b/crux-mir/lib/gimli/fixtures/self/debug_loc new file mode 100644 index 0000000000000000000000000000000000000000..3fcdb32ba360132f00a979bdfc6f4dc96fdad96d GIT binary patch literal 283588 zcmeEvcYqwl(fytz5J@0FS@3a0;zo~v*Rb5rx(=)qw*Eg5^TODmnu1t8k zh${27B=|72ExF2_fZuo#_|o(RS`}^gTxo0%UTi4B`q&=4pvmsJ`q&m4-3S&UVGmx= zlb9L$5(}YUn!Z3Qq3xb4jP1b-I?z2=^WK~-%$ntmV`^_4Q+wltP$f-YNCryyiqZ#= zfkL-;#1NMkx^$o@&XTGE6r>b|?sQJ^XK>?lwB2+0u{}c71$36h=X^+E*U-k4eKF?G zXj^inQDC!LoG4VxIC)BIBMP;3%k|C?`(aA+Xy`c+tvPj)4hN|v*B+WDg^RRyfERkvEr54=5tK`F575;4 z{5G_FgSI8ta9X68)9M;;rX?+zvYClwq+GHHz==O1Y3NJ=8p&i$UDA?hHuBrKQP)Eu zE43x3Jm$Q6itSoBZ%t8JIGU|tsCTzS??PzZt(@>=5fyuMPgv`rZON${PpBxqZ&TG` zP!$O=BR?H>4%YVOX>6}^qy7LJ-j-Ukn}D*rxTE6Qfv%xKQCjvv*HLKAmCsJLEv?r5cmseEp`PHZ*pzHMAUykW3wAGPHal0WSO9C_yyvlu(K<1HR`) zRP=)vg^D=pb)oRoEX;G!HLfw`8gx-xORi0Z15AOI2gJ}cttF4Ds-f+kE068Li%Uf4 zo~Z7`#oK9N7@!-`c@tXW9=0R84iJhej6Kjb<@O)I$VWwa@dZHoZMZZNlta;W&lUBf z&`IcIYwpVIG(D$N@uO@QxeK74Bb}SZWWrHQ3aQZp5HZae6J|`_gi%K!^kGu_)>UBB zs5{1l{!9kt2T@!mTmcW4ExFnNz)ifUk)rOP#3!AQG(S2Q6FthIMbNdpP}HG{4mLG4 z6UNUbddh16uH!|agPbU-QF@4*44VrYIX{=4jUe(NQ}dugQX^x;O@=KE&8hr^)5wFe zGg_B;!bPS_$St{QqftXKDo!CUo)=-;*dDxC5X!pchHpx|1!UcGj<_{09!Lv$@q`H7 z+bCX46rmd)dbm3z=b&|QCLAZCGU4SSQlalMMSH`>(P-nC@q&&-i6y2}G4V!>`a(v# zS311tjYzoQ4kBju;=PO)Y|SM{2mgoNFs`Fu-{;O{MCZ;4!;wrETen$+6e>SVCTZvi zLuQoc$r#T_8kBcHnE|0QHavDQXoi*y6ejLonUnD@Zf$rmxUn~{q3iv|l-%@5Kxaqm z=0v!zh>C<;Lgb1jPdwVw2!Bn{h`aYp47o)`_y|OiHsWW6mj4`Zt{G$`;XsI@kzqj` ziDAcyCiRMKUiil;#4oHvJ^euXUMADKg;*vPoeVe~?^JK~yuBop3`A>`&TZ)WCtCB= z@i01{Xsm}xI0}?#dmuhuhN-*xSah8!l=3*hzk5*_B(h13_oCk(b~z|ldd-sn-}R!< z82#Yr$QW@-Q#g$ISqKHuJy#vugO#)xDk&C*Bl}$BR7eg4BG?7 z_VIQ57eJP@`_v$GaeYuWL2DjEOXy^4l4>CR39U0`xD?tao78Bs#HpEt!!pdwpfC<> zjkkk;h4+7xS*SDMh3L9QC@QK~YRd9Rd}@`4lgd4gVW~|lhwUrNHuq-TcC#| zx2^VrhxF8mLhCF)Qlo4UHyQRajJ<)@DEf9R+Z|>3R=x*&2KZ%EFSmG4QTq}p)E2o+ z?1!lg4uc=u4UJ&XRZmfei(l0gYP~TVu5#l@Lbp6c-K1uQ7HW%TMeK(uf40Z4T_vPt z!c#?5JO}V%FA57`Pju}s6y@Sk=<n$#=>!PCl|M% zjfK4EgP6H2UN)K+S0QRKlNZm5&@p+zxW>#dvav9XZ7ggi7)L?(;4d9=o;UodJ3q2$9!8j3?o zPt2efTJw<3&Uu=V`k`|rw8k-+T^kEUxlGEcLTL?Mz8qcLY>e2Ph?ty@wp(uh+~>gD z3?o_i?_i|*ShNx6W8j}svT)qz*fzUiQ;fz{fOksC!f`!kNhI^4jmGT`d_YPTjvE1c zN7Jqw_Y&}XDOot~b8KqeP&<*m4q)DABU$+G8Qi}9$8C%bUI<6y4f+#Yn0Ks4WJ7j0 zyw7m)KW<}FeskIz(~Yh3{{E@loSY#pPVR{@llP$oWeal5&fjv*#CWB zH{FcRpu}|9(L#rvi0;zEXe2T3iH;WLZ4m^AMXThrb8CB@cV=ON^>O@@EsY79?=mRK7 zDGDi_Q@k(^0p4~YRNNnw0dj~iX39?Li31mE`WYwb>5&KH)AJT|^%08lXL00v8My%# zHwN6+i$dEODC@%}%f;QzSPv`#Ct4Jz&(+a7(FTA!deJd}XL?asF>YY!WEK9W9$Z)_ zsi%){)cx?Ae(hX3}1M^8y)s|=4fSN$9w%qmCJ>N0>EcoBoh zb!5EkKLTcG!@1N#OpTG7#8esah?`Akk_uJ+xnOm>1=%PFm+Bxe_HHipX;nYtP|=i5 zFEthQiu_DCzvYs6eb zQFe*BVj>=KDkPx?xm}?D474tl8Xbejeogq*L3-exW?z z8s?~Y4f7&pPKD(}#DE!cnYk#qdl|_(A~+btJ9>CDh=n{Hn1`hKu&2b#{~nUM^pJEx zkfP{#0a17V{^|h0)GIB0PRsJ1sL4^$n`_?z=4jTH7U4v3s-5r+5tX%5?HLY=lQ3aU z+&z424+;;c7^sJ{o1o|cw9aon0QvWDC7!xhSTI>;#=TzQuZn(LQ9JAaHA>y(HU%Xw ziq#BgldTblWigrtjY;-b6y?hqpxi6IRP-9kvaqRut>Ml^+E$o70A-;I#=K zgN=jpPoVUWw}EPX_e7W1qA0IH3k@1GcvDf-A)dFx)Q1Cnisd7}+o<_m%nijjb`Hlu zk}rUbLTUpgWh+ce1}&X)Z$mty`$?$^t$;EVtuce0l@_(2)&YeEjkF9_6y?kL2-QV$ zJRfo&AP+7rxglZnGEgQ$3ttPxUqsQ4)dFUnqv-R)Mi?hdmnZZ^Cf8aJJ8 zIA;0dmQ&5+){i(;#!m313tv&2-<#=Tb-e@c+ypLzPglu-S7nUulNvo55H}fS`X1)> zUYLDG%&U7yCOtWOT0E8|-^3}+u7&M=pggG5;`limi0Np;CwLQP+lz(28~UzU}vD0@ONS9+8W;ufEy z95Gi(#3N2cEu33!4D?@%)+K2Yy!l+bs!%&C!ARwlI+}7yrIJpK^-g8*fb8=yDk^7U zKRp%U43t|C3FgaXcYnnG2(-rP-2yM94n}KG_|1i3xOGJ?6Z37I8^*`*RLoQ&55Ok= zl5FPn0m^|oUrNsagYJ`ezTxM)IVa*fL+?VJF7R+?d}Mz#EYoaMg2XfuO^}!>BOY19wFZou3#Y#`Hh8|FHmaFlq2YELC6<_9Jc$Z`m^X%qN1TdU=t1s`K8fgQ zv~CwVLU~o8_FvIu>{V0hXv!&-N;<_a4LNi4z<;Br63&Hee#IyX3-MzzJ^5~qpNo2I z??6;zOy_GGU*-oO_9Ikci5J*Ndn`c$0|Agg=R>O4w5hyuey! z?Zlq(g0o00DwD%#j@$cN? z#f8`e;-8TTms%_wv}7*+$K@0Y=`-ss7KRlU3payn%zZ3OH2VK%fEVBNO=Rwf&lvmt zqW02TI04r(NxezoEv~yP}aEb z;M0zXdHh2>y3UQCRqg!9>0zsv&okiy;>PfCqj}NS^TO26i7ry3QwwpEVf3BD%^zJ_ z4!YM8^X1TdCOrl08Xaiedii6pkvLJPdIMP96s`FdvW!l)CMlDX6=-4|IFAg!Lnpan zd5AIQ!CIoWC_}`lh=iImIh`Slr0n5?Pq>eW)T_$95sg2iHC|O-i*5cUIeFHf0-WeY z;j}VPet}KitKypsZ;p(ndFNRO>wPJ-?xwmS=E!lzsrqAhJDZ$BR4x7j$_zqL5t|EL z%b_(Zk)3Sa2TJ2G+lwXAt+xa6reb{T)HOc$g0Fplds?KJ)9M;;rX?+zvYCm*RI_c8 z%&%nGbfy4}WU{6%^R*SO$Igwq5eiwU#-!FlP4UswJzWy=!83lG-~}H&n?Hr}DKj3* z3qEJM&6kjeU*{N)x_n%81_*h&bejSX`g}5UEJ8jRIu_vwc?S4*J|yz-A)Y)h_;}|g zPnbF@o;*5CI8Ws{|2%*6XkP3pk406&eMD3wd{ac#PXKwqci%>3R(uMoOo5MgmGSnG ztn$H%r+KG~ljP&7@x@&}Z7d{7JHUtI#TVH@PgN$yVU_}S5S6G8-EZH?lA169+NaP_PIWd z_q*B9wU|)U{ca4xa2#6G)U9|2@1ITDLYHsGeE8XyyR(OsD?oWmrtW?&^;cZ*MrZyU z>KP!>h!OhBK%o&+4?C&HB%O_8%w=eeJK=fgx>hJEt^*OWihTZ|?nYaoYdfK+d)iQR z@k5aAxmNu{%j{(9cG8FOqTva7gOPs+(0|mT9O@dMIpbk6EoF$)BE_6m*LX86Y2mb) zD0AAGNYudAILL6~8#GQsh5|H_$(nlF*H*aFJ2&ceC}gD?lNyKD(l4NOyA0ty%Ozg8 zqt^J*yjXVmMCk6}ggc6;dNAN$MO3^1@L~~_36EbPWq=naL+BhKd@9Z3#fB@2BUQlj zy@>EG5mgCi9gwohi^U*x(;~bs&Ev&Ko=3RcN-2H37!IM+NBES8s)Qe|oEn-y|35SH zl4PdDSiaRip}A85!^W^U(Kz@>pPeZ5)rmHN?Mu-*(FDK;y~w+hAUbseoQiL5y!fjW zjQfz1zK)ROIbrB+{Ar&KxMPMM$4thJKF7k)Yx5JtbKx3W_2#rF<;_qA5l zwN_=ECrWKpSqA(0oQILT1rhvGBAE)>O;{|+T#)TK(Z<5VSOUKp-mo0v;q6HAsc2(i zrbyloe^X@{X4S62FaVr5TjQ1LGul=sv~TzgkZ;#;iW30KuK{+0+Vf!Sdr8S(vj-b$>p=DP^QR{yeY|RUyjlRA$44CH|>nmNa zpyEG78*v80+$LzPas9Cz)n@YrMP1!Z1#gG4gvkFPE-| zgB)k9uCEY-AJ7`7s_{5Y@%dru9wvAj^#+UjKvpyk|BMLFn(kcAV;gS?P83GvJ~;oj z90?2h38%nQqIasG%@so3&m*7SyrR4X%DZUIuNtwFdQ8$@Nb7-;-hAlKzUVqaD9WFO zam!gsZapeuJ}ktwEv@FeZrRdWg9;LT|37&DFHInl`7tWU>{^RK%RRpZx`0hc%|GkK!mu zkti+Kp|pIpDawPOa3*e#{TZU!{*Xk+xb?pY%2#NO#~nj)81`dO<|m;wzc9$ovzn2v zMCbKrjY9;x9ukUL<)qvsl-AJYSZGF3zmLfDQ(8yy4Q0RAK;3st{^RDX)rIzD`d z@zcY@i1SC_VJTUtdkow@+t&)=Z6Yc@0{H$K$#_=_zW}zcC8E6n-|(VF=FSk^B|>km zuFci8xw>_s+cPfdUG{xuBOe^v^lkWpN5%9Vy+wpAcwH^lt?Pd_6Q#h8msIj zjQ06ZT0RAZGjSA*yC6mZjPY)a#j)j8)9-5+GvTjK;z74t))2vs*gq+ah?I@QnAL(4V_D* zbdLc(F(nJ5-+S#uG61bhGvRh3s_g`Lu@_wd_|-az0{2KT^STKfwVsHM1KfL%h!Wuc zrw9E-cZtw@P}Ck2wFgD*K~Z~9^tKa35BiA*1w!vZReMm?9#pjlRqcT~7tw>I#e-Ht z??FX-P|+S#v zCNHk>Ji;qco-vOXH%iU66W*ET@#0?3BfNA_61iAk7{q;Ge~kE{_Jwii@+(PEhQq`W zXpNosJvinUpVIOpC`^Xi(pSRz+yJeicrJ7;BNVki8j8a>+<{epFIsa;XXoS1NR!a{ zK3Zc-XV*_cQ7)76kx$DN>D5BbZfX|4iOgMV;lr~=60iin-5k4cL_BR3N-ZC|m7kwdgLkZtX^LWvB zt7LG*1KdDF?dJf#fHs~hFMbtXjc~fHsjYo4!0*t;LwV7A8{rYoE20|V<|1k*+zp~w z8!tu(uXZHhC21ZnZt}c;0zQ}K@!|{5n+HpBezdM|2&*Eh5gsU__A>y_6H)C!z{k8O ziGu&1)w~gCUBC^5Rdz z%d-YqV=<2xD@hIJSx9W%D;mN}eSu6K&Wq2bkZOcXZES)UY+d;hUV|+%=K0Vj$cSY~ zF*b&f@*raTgv2^en6f_Rz;N7u*82UMB>p^y*4>iUK`9PJ8^??n`%0avo6hyv(QlVK zeQDa7>mXG39_iBO#YicAHL?F}hH|()%`CU6!gp9hOZy#o@^D_PDLb(G0N_76kykK* z=}~HNUGZWeslUR)SU~+{Ax&cw`Ok*&KWqIoZ1eY}?*Y8v#&4W!|Fd3aI5Qd!Ho2ir zl-jI?dz^8r(2B~uGkZnqnQ#znY%I3&hVBE=OB*isglydjnJ{&VJq80$5%tBffOO6c zBy8g6tJv>*;AkD4J@KRAytv4xg+4e5;>ZV~ z!F;IvGEnb+HlE5?gL2~+l0Jn|FQRpnPXVdW7)3`oI?5SwN>ezGIpGKWGM;-Ghi%1W zb?2m37H$09YhKKN-bmO&Cffx7hN3l6yO^T1aWvau0Q;kj?epTGlu+4!3Z1W^b#+5H zH^?s7TL5nBMc_;G6VR$cDRNZJi-Zo)WR$n1-b4(8!ZD(_O!$O|>Y*);8Yz?#;cX&P zwkDzLXQ7k{SL~LUa5*C!BBBc6Lfy$K6OMw&oz=#S=Ee2Gs}l}F3AyioZUeZth$@7K zi>OR^v54w~{}fRh;Y1Nt2!9e$nQ+eO=zpDXJrSu&WVn8iaFqxzm)H~uPxgVBJc<`T zOCZXeHRt74Cm|={3YL%_I~Jk$6pPmv1wH*%JS`9&C?3?c2X$>ul^S(_CAwP)z3!^k zUDdj)-s1!o|6)tOTfn6mZvu9TgW)810$)*@o)xF+21X(Xg`uIR$F&?%O-D?&o1Cwep+#S2c| z@o3O%SG&;8n8bZ(;|;!%3*32}>dsl)2v$;xLR+0v`~d>*XNnuUSN}CA-=Z~V%1-Jr zNkgDvC|cvYN_BMYBNTOOzZ$XN-tAIP$jv3*#{CQu#BuYxV{3T* zxuW2t=XGrhtGu~P!JUSM$s32e{S&fK-TjD>=Pkjp2su6DrxrTk7WXb|%Q*%d-VWkb z!;93tfqr_YUWwbgQ*73%#adsMr)|}hPi2Rk30ri-8D!Q%~irTuEM_(fqJx_d`Lu9Lhq+a8T~AYQw2irRGm)!B&<4}^19Wb^a!xMq9PS>wEN7B@Lch` zNXRKU7U83wMd&?f53|q;Hf>njf0LB2D8{m+Tr28i+#W{U`oMhmTv@-JGOTfyJ|HS3 z<5Wr%C4&|w^g(Dd9sB^^f7EOiN22p2w8mOL7G38HMIGT-mir2&5>^5!2MI+@d;@f` zHGE+~tR-61zLSA68KZmj zkP6+|zR{>(d^8A`6-6o&>^e*+YO1947mCtyAiCHZ$r`38Dp~ZQN!Up_N_0N00zOk_n6-DjAV_@PuwC*5GSehxxlKW9@{Y%dl>%n)s?0>`z6k8gWK^E;Gl<+w zP9DvRFNIg6_6oGeJYH-sCRLSw2o>JZR;uzLSKDAeC?wS-iZ=Wxl_=$$Z zN_u0oZrKp7Jqwpjh42&+RSB;ZQJHYFUL07h01ls3M8^UiHJgYM;Qyyn8;DaSLbmY| z9EpCaGS?-Qts%+^glCGVNVqem<5YbxiWj@d9P6B83nmuxc+o>9UgyMjPxE;3zUMI- z6D2_PSrC}5B|z$c!a1!V6KEwoM?`hPOT7=1M)6`bF<7P#_d?|6G-(ts=#483!hUcp z=JA3#a6F&L1ku~&#I-iUem>ljNAY4?A8tYkd#*dU_}?3X-%TOxo=}X=6t7 zf|+uE^O$gPpETzRE_~94gh^Wuv)LMcL?15`m%Xfon5rUP_^-*1QuIL!M=|7o5K4{E zk5aqo*GLpwW1Id&QPdsj3E1?B>Eu@S3=~a9Yt>j!gmXc(#w%L-!%BA)Lhqs~`Y9;h zH>JMSyI(GkM`Q}UTZlgp*{~d^n5qQzV)uGX(eHrD94QZgYQ8TVicapR@ixp0ZgDP1 zTqRSsy&eSZDMev%c24m?tUJH8t8?ropll}VwV=<{?4%wuMwen|7%!WKI_F=3uKR_e zPD0Ni7JlogGML<2+;NSLF!#o9CHfVL2Edm2z(8w1fb8xj+FDs#Q}-MeSV~ct!pyWv(dWT5nd%CwO_OAmd2F3(e;#2%7mNZW@Ee^;;6@EREcmp8C9PRa2YQOV@z9< zM3Fj=eTlAqqD9TK54xIA7FCqe77z^)#re>^VdxqmlrrJTA}XEj$Bf346_+kd2hGYre*mzUW#>C}qNB=4hyxqgIenCBnl* zq_R%Mn>VIxi>}v&q6&kQCWfBJ=-!QI@cD03dSbNKQd$n8wM4PXR32SJg;FN`SVWa? z0neS2F|Pa_a6lgstp}L*qNLCNKN5=0MB6PlI=4YEEIPY~{}u2~hboTI0`= zUWBgkVtxNmOKyA2%a|&iFf{)PjlD41ItTs;&L4%dpR&FT`p9u-oLwRE4yj7L4U}il z8ZQFwMOTwKEPxn$ii+{wn8I^tjd#CH{`S!3ZWDwPMN|wY&QVoR;?n~!I6nTRSG?Fq zn05Uv)RE|%(AeTxzfHMDQA)2s^sy*bD=IfvGP#A(x&RRN7g3RL>$wxQ`^E(!TQ`vK zCC?&c>sW**%_F)9zn(7NI&?+6+Lx-#5}UK0>|7gZp~AP=XaH4X(AqH6(J0_}tc zLF7v2ZWf<`8sXGf3^RpOIS~B>Ofx=09Lv4NFUXwU{0=N9A zI!7%j(NmM^+$cuNyQ36u0GrtsxDUHpx=fy*mkPV#4$XVT%0t%i3-Hu>okxx@>@2l@ zIZ`oRDn-5exePa>tI--CWLM@*_S!4`ee2{=yzt}N!yWl+tjl>swF=mZP`wjcceEmW z5F)ohoH>dYvx}-Sq1WCKI#7m+&qcdR>K*91TPSKj%s`m|ZVg83GT1`Av3EK2ugUCF zciuDAo%e18HYZLO2vg&}8yVVTLdMFtDM?65wRUMOsB9C}t%Rv@A1IePJC3RKQ&Fik zX3H%5no$AxcaC}+!RjF;Qp%9B43lV5SX@b6u&h$Cr%z2*TIT}$`_5!q^vS5~4AEua zzmoN**29nJ^2M(xMm)lTTobjrw$!ZJcBdsR7r{;5uH4s3!x{C!lNX<{rucE|R(ZBk zcUS6ON}lf2hBGg^Jf(ggL|(BvGu!~1o}v_I(6T7B{2o*9k4X%WsjG6^23_NXQVx?- zMA!C0QP1{=pz9i;s9lskG@)cLl!20AMhC;qk>YM!IO$19OK!oW$GuJMZzM%XuhoBKQqCBr78HQu#-fUb{) zQp`amTjPnEiu=kaRm7*Fiv|oW>`L1X=GhNgc0n6GY!L4bSsd4WaKWHiPbI^_p=*Xq zD}b^ATK5`sJ3yK>-i&4l$5YVqoL3x7HJ5?HlrZbY44y|}{37+M3iBK!c*+*yx){Zk z_8Y6Nf8$j*u2c3e!l+x&8ZRu~MpqNt@BTJrR(Re-%MyxG+6zh=Ad!P3h;F61rXv-xMIABQ;CBXkr-G__r5~0^!r|xY;cYOiqUJtE%rq$-^-cW)r zftg#;#=oBADO}KCETqX1izXvu;moklImW`!Y%B~7->_IBp$2#S;SAQ28C1hj3^Uy| zemGH6vQ0XJDaK`&z6se`KOK;vO6C0w1j>oRh!}2P8zjP|Pn)Z2b5aW4+*IeC^NSM~ znv8{JF@&=;^UjfTNOV+^jZlQi$w0YL1yrr$Z$aUF?R?Tx7JjyTQui;g;dVJfuKsA_ z4JX!gJ<`v{c?o@unVIr(o+{)l;iBpzWy#hi0RG@b;Us_e*o1DyVF(>F%X19Ez^XUi zn(Hi$=Qu=#J7;ZO;!Tw09K$VlBC>Q)A zCFM@%Y`w3FQk`?Z1m$bAW?0$D;xqi&H$UG8Sa^WWXD`LNhlZ>jhv z;V*#!Lkn%++L-bz49o*c^u$d(KV)$l`r^W$UwVgO&n*cKT{B#I5%&Hi6D#z@D4I3i zZ)6C^q0n+9TH^sf17!)+Z-1#{mGca~Xf3KNHirRU1Ip0qpfq_R%0Rgg4{6)&OM-wC*mtGa$_x34Q~Vw`Eiz zoNEROT^}S{beV97h*S_cY8|1J2=@_@+VMDw2I4RH^I{8(i-axVL^ed{E}~o&(8=h! zNGN5(OO{IX8Q);!sE1@!iSS&Ea?dV#;%h{cs`1O9;h#cL^CyKC+}bCkfwZynMN>m# zpTKCekr6K?e)n!6-rCD@#7B8q3-O~~7N#Ia?@t29c(R`Wcb4^PT8LMYVZ~zsy(U#) z_d%1VsDwWU3g>7%zf2E9o>B>?MMIvV46%!?v4VGoA;!_DG#aAY?yX1Y%j!ytipx`= zyouJmdnf#AX;yh{&Set#XS7DDsOWQ1RN|=JWt6JB9Bj;i*8Fi7Iy7afKZPTyQDJ;C zIJk?Nx!M%>2W2=~a}Mle1WnQ+(69_zgp7@@EaiGL^q@;tcy%phnA`_&6_X zA->wna>W1gvKHbo@XGq+VKMMeOpPNx9Wu8PJ_GdLtD2=IPf@FIMWsb8j)9dA| zNm0}mOhaCaGQ=*n#&UlUh8Ra#EYH>++jICGORQjJWmXk2uF#g3YD({O~!UdMdv_Ke6Q!loy#V6&0qe_k*CVY z*0DW!!P&>ZsG1kw3$r@5g%`7mjp4VC-VhgCriDA;g7I)4THZc749Ixs{u-^T2EvCi zK6<%HytD+VO1PPb3e3Q_UKHBf8%lpbYt+r67=4szsp^DOH$iT;ksPU44~*Z;uw|wr z6RAm*7~LpJ;beD*VNTkZ>|2OCeRPzaF^Z#&QFNMAqd-50;-4h3B|^IF3Szt}i0NV7 zJ5=ol5v$ACn&t*&eYD1{A_GMmQah=psI+%>=2ye_>tqh+aDB|1u#MhqT-mB+occLCWNhiL}o zbI!^7%QBfrj0!aw#yBxgQM`LG_> z${27UTH}+rp6FUoD23GlY1TbBgkEL{$7RrRrB@tk$w1i@<-NC*xq5nIiF_)C6y-6D zZhm(O-FvQ6qb6e1C*rZXxj%tz;YGA=Nqhr1y=*~h6nCknyZIoDuZo^B;YI_J3AW@q zihx&p5n*AagcTjGi1&xgjU+7n!PAImT#W{*%LA@0qK<6=@9`qShecE*d`mNMFgAK*q_M7WiRiiE8q>LA=jL{-AKMASj}k%)?f>#reB5dKL-MZ#l6)OJ4L z?IP+RJZeoJSK^Z)bNl%C={btJ9{@c(mtcgXrK z5uPuiI^jeSmAK@elqFy1^8S-7?>g7*Dzg0QgnlWPxRfstR*5U-Sy>+S&j9CI^*#Q+C*k&j7&7@6B-&IQi-a+!01b3R4pTp%2|a-zuHw+T01g`#%C zRuL5mcMwrKVSj9o?l5)PXkP3Ap}U_FJ})8(y^nK&aXv}nTwt7Ek~kL_=U=^uaptb# zJ}CSPwh6zPl=Ws(=4R4EHh==*93rX_atCqh3H@eL*O@IavrMNOs25PVQD(Q3%(fDq zB%%tFy`Lnz!eswhl3n=+f^>@ntHL>R3hv4OSIl*8DYr@^r+9abpImsc5PXQAhUi!y zco7TfS}iT)1y`ZlIR+84EaK5DjQvpdHY-vfr(YJ+8#&? zS#R575ywnc+=Xc&%kS>AkQcL{AR~{6XNN3Sxfw3XX(7v)#TAVv=B+9o%~I=*6vjf< z9dAvsFudEuLf#O#aYuHD-@%R@6M6B42y0_oc)^W4W;UvzlGJcJVTTl~agwSs`$r&u z?p(m!`}0th3FYy|v6#*$rTDAdM;KSfVmRlO;;%Ap{#IW7Cwl#TwaS?J+oRkI83?C7 z48um4>y|R-;#kzn*=XGhoOMtH8=`fhiSXbJ@v}%XJ>>?fK0M)$ah*iH5H2ydD>1~w zB*X=Vg;nNQ;Z*(xohjWRrORyXi7m>zqxQ`ap!@}G{Oy#`A$C%aNm>tK*i^!i5A|$- zE=JCHXuKMfXGDv7hUU$CW^AC*LoM+iWMeO~+_{5AG~!=Gdyz|kMIOvEsn248`<3OFqNS+QO z8u7ATJI{-JqY%-EdB$^v#>aEN%6f;h%RPrY(sHyr64Okio|rl#9&t1M->cZn$N6dnWc*Lp9hkoZ)g#J-zo#XVzAMfkb?vM9V?P$s=RgQFOJ?|7f;F~|A zjL6;!qoS}7A1s>k)Zpim52M0Se5N%%0$&{|TSrpY!EnEi#Jf8iAz$UWMMTJl6UPe8 zGrvt_4#*T05YuUO^i5sczKX#Cw)*1`D_FnPA~qtsd8AK0FI%1g64;U0J$7Jrb) zi)~j;=IkB|2x}rL5)KzpneZbKRSA!kmu4lxIpxq_`1o^Hp_1bZ?1hxG*K)!x-A9EA(^`k2> zLcV@@p0#==9frC|y^=c=q8(+{>Sk~xDEkOS6=ZjG`Pg+Z zc2~=&4#I7Hjh;N37i^6My0Fq-yci;GYc9a!Mu$n>3#9$Uh1QU=8|KEyIO{Aoevacr zg!~-GC7qCoFx=*Qq|_V!(EQHIe0?sQAiZL>uP`Bux^bVq1d6Xf>)d+~aFU3Mgdd4W zZI0KY>ouXM?K=adbGA88KM7TMMrnjG&Oj-mKz#WQ`HyMYNcmFWQo2>@xDdLRAyizx zev(xx_G?cC*a!_reIxFuXq}F(7lqOqdVD9k24XWX9_StdWeuUUg<75iWnrPHO|Nq^ zxEYEcM{Ar(emBLR!sd@%ZM$9Kf{M@3YH7wd` ziPE`Up>v}aM@W{E(5QRKiYma0au_H@Ih%G6?jWKnAzS1AY)m4|6wh&9i^}&EaOZ28 zlsar)hpy*^q9%1Gy1XH^r``|BmNH4T8$JWd(n3);icQhw4Jmgs%qRopX87_bT4R6j z8ZDpDfK0j=rZIz|X^E1d>M~GT5RV>><1#C{_)uuPbYjh}Ct6hP&W*15grZIn87LgR zU}G&Sp^MAiIQM37cUqz_jr+2Oa}_Sk{Ui=*%efg{4+ur=1Q{q_Vf2q^<3}i7%pgzs zY6n>iT}w?BrH;`jqBZ}r98H*y(QRVCob8!8!vpd_=uGJj_mVENHGb_xHD6g9C(JuQ zxf`t+Bz97dNg9jSDtz-|7&}8zROzgL>+|fBB~SaDxwa;TSQcUAMjfdsYm|%;0(>WZFVQHy`pM3l`Sxpb7Z#a1VhT! zLQ!Ge4PAQ)MHL~nOr4>Bo0bfv?jH-p1eHXdtBNUak&p`P? z(xyuNTXg**6m?8H90vRhekWhLW=373WhP8|JBe<&3V4tg=}eZxGz|i~B%^9nw7(Zo z(HFA*YoQ{VWo!H>%?ePsvM5%|Gy}z$dgx0A$}{lcC-FsXJS=j?!+02MvgsWpp{j=N zag?{+kU}?&Lu&^2Xu$8uYJt(0CWaQlOqP)usg=AQx;AJ`=^8B=+|3Y|iExu;ZTx&& zZcvi7`O&(MkqFNbQRN1}d%Y+Z{zKP1LMahmAfg%-Eju_FZmb@9)JH}Y2uFxW6%t3y zE|gZnT|`tT^orEVq@pEcR9i0i#{hXPwSbY+6y>p|)EjAO;tRXh8I%k*K1J1r8u~(Q zhL&YvpuZSWtD|cS{ePx~Ak)A2Dbtp!i)i}8ipsR`Phkl## z`EOHtVshS4X?cj&mU*k%+a6sz3#CN3pNR4Y03Pl|x!~V8bX_ZyGU2fG6R!KZ$8f-R zyeQCnk;Vzy2jVRai%Bi<3R$S7ccNjXgs@#i#q9x^t(M#n!spQy zDI;bwjnBxK%cUik1;V2wmleXy?f^+rJM&hOj8sko zJPU1XlNWy%UOVA?BC31^*dI&6{XEctfIf&7M$AV?oe&tEB_ukngbPb_@|^WCQq_4q z>pZ9Urp!98YeMa+^k6^nppE7>6LW2TZ6=7L24Ph9T>ID_ytp?dY*`-D*cffZY~ckh zxzJA-&5IMH1{{;A7=Kq z8{x*VH4Lp=-JGJ2tjY)3Y>ihWty*z7PmX45XvyG9U(DaTtBxpZf-<;qs5VquR5a%U zWpUA>z8AC*y6BVfB*GFtOak6U&zVv8W+N;f<7J1TC=4(QiLLQ709~tP%+yY5dj!yj zLM8KHP!5qasG>exX;GARP-xKj46&*x$`{_6pOu|ny}3RC2cy5s0ZC~&8(rt3H9!Al zC-s=5GBgYoW2!{k(Y32klt2GM7JJENQ=wJvbnafFw{3Of_+BdtOSE%}Kg98YX9suF z4HbV2$`5GGUpCE7>M=>zAyW59#8gmkLf0cgQ9)fF!Q_F^9aadrJx7NXVjc(`jgaTT zh(^rT(U=`=E{CdRC6LLkM3HQH{_aep`9? z{Xtl*glEfxjC$GFj=7#8bIucTqOMT$#8=4-^Mt=Z6w$h2wm+baWW+y9B+7(-ElY41 ziD)D-kB}~gjLs|maV<~0kvw4)SUNoYyOD%Eheig%9kfX>Xe1g&oHCe2d)JnsQB4dE z>&5LRgsc~L;3S;t(%BIu!%f|pBjgjbi`JArP5Yn@F@lOyhfmxMaDqElqwPzFtDWKE zgQVN!2z_kinbP|u&y?J~-Co6KN3)tgM5VCELhL z6ZPGp&C#_LTJ!I~vy*yE(&sqCb{Au6R$qfMolulNJELp&X8vR%O+ocFsTD2mE>h8Z>49(l`-x5sc>T>;jc>~g=O_5Zc18P0QT|UR?%!65N2>&=?eJPqUPK%JdiDwU zq=Ki)Sa>cz)#14^7KU5k^H4LVy!df{C?O=#t*b$HCsdxVX*qR{rCsA3%fd=23T<^x z@zI#8x2*zqIw)tNHN(M9>M==2A=<~uM@8!H$gcAmQ|=DZW4!%-9F(WTWg|K4^l3Io z55bQY#2qN7_ z{_JRt70FR4>@HLz`x%=sh@ohd*gncSB9K%Zf| zbE6i8LRPAwn6JnA2~Oq@bHWOwBEC^Jjx`w`mouE3AH~aXeuCvZCgjWR=$$ArTSud7 zT(A)>oW@Jg{-u8lA5ywz8gt-XeOcbwtG%j#t}}$9?xq}lkWiGC9ndwKe4MO4*4zb@ z&qRxQ5rH=v6jAG+_ThI?Ix9C7P;OAM(k=)YRxOslxe zh%MRgnCkmsk07d#OLP_GMRYaM@^4E(hTG=~m^5$9@eg(I%MAP+Iu@>nM{K_9Fn|1` z73xkvYkvH|K&QT0!NUM8yYtGgcw9<-8zNi}jB!d)Na>v7ewZugY#h~A2W22yGaT%s z9+R{HJ{4L*J{?k9{37TYC=_+S|6`CI!{xm}*;iaPUb?W;r`aG4g&%v0Kg#7@(Di4b zD3@7t&42tL%qT}agc5UKb>MgNzVIUba}33K@sSq4vg`)(i#(^wr(DIm0QvPy#|k}o z1ZJkp?asO<;9#ZfKp+q{Hph^R<7Mnr9dSBj`e_<)Gop9JiIL%jRy2H_wP)!P6M z7EwFluOccE&V%!X)72kvxEB$wB-tnu?k%D=!jnW)Bs^0@ZG@MLs7QF9h}sC>6;YM2 z2kO8@m2d$O6$w`tQ5)e#BB~LxrkpOq`-D{_d{RVh?*Xoh$1oQ!!p%ffB-~m=?S%V@ zs7^RWL~Vqph^R(*x`^5cFB4Jw&4AMaqc%CfCkQZXv%h5w0tu8sYvTYA3u=L^VRc$rZWD zy&C>VYFsF| z#XKymcEVRg)Iqq2>?iGndqNatg!nX}wGo~xq8i~NB5EgmMnpwI?pw}R!sTRNs1j}o zQRD^jK|*UM93!G4A$KXKpYTS{;)XR*MD2u=L{uc?F6DF)t|%LWu{+5obBD0n3GWwC zk?>OywG)0VqBg>xMN}p9TM<&~4_Mra&XawojVtm_S&?nJBpNE(P!zQja))x?)TM5} z+H1T;v-TZ}Madd-B7I}3aie7{9LvuuL7d(7a4PzFn-=bI^J5c=f2jo>n}|ur%y8PV zFtl+RbpJKAvl91}eXc~fmWZl^brH1^-U3ln&%}NoEpZ?9C&SVfaGWPZceZ8%xt~VL zh`FIU8sQ;08b&nY*<|Z05pF7?8sWAgY9-uFL={4A&XIoN;X*4B9xkGGLhjemNaArq zD-r$zQA8u=?(F8d2sYJKWpiyMWE+hnW?iNpQdpu9jrc*)RweXXbt_>%*-J}=8;ht; z*de0QZh+j_ofm{7gw;xTqKGPlevfbEa$-40?uR>iB;#@$D!Xur@IDb$2-!v>iMc~N zZ@DwCF56^@a8nW02)Vr7SFs6s?u_&kAL~aFa?^Gr3At%IF9;u%fvtpZi>OA(P21@r z+)sAX65%l-suA-1=qz4^UY;MFh>#mqG$UfaHLGveAB&B}pW^B~#rYFkox9mSvWe8W z?)@fP=Ne}{x*sBCIr&1VhlS!;tP{r8i5OFc$cZ>ZKO6P+=N;gm-^HqW7prm?E6Tm6 zO6WJXJmLPHMaX^Jz1-mRfAB0qJ|H_5;T-Z{TqX36&w0IVRvF4S{RTkD1DaD$$Y{GC zhho6}gI<-8tsBUIGumz-1I}n0zr(PMnqlp@M_@*TRpCV32$)r0k(> zg%plvYiQ}5d#53%*Gq2UjQ8MbbloHrHG^Bw*#|OXK)!WpG(EBM`ci>qIq|^~hFX9^W%njZR%y4*GAA5eA?ET84Ji3Z#yXR{9 zYntq&9+Pw|mfu)ehAN0BqU!>oD1Yw33E&HVI0)l>8R9&noUnT5h&Pn8W{-xJ&q~SXUAzF zri0NqV(N@|#Le{c9i5MA{t4XWSVqQHG^JjpJZK4LI0-ZK_q_rq${85xR z-$+=0!jnaC5x9+B!iTyifY0UR%s;9Ut~~kxVdgrI%8Qs49hD(5k3bQRnDLBw;lCfD zpN|-><}O5Lu4jgF@hrVw_Z1tci(6HP{NPWLwUhlw6Pwh zG(4$LsP({5%p&`o%$tX7!vnJC$EYaC#9ND|JT>?T<{3dBnLH!oXRJ1C>R!U@4s&`S z5;sZ`m?z|qno!*#o7P|Nbz?k=LpzJ4FRHpVq z7kx6cjD&Y*`+0ju6LcE7&J_xPCys-Jtzl>%P!13+YAv&iJ{cu5J?6;Ptx@kqg4o5@ zpnQu*kv@J$2(`?Hu6cx_49$-&f8te?k74LDd3T~F%`W<6OnPnTO&!2Pcee+{pX}AI z#pb6c{HWMHkP-5uV)ro;p?`$#{HMlzIOgN8Q%~q0d*zb`ej4gtY!hxKVaxczD38Pb z8CPDo^IqcAhrv4nM{<6cSDo80s#L`ZO8h-!q;| zEL}Znqx+OkZ#bED&lUB-^Oh1c2kdx?ntoqU z=$dJroz!EJHiC;=qcv_go1v>B6y*=2!S6n}ZIzJUeQ7-uI`>8!J+2V%Bf3?+jzQNrp&(M;>9c;Dasw!Lp^c0a-y@op z@u$%By-<|#FMgYn!vSGQwC;(4@JD>cX*}L>)M7HKL^xPRsTZ6awTO%=5c*N-gH?`N zP)4;9`cW$F95tVeY9ss`9E%>ae4>CrpG$_d6Q+jYOP8s^pq~KM#^=Lys37rCi;0wL zg`!T9%fgVKIRxGpO;L%^6cu$%Q6bh8<;!Z2tcx}(ZQ@NKGk>rzgYJ7U{?U|f;8+ac&pFlfvuJ^C}*Fxb_%`~v{O(B)X_Ta^k z?UUS_yZqKWB%Rx#jS@h7BxJ_U|1rA0Nev_J{>Mb6aZe>>{>GF&h9;wrK^r+lJpF!2 zpOdvD$p610_=C_Ik8@o^X@vngYDA}NDDT413_B*XbcVLtD}g7Xb)q5rCZa>o8txd1 zeB2U5%5+m}YZTl#RoqcGonz5;JX&)xv6FgCQWas?UBb~CPHYEs?JpGN&wQBk5@=0R znMl;IoR4DY54$$>6ejaYaWT(X-zrg5cXvZkBU^)~*4E8%;}LNOXJc>iUUc1$)(j#$ zsmCOphcH|z;b`?{67+X;-6$00&uN(R-(>EJ@^^GiL~H)8-iy$AiIDR8L9T+%-O(C5 z-?jTBu0Mmedv1u<)4F#;S_UNJl+Xc6?_mkmSne~SYcZjy08-1Y(B?MFcK|>4qQrv# zKLe$6iZ6l9TgA3oq&J~!0$TI8RkM?NOwyjv@E0+r&RNU@XJh1ni*+s0Qd}Ha8!g#W zC!$l)bxvc--soa$B$&2ekn+#3hf!OIVzps(ZqyQ(pP#2%N2`Le8d@{z?4%x(^a>{Q zj-QyXz68C2u1|%c{K*iDO|Ud_jl=Z=D02u!Ws_RA^$YDez&E`poI&SCjfKKXMX_2U z7ozK8v}TCeNj)a10~&S}V})=Q87BH6BKI$eU_R9HJ?<7iHoI%|L+2W3jg#F9=o%~( zRTazQ>^R^4i4)Pel6XnTjMokw7`8WBXZ%XQo4qKU*W6G@1BT-FF#223qFl~E8303h znL(NG`U4WL-1nR&0B(1Xh_(kjeT0a@Nl^EA(OsGV_=p!Z8nqQDae?uIzQx~u;KlOr zE)urz;(qvL9rfwh=3=6CY!6;=p!*1R(x^tx(RBA*+t?nwsE8kJ`VR||w5QCxG9J(e zx}pPJVW5vcn1QaG2CCVz_88h1ywa ztS{p<$y)FfmF1fdbS_!J?;_Vw_= zm}CZu5xp=D-@(A5Qo?xo;)yJLPbvD&1ogsbw#ISq2N;@O45_!tGBQbi6Xi=G_fn$X7I~mFl-3;zdOIjGR;~R&L3-#H?l=X4) z;Q7e-`GyRWHVTtR$wL|+DS4ygeaL;Jdp6jaj%nM;wB(rfnHTYBxs9Be%Hf8_6PZ^e zCo-?7q!pDyMaKMjy7o^`>d;}#pC{ILN`74DMzy)HPZ%xcdmw=hwHopS}zVpDB$8#SHFd z@Fjzm43y8*jMQl*gO;^$6Yw_!rDbD9QH!K&3{6W~PR69W`@D`7c#iaIi8nAY!@iVW7bP;*dQ<{_XwiMDggDuXZo zf`P&EqPsJvlV6}^plDHxb53-T(k+Jz6sW5kg9 z%H778(m^t7dpop#=grXj~Nofbg^kyhJcNaZs`y^#&p{TO?4qZF^B?+Wae43)PEQHYm zM2kvze{^|6s`xeoWvj+oG8k%ChLkU)4E2Ui1DxzdVPzSk8d3Zt^BEL>51T)sHPd|o zI>(EHs`xHO*Ns9^#kU{24iSnfK2i<_#q{QIbRH*qRPm8=lu%UJ^d6Z6@Cba!=pNiJ z1AM}ZUI(0IG)uHR7vQyCbPwPfV}x});FVr9$014Y+Gw4{eE`RK(ffe^@}kcHzwx3( zlmDNySyx;y?FzV$7u^W>uorn130f0jSr@JQT?OX@UgkyNsB_UpsvC8Q8dducqRqv8 zJDv1dXs46+d)7yQUy7&}dax68?~c~2oLWFKHk^$VMV0y<@Pg&yKHIwm1;9cuKJH^} z3?7rX-h{G!V~Vb!jxZ8W!&%jf_$|7rVX}O6zJ+$jKSM+2SWq zdP(t?2ss(!$Au3D`vgBKoaf1K<8pCFO^Ty$_gc7?zW1U~aqmMD$#}Fzm299W?V*j*`8 zXee%EJLNHLHu3mFc*@BcUkl*qW5qpH*^|)qi%`mhqYh^P+X+t-QI+sT5ve+$?jJ>W ziEu6%RSBcPQQyg^0^uPts)O((5fuq<5K#x=Z6YcX-gQJ`*uBagKbjZ+gwS}aM7Q>W zs_6Gx_Ta@|A&h>$iui&wkr(%-ge{F`{xxPEf0v6F?|Of4$Fle1nX*}--hI8d1Mp3m zlDcth0m=a~s!Vu1L~cTP;)^|v@JJ|+Xv8;(L3K8{3L35zO?kqLyrx0KFNh{}X1xm< zJf-TRk)T5{`Zm#`il1R?QjLre!)biTK`p7^hv_!HZfz*$C`Q-WB4qZXK!+2}BNioz zYMn4OO#J|#;dZFm@G+z?5iD%ut=&#Yz`l|QmCg*lOiPsGFq1Q7M(U)L!QKCpTCPUO z{K8dt%nYH+P?-PK;+CqhTXB2lEidc+kGIDA8`u1kDz|D`Oa^7{Ba@B9IY_vUh-!o% zim3b@;6+Etz{>z{_M&iiqPaOmQHgM%i0XviTuGZNX>;oSMsvMIcY$zq5fur&xel7^ zkg2F=8|v;QidqQ=L*#A_n-Y8FHLYBo1gLyQQQk&KMNau3YU&0zBN_2@;`)$AS(r|= zw-cV>UC$HKqI))?!8t^^+AUUu20yC8Q9hdLkogNn`EpXHSJow~!+n52xS32ul>kRE z$?n5PLO)6+cR48JWQ;#IO2zcy_jKoft1V{Wc*&o74aW?4imDy0MOA^OsFc&dhmCz9 zg$pA({1N+MYDK&ZjckqAEzc^7+No#5lG|98o}zG7`gNv!=^Dy>@WC5W5zk;~=`PVi zU;0U0)Pa0ebge5Cb>JnX$&5I9@c$((thT9(ONJcf5mj$Ul{dTmJwojktU6CoD6^-?0-_GliO+P4Qh(Tm0cZhDNc!aI1VJN^;4P(u&b6GxOIDFi77^?vyvvLC+lGjhZ$o zb9sscNzz?n=2!Qku-XhojVLL1I%jJT+}HrE8K0LhqfcaJDiyCOin_0?0ZJaNX=*=o zruKkv?s>R6R3ca=%pI4I-8-$>0gv&b69Av^qL%?@I$j2b=5~jfF=9^5Y?z{`r{ zXy6uTi+*OB_z966oFiTkK1aOp-+Y*tpM870TxoMR}M(%fm{GN_++_+84FAWYCgadV2fv{zUnav%Pf_*J7aRLJa#Jc3-Xo%R z!e>NOB>X`{>Rd?O1BFr|EQv_HR^X`Bg;F5=R77gcan!1y#2-uYVwMwh?=N(PDXG2I&Tp_DwELMaTy5$s~1QEkzd?J%Uao12L!l zxwXLgr>IeT^26wQMJVc4N-gUNB~Q4ih*}9-MN}ZHh^UotClM70hl!|_aD<2ogd;_y zs*YW!Ocf=JtMOGZ#<>rGOBmt2B2s(Zg6JBFFPOOF=NQ0iyy#=VRl6m0j3ZfqVE8U;$nUe!2P_46ZxALaZY!7(c^%xdJ(is zGaOw$NOkE0U>rBWwTn=Opc$h=#Z<`Fc%{;{T9%!n8T^dU%r2Fo7V})_TBtFlFS?d& zO!*duer%j3cF`vzR9ULDt2nzLko!wO)y6`ofzH$rI%1Dp*x>Jkg^?8xOC~cO9LTW!@Z4Qb91q++*?*r)b_EdqSQn8G(}C4 zdx&4}$`E%RZaYS;Z3kaGMH#vY6sFI(;Sa)+D9O?&6CNQV2zM5d3Lq6Xpdv@l3Zp^Xl-#LGZt#A6V;HW7;2U`W|qDC!`kDXnC4D>uGw&93c4 zv9e7HKh18U#Zy`dw-?3AV|MN3DF*?L@*;e}F^6-P`+8GDMND%19a?CRW++fo5{=Mn_*n%1p$!<(S#-7y|rdAZM)mzBx;2K>uUM;YTt?BRu&~q_b!<&22^|DaZt?MClJt34l z*=!BP#wbZIz_WqvXv?5RwuWYQz2p4}HUEq*A7eEu7UXu42DLeolB%!J&_)mkGDWp};*O z4g);Ii$WKv=nGL)`W|q88C6;ra7QoN70~NeIlc>us4!kf-7u~vT61x61j$AO*hM`C zWdKajd*ijnZE%sU8oL3z*qRPs4n0?+HN5#Yh=xIV43xJ;i@F^?jjk7kk|&$3q1YG| z7Q(B6?Wk6%k*%SbT_1XXLe0HY8q};PgQCAc$N;!;PmSUQL*S~7kZsJH9iidWVj(Z+iE%Tb zA6ttbt%QA{(y5v>iWfs91$9Ddk9jgd0P1Jq#1E9bxEn?zVGA!fW2f#uHT{^$iF0}} zlNa6_-KzUoDEil*I1yP-kj(_WgpMa-26y}1g_Juc=dgac(+ zv=ScRm(PSzyg1wQ2x>F#@9xN2x4P9|j{JGu>O}!< zJu9|q2O>O2NSJDCAQpoqCbcU8*&2J%&(OJwWVy=3`+c*@DS311b@2*Nz*=(mDMBiA zMvq2Pk1LL6Z3fSq|C717t`5IW~idM7os^*Rz;p^`PCcLX;B9wLs7W>oBj*3toTi;is#Tvw&DL;-=Oh;)LVvGvN2aiJr$jfWYTSde zsOh4aW-&DZ(co5UB_|6fgUeaeFmODay5tZxO->fwSvXxZjVz{&J5*S5ywS`oTY|cU z@HWNPSnnHQ>pw?s0cz*wjmY286s2plFo`eAS)xo>oF$pQF$0czQAU*r{V25=aMW`$ zszB&Rsep3S(=w`+(2r7YD>>>38P!I(JLcu~Q2m?TIqpFj*G~Aei0XuY!bgx!)x=S} z@Ls7q`wB3UrGR=g12y4b2F9YQpxbL;7H~kqX3#l0>Et;Y#K~q%PXIF+) zj&q8l0zM3s(P-Vdp73T7Dfc+)FG490-XS9Ox`Lxl6N)-7kU}5ghdo|gigEEHT)6Yx zjn1z{d6}^P>R3w~99IMlzpCuM`LV9AXU3$QL#zn!4@8DKk6ud|U_Fp6aA9L>=AXRyOkI&NF z1kn|-qhdt`1uJ&ZsGwMOX79uq+?`QuSmRnTDxgupj$$tn6O0Cp9YyS&SR!_fjU*b3 z1x3YPzW45&bDsI#=bin`hw%OW^bg-N=RME!KIOgdy)!$@T1hL2?p{?wMZ)vd%T~fG zt=36zlAGtH3T@-4|<#s5EqFcD375@X0 z*H!wM3tW0>$=c4#|D}cMh@}n*&llU1uA7TS!qe(nJGIiK+Y~+8RwsOOTGEUg?a_Mt z$P@n1yCM#3Sy~+_tBBC7++!TA4wqGy@Dfoa!tX&Pm(f_hFz;>?(EA~Bl_NAO*9%(x z9?5pxwr|CYnbqXec=1U!*=^}cjh4jNVP31o2PvcY`+|JIYIn`FP431QS89#6$@qGd zIi=`Shyp)utwVA?f-x`dXsAQdguDl1jGxR2NA}_NaW_F)tuL#H@Yk|(;m~S5S!D?e zvT_ZkRexFK2wOyz2=4*avFZ4tRo>mEqxUss7yn^pz8EaBJh>ZRl;AA+Z9CUminh~= zE!Ou6szb6H)-^A#19eD_L=0CbmOSCCW!PF59<45wRYdqVS-CRO>LOWX37?UbOM+Gx z$SOzpl&su_pw)S@DiFR1tB&rc3s1WVvMUnaFRDzqhPireEMLr4RPJnKY5y#VD^eYj zyAX(ZaSf_N!pv6#h%aiwtwcG;v|2@yi11NSZLO3*^*&sWKatt_s->q34o z#`xH$r}vBk(!lI&{Whv?Eq7(@4SN>3zIs0lD_g6@ujTSO5LS0!3_QepFRcuoW8fxu z798@|QUY%>u6Ao@!MI@nnD}BKJV_Mm5&jEQ ze4xEHmfq7NFRpk86IR*z4_8^DBj70hCQ=V#mL&Nm5;2`6y7b!o-VwDO-{51e9D`r$ zyA-abU<`D3BBsulBnPCSJ8r?tV~o42Ie?LvdIMwf-7wKZO_D@}i+8Imk(@QetAmSwj!C$6m5lfplM(jnZr*BzeuIL@5@yZ& z&^&OJtD&`AO_i%0;l^Dn;`m@DJONa^iKmR?3nmup2sg#b9Xh_aQ%SbI0{DSZ>A?8S zarPwKo-uG>zFo)eLuL24>K?Vz9donzOVCR|W!V2n2u;bx**b_V>lQA!Yij`~Rw5t^ei9gURA zHH(f`mLyAPj!JaIudoHmx(^&okfS`|zBuPQu{upw5#ipla?Rjcr^qTxI0jbnRX<}~ zx@xqu@8IMJX_q|i)iOIl%M}Q#%!b!8J62{z!YVVjg_lMKt0;dbSq)QbOX8ja9>bzf zX>qq1Kf=_PwIw}VEY3?{Av&aW;99vAj=xt1`Cj<(`;z)Gyu0_(>Tj}&UI+ZfDDNmr zw}y^>k`-wN;)qs{@GY?R^vprf&pM6{Y_R4uFZ zB*_xeI&e{Pt^Tsg5snh&PZCV6DM^8lDF;3}&7sx9vMLh(Ta;@Kr#_aX`5QpX%UvmKf`#K+x(xC&FRrD{khH55`BMPz zGwMmezZsQo4jtViNkpsnjH+dIh$LA;S_fLnwGNV1j_`U>uBEivUy=eLQx3G0R{P4T zNO*=Q*V1z_b(18nf2_Ez?fN$z)>aDFzfU1qO5dW&uLjs^)Ioqp7?th>9Tg>sXmz?# zwXA+ANtTe-f&Ou=#blKu945;3k5=6zDG)N{K>ujfAgdzb+M-pE9;2uV$J3&WV zND|TN0HbPIeW2yDgtQLyk8919RgSQ~)^h!$)w{AP2+GR6b)?llWK|^WAuHFvUYHsv ziR+&#E{W^krkJu)xc(gp$tmXLBEWl$N|&8hzm+6Qc%vv+3a-Hg;*F7ZJR8>Y%XilO$>bJkh9HYt7SIS;F;XRVJjvPJ;eWj&g+jif;1pDDhTu?%v=rk-73z zAZ(PCt1GSEl~s|jm#o~62v)M%A+Vz1GSSzA36qNQZ&0a``J|l_Pu)RMJ)Ak4;u9q)R0!5Y7*-)fvT8-1@B&e;r?i?bN%Rol z+eW1eNUJ?0$r7F;s!T|Sfu7RQ2wCL_F9wzLl=y0s)e32tBn86JqFhh8{*ICq2@e$Tc%r z!mVXhCZxkaPwD78Im!|41V>3viFcQCcly%PSMpRKTwYeLr?mP)Rz*U7T`%wmvJo5> z*>CbSb1DeCqZ(X7RnRO7v*|Nt7Y`fy{Mo2GM|4p1cx`vahC8SQDs6p z4D^)C-zcjb;Q~-ePlVfW+2z zJzWacRvy>Wp^)sWh4O^gigG=r)#H*xvjOKBl`bHyN|Iy=uM$-zq{Bc@>1c0R zmGqSOFD9!M(q57j2*-$WJ>~k5Bt^oLMY*0{h^ZNpxSo#2l&$T0`YNi-ht_eP})NUJ3z$rA1+s!T|Sfu7P)Pg&&%^PrNR61SPGR!ECU zQXpJMlhj(4`YlgD7X~c?0QP8 zwIqoK1MXo|I$~NqptZ7u--{{}(qW*dbabz*a)jNne9}|mUJA+GPwDAyc`6XTCd&1c zR(DELBz#Mh>*;5hT3+Earzn$; zs9I|sAxW0-CQ)TVIt=ub%O5JM9N~0ONl%IIH(9Na`1SGlOIw5!W#xLx_4k*gNO-O& z*V8{>YL+Cfr(-Z>YrCF44~dm0n@%dzyTYu7F|HutzM@=DX?40J(Imi|jjFZQDw1Rg zTSb)#=`he!F29nja)bwfN_t9sxXEgTw45Xb!p%jwo^t&@k`xKI5#@TC!_*i_Tu;}* zl&$T0dI}^~9@o>^kj&RYdBW|Mu2dlKSWc_`B#DjyJjbYX0ckZ)Yh?+ylvSCK4g)=< zqYvdMNBAo^N_t8>Le6uwLYga21;R#IwbZhDS5`&BUb1o@R`OQ@el3aX>DO>*YrCG7 zVQuAcJ)H!}ZCWT#nCVlqr?gsAl4ua%?nb3cL93g!R+jKPQDs6p4D^(aZj@Dyup5?7 zdP=;wLUKK&r|adZKsZZOORYqvN>U_zR+Q`Md`vB;a9mGkKw@jVp7w*amB;n88IpD_ zlqY;plsiUf^)E@Hd4P*A!<`VNOF^q6WR)eHDXL6Jhk>5b(V?=+5k3kk=_&EkCaV?F zL6Q^*CyHvRWwpN~MZ$AMxt`vNsaGU%JspE7Tif;Y9Z0M^uBT0C!G>x@o^ZS<*Hc)!m=pWQ(FB_lIT3Zn~X|_ zL#rhu$r6qfRVJjvKu_tYr>t^>`+-V&N_?ovYK63zBn85CM7f@F{ce&J3D+0pdfJSs z<0NrCT^>`mw(IFQNUS`rr?*1#ycWt6_FIl^c0Hxlj*>)s0v>Kux`4F$o7Tz_t|Y56 zAsq&KN=Hx2QI2p;I7)g-JW$Tv{gj@bl&1pW`=VS=Y4w;SMZ%9oxt=bCFoq}`*VETv zZEL%p4u!Rq$My6`NG{MqdBQoOTu*7$4Zolk{{tFJ0G(nOA;SHi(Pr3eSk`xL5BFgpjc}#sJiR1c$ka)eVsB|Rm+ z$z-)c8YW4BaI`4bQ?9?GBt^o*MY*0{fT_DBaXl?y%GP#0eFzdOkLzjE3Y9cB#28nQ z@I+Crr?k38l4u&>EThr|q}2wJWC@QGRVJjvKu_stfUI(a<3J@nC7xiiS|Rn9q(C@a zl zhX77ADqTRXHBW102{)5fnUD?xJ*A@$g0$#`n|p0EZiu z?kTNqmg6kpm!isqbQtI<9o;Ca9N|x(lAaPTrjT4u>FIiTDiF>R<$6l1sge{4pB3eL z`VppjD;(F;8Iai8uBR)(+REd4+6+m%7RnReE6VkhR_{m>eE`_lmsfX`?kTN~kX4rO zE>UGdIt=ubjt-Soj_^THNl%HNG+C{X4w9rmI8l`ADc9d$k|N=`qFhgJ!qf|rxSo!| zl&$T0`UWIc9@o<*v|vNEB2RdtDA!Y3T_Z^}4RDrG=>pPf14*)k$B8Nv(qW*dbTmL# zIl^(EI_{^y6HHbsr2di=2#1SuJ>~jqN>U`;U6kwTVVJs564%qsF=cDJo?Z!wmB;mT zHYD@4P@ZtBl`9np-A|z|OA;LdIMJwd0lC&Zt(7I*Ojcz=It=ubjy{y59N~6w)Nwxr z9wz7R?4_r<@>C#fl$Gl#t=^SYk+7Gn+S2I{#8gfa*VC`z(AIW6-5b_c9@o=Jkldz) z@`T@sa@Q%XR$YZ{kJbYmZd5ucTHP$iS;8+xl?mxE&{I0PQC2y^pFnloPk|RxNUo>! zbiF(k2xp0MJ*CxDNs5HeifT)z{}HBoD;(F;8Iai8uBR)(+REd4+6+m%7RnReE2@+Z zk5=zU5`6&JxT@M*%jyVOWeM*RRVJjvKu@{+p|Z*mJ_xGgehU1g$!djkkR%1diK1Lj zx&Ho=6ba82<$8J(re2W5^>hrTY;D)mHz2X{xSlql1skdrdBPJ#mC~i4)isht(*S1~ zRm*AvNwS2;i7FG)VW6j6et@iUgyTSU+)sfgn5ANuvD#PcbT!`j1xgv{sgI zkgUptbQtI<9epTAIl|51DCsHjP&s#JFFnnbrvhQ4tXxlN^{%XnguP_tE`+r(H9`{C z)34#s)^?nqpWg-UxP||N}N$huBY^Ly*w2NXNht>rPWkPiiFRKay@+yQ#}=q>*)+g zY;D)mKCrg(WHYJ%njvY|LV3bFMY*2R>NQECe-W-;St0Q6V$TJ0}Mk?>qmuBX>w>S;+_Psd=&)^uD2Ou%TL!Cp<=!>nW`+lO*~h;KN3x3rMRCB*_x~MpT)Q4g=RI9Sx9Gj&Ka9 zq^HE=O;#(U{*n|3hl_GO<@#$%QY746lIvT{9L9aF<4aXtMS4sC7M(=4p5Jg%qXA-Pryqb9%&j7s;ER==0yEa97?%7kTBh3Jsk{5R;|brULeZ#lvdLvi5>!c+o*H_ zX|;zWS;BKfl?mxE&{H}ZA*&qW#h{X&5?^hyS|JUSq(C@YlHtZiBLUAdDqTQY{m3=r-&F+M zT2^I3It=ubj=qzl9N|uIl=PH%cR6=wFFk!FPX)r|W#xKGt1o0#BwSfmuBRJesvwE$ zX(Og=ZP(L%A+hqfo=$?~HZ7DV{6duLDXo@Yn>!&|18{qz(mkcs&2pS2{6tilkPZVq zrK1~Vl_UHbRMJ!8j6!lfrKjuVsX#bOl3f*!sc>9RXFy_WyPo!e zwUx*9v>B3iEtDs`QPnpR6k`xKg73F$*9j2a^#PxIxrfhB3)0ZHz^0=Nhp#>YN z6?wvAM7f^Q>M}{9KLS2%RJwq)+CY*l;crBh3F$C!ozl?&S>*`FfJ%BwJlc9)PK{C2>9798*+<1Sb1DeXG1by3*`wnS*KEg&}V~C z7bJ=H13bm3bOE{6Jgt=_93-nUAsq&KN=F~cQI2qPI7)g-JXFrz*-KAz<*7i}C@a@f zTD>c)B4ICCxt^|tsS%R6o_-C7wzlgj|3P;vkL&3qEO46^$`j5P<*rj&Ewe7$9`ys< z)~Iw(X?3$4X9+(NRVJjvKu_uDMp@+u7l2B7O8kS#YK3&YBn84*qFhh8{!~edgwKj{ zJ^d%98dV|J(;1N1+ODTd!P?5>dO8%6eYH@Y@LEx>r?h%pl4v&IJfqSDq*Y0hEa6q6 z%7kib)b@-692_ywL;oUk^*LlW21(U`Ke zT~8l|#LDA(y3cx*G*7`8SMUPB2aQUXf>ytkBuh9$luL?M$4inUyhT*G)>_9%QXpic zfq%P>YqiU&NO*}T_v5hXm|_yaf0&A;vejMT=WvMum2BdwE(7=nqpk=1z^HWcbhN7^ zQ48R4MvVuYVAKVGR~p6TUN!1pgaa9;`w|%=_?9VUvV?RP_#zCgzLHgra4^^O|Lhj6 zzK~UckmU=snpX2=RU}+RYq?f8V9Ff3E;gYmB~>SX1mLMgO#-~!sB{#++M74H-&%W~Q?zD6?LsmJ$S4Fu#(`vdT1wvLK&}UlRE~_HphoW4c zGYEb;j7gt~S=UgXpMp!PhwF1sS0C5s!H}GcG4AtufRl_$H)X}HI;{Z%DW6uMSqu__D@XiY*Es{i3pKH{$ zfPXZq)>@M#sTt0(lH}4}Mqz5aByOwhg{jdPgY~OR#Ko9!;45xtVv(yAPjfot*^Fc3 z%1nHb*OGi8FHu*h*N{~-1aJ$ZYB^e05?5q~(^HaM+RN7nV_8XDktt!AfzqrB>y72b zm1ZAIT`Y+!4JBtt5>Y+fsJ{X}ZB#lbI=WX9R~kyLmc*6jEKJ=Xi7U-*n0j0im-lQ; zb=#y8WoS!4@};~)RIeecXb9jIMy2!STI)*U@}^{QNnGCFB8+7r3EuHm#pK5F<1!zL zspBMZ9jD|#Ng}E*F=`s%O-7|Nr=zPRahX#xRuY%_p_n>Tk|@2)rQZ3lR*QMUm8+o)Q;&y(*g;R~Y5gk9k% z{z1f0VlEPY_VTP-&$XH9o-OA&!c)xoaN;u+c`ofd`C&+{Y$nKif$*=QT54IHCP|U- zAyMw}h7B90u$#IP?*gvlmdzLYR|$zZb_n@`?TY_MY}a(Gk0PKSEU&$dzB z@p%^{Tp@65dwC0v$ETQIRkbRq~G9{b4<*tw!!rKMa!J zVN5>PC1%k=pX;s!y>-9k8c;L~yl-O6lqMUNxD&ID;MwwUx8dU8*pvi zZz;nC%>o;iF;kjsSmI91K681v4LbyjTkqY5Jra^@Fa|a(B`-)4y#aXn7TUwt0)A!G z_ke3`DOGw8)AvR4oh3BiZCAn3zvVms0U+0i<6aHRY^>$nZsc+oXt^ArEmvkLw`jSt z_w6<)eV-@a1;Ux4S_!w7@2vM-vrA{oQIYUrQ1PGnzF{n1%#*Iwdv_Q6Wl*!LT{D3P zl?@Q^Bn2L&o5?+Nw^n!0GIcG*jmCK5=j?S|Z{6xll+xrfBDO8z%-9I#)g{xO&MWgd za<|DuC<|%L>Rkf@{_yq#m6Gs{DUMdvl7KWZ{Q1h=;$n8 zn7h*ChMrqh!rcL5qHH(9h{Uf@3iTvYL#kYZcqknI36B(=mCs{6tRzK4Vnc z>Mxk0b<%a>Do<&%cVPA(dCq5%;|Pts>Gd`tp(5PX|eptZZUVEPJv{WS-lSUtx=6zR|>m0#<*sE0XH;i2f&?;Y6aZS zsG|VK8g&lf{YE_r_&1~81^m#c%r=#{x?_yvS`lzHqlN+QVN@&NenyP}JlUuV0WUY| zA;2e%dKvIdqvipAZdAjzl~5PM7>7C#aATu(1RP;h9xBM3fdGR%5+UUvxZL8(zPB|(P_ESu)gxp_oOjE}3#qp-&f-GA6Ap{|d z)?vyQ7FcUKFm7S)-!7SB(vl1}Fn4dF;dbS^Avvxq=EY>4Nu~_qs%B22dWzJJ&vV;Q_XWJ2g zbQqs!DL)xHI$Mr%gj_y8Zf+e{%Wi_~3WOhsDig9r9S1RAuu1VJ9t@PVjwwP`vBQ)v zETM9(z)n*XMZ%9ol?Yc+vC4$QMU{M5rF2+vq&}!pI;f4|lUuZBrs*gC*Fch+HTd&D zZd>lz72oS(j8B^_0VAVE z0Ul`7aexzyx(x6tqizR$->A<4yYEn0IbLx^z*UXf1aM2E_5~bm)Nz0(8+9SzrAFNZ zc!yDc2mFUop8$SoRO7EIA@;%;hq@BrYDNtM+{36c;DJUR33$9w73Wp|M`eysu_9J^ zs!=TI8%Dhc_=!=h@-LK4#47J(6ia%TQ7q{xMzN$17{&U%XB6xAtx>GsK;;~GM^O|BPZm`oyjfJ4@HJ5-A66+HRvf7h${nAN!Y8-r z|L6E5$;}!(KDlkVXOp9go0Ktyjx%mP?z}|9t;Q;b?6}*x)e{Z(HWLonO~RgiOnW!~ z9N<5V`W$e9QRy8{N4H55-3R!NQQndJM39c&la+g_rsS`ZxNm+ijF%uun$*BRJ^>jj zNM^@MW_x0ckIa1lM;VpQo>r?#5)B00%_#54Wlu*rS-I>f*+deTJ;T^e61UKBOtnhl zetvNzrXG|eKO67`qtd0I)diA7R{`EKRE~ zDdu2m;7*lv<6W^0;C4o(OF^rQB+)Mc*EPyJauL$eAX&LmP|{lxR|D+V21B`a49O2$g!ig75W&XmM0bRnkx zB#A4C}xry7+m1+BK1B-#z|Fr&O97a<)TB`a47N+L;IDHujs61UK3 zOr0c&E5#&CJtj&1IlzA!^*P`Iqtex(quV5j?gMqy8neB-&xv2tf*+deTJ;T^e z61UKBOtnhl_QsKznkq^D7Qjc0dJ*ttqtex(qYEU7t^$0}DDTJxO-GN*%2kJwKTG1O z!!YiIq+@R|kmo>#_Qo8{4H{m_Fy0$O0e3Jeojt7@B#9OW9B7nx3iYf}~?_Fpyh72KNT$`s|{;u`1w3M*RwKC!^8@nZw{Di53SOWR!R03PMMl z%E}dlk`*Lz1!=(4nix9@!a%kI87fFK=B~jQUo5u*-fmPnds?+i5}gEil~LZ2%bt$@ zAS;(WC8tT^vS%3QOX3!~3{y8r;x3kbcCDm&1ID-%_W<5&RJs(jI!lu1V!-Q+@{U}D zbabPvTq!6yUlLb}<5O$LUn*!|9Mdru+6NC|u5m;q!8rL909P_9ojk2RP(aZF!0xh& zyd#%99rcuzOP-QXw3bW$)zsR_H!2v%GVmNqz8~gBYxVpIfF~N2PM%h~N)qJ(4=~C* za>>)tp|WzxQ?jQdF8Lv;wUf>>j$<(wN`5@%X32H_b-*`_N+(aNX_7?K0UtNYJ95d> z(bKYW$y0KRBrf?&Q)?%kXB;nME|mNn%njU4+kP9s?Tkt%PpcmlQ1lC^*OgV|9l7M` zXppR2@{}wkiA#PhOl={FJKKh0YN90h%K@)6DxD{-4v{1}7VvbVyd#$<9i1&JmnS8^ zk;LV>S8DBKhck|gF&Ap`6wJLZ*ZFS%zcVVGJguIRBzhU}J)^uMmpmPPC@YsdCC^FX zlD|8(cCy16#{$fSlF#g3No+XA_+T#r?rl^$d0MR}Nwg{8PDXi0E_pf{AuE?WC4(e! z$uE~$J4v2#@KUwJ95d>QBPU98S-IpX*;5ji{E*bz zN%D;2Sj>fzACI|Na-Dx2@C~EV$8jbt+S`L{T6^*8I?|+R=<=aS_W_}qr4-RJRJ>?l}nzIB_(mme}yEtc9J~t zrZ5X7KNNElk;FU(Dlc&`ol0?S>o^F(PGT?hgc}FgJI{HvnE_q6xlf)%|cWUhP6s^Ss40L`jk+E1DWm=Y_?1x^WV}7bIEWlzn^F4$jx*{s zz^jb94e&0bo(6o;sBZu>C}h0i5`aq?H4t!PqlN?SZqz8i(MFvIc(GA`1iaIz`v6}z zY98R{Ms;hcY`vZs<51TJ+{CCI0CzE}1#qNM=}pY;pP*hu?CVWNv57Aj#U|!e(unOj z!l*HTQ;cGJ-ZqN8=%$<__F{-pEd0?%vC5Yl#VS8*)C+)b8O16utI9+y>E=eUr289H ztA4AhGFifHMYRxe$H!kunKF(qnpB89E6B3NoxT{eEJ*wfazcjH;jLC+eHBHH@JCRw z-DTtWqES&abIB_ImyN6Ce|h;Y5Wa5yZy3iHpGz0=e?v|GePmZ8{8UscLuJ?FuZ*+X zY+m43*}09kLC(pCp54+3QpCVJM0~-qfGmvh8vrO3EA;-4vA*pPR}J3yc-IRQ>`0tdf7Y3mp9%P;QS$*87}XumB||XAzIOuL%c#Qu+l@K{ z@EoJ=0Gw&mEWqcCdKd6NMtuv|1*ykD3;^87sG)$vjLHHQj7ry+m9a8JEW-~fL&P%l zRZ!9DfSVY_%Is|vD|4w)EW-mvu?#O7#WM6#=n-??%qZqO%BWfiPEa;kLd&_#oVQm5 zM z9MZ7J%s!+UjL@uT?ie}}jrp@0l%EOsrcv_&7Z}z3*OfenU`#GNz`cw*46xm(GXT#q z>JGq}M$H0z&Zu_*|6|m*fL->g1e08NfEyV#6mXbPS-^r(*0u^_WvmPl%kYEB5U~uK zD5!{K*xM+U;ZmcR^8-dP=NFA)&b<_8!~{1piV2P~s#Z1=6nd7>5-c;p?G-_}8IWs) zt}^&MO}=x4=DS4StIK!E`}S8E)cs|JOW6`+omu3Ls|<3MG<0>a%&c)p!y+^LkY+H} zvZDE`42}7-8kC<2_@+_w0T&q6y->+>2*%_p1Ki7~!vNcjIs@<=qwWBlY1Ayh=Ztz6 z@IOX<3)rPt2`0J905>vfDBv)ovVaAntZfx^q$@+jGW?)2L@YyJ1r@ChxQS7$%-%+^ zGM5^~GCW`u%kZL6EJH7a9x><5jAG8CjH;F31Z9&YwAPoI^Y)6M+ziMyLKhxq@k;WWoC^-8Wx$^hctt+mKDuiWs}gDyVaolEWnSA`VR0Xqn5;b z-u@Wl4ZbKwoejd~F9 z38P*De8Z?u0T&qcOYDQCFve+b3%HX}Er279`VHVQMx666ivjJZ`Pfw4BU`FH5jVuXMsY)2Z4|rzlu_*dJ4UhleO2X%t=`2bwz^~#TYZI5 z?CZluv9GTf#l9}B`bBKwmPWCOdm6cxbbIRgv`Ff8(%Q9z%76qZkjU65k4cTMA!$h z#NKZh#}|DSYtF}7VyvuchiR?4Uaut!gcqw~ZS>Au<6naH86p^4ff-Jf?;_#S^4&_v zORJ-Pd~v$z7z5LaOHasHYLp&6EHGY|>~-iCm50?xjKP2Po>wz3<-`%MT?Q9&&MSnM zN21~Nz%3K9;~~HkAkpxWXKzAwJOg+DBpP1vyxc=}JOX(Ahcr9@*o2VgQ(V~n_Ng=> zz6q=kIKrqh;C@CO1$eAc;{nez>KeeQM%@GWpi%Dueq_|Qg!^VX{)^*U7H}n_1_Ewu zR35O^s6znTjXDwVRHG{E*#B7>D?`LG9J)_M7XQN{EW=4gu?!a)#WFl)6m$O6DCXQ< zK}F1YTceoZK1MOYbBxNQ{$n=FYB~1?kMT0tWV4VUF?RLld40_%OzJ?au+%lH+^?c&1V2qJmCKNxEj@jeG@2 zjvZv$5^q*B3?&ZXVQ3giOl<;JyI>3yiH`PxbzG6nq25VWu1IxA7S8c|2;+T)<1VUs zn3|8VYbNV|P|isfBPhbi3K)Y+)g|JNIaq~vdz$@z*y3S| zw)L_M<#0K2QPa^ya+GU@+Lm|aSU9WY;CMxa(}Gns^uyE|7=uR_Cz)wbbUA$9Xo+bR zfU7b7MG_ZJ9kqA}p#!VlDHlR9M@XADq^nLNyL9gq)aaGcX4Qaj>2=N`lpy`U#t>UT&=4B zuQTc$z|W0JC$Vbx3bPKzhD`BqJu1|z7-K4J^$eycjc*4pJFC))SoUNsKaxS-V-$n@ z#HeK&D+{iIF?OGhUA} zJR4!WsBqka_-*Zly1{xCdC5Ko_@+_mYBgeNJxSbw$f@-u$sP{K$OCx~L8QA{#Dq*H zKEu-C{2uB%%qlH;80CLUZE#*_&Gy9MZP>HoXKRdsh?_ChCP{uA;2B1x!|_%TtB^zK2KK8@fMicRuXrWOu^KK zy(-BC-mkypB<|_!2S~cWOYjif5tGAgDZ;ZwHBSP3->CH7rPb~xq18)9rLF#esRt!- z_n&~{v=>UMBIcT%M0^O=WSD`EiD|v~5^RL))jpVdeMw64-vh4EBx)1DR-+C8TxKb$ z)&{)RsILH*?=97-fPXRSpMZUqmZ}YKlu@Gr4>zhJxBoMn3zSWCAK>Fg{Su+CqHNr? zaVE-go+>x;W56$svV|+?LrhWHkQw=fvnp+gNUl~yMFweIC^EMdWml)EyUpqRDfw${iSItX7G=IRgLSjKWVwx&)Sk(j zzJj)dx>Z&#JWiD*$z21;$m5Qs9Up=B@|9JEur;b zZTW~-LAz0DE563_TBhTT6<@4^*@VQ0rqtU=aYLp+NV~xMuPB04LXy^yDImY-=hwxHW zx$J`~`=H7`sIm{L?1RF$G!#>YL2U}EYo;){8(*;M@lS>C8)r5ZM9ZgfUB#CQ;ftbL z2!|`f0^!EkPqFt~$MFT*6nkSOc2_S7gd;(9nDRxt=?Gc54jo_oN4g^6UV48Thr$=# zaTvs*^a5PgsC59hHmVuWJ{v0%o-S1j;WeU)gx8BI5#A!INVpT;|8yk97rW|BQIYW1 z)jGbo+H`jU?u3n9@rM_?YPS~&e_gHP3tqEvk+>)MZZDao*CgL+^8UlC-V@wS~0!%2T=<{(krk*orRXC!<_y%wRsLNC&_paw^M8e48lgc~F zje~=Ua?}Dl4Mzk04r9DLufk_^6-NH#a!;rMM_L4cL30eelU%f3#{aiT-GUOls#3U4 zJc+62Byl_X1?R;{Zh(Xf21>(E9!=t6c?J0{rGB?Da-O5{K7~lbJfNPka?S05sU;+F z8E{RGfehwiAubq@eCQ-DmMmg8NwF;Cl}8601D|=7{8}m-0IfL&ye!Ix?>E!dme~Mt5@FF%N-Rj z;UW72J#4$Tr;Obyq28F4%lr{Id_s<0T3q{UN!;4+V#*eZz651f?q~>DX@dZK;=H)< zDB*(1cQ=VUS-dCuuqnpCM;2RPYCB1C!vL*&E}no@Ix9M)w2zTG-=PP zK{7zsm5Xn4OtoMP{@j$49OLTbkvey4O!0?ed~yraaM8rfjCWL8ZVNIkN`h%EnrMDb zUYIr;6->J>5j}A1q?va7yEfFZ0RzqCp?Zw2H0OBXtd566?$kh1jo4F`R0~VMk8uJI z36vCT??_Iy{?8IFRO7Rw^i_T&Jn}Q8IKBl))IMjadCTXGqLcf3CldZ}c}4w@+MO_q z=7zfm-c9q3cHlYZGrX5yfH8Q(?F&C!=#6?l{f0j9^SH<1@(qlEjl&{U-Eh;5x&zjC z%Zn=(7y6GRZtK5{DT^hV3(BnA-8Ep94sVY0;=-eZ3nsTJ;!YO-4xZOi(Op*SVrq~i zxgmhoDi=?{DxDP_(pwTTaVMVFz=qyQ8{37v8K&%eLN31DAsLM^cph_-W4yVxN9x>~ zF?ATIb`{74v}j^x#%;ATXIhj5(^@pq{G7ZnZ8j>Hc3mQR;21bhnE1fjZKz`d2Aate zv{-v1Eu7VXaLAn+NNQyyV@Z`*g5N1m_j5=}4wb}heNMIiSCZet`gVEA^7ChY_#FTA zgI6Fw;0^r1?p)`n;2q%~O0fg$8d%+>Tz$e6X{#3Q|w4Vi-?2bjlKcnEDU}%Al%vo03-3!wk zZVc<6+XZz<7T|NBM+GCi;13CcWjzjZdp{OZ%CONej@OBb;L3fouCbJZ?Xu2YJ_w-#dx>+_nxci{`=kW5i-V zZE;U2i)NucuoL*BtI!iV_R#{2;U{$X*+M%jpZ*b9;0gUHxSWGAaPz6-34I2v?~@l- zEG{%p61R!|jwy>J;&)ul%3TNntMmqb501@?3y%^mm^|(ho0(huJ9u78MR!@Pi>X19 zN0lpRlo4!_T<`9q)~)zLNO(FtsDb;4>X3ImSzEj{vzf zW9l$a{4r%9phXiiGu}dJ`DS5?l3-ekCYqmJ`Pw<-(fsb=f!_S%icUBW&bsfgQ)s?}Q^>@;j_Va{C<9oF6*M#Pb<%>J? zt={}hz$v{e_sLG4rdGQ5z#YP{Fqy z4mj4R(*bWYY6jrfM%7AiWhIy;{8&_Z5`OXES$u~&{&&oL&Sjr-nK@sloXay^8Okif ziYh~yW%%4ECV0KFDKo*jM%7AiMP-vCv;<2`@G~SDc+1VLu$&wf2+dI|9gWeiKeYBk z(X4o_%VKxfhG}K-R#c^0eWhA`rCKjR?pBu8*FafW+|S6F&9=yj212zW7!@qKG)KxQAfl?ANQju&@ca$C8n37-S3n~>3qODjN(1;E!`&E6U-UT%6$cg z*TXe(?Di9{TVAs9y^Y68C%2WgaRLq2!R#C_iEDpTodM00yX}A%`~|$=Rd57E7LvQ* zdP2fQ0v8;oI0l#c7Dw5S7z0m3eX$ht2wZThLDC;%e06Q?tfKTr2w0^h1DzMQ!71T_ z$?he#klo@V;Q1tl>@K+Rm^wp}9DhmDI^*IASf#U~LwZX>Cho*@y$^orq_Bu1bXFy_Mei7!!CHYiOz@ zOy8k@$*&u#)JuiSvCQikl`dSsDlK7cIR@hUzviWm3^qk_c_ry~CV!>3S&|Z=S+%6| zZgGy?o*D$n#&YZq08TN?z|njPtoh^|-*fnO<0Wm@HohO=190Gb<(+KtPPgtK5x`$D zcHA3Yz>Alw$#ji=#H59;_q-Om>&G>);<|yFaEfE_c6A>7T!=AnySf!qOf7J`nhwdm z@)AAitfF-00jsp+A?L;I4obLSa`qEj$ZqjB;Msny+ug33@L;^Go}F{609vzLJOQh8 zR&+>jNyx;Vc%I{q$D%*@+;1;+vZHxU*E+P*ar5rMt1a*m(jr{#$KI9oJO5lG9pkNd zyq&4XTz3oYhpC-0c3c=ayvSFRY3E}xX(74i$i9$pk-&w?DUQJlvnTv4i7{|tu8%3^ z5x6inf@E`fiFR~W?%D`gr7N+G^WrW9O1NOM%ZV*yxA@-hJW(OL3-cUIT_{QJ3P9_O zizi@}&WaA{EeV;p6VIID$Bi9t-1vg;yTYG6rX!%MJCs`KK|pg<@*iTl`|4fJD)iB3 zdU;+o&uMvg^-RHolzkGVkMZo2sL%&=>9xG0&_`N<&#%&s_^&(kM`ZD(vOeJ4>c>*P zun!Gga%;kFx?GhA9|sjT_qDNnVMQqUkK;;={WW`dC9Haw@E=+Agr4;bJ*%#ZI}tzL zil1fpX!{2R-^M2$`>4--Bcqe*RR!rE)<+~EZvyEz1`(#09=ytspm5-ymW zQp6UrTl{QzzDFUqq!+&*Q;$lLdm7N1<>Cogr5)2Dy(J+Ncj9?r_vRd4K)ox01is|I zu&$Sa%c>v1@ZzfaIZ8VJg|p(n?sK=&XihsGh^Y}6E6*}`k*_AxmM3A-5^u+$Jpwvm_YF3!sB`hZos5<56Axmpq~nCxF-3)wBc zH$0z?F_6`nm^xpQ+@*llnOr)afK^&ThxC?&Ox%fQPVq;q9nUg+QT6lI^m4SjRe6-| z12n7BPk652Cuo5WLhf)@?#=JrkUXQc+;x#zrV?NujPcVK;oi$ursB1a1iZ_rmjVB6 zR614q&dYa1xHqWeSD1)LnT&8TI8S86t7^e4VWCP!%#h=2h;U4mjQB6|*5WgA7dkU4 zD%30?vxq|-N_?Qn2zy!#Ly5On3}wQSs8&XQk5N8ScSSK$HnZakUVM>7TnX907YsVy z$oGxoi&C|cFTNBREBRt)gxB#y%yfVq3#GTmY`9_0ft$}ukg!sLzw-41B-{&KGGo#U zWtjIK(8fEO3DPX4(m@BVIfiUj?#I8IVGXX;anN(MV=>u5o{ATzSCi@3Z^q=?7~|_R zE!hZDTsa`&6vyD*XEpd)OG~-a;#Zhra`8LqbZUV&*TbDv@n}#iNZ_mS0jqT8hdM90 zS`sdpobJS(EPggT&%hYS>OM?8B1!Hkz-Np~#}lwhOX!f^l8}iz@yxy1$>r1^*QiN! z;QY!?mvlUnma8z6F$SJWmc`UI7(4dY4tTLsHJNsP1ST)W82CbcZ%lEKz#AG)aSZP7 z2Kec&rQH5r15?Z+(Drp8*+5>3TRAKD;bOolU5SmI7q^)y;eyG|AhwX*;v?XBj6!yw z6rF^rQzglr1$d58>5v0fX$c+DTM{yHC!TM`qOWW5JYjLA${O*F*7c8O&&w<#Y+Alz z7Qg>r9&mM|(qYrl-{dGuxU{TFgmf5R8HDD$vT41euvAB^#xmGBr*r3rx3M70vK#(`TPEAx&jlOtrJ$8wE7mdGMk;1jl1vD_ha&1 zBsAadN6@q~NzpG~TnEce)beFQ+d;*pQD5$)_?{YpV(H`OCqgTEktN?i#VitTCaQ(7 zSyYj*AgV;j{T2to_E}?!ghQ-Xgu9C>66Qs<650kWrZ3&iP$a8N8!KbiR@)F%hGTr` z@^Z7vw6QXF)wD4ii@<$S$X#-P+U0i3XV@jycegeCK5z?piCf4^T)`MF52g5`C$t)N zc0QemTE>RFpm`|86rq)M;m@aZuOn1AaLQOv3xCR_mt#X&p1Lc=>(GMwe_g55|KC>X zSZoPP*loLG5z(c(O0p}#%C6Hak1KPO<^zy>J)lnrWTfjSmaF#dMHq%iqB{mXJvg5ocS)X z?pC}Ki${m?Ei7Gun6lI>XgT=aS_Nn!WJIC2V@&sdS2rfIy%uxvT!NC`uWLH~hVvtM z@pv_v&SV}Y&AGc8E`)@O1U_%z6vyD_4X41*=@00hS-C?m zV3n@K9nOpUynzxfn7nx>wvgT8Ps8)q3b~wK{CiAwQIB)K1YF#xI;_$XI;6KGWa3Ud zuY^Ubc52#l9ozC|c)7cF3NPWfdrQZjtNLb`%46)-Vdq{@KOSZ*Xxyw3Wl@4z= z=fyRL5-ynRFJcSXEq*9GU#5`VC2fa~ z=wY}mT^$DfD_PYH$0Tm;pOcpenVgBSYo={-H$JD&LM1xLbS-GPf3;;=^jmn~+o9zB z7%^7~e0+XBeA+vV(7PADvoJ@k>GgYIeS3q`#-v`eHgDapVs`|__$N1)06t_?MPvVG z8|`XZn9abAX#1tjmZ+dQB!N8BUcNxS%k`_wbTsuuykNS)PWN#|j9A3?Rm8{_F``$# za|`@Na=>>w@H!+&%V<6EvNbsc9DuKqI!%PL7Zd59X%y>*)BuObP;nW{$OXmLHTKgARa}dUk_YNE3 z#irF{I_eT8PsbQIlDcDxiv-@QbBbf|y~`KK{%eeZ<}8b)m`C8d0V_eWngWj2cUJEG zSHLP=iM5><_n=J)7fhb=i7jNe_*U>dS|Pg|<6)RON|M}f0d1qYcmh`Gtmu&5l8}iz z@jMBBF2fi&zh-0VZAtQkYc^E4z^7bwNE%?>i5D!%FW_b3ix^)A%Iv9d+!G$HdP@=! zjuKU-isNN!-bMCZ?uf48BR{B**xM$31g~GHsp#c)w9_s$tQ@%#1IF-@yHs zX;BhPYtcmWbMnHp*{ERJb&2SKW1z-Ne01$L)Ug2r&E!>dUG0sua8|d&A$MvZDdu~h zl5%GuC4ZE}U3k+lWnP@*4oK$7OO_W753Rt@!k=(f1s*VFI6Vf&T<&@uT>Bz%_I) zj_B2H&+ch1;5%LCIwTXZ=GmR4@dEVjNnF4D!voJw#mz^K&9pH>j~tQ% zut?Wy?L6TZqDB&aFRF#`V*T@?Bl$0l+8@dqIUaxc>CYJBzd=uUo2U}uYobOHz9Xtc z_=TvE-vTa&Kadc+C+sV#v?}0nqDJn3zoc-0{-VN2!ox+C2u~3;lJHDXZG?XlHIk5j z<|w{M3Ewsq;lD(c{tY-^)JVc_M70nuj(;XSPPbKSVLx@OOgIHp^2sbQTNgj@ z5`K+xCo*ETAeIrXi8?1T;_p=ZGXK2xiAtzUcr~cRlGv(K{t(lax4R1NM!tQOWE;2T zS4J@^+Y#>h`Ar0DfwvKI47^w3@~^4vT7Iryo`0|?@DCmZ9Hm=K$z%WRTEmd4h2zej zJR}8-!N1GFN%{$j?m`(JR5{#vI1^LPO5);SeR(j&R{$XornozVOUjYEPToZE{-Gka z{sd@N?#4F{60W(>%h$2iCAjEj>ezF)Gj4YN`MczkEn+(dBVJ{^d=gpuzdrWoRSWs=Oibdd3`42i(b<;GL71RL9lGE#Ppikl+vDQzDF$a{aXhlu$% z5R<(|%)3pZBd%&tI)Y4p_&*3^e5xPmqH3deyREsN{ci)U3d-#o1~tTjVg!#OxWL_q z>K$N}l!bVdJZ0&@a&ZSWt#W3SF5I4ogDs3(_dO_P7U<^nkT9FjGY~G!%XERd53hcY zsRtx+ccZ^x>TXG#m&Y*m3dZ2id^pK5{^(#GLSmjT!iWVbR2VA7Xv_e@=^% zU|NeNnxB)Gy1H4Hh^^+>Ni+YbHZ05r3^bE_YJKf3w`~{BYD+leP7Nf*eC{(M{P zjj$K~ap}Yx@xI_<*)xFO7}XvBD0({e`_W<|CKaQFBC+0ut75IuitFxv` z5fM%k)kZiURN|cYQI(?YX~3_Pb{kXNPx-VR4tT#&%=$Z{m~|he*tQDb7Do9(w6PFd zpb+t`YiB^KgS($jKxVcjO575bV2KINQ-UQQL5UIY-^Y#L*|QV&ubP%#L{Qu6AM7ga z0%&o%Yhn`Od{oV8CS>SwbDD`4S96*PxrGxM@g$QGUIQwT5p%1@mowpas6-+o=HC>F zWxm(V>>>*ns|eYr#2GOw7t07)o@P~e{--p}fjmPD8;N#ahuLoszK#^7K5<0Qv8 zLjF;V*R_wE-vj)>sB}P!CT3=QywY;TVOo>~(^@pq{G7ZnZ8j>Hc3mQR;25{ORx@|4 z-G(|gV4#`YQ~Vn%fwdRT>K*J%?$m(ewISi<9RJLS=QA(oz>kr(a8~Z;Hgp{6>TvH4 z=*1*IYcF@9?+>c~E$J=t3a0)oiF-56+YyiZz)x`MSZGO{;e3**dZ^(sAD^MaiFu~S zGD1GcBr;+?$;2{3-sKY+ac{MB5k9WeQI{2wbhF*l2!AW8g^-EH|HRmgaeQ&Rbj^g! zszb*YmZBa5cyH>^r5n$`ym`3%yTMKdy{VRjJ@@9HYaUZHIjj5F- zabDKL)b<#IKj-Hp$N2OpHc3mQR;25}JF!5h#cP-2Y3^emtY@{)^x6)VH!daaThuo=wq?m83l5*E3B|A#u zw%@LpGA~Zj0tt`T_}uNoF_oGT zo(k*0-Jju@#I5~v^1?r({tPnenki52mVPJKL8=GPx+3ugG?vNx7~dI7WmAej#}J|t@nVoV8nNS zCnBWy^IpD%?yk4ck$(&A4xWJTbP{z)PR5$!F$RnA4q|;@F}rL1Uzqw#64!M;W^fFY zln*_tYJ0hb&cXB8Bt4h8g(e}FYgNxY;mh~~Ch>zP|CY(eMtx7%yHd%(_TuDy{K&T1Ptdv3eS6(eW>pJ{U(y5utEAtIw5~x&~vgK8%^7!0JrrQ7x4tWE6q=umvjv z#PuP(TvRLJ%JN+r0{AOYt%M^)~4w-Vkis!aHhs8+&mOIJz}xTmvzE2>x#;ZdSWgjT;6)^BGOt3+s(X<=o0 zsthfJ_lPPHuBs0yS_s?pZoW*opQwnix1RD^2wBJYtMY`bW4!qY_foOSglCItA-q9U zMEJC*7Q%n)-qI$h!nd%{=@I`}bo{I2S;Dtec=zp!0hoFWWAb%L;wOyD5bxWkLMA#) zYoM>cnq33&)5c|p>+v>7ygGcH41kFUGTu7P-laT(%zeErSr8i+Y2-?t>L$Jc{Z zzK9<-E<@azuZ^MK+3XsL_Yv2gA+GX;pLIeM4}46aw&RL42Qv>mK;@UIB4~T1OAbv(%dxm&Rd^nn{ zOFUd$dxrQFaZL@x=ZkC45YH0V)Ij`-xb_V32Dm>Y>k`GsOQE*VIVN-?&L)BOYL9KqK)M;@UIBBg8c|5*Nj_ zXNcd^_0-fz{JFUH4Dp%ThfR&dmx^o85O>$UX=)^1T3maExUBE?Zaa-7XtGDWszV2sb3M&ftH#b-Jm6+7!G|DBQ*)YGO0V(W8!KlHw^l0`Qib>+Pz6#M$9 zDen&<4ZrrXrE1XBjd(Y4@x9l_gq>$Godo^mvTJH2zENCzX0G>zNX^%y(9e-wQzNnU zt3C5o9ln}y98#vgxd%U7F7JD%=eGy zixH+HJg{2F7sr{7@PcX`UtDiG!spkmgwnAtU);N%;v?*_ewB_d4h9)#OE{%k#}}_k zS0=n?aFsv4SbIaR+d|l~X_c-5?f>k}v+7MmXjN}x)sI&r+6b3JqH*@G9nTjJDEl_T zp9U&0!rv&THp0tAwGb|`0S(&-Z%5{F(Fg}B?>552MYUMI6~wHcLe?F>E5H|Hu_uFn zhKQE4WSN`XjW4Py{l)RM)_g>k@lP}OqRR56nwIy(LI( zF^Qqu#{Y92hQn@x`DzTC9*){bb-b-A2hPbo1)`xzq?3x;hFA#TB=FB?cdK>y4%h?%0ytKIZ!zi#)g~xysfySGVj+B7xf3quBiv{ z(c;=O#GS>p3-tV(h{?H1%)gJA?ECNPi0d=>;vcF`_7`z~{CqsA7jb8C{QzGhWEX!R z4qOsSFYicC^gSewI z!|I6Z73hDIUDN-m?##pOobEloGuU=w+7YcOG>9R_m|8=liRp%F?0wGOBqup%2d!IL z_d3TALwbU?x#kBk#}JB=R18&*ctmTepoSXaqGnOt61TXG%fnsoyT6~`+TOL!?!KXY zIDee=UBC64*7RHJeb@WGI}cuA81t$-2X5xq%hIux$rTIWU5(b0>uA9*_zyL?Vjeuz zXx%w*Grztk{#hnhEP(kjy<|Qf)`DN~FY>zb;8%^-odY-XYrgoG*KFVcH5v{G*M~d(o|v6gf_f|J2T@Y zcaqmth{y$QUL*cHyj=cSVba8hZCmi+Cdn=Kx&}t%g6ncR#f_C_^_)qe3`Pssxi4PN7@L{UtZuPnf5xKyJXT|@nm&>obVZCig6CZxu zf)9(OV}#_LTp=PC_%KfVU0yCfO_((CVR{Qbv`g+Bud5J|3w*d${D1Ir`L~2g6CciL z!H2sfm)oe`j!8R2}=NfRGd55rNuZpr`uw&afYx(X4wz=zAl{~Irt ze?piv@nO>ze3&D-H@&VxL@w}QC55pKlqUAZ3X>+Zt6S)cH6*vc*Hwtf1wI@v{I z1PaJ8qtg!Ln9*?oIc9WRKyNPCO&ljrPE{H`@7`C;u8a8!g-^PpcXEY@TreDV5r4(Y<$Hul z6Cdts!H2yhce2-2h{y##TqpkDdb#}b!la20PqyI0Es}fB>ncR#0v|Tk(f4SjiM>g} zqzUbz7N+Jlk~`SzDn#T0A5InjY%iC;QKxT;Ky^bcld^Hw?JI zNbPWu>NG7k$yh2qg3~++ZqmmD-Zqvha2su_-HhKsu;xj?QvbT73s^T!Cv&!lPIRcI z+z6$Pgl$rM@CGXwBy~+NTDIHFhYCKt=SiOk?qeQQIt9H`mDwWk?5|V5<$$_7&nQh^ z$dk(h&Gkwls7-}hkW!nsL~!o}0euB)RLnb7!zpjB7B*fG*fIBKnM*&oKxQqqM*59- zGUupcGhVfUN{nW1LbT_KV=(`+1l?Q}BKHhpX%G#t6W zWPF}48*V!0zzaOBANX-~cESh#(9`;X8KDVZ_HU5%(u2`oN~TX&@XB-Gl|)Mg$?U8K zNi2d#TO+w&6erWnHLUU;gZt{>81PMU+g&dLy{eAbcDnDaL@S?1v%|4cug=&3)Y}zmaYR zg%cbpGemh9FJk{DW zaH683ZeuBBM+GhsM^l$j+0@Uqq;!N=&RaSa>(9X6NOAj~YMU7JL0i_#*1CMz`ghst zF-mL2HA>y%oHq)RHjoZmu3jV8akot;Ghale`KEXj?SJ=<^>~`XAaC-a8;T;y)S1`o32}Aeq@H-YDA{L>fqE}GMx++ zOP8q7uoKT3rb%8}H%maTxo8A8F$^D{lm-w-#ON}LnpjbaeEU9%=Hg&0_Id-tci}F!|CwS8Nf;V_lwlh&Q z#aP;aNV`>QR-h<|MJVz=k><9UWPT|5^u=g0ha1s7a@?0+9p=|Wp!cf_zvdWM(Cu); z8u_fnvQ_vr*~EZTj2phagD$^OQol2gtH2M96f`bU3yh@<9K3wJ=^~$-XhHFPQrJYx za2fcP8L9&B(Iv6l5wU1zK8CbM4! z9y^#83NFf!nrBiK;3Fm#3@H@9Z2lB;;0}qoRcHkGAkhZosxt?YdAflyJNS8Z?tSy2 z3LLcpmFe0;@O)3&XvKQzwksJ|R`*ZDeX2>K5IC^cU?LmMV3m?Q}EhQl*idD2m#o(3uDF z1;k{sfjdljJJ5eiu#5}Xi%rwRW$1d-)b)_yv!0aI6p3vpI?g=r0`j7nYwl~+K)#{M zVXKrf-Im)i@F)lz=Za;K(qz*KrXtZ#YU1Q8^4kfH0q-bp2jq$;3@5`^9}`CKDc0|m zY;_X0Y<+H#E&mnS^19NnwP`9JDd?LoxOO{LiY~TBH~T-dK^9Ddh*V!5;%Q`y`^9Rx z;Qv*FF{&K5cO&ji5X%&6hjO-(2dT(6Rlu^NSawtDw$@oKqg8}Y5nmW%ygeF;J1wyA zu~t5t6)`kaLuPG1in8u9&`2X%-dA6IVto=+dZ~)=DWau?8nwXEqneOLI^dtCK2Ok6#D#%6U_X%lJem<2&9C1 zucef&BrF(AVlCK0DztL$@D_TE%K~gWvLvJhSz4yej zx>X`*!Muu!#u8}Z>)hLlCA#OMkkkd1)l}m(t>&ez1bq_*4t+5`e@(t_^7pF+OSD>F zj5l8w6?nIig1ZmbtH`qi!S*q+{Mocr zfd{Wz_rh)BfuD+$R0MqbP}tfp7Mx?GF5m)@5=ro1jjcVmT3yKsN+Z37r>~i(72rqa z>14d(BOvZ}241~y<|boqB{MfU+u}=QX0YjQ2mac(!Z`4q##RQdDdn!puXNc5)SD_b`#%K(gYCS&ShQylc~vPnHFg^bilrKs-| zvmMM$SjHJkP;ZLHe=e5AlI9a*kqeLg#X|H=SQHS6TM?;41E~qwnTp~Qtp?>R3tBwO zms-mxwaZvrjjF&|Myf3qe8-cr^+wSy#!>|JRte66l9`i=bID%`0V(>GOkfYQi13Cnl;W;szG z7Ht+apc@NBhmtE+5lrEX)$ z&lw;VIr`c(kP0fw=O(w;Q#CQ;(^Pgw6Q0p3+h3{Mb{~@wi`-7Ry8L^0i>jzSSgKvpt znCcR9u%u(l1MGGsP)BQn?F=w< z@OJS$X~%H24>DqrG5ByG6*zc-j6TrB_`l2f6Pi?lligld_YE+w4tP z5YZD0?gucc!1Yz;Jls%lcTYl)v;`Ih6df7aWxz7g3{`-~7^xkbq-P0SVJubPrncJE zwi4XNlXe#TnJ1x>6y2AZ?h4Q=>IxMFV;n{AOHukv<`Av_la!{!e^uttMj{ptm#Zl! z>puRcL_8;*C2)mO*X+Q1j8p@Xx>W$kh!#kYQV)t zs_+bOYrS@`?=>J#HM{*qwrwIYuSv^X?OTQ)@8Gv~BJ1O);-<$|$Q3{3uOGxks%LV+ z0e;5n-u9}XZ;tZ2BGD|LQ48_Y2DRY4wM^;jf(((*=#bl%l%)%oovQ^fP3&lc!6W1Bof_#nLk?t4iVdJU*A2m`ntDCy>az^Az|18z}4r_`kaFI5n?itd@ zddaZ0bmw}&-)mEt@{##p5#4YC9;B^L%136l?EpK0eB?dlBlD5*)qs226KGLoKx#D2Ms>4M*}$khSt@e%H&UNVF=Ckh?Om>?7|U+!7Q zlYozil)6EN4ToEGfL(qAv9Om6y9>t$-#;$!d^0LI9$&H4{qcm!-m800r#|kQU$Wz=$D%SWM|I?wK`(4MvUNUS+93Rj}wla~8-G{RYyvc&y1mJB(ssh<5 zq`g39c?)+HAUlPWkIZhiM<@e_`L%jsFPZO)=)41-sQ8uYBjZC}IShH_FyxiPkXI(; zl`RUEfj{@_^uk^;w^)P=!q-&_TMZG8chLvfI$eHQaH)~nj?$9sSLrrZX;Rk|Llm+$ zRq9Av3bKH^n^y#`!4?|Y#tIT{oZvi7&fAna5|F9tbX~2*j(&Dr*^o_LUsc_ct68w$ zuXF*|R@_P0z{DhX9~XFJ0~?sjCU;g1BuXXPz=NztrM(3AF;W|_Y@`x!fp#2;GVtBD z3n>9fowvYY3POn_xWfWdiA~DYM(P5xX>oM|UZ**iC+`PzKJ~G%o>3lY8(gJydDp2KX0Vl31{#Ha`g)co*9cm4Kwq z0Pw;FN$}HNk{~i!`=-PI*q^bMfTYd<@YV)N@W-|>EA63BbiH;&i2?A!@f;V<~4@cxvakzFE38*niqE=yi+HYh&@5 zXEiBoz&+$kK6CCI!x1&DOoa*nLh#pY+Xgk7DT)N+YMLEwJ=c*!AJJ&jI=s4DFEyctg##r0-^HGG>{X zZjyc}*Ro4AW-qrHSpgEJ+~wVkz1bnjW_FbJJnqDeyWNvLDJwNYo8}8_^BfH1KdR`{ zzFD}du#Xn3eNE9W0(> zR8%&Wpx=&D5x-~^xh_3KoAg22S|zOyUe&f+6Lu8b*^~AbBy~4=fY)kgohSqQ`vDWO z{q%il;)i~mrYsAa!iUA$KR09x@MEG4$W5Ht?{#I^DyGD&wj~)RcMmAb@`a8a-G7>4 zH`A-1dw1;Hsz#OBB}Y!||4kL1s?^;tJ5_M5k;=g9ja2=IHjB3E|D-DfE+ALZ;B?VH zTf4J$y?Lcwod#3tAr-yRHwzc@Cl9IXcXtbTpZHo&%G!QW5&B(=)Pe2clHS*}1T5Lh z*2qH_+3VBDaA_Y__ijI>(d*nPDm+H18&kl+>sZaR)kNwjld1r{RPYec1LF29?d!`1 zzdYkwIctj|Qg?qK%W}B1{M02^AGl^!b0Lr=ln*KHu|y&Z&EEgst(DggrO06|JS8r6i^;Wae zYu39RZ@5xfb<$xc9eqUQ$m330b<&Atp{^&wsV<`KE_Z>{xUr_>_AVEoI5$=|xZU9bsc~aX$?f|tKyhxYZg6|h1ybY2nv&c0+c?CV z2E>{Ys9*saJL|0a>|EoVbBaxWy{BojxYRz`qFor{u;(_w0bgnO&6)1%O0iIBt+50@ zgBXr#x&v!GSzq$J#2qePp$ua1yulq<+-nZCb?NiOlI=|had8c*UWgvs|m-_?P z_%ep%`#^t?@%G3-2H(H+2d?pD49Rzw$HZGou^<03kimEV{=hZ9j3N0RXMqe_w)yH2 z#Y5PY1o2_%Lm1T{KQjPJT1gmCmgM{a;K##;QH_$kWdN46k}#kw$sIQQA?{le>m;>* zVL+o0_s-9V#-Fhy|G!zFwcdQnpX|+4s*{s#e_f?DJ>BuUIKQ{V5hEGIcUfv_Vd_|s zfUYrS5Xty?ePYJrg^>)#FYFUtW6U6u@w5fAR8qztyBE^OAqF+bpFe*?S~(a`mg7bX zd}geSlq|8rNCx>&_ld4CW)R8vdwpWYV}+3n#((G&U1Q83lJU;l#am2ftYpnu7}OyD zy!ji_%E5TD9LHMV3+kM!IyHsU+fJoZ-{jk8Eev6>_FZ8$j;t*?p6d!04-1AcI9}%p zt8rv)$?*#ouxQoB8){dZGk#^d-I~=`Z&*#D{L+F_5~Gw@y@`!gtNrSf-%eb~VL+qY z|D2x@%?per`S0;qORJayy2%U!i`CnQXLbYdlN*For@BH-q2}&bS&&R+lVqc1DZW5MvlDzvPUov1E10@~6&d zah8l>u>7ktt|gKfL;>bV+Ia^{8%nKxt^BMm%U^Irv%P6ChC-8K=@@5RjU}r~mQ&7X z@mgXGgXKBSxEf1Vmn^SwMvJp#41?u|oN+C^YV$I@4wk+t%(x-9>19noxWVvM2C zq*&Vaac&V`(&wJ6E?GX!87*E*jA5`GcE;6w$?B5j@y=**mW*MroN>mr#1exjz#L2G zJ0ol5XLVWr8*OOX`&u+Z5@TGv_za=2q)_^_E3C$mwI#>zxq`(xGK9hL_pY!SN7j}c zx7*RucHI2P5C+F*y24tbh%ppgj-oMFV6E`1Eerqh{9senZq+tVVhDvK1yRQpR^!Op zlH--GVDTVg2!rE$TwyI0#25-LSJ0iVz*^y1TNeIjZeX^_2SX?%se&GLh1EE+w&b|$ zPPtnU1(A&-Ll_+QcZJnBvbN;-LRYYO5HW)Zg=cM9_zgBN^)#zoBK9}TwQ-f$GN3VtLX*Pib?(3#Pu7<_Z*qr=R}+I6 zJnwP`)_AhMOBW1T;EL*=rwpm- zMs=prZCj6sGK@l%g6imgpf#?HAh}NV0~oI`hB3IF)ep3#${0t%<|=z*Kj4}p7(qJX z<2FFgx>dxLHeGFWY)4}mM`25~_N~6KHMR^P+5WmO#CW|ij=^@DUE-~wnCS^i#W9cq z&K0+3U+9`I7()8uARD53RLy8^b96Yn*|l`zjHD2zx;wc~bd52CNXE@RG2`{eNCx9e z`b5_lGl*n-L!X#&#*Abz{$!u%mY8EG1)gK>o<8w4k1&Yz$X{%j>d`h^jGU zWy$awN33{+F@nMHLyoAHN@55FmMiJ2j<{OMSy`6+w@z5L9T-MXC{o>QyL-GDB-ePs zI8;E&lHt=FvEp^Z2nNGpM^w#+tSlKG?}!y=$Os0*8Anu0^e}`1%h7YbBd%6*R+c5d z(Fx18J;VqKMT(wJJECd~Sy?jto+DPgZWzH}_KrLEX}$Os0*XF8%< zqK6?ASdN}CM_jGstSn3Zawja?z7QiQ6e)T-j;I<#R+bE}bi|5B4gpI73D-82-Q!)e=1np}=zV{Lv9tD>*C6lJELtOS>7juD{@{B56~IQ52dKLHqZC ztTAN($@GPN;Kl2SQ4FSM^nt7~WdO-^T_1RHri@}ReQh7emZ)MF1)8Jk{e56-USI&} zg}W_y3F}!HS@bIc`_Cj9Mm&Y)@!x8A7ss zeqV@jwv1!2J*qEkOO!E?0?twP%D&JwUoeF9#f9#O);C4QQRq^fy|piFjV(h+wjb>a zFcVLYt>r0;3yTiqKGKj(R zc6VS)R56Nz%u)4icWBK8tS?>gkPXiMOPLP-+^#ue**swkg(pSSj!%sTe<2lDW6A20 z<#U|T;`PKB2Frt;aW$5#E?J)9j236f7zWFhGp;3=7(@Z)V7k;9St~!Q%ktl3L$hzD zV=T?t=UNP+@T6e6%NQ42eKuw7)F8S!1{0>*qRp@ zKziXm3tn{ZN2%#==<2h3mY3=er>Ph5w$}~agZKvHi3xFq@x;Wj^=`M}moZGM&6Q;! zgiJBG_i6D4;E9VB=>4HnsdxRhWtzu&iVrE|AIycNIgM3hskeW6e{5LKV6%5FEK7K> zq5{U@F_8;RD;ulGvaNDQU=@kb$b}GZO01_8i0bO%Tv%FpSVgkAJ{Llq4eJ?fZqJ2f zsVi1gz__lyoeNDX8>`5&{l*=ERV2caXY?1ztfv%+>T0iCSXy~lMY1X7LWtKD>ltj0 z%Y|jBD^^s%xUQyip=o7f6dMIzjt3n5-Ztfx4Y;^h~) zur!~tie&TnXU3y98oWkpy>^>UPHG6OA`$j+M_?6+@ZvNCuRuc^-aR(iz-5Ti&{=2I z8m&&R?Voc_#;{1Rd8Ogv&BK9Ub91=c!l6*8cJOB5ZUw8My|J>sj(4^)!k@abB!!@Om&mpvH^! zB(LrFo=1GKvcYTb{D2xS)|0#@^25aa#mWY+RrvumUaTj1HS)v6d9kv=>*D-?8ZXw9 zyspm=6X(Us2Cv)m18TfjPxAV9ewa8fRyKJ3CO@FYi}fV0CC|zo_ZkfANtnI*gke1i zbFc-bC{}84J`(lv!Hl5NY=x79%3Rm>0?T-Y<4&2E4b2G*CE2$Ihs;nC`O?85Gn7QW z$%0%GKLE0MoF^fn`|UnwM8oHI&C`nJbq13qxPOq?3?{K39wat{N$jUQ+tTvXwf^qL zIUnPDrdjRcoY#K)E`u1(9&!iP{LA{1=h5zP@g9sp44yT2V2vm1OP&{7;EL*r0+*aDhv@37J6* zo;SM#Ydl$B^8AuJT-=`wV(|Q_JFv!+^(D{0y2HhJGKj%*=jS@KBT-QrPu7<__j8Af z^JEZ%=MnC}8c)`jJbeqC_PqnO|4i-l8Ob1fUZ3b1V+N6o-gK%bLII6J7H+gGhfbT^?^3@d(M|Z$>g0KeJDCjWL5r z#-l6ZjH9L5A}))r&sYE@fGjC!r&%eDm|q*{WY{D)Hy!woGzP_S`Q3yJ(}a^#7t9nG+N5jDixRyB@UDn3O#A%PkoM z2r0KX+?r;$I6U2I&sMXJEf7WyLHKQ|tr6N7sry;|5`pBAgwjw--IhxA4t?RVJ{8>W zkl|DUbJ2izrCE3A3kfH0U-)bZ4IswHf8Wfs9d;OIgcDHbIW z68&A8s8?8B?Nn;3>ka&dQM1g#Y6_fGOffX<&(3u>mRX4Zgmr1+BlRZj)sGv@JSqbI=q+On<7gxyt&Zf6KSP#XrzRbw~-z>mg31`iBE84G*W`d zmV?iwTFNo)BmiueschhX(yWvU;vZGL5OezQOiU6X%s-`xGA2nNX7`9pOcEi?f2N5tCP^UX zg)hv+BoV?~T8@hr?B6r)lv$d87$1AHhQAyc2a<~fc}@=#0=4th?f@=6D$Py_KS1cG z56WeR$d-f8S}YBZrGEP`m3o;G``3p(UmP!bX(VWFv_2=^+cHQHikl9caE!(9Z~#&) zRB&S38!2VjB@ibSxZGm47*s&EJ<<;BWpIPLEQU%2p&Wb!JEL%fkY66!$HpBRmCPH- zFh<%z$ueZ&gBC->C82Ejf^ddmLhDZ#o_Jim#DyS)EJR#R?h$nElY31I4=;!*b=9E^ zBq<83ro#s;07^esZ)Pr|5kjUI{BGWM2qD|f^N){Lj4Y4mug!=F($5frQ%*>;QNVF1 z!w3OnNx?xcNwLDiBT3-!cuSg~m~M~=4Uc0lO~EW?dx!M1yz}E!8#%l~0GZxtpBOJm z82G&=9qjO@Z~&Ps14#zmi;@n%vjGI7kE%{`LS3f9^s2b^LNRDw{GE95idCt#rf_OX z;t83s(@7SnptDxrzAD}7Oqr!ENSFt=S%8Y3MOa;?DYIIJ5WMo_ILD|_aVU}mhT?b9 z1Vt#42!*0I8OJOP1x%t3t%=1MlC|Z)0~U~on6O78WbftEdF;`;X_;kBdz1q^%)~(q zdn7{k-f9sojGL&PIloq)uGjd5yv$0eBJicvaR$X;1Z?&ijf%Xp!oVg?ft@A?57aFN zV)de7<4l^gQi*rqr)RnYd!8~9mKj1xfSuRO8{IK6rn|m2Z zGH|KVqrjE3@iK>9kEmEhy3FEgtt**OXvU$uFhmtWtX~q1lj*>|t(*{sTS!lJ%gg)3 zAw3o6y1sA>as;A+{=@>R9U|l1?O>M)-yd3Q%0N2Goog`(xM^+NbBUmQprp(?37{+i z>pOAKlKPQvl}E3+rrBEGtk5m)?R7X@28-JSl3>NI0)Mxd)y@E@X5}f7GAkySl45X9 zH(o03@SzHij*zNl)=jXWNdojNM(isGmEa|cE1LA!fVqaCNWmHF;^0BOB&vnJ4vdu< zB;Ass0_F7O9g{)T61@*wN`WDI!i8K@wMm~(69S1C_--3xXB=vY#ne`R;RB2d#PM^lO z8!oec4r440-nnsOQDpP%nj~;`z2brtK{4AU5t@D8cTtKcW0C}7o^^4Gpui*%!hHOz z;b3U&jfF!Di5ZFTh)s1Qkis{)P?zM)RCxO(ac&70Yg1sA zSwX2GIcuhD*zdA9Tq&4kN1_=o`GjFiiBmOT|I6b>qc|uQGuJ-KjF2e?EsGs5Si6b} z=dxl&fXaL+y_A0(v~>c^ZGG((aUajC>;#zG`a^@ZPJp?scdi_8*$FVW^}h_-IsxXk z-n(kRWhcPg)*l(Pbpp(7ef?_&Ty_G?ZT)Y9woZV#t#5ekfXhyRxvl?g(AEhsxAl!z z54h|EnA`fJgSJk9xvg(}-Jr{kfVr(dI%w+znA`fs*AKev2${d{@Pja>A_3I}{=2 zF7?$G^KNg7gpmEF_#YNK(!03Q%`3(k-X)OGySY^0tT)HK8yzPp7BcLSws`OEWyr!E z7DKWcL$WT_NQk+sap$+h%PFfd*V!+E$Z_`Tq*O~e7EcKvbMf?_7ApytluzR4zt?zW zAQkH}j1Yh;-WGRJ8YC_!lS(pV0iJTb8zio?l1egU3EuS%3#7K#8Et5Dxyr1Oj0i7y zXKui$YPX7xBn>Md=fg2KIsu{^Nh1-k06%z_6CgT~G!g+zPLn%7YFx$NFxSQ=Q)V42WFP?#e=rW3 z_04p=>K3)Eu+ewbVV7Ghz*%3jF0)!PGJpIV4!G3-(#$e|TR7jjEp8xTc^l6QV|PFbR$R)4-+#mf4m$%< zNP;WC#*ewcVW~+W39bZ7Z+C&ij(`-B;Ktx}pRmB$FDOUn1YH3sB|(bb;P_9)VKYAg zRR&T^0;a78+!2NQKJ5%mu|#UA&=^S5pNT_jrw(LOKxg2IH*_!QB;k?-Janf8H0|Rg zWmd6bim#%;xu3H@+PlW!+ZSclNy4PBW5DCS;08)gC7onY^yoErSwQXK72bbeW+f|X zC;~73QXDpRt9Y5k7>MdkTs02fSL3So+dRuG%lhhlWO&afT=`m@ zaC1+PznVs(SmIy%WV{U!M7A7MzZoy?uxGy_Hp3!8^`Qu__*R@k?(6x=tRAb$@G?35 zE_w+SJ8)neLW#~Y22b?LPTGIy`_C5!#US_=-0kWNCa zi9;e6{xI%5?TnC8by~2IVgj-fX-j{U2cFN_>BnkF(S(nQ!L#p+!w&~%)0w!Uk`Z_W zrPO6!Mfk#x;}m4{3Ym`||Nr2lD^@dw`9&x(ST2I%!urA3OiSV2UlI#}Nx2tn=d`C;pPH-5@V!KgVgrBaO-|e7w_AAXnd25_z!nm_;Z^w3-F)t z>t86M*l?7f5Qiwcy*bh#IVAy}_y3ag3PKDg{JmPY%z|nPG!iw3gj}QEK|K5`n10H00L zD`2aNhrp413lrjyUWx!b_Lm9OHZH7>ln{i3qDMF(NyRJTC$GYqHY#;rvtsxQR$14% ztXQ@${_HqbtDFbN3{<0Cwfq%tgY`tl`OAzZf7 z9{}w0tAt?jhM1UW;{&RB+@nmXV4^bc3QjW#t)#*i$80`*p+DK1IgEdzbi(v>$M2&5 z2_l~?+@563D=FP+hO0$8Il4L5OCJPc?I=b*r6!1J?wZ&FjNqS1t`qgd(?gQsn!qPG z>emTjJ(A*Tz7%W4vffyVYX~^mHQ5O8+9YP482Ja<;EYbeW!z2i0Tm(H2Zqgjmwd&- zbs$vX2T9Hr6Bu)rby&DziVd4oIh>60NEvwMZxVvcEAk{RS5-!MeYELFH>I+OWbVp} z05=O6m`QS&7z1&NjgBWFc04`8hmuqZBP7)6PEAbITN4vL%?c+b#B8C=YRUQV*Cf<& zrQ-US*hc<~#j~8Xl5JTbo5ctm^4p}y6jRdh`ahCmUyB5|BuTHpeleOt=xov=6l*qo zD+yCHlEgN6O106WX*20FM*}|cwGf1-{C9ep>%laf3hKoc?0+0Puf)9$L7BaE-H{?wSxkTZ_BwK8X-=9c? zx#8>6+mpl#BNlt23q(GomYfg2PC^}K7a6TO-}r`B3D#mYr#=)PN~i{Jh)~0+5{b!Y zI5$aaoP>hgl}RXigs&y3$T}bqQlMAZ-T$1xYnV-l_C8x}ukmr-`;ygFyfba~g?Iqa zoQ7$imq&LIF4^c404hnmiaEBkNSTZqb**uPo02SeLmVj9@yd`^>$p>;`N~+D z*$fup;iRG$hAnOjC*Y)&m<@;iB_WWC^h`8|P_==|NFBeJowY z=2qljGH!G5Ag#a-e@pQHxUFx_ZQ6+1SoyGcWZ6qG?hju7R^D@l@ilmur#n&4Tl;MuA*XKb6h(Ju20iYlVDVCLZ~iB9qQpMslxq>D?? zBgBb3V#I9D`Y^PdY{Th)PlyqAMxEb6Nrvdr6TCM`fv4n?tyTjIL_(y&kMO4?6*0yb z{AsBrT3RDji)R!WIOu;8N`PN&iu;uNfuoz#t;huF=MaQT67ASt2-%LZ)`m_YfFwhz zB;1o|$@ZepmR5D!QGrN3$&QqRr~f0#Z#FB@TrymfCUuJyLk?CaSt)S57|dND}tAwR?-e`xqW0UFyKdE-^C)-$-bF2H^ja$~9)|pqbecyYuUEk%0|p2y>4zfo%xE!UO^&z_Nwtu}K;qZu$c?ZQ`*-N1vb zQJkD?46R&bu+oW)rP4R2Q8;~@gg6+sE&Svy387j_l2098v_x_m5lgc&Jd1_d>fAy5 z%9ly=C54wxSD5h|4M{l$d^4CL-ThYJPyaf-Hm#l#$;H3~9@-kbMI;x6xBj$ka%_!_ z;iU(DEjA*$%-gKdh~YwZHy1iwwluj=-q;8`-sqEZb}Bpzv6d;;!g7yIX7l3VPd$9A zHZv;P*2^D?+ojcvw`EEe^0e~C3X18zhr$-skYAgxa6__itU3+wRP5qxBb6j zeNd^k&ieTAzhr$-skYAgxOj)O_A$I=3N^o(7QKW8hkVtE7L{zPr4JW9F0EeXH_xJ< zwwbeM`b|P?20ZY27L{sVb+S;@;fFh>)JL(%g$iHc=@k3I_?K8N3Mz2nPFsV&h~?t& z*Zxn)?JwNkU8wjXma4xNj0${r=M;aH##NF=Q;a$_?etudM}sykc6E$jK;;5nsg6m+ zdQnn?dv;0jr-!$v$R)~s@u%-b(Ss$*!8(g5ZF#$J)~+d@7}^Y;P(CE71q%@weEIFx z;G4Upl)5n5YFGS5$B&k&=hYsp!DCI7tJB19fl0tOc2DzBYM-NceGrXDfzv{U7^)#- zI904kiZ3s3J{s_pC#U#nXecMaP6ES}GIf;g3?i{5EWu7sNhz(@U(3_UoJIon6ci7N4E|%oLxE zi4|@0B_rpxi3>QDE(4W|eZ2LNho#H@HS2`RwYApCWA;v~llkq$6$=aX_DiLjPmSQG z;}(WG-1;wBA5^NXvp!CH*1ux=pi*s}^|9pH|BCfNrP?~{W1HtB*GF+UI6}>LajZzX zB9b6-2*U4EObr+9r?sP{1?!$39`q*AzEt7j=O&kYxPUkJx7L`y)yiZ;nEhzN-DzZs zy(WGAz?lt!_N5B%U!LNd(FA`hYxA9?>LTF2)!~|bl4*@6{F;D|yU*ZP5$NaI?b#~b zv0rX>+Z#=HYF8&E@h1j${)4mwL;I!_bkrJQgS1~}0-N0o)b>j*=9ukw_k;Lp2pP5f zSZe;zF!sFU;&`6**W`JZ*3<6xvCnj|ZzMkR`H^2AM&a@MC;MQ`df7}~7tlRBNzIb1 z&Znfw*zlhz1@b)p)tSktCxl21l8xKc+le?R+}D-<~j4* z&@mhwUb07oY(CnYR$=4?$zB-dOUewk5gU78mKl+X;id;W4kZ&AAH@yZrf)(Hc4#L8 zo81gtol?wsjsr;tOtD`ftG3X}+x+Mf2zz~l?6jHn)u1qpJmArLe6gAh6*b^$P%sg!lS^zSzb1~+#;)r-Zt4vqx4Sy=}D-5KX zM-r~E*cjF|4gZFrM4CAS;mnd!3UTCExh)`gHwrKFBw;NH?oCw}1cJLi@rF?p){@}t z;S}p0y|p)~1I=^IjmvnSe6bD@+R|3x)TXcDF?8rgcSQ3mKS$1;FbeZ>ZuftOZ<|JVM5Y+8vH5?fV=b%fTY82UZla^2c<9w z7xhDZ5)NBC-|P>cNKqf9h0ff=xqM$UFFX-594tQ>8z>|^ltn-Hh?s*u84JCiXH?6C zgcS!TS6(q0 z1fjYzIWX|rXNP+1bL}Fr4HL5kGY{WM^^4Q%$DkcU)Lc5Z!P+rv^A z3*O1?> z^NRjj{zBG@;4@hyuO-0_FO+_@AQT#1{D1|@G`et1hUko6W|#x z2?YSP(JP_cY_BVINmx&V-z4jg*PBy5x?>FcSrUsL zp>$MY=~N{WlS=31Qz9{`ba#pp#vQISF+CEC9^n};N~Y(p?NuA1Oe7XP!WGF%_@&Mb z{9!lz+!acNPjK4NiFU^7!Dk%HEEPV%4lhnr@d}-}M!j392Bt_TnP|k;XdkAwVOgPItb5M4ZfGcqCo8;uXzH>FAqB(lUN6( zVJrqUwY;VY%Si=}O}4JK9TJnx;M!y*Z*<1*R(j3aY9H6WrbNY)d@LFMGZ{rSO=8mY z0mmkK%PX=N^)bRM$!bZkAw_#6jBW`);$|xRDjB0x5=8S>7eTpbK;gK=(!1IwC@0(J zB-<`{TJ1^)l8!YCz7|8n?*>tS(XOtq%vIanj;KSDi>Jd=kB>QG6@9R_yrJvSJApqt z*WDO?Jd=d=G?GJb8-$kHiVy2#@&^Sb##Xne4G!Wu|cE4jxSZb zu(?Y>#s*%3hFSs~7Nd$U73lNWMr0*U4+ti|Jp4MzDt|E@H)Jr}NQNNk38qd|9IetE zWf7Xcl1T6XUr7=#m4pZLmWzN~G+^6R2}HdhrbXdMOyF|y4oT7)4}_?IBosZu*@jA~ zw7!jV(A+gf`Ysx<{3N|(-jGli1)^N}3}+g`#RPWB@qR2JY1uica&W5wUK(rrbB$`v z&+0H%;V$xva4!^15r(T4JYpF33L%7W0_CeIu<~TxXGQ!JhG1$2pjyB(>*bMzR~rV# zgF2vip}go3&Ob#`8O05-O@6x-HTGdo1lIFvxL@2g*U1xGJz@5t2=}FN@kTXB;aSp% zB}3`dlrniEx&pa`6wFk3pMp}N0|${?iZt=gK5|fIh+LxZNV2g)B7)fJME<0mB?UvL zNv{~?DRh#`6PY0GEGg)wn4msbEUFo&G$IqEoh1dIPcf0@f--AoNx}c7n8@N+X6-B~ zIPGOgb>WqYlPx@WpWv)5zbuL2MQj%tDYz-cgQJ2hPok|Mw;*05#*2^`Jn`j;u{cC+ zB|KkAa`G9DPEn(dB2*(e`3yIts7drP%gJYWFj=j{EEi8b6IEiqSQx<>rzcf#5j(0f z=i?vniwrr+C%LD+A_=cF)o#s|S%Ch6S0+m0&uPSfyI8zbX0nTm6TB_af5nozQIs^I zl76-v{4mi>P@yEi6}s;kNmaiHE0oACvI?C_@t>HNF0w(32(u3t6)(6`^&=f)Eq3a2 z(fr(_eyy^;-d$Z;<8SQX_gYb0OM=t>O$p=}KuHG(x^i(`77a~Itg6p>#S>e-=2VB~ zM+m0Drz^ncQ;ipg@#t4Hek2+tkQ{mvyaOx7(T(zQ?*8>>&<#+HJ(&PG4VI-T1~T43{n-F3m|;<`-T2=wb$sr z2nBsw&?QV)Etocmj0n8pBF6z0z>Ng~_N59-Yl=^3|8=C>uGYJ99l2PjH@mG0Z_SGM z60wwCHAk`YO_ zKNOhFV(JZf(uh*kA_`8buJKP@jX!L(t#oIXV#P9?C-dij(fveai*MO;zwWie;7z9^eF1i;t{SyRG&Z1thv?A zV5BoTNae>ZgqBkncA7T$S}c$HdD?TCv2uyRi?XebjN-UH)o190k%J$OMoR+#f`_vi zkB_qy?!tgi)^w<631wLqD9stnIW;iZ-v&|B-*liO7#fO6)7vT0- z2}9QxaZ?;CR~eLy{u0@dc9sF7gHy{S6ibHx zHC4R`nYIThTSLBq6xzz{#f3(Q2bU(d)!v%hl(9_@+Q_1VB$KSt(8kDUhz?0VK{wxgjn)nXits~h@bg+@&u^vdymIH?y z0A6vr)t0*lWfstqV1uDjOsqqF3!e1C(wcr(CE?wM0k0UjUhG|yfR+T`HdKnqRjisS zvtk65%IY7632)4BD7P(=il@VV4ZWU91zjl3a+$UB%fl(AF)uv%AjS`4&0G1MJVpvB z=N5-|8m5Y49GyArkY+wv*kRUi-l*=39M%ZJQ3*y&vqM2U$YGCk^GZX*Ff+5L8PrR|d1_fXaz+47D)3XoG;V;p5G3Gjd+UMwTvrI5a2Pd^0 zg0R+r_o8!;{6Z4Ud`T+oIb}NBVJeI8l@etZ;X=wpc+OhANChJVrD&OD^&peE@KOT^ zZ#fmdkuqf#iY3D(hU#!3kU6}y;h@t^-{N9sd@6nhEx!{^Kbr6vgMe2mg&ww!t|Br0 zVu!*0rW8v@Bo`Yi*rBb5k5>|J2#d#U#E^r2u~34x>9b+uHEFRh#T&1Xs@~!k3paR& z>BUm`uDDnzmYHMFAuSeCaIfjNLiEDPzr&?`H3c5ukxhJ<4xs^mk_h5P)fNa9gsSkrAuD%M}aQPm_W9jbm5lB`^fM@s8yv4t16h|EDt%(VrE>FkJ>EI4^(q$}u zp@|8=8EM!>wS3wQe(U_&#Kb9njefw1F+?aznUh7~X2sr!IF%e#yypdny~V7WM3n@1 zNKq}~PXHr5y8Li*W7o$6S0ocY!3)+&zZG$FFH#uB&2l=_LIV6Jygy0QE6`U-=J3-R z72L)wv!I#+Q|sBtNW-7>3S&4At4?`%8EprbNO1)*tWuIP5@DAOaZNlGU^<_?m; zn>)U$3mR9C3{h=X{0-F_-XKDv@)<_Y(M;%?;%85lYNJtRL2;f4AK{iXweV0J39$-X zer{w1$0K=%W{J*84f8dWkvoQm=q#W<{Xxf?SADB@MPkz>3+{Sw?s@r%mAqPYu3D?( zueG!J5O8jBxXXpcc-voBnZ?sbl}Wy|Qyf1^8-2@cz*!CUJKw=OxS`(ebOTQG+xQ$_ zA_LB9@Er$QMc&2AL4`nRV!0dFT~M9ARXLnIf%GF<8AojKoM zv*m+_NG^nBxX;Nu!#TQ)!G{iUTmZvx+C>iSsxVaTR>Y+-TfA)%(S@=Oce{9r-u(Ec zvh)FB@G(Rz7sNE2cdH;AaKO%x?=DgViMKb+`pq1#--B#RM0BC7!^JLM@)lskCW2dh#D3VxE8=%}Vs&xL zwk+aFJ21gA;+8ZHM_lgEzQmpxeXJ^T`E{^!8)C^G+R~6iRMA0PVXBvJWD% z3&{Z9VbD$@t_w&24!bgcyDN^;uky0GP24Xd z&|ISMUk>JRxj*N;Xwa2JEEmKycvt1G9$fCv_mVMa_Yld2ungz9c_-h*Xwcpvk_%xO z9(3{!w(ao4iYdFjgI}yAmJ4DU4t$ME+Zye3>9Us0H{?Nw53yVj({P!Ke+t7qiec$< zPFn}T8^O%A6k71GXEr2fO zFoX{|)g0b4P2U&IcaCDxv1M^r z8x?qKF9lrxru?m~I7*+M$m%3<>4HFWiNa;qxR}q=xuJOi%_Ry4T$|r~X*~F55YKAF zcR|yK3#38@oYi2?ffoOGlN9jBn-4nZ;RdVtBuCbmH}Kz65z7TJ4F_JAzq;|a%dQnS z`28{>ywEJ*(rmAJqqS!y43* zaZ-wEW-4sT#u^@;rMrrwg}9_4g}=XmUjvwG&93TVF|h3tdY{U0=37!qSwv&ekCexT z`SU15&~In;tt%Yp6NSwJ4Ugn5T;3n85@L>H_A6nCe-(Ht_CdZ{nWPaeVPnSjVrnYpe6j?{e4U z3Y=r%fI=)b7dpH*e-TIZQ7!ERP~LhBN=PI&6FBU2L+(2C_{Qn<$+>Igk@#HsMhrFwGVJlry!9nMeo68;2Y|zK>v@IQdb1MVje>WJqo$ey#T#=|z}0u^ z=d!$ut(ZV_3d5>*W$DhNE9O&J)~r&!cI3wNaJ(WX1%rsuPO9KC;b@=OHK%H$0NyDGF%|^ zOLW((^?pPW5cebAc`KWWiz@dcc6yWe#ak4N?nlHFc*%!ZQ@g#ch}TC(Ra3&BlCC#z z`=n0*#&69=M-W_oc{nH2dU39YN8in8Pclz<4hm4bEu-|qMY^4nZhgZVYj-=jnGkYc zs&I1_CvTJ&(!!Z0-SJswtynVrIt#1d;d-WztDDS~zfL9aNYk1H@AycD&pf#_M9fG& zTGSa^3BSzJ7mF~YN}{}cf@;tDXr@_Q=*4?OaRG;PNnS~SdY1ke2M+6!Z1@B}%Tg>A zd3Pyqj1r!M0_^dzjDnYNij0>X@fty15(Mh11+{D@@|)m!nGmR}7JM#~NpSWhFAD+` zr%du`z>Xi!s+E%1Rmz7~WLlybJMOhf*!ja6Kgri6Jy+w7HyFI+JwYH|jH!4?f z{L9_Z6C%GH=2wCPKN0ty!onNEQJvtp03)kbMx>VLE*EQiq=KFPEb zOpnlY#noxQ(ri_zx+~L->J09e`BNQ!&h1e$GY}{^D3x%jFxPG8W<^-}<>BUZ)5BA( z9{uJqZ|^{GPAYKf9ch)-_q>K+`)TYFEgpn-=m!_KX zhJvtSNRpM$@W#)jmCSM8LtrlRo~M64&A2kYkYF^vyw9fPdE8r8W_r~&UJ_3-N&?)k z>XcUd)j4n2-%wjkpWLp@vSgWsj6^u_3kpS69AjzRu`hFG_yn6`l3p=Fn{N9kPa;|p z+!a?En({+Ty!;yTJ5d?rLK|0ugP1Q4o`x$QkktkA%ZFyp`FC-VlC0BI-~{K|aH#9Qq}c zryq((7^I}9!N+sp6wpup>>OR0Pc9%8H3j~d2Ltaht9L6^+~y$+Qc+Xj!2if}q&Hq` z&25amypyIS11`))9HY-;3qm9vKEa*Ys@_OQUrydh%}j+IzL8xpZ3;{Z$`rVqtzV!` zSNzyQZM9!pgM&HNxhNFrTC?ElY*cSlQx%A$7fXiU3AA_*R~iYMSl;fHpxks|-)|rX+rWKG#8c(v$E)AQiMr?#P4e;VG%C*d-U;T{JuOXM9=Ut zSAABH`e$e97Yn+5R*?GFxazZl)c<6b{zyr;&k9oiyRQ1IAoWolc6bcc$bCQA29A3@ zzzZ1)t{)GRNDI3HL9d^ff37Bp2+bwvWd~R0TezV$vqeK#~CKdQ=?YOrF zJ&{V!1+maXsUN~r{Pfv$eWo(kZcPKey%4-GR-qiYS0puSxK<3m!v|ref(2d!hhc1^ zd|-L)t^ffGJVo5CHsIAtO10N*!CNt*QN_>G<4(gAd;p)iohiKPKm6W(8?M|P@H`Xw zv(lk2G0_4`2i}RP)gkz}%p{!(Ufk35;m?@LkIZTNCq-q7yoN~%ChUjLM0+Uk>MQpo z^t!0U@-u_XDe=5%8D|Dh-+T_|YQ?PJ$OvC~u5<+#2zJ|7NtvzBw2AA?^OVHyRJB_r zZQQYMPr*wMQ1WWHr`w@Z)R?i|FlUHeMGHC3rK{A#l|2cDKMI~yr+g^=Kiiqy^; zk@h+jK_-M;Hp+QXIhkpgd5p*mB0(rlZk(%jIuZ75hqJzL@CI0WIzP7Ar5oUmHT>8S zPTc^fV}lCfuGVe9*Vn6$&FZY*ZOzsv;SlV3q}b#}O8e~jlor%3Ml61mtyk-|+OYnT zsM0w&$dg#*9Ju}zG_MA__X$5*HrK{c24|@LhI$uU9KZ$JQ)jTx+!DC(QGoq11&vZ? zFxSJ#g_=m<2mF-N6fOoL!w$S%3N~gcQ}uRtBTmU0UHG=lnDT4Yjg@YF)`wqWs%#2R z#+IdNRl?&R`(ut&i;@o&?NZbzodKvfr(3X>Xpnf?9Y4s1oq;ExXX~9>@Jj(WN@=&) z+BdX&O*liO2}+n&YhZmsN2H=a1-mcoyf#tyumLW^^k#!)u;}lgr`6%p$M6XiI+g0W zYP}(dd|yv%hO)oK^bXZtg-lSNwBxp6N(!o4)PQJPzfv=l zs3**8p08jkc?9^`p|JD{8nzW|ZSXBm(GzB#Jr;_KR!j#B(2res-4TjxS47Bk9ix+D z6b^WCs7Ea>=ybS?e%pzPqNrS&DFk}oS*7F#m8)cLf3lJ-X9*QWF!2NW7Wv2T1O_Ol{N6cLeEp^ps0-511RgoplqQ9Or> zA;>=K;&`^`adZY(p`6A*2&o+I#Qr^sz<{SPa`{zCP7vX^U&C$Je_t#Q{(75|Hid)O zT-&elpuYMewj2t$_o+F0{0AL3104NP1;8#wXzx+Ql+ae89`i8;$|6HJQy*7kMBAVB zY0UcLCzOn)m`HS(;VC$qj@%gK)SESU(3*pm;q9MNb?^Z3k)Kr(CR?p8^u7!bj!N9M z^GuX}n+AK{9o8-N>Z0c0dp`^l#h?*%$hY68CkH+71NVo?JZj)P8*fGA`mH?>%L$+^ z`MH`S)@wvmt@aypejD!jued;f`m5inS)zGsy3hP$D4=xju)`x^D$x`sMKhWDTbLti z66b7F4SX>WGe?{mz|Ht?34EZwEsWvlL2o3JRh$pbb~gYvG4UW=)!Bqo>0j*}q|vz3 zfq#UL%#|jKu*)ui1pT&U3J)>E@-PLb3v(#(L@`FP9noD}wFlQLr-*Q(j(86wgdxMJ zT-nG6#_``0`1ys~LKSBw!5b8Q6qcqUG&K<=Ll?7Zm>$fyZpK-_5;**#ZQ(x;j+AM} zg{=`z5jymUEr|UazG?|)<4k!CZoi7Nvv3k(Av@0MRN7s730sdZ8c`ZfVc;qw0sHdK z8hDqIT4~NzXV8;B*$pQWJYNvi9-7$+CjlZo=zGG9)lo*RHH%|PMC{%u1M6JRx(H_u z0!)lYA=dRz3DZUJgy~PldJ*#6ydx9e5%ndqgj=Y>E)x}|CR1;*O1$Za)^5| z%;7{L^c4a}>F;^2k}iBf5de=_9!uwO6HePNB#GVWMtpYw5F@cLC?NQp9$NioK|u*T zDq!v^DloX;0B^%klLQ3}j}`F9L_8-naOgowp73&*wFYA@0_1?c0#le}5J0%yAIB(M{401vh(-?aI&dN8M5s|7RmUyX9I6re@1^)}3G8;)w(vheLki}x zv%MxZCb$RlBV1AXSxOJy-Gy@l4{`>9-hfSe2|VFT+rr&gr{mrVrPi_42^b+7!lcFFXLXavbWlF2j|;gURP;_xg*rBs!`)~ZM0xib!u&@C5lh|?-ML`c>Y|gwi@1mPt=2A!sJHktl`spihNoqfw~eBFlqjKEk3XF+nss~e=dhhT9+@weFL)2Pj~Gd_9R%! zZXLe{PBQT9FqOM?vBZbz2jhEt(C(qmxkdR@r>xrfQ&p2M?6E31~|$(3Saz%sS1I)doM{-jO)(!J-H+jH!_1wnPOYos+?(A3!2zCEz6tB&xX5ZSkyy=eiiCoH9!nE!`)hlC2bVbsa|kC$ ziI}xsJ2XJDvj@gx!_iKZ;3Q;Gr?G zV5Al7A5P3y$rlb!LQ%Q5j!2Z!sNej;BspU4&xlI;O*u&lH)?mrhC&(s(Vw7h8jun3Czd4#AZ_tC-5`?9-Bj z6keZvS(1>(>yeko#G)Za=|_vNNEXveFn2~Uc?>tOL`TlMGSpQ65#}pB{H6aE=uGuy z=eY0My%MC%L`gSoQezwj_JwnKD)F>A$chN)LVwMbbOXMyWzL znjTzd$q~zYsCIudQVVA2qQ70~CF4k+v z&|lA%c}xnvcz*OUQNtsTc;^?%?ft4Kb(%VTPP`wN2ZPALmB4PhwULvreYJSi-~$bu za``19B`nGXmx^aBh9E>HHe43Gp#(8WX|2men)KH+t|A2Mb8_F4jhu0%$T3QP(p4f^ zbZlY=mY|P%jY#J$14>x=T9H7f9T?ENS|ljjBqX=*>qK&Jnu105dc8;!uSJ~nz91=h zq2jdS8-!TkG(ubcMv;WBFdE_0-z0LlD_yIS@vcxMc=Pr9!lq*7#xbF7uC{c-x?cO6(gUQ&VGD(TG-@P{y z3}4pS)&uX0($z3r`hn>ER&+>2;h39cVsOS?ylJ#`r>uWx6MSOwUkadDX|GtiVCIyNifRJrgp?*gf42C*`cIr(-Sg zp@3POX@~e#o&&X5=yoZ2J^>yJYExAr7gz9sRjex(tPyp&vxa@`N>a+(S zVS~gkFHsYN(;dhPF2H|F;4`}}g)d{7ZxKX!9}C?!2d4-yksx*xZdQ}Wdj$bEMN_3>H zF{bHHwW~N;rIVZV-|0;C!O5kt*HZxAh)?J*-hAX9>jcpwi3AP>-m=Cnf{$u^{7H&@ zZ8hED!#`}{OoFd5{Npw)h2J22&=P3IrA(m0Q>wuLCi-buA}UywtL~(|xrqaCWChN} zx?ci+e%(@dByc#Eg3O5SJ+u%^_2kqD&O>Ds4N`a(Psh>j!eq2uTZ2w$%yzi5La`>l zslXpJrRd06(2@jJ2WaOe_!|ST(}l-i*o6ta0}fBc6g9@4htD#`4kmeH;r)8y@PMzs z4!6TaO&o&m)e9m9EwOF?`ls4+*%+u6(^ayN4 zVhGXl@oQvZ_7BA{5oC+{6o)&OJxjHVJB*?z{~=;KP!O*TP#RKY!vmck_ClB9jxWD6 zz*QrvU^K1=Cr&@1=HgNlzmve7ql!)kkM80-3G9N+Ch{n4(+m9N7+_JdSTo0Pfc-FB zm%wknz7*P6DqLq~p`i9E__M_+_-_=U`lOz|67J!8qJf3!c$(oTOu=J@d_uMZuaNmD z1DE2X858dkBEk4VxxpCugCJNiC3M&=qPJ0GiOmrFRMfGN?!mU$_fToWg78mvu#Mx| z8Ep~3`;GkI)~u%lH7hMg1tw>G4gW2H)Bms(p1X6@D+IRjL}~}u6aXjYWI0F`S9v{( zY3tkFP-I_pD2hz>ru}vsPtM|(2Ne~pu5PPGXFlKj7&U!er^)Ya)9~8ItLahl`=6jD Q*8<`3C#tDIBJzR%57~w<2LJ#7 literal 0 HcmV?d00001 diff --git a/crux-mir/lib/gimli/fixtures/self/debug_pubtypes b/crux-mir/lib/gimli/fixtures/self/debug_pubtypes new file mode 100644 index 0000000000000000000000000000000000000000..68b4e0405f410233a242a8e4d17b0782208ee947 GIT binary patch literal 52984 zcmdsg2bfjW)%Kb}Vu@k|tg#R^$V7|*8;s+qC`2rb0-~aRlRGmPxN>Lic=s8aMh!xu zCiWI2cI?;_OJdYmBiK#sU1E$SmKcqi#Q$BpoxS%xXXXrCBhT;i;92Y4y{&icefGJt zWEN2^@qgcVu7*YqCK_EXm&&vN(wI`YJr~i95O;5BDd%Sta#5kwF)=rlMh@|@y>eoY zX@z22Ip0ACa99+T3oTtyeq63SPxpP#K{!{MvbBRu$`!lvbT^=V3sF?ekM3wIKV~M9&#;LR8k!y9QE~)aaK69osRbq_O@(i4N>2L|XkWR9OH= zX_#7QFBa&y5zV_1Ft?&Yan?fH)AaC#6?bdlY}qh*L@5>cPm ziS{pv0uKpo1Z7-)Rz%yl=x8XrN7ROgGGRR9=?X@Rg{d7}HZ6s=N2v?aN07z}TIUS| z>;P&1TshZH6CrW-N}CF8YflJE=6i(=BdhDsw;DkQyH^< zy6L!W){;%x7Y}aAbUbxafZK7vXqEB?9b^FWuH~RmP65np(i0RRJIC0tS4`wl=NiuEU#}?e~xmI%F&! z+#%DlXUo$#HQQrVwioDZf1f>D&lgP1|Cn`Twa>?ct9=KaX0_=FJ|x`?eRqINyeA%< zxE0SJacikq%(pV33wQ|m*82KM`PPshtV8EjxiPiQb5yYlbg{Q370a$ksgnz?uWypi z6)ko4K-t(|;=zr*70;lt-OTvZwsfp3b3GXk&UG=KCRd~xQrAfLsu-{97&Ys~G4u#^ zQ?5uY_%`H&`T7hVT<|BA*(z_MhfZ|17wH3b`}(r1XYt^&KE=~CQa3?9QtNutrZW8# zcyRg;@HFYewwtxO9IRlcVQM{A-AvYFFdkfw(RilRBdd?neYYGwJupFY|w;Ed(qVD4MKfjgLcJ(yK2qtp-Nj#TgGBTEKamygPf9$ zwEYe>G_KTx9mU5hdVS&P#l=vs8Y zQEtQ`hE8*4)Z(+W|+H6OXr#tSH~&RDi!5gitt{>bQO!Ex>~2_qlTX9Hk6NburWwzq0Vf4 zZstVJw#Te^v>JBS&1-^?GV$D=p%#;%D^4&(lg3~iFf`H$CdR@zhUC6RI<-NUb_oPs zT9%b-=YHz4+^|uP*H8#@y!L_6iz2E@42?8qS6!b&Ajta6g5cFhEv``|F!D?seKyk1 zP1Fl|h`L`_)JQj)*!RV;3w)HA z{jpOfdkxa)OFpV9TJm@2Uq|hS9xA&u*@O8yDzpzOWQ{-Fq^bwYzVg)Y3Acu*87)2> zY*FaM$=kGidDwRJU+7BFl}@uw|N1{9jB!k8_(u?72q4!lSzBeX%jb5VmK ziiK8qMv{}T{IUKKb7F3K{vf=dPP8dKzpleulVVA9l|hk1X4DXj0KJ1}>$F^%K7vpf zL49V*{Rv*3f7?pKA&o6z*8=na$W1fyQG=`VRydzE7c}jgC6Xa2QRDy=`iH-%TR;CVkY1sEwwn~Bv=mYL)qZaJy4WsNO+^>?r$6EHAKZ~QgzOHh2~dfc=3bCqkb%W1uy_f@XJMASgdeE~_%zB9CtxmS8v0=?UGSGRqQH2+M~#Tl&1 zO5Kv#XPK5{HIgfJb528Z)XclyNdJG#xH)I*h4j$^w2%+NXpQq~Xi}~fJ9DWk_hWN_ zV>@sSrXyEW(RKsEW*1jgR$#Kq12}9!+e+=Spho)qA~wwypoWWUaGz#M4mV$Lf{UY! z{Y&ryXVYBZ8Lj=L^(=cpGU{_tEN?7%^+&Klv%LuAOT8 zR_~x1`F7kkTr87#Qf9Oxhq|{M;|d>62g*R) zprgoS!uRRuOlU2oQjyMsU@ny42%K(bJvf|N&X;bzd1qBe&#pkhiN}7pX70!=~U23in!L8jhXmTdyDJZ6q)=Lm9f@ z=rvO|jp)Do;>o@?WNA7N&6N6Z`o5ERcTOxPDecW-);tP`Y~Cxfk*%W>97G;A*U=T6 zW2Q>c4V+ppwlIbBg0qeuLqz?4fe#3`ydNs_1}n2!QJ3qx$%7t6G*fr|YE(7hGZ~Oz zMct)XM;kn#JG^SNPv7AztC!vPklm2M=n^n=H$72DW4T=Jy74S@6O|4{1h@bWuLo0i z1@0sixNx-|)LYkHz27M#b?oub=}tu*_(Cqj(XZrGJ@mMAUo-D6tL$G((y8EcenG zy8>CrZgD`{xK%4oJT5y0`?VrM<~nds3w!X_el@t zddXtde6(U6%|sAyx-zmPTnw&8Fk6=I#V=i$kdFDMSM7mZ-8<$o8>kHCrZ888|?5HvgpB~Fsw_h966;t-dPk8 z{v8q=D_$n@gr%hmAAsk}_=ZD{hWd7q%=pa*8d1oyBvV^sHY5T^65c zd}61ok*?yD-M5H8xV(v@$#^|=`#b9EnMM00qSU4|xIw4NBnN6=G zB$b1u%(ea!`>ljnc6QQ-2u<%&$)1o-Pyl`bgocPio-O0gWNA=42eUNiVT;y41S8s}{ z;2z^wJf6@-*;PINyu@>yl#Nd?SAu$Fn3=D~{m`B(PiME(y+9k}?_Z342Wj~%eMJVM ziU;qjJi(KK-w>=NrJ(MWc56MAU<7TJ6knUsAxU8w&F3(gR-H-JOIb~>Uib7KNF2Yl zSPmyMm>A(rS#Mm^CTtal>M-t%|ItFDywmgDZyc)7OVlAt3+A4(jWan&x&h8ny9zpv zgLpx!$#PC=OnL9`LJkg!u!JjrLFdZuV(`qS8#zYznf$ifC^DCOIg0Xvmj9bvdYE%j z)1fnh1?t^(?mx$=J4zjMaBaK39v|&2$NlY>O>0v8*|aYAJaAlwR~1@;4*9@TM0HB3 z6Hcmf2sD#?grDV9(|MJra7bX>cm%Z17l#1r6%3S*rdl)?yZi&9J_KF z!LM(mZR4=CNh0Ho;CMWt(|X8!`_$Weu)A}{sr_zp&HP0b?alf1P|55EzCC1=eRI-C z6Eak+cQ@jySh>Scxy0?IbGAx(Zb61QkiB~|WY(K5?x=bNtJuXe?f6I2)Q^zhEFy=m zI}l33hsvcM2^{GtTQ-P1+BTGyU>&vT5~6ophYHDTF-oCmhxKJ0%g~4 zVL7b^+Gg}_Eme6Y-j$((U)wVIqZz|2i#hxHqbD*n`fGjRM$e;$a%U!|Q57Bcf0!;+ z3vrtK;8}hmsP4n{@OjR^P4cukEmxI)!>;dB+8S3B>3$*9FFlt09B6$G??tJDIDWS! z)+NTs)D6SCtIYB&>$HX~PhQnsEcl*kXKrlO+4431MQ^iY!)SsdHksi!&FItg96-QyVE^{U6gDSS{3?AChf>v=9 zX(1Qzm6ouGOGq9WdFbEmuO?rsM;5oss@SU!=vOsdy4@N`7a~M zH4RR3vKsCnS#RF{q%Qka|E$PlWv^%z$8HdsT(NTg7h^zmT=Lgb^sZbqfA+E$3-aTv z17vL&n=B;>wn{OWY0gcCVp6!h1)y(CbfrdeuCUQDsj=d_7e7miV_zMr8z!f98;2Hf zaHi>P^IZ<2wP{mV+9g~MK2~u2ukd2-HHee68M3)oaT(mF%li7erizVvi90#uUE6yr zG_Ld}=lxY>{=UpSbiR6H;;H}Bdh+1LQx&^(Yn`n;aS1O1RI!{rQBJ%*3d=q^@Y)m5=O9^;ZRBX=HzkDH&h%1*-$r@ynN z?>By`%MThRXIm9(^Q~u4WKf2)OY8Qo6m7_fnT3K3_^bPje|~3k&LYcK}K z;W%opy4UeI=-fCc^%@nTU8JHV-GJjuA7JG3P zi9{Wa){31>I-B~`u{=PJWuoKaWKhL1I6@b>a`ZqI8+JU8>sL2~^b5}at1RGpE+Ese zWL+|WH=F5$q13s0n{5B`WJ!NO(s-9wC6l#0HA!n+6{q3z+)d#${7MIm-jE~nzb*4! zJ*1)K$~4ES*xmd$jPXZF@z;#`FBv~yEB_*_`f+6%2V8xak%we{l6f9v`;*L1Lcte% z&+$Y4UDH=t!r|{Hc*>t*?pLETk>70m!f13MBpKlr2>7jLoK%_LYW_GDH$sJ+Uq#y- zD*s|rorc5HBlC+*k0acCXo!l?!T488uf9-2pCj)Npt)c4tqlyf;>z@W2A<3V*{tZXMcg)QZ-OZuR zZMpKyLPxX4mI5o5I;IZl>Ja!ltNFG;1^j(g`~eB_fDQnhBI2*1&*xA?8-YBpbYsc_ zYtZ_3>De}8zm=bbNNa|Wpe+D3O_9j3W6|F4j z^;HZmaIH`v^t}KP^vAxciaTzP;HRL-HdZ$D6$I(Z7IWpP`DT;l+sOffe((*OVp^Az zVCO_!kN_UF=31xm2pkhDg8t4*0e=UkjF8V_MbM@gR51s?E{=d5AnWuk7&eTJ4+uIn z0CcCEmjnyC6>ysn&7I|ZD{@-~RdsUfw62cnK;91!K?7GK!e7+QcSOw{r8aKIW&t8- z+Ui7|<8gV6G3r!R)ki#C&>?G?s><+ko?ATAfGwQk zTt^dhv(aSRc?NNc7l5jsEn_J)w-w6K?B-%Y_VJskVS={rXVR$=!FAl`7W_SOWVvfv zkf2Ej5_(PNAjj)mgGGRH{`2}4Q1C~i3+-}J#)_beq3E%WRLk!IqG~d&RGKcUv9>|f zGLLAEMcNE>xrH!P?FH_&XDrl~x8PeA?CjzNh^1f=fj`{L3Ag>0DG+};K*lKoQJt$6 z#Y|7f7+W2kj~pKHbo@5|Iz;GH?%Rmghhj&nSvR}hg7Gl47X;6=(P=?~oM&lYg_Ote@sh)jv1gZGmWl+t|}Mlm2*pg2zrduC>sNFiN6NLS$+cW$gW%& zzxP|{;C`(c=mCNb^MO*!5r{Rl5)`x)pl((Hf1n)E&e&2JSZT2@-$~VZW;xf{Db3l_ z8j-PE@bL*-me(plLA&FRs&}fdR1jrfXm;wx@L>#>I&=5~GtDu2c2Z>g<)OAB>Hzab zTkUH5g-4@+KMapDh9{tczSzyg)layc4o*r~w}WXxAw6HN?FNNH&hF$ei7VNH3-{Ka-~Z9G*0W~FWv zAc8&zlISAz1;r%<2-+tO@LYsRiMXIsfLj+~L97T`9VBU8gn_XlXf!Ko$tw%EzYhqy zFaWe$aCH(a=t01aTks52)iK5`cs)P_4FEa2aU7!BG%ZNb4mPNg7I#Yx6ZB(*S;{&g zRs>xG#iy)?fcTU(796h-!%xRr#8TEZ7P9R0qJz{8fZHin}5UPe|-o#Y;9%G%q;*OdbC$sj>5A;|UI-tu%o zmx7K}xaObhd_d5fAXnwNeW)ctIWQh$+H;$y)!1{pOd-stHy9E7Fo=%l_NuE28V;6} zq1toX#nS~nWOU_(s$Ti40j1~mk)vT30*;K^ZYcNrK=s0QNql4yENJcBu@WVTZ=@wu z#7No~d=9!AE<@tW0z}Y%I88|6YmM?aJE9GI0Q(CcP$Yg#B`9dt1ZA0d+jcz5#^6h) zdiHW#yu+1mB%p%+^rFfMZzeP!UI^=4OIH>8(1;C*_5^hW_PHs91Wf_t*yl1-W1oMl zWLkiyzqId&5xp0AFhB%-tJd~5ecQ@{P6xr*Mzi_*)n#{6FzAn?b~GR@g1LJVp6 zEMS%7wh*W#!OKmQ%6K;+D1kyE`h9YMpg!NQDLiiJmxv1*1Ke`U0kIwB zH%@{!Mv!r`wt^gwq8{4e0Oj!Tv>3@Tq%*S{Bx`1aJ={L2VS-LTm?f(-V@1#{P*nBI zZfZvmzCsK&9cd9uP3K$4Qq!Xra;WJQ3mR(rz#@j427}3U4{2&@wcM_#sl(F+-EMV7 zP4{^^CMEdRQd2)Dw$IFNYM2G%vC-7jkQM|#4?(ecj_7$$7j!mgSA}b8y2u9vJqr?* z=Is?00&b*Vy_qpmX|_IMCwsp@p$Wmm=dkYPv7Q9FA&Xx`{9 zNBzZslA}IwG(q11r^cONIBG1=1df`R1PfS5tYZADrPt>6pNK05i1tbg z67&LsV!IpBo1Tu7Qt*gJK8t-o&@#mtSi|pYNl-IvyTds@(`w8)S1N=#=Vl{<#~?bK z^MlKmw!M z6hfJ{JoAuB)$Arntx>VL1AkDZ&AfT}eI*Es&Q!!P`SelHHj2ynC;F`{`; zb=yyc7`C0cv9j$~=tS~W;D#V4E*bBt^0VSvZkYrNdJ7!Iv3=6`M*=G7X)u;CRIfpf zE2nGxE(6tSpFqZijOF!vLyihH^Iv5J97WxF^&R?H3?|&hboVL>GHHUb?{8$)3@+ zda9=j`iIe7TfGjBo@63vt2cHuLHilav(;TdRdXtt`?%CFL66$7m@C1q{7&S#v>-tr zAjtCl7qNm@*?4p0^L+ylmABae-(eBs2!Co3OB-9PYm!+iVW! zb8uJ;5JB^S=(QWa{aXysd;52*4+!eNffqv^9Bxtx;-LRnWo+WtdwXcad%ZWV5)^bD z!1#^b?a-_tkKu09HwaKL!!9lY9!EzE!}VT=fv#Spgv$BlMn?1+~VM zVlK;cgb+p;kh2#%9je)j{R*x*f>-Pw>|`jiEtL&gfS|-Z*rMbBL2t(a z9`XG%5r=02pGi;Eh(^Q;{@MsD<}~u=Sb-NEM^kH)FQ_3rm z3FMcMRQZ*5d=2n?a+shWBFtLk@v$Ok4iw)av%lb5%FKGs6kBHv&k z$0Gm1g2p00X%S7R#z7Ja!&`nL$?-t@B1m~Q>h{nbo-X2xLdIa*yFv;O^jqExPJ?^?2fIIf*zOM-;FUQ0 z!#1tPF3^01unTmW5e5AQqO6s2f$nxyn0k0AuFt`qO{~MWGrH?@JZM1ab3Ey2IPmqI z8muh7VA2n0!v1i>Bv{a5z-*P`JszJz)l%OpV6dmUTeQisbmyLD(63F7;EaE0;K;mtzIj60o(f_Vf_54XL!JKSGeTY13l`XE#uc z1z)ISx{f2bF8J*}09PKC-HBlg{(vM{(6)fBo!c3Tnzj0*e2fnWnjM>L!$k8!2&)c| z!$gar%Bg%a3-WmVPA#buL}w~b!sxOZPvw~k;i){=h=Q(w=uG9CTvgC2)*gDdH)uXg zidxL3TjKLEg$U||6-eZ_qml#gjw}xF`0e3DT+o-m9XjEoDc8b86rWEiYX{E~aeVJ# z?*PHR$n0p;`lJgy9XA$XKH~KpQJbg3JvX|$zWW&)A=meYjwWbtqj`IZ1A(gMRC3Xe zQp4Z_+AwY6VQBT9;`X#4L60NIFy;%8D;V=lfR-`egJSvaU$G+S+pt_d-;D&KYGQUE zw_C*U-NzQOd^ZRNOC_^>cYuW)zH70d;X7dw!*^F%#Oy%c1fn`u@f{zZ#k!yd>*)b@&aIXNnhkf5M2UyhCAB$AmSPsVA_I8({x$$ z-}moJ4-6i*^_`uDmz`LmivRtp>PxK-DAA zv2eBqqQZtl(uM58plO}P_5xN}a1$reG1$|IQ)w`7l5s`2LALL0ODc0c2JvKb=0o6& zWOI1jy;%~`xgi954ak`|zlCZ#`+g1Ve^E==a)e&i1OSc$aFkYK2jEu?_M3}1+RQC8jg#wk*nfU1gOfn6Iv3qIr~!1d1+JZTSYWER&cA8 z6|)ijbFAR%2^8Z|t^@bUam6Cq*arkn4*-2$IwJ`dbS_|RCFv3?vyPKm_dpZ%K`- z+DeV{bliSjQ-|xLQvUoSc5`5(jqbWNcN$Q-H4ix&M$l-UTeCU5Db<`xw`QBvFhNJz zu$WJ^Tl3SjAVKFN$ePtFV+EJ&q4Q>MrG|T069eP zAXGyHPb!(7!&8-LI(%7x2)YY~({dKuS4mfuoW-yF5)B!@|4RrEv=vOH4)6@w@I+kD z5y16`M${E6I4)wvn0(HBZG*(l@=uEf@GfHGf-_x zV*0?>56l7b<@|I3V&?Ma1vhdU@AHBmU@z!whtq>efrTAPm;hSp9~GM$U38d5`RmjnpDNP?-=TeK4(#>-i~NIVl*CC6O|G=a(@ zx;i-kM=EiE$8ldI;)1q=eYYIvycz@BaEReJevM^0?qw*3yk-pwSj*-6G zg2qTcVG(1bR|Rir#?o)u_Owi<#=6PVamm)|%1B@4>3BH{-5Tlly+CcG`8JL_#Lio` zP11q{-GHFjNr~upo-Swx=*Dwee&PdoO$)}52O#_|OgSw}wIpa~7<6(8t!T)P%jX8bk zZ(xP>R0QH*1--GK+`7`ji&Wd?9G9>LKY5=NA}- z*%y^vzZQZlw`>qAf_8yodVMb?Qxl$sZ>Df$_r2$lJ|O7s0MLB1GzrEfQ^4Fg^1$<8 zI0wpoYt<6ohGRGBx^KHe_iV}DJ^-f2elTz=P6#1$0Xj+{Y=C%6iMV_;dJRO!0NvuMct4DpsZ4D|{F$DTaXAeWU(34d&I%Ag z5!flwFX3Ar2?4m59|w4T$>WK*`bf;7{vUvyTwaK1=K#SiS0KJCI6XjMlR-MMk&Ngr zPseo^!yMWw&-ZloS+$d1eKWkRVH0VU>m3aTyGHZ2l4k-{&8hStE=Uc-X_*bv#2mjH z{WL8IFPFe%*6-j~w_3q#LMXo9F_~FU{NgM#_w2S5wkg56-1}HOZgrBJsn#tt1G`_ny2HE0Cekj+~fsnzk|OdcjuSm zckox`i9v!ol2|mN*-0?w4meO{WHx?$HP=P(Z3Yl+-r|klVE~U&%qUToS`aGF)*x^XjC??(Iy%fBfTB zwe+{8eW|>;`k53qys}=?@;UyMntc(=$MAa?E@pl_hQCs@5fv!tV}Nl*zl4@he@T%Q z_1^=;W_i5{N<~VyG5nKf(wHxTVA0<@5yY|bsC|=BHBYsf<7{m_6av*ax-_8z85QS;GtOlz6?qVe=p`lW^c%^f9>AzMJR@= z{|ec2D)?7iJo4rrad7~=YYG6(K+Q=o91)*-;f(kIu4!2|ebln)NbsQchxUYMM^a-= zz$!hKLxDOJe7sg;3g%m{l6NDz-iSEwgy>N4E3PVNn5Er_8mx@Z?gGNa7ybc)H$yRT z`tR|sOb);=$i)F31%H-^3mOXcbSU@`sIDEv#i0O!?*PQ7;Ije*UsigQ8_`>yjx#im za`7#-rwg+D>TZk<2K&i=mK5CTXt*C?G>?Ms0IHf(Nx=`MhT(u8bm(@2bHFetEeK!W zBFJnU_-Hepbb8k92hei#!LfoDhfsWuUI4`B=nXAp8jj{qy916s&O(->Z?urZ(fn&@ zDyiYjHOA%(R|rS^%LuYer0vV(ag!V{}IkNnk+}h*O_EwGwZBrR$H literal 0 HcmV?d00001 diff --git a/crux-mir/lib/gimli/fixtures/self/debug_ranges b/crux-mir/lib/gimli/fixtures/self/debug_ranges new file mode 100644 index 0000000000000000000000000000000000000000..a5f52ed4aed1bd8a26b97d354a584e4eba5d3042 GIT binary patch literal 186016 zcmeFad7O^r`~QEFwJbwavW+b}V~@yKBKz1W%TUtFAQe)Rlu~J0q)n!T(qe2ail&f~ zm_!T8BukVGElP{usZh8TJC#pGha15BlQ3O z`xW+SCgWl7UkCr6FQb~vAijmL0c;BYAL+loePOxytNWU;E$j%bekAivUl)A`*aces z&=k?-8zbxr?;R`SNcFv*5I^JKmvHp#-2DgDFIp&m&U#6B2tKeVcmF~4BVH9hsqiW2 z@_)m*(y=Gc6K0kXHZ5yADdV8}siKMRu(p`BK#EIS2a5NA$`Qp!|-p|s;|tCfTN&|T49*X^n$D4U$EM6IbIK1 z`;8yTOy>>4%i!9LR{8J8r=i;g|N2;tUxJ;>KB&Irx8is7e&I|w2g>Y;uZer*`{(uv zulP>b-=VKKCeObFufuilyqjfStWSK^UK{y{CfhRJP@8%o(+l;-{-*CJ552XtC!1Z@AZw%UVd^XHMH{|Zz^>0m=`JVR) zo8K?2@KEmhndp9m?a?i9=s%bxe*S{1AD40Ar*rq8!0~%weRQ)O`j_U3pHJbF^JN_O zLhk;HaeO3fitc%b{-s6uhfikUe~H6C$4A1Z=$?1zL(9caWw;f#c|CXkT{-?79EonL zL;vDi;-|se!p~sWRk{1`$?>^xEV^ume&KuK=e*UzOxSd7?*4=7FT~$;I0(P%9saj{ zBXK|7C7kiC@WOq$=kpV~OVABLSM$5v^#%9Kd~p~L&pV#Gej3N$g5`b_UE$wz*MI!0 z%r`zNEd8hO{l9Y8>+u`WEsC)%0F0$nY1zc}>$M~a`qR|{9bgllv6Uz6jLU_W$~uFGBj+VwKO=mz1n zqlBf#=C0p_t`fQy=<<)tU4PAuGCzF0u>4KJbrW*ePeHc@9z^%X#N71>x5|9Y$->=m z#vQrqJ8}FexD4IxQ*zfoJ5}cYfUn&p<16pYU0;^tH^AxW`cBVXf5#Itza2jIq>S4? zle_*8bluQhi>}F>-1R>_C-cYW3RgTYyz+(I^%Xfj22MxU_r=`xuP>DOoiObs85ejZ zcl~j6J<-)cchsTpzexPt4ihqDJb!WS{^y`O3^$;gwIp}_v86KKXPL10a^VYayy?r__0!SqfWM&2+?Kn(*$$b12zLES#=mCgu3wDq40Pqb z5#2s$`=4jG9KRf%vq#2jzRTVJBy=Cc1L&6T&s|^rdzrrjw){cHdk^KVe-_u(Op`!y+fgthYux8@hxdehMDf`6cUzd-K#W`$(_A=tIBjDIbXyM8gcGtiYQD!P5p z=F_LR9DfyF6DQ-dOXco=Bf5BW&C%s6ox6Ss^Rv&9<1d{n%z!qZqs;d_Pjp4f2v?L7 zTKyPwt6?^}m&)g^KN>Id6)OqrR~9a*lDmEsx^-|px)-YEu79DL%xA&u3uJsnjokGW zIerVAgl=HX-1P%%$^0Ggq1rMoRX2Bi7P^|~>YzKbUhevJ@5%fj_}pq4$F0p>UxwqO zVN-O^JM`5*5vLBrd;=S!W{SuwEACoAOGxe5~n@94$gp2Kx;p9g4ivEyWv?AWxgJ4 z4y}EWn?#ogyTM-2>ZjF_{(aX_coWuws4w$R!YuerLhkyt z4P-tWzC%7+eC8L2?pBx%)sL;GcXL_a8{q@+IcW8D?w0=cf|KFn(CVMg5dC}bAS|?4 z=B!@Dtc_pjFDazzpe61y~Ore=zsw zfquNs8U=5Ik35^Zep{N%AA_|XlJP~*X4UW)nc4pSNgII<~>8kg`w5= zuPn#Ega_auX!SSWAo}0onWJP}99n(5@^ZW@OoG?JNcA%Zi2p;d@<17vzf5TTS3E2; z6@C`hgN>lo*ZV_en!$_U02rzM!{U?pdmuMu)@ z`6uC4_+C$$-vnp%mGP_amVUYWtARdT`=IgO>MeQv21oaivF7`ZN4}~rf$mDE_O@R^ z^$o9)eD=VLu9a~it`FA#Z=cCbp3jB(VJUbHtPHJvE9O_hjqp>4zVQQc-}wxF4~xvm z{eBcw-*%?>=>mJeAut(U4{bfum_G{hJtqDNL#vg^BT)|3p@t%G?960-+MZG z_%%G^zTEYtzZ4JM;bm|ZwEpj)uXn@g@Co=7TmY^8u?HoOMQOs-@B?V|m&_8~ZE!kV z2d!SmRrn?WR+McU;_?-$%{T$-j{`Fpx8wGn1h0XqZ~@GKAHXcg+ZiF=P6+X`G?XCa zgnGkds2`U}W&99KhrApMWieivPYTO~AHgiR17^cB@{2wW<|)7!mWA=K9!!9}VKN*G zQ{gi(9li-O;r9h4UP2*Z%_71SI3H%fTE)=A;>Beg2b;h|_z+Boi(w{Q7$^1_a2L#h zZA!@TB-o;)j1%EQFda@mTaKr}Z(t7GR!WX%!v>}C2Wy=p;{^C%CH%onFbmGED#z2| zdYAd!umMbj$Lq-PP+j4u1Yruix&dSO5zK;nU=Dn`vFOv`oMwz+oeO2007t+S z_zX;kU&Cxzw7J;F!QL<#>gU}O`P|#>Fcnt1i1=^|%!U`Wl;er;-BvQrf?Y3`aT44Q zb6}0u_=nfR6gW0fj;F#WU^-k5Ghwkdp@030gTrA8yuE`QPld0+On3x_I*P6#OoU@$ zDqI3H;Z2v~A1;BJaA;>ao(vy`>F_g{4cl}PeG;7575{Jn%z(?f$?;715zK=7VGcZ} zyXfO#tsXK?fWu)5d=X~A@;$MKyg&P*!R0beghOF6d<>?;W`oefaWEBj8!X3jh6u|J6~@CZFbNKWamk{)AEv?X!(~1R zHoQv4Y4DGcG7eoWY5q>p6#@TStL>VW;60>C-563K*aVi}8Huf;z8X3pIwQFUZ z1@peoao83n!5d*J{PhFThdva(%IEMh;J%GAp93pwl5sp74^!cNFb(#}5`8jU4&y$N z`Tj5&-UHL%I+z6uZWg;Z*c2whD_{z|8>Yb*VFug^6F(Kd1-IZImW2th^j0|@4=2M^ zIGWc1Deyy>1@(D8o9Ft2Uy8pRc+oZ)C&Df;2|fbT;YOGRe}SRx;-?BsfNfzCydI{& z1uz5dgE_GD4(f+(VG_Ie*8vtfa6LjU>~2iw9VI0dG`=XQxc1O5nepnje-?ttiq ze=kgdD-X#y6Mh7<;C7e|zlS++SPuT+GcX-4hH-~Qr|%yn{w(8%VLB}Ri_FJEeg7`* zDE2TB_JhgrR~Y(D^xI%IZ2G&*C&D*jCfoyaVBKS)Pk;sflyMw9AI8I)Fb(d4IWW&( z*uzRN0k(xnu)^P>kB9AH61*O!!Y5!lEOMN9@ZwOOe@!I9QF&yX0;j+<_&&^nRr88I z0qXmualCI@4<^9AFc}ulC-!mhCYT8~!YnvHzvwey#{x1=f&*bPoC(w6?=VzQ>@I`J z@HUtVpMmMHR$;MAfbYUAxc*Ezo&^uV9C#5ww@QTi^Qt6%ZZ!#}!gG0_KOSy^Sy12a z&)|LkH(@5Md_MWYzhS7X%s&q^;0BllYnQ_w-Urj*dA!e=0Dq_?`W)Dt_rnvRzCWJB zIBylv$H6zM$v6{!0<+;SFm!?FdckD)Kn?byg4e+m zcpprI<2#65D%AIfb2`g>-Y&v8I1FaO-7p7!)>ZV`@JE;fckq67Hk`uy$7ygL%z)MU zi@yZ81!luPVQ7HpdcY*uke_cP!ig{yegU)L0hj~xUQYe61x$qZ!!-CLOoulO68luB zKYz;@Ec1DX;vYT=)8WQsIi3Z-9wy^#_{Eis;c*xmA@l8E5?l;3;bxc(3tT03aj+ju zhJV8Jk)o@7wJ-tBg6UA--;U#b?tw5F>d(XC__sI0(=yJ0tDeCg9)Y1bGQS9Bz%FxTJ_#0^C*wHy$_p~i zfX^(HaXQ=#v*Cj;$?-H;{}mZ0z`~1U90y-nBI67=Wf}JH>(^zR4L80i<1CnWg^c6i zJ1`UOUnR$L;Hpd+XTl>e^sdY=f*J7BwKAU#@7#zzyyFYT@CXcjDf10sA{+}-;S!h$ zkHFA2>|r7t22)_+?V^i=Yj?;v3+DZbz9nUx4CkMX zJ^ZY+jI&{Z^JE+co1QP@M0iy>8K=O}6&S-h6=j?Nd&J8)2^Og&<2cx_vW%1A(5j4K z)oLdJh@dct_PALhW~ z335CReg?DQ@dk1{)KK^~%!E4|$$U0^y|Iil;m#%;hs&CB9PVt!ak%V4j>DaN-Y6Ru zY$3%qMC%{yA_C@%EU&Cy89EMtot{zN)Jz+9@2WG;Zt;8-HzJ4+B;Lg?@hszQ< z4okL?aXh>cro!(005J(pfoZUHd$CW1FL#h}2HXX6;Ii(-hg18>I1RQLBI6|Z_Xrt> zt`gR|PM83D!DLu$oE(pX?@p9)7QAPQjMLzd=`v1+Q(zjLf1e!BfagCT<9K)zOoiz% z2Nq5feH>i-u#B@{_eW)%1fPWI@O_vC3(UkH{O9wIkI8&Bm;gt^6u1Ls!yjP|{2PX5 ziNC903cPrZ%qPOx&&oI*4tY_=$?&;1Wt;&Iu9I;Noc}4u;fLE~oCR0ymvJWi>!^%F zzX{797skU)1>^_IS+Hnn8OOn8eEvEUp2_DE;^6%-4eI9;;`p3GFPIEpfN@PF&JdUk zZ*C#;sc;5NgP*``*y1A5C&G#?WgHJ5XeHw`_#VuHx3wl7d>LlIBQTUGx;iic-V9S= z{x+hEgBym)I165Xg^W|+!&k~U9cIF~5i)<-NMSO(`Dz(w!!xdxaU3jjos8q*%P<4p zcfA}>gN<*HaUy(~42!o;=v0h$v7L9yj8{ta1cy} z3t$F(7iPn*lgS^Jy-mjPuqI4^3tBv|2o z@`JTu0vrO9VMRW#6c4+=B={+xTgry|d8I5qxAd!ieu)q6RDlVw15AR$VG6t#rop#h zCT#Yw)RPD&!BqGp%z@1wK@YEnDR2RddsK8=U=A!flm0>dyjt92GGG32j>G;i86Jb7 zCq#GqY+)+g46|XwXXJPyydI{&*I_2y^P=c;VCRK0PJ&Ao$v6{sUCePf1SZ3KVHzyH zMD%fR;8GbUEED#F$*}Trj>G9N4eIY(N_tIn`umiUUYGH$Fcq$WS+F+$ewiEGhaHr03cUD`j1%Elml=1Lpl%#&NJFl%HDti%96JkLVv&zk7c#t|Frz~nPv>z!!r1}ME(7{ zJmk{1h>~9O7(rakgSRb1ynPk=)e+C7--4dEb7E$%c~2t$3B!c6?%vXzxt=c)W3zD<}<@lulj$(r(XT$!Jqnd)f-g*E_tZ`$B5(dzuxG1yZE1d z$>D#%#|uL0-%DSf^Ol#Xp?q>#2(^a)IOZGPehLNE^R~i2{)>C~=WPs6z51O-9qQMW ze^7l#{Hp(RJ^GVm^i@6dV*={6?{8!G4%BtC>y^%NzxsI&Juk~cM?B(PPrdVDCj1uK z{2p}Zc?_`f(6a&c$IutgC$1X83!x2gCi8yvsypBK?db4dnt9c?X6({8%P$#Dh4;XD z(B>ah-x5E);dppEd;mTHZM;bBHICM&aa{EU)t4wB8SgA8{28_=WGj(z1@v9v9mf7# z>|FMB9rnkWw^7?U^oL`tpN9Xp;6_J0tzY#Ydg$-x_{&D;swb%adE@^x#(SXcuhnOJ z=+9t(ybN`{E3@wPJ=S0C%RB6Cex=a))jw_W^U5cver`bji_m`xtJ0r3(AKN^RX+7^ zqVucQ^Y(VuSW7mPybWU z`PH9WSUT1aeto8l|2#{0WeMRQC53y>6<&3o@JIM@85!SIUbq0t=)~7k6|btPuLAYG zOnrA!pRMOa=JK8PCtQ1tw~DxL(l_m&jitx4^bv-WskFw5@_DlU=%=z#>{1{sQ%{bpLhF^+06Z4&x`RzWT)(n6KY>a z^L(Rm?0!;zJstjvvhNnAeS7R&_BNj8sr75UHc!9x4j>!ABt?V^2u zyO_NG+vD9ro!Zao^k1J3dhk48*F)`QVXt<&9(}M2_xZy5_wr}+v-x;E&&;6iSy1&0 z9M3;mx9$(S|7!Ght{KhMXU+uqFyeEnGGyDVo z4dZydakBF{NPJtrtwlelh)80loTln0WU2K-X;w_aE2&{#b-9dRUgz~WUEh~E&hxrH2kE}ned&6CSM$GsIyC=&)M2k5wa|5h1JOMRHNUnz zA6t8uzB2kcFjn@OcM|!}CGSDJuc&dm(ogMQ6Mk+!7`poTfBs~}PuKo^!v50yin5-% zzIMN6nEeqg`?Bn>3!vY8-XWjgq2{xVx>^~3wx0IvbKUP6U+rS0uR~sb{k!y9r}ocl z{leAnq@UlyAE3so&c3nr==I}$^m_eJJFn|UckE7;eG>5>G5Ke3zsy73T92(??G|IN zc89U|sy|%)a{Rvu-!t)UcC44?sr}YGwN6`KwDd>G&#!-%ejxWTeg4q%+WuTLTF)PP zo$~wqq1RPicU$lO#m!Q_yoOt&2lkLw&)1N}T?$YxwR{CPZ@$28E*E+SoUi}JJe=+@R z3%fv#*V%FZ()*8IKfKO=ulvK*T!&6qf7g=V0;u)&<$3pG_@(3dSM5H*UhVY$`Zad? z`rfNw)}OVv`_IPn@~`^K$ZH533Dv%b<9SQ#)b(7Bo%YG*7cKot;`sIN(tl1}d*Hgt z;_>Dx!d2CT`7aPIt}d)uLzoV$)RggAKnbtUbS6{!2UjKS+Nb zgI@k@eqQ!R^*PBA-^SBCwI5o)jpMf;l}vw6SAV_9(`$WepbM&R>C=BBbbj@khu`(G z>m8~6W#k)fe6=5gAFusk^S>IqaP5P}yN7sBK)p^pSXDCX!t;$jN7;OX>VM-pG|8v` z0_gneO9#}K4XCdgP~Q;!25KaJ(yK0LA1Bez45+UwHLv~TS;)MPpwF$k&L=w0x`nx}r#hc1j()_d z{$JSNw*Nun4ffgZ|MYk(sPkR827czKe`&nz$I|TIKJX*>6&%a^WBR^@?(ZhNKCZ-d z+qK`d|4Yfk#2aP22!KR?X)LGul&*SfscuNCWM*T?ot*W0gtCw-~t(|@e> z1Ic3+wENF%zYU9${rzFa51MaK{W)A0?Q?+F^-K36*7Hs4w|;fK%;hok zjiY~RA1nPH`s~-gORvw{uIr2S?{$9L>)nkW_m^nNlTuM!%8Q;re>q=6_{C{hPd=`3WwpBOdeA6Q;o|n4Tc> zOW*-myMfGiY$#j+-+=3&96j;%RL5Uy>i4SmMCS5cto5(39tG=4ykbz-C({0_{}6ty zzemk_+WOxjzi{n8#QrB3G=8l0T9@tro%Boh`#{J3u7J+3zB>BhKK;j9KbJgqL9J8w zh1dE0Qk3jJ3p0Mue1q!sx;eGBoEdsvPUL!7nd_roH?z!jQm>2q9s5W9pY8BJ2LHD} zzy69v$zOi_xcqtLWBcW$??qonz%S^VtAE-zT_0T!+c($x*y~*->OI~4^&(HZ-)ueA z(FN6Cx`-nFiZm2IA72|rc zfb&bQ^ETgD>$e2V=j(v_9Q40KuX?pl5A(dD*ZKRo9_e%PJ0=fZ7uR#Mo~O3I&#^B5 zt?@EUebMfp`oEL)x*KXgHQoW%?=aN$E5UgX?)t5vzEc(dV~_Z|sW(>jpX&JAJnC=H zd8+Ge&rf?kM%&-FX`$*>T&5mVBc?s`Z~+@BT{?K+tzRW>iXFD zBdFVDA1i%V`s>%fORssmt`F9~>wMLHW3N~GzKrYs5^aBZ>5Jy$6+c?~p}elL{c`zl zgT4!lm3>?z$*2nK0LL}6naVg^eHr8bPR0*57QZ(%5vDX1_H@M8{KB=@c+V2|BI3B> zsV*=1m4We4vOV!t^_%d&1NybkM6dS08v7HGww>SFLGQry!M}62iJa#wST?NMJr$RH<-S-&Og8P_rXuJ z{6(72lU%P}f$L$OmiA1Nak%=q#(%*J#ouf2q2@B4(Ly--BBA}N`Gsq*@jf8#K;pRK zsqRP0?klawOTUZ#`YX)ws8{=@{nt3QZ}z;_JnZ?? zlzLBhe>ae)UN>Czv_uzF-^-`}PU!sVH4nf2wd)h9{W$UsH@@26gCDQ`X7j%tyKwD; z#!KKn+74a|7bM!uWE`%(x$!@b@##BvpE~V%?KgkD4*1oNB%Zx)%w^uS|Li)r_G7I5>vb+#`KrHrqf~z?exl_s()C!z z^=dQR2}iWIXOfIn|AEn$Ya_=WhojrdIJup$ox`8T3)f!bEhKJj;<)0e?hEvL;92A= z*`E0NA;(pJ+(WN^Z2grS^}FOPk|guL!u%2X-%OnH9mH={*c(0r=fRg@^^T%{4SowNkms3Q<@h)G|3XV$-7hjy{A^}4zRd%X_Ev(G<-dARO@m@Cli?m;-+Ftif$m6>42FG`^ukK|()$bxZpebwx+ryXO61X4c zz@zYQn85kp%v=Y~Wxu7uY0zt*+Wi@{-tyG>zop*9|L}TU&-J>0W4&J^-Jk#I`RjG$ zEUp*(*-yng%bBS6o#xzcCOFRPS*ZPmPzu&mNM=QVT)cN1)FX;LO)$4g%-aMD< z`v>u?R|6%_XI&l7qh9{(e%sDIcEz{)&FpvG_bJ@hg5C%9eTH`IAAO&}#yeI1zof2k z^V@Ke?C*2*-*5a#?Pq($x9b^Hza9Tx&sDmgY`jx#ulI!(+#ilYulsZob?f_X*K?lO z`l9Wx5^??JXY)DL_LX_Aw$Is}nb+5K`hKjpYZJbqCX$X=)~7k74PYjT;Juyv;DLEI?);V&ZXZ@fA^Sv zN8A2Y@?Htoz`X1S8_%Ubj9sMq7R0st@*wj+!$kTL?mlhp*sq=cm&YGPhlfoj97$EXJ<*^^N|yQ>=gSEa~{0FcXG~*p-m6OMg)9j9E~9cjESjgW=n7 zHOzrW;bTQ5@B(;KF&R&XkHhETwBjaFhHv1{D_$q!41)hH@xP<~?@fPWRd2NWcb#KB zDu%fqPL{v2QO<80eQ@pnFNjx!^Ps!qe6aOL>d&uzUGiz>kzcg^Y2S1`Tzzs~Kh*vh z{SRvI(vP4IqoFH)r1sC3lnY@7JPxar%Kdt|_iULj$njFJtV4fYY0=+uj_^MCE?f(% zoh!%d!@lrxI2`VR`{5bqiQSoS5WE6D0-uB}&li0=*bfeda`eR4|MdJHjj;bGyWevI z=6ka9%P{$Q#SgmQ*5UstiQ>emmf!Xp04D4 zT?f<3NTlas$Z@_tdXnPr(mYH9UOP$gH=pA8uX@DKjM4g=c89dz8ZX3tvFA;F<~P94 zp!L_Bx;nrVIL%S7>RTH9M8>znDGqzJ(|9gBSA0DWysk5AIA6YlE6Uo+8FM^Z{?7C1 z??ufE+I$x~@>Tx{*flyy@h*rkU$w79UH5p@XZ3X*de?J}_Sdd@Z|bo3l^dD2`qt%S zW)<8BTUM}28LPery0P#Pn7=%B4tuMwg+52^OjtcWisNJ8Lk@jC=G(#k@JffipQDbK znb&^&QZhx!eh{tUvKid9Hzpc{z_ukGE93{ywAvgw?{pZ+H2mfeuT`w?$l}Z z$;@93wV%d3r2SU?IP|wdt6##r-~FTe;0@C^*Z$W1cts_Nd>wS{&!GM*<1Y~gwRh>` zIA70)uKFUi*S!7q$F5&5>bCYbGHsjoG)8VJb{_?B@%a?*d1{aSnYV*Mp8Q30@6t zKVM+}1Goh?xImQ6;2iiq{0N5Y??H$C8$R>#vVYf6PcPyoJK}5o7jQgQ`aR@#5C)AG zuD!0Ot;^L>9#J(C%+NemTcSL|LDgJnF+1 z@G(f=e>*m56T4rH*_$I{F_}|6V}<4-ofZ_!yi6@2)2+^aQl)rMf@s%8_E| z%0m77&gy3Y^Jmw$o@Bg?xY6==Mji3n4JN^Bpv|Hn^W|VO*v_Hf!~BnMDt$WKLquBImfZy zs^5bCi-7h=&}%>J`RG-@%f7!OU!B+U-OK+~__6u|j_X5E{kVYsA0%$?1UWP6G!^!0 zCj1f}f}LB){QK~#mNLEv%IL&b^}mET>R@Nf5@m%W$1nyJSte;e;V-M%^X>-9y~u>||T)*CH-fq?#9 zdfm5rzIfda!qq>^{+dbv|D4wg1)G z$GV@tcAT%TntD%`|NQK$GSIc(g4VN=dX7S^=R@|9?&rS5wdYq}&Wk!w^Q(zo?eu)G z`MLa^ps){?REZztKZ0e z{tWJf8t+rb`EIZGUe|NIj_CdIpX-R@dVZYylipAM)BNY9kFBBRUz_`y_M?%bziqL* z4XWMM*sGo2ep-Ll-kuLOp4WP*K7sri!&Xr3k2>nrI(5C@!%m+EY<|(wuP2UQ|1P~g zN80CA`<$ll%jk7qpI`MkP4|hev-UyXr_uA$=6kB+57GMt&-ePguJzk`gVv}1vx)Nq z?A1s*(GR-%AFcRT5O*X@fq!$|KbPljyFQPjv-uxm-s*M!jD~Wg9DUXLtiA%e!SHVS z@c?|Zu~<9>|AzUSSWhypgrD9~@;8h;Zi17c&2t!e*{sH%VtZG{?f#6mzpu%!u(|$U#&evFuljGW+Y``U<7hvE z*6*_4QjV z;VW=oYdL-t+WlbT9m4Msc(V1{7p>RU=T(ojFGe34!e+1qwDnp066oqV^xEe)IKBqH zNx#2|E90R0M)>P#{Eua9qo}?q$6Exn*Em|w z1ngb)xb(L<^6NlfQedeLQb8wZ>lugc8K}B@(QoQ37T-W?A8WnlrFCdtM=q62@^unw zp8LtOZj#MZ#w$!;UZKut`^zA|B=Q?iUS9dFCcg@oNWN;n2S01O$^3C>tBlls1Mz== z8vi-s+jy!gO1`S|o3Bg1gZlP*)ElXN9rD)wrTanGPwTMjXV<~C-hILDU+OM0t%kjTpz8*4m_3s)}&*>7cA$4gzzf;c$>e&kI`Lcj|wm|)R z9J}b7)vrLeA)tQUX`C9_ak) z2YBc;egZoEJ1iQwcR{_AA+>&db&O5AYcmofIb&s_C33aHoDBP)4+(ASS=Ii45SqAS@^ z?rdeCzMfILnb_;=NWD(m`=R>#2*2vDGJb2qcCa(d%j;k3&#!$?{8;-h$>S&Jm4~(0 zyylW$(0)YAzLtsK$E-(d?yoj~&DZM7qpuEQWnYiHyF<-G_mjqT?I)MMFL@7zv9j0m zU^)4J0PXuf`o4$0?$`I1?0Q7opXRfNy!8F8!jAV}RKFg*{+&I)_NvqR&m)fC`d#+= zceHK4bl$K3nvQ(lalD=gs&5+5|4^PIro#z+q{3D3!vQkh4sHFh)<0wNdXKzbAfMNv z&9l@Xo2iT!8GqkL=XOD~ssS3Mg48}hiJ zm*{Oimyypi(3M}sfnwj2_*X-({C1G%j~@AIeh-tU_UkSBrtt@mhh%f&t4rSmeQOvi zd(B(>t9kqFujUo2{<`uT-A{V>I6PhQeah@t?dN|=zCruBjs02MoZou>oY?8|-3z8a zrz+kN_DvD4AAa{w(0X5|-lI_KeV+K;O?=z0!d!>jL(R9k&+DJf*X6&Y!+%hFFMVxO zkJt6v=KG&+--Y`1daLWG`@q&4Eqz_~i(mgPz3y8*fAoItI^V<9XK;SJ0pEohZzK0* zTaUfo>ix;?OTYIkulxT;+~@WB{@)gFBlQ)b|61Rb=DOdE>%U!(;@I_rYPS@7wTsqz z*myzx+x%_5Ui(4yqp9y^I2EdW0mt*a_DT1X_DT1z%`aN|mE`Bwze}&r?HBbFPi0Gqr48$dV<>Lr`~!nsC}&UuJu_( zzIHuqe{?WASs-KYOp>pvxr66{BPUqJVp*Z$fbCHn&5#^-go?|cL5@8Wqy-xtyM z7xZvgxjpW*d7+wP~69q%zy-*MhQiB)|XZ#Z#AK<%%_FVB6qDs<)Z1nU`Yzvol` z>5|Wz9{HS0KTcMEPPcsC_vnv3AMN>P&$np%+eCf?*w;bxQTre9vzON`)!FA>^|@cj5R5 zc)dgaJM-s`5KrAk31`5|$IAGAXydu`W6)2Ev3(Qr?hQ5XP2~S5@z*=@cj*_RUlwEg zO ze)`pSWqlUFE$}e3{`EYm%6X&b!FEUfvDSY>+&|#?T(9)Ht=GLj9M?aWJ{$c37%Tfy z)@#9yoehy(*l|Fj)xd;873~k?&=ywnL+z&o6 zQTmn+Z9l`!=PdFF8qaUN>fh$;+K125N2{-@dHU?1wfE|`)z5RR&!5a&{dm^Ro;RDA zw|bqo=apTDHjee)>o_k@mi^HPVI#@-C-jBTmxj-tBKvsaCY$)HQzhS$)G-0>gNJ?A*CfXN+Zz9E z$oqbie;4u{3dg{p_4G3Fv_Id=0AC^YlWqF1o+8Z#J&( zw-b@&yDj8x{msR{)$93H#;lv3ZyT`FK5AUMj;{0hD*Q&v-?jJ~18sh9n|*8R9dGi{ zdelzOTUR}IMra>2-kK>=!8%y{P8s)vw!V)Yy6F!6j)3~T0rmR>>Pt_Rin_q|cQJM!(xknTq!#wxOI1XC-SnD;fcI49qj`qm&Qb)e|v8x)Uy{j+P ziPr$S;zw$)dAs^y*QY6UT744p$x!=g?X};kzXJVr(CR;={tDCO$bIlpX!RQd>Ro=z z+-E(@I9mF32jDh`egX3@!Nu?+pZfPb^nLD^zD}J z!(Q{(JT!k-9=3mKUyC?uKi%Z&}Far zUd1>1W27~Vv*7i4;bJXAo&Iq+aX#_;ySGERj>&ysN*oK{@MX>cXX zgyl=f@p#z1q>Pi`U!`OmDlObzR+tS(SCnxIyrY_o)8I!J$T$lQtBoFBTwlhCuuLl% z$HV$C0Um8F$3uxi{r#88ZDc$ZrolNd9qR9|%xNon{r!Y-?PaXL-yk7L#`^y);|IyO z3rvES!(=!Hro!i827DK0!M|W=u=sy^s4yMA0yAKtE9H0`EWppr;>^#_68JgUTHg1} zg8F`84(}W4`)_fR#qU?VKFWrB?vnW&sLy|yJm=~2NfP(pa$J|=;dfj=a^Uxl^RJwQ z33Y_u=aFw3=Vih`U&dX6ZYxwfBM&{#L{NP`{1$|M?GF*}H|UjLK6K~7cvu5k1l4nT zhy40ydxiM1Na$`7a4cUasGc8N`}Y4R`=I|BlE^ji0ciUZuKqRRsQ-x$|Ek-KUUjZ~ zK0vn-ZiU;R&0pvJ`rnVABk(BHep&zV)Kwq$gf4w`j@S07pN(z=8#M*q0c}32e+N4+ z{U_)?hhM{Qq4obe^LdG15Ndu_pTO~p;cyRqQ;xUrsUMYJBF=&vxSyX{Namk8Q3{(E+z(CK3TWx#xc>aW261gQ45A79a@Wn8~L zg9o71|AKz5PyJb3f8T_au^Z~}@6z{m=&R!QYB&MDY~p#@>-|vkYvPFaHP^*`@Gvaw zxIPYKemEQl?|10`X1*XNXbD(dPgt`c&oI9LE{5wI`ir@ac7#3QNNC)LzGi+OJPZpu zuJ5-oKMg(v7dZ44c)eO3Hh>+W&F8`*GSdcjhJ&HiA7K6`_!o@hx^MMQGCvo-0y7=@ z&Rlp73%w3R-<(-lr%D%fb54>OWxq6Sy7z=+NKG z`yUU(r{N-K{a<~K%#4AP;0$Q>HP4lqhOi~<0j+*N^GD#Hu(-a!GX^g*zXZMwH#ziE z%ZP&q;4HWh8uy|0=gUkt*dJaCtv(+Q8b#r`u$JQiWDWBl!_VOXhkhz=C_DgX!G-!p zgGs1;dGd$-;kD4nL-{JmOi_3)tOc!p4f7wv&*1@weriSW@Bo|z7eed5RlLk}fJyKQ zC`V6x{fqemm1HgsR)Hs!^4&`2--lW7TZevfWwE;(rop+;`fpK1X4=7SFd16?0p@># zf5Et_Rw?5rnV$<^fte0{=W1ft8xDpk&_?--`2rWnkvLcdTK!7q--r7C=(i62W?vhMol?V6vjiVU&#DYxDsYL^tJ1Vhc57Dct5oMn*`Kf ze46Szl9yM#Z=u@`3)Yp6l!LZEYXj;xo~HWGeb#$9eYg^ih0`7Vd5ifqa5FsM&{wJ_ z9j*l%!>-Wwe>(Gzz;yVALw|_*-(a5l;;{_0{(CS#01kr_9QsAfzX9KcUpn+v6C~oZ z4TSH(4bbKjRR0A2_3zMo`Cn-A_=NERM?OLI%L4jW{Yvb-@)_GuIyMR31!qIs&q9r4 zrUWbt8$qjolKHvt6}Z-+FWgu>l!RqrBWV3kVqX7V?F`2A9r`nxiiay<7TgQ1|02wH z3aG!p=yy2$muKFu{%-US!YAM&X!9@HOlIQY6YzOx^<@L<)o*+3#=(0W{%bMsSKstB z)hCjdSG^yj%Z39ll#X8sZU46i)T`fo_-h0&b@)Gkf4}-)Pg8x!XT8s(`?I-px=IUS zBWU~oF8b|I^)(&(%>nhh(WjBm({K?y1T|lqU##`>1LhM{|0w>SFmd)Ww)Kvp??W$= zBPnnijFx^3ac+ZW;lDG~_%_}m^smEpa09gZFPPsC)qg)nJvI-o`c+>6f2zO9;lB*? ze)X!Wg`N7H;_$zU`9UqE6T9J67u(LrSoL3GmkoC-u^-e*^zHHE*Pq7sYoA42%_pdR zP<>InOK)VIUYC+)Yb({}wJ_OaHNW8Jozb+h>#C7+=BLO%T;$G=~F72R)2^4B_S z{z3J1)vwaSzv|oi)Hg@xSN}Bnd2k6_2W`DG*q@bK%aKv=aH3VpSoI6B)4w;qn6cXL z#m^@E`1Kdm{#oK`K0)n+>Q_?NCotQiKF!PaN&9X4WV^n?tWT`;dxNorJ{~~z`=tH0b=t0XroLF~Zz9ei z>e6+y`6QtWsvqvt|3Gwp^^?)t{%9RG|DgIi1Nv9})!($!FaqwzkvDWa4p>G&=>4Md{_-O zfi|C!%#Vk6!I=*II_9^+J@6NYzG9L@TmonIlyR|M!a8s++zhYmZ8Mdz`fKa(*A_c3 z`xar^H^44jdt1L(zI*A{@36N=ecBi8x2wMz*LJxF^_^<_yU0WLgDc;D=z{8R@ag|b zbbj@khpT_K-$C{FlaF8js!zwx?gyLCEOdVLuc3bj>Umv(^Vs^=zXuY>e!dFc0wdMy z-wW~UU-iX&=A(WGqn~fCTQ>jm(fQR^LEp;wf7{W|#sT%uaQ(>NM>0)`G%-4kkh`e=DNoZ!vyc{=D+B^?2#$Q1?>U)1zMPoA%qQZ*~>doAo+X_FKqf z2Rsa2`P8N_ww_aM|6PRn2d&rDH{B1~C*60pUy<7DI{A%PhJ9Tb)`4xH&F>l3Ie$Mn zk_3nMw@MkSeie4A{{Xwv*p-8c(97S7DEV8AAD2I`d~7{l`Z?6S6!!F}SNo>@_UfBm zh4p5=PL=%@^4I|nLsvev>5HxBRNH?SVg5nub@fg6gZ4@Ho$Xho_PS1fzh|0%fE-DJ!v|WWj8(si{h|5~uq%ySIhY8&{H=(Rzs2}*`SZ%h*5jq0L)}YZ zPmg-FZ`yCKzS&h+PkpD_ehYc*fQO;0p4#-q)^n=uzl$*cp!K@?rt7JF(tT(96{)?h zlizq{*w>X|9oPoi{GMT*^Is-MlHl;mty0FSUxl6OKftaucI99q^zyeNO8yq($K}r} zA6t)?ehzgng*`p$)xK%Jz4~TXVZB+eQ)Rz}Ja)jt(3MYZ`eN%j)%M>-n19fEU47I2 zpncMPXZsbYy{?nrcxBkvm0=y&2HN~yVV!#ok|TZLHE;%;3HuEeUGE{n_Ctl=KpCC* zYW+vcUlw_P3lG5ZEp*_>Rx|J~&PCd51hSA6>6`)cZ2j`n~Fl z)L!GgL)teY1sQrc1>2?0udg@~ru6@vWHxSR>cM5YoveyZl z@1zLr?~Tx2@4G+a$17jeziRX!aeuS*seYkP{j=!&>NO9|U-QuO%I32I|FPDuAQVQ?E3eT*Eyb@{uHyt;ejr~ZQK_4iP_{JZk|g8X+ut>4xgslC>v>zhO$bUkgn zpn6x|b^qvoxzW*oFZ)5*jSXmjCi+rP^Re~1;`_BfC$D%s2J@XE<0?>pU-JOQgP{Ii zYnzYy8;rk4;97@&mwkEkHKEI2A@V5?z5E5$m-6uM%5NcgFN0dY%`Z}W+oy`;tLtau z1=YLyuKPvzM>|LVz3lb(fVT-~zmvY|?My9irH6l4eyhp*W2p7pdLy;hy3|ic^40aT@q+4Ieb@b>`{O!C z|Gn&cV|PtJ``_qmUhFgd>C#WN6{x=%`+K>BAk-wnvN09$$X#1meXn*W_2Gxh?`)IQcx}P+T z?k5|ski&il^H#6tS97yYdcGB4owRQnSFb~QKHGiaI=^QTCszKS!vAy7=5O=0{g`j^ zv-RqE>#A>gg!Vz>J;n88AzTALg|@z=A~JIwOodaS)weGyGaX?X+ybq>n?qNmm{rPn zP(b}v0rl4g)Gr{aqTV_9s$*LF3OP|2fe1%l60a;-GrnPtTfl(f#FG7u|0*ehd6q zf5ll(tJm}E8nbSCzBR{A`>1j4I=as1b@+{zzmM>@8QT03IPWw+TkmHkAFW62^t^S| zvnN9Ppz(TBz+(6Yd<%XIpXGhm$9bK&AKt}t*IL++=c)E^Jk-BegS&sA@fR|q{y)dR z_4kh<-;99a+K%JCVmd(C$C4 z{nj%|_P2%^KWM%|^_k=~koSZ2?_GYwcopxvPT~Ce8|HVsU-Vg&{q2mAzd|1McB0P9 z;0T}fYrgqoayUFZt$&e(Tx7{tVZCH}?AX^$s)s4ch(% z&94siG=aLGT~-xN-uJBmm+?M^-{+QJc#g60^}6|=?tcu||3&m?4E=G{8&qH3 z_}BO2Iuob6N4)dHv@d~Oxc1r)zxkdl{Snsb7w9*?wdljO--`WqxC`!wcE1G8uN?cJ z8uZ#9y05Ngon3u!?dM4Cs}R=}Uw^*gieJard+ECc)a!X_uTL#muhvla?M?K3HcW@n zx?WBs|I;O(J54?(yB~8-ar`A7@$LB>sXoc0^Dq-5uleInw8+XxZ=Q{U?2&$@RRZ$2ajj_a%&${!N}k?d$ICeC|lEqhlODuiOx! zy}r)(8&CD!eC8Lf{&(uq`kPaKXV~4*4~_S0nDz&;3)enqz9&n6g!B3r=(nHi(T8il z4f|}k5B>=4^)_gJ<=6++eC`k3SMAxqu0FWVw@B@ev%g&N`%#B0zJC5z?Y;Chsnf6i zX5RPE_lfj%zP@j*dEMnW|01%=;^!Eg`*Em|QSA8zMey+vt=S%q9 zK^mVY_?Vxc>)(%`%+GJ8!cX{l(O0lcDe>47-T-fd``~ZT#=nmE55c+cYq$rx@{3hI zGl*;ZVdL3)ZGF+Qzn;$>+zL~n-}tB6zB2h%gOTQQh<)l+zv}$Px9gXOd<(*2ur#!K zullUMJAMX0yS`RG&gkxBtmmuUpV7A0cy|+bI$Q{CJUx&6p5OL*;&uHvXs!>D+E?Z} zb%D?8myLg_?Avnvz69#^GTeNMa+S$__-C@eptQk zAJx@lAL#upQoZ`CgWm+$6YtVjuMZau*w~(jp zuj-Frmk(VzSOu!Tp!Qz+q9&e8znlD2UkW>`SDn_QI<3d*@4%01{cQhZWxts`wnizR zx%lzQUv+-t+xnLi=Pmdi{195bSAAChH-7ZgO8WQpT?PVYKaC@qQ%_8&CDGaeRd%Uzfe>IYD2C>T`m9U4K6Jx%Duc``dBimt^1Q zbHQTA^FU3WOa8aT)6e%;=Q+%)za{9e_H#kNermjvWj}>=z00w_LG$T?AFupX=QqAR zp9T}>3V1CX1FhbxKC6EjKW{=kzqB7#uj}qQzjPnC&M&opg8lO%diDc*b$;X9^=n3)R;@UQx6=xRXCPwlT`ek@ErU-G*Gy7C`SKC$wDn}`1q zrs$;UG}lkZy>H;|1Q1e?K*$0f7kh@`$n%9cHLdqk7)b5)9hnCk6iJi zrT^M-zPkKpqF)bVWq%|0opZ~{N*1mlJRjQi4Ojmjan^88Fo~`Ev{Jsib zgDauc|H%ArFpj=y|E&H=j=x@Bj%Z&mswm@uaC5x1l(E{MY3$!*eX`(humOHro@9S3 zs8j2E+2pq+V1BPh$=|Em|0wy3w7%2DemLtf0*1R@T2E7->#I7y@$FgBhBzHzci0zN zy;pr!KLbCHLp{HA|60B7`#hYV+DAP<)lSdPXxnG9uU+xKq7GO54~@N-{y;#zUbpo6 zx6<_WFUD254%UWxeN1L-uWPRBWhvI-WW_6Q;zhfj<}vo)!T8sHe?Z^uehsScYW%0* z*X6%YK>Zwteug9eXz6bY=-;LH+ArECzx%%^dAZJ~p#Dyl{yXw+8816-6nqT2&Ucso zW$a#yvAterzGojEgZiAR*D-z03z>hB+P_V_uT1>BTvtuFkm}z<|9(JwjidE?)#uW; zb(}xDS+Bb)N#@VP<uD&7un!up(3S;M$pXyxkY&~W1TM5>H^`X@dVg71(E7W?d zUf))3!GYFr7JLV;ffm;O5(MGe_XuboYyI$m`RqquBaeL361IUH{s%t=)Zc)gPv92V ziT{`Jk7K_1oZ~y7{(g_@`u{kf%YGMjvC`}B1M%x$>*K#o>hK@d=DYsneHBdO|B?J7 z-wqxiyis9^vwh6YeY%!wI8%kuKl~&WB+^E>+$*2a~SI1 zv*^Qh+!f!YzZSb2V{ET~C*lj@%;W#_?SoahaSe9dz+Czk=-bEGz5sdJJjxMw1l$3O z@_PY_IQ~DZUw=2@uWf|>yzGm6)ECq~mH*eN{h37mvmL)LA>PD~mi-FycrRc+^~q-h z)O_y6&oNkl_4((*%QyP}Rdv7YME^64m0o{OmS6uaz1FGy^IE^4`UmL4V^IA+=-3~6 zozQjA{it@juUzNbQL~OG+x{5wn^CXE??Sz=!_|)Z)ovvAYNz#ofSs#;Yp?om9r3LH z?>+qY;JE&N6V?Ci@UL~~dg}UVAME~(mVQCNd|dil9M^kYSDXLjF!hgPr}vZd9QmvN znLhRE_cQeB*Oh-z{WJJg|MBGQ@;}$;YkBC`1k^X+KCFMgWDCFNU_1PS-~W=A{};Of z9)NZ|qwTMa$RZ19@}`nBN8H z!?myPVV|GAxcZ}g-AjL`vJcbX9QYE{eOZ*aw%#qC9<~56aSHXAT zM)(Qzdp`CizmpYjkck)VdTxoa|9g#p?RRsod-nVYs(;Y<|Bm%^`JWX~|E+{1T>e0ILo4Q=j z7cTu!?5}~bvM0kA&jJ`boueUC|6xx17%l-%Q)_xT9h_~OTUj6FdM^nFk>x2zhG!IO_5n z{|{sDwI1bIM~&yD*LBzZV)N=ozT@F7@E&*{bnV}cVFEqF8})b%QPP^{Z@znR~`9BOaE*@|1Q1P`e~nB`*$#Txc0x7 zy&gY@-!m~BJ_^@DSA3U#J9c|wY+s%G(s1|?e9m#dQhjfZ_k(`z_5YiPYu}Q1F8ilA z?#lloPRRZ6XZV{vQ5o0f=fnE>oI#97Lu(dP{|J6o!w;Z-?#cQOsxOJZ@^C7=n&+Sj zXG??za2*`x`28KzBeY+R{cfo7J|}K};=1DhiCt}e&aQU*u($mx&+D(Q(AI19x1xIq zs^6my|Etj*fvPL$=wBSqrLCdr&UNUgpj!-8SKXnn$a8KS>)Q;S{$4y=ZptGgdEb_G(tQ;z{gdRQzu!s!PNem(x((=6H_5So zRHwgpOm#CHdetTH+^D)|9eULbN3XhN4!!CgMXx$nKmSIje_vMbhnd{BbUn0hwqGTA z{s`B;eT4Rz-2Xp-nz!yJn{O8T{A*aCmMF@=`|Ha1Irtj9p@GcrgYgYzd>EeFNXE^Z z2s=U^(2?8pjW=CbLH#Rj}w*4caw>${=&8Y5kG2w zj6T_TsvA^aj;O9E>t^+;TaR9Ky53g50iD)k_k-1M3sc`JK|HDdeC#*tUv)2`SDo%p ztJm`&jt|zi!fz7n#6Irr*w6o8b>|;P?rIn>cikXgYX zDnY9#usY@p3{qID0?J%4wsQ?)Y+#^OL=-Tfq!W=5g*in~x`E1YQ*j%*vH%Y2gkf}l z+|L){Nlxx_Z`(Eg({JC~&pFTe@x0Hy@4ffEmEB#~*U^WiXeEBCop_gGm(6ox#ML_d zIdBi4;+8r5dvI6xj4~Iu&f)9f{)>v+>hR*G@_Y$#Z#lfUUGU=m>G0ypcy55W;|?!w zHN3cPLv81y*GJ%P=6n9)`Z~NgJr_XSP=^UUQce%rhTMjSo28VweZUo;86*t}C z#p!$5;^sTNxL$luN!)`DFYaD=aVs7E1-K%074(hl#C37j7jgH)i_3TTwQzR7KA8Qx)}^(qPg>V^nEkk2 z2epnBGp_cVQ5GkE{*Liyo)c|bh#n}8Ubmv172kg1%O$S3e7*-)iE6)~dYsHW`b}J1 z{->twhaJz>H)(%8f7$(_zgT~k?*ga(#?2>d-?NSOBe7qJYWyM2@r}4Hg;6Bpe(3Px zrooG=aCmX=z>Aya@Z!q*NAZZe+u_AM2QO}s!;32{iu{Xv!r{er9T4$4Zqj-XZ9Vz3 z6u(LNM(pJOJI?-GocxRP;t@B8_?74LPQKz=;KgY@Jeg_qo7Pe9bCT9o9hb|Gj$5^E zYu&N_FJWDn&F7>V^cQFy+EL@rVf-i2xOl~R@y6A|=D*G4pVq!?VB}f$Tb$2F;#R?n zd(+{?^&AxW7pH#PeDdK`A3Gl`KOiMvhn@Vt=gfC;N8!b3e%g3GWkcpE(Jy)2Z(fL6 zJ`e7fnbBV!MXR!+*YnT?sI~8l{RQY8bRqgY{`7wgt^F{#NoYB|;_>+5hOfaclphB_ z9`)?UnE1qb@rl0%yEy(OxEAzdRNrsY|D(&N{srhyoqoLP_$wvPOzf<`DUQEm_>;Z< z-<-#HWgPLt@$2!u;EU1G@D)xxs>>*N)gg4gUd#JnGI|O3k6(e>`BMw`3_6|fT~woY zpwFP5eLeQ#A5X~7h1d5aJo{xPUU6Q0;vdH@j-O7Q8qlTaI;VaQasU2j=xMr-936o! zK{uhgf7sSD=UdU1O0*h%0o{)7M6G=*+h=lp<9=P|$ce}z^bvF;s{5`ye>dRgCUhF= z`5V$T^3=Oqq@n)X%U=&CzR-AdUCG81XP?cuY3-Aar~7wn>Eqy-daEw#r~0pc+5Q)h zPe=2`j8lT@|4u5tKFlW@Uq|!bPZ*Dvhw7(1yn46QUUdr{&#n)OOY_{;J1O5l{dzJ# z{TA%R-|F!C zzbT=79e$ev{mZY-r<8nb{MvpIe;A=#SES^hcKrX9 z?KW=x-&Pxs_^~F>{Z77V`QyZ;I{r4s|ItAHOSn<^lV30Xr2IGVEB~8$?fLI!`0X+L z`3ZUTsRp|{QT1E*7kd4^4nD2@(P(UL#UmfwF&u^XGR{H(F5qE z+;?I9TVD5l6v3CE9c8cQ@9BJQch1NC%{f1IDGAI%#ebLcWmjT%Z;buKl=jzRm)1UM zyy8o#%Z2DTbUa#tPDUG1)nlSlkA85W{^R^9zR`i>C*}9!SN+y=y}W+!reB(0vQr+i z^X8lE5924P{fF44wNE;p`0IE-Ohl)lGtr-;bJ0zx>e1lL7d2z`x|w<224Ik3m1+{O%Ey_MecX{9ycE7U*AbG{f(A?6P={!HK~5 zMfR1(KAZF3y*bZ40ac!H{wB4PzX|whFMp~}+Wbz~bN%9Uv;Zwci_sEv2C8}vqMo)N z$H+GuKF)ufKiT&R96u?)0l%u}#b&&06G zmz4iip#OW|LV3lzp1Ac~vR8S%1N|eq2R)1R?MZZlv)-+M5A`4CPw{QRZ+r7Mo4o$O zYptNkkaopyZLZ^n=J{`2B*!Y-|S((%L>vp+fyEkj43m!o6RU!z)2 zrZRtQKfVnY>Oan(;u{t?eo}rnezjiCCNHmFMeO&dqFJ2B?T^}ej4^zV7+!u4VwcJF zXpLk2i=S`!b2(pL#(DDFo%7|gpOezQ3cIxS$}cot@rBf-1pNVe8F~de4%KsYRFAPv zJmy!!{Zt?0 z=AY_p*UPrrze2y$jxT#Xr_X!;dGYUJ{IvE-#}hxA{mE76B(xH}6`g^uMYW#X$NaJV z(ETBy{^R^9zKX!{lk%V9SL_1lI-ie4Guu)_Bs#_~i+-70uhU@+^<(nuEcGMvgfARD1 zSBHk$KMJqB^;~K@-*w;E>&&a&sIAXV{QU*3rHL$i`%8Y};?r~8Z2spGN1Xk$&hdhtrz3mS!{hU* zQy+8y>c!VeU%pBH7oins4t-Exh7qUY@b;TS*?$g4M>+Qo+```*&?V?9)Yh|w?O&j0 za(}`U)bh`Trbq9Y+U}zv^J~oI#%RI6hl|ddDwq9j~78mZJyJQl58l zJvs%wh38cK2etM3f5-DP;&=*OgT9RJK|erkyhC{2MLBvCx*oOsQMR8-U;3fdsO4{D z{x_pqhjiR%`B@I1!SSW#7sEem{CoUu_?d;yN4@7}|MAa~2QHtklA)a}Gs z?Eikhsv;?hq9m%_j=hHvoQ zN0M(c@n?(=ot)=U^3$8Q=gHeQzu++x%R!y#Jo@K9^PBbZU;m0$*~b=&6a96HuW=(E zQ^@3UrIKZ~y4qYP(@9c%XyKWh^nv@T<<_MiK?r3>vr&{W_*7kHr{LnS4d^zRWi53QVUR6{zzB5FLUggiy3kU2con z)1wYr8`(n04MWtX*i{>e(pEkxp86Grl_iAIVI(7{)IzDKZZTNNUS}6F_eWmT`pk z^~u@H>XUw&t?U8hk3=}8CP%aJEI*nB&^*X;U>QfK#j_U8gB%*G4o4#Fs!6(7V6?1~ zD*v4M6yn6zx(;vvlY6r_Xe!gyRhC?q#Z8NM#(0VkYN*b(X(^EFEL~(}he4&Zj&%9n ztO2zhfviz?t2cI$z(!DuGeHG%)_qYg$hxRcH=8oQ*lx0fM5RUU#5TN|umSY-Lhg7mRf3updY2TZ5RK426+@uHj zlD>L{j3WK{cW$p)JgQ!8F>I)9(`Ct$r;+8sxft9;N9ya*>|av z=Ykn5D$EBHg65ue1|gw!F0)II z+3BCO5^O-Lc1&71cW=e|GLEQ^$K>p+PmsmglY7Vdgxj0on#bJiq$tSR?8&`jeMZTf z$JFek=;U7VHe_h_6gxuRI!w(|@92m247N50F!Z<8K;{wkBfpk+8NffdqJ}C*K-*0R zarD-5IQ!Uo`V@aZjbU1U5XXJ$IZoeK`IV;&`b}OmCV>aEWMi4WHVHI>-s%nH4Q;~; zp4in@%R1}uR|#}T&HT}k7I_NiSBvq&|g&;3pv13N`H z(5KCJ9I|wF#xvmZ?Cfp6*(|ejFf3hte~x&f0g$N5mX{H!nJ8N=BFxJ+Q5H#lS-hZc zRg!NIN#%OiV;T2rhyaGS^YpT~4env;d|Q^j>7?P+v4Uu_mY9RZ%ZsH?*_piY#D<$4 z{PUmlWl?R*>|g&HV16hh!oLhkYJn)N5QDTyP^I!N`IsZ3Yn9%#{Q{GTA26U5IdG$d>E2bZX+^mdze zf6f3p!*PX?&O)N8@ew~0yU9nqn>T>`G1%G_5O3Y)Zk>VOSF3b`&g6|But&xgcmSD= zp{tD`kg_^>s4WtL+C^t8hbE+ zAcV5It=dg&`vSRTRK4pw0o-diHaoEN5=t@4NpCbtyNFhvod|7X@r=MNUD@JV8%?9W z=Ve)x(GTh3j}_^JZ1Fa`h$knM^3HyL#W8g`2XenI7TJGiWfA=$vAgfnD}w)9TD`u` zmxxVT=`X*~@Ts7FD$1Ma{L>otWVYb!KW^8{e4cJH7rD;zyNU^Z%i+~J^-l?0$7{a% z8^;OB_xDBd?z$*98qUUwG_1UE(dWfwM6{%v=*N#~c?sK{e-S5W=b%cxD$DflJ@jSt zdWj*QpU%lR_^rs{ML%C{Z?ckE!k-xH^0%}^t;uYAbF&6H+2Y4+zCi)Knb|DIzE1aXTj+E^ph5ve<6& zmAq2gBD*9e6)Ofn1A_tiWu7eziBfPpM6TOqh-i*X3o?1=DrBi8nx@1An!o@FO&mw2}dUu_cs=Abg zzy8i4^-Y=540yH7(<*vf(2kDiZ#nKZ?9AuQYUDvN&1t9dDZ zxjmvka~N9rGT+=OKB>;2w1Y^ksU=DD`p3M)FGNQ$;Bk}YE6~O4{1J}6?NWUc!H(Wk zzpkkoDLQHKF)M#4wh%2e<6=9fPFfb5Vnclk4T`?EdPOmF?>1@f+~l0UO?@w z*E;*F3$%SY-~3wUSNTc_zpe{mKwn@D1C6sm51pftp;4tnRMG2nNmC_Ie~YOJ@%S~U zg8D*)fb=-SeCkoJi{5O{`{&rRu%M@ufRRPw}BURl%u#5@QMLW+U6!^#H@`USz zfekGCos>`&y+_^F1?ezmtKt=Ln0ZAHidT@noJ9d`x=PEtcTm{O!~7;)%F8=EEs3q1 zgy9n6dX;aab$&x~=0CRuda?PUm*CmFmit{gMIK&7Dwf^hHKQK6{AwanSh`q!K zuosQ;nQNaVL!M1wOM@Mx5L1$C@78*!%YYb^XIrDVk^GDgjU4#xe|^kp8MS(sVB zO(Oom&0LHgYJ#Xn%QM#B&_hVd0FR2F)GA2&H4(|?0KTI+um^yI)Vz=gR^|<)(X;Ka zkvt5h3Qv*)Nd&Y?V!Wan?G!3Wy_2$aKlQpQB8lVAfg-`uGk0y*qBH)GDf}THd(n~G zsUM?JEL%XR?TV4!o#fd~ z5~VUK@g9TdQp{!aKv-H;(#xIMiEK-_w0B1K(sUk;&Ed0N6?fa>gNV{;OJuz4-{I+eiTf!b<{S@3d8!!{JhvcCtNN8|JOv-I#dwkt8f z;b742@}`F(lLI9M0QHn{qy&DTkPs+1`MK{z()==yZu?)2UYpJGigv>03l+pPK*Jd5 z$N`QDvCGShW-qr{vMJaFuDd!WUYO*Nfsk(AkjCkyrH7w0%$BdR-rVQWFpPlIHbB$# zT)}o_h1Ej{hRO5p&oe^N_noBbyx}as=mv{_1M}xPt&+L+g3tmDZ>{NTY0BDD+7wuO zKF1Tg#6G}-nwlo@SodI^)I;PUrpueV^N4@CId_z=XaVIM0~7r!ye#{7-MOY$8M5F{ z^!p$0-v7t>FMs^}A3wf%_j}#K5qg3r2-WnKL;j9hX?}-@Y0?g}G6-h}gJSsd`TuNB z{ufJqp^(x0ZWf%J3}bN?9A&9+$a)H*Z#T?hxQLaESuLB~q#`icIwv1lZ$IaiPwZgC z>?e!TOISJyiD$(_Xg8mxGluF#eJ;3rVFZNbCBV$GEdPG24MS?spr1_|gF!E9RLXL| zR=uz=ta+??&8v9*p!0NojWZW8tL$IMogOKWUUcVC2vj76(uym*FFlQYtk)?A3Jn6| zCHaIFJr~(>lO~I7y}6F)*a;%SyUbpzgP4m@>wF>9gn+_&Z!W{6!3JSfjwL44UZJ?(M$t-Vqy&NA=cej81?w4 zn}F%%4Mqn1!6Yy(uP`US*kXYJez;Zz*L3DgvOPJE^ivx~pXV~k{=!i97m$+eoW#JZ z9uMfzT@&}83{jn1vv*coRiu{~BKY~PH)OxTj=q!!t^sDURFQ+I*ubU!F^5l0&ly_f z8fHyrsBt!A;}64tUZ81}b9f>gb0_F()oVc7ch9RDs-svM@}sP5xAaVo2^ulGR02^? ze0!^;<);R3acp*|P~yBFn0wo!SuaK-l40PQ00Z4Bf2vw6xZnW-b%~xHJDW&V)kHR^ zo*}TRNbP}e`<$s^d=yk96k0Ewu3>!CV&>sYYSZaYQ#e8`BE_Bn{Y3+)56#4t%5OY{!Q-3Z-8fD0s%}!9eqC zx5m@h1K6Ar4*;}KP7YKKJ)7hxKvSniYCvZ37Pu%WwVnCY1yf7*vaHr7q47DQw-(N+ z(r>Hna>;WnIqRd5QzPz!X(Mj2seu#rQEz41-iGM_!Z^0#HQSz!QP~|_?TyyDC)t8& zYGh5tT5QpsLU?IG>fEglEfld911opUPCGFyt5aFG3weucD9!{uX$T*<~>>+3D=!JF%nrLwc7k)lZc#vg%ON+X$7{$uGHYANPs;QRI1>zGv zL>%Rych+NuEs=ZC9Ns0zJe!%;!S-hMS@+hfZo43Gy=I5WkZNkTeACPZWK5M^Y;|oQ z{bOnq1a4}hpy{fQ=$d+W|NwMgVISRWnXgb05Za?B`=?irZbM58R`Y`@lWQ zw5be@am&PbXt75_({U2&0zS&Sp?!pRLxyckcB_AZK=Rqzo~SW_E*=tL7w4*#Cq`g{^k7JIy{iLwv+@MIc1@xf;rOb2uM z#`cau1!~nNU!D8ked5&#?31oeV4raLw3!Ac+w0q=sA0a>qEyM|-Er_<^8H-ZdraE}w6-j^K_=d9YI<=^BAaY06fP=+=25&8 z7Phc+uB8%;k0+kmGsWsY!-K&&o$>m}z~D9v)1vw?_HSSuC~1dGk?f*`?}3Pj<9V@M zN}yvkg`}9Op1_aJ7A@LBJ+-I~-n-Xs_YwQ%v@me1mqpQDplk?!a}mxz2~gAdCg=dK z#bg`P?C0;t< z8`NIgtIhvLO2B(yAj@SIA%yNW#-1R_7p;p*HhKM~AAZ3}3W#D- z!I|&K|9_c7mJv9L%_wBmR}nlDbL^bL7MPFu0wFK2=o|0Zd5hSD=ywWLh+cy(gjJ}m zn%D(}eRH%=jtY<`>^ECt>!2!%fMF70Au7UuOMX#ftIvonwZT45Fw%OdIo^Pu2ngB_ z8yj^7dF#k=AUSB;pN8ixfTj=I_)i7+CS7Y-JVwuNwwq^v$DA)Soc=RbA@5X8FRG(M z+G_FKiJW^~Z=v{OC=#C!3ArbV=u^b;93tO#RpaRa5hxN?@HVEiGuhdu5+(5I?Cf_A zfO5x*F*SMl%wYrXDgOR@Rl~=n!QXr_0=mF^*7v$0`T@f3&C=$3RE?S)fjSWM{4XD?7(tJQ?yufA%1P|Yd3i=;IllMX+d z6i4GD!%;l?K@SCXC*Eo7yJhy%6CW@^d*v(~ykD{*`-<~ddV1pPQ9sh&y;r=#`73Wq z=-_u7&RnYng(^Z=ww=ZKD?Rnv_B^P+r-K8A4G|m6Uq(C) zH}UQ_@>{fY@?nG}|M0|gFzk@+kf8A(R5Y9(M4TF&rSF4>k3yfHt-@7T;RJD_Ll1@J z#mPwdLT}!w)Nvc+6@t4eTQ0D3hW3}rX`9^h-mwdachf2ePUZt^^3}?9%cyNZm2o{y zXK-lIVFj4vpS%|0c57$*8LdedZOtEl*(F3S-yqom0*N9X$R?nnkci(s9HsTF({Nx8bnh~TUFJmSMs)YMs@KYvLV3!S7rqg3HJ@!MM{h)^oD_Zy_Ek9(Khx~SpgI%f$IO0urU4sr8p=-h z>G*&f%xji7tvIHKbzeISnB#|QY2oq#&-04-MAq1-xTo;W?IF=TXq_hUWcKDleVkhA`tP0J`c zX!G@zKt==Yy&8)0`>8tRVcX@HG{O$vyO!jHJ)_f2>>tL19!DZqy6D8V;Df?1Rv**^l$otBy>r9fm8Dw@cv|fcqi$*XP8iQO1EBU8G2 zq43kjOqO*)66Wq@!DAjGL#D8%jqX1`9Yp1?luU*6`W4lyZ~5w;W-(TdREOQw6o4K7 zRmV7SbRV2T0K8_uytFL$8vkou+{zD^8YPp?^dlR@t0{4qfhs$iyyo( z_Dpt9I~;zQcDP^9AnQN{y#2d%3@_>TRlKy$FY7OkK=u0A118~4P-L_MV z_ET?vNv>oxVw7*Q`Ptc@+1&MvbasOnHeGPRlHZ`y@|D( z?U4zHRQb`bZeu%i+vC=FjXdeq$d>f&LvdK>I=fA8?VOK0=q^7AhQ(mu3oMR>kMY%b zmLDA0D%q)$-I~Dde3c42iH$5Be&W6?_bJ8YR8TK%w!v7{5bvfMnuR+2@bXm>@2L>( zTdd1qwo%5*+*BMp%ED1dyi9N%(q`1ytIO>13F$T%V)2aZKw_x#Uh%&cEn6XsX`CGk zG(zP_z@FN=qep9{>J&P%4jsrlEr}CxUEHrwH}@BGcDO|cdb-9)(Vm)Ji*uKBg*G&u ziT0)4nN+09>8Ibw%&Es9TQ@Op`RMdVp;};-?0yd!_Pu_>epI!sM7CnK4v5fn zI6N={Jywhkr~aki8lh7X+Xz^b-zi(?5Og@fIfccC&Iz1XR6)fqxg=9JDbwwWPIMei z%TC#F&ma}n)vT5=RgG@ihQ%RvJ5urh8gsjz9zjZH*q;PkcKE#)P|cDN9c<{r%B) zifSBtH!Pvtow;~TU#h4yt@4WHSEUQS&=yAI7gSK_v@(Q zzL6ayej`*xd4BO?JNY0d9q45VIzO+kebhv{6RfkCSavkK^$ryrho&)|yT3vX;>V+> zn{zRTmb;uYX6*fZId1)FqSPUy4_RVo?GJrh*bF6ovg*`qAWtU&r^lEJuQJzW-lx%^ zZY6)nB08&gPkr7MKiMHdq1~%9F0yjzn?yD7+wzMzLEK%OYLkSIi(05~Y`Lx{_(>5)uKlgl(;W{b|_Dnp>u99N;D+vDZN9zGsjN$B*j ziOH#&=2id@g7i0Zs=l169{oc16}&5MGL_&S+HoY{)}{yB3xgioc&*vg$|j^K^yjM1 zx-=o?iyiUaO=e7vmKCXLS-2{BL&-xPH_!9QFIpmB9b#Wo(Q0GnLIt7tvRwG0B@!2? zz#dtMm27`GQTW0I>OS9p?*~@D|9JQQKhA%7{p*kCKjQa~pa0_k|5$aXo{QrO{a_4a z{JxG>kXQpc>hOhf9OK!I!s`62sdsdtvC*2rT^iauakf{?X|>-|-}pbKT^6n^_3}c!*DURtPG~Cel zomD};63t1YThfrEQ9rl9yK$eYp~HqY1Bjid(Sp|R^>goujpsOQ^N!9SY&lYGf#_+e z<0I8$_fm{$w7p={9B?=7H>#$Ax=Q35MbFND!c9{@Y;lhipWWEYt(SBo76P9e5omHs zrb706`tdzoR2sl|e^;D{>gm!d@hPwH4p8w}Rq8+RQ!kmyu7-c>DrnJi)w8um;%{ER zp0!5^f4;amBJmlI-CAX)4L$Zo!H{jf_h)2F#mZ>CcW2Z~i8`NB9rCKV6=G|c5w6t{ zWggHyvK-U;NU*UvSWeBfFF;+pV(edG7x>_Wg>ueQd z{SR?l9p>CWks3x~HoHGz;v!Akn9jVT5757W*WL5u$(Y&9eu)|-x%(};Tg)e;X8QZ3 zYMkEOZ`lJJ308~yrHVQ7NM$#Bt4!p6DBy+r_snKEd#5FRpoj#J>BPMm_`X_lpK%UN>R=Y8OJ~t9 zJ!J8zVIkw-m^HpGYwEm{kiDKs*4YS?16e!@sqemf89=6lO--+U0OFUdqA#m0qj-nBlKa({;CDU_nq#L#f>4l zc?l%OhL2Ci6<$B%5S>ra?Zk!6S2e78j&5)%HQx;q@zo$A9el!(zIpf0`1=SqP)CUz zUN3OusN)Royv)0+vDOWTT$W+|@e=kSA62U!4ABl?d;p&hs1V@#nOs3koH~En%{Ie# zTyfjTYQ0l-_1abPX}(%yU^88}Z64*RGYbR=-bl`39(#bdsk)gv`%67W{DiB;tsA+W zs^N;DPw=A zqRM`Y^b6$8CD8~&d^#`IcS0+RiPX-Mhzp_{fKgS>yXr7It=rgL);1G{rjwU_j#^6Y zC*ZRQ33eC`>foWUowYrb4iQ}BDDbXQtZW`7`xS zLDzVd{zo4>^B-xEt5LWqMk@Od&1TZ;L)&l+bQkfHxWTHr2HxZ*Up;lfsn*p0%k*Qi zPRmUt3~i@H&7|nvGM{JXxMW?AI;zL(^q8zlzPw6bh0=T{XBdBu-et@Bu5Q7Cv;7)G zaV0~UymMBkiXLViPMgSsd(uwX9M(=IO)PcG`_nPh(}gmTeV>e4t7(nlHjKmP#+@@w zWEfx?2~N%q4cf@kS~}>k5o`i^@x@LES#Y!ApVMr?P7upRmJq5O7j3=E*YWiCV!K*Y zO-QP;mUtZLMhrG%)HKF`?(m?&Jy>xCyj~@|ICl}nioB`EyD8EPrjz3GJFIa~gqwcC zgDJQHEBbWLc*vIy}$=N7iHWNV$YalI_VbJ~(W7S!N zAyRsSj}7`x6#&)8JT?%|W*A~VS20CKkXi?R+08w{Jr&FnW_{?Lp!KB+3n(36k zxxBJh@Xn(_kOs>5K&Thq)z1x}X$M1^1^d}x!!(f2PReY(Oy}8BO%x_E0c{y6Pg`pdNyV0MNxi9{9zibb_-C zdK!iLAO;&!*AVLm-5iz@`qxGG>(2c?y5MDv4%Wo8seJcW{t@UsR|G#9f;u^OES%$6 zpI3Aqk$X>yDAl4oedoU6ToT#TGw}iG837TEG5}7RL4()Nc8666hIH{Gv{$EaF%zDS zRCTKU$KV-M?9X4SZ(DW@1&&au1u@@M!)pGt#9hH6mNLHKM@7HimD_}c-2t(js5 zJ(X%{EAgq&)4{jt%wA&cvPgzuI?%^6u}+`V7@(TwNzo)za~_t+(})GUbp$^n%>%PC zo<@{tIg@7eJ_V4m%wy`@$;W(ioqWjdD)z3_uMyG?=uaF8O4a=d&Dk07I9|;U0FI+- zlWwwTgDqy!ivQ}#dAs;ov-p1W4srd!(YsoWKW`5Shdi0B*WlZ{nF>gMHT9%e^hHNauwa`lvIUm=E;msg-9!ac=1ty1-N`&D z*;9@Ex$3D0`sgxW<<&KA+vDFJ$f*Jys+1t=hw3TjWMRnHPi^QpUKZ&h!OS_~V2App z3F|~}%ua~Qo!%H6fp>QHV|JNtmz%$F?}OuFY>wz}s=qZUITD12Qk1&WfQLp z6&9VZwl`Uc&i&nUtc0rDsd(l5?2Lmm;3xe)dY^vC{(g=9r9RJK#c+)bTd!2q zY;?IyucCFCE%G^QlrqgXb!5$qLIjRBoM`mfbOQHzk?k?HhN0t&Xf(iS8l} z!QkOi@aa$S?4JS0!F_zY4((PoQ3pOgy4pN)lfs_9v#CxSnN2u=)r}J zmHQ$ggQs3@=K-nBrqfd<_dx{GYSu4M$WX%^Pd6C*laKcp{Ojh~8!F61G*Mj@_~$<< zLib<)nqb)%L5|W?ZNs7G;}GUUtfK?-ZYoyklHD>7QFba?>p{d*Yr%&>?$k^-;uLEo zV0Dj)s`%Lh(SiC%6fky(}|q{ zZ(V>Ke_=2;F1jWXB=)zsOl}citYX_0be*J$p_FDhvB6A>k=9ECY<8r-c$y)<*lsd6 zxONdC=& zg=zXGN5P+Q29VC?OT~FN7H9)aJiRPRdzqv08p&HD$F#HI?BIvG;a!H?Pz}WKRP#`U zhBRIqpwsU5w9&&5>uPoztbsZ{X)sHX(oSn+noXJjeRZb8BdpeF31lZ_vslMwh#q#_ zwbp4C=yt)Nje1N^SOf#`hruqK{9x<@Sdi9i0@_{t!0EJ#l*D{)%zC;r=i;D-bK$+8=&Qc|>LJ4`qYo*CK12-t72E3&;Xw}B2BO$`UKjvTv zj2rjIft}B*+pMf3td3QJVeRCIwhFqlQGK*;_cz`dY1rFF%j-tG_v|H&mgvaZc@e zG)^fr_vF&*d!ugMV5XcSi2ydO8OM^K+!xjh!g{P-X?QVt=oHP1yG@4ZA=sP-^%Sc< z1}2`+g-Gjsb(LU^i=tNQw`hKymQj9!_!PQ>9xjM<5xpt6#iL&#eV5DPBcVMK0GKlA zBF53u6l~pkK$Bch2*K|yYzT^K;A*_W7fBx@s-%@OKGF!` zA7l<-N4+XSr&!<;&J*KYP>q~|D3opMNsK;5e{W}pk5ve@GPjbVg*qr%aV$Qx=|Uql z38hwpJVtqeD9?2n{aQt@x7DV&iMS2^ix#*&oj=*9>l78vZ!_|Lv6xT6kV#$~0vyY%dToJsL#kc0AJp2PA9A7& zSkYgdBVZ1wGD-F+o#Ql@`kC6>pF{RRCFydhLcZv+8(fE%r8gU>@a0k(sKQzpvy~4; zm7ulroAu@{QX6qYk2sCy={lX~$Vj1083*MM-55yg!Z@VukV`ZSJUpOy!{#I$;^$sw z(HJ8o?r&TktiC(NC`uFZpHoO9&e5hP$c}ot0Yi&9*oqEzldU$K0sZ)rM7V$s(n}J_ z1+y=@*j`>raAi={XR&4)N9H&qyHn#}P%Q$3oS$LygJ8ieJ3WH1 z52E88Z$N0OJDzqd+=wN;a7tY`a>D{_KoKfwQymCuIlMnXW9O z?mATGmwP4<*7{=5oY-8lN(>U#1Y&;3~|7SzF%g1(E>;C;z*B(Us7@ru^4Q6?5`Qr zkj9Qgu5IgxIBPv@uU9NZ?6wS_w(hack97sp2oXepp`4S}=U08X#~ZK2hy5i5ehfEw z`|Y5Sxjzw68-BlFiQ(Ls9ANUyXRwiv`%AVM&QExxk{boXZBIACg>x*H4WW|l%QB3& zPJa1~?VueRkYNX>l)lCIMyRK`$aKILAUYPqK_zuIz z^3G0Xz`A67841(9vkWrG|55f{8XZ+=r^U{gNs1GLKYvFps9k zw1;(Q>_PBP5Lj@rag0-4Lk!7%x5;CaI)W=`4Sf-8&+_@cJ=oN`k5KFwzEm4J8fVb% z-M6?rO0jgz6;9j=jSYw=hpgsM_dpYA-&}gs;>BFjl^;8M-!Ch|+blj@({(UkdU@gz z3XM?aSZ~grTV<4AWni*C3fG=P26)p|&;r=xARc8lKjlh~w85WpRl@bd=`_fof6A2& zoE<#nDvW3^b4$UHPJhbvt#{7-JJ%w*rt1R}>OlNJ5pa~Cjs*!)pz$fr4-JTVdfKh| zZ(oJr4mdfGWB&r@`cJ#r#&|8Ba!rC;FFqAdc-S=bDOXu}eg6F`BGW3a$B1TBH%$IJ zj~=0+GEZ3w!vn&ebS;8CAg9xy2%@K4*}#ybr(A`Rv-~Grm2p8JPr54My=W5R9(u}k zMJEdIsy*cz1dp}TW4e4~$b*Q5n;@_e#fHo&P;3xbaIv#f*)N{3LBbbh4A>YTVRV*O zj$r!iJ-LIM5D}d&SHlccVgJiKk5X(f6R|^oY?JMuMUPFv5VzvvdAgi$m$8b;^=3GSXqf^!f!30vMbDIM8q zZE+qLbq)`2=Xe7mL- z2mVawDfVZ}4}arQqhGM+9@{T*x&Z&Yq0RoQbSb&t)8gryO18WI4PK|8>CDuDV4atp zdmUR<7J4tX`4asX+h}QzZ?Z0MrG3xN&MDBCz40f9d=C$Cdw5sv%MoqyAZQ~CYqyv- zDtOF?qYsUSV`>(%u{a!GU^PPbTX%M@CR0ukhgsh3lPh~h7|50r95Y*Gf$vMeDb#dU1Rn-*ldlq-yubujm3Rae`w>Le3@_ z*@6M59fV_4s+HFY#PLO|2Hbp?+xke9wtd&=b*G5NfMjiWY|Aj-mSlDD-QvA5^CPtG z0~Y|-pQi$JFpG+CuCOCBwB1jtEG_5PAtrlpOTEP43Pa$bFk6VDzoP#;w5G>R!=x{M zE3~auMIQ)p^#6obE#~9c2ZB{bR%-7kLn|{Ax!IdWZAjyOu_{SKT(>cR18XTatE$`L zerMJD+Zc;6o~RLv&tihnE#sG*%(Sh@;uX!vCNS5!Z->_R#>5VVr_)YsJ3OF>Fso>a zr6y8uLoEdb69{ON$C$6t63``lp+}kZgljb{2KYUA#kWwi(BPQy{ZXoxEgz8rX+Vd7oqE>(xDu#q zPvV8#$~mM`f(;zV{@zh)jDeMci;`~|<&$mwAv2B;vx7pc-V)ANa#yIgz#{d@f?>ME zTn*a`9^+_wF+kb*F>HqKH)PY?s${Iy$0b9Xwn9aeSIJGfnP1~7)Z!w8fA0dr{E%%w zn+n`@Q4F#Zf;>R&?^9qOYw+ZidH6F*)d9v z9DVmf!LGunS{4J;#z#~mt0=ZEnkmsbUFCC}y?Ukf)u~B|k1w(-^bJnnGopc@m1L*j zoHdxp%Y4P>`X2HLpobU5r)*(ZXdn5CtJR_xUq%0kGq5Xt|MBI`=IrcMd4;QySDQcM z=(!x4&dGlkIXc7Yg`1`_*XTu zBBnW&-K~r!u7U9TuOeiK{>?;r$6;4I50&t~vWDnqe=4tc`@W z%{cgh(kHY&5n6J;upZ%nZJd}Jj&2-$e?<22yb)=kGGTQ8XMunhF) zKVkL4dG%9&hS`(@t3u^;1n@w)@D<@CKVDA>EqrW_AqqH*XcPrwa|}?R!puYK-zzz@ z850-WJA09x@C__1M*1n@P@C>yFrL=T;f#&feny-~a3A#!{nVd99X%#)v4_CO2Vg6wzXY{~U2<9X^J;O7IhY;}5r#4G1O5y`}jR`H&T|Y7&`SZ~T;V#8s3i^O;rTx2+GUy+B4VoN7zgleQw;J9;#kn;rqvgk%( zB_N!mN#t%3j#B6r%U#XgAmroiMY_vJRUlj75w=8`azalPj?)`l@q>tu1^R)6)dcf< z=d3VP{@Uh<-(c#bA%+Ou)$UU|2x_ZG)Vy|IH`fp?idaO_^y;Axb<;%aMmm{z_{oL% z)1!8B4;ja#1sI!p=z*Q$s?|QtBzT5ldWwh-RA*(9>gS#;BW*+6W+u0s^`Nvy`c}Kd zv2@d`ZCc_2H0ve0&Y&esh8exahuJ4wi?+JT;QOaB@EoRmBf-gV!cYj7Rj4U!)NbDO7$3hs4g!F!hxrzZv^q+L{3}sY~_?~k7xTjsRWl>m{ z*@GUwPn$zE+XmvwRU__g0V&$wS-xmJuof#6J=F&J#Qr`ITK2HkG8|D?3nKGuxjZ}j zg9nG#c>YdSEF55K$?JP8cnHk+tZFR)gnwm-t5HYKt>+g54Kr@v;eHpZzsDNQuyk?z z3{^uPTpEJ$1=m6(gSub8bm$JsnR-A&R7GWb*aOt2-G|#MYs{ilYk|l(4nq~NTDN!q1xc6JU6 zDl1_qd}n=f@HEyl$7B=nxEyVYLsbxPTdoR%EQy0N22S^4rv)@Xspr!wa$GYgC*H6j`B zu{Z7SSPb+|=qnaYhlP!X!fLv#ls2loZ`AHu`u$dV-`}mXN-y-fs6+d*!xX~#2gO}D z=?!Lm#g`xRE(tQJPU$LSelzy^2}P?USl#9|fNp^4WB`C}C#EnMzGhe1dM8w9!#gXk zL{obA0)zt?AKFZYH zbY86Q?DfcyTxthi2>v5YPPZT0C8^=l9pLSRwkzN!;Q4ii^Y|Cxg&a5X39m>?Y#_%X zE{-6Z;kB{k9zExY>)Dxd4s*=#IPY?wK+aAr(Fr056OI9v&F;16>NM64O94ZZ3~f(# zvKA5={U5E6CADX^Iy>Vb?k!#J_?+i>L3{(O#AAg*8{<^vqp1uRH#GYgq8vMlG2bTi z1M>&?5P3WDV4ypOCLWbrMuiL`_=<=$*KP?L2U^)$p+qeR(~)}E#dfBiqVwS0&DzJX zdZ=|gf--C_D3L^Ll$JMoWLd(m2PH@C{FyGd2*pxnj043Sw}$iGglfaF^OXv zot|5W*ywAea{2Ezr5F_c&t&^{d?%wpe{vZxYMP~{^gCFaV~Xjvgni9I_3pKZr0t!o z4m}}KC7s|VEkDpC={NX%H^t45=_c(jouW$b!}NWIpo$i9sqyF&iY#nVtyP(uKU9rr z52uo%ojB>${2}VeY4F}Q>k`0yHpLgB8Ljx`N^<&`(q(Qis!Aq>tX$q+i2p*EO-s|6 zfsGjFJA@)%XHK!r8?Qy|FU8 z{kgc! z8X`QwO|5$J-iRAOG={Cx-m=~5?2PwYot?e)?ayjxs>V`pz1C!Urepm!AX=KN)r!s- z>G+)cn&#oOoSwQ4@P1tX=&Q60GGI?-xo(r?ztMk?X;_@Zsye$6B>yX^+%6XV|bp#f27|y;lkcYhrzl>QSMfLPV1V&AA%iwtWaxA+z;WN-nTczuB47p$s#Bm zC1Xj`{U?s>wXjY^;<@lXn~D4}OfjF-r55p=r2_Ae(j}K21_OK?f=2e$bZ^Gt;dksd z9G$7|Mh{Viee!)DJVj^N4;sPZuuOO=f~{<#y@@vA`#+!2A#wb09Me8RiD9EAbYyGq zLhs?*b*zQZB*YJG;(HYO>%|`-@`s`pSOsMwt0?s`InSXAhoh16!t!xy$szTSry9!oxS9!;#& zLOsB+`}xYOiEwY*x!uE^MemX4!J_iuHQ!^UT9d{75$_SReIoY`;JbDUXUquNoY5(l zEu9^Nj?MiiTd9W>xlrY^QzLn#@L|jI4cb8hutyeSv*FP)yTH9BGupaqw$je%OvHgb zy75tbko@%K?RoO{%`fKzz|+YH;GbFzX8>U6={HB;98|YBCi>eya87RZZMm&Byw&jAs+{xN z*Zg`>Y`(3Q>23DyD!;nkEbqQvURP z$x(7N`#Pmsf4xY{kNN7`a`UY&EBXiSlU-wvFfPEwVfsJN$?YoXAKTRj`x&>qA~v;= zLdQPoguieM6}B4D$@dgFp8#7T>dp9|zu1#fXG z+}yl<5)SE)w9`O8uTq@aomO;AVFM>0d^!;-(hGv7=ZTz$ikQx5i^5Mpk;#aIu*wLV zJ}>jD1eao5QXVNcp;UTZ&2d6(Bqb(b#0T6gOAMiX#yFUD(|oPynuQP1ukYTi^VM5= zM3K`8z*M~z>HLFK`Na);*i5pLP~eI#!`M{OW}ae8bMn_R`fGWeT;?Tqvl5L-ii?A8 zXy1ILj;cy1Ll!M6Y)CJ0x@Cm}h_h&Ul`Qgda|c3~$hj@a*$}N#Y?rvrBAmpJZ&3+W z2v{_~M#EP)?GB~l_LzBkmuzzEVv$FXk1IKGfrPwf>jL+P$n!sPx^F-e!5MMR*McuY z5C~P6yj28VCb z#cf_?%z7l>fUE_!<3Yd}%)GNWp#c|imK!}KOS)j0P_U?}w?*TPjw;M5kRe|UmV~*d zv&xIg$uCP0uWR|CE^#zNRk)Y7yQ)nAH!)S)lUQkDAz4gHw9*7d^KD6Trn3{154c&yk&w-X%K}9M? zaOu?fU_Wptlre^f8&0wY9?nZ_*O~bl(guOND7Y;u;W{_Oh2}Nqfc1cE6gS!50|R16XT%??xCXSb3NN zoYK0|J!w4Ueky|xECL(|BcrC6bI2gs)ySYO9IMZ4s z>2^~@JYwUT2D#1#U%Mt!!sOBVH zw!A4q_yq^iuG2~vg)7%&2!c|v%zCDVyOpAhpSNfzO}ErTx|r^Ib*{Q|)oEpxW8f@T zWb7zm5Q4P$$QyKsGDLZ3SEH*UlNcpqwobTAm$0{-#7&8VGn_Pa+hJ7K63DC<(Yfg- zR(t%%`iS=?G$k182Dg)a*X@kTb#F z$BJZ52Zr0p^o_M6*voJ{Em1QaJ+)F9h+>-Qqz97FK!iHrQ$3GrzF|UCeAuozH_gcs z9H}Jb<(C+vVHXAfOgBe%{>hHHFGnuJfj1RC1>la7i%8 zD|ugJHwDIEnp=<}A;jv7fh0pHTi!6OCJ`Ej(%0Tpf1>Cr{-$zsadcmYOhRksfE1O; zHNwFh`8-J%IOdGaZzu6HzYELsq=9Qky@j&Tchz_@3+2}3$428}b_;lJ!3Owt#TkI5 zZdfwx)|k?w$1O2Q$yekhp<*h`c#B1JRjzqTNnhz9Bh6wS;iNK2 zg-R)efrt`0$*<@{%&1r{c>K)Z?^)0rf2NxQ=`3T)pk21uq)QzgUFTJZ`UjYVFnI)}vr!Q^7t zE5ZkD!`e4m!MEP!gWFYR~ZUc0_$anc+l`y z5lurBCdLU|g9KQ6@^lUyDzsRFT=Fs{u->=q_e&szgoS71s$$f6)#GJGch0I8CAFB@ zYOkUinkmB^{$mKS0#zSW4)w;_l8UnP>|T%T)pkD5YD^O$S;SsJ$F3LEM@x%$Fb?v&n`Q5*6sA)g79ZAJtB&5M8=4gp8&DP?DA9s)*?C zmUVZPPPXuS!H-MVo>G-6B*>3tT_CYJK&fGhhbKViYpy<-5A;^@+lCY%jc}BvBtUe% zB|xC=k|wO|Y$YR~-fPp+q`rc`46iSF^CIc8)M=t3WOT6xs7R3u8{ zc9ilf)&sCj6g}i?(_i2tme%^DtV^n=q%hkQ2jm%7_d-MPv&v+%1>(RQC;v~G!vDR# z(_G4{zc?#_i%?ee=$NZ##w#rTL@Z9q&zzqg<;WbGe1m{bu;q9rrvQLdpuI}SYZuk#ghzJEkUKtuKFCzS3(;o(|0WR*;NA(SPrRLIsH|K11DRYxeo0@E--41VEa>3M8WvkhWB*XLy5(Q$8W0CGlZ~16AYo3{(Fghu zJove#|MCb=O19qMt~zwfj2ayX8z@kom{K&NO5v&mR`<%VP<8#w2>9I3^N)f#s4Hb^=fO>;!Rh z0-1$^8lchz!1}ZnJ5;H9?g)oo<2$+`Uv_UMmvFS`BxDu2*B!sML9}LQekaF92G<{{ z8MUCLU+6Y-wQ9oN4FZ8z9F7C!(qPqtgoC8uJ9q7Lu*iZ!DKJbRtE(%Fx@)dCIYlmV ztb>7OdS*9=LS(7|=_J1f-o@#FcdU0}phOB0g#iSPO03(FR$DFk8+kIW^%MCr5_WD& zVb=Rf!|-s-=*ayjZ>r!%8FS1P5&~(Y^0a}IL6hyEmKLQ>huNI;S1$yXXfmu~^4w6b zW=SI{9~Uo!$$Asj+x{kc|0cb=pb`WRuYh?=Dlxmn;#m(41tF_z29Evhj$-9BJqckW zo)vq9+ObRa)$yhXJ@s(n_ zsDMX%Q8=W~go>tJO;Xe}t4T^@?&VUU&~njwgl0{q7+@Hw8nmY0Z3`Keq-qtG79sUE za`jnA^CNFyxP~VE5N#Lvh~Xs~UZl1m8Vk#f7aNLKu+eL7(-ndaR^~yJ4m&{_<$9N`u~>uU z-g+H7QBW$U%|;p1WF;JaXj=3(4Pdh#qKTCo(H}C}vBs@n9b-`&OB%GeoUjm9n7yl1 zm^gG4(E{rcWIpm=nU4u1B`O6-!vwvW4`$vb?19S&S7&M{L!(NdZ;50^xZz&NwySW_eJI_)lkj3^o`mx)eE z*bu6@t*K=Svz6Zu#UnSW!wwQ576T!-4HGRcOpw<@oJG0C38^#K=m>%6G>X1avlGVx zAr6wIA;SosXzEak<^^Aru~1!ZVK>l}1_=>6UtwA#OP*6{7_jRKhC*~@af{Fow0m8c zFhqA~T17W4QzzDG#yJEF3G)HSchfyw=TWW@)X~ZGY0%fufQl(m-{NSFmLtLl36 zjzIWJ+`%S>NqJWbqx-S`!d#%`HHo*u8YP7~Np&d~zE|oSYE00^fFk2YK~lQ7rY;bn zfk&;v=!3Mg7)jUJ65byxm-Vqu)%F*WY51)BV@mM0u>h>c*>@mAZC@Z;aYotnbvtpE?K^`sD*if3V=Ha%t z_`r)BuqGsq!_r2kP{d5ZqlP~$68xa6QgfLnpAM6SD|KP2c}d`?4Jnuj&Pr%Xn!nJ% z(;~}EAHXJuG*WoVMmE`qWTpPh{EAI8K#;EcZ0kDoSM(bAVxTB%QyS4DRWZ+z0KLh* zT6F@8c%d3TmNkTeE26Y*vo-4+b;VSgC@Vv^N@c-zc&I*cs>3y5egmLEnV!zr5L$Ie z*O#)`F$3v@lY<97n(wKGwZIZgFF!_?I99n3@#8U@1@SY5#YQYa(Pyk0vv&7V%RL$ z${(RYDeCk>ehh7a92>e|cY#5(NxMM> zuj?Eu&tm&VoAQng1O}ExuKFFL{v{4BuM-Gbd_($^ys3=ph5Y2)hfVwF9w)VG4klc` zhR|#1hM^Aqs5%vRGB46)h6zHnmRhcvL>j;3N$gFkvOH19i$L;|B(6;&KS^R;h75eJ zFU3n6O(Li-xS1O^3eRNpFkwNm7K(Un3&W3m&}Ng(8=}_8m@Hxou(-V%Wl2)Q!lLZ( z$CSK45h1=;SpXO2iQL54tO=K1O&y0YDLfnlLH?jM#AkDjC_aigq$dwSGXh29^)#hePuDLHZsA70U+_W{g+_!~7zHLtZh* z%onttCjfZXme5)d3?`9x5F(tH5LRmihLfwL zKn35`_>5rD2sq3pa7MbCy>4U}D!~@!Zay(p|6)tH~&+1t} zVY)uQgB>jH!opd8oJ!hwK2 zM%sN+b6k}z9me2MsTz?<=Mg}$MkJ@!Rp{Z%o9dy4Q>hD-uxMm6_cd$i1^{+IHrKO6 zCzb4qha4`F8`v?RP=zVqY_W-L-4F@_NH|_u<`jmtA{+F%)i|8^P-JhGMDJ$FMoP7% zP(IqzqWT@uiNlDlWUGS#tpo}wqLBxyyQrmpKtK#3%qq44D z)n&7DrdfI;iH0zuSP-!gS4l4U1JcwUJq7Q>q#^#{#o7OL2@6)$#Y?4p_hat+2A4x0 zTK??E+VhFmZ`&=AtMw`07ctd+UyRp=KawnnVIWNcv zx!#HcwH1g?y_y>(xJ*Cav4Ny>(lVHSUSt>BEBzv?WW~iYzs(D5Sy!>BOu2fl8&@UR zRVJ~LZQZPXjNW2}ifFezDQakBkS7uJDzidSRh-#YY2-7{J}F(qFluuIkyov; z$cqqT_&pYIveEyHLhsg0d(MAJ)1wd~LE->ag2esl7xK}m(;Wz$f(p@E;L!Sx5XzcA zeSkepAEZlxbF?R(b11#1r*U?g3uO8!A(VW2(r0DEkc3U)R)Uh+Tv*an-4J7G8SKM1 zNKFPBwmr#B53~os@-KV7D&^tBEc(RvrMp(Li0{h-HN+d@ecxnF-S?#)C0K-9e$}6~ zmh)860@BuEo4Xc}(TTmd5WA`Kc;NIH0J_Og0DW|bk<&|ufb`Iz0J`Z=0DW{ARo$((3`mn&9x~Nv zsfGy}=vcV;n>BXuJ8d!NuaHyhh{TA=$-Ljc&V_*jjt^>kXm52$0K@4OZT*cC1%Q*( zV4$~a5e7XzSW|jRh7e=^tv{`o(4lCQluRkDQdJglXVKd%+{XiGN% z)a+8=vVuql8<{+gAQPb2WV{)|fN|^rF@ONb31q%lu}Av5iDgpoV(FV4Y9FY3Nh4-jibeX+1?61jf<+Sa7xEXj;VD%?Xihj7iKaRq zD;5W`RyKgu`#B(~ILytwt58#%wZefd^#^Hl)RT_bn)|<2>MO8mJP+hT6sNTKWtD@P z*P;0ABnp>Ni8??*zw>trY(d0<#Jyao-&9tLF60M{Y@@BfD%q3U`lccUeA(1Wzw-(4 zE*6=1W1`wX>_TrR`$-|BonQPUB8wvMfm({pZfPerz(q;PF?Hmk(2M-y*{5Csr$z?1 zhiq$=!B%-a#Db?xs`qe$I^zO75*$!0s1l4gq{zTeK_p`C!qmXv+sLj0F;le#6M>L& zXcyydM$y4Ok*zTj3&=50>jI(5Dht|5KUN%C>BV6J^*mpNFHWhlIff4SP|GjO6zd&J zbdqFabb|b>PZskf&J&E{&oWsczVWL%0px+?g3sL$!B7L29LgaMF~l$ML6Hx+{4Zxj z2?_G7ym5%WlXIkqmI>`Q#W7O&1IDNiIIpOF;e#IR*Bd@bvQ9x@OoacCYAYA7ePf7_ zMey{x`86jiEnhG1BK43sMEBLwNiMx&hY`s z$D+j1kmfvE=>yjL0+IR%f93->F6eT08<+#1u^DKM&>*bK(eFislbwkB77kr~D~Zu2 zbdc=>@q|YMu;ZGdx3AOanoqG&E6A`ADB=W_MKDatN7kws-Fb=_E(kZmxNs({Iv)eI zu|BWupcmFxScT>#RKc#oT6h$aAT;EcPjxhde0ZR`vw0?>K(iFF-DH0|NRe0Nzo(Y6 zzsq6)rThpDV0RQ^%&Q+|au?JO$1hI#rfjo;bH) zxy)_%b|Q`vsqBhh8lSb-L;}%1B*jE5y2fQ|ml^GAW2sY8(C^;5+ zR|gQ{XvUSEHTfV?n^?Z~5AoHo3 z8p%=b!?BQQ64%?#lK7*U5{Gd^&N ziB@Tl7o6n4M`W=vNU->C&XPjV9NS6iRCj$Alq7>e%IKwRYEDF4!yD6uJ=*9h>Hwkw zfdr0u969UIz=?G$qpvdUlpmW15w_6decH8ZplMZ;XFFqfmCNPFFSp>Yg=b!TM_iwi&h?2Mn!dWt`t7%s z|1ONUR$Uh9T8gmU&9`)H<7zVF1n}QPEZ?&f=Nkg!G7fv`?WMq$c*17z*P!&53oI@h zSHLgNNO$lOvUfH7^1O6?8s|a3+pCnP%8C9o@YkSpn`J!X#ut_Si6d@N`V%zmr{U(L z`;^Az;e3ChJazG^aGlbr`XlP?-p1t`mabRRZUAnE;i7Q9KU03XcuBZkhReZCNO!0v zzbUw~bSG(C1J3mv<+o1bT2{MU%?uZYYh$=RxK4)4!1XX(0d9cdYH-60=W)ASV+7URZieAfaPtf|4rk0dk6Q)K$8d{q?F{FCkINO2?i9^@>4F=QZllH};3gO@3pc}X zB{u{|M=f&n8kxsXcLU2P27l$h{+D*gFF3Qe=R^i$i&b=Mhn{>MU zAOJVOa8bBnhD*YYF&n`gKd1MBSt=k*qb^D$f>Tsy;M;3Co;t64_{ zxB-T%!HqJU=e@XpFkBFBp5bC}%{8i@t{*A54u%_t>t(nK+%Ur}!c8)q|9vjkEW>rd z8Gm)2F9|pw!)4*x8Lk8ukxp0VI^2MR6X&}M+^BSC$@ynISG7Ig3GZHm`dIyAY9EC6 zy-0Ohj(@+}$KiXd{w}r8z>iw}ZE9Z<`CI)~wQq?0U$W$Xh1&Z)$lvNW+j~FinSmdb z{zzGW+w(u%q;$G{y9PHco$CG_Ks!@##vJu)PPS{j^sI8+7valR|GL@-w#WVQZxp`^ zXLE5G@2GtYe(diSzsA09BL6ge`DKgWS?vq(b9IZ~Q|;^U#y>6oK(%k#0q2cZEPkEZ zcfohgTl@yKPr~=UYVjx8doi!-@I$YW|F*747%I%`jXBuE=l&xH0LlB$e&f;D#CTJWbdi8S#Q}J&brUxK4&k!L>2m zI9xNsRp4CG>FTlwH}?jOo6h;)kNrbB-S~FFO)}yo;6@qovT#ETSAy$jv|ESkX2kRE zi1P@;h2UBl?Z)An7%mOB&~RQydAM1ItHPC~(~YltCp?caTmWu_(QXuOkl~VWy^MBq za1lnlDY$k4>qG5v#TAA|4y5BYY} zuYTG#ZxKHY-}^TCgH69)?F;Y&u9ucx!$wTMMeXbGBdaa`LA7uBAojy;$#U*`F8jXLfkeD`kT4^ZSQ`sHuN z{a(75#&yAsNY|%v3AkD5bm#Ug+=6tU*TgHqb@-?s+UGyGUWW622>S)Yh2SO`E)F-# zaA`PWcjtcO;o2Ck3fIkW?hoUB#c%<*F@}r6O*32)Zh_%)aK1g9>oNt`$#4z0euit= z1@oEV!f-`~>w}wPxC~s=p48t174u%K_at1ibQ{g}e$JD&`;+kF@a=wzyX;R=`zn0* zUgYmI@u7;Z|}-nQ_-1I#hzQb_74vM#@IX;)$w=aA` z_)n1kkLfq5eGY#9X!5(6{&cmkz|XEH|0LQIm$6ap-MeEy3zMHD?lRV?eGuMvjK%M# z_HpHlHI-$m^+@cqY9{AIsJ?Mv`&8!Y*}^#|qrYQPtzf5Xh@WwrP3f%<8YnQ z9j$Q{xPFFPgd1TvKL({J-BFr$yWnP|)Ab_(H!s}-ns`~bk&QGDj@P)7$WOW+jjO{g zFr0TUj7Jx>tBV(c>t(n&+^BR(O@3*(1%}JRMNXjnbn&WiV+`m1D3bc5bG!gtH^W8Y zCKxUWXPl^Scj^2G*CXBO+fV_Gryo`BPZ{{(Pf^@u|Ag9?;EN|&{9|h0fS*6v;vZ3a z|K8Xyx-I@8wU5F_K4b9@s(lK6FlO=ht9>4R;tY$wSM6)?vpp7nkJ@`%vEQC!@pr3z z7{23Ni@!_l6Y%}#kzYQ)-l_IE_=(S3;@_e675Ldr7Js|iyZ1qVFChQAd4Ic2?St^M zn=SrkwU5J3Tu8oQ#=ptlV~cZD;0G@zk6RyIHs`ZNxFLq~?~C=sa9waC43~f#Ww`H_33`{jeV}TnMf#-KR8lj>EYUG#)yahHGcIJX}A+RpG`M&W%lWmf-?$ z&0loxM-(o?a7nl!hReYf8Ey)0!NGyJ^7vcZCDe~mb3I-Bi1L0?gCD(={EnvItoGjh zv0etq*G<1k?Zfbq%gFCx-rvqs`-JdG@+X*HlYb7r|10FbYUXpTI(`Ly`U>*DH2pPd z@5amHo-4`!$@E*)J_z5mh5SpVze4Tf@Y5;s=b2uUe@4XrD*5I9YVt3^_g_Q)lV*HP z{tfuyA&b}K@5f8LiLa4g?(gO5{zl>ZueHS2S#-w{~JA%e-zx(ff zQ+^&L=1mN72fj^lv8B*unKvSC7;aFy!wh1^J(jrg$AwS84}X{9F8kdc{_i}b7kSj+ zM}H{2{B^tc37$hxf9WpLxFB59k0{>n6;9;YBH~H+bLot=_IV251>bom#eLP>*Vm|h z62AE-93UQ-mVVbAkG7KQQXhUxOOfC*E~Va+-I#$^ve%7_)~J{ zn*F+2J-$)+;@#wbYWC}XdoTJ`f{*0M|5cN3UF7?7XXpJG`p)Yi76voR8tMaIFki zf@@>AI$S%$c@M|@W4I7pC&R_zA`F*?>t?t-To1!l;d&X)jYDWZ!v)|57%mDo$Z$!x zA?d!SxgY1?njWC}r*l(qBfoZb4LD!P*|mhQet+Za!f%7B!`d2)%zb@nc)&}MTX15O-gr~rri?UymUT|tHaGnx2>iwUVO+k z{S5UttBDta>wK2nu^JbL>z8g$ggdhq3xT+vhmp``}xvRPTi8-}#lT7x&{T{J@{ctMU-e{R!kN zov!WyxL)bJ__?LaQYVq8cSXEyR}(XCw9ixcFnq@><+<#)s(k|9_kzV=tM)l~*Nfzz zHJ`t(QTqz~#2onzX8xC|z58gamzT-EYWj=SJ_tYhiY5Om)IJVB_?ji3%hf(3;=e(D zxxbqHOYo5eOFrkR<2T^j-m>I#j@tXzBmVzd@;OuOqwua(b4%aLUY-vps(lK6b~X9s z@!F{Nc@cj*i{GI3HF)3qEcvWgdv6%=x3~DC)IJP9yVl~@seJ;zxTD1%ruI4b(GOVs zA!=WNA8aPST+aj5-hB+>?_!C+pV|lE2Y0pjz12Pr-?O_VpFPz+BjWqXFVFwo)V>7Y zwztKvRr?0~>^_!!-YMAb!(xB)i<{&A%FP{_GNQ8F#CFAN<%6 z2Fs1MfibZE&e984|F2`SzT3?IVzn>8H-C!!ay>VzeO=^p68S%@lJ(rA_ASSuJ||o9xm4}D;K#bj zKWxU=o0sl5bDr&IA6LwK_wjf>J(J=t ze?NN{wGRs4L;m|_Ki8;z9DevL^4FSvXSL73&!0_xIsZM?z9jNFm;5iy{_dpq4fxUX z$iLt8tJU7W5&3`K;$OelcE1$klz|`IL|*)vOG}q!oP;aD4V_Q!1j(%L--WBeO-Q%9 z#(BE%yd~XU8W)5sN_UCI#o(GRp!{?$1=r7T<8Wn$tHAj-JGZ+CH^6Xy6nC28y5L&- zsa;+D5^!zOwJPctz^}*P+NJxAbjGv4w4FD?r{QO;{tGuS}J$3vP_z5^zO^%fiht@+-m3Gh7|cxX8J`-V-r@7%l|YE?rntmpELHbeM9= z`laE97%mSt!DzP%H^YeMjv%FrsV=z1Df0`!wK7~3F2Zn0xB-UC!HqK96kJ(4-FP(M z=B3k(N6V+MeAl z!*D6Mrc0dj8;9#;xC-1L!!5#1GMxV;%om31f@{6hxgQC*9)`=pjWApZZkplhaK?ah ze%>hFM=@LoZh+z9a1#udhMQx!Je==K&i$yubu*m%WZd5vE&x|%xG0?KGUxn~aP172 zgX?FwDY!9)YrxGiT+1og|1Nj#M;I=`aD8w?43~i`GF$;}f#GU!ZAs^TcuvLs$8bTo zQHG1b%`jXFuIbCp`HjPMGF%02kl_~LCK=A(js1_|y5L&B;@pn}To1!#;YJv)1UJoa zbvWY+=ls0*u&IOLLU02N7l)f*xHQ}x!{y<8gUq_VR zl5p(|mxJqPxGA_XhHJphGF;2)*#E9_?nf9d!f<_XLkyRJD>7UGZh_%yaBW+h`{9XU z|6{ly+$h7v;AR*u1=o~v&Tky9li@0GgABI_H_34R&tm^$xGuQXtDXCifa_tnEZhjg zmEfist`28>)j2=!8QA|AE(ABgaB;W^hD*cEFP~Q(+5BHb&7kx`5yEnwa<$9X^THo?Wf?&->~>iYQG5I zz189`Rr^2>#&_7_x2Sy#e&7a+->UX$cq3!+H{1KA_h0ZM-z1-N$;FT=Ud!TEyW0&oM;?W(CubVa-!H7*Hf=C@Yka&Tt-bn&L(I%R%3 z*MRGgPS;DzgpQHA9_{g`(-)MSG z{x$fb^jXtu^7noY`^R@EJ}3V${JiuJoAEXIC*X%~ruciTq8~8MRQES0e3raZ{(gk> zo`=8PLLMKH&}Fkfhu~VJyG`TbaHEWPX}GdH_vbd zxH*Qa!Ob$9=K`#Mh6}=(>r>Z{7+jeVF9la*xN*2ihO59$Fx(>C7{mEDN|u zxORq%!nH}aujYP~ToG?Cjmxdb@823X1!wMmmug%C&b+^UP~%!IdiVaezs7~(%yU6+k)N)9wH5Jp)Wq{#{BD2u(YPR-*2&eN;U=ZSt82P!o=Yll^V037af@&Z(&^@bKk;rnT|c_uj8Qs=>H3j?^GSEDCciA) z5W|&3evEeOBERoC=jZ(*@{{feO@1M`cIohsblHqY9RG1xI=nhix-{GjBVHb^?R(S@ zUAtBM$06w^HSM}DL4R+jc<)!Z0DirWc+JwCB%N{BHR++<;<+vhKP0_w9hTrm8LkdDDczxpdA{`i z=Sw)>N_V)%h2Z8GE)M7V0o6&@k2G8}!{y=H7_JJ}DV?q#?#pmKW4Hj^0K-M$h8g)K z;l>y)2UldYI|Vl*-BFrxYrxGjT+8KH|98;1>Eeaqd<@qI*UoSmxCp});CdOZ1~jTVS{fT+^8IxGlo9GMpbTxjLlNt=}%VZs`u!%)bO& zzjV5NC<`~lh*yFeW#m_fn`AieSFryv@(aPuG2+GHTtB4o(A6ak*UWHvxHg8X!gVs7 z`wHxT3>SbKknRZ0_(tJ|8S#>EV+@yrD>B-hf}3H)YrxGjT+1N#zaKf@x598fhUt(na+#thwuEhSwa6z~UhKs?K87>7k%W&gx3k+9*Yr2!M z|G~8~oc}89e+<_J*UfMVxPFGq!VNK832v0(>Tr_`=iP$+kKsaaa|{=UbB#NXM;fkK zIuCwi>9U#k`4#bWuDT*WopYz&?XS)S;LQ5zTole651mWGnd7T-IXH8^=-d>XIsbI7 z0cXx0BS2xqfvn1843BI#+-*_ZOY3!I}G&&UwE2?*6B9K{#_i z)wvj)xj*Y%3eMc`Gn#on4rlfkuX5Kqn8|FUrH(&1LObQ!m$lBUK9d&i7!p$(88!w@Y3>SbKW4I{XFvBI`1{f{}*TZmAaOQl})ujQ~CgYt!)6n++ z-_?@Fe35QYp8e#=^m8M*z-!| zK2U*gyPMYCou)rf?cLwN{Z;zq`0La@2p`E){O_CbH`sfz59HvdrJt1E?)xoMaJ@gL zxVq2b8gTs#*K!@^C&Pu|1{tmoZiwMBaKj8&fE!`B8r&$udA4GGGh7gEg5hFtlMI)F zD>B?TT$$l2aMRN5s+li~E8^*#fB4<}PE*7a>&yeUAlv-=y}7BL90V`Jb-#f$NdK)o)b$82qHwuT%Rp{DRf* zr}hQ-_DM_ryQqC#2&Wk``~7zQ$4qcd76QnXSf2KQKbBI@oI2BhVy(2Y!*F%D5r*@QV83Fx z5S;O#bAEBS4u(s^4KQ3DZi3;eaB~dj{x;S}$+;f^xNe4v!VNQA60Xc}IXKsEob#K4 zYiGCyTtCCLdGaPZkFNda0?9Qy#@D=hp8^Q{6cW83>Sy%knUJbf75Wi43~!+X1FTcB*VFHMg64H z^)~>wAYG@XA5l2pBUC?~OTu+BTn?^ZI`hwow>uY4!F5kjyE@l^>-fF1Yq<^M`>3-E z!!-{d~|C7}IWv0JP?Zfc> zPm$kZ`mJi8fS-Pv{08JFF5?P&FP7i}Xim@{Pmw zNcRbiOT!IGr>kcku3x$}ns`;XZs|_cICl=uH_{!iaH5`d#2b@txt`aYZ(C1MpBUnn zrB5p23YUWG|0B&iRr|t?!;MI%t9J!%LON9)MSlt_;w|^*={{wD0(e$>Sl5V;EOV#a!*Ef!qJtB8hTzQp zEcgGOO}6zHz7M`^?bjXlUgTSapObz^{QJ^nv+tGQX035ItK$amK%{4AzN+SzXg>yL zwy&BmqF-US|2JN1)$Qlu&G9;NsZ;oyrC&?^hcjfJs&ysWFTu4*w?W1;?)rjq92)Qu ztG`X{{bN|)R=-v4qwvF4e}%ml{Vc&xNU!RraCNvD=~VlNs7D2E-rD|d>h{Aw#QE=8 znhyub`dPYrBoG<>hsKVVd!Hs= z0IpLy-Mosfh^N{QL_d;n12Uefj-ozsxKZietmz&yzG(H&*?WK&qE}MDTfNNvKYq<;KEuF3( zVK_71iHdf`xCG$3WxVBaJnvlFaTN8*!VfU|Q-T|oE~aQ-#H+)VrPIx~3f#Ok&*$RG zJi`;X|NWW9M>T(z`VY5YjeCbWZWg|o5jO?rl5V-Ko7Hg}@IL7~H1%rvDfSQPRObaz z*Dzc}I#peT>w_DRPFJTWT#t0i{ki`f+xm;Trr-yy{w{kj@@T-1O1~EWzI0jEiHKW; z8?wf|R2{eLXSknBud7!At|;AUvU%%yN#vP@TVS{nT-$Rr|5Wirf5zc@r8`5mYdmze zvaVJ55$V5Z`g`oX$hYfmocE>QReHOACE#YIJ6hwiaKt;A_9`lLe zLU3aY7l)f>xHQ}X!{y<8e{rr$6|R%v+&{7UXZjRw{a81uU_jd}e zo#7gAJq*`^m&!v77lxZ)xIVZUhRcYT*)KTvw*c45a5cCH!+Gw({>*SexKW0S!Ic>< z1vk%d<8aM2s>`vOeWL=`A)RjDScL0kIR7Nh7Yx@0H_31bxLJnF!Wn;co(CnkHioOi zbu*mzm)QRpE(ABmaB;Y4hD*aOFkBwa_o8!Ms&JhQ=l&n;e+(CZ8)3L8T#@0DaB~co zgKK)pdEQUKwM(a)_YJrnhHJSO`ya!F;U*Za4{nCxGH|Xr=kYDTwK7}{F2Zo00`@uTnw(va4EQXh8u@#{+n}MDsUYPw+PqEaQt;Cbec1mPE(ABmaB;Y4hD*aOFkBwa_p)U z^E`n4kKuxFqYM{=D>GaQZl2-B;hO*HJP#^x9SpY!*UNDJUt|AcxGuO!hD*TBGF%qU zc*VKDCAc<*tHX6OocBTOe+(Cb8)LXQ+%&_b;T9M!59gbAu1giJli}PY?0*awfE!`B zC|r@@l5lejmxF71)w#b@aP17&fa_tnmfv9ilTNn}h2bU`@%rFq7%l_ndd<1N1-Mp* ztHDJW&huOBe+(Cd8)di{T$$lgaPtf|4%hrT)#X@}Xz8-NxAfx|D{vjs9W9yNd$>io zUWW5Og#C}w}wNxD1@@P3Qg=;94231{YyC&m-9X7%m7m%5X8bGQ*|d<{54ru6e<^E)}>A zhFgT|WjOy7_CJQ}f}3Qx1l%mcW#No}JNLH)*T!&lxNe5?{vP`u!-e3+q}yF{pNYdw zNVl)XrQs$SE)Q2^xGG$k;oOg6e`dG<+zi7-;bs{w2{*@ZIkYQg9>x1EfaKj82gDW#! z3eL6YoZmQHyL3-#`dfjUm+tXxB(ymvF2aqyw;@`{p}rb>FS(-i%55e zrXN|j8R-t!xDwnv!`0!8Rg|ADp7#%!FANugYiGDPT!i7$aJ>wdhZ|(LD%=Rexu3xP z%y0p?GQ&mTW*IICx4>{YxTbBK$88F(mEjt29Sqk}!G6GSVYq&V>w_C&xD4E=bSJJ- zuipY(c{SBV=W1~CZfEDgrqaBvvkSs?Z0GD^aJ`1JOTi7l*V&E3O}@|BRp4gVIJ-qS zV|!=ke+uWT9h_YkT=!aMmw+2=a&}p`vG+T>65RBT&aMu(u#>a%K8^kF1I{i4*ZDzb z7l-TLncRqGA5X*0FaR6}ZTUXnjm*+#=kJbn7(E|19#8E~If?aK?wJU7bt7`J_8d6E6$bE!|NX zSAy%84j*yQWpm!G!!_?h`S~=?Tg7=py5lr11lKN|t{-u@euhiKjWJvvZdSUGCci4& z0wbRLPq@FeIFDNZu2VW)f1_}N43~tPWVjsMymY$pn1XBm2=zmkUjr_}a4mnv{>*S; zxT18r@#uq_VYm$3Ji`^>j9s0_tp?|lPFEMtb69T-7la#TxENfS;Zks}-JJU|4%aN5 zZXQ(NIvDX5;kp^lKa2G%ovwaeaAS;k3AiG|W#ML|^SR_@v#*ulrWyIw;S8U1oxOj- ze34GKpN8Pd((SLwFAg`&aA~+1>2#lytKo*bnxgct6#|3&71u zcZ82G`W;9M4n3elOjqrrjW%Yai$S#^44RE(JHwaN}^j`%=5Q{3>v> z(&_552-m%zb3gol#rkEqF1XHsbG!sxk>Rp%?fW~&E5S`jca>(`>Tt~mP`oc_ocBen zUxo|8jWJvtZeF^Srrk7L_kon3&gJ2X(w(J=SA{d$D4x!_U&8*vZ~?dx={}=rHwrh+ za7j4hAj(e{F9+8l-6fiKr$l}X*MOU6xRyE0=YuJ~^_q6WaQ)JKTjToR<{0rZa7~9$ zyBBKW72tZM(~WNpZbG^qO+3%vkRQVZ;e0{LPZuu+*UfM#xM7AHhbv3>QB6N8aMRLt zYuqB7@iFJ|_5U6BH|ciQ#Os1u{g&SnJB-{wY<=`e5ZVIlu5Z5`@teycZi&BKk(y2 zc;6A^ey7Q=3$84kZr@12^&Cm@bo)jY&NzzPk(zc(B0uSL`$iqEo#DLm*bf*k1lP-O zakxQ-OT&#w7uNJQ4_9QwtHRAN;<;Z%V;xjyUHt-Z%?uZXYh$=1Tqncj;CiIf)ny89 zh!L*=H_C7=uVMX4r~C6X!*HXYp!)q=Gj4rw14om4OXD(dk@e2;3UIAqa=QF#a81WJ zJJ0Lb?-?!#H+!seyck@01GxoFe^YRyoz89?Zs0g_x;j_jBF8(sMYz_D?7f5gSz4Q!R+sPDn4~;9pwM(Zvm)GDrq|0dH zdH#jTn|r=M|gP>CW*&aJ>u{hntozs;OTZ&WKUFI+us* zV7MyW0K>VlnNBcV0B(-qqHw;?I`<<9*UfM_xM7Byf-5sz1I~4ZbAB!V!T!Qw_C(xD4DZ!xi9~&vfob4KBiP9vn)B7%m7`WVjgIf^@p~XDK*e5A|a|&H5OJYnAQ? z8drgvkZy0y{csU(@+@k%UE}(}F2ADoePj+cS!V7LO@ z0K?VbCK%4MdX;OA;ev3!Z#efO2G`ASDY#*V8;2`1Tm{Z`opXMRaP17|cdv5wGh7$k z7{evtW~DR#KiGE9!CAP4t<;YW&H1DR*K|EOUA#J6>kZD%`yR}{8_9(=?S|lbGtT+N z;Re6y94`$w_ATf9@^HnQoLv=eX2dx^_qMBC^WS!M0XXAka;mzBebNipmL<2dte>&| z{mOkc4DY*z{9jCeh`ks4QXYQxR`Qo>?kiQe_V1A^;pdhvoBN)7JH(UjIE@Ry8MjkB zor}V?NvC?xB1ee?jpM)x?X!wMq9ejZ4CHO4p`wIk<>)`)k}3T(@+( z&(RuiJ<{!{iP!SJcjKv?7(X|hncrP9o^k40pI?%@Mspts!%h5;v+ILvE|9xT@fShMknR}G{D{Mi{+jx$b7{EpgXE6X#LL6YmYm~N z;f&upJNFJ4kB6LH0Is|2?4odkzjJm;xUq+wT@G&g5ob3Aw=m`G8gRZxon6aX%)iH+ zT^O!^+S&ENjXdt`GH}H|IJ*Me+!M~O2G{hYv-33Jy!4c_3&Qn0?d)Q3Lo?1U1vl|W zXEzQv^Nh2rz`3f zxqn3AMj0*%Hz%FyJSpz4aX8mMsoneJeZjbWTig4wxW885hhHK8M@^oKaGmqyZqPWt z7yA*zb-}g2O7RYtdD=a1B;X>ik-J6HZWgZ0a3#2o*PY|l;U=YvY1;LE0P9IQQO~8z z=D8sRx4>|5I9G%6Q^gbG+y!SyccD4XZ@O*AS@oLwJWvvj(1Y6i~7a0R$lhO5D~Nrx#wm(6&3@Uo)y-_#FP zexmLT5l^}!&G*K8+mDl|V;pfi7AdZ7-lgFNr0bw8ZU4VkdAMQeRQrZ#w+c5Vo$7s! zaPDTDuimEoRQrJFcOCIM|CiixX1}+r{_pvA{1*3h_;13uAa3ufzc2lHIMw#fX9wf6lHz69U459PV+*Qk91e%R`_Q+xlem_Js(TJ59ok$o-syz{@d^G)QR zf}gPZx70ol-?X14{+nuFgYUQc*VW#;8|rWMuc&<(zAa$M=Vi4|zz4PgXTn28I;R)sRVN%}(+c?%bS8p0k<>2NIb9Pg3#>bsq1I~B2vup9={I|~8h2bJ0XV(YUdxW#gzzrVh>)G1+(d`7^X!HF=@ZT_2seAQvx~tktao-PxTa&A-8fw9vCggn*RjFbEy8tolDkha z@5Oqp!Octern&xZ`?qqvdOwQ!-$ikk{bOn$h98bt{5xu&fbWS~{DJ>b=97bO>$dna z)xHAnidpn-zE6zBzfKbU5A@vIPX3f|1Ueo z3&AxrTpX^A;nHxO43~%NVYn*X0K>WW#rk8o0NfbEMd6AJmxP;PxE$O(!%e{%UvVC{ z2Aq%KTK2>JkKw{_5r*r7>t(nM+#tgh;6@m(1~So(mF^V9eM6j=VsOn@ z(0Gh$TncVjx^HXTIGiy^@pP^NHz=L%^MysY1%~tQkNx*bYImI`zb?2T>2%*~NWhIU z;$`6`rMptoZVAqJ73HUTk0ZvT4%f+W-UHBIh6}-sNVh?gUmUI|-9O}hzWiL!xn=1@ zA@WPZ4P8zB_>I-c_`>Dk+J?w=zDNECoA*#vxbAN_JNJQ@FE@}oONLv{Z$-NSxUzJ| z;1;oT*|Zylo4=9b-LG*;xS?;#cDJ>}>)P_3>)cyci~i=|rf()!(8Qa9>%GO%rQ3%_9XJE{E?{Ls^u_^a)`n3ti0vA@rd-&?a@<8Z!boLw5OwMs6oiI<1#|FhnS zdS&4zXUTQTeZn~Dwg0Y{oKM0}!O#8G;y0=NB7F197QaR91BZyu+sTib`P{7bG5Ea+N3)qWh_*vFF3y=q^DFCS!y|FYV9K8Ex0;TFG3Lpk3< z@I&h@exus=!S{Aq^1oc|v+%7aTjJko@5OrdVDmA~AioCcVW1QblaQn7tXQdDSux01bo|Bl;<<@*Nq$Pz35jBepdSTYW88zVHoeTDejz% zYu(pH`whe!K8M_uGM;gwefz@u5qCaLJ|*KC=c#=ZK5{PkyX3jRP}D>8uLM86iTnvN zzBO;*>TnAT=WWM+-AD1>*3>Hmw{SkWi#6@W;UX81(|wOO4c9H*k(zjUxPIv#)wn8L z^JZ#S_xm2*AIJDe_jOIY090h_^ zVjS}#zVxc^YY10`o00CbnsIa=j`e&owf{TK`7Ho9c?r3<*2qNd&cAT8SCRXJWOnm0 z2{*EZobG&-gDa-U;Sq)|n|7z*%2$&!-}~FO+ko?ZmE6;s{94wbe$w5jX*UcP8KQXq z(YQXi)~}O0SkrC>u0y)-YvL8)=B}l9Q<`zB!A)!>x4Xu9Lg?=e=9`OmDSLK{cDs`NGZ`KvAdMEm_&jF@VN6_%ki%#ueimhL7aPH_Uo& zR-X?NBK{rZ-!Ip*ahuxb;G2GA@sFu}1>W^zi+@M$-5sdUUF7dE^EvPhWj%xNgLjkv zf%*IWHmiLc-gO`O>&@q;cho)u-~XV+Z~T`spA!6Rnf$3{{F~Ii0pIi}`Q`C?LGAsY zK>TU)%j31vo63Bm@Y7FN{Cc%d!55#h_>0v(4?poo^1m_9m$$2Z4L&kUzHIt63(EYx zM`J#`Xo-KZ+K1tX|3N-)*8f?xPr&+_cS^Evl=#NU^E$SwDiYt^4`1>qwHkWZNYUbTKaBBOPyQ6sYrgLhg^z3?|4Gw7tB#+7 z@83v1WBN63E9Yk(e)dH2XPbVV+Sf$>QH$TC_Fmkid%7+DX0;E)kH#$dKce;t_@=YS zFOTnAYM&GST#Mh&{`*{FK2(JNyd|I0)!uz9_Ve?}FW2*0wGYA~3_t&4@_#nh&KL1xC&_Fn8$Q}EpZ@>j}vZu|ZVt|;9n zHLe9OC*~OO!f=uOsa;*XKDYttbopiAW~I}8pP>L}9N^sF8eG40x_F)wu^%wv1>sx= zQoFi%F}POgbp1`i4NIr1-#FYPBVGm0*XBGPi*TLN>GJbOaNc3W>w=qM#7n?+9ORr| z7Oq!1U4A9FX-2#{+yWz>_ft5(9PFH52yRF^U4C)6dFgcXB@Nen2(_z=mxmjaP8Y8V zH^zwP{xr_}LFfJk;M%3rP9o7H&?uA8F#1 z;Q9}zc6F`}w;)}ICZ6{c)NdWd)432_t8@uXyg1w-!=>S77%mUj7NY!g?N;H2r2D)k zKliD)e=uACuK5URSNG?zMd5r5mxODT?g&kOIk*mnn}X|>?wIXmK%39y8X~`q)Q=nF zxz+BR*wT&pcOp6ODs{Xt+-$_z^})@2irfL3co`Az)6T9S@;k}d)!^o%p2bU z<78(Sgmazh>|$_6w{w0exTe#b-8fwHXPjLH&UZRFk9^Ls8{b8d-)Ehj|1+rH8P2W? zZl;G^lcwDST=8sYmxUXPJG&Cx@aLRe9d4kPT$^Sdcu&WE^?7F(f*bk*xmz{sHxB38 z0Y=F>0Xs} zHl9#_z7&RU`Xa?$_K(?n@tm87pS^_qgeKoAT<@jiL|#jm&3@?qEbgDuWi&1THz(Z> zHTz~%#7k1UI+ui77$o;&`5xA8o#x;MuOau8#!bPsUq^06*4-{%18#mRxr)ZMoPqUs zJ-IoJ3&Zu_NbX8a{rcc$q)`g z_n<%0bts%@zm9k#(!C?=Xk4LgKZdxyqtw3Y94_L9Mf=hf&CesO-)oh5iG1_$O?OaS zyn>_4W`0!R+Q-Nps&JzH5?thmY_;V0yl-_NhH&r|p;eDBZ5f5H47%{q%;@%t$}XJftJOa8Z-eg)wo1#-IQ zjTqdFbUm7QDY&LzQM}zXZXB*zx;-?m0_T%XSNBCY<9=%QHTk^*>o|%!*5GDKs3YFYU&uXVeosU5z2_Fh9e9cSug!klqwZH4zWMLub^Xf2 zP5gtLu3uHSnSYWyNU^S#_P=v+f0!qCi>6KixZc;CT@-Hib#i~!#7n}Byg}}DjmyFH zze!HF-%W{l3*;Wq#B0EH{D<5ZJ?eSU@;SuY_LZg2PXcoPu+B5FKHPAlYsuxz-xqR} zdcFtYea+-AHTU}~)IJX1zbpA4nEoBL&%n?7$&Z>|bDwgbhxfhvk-t)w!`e?VPXcgd z=~VY~Q4cTNymSYe^ZqXNyiW+kQed@o>W{eK?&>BaPO9Wo7T{H5`SYn4t{*C<>>I^DWXig;h7c7LYX z$8&JS0dj9^@|%Ji`4YMNHSrp7ZC@sLyCz=C7jPby?pt!)?dlSSYrBf#?WT#>2iMAQ z891MGqV7wV&AccuO8zaU(bbgFZ$$U6pS_E&dKOu==?cse%@*Dc*@ z&3esF z2OlT@rg?ulUHyIbn#iX@{!}ymscP@N0P&w8-*5Vt)IJPfoF)Gy(_gB-zbD|EULrqd zokN=8hGPw%lFe){65zp zeB0aP-!kLBVDS`Y>3d>vc$a&A=^WN!K7U-H{=RtzzG)5llTE*i{rj|~`48W*Bl+je zd>&N$hREkbkZwXeVr9zi~5=KrSJ zyD!3g=pcWK>DQ{iFCT>OKbrhQrr%rb!ODzT*V)4Kx0oYVW@o`G1Q1&rScV+DG9>qn3QusK4)@67f$ZKWyf|r`qS?`%fc( zgX!0)eGPv0bn*q$pQQHQ1md4T{#nyss`g=c-&vM?ZdUsQ{O~#Cm&faFwa>x3K1cp0 zGoMG)z9RDPCI3Cszo7Q+FXH*_3*=Xu5tFfSS2kIVCbwZGy%8i1R; zp4{*UDL><8^*$JbZyqCmkLll3`!xL6edO;n&n1d|O3b50_}1T&KSnb@{9nTP_aSok zYg`xHT$$XjTsPel6$YF zAKuHb9$q5%Dh-qEp92ztGv>&RYvRS>hW|$H+$I^&W*w&CX8uiX-NP2Q;&X~T-26kY zE}e&O*39!NT<;XQs}yw+^U{4e*2AOZK4LzAoB3Iudxa0e7me4J+JC~@zU90r#xV}x zzdiZ?_kEic&#P&;x%ZQMpS+LQjdxz;>m_I2KdesVS%qugSvu=Hv*anfCyD)NSMpEF z=YZw7Hg#A;{La0|Z(Uw*_UC#LKY;jS0rDR+{atDwgP%Nv{KK-&hT^$F#81O_x0Byu zdd<1MAmWF}FaLgy)79tPI{e&G^146oAqD4SxN*34=|q2* zE_0sBe--ns0@oqksQmTiaa`dR;U=YfOtXIdU%~s|W2hhI^01571venw`Q{heEcuDN zgCgD*ig)Xdl#p?){X7u;$-z&5ll)puo>Op>-y-*CjcdU5kC4;t=Pg%Y{Jul(d|5ZU zeuv>A(y9KxxS}6@a6{6CHSscV6Vi2STmf!Qy7LrH)H?^)b~E+kujc2#ci2DQi#)x9 zc%Jw^d2FF{S;kSg5M1vM$UP&OwLYSKKb-Fla%B%e;~h);E7na0zIZqJjCJ3Ze_o%6 zTYzi-1-W~C6nD9OoBgT|Uw(@GjP=j2v3SwHmMbxzpC|ve_4BV4e{NtHE>b5q>bB&u zBHsX9d7fNBI^!No9psOT{$=62-yr{WM!r+G^H}jdrU5tj9&-0tKTol# zga0bbhaJdolKCyyc}2cmaNW{7F+daMOEJySnF%EZjgVIovxjChN1Wkx*zHMoDqo%8F0>y}QJUjnYkh?j+%W4IDr)47!2204B0)?pp4 zUAiAz_fPqAVjg&hkl%R}&t<*utZ*T?k&DQkq=^@Yn@*6s(=7wotgkd&_m{}2){V$7 z57(0&cMQ^}};5_Sc)7;|1X+M#$-&Gh%SP-y!!Q&Adv%^<>F;tn=!)cmEvl zagpCGeUe@gA_A7v;~1b=d{C@O{cJs9EO;xTYVG z`+tAF_lkH~xbB~jJ6YcE?D|`RbKOtw0_!<)MZ7v(`B8GZbC>rUSig^v)4lf$!7WI) z-A80YHv3QODBdR&=L<0&Rk+^Q$jw;im5eW(`#P+* z2D#s9+6};!rR&y=Zxqh;Z;CgoX*UTs`5$sW)3_X5WRcuYHEs%S?rn1a+Cg^2W;_~j z9osjSo|8s3{cYKb{G`+Ue`UgO)6%_ReeRL*MVS&I zB0h4eKVL|=akw(WRp2^yr+C+C>a++qCS6hE{5K%p9u!ZvZ+5}WNvHe$a0$4cJt>~< z|KpN{o00DOn*2&4o}c19pm1Wm$Ki~<$X#TP_og}J__f@K`_D(o>z*IOaIU?{J+5ef zY5l`ZGF%3(zm?+Y{(RyBTeoy(iwKY!>Hv!ji2)S?UL(@?x+mQd+T7== za04-l_p-*hzlD09MGmE+%O+j`Zt`q$@71^{+~_&vbo+S{ZXixh_ntWiH+?R-g7yBo z;`6g9xZXZ;&5HRW#NSrQqhJlV97no%iE# z3(~FEv|EAmT}=ImYTP1RH^ceAjq#A~Sk1ief}2cGyJu@$0?u^_xsx<53)g-rIo)$Z zN#rLTUTM)~vwrJv3j-8S_kC^ecd&lHL{8V=5M22(a=LcoaBWHFex%`sq|?$c~ z=XR@b?N^XXXy$?YX4Fr*Q#ADpz?BD`+l|7tUFGbOa3fpDg*ENw;EE}7pVqi3xP`09 z>FU>j>$t`_UJGtk14HC=^B@d2`ZaR8dC&)ETuUyh=|={xBTY`%j{@A#H=Nt8!OdSs zPPdLcx8VKCR%aK486J;l?FZwjvM26DZcx{SjO+(=H>ZUt^OL#|ab?-$|b z7|wqy>L;Dk8BgaDaAtlwmxY^Qv|ECkX1F?Bnc=**VZAY22yT+$;&2lTmxdc- zxIEk_!&Tu%7|uP4{g2@Sa6{7VrlbM-6?OHX^w5rLoogrSIh;yy@*>3rBy5Hv(hU;gzKDZ(2HsYU`E}L^o2LEA# z5w8GOX1E&MtaQgNHHyF4wCh2?jqg&ORQ(k-5!b5AB3(-r+wSHc<1_Lmvu@&fQ`F8S zNY{m*5tp&EeO%$I z_-pce)c<9_r@a?>gs>K8ZYM7vt8MEl{$GwohD*bZNoRiSXg5xIxM4=TD%=3Wx$zFY zhtX~Tu9Fcj3fJb~M7_nI$}!{+$B^0acEB;hA>RPSR9Wid;oNSl59w6%Ubq0tZez8_G#I+uj&WVjq$Kf_JIjWApTt|;B|erTzS z7@s=oSC-EF{-5#YQ?~UMz6EiOJE-4xn&b1j+IPVZjamFVYA@<#{D}Ml+7p+tMm=A~ z;b){@&VOgMufjLqN%1$D`R}Cm9=u~|AGi3`Y9E5{{xSJ;%=nu6^udq(g#2!%e?eWJ zEd0=2jCJZfD~LP#bBenMaoao>;l>!wkI#H3r2CX2-qQR>ygBJ! zH0$}&leX(c)H8{=WA{*=x_w9dnIc8$_Eoem;!VMor8`yOM4lz&>6)Z=U&Wp(F5`0h zJcVz-PyY}3txFpI8nqYoYP*;GPffqY-iy3b_-<)Y`oBtVci$U_8!AxT&uUyn#ACQc zxCQB+*TnPV=3e|2wQKGpcKLO|8TXOX-ER_bO$?WXYnJYNaz5F$TY?+CpW1yy|3Vjf?!GJ4NFvBEN?z zzmIC%B3v`W`Pbq+%Wz$AtqhldYm-j?_Lu)Rzr?)hLw;twtn7zz>a^`TTDt!s?ubKN z;Tmwm(&_fW7JTPwh~dI;gVKFiQ|~^w0fx)K^)p-nu9x9za6Js?c|Z0y>Gs$3BM4{4 zJ3!-NaAtlV(zw)$cs`9AUy+|~U#hH#rKZ2P3-B07X;LQGhP~#FS z;$5q8S-7$s51lK)wLL=XUH3k(4%aE&7d7pA@tuNEh6}+hFkBoiGDZ2JNm9_ zp3XH^SQ;_sB>nTBtF+TxE=`vQFL40-eSIT;7q`=#-3#{T<9^1AoTqF%GoZIJyl z-YnbZE#f8+*Yym=JwbZAx@F;7r8`mKMBOrIKO)_7`!}iEZy;{J^msI&%Vs}m!A-kc zIvkm7-O~I=ybf#ohpF37AZ`z%{VZIkbhvfdwlC^gf-~D!Wcp8&J*sIcjKwn zxtKozI5R(;i^6rte(36$ToF&_ax3!FxhXhvKB$}+m(q&yTOOB}O19%7d;{KP9ml8D z-j6(dR{xONudJ7OE->z~_hKCKAHn`5w_DS zPWQQ32CgXG1OM09`M_0HReKy~u6is?N>oa67{wuVna%b4kjik zIoOcH$DsxdUsUwMCo1VRsVFH)DKSy6O3EZDCeNWaXrw$v2OE5%@)Gm?)?I5KHj8ui zJ^S;yGaUG?-&%X^efHkx?{(-6fU9|t^Wx$r!DalFxvx6((sMB1Mc@j-#f58j=v9LA zCfKiwYX(;*+{F&P9&r5;xKVIve`CMrJM`wk6-D53S0X;)zTwa-12+_bs|T0!636G7 zH#@N*w!nww^*b%6&aK}69kq@pS0#^>MPPiQoy+&|Z|HJio$ia1i>lN+^ z2RBUdy~27YJGfbjF9Mg1&d?HpD*+dez}1557tWQJc5qd%rq0U%xH;imd6}g6rc&o6 z9VgZLg)4F7uMpgXaNpibz<~L;5?tx)?ALXkp&494xK@YX9&mZPS?>o9ZWLUbaODne z9^6m_E|+$)Y4+>d-^%buF9KH&E+Ych2`(!FHwZ3AI9GkA!Bs@yGLJ(3ggeENzt!OS zg>#LQ)!@?J;QYDPO|9Uvgj?zG+XpU3xU~*$99*ezCpx$oE?!kd@S6{=Mz}9J^vc1- zBXEu2#)Mnt(CY#>BixCO{cRYWw}G#d3N3n3HFJsI*qCc?xi0=bm^?$6}Z1_##{3Q6q zd(4}4lJ}!|Q{!a?_ zPkbf*=@P{_MZW@IMIW zCrSS!%zwy?mp>2ZGZA-N2J;W^E&14~c$ig+`4`215Y7*RpE#QN4Mu-W@cM`HISW4R(~92~%+r2Uj)Ul&0_OLT{OkRQ zxJGbZ1g;C*%(1NJ8kdGGz0DzhsXl|aSWqQ?zi#^9is1TWijIChhIDY6H?8A%j_Y{Q z@#}f!_I+FT`JVsHBUIlqe7Xynzh3r3pFd@LvL%SC$0y@N=J3iQE~kGaiLPD~0$Q`L0;QdV2^)@wTXbbjITE zQz4sN#F$acb}uBY@Q;aO{X)FW`1O!*6gw49&xcY!il+~sqElEm5hEJBo(}`^OwY$8 z*0J5CY_<^3sS9}?2j`u}++CWJZOU^DZ5tPE$@fSq|HRiqx3^T{79HuuZa)taH*xLY62gVWK^%RD=#+37*iIJHFJ+V3 zYZBaua8+o#WHH?7OZVeU>WgP_oHq&{6z6JiHNv$zxN2}C!hO%dwSp@;oBf9AQU2+< zmAG&PCjVQ0w|Jct&~J00-*FD><5iK#;@zX{toPAnW3He3H|E#rxTW7vJoMa3&biE8 zvcKqVSGuY{kX;|_DyrD-9uv<|YCj4XkNObT$i-||#qJjNf2$vjgYzz7uF}E9Fo_il zSL5LF!LP(*TH!Y#-18sXS8u9;8XQp6#9| zBPynUtM@FEAF4;M@as+gd0F{U{fF$TVK?**_H+5ZN)glye$KZ z<5d1A|MZ+sQybeIXX>}{p~ZIc9vk9&z_&Uu_{MSM+F5Eq0?`={3t@fbjV`hHBx_&=9Yw3EIo{uTI zi@9#m_3OWW@w$}am_;1@!kc)#L&M{khQ8Ou`b&-gmpc*gKxJGgJP(FUncE1pP?;d4m#nXj&>bjX@ZK|OUa;Obmi-a*$9tH+S?gQ%ECJUm+#Mkt`R{`N zyk6FO!St8!D*v+IP+msCkKN0BLUg@X6|d?^cJ!Q2)qQMtrtz~&*#*pFRF8(=vfZi1 z-+`(fq~27Ydhi{>UupQ|inr?11%6EU&zrd3a>qsW8He4({T!D$4)=~xegg84c>wyy z18nzq)345T+flvfIiL80YHTd^SkE&G%7z2Pj^(6SZ$U z?DB><4^Nu)4KelZKa>N)1k zc_W|4->DFneIVus;mr5S`kX!ffh&GKwO%c_q6l0&xPlZM<(KM_CtQI!UP`fEC%b&; z=lq4^%{TkqMecSXJ9^G1YZu$48#_IosP-YdPT0+iv)%J1&h>78)UKVG7+)vY4zGIR zV)pZZ{zlLFcrP*UuM00weggVK>>$ivVi$HEh5AE2xDMf5<4!raCgEK7Wi^89kHB?- zn~C6e7~Gg}VfCQA&Vow_cbBw>->+5s5ubZ7`u8iG*Q?}s(&q!(p%wZ~yP2;t{p166 zoUHs%e(8CS?l;(uzQQ3{OnY1N=rr^*_Aqyn(SORVZ|aftCfoVr-Y#V);~w$lh^IyP zof3|>Gnl9N%kZ2=(f_jkWsdlJpxg8oa~sTjdAHIH=%+o<@0evf*Ss9%X*vj?lb$PKA#aVZ`YFk9NXO~JA$`G#jpAW<+F?Y?ek9ZI=uhA>W8THkE}mw z+@N|C?a%x>>0A*{pEn8cGn$3*?GtR*sM{f+-^D(TMN&tz^4P5hE4v17#QBFCA?0=h*HqX5Q83V*>0- zU^iFHcGc^}-)6Or3$Sa1T|o=meL)5u??82be}G*d>`FV>?unTATmFZ|=XC zPPQw>SU|;FpV}@Lizn}1w!6ff7yFTV{wBa*CG3jtXS?^<9oj+J1@x;*JXhiMvt8JD zO#OqN^O+I-+i^@!#oMLy1LB&6UBzRpe?HEUCGA$I=duFq@)1|l4z{})xlGz^blaJA zl68IUrn3XCug$&0x(U;sYt?g-Do->n(7EZRSK02GWHhM%Q_4=pEllyrqOVud_)9VN zi%wX+i7UY;F5Fi9oGfOZvDS69_`{rVOWs>*9sguwj46JNr4O zce9;&uZwqbYCF1Sy5SABqia@Fyp3)d8h;T5XM;;m8o z*6~3$>fH7w>lT~(>*sV;{i)r{!FIgGcK-F4+k*X2{PT!qZkG9#9M-}(ZC&@wr5N63 zF2})@firq8t{&V>1izh@-a749`Y+XE5L}MLSED(r|4oCd5$+-fmx(?Q7p^>nqy9Ds zZcMl%p-sixsGg5i?L_g`!*1kV&gWp#3jdgXW1Ua$1lKUf+{CdWFN3 z#Kq)aHgoa+0J!cH9E}T;;Nrp+==N0gCO#8uocKSv-sbv~-%bI13HaU*n2(}Q@2Bfc zdpEgmU8dtv{fpx10^hb5^KYi$X}_$*++DSt`G4SA4;62B>h(-J?DEpt?w4l%U!$JC zRr5c^N7wn{2eRF*G7k9pQar^+@6pRRn0f!X{Y}b`$^*4GUFR!0gza4CPw0JqIiFze zb=W0KK>wjQri8mv=hd2zdXU3{Y}WJ71N>FxGoW8o!fs6L9@P4Zr~0*muQ`nUxb~wy za2tfHkm~vQmTl6b>wGOKIBLHbu74$jdq?Z3>qMkW*ZJC3xZ|XDo&?t*+?bgko>S|1 zb(})+WMj>mb2#holz6;7%1-79@@8xg+F_}Q&^<%jH=VdovmeqP5paVp*ss$Et8 zrT7M5msrJik0q^OcarMAR($jxy{bayZ#bC^yj6-)6zo-0I>xE&`7kYiphGzAxxf8rX!Rixl3j(iLT#ll@@`uAn2;~0m2 z%LvC|j<@`A!dlB#4 zTo3L9U;G!={T_Z!7QEFd_Wc zwXRif(k*&{b+IdOG3_Foq*nyB|m99l)rQm44qxY?62zM58K*hUT%>$}`lih0A z<-N%9X2?L~{Ycpbj9=MwJT=93kBUEkzg)L)A8rY_?$<=0f}#?T54umaO*q$jh3-=w zN})&XOUGxu!X0e#agnN*b>2OBeD*rWSz-7sinq?o)OJfidT7~xXNKS z^>?;AMdwYeKZtJz@69m(Gjc-3+sXfy%=C%lbhN;IPDlTP5e(1O_{wDh|_wW0D z0{Tro{LH+?{5s=jmpYH4{7`&7u=Cz#yVH!_Zns@7?CAc^oOjsn6>KMq$%AYr>yI_i zoN%3Fh~hn`^i{i%9UY%l9Xgjh-ug1u{8YTHe_Fim2(W8}UCZHYw;y6m+CAd7%VaEc zzAHV7_5MRRza6acumpcg2v@5)YdoUAd3mh2NpqH7JN~voxUhMO(m8-nQ3@_#K7Qe| z0rPR1=-eZ5dCOJa0*)(aUTF|J*LhT$S4M<$%|A4+ct>-7$|a7VeAB#AEZm^xto7wM zxTy$S4C8v+r`WH5o=Np*%6C4vp$HtE*GxNx^|tBwWSjKp`z|Vk+aAKvcrb{1B!qiJ zxFzG8+%HPHqtM-upW6RCxb6sCE{&<5cI#1`wNxMBUNdpNr}lTNeHx*g^%>T6)*re} z!aZPg9}m{0IA+0T9Lu_I8ved;p6WLt{4T@arFfa2D8Gd<^p^tG_xnr0c}c3@kZ_M0 z-7CU%>G_A6;pAX#z;b{z_;_XuBhpq8! zHOAN};a%&iYH%YFIGR`bBXIQmLtMCnILw84Shguo^!!7IaAE5@s&@?YMw@U~YdtmJ zl5QdRi3o94f*VTVpY)o+B_incfa{LH(Q^Z)K4JAHzw_W)MDJU=-qw0M7jy7TgnE?G z`MC&OJvfuUKWo3TP4RVt>sZD8w?K2&I6~id5s$!4gX@mKW#T+sZwik3&1!J{!bLe> zSPQ=N1g?*3e`yC-5P=&2mnEEQojwU}Lj*1z=l@zFaCAN~ks?0IPbIi+;i8x&kZyQk5IU%^;RK%OXqECj4tMMD&7^3FFp=X`eaA->-{Xp{TpL2cpS=j{~-wngAV=deH_a$srcB)+|pY;78Y3tbT^Ae|@cbP|$?GG+XxEie8 zsCaAK`eauPyA7qRKa#Y9-DS#7<$>aA#5Ig5u{%!e{J3PBIC^d%?F+1bx#p~KXBb?m zaOa0``S^V%xF+HDHvM7eV~h7I>wFA-KS|~ti5q}G#d z%3}$*rU><@1y>WH9`xLR$+xS%1K^5M=+XX3&v)bq_YaBNTbo$i-_`y}cB^4GC3a0_ zoH|L_squ#TLHWnf|IXq5Y@Sc|_m=>^nfP;=-=X&}#Z$ifz>l2AeAs?Y+&DPzi_D#= z=Or2UDBc+E&u`ku+;!Ma7Sq2}UP!kP{D|-?1^4SG+r(9Z8x!u*5RUwJA;sg8?;-tdNrEK?S#7o6HN!eNXs|Fwc3iEWXF3GP|yz)bN z=!9KS4ck3!?DTWs%8v3d3cHcZ+3s&958qM#BoEYICl5yZevNqyXK-~0v$6TGVds*38?ML6|k+qq5|302QiVx^tv#?8O zJMBl=QC`z=kW*97{w~z@52(*-@EzAM|EM0f)w+oCTMNFTf%#V??`g|kTy!4;aqZv= zzR6sMjzjDy-T`n~*D`0G@ATU%pnlWf=e9DR7ZV}V4^+IQpN)INvznNvdzF*L@B!8MI=UNBQ_k;8bm+sJ`=LR|=aP<8kO%eRk z_k+|#;OP57HVB6yiHpgjHD9!XGx@tpbJp>DAGkKr3tPWY`^}@i-NIchdfvY3c{uS; z;}LzI$DHu58~$DOyou^p)J`Rs1be^D?S7irrEOW%t&`7F)`A-ouE20657vCx4sJrY z^$u7sR}9>Y=pA6jt2L@#LHjrChMHM_mGpn_0%d2-w`I84Hg-MpOV$}u z56V|PU6i|lxv=ve$^H*+O7xc8A0_h-af9FzqIZJ$@%K9`zG;e2xF!9@%6}#n`!mAj zXg#aHt_C+HT%H~uw4IyNfV4C5!1VYP_O6 z41@RL%)=!Y!^A($JyZ|Bb-!T0zc6{bL*-4iALXwV`crqa-9L4_YX79T>H9(Ydzeqx zylj&$eLqNCxaAU$zkeuQ;xltF{{NbFUlyG(p5iD0UwjYqSvoJ)I;<94Q7?0@{k|Ps zj&M7mpDd;y+wTQJy@rIlNw6jF3zq&s`(b(x{QrjiZ;G*jr{5Q>?t7v7<*r0~{+8`- zX0wI8G+uTs61} z;g%c+TU;x+tjF1Jk^J7zuWZx)If}d$2$zuGd%IP=tUOOreS}{pysvAWUra}^Q^H-U zIjf)3_n#CDa-1*j#h`by@*l7t(f5Ol{ekT^YJJ(J`t?CS;|b=>^C{jorEiV5RKJWT zng8rEiATS0PmOcbF7x;&YX{q5DB%*&ez`|NU$~cQoR z-X6)H-yW(zW{OPM{m5f~QBsf>iDF; znElnd&-r@9y5xPcQ%|!_C3{WtR;c$vs&xvO#f8lI@O^E%*gA^hwZndI|^<{xN25i7{^uqh^PDDyxr02OL5Ts z)uqCR#X%h1UtKKRRvm}*pH%&?N$kh8y=;@85`2oLIeyo9u3B&f5jg69dBV{pxMVTy zBAeuQ0G|xuJ}tjb`_j^JZxVmY5^k$*XVouK}vQ{#;!w8zV)x1FaM?F6M zCoq4r8E_qZ7x=u>m_H8pq9luh9|cf!HuE*6VGXbP2eoeueW>9)=3is=h5cK$iOUC9 z{6*&8(wucXPv`iuHZu2w?w>O6Q66Z#N>ngszOTR^7ggTKKaE!t7chT{_AlGS(Rk%m zGKWKBE~frgob%w)gximPreZkWN{;opX!|MQZkONtoHdV@fvfs5`}OzV0KIx}HNyGp zpz{~+=TwhQaCO36s{LB=4T9?zZpn3hi<<_Qb|J@iy+e=Q15hK}g$|CMGl>iLRR>oM zZYl*w{h<|H!9^V3CAiltSxo){`t1Pty33e%waX;=7tZWHe*1|V@}C~Vd?=iGAC%9j zd5+qL-djF#Is3oG(SNGJRb0WG>v)~++btIE4vEY6FWcmo?%VB;phx%Z#)a!}=*9NN z{#q;baB%tH+Jw8!!IgvazLvVajo@ZPui2s31+GH$Zgp_O;6@^Fv*1dvOdVe~#?FEW z96e`}6@jazu{Z+P4sJ^7v1HvY^{0L?0Io{(_Hy*UNpPiCaeY6gIqUc~9S7gVbr+})0P(EA8VujBY!<5oHb=Y(*sT%7|`MVNa)i(Bf zp5e@VCfl?h)B6ZIgmbO0d%!hC;6}mK2v_0oI}a`{9Dd{y5FfpdphY-y`6OswDx>>x zg$p}BNBbw;x7#n=4>30NtiYxj4Vf1lP! zR2=cp`@S2#&-~e%4;Wvn!FRMVU(V_a`m#;^pcUMRaISvP2d-Z@+@iq6#4DSmNACgX zji5*G0Wk5o{L=dfnndp)@#E*o@>>qBLAY-@xJGaj!nyiY7q~IuJ`7ylU2ut;OF$lTanSAw_k!V;Ee6Sd8MuUSx0)TwU;)Q7bl+}4oc+7H{%%)D`3&fezU zxBHo6M{cv}?vLHHDXwk|(Ldj2Q_m^pe{*B#^cAnTF=W~uXIZ`S#!wCYuf8!HAAHS? zq38IuH-^pcxy`2d_2#qu%#9&u#W{{xvEw1eYd40Y%z2tMFAx2^HuQh?#<1nR`njDo zcd_Fh#-F<}>>K)dVCd)Jp`XhvK7V6K-+!A;*%xmNm$cI_-5AC!Ik9A3Z(T-Tww`h2 zSFEqye^s7LzIJ2SKaAtS;W?KEzkXwwH@^9XxV1;$zALQ!^FRcg9!^xvtrJM zb8K0$=OUwfi;Kl%8#<2f(}qnSH1B@mU~&I8>|yo*^H_3`4M*&_!YGjsQ*N+e{2}$@ z6kBE-bC$^i+fXv+0&6yGxy&&;rVo-I3q~X3ILVGF;|Hq~GZrj4&xSQSE-~8f{Fri$ zCC6+y@vp`)Ve}AnW6GQb=h(1f&qc-$l|M7Cuwu^a!{z@G>u{0{Q+AwT^a%MgbtuQ+gNcYd+r{dKeG+jv*rzKcxdS7QAW=aH*@x^xWN(Qj~T})M!)JFFy$-@ zmaMtJmJP>TW|F%H%(>2r(Z{XJN%l+`J=;BC!h#v+S+HisC63s#=NjYZxChKQ@gM5M zgbk&L4w?C&iV#c1cjGrfeW?W#&h7Fh5v19an z`47IJ4Vyo~S#b`&unoHgUt~QdESYhE1shgeX2*{GZ(HX-#s7!u#1U(@Tw>3bW3I9I zBl)moe4}}sV#|zU&a(Vt`?KM~@SF{^(*8p~yP==!Y`Liw7q_zK4n}|CePPPIEVzGo z{%7tToBeG##^lf2aEbX##K&yjIw$1I9c+0W$K1>GFRjCp2N}Jz4M$k>IOCUDZ)o%C z(7r*PPHJ--d+ucVSKbSDyq@J>doLKhTzsr~lu6~i;E0#myh6UG^dGPun^!qM_ODhy zCa)16TK?3w(%IGHnEu|KCc zV#c1cjQ_!XX2u1UY}jy_9Xm$va-W$pI%|JUvS!MbGaR#E(#W4VC%?>htAlpQB{po? zagE78s>9IFUB+ks#7)Sr?Tv@)yz5R-H-|IbKxg;N^?^7Q}@3#*-b{v1e zz5J^AN87NS*}sUF@dxG4o{MZgBrcXC`7-%e^}Ax+hn)vI7OXy^Ud)fF|Iq($@?gv8 zYwGgv>do|{&W~fxvsqSO<{xujj6N=JjyU;s&;P^uFh1UfeeAi&_7mPSR{!a}WpbnM zz&AYSE|#r&u{fd5j7~Z~j@S>+Pq|;~=ACw4>^RHvjQ5)HC%umxafSI!_TiXYziFQp zaj`n<9x}bzKJ2+Xv{(JkGCA)(`srR-g4f8J>SmT*Gs2`HttG_kCk> z+4qti4-C)0;CsgAi{fJbC2@V%xZeF@&pm9utX?d>;(jsys{6&38^iM};`(pTza}n@ zILG|!?lH$a#`+uf8J@4(XKeg8<&6QSdX z8%*zhA|(GS4$d&X$B9s~;v&b~tM19x6Jh^Qe;#DTBOLKKdtPFE&lBM)Gj6`c`EnZ@ z?qtW^jP9k*OnCzf9%9X-Y%Fx4eN87ec~Rmd*F$%!1h5W!ib{>p9pJgwx0;mkF5KU6Jd(gLr;X9 z<-<;dc@|SALc{#wC&CJ&N1O;_jyd^%<~`CqV!FdUV#k{0que9Lk9LpPaw6FGG42ub z)VYa2eysb%iu24Kr>-3FIFrXaC)V8bU!HR-JMQ3^*D>Cyo=mx)IS;br5!O7;5ihah zRgSs&lKD?iPo~_-oV!`_de*#wBOYSUql}*DJ~88Emb}J_TRx?JKO z*?Y(Asp4kxE6#<{jCk2IS~LG??me5QdtX^S!+mG5+j%m3rhCrhS?&#^Uv(azHJ=H) z+&OUcZ1;fmbJU;ZbKN)Qd(@rj^L!r|KVQ8+r~b^?yudwRH7j1mFLYmrHka6Qh53uz z^P$a&&ujmheA#klXz#T@J1#JLvH3$kSBCyyH=iY=%i`oD8>VbI!=44B-*8?`STo}i z3%0Dd#u3NtIq?Pi7wXCOH|59Z|GA&cueTnX-*z6Xe@A`T{jTo^liw2`%injNUo`&@ z)R&`uzDJDyNS&DfvHQTX6c_71aUa?KsXQ6&cb}O5xjerl?w6=H>%Z__U_b9ZGJ2_c zGkKZwVsV4~SpAiI&F-(=FOFaCynEySM!h)ZPWG>GKiU1Q`@-av>dyF8-aDo|&Wx8> z@G472{#{*jHpq|H-}$ziMz1!1a6vwUuMwaA^tGO|tJP~5|7O28Ob)5n@SM@6+iq&! zB2Q+E@?`XO@v(Y`b=dRZ(Em<(vf*h)hvmtFn{IR4O)V#x{JlI`F=x+x%>F?G&*RMg$-QF3YmAQguHVjl?qJIq zCjYE%tXQ$jH3`^CqEi_AD;!4+2Q*>Hm`j{d-mL5^bz_0g!!Ce#*77LS+Qip1-5M1bD7aG`7_}< zD{h|9&xG;6iIXXF=A2{6iZvHG;)oqrIA+iI-^Dq!x%E!k%ie3I+|8DAOg`p)WW|Q@ z$GwkCk9)6Ke8Ts^{VPxU&amdr;XB3M?6|@FlzQC7b55~3<2@VN+{^5f&W$-_s?_{W1bGPU115bvM z)k98(Lo6PCGAuKD)X8v}<0qaBQ7n$k`plnxGR&}j?#Zx^@o!s?$s10FV@%(AGORH= zd@^jhoBn@184~vIJsGZJ{x2uP9Mg}S3-V1uC95;1!XXx)ITe1l z`SPg{-N$-gKNYq!|Msac!{Ym=!ag=XJQZptw>=$>4efiJ4r|PJo(`MtYyT&m4lCBH zcAXCE9RI@UFyUUe+3Aqz@1AiwT*vsir^A%z`SVVP8K$$R!<^^iUppNRviYskVZ`?L zPKQe@%hO?l{Yy@VZTA!J%TI@l)vHd2>p6PE>9D~1ZKuOgChtBSI%e-X9j#{#4}+$w4Zb)Y<+^NfbOJ~9gbM~ycG4wM| z?EmC5VTvg;7Mx|xk}Ve)J>^Vjm~ojUyP==!jITQrq6b=slWdQ(B_^wsLcp6jd^&V-5W;(d)eam*PuuQi|f>&$0#(0q=#!s7Mj zGk$~l53w(&SiaGG=C%1u|IU2IZ!(`_uCRNv`5YZG|Dob|i}@_yYCiL~na_05e8z7# zpJT4Ddx!ZPz0>@MX*1!N(~J*07p9zJ&Wa@$S@X*9{OO+z+osg(HJ=Oxv+sN|EHVDU zC&Ng8eC?BAb?|>UJio_HVdCNHdB2;&G{?-D-2bL9&w}+}a#J|U=Ak!*j``G0;VPp? znEwdlnXrH4P2oB=kGd%o>^RTtF*k+7EP0G$c8t=S!a5U1kF*abSuthH8TKrgJl1;5 zSTldzO<~Ipv^YI)c$z&gGkMZY;TkjM#+967!-^dj8U5@{VZ@XxEZDQ=>7U`x z-4r&o<6Sp}_)*s56ia4oILnSD$6R3i^EZWtDVLeEW65>aj2`X$Imwyu`@v-DGTXu|}DLxjA9&cSvGJ2Ny zm~(~|i=m(MY+1A85~E*rK1_4#4fkkPy{4`E;+w*LHh-bM`g?8+12?yBxsd?fV zb0@PuTM4^a@p`trfgKOA=TR2>S3>=B;{5ZKu*8Nlzp8#OSqZIn%{8_hbIgfd>iHKd zAz{X8mdsgmZfL($Tx@uF@MSAunJqiUH>`wp7L0!0JWjG@%ARxLX#Q#?9Axp=D`CX^ zZzDws z9Qxm(ohv7r-X$L|%M;c{2T=`@`l#>c;qASHdcb54(pPePktU-DBKG#bKQh z*O@N6PrsziNv0ojpN2MPhW5v;KeRc|=s(13otD?LIxbFTpO7cx|Fk}n8|BHKlg|@Z zD_#~StjFY}c-eD_qf_E#d0M~1K;D{wxSg~i#4UQN;#eSUPm>J_0=f#vIb1tytw79EtD`79|3-)7k(LLno zl6}~IW+hx@_c`moKptOk&gMs7bk0oJG2=Q5M%P)FlWdr>gWpFVp|= zob{%&VUhVA&W7VG?tC`%Z0~k9Y<{7AC(ni{)(D~EFP;r^jJ~2SOulA+X6t9e z)*0iNGW(`_4{gp3?Qh9vXtNpG-*#R@n`>;oD}Pqs6W`OsaZR4g{`YK{W%7S#!#tzg z+#D9!-|ptH%=Y#-hcz~Lyg6*Jnz%VkKHdDg+#IHv-Sy@$%j9l1hj~U@&1Ziv^V#0V zd^X$6XY~N{pJD!k%xCrx^O;PU&*+invwxKNY*X{uJl=d(Pc(nG`A;&R*)H>0vSh;r zc5E2^ymgqeW5IRSjGk#6CmH|3%^_vZ8P+U@e$Eg5zj$+~hkh=xV9SzgtT<-PiD!v} z2}hh}%S+C&$Zih#(9dhb`To+)VNSbz^3CDEFzzYhW_q2thjHxKah=gGi`%~^Qce#4 zu3*Y$X#a+BLz~f4<-wE%OU|=l&5lcqe?=b5xWGoW5Rq!9z&Zsqo;|RDJy1N zWXTb0u5iSj9XA+1{pJw=rtzF&$&3wW*|B8w4Cly{4RbEDWXGE895MPW_l1*;cgvp{ zXIQde!+CbB89h_}ta+MaUS{zu=faL#{-1HbdUM#$l)G4P4{PpY%L9YCI0v77b2vWu z9Q9=KT=iteiR;a0!iv-Em^0d=o=jP>;38{|*m8yW^KK4ThJJ4PZS$UgbJ)t1J6P~K zj<}co3vLcKFn;09;V?5EW5wq8t@rCUhgI$9rRw@S#&Mm^>&^RJ>z;HTzbB8g`k9@# zzj@_n)#DG0``XRn5TpOr&tgN}_F4b-=fYk#Th4`}Z0~z6MCL{JI~OJyGadHl3=0+< zZ95kZFnO?fe{B4(oC{fLz1`=+*g7Ll{Gs{JJQot?oMy+I(X-BlIi@^hUi|EHq0+9o z$oM(u!f0r7h57T(g)2jwoBqf;+{*X`=fV!=ypA>Z4*lFe^v|9P2Zw$h8TxsA=;tLy zFFY5nGUaA@)ZE7L-gBW9*Z9TuW%JT=VYlZUuV?bIbKwRSJj9mQ|B3N0KNr^6y;_`q z>RjI_ewQ{2%syjZrk_94H+MR{<{U97((52n35*zqvSFPqQ!E9P^=O@D3vSIy^` zyI5aw9`xLzkM!jXLi*+X7wHK2ix!3cjzCV3#W(v z?|C1F{_neYmHr>tkJ10Q-^_mK{OwzC%#suH;$gxO4;q*LuQ)j3wC5djjycD8!+l`N zMdlo_8lK%nYImHpPq5WX_vt!9I7l!sjSHlq|4_gh#ne!4W zULBkg=PUW})v%oncd_Ljj=7J?BUZx!<~+=b$2j6D`$t-j&5qTucRS{;$ku5`7n>0-faEc{bc=Y z_sH{xi|jaJ^c;EXAMpsQz1A7}xuzYz*!|KjxG}8r>(p)-zgE1QV*FC?12YyZInRbQ zJ1#MLnfHMy*Er&}p?!n(UMJ7LS`9l`{k8Sjv0(jj=RZ8J#L4n+oC}Lrn9uxgSHreL z+8^OTuwuZGb;P^MrhpU5yI=lbrZ>lq!-x3e2>(yC4BhIsD&G@&)!;CFU zuCd{m$?sV2?b=M(ahlQZswZ>Kv1G-Xi)=aKm@7=?)QveeSTcTx@tk7IjAPC+`8{=G z&IOiiSaX>@JGT4e&G-+`ht2Pl2e+~0&cQzt=inco5Bpg207pE`p2wJ$=ff&1UK#qi z>99Qi5A!(jE^#nn%W008 zGk%?OW5$Xl7a1K?4;EZy#VbSmjqYb-Jx&hR;^v6E+4Fj)eicf_ywy5v-(eo3!{@^hmOMT@=cVEK->VlhZvIE}xQzvOvgf8H>%L2VLqF$+=d3t- zxASK8PwHV@@gDbRXtQI@bvBIt$#_n(XUgck=fez>_sNsv_gk0kQR_1QpnGB6>cieI z=F95mdBH2Jx#@`Yxs@$gU0L%Q8&2y_Z$2Nk{4>wVgX2~4Gk?T|aG1%C3t@S9{-_J#665qj7_;QWdpv*q z1%F3EJUcIh>zF?ALMVnd=UKC6@-r9w9SKff2rJClv*)JwT8CQ)pL8Ll%zySmm>Jr< ze(>imgo^EqxL7~uLTEYq-3#H$@SL;qN#-tuk~tSxvEhiz?AbB?y$k-ngZskh{qo@? z8>Z|y!|3cIRB7s3IyZ!(_g z+tkl~Ee|taR1cP1)nC3{+#J2jzMqiKKdY1e_&w^xl+jW1ILVSJYtC@Qf*t2MX3hA$ z_GikLIoDWn%$gJbVm=dgoM!*N3tl7l4GWfKQ4c!ESPhiC2Q7P;)pFfu5rvU|Cwu;Vhv z>=>UBCsRfrHJ_6#nX={#<4^jIGUW{%agiNI%y04?V96^SGg>xp#d=JcGUp6S7OXkX z5o>l_;+QStv(B9<$ILnLG4q+Q<}^pl*>R3zR*Y{JCsU4?bA=^)*4*HTvF~QbDUO*j zKBt~cSu*DW%i+22+-UVe*gt&l&WoSvMRBvdBwl8p_8s|8>#WI}DGOGdXZE8D;W)>4 zyT#vwaL#wX#ovQ)zRXy2mMu%B_qfI1gK*w#7;nAB--ECoJC3=|?4GxTs1+Y4ncVA^ zkh10sqkG>H3YMH_&zkwBWCP4%P~vF+vU%c4RbEDV#g8J*)#g2@eh$dGo~y#!-fSr&NF(b{F!oz z1zXl!8`=+(|Ij}r|C{W~ge9k0GiS>=j#)8zxcr%O#EL5%v1iW>#*dKyihVf6irLV9 z)Gc8zYwl;ugBLAE%x}KbMDo zc0)hcSw8-ju=!?n+9_`~oMy*-c>V-=GiAl%iPmGyWscaf-^D6cnaf8uGb?J=b6bojo zILn45TQ0C?!|0UtnXqHVbry^+n$Jl#OxbdVJqt#s<-vqCGcK`U%Zh7^&$wqyIdQ9d z!-N&5*)V7E$y>rc);z$Lhnd}UOE|`os~qzRla*Wicf{7?Ruz*;=ezrWw zF^@30**#;!OYC@+JvU#n-nm=+cf{i1PBz@lWc3#R9kG3R11lb4&7+LZ-{QX`wl6O; z=QVcR@+oz@a7)A3e{8{icdtPSzY56nZ zmNj+cc6Qvw=ri(X%zdo6z?KciTxPyz+~@7{S@%`D#)QNxeTO&M(`CId^i*-AulsE-ZNi%>3)#V`ks5?q%cF)qRME+qLs=+DAM4mh;q(zb!BA>Z*LS8&+(& z$etrc-?2Uu_RP4!g7FvR&ndRdIOZ&q@A^(K=K?D>9C4XFJI4R5?#vi{QQVwl!;~Fo z7>#`=m~fsMYZhE$#g+}%*mBIC6JN5=_k1UqahfG_Hk@O}iqZGw&y*t;Tw%?gEjKu3 z+>7T2@@LMB6=&J7%tD^yp9d`a?JhAqAna{%_Ho2oav8s;SwueWyj4|)baLR*v67O*>X3>yng7vgE+rt z{^l+$F}hXv<33h*buLVQO1;=~m31s%W_L3`zMcQQVLqF? zo6r0n<}=!AK1V#t;-2O+zL)uIx#e#5zjqhnb@6bD@njb=7M$gXC8PUvVSzatHr!!; zeqZ}Cx}SYH;z1VM?92H6_GQbfEPvX*cbD%2y0AQ~$Bq@(Ib!rp=fz3JNf%OPoMFj= z4d>ahX7oVk$doM$uCeBrEhoO^{J6vZ*@N9D*4y1rb`Le5<&=0BKf*b2^hoD^5BWV> zo-7{YyME7s2^{2gWte>PV?0?q%XSK`w#pD;fuPlF2J?^PK znfuE4De_}+oqBWhRPQ9}d%}!YSaQ>M?9Z+2xP#FP#L1LDnS$Q+(RaV^mUHfwz zd+ucXLg&kj*R$jeYk~gs7A$B~m-|-%?i^RA9}9_|43a2|Jd)uU@8B>Ke686 zpE{?(KNIHOI7Hm1so;9Pt^u1-mmKoPraLkGmKQf;Q zTTZiQ&giAScT89@<01==SaF37d$!zQ&-nk$f0_K5Fk{A97A#qDfejnBTxQRX(GBuv z!YG)}Nfu05afa<*iIdS^i<21_hklNRey$AtFBj*~&yAs<@g~0uoMKdolNo1Ou^jrj zF!cY8IGJ;q4Lgpx&g>Q9yp7)@PO@jp^l!z-#E!{Zz1OTcaeM2$&3ny~8GG(!zNr3ed5G!T)rk#P8Nb7M zvf}1D$b$*oESHcvyT$JRFV0!|21}Vg3>E+)Rc-d(Y@5_mlaG-z7F@-6M`~c8`9-{;Td0 z>+|jr`wQ+7)6P9&bD@>A{++fTbkjMvuwd?h&KQ?h*4Z zxJPWh=pJ$WCHIKgm)#@QUvZDvf7Ly@ll{Nu9n9J=k7zK7&8>UK-bP;?CmxU+2M!(?feh{!BQ>hSkvik$b|7 zBi8@teAx%*%k(xE!{)nK=e8F^!tQn#!!9O2b}`Jdy#2+npY0tk`g=Iyz2ilH4@bN| zele_Y^b;4u6-IZu7@{reb?1v=8*A=l$K8zYaxq*#^z(+HpNEEi9v%9(Tnwj&eqJ8> zd2Q(DmY>xBlNbFR9R1wIntRxB-_U>8i{Ze~&%;AMj}84?WzU{tUSsrA7yX?Q^SF%( zQ>MI*8E2Vu9}8A2d59I4So1g=b{z2vTW+x9vh#~#{Y<&eg3-Or=OkOE9CL=r-7bcL zIpw|*)qQS#jwVVW0stl6b}=2oMve(H!}@pBozd^PPb~jHy%YQYp?k!${SKFV}YmBqshz)x#4{^NKy<^68mW=Le9Zs@i%IJ0CWXggC z=UKC6%O#H4GC3#?<{Y!)gnKk%!j97%GiUsIb!N(3yv5LF!-^dj8NETCOu52>J!@{T zW&B|4a*D|t)txzKS+V4Z3+&l2uGNDXJCGY?y_hh1 zyZo6lXTdqvtk`mq)jQvD=?W{eNZpBYP*Twue79hVvXz5JPS zodu&uSf7(@nR3h-CjTIR=A386nhlrOvSt4+-x)^l7AI52kF-9gSTkeGS&mr_{^P}P zaPXh3H+aPNb?~3%&Hg^!mlgL9E{SjOeSTNj@ide7i;pF* zvE`OW*@xSie8BH2bM9fqeXMzaBOYeUW9+%gF|V*bD$YmC`(MP#jyo8AP@GJ-ml^jn z=Rp=c!uUhZn=P+08u`vYM!ej{nmak-ZuY#M#lQM}8v1#NEsrw$u)JCFG89UJat(#n76=fR<$ zM>yv3q5p*Z+3+e`ZhnG(Zew{;{!CAab7-IT{xHAjyi@`XXVd?^UPSY z;1VmgY`DgjWA>cbW&Y>n&xF&=n6uyvEDMQ#PDo$AZxp<EZY`Mse zBlcY3m_4Jf%bzjhr&ym;Oqns`EOVADxWJMPD=xEU$A;@1F}lwFoMgw8J!d#(!RQ>~v*5%p+n)(5PP1mthI1UTV#`H#9I@vL$Ltxc%bzjhr`n%WOqns` zEOVADxWJMPD=xEU$A;@1G5QtzbCMlX_MG9E1*31upD}ADTw=`NRaDgQoR$OMyjt$p2V)S(T zbCNw%Cf}An3l^+6&xSQyF0sBU-=WR+8RGqpyqSO3?=<`W*4}O0_x$c_N8cAO3-0i| z`GMc*;rTV|vt#ag|3kmqL;r?<2YBB8NPV>P;CFlI-*ju}Ip#G+x4AV;JX3wRjR{kx zyp9=XnR6ctRxEjl6_;4^I7ht1mh0@e=~>ph?X6)eYwqBf*D=4{tzj=m+|M|=H5_Eg zBkXvb>5o~LHLr5a&A%$X+l!kGcQU%etzkD~7EHLGDHm9=VaMg6fAg)OW5#tBjB?{R z$%ZLg&ah{}=#ICBc_yrxF@B+SnX>rtTSLwmB6;6g zJ*`*sI7jT*@e0S>V0@Qb!C&l&P~s^4r5l_&W6)$xtl!;MqAaJ2^X00 zFbj@YbA>JYq5q!Ne}Qx5R*so6y_fh|uwch|M)y|lp`S}bKii?7mxumI^=8F*Ry{ez z^giBe7Tn98^NjCnJ=Q$RmZzECPkb!6G4wO_9*j7{js?e@XS~fiOu59IElaMk=9mp9 ze$6^e*m9a3bM~BLbbsG7CW9|Fjt$EPI=5lmgKiD0+Qmb>kJ=?KvEfyYxp^-iYTsf0 z!|kUXKf->jnX`DLd%+PGm^{k&jVX^a;2>Pe!uhi+}C6M>>kF~PVcD>RqKBA=8B^w* zVZnmIwE38FaqRa{_k_vAyys7}FMHi*Mvrtq>rW zTx7wP71!DDsJ!ka_J_eM+*d}&tb_5Jz3)#m{yUt*ldYHASTknJY4*$*zSDj&X33Nb z%-OKyDr@#^Iq?+z88JLz{~0r7${FS?SaO~HN^M zWcF<1>^MIhWqJ4W!^AW7W5kA2Y?-p-40|rfuYdFWP>gM!(@ySjewfz|?|FV$WXn~S z@%dq6^j_zOtc;ES9j=`hP52wa9&obdK3O zG5pK(!z>e)%(=jd4O^}?oLv0%lTi|p7ke8Tx*oe6{IStqx#V$7D)3^MgI<}5RoEV;mj z4STLKf1>)?a^m^={gwJza*91urcY8oYZeR-s-HP)wp?QTWc9P;2787t(ElmwXU&A+ zQ`OI$Ia|&#o>4zbF0yCK^l9p6&0tplr>mbiW44@T{0#N8d8A`ae_sEIGxVDbr`EpEV1Hx%!#2X3HhU&sIN6Zm?(gBK@DEe%4GFK3DzBnX}~_ zSxX1 zW%|Ef{mdD&;mD3x>zl z&zv<|E-`+K`dM;=J;Rsl|5o+0X2S4o>SxZJE$0}|tDhwo*|TN(cJ;Gna8&rU@oMO+E>3h`Angzp3{mfajaBYN%gbjA{(|0K4_hcIPogq1B{q)iUm_voMFR) z9p@Q*$T}Hui3vMq++e|QPJd3aVZx6487x>QBhE2p#e$2h*|O(4;}1KhSF4Y+3_fBV zEV+#pW7eEz$83y$)OhT<&hTT#WyJ6`#{IZ@+11v?=#$pPob$)5=Tpv?J?A_RK7D>T z!I1Ts&n2epSdPjsKXZQA{yO=ewI5?Xx4c$9BgUL!&Xg5*%dZ#x{le&T_I=Fb5hg5| za)AXKHasdn`~3OgiOXiV;*J76U=#b1djGNwWUEI##vUM?K#)7kBoFzLhF#IRy!IGER z@EW6kR_8nP=eBoy{x9m)?zx*`r(VW9H0JRrbDm(y(+vJqy{!P6s zxrZGOF#7lN!(pa8#)2nV^9(yKGrZ!wnQ-$7<1n$_@(0$%cHMnu_Cx16`XlcJs~_8^ z_wXmwui-9s+{@sn?gtYdVZq~UcxsIQ%y?s*m&Z7-vF4U{nU_1*b2r0l_KPVGvE)&< zJi(r)8UEaUG3J#q&cJ$uUw9v^H)qCKJjXT{#`cEuWW!a)|EpeBoT!Zde?A9T zaEd)sW`oZGwk()zS{~+EGkU-FuPhG<^Xn`R85n{)C zd)3QHhMSGUiu)PgV0p;daE|E>mxqcS7a84Xd1#q&ofU)kiE|sHiRB??#%Wf}#yDrk z_^&MwGFubYrV9c5+mzc9-$qm*FKd3(^*)w5yGyBb$IWx|& zV8x1yY}m5nI)j_5|3mhR+n6wB#%Y$!SaX&gO9r9(8L?r?Rp#tja$-S0Mr=97o+-my zsGl(lrkrQankAQ5vt!E*_6$F)|1H(emfGrW`fnXqHQ4b}`lA&-*`@2q~N z+|PnJYtFG_#V}Go6SmB`&Wb^;4sK&`7xgpdG&5!_Im?D6doD1#tNNL8l?8j&ocN^q z8L{USgS)ApA!it|V9a?YteJ9&89Nr-V9D@P#^)qkChWMMJ#&UT)X#_&V=gjf%Z%&H z8GPFK+{TJAYfiIe#*VY>Su*%_^)qC{n5#_KGv&l-^Dtu0DHcpwa)uQP)|_Xu zv188-2EU>HzcVi<88KnZ{Y;oM}nQ(zQ8&+Ip!=4={K5yRNRzG7- zF=fh}Gb~xK;XHfR40o!Z2|MQ8V9D?c@;S+l3B%t}KV#<1ILCq&D=xBS%bx3u?xFrO z*2Qfs7_;UyJ7x^;seUFbnR9^^8@60!5UZatC%$MMj97At4O8}f{*k&FGiJ(Z7R;F6U)^kYoOPmZ zwp?P*j>!X@4|9fpZ$3`4VZx6489dPWFy`FogO-PrEO};hkGk2gXYj}BK5KkN%sIu1 zDSIAb^x);;C<|7sxyaxl>SoGymJGfo&TVWMv*R>-W(=pD4-=NmxxkVQ8?G{Z=<;xl z3AcP*oRe&rF!&R7Ghxo0b1Yf0;vyTiY`Mtk|;QI$H+cG)`)rY#1|sq;;}l zHa_PpdzOqIWj~p*Vae4o&VGFUXU_dw`Y~d(Z+Y0wiu>6zXYl9N$(R*0F0y3Hmh0>o zd|Te5t&=HZ=A35pmp+eJKX!RI@IB}H6ys`VPqj|QGwwTkUS)9Dd7m@S$>m`utA*uZ zFPl^DBg0QF568ymr=5rKqR)Fz*nLIY^Zcy-t@*yOJS=NxOU7mMO?7`q|8J?A1*h0D zW%O<99OLJ#kI8qej|Cg%KXKj+u5)47U_EhR*!o@Zn_d{E%vas=!Vqg$cfBy|V|Moo z!;vw5?+bocP`~?J@VkQcXX?UmnaRU141smGkGL@GVDjh-!;Eoy7UOu2xiFkyvj4(x znkCPT&v}I{H~oYB$6gqAv3%Tx;Q$*R{=WTqf_d3IQ9o8sy5M&M)p5|cjGuhL?*;15 zE3BV#!S4i$KUIB9pJrX>9yUC{@X&=J74M#>PS!8IFf6cm^##A{r~mtm zw`^a(aADZZ{``d@XLI$!aFXqhE(}ZK^ZvqcmEoq9VcP}!_A4uXpU-&LSs5}0!HVDI zGv0Mqh6PsFTk(5*`dxoz=$YJL#qaQ`|As5W6uTR(`29WW{!xgsC$}n-s{C8a$b~3-)ir>jI z|BjX6FvH(i8RnVYePuYqdU9px82r}CuzAh=zr8X{vf8=gckj&qJ1axZj=g$1` z%CN}pUMs_8#`jqnf`2ssF8vtacV(Dn#*D$k?JLtix38m*Ss9j@@7M3L@&3X(S^uSd zVf$G5><_H?y*ulDym=TuL0$i(e`X(&FaPWjmgW@Kl-Ybp=A6z>teueePt9>lpTgHr;ahfGFHk@V8lF=LOH&Zq& zxXPM6TTc9&ao@D!_vnnvDP~Mra)u2H_MB(*X7w}W5({>$xxtR%zpLxG`k63c&iyQz zv*sLIR_wXR@Ga_R%yp&=u9%10STknFX@+lAKNHR}XUU2SY}qh)oBA2EXU2*DupUNi zIK`eRlX>+sXTgf|Y*>%&clvx_%PR~{tPGoeV0><6#+@v=hYb&~=i$+JStq-9uMDR~ z-(x>VE8~s6SKa@~_pJ=uN8fLK3{F}fBhD~k!Hn}PShM018+Pou!RQ0dXWc$>CzB7_ zM|R9ue8@hIah7bkz@81m1^dXDJyTBnmwt>`bBaAvMjy6LrYu-+o;7QBTw?eU>twwY@c$EShF3UbDinm~o0FQwFvAnX_QW`LX>;^^fsSsh=e~MxRzc6NW!B9w(W6 z#yw*6IqPK3Syn9Ba$)rIzDGvCU_8cWjK`GW|LDg_R!rD(KZ7sYPsW^M#)>5u*|25L zbw*!S_m7R+7>^ahpBv|E*2$cS=LPq%e%(GW;u3Qn9LHI*{_#2IwA*vO zuUPy*J)Us zyz|9j(}w;#E)EI%yI&j*Gmb9~HOq%y98UhP_McoFuE{UC<(l!hgW03>8-1L4867mf z{Nic)Yj-@y>KXboc&7FLpZsU($AADW9A)wo%e*n>n{$cN8eyy z*l}aDyf|#XRp9R^>t^*P`^)Cd<{jI|&CBvF?mzRlUL3A4eVh5LujDjaW(?<@3saV3 zoC{-|%@}{X`Nuf>G0utWi8EsO4(Gy@DND|baTa6zoz7*9vmWDI8sqH7_z885afa6y z=OkMu4BzEkm@*&ZoEzh;#`wFP%NS=n#<@Pm8Eh7Rk8@$lm?fv#G8^NS`o}oSG0ufC z&Ss3iSN&t0{TS!O4a6BSe4qN6GG)n`G0tL)zhC`hob?#z(imqq#!sq$j5EBUI49XM zVfX>{Gi5%;IXA{xjqwkve~hyo<6Iw~f9T?{`9^`ipRAMVf_1XxK2{$#9`lb}9F8&g zsCzTU*^Kd1K9`w&%=*}I>x6suar?;R6BqqHnLHk3Q2U%`$9ZO-bbs0K9IH=R*ZBO? z*7a-h7%@A2aoElH@6^eX`S|=Z*2Uzr&V%8i`#i?kG5wtHiyQ0rdFRXE3m1o7Y?!h< zb8$Gto=2H}@#1iT@t5q^7-!4;%ihcJdE;DfqCa=A<8Fpuv0qGhh#8Nv;0abd&4%aL z@e0GQTHhAqaw}8rWX?S-d4M$!v*j`NJjw9yFAirIbD1fxGUw)->d)=0xr;6LGC1oz z81V=b9%sf=EP0kSFSF$}_S|wa^L|bJjJcZ`_p#t1Ry@jvC)o2e!>_BKF|RP?rkh(Y zx3c0+w%o&>2N-@s{fv2xDNi!z8J1jT&8uv=Inlm$<)=4rM($6(dHW5i9j(~n!3b0;hAVao&Td6>~9^)uy3 z<~+la%dB~o9XH?JJZtJ_%w0^mmpKoz;t@7H&W@)T{-gRC^D>< zn-MD}Y?(3qfqP+p5*A|~k89Vgv^zFq-hUd834>?JW5kLHTV@Q_-K%jt7GoZ(F^|od z$8OB~FXQC)n-Mdf)vsWpT{C0Hg5iG~hY1_z>{u~)wme1*f2cnbX3SZzV#SsX!>jJQ z`BGkE&ThBh7QwE2{8L?%;;3xVsX2FaVOEzrSv1jl+uwlWD z4TI+!j}fD5`ZHz5f(2_7n+w5ga6f^F*9Z?SaP3v8&=vq4{68$XS^|w-I&Lt zp4SXsWPU~r2jenj#)1WFR_xd?+_XA$Oc=b_IvKHI!j>6>Us)Yam?vVPow8!VhBZ5O z42~H0I>uqbggG--EZDMQ@T>YWX2*=dOXM+P%Y;F&I-DNoXQrL9V9AON8}>Xm=3Uo# zV;+OQu}(&8ctt$8o^iEfZo0EPW@8?UF^?64>l=5>W5md2Gf!c4HoQdfqd5nR;(zJf=)oFk{VvJuAi& z#$nEm755mYVeq%+{Waq-X2OgaOBQTcF}SgD7_(!>;N`|+#Fht)8{EV=+A%X`ELgH) z!-l~Y<1l7$R30OiOxQ4E&w|lS^=Hn*<|)}|x9k|)%s8(w9wTNS31v}OZ=GDW9;XUBOF=EYx9W#da(w`|S7Hrt?xc%tZjd}Mr&O7WQBj!w4F=NMq;eCw5 zgbj0ctQfr0J~CpkOMk}9n6Y5VQ{y?Z((c$WysvSXGB{zKj94>a$Bg0c8iyGxmTcIt zW6$7S_UHHXXUc>HGuABFv0^x-KNEJ$8N6FPjMy?^@ca5RX2FaVOEzrTG57=h-y@F^ zGbSvVv0=fU6{A1YpD8;Q3@Uky*fU|YTYsi3Sh8ZnhCMq*_tXEq@))sT!kQU777YJL zeyuxH08(f|GCXT*YMBjYmBZkaKB!0K?>^O(h$ z$BH!@cI?Ky2O96BbuwbkgcUQkEEqgUf5vQ>v17^L1J=oiJrhQIR)=fmPnl^KELgK* z$A;k_8;2=_56WZ2nh85*3?IBYY`Kedat8|*V;-w9kIk6(5aTgn@FDduV#S0lGX~SE z!*1imEVMILEZMMO$DYB0dLC*Vrc78cW5a#M=~!roe_|XaJmh)CM!RG;=JTlMEl-U7 z7<}0Hd;Pty9rGL$USY~jcQp>TGUrZKoMz384QJW1WXH(3@x!f`6*C5pSREF`LpF@K z%9uS9PTbABjF@qXIa3y#VabBk=s515t`76sA!|mgJg>Q=-LPZNO*`bJ>S4s4jJbyi z53t}YYaVCAQ*3#b9WS%zH3pA#KYrbK+`*W;nQ$Ld9%9C$%z1(ZPqXAX*1W=&n|{Oi z+{)ll*2$2Kx-#z3&Ut_Z53}SkRy@g?XV`F=Ew8fU=DQo8+Zq0uzb_bZFH_DiW5I&+ zELpSS5^HvBxWRU8TVJ$qbqFWz11FgrBV^Sl8pt$DYAG^nZeTqn&b_cE*@Fr&%y##aZ?|&LDHHOt{39!AIpWV$W{O zd!qZvm|O2DpF5dy4>KNM&ciHtj3rO9;u+RlX2Znz!IPY~cF4`K@wuHTcQNB$=FH@G zJgD9C2!n&_V8~NUxWtqlGj6ct)_Yk8ce3FgwmiU|hZ#KC{V-3;W7-){GUpif&}btn_bYtY5q3UWQL|PZ_ab%z0)!&79|0@Cr+Ax{qn`i!4o2L~nERM9XU;hmtXOi96+7$b*=h%e+)suK z?rS`5W6YQ-r!vE~#T zrffOGjs<(pGst}oGvpEzUSZ12d=)qSp7FSq4R^BT9(Fvyo`;z{+q#)@ff*YXTxHE` zY`A60INZUGyBR#kzBA+@Mm)-xh4mDySU=bIp63luv*kJVyu#qHb^pF~a2qqmEZFGZ za$37%#^8C*(es?a$Bh4c`^ex0&PP0C$%qS#*)ZWMQ})a_@dw6X#DY^SnX=*xYZh!c z&z3biF0p6FVAgxfh~Xcamy=AGFy{dlJj{~ESn(ulo?*jfw!F%oiTaWkdT)0dhY>4g zo`)}T9@-K2G3FsAJj#?OnDI1oo@2p6|CU#@J8rt4I$rGlFyu}~+{2g$nD8)jmMply zk_~IFvSH7T)ws@=*zZ3w9wR24V#bs?XIQXc$$3_+S+g-t@Hh5dJ7mX*8)KYX?{8l2 zWXnD5cz`_*GkU3YGv)#lHcYw7j6HKsB-X=-C8t<1WzEidqQbhhV-`#}&y+PY1|OHt zCGCbCTW+vp_yFU)%(@wK4-+0>%4qD*!`clK?cn9^k9NqDjCh7IGx352)1&T_=NYdu z=jI2Bb304!V#P{+&%N5gD|~M;*mEy~(m65Y5ym{ugr}JDEHg&NZFyO{W1^kD z(dYaa=az@4gF9GoH%sng#Y3!llr2xN<7xIh$KXx&hcSa`>)WW>X^r#87CfYos3v;ia}*vma)uoX_MB(%DeGp)B}VK%?fIuUKIg{x zoZ&uoaFQt#7CgX`hgtC$Yo27wGwiv{;Iz+o#tcsD|995?=lU~Z#VIypTRixz^=pTm zVZ?$7=b7^~3nucLMf=F?bMjd8oOr`4>=``TIG^{u!kD|5a4%CHWX4MW>I=SKv}+z` z!&7W|mK`s%=QSp0?9*e6!%1dLSg|o)&jZ@Q7ws2A9%ICljCqDBmznV@b8g32cj0I1!WMai zX2&J=>=>N&KC$4|$LY_Vtht8`53uE7c09(OCmDRrJl31?jCRgt7QD)mn;&mnZfCm53*-voaF27hjzx}tl5nF@eS+Oj=02_9TRRa8{6_~hEFgLC)qM#$KW&iEx9+^ zDI@Kc3G;8dN1hkVw9{|7Ph*>fcJOWM8~bsQ6J@+&DzWrdyIX0{qo>%`EXUm-HtQb5=e{N&6tbV4PX2Fa#XBk{j{}|`O7-uua zxjM#I)IY{KanSgT*s^jS{Y9S_%r2RqH77o+-MU+w3UwK>WB-|Xku=Kit$@Ah+Sb8c*3 zQODTk;@JKV`#H9`KDK{gKcAw_ZOs4EevWNUkL`8)Ikq`Fw*Sk1j%_ZC?f(Y;)qN+E?u-drpn*AGuFsn=@nkf7Cy=IX||4tp2ghrLo&E9PuD$Kd*x zhKey4nXzTbbv6vXV1906v|0U3In9C@YtFJ`$?yj1XTpXVS6Q-W!-+HIzoGh>aEdw8 z(HmVF4zc7>Hax+eJFalT{0wd^Z)~$8jg(5d6LoAOT#&~yvFR-m;AeT>);+%x3Ml}w{>m| zx7i0aTxW5+OT*T6`+oaN!)|sw$ovkMhGPu2Um8xc>Qay z@)_Jk-7Gor-`06o=f#c(SlvxNvmNr;^D^sSm;XcK{DyqC+|Tmv_Gx@RX`jaDTxRy0 z_UWqr+`;;{>=X0fwoiRXdVfy>-BZEJ1zkBQCP8NTt zZic(<&v*~HJl;EAV{$*|_C@hOGCn&VV10k@>G(Y1IL-s?7yAdAXY@grhRr{bzsEdm z|M-%BpKaa;%V#_-fApd98U2ZM{nYv%<{mQNE1%iJ?Jv_u*k2}pYJH4T`9IU|k;Z5F zD0MK{=ls~S9@~$0-ea4auUXGyoHyJ3>R|mB@)xQcV)qiC*9=}NpHbmn|DW@F*`;AG zv%fVyiP1p7Rufo1Fc!T;_ah_#)X*kQ`jpkwcCg=4l z*Y&>(+7Gro#OAns)^Cx|>aFrwzRmt!NB?=_Gk?4BnZ3jQF@2|eMkg)}SJ`muuNwbd zK0g`1+qzk^9NX`4pV+cvRjE5L&wJe;#_#ifFnYgxz?!E z%+A;+#$U8g48CNaZYZC-*nHXh#j^33>?`9X`4i^(rq3yM+|TA)@>zacKEveVSpQ}D?6{xx1^1c7iuZ%*Mei4*Ro^r08QsJ@m%LxB*1TUV{?Yrz?6Pxa{7=rA z!9P3aEym|G>wj_1EIap=*}uB4jQ`C!Gx&Gsa8vnASYL5pS^S4{VD^Jc!x<+3>D(Bu z`yRWQd`_|XFXzVczx{o{?1#>Q@l|y*{E>6JxjMLu?f*D8RzG%b%zOL9^e6He|5Sdc z-_PW;;{n#!>?@0(I|pXJa1Km#Yqt8C`#E*w0|Ie0Hqa++c0E%<6_~ z{ynXJH!?nx3HgkEO+I^`VsqoQVVUJk)`m^DmcM0fm}Gj>wP7Elo2?B;*z+XYo39Nm z>rnn}?87bAh8@gqxi+LsZY7`5*0o`Q!L5zYmYZ&?-))S~^0vljwoN{h+sS8md*id? z2Aew=f17%?8=v_d*M?apcUl`NhId{Yme}(en`mv=emnVhSsSKV+;z>rM>Wsg+;z%y1jnCA)nRV<+GTS&*V3a&*-=0v*-3ZsOPukv)Z{fev!?^rz8J!19{_lU`~e1;E|&yK-e^!pR}EFb2aneSa2 z3MLPC&Ws-6Ua;q;yUPDl=gcx)8}>4Lq;p{WsI}n)gFmycqx;tU-&d-i+wNvve{Nq{ zJlei8dyI2nynk(I82*KGV8^XH%=4G>Sw41cm|=E6zVGqiaq3|FcvO2N@l-Ure9uJ!0_`_m=fj-4k{**7p^EFFk#2*v;e_&V~6Q`^M_8?E~9q zx{nNUbuoUnb$-?S&vE`NpX+?s9JYV#pXWVf^aA&i>8$!#yl~C$yqf<-?g6_OJ12%O z@qRJ+8~2&{OYI}8Vr^Jw`!e_EtodK=J}^G&yqLYhd&csW_MgqG>?gZ9_mtsl+@r4< z_qEQM`7!S&tJgUq5Fr)W+@0jrCwDO8ZkF7~mWLRAQ2k7KVvO_j80Wb${vq`<;ihk!hg;cjCxZp` zGv)#2Jj|NM*z@EV|FHU*ahVmbvg77+_Uj|+XU1KuxR)IdGW@9eneaFZo?^qZ3{I(k zjPu$U=a%nS2Y0aJZblzdKQkU;#iL`KC&u{4)j!60ZjAHF80V(%TE{2U&y+h^at~V` z7~{42neZ43o@B!_3_hv;G0v-FoSVNV&g~37rG6&d%Yp~TIFF3+Ppf~7^VAsU*)h(` zWBj!G$2hmN;@rW8yV-LeymsD36q!GfpR@EkjK48CMsMx0od&xjd!vEW`-%-HZS zJ04^3W&6O0i%fWq89Nr-V8!r){@lTiQw$oPlZ<$f31^wHWWkfHc!mv^*>j!YS9}hx zI5%!%%AG8ju;M;8Jj9L#gRk0eMm)uY4KpsY%9J^0Sn&uO z9%s)5hG(sxFnXq8a zPV#ky{4>J6g`@@*Wn6hHd(=2(8H9NMv#-0pJiI8pkOPT*+jj z{>-?61-G)|4mRA&jvad*Vf+*4pDE|{;^iWiT*jIiJFexxO-z64-=m*ni6T8~bORZ$@XF3ub5KJ!9T; z;$nB+elfZrUuGAb=du5i_w3j|r;cO)WqFSMS3HMf|6u=5>;I+tG5VE#VR6-amCdi! zjs5@GZzk8A*RlV%o`X?4Wah$^~p_d2>Zr=w?{XGql~V*A~DEP*vM+Z4PiTz-@hTWEP03xd*-(>{@(86A~syc30E;%c!SR+ z8PCnkxt$evv*iKCx4j`;9j`z5hOqcg-Ty-IY3DD#A*|Gnw%ia3b|1VUY#ZAycf2M4x{|3O|%*)!)gmW=P?zjrvrmKi5p z$AKl|FWwNQnX+Nd{VX|Q%~`gL?rS_3v;Fc7VFlx_n~&LdZU|eLAGsmyWb@zV89k~V zqyNMEsly3%UuK_X#OZo<+I+6(=gh~+1@kewr0z`T>_4+XzHEPIzsK=h!8qIya%SAf zjui*)9Q)@T3+>p?LoC^|<~4SV|IGcIVs(>aVHJC>XMEFRVe{zCj)j^j_po5cmPgrh zU>qF_3+`_{CM>yv&CQR6b(~yh{s&m+4&q>Y|6^f)s{e_{d@jg5tT}KGqt(`7#-q$R zu;7Am9wscWKNdDV&^)ZzaVH11Or9)%X6#w=8XLwBazCfoJXL%gxSq+=S8@MCj)&x-uCF>C3dYYq9-6Vu1FWBO+~-fszwWrtpXh(? z@er>V<2oLev3%a~u$tNPkB1G6*B=iRd+uWUg5zO7s}~**N0`6pco-NJ$HT&h>E{%S z7atF+7{5du?6{TnOOJ=0WB-QZVc*#Qvg2Vg_H&Nq%a8lqh_z-vtYQk;*p9;a9_WBYpb zWAcXMVcVnJ|Ni4)#iPyhCHpeo_hsiG6UQ&atDSMeoU<$#8P{+zTc+%|nmr3nj#>9H z<~eR%rd-E@C2LNzW5a>_nVhg5bI!72^jPa~F?*(rPg;)|3zpo%hBYVL!>B(VI;K3z zf&*(Vc%1c_aNr6ir;dl5IXALm#g;qSvt>M^?#$S;L4<@(K5y_1rK$uO8!drc7tmV{Ee++ZWtFwpowui|R49*^TW> z>M^!CjO{t~c!D+)rZ=j`*k(SqFRRDcW;M32sK?l5JGKY)7~AZ}_Ak|AY%_kM_W!B} zTV`YXSL!jgS&r?i>M^$2jO|~m$Jpj%Z2zBnjBQ4%wSOajwoJ$NHTjQi7GwLj@*mr* z$M)~!KepM8?I8cL&0%cMI}sK>S(d>_>9l z3BSv0U+0_WllJ{qC;T3-b8&0q+27_w*v$NP#<7b}`2Ah;{Lu;DyW!m2;e_Aab*>hl z2ve-?e8TVN+V3SNLdoLpC&CU+?r|dQXSVc2IL7wgCw$+A_3v}S@8nwleka0eM)yDA z_i?SCo(MJj2c7V{xYmF0iExzNLr;WxpSJ$PPlQ$;IS;bnQC6H|!{~bJa1ke*;=q-R z9w9#_+`x=mSa3TlHmtdiEe~^;av5`GEVz~>H?rndHmuokH+$~q zz{89mslH5kl{puzF%OrpX3CbU*l`^v+{B(02kv0}s1soi6CPm32@9TK#cOQ1@G0hF z!U>mi;A+N?J`vV4Wyze|SaK&T?q$u6EswC{EPKv-s`(f*&g?%^u3*kJELpJTW;UE= z$6f5%GJ1^tXTqaQImevQ)8xlREIGxRE7>w<{#g6Mfz{amIC(H#WqwY0h~?w$2jeHG z6I;f)@lP}_2WG5S+XtpkQXft@&GLHZiSd)|Gh0rWuTd8cjGk`%Q=B8FPqn|Ca5c-P zsR!fSIc3Y5`P1zy2X?HVp$<%+>3nj+1kt^H*DEOlnfocXim&w&-|=g6PwI{9DnH2(SW=fI5hdigVbf&4k)G|LytpYe<2&z2MBh5XqwTI;?S%b(Fp zi~Y`KNm2KjTsJuF@(e-1p#>gDoh{0jL$%lTo#bff&)F=zft`Lk!m@-O7i z=vDG(%|lE!$)7E+F?+TApKX0kv3QOAIdC1T*UF!9DStL>n7&T_>^Ncmm-1)N=sDJZ zz5E$%mOpE*X7UF4v*i|MZmA+)Y`BhD<$cBp8y4^MJdW*msn^)Q&GY^|alcz$O#a$< z*3;H!^giz^mhV?5_S>z`{Dbmg_aX0FrXN;!HXrdkKVM!u)Q8pIcwcb%m^@hQ^giO` zEe?~s*t^Zl?YnFef&g?(u?*a4As|UNk_r77e$G)-o2k$e+ zUs7LId%Z7TVEun|4p@A}`pF=fw!*H|-t zvGqB{ffR-9(bhCTN){x9)-+Pcg*VZrpL;$Y2&13TtNtk0IwXN>=u@hq8f z!h-348_${z2X@SVZaiB?jqyi~XUU8cPQ+FI!aj`g{g3-z;(jh>euMMMfvZ^`vmVpq z_LU8HFg~Gftaz9MXIY%IuP?O@6K1`AWy^I;PT5!1tQpPtJIs=Y*z*kY)8g8o?p(t3 zjQwQ8wT#cIJ1cJIz&$L^d9FEO&+NSC`eoMVVkWbmYu3yeUGQA9vgevDk21O9xn|Ay71kd-*DSf3JvT7_rTt~c zT}=Pid1AvO<8{ul=lqT4`IR~`;c{kN!-5-FaVs0{V8^}ed63anc{1TuW?cA6?oJ_cn84t7I8CIP47shiDJ1%3-RgC`6`-2HLGvjs^+|7yy*zgED z&avnGR~i2s`7_~iW?aL98(48G8}4Aoz3h3A(KY!q;Z?~E8huV5=D(FM?!%_hSM}k@ z=o|X5ax34_hux#^?Zd*i@uPj%Jo>3V92@;YA6C7cU+u%*(SPp4qId9neb_qs<35}j z{Y4+vR6N~>eJpsG70+{}{OS#vjA9$?QSjIYR_8Rt*CpG#PCIXkZ5zzs|W`7`4VmfXvl z2ifr$2VQ0TOZmT79$dl%`nCL-a07F0WyKwAxR)Idvga{I|0jPY zT)15xT*`tgS#vF0Zeq`EjDI74X57b;huQE9JI?!n`?-kmHTg5;D&}0ziksPRJ15-D z=(qA`!XwN%$BOem=zcEYgv&W_4ddU*pDDMp;11T@%a#Y(^BAKbe-bKn}rH#-$JFymGh+`)=_+3+AIJjN(G6|OSn!jD*oOIdLxTdw7V zn>cVA+$x^1ZXhGy1G?Oc>8U<$Gtv z&w?#0Mz=7YIXgD&IWYP=@!ZmQ*38(mV0tU#S+U`S9g_vdvt;x+>oZ||YvWmPkvyt} z>cQp@)R$4LzAU(7Y~TJ=*gy937_&b#-wx|@DXTv^6;?A@q>k)Zv$%u$G5X_E;V2u< z`y1o$C=T{q#cJ`Xu#w4~oD+6zS=`zAV04#L;Tjt*`Izxb#Lu1^Sl#thm}YV}=YSn2 zEbeaq86~H};=eVXE1BIx-Pm&*t3Npv_At4peP+jj#ZvXzY5cuTg%xbLp4q+Cn>}~4 z`qNY45R)nAm>uJf8-E|?m(hJsg>`JWmD&C56MG(Hwd_=wW%6gv=_ib5%HsacC!+_5 z*M243!i?L;{o`&cH zZup{gAEHiddBpVz=Qwcw=bhJw+84H5&I#9W;0DGIvp-B9ZhzSF(Aa*2=ZP7wG3V}a zot5hF_vYbJ4m{*~{Al~a^fC6)^@3|zbMcqd>B-h*xz;&!J$kltIQBosIb^cV{<7s6 zX3swr7JR`vFHj$5FIE?}8`Oh6cQbjJda&RT)~u{E*=SzIo7BVgjPv(cpG#PCIa{t_ z_GR#X6i~@)7S7=3K`KC#`sQs58@#d0%~1JUi_pvrk&reI=(kuwlAO zTYvXy`^9FTI9%_!>uc`&rg^oCZ^@G#54v7`U){bgjvshF*)g(ia!CHHn6l$)Mn80p z+*h#}`~Tf}VsZFX*g5w9r@D^*$o%_U|GD*8T@;^j(IxxM_(tdC8`imIewM-gX%V)w8Mh~&>*nZed*v#P(Ghyd={ZZm${@9t&vw6Zyi1zEhekLSrpKe{o>#WQ4 zxiev!>o%{_*+?XTmJ2VkRv9C%$CH_W`KGtK84FbU!C=obma8?YGQ?39Glygn{|H zW_<47{qLCxDTnvYgq)KP%=o;&`h3uMrXQXOdzpM}CLChE%Xn5?@XzvR!k#OcerhJH zW5vzmb=HhOJ>&EJ`q{DMQPvz-ea3$LoAp086BZtDjz8~y?cqx^VQOsuqdKzs@=Q4F zdi-_cn0!mVuIKw_!lHxXJ}?t1?fPHTL%aFTOlY(x-<=8TwfpZ|UpqeJzOnri_1A8W zsQ9C%~%F|&Bd)8z7 zk*C8R4(!-G%6v>8eL7ra&$yHKV@`)9Cmb(@uw!d44*qVb2X?|I^Jk_CMov*uj>2IXv^U&kKp`S*OF9vHv-z!+hr{ectKN zj{7&Qj?5374*B=3`|su%{jq&ybmVk6$l>SWVR=GcKhT~L2dnd^!&*)*sWYSBs0;hs zp9xE?Q%#)-tC&6dOxVEa@n=G(Kjof7+N;lmo+XdEUU7~!Bi9=)X3v!IlU)Cy{J45- zUw=6B74D@3cOLZ=MMo*&R6(wzK))GofWZ?`$~2>^5h^f`2!j2`9Hb z>+?3oaV_K9o%MMeduzC#ohA&<3F`O%s9uA^Z!%4T*3*LGnzUZ z)-dG;7Tn63JJ@kA2d=VCb07I@Pk3y+&Z~^?Yu|okJuYR*m29|{9XGM(Hb(ceZ%nw4 z84t7I8P=TlW8=7pJ(n?BCJv@t&zzfCayx78X3GQYd4%zwsRJ|4pSYh(*l;-~T*K)8 z>cEs+nR5qA?q$t`YdGmW50*Ehhlr0g zw=;dHd>OAe8zyWxjMpFLyq++gDf5S)4LQ?Ch-19Y9iuCq*RlVRo^QsFlFv!;GGXy( z>xjEz!-o6Wa>9i#af5 z^ceFp<$C5US#ldI?qbb-9C(=dW6y?6Uh(6^Gv>*P^(y%>f4ukI*#CsHp&$Dh_4=Qv z9?VzU_i=zRrVa7QYoIn1(;ga$13=?_wFLgedyi8qKyxiXhMz2s8 zHeBxh3D+>%==mG3bKV*AztTJ$UTYpErTt~jEv#6xH#{dltOLn(;Yx;uHsFOg5hl>zK1-#c8%|*mFPQH#k?! zILngJdFyjAlQ%kFEV+8T&VmzeVe}^Xv)~?f>^Sf!lP&UR#s#zPW5Sv%*fHn8jf~$c zf2Q2Yf-P$vV#j{$e~bOP?mFL}FFy`jo#(OrHhD39yZX2SxP~n^F#0R+1*Y7=oO@Yq^E|NO8Frj^ z$vp4&Jh0+24qV0TJ)TGTG#~I>XivDAJ-0LYpgR8Eb-rg?+}ia{>uIN-@;qzjJjRAs z8GYJ*>+g7#)o1MU{Ojgrjpv&6XT{I{@66A1x96DE=gl+rf5AT9!g_nf$MmcAi~ZN+ z$9kXl#hmrHlr2}X<62I*iRm}Y&zgJ3>paZ(o9fJr^KZ0oT*8{m*>Vj#Zs3Gl*>eX6 z?q&2X`7`A)=Df<13oo08OF7|6_FT(>N99p{+xoY>&gXuee>VS5{aO5nbI9c9;<=T) zXFN|#F5ABa`hV*=V>AC;n0M>z=H&~{g%ykzo(r4V-|k%4!|wLyd|!_H7M%<6@2lJ0 z&xIYV?t3mQzRh+1opCN~5>Lf#W1QUOdh*b7p&R>GoC{~zJ@ngU zeXKlLJ)%!G@GZ(o82?cg`=#Vch2|kSpOyFWBvQ1{hrc}+mj0o6K65Tq%)Vfs*ne5Pe`KC-syDNLJr|-yu7B@b*ud&P zjc5F`b7B6Vp1)9E<|ovHC3msrKK4A!^rZFfAfB0XVIAYM_HDe*dA~HC+gy2$59s7T6z0n~4U-9PW!*UK>!|b~AVFPP! zWzQWo^u;3?qYV^^I`vZ|Lx9)#C>giKCEK#N7i>eS#&-O?78sQ;<&^4FvaAK z)?u~yeAvSL&f*`hFF7AtmUlfL4zs)4`LNM^!#%7w#&NInzL!W{SDX*)7duaD&WBm| zrB4y(|5=BNnLSlLY&h$_BDemXjDLo8$9`^P!HO++GJ2-~%uDS2!=fkw|{VUFg zz2iKulsAW0iQ~?mtJkQv`zo$r$0^s_*IAd%W_ga+S!-8sa4yDiZG86V{w%%Xz=`o@317&DP<-jLBQhhjq+()cpN!`7e?GkHvpi{zN@Te|p~c44U_d zIl9^Z(9uPI1DFJ=bwy$!NyBjM*^de&(Ev?bGVW=8Sm8b^hWMtQr$3wx%h}&yHmn=H(`?u?de_--jp?7v zhGjQ1{@$};#oaug_n8fa_Hf_Xut~dIHXF8Tr}v)?yT|Jfm<@-SKWH}0vRytKYWF1% zne}~B)_dq|m>=nX*lb7`KVmklwB@x%VzX5v!P?!n2$ZLvfQoCi8wjM?2EHuHT$p3hK)>H z`@)htS+nKjtFwMz)_uIn_-nI%pH=+dm<+VxOusW5O6Htq$KC8X z^(XT9uKSsA9W$0JIn9O*C*05I-)6&vDQ8(Qx~KL*`@ozjORi?af)j3G^gZW*Dfh5o z$C^jkap1rOOZolTuyphX;$(bCoJ?7=;51t{jD9Fi=A5wNa6k7So()Tunde91V)K(( zzYpoYBgQfMx%0~Q7tTA2W7hpM>z%aU>`pn~EYFCW$@$rEg#88i-(UQ5o)2bM#K+;6 z&L5kr_L0SJ>?4!k$?E~?G4DcH&gy0tLc#313x0=D9Jja-_OrXyg>YtU|M7*8r23a$ z2ab3^o6kCLHa*-!RNL0@4n!7 z4fTKRLO8(WI~T&NIyH>$HJ%Uiv1iKcyBES57ThpiXT|8>E`*(|xQ`7FGdXBJW{mD_ zJ}zO;^#Nku9%@WAc6TY7gAW_y@-Qx$%cCgjTy}Hn#s=9Bd9>2!~wn*famn z3t`@$x{r$){Yd<*xrWJ)t;d$zIN>g46ZIai^T>FebK~`&sN0n5T*?Voviz?Le%Doe zKfMq(Gv%~+h9mZGy#BKbVK*ywYTzZV970`XD|5uR`K%~ zTV7>;US00%`mFurgzK1JxZrnNwK>g!4daXQ8?VpV7uHwQ$MeuK<>Z&fb6`0>C;zKH z%(;u*O)mOf$?=?C40{u1jiuQ9&vVu;#ape*$=fc5Gt4UI;!);# z*Tt}m#k-vcCe!M{@_p(&_HTEt#_Jza-|_l~FNXP#7RN{JJJXL{3~LyD+-lJYDzj!fh;;>hpn0)18*vI0l>caZ#7sI^Ai2ED% zk=g!>eqYhJZ#%ba4_x#+ipGD}d0_fI=bz0FoC8KTy%eS%t3J%wa2+QsIdGcE%`Sz8 zIrp>Vgf(Z`GJ2f&xtMWuDWuG}nk5T1+`dqB{{2gS@6fzVIN=Ho z%-P)LQrO6R;iXV9y6vT~lPy~ox4RS$G5&)~p=ZZyEaOWde!P5cFK_05c**Y=IwyZ5 zKDLW4g{|!GU_QooycG5^TP!Y?ce>oP79S^5=3#eVd9qz5E;jc!o^`6OZ#T~a?K^AkVe}yNWXW0P%fC8H->pNlzR%IFc!D^nIMxP>)qcHF~(9g~%p!cpcNSaHD`>oZ}`6^tJ# ze`egsk`){7^!p z=VDHn+0Wr|&Mo67ifiIWy$F2>cX528}8@83A1OY z3u{JCw>}p$eWv_baWw}PEY`}O6V}Y0C4aW;m^@qltT`}xj{Kh?UMB3hg84f6v*Sjl z&y_zL9#sGQ`Ql>qf=gjxW&Rh6hehE%!}ulQV70+>%1#50$!)5m8z-dMwkUwMYXTk|n&N5^49P4v23#Kf&niUJy z+`@)6Tkc`UjuReb&w&FMth4?H<CgE{9|F?yc$xrjZd7=P5?M`p}f zaswM~;e^{6?NEQF+{c24So0V=4jefD`PTm%`@o#bSTSSEwd}c(@yGlbZ(OuHMW);XTdalkR8nDeO0O8hwplxm>pCPw%=24_TRUk z*5?woT+W_r7@wCvGj3(c9c;Ll6CPwVD}ScE%7P0E>vQUT z*1f0>+Q}t#V9vFyxQQ*dvF9$vbLzm1hgtFr8_s*Nb-9Sqjq1RZt5|S7Yi?%8?Hst9 z$z^q5&LgZi$CmS7Vtp=Qd_^6YaScmuV8gAPa0jD79hmYU3m#+5tL(V&rPk+CCcl(F zbFO8@O>DW1J$EtwU->iRVU|3@hVwR9pNkm%O8!i_iUrrR=4N)>&Vjp`T$MjF9$~>b zR-FGb^K%J1E@#g*jD9VDCfv%5J6LcpD;{LSW9)d9Jr};*`2UkX6Ru>&wJf-a6}PeF zE=IqRKXV>t!!sN>?-j;hlRs-NW6xDge=C1h+{_8LGy0wU$9^7Q!6U3W$By$inx9LU z1o?Bqd~DB~3maLoV#l3~Z!#BJ7Cgk3J)@h>g=@?izf%01;)EHao6UuFOj)wvG+Q=f ze>4~NGv|a2XE`wX3;ExCE-Yrvls#87y>2cPthj{})=YkHF6?2!jt!5p=P>rqp9>3K zrJo5Ku3*QUJvTDFh5T7@Cns!~+*1B5*|Xy{#Gm7QUobhX|&nXVfnB89ftXZ<>G}AwnKP&F%gcBxzB!8BS zO6zkm<3;jk!PRV8FuH^MnX_iYJsj9E`(ydD=D?l{UT6I~%AXZiaKfC)V)?UV#g01} z-%0)~c!(`~Mt7D!bH;yZeNJ&;#_TTgXU&p5r$`&iz6&gb9VpUj1;Y`EzC#@%Di=iZIuTIPRZJbUh7c~9dRFEySGuQI)t@!O5# za_09oo;^3S{8Qr@O&QOcN0{8lc(z>h0r}q7cuu&M#r=%uz#XiX8PE97jAz5EOz&^} z2d&TL%pYJpdv0c#8qer~#6Ef;;rxIZ_Z$%ECM{X^!$RyHfto5jQD!a=4h z)tS+w=EA~SypMLi*gVF%EFR}vF@5}8*v06H&J!oA=fW8_*E>facHf%0u#D+bogWTQ z6EC}`J2$MKITsGFc$V|RuN>hXN%oHbJpT+MvFbIu93j_nti zmn~bSFLWMRvuFGw=aD7j&Bk$xdEq>=?6mPb|2REh|P_?LTw2Yf$^Kn^EUg>lo<=IW5tp!r`f+_F6?If zE^)HhHW!YK{qHd!qxX8gwwUMr@@Dk`b!7V?@iF?yT-Y}DbJuwNqs|e#9iG#%{Wt14 z`Z4GK&Eo#sxv+%U$K}iP6Y^#IN&7OkKP4^>Y{u)KHXqB+IG1C8<6MsYjNYREv-XAM z--(au=j;my7Grz2eHr^%v*n(#pWWF1dHXW0bH2&3DSeP6?cx;xqrOAPrRIPZuA@G{abl{b1p0${nlJq#csbi$M!#olkK<7Gq(R(J(>QC z=aL1_u;#p-=Hnuk|LXZ<%TRD=lA8$lqqwrX2pUnx3Fi;=m+v=!j2h_vgE*y z3qIw3CX5crp9yp3+{lg;Sf{yEO?DIFtT@Y-(dXsG#f)!uV@R2DHFFlMxP>ii_T0lb zx-oRjc$6iF(VO4s`$2e}I9W4e%XN%?Pn=8{eaSqx5U+O5W!n9d#I2oNe`A>UkLF(^ ze(i!wwd-6S+S$|P!HUrr%+JLvpK)VY!G^2dpMS|bt^4=h7;@LEuiO}_(e}o$i<7V4 z7!GsTccbqK;Wus!3-<7vH-;s~<^N<|?fl#F82#rP!}`$!>caNB>cH;b^t-?LiMX`; z|GF{E{|9*-xiKWne|BS7#cW1hS>NPx*v2Zl92zFKxEv0S-tw~V_0WH-%OU!rd~SC+ zEMaxW%OPXSb?jL(UVJ%BGh@Sw``K|aUcb}jFgvzci?8P%Mt8m(Iwm~Ii~~z9_^S1p zu;mI)m~-GpCU?0UD(2kDiY;3nV$YuOlFQ*5Gsa&tKd0C*W5;zISTeb*e3-Lg#ru{_;m(KthRZo{4YNmF4jWi=D|_x>y7F?^%Zdj%;V~wUyd18wT+1gf@|4w6Qf5vzs$Ld4fk>2;j#ZQ@?iYf%VGY0u2PrLCx~x!wR((RFRsxw z>cD|VIC+Zm$n2^1>z}O41M;2Z_Dwr|x_x8KoY6Dv8%u6u&t1%)DXy`9t+?3o42x&k zKl3%u6W>AgST8=tFBKn$SBh_Jzxr~R_iguc5%brG*Y#$z^QPUu!TLcx!vSn@C%p5efG|Du1ZI<=awB_I%-?VS z*|6onL(H~2cdU7h#Rug79qV%$vk%IjE!Q*oko;M5JEL0uOnHC>kBt4CWBy@repkL9 z5ho{H&U}aY#`fRXZ`p!k+cm{~3RG*s){%S?6_Z^D6Vd^Zxw4`#3fFIdx#i^&GgFtiCQ!4*QH}@-6uv5+74`T+M+6ll}5y!cHG2)+eZJ{-v#Ce#Kn>mHk=*%8U4F;{zd(n zGiAlqZ2wgqeHkP{wbR9x{pwASaspPG+LS#u>juI0c@OkRA&@6gJVyI64_TOMZ5GmKv%?<1~r z5%ZT`3ClQe73&S+WXH|pb#7<$vMYXnR-8OAwqI^vS#XXu=l{$&E@8P*oJ{`0e5_t2 zZ}ywy&Ft0kX2XMwUL$YDyvmdd|66=q%7QCdaxEKf;)L56y;j~#xQ`hRkL^<4%sKDp z;@~2-T*jWOnEd6Huwm@y*6})bFnzr^$Ll=EVzW4zy+NGJ-zeXsuD{7VoNzs(H;aqu zTdw%MQ2lRp9$0U6zSzD^J=nkfN@y5W&J&Y&I`7QibtNo5>HfcRe%Njkhxy8PtNS?b zd*t&A`@^N9e|^RG6mZ(UjlNesM&Bpy(f8Y@(e3j8AAUeRM?Wa<(GS_@(OUgRKP=DD zkJ#56_)+y5-J$;MKIZulSNU=0g2^s*b3Nk@7Tn8<2ifo#J6>h-8F3%e|5@{~{<|xo zWb-+3FxhQ9a~@>HV{AX~Ib-(s@;I*EOj&$E{v5cG`5wdfSe&IRKy zd49#;@ESYDy?x;n)4l4?itE_09Q!%V>>uqPYwl;u3G=VWlMNT1GB1~M;7Vq#yjgP- zdv0U;Rr|(@``Ga?qpx`{p5eEzgjwzSfcN2y_IKsa>Y#HqwtpZ#c0cr99j_mjH?tp$ zYwZ7-Jjee3@g6R&>b;lQ&8Qozv+^4IXRSN-Us9*>`YneLi6gu9;QKFJUpRz} z-R%b7f1&^OL#W37MMG%D{yPrg8k4&WVewgUb2*EL4Ph;ZM+{*z`$rC82lGb_VK1X> z2#3dh&a!#j5Eh){;|Jeo;r=HMVIzm@hp?TKrwpNG_0+-lR@m344`KCq{aMy$zgfKJ z-Om+_-Y|rmF*mYc#g;qSvE_t^*t6%rYmDAFgm~6?PBCFN`lcbQXTDV)WBnWTxR(tNj{Q6~_P=KcS6Ol4Mg3gL30HF9S|)!zgiXx2 zjRkkH;yzAzn920u`zGAS=#sd(m?cv-T+NOJ2X0~f-XYXXxrYTiRy@j<1A8u*bN~C) zpBY!MV9tsg*|1{AogCOQd%ya#X3w72m~L198;$1_C(M|9UQ9mN}yx z@@LM94R>;2%j|FD&ze1ZUSs+(`486T6erA>{H^?1vSi0;#yjQDg8SKW!sz4jXU^!C z*5_glOqqQ`{;XNB=N6`)ls_x(;e;KNUGitif&agv`wz1tof8F~<+zh2$hCEMUcuQFv=tN^1X&6}BPawJVz&?k845vL!C`_RV`@ea1ZPIjOmGk;Ool?xh*b!J z4AtGIyHD@CpZBlN^L>AQ-|t)J^gJGwGvu0i`(NSw%sI&cGj=}c{LEOg;$gNNFul_G z*=NIsWA-ZNU$8$XSTkk!D(7dxfRy@p>1EwEye)ie0;h4RT zJO7gXIl-DKyPt4=7A!dAG?U}b&+L=FPuTyIbF==m?~!Hw*Elb;YkhyP{~70GeZu!f z?Ejqe#s1IxzKH!Z&NbHmg*I$ra-H)qJL&tJ{VzF3?Ei8bPR9PB{A2%D+pu~?|JUTr z`s=<2nS4XO%)aUSFZN&Wo?`!L-+QtD+wSMT^xxq7j{P(4h4qcT=a_ugeZ>CS_gn1$ zo_mP>-*3aJp#QA;Gy7rdJ$U^;Qg_xr_V*+9|3tlG|4-X+HunEaon!yGHf&gRiQg+y zUuM7Xea(KOuB?CQdph?2+Vd3q&-;Fk{lD=X{e}MD`d(&o!SloH_r8zW|AXfy_W#j5 zw!FaPqI2F%%n4>U*_Q>USaJ{hELrgo2Rz2kpY6}KHSgw^`2X3d3sW9t!Bec3JU<=d zmpzY6xc~P0SNvUI8r185U$UxMH5Uq&H=7GXRyUsu+l_BJbDaxW~BOqvyg_amk^$vwbd1 ziF2;`Tjzh`T-X_%vM-CLxSu$l2RUG)y?okSsKkA4xRZT$T89M>vE>QRO+>m{7e(78|$3D--_1MPoFR;(w$&0IRDSn~nf(?(b z<@VUWM;*oe7r9??^wk^=XU2j<9^l}$ zbN>AVmd?Z8>)fmMl;>G+ti62$#oYt;xs7q`GUaCG+{OX7Y}Efo`^A11v7HB4^DtW; zXXj0xS0-<^?roiyUG}CuKTNsLdPA1tmdC~ITRe}k|LyW*!7(ddjQhXKdTaG_BfFQo zM|KXWKN}um@7>m8$y2O(j@|e8+{J#b`JdL|24>vMoZDmjd!2{f!=B&h`{qJ=SMBeY zFRKsuUb>z2kGL=9SEvWOAM`x2bEVHu?B}Y#a!wA-D=P8d>%Yo#&F+WnuRrG&Ry?dd zIjYX$jQd%z&yq*k=L{>J;eZW?9I-y;Ikir4Lf(nGe%3nde9k&dc$7U(+|BqeSTBy_ znK+@W zd?;C#^I;VCd-HtQ`PbsN&4)AEhwqpV>+Wg456p+%tge_3$C+F?A5O(_ydW+D4;Rhvvlb_xm)AeKrf&c$qLK1|%(zAN(~XZ>IEpb6Edzo|>LKD5S{`+XZ+d|l8TrsiW-$T&; z@P%-W?Y0H~-huUZFN6ae6boUNz5YTtWPHkF%y=@+=cfC~k9CaSwGdYO?}`59)?xZ? z`#dy$o)-K&2l`oxllK|V;rkcDe(g28|04enEco{i%sb+I%(zYa;Dg$kRnE!cD(BWe zxZ1o)>u@t`ZezopOg^M;?D7Cp9%jb%Ta7=u5RQunA9hZ*+@Zbr$U^9|Wvjja(S@+( z{+=J6(cV3_5HfMfT^xLDA*^|rI(=L{#Tkz<=YR!IvE(d=95eaEg7?FX<6ah@UI?c- zR30PB5L-t6%xGtR{La|@xf#Xele{^u7$ zYMo@pzT%cAnSQ}@z>FSPcZ27U*_nkhn0(Lk5dFT-Pn`Ed`#n;gv*v5h zf9hUY|4iQ6t6wgJjgN|-Z}-6bqB`jB-=q!?aj$<`2&*1#ojLh4nU@a-3+lo8W+U%+ z%eONMJEQj*d2ieJEu&DHUu+%uU30#06ej=GyqAr_L1r%>g_CS~k;8qXaK!lfRim)s zF~*mpa6w#hmCs$Dh4$*eC?w(mH^t8*PigPHbrjY=R=v211yh#X!HT;%;C^NYN8X>d zKi6f(zikvYv*b2b+{v0Jo@o8I+h3gX06Xs(c~4q9&&PI-ne?6Waqfv33+`sglKJJM zFvIlFD74n?zuUYx?meTh@$q8zo+Q5`))CiNjKW?voMy`-Og^a2?C}iyJYZh?dFzS0 zGo!Hf3HD)^{p;k<>MNr#|FkHm3* zV?Mj<7X41O@oZUhO)kFuV%Yy2``=;F?^Vld{bJb69=9>&PBz@j^8YM`gG}zY7>=^y z$rx|2e(e9-#W3Q4YoBG`J1zRXYI$=jhuq1Qd!rMJ;av2e^gr7=_twvpdztefOCDv# zldO4`oz3dP9xLZ4xtiVk*q1|YXL{eouq(!#-DST?brC1`w+?e&WWm+?iw7)*9tYgU zbjxDMS#uxr2QG#_TOMQiAnP%C@M7pZS3lQtz+Q|~>o9qUJYqZdv-42*%#6ob@l@>R zIW`=zyVZKnv(Cd7y+>%?Bdo(7_jABQG2Uh!raT?ndEip{J#sNL;vvUunLOXRk5VUQ zOj&Y@0~TyJ&E(OZC+l>!tFySr8Ro26@&YS%cB>QDv-7Xc&xG69WzLlQn6uB4$5``} zeY%fX^n1+?u$LHHZI9LzwB`{D{hbRu%$fsscBmhFY?!mR$9!hYpQw&;JPUF4B=cE3S)S}psqYIt_nctGlmkw2$bt>0 z*?o$-GUW_w*0KGm>dJy0>y}*4J{t}=X2aw~>hLuCF=fhvQ|z-~#c4J?%9bbD+39n@ zgcq4|&A)jr*k!>^hyGV6c^kruGz$K9LI4?{@wo1kT)}K zWzL-}xR)glvd?Mj6wj1@Z0E_?&az?8{8b>IYRrc7Vr{LERfXUiTtd!3&N zb9T9pJ@%RN82cP@z**L8*?HMw*z;=bJjnLt>hUV;yu$g!#XjR%y;^;=5BB@}!-hMU zTxLE~9$?NR%u45A$vXNvpG#I;ZM~Ws*nPe4YnE?N@7R98{`=*@E|WL95B8WbV||&t z-Xwnx-|T+0ccdyW0xAwTxtss8M|OZ{1}&xXg?yyW?e?7rJNrF`F`&iXU9>~oF& z>ahISd7u1Pawi+^jqUGuZ|w0XGoEC@N$U=oG5>(`aKP!f-x23w$r<*pkXLN~p#3@I z+Sl3tO8Yabd@r-T+P=(>n$Mau?0m$0c6pH*SHIr++`s`hv-44ZhYuL{F@K-M1$V`9 z+|ND_amZtAxy|_e6V4U;c{cX*V(jPYH(2kueVB4Hb8hE=yI6BSyPvdv?C0^=&r|Gt z%5!X==F|SWEA|iEpSX98dx|mFiU-{MCi`6L9@z2#hoAAhF+brR+4-!`>zn1v33fl{ z9$0d3j6d&l!j`95&)A29FL>^!jr*c=GrP{cv3JsVCSNk14Ox$E&B z?tRlevva-s<&Y!xPrFy+2HeQ&ZS#da2~In9^G@TLGW(@7~rRu}-VeCg{d36eY?!mFiir`kcQE5_ zCb{|SbCyHq=2y>N@_W?gJ=c8gDc4*je{N*KN%pye0~T!AXZ}3rV!_#%&+etLe#SbN zIzO`)Sf4}Avfks|Uofxm{6t@}6sqV;^)r2$`CoMIS9(69mo0^%^%GX&>a|N@6x(0# z^Ku>E=v>ih&kysrF8Q5k;}5DI8=m0cZRW9j`%*Y-y|TX)x>w8Zoz{t7?zv&-J)Scb z?^O>Dk9ZD07~%bm;-UkSytD`=gaoJ);=G$FV`{U&e;AL&l}6n`uwt)G5#z1zi3}J z>~naX?*a3Ylj_BSIR~7H{a^MR4BgjI9pkvK%3qv))AtIC>+PexnS+n8A82{NmA2Y61H{*wk))%+j!F*}Sd(fVTvHm!Yr&;m> zhg|n@c?9QUwQ4!+=YXqD>A%@>=!-kISPo~HaJ%;Y>g8~P*(qrJ}&t6MFH4WBTd zJ$7%s9LC1Aw_OgC-;mGkmP4Vv;t@7H&6WrC*Y~y_hxc9fyV&xbTn^{r`1|V@H;>TI z!S>~_`FiU;b~$v9tA8dhHr&nZam(QVa~_WU+-LmYiOvyy(sDSiz2=Y&&$9c}<*?3w ze|4W`J!Z^VaxZI6v*8iu&sYx0Y3p+%OHQ)S{cmGSIeZ9G#RV1A9~l>IMz zZrJ(Sa#&j%$1YP&vd^8YSTO&(eA)0cvs3EL^w;Kn&%H2X_xy6$6=NRc;HKqpGRCZ< zw;qSi_j%i~-(Pl~`;NU2XZ`z)!!$b&8vEU1>+no$fBrbE`hoK>Va1KCcaKBH&ZXn9 zEA|)TaEQIj#^DqT&a(fSaaem+`)kKxlGPi=VK;{?*?;Re4457qhjT2s=7-w3fkXD# za4VCyjl)#zfBV?)1Dkin*n4W$zgqu~j5|6G``G{ZI9y<7Fb>;(EZ=MF$HBGoV|JZ$ z%sSsmd9l9Ucvd%z!;YWW_r`HJ%#CT3! zn9h&GX6y78#$o+`TA!PkGmS2dL&3pqR>F|!9alo~3*$Gg_}ypy|9>SMXL0wHaF*?+ z74K(hzsE{g+t`mOTkc?T&y}#7J?>}CKGS=xgrhO$Nj9w6@;o~|>oVcmU#cfJvgIU; ze_ZkV%kpF%+wUzeR`)S4#$5L+``&NG`&Y(sAM44LaDokMCik}=Q;wPa^GewKYwJ8< z#qSSWhy7@};yo+lAHEWL=iU3GR=iJT{l{8|oopqn`i*tCju|J|dz?D3hU0JjX6u_PF|j``{*4oMOY0 z-RG=?W3it#OP*(+W7b^zd*^-rir?9_{_d49&GH2+eot53U#R~N>bXZ9*n81R*cxL_ zvF0AOJizP~_F>NREI4LwzdHX>`(^6Dg2!2M$Zomf_ixSTnv2FASn-~cbHCa89K6Lm zW^Y{yrj(GlA@eu1P%xC9HpCk4v^O;}e z9IgGXHlM=}d4AY%H#;BkJhI|ZCdXF%E-62@;yoef;yR|BV8$&hn8o7?B8M% literal 0 HcmV?d00001 diff --git a/crux-mir/lib/gimli/rustfmt.toml b/crux-mir/lib/gimli/rustfmt.toml new file mode 100644 index 000000000..e69de29bb diff --git a/crux-mir/lib/gimli/src/arch.rs b/crux-mir/lib/gimli/src/arch.rs new file mode 100644 index 000000000..f5b2e5ed8 --- /dev/null +++ b/crux-mir/lib/gimli/src/arch.rs @@ -0,0 +1,603 @@ +use crate::common::Register; + +macro_rules! registers { + ($struct_name:ident, { $($name:ident = ($val:expr, $disp:expr)),+ $(,)? } + $(, aliases { $($alias_name:ident = ($alias_val:expr, $alias_disp:expr)),+ $(,)? })?) => { + #[allow(missing_docs)] + impl $struct_name { + $( + pub const $name: Register = Register($val); + )+ + $( + $(pub const $alias_name: Register = Register($alias_val);)+ + )* + } + + impl $struct_name { + /// The name of a register, or `None` if the register number is unknown. + /// + /// Only returns the primary name for registers that alias with others. + pub fn register_name(register: Register) -> Option<&'static str> { + match register { + $( + Self::$name => Some($disp), + )+ + _ => return None, + } + } + + /// Converts a register name into a register number. + pub fn name_to_register(value: &str) -> Option { + match value { + $( + $disp => Some(Self::$name), + )+ + $( + $($alias_disp => Some(Self::$alias_name),)+ + )* + _ => return None, + } + } + } + }; +} + +/// ARM architecture specific definitions. +/// +/// See [DWARF for the ARM Architecture](https://developer.arm.com/documentation/ihi0040/c/). +#[derive(Debug, Clone, Copy)] +pub struct Arm; + +registers!(Arm, { + R0 = (0, "R0"), + R1 = (1, "R1"), + R2 = (2, "R2"), + R3 = (3, "R3"), + R4 = (4, "R4"), + R5 = (5, "R5"), + R6 = (6, "R6"), + R7 = (7, "R7"), + R8 = (8, "R8"), + R9 = (9, "R9"), + R10 = (10, "R10"), + R11 = (11, "R11"), + R12 = (12, "R12"), + R13 = (13, "R13"), + R14 = (14, "R14"), + R15 = (15, "R15"), + + WCGR0 = (104, "wCGR0"), + WCGR1 = (105, "wCGR1"), + WCGR2 = (106, "wCGR2"), + WCGR3 = (107, "wCGR3"), + WCGR4 = (108, "wCGR4"), + WCGR5 = (109, "wCGR5"), + WCGR6 = (110, "wCGR6"), + WCGR7 = (111, "wCGR7"), + + WR0 = (112, "wR0"), + WR1 = (113, "wR1"), + WR2 = (114, "wR2"), + WR3 = (115, "wR3"), + WR4 = (116, "wR4"), + WR5 = (117, "wR5"), + WR6 = (118, "wR6"), + WR7 = (119, "wR7"), + WR8 = (120, "wR8"), + WR9 = (121, "wR9"), + WR10 = (122, "wR10"), + WR11 = (123, "wR11"), + WR12 = (124, "wR12"), + WR13 = (125, "wR13"), + WR14 = (126, "wR14"), + WR15 = (127, "wR15"), + + SPSR = (128, "SPSR"), + SPSR_FIQ = (129, "SPSR_FIQ"), + SPSR_IRQ = (130, "SPSR_IRQ"), + SPSR_ABT = (131, "SPSR_ABT"), + SPSR_UND = (132, "SPSR_UND"), + SPSR_SVC = (133, "SPSR_SVC"), + + R8_USR = (144, "R8_USR"), + R9_USR = (145, "R9_USR"), + R10_USR = (146, "R10_USR"), + R11_USR = (147, "R11_USR"), + R12_USR = (148, "R12_USR"), + R13_USR = (149, "R13_USR"), + R14_USR = (150, "R14_USR"), + + R8_FIQ = (151, "R8_FIQ"), + R9_FIQ = (152, "R9_FIQ"), + R10_FIQ = (153, "R10_FIQ"), + R11_FIQ = (154, "R11_FIQ"), + R12_FIQ = (155, "R12_FIQ"), + R13_FIQ = (156, "R13_FIQ"), + R14_FIQ = (157, "R14_FIQ"), + + R13_IRQ = (158, "R13_IRQ"), + R14_IRQ = (159, "R14_IRQ"), + + R13_ABT = (160, "R13_ABT"), + R14_ABT = (161, "R14_ABT"), + + R13_UND = (162, "R13_UND"), + R14_UND = (163, "R14_UND"), + + R13_SVC = (164, "R13_SVC"), + R14_SVC = (165, "R14_SVC"), + + WC0 = (192, "wC0"), + WC1 = (193, "wC1"), + WC2 = (194, "wC2"), + WC3 = (195, "wC3"), + WC4 = (196, "wC4"), + WC5 = (197, "wC5"), + WC6 = (198, "wC6"), + WC7 = (199, "wC7"), + + D0 = (256, "D0"), + D1 = (257, "D1"), + D2 = (258, "D2"), + D3 = (259, "D3"), + D4 = (260, "D4"), + D5 = (261, "D5"), + D6 = (262, "D6"), + D7 = (263, "D7"), + D8 = (264, "D8"), + D9 = (265, "D9"), + D10 = (266, "D10"), + D11 = (267, "D11"), + D12 = (268, "D12"), + D13 = (269, "D13"), + D14 = (270, "D14"), + D15 = (271, "D15"), + D16 = (272, "D16"), + D17 = (273, "D17"), + D18 = (274, "D18"), + D19 = (275, "D19"), + D20 = (276, "D20"), + D21 = (277, "D21"), + D22 = (278, "D22"), + D23 = (279, "D23"), + D24 = (280, "D24"), + D25 = (281, "D25"), + D26 = (282, "D26"), + D27 = (283, "D27"), + D28 = (284, "D28"), + D29 = (285, "D29"), + D30 = (286, "D30"), + D31 = (287, "D31"), +}, +aliases { + SP = (13, "SP"), + LR = (14, "LR"), + PC = (15, "PC"), + + ACC0 = (104, "ACC0"), + ACC1 = (105, "ACC1"), + ACC2 = (106, "ACC2"), + ACC3 = (107, "ACC3"), + ACC4 = (108, "ACC4"), + ACC5 = (109, "ACC5"), + ACC6 = (110, "ACC6"), + ACC7 = (111, "ACC7"), + + S0 = (256, "S0"), + S1 = (256, "S1"), + S2 = (257, "S2"), + S3 = (257, "S3"), + S4 = (258, "S4"), + S5 = (258, "S5"), + S6 = (259, "S6"), + S7 = (259, "S7"), + S8 = (260, "S8"), + S9 = (260, "S9"), + S10 = (261, "S10"), + S11 = (261, "S11"), + S12 = (262, "S12"), + S13 = (262, "S13"), + S14 = (263, "S14"), + S15 = (263, "S15"), + S16 = (264, "S16"), + S17 = (264, "S17"), + S18 = (265, "S18"), + S19 = (265, "S19"), + S20 = (266, "S20"), + S21 = (266, "S21"), + S22 = (267, "S22"), + S23 = (267, "S23"), + S24 = (268, "S24"), + S25 = (268, "S25"), + S26 = (269, "S26"), + S27 = (269, "S27"), + S28 = (270, "S28"), + S29 = (270, "S29"), + S30 = (271, "S30"), + S31 = (271, "S31"), +}); + +/// ARM 64-bit (AArch64) architecture specific definitions. +/// +/// See [DWARF for the ARM 64-bit Architecture](https://developer.arm.com/documentation/ihi0057/b/). +#[derive(Debug, Clone, Copy)] +pub struct AArch64; + +registers!(AArch64, { + X0 = (0, "X0"), + X1 = (1, "X1"), + X2 = (2, "X2"), + X3 = (3, "X3"), + X4 = (4, "X4"), + X5 = (5, "X5"), + X6 = (6, "X6"), + X7 = (7, "X7"), + X8 = (8, "X8"), + X9 = (9, "X9"), + X10 = (10, "X10"), + X11 = (11, "X11"), + X12 = (12, "X12"), + X13 = (13, "X13"), + X14 = (14, "X14"), + X15 = (15, "X15"), + X16 = (16, "X16"), + X17 = (17, "X17"), + X18 = (18, "X18"), + X19 = (19, "X19"), + X20 = (20, "X20"), + X21 = (21, "X21"), + X22 = (22, "X22"), + X23 = (23, "X23"), + X24 = (24, "X24"), + X25 = (25, "X25"), + X26 = (26, "X26"), + X27 = (27, "X27"), + X28 = (28, "X28"), + X29 = (29, "X29"), + X30 = (30, "X30"), + SP = (31, "SP"), + + V0 = (64, "V0"), + V1 = (65, "V1"), + V2 = (66, "V2"), + V3 = (67, "V3"), + V4 = (68, "V4"), + V5 = (69, "V5"), + V6 = (70, "V6"), + V7 = (71, "V7"), + V8 = (72, "V8"), + V9 = (73, "V9"), + V10 = (74, "V10"), + V11 = (75, "V11"), + V12 = (76, "V12"), + V13 = (77, "V13"), + V14 = (78, "V14"), + V15 = (79, "V15"), + V16 = (80, "V16"), + V17 = (81, "V17"), + V18 = (82, "V18"), + V19 = (83, "V19"), + V20 = (84, "V20"), + V21 = (85, "V21"), + V22 = (86, "V22"), + V23 = (87, "V23"), + V24 = (88, "V24"), + V25 = (89, "V25"), + V26 = (90, "V26"), + V27 = (91, "V27"), + V28 = (92, "V28"), + V29 = (93, "V29"), + V30 = (94, "V30"), + V31 = (95, "V31"), +}); + +/// RISC-V architecture specific definitions. +/// +/// See [RISC-V ELF psABI specification](https://github.com/riscv/riscv-elf-psabi-doc). +#[derive(Debug, Clone, Copy)] +pub struct RiscV; + +registers!(RiscV, { + X0 = (0, "x0"), + X1 = (1, "x1"), + X2 = (2, "x2"), + X3 = (3, "x3"), + X4 = (4, "x4"), + X5 = (5, "x5"), + X6 = (6, "x6"), + X7 = (7, "x7"), + X8 = (8, "x8"), + X9 = (9, "x9"), + X10 = (10, "x10"), + X11 = (11, "x11"), + X12 = (12, "x12"), + X13 = (13, "x13"), + X14 = (14, "x14"), + X15 = (15, "x15"), + X16 = (16, "x16"), + X17 = (17, "x17"), + X18 = (18, "x18"), + X19 = (19, "x19"), + X20 = (20, "x20"), + X21 = (21, "x21"), + X22 = (22, "x22"), + X23 = (23, "x23"), + X24 = (24, "x24"), + X25 = (25, "x25"), + X26 = (26, "x26"), + X27 = (27, "x27"), + X28 = (28, "x28"), + X29 = (29, "x29"), + X30 = (30, "x30"), + X31 = (31, "x31"), + + F0 = (32, "f0"), + F1 = (33, "f1"), + F2 = (34, "f2"), + F3 = (35, "f3"), + F4 = (36, "f4"), + F5 = (37, "f5"), + F6 = (38, "f6"), + F7 = (39, "f7"), + F8 = (40, "f8"), + F9 = (41, "f9"), + F10 = (42, "f10"), + F11 = (43, "f11"), + F12 = (44, "f12"), + F13 = (45, "f13"), + F14 = (46, "f14"), + F15 = (47, "f15"), + F16 = (48, "f16"), + F17 = (49, "f17"), + F18 = (50, "f18"), + F19 = (51, "f19"), + F20 = (52, "f20"), + F21 = (53, "f21"), + F22 = (54, "f22"), + F23 = (55, "f23"), + F24 = (56, "f24"), + F25 = (57, "f25"), + F26 = (58, "f26"), + F27 = (59, "f27"), + F28 = (60, "f28"), + F29 = (61, "f29"), + F30 = (62, "f30"), + F31 = (63, "f31"), +}, +aliases { + ZERO = (0, "zero"), + RA = (1, "ra"), + SP = (2, "sp"), + GP = (3, "gp"), + TP = (4, "tp"), + T0 = (5, "t0"), + T1 = (6, "t1"), + T2 = (7, "t2"), + S0 = (8, "s0"), + S1 = (9, "s1"), + A0 = (10, "a0"), + A1 = (11, "a1"), + A2 = (12, "a2"), + A3 = (13, "a3"), + A4 = (14, "a4"), + A5 = (15, "a5"), + A6 = (16, "a6"), + A7 = (17, "a7"), + S2 = (18, "s2"), + S3 = (19, "s3"), + S4 = (20, "s4"), + S5 = (21, "s5"), + S6 = (22, "s6"), + S7 = (23, "s7"), + S8 = (24, "s8"), + S9 = (25, "s9"), + S10 = (26, "s10"), + S11 = (27, "s11"), + T3 = (28, "t3"), + T4 = (29, "t4"), + T5 = (30, "t5"), + T6 = (31, "t6"), + + FT0 = (32, "ft0"), + FT1 = (33, "ft1"), + FT2 = (34, "ft2"), + FT3 = (35, "ft3"), + FT4 = (36, "ft4"), + FT5 = (37, "ft5"), + FT6 = (38, "ft6"), + FT7 = (39, "ft7"), + FS0 = (40, "fs0"), + FS1 = (41, "fs1"), + FA0 = (42, "fa0"), + FA1 = (43, "fa1"), + FA2 = (44, "fa2"), + FA3 = (45, "fa3"), + FA4 = (46, "fa4"), + FA5 = (47, "fa5"), + FA6 = (48, "fa6"), + FA7 = (49, "fa7"), + FS2 = (50, "fs2"), + FS3 = (51, "fs3"), + FS4 = (52, "fs4"), + FS5 = (53, "fs5"), + FS6 = (54, "fs6"), + FS7 = (55, "fs7"), + FS8 = (56, "fs8"), + FS9 = (57, "fs9"), + FS10 = (58, "fs10"), + FS11 = (59, "fs11"), + FT8 = (60, "ft8"), + FT9 = (61, "ft9"), + FT10 = (62, "ft10"), + FT11 = (63, "ft11"), +}); + +/// Intel i386 architecture specific definitions. +/// +/// See Intel386 psABi version 1.1 at the [X86 psABI wiki](https://github.com/hjl-tools/x86-psABI/wiki/X86-psABI). +#[derive(Debug, Clone, Copy)] +pub struct X86; + +registers!(X86, { + EAX = (0, "eax"), + ECX = (1, "ecx"), + EDX = (2, "edx"), + EBX = (3, "ebx"), + ESP = (4, "esp"), + EBP = (5, "ebp"), + ESI = (6, "esi"), + EDI = (7, "edi"), + + // Return Address register. This is stored in `0(%esp, "")` and is not a physical register. + RA = (8, "RA"), + + ST0 = (11, "st0"), + ST1 = (12, "st1"), + ST2 = (13, "st2"), + ST3 = (14, "st3"), + ST4 = (15, "st4"), + ST5 = (16, "st5"), + ST6 = (17, "st6"), + ST7 = (18, "st7"), + + XMM0 = (21, "xmm0"), + XMM1 = (22, "xmm1"), + XMM2 = (23, "xmm2"), + XMM3 = (24, "xmm3"), + XMM4 = (25, "xmm4"), + XMM5 = (26, "xmm5"), + XMM6 = (27, "xmm6"), + XMM7 = (28, "xmm7"), + + MM0 = (29, "mm0"), + MM1 = (30, "mm1"), + MM2 = (31, "mm2"), + MM3 = (32, "mm3"), + MM4 = (33, "mm4"), + MM5 = (34, "mm5"), + MM6 = (35, "mm6"), + MM7 = (36, "mm7"), + + MXCSR = (39, "mxcsr"), + + ES = (40, "es"), + CS = (41, "cs"), + SS = (42, "ss"), + DS = (43, "ds"), + FS = (44, "fs"), + GS = (45, "gs"), + + TR = (48, "tr"), + LDTR = (49, "ldtr"), + + FS_BASE = (93, "fs.base"), + GS_BASE = (94, "gs.base"), +}); + +/// AMD64 architecture specific definitions. +/// +/// See x86-64 psABI version 1.0 at the [X86 psABI wiki](https://github.com/hjl-tools/x86-psABI/wiki/X86-psABI). +#[derive(Debug, Clone, Copy)] +pub struct X86_64; + +registers!(X86_64, { + RAX = (0, "rax"), + RDX = (1, "rdx"), + RCX = (2, "rcx"), + RBX = (3, "rbx"), + RSI = (4, "rsi"), + RDI = (5, "rdi"), + RBP = (6, "rbp"), + RSP = (7, "rsp"), + + R8 = (8, "r8"), + R9 = (9, "r9"), + R10 = (10, "r10"), + R11 = (11, "r11"), + R12 = (12, "r12"), + R13 = (13, "r13"), + R14 = (14, "r14"), + R15 = (15, "r15"), + + // Return Address register. This is stored in `0(%rsp, "")` and is not a physical register. + RA = (16, "RA"), + + XMM0 = (17, "xmm0"), + XMM1 = (18, "xmm1"), + XMM2 = (19, "xmm2"), + XMM3 = (20, "xmm3"), + XMM4 = (21, "xmm4"), + XMM5 = (22, "xmm5"), + XMM6 = (23, "xmm6"), + XMM7 = (24, "xmm7"), + + XMM8 = (25, "xmm8"), + XMM9 = (26, "xmm9"), + XMM10 = (27, "xmm10"), + XMM11 = (28, "xmm11"), + XMM12 = (29, "xmm12"), + XMM13 = (30, "xmm13"), + XMM14 = (31, "xmm14"), + XMM15 = (32, "xmm15"), + + ST0 = (33, "st0"), + ST1 = (34, "st1"), + ST2 = (35, "st2"), + ST3 = (36, "st3"), + ST4 = (37, "st4"), + ST5 = (38, "st5"), + ST6 = (39, "st6"), + ST7 = (40, "st7"), + + MM0 = (41, "mm0"), + MM1 = (42, "mm1"), + MM2 = (43, "mm2"), + MM3 = (44, "mm3"), + MM4 = (45, "mm4"), + MM5 = (46, "mm5"), + MM6 = (47, "mm6"), + MM7 = (48, "mm7"), + + RFLAGS = (49, "rFLAGS"), + ES = (50, "es"), + CS = (51, "cs"), + SS = (52, "ss"), + DS = (53, "ds"), + FS = (54, "fs"), + GS = (55, "gs"), + + FS_BASE = (58, "fs.base"), + GS_BASE = (59, "gs.base"), + + TR = (62, "tr"), + LDTR = (63, "ldtr"), + MXCSR = (64, "mxcsr"), + FCW = (65, "fcw"), + FSW = (66, "fsw"), + + XMM16 = (67, "xmm16"), + XMM17 = (68, "xmm17"), + XMM18 = (69, "xmm18"), + XMM19 = (70, "xmm19"), + XMM20 = (71, "xmm20"), + XMM21 = (72, "xmm21"), + XMM22 = (73, "xmm22"), + XMM23 = (74, "xmm23"), + XMM24 = (75, "xmm24"), + XMM25 = (76, "xmm25"), + XMM26 = (77, "xmm26"), + XMM27 = (78, "xmm27"), + XMM28 = (79, "xmm28"), + XMM29 = (80, "xmm29"), + XMM30 = (81, "xmm30"), + XMM31 = (82, "xmm31"), + + K0 = (118, "k0"), + K1 = (119, "k1"), + K2 = (120, "k2"), + K3 = (121, "k3"), + K4 = (122, "k4"), + K5 = (123, "k5"), + K6 = (124, "k6"), + K7 = (125, "k7"), +}); diff --git a/crux-mir/lib/gimli/src/common.rs b/crux-mir/lib/gimli/src/common.rs new file mode 100644 index 000000000..79cf76616 --- /dev/null +++ b/crux-mir/lib/gimli/src/common.rs @@ -0,0 +1,363 @@ +/// Whether the format of a compilation unit is 32- or 64-bit. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Format { + /// 64-bit DWARF + Dwarf64 = 8, + /// 32-bit DWARF + Dwarf32 = 4, +} + +impl Format { + /// Return the serialized size of an initial length field for the format. + #[inline] + pub fn initial_length_size(self) -> u8 { + match self { + Format::Dwarf32 => 4, + Format::Dwarf64 => 12, + } + } + + /// Return the natural word size for the format + #[inline] + pub fn word_size(self) -> u8 { + match self { + Format::Dwarf32 => 4, + Format::Dwarf64 => 8, + } + } +} + +/// Encoding parameters that are commonly used for multiple DWARF sections. +/// +/// This is intended to be small enough to pass by value. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +// `address_size` and `format` are used more often than `version`, so keep +// them first. +#[repr(C)] +pub struct Encoding { + /// The size of an address. + pub address_size: u8, + + // The size of a segment selector. + // TODO: pub segment_size: u8, + /// Whether the DWARF format is 32- or 64-bit. + pub format: Format, + + /// The DWARF version of the header. + pub version: u16, +} + +/// Encoding parameters for a line number program. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct LineEncoding { + /// The size in bytes of the smallest target machine instruction. + pub minimum_instruction_length: u8, + + /// The maximum number of individual operations that may be encoded in an + /// instruction. + pub maximum_operations_per_instruction: u8, + + /// The initial value of the `is_stmt` register. + pub default_is_stmt: bool, + + /// The minimum value which a special opcode can add to the line register. + pub line_base: i8, + + /// The range of values which a special opcode can add to the line register. + pub line_range: u8, +} + +impl Default for LineEncoding { + fn default() -> Self { + // Values from LLVM. + LineEncoding { + minimum_instruction_length: 1, + maximum_operations_per_instruction: 1, + default_is_stmt: true, + line_base: -5, + line_range: 14, + } + } +} + +/// A DWARF register number. +/// +/// The meaning of this value is ABI dependent. This is generally encoded as +/// a ULEB128, but supported architectures need 16 bits at most. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Register(pub u16); + +/// An offset into the `.debug_abbrev` section. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct DebugAbbrevOffset(pub T); + +/// An offset to a set of entries in the `.debug_addr` section. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DebugAddrBase(pub T); + +/// An index into a set of addresses in the `.debug_addr` section. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DebugAddrIndex(pub T); + +/// An offset into the `.debug_aranges` section. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DebugArangesOffset(pub T); + +/// An offset into the `.debug_info` section. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash)] +pub struct DebugInfoOffset(pub T); + +/// An offset into the `.debug_line` section. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DebugLineOffset(pub T); + +/// An offset into the `.debug_line_str` section. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DebugLineStrOffset(pub T); + +/// An offset into either the `.debug_loc` section or the `.debug_loclists` section, +/// depending on the version of the unit the offset was contained in. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct LocationListsOffset(pub T); + +/// An offset to a set of location list offsets in the `.debug_loclists` section. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DebugLocListsBase(pub T); + +/// An index into a set of location list offsets in the `.debug_loclists` section. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DebugLocListsIndex(pub T); + +/// An offset into the `.debug_macinfo` section. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct DebugMacinfoOffset(pub T); + +/// An offset into the `.debug_macro` section. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct DebugMacroOffset(pub T); + +/// An offset into either the `.debug_ranges` section or the `.debug_rnglists` section, +/// depending on the version of the unit the offset was contained in. +/// +/// If this is from a DWARF 4 DWO file, then it must additionally be offset by the +/// value of `DW_AT_GNU_ranges_base`. You can use `Dwarf::ranges_offset_from_raw` to do this. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct RawRangeListsOffset(pub T); + +/// An offset into either the `.debug_ranges` section or the `.debug_rnglists` section, +/// depending on the version of the unit the offset was contained in. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct RangeListsOffset(pub T); + +/// An offset to a set of range list offsets in the `.debug_rnglists` section. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DebugRngListsBase(pub T); + +/// An index into a set of range list offsets in the `.debug_rnglists` section. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DebugRngListsIndex(pub T); + +/// An offset into the `.debug_str` section. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DebugStrOffset(pub T); + +/// An offset to a set of entries in the `.debug_str_offsets` section. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DebugStrOffsetsBase(pub T); + +/// An index into a set of entries in the `.debug_str_offsets` section. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DebugStrOffsetsIndex(pub T); + +/// An offset into the `.debug_types` section. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash)] +pub struct DebugTypesOffset(pub T); + +/// A type signature as used in the `.debug_types` section. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct DebugTypeSignature(pub u64); + +/// An offset into the `.debug_frame` section. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct DebugFrameOffset(pub T); + +impl From for DebugFrameOffset { + #[inline] + fn from(o: T) -> Self { + DebugFrameOffset(o) + } +} + +/// An offset into the `.eh_frame` section. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct EhFrameOffset(pub T); + +impl From for EhFrameOffset { + #[inline] + fn from(o: T) -> Self { + EhFrameOffset(o) + } +} + +/// An offset into the `.debug_info` or `.debug_types` sections. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash)] +pub enum UnitSectionOffset { + /// An offset into the `.debug_info` section. + DebugInfoOffset(DebugInfoOffset), + /// An offset into the `.debug_types` section. + DebugTypesOffset(DebugTypesOffset), +} + +impl From> for UnitSectionOffset { + fn from(offset: DebugInfoOffset) -> Self { + UnitSectionOffset::DebugInfoOffset(offset) + } +} + +impl From> for UnitSectionOffset { + fn from(offset: DebugTypesOffset) -> Self { + UnitSectionOffset::DebugTypesOffset(offset) + } +} + +impl UnitSectionOffset +where + T: Clone, +{ + /// Returns the `DebugInfoOffset` inside, or `None` otherwise. + pub fn as_debug_info_offset(&self) -> Option> { + match self { + UnitSectionOffset::DebugInfoOffset(offset) => Some(offset.clone()), + UnitSectionOffset::DebugTypesOffset(_) => None, + } + } + /// Returns the `DebugTypesOffset` inside, or `None` otherwise. + pub fn as_debug_types_offset(&self) -> Option> { + match self { + UnitSectionOffset::DebugInfoOffset(_) => None, + UnitSectionOffset::DebugTypesOffset(offset) => Some(offset.clone()), + } + } +} + +/// An identifier for a DWARF section. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash)] +pub enum SectionId { + /// The `.debug_abbrev` section. + DebugAbbrev, + /// The `.debug_addr` section. + DebugAddr, + /// The `.debug_aranges` section. + DebugAranges, + /// The `.debug_cu_index` section. + DebugCuIndex, + /// The `.debug_frame` section. + DebugFrame, + /// The `.eh_frame` section. + EhFrame, + /// The `.eh_frame_hdr` section. + EhFrameHdr, + /// The `.debug_info` section. + DebugInfo, + /// The `.debug_line` section. + DebugLine, + /// The `.debug_line_str` section. + DebugLineStr, + /// The `.debug_loc` section. + DebugLoc, + /// The `.debug_loclists` section. + DebugLocLists, + /// The `.debug_macinfo` section. + DebugMacinfo, + /// The `.debug_macro` section. + DebugMacro, + /// The `.debug_pubnames` section. + DebugPubNames, + /// The `.debug_pubtypes` section. + DebugPubTypes, + /// The `.debug_ranges` section. + DebugRanges, + /// The `.debug_rnglists` section. + DebugRngLists, + /// The `.debug_str` section. + DebugStr, + /// The `.debug_str_offsets` section. + DebugStrOffsets, + /// The `.debug_tu_index` section. + DebugTuIndex, + /// The `.debug_types` section. + DebugTypes, +} + +impl SectionId { + /// Returns the ELF section name for this kind. + pub fn name(self) -> &'static str { + match self { + SectionId::DebugAbbrev => ".debug_abbrev", + SectionId::DebugAddr => ".debug_addr", + SectionId::DebugAranges => ".debug_aranges", + SectionId::DebugCuIndex => ".debug_cu_index", + SectionId::DebugFrame => ".debug_frame", + SectionId::EhFrame => ".eh_frame", + SectionId::EhFrameHdr => ".eh_frame_hdr", + SectionId::DebugInfo => ".debug_info", + SectionId::DebugLine => ".debug_line", + SectionId::DebugLineStr => ".debug_line_str", + SectionId::DebugLoc => ".debug_loc", + SectionId::DebugLocLists => ".debug_loclists", + SectionId::DebugMacinfo => ".debug_macinfo", + SectionId::DebugMacro => ".debug_macro", + SectionId::DebugPubNames => ".debug_pubnames", + SectionId::DebugPubTypes => ".debug_pubtypes", + SectionId::DebugRanges => ".debug_ranges", + SectionId::DebugRngLists => ".debug_rnglists", + SectionId::DebugStr => ".debug_str", + SectionId::DebugStrOffsets => ".debug_str_offsets", + SectionId::DebugTuIndex => ".debug_tu_index", + SectionId::DebugTypes => ".debug_types", + } + } + + /// Returns the ELF section name for this kind, when found in a .dwo or .dwp file. + pub fn dwo_name(self) -> Option<&'static str> { + Some(match self { + SectionId::DebugAbbrev => ".debug_abbrev.dwo", + SectionId::DebugCuIndex => ".debug_cu_index", + SectionId::DebugInfo => ".debug_info.dwo", + SectionId::DebugLine => ".debug_line.dwo", + // The debug_loc section can be present in the dwo when using the + // GNU split-dwarf extension to DWARF4. + SectionId::DebugLoc => ".debug_loc.dwo", + SectionId::DebugLocLists => ".debug_loclists.dwo", + SectionId::DebugMacro => ".debug_macro.dwo", + SectionId::DebugRngLists => ".debug_rnglists.dwo", + SectionId::DebugStr => ".debug_str.dwo", + SectionId::DebugStrOffsets => ".debug_str_offsets.dwo", + SectionId::DebugTuIndex => ".debug_tu_index", + SectionId::DebugTypes => ".debug_types.dwo", + _ => return None, + }) + } +} + +/// An optionally-provided implementation-defined compilation unit ID to enable +/// split DWARF and linking a split compilation unit back together. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct DwoId(pub u64); + +/// The "type" of file with DWARF debugging information. This determines, among other things, +/// which files DWARF sections should be loaded from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DwarfFileType { + /// A normal executable or object file. + Main, + /// A .dwo split DWARF file. + Dwo, + // TODO: Supplementary files, .dwps? +} + +impl Default for DwarfFileType { + fn default() -> Self { + DwarfFileType::Main + } +} diff --git a/crux-mir/lib/gimli/src/constants.rs b/crux-mir/lib/gimli/src/constants.rs new file mode 100644 index 000000000..c617f4e2e --- /dev/null +++ b/crux-mir/lib/gimli/src/constants.rs @@ -0,0 +1,1425 @@ +// This file originally from https://github.com/philipc/rust-dwarf/ and +// distributed under either MIT or Apache 2.0 licenses. +// +// Copyright 2016 The rust-dwarf Developers +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Constant definitions. +//! +//! The DWARF spec's `DW_AT_*` type is represented as `struct DwAt(u16)`, +//! `DW_FORM_*` as `DwForm(u16)`, etc. +//! +//! There are also exported const definitions for each constant. + +#![allow(non_upper_case_globals)] +#![allow(missing_docs)] + +use core::fmt; + +// The `dw!` macro turns this: +// +// dw!(DwFoo(u32) { +// DW_FOO_bar = 0, +// DW_FOO_baz = 1, +// DW_FOO_bang = 2, +// }); +// +// into this: +// +// #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +// pub struct DwFoo(pub u32); +// +// pub const DW_FOO_bar: DwFoo = DwFoo(0); +// pub const DW_FOO_baz: DwFoo = DwFoo(1); +// pub const DW_FOO_bang: DwFoo = DwFoo(2); +// +// impl DwFoo { +// pub fn static_string(&self) -> Option<&'static str> { +// ... +// } +// } +// +// impl fmt::Display for DwFoo { +// fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { +// ... +// } +// } +macro_rules! dw { + ($(#[$meta:meta])* $struct_name:ident($struct_type:ty) { $($name:ident = $val:expr),+ $(,)? }) => { + $(#[$meta])* + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] + pub struct $struct_name(pub $struct_type); + + $( + pub const $name: $struct_name = $struct_name($val); + )+ + + impl $struct_name { + pub fn static_string(&self) -> Option<&'static str> { + Some(match *self { + $( + $name => stringify!($name), + )+ + _ => return None, + }) + } + } + + impl fmt::Display for $struct_name { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + if let Some(s) = self.static_string() { + f.pad(s) + } else { + #[cfg(feature = "read")] + { + f.pad(&format!("Unknown {}: {}", stringify!($struct_name), self.0)) + } + #[cfg(not(feature = "read"))] + { + write!(f, "Unknown {}: {}", stringify!($struct_name), self.0) + } + } + } + } + }; +} + +dw!( +/// The section type field in a `.dwp` unit index. +/// +/// This is used for version 5 and later. +/// +/// See Section 7.3.5. +DwSect(u32) { + DW_SECT_INFO = 1, + DW_SECT_ABBREV = 3, + DW_SECT_LINE = 4, + DW_SECT_LOCLISTS = 5, + DW_SECT_STR_OFFSETS = 6, + DW_SECT_MACRO = 7, + DW_SECT_RNGLISTS = 8, +}); + +dw!( +/// The section type field in a `.dwp` unit index with version 2. +DwSectV2(u32) { + DW_SECT_V2_INFO = 1, + DW_SECT_V2_TYPES = 2, + DW_SECT_V2_ABBREV = 3, + DW_SECT_V2_LINE = 4, + DW_SECT_V2_LOC = 5, + DW_SECT_V2_STR_OFFSETS = 6, + DW_SECT_V2_MACINFO = 7, + DW_SECT_V2_MACRO = 8, +}); + +dw!( +/// The unit type field in a unit header. +/// +/// See Section 7.5.1, Table 7.2. +DwUt(u8) { + DW_UT_compile = 0x01, + DW_UT_type = 0x02, + DW_UT_partial = 0x03, + DW_UT_skeleton = 0x04, + DW_UT_split_compile = 0x05, + DW_UT_split_type = 0x06, + DW_UT_lo_user = 0x80, + DW_UT_hi_user = 0xff, +}); + +dw!( +/// The opcode for a call frame instruction. +/// +/// Section 7.24: +/// > Call frame instructions are encoded in one or more bytes. The primary +/// > opcode is encoded in the high order two bits of the first byte (that is, +/// > opcode = byte >> 6). An operand or extended opcode may be encoded in the +/// > low order 6 bits. Additional operands are encoded in subsequent bytes. +DwCfa(u8) { + DW_CFA_advance_loc = 0x01 << 6, + DW_CFA_offset = 0x02 << 6, + DW_CFA_restore = 0x03 << 6, + DW_CFA_nop = 0, + DW_CFA_set_loc = 0x01, + DW_CFA_advance_loc1 = 0x02, + DW_CFA_advance_loc2 = 0x03, + DW_CFA_advance_loc4 = 0x04, + DW_CFA_offset_extended = 0x05, + DW_CFA_restore_extended = 0x06, + DW_CFA_undefined = 0x07, + DW_CFA_same_value = 0x08, + DW_CFA_register = 0x09, + DW_CFA_remember_state = 0x0a, + DW_CFA_restore_state = 0x0b, + DW_CFA_def_cfa = 0x0c, + DW_CFA_def_cfa_register = 0x0d, + DW_CFA_def_cfa_offset = 0x0e, + DW_CFA_def_cfa_expression = 0x0f, + DW_CFA_expression = 0x10, + DW_CFA_offset_extended_sf = 0x11, + DW_CFA_def_cfa_sf = 0x12, + DW_CFA_def_cfa_offset_sf = 0x13, + DW_CFA_val_offset = 0x14, + DW_CFA_val_offset_sf = 0x15, + DW_CFA_val_expression = 0x16, + + DW_CFA_lo_user = 0x1c, + DW_CFA_hi_user = 0x3f, + + DW_CFA_MIPS_advance_loc8 = 0x1d, + DW_CFA_GNU_window_save = 0x2d, + DW_CFA_GNU_args_size = 0x2e, + DW_CFA_GNU_negative_offset_extended = 0x2f, +}); + +dw!( +/// The child determination encodings for DIE attributes. +/// +/// See Section 7.5.3, Table 7.4. +DwChildren(u8) { + DW_CHILDREN_no = 0, + DW_CHILDREN_yes = 1, +}); + +dw!( +/// The tag encodings for DIE attributes. +/// +/// See Section 7.5.3, Table 7.3. +DwTag(u16) { + DW_TAG_null = 0x00, + + DW_TAG_array_type = 0x01, + DW_TAG_class_type = 0x02, + DW_TAG_entry_point = 0x03, + DW_TAG_enumeration_type = 0x04, + DW_TAG_formal_parameter = 0x05, + DW_TAG_imported_declaration = 0x08, + DW_TAG_label = 0x0a, + DW_TAG_lexical_block = 0x0b, + DW_TAG_member = 0x0d, + DW_TAG_pointer_type = 0x0f, + DW_TAG_reference_type = 0x10, + DW_TAG_compile_unit = 0x11, + DW_TAG_string_type = 0x12, + DW_TAG_structure_type = 0x13, + DW_TAG_subroutine_type = 0x15, + DW_TAG_typedef = 0x16, + DW_TAG_union_type = 0x17, + DW_TAG_unspecified_parameters = 0x18, + DW_TAG_variant = 0x19, + DW_TAG_common_block = 0x1a, + DW_TAG_common_inclusion = 0x1b, + DW_TAG_inheritance = 0x1c, + DW_TAG_inlined_subroutine = 0x1d, + DW_TAG_module = 0x1e, + DW_TAG_ptr_to_member_type = 0x1f, + DW_TAG_set_type = 0x20, + DW_TAG_subrange_type = 0x21, + DW_TAG_with_stmt = 0x22, + DW_TAG_access_declaration = 0x23, + DW_TAG_base_type = 0x24, + DW_TAG_catch_block = 0x25, + DW_TAG_const_type = 0x26, + DW_TAG_constant = 0x27, + DW_TAG_enumerator = 0x28, + DW_TAG_file_type = 0x29, + DW_TAG_friend = 0x2a, + DW_TAG_namelist = 0x2b, + DW_TAG_namelist_item = 0x2c, + DW_TAG_packed_type = 0x2d, + DW_TAG_subprogram = 0x2e, + DW_TAG_template_type_parameter = 0x2f, + DW_TAG_template_value_parameter = 0x30, + DW_TAG_thrown_type = 0x31, + DW_TAG_try_block = 0x32, + DW_TAG_variant_part = 0x33, + DW_TAG_variable = 0x34, + DW_TAG_volatile_type = 0x35, + +// DWARF 3. + DW_TAG_dwarf_procedure = 0x36, + DW_TAG_restrict_type = 0x37, + DW_TAG_interface_type = 0x38, + DW_TAG_namespace = 0x39, + DW_TAG_imported_module = 0x3a, + DW_TAG_unspecified_type = 0x3b, + DW_TAG_partial_unit = 0x3c, + DW_TAG_imported_unit = 0x3d, + DW_TAG_condition = 0x3f, + DW_TAG_shared_type = 0x40, + +// DWARF 4. + DW_TAG_type_unit = 0x41, + DW_TAG_rvalue_reference_type = 0x42, + DW_TAG_template_alias = 0x43, + +// DWARF 5. + DW_TAG_coarray_type = 0x44, + DW_TAG_generic_subrange = 0x45, + DW_TAG_dynamic_type = 0x46, + DW_TAG_atomic_type = 0x47, + DW_TAG_call_site = 0x48, + DW_TAG_call_site_parameter = 0x49, + DW_TAG_skeleton_unit = 0x4a, + DW_TAG_immutable_type = 0x4b, + + DW_TAG_lo_user = 0x4080, + DW_TAG_hi_user = 0xffff, + +// SGI/MIPS extensions. + DW_TAG_MIPS_loop = 0x4081, + +// HP extensions. + DW_TAG_HP_array_descriptor = 0x4090, + DW_TAG_HP_Bliss_field = 0x4091, + DW_TAG_HP_Bliss_field_set = 0x4092, + +// GNU extensions. + DW_TAG_format_label = 0x4101, + DW_TAG_function_template = 0x4102, + DW_TAG_class_template = 0x4103, + DW_TAG_GNU_BINCL = 0x4104, + DW_TAG_GNU_EINCL = 0x4105, + DW_TAG_GNU_template_template_param = 0x4106, + DW_TAG_GNU_template_parameter_pack = 0x4107, + DW_TAG_GNU_formal_parameter_pack = 0x4108, + DW_TAG_GNU_call_site = 0x4109, + DW_TAG_GNU_call_site_parameter = 0x410a, + + DW_TAG_APPLE_property = 0x4200, + +// SUN extensions. + DW_TAG_SUN_function_template = 0x4201, + DW_TAG_SUN_class_template = 0x4202, + DW_TAG_SUN_struct_template = 0x4203, + DW_TAG_SUN_union_template = 0x4204, + DW_TAG_SUN_indirect_inheritance = 0x4205, + DW_TAG_SUN_codeflags = 0x4206, + DW_TAG_SUN_memop_info = 0x4207, + DW_TAG_SUN_omp_child_func = 0x4208, + DW_TAG_SUN_rtti_descriptor = 0x4209, + DW_TAG_SUN_dtor_info = 0x420a, + DW_TAG_SUN_dtor = 0x420b, + DW_TAG_SUN_f90_interface = 0x420c, + DW_TAG_SUN_fortran_vax_structure = 0x420d, + +// ALTIUM extensions. + DW_TAG_ALTIUM_circ_type = 0x5101, + DW_TAG_ALTIUM_mwa_circ_type = 0x5102, + DW_TAG_ALTIUM_rev_carry_type = 0x5103, + DW_TAG_ALTIUM_rom = 0x5111, + +// Extensions for UPC. + DW_TAG_upc_shared_type = 0x8765, + DW_TAG_upc_strict_type = 0x8766, + DW_TAG_upc_relaxed_type = 0x8767, + +// PGI (STMicroelectronics) extensions. + DW_TAG_PGI_kanji_type = 0xa000, + DW_TAG_PGI_interface_block = 0xa020, + +// Borland extensions. + DW_TAG_BORLAND_property = 0xb000, + DW_TAG_BORLAND_Delphi_string = 0xb001, + DW_TAG_BORLAND_Delphi_dynamic_array = 0xb002, + DW_TAG_BORLAND_Delphi_set = 0xb003, + DW_TAG_BORLAND_Delphi_variant = 0xb004, +}); + +dw!( +/// The attribute encodings for DIE attributes. +/// +/// See Section 7.5.4, Table 7.5. +DwAt(u16) { + DW_AT_null = 0x00, + + DW_AT_sibling = 0x01, + DW_AT_location = 0x02, + DW_AT_name = 0x03, + DW_AT_ordering = 0x09, + DW_AT_byte_size = 0x0b, + DW_AT_bit_offset = 0x0c, + DW_AT_bit_size = 0x0d, + DW_AT_stmt_list = 0x10, + DW_AT_low_pc = 0x11, + DW_AT_high_pc = 0x12, + DW_AT_language = 0x13, + DW_AT_discr = 0x15, + DW_AT_discr_value = 0x16, + DW_AT_visibility = 0x17, + DW_AT_import = 0x18, + DW_AT_string_length = 0x19, + DW_AT_common_reference = 0x1a, + DW_AT_comp_dir = 0x1b, + DW_AT_const_value = 0x1c, + DW_AT_containing_type = 0x1d, + DW_AT_default_value = 0x1e, + DW_AT_inline = 0x20, + DW_AT_is_optional = 0x21, + DW_AT_lower_bound = 0x22, + DW_AT_producer = 0x25, + DW_AT_prototyped = 0x27, + DW_AT_return_addr = 0x2a, + DW_AT_start_scope = 0x2c, + DW_AT_bit_stride = 0x2e, + DW_AT_upper_bound = 0x2f, + DW_AT_abstract_origin = 0x31, + DW_AT_accessibility = 0x32, + DW_AT_address_class = 0x33, + DW_AT_artificial = 0x34, + DW_AT_base_types = 0x35, + DW_AT_calling_convention = 0x36, + DW_AT_count = 0x37, + DW_AT_data_member_location = 0x38, + DW_AT_decl_column = 0x39, + DW_AT_decl_file = 0x3a, + DW_AT_decl_line = 0x3b, + DW_AT_declaration = 0x3c, + DW_AT_discr_list = 0x3d, + DW_AT_encoding = 0x3e, + DW_AT_external = 0x3f, + DW_AT_frame_base = 0x40, + DW_AT_friend = 0x41, + DW_AT_identifier_case = 0x42, + DW_AT_macro_info = 0x43, + DW_AT_namelist_item = 0x44, + DW_AT_priority = 0x45, + DW_AT_segment = 0x46, + DW_AT_specification = 0x47, + DW_AT_static_link = 0x48, + DW_AT_type = 0x49, + DW_AT_use_location = 0x4a, + DW_AT_variable_parameter = 0x4b, + DW_AT_virtuality = 0x4c, + DW_AT_vtable_elem_location = 0x4d, + +// DWARF 3. + DW_AT_allocated = 0x4e, + DW_AT_associated = 0x4f, + DW_AT_data_location = 0x50, + DW_AT_byte_stride = 0x51, + DW_AT_entry_pc = 0x52, + DW_AT_use_UTF8 = 0x53, + DW_AT_extension = 0x54, + DW_AT_ranges = 0x55, + DW_AT_trampoline = 0x56, + DW_AT_call_column = 0x57, + DW_AT_call_file = 0x58, + DW_AT_call_line = 0x59, + DW_AT_description = 0x5a, + DW_AT_binary_scale = 0x5b, + DW_AT_decimal_scale = 0x5c, + DW_AT_small = 0x5d, + DW_AT_decimal_sign = 0x5e, + DW_AT_digit_count = 0x5f, + DW_AT_picture_string = 0x60, + DW_AT_mutable = 0x61, + DW_AT_threads_scaled = 0x62, + DW_AT_explicit = 0x63, + DW_AT_object_pointer = 0x64, + DW_AT_endianity = 0x65, + DW_AT_elemental = 0x66, + DW_AT_pure = 0x67, + DW_AT_recursive = 0x68, + +// DWARF 4. + DW_AT_signature = 0x69, + DW_AT_main_subprogram = 0x6a, + DW_AT_data_bit_offset = 0x6b, + DW_AT_const_expr = 0x6c, + DW_AT_enum_class = 0x6d, + DW_AT_linkage_name = 0x6e, + +// DWARF 5. + DW_AT_string_length_bit_size = 0x6f, + DW_AT_string_length_byte_size = 0x70, + DW_AT_rank = 0x71, + DW_AT_str_offsets_base = 0x72, + DW_AT_addr_base = 0x73, + DW_AT_rnglists_base = 0x74, + DW_AT_dwo_name = 0x76, + DW_AT_reference = 0x77, + DW_AT_rvalue_reference = 0x78, + DW_AT_macros = 0x79, + DW_AT_call_all_calls = 0x7a, + DW_AT_call_all_source_calls = 0x7b, + DW_AT_call_all_tail_calls = 0x7c, + DW_AT_call_return_pc = 0x7d, + DW_AT_call_value = 0x7e, + DW_AT_call_origin = 0x7f, + DW_AT_call_parameter = 0x80, + DW_AT_call_pc = 0x81, + DW_AT_call_tail_call = 0x82, + DW_AT_call_target = 0x83, + DW_AT_call_target_clobbered = 0x84, + DW_AT_call_data_location = 0x85, + DW_AT_call_data_value = 0x86, + DW_AT_noreturn = 0x87, + DW_AT_alignment = 0x88, + DW_AT_export_symbols = 0x89, + DW_AT_deleted = 0x8a, + DW_AT_defaulted = 0x8b, + DW_AT_loclists_base = 0x8c, + + DW_AT_lo_user = 0x2000, + DW_AT_hi_user = 0x3fff, + +// SGI/MIPS extensions. + DW_AT_MIPS_fde = 0x2001, + DW_AT_MIPS_loop_begin = 0x2002, + DW_AT_MIPS_tail_loop_begin = 0x2003, + DW_AT_MIPS_epilog_begin = 0x2004, + DW_AT_MIPS_loop_unroll_factor = 0x2005, + DW_AT_MIPS_software_pipeline_depth = 0x2006, + DW_AT_MIPS_linkage_name = 0x2007, + DW_AT_MIPS_stride = 0x2008, + DW_AT_MIPS_abstract_name = 0x2009, + DW_AT_MIPS_clone_origin = 0x200a, + DW_AT_MIPS_has_inlines = 0x200b, + DW_AT_MIPS_stride_byte = 0x200c, + DW_AT_MIPS_stride_elem = 0x200d, + DW_AT_MIPS_ptr_dopetype = 0x200e, + DW_AT_MIPS_allocatable_dopetype = 0x200f, + DW_AT_MIPS_assumed_shape_dopetype = 0x2010, + +// This one appears to have only been implemented by Open64 for +// fortran and may conflict with other extensions. + DW_AT_MIPS_assumed_size = 0x2011, + +// TODO: HP/CPQ extensions. +// These conflict with the MIPS extensions. + + DW_AT_INTEL_other_endian = 0x2026, + +// GNU extensions + DW_AT_sf_names = 0x2101, + DW_AT_src_info = 0x2102, + DW_AT_mac_info = 0x2103, + DW_AT_src_coords = 0x2104, + DW_AT_body_begin = 0x2105, + DW_AT_body_end = 0x2106, + DW_AT_GNU_vector = 0x2107, + DW_AT_GNU_guarded_by = 0x2108, + DW_AT_GNU_pt_guarded_by = 0x2109, + DW_AT_GNU_guarded = 0x210a, + DW_AT_GNU_pt_guarded = 0x210b, + DW_AT_GNU_locks_excluded = 0x210c, + DW_AT_GNU_exclusive_locks_required = 0x210d, + DW_AT_GNU_shared_locks_required = 0x210e, + DW_AT_GNU_odr_signature = 0x210f, + DW_AT_GNU_template_name = 0x2110, + DW_AT_GNU_call_site_value = 0x2111, + DW_AT_GNU_call_site_data_value = 0x2112, + DW_AT_GNU_call_site_target = 0x2113, + DW_AT_GNU_call_site_target_clobbered = 0x2114, + DW_AT_GNU_tail_call = 0x2115, + DW_AT_GNU_all_tail_call_sites = 0x2116, + DW_AT_GNU_all_call_sites = 0x2117, + DW_AT_GNU_all_source_call_sites = 0x2118, + DW_AT_GNU_macros = 0x2119, + +// Extensions for Fission proposal. + DW_AT_GNU_dwo_name = 0x2130, + DW_AT_GNU_dwo_id = 0x2131, + DW_AT_GNU_ranges_base = 0x2132, + DW_AT_GNU_addr_base = 0x2133, + DW_AT_GNU_pubnames = 0x2134, + DW_AT_GNU_pubtypes = 0x2135, + DW_AT_GNU_discriminator = 0x2136, + DW_AT_GNU_locviews = 0x2137, + DW_AT_GNU_entry_view = 0x2138, + +// Conflict with Sun. +// DW_AT_VMS_rtnbeg_pd_address = 0x2201, + +// Sun extensions. + DW_AT_SUN_template = 0x2201, + DW_AT_SUN_alignment = 0x2202, + DW_AT_SUN_vtable = 0x2203, + DW_AT_SUN_count_guarantee = 0x2204, + DW_AT_SUN_command_line = 0x2205, + DW_AT_SUN_vbase = 0x2206, + DW_AT_SUN_compile_options = 0x2207, + DW_AT_SUN_language = 0x2208, + DW_AT_SUN_browser_file = 0x2209, + DW_AT_SUN_vtable_abi = 0x2210, + DW_AT_SUN_func_offsets = 0x2211, + DW_AT_SUN_cf_kind = 0x2212, + DW_AT_SUN_vtable_index = 0x2213, + DW_AT_SUN_omp_tpriv_addr = 0x2214, + DW_AT_SUN_omp_child_func = 0x2215, + DW_AT_SUN_func_offset = 0x2216, + DW_AT_SUN_memop_type_ref = 0x2217, + DW_AT_SUN_profile_id = 0x2218, + DW_AT_SUN_memop_signature = 0x2219, + DW_AT_SUN_obj_dir = 0x2220, + DW_AT_SUN_obj_file = 0x2221, + DW_AT_SUN_original_name = 0x2222, + DW_AT_SUN_hwcprof_signature = 0x2223, + DW_AT_SUN_amd64_parmdump = 0x2224, + DW_AT_SUN_part_link_name = 0x2225, + DW_AT_SUN_link_name = 0x2226, + DW_AT_SUN_pass_with_const = 0x2227, + DW_AT_SUN_return_with_const = 0x2228, + DW_AT_SUN_import_by_name = 0x2229, + DW_AT_SUN_f90_pointer = 0x222a, + DW_AT_SUN_pass_by_ref = 0x222b, + DW_AT_SUN_f90_allocatable = 0x222c, + DW_AT_SUN_f90_assumed_shape_array = 0x222d, + DW_AT_SUN_c_vla = 0x222e, + DW_AT_SUN_return_value_ptr = 0x2230, + DW_AT_SUN_dtor_start = 0x2231, + DW_AT_SUN_dtor_length = 0x2232, + DW_AT_SUN_dtor_state_initial = 0x2233, + DW_AT_SUN_dtor_state_final = 0x2234, + DW_AT_SUN_dtor_state_deltas = 0x2235, + DW_AT_SUN_import_by_lname = 0x2236, + DW_AT_SUN_f90_use_only = 0x2237, + DW_AT_SUN_namelist_spec = 0x2238, + DW_AT_SUN_is_omp_child_func = 0x2239, + DW_AT_SUN_fortran_main_alias = 0x223a, + DW_AT_SUN_fortran_based = 0x223b, + + DW_AT_ALTIUM_loclist = 0x2300, + + DW_AT_use_GNAT_descriptive_type = 0x2301, + DW_AT_GNAT_descriptive_type = 0x2302, + DW_AT_GNU_numerator = 0x2303, + DW_AT_GNU_denominator = 0x2304, + DW_AT_GNU_bias = 0x2305, + + DW_AT_upc_threads_scaled = 0x3210, + +// PGI (STMicroelectronics) extensions. + DW_AT_PGI_lbase = 0x3a00, + DW_AT_PGI_soffset = 0x3a01, + DW_AT_PGI_lstride = 0x3a02, + +// Borland extensions. + DW_AT_BORLAND_property_read = 0x3b11, + DW_AT_BORLAND_property_write = 0x3b12, + DW_AT_BORLAND_property_implements = 0x3b13, + DW_AT_BORLAND_property_index = 0x3b14, + DW_AT_BORLAND_property_default = 0x3b15, + DW_AT_BORLAND_Delphi_unit = 0x3b20, + DW_AT_BORLAND_Delphi_class = 0x3b21, + DW_AT_BORLAND_Delphi_record = 0x3b22, + DW_AT_BORLAND_Delphi_metaclass = 0x3b23, + DW_AT_BORLAND_Delphi_constructor = 0x3b24, + DW_AT_BORLAND_Delphi_destructor = 0x3b25, + DW_AT_BORLAND_Delphi_anonymous_method = 0x3b26, + DW_AT_BORLAND_Delphi_interface = 0x3b27, + DW_AT_BORLAND_Delphi_ABI = 0x3b28, + DW_AT_BORLAND_Delphi_return = 0x3b29, + DW_AT_BORLAND_Delphi_frameptr = 0x3b30, + DW_AT_BORLAND_closure = 0x3b31, + +// LLVM project extensions. + DW_AT_LLVM_include_path = 0x3e00, + DW_AT_LLVM_config_macros = 0x3e01, + DW_AT_LLVM_isysroot = 0x3e02, + +// Apple extensions. + DW_AT_APPLE_optimized = 0x3fe1, + DW_AT_APPLE_flags = 0x3fe2, + DW_AT_APPLE_isa = 0x3fe3, + DW_AT_APPLE_block = 0x3fe4, + DW_AT_APPLE_major_runtime_vers = 0x3fe5, + DW_AT_APPLE_runtime_class = 0x3fe6, + DW_AT_APPLE_omit_frame_ptr = 0x3fe7, + DW_AT_APPLE_property_name = 0x3fe8, + DW_AT_APPLE_property_getter = 0x3fe9, + DW_AT_APPLE_property_setter = 0x3fea, + DW_AT_APPLE_property_attribute = 0x3feb, + DW_AT_APPLE_objc_complete_type = 0x3fec, + DW_AT_APPLE_property = 0x3fed +}); + +dw!( +/// The attribute form encodings for DIE attributes. +/// +/// See Section 7.5.6, Table 7.6. +DwForm(u16) { + DW_FORM_null = 0x00, + + DW_FORM_addr = 0x01, + DW_FORM_block2 = 0x03, + DW_FORM_block4 = 0x04, + DW_FORM_data2 = 0x05, + DW_FORM_data4 = 0x06, + DW_FORM_data8 = 0x07, + DW_FORM_string = 0x08, + DW_FORM_block = 0x09, + DW_FORM_block1 = 0x0a, + DW_FORM_data1 = 0x0b, + DW_FORM_flag = 0x0c, + DW_FORM_sdata = 0x0d, + DW_FORM_strp = 0x0e, + DW_FORM_udata = 0x0f, + DW_FORM_ref_addr = 0x10, + DW_FORM_ref1 = 0x11, + DW_FORM_ref2 = 0x12, + DW_FORM_ref4 = 0x13, + DW_FORM_ref8 = 0x14, + DW_FORM_ref_udata = 0x15, + DW_FORM_indirect = 0x16, + +// DWARF 4. + DW_FORM_sec_offset = 0x17, + DW_FORM_exprloc = 0x18, + DW_FORM_flag_present = 0x19, + DW_FORM_ref_sig8 = 0x20, + +// DWARF 5. + DW_FORM_strx = 0x1a, + DW_FORM_addrx = 0x1b, + DW_FORM_ref_sup4 = 0x1c, + DW_FORM_strp_sup = 0x1d, + DW_FORM_data16 = 0x1e, + DW_FORM_line_strp = 0x1f, + DW_FORM_implicit_const = 0x21, + DW_FORM_loclistx = 0x22, + DW_FORM_rnglistx = 0x23, + DW_FORM_ref_sup8 = 0x24, + DW_FORM_strx1 = 0x25, + DW_FORM_strx2 = 0x26, + DW_FORM_strx3 = 0x27, + DW_FORM_strx4 = 0x28, + DW_FORM_addrx1 = 0x29, + DW_FORM_addrx2 = 0x2a, + DW_FORM_addrx3 = 0x2b, + DW_FORM_addrx4 = 0x2c, + +// Extensions for Fission proposal + DW_FORM_GNU_addr_index = 0x1f01, + DW_FORM_GNU_str_index = 0x1f02, + +// Alternate debug sections proposal (output of "dwz" tool). + DW_FORM_GNU_ref_alt = 0x1f20, + DW_FORM_GNU_strp_alt = 0x1f21 +}); + +dw!( +/// The encodings of the constants used in the `DW_AT_encoding` attribute. +/// +/// See Section 7.8, Table 7.11. +DwAte(u8) { + DW_ATE_address = 0x01, + DW_ATE_boolean = 0x02, + DW_ATE_complex_float = 0x03, + DW_ATE_float = 0x04, + DW_ATE_signed = 0x05, + DW_ATE_signed_char = 0x06, + DW_ATE_unsigned = 0x07, + DW_ATE_unsigned_char = 0x08, + +// DWARF 3. + DW_ATE_imaginary_float = 0x09, + DW_ATE_packed_decimal = 0x0a, + DW_ATE_numeric_string = 0x0b, + DW_ATE_edited = 0x0c, + DW_ATE_signed_fixed = 0x0d, + DW_ATE_unsigned_fixed = 0x0e, + DW_ATE_decimal_float = 0x0f , + +// DWARF 4. + DW_ATE_UTF = 0x10, + DW_ATE_UCS = 0x11, + DW_ATE_ASCII = 0x12, + + DW_ATE_lo_user = 0x80, + DW_ATE_hi_user = 0xff, +}); + +dw!( +/// The encodings of the constants used in location list entries. +/// +/// See Section 7.7.3, Table 7.10. +DwLle(u8) { + DW_LLE_end_of_list = 0x00, + DW_LLE_base_addressx = 0x01, + DW_LLE_startx_endx = 0x02, + DW_LLE_startx_length = 0x03, + DW_LLE_offset_pair = 0x04, + DW_LLE_default_location = 0x05, + DW_LLE_base_address = 0x06, + DW_LLE_start_end = 0x07, + DW_LLE_start_length = 0x08, + DW_LLE_GNU_view_pair = 0x09, +}); + +dw!( +/// The encodings of the constants used in the `DW_AT_decimal_sign` attribute. +/// +/// See Section 7.8, Table 7.12. +DwDs(u8) { + DW_DS_unsigned = 0x01, + DW_DS_leading_overpunch = 0x02, + DW_DS_trailing_overpunch = 0x03, + DW_DS_leading_separate = 0x04, + DW_DS_trailing_separate = 0x05, +}); + +dw!( +/// The encodings of the constants used in the `DW_AT_endianity` attribute. +/// +/// See Section 7.8, Table 7.13. +DwEnd(u8) { + DW_END_default = 0x00, + DW_END_big = 0x01, + DW_END_little = 0x02, + DW_END_lo_user = 0x40, + DW_END_hi_user = 0xff, +}); + +dw!( +/// The encodings of the constants used in the `DW_AT_accessibility` attribute. +/// +/// See Section 7.9, Table 7.14. +DwAccess(u8) { + DW_ACCESS_public = 0x01, + DW_ACCESS_protected = 0x02, + DW_ACCESS_private = 0x03, +}); + +dw!( +/// The encodings of the constants used in the `DW_AT_visibility` attribute. +/// +/// See Section 7.10, Table 7.15. +DwVis(u8) { + DW_VIS_local = 0x01, + DW_VIS_exported = 0x02, + DW_VIS_qualified = 0x03, +}); + +dw!( +/// The encodings of the constants used in the `DW_AT_virtuality` attribute. +/// +/// See Section 7.11, Table 7.16. +DwVirtuality(u8) { + DW_VIRTUALITY_none = 0x00, + DW_VIRTUALITY_virtual = 0x01, + DW_VIRTUALITY_pure_virtual = 0x02, +}); + +dw!( +/// The encodings of the constants used in the `DW_AT_language` attribute. +/// +/// See Section 7.12, Table 7.17. +DwLang(u16) { + DW_LANG_C89 = 0x0001, + DW_LANG_C = 0x0002, + DW_LANG_Ada83 = 0x0003, + DW_LANG_C_plus_plus = 0x0004, + DW_LANG_Cobol74 = 0x0005, + DW_LANG_Cobol85 = 0x0006, + DW_LANG_Fortran77 = 0x0007, + DW_LANG_Fortran90 = 0x0008, + DW_LANG_Pascal83 = 0x0009, + DW_LANG_Modula2 = 0x000a, + DW_LANG_Java = 0x000b, + DW_LANG_C99 = 0x000c, + DW_LANG_Ada95 = 0x000d, + DW_LANG_Fortran95 = 0x000e, + DW_LANG_PLI = 0x000f, + DW_LANG_ObjC = 0x0010, + DW_LANG_ObjC_plus_plus = 0x0011, + DW_LANG_UPC = 0x0012, + DW_LANG_D = 0x0013, + DW_LANG_Python = 0x0014, + DW_LANG_OpenCL = 0x0015, + DW_LANG_Go = 0x0016, + DW_LANG_Modula3 = 0x0017, + DW_LANG_Haskell = 0x0018, + DW_LANG_C_plus_plus_03 = 0x0019, + DW_LANG_C_plus_plus_11 = 0x001a, + DW_LANG_OCaml = 0x001b, + DW_LANG_Rust = 0x001c, + DW_LANG_C11 = 0x001d, + DW_LANG_Swift = 0x001e, + DW_LANG_Julia = 0x001f, + DW_LANG_Dylan = 0x0020, + DW_LANG_C_plus_plus_14 = 0x0021, + DW_LANG_Fortran03 = 0x0022, + DW_LANG_Fortran08 = 0x0023, + DW_LANG_RenderScript = 0x0024, + DW_LANG_BLISS = 0x0025, + DW_LANG_Kotlin = 0x0026, + DW_LANG_Zig = 0x0027, + DW_LANG_Crystal = 0x0028, + DW_LANG_C_plus_plus_17 = 0x002a, + DW_LANG_C_plus_plus_20 = 0x002b, + DW_LANG_C17 = 0x002c, + DW_LANG_Fortran18 = 0x002d, + DW_LANG_Ada2005 = 0x002e, + DW_LANG_Ada2012 = 0x002f, + + DW_LANG_lo_user = 0x8000, + DW_LANG_hi_user = 0xffff, + + DW_LANG_Mips_Assembler = 0x8001, + DW_LANG_GOOGLE_RenderScript = 0x8e57, + DW_LANG_SUN_Assembler = 0x9001, + DW_LANG_ALTIUM_Assembler = 0x9101, + DW_LANG_BORLAND_Delphi = 0xb000, +}); + +impl DwLang { + /// Get the default DW_AT_lower_bound for this language. + pub fn default_lower_bound(self) -> Option { + match self { + DW_LANG_C89 + | DW_LANG_C + | DW_LANG_C_plus_plus + | DW_LANG_Java + | DW_LANG_C99 + | DW_LANG_ObjC + | DW_LANG_ObjC_plus_plus + | DW_LANG_UPC + | DW_LANG_D + | DW_LANG_Python + | DW_LANG_OpenCL + | DW_LANG_Go + | DW_LANG_Haskell + | DW_LANG_C_plus_plus_03 + | DW_LANG_C_plus_plus_11 + | DW_LANG_OCaml + | DW_LANG_Rust + | DW_LANG_C11 + | DW_LANG_Swift + | DW_LANG_Dylan + | DW_LANG_C_plus_plus_14 + | DW_LANG_RenderScript + | DW_LANG_BLISS => Some(0), + DW_LANG_Ada83 | DW_LANG_Cobol74 | DW_LANG_Cobol85 | DW_LANG_Fortran77 + | DW_LANG_Fortran90 | DW_LANG_Pascal83 | DW_LANG_Modula2 | DW_LANG_Ada95 + | DW_LANG_Fortran95 | DW_LANG_PLI | DW_LANG_Modula3 | DW_LANG_Julia + | DW_LANG_Fortran03 | DW_LANG_Fortran08 => Some(1), + _ => None, + } + } +} + +dw!( +/// The encodings of the constants used in the `DW_AT_address_class` attribute. +/// +/// There is only one value that is common to all target architectures. +/// See Section 7.13. +DwAddr(u64) { + DW_ADDR_none = 0x00, +}); + +dw!( +/// The encodings of the constants used in the `DW_AT_identifier_case` attribute. +/// +/// See Section 7.14, Table 7.18. +DwId(u8) { + DW_ID_case_sensitive = 0x00, + DW_ID_up_case = 0x01, + DW_ID_down_case = 0x02, + DW_ID_case_insensitive = 0x03, +}); + +dw!( +/// The encodings of the constants used in the `DW_AT_calling_convention` attribute. +/// +/// See Section 7.15, Table 7.19. +DwCc(u8) { + DW_CC_normal = 0x01, + DW_CC_program = 0x02, + DW_CC_nocall = 0x03, + DW_CC_pass_by_reference = 0x04, + DW_CC_pass_by_value = 0x05, + DW_CC_lo_user = 0x40, + DW_CC_hi_user = 0xff, +}); + +dw!( +/// The encodings of the constants used in the `DW_AT_inline` attribute. +/// +/// See Section 7.16, Table 7.20. +DwInl(u8) { + DW_INL_not_inlined = 0x00, + DW_INL_inlined = 0x01, + DW_INL_declared_not_inlined = 0x02, + DW_INL_declared_inlined = 0x03, +}); + +dw!( +/// The encodings of the constants used in the `DW_AT_ordering` attribute. +/// +/// See Section 7.17, Table 7.17. +DwOrd(u8) { + DW_ORD_row_major = 0x00, + DW_ORD_col_major = 0x01, +}); + +dw!( +/// The encodings of the constants used in the `DW_AT_discr_list` attribute. +/// +/// See Section 7.18, Table 7.22. +DwDsc(u8) { + DW_DSC_label = 0x00, + DW_DSC_range = 0x01, +}); + +dw!( +/// Name index attribute encodings. +/// +/// See Section 7.19, Table 7.23. +DwIdx(u16) { + DW_IDX_compile_unit = 1, + DW_IDX_type_unit = 2, + DW_IDX_die_offset = 3, + DW_IDX_parent = 4, + DW_IDX_type_hash = 5, + DW_IDX_lo_user = 0x2000, + DW_IDX_hi_user = 0x3fff, +}); + +dw!( +/// The encodings of the constants used in the `DW_AT_defaulted` attribute. +/// +/// See Section 7.20, Table 7.24. +DwDefaulted(u8) { + DW_DEFAULTED_no = 0x00, + DW_DEFAULTED_in_class = 0x01, + DW_DEFAULTED_out_of_class = 0x02, +}); + +dw!( +/// The encodings for the standard opcodes for line number information. +/// +/// See Section 7.22, Table 7.25. +DwLns(u8) { + DW_LNS_copy = 0x01, + DW_LNS_advance_pc = 0x02, + DW_LNS_advance_line = 0x03, + DW_LNS_set_file = 0x04, + DW_LNS_set_column = 0x05, + DW_LNS_negate_stmt = 0x06, + DW_LNS_set_basic_block = 0x07, + DW_LNS_const_add_pc = 0x08, + DW_LNS_fixed_advance_pc = 0x09, + DW_LNS_set_prologue_end = 0x0a, + DW_LNS_set_epilogue_begin = 0x0b, + DW_LNS_set_isa = 0x0c, +}); + +dw!( +/// The encodings for the extended opcodes for line number information. +/// +/// See Section 7.22, Table 7.26. +DwLne(u8) { + DW_LNE_end_sequence = 0x01, + DW_LNE_set_address = 0x02, + DW_LNE_define_file = 0x03, + DW_LNE_set_discriminator = 0x04, + + DW_LNE_lo_user = 0x80, + DW_LNE_hi_user = 0xff, +}); + +dw!( +/// The encodings for the line number header entry formats. +/// +/// See Section 7.22, Table 7.27. +DwLnct(u16) { + DW_LNCT_path = 0x1, + DW_LNCT_directory_index = 0x2, + DW_LNCT_timestamp = 0x3, + DW_LNCT_size = 0x4, + DW_LNCT_MD5 = 0x5, + DW_LNCT_lo_user = 0x2000, + DW_LNCT_hi_user = 0x3fff, +}); + +dw!( +/// The encodings for macro information entry types. +/// +/// See Section 7.23, Table 7.28. +DwMacro(u8) { + DW_MACRO_define = 0x01, + DW_MACRO_undef = 0x02, + DW_MACRO_start_file = 0x03, + DW_MACRO_end_file = 0x04, + DW_MACRO_define_strp = 0x05, + DW_MACRO_undef_strp = 0x06, + DW_MACRO_import = 0x07, + DW_MACRO_define_sup = 0x08, + DW_MACRO_undef_sup = 0x09, + DW_MACRO_import_sup = 0x0a, + DW_MACRO_define_strx = 0x0b, + DW_MACRO_undef_strx = 0x0c, + DW_MACRO_lo_user = 0xe0, + DW_MACRO_hi_user = 0xff, +}); + +dw!( +/// Range list entry encoding values. +/// +/// See Section 7.25, Table 7.30. +DwRle(u8) { + DW_RLE_end_of_list = 0x00, + DW_RLE_base_addressx = 0x01, + DW_RLE_startx_endx = 0x02, + DW_RLE_startx_length = 0x03, + DW_RLE_offset_pair = 0x04, + DW_RLE_base_address = 0x05, + DW_RLE_start_end = 0x06, + DW_RLE_start_length = 0x07, +}); + +dw!( +/// The encodings for DWARF expression operations. +/// +/// See Section 7.7.1, Table 7.9. +DwOp(u8) { + DW_OP_addr = 0x03, + DW_OP_deref = 0x06, + DW_OP_const1u = 0x08, + DW_OP_const1s = 0x09, + DW_OP_const2u = 0x0a, + DW_OP_const2s = 0x0b, + DW_OP_const4u = 0x0c, + DW_OP_const4s = 0x0d, + DW_OP_const8u = 0x0e, + DW_OP_const8s = 0x0f, + DW_OP_constu = 0x10, + DW_OP_consts = 0x11, + DW_OP_dup = 0x12, + DW_OP_drop = 0x13, + DW_OP_over = 0x14, + DW_OP_pick = 0x15, + DW_OP_swap = 0x16, + DW_OP_rot = 0x17, + DW_OP_xderef = 0x18, + DW_OP_abs = 0x19, + DW_OP_and = 0x1a, + DW_OP_div = 0x1b, + DW_OP_minus = 0x1c, + DW_OP_mod = 0x1d, + DW_OP_mul = 0x1e, + DW_OP_neg = 0x1f, + DW_OP_not = 0x20, + DW_OP_or = 0x21, + DW_OP_plus = 0x22, + DW_OP_plus_uconst = 0x23, + DW_OP_shl = 0x24, + DW_OP_shr = 0x25, + DW_OP_shra = 0x26, + DW_OP_xor = 0x27, + DW_OP_bra = 0x28, + DW_OP_eq = 0x29, + DW_OP_ge = 0x2a, + DW_OP_gt = 0x2b, + DW_OP_le = 0x2c, + DW_OP_lt = 0x2d, + DW_OP_ne = 0x2e, + DW_OP_skip = 0x2f, + DW_OP_lit0 = 0x30, + DW_OP_lit1 = 0x31, + DW_OP_lit2 = 0x32, + DW_OP_lit3 = 0x33, + DW_OP_lit4 = 0x34, + DW_OP_lit5 = 0x35, + DW_OP_lit6 = 0x36, + DW_OP_lit7 = 0x37, + DW_OP_lit8 = 0x38, + DW_OP_lit9 = 0x39, + DW_OP_lit10 = 0x3a, + DW_OP_lit11 = 0x3b, + DW_OP_lit12 = 0x3c, + DW_OP_lit13 = 0x3d, + DW_OP_lit14 = 0x3e, + DW_OP_lit15 = 0x3f, + DW_OP_lit16 = 0x40, + DW_OP_lit17 = 0x41, + DW_OP_lit18 = 0x42, + DW_OP_lit19 = 0x43, + DW_OP_lit20 = 0x44, + DW_OP_lit21 = 0x45, + DW_OP_lit22 = 0x46, + DW_OP_lit23 = 0x47, + DW_OP_lit24 = 0x48, + DW_OP_lit25 = 0x49, + DW_OP_lit26 = 0x4a, + DW_OP_lit27 = 0x4b, + DW_OP_lit28 = 0x4c, + DW_OP_lit29 = 0x4d, + DW_OP_lit30 = 0x4e, + DW_OP_lit31 = 0x4f, + DW_OP_reg0 = 0x50, + DW_OP_reg1 = 0x51, + DW_OP_reg2 = 0x52, + DW_OP_reg3 = 0x53, + DW_OP_reg4 = 0x54, + DW_OP_reg5 = 0x55, + DW_OP_reg6 = 0x56, + DW_OP_reg7 = 0x57, + DW_OP_reg8 = 0x58, + DW_OP_reg9 = 0x59, + DW_OP_reg10 = 0x5a, + DW_OP_reg11 = 0x5b, + DW_OP_reg12 = 0x5c, + DW_OP_reg13 = 0x5d, + DW_OP_reg14 = 0x5e, + DW_OP_reg15 = 0x5f, + DW_OP_reg16 = 0x60, + DW_OP_reg17 = 0x61, + DW_OP_reg18 = 0x62, + DW_OP_reg19 = 0x63, + DW_OP_reg20 = 0x64, + DW_OP_reg21 = 0x65, + DW_OP_reg22 = 0x66, + DW_OP_reg23 = 0x67, + DW_OP_reg24 = 0x68, + DW_OP_reg25 = 0x69, + DW_OP_reg26 = 0x6a, + DW_OP_reg27 = 0x6b, + DW_OP_reg28 = 0x6c, + DW_OP_reg29 = 0x6d, + DW_OP_reg30 = 0x6e, + DW_OP_reg31 = 0x6f, + DW_OP_breg0 = 0x70, + DW_OP_breg1 = 0x71, + DW_OP_breg2 = 0x72, + DW_OP_breg3 = 0x73, + DW_OP_breg4 = 0x74, + DW_OP_breg5 = 0x75, + DW_OP_breg6 = 0x76, + DW_OP_breg7 = 0x77, + DW_OP_breg8 = 0x78, + DW_OP_breg9 = 0x79, + DW_OP_breg10 = 0x7a, + DW_OP_breg11 = 0x7b, + DW_OP_breg12 = 0x7c, + DW_OP_breg13 = 0x7d, + DW_OP_breg14 = 0x7e, + DW_OP_breg15 = 0x7f, + DW_OP_breg16 = 0x80, + DW_OP_breg17 = 0x81, + DW_OP_breg18 = 0x82, + DW_OP_breg19 = 0x83, + DW_OP_breg20 = 0x84, + DW_OP_breg21 = 0x85, + DW_OP_breg22 = 0x86, + DW_OP_breg23 = 0x87, + DW_OP_breg24 = 0x88, + DW_OP_breg25 = 0x89, + DW_OP_breg26 = 0x8a, + DW_OP_breg27 = 0x8b, + DW_OP_breg28 = 0x8c, + DW_OP_breg29 = 0x8d, + DW_OP_breg30 = 0x8e, + DW_OP_breg31 = 0x8f, + DW_OP_regx = 0x90, + DW_OP_fbreg = 0x91, + DW_OP_bregx = 0x92, + DW_OP_piece = 0x93, + DW_OP_deref_size = 0x94, + DW_OP_xderef_size = 0x95, + DW_OP_nop = 0x96, + DW_OP_push_object_address = 0x97, + DW_OP_call2 = 0x98, + DW_OP_call4 = 0x99, + DW_OP_call_ref = 0x9a, + DW_OP_form_tls_address = 0x9b, + DW_OP_call_frame_cfa = 0x9c, + DW_OP_bit_piece = 0x9d, + DW_OP_implicit_value = 0x9e, + DW_OP_stack_value = 0x9f, + DW_OP_implicit_pointer = 0xa0, + DW_OP_addrx = 0xa1, + DW_OP_constx = 0xa2, + DW_OP_entry_value = 0xa3, + DW_OP_const_type = 0xa4, + DW_OP_regval_type = 0xa5, + DW_OP_deref_type = 0xa6, + DW_OP_xderef_type = 0xa7, + DW_OP_convert = 0xa8, + DW_OP_reinterpret = 0xa9, + + // GNU extensions + DW_OP_GNU_push_tls_address = 0xe0, + DW_OP_GNU_implicit_pointer = 0xf2, + DW_OP_GNU_entry_value = 0xf3, + DW_OP_GNU_const_type = 0xf4, + DW_OP_GNU_regval_type = 0xf5, + DW_OP_GNU_deref_type = 0xf6, + DW_OP_GNU_convert = 0xf7, + DW_OP_GNU_reinterpret = 0xf9, + DW_OP_GNU_parameter_ref = 0xfa, + DW_OP_GNU_addr_index = 0xfb, + DW_OP_GNU_const_index = 0xfc, + + // Wasm extensions + DW_OP_WASM_location = 0xed, +}); + +dw!( +/// Pointer encoding used by `.eh_frame`. +/// +/// The four lower bits describe the +/// format of the pointer, the upper four bits describe how the encoding should +/// be applied. +/// +/// Defined in https://refspecs.linuxfoundation.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/dwarfext.html +DwEhPe(u8) { +// Format of pointer encoding. + +// "Unsigned value is encoded using the Little Endian Base 128" + DW_EH_PE_uleb128 = 0x1, +// "A 2 bytes unsigned value." + DW_EH_PE_udata2 = 0x2, +// "A 4 bytes unsigned value." + DW_EH_PE_udata4 = 0x3, +// "An 8 bytes unsigned value." + DW_EH_PE_udata8 = 0x4, +// "Signed value is encoded using the Little Endian Base 128" + DW_EH_PE_sleb128 = 0x9, +// "A 2 bytes signed value." + DW_EH_PE_sdata2 = 0x0a, +// "A 4 bytes signed value." + DW_EH_PE_sdata4 = 0x0b, +// "An 8 bytes signed value." + DW_EH_PE_sdata8 = 0x0c, + +// How the pointer encoding should be applied. + +// `DW_EH_PE_pcrel` pointers are relative to their own location. + DW_EH_PE_pcrel = 0x10, +// "Value is relative to the beginning of the .text section." + DW_EH_PE_textrel = 0x20, +// "Value is relative to the beginning of the .got or .eh_frame_hdr section." + DW_EH_PE_datarel = 0x30, +// "Value is relative to the beginning of the function." + DW_EH_PE_funcrel = 0x40, +// "Value is aligned to an address unit sized boundary." + DW_EH_PE_aligned = 0x50, + +// This bit can be set for any of the above encoding applications. When set, +// the encoded value is the address of the real pointer result, not the +// pointer result itself. +// +// This isn't defined in the DWARF or the `.eh_frame` standards, but is +// generated by both GNU/Linux and macOS tooling. + DW_EH_PE_indirect = 0x80, + +// These constants apply to both the lower and upper bits. + +// "The Value is a literal pointer whose size is determined by the +// architecture." + DW_EH_PE_absptr = 0x0, +// The absence of a pointer and encoding. + DW_EH_PE_omit = 0xff, +}); + +const DW_EH_PE_FORMAT_MASK: u8 = 0b0000_1111; + +// Ignores indirection bit. +const DW_EH_PE_APPLICATION_MASK: u8 = 0b0111_0000; + +impl DwEhPe { + /// Get the pointer encoding's format. + #[inline] + pub fn format(self) -> DwEhPe { + DwEhPe(self.0 & DW_EH_PE_FORMAT_MASK) + } + + /// Get the pointer encoding's application. + #[inline] + pub fn application(self) -> DwEhPe { + DwEhPe(self.0 & DW_EH_PE_APPLICATION_MASK) + } + + /// Is this encoding the absent pointer encoding? + #[inline] + pub fn is_absent(self) -> bool { + self == DW_EH_PE_omit + } + + /// Is this coding indirect? If so, its encoded value is the address of the + /// real pointer result, not the pointer result itself. + #[inline] + pub fn is_indirect(self) -> bool { + self.0 & DW_EH_PE_indirect.0 != 0 + } + + /// Is this a known, valid pointer encoding? + pub fn is_valid_encoding(self) -> bool { + if self.is_absent() { + return true; + } + + match self.format() { + DW_EH_PE_absptr | DW_EH_PE_uleb128 | DW_EH_PE_udata2 | DW_EH_PE_udata4 + | DW_EH_PE_udata8 | DW_EH_PE_sleb128 | DW_EH_PE_sdata2 | DW_EH_PE_sdata4 + | DW_EH_PE_sdata8 => {} + _ => return false, + } + + match self.application() { + DW_EH_PE_absptr | DW_EH_PE_pcrel | DW_EH_PE_textrel | DW_EH_PE_datarel + | DW_EH_PE_funcrel | DW_EH_PE_aligned => {} + _ => return false, + } + + true + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_dw_eh_pe_format() { + let encoding = DwEhPe(DW_EH_PE_pcrel.0 | DW_EH_PE_uleb128.0); + assert_eq!(encoding.format(), DW_EH_PE_uleb128); + } + + #[test] + fn test_dw_eh_pe_application() { + let encoding = DwEhPe(DW_EH_PE_pcrel.0 | DW_EH_PE_uleb128.0); + assert_eq!(encoding.application(), DW_EH_PE_pcrel); + } + + #[test] + fn test_dw_eh_pe_is_absent() { + assert_eq!(DW_EH_PE_absptr.is_absent(), false); + assert_eq!(DW_EH_PE_omit.is_absent(), true); + } + + #[test] + fn test_dw_eh_pe_is_valid_encoding_ok() { + let encoding = DwEhPe(DW_EH_PE_uleb128.0 | DW_EH_PE_pcrel.0); + assert!(encoding.is_valid_encoding()); + assert!(DW_EH_PE_absptr.is_valid_encoding()); + assert!(DW_EH_PE_omit.is_valid_encoding()); + } + + #[test] + fn test_dw_eh_pe_is_valid_encoding_bad_format() { + let encoding = DwEhPe((DW_EH_PE_sdata8.0 + 1) | DW_EH_PE_pcrel.0); + assert_eq!(encoding.is_valid_encoding(), false); + } + + #[test] + fn test_dw_eh_pe_is_valid_encoding_bad_application() { + let encoding = DwEhPe(DW_EH_PE_sdata8.0 | (DW_EH_PE_aligned.0 + 1)); + assert_eq!(encoding.is_valid_encoding(), false); + } +} diff --git a/crux-mir/lib/gimli/src/endianity.rs b/crux-mir/lib/gimli/src/endianity.rs new file mode 100644 index 000000000..3201551f1 --- /dev/null +++ b/crux-mir/lib/gimli/src/endianity.rs @@ -0,0 +1,256 @@ +//! Types for compile-time and run-time endianity. + +use core::convert::TryInto; +use core::fmt::Debug; + +/// A trait describing the endianity of some buffer. +pub trait Endianity: Debug + Default + Clone + Copy + PartialEq + Eq { + /// Return true for big endian byte order. + fn is_big_endian(self) -> bool; + + /// Return true for little endian byte order. + #[inline] + fn is_little_endian(self) -> bool { + !self.is_big_endian() + } + + /// Reads an unsigned 16 bit integer from `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 2`. + #[inline] + fn read_u16(self, buf: &[u8]) -> u16 { + let bytes: &[u8; 2] = buf[..2].try_into().unwrap(); + if self.is_big_endian() { + u16::from_be_bytes(*bytes) + } else { + u16::from_le_bytes(*bytes) + } + } + + /// Reads an unsigned 32 bit integer from `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 4`. + #[inline] + fn read_u32(self, buf: &[u8]) -> u32 { + let bytes: &[u8; 4] = buf[..4].try_into().unwrap(); + if self.is_big_endian() { + u32::from_be_bytes(*bytes) + } else { + u32::from_le_bytes(*bytes) + } + } + + /// Reads an unsigned 64 bit integer from `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 8`. + #[inline] + fn read_u64(self, buf: &[u8]) -> u64 { + let bytes: &[u8; 8] = buf[..8].try_into().unwrap(); + if self.is_big_endian() { + u64::from_be_bytes(*bytes) + } else { + u64::from_le_bytes(*bytes) + } + } + + /// Read an unsigned n-bytes integer u64. + /// + /// # Panics + /// + /// Panics when `buf.len() < 1` or `buf.len() > 8`. + #[inline] + fn read_uint(&mut self, buf: &[u8]) -> u64 { + let mut tmp = [0; 8]; + if self.is_big_endian() { + tmp[8 - buf.len()..].copy_from_slice(buf); + } else { + tmp[..buf.len()].copy_from_slice(buf); + } + self.read_u64(&tmp) + } + + /// Reads a signed 16 bit integer from `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 2`. + #[inline] + fn read_i16(self, buf: &[u8]) -> i16 { + self.read_u16(buf) as i16 + } + + /// Reads a signed 32 bit integer from `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 4`. + #[inline] + fn read_i32(self, buf: &[u8]) -> i32 { + self.read_u32(buf) as i32 + } + + /// Reads a signed 64 bit integer from `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 8`. + #[inline] + fn read_i64(self, buf: &[u8]) -> i64 { + self.read_u64(buf) as i64 + } + + /// Reads a 32 bit floating point number from `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 8`. + #[inline] + fn read_f32(self, buf: &[u8]) -> f32 { + f32::from_bits(self.read_u32(buf)) + } + + /// Reads a 32 bit floating point number from `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 8`. + #[inline] + fn read_f64(self, buf: &[u8]) -> f64 { + f64::from_bits(self.read_u64(buf)) + } + + /// Writes an unsigned 16 bit integer `n` to `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 2`. + #[inline] + fn write_u16(self, buf: &mut [u8], n: u16) { + let bytes = if self.is_big_endian() { + n.to_be_bytes() + } else { + n.to_le_bytes() + }; + buf[..2].copy_from_slice(&bytes); + } + + /// Writes an unsigned 32 bit integer `n` to `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 4`. + #[inline] + fn write_u32(self, buf: &mut [u8], n: u32) { + let bytes = if self.is_big_endian() { + n.to_be_bytes() + } else { + n.to_le_bytes() + }; + buf[..4].copy_from_slice(&bytes); + } + + /// Writes an unsigned 64 bit integer `n` to `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 8`. + #[inline] + fn write_u64(self, buf: &mut [u8], n: u64) { + let bytes = if self.is_big_endian() { + n.to_be_bytes() + } else { + n.to_le_bytes() + }; + buf[..8].copy_from_slice(&bytes); + } +} + +/// Byte order that is selectable at runtime. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum RunTimeEndian { + /// Little endian byte order. + Little, + /// Big endian byte order. + Big, +} + +impl Default for RunTimeEndian { + #[cfg(target_endian = "little")] + #[inline] + fn default() -> RunTimeEndian { + RunTimeEndian::Little + } + + #[cfg(target_endian = "big")] + #[inline] + fn default() -> RunTimeEndian { + RunTimeEndian::Big + } +} + +impl Endianity for RunTimeEndian { + #[inline] + fn is_big_endian(self) -> bool { + self != RunTimeEndian::Little + } +} + +/// Little endian byte order. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct LittleEndian; + +impl Default for LittleEndian { + #[inline] + fn default() -> LittleEndian { + LittleEndian + } +} + +impl Endianity for LittleEndian { + #[inline] + fn is_big_endian(self) -> bool { + false + } +} + +/// Big endian byte order. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct BigEndian; + +impl Default for BigEndian { + #[inline] + fn default() -> BigEndian { + BigEndian + } +} + +impl Endianity for BigEndian { + #[inline] + fn is_big_endian(self) -> bool { + true + } +} + +/// The native endianity for the target platform. +#[cfg(target_endian = "little")] +pub type NativeEndian = LittleEndian; + +#[cfg(target_endian = "little")] +#[allow(non_upper_case_globals)] +#[doc(hidden)] +pub const NativeEndian: LittleEndian = LittleEndian; + +/// The native endianity for the target platform. +#[cfg(target_endian = "big")] +pub type NativeEndian = BigEndian; + +#[cfg(target_endian = "big")] +#[allow(non_upper_case_globals)] +#[doc(hidden)] +pub const NativeEndian: BigEndian = BigEndian; diff --git a/crux-mir/lib/gimli/src/leb128.rs b/crux-mir/lib/gimli/src/leb128.rs new file mode 100644 index 000000000..de81cfdcf --- /dev/null +++ b/crux-mir/lib/gimli/src/leb128.rs @@ -0,0 +1,612 @@ +//! Read and write DWARF's "Little Endian Base 128" (LEB128) variable length +//! integer encoding. +//! +//! The implementation is a direct translation of the psuedocode in the DWARF 4 +//! standard's appendix C. +//! +//! Read and write signed integers: +//! +//! ``` +//! # #[cfg(all(feature = "read", feature = "write"))] { +//! use gimli::{EndianSlice, NativeEndian, leb128}; +//! +//! let mut buf = [0; 1024]; +//! +//! // Write to anything that implements `std::io::Write`. +//! { +//! let mut writable = &mut buf[..]; +//! leb128::write::signed(&mut writable, -12345).expect("Should write number"); +//! } +//! +//! // Read from anything that implements `gimli::Reader`. +//! let mut readable = EndianSlice::new(&buf[..], NativeEndian); +//! let val = leb128::read::signed(&mut readable).expect("Should read number"); +//! assert_eq!(val, -12345); +//! # } +//! ``` +//! +//! Or read and write unsigned integers: +//! +//! ``` +//! # #[cfg(all(feature = "read", feature = "write"))] { +//! use gimli::{EndianSlice, NativeEndian, leb128}; +//! +//! let mut buf = [0; 1024]; +//! +//! { +//! let mut writable = &mut buf[..]; +//! leb128::write::unsigned(&mut writable, 98765).expect("Should write number"); +//! } +//! +//! let mut readable = EndianSlice::new(&buf[..], NativeEndian); +//! let val = leb128::read::unsigned(&mut readable).expect("Should read number"); +//! assert_eq!(val, 98765); +//! # } +//! ``` + +const CONTINUATION_BIT: u8 = 1 << 7; +#[cfg(feature = "read-core")] +const SIGN_BIT: u8 = 1 << 6; + +#[inline] +fn low_bits_of_byte(byte: u8) -> u8 { + byte & !CONTINUATION_BIT +} + +#[inline] +#[allow(dead_code)] +fn low_bits_of_u64(val: u64) -> u8 { + let byte = val & u64::from(core::u8::MAX); + low_bits_of_byte(byte as u8) +} + +/// A module for reading signed and unsigned integers that have been LEB128 +/// encoded. +#[cfg(feature = "read-core")] +pub mod read { + use super::{low_bits_of_byte, CONTINUATION_BIT, SIGN_BIT}; + use crate::read::{Error, Reader, Result}; + + /// Read bytes until the LEB128 continuation bit is not set. + pub fn skip(r: &mut R) -> Result<()> { + loop { + let byte = r.read_u8()?; + if byte & CONTINUATION_BIT == 0 { + return Ok(()); + } + } + } + + /// Read an unsigned LEB128 number from the given `Reader` and + /// return it or an error if reading failed. + pub fn unsigned(r: &mut R) -> Result { + let mut result = 0; + let mut shift = 0; + + loop { + let byte = r.read_u8()?; + if shift == 63 && byte != 0x00 && byte != 0x01 { + return Err(Error::BadUnsignedLeb128); + } + + let low_bits = u64::from(low_bits_of_byte(byte)); + result |= low_bits << shift; + + if byte & CONTINUATION_BIT == 0 { + return Ok(result); + } + + shift += 7; + } + } + + /// Read an LEB128 u16 from the given `Reader` and + /// return it or an error if reading failed. + pub fn u16(r: &mut R) -> Result { + let byte = r.read_u8()?; + let mut result = u16::from(low_bits_of_byte(byte)); + if byte & CONTINUATION_BIT == 0 { + return Ok(result); + } + + let byte = r.read_u8()?; + result |= u16::from(low_bits_of_byte(byte)) << 7; + if byte & CONTINUATION_BIT == 0 { + return Ok(result); + } + + let byte = r.read_u8()?; + if byte > 0x03 { + return Err(Error::BadUnsignedLeb128); + } + result += u16::from(byte) << 14; + Ok(result) + } + + /// Read a signed LEB128 number from the given `Reader` and + /// return it or an error if reading failed. + pub fn signed(r: &mut R) -> Result { + let mut result = 0; + let mut shift = 0; + let size = 64; + let mut byte; + + loop { + byte = r.read_u8()?; + if shift == 63 && byte != 0x00 && byte != 0x7f { + return Err(Error::BadSignedLeb128); + } + + let low_bits = i64::from(low_bits_of_byte(byte)); + result |= low_bits << shift; + shift += 7; + + if byte & CONTINUATION_BIT == 0 { + break; + } + } + + if shift < size && (SIGN_BIT & byte) == SIGN_BIT { + // Sign extend the result. + result |= !0 << shift; + } + + Ok(result) + } +} + +/// A module for writing integers encoded as LEB128. +#[cfg(feature = "write")] +pub mod write { + use super::{low_bits_of_u64, CONTINUATION_BIT}; + use std::io; + + /// Write the given unsigned number using the LEB128 encoding to the given + /// `std::io::Write`able. Returns the number of bytes written to `w`, or an + /// error if writing failed. + pub fn unsigned(w: &mut W, mut val: u64) -> Result + where + W: io::Write, + { + let mut bytes_written = 0; + loop { + let mut byte = low_bits_of_u64(val); + val >>= 7; + if val != 0 { + // More bytes to come, so set the continuation bit. + byte |= CONTINUATION_BIT; + } + + let buf = [byte]; + w.write_all(&buf)?; + bytes_written += 1; + + if val == 0 { + return Ok(bytes_written); + } + } + } + + /// Return the size of the LEB128 encoding of the given unsigned number. + pub fn uleb128_size(mut val: u64) -> usize { + let mut size = 0; + loop { + val >>= 7; + size += 1; + if val == 0 { + return size; + } + } + } + + /// Write the given signed number using the LEB128 encoding to the given + /// `std::io::Write`able. Returns the number of bytes written to `w`, or an + /// error if writing failed. + pub fn signed(w: &mut W, mut val: i64) -> Result + where + W: io::Write, + { + let mut bytes_written = 0; + loop { + let mut byte = val as u8; + // Keep the sign bit for testing + val >>= 6; + let done = val == 0 || val == -1; + if done { + byte &= !CONTINUATION_BIT; + } else { + // Remove the sign bit + val >>= 1; + // More bytes to come, so set the continuation bit. + byte |= CONTINUATION_BIT; + } + + let buf = [byte]; + w.write_all(&buf)?; + bytes_written += 1; + + if done { + return Ok(bytes_written); + } + } + } + + /// Return the size of the LEB128 encoding of the given signed number. + pub fn sleb128_size(mut val: i64) -> usize { + let mut size = 0; + loop { + val >>= 6; + let done = val == 0 || val == -1; + val >>= 1; + size += 1; + if done { + return size; + } + } + } +} + +#[cfg(test)] +#[cfg(all(feature = "read", feature = "write"))] +mod tests { + use super::{low_bits_of_byte, low_bits_of_u64, read, write, CONTINUATION_BIT}; + use crate::endianity::NativeEndian; + use crate::read::{EndianSlice, Error, ReaderOffsetId}; + + trait ResultExt { + fn map_eof(self, input: &[u8]) -> Self; + } + + impl ResultExt for Result { + fn map_eof(self, input: &[u8]) -> Self { + match self { + Err(Error::UnexpectedEof(id)) => { + let id = ReaderOffsetId(id.0 - input.as_ptr() as u64); + Err(Error::UnexpectedEof(id)) + } + r => r, + } + } + } + + #[test] + fn test_low_bits_of_byte() { + for i in 0..127 { + assert_eq!(i, low_bits_of_byte(i)); + assert_eq!(i, low_bits_of_byte(i | CONTINUATION_BIT)); + } + } + + #[test] + fn test_low_bits_of_u64() { + for i in 0u64..127 { + assert_eq!(i as u8, low_bits_of_u64(1 << 16 | i)); + assert_eq!( + i as u8, + low_bits_of_u64(i << 16 | i | (u64::from(CONTINUATION_BIT))) + ); + } + } + + // Examples from the DWARF 4 standard, section 7.6, figure 22. + #[test] + fn test_read_unsigned() { + let buf = [2u8]; + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + assert_eq!( + 2, + read::unsigned(&mut readable).expect("Should read number") + ); + + let buf = [127u8]; + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + assert_eq!( + 127, + read::unsigned(&mut readable).expect("Should read number") + ); + + let buf = [CONTINUATION_BIT, 1]; + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + assert_eq!( + 128, + read::unsigned(&mut readable).expect("Should read number") + ); + + let buf = [1u8 | CONTINUATION_BIT, 1]; + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + assert_eq!( + 129, + read::unsigned(&mut readable).expect("Should read number") + ); + + let buf = [2u8 | CONTINUATION_BIT, 1]; + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + assert_eq!( + 130, + read::unsigned(&mut readable).expect("Should read number") + ); + + let buf = [57u8 | CONTINUATION_BIT, 100]; + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + assert_eq!( + 12857, + read::unsigned(&mut readable).expect("Should read number") + ); + } + + // Examples from the DWARF 4 standard, section 7.6, figure 23. + #[test] + fn test_read_signed() { + let buf = [2u8]; + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + assert_eq!(2, read::signed(&mut readable).expect("Should read number")); + + let buf = [0x7eu8]; + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + assert_eq!(-2, read::signed(&mut readable).expect("Should read number")); + + let buf = [127u8 | CONTINUATION_BIT, 0]; + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + assert_eq!( + 127, + read::signed(&mut readable).expect("Should read number") + ); + + let buf = [1u8 | CONTINUATION_BIT, 0x7f]; + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + assert_eq!( + -127, + read::signed(&mut readable).expect("Should read number") + ); + + let buf = [CONTINUATION_BIT, 1]; + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + assert_eq!( + 128, + read::signed(&mut readable).expect("Should read number") + ); + + let buf = [CONTINUATION_BIT, 0x7f]; + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + assert_eq!( + -128, + read::signed(&mut readable).expect("Should read number") + ); + + let buf = [1u8 | CONTINUATION_BIT, 1]; + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + assert_eq!( + 129, + read::signed(&mut readable).expect("Should read number") + ); + + let buf = [0x7fu8 | CONTINUATION_BIT, 0x7e]; + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + assert_eq!( + -129, + read::signed(&mut readable).expect("Should read number") + ); + } + + #[test] + fn test_read_signed_63_bits() { + let buf = [ + CONTINUATION_BIT, + CONTINUATION_BIT, + CONTINUATION_BIT, + CONTINUATION_BIT, + CONTINUATION_BIT, + CONTINUATION_BIT, + CONTINUATION_BIT, + CONTINUATION_BIT, + 0x40, + ]; + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + assert_eq!( + -0x4000_0000_0000_0000, + read::signed(&mut readable).expect("Should read number") + ); + } + + #[test] + fn test_read_unsigned_not_enough_data() { + let buf = [CONTINUATION_BIT]; + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + assert_eq!( + read::unsigned(&mut readable).map_eof(&buf), + Err(Error::UnexpectedEof(ReaderOffsetId(1))) + ); + } + + #[test] + fn test_read_signed_not_enough_data() { + let buf = [CONTINUATION_BIT]; + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + assert_eq!( + read::signed(&mut readable).map_eof(&buf), + Err(Error::UnexpectedEof(ReaderOffsetId(1))) + ); + } + + #[test] + fn test_write_unsigned_not_enough_space() { + let mut buf = [0; 1]; + let mut writable = &mut buf[..]; + match write::unsigned(&mut writable, 128) { + Err(e) => assert_eq!(e.kind(), std::io::ErrorKind::WriteZero), + otherwise => panic!("Unexpected: {:?}", otherwise), + } + } + + #[test] + fn test_write_signed_not_enough_space() { + let mut buf = [0; 1]; + let mut writable = &mut buf[..]; + match write::signed(&mut writable, 128) { + Err(e) => assert_eq!(e.kind(), std::io::ErrorKind::WriteZero), + otherwise => panic!("Unexpected: {:?}", otherwise), + } + } + + #[test] + fn dogfood_signed() { + fn inner(i: i64) { + let mut buf = [0u8; 1024]; + + { + let mut writable = &mut buf[..]; + write::signed(&mut writable, i).expect("Should write signed number"); + } + + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + let result = read::signed(&mut readable).expect("Should be able to read it back again"); + assert_eq!(i, result); + } + for i in -513..513 { + inner(i); + } + inner(core::i64::MIN); + } + + #[test] + fn dogfood_unsigned() { + for i in 0..1025 { + let mut buf = [0u8; 1024]; + + { + let mut writable = &mut buf[..]; + write::unsigned(&mut writable, i).expect("Should write signed number"); + } + + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + let result = + read::unsigned(&mut readable).expect("Should be able to read it back again"); + assert_eq!(i, result); + } + } + + #[test] + fn test_read_unsigned_overflow() { + let buf = [ + 2u8 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 1, + ]; + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + assert!(read::unsigned(&mut readable).is_err()); + } + + #[test] + fn test_read_signed_overflow() { + let buf = [ + 2u8 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 2 | CONTINUATION_BIT, + 1, + ]; + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + assert!(read::signed(&mut readable).is_err()); + } + + #[test] + fn test_read_multiple() { + let buf = [2u8 | CONTINUATION_BIT, 1u8, 1u8]; + + let mut readable = EndianSlice::new(&buf[..], NativeEndian); + assert_eq!( + read::unsigned(&mut readable).expect("Should read first number"), + 130u64 + ); + assert_eq!( + read::unsigned(&mut readable).expect("Should read first number"), + 1u64 + ); + } + + #[test] + fn test_read_u16() { + for (buf, val) in [ + (&[2][..], 2), + (&[0x7f][..], 0x7f), + (&[0x80, 1][..], 0x80), + (&[0x81, 1][..], 0x81), + (&[0x82, 1][..], 0x82), + (&[0xff, 0x7f][..], 0x3fff), + (&[0x80, 0x80, 1][..], 0x4000), + (&[0xff, 0xff, 1][..], 0x7fff), + (&[0xff, 0xff, 3][..], 0xffff), + ] + .iter() + { + let mut readable = EndianSlice::new(buf, NativeEndian); + assert_eq!(*val, read::u16(&mut readable).expect("Should read number")); + } + + for buf in [ + &[0x80][..], + &[0x80, 0x80][..], + &[0x80, 0x80, 4][..], + &[0x80, 0x80, 0x80, 3][..], + ] + .iter() + { + let mut readable = EndianSlice::new(buf, NativeEndian); + assert!(read::u16(&mut readable).is_err(), "{:?}", buf); + } + } +} diff --git a/crux-mir/lib/gimli/src/lib.rs b/crux-mir/lib/gimli/src/lib.rs new file mode 100644 index 000000000..ed1af9cbd --- /dev/null +++ b/crux-mir/lib/gimli/src/lib.rs @@ -0,0 +1,76 @@ +//! `gimli` is a library for reading and writing the +//! [DWARF debugging format](https://dwarfstd.org/). +//! +//! See the [read](./read/index.html) and [write](./write/index.html) modules +//! for examples and API documentation. +//! +//! ## Cargo Features +//! +//! Cargo features that can be enabled with `gimli`: +//! +//! * `std`: Enabled by default. Use the `std` library. Disabling this feature +//! allows using `gimli` in embedded environments that do not have access to +//! `std`. Note that even when `std` is disabled, `gimli` still requires an +//! implementation of the `alloc` crate. +//! +//! * `read`: Enabled by default. Enables the `read` module. Use of `std` is +//! optional. +//! +//! * `write`: Enabled by default. Enables the `write` module. Always uses +//! the `std` library. +#![deny(missing_docs)] +#![deny(missing_debug_implementations)] +// Selectively enable rust 2018 warnings +#![warn(bare_trait_objects)] +#![warn(unused_extern_crates)] +#![warn(ellipsis_inclusive_range_patterns)] +//#![warn(elided_lifetimes_in_paths)] +#![warn(explicit_outlives_requirements)] +// Allow clippy warnings when we aren't building with clippy. +#![allow(unknown_lints)] +// False positives with `fallible_iterator`. +#![allow(clippy::should_implement_trait)] +// Many false positives involving `continue`. +#![allow(clippy::never_loop)] +// False positives when block expressions are used inside an assertion. +#![allow(clippy::panic_params)] +#![no_std] + +#[allow(unused_imports)] +#[cfg(any(feature = "read", feature = "write"))] +#[macro_use] +extern crate alloc; + +#[cfg(any(feature = "std", feature = "write"))] +#[macro_use] +extern crate std; + +#[cfg(feature = "stable_deref_trait")] +pub use stable_deref_trait::{CloneStableDeref, StableDeref}; + +mod common; +pub use crate::common::*; + +mod arch; +pub use crate::arch::*; + +pub mod constants; +// For backwards compat. +pub use crate::constants::*; + +mod endianity; +pub use crate::endianity::{BigEndian, Endianity, LittleEndian, NativeEndian, RunTimeEndian}; + +pub mod leb128; + +#[cfg(feature = "read-core")] +pub mod read; +// For backwards compat. +#[cfg(feature = "read-core")] +pub use crate::read::*; + +#[cfg(feature = "write")] +pub mod write; + +#[cfg(test)] +mod test_util; diff --git a/crux-mir/lib/gimli/src/read/abbrev.rs b/crux-mir/lib/gimli/src/read/abbrev.rs new file mode 100644 index 000000000..1a24835a7 --- /dev/null +++ b/crux-mir/lib/gimli/src/read/abbrev.rs @@ -0,0 +1,996 @@ +//! Functions for parsing DWARF debugging abbreviations. + +use alloc::collections::btree_map; +use alloc::vec::Vec; +use core::convert::TryFrom; +use core::fmt::{self, Debug}; +use core::iter::FromIterator; +use core::ops::Deref; + +use crate::common::{DebugAbbrevOffset, Encoding, SectionId}; +use crate::constants; +use crate::endianity::Endianity; +use crate::read::{EndianSlice, Error, Reader, Result, Section, UnitHeader}; + +/// The `DebugAbbrev` struct represents the abbreviations describing +/// `DebuggingInformationEntry`s' attribute names and forms found in the +/// `.debug_abbrev` section. +#[derive(Debug, Default, Clone, Copy)] +pub struct DebugAbbrev { + debug_abbrev_section: R, +} + +impl<'input, Endian> DebugAbbrev> +where + Endian: Endianity, +{ + /// Construct a new `DebugAbbrev` instance from the data in the `.debug_abbrev` + /// section. + /// + /// It is the caller's responsibility to read the `.debug_abbrev` section and + /// present it as a `&[u8]` slice. That means using some ELF loader on + /// Linux, a Mach-O loader on macOS, etc. + /// + /// ``` + /// use gimli::{DebugAbbrev, LittleEndian}; + /// + /// # let buf = [0x00, 0x01, 0x02, 0x03]; + /// # let read_debug_abbrev_section_somehow = || &buf; + /// let debug_abbrev = DebugAbbrev::new(read_debug_abbrev_section_somehow(), LittleEndian); + /// ``` + pub fn new(debug_abbrev_section: &'input [u8], endian: Endian) -> Self { + Self::from(EndianSlice::new(debug_abbrev_section, endian)) + } +} + +impl DebugAbbrev { + /// Parse the abbreviations at the given `offset` within this + /// `.debug_abbrev` section. + /// + /// The `offset` should generally be retrieved from a unit header. + pub fn abbreviations( + &self, + debug_abbrev_offset: DebugAbbrevOffset, + ) -> Result { + let input = &mut self.debug_abbrev_section.clone(); + input.skip(debug_abbrev_offset.0)?; + Abbreviations::parse(input) + } +} + +impl DebugAbbrev { + /// Create a `DebugAbbrev` section that references the data in `self`. + /// + /// This is useful when `R` implements `Reader` but `T` does not. + /// + /// ## Example Usage + /// + /// ```rust,no_run + /// # let load_section = || unimplemented!(); + /// // Read the DWARF section into a `Vec` with whatever object loader you're using. + /// let owned_section: gimli::DebugAbbrev> = load_section(); + /// // Create a reference to the DWARF section. + /// let section = owned_section.borrow(|section| { + /// gimli::EndianSlice::new(§ion, gimli::LittleEndian) + /// }); + /// ``` + pub fn borrow<'a, F, R>(&'a self, mut borrow: F) -> DebugAbbrev + where + F: FnMut(&'a T) -> R, + { + borrow(&self.debug_abbrev_section).into() + } +} + +impl Section for DebugAbbrev { + fn id() -> SectionId { + SectionId::DebugAbbrev + } + + fn reader(&self) -> &R { + &self.debug_abbrev_section + } +} + +impl From for DebugAbbrev { + fn from(debug_abbrev_section: R) -> Self { + DebugAbbrev { + debug_abbrev_section, + } + } +} + +/// A set of type abbreviations. +/// +/// Construct an `Abbreviations` instance with the +/// [`abbreviations()`](struct.UnitHeader.html#method.abbreviations) +/// method. +#[derive(Debug, Default, Clone)] +pub struct Abbreviations { + vec: Vec, + map: btree_map::BTreeMap, +} + +impl Abbreviations { + /// Construct a new, empty set of abbreviations. + fn empty() -> Abbreviations { + Abbreviations { + vec: Vec::new(), + map: btree_map::BTreeMap::new(), + } + } + + /// Insert an abbreviation into the set. + /// + /// Returns `Ok` if it is the first abbreviation in the set with its code, + /// `Err` if the code is a duplicate and there already exists an + /// abbreviation in the set with the given abbreviation's code. + fn insert(&mut self, abbrev: Abbreviation) -> ::core::result::Result<(), ()> { + let code_usize = abbrev.code as usize; + if code_usize as u64 == abbrev.code { + // Optimize for sequential abbreviation codes by storing them + // in a Vec, as long as the map doesn't already contain them. + // A potential further optimization would be to allow some + // holes in the Vec, but there's no need for that yet. + if code_usize - 1 < self.vec.len() { + return Err(()); + } else if code_usize - 1 == self.vec.len() { + if !self.map.is_empty() && self.map.contains_key(&abbrev.code) { + return Err(()); + } else { + self.vec.push(abbrev); + return Ok(()); + } + } + } + match self.map.entry(abbrev.code) { + btree_map::Entry::Occupied(_) => Err(()), + btree_map::Entry::Vacant(entry) => { + entry.insert(abbrev); + Ok(()) + } + } + } + + /// Get the abbreviation associated with the given code. + #[inline] + pub fn get(&self, code: u64) -> Option<&Abbreviation> { + if let Ok(code) = usize::try_from(code) { + let index = code.checked_sub(1)?; + if index < self.vec.len() { + return Some(&self.vec[index]); + } + } + + self.map.get(&code) + } + + /// Parse a series of abbreviations, terminated by a null abbreviation. + fn parse(input: &mut R) -> Result { + let mut abbrevs = Abbreviations::empty(); + + while let Some(abbrev) = Abbreviation::parse(input)? { + if abbrevs.insert(abbrev).is_err() { + return Err(Error::DuplicateAbbreviationCode); + } + } + + Ok(abbrevs) + } +} + +/// An abbreviation describes the shape of a `DebuggingInformationEntry`'s type: +/// its code, tag type, whether it has children, and its set of attributes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Abbreviation { + code: u64, + tag: constants::DwTag, + has_children: constants::DwChildren, + attributes: Attributes, +} + +impl Abbreviation { + /// Construct a new `Abbreviation`. + /// + /// ### Panics + /// + /// Panics if `code` is `0`. + pub(crate) fn new( + code: u64, + tag: constants::DwTag, + has_children: constants::DwChildren, + attributes: Attributes, + ) -> Abbreviation { + assert_ne!(code, 0); + Abbreviation { + code, + tag, + has_children, + attributes, + } + } + + /// Get this abbreviation's code. + #[inline] + pub fn code(&self) -> u64 { + self.code + } + + /// Get this abbreviation's tag. + #[inline] + pub fn tag(&self) -> constants::DwTag { + self.tag + } + + /// Return true if this abbreviation's type has children, false otherwise. + #[inline] + pub fn has_children(&self) -> bool { + self.has_children == constants::DW_CHILDREN_yes + } + + /// Get this abbreviation's attributes. + #[inline] + pub fn attributes(&self) -> &[AttributeSpecification] { + &self.attributes[..] + } + + /// Parse an abbreviation's tag. + fn parse_tag(input: &mut R) -> Result { + let val = input.read_uleb128_u16()?; + if val == 0 { + Err(Error::AbbreviationTagZero) + } else { + Ok(constants::DwTag(val)) + } + } + + /// Parse an abbreviation's "does the type have children?" byte. + fn parse_has_children(input: &mut R) -> Result { + let val = input.read_u8()?; + let val = constants::DwChildren(val); + if val == constants::DW_CHILDREN_no || val == constants::DW_CHILDREN_yes { + Ok(val) + } else { + Err(Error::BadHasChildren) + } + } + + /// Parse a series of attribute specifications, terminated by a null attribute + /// specification. + fn parse_attributes(input: &mut R) -> Result { + let mut attrs = Attributes::new(); + + while let Some(attr) = AttributeSpecification::parse(input)? { + attrs.push(attr); + } + + Ok(attrs) + } + + /// Parse an abbreviation. Return `None` for the null abbreviation, `Some` + /// for an actual abbreviation. + fn parse(input: &mut R) -> Result> { + let code = input.read_uleb128()?; + if code == 0 { + return Ok(None); + } + + let tag = Self::parse_tag(input)?; + let has_children = Self::parse_has_children(input)?; + let attributes = Self::parse_attributes(input)?; + let abbrev = Abbreviation::new(code, tag, has_children, attributes); + Ok(Some(abbrev)) + } +} + +/// A list of attributes found in an `Abbreviation` +#[derive(Clone)] +pub(crate) enum Attributes { + Inline { + buf: [AttributeSpecification; MAX_ATTRIBUTES_INLINE], + len: usize, + }, + Heap(Vec), +} + +// Length of 5 based on benchmark results for both x86-64 and i686. +const MAX_ATTRIBUTES_INLINE: usize = 5; + +impl Attributes { + /// Returns a new empty list of attributes + fn new() -> Attributes { + let default = + AttributeSpecification::new(constants::DW_AT_null, constants::DW_FORM_null, None); + Attributes::Inline { + buf: [default; 5], + len: 0, + } + } + + /// Pushes a new value onto this list of attributes. + fn push(&mut self, attr: AttributeSpecification) { + match self { + Attributes::Heap(list) => return list.push(attr), + Attributes::Inline { + buf, + len: MAX_ATTRIBUTES_INLINE, + } => { + let mut list = buf.to_vec(); + list.push(attr); + *self = Attributes::Heap(list); + } + Attributes::Inline { buf, len } => { + buf[*len] = attr; + *len += 1; + } + } + } +} + +impl Debug for Attributes { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + (&**self).fmt(f) + } +} + +impl PartialEq for Attributes { + fn eq(&self, other: &Attributes) -> bool { + &**self == &**other + } +} + +impl Eq for Attributes {} + +impl Deref for Attributes { + type Target = [AttributeSpecification]; + fn deref(&self) -> &[AttributeSpecification] { + match self { + Attributes::Inline { buf, len } => &buf[..*len], + Attributes::Heap(list) => list, + } + } +} + +impl FromIterator for Attributes { + fn from_iter(iter: I) -> Attributes + where + I: IntoIterator, + { + let mut list = Attributes::new(); + for item in iter { + list.push(item); + } + return list; + } +} + +impl From> for Attributes { + fn from(list: Vec) -> Attributes { + Attributes::Heap(list) + } +} + +/// The description of an attribute in an abbreviated type. It is a pair of name +/// and form. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AttributeSpecification { + name: constants::DwAt, + form: constants::DwForm, + implicit_const_value: i64, +} + +impl AttributeSpecification { + /// Construct a new `AttributeSpecification` from the given name and form + /// and implicit const value. + #[inline] + pub fn new( + name: constants::DwAt, + form: constants::DwForm, + implicit_const_value: Option, + ) -> AttributeSpecification { + debug_assert!( + (form == constants::DW_FORM_implicit_const && implicit_const_value.is_some()) + || (form != constants::DW_FORM_implicit_const && implicit_const_value.is_none()) + ); + AttributeSpecification { + name, + form, + implicit_const_value: implicit_const_value.unwrap_or(0), + } + } + + /// Get the attribute's name. + #[inline] + pub fn name(&self) -> constants::DwAt { + self.name + } + + /// Get the attribute's form. + #[inline] + pub fn form(&self) -> constants::DwForm { + self.form + } + + /// Get the attribute's implicit const value. + #[inline] + pub fn implicit_const_value(&self) -> Option { + if self.form == constants::DW_FORM_implicit_const { + Some(self.implicit_const_value) + } else { + None + } + } + + /// Return the size of the attribute, in bytes. + /// + /// Note that because some attributes are variably sized, the size cannot + /// always be known without parsing, in which case we return `None`. + pub fn size(&self, header: &UnitHeader) -> Option { + get_attribute_size(self.form, header.encoding()).map(usize::from) + } + + /// Parse an attribute's form. + fn parse_form(input: &mut R) -> Result { + let val = input.read_uleb128_u16()?; + if val == 0 { + Err(Error::AttributeFormZero) + } else { + Ok(constants::DwForm(val)) + } + } + + /// Parse an attribute specification. Returns `None` for the null attribute + /// specification, `Some` for an actual attribute specification. + fn parse(input: &mut R) -> Result> { + let name = input.read_uleb128_u16()?; + if name == 0 { + // Parse the null attribute specification. + let form = input.read_uleb128_u16()?; + return if form == 0 { + Ok(None) + } else { + Err(Error::ExpectedZero) + }; + } + + let name = constants::DwAt(name); + let form = Self::parse_form(input)?; + let implicit_const_value = if form == constants::DW_FORM_implicit_const { + Some(input.read_sleb128()?) + } else { + None + }; + let spec = AttributeSpecification::new(name, form, implicit_const_value); + Ok(Some(spec)) + } +} + +#[inline] +pub(crate) fn get_attribute_size(form: constants::DwForm, encoding: Encoding) -> Option { + match form { + constants::DW_FORM_addr => Some(encoding.address_size), + + constants::DW_FORM_implicit_const | + constants::DW_FORM_flag_present => Some(0), + + constants::DW_FORM_data1 + | constants::DW_FORM_flag + | constants::DW_FORM_strx1 + | constants::DW_FORM_ref1 + | constants::DW_FORM_addrx1 => Some(1), + + constants::DW_FORM_data2 + | constants::DW_FORM_ref2 + | constants::DW_FORM_addrx2 + | constants::DW_FORM_strx2 => Some(2), + + constants::DW_FORM_addrx3 | constants::DW_FORM_strx3 => Some(3), + + constants::DW_FORM_data4 + | constants::DW_FORM_ref_sup4 + | constants::DW_FORM_ref4 + | constants::DW_FORM_strx4 + | constants::DW_FORM_addrx4 => Some(4), + + constants::DW_FORM_data8 + | constants::DW_FORM_ref8 + | constants::DW_FORM_ref_sig8 + | constants::DW_FORM_ref_sup8 => Some(8), + + constants::DW_FORM_data16 => Some(16), + + constants::DW_FORM_sec_offset + | constants::DW_FORM_GNU_ref_alt + | constants::DW_FORM_strp + | constants::DW_FORM_strp_sup + | constants::DW_FORM_GNU_strp_alt + | constants::DW_FORM_line_strp => Some(encoding.format.word_size()), + + constants::DW_FORM_ref_addr => { + // This is an offset, but DWARF version 2 specifies that DW_FORM_ref_addr + // has the same size as an address on the target system. This was changed + // in DWARF version 3. + Some(if encoding.version == 2 { + encoding.address_size + } else { + encoding.format.word_size() + }) + } + + // Variably sized forms. + constants::DW_FORM_block | + constants::DW_FORM_block1 | + constants::DW_FORM_block2 | + constants::DW_FORM_block4 | + constants::DW_FORM_exprloc | + constants::DW_FORM_ref_udata | + constants::DW_FORM_string | + constants::DW_FORM_sdata | + constants::DW_FORM_udata | + constants::DW_FORM_indirect | + + // We don't know the size of unknown forms. + _ => None, + } +} + +#[cfg(test)] +pub mod tests { + use super::*; + use crate::constants; + use crate::endianity::LittleEndian; + use crate::read::{EndianSlice, Error}; + use crate::test_util::GimliSectionMethods; + #[cfg(target_pointer_width = "32")] + use core::u32; + use test_assembler::Section; + + pub trait AbbrevSectionMethods { + fn abbrev(self, code: u64, tag: constants::DwTag, children: constants::DwChildren) -> Self; + fn abbrev_null(self) -> Self; + fn abbrev_attr(self, name: constants::DwAt, form: constants::DwForm) -> Self; + fn abbrev_attr_implicit_const(self, name: constants::DwAt, value: i64) -> Self; + fn abbrev_attr_null(self) -> Self; + } + + impl AbbrevSectionMethods for Section { + fn abbrev(self, code: u64, tag: constants::DwTag, children: constants::DwChildren) -> Self { + self.uleb(code).uleb(tag.0.into()).D8(children.0) + } + + fn abbrev_null(self) -> Self { + self.D8(0) + } + + fn abbrev_attr(self, name: constants::DwAt, form: constants::DwForm) -> Self { + self.uleb(name.0.into()).uleb(form.0.into()) + } + + fn abbrev_attr_implicit_const(self, name: constants::DwAt, value: i64) -> Self { + self.uleb(name.0.into()) + .uleb(constants::DW_FORM_implicit_const.0.into()) + .sleb(value) + } + + fn abbrev_attr_null(self) -> Self { + self.D8(0).D8(0) + } + } + + #[test] + fn test_debug_abbrev_ok() { + let extra_start = [1, 2, 3, 4]; + let expected_rest = [5, 6, 7, 8]; + #[rustfmt::skip] + let buf = Section::new() + .append_bytes(&extra_start) + .abbrev(2, constants::DW_TAG_subprogram, constants::DW_CHILDREN_no) + .abbrev_attr(constants::DW_AT_name, constants::DW_FORM_string) + .abbrev_attr_null() + .abbrev(1, constants::DW_TAG_compile_unit, constants::DW_CHILDREN_yes) + .abbrev_attr(constants::DW_AT_producer, constants::DW_FORM_strp) + .abbrev_attr(constants::DW_AT_language, constants::DW_FORM_data2) + .abbrev_attr_null() + .abbrev_null() + .append_bytes(&expected_rest) + .get_contents() + .unwrap(); + + let abbrev1 = Abbreviation::new( + 1, + constants::DW_TAG_compile_unit, + constants::DW_CHILDREN_yes, + vec![ + AttributeSpecification::new( + constants::DW_AT_producer, + constants::DW_FORM_strp, + None, + ), + AttributeSpecification::new( + constants::DW_AT_language, + constants::DW_FORM_data2, + None, + ), + ] + .into(), + ); + + let abbrev2 = Abbreviation::new( + 2, + constants::DW_TAG_subprogram, + constants::DW_CHILDREN_no, + vec![AttributeSpecification::new( + constants::DW_AT_name, + constants::DW_FORM_string, + None, + )] + .into(), + ); + + let debug_abbrev = DebugAbbrev::new(&buf, LittleEndian); + let debug_abbrev_offset = DebugAbbrevOffset(extra_start.len()); + let abbrevs = debug_abbrev + .abbreviations(debug_abbrev_offset) + .expect("Should parse abbreviations"); + assert_eq!(abbrevs.get(1), Some(&abbrev1)); + assert_eq!(abbrevs.get(2), Some(&abbrev2)); + } + + #[test] + fn test_abbreviations_insert() { + fn abbrev(code: u16) -> Abbreviation { + Abbreviation::new( + code.into(), + constants::DwTag(code), + constants::DW_CHILDREN_no, + vec![].into(), + ) + } + + fn assert_abbrev(abbrevs: &Abbreviations, code: u16) { + let abbrev = abbrevs.get(code.into()).unwrap(); + assert_eq!(abbrev.tag(), constants::DwTag(code)); + } + + // Sequential insert. + let mut abbrevs = Abbreviations::empty(); + abbrevs.insert(abbrev(1)).unwrap(); + abbrevs.insert(abbrev(2)).unwrap(); + assert_eq!(abbrevs.vec.len(), 2); + assert!(abbrevs.map.is_empty()); + assert_abbrev(&abbrevs, 1); + assert_abbrev(&abbrevs, 2); + + // Out of order insert. + let mut abbrevs = Abbreviations::empty(); + abbrevs.insert(abbrev(2)).unwrap(); + abbrevs.insert(abbrev(3)).unwrap(); + assert!(abbrevs.vec.is_empty()); + assert_abbrev(&abbrevs, 2); + assert_abbrev(&abbrevs, 3); + + // Mixed order insert. + let mut abbrevs = Abbreviations::empty(); + abbrevs.insert(abbrev(1)).unwrap(); + abbrevs.insert(abbrev(3)).unwrap(); + abbrevs.insert(abbrev(2)).unwrap(); + assert_eq!(abbrevs.vec.len(), 2); + assert_abbrev(&abbrevs, 1); + assert_abbrev(&abbrevs, 2); + assert_abbrev(&abbrevs, 3); + + // Duplicate code in vec. + let mut abbrevs = Abbreviations::empty(); + abbrevs.insert(abbrev(1)).unwrap(); + abbrevs.insert(abbrev(2)).unwrap(); + assert_eq!(abbrevs.insert(abbrev(1)), Err(())); + assert_eq!(abbrevs.insert(abbrev(2)), Err(())); + + // Duplicate code in map when adding to map. + let mut abbrevs = Abbreviations::empty(); + abbrevs.insert(abbrev(2)).unwrap(); + assert_eq!(abbrevs.insert(abbrev(2)), Err(())); + + // Duplicate code in map when adding to vec. + let mut abbrevs = Abbreviations::empty(); + abbrevs.insert(abbrev(2)).unwrap(); + abbrevs.insert(abbrev(1)).unwrap(); + assert_eq!(abbrevs.insert(abbrev(2)), Err(())); + + // 32-bit usize conversions. + let mut abbrevs = Abbreviations::empty(); + abbrevs.insert(abbrev(2)).unwrap(); + } + + #[test] + #[cfg(target_pointer_width = "32")] + fn test_abbreviations_insert_32() { + fn abbrev(code: u64) -> Abbreviation { + Abbreviation::new( + code, + constants::DwTag(code as u16), + constants::DW_CHILDREN_no, + vec![].into(), + ) + } + + fn assert_abbrev(abbrevs: &Abbreviations, code: u64) { + let abbrev = abbrevs.get(code).unwrap(); + assert_eq!(abbrev.tag(), constants::DwTag(code as u16)); + } + + let mut abbrevs = Abbreviations::empty(); + abbrevs.insert(abbrev(1)).unwrap(); + + let wrap_code = (u32::MAX as u64 + 1) + 1; + // `get` should not treat the wrapped code as `1`. + assert_eq!(abbrevs.get(wrap_code), None); + // `insert` should not treat the wrapped code as `1`. + abbrevs.insert(abbrev(wrap_code)).unwrap(); + assert_abbrev(&abbrevs, 1); + assert_abbrev(&abbrevs, wrap_code); + } + + #[test] + fn test_parse_abbreviations_ok() { + let expected_rest = [1, 2, 3, 4]; + #[rustfmt::skip] + let buf = Section::new() + .abbrev(2, constants::DW_TAG_subprogram, constants::DW_CHILDREN_no) + .abbrev_attr(constants::DW_AT_name, constants::DW_FORM_string) + .abbrev_attr_null() + .abbrev(1, constants::DW_TAG_compile_unit, constants::DW_CHILDREN_yes) + .abbrev_attr(constants::DW_AT_producer, constants::DW_FORM_strp) + .abbrev_attr(constants::DW_AT_language, constants::DW_FORM_data2) + .abbrev_attr_null() + .abbrev_null() + .append_bytes(&expected_rest) + .get_contents() + .unwrap(); + let rest = &mut EndianSlice::new(&*buf, LittleEndian); + + let abbrev1 = Abbreviation::new( + 1, + constants::DW_TAG_compile_unit, + constants::DW_CHILDREN_yes, + vec![ + AttributeSpecification::new( + constants::DW_AT_producer, + constants::DW_FORM_strp, + None, + ), + AttributeSpecification::new( + constants::DW_AT_language, + constants::DW_FORM_data2, + None, + ), + ] + .into(), + ); + + let abbrev2 = Abbreviation::new( + 2, + constants::DW_TAG_subprogram, + constants::DW_CHILDREN_no, + vec![AttributeSpecification::new( + constants::DW_AT_name, + constants::DW_FORM_string, + None, + )] + .into(), + ); + + let abbrevs = Abbreviations::parse(rest).expect("Should parse abbreviations"); + assert_eq!(abbrevs.get(1), Some(&abbrev1)); + assert_eq!(abbrevs.get(2), Some(&abbrev2)); + assert_eq!(*rest, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_abbreviations_duplicate() { + let expected_rest = [1, 2, 3, 4]; + #[rustfmt::skip] + let buf = Section::new() + .abbrev(1, constants::DW_TAG_subprogram, constants::DW_CHILDREN_no) + .abbrev_attr(constants::DW_AT_name, constants::DW_FORM_string) + .abbrev_attr_null() + .abbrev(1, constants::DW_TAG_compile_unit, constants::DW_CHILDREN_yes) + .abbrev_attr(constants::DW_AT_producer, constants::DW_FORM_strp) + .abbrev_attr(constants::DW_AT_language, constants::DW_FORM_data2) + .abbrev_attr_null() + .abbrev_null() + .append_bytes(&expected_rest) + .get_contents() + .unwrap(); + let buf = &mut EndianSlice::new(&*buf, LittleEndian); + + match Abbreviations::parse(buf) { + Err(Error::DuplicateAbbreviationCode) => {} + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + fn test_parse_abbreviation_tag_ok() { + let buf = [0x01, 0x02]; + let rest = &mut EndianSlice::new(&buf, LittleEndian); + let tag = Abbreviation::parse_tag(rest).expect("Should parse tag"); + assert_eq!(tag, constants::DW_TAG_array_type); + assert_eq!(*rest, EndianSlice::new(&buf[1..], LittleEndian)); + } + + #[test] + fn test_parse_abbreviation_tag_zero() { + let buf = [0x00]; + let buf = &mut EndianSlice::new(&buf, LittleEndian); + match Abbreviation::parse_tag(buf) { + Err(Error::AbbreviationTagZero) => {} + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + fn test_parse_abbreviation_has_children() { + let buf = [0x00, 0x01, 0x02]; + let rest = &mut EndianSlice::new(&buf, LittleEndian); + let val = Abbreviation::parse_has_children(rest).expect("Should parse children"); + assert_eq!(val, constants::DW_CHILDREN_no); + let val = Abbreviation::parse_has_children(rest).expect("Should parse children"); + assert_eq!(val, constants::DW_CHILDREN_yes); + match Abbreviation::parse_has_children(rest) { + Err(Error::BadHasChildren) => {} + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + fn test_parse_abbreviation_ok() { + let expected_rest = [0x01, 0x02, 0x03, 0x04]; + let buf = Section::new() + .abbrev(1, constants::DW_TAG_subprogram, constants::DW_CHILDREN_no) + .abbrev_attr(constants::DW_AT_name, constants::DW_FORM_string) + .abbrev_attr_null() + .append_bytes(&expected_rest) + .get_contents() + .unwrap(); + let rest = &mut EndianSlice::new(&*buf, LittleEndian); + + let expect = Some(Abbreviation::new( + 1, + constants::DW_TAG_subprogram, + constants::DW_CHILDREN_no, + vec![AttributeSpecification::new( + constants::DW_AT_name, + constants::DW_FORM_string, + None, + )] + .into(), + )); + + let abbrev = Abbreviation::parse(rest).expect("Should parse abbreviation"); + assert_eq!(abbrev, expect); + assert_eq!(*rest, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_abbreviation_implicit_const_ok() { + let expected_rest = [0x01, 0x02, 0x03, 0x04]; + let buf = Section::new() + .abbrev(1, constants::DW_TAG_subprogram, constants::DW_CHILDREN_no) + .abbrev_attr_implicit_const(constants::DW_AT_name, -42) + .abbrev_attr_null() + .append_bytes(&expected_rest) + .get_contents() + .unwrap(); + let rest = &mut EndianSlice::new(&*buf, LittleEndian); + + let expect = Some(Abbreviation::new( + 1, + constants::DW_TAG_subprogram, + constants::DW_CHILDREN_no, + vec![AttributeSpecification::new( + constants::DW_AT_name, + constants::DW_FORM_implicit_const, + Some(-42), + )] + .into(), + )); + + let abbrev = Abbreviation::parse(rest).expect("Should parse abbreviation"); + assert_eq!(abbrev, expect); + assert_eq!(*rest, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_abbreviation_implicit_const_no_const() { + let buf = Section::new() + .abbrev(1, constants::DW_TAG_subprogram, constants::DW_CHILDREN_no) + .abbrev_attr(constants::DW_AT_name, constants::DW_FORM_implicit_const) + .get_contents() + .unwrap(); + let buf = &mut EndianSlice::new(&*buf, LittleEndian); + + match Abbreviation::parse(buf) { + Err(Error::UnexpectedEof(_)) => {} + otherwise => panic!("Unexpected result: {:?}", otherwise), + } + } + + #[test] + fn test_parse_null_abbreviation_ok() { + let expected_rest = [0x01, 0x02, 0x03, 0x04]; + let buf = Section::new() + .abbrev_null() + .append_bytes(&expected_rest) + .get_contents() + .unwrap(); + let rest = &mut EndianSlice::new(&*buf, LittleEndian); + + let abbrev = Abbreviation::parse(rest).expect("Should parse null abbreviation"); + assert!(abbrev.is_none()); + assert_eq!(*rest, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_attribute_form_ok() { + let buf = [0x01, 0x02]; + let rest = &mut EndianSlice::new(&buf, LittleEndian); + let tag = AttributeSpecification::parse_form(rest).expect("Should parse form"); + assert_eq!(tag, constants::DW_FORM_addr); + assert_eq!(*rest, EndianSlice::new(&buf[1..], LittleEndian)); + } + + #[test] + fn test_parse_attribute_form_zero() { + let buf = [0x00]; + let buf = &mut EndianSlice::new(&buf, LittleEndian); + match AttributeSpecification::parse_form(buf) { + Err(Error::AttributeFormZero) => {} + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + fn test_parse_null_attribute_specification_ok() { + let buf = [0x00, 0x00, 0x01]; + let rest = &mut EndianSlice::new(&buf, LittleEndian); + let attr = + AttributeSpecification::parse(rest).expect("Should parse null attribute specification"); + assert!(attr.is_none()); + assert_eq!(*rest, EndianSlice::new(&buf[2..], LittleEndian)); + } + + #[test] + fn test_parse_attribute_specifications_name_zero() { + let buf = [0x00, 0x01, 0x00, 0x00]; + let buf = &mut EndianSlice::new(&buf, LittleEndian); + match AttributeSpecification::parse(buf) { + Err(Error::ExpectedZero) => {} + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + fn test_parse_attribute_specifications_form_zero() { + let buf = [0x01, 0x00, 0x00, 0x00]; + let buf = &mut EndianSlice::new(&buf, LittleEndian); + match AttributeSpecification::parse(buf) { + Err(Error::AttributeFormZero) => {} + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + fn test_get_abbrev_zero() { + let mut abbrevs = Abbreviations::empty(); + abbrevs + .insert(Abbreviation::new( + 1, + constants::DwTag(1), + constants::DW_CHILDREN_no, + vec![].into(), + )) + .unwrap(); + assert!(abbrevs.get(0).is_none()); + } +} diff --git a/crux-mir/lib/gimli/src/read/addr.rs b/crux-mir/lib/gimli/src/read/addr.rs new file mode 100644 index 000000000..593f9fe3c --- /dev/null +++ b/crux-mir/lib/gimli/src/read/addr.rs @@ -0,0 +1,128 @@ +use crate::common::{DebugAddrBase, DebugAddrIndex, SectionId}; +use crate::read::{Reader, ReaderOffset, Result, Section}; + +/// The raw contents of the `.debug_addr` section. +#[derive(Debug, Default, Clone, Copy)] +pub struct DebugAddr { + section: R, +} + +impl DebugAddr { + // TODO: add an iterator over the sets of addresses in the section. + // This is not needed for common usage of the section though. + + /// Returns the address at the given `base` and `index`. + /// + /// A set of addresses in the `.debug_addr` section consists of a header + /// followed by a series of addresses. + /// + /// The `base` must be the `DW_AT_addr_base` value from the compilation unit DIE. + /// This is an offset that points to the first address following the header. + /// + /// The `index` is the value of a `DW_FORM_addrx` attribute. + /// + /// The `address_size` must be the size of the address for the compilation unit. + /// This value must also match the header. However, note that we do not parse the + /// header to validate this, since locating the header is unreliable, and the GNU + /// extensions do not emit it. + pub fn get_address( + &self, + address_size: u8, + base: DebugAddrBase, + index: DebugAddrIndex, + ) -> Result { + let input = &mut self.section.clone(); + input.skip(base.0)?; + input.skip(R::Offset::from_u64( + index.0.into_u64() * u64::from(address_size), + )?)?; + input.read_address(address_size) + } +} + +impl DebugAddr { + /// Create a `DebugAddr` section that references the data in `self`. + /// + /// This is useful when `R` implements `Reader` but `T` does not. + /// + /// ## Example Usage + /// + /// ```rust,no_run + /// # let load_section = || unimplemented!(); + /// // Read the DWARF section into a `Vec` with whatever object loader you're using. + /// let owned_section: gimli::DebugAddr> = load_section(); + /// // Create a reference to the DWARF section. + /// let section = owned_section.borrow(|section| { + /// gimli::EndianSlice::new(§ion, gimli::LittleEndian) + /// }); + /// ``` + pub fn borrow<'a, F, R>(&'a self, mut borrow: F) -> DebugAddr + where + F: FnMut(&'a T) -> R, + { + borrow(&self.section).into() + } +} + +impl Section for DebugAddr { + fn id() -> SectionId { + SectionId::DebugAddr + } + + fn reader(&self) -> &R { + &self.section + } +} + +impl From for DebugAddr { + fn from(section: R) -> Self { + DebugAddr { section } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::read::EndianSlice; + use crate::test_util::GimliSectionMethods; + use crate::{Format, LittleEndian}; + use test_assembler::{Endian, Label, LabelMaker, Section}; + + #[test] + fn test_get_address() { + for format in vec![Format::Dwarf32, Format::Dwarf64] { + for address_size in vec![4, 8] { + let zero = Label::new(); + let length = Label::new(); + let start = Label::new(); + let first = Label::new(); + let end = Label::new(); + let mut section = Section::with_endian(Endian::Little) + .mark(&zero) + .initial_length(format, &length, &start) + .D16(5) + .D8(address_size) + .D8(0) + .mark(&first); + for i in 0..20 { + section = section.word(address_size, 1000 + i); + } + section = section.mark(&end); + length.set_const((&end - &start) as u64); + + let section = section.get_contents().unwrap(); + let debug_addr = DebugAddr::from(EndianSlice::new(§ion, LittleEndian)); + let base = DebugAddrBase((&first - &zero) as usize); + + assert_eq!( + debug_addr.get_address(address_size, base, DebugAddrIndex(0)), + Ok(1000) + ); + assert_eq!( + debug_addr.get_address(address_size, base, DebugAddrIndex(19)), + Ok(1019) + ); + } + } + } +} diff --git a/crux-mir/lib/gimli/src/read/aranges.rs b/crux-mir/lib/gimli/src/read/aranges.rs new file mode 100644 index 000000000..83159b69b --- /dev/null +++ b/crux-mir/lib/gimli/src/read/aranges.rs @@ -0,0 +1,660 @@ +use crate::common::{DebugArangesOffset, DebugInfoOffset, Encoding, SectionId}; +use crate::endianity::Endianity; +use crate::read::{EndianSlice, Error, Range, Reader, ReaderOffset, Result, Section}; + +/// The `DebugAranges` struct represents the DWARF address range information +/// found in the `.debug_aranges` section. +#[derive(Debug, Default, Clone, Copy)] +pub struct DebugAranges { + section: R, +} + +impl<'input, Endian> DebugAranges> +where + Endian: Endianity, +{ + /// Construct a new `DebugAranges` instance from the data in the `.debug_aranges` + /// section. + /// + /// It is the caller's responsibility to read the `.debug_aranges` section and + /// present it as a `&[u8]` slice. That means using some ELF loader on + /// Linux, a Mach-O loader on macOS, etc. + /// + /// ``` + /// use gimli::{DebugAranges, LittleEndian}; + /// + /// # let buf = []; + /// # let read_debug_aranges_section = || &buf; + /// let debug_aranges = + /// DebugAranges::new(read_debug_aranges_section(), LittleEndian); + /// ``` + pub fn new(section: &'input [u8], endian: Endian) -> Self { + DebugAranges { + section: EndianSlice::new(section, endian), + } + } +} + +impl DebugAranges { + /// Iterate the sets of entries in the `.debug_aranges` section. + /// + /// Each set of entries belongs to a single unit. + pub fn headers(&self) -> ArangeHeaderIter { + ArangeHeaderIter { + input: self.section.clone(), + offset: DebugArangesOffset(R::Offset::from_u8(0)), + } + } + + /// Get the header at the given offset. + pub fn header(&self, offset: DebugArangesOffset) -> Result> { + let mut input = self.section.clone(); + input.skip(offset.0)?; + ArangeHeader::parse(&mut input, offset) + } +} + +impl DebugAranges { + /// Create a `DebugAranges` section that references the data in `self`. + /// + /// This is useful when `R` implements `Reader` but `T` does not. + /// + /// ## Example Usage + /// + /// ```rust,no_run + /// # let load_section = || unimplemented!(); + /// // Read the DWARF section into a `Vec` with whatever object loader you're using. + /// let owned_section: gimli::DebugAranges> = load_section(); + /// // Create a reference to the DWARF section. + /// let section = owned_section.borrow(|section| { + /// gimli::EndianSlice::new(§ion, gimli::LittleEndian) + /// }); + /// ``` + pub fn borrow<'a, F, R>(&'a self, mut borrow: F) -> DebugAranges + where + F: FnMut(&'a T) -> R, + { + borrow(&self.section).into() + } +} + +impl Section for DebugAranges { + fn id() -> SectionId { + SectionId::DebugAranges + } + + fn reader(&self) -> &R { + &self.section + } +} + +impl From for DebugAranges { + fn from(section: R) -> Self { + DebugAranges { section } + } +} + +/// An iterator over the headers of a `.debug_aranges` section. +#[derive(Clone, Debug)] +pub struct ArangeHeaderIter { + input: R, + offset: DebugArangesOffset, +} + +impl ArangeHeaderIter { + /// Advance the iterator to the next header. + pub fn next(&mut self) -> Result>> { + if self.input.is_empty() { + return Ok(None); + } + + let len = self.input.len(); + match ArangeHeader::parse(&mut self.input, self.offset) { + Ok(header) => { + self.offset.0 += len - self.input.len(); + Ok(Some(header)) + } + Err(e) => { + self.input.empty(); + Err(e) + } + } + } +} + +#[cfg(feature = "fallible-iterator")] +impl fallible_iterator::FallibleIterator for ArangeHeaderIter { + type Item = ArangeHeader; + type Error = Error; + + fn next(&mut self) -> ::core::result::Result, Self::Error> { + ArangeHeaderIter::next(self) + } +} + +/// A header for a set of entries in the `.debug_arange` section. +/// +/// These entries all belong to a single unit. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ArangeHeader::Offset> +where + R: Reader, + Offset: ReaderOffset, +{ + offset: DebugArangesOffset, + encoding: Encoding, + length: Offset, + debug_info_offset: DebugInfoOffset, + segment_size: u8, + entries: R, +} + +impl ArangeHeader +where + R: Reader, + Offset: ReaderOffset, +{ + fn parse(input: &mut R, offset: DebugArangesOffset) -> Result { + let (length, format) = input.read_initial_length()?; + let mut rest = input.split(length)?; + + // Check the version. The DWARF 5 spec says that this is always 2, but version 3 + // has been observed in the wild, potentially due to a bug; see + // https://github.com/gimli-rs/gimli/issues/559 for more information. + // lldb allows versions 2 through 5, possibly by mistake. + let version = rest.read_u16()?; + if version != 2 && version != 3 { + return Err(Error::UnknownVersion(u64::from(version))); + } + + let debug_info_offset = rest.read_offset(format).map(DebugInfoOffset)?; + let address_size = rest.read_u8()?; + let segment_size = rest.read_u8()?; + + // unit_length + version + offset + address_size + segment_size + let header_length = format.initial_length_size() + 2 + format.word_size() + 1 + 1; + + // The first tuple following the header in each set begins at an offset that is + // a multiple of the size of a single tuple (that is, the size of a segment selector + // plus twice the size of an address). + let tuple_length = address_size + .checked_mul(2) + .and_then(|x| x.checked_add(segment_size)) + .ok_or(Error::InvalidAddressRange)?; + if tuple_length == 0 { + return Err(Error::InvalidAddressRange)?; + } + let padding = if header_length % tuple_length == 0 { + 0 + } else { + tuple_length - header_length % tuple_length + }; + rest.skip(R::Offset::from_u8(padding))?; + + let encoding = Encoding { + format, + version, + address_size, + // TODO: segment_size + }; + Ok(ArangeHeader { + offset, + encoding, + length, + debug_info_offset, + segment_size, + entries: rest, + }) + } + + /// Return the offset of this header within the `.debug_aranges` section. + #[inline] + pub fn offset(&self) -> DebugArangesOffset { + self.offset + } + + /// Return the length of this set of entries, including the header. + #[inline] + pub fn length(&self) -> Offset { + self.length + } + + /// Return the encoding parameters for this set of entries. + #[inline] + pub fn encoding(&self) -> Encoding { + self.encoding + } + + /// Return the segment size for this set of entries. + #[inline] + pub fn segment_size(&self) -> u8 { + self.segment_size + } + + /// Return the offset into the .debug_info section for this set of arange entries. + #[inline] + pub fn debug_info_offset(&self) -> DebugInfoOffset { + self.debug_info_offset + } + + /// Return the arange entries in this set. + #[inline] + pub fn entries(&self) -> ArangeEntryIter { + ArangeEntryIter { + input: self.entries.clone(), + encoding: self.encoding, + segment_size: self.segment_size, + } + } +} + +/// An iterator over the aranges from a `.debug_aranges` section. +/// +/// Can be [used with +/// `FallibleIterator`](./index.html#using-with-fallibleiterator). +#[derive(Debug, Clone)] +pub struct ArangeEntryIter { + input: R, + encoding: Encoding, + segment_size: u8, +} + +impl ArangeEntryIter { + /// Advance the iterator and return the next arange. + /// + /// Returns the newly parsed arange as `Ok(Some(arange))`. Returns `Ok(None)` + /// when iteration is complete and all aranges have already been parsed and + /// yielded. If an error occurs while parsing the next arange, then this error + /// is returned as `Err(e)`, and all subsequent calls return `Ok(None)`. + pub fn next(&mut self) -> Result> { + if self.input.is_empty() { + return Ok(None); + } + + match ArangeEntry::parse(&mut self.input, self.encoding, self.segment_size) { + Ok(Some(entry)) => Ok(Some(entry)), + Ok(None) => { + self.input.empty(); + Ok(None) + } + Err(e) => { + self.input.empty(); + Err(e) + } + } + } +} + +#[cfg(feature = "fallible-iterator")] +impl fallible_iterator::FallibleIterator for ArangeEntryIter { + type Item = ArangeEntry; + type Error = Error; + + fn next(&mut self) -> ::core::result::Result, Self::Error> { + ArangeEntryIter::next(self) + } +} + +/// A single parsed arange. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct ArangeEntry { + segment: Option, + address: u64, + length: u64, +} + +impl ArangeEntry { + /// Parse a single arange. Return `None` for the null arange, `Some` for an actual arange. + fn parse( + input: &mut R, + encoding: Encoding, + segment_size: u8, + ) -> Result> { + let address_size = encoding.address_size; + + let tuple_length = R::Offset::from_u8(2 * address_size + segment_size); + if tuple_length > input.len() { + input.empty(); + return Ok(None); + } + + let segment = if segment_size != 0 { + input.read_address(segment_size)? + } else { + 0 + }; + let address = input.read_address(address_size)?; + let length = input.read_address(address_size)?; + + match (segment, address, length) { + // This is meant to be a null terminator, but in practice it can occur + // before the end, possibly due to a linker omitting a function and + // leaving an unrelocated entry. + (0, 0, 0) => Self::parse(input, encoding, segment_size), + _ => Ok(Some(ArangeEntry { + segment: if segment_size != 0 { + Some(segment) + } else { + None + }, + address, + length, + })), + } + } + + /// Return the segment selector of this arange. + #[inline] + pub fn segment(&self) -> Option { + self.segment + } + + /// Return the beginning address of this arange. + #[inline] + pub fn address(&self) -> u64 { + self.address + } + + /// Return the length of this arange. + #[inline] + pub fn length(&self) -> u64 { + self.length + } + + /// Return the range. + #[inline] + pub fn range(&self) -> Range { + Range { + begin: self.address, + end: self.address.wrapping_add(self.length), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::common::{DebugInfoOffset, Format}; + use crate::endianity::LittleEndian; + use crate::read::EndianSlice; + + #[test] + fn test_iterate_headers() { + #[rustfmt::skip] + let buf = [ + // 32-bit length = 28. + 0x1c, 0x00, 0x00, 0x00, + // Version. + 0x02, 0x00, + // Offset. + 0x01, 0x02, 0x03, 0x04, + // Address size. + 0x04, + // Segment size. + 0x00, + // Dummy padding and arange tuples. + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // 32-bit length = 36. + 0x24, 0x00, 0x00, 0x00, + // Version. + 0x02, 0x00, + // Offset. + 0x11, 0x12, 0x13, 0x14, + // Address size. + 0x04, + // Segment size. + 0x00, + // Dummy padding and arange tuples. + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + + let debug_aranges = DebugAranges::new(&buf, LittleEndian); + let mut headers = debug_aranges.headers(); + + let header = headers + .next() + .expect("should parse header ok") + .expect("should have a header"); + assert_eq!(header.offset(), DebugArangesOffset(0)); + assert_eq!(header.debug_info_offset(), DebugInfoOffset(0x0403_0201)); + + let header = headers + .next() + .expect("should parse header ok") + .expect("should have a header"); + assert_eq!(header.offset(), DebugArangesOffset(0x20)); + assert_eq!(header.debug_info_offset(), DebugInfoOffset(0x1413_1211)); + } + + #[test] + fn test_parse_header_ok() { + #[rustfmt::skip] + let buf = [ + // 32-bit length = 32. + 0x20, 0x00, 0x00, 0x00, + // Version. + 0x02, 0x00, + // Offset. + 0x01, 0x02, 0x03, 0x04, + // Address size. + 0x08, + // Segment size. + 0x04, + // Length to here = 12, tuple length = 20. + // Padding to tuple length multiple = 4. + 0x10, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + + // Dummy arange tuple data. + 0x20, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + + // Dummy next arange. + 0x30, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ]; + + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + let header = + ArangeHeader::parse(rest, DebugArangesOffset(0x10)).expect("should parse header ok"); + + assert_eq!( + *rest, + EndianSlice::new(&buf[buf.len() - 16..], LittleEndian) + ); + assert_eq!( + header, + ArangeHeader { + offset: DebugArangesOffset(0x10), + encoding: Encoding { + format: Format::Dwarf32, + version: 2, + address_size: 8, + }, + length: 0x20, + debug_info_offset: DebugInfoOffset(0x0403_0201), + segment_size: 4, + entries: EndianSlice::new(&buf[buf.len() - 32..buf.len() - 16], LittleEndian), + } + ); + } + + #[test] + fn test_parse_header_overflow_error() { + #[rustfmt::skip] + let buf = [ + // 32-bit length = 32. + 0x20, 0x00, 0x00, 0x00, + // Version. + 0x02, 0x00, + // Offset. + 0x01, 0x02, 0x03, 0x04, + // Address size. + 0xff, + // Segment size. + 0xff, + // Length to here = 12, tuple length = 20. + // Padding to tuple length multiple = 4. + 0x10, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + + // Dummy arange tuple data. + 0x20, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + + // Dummy next arange. + 0x30, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ]; + + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + let error = ArangeHeader::parse(rest, DebugArangesOffset(0x10)) + .expect_err("should fail to parse header"); + assert_eq!(error, Error::InvalidAddressRange); + } + + #[test] + fn test_parse_header_div_by_zero_error() { + #[rustfmt::skip] + let buf = [ + // 32-bit length = 32. + 0x20, 0x00, 0x00, 0x00, + // Version. + 0x02, 0x00, + // Offset. + 0x01, 0x02, 0x03, 0x04, + // Address size = 0. Could cause a division by zero if we aren't + // careful. + 0x00, + // Segment size. + 0x00, + // Length to here = 12, tuple length = 20. + // Padding to tuple length multiple = 4. + 0x10, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + + // Dummy arange tuple data. + 0x20, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + + // Dummy next arange. + 0x30, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ]; + + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + let error = ArangeHeader::parse(rest, DebugArangesOffset(0x10)) + .expect_err("should fail to parse header"); + assert_eq!(error, Error::InvalidAddressRange); + } + + #[test] + fn test_parse_entry_ok() { + let encoding = Encoding { + format: Format::Dwarf32, + version: 2, + address_size: 4, + }; + let segment_size = 0; + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09]; + let rest = &mut EndianSlice::new(&buf, LittleEndian); + let entry = + ArangeEntry::parse(rest, encoding, segment_size).expect("should parse entry ok"); + assert_eq!(*rest, EndianSlice::new(&buf[buf.len() - 1..], LittleEndian)); + assert_eq!( + entry, + Some(ArangeEntry { + segment: None, + address: 0x0403_0201, + length: 0x0807_0605, + }) + ); + } + + #[test] + fn test_parse_entry_segment() { + let encoding = Encoding { + format: Format::Dwarf32, + version: 2, + address_size: 4, + }; + let segment_size = 8; + #[rustfmt::skip] + let buf = [ + // Segment. + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + // Address. + 0x01, 0x02, 0x03, 0x04, + // Length. + 0x05, 0x06, 0x07, 0x08, + // Next tuple. + 0x09 + ]; + let rest = &mut EndianSlice::new(&buf, LittleEndian); + let entry = + ArangeEntry::parse(rest, encoding, segment_size).expect("should parse entry ok"); + assert_eq!(*rest, EndianSlice::new(&buf[buf.len() - 1..], LittleEndian)); + assert_eq!( + entry, + Some(ArangeEntry { + segment: Some(0x1817_1615_1413_1211), + address: 0x0403_0201, + length: 0x0807_0605, + }) + ); + } + + #[test] + fn test_parse_entry_zero() { + let encoding = Encoding { + format: Format::Dwarf32, + version: 2, + address_size: 4, + }; + let segment_size = 0; + #[rustfmt::skip] + let buf = [ + // Zero tuple. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // Address. + 0x01, 0x02, 0x03, 0x04, + // Length. + 0x05, 0x06, 0x07, 0x08, + // Next tuple. + 0x09 + ]; + let rest = &mut EndianSlice::new(&buf, LittleEndian); + let entry = + ArangeEntry::parse(rest, encoding, segment_size).expect("should parse entry ok"); + assert_eq!(*rest, EndianSlice::new(&buf[buf.len() - 1..], LittleEndian)); + assert_eq!( + entry, + Some(ArangeEntry { + segment: None, + address: 0x0403_0201, + length: 0x0807_0605, + }) + ); + } +} diff --git a/crux-mir/lib/gimli/src/read/cfi.rs b/crux-mir/lib/gimli/src/read/cfi.rs new file mode 100644 index 000000000..2e5167349 --- /dev/null +++ b/crux-mir/lib/gimli/src/read/cfi.rs @@ -0,0 +1,7585 @@ +#[cfg(feature = "read")] +use alloc::vec::Vec; + +use core::cmp::{Ord, Ordering}; +use core::fmt::{self, Debug}; +use core::iter::FromIterator; +use core::mem; +use core::num::Wrapping; + +use super::util::{ArrayLike, ArrayVec}; +use crate::common::{DebugFrameOffset, EhFrameOffset, Encoding, Format, Register, SectionId}; +use crate::constants::{self, DwEhPe}; +use crate::endianity::Endianity; +use crate::read::{ + EndianSlice, Error, Expression, Reader, ReaderOffset, Result, Section, StoreOnHeap, +}; + +/// `DebugFrame` contains the `.debug_frame` section's frame unwinding +/// information required to unwind to and recover registers from older frames on +/// the stack. For example, this is useful for a debugger that wants to print +/// locals in a backtrace. +/// +/// Most interesting methods are defined in the +/// [`UnwindSection`](trait.UnwindSection.html) trait. +/// +/// ### Differences between `.debug_frame` and `.eh_frame` +/// +/// While the `.debug_frame` section's information has a lot of overlap with the +/// `.eh_frame` section's information, the `.eh_frame` information tends to only +/// encode the subset of information needed for exception handling. Often, only +/// one of `.eh_frame` or `.debug_frame` will be present in an object file. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DebugFrame { + section: R, + address_size: u8, + segment_size: u8, +} + +impl DebugFrame { + /// Set the size of a target address in bytes. + /// + /// This defaults to the native word size. + /// This is only used if the CIE version is less than 4. + pub fn set_address_size(&mut self, address_size: u8) { + self.address_size = address_size + } + + /// Set the size of a segment selector in bytes. + /// + /// This defaults to 0. + /// This is only used if the CIE version is less than 4. + pub fn set_segment_size(&mut self, segment_size: u8) { + self.segment_size = segment_size + } +} + +impl<'input, Endian> DebugFrame> +where + Endian: Endianity, +{ + /// Construct a new `DebugFrame` instance from the data in the + /// `.debug_frame` section. + /// + /// It is the caller's responsibility to read the section and present it as + /// a `&[u8]` slice. That means using some ELF loader on Linux, a Mach-O + /// loader on macOS, etc. + /// + /// ``` + /// use gimli::{DebugFrame, NativeEndian}; + /// + /// // Use with `.debug_frame` + /// # let buf = [0x00, 0x01, 0x02, 0x03]; + /// # let read_debug_frame_section_somehow = || &buf; + /// let debug_frame = DebugFrame::new(read_debug_frame_section_somehow(), NativeEndian); + /// ``` + pub fn new(section: &'input [u8], endian: Endian) -> Self { + Self::from(EndianSlice::new(section, endian)) + } +} + +impl Section for DebugFrame { + fn id() -> SectionId { + SectionId::DebugFrame + } + + fn reader(&self) -> &R { + &self.section + } +} + +impl From for DebugFrame { + fn from(section: R) -> Self { + // Default to no segments and native word size. + DebugFrame { + section, + address_size: mem::size_of::() as u8, + segment_size: 0, + } + } +} + +/// `EhFrameHdr` contains the information about the `.eh_frame_hdr` section. +/// +/// A pointer to the start of the `.eh_frame` data, and optionally, a binary +/// search table of pointers to the `.eh_frame` records that are found in this section. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct EhFrameHdr(R); + +/// `ParsedEhFrameHdr` contains the parsed information from the `.eh_frame_hdr` section. +#[derive(Clone, Debug)] +pub struct ParsedEhFrameHdr { + address_size: u8, + section: R, + + eh_frame_ptr: Pointer, + fde_count: u64, + table_enc: DwEhPe, + table: R, +} + +impl<'input, Endian> EhFrameHdr> +where + Endian: Endianity, +{ + /// Constructs a new `EhFrameHdr` instance from the data in the `.eh_frame_hdr` section. + pub fn new(section: &'input [u8], endian: Endian) -> Self { + Self::from(EndianSlice::new(section, endian)) + } +} + +impl EhFrameHdr { + /// Parses this `EhFrameHdr` to a `ParsedEhFrameHdr`. + pub fn parse(&self, bases: &BaseAddresses, address_size: u8) -> Result> { + let mut reader = self.0.clone(); + let version = reader.read_u8()?; + if version != 1 { + return Err(Error::UnknownVersion(u64::from(version))); + } + + let eh_frame_ptr_enc = parse_pointer_encoding(&mut reader)?; + let fde_count_enc = parse_pointer_encoding(&mut reader)?; + let table_enc = parse_pointer_encoding(&mut reader)?; + + let parameters = PointerEncodingParameters { + bases: &bases.eh_frame_hdr, + func_base: None, + address_size, + section: &self.0, + }; + + // Omitting this pointer is not valid (defeats the purpose of .eh_frame_hdr entirely) + if eh_frame_ptr_enc == constants::DW_EH_PE_omit { + return Err(Error::CannotParseOmitPointerEncoding); + } + let eh_frame_ptr = parse_encoded_pointer(eh_frame_ptr_enc, ¶meters, &mut reader)?; + + let fde_count; + if fde_count_enc == constants::DW_EH_PE_omit || table_enc == constants::DW_EH_PE_omit { + fde_count = 0 + } else { + let ptr = parse_encoded_pointer(fde_count_enc, ¶meters, &mut reader)?; + fde_count = match ptr { + Pointer::Direct(c) => c, + Pointer::Indirect(_) => return Err(Error::UnsupportedPointerEncoding), + } + } + + Ok(ParsedEhFrameHdr { + address_size, + section: self.0.clone(), + + eh_frame_ptr, + fde_count, + table_enc, + table: reader, + }) + } +} + +impl Section for EhFrameHdr { + fn id() -> SectionId { + SectionId::EhFrameHdr + } + + fn reader(&self) -> &R { + &self.0 + } +} + +impl From for EhFrameHdr { + fn from(section: R) -> Self { + EhFrameHdr(section) + } +} + +impl ParsedEhFrameHdr { + /// Returns the address of the binary's `.eh_frame` section. + pub fn eh_frame_ptr(&self) -> Pointer { + self.eh_frame_ptr + } + + /// Retrieves the CFI binary search table, if there is one. + pub fn table(&self) -> Option> { + // There are two big edge cases here: + // * You search the table for an invalid address. As this is just a binary + // search table, we always have to return a valid result for that (unless + // you specify an address that is lower than the first address in the + // table). Since this means that you have to recheck that the FDE contains + // your address anyways, we just return the first FDE even when the address + // is too low. After all, we're just doing a normal binary search. + // * This falls apart when the table is empty - there is no entry we could + // return. We conclude that an empty table is not really a table at all. + if self.fde_count == 0 { + None + } else { + Some(EhHdrTable { hdr: self }) + } + } +} + +/// An iterator for `.eh_frame_hdr` section's binary search table. +/// +/// Each table entry consists of a tuple containing an `initial_location` and `address`. +/// The `initial location` represents the first address that the targeted FDE +/// is able to decode. The `address` is the address of the FDE in the `.eh_frame` section. +/// The `address` can be converted with `EhHdrTable::pointer_to_offset` and `EhFrame::fde_from_offset` to an FDE. +#[derive(Debug)] +pub struct EhHdrTableIter<'a, 'bases, R: Reader> { + hdr: &'a ParsedEhFrameHdr, + table: R, + bases: &'bases BaseAddresses, + remain: u64, +} + +impl<'a, 'bases, R: Reader> EhHdrTableIter<'a, 'bases, R> { + /// Yield the next entry in the `EhHdrTableIter`. + pub fn next(&mut self) -> Result> { + if self.remain == 0 { + return Ok(None); + } + + let parameters = PointerEncodingParameters { + bases: &self.bases.eh_frame_hdr, + func_base: None, + address_size: self.hdr.address_size, + section: &self.hdr.section, + }; + + self.remain -= 1; + let from = parse_encoded_pointer(self.hdr.table_enc, ¶meters, &mut self.table)?; + let to = parse_encoded_pointer(self.hdr.table_enc, ¶meters, &mut self.table)?; + Ok(Some((from, to))) + } + /// Yield the nth entry in the `EhHdrTableIter` + pub fn nth(&mut self, n: usize) -> Result> { + use core::convert::TryFrom; + let size = match self.hdr.table_enc.format() { + constants::DW_EH_PE_uleb128 | constants::DW_EH_PE_sleb128 => { + return Err(Error::VariableLengthSearchTable); + } + constants::DW_EH_PE_sdata2 | constants::DW_EH_PE_udata2 => 2, + constants::DW_EH_PE_sdata4 | constants::DW_EH_PE_udata4 => 4, + constants::DW_EH_PE_sdata8 | constants::DW_EH_PE_udata8 => 8, + _ => return Err(Error::UnknownPointerEncoding), + }; + + let row_size = size * 2; + let n = u64::try_from(n).map_err(|_| Error::UnsupportedOffset)?; + self.remain = self.remain.saturating_sub(n); + self.table.skip(R::Offset::from_u64(n * row_size)?)?; + self.next() + } +} + +#[cfg(feature = "fallible-iterator")] +impl<'a, 'bases, R: Reader> fallible_iterator::FallibleIterator for EhHdrTableIter<'a, 'bases, R> { + type Item = (Pointer, Pointer); + type Error = Error; + fn next(&mut self) -> Result> { + EhHdrTableIter::next(self) + } + + fn size_hint(&self) -> (usize, Option) { + use core::convert::TryInto; + ( + self.remain.try_into().unwrap_or(0), + self.remain.try_into().ok(), + ) + } + + fn nth(&mut self, n: usize) -> Result> { + EhHdrTableIter::nth(self, n) + } +} + +/// The CFI binary search table that is an optional part of the `.eh_frame_hdr` section. +#[derive(Debug, Clone)] +pub struct EhHdrTable<'a, R: Reader> { + hdr: &'a ParsedEhFrameHdr, +} + +impl<'a, R: Reader + 'a> EhHdrTable<'a, R> { + /// Return an iterator that can walk the `.eh_frame_hdr` table. + /// + /// Each table entry consists of a tuple containing an `initial_location` and `address`. + /// The `initial location` represents the first address that the targeted FDE + /// is able to decode. The `address` is the address of the FDE in the `.eh_frame` section. + /// The `address` can be converted with `EhHdrTable::pointer_to_offset` and `EhFrame::fde_from_offset` to an FDE. + pub fn iter<'bases>(&self, bases: &'bases BaseAddresses) -> EhHdrTableIter<'_, 'bases, R> { + EhHdrTableIter { + hdr: self.hdr, + bases, + remain: self.hdr.fde_count, + table: self.hdr.table.clone(), + } + } + /// *Probably* returns a pointer to the FDE for the given address. + /// + /// This performs a binary search, so if there is no FDE for the given address, + /// this function **will** return a pointer to any other FDE that's close by. + /// + /// To be sure, you **must** call `contains` on the FDE. + pub fn lookup(&self, address: u64, bases: &BaseAddresses) -> Result { + let size = match self.hdr.table_enc.format() { + constants::DW_EH_PE_uleb128 | constants::DW_EH_PE_sleb128 => { + return Err(Error::VariableLengthSearchTable); + } + constants::DW_EH_PE_sdata2 | constants::DW_EH_PE_udata2 => 2, + constants::DW_EH_PE_sdata4 | constants::DW_EH_PE_udata4 => 4, + constants::DW_EH_PE_sdata8 | constants::DW_EH_PE_udata8 => 8, + _ => return Err(Error::UnknownPointerEncoding), + }; + + let row_size = size * 2; + + let mut len = self.hdr.fde_count; + + let mut reader = self.hdr.table.clone(); + + let parameters = PointerEncodingParameters { + bases: &bases.eh_frame_hdr, + func_base: None, + address_size: self.hdr.address_size, + section: &self.hdr.section, + }; + + while len > 1 { + let head = reader.split(R::Offset::from_u64((len / 2) * row_size)?)?; + let tail = reader.clone(); + + let pivot = parse_encoded_pointer(self.hdr.table_enc, ¶meters, &mut reader)?; + let pivot = match pivot { + Pointer::Direct(x) => x, + Pointer::Indirect(_) => return Err(Error::UnsupportedPointerEncoding), + }; + + match pivot.cmp(&address) { + Ordering::Equal => { + reader = tail; + break; + } + Ordering::Less => { + reader = tail; + len = len - (len / 2); + } + Ordering::Greater => { + reader = head; + len /= 2; + } + } + } + + reader.skip(R::Offset::from_u64(size)?)?; + + parse_encoded_pointer(self.hdr.table_enc, ¶meters, &mut reader) + } + + /// Convert a `Pointer` to a section offset. + /// + /// This does not support indirect pointers. + pub fn pointer_to_offset(&self, ptr: Pointer) -> Result> { + let ptr = match ptr { + Pointer::Direct(x) => x, + _ => return Err(Error::UnsupportedPointerEncoding), + }; + + let eh_frame_ptr = match self.hdr.eh_frame_ptr() { + Pointer::Direct(x) => x, + _ => return Err(Error::UnsupportedPointerEncoding), + }; + + // Calculate the offset in the EhFrame section + R::Offset::from_u64(ptr - eh_frame_ptr).map(EhFrameOffset) + } + + /// Returns a parsed FDE for the given address, or `NoUnwindInfoForAddress` + /// if there are none. + /// + /// You must provide a function to get its associated CIE. See + /// `PartialFrameDescriptionEntry::parse` for more information. + /// + /// # Example + /// + /// ``` + /// # use gimli::{BaseAddresses, EhFrame, ParsedEhFrameHdr, EndianSlice, NativeEndian, Error, UnwindSection}; + /// # fn foo() -> Result<(), Error> { + /// # let eh_frame: EhFrame> = unreachable!(); + /// # let eh_frame_hdr: ParsedEhFrameHdr> = unimplemented!(); + /// # let addr = 0; + /// # let bases = unimplemented!(); + /// let table = eh_frame_hdr.table().unwrap(); + /// let fde = table.fde_for_address(&eh_frame, &bases, addr, EhFrame::cie_from_offset)?; + /// # Ok(()) + /// # } + /// ``` + pub fn fde_for_address( + &self, + frame: &EhFrame, + bases: &BaseAddresses, + address: u64, + get_cie: F, + ) -> Result> + where + F: FnMut( + &EhFrame, + &BaseAddresses, + EhFrameOffset, + ) -> Result>, + { + let fdeptr = self.lookup(address, bases)?; + let offset = self.pointer_to_offset(fdeptr)?; + let entry = frame.fde_from_offset(bases, offset, get_cie)?; + if entry.contains(address) { + Ok(entry) + } else { + Err(Error::NoUnwindInfoForAddress) + } + } + + #[inline] + #[doc(hidden)] + #[deprecated(note = "Method renamed to fde_for_address; use that instead.")] + pub fn lookup_and_parse( + &self, + address: u64, + bases: &BaseAddresses, + frame: EhFrame, + get_cie: F, + ) -> Result> + where + F: FnMut( + &EhFrame, + &BaseAddresses, + EhFrameOffset, + ) -> Result>, + { + self.fde_for_address(&frame, bases, address, get_cie) + } + + /// Returns the frame unwind information for the given address, + /// or `NoUnwindInfoForAddress` if there are none. + /// + /// You must provide a function to get the associated CIE. See + /// `PartialFrameDescriptionEntry::parse` for more information. + pub fn unwind_info_for_address<'ctx, F, A: UnwindContextStorage>( + &self, + frame: &EhFrame, + bases: &BaseAddresses, + ctx: &'ctx mut UnwindContext, + address: u64, + get_cie: F, + ) -> Result<&'ctx UnwindTableRow> + where + F: FnMut( + &EhFrame, + &BaseAddresses, + EhFrameOffset, + ) -> Result>, + { + let fde = self.fde_for_address(frame, bases, address, get_cie)?; + fde.unwind_info_for_address(frame, bases, ctx, address) + } +} + +/// `EhFrame` contains the frame unwinding information needed during exception +/// handling found in the `.eh_frame` section. +/// +/// Most interesting methods are defined in the +/// [`UnwindSection`](trait.UnwindSection.html) trait. +/// +/// See +/// [`DebugFrame`](./struct.DebugFrame.html#differences-between-debug_frame-and-eh_frame) +/// for some discussion on the differences between `.debug_frame` and +/// `.eh_frame`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct EhFrame { + section: R, + address_size: u8, +} + +impl EhFrame { + /// Set the size of a target address in bytes. + /// + /// This defaults to the native word size. + pub fn set_address_size(&mut self, address_size: u8) { + self.address_size = address_size + } +} + +impl<'input, Endian> EhFrame> +where + Endian: Endianity, +{ + /// Construct a new `EhFrame` instance from the data in the + /// `.eh_frame` section. + /// + /// It is the caller's responsibility to read the section and present it as + /// a `&[u8]` slice. That means using some ELF loader on Linux, a Mach-O + /// loader on macOS, etc. + /// + /// ``` + /// use gimli::{EhFrame, EndianSlice, NativeEndian}; + /// + /// // Use with `.eh_frame` + /// # let buf = [0x00, 0x01, 0x02, 0x03]; + /// # let read_eh_frame_section_somehow = || &buf; + /// let eh_frame = EhFrame::new(read_eh_frame_section_somehow(), NativeEndian); + /// ``` + pub fn new(section: &'input [u8], endian: Endian) -> Self { + Self::from(EndianSlice::new(section, endian)) + } +} + +impl Section for EhFrame { + fn id() -> SectionId { + SectionId::EhFrame + } + + fn reader(&self) -> &R { + &self.section + } +} + +impl From for EhFrame { + fn from(section: R) -> Self { + // Default to native word size. + EhFrame { + section, + address_size: mem::size_of::() as u8, + } + } +} + +// This has to be `pub` to silence a warning (that is deny(..)'d by default) in +// rustc. Eventually, not having this `pub` will become a hard error. +#[doc(hidden)] +#[allow(missing_docs)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CieOffsetEncoding { + U32, + U64, +} + +/// An offset into an `UnwindSection`. +// +// Needed to avoid conflicting implementations of `Into`. +pub trait UnwindOffset: Copy + Debug + Eq + From +where + T: ReaderOffset, +{ + /// Convert an `UnwindOffset` into a `T`. + fn into(self) -> T; +} + +impl UnwindOffset for DebugFrameOffset +where + T: ReaderOffset, +{ + #[inline] + fn into(self) -> T { + self.0 + } +} + +impl UnwindOffset for EhFrameOffset +where + T: ReaderOffset, +{ + #[inline] + fn into(self) -> T { + self.0 + } +} + +/// This trait completely encapsulates everything that is different between +/// `.eh_frame` and `.debug_frame`, as well as all the bits that can change +/// between DWARF versions. +#[doc(hidden)] +pub trait _UnwindSectionPrivate { + /// Get the underlying section data. + fn section(&self) -> &R; + + /// Returns true if the given length value should be considered an + /// end-of-entries sentinel. + fn length_value_is_end_of_entries(length: R::Offset) -> bool; + + /// Return true if the given offset if the CIE sentinel, false otherwise. + fn is_cie(format: Format, id: u64) -> bool; + + /// Return the CIE offset/ID encoding used by this unwind section with the + /// given DWARF format. + fn cie_offset_encoding(format: Format) -> CieOffsetEncoding; + + /// For `.eh_frame`, CIE offsets are relative to the current position. For + /// `.debug_frame`, they are relative to the start of the section. We always + /// internally store them relative to the section, so we handle translating + /// `.eh_frame`'s relative offsets in this method. If the offset calculation + /// underflows, return `None`. + fn resolve_cie_offset(&self, base: R::Offset, offset: R::Offset) -> Option; + + /// Does this version of this unwind section encode address and segment + /// sizes in its CIEs? + fn has_address_and_segment_sizes(version: u8) -> bool; + + /// The address size to use if `has_address_and_segment_sizes` returns false. + fn address_size(&self) -> u8; + + /// The segment size to use if `has_address_and_segment_sizes` returns false. + fn segment_size(&self) -> u8; +} + +/// A section holding unwind information: either `.debug_frame` or +/// `.eh_frame`. See [`DebugFrame`](./struct.DebugFrame.html) and +/// [`EhFrame`](./struct.EhFrame.html) respectively. +pub trait UnwindSection: Clone + Debug + _UnwindSectionPrivate { + /// The offset type associated with this CFI section. Either + /// `DebugFrameOffset` or `EhFrameOffset`. + type Offset: UnwindOffset; + + /// Iterate over the `CommonInformationEntry`s and `FrameDescriptionEntry`s + /// in this `.debug_frame` section. + /// + /// Can be [used with + /// `FallibleIterator`](./index.html#using-with-fallibleiterator). + fn entries<'bases>(&self, bases: &'bases BaseAddresses) -> CfiEntriesIter<'bases, Self, R> { + CfiEntriesIter { + section: self.clone(), + bases, + input: self.section().clone(), + } + } + + /// Parse the `CommonInformationEntry` at the given offset. + fn cie_from_offset( + &self, + bases: &BaseAddresses, + offset: Self::Offset, + ) -> Result> { + let offset = UnwindOffset::into(offset); + let input = &mut self.section().clone(); + input.skip(offset)?; + CommonInformationEntry::parse(bases, self, input) + } + + /// Parse the `PartialFrameDescriptionEntry` at the given offset. + fn partial_fde_from_offset<'bases>( + &self, + bases: &'bases BaseAddresses, + offset: Self::Offset, + ) -> Result> { + let offset = UnwindOffset::into(offset); + let input = &mut self.section().clone(); + input.skip(offset)?; + PartialFrameDescriptionEntry::parse_partial(self, bases, input) + } + + /// Parse the `FrameDescriptionEntry` at the given offset. + fn fde_from_offset( + &self, + bases: &BaseAddresses, + offset: Self::Offset, + get_cie: F, + ) -> Result> + where + F: FnMut(&Self, &BaseAddresses, Self::Offset) -> Result>, + { + let partial = self.partial_fde_from_offset(bases, offset)?; + partial.parse(get_cie) + } + + /// Find the `FrameDescriptionEntry` for the given address. + /// + /// If found, the FDE is returned. If not found, + /// `Err(gimli::Error::NoUnwindInfoForAddress)` is returned. + /// If parsing fails, the error is returned. + /// + /// You must provide a function to get its associated CIE. See + /// `PartialFrameDescriptionEntry::parse` for more information. + /// + /// Note: this iterates over all FDEs. If available, it is possible + /// to do a binary search with `EhFrameHdr::fde_for_address` instead. + fn fde_for_address( + &self, + bases: &BaseAddresses, + address: u64, + mut get_cie: F, + ) -> Result> + where + F: FnMut(&Self, &BaseAddresses, Self::Offset) -> Result>, + { + let mut entries = self.entries(bases); + while let Some(entry) = entries.next()? { + match entry { + CieOrFde::Cie(_) => {} + CieOrFde::Fde(partial) => { + let fde = partial.parse(&mut get_cie)?; + if fde.contains(address) { + return Ok(fde); + } + } + } + } + Err(Error::NoUnwindInfoForAddress) + } + + /// Find the frame unwind information for the given address. + /// + /// If found, the unwind information is returned. If not found, + /// `Err(gimli::Error::NoUnwindInfoForAddress)` is returned. If parsing or + /// CFI evaluation fails, the error is returned. + /// + /// ``` + /// use gimli::{BaseAddresses, EhFrame, EndianSlice, NativeEndian, UnwindContext, + /// UnwindSection}; + /// + /// # fn foo() -> gimli::Result<()> { + /// # let read_eh_frame_section = || unimplemented!(); + /// // Get the `.eh_frame` section from the object file. Alternatively, + /// // use `EhFrame` with the `.eh_frame` section of the object file. + /// let eh_frame = EhFrame::new(read_eh_frame_section(), NativeEndian); + /// + /// # let get_frame_pc = || unimplemented!(); + /// // Get the address of the PC for a frame you'd like to unwind. + /// let address = get_frame_pc(); + /// + /// // This context is reusable, which cuts down on heap allocations. + /// let ctx = UnwindContext::new(); + /// + /// // Optionally provide base addresses for any relative pointers. If a + /// // base address isn't provided and a pointer is found that is relative to + /// // it, we will return an `Err`. + /// # let address_of_text_section_in_memory = unimplemented!(); + /// # let address_of_got_section_in_memory = unimplemented!(); + /// let bases = BaseAddresses::default() + /// .set_text(address_of_text_section_in_memory) + /// .set_got(address_of_got_section_in_memory); + /// + /// let unwind_info = eh_frame.unwind_info_for_address( + /// &bases, + /// &mut ctx, + /// address, + /// EhFrame::cie_from_offset, + /// )?; + /// + /// # let do_stuff_with = |_| unimplemented!(); + /// do_stuff_with(unwind_info); + /// # let _ = ctx; + /// # unreachable!() + /// # } + /// ``` + #[inline] + fn unwind_info_for_address<'ctx, F, A: UnwindContextStorage>( + &self, + bases: &BaseAddresses, + ctx: &'ctx mut UnwindContext, + address: u64, + get_cie: F, + ) -> Result<&'ctx UnwindTableRow> + where + F: FnMut(&Self, &BaseAddresses, Self::Offset) -> Result>, + { + let fde = self.fde_for_address(bases, address, get_cie)?; + fde.unwind_info_for_address(self, bases, ctx, address) + } +} + +impl _UnwindSectionPrivate for DebugFrame { + fn section(&self) -> &R { + &self.section + } + + fn length_value_is_end_of_entries(_: R::Offset) -> bool { + false + } + + fn is_cie(format: Format, id: u64) -> bool { + match format { + Format::Dwarf32 => id == 0xffff_ffff, + Format::Dwarf64 => id == 0xffff_ffff_ffff_ffff, + } + } + + fn cie_offset_encoding(format: Format) -> CieOffsetEncoding { + match format { + Format::Dwarf32 => CieOffsetEncoding::U32, + Format::Dwarf64 => CieOffsetEncoding::U64, + } + } + + fn resolve_cie_offset(&self, _: R::Offset, offset: R::Offset) -> Option { + Some(offset) + } + + fn has_address_and_segment_sizes(version: u8) -> bool { + version == 4 + } + + fn address_size(&self) -> u8 { + self.address_size + } + + fn segment_size(&self) -> u8 { + self.segment_size + } +} + +impl UnwindSection for DebugFrame { + type Offset = DebugFrameOffset; +} + +impl _UnwindSectionPrivate for EhFrame { + fn section(&self) -> &R { + &self.section + } + + fn length_value_is_end_of_entries(length: R::Offset) -> bool { + length.into_u64() == 0 + } + + fn is_cie(_: Format, id: u64) -> bool { + id == 0 + } + + fn cie_offset_encoding(_format: Format) -> CieOffsetEncoding { + // `.eh_frame` offsets are always 4 bytes, regardless of the DWARF + // format. + CieOffsetEncoding::U32 + } + + fn resolve_cie_offset(&self, base: R::Offset, offset: R::Offset) -> Option { + base.checked_sub(offset) + } + + fn has_address_and_segment_sizes(_version: u8) -> bool { + false + } + + fn address_size(&self) -> u8 { + self.address_size + } + + fn segment_size(&self) -> u8 { + 0 + } +} + +impl UnwindSection for EhFrame { + type Offset = EhFrameOffset; +} + +/// Optional base addresses for the relative `DW_EH_PE_*` encoded pointers. +/// +/// During CIE/FDE parsing, if a relative pointer is encountered for a base +/// address that is unknown, an Err will be returned. +/// +/// ``` +/// use gimli::BaseAddresses; +/// +/// # fn foo() { +/// # let address_of_eh_frame_hdr_section_in_memory = unimplemented!(); +/// # let address_of_eh_frame_section_in_memory = unimplemented!(); +/// # let address_of_text_section_in_memory = unimplemented!(); +/// # let address_of_got_section_in_memory = unimplemented!(); +/// # let address_of_the_start_of_current_func = unimplemented!(); +/// let bases = BaseAddresses::default() +/// .set_eh_frame_hdr(address_of_eh_frame_hdr_section_in_memory) +/// .set_eh_frame(address_of_eh_frame_section_in_memory) +/// .set_text(address_of_text_section_in_memory) +/// .set_got(address_of_got_section_in_memory); +/// # let _ = bases; +/// # } +/// ``` +#[derive(Clone, Default, Debug, PartialEq, Eq)] +pub struct BaseAddresses { + /// The base addresses to use for pointers in the `.eh_frame_hdr` section. + pub eh_frame_hdr: SectionBaseAddresses, + + /// The base addresses to use for pointers in the `.eh_frame` section. + pub eh_frame: SectionBaseAddresses, +} + +/// Optional base addresses for the relative `DW_EH_PE_*` encoded pointers +/// in a particular section. +/// +/// See `BaseAddresses` for methods that are helpful in setting these addresses. +#[derive(Clone, Default, Debug, PartialEq, Eq)] +pub struct SectionBaseAddresses { + /// The address of the section containing the pointer. + pub section: Option, + + /// The base address for text relative pointers. + /// This is generally the address of the `.text` section. + pub text: Option, + + /// The base address for data relative pointers. + /// + /// For pointers in the `.eh_frame_hdr` section, this is the address + /// of the `.eh_frame_hdr` section + /// + /// For pointers in the `.eh_frame` section, this is generally the + /// global pointer, such as the address of the `.got` section. + pub data: Option, +} + +impl BaseAddresses { + /// Set the `.eh_frame_hdr` section base address. + #[inline] + pub fn set_eh_frame_hdr(mut self, addr: u64) -> Self { + self.eh_frame_hdr.section = Some(addr); + self.eh_frame_hdr.data = Some(addr); + self + } + + /// Set the `.eh_frame` section base address. + #[inline] + pub fn set_eh_frame(mut self, addr: u64) -> Self { + self.eh_frame.section = Some(addr); + self + } + + /// Set the `.text` section base address. + #[inline] + pub fn set_text(mut self, addr: u64) -> Self { + self.eh_frame_hdr.text = Some(addr); + self.eh_frame.text = Some(addr); + self + } + + /// Set the `.got` section base address. + #[inline] + pub fn set_got(mut self, addr: u64) -> Self { + self.eh_frame.data = Some(addr); + self + } +} + +/// An iterator over CIE and FDE entries in a `.debug_frame` or `.eh_frame` +/// section. +/// +/// Some pointers may be encoded relative to various base addresses. Use the +/// [`BaseAddresses`](./struct.BaseAddresses.html) parameter to provide them. By +/// default, none are provided. If a relative pointer is encountered for a base +/// address that is unknown, an `Err` will be returned and iteration will abort. +/// +/// Can be [used with +/// `FallibleIterator`](./index.html#using-with-fallibleiterator). +/// +/// ``` +/// use gimli::{BaseAddresses, EhFrame, EndianSlice, NativeEndian, UnwindSection}; +/// +/// # fn foo() -> gimli::Result<()> { +/// # let read_eh_frame_somehow = || unimplemented!(); +/// let eh_frame = EhFrame::new(read_eh_frame_somehow(), NativeEndian); +/// +/// # let address_of_eh_frame_hdr_section_in_memory = unimplemented!(); +/// # let address_of_eh_frame_section_in_memory = unimplemented!(); +/// # let address_of_text_section_in_memory = unimplemented!(); +/// # let address_of_got_section_in_memory = unimplemented!(); +/// # let address_of_the_start_of_current_func = unimplemented!(); +/// // Provide base addresses for relative pointers. +/// let bases = BaseAddresses::default() +/// .set_eh_frame_hdr(address_of_eh_frame_hdr_section_in_memory) +/// .set_eh_frame(address_of_eh_frame_section_in_memory) +/// .set_text(address_of_text_section_in_memory) +/// .set_got(address_of_got_section_in_memory); +/// +/// let mut entries = eh_frame.entries(&bases); +/// +/// # let do_stuff_with = |_| unimplemented!(); +/// while let Some(entry) = entries.next()? { +/// do_stuff_with(entry) +/// } +/// # unreachable!() +/// # } +/// ``` +#[derive(Clone, Debug)] +pub struct CfiEntriesIter<'bases, Section, R> +where + R: Reader, + Section: UnwindSection, +{ + section: Section, + bases: &'bases BaseAddresses, + input: R, +} + +impl<'bases, Section, R> CfiEntriesIter<'bases, Section, R> +where + R: Reader, + Section: UnwindSection, +{ + /// Advance the iterator to the next entry. + pub fn next(&mut self) -> Result>> { + if self.input.is_empty() { + return Ok(None); + } + + match parse_cfi_entry(self.bases, &self.section, &mut self.input) { + Err(e) => { + self.input.empty(); + Err(e) + } + Ok(None) => { + self.input.empty(); + Ok(None) + } + Ok(Some(entry)) => Ok(Some(entry)), + } + } +} + +#[cfg(feature = "fallible-iterator")] +impl<'bases, Section, R> fallible_iterator::FallibleIterator for CfiEntriesIter<'bases, Section, R> +where + R: Reader, + Section: UnwindSection, +{ + type Item = CieOrFde<'bases, Section, R>; + type Error = Error; + + fn next(&mut self) -> ::core::result::Result, Self::Error> { + CfiEntriesIter::next(self) + } +} + +/// Either a `CommonInformationEntry` (CIE) or a `FrameDescriptionEntry` (FDE). +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CieOrFde<'bases, Section, R> +where + R: Reader, + Section: UnwindSection, +{ + /// This CFI entry is a `CommonInformationEntry`. + Cie(CommonInformationEntry), + /// This CFI entry is a `FrameDescriptionEntry`, however fully parsing it + /// requires parsing its CIE first, so it is left in a partially parsed + /// state. + Fde(PartialFrameDescriptionEntry<'bases, Section, R>), +} + +#[allow(clippy::type_complexity)] +fn parse_cfi_entry<'bases, Section, R>( + bases: &'bases BaseAddresses, + section: &Section, + input: &mut R, +) -> Result>> +where + R: Reader, + Section: UnwindSection, +{ + let (offset, length, format) = loop { + let offset = input.offset_from(section.section()); + let (length, format) = input.read_initial_length()?; + + if Section::length_value_is_end_of_entries(length) { + return Ok(None); + } + + // Hack: skip zero padding inserted by buggy compilers/linkers. + // We require that the padding is a multiple of 32-bits, otherwise + // there is no reliable way to determine when the padding ends. This + // should be okay since CFI entries must be aligned to the address size. + + if length.into_u64() != 0 || format != Format::Dwarf32 { + break (offset, length, format); + } + }; + + let mut rest = input.split(length)?; + let cie_offset_base = rest.offset_from(section.section()); + let cie_id_or_offset = match Section::cie_offset_encoding(format) { + CieOffsetEncoding::U32 => rest.read_u32().map(u64::from)?, + CieOffsetEncoding::U64 => rest.read_u64()?, + }; + + if Section::is_cie(format, cie_id_or_offset) { + let cie = CommonInformationEntry::parse_rest(offset, length, format, bases, section, rest)?; + Ok(Some(CieOrFde::Cie(cie))) + } else { + let cie_offset = R::Offset::from_u64(cie_id_or_offset)?; + let cie_offset = match section.resolve_cie_offset(cie_offset_base, cie_offset) { + None => return Err(Error::OffsetOutOfBounds), + Some(cie_offset) => cie_offset, + }; + + let fde = PartialFrameDescriptionEntry { + offset, + length, + format, + cie_offset: cie_offset.into(), + rest, + section: section.clone(), + bases, + }; + + Ok(Some(CieOrFde::Fde(fde))) + } +} + +/// We support the z-style augmentation [defined by `.eh_frame`][ehframe]. +/// +/// [ehframe]: https://refspecs.linuxfoundation.org/LSB_3.0.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub struct Augmentation { + /// > A 'L' may be present at any position after the first character of the + /// > string. This character may only be present if 'z' is the first character + /// > of the string. If present, it indicates the presence of one argument in + /// > the Augmentation Data of the CIE, and a corresponding argument in the + /// > Augmentation Data of the FDE. The argument in the Augmentation Data of + /// > the CIE is 1-byte and represents the pointer encoding used for the + /// > argument in the Augmentation Data of the FDE, which is the address of a + /// > language-specific data area (LSDA). The size of the LSDA pointer is + /// > specified by the pointer encoding used. + lsda: Option, + + /// > A 'P' may be present at any position after the first character of the + /// > string. This character may only be present if 'z' is the first character + /// > of the string. If present, it indicates the presence of two arguments in + /// > the Augmentation Data of the CIE. The first argument is 1-byte and + /// > represents the pointer encoding used for the second argument, which is + /// > the address of a personality routine handler. The size of the + /// > personality routine pointer is specified by the pointer encoding used. + personality: Option<(constants::DwEhPe, Pointer)>, + + /// > A 'R' may be present at any position after the first character of the + /// > string. This character may only be present if 'z' is the first character + /// > of the string. If present, The Augmentation Data shall include a 1 byte + /// > argument that represents the pointer encoding for the address pointers + /// > used in the FDE. + fde_address_encoding: Option, + + /// True if this CIE's FDEs are trampolines for signal handlers. + is_signal_trampoline: bool, +} + +impl Augmentation { + fn parse( + augmentation_str: &mut R, + bases: &BaseAddresses, + address_size: u8, + section: &Section, + input: &mut R, + ) -> Result + where + R: Reader, + Section: UnwindSection, + { + debug_assert!( + !augmentation_str.is_empty(), + "Augmentation::parse should only be called if we have an augmentation" + ); + + let mut augmentation = Augmentation::default(); + + let mut parsed_first = false; + let mut data = None; + + while !augmentation_str.is_empty() { + let ch = augmentation_str.read_u8()?; + match ch { + b'z' => { + if parsed_first { + return Err(Error::UnknownAugmentation); + } + + let augmentation_length = input.read_uleb128().and_then(R::Offset::from_u64)?; + data = Some(input.split(augmentation_length)?); + } + b'L' => { + let rest = data.as_mut().ok_or(Error::UnknownAugmentation)?; + let encoding = parse_pointer_encoding(rest)?; + augmentation.lsda = Some(encoding); + } + b'P' => { + let rest = data.as_mut().ok_or(Error::UnknownAugmentation)?; + let encoding = parse_pointer_encoding(rest)?; + let parameters = PointerEncodingParameters { + bases: &bases.eh_frame, + func_base: None, + address_size, + section: section.section(), + }; + + let personality = parse_encoded_pointer(encoding, ¶meters, rest)?; + augmentation.personality = Some((encoding, personality)); + } + b'R' => { + let rest = data.as_mut().ok_or(Error::UnknownAugmentation)?; + let encoding = parse_pointer_encoding(rest)?; + augmentation.fde_address_encoding = Some(encoding); + } + b'S' => augmentation.is_signal_trampoline = true, + _ => return Err(Error::UnknownAugmentation), + } + + parsed_first = true; + } + + Ok(augmentation) + } +} + +/// Parsed augmentation data for a `FrameDescriptEntry`. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct AugmentationData { + lsda: Option, +} + +impl AugmentationData { + fn parse( + augmentation: &Augmentation, + encoding_parameters: &PointerEncodingParameters, + input: &mut R, + ) -> Result { + // In theory, we should be iterating over the original augmentation + // string, interpreting each character, and reading the appropriate bits + // out of the augmentation data as we go. However, the only character + // that defines augmentation data in the FDE is the 'L' character, so we + // can just check for its presence directly. + + let aug_data_len = input.read_uleb128().and_then(R::Offset::from_u64)?; + let rest = &mut input.split(aug_data_len)?; + let mut augmentation_data = AugmentationData::default(); + if let Some(encoding) = augmentation.lsda { + let lsda = parse_encoded_pointer(encoding, encoding_parameters, rest)?; + augmentation_data.lsda = Some(lsda); + } + Ok(augmentation_data) + } +} + +/// > A Common Information Entry holds information that is shared among many +/// > Frame Description Entries. There is at least one CIE in every non-empty +/// > `.debug_frame` section. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CommonInformationEntry::Offset> +where + R: Reader, + Offset: ReaderOffset, +{ + /// The offset of this entry from the start of its containing section. + offset: Offset, + + /// > A constant that gives the number of bytes of the CIE structure, not + /// > including the length field itself (see Section 7.2.2). The size of the + /// > length field plus the value of length must be an integral multiple of + /// > the address size. + length: Offset, + + format: Format, + + /// > A version number (see Section 7.23). This number is specific to the + /// > call frame information and is independent of the DWARF version number. + version: u8, + + /// The parsed augmentation, if any. + augmentation: Option, + + /// > The size of a target address in this CIE and any FDEs that use it, in + /// > bytes. If a compilation unit exists for this frame, its address size + /// > must match the address size here. + address_size: u8, + + /// "The size of a segment selector in this CIE and any FDEs that use it, in + /// bytes." + segment_size: u8, + + /// "A constant that is factored out of all advance location instructions + /// (see Section 6.4.2.1)." + code_alignment_factor: u64, + + /// > A constant that is factored out of certain offset instructions (see + /// > below). The resulting value is (operand * data_alignment_factor). + data_alignment_factor: i64, + + /// > An unsigned LEB128 constant that indicates which column in the rule + /// > table represents the return address of the function. Note that this + /// > column might not correspond to an actual machine register. + return_address_register: Register, + + /// > A sequence of rules that are interpreted to create the initial setting + /// > of each column in the table. + /// + /// > The default rule for all columns before interpretation of the initial + /// > instructions is the undefined rule. However, an ABI authoring body or a + /// > compilation system authoring body may specify an alternate default + /// > value for any or all columns. + /// + /// This is followed by `DW_CFA_nop` padding until the end of `length` bytes + /// in the input. + initial_instructions: R, +} + +impl CommonInformationEntry { + fn parse>( + bases: &BaseAddresses, + section: &Section, + input: &mut R, + ) -> Result> { + match parse_cfi_entry(bases, section, input)? { + Some(CieOrFde::Cie(cie)) => Ok(cie), + Some(CieOrFde::Fde(_)) => Err(Error::NotCieId), + None => Err(Error::NoEntryAtGivenOffset), + } + } + + fn parse_rest>( + offset: R::Offset, + length: R::Offset, + format: Format, + bases: &BaseAddresses, + section: &Section, + mut rest: R, + ) -> Result> { + let version = rest.read_u8()?; + + // Version 1 of `.debug_frame` corresponds to DWARF 2, and then for + // DWARF 3 and 4, I think they decided to just match the standard's + // version. + match version { + 1 | 3 | 4 => (), + _ => return Err(Error::UnknownVersion(u64::from(version))), + } + + let mut augmentation_string = rest.read_null_terminated_slice()?; + + let (address_size, segment_size) = if Section::has_address_and_segment_sizes(version) { + let address_size = rest.read_u8()?; + let segment_size = rest.read_u8()?; + (address_size, segment_size) + } else { + (section.address_size(), section.segment_size()) + }; + + let code_alignment_factor = rest.read_uleb128()?; + let data_alignment_factor = rest.read_sleb128()?; + + let return_address_register = if version == 1 { + Register(rest.read_u8()?.into()) + } else { + rest.read_uleb128().and_then(Register::from_u64)? + }; + + let augmentation = if augmentation_string.is_empty() { + None + } else { + Some(Augmentation::parse( + &mut augmentation_string, + bases, + address_size, + section, + &mut rest, + )?) + }; + + let entry = CommonInformationEntry { + offset, + length, + format, + version, + augmentation, + address_size, + segment_size, + code_alignment_factor, + data_alignment_factor, + return_address_register, + initial_instructions: rest, + }; + + Ok(entry) + } +} + +/// # Signal Safe Methods +/// +/// These methods are guaranteed not to allocate, acquire locks, or perform any +/// other signal-unsafe operations. +impl CommonInformationEntry { + /// Get the offset of this entry from the start of its containing section. + pub fn offset(&self) -> R::Offset { + self.offset + } + + /// Return the encoding parameters for this CIE. + pub fn encoding(&self) -> Encoding { + Encoding { + format: self.format, + version: u16::from(self.version), + address_size: self.address_size, + } + } + + /// The size of addresses (in bytes) in this CIE. + pub fn address_size(&self) -> u8 { + self.address_size + } + + /// Iterate over this CIE's initial instructions. + /// + /// Can be [used with + /// `FallibleIterator`](./index.html#using-with-fallibleiterator). + pub fn instructions<'a, Section>( + &self, + section: &'a Section, + bases: &'a BaseAddresses, + ) -> CallFrameInstructionIter<'a, R> + where + Section: UnwindSection, + { + CallFrameInstructionIter { + input: self.initial_instructions.clone(), + address_encoding: None, + parameters: PointerEncodingParameters { + bases: &bases.eh_frame, + func_base: None, + address_size: self.address_size, + section: section.section(), + }, + } + } + + /// > A constant that gives the number of bytes of the CIE structure, not + /// > including the length field itself (see Section 7.2.2). The size of the + /// > length field plus the value of length must be an integral multiple of + /// > the address size. + pub fn entry_len(&self) -> R::Offset { + self.length + } + + /// > A version number (see Section 7.23). This number is specific to the + /// > call frame information and is independent of the DWARF version number. + pub fn version(&self) -> u8 { + self.version + } + + /// Get the augmentation data, if any exists. + /// + /// The only augmentation understood by `gimli` is that which is defined by + /// `.eh_frame`. + pub fn augmentation(&self) -> Option<&Augmentation> { + self.augmentation.as_ref() + } + + /// True if this CIE's FDEs have a LSDA. + pub fn has_lsda(&self) -> bool { + self.augmentation.map_or(false, |a| a.lsda.is_some()) + } + + /// Return the encoding of the LSDA address for this CIE's FDEs. + pub fn lsda_encoding(&self) -> Option { + self.augmentation.and_then(|a| a.lsda) + } + + /// Return the encoding and address of the personality routine handler + /// for this CIE's FDEs. + pub fn personality_with_encoding(&self) -> Option<(constants::DwEhPe, Pointer)> { + self.augmentation.as_ref().and_then(|a| a.personality) + } + + /// Return the address of the personality routine handler + /// for this CIE's FDEs. + pub fn personality(&self) -> Option { + self.augmentation + .as_ref() + .and_then(|a| a.personality) + .map(|(_, p)| p) + } + + /// Return the encoding of the addresses for this CIE's FDEs. + pub fn fde_address_encoding(&self) -> Option { + self.augmentation.and_then(|a| a.fde_address_encoding) + } + + /// True if this CIE's FDEs are trampolines for signal handlers. + pub fn is_signal_trampoline(&self) -> bool { + self.augmentation.map_or(false, |a| a.is_signal_trampoline) + } + + /// > A constant that is factored out of all advance location instructions + /// > (see Section 6.4.2.1). + pub fn code_alignment_factor(&self) -> u64 { + self.code_alignment_factor + } + + /// > A constant that is factored out of certain offset instructions (see + /// > below). The resulting value is (operand * data_alignment_factor). + pub fn data_alignment_factor(&self) -> i64 { + self.data_alignment_factor + } + + /// > An unsigned ... constant that indicates which column in the rule + /// > table represents the return address of the function. Note that this + /// > column might not correspond to an actual machine register. + pub fn return_address_register(&self) -> Register { + self.return_address_register + } +} + +/// A partially parsed `FrameDescriptionEntry`. +/// +/// Fully parsing this FDE requires first parsing its CIE. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PartialFrameDescriptionEntry<'bases, Section, R> +where + R: Reader, + Section: UnwindSection, +{ + offset: R::Offset, + length: R::Offset, + format: Format, + cie_offset: Section::Offset, + rest: R, + section: Section, + bases: &'bases BaseAddresses, +} + +impl<'bases, Section, R> PartialFrameDescriptionEntry<'bases, Section, R> +where + R: Reader, + Section: UnwindSection, +{ + fn parse_partial( + section: &Section, + bases: &'bases BaseAddresses, + input: &mut R, + ) -> Result> { + match parse_cfi_entry(bases, section, input)? { + Some(CieOrFde::Cie(_)) => Err(Error::NotFdePointer), + Some(CieOrFde::Fde(partial)) => Ok(partial), + None => Err(Error::NoEntryAtGivenOffset), + } + } + + /// Fully parse this FDE. + /// + /// You must provide a function get its associated CIE (either by parsing it + /// on demand, or looking it up in some table mapping offsets to CIEs that + /// you've already parsed, etc.) + pub fn parse(&self, get_cie: F) -> Result> + where + F: FnMut(&Section, &BaseAddresses, Section::Offset) -> Result>, + { + FrameDescriptionEntry::parse_rest( + self.offset, + self.length, + self.format, + self.cie_offset, + self.rest.clone(), + &self.section, + self.bases, + get_cie, + ) + } +} + +/// A `FrameDescriptionEntry` is a set of CFA instructions for an address range. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FrameDescriptionEntry::Offset> +where + R: Reader, + Offset: ReaderOffset, +{ + /// The start of this entry within its containing section. + offset: Offset, + + /// > A constant that gives the number of bytes of the header and + /// > instruction stream for this function, not including the length field + /// > itself (see Section 7.2.2). The size of the length field plus the value + /// > of length must be an integral multiple of the address size. + length: Offset, + + format: Format, + + /// "A constant offset into the .debug_frame section that denotes the CIE + /// that is associated with this FDE." + /// + /// This is the CIE at that offset. + cie: CommonInformationEntry, + + /// > The address of the first location associated with this table entry. If + /// > the segment_size field of this FDE's CIE is non-zero, the initial + /// > location is preceded by a segment selector of the given length. + initial_segment: u64, + initial_address: u64, + + /// "The number of bytes of program instructions described by this entry." + address_range: u64, + + /// The parsed augmentation data, if we have any. + augmentation: Option, + + /// "A sequence of table defining instructions that are described below." + /// + /// This is followed by `DW_CFA_nop` padding until `length` bytes of the + /// input are consumed. + instructions: R, +} + +impl FrameDescriptionEntry { + #[allow(clippy::too_many_arguments)] + fn parse_rest( + offset: R::Offset, + length: R::Offset, + format: Format, + cie_pointer: Section::Offset, + mut rest: R, + section: &Section, + bases: &BaseAddresses, + mut get_cie: F, + ) -> Result> + where + Section: UnwindSection, + F: FnMut(&Section, &BaseAddresses, Section::Offset) -> Result>, + { + let cie = get_cie(section, bases, cie_pointer)?; + + let initial_segment = if cie.segment_size > 0 { + rest.read_address(cie.segment_size)? + } else { + 0 + }; + + let mut parameters = PointerEncodingParameters { + bases: &bases.eh_frame, + func_base: None, + address_size: cie.address_size, + section: section.section(), + }; + + let (initial_address, address_range) = Self::parse_addresses(&mut rest, &cie, ¶meters)?; + parameters.func_base = Some(initial_address); + + let aug_data = if let Some(ref augmentation) = cie.augmentation { + Some(AugmentationData::parse( + augmentation, + ¶meters, + &mut rest, + )?) + } else { + None + }; + + let entry = FrameDescriptionEntry { + offset, + length, + format, + cie, + initial_segment, + initial_address, + address_range, + augmentation: aug_data, + instructions: rest, + }; + + Ok(entry) + } + + fn parse_addresses( + input: &mut R, + cie: &CommonInformationEntry, + parameters: &PointerEncodingParameters, + ) -> Result<(u64, u64)> { + let encoding = cie.augmentation().and_then(|a| a.fde_address_encoding); + if let Some(encoding) = encoding { + let initial_address = parse_encoded_pointer(encoding, parameters, input)?; + + // Ignore indirection. + let initial_address = initial_address.into(); + + // Address ranges cannot be relative to anything, so just grab the + // data format bits from the encoding. + let address_range = parse_encoded_pointer(encoding.format(), parameters, input)?; + Ok((initial_address, address_range.into())) + } else { + let initial_address = input.read_address(cie.address_size)?; + let address_range = input.read_address(cie.address_size)?; + Ok((initial_address, address_range)) + } + } + + /// Return the table of unwind information for this FDE. + #[inline] + pub fn rows<'a, 'ctx, Section: UnwindSection, A: UnwindContextStorage>( + &self, + section: &'a Section, + bases: &'a BaseAddresses, + ctx: &'ctx mut UnwindContext, + ) -> Result> { + UnwindTable::new(section, bases, ctx, self) + } + + /// Find the frame unwind information for the given address. + /// + /// If found, the unwind information is returned along with the reset + /// context in the form `Ok((unwind_info, context))`. If not found, + /// `Err(gimli::Error::NoUnwindInfoForAddress)` is returned. If parsing or + /// CFI evaluation fails, the error is returned. + pub fn unwind_info_for_address<'ctx, Section: UnwindSection, A: UnwindContextStorage>( + &self, + section: &Section, + bases: &BaseAddresses, + ctx: &'ctx mut UnwindContext, + address: u64, + ) -> Result<&'ctx UnwindTableRow> { + let mut table = self.rows(section, bases, ctx)?; + while let Some(row) = table.next_row()? { + if row.contains(address) { + return Ok(table.ctx.row()); + } + } + Err(Error::NoUnwindInfoForAddress) + } +} + +/// # Signal Safe Methods +/// +/// These methods are guaranteed not to allocate, acquire locks, or perform any +/// other signal-unsafe operations. +#[allow(clippy::len_without_is_empty)] +impl FrameDescriptionEntry { + /// Get the offset of this entry from the start of its containing section. + pub fn offset(&self) -> R::Offset { + self.offset + } + + /// Get a reference to this FDE's CIE. + pub fn cie(&self) -> &CommonInformationEntry { + &self.cie + } + + /// > A constant that gives the number of bytes of the header and + /// > instruction stream for this function, not including the length field + /// > itself (see Section 7.2.2). The size of the length field plus the value + /// > of length must be an integral multiple of the address size. + pub fn entry_len(&self) -> R::Offset { + self.length + } + + /// Iterate over this FDE's instructions. + /// + /// Will not include the CIE's initial instructions, if you want those do + /// `fde.cie().instructions()` first. + /// + /// Can be [used with + /// `FallibleIterator`](./index.html#using-with-fallibleiterator). + pub fn instructions<'a, Section>( + &self, + section: &'a Section, + bases: &'a BaseAddresses, + ) -> CallFrameInstructionIter<'a, R> + where + Section: UnwindSection, + { + CallFrameInstructionIter { + input: self.instructions.clone(), + address_encoding: self.cie.augmentation().and_then(|a| a.fde_address_encoding), + parameters: PointerEncodingParameters { + bases: &bases.eh_frame, + func_base: None, + address_size: self.cie.address_size, + section: section.section(), + }, + } + } + + /// The first address for which this entry has unwind information for. + pub fn initial_address(&self) -> u64 { + self.initial_address + } + + /// The number of bytes of instructions that this entry has unwind + /// information for. + pub fn len(&self) -> u64 { + self.address_range + } + + /// Return `true` if the given address is within this FDE, `false` + /// otherwise. + /// + /// This is equivalent to `entry.initial_address() <= address < + /// entry.initial_address() + entry.len()`. + pub fn contains(&self, address: u64) -> bool { + let start = self.initial_address(); + let end = start + self.len(); + start <= address && address < end + } + + /// The address of this FDE's language-specific data area (LSDA), if it has + /// any. + pub fn lsda(&self) -> Option { + self.augmentation.as_ref().and_then(|a| a.lsda) + } + + /// Return true if this FDE's function is a trampoline for a signal handler. + #[inline] + pub fn is_signal_trampoline(&self) -> bool { + self.cie().is_signal_trampoline() + } + + /// Return the address of the FDE's function's personality routine + /// handler. The personality routine does language-specific clean up when + /// unwinding the stack frames with the intent to not run them again. + #[inline] + pub fn personality(&self) -> Option { + self.cie().personality() + } +} + +/// Specification of what storage should be used for [`UnwindContext`]. +/// +#[cfg_attr( + feature = "read", + doc = " +Normally you would only need to use [`StoreOnHeap`], which places the stack +on the heap using [`Vec`]. This is the default storage type parameter for [`UnwindContext`]. +" +)] +/// +/// If you need to avoid [`UnwindContext`] from allocating memory, e.g. for signal safety, +/// you can provide you own storage specification: +/// ```rust,no_run +/// # use gimli::*; +/// # +/// # fn foo<'a>(some_fde: gimli::FrameDescriptionEntry>) +/// # -> gimli::Result<()> { +/// # let eh_frame: gimli::EhFrame<_> = unreachable!(); +/// # let bases = unimplemented!(); +/// # +/// struct StoreOnStack; +/// +/// impl UnwindContextStorage for StoreOnStack { +/// type Rules = [(Register, RegisterRule); 192]; +/// type Stack = [UnwindTableRow; 4]; +/// } +/// +/// let mut ctx = UnwindContext::<_, StoreOnStack>::new_in(); +/// +/// // Initialize the context by evaluating the CIE's initial instruction program, +/// // and generate the unwind table. +/// let mut table = some_fde.rows(&eh_frame, &bases, &mut ctx)?; +/// while let Some(row) = table.next_row()? { +/// // Do stuff with each row... +/// # let _ = row; +/// } +/// # unreachable!() +/// # } +/// ``` +pub trait UnwindContextStorage: Sized { + /// The storage used for register rules in a unwind table row. + /// + /// Note that this is nested within the stack. + type Rules: ArrayLike)>; + + /// The storage used for unwind table row stack. + type Stack: ArrayLike>; +} + +#[cfg(feature = "read")] +const MAX_RULES: usize = 192; + +#[cfg(feature = "read")] +impl UnwindContextStorage for StoreOnHeap { + type Rules = [(Register, RegisterRule); MAX_RULES]; + type Stack = Vec>; +} + +/// Common context needed when evaluating the call frame unwinding information. +/// +/// This structure can be large so it is advisable to place it on the heap. +/// To avoid re-allocating the context multiple times when evaluating multiple +/// CFI programs, it can be reused. +/// +/// ``` +/// use gimli::{UnwindContext, UnwindTable}; +/// +/// # fn foo<'a>(some_fde: gimli::FrameDescriptionEntry>) +/// # -> gimli::Result<()> { +/// # let eh_frame: gimli::EhFrame<_> = unreachable!(); +/// # let bases = unimplemented!(); +/// // An uninitialized context. +/// let mut ctx = Box::new(UnwindContext::new()); +/// +/// // Initialize the context by evaluating the CIE's initial instruction program, +/// // and generate the unwind table. +/// let mut table = some_fde.rows(&eh_frame, &bases, &mut ctx)?; +/// while let Some(row) = table.next_row()? { +/// // Do stuff with each row... +/// # let _ = row; +/// } +/// # unreachable!() +/// # } +/// ``` +#[derive(Clone, PartialEq, Eq)] +pub struct UnwindContext = StoreOnHeap> { + // Stack of rows. The last row is the row currently being built by the + // program. There is always at least one row. The vast majority of CFI + // programs will only ever have one row on the stack. + stack: ArrayVec, + + // If we are evaluating an FDE's instructions, then `is_initialized` will be + // `true`. If `initial_rule` is `Some`, then the initial register rules are either + // all default rules or have just 1 non-default rule, stored in `initial_rule`. + // If it's `None`, `stack[0]` will contain the initial register rules + // described by the CIE's initial instructions. These rules are used by + // `DW_CFA_restore`. Otherwise, when we are currently evaluating a CIE's + // initial instructions, `is_initialized` will be `false` and initial rules + // cannot be read. + initial_rule: Option<(Register, RegisterRule)>, + + is_initialized: bool, +} + +impl> Debug for UnwindContext { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("UnwindContext") + .field("stack", &self.stack) + .field("initial_rule", &self.initial_rule) + .field("is_initialized", &self.is_initialized) + .finish() + } +} + +impl> Default for UnwindContext { + fn default() -> Self { + Self::new_in() + } +} + +#[cfg(feature = "read")] +impl UnwindContext { + /// Construct a new call frame unwinding context. + pub fn new() -> Self { + Self::new_in() + } +} + +/// # Signal Safe Methods +/// +/// These methods are guaranteed not to allocate, acquire locks, or perform any +/// other signal-unsafe operations, if an non-allocating storage is used. +impl> UnwindContext { + /// Construct a new call frame unwinding context. + pub fn new_in() -> Self { + let mut ctx = UnwindContext { + stack: Default::default(), + initial_rule: None, + is_initialized: false, + }; + ctx.reset(); + ctx + } + + /// Run the CIE's initial instructions and initialize this `UnwindContext`. + fn initialize>( + &mut self, + section: &Section, + bases: &BaseAddresses, + cie: &CommonInformationEntry, + ) -> Result<()> { + if self.is_initialized { + self.reset(); + } + + let mut table = UnwindTable::new_for_cie(section, bases, self, cie); + while let Some(_) = table.next_row()? {} + + self.save_initial_rules()?; + Ok(()) + } + + fn reset(&mut self) { + self.stack.clear(); + self.stack.try_push(UnwindTableRow::default()).unwrap(); + debug_assert!(self.stack[0].is_default()); + self.initial_rule = None; + self.is_initialized = false; + } + + fn row(&self) -> &UnwindTableRow { + self.stack.last().unwrap() + } + + fn row_mut(&mut self) -> &mut UnwindTableRow { + self.stack.last_mut().unwrap() + } + + fn save_initial_rules(&mut self) -> Result<()> { + assert_eq!(self.is_initialized, false); + self.initial_rule = match *self.stack.last().unwrap().registers.rules { + // All rules are default (undefined). In this case just synthesize + // an undefined rule. + [] => Some((Register(0), RegisterRule::Undefined)), + [ref rule] => Some(rule.clone()), + _ => { + let rules = self.stack.last().unwrap().clone(); + self.stack + .try_insert(0, rules) + .map_err(|_| Error::StackFull)?; + None + } + }; + self.is_initialized = true; + Ok(()) + } + + fn start_address(&self) -> u64 { + self.row().start_address + } + + fn set_start_address(&mut self, start_address: u64) { + let row = self.row_mut(); + row.start_address = start_address; + } + + fn set_register_rule(&mut self, register: Register, rule: RegisterRule) -> Result<()> { + let row = self.row_mut(); + row.registers.set(register, rule) + } + + /// Returns `None` if we have not completed evaluation of a CIE's initial + /// instructions. + fn get_initial_rule(&self, register: Register) -> Option> { + if !self.is_initialized { + return None; + } + Some(match self.initial_rule { + None => self.stack[0].registers.get(register), + Some((r, ref rule)) if r == register => rule.clone(), + _ => RegisterRule::Undefined, + }) + } + + fn set_cfa(&mut self, cfa: CfaRule) { + self.row_mut().cfa = cfa; + } + + fn cfa_mut(&mut self) -> &mut CfaRule { + &mut self.row_mut().cfa + } + + fn push_row(&mut self) -> Result<()> { + let new_row = self.row().clone(); + self.stack.try_push(new_row).map_err(|_| Error::StackFull) + } + + fn pop_row(&mut self) -> Result<()> { + let min_size = if self.is_initialized && self.initial_rule.is_none() { + 2 + } else { + 1 + }; + if self.stack.len() <= min_size { + return Err(Error::PopWithEmptyStack); + } + self.stack.pop().unwrap(); + Ok(()) + } +} + +/// The `UnwindTable` iteratively evaluates a `FrameDescriptionEntry`'s +/// `CallFrameInstruction` program, yielding the each row one at a time. +/// +/// > 6.4.1 Structure of Call Frame Information +/// > +/// > DWARF supports virtual unwinding by defining an architecture independent +/// > basis for recording how procedures save and restore registers during their +/// > lifetimes. This basis must be augmented on some machines with specific +/// > information that is defined by an architecture specific ABI authoring +/// > committee, a hardware vendor, or a compiler producer. The body defining a +/// > specific augmentation is referred to below as the “augmenter.” +/// > +/// > Abstractly, this mechanism describes a very large table that has the +/// > following structure: +/// > +/// > +/// > +/// > +/// > +/// > +/// > +/// > +/// > +/// > +/// > +/// > +/// > +/// > +/// > +/// > +/// > +/// >
LOCCFAR0R1...RN
L0
L1
...
LN
+/// > +/// > The first column indicates an address for every location that contains code +/// > in a program. (In shared objects, this is an object-relative offset.) The +/// > remaining columns contain virtual unwinding rules that are associated with +/// > the indicated location. +/// > +/// > The CFA column defines the rule which computes the Canonical Frame Address +/// > value; it may be either a register and a signed offset that are added +/// > together, or a DWARF expression that is evaluated. +/// > +/// > The remaining columns are labeled by register number. This includes some +/// > registers that have special designation on some architectures such as the PC +/// > and the stack pointer register. (The actual mapping of registers for a +/// > particular architecture is defined by the augmenter.) The register columns +/// > contain rules that describe whether a given register has been saved and the +/// > rule to find the value for the register in the previous frame. +/// > +/// > ... +/// > +/// > This table would be extremely large if actually constructed as +/// > described. Most of the entries at any point in the table are identical to +/// > the ones above them. The whole table can be represented quite compactly by +/// > recording just the differences starting at the beginning address of each +/// > subroutine in the program. +#[derive(Debug)] +pub struct UnwindTable<'a, 'ctx, R: Reader, A: UnwindContextStorage = StoreOnHeap> { + code_alignment_factor: Wrapping, + data_alignment_factor: Wrapping, + next_start_address: u64, + last_end_address: u64, + returned_last_row: bool, + current_row_valid: bool, + instructions: CallFrameInstructionIter<'a, R>, + ctx: &'ctx mut UnwindContext, +} + +/// # Signal Safe Methods +/// +/// These methods are guaranteed not to allocate, acquire locks, or perform any +/// other signal-unsafe operations. +impl<'a, 'ctx, R: Reader, A: UnwindContextStorage> UnwindTable<'a, 'ctx, R, A> { + /// Construct a new `UnwindTable` for the given + /// `FrameDescriptionEntry`'s CFI unwinding program. + pub fn new>( + section: &'a Section, + bases: &'a BaseAddresses, + ctx: &'ctx mut UnwindContext, + fde: &FrameDescriptionEntry, + ) -> Result { + ctx.initialize(section, bases, fde.cie())?; + Ok(Self::new_for_fde(section, bases, ctx, fde)) + } + + fn new_for_fde>( + section: &'a Section, + bases: &'a BaseAddresses, + ctx: &'ctx mut UnwindContext, + fde: &FrameDescriptionEntry, + ) -> Self { + assert!(ctx.stack.len() >= 1); + UnwindTable { + code_alignment_factor: Wrapping(fde.cie().code_alignment_factor()), + data_alignment_factor: Wrapping(fde.cie().data_alignment_factor()), + next_start_address: fde.initial_address(), + last_end_address: fde.initial_address().wrapping_add(fde.len()), + returned_last_row: false, + current_row_valid: false, + instructions: fde.instructions(section, bases), + ctx, + } + } + + fn new_for_cie>( + section: &'a Section, + bases: &'a BaseAddresses, + ctx: &'ctx mut UnwindContext, + cie: &CommonInformationEntry, + ) -> Self { + assert!(ctx.stack.len() >= 1); + UnwindTable { + code_alignment_factor: Wrapping(cie.code_alignment_factor()), + data_alignment_factor: Wrapping(cie.data_alignment_factor()), + next_start_address: 0, + last_end_address: 0, + returned_last_row: false, + current_row_valid: false, + instructions: cie.instructions(section, bases), + ctx, + } + } + + /// Evaluate call frame instructions until the next row of the table is + /// completed, and return it. + /// + /// Unfortunately, this cannot be used with `FallibleIterator` because of + /// the restricted lifetime of the yielded item. + pub fn next_row(&mut self) -> Result>> { + assert!(self.ctx.stack.len() >= 1); + self.ctx.set_start_address(self.next_start_address); + self.current_row_valid = false; + + loop { + match self.instructions.next() { + Err(e) => return Err(e), + + Ok(None) => { + if self.returned_last_row { + return Ok(None); + } + + let row = self.ctx.row_mut(); + row.end_address = self.last_end_address; + + self.returned_last_row = true; + self.current_row_valid = true; + return Ok(Some(row)); + } + + Ok(Some(instruction)) => { + if self.evaluate(instruction)? { + self.current_row_valid = true; + return Ok(Some(self.ctx.row())); + } + } + }; + } + } + + /// Returns the current row with the lifetime of the context. + pub fn into_current_row(self) -> Option<&'ctx UnwindTableRow> { + if self.current_row_valid { + Some(self.ctx.row()) + } else { + None + } + } + + /// Evaluate one call frame instruction. Return `Ok(true)` if the row is + /// complete, `Ok(false)` otherwise. + fn evaluate(&mut self, instruction: CallFrameInstruction) -> Result { + use crate::CallFrameInstruction::*; + + match instruction { + // Instructions that complete the current row and advance the + // address for the next row. + SetLoc { address } => { + if address < self.ctx.start_address() { + return Err(Error::InvalidAddressRange); + } + + self.next_start_address = address; + self.ctx.row_mut().end_address = self.next_start_address; + return Ok(true); + } + AdvanceLoc { delta } => { + let delta = Wrapping(u64::from(delta)) * self.code_alignment_factor; + self.next_start_address = (Wrapping(self.ctx.start_address()) + delta).0; + self.ctx.row_mut().end_address = self.next_start_address; + return Ok(true); + } + + // Instructions that modify the CFA. + DefCfa { register, offset } => { + self.ctx.set_cfa(CfaRule::RegisterAndOffset { + register, + offset: offset as i64, + }); + } + DefCfaSf { + register, + factored_offset, + } => { + let data_align = self.data_alignment_factor; + self.ctx.set_cfa(CfaRule::RegisterAndOffset { + register, + offset: (Wrapping(factored_offset) * data_align).0, + }); + } + DefCfaRegister { register } => { + if let CfaRule::RegisterAndOffset { + register: ref mut reg, + .. + } = *self.ctx.cfa_mut() + { + *reg = register; + } else { + return Err(Error::CfiInstructionInInvalidContext); + } + } + DefCfaOffset { offset } => { + if let CfaRule::RegisterAndOffset { + offset: ref mut off, + .. + } = *self.ctx.cfa_mut() + { + *off = offset as i64; + } else { + return Err(Error::CfiInstructionInInvalidContext); + } + } + DefCfaOffsetSf { factored_offset } => { + if let CfaRule::RegisterAndOffset { + offset: ref mut off, + .. + } = *self.ctx.cfa_mut() + { + let data_align = self.data_alignment_factor; + *off = (Wrapping(factored_offset) * data_align).0; + } else { + return Err(Error::CfiInstructionInInvalidContext); + } + } + DefCfaExpression { expression } => { + self.ctx.set_cfa(CfaRule::Expression(expression)); + } + + // Instructions that define register rules. + Undefined { register } => { + self.ctx + .set_register_rule(register, RegisterRule::Undefined)?; + } + SameValue { register } => { + self.ctx + .set_register_rule(register, RegisterRule::SameValue)?; + } + Offset { + register, + factored_offset, + } => { + let offset = Wrapping(factored_offset as i64) * self.data_alignment_factor; + self.ctx + .set_register_rule(register, RegisterRule::Offset(offset.0))?; + } + OffsetExtendedSf { + register, + factored_offset, + } => { + let offset = Wrapping(factored_offset) * self.data_alignment_factor; + self.ctx + .set_register_rule(register, RegisterRule::Offset(offset.0))?; + } + ValOffset { + register, + factored_offset, + } => { + let offset = Wrapping(factored_offset as i64) * self.data_alignment_factor; + self.ctx + .set_register_rule(register, RegisterRule::ValOffset(offset.0))?; + } + ValOffsetSf { + register, + factored_offset, + } => { + let offset = Wrapping(factored_offset) * self.data_alignment_factor; + self.ctx + .set_register_rule(register, RegisterRule::ValOffset(offset.0))?; + } + Register { + dest_register, + src_register, + } => { + self.ctx + .set_register_rule(dest_register, RegisterRule::Register(src_register))?; + } + Expression { + register, + expression, + } => { + let expression = RegisterRule::Expression(expression); + self.ctx.set_register_rule(register, expression)?; + } + ValExpression { + register, + expression, + } => { + let expression = RegisterRule::ValExpression(expression); + self.ctx.set_register_rule(register, expression)?; + } + Restore { register } => { + let initial_rule = if let Some(rule) = self.ctx.get_initial_rule(register) { + rule + } else { + // Can't restore the initial rule when we are + // evaluating the initial rules! + return Err(Error::CfiInstructionInInvalidContext); + }; + + self.ctx.set_register_rule(register, initial_rule)?; + } + + // Row push and pop instructions. + RememberState => { + self.ctx.push_row()?; + } + RestoreState => { + // Pop state while preserving current location. + let start_address = self.ctx.start_address(); + self.ctx.pop_row()?; + self.ctx.set_start_address(start_address); + } + + // GNU Extension. Save the size somewhere so the unwinder can use + // it when restoring IP + ArgsSize { size } => { + self.ctx.row_mut().saved_args_size = size; + } + + // No operation. + Nop => {} + }; + + Ok(false) + } +} + +// We tend to have very few register rules: usually only a couple. Even if we +// have a rule for every register, on x86-64 with SSE and everything we're +// talking about ~100 rules. So rather than keeping the rules in a hash map, or +// a vector indexed by register number (which would lead to filling lots of +// empty entries), we store them as a vec of (register number, register rule) +// pairs. +// +// Additionally, because every register's default rule is implicitly +// `RegisterRule::Undefined`, we never store a register's rule in this vec if it +// is undefined and save a little bit more space and do a little fewer +// comparisons that way. +// +// The maximum number of rules preallocated by libunwind is 97 for AArch64, 128 +// for ARM, and even 188 for MIPS. It is extremely unlikely to encounter this +// many register rules in practice. +// +// See: +// - https://github.com/libunwind/libunwind/blob/11fd461095ea98f4b3e3a361f5a8a558519363fa/include/tdep-x86_64/dwarf-config.h#L36 +// - https://github.com/libunwind/libunwind/blob/11fd461095ea98f4b3e3a361f5a8a558519363fa/include/tdep-aarch64/dwarf-config.h#L32 +// - https://github.com/libunwind/libunwind/blob/11fd461095ea98f4b3e3a361f5a8a558519363fa/include/tdep-arm/dwarf-config.h#L31 +// - https://github.com/libunwind/libunwind/blob/11fd461095ea98f4b3e3a361f5a8a558519363fa/include/tdep-mips/dwarf-config.h#L31 +struct RegisterRuleMap = StoreOnHeap> { + rules: ArrayVec, +} + +impl> Debug for RegisterRuleMap { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("RegisterRuleMap") + .field("rules", &self.rules) + .finish() + } +} + +impl> Clone for RegisterRuleMap { + fn clone(&self) -> Self { + Self { + rules: self.rules.clone(), + } + } +} + +impl> Default for RegisterRuleMap { + fn default() -> Self { + RegisterRuleMap { + rules: Default::default(), + } + } +} + +/// # Signal Safe Methods +/// +/// These methods are guaranteed not to allocate, acquire locks, or perform any +/// other signal-unsafe operations. +impl> RegisterRuleMap { + fn is_default(&self) -> bool { + self.rules.is_empty() + } + + fn get(&self, register: Register) -> RegisterRule { + self.rules + .iter() + .find(|rule| rule.0 == register) + .map(|r| { + debug_assert!(r.1.is_defined()); + r.1.clone() + }) + .unwrap_or(RegisterRule::Undefined) + } + + fn set(&mut self, register: Register, rule: RegisterRule) -> Result<()> { + if !rule.is_defined() { + let idx = self + .rules + .iter() + .enumerate() + .find(|&(_, r)| r.0 == register) + .map(|(i, _)| i); + if let Some(idx) = idx { + self.rules.swap_remove(idx); + } + return Ok(()); + } + + for &mut (reg, ref mut old_rule) in &mut *self.rules { + debug_assert!(old_rule.is_defined()); + if reg == register { + *old_rule = rule; + return Ok(()); + } + } + + self.rules + .try_push((register, rule)) + .map_err(|_| Error::TooManyRegisterRules) + } + + fn iter(&self) -> RegisterRuleIter { + RegisterRuleIter(self.rules.iter()) + } +} + +impl<'a, R, S: UnwindContextStorage> FromIterator<&'a (Register, RegisterRule)> + for RegisterRuleMap +where + R: 'a + Reader, +{ + fn from_iter(iter: T) -> Self + where + T: IntoIterator)>, + { + let iter = iter.into_iter(); + let mut rules = RegisterRuleMap::default(); + for &(reg, ref rule) in iter.filter(|r| r.1.is_defined()) { + rules.set(reg, rule.clone()).expect( + "This is only used in tests, impl isn't exposed publicly. + If you trip this, fix your test", + ); + } + rules + } +} + +impl> PartialEq for RegisterRuleMap +where + R: Reader + PartialEq, +{ + fn eq(&self, rhs: &Self) -> bool { + for &(reg, ref rule) in &*self.rules { + debug_assert!(rule.is_defined()); + if *rule != rhs.get(reg) { + return false; + } + } + + for &(reg, ref rhs_rule) in &*rhs.rules { + debug_assert!(rhs_rule.is_defined()); + if *rhs_rule != self.get(reg) { + return false; + } + } + + true + } +} + +impl> Eq for RegisterRuleMap where R: Reader + Eq {} + +/// An unordered iterator for register rules. +#[derive(Debug, Clone)] +pub struct RegisterRuleIter<'iter, R>(::core::slice::Iter<'iter, (Register, RegisterRule)>) +where + R: Reader; + +impl<'iter, R: Reader> Iterator for RegisterRuleIter<'iter, R> { + type Item = &'iter (Register, RegisterRule); + + fn next(&mut self) -> Option { + self.0.next() + } +} + +/// A row in the virtual unwind table that describes how to find the values of +/// the registers in the *previous* frame for a range of PC addresses. +#[derive(PartialEq, Eq)] +pub struct UnwindTableRow = StoreOnHeap> { + start_address: u64, + end_address: u64, + saved_args_size: u64, + cfa: CfaRule, + registers: RegisterRuleMap, +} + +impl> Debug for UnwindTableRow { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("UnwindTableRow") + .field("start_address", &self.start_address) + .field("end_address", &self.end_address) + .field("saved_args_size", &self.saved_args_size) + .field("cfa", &self.cfa) + .field("registers", &self.registers) + .finish() + } +} + +impl> Clone for UnwindTableRow { + fn clone(&self) -> Self { + Self { + start_address: self.start_address, + end_address: self.end_address, + saved_args_size: self.saved_args_size, + cfa: self.cfa.clone(), + registers: self.registers.clone(), + } + } +} + +impl> Default for UnwindTableRow { + fn default() -> Self { + UnwindTableRow { + start_address: 0, + end_address: 0, + saved_args_size: 0, + cfa: Default::default(), + registers: Default::default(), + } + } +} + +impl> UnwindTableRow { + fn is_default(&self) -> bool { + self.start_address == 0 + && self.end_address == 0 + && self.cfa.is_default() + && self.registers.is_default() + } + + /// Get the starting PC address that this row applies to. + pub fn start_address(&self) -> u64 { + self.start_address + } + + /// Get the end PC address where this row's register rules become + /// unapplicable. + /// + /// In other words, this row describes how to recover the last frame's + /// registers for all PCs where `row.start_address() <= PC < + /// row.end_address()`. This row does NOT describe how to recover registers + /// when `PC == row.end_address()`. + pub fn end_address(&self) -> u64 { + self.end_address + } + + /// Return `true` if the given `address` is within this row's address range, + /// `false` otherwise. + pub fn contains(&self, address: u64) -> bool { + self.start_address <= address && address < self.end_address + } + + /// Returns the amount of args currently on the stack. + /// + /// When unwinding, if the personality function requested a change in IP, + /// the SP needs to be adjusted by saved_args_size. + pub fn saved_args_size(&self) -> u64 { + self.saved_args_size + } + + /// Get the canonical frame address (CFA) recovery rule for this row. + pub fn cfa(&self) -> &CfaRule { + &self.cfa + } + + /// Get the register recovery rule for the given register number. + /// + /// The register number mapping is architecture dependent. For example, in + /// the x86-64 ABI the register number mapping is defined in Figure 3.36: + /// + /// > Figure 3.36: DWARF Register Number Mapping + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// > + /// >
Register Name Number Abbreviation
General Purpose Register RAX 0 %rax
General Purpose Register RDX 1 %rdx
General Purpose Register RCX 2 %rcx
General Purpose Register RBX 3 %rbx
General Purpose Register RSI 4 %rsi
General Purpose Register RDI 5 %rdi
General Purpose Register RBP 6 %rbp
Stack Pointer Register RSP 7 %rsp
Extended Integer Registers 8-15 8-15 %r8-%r15
Return Address RA 16
Vector Registers 0–7 17-24 %xmm0–%xmm7
Extended Vector Registers 8–15 25-32 %xmm8–%xmm15
Floating Point Registers 0–7 33-40 %st0–%st7
MMX Registers 0–7 41-48 %mm0–%mm7
Flag Register 49 %rFLAGS
Segment Register ES 50 %es
Segment Register CS 51 %cs
Segment Register SS 52 %ss
Segment Register DS 53 %ds
Segment Register FS 54 %fs
Segment Register GS 55 %gs
Reserved 56-57
FS Base address 58 %fs.base
GS Base address 59 %gs.base
Reserved 60-61
Task Register 62 %tr
LDT Register 63 %ldtr
128-bit Media Control and Status 64 %mxcsr
x87 Control Word 65 %fcw
x87 Status Word 66 %fsw
Upper Vector Registers 16–31 67-82 %xmm16–%xmm31
Reserved 83-117
Vector Mask Registers 0–7 118-125 %k0–%k7
Reserved 126-129
+ pub fn register(&self, register: Register) -> RegisterRule { + self.registers.get(register) + } + + /// Iterate over all defined register `(number, rule)` pairs. + /// + /// The rules are not iterated in any guaranteed order. Any register that + /// does not make an appearance in the iterator implicitly has the rule + /// `RegisterRule::Undefined`. + /// + /// ``` + /// # use gimli::{EndianSlice, LittleEndian, UnwindTableRow}; + /// # fn foo<'input>(unwind_table_row: UnwindTableRow>) { + /// for &(register, ref rule) in unwind_table_row.registers() { + /// // ... + /// # drop(register); drop(rule); + /// } + /// # } + /// ``` + pub fn registers(&self) -> RegisterRuleIter { + self.registers.iter() + } +} + +/// The canonical frame address (CFA) recovery rules. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CfaRule { + /// The CFA is given offset from the given register's value. + RegisterAndOffset { + /// The register containing the base value. + register: Register, + /// The offset from the register's base value. + offset: i64, + }, + /// The CFA is obtained by evaluating this `Reader` as a DWARF expression + /// program. + Expression(Expression), +} + +impl Default for CfaRule { + fn default() -> Self { + CfaRule::RegisterAndOffset { + register: Register(0), + offset: 0, + } + } +} + +impl CfaRule { + fn is_default(&self) -> bool { + match *self { + CfaRule::RegisterAndOffset { register, offset } => { + register == Register(0) && offset == 0 + } + _ => false, + } + } +} + +/// An entry in the abstract CFI table that describes how to find the value of a +/// register. +/// +/// "The register columns contain rules that describe whether a given register +/// has been saved and the rule to find the value for the register in the +/// previous frame." +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RegisterRule { + /// > A register that has this rule has no recoverable value in the previous + /// > frame. (By convention, it is not preserved by a callee.) + Undefined, + + /// > This register has not been modified from the previous frame. (By + /// > convention, it is preserved by the callee, but the callee has not + /// > modified it.) + SameValue, + + /// "The previous value of this register is saved at the address CFA+N where + /// CFA is the current CFA value and N is a signed offset." + Offset(i64), + + /// "The previous value of this register is the value CFA+N where CFA is the + /// current CFA value and N is a signed offset." + ValOffset(i64), + + /// "The previous value of this register is stored in another register + /// numbered R." + Register(Register), + + /// "The previous value of this register is located at the address produced + /// by executing the DWARF expression." + Expression(Expression), + + /// "The previous value of this register is the value produced by executing + /// the DWARF expression." + ValExpression(Expression), + + /// "The rule is defined externally to this specification by the augmenter." + Architectural, +} + +impl RegisterRule { + fn is_defined(&self) -> bool { + match *self { + RegisterRule::Undefined => false, + _ => true, + } + } +} + +/// A parsed call frame instruction. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CallFrameInstruction { + // 6.4.2.1 Row Creation Methods + /// > 1. DW_CFA_set_loc + /// > + /// > The DW_CFA_set_loc instruction takes a single operand that represents + /// > a target address. The required action is to create a new table row + /// > using the specified address as the location. All other values in the + /// > new row are initially identical to the current row. The new location + /// > value is always greater than the current one. If the segment_size + /// > field of this FDE's CIE is non- zero, the initial location is preceded + /// > by a segment selector of the given length. + SetLoc { + /// The target address. + address: u64, + }, + + /// The `AdvanceLoc` instruction is used for all of `DW_CFA_advance_loc` and + /// `DW_CFA_advance_loc{1,2,4}`. + /// + /// > 2. DW_CFA_advance_loc + /// > + /// > The DW_CFA_advance instruction takes a single operand (encoded with + /// > the opcode) that represents a constant delta. The required action is + /// > to create a new table row with a location value that is computed by + /// > taking the current entry’s location value and adding the value of + /// > delta * code_alignment_factor. All other values in the new row are + /// > initially identical to the current row. + AdvanceLoc { + /// The delta to be added to the current address. + delta: u32, + }, + + // 6.4.2.2 CFA Definition Methods + /// > 1. DW_CFA_def_cfa + /// > + /// > The DW_CFA_def_cfa instruction takes two unsigned LEB128 operands + /// > representing a register number and a (non-factored) offset. The + /// > required action is to define the current CFA rule to use the provided + /// > register and offset. + DefCfa { + /// The target register's number. + register: Register, + /// The non-factored offset. + offset: u64, + }, + + /// > 2. DW_CFA_def_cfa_sf + /// > + /// > The DW_CFA_def_cfa_sf instruction takes two operands: an unsigned + /// > LEB128 value representing a register number and a signed LEB128 + /// > factored offset. This instruction is identical to DW_CFA_def_cfa + /// > except that the second operand is signed and factored. The resulting + /// > offset is factored_offset * data_alignment_factor. + DefCfaSf { + /// The target register's number. + register: Register, + /// The factored offset. + factored_offset: i64, + }, + + /// > 3. DW_CFA_def_cfa_register + /// > + /// > The DW_CFA_def_cfa_register instruction takes a single unsigned LEB128 + /// > operand representing a register number. The required action is to + /// > define the current CFA rule to use the provided register (but to keep + /// > the old offset). This operation is valid only if the current CFA rule + /// > is defined to use a register and offset. + DefCfaRegister { + /// The target register's number. + register: Register, + }, + + /// > 4. DW_CFA_def_cfa_offset + /// > + /// > The DW_CFA_def_cfa_offset instruction takes a single unsigned LEB128 + /// > operand representing a (non-factored) offset. The required action is + /// > to define the current CFA rule to use the provided offset (but to keep + /// > the old register). This operation is valid only if the current CFA + /// > rule is defined to use a register and offset. + DefCfaOffset { + /// The non-factored offset. + offset: u64, + }, + + /// > 5. DW_CFA_def_cfa_offset_sf + /// > + /// > The DW_CFA_def_cfa_offset_sf instruction takes a signed LEB128 operand + /// > representing a factored offset. This instruction is identical to + /// > DW_CFA_def_cfa_offset except that the operand is signed and + /// > factored. The resulting offset is factored_offset * + /// > data_alignment_factor. This operation is valid only if the current CFA + /// > rule is defined to use a register and offset. + DefCfaOffsetSf { + /// The factored offset. + factored_offset: i64, + }, + + /// > 6. DW_CFA_def_cfa_expression + /// > + /// > The DW_CFA_def_cfa_expression instruction takes a single operand + /// > encoded as a DW_FORM_exprloc value representing a DWARF + /// > expression. The required action is to establish that expression as the + /// > means by which the current CFA is computed. + DefCfaExpression { + /// The DWARF expression. + expression: Expression, + }, + + // 6.4.2.3 Register Rule Instructions + /// > 1. DW_CFA_undefined + /// > + /// > The DW_CFA_undefined instruction takes a single unsigned LEB128 + /// > operand that represents a register number. The required action is to + /// > set the rule for the specified register to “undefined.” + Undefined { + /// The target register's number. + register: Register, + }, + + /// > 2. DW_CFA_same_value + /// > + /// > The DW_CFA_same_value instruction takes a single unsigned LEB128 + /// > operand that represents a register number. The required action is to + /// > set the rule for the specified register to “same value.” + SameValue { + /// The target register's number. + register: Register, + }, + + /// The `Offset` instruction represents both `DW_CFA_offset` and + /// `DW_CFA_offset_extended`. + /// + /// > 3. DW_CFA_offset + /// > + /// > The DW_CFA_offset instruction takes two operands: a register number + /// > (encoded with the opcode) and an unsigned LEB128 constant representing + /// > a factored offset. The required action is to change the rule for the + /// > register indicated by the register number to be an offset(N) rule + /// > where the value of N is factored offset * data_alignment_factor. + Offset { + /// The target register's number. + register: Register, + /// The factored offset. + factored_offset: u64, + }, + + /// > 5. DW_CFA_offset_extended_sf + /// > + /// > The DW_CFA_offset_extended_sf instruction takes two operands: an + /// > unsigned LEB128 value representing a register number and a signed + /// > LEB128 factored offset. This instruction is identical to + /// > DW_CFA_offset_extended except that the second operand is signed and + /// > factored. The resulting offset is factored_offset * + /// > data_alignment_factor. + OffsetExtendedSf { + /// The target register's number. + register: Register, + /// The factored offset. + factored_offset: i64, + }, + + /// > 6. DW_CFA_val_offset + /// > + /// > The DW_CFA_val_offset instruction takes two unsigned LEB128 operands + /// > representing a register number and a factored offset. The required + /// > action is to change the rule for the register indicated by the + /// > register number to be a val_offset(N) rule where the value of N is + /// > factored_offset * data_alignment_factor. + ValOffset { + /// The target register's number. + register: Register, + /// The factored offset. + factored_offset: u64, + }, + + /// > 7. DW_CFA_val_offset_sf + /// > + /// > The DW_CFA_val_offset_sf instruction takes two operands: an unsigned + /// > LEB128 value representing a register number and a signed LEB128 + /// > factored offset. This instruction is identical to DW_CFA_val_offset + /// > except that the second operand is signed and factored. The resulting + /// > offset is factored_offset * data_alignment_factor. + ValOffsetSf { + /// The target register's number. + register: Register, + /// The factored offset. + factored_offset: i64, + }, + + /// > 8. DW_CFA_register + /// > + /// > The DW_CFA_register instruction takes two unsigned LEB128 operands + /// > representing register numbers. The required action is to set the rule + /// > for the first register to be register(R) where R is the second + /// > register. + Register { + /// The number of the register whose rule is being changed. + dest_register: Register, + /// The number of the register where the other register's value can be + /// found. + src_register: Register, + }, + + /// > 9. DW_CFA_expression + /// > + /// > The DW_CFA_expression instruction takes two operands: an unsigned + /// > LEB128 value representing a register number, and a DW_FORM_block value + /// > representing a DWARF expression. The required action is to change the + /// > rule for the register indicated by the register number to be an + /// > expression(E) rule where E is the DWARF expression. That is, the DWARF + /// > expression computes the address. The value of the CFA is pushed on the + /// > DWARF evaluation stack prior to execution of the DWARF expression. + Expression { + /// The target register's number. + register: Register, + /// The DWARF expression. + expression: Expression, + }, + + /// > 10. DW_CFA_val_expression + /// > + /// > The DW_CFA_val_expression instruction takes two operands: an unsigned + /// > LEB128 value representing a register number, and a DW_FORM_block value + /// > representing a DWARF expression. The required action is to change the + /// > rule for the register indicated by the register number to be a + /// > val_expression(E) rule where E is the DWARF expression. That is, the + /// > DWARF expression computes the value of the given register. The value + /// > of the CFA is pushed on the DWARF evaluation stack prior to execution + /// > of the DWARF expression. + ValExpression { + /// The target register's number. + register: Register, + /// The DWARF expression. + expression: Expression, + }, + + /// The `Restore` instruction represents both `DW_CFA_restore` and + /// `DW_CFA_restore_extended`. + /// + /// > 11. DW_CFA_restore + /// > + /// > The DW_CFA_restore instruction takes a single operand (encoded with + /// > the opcode) that represents a register number. The required action is + /// > to change the rule for the indicated register to the rule assigned it + /// > by the initial_instructions in the CIE. + Restore { + /// The register to be reset. + register: Register, + }, + + // 6.4.2.4 Row State Instructions + /// > 1. DW_CFA_remember_state + /// > + /// > The DW_CFA_remember_state instruction takes no operands. The required + /// > action is to push the set of rules for every register onto an implicit + /// > stack. + RememberState, + + /// > 2. DW_CFA_restore_state + /// > + /// > The DW_CFA_restore_state instruction takes no operands. The required + /// > action is to pop the set of rules off the implicit stack and place + /// > them in the current row. + RestoreState, + + /// > DW_CFA_GNU_args_size + /// > + /// > GNU Extension + /// > + /// > The DW_CFA_GNU_args_size instruction takes an unsigned LEB128 operand + /// > representing an argument size. This instruction specifies the total of + /// > the size of the arguments which have been pushed onto the stack. + ArgsSize { + /// The size of the arguments which have been pushed onto the stack + size: u64, + }, + + // 6.4.2.5 Padding Instruction + /// > 1. DW_CFA_nop + /// > + /// > The DW_CFA_nop instruction has no operands and no required actions. It + /// > is used as padding to make a CIE or FDE an appropriate size. + Nop, +} + +const CFI_INSTRUCTION_HIGH_BITS_MASK: u8 = 0b1100_0000; +const CFI_INSTRUCTION_LOW_BITS_MASK: u8 = !CFI_INSTRUCTION_HIGH_BITS_MASK; + +impl CallFrameInstruction { + fn parse( + input: &mut R, + address_encoding: Option, + parameters: &PointerEncodingParameters, + ) -> Result> { + let instruction = input.read_u8()?; + let high_bits = instruction & CFI_INSTRUCTION_HIGH_BITS_MASK; + + if high_bits == constants::DW_CFA_advance_loc.0 { + let delta = instruction & CFI_INSTRUCTION_LOW_BITS_MASK; + return Ok(CallFrameInstruction::AdvanceLoc { + delta: u32::from(delta), + }); + } + + if high_bits == constants::DW_CFA_offset.0 { + let register = Register((instruction & CFI_INSTRUCTION_LOW_BITS_MASK).into()); + let offset = input.read_uleb128()?; + return Ok(CallFrameInstruction::Offset { + register, + factored_offset: offset, + }); + } + + if high_bits == constants::DW_CFA_restore.0 { + let register = Register((instruction & CFI_INSTRUCTION_LOW_BITS_MASK).into()); + return Ok(CallFrameInstruction::Restore { register }); + } + + debug_assert_eq!(high_bits, 0); + let instruction = constants::DwCfa(instruction); + + match instruction { + constants::DW_CFA_nop => Ok(CallFrameInstruction::Nop), + + constants::DW_CFA_set_loc => { + let address = if let Some(encoding) = address_encoding { + match parse_encoded_pointer(encoding, parameters, input)? { + Pointer::Direct(x) => x, + _ => return Err(Error::UnsupportedPointerEncoding), + } + } else { + input.read_address(parameters.address_size)? + }; + Ok(CallFrameInstruction::SetLoc { address }) + } + + constants::DW_CFA_advance_loc1 => { + let delta = input.read_u8()?; + Ok(CallFrameInstruction::AdvanceLoc { + delta: u32::from(delta), + }) + } + + constants::DW_CFA_advance_loc2 => { + let delta = input.read_u16()?; + Ok(CallFrameInstruction::AdvanceLoc { + delta: u32::from(delta), + }) + } + + constants::DW_CFA_advance_loc4 => { + let delta = input.read_u32()?; + Ok(CallFrameInstruction::AdvanceLoc { delta }) + } + + constants::DW_CFA_offset_extended => { + let register = input.read_uleb128().and_then(Register::from_u64)?; + let offset = input.read_uleb128()?; + Ok(CallFrameInstruction::Offset { + register, + factored_offset: offset, + }) + } + + constants::DW_CFA_restore_extended => { + let register = input.read_uleb128().and_then(Register::from_u64)?; + Ok(CallFrameInstruction::Restore { register }) + } + + constants::DW_CFA_undefined => { + let register = input.read_uleb128().and_then(Register::from_u64)?; + Ok(CallFrameInstruction::Undefined { register }) + } + + constants::DW_CFA_same_value => { + let register = input.read_uleb128().and_then(Register::from_u64)?; + Ok(CallFrameInstruction::SameValue { register }) + } + + constants::DW_CFA_register => { + let dest = input.read_uleb128().and_then(Register::from_u64)?; + let src = input.read_uleb128().and_then(Register::from_u64)?; + Ok(CallFrameInstruction::Register { + dest_register: dest, + src_register: src, + }) + } + + constants::DW_CFA_remember_state => Ok(CallFrameInstruction::RememberState), + + constants::DW_CFA_restore_state => Ok(CallFrameInstruction::RestoreState), + + constants::DW_CFA_def_cfa => { + let register = input.read_uleb128().and_then(Register::from_u64)?; + let offset = input.read_uleb128()?; + Ok(CallFrameInstruction::DefCfa { register, offset }) + } + + constants::DW_CFA_def_cfa_register => { + let register = input.read_uleb128().and_then(Register::from_u64)?; + Ok(CallFrameInstruction::DefCfaRegister { register }) + } + + constants::DW_CFA_def_cfa_offset => { + let offset = input.read_uleb128()?; + Ok(CallFrameInstruction::DefCfaOffset { offset }) + } + + constants::DW_CFA_def_cfa_expression => { + let len = input.read_uleb128().and_then(R::Offset::from_u64)?; + let expression = input.split(len)?; + Ok(CallFrameInstruction::DefCfaExpression { + expression: Expression(expression), + }) + } + + constants::DW_CFA_expression => { + let register = input.read_uleb128().and_then(Register::from_u64)?; + let len = input.read_uleb128().and_then(R::Offset::from_u64)?; + let expression = input.split(len)?; + Ok(CallFrameInstruction::Expression { + register, + expression: Expression(expression), + }) + } + + constants::DW_CFA_offset_extended_sf => { + let register = input.read_uleb128().and_then(Register::from_u64)?; + let offset = input.read_sleb128()?; + Ok(CallFrameInstruction::OffsetExtendedSf { + register, + factored_offset: offset, + }) + } + + constants::DW_CFA_def_cfa_sf => { + let register = input.read_uleb128().and_then(Register::from_u64)?; + let offset = input.read_sleb128()?; + Ok(CallFrameInstruction::DefCfaSf { + register, + factored_offset: offset, + }) + } + + constants::DW_CFA_def_cfa_offset_sf => { + let offset = input.read_sleb128()?; + Ok(CallFrameInstruction::DefCfaOffsetSf { + factored_offset: offset, + }) + } + + constants::DW_CFA_val_offset => { + let register = input.read_uleb128().and_then(Register::from_u64)?; + let offset = input.read_uleb128()?; + Ok(CallFrameInstruction::ValOffset { + register, + factored_offset: offset, + }) + } + + constants::DW_CFA_val_offset_sf => { + let register = input.read_uleb128().and_then(Register::from_u64)?; + let offset = input.read_sleb128()?; + Ok(CallFrameInstruction::ValOffsetSf { + register, + factored_offset: offset, + }) + } + + constants::DW_CFA_val_expression => { + let register = input.read_uleb128().and_then(Register::from_u64)?; + let len = input.read_uleb128().and_then(R::Offset::from_u64)?; + let expression = input.split(len)?; + Ok(CallFrameInstruction::ValExpression { + register, + expression: Expression(expression), + }) + } + + constants::DW_CFA_GNU_args_size => { + let size = input.read_uleb128()?; + Ok(CallFrameInstruction::ArgsSize { size }) + } + + otherwise => Err(Error::UnknownCallFrameInstruction(otherwise)), + } + } +} + +/// A lazy iterator parsing call frame instructions. +/// +/// Can be [used with +/// `FallibleIterator`](./index.html#using-with-fallibleiterator). +#[derive(Clone, Debug)] +pub struct CallFrameInstructionIter<'a, R: Reader> { + input: R, + address_encoding: Option, + parameters: PointerEncodingParameters<'a, R>, +} + +impl<'a, R: Reader> CallFrameInstructionIter<'a, R> { + /// Parse the next call frame instruction. + pub fn next(&mut self) -> Result>> { + if self.input.is_empty() { + return Ok(None); + } + + match CallFrameInstruction::parse(&mut self.input, self.address_encoding, &self.parameters) + { + Ok(instruction) => Ok(Some(instruction)), + Err(e) => { + self.input.empty(); + Err(e) + } + } + } +} + +#[cfg(feature = "fallible-iterator")] +impl<'a, R: Reader> fallible_iterator::FallibleIterator for CallFrameInstructionIter<'a, R> { + type Item = CallFrameInstruction; + type Error = Error; + + fn next(&mut self) -> ::core::result::Result, Self::Error> { + CallFrameInstructionIter::next(self) + } +} + +/// Parse a `DW_EH_PE_*` pointer encoding. +#[doc(hidden)] +#[inline] +fn parse_pointer_encoding(input: &mut R) -> Result { + let eh_pe = input.read_u8()?; + let eh_pe = constants::DwEhPe(eh_pe); + + if eh_pe.is_valid_encoding() { + Ok(eh_pe) + } else { + Err(Error::UnknownPointerEncoding) + } +} + +/// A decoded pointer. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum Pointer { + /// This value is the decoded pointer value. + Direct(u64), + + /// This value is *not* the pointer value, but points to the address of + /// where the real pointer value lives. In other words, deref this pointer + /// to get the real pointer value. + /// + /// Chase this pointer at your own risk: do you trust the DWARF data it came + /// from? + Indirect(u64), +} + +impl Default for Pointer { + #[inline] + fn default() -> Self { + Pointer::Direct(0) + } +} + +impl Into for Pointer { + #[inline] + fn into(self) -> u64 { + match self { + Pointer::Direct(p) | Pointer::Indirect(p) => p, + } + } +} + +impl Pointer { + #[inline] + fn new(encoding: constants::DwEhPe, address: u64) -> Pointer { + if encoding.is_indirect() { + Pointer::Indirect(address) + } else { + Pointer::Direct(address) + } + } +} + +#[derive(Clone, Debug)] +struct PointerEncodingParameters<'a, R: Reader> { + bases: &'a SectionBaseAddresses, + func_base: Option, + address_size: u8, + section: &'a R, +} + +fn parse_encoded_pointer( + encoding: constants::DwEhPe, + parameters: &PointerEncodingParameters, + input: &mut R, +) -> Result { + // TODO: check this once only in parse_pointer_encoding + if !encoding.is_valid_encoding() { + return Err(Error::UnknownPointerEncoding); + } + + if encoding == constants::DW_EH_PE_omit { + return Err(Error::CannotParseOmitPointerEncoding); + } + + let base = match encoding.application() { + constants::DW_EH_PE_absptr => 0, + constants::DW_EH_PE_pcrel => { + if let Some(section_base) = parameters.bases.section { + let offset_from_section = input.offset_from(parameters.section); + section_base.wrapping_add(offset_from_section.into_u64()) + } else { + return Err(Error::PcRelativePointerButSectionBaseIsUndefined); + } + } + constants::DW_EH_PE_textrel => { + if let Some(text) = parameters.bases.text { + text + } else { + return Err(Error::TextRelativePointerButTextBaseIsUndefined); + } + } + constants::DW_EH_PE_datarel => { + if let Some(data) = parameters.bases.data { + data + } else { + return Err(Error::DataRelativePointerButDataBaseIsUndefined); + } + } + constants::DW_EH_PE_funcrel => { + if let Some(func) = parameters.func_base { + func + } else { + return Err(Error::FuncRelativePointerInBadContext); + } + } + constants::DW_EH_PE_aligned => return Err(Error::UnsupportedPointerEncoding), + _ => unreachable!(), + }; + + let offset = match encoding.format() { + // Unsigned variants. + constants::DW_EH_PE_absptr => input.read_address(parameters.address_size), + constants::DW_EH_PE_uleb128 => input.read_uleb128(), + constants::DW_EH_PE_udata2 => input.read_u16().map(u64::from), + constants::DW_EH_PE_udata4 => input.read_u32().map(u64::from), + constants::DW_EH_PE_udata8 => input.read_u64(), + + // Signed variants. Here we sign extend the values (happens by + // default when casting a signed integer to a larger range integer + // in Rust), return them as u64, and rely on wrapping addition to do + // the right thing when adding these offsets to their bases. + constants::DW_EH_PE_sleb128 => input.read_sleb128().map(|a| a as u64), + constants::DW_EH_PE_sdata2 => input.read_i16().map(|a| a as u64), + constants::DW_EH_PE_sdata4 => input.read_i32().map(|a| a as u64), + constants::DW_EH_PE_sdata8 => input.read_i64().map(|a| a as u64), + + // That was all of the valid encoding formats. + _ => unreachable!(), + }?; + + Ok(Pointer::new(encoding, base.wrapping_add(offset))) +} + +#[cfg(test)] +mod tests { + use super::*; + use super::{parse_cfi_entry, AugmentationData, RegisterRuleMap, UnwindContext}; + use crate::common::Format; + use crate::constants; + use crate::endianity::{BigEndian, Endianity, LittleEndian, NativeEndian}; + use crate::read::{ + EndianSlice, Error, Expression, Pointer, ReaderOffsetId, Result, Section as ReadSection, + }; + use crate::test_util::GimliSectionMethods; + use alloc::boxed::Box; + use alloc::vec::Vec; + use core::marker::PhantomData; + use core::mem; + use core::u64; + use test_assembler::{Endian, Label, LabelMaker, LabelOrNum, Section, ToLabelOrNum}; + + // Ensure each test tries to read the same section kind that it wrote. + #[derive(Clone, Copy)] + struct SectionKind
(PhantomData
); + + impl SectionKind { + fn endian<'input, E>(self) -> Endian + where + E: Endianity, + T: UnwindSection>, + T::Offset: UnwindOffset, + { + if E::default().is_big_endian() { + Endian::Big + } else { + Endian::Little + } + } + + fn section<'input, E>(self, contents: &'input [u8]) -> T + where + E: Endianity, + T: UnwindSection> + ReadSection>, + T::Offset: UnwindOffset, + { + EndianSlice::new(contents, E::default()).into() + } + } + + fn debug_frame_le<'a>() -> SectionKind>> { + SectionKind(PhantomData) + } + + fn debug_frame_be<'a>() -> SectionKind>> { + SectionKind(PhantomData) + } + + fn eh_frame_le<'a>() -> SectionKind>> { + SectionKind(PhantomData) + } + + fn parse_fde( + section: Section, + input: &mut R, + get_cie: F, + ) -> Result> + where + R: Reader, + Section: UnwindSection, + O: UnwindOffset, + F: FnMut(&Section, &BaseAddresses, O) -> Result>, + { + let bases = Default::default(); + match parse_cfi_entry(&bases, §ion, input) { + Ok(Some(CieOrFde::Fde(partial))) => partial.parse(get_cie), + Ok(_) => Err(Error::NoEntryAtGivenOffset), + Err(e) => Err(e), + } + } + + // Mixin methods for `Section` to help define binary test data. + + trait CfiSectionMethods: GimliSectionMethods { + fn cie<'aug, 'input, E, T>( + self, + _kind: SectionKind, + augmentation: Option<&'aug str>, + cie: &mut CommonInformationEntry>, + ) -> Self + where + E: Endianity, + T: UnwindSection>, + T::Offset: UnwindOffset; + fn fde<'a, 'input, E, T, L>( + self, + _kind: SectionKind, + cie_offset: L, + fde: &mut FrameDescriptionEntry>, + ) -> Self + where + E: Endianity, + T: UnwindSection>, + T::Offset: UnwindOffset, + L: ToLabelOrNum<'a, u64>; + } + + impl CfiSectionMethods for Section { + fn cie<'aug, 'input, E, T>( + self, + _kind: SectionKind, + augmentation: Option<&'aug str>, + cie: &mut CommonInformationEntry>, + ) -> Self + where + E: Endianity, + T: UnwindSection>, + T::Offset: UnwindOffset, + { + cie.offset = self.size() as _; + let length = Label::new(); + let start = Label::new(); + let end = Label::new(); + + let section = match cie.format { + Format::Dwarf32 => self.D32(&length).mark(&start).D32(0xffff_ffff), + Format::Dwarf64 => { + let section = self.D32(0xffff_ffff); + section.D64(&length).mark(&start).D64(0xffff_ffff_ffff_ffff) + } + }; + + let mut section = section.D8(cie.version); + + if let Some(augmentation) = augmentation { + section = section.append_bytes(augmentation.as_bytes()); + } + + // Null terminator for augmentation string. + let section = section.D8(0); + + let section = if T::has_address_and_segment_sizes(cie.version) { + section.D8(cie.address_size).D8(cie.segment_size) + } else { + section + }; + + let section = section + .uleb(cie.code_alignment_factor) + .sleb(cie.data_alignment_factor) + .uleb(cie.return_address_register.0.into()) + .append_bytes(cie.initial_instructions.into()) + .mark(&end); + + cie.length = (&end - &start) as usize; + length.set_const(cie.length as u64); + + section + } + + fn fde<'a, 'input, E, T, L>( + self, + _kind: SectionKind, + cie_offset: L, + fde: &mut FrameDescriptionEntry>, + ) -> Self + where + E: Endianity, + T: UnwindSection>, + T::Offset: UnwindOffset, + L: ToLabelOrNum<'a, u64>, + { + fde.offset = self.size() as _; + let length = Label::new(); + let start = Label::new(); + let end = Label::new(); + + assert_eq!(fde.format, fde.cie.format); + + let section = match T::cie_offset_encoding(fde.format) { + CieOffsetEncoding::U32 => { + let section = self.D32(&length).mark(&start); + match cie_offset.to_labelornum() { + LabelOrNum::Label(ref l) => section.D32(l), + LabelOrNum::Num(o) => section.D32(o as u32), + } + } + CieOffsetEncoding::U64 => { + let section = self.D32(0xffff_ffff); + section.D64(&length).mark(&start).D64(cie_offset) + } + }; + + let section = match fde.cie.segment_size { + 0 => section, + 4 => section.D32(fde.initial_segment as u32), + 8 => section.D64(fde.initial_segment), + x => panic!("Unsupported test segment size: {}", x), + }; + + let section = match fde.cie.address_size { + 4 => section + .D32(fde.initial_address() as u32) + .D32(fde.len() as u32), + 8 => section.D64(fde.initial_address()).D64(fde.len()), + x => panic!("Unsupported address size: {}", x), + }; + + let section = if let Some(ref augmentation) = fde.augmentation { + let cie_aug = fde + .cie + .augmentation + .expect("FDE has augmentation, but CIE doesn't"); + + if let Some(lsda) = augmentation.lsda { + // We only support writing `DW_EH_PE_absptr` here. + assert_eq!( + cie_aug + .lsda + .expect("FDE has lsda, but CIE doesn't") + .format(), + constants::DW_EH_PE_absptr + ); + + // Augmentation data length + let section = section.uleb(u64::from(fde.cie.address_size)); + match fde.cie.address_size { + 4 => section.D32({ + let x: u64 = lsda.into(); + x as u32 + }), + 8 => section.D64({ + let x: u64 = lsda.into(); + x + }), + x => panic!("Unsupported address size: {}", x), + } + } else { + // Even if we don't have any augmentation data, if there is + // an augmentation defined, we need to put the length in. + section.uleb(0) + } + } else { + section + }; + + let section = section.append_bytes(fde.instructions.into()).mark(&end); + + fde.length = (&end - &start) as usize; + length.set_const(fde.length as u64); + + section + } + } + + trait ResultExt { + fn map_eof(self, input: &[u8]) -> Self; + } + + impl ResultExt for Result { + fn map_eof(self, input: &[u8]) -> Self { + match self { + Err(Error::UnexpectedEof(id)) => { + let id = ReaderOffsetId(id.0 - input.as_ptr() as u64); + Err(Error::UnexpectedEof(id)) + } + r => r, + } + } + } + + #[allow(clippy::type_complexity)] + #[allow(clippy::needless_pass_by_value)] + fn assert_parse_cie<'input, E>( + kind: SectionKind>>, + section: Section, + address_size: u8, + expected: Result<( + EndianSlice<'input, E>, + CommonInformationEntry>, + )>, + ) where + E: Endianity, + { + let section = section.get_contents().unwrap(); + let mut debug_frame = kind.section(§ion); + debug_frame.set_address_size(address_size); + let input = &mut EndianSlice::new(§ion, E::default()); + let bases = Default::default(); + let result = CommonInformationEntry::parse(&bases, &debug_frame, input); + let result = result.map(|cie| (*input, cie)).map_eof(§ion); + assert_eq!(result, expected); + } + + #[test] + fn test_parse_cie_incomplete_length_32() { + let kind = debug_frame_le(); + let section = Section::with_endian(kind.endian()).L16(5); + assert_parse_cie( + kind, + section, + 8, + Err(Error::UnexpectedEof(ReaderOffsetId(0))), + ); + } + + #[test] + fn test_parse_cie_incomplete_length_64() { + let kind = debug_frame_le(); + let section = Section::with_endian(kind.endian()) + .L32(0xffff_ffff) + .L32(12345); + assert_parse_cie( + kind, + section, + 8, + Err(Error::UnexpectedEof(ReaderOffsetId(4))), + ); + } + + #[test] + fn test_parse_cie_incomplete_id_32() { + let kind = debug_frame_be(); + let section = Section::with_endian(kind.endian()) + // The length is not large enough to contain the ID. + .B32(3) + .B32(0xffff_ffff); + assert_parse_cie( + kind, + section, + 8, + Err(Error::UnexpectedEof(ReaderOffsetId(4))), + ); + } + + #[test] + fn test_parse_cie_bad_id_32() { + let kind = debug_frame_be(); + let section = Section::with_endian(kind.endian()) + // Initial length + .B32(4) + // Not the CIE Id. + .B32(0xbad1_bad2); + assert_parse_cie(kind, section, 8, Err(Error::NotCieId)); + } + + #[test] + fn test_parse_cie_32_bad_version() { + let mut cie = CommonInformationEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + version: 99, + augmentation: None, + address_size: 4, + segment_size: 0, + code_alignment_factor: 1, + data_alignment_factor: 2, + return_address_register: Register(3), + initial_instructions: EndianSlice::new(&[], LittleEndian), + }; + + let kind = debug_frame_le(); + let section = Section::with_endian(kind.endian()).cie(kind, None, &mut cie); + assert_parse_cie(kind, section, 4, Err(Error::UnknownVersion(99))); + } + + #[test] + fn test_parse_cie_unknown_augmentation() { + let length = Label::new(); + let start = Label::new(); + let end = Label::new(); + + let augmentation = Some("replicant"); + let expected_rest = [1, 2, 3]; + + let kind = debug_frame_le(); + let section = Section::with_endian(kind.endian()) + // Initial length + .L32(&length) + .mark(&start) + // CIE Id + .L32(0xffff_ffff) + // Version + .D8(4) + // Augmentation + .append_bytes(augmentation.unwrap().as_bytes()) + // Null terminator + .D8(0) + // Extra augmented data that we can't understand. + .L32(1) + .L32(2) + .L32(3) + .L32(4) + .L32(5) + .L32(6) + .mark(&end) + .append_bytes(&expected_rest); + + let expected_length = (&end - &start) as u64; + length.set_const(expected_length); + + assert_parse_cie(kind, section, 8, Err(Error::UnknownAugmentation)); + } + + fn test_parse_cie(format: Format, version: u8, address_size: u8) { + let expected_rest = [1, 2, 3, 4, 5, 6, 7, 8, 9]; + let expected_instrs: Vec<_> = (0..4).map(|_| constants::DW_CFA_nop.0).collect(); + + let mut cie = CommonInformationEntry { + offset: 0, + length: 0, + format, + version, + augmentation: None, + address_size, + segment_size: 0, + code_alignment_factor: 16, + data_alignment_factor: 32, + return_address_register: Register(1), + initial_instructions: EndianSlice::new(&expected_instrs, LittleEndian), + }; + + let kind = debug_frame_le(); + let section = Section::with_endian(kind.endian()) + .cie(kind, None, &mut cie) + .append_bytes(&expected_rest); + + assert_parse_cie( + kind, + section, + address_size, + Ok((EndianSlice::new(&expected_rest, LittleEndian), cie)), + ); + } + + #[test] + fn test_parse_cie_32_ok() { + test_parse_cie(Format::Dwarf32, 1, 4); + test_parse_cie(Format::Dwarf32, 1, 8); + test_parse_cie(Format::Dwarf32, 4, 4); + test_parse_cie(Format::Dwarf32, 4, 8); + } + + #[test] + fn test_parse_cie_64_ok() { + test_parse_cie(Format::Dwarf64, 1, 4); + test_parse_cie(Format::Dwarf64, 1, 8); + test_parse_cie(Format::Dwarf64, 4, 4); + test_parse_cie(Format::Dwarf64, 4, 8); + } + + #[test] + fn test_parse_cie_length_too_big() { + let expected_instrs: Vec<_> = (0..13).map(|_| constants::DW_CFA_nop.0).collect(); + + let mut cie = CommonInformationEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + version: 4, + augmentation: None, + address_size: 4, + segment_size: 0, + code_alignment_factor: 0, + data_alignment_factor: 0, + return_address_register: Register(3), + initial_instructions: EndianSlice::new(&expected_instrs, LittleEndian), + }; + + let kind = debug_frame_le(); + let section = Section::with_endian(kind.endian()).cie(kind, None, &mut cie); + + let mut contents = section.get_contents().unwrap(); + + // Overwrite the length to be too big. + contents[0] = 0; + contents[1] = 0; + contents[2] = 0; + contents[3] = 255; + + let debug_frame = DebugFrame::new(&contents, LittleEndian); + let bases = Default::default(); + assert_eq!( + CommonInformationEntry::parse( + &bases, + &debug_frame, + &mut EndianSlice::new(&contents, LittleEndian) + ) + .map_eof(&contents), + Err(Error::UnexpectedEof(ReaderOffsetId(4))) + ); + } + + #[test] + fn test_parse_fde_incomplete_length_32() { + let kind = debug_frame_le(); + let section = Section::with_endian(kind.endian()).L16(5); + let section = section.get_contents().unwrap(); + let debug_frame = kind.section(§ion); + let rest = &mut EndianSlice::new(§ion, LittleEndian); + assert_eq!( + parse_fde(debug_frame, rest, UnwindSection::cie_from_offset).map_eof(§ion), + Err(Error::UnexpectedEof(ReaderOffsetId(0))) + ); + } + + #[test] + fn test_parse_fde_incomplete_length_64() { + let kind = debug_frame_le(); + let section = Section::with_endian(kind.endian()) + .L32(0xffff_ffff) + .L32(12345); + let section = section.get_contents().unwrap(); + let debug_frame = kind.section(§ion); + let rest = &mut EndianSlice::new(§ion, LittleEndian); + assert_eq!( + parse_fde(debug_frame, rest, UnwindSection::cie_from_offset).map_eof(§ion), + Err(Error::UnexpectedEof(ReaderOffsetId(4))) + ); + } + + #[test] + fn test_parse_fde_incomplete_cie_pointer_32() { + let kind = debug_frame_be(); + let section = Section::with_endian(kind.endian()) + // The length is not large enough to contain the CIE pointer. + .B32(3) + .B32(1994); + let section = section.get_contents().unwrap(); + let debug_frame = kind.section(§ion); + let rest = &mut EndianSlice::new(§ion, BigEndian); + assert_eq!( + parse_fde(debug_frame, rest, UnwindSection::cie_from_offset).map_eof(§ion), + Err(Error::UnexpectedEof(ReaderOffsetId(4))) + ); + } + + #[test] + fn test_parse_fde_32_ok() { + let expected_rest = [1, 2, 3, 4, 5, 6, 7, 8, 9]; + let cie_offset = 0xbad0_bad1; + let expected_instrs: Vec<_> = (0..7).map(|_| constants::DW_CFA_nop.0).collect(); + + let cie = CommonInformationEntry { + offset: 0, + length: 100, + format: Format::Dwarf32, + version: 4, + augmentation: None, + // DWARF32 with a 64 bit address size! Holy moly! + address_size: 8, + segment_size: 0, + code_alignment_factor: 3, + data_alignment_factor: 2, + return_address_register: Register(1), + initial_instructions: EndianSlice::new(&[], LittleEndian), + }; + + let mut fde = FrameDescriptionEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + cie: cie.clone(), + initial_segment: 0, + initial_address: 0xfeed_beef, + address_range: 39, + augmentation: None, + instructions: EndianSlice::new(&expected_instrs, LittleEndian), + }; + + let kind = debug_frame_le(); + let section = Section::with_endian(kind.endian()) + .fde(kind, cie_offset, &mut fde) + .append_bytes(&expected_rest); + + let section = section.get_contents().unwrap(); + let debug_frame = kind.section(§ion); + let rest = &mut EndianSlice::new(§ion, LittleEndian); + + let get_cie = |_: &_, _: &_, offset| { + assert_eq!(offset, DebugFrameOffset(cie_offset as usize)); + Ok(cie.clone()) + }; + + assert_eq!(parse_fde(debug_frame, rest, get_cie), Ok(fde)); + assert_eq!(*rest, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_fde_32_with_segment_ok() { + let expected_rest = [1, 2, 3, 4, 5, 6, 7, 8, 9]; + let cie_offset = 0xbad0_bad1; + let expected_instrs: Vec<_> = (0..92).map(|_| constants::DW_CFA_nop.0).collect(); + + let cie = CommonInformationEntry { + offset: 0, + length: 100, + format: Format::Dwarf32, + version: 4, + augmentation: None, + address_size: 4, + segment_size: 4, + code_alignment_factor: 3, + data_alignment_factor: 2, + return_address_register: Register(1), + initial_instructions: EndianSlice::new(&[], LittleEndian), + }; + + let mut fde = FrameDescriptionEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + cie: cie.clone(), + initial_segment: 0xbadb_ad11, + initial_address: 0xfeed_beef, + address_range: 999, + augmentation: None, + instructions: EndianSlice::new(&expected_instrs, LittleEndian), + }; + + let kind = debug_frame_le(); + let section = Section::with_endian(kind.endian()) + .fde(kind, cie_offset, &mut fde) + .append_bytes(&expected_rest); + + let section = section.get_contents().unwrap(); + let debug_frame = kind.section(§ion); + let rest = &mut EndianSlice::new(§ion, LittleEndian); + + let get_cie = |_: &_, _: &_, offset| { + assert_eq!(offset, DebugFrameOffset(cie_offset as usize)); + Ok(cie.clone()) + }; + + assert_eq!(parse_fde(debug_frame, rest, get_cie), Ok(fde)); + assert_eq!(*rest, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_fde_64_ok() { + let expected_rest = [1, 2, 3, 4, 5, 6, 7, 8, 9]; + let cie_offset = 0xbad0_bad1; + let expected_instrs: Vec<_> = (0..7).map(|_| constants::DW_CFA_nop.0).collect(); + + let cie = CommonInformationEntry { + offset: 0, + length: 100, + format: Format::Dwarf64, + version: 4, + augmentation: None, + address_size: 8, + segment_size: 0, + code_alignment_factor: 3, + data_alignment_factor: 2, + return_address_register: Register(1), + initial_instructions: EndianSlice::new(&[], LittleEndian), + }; + + let mut fde = FrameDescriptionEntry { + offset: 0, + length: 0, + format: Format::Dwarf64, + cie: cie.clone(), + initial_segment: 0, + initial_address: 0xfeed_beef, + address_range: 999, + augmentation: None, + instructions: EndianSlice::new(&expected_instrs, LittleEndian), + }; + + let kind = debug_frame_le(); + let section = Section::with_endian(kind.endian()) + .fde(kind, cie_offset, &mut fde) + .append_bytes(&expected_rest); + + let section = section.get_contents().unwrap(); + let debug_frame = kind.section(§ion); + let rest = &mut EndianSlice::new(§ion, LittleEndian); + + let get_cie = |_: &_, _: &_, offset| { + assert_eq!(offset, DebugFrameOffset(cie_offset as usize)); + Ok(cie.clone()) + }; + + assert_eq!(parse_fde(debug_frame, rest, get_cie), Ok(fde)); + assert_eq!(*rest, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_entry_on_cie_32_ok() { + let expected_rest = [1, 2, 3, 4, 5, 6, 7, 8, 9]; + let expected_instrs: Vec<_> = (0..4).map(|_| constants::DW_CFA_nop.0).collect(); + + let mut cie = CommonInformationEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + version: 4, + augmentation: None, + address_size: 4, + segment_size: 0, + code_alignment_factor: 16, + data_alignment_factor: 32, + return_address_register: Register(1), + initial_instructions: EndianSlice::new(&expected_instrs, BigEndian), + }; + + let kind = debug_frame_be(); + let section = Section::with_endian(kind.endian()) + .cie(kind, None, &mut cie) + .append_bytes(&expected_rest); + let section = section.get_contents().unwrap(); + let debug_frame = kind.section(§ion); + let rest = &mut EndianSlice::new(§ion, BigEndian); + + let bases = Default::default(); + assert_eq!( + parse_cfi_entry(&bases, &debug_frame, rest), + Ok(Some(CieOrFde::Cie(cie))) + ); + assert_eq!(*rest, EndianSlice::new(&expected_rest, BigEndian)); + } + + #[test] + fn test_parse_cfi_entry_on_fde_32_ok() { + let cie_offset = 0x1234_5678; + let expected_rest = [1, 2, 3, 4, 5, 6, 7, 8, 9]; + let expected_instrs: Vec<_> = (0..4).map(|_| constants::DW_CFA_nop.0).collect(); + + let cie = CommonInformationEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + version: 4, + augmentation: None, + address_size: 4, + segment_size: 0, + code_alignment_factor: 16, + data_alignment_factor: 32, + return_address_register: Register(1), + initial_instructions: EndianSlice::new(&[], BigEndian), + }; + + let mut fde = FrameDescriptionEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + cie: cie.clone(), + initial_segment: 0, + initial_address: 0xfeed_beef, + address_range: 39, + augmentation: None, + instructions: EndianSlice::new(&expected_instrs, BigEndian), + }; + + let kind = debug_frame_be(); + let section = Section::with_endian(kind.endian()) + .fde(kind, cie_offset, &mut fde) + .append_bytes(&expected_rest); + + let section = section.get_contents().unwrap(); + let debug_frame = kind.section(§ion); + let rest = &mut EndianSlice::new(§ion, BigEndian); + + let bases = Default::default(); + match parse_cfi_entry(&bases, &debug_frame, rest) { + Ok(Some(CieOrFde::Fde(partial))) => { + assert_eq!(*rest, EndianSlice::new(&expected_rest, BigEndian)); + + assert_eq!(partial.length, fde.length); + assert_eq!(partial.format, fde.format); + assert_eq!(partial.cie_offset, DebugFrameOffset(cie_offset as usize)); + + let get_cie = |_: &_, _: &_, offset| { + assert_eq!(offset, DebugFrameOffset(cie_offset as usize)); + Ok(cie.clone()) + }; + + assert_eq!(partial.parse(get_cie), Ok(fde)); + } + otherwise => panic!("Unexpected result: {:#?}", otherwise), + } + } + + #[test] + fn test_cfi_entries_iter() { + let expected_instrs1: Vec<_> = (0..4).map(|_| constants::DW_CFA_nop.0).collect(); + + let expected_instrs2: Vec<_> = (0..8).map(|_| constants::DW_CFA_nop.0).collect(); + + let expected_instrs3: Vec<_> = (0..12).map(|_| constants::DW_CFA_nop.0).collect(); + + let expected_instrs4: Vec<_> = (0..16).map(|_| constants::DW_CFA_nop.0).collect(); + + let mut cie1 = CommonInformationEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + version: 4, + augmentation: None, + address_size: 4, + segment_size: 0, + code_alignment_factor: 1, + data_alignment_factor: 2, + return_address_register: Register(3), + initial_instructions: EndianSlice::new(&expected_instrs1, BigEndian), + }; + + let mut cie2 = CommonInformationEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + version: 4, + augmentation: None, + address_size: 4, + segment_size: 0, + code_alignment_factor: 3, + data_alignment_factor: 2, + return_address_register: Register(1), + initial_instructions: EndianSlice::new(&expected_instrs2, BigEndian), + }; + + let cie1_location = Label::new(); + let cie2_location = Label::new(); + + // Write the CIEs first so that their length gets set before we clone + // them into the FDEs and our equality assertions down the line end up + // with all the CIEs always having he correct length. + let kind = debug_frame_be(); + let section = Section::with_endian(kind.endian()) + .mark(&cie1_location) + .cie(kind, None, &mut cie1) + .mark(&cie2_location) + .cie(kind, None, &mut cie2); + + let mut fde1 = FrameDescriptionEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + cie: cie1.clone(), + initial_segment: 0, + initial_address: 0xfeed_beef, + address_range: 39, + augmentation: None, + instructions: EndianSlice::new(&expected_instrs3, BigEndian), + }; + + let mut fde2 = FrameDescriptionEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + cie: cie2.clone(), + initial_segment: 0, + initial_address: 0xfeed_face, + address_range: 9000, + augmentation: None, + instructions: EndianSlice::new(&expected_instrs4, BigEndian), + }; + + let section = + section + .fde(kind, &cie1_location, &mut fde1) + .fde(kind, &cie2_location, &mut fde2); + + section.start().set_const(0); + + let cie1_offset = cie1_location.value().unwrap() as usize; + let cie2_offset = cie2_location.value().unwrap() as usize; + + let contents = section.get_contents().unwrap(); + let debug_frame = kind.section(&contents); + + let bases = Default::default(); + let mut entries = debug_frame.entries(&bases); + + assert_eq!(entries.next(), Ok(Some(CieOrFde::Cie(cie1.clone())))); + assert_eq!(entries.next(), Ok(Some(CieOrFde::Cie(cie2.clone())))); + + match entries.next() { + Ok(Some(CieOrFde::Fde(partial))) => { + assert_eq!(partial.length, fde1.length); + assert_eq!(partial.format, fde1.format); + assert_eq!(partial.cie_offset, DebugFrameOffset(cie1_offset)); + + let get_cie = |_: &_, _: &_, offset| { + assert_eq!(offset, DebugFrameOffset(cie1_offset)); + Ok(cie1.clone()) + }; + assert_eq!(partial.parse(get_cie), Ok(fde1)); + } + otherwise => panic!("Unexpected result: {:#?}", otherwise), + } + + match entries.next() { + Ok(Some(CieOrFde::Fde(partial))) => { + assert_eq!(partial.length, fde2.length); + assert_eq!(partial.format, fde2.format); + assert_eq!(partial.cie_offset, DebugFrameOffset(cie2_offset)); + + let get_cie = |_: &_, _: &_, offset| { + assert_eq!(offset, DebugFrameOffset(cie2_offset)); + Ok(cie2.clone()) + }; + assert_eq!(partial.parse(get_cie), Ok(fde2)); + } + otherwise => panic!("Unexpected result: {:#?}", otherwise), + } + + assert_eq!(entries.next(), Ok(None)); + } + + #[test] + fn test_parse_cie_from_offset() { + let filler = [1, 2, 3, 4, 5, 6, 7, 8, 9]; + let instrs: Vec<_> = (0..5).map(|_| constants::DW_CFA_nop.0).collect(); + + let mut cie = CommonInformationEntry { + offset: 0, + length: 0, + format: Format::Dwarf64, + version: 4, + augmentation: None, + address_size: 4, + segment_size: 0, + code_alignment_factor: 4, + data_alignment_factor: 8, + return_address_register: Register(12), + initial_instructions: EndianSlice::new(&instrs, LittleEndian), + }; + + let cie_location = Label::new(); + + let kind = debug_frame_le(); + let section = Section::with_endian(kind.endian()) + .append_bytes(&filler) + .mark(&cie_location) + .cie(kind, None, &mut cie) + .append_bytes(&filler); + + section.start().set_const(0); + + let cie_offset = DebugFrameOffset(cie_location.value().unwrap() as usize); + + let contents = section.get_contents().unwrap(); + let debug_frame = kind.section(&contents); + let bases = Default::default(); + + assert_eq!(debug_frame.cie_from_offset(&bases, cie_offset), Ok(cie)); + } + + fn parse_cfi_instruction( + input: &mut R, + address_size: u8, + ) -> Result> { + let parameters = &PointerEncodingParameters { + bases: &SectionBaseAddresses::default(), + func_base: None, + address_size, + section: &R::default(), + }; + CallFrameInstruction::parse(input, None, parameters) + } + + #[test] + fn test_parse_cfi_instruction_advance_loc() { + let expected_rest = [1, 2, 3, 4]; + let expected_delta = 42; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_advance_loc.0 | expected_delta) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::AdvanceLoc { + delta: u32::from(expected_delta), + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_offset() { + let expected_rest = [1, 2, 3, 4]; + let expected_reg = 3; + let expected_offset = 1997; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_offset.0 | expected_reg) + .uleb(expected_offset) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::Offset { + register: Register(expected_reg.into()), + factored_offset: expected_offset, + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_restore() { + let expected_rest = [1, 2, 3, 4]; + let expected_reg = 3; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_restore.0 | expected_reg) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::Restore { + register: Register(expected_reg.into()), + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_nop() { + let expected_rest = [1, 2, 3, 4]; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_nop.0) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::Nop) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_set_loc() { + let expected_rest = [1, 2, 3, 4]; + let expected_addr = 0xdead_beef; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_set_loc.0) + .L64(expected_addr) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::SetLoc { + address: expected_addr, + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_set_loc_encoding() { + let text_base = 0xfeed_face; + let addr_offset = 0xbeef; + let expected_addr = text_base + addr_offset; + let expected_rest = [1, 2, 3, 4]; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_set_loc.0) + .L64(addr_offset) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + let parameters = &PointerEncodingParameters { + bases: &BaseAddresses::default().set_text(text_base).eh_frame, + func_base: None, + address_size: 8, + section: &EndianSlice::new(&[], LittleEndian), + }; + assert_eq!( + CallFrameInstruction::parse(input, Some(constants::DW_EH_PE_textrel), parameters), + Ok(CallFrameInstruction::SetLoc { + address: expected_addr, + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_advance_loc1() { + let expected_rest = [1, 2, 3, 4]; + let expected_delta = 8; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_advance_loc1.0) + .D8(expected_delta) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::AdvanceLoc { + delta: u32::from(expected_delta), + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_advance_loc2() { + let expected_rest = [1, 2, 3, 4]; + let expected_delta = 500; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_advance_loc2.0) + .L16(expected_delta) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::AdvanceLoc { + delta: u32::from(expected_delta), + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_advance_loc4() { + let expected_rest = [1, 2, 3, 4]; + let expected_delta = 1 << 20; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_advance_loc4.0) + .L32(expected_delta) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::AdvanceLoc { + delta: expected_delta, + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_offset_extended() { + let expected_rest = [1, 2, 3, 4]; + let expected_reg = 7; + let expected_offset = 33; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_offset_extended.0) + .uleb(expected_reg.into()) + .uleb(expected_offset) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::Offset { + register: Register(expected_reg), + factored_offset: expected_offset, + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_restore_extended() { + let expected_rest = [1, 2, 3, 4]; + let expected_reg = 7; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_restore_extended.0) + .uleb(expected_reg.into()) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::Restore { + register: Register(expected_reg), + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_undefined() { + let expected_rest = [1, 2, 3, 4]; + let expected_reg = 7; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_undefined.0) + .uleb(expected_reg.into()) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::Undefined { + register: Register(expected_reg), + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_same_value() { + let expected_rest = [1, 2, 3, 4]; + let expected_reg = 7; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_same_value.0) + .uleb(expected_reg.into()) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::SameValue { + register: Register(expected_reg), + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_register() { + let expected_rest = [1, 2, 3, 4]; + let expected_dest_reg = 7; + let expected_src_reg = 8; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_register.0) + .uleb(expected_dest_reg.into()) + .uleb(expected_src_reg.into()) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::Register { + dest_register: Register(expected_dest_reg), + src_register: Register(expected_src_reg), + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_remember_state() { + let expected_rest = [1, 2, 3, 4]; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_remember_state.0) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::RememberState) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_restore_state() { + let expected_rest = [1, 2, 3, 4]; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_restore_state.0) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::RestoreState) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_def_cfa() { + let expected_rest = [1, 2, 3, 4]; + let expected_reg = 2; + let expected_offset = 0; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_def_cfa.0) + .uleb(expected_reg.into()) + .uleb(expected_offset) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::DefCfa { + register: Register(expected_reg), + offset: expected_offset, + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_def_cfa_register() { + let expected_rest = [1, 2, 3, 4]; + let expected_reg = 2; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_def_cfa_register.0) + .uleb(expected_reg.into()) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::DefCfaRegister { + register: Register(expected_reg), + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_def_cfa_offset() { + let expected_rest = [1, 2, 3, 4]; + let expected_offset = 23; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_def_cfa_offset.0) + .uleb(expected_offset) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::DefCfaOffset { + offset: expected_offset, + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_def_cfa_expression() { + let expected_rest = [1, 2, 3, 4]; + let expected_expr = [10, 9, 8, 7, 6, 5, 4, 3, 2, 1]; + + let length = Label::new(); + let start = Label::new(); + let end = Label::new(); + + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_def_cfa_expression.0) + .D8(&length) + .mark(&start) + .append_bytes(&expected_expr) + .mark(&end) + .append_bytes(&expected_rest); + + length.set_const((&end - &start) as u64); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::DefCfaExpression { + expression: Expression(EndianSlice::new(&expected_expr, LittleEndian)), + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_expression() { + let expected_rest = [1, 2, 3, 4]; + let expected_reg = 99; + let expected_expr = [10, 9, 8, 7, 6, 5, 4, 3, 2, 1]; + + let length = Label::new(); + let start = Label::new(); + let end = Label::new(); + + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_expression.0) + .uleb(expected_reg.into()) + .D8(&length) + .mark(&start) + .append_bytes(&expected_expr) + .mark(&end) + .append_bytes(&expected_rest); + + length.set_const((&end - &start) as u64); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::Expression { + register: Register(expected_reg), + expression: Expression(EndianSlice::new(&expected_expr, LittleEndian)), + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_offset_extended_sf() { + let expected_rest = [1, 2, 3, 4]; + let expected_reg = 7; + let expected_offset = -33; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_offset_extended_sf.0) + .uleb(expected_reg.into()) + .sleb(expected_offset) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::OffsetExtendedSf { + register: Register(expected_reg), + factored_offset: expected_offset, + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_def_cfa_sf() { + let expected_rest = [1, 2, 3, 4]; + let expected_reg = 2; + let expected_offset = -9999; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_def_cfa_sf.0) + .uleb(expected_reg.into()) + .sleb(expected_offset) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::DefCfaSf { + register: Register(expected_reg), + factored_offset: expected_offset, + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_def_cfa_offset_sf() { + let expected_rest = [1, 2, 3, 4]; + let expected_offset = -123; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_def_cfa_offset_sf.0) + .sleb(expected_offset) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::DefCfaOffsetSf { + factored_offset: expected_offset, + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_val_offset() { + let expected_rest = [1, 2, 3, 4]; + let expected_reg = 50; + let expected_offset = 23; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_val_offset.0) + .uleb(expected_reg.into()) + .uleb(expected_offset) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::ValOffset { + register: Register(expected_reg), + factored_offset: expected_offset, + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_val_offset_sf() { + let expected_rest = [1, 2, 3, 4]; + let expected_reg = 50; + let expected_offset = -23; + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_val_offset_sf.0) + .uleb(expected_reg.into()) + .sleb(expected_offset) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::ValOffsetSf { + register: Register(expected_reg), + factored_offset: expected_offset, + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_val_expression() { + let expected_rest = [1, 2, 3, 4]; + let expected_reg = 50; + let expected_expr = [2, 2, 1, 1, 5, 5]; + + let length = Label::new(); + let start = Label::new(); + let end = Label::new(); + + let section = Section::with_endian(Endian::Little) + .D8(constants::DW_CFA_val_expression.0) + .uleb(expected_reg.into()) + .D8(&length) + .mark(&start) + .append_bytes(&expected_expr) + .mark(&end) + .append_bytes(&expected_rest); + + length.set_const((&end - &start) as u64); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + + assert_eq!( + parse_cfi_instruction(input, 8), + Ok(CallFrameInstruction::ValExpression { + register: Register(expected_reg), + expression: Expression(EndianSlice::new(&expected_expr, LittleEndian)), + }) + ); + assert_eq!(*input, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_cfi_instruction_unknown_instruction() { + let expected_rest = [1, 2, 3, 4]; + let unknown_instr = constants::DwCfa(0b0011_1111); + let section = Section::with_endian(Endian::Little) + .D8(unknown_instr.0) + .append_bytes(&expected_rest); + let contents = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&contents, LittleEndian); + assert_eq!( + parse_cfi_instruction(input, 8), + Err(Error::UnknownCallFrameInstruction(unknown_instr)) + ); + } + + #[test] + fn test_call_frame_instruction_iter_ok() { + let expected_reg = 50; + let expected_expr = [2, 2, 1, 1, 5, 5]; + let expected_delta = 230; + + let length = Label::new(); + let start = Label::new(); + let end = Label::new(); + + let section = Section::with_endian(Endian::Big) + .D8(constants::DW_CFA_val_expression.0) + .uleb(expected_reg.into()) + .D8(&length) + .mark(&start) + .append_bytes(&expected_expr) + .mark(&end) + .D8(constants::DW_CFA_advance_loc1.0) + .D8(expected_delta); + + length.set_const((&end - &start) as u64); + let contents = section.get_contents().unwrap(); + let input = EndianSlice::new(&contents, BigEndian); + let parameters = PointerEncodingParameters { + bases: &SectionBaseAddresses::default(), + func_base: None, + address_size: 8, + section: &EndianSlice::default(), + }; + let mut iter = CallFrameInstructionIter { + input, + address_encoding: None, + parameters, + }; + + assert_eq!( + iter.next(), + Ok(Some(CallFrameInstruction::ValExpression { + register: Register(expected_reg), + expression: Expression(EndianSlice::new(&expected_expr, BigEndian)), + })) + ); + + assert_eq!( + iter.next(), + Ok(Some(CallFrameInstruction::AdvanceLoc { + delta: u32::from(expected_delta), + })) + ); + + assert_eq!(iter.next(), Ok(None)); + } + + #[test] + fn test_call_frame_instruction_iter_err() { + // DW_CFA_advance_loc1 without an operand. + let section = Section::with_endian(Endian::Big).D8(constants::DW_CFA_advance_loc1.0); + + let contents = section.get_contents().unwrap(); + let input = EndianSlice::new(&contents, BigEndian); + let parameters = PointerEncodingParameters { + bases: &SectionBaseAddresses::default(), + func_base: None, + address_size: 8, + section: &EndianSlice::default(), + }; + let mut iter = CallFrameInstructionIter { + input, + address_encoding: None, + parameters, + }; + + assert_eq!( + iter.next().map_eof(&contents), + Err(Error::UnexpectedEof(ReaderOffsetId(1))) + ); + assert_eq!(iter.next(), Ok(None)); + } + + #[allow(clippy::needless_pass_by_value)] + fn assert_eval<'a, I>( + mut initial_ctx: UnwindContext>, + expected_ctx: UnwindContext>, + cie: CommonInformationEntry>, + fde: Option>>, + instructions: I, + ) where + I: AsRef< + [( + Result, + CallFrameInstruction>, + )], + >, + { + { + let section = &DebugFrame::from(EndianSlice::default()); + let bases = &BaseAddresses::default(); + let mut table = match fde { + Some(fde) => UnwindTable::new_for_fde(section, bases, &mut initial_ctx, &fde), + None => UnwindTable::new_for_cie(section, bases, &mut initial_ctx, &cie), + }; + for &(ref expected_result, ref instruction) in instructions.as_ref() { + assert_eq!(*expected_result, table.evaluate(instruction.clone())); + } + } + + assert_eq!(expected_ctx, initial_ctx); + } + + fn make_test_cie<'a>() -> CommonInformationEntry> { + CommonInformationEntry { + offset: 0, + format: Format::Dwarf64, + length: 0, + return_address_register: Register(0), + version: 4, + address_size: mem::size_of::() as u8, + initial_instructions: EndianSlice::new(&[], LittleEndian), + augmentation: None, + segment_size: 0, + data_alignment_factor: 2, + code_alignment_factor: 3, + } + } + + #[test] + fn test_eval_set_loc() { + let cie = make_test_cie(); + let ctx = UnwindContext::new(); + let mut expected = ctx.clone(); + expected.row_mut().end_address = 42; + let instructions = [(Ok(true), CallFrameInstruction::SetLoc { address: 42 })]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_set_loc_backwards() { + let cie = make_test_cie(); + let mut ctx = UnwindContext::new(); + ctx.row_mut().start_address = 999; + let expected = ctx.clone(); + let instructions = [( + Err(Error::InvalidAddressRange), + CallFrameInstruction::SetLoc { address: 42 }, + )]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_advance_loc() { + let cie = make_test_cie(); + let mut ctx = UnwindContext::new(); + ctx.row_mut().start_address = 3; + let mut expected = ctx.clone(); + expected.row_mut().end_address = 3 + 2 * cie.code_alignment_factor; + let instructions = [(Ok(true), CallFrameInstruction::AdvanceLoc { delta: 2 })]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_advance_loc_overflow() { + let cie = make_test_cie(); + let mut ctx = UnwindContext::new(); + ctx.row_mut().start_address = u64::MAX; + let mut expected = ctx.clone(); + expected.row_mut().end_address = 42 * cie.code_alignment_factor - 1; + let instructions = [(Ok(true), CallFrameInstruction::AdvanceLoc { delta: 42 })]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_def_cfa() { + let cie = make_test_cie(); + let ctx = UnwindContext::new(); + let mut expected = ctx.clone(); + expected.set_cfa(CfaRule::RegisterAndOffset { + register: Register(42), + offset: 36, + }); + let instructions = [( + Ok(false), + CallFrameInstruction::DefCfa { + register: Register(42), + offset: 36, + }, + )]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_def_cfa_sf() { + let cie = make_test_cie(); + let ctx = UnwindContext::new(); + let mut expected = ctx.clone(); + expected.set_cfa(CfaRule::RegisterAndOffset { + register: Register(42), + offset: 36 * cie.data_alignment_factor as i64, + }); + let instructions = [( + Ok(false), + CallFrameInstruction::DefCfaSf { + register: Register(42), + factored_offset: 36, + }, + )]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_def_cfa_register() { + let cie = make_test_cie(); + let mut ctx = UnwindContext::new(); + ctx.set_cfa(CfaRule::RegisterAndOffset { + register: Register(3), + offset: 8, + }); + let mut expected = ctx.clone(); + expected.set_cfa(CfaRule::RegisterAndOffset { + register: Register(42), + offset: 8, + }); + let instructions = [( + Ok(false), + CallFrameInstruction::DefCfaRegister { + register: Register(42), + }, + )]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_def_cfa_register_invalid_context() { + let cie = make_test_cie(); + let mut ctx = UnwindContext::new(); + ctx.set_cfa(CfaRule::Expression(Expression(EndianSlice::new( + &[], + LittleEndian, + )))); + let expected = ctx.clone(); + let instructions = [( + Err(Error::CfiInstructionInInvalidContext), + CallFrameInstruction::DefCfaRegister { + register: Register(42), + }, + )]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_def_cfa_offset() { + let cie = make_test_cie(); + let mut ctx = UnwindContext::new(); + ctx.set_cfa(CfaRule::RegisterAndOffset { + register: Register(3), + offset: 8, + }); + let mut expected = ctx.clone(); + expected.set_cfa(CfaRule::RegisterAndOffset { + register: Register(3), + offset: 42, + }); + let instructions = [(Ok(false), CallFrameInstruction::DefCfaOffset { offset: 42 })]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_def_cfa_offset_invalid_context() { + let cie = make_test_cie(); + let mut ctx = UnwindContext::new(); + ctx.set_cfa(CfaRule::Expression(Expression(EndianSlice::new( + &[], + LittleEndian, + )))); + let expected = ctx.clone(); + let instructions = [( + Err(Error::CfiInstructionInInvalidContext), + CallFrameInstruction::DefCfaOffset { offset: 1993 }, + )]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_def_cfa_expression() { + let expr = [1, 2, 3, 4]; + let cie = make_test_cie(); + let ctx = UnwindContext::new(); + let mut expected = ctx.clone(); + expected.set_cfa(CfaRule::Expression(Expression(EndianSlice::new( + &expr, + LittleEndian, + )))); + let instructions = [( + Ok(false), + CallFrameInstruction::DefCfaExpression { + expression: Expression(EndianSlice::new(&expr, LittleEndian)), + }, + )]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_undefined() { + let cie = make_test_cie(); + let ctx = UnwindContext::new(); + let mut expected = ctx.clone(); + expected + .set_register_rule(Register(5), RegisterRule::Undefined) + .unwrap(); + let instructions = [( + Ok(false), + CallFrameInstruction::Undefined { + register: Register(5), + }, + )]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_same_value() { + let cie = make_test_cie(); + let ctx = UnwindContext::new(); + let mut expected = ctx.clone(); + expected + .set_register_rule(Register(0), RegisterRule::SameValue) + .unwrap(); + let instructions = [( + Ok(false), + CallFrameInstruction::SameValue { + register: Register(0), + }, + )]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_offset() { + let cie = make_test_cie(); + let ctx = UnwindContext::new(); + let mut expected = ctx.clone(); + expected + .set_register_rule( + Register(2), + RegisterRule::Offset(3 * cie.data_alignment_factor), + ) + .unwrap(); + let instructions = [( + Ok(false), + CallFrameInstruction::Offset { + register: Register(2), + factored_offset: 3, + }, + )]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_offset_extended_sf() { + let cie = make_test_cie(); + let ctx = UnwindContext::new(); + let mut expected = ctx.clone(); + expected + .set_register_rule( + Register(4), + RegisterRule::Offset(-3 * cie.data_alignment_factor), + ) + .unwrap(); + let instructions = [( + Ok(false), + CallFrameInstruction::OffsetExtendedSf { + register: Register(4), + factored_offset: -3, + }, + )]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_val_offset() { + let cie = make_test_cie(); + let ctx = UnwindContext::new(); + let mut expected = ctx.clone(); + expected + .set_register_rule( + Register(5), + RegisterRule::ValOffset(7 * cie.data_alignment_factor), + ) + .unwrap(); + let instructions = [( + Ok(false), + CallFrameInstruction::ValOffset { + register: Register(5), + factored_offset: 7, + }, + )]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_val_offset_sf() { + let cie = make_test_cie(); + let ctx = UnwindContext::new(); + let mut expected = ctx.clone(); + expected + .set_register_rule( + Register(5), + RegisterRule::ValOffset(-7 * cie.data_alignment_factor), + ) + .unwrap(); + let instructions = [( + Ok(false), + CallFrameInstruction::ValOffsetSf { + register: Register(5), + factored_offset: -7, + }, + )]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_expression() { + let expr = [1, 2, 3, 4]; + let cie = make_test_cie(); + let ctx = UnwindContext::new(); + let mut expected = ctx.clone(); + expected + .set_register_rule( + Register(9), + RegisterRule::Expression(Expression(EndianSlice::new(&expr, LittleEndian))), + ) + .unwrap(); + let instructions = [( + Ok(false), + CallFrameInstruction::Expression { + register: Register(9), + expression: Expression(EndianSlice::new(&expr, LittleEndian)), + }, + )]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_val_expression() { + let expr = [1, 2, 3, 4]; + let cie = make_test_cie(); + let ctx = UnwindContext::new(); + let mut expected = ctx.clone(); + expected + .set_register_rule( + Register(9), + RegisterRule::ValExpression(Expression(EndianSlice::new(&expr, LittleEndian))), + ) + .unwrap(); + let instructions = [( + Ok(false), + CallFrameInstruction::ValExpression { + register: Register(9), + expression: Expression(EndianSlice::new(&expr, LittleEndian)), + }, + )]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_restore() { + let cie = make_test_cie(); + let fde = FrameDescriptionEntry { + offset: 0, + format: Format::Dwarf64, + length: 0, + address_range: 0, + augmentation: None, + initial_address: 0, + initial_segment: 0, + cie: cie.clone(), + instructions: EndianSlice::new(&[], LittleEndian), + }; + + let mut ctx = UnwindContext::new(); + ctx.set_register_rule(Register(0), RegisterRule::Offset(1)) + .unwrap(); + ctx.save_initial_rules().unwrap(); + let expected = ctx.clone(); + ctx.set_register_rule(Register(0), RegisterRule::Offset(2)) + .unwrap(); + + let instructions = [( + Ok(false), + CallFrameInstruction::Restore { + register: Register(0), + }, + )]; + assert_eval(ctx, expected, cie, Some(fde), instructions); + } + + #[test] + fn test_eval_restore_havent_saved_initial_context() { + let cie = make_test_cie(); + let ctx = UnwindContext::new(); + let expected = ctx.clone(); + let instructions = [( + Err(Error::CfiInstructionInInvalidContext), + CallFrameInstruction::Restore { + register: Register(0), + }, + )]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_remember_state() { + let cie = make_test_cie(); + let ctx = UnwindContext::new(); + let mut expected = ctx.clone(); + expected.push_row().unwrap(); + let instructions = [(Ok(false), CallFrameInstruction::RememberState)]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_restore_state() { + let cie = make_test_cie(); + + let mut ctx = UnwindContext::new(); + ctx.set_start_address(1); + ctx.set_register_rule(Register(0), RegisterRule::SameValue) + .unwrap(); + let mut expected = ctx.clone(); + ctx.push_row().unwrap(); + ctx.set_start_address(2); + ctx.set_register_rule(Register(0), RegisterRule::Offset(16)) + .unwrap(); + + // Restore state should preserve current location. + expected.set_start_address(2); + + let instructions = [ + // First one pops just fine. + (Ok(false), CallFrameInstruction::RestoreState), + // Second pop would try to pop out of bounds. + ( + Err(Error::PopWithEmptyStack), + CallFrameInstruction::RestoreState, + ), + ]; + + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_eval_nop() { + let cie = make_test_cie(); + let ctx = UnwindContext::new(); + let expected = ctx.clone(); + let instructions = [(Ok(false), CallFrameInstruction::Nop)]; + assert_eval(ctx, expected, cie, None, instructions); + } + + #[test] + fn test_unwind_table_cie_no_rule() { + #[allow(clippy::identity_op)] + let initial_instructions = Section::with_endian(Endian::Little) + // The CFA is -12 from register 4. + .D8(constants::DW_CFA_def_cfa_sf.0) + .uleb(4) + .sleb(-12) + .append_repeated(constants::DW_CFA_nop.0, 4); + let initial_instructions = initial_instructions.get_contents().unwrap(); + + let cie = CommonInformationEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + version: 4, + augmentation: None, + address_size: 8, + segment_size: 0, + code_alignment_factor: 1, + data_alignment_factor: 1, + return_address_register: Register(3), + initial_instructions: EndianSlice::new(&initial_instructions, LittleEndian), + }; + + let instructions = Section::with_endian(Endian::Little) + // A bunch of nop padding. + .append_repeated(constants::DW_CFA_nop.0, 8); + let instructions = instructions.get_contents().unwrap(); + + let fde = FrameDescriptionEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + cie: cie.clone(), + initial_segment: 0, + initial_address: 0, + address_range: 100, + augmentation: None, + instructions: EndianSlice::new(&instructions, LittleEndian), + }; + + let section = &DebugFrame::from(EndianSlice::default()); + let bases = &BaseAddresses::default(); + let mut ctx = Box::new(UnwindContext::new()); + + let mut table = fde + .rows(section, bases, &mut ctx) + .expect("Should run initial program OK"); + assert!(table.ctx.is_initialized); + let expected_initial_rule = (Register(0), RegisterRule::Undefined); + assert_eq!(table.ctx.initial_rule, Some(expected_initial_rule)); + + { + let row = table.next_row().expect("Should evaluate first row OK"); + let expected = UnwindTableRow { + start_address: 0, + end_address: 100, + saved_args_size: 0, + cfa: CfaRule::RegisterAndOffset { + register: Register(4), + offset: -12, + }, + registers: [].iter().collect(), + }; + assert_eq!(Some(&expected), row); + } + + // All done! + assert_eq!(Ok(None), table.next_row()); + assert_eq!(Ok(None), table.next_row()); + } + + #[test] + fn test_unwind_table_cie_single_rule() { + #[allow(clippy::identity_op)] + let initial_instructions = Section::with_endian(Endian::Little) + // The CFA is -12 from register 4. + .D8(constants::DW_CFA_def_cfa_sf.0) + .uleb(4) + .sleb(-12) + // Register 3 is 4 from the CFA. + .D8(constants::DW_CFA_offset.0 | 3) + .uleb(4) + .append_repeated(constants::DW_CFA_nop.0, 4); + let initial_instructions = initial_instructions.get_contents().unwrap(); + + let cie = CommonInformationEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + version: 4, + augmentation: None, + address_size: 8, + segment_size: 0, + code_alignment_factor: 1, + data_alignment_factor: 1, + return_address_register: Register(3), + initial_instructions: EndianSlice::new(&initial_instructions, LittleEndian), + }; + + let instructions = Section::with_endian(Endian::Little) + // A bunch of nop padding. + .append_repeated(constants::DW_CFA_nop.0, 8); + let instructions = instructions.get_contents().unwrap(); + + let fde = FrameDescriptionEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + cie: cie.clone(), + initial_segment: 0, + initial_address: 0, + address_range: 100, + augmentation: None, + instructions: EndianSlice::new(&instructions, LittleEndian), + }; + + let section = &DebugFrame::from(EndianSlice::default()); + let bases = &BaseAddresses::default(); + let mut ctx = Box::new(UnwindContext::new()); + + let mut table = fde + .rows(section, bases, &mut ctx) + .expect("Should run initial program OK"); + assert!(table.ctx.is_initialized); + let expected_initial_rule = (Register(3), RegisterRule::Offset(4)); + assert_eq!(table.ctx.initial_rule, Some(expected_initial_rule)); + + { + let row = table.next_row().expect("Should evaluate first row OK"); + let expected = UnwindTableRow { + start_address: 0, + end_address: 100, + saved_args_size: 0, + cfa: CfaRule::RegisterAndOffset { + register: Register(4), + offset: -12, + }, + registers: [(Register(3), RegisterRule::Offset(4))].iter().collect(), + }; + assert_eq!(Some(&expected), row); + } + + // All done! + assert_eq!(Ok(None), table.next_row()); + assert_eq!(Ok(None), table.next_row()); + } + + #[test] + fn test_unwind_table_next_row() { + #[allow(clippy::identity_op)] + let initial_instructions = Section::with_endian(Endian::Little) + // The CFA is -12 from register 4. + .D8(constants::DW_CFA_def_cfa_sf.0) + .uleb(4) + .sleb(-12) + // Register 0 is 8 from the CFA. + .D8(constants::DW_CFA_offset.0 | 0) + .uleb(8) + // Register 3 is 4 from the CFA. + .D8(constants::DW_CFA_offset.0 | 3) + .uleb(4) + .append_repeated(constants::DW_CFA_nop.0, 4); + let initial_instructions = initial_instructions.get_contents().unwrap(); + + let cie = CommonInformationEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + version: 4, + augmentation: None, + address_size: 8, + segment_size: 0, + code_alignment_factor: 1, + data_alignment_factor: 1, + return_address_register: Register(3), + initial_instructions: EndianSlice::new(&initial_instructions, LittleEndian), + }; + + let instructions = Section::with_endian(Endian::Little) + // Initial instructions form a row, advance the address by 1. + .D8(constants::DW_CFA_advance_loc1.0) + .D8(1) + // Register 0 is -16 from the CFA. + .D8(constants::DW_CFA_offset_extended_sf.0) + .uleb(0) + .sleb(-16) + // Finish this row, advance the address by 32. + .D8(constants::DW_CFA_advance_loc1.0) + .D8(32) + // Register 3 is -4 from the CFA. + .D8(constants::DW_CFA_offset_extended_sf.0) + .uleb(3) + .sleb(-4) + // Finish this row, advance the address by 64. + .D8(constants::DW_CFA_advance_loc1.0) + .D8(64) + // Register 5 is 4 from the CFA. + .D8(constants::DW_CFA_offset.0 | 5) + .uleb(4) + // A bunch of nop padding. + .append_repeated(constants::DW_CFA_nop.0, 8); + let instructions = instructions.get_contents().unwrap(); + + let fde = FrameDescriptionEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + cie: cie.clone(), + initial_segment: 0, + initial_address: 0, + address_range: 100, + augmentation: None, + instructions: EndianSlice::new(&instructions, LittleEndian), + }; + + let section = &DebugFrame::from(EndianSlice::default()); + let bases = &BaseAddresses::default(); + let mut ctx = Box::new(UnwindContext::new()); + + let mut table = fde + .rows(section, bases, &mut ctx) + .expect("Should run initial program OK"); + assert!(table.ctx.is_initialized); + assert!(table.ctx.initial_rule.is_none()); + let expected_initial_rules: RegisterRuleMap<_> = [ + (Register(0), RegisterRule::Offset(8)), + (Register(3), RegisterRule::Offset(4)), + ] + .iter() + .collect(); + assert_eq!(table.ctx.stack[0].registers, expected_initial_rules); + + { + let row = table.next_row().expect("Should evaluate first row OK"); + let expected = UnwindTableRow { + start_address: 0, + end_address: 1, + saved_args_size: 0, + cfa: CfaRule::RegisterAndOffset { + register: Register(4), + offset: -12, + }, + registers: [ + (Register(0), RegisterRule::Offset(8)), + (Register(3), RegisterRule::Offset(4)), + ] + .iter() + .collect(), + }; + assert_eq!(Some(&expected), row); + } + + { + let row = table.next_row().expect("Should evaluate second row OK"); + let expected = UnwindTableRow { + start_address: 1, + end_address: 33, + saved_args_size: 0, + cfa: CfaRule::RegisterAndOffset { + register: Register(4), + offset: -12, + }, + registers: [ + (Register(0), RegisterRule::Offset(-16)), + (Register(3), RegisterRule::Offset(4)), + ] + .iter() + .collect(), + }; + assert_eq!(Some(&expected), row); + } + + { + let row = table.next_row().expect("Should evaluate third row OK"); + let expected = UnwindTableRow { + start_address: 33, + end_address: 97, + saved_args_size: 0, + cfa: CfaRule::RegisterAndOffset { + register: Register(4), + offset: -12, + }, + registers: [ + (Register(0), RegisterRule::Offset(-16)), + (Register(3), RegisterRule::Offset(-4)), + ] + .iter() + .collect(), + }; + assert_eq!(Some(&expected), row); + } + + { + let row = table.next_row().expect("Should evaluate fourth row OK"); + let expected = UnwindTableRow { + start_address: 97, + end_address: 100, + saved_args_size: 0, + cfa: CfaRule::RegisterAndOffset { + register: Register(4), + offset: -12, + }, + registers: [ + (Register(0), RegisterRule::Offset(-16)), + (Register(3), RegisterRule::Offset(-4)), + (Register(5), RegisterRule::Offset(4)), + ] + .iter() + .collect(), + }; + assert_eq!(Some(&expected), row); + } + + // All done! + assert_eq!(Ok(None), table.next_row()); + assert_eq!(Ok(None), table.next_row()); + } + + #[test] + fn test_unwind_info_for_address_ok() { + let instrs1 = Section::with_endian(Endian::Big) + // The CFA is -12 from register 4. + .D8(constants::DW_CFA_def_cfa_sf.0) + .uleb(4) + .sleb(-12); + let instrs1 = instrs1.get_contents().unwrap(); + + let instrs2: Vec<_> = (0..8).map(|_| constants::DW_CFA_nop.0).collect(); + + let instrs3 = Section::with_endian(Endian::Big) + // Initial instructions form a row, advance the address by 100. + .D8(constants::DW_CFA_advance_loc1.0) + .D8(100) + // Register 0 is -16 from the CFA. + .D8(constants::DW_CFA_offset_extended_sf.0) + .uleb(0) + .sleb(-16); + let instrs3 = instrs3.get_contents().unwrap(); + + let instrs4: Vec<_> = (0..16).map(|_| constants::DW_CFA_nop.0).collect(); + + let mut cie1 = CommonInformationEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + version: 4, + augmentation: None, + address_size: 8, + segment_size: 0, + code_alignment_factor: 1, + data_alignment_factor: 1, + return_address_register: Register(3), + initial_instructions: EndianSlice::new(&instrs1, BigEndian), + }; + + let mut cie2 = CommonInformationEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + version: 4, + augmentation: None, + address_size: 4, + segment_size: 0, + code_alignment_factor: 1, + data_alignment_factor: 1, + return_address_register: Register(1), + initial_instructions: EndianSlice::new(&instrs2, BigEndian), + }; + + let cie1_location = Label::new(); + let cie2_location = Label::new(); + + // Write the CIEs first so that their length gets set before we clone + // them into the FDEs and our equality assertions down the line end up + // with all the CIEs always having he correct length. + let kind = debug_frame_be(); + let section = Section::with_endian(kind.endian()) + .mark(&cie1_location) + .cie(kind, None, &mut cie1) + .mark(&cie2_location) + .cie(kind, None, &mut cie2); + + let mut fde1 = FrameDescriptionEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + cie: cie1.clone(), + initial_segment: 0, + initial_address: 0xfeed_beef, + address_range: 200, + augmentation: None, + instructions: EndianSlice::new(&instrs3, BigEndian), + }; + + let mut fde2 = FrameDescriptionEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + cie: cie2.clone(), + initial_segment: 0, + initial_address: 0xfeed_face, + address_range: 9000, + augmentation: None, + instructions: EndianSlice::new(&instrs4, BigEndian), + }; + + let section = + section + .fde(kind, &cie1_location, &mut fde1) + .fde(kind, &cie2_location, &mut fde2); + section.start().set_const(0); + + let contents = section.get_contents().unwrap(); + let debug_frame = kind.section(&contents); + + // Get the second row of the unwind table in `instrs3`. + let bases = Default::default(); + let mut ctx = Box::new(UnwindContext::new()); + let result = debug_frame.unwind_info_for_address( + &bases, + &mut ctx, + 0xfeed_beef + 150, + DebugFrame::cie_from_offset, + ); + assert!(result.is_ok()); + let unwind_info = result.unwrap(); + + assert_eq!( + *unwind_info, + UnwindTableRow { + start_address: fde1.initial_address() + 100, + end_address: fde1.initial_address() + fde1.len(), + saved_args_size: 0, + cfa: CfaRule::RegisterAndOffset { + register: Register(4), + offset: -12, + }, + registers: [(Register(0), RegisterRule::Offset(-16))].iter().collect(), + } + ); + } + + #[test] + fn test_unwind_info_for_address_not_found() { + let debug_frame = DebugFrame::new(&[], NativeEndian); + let bases = Default::default(); + let mut ctx = Box::new(UnwindContext::new()); + let result = debug_frame.unwind_info_for_address( + &bases, + &mut ctx, + 0xbadb_ad99, + DebugFrame::cie_from_offset, + ); + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), Error::NoUnwindInfoForAddress); + } + + #[test] + fn test_eh_frame_hdr_unknown_version() { + let bases = BaseAddresses::default(); + let buf = &[42]; + let result = EhFrameHdr::new(buf, NativeEndian).parse(&bases, 8); + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), Error::UnknownVersion(42)); + } + + #[test] + fn test_eh_frame_hdr_omit_ehptr() { + let section = Section::with_endian(Endian::Little) + .L8(1) + .L8(0xff) + .L8(0x03) + .L8(0x0b) + .L32(2) + .L32(10) + .L32(1) + .L32(20) + .L32(2) + .L32(0); + let section = section.get_contents().unwrap(); + let bases = BaseAddresses::default(); + let result = EhFrameHdr::new(§ion, LittleEndian).parse(&bases, 8); + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), Error::CannotParseOmitPointerEncoding); + } + + #[test] + fn test_eh_frame_hdr_omit_count() { + let section = Section::with_endian(Endian::Little) + .L8(1) + .L8(0x0b) + .L8(0xff) + .L8(0x0b) + .L32(0x12345); + let section = section.get_contents().unwrap(); + let bases = BaseAddresses::default(); + let result = EhFrameHdr::new(§ion, LittleEndian).parse(&bases, 8); + assert!(result.is_ok()); + let result = result.unwrap(); + assert_eq!(result.eh_frame_ptr(), Pointer::Direct(0x12345)); + assert!(result.table().is_none()); + } + + #[test] + fn test_eh_frame_hdr_omit_table() { + let section = Section::with_endian(Endian::Little) + .L8(1) + .L8(0x0b) + .L8(0x03) + .L8(0xff) + .L32(0x12345) + .L32(2); + let section = section.get_contents().unwrap(); + let bases = BaseAddresses::default(); + let result = EhFrameHdr::new(§ion, LittleEndian).parse(&bases, 8); + assert!(result.is_ok()); + let result = result.unwrap(); + assert_eq!(result.eh_frame_ptr(), Pointer::Direct(0x12345)); + assert!(result.table().is_none()); + } + + #[test] + fn test_eh_frame_hdr_varlen_table() { + let section = Section::with_endian(Endian::Little) + .L8(1) + .L8(0x0b) + .L8(0x03) + .L8(0x01) + .L32(0x12345) + .L32(2); + let section = section.get_contents().unwrap(); + let bases = BaseAddresses::default(); + let result = EhFrameHdr::new(§ion, LittleEndian).parse(&bases, 8); + assert!(result.is_ok()); + let result = result.unwrap(); + assert_eq!(result.eh_frame_ptr(), Pointer::Direct(0x12345)); + let table = result.table(); + assert!(table.is_some()); + let table = table.unwrap(); + assert_eq!( + table.lookup(0, &bases), + Err(Error::VariableLengthSearchTable) + ); + } + + #[test] + fn test_eh_frame_hdr_indirect_length() { + let section = Section::with_endian(Endian::Little) + .L8(1) + .L8(0x0b) + .L8(0x83) + .L8(0x0b) + .L32(0x12345) + .L32(2); + let section = section.get_contents().unwrap(); + let bases = BaseAddresses::default(); + let result = EhFrameHdr::new(§ion, LittleEndian).parse(&bases, 8); + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), Error::UnsupportedPointerEncoding); + } + + #[test] + fn test_eh_frame_hdr_indirect_ptrs() { + let section = Section::with_endian(Endian::Little) + .L8(1) + .L8(0x8b) + .L8(0x03) + .L8(0x8b) + .L32(0x12345) + .L32(2) + .L32(10) + .L32(1) + .L32(20) + .L32(2); + let section = section.get_contents().unwrap(); + let bases = BaseAddresses::default(); + let result = EhFrameHdr::new(§ion, LittleEndian).parse(&bases, 8); + assert!(result.is_ok()); + let result = result.unwrap(); + assert_eq!(result.eh_frame_ptr(), Pointer::Indirect(0x12345)); + let table = result.table(); + assert!(table.is_some()); + let table = table.unwrap(); + assert_eq!( + table.lookup(0, &bases), + Err(Error::UnsupportedPointerEncoding) + ); + } + + #[test] + fn test_eh_frame_hdr_good() { + let section = Section::with_endian(Endian::Little) + .L8(1) + .L8(0x0b) + .L8(0x03) + .L8(0x0b) + .L32(0x12345) + .L32(2) + .L32(10) + .L32(1) + .L32(20) + .L32(2); + let section = section.get_contents().unwrap(); + let bases = BaseAddresses::default(); + let result = EhFrameHdr::new(§ion, LittleEndian).parse(&bases, 8); + assert!(result.is_ok()); + let result = result.unwrap(); + assert_eq!(result.eh_frame_ptr(), Pointer::Direct(0x12345)); + let table = result.table(); + assert!(table.is_some()); + let table = table.unwrap(); + assert_eq!(table.lookup(0, &bases), Ok(Pointer::Direct(1))); + assert_eq!(table.lookup(9, &bases), Ok(Pointer::Direct(1))); + assert_eq!(table.lookup(10, &bases), Ok(Pointer::Direct(1))); + assert_eq!(table.lookup(11, &bases), Ok(Pointer::Direct(1))); + assert_eq!(table.lookup(19, &bases), Ok(Pointer::Direct(1))); + assert_eq!(table.lookup(20, &bases), Ok(Pointer::Direct(2))); + assert_eq!(table.lookup(21, &bases), Ok(Pointer::Direct(2))); + assert_eq!(table.lookup(100_000, &bases), Ok(Pointer::Direct(2))); + } + + #[test] + fn test_eh_frame_fde_for_address_good() { + // First, setup eh_frame + // Write the CIE first so that its length gets set before we clone it + // into the FDE. + let mut cie = make_test_cie(); + cie.format = Format::Dwarf32; + cie.version = 1; + + let start_of_cie = Label::new(); + let end_of_cie = Label::new(); + + let kind = eh_frame_le(); + let section = Section::with_endian(kind.endian()) + .append_repeated(0, 16) + .mark(&start_of_cie) + .cie(kind, None, &mut cie) + .mark(&end_of_cie); + + let mut fde1 = FrameDescriptionEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + cie: cie.clone(), + initial_segment: 0, + initial_address: 9, + address_range: 4, + augmentation: None, + instructions: EndianSlice::new(&[], LittleEndian), + }; + let mut fde2 = FrameDescriptionEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + cie: cie.clone(), + initial_segment: 0, + initial_address: 20, + address_range: 8, + augmentation: None, + instructions: EndianSlice::new(&[], LittleEndian), + }; + + let start_of_fde1 = Label::new(); + let start_of_fde2 = Label::new(); + + let section = section + // +4 for the FDE length before the CIE offset. + .mark(&start_of_fde1) + .fde(kind, (&start_of_fde1 - &start_of_cie + 4) as u64, &mut fde1) + .mark(&start_of_fde2) + .fde(kind, (&start_of_fde2 - &start_of_cie + 4) as u64, &mut fde2); + + section.start().set_const(0); + let section = section.get_contents().unwrap(); + let section = EndianSlice::new(§ion, LittleEndian); + let eh_frame = kind.section(§ion); + + // Setup eh_frame_hdr + let section = Section::with_endian(kind.endian()) + .L8(1) + .L8(0x0b) + .L8(0x03) + .L8(0x0b) + .L32(0x12345) + .L32(2) + .L32(10) + .L32(0x12345 + start_of_fde1.value().unwrap() as u32) + .L32(20) + .L32(0x12345 + start_of_fde2.value().unwrap() as u32); + + let section = section.get_contents().unwrap(); + let bases = BaseAddresses::default(); + let eh_frame_hdr = EhFrameHdr::new(§ion, LittleEndian).parse(&bases, 8); + assert!(eh_frame_hdr.is_ok()); + let eh_frame_hdr = eh_frame_hdr.unwrap(); + + let table = eh_frame_hdr.table(); + assert!(table.is_some()); + let table = table.unwrap(); + + let bases = Default::default(); + let mut iter = table.iter(&bases); + assert_eq!( + iter.next(), + Ok(Some(( + Pointer::Direct(10), + Pointer::Direct(0x12345 + start_of_fde1.value().unwrap() as u64) + ))) + ); + assert_eq!( + iter.next(), + Ok(Some(( + Pointer::Direct(20), + Pointer::Direct(0x12345 + start_of_fde2.value().unwrap() as u64) + ))) + ); + assert_eq!(iter.next(), Ok(None)); + + assert_eq!( + table.iter(&bases).nth(0), + Ok(Some(( + Pointer::Direct(10), + Pointer::Direct(0x12345 + start_of_fde1.value().unwrap() as u64) + ))) + ); + + assert_eq!( + table.iter(&bases).nth(1), + Ok(Some(( + Pointer::Direct(20), + Pointer::Direct(0x12345 + start_of_fde2.value().unwrap() as u64) + ))) + ); + assert_eq!(table.iter(&bases).nth(2), Ok(None)); + + let f = |_: &_, _: &_, o: EhFrameOffset| { + assert_eq!(o, EhFrameOffset(start_of_cie.value().unwrap() as usize)); + Ok(cie.clone()) + }; + assert_eq!( + table.fde_for_address(&eh_frame, &bases, 9, f), + Ok(fde1.clone()) + ); + assert_eq!( + table.fde_for_address(&eh_frame, &bases, 10, f), + Ok(fde1.clone()) + ); + assert_eq!(table.fde_for_address(&eh_frame, &bases, 11, f), Ok(fde1)); + assert_eq!( + table.fde_for_address(&eh_frame, &bases, 19, f), + Err(Error::NoUnwindInfoForAddress) + ); + assert_eq!( + table.fde_for_address(&eh_frame, &bases, 20, f), + Ok(fde2.clone()) + ); + assert_eq!(table.fde_for_address(&eh_frame, &bases, 21, f), Ok(fde2)); + assert_eq!( + table.fde_for_address(&eh_frame, &bases, 100_000, f), + Err(Error::NoUnwindInfoForAddress) + ); + } + + #[test] + fn test_eh_frame_stops_at_zero_length() { + let section = Section::with_endian(Endian::Little).L32(0); + let section = section.get_contents().unwrap(); + let rest = &mut EndianSlice::new(§ion, LittleEndian); + let bases = Default::default(); + + assert_eq!( + parse_cfi_entry(&bases, &EhFrame::new(&*section, LittleEndian), rest), + Ok(None) + ); + + assert_eq!( + EhFrame::new(§ion, LittleEndian).cie_from_offset(&bases, EhFrameOffset(0)), + Err(Error::NoEntryAtGivenOffset) + ); + } + + fn resolve_cie_offset(buf: &[u8], cie_offset: usize) -> Result { + let mut fde = FrameDescriptionEntry { + offset: 0, + length: 0, + format: Format::Dwarf64, + cie: make_test_cie(), + initial_segment: 0, + initial_address: 0xfeed_beef, + address_range: 39, + augmentation: None, + instructions: EndianSlice::new(&[], LittleEndian), + }; + + let kind = eh_frame_le(); + let section = Section::with_endian(kind.endian()) + .append_bytes(&buf) + .fde(kind, cie_offset as u64, &mut fde) + .append_bytes(&buf); + + let section = section.get_contents().unwrap(); + let eh_frame = kind.section(§ion); + let input = &mut EndianSlice::new(§ion[buf.len()..], LittleEndian); + + let bases = Default::default(); + match parse_cfi_entry(&bases, &eh_frame, input) { + Ok(Some(CieOrFde::Fde(partial))) => Ok(partial.cie_offset.0), + Err(e) => Err(e), + otherwise => panic!("Unexpected result: {:#?}", otherwise), + } + } + + #[test] + fn test_eh_frame_resolve_cie_offset_ok() { + let buf = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + let cie_offset = 2; + // + 4 for size of length field + assert_eq!( + resolve_cie_offset(&buf, buf.len() + 4 - cie_offset), + Ok(cie_offset) + ); + } + + #[test] + fn test_eh_frame_resolve_cie_offset_out_of_bounds() { + let buf = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + assert_eq!( + resolve_cie_offset(&buf, buf.len() + 4 + 2), + Err(Error::OffsetOutOfBounds) + ); + } + + #[test] + fn test_eh_frame_resolve_cie_offset_underflow() { + let buf = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + assert_eq!( + resolve_cie_offset(&buf, ::core::usize::MAX), + Err(Error::OffsetOutOfBounds) + ); + } + + #[test] + fn test_eh_frame_fde_ok() { + let mut cie = make_test_cie(); + cie.format = Format::Dwarf32; + cie.version = 1; + + let start_of_cie = Label::new(); + let end_of_cie = Label::new(); + + // Write the CIE first so that its length gets set before we clone it + // into the FDE. + let kind = eh_frame_le(); + let section = Section::with_endian(kind.endian()) + .append_repeated(0, 16) + .mark(&start_of_cie) + .cie(kind, None, &mut cie) + .mark(&end_of_cie); + + let mut fde = FrameDescriptionEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + cie: cie.clone(), + initial_segment: 0, + initial_address: 0xfeed_beef, + address_range: 999, + augmentation: None, + instructions: EndianSlice::new(&[], LittleEndian), + }; + + let section = section + // +4 for the FDE length before the CIE offset. + .fde(kind, (&end_of_cie - &start_of_cie + 4) as u64, &mut fde); + + section.start().set_const(0); + let section = section.get_contents().unwrap(); + let eh_frame = kind.section(§ion); + let section = EndianSlice::new(§ion, LittleEndian); + + let mut offset = None; + match parse_fde( + eh_frame, + &mut section.range_from(end_of_cie.value().unwrap() as usize..), + |_, _, o| { + offset = Some(o); + assert_eq!(o, EhFrameOffset(start_of_cie.value().unwrap() as usize)); + Ok(cie.clone()) + }, + ) { + Ok(actual) => assert_eq!(actual, fde), + otherwise => panic!("Unexpected result {:?}", otherwise), + } + assert!(offset.is_some()); + } + + #[test] + fn test_eh_frame_fde_out_of_bounds() { + let mut cie = make_test_cie(); + cie.version = 1; + + let end_of_cie = Label::new(); + + let mut fde = FrameDescriptionEntry { + offset: 0, + length: 0, + format: Format::Dwarf64, + cie: cie.clone(), + initial_segment: 0, + initial_address: 0xfeed_beef, + address_range: 999, + augmentation: None, + instructions: EndianSlice::new(&[], LittleEndian), + }; + + let kind = eh_frame_le(); + let section = Section::with_endian(kind.endian()) + .cie(kind, None, &mut cie) + .mark(&end_of_cie) + .fde(kind, 99_999_999_999_999, &mut fde); + + section.start().set_const(0); + let section = section.get_contents().unwrap(); + let eh_frame = kind.section(§ion); + let section = EndianSlice::new(§ion, LittleEndian); + + let result = parse_fde( + eh_frame, + &mut section.range_from(end_of_cie.value().unwrap() as usize..), + UnwindSection::cie_from_offset, + ); + assert_eq!(result, Err(Error::OffsetOutOfBounds)); + } + + #[test] + fn test_augmentation_parse_not_z_augmentation() { + let augmentation = &mut EndianSlice::new(b"wtf", NativeEndian); + let bases = Default::default(); + let address_size = 8; + let section = EhFrame::new(&[], NativeEndian); + let input = &mut EndianSlice::new(&[], NativeEndian); + assert_eq!( + Augmentation::parse(augmentation, &bases, address_size, §ion, input), + Err(Error::UnknownAugmentation) + ); + } + + #[test] + fn test_augmentation_parse_just_signal_trampoline() { + let aug_str = &mut EndianSlice::new(b"S", LittleEndian); + let bases = Default::default(); + let address_size = 8; + let section = EhFrame::new(&[], LittleEndian); + let input = &mut EndianSlice::new(&[], LittleEndian); + + let mut augmentation = Augmentation::default(); + augmentation.is_signal_trampoline = true; + + assert_eq!( + Augmentation::parse(aug_str, &bases, address_size, §ion, input), + Ok(augmentation) + ); + } + + #[test] + fn test_augmentation_parse_unknown_part_of_z_augmentation() { + // The 'Z' character is not defined by the z-style augmentation. + let bases = Default::default(); + let address_size = 8; + let section = Section::with_endian(Endian::Little) + .uleb(4) + .append_repeated(4, 4) + .get_contents() + .unwrap(); + let section = EhFrame::new(§ion, LittleEndian); + let input = &mut section.section().clone(); + let augmentation = &mut EndianSlice::new(b"zZ", LittleEndian); + assert_eq!( + Augmentation::parse(augmentation, &bases, address_size, §ion, input), + Err(Error::UnknownAugmentation) + ); + } + + #[test] + #[allow(non_snake_case)] + fn test_augmentation_parse_L() { + let bases = Default::default(); + let address_size = 8; + let rest = [9, 8, 7, 6, 5, 4, 3, 2, 1]; + + let section = Section::with_endian(Endian::Little) + .uleb(1) + .D8(constants::DW_EH_PE_uleb128.0) + .append_bytes(&rest) + .get_contents() + .unwrap(); + let section = EhFrame::new(§ion, LittleEndian); + let input = &mut section.section().clone(); + let aug_str = &mut EndianSlice::new(b"zL", LittleEndian); + + let mut augmentation = Augmentation::default(); + augmentation.lsda = Some(constants::DW_EH_PE_uleb128); + + assert_eq!( + Augmentation::parse(aug_str, &bases, address_size, §ion, input), + Ok(augmentation) + ); + assert_eq!(*input, EndianSlice::new(&rest, LittleEndian)); + } + + #[test] + #[allow(non_snake_case)] + fn test_augmentation_parse_P() { + let bases = Default::default(); + let address_size = 8; + let rest = [9, 8, 7, 6, 5, 4, 3, 2, 1]; + + let section = Section::with_endian(Endian::Little) + .uleb(9) + .D8(constants::DW_EH_PE_udata8.0) + .L64(0xf00d_f00d) + .append_bytes(&rest) + .get_contents() + .unwrap(); + let section = EhFrame::new(§ion, LittleEndian); + let input = &mut section.section().clone(); + let aug_str = &mut EndianSlice::new(b"zP", LittleEndian); + + let mut augmentation = Augmentation::default(); + augmentation.personality = Some((constants::DW_EH_PE_udata8, Pointer::Direct(0xf00d_f00d))); + + assert_eq!( + Augmentation::parse(aug_str, &bases, address_size, §ion, input), + Ok(augmentation) + ); + assert_eq!(*input, EndianSlice::new(&rest, LittleEndian)); + } + + #[test] + #[allow(non_snake_case)] + fn test_augmentation_parse_R() { + let bases = Default::default(); + let address_size = 8; + let rest = [9, 8, 7, 6, 5, 4, 3, 2, 1]; + + let section = Section::with_endian(Endian::Little) + .uleb(1) + .D8(constants::DW_EH_PE_udata4.0) + .append_bytes(&rest) + .get_contents() + .unwrap(); + let section = EhFrame::new(§ion, LittleEndian); + let input = &mut section.section().clone(); + let aug_str = &mut EndianSlice::new(b"zR", LittleEndian); + + let mut augmentation = Augmentation::default(); + augmentation.fde_address_encoding = Some(constants::DW_EH_PE_udata4); + + assert_eq!( + Augmentation::parse(aug_str, &bases, address_size, §ion, input), + Ok(augmentation) + ); + assert_eq!(*input, EndianSlice::new(&rest, LittleEndian)); + } + + #[test] + #[allow(non_snake_case)] + fn test_augmentation_parse_S() { + let bases = Default::default(); + let address_size = 8; + let rest = [9, 8, 7, 6, 5, 4, 3, 2, 1]; + + let section = Section::with_endian(Endian::Little) + .uleb(0) + .append_bytes(&rest) + .get_contents() + .unwrap(); + let section = EhFrame::new(§ion, LittleEndian); + let input = &mut section.section().clone(); + let aug_str = &mut EndianSlice::new(b"zS", LittleEndian); + + let mut augmentation = Augmentation::default(); + augmentation.is_signal_trampoline = true; + + assert_eq!( + Augmentation::parse(aug_str, &bases, address_size, §ion, input), + Ok(augmentation) + ); + assert_eq!(*input, EndianSlice::new(&rest, LittleEndian)); + } + + #[test] + fn test_augmentation_parse_all() { + let bases = Default::default(); + let address_size = 8; + let rest = [9, 8, 7, 6, 5, 4, 3, 2, 1]; + + let section = Section::with_endian(Endian::Little) + .uleb(1 + 9 + 1) + // L + .D8(constants::DW_EH_PE_uleb128.0) + // P + .D8(constants::DW_EH_PE_udata8.0) + .L64(0x1bad_f00d) + // R + .D8(constants::DW_EH_PE_uleb128.0) + .append_bytes(&rest) + .get_contents() + .unwrap(); + let section = EhFrame::new(§ion, LittleEndian); + let input = &mut section.section().clone(); + let aug_str = &mut EndianSlice::new(b"zLPRS", LittleEndian); + + let augmentation = Augmentation { + lsda: Some(constants::DW_EH_PE_uleb128), + personality: Some((constants::DW_EH_PE_udata8, Pointer::Direct(0x1bad_f00d))), + fde_address_encoding: Some(constants::DW_EH_PE_uleb128), + is_signal_trampoline: true, + }; + + assert_eq!( + Augmentation::parse(aug_str, &bases, address_size, §ion, input), + Ok(augmentation) + ); + assert_eq!(*input, EndianSlice::new(&rest, LittleEndian)); + } + + #[test] + fn test_eh_frame_fde_no_augmentation() { + let instrs = [1, 2, 3, 4]; + let cie_offset = 1; + + let mut cie = make_test_cie(); + cie.format = Format::Dwarf32; + cie.version = 1; + + let mut fde = FrameDescriptionEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + cie: cie.clone(), + initial_segment: 0, + initial_address: 0xfeed_face, + address_range: 9000, + augmentation: None, + instructions: EndianSlice::new(&instrs, LittleEndian), + }; + + let rest = [1, 2, 3, 4]; + + let kind = eh_frame_le(); + let section = Section::with_endian(kind.endian()) + .fde(kind, cie_offset, &mut fde) + .append_bytes(&rest) + .get_contents() + .unwrap(); + let section = kind.section(§ion); + let input = &mut section.section().clone(); + + let result = parse_fde(section, input, |_, _, _| Ok(cie.clone())); + assert_eq!(result, Ok(fde)); + assert_eq!(*input, EndianSlice::new(&rest, LittleEndian)); + } + + #[test] + fn test_eh_frame_fde_empty_augmentation() { + let instrs = [1, 2, 3, 4]; + let cie_offset = 1; + + let mut cie = make_test_cie(); + cie.format = Format::Dwarf32; + cie.version = 1; + cie.augmentation = Some(Augmentation::default()); + + let mut fde = FrameDescriptionEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + cie: cie.clone(), + initial_segment: 0, + initial_address: 0xfeed_face, + address_range: 9000, + augmentation: Some(AugmentationData::default()), + instructions: EndianSlice::new(&instrs, LittleEndian), + }; + + let rest = [1, 2, 3, 4]; + + let kind = eh_frame_le(); + let section = Section::with_endian(kind.endian()) + .fde(kind, cie_offset, &mut fde) + .append_bytes(&rest) + .get_contents() + .unwrap(); + let section = kind.section(§ion); + let input = &mut section.section().clone(); + + let result = parse_fde(section, input, |_, _, _| Ok(cie.clone())); + assert_eq!(result, Ok(fde)); + assert_eq!(*input, EndianSlice::new(&rest, LittleEndian)); + } + + #[test] + fn test_eh_frame_fde_lsda_augmentation() { + let instrs = [1, 2, 3, 4]; + let cie_offset = 1; + + let mut cie = make_test_cie(); + cie.format = Format::Dwarf32; + cie.version = 1; + cie.augmentation = Some(Augmentation::default()); + cie.augmentation.as_mut().unwrap().lsda = Some(constants::DW_EH_PE_absptr); + + let mut fde = FrameDescriptionEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + cie: cie.clone(), + initial_segment: 0, + initial_address: 0xfeed_face, + address_range: 9000, + augmentation: Some(AugmentationData { + lsda: Some(Pointer::Direct(0x1122_3344)), + }), + instructions: EndianSlice::new(&instrs, LittleEndian), + }; + + let rest = [1, 2, 3, 4]; + + let kind = eh_frame_le(); + let section = Section::with_endian(kind.endian()) + .fde(kind, cie_offset, &mut fde) + .append_bytes(&rest) + .get_contents() + .unwrap(); + let section = kind.section(§ion); + let input = &mut section.section().clone(); + + let result = parse_fde(section, input, |_, _, _| Ok(cie.clone())); + assert_eq!(result, Ok(fde)); + assert_eq!(*input, EndianSlice::new(&rest, LittleEndian)); + } + + #[test] + fn test_eh_frame_fde_lsda_function_relative() { + let instrs = [1, 2, 3, 4]; + let cie_offset = 1; + + let mut cie = make_test_cie(); + cie.format = Format::Dwarf32; + cie.version = 1; + cie.augmentation = Some(Augmentation::default()); + cie.augmentation.as_mut().unwrap().lsda = Some(constants::DwEhPe( + constants::DW_EH_PE_funcrel.0 | constants::DW_EH_PE_absptr.0, + )); + + let mut fde = FrameDescriptionEntry { + offset: 0, + length: 0, + format: Format::Dwarf32, + cie: cie.clone(), + initial_segment: 0, + initial_address: 0xfeed_face, + address_range: 9000, + augmentation: Some(AugmentationData { + lsda: Some(Pointer::Direct(0xbeef)), + }), + instructions: EndianSlice::new(&instrs, LittleEndian), + }; + + let rest = [1, 2, 3, 4]; + + let kind = eh_frame_le(); + let section = Section::with_endian(kind.endian()) + .append_repeated(10, 10) + .fde(kind, cie_offset, &mut fde) + .append_bytes(&rest) + .get_contents() + .unwrap(); + let section = kind.section(§ion); + let input = &mut section.section().range_from(10..); + + // Adjust the FDE's augmentation to be relative to the function. + fde.augmentation.as_mut().unwrap().lsda = Some(Pointer::Direct(0xfeed_face + 0xbeef)); + + let result = parse_fde(section, input, |_, _, _| Ok(cie.clone())); + assert_eq!(result, Ok(fde)); + assert_eq!(*input, EndianSlice::new(&rest, LittleEndian)); + } + + #[test] + fn test_eh_frame_cie_personality_function_relative_bad_context() { + let instrs = [1, 2, 3, 4]; + + let length = Label::new(); + let start = Label::new(); + let end = Label::new(); + + let aug_len = Label::new(); + let aug_start = Label::new(); + let aug_end = Label::new(); + + let section = Section::with_endian(Endian::Little) + // Length + .L32(&length) + .mark(&start) + // CIE ID + .L32(0) + // Version + .D8(1) + // Augmentation + .append_bytes(b"zP\0") + // Code alignment factor + .uleb(1) + // Data alignment factor + .sleb(1) + // Return address register + .uleb(1) + // Augmentation data length. This is a uleb, be we rely on the value + // being less than 2^7 and therefore a valid uleb (can't use Label + // with uleb). + .D8(&aug_len) + .mark(&aug_start) + // Augmentation data. Personality encoding and then encoded pointer. + .D8(constants::DW_EH_PE_funcrel.0 | constants::DW_EH_PE_uleb128.0) + .uleb(1) + .mark(&aug_end) + // Initial instructions + .append_bytes(&instrs) + .mark(&end); + + length.set_const((&end - &start) as u64); + aug_len.set_const((&aug_end - &aug_start) as u64); + + let section = section.get_contents().unwrap(); + let section = EhFrame::new(§ion, LittleEndian); + + let bases = BaseAddresses::default(); + let mut iter = section.entries(&bases); + assert_eq!(iter.next(), Err(Error::FuncRelativePointerInBadContext)); + } + + #[test] + fn register_rule_map_eq() { + // Different order, but still equal. + let map1: RegisterRuleMap> = [ + (Register(0), RegisterRule::SameValue), + (Register(3), RegisterRule::Offset(1)), + ] + .iter() + .collect(); + let map2: RegisterRuleMap> = [ + (Register(3), RegisterRule::Offset(1)), + (Register(0), RegisterRule::SameValue), + ] + .iter() + .collect(); + assert_eq!(map1, map2); + assert_eq!(map2, map1); + + // Not equal. + let map3: RegisterRuleMap> = [ + (Register(0), RegisterRule::SameValue), + (Register(2), RegisterRule::Offset(1)), + ] + .iter() + .collect(); + let map4: RegisterRuleMap> = [ + (Register(3), RegisterRule::Offset(1)), + (Register(0), RegisterRule::SameValue), + ] + .iter() + .collect(); + assert!(map3 != map4); + assert!(map4 != map3); + + // One has undefined explicitly set, other implicitly has undefined. + let mut map5 = RegisterRuleMap::>::default(); + map5.set(Register(0), RegisterRule::SameValue).unwrap(); + map5.set(Register(0), RegisterRule::Undefined).unwrap(); + let map6 = RegisterRuleMap::>::default(); + assert_eq!(map5, map6); + assert_eq!(map6, map5); + } + + #[test] + fn iter_register_rules() { + let mut row = UnwindTableRow::>::default(); + row.registers = [ + (Register(0), RegisterRule::SameValue), + (Register(1), RegisterRule::Offset(1)), + (Register(2), RegisterRule::ValOffset(2)), + ] + .iter() + .collect(); + + let mut found0 = false; + let mut found1 = false; + let mut found2 = false; + + for &(register, ref rule) in row.registers() { + match register.0 { + 0 => { + assert_eq!(found0, false); + found0 = true; + assert_eq!(*rule, RegisterRule::SameValue); + } + 1 => { + assert_eq!(found1, false); + found1 = true; + assert_eq!(*rule, RegisterRule::Offset(1)); + } + 2 => { + assert_eq!(found2, false); + found2 = true; + assert_eq!(*rule, RegisterRule::ValOffset(2)); + } + x => panic!("Unexpected register rule: ({}, {:?})", x, rule), + } + } + + assert_eq!(found0, true); + assert_eq!(found1, true); + assert_eq!(found2, true); + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn size_of_unwind_ctx() { + use core::mem; + let size = mem::size_of::>>(); + let max_size = 30968; + if size > max_size { + assert_eq!(size, max_size); + } + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn size_of_register_rule_map() { + use core::mem; + let size = mem::size_of::>>(); + let max_size = 6152; + if size > max_size { + assert_eq!(size, max_size); + } + } + + #[test] + fn test_parse_pointer_encoding_ok() { + use crate::endianity::NativeEndian; + let expected = + constants::DwEhPe(constants::DW_EH_PE_uleb128.0 | constants::DW_EH_PE_pcrel.0); + let input = [expected.0, 1, 2, 3, 4]; + let input = &mut EndianSlice::new(&input, NativeEndian); + assert_eq!(parse_pointer_encoding(input), Ok(expected)); + assert_eq!(*input, EndianSlice::new(&[1, 2, 3, 4], NativeEndian)); + } + + #[test] + fn test_parse_pointer_encoding_bad_encoding() { + use crate::endianity::NativeEndian; + let expected = + constants::DwEhPe((constants::DW_EH_PE_sdata8.0 + 1) | constants::DW_EH_PE_pcrel.0); + let input = [expected.0, 1, 2, 3, 4]; + let input = &mut EndianSlice::new(&input, NativeEndian); + assert_eq!( + Err(Error::UnknownPointerEncoding), + parse_pointer_encoding(input) + ); + } + + #[test] + fn test_parse_encoded_pointer_absptr() { + let encoding = constants::DW_EH_PE_absptr; + let expected_rest = [1, 2, 3, 4]; + + let input = Section::with_endian(Endian::Little) + .L32(0xf00d_f00d) + .append_bytes(&expected_rest); + let input = input.get_contents().unwrap(); + let input = EndianSlice::new(&input, LittleEndian); + let mut rest = input; + + let parameters = PointerEncodingParameters { + bases: &SectionBaseAddresses::default(), + func_base: None, + address_size: 4, + section: &input, + }; + assert_eq!( + parse_encoded_pointer(encoding, ¶meters, &mut rest), + Ok(Pointer::Direct(0xf00d_f00d)) + ); + assert_eq!(rest, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_encoded_pointer_pcrel() { + let encoding = constants::DW_EH_PE_pcrel; + let expected_rest = [1, 2, 3, 4]; + + let input = Section::with_endian(Endian::Little) + .append_repeated(0, 0x10) + .L32(0x1) + .append_bytes(&expected_rest); + let input = input.get_contents().unwrap(); + let input = EndianSlice::new(&input, LittleEndian); + let mut rest = input.range_from(0x10..); + + let parameters = PointerEncodingParameters { + bases: &BaseAddresses::default().set_eh_frame(0x100).eh_frame, + func_base: None, + address_size: 4, + section: &input, + }; + assert_eq!( + parse_encoded_pointer(encoding, ¶meters, &mut rest), + Ok(Pointer::Direct(0x111)) + ); + assert_eq!(rest, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_encoded_pointer_pcrel_undefined() { + let encoding = constants::DW_EH_PE_pcrel; + + let input = Section::with_endian(Endian::Little).L32(0x1); + let input = input.get_contents().unwrap(); + let input = EndianSlice::new(&input, LittleEndian); + let mut rest = input; + + let parameters = PointerEncodingParameters { + bases: &SectionBaseAddresses::default(), + func_base: None, + address_size: 4, + section: &input, + }; + assert_eq!( + parse_encoded_pointer(encoding, ¶meters, &mut rest), + Err(Error::PcRelativePointerButSectionBaseIsUndefined) + ); + } + + #[test] + fn test_parse_encoded_pointer_textrel() { + let encoding = constants::DW_EH_PE_textrel; + let expected_rest = [1, 2, 3, 4]; + + let input = Section::with_endian(Endian::Little) + .L32(0x1) + .append_bytes(&expected_rest); + let input = input.get_contents().unwrap(); + let input = EndianSlice::new(&input, LittleEndian); + let mut rest = input; + + let parameters = PointerEncodingParameters { + bases: &BaseAddresses::default().set_text(0x10).eh_frame, + func_base: None, + address_size: 4, + section: &input, + }; + assert_eq!( + parse_encoded_pointer(encoding, ¶meters, &mut rest), + Ok(Pointer::Direct(0x11)) + ); + assert_eq!(rest, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_encoded_pointer_textrel_undefined() { + let encoding = constants::DW_EH_PE_textrel; + + let input = Section::with_endian(Endian::Little).L32(0x1); + let input = input.get_contents().unwrap(); + let input = EndianSlice::new(&input, LittleEndian); + let mut rest = input; + + let parameters = PointerEncodingParameters { + bases: &SectionBaseAddresses::default(), + func_base: None, + address_size: 4, + section: &input, + }; + assert_eq!( + parse_encoded_pointer(encoding, ¶meters, &mut rest), + Err(Error::TextRelativePointerButTextBaseIsUndefined) + ); + } + + #[test] + fn test_parse_encoded_pointer_datarel() { + let encoding = constants::DW_EH_PE_datarel; + let expected_rest = [1, 2, 3, 4]; + + let input = Section::with_endian(Endian::Little) + .L32(0x1) + .append_bytes(&expected_rest); + let input = input.get_contents().unwrap(); + let input = EndianSlice::new(&input, LittleEndian); + let mut rest = input; + + let parameters = PointerEncodingParameters { + bases: &BaseAddresses::default().set_got(0x10).eh_frame, + func_base: None, + address_size: 4, + section: &input, + }; + assert_eq!( + parse_encoded_pointer(encoding, ¶meters, &mut rest), + Ok(Pointer::Direct(0x11)) + ); + assert_eq!(rest, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_encoded_pointer_datarel_undefined() { + let encoding = constants::DW_EH_PE_datarel; + + let input = Section::with_endian(Endian::Little).L32(0x1); + let input = input.get_contents().unwrap(); + let input = EndianSlice::new(&input, LittleEndian); + let mut rest = input; + + let parameters = PointerEncodingParameters { + bases: &SectionBaseAddresses::default(), + func_base: None, + address_size: 4, + section: &input, + }; + assert_eq!( + parse_encoded_pointer(encoding, ¶meters, &mut rest), + Err(Error::DataRelativePointerButDataBaseIsUndefined) + ); + } + + #[test] + fn test_parse_encoded_pointer_funcrel() { + let encoding = constants::DW_EH_PE_funcrel; + let expected_rest = [1, 2, 3, 4]; + + let input = Section::with_endian(Endian::Little) + .L32(0x1) + .append_bytes(&expected_rest); + let input = input.get_contents().unwrap(); + let input = EndianSlice::new(&input, LittleEndian); + let mut rest = input; + + let parameters = PointerEncodingParameters { + bases: &SectionBaseAddresses::default(), + func_base: Some(0x10), + address_size: 4, + section: &input, + }; + assert_eq!( + parse_encoded_pointer(encoding, ¶meters, &mut rest), + Ok(Pointer::Direct(0x11)) + ); + assert_eq!(rest, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_encoded_pointer_funcrel_undefined() { + let encoding = constants::DW_EH_PE_funcrel; + + let input = Section::with_endian(Endian::Little).L32(0x1); + let input = input.get_contents().unwrap(); + let input = EndianSlice::new(&input, LittleEndian); + let mut rest = input; + + let parameters = PointerEncodingParameters { + bases: &SectionBaseAddresses::default(), + func_base: None, + address_size: 4, + section: &input, + }; + assert_eq!( + parse_encoded_pointer(encoding, ¶meters, &mut rest), + Err(Error::FuncRelativePointerInBadContext) + ); + } + + #[test] + fn test_parse_encoded_pointer_uleb128() { + let encoding = + constants::DwEhPe(constants::DW_EH_PE_absptr.0 | constants::DW_EH_PE_uleb128.0); + let expected_rest = [1, 2, 3, 4]; + + let input = Section::with_endian(Endian::Little) + .uleb(0x12_3456) + .append_bytes(&expected_rest); + let input = input.get_contents().unwrap(); + let input = EndianSlice::new(&input, LittleEndian); + let mut rest = input; + + let parameters = PointerEncodingParameters { + bases: &SectionBaseAddresses::default(), + func_base: None, + address_size: 4, + section: &input, + }; + assert_eq!( + parse_encoded_pointer(encoding, ¶meters, &mut rest), + Ok(Pointer::Direct(0x12_3456)) + ); + assert_eq!(rest, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_encoded_pointer_udata2() { + let encoding = + constants::DwEhPe(constants::DW_EH_PE_absptr.0 | constants::DW_EH_PE_udata2.0); + let expected_rest = [1, 2, 3, 4]; + + let input = Section::with_endian(Endian::Little) + .L16(0x1234) + .append_bytes(&expected_rest); + let input = input.get_contents().unwrap(); + let input = EndianSlice::new(&input, LittleEndian); + let mut rest = input; + + let parameters = PointerEncodingParameters { + bases: &SectionBaseAddresses::default(), + func_base: None, + address_size: 4, + section: &input, + }; + assert_eq!( + parse_encoded_pointer(encoding, ¶meters, &mut rest), + Ok(Pointer::Direct(0x1234)) + ); + assert_eq!(rest, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_encoded_pointer_udata4() { + let encoding = + constants::DwEhPe(constants::DW_EH_PE_absptr.0 | constants::DW_EH_PE_udata4.0); + let expected_rest = [1, 2, 3, 4]; + + let input = Section::with_endian(Endian::Little) + .L32(0x1234_5678) + .append_bytes(&expected_rest); + let input = input.get_contents().unwrap(); + let input = EndianSlice::new(&input, LittleEndian); + let mut rest = input; + + let parameters = PointerEncodingParameters { + bases: &SectionBaseAddresses::default(), + func_base: None, + address_size: 4, + section: &input, + }; + assert_eq!( + parse_encoded_pointer(encoding, ¶meters, &mut rest), + Ok(Pointer::Direct(0x1234_5678)) + ); + assert_eq!(rest, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_encoded_pointer_udata8() { + let encoding = + constants::DwEhPe(constants::DW_EH_PE_absptr.0 | constants::DW_EH_PE_udata8.0); + let expected_rest = [1, 2, 3, 4]; + + let input = Section::with_endian(Endian::Little) + .L64(0x1234_5678_1234_5678) + .append_bytes(&expected_rest); + let input = input.get_contents().unwrap(); + let input = EndianSlice::new(&input, LittleEndian); + let mut rest = input; + + let parameters = PointerEncodingParameters { + bases: &SectionBaseAddresses::default(), + func_base: None, + address_size: 4, + section: &input, + }; + assert_eq!( + parse_encoded_pointer(encoding, ¶meters, &mut rest), + Ok(Pointer::Direct(0x1234_5678_1234_5678)) + ); + assert_eq!(rest, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_encoded_pointer_sleb128() { + let encoding = + constants::DwEhPe(constants::DW_EH_PE_textrel.0 | constants::DW_EH_PE_sleb128.0); + let expected_rest = [1, 2, 3, 4]; + + let input = Section::with_endian(Endian::Little) + .sleb(-0x1111) + .append_bytes(&expected_rest); + let input = input.get_contents().unwrap(); + let input = EndianSlice::new(&input, LittleEndian); + let mut rest = input; + + let parameters = PointerEncodingParameters { + bases: &BaseAddresses::default().set_text(0x1111_1111).eh_frame, + func_base: None, + address_size: 4, + section: &input, + }; + assert_eq!( + parse_encoded_pointer(encoding, ¶meters, &mut rest), + Ok(Pointer::Direct(0x1111_0000)) + ); + assert_eq!(rest, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_encoded_pointer_sdata2() { + let encoding = + constants::DwEhPe(constants::DW_EH_PE_absptr.0 | constants::DW_EH_PE_sdata2.0); + let expected_rest = [1, 2, 3, 4]; + let expected = 0x111 as i16; + + let input = Section::with_endian(Endian::Little) + .L16(expected as u16) + .append_bytes(&expected_rest); + let input = input.get_contents().unwrap(); + let input = EndianSlice::new(&input, LittleEndian); + let mut rest = input; + + let parameters = PointerEncodingParameters { + bases: &SectionBaseAddresses::default(), + func_base: None, + address_size: 4, + section: &input, + }; + assert_eq!( + parse_encoded_pointer(encoding, ¶meters, &mut rest), + Ok(Pointer::Direct(expected as u64)) + ); + assert_eq!(rest, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_encoded_pointer_sdata4() { + let encoding = + constants::DwEhPe(constants::DW_EH_PE_absptr.0 | constants::DW_EH_PE_sdata4.0); + let expected_rest = [1, 2, 3, 4]; + let expected = 0x111_1111 as i32; + + let input = Section::with_endian(Endian::Little) + .L32(expected as u32) + .append_bytes(&expected_rest); + let input = input.get_contents().unwrap(); + let input = EndianSlice::new(&input, LittleEndian); + let mut rest = input; + + let parameters = PointerEncodingParameters { + bases: &SectionBaseAddresses::default(), + func_base: None, + address_size: 4, + section: &input, + }; + assert_eq!( + parse_encoded_pointer(encoding, ¶meters, &mut rest), + Ok(Pointer::Direct(expected as u64)) + ); + assert_eq!(rest, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_encoded_pointer_sdata8() { + let encoding = + constants::DwEhPe(constants::DW_EH_PE_absptr.0 | constants::DW_EH_PE_sdata8.0); + let expected_rest = [1, 2, 3, 4]; + let expected = -0x11_1111_1222_2222 as i64; + + let input = Section::with_endian(Endian::Little) + .L64(expected as u64) + .append_bytes(&expected_rest); + let input = input.get_contents().unwrap(); + let input = EndianSlice::new(&input, LittleEndian); + let mut rest = input; + + let parameters = PointerEncodingParameters { + bases: &SectionBaseAddresses::default(), + func_base: None, + address_size: 4, + section: &input, + }; + assert_eq!( + parse_encoded_pointer(encoding, ¶meters, &mut rest), + Ok(Pointer::Direct(expected as u64)) + ); + assert_eq!(rest, EndianSlice::new(&expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_encoded_pointer_omit() { + let encoding = constants::DW_EH_PE_omit; + + let input = Section::with_endian(Endian::Little).L32(0x1); + let input = input.get_contents().unwrap(); + let input = EndianSlice::new(&input, LittleEndian); + let mut rest = input; + + let parameters = PointerEncodingParameters { + bases: &SectionBaseAddresses::default(), + func_base: None, + address_size: 4, + section: &input, + }; + assert_eq!( + parse_encoded_pointer(encoding, ¶meters, &mut rest), + Err(Error::CannotParseOmitPointerEncoding) + ); + assert_eq!(rest, input); + } + + #[test] + fn test_parse_encoded_pointer_bad_encoding() { + let encoding = constants::DwEhPe(constants::DW_EH_PE_sdata8.0 + 1); + + let input = Section::with_endian(Endian::Little).L32(0x1); + let input = input.get_contents().unwrap(); + let input = EndianSlice::new(&input, LittleEndian); + let mut rest = input; + + let parameters = PointerEncodingParameters { + bases: &SectionBaseAddresses::default(), + func_base: None, + address_size: 4, + section: &input, + }; + assert_eq!( + parse_encoded_pointer(encoding, ¶meters, &mut rest), + Err(Error::UnknownPointerEncoding) + ); + } + + #[test] + fn test_parse_encoded_pointer_aligned() { + // FIXME: support this encoding! + + let encoding = constants::DW_EH_PE_aligned; + + let input = Section::with_endian(Endian::Little).L32(0x1); + let input = input.get_contents().unwrap(); + let input = EndianSlice::new(&input, LittleEndian); + let mut rest = input; + + let parameters = PointerEncodingParameters { + bases: &SectionBaseAddresses::default(), + func_base: None, + address_size: 4, + section: &input, + }; + assert_eq!( + parse_encoded_pointer(encoding, ¶meters, &mut rest), + Err(Error::UnsupportedPointerEncoding) + ); + } + + #[test] + fn test_parse_encoded_pointer_indirect() { + let expected_rest = [1, 2, 3, 4]; + let encoding = constants::DW_EH_PE_indirect; + + let input = Section::with_endian(Endian::Little) + .L32(0x1234_5678) + .append_bytes(&expected_rest); + let input = input.get_contents().unwrap(); + let input = EndianSlice::new(&input, LittleEndian); + let mut rest = input; + + let parameters = PointerEncodingParameters { + bases: &SectionBaseAddresses::default(), + func_base: None, + address_size: 4, + section: &input, + }; + assert_eq!( + parse_encoded_pointer(encoding, ¶meters, &mut rest), + Ok(Pointer::Indirect(0x1234_5678)) + ); + assert_eq!(rest, EndianSlice::new(&expected_rest, LittleEndian)); + } +} diff --git a/crux-mir/lib/gimli/src/read/dwarf.rs b/crux-mir/lib/gimli/src/read/dwarf.rs new file mode 100644 index 000000000..b63526941 --- /dev/null +++ b/crux-mir/lib/gimli/src/read/dwarf.rs @@ -0,0 +1,1143 @@ +use alloc::string::String; +use alloc::sync::Arc; + +use crate::common::{ + DebugAddrBase, DebugAddrIndex, DebugInfoOffset, DebugLineStrOffset, DebugLocListsBase, + DebugLocListsIndex, DebugRngListsBase, DebugRngListsIndex, DebugStrOffset, DebugStrOffsetsBase, + DebugStrOffsetsIndex, DebugTypeSignature, DebugTypesOffset, DwarfFileType, DwoId, Encoding, + LocationListsOffset, RangeListsOffset, RawRangeListsOffset, SectionId, UnitSectionOffset, +}; +use crate::constants; +use crate::read::{ + Abbreviations, AttributeValue, DebugAbbrev, DebugAddr, DebugAranges, DebugCuIndex, DebugInfo, + DebugInfoUnitHeadersIter, DebugLine, DebugLineStr, DebugLoc, DebugLocLists, DebugRngLists, + DebugStr, DebugStrOffsets, DebugTuIndex, DebugTypes, DebugTypesUnitHeadersIter, + DebuggingInformationEntry, EntriesCursor, EntriesRaw, EntriesTree, Error, + IncompleteLineProgram, LocListIter, LocationLists, Range, RangeLists, RawLocListIter, + RawRngListIter, Reader, ReaderOffset, ReaderOffsetId, Result, RngListIter, Section, UnitHeader, + UnitIndex, UnitIndexSectionIterator, UnitOffset, UnitType, +}; + +/// All of the commonly used DWARF sections, and other common information. +#[derive(Debug, Default)] +pub struct Dwarf { + /// The `.debug_abbrev` section. + pub debug_abbrev: DebugAbbrev, + + /// The `.debug_addr` section. + pub debug_addr: DebugAddr, + + /// The `.debug_aranges` section. + pub debug_aranges: DebugAranges, + + /// The `.debug_info` section. + pub debug_info: DebugInfo, + + /// The `.debug_line` section. + pub debug_line: DebugLine, + + /// The `.debug_line_str` section. + pub debug_line_str: DebugLineStr, + + /// The `.debug_str` section. + pub debug_str: DebugStr, + + /// The `.debug_str_offsets` section. + pub debug_str_offsets: DebugStrOffsets, + + /// The `.debug_types` section. + pub debug_types: DebugTypes, + + /// The location lists in the `.debug_loc` and `.debug_loclists` sections. + pub locations: LocationLists, + + /// The range lists in the `.debug_ranges` and `.debug_rnglists` sections. + pub ranges: RangeLists, + + /// The type of this file. + pub file_type: DwarfFileType, + + /// The DWARF sections for a supplementary object file. + pub sup: Option>>, +} + +impl Dwarf { + /// Try to load the DWARF sections using the given loader function. + /// + /// `section` loads a DWARF section from the object file. + /// It should return an empty section if the section does not exist. + /// + /// `section` may either directly return a `Reader` instance (such as + /// `EndianSlice`), or it may return some other type and then convert + /// that type into a `Reader` using `Dwarf::borrow`. + /// + /// After loading, the user should set the `file_type` field and + /// call `load_sup` if required. + pub fn load(mut section: F) -> core::result::Result + where + F: FnMut(SectionId) -> core::result::Result, + { + // Section types are inferred. + let debug_loc = Section::load(&mut section)?; + let debug_loclists = Section::load(&mut section)?; + let debug_ranges = Section::load(&mut section)?; + let debug_rnglists = Section::load(&mut section)?; + Ok(Dwarf { + debug_abbrev: Section::load(&mut section)?, + debug_addr: Section::load(&mut section)?, + debug_aranges: Section::load(&mut section)?, + debug_info: Section::load(&mut section)?, + debug_line: Section::load(&mut section)?, + debug_line_str: Section::load(&mut section)?, + debug_str: Section::load(&mut section)?, + debug_str_offsets: Section::load(&mut section)?, + debug_types: Section::load(&mut section)?, + locations: LocationLists::new(debug_loc, debug_loclists), + ranges: RangeLists::new(debug_ranges, debug_rnglists), + file_type: DwarfFileType::Main, + sup: None, + }) + } + + /// Load the DWARF sections from the supplementary object file. + /// + /// `section` operates the same as for `load`. + /// + /// Sets `self.sup`, replacing any previous value. + pub fn load_sup(&mut self, section: F) -> core::result::Result<(), E> + where + F: FnMut(SectionId) -> core::result::Result, + { + self.sup = Some(Arc::new(Self::load(section)?)); + Ok(()) + } + + /// Create a `Dwarf` structure that references the data in `self`. + /// + /// This is useful when `R` implements `Reader` but `T` does not. + /// + /// ## Example Usage + /// + /// It can be useful to load DWARF sections into owned data structures, + /// such as `Vec`. However, we do not implement the `Reader` trait + /// for `Vec`, because it would be very inefficient, but this trait + /// is required for all of the methods that parse the DWARF data. + /// So we first load the DWARF sections into `Vec`s, and then use + /// `borrow` to create `Reader`s that reference the data. + /// + /// ```rust,no_run + /// # fn example() -> Result<(), gimli::Error> { + /// # let loader = |name| -> Result<_, gimli::Error> { unimplemented!() }; + /// # let sup_loader = |name| -> Result<_, gimli::Error> { unimplemented!() }; + /// // Read the DWARF sections into `Vec`s with whatever object loader you're using. + /// let mut owned_dwarf: gimli::Dwarf> = gimli::Dwarf::load(loader)?; + /// owned_dwarf.load_sup(sup_loader)?; + /// // Create references to the DWARF sections. + /// let dwarf = owned_dwarf.borrow(|section| { + /// gimli::EndianSlice::new(§ion, gimli::LittleEndian) + /// }); + /// # unreachable!() + /// # } + /// ``` + pub fn borrow<'a, F, R>(&'a self, mut borrow: F) -> Dwarf + where + F: FnMut(&'a T) -> R, + { + Dwarf { + debug_abbrev: self.debug_abbrev.borrow(&mut borrow), + debug_addr: self.debug_addr.borrow(&mut borrow), + debug_aranges: self.debug_aranges.borrow(&mut borrow), + debug_info: self.debug_info.borrow(&mut borrow), + debug_line: self.debug_line.borrow(&mut borrow), + debug_line_str: self.debug_line_str.borrow(&mut borrow), + debug_str: self.debug_str.borrow(&mut borrow), + debug_str_offsets: self.debug_str_offsets.borrow(&mut borrow), + debug_types: self.debug_types.borrow(&mut borrow), + locations: self.locations.borrow(&mut borrow), + ranges: self.ranges.borrow(&mut borrow), + file_type: self.file_type, + sup: self.sup().map(|sup| Arc::new(sup.borrow(borrow))), + } + } + + /// Return a reference to the DWARF sections for supplementary object file. + pub fn sup(&self) -> Option<&Dwarf> { + self.sup.as_ref().map(Arc::as_ref) + } +} + +impl Dwarf { + /// Iterate the unit headers in the `.debug_info` section. + /// + /// Can be [used with + /// `FallibleIterator`](./index.html#using-with-fallibleiterator). + #[inline] + pub fn units(&self) -> DebugInfoUnitHeadersIter { + self.debug_info.units() + } + + /// Construct a new `Unit` from the given unit header. + #[inline] + pub fn unit(&self, header: UnitHeader) -> Result> { + Unit::new(self, header) + } + + /// Iterate the type-unit headers in the `.debug_types` section. + /// + /// Can be [used with + /// `FallibleIterator`](./index.html#using-with-fallibleiterator). + #[inline] + pub fn type_units(&self) -> DebugTypesUnitHeadersIter { + self.debug_types.units() + } + + /// Parse the abbreviations for a compilation unit. + // TODO: provide caching of abbreviations + #[inline] + pub fn abbreviations(&self, unit: &UnitHeader) -> Result { + unit.abbreviations(&self.debug_abbrev) + } + + /// Return the string offset at the given index. + #[inline] + pub fn string_offset( + &self, + unit: &Unit, + index: DebugStrOffsetsIndex, + ) -> Result> { + self.debug_str_offsets + .get_str_offset(unit.header.format(), unit.str_offsets_base, index) + } + + /// Return the string at the given offset in `.debug_str`. + #[inline] + pub fn string(&self, offset: DebugStrOffset) -> Result { + self.debug_str.get_str(offset) + } + + /// Return the string at the given offset in `.debug_line_str`. + #[inline] + pub fn line_string(&self, offset: DebugLineStrOffset) -> Result { + self.debug_line_str.get_str(offset) + } + + /// Return an attribute value as a string slice. + /// + /// If the attribute value is one of: + /// + /// - an inline `DW_FORM_string` string + /// - a `DW_FORM_strp` reference to an offset into the `.debug_str` section + /// - a `DW_FORM_strp_sup` reference to an offset into a supplementary + /// object file + /// - a `DW_FORM_line_strp` reference to an offset into the `.debug_line_str` + /// section + /// - a `DW_FORM_strx` index into the `.debug_str_offsets` entries for the unit + /// + /// then return the attribute's string value. Returns an error if the attribute + /// value does not have a string form, or if a string form has an invalid value. + pub fn attr_string(&self, unit: &Unit, attr: AttributeValue) -> Result { + match attr { + AttributeValue::String(string) => Ok(string), + AttributeValue::DebugStrRef(offset) => self.debug_str.get_str(offset), + AttributeValue::DebugStrRefSup(offset) => { + if let Some(sup) = self.sup() { + sup.debug_str.get_str(offset) + } else { + Err(Error::ExpectedStringAttributeValue) + } + } + AttributeValue::DebugLineStrRef(offset) => self.debug_line_str.get_str(offset), + AttributeValue::DebugStrOffsetsIndex(index) => { + let offset = self.debug_str_offsets.get_str_offset( + unit.header.format(), + unit.str_offsets_base, + index, + )?; + self.debug_str.get_str(offset) + } + _ => Err(Error::ExpectedStringAttributeValue), + } + } + + /// Return the address at the given index. + pub fn address(&self, unit: &Unit, index: DebugAddrIndex) -> Result { + self.debug_addr + .get_address(unit.encoding().address_size, unit.addr_base, index) + } + + /// Try to return an attribute value as an address. + /// + /// If the attribute value is one of: + /// + /// - a `DW_FORM_addr` + /// - a `DW_FORM_addrx` index into the `.debug_addr` entries for the unit + /// + /// then return the address. + /// Returns `None` for other forms. + pub fn attr_address(&self, unit: &Unit, attr: AttributeValue) -> Result> { + match attr { + AttributeValue::Addr(addr) => Ok(Some(addr)), + AttributeValue::DebugAddrIndex(index) => self.address(unit, index).map(Some), + _ => Ok(None), + } + } + + /// Return the range list offset for the given raw offset. + /// + /// This handles adding `DW_AT_GNU_ranges_base` if required. + pub fn ranges_offset_from_raw( + &self, + unit: &Unit, + offset: RawRangeListsOffset, + ) -> RangeListsOffset { + if self.file_type == DwarfFileType::Dwo && unit.header.version() < 5 { + RangeListsOffset(offset.0.wrapping_add(unit.rnglists_base.0)) + } else { + RangeListsOffset(offset.0) + } + } + + /// Return the range list offset at the given index. + pub fn ranges_offset( + &self, + unit: &Unit, + index: DebugRngListsIndex, + ) -> Result> { + self.ranges + .get_offset(unit.encoding(), unit.rnglists_base, index) + } + + /// Iterate over the `RangeListEntry`s starting at the given offset. + pub fn ranges( + &self, + unit: &Unit, + offset: RangeListsOffset, + ) -> Result> { + self.ranges.ranges( + offset, + unit.encoding(), + unit.low_pc, + &self.debug_addr, + unit.addr_base, + ) + } + + /// Iterate over the `RawRngListEntry`ies starting at the given offset. + pub fn raw_ranges( + &self, + unit: &Unit, + offset: RangeListsOffset, + ) -> Result> { + self.ranges.raw_ranges(offset, unit.encoding()) + } + + /// Try to return an attribute value as a range list offset. + /// + /// If the attribute value is one of: + /// + /// - a `DW_FORM_sec_offset` reference to the `.debug_ranges` or `.debug_rnglists` sections + /// - a `DW_FORM_rnglistx` index into the `.debug_rnglists` entries for the unit + /// + /// then return the range list offset of the range list. + /// Returns `None` for other forms. + pub fn attr_ranges_offset( + &self, + unit: &Unit, + attr: AttributeValue, + ) -> Result>> { + match attr { + AttributeValue::RangeListsRef(offset) => { + Ok(Some(self.ranges_offset_from_raw(unit, offset))) + } + AttributeValue::DebugRngListsIndex(index) => self.ranges_offset(unit, index).map(Some), + _ => Ok(None), + } + } + + /// Try to return an attribute value as a range list entry iterator. + /// + /// If the attribute value is one of: + /// + /// - a `DW_FORM_sec_offset` reference to the `.debug_ranges` or `.debug_rnglists` sections + /// - a `DW_FORM_rnglistx` index into the `.debug_rnglists` entries for the unit + /// + /// then return an iterator over the entries in the range list. + /// Returns `None` for other forms. + pub fn attr_ranges( + &self, + unit: &Unit, + attr: AttributeValue, + ) -> Result>> { + match self.attr_ranges_offset(unit, attr)? { + Some(offset) => Ok(Some(self.ranges(unit, offset)?)), + None => Ok(None), + } + } + + /// Return an iterator for the address ranges of a `DebuggingInformationEntry`. + /// + /// This uses `DW_AT_low_pc`, `DW_AT_high_pc` and `DW_AT_ranges`. + pub fn die_ranges( + &self, + unit: &Unit, + entry: &DebuggingInformationEntry, + ) -> Result> { + let mut low_pc = None; + let mut high_pc = None; + let mut size = None; + let mut attrs = entry.attrs(); + while let Some(attr) = attrs.next()? { + match attr.name() { + constants::DW_AT_low_pc => { + low_pc = Some( + self.attr_address(unit, attr.value())? + .ok_or(Error::UnsupportedAttributeForm)?, + ); + } + constants::DW_AT_high_pc => match attr.value() { + AttributeValue::Udata(val) => size = Some(val), + attr => { + high_pc = Some( + self.attr_address(unit, attr)? + .ok_or(Error::UnsupportedAttributeForm)?, + ); + } + }, + constants::DW_AT_ranges => { + if let Some(list) = self.attr_ranges(unit, attr.value())? { + return Ok(RangeIter(RangeIterInner::List(list))); + } + } + _ => {} + } + } + let range = low_pc.and_then(|begin| { + let end = size.map(|size| begin + size).or(high_pc); + // TODO: perhaps return an error if `end` is `None` + end.map(|end| Range { begin, end }) + }); + Ok(RangeIter(RangeIterInner::Single(range))) + } + + /// Return an iterator for the address ranges of a `Unit`. + /// + /// This uses `DW_AT_low_pc`, `DW_AT_high_pc` and `DW_AT_ranges` of the + /// root `DebuggingInformationEntry`. + pub fn unit_ranges(&self, unit: &Unit) -> Result> { + let mut cursor = unit.header.entries(&unit.abbreviations); + cursor.next_dfs()?; + let root = cursor.current().ok_or(Error::MissingUnitDie)?; + self.die_ranges(unit, root) + } + + /// Return the location list offset at the given index. + pub fn locations_offset( + &self, + unit: &Unit, + index: DebugLocListsIndex, + ) -> Result> { + self.locations + .get_offset(unit.encoding(), unit.loclists_base, index) + } + + /// Iterate over the `LocationListEntry`s starting at the given offset. + pub fn locations( + &self, + unit: &Unit, + offset: LocationListsOffset, + ) -> Result> { + match self.file_type { + DwarfFileType::Main => self.locations.locations( + offset, + unit.encoding(), + unit.low_pc, + &self.debug_addr, + unit.addr_base, + ), + DwarfFileType::Dwo => self.locations.locations_dwo( + offset, + unit.encoding(), + unit.low_pc, + &self.debug_addr, + unit.addr_base, + ), + } + } + + /// Iterate over the raw `LocationListEntry`s starting at the given offset. + pub fn raw_locations( + &self, + unit: &Unit, + offset: LocationListsOffset, + ) -> Result> { + match self.file_type { + DwarfFileType::Main => self.locations.raw_locations(offset, unit.encoding()), + DwarfFileType::Dwo => self.locations.raw_locations_dwo(offset, unit.encoding()), + } + } + + /// Try to return an attribute value as a location list offset. + /// + /// If the attribute value is one of: + /// + /// - a `DW_FORM_sec_offset` reference to the `.debug_loc` or `.debug_loclists` sections + /// - a `DW_FORM_loclistx` index into the `.debug_loclists` entries for the unit + /// + /// then return the location list offset of the location list. + /// Returns `None` for other forms. + pub fn attr_locations_offset( + &self, + unit: &Unit, + attr: AttributeValue, + ) -> Result>> { + match attr { + AttributeValue::LocationListsRef(offset) => Ok(Some(offset)), + AttributeValue::DebugLocListsIndex(index) => { + self.locations_offset(unit, index).map(Some) + } + _ => Ok(None), + } + } + + /// Try to return an attribute value as a location list entry iterator. + /// + /// If the attribute value is one of: + /// + /// - a `DW_FORM_sec_offset` reference to the `.debug_loc` or `.debug_loclists` sections + /// - a `DW_FORM_loclistx` index into the `.debug_loclists` entries for the unit + /// + /// then return an iterator over the entries in the location list. + /// Returns `None` for other forms. + pub fn attr_locations( + &self, + unit: &Unit, + attr: AttributeValue, + ) -> Result>> { + match self.attr_locations_offset(unit, attr)? { + Some(offset) => Ok(Some(self.locations(unit, offset)?)), + None => Ok(None), + } + } + + /// Call `Reader::lookup_offset_id` for each section, and return the first match. + /// + /// The first element of the tuple is `true` for supplementary sections. + pub fn lookup_offset_id(&self, id: ReaderOffsetId) -> Option<(bool, SectionId, R::Offset)> { + None.or_else(|| self.debug_abbrev.lookup_offset_id(id)) + .or_else(|| self.debug_addr.lookup_offset_id(id)) + .or_else(|| self.debug_aranges.lookup_offset_id(id)) + .or_else(|| self.debug_info.lookup_offset_id(id)) + .or_else(|| self.debug_line.lookup_offset_id(id)) + .or_else(|| self.debug_line_str.lookup_offset_id(id)) + .or_else(|| self.debug_str.lookup_offset_id(id)) + .or_else(|| self.debug_str_offsets.lookup_offset_id(id)) + .or_else(|| self.debug_types.lookup_offset_id(id)) + .or_else(|| self.locations.lookup_offset_id(id)) + .or_else(|| self.ranges.lookup_offset_id(id)) + .map(|(id, offset)| (false, id, offset)) + .or_else(|| { + self.sup() + .and_then(|sup| sup.lookup_offset_id(id)) + .map(|(_, id, offset)| (true, id, offset)) + }) + } + + /// Returns a string representation of the given error. + /// + /// This uses information from the DWARF sections to provide more information in some cases. + pub fn format_error(&self, err: Error) -> String { + #[allow(clippy::single_match)] + match err { + Error::UnexpectedEof(id) => match self.lookup_offset_id(id) { + Some((sup, section, offset)) => { + return format!( + "{} at {}{}+0x{:x}", + err, + section.name(), + if sup { "(sup)" } else { "" }, + offset.into_u64(), + ); + } + None => {} + }, + _ => {} + } + err.description().into() + } +} + +/// The sections from a `.dwp` file. +#[derive(Debug)] +pub struct DwarfPackage { + /// The compilation unit index in the `.debug_cu_index` section. + pub cu_index: UnitIndex, + + /// The type unit index in the `.debug_tu_index` section. + pub tu_index: UnitIndex, + + /// The `.debug_abbrev.dwo` section. + pub debug_abbrev: DebugAbbrev, + + /// The `.debug_info.dwo` section. + pub debug_info: DebugInfo, + + /// The `.debug_line.dwo` section. + pub debug_line: DebugLine, + + /// The `.debug_str.dwo` section. + pub debug_str: DebugStr, + + /// The `.debug_str_offsets.dwo` section. + pub debug_str_offsets: DebugStrOffsets, + + /// The `.debug_loc.dwo` section. + /// + /// Only present when using GNU split-dwarf extension to DWARF 4. + pub debug_loc: DebugLoc, + + /// The `.debug_loclists.dwo` section. + pub debug_loclists: DebugLocLists, + + /// The `.debug_rnglists.dwo` section. + pub debug_rnglists: DebugRngLists, + + /// The `.debug_types.dwo` section. + /// + /// Only present when using GNU split-dwarf extension to DWARF 4. + pub debug_types: DebugTypes, + + /// An empty section. + /// + /// Used when creating `Dwarf`. + pub empty: R, +} + +impl DwarfPackage { + /// Try to load the `.dwp` sections using the given loader function. + /// + /// `section` loads a DWARF section from the object file. + /// It should return an empty section if the section does not exist. + pub fn load(mut section: F, empty: R) -> core::result::Result + where + F: FnMut(SectionId) -> core::result::Result, + E: From, + { + Ok(DwarfPackage { + cu_index: DebugCuIndex::load(&mut section)?.index()?, + tu_index: DebugTuIndex::load(&mut section)?.index()?, + // Section types are inferred. + debug_abbrev: Section::load(&mut section)?, + debug_info: Section::load(&mut section)?, + debug_line: Section::load(&mut section)?, + debug_str: Section::load(&mut section)?, + debug_str_offsets: Section::load(&mut section)?, + debug_loc: Section::load(&mut section)?, + debug_loclists: Section::load(&mut section)?, + debug_rnglists: Section::load(&mut section)?, + debug_types: Section::load(&mut section)?, + empty, + }) + } + + /// Find the compilation unit with the given DWO identifier and return its section + /// contributions. + pub fn find_cu(&self, id: DwoId, parent: &Dwarf) -> Result>> { + let row = match self.cu_index.find(id.0) { + Some(row) => row, + None => return Ok(None), + }; + self.cu_sections(row, parent).map(Some) + } + + /// Find the type unit with the given type signature and return its section + /// contributions. + pub fn find_tu( + &self, + signature: DebugTypeSignature, + parent: &Dwarf, + ) -> Result>> { + let row = match self.tu_index.find(signature.0) { + Some(row) => row, + None => return Ok(None), + }; + self.tu_sections(row, parent).map(Some) + } + + /// Return the section contributions of the compilation unit at the given index. + /// + /// The index must be in the range `1..cu_index.unit_count`. + /// + /// This function should only be needed by low level parsers. + pub fn cu_sections(&self, index: u32, parent: &Dwarf) -> Result> { + self.sections(self.cu_index.sections(index)?, parent) + } + + /// Return the section contributions of the compilation unit at the given index. + /// + /// The index must be in the range `1..tu_index.unit_count`. + /// + /// This function should only be needed by low level parsers. + pub fn tu_sections(&self, index: u32, parent: &Dwarf) -> Result> { + self.sections(self.tu_index.sections(index)?, parent) + } + + /// Return the section contributions of a unit. + /// + /// This function should only be needed by low level parsers. + pub fn sections( + &self, + sections: UnitIndexSectionIterator, + parent: &Dwarf, + ) -> Result> { + let mut abbrev_offset = 0; + let mut abbrev_size = 0; + let mut info_offset = 0; + let mut info_size = 0; + let mut line_offset = 0; + let mut line_size = 0; + let mut loc_offset = 0; + let mut loc_size = 0; + let mut loclists_offset = 0; + let mut loclists_size = 0; + let mut str_offsets_offset = 0; + let mut str_offsets_size = 0; + let mut rnglists_offset = 0; + let mut rnglists_size = 0; + let mut types_offset = 0; + let mut types_size = 0; + for section in sections { + match section.section { + SectionId::DebugAbbrev => { + abbrev_offset = section.offset; + abbrev_size = section.size; + } + SectionId::DebugInfo => { + info_offset = section.offset; + info_size = section.size; + } + SectionId::DebugLine => { + line_offset = section.offset; + line_size = section.size; + } + SectionId::DebugLoc => { + loc_offset = section.offset; + loc_size = section.size; + } + SectionId::DebugLocLists => { + loclists_offset = section.offset; + loclists_size = section.size; + } + SectionId::DebugStrOffsets => { + str_offsets_offset = section.offset; + str_offsets_size = section.size; + } + SectionId::DebugRngLists => { + rnglists_offset = section.offset; + rnglists_size = section.size; + } + SectionId::DebugTypes => { + types_offset = section.offset; + types_size = section.size; + } + SectionId::DebugMacro | SectionId::DebugMacinfo => { + // These are valid but we can't parse these yet. + } + _ => return Err(Error::UnknownIndexSection), + } + } + + let debug_abbrev = self.debug_abbrev.dwp_range(abbrev_offset, abbrev_size)?; + let debug_info = self.debug_info.dwp_range(info_offset, info_size)?; + let debug_line = self.debug_line.dwp_range(line_offset, line_size)?; + let debug_loc = self.debug_loc.dwp_range(loc_offset, loc_size)?; + let debug_loclists = self + .debug_loclists + .dwp_range(loclists_offset, loclists_size)?; + let debug_str_offsets = self + .debug_str_offsets + .dwp_range(str_offsets_offset, str_offsets_size)?; + let debug_rnglists = self + .debug_rnglists + .dwp_range(rnglists_offset, rnglists_size)?; + let debug_types = self.debug_types.dwp_range(types_offset, types_size)?; + + let debug_str = self.debug_str.clone(); + + let debug_addr = parent.debug_addr.clone(); + let debug_ranges = parent.ranges.debug_ranges().clone(); + + let debug_aranges = self.empty.clone().into(); + let debug_line_str = self.empty.clone().into(); + + Ok(Dwarf { + debug_abbrev, + debug_addr, + debug_aranges, + debug_info, + debug_line, + debug_line_str, + debug_str, + debug_str_offsets, + debug_types, + locations: LocationLists::new(debug_loc, debug_loclists), + ranges: RangeLists::new(debug_ranges, debug_rnglists), + file_type: DwarfFileType::Dwo, + sup: None, + }) + } +} + +/// All of the commonly used information for a unit in the `.debug_info` or `.debug_types` +/// sections. +#[derive(Debug)] +pub struct Unit::Offset> +where + R: Reader, + Offset: ReaderOffset, +{ + /// The header of the unit. + pub header: UnitHeader, + + /// The parsed abbreviations for the unit. + pub abbreviations: Abbreviations, + + /// The `DW_AT_name` attribute of the unit. + pub name: Option, + + /// The `DW_AT_comp_dir` attribute of the unit. + pub comp_dir: Option, + + /// The `DW_AT_low_pc` attribute of the unit. Defaults to 0. + pub low_pc: u64, + + /// The `DW_AT_str_offsets_base` attribute of the unit. Defaults to 0. + pub str_offsets_base: DebugStrOffsetsBase, + + /// The `DW_AT_addr_base` attribute of the unit. Defaults to 0. + pub addr_base: DebugAddrBase, + + /// The `DW_AT_loclists_base` attribute of the unit. Defaults to 0. + pub loclists_base: DebugLocListsBase, + + /// The `DW_AT_rnglists_base` attribute of the unit. Defaults to 0. + pub rnglists_base: DebugRngListsBase, + + /// The line number program of the unit. + pub line_program: Option>, + + /// The DWO ID of a skeleton unit or split compilation unit. + pub dwo_id: Option, +} + +impl Unit { + /// Construct a new `Unit` from the given unit header. + #[inline] + pub fn new(dwarf: &Dwarf, header: UnitHeader) -> Result { + let abbreviations = header.abbreviations(&dwarf.debug_abbrev)?; + let mut unit = Unit { + abbreviations, + name: None, + comp_dir: None, + low_pc: 0, + str_offsets_base: DebugStrOffsetsBase::default_for_encoding_and_file( + header.encoding(), + dwarf.file_type, + ), + // NB: Because the .debug_addr section never lives in a .dwo, we can assume its base is always 0 or provided. + addr_base: DebugAddrBase(R::Offset::from_u8(0)), + loclists_base: DebugLocListsBase::default_for_encoding_and_file( + header.encoding(), + dwarf.file_type, + ), + rnglists_base: DebugRngListsBase::default_for_encoding_and_file( + header.encoding(), + dwarf.file_type, + ), + line_program: None, + dwo_id: match header.type_() { + UnitType::Skeleton(dwo_id) | UnitType::SplitCompilation(dwo_id) => Some(dwo_id), + _ => None, + }, + header, + }; + let mut name = None; + let mut comp_dir = None; + let mut line_program_offset = None; + let mut low_pc_attr = None; + + { + let mut cursor = unit.header.entries(&unit.abbreviations); + cursor.next_dfs()?; + let root = cursor.current().ok_or(Error::MissingUnitDie)?; + let mut attrs = root.attrs(); + while let Some(attr) = attrs.next()? { + match attr.name() { + constants::DW_AT_name => { + name = Some(attr.value()); + } + constants::DW_AT_comp_dir => { + comp_dir = Some(attr.value()); + } + constants::DW_AT_low_pc => { + low_pc_attr = Some(attr.value()); + } + constants::DW_AT_stmt_list => { + if let AttributeValue::DebugLineRef(offset) = attr.value() { + line_program_offset = Some(offset); + } + } + constants::DW_AT_str_offsets_base => { + if let AttributeValue::DebugStrOffsetsBase(base) = attr.value() { + unit.str_offsets_base = base; + } + } + constants::DW_AT_addr_base | constants::DW_AT_GNU_addr_base => { + if let AttributeValue::DebugAddrBase(base) = attr.value() { + unit.addr_base = base; + } + } + constants::DW_AT_loclists_base => { + if let AttributeValue::DebugLocListsBase(base) = attr.value() { + unit.loclists_base = base; + } + } + constants::DW_AT_rnglists_base | constants::DW_AT_GNU_ranges_base => { + if let AttributeValue::DebugRngListsBase(base) = attr.value() { + unit.rnglists_base = base; + } + } + constants::DW_AT_GNU_dwo_id => { + if unit.dwo_id.is_none() { + if let AttributeValue::DwoId(dwo_id) = attr.value() { + unit.dwo_id = Some(dwo_id); + } + } + } + _ => {} + } + } + } + + unit.name = match name { + Some(val) => dwarf.attr_string(&unit, val).ok(), + None => None, + }; + unit.comp_dir = match comp_dir { + Some(val) => dwarf.attr_string(&unit, val).ok(), + None => None, + }; + unit.line_program = match line_program_offset { + Some(offset) => Some(dwarf.debug_line.program( + offset, + unit.header.address_size(), + unit.comp_dir.clone(), + unit.name.clone(), + )?), + None => None, + }; + if let Some(low_pc_attr) = low_pc_attr { + if let Some(addr) = dwarf.attr_address(&unit, low_pc_attr)? { + unit.low_pc = addr; + } + } + Ok(unit) + } + + /// Return the encoding parameters for this unit. + #[inline] + pub fn encoding(&self) -> Encoding { + self.header.encoding() + } + + /// Read the `DebuggingInformationEntry` at the given offset. + pub fn entry(&self, offset: UnitOffset) -> Result> { + self.header.entry(&self.abbreviations, offset) + } + + /// Navigate this unit's `DebuggingInformationEntry`s. + #[inline] + pub fn entries(&self) -> EntriesCursor { + self.header.entries(&self.abbreviations) + } + + /// Navigate this unit's `DebuggingInformationEntry`s + /// starting at the given offset. + #[inline] + pub fn entries_at_offset(&self, offset: UnitOffset) -> Result> { + self.header.entries_at_offset(&self.abbreviations, offset) + } + + /// Navigate this unit's `DebuggingInformationEntry`s as a tree + /// starting at the given offset. + #[inline] + pub fn entries_tree(&self, offset: Option>) -> Result> { + self.header.entries_tree(&self.abbreviations, offset) + } + + /// Read the raw data that defines the Debugging Information Entries. + #[inline] + pub fn entries_raw(&self, offset: Option>) -> Result> { + self.header.entries_raw(&self.abbreviations, offset) + } + + /// Copy attributes that are subject to relocation from another unit. This is intended + /// to be used to copy attributes from a skeleton compilation unit to the corresponding + /// split compilation unit. + pub fn copy_relocated_attributes(&mut self, other: &Unit) { + self.low_pc = other.low_pc; + self.addr_base = other.addr_base; + if self.header.version() < 5 { + self.rnglists_base = other.rnglists_base; + } + } +} + +impl UnitSectionOffset { + /// Convert an offset to be relative to the start of the given unit, + /// instead of relative to the start of the section. + /// Returns `None` if the offset is not within the unit entries. + pub fn to_unit_offset(&self, unit: &Unit) -> Option> + where + R: Reader, + { + let (offset, unit_offset) = match (self, unit.header.offset()) { + ( + UnitSectionOffset::DebugInfoOffset(offset), + UnitSectionOffset::DebugInfoOffset(unit_offset), + ) => (offset.0, unit_offset.0), + ( + UnitSectionOffset::DebugTypesOffset(offset), + UnitSectionOffset::DebugTypesOffset(unit_offset), + ) => (offset.0, unit_offset.0), + _ => return None, + }; + let offset = match offset.checked_sub(unit_offset) { + Some(offset) => UnitOffset(offset), + None => return None, + }; + if !unit.header.is_valid_offset(offset) { + return None; + } + Some(offset) + } +} + +impl UnitOffset { + /// Convert an offset to be relative to the start of the .debug_info section, + /// instead of relative to the start of the given compilation unit. + /// + /// Does not check that the offset is valid. + pub fn to_unit_section_offset(&self, unit: &Unit) -> UnitSectionOffset + where + R: Reader, + { + match unit.header.offset() { + UnitSectionOffset::DebugInfoOffset(unit_offset) => { + DebugInfoOffset(unit_offset.0 + self.0).into() + } + UnitSectionOffset::DebugTypesOffset(unit_offset) => { + DebugTypesOffset(unit_offset.0 + self.0).into() + } + } + } +} + +/// An iterator for the address ranges of a `DebuggingInformationEntry`. +/// +/// Returned by `Dwarf::die_ranges` and `Dwarf::unit_ranges`. +#[derive(Debug)] +pub struct RangeIter(RangeIterInner); + +#[derive(Debug)] +enum RangeIterInner { + Single(Option), + List(RngListIter), +} + +impl Default for RangeIter { + fn default() -> Self { + RangeIter(RangeIterInner::Single(None)) + } +} + +impl RangeIter { + /// Advance the iterator to the next range. + pub fn next(&mut self) -> Result> { + match self.0 { + RangeIterInner::Single(ref mut range) => Ok(range.take()), + RangeIterInner::List(ref mut list) => list.next(), + } + } +} + +#[cfg(feature = "fallible-iterator")] +impl fallible_iterator::FallibleIterator for RangeIter { + type Item = Range; + type Error = Error; + + #[inline] + fn next(&mut self) -> ::core::result::Result, Self::Error> { + RangeIter::next(self) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::read::EndianSlice; + use crate::{Endianity, LittleEndian}; + + /// Ensure that `Dwarf` is covariant wrt R. + #[test] + fn test_dwarf_variance() { + /// This only needs to compile. + fn _f<'a: 'b, 'b, E: Endianity>(x: Dwarf>) -> Dwarf> { + x + } + } + + /// Ensure that `Unit` is covariant wrt R. + #[test] + fn test_dwarf_unit_variance() { + /// This only needs to compile. + fn _f<'a: 'b, 'b, E: Endianity>(x: Unit>) -> Unit> { + x + } + } + + #[test] + fn test_send() { + fn assert_is_send() {} + assert_is_send::>>(); + assert_is_send::>>(); + } + + #[test] + fn test_format_error() { + let mut owned_dwarf = Dwarf::load(|_| -> Result<_> { Ok(vec![1, 2]) }).unwrap(); + owned_dwarf + .load_sup(|_| -> Result<_> { Ok(vec![1, 2]) }) + .unwrap(); + let dwarf = owned_dwarf.borrow(|section| EndianSlice::new(§ion, LittleEndian)); + + match dwarf.debug_str.get_str(DebugStrOffset(1)) { + Ok(r) => panic!("Unexpected str {:?}", r), + Err(e) => { + assert_eq!( + dwarf.format_error(e), + "Hit the end of input before it was expected at .debug_str+0x1" + ); + } + } + match dwarf.sup().unwrap().debug_str.get_str(DebugStrOffset(1)) { + Ok(r) => panic!("Unexpected str {:?}", r), + Err(e) => { + assert_eq!( + dwarf.format_error(e), + "Hit the end of input before it was expected at .debug_str(sup)+0x1" + ); + } + } + assert_eq!(dwarf.format_error(Error::Io), Error::Io.description()); + } +} diff --git a/crux-mir/lib/gimli/src/read/endian_reader.rs b/crux-mir/lib/gimli/src/read/endian_reader.rs new file mode 100644 index 000000000..8852b3804 --- /dev/null +++ b/crux-mir/lib/gimli/src/read/endian_reader.rs @@ -0,0 +1,639 @@ +//! Defining custom `Reader`s quickly. + +use alloc::borrow::Cow; +use alloc::rc::Rc; +use alloc::string::String; +use alloc::sync::Arc; +use core::fmt::Debug; +use core::ops::{Deref, Index, Range, RangeFrom, RangeTo}; +use core::slice; +use core::str; +use stable_deref_trait::CloneStableDeref; + +use crate::endianity::Endianity; +use crate::read::{Error, Reader, ReaderOffsetId, Result}; + +/// A reference counted, non-thread-safe slice of bytes and associated +/// endianity. +/// +/// ``` +/// # #[cfg(feature = "std")] { +/// use std::rc::Rc; +/// +/// let buf = Rc::from(&[1, 2, 3, 4][..]); +/// let reader = gimli::EndianRcSlice::new(buf, gimli::NativeEndian); +/// # let _ = reader; +/// # } +/// ``` +pub type EndianRcSlice = EndianReader>; + +/// An atomically reference counted, thread-safe slice of bytes and associated +/// endianity. +/// +/// ``` +/// # #[cfg(feature = "std")] { +/// use std::sync::Arc; +/// +/// let buf = Arc::from(&[1, 2, 3, 4][..]); +/// let reader = gimli::EndianArcSlice::new(buf, gimli::NativeEndian); +/// # let _ = reader; +/// # } +/// ``` +pub type EndianArcSlice = EndianReader>; + +/// An easy way to define a custom `Reader` implementation with a reference to a +/// generic buffer of bytes and an associated endianity. +/// +/// Note that the whole original buffer is kept alive in memory even if there is +/// only one reader that references only a handful of bytes from that original +/// buffer. That is, `EndianReader` will not do any copying, moving, or +/// compacting in order to free up unused regions of the original buffer. If you +/// require this kind of behavior, it is up to you to implement `Reader` +/// directly by-hand. +/// +/// # Example +/// +/// Say you have an `mmap`ed file that you want to serve as a `gimli::Reader`. +/// You can wrap that `mmap`ed file up in a `MmapFile` type and use +/// `EndianReader>` or `EndianReader>` as readers as +/// long as `MmapFile` dereferences to the underlying `[u8]` data. +/// +/// ``` +/// use std::io; +/// use std::ops::Deref; +/// use std::path::Path; +/// use std::slice; +/// use std::sync::Arc; +/// +/// /// A type that represents an `mmap`ed file. +/// #[derive(Debug)] +/// pub struct MmapFile { +/// ptr: *const u8, +/// len: usize, +/// } +/// +/// impl MmapFile { +/// pub fn new(path: &Path) -> io::Result { +/// // Call `mmap` and check for errors and all that... +/// # unimplemented!() +/// } +/// } +/// +/// impl Drop for MmapFile { +/// fn drop(&mut self) { +/// // Call `munmap` to clean up after ourselves... +/// # unimplemented!() +/// } +/// } +/// +/// // And `MmapFile` can deref to a slice of the `mmap`ed region of memory. +/// impl Deref for MmapFile { +/// type Target = [u8]; +/// fn deref(&self) -> &[u8] { +/// unsafe { +/// slice::from_raw_parts(self.ptr, self.len) +/// } +/// } +/// } +/// +/// /// A type that represents a shared `mmap`ed file. +/// #[derive(Debug, Clone)] +/// pub struct ArcMmapFile(Arc); +/// +/// // And `ArcMmapFile` can deref to a slice of the `mmap`ed region of memory. +/// impl Deref for ArcMmapFile { +/// type Target = [u8]; +/// fn deref(&self) -> &[u8] { +/// &self.0 +/// } +/// } +/// +/// // These are both valid for any `Rc` or `Arc`. +/// unsafe impl gimli::StableDeref for ArcMmapFile {} +/// unsafe impl gimli::CloneStableDeref for ArcMmapFile {} +/// +/// /// A `gimli::Reader` that is backed by an `mmap`ed file! +/// pub type MmapFileReader = gimli::EndianReader; +/// # fn test(_: &MmapFileReader) { } +/// ``` +#[derive(Debug, Clone, Copy, Hash)] +pub struct EndianReader +where + Endian: Endianity, + T: CloneStableDeref + Debug, +{ + range: SubRange, + endian: Endian, +} + +impl PartialEq> for EndianReader +where + Endian: Endianity, + T1: CloneStableDeref + Debug, + T2: CloneStableDeref + Debug, +{ + fn eq(&self, rhs: &EndianReader) -> bool { + self.bytes() == rhs.bytes() + } +} + +impl Eq for EndianReader +where + Endian: Endianity, + T: CloneStableDeref + Debug, +{ +} + +// This is separated out from `EndianReader` so that we can avoid running afoul +// of borrowck. We need to `read_slice(&mut self, ...) -> &[u8]` and then call +// `self.endian.read_whatever` on the result. The problem is that the returned +// slice keeps the `&mut self` borrow active, so we wouldn't be able to access +// `self.endian`. Splitting the sub-range out from the endian lets us work +// around this, making it so that only the `self.range` borrow is held active, +// not all of `self`. +// +// This also serves to encapsulate the unsafe code concerning `CloneStableDeref`. +// The `bytes` member is held so that the bytes live long enough, and the +// `CloneStableDeref` ensures these bytes never move. The `ptr` and `len` +// members point inside `bytes`, and are updated during read operations. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct SubRange +where + T: CloneStableDeref + Debug, +{ + bytes: T, + ptr: *const u8, + len: usize, +} + +unsafe impl Send for SubRange where T: CloneStableDeref + Debug + Send {} + +unsafe impl Sync for SubRange where T: CloneStableDeref + Debug + Sync {} + +impl SubRange +where + T: CloneStableDeref + Debug, +{ + #[inline] + fn new(bytes: T) -> Self { + let ptr = bytes.as_ptr(); + let len = bytes.len(); + SubRange { bytes, ptr, len } + } + + #[inline] + fn bytes(&self) -> &[u8] { + // Safe because `T` implements `CloneStableDeref`, `bytes` can't be modified, + // and all operations that modify `ptr` and `len` ensure they stay in range. + unsafe { slice::from_raw_parts(self.ptr, self.len) } + } + + #[inline] + fn len(&self) -> usize { + self.len + } + + #[inline] + fn truncate(&mut self, len: usize) { + assert!(len <= self.len); + self.len = len; + } + + #[inline] + fn skip(&mut self, len: usize) { + assert!(len <= self.len); + self.ptr = unsafe { self.ptr.add(len) }; + self.len -= len; + } + + #[inline] + fn read_slice(&mut self, len: usize) -> Option<&[u8]> { + if self.len() < len { + None + } else { + // Same as for `bytes()`. + let bytes = unsafe { slice::from_raw_parts(self.ptr, len) }; + self.skip(len); + Some(bytes) + } + } +} + +impl EndianReader +where + Endian: Endianity, + T: CloneStableDeref + Debug, +{ + /// Construct a new `EndianReader` with the given bytes. + #[inline] + pub fn new(bytes: T, endian: Endian) -> EndianReader { + EndianReader { + range: SubRange::new(bytes), + endian, + } + } + + /// Return a reference to the raw bytes underlying this reader. + #[inline] + pub fn bytes(&self) -> &[u8] { + self.range.bytes() + } +} + +/// # Range Methods +/// +/// Unfortunately, `std::ops::Index` *must* return a reference, so we can't +/// implement `Index>` to return a new `EndianReader` the way we +/// would like to. Instead, we abandon fancy indexing operators and have these +/// plain old methods. +impl EndianReader +where + Endian: Endianity, + T: CloneStableDeref + Debug, +{ + /// Take the given `start..end` range of the underlying buffer and return a + /// new `EndianReader`. + /// + /// ``` + /// # #[cfg(feature = "std")] { + /// use gimli::{EndianReader, LittleEndian}; + /// use std::sync::Arc; + /// + /// let buf = Arc::<[u8]>::from(&[0x01, 0x02, 0x03, 0x04][..]); + /// let reader = EndianReader::new(buf.clone(), LittleEndian); + /// assert_eq!(reader.range(1..3), + /// EndianReader::new(&buf[1..3], LittleEndian)); + /// # } + /// ``` + /// + /// # Panics + /// + /// Panics if the range is out of bounds. + pub fn range(&self, idx: Range) -> EndianReader { + let mut r = self.clone(); + r.range.skip(idx.start); + r.range.truncate(idx.len()); + r + } + + /// Take the given `start..` range of the underlying buffer and return a new + /// `EndianReader`. + /// + /// ``` + /// # #[cfg(feature = "std")] { + /// use gimli::{EndianReader, LittleEndian}; + /// use std::sync::Arc; + /// + /// let buf = Arc::<[u8]>::from(&[0x01, 0x02, 0x03, 0x04][..]); + /// let reader = EndianReader::new(buf.clone(), LittleEndian); + /// assert_eq!(reader.range_from(2..), + /// EndianReader::new(&buf[2..], LittleEndian)); + /// # } + /// ``` + /// + /// # Panics + /// + /// Panics if the range is out of bounds. + pub fn range_from(&self, idx: RangeFrom) -> EndianReader { + let mut r = self.clone(); + r.range.skip(idx.start); + r + } + + /// Take the given `..end` range of the underlying buffer and return a new + /// `EndianReader`. + /// + /// ``` + /// # #[cfg(feature = "std")] { + /// use gimli::{EndianReader, LittleEndian}; + /// use std::sync::Arc; + /// + /// let buf = Arc::<[u8]>::from(&[0x01, 0x02, 0x03, 0x04][..]); + /// let reader = EndianReader::new(buf.clone(), LittleEndian); + /// assert_eq!(reader.range_to(..3), + /// EndianReader::new(&buf[..3], LittleEndian)); + /// # } + /// ``` + /// + /// # Panics + /// + /// Panics if the range is out of bounds. + pub fn range_to(&self, idx: RangeTo) -> EndianReader { + let mut r = self.clone(); + r.range.truncate(idx.end); + r + } +} + +impl Index for EndianReader +where + Endian: Endianity, + T: CloneStableDeref + Debug, +{ + type Output = u8; + fn index(&self, idx: usize) -> &Self::Output { + &self.bytes()[idx] + } +} + +impl Index> for EndianReader +where + Endian: Endianity, + T: CloneStableDeref + Debug, +{ + type Output = [u8]; + fn index(&self, idx: RangeFrom) -> &Self::Output { + &self.bytes()[idx] + } +} + +impl Deref for EndianReader +where + Endian: Endianity, + T: CloneStableDeref + Debug, +{ + type Target = [u8]; + fn deref(&self) -> &Self::Target { + self.bytes() + } +} + +impl Reader for EndianReader +where + Endian: Endianity, + T: CloneStableDeref + Debug, +{ + type Endian = Endian; + type Offset = usize; + + #[inline] + fn endian(&self) -> Endian { + self.endian + } + + #[inline] + fn len(&self) -> usize { + self.range.len() + } + + #[inline] + fn empty(&mut self) { + self.range.truncate(0); + } + + #[inline] + fn truncate(&mut self, len: usize) -> Result<()> { + if self.len() < len { + Err(Error::UnexpectedEof(self.offset_id())) + } else { + self.range.truncate(len); + Ok(()) + } + } + + #[inline] + fn offset_from(&self, base: &EndianReader) -> usize { + let base_ptr = base.bytes().as_ptr() as *const u8 as usize; + let ptr = self.bytes().as_ptr() as *const u8 as usize; + debug_assert!(base_ptr <= ptr); + debug_assert!(ptr + self.bytes().len() <= base_ptr + base.bytes().len()); + ptr - base_ptr + } + + #[inline] + fn offset_id(&self) -> ReaderOffsetId { + ReaderOffsetId(self.bytes().as_ptr() as u64) + } + + #[inline] + fn lookup_offset_id(&self, id: ReaderOffsetId) -> Option { + let id = id.0; + let self_id = self.bytes().as_ptr() as u64; + let self_len = self.bytes().len() as u64; + if id >= self_id && id <= self_id + self_len { + Some((id - self_id) as usize) + } else { + None + } + } + + #[inline] + fn find(&self, byte: u8) -> Result { + self.bytes() + .iter() + .position(|x| *x == byte) + .ok_or_else(|| Error::UnexpectedEof(self.offset_id())) + } + + #[inline] + fn skip(&mut self, len: usize) -> Result<()> { + if self.len() < len { + Err(Error::UnexpectedEof(self.offset_id())) + } else { + self.range.skip(len); + Ok(()) + } + } + + #[inline] + fn split(&mut self, len: usize) -> Result { + if self.len() < len { + Err(Error::UnexpectedEof(self.offset_id())) + } else { + let mut r = self.clone(); + r.range.truncate(len); + self.range.skip(len); + Ok(r) + } + } + + #[inline] + fn to_slice(&self) -> Result> { + Ok(self.bytes().into()) + } + + #[inline] + fn to_string(&self) -> Result> { + match str::from_utf8(self.bytes()) { + Ok(s) => Ok(s.into()), + _ => Err(Error::BadUtf8), + } + } + + #[inline] + fn to_string_lossy(&self) -> Result> { + Ok(String::from_utf8_lossy(self.bytes())) + } + + #[inline] + fn read_slice(&mut self, buf: &mut [u8]) -> Result<()> { + match self.range.read_slice(buf.len()) { + Some(slice) => { + buf.copy_from_slice(slice); + Ok(()) + } + None => Err(Error::UnexpectedEof(self.offset_id())), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::endianity::NativeEndian; + use crate::read::Reader; + + fn native_reader + Debug>( + bytes: T, + ) -> EndianReader { + EndianReader::new(bytes, NativeEndian) + } + + const BUF: &[u8] = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 0]; + + #[test] + fn test_reader_split() { + let mut reader = native_reader(BUF); + let left = reader.split(3).unwrap(); + assert_eq!(left, native_reader(&BUF[..3])); + assert_eq!(reader, native_reader(&BUF[3..])); + } + + #[test] + fn test_reader_split_out_of_bounds() { + let mut reader = native_reader(BUF); + assert!(reader.split(30).is_err()); + } + + #[test] + fn bytes_and_len_and_range_and_eq() { + let reader = native_reader(BUF); + assert_eq!(reader.len(), BUF.len()); + assert_eq!(reader.bytes(), BUF); + assert_eq!(reader, native_reader(BUF)); + + let range = reader.range(2..8); + let buf_range = &BUF[2..8]; + assert_eq!(range.len(), buf_range.len()); + assert_eq!(range.bytes(), buf_range); + assert_ne!(range, native_reader(BUF)); + assert_eq!(range, native_reader(buf_range)); + + let range_from = range.range_from(1..); + let buf_range_from = &buf_range[1..]; + assert_eq!(range_from.len(), buf_range_from.len()); + assert_eq!(range_from.bytes(), buf_range_from); + assert_ne!(range_from, native_reader(BUF)); + assert_eq!(range_from, native_reader(buf_range_from)); + + let range_to = range_from.range_to(..4); + let buf_range_to = &buf_range_from[..4]; + assert_eq!(range_to.len(), buf_range_to.len()); + assert_eq!(range_to.bytes(), buf_range_to); + assert_ne!(range_to, native_reader(BUF)); + assert_eq!(range_to, native_reader(buf_range_to)); + } + + #[test] + fn find() { + let mut reader = native_reader(BUF); + reader.skip(2).unwrap(); + assert_eq!( + reader.find(5), + Ok(BUF[2..].iter().position(|x| *x == 5).unwrap()) + ); + } + + #[test] + fn indexing() { + let mut reader = native_reader(BUF); + reader.skip(2).unwrap(); + assert_eq!(reader[0], BUF[2]); + } + + #[test] + #[should_panic] + fn indexing_out_of_bounds() { + let mut reader = native_reader(BUF); + reader.skip(2).unwrap(); + let _ = reader[900]; + } + + #[test] + fn endian() { + let reader = native_reader(BUF); + assert_eq!(reader.endian(), NativeEndian); + } + + #[test] + fn empty() { + let mut reader = native_reader(BUF); + assert!(!reader.is_empty()); + reader.empty(); + assert!(reader.is_empty()); + assert!(reader.bytes().is_empty()); + } + + #[test] + fn truncate() { + let reader = native_reader(BUF); + let mut reader = reader.range(2..8); + reader.truncate(2).unwrap(); + assert_eq!(reader.bytes(), &BUF[2..4]); + } + + #[test] + fn offset_from() { + let reader = native_reader(BUF); + let sub = reader.range(2..8); + assert_eq!(sub.offset_from(&reader), 2); + } + + #[test] + fn skip() { + let mut reader = native_reader(BUF); + reader.skip(2).unwrap(); + assert_eq!(reader.bytes(), &BUF[2..]); + } + + #[test] + fn to_slice() { + assert_eq!( + native_reader(BUF).range(2..5).to_slice(), + Ok(Cow::from(&BUF[2..5])) + ); + } + + #[test] + fn to_string_ok() { + let buf = b"hello, world!"; + let reader = native_reader(&buf[..]); + let reader = reader.range_from(7..); + assert_eq!(reader.to_string(), Ok(Cow::from("world!"))); + } + + // The rocket emoji (🚀 = [0xf0, 0x9f, 0x9a, 0x80]) but rotated left by one + // to make it invalid UTF-8. + const BAD_UTF8: &[u8] = &[0x9f, 0x9a, 0x80, 0xf0]; + + #[test] + fn to_string_err() { + let reader = native_reader(BAD_UTF8); + assert!(reader.to_string().is_err()); + } + + #[test] + fn to_string_lossy() { + let reader = native_reader(BAD_UTF8); + assert_eq!(reader.to_string_lossy(), Ok(Cow::from("����"))); + } + + #[test] + fn read_u8_array() { + let mut reader = native_reader(BAD_UTF8); + reader.skip(1).unwrap(); + let arr: [u8; 2] = reader.read_u8_array().unwrap(); + assert_eq!(arr, &BAD_UTF8[1..3]); + assert_eq!(reader.bytes(), &BAD_UTF8[3..]); + } +} diff --git a/crux-mir/lib/gimli/src/read/endian_slice.rs b/crux-mir/lib/gimli/src/read/endian_slice.rs new file mode 100644 index 000000000..05262cdec --- /dev/null +++ b/crux-mir/lib/gimli/src/read/endian_slice.rs @@ -0,0 +1,350 @@ +//! Working with byte slices that have an associated endianity. + +#[cfg(feature = "read")] +use alloc::borrow::Cow; +#[cfg(feature = "read")] +use alloc::string::String; +use core::ops::{Deref, Index, Range, RangeFrom, RangeTo}; +use core::str; + +use crate::endianity::Endianity; +use crate::read::{Error, Reader, ReaderOffsetId, Result}; + +/// A `&[u8]` slice with endianity metadata. +/// +/// This implements the `Reader` trait, which is used for all reading of DWARF sections. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] +pub struct EndianSlice<'input, Endian> +where + Endian: Endianity, +{ + slice: &'input [u8], + endian: Endian, +} + +impl<'input, Endian> EndianSlice<'input, Endian> +where + Endian: Endianity, +{ + /// Construct a new `EndianSlice` with the given slice and endianity. + #[inline] + pub fn new(slice: &'input [u8], endian: Endian) -> EndianSlice<'input, Endian> { + EndianSlice { slice, endian } + } + + /// Return a reference to the raw slice. + #[inline] + #[doc(hidden)] + #[deprecated(note = "Method renamed to EndianSlice::slice; use that instead.")] + pub fn buf(&self) -> &'input [u8] { + self.slice + } + + /// Return a reference to the raw slice. + #[inline] + pub fn slice(&self) -> &'input [u8] { + self.slice + } + + /// Split the slice in two at the given index, resulting in the tuple where + /// the first item has range [0, idx), and the second has range [idx, + /// len). Panics if the index is out of bounds. + #[inline] + pub fn split_at( + &self, + idx: usize, + ) -> (EndianSlice<'input, Endian>, EndianSlice<'input, Endian>) { + (self.range_to(..idx), self.range_from(idx..)) + } + + /// Find the first occurence of a byte in the slice, and return its index. + #[inline] + pub fn find(&self, byte: u8) -> Option { + self.slice.iter().position(|ch| *ch == byte) + } + + /// Return the offset of the start of the slice relative to the start + /// of the given slice. + #[inline] + pub fn offset_from(&self, base: EndianSlice<'input, Endian>) -> usize { + let base_ptr = base.slice.as_ptr() as *const u8 as usize; + let ptr = self.slice.as_ptr() as *const u8 as usize; + debug_assert!(base_ptr <= ptr); + debug_assert!(ptr + self.slice.len() <= base_ptr + base.slice.len()); + ptr - base_ptr + } + + /// Converts the slice to a string using `str::from_utf8`. + /// + /// Returns an error if the slice contains invalid characters. + #[inline] + pub fn to_string(&self) -> Result<&'input str> { + str::from_utf8(self.slice).map_err(|_| Error::BadUtf8) + } + + /// Converts the slice to a string, including invalid characters, + /// using `String::from_utf8_lossy`. + #[cfg(feature = "read")] + #[inline] + pub fn to_string_lossy(&self) -> Cow<'input, str> { + String::from_utf8_lossy(self.slice) + } + + #[inline] + fn read_slice(&mut self, len: usize) -> Result<&'input [u8]> { + if self.slice.len() < len { + Err(Error::UnexpectedEof(self.offset_id())) + } else { + let val = &self.slice[..len]; + self.slice = &self.slice[len..]; + Ok(val) + } + } +} + +/// # Range Methods +/// +/// Unfortunately, `std::ops::Index` *must* return a reference, so we can't +/// implement `Index>` to return a new `EndianSlice` the way we would +/// like to. Instead, we abandon fancy indexing operators and have these plain +/// old methods. +impl<'input, Endian> EndianSlice<'input, Endian> +where + Endian: Endianity, +{ + /// Take the given `start..end` range of the underlying slice and return a + /// new `EndianSlice`. + /// + /// ``` + /// use gimli::{EndianSlice, LittleEndian}; + /// + /// let slice = &[0x01, 0x02, 0x03, 0x04]; + /// let endian_slice = EndianSlice::new(slice, LittleEndian); + /// assert_eq!(endian_slice.range(1..3), + /// EndianSlice::new(&slice[1..3], LittleEndian)); + /// ``` + pub fn range(&self, idx: Range) -> EndianSlice<'input, Endian> { + EndianSlice { + slice: &self.slice[idx], + endian: self.endian, + } + } + + /// Take the given `start..` range of the underlying slice and return a new + /// `EndianSlice`. + /// + /// ``` + /// use gimli::{EndianSlice, LittleEndian}; + /// + /// let slice = &[0x01, 0x02, 0x03, 0x04]; + /// let endian_slice = EndianSlice::new(slice, LittleEndian); + /// assert_eq!(endian_slice.range_from(2..), + /// EndianSlice::new(&slice[2..], LittleEndian)); + /// ``` + pub fn range_from(&self, idx: RangeFrom) -> EndianSlice<'input, Endian> { + EndianSlice { + slice: &self.slice[idx], + endian: self.endian, + } + } + + /// Take the given `..end` range of the underlying slice and return a new + /// `EndianSlice`. + /// + /// ``` + /// use gimli::{EndianSlice, LittleEndian}; + /// + /// let slice = &[0x01, 0x02, 0x03, 0x04]; + /// let endian_slice = EndianSlice::new(slice, LittleEndian); + /// assert_eq!(endian_slice.range_to(..3), + /// EndianSlice::new(&slice[..3], LittleEndian)); + /// ``` + pub fn range_to(&self, idx: RangeTo) -> EndianSlice<'input, Endian> { + EndianSlice { + slice: &self.slice[idx], + endian: self.endian, + } + } +} + +impl<'input, Endian> Index for EndianSlice<'input, Endian> +where + Endian: Endianity, +{ + type Output = u8; + fn index(&self, idx: usize) -> &Self::Output { + &self.slice[idx] + } +} + +impl<'input, Endian> Index> for EndianSlice<'input, Endian> +where + Endian: Endianity, +{ + type Output = [u8]; + fn index(&self, idx: RangeFrom) -> &Self::Output { + &self.slice[idx] + } +} + +impl<'input, Endian> Deref for EndianSlice<'input, Endian> +where + Endian: Endianity, +{ + type Target = [u8]; + fn deref(&self) -> &Self::Target { + self.slice + } +} + +impl<'input, Endian> Into<&'input [u8]> for EndianSlice<'input, Endian> +where + Endian: Endianity, +{ + fn into(self) -> &'input [u8] { + self.slice + } +} + +impl<'input, Endian> Reader for EndianSlice<'input, Endian> +where + Endian: Endianity, +{ + type Endian = Endian; + type Offset = usize; + + #[inline] + fn endian(&self) -> Endian { + self.endian + } + + #[inline] + fn len(&self) -> usize { + self.slice.len() + } + + #[inline] + fn is_empty(&self) -> bool { + self.slice.is_empty() + } + + #[inline] + fn empty(&mut self) { + self.slice = &[]; + } + + #[inline] + fn truncate(&mut self, len: usize) -> Result<()> { + if self.slice.len() < len { + Err(Error::UnexpectedEof(self.offset_id())) + } else { + self.slice = &self.slice[..len]; + Ok(()) + } + } + + #[inline] + fn offset_from(&self, base: &Self) -> usize { + self.offset_from(*base) + } + + #[inline] + fn offset_id(&self) -> ReaderOffsetId { + ReaderOffsetId(self.slice.as_ptr() as u64) + } + + #[inline] + fn lookup_offset_id(&self, id: ReaderOffsetId) -> Option { + let id = id.0; + let self_id = self.slice.as_ptr() as u64; + let self_len = self.slice.len() as u64; + if id >= self_id && id <= self_id + self_len { + Some((id - self_id) as usize) + } else { + None + } + } + + #[inline] + fn find(&self, byte: u8) -> Result { + self.find(byte) + .ok_or_else(|| Error::UnexpectedEof(self.offset_id())) + } + + #[inline] + fn skip(&mut self, len: usize) -> Result<()> { + if self.slice.len() < len { + Err(Error::UnexpectedEof(self.offset_id())) + } else { + self.slice = &self.slice[len..]; + Ok(()) + } + } + + #[inline] + fn split(&mut self, len: usize) -> Result { + let slice = self.read_slice(len)?; + Ok(EndianSlice::new(slice, self.endian)) + } + + #[cfg(not(feature = "read"))] + fn cannot_implement() -> super::reader::seal_if_no_alloc::Sealed { + super::reader::seal_if_no_alloc::Sealed + } + + #[cfg(feature = "read")] + #[inline] + fn to_slice(&self) -> Result> { + Ok(self.slice.into()) + } + + #[cfg(feature = "read")] + #[inline] + fn to_string(&self) -> Result> { + match str::from_utf8(self.slice) { + Ok(s) => Ok(s.into()), + _ => Err(Error::BadUtf8), + } + } + + #[cfg(feature = "read")] + #[inline] + fn to_string_lossy(&self) -> Result> { + Ok(String::from_utf8_lossy(self.slice)) + } + + #[inline] + fn read_slice(&mut self, buf: &mut [u8]) -> Result<()> { + let slice = self.read_slice(buf.len())?; + buf.copy_from_slice(slice); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::endianity::NativeEndian; + + #[test] + fn test_endian_slice_split_at() { + let endian = NativeEndian; + let slice = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 0]; + let eb = EndianSlice::new(slice, endian); + assert_eq!( + eb.split_at(3), + ( + EndianSlice::new(&slice[..3], endian), + EndianSlice::new(&slice[3..], endian) + ) + ); + } + + #[test] + #[should_panic] + fn test_endian_slice_split_at_out_of_bounds() { + let slice = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 0]; + let eb = EndianSlice::new(slice, NativeEndian); + eb.split_at(30); + } +} diff --git a/crux-mir/lib/gimli/src/read/index.rs b/crux-mir/lib/gimli/src/read/index.rs new file mode 100644 index 000000000..129eb2fb1 --- /dev/null +++ b/crux-mir/lib/gimli/src/read/index.rs @@ -0,0 +1,535 @@ +use core::slice; + +use crate::common::SectionId; +use crate::constants; +use crate::endianity::Endianity; +use crate::read::{EndianSlice, Error, Reader, ReaderOffset, Result, Section}; + +/// The data in the `.debug_cu_index` section of a `.dwp` file. +/// +/// This section contains the compilation unit index. +#[derive(Debug, Default, Clone, Copy)] +pub struct DebugCuIndex { + section: R, +} + +impl<'input, Endian> DebugCuIndex> +where + Endian: Endianity, +{ + /// Construct a new `DebugCuIndex` instance from the data in the `.debug_cu_index` + /// section. + pub fn new(section: &'input [u8], endian: Endian) -> Self { + Self::from(EndianSlice::new(section, endian)) + } +} + +impl Section for DebugCuIndex { + fn id() -> SectionId { + SectionId::DebugCuIndex + } + + fn reader(&self) -> &R { + &self.section + } +} + +impl From for DebugCuIndex { + fn from(section: R) -> Self { + DebugCuIndex { section } + } +} + +impl DebugCuIndex { + /// Parse the index header. + pub fn index(self) -> Result> { + UnitIndex::parse(self.section) + } +} + +/// The data in the `.debug_tu_index` section of a `.dwp` file. +/// +/// This section contains the type unit index. +#[derive(Debug, Default, Clone, Copy)] +pub struct DebugTuIndex { + section: R, +} + +impl<'input, Endian> DebugTuIndex> +where + Endian: Endianity, +{ + /// Construct a new `DebugTuIndex` instance from the data in the `.debug_tu_index` + /// section. + pub fn new(section: &'input [u8], endian: Endian) -> Self { + Self::from(EndianSlice::new(section, endian)) + } +} + +impl Section for DebugTuIndex { + fn id() -> SectionId { + SectionId::DebugTuIndex + } + + fn reader(&self) -> &R { + &self.section + } +} + +impl From for DebugTuIndex { + fn from(section: R) -> Self { + DebugTuIndex { section } + } +} + +impl DebugTuIndex { + /// Parse the index header. + pub fn index(self) -> Result> { + UnitIndex::parse(self.section) + } +} + +const SECTION_COUNT_MAX: u8 = 8; + +/// The partially parsed index from a `DebugCuIndex` or `DebugTuIndex`. +#[derive(Debug, Clone)] +pub struct UnitIndex { + version: u16, + section_count: u32, + unit_count: u32, + slot_count: u32, + hash_ids: R, + hash_rows: R, + // Only `section_count` values are valid. + sections: [SectionId; SECTION_COUNT_MAX as usize], + offsets: R, + sizes: R, +} + +impl UnitIndex { + fn parse(mut input: R) -> Result> { + if input.is_empty() { + return Ok(UnitIndex { + version: 5, + section_count: 0, + unit_count: 0, + slot_count: 0, + hash_ids: input.clone(), + hash_rows: input.clone(), + sections: [SectionId::DebugAbbrev; SECTION_COUNT_MAX as usize], + offsets: input.clone(), + sizes: input.clone(), + }); + } + + // GNU split-dwarf extension to DWARF 4 uses a 32-bit version, + // but DWARF 5 uses a 16-bit version followed by 16-bit padding. + let mut original_input = input.clone(); + let version; + if input.read_u32()? == 2 { + version = 2 + } else { + version = original_input.read_u16()?; + if version != 5 { + return Err(Error::UnknownVersion(version.into())); + } + } + + let section_count = input.read_u32()?; + let unit_count = input.read_u32()?; + let slot_count = input.read_u32()?; + if slot_count == 0 || slot_count & (slot_count - 1) != 0 || slot_count <= unit_count { + return Err(Error::InvalidIndexSlotCount); + } + + let hash_ids = input.split(R::Offset::from_u64(u64::from(slot_count) * 8)?)?; + let hash_rows = input.split(R::Offset::from_u64(u64::from(slot_count) * 4)?)?; + + let mut sections = [SectionId::DebugAbbrev; SECTION_COUNT_MAX as usize]; + if section_count > SECTION_COUNT_MAX.into() { + return Err(Error::InvalidIndexSectionCount); + } + for i in 0..section_count { + let section = input.read_u32()?; + sections[i as usize] = if version == 2 { + match constants::DwSectV2(section) { + constants::DW_SECT_V2_INFO => SectionId::DebugInfo, + constants::DW_SECT_V2_TYPES => SectionId::DebugTypes, + constants::DW_SECT_V2_ABBREV => SectionId::DebugAbbrev, + constants::DW_SECT_V2_LINE => SectionId::DebugLine, + constants::DW_SECT_V2_LOC => SectionId::DebugLoc, + constants::DW_SECT_V2_STR_OFFSETS => SectionId::DebugStrOffsets, + constants::DW_SECT_V2_MACINFO => SectionId::DebugMacinfo, + constants::DW_SECT_V2_MACRO => SectionId::DebugMacro, + _ => return Err(Error::UnknownIndexSection), + } + } else { + match constants::DwSect(section) { + constants::DW_SECT_INFO => SectionId::DebugInfo, + constants::DW_SECT_ABBREV => SectionId::DebugAbbrev, + constants::DW_SECT_LINE => SectionId::DebugLine, + constants::DW_SECT_LOCLISTS => SectionId::DebugLocLists, + constants::DW_SECT_STR_OFFSETS => SectionId::DebugStrOffsets, + constants::DW_SECT_MACRO => SectionId::DebugMacro, + constants::DW_SECT_RNGLISTS => SectionId::DebugRngLists, + _ => return Err(Error::UnknownIndexSection), + } + }; + } + + let offsets = input.split(R::Offset::from_u64( + u64::from(unit_count) * u64::from(section_count) * 4, + )?)?; + let sizes = input.split(R::Offset::from_u64( + u64::from(unit_count) * u64::from(section_count) * 4, + )?)?; + + Ok(UnitIndex { + version, + section_count, + unit_count, + slot_count, + hash_ids, + hash_rows, + sections, + offsets, + sizes, + }) + } + + /// Find `id` in the index hash table, and return the row index. + /// + /// `id` may be a compilation unit ID if this index is from `.debug_cu_index`, + /// or a type signature if this index is from `.debug_tu_index`. + pub fn find(&self, id: u64) -> Option { + if self.slot_count == 0 { + return None; + } + let mask = u64::from(self.slot_count - 1); + let mut hash1 = id & mask; + let hash2 = ((id >> 32) & mask) | 1; + for _ in 0..self.slot_count { + // The length of these arrays was validated in `UnitIndex::parse`. + let mut hash_ids = self.hash_ids.clone(); + hash_ids.skip(R::Offset::from_u64(hash1 * 8).ok()?).ok()?; + let hash_id = hash_ids.read_u64().ok()?; + if hash_id == id { + let mut hash_rows = self.hash_rows.clone(); + hash_rows.skip(R::Offset::from_u64(hash1 * 4).ok()?).ok()?; + let hash_row = hash_rows.read_u32().ok()?; + return Some(hash_row); + } + if hash_id == 0 { + return None; + } + hash1 = (hash1 + hash2) & mask; + } + None + } + + /// Return the section offsets and sizes for the given row index. + pub fn sections(&self, mut row: u32) -> Result> { + if row == 0 { + return Err(Error::InvalidIndexRow); + } + row -= 1; + if row >= self.unit_count { + return Err(Error::InvalidIndexRow); + } + let mut offsets = self.offsets.clone(); + offsets.skip(R::Offset::from_u64( + u64::from(row) * u64::from(self.section_count) * 4, + )?)?; + let mut sizes = self.sizes.clone(); + sizes.skip(R::Offset::from_u64( + u64::from(row) * u64::from(self.section_count) * 4, + )?)?; + Ok(UnitIndexSectionIterator { + sections: self.sections[..self.section_count as usize].iter(), + offsets, + sizes, + }) + } + + /// Return the version. + pub fn version(&self) -> u16 { + self.version + } + + /// Return the number of sections. + pub fn section_count(&self) -> u32 { + self.section_count + } + + /// Return the number of units. + pub fn unit_count(&self) -> u32 { + self.unit_count + } + + /// Return the number of slots. + pub fn slot_count(&self) -> u32 { + self.slot_count + } +} + +/// An iterator over the section offsets and sizes for a row in a `UnitIndex`. +#[derive(Debug, Clone)] +pub struct UnitIndexSectionIterator<'index, R: Reader> { + sections: slice::Iter<'index, SectionId>, + offsets: R, + sizes: R, +} + +impl<'index, R: Reader> Iterator for UnitIndexSectionIterator<'index, R> { + type Item = UnitIndexSection; + + fn next(&mut self) -> Option { + let section = *self.sections.next()?; + // The length of these arrays was validated in `UnitIndex::parse`. + let offset = self.offsets.read_u32().ok()?; + let size = self.sizes.read_u32().ok()?; + Some(UnitIndexSection { + section, + offset, + size, + }) + } +} + +/// Information about a unit's contribution to a section in a `.dwp` file. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct UnitIndexSection { + /// The section kind. + pub section: SectionId, + /// The base offset of the unit's contribution to the section. + pub offset: u32, + /// The size of the unit's contribution to the section. + pub size: u32, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::endianity::BigEndian; + use test_assembler::{Endian, Section}; + + #[test] + fn test_empty() { + let buf = EndianSlice::new(&[], BigEndian); + let index = UnitIndex::parse(buf).unwrap(); + assert!(index.find(0).is_none()); + } + + #[test] + fn test_version_2() { + #[rustfmt::skip] + let section = Section::with_endian(Endian::Big) + // Header. + .D32(2).D32(0).D32(0).D32(1) + // Slots. + .D64(0).D32(0); + let buf = section.get_contents().unwrap(); + let buf = EndianSlice::new(&buf, BigEndian); + let index = UnitIndex::parse(buf).unwrap(); + assert_eq!(index.version, 2); + } + + #[test] + fn test_version_5() { + #[rustfmt::skip] + let section = Section::with_endian(Endian::Big) + // Header. + .D16(5).D16(0).D32(0).D32(0).D32(1) + // Slots. + .D64(0).D32(0); + let buf = section.get_contents().unwrap(); + let buf = EndianSlice::new(&buf, BigEndian); + let index = UnitIndex::parse(buf).unwrap(); + assert_eq!(index.version, 5); + } + + #[test] + fn test_version_5_invalid() { + #[rustfmt::skip] + let section = Section::with_endian(Endian::Big) + // Header. + .D32(5).D32(0).D32(0).D32(1) + // Slots. + .D64(0).D32(0); + let buf = section.get_contents().unwrap(); + let buf = EndianSlice::new(&buf, BigEndian); + assert!(UnitIndex::parse(buf).is_err()); + } + + #[test] + fn test_version_2_sections() { + #[rustfmt::skip] + let section = Section::with_endian(Endian::Big) + // Header. + .D32(2).D32(8).D32(1).D32(2) + // Slots. + .D64(0).D64(0).D32(0).D32(0) + // Sections. + .D32(constants::DW_SECT_V2_INFO.0) + .D32(constants::DW_SECT_V2_TYPES.0) + .D32(constants::DW_SECT_V2_ABBREV.0) + .D32(constants::DW_SECT_V2_LINE.0) + .D32(constants::DW_SECT_V2_LOC.0) + .D32(constants::DW_SECT_V2_STR_OFFSETS.0) + .D32(constants::DW_SECT_V2_MACINFO.0) + .D32(constants::DW_SECT_V2_MACRO.0) + // Offsets. + .D32(11).D32(12).D32(13).D32(14).D32(15).D32(16).D32(17).D32(18) + // Sizes. + .D32(21).D32(22).D32(23).D32(24).D32(25).D32(26).D32(27).D32(28); + let buf = section.get_contents().unwrap(); + let buf = EndianSlice::new(&buf, BigEndian); + let index = UnitIndex::parse(buf).unwrap(); + assert_eq!(index.section_count, 8); + assert_eq!( + index.sections, + [ + SectionId::DebugInfo, + SectionId::DebugTypes, + SectionId::DebugAbbrev, + SectionId::DebugLine, + SectionId::DebugLoc, + SectionId::DebugStrOffsets, + SectionId::DebugMacinfo, + SectionId::DebugMacro, + ] + ); + #[rustfmt::skip] + let expect = [ + UnitIndexSection { section: SectionId::DebugInfo, offset: 11, size: 21 }, + UnitIndexSection { section: SectionId::DebugTypes, offset: 12, size: 22 }, + UnitIndexSection { section: SectionId::DebugAbbrev, offset: 13, size: 23 }, + UnitIndexSection { section: SectionId::DebugLine, offset: 14, size: 24 }, + UnitIndexSection { section: SectionId::DebugLoc, offset: 15, size: 25 }, + UnitIndexSection { section: SectionId::DebugStrOffsets, offset: 16, size: 26 }, + UnitIndexSection { section: SectionId::DebugMacinfo, offset: 17, size: 27 }, + UnitIndexSection { section: SectionId::DebugMacro, offset: 18, size: 28 }, + ]; + let mut sections = index.sections(1).unwrap(); + for section in &expect { + assert_eq!(*section, sections.next().unwrap()); + } + assert!(sections.next().is_none()); + } + + #[test] + fn test_version_5_sections() { + #[rustfmt::skip] + let section = Section::with_endian(Endian::Big) + // Header. + .D16(5).D16(0).D32(7).D32(1).D32(2) + // Slots. + .D64(0).D64(0).D32(0).D32(0) + // Sections. + .D32(constants::DW_SECT_INFO.0) + .D32(constants::DW_SECT_ABBREV.0) + .D32(constants::DW_SECT_LINE.0) + .D32(constants::DW_SECT_LOCLISTS.0) + .D32(constants::DW_SECT_STR_OFFSETS.0) + .D32(constants::DW_SECT_MACRO.0) + .D32(constants::DW_SECT_RNGLISTS.0) + // Offsets. + .D32(11).D32(12).D32(13).D32(14).D32(15).D32(16).D32(17) + // Sizes. + .D32(21).D32(22).D32(23).D32(24).D32(25).D32(26).D32(27); + let buf = section.get_contents().unwrap(); + let buf = EndianSlice::new(&buf, BigEndian); + let index = UnitIndex::parse(buf).unwrap(); + assert_eq!(index.section_count, 7); + assert_eq!( + index.sections[..7], + [ + SectionId::DebugInfo, + SectionId::DebugAbbrev, + SectionId::DebugLine, + SectionId::DebugLocLists, + SectionId::DebugStrOffsets, + SectionId::DebugMacro, + SectionId::DebugRngLists, + ] + ); + #[rustfmt::skip] + let expect = [ + UnitIndexSection { section: SectionId::DebugInfo, offset: 11, size: 21 }, + UnitIndexSection { section: SectionId::DebugAbbrev, offset: 12, size: 22 }, + UnitIndexSection { section: SectionId::DebugLine, offset: 13, size: 23 }, + UnitIndexSection { section: SectionId::DebugLocLists, offset: 14, size: 24 }, + UnitIndexSection { section: SectionId::DebugStrOffsets, offset: 15, size: 25 }, + UnitIndexSection { section: SectionId::DebugMacro, offset: 16, size: 26 }, + UnitIndexSection { section: SectionId::DebugRngLists, offset: 17, size: 27 }, + ]; + let mut sections = index.sections(1).unwrap(); + for section in &expect { + assert_eq!(*section, sections.next().unwrap()); + } + assert!(sections.next().is_none()); + + assert!(index.sections(0).is_err()); + assert!(index.sections(2).is_err()); + } + + #[test] + fn test_hash() { + #[rustfmt::skip] + let section = Section::with_endian(Endian::Big) + // Header. + .D16(5).D16(0).D32(2).D32(3).D32(4) + // Slots. + .D64(0xffff_fff2_ffff_fff1) + .D64(0xffff_fff0_ffff_fff1) + .D64(0xffff_fff1_ffff_fff1) + .D64(0) + .D32(3).D32(1).D32(2).D32(0) + // Sections. + .D32(constants::DW_SECT_INFO.0) + .D32(constants::DW_SECT_ABBREV.0) + // Offsets. + .D32(0).D32(0).D32(0).D32(0).D32(0).D32(0) + // Sizes. + .D32(0).D32(0).D32(0).D32(0).D32(0).D32(0); + let buf = section.get_contents().unwrap(); + let buf = EndianSlice::new(&buf, BigEndian); + let index = UnitIndex::parse(buf).unwrap(); + assert_eq!(index.version(), 5); + assert_eq!(index.slot_count(), 4); + assert_eq!(index.unit_count(), 3); + assert_eq!(index.section_count(), 2); + assert_eq!(index.find(0xffff_fff0_ffff_fff1), Some(1)); + assert_eq!(index.find(0xffff_fff1_ffff_fff1), Some(2)); + assert_eq!(index.find(0xffff_fff2_ffff_fff1), Some(3)); + assert_eq!(index.find(0xffff_fff3_ffff_fff1), None); + } + + #[test] + fn test_cu_index() { + #[rustfmt::skip] + let section = Section::with_endian(Endian::Big) + // Header. + .D16(5).D16(0).D32(0).D32(0).D32(1) + // Slots. + .D64(0).D32(0); + let buf = section.get_contents().unwrap(); + let cu_index = DebugCuIndex::new(&buf, BigEndian); + let index = cu_index.index().unwrap(); + assert_eq!(index.version, 5); + } + + #[test] + fn test_tu_index() { + #[rustfmt::skip] + let section = Section::with_endian(Endian::Big) + // Header. + .D16(5).D16(0).D32(0).D32(0).D32(1) + // Slots. + .D64(0).D32(0); + let buf = section.get_contents().unwrap(); + let tu_index = DebugTuIndex::new(&buf, BigEndian); + let index = tu_index.index().unwrap(); + assert_eq!(index.version, 5); + } +} diff --git a/crux-mir/lib/gimli/src/read/line.rs b/crux-mir/lib/gimli/src/read/line.rs new file mode 100644 index 000000000..0e7380bb9 --- /dev/null +++ b/crux-mir/lib/gimli/src/read/line.rs @@ -0,0 +1,3030 @@ +use alloc::vec::Vec; +use core::fmt; +use core::num::{NonZeroU64, Wrapping}; +use core::result; + +use crate::common::{ + DebugLineOffset, DebugLineStrOffset, DebugStrOffset, DebugStrOffsetsIndex, Encoding, Format, + LineEncoding, SectionId, +}; +use crate::constants; +use crate::endianity::Endianity; +use crate::read::{AttributeValue, EndianSlice, Error, Reader, ReaderOffset, Result, Section}; + +/// The `DebugLine` struct contains the source location to instruction mapping +/// found in the `.debug_line` section. +#[derive(Debug, Default, Clone, Copy)] +pub struct DebugLine { + debug_line_section: R, +} + +impl<'input, Endian> DebugLine> +where + Endian: Endianity, +{ + /// Construct a new `DebugLine` instance from the data in the `.debug_line` + /// section. + /// + /// It is the caller's responsibility to read the `.debug_line` section and + /// present it as a `&[u8]` slice. That means using some ELF loader on + /// Linux, a Mach-O loader on macOS, etc. + /// + /// ``` + /// use gimli::{DebugLine, LittleEndian}; + /// + /// # let buf = [0x00, 0x01, 0x02, 0x03]; + /// # let read_debug_line_section_somehow = || &buf; + /// let debug_line = DebugLine::new(read_debug_line_section_somehow(), LittleEndian); + /// ``` + pub fn new(debug_line_section: &'input [u8], endian: Endian) -> Self { + Self::from(EndianSlice::new(debug_line_section, endian)) + } +} + +impl DebugLine { + /// Parse the line number program whose header is at the given `offset` in the + /// `.debug_line` section. + /// + /// The `address_size` must match the compilation unit that the lines apply to. + /// The `comp_dir` should be from the `DW_AT_comp_dir` attribute of the compilation + /// unit. The `comp_name` should be from the `DW_AT_name` attribute of the + /// compilation unit. + /// + /// ```rust,no_run + /// use gimli::{DebugLine, DebugLineOffset, IncompleteLineProgram, EndianSlice, LittleEndian}; + /// + /// # let buf = []; + /// # let read_debug_line_section_somehow = || &buf; + /// let debug_line = DebugLine::new(read_debug_line_section_somehow(), LittleEndian); + /// + /// // In a real example, we'd grab the offset via a compilation unit + /// // entry's `DW_AT_stmt_list` attribute, and the address size from that + /// // unit directly. + /// let offset = DebugLineOffset(0); + /// let address_size = 8; + /// + /// let program = debug_line.program(offset, address_size, None, None) + /// .expect("should have found a header at that offset, and parsed it OK"); + /// ``` + pub fn program( + &self, + offset: DebugLineOffset, + address_size: u8, + comp_dir: Option, + comp_name: Option, + ) -> Result> { + let input = &mut self.debug_line_section.clone(); + input.skip(offset.0)?; + let header = LineProgramHeader::parse(input, offset, address_size, comp_dir, comp_name)?; + let program = IncompleteLineProgram { header }; + Ok(program) + } +} + +impl DebugLine { + /// Create a `DebugLine` section that references the data in `self`. + /// + /// This is useful when `R` implements `Reader` but `T` does not. + /// + /// ## Example Usage + /// + /// ```rust,no_run + /// # let load_section = || unimplemented!(); + /// // Read the DWARF section into a `Vec` with whatever object loader you're using. + /// let owned_section: gimli::DebugLine> = load_section(); + /// // Create a reference to the DWARF section. + /// let section = owned_section.borrow(|section| { + /// gimli::EndianSlice::new(§ion, gimli::LittleEndian) + /// }); + /// ``` + pub fn borrow<'a, F, R>(&'a self, mut borrow: F) -> DebugLine + where + F: FnMut(&'a T) -> R, + { + borrow(&self.debug_line_section).into() + } +} + +impl Section for DebugLine { + fn id() -> SectionId { + SectionId::DebugLine + } + + fn reader(&self) -> &R { + &self.debug_line_section + } +} + +impl From for DebugLine { + fn from(debug_line_section: R) -> Self { + DebugLine { debug_line_section } + } +} + +/// Deprecated. `LineNumberProgram` has been renamed to `LineProgram`. +#[deprecated(note = "LineNumberProgram has been renamed to LineProgram, use that instead.")] +pub type LineNumberProgram = dyn LineProgram; + +/// A `LineProgram` provides access to a `LineProgramHeader` and +/// a way to add files to the files table if necessary. Gimli consumers should +/// never need to use or see this trait. +pub trait LineProgram::Offset> +where + R: Reader, + Offset: ReaderOffset, +{ + /// Get a reference to the held `LineProgramHeader`. + fn header(&self) -> &LineProgramHeader; + /// Add a file to the file table if necessary. + fn add_file(&mut self, file: FileEntry); +} + +impl LineProgram for IncompleteLineProgram +where + R: Reader, + Offset: ReaderOffset, +{ + fn header(&self) -> &LineProgramHeader { + &self.header + } + fn add_file(&mut self, file: FileEntry) { + self.header.file_names.push(file); + } +} + +impl<'program, R, Offset> LineProgram for &'program CompleteLineProgram +where + R: Reader, + Offset: ReaderOffset, +{ + fn header(&self) -> &LineProgramHeader { + &self.header + } + fn add_file(&mut self, _: FileEntry) { + // Nop. Our file table is already complete. + } +} + +/// Deprecated. `StateMachine` has been renamed to `LineRows`. +#[deprecated(note = "StateMachine has been renamed to LineRows, use that instead.")] +pub type StateMachine = LineRows; + +/// Executes a `LineProgram` to iterate over the rows in the matrix of line number information. +/// +/// "The hypothetical machine used by a consumer of the line number information +/// to expand the byte-coded instruction stream into a matrix of line number +/// information." -- Section 6.2.1 +#[derive(Debug, Clone)] +pub struct LineRows::Offset> +where + Program: LineProgram, + R: Reader, + Offset: ReaderOffset, +{ + program: Program, + row: LineRow, + instructions: LineInstructions, +} + +type OneShotLineRows::Offset> = + LineRows, Offset>; + +type ResumedLineRows<'program, R, Offset = ::Offset> = + LineRows, Offset>; + +impl LineRows +where + Program: LineProgram, + R: Reader, + Offset: ReaderOffset, +{ + #[allow(clippy::new_ret_no_self)] + fn new(program: IncompleteLineProgram) -> OneShotLineRows { + let row = LineRow::new(program.header()); + let instructions = LineInstructions { + input: program.header().program_buf.clone(), + }; + LineRows { + program, + row, + instructions, + } + } + + fn resume<'program>( + program: &'program CompleteLineProgram, + sequence: &LineSequence, + ) -> ResumedLineRows<'program, R, Offset> { + let row = LineRow::new(program.header()); + let instructions = sequence.instructions.clone(); + LineRows { + program, + row, + instructions, + } + } + + /// Get a reference to the header for this state machine's line number + /// program. + #[inline] + pub fn header(&self) -> &LineProgramHeader { + self.program.header() + } + + /// Parse and execute the next instructions in the line number program until + /// another row in the line number matrix is computed. + /// + /// The freshly computed row is returned as `Ok(Some((header, row)))`. + /// If the matrix is complete, and there are no more new rows in the line + /// number matrix, then `Ok(None)` is returned. If there was an error parsing + /// an instruction, then `Err(e)` is returned. + /// + /// Unfortunately, the references mean that this cannot be a + /// `FallibleIterator`. + pub fn next_row(&mut self) -> Result, &LineRow)>> { + // Perform any reset that was required after copying the previous row. + self.row.reset(self.program.header()); + + loop { + // Split the borrow here, rather than calling `self.header()`. + match self.instructions.next_instruction(self.program.header()) { + Err(err) => return Err(err), + Ok(None) => return Ok(None), + Ok(Some(instruction)) => { + if self.row.execute(instruction, &mut self.program) { + return Ok(Some((self.header(), &self.row))); + } + // Fall through, parse the next instruction, and see if that + // yields a row. + } + } + } + } +} + +/// Deprecated. `Opcode` has been renamed to `LineInstruction`. +#[deprecated(note = "Opcode has been renamed to LineInstruction, use that instead.")] +pub type Opcode = LineInstruction::Offset>; + +/// A parsed line number program instruction. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LineInstruction::Offset> +where + R: Reader, + Offset: ReaderOffset, +{ + /// > ### 6.2.5.1 Special Opcodes + /// > + /// > Each ubyte special opcode has the following effect on the state machine: + /// > + /// > 1. Add a signed integer to the line register. + /// > + /// > 2. Modify the operation pointer by incrementing the address and + /// > op_index registers as described below. + /// > + /// > 3. Append a row to the matrix using the current values of the state + /// > machine registers. + /// > + /// > 4. Set the basic_block register to “false.” + /// > + /// > 5. Set the prologue_end register to “false.” + /// > + /// > 6. Set the epilogue_begin register to “false.” + /// > + /// > 7. Set the discriminator register to 0. + /// > + /// > All of the special opcodes do those same seven things; they differ from + /// > one another only in what values they add to the line, address and + /// > op_index registers. + Special(u8), + + /// "[`LineInstruction::Copy`] appends a row to the matrix using the current + /// values of the state machine registers. Then it sets the discriminator + /// register to 0, and sets the basic_block, prologue_end and epilogue_begin + /// registers to “false.”" + Copy, + + /// "The DW_LNS_advance_pc opcode takes a single unsigned LEB128 operand as + /// the operation advance and modifies the address and op_index registers + /// [the same as `LineInstruction::Special`]" + AdvancePc(u64), + + /// "The DW_LNS_advance_line opcode takes a single signed LEB128 operand and + /// adds that value to the line register of the state machine." + AdvanceLine(i64), + + /// "The DW_LNS_set_file opcode takes a single unsigned LEB128 operand and + /// stores it in the file register of the state machine." + SetFile(u64), + + /// "The DW_LNS_set_column opcode takes a single unsigned LEB128 operand and + /// stores it in the column register of the state machine." + SetColumn(u64), + + /// "The DW_LNS_negate_stmt opcode takes no operands. It sets the is_stmt + /// register of the state machine to the logical negation of its current + /// value." + NegateStatement, + + /// "The DW_LNS_set_basic_block opcode takes no operands. It sets the + /// basic_block register of the state machine to “true.”" + SetBasicBlock, + + /// > The DW_LNS_const_add_pc opcode takes no operands. It advances the + /// > address and op_index registers by the increments corresponding to + /// > special opcode 255. + /// > + /// > When the line number program needs to advance the address by a small + /// > amount, it can use a single special opcode, which occupies a single + /// > byte. When it needs to advance the address by up to twice the range of + /// > the last special opcode, it can use DW_LNS_const_add_pc followed by a + /// > special opcode, for a total of two bytes. Only if it needs to advance + /// > the address by more than twice that range will it need to use both + /// > DW_LNS_advance_pc and a special opcode, requiring three or more bytes. + ConstAddPc, + + /// > The DW_LNS_fixed_advance_pc opcode takes a single uhalf (unencoded) + /// > operand and adds it to the address register of the state machine and + /// > sets the op_index register to 0. This is the only standard opcode whose + /// > operand is not a variable length number. It also does not multiply the + /// > operand by the minimum_instruction_length field of the header. + FixedAddPc(u16), + + /// "[`LineInstruction::SetPrologueEnd`] sets the prologue_end register to “true”." + SetPrologueEnd, + + /// "[`LineInstruction::SetEpilogueBegin`] sets the epilogue_begin register to + /// “true”." + SetEpilogueBegin, + + /// "The DW_LNS_set_isa opcode takes a single unsigned LEB128 operand and + /// stores that value in the isa register of the state machine." + SetIsa(u64), + + /// An unknown standard opcode with zero operands. + UnknownStandard0(constants::DwLns), + + /// An unknown standard opcode with one operand. + UnknownStandard1(constants::DwLns, u64), + + /// An unknown standard opcode with multiple operands. + UnknownStandardN(constants::DwLns, R), + + /// > [`LineInstruction::EndSequence`] sets the end_sequence register of the state + /// > machine to “true” and appends a row to the matrix using the current + /// > values of the state-machine registers. Then it resets the registers to + /// > the initial values specified above (see Section 6.2.2). Every line + /// > number program sequence must end with a DW_LNE_end_sequence instruction + /// > which creates a row whose address is that of the byte after the last + /// > target machine instruction of the sequence. + EndSequence, + + /// > The DW_LNE_set_address opcode takes a single relocatable address as an + /// > operand. The size of the operand is the size of an address on the target + /// > machine. It sets the address register to the value given by the + /// > relocatable address and sets the op_index register to 0. + /// > + /// > All of the other line number program opcodes that affect the address + /// > register add a delta to it. This instruction stores a relocatable value + /// > into it instead. + SetAddress(u64), + + /// Defines a new source file in the line number program and appends it to + /// the line number program header's list of source files. + DefineFile(FileEntry), + + /// "The DW_LNE_set_discriminator opcode takes a single parameter, an + /// unsigned LEB128 integer. It sets the discriminator register to the new + /// value." + SetDiscriminator(u64), + + /// An unknown extended opcode and the slice of its unparsed operands. + UnknownExtended(constants::DwLne, R), +} + +impl LineInstruction +where + R: Reader, + Offset: ReaderOffset, +{ + fn parse<'header>( + header: &'header LineProgramHeader, + input: &mut R, + ) -> Result> + where + R: 'header, + { + let opcode = input.read_u8()?; + if opcode == 0 { + let length = input.read_uleb128().and_then(R::Offset::from_u64)?; + let mut instr_rest = input.split(length)?; + let opcode = instr_rest.read_u8()?; + + match constants::DwLne(opcode) { + constants::DW_LNE_end_sequence => Ok(LineInstruction::EndSequence), + + constants::DW_LNE_set_address => { + let address = instr_rest.read_address(header.address_size())?; + Ok(LineInstruction::SetAddress(address)) + } + + constants::DW_LNE_define_file => { + if header.version() <= 4 { + let path_name = instr_rest.read_null_terminated_slice()?; + let entry = FileEntry::parse(&mut instr_rest, path_name)?; + Ok(LineInstruction::DefineFile(entry)) + } else { + Ok(LineInstruction::UnknownExtended( + constants::DW_LNE_define_file, + instr_rest, + )) + } + } + + constants::DW_LNE_set_discriminator => { + let discriminator = instr_rest.read_uleb128()?; + Ok(LineInstruction::SetDiscriminator(discriminator)) + } + + otherwise => Ok(LineInstruction::UnknownExtended(otherwise, instr_rest)), + } + } else if opcode >= header.opcode_base { + Ok(LineInstruction::Special(opcode)) + } else { + match constants::DwLns(opcode) { + constants::DW_LNS_copy => Ok(LineInstruction::Copy), + + constants::DW_LNS_advance_pc => { + let advance = input.read_uleb128()?; + Ok(LineInstruction::AdvancePc(advance)) + } + + constants::DW_LNS_advance_line => { + let increment = input.read_sleb128()?; + Ok(LineInstruction::AdvanceLine(increment)) + } + + constants::DW_LNS_set_file => { + let file = input.read_uleb128()?; + Ok(LineInstruction::SetFile(file)) + } + + constants::DW_LNS_set_column => { + let column = input.read_uleb128()?; + Ok(LineInstruction::SetColumn(column)) + } + + constants::DW_LNS_negate_stmt => Ok(LineInstruction::NegateStatement), + + constants::DW_LNS_set_basic_block => Ok(LineInstruction::SetBasicBlock), + + constants::DW_LNS_const_add_pc => Ok(LineInstruction::ConstAddPc), + + constants::DW_LNS_fixed_advance_pc => { + let advance = input.read_u16()?; + Ok(LineInstruction::FixedAddPc(advance)) + } + + constants::DW_LNS_set_prologue_end => Ok(LineInstruction::SetPrologueEnd), + + constants::DW_LNS_set_epilogue_begin => Ok(LineInstruction::SetEpilogueBegin), + + constants::DW_LNS_set_isa => { + let isa = input.read_uleb128()?; + Ok(LineInstruction::SetIsa(isa)) + } + + otherwise => { + let mut opcode_lengths = header.standard_opcode_lengths().clone(); + opcode_lengths.skip(R::Offset::from_u8(opcode - 1))?; + let num_args = opcode_lengths.read_u8()? as usize; + match num_args { + 0 => Ok(LineInstruction::UnknownStandard0(otherwise)), + 1 => { + let arg = input.read_uleb128()?; + Ok(LineInstruction::UnknownStandard1(otherwise, arg)) + } + _ => { + let mut args = input.clone(); + for _ in 0..num_args { + input.read_uleb128()?; + } + let len = input.offset_from(&args); + args.truncate(len)?; + Ok(LineInstruction::UnknownStandardN(otherwise, args)) + } + } + } + } + } + } +} + +impl fmt::Display for LineInstruction +where + R: Reader, + Offset: ReaderOffset, +{ + fn fmt(&self, f: &mut fmt::Formatter) -> result::Result<(), fmt::Error> { + match *self { + LineInstruction::Special(opcode) => write!(f, "Special opcode {}", opcode), + LineInstruction::Copy => write!(f, "{}", constants::DW_LNS_copy), + LineInstruction::AdvancePc(advance) => { + write!(f, "{} by {}", constants::DW_LNS_advance_pc, advance) + } + LineInstruction::AdvanceLine(increment) => { + write!(f, "{} by {}", constants::DW_LNS_advance_line, increment) + } + LineInstruction::SetFile(file) => { + write!(f, "{} to {}", constants::DW_LNS_set_file, file) + } + LineInstruction::SetColumn(column) => { + write!(f, "{} to {}", constants::DW_LNS_set_column, column) + } + LineInstruction::NegateStatement => write!(f, "{}", constants::DW_LNS_negate_stmt), + LineInstruction::SetBasicBlock => write!(f, "{}", constants::DW_LNS_set_basic_block), + LineInstruction::ConstAddPc => write!(f, "{}", constants::DW_LNS_const_add_pc), + LineInstruction::FixedAddPc(advance) => { + write!(f, "{} by {}", constants::DW_LNS_fixed_advance_pc, advance) + } + LineInstruction::SetPrologueEnd => write!(f, "{}", constants::DW_LNS_set_prologue_end), + LineInstruction::SetEpilogueBegin => { + write!(f, "{}", constants::DW_LNS_set_epilogue_begin) + } + LineInstruction::SetIsa(isa) => write!(f, "{} to {}", constants::DW_LNS_set_isa, isa), + LineInstruction::UnknownStandard0(opcode) => write!(f, "Unknown {}", opcode), + LineInstruction::UnknownStandard1(opcode, arg) => { + write!(f, "Unknown {} with operand {}", opcode, arg) + } + LineInstruction::UnknownStandardN(opcode, ref args) => { + write!(f, "Unknown {} with operands {:?}", opcode, args) + } + LineInstruction::EndSequence => write!(f, "{}", constants::DW_LNE_end_sequence), + LineInstruction::SetAddress(address) => { + write!(f, "{} to {}", constants::DW_LNE_set_address, address) + } + LineInstruction::DefineFile(_) => write!(f, "{}", constants::DW_LNE_define_file), + LineInstruction::SetDiscriminator(discr) => { + write!(f, "{} to {}", constants::DW_LNE_set_discriminator, discr) + } + LineInstruction::UnknownExtended(opcode, _) => write!(f, "Unknown {}", opcode), + } + } +} + +/// Deprecated. `OpcodesIter` has been renamed to `LineInstructions`. +#[deprecated(note = "OpcodesIter has been renamed to LineInstructions, use that instead.")] +pub type OpcodesIter = LineInstructions; + +/// An iterator yielding parsed instructions. +/// +/// See +/// [`LineProgramHeader::instructions`](./struct.LineProgramHeader.html#method.instructions) +/// for more details. +#[derive(Clone, Debug)] +pub struct LineInstructions { + input: R, +} + +impl LineInstructions { + fn remove_trailing(&self, other: &LineInstructions) -> Result> { + let offset = other.input.offset_from(&self.input); + let mut input = self.input.clone(); + input.truncate(offset)?; + Ok(LineInstructions { input }) + } +} + +impl LineInstructions { + /// Advance the iterator and return the next instruction. + /// + /// Returns the newly parsed instruction as `Ok(Some(instruction))`. Returns + /// `Ok(None)` when iteration is complete and all instructions have already been + /// parsed and yielded. If an error occurs while parsing the next attribute, + /// then this error is returned as `Err(e)`, and all subsequent calls return + /// `Ok(None)`. + /// + /// Unfortunately, the `header` parameter means that this cannot be a + /// `FallibleIterator`. + #[allow(clippy::inline_always)] + #[inline(always)] + pub fn next_instruction( + &mut self, + header: &LineProgramHeader, + ) -> Result>> { + if self.input.is_empty() { + return Ok(None); + } + + match LineInstruction::parse(header, &mut self.input) { + Ok(instruction) => Ok(Some(instruction)), + Err(e) => { + self.input.empty(); + Err(e) + } + } + } +} + +/// Deprecated. `LineNumberRow` has been renamed to `LineRow`. +#[deprecated(note = "LineNumberRow has been renamed to LineRow, use that instead.")] +pub type LineNumberRow = LineRow; + +/// A row in the line number program's resulting matrix. +/// +/// Each row is a copy of the registers of the state machine, as defined in section 6.2.2. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LineRow { + address: Wrapping, + op_index: Wrapping, + file: u64, + line: Wrapping, + column: u64, + is_stmt: bool, + basic_block: bool, + end_sequence: bool, + prologue_end: bool, + epilogue_begin: bool, + isa: u64, + discriminator: u64, +} + +impl LineRow { + /// Create a line number row in the initial state for the given program. + pub fn new(header: &LineProgramHeader) -> Self { + LineRow { + // "At the beginning of each sequence within a line number program, the + // state of the registers is:" -- Section 6.2.2 + address: Wrapping(0), + op_index: Wrapping(0), + file: 1, + line: Wrapping(1), + column: 0, + // "determined by default_is_stmt in the line number program header" + is_stmt: header.line_encoding.default_is_stmt, + basic_block: false, + end_sequence: false, + prologue_end: false, + epilogue_begin: false, + // "The isa value 0 specifies that the instruction set is the + // architecturally determined default instruction set. This may be fixed + // by the ABI, or it may be specified by other means, for example, by + // the object file description." + isa: 0, + discriminator: 0, + } + } + + /// "The program-counter value corresponding to a machine instruction + /// generated by the compiler." + #[inline] + pub fn address(&self) -> u64 { + self.address.0 + } + + /// > An unsigned integer representing the index of an operation within a VLIW + /// > instruction. The index of the first operation is 0. For non-VLIW + /// > architectures, this register will always be 0. + /// > + /// > The address and op_index registers, taken together, form an operation + /// > pointer that can reference any individual operation with the + /// > instruction stream. + #[inline] + pub fn op_index(&self) -> u64 { + self.op_index.0 + } + + /// "An unsigned integer indicating the identity of the source file + /// corresponding to a machine instruction." + #[inline] + pub fn file_index(&self) -> u64 { + self.file + } + + /// The source file corresponding to the current machine instruction. + #[inline] + pub fn file<'header, R: Reader>( + &self, + header: &'header LineProgramHeader, + ) -> Option<&'header FileEntry> { + header.file(self.file) + } + + /// "An unsigned integer indicating a source line number. Lines are numbered + /// beginning at 1. The compiler may emit the value 0 in cases where an + /// instruction cannot be attributed to any source line." + /// Line number values of 0 are represented as `None`. + #[inline] + pub fn line(&self) -> Option { + NonZeroU64::new(self.line.0) + } + + /// "An unsigned integer indicating a column number within a source + /// line. Columns are numbered beginning at 1. The value 0 is reserved to + /// indicate that a statement begins at the “left edge” of the line." + #[inline] + pub fn column(&self) -> ColumnType { + NonZeroU64::new(self.column) + .map(ColumnType::Column) + .unwrap_or(ColumnType::LeftEdge) + } + + /// "A boolean indicating that the current instruction is a recommended + /// breakpoint location. A recommended breakpoint location is intended to + /// “represent” a line, a statement and/or a semantically distinct subpart + /// of a statement." + #[inline] + pub fn is_stmt(&self) -> bool { + self.is_stmt + } + + /// "A boolean indicating that the current instruction is the beginning of a + /// basic block." + #[inline] + pub fn basic_block(&self) -> bool { + self.basic_block + } + + /// "A boolean indicating that the current address is that of the first byte + /// after the end of a sequence of target machine instructions. end_sequence + /// terminates a sequence of lines; therefore other information in the same + /// row is not meaningful." + #[inline] + pub fn end_sequence(&self) -> bool { + self.end_sequence + } + + /// "A boolean indicating that the current address is one (of possibly many) + /// where execution should be suspended for an entry breakpoint of a + /// function." + #[inline] + pub fn prologue_end(&self) -> bool { + self.prologue_end + } + + /// "A boolean indicating that the current address is one (of possibly many) + /// where execution should be suspended for an exit breakpoint of a + /// function." + #[inline] + pub fn epilogue_begin(&self) -> bool { + self.epilogue_begin + } + + /// Tag for the current instruction set architecture. + /// + /// > An unsigned integer whose value encodes the applicable instruction set + /// > architecture for the current instruction. + /// > + /// > The encoding of instruction sets should be shared by all users of a + /// > given architecture. It is recommended that this encoding be defined by + /// > the ABI authoring committee for each architecture. + #[inline] + pub fn isa(&self) -> u64 { + self.isa + } + + /// "An unsigned integer identifying the block to which the current + /// instruction belongs. Discriminator values are assigned arbitrarily by + /// the DWARF producer and serve to distinguish among multiple blocks that + /// may all be associated with the same source file, line, and column. Where + /// only one block exists for a given source position, the discriminator + /// value should be zero." + #[inline] + pub fn discriminator(&self) -> u64 { + self.discriminator + } + + /// Execute the given instruction, and return true if a new row in the + /// line number matrix needs to be generated. + /// + /// Unknown opcodes are treated as no-ops. + #[inline] + pub fn execute( + &mut self, + instruction: LineInstruction, + program: &mut Program, + ) -> bool + where + Program: LineProgram, + R: Reader, + { + match instruction { + LineInstruction::Special(opcode) => { + self.exec_special_opcode(opcode, program.header()); + true + } + + LineInstruction::Copy => true, + + LineInstruction::AdvancePc(operation_advance) => { + self.apply_operation_advance(operation_advance, program.header()); + false + } + + LineInstruction::AdvanceLine(line_increment) => { + self.apply_line_advance(line_increment); + false + } + + LineInstruction::SetFile(file) => { + self.file = file; + false + } + + LineInstruction::SetColumn(column) => { + self.column = column; + false + } + + LineInstruction::NegateStatement => { + self.is_stmt = !self.is_stmt; + false + } + + LineInstruction::SetBasicBlock => { + self.basic_block = true; + false + } + + LineInstruction::ConstAddPc => { + let adjusted = self.adjust_opcode(255, program.header()); + let operation_advance = adjusted / program.header().line_encoding.line_range; + self.apply_operation_advance(u64::from(operation_advance), program.header()); + false + } + + LineInstruction::FixedAddPc(operand) => { + self.address += Wrapping(u64::from(operand)); + self.op_index.0 = 0; + false + } + + LineInstruction::SetPrologueEnd => { + self.prologue_end = true; + false + } + + LineInstruction::SetEpilogueBegin => { + self.epilogue_begin = true; + false + } + + LineInstruction::SetIsa(isa) => { + self.isa = isa; + false + } + + LineInstruction::EndSequence => { + self.end_sequence = true; + true + } + + LineInstruction::SetAddress(address) => { + self.address.0 = address; + self.op_index.0 = 0; + false + } + + LineInstruction::DefineFile(entry) => { + program.add_file(entry); + false + } + + LineInstruction::SetDiscriminator(discriminator) => { + self.discriminator = discriminator; + false + } + + // Compatibility with future opcodes. + LineInstruction::UnknownStandard0(_) + | LineInstruction::UnknownStandard1(_, _) + | LineInstruction::UnknownStandardN(_, _) + | LineInstruction::UnknownExtended(_, _) => false, + } + } + + /// Perform any reset that was required after copying the previous row. + #[inline] + pub fn reset(&mut self, header: &LineProgramHeader) { + if self.end_sequence { + // Previous instruction was EndSequence, so reset everything + // as specified in Section 6.2.5.3. + *self = Self::new(header); + } else { + // Previous instruction was one of: + // - Special - specified in Section 6.2.5.1, steps 4-7 + // - Copy - specified in Section 6.2.5.2 + // The reset behaviour is the same in both cases. + self.discriminator = 0; + self.basic_block = false; + self.prologue_end = false; + self.epilogue_begin = false; + } + } + + /// Step 1 of section 6.2.5.1 + fn apply_line_advance(&mut self, line_increment: i64) { + if line_increment < 0 { + let decrement = -line_increment as u64; + if decrement <= self.line.0 { + self.line.0 -= decrement; + } else { + self.line.0 = 0; + } + } else { + self.line += Wrapping(line_increment as u64); + } + } + + /// Step 2 of section 6.2.5.1 + fn apply_operation_advance( + &mut self, + operation_advance: u64, + header: &LineProgramHeader, + ) { + let operation_advance = Wrapping(operation_advance); + + let minimum_instruction_length = u64::from(header.line_encoding.minimum_instruction_length); + let minimum_instruction_length = Wrapping(minimum_instruction_length); + + let maximum_operations_per_instruction = + u64::from(header.line_encoding.maximum_operations_per_instruction); + let maximum_operations_per_instruction = Wrapping(maximum_operations_per_instruction); + + if maximum_operations_per_instruction.0 == 1 { + self.address += minimum_instruction_length * operation_advance; + self.op_index.0 = 0; + } else { + let op_index_with_advance = self.op_index + operation_advance; + self.address += minimum_instruction_length + * (op_index_with_advance / maximum_operations_per_instruction); + self.op_index = op_index_with_advance % maximum_operations_per_instruction; + } + } + + #[inline] + fn adjust_opcode(&self, opcode: u8, header: &LineProgramHeader) -> u8 { + opcode - header.opcode_base + } + + /// Section 6.2.5.1 + fn exec_special_opcode(&mut self, opcode: u8, header: &LineProgramHeader) { + let adjusted_opcode = self.adjust_opcode(opcode, header); + + let line_range = header.line_encoding.line_range; + let line_advance = adjusted_opcode % line_range; + let operation_advance = adjusted_opcode / line_range; + + // Step 1 + let line_base = i64::from(header.line_encoding.line_base); + self.apply_line_advance(line_base + i64::from(line_advance)); + + // Step 2 + self.apply_operation_advance(u64::from(operation_advance), header); + } +} + +/// The type of column that a row is referring to. +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum ColumnType { + /// The `LeftEdge` means that the statement begins at the start of the new + /// line. + LeftEdge, + /// A column number, whose range begins at 1. + Column(NonZeroU64), +} + +/// Deprecated. `LineNumberSequence` has been renamed to `LineSequence`. +#[deprecated(note = "LineNumberSequence has been renamed to LineSequence, use that instead.")] +pub type LineNumberSequence = LineSequence; + +/// A sequence within a line number program. A sequence, as defined in section +/// 6.2.5 of the standard, is a linear subset of a line number program within +/// which addresses are monotonically increasing. +#[derive(Clone, Debug)] +pub struct LineSequence { + /// The first address that is covered by this sequence within the line number + /// program. + pub start: u64, + /// The first address that is *not* covered by this sequence within the line + /// number program. + pub end: u64, + instructions: LineInstructions, +} + +/// Deprecated. `LineNumberProgramHeader` has been renamed to `LineProgramHeader`. +#[deprecated( + note = "LineNumberProgramHeader has been renamed to LineProgramHeader, use that instead." +)] +pub type LineNumberProgramHeader = LineProgramHeader; + +/// A header for a line number program in the `.debug_line` section, as defined +/// in section 6.2.4 of the standard. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LineProgramHeader::Offset> +where + R: Reader, + Offset: ReaderOffset, +{ + encoding: Encoding, + offset: DebugLineOffset, + unit_length: Offset, + + header_length: Offset, + + line_encoding: LineEncoding, + + /// "The number assigned to the first special opcode." + opcode_base: u8, + + /// "This array specifies the number of LEB128 operands for each of the + /// standard opcodes. The first element of the array corresponds to the + /// opcode whose value is 1, and the last element corresponds to the opcode + /// whose value is `opcode_base - 1`." + standard_opcode_lengths: R, + + /// "A sequence of directory entry format descriptions." + directory_entry_format: Vec, + + /// > Entries in this sequence describe each path that was searched for + /// > included source files in this compilation. (The paths include those + /// > directories specified explicitly by the user for the compiler to search + /// > and those the compiler searches without explicit direction.) Each path + /// > entry is either a full path name or is relative to the current directory + /// > of the compilation. + /// > + /// > The last entry is followed by a single null byte. + include_directories: Vec>, + + /// "A sequence of file entry format descriptions." + file_name_entry_format: Vec, + + /// "Entries in this sequence describe source files that contribute to the + /// line number information for this compilation unit or is used in other + /// contexts." + file_names: Vec>, + + /// The encoded line program instructions. + program_buf: R, + + /// The current directory of the compilation. + comp_dir: Option, + + /// The primary source file. + comp_file: Option>, +} + +impl LineProgramHeader +where + R: Reader, + Offset: ReaderOffset, +{ + /// Return the offset of the line number program header in the `.debug_line` section. + pub fn offset(&self) -> DebugLineOffset { + self.offset + } + + /// Return the length of the line number program and header, not including + /// the length of the encoded length itself. + pub fn unit_length(&self) -> R::Offset { + self.unit_length + } + + /// Return the encoding parameters for this header's line program. + pub fn encoding(&self) -> Encoding { + self.encoding + } + + /// Get the version of this header's line program. + pub fn version(&self) -> u16 { + self.encoding.version + } + + /// Get the length of the encoded line number program header, not including + /// the length of the encoded length itself. + pub fn header_length(&self) -> R::Offset { + self.header_length + } + + /// Get the size in bytes of a target machine address. + pub fn address_size(&self) -> u8 { + self.encoding.address_size + } + + /// Whether this line program is encoded in 64- or 32-bit DWARF. + pub fn format(&self) -> Format { + self.encoding.format + } + + /// Get the line encoding parameters for this header's line program. + pub fn line_encoding(&self) -> LineEncoding { + self.line_encoding + } + + /// Get the minimum instruction length any instruction in this header's line + /// program may have. + pub fn minimum_instruction_length(&self) -> u8 { + self.line_encoding.minimum_instruction_length + } + + /// Get the maximum number of operations each instruction in this header's + /// line program may have. + pub fn maximum_operations_per_instruction(&self) -> u8 { + self.line_encoding.maximum_operations_per_instruction + } + + /// Get the default value of the `is_stmt` register for this header's line + /// program. + pub fn default_is_stmt(&self) -> bool { + self.line_encoding.default_is_stmt + } + + /// Get the line base for this header's line program. + pub fn line_base(&self) -> i8 { + self.line_encoding.line_base + } + + /// Get the line range for this header's line program. + pub fn line_range(&self) -> u8 { + self.line_encoding.line_range + } + + /// Get opcode base for this header's line program. + pub fn opcode_base(&self) -> u8 { + self.opcode_base + } + + /// An array of `u8` that specifies the number of LEB128 operands for + /// each of the standard opcodes. + pub fn standard_opcode_lengths(&self) -> &R { + &self.standard_opcode_lengths + } + + /// Get the format of a directory entry. + pub fn directory_entry_format(&self) -> &[FileEntryFormat] { + &self.directory_entry_format[..] + } + + /// Get the set of include directories for this header's line program. + /// + /// For DWARF version <= 4, the compilation's current directory is not included + /// in the return value, but is implicitly considered to be in the set per spec. + pub fn include_directories(&self) -> &[AttributeValue] { + &self.include_directories[..] + } + + /// The include directory with the given directory index. + /// + /// A directory index of 0 corresponds to the compilation unit directory. + pub fn directory(&self, directory: u64) -> Option> { + if self.encoding.version <= 4 { + if directory == 0 { + self.comp_dir.clone().map(AttributeValue::String) + } else { + let directory = directory as usize - 1; + self.include_directories.get(directory).cloned() + } + } else { + self.include_directories.get(directory as usize).cloned() + } + } + + /// Get the format of a file name entry. + pub fn file_name_entry_format(&self) -> &[FileEntryFormat] { + &self.file_name_entry_format[..] + } + + /// Return true if the file entries may have valid timestamps. + /// + /// Only returns false if we definitely know that all timestamp fields + /// are invalid. + pub fn file_has_timestamp(&self) -> bool { + self.encoding.version <= 4 + || self + .file_name_entry_format + .iter() + .any(|x| x.content_type == constants::DW_LNCT_timestamp) + } + + /// Return true if the file entries may have valid sizes. + /// + /// Only returns false if we definitely know that all size fields + /// are invalid. + pub fn file_has_size(&self) -> bool { + self.encoding.version <= 4 + || self + .file_name_entry_format + .iter() + .any(|x| x.content_type == constants::DW_LNCT_size) + } + + /// Return true if the file name entry format contains an MD5 field. + pub fn file_has_md5(&self) -> bool { + self.file_name_entry_format + .iter() + .any(|x| x.content_type == constants::DW_LNCT_MD5) + } + + /// Get the list of source files that appear in this header's line program. + pub fn file_names(&self) -> &[FileEntry] { + &self.file_names[..] + } + + /// The source file with the given file index. + /// + /// A file index of 0 corresponds to the compilation unit file. + /// Note that a file index of 0 is invalid for DWARF version <= 4, + /// but we support it anyway. + pub fn file(&self, file: u64) -> Option<&FileEntry> { + if self.encoding.version <= 4 { + if file == 0 { + self.comp_file.as_ref() + } else { + let file = file as usize - 1; + self.file_names.get(file) + } + } else { + self.file_names.get(file as usize) + } + } + + /// Get the raw, un-parsed `EndianSlice` containing this header's line number + /// program. + /// + /// ``` + /// # fn foo() { + /// use gimli::{LineProgramHeader, EndianSlice, NativeEndian}; + /// + /// fn get_line_number_program_header<'a>() -> LineProgramHeader> { + /// // Get a line number program header from some offset in a + /// // `.debug_line` section... + /// # unimplemented!() + /// } + /// + /// let header = get_line_number_program_header(); + /// let raw_program = header.raw_program_buf(); + /// println!("The length of the raw program in bytes is {}", raw_program.len()); + /// # } + /// ``` + pub fn raw_program_buf(&self) -> R { + self.program_buf.clone() + } + + /// Iterate over the instructions in this header's line number program, parsing + /// them as we go. + pub fn instructions(&self) -> LineInstructions { + LineInstructions { + input: self.program_buf.clone(), + } + } + + fn parse( + input: &mut R, + offset: DebugLineOffset, + mut address_size: u8, + mut comp_dir: Option, + comp_name: Option, + ) -> Result> { + let (unit_length, format) = input.read_initial_length()?; + let rest = &mut input.split(unit_length)?; + + let version = rest.read_u16()?; + if version < 2 || version > 5 { + return Err(Error::UnknownVersion(u64::from(version))); + } + + if version >= 5 { + address_size = rest.read_u8()?; + let segment_selector_size = rest.read_u8()?; + if segment_selector_size != 0 { + return Err(Error::UnsupportedSegmentSize); + } + } + + let encoding = Encoding { + format, + version, + address_size, + }; + + let header_length = rest.read_length(format)?; + + let mut program_buf = rest.clone(); + program_buf.skip(header_length)?; + rest.truncate(header_length)?; + + let minimum_instruction_length = rest.read_u8()?; + if minimum_instruction_length == 0 { + return Err(Error::MinimumInstructionLengthZero); + } + + // This field did not exist before DWARF 4, but is specified to be 1 for + // non-VLIW architectures, which makes it a no-op. + let maximum_operations_per_instruction = if version >= 4 { rest.read_u8()? } else { 1 }; + if maximum_operations_per_instruction == 0 { + return Err(Error::MaximumOperationsPerInstructionZero); + } + + let default_is_stmt = rest.read_u8()? != 0; + let line_base = rest.read_i8()?; + let line_range = rest.read_u8()?; + if line_range == 0 { + return Err(Error::LineRangeZero); + } + let line_encoding = LineEncoding { + minimum_instruction_length, + maximum_operations_per_instruction, + default_is_stmt, + line_base, + line_range, + }; + + let opcode_base = rest.read_u8()?; + if opcode_base == 0 { + return Err(Error::OpcodeBaseZero); + } + + let standard_opcode_count = R::Offset::from_u8(opcode_base - 1); + let standard_opcode_lengths = rest.split(standard_opcode_count)?; + + let directory_entry_format; + let mut include_directories = Vec::new(); + if version <= 4 { + directory_entry_format = Vec::new(); + loop { + let directory = rest.read_null_terminated_slice()?; + if directory.is_empty() { + break; + } + include_directories.push(AttributeValue::String(directory)); + } + } else { + comp_dir = None; + directory_entry_format = FileEntryFormat::parse(rest)?; + let count = rest.read_uleb128()?; + for _ in 0..count { + include_directories.push(parse_directory_v5( + rest, + encoding, + &directory_entry_format, + )?); + } + } + + let comp_file; + let file_name_entry_format; + let mut file_names = Vec::new(); + if version <= 4 { + comp_file = comp_name.map(|name| FileEntry { + path_name: AttributeValue::String(name), + directory_index: 0, + timestamp: 0, + size: 0, + md5: [0; 16], + }); + + file_name_entry_format = Vec::new(); + loop { + let path_name = rest.read_null_terminated_slice()?; + if path_name.is_empty() { + break; + } + file_names.push(FileEntry::parse(rest, path_name)?); + } + } else { + comp_file = None; + file_name_entry_format = FileEntryFormat::parse(rest)?; + let count = rest.read_uleb128()?; + for _ in 0..count { + file_names.push(parse_file_v5(rest, encoding, &file_name_entry_format)?); + } + } + + let header = LineProgramHeader { + encoding, + offset, + unit_length, + header_length, + line_encoding, + opcode_base, + standard_opcode_lengths, + directory_entry_format, + include_directories, + file_name_entry_format, + file_names, + program_buf, + comp_dir, + comp_file, + }; + Ok(header) + } +} + +/// Deprecated. `IncompleteLineNumberProgram` has been renamed to `IncompleteLineProgram`. +#[deprecated( + note = "IncompleteLineNumberProgram has been renamed to IncompleteLineProgram, use that instead." +)] +pub type IncompleteLineNumberProgram = IncompleteLineProgram; + +/// A line number program that has not been run to completion. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct IncompleteLineProgram::Offset> +where + R: Reader, + Offset: ReaderOffset, +{ + header: LineProgramHeader, +} + +impl IncompleteLineProgram +where + R: Reader, + Offset: ReaderOffset, +{ + /// Retrieve the `LineProgramHeader` for this program. + pub fn header(&self) -> &LineProgramHeader { + &self.header + } + + /// Construct a new `LineRows` for executing this program to iterate + /// over rows in the line information matrix. + pub fn rows(self) -> OneShotLineRows { + OneShotLineRows::new(self) + } + + /// Execute the line number program, completing the `IncompleteLineProgram` + /// into a `CompleteLineProgram` and producing an array of sequences within + /// the line number program that can later be used with + /// `CompleteLineProgram::resume_from`. + /// + /// ``` + /// # fn foo() { + /// use gimli::{IncompleteLineProgram, EndianSlice, NativeEndian}; + /// + /// fn get_line_number_program<'a>() -> IncompleteLineProgram> { + /// // Get a line number program from some offset in a + /// // `.debug_line` section... + /// # unimplemented!() + /// } + /// + /// let program = get_line_number_program(); + /// let (program, sequences) = program.sequences().unwrap(); + /// println!("There are {} sequences in this line number program", sequences.len()); + /// # } + /// ``` + #[allow(clippy::type_complexity)] + pub fn sequences(self) -> Result<(CompleteLineProgram, Vec>)> { + let mut sequences = Vec::new(); + let mut rows = self.rows(); + let mut instructions = rows.instructions.clone(); + let mut sequence_start_addr = None; + loop { + let sequence_end_addr; + if rows.next_row()?.is_none() { + break; + } + + let row = &rows.row; + if row.end_sequence() { + sequence_end_addr = row.address(); + } else if sequence_start_addr.is_none() { + sequence_start_addr = Some(row.address()); + continue; + } else { + continue; + } + + // We just finished a sequence. + sequences.push(LineSequence { + // In theory one could have multiple DW_LNE_end_sequence instructions + // in a row. + start: sequence_start_addr.unwrap_or(0), + end: sequence_end_addr, + instructions: instructions.remove_trailing(&rows.instructions)?, + }); + sequence_start_addr = None; + instructions = rows.instructions.clone(); + } + + let program = CompleteLineProgram { + header: rows.program.header, + }; + Ok((program, sequences)) + } +} + +/// Deprecated. `CompleteLineNumberProgram` has been renamed to `CompleteLineProgram`. +#[deprecated( + note = "CompleteLineNumberProgram has been renamed to CompleteLineProgram, use that instead." +)] +pub type CompleteLineNumberProgram = CompleteLineProgram; + +/// A line number program that has previously been run to completion. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CompleteLineProgram::Offset> +where + R: Reader, + Offset: ReaderOffset, +{ + header: LineProgramHeader, +} + +impl CompleteLineProgram +where + R: Reader, + Offset: ReaderOffset, +{ + /// Retrieve the `LineProgramHeader` for this program. + pub fn header(&self) -> &LineProgramHeader { + &self.header + } + + /// Construct a new `LineRows` for executing the subset of the line + /// number program identified by 'sequence' and generating the line information + /// matrix. + /// + /// ``` + /// # fn foo() { + /// use gimli::{IncompleteLineProgram, EndianSlice, NativeEndian}; + /// + /// fn get_line_number_program<'a>() -> IncompleteLineProgram> { + /// // Get a line number program from some offset in a + /// // `.debug_line` section... + /// # unimplemented!() + /// } + /// + /// let program = get_line_number_program(); + /// let (program, sequences) = program.sequences().unwrap(); + /// for sequence in &sequences { + /// let mut sm = program.resume_from(sequence); + /// } + /// # } + /// ``` + pub fn resume_from<'program>( + &'program self, + sequence: &LineSequence, + ) -> ResumedLineRows<'program, R, Offset> { + ResumedLineRows::resume(self, sequence) + } +} + +/// An entry in the `LineProgramHeader`'s `file_names` set. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub struct FileEntry::Offset> +where + R: Reader, + Offset: ReaderOffset, +{ + path_name: AttributeValue, + directory_index: u64, + timestamp: u64, + size: u64, + md5: [u8; 16], +} + +impl FileEntry +where + R: Reader, + Offset: ReaderOffset, +{ + // version 2-4 + fn parse(input: &mut R, path_name: R) -> Result> { + let directory_index = input.read_uleb128()?; + let timestamp = input.read_uleb128()?; + let size = input.read_uleb128()?; + + let entry = FileEntry { + path_name: AttributeValue::String(path_name), + directory_index, + timestamp, + size, + md5: [0; 16], + }; + + Ok(entry) + } + + /// > A slice containing the full or relative path name of + /// > a source file. If the entry contains a file name or a relative path + /// > name, the file is located relative to either the compilation directory + /// > (as specified by the DW_AT_comp_dir attribute given in the compilation + /// > unit) or one of the directories in the include_directories section. + pub fn path_name(&self) -> AttributeValue { + self.path_name.clone() + } + + /// > An unsigned LEB128 number representing the directory index of the + /// > directory in which the file was found. + /// > + /// > ... + /// > + /// > The directory index represents an entry in the include_directories + /// > section of the line number program header. The index is 0 if the file + /// > was found in the current directory of the compilation, 1 if it was found + /// > in the first directory in the include_directories section, and so + /// > on. The directory index is ignored for file names that represent full + /// > path names. + pub fn directory_index(&self) -> u64 { + self.directory_index + } + + /// Get this file's directory. + /// + /// A directory index of 0 corresponds to the compilation unit directory. + pub fn directory(&self, header: &LineProgramHeader) -> Option> { + header.directory(self.directory_index) + } + + /// The implementation-defined time of last modification of the file, + /// or 0 if not available. + pub fn timestamp(&self) -> u64 { + self.timestamp + } + + /// "An unsigned LEB128 number representing the time of last modification of + /// the file, or 0 if not available." + // Terminology changed in DWARF version 5. + #[doc(hidden)] + pub fn last_modification(&self) -> u64 { + self.timestamp + } + + /// The size of the file in bytes, or 0 if not available. + pub fn size(&self) -> u64 { + self.size + } + + /// "An unsigned LEB128 number representing the length in bytes of the file, + /// or 0 if not available." + // Terminology changed in DWARF version 5. + #[doc(hidden)] + pub fn length(&self) -> u64 { + self.size + } + + /// A 16-byte MD5 digest of the file contents. + /// + /// Only valid if `LineProgramHeader::file_has_md5` returns `true`. + pub fn md5(&self) -> &[u8; 16] { + &self.md5 + } +} + +/// The format of a component of an include directory or file name entry. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub struct FileEntryFormat { + /// The type of information that is represented by the component. + pub content_type: constants::DwLnct, + + /// The encoding form of the component value. + pub form: constants::DwForm, +} + +impl FileEntryFormat { + fn parse(input: &mut R) -> Result> { + let format_count = input.read_u8()? as usize; + let mut format = Vec::with_capacity(format_count); + let mut path_count = 0; + for _ in 0..format_count { + let content_type = input.read_uleb128()?; + let content_type = if content_type > u64::from(u16::max_value()) { + constants::DwLnct(u16::max_value()) + } else { + constants::DwLnct(content_type as u16) + }; + if content_type == constants::DW_LNCT_path { + path_count += 1; + } + + let form = constants::DwForm(input.read_uleb128_u16()?); + + format.push(FileEntryFormat { content_type, form }); + } + if path_count != 1 { + return Err(Error::MissingFileEntryFormatPath); + } + Ok(format) + } +} + +fn parse_directory_v5( + input: &mut R, + encoding: Encoding, + formats: &[FileEntryFormat], +) -> Result> { + let mut path_name = None; + + for format in formats { + let value = parse_attribute(input, encoding, format.form)?; + if format.content_type == constants::DW_LNCT_path { + path_name = Some(value); + } + } + + Ok(path_name.unwrap()) +} + +fn parse_file_v5( + input: &mut R, + encoding: Encoding, + formats: &[FileEntryFormat], +) -> Result> { + let mut path_name = None; + let mut directory_index = 0; + let mut timestamp = 0; + let mut size = 0; + let mut md5 = [0; 16]; + + for format in formats { + let value = parse_attribute(input, encoding, format.form)?; + match format.content_type { + constants::DW_LNCT_path => path_name = Some(value), + constants::DW_LNCT_directory_index => { + if let Some(value) = value.udata_value() { + directory_index = value; + } + } + constants::DW_LNCT_timestamp => { + if let Some(value) = value.udata_value() { + timestamp = value; + } + } + constants::DW_LNCT_size => { + if let Some(value) = value.udata_value() { + size = value; + } + } + constants::DW_LNCT_MD5 => { + if let AttributeValue::Block(mut value) = value { + if value.len().into_u64() == 16 { + md5 = value.read_u8_array()?; + } + } + } + // Ignore unknown content types. + _ => {} + } + } + + Ok(FileEntry { + path_name: path_name.unwrap(), + directory_index, + timestamp, + size, + md5, + }) +} + +// TODO: this should be shared with unit::parse_attribute(), but that is hard to do. +fn parse_attribute( + input: &mut R, + encoding: Encoding, + form: constants::DwForm, +) -> Result> { + Ok(match form { + constants::DW_FORM_block1 => { + let len = input.read_u8().map(R::Offset::from_u8)?; + let block = input.split(len)?; + AttributeValue::Block(block) + } + constants::DW_FORM_block2 => { + let len = input.read_u16().map(R::Offset::from_u16)?; + let block = input.split(len)?; + AttributeValue::Block(block) + } + constants::DW_FORM_block4 => { + let len = input.read_u32().map(R::Offset::from_u32)?; + let block = input.split(len)?; + AttributeValue::Block(block) + } + constants::DW_FORM_block => { + let len = input.read_uleb128().and_then(R::Offset::from_u64)?; + let block = input.split(len)?; + AttributeValue::Block(block) + } + constants::DW_FORM_data1 => { + let data = input.read_u8()?; + AttributeValue::Data1(data) + } + constants::DW_FORM_data2 => { + let data = input.read_u16()?; + AttributeValue::Data2(data) + } + constants::DW_FORM_data4 => { + let data = input.read_u32()?; + AttributeValue::Data4(data) + } + constants::DW_FORM_data8 => { + let data = input.read_u64()?; + AttributeValue::Data8(data) + } + constants::DW_FORM_data16 => { + let block = input.split(R::Offset::from_u8(16))?; + AttributeValue::Block(block) + } + constants::DW_FORM_udata => { + let data = input.read_uleb128()?; + AttributeValue::Udata(data) + } + constants::DW_FORM_sdata => { + let data = input.read_sleb128()?; + AttributeValue::Sdata(data) + } + constants::DW_FORM_flag => { + let present = input.read_u8()?; + AttributeValue::Flag(present != 0) + } + constants::DW_FORM_sec_offset => { + let offset = input.read_offset(encoding.format)?; + AttributeValue::SecOffset(offset) + } + constants::DW_FORM_string => { + let string = input.read_null_terminated_slice()?; + AttributeValue::String(string) + } + constants::DW_FORM_strp => { + let offset = input.read_offset(encoding.format)?; + AttributeValue::DebugStrRef(DebugStrOffset(offset)) + } + constants::DW_FORM_strp_sup | constants::DW_FORM_GNU_strp_alt => { + let offset = input.read_offset(encoding.format)?; + AttributeValue::DebugStrRefSup(DebugStrOffset(offset)) + } + constants::DW_FORM_line_strp => { + let offset = input.read_offset(encoding.format)?; + AttributeValue::DebugLineStrRef(DebugLineStrOffset(offset)) + } + constants::DW_FORM_strx | constants::DW_FORM_GNU_str_index => { + let index = input.read_uleb128().and_then(R::Offset::from_u64)?; + AttributeValue::DebugStrOffsetsIndex(DebugStrOffsetsIndex(index)) + } + constants::DW_FORM_strx1 => { + let index = input.read_u8().map(R::Offset::from_u8)?; + AttributeValue::DebugStrOffsetsIndex(DebugStrOffsetsIndex(index)) + } + constants::DW_FORM_strx2 => { + let index = input.read_u16().map(R::Offset::from_u16)?; + AttributeValue::DebugStrOffsetsIndex(DebugStrOffsetsIndex(index)) + } + constants::DW_FORM_strx3 => { + let index = input.read_uint(3).and_then(R::Offset::from_u64)?; + AttributeValue::DebugStrOffsetsIndex(DebugStrOffsetsIndex(index)) + } + constants::DW_FORM_strx4 => { + let index = input.read_u32().map(R::Offset::from_u32)?; + AttributeValue::DebugStrOffsetsIndex(DebugStrOffsetsIndex(index)) + } + _ => { + return Err(Error::UnknownForm); + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::constants; + use crate::endianity::LittleEndian; + use crate::read::{EndianSlice, Error}; + use crate::test_util::GimliSectionMethods; + use core::u64; + use core::u8; + use test_assembler::{Endian, Label, LabelMaker, Section}; + + #[test] + fn test_parse_debug_line_32_ok() { + #[rustfmt::skip] + let buf = [ + // 32-bit length = 62. + 0x3e, 0x00, 0x00, 0x00, + // Version. + 0x04, 0x00, + // Header length = 40. + 0x28, 0x00, 0x00, 0x00, + // Minimum instruction length. + 0x01, + // Maximum operations per byte. + 0x01, + // Default is_stmt. + 0x01, + // Line base. + 0x00, + // Line range. + 0x01, + // Opcode base. + 0x03, + // Standard opcode lengths for opcodes 1 .. opcode base - 1. + 0x01, 0x02, + // Include directories = '/', 'i', 'n', 'c', '\0', '/', 'i', 'n', 'c', '2', '\0', '\0' + 0x2f, 0x69, 0x6e, 0x63, 0x00, 0x2f, 0x69, 0x6e, 0x63, 0x32, 0x00, 0x00, + // File names + // foo.rs + 0x66, 0x6f, 0x6f, 0x2e, 0x72, 0x73, 0x00, + 0x00, + 0x00, + 0x00, + // bar.h + 0x62, 0x61, 0x72, 0x2e, 0x68, 0x00, + 0x01, + 0x00, + 0x00, + // End file names. + 0x00, + + // Dummy line program data. + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + + // Dummy next line program. + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ]; + + let rest = &mut EndianSlice::new(&buf, LittleEndian); + let comp_dir = EndianSlice::new(b"/comp_dir", LittleEndian); + let comp_name = EndianSlice::new(b"/comp_name", LittleEndian); + + let header = + LineProgramHeader::parse(rest, DebugLineOffset(0), 4, Some(comp_dir), Some(comp_name)) + .expect("should parse header ok"); + + assert_eq!( + *rest, + EndianSlice::new(&buf[buf.len() - 16..], LittleEndian) + ); + + assert_eq!(header.offset, DebugLineOffset(0)); + assert_eq!(header.version(), 4); + assert_eq!(header.minimum_instruction_length(), 1); + assert_eq!(header.maximum_operations_per_instruction(), 1); + assert_eq!(header.default_is_stmt(), true); + assert_eq!(header.line_base(), 0); + assert_eq!(header.line_range(), 1); + assert_eq!(header.opcode_base(), 3); + assert_eq!(header.directory(0), Some(AttributeValue::String(comp_dir))); + assert_eq!( + header.file(0).unwrap().path_name, + AttributeValue::String(comp_name) + ); + + let expected_lengths = [1, 2]; + assert_eq!(header.standard_opcode_lengths().slice(), &expected_lengths); + + let expected_include_directories = [ + AttributeValue::String(EndianSlice::new(b"/inc", LittleEndian)), + AttributeValue::String(EndianSlice::new(b"/inc2", LittleEndian)), + ]; + assert_eq!(header.include_directories(), &expected_include_directories); + + let expected_file_names = [ + FileEntry { + path_name: AttributeValue::String(EndianSlice::new(b"foo.rs", LittleEndian)), + directory_index: 0, + timestamp: 0, + size: 0, + md5: [0; 16], + }, + FileEntry { + path_name: AttributeValue::String(EndianSlice::new(b"bar.h", LittleEndian)), + directory_index: 1, + timestamp: 0, + size: 0, + md5: [0; 16], + }, + ]; + assert_eq!(&*header.file_names(), &expected_file_names); + } + + #[test] + fn test_parse_debug_line_header_length_too_short() { + #[rustfmt::skip] + let buf = [ + // 32-bit length = 62. + 0x3e, 0x00, 0x00, 0x00, + // Version. + 0x04, 0x00, + // Header length = 20. TOO SHORT!!! + 0x15, 0x00, 0x00, 0x00, + // Minimum instruction length. + 0x01, + // Maximum operations per byte. + 0x01, + // Default is_stmt. + 0x01, + // Line base. + 0x00, + // Line range. + 0x01, + // Opcode base. + 0x03, + // Standard opcode lengths for opcodes 1 .. opcode base - 1. + 0x01, 0x02, + // Include directories = '/', 'i', 'n', 'c', '\0', '/', 'i', 'n', 'c', '2', '\0', '\0' + 0x2f, 0x69, 0x6e, 0x63, 0x00, 0x2f, 0x69, 0x6e, 0x63, 0x32, 0x00, 0x00, + // File names + // foo.rs + 0x66, 0x6f, 0x6f, 0x2e, 0x72, 0x73, 0x00, + 0x00, + 0x00, + 0x00, + // bar.h + 0x62, 0x61, 0x72, 0x2e, 0x68, 0x00, + 0x01, + 0x00, + 0x00, + // End file names. + 0x00, + + // Dummy line program data. + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + + // Dummy next line program. + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ]; + + let input = &mut EndianSlice::new(&buf, LittleEndian); + + match LineProgramHeader::parse(input, DebugLineOffset(0), 4, None, None) { + Err(Error::UnexpectedEof(_)) => return, + otherwise => panic!("Unexpected result: {:?}", otherwise), + } + } + + #[test] + fn test_parse_debug_line_unit_length_too_short() { + #[rustfmt::skip] + let buf = [ + // 32-bit length = 40. TOO SHORT!!! + 0x28, 0x00, 0x00, 0x00, + // Version. + 0x04, 0x00, + // Header length = 40. + 0x28, 0x00, 0x00, 0x00, + // Minimum instruction length. + 0x01, + // Maximum operations per byte. + 0x01, + // Default is_stmt. + 0x01, + // Line base. + 0x00, + // Line range. + 0x01, + // Opcode base. + 0x03, + // Standard opcode lengths for opcodes 1 .. opcode base - 1. + 0x01, 0x02, + // Include directories = '/', 'i', 'n', 'c', '\0', '/', 'i', 'n', 'c', '2', '\0', '\0' + 0x2f, 0x69, 0x6e, 0x63, 0x00, 0x2f, 0x69, 0x6e, 0x63, 0x32, 0x00, 0x00, + // File names + // foo.rs + 0x66, 0x6f, 0x6f, 0x2e, 0x72, 0x73, 0x00, + 0x00, + 0x00, + 0x00, + // bar.h + 0x62, 0x61, 0x72, 0x2e, 0x68, 0x00, + 0x01, + 0x00, + 0x00, + // End file names. + 0x00, + + // Dummy line program data. + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + + // Dummy next line program. + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ]; + + let input = &mut EndianSlice::new(&buf, LittleEndian); + + match LineProgramHeader::parse(input, DebugLineOffset(0), 4, None, None) { + Err(Error::UnexpectedEof(_)) => return, + otherwise => panic!("Unexpected result: {:?}", otherwise), + } + } + + const OPCODE_BASE: u8 = 13; + const STANDARD_OPCODE_LENGTHS: &[u8] = &[0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1]; + + fn make_test_header( + buf: EndianSlice, + ) -> LineProgramHeader> { + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 8, + }; + let line_encoding = LineEncoding { + line_base: -3, + line_range: 12, + ..Default::default() + }; + LineProgramHeader { + encoding, + offset: DebugLineOffset(0), + unit_length: 1, + header_length: 1, + line_encoding, + opcode_base: OPCODE_BASE, + standard_opcode_lengths: EndianSlice::new(STANDARD_OPCODE_LENGTHS, LittleEndian), + file_names: vec![ + FileEntry { + path_name: AttributeValue::String(EndianSlice::new(b"foo.c", LittleEndian)), + directory_index: 0, + timestamp: 0, + size: 0, + md5: [0; 16], + }, + FileEntry { + path_name: AttributeValue::String(EndianSlice::new(b"bar.rs", LittleEndian)), + directory_index: 0, + timestamp: 0, + size: 0, + md5: [0; 16], + }, + ], + include_directories: vec![], + directory_entry_format: vec![], + file_name_entry_format: vec![], + program_buf: buf, + comp_dir: None, + comp_file: None, + } + } + + fn make_test_program( + buf: EndianSlice, + ) -> IncompleteLineProgram> { + IncompleteLineProgram { + header: make_test_header(buf), + } + } + + #[test] + fn test_parse_special_opcodes() { + for i in OPCODE_BASE..u8::MAX { + let input = [i, 0, 0, 0]; + let input = EndianSlice::new(&input, LittleEndian); + let header = make_test_header(input); + + let mut rest = input; + let opcode = + LineInstruction::parse(&header, &mut rest).expect("Should parse the opcode OK"); + + assert_eq!(*rest, *input.range_from(1..)); + assert_eq!(opcode, LineInstruction::Special(i)); + } + } + + #[test] + fn test_parse_standard_opcodes() { + fn test( + raw: constants::DwLns, + operands: Operands, + expected: LineInstruction>, + ) where + Operands: AsRef<[u8]>, + { + let mut input = Vec::new(); + input.push(raw.0); + input.extend_from_slice(operands.as_ref()); + + let expected_rest = [0, 1, 2, 3, 4]; + input.extend_from_slice(&expected_rest); + + let input = EndianSlice::new(&*input, LittleEndian); + let header = make_test_header(input); + + let mut rest = input; + let opcode = + LineInstruction::parse(&header, &mut rest).expect("Should parse the opcode OK"); + + assert_eq!(opcode, expected); + assert_eq!(*rest, expected_rest); + } + + test(constants::DW_LNS_copy, [], LineInstruction::Copy); + test( + constants::DW_LNS_advance_pc, + [42], + LineInstruction::AdvancePc(42), + ); + test( + constants::DW_LNS_advance_line, + [9], + LineInstruction::AdvanceLine(9), + ); + test(constants::DW_LNS_set_file, [7], LineInstruction::SetFile(7)); + test( + constants::DW_LNS_set_column, + [1], + LineInstruction::SetColumn(1), + ); + test( + constants::DW_LNS_negate_stmt, + [], + LineInstruction::NegateStatement, + ); + test( + constants::DW_LNS_set_basic_block, + [], + LineInstruction::SetBasicBlock, + ); + test( + constants::DW_LNS_const_add_pc, + [], + LineInstruction::ConstAddPc, + ); + test( + constants::DW_LNS_fixed_advance_pc, + [42, 0], + LineInstruction::FixedAddPc(42), + ); + test( + constants::DW_LNS_set_prologue_end, + [], + LineInstruction::SetPrologueEnd, + ); + test( + constants::DW_LNS_set_isa, + [57 + 0x80, 100], + LineInstruction::SetIsa(12857), + ); + } + + #[test] + fn test_parse_unknown_standard_opcode_no_args() { + let input = [OPCODE_BASE, 1, 2, 3]; + let input = EndianSlice::new(&input, LittleEndian); + let mut standard_opcode_lengths = Vec::new(); + let mut header = make_test_header(input); + standard_opcode_lengths.extend(header.standard_opcode_lengths.slice()); + standard_opcode_lengths.push(0); + header.opcode_base += 1; + header.standard_opcode_lengths = EndianSlice::new(&standard_opcode_lengths, LittleEndian); + + let mut rest = input; + let opcode = + LineInstruction::parse(&header, &mut rest).expect("Should parse the opcode OK"); + + assert_eq!( + opcode, + LineInstruction::UnknownStandard0(constants::DwLns(OPCODE_BASE)) + ); + assert_eq!(*rest, *input.range_from(1..)); + } + + #[test] + fn test_parse_unknown_standard_opcode_one_arg() { + let input = [OPCODE_BASE, 1, 2, 3]; + let input = EndianSlice::new(&input, LittleEndian); + let mut standard_opcode_lengths = Vec::new(); + let mut header = make_test_header(input); + standard_opcode_lengths.extend(header.standard_opcode_lengths.slice()); + standard_opcode_lengths.push(1); + header.opcode_base += 1; + header.standard_opcode_lengths = EndianSlice::new(&standard_opcode_lengths, LittleEndian); + + let mut rest = input; + let opcode = + LineInstruction::parse(&header, &mut rest).expect("Should parse the opcode OK"); + + assert_eq!( + opcode, + LineInstruction::UnknownStandard1(constants::DwLns(OPCODE_BASE), 1) + ); + assert_eq!(*rest, *input.range_from(2..)); + } + + #[test] + fn test_parse_unknown_standard_opcode_many_args() { + let input = [OPCODE_BASE, 1, 2, 3]; + let input = EndianSlice::new(&input, LittleEndian); + let args = EndianSlice::new(&input[1..], LittleEndian); + let mut standard_opcode_lengths = Vec::new(); + let mut header = make_test_header(input); + standard_opcode_lengths.extend(header.standard_opcode_lengths.slice()); + standard_opcode_lengths.push(3); + header.opcode_base += 1; + header.standard_opcode_lengths = EndianSlice::new(&standard_opcode_lengths, LittleEndian); + + let mut rest = input; + let opcode = + LineInstruction::parse(&header, &mut rest).expect("Should parse the opcode OK"); + + assert_eq!( + opcode, + LineInstruction::UnknownStandardN(constants::DwLns(OPCODE_BASE), args) + ); + assert_eq!(*rest, []); + } + + #[test] + fn test_parse_extended_opcodes() { + fn test( + raw: constants::DwLne, + operands: Operands, + expected: LineInstruction>, + ) where + Operands: AsRef<[u8]>, + { + let mut input = Vec::new(); + input.push(0); + + let operands = operands.as_ref(); + input.push(1 + operands.len() as u8); + + input.push(raw.0); + input.extend_from_slice(operands); + + let expected_rest = [0, 1, 2, 3, 4]; + input.extend_from_slice(&expected_rest); + + let input = EndianSlice::new(&input, LittleEndian); + let header = make_test_header(input); + + let mut rest = input; + let opcode = + LineInstruction::parse(&header, &mut rest).expect("Should parse the opcode OK"); + + assert_eq!(opcode, expected); + assert_eq!(*rest, expected_rest); + } + + test( + constants::DW_LNE_end_sequence, + [], + LineInstruction::EndSequence, + ); + test( + constants::DW_LNE_set_address, + [1, 2, 3, 4, 5, 6, 7, 8], + LineInstruction::SetAddress(578_437_695_752_307_201), + ); + test( + constants::DW_LNE_set_discriminator, + [42], + LineInstruction::SetDiscriminator(42), + ); + + let mut file = Vec::new(); + // "foo.c" + let path_name = [b'f', b'o', b'o', b'.', b'c', 0]; + file.extend_from_slice(&path_name); + // Directory index. + file.push(0); + // Last modification of file. + file.push(1); + // Size of file. + file.push(2); + + test( + constants::DW_LNE_define_file, + file, + LineInstruction::DefineFile(FileEntry { + path_name: AttributeValue::String(EndianSlice::new(b"foo.c", LittleEndian)), + directory_index: 0, + timestamp: 1, + size: 2, + md5: [0; 16], + }), + ); + + // Unknown extended opcode. + let operands = [1, 2, 3, 4, 5, 6]; + let opcode = constants::DwLne(99); + test( + opcode, + operands, + LineInstruction::UnknownExtended(opcode, EndianSlice::new(&operands, LittleEndian)), + ); + } + + #[test] + fn test_file_entry_directory() { + let path_name = [b'f', b'o', b'o', b'.', b'r', b's', 0]; + + let mut file = FileEntry { + path_name: AttributeValue::String(EndianSlice::new(&path_name, LittleEndian)), + directory_index: 1, + timestamp: 0, + size: 0, + md5: [0; 16], + }; + + let mut header = make_test_header(EndianSlice::new(&[], LittleEndian)); + + let dir = AttributeValue::String(EndianSlice::new(b"dir", LittleEndian)); + header.include_directories.push(dir); + + assert_eq!(file.directory(&header), Some(dir)); + + // Now test the compilation's current directory. + file.directory_index = 0; + assert_eq!(file.directory(&header), None); + } + + fn assert_exec_opcode<'input>( + header: LineProgramHeader>, + mut registers: LineRow, + opcode: LineInstruction>, + expected_registers: LineRow, + expect_new_row: bool, + ) { + let mut program = IncompleteLineProgram { header }; + let is_new_row = registers.execute(opcode, &mut program); + + assert_eq!(is_new_row, expect_new_row); + assert_eq!(registers, expected_registers); + } + + #[test] + fn test_exec_special_noop() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + + let initial_registers = LineRow::new(&header); + let opcode = LineInstruction::Special(16); + let expected_registers = initial_registers; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, true); + } + + #[test] + fn test_exec_special_negative_line_advance() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + + let mut initial_registers = LineRow::new(&header); + initial_registers.line.0 = 10; + + let opcode = LineInstruction::Special(13); + + let mut expected_registers = initial_registers; + expected_registers.line.0 -= 3; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, true); + } + + #[test] + fn test_exec_special_positive_line_advance() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + + let initial_registers = LineRow::new(&header); + + let opcode = LineInstruction::Special(19); + + let mut expected_registers = initial_registers; + expected_registers.line.0 += 3; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, true); + } + + #[test] + fn test_exec_special_positive_address_advance() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + + let initial_registers = LineRow::new(&header); + + let opcode = LineInstruction::Special(52); + + let mut expected_registers = initial_registers; + expected_registers.address.0 += 3; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, true); + } + + #[test] + fn test_exec_special_positive_address_and_line_advance() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + + let initial_registers = LineRow::new(&header); + + let opcode = LineInstruction::Special(55); + + let mut expected_registers = initial_registers; + expected_registers.address.0 += 3; + expected_registers.line.0 += 3; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, true); + } + + #[test] + fn test_exec_special_positive_address_and_negative_line_advance() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + + let mut initial_registers = LineRow::new(&header); + initial_registers.line.0 = 10; + + let opcode = LineInstruction::Special(49); + + let mut expected_registers = initial_registers; + expected_registers.address.0 += 3; + expected_registers.line.0 -= 3; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, true); + } + + #[test] + fn test_exec_special_line_underflow() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + + let mut initial_registers = LineRow::new(&header); + initial_registers.line.0 = 2; + + // -3 line advance. + let opcode = LineInstruction::Special(13); + + let mut expected_registers = initial_registers; + // Clamp at 0. No idea if this is the best way to handle this situation + // or not... + expected_registers.line.0 = 0; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, true); + } + + #[test] + fn test_exec_copy() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + + let mut initial_registers = LineRow::new(&header); + initial_registers.address.0 = 1337; + initial_registers.line.0 = 42; + + let opcode = LineInstruction::Copy; + + let expected_registers = initial_registers; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, true); + } + + #[test] + fn test_exec_advance_pc() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + let initial_registers = LineRow::new(&header); + let opcode = LineInstruction::AdvancePc(42); + + let mut expected_registers = initial_registers; + expected_registers.address.0 += 42; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, false); + } + + #[test] + fn test_exec_advance_pc_overflow() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + let opcode = LineInstruction::AdvancePc(42); + + let mut initial_registers = LineRow::new(&header); + initial_registers.address.0 = u64::MAX; + + let mut expected_registers = initial_registers; + expected_registers.address.0 = 41; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, false); + } + + #[test] + fn test_exec_advance_line() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + let initial_registers = LineRow::new(&header); + let opcode = LineInstruction::AdvanceLine(42); + + let mut expected_registers = initial_registers; + expected_registers.line.0 += 42; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, false); + } + + #[test] + fn test_exec_advance_line_overflow() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + let opcode = LineInstruction::AdvanceLine(42); + + let mut initial_registers = LineRow::new(&header); + initial_registers.line.0 = u64::MAX; + + let mut expected_registers = initial_registers; + expected_registers.line.0 = 41; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, false); + } + + #[test] + fn test_exec_set_file_in_bounds() { + for file_idx in 1..3 { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + let initial_registers = LineRow::new(&header); + let opcode = LineInstruction::SetFile(file_idx); + + let mut expected_registers = initial_registers; + expected_registers.file = file_idx; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, false); + } + } + + #[test] + fn test_exec_set_file_out_of_bounds() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + let initial_registers = LineRow::new(&header); + let opcode = LineInstruction::SetFile(100); + + // The spec doesn't say anything about rejecting input programs + // that set the file register out of bounds of the actual number + // of files that have been defined. Instead, we cross our + // fingers and hope that one gets defined before + // `LineRow::file` gets called and handle the error at + // that time if need be. + let mut expected_registers = initial_registers; + expected_registers.file = 100; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, false); + } + + #[test] + fn test_file_entry_file_index_out_of_bounds() { + // These indices are 1-based, so 0 is invalid. 100 is way more than the + // number of files defined in the header. + let out_of_bounds_indices = [0, 100]; + + for file_idx in &out_of_bounds_indices[..] { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + let mut row = LineRow::new(&header); + + row.file = *file_idx; + + assert_eq!(row.file(&header), None); + } + } + + #[test] + fn test_file_entry_file_index_in_bounds() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + let mut row = LineRow::new(&header); + + row.file = 2; + + assert_eq!(row.file(&header), Some(&header.file_names()[1])); + } + + #[test] + fn test_exec_set_column() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + let initial_registers = LineRow::new(&header); + let opcode = LineInstruction::SetColumn(42); + + let mut expected_registers = initial_registers; + expected_registers.column = 42; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, false); + } + + #[test] + fn test_exec_negate_statement() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + let initial_registers = LineRow::new(&header); + let opcode = LineInstruction::NegateStatement; + + let mut expected_registers = initial_registers; + expected_registers.is_stmt = !initial_registers.is_stmt; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, false); + } + + #[test] + fn test_exec_set_basic_block() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + + let mut initial_registers = LineRow::new(&header); + initial_registers.basic_block = false; + + let opcode = LineInstruction::SetBasicBlock; + + let mut expected_registers = initial_registers; + expected_registers.basic_block = true; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, false); + } + + #[test] + fn test_exec_const_add_pc() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + let initial_registers = LineRow::new(&header); + let opcode = LineInstruction::ConstAddPc; + + let mut expected_registers = initial_registers; + expected_registers.address.0 += 20; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, false); + } + + #[test] + fn test_exec_fixed_add_pc() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + + let mut initial_registers = LineRow::new(&header); + initial_registers.op_index.0 = 1; + + let opcode = LineInstruction::FixedAddPc(10); + + let mut expected_registers = initial_registers; + expected_registers.address.0 += 10; + expected_registers.op_index.0 = 0; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, false); + } + + #[test] + fn test_exec_set_prologue_end() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + + let mut initial_registers = LineRow::new(&header); + initial_registers.prologue_end = false; + + let opcode = LineInstruction::SetPrologueEnd; + + let mut expected_registers = initial_registers; + expected_registers.prologue_end = true; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, false); + } + + #[test] + fn test_exec_set_isa() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + let initial_registers = LineRow::new(&header); + let opcode = LineInstruction::SetIsa(1993); + + let mut expected_registers = initial_registers; + expected_registers.isa = 1993; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, false); + } + + #[test] + fn test_exec_unknown_standard_0() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + let initial_registers = LineRow::new(&header); + let opcode = LineInstruction::UnknownStandard0(constants::DwLns(111)); + let expected_registers = initial_registers; + assert_exec_opcode(header, initial_registers, opcode, expected_registers, false); + } + + #[test] + fn test_exec_unknown_standard_1() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + let initial_registers = LineRow::new(&header); + let opcode = LineInstruction::UnknownStandard1(constants::DwLns(111), 2); + let expected_registers = initial_registers; + assert_exec_opcode(header, initial_registers, opcode, expected_registers, false); + } + + #[test] + fn test_exec_unknown_standard_n() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + let initial_registers = LineRow::new(&header); + let opcode = LineInstruction::UnknownStandardN( + constants::DwLns(111), + EndianSlice::new(&[2, 2, 2], LittleEndian), + ); + let expected_registers = initial_registers; + assert_exec_opcode(header, initial_registers, opcode, expected_registers, false); + } + + #[test] + fn test_exec_end_sequence() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + let initial_registers = LineRow::new(&header); + let opcode = LineInstruction::EndSequence; + + let mut expected_registers = initial_registers; + expected_registers.end_sequence = true; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, true); + } + + #[test] + fn test_exec_set_address() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + let initial_registers = LineRow::new(&header); + let opcode = LineInstruction::SetAddress(3030); + + let mut expected_registers = initial_registers; + expected_registers.address.0 = 3030; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, false); + } + + #[test] + fn test_exec_define_file() { + let mut program = make_test_program(EndianSlice::new(&[], LittleEndian)); + let mut row = LineRow::new(program.header()); + + let file = FileEntry { + path_name: AttributeValue::String(EndianSlice::new(b"test.cpp", LittleEndian)), + directory_index: 0, + timestamp: 0, + size: 0, + md5: [0; 16], + }; + + let opcode = LineInstruction::DefineFile(file); + let is_new_row = row.execute(opcode, &mut program); + + assert_eq!(is_new_row, false); + assert_eq!(Some(&file), program.header().file_names.last()); + } + + #[test] + fn test_exec_set_discriminator() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + let initial_registers = LineRow::new(&header); + let opcode = LineInstruction::SetDiscriminator(9); + + let mut expected_registers = initial_registers; + expected_registers.discriminator = 9; + + assert_exec_opcode(header, initial_registers, opcode, expected_registers, false); + } + + #[test] + fn test_exec_unknown_extended() { + let header = make_test_header(EndianSlice::new(&[], LittleEndian)); + let initial_registers = LineRow::new(&header); + let opcode = LineInstruction::UnknownExtended( + constants::DwLne(74), + EndianSlice::new(&[], LittleEndian), + ); + let expected_registers = initial_registers; + assert_exec_opcode(header, initial_registers, opcode, expected_registers, false); + } + + /// Ensure that `LineRows` is covariant wrt R. + /// This only needs to compile. + #[allow(dead_code, unreachable_code, unused_variables)] + fn test_line_rows_variance<'a, 'b>(_: &'a [u8], _: &'b [u8]) + where + 'a: 'b, + { + let a: &OneShotLineRows> = unimplemented!(); + let _: &OneShotLineRows> = a; + } + + #[test] + fn test_parse_debug_line_v5_ok() { + let expected_lengths = &[1, 2]; + let expected_program = &[0, 1, 2, 3, 4]; + let expected_rest = &[5, 6, 7, 8, 9]; + let expected_include_directories = [ + AttributeValue::String(EndianSlice::new(b"dir1", LittleEndian)), + AttributeValue::String(EndianSlice::new(b"dir2", LittleEndian)), + ]; + let expected_file_names = [ + FileEntry { + path_name: AttributeValue::String(EndianSlice::new(b"file1", LittleEndian)), + directory_index: 0, + timestamp: 0, + size: 0, + md5: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + }, + FileEntry { + path_name: AttributeValue::String(EndianSlice::new(b"file2", LittleEndian)), + directory_index: 1, + timestamp: 0, + size: 0, + md5: [ + 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, + ], + }, + ]; + + for format in vec![Format::Dwarf32, Format::Dwarf64] { + let length = Label::new(); + let header_length = Label::new(); + let start = Label::new(); + let header_start = Label::new(); + let end = Label::new(); + let header_end = Label::new(); + let section = Section::with_endian(Endian::Little) + .initial_length(format, &length, &start) + .D16(5) + // Address size. + .D8(4) + // Segment selector size. + .D8(0) + .word_label(format.word_size(), &header_length) + .mark(&header_start) + // Minimum instruction length. + .D8(1) + // Maximum operations per byte. + .D8(1) + // Default is_stmt. + .D8(1) + // Line base. + .D8(0) + // Line range. + .D8(1) + // Opcode base. + .D8(expected_lengths.len() as u8 + 1) + // Standard opcode lengths for opcodes 1 .. opcode base - 1. + .append_bytes(expected_lengths) + // Directory entry format count. + .D8(1) + .uleb(constants::DW_LNCT_path.0 as u64) + .uleb(constants::DW_FORM_string.0 as u64) + // Directory count. + .D8(2) + .append_bytes(b"dir1\0") + .append_bytes(b"dir2\0") + // File entry format count. + .D8(3) + .uleb(constants::DW_LNCT_path.0 as u64) + .uleb(constants::DW_FORM_string.0 as u64) + .uleb(constants::DW_LNCT_directory_index.0 as u64) + .uleb(constants::DW_FORM_data1.0 as u64) + .uleb(constants::DW_LNCT_MD5.0 as u64) + .uleb(constants::DW_FORM_data16.0 as u64) + // File count. + .D8(2) + .append_bytes(b"file1\0") + .D8(0) + .append_bytes(&expected_file_names[0].md5) + .append_bytes(b"file2\0") + .D8(1) + .append_bytes(&expected_file_names[1].md5) + .mark(&header_end) + // Dummy line program data. + .append_bytes(expected_program) + .mark(&end) + // Dummy trailing data. + .append_bytes(expected_rest); + length.set_const((&end - &start) as u64); + header_length.set_const((&header_end - &header_start) as u64); + let section = section.get_contents().unwrap(); + + let input = &mut EndianSlice::new(§ion, LittleEndian); + + let header = LineProgramHeader::parse(input, DebugLineOffset(0), 0, None, None) + .expect("should parse header ok"); + + assert_eq!(header.raw_program_buf().slice(), expected_program); + assert_eq!(input.slice(), expected_rest); + + assert_eq!(header.offset, DebugLineOffset(0)); + assert_eq!(header.version(), 5); + assert_eq!(header.address_size(), 4); + assert_eq!(header.minimum_instruction_length(), 1); + assert_eq!(header.maximum_operations_per_instruction(), 1); + assert_eq!(header.default_is_stmt(), true); + assert_eq!(header.line_base(), 0); + assert_eq!(header.line_range(), 1); + assert_eq!(header.opcode_base(), expected_lengths.len() as u8 + 1); + assert_eq!(header.standard_opcode_lengths().slice(), expected_lengths); + assert_eq!( + header.directory_entry_format(), + &[FileEntryFormat { + content_type: constants::DW_LNCT_path, + form: constants::DW_FORM_string, + }] + ); + assert_eq!(header.include_directories(), expected_include_directories); + assert_eq!(header.directory(0), Some(expected_include_directories[0])); + assert_eq!( + header.file_name_entry_format(), + &[ + FileEntryFormat { + content_type: constants::DW_LNCT_path, + form: constants::DW_FORM_string, + }, + FileEntryFormat { + content_type: constants::DW_LNCT_directory_index, + form: constants::DW_FORM_data1, + }, + FileEntryFormat { + content_type: constants::DW_LNCT_MD5, + form: constants::DW_FORM_data16, + } + ] + ); + assert_eq!(header.file_names(), expected_file_names); + assert_eq!(header.file(0), Some(&expected_file_names[0])); + } + } +} diff --git a/crux-mir/lib/gimli/src/read/lists.rs b/crux-mir/lib/gimli/src/read/lists.rs new file mode 100644 index 000000000..898a757d3 --- /dev/null +++ b/crux-mir/lib/gimli/src/read/lists.rs @@ -0,0 +1,68 @@ +use crate::common::{Encoding, Format}; +use crate::read::{Error, Reader, Result}; + +#[derive(Debug, Clone, Copy)] +pub(crate) struct ListsHeader { + encoding: Encoding, + #[allow(dead_code)] + offset_entry_count: u32, +} + +impl Default for ListsHeader { + fn default() -> Self { + ListsHeader { + encoding: Encoding { + format: Format::Dwarf32, + version: 5, + address_size: 0, + }, + offset_entry_count: 0, + } + } +} + +impl ListsHeader { + /// Return the serialized size of the table header. + #[allow(dead_code)] + #[inline] + fn size(self) -> u8 { + // initial_length + version + address_size + segment_selector_size + offset_entry_count + ListsHeader::size_for_encoding(self.encoding) + } + + /// Return the serialized size of the table header. + #[inline] + pub(crate) fn size_for_encoding(encoding: Encoding) -> u8 { + // initial_length + version + address_size + segment_selector_size + offset_entry_count + encoding.format.initial_length_size() + 2 + 1 + 1 + 4 + } +} + +// TODO: add an iterator over headers in the appropriate sections section +#[allow(dead_code)] +fn parse_header(input: &mut R) -> Result { + let (length, format) = input.read_initial_length()?; + input.truncate(length)?; + + let version = input.read_u16()?; + if version != 5 { + return Err(Error::UnknownVersion(u64::from(version))); + } + + let address_size = input.read_u8()?; + let segment_selector_size = input.read_u8()?; + if segment_selector_size != 0 { + return Err(Error::UnsupportedSegmentSize); + } + let offset_entry_count = input.read_u32()?; + + let encoding = Encoding { + format, + version, + address_size, + }; + Ok(ListsHeader { + encoding, + offset_entry_count, + }) +} diff --git a/crux-mir/lib/gimli/src/read/loclists.rs b/crux-mir/lib/gimli/src/read/loclists.rs new file mode 100644 index 000000000..3902c181b --- /dev/null +++ b/crux-mir/lib/gimli/src/read/loclists.rs @@ -0,0 +1,1514 @@ +use crate::common::{ + DebugAddrBase, DebugAddrIndex, DebugLocListsBase, DebugLocListsIndex, DwarfFileType, Encoding, + LocationListsOffset, SectionId, +}; +use crate::constants; +use crate::endianity::Endianity; +use crate::read::{ + lists::ListsHeader, DebugAddr, EndianSlice, Error, Expression, Range, RawRange, Reader, + ReaderOffset, ReaderOffsetId, Result, Section, +}; + +/// The raw contents of the `.debug_loc` section. +#[derive(Debug, Default, Clone, Copy)] +pub struct DebugLoc { + pub(crate) section: R, +} + +impl<'input, Endian> DebugLoc> +where + Endian: Endianity, +{ + /// Construct a new `DebugLoc` instance from the data in the `.debug_loc` + /// section. + /// + /// It is the caller's responsibility to read the `.debug_loc` section and + /// present it as a `&[u8]` slice. That means using some ELF loader on + /// Linux, a Mach-O loader on macOS, etc. + /// + /// ``` + /// use gimli::{DebugLoc, LittleEndian}; + /// + /// # let buf = [0x00, 0x01, 0x02, 0x03]; + /// # let read_debug_loc_section_somehow = || &buf; + /// let debug_loc = DebugLoc::new(read_debug_loc_section_somehow(), LittleEndian); + /// ``` + pub fn new(section: &'input [u8], endian: Endian) -> Self { + Self::from(EndianSlice::new(section, endian)) + } +} + +impl Section for DebugLoc { + fn id() -> SectionId { + SectionId::DebugLoc + } + + fn reader(&self) -> &R { + &self.section + } +} + +impl From for DebugLoc { + fn from(section: R) -> Self { + DebugLoc { section } + } +} + +/// The `DebugLocLists` struct represents the DWARF data +/// found in the `.debug_loclists` section. +#[derive(Debug, Default, Clone, Copy)] +pub struct DebugLocLists { + section: R, +} + +impl<'input, Endian> DebugLocLists> +where + Endian: Endianity, +{ + /// Construct a new `DebugLocLists` instance from the data in the `.debug_loclists` + /// section. + /// + /// It is the caller's responsibility to read the `.debug_loclists` section and + /// present it as a `&[u8]` slice. That means using some ELF loader on + /// Linux, a Mach-O loader on macOS, etc. + /// + /// ``` + /// use gimli::{DebugLocLists, LittleEndian}; + /// + /// # let buf = [0x00, 0x01, 0x02, 0x03]; + /// # let read_debug_loclists_section_somehow = || &buf; + /// let debug_loclists = DebugLocLists::new(read_debug_loclists_section_somehow(), LittleEndian); + /// ``` + pub fn new(section: &'input [u8], endian: Endian) -> Self { + Self::from(EndianSlice::new(section, endian)) + } +} + +impl Section for DebugLocLists { + fn id() -> SectionId { + SectionId::DebugLocLists + } + + fn reader(&self) -> &R { + &self.section + } +} + +impl From for DebugLocLists { + fn from(section: R) -> Self { + DebugLocLists { section } + } +} + +pub(crate) type LocListsHeader = ListsHeader; + +impl DebugLocListsBase +where + Offset: ReaderOffset, +{ + /// Returns a `DebugLocListsBase` with the default value of DW_AT_loclists_base + /// for the given `Encoding` and `DwarfFileType`. + pub fn default_for_encoding_and_file( + encoding: Encoding, + file_type: DwarfFileType, + ) -> DebugLocListsBase { + if encoding.version >= 5 && file_type == DwarfFileType::Dwo { + // In .dwo files, the compiler omits the DW_AT_loclists_base attribute (because there is + // only a single unit in the file) but we must skip past the header, which the attribute + // would normally do for us. + DebugLocListsBase(Offset::from_u8(LocListsHeader::size_for_encoding(encoding))) + } else { + DebugLocListsBase(Offset::from_u8(0)) + } + } +} + +/// The DWARF data found in `.debug_loc` and `.debug_loclists` sections. +#[derive(Debug, Default, Clone, Copy)] +pub struct LocationLists { + debug_loc: DebugLoc, + debug_loclists: DebugLocLists, +} + +impl LocationLists { + /// Construct a new `LocationLists` instance from the data in the `.debug_loc` and + /// `.debug_loclists` sections. + pub fn new(debug_loc: DebugLoc, debug_loclists: DebugLocLists) -> LocationLists { + LocationLists { + debug_loc, + debug_loclists, + } + } +} + +impl LocationLists { + /// Create a `LocationLists` that references the data in `self`. + /// + /// This is useful when `R` implements `Reader` but `T` does not. + /// + /// ## Example Usage + /// + /// ```rust,no_run + /// # let load_section = || unimplemented!(); + /// // Read the DWARF section into a `Vec` with whatever object loader you're using. + /// let owned_section: gimli::LocationLists> = load_section(); + /// // Create a reference to the DWARF section. + /// let section = owned_section.borrow(|section| { + /// gimli::EndianSlice::new(§ion, gimli::LittleEndian) + /// }); + /// ``` + pub fn borrow<'a, F, R>(&'a self, mut borrow: F) -> LocationLists + where + F: FnMut(&'a T) -> R, + { + LocationLists { + debug_loc: borrow(&self.debug_loc.section).into(), + debug_loclists: borrow(&self.debug_loclists.section).into(), + } + } +} + +impl LocationLists { + /// Iterate over the `LocationListEntry`s starting at the given offset. + /// + /// The `unit_encoding` must match the compilation unit that the + /// offset was contained in. + /// + /// The `base_address` should be obtained from the `DW_AT_low_pc` attribute in the + /// `DW_TAG_compile_unit` entry for the compilation unit that contains this location + /// list. + /// + /// Can be [used with + /// `FallibleIterator`](./index.html#using-with-fallibleiterator). + pub fn locations( + &self, + offset: LocationListsOffset, + unit_encoding: Encoding, + base_address: u64, + debug_addr: &DebugAddr, + debug_addr_base: DebugAddrBase, + ) -> Result> { + Ok(LocListIter::new( + self.raw_locations(offset, unit_encoding)?, + base_address, + debug_addr.clone(), + debug_addr_base, + )) + } + + /// Similar to `locations`, but with special handling for .dwo files. + /// This should only been used when this `LocationLists` was loaded from a + /// .dwo file. + pub fn locations_dwo( + &self, + offset: LocationListsOffset, + unit_encoding: Encoding, + base_address: u64, + debug_addr: &DebugAddr, + debug_addr_base: DebugAddrBase, + ) -> Result> { + Ok(LocListIter::new( + self.raw_locations_dwo(offset, unit_encoding)?, + base_address, + debug_addr.clone(), + debug_addr_base, + )) + } + + /// Iterate over the raw `LocationListEntry`s starting at the given offset. + /// + /// The `unit_encoding` must match the compilation unit that the + /// offset was contained in. + /// + /// This iterator does not perform any processing of the location entries, + /// such as handling base addresses. + /// + /// Can be [used with + /// `FallibleIterator`](./index.html#using-with-fallibleiterator). + pub fn raw_locations( + &self, + offset: LocationListsOffset, + unit_encoding: Encoding, + ) -> Result> { + let (mut input, format) = if unit_encoding.version <= 4 { + (self.debug_loc.section.clone(), LocListsFormat::Bare) + } else { + (self.debug_loclists.section.clone(), LocListsFormat::LLE) + }; + input.skip(offset.0)?; + Ok(RawLocListIter::new(input, unit_encoding, format)) + } + + /// Similar to `raw_locations`, but with special handling for .dwo files. + /// This should only been used when this `LocationLists` was loaded from a + /// .dwo file. + pub fn raw_locations_dwo( + &self, + offset: LocationListsOffset, + unit_encoding: Encoding, + ) -> Result> { + let mut input = if unit_encoding.version <= 4 { + // In the GNU split dwarf extension the locations are present in the + // .debug_loc section but are encoded with the DW_LLE values used + // for the DWARF 5 .debug_loclists section. + self.debug_loc.section.clone() + } else { + self.debug_loclists.section.clone() + }; + input.skip(offset.0)?; + Ok(RawLocListIter::new( + input, + unit_encoding, + LocListsFormat::LLE, + )) + } + + /// Returns the `.debug_loclists` offset at the given `base` and `index`. + /// + /// The `base` must be the `DW_AT_loclists_base` value from the compilation unit DIE. + /// This is an offset that points to the first entry following the header. + /// + /// The `index` is the value of a `DW_FORM_loclistx` attribute. + pub fn get_offset( + &self, + unit_encoding: Encoding, + base: DebugLocListsBase, + index: DebugLocListsIndex, + ) -> Result> { + let format = unit_encoding.format; + let input = &mut self.debug_loclists.section.clone(); + input.skip(base.0)?; + input.skip(R::Offset::from_u64( + index.0.into_u64() * u64::from(format.word_size()), + )?)?; + input + .read_offset(format) + .map(|x| LocationListsOffset(base.0 + x)) + } + + /// Call `Reader::lookup_offset_id` for each section, and return the first match. + pub fn lookup_offset_id(&self, id: ReaderOffsetId) -> Option<(SectionId, R::Offset)> { + self.debug_loc + .lookup_offset_id(id) + .or_else(|| self.debug_loclists.lookup_offset_id(id)) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LocListsFormat { + /// The bare location list format used before DWARF 5. + Bare, + /// The DW_LLE encoded range list format used in DWARF 5 and the non-standard GNU + /// split dwarf extension. + LLE, +} + +/// A raw iterator over a location list. +/// +/// This iterator does not perform any processing of the location entries, +/// such as handling base addresses. +#[derive(Debug)] +pub struct RawLocListIter { + input: R, + encoding: Encoding, + format: LocListsFormat, +} + +/// A raw entry in .debug_loclists. +#[derive(Clone, Debug)] +pub enum RawLocListEntry { + /// A location from DWARF version <= 4. + AddressOrOffsetPair { + /// Start of range. May be an address or an offset. + begin: u64, + /// End of range. May be an address or an offset. + end: u64, + /// expression + data: Expression, + }, + /// DW_LLE_base_address + BaseAddress { + /// base address + addr: u64, + }, + /// DW_LLE_base_addressx + BaseAddressx { + /// base address + addr: DebugAddrIndex, + }, + /// DW_LLE_startx_endx + StartxEndx { + /// start of range + begin: DebugAddrIndex, + /// end of range + end: DebugAddrIndex, + /// expression + data: Expression, + }, + /// DW_LLE_startx_length + StartxLength { + /// start of range + begin: DebugAddrIndex, + /// length of range + length: u64, + /// expression + data: Expression, + }, + /// DW_LLE_offset_pair + OffsetPair { + /// start of range + begin: u64, + /// end of range + end: u64, + /// expression + data: Expression, + }, + /// DW_LLE_default_location + DefaultLocation { + /// expression + data: Expression, + }, + /// DW_LLE_start_end + StartEnd { + /// start of range + begin: u64, + /// end of range + end: u64, + /// expression + data: Expression, + }, + /// DW_LLE_start_length + StartLength { + /// start of range + begin: u64, + /// length of range + length: u64, + /// expression + data: Expression, + }, +} + +fn parse_data(input: &mut R, encoding: Encoding) -> Result> { + if encoding.version >= 5 { + let len = R::Offset::from_u64(input.read_uleb128()?)?; + Ok(Expression(input.split(len)?)) + } else { + // In the GNU split-dwarf extension this is a fixed 2 byte value. + let len = R::Offset::from_u16(input.read_u16()?); + Ok(Expression(input.split(len)?)) + } +} + +impl RawLocListEntry { + /// Parse a location list entry from `.debug_loclists` + fn parse(input: &mut R, encoding: Encoding, format: LocListsFormat) -> Result> { + match format { + LocListsFormat::Bare => { + let range = RawRange::parse(input, encoding.address_size)?; + return Ok(if range.is_end() { + None + } else if range.is_base_address(encoding.address_size) { + Some(RawLocListEntry::BaseAddress { addr: range.end }) + } else { + let len = R::Offset::from_u16(input.read_u16()?); + let data = Expression(input.split(len)?); + Some(RawLocListEntry::AddressOrOffsetPair { + begin: range.begin, + end: range.end, + data, + }) + }); + } + LocListsFormat::LLE => Ok(match constants::DwLle(input.read_u8()?) { + constants::DW_LLE_end_of_list => None, + constants::DW_LLE_base_addressx => Some(RawLocListEntry::BaseAddressx { + addr: DebugAddrIndex(input.read_uleb128().and_then(R::Offset::from_u64)?), + }), + constants::DW_LLE_startx_endx => Some(RawLocListEntry::StartxEndx { + begin: DebugAddrIndex(input.read_uleb128().and_then(R::Offset::from_u64)?), + end: DebugAddrIndex(input.read_uleb128().and_then(R::Offset::from_u64)?), + data: parse_data(input, encoding)?, + }), + constants::DW_LLE_startx_length => Some(RawLocListEntry::StartxLength { + begin: DebugAddrIndex(input.read_uleb128().and_then(R::Offset::from_u64)?), + length: if encoding.version >= 5 { + input.read_uleb128()? + } else { + // In the GNU split-dwarf extension this is a fixed 4 byte value. + input.read_u32()? as u64 + }, + data: parse_data(input, encoding)?, + }), + constants::DW_LLE_offset_pair => Some(RawLocListEntry::OffsetPair { + begin: input.read_uleb128()?, + end: input.read_uleb128()?, + data: parse_data(input, encoding)?, + }), + constants::DW_LLE_default_location => Some(RawLocListEntry::DefaultLocation { + data: parse_data(input, encoding)?, + }), + constants::DW_LLE_base_address => Some(RawLocListEntry::BaseAddress { + addr: input.read_address(encoding.address_size)?, + }), + constants::DW_LLE_start_end => Some(RawLocListEntry::StartEnd { + begin: input.read_address(encoding.address_size)?, + end: input.read_address(encoding.address_size)?, + data: parse_data(input, encoding)?, + }), + constants::DW_LLE_start_length => Some(RawLocListEntry::StartLength { + begin: input.read_address(encoding.address_size)?, + length: input.read_uleb128()?, + data: parse_data(input, encoding)?, + }), + _ => { + return Err(Error::InvalidAddressRange); + } + }), + } + } +} + +impl RawLocListIter { + /// Construct a `RawLocListIter`. + fn new(input: R, encoding: Encoding, format: LocListsFormat) -> RawLocListIter { + RawLocListIter { + input, + encoding, + format, + } + } + + /// Advance the iterator to the next location. + pub fn next(&mut self) -> Result>> { + if self.input.is_empty() { + return Ok(None); + } + + match RawLocListEntry::parse(&mut self.input, self.encoding, self.format) { + Ok(entry) => { + if entry.is_none() { + self.input.empty(); + } + Ok(entry) + } + Err(e) => { + self.input.empty(); + Err(e) + } + } + } +} + +#[cfg(feature = "fallible-iterator")] +impl fallible_iterator::FallibleIterator for RawLocListIter { + type Item = RawLocListEntry; + type Error = Error; + + fn next(&mut self) -> ::core::result::Result, Self::Error> { + RawLocListIter::next(self) + } +} + +/// An iterator over a location list. +/// +/// This iterator internally handles processing of base address selection entries +/// and list end entries. Thus, it only returns location entries that are valid +/// and already adjusted for the base address. +#[derive(Debug)] +pub struct LocListIter { + raw: RawLocListIter, + base_address: u64, + debug_addr: DebugAddr, + debug_addr_base: DebugAddrBase, +} + +impl LocListIter { + /// Construct a `LocListIter`. + fn new( + raw: RawLocListIter, + base_address: u64, + debug_addr: DebugAddr, + debug_addr_base: DebugAddrBase, + ) -> LocListIter { + LocListIter { + raw, + base_address, + debug_addr, + debug_addr_base, + } + } + + #[inline] + fn get_address(&self, index: DebugAddrIndex) -> Result { + self.debug_addr + .get_address(self.raw.encoding.address_size, self.debug_addr_base, index) + } + + /// Advance the iterator to the next location. + pub fn next(&mut self) -> Result>> { + loop { + let raw_loc = match self.raw.next()? { + Some(loc) => loc, + None => return Ok(None), + }; + + let (range, data) = match raw_loc { + RawLocListEntry::BaseAddress { addr } => { + self.base_address = addr; + continue; + } + RawLocListEntry::BaseAddressx { addr } => { + self.base_address = self.get_address(addr)?; + continue; + } + RawLocListEntry::StartxEndx { begin, end, data } => { + let begin = self.get_address(begin)?; + let end = self.get_address(end)?; + (Range { begin, end }, data) + } + RawLocListEntry::StartxLength { + begin, + length, + data, + } => { + let begin = self.get_address(begin)?; + let end = begin + length; + (Range { begin, end }, data) + } + RawLocListEntry::DefaultLocation { data } => ( + Range { + begin: 0, + end: u64::max_value(), + }, + data, + ), + RawLocListEntry::AddressOrOffsetPair { begin, end, data } + | RawLocListEntry::OffsetPair { begin, end, data } => { + let mut range = Range { begin, end }; + range.add_base_address(self.base_address, self.raw.encoding.address_size); + (range, data) + } + RawLocListEntry::StartEnd { begin, end, data } => (Range { begin, end }, data), + RawLocListEntry::StartLength { + begin, + length, + data, + } => ( + Range { + begin, + end: begin + length, + }, + data, + ), + }; + + if range.begin > range.end { + self.raw.input.empty(); + return Err(Error::InvalidLocationAddressRange); + } + + return Ok(Some(LocationListEntry { range, data })); + } + } +} + +#[cfg(feature = "fallible-iterator")] +impl fallible_iterator::FallibleIterator for LocListIter { + type Item = LocationListEntry; + type Error = Error; + + fn next(&mut self) -> ::core::result::Result, Self::Error> { + LocListIter::next(self) + } +} + +/// A location list entry from the `.debug_loc` or `.debug_loclists` sections. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct LocationListEntry { + /// The address range that this location is valid for. + pub range: Range, + + /// The data containing a single location description. + pub data: Expression, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::common::Format; + use crate::endianity::LittleEndian; + use crate::read::{EndianSlice, Range}; + use crate::test_util::GimliSectionMethods; + use test_assembler::{Endian, Label, LabelMaker, Section}; + + #[test] + fn test_loclists_32() { + let encoding = Encoding { + format: Format::Dwarf32, + version: 5, + address_size: 4, + }; + + let section = Section::with_endian(Endian::Little) + .L32(0x0300_0000) + .L32(0x0301_0300) + .L32(0x0301_0400) + .L32(0x0301_0500); + let buf = section.get_contents().unwrap(); + let debug_addr = &DebugAddr::from(EndianSlice::new(&buf, LittleEndian)); + let debug_addr_base = DebugAddrBase(0); + + let start = Label::new(); + let first = Label::new(); + let size = Label::new(); + #[rustfmt::skip] + let section = Section::with_endian(Endian::Little) + // Header + .mark(&start) + .L32(&size) + .L16(encoding.version) + .L8(encoding.address_size) + .L8(0) + .L32(0) + .mark(&first) + // OffsetPair + .L8(4).uleb(0x10200).uleb(0x10300).uleb(4).L32(2) + // A base address selection followed by an OffsetPair. + .L8(6).L32(0x0200_0000) + .L8(4).uleb(0x10400).uleb(0x10500).uleb(4).L32(3) + // An empty OffsetPair followed by a normal OffsetPair. + .L8(4).uleb(0x10600).uleb(0x10600).uleb(4).L32(4) + .L8(4).uleb(0x10800).uleb(0x10900).uleb(4).L32(5) + // A StartEnd + .L8(7).L32(0x201_0a00).L32(0x201_0b00).uleb(4).L32(6) + // A StartLength + .L8(8).L32(0x201_0c00).uleb(0x100).uleb(4).L32(7) + // An OffsetPair that starts at 0. + .L8(4).uleb(0).uleb(1).uleb(4).L32(8) + // An OffsetPair that ends at -1. + .L8(6).L32(0) + .L8(4).uleb(0).uleb(0xffff_ffff).uleb(4).L32(9) + // A DefaultLocation + .L8(5).uleb(4).L32(10) + // A BaseAddressx + OffsetPair + .L8(1).uleb(0) + .L8(4).uleb(0x10100).uleb(0x10200).uleb(4).L32(11) + // A StartxEndx + .L8(2).uleb(1).uleb(2).uleb(4).L32(12) + // A StartxLength + .L8(3).uleb(3).uleb(0x100).uleb(4).L32(13) + // A range end. + .L8(0) + // Some extra data. + .L32(0xffff_ffff); + size.set_const((§ion.here() - &start - 4) as u64); + + let buf = section.get_contents().unwrap(); + let debug_loc = DebugLoc::new(&[], LittleEndian); + let debug_loclists = DebugLocLists::new(&buf, LittleEndian); + let loclists = LocationLists::new(debug_loc, debug_loclists); + let offset = LocationListsOffset((&first - &start) as usize); + let mut locations = loclists + .locations(offset, encoding, 0x0100_0000, debug_addr, debug_addr_base) + .unwrap(); + + // A normal location. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0101_0200, + end: 0x0101_0300, + }, + data: Expression(EndianSlice::new(&[2, 0, 0, 0], LittleEndian)), + })) + ); + + // A base address selection followed by a normal location. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0201_0400, + end: 0x0201_0500, + }, + data: Expression(EndianSlice::new(&[3, 0, 0, 0], LittleEndian)), + })) + ); + + // An empty location range followed by a normal location. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0201_0600, + end: 0x0201_0600, + }, + data: Expression(EndianSlice::new(&[4, 0, 0, 0], LittleEndian)), + })) + ); + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0201_0800, + end: 0x0201_0900, + }, + data: Expression(EndianSlice::new(&[5, 0, 0, 0], LittleEndian)), + })) + ); + + // A normal location. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0201_0a00, + end: 0x0201_0b00, + }, + data: Expression(EndianSlice::new(&[6, 0, 0, 0], LittleEndian)), + })) + ); + + // A normal location. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0201_0c00, + end: 0x0201_0d00, + }, + data: Expression(EndianSlice::new(&[7, 0, 0, 0], LittleEndian)), + })) + ); + + // A location range that starts at 0. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0200_0000, + end: 0x0200_0001, + }, + data: Expression(EndianSlice::new(&[8, 0, 0, 0], LittleEndian)), + })) + ); + + // A location range that ends at -1. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0000_0000, + end: 0xffff_ffff, + }, + data: Expression(EndianSlice::new(&[9, 0, 0, 0], LittleEndian)), + })) + ); + + // A DefaultLocation. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0, + end: u64::max_value(), + }, + data: Expression(EndianSlice::new(&[10, 0, 0, 0], LittleEndian)), + })) + ); + + // A BaseAddressx + OffsetPair + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0301_0100, + end: 0x0301_0200, + }, + data: Expression(EndianSlice::new(&[11, 0, 0, 0], LittleEndian)), + })) + ); + + // A StartxEndx + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0301_0300, + end: 0x0301_0400, + }, + data: Expression(EndianSlice::new(&[12, 0, 0, 0], LittleEndian)), + })) + ); + + // A StartxLength + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0301_0500, + end: 0x0301_0600, + }, + data: Expression(EndianSlice::new(&[13, 0, 0, 0], LittleEndian)), + })) + ); + + // A location list end. + assert_eq!(locations.next(), Ok(None)); + + // An offset at the end of buf. + let mut locations = loclists + .locations( + LocationListsOffset(buf.len()), + encoding, + 0x0100_0000, + debug_addr, + debug_addr_base, + ) + .unwrap(); + assert_eq!(locations.next(), Ok(None)); + } + + #[test] + fn test_loclists_64() { + let encoding = Encoding { + format: Format::Dwarf64, + version: 5, + address_size: 8, + }; + + let section = Section::with_endian(Endian::Little) + .L64(0x0300_0000) + .L64(0x0301_0300) + .L64(0x0301_0400) + .L64(0x0301_0500); + let buf = section.get_contents().unwrap(); + let debug_addr = &DebugAddr::from(EndianSlice::new(&buf, LittleEndian)); + let debug_addr_base = DebugAddrBase(0); + + let start = Label::new(); + let first = Label::new(); + let size = Label::new(); + #[rustfmt::skip] + let section = Section::with_endian(Endian::Little) + // Header + .mark(&start) + .L32(0xffff_ffff) + .L64(&size) + .L16(encoding.version) + .L8(encoding.address_size) + .L8(0) + .L32(0) + .mark(&first) + // OffsetPair + .L8(4).uleb(0x10200).uleb(0x10300).uleb(4).L32(2) + // A base address selection followed by an OffsetPair. + .L8(6).L64(0x0200_0000) + .L8(4).uleb(0x10400).uleb(0x10500).uleb(4).L32(3) + // An empty OffsetPair followed by a normal OffsetPair. + .L8(4).uleb(0x10600).uleb(0x10600).uleb(4).L32(4) + .L8(4).uleb(0x10800).uleb(0x10900).uleb(4).L32(5) + // A StartEnd + .L8(7).L64(0x201_0a00).L64(0x201_0b00).uleb(4).L32(6) + // A StartLength + .L8(8).L64(0x201_0c00).uleb(0x100).uleb(4).L32(7) + // An OffsetPair that starts at 0. + .L8(4).uleb(0).uleb(1).uleb(4).L32(8) + // An OffsetPair that ends at -1. + .L8(6).L64(0) + .L8(4).uleb(0).uleb(0xffff_ffff).uleb(4).L32(9) + // A DefaultLocation + .L8(5).uleb(4).L32(10) + // A BaseAddressx + OffsetPair + .L8(1).uleb(0) + .L8(4).uleb(0x10100).uleb(0x10200).uleb(4).L32(11) + // A StartxEndx + .L8(2).uleb(1).uleb(2).uleb(4).L32(12) + // A StartxLength + .L8(3).uleb(3).uleb(0x100).uleb(4).L32(13) + // A range end. + .L8(0) + // Some extra data. + .L32(0xffff_ffff); + size.set_const((§ion.here() - &start - 12) as u64); + + let buf = section.get_contents().unwrap(); + let debug_loc = DebugLoc::new(&[], LittleEndian); + let debug_loclists = DebugLocLists::new(&buf, LittleEndian); + let loclists = LocationLists::new(debug_loc, debug_loclists); + let offset = LocationListsOffset((&first - &start) as usize); + let mut locations = loclists + .locations(offset, encoding, 0x0100_0000, debug_addr, debug_addr_base) + .unwrap(); + + // A normal location. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0101_0200, + end: 0x0101_0300, + }, + data: Expression(EndianSlice::new(&[2, 0, 0, 0], LittleEndian)), + })) + ); + + // A base address selection followed by a normal location. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0201_0400, + end: 0x0201_0500, + }, + data: Expression(EndianSlice::new(&[3, 0, 0, 0], LittleEndian)), + })) + ); + + // An empty location range followed by a normal location. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0201_0600, + end: 0x0201_0600, + }, + data: Expression(EndianSlice::new(&[4, 0, 0, 0], LittleEndian)), + })) + ); + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0201_0800, + end: 0x0201_0900, + }, + data: Expression(EndianSlice::new(&[5, 0, 0, 0], LittleEndian)), + })) + ); + + // A normal location. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0201_0a00, + end: 0x0201_0b00, + }, + data: Expression(EndianSlice::new(&[6, 0, 0, 0], LittleEndian)), + })) + ); + + // A normal location. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0201_0c00, + end: 0x0201_0d00, + }, + data: Expression(EndianSlice::new(&[7, 0, 0, 0], LittleEndian)), + })) + ); + + // A location range that starts at 0. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0200_0000, + end: 0x0200_0001, + }, + data: Expression(EndianSlice::new(&[8, 0, 0, 0], LittleEndian)), + })) + ); + + // A location range that ends at -1. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0000_0000, + end: 0xffff_ffff, + }, + data: Expression(EndianSlice::new(&[9, 0, 0, 0], LittleEndian)), + })) + ); + + // A DefaultLocation. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0, + end: u64::max_value(), + }, + data: Expression(EndianSlice::new(&[10, 0, 0, 0], LittleEndian)), + })) + ); + + // A BaseAddressx + OffsetPair + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0301_0100, + end: 0x0301_0200, + }, + data: Expression(EndianSlice::new(&[11, 0, 0, 0], LittleEndian)), + })) + ); + + // A StartxEndx + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0301_0300, + end: 0x0301_0400, + }, + data: Expression(EndianSlice::new(&[12, 0, 0, 0], LittleEndian)), + })) + ); + + // A StartxLength + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0301_0500, + end: 0x0301_0600, + }, + data: Expression(EndianSlice::new(&[13, 0, 0, 0], LittleEndian)), + })) + ); + + // A location list end. + assert_eq!(locations.next(), Ok(None)); + + // An offset at the end of buf. + let mut locations = loclists + .locations( + LocationListsOffset(buf.len()), + encoding, + 0x0100_0000, + debug_addr, + debug_addr_base, + ) + .unwrap(); + assert_eq!(locations.next(), Ok(None)); + } + + #[test] + fn test_location_list_32() { + let start = Label::new(); + let first = Label::new(); + #[rustfmt::skip] + let section = Section::with_endian(Endian::Little) + // A location before the offset. + .mark(&start) + .L32(0x10000).L32(0x10100).L16(4).L32(1) + .mark(&first) + // A normal location. + .L32(0x10200).L32(0x10300).L16(4).L32(2) + // A base address selection followed by a normal location. + .L32(0xffff_ffff).L32(0x0200_0000) + .L32(0x10400).L32(0x10500).L16(4).L32(3) + // An empty location range followed by a normal location. + .L32(0x10600).L32(0x10600).L16(4).L32(4) + .L32(0x10800).L32(0x10900).L16(4).L32(5) + // A location range that starts at 0. + .L32(0).L32(1).L16(4).L32(6) + // A location range that ends at -1. + .L32(0xffff_ffff).L32(0x0000_0000) + .L32(0).L32(0xffff_ffff).L16(4).L32(7) + // A location list end. + .L32(0).L32(0) + // Some extra data. + .L32(0); + + let buf = section.get_contents().unwrap(); + let debug_loc = DebugLoc::new(&buf, LittleEndian); + let debug_loclists = DebugLocLists::new(&[], LittleEndian); + let loclists = LocationLists::new(debug_loc, debug_loclists); + let offset = LocationListsOffset((&first - &start) as usize); + let debug_addr = &DebugAddr::from(EndianSlice::new(&[], LittleEndian)); + let debug_addr_base = DebugAddrBase(0); + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 4, + }; + let mut locations = loclists + .locations(offset, encoding, 0x0100_0000, debug_addr, debug_addr_base) + .unwrap(); + + // A normal location. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0101_0200, + end: 0x0101_0300, + }, + data: Expression(EndianSlice::new(&[2, 0, 0, 0], LittleEndian)), + })) + ); + + // A base address selection followed by a normal location. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0201_0400, + end: 0x0201_0500, + }, + data: Expression(EndianSlice::new(&[3, 0, 0, 0], LittleEndian)), + })) + ); + + // An empty location range followed by a normal location. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0201_0600, + end: 0x0201_0600, + }, + data: Expression(EndianSlice::new(&[4, 0, 0, 0], LittleEndian)), + })) + ); + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0201_0800, + end: 0x0201_0900, + }, + data: Expression(EndianSlice::new(&[5, 0, 0, 0], LittleEndian)), + })) + ); + + // A location range that starts at 0. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0200_0000, + end: 0x0200_0001, + }, + data: Expression(EndianSlice::new(&[6, 0, 0, 0], LittleEndian)), + })) + ); + + // A location range that ends at -1. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0000_0000, + end: 0xffff_ffff, + }, + data: Expression(EndianSlice::new(&[7, 0, 0, 0], LittleEndian)), + })) + ); + + // A location list end. + assert_eq!(locations.next(), Ok(None)); + + // An offset at the end of buf. + let mut locations = loclists + .locations( + LocationListsOffset(buf.len()), + encoding, + 0x0100_0000, + debug_addr, + debug_addr_base, + ) + .unwrap(); + assert_eq!(locations.next(), Ok(None)); + } + + #[test] + fn test_location_list_64() { + let start = Label::new(); + let first = Label::new(); + #[rustfmt::skip] + let section = Section::with_endian(Endian::Little) + // A location before the offset. + .mark(&start) + .L64(0x10000).L64(0x10100).L16(4).L32(1) + .mark(&first) + // A normal location. + .L64(0x10200).L64(0x10300).L16(4).L32(2) + // A base address selection followed by a normal location. + .L64(0xffff_ffff_ffff_ffff).L64(0x0200_0000) + .L64(0x10400).L64(0x10500).L16(4).L32(3) + // An empty location range followed by a normal location. + .L64(0x10600).L64(0x10600).L16(4).L32(4) + .L64(0x10800).L64(0x10900).L16(4).L32(5) + // A location range that starts at 0. + .L64(0).L64(1).L16(4).L32(6) + // A location range that ends at -1. + .L64(0xffff_ffff_ffff_ffff).L64(0x0000_0000) + .L64(0).L64(0xffff_ffff_ffff_ffff).L16(4).L32(7) + // A location list end. + .L64(0).L64(0) + // Some extra data. + .L64(0); + + let buf = section.get_contents().unwrap(); + let debug_loc = DebugLoc::new(&buf, LittleEndian); + let debug_loclists = DebugLocLists::new(&[], LittleEndian); + let loclists = LocationLists::new(debug_loc, debug_loclists); + let offset = LocationListsOffset((&first - &start) as usize); + let debug_addr = &DebugAddr::from(EndianSlice::new(&[], LittleEndian)); + let debug_addr_base = DebugAddrBase(0); + let encoding = Encoding { + format: Format::Dwarf64, + version: 4, + address_size: 8, + }; + let mut locations = loclists + .locations(offset, encoding, 0x0100_0000, debug_addr, debug_addr_base) + .unwrap(); + + // A normal location. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0101_0200, + end: 0x0101_0300, + }, + data: Expression(EndianSlice::new(&[2, 0, 0, 0], LittleEndian)), + })) + ); + + // A base address selection followed by a normal location. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0201_0400, + end: 0x0201_0500, + }, + data: Expression(EndianSlice::new(&[3, 0, 0, 0], LittleEndian)), + })) + ); + + // An empty location range followed by a normal location. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0201_0600, + end: 0x0201_0600, + }, + data: Expression(EndianSlice::new(&[4, 0, 0, 0], LittleEndian)), + })) + ); + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0201_0800, + end: 0x0201_0900, + }, + data: Expression(EndianSlice::new(&[5, 0, 0, 0], LittleEndian)), + })) + ); + + // A location range that starts at 0. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0200_0000, + end: 0x0200_0001, + }, + data: Expression(EndianSlice::new(&[6, 0, 0, 0], LittleEndian)), + })) + ); + + // A location range that ends at -1. + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0, + end: 0xffff_ffff_ffff_ffff, + }, + data: Expression(EndianSlice::new(&[7, 0, 0, 0], LittleEndian)), + })) + ); + + // A location list end. + assert_eq!(locations.next(), Ok(None)); + + // An offset at the end of buf. + let mut locations = loclists + .locations( + LocationListsOffset(buf.len()), + encoding, + 0x0100_0000, + debug_addr, + debug_addr_base, + ) + .unwrap(); + assert_eq!(locations.next(), Ok(None)); + } + + #[test] + fn test_locations_invalid() { + #[rustfmt::skip] + let section = Section::with_endian(Endian::Little) + // An invalid location range. + .L32(0x20000).L32(0x10000).L16(4).L32(1) + // An invalid range after wrapping. + .L32(0x20000).L32(0xff01_0000).L16(4).L32(2); + + let buf = section.get_contents().unwrap(); + let debug_loc = DebugLoc::new(&buf, LittleEndian); + let debug_loclists = DebugLocLists::new(&[], LittleEndian); + let loclists = LocationLists::new(debug_loc, debug_loclists); + let debug_addr = &DebugAddr::from(EndianSlice::new(&[], LittleEndian)); + let debug_addr_base = DebugAddrBase(0); + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 4, + }; + + // An invalid location range. + let mut locations = loclists + .locations( + LocationListsOffset(0x0), + encoding, + 0x0100_0000, + debug_addr, + debug_addr_base, + ) + .unwrap(); + assert_eq!(locations.next(), Err(Error::InvalidLocationAddressRange)); + + // An invalid location range after wrapping. + let mut locations = loclists + .locations( + LocationListsOffset(14), + encoding, + 0x0100_0000, + debug_addr, + debug_addr_base, + ) + .unwrap(); + assert_eq!(locations.next(), Err(Error::InvalidLocationAddressRange)); + + // An invalid offset. + match loclists.locations( + LocationListsOffset(buf.len() + 1), + encoding, + 0x0100_0000, + debug_addr, + debug_addr_base, + ) { + Err(Error::UnexpectedEof(_)) => {} + otherwise => panic!("Unexpected result: {:?}", otherwise), + } + } + + #[test] + fn test_get_offset() { + for format in vec![Format::Dwarf32, Format::Dwarf64] { + let encoding = Encoding { + format, + version: 5, + address_size: 4, + }; + + let zero = Label::new(); + let length = Label::new(); + let start = Label::new(); + let first = Label::new(); + let end = Label::new(); + let mut section = Section::with_endian(Endian::Little) + .mark(&zero) + .initial_length(format, &length, &start) + .D16(encoding.version) + .D8(encoding.address_size) + .D8(0) + .D32(20) + .mark(&first); + for i in 0..20 { + section = section.word(format.word_size(), 1000 + i); + } + section = section.mark(&end); + length.set_const((&end - &start) as u64); + let section = section.get_contents().unwrap(); + + let debug_loc = DebugLoc::from(EndianSlice::new(&[], LittleEndian)); + let debug_loclists = DebugLocLists::from(EndianSlice::new(§ion, LittleEndian)); + let locations = LocationLists::new(debug_loc, debug_loclists); + + let base = DebugLocListsBase((&first - &zero) as usize); + assert_eq!( + locations.get_offset(encoding, base, DebugLocListsIndex(0)), + Ok(LocationListsOffset(base.0 + 1000)) + ); + assert_eq!( + locations.get_offset(encoding, base, DebugLocListsIndex(19)), + Ok(LocationListsOffset(base.0 + 1019)) + ); + } + } + + #[test] + fn test_loclists_gnu_v4_split_dwarf() { + #[rustfmt::skip] + let buf = [ + 0x03, // DW_LLE_startx_length + 0x00, // ULEB encoded b7 + 0x08, 0x00, 0x00, 0x00, // Fixed 4 byte length of 8 + 0x03, 0x00, // Fixed two byte length of the location + 0x11, 0x00, // DW_OP_constu 0 + 0x9f, // DW_OP_stack_value + // Padding data + //0x99, 0x99, 0x99, 0x99 + ]; + let data_buf = [0x11, 0x00, 0x9f]; + let expected_data = EndianSlice::new(&data_buf, LittleEndian); + let debug_loc = DebugLoc::new(&buf, LittleEndian); + let debug_loclists = DebugLocLists::new(&[], LittleEndian); + let loclists = LocationLists::new(debug_loc, debug_loclists); + let debug_addr = + &DebugAddr::from(EndianSlice::new(&[0x01, 0x02, 0x03, 0x04], LittleEndian)); + let debug_addr_base = DebugAddrBase(0); + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 4, + }; + + // An invalid location range. + let mut locations = loclists + .locations_dwo( + LocationListsOffset(0x0), + encoding, + 0, + debug_addr, + debug_addr_base, + ) + .unwrap(); + assert_eq!( + locations.next(), + Ok(Some(LocationListEntry { + range: Range { + begin: 0x0403_0201, + end: 0x0403_0209 + }, + data: Expression(expected_data), + })) + ); + } +} diff --git a/crux-mir/lib/gimli/src/read/lookup.rs b/crux-mir/lib/gimli/src/read/lookup.rs new file mode 100644 index 000000000..1d082f24f --- /dev/null +++ b/crux-mir/lib/gimli/src/read/lookup.rs @@ -0,0 +1,202 @@ +use core::marker::PhantomData; + +use crate::common::{DebugInfoOffset, Format}; +use crate::read::{parse_debug_info_offset, Error, Reader, ReaderOffset, Result, UnitOffset}; + +// The various "Accelerated Access" sections (DWARF standard v4 Section 6.1) all have +// similar structures. They consist of a header with metadata and an offset into the +// .debug_info section for the entire compilation unit, and a series +// of following entries that list addresses (for .debug_aranges) or names +// (for .debug_pubnames and .debug_pubtypes) that are covered. +// +// Because these three tables all have similar structures, we abstract out some of +// the parsing mechanics. + +pub trait LookupParser { + /// The type of the produced header. + type Header; + /// The type of the produced entry. + type Entry; + + /// Parse a header from `input`. Returns a tuple of `input` sliced to contain just the entries + /// corresponding to this header (without the header itself), and the parsed representation of + /// the header itself. + fn parse_header(input: &mut R) -> Result<(R, Self::Header)>; + + /// Parse a single entry from `input`. Returns either a parsed representation of the entry + /// or None if `input` is exhausted. + fn parse_entry(input: &mut R, header: &Self::Header) -> Result>; +} + +#[derive(Clone, Debug)] +pub struct DebugLookup +where + R: Reader, + Parser: LookupParser, +{ + input_buffer: R, + phantom: PhantomData, +} + +impl From for DebugLookup +where + R: Reader, + Parser: LookupParser, +{ + fn from(input_buffer: R) -> Self { + DebugLookup { + input_buffer, + phantom: PhantomData, + } + } +} + +impl DebugLookup +where + R: Reader, + Parser: LookupParser, +{ + pub fn items(&self) -> LookupEntryIter { + LookupEntryIter { + current_set: None, + remaining_input: self.input_buffer.clone(), + } + } + + pub fn reader(&self) -> &R { + &self.input_buffer + } +} + +#[derive(Clone, Debug)] +pub struct LookupEntryIter +where + R: Reader, + Parser: LookupParser, +{ + current_set: Option<(R, Parser::Header)>, // Only none at the very beginning and end. + remaining_input: R, +} + +impl LookupEntryIter +where + R: Reader, + Parser: LookupParser, +{ + /// Advance the iterator and return the next entry. + /// + /// Returns the newly parsed entry as `Ok(Some(Parser::Entry))`. Returns + /// `Ok(None)` when iteration is complete and all entries have already been + /// parsed and yielded. If an error occurs while parsing the next entry, + /// then this error is returned as `Err(e)`, and all subsequent calls return + /// `Ok(None)`. + /// + /// Can be [used with `FallibleIterator`](./index.html#using-with-fallibleiterator). + pub fn next(&mut self) -> Result> { + loop { + if let Some((ref mut input, ref header)) = self.current_set { + if !input.is_empty() { + match Parser::parse_entry(input, header) { + Ok(Some(entry)) => return Ok(Some(entry)), + Ok(None) => {} + Err(e) => { + input.empty(); + self.remaining_input.empty(); + return Err(e); + } + } + } + } + if self.remaining_input.is_empty() { + self.current_set = None; + return Ok(None); + } + match Parser::parse_header(&mut self.remaining_input) { + Ok(set) => { + self.current_set = Some(set); + } + Err(e) => { + self.current_set = None; + self.remaining_input.empty(); + return Err(e); + } + } + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PubStuffHeader { + format: Format, + length: T, + version: u16, + unit_offset: DebugInfoOffset, + unit_length: T, +} + +pub trait PubStuffEntry { + fn new( + die_offset: UnitOffset, + name: R, + unit_header_offset: DebugInfoOffset, + ) -> Self; +} + +#[derive(Clone, Debug)] +pub struct PubStuffParser +where + R: Reader, + Entry: PubStuffEntry, +{ + // This struct is never instantiated. + phantom: PhantomData<(R, Entry)>, +} + +impl LookupParser for PubStuffParser +where + R: Reader, + Entry: PubStuffEntry, +{ + type Header = PubStuffHeader; + type Entry = Entry; + + /// Parse an pubthings set header. Returns a tuple of the + /// pubthings to be parsed for this set, and the newly created PubThingHeader struct. + fn parse_header(input: &mut R) -> Result<(R, Self::Header)> { + let (length, format) = input.read_initial_length()?; + let mut rest = input.split(length)?; + + let version = rest.read_u16()?; + if version != 2 { + return Err(Error::UnknownVersion(u64::from(version))); + } + + let unit_offset = parse_debug_info_offset(&mut rest, format)?; + let unit_length = rest.read_length(format)?; + + let header = PubStuffHeader { + format, + length, + version, + unit_offset, + unit_length, + }; + Ok((rest, header)) + } + + /// Parse a single pubthing. Return `None` for the null pubthing, `Some` for an actual pubthing. + fn parse_entry(input: &mut R, header: &Self::Header) -> Result> { + let offset = input.read_offset(header.format)?; + if offset.into_u64() == 0 { + input.empty(); + Ok(None) + } else { + let name = input.read_null_terminated_slice()?; + Ok(Some(Self::Entry::new( + UnitOffset(offset), + name, + header.unit_offset, + ))) + } + } +} diff --git a/crux-mir/lib/gimli/src/read/mod.rs b/crux-mir/lib/gimli/src/read/mod.rs new file mode 100644 index 000000000..3110957c2 --- /dev/null +++ b/crux-mir/lib/gimli/src/read/mod.rs @@ -0,0 +1,821 @@ +//! Read DWARF debugging information. +//! +//! * [Example Usage](#example-usage) +//! * [API Structure](#api-structure) +//! * [Using with `FallibleIterator`](#using-with-fallibleiterator) +//! +//! ## Example Usage +//! +//! Print out all of the functions in the debuggee program: +//! +//! ```rust,no_run +//! # fn example() -> Result<(), gimli::Error> { +//! # type R = gimli::EndianSlice<'static, gimli::LittleEndian>; +//! # let get_file_section_reader = |name| -> Result { unimplemented!() }; +//! # let get_sup_file_section_reader = |name| -> Result { unimplemented!() }; +//! // Read the DWARF sections with whatever object loader you're using. +//! // These closures should return a `Reader` instance (e.g. `EndianSlice`). +//! let loader = |section: gimli::SectionId| { get_file_section_reader(section.name()) }; +//! let sup_loader = |section: gimli::SectionId| { get_sup_file_section_reader(section.name()) }; +//! let mut dwarf = gimli::Dwarf::load(loader)?; +//! dwarf.load_sup(sup_loader)?; +//! +//! // Iterate over all compilation units. +//! let mut iter = dwarf.units(); +//! while let Some(header) = iter.next()? { +//! // Parse the abbreviations and other information for this compilation unit. +//! let unit = dwarf.unit(header)?; +//! +//! // Iterate over all of this compilation unit's entries. +//! let mut entries = unit.entries(); +//! while let Some((_, entry)) = entries.next_dfs()? { +//! // If we find an entry for a function, print it. +//! if entry.tag() == gimli::DW_TAG_subprogram { +//! println!("Found a function: {:?}", entry); +//! } +//! } +//! } +//! # unreachable!() +//! # } +//! ``` +//! +//! Full example programs: +//! +//! * [A simple parser](https://github.com/gimli-rs/gimli/blob/master/examples/simple.rs) +//! +//! * [A `dwarfdump` +//! clone](https://github.com/gimli-rs/gimli/blob/master/examples/dwarfdump.rs) +//! +//! * [An `addr2line` clone](https://github.com/gimli-rs/addr2line) +//! +//! * [`ddbug`](https://github.com/gimli-rs/ddbug), a utility giving insight into +//! code generation by making debugging information readable +//! +//! * [`dwprod`](https://github.com/fitzgen/dwprod), a tiny utility to list the +//! compilers used to create each compilation unit within a shared library or +//! executable (via `DW_AT_producer`) +//! +//! * [`dwarf-validate`](https://github.com/gimli-rs/gimli/blob/master/examples/dwarf-validate.rs), +//! a program to validate the integrity of some DWARF and its references +//! between sections and compilation units. +//! +//! ## API Structure +//! +//! * Basic familiarity with DWARF is assumed. +//! +//! * The [`Dwarf`](./struct.Dwarf.html) type contains the commonly used DWARF +//! sections. It has methods that simplify access to debugging data that spans +//! multiple sections. Use of this type is optional, but recommended. +//! +//! * Each section gets its own type. Consider these types the entry points to +//! the library: +//! +//! * [`DebugAbbrev`](./struct.DebugAbbrev.html): The `.debug_abbrev` section. +//! +//! * [`DebugAddr`](./struct.DebugAddr.html): The `.debug_addr` section. +//! +//! * [`DebugAranges`](./struct.DebugAranges.html): The `.debug_aranges` +//! section. +//! +//! * [`DebugFrame`](./struct.DebugFrame.html): The `.debug_frame` section. +//! +//! * [`DebugInfo`](./struct.DebugInfo.html): The `.debug_info` section. +//! +//! * [`DebugLine`](./struct.DebugLine.html): The `.debug_line` section. +//! +//! * [`DebugLineStr`](./struct.DebugLineStr.html): The `.debug_line_str` section. +//! +//! * [`DebugLoc`](./struct.DebugLoc.html): The `.debug_loc` section. +//! +//! * [`DebugLocLists`](./struct.DebugLocLists.html): The `.debug_loclists` section. +//! +//! * [`DebugPubNames`](./struct.DebugPubNames.html): The `.debug_pubnames` +//! section. +//! +//! * [`DebugPubTypes`](./struct.DebugPubTypes.html): The `.debug_pubtypes` +//! section. +//! +//! * [`DebugRanges`](./struct.DebugRanges.html): The `.debug_ranges` section. +//! +//! * [`DebugRngLists`](./struct.DebugRngLists.html): The `.debug_rnglists` section. +//! +//! * [`DebugStr`](./struct.DebugStr.html): The `.debug_str` section. +//! +//! * [`DebugStrOffsets`](./struct.DebugStrOffsets.html): The `.debug_str_offsets` section. +//! +//! * [`DebugTypes`](./struct.DebugTypes.html): The `.debug_types` section. +//! +//! * [`DebugCuIndex`](./struct.DebugCuIndex.html): The `.debug_cu_index` section. +//! +//! * [`DebugTuIndex`](./struct.DebugTuIndex.html): The `.debug_tu_index` section. +//! +//! * [`EhFrame`](./struct.EhFrame.html): The `.eh_frame` section. +//! +//! * [`EhFrameHdr`](./struct.EhFrameHdr.html): The `.eh_frame_hdr` section. +//! +//! * Each section type exposes methods for accessing the debugging data encoded +//! in that section. For example, the [`DebugInfo`](./struct.DebugInfo.html) +//! struct has the [`units`](./struct.DebugInfo.html#method.units) method for +//! iterating over the compilation units defined within it. +//! +//! * Offsets into a section are strongly typed: an offset into `.debug_info` is +//! the [`DebugInfoOffset`](./struct.DebugInfoOffset.html) type. It cannot be +//! used to index into the [`DebugLine`](./struct.DebugLine.html) type because +//! `DebugLine` represents the `.debug_line` section. There are similar types +//! for offsets relative to a compilation unit rather than a section. +//! +//! ## Using with `FallibleIterator` +//! +//! The standard library's `Iterator` trait and related APIs do not play well +//! with iterators where the `next` operation is fallible. One can make the +//! `Iterator`'s associated `Item` type be a `Result`, however the +//! provided methods cannot gracefully handle the case when an `Err` is +//! returned. +//! +//! This situation led to the +//! [`fallible-iterator`](https://crates.io/crates/fallible-iterator) crate's +//! existence. You can read more of the rationale for its existence in its +//! docs. The crate provides the helpers you have come to expect (eg `map`, +//! `filter`, etc) for iterators that can fail. +//! +//! `gimli`'s many lazy parsing iterators are a perfect match for the +//! `fallible-iterator` crate's `FallibleIterator` trait because parsing is not +//! done eagerly. Parse errors later in the input might only be discovered after +//! having iterated through many items. +//! +//! To use `gimli` iterators with `FallibleIterator`, import the crate and trait +//! into your code: +//! +//! ``` +//! # #[cfg(feature = "fallible-iterator")] +//! # fn foo() { +//! // Use the `FallibleIterator` trait so its methods are in scope! +//! use fallible_iterator::FallibleIterator; +//! use gimli::{DebugAranges, EndianSlice, LittleEndian}; +//! +//! fn find_sum_of_address_range_lengths(aranges: DebugAranges>) +//! -> gimli::Result +//! { +//! // `DebugAranges::headers` returns a `FallibleIterator`! +//! aranges.headers() +//! // `flat_map` is provided by `FallibleIterator`! +//! .flat_map(|header| Ok(header.entries())) +//! // `map` is provided by `FallibleIterator`! +//! .map(|arange| Ok(arange.length())) +//! // `fold` is provided by `FallibleIterator`! +//! .fold(0, |sum, len| Ok(sum + len)) +//! } +//! # } +//! # fn main() {} +//! ``` + +use core::fmt::{self, Debug}; +use core::result; +#[cfg(feature = "std")] +use std::{error, io}; + +use crate::common::{Register, SectionId}; +use crate::constants; + +mod util; +pub use util::*; + +mod addr; +pub use self::addr::*; + +mod cfi; +pub use self::cfi::*; + +#[cfg(feature = "read")] +mod dwarf; +#[cfg(feature = "read")] +pub use self::dwarf::*; + +mod endian_slice; +pub use self::endian_slice::*; + +#[cfg(feature = "endian-reader")] +mod endian_reader; +#[cfg(feature = "endian-reader")] +pub use self::endian_reader::*; + +mod reader; +pub use self::reader::*; + +#[cfg(feature = "read")] +mod abbrev; +#[cfg(feature = "read")] +pub use self::abbrev::*; + +mod aranges; +pub use self::aranges::*; + +mod index; +pub use self::index::*; + +#[cfg(feature = "read")] +mod line; +#[cfg(feature = "read")] +pub use self::line::*; + +mod lists; + +mod loclists; +pub use self::loclists::*; + +#[cfg(feature = "read")] +mod lookup; + +mod op; +pub use self::op::*; + +#[cfg(feature = "read")] +mod pubnames; +#[cfg(feature = "read")] +pub use self::pubnames::*; + +#[cfg(feature = "read")] +mod pubtypes; +#[cfg(feature = "read")] +pub use self::pubtypes::*; + +mod rnglists; +pub use self::rnglists::*; + +mod str; +pub use self::str::*; + +/// An offset into the current compilation or type unit. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash)] +pub struct UnitOffset(pub T); + +#[cfg(feature = "read")] +mod unit; +#[cfg(feature = "read")] +pub use self::unit::*; + +mod value; +pub use self::value::*; + +/// Indicates that storage should be allocated on heap. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StoreOnHeap; + +/// `EndianBuf` has been renamed to `EndianSlice`. For ease of upgrading across +/// `gimli` versions, we export this type alias. +#[deprecated(note = "EndianBuf has been renamed to EndianSlice, use that instead.")] +pub type EndianBuf<'input, Endian> = EndianSlice<'input, Endian>; + +/// An error that occurred when parsing. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Error { + /// An I/O error occurred while reading. + Io, + /// Found a PC relative pointer, but the section base is undefined. + PcRelativePointerButSectionBaseIsUndefined, + /// Found a `.text` relative pointer, but the `.text` base is undefined. + TextRelativePointerButTextBaseIsUndefined, + /// Found a data relative pointer, but the data base is undefined. + DataRelativePointerButDataBaseIsUndefined, + /// Found a function relative pointer in a context that does not have a + /// function base. + FuncRelativePointerInBadContext, + /// Cannot parse a pointer with a `DW_EH_PE_omit` encoding. + CannotParseOmitPointerEncoding, + /// An error parsing an unsigned LEB128 value. + BadUnsignedLeb128, + /// An error parsing a signed LEB128 value. + BadSignedLeb128, + /// An abbreviation declared that its tag is zero, but zero is reserved for + /// null records. + AbbreviationTagZero, + /// An attribute specification declared that its form is zero, but zero is + /// reserved for null records. + AttributeFormZero, + /// The abbreviation's has-children byte was not one of + /// `DW_CHILDREN_{yes,no}`. + BadHasChildren, + /// The specified length is impossible. + BadLength, + /// Found an unknown `DW_FORM_*` type. + UnknownForm, + /// Expected a zero, found something else. + ExpectedZero, + /// Found an abbreviation code that has already been used. + DuplicateAbbreviationCode, + /// Found a duplicate arange. + DuplicateArange, + /// Found an unknown reserved length value. + UnknownReservedLength, + /// Found an unknown DWARF version. + UnknownVersion(u64), + /// Found a record with an unknown abbreviation code. + UnknownAbbreviation, + /// Hit the end of input before it was expected. + UnexpectedEof(ReaderOffsetId), + /// Read a null entry before it was expected. + UnexpectedNull, + /// Found an unknown standard opcode. + UnknownStandardOpcode(constants::DwLns), + /// Found an unknown extended opcode. + UnknownExtendedOpcode(constants::DwLne), + /// The specified address size is not supported. + UnsupportedAddressSize(u8), + /// The specified offset size is not supported. + UnsupportedOffsetSize(u8), + /// The specified field size is not supported. + UnsupportedFieldSize(u8), + /// The minimum instruction length must not be zero. + MinimumInstructionLengthZero, + /// The maximum operations per instruction must not be zero. + MaximumOperationsPerInstructionZero, + /// The line range must not be zero. + LineRangeZero, + /// The opcode base must not be zero. + OpcodeBaseZero, + /// Found an invalid UTF-8 string. + BadUtf8, + /// Expected to find the CIE ID, but found something else. + NotCieId, + /// Expected to find a pointer to a CIE, but found the CIE ID instead. + NotCiePointer, + /// Expected to find a pointer to an FDE, but found a CIE instead. + NotFdePointer, + /// Invalid branch target for a DW_OP_bra or DW_OP_skip. + BadBranchTarget(u64), + /// DW_OP_push_object_address used but no address passed in. + InvalidPushObjectAddress, + /// Not enough items on the stack when evaluating an expression. + NotEnoughStackItems, + /// Too many iterations to compute the expression. + TooManyIterations, + /// An unrecognized operation was found while parsing a DWARF + /// expression. + InvalidExpression(constants::DwOp), + /// An unsupported operation was found while evaluating a DWARF expression. + UnsupportedEvaluation, + /// The expression had a piece followed by an expression + /// terminator without a piece. + InvalidPiece, + /// An expression-terminating operation was followed by something + /// other than the end of the expression or a piece operation. + InvalidExpressionTerminator(u64), + /// Division or modulus by zero when evaluating an expression. + DivisionByZero, + /// An expression operation used mismatching types. + TypeMismatch, + /// An expression operation required an integral type but saw a + /// floating point type. + IntegralTypeRequired, + /// An expression operation used types that are not supported. + UnsupportedTypeOperation, + /// The shift value in an expression must be a non-negative integer. + InvalidShiftExpression, + /// An unknown DW_CFA_* instruction. + UnknownCallFrameInstruction(constants::DwCfa), + /// The end of an address range was before the beginning. + InvalidAddressRange, + /// The end offset of a loc list entry was before the beginning. + InvalidLocationAddressRange, + /// Encountered a call frame instruction in a context in which it is not + /// valid. + CfiInstructionInInvalidContext, + /// When evaluating call frame instructions, found a `DW_CFA_restore_state` + /// stack pop instruction, but the stack was empty, and had nothing to pop. + PopWithEmptyStack, + /// Do not have unwind info for the given address. + NoUnwindInfoForAddress, + /// An offset value was larger than the maximum supported value. + UnsupportedOffset, + /// The given pointer encoding is either unknown or invalid. + UnknownPointerEncoding, + /// Did not find an entry at the given offset. + NoEntryAtGivenOffset, + /// The given offset is out of bounds. + OffsetOutOfBounds, + /// Found an unknown CFI augmentation. + UnknownAugmentation, + /// We do not support the given pointer encoding yet. + UnsupportedPointerEncoding, + /// Registers larger than `u16` are not supported. + UnsupportedRegister(u64), + /// The CFI program defined more register rules than we have storage for. + TooManyRegisterRules, + /// Attempted to push onto the CFI or evaluation stack, but it was already + /// at full capacity. + StackFull, + /// The `.eh_frame_hdr` binary search table claims to be variable-length encoded, + /// which makes binary search impossible. + VariableLengthSearchTable, + /// The `DW_UT_*` value for this unit is not supported yet. + UnsupportedUnitType, + /// Ranges using AddressIndex are not supported yet. + UnsupportedAddressIndex, + /// Nonzero segment selector sizes aren't supported yet. + UnsupportedSegmentSize, + /// A compilation unit or type unit is missing its top level DIE. + MissingUnitDie, + /// A DIE attribute used an unsupported form. + UnsupportedAttributeForm, + /// Missing DW_LNCT_path in file entry format. + MissingFileEntryFormatPath, + /// Expected an attribute value to be a string form. + ExpectedStringAttributeValue, + /// `DW_FORM_implicit_const` used in an invalid context. + InvalidImplicitConst, + /// Invalid section count in `.dwp` index. + InvalidIndexSectionCount, + /// Invalid slot count in `.dwp` index. + InvalidIndexSlotCount, + /// Invalid hash row in `.dwp` index. + InvalidIndexRow, + /// Unknown section type in `.dwp` index. + UnknownIndexSection, +} + +impl fmt::Display for Error { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter) -> ::core::result::Result<(), fmt::Error> { + write!(f, "{}", self.description()) + } +} + +impl Error { + /// A short description of the error. + pub fn description(&self) -> &str { + match *self { + Error::Io => "An I/O error occurred while reading.", + Error::PcRelativePointerButSectionBaseIsUndefined => { + "Found a PC relative pointer, but the section base is undefined." + } + Error::TextRelativePointerButTextBaseIsUndefined => { + "Found a `.text` relative pointer, but the `.text` base is undefined." + } + Error::DataRelativePointerButDataBaseIsUndefined => { + "Found a data relative pointer, but the data base is undefined." + } + Error::FuncRelativePointerInBadContext => { + "Found a function relative pointer in a context that does not have a function base." + } + Error::CannotParseOmitPointerEncoding => { + "Cannot parse a pointer with a `DW_EH_PE_omit` encoding." + } + Error::BadUnsignedLeb128 => "An error parsing an unsigned LEB128 value", + Error::BadSignedLeb128 => "An error parsing a signed LEB128 value", + Error::AbbreviationTagZero => { + "An abbreviation declared that its tag is zero, + but zero is reserved for null records" + } + Error::AttributeFormZero => { + "An attribute specification declared that its form is zero, + but zero is reserved for null records" + } + Error::BadHasChildren => { + "The abbreviation's has-children byte was not one of + `DW_CHILDREN_{yes,no}`" + } + Error::BadLength => "The specified length is impossible", + Error::UnknownForm => "Found an unknown `DW_FORM_*` type", + Error::ExpectedZero => "Expected a zero, found something else", + Error::DuplicateAbbreviationCode => { + "Found an abbreviation code that has already been used" + } + Error::DuplicateArange => "Found a duplicate arange", + Error::UnknownReservedLength => "Found an unknown reserved length value", + Error::UnknownVersion(_) => "Found an unknown DWARF version", + Error::UnknownAbbreviation => "Found a record with an unknown abbreviation code", + Error::UnexpectedEof(_) => "Hit the end of input before it was expected", + Error::UnexpectedNull => "Read a null entry before it was expected.", + Error::UnknownStandardOpcode(_) => "Found an unknown standard opcode", + Error::UnknownExtendedOpcode(_) => "Found an unknown extended opcode", + Error::UnsupportedAddressSize(_) => "The specified address size is not supported", + Error::UnsupportedOffsetSize(_) => "The specified offset size is not supported", + Error::UnsupportedFieldSize(_) => "The specified field size is not supported", + Error::MinimumInstructionLengthZero => { + "The minimum instruction length must not be zero." + } + Error::MaximumOperationsPerInstructionZero => { + "The maximum operations per instruction must not be zero." + } + Error::LineRangeZero => "The line range must not be zero.", + Error::OpcodeBaseZero => "The opcode base must not be zero.", + Error::BadUtf8 => "Found an invalid UTF-8 string.", + Error::NotCieId => "Expected to find the CIE ID, but found something else.", + Error::NotCiePointer => "Expected to find a CIE pointer, but found the CIE ID instead.", + Error::NotFdePointer => { + "Expected to find an FDE pointer, but found a CIE pointer instead." + } + Error::BadBranchTarget(_) => "Invalid branch target in DWARF expression", + Error::InvalidPushObjectAddress => { + "DW_OP_push_object_address used but no object address given" + } + Error::NotEnoughStackItems => "Not enough items on stack when evaluating expression", + Error::TooManyIterations => "Too many iterations to evaluate DWARF expression", + Error::InvalidExpression(_) => "Invalid opcode in DWARF expression", + Error::UnsupportedEvaluation => "Unsupported operation when evaluating expression", + Error::InvalidPiece => { + "DWARF expression has piece followed by non-piece expression at end" + } + Error::InvalidExpressionTerminator(_) => "Expected DW_OP_piece or DW_OP_bit_piece", + Error::DivisionByZero => "Division or modulus by zero when evaluating expression", + Error::TypeMismatch => "Type mismatch when evaluating expression", + Error::IntegralTypeRequired => "Integral type expected when evaluating expression", + Error::UnsupportedTypeOperation => { + "An expression operation used types that are not supported" + } + Error::InvalidShiftExpression => { + "The shift value in an expression must be a non-negative integer." + } + Error::UnknownCallFrameInstruction(_) => "An unknown DW_CFA_* instructiion", + Error::InvalidAddressRange => { + "The end of an address range must not be before the beginning." + } + Error::InvalidLocationAddressRange => { + "The end offset of a location list entry must not be before the beginning." + } + Error::CfiInstructionInInvalidContext => { + "Encountered a call frame instruction in a context in which it is not valid." + } + Error::PopWithEmptyStack => { + "When evaluating call frame instructions, found a `DW_CFA_restore_state` stack pop \ + instruction, but the stack was empty, and had nothing to pop." + } + Error::NoUnwindInfoForAddress => "Do not have unwind info for the given address.", + Error::UnsupportedOffset => { + "An offset value was larger than the maximum supported value." + } + Error::UnknownPointerEncoding => { + "The given pointer encoding is either unknown or invalid." + } + Error::NoEntryAtGivenOffset => "Did not find an entry at the given offset.", + Error::OffsetOutOfBounds => "The given offset is out of bounds.", + Error::UnknownAugmentation => "Found an unknown CFI augmentation.", + Error::UnsupportedPointerEncoding => { + "We do not support the given pointer encoding yet." + } + Error::UnsupportedRegister(_) => "Registers larger than `u16` are not supported.", + Error::TooManyRegisterRules => { + "The CFI program defined more register rules than we have storage for." + } + Error::StackFull => { + "Attempted to push onto the CFI stack, but it was already at full capacity." + } + Error::VariableLengthSearchTable => { + "The `.eh_frame_hdr` binary search table claims to be variable-length encoded, \ + which makes binary search impossible." + } + Error::UnsupportedUnitType => "The `DW_UT_*` value for this unit is not supported yet", + Error::UnsupportedAddressIndex => "Ranges involving AddressIndex are not supported yet", + Error::UnsupportedSegmentSize => "Nonzero segment size not supported yet", + Error::MissingUnitDie => { + "A compilation unit or type unit is missing its top level DIE." + } + Error::UnsupportedAttributeForm => "A DIE attribute used an unsupported form.", + Error::MissingFileEntryFormatPath => "Missing DW_LNCT_path in file entry format.", + Error::ExpectedStringAttributeValue => { + "Expected an attribute value to be a string form." + } + Error::InvalidImplicitConst => "DW_FORM_implicit_const used in an invalid context.", + Error::InvalidIndexSectionCount => "Invalid section count in `.dwp` index.", + Error::InvalidIndexSlotCount => "Invalid slot count in `.dwp` index.", + Error::InvalidIndexRow => "Invalid hash row in `.dwp` index.", + Error::UnknownIndexSection => "Unknown section type in `.dwp` index.", + } + } +} + +#[cfg(feature = "std")] +impl error::Error for Error {} + +#[cfg(feature = "std")] +impl From for Error { + fn from(_: io::Error) -> Self { + Error::Io + } +} + +/// The result of a parse. +pub type Result = result::Result; + +/// A convenience trait for loading DWARF sections from object files. To be +/// used like: +/// +/// ``` +/// use gimli::{DebugInfo, EndianSlice, LittleEndian, Reader, Section}; +/// +/// let buf = [0x00, 0x01, 0x02, 0x03]; +/// let reader = EndianSlice::new(&buf, LittleEndian); +/// let loader = |name| -> Result<_, ()> { Ok(reader) }; +/// +/// let debug_info: DebugInfo<_> = Section::load(loader).unwrap(); +/// ``` +pub trait Section: From { + /// Returns the section id for this type. + fn id() -> SectionId; + + /// Returns the ELF section name for this type. + fn section_name() -> &'static str { + Self::id().name() + } + + /// Returns the ELF section name (if any) for this type when used in a dwo + /// file. + fn dwo_section_name() -> Option<&'static str> { + Self::id().dwo_name() + } + + /// Try to load the section using the given loader function. + fn load(f: F) -> core::result::Result + where + F: FnOnce(SectionId) -> core::result::Result, + { + f(Self::id()).map(From::from) + } + + /// Returns the `Reader` for this section. + fn reader(&self) -> &R + where + R: Reader; + + /// Returns the subrange of the section that is the contribution of + /// a unit in a `.dwp` file. + fn dwp_range(&self, offset: u32, size: u32) -> Result + where + R: Reader, + { + let mut data = self.reader().clone(); + data.skip(R::Offset::from_u32(offset))?; + data.truncate(R::Offset::from_u32(size))?; + Ok(data.into()) + } + + /// Returns the `Reader` for this section. + fn lookup_offset_id(&self, id: ReaderOffsetId) -> Option<(SectionId, R::Offset)> + where + R: Reader, + { + self.reader() + .lookup_offset_id(id) + .map(|offset| (Self::id(), offset)) + } +} + +impl Register { + pub(crate) fn from_u64(x: u64) -> Result { + let y = x as u16; + if u64::from(y) == x { + Ok(Register(y)) + } else { + Err(Error::UnsupportedRegister(x)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::common::Format; + use crate::endianity::LittleEndian; + use test_assembler::{Endian, Section}; + + #[test] + fn test_parse_initial_length_32_ok() { + let section = Section::with_endian(Endian::Little).L32(0x7856_3412); + let buf = section.get_contents().unwrap(); + + let input = &mut EndianSlice::new(&buf, LittleEndian); + match input.read_initial_length() { + Ok((length, format)) => { + assert_eq!(input.len(), 0); + assert_eq!(format, Format::Dwarf32); + assert_eq!(0x7856_3412, length); + } + otherwise => panic!("Unexpected result: {:?}", otherwise), + } + } + + #[test] + fn test_parse_initial_length_64_ok() { + let section = Section::with_endian(Endian::Little) + // Dwarf_64_INITIAL_UNIT_LENGTH + .L32(0xffff_ffff) + // Actual length + .L64(0xffde_bc9a_7856_3412); + let buf = section.get_contents().unwrap(); + let input = &mut EndianSlice::new(&buf, LittleEndian); + + #[cfg(target_pointer_width = "64")] + match input.read_initial_length() { + Ok((length, format)) => { + assert_eq!(input.len(), 0); + assert_eq!(format, Format::Dwarf64); + assert_eq!(0xffde_bc9a_7856_3412, length); + } + otherwise => panic!("Unexpected result: {:?}", otherwise), + } + + #[cfg(target_pointer_width = "32")] + match input.read_initial_length() { + Err(Error::UnsupportedOffset) => {} + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + fn test_parse_initial_length_unknown_reserved_value() { + let section = Section::with_endian(Endian::Little).L32(0xffff_fffe); + let buf = section.get_contents().unwrap(); + + let input = &mut EndianSlice::new(&buf, LittleEndian); + match input.read_initial_length() { + Err(Error::UnknownReservedLength) => assert!(true), + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + fn test_parse_initial_length_incomplete() { + let buf = [0xff, 0xff, 0xff]; // Need at least 4 bytes. + + let input = &mut EndianSlice::new(&buf, LittleEndian); + match input.read_initial_length() { + Err(Error::UnexpectedEof(_)) => assert!(true), + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + fn test_parse_initial_length_64_incomplete() { + let section = Section::with_endian(Endian::Little) + // Dwarf_64_INITIAL_UNIT_LENGTH + .L32(0xffff_ffff) + // Actual length is not long enough. + .L32(0x7856_3412); + let buf = section.get_contents().unwrap(); + + let input = &mut EndianSlice::new(&buf, LittleEndian); + match input.read_initial_length() { + Err(Error::UnexpectedEof(_)) => assert!(true), + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + fn test_parse_offset_32() { + let section = Section::with_endian(Endian::Little).L32(0x0123_4567); + let buf = section.get_contents().unwrap(); + + let input = &mut EndianSlice::new(&buf, LittleEndian); + match input.read_offset(Format::Dwarf32) { + Ok(val) => { + assert_eq!(input.len(), 0); + assert_eq!(val, 0x0123_4567); + } + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + fn test_parse_offset_64_small() { + let section = Section::with_endian(Endian::Little).L64(0x0123_4567); + let buf = section.get_contents().unwrap(); + + let input = &mut EndianSlice::new(&buf, LittleEndian); + match input.read_offset(Format::Dwarf64) { + Ok(val) => { + assert_eq!(input.len(), 0); + assert_eq!(val, 0x0123_4567); + } + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_parse_offset_64_large() { + let section = Section::with_endian(Endian::Little).L64(0x0123_4567_89ab_cdef); + let buf = section.get_contents().unwrap(); + + let input = &mut EndianSlice::new(&buf, LittleEndian); + match input.read_offset(Format::Dwarf64) { + Ok(val) => { + assert_eq!(input.len(), 0); + assert_eq!(val, 0x0123_4567_89ab_cdef); + } + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + #[cfg(target_pointer_width = "32")] + fn test_parse_offset_64_large() { + let section = Section::with_endian(Endian::Little).L64(0x0123_4567_89ab_cdef); + let buf = section.get_contents().unwrap(); + + let input = &mut EndianSlice::new(&buf, LittleEndian); + match input.read_offset(Format::Dwarf64) { + Err(Error::UnsupportedOffset) => assert!(true), + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } +} diff --git a/crux-mir/lib/gimli/src/read/op.rs b/crux-mir/lib/gimli/src/read/op.rs new file mode 100644 index 000000000..88ea20297 --- /dev/null +++ b/crux-mir/lib/gimli/src/read/op.rs @@ -0,0 +1,4114 @@ +//! Functions for parsing and evaluating DWARF expressions. + +#[cfg(feature = "read")] +use alloc::vec::Vec; +use core::mem; + +use super::util::{ArrayLike, ArrayVec}; +use crate::common::{DebugAddrIndex, DebugInfoOffset, Encoding, Register}; +use crate::constants; +use crate::read::{Error, Reader, ReaderOffset, Result, StoreOnHeap, UnitOffset, Value, ValueType}; + +/// A reference to a DIE, either relative to the current CU or +/// relative to the section. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DieReference { + /// A CU-relative reference. + UnitRef(UnitOffset), + /// A section-relative reference. + DebugInfoRef(DebugInfoOffset), +} + +/// A single decoded DWARF expression operation. +/// +/// DWARF expression evaluation is done in two parts: first the raw +/// bytes of the next part of the expression are decoded; and then the +/// decoded operation is evaluated. This approach lets other +/// consumers inspect the DWARF expression without reimplementing the +/// decoding operation. +/// +/// Multiple DWARF opcodes may decode into a single `Operation`. For +/// example, both `DW_OP_deref` and `DW_OP_xderef` are represented +/// using `Operation::Deref`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Operation::Offset> +where + R: Reader, + Offset: ReaderOffset, +{ + /// Dereference the topmost value of the stack. + Deref { + /// The DIE of the base type or 0 to indicate the generic type + base_type: UnitOffset, + /// The size of the data to dereference. + size: u8, + /// True if the dereference operation takes an address space + /// argument from the stack; false otherwise. + space: bool, + }, + /// Drop an item from the stack. + Drop, + /// Pick an item from the stack and push it on top of the stack. + /// This operation handles `DW_OP_pick`, `DW_OP_dup`, and + /// `DW_OP_over`. + Pick { + /// The index, from the top of the stack, of the item to copy. + index: u8, + }, + /// Swap the top two stack items. + Swap, + /// Rotate the top three stack items. + Rot, + /// Take the absolute value of the top of the stack. + Abs, + /// Bitwise `and` of the top two values on the stack. + And, + /// Divide the top two values on the stack. + Div, + /// Subtract the top two values on the stack. + Minus, + /// Modulus of the top two values on the stack. + Mod, + /// Multiply the top two values on the stack. + Mul, + /// Negate the top of the stack. + Neg, + /// Bitwise `not` of the top of the stack. + Not, + /// Bitwise `or` of the top two values on the stack. + Or, + /// Add the top two values on the stack. + Plus, + /// Add a constant to the topmost value on the stack. + PlusConstant { + /// The value to add. + value: u64, + }, + /// Logical left shift of the 2nd value on the stack by the number + /// of bits given by the topmost value on the stack. + Shl, + /// Right shift of the 2nd value on the stack by the number of + /// bits given by the topmost value on the stack. + Shr, + /// Arithmetic left shift of the 2nd value on the stack by the + /// number of bits given by the topmost value on the stack. + Shra, + /// Bitwise `xor` of the top two values on the stack. + Xor, + /// Branch to the target location if the top of stack is nonzero. + Bra { + /// The relative offset to the target bytecode. + target: i16, + }, + /// Compare the top two stack values for equality. + Eq, + /// Compare the top two stack values using `>=`. + Ge, + /// Compare the top two stack values using `>`. + Gt, + /// Compare the top two stack values using `<=`. + Le, + /// Compare the top two stack values using `<`. + Lt, + /// Compare the top two stack values using `!=`. + Ne, + /// Unconditional branch to the target location. + Skip { + /// The relative offset to the target bytecode. + target: i16, + }, + /// Push an unsigned constant value on the stack. This handles multiple + /// DWARF opcodes. + UnsignedConstant { + /// The value to push. + value: u64, + }, + /// Push a signed constant value on the stack. This handles multiple + /// DWARF opcodes. + SignedConstant { + /// The value to push. + value: i64, + }, + /// Indicate that this piece's location is in the given register. + /// + /// Completes the piece or expression. + Register { + /// The register number. + register: Register, + }, + /// Find the value of the given register, add the offset, and then + /// push the resulting sum on the stack. + RegisterOffset { + /// The register number. + register: Register, + /// The offset to add. + offset: i64, + /// The DIE of the base type or 0 to indicate the generic type + base_type: UnitOffset, + }, + /// Compute the frame base (using `DW_AT_frame_base`), add the + /// given offset, and then push the resulting sum on the stack. + FrameOffset { + /// The offset to add. + offset: i64, + }, + /// No operation. + Nop, + /// Push the object address on the stack. + PushObjectAddress, + /// Evaluate a DWARF expression as a subroutine. The expression + /// comes from the `DW_AT_location` attribute of the indicated + /// DIE. + Call { + /// The DIE to use. + offset: DieReference, + }, + /// Compute the address of a thread-local variable and push it on + /// the stack. + TLS, + /// Compute the call frame CFA and push it on the stack. + CallFrameCFA, + /// Terminate a piece. + Piece { + /// The size of this piece in bits. + size_in_bits: u64, + /// The bit offset of this piece. If `None`, then this piece + /// was specified using `DW_OP_piece` and should start at the + /// next byte boundary. + bit_offset: Option, + }, + /// The object has no location, but has a known constant value. + /// + /// Represents `DW_OP_implicit_value`. + /// Completes the piece or expression. + ImplicitValue { + /// The implicit value to use. + data: R, + }, + /// The object has no location, but its value is at the top of the stack. + /// + /// Represents `DW_OP_stack_value`. + /// Completes the piece or expression. + StackValue, + /// The object is a pointer to a value which has no actual location, + /// such as an implicit value or a stack value. + /// + /// Represents `DW_OP_implicit_pointer`. + /// Completes the piece or expression. + ImplicitPointer { + /// The `.debug_info` offset of the value that this is an implicit pointer into. + value: DebugInfoOffset, + /// The byte offset into the value that the implicit pointer points to. + byte_offset: i64, + }, + /// Evaluate an expression at the entry to the current subprogram, and push it on the stack. + /// + /// Represents `DW_OP_entry_value`. + EntryValue { + /// The expression to be evaluated. + expression: R, + }, + /// This represents a parameter that was optimized out. + /// + /// The offset points to the definition of the parameter, and is + /// matched to the `DW_TAG_GNU_call_site_parameter` in the caller that also + /// points to the same definition of the parameter. + /// + /// Represents `DW_OP_GNU_parameter_ref`. + ParameterRef { + /// The DIE to use. + offset: UnitOffset, + }, + /// Relocate the address if needed, and push it on the stack. + /// + /// Represents `DW_OP_addr`. + Address { + /// The offset to add. + address: u64, + }, + /// Read the address at the given index in `.debug_addr, relocate the address if needed, + /// and push it on the stack. + /// + /// Represents `DW_OP_addrx`. + AddressIndex { + /// The index of the address in `.debug_addr`. + index: DebugAddrIndex, + }, + /// Read the address at the given index in `.debug_addr, and push it on the stack. + /// Do not relocate the address. + /// + /// Represents `DW_OP_constx`. + ConstantIndex { + /// The index of the address in `.debug_addr`. + index: DebugAddrIndex, + }, + /// Interpret the value bytes as a constant of a given type, and push it on the stack. + /// + /// Represents `DW_OP_const_type`. + TypedLiteral { + /// The DIE of the base type. + base_type: UnitOffset, + /// The value bytes. + value: R, + }, + /// Pop the top stack entry, convert it to a different type, and push it on the stack. + /// + /// Represents `DW_OP_convert`. + Convert { + /// The DIE of the base type. + base_type: UnitOffset, + }, + /// Pop the top stack entry, reinterpret the bits in its value as a different type, + /// and push it on the stack. + /// + /// Represents `DW_OP_reinterpret`. + Reinterpret { + /// The DIE of the base type. + base_type: UnitOffset, + }, + /// The index of a local in the currently executing function. + /// + /// Represents `DW_OP_WASM_location 0x00`. + /// Completes the piece or expression. + WasmLocal { + /// The index of the local. + index: u32, + }, + /// The index of a global. + /// + /// Represents `DW_OP_WASM_location 0x01` or `DW_OP_WASM_location 0x03`. + /// Completes the piece or expression. + WasmGlobal { + /// The index of the global. + index: u32, + }, + /// The index of an item on the operand stack. + /// + /// Represents `DW_OP_WASM_location 0x02`. + /// Completes the piece or expression. + WasmStack { + /// The index of the stack item. 0 is the bottom of the operand stack. + index: u32, + }, +} + +#[derive(Debug)] +enum OperationEvaluationResult { + Piece, + Incomplete, + Complete { location: Location }, + Waiting(EvaluationWaiting, EvaluationResult), +} + +/// A single location of a piece of the result of a DWARF expression. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Location::Offset> +where + R: Reader, + Offset: ReaderOffset, +{ + /// The piece is empty. Ordinarily this means the piece has been + /// optimized away. + Empty, + /// The piece is found in a register. + Register { + /// The register number. + register: Register, + }, + /// The piece is found in memory. + Address { + /// The address. + address: u64, + }, + /// The piece has no location but its value is known. + Value { + /// The value. + value: Value, + }, + /// The piece is represented by some constant bytes. + Bytes { + /// The value. + value: R, + }, + /// The piece is a pointer to a value which has no actual location. + ImplicitPointer { + /// The `.debug_info` offset of the value that this is an implicit pointer into. + value: DebugInfoOffset, + /// The byte offset into the value that the implicit pointer points to. + byte_offset: i64, + }, +} + +impl Location +where + R: Reader, + Offset: ReaderOffset, +{ + /// Return true if the piece is empty. + pub fn is_empty(&self) -> bool { + match *self { + Location::Empty => true, + _ => false, + } + } +} + +/// The description of a single piece of the result of a DWARF +/// expression. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Piece::Offset> +where + R: Reader, + Offset: ReaderOffset, +{ + /// If given, the size of the piece in bits. If `None`, there + /// must be only one piece whose size is all of the object. + pub size_in_bits: Option, + /// If given, the bit offset of the piece within the location. + /// If the location is a `Location::Register` or `Location::Value`, + /// then this offset is from the least significant bit end of + /// the register or value. + /// If the location is a `Location::Address` then the offset uses + /// the bit numbering and direction conventions of the language + /// and target system. + /// + /// If `None`, the piece starts at the location. If the + /// location is a register whose size is larger than the piece, + /// then placement within the register is defined by the ABI. + pub bit_offset: Option, + /// Where this piece is to be found. + pub location: Location, +} + +// A helper function to handle branch offsets. +fn compute_pc(pc: &R, bytecode: &R, offset: i16) -> Result { + let pc_offset = pc.offset_from(bytecode); + let new_pc_offset = pc_offset.wrapping_add(R::Offset::from_i16(offset)); + if new_pc_offset > bytecode.len() { + Err(Error::BadBranchTarget(new_pc_offset.into_u64())) + } else { + let mut new_pc = bytecode.clone(); + new_pc.skip(new_pc_offset)?; + Ok(new_pc) + } +} + +fn generic_type() -> UnitOffset { + UnitOffset(O::from_u64(0).unwrap()) +} + +impl Operation +where + R: Reader, + Offset: ReaderOffset, +{ + /// Parse a single DWARF expression operation. + /// + /// This is useful when examining a DWARF expression for reasons other + /// than direct evaluation. + /// + /// `bytes` points to a the operation to decode. It should point into + /// the same array as `bytecode`, which should be the entire + /// expression. + pub fn parse(bytes: &mut R, encoding: Encoding) -> Result> { + let opcode = bytes.read_u8()?; + let name = constants::DwOp(opcode); + match name { + constants::DW_OP_addr => { + let address = bytes.read_address(encoding.address_size)?; + Ok(Operation::Address { address }) + } + constants::DW_OP_deref => Ok(Operation::Deref { + base_type: generic_type(), + size: encoding.address_size, + space: false, + }), + constants::DW_OP_const1u => { + let value = bytes.read_u8()?; + Ok(Operation::UnsignedConstant { + value: u64::from(value), + }) + } + constants::DW_OP_const1s => { + let value = bytes.read_i8()?; + Ok(Operation::SignedConstant { + value: i64::from(value), + }) + } + constants::DW_OP_const2u => { + let value = bytes.read_u16()?; + Ok(Operation::UnsignedConstant { + value: u64::from(value), + }) + } + constants::DW_OP_const2s => { + let value = bytes.read_i16()?; + Ok(Operation::SignedConstant { + value: i64::from(value), + }) + } + constants::DW_OP_const4u => { + let value = bytes.read_u32()?; + Ok(Operation::UnsignedConstant { + value: u64::from(value), + }) + } + constants::DW_OP_const4s => { + let value = bytes.read_i32()?; + Ok(Operation::SignedConstant { + value: i64::from(value), + }) + } + constants::DW_OP_const8u => { + let value = bytes.read_u64()?; + Ok(Operation::UnsignedConstant { value }) + } + constants::DW_OP_const8s => { + let value = bytes.read_i64()?; + Ok(Operation::SignedConstant { value }) + } + constants::DW_OP_constu => { + let value = bytes.read_uleb128()?; + Ok(Operation::UnsignedConstant { value }) + } + constants::DW_OP_consts => { + let value = bytes.read_sleb128()?; + Ok(Operation::SignedConstant { value }) + } + constants::DW_OP_dup => Ok(Operation::Pick { index: 0 }), + constants::DW_OP_drop => Ok(Operation::Drop), + constants::DW_OP_over => Ok(Operation::Pick { index: 1 }), + constants::DW_OP_pick => { + let value = bytes.read_u8()?; + Ok(Operation::Pick { index: value }) + } + constants::DW_OP_swap => Ok(Operation::Swap), + constants::DW_OP_rot => Ok(Operation::Rot), + constants::DW_OP_xderef => Ok(Operation::Deref { + base_type: generic_type(), + size: encoding.address_size, + space: true, + }), + constants::DW_OP_abs => Ok(Operation::Abs), + constants::DW_OP_and => Ok(Operation::And), + constants::DW_OP_div => Ok(Operation::Div), + constants::DW_OP_minus => Ok(Operation::Minus), + constants::DW_OP_mod => Ok(Operation::Mod), + constants::DW_OP_mul => Ok(Operation::Mul), + constants::DW_OP_neg => Ok(Operation::Neg), + constants::DW_OP_not => Ok(Operation::Not), + constants::DW_OP_or => Ok(Operation::Or), + constants::DW_OP_plus => Ok(Operation::Plus), + constants::DW_OP_plus_uconst => { + let value = bytes.read_uleb128()?; + Ok(Operation::PlusConstant { value }) + } + constants::DW_OP_shl => Ok(Operation::Shl), + constants::DW_OP_shr => Ok(Operation::Shr), + constants::DW_OP_shra => Ok(Operation::Shra), + constants::DW_OP_xor => Ok(Operation::Xor), + constants::DW_OP_bra => { + let target = bytes.read_i16()?; + Ok(Operation::Bra { target }) + } + constants::DW_OP_eq => Ok(Operation::Eq), + constants::DW_OP_ge => Ok(Operation::Ge), + constants::DW_OP_gt => Ok(Operation::Gt), + constants::DW_OP_le => Ok(Operation::Le), + constants::DW_OP_lt => Ok(Operation::Lt), + constants::DW_OP_ne => Ok(Operation::Ne), + constants::DW_OP_skip => { + let target = bytes.read_i16()?; + Ok(Operation::Skip { target }) + } + constants::DW_OP_lit0 + | constants::DW_OP_lit1 + | constants::DW_OP_lit2 + | constants::DW_OP_lit3 + | constants::DW_OP_lit4 + | constants::DW_OP_lit5 + | constants::DW_OP_lit6 + | constants::DW_OP_lit7 + | constants::DW_OP_lit8 + | constants::DW_OP_lit9 + | constants::DW_OP_lit10 + | constants::DW_OP_lit11 + | constants::DW_OP_lit12 + | constants::DW_OP_lit13 + | constants::DW_OP_lit14 + | constants::DW_OP_lit15 + | constants::DW_OP_lit16 + | constants::DW_OP_lit17 + | constants::DW_OP_lit18 + | constants::DW_OP_lit19 + | constants::DW_OP_lit20 + | constants::DW_OP_lit21 + | constants::DW_OP_lit22 + | constants::DW_OP_lit23 + | constants::DW_OP_lit24 + | constants::DW_OP_lit25 + | constants::DW_OP_lit26 + | constants::DW_OP_lit27 + | constants::DW_OP_lit28 + | constants::DW_OP_lit29 + | constants::DW_OP_lit30 + | constants::DW_OP_lit31 => Ok(Operation::UnsignedConstant { + value: (opcode - constants::DW_OP_lit0.0).into(), + }), + constants::DW_OP_reg0 + | constants::DW_OP_reg1 + | constants::DW_OP_reg2 + | constants::DW_OP_reg3 + | constants::DW_OP_reg4 + | constants::DW_OP_reg5 + | constants::DW_OP_reg6 + | constants::DW_OP_reg7 + | constants::DW_OP_reg8 + | constants::DW_OP_reg9 + | constants::DW_OP_reg10 + | constants::DW_OP_reg11 + | constants::DW_OP_reg12 + | constants::DW_OP_reg13 + | constants::DW_OP_reg14 + | constants::DW_OP_reg15 + | constants::DW_OP_reg16 + | constants::DW_OP_reg17 + | constants::DW_OP_reg18 + | constants::DW_OP_reg19 + | constants::DW_OP_reg20 + | constants::DW_OP_reg21 + | constants::DW_OP_reg22 + | constants::DW_OP_reg23 + | constants::DW_OP_reg24 + | constants::DW_OP_reg25 + | constants::DW_OP_reg26 + | constants::DW_OP_reg27 + | constants::DW_OP_reg28 + | constants::DW_OP_reg29 + | constants::DW_OP_reg30 + | constants::DW_OP_reg31 => Ok(Operation::Register { + register: Register((opcode - constants::DW_OP_reg0.0).into()), + }), + constants::DW_OP_breg0 + | constants::DW_OP_breg1 + | constants::DW_OP_breg2 + | constants::DW_OP_breg3 + | constants::DW_OP_breg4 + | constants::DW_OP_breg5 + | constants::DW_OP_breg6 + | constants::DW_OP_breg7 + | constants::DW_OP_breg8 + | constants::DW_OP_breg9 + | constants::DW_OP_breg10 + | constants::DW_OP_breg11 + | constants::DW_OP_breg12 + | constants::DW_OP_breg13 + | constants::DW_OP_breg14 + | constants::DW_OP_breg15 + | constants::DW_OP_breg16 + | constants::DW_OP_breg17 + | constants::DW_OP_breg18 + | constants::DW_OP_breg19 + | constants::DW_OP_breg20 + | constants::DW_OP_breg21 + | constants::DW_OP_breg22 + | constants::DW_OP_breg23 + | constants::DW_OP_breg24 + | constants::DW_OP_breg25 + | constants::DW_OP_breg26 + | constants::DW_OP_breg27 + | constants::DW_OP_breg28 + | constants::DW_OP_breg29 + | constants::DW_OP_breg30 + | constants::DW_OP_breg31 => { + let value = bytes.read_sleb128()?; + Ok(Operation::RegisterOffset { + register: Register((opcode - constants::DW_OP_breg0.0).into()), + offset: value, + base_type: generic_type(), + }) + } + constants::DW_OP_regx => { + let register = bytes.read_uleb128().and_then(Register::from_u64)?; + Ok(Operation::Register { register }) + } + constants::DW_OP_fbreg => { + let value = bytes.read_sleb128()?; + Ok(Operation::FrameOffset { offset: value }) + } + constants::DW_OP_bregx => { + let register = bytes.read_uleb128().and_then(Register::from_u64)?; + let offset = bytes.read_sleb128()?; + Ok(Operation::RegisterOffset { + register, + offset, + base_type: generic_type(), + }) + } + constants::DW_OP_piece => { + let size = bytes.read_uleb128()?; + Ok(Operation::Piece { + size_in_bits: 8 * size, + bit_offset: None, + }) + } + constants::DW_OP_deref_size => { + let size = bytes.read_u8()?; + Ok(Operation::Deref { + base_type: generic_type(), + size, + space: false, + }) + } + constants::DW_OP_xderef_size => { + let size = bytes.read_u8()?; + Ok(Operation::Deref { + base_type: generic_type(), + size, + space: true, + }) + } + constants::DW_OP_nop => Ok(Operation::Nop), + constants::DW_OP_push_object_address => Ok(Operation::PushObjectAddress), + constants::DW_OP_call2 => { + let value = bytes.read_u16().map(R::Offset::from_u16)?; + Ok(Operation::Call { + offset: DieReference::UnitRef(UnitOffset(value)), + }) + } + constants::DW_OP_call4 => { + let value = bytes.read_u32().map(R::Offset::from_u32)?; + Ok(Operation::Call { + offset: DieReference::UnitRef(UnitOffset(value)), + }) + } + constants::DW_OP_call_ref => { + let value = bytes.read_offset(encoding.format)?; + Ok(Operation::Call { + offset: DieReference::DebugInfoRef(DebugInfoOffset(value)), + }) + } + constants::DW_OP_form_tls_address | constants::DW_OP_GNU_push_tls_address => { + Ok(Operation::TLS) + } + constants::DW_OP_call_frame_cfa => Ok(Operation::CallFrameCFA), + constants::DW_OP_bit_piece => { + let size = bytes.read_uleb128()?; + let offset = bytes.read_uleb128()?; + Ok(Operation::Piece { + size_in_bits: size, + bit_offset: Some(offset), + }) + } + constants::DW_OP_implicit_value => { + let len = bytes.read_uleb128().and_then(R::Offset::from_u64)?; + let data = bytes.split(len)?; + Ok(Operation::ImplicitValue { data }) + } + constants::DW_OP_stack_value => Ok(Operation::StackValue), + constants::DW_OP_implicit_pointer | constants::DW_OP_GNU_implicit_pointer => { + let value = if encoding.version == 2 { + bytes + .read_address(encoding.address_size) + .and_then(Offset::from_u64)? + } else { + bytes.read_offset(encoding.format)? + }; + let byte_offset = bytes.read_sleb128()?; + Ok(Operation::ImplicitPointer { + value: DebugInfoOffset(value), + byte_offset, + }) + } + constants::DW_OP_addrx | constants::DW_OP_GNU_addr_index => { + let index = bytes.read_uleb128().and_then(R::Offset::from_u64)?; + Ok(Operation::AddressIndex { + index: DebugAddrIndex(index), + }) + } + constants::DW_OP_constx | constants::DW_OP_GNU_const_index => { + let index = bytes.read_uleb128().and_then(R::Offset::from_u64)?; + Ok(Operation::ConstantIndex { + index: DebugAddrIndex(index), + }) + } + constants::DW_OP_entry_value | constants::DW_OP_GNU_entry_value => { + let len = bytes.read_uleb128().and_then(R::Offset::from_u64)?; + let expression = bytes.split(len)?; + Ok(Operation::EntryValue { expression }) + } + constants::DW_OP_GNU_parameter_ref => { + let value = bytes.read_u32().map(R::Offset::from_u32)?; + Ok(Operation::ParameterRef { + offset: UnitOffset(value), + }) + } + constants::DW_OP_const_type | constants::DW_OP_GNU_const_type => { + let base_type = bytes.read_uleb128().and_then(R::Offset::from_u64)?; + let len = bytes.read_u8()?; + let value = bytes.split(R::Offset::from_u8(len))?; + Ok(Operation::TypedLiteral { + base_type: UnitOffset(base_type), + value, + }) + } + constants::DW_OP_regval_type | constants::DW_OP_GNU_regval_type => { + let register = bytes.read_uleb128().and_then(Register::from_u64)?; + let base_type = bytes.read_uleb128().and_then(R::Offset::from_u64)?; + Ok(Operation::RegisterOffset { + register, + offset: 0, + base_type: UnitOffset(base_type), + }) + } + constants::DW_OP_deref_type | constants::DW_OP_GNU_deref_type => { + let size = bytes.read_u8()?; + let base_type = bytes.read_uleb128().and_then(R::Offset::from_u64)?; + Ok(Operation::Deref { + base_type: UnitOffset(base_type), + size, + space: false, + }) + } + constants::DW_OP_xderef_type => { + let size = bytes.read_u8()?; + let base_type = bytes.read_uleb128().and_then(R::Offset::from_u64)?; + Ok(Operation::Deref { + base_type: UnitOffset(base_type), + size, + space: true, + }) + } + constants::DW_OP_convert | constants::DW_OP_GNU_convert => { + let base_type = bytes.read_uleb128().and_then(R::Offset::from_u64)?; + Ok(Operation::Convert { + base_type: UnitOffset(base_type), + }) + } + constants::DW_OP_reinterpret | constants::DW_OP_GNU_reinterpret => { + let base_type = bytes.read_uleb128().and_then(R::Offset::from_u64)?; + Ok(Operation::Reinterpret { + base_type: UnitOffset(base_type), + }) + } + constants::DW_OP_WASM_location => match bytes.read_u8()? { + 0x0 => { + let index = bytes.read_uleb128_u32()?; + Ok(Operation::WasmLocal { index }) + } + 0x1 => { + let index = bytes.read_uleb128_u32()?; + Ok(Operation::WasmGlobal { index }) + } + 0x2 => { + let index = bytes.read_uleb128_u32()?; + Ok(Operation::WasmStack { index }) + } + 0x3 => { + let index = bytes.read_u32()?; + Ok(Operation::WasmGlobal { index }) + } + _ => Err(Error::InvalidExpression(name)), + }, + _ => Err(Error::InvalidExpression(name)), + } + } +} + +#[derive(Debug)] +enum EvaluationState { + Start(Option), + Ready, + Error(Error), + Complete, + Waiting(EvaluationWaiting), +} + +#[derive(Debug)] +enum EvaluationWaiting { + Memory, + Register { offset: i64 }, + FrameBase { offset: i64 }, + Tls, + Cfa, + AtLocation, + EntryValue, + ParameterRef, + RelocatedAddress, + IndexedAddress, + TypedLiteral { value: R }, + Convert, + Reinterpret, +} + +/// The state of an `Evaluation` after evaluating a DWARF expression. +/// The evaluation is either `Complete`, or it requires more data +/// to continue, as described by the variant. +#[derive(Debug, PartialEq)] +pub enum EvaluationResult { + /// The `Evaluation` is complete, and `Evaluation::result()` can be called. + Complete, + /// The `Evaluation` needs a value from memory to proceed further. Once the + /// caller determines what value to provide it should resume the `Evaluation` + /// by calling `Evaluation::resume_with_memory`. + RequiresMemory { + /// The address of the value required. + address: u64, + /// The size of the value required. This is guaranteed to be at most the + /// word size of the target architecture. + size: u8, + /// If not `None`, a target-specific address space value. + space: Option, + /// The DIE of the base type or 0 to indicate the generic type + base_type: UnitOffset, + }, + /// The `Evaluation` needs a value from a register to proceed further. Once + /// the caller determines what value to provide it should resume the + /// `Evaluation` by calling `Evaluation::resume_with_register`. + RequiresRegister { + /// The register number. + register: Register, + /// The DIE of the base type or 0 to indicate the generic type + base_type: UnitOffset, + }, + /// The `Evaluation` needs the frame base address to proceed further. Once + /// the caller determines what value to provide it should resume the + /// `Evaluation` by calling `Evaluation::resume_with_frame_base`. The frame + /// base address is the address produced by the location description in the + /// `DW_AT_frame_base` attribute of the current function. + RequiresFrameBase, + /// The `Evaluation` needs a value from TLS to proceed further. Once the + /// caller determines what value to provide it should resume the + /// `Evaluation` by calling `Evaluation::resume_with_tls`. + RequiresTls(u64), + /// The `Evaluation` needs the CFA to proceed further. Once the caller + /// determines what value to provide it should resume the `Evaluation` by + /// calling `Evaluation::resume_with_call_frame_cfa`. + RequiresCallFrameCfa, + /// The `Evaluation` needs the DWARF expression at the given location to + /// proceed further. Once the caller determines what value to provide it + /// should resume the `Evaluation` by calling + /// `Evaluation::resume_with_at_location`. + RequiresAtLocation(DieReference), + /// The `Evaluation` needs the value produced by evaluating a DWARF + /// expression at the entry point of the current subprogram. Once the + /// caller determines what value to provide it should resume the + /// `Evaluation` by calling `Evaluation::resume_with_entry_value`. + RequiresEntryValue(Expression), + /// The `Evaluation` needs the value of the parameter at the given location + /// in the current function's caller. Once the caller determines what value + /// to provide it should resume the `Evaluation` by calling + /// `Evaluation::resume_with_parameter_ref`. + RequiresParameterRef(UnitOffset), + /// The `Evaluation` needs an address to be relocated to proceed further. + /// Once the caller determines what value to provide it should resume the + /// `Evaluation` by calling `Evaluation::resume_with_relocated_address`. + RequiresRelocatedAddress(u64), + /// The `Evaluation` needs an address from the `.debug_addr` section. + /// This address may also need to be relocated. + /// Once the caller determines what value to provide it should resume the + /// `Evaluation` by calling `Evaluation::resume_with_indexed_address`. + RequiresIndexedAddress { + /// The index of the address in the `.debug_addr` section, + /// relative to the `DW_AT_addr_base` of the compilation unit. + index: DebugAddrIndex, + /// Whether the address also needs to be relocated. + relocate: bool, + }, + /// The `Evaluation` needs the `ValueType` for the base type DIE at + /// the give unit offset. Once the caller determines what value to provide it + /// should resume the `Evaluation` by calling + /// `Evaluation::resume_with_base_type`. + RequiresBaseType(UnitOffset), +} + +/// The bytecode for a DWARF expression or location description. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Expression(pub R); + +impl Expression { + /// Create an evaluation for this expression. + /// + /// The `encoding` is determined by the + /// [`CompilationUnitHeader`](struct.CompilationUnitHeader.html) or + /// [`TypeUnitHeader`](struct.TypeUnitHeader.html) that this expression + /// relates to. + /// + /// # Examples + /// ```rust,no_run + /// use gimli::Expression; + /// # let endian = gimli::LittleEndian; + /// # let debug_info = gimli::DebugInfo::from(gimli::EndianSlice::new(&[], endian)); + /// # let unit = debug_info.units().next().unwrap().unwrap(); + /// # let bytecode = gimli::EndianSlice::new(&[], endian); + /// let expression = gimli::Expression(bytecode); + /// let mut eval = expression.evaluation(unit.encoding()); + /// let mut result = eval.evaluate().unwrap(); + /// ``` + #[cfg(feature = "read")] + #[inline] + pub fn evaluation(self, encoding: Encoding) -> Evaluation { + Evaluation::new(self.0, encoding) + } + + /// Return an iterator for the operations in the expression. + pub fn operations(self, encoding: Encoding) -> OperationIter { + OperationIter { + input: self.0, + encoding, + } + } +} + +/// An iterator for the operations in an expression. +#[derive(Debug, Clone, Copy)] +pub struct OperationIter { + input: R, + encoding: Encoding, +} + +impl OperationIter { + /// Read the next operation in an expression. + pub fn next(&mut self) -> Result>> { + if self.input.is_empty() { + return Ok(None); + } + match Operation::parse(&mut self.input, self.encoding) { + Ok(op) => Ok(Some(op)), + Err(e) => { + self.input.empty(); + Err(e) + } + } + } + + /// Return the current byte offset of the iterator. + pub fn offset_from(&self, expression: &Expression) -> R::Offset { + self.input.offset_from(&expression.0) + } +} + +/// Specification of what storage should be used for [`Evaluation`]. +/// +#[cfg_attr( + feature = "read", + doc = " +Normally you would only need to use [`StoreOnHeap`], which places the stacks and the results +on the heap using [`Vec`]. This is the default storage type parameter for [`Evaluation`]. +" +)] +/// +/// If you need to avoid [`Evaluation`] from allocating memory, e.g. for signal safety, +/// you can provide you own storage specification: +/// ```rust,no_run +/// # use gimli::*; +/// # let bytecode = EndianSlice::new(&[], LittleEndian); +/// # let encoding = unimplemented!(); +/// # let get_register_value = |_, _| Value::Generic(42); +/// # let get_frame_base = || 0xdeadbeef; +/// # +/// struct StoreOnStack; +/// +/// impl EvaluationStorage for StoreOnStack { +/// type Stack = [Value; 64]; +/// type ExpressionStack = [(R, R); 4]; +/// type Result = [Piece; 1]; +/// } +/// +/// let mut eval = Evaluation::<_, StoreOnStack>::new_in(bytecode, encoding); +/// let mut result = eval.evaluate().unwrap(); +/// while result != EvaluationResult::Complete { +/// match result { +/// EvaluationResult::RequiresRegister { register, base_type } => { +/// let value = get_register_value(register, base_type); +/// result = eval.resume_with_register(value).unwrap(); +/// }, +/// EvaluationResult::RequiresFrameBase => { +/// let frame_base = get_frame_base(); +/// result = eval.resume_with_frame_base(frame_base).unwrap(); +/// }, +/// _ => unimplemented!(), +/// }; +/// } +/// +/// let result = eval.as_result(); +/// println!("{:?}", result); +/// ``` +pub trait EvaluationStorage { + /// The storage used for the evaluation stack. + type Stack: ArrayLike; + /// The storage used for the expression stack. + type ExpressionStack: ArrayLike; + /// The storage used for the results. + type Result: ArrayLike>; +} + +#[cfg(feature = "read")] +impl EvaluationStorage for StoreOnHeap { + type Stack = Vec; + type ExpressionStack = Vec<(R, R)>; + type Result = Vec>; +} + +/// A DWARF expression evaluator. +/// +/// # Usage +/// A DWARF expression may require additional data to produce a final result, +/// such as the value of a register or a memory location. Once initial setup +/// is complete (i.e. `set_initial_value()`, `set_object_address()`) the +/// consumer calls the `evaluate()` method. That returns an `EvaluationResult`, +/// which is either `EvaluationResult::Complete` or a value indicating what +/// data is needed to resume the `Evaluation`. The consumer is responsible for +/// producing that data and resuming the computation with the correct method, +/// as documented for `EvaluationResult`. Only once an `EvaluationResult::Complete` +/// is returned can the consumer call `result()`. +/// +/// This design allows the consumer of `Evaluation` to decide how and when to +/// produce the required data and resume the computation. The `Evaluation` can +/// be driven synchronously (as shown below) or by some asynchronous mechanism +/// such as futures. +/// +/// # Examples +/// ```rust,no_run +/// use gimli::{EndianSlice, Evaluation, EvaluationResult, Format, LittleEndian, Value}; +/// # let bytecode = EndianSlice::new(&[], LittleEndian); +/// # let encoding = unimplemented!(); +/// # let get_register_value = |_, _| Value::Generic(42); +/// # let get_frame_base = || 0xdeadbeef; +/// +/// let mut eval = Evaluation::new(bytecode, encoding); +/// let mut result = eval.evaluate().unwrap(); +/// while result != EvaluationResult::Complete { +/// match result { +/// EvaluationResult::RequiresRegister { register, base_type } => { +/// let value = get_register_value(register, base_type); +/// result = eval.resume_with_register(value).unwrap(); +/// }, +/// EvaluationResult::RequiresFrameBase => { +/// let frame_base = get_frame_base(); +/// result = eval.resume_with_frame_base(frame_base).unwrap(); +/// }, +/// _ => unimplemented!(), +/// }; +/// } +/// +/// let result = eval.result(); +/// println!("{:?}", result); +/// ``` +#[derive(Debug)] +pub struct Evaluation = StoreOnHeap> { + bytecode: R, + encoding: Encoding, + object_address: Option, + max_iterations: Option, + iteration: u32, + state: EvaluationState, + + // Stack operations are done on word-sized values. We do all + // operations on 64-bit values, and then mask the results + // appropriately when popping. + addr_mask: u64, + + // The stack. + stack: ArrayVec, + + // The next operation to decode and evaluate. + pc: R, + + // If we see a DW_OP_call* operation, the previous PC and bytecode + // is stored here while evaluating the subroutine. + expression_stack: ArrayVec, + + result: ArrayVec, +} + +#[cfg(feature = "read")] +impl Evaluation { + /// Create a new DWARF expression evaluator. + /// + /// The new evaluator is created without an initial value, without + /// an object address, and without a maximum number of iterations. + pub fn new(bytecode: R, encoding: Encoding) -> Self { + Self::new_in(bytecode, encoding) + } + + /// Get the result of this `Evaluation`. + /// + /// # Panics + /// Panics if this `Evaluation` has not been driven to completion. + pub fn result(self) -> Vec> { + match self.state { + EvaluationState::Complete => self.result.into_vec(), + _ => { + panic!("Called `Evaluation::result` on an `Evaluation` that has not been completed") + } + } + } +} + +impl> Evaluation { + /// Create a new DWARF expression evaluator. + /// + /// The new evaluator is created without an initial value, without + /// an object address, and without a maximum number of iterations. + pub fn new_in(bytecode: R, encoding: Encoding) -> Self { + let pc = bytecode.clone(); + Evaluation { + bytecode, + encoding, + object_address: None, + max_iterations: None, + iteration: 0, + state: EvaluationState::Start(None), + addr_mask: if encoding.address_size == 8 { + !0u64 + } else { + (1 << (8 * u64::from(encoding.address_size))) - 1 + }, + stack: Default::default(), + expression_stack: Default::default(), + pc, + result: Default::default(), + } + } + + /// Set an initial value to be pushed on the DWARF expression + /// evaluator's stack. This can be used in cases like + /// `DW_AT_vtable_elem_location`, which require a value on the + /// stack before evaluation commences. If no initial value is + /// set, and the expression uses an opcode requiring the initial + /// value, then evaluation will fail with an error. + /// + /// # Panics + /// Panics if `set_initial_value()` has already been called, or if + /// `evaluate()` has already been called. + pub fn set_initial_value(&mut self, value: u64) { + match self.state { + EvaluationState::Start(None) => { + self.state = EvaluationState::Start(Some(value)); + } + _ => panic!( + "`Evaluation::set_initial_value` was called twice, or after evaluation began." + ), + }; + } + + /// Set the enclosing object's address, as used by + /// `DW_OP_push_object_address`. If no object address is set, and + /// the expression uses an opcode requiring the object address, + /// then evaluation will fail with an error. + pub fn set_object_address(&mut self, value: u64) { + self.object_address = Some(value); + } + + /// Set the maximum number of iterations to be allowed by the + /// expression evaluator. + /// + /// An iteration corresponds approximately to the evaluation of a + /// single operation in an expression ("approximately" because the + /// implementation may allow two such operations in some cases). + /// The default is not to have a maximum; once set, it's not + /// possible to go back to this default state. This value can be + /// set to avoid denial of service attacks by bad DWARF bytecode. + pub fn set_max_iterations(&mut self, value: u32) { + self.max_iterations = Some(value); + } + + fn pop(&mut self) -> Result { + match self.stack.pop() { + Some(value) => Ok(value), + None => Err(Error::NotEnoughStackItems), + } + } + + fn push(&mut self, value: Value) -> Result<()> { + self.stack.try_push(value).map_err(|_| Error::StackFull) + } + + #[allow(clippy::cyclomatic_complexity)] + fn evaluate_one_operation(&mut self) -> Result> { + let operation = Operation::parse(&mut self.pc, self.encoding)?; + + match operation { + Operation::Deref { + base_type, + size, + space, + } => { + let entry = self.pop()?; + let addr = entry.to_u64(self.addr_mask)?; + let addr_space = if space { + let entry = self.pop()?; + let value = entry.to_u64(self.addr_mask)?; + Some(value) + } else { + None + }; + return Ok(OperationEvaluationResult::Waiting( + EvaluationWaiting::Memory, + EvaluationResult::RequiresMemory { + address: addr, + size, + space: addr_space, + base_type, + }, + )); + } + + Operation::Drop => { + self.pop()?; + } + Operation::Pick { index } => { + let len = self.stack.len(); + let index = index as usize; + if index >= len { + return Err(Error::NotEnoughStackItems); + } + let value = self.stack[len - index - 1]; + self.push(value)?; + } + Operation::Swap => { + let top = self.pop()?; + let next = self.pop()?; + self.push(top)?; + self.push(next)?; + } + Operation::Rot => { + let one = self.pop()?; + let two = self.pop()?; + let three = self.pop()?; + self.push(one)?; + self.push(three)?; + self.push(two)?; + } + + Operation::Abs => { + let value = self.pop()?; + let result = value.abs(self.addr_mask)?; + self.push(result)?; + } + Operation::And => { + let rhs = self.pop()?; + let lhs = self.pop()?; + let result = lhs.and(rhs, self.addr_mask)?; + self.push(result)?; + } + Operation::Div => { + let rhs = self.pop()?; + let lhs = self.pop()?; + let result = lhs.div(rhs, self.addr_mask)?; + self.push(result)?; + } + Operation::Minus => { + let rhs = self.pop()?; + let lhs = self.pop()?; + let result = lhs.sub(rhs, self.addr_mask)?; + self.push(result)?; + } + Operation::Mod => { + let rhs = self.pop()?; + let lhs = self.pop()?; + let result = lhs.rem(rhs, self.addr_mask)?; + self.push(result)?; + } + Operation::Mul => { + let rhs = self.pop()?; + let lhs = self.pop()?; + let result = lhs.mul(rhs, self.addr_mask)?; + self.push(result)?; + } + Operation::Neg => { + let v = self.pop()?; + let result = v.neg(self.addr_mask)?; + self.push(result)?; + } + Operation::Not => { + let value = self.pop()?; + let result = value.not(self.addr_mask)?; + self.push(result)?; + } + Operation::Or => { + let rhs = self.pop()?; + let lhs = self.pop()?; + let result = lhs.or(rhs, self.addr_mask)?; + self.push(result)?; + } + Operation::Plus => { + let rhs = self.pop()?; + let lhs = self.pop()?; + let result = lhs.add(rhs, self.addr_mask)?; + self.push(result)?; + } + Operation::PlusConstant { value } => { + let lhs = self.pop()?; + let rhs = Value::from_u64(lhs.value_type(), value)?; + let result = lhs.add(rhs, self.addr_mask)?; + self.push(result)?; + } + Operation::Shl => { + let rhs = self.pop()?; + let lhs = self.pop()?; + let result = lhs.shl(rhs, self.addr_mask)?; + self.push(result)?; + } + Operation::Shr => { + let rhs = self.pop()?; + let lhs = self.pop()?; + let result = lhs.shr(rhs, self.addr_mask)?; + self.push(result)?; + } + Operation::Shra => { + let rhs = self.pop()?; + let lhs = self.pop()?; + let result = lhs.shra(rhs, self.addr_mask)?; + self.push(result)?; + } + Operation::Xor => { + let rhs = self.pop()?; + let lhs = self.pop()?; + let result = lhs.xor(rhs, self.addr_mask)?; + self.push(result)?; + } + + Operation::Bra { target } => { + let entry = self.pop()?; + let v = entry.to_u64(self.addr_mask)?; + if v != 0 { + self.pc = compute_pc(&self.pc, &self.bytecode, target)?; + } + } + + Operation::Eq => { + let rhs = self.pop()?; + let lhs = self.pop()?; + let result = lhs.eq(rhs, self.addr_mask)?; + self.push(result)?; + } + Operation::Ge => { + let rhs = self.pop()?; + let lhs = self.pop()?; + let result = lhs.ge(rhs, self.addr_mask)?; + self.push(result)?; + } + Operation::Gt => { + let rhs = self.pop()?; + let lhs = self.pop()?; + let result = lhs.gt(rhs, self.addr_mask)?; + self.push(result)?; + } + Operation::Le => { + let rhs = self.pop()?; + let lhs = self.pop()?; + let result = lhs.le(rhs, self.addr_mask)?; + self.push(result)?; + } + Operation::Lt => { + let rhs = self.pop()?; + let lhs = self.pop()?; + let result = lhs.lt(rhs, self.addr_mask)?; + self.push(result)?; + } + Operation::Ne => { + let rhs = self.pop()?; + let lhs = self.pop()?; + let result = lhs.ne(rhs, self.addr_mask)?; + self.push(result)?; + } + + Operation::Skip { target } => { + self.pc = compute_pc(&self.pc, &self.bytecode, target)?; + } + + Operation::UnsignedConstant { value } => { + self.push(Value::Generic(value))?; + } + + Operation::SignedConstant { value } => { + self.push(Value::Generic(value as u64))?; + } + + Operation::RegisterOffset { + register, + offset, + base_type, + } => { + return Ok(OperationEvaluationResult::Waiting( + EvaluationWaiting::Register { offset }, + EvaluationResult::RequiresRegister { + register, + base_type, + }, + )); + } + + Operation::FrameOffset { offset } => { + return Ok(OperationEvaluationResult::Waiting( + EvaluationWaiting::FrameBase { offset }, + EvaluationResult::RequiresFrameBase, + )); + } + + Operation::Nop => {} + + Operation::PushObjectAddress => { + if let Some(value) = self.object_address { + self.push(Value::Generic(value))?; + } else { + return Err(Error::InvalidPushObjectAddress); + } + } + + Operation::Call { offset } => { + return Ok(OperationEvaluationResult::Waiting( + EvaluationWaiting::AtLocation, + EvaluationResult::RequiresAtLocation(offset), + )); + } + + Operation::TLS => { + let entry = self.pop()?; + let index = entry.to_u64(self.addr_mask)?; + return Ok(OperationEvaluationResult::Waiting( + EvaluationWaiting::Tls, + EvaluationResult::RequiresTls(index), + )); + } + + Operation::CallFrameCFA => { + return Ok(OperationEvaluationResult::Waiting( + EvaluationWaiting::Cfa, + EvaluationResult::RequiresCallFrameCfa, + )); + } + + Operation::Register { register } => { + let location = Location::Register { register }; + return Ok(OperationEvaluationResult::Complete { location }); + } + + Operation::ImplicitValue { ref data } => { + let location = Location::Bytes { + value: data.clone(), + }; + return Ok(OperationEvaluationResult::Complete { location }); + } + + Operation::StackValue => { + let value = self.pop()?; + let location = Location::Value { value }; + return Ok(OperationEvaluationResult::Complete { location }); + } + + Operation::ImplicitPointer { value, byte_offset } => { + let location = Location::ImplicitPointer { value, byte_offset }; + return Ok(OperationEvaluationResult::Complete { location }); + } + + Operation::EntryValue { ref expression } => { + return Ok(OperationEvaluationResult::Waiting( + EvaluationWaiting::EntryValue, + EvaluationResult::RequiresEntryValue(Expression(expression.clone())), + )); + } + + Operation::ParameterRef { offset } => { + return Ok(OperationEvaluationResult::Waiting( + EvaluationWaiting::ParameterRef, + EvaluationResult::RequiresParameterRef(offset), + )); + } + + Operation::Address { address } => { + return Ok(OperationEvaluationResult::Waiting( + EvaluationWaiting::RelocatedAddress, + EvaluationResult::RequiresRelocatedAddress(address), + )); + } + + Operation::AddressIndex { index } => { + return Ok(OperationEvaluationResult::Waiting( + EvaluationWaiting::IndexedAddress, + EvaluationResult::RequiresIndexedAddress { + index, + relocate: true, + }, + )); + } + + Operation::ConstantIndex { index } => { + return Ok(OperationEvaluationResult::Waiting( + EvaluationWaiting::IndexedAddress, + EvaluationResult::RequiresIndexedAddress { + index, + relocate: false, + }, + )); + } + + Operation::Piece { + size_in_bits, + bit_offset, + } => { + let location = if self.stack.is_empty() { + Location::Empty + } else { + let entry = self.pop()?; + let address = entry.to_u64(self.addr_mask)?; + Location::Address { address } + }; + self.result + .try_push(Piece { + size_in_bits: Some(size_in_bits), + bit_offset, + location, + }) + .map_err(|_| Error::StackFull)?; + return Ok(OperationEvaluationResult::Piece); + } + + Operation::TypedLiteral { base_type, value } => { + return Ok(OperationEvaluationResult::Waiting( + EvaluationWaiting::TypedLiteral { value }, + EvaluationResult::RequiresBaseType(base_type), + )); + } + Operation::Convert { base_type } => { + return Ok(OperationEvaluationResult::Waiting( + EvaluationWaiting::Convert, + EvaluationResult::RequiresBaseType(base_type), + )); + } + Operation::Reinterpret { base_type } => { + return Ok(OperationEvaluationResult::Waiting( + EvaluationWaiting::Reinterpret, + EvaluationResult::RequiresBaseType(base_type), + )); + } + Operation::WasmLocal { .. } + | Operation::WasmGlobal { .. } + | Operation::WasmStack { .. } => { + return Err(Error::UnsupportedEvaluation); + } + } + + Ok(OperationEvaluationResult::Incomplete) + } + + /// Get the result of this `Evaluation`. + /// + /// # Panics + /// Panics if this `Evaluation` has not been driven to completion. + pub fn as_result(&self) -> &[Piece] { + match self.state { + EvaluationState::Complete => &self.result, + _ => { + panic!("Called `Evaluation::result` on an `Evaluation` that has not been completed") + } + } + } + + /// Evaluate a DWARF expression. This method should only ever be called + /// once. If the returned `EvaluationResult` is not + /// `EvaluationResult::Complete`, the caller should provide the required + /// value and resume the evaluation by calling the appropriate resume_with + /// method on `Evaluation`. + pub fn evaluate(&mut self) -> Result> { + match self.state { + EvaluationState::Start(initial_value) => { + if let Some(value) = initial_value { + self.push(Value::Generic(value))?; + } + self.state = EvaluationState::Ready; + } + EvaluationState::Ready => {} + EvaluationState::Error(err) => return Err(err), + EvaluationState::Complete => return Ok(EvaluationResult::Complete), + EvaluationState::Waiting(_) => panic!(), + }; + + match self.evaluate_internal() { + Ok(r) => Ok(r), + Err(e) => { + self.state = EvaluationState::Error(e); + Err(e) + } + } + } + + /// Resume the `Evaluation` with the provided memory `value`. This will apply + /// the provided memory value to the evaluation and continue evaluating + /// opcodes until the evaluation is completed, reaches an error, or needs + /// more information again. + /// + /// # Panics + /// Panics if this `Evaluation` did not previously stop with `EvaluationResult::RequiresMemory`. + pub fn resume_with_memory(&mut self, value: Value) -> Result> { + match self.state { + EvaluationState::Error(err) => return Err(err), + EvaluationState::Waiting(EvaluationWaiting::Memory) => { + self.push(value)?; + } + _ => panic!( + "Called `Evaluation::resume_with_memory` without a preceding `EvaluationResult::RequiresMemory`" + ), + }; + + self.evaluate_internal() + } + + /// Resume the `Evaluation` with the provided `register` value. This will apply + /// the provided register value to the evaluation and continue evaluating + /// opcodes until the evaluation is completed, reaches an error, or needs + /// more information again. + /// + /// # Panics + /// Panics if this `Evaluation` did not previously stop with `EvaluationResult::RequiresRegister`. + pub fn resume_with_register(&mut self, value: Value) -> Result> { + match self.state { + EvaluationState::Error(err) => return Err(err), + EvaluationState::Waiting(EvaluationWaiting::Register { offset }) => { + let offset = Value::from_u64(value.value_type(), offset as u64)?; + let value = value.add(offset, self.addr_mask)?; + self.push(value)?; + } + _ => panic!( + "Called `Evaluation::resume_with_register` without a preceding `EvaluationResult::RequiresRegister`" + ), + }; + + self.evaluate_internal() + } + + /// Resume the `Evaluation` with the provided `frame_base`. This will + /// apply the provided frame base value to the evaluation and continue + /// evaluating opcodes until the evaluation is completed, reaches an error, + /// or needs more information again. + /// + /// # Panics + /// Panics if this `Evaluation` did not previously stop with `EvaluationResult::RequiresFrameBase`. + pub fn resume_with_frame_base(&mut self, frame_base: u64) -> Result> { + match self.state { + EvaluationState::Error(err) => return Err(err), + EvaluationState::Waiting(EvaluationWaiting::FrameBase { offset }) => { + self.push(Value::Generic(frame_base.wrapping_add(offset as u64)))?; + } + _ => panic!( + "Called `Evaluation::resume_with_frame_base` without a preceding `EvaluationResult::RequiresFrameBase`" + ), + }; + + self.evaluate_internal() + } + + /// Resume the `Evaluation` with the provided `value`. This will apply + /// the provided TLS value to the evaluation and continue evaluating + /// opcodes until the evaluation is completed, reaches an error, or needs + /// more information again. + /// + /// # Panics + /// Panics if this `Evaluation` did not previously stop with `EvaluationResult::RequiresTls`. + pub fn resume_with_tls(&mut self, value: u64) -> Result> { + match self.state { + EvaluationState::Error(err) => return Err(err), + EvaluationState::Waiting(EvaluationWaiting::Tls) => { + self.push(Value::Generic(value))?; + } + _ => panic!( + "Called `Evaluation::resume_with_tls` without a preceding `EvaluationResult::RequiresTls`" + ), + }; + + self.evaluate_internal() + } + + /// Resume the `Evaluation` with the provided `cfa`. This will + /// apply the provided CFA value to the evaluation and continue evaluating + /// opcodes until the evaluation is completed, reaches an error, or needs + /// more information again. + /// + /// # Panics + /// Panics if this `Evaluation` did not previously stop with `EvaluationResult::RequiresCallFrameCfa`. + pub fn resume_with_call_frame_cfa(&mut self, cfa: u64) -> Result> { + match self.state { + EvaluationState::Error(err) => return Err(err), + EvaluationState::Waiting(EvaluationWaiting::Cfa) => { + self.push(Value::Generic(cfa))?; + } + _ => panic!( + "Called `Evaluation::resume_with_call_frame_cfa` without a preceding `EvaluationResult::RequiresCallFrameCfa`" + ), + }; + + self.evaluate_internal() + } + + /// Resume the `Evaluation` with the provided `bytes`. This will + /// continue processing the evaluation with the new expression provided + /// until the evaluation is completed, reaches an error, or needs more + /// information again. + /// + /// # Panics + /// Panics if this `Evaluation` did not previously stop with `EvaluationResult::RequiresAtLocation`. + pub fn resume_with_at_location(&mut self, mut bytes: R) -> Result> { + match self.state { + EvaluationState::Error(err) => return Err(err), + EvaluationState::Waiting(EvaluationWaiting::AtLocation) => { + if !bytes.is_empty() { + let mut pc = bytes.clone(); + mem::swap(&mut pc, &mut self.pc); + mem::swap(&mut bytes, &mut self.bytecode); + self.expression_stack.try_push((pc, bytes)).map_err(|_| Error::StackFull)?; + } + } + _ => panic!( + "Called `Evaluation::resume_with_at_location` without a precedeing `EvaluationResult::RequiresAtLocation`" + ), + }; + + self.evaluate_internal() + } + + /// Resume the `Evaluation` with the provided `entry_value`. This will + /// apply the provided entry value to the evaluation and continue evaluating + /// opcodes until the evaluation is completed, reaches an error, or needs + /// more information again. + /// + /// # Panics + /// Panics if this `Evaluation` did not previously stop with `EvaluationResult::RequiresEntryValue`. + pub fn resume_with_entry_value(&mut self, entry_value: Value) -> Result> { + match self.state { + EvaluationState::Error(err) => return Err(err), + EvaluationState::Waiting(EvaluationWaiting::EntryValue) => { + self.push(entry_value)?; + } + _ => panic!( + "Called `Evaluation::resume_with_entry_value` without a preceding `EvaluationResult::RequiresEntryValue`" + ), + }; + + self.evaluate_internal() + } + + /// Resume the `Evaluation` with the provided `parameter_value`. This will + /// apply the provided parameter value to the evaluation and continue evaluating + /// opcodes until the evaluation is completed, reaches an error, or needs + /// more information again. + /// + /// # Panics + /// Panics if this `Evaluation` did not previously stop with `EvaluationResult::RequiresParameterRef`. + pub fn resume_with_parameter_ref( + &mut self, + parameter_value: u64, + ) -> Result> { + match self.state { + EvaluationState::Error(err) => return Err(err), + EvaluationState::Waiting(EvaluationWaiting::ParameterRef) => { + self.push(Value::Generic(parameter_value))?; + } + _ => panic!( + "Called `Evaluation::resume_with_parameter_ref` without a preceding `EvaluationResult::RequiresParameterRef`" + ), + }; + + self.evaluate_internal() + } + + /// Resume the `Evaluation` with the provided relocated `address`. This will use the + /// provided relocated address for the operation that required it, and continue evaluating + /// opcodes until the evaluation is completed, reaches an error, or needs + /// more information again. + /// + /// # Panics + /// Panics if this `Evaluation` did not previously stop with + /// `EvaluationResult::RequiresRelocatedAddress`. + pub fn resume_with_relocated_address(&mut self, address: u64) -> Result> { + match self.state { + EvaluationState::Error(err) => return Err(err), + EvaluationState::Waiting(EvaluationWaiting::RelocatedAddress) => { + self.push(Value::Generic(address))?; + } + _ => panic!( + "Called `Evaluation::resume_with_relocated_address` without a preceding `EvaluationResult::RequiresRelocatedAddress`" + ), + }; + + self.evaluate_internal() + } + + /// Resume the `Evaluation` with the provided indexed `address`. This will use the + /// provided indexed address for the operation that required it, and continue evaluating + /// opcodes until the evaluation is completed, reaches an error, or needs + /// more information again. + /// + /// # Panics + /// Panics if this `Evaluation` did not previously stop with + /// `EvaluationResult::RequiresIndexedAddress`. + pub fn resume_with_indexed_address(&mut self, address: u64) -> Result> { + match self.state { + EvaluationState::Error(err) => return Err(err), + EvaluationState::Waiting(EvaluationWaiting::IndexedAddress) => { + self.push(Value::Generic(address))?; + } + _ => panic!( + "Called `Evaluation::resume_with_indexed_address` without a preceding `EvaluationResult::RequiresIndexedAddress`" + ), + }; + + self.evaluate_internal() + } + + /// Resume the `Evaluation` with the provided `base_type`. This will use the + /// provided base type for the operation that required it, and continue evaluating + /// opcodes until the evaluation is completed, reaches an error, or needs + /// more information again. + /// + /// # Panics + /// Panics if this `Evaluation` did not previously stop with `EvaluationResult::RequiresBaseType`. + pub fn resume_with_base_type(&mut self, base_type: ValueType) -> Result> { + let value = match self.state { + EvaluationState::Error(err) => return Err(err), + EvaluationState::Waiting(EvaluationWaiting::TypedLiteral { ref value }) => { + Value::parse(base_type, value.clone())? + } + EvaluationState::Waiting(EvaluationWaiting::Convert) => { + let entry = self.pop()?; + entry.convert(base_type, self.addr_mask)? + } + EvaluationState::Waiting(EvaluationWaiting::Reinterpret) => { + let entry = self.pop()?; + entry.reinterpret(base_type, self.addr_mask)? + } + _ => panic!( + "Called `Evaluation::resume_with_base_type` without a preceding `EvaluationResult::RequiresBaseType`" + ), + }; + self.push(value)?; + self.evaluate_internal() + } + + fn end_of_expression(&mut self) -> bool { + while self.pc.is_empty() { + match self.expression_stack.pop() { + Some((newpc, newbytes)) => { + self.pc = newpc; + self.bytecode = newbytes; + } + None => return true, + } + } + false + } + + fn evaluate_internal(&mut self) -> Result> { + while !self.end_of_expression() { + self.iteration += 1; + if let Some(max_iterations) = self.max_iterations { + if self.iteration > max_iterations { + return Err(Error::TooManyIterations); + } + } + + let op_result = self.evaluate_one_operation()?; + match op_result { + OperationEvaluationResult::Piece => {} + OperationEvaluationResult::Incomplete => { + if self.end_of_expression() && !self.result.is_empty() { + // We saw a piece earlier and then some + // unterminated piece. It's not clear this is + // well-defined. + return Err(Error::InvalidPiece); + } + } + OperationEvaluationResult::Complete { location } => { + if self.end_of_expression() { + if !self.result.is_empty() { + // We saw a piece earlier and then some + // unterminated piece. It's not clear this is + // well-defined. + return Err(Error::InvalidPiece); + } + self.result + .try_push(Piece { + size_in_bits: None, + bit_offset: None, + location, + }) + .map_err(|_| Error::StackFull)?; + } else { + // If there are more operations, then the next operation must + // be a Piece. + match Operation::parse(&mut self.pc, self.encoding)? { + Operation::Piece { + size_in_bits, + bit_offset, + } => { + self.result + .try_push(Piece { + size_in_bits: Some(size_in_bits), + bit_offset, + location, + }) + .map_err(|_| Error::StackFull)?; + } + _ => { + let value = + self.bytecode.len().into_u64() - self.pc.len().into_u64() - 1; + return Err(Error::InvalidExpressionTerminator(value)); + } + } + } + } + OperationEvaluationResult::Waiting(waiting, result) => { + self.state = EvaluationState::Waiting(waiting); + return Ok(result); + } + }; + } + + // If no pieces have been seen, use the stack top as the + // result. + if self.result.is_empty() { + let entry = self.pop()?; + let addr = entry.to_u64(self.addr_mask)?; + self.result + .try_push(Piece { + size_in_bits: None, + bit_offset: None, + location: Location::Address { address: addr }, + }) + .map_err(|_| Error::StackFull)?; + } + + self.state = EvaluationState::Complete; + Ok(EvaluationResult::Complete) + } +} + +#[cfg(test)] +// Tests require leb128::write. +#[cfg(feature = "write")] +mod tests { + use super::*; + use crate::common::Format; + use crate::constants; + use crate::endianity::LittleEndian; + use crate::leb128; + use crate::read::{EndianSlice, Error, Result, UnitOffset}; + use crate::test_util::GimliSectionMethods; + use core::usize; + use test_assembler::{Endian, Section}; + + fn encoding4() -> Encoding { + Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 4, + } + } + + fn encoding8() -> Encoding { + Encoding { + format: Format::Dwarf64, + version: 4, + address_size: 8, + } + } + + #[test] + fn test_compute_pc() { + // Contents don't matter for this test, just length. + let bytes = [0, 1, 2, 3, 4]; + let bytecode = &bytes[..]; + let ebuf = &EndianSlice::new(bytecode, LittleEndian); + + assert_eq!(compute_pc(ebuf, ebuf, 0), Ok(*ebuf)); + assert_eq!( + compute_pc(ebuf, ebuf, -1), + Err(Error::BadBranchTarget(usize::MAX as u64)) + ); + assert_eq!(compute_pc(ebuf, ebuf, 5), Ok(ebuf.range_from(5..))); + assert_eq!( + compute_pc(&ebuf.range_from(3..), ebuf, -2), + Ok(ebuf.range_from(1..)) + ); + assert_eq!( + compute_pc(&ebuf.range_from(2..), ebuf, 2), + Ok(ebuf.range_from(4..)) + ); + } + + fn check_op_parse_simple<'input>( + input: &'input [u8], + expect: &Operation>, + encoding: Encoding, + ) { + let buf = EndianSlice::new(input, LittleEndian); + let mut pc = buf; + let value = Operation::parse(&mut pc, encoding); + match value { + Ok(val) => { + assert_eq!(val, *expect); + assert_eq!(pc.len(), 0); + } + _ => panic!("Unexpected result"), + } + } + + fn check_op_parse_eof(input: &[u8], encoding: Encoding) { + let buf = EndianSlice::new(input, LittleEndian); + let mut pc = buf; + match Operation::parse(&mut pc, encoding) { + Err(Error::UnexpectedEof(id)) => { + assert!(buf.lookup_offset_id(id).is_some()); + } + + _ => panic!("Unexpected result"), + } + } + + fn check_op_parse( + input: F, + expect: &Operation>, + encoding: Encoding, + ) where + F: Fn(Section) -> Section, + { + let input = input(Section::with_endian(Endian::Little)) + .get_contents() + .unwrap(); + for i in 1..input.len() { + check_op_parse_eof(&input[..i], encoding); + } + check_op_parse_simple(&input, expect, encoding); + } + + #[test] + fn test_op_parse_onebyte() { + // Doesn't matter for this test. + let encoding = encoding4(); + + // Test all single-byte opcodes. + #[rustfmt::skip] + let inputs = [ + ( + constants::DW_OP_deref, + Operation::Deref { + base_type: generic_type(), + size: encoding.address_size, + space: false, + }, + ), + (constants::DW_OP_dup, Operation::Pick { index: 0 }), + (constants::DW_OP_drop, Operation::Drop), + (constants::DW_OP_over, Operation::Pick { index: 1 }), + (constants::DW_OP_swap, Operation::Swap), + (constants::DW_OP_rot, Operation::Rot), + ( + constants::DW_OP_xderef, + Operation::Deref { + base_type: generic_type(), + size: encoding.address_size, + space: true, + }, + ), + (constants::DW_OP_abs, Operation::Abs), + (constants::DW_OP_and, Operation::And), + (constants::DW_OP_div, Operation::Div), + (constants::DW_OP_minus, Operation::Minus), + (constants::DW_OP_mod, Operation::Mod), + (constants::DW_OP_mul, Operation::Mul), + (constants::DW_OP_neg, Operation::Neg), + (constants::DW_OP_not, Operation::Not), + (constants::DW_OP_or, Operation::Or), + (constants::DW_OP_plus, Operation::Plus), + (constants::DW_OP_shl, Operation::Shl), + (constants::DW_OP_shr, Operation::Shr), + (constants::DW_OP_shra, Operation::Shra), + (constants::DW_OP_xor, Operation::Xor), + (constants::DW_OP_eq, Operation::Eq), + (constants::DW_OP_ge, Operation::Ge), + (constants::DW_OP_gt, Operation::Gt), + (constants::DW_OP_le, Operation::Le), + (constants::DW_OP_lt, Operation::Lt), + (constants::DW_OP_ne, Operation::Ne), + (constants::DW_OP_lit0, Operation::UnsignedConstant { value: 0 }), + (constants::DW_OP_lit1, Operation::UnsignedConstant { value: 1 }), + (constants::DW_OP_lit2, Operation::UnsignedConstant { value: 2 }), + (constants::DW_OP_lit3, Operation::UnsignedConstant { value: 3 }), + (constants::DW_OP_lit4, Operation::UnsignedConstant { value: 4 }), + (constants::DW_OP_lit5, Operation::UnsignedConstant { value: 5 }), + (constants::DW_OP_lit6, Operation::UnsignedConstant { value: 6 }), + (constants::DW_OP_lit7, Operation::UnsignedConstant { value: 7 }), + (constants::DW_OP_lit8, Operation::UnsignedConstant { value: 8 }), + (constants::DW_OP_lit9, Operation::UnsignedConstant { value: 9 }), + (constants::DW_OP_lit10, Operation::UnsignedConstant { value: 10 }), + (constants::DW_OP_lit11, Operation::UnsignedConstant { value: 11 }), + (constants::DW_OP_lit12, Operation::UnsignedConstant { value: 12 }), + (constants::DW_OP_lit13, Operation::UnsignedConstant { value: 13 }), + (constants::DW_OP_lit14, Operation::UnsignedConstant { value: 14 }), + (constants::DW_OP_lit15, Operation::UnsignedConstant { value: 15 }), + (constants::DW_OP_lit16, Operation::UnsignedConstant { value: 16 }), + (constants::DW_OP_lit17, Operation::UnsignedConstant { value: 17 }), + (constants::DW_OP_lit18, Operation::UnsignedConstant { value: 18 }), + (constants::DW_OP_lit19, Operation::UnsignedConstant { value: 19 }), + (constants::DW_OP_lit20, Operation::UnsignedConstant { value: 20 }), + (constants::DW_OP_lit21, Operation::UnsignedConstant { value: 21 }), + (constants::DW_OP_lit22, Operation::UnsignedConstant { value: 22 }), + (constants::DW_OP_lit23, Operation::UnsignedConstant { value: 23 }), + (constants::DW_OP_lit24, Operation::UnsignedConstant { value: 24 }), + (constants::DW_OP_lit25, Operation::UnsignedConstant { value: 25 }), + (constants::DW_OP_lit26, Operation::UnsignedConstant { value: 26 }), + (constants::DW_OP_lit27, Operation::UnsignedConstant { value: 27 }), + (constants::DW_OP_lit28, Operation::UnsignedConstant { value: 28 }), + (constants::DW_OP_lit29, Operation::UnsignedConstant { value: 29 }), + (constants::DW_OP_lit30, Operation::UnsignedConstant { value: 30 }), + (constants::DW_OP_lit31, Operation::UnsignedConstant { value: 31 }), + (constants::DW_OP_reg0, Operation::Register { register: Register(0) }), + (constants::DW_OP_reg1, Operation::Register { register: Register(1) }), + (constants::DW_OP_reg2, Operation::Register { register: Register(2) }), + (constants::DW_OP_reg3, Operation::Register { register: Register(3) }), + (constants::DW_OP_reg4, Operation::Register { register: Register(4) }), + (constants::DW_OP_reg5, Operation::Register { register: Register(5) }), + (constants::DW_OP_reg6, Operation::Register { register: Register(6) }), + (constants::DW_OP_reg7, Operation::Register { register: Register(7) }), + (constants::DW_OP_reg8, Operation::Register { register: Register(8) }), + (constants::DW_OP_reg9, Operation::Register { register: Register(9) }), + (constants::DW_OP_reg10, Operation::Register { register: Register(10) }), + (constants::DW_OP_reg11, Operation::Register { register: Register(11) }), + (constants::DW_OP_reg12, Operation::Register { register: Register(12) }), + (constants::DW_OP_reg13, Operation::Register { register: Register(13) }), + (constants::DW_OP_reg14, Operation::Register { register: Register(14) }), + (constants::DW_OP_reg15, Operation::Register { register: Register(15) }), + (constants::DW_OP_reg16, Operation::Register { register: Register(16) }), + (constants::DW_OP_reg17, Operation::Register { register: Register(17) }), + (constants::DW_OP_reg18, Operation::Register { register: Register(18) }), + (constants::DW_OP_reg19, Operation::Register { register: Register(19) }), + (constants::DW_OP_reg20, Operation::Register { register: Register(20) }), + (constants::DW_OP_reg21, Operation::Register { register: Register(21) }), + (constants::DW_OP_reg22, Operation::Register { register: Register(22) }), + (constants::DW_OP_reg23, Operation::Register { register: Register(23) }), + (constants::DW_OP_reg24, Operation::Register { register: Register(24) }), + (constants::DW_OP_reg25, Operation::Register { register: Register(25) }), + (constants::DW_OP_reg26, Operation::Register { register: Register(26) }), + (constants::DW_OP_reg27, Operation::Register { register: Register(27) }), + (constants::DW_OP_reg28, Operation::Register { register: Register(28) }), + (constants::DW_OP_reg29, Operation::Register { register: Register(29) }), + (constants::DW_OP_reg30, Operation::Register { register: Register(30) }), + (constants::DW_OP_reg31, Operation::Register { register: Register(31) }), + (constants::DW_OP_nop, Operation::Nop), + (constants::DW_OP_push_object_address, Operation::PushObjectAddress), + (constants::DW_OP_form_tls_address, Operation::TLS), + (constants::DW_OP_GNU_push_tls_address, Operation::TLS), + (constants::DW_OP_call_frame_cfa, Operation::CallFrameCFA), + (constants::DW_OP_stack_value, Operation::StackValue), + ]; + + let input = []; + check_op_parse_eof(&input[..], encoding); + + for item in inputs.iter() { + let (opcode, ref result) = *item; + check_op_parse(|s| s.D8(opcode.0), result, encoding); + } + } + + #[test] + fn test_op_parse_twobyte() { + // Doesn't matter for this test. + let encoding = encoding4(); + + let inputs = [ + ( + constants::DW_OP_const1u, + 23, + Operation::UnsignedConstant { value: 23 }, + ), + ( + constants::DW_OP_const1s, + (-23i8) as u8, + Operation::SignedConstant { value: -23 }, + ), + (constants::DW_OP_pick, 7, Operation::Pick { index: 7 }), + ( + constants::DW_OP_deref_size, + 19, + Operation::Deref { + base_type: generic_type(), + size: 19, + space: false, + }, + ), + ( + constants::DW_OP_xderef_size, + 19, + Operation::Deref { + base_type: generic_type(), + size: 19, + space: true, + }, + ), + ]; + + for item in inputs.iter() { + let (opcode, arg, ref result) = *item; + check_op_parse(|s| s.D8(opcode.0).D8(arg), result, encoding); + } + } + + #[test] + fn test_op_parse_threebyte() { + // Doesn't matter for this test. + let encoding = encoding4(); + + // While bra and skip are 3-byte opcodes, they aren't tested here, + // but rather specially in their own function. + let inputs = [ + ( + constants::DW_OP_const2u, + 23, + Operation::UnsignedConstant { value: 23 }, + ), + ( + constants::DW_OP_const2s, + (-23i16) as u16, + Operation::SignedConstant { value: -23 }, + ), + ( + constants::DW_OP_call2, + 1138, + Operation::Call { + offset: DieReference::UnitRef(UnitOffset(1138)), + }, + ), + ( + constants::DW_OP_bra, + (-23i16) as u16, + Operation::Bra { target: -23 }, + ), + ( + constants::DW_OP_skip, + (-23i16) as u16, + Operation::Skip { target: -23 }, + ), + ]; + + for item in inputs.iter() { + let (opcode, arg, ref result) = *item; + check_op_parse(|s| s.D8(opcode.0).L16(arg), result, encoding); + } + } + + #[test] + fn test_op_parse_fivebyte() { + // There are some tests here that depend on address size. + let encoding = encoding4(); + + let inputs = [ + ( + constants::DW_OP_addr, + 0x1234_5678, + Operation::Address { + address: 0x1234_5678, + }, + ), + ( + constants::DW_OP_const4u, + 0x1234_5678, + Operation::UnsignedConstant { value: 0x1234_5678 }, + ), + ( + constants::DW_OP_const4s, + (-23i32) as u32, + Operation::SignedConstant { value: -23 }, + ), + ( + constants::DW_OP_call4, + 0x1234_5678, + Operation::Call { + offset: DieReference::UnitRef(UnitOffset(0x1234_5678)), + }, + ), + ( + constants::DW_OP_call_ref, + 0x1234_5678, + Operation::Call { + offset: DieReference::DebugInfoRef(DebugInfoOffset(0x1234_5678)), + }, + ), + ]; + + for item in inputs.iter() { + let (op, arg, ref expect) = *item; + check_op_parse(|s| s.D8(op.0).L32(arg), expect, encoding); + } + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_op_parse_ninebyte() { + // There are some tests here that depend on address size. + let encoding = encoding8(); + + let inputs = [ + ( + constants::DW_OP_addr, + 0x1234_5678_1234_5678, + Operation::Address { + address: 0x1234_5678_1234_5678, + }, + ), + ( + constants::DW_OP_const8u, + 0x1234_5678_1234_5678, + Operation::UnsignedConstant { + value: 0x1234_5678_1234_5678, + }, + ), + ( + constants::DW_OP_const8s, + (-23i64) as u64, + Operation::SignedConstant { value: -23 }, + ), + ( + constants::DW_OP_call_ref, + 0x1234_5678_1234_5678, + Operation::Call { + offset: DieReference::DebugInfoRef(DebugInfoOffset(0x1234_5678_1234_5678)), + }, + ), + ]; + + for item in inputs.iter() { + let (op, arg, ref expect) = *item; + check_op_parse(|s| s.D8(op.0).L64(arg), expect, encoding); + } + } + + #[test] + fn test_op_parse_sleb() { + // Doesn't matter for this test. + let encoding = encoding4(); + + let values = [ + -1i64, + 0, + 1, + 0x100, + 0x1eee_eeee, + 0x7fff_ffff_ffff_ffff, + -0x100, + -0x1eee_eeee, + -0x7fff_ffff_ffff_ffff, + ]; + for value in values.iter() { + let mut inputs = vec![ + ( + constants::DW_OP_consts.0, + Operation::SignedConstant { value: *value }, + ), + ( + constants::DW_OP_fbreg.0, + Operation::FrameOffset { offset: *value }, + ), + ]; + + for i in 0..32 { + inputs.push(( + constants::DW_OP_breg0.0 + i, + Operation::RegisterOffset { + register: Register(i.into()), + offset: *value, + base_type: UnitOffset(0), + }, + )); + } + + for item in inputs.iter() { + let (op, ref expect) = *item; + check_op_parse(|s| s.D8(op).sleb(*value), expect, encoding); + } + } + } + + #[test] + fn test_op_parse_uleb() { + // Doesn't matter for this test. + let encoding = encoding4(); + + let values = [ + 0, + 1, + 0x100, + (!0u16).into(), + 0x1eee_eeee, + 0x7fff_ffff_ffff_ffff, + !0u64, + ]; + for value in values.iter() { + let mut inputs = vec![ + ( + constants::DW_OP_constu, + Operation::UnsignedConstant { value: *value }, + ), + ( + constants::DW_OP_plus_uconst, + Operation::PlusConstant { value: *value }, + ), + ]; + + if *value <= (!0u16).into() { + inputs.push(( + constants::DW_OP_regx, + Operation::Register { + register: Register::from_u64(*value).unwrap(), + }, + )); + } + + if *value <= (!0u32).into() { + inputs.extend(&[ + ( + constants::DW_OP_addrx, + Operation::AddressIndex { + index: DebugAddrIndex(*value as usize), + }, + ), + ( + constants::DW_OP_constx, + Operation::ConstantIndex { + index: DebugAddrIndex(*value as usize), + }, + ), + ]); + } + + // FIXME + if *value < !0u64 / 8 { + inputs.push(( + constants::DW_OP_piece, + Operation::Piece { + size_in_bits: 8 * value, + bit_offset: None, + }, + )); + } + + for item in inputs.iter() { + let (op, ref expect) = *item; + let input = Section::with_endian(Endian::Little) + .D8(op.0) + .uleb(*value) + .get_contents() + .unwrap(); + check_op_parse_simple(&input, expect, encoding); + } + } + } + + #[test] + fn test_op_parse_bregx() { + // Doesn't matter for this test. + let encoding = encoding4(); + + let uvalues = [0, 1, 0x100, !0u16]; + let svalues = [ + -1i64, + 0, + 1, + 0x100, + 0x1eee_eeee, + 0x7fff_ffff_ffff_ffff, + -0x100, + -0x1eee_eeee, + -0x7fff_ffff_ffff_ffff, + ]; + + for v1 in uvalues.iter() { + for v2 in svalues.iter() { + check_op_parse( + |s| s.D8(constants::DW_OP_bregx.0).uleb((*v1).into()).sleb(*v2), + &Operation::RegisterOffset { + register: Register(*v1), + offset: *v2, + base_type: UnitOffset(0), + }, + encoding, + ); + } + } + } + + #[test] + fn test_op_parse_bit_piece() { + // Doesn't matter for this test. + let encoding = encoding4(); + + let values = [0, 1, 0x100, 0x1eee_eeee, 0x7fff_ffff_ffff_ffff, !0u64]; + + for v1 in values.iter() { + for v2 in values.iter() { + let input = Section::with_endian(Endian::Little) + .D8(constants::DW_OP_bit_piece.0) + .uleb(*v1) + .uleb(*v2) + .get_contents() + .unwrap(); + check_op_parse_simple( + &input, + &Operation::Piece { + size_in_bits: *v1, + bit_offset: Some(*v2), + }, + encoding, + ); + } + } + } + + #[test] + fn test_op_parse_implicit_value() { + // Doesn't matter for this test. + let encoding = encoding4(); + + let data = b"hello"; + + check_op_parse( + |s| { + s.D8(constants::DW_OP_implicit_value.0) + .uleb(data.len() as u64) + .append_bytes(&data[..]) + }, + &Operation::ImplicitValue { + data: EndianSlice::new(&data[..], LittleEndian), + }, + encoding, + ); + } + + #[test] + fn test_op_parse_const_type() { + // Doesn't matter for this test. + let encoding = encoding4(); + + let data = b"hello"; + + check_op_parse( + |s| { + s.D8(constants::DW_OP_const_type.0) + .uleb(100) + .D8(data.len() as u8) + .append_bytes(&data[..]) + }, + &Operation::TypedLiteral { + base_type: UnitOffset(100), + value: EndianSlice::new(&data[..], LittleEndian), + }, + encoding, + ); + check_op_parse( + |s| { + s.D8(constants::DW_OP_GNU_const_type.0) + .uleb(100) + .D8(data.len() as u8) + .append_bytes(&data[..]) + }, + &Operation::TypedLiteral { + base_type: UnitOffset(100), + value: EndianSlice::new(&data[..], LittleEndian), + }, + encoding, + ); + } + + #[test] + fn test_op_parse_regval_type() { + // Doesn't matter for this test. + let encoding = encoding4(); + + check_op_parse( + |s| s.D8(constants::DW_OP_regval_type.0).uleb(1).uleb(100), + &Operation::RegisterOffset { + register: Register(1), + offset: 0, + base_type: UnitOffset(100), + }, + encoding, + ); + check_op_parse( + |s| s.D8(constants::DW_OP_GNU_regval_type.0).uleb(1).uleb(100), + &Operation::RegisterOffset { + register: Register(1), + offset: 0, + base_type: UnitOffset(100), + }, + encoding, + ); + } + + #[test] + fn test_op_parse_deref_type() { + // Doesn't matter for this test. + let encoding = encoding4(); + + check_op_parse( + |s| s.D8(constants::DW_OP_deref_type.0).D8(8).uleb(100), + &Operation::Deref { + base_type: UnitOffset(100), + size: 8, + space: false, + }, + encoding, + ); + check_op_parse( + |s| s.D8(constants::DW_OP_GNU_deref_type.0).D8(8).uleb(100), + &Operation::Deref { + base_type: UnitOffset(100), + size: 8, + space: false, + }, + encoding, + ); + check_op_parse( + |s| s.D8(constants::DW_OP_xderef_type.0).D8(8).uleb(100), + &Operation::Deref { + base_type: UnitOffset(100), + size: 8, + space: true, + }, + encoding, + ); + } + + #[test] + fn test_op_convert() { + // Doesn't matter for this test. + let encoding = encoding4(); + + check_op_parse( + |s| s.D8(constants::DW_OP_convert.0).uleb(100), + &Operation::Convert { + base_type: UnitOffset(100), + }, + encoding, + ); + check_op_parse( + |s| s.D8(constants::DW_OP_GNU_convert.0).uleb(100), + &Operation::Convert { + base_type: UnitOffset(100), + }, + encoding, + ); + } + + #[test] + fn test_op_reinterpret() { + // Doesn't matter for this test. + let encoding = encoding4(); + + check_op_parse( + |s| s.D8(constants::DW_OP_reinterpret.0).uleb(100), + &Operation::Reinterpret { + base_type: UnitOffset(100), + }, + encoding, + ); + check_op_parse( + |s| s.D8(constants::DW_OP_GNU_reinterpret.0).uleb(100), + &Operation::Reinterpret { + base_type: UnitOffset(100), + }, + encoding, + ); + } + + #[test] + fn test_op_parse_implicit_pointer() { + for op in &[ + constants::DW_OP_implicit_pointer, + constants::DW_OP_GNU_implicit_pointer, + ] { + check_op_parse( + |s| s.D8(op.0).D32(0x1234_5678).sleb(0x123), + &Operation::ImplicitPointer { + value: DebugInfoOffset(0x1234_5678), + byte_offset: 0x123, + }, + encoding4(), + ); + + check_op_parse( + |s| s.D8(op.0).D64(0x1234_5678).sleb(0x123), + &Operation::ImplicitPointer { + value: DebugInfoOffset(0x1234_5678), + byte_offset: 0x123, + }, + encoding8(), + ); + + check_op_parse( + |s| s.D8(op.0).D64(0x1234_5678).sleb(0x123), + &Operation::ImplicitPointer { + value: DebugInfoOffset(0x1234_5678), + byte_offset: 0x123, + }, + Encoding { + format: Format::Dwarf32, + version: 2, + address_size: 8, + }, + ) + } + } + + #[test] + fn test_op_parse_entry_value() { + for op in &[ + constants::DW_OP_entry_value, + constants::DW_OP_GNU_entry_value, + ] { + let data = b"hello"; + check_op_parse( + |s| s.D8(op.0).uleb(data.len() as u64).append_bytes(&data[..]), + &Operation::EntryValue { + expression: EndianSlice::new(&data[..], LittleEndian), + }, + encoding4(), + ); + } + } + + #[test] + fn test_op_parse_gnu_parameter_ref() { + check_op_parse( + |s| s.D8(constants::DW_OP_GNU_parameter_ref.0).D32(0x1234_5678), + &Operation::ParameterRef { + offset: UnitOffset(0x1234_5678), + }, + encoding4(), + ) + } + + #[test] + fn test_op_wasm() { + // Doesn't matter for this test. + let encoding = encoding4(); + + check_op_parse( + |s| s.D8(constants::DW_OP_WASM_location.0).D8(0).uleb(1000), + &Operation::WasmLocal { index: 1000 }, + encoding, + ); + check_op_parse( + |s| s.D8(constants::DW_OP_WASM_location.0).D8(1).uleb(1000), + &Operation::WasmGlobal { index: 1000 }, + encoding, + ); + check_op_parse( + |s| s.D8(constants::DW_OP_WASM_location.0).D8(2).uleb(1000), + &Operation::WasmStack { index: 1000 }, + encoding, + ); + check_op_parse( + |s| s.D8(constants::DW_OP_WASM_location.0).D8(3).D32(1000), + &Operation::WasmGlobal { index: 1000 }, + encoding, + ); + } + + enum AssemblerEntry { + Op(constants::DwOp), + Mark(u8), + Branch(u8), + U8(u8), + U16(u16), + U32(u32), + U64(u64), + Uleb(u64), + Sleb(u64), + } + + fn assemble(entries: &[AssemblerEntry]) -> Vec { + let mut result = Vec::new(); + + struct Marker(Option, Vec); + + let mut markers = Vec::new(); + for _ in 0..256 { + markers.push(Marker(None, Vec::new())); + } + + fn write(stack: &mut Vec, index: usize, mut num: u64, nbytes: u8) { + for i in 0..nbytes as usize { + stack[index + i] = (num & 0xff) as u8; + num >>= 8; + } + } + + fn push(stack: &mut Vec, num: u64, nbytes: u8) { + let index = stack.len(); + for _ in 0..nbytes { + stack.push(0); + } + write(stack, index, num, nbytes); + } + + for item in entries { + match *item { + AssemblerEntry::Op(op) => result.push(op.0), + AssemblerEntry::Mark(num) => { + assert!(markers[num as usize].0.is_none()); + markers[num as usize].0 = Some(result.len()); + } + AssemblerEntry::Branch(num) => { + markers[num as usize].1.push(result.len()); + push(&mut result, 0, 2); + } + AssemblerEntry::U8(num) => result.push(num), + AssemblerEntry::U16(num) => push(&mut result, u64::from(num), 2), + AssemblerEntry::U32(num) => push(&mut result, u64::from(num), 4), + AssemblerEntry::U64(num) => push(&mut result, num, 8), + AssemblerEntry::Uleb(num) => { + leb128::write::unsigned(&mut result, num).unwrap(); + } + AssemblerEntry::Sleb(num) => { + leb128::write::signed(&mut result, num as i64).unwrap(); + } + } + } + + // Update all the branches. + for marker in markers { + if let Some(offset) = marker.0 { + for branch_offset in marker.1 { + let delta = offset.wrapping_sub(branch_offset + 2) as u64; + write(&mut result, branch_offset, delta, 2); + } + } + } + + result + } + + #[allow(clippy::too_many_arguments)] + fn check_eval_with_args( + program: &[AssemblerEntry], + expect: Result<&[Piece>]>, + encoding: Encoding, + object_address: Option, + initial_value: Option, + max_iterations: Option, + f: F, + ) where + for<'a> F: Fn( + &mut Evaluation>, + EvaluationResult>, + ) -> Result>>, + { + let bytes = assemble(program); + let bytes = EndianSlice::new(&bytes, LittleEndian); + + let mut eval = Evaluation::new(bytes, encoding); + + if let Some(val) = object_address { + eval.set_object_address(val); + } + if let Some(val) = initial_value { + eval.set_initial_value(val); + } + if let Some(val) = max_iterations { + eval.set_max_iterations(val); + } + + let result = match eval.evaluate() { + Err(e) => Err(e), + Ok(r) => f(&mut eval, r), + }; + + match (result, expect) { + (Ok(EvaluationResult::Complete), Ok(pieces)) => { + let vec = eval.result(); + assert_eq!(vec.len(), pieces.len()); + for i in 0..pieces.len() { + assert_eq!(vec[i], pieces[i]); + } + } + (Err(f1), Err(f2)) => { + assert_eq!(f1, f2); + } + otherwise => panic!("Unexpected result: {:?}", otherwise), + } + } + + fn check_eval( + program: &[AssemblerEntry], + expect: Result<&[Piece>]>, + encoding: Encoding, + ) { + check_eval_with_args(program, expect, encoding, None, None, None, |_, result| { + Ok(result) + }); + } + + #[test] + fn test_eval_arith() { + // It's nice if an operation and its arguments can fit on a single + // line in the test program. + use self::AssemblerEntry::*; + use crate::constants::*; + + // Indices of marks in the assembly. + let done = 0; + let fail = 1; + + #[rustfmt::skip] + let program = [ + Op(DW_OP_const1u), U8(23), + Op(DW_OP_const1s), U8((-23i8) as u8), + Op(DW_OP_plus), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_const2u), U16(23), + Op(DW_OP_const2s), U16((-23i16) as u16), + Op(DW_OP_plus), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_const4u), U32(0x1111_2222), + Op(DW_OP_const4s), U32((-0x1111_2222i32) as u32), + Op(DW_OP_plus), + Op(DW_OP_bra), Branch(fail), + + // Plus should overflow. + Op(DW_OP_const1s), U8(0xff), + Op(DW_OP_const1u), U8(1), + Op(DW_OP_plus), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_const1s), U8(0xff), + Op(DW_OP_plus_uconst), Uleb(1), + Op(DW_OP_bra), Branch(fail), + + // Minus should underflow. + Op(DW_OP_const1s), U8(0), + Op(DW_OP_const1u), U8(1), + Op(DW_OP_minus), + Op(DW_OP_const1s), U8(0xff), + Op(DW_OP_ne), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_const1s), U8(0xff), + Op(DW_OP_abs), + Op(DW_OP_const1u), U8(1), + Op(DW_OP_minus), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_const4u), U32(0xf078_fffe), + Op(DW_OP_const4u), U32(0x0f87_0001), + Op(DW_OP_and), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_const4u), U32(0xf078_fffe), + Op(DW_OP_const4u), U32(0xf000_00fe), + Op(DW_OP_and), + Op(DW_OP_const4u), U32(0xf000_00fe), + Op(DW_OP_ne), + Op(DW_OP_bra), Branch(fail), + + // Division is signed. + Op(DW_OP_const1s), U8(0xfe), + Op(DW_OP_const1s), U8(2), + Op(DW_OP_div), + Op(DW_OP_plus_uconst), Uleb(1), + Op(DW_OP_bra), Branch(fail), + + // Mod is unsigned. + Op(DW_OP_const1s), U8(0xfd), + Op(DW_OP_const1s), U8(2), + Op(DW_OP_mod), + Op(DW_OP_neg), + Op(DW_OP_plus_uconst), Uleb(1), + Op(DW_OP_bra), Branch(fail), + + // Overflow is defined for multiplication. + Op(DW_OP_const4u), U32(0x8000_0001), + Op(DW_OP_lit2), + Op(DW_OP_mul), + Op(DW_OP_lit2), + Op(DW_OP_ne), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_const4u), U32(0xf0f0_f0f0), + Op(DW_OP_const4u), U32(0xf0f0_f0f0), + Op(DW_OP_xor), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_const4u), U32(0xf0f0_f0f0), + Op(DW_OP_const4u), U32(0x0f0f_0f0f), + Op(DW_OP_or), + Op(DW_OP_not), + Op(DW_OP_bra), Branch(fail), + + // In 32 bit mode, values are truncated. + Op(DW_OP_const8u), U64(0xffff_ffff_0000_0000), + Op(DW_OP_lit2), + Op(DW_OP_div), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_const1u), U8(0xff), + Op(DW_OP_lit1), + Op(DW_OP_shl), + Op(DW_OP_const2u), U16(0x1fe), + Op(DW_OP_ne), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_const1u), U8(0xff), + Op(DW_OP_const1u), U8(50), + Op(DW_OP_shl), + Op(DW_OP_bra), Branch(fail), + + // Absurd shift. + Op(DW_OP_const1u), U8(0xff), + Op(DW_OP_const1s), U8(0xff), + Op(DW_OP_shl), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_const1s), U8(0xff), + Op(DW_OP_lit1), + Op(DW_OP_shr), + Op(DW_OP_const4u), U32(0x7fff_ffff), + Op(DW_OP_ne), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_const1s), U8(0xff), + Op(DW_OP_const1u), U8(0xff), + Op(DW_OP_shr), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_const1s), U8(0xff), + Op(DW_OP_lit1), + Op(DW_OP_shra), + Op(DW_OP_const1s), U8(0xff), + Op(DW_OP_ne), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_const1s), U8(0xff), + Op(DW_OP_const1u), U8(0xff), + Op(DW_OP_shra), + Op(DW_OP_const1s), U8(0xff), + Op(DW_OP_ne), + Op(DW_OP_bra), Branch(fail), + + // Success. + Op(DW_OP_lit0), + Op(DW_OP_nop), + Op(DW_OP_skip), Branch(done), + + Mark(fail), + Op(DW_OP_lit1), + + Mark(done), + Op(DW_OP_stack_value), + ]; + + let result = [Piece { + size_in_bits: None, + bit_offset: None, + location: Location::Value { + value: Value::Generic(0), + }, + }]; + + check_eval(&program, Ok(&result), encoding4()); + } + + #[test] + fn test_eval_arith64() { + // It's nice if an operation and its arguments can fit on a single + // line in the test program. + use self::AssemblerEntry::*; + use crate::constants::*; + + // Indices of marks in the assembly. + let done = 0; + let fail = 1; + + #[rustfmt::skip] + let program = [ + Op(DW_OP_const8u), U64(0x1111_2222_3333_4444), + Op(DW_OP_const8s), U64((-0x1111_2222_3333_4444i64) as u64), + Op(DW_OP_plus), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_constu), Uleb(0x1111_2222_3333_4444), + Op(DW_OP_consts), Sleb((-0x1111_2222_3333_4444i64) as u64), + Op(DW_OP_plus), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_lit1), + Op(DW_OP_plus_uconst), Uleb(!0u64), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_lit1), + Op(DW_OP_neg), + Op(DW_OP_not), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_const8u), U64(0x8000_0000_0000_0000), + Op(DW_OP_const1u), U8(63), + Op(DW_OP_shr), + Op(DW_OP_lit1), + Op(DW_OP_ne), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_const8u), U64(0x8000_0000_0000_0000), + Op(DW_OP_const1u), U8(62), + Op(DW_OP_shra), + Op(DW_OP_plus_uconst), Uleb(2), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_lit1), + Op(DW_OP_const1u), U8(63), + Op(DW_OP_shl), + Op(DW_OP_const8u), U64(0x8000_0000_0000_0000), + Op(DW_OP_ne), + Op(DW_OP_bra), Branch(fail), + + // Success. + Op(DW_OP_lit0), + Op(DW_OP_nop), + Op(DW_OP_skip), Branch(done), + + Mark(fail), + Op(DW_OP_lit1), + + Mark(done), + Op(DW_OP_stack_value), + ]; + + let result = [Piece { + size_in_bits: None, + bit_offset: None, + location: Location::Value { + value: Value::Generic(0), + }, + }]; + + check_eval(&program, Ok(&result), encoding8()); + } + + #[test] + fn test_eval_compare() { + // It's nice if an operation and its arguments can fit on a single + // line in the test program. + use self::AssemblerEntry::*; + use crate::constants::*; + + // Indices of marks in the assembly. + let done = 0; + let fail = 1; + + #[rustfmt::skip] + let program = [ + // Comparisons are signed. + Op(DW_OP_const1s), U8(1), + Op(DW_OP_const1s), U8(0xff), + Op(DW_OP_lt), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_const1s), U8(0xff), + Op(DW_OP_const1s), U8(1), + Op(DW_OP_gt), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_const1s), U8(1), + Op(DW_OP_const1s), U8(0xff), + Op(DW_OP_le), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_const1s), U8(0xff), + Op(DW_OP_const1s), U8(1), + Op(DW_OP_ge), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_const1s), U8(0xff), + Op(DW_OP_const1s), U8(1), + Op(DW_OP_eq), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_const4s), U32(1), + Op(DW_OP_const1s), U8(1), + Op(DW_OP_ne), + Op(DW_OP_bra), Branch(fail), + + // Success. + Op(DW_OP_lit0), + Op(DW_OP_nop), + Op(DW_OP_skip), Branch(done), + + Mark(fail), + Op(DW_OP_lit1), + + Mark(done), + Op(DW_OP_stack_value), + ]; + + let result = [Piece { + size_in_bits: None, + bit_offset: None, + location: Location::Value { + value: Value::Generic(0), + }, + }]; + + check_eval(&program, Ok(&result), encoding4()); + } + + #[test] + fn test_eval_stack() { + // It's nice if an operation and its arguments can fit on a single + // line in the test program. + use self::AssemblerEntry::*; + use crate::constants::*; + + #[rustfmt::skip] + let program = [ + Op(DW_OP_lit17), // -- 17 + Op(DW_OP_dup), // -- 17 17 + Op(DW_OP_over), // -- 17 17 17 + Op(DW_OP_minus), // -- 17 0 + Op(DW_OP_swap), // -- 0 17 + Op(DW_OP_dup), // -- 0 17 17 + Op(DW_OP_plus_uconst), Uleb(1), // -- 0 17 18 + Op(DW_OP_rot), // -- 18 0 17 + Op(DW_OP_pick), U8(2), // -- 18 0 17 18 + Op(DW_OP_pick), U8(3), // -- 18 0 17 18 18 + Op(DW_OP_minus), // -- 18 0 17 0 + Op(DW_OP_drop), // -- 18 0 17 + Op(DW_OP_swap), // -- 18 17 0 + Op(DW_OP_drop), // -- 18 17 + Op(DW_OP_minus), // -- 1 + Op(DW_OP_stack_value), + ]; + + let result = [Piece { + size_in_bits: None, + bit_offset: None, + location: Location::Value { + value: Value::Generic(1), + }, + }]; + + check_eval(&program, Ok(&result), encoding4()); + } + + #[test] + fn test_eval_lit_and_reg() { + // It's nice if an operation and its arguments can fit on a single + // line in the test program. + use self::AssemblerEntry::*; + use crate::constants::*; + + let mut program = Vec::new(); + program.push(Op(DW_OP_lit0)); + for i in 0..32 { + program.push(Op(DwOp(DW_OP_lit0.0 + i))); + program.push(Op(DwOp(DW_OP_breg0.0 + i))); + program.push(Sleb(u64::from(i))); + program.push(Op(DW_OP_plus)); + program.push(Op(DW_OP_plus)); + } + + program.push(Op(DW_OP_bregx)); + program.push(Uleb(0x1234)); + program.push(Sleb(0x1234)); + program.push(Op(DW_OP_plus)); + + program.push(Op(DW_OP_stack_value)); + + let result = [Piece { + size_in_bits: None, + bit_offset: None, + location: Location::Value { + value: Value::Generic(496), + }, + }]; + + check_eval_with_args( + &program, + Ok(&result), + encoding4(), + None, + None, + None, + |eval, mut result| { + while result != EvaluationResult::Complete { + result = eval.resume_with_register(match result { + EvaluationResult::RequiresRegister { + register, + base_type, + } => { + assert_eq!(base_type, UnitOffset(0)); + Value::Generic(u64::from(register.0).wrapping_neg()) + } + _ => panic!(), + })?; + } + Ok(result) + }, + ); + } + + #[test] + fn test_eval_memory() { + // It's nice if an operation and its arguments can fit on a single + // line in the test program. + use self::AssemblerEntry::*; + use crate::constants::*; + + // Indices of marks in the assembly. + let done = 0; + let fail = 1; + + #[rustfmt::skip] + let program = [ + Op(DW_OP_addr), U32(0x7fff_ffff), + Op(DW_OP_deref), + Op(DW_OP_const4u), U32(0xffff_fffc), + Op(DW_OP_ne), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_addr), U32(0x7fff_ffff), + Op(DW_OP_deref_size), U8(2), + Op(DW_OP_const4u), U32(0xfffc), + Op(DW_OP_ne), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_lit1), + Op(DW_OP_addr), U32(0x7fff_ffff), + Op(DW_OP_xderef), + Op(DW_OP_const4u), U32(0xffff_fffd), + Op(DW_OP_ne), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_lit1), + Op(DW_OP_addr), U32(0x7fff_ffff), + Op(DW_OP_xderef_size), U8(2), + Op(DW_OP_const4u), U32(0xfffd), + Op(DW_OP_ne), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_lit17), + Op(DW_OP_form_tls_address), + Op(DW_OP_constu), Uleb(!17), + Op(DW_OP_ne), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_lit17), + Op(DW_OP_GNU_push_tls_address), + Op(DW_OP_constu), Uleb(!17), + Op(DW_OP_ne), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_addrx), Uleb(0x10), + Op(DW_OP_deref), + Op(DW_OP_const4u), U32(0x4040), + Op(DW_OP_ne), + Op(DW_OP_bra), Branch(fail), + + Op(DW_OP_constx), Uleb(17), + Op(DW_OP_form_tls_address), + Op(DW_OP_constu), Uleb(!27), + Op(DW_OP_ne), + Op(DW_OP_bra), Branch(fail), + + // Success. + Op(DW_OP_lit0), + Op(DW_OP_nop), + Op(DW_OP_skip), Branch(done), + + Mark(fail), + Op(DW_OP_lit1), + + Mark(done), + Op(DW_OP_stack_value), + ]; + + let result = [Piece { + size_in_bits: None, + bit_offset: None, + location: Location::Value { + value: Value::Generic(0), + }, + }]; + + check_eval_with_args( + &program, + Ok(&result), + encoding4(), + None, + None, + None, + |eval, mut result| { + while result != EvaluationResult::Complete { + result = match result { + EvaluationResult::RequiresMemory { + address, + size, + space, + base_type, + } => { + assert_eq!(base_type, UnitOffset(0)); + let mut v = address << 2; + if let Some(value) = space { + v += value; + } + v &= (1u64 << (8 * size)) - 1; + eval.resume_with_memory(Value::Generic(v))? + } + EvaluationResult::RequiresTls(slot) => eval.resume_with_tls(!slot)?, + EvaluationResult::RequiresRelocatedAddress(address) => { + eval.resume_with_relocated_address(address)? + } + EvaluationResult::RequiresIndexedAddress { index, relocate } => { + if relocate { + eval.resume_with_indexed_address(0x1000 + index.0 as u64)? + } else { + eval.resume_with_indexed_address(10 + index.0 as u64)? + } + } + _ => panic!(), + }; + } + + Ok(result) + }, + ); + } + + #[test] + fn test_eval_register() { + // It's nice if an operation and its arguments can fit on a single + // line in the test program. + use self::AssemblerEntry::*; + use crate::constants::*; + + for i in 0..32 { + #[rustfmt::skip] + let program = [ + Op(DwOp(DW_OP_reg0.0 + i)), + // Included only in the "bad" run. + Op(DW_OP_lit23), + ]; + let ok_result = [Piece { + size_in_bits: None, + bit_offset: None, + location: Location::Register { + register: Register(i.into()), + }, + }]; + + check_eval(&program[..1], Ok(&ok_result), encoding4()); + + check_eval( + &program, + Err(Error::InvalidExpressionTerminator(1)), + encoding4(), + ); + } + + #[rustfmt::skip] + let program = [ + Op(DW_OP_regx), Uleb(0x1234) + ]; + + let result = [Piece { + size_in_bits: None, + bit_offset: None, + location: Location::Register { + register: Register(0x1234), + }, + }]; + + check_eval(&program, Ok(&result), encoding4()); + } + + #[test] + fn test_eval_context() { + // It's nice if an operation and its arguments can fit on a single + // line in the test program. + use self::AssemblerEntry::*; + use crate::constants::*; + + // Test `frame_base` and `call_frame_cfa` callbacks. + #[rustfmt::skip] + let program = [ + Op(DW_OP_fbreg), Sleb((-8i8) as u64), + Op(DW_OP_call_frame_cfa), + Op(DW_OP_plus), + Op(DW_OP_neg), + Op(DW_OP_stack_value) + ]; + + let result = [Piece { + size_in_bits: None, + bit_offset: None, + location: Location::Value { + value: Value::Generic(9), + }, + }]; + + check_eval_with_args( + &program, + Ok(&result), + encoding8(), + None, + None, + None, + |eval, result| { + match result { + EvaluationResult::RequiresFrameBase => {} + _ => panic!(), + }; + match eval.resume_with_frame_base(0x0123_4567_89ab_cdef)? { + EvaluationResult::RequiresCallFrameCfa => {} + _ => panic!(), + }; + eval.resume_with_call_frame_cfa(0xfedc_ba98_7654_3210) + }, + ); + + // Test `evaluate_entry_value` callback. + #[rustfmt::skip] + let program = [ + Op(DW_OP_entry_value), Uleb(8), U64(0x1234_5678), + Op(DW_OP_stack_value) + ]; + + let result = [Piece { + size_in_bits: None, + bit_offset: None, + location: Location::Value { + value: Value::Generic(0x1234_5678), + }, + }]; + + check_eval_with_args( + &program, + Ok(&result), + encoding8(), + None, + None, + None, + |eval, result| { + let entry_value = match result { + EvaluationResult::RequiresEntryValue(mut expression) => { + expression.0.read_u64()? + } + _ => panic!(), + }; + eval.resume_with_entry_value(Value::Generic(entry_value)) + }, + ); + + // Test missing `object_address` field. + #[rustfmt::skip] + let program = [ + Op(DW_OP_push_object_address), + ]; + + check_eval_with_args( + &program, + Err(Error::InvalidPushObjectAddress), + encoding4(), + None, + None, + None, + |_, _| panic!(), + ); + + // Test `object_address` field. + #[rustfmt::skip] + let program = [ + Op(DW_OP_push_object_address), + Op(DW_OP_stack_value), + ]; + + let result = [Piece { + size_in_bits: None, + bit_offset: None, + location: Location::Value { + value: Value::Generic(0xff), + }, + }]; + + check_eval_with_args( + &program, + Ok(&result), + encoding8(), + Some(0xff), + None, + None, + |_, result| Ok(result), + ); + + // Test `initial_value` field. + #[rustfmt::skip] + let program = [ + ]; + + let result = [Piece { + size_in_bits: None, + bit_offset: None, + location: Location::Address { + address: 0x1234_5678, + }, + }]; + + check_eval_with_args( + &program, + Ok(&result), + encoding8(), + None, + Some(0x1234_5678), + None, + |_, result| Ok(result), + ); + } + + #[test] + fn test_eval_empty_stack() { + // It's nice if an operation and its arguments can fit on a single + // line in the test program. + use self::AssemblerEntry::*; + use crate::constants::*; + + #[rustfmt::skip] + let program = [ + Op(DW_OP_stack_value) + ]; + + check_eval(&program, Err(Error::NotEnoughStackItems), encoding4()); + } + + #[test] + fn test_eval_call() { + // It's nice if an operation and its arguments can fit on a single + // line in the test program. + use self::AssemblerEntry::*; + use crate::constants::*; + + #[rustfmt::skip] + let program = [ + Op(DW_OP_lit23), + Op(DW_OP_call2), U16(0x7755), + Op(DW_OP_call4), U32(0x7755_aaee), + Op(DW_OP_call_ref), U32(0x7755_aaee), + Op(DW_OP_stack_value) + ]; + + let result = [Piece { + size_in_bits: None, + bit_offset: None, + location: Location::Value { + value: Value::Generic(23), + }, + }]; + + check_eval_with_args( + &program, + Ok(&result), + encoding4(), + None, + None, + None, + |eval, result| { + let buf = EndianSlice::new(&[], LittleEndian); + match result { + EvaluationResult::RequiresAtLocation(_) => {} + _ => panic!(), + }; + + eval.resume_with_at_location(buf)?; + + match result { + EvaluationResult::RequiresAtLocation(_) => {} + _ => panic!(), + }; + + eval.resume_with_at_location(buf)?; + + match result { + EvaluationResult::RequiresAtLocation(_) => {} + _ => panic!(), + }; + + eval.resume_with_at_location(buf) + }, + ); + + // DW_OP_lit2 DW_OP_mul + const SUBR: &[u8] = &[0x32, 0x1e]; + + let result = [Piece { + size_in_bits: None, + bit_offset: None, + location: Location::Value { + value: Value::Generic(184), + }, + }]; + + check_eval_with_args( + &program, + Ok(&result), + encoding4(), + None, + None, + None, + |eval, result| { + let buf = EndianSlice::new(SUBR, LittleEndian); + match result { + EvaluationResult::RequiresAtLocation(_) => {} + _ => panic!(), + }; + + eval.resume_with_at_location(buf)?; + + match result { + EvaluationResult::RequiresAtLocation(_) => {} + _ => panic!(), + }; + + eval.resume_with_at_location(buf)?; + + match result { + EvaluationResult::RequiresAtLocation(_) => {} + _ => panic!(), + }; + + eval.resume_with_at_location(buf) + }, + ); + } + + #[test] + fn test_eval_pieces() { + // It's nice if an operation and its arguments can fit on a single + // line in the test program. + use self::AssemblerEntry::*; + use crate::constants::*; + + // Example from DWARF 2.6.1.3. + #[rustfmt::skip] + let program = [ + Op(DW_OP_reg3), + Op(DW_OP_piece), Uleb(4), + Op(DW_OP_reg4), + Op(DW_OP_piece), Uleb(2), + ]; + + let result = [ + Piece { + size_in_bits: Some(32), + bit_offset: None, + location: Location::Register { + register: Register(3), + }, + }, + Piece { + size_in_bits: Some(16), + bit_offset: None, + location: Location::Register { + register: Register(4), + }, + }, + ]; + + check_eval(&program, Ok(&result), encoding4()); + + // Example from DWARF 2.6.1.3 (but hacked since dealing with fbreg + // in the tests is a pain). + #[rustfmt::skip] + let program = [ + Op(DW_OP_reg0), + Op(DW_OP_piece), Uleb(4), + Op(DW_OP_piece), Uleb(4), + Op(DW_OP_addr), U32(0x7fff_ffff), + Op(DW_OP_piece), Uleb(4), + ]; + + let result = [ + Piece { + size_in_bits: Some(32), + bit_offset: None, + location: Location::Register { + register: Register(0), + }, + }, + Piece { + size_in_bits: Some(32), + bit_offset: None, + location: Location::Empty, + }, + Piece { + size_in_bits: Some(32), + bit_offset: None, + location: Location::Address { + address: 0x7fff_ffff, + }, + }, + ]; + + check_eval_with_args( + &program, + Ok(&result), + encoding4(), + None, + None, + None, + |eval, mut result| { + while result != EvaluationResult::Complete { + result = match result { + EvaluationResult::RequiresRelocatedAddress(address) => { + eval.resume_with_relocated_address(address)? + } + _ => panic!(), + }; + } + + Ok(result) + }, + ); + + #[rustfmt::skip] + let program = [ + Op(DW_OP_implicit_value), Uleb(5), + U8(23), U8(24), U8(25), U8(26), U8(0), + ]; + + const BYTES: &[u8] = &[23, 24, 25, 26, 0]; + + let result = [Piece { + size_in_bits: None, + bit_offset: None, + location: Location::Bytes { + value: EndianSlice::new(BYTES, LittleEndian), + }, + }]; + + check_eval(&program, Ok(&result), encoding4()); + + #[rustfmt::skip] + let program = [ + Op(DW_OP_lit7), + Op(DW_OP_stack_value), + Op(DW_OP_bit_piece), Uleb(5), Uleb(0), + Op(DW_OP_bit_piece), Uleb(3), Uleb(0), + ]; + + let result = [ + Piece { + size_in_bits: Some(5), + bit_offset: Some(0), + location: Location::Value { + value: Value::Generic(7), + }, + }, + Piece { + size_in_bits: Some(3), + bit_offset: Some(0), + location: Location::Empty, + }, + ]; + + check_eval(&program, Ok(&result), encoding4()); + + #[rustfmt::skip] + let program = [ + Op(DW_OP_lit7), + ]; + + let result = [Piece { + size_in_bits: None, + bit_offset: None, + location: Location::Address { address: 7 }, + }]; + + check_eval(&program, Ok(&result), encoding4()); + + #[rustfmt::skip] + let program = [ + Op(DW_OP_implicit_pointer), U32(0x1234_5678), Sleb(0x123), + ]; + + let result = [Piece { + size_in_bits: None, + bit_offset: None, + location: Location::ImplicitPointer { + value: DebugInfoOffset(0x1234_5678), + byte_offset: 0x123, + }, + }]; + + check_eval(&program, Ok(&result), encoding4()); + + #[rustfmt::skip] + let program = [ + Op(DW_OP_reg3), + Op(DW_OP_piece), Uleb(4), + Op(DW_OP_reg4), + ]; + + check_eval(&program, Err(Error::InvalidPiece), encoding4()); + + #[rustfmt::skip] + let program = [ + Op(DW_OP_reg3), + Op(DW_OP_piece), Uleb(4), + Op(DW_OP_lit0), + ]; + + check_eval(&program, Err(Error::InvalidPiece), encoding4()); + } + + #[test] + fn test_eval_max_iterations() { + // It's nice if an operation and its arguments can fit on a single + // line in the test program. + use self::AssemblerEntry::*; + use crate::constants::*; + + #[rustfmt::skip] + let program = [ + Mark(1), + Op(DW_OP_skip), Branch(1), + ]; + + check_eval_with_args( + &program, + Err(Error::TooManyIterations), + encoding4(), + None, + None, + Some(150), + |_, _| panic!(), + ); + } + + #[test] + fn test_eval_typed_stack() { + use self::AssemblerEntry::*; + use crate::constants::*; + + let base_types = [ + ValueType::Generic, + ValueType::U16, + ValueType::U32, + ValueType::F32, + ]; + + // TODO: convert, reinterpret + #[rustfmt::skip] + let tests = [ + ( + &[ + Op(DW_OP_const_type), Uleb(1), U8(2), U16(0x1234), + Op(DW_OP_stack_value), + ][..], + Value::U16(0x1234), + ), + ( + &[ + Op(DW_OP_regval_type), Uleb(0x1234), Uleb(1), + Op(DW_OP_stack_value), + ][..], + Value::U16(0x2340), + ), + ( + &[ + Op(DW_OP_addr), U32(0x7fff_ffff), + Op(DW_OP_deref_type), U8(2), Uleb(1), + Op(DW_OP_stack_value), + ][..], + Value::U16(0xfff0), + ), + ( + &[ + Op(DW_OP_lit1), + Op(DW_OP_addr), U32(0x7fff_ffff), + Op(DW_OP_xderef_type), U8(2), Uleb(1), + Op(DW_OP_stack_value), + ][..], + Value::U16(0xfff1), + ), + ( + &[ + Op(DW_OP_const_type), Uleb(1), U8(2), U16(0x1234), + Op(DW_OP_convert), Uleb(2), + Op(DW_OP_stack_value), + ][..], + Value::U32(0x1234), + ), + ( + &[ + Op(DW_OP_const_type), Uleb(2), U8(4), U32(0x3f80_0000), + Op(DW_OP_reinterpret), Uleb(3), + Op(DW_OP_stack_value), + ][..], + Value::F32(1.0), + ), + ]; + for &(program, value) in &tests { + let result = [Piece { + size_in_bits: None, + bit_offset: None, + location: Location::Value { value }, + }]; + + check_eval_with_args( + program, + Ok(&result), + encoding4(), + None, + None, + None, + |eval, mut result| { + while result != EvaluationResult::Complete { + result = match result { + EvaluationResult::RequiresMemory { + address, + size, + space, + base_type, + } => { + let mut v = address << 4; + if let Some(value) = space { + v += value; + } + v &= (1u64 << (8 * size)) - 1; + let v = Value::from_u64(base_types[base_type.0], v)?; + eval.resume_with_memory(v)? + } + EvaluationResult::RequiresRegister { + register, + base_type, + } => { + let v = Value::from_u64( + base_types[base_type.0], + u64::from(register.0) << 4, + )?; + eval.resume_with_register(v)? + } + EvaluationResult::RequiresBaseType(offset) => { + eval.resume_with_base_type(base_types[offset.0])? + } + EvaluationResult::RequiresRelocatedAddress(address) => { + eval.resume_with_relocated_address(address)? + } + _ => panic!("Unexpected result {:?}", result), + } + } + Ok(result) + }, + ); + } + } +} diff --git a/crux-mir/lib/gimli/src/read/pubnames.rs b/crux-mir/lib/gimli/src/read/pubnames.rs new file mode 100644 index 000000000..e8b7e5528 --- /dev/null +++ b/crux-mir/lib/gimli/src/read/pubnames.rs @@ -0,0 +1,141 @@ +use crate::common::{DebugInfoOffset, SectionId}; +use crate::endianity::Endianity; +use crate::read::lookup::{DebugLookup, LookupEntryIter, PubStuffEntry, PubStuffParser}; +use crate::read::{EndianSlice, Reader, Result, Section, UnitOffset}; + +/// A single parsed pubname. +#[derive(Debug, Clone)] +pub struct PubNamesEntry { + unit_header_offset: DebugInfoOffset, + die_offset: UnitOffset, + name: R, +} + +impl PubNamesEntry { + /// Returns the name this entry refers to. + pub fn name(&self) -> &R { + &self.name + } + + /// Returns the offset into the .debug_info section for the header of the compilation unit + /// which contains this name. + pub fn unit_header_offset(&self) -> DebugInfoOffset { + self.unit_header_offset + } + + /// Returns the offset into the compilation unit for the debugging information entry which + /// has this name. + pub fn die_offset(&self) -> UnitOffset { + self.die_offset + } +} + +impl PubStuffEntry for PubNamesEntry { + fn new( + die_offset: UnitOffset, + name: R, + unit_header_offset: DebugInfoOffset, + ) -> Self { + PubNamesEntry { + unit_header_offset, + die_offset, + name, + } + } +} + +/// The `DebugPubNames` struct represents the DWARF public names information +/// found in the `.debug_pubnames` section. +#[derive(Debug, Clone)] +pub struct DebugPubNames(DebugLookup>>); + +impl<'input, Endian> DebugPubNames> +where + Endian: Endianity, +{ + /// Construct a new `DebugPubNames` instance from the data in the `.debug_pubnames` + /// section. + /// + /// It is the caller's responsibility to read the `.debug_pubnames` section and + /// present it as a `&[u8]` slice. That means using some ELF loader on + /// Linux, a Mach-O loader on macOS, etc. + /// + /// ``` + /// use gimli::{DebugPubNames, LittleEndian}; + /// + /// # let buf = []; + /// # let read_debug_pubnames_section_somehow = || &buf; + /// let debug_pubnames = + /// DebugPubNames::new(read_debug_pubnames_section_somehow(), LittleEndian); + /// ``` + pub fn new(debug_pubnames_section: &'input [u8], endian: Endian) -> Self { + Self::from(EndianSlice::new(debug_pubnames_section, endian)) + } +} + +impl DebugPubNames { + /// Iterate the pubnames in the `.debug_pubnames` section. + /// + /// ``` + /// use gimli::{DebugPubNames, EndianSlice, LittleEndian}; + /// + /// # let buf = []; + /// # let read_debug_pubnames_section_somehow = || &buf; + /// let debug_pubnames = + /// DebugPubNames::new(read_debug_pubnames_section_somehow(), LittleEndian); + /// + /// let mut iter = debug_pubnames.items(); + /// while let Some(pubname) = iter.next().unwrap() { + /// println!("pubname {} found!", pubname.name().to_string_lossy()); + /// } + /// ``` + pub fn items(&self) -> PubNamesEntryIter { + PubNamesEntryIter(self.0.items()) + } +} + +impl Section for DebugPubNames { + fn id() -> SectionId { + SectionId::DebugPubNames + } + + fn reader(&self) -> &R { + self.0.reader() + } +} + +impl From for DebugPubNames { + fn from(debug_pubnames_section: R) -> Self { + DebugPubNames(DebugLookup::from(debug_pubnames_section)) + } +} + +/// An iterator over the pubnames from a `.debug_pubnames` section. +/// +/// Can be [used with +/// `FallibleIterator`](./index.html#using-with-fallibleiterator). +#[derive(Debug, Clone)] +pub struct PubNamesEntryIter(LookupEntryIter>>); + +impl PubNamesEntryIter { + /// Advance the iterator and return the next pubname. + /// + /// Returns the newly parsed pubname as `Ok(Some(pubname))`. Returns + /// `Ok(None)` when iteration is complete and all pubnames have already been + /// parsed and yielded. If an error occurs while parsing the next pubname, + /// then this error is returned as `Err(e)`, and all subsequent calls return + /// `Ok(None)`. + pub fn next(&mut self) -> Result>> { + self.0.next() + } +} + +#[cfg(feature = "fallible-iterator")] +impl fallible_iterator::FallibleIterator for PubNamesEntryIter { + type Item = PubNamesEntry; + type Error = crate::read::Error; + + fn next(&mut self) -> ::core::result::Result, Self::Error> { + self.0.next() + } +} diff --git a/crux-mir/lib/gimli/src/read/pubtypes.rs b/crux-mir/lib/gimli/src/read/pubtypes.rs new file mode 100644 index 000000000..6723b4222 --- /dev/null +++ b/crux-mir/lib/gimli/src/read/pubtypes.rs @@ -0,0 +1,141 @@ +use crate::common::{DebugInfoOffset, SectionId}; +use crate::endianity::Endianity; +use crate::read::lookup::{DebugLookup, LookupEntryIter, PubStuffEntry, PubStuffParser}; +use crate::read::{EndianSlice, Reader, Result, Section, UnitOffset}; + +/// A single parsed pubtype. +#[derive(Debug, Clone)] +pub struct PubTypesEntry { + unit_header_offset: DebugInfoOffset, + die_offset: UnitOffset, + name: R, +} + +impl PubTypesEntry { + /// Returns the name of the type this entry refers to. + pub fn name(&self) -> &R { + &self.name + } + + /// Returns the offset into the .debug_info section for the header of the compilation unit + /// which contains the type with this name. + pub fn unit_header_offset(&self) -> DebugInfoOffset { + self.unit_header_offset + } + + /// Returns the offset into the compilation unit for the debugging information entry which + /// the type with this name. + pub fn die_offset(&self) -> UnitOffset { + self.die_offset + } +} + +impl PubStuffEntry for PubTypesEntry { + fn new( + die_offset: UnitOffset, + name: R, + unit_header_offset: DebugInfoOffset, + ) -> Self { + PubTypesEntry { + unit_header_offset, + die_offset, + name, + } + } +} + +/// The `DebugPubTypes` struct represents the DWARF public types information +/// found in the `.debug_info` section. +#[derive(Debug, Clone)] +pub struct DebugPubTypes(DebugLookup>>); + +impl<'input, Endian> DebugPubTypes> +where + Endian: Endianity, +{ + /// Construct a new `DebugPubTypes` instance from the data in the `.debug_pubtypes` + /// section. + /// + /// It is the caller's responsibility to read the `.debug_pubtypes` section and + /// present it as a `&[u8]` slice. That means using some ELF loader on + /// Linux, a Mach-O loader on macOS, etc. + /// + /// ``` + /// use gimli::{DebugPubTypes, LittleEndian}; + /// + /// # let buf = []; + /// # let read_debug_pubtypes_somehow = || &buf; + /// let debug_pubtypes = + /// DebugPubTypes::new(read_debug_pubtypes_somehow(), LittleEndian); + /// ``` + pub fn new(debug_pubtypes_section: &'input [u8], endian: Endian) -> Self { + Self::from(EndianSlice::new(debug_pubtypes_section, endian)) + } +} + +impl DebugPubTypes { + /// Iterate the pubtypes in the `.debug_pubtypes` section. + /// + /// ``` + /// use gimli::{DebugPubTypes, EndianSlice, LittleEndian}; + /// + /// # let buf = []; + /// # let read_debug_pubtypes_section_somehow = || &buf; + /// let debug_pubtypes = + /// DebugPubTypes::new(read_debug_pubtypes_section_somehow(), LittleEndian); + /// + /// let mut iter = debug_pubtypes.items(); + /// while let Some(pubtype) = iter.next().unwrap() { + /// println!("pubtype {} found!", pubtype.name().to_string_lossy()); + /// } + /// ``` + pub fn items(&self) -> PubTypesEntryIter { + PubTypesEntryIter(self.0.items()) + } +} + +impl Section for DebugPubTypes { + fn id() -> SectionId { + SectionId::DebugPubTypes + } + + fn reader(&self) -> &R { + self.0.reader() + } +} + +impl From for DebugPubTypes { + fn from(debug_pubtypes_section: R) -> Self { + DebugPubTypes(DebugLookup::from(debug_pubtypes_section)) + } +} + +/// An iterator over the pubtypes from a `.debug_pubtypes` section. +/// +/// Can be [used with +/// `FallibleIterator`](./index.html#using-with-fallibleiterator). +#[derive(Debug, Clone)] +pub struct PubTypesEntryIter(LookupEntryIter>>); + +impl PubTypesEntryIter { + /// Advance the iterator and return the next pubtype. + /// + /// Returns the newly parsed pubtype as `Ok(Some(pubtype))`. Returns + /// `Ok(None)` when iteration is complete and all pubtypes have already been + /// parsed and yielded. If an error occurs while parsing the next pubtype, + /// then this error is returned as `Err(e)`, and all subsequent calls return + /// `Ok(None)`. + pub fn next(&mut self) -> Result>> { + self.0.next() + } +} + +#[cfg(feature = "fallible-iterator")] +impl fallible_iterator::FallibleIterator for PubTypesEntryIter { + type Item = PubTypesEntry; + type Error = crate::read::Error; + + fn next(&mut self) -> ::core::result::Result, Self::Error> { + self.0.next() + } +} diff --git a/crux-mir/lib/gimli/src/read/reader.rs b/crux-mir/lib/gimli/src/read/reader.rs new file mode 100644 index 000000000..1bb748bb8 --- /dev/null +++ b/crux-mir/lib/gimli/src/read/reader.rs @@ -0,0 +1,502 @@ +#[cfg(feature = "read")] +use alloc::borrow::Cow; +use core::convert::TryInto; +use core::fmt::Debug; +use core::hash::Hash; +use core::ops::{Add, AddAssign, Sub}; + +use crate::common::Format; +use crate::endianity::Endianity; +use crate::leb128; +use crate::read::{Error, Result}; + +/// An identifier for an offset within a section reader. +/// +/// This is used for error reporting. The meaning of this value is specific to +/// each reader implementation. The values should be chosen to be unique amongst +/// all readers. If values are not unique then errors may point to the wrong reader. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReaderOffsetId(pub u64); + +/// A trait for offsets with a DWARF section. +/// +/// This allows consumers to choose a size that is appropriate for their address space. +pub trait ReaderOffset: + Debug + Copy + Eq + Ord + Hash + Add + AddAssign + Sub +{ + /// Convert a u8 to an offset. + fn from_u8(offset: u8) -> Self; + + /// Convert a u16 to an offset. + fn from_u16(offset: u16) -> Self; + + /// Convert an i16 to an offset. + fn from_i16(offset: i16) -> Self; + + /// Convert a u32 to an offset. + fn from_u32(offset: u32) -> Self; + + /// Convert a u64 to an offset. + /// + /// Returns `Error::UnsupportedOffset` if the value is too large. + fn from_u64(offset: u64) -> Result; + + /// Convert an offset to a u64. + fn into_u64(self) -> u64; + + /// Wrapping (modular) addition. Computes `self + other`. + fn wrapping_add(self, other: Self) -> Self; + + /// Checked subtraction. Computes `self - other`. + fn checked_sub(self, other: Self) -> Option; +} + +impl ReaderOffset for u64 { + #[inline] + fn from_u8(offset: u8) -> Self { + u64::from(offset) + } + + #[inline] + fn from_u16(offset: u16) -> Self { + u64::from(offset) + } + + #[inline] + fn from_i16(offset: i16) -> Self { + offset as u64 + } + + #[inline] + fn from_u32(offset: u32) -> Self { + u64::from(offset) + } + + #[inline] + fn from_u64(offset: u64) -> Result { + Ok(offset) + } + + #[inline] + fn into_u64(self) -> u64 { + self + } + + #[inline] + fn wrapping_add(self, other: Self) -> Self { + self.wrapping_add(other) + } + + #[inline] + fn checked_sub(self, other: Self) -> Option { + self.checked_sub(other) + } +} + +impl ReaderOffset for u32 { + #[inline] + fn from_u8(offset: u8) -> Self { + u32::from(offset) + } + + #[inline] + fn from_u16(offset: u16) -> Self { + u32::from(offset) + } + + #[inline] + fn from_i16(offset: i16) -> Self { + offset as u32 + } + + #[inline] + fn from_u32(offset: u32) -> Self { + offset + } + + #[inline] + fn from_u64(offset64: u64) -> Result { + let offset = offset64 as u32; + if u64::from(offset) == offset64 { + Ok(offset) + } else { + Err(Error::UnsupportedOffset) + } + } + + #[inline] + fn into_u64(self) -> u64 { + u64::from(self) + } + + #[inline] + fn wrapping_add(self, other: Self) -> Self { + self.wrapping_add(other) + } + + #[inline] + fn checked_sub(self, other: Self) -> Option { + self.checked_sub(other) + } +} + +impl ReaderOffset for usize { + #[inline] + fn from_u8(offset: u8) -> Self { + offset as usize + } + + #[inline] + fn from_u16(offset: u16) -> Self { + offset as usize + } + + #[inline] + fn from_i16(offset: i16) -> Self { + offset as usize + } + + #[inline] + fn from_u32(offset: u32) -> Self { + offset as usize + } + + #[inline] + fn from_u64(offset64: u64) -> Result { + let offset = offset64 as usize; + if offset as u64 == offset64 { + Ok(offset) + } else { + Err(Error::UnsupportedOffset) + } + } + + #[inline] + fn into_u64(self) -> u64 { + self as u64 + } + + #[inline] + fn wrapping_add(self, other: Self) -> Self { + self.wrapping_add(other) + } + + #[inline] + fn checked_sub(self, other: Self) -> Option { + self.checked_sub(other) + } +} + +#[cfg(not(feature = "read"))] +pub(crate) mod seal_if_no_alloc { + #[derive(Debug)] + pub struct Sealed; +} + +/// A trait for reading the data from a DWARF section. +/// +/// All read operations advance the section offset of the reader +/// unless specified otherwise. +/// +/// ## Choosing a `Reader` Implementation +/// +/// `gimli` comes with a few different `Reader` implementations and lets you +/// choose the one that is right for your use case. A `Reader` is essentially a +/// view into the raw bytes that make up some DWARF, but this view might borrow +/// the underlying data or use reference counting ownership, and it might be +/// thread safe or not. +/// +/// | Implementation | Ownership | Thread Safe | Notes | +/// |:------------------|:------------------|:------------|:------| +/// | [`EndianSlice`](./struct.EndianSlice.html) | Borrowed | Yes | Fastest, but requires that all of your code work with borrows. | +/// | [`EndianRcSlice`](./struct.EndianRcSlice.html) | Reference counted | No | Shared ownership via reference counting, which alleviates the borrow restrictions of `EndianSlice` but imposes reference counting increments and decrements. Cannot be sent across threads, because the reference count is not atomic. | +/// | [`EndianArcSlice`](./struct.EndianArcSlice.html) | Reference counted | Yes | The same as `EndianRcSlice`, but uses atomic reference counting, and therefore reference counting operations are slower but `EndianArcSlice`s may be sent across threads. | +/// | [`EndianReader`](./struct.EndianReader.html) | Same as `T` | Same as `T` | Escape hatch for easily defining your own type of `Reader`. | +pub trait Reader: Debug + Clone { + /// The endianity of bytes that are read. + type Endian: Endianity; + + /// The type used for offsets and lengths. + type Offset: ReaderOffset; + + /// Return the endianity of bytes that are read. + fn endian(&self) -> Self::Endian; + + /// Return the number of bytes remaining. + fn len(&self) -> Self::Offset; + + /// Set the number of bytes remaining to zero. + fn empty(&mut self); + + /// Set the number of bytes remaining to the specified length. + fn truncate(&mut self, len: Self::Offset) -> Result<()>; + + /// Return the offset of this reader's data relative to the start of + /// the given base reader's data. + /// + /// May panic if this reader's data is not contained within the given + /// base reader's data. + fn offset_from(&self, base: &Self) -> Self::Offset; + + /// Return an identifier for the current reader offset. + fn offset_id(&self) -> ReaderOffsetId; + + /// Return the offset corresponding to the given `id` if + /// it is associated with this reader. + fn lookup_offset_id(&self, id: ReaderOffsetId) -> Option; + + /// Find the index of the first occurence of the given byte. + /// The offset of the reader is not changed. + fn find(&self, byte: u8) -> Result; + + /// Discard the specified number of bytes. + fn skip(&mut self, len: Self::Offset) -> Result<()>; + + /// Split a reader in two. + /// + /// A new reader is returned that can be used to read the next + /// `len` bytes, and `self` is advanced so that it reads the remainder. + fn split(&mut self, len: Self::Offset) -> Result; + + /// This trait cannot be implemented if "read" feature is not enabled. + /// + /// `Reader` trait has a few methods that depend on `alloc` crate. + /// Disallowing `Reader` trait implementation prevents a crate that only depends on + /// "read-core" from being broken if another crate depending on `gimli` enables + /// "read" feature. + #[cfg(not(feature = "read"))] + fn cannot_implement() -> seal_if_no_alloc::Sealed; + + /// Return all remaining data as a clone-on-write slice. + /// + /// The slice will be borrowed where possible, but some readers may + /// always return an owned vector. + /// + /// Does not advance the reader. + #[cfg(feature = "read")] + fn to_slice(&self) -> Result>; + + /// Convert all remaining data to a clone-on-write string. + /// + /// The string will be borrowed where possible, but some readers may + /// always return an owned string. + /// + /// Does not advance the reader. + /// + /// Returns an error if the data contains invalid characters. + #[cfg(feature = "read")] + fn to_string(&self) -> Result>; + + /// Convert all remaining data to a clone-on-write string, including invalid characters. + /// + /// The string will be borrowed where possible, but some readers may + /// always return an owned string. + /// + /// Does not advance the reader. + #[cfg(feature = "read")] + fn to_string_lossy(&self) -> Result>; + + /// Read exactly `buf.len()` bytes into `buf`. + fn read_slice(&mut self, buf: &mut [u8]) -> Result<()>; + + /// Read a u8 array. + #[inline] + fn read_u8_array(&mut self) -> Result + where + A: Sized + Default + AsMut<[u8]>, + { + let mut val = Default::default(); + self.read_slice(>::as_mut(&mut val))?; + Ok(val) + } + + /// Return true if the number of bytes remaining is zero. + #[inline] + fn is_empty(&self) -> bool { + self.len() == Self::Offset::from_u8(0) + } + + /// Read a u8. + #[inline] + fn read_u8(&mut self) -> Result { + let a: [u8; 1] = self.read_u8_array()?; + Ok(a[0]) + } + + /// Read an i8. + #[inline] + fn read_i8(&mut self) -> Result { + let a: [u8; 1] = self.read_u8_array()?; + Ok(a[0] as i8) + } + + /// Read a u16. + #[inline] + fn read_u16(&mut self) -> Result { + let a: [u8; 2] = self.read_u8_array()?; + Ok(self.endian().read_u16(&a)) + } + + /// Read an i16. + #[inline] + fn read_i16(&mut self) -> Result { + let a: [u8; 2] = self.read_u8_array()?; + Ok(self.endian().read_i16(&a)) + } + + /// Read a u32. + #[inline] + fn read_u32(&mut self) -> Result { + let a: [u8; 4] = self.read_u8_array()?; + Ok(self.endian().read_u32(&a)) + } + + /// Read an i32. + #[inline] + fn read_i32(&mut self) -> Result { + let a: [u8; 4] = self.read_u8_array()?; + Ok(self.endian().read_i32(&a)) + } + + /// Read a u64. + #[inline] + fn read_u64(&mut self) -> Result { + let a: [u8; 8] = self.read_u8_array()?; + Ok(self.endian().read_u64(&a)) + } + + /// Read an i64. + #[inline] + fn read_i64(&mut self) -> Result { + let a: [u8; 8] = self.read_u8_array()?; + Ok(self.endian().read_i64(&a)) + } + + /// Read a f32. + #[inline] + fn read_f32(&mut self) -> Result { + let a: [u8; 4] = self.read_u8_array()?; + Ok(self.endian().read_f32(&a)) + } + + /// Read a f64. + #[inline] + fn read_f64(&mut self) -> Result { + let a: [u8; 8] = self.read_u8_array()?; + Ok(self.endian().read_f64(&a)) + } + + /// Read an unsigned n-bytes integer u64. + /// + /// # Panics + /// + /// Panics when nbytes < 1 or nbytes > 8 + #[inline] + fn read_uint(&mut self, n: usize) -> Result { + let mut buf = [0; 8]; + self.read_slice(&mut buf[..n])?; + Ok(self.endian().read_uint(&buf[..n])) + } + + /// Read a null-terminated slice, and return it (excluding the null). + fn read_null_terminated_slice(&mut self) -> Result { + let idx = self.find(0)?; + let val = self.split(idx)?; + self.skip(Self::Offset::from_u8(1))?; + Ok(val) + } + + /// Skip a LEB128 encoded integer. + fn skip_leb128(&mut self) -> Result<()> { + leb128::read::skip(self) + } + + /// Read an unsigned LEB128 encoded integer. + fn read_uleb128(&mut self) -> Result { + leb128::read::unsigned(self) + } + + /// Read an unsigned LEB128 encoded u32. + fn read_uleb128_u32(&mut self) -> Result { + leb128::read::unsigned(self)? + .try_into() + .map_err(|_| Error::BadUnsignedLeb128) + } + + /// Read an unsigned LEB128 encoded u16. + fn read_uleb128_u16(&mut self) -> Result { + leb128::read::u16(self) + } + + /// Read a signed LEB128 encoded integer. + fn read_sleb128(&mut self) -> Result { + leb128::read::signed(self) + } + + /// Read an initial length field. + /// + /// This field is encoded as either a 32-bit length or + /// a 64-bit length, and the returned `Format` indicates which. + fn read_initial_length(&mut self) -> Result<(Self::Offset, Format)> { + const MAX_DWARF_32_UNIT_LENGTH: u32 = 0xffff_fff0; + const DWARF_64_INITIAL_UNIT_LENGTH: u32 = 0xffff_ffff; + + let val = self.read_u32()?; + if val < MAX_DWARF_32_UNIT_LENGTH { + Ok((Self::Offset::from_u32(val), Format::Dwarf32)) + } else if val == DWARF_64_INITIAL_UNIT_LENGTH { + let val = self.read_u64().and_then(Self::Offset::from_u64)?; + Ok((val, Format::Dwarf64)) + } else { + Err(Error::UnknownReservedLength) + } + } + + /// Read an address-sized integer, and return it as a `u64`. + fn read_address(&mut self, address_size: u8) -> Result { + match address_size { + 1 => self.read_u8().map(u64::from), + 2 => self.read_u16().map(u64::from), + 4 => self.read_u32().map(u64::from), + 8 => self.read_u64(), + otherwise => Err(Error::UnsupportedAddressSize(otherwise)), + } + } + + /// Parse a word-sized integer according to the DWARF format. + /// + /// These are always used to encode section offsets or lengths, + /// and so have a type of `Self::Offset`. + fn read_word(&mut self, format: Format) -> Result { + match format { + Format::Dwarf32 => self.read_u32().map(Self::Offset::from_u32), + Format::Dwarf64 => self.read_u64().and_then(Self::Offset::from_u64), + } + } + + /// Parse a word-sized section length according to the DWARF format. + #[inline] + fn read_length(&mut self, format: Format) -> Result { + self.read_word(format) + } + + /// Parse a word-sized section offset according to the DWARF format. + #[inline] + fn read_offset(&mut self, format: Format) -> Result { + self.read_word(format) + } + + /// Parse a section offset of the given size. + /// + /// This is used for `DW_FORM_ref_addr` values in DWARF version 2. + fn read_sized_offset(&mut self, size: u8) -> Result { + match size { + 1 => self.read_u8().map(u64::from), + 2 => self.read_u16().map(u64::from), + 4 => self.read_u32().map(u64::from), + 8 => self.read_u64(), + otherwise => Err(Error::UnsupportedOffsetSize(otherwise)), + } + .and_then(Self::Offset::from_u64) + } +} diff --git a/crux-mir/lib/gimli/src/read/rnglists.rs b/crux-mir/lib/gimli/src/read/rnglists.rs new file mode 100644 index 000000000..d8d49042f --- /dev/null +++ b/crux-mir/lib/gimli/src/read/rnglists.rs @@ -0,0 +1,1354 @@ +use crate::common::{ + DebugAddrBase, DebugAddrIndex, DebugRngListsBase, DebugRngListsIndex, DwarfFileType, Encoding, + RangeListsOffset, SectionId, +}; +use crate::constants; +use crate::endianity::Endianity; +use crate::read::{ + lists::ListsHeader, DebugAddr, EndianSlice, Error, Reader, ReaderOffset, ReaderOffsetId, + Result, Section, +}; + +/// The raw contents of the `.debug_ranges` section. +#[derive(Debug, Default, Clone, Copy)] +pub struct DebugRanges { + pub(crate) section: R, +} + +impl<'input, Endian> DebugRanges> +where + Endian: Endianity, +{ + /// Construct a new `DebugRanges` instance from the data in the `.debug_ranges` + /// section. + /// + /// It is the caller's responsibility to read the `.debug_ranges` section and + /// present it as a `&[u8]` slice. That means using some ELF loader on + /// Linux, a Mach-O loader on macOS, etc. + /// + /// ``` + /// use gimli::{DebugRanges, LittleEndian}; + /// + /// # let buf = [0x00, 0x01, 0x02, 0x03]; + /// # let read_debug_ranges_section_somehow = || &buf; + /// let debug_ranges = DebugRanges::new(read_debug_ranges_section_somehow(), LittleEndian); + /// ``` + pub fn new(section: &'input [u8], endian: Endian) -> Self { + Self::from(EndianSlice::new(section, endian)) + } +} + +impl Section for DebugRanges { + fn id() -> SectionId { + SectionId::DebugRanges + } + + fn reader(&self) -> &R { + &self.section + } +} + +impl From for DebugRanges { + fn from(section: R) -> Self { + DebugRanges { section } + } +} + +/// The `DebugRngLists` struct represents the contents of the +/// `.debug_rnglists` section. +#[derive(Debug, Default, Clone, Copy)] +pub struct DebugRngLists { + section: R, +} + +impl<'input, Endian> DebugRngLists> +where + Endian: Endianity, +{ + /// Construct a new `DebugRngLists` instance from the data in the + /// `.debug_rnglists` section. + /// + /// It is the caller's responsibility to read the `.debug_rnglists` + /// section and present it as a `&[u8]` slice. That means using some ELF + /// loader on Linux, a Mach-O loader on macOS, etc. + /// + /// ``` + /// use gimli::{DebugRngLists, LittleEndian}; + /// + /// # let buf = [0x00, 0x01, 0x02, 0x03]; + /// # let read_debug_rnglists_section_somehow = || &buf; + /// let debug_rnglists = + /// DebugRngLists::new(read_debug_rnglists_section_somehow(), LittleEndian); + /// ``` + pub fn new(section: &'input [u8], endian: Endian) -> Self { + Self::from(EndianSlice::new(section, endian)) + } +} + +impl Section for DebugRngLists { + fn id() -> SectionId { + SectionId::DebugRngLists + } + + fn reader(&self) -> &R { + &self.section + } +} + +impl From for DebugRngLists { + fn from(section: R) -> Self { + DebugRngLists { section } + } +} + +#[allow(unused)] +pub(crate) type RngListsHeader = ListsHeader; + +impl DebugRngListsBase +where + Offset: ReaderOffset, +{ + /// Returns a `DebugRngListsBase` with the default value of DW_AT_rnglists_base + /// for the given `Encoding` and `DwarfFileType`. + pub fn default_for_encoding_and_file( + encoding: Encoding, + file_type: DwarfFileType, + ) -> DebugRngListsBase { + if encoding.version >= 5 && file_type == DwarfFileType::Dwo { + // In .dwo files, the compiler omits the DW_AT_rnglists_base attribute (because there is + // only a single unit in the file) but we must skip past the header, which the attribute + // would normally do for us. + DebugRngListsBase(Offset::from_u8(RngListsHeader::size_for_encoding(encoding))) + } else { + DebugRngListsBase(Offset::from_u8(0)) + } + } +} + +/// The DWARF data found in `.debug_ranges` and `.debug_rnglists` sections. +#[derive(Debug, Default, Clone, Copy)] +pub struct RangeLists { + debug_ranges: DebugRanges, + debug_rnglists: DebugRngLists, +} + +impl RangeLists { + /// Construct a new `RangeLists` instance from the data in the `.debug_ranges` and + /// `.debug_rnglists` sections. + pub fn new(debug_ranges: DebugRanges, debug_rnglists: DebugRngLists) -> RangeLists { + RangeLists { + debug_ranges, + debug_rnglists, + } + } + + /// Return the `.debug_ranges` section. + pub fn debug_ranges(&self) -> &DebugRanges { + &self.debug_ranges + } + + /// Replace the `.debug_ranges` section. + /// + /// This is useful for `.dwo` files when using the GNU split-dwarf extension to DWARF 4. + pub fn set_debug_ranges(&mut self, debug_ranges: DebugRanges) { + self.debug_ranges = debug_ranges; + } + + /// Return the `.debug_rnglists` section. + pub fn debug_rnglists(&self) -> &DebugRngLists { + &self.debug_rnglists + } +} + +impl RangeLists { + /// Create a `RangeLists` that references the data in `self`. + /// + /// This is useful when `R` implements `Reader` but `T` does not. + /// + /// ## Example Usage + /// + /// ```rust,no_run + /// # let load_section = || unimplemented!(); + /// // Read the DWARF section into a `Vec` with whatever object loader you're using. + /// let owned_section: gimli::RangeLists> = load_section(); + /// // Create a reference to the DWARF section. + /// let section = owned_section.borrow(|section| { + /// gimli::EndianSlice::new(§ion, gimli::LittleEndian) + /// }); + /// ``` + pub fn borrow<'a, F, R>(&'a self, mut borrow: F) -> RangeLists + where + F: FnMut(&'a T) -> R, + { + RangeLists { + debug_ranges: borrow(&self.debug_ranges.section).into(), + debug_rnglists: borrow(&self.debug_rnglists.section).into(), + } + } +} + +impl RangeLists { + /// Iterate over the `Range` list entries starting at the given offset. + /// + /// The `unit_version` and `address_size` must match the compilation unit that the + /// offset was contained in. + /// + /// The `base_address` should be obtained from the `DW_AT_low_pc` attribute in the + /// `DW_TAG_compile_unit` entry for the compilation unit that contains this range list. + /// + /// Can be [used with + /// `FallibleIterator`](./index.html#using-with-fallibleiterator). + pub fn ranges( + &self, + offset: RangeListsOffset, + unit_encoding: Encoding, + base_address: u64, + debug_addr: &DebugAddr, + debug_addr_base: DebugAddrBase, + ) -> Result> { + Ok(RngListIter::new( + self.raw_ranges(offset, unit_encoding)?, + base_address, + debug_addr.clone(), + debug_addr_base, + )) + } + + /// Iterate over the `RawRngListEntry`ies starting at the given offset. + /// + /// The `unit_encoding` must match the compilation unit that the + /// offset was contained in. + /// + /// This iterator does not perform any processing of the range entries, + /// such as handling base addresses. + /// + /// Can be [used with + /// `FallibleIterator`](./index.html#using-with-fallibleiterator). + pub fn raw_ranges( + &self, + offset: RangeListsOffset, + unit_encoding: Encoding, + ) -> Result> { + let (mut input, format) = if unit_encoding.version <= 4 { + (self.debug_ranges.section.clone(), RangeListsFormat::Bare) + } else { + (self.debug_rnglists.section.clone(), RangeListsFormat::RLE) + }; + input.skip(offset.0)?; + Ok(RawRngListIter::new(input, unit_encoding, format)) + } + + /// Returns the `.debug_rnglists` offset at the given `base` and `index`. + /// + /// The `base` must be the `DW_AT_rnglists_base` value from the compilation unit DIE. + /// This is an offset that points to the first entry following the header. + /// + /// The `index` is the value of a `DW_FORM_rnglistx` attribute. + /// + /// The `unit_encoding` must match the compilation unit that the + /// index was contained in. + pub fn get_offset( + &self, + unit_encoding: Encoding, + base: DebugRngListsBase, + index: DebugRngListsIndex, + ) -> Result> { + let format = unit_encoding.format; + let input = &mut self.debug_rnglists.section.clone(); + input.skip(base.0)?; + input.skip(R::Offset::from_u64( + index.0.into_u64() * u64::from(format.word_size()), + )?)?; + input + .read_offset(format) + .map(|x| RangeListsOffset(base.0 + x)) + } + + /// Call `Reader::lookup_offset_id` for each section, and return the first match. + pub fn lookup_offset_id(&self, id: ReaderOffsetId) -> Option<(SectionId, R::Offset)> { + self.debug_ranges + .lookup_offset_id(id) + .or_else(|| self.debug_rnglists.lookup_offset_id(id)) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RangeListsFormat { + /// The bare range list format used before DWARF 5. + Bare, + /// The DW_RLE encoded range list format used in DWARF 5. + RLE, +} + +/// A raw iterator over an address range list. +/// +/// This iterator does not perform any processing of the range entries, +/// such as handling base addresses. +#[derive(Debug)] +pub struct RawRngListIter { + input: R, + encoding: Encoding, + format: RangeListsFormat, +} + +/// A raw entry in .debug_rnglists +#[derive(Clone, Debug)] +pub enum RawRngListEntry { + /// A range from DWARF version <= 4. + AddressOrOffsetPair { + /// Start of range. May be an address or an offset. + begin: u64, + /// End of range. May be an address or an offset. + end: u64, + }, + /// DW_RLE_base_address + BaseAddress { + /// base address + addr: u64, + }, + /// DW_RLE_base_addressx + BaseAddressx { + /// base address + addr: DebugAddrIndex, + }, + /// DW_RLE_startx_endx + StartxEndx { + /// start of range + begin: DebugAddrIndex, + /// end of range + end: DebugAddrIndex, + }, + /// DW_RLE_startx_length + StartxLength { + /// start of range + begin: DebugAddrIndex, + /// length of range + length: u64, + }, + /// DW_RLE_offset_pair + OffsetPair { + /// start of range + begin: u64, + /// end of range + end: u64, + }, + /// DW_RLE_start_end + StartEnd { + /// start of range + begin: u64, + /// end of range + end: u64, + }, + /// DW_RLE_start_length + StartLength { + /// start of range + begin: u64, + /// length of range + length: u64, + }, +} + +impl RawRngListEntry { + /// Parse a range entry from `.debug_rnglists` + fn parse>( + input: &mut R, + encoding: Encoding, + format: RangeListsFormat, + ) -> Result> { + match format { + RangeListsFormat::Bare => { + let range = RawRange::parse(input, encoding.address_size)?; + return Ok(if range.is_end() { + None + } else if range.is_base_address(encoding.address_size) { + Some(RawRngListEntry::BaseAddress { addr: range.end }) + } else { + Some(RawRngListEntry::AddressOrOffsetPair { + begin: range.begin, + end: range.end, + }) + }); + } + RangeListsFormat::RLE => Ok(match constants::DwRle(input.read_u8()?) { + constants::DW_RLE_end_of_list => None, + constants::DW_RLE_base_addressx => Some(RawRngListEntry::BaseAddressx { + addr: DebugAddrIndex(input.read_uleb128().and_then(R::Offset::from_u64)?), + }), + constants::DW_RLE_startx_endx => Some(RawRngListEntry::StartxEndx { + begin: DebugAddrIndex(input.read_uleb128().and_then(R::Offset::from_u64)?), + end: DebugAddrIndex(input.read_uleb128().and_then(R::Offset::from_u64)?), + }), + constants::DW_RLE_startx_length => Some(RawRngListEntry::StartxLength { + begin: DebugAddrIndex(input.read_uleb128().and_then(R::Offset::from_u64)?), + length: input.read_uleb128()?, + }), + constants::DW_RLE_offset_pair => Some(RawRngListEntry::OffsetPair { + begin: input.read_uleb128()?, + end: input.read_uleb128()?, + }), + constants::DW_RLE_base_address => Some(RawRngListEntry::BaseAddress { + addr: input.read_address(encoding.address_size)?, + }), + constants::DW_RLE_start_end => Some(RawRngListEntry::StartEnd { + begin: input.read_address(encoding.address_size)?, + end: input.read_address(encoding.address_size)?, + }), + constants::DW_RLE_start_length => Some(RawRngListEntry::StartLength { + begin: input.read_address(encoding.address_size)?, + length: input.read_uleb128()?, + }), + _ => { + return Err(Error::InvalidAddressRange); + } + }), + } + } +} + +impl RawRngListIter { + /// Construct a `RawRngListIter`. + fn new(input: R, encoding: Encoding, format: RangeListsFormat) -> RawRngListIter { + RawRngListIter { + input, + encoding, + format, + } + } + + /// Advance the iterator to the next range. + pub fn next(&mut self) -> Result>> { + if self.input.is_empty() { + return Ok(None); + } + + match RawRngListEntry::parse(&mut self.input, self.encoding, self.format) { + Ok(range) => { + if range.is_none() { + self.input.empty(); + } + Ok(range) + } + Err(e) => { + self.input.empty(); + Err(e) + } + } + } +} + +#[cfg(feature = "fallible-iterator")] +impl fallible_iterator::FallibleIterator for RawRngListIter { + type Item = RawRngListEntry; + type Error = Error; + + fn next(&mut self) -> ::core::result::Result, Self::Error> { + RawRngListIter::next(self) + } +} + +/// An iterator over an address range list. +/// +/// This iterator internally handles processing of base addresses and different +/// entry types. Thus, it only returns range entries that are valid +/// and already adjusted for the base address. +#[derive(Debug)] +pub struct RngListIter { + raw: RawRngListIter, + base_address: u64, + debug_addr: DebugAddr, + debug_addr_base: DebugAddrBase, +} + +impl RngListIter { + /// Construct a `RngListIter`. + fn new( + raw: RawRngListIter, + base_address: u64, + debug_addr: DebugAddr, + debug_addr_base: DebugAddrBase, + ) -> RngListIter { + RngListIter { + raw, + base_address, + debug_addr, + debug_addr_base, + } + } + + #[inline] + fn get_address(&self, index: DebugAddrIndex) -> Result { + self.debug_addr + .get_address(self.raw.encoding.address_size, self.debug_addr_base, index) + } + + /// Advance the iterator to the next range. + pub fn next(&mut self) -> Result> { + loop { + let raw_range = match self.raw.next()? { + Some(range) => range, + None => return Ok(None), + }; + + let range = match raw_range { + RawRngListEntry::BaseAddress { addr } => { + self.base_address = addr; + continue; + } + RawRngListEntry::BaseAddressx { addr } => { + self.base_address = self.get_address(addr)?; + continue; + } + RawRngListEntry::StartxEndx { begin, end } => { + let begin = self.get_address(begin)?; + let end = self.get_address(end)?; + Range { begin, end } + } + RawRngListEntry::StartxLength { begin, length } => { + let begin = self.get_address(begin)?; + let end = begin + length; + Range { begin, end } + } + RawRngListEntry::AddressOrOffsetPair { begin, end } + | RawRngListEntry::OffsetPair { begin, end } => { + let mut range = Range { begin, end }; + range.add_base_address(self.base_address, self.raw.encoding.address_size); + range + } + RawRngListEntry::StartEnd { begin, end } => Range { begin, end }, + RawRngListEntry::StartLength { begin, length } => Range { + begin, + end: begin + length, + }, + }; + + if range.begin > range.end { + self.raw.input.empty(); + return Err(Error::InvalidAddressRange); + } + + return Ok(Some(range)); + } + } +} + +#[cfg(feature = "fallible-iterator")] +impl fallible_iterator::FallibleIterator for RngListIter { + type Item = Range; + type Error = Error; + + fn next(&mut self) -> ::core::result::Result, Self::Error> { + RngListIter::next(self) + } +} + +/// A raw address range from the `.debug_ranges` section. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct RawRange { + /// The beginning address of the range. + pub begin: u64, + + /// The first address past the end of the range. + pub end: u64, +} + +impl RawRange { + /// Check if this is a range end entry. + /// + /// This will only occur for raw ranges. + #[inline] + pub fn is_end(&self) -> bool { + self.begin == 0 && self.end == 0 + } + + /// Check if this is a base address selection entry. + /// + /// A base address selection entry changes the base address that subsequent + /// range entries are relative to. This will only occur for raw ranges. + #[inline] + pub fn is_base_address(&self, address_size: u8) -> bool { + self.begin == !0 >> (64 - address_size * 8) + } + + /// Parse an address range entry from `.debug_ranges` or `.debug_loc`. + #[doc(hidden)] + #[inline] + pub fn parse(input: &mut R, address_size: u8) -> Result { + let begin = input.read_address(address_size)?; + let end = input.read_address(address_size)?; + let range = RawRange { begin, end }; + Ok(range) + } +} + +/// An address range from the `.debug_ranges`, `.debug_rnglists`, or `.debug_aranges` sections. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Range { + /// The beginning address of the range. + pub begin: u64, + + /// The first address past the end of the range. + pub end: u64, +} + +impl Range { + /// Add a base address to this range. + #[inline] + pub(crate) fn add_base_address(&mut self, base_address: u64, address_size: u8) { + let mask = !0 >> (64 - address_size * 8); + self.begin = base_address.wrapping_add(self.begin) & mask; + self.end = base_address.wrapping_add(self.end) & mask; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::common::Format; + use crate::endianity::LittleEndian; + use crate::test_util::GimliSectionMethods; + use test_assembler::{Endian, Label, LabelMaker, Section}; + + #[test] + fn test_rnglists_32() { + let encoding = Encoding { + format: Format::Dwarf32, + version: 5, + address_size: 4, + }; + let section = Section::with_endian(Endian::Little) + .L32(0x0300_0000) + .L32(0x0301_0300) + .L32(0x0301_0400) + .L32(0x0301_0500); + let buf = section.get_contents().unwrap(); + let debug_addr = &DebugAddr::from(EndianSlice::new(&buf, LittleEndian)); + let debug_addr_base = DebugAddrBase(0); + + let start = Label::new(); + let first = Label::new(); + let size = Label::new(); + #[rustfmt::skip] + let section = Section::with_endian(Endian::Little) + // Header + .mark(&start) + .L32(&size) + .L16(encoding.version) + .L8(encoding.address_size) + .L8(0) + .L32(0) + .mark(&first) + // OffsetPair + .L8(4).uleb(0x10200).uleb(0x10300) + // A base address selection followed by an OffsetPair. + .L8(5).L32(0x0200_0000) + .L8(4).uleb(0x10400).uleb(0x10500) + // An empty OffsetPair followed by a normal OffsetPair. + .L8(4).uleb(0x10600).uleb(0x10600) + .L8(4).uleb(0x10800).uleb(0x10900) + // A StartEnd + .L8(6).L32(0x201_0a00).L32(0x201_0b00) + // A StartLength + .L8(7).L32(0x201_0c00).uleb(0x100) + // An OffsetPair that starts at 0. + .L8(4).uleb(0).uleb(1) + // An OffsetPair that starts and ends at 0. + .L8(4).uleb(0).uleb(0) + // An OffsetPair that ends at -1. + .L8(5).L32(0) + .L8(4).uleb(0).uleb(0xffff_ffff) + // A BaseAddressx + OffsetPair + .L8(1).uleb(0) + .L8(4).uleb(0x10100).uleb(0x10200) + // A StartxEndx + .L8(2).uleb(1).uleb(2) + // A StartxLength + .L8(3).uleb(3).uleb(0x100) + // A range end. + .L8(0) + // Some extra data. + .L32(0xffff_ffff); + size.set_const((§ion.here() - &start - 4) as u64); + + let buf = section.get_contents().unwrap(); + let debug_ranges = DebugRanges::new(&[], LittleEndian); + let debug_rnglists = DebugRngLists::new(&buf, LittleEndian); + let rnglists = RangeLists::new(debug_ranges, debug_rnglists); + let offset = RangeListsOffset((&first - &start) as usize); + let mut ranges = rnglists + .ranges(offset, encoding, 0x0100_0000, debug_addr, debug_addr_base) + .unwrap(); + + // A normal range. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0101_0200, + end: 0x0101_0300, + })) + ); + + // A base address selection followed by a normal range. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0201_0400, + end: 0x0201_0500, + })) + ); + + // An empty range followed by a normal range. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0201_0600, + end: 0x0201_0600, + })) + ); + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0201_0800, + end: 0x0201_0900, + })) + ); + + // A normal range. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0201_0a00, + end: 0x0201_0b00, + })) + ); + + // A normal range. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0201_0c00, + end: 0x0201_0d00, + })) + ); + + // A range that starts at 0. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0200_0000, + end: 0x0200_0001, + })) + ); + + // A range that starts and ends at 0. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0200_0000, + end: 0x0200_0000, + })) + ); + + // A range that ends at -1. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0000_0000, + end: 0xffff_ffff, + })) + ); + + // A BaseAddressx + OffsetPair + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0301_0100, + end: 0x0301_0200, + })) + ); + + // A StartxEndx + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0301_0300, + end: 0x0301_0400, + })) + ); + + // A StartxLength + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0301_0500, + end: 0x0301_0600, + })) + ); + + // A range end. + assert_eq!(ranges.next(), Ok(None)); + + // An offset at the end of buf. + let mut ranges = rnglists + .ranges( + RangeListsOffset(buf.len()), + encoding, + 0x0100_0000, + debug_addr, + debug_addr_base, + ) + .unwrap(); + assert_eq!(ranges.next(), Ok(None)); + } + + #[test] + fn test_rnglists_64() { + let encoding = Encoding { + format: Format::Dwarf64, + version: 5, + address_size: 8, + }; + let section = Section::with_endian(Endian::Little) + .L64(0x0300_0000) + .L64(0x0301_0300) + .L64(0x0301_0400) + .L64(0x0301_0500); + let buf = section.get_contents().unwrap(); + let debug_addr = &DebugAddr::from(EndianSlice::new(&buf, LittleEndian)); + let debug_addr_base = DebugAddrBase(0); + + let start = Label::new(); + let first = Label::new(); + let size = Label::new(); + #[rustfmt::skip] + let section = Section::with_endian(Endian::Little) + // Header + .mark(&start) + .L32(0xffff_ffff) + .L64(&size) + .L16(encoding.version) + .L8(encoding.address_size) + .L8(0) + .L32(0) + .mark(&first) + // OffsetPair + .L8(4).uleb(0x10200).uleb(0x10300) + // A base address selection followed by an OffsetPair. + .L8(5).L64(0x0200_0000) + .L8(4).uleb(0x10400).uleb(0x10500) + // An empty OffsetPair followed by a normal OffsetPair. + .L8(4).uleb(0x10600).uleb(0x10600) + .L8(4).uleb(0x10800).uleb(0x10900) + // A StartEnd + .L8(6).L64(0x201_0a00).L64(0x201_0b00) + // A StartLength + .L8(7).L64(0x201_0c00).uleb(0x100) + // An OffsetPair that starts at 0. + .L8(4).uleb(0).uleb(1) + // An OffsetPair that starts and ends at 0. + .L8(4).uleb(0).uleb(0) + // An OffsetPair that ends at -1. + .L8(5).L64(0) + .L8(4).uleb(0).uleb(0xffff_ffff) + // A BaseAddressx + OffsetPair + .L8(1).uleb(0) + .L8(4).uleb(0x10100).uleb(0x10200) + // A StartxEndx + .L8(2).uleb(1).uleb(2) + // A StartxLength + .L8(3).uleb(3).uleb(0x100) + // A range end. + .L8(0) + // Some extra data. + .L32(0xffff_ffff); + size.set_const((§ion.here() - &start - 12) as u64); + + let buf = section.get_contents().unwrap(); + let debug_ranges = DebugRanges::new(&[], LittleEndian); + let debug_rnglists = DebugRngLists::new(&buf, LittleEndian); + let rnglists = RangeLists::new(debug_ranges, debug_rnglists); + let offset = RangeListsOffset((&first - &start) as usize); + let mut ranges = rnglists + .ranges(offset, encoding, 0x0100_0000, debug_addr, debug_addr_base) + .unwrap(); + + // A normal range. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0101_0200, + end: 0x0101_0300, + })) + ); + + // A base address selection followed by a normal range. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0201_0400, + end: 0x0201_0500, + })) + ); + + // An empty range followed by a normal range. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0201_0600, + end: 0x0201_0600, + })) + ); + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0201_0800, + end: 0x0201_0900, + })) + ); + + // A normal range. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0201_0a00, + end: 0x0201_0b00, + })) + ); + + // A normal range. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0201_0c00, + end: 0x0201_0d00, + })) + ); + + // A range that starts at 0. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0200_0000, + end: 0x0200_0001, + })) + ); + + // A range that starts and ends at 0. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0200_0000, + end: 0x0200_0000, + })) + ); + + // A range that ends at -1. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0000_0000, + end: 0xffff_ffff, + })) + ); + + // A BaseAddressx + OffsetPair + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0301_0100, + end: 0x0301_0200, + })) + ); + + // A StartxEndx + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0301_0300, + end: 0x0301_0400, + })) + ); + + // A StartxLength + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0301_0500, + end: 0x0301_0600, + })) + ); + + // A range end. + assert_eq!(ranges.next(), Ok(None)); + + // An offset at the end of buf. + let mut ranges = rnglists + .ranges( + RangeListsOffset(buf.len()), + encoding, + 0x0100_0000, + debug_addr, + debug_addr_base, + ) + .unwrap(); + assert_eq!(ranges.next(), Ok(None)); + } + + #[test] + fn test_raw_range() { + let range = RawRange { + begin: 0, + end: 0xffff_ffff, + }; + assert!(!range.is_end()); + assert!(!range.is_base_address(4)); + assert!(!range.is_base_address(8)); + + let range = RawRange { begin: 0, end: 0 }; + assert!(range.is_end()); + assert!(!range.is_base_address(4)); + assert!(!range.is_base_address(8)); + + let range = RawRange { + begin: 0xffff_ffff, + end: 0, + }; + assert!(!range.is_end()); + assert!(range.is_base_address(4)); + assert!(!range.is_base_address(8)); + + let range = RawRange { + begin: 0xffff_ffff_ffff_ffff, + end: 0, + }; + assert!(!range.is_end()); + assert!(!range.is_base_address(4)); + assert!(range.is_base_address(8)); + } + + #[test] + fn test_ranges_32() { + let start = Label::new(); + let first = Label::new(); + #[rustfmt::skip] + let section = Section::with_endian(Endian::Little) + // A range before the offset. + .mark(&start) + .L32(0x10000).L32(0x10100) + .mark(&first) + // A normal range. + .L32(0x10200).L32(0x10300) + // A base address selection followed by a normal range. + .L32(0xffff_ffff).L32(0x0200_0000) + .L32(0x10400).L32(0x10500) + // An empty range followed by a normal range. + .L32(0x10600).L32(0x10600) + .L32(0x10800).L32(0x10900) + // A range that starts at 0. + .L32(0).L32(1) + // A range that ends at -1. + .L32(0xffff_ffff).L32(0x0000_0000) + .L32(0).L32(0xffff_ffff) + // A range end. + .L32(0).L32(0) + // Some extra data. + .L32(0); + + let buf = section.get_contents().unwrap(); + let debug_ranges = DebugRanges::new(&buf, LittleEndian); + let debug_rnglists = DebugRngLists::new(&[], LittleEndian); + let rnglists = RangeLists::new(debug_ranges, debug_rnglists); + let offset = RangeListsOffset((&first - &start) as usize); + let debug_addr = &DebugAddr::from(EndianSlice::new(&[], LittleEndian)); + let debug_addr_base = DebugAddrBase(0); + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 4, + }; + let mut ranges = rnglists + .ranges(offset, encoding, 0x0100_0000, debug_addr, debug_addr_base) + .unwrap(); + + // A normal range. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0101_0200, + end: 0x0101_0300, + })) + ); + + // A base address selection followed by a normal range. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0201_0400, + end: 0x0201_0500, + })) + ); + + // An empty range followed by a normal range. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0201_0600, + end: 0x0201_0600, + })) + ); + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0201_0800, + end: 0x0201_0900, + })) + ); + + // A range that starts at 0. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0200_0000, + end: 0x0200_0001, + })) + ); + + // A range that ends at -1. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0000_0000, + end: 0xffff_ffff, + })) + ); + + // A range end. + assert_eq!(ranges.next(), Ok(None)); + + // An offset at the end of buf. + let mut ranges = rnglists + .ranges( + RangeListsOffset(buf.len()), + encoding, + 0x0100_0000, + debug_addr, + debug_addr_base, + ) + .unwrap(); + assert_eq!(ranges.next(), Ok(None)); + } + + #[test] + fn test_ranges_64() { + let start = Label::new(); + let first = Label::new(); + #[rustfmt::skip] + let section = Section::with_endian(Endian::Little) + // A range before the offset. + .mark(&start) + .L64(0x10000).L64(0x10100) + .mark(&first) + // A normal range. + .L64(0x10200).L64(0x10300) + // A base address selection followed by a normal range. + .L64(0xffff_ffff_ffff_ffff).L64(0x0200_0000) + .L64(0x10400).L64(0x10500) + // An empty range followed by a normal range. + .L64(0x10600).L64(0x10600) + .L64(0x10800).L64(0x10900) + // A range that starts at 0. + .L64(0).L64(1) + // A range that ends at -1. + .L64(0xffff_ffff_ffff_ffff).L64(0x0000_0000) + .L64(0).L64(0xffff_ffff_ffff_ffff) + // A range end. + .L64(0).L64(0) + // Some extra data. + .L64(0); + + let buf = section.get_contents().unwrap(); + let debug_ranges = DebugRanges::new(&buf, LittleEndian); + let debug_rnglists = DebugRngLists::new(&[], LittleEndian); + let rnglists = RangeLists::new(debug_ranges, debug_rnglists); + let offset = RangeListsOffset((&first - &start) as usize); + let debug_addr = &DebugAddr::from(EndianSlice::new(&[], LittleEndian)); + let debug_addr_base = DebugAddrBase(0); + let encoding = Encoding { + format: Format::Dwarf64, + version: 4, + address_size: 8, + }; + let mut ranges = rnglists + .ranges(offset, encoding, 0x0100_0000, debug_addr, debug_addr_base) + .unwrap(); + + // A normal range. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0101_0200, + end: 0x0101_0300, + })) + ); + + // A base address selection followed by a normal range. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0201_0400, + end: 0x0201_0500, + })) + ); + + // An empty range followed by a normal range. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0201_0600, + end: 0x0201_0600, + })) + ); + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0201_0800, + end: 0x0201_0900, + })) + ); + + // A range that starts at 0. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0200_0000, + end: 0x0200_0001, + })) + ); + + // A range that ends at -1. + assert_eq!( + ranges.next(), + Ok(Some(Range { + begin: 0x0, + end: 0xffff_ffff_ffff_ffff, + })) + ); + + // A range end. + assert_eq!(ranges.next(), Ok(None)); + + // An offset at the end of buf. + let mut ranges = rnglists + .ranges( + RangeListsOffset(buf.len()), + encoding, + 0x0100_0000, + debug_addr, + debug_addr_base, + ) + .unwrap(); + assert_eq!(ranges.next(), Ok(None)); + } + + #[test] + fn test_ranges_invalid() { + #[rustfmt::skip] + let section = Section::with_endian(Endian::Little) + // An invalid range. + .L32(0x20000).L32(0x10000) + // An invalid range after wrapping. + .L32(0x20000).L32(0xff01_0000); + + let buf = section.get_contents().unwrap(); + let debug_ranges = DebugRanges::new(&buf, LittleEndian); + let debug_rnglists = DebugRngLists::new(&[], LittleEndian); + let rnglists = RangeLists::new(debug_ranges, debug_rnglists); + let debug_addr = &DebugAddr::from(EndianSlice::new(&[], LittleEndian)); + let debug_addr_base = DebugAddrBase(0); + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 4, + }; + + // An invalid range. + let mut ranges = rnglists + .ranges( + RangeListsOffset(0x0), + encoding, + 0x0100_0000, + debug_addr, + debug_addr_base, + ) + .unwrap(); + assert_eq!(ranges.next(), Err(Error::InvalidAddressRange)); + + // An invalid range after wrapping. + let mut ranges = rnglists + .ranges( + RangeListsOffset(0x8), + encoding, + 0x0100_0000, + debug_addr, + debug_addr_base, + ) + .unwrap(); + assert_eq!(ranges.next(), Err(Error::InvalidAddressRange)); + + // An invalid offset. + match rnglists.ranges( + RangeListsOffset(buf.len() + 1), + encoding, + 0x0100_0000, + debug_addr, + debug_addr_base, + ) { + Err(Error::UnexpectedEof(_)) => {} + otherwise => panic!("Unexpected result: {:?}", otherwise), + } + } + + #[test] + fn test_get_offset() { + for format in vec![Format::Dwarf32, Format::Dwarf64] { + let encoding = Encoding { + format, + version: 5, + address_size: 4, + }; + + let zero = Label::new(); + let length = Label::new(); + let start = Label::new(); + let first = Label::new(); + let end = Label::new(); + let mut section = Section::with_endian(Endian::Little) + .mark(&zero) + .initial_length(format, &length, &start) + .D16(encoding.version) + .D8(encoding.address_size) + .D8(0) + .D32(20) + .mark(&first); + for i in 0..20 { + section = section.word(format.word_size(), 1000 + i); + } + section = section.mark(&end); + length.set_const((&end - &start) as u64); + let section = section.get_contents().unwrap(); + + let debug_ranges = DebugRanges::from(EndianSlice::new(&[], LittleEndian)); + let debug_rnglists = DebugRngLists::from(EndianSlice::new(§ion, LittleEndian)); + let ranges = RangeLists::new(debug_ranges, debug_rnglists); + + let base = DebugRngListsBase((&first - &zero) as usize); + assert_eq!( + ranges.get_offset(encoding, base, DebugRngListsIndex(0)), + Ok(RangeListsOffset(base.0 + 1000)) + ); + assert_eq!( + ranges.get_offset(encoding, base, DebugRngListsIndex(19)), + Ok(RangeListsOffset(base.0 + 1019)) + ); + } + } +} diff --git a/crux-mir/lib/gimli/src/read/str.rs b/crux-mir/lib/gimli/src/read/str.rs new file mode 100644 index 000000000..c6b87d8f9 --- /dev/null +++ b/crux-mir/lib/gimli/src/read/str.rs @@ -0,0 +1,321 @@ +use crate::common::{ + DebugLineStrOffset, DebugStrOffset, DebugStrOffsetsBase, DebugStrOffsetsIndex, DwarfFileType, + Encoding, SectionId, +}; +use crate::endianity::Endianity; +use crate::read::{EndianSlice, Reader, ReaderOffset, Result, Section}; +use crate::Format; + +/// The `DebugStr` struct represents the DWARF strings +/// found in the `.debug_str` section. +#[derive(Debug, Default, Clone, Copy)] +pub struct DebugStr { + debug_str_section: R, +} + +impl<'input, Endian> DebugStr> +where + Endian: Endianity, +{ + /// Construct a new `DebugStr` instance from the data in the `.debug_str` + /// section. + /// + /// It is the caller's responsibility to read the `.debug_str` section and + /// present it as a `&[u8]` slice. That means using some ELF loader on + /// Linux, a Mach-O loader on macOS, etc. + /// + /// ``` + /// use gimli::{DebugStr, LittleEndian}; + /// + /// # let buf = [0x00, 0x01, 0x02, 0x03]; + /// # let read_debug_str_section_somehow = || &buf; + /// let debug_str = DebugStr::new(read_debug_str_section_somehow(), LittleEndian); + /// ``` + pub fn new(debug_str_section: &'input [u8], endian: Endian) -> Self { + Self::from(EndianSlice::new(debug_str_section, endian)) + } +} + +impl DebugStr { + /// Lookup a string from the `.debug_str` section by DebugStrOffset. + /// + /// ``` + /// use gimli::{DebugStr, DebugStrOffset, LittleEndian}; + /// + /// # let buf = [0x01, 0x02, 0x00]; + /// # let offset = DebugStrOffset(0); + /// # let read_debug_str_section_somehow = || &buf; + /// # let debug_str_offset_somehow = || offset; + /// let debug_str = DebugStr::new(read_debug_str_section_somehow(), LittleEndian); + /// println!("Found string {:?}", debug_str.get_str(debug_str_offset_somehow())); + /// ``` + pub fn get_str(&self, offset: DebugStrOffset) -> Result { + let input = &mut self.debug_str_section.clone(); + input.skip(offset.0)?; + input.read_null_terminated_slice() + } +} + +impl DebugStr { + /// Create a `DebugStr` section that references the data in `self`. + /// + /// This is useful when `R` implements `Reader` but `T` does not. + /// + /// ## Example Usage + /// + /// ```rust,no_run + /// # let load_section = || unimplemented!(); + /// // Read the DWARF section into a `Vec` with whatever object loader you're using. + /// let owned_section: gimli::DebugStr> = load_section(); + /// // Create a reference to the DWARF section. + /// let section = owned_section.borrow(|section| { + /// gimli::EndianSlice::new(§ion, gimli::LittleEndian) + /// }); + /// ``` + pub fn borrow<'a, F, R>(&'a self, mut borrow: F) -> DebugStr + where + F: FnMut(&'a T) -> R, + { + borrow(&self.debug_str_section).into() + } +} + +impl Section for DebugStr { + fn id() -> SectionId { + SectionId::DebugStr + } + + fn reader(&self) -> &R { + &self.debug_str_section + } +} + +impl From for DebugStr { + fn from(debug_str_section: R) -> Self { + DebugStr { debug_str_section } + } +} + +/// The raw contents of the `.debug_str_offsets` section. +#[derive(Debug, Default, Clone, Copy)] +pub struct DebugStrOffsets { + section: R, +} + +impl DebugStrOffsets { + // TODO: add an iterator over the sets of entries in the section. + // This is not needed for common usage of the section though. + + /// Returns the `.debug_str` offset at the given `base` and `index`. + /// + /// A set of entries in the `.debug_str_offsets` section consists of a header + /// followed by a series of string table offsets. + /// + /// The `base` must be the `DW_AT_str_offsets_base` value from the compilation unit DIE. + /// This is an offset that points to the first entry following the header. + /// + /// The `index` is the value of a `DW_FORM_strx` attribute. + /// + /// The `format` must be the DWARF format of the compilation unit. This format must + /// match the header. However, note that we do not parse the header to validate this, + /// since locating the header is unreliable, and the GNU extensions do not emit it. + pub fn get_str_offset( + &self, + format: Format, + base: DebugStrOffsetsBase, + index: DebugStrOffsetsIndex, + ) -> Result> { + let input = &mut self.section.clone(); + input.skip(base.0)?; + input.skip(R::Offset::from_u64( + index.0.into_u64() * u64::from(format.word_size()), + )?)?; + input.read_offset(format).map(DebugStrOffset) + } +} + +impl DebugStrOffsets { + /// Create a `DebugStrOffsets` section that references the data in `self`. + /// + /// This is useful when `R` implements `Reader` but `T` does not. + /// + /// ## Example Usage + /// + /// ```rust,no_run + /// # let load_section = || unimplemented!(); + /// // Read the DWARF section into a `Vec` with whatever object loader you're using. + /// let owned_section: gimli::DebugStrOffsets> = load_section(); + /// // Create a reference to the DWARF section. + /// let section = owned_section.borrow(|section| { + /// gimli::EndianSlice::new(§ion, gimli::LittleEndian) + /// }); + /// ``` + pub fn borrow<'a, F, R>(&'a self, mut borrow: F) -> DebugStrOffsets + where + F: FnMut(&'a T) -> R, + { + borrow(&self.section).into() + } +} + +impl Section for DebugStrOffsets { + fn id() -> SectionId { + SectionId::DebugStrOffsets + } + + fn reader(&self) -> &R { + &self.section + } +} + +impl From for DebugStrOffsets { + fn from(section: R) -> Self { + DebugStrOffsets { section } + } +} + +impl DebugStrOffsetsBase +where + Offset: ReaderOffset, +{ + /// Returns a `DebugStrOffsetsBase` with the default value of DW_AT_str_offsets_base + /// for the given `Encoding` and `DwarfFileType`. + pub fn default_for_encoding_and_file( + encoding: Encoding, + file_type: DwarfFileType, + ) -> DebugStrOffsetsBase { + if encoding.version >= 5 && file_type == DwarfFileType::Dwo { + // In .dwo files, the compiler omits the DW_AT_str_offsets_base attribute (because there is + // only a single unit in the file) but we must skip past the header, which the attribute + // would normally do for us. + // initial_length_size + version + 2 bytes of padding. + DebugStrOffsetsBase(Offset::from_u8( + encoding.format.initial_length_size() + 2 + 2, + )) + } else { + DebugStrOffsetsBase(Offset::from_u8(0)) + } + } +} + +/// The `DebugLineStr` struct represents the DWARF strings +/// found in the `.debug_line_str` section. +#[derive(Debug, Default, Clone, Copy)] +pub struct DebugLineStr { + section: R, +} + +impl<'input, Endian> DebugLineStr> +where + Endian: Endianity, +{ + /// Construct a new `DebugLineStr` instance from the data in the `.debug_line_str` + /// section. + /// + /// It is the caller's responsibility to read the `.debug_line_str` section and + /// present it as a `&[u8]` slice. That means using some ELF loader on + /// Linux, a Mach-O loader on macOS, etc. + /// + /// ``` + /// use gimli::{DebugLineStr, LittleEndian}; + /// + /// # let buf = [0x00, 0x01, 0x02, 0x03]; + /// # let read_debug_line_str_section_somehow = || &buf; + /// let debug_str = DebugLineStr::new(read_debug_line_str_section_somehow(), LittleEndian); + /// ``` + pub fn new(debug_line_str_section: &'input [u8], endian: Endian) -> Self { + Self::from(EndianSlice::new(debug_line_str_section, endian)) + } +} + +impl DebugLineStr { + /// Lookup a string from the `.debug_line_str` section by DebugLineStrOffset. + pub fn get_str(&self, offset: DebugLineStrOffset) -> Result { + let input = &mut self.section.clone(); + input.skip(offset.0)?; + input.read_null_terminated_slice() + } +} + +impl DebugLineStr { + /// Create a `DebugLineStr` section that references the data in `self`. + /// + /// This is useful when `R` implements `Reader` but `T` does not. + /// + /// ## Example Usage + /// + /// ```rust,no_run + /// # let load_section = || unimplemented!(); + /// // Read the DWARF section into a `Vec` with whatever object loader you're using. + /// let owned_section: gimli::DebugLineStr> = load_section(); + /// // Create a reference to the DWARF section. + /// let section = owned_section.borrow(|section| { + /// gimli::EndianSlice::new(§ion, gimli::LittleEndian) + /// }); + /// ``` + pub fn borrow<'a, F, R>(&'a self, mut borrow: F) -> DebugLineStr + where + F: FnMut(&'a T) -> R, + { + borrow(&self.section).into() + } +} + +impl Section for DebugLineStr { + fn id() -> SectionId { + SectionId::DebugLineStr + } + + fn reader(&self) -> &R { + &self.section + } +} + +impl From for DebugLineStr { + fn from(section: R) -> Self { + DebugLineStr { section } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_util::GimliSectionMethods; + use crate::LittleEndian; + use test_assembler::{Endian, Label, LabelMaker, Section}; + + #[test] + fn test_get_str_offset() { + for format in vec![Format::Dwarf32, Format::Dwarf64] { + let zero = Label::new(); + let length = Label::new(); + let start = Label::new(); + let first = Label::new(); + let end = Label::new(); + let mut section = Section::with_endian(Endian::Little) + .mark(&zero) + .initial_length(format, &length, &start) + .D16(5) + .D16(0) + .mark(&first); + for i in 0..20 { + section = section.word(format.word_size(), 1000 + i); + } + section = section.mark(&end); + length.set_const((&end - &start) as u64); + + let section = section.get_contents().unwrap(); + let debug_str_offsets = DebugStrOffsets::from(EndianSlice::new(§ion, LittleEndian)); + let base = DebugStrOffsetsBase((&first - &zero) as usize); + + assert_eq!( + debug_str_offsets.get_str_offset(format, base, DebugStrOffsetsIndex(0)), + Ok(DebugStrOffset(1000)) + ); + assert_eq!( + debug_str_offsets.get_str_offset(format, base, DebugStrOffsetsIndex(19)), + Ok(DebugStrOffset(1019)) + ); + } + } +} diff --git a/crux-mir/lib/gimli/src/read/unit.rs b/crux-mir/lib/gimli/src/read/unit.rs new file mode 100644 index 000000000..670e55efd --- /dev/null +++ b/crux-mir/lib/gimli/src/read/unit.rs @@ -0,0 +1,6146 @@ +//! Functions for parsing DWARF `.debug_info` and `.debug_types` sections. + +use core::cell::Cell; +use core::ops::{Range, RangeFrom, RangeTo}; +use core::{u16, u8}; + +use crate::common::{ + DebugAbbrevOffset, DebugAddrBase, DebugAddrIndex, DebugInfoOffset, DebugLineOffset, + DebugLineStrOffset, DebugLocListsBase, DebugLocListsIndex, DebugMacinfoOffset, + DebugMacroOffset, DebugRngListsBase, DebugRngListsIndex, DebugStrOffset, DebugStrOffsetsBase, + DebugStrOffsetsIndex, DebugTypeSignature, DebugTypesOffset, DwoId, Encoding, Format, + LocationListsOffset, RawRangeListsOffset, SectionId, UnitSectionOffset, +}; +use crate::constants; +use crate::endianity::Endianity; +use crate::read::abbrev::get_attribute_size; +use crate::read::{ + Abbreviation, Abbreviations, AttributeSpecification, DebugAbbrev, DebugStr, EndianSlice, Error, + Expression, Reader, ReaderOffset, Result, Section, UnitOffset, +}; + +impl DebugTypesOffset { + /// Convert an offset to be relative to the start of the given unit, + /// instead of relative to the start of the .debug_types section. + /// Returns `None` if the offset is not within the unit entries. + pub fn to_unit_offset(&self, unit: &UnitHeader) -> Option> + where + R: Reader, + { + let unit_offset = unit.offset().as_debug_types_offset()?; + let offset = UnitOffset(self.0.checked_sub(unit_offset.0)?); + if !unit.is_valid_offset(offset) { + return None; + } + Some(offset) + } +} + +impl DebugInfoOffset { + /// Convert an offset to be relative to the start of the given unit, + /// instead of relative to the start of the .debug_info section. + /// Returns `None` if the offset is not within this unit entries. + pub fn to_unit_offset(&self, unit: &UnitHeader) -> Option> + where + R: Reader, + { + let unit_offset = unit.offset().as_debug_info_offset()?; + let offset = UnitOffset(self.0.checked_sub(unit_offset.0)?); + if !unit.is_valid_offset(offset) { + return None; + } + Some(offset) + } +} + +impl UnitOffset { + /// Convert an offset to be relative to the start of the .debug_info section, + /// instead of relative to the start of the given unit. Returns None if the + /// provided unit lives in the .debug_types section. + pub fn to_debug_info_offset(&self, unit: &UnitHeader) -> Option> + where + R: Reader, + { + let unit_offset = unit.offset().as_debug_info_offset()?; + Some(DebugInfoOffset(unit_offset.0 + self.0)) + } + + /// Convert an offset to be relative to the start of the .debug_types section, + /// instead of relative to the start of the given unit. Returns None if the + /// provided unit lives in the .debug_info section. + pub fn to_debug_types_offset(&self, unit: &UnitHeader) -> Option> + where + R: Reader, + { + let unit_offset = unit.offset().as_debug_types_offset()?; + Some(DebugTypesOffset(unit_offset.0 + self.0)) + } +} + +/// The `DebugInfo` struct represents the DWARF debugging information found in +/// the `.debug_info` section. +#[derive(Debug, Default, Clone, Copy)] +pub struct DebugInfo { + debug_info_section: R, +} + +impl<'input, Endian> DebugInfo> +where + Endian: Endianity, +{ + /// Construct a new `DebugInfo` instance from the data in the `.debug_info` + /// section. + /// + /// It is the caller's responsibility to read the `.debug_info` section and + /// present it as a `&[u8]` slice. That means using some ELF loader on + /// Linux, a Mach-O loader on macOS, etc. + /// + /// ``` + /// use gimli::{DebugInfo, LittleEndian}; + /// + /// # let buf = [0x00, 0x01, 0x02, 0x03]; + /// # let read_debug_info_section_somehow = || &buf; + /// let debug_info = DebugInfo::new(read_debug_info_section_somehow(), LittleEndian); + /// ``` + pub fn new(debug_info_section: &'input [u8], endian: Endian) -> Self { + Self::from(EndianSlice::new(debug_info_section, endian)) + } +} + +impl DebugInfo { + /// Iterate the units in this `.debug_info` section. + /// + /// ``` + /// use gimli::{DebugInfo, LittleEndian}; + /// + /// # let buf = []; + /// # let read_debug_info_section_somehow = || &buf; + /// let debug_info = DebugInfo::new(read_debug_info_section_somehow(), LittleEndian); + /// + /// let mut iter = debug_info.units(); + /// while let Some(unit) = iter.next().unwrap() { + /// println!("unit's length is {}", unit.unit_length()); + /// } + /// ``` + /// + /// Can be [used with + /// `FallibleIterator`](./index.html#using-with-fallibleiterator). + pub fn units(&self) -> DebugInfoUnitHeadersIter { + DebugInfoUnitHeadersIter { + input: self.debug_info_section.clone(), + offset: DebugInfoOffset(R::Offset::from_u8(0)), + } + } + + /// Get the UnitHeader located at offset from this .debug_info section. + /// + /// + pub fn header_from_offset(&self, offset: DebugInfoOffset) -> Result> { + let input = &mut self.debug_info_section.clone(); + input.skip(offset.0)?; + parse_unit_header(input, offset.into()) + } +} + +impl DebugInfo { + /// Create a `DebugInfo` section that references the data in `self`. + /// + /// This is useful when `R` implements `Reader` but `T` does not. + /// + /// ## Example Usage + /// + /// ```rust,no_run + /// # let load_section = || unimplemented!(); + /// // Read the DWARF section into a `Vec` with whatever object loader you're using. + /// let owned_section: gimli::DebugInfo> = load_section(); + /// // Create a reference to the DWARF section. + /// let section = owned_section.borrow(|section| { + /// gimli::EndianSlice::new(§ion, gimli::LittleEndian) + /// }); + /// ``` + pub fn borrow<'a, F, R>(&'a self, mut borrow: F) -> DebugInfo + where + F: FnMut(&'a T) -> R, + { + borrow(&self.debug_info_section).into() + } +} + +impl Section for DebugInfo { + fn id() -> SectionId { + SectionId::DebugInfo + } + + fn reader(&self) -> &R { + &self.debug_info_section + } +} + +impl From for DebugInfo { + fn from(debug_info_section: R) -> Self { + DebugInfo { debug_info_section } + } +} + +/// An iterator over the units of a .debug_info section. +/// +/// See the [documentation on +/// `DebugInfo::units`](./struct.DebugInfo.html#method.units) for more detail. +#[derive(Clone, Debug)] +pub struct DebugInfoUnitHeadersIter { + input: R, + offset: DebugInfoOffset, +} + +impl DebugInfoUnitHeadersIter { + /// Advance the iterator to the next unit header. + pub fn next(&mut self) -> Result>> { + if self.input.is_empty() { + Ok(None) + } else { + let len = self.input.len(); + match parse_unit_header(&mut self.input, self.offset.into()) { + Ok(header) => { + self.offset.0 += len - self.input.len(); + Ok(Some(header)) + } + Err(e) => { + self.input.empty(); + Err(e) + } + } + } + } +} + +#[cfg(feature = "fallible-iterator")] +impl fallible_iterator::FallibleIterator for DebugInfoUnitHeadersIter { + type Item = UnitHeader; + type Error = Error; + + fn next(&mut self) -> ::core::result::Result, Self::Error> { + DebugInfoUnitHeadersIter::next(self) + } +} + +/// Parse the unit type from the unit header. +fn parse_unit_type(input: &mut R) -> Result { + let val = input.read_u8()?; + Ok(constants::DwUt(val)) +} + +/// Parse the `debug_abbrev_offset` in the compilation unit header. +fn parse_debug_abbrev_offset( + input: &mut R, + format: Format, +) -> Result> { + input.read_offset(format).map(DebugAbbrevOffset) +} + +/// Parse the `debug_info_offset` in the arange header. +pub(crate) fn parse_debug_info_offset( + input: &mut R, + format: Format, +) -> Result> { + input.read_offset(format).map(DebugInfoOffset) +} + +/// This enum specifies the type of the unit and any type +/// specific data carried in the header (e.g. the type +/// signature/type offset of a type unit). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UnitType +where + Offset: ReaderOffset, +{ + /// In DWARF5, a unit with type `DW_UT_compile`. In previous DWARF versions, + /// any unit appearing in the .debug_info section. + Compilation, + /// In DWARF5, a unit with type `DW_UT_type`. In DWARF4, any unit appearing + /// in the .debug_types section. + Type { + /// The unique type signature for this type unit. + type_signature: DebugTypeSignature, + /// The offset within this type unit where the type is defined. + type_offset: UnitOffset, + }, + /// A unit with type `DW_UT_partial`. The root DIE of this unit should be a + /// `DW_TAG_partial_unit`. + Partial, + /// A unit with type `DW_UT_skeleton`. The enclosed dwo_id can be used to + /// link this with the corresponding `SplitCompilation` unit in a dwo file. + /// NB: The non-standard GNU split DWARF extension to DWARF 4 will instead + /// be a `Compilation` unit with the dwo_id present as an attribute on the + /// root DIE. + Skeleton(DwoId), + /// A unit with type `DW_UT_split_compile`. The enclosed dwo_id can be used to + /// link this with the corresponding `Skeleton` unit in the original binary. + /// NB: The non-standard GNU split DWARF extension to DWARF 4 will instead + /// be a `Compilation` unit with the dwo_id present as an attribute on the + /// root DIE. + SplitCompilation(DwoId), + /// A unit with type `DW_UT_split_type`. A split type unit is identical to a + /// conventional type unit except for the section in which it appears. + SplitType { + /// The unique type signature for this type unit. + type_signature: DebugTypeSignature, + /// The offset within this type unit where the type is defined. + type_offset: UnitOffset, + }, +} + +impl UnitType +where + Offset: ReaderOffset, +{ + // TODO: This will be used by the DWARF writing code once it + // supports unit types other than simple compilation units. + #[allow(unused)] + pub(crate) fn dw_ut(&self) -> constants::DwUt { + match self { + UnitType::Compilation => constants::DW_UT_compile, + UnitType::Type { .. } => constants::DW_UT_type, + UnitType::Partial => constants::DW_UT_partial, + UnitType::Skeleton(_) => constants::DW_UT_skeleton, + UnitType::SplitCompilation(_) => constants::DW_UT_split_compile, + UnitType::SplitType { .. } => constants::DW_UT_split_type, + } + } +} + +/// The common fields for the headers of compilation units and +/// type units. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct UnitHeader::Offset> +where + R: Reader, + Offset: ReaderOffset, +{ + encoding: Encoding, + unit_length: Offset, + unit_type: UnitType, + debug_abbrev_offset: DebugAbbrevOffset, + unit_offset: UnitSectionOffset, + entries_buf: R, +} + +/// Static methods. +impl UnitHeader +where + R: Reader, + Offset: ReaderOffset, +{ + /// Construct a new `UnitHeader`. + pub fn new( + encoding: Encoding, + unit_length: Offset, + unit_type: UnitType, + debug_abbrev_offset: DebugAbbrevOffset, + unit_offset: UnitSectionOffset, + entries_buf: R, + ) -> Self { + UnitHeader { + encoding, + unit_length, + unit_type, + debug_abbrev_offset, + unit_offset, + entries_buf, + } + } +} + +/// Instance methods. +impl UnitHeader +where + R: Reader, + Offset: ReaderOffset, +{ + /// Get the offset of this unit within its section. + pub fn offset(&self) -> UnitSectionOffset { + self.unit_offset + } + + /// Return the serialized size of the common unit header for the given + /// DWARF format. + pub fn size_of_header(&self) -> usize { + let unit_length_size = self.encoding.format.initial_length_size() as usize; + let version_size = 2; + let debug_abbrev_offset_size = self.encoding.format.word_size() as usize; + let address_size_size = 1; + let unit_type_size = if self.encoding.version == 5 { 1 } else { 0 }; + let type_specific_size = match self.unit_type { + UnitType::Compilation | UnitType::Partial => 0, + UnitType::Type { .. } | UnitType::SplitType { .. } => { + let type_signature_size = 8; + let type_offset_size = self.encoding.format.word_size() as usize; + type_signature_size + type_offset_size + } + UnitType::Skeleton(_) | UnitType::SplitCompilation(_) => 8, + }; + + unit_length_size + + version_size + + debug_abbrev_offset_size + + address_size_size + + unit_type_size + + type_specific_size + } + + /// Get the length of the debugging info for this compilation unit, not + /// including the byte length of the encoded length itself. + pub fn unit_length(&self) -> Offset { + self.unit_length + } + + /// Get the length of the debugging info for this compilation unit, + /// including the byte length of the encoded length itself. + pub fn length_including_self(&self) -> Offset { + Offset::from_u8(self.format().initial_length_size()) + self.unit_length + } + + /// Return the encoding parameters for this unit. + pub fn encoding(&self) -> Encoding { + self.encoding + } + + /// Get the DWARF version of the debugging info for this compilation unit. + pub fn version(&self) -> u16 { + self.encoding.version + } + + /// Get the UnitType of this unit. + pub fn type_(&self) -> UnitType { + self.unit_type + } + + /// The offset into the `.debug_abbrev` section for this compilation unit's + /// debugging information entries' abbreviations. + pub fn debug_abbrev_offset(&self) -> DebugAbbrevOffset { + self.debug_abbrev_offset + } + + /// The size of addresses (in bytes) in this compilation unit. + pub fn address_size(&self) -> u8 { + self.encoding.address_size + } + + /// Whether this compilation unit is encoded in 64- or 32-bit DWARF. + pub fn format(&self) -> Format { + self.encoding.format + } + + /// The serialized size of the header for this compilation unit. + pub fn header_size(&self) -> Offset { + self.length_including_self() - self.entries_buf.len() + } + + pub(crate) fn is_valid_offset(&self, offset: UnitOffset) -> bool { + let size_of_header = self.header_size(); + if offset.0 < size_of_header { + return false; + } + + let relative_to_entries_buf = offset.0 - size_of_header; + relative_to_entries_buf < self.entries_buf.len() + } + + /// Get the underlying bytes for the supplied range. + pub fn range(&self, idx: Range>) -> Result { + if !self.is_valid_offset(idx.start) { + return Err(Error::OffsetOutOfBounds); + } + if !self.is_valid_offset(idx.end) { + return Err(Error::OffsetOutOfBounds); + } + assert!(idx.start <= idx.end); + let size_of_header = self.header_size(); + let start = idx.start.0 - size_of_header; + let end = idx.end.0 - size_of_header; + let mut input = self.entries_buf.clone(); + input.skip(start)?; + input.truncate(end - start)?; + Ok(input) + } + + /// Get the underlying bytes for the supplied range. + pub fn range_from(&self, idx: RangeFrom>) -> Result { + if !self.is_valid_offset(idx.start) { + return Err(Error::OffsetOutOfBounds); + } + let start = idx.start.0 - self.header_size(); + let mut input = self.entries_buf.clone(); + input.skip(start)?; + Ok(input) + } + + /// Get the underlying bytes for the supplied range. + pub fn range_to(&self, idx: RangeTo>) -> Result { + if !self.is_valid_offset(idx.end) { + return Err(Error::OffsetOutOfBounds); + } + let end = idx.end.0 - self.header_size(); + let mut input = self.entries_buf.clone(); + input.truncate(end)?; + Ok(input) + } + + /// Read the `DebuggingInformationEntry` at the given offset. + pub fn entry<'me, 'abbrev>( + &'me self, + abbreviations: &'abbrev Abbreviations, + offset: UnitOffset, + ) -> Result> { + let mut input = self.range_from(offset..)?; + let entry = DebuggingInformationEntry::parse(&mut input, self, abbreviations)?; + entry.ok_or(Error::NoEntryAtGivenOffset) + } + + /// Navigate this unit's `DebuggingInformationEntry`s. + pub fn entries<'me, 'abbrev>( + &'me self, + abbreviations: &'abbrev Abbreviations, + ) -> EntriesCursor<'abbrev, 'me, R> { + EntriesCursor { + unit: self, + input: self.entries_buf.clone(), + abbreviations, + cached_current: None, + delta_depth: 0, + } + } + + /// Navigate this compilation unit's `DebuggingInformationEntry`s + /// starting at the given offset. + pub fn entries_at_offset<'me, 'abbrev>( + &'me self, + abbreviations: &'abbrev Abbreviations, + offset: UnitOffset, + ) -> Result> { + let input = self.range_from(offset..)?; + Ok(EntriesCursor { + unit: self, + input, + abbreviations, + cached_current: None, + delta_depth: 0, + }) + } + + /// Navigate this unit's `DebuggingInformationEntry`s as a tree + /// starting at the given offset. + pub fn entries_tree<'me, 'abbrev>( + &'me self, + abbreviations: &'abbrev Abbreviations, + offset: Option>, + ) -> Result> { + let input = match offset { + Some(offset) => self.range_from(offset..)?, + None => self.entries_buf.clone(), + }; + Ok(EntriesTree::new(input, self, abbreviations)) + } + + /// Read the raw data that defines the Debugging Information Entries. + pub fn entries_raw<'me, 'abbrev>( + &'me self, + abbreviations: &'abbrev Abbreviations, + offset: Option>, + ) -> Result> { + let input = match offset { + Some(offset) => self.range_from(offset..)?, + None => self.entries_buf.clone(), + }; + Ok(EntriesRaw { + input, + unit: self, + abbreviations, + depth: 0, + }) + } + + /// Parse this unit's abbreviations. + pub fn abbreviations(&self, debug_abbrev: &DebugAbbrev) -> Result { + debug_abbrev.abbreviations(self.debug_abbrev_offset()) + } +} + +/// Parse a unit header. +fn parse_unit_header( + input: &mut R, + unit_offset: UnitSectionOffset, +) -> Result> +where + R: Reader, + Offset: ReaderOffset, +{ + let (unit_length, format) = input.read_initial_length()?; + let mut rest = input.split(unit_length)?; + + let version = rest.read_u16()?; + let abbrev_offset; + let address_size; + let unit_type; + // DWARF 1 was very different, and is obsolete, so isn't supported by this + // reader. + if 2 <= version && version <= 4 { + abbrev_offset = parse_debug_abbrev_offset(&mut rest, format)?; + address_size = rest.read_u8()?; + // Before DWARF5, all units in the .debug_info section are compilation + // units, and all units in the .debug_types section are type units. + unit_type = match unit_offset { + UnitSectionOffset::DebugInfoOffset(_) => constants::DW_UT_compile, + UnitSectionOffset::DebugTypesOffset(_) => constants::DW_UT_type, + }; + } else if version == 5 { + unit_type = parse_unit_type(&mut rest)?; + address_size = rest.read_u8()?; + abbrev_offset = parse_debug_abbrev_offset(&mut rest, format)?; + } else { + return Err(Error::UnknownVersion(u64::from(version))); + } + let encoding = Encoding { + format, + version, + address_size, + }; + + // Parse any data specific to this type of unit. + let unit_type = match unit_type { + constants::DW_UT_compile => UnitType::Compilation, + constants::DW_UT_type => { + let type_signature = parse_type_signature(&mut rest)?; + let type_offset = parse_type_offset(&mut rest, format)?; + UnitType::Type { + type_signature, + type_offset, + } + } + constants::DW_UT_partial => UnitType::Partial, + constants::DW_UT_skeleton => { + let dwo_id = parse_dwo_id(&mut rest)?; + UnitType::Skeleton(dwo_id) + } + constants::DW_UT_split_compile => { + let dwo_id = parse_dwo_id(&mut rest)?; + UnitType::SplitCompilation(dwo_id) + } + constants::DW_UT_split_type => { + let type_signature = parse_type_signature(&mut rest)?; + let type_offset = parse_type_offset(&mut rest, format)?; + UnitType::SplitType { + type_signature, + type_offset, + } + } + _ => return Err(Error::UnsupportedUnitType), + }; + + Ok(UnitHeader::new( + encoding, + unit_length, + unit_type, + abbrev_offset, + unit_offset, + rest, + )) +} + +/// Parse a dwo_id from a header +fn parse_dwo_id(input: &mut R) -> Result { + Ok(DwoId(input.read_u64()?)) +} + +/// A Debugging Information Entry (DIE). +/// +/// DIEs have a set of attributes and optionally have children DIEs as well. +#[derive(Clone, Debug)] +pub struct DebuggingInformationEntry<'abbrev, 'unit, R, Offset = ::Offset> +where + R: Reader, + Offset: ReaderOffset, +{ + offset: UnitOffset, + attrs_slice: R, + attrs_len: Cell>, + abbrev: &'abbrev Abbreviation, + unit: &'unit UnitHeader, +} + +impl<'abbrev, 'unit, R, Offset> DebuggingInformationEntry<'abbrev, 'unit, R, Offset> +where + R: Reader, + Offset: ReaderOffset, +{ + /// Construct a new `DebuggingInformationEntry`. + pub fn new( + offset: UnitOffset, + attrs_slice: R, + abbrev: &'abbrev Abbreviation, + unit: &'unit UnitHeader, + ) -> Self { + DebuggingInformationEntry { + offset, + attrs_slice, + attrs_len: Cell::new(None), + abbrev, + unit, + } + } + + /// Get this entry's code. + pub fn code(&self) -> u64 { + self.abbrev.code() + } + + /// Get this entry's offset. + pub fn offset(&self) -> UnitOffset { + self.offset + } + + /// Get this entry's `DW_TAG_whatever` tag. + /// + /// ``` + /// # use gimli::{DebugAbbrev, DebugInfo, LittleEndian}; + /// # let info_buf = [ + /// # // Comilation unit header + /// # + /// # // 32-bit unit length = 12 + /// # 0x0c, 0x00, 0x00, 0x00, + /// # // Version 4 + /// # 0x04, 0x00, + /// # // debug_abbrev_offset + /// # 0x00, 0x00, 0x00, 0x00, + /// # // Address size + /// # 0x04, + /// # + /// # // DIEs + /// # + /// # // Abbreviation code + /// # 0x01, + /// # // Attribute of form DW_FORM_string = "foo\0" + /// # 0x66, 0x6f, 0x6f, 0x00, + /// # ]; + /// # let debug_info = DebugInfo::new(&info_buf, LittleEndian); + /// # let abbrev_buf = [ + /// # // Code + /// # 0x01, + /// # // DW_TAG_subprogram + /// # 0x2e, + /// # // DW_CHILDREN_no + /// # 0x00, + /// # // Begin attributes + /// # // Attribute name = DW_AT_name + /// # 0x03, + /// # // Attribute form = DW_FORM_string + /// # 0x08, + /// # // End attributes + /// # 0x00, + /// # 0x00, + /// # // Null terminator + /// # 0x00 + /// # ]; + /// # let debug_abbrev = DebugAbbrev::new(&abbrev_buf, LittleEndian); + /// # let unit = debug_info.units().next().unwrap().unwrap(); + /// # let abbrevs = unit.abbreviations(&debug_abbrev).unwrap(); + /// # let mut cursor = unit.entries(&abbrevs); + /// # let (_, entry) = cursor.next_dfs().unwrap().unwrap(); + /// # let mut get_some_entry = || entry; + /// let entry = get_some_entry(); + /// + /// match entry.tag() { + /// gimli::DW_TAG_subprogram => + /// println!("this entry contains debug info about a function"), + /// gimli::DW_TAG_inlined_subroutine => + /// println!("this entry contains debug info about a particular instance of inlining"), + /// gimli::DW_TAG_variable => + /// println!("this entry contains debug info about a local variable"), + /// gimli::DW_TAG_formal_parameter => + /// println!("this entry contains debug info about a function parameter"), + /// otherwise => + /// println!("this entry is some other kind of data: {:?}", otherwise), + /// }; + /// ``` + pub fn tag(&self) -> constants::DwTag { + self.abbrev.tag() + } + + /// Return true if this entry's type can have children, false otherwise. + pub fn has_children(&self) -> bool { + self.abbrev.has_children() + } + + /// Iterate over this entry's set of attributes. + /// + /// ``` + /// use gimli::{DebugAbbrev, DebugInfo, LittleEndian}; + /// + /// // Read the `.debug_info` section. + /// + /// # let info_buf = [ + /// # // Comilation unit header + /// # + /// # // 32-bit unit length = 12 + /// # 0x0c, 0x00, 0x00, 0x00, + /// # // Version 4 + /// # 0x04, 0x00, + /// # // debug_abbrev_offset + /// # 0x00, 0x00, 0x00, 0x00, + /// # // Address size + /// # 0x04, + /// # + /// # // DIEs + /// # + /// # // Abbreviation code + /// # 0x01, + /// # // Attribute of form DW_FORM_string = "foo\0" + /// # 0x66, 0x6f, 0x6f, 0x00, + /// # ]; + /// # let read_debug_info_section_somehow = || &info_buf; + /// let debug_info = DebugInfo::new(read_debug_info_section_somehow(), LittleEndian); + /// + /// // Get the data about the first compilation unit out of the `.debug_info`. + /// + /// let unit = debug_info.units().next() + /// .expect("Should have at least one compilation unit") + /// .expect("and it should parse ok"); + /// + /// // Read the `.debug_abbrev` section and parse the + /// // abbreviations for our compilation unit. + /// + /// # let abbrev_buf = [ + /// # // Code + /// # 0x01, + /// # // DW_TAG_subprogram + /// # 0x2e, + /// # // DW_CHILDREN_no + /// # 0x00, + /// # // Begin attributes + /// # // Attribute name = DW_AT_name + /// # 0x03, + /// # // Attribute form = DW_FORM_string + /// # 0x08, + /// # // End attributes + /// # 0x00, + /// # 0x00, + /// # // Null terminator + /// # 0x00 + /// # ]; + /// # let read_debug_abbrev_section_somehow = || &abbrev_buf; + /// let debug_abbrev = DebugAbbrev::new(read_debug_abbrev_section_somehow(), LittleEndian); + /// let abbrevs = unit.abbreviations(&debug_abbrev).unwrap(); + /// + /// // Get the first entry from that compilation unit. + /// + /// let mut cursor = unit.entries(&abbrevs); + /// let (_, entry) = cursor.next_dfs() + /// .expect("Should parse next entry") + /// .expect("Should have at least one entry"); + /// + /// // Finally, print the first entry's attributes. + /// + /// let mut attrs = entry.attrs(); + /// while let Some(attr) = attrs.next().unwrap() { + /// println!("Attribute name = {:?}", attr.name()); + /// println!("Attribute value = {:?}", attr.value()); + /// } + /// ``` + /// + /// Can be [used with + /// `FallibleIterator`](./index.html#using-with-fallibleiterator). + pub fn attrs<'me>(&'me self) -> AttrsIter<'abbrev, 'me, 'unit, R> { + AttrsIter { + input: self.attrs_slice.clone(), + attributes: self.abbrev.attributes(), + entry: self, + } + } + + /// Find the first attribute in this entry which has the given name, + /// and return it. Returns `Ok(None)` if no attribute is found. + pub fn attr(&self, name: constants::DwAt) -> Result>> { + let mut attrs = self.attrs(); + while let Some(attr) = attrs.next()? { + if attr.name() == name { + return Ok(Some(attr)); + } + } + Ok(None) + } + + /// Find the first attribute in this entry which has the given name, + /// and return its raw value. Returns `Ok(None)` if no attribute is found. + pub fn attr_value_raw(&self, name: constants::DwAt) -> Result>> { + self.attr(name) + .map(|attr| attr.map(|attr| attr.raw_value())) + } + + /// Find the first attribute in this entry which has the given name, + /// and return its normalized value. Returns `Ok(None)` if no + /// attribute is found. + pub fn attr_value(&self, name: constants::DwAt) -> Result>> { + self.attr(name).map(|attr| attr.map(|attr| attr.value())) + } + + /// Return the input buffer after the last attribute. + #[allow(clippy::inline_always)] + #[inline(always)] + fn after_attrs(&self) -> Result { + if let Some(attrs_len) = self.attrs_len.get() { + let mut input = self.attrs_slice.clone(); + input.skip(attrs_len)?; + Ok(input) + } else { + let mut attrs = self.attrs(); + while let Some(_) = attrs.next()? {} + Ok(attrs.input) + } + } + + /// Use the `DW_AT_sibling` attribute to find the input buffer for the + /// next sibling. Returns `None` if the attribute is missing or invalid. + fn sibling(&self) -> Option { + let attr = self.attr_value(constants::DW_AT_sibling); + if let Ok(Some(AttributeValue::UnitRef(offset))) = attr { + if offset.0 > self.offset.0 { + if let Ok(input) = self.unit.range_from(offset..) { + return Some(input); + } + } + } + None + } + + /// Parse an entry. Returns `Ok(None)` for null entries. + #[allow(clippy::inline_always)] + #[inline(always)] + fn parse( + input: &mut R, + unit: &'unit UnitHeader, + abbreviations: &'abbrev Abbreviations, + ) -> Result> { + let offset = unit.header_size() + input.offset_from(&unit.entries_buf); + let code = input.read_uleb128()?; + if code == 0 { + return Ok(None); + }; + let abbrev = abbreviations.get(code).ok_or(Error::UnknownAbbreviation)?; + Ok(Some(DebuggingInformationEntry { + offset: UnitOffset(offset), + attrs_slice: input.clone(), + attrs_len: Cell::new(None), + abbrev, + unit, + })) + } +} + +/// The value of an attribute in a `DebuggingInformationEntry`. +// +// Set the discriminant size so that all variants use the same alignment +// for their data. This gives better code generation in `parse_attribute`. +#[repr(u64)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum AttributeValue::Offset> +where + R: Reader, + Offset: ReaderOffset, +{ + /// "Refers to some location in the address space of the described program." + Addr(u64), + + /// A slice of an arbitrary number of bytes. + Block(R), + + /// A one byte constant data value. How to interpret the byte depends on context. + /// + /// From section 7 of the standard: "Depending on context, it may be a + /// signed integer, an unsigned integer, a floating-point constant, or + /// anything else." + Data1(u8), + + /// A two byte constant data value. How to interpret the bytes depends on context. + /// + /// These bytes have been converted from `R::Endian`. This may need to be reversed + /// if this was not required. + /// + /// From section 7 of the standard: "Depending on context, it may be a + /// signed integer, an unsigned integer, a floating-point constant, or + /// anything else." + Data2(u16), + + /// A four byte constant data value. How to interpret the bytes depends on context. + /// + /// These bytes have been converted from `R::Endian`. This may need to be reversed + /// if this was not required. + /// + /// From section 7 of the standard: "Depending on context, it may be a + /// signed integer, an unsigned integer, a floating-point constant, or + /// anything else." + Data4(u32), + + /// An eight byte constant data value. How to interpret the bytes depends on context. + /// + /// These bytes have been converted from `R::Endian`. This may need to be reversed + /// if this was not required. + /// + /// From section 7 of the standard: "Depending on context, it may be a + /// signed integer, an unsigned integer, a floating-point constant, or + /// anything else." + Data8(u64), + + /// A signed integer constant. + Sdata(i64), + + /// An unsigned integer constant. + Udata(u64), + + /// "The information bytes contain a DWARF expression (see Section 2.5) or + /// location description (see Section 2.6)." + Exprloc(Expression), + + /// A boolean that indicates presence or absence of the attribute. + Flag(bool), + + /// An offset into another section. Which section this is an offset into + /// depends on context. + SecOffset(Offset), + + /// An offset to a set of addresses in the `.debug_addr` section. + DebugAddrBase(DebugAddrBase), + + /// An index into a set of addresses in the `.debug_addr` section. + DebugAddrIndex(DebugAddrIndex), + + /// An offset into the current compilation unit. + UnitRef(UnitOffset), + + /// An offset into the current `.debug_info` section, but possibly a + /// different compilation unit from the current one. + DebugInfoRef(DebugInfoOffset), + + /// An offset into the `.debug_info` section of the supplementary object file. + DebugInfoRefSup(DebugInfoOffset), + + /// An offset into the `.debug_line` section. + DebugLineRef(DebugLineOffset), + + /// An offset into either the `.debug_loc` section or the `.debug_loclists` section. + LocationListsRef(LocationListsOffset), + + /// An offset to a set of offsets in the `.debug_loclists` section. + DebugLocListsBase(DebugLocListsBase), + + /// An index into a set of offsets in the `.debug_loclists` section. + DebugLocListsIndex(DebugLocListsIndex), + + /// An offset into the `.debug_macinfo` section. + DebugMacinfoRef(DebugMacinfoOffset), + + /// An offset into the `.debug_macro` section. + DebugMacroRef(DebugMacroOffset), + + /// An offset into the `.debug_ranges` section. + RangeListsRef(RawRangeListsOffset), + + /// An offset to a set of offsets in the `.debug_rnglists` section. + DebugRngListsBase(DebugRngListsBase), + + /// An index into a set of offsets in the `.debug_rnglists` section. + DebugRngListsIndex(DebugRngListsIndex), + + /// A type signature. + DebugTypesRef(DebugTypeSignature), + + /// An offset into the `.debug_str` section. + DebugStrRef(DebugStrOffset), + + /// An offset into the `.debug_str` section of the supplementary object file. + DebugStrRefSup(DebugStrOffset), + + /// An offset to a set of entries in the `.debug_str_offsets` section. + DebugStrOffsetsBase(DebugStrOffsetsBase), + + /// An index into a set of entries in the `.debug_str_offsets` section. + DebugStrOffsetsIndex(DebugStrOffsetsIndex), + + /// An offset into the `.debug_line_str` section. + DebugLineStrRef(DebugLineStrOffset), + + /// A slice of bytes representing a string. Does not include a final null byte. + /// Not guaranteed to be UTF-8 or anything like that. + String(R), + + /// The value of a `DW_AT_encoding` attribute. + Encoding(constants::DwAte), + + /// The value of a `DW_AT_decimal_sign` attribute. + DecimalSign(constants::DwDs), + + /// The value of a `DW_AT_endianity` attribute. + Endianity(constants::DwEnd), + + /// The value of a `DW_AT_accessibility` attribute. + Accessibility(constants::DwAccess), + + /// The value of a `DW_AT_visibility` attribute. + Visibility(constants::DwVis), + + /// The value of a `DW_AT_virtuality` attribute. + Virtuality(constants::DwVirtuality), + + /// The value of a `DW_AT_language` attribute. + Language(constants::DwLang), + + /// The value of a `DW_AT_address_class` attribute. + AddressClass(constants::DwAddr), + + /// The value of a `DW_AT_identifier_case` attribute. + IdentifierCase(constants::DwId), + + /// The value of a `DW_AT_calling_convention` attribute. + CallingConvention(constants::DwCc), + + /// The value of a `DW_AT_inline` attribute. + Inline(constants::DwInl), + + /// The value of a `DW_AT_ordering` attribute. + Ordering(constants::DwOrd), + + /// An index into the filename entries from the line number information + /// table for the compilation unit containing this value. + FileIndex(u64), + + /// An implementation-defined identifier uniquely identifying a compilation + /// unit. + DwoId(DwoId), +} + +/// An attribute in a `DebuggingInformationEntry`, consisting of a name and +/// associated value. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct Attribute { + name: constants::DwAt, + value: AttributeValue, +} + +impl Attribute { + /// Get this attribute's name. + pub fn name(&self) -> constants::DwAt { + self.name + } + + /// Get this attribute's raw value. + pub fn raw_value(&self) -> AttributeValue { + self.value.clone() + } + + /// Get this attribute's normalized value. + /// + /// Attribute values can potentially be encoded in multiple equivalent forms, + /// and may have special meaning depending on the attribute name. This method + /// converts the attribute value to a normalized form based on the attribute + /// name. + /// + /// See "Table 7.5: Attribute encodings" and "Table 7.6: Attribute form encodings". + #[allow(clippy::cyclomatic_complexity)] + #[allow(clippy::match_same_arms)] + pub fn value(&self) -> AttributeValue { + // Table 7.5 shows the possible attribute classes for each name. + // Table 7.6 shows the possible attribute classes for each form. + // For each attribute name, we need to match on the form, and + // convert it to one of the classes that is allowed for both + // the name and the form. + // + // The individual class conversions rarely vary for each name, + // so for each class conversion we define a macro that matches + // on the allowed forms for that class. + // + // For some classes, we don't need to do any conversion, so their + // macro is empty. In the future we may want to fill them in to + // provide strict checking of the forms for each class. For now, + // they simply provide a way to document the allowed classes for + // each name. + + // DW_FORM_addr + // DW_FORM_addrx + // DW_FORM_addrx1 + // DW_FORM_addrx2 + // DW_FORM_addrx3 + // DW_FORM_addrx4 + macro_rules! address { + () => {}; + } + // DW_FORM_sec_offset + macro_rules! addrptr { + () => { + if let Some(offset) = self.offset_value() { + return AttributeValue::DebugAddrBase(DebugAddrBase(offset)); + } + }; + } + // DW_FORM_block + // DW_FORM_block1 + // DW_FORM_block2 + // DW_FORM_block4 + macro_rules! block { + () => {}; + } + // DW_FORM_sdata + // DW_FORM_udata + // DW_FORM_data1 + // DW_FORM_data2 + // DW_FORM_data4 + // DW_FORM_data8 + // DW_FORM_data16 + // DW_FORM_implicit_const + macro_rules! constant { + ($value:ident, $variant:ident) => { + if let Some(value) = self.$value() { + return AttributeValue::$variant(value); + } + }; + ($value:ident, $variant:ident, $constant:ident) => { + if let Some(value) = self.$value() { + return AttributeValue::$variant(constants::$constant(value)); + } + }; + } + // DW_FORM_exprloc + macro_rules! exprloc { + () => { + if let Some(value) = self.exprloc_value() { + return AttributeValue::Exprloc(value); + } + }; + } + // DW_FORM_flag + // DW_FORM_flag_present + macro_rules! flag { + () => {}; + } + // DW_FORM_sec_offset + macro_rules! lineptr { + () => { + if let Some(offset) = self.offset_value() { + return AttributeValue::DebugLineRef(DebugLineOffset(offset)); + } + }; + } + // This also covers `loclist` in DWARF version 5. + // DW_FORM_sec_offset + // DW_FORM_loclistx + macro_rules! loclistptr { + () => { + // DebugLocListsIndex is also an allowed form in DWARF version 5. + if let Some(offset) = self.offset_value() { + return AttributeValue::LocationListsRef(LocationListsOffset(offset)); + } + }; + } + // DW_FORM_sec_offset + macro_rules! loclistsptr { + () => { + if let Some(offset) = self.offset_value() { + return AttributeValue::DebugLocListsBase(DebugLocListsBase(offset)); + } + }; + } + // DWARF version <= 4. + // DW_FORM_sec_offset + macro_rules! macinfoptr { + () => { + if let Some(offset) = self.offset_value() { + return AttributeValue::DebugMacinfoRef(DebugMacinfoOffset(offset)); + } + }; + } + // DWARF version >= 5. + // DW_FORM_sec_offset + macro_rules! macroptr { + () => { + if let Some(offset) = self.offset_value() { + return AttributeValue::DebugMacroRef(DebugMacroOffset(offset)); + } + }; + } + // DW_FORM_ref_addr + // DW_FORM_ref1 + // DW_FORM_ref2 + // DW_FORM_ref4 + // DW_FORM_ref8 + // DW_FORM_ref_udata + // DW_FORM_ref_sig8 + // DW_FORM_ref_sup4 + // DW_FORM_ref_sup8 + macro_rules! reference { + () => {}; + } + // This also covers `rnglist` in DWARF version 5. + // DW_FORM_sec_offset + // DW_FORM_rnglistx + macro_rules! rangelistptr { + () => { + // DebugRngListsIndex is also an allowed form in DWARF version 5. + if let Some(offset) = self.offset_value() { + return AttributeValue::RangeListsRef(RawRangeListsOffset(offset)); + } + }; + } + // DW_FORM_sec_offset + macro_rules! rnglistsptr { + () => { + if let Some(offset) = self.offset_value() { + return AttributeValue::DebugRngListsBase(DebugRngListsBase(offset)); + } + }; + } + // DW_FORM_string + // DW_FORM_strp + // DW_FORM_strx + // DW_FORM_strx1 + // DW_FORM_strx2 + // DW_FORM_strx3 + // DW_FORM_strx4 + // DW_FORM_strp_sup + // DW_FORM_line_strp + macro_rules! string { + () => {}; + } + // DW_FORM_sec_offset + macro_rules! stroffsetsptr { + () => { + if let Some(offset) = self.offset_value() { + return AttributeValue::DebugStrOffsetsBase(DebugStrOffsetsBase(offset)); + } + }; + } + // This isn't a separate form but it's useful to distinguish it from a generic udata. + macro_rules! dwoid { + () => { + if let Some(value) = self.udata_value() { + return AttributeValue::DwoId(DwoId(value)); + } + }; + } + + // Perform the allowed class conversions for each attribute name. + match self.name { + constants::DW_AT_sibling => { + reference!(); + } + constants::DW_AT_location => { + exprloc!(); + loclistptr!(); + } + constants::DW_AT_name => { + string!(); + } + constants::DW_AT_ordering => { + constant!(u8_value, Ordering, DwOrd); + } + constants::DW_AT_byte_size + | constants::DW_AT_bit_offset + | constants::DW_AT_bit_size => { + constant!(udata_value, Udata); + exprloc!(); + reference!(); + } + constants::DW_AT_stmt_list => { + lineptr!(); + } + constants::DW_AT_low_pc => { + address!(); + } + constants::DW_AT_high_pc => { + address!(); + constant!(udata_value, Udata); + } + constants::DW_AT_language => { + constant!(u16_value, Language, DwLang); + } + constants::DW_AT_discr => { + reference!(); + } + constants::DW_AT_discr_value => { + // constant: depends on type of DW_TAG_variant_part, + // so caller must normalize. + } + constants::DW_AT_visibility => { + constant!(u8_value, Visibility, DwVis); + } + constants::DW_AT_import => { + reference!(); + } + constants::DW_AT_string_length => { + exprloc!(); + loclistptr!(); + reference!(); + } + constants::DW_AT_common_reference => { + reference!(); + } + constants::DW_AT_comp_dir => { + string!(); + } + constants::DW_AT_const_value => { + // TODO: constant: sign depends on DW_AT_type. + block!(); + string!(); + } + constants::DW_AT_containing_type => { + reference!(); + } + constants::DW_AT_default_value => { + // TODO: constant: sign depends on DW_AT_type. + reference!(); + flag!(); + } + constants::DW_AT_inline => { + constant!(u8_value, Inline, DwInl); + } + constants::DW_AT_is_optional => { + flag!(); + } + constants::DW_AT_lower_bound => { + // TODO: constant: sign depends on DW_AT_type. + exprloc!(); + reference!(); + } + constants::DW_AT_producer => { + string!(); + } + constants::DW_AT_prototyped => { + flag!(); + } + constants::DW_AT_return_addr => { + exprloc!(); + loclistptr!(); + } + constants::DW_AT_start_scope => { + // TODO: constant + rangelistptr!(); + } + constants::DW_AT_bit_stride => { + constant!(udata_value, Udata); + exprloc!(); + reference!(); + } + constants::DW_AT_upper_bound => { + // TODO: constant: sign depends on DW_AT_type. + exprloc!(); + reference!(); + } + constants::DW_AT_abstract_origin => { + reference!(); + } + constants::DW_AT_accessibility => { + constant!(u8_value, Accessibility, DwAccess); + } + constants::DW_AT_address_class => { + constant!(udata_value, AddressClass, DwAddr); + } + constants::DW_AT_artificial => { + flag!(); + } + constants::DW_AT_base_types => { + reference!(); + } + constants::DW_AT_calling_convention => { + constant!(u8_value, CallingConvention, DwCc); + } + constants::DW_AT_count => { + // TODO: constant + exprloc!(); + reference!(); + } + constants::DW_AT_data_member_location => { + // Constants must be handled before loclistptr so that DW_FORM_data4/8 + // are correctly interpreted for DWARF version 4+. + constant!(udata_value, Udata); + exprloc!(); + loclistptr!(); + } + constants::DW_AT_decl_column => { + constant!(udata_value, Udata); + } + constants::DW_AT_decl_file => { + constant!(udata_value, FileIndex); + } + constants::DW_AT_decl_line => { + constant!(udata_value, Udata); + } + constants::DW_AT_declaration => { + flag!(); + } + constants::DW_AT_discr_list => { + block!(); + } + constants::DW_AT_encoding => { + constant!(u8_value, Encoding, DwAte); + } + constants::DW_AT_external => { + flag!(); + } + constants::DW_AT_frame_base => { + exprloc!(); + loclistptr!(); + } + constants::DW_AT_friend => { + reference!(); + } + constants::DW_AT_identifier_case => { + constant!(u8_value, IdentifierCase, DwId); + } + constants::DW_AT_macro_info => { + macinfoptr!(); + } + constants::DW_AT_namelist_item => { + reference!(); + } + constants::DW_AT_priority => { + reference!(); + } + constants::DW_AT_segment => { + exprloc!(); + loclistptr!(); + } + constants::DW_AT_specification => { + reference!(); + } + constants::DW_AT_static_link => { + exprloc!(); + loclistptr!(); + } + constants::DW_AT_type => { + reference!(); + } + constants::DW_AT_use_location => { + exprloc!(); + loclistptr!(); + } + constants::DW_AT_variable_parameter => { + flag!(); + } + constants::DW_AT_virtuality => { + constant!(u8_value, Virtuality, DwVirtuality); + } + constants::DW_AT_vtable_elem_location => { + exprloc!(); + loclistptr!(); + } + constants::DW_AT_allocated => { + // TODO: constant + exprloc!(); + reference!(); + } + constants::DW_AT_associated => { + // TODO: constant + exprloc!(); + reference!(); + } + constants::DW_AT_data_location => { + exprloc!(); + } + constants::DW_AT_byte_stride => { + constant!(udata_value, Udata); + exprloc!(); + reference!(); + } + constants::DW_AT_entry_pc => { + // TODO: constant + address!(); + } + constants::DW_AT_use_UTF8 => { + flag!(); + } + constants::DW_AT_extension => { + reference!(); + } + constants::DW_AT_ranges => { + rangelistptr!(); + } + constants::DW_AT_trampoline => { + address!(); + flag!(); + reference!(); + string!(); + } + constants::DW_AT_call_column => { + constant!(udata_value, Udata); + } + constants::DW_AT_call_file => { + constant!(udata_value, FileIndex); + } + constants::DW_AT_call_line => { + constant!(udata_value, Udata); + } + constants::DW_AT_description => { + string!(); + } + constants::DW_AT_binary_scale => { + // TODO: constant + } + constants::DW_AT_decimal_scale => { + // TODO: constant + } + constants::DW_AT_small => { + reference!(); + } + constants::DW_AT_decimal_sign => { + constant!(u8_value, DecimalSign, DwDs); + } + constants::DW_AT_digit_count => { + // TODO: constant + } + constants::DW_AT_picture_string => { + string!(); + } + constants::DW_AT_mutable => { + flag!(); + } + constants::DW_AT_threads_scaled => { + flag!(); + } + constants::DW_AT_explicit => { + flag!(); + } + constants::DW_AT_object_pointer => { + reference!(); + } + constants::DW_AT_endianity => { + constant!(u8_value, Endianity, DwEnd); + } + constants::DW_AT_elemental => { + flag!(); + } + constants::DW_AT_pure => { + flag!(); + } + constants::DW_AT_recursive => { + flag!(); + } + constants::DW_AT_signature => { + reference!(); + } + constants::DW_AT_main_subprogram => { + flag!(); + } + constants::DW_AT_data_bit_offset => { + // TODO: constant + } + constants::DW_AT_const_expr => { + flag!(); + } + constants::DW_AT_enum_class => { + flag!(); + } + constants::DW_AT_linkage_name => { + string!(); + } + constants::DW_AT_string_length_bit_size => { + // TODO: constant + } + constants::DW_AT_string_length_byte_size => { + // TODO: constant + } + constants::DW_AT_rank => { + // TODO: constant + exprloc!(); + } + constants::DW_AT_str_offsets_base => { + stroffsetsptr!(); + } + constants::DW_AT_addr_base | constants::DW_AT_GNU_addr_base => { + addrptr!(); + } + constants::DW_AT_rnglists_base | constants::DW_AT_GNU_ranges_base => { + rnglistsptr!(); + } + constants::DW_AT_dwo_name => { + string!(); + } + constants::DW_AT_reference => { + flag!(); + } + constants::DW_AT_rvalue_reference => { + flag!(); + } + constants::DW_AT_macros => { + macroptr!(); + } + constants::DW_AT_call_all_calls => { + flag!(); + } + constants::DW_AT_call_all_source_calls => { + flag!(); + } + constants::DW_AT_call_all_tail_calls => { + flag!(); + } + constants::DW_AT_call_return_pc => { + address!(); + } + constants::DW_AT_call_value => { + exprloc!(); + } + constants::DW_AT_call_origin => { + exprloc!(); + } + constants::DW_AT_call_parameter => { + reference!(); + } + constants::DW_AT_call_pc => { + address!(); + } + constants::DW_AT_call_tail_call => { + flag!(); + } + constants::DW_AT_call_target => { + exprloc!(); + } + constants::DW_AT_call_target_clobbered => { + exprloc!(); + } + constants::DW_AT_call_data_location => { + exprloc!(); + } + constants::DW_AT_call_data_value => { + exprloc!(); + } + constants::DW_AT_noreturn => { + flag!(); + } + constants::DW_AT_alignment => { + // TODO: constant + } + constants::DW_AT_export_symbols => { + flag!(); + } + constants::DW_AT_deleted => { + flag!(); + } + constants::DW_AT_defaulted => { + // TODO: constant + } + constants::DW_AT_loclists_base => { + loclistsptr!(); + } + constants::DW_AT_GNU_dwo_id => { + dwoid!(); + } + _ => {} + } + self.value.clone() + } + + /// Try to convert this attribute's value to a u8. + #[inline] + pub fn u8_value(&self) -> Option { + self.value.u8_value() + } + + /// Try to convert this attribute's value to a u16. + #[inline] + pub fn u16_value(&self) -> Option { + self.value.u16_value() + } + + /// Try to convert this attribute's value to an unsigned integer. + #[inline] + pub fn udata_value(&self) -> Option { + self.value.udata_value() + } + + /// Try to convert this attribute's value to a signed integer. + #[inline] + pub fn sdata_value(&self) -> Option { + self.value.sdata_value() + } + + /// Try to convert this attribute's value to an offset. + #[inline] + pub fn offset_value(&self) -> Option { + self.value.offset_value() + } + + /// Try to convert this attribute's value to an expression or location buffer. + /// + /// Expressions and locations may be `DW_FORM_block*` or `DW_FORM_exprloc`. + /// The standard doesn't mention `DW_FORM_block*` as a possible form, but + /// it is encountered in practice. + #[inline] + pub fn exprloc_value(&self) -> Option> { + self.value.exprloc_value() + } + + /// Try to return this attribute's value as a string slice. + /// + /// If this attribute's value is either an inline `DW_FORM_string` string, + /// or a `DW_FORM_strp` reference to an offset into the `.debug_str` + /// section, return the attribute's string value as `Some`. Other attribute + /// value forms are returned as `None`. + /// + /// Warning: this function does not handle all possible string forms. + /// Use `Dwarf::attr_string` instead. + #[inline] + pub fn string_value(&self, debug_str: &DebugStr) -> Option { + self.value.string_value(debug_str) + } + + /// Try to return this attribute's value as a string slice. + /// + /// If this attribute's value is either an inline `DW_FORM_string` string, + /// or a `DW_FORM_strp` reference to an offset into the `.debug_str` + /// section, or a `DW_FORM_strp_sup` reference to an offset into a supplementary + /// object file, return the attribute's string value as `Some`. Other attribute + /// value forms are returned as `None`. + /// + /// Warning: this function does not handle all possible string forms. + /// Use `Dwarf::attr_string` instead. + #[inline] + pub fn string_value_sup( + &self, + debug_str: &DebugStr, + debug_str_sup: Option<&DebugStr>, + ) -> Option { + self.value.string_value_sup(debug_str, debug_str_sup) + } +} + +impl AttributeValue +where + R: Reader, + Offset: ReaderOffset, +{ + /// Try to convert this attribute's value to a u8. + pub fn u8_value(&self) -> Option { + if let Some(value) = self.udata_value() { + if value <= u64::from(u8::MAX) { + return Some(value as u8); + } + } + None + } + + /// Try to convert this attribute's value to a u16. + pub fn u16_value(&self) -> Option { + if let Some(value) = self.udata_value() { + if value <= u64::from(u16::MAX) { + return Some(value as u16); + } + } + None + } + + /// Try to convert this attribute's value to an unsigned integer. + pub fn udata_value(&self) -> Option { + Some(match *self { + AttributeValue::Data1(data) => u64::from(data), + AttributeValue::Data2(data) => u64::from(data), + AttributeValue::Data4(data) => u64::from(data), + AttributeValue::Data8(data) => data, + AttributeValue::Udata(data) => data, + AttributeValue::Sdata(data) => { + if data < 0 { + // Maybe we should emit a warning here + return None; + } + data as u64 + } + _ => return None, + }) + } + + /// Try to convert this attribute's value to a signed integer. + pub fn sdata_value(&self) -> Option { + Some(match *self { + AttributeValue::Data1(data) => i64::from(data as i8), + AttributeValue::Data2(data) => i64::from(data as i16), + AttributeValue::Data4(data) => i64::from(data as i32), + AttributeValue::Data8(data) => data as i64, + AttributeValue::Sdata(data) => data, + AttributeValue::Udata(data) => { + if data > i64::max_value() as u64 { + // Maybe we should emit a warning here + return None; + } + data as i64 + } + _ => return None, + }) + } + + /// Try to convert this attribute's value to an offset. + pub fn offset_value(&self) -> Option { + // While offsets will be DW_FORM_data4/8 in DWARF version 2/3, + // these have already been converted to `SecOffset. + if let AttributeValue::SecOffset(offset) = *self { + Some(offset) + } else { + None + } + } + + /// Try to convert this attribute's value to an expression or location buffer. + /// + /// Expressions and locations may be `DW_FORM_block*` or `DW_FORM_exprloc`. + /// The standard doesn't mention `DW_FORM_block*` as a possible form, but + /// it is encountered in practice. + pub fn exprloc_value(&self) -> Option> { + Some(match *self { + AttributeValue::Block(ref data) => Expression(data.clone()), + AttributeValue::Exprloc(ref data) => data.clone(), + _ => return None, + }) + } + + /// Try to return this attribute's value as a string slice. + /// + /// If this attribute's value is either an inline `DW_FORM_string` string, + /// or a `DW_FORM_strp` reference to an offset into the `.debug_str` + /// section, return the attribute's string value as `Some`. Other attribute + /// value forms are returned as `None`. + /// + /// Warning: this function does not handle all possible string forms. + /// Use `Dwarf::attr_string` instead. + pub fn string_value(&self, debug_str: &DebugStr) -> Option { + match *self { + AttributeValue::String(ref string) => Some(string.clone()), + AttributeValue::DebugStrRef(offset) => debug_str.get_str(offset).ok(), + _ => None, + } + } + + /// Try to return this attribute's value as a string slice. + /// + /// If this attribute's value is either an inline `DW_FORM_string` string, + /// or a `DW_FORM_strp` reference to an offset into the `.debug_str` + /// section, or a `DW_FORM_strp_sup` reference to an offset into a supplementary + /// object file, return the attribute's string value as `Some`. Other attribute + /// value forms are returned as `None`. + /// + /// Warning: this function does not handle all possible string forms. + /// Use `Dwarf::attr_string` instead. + pub fn string_value_sup( + &self, + debug_str: &DebugStr, + debug_str_sup: Option<&DebugStr>, + ) -> Option { + match *self { + AttributeValue::String(ref string) => Some(string.clone()), + AttributeValue::DebugStrRef(offset) => debug_str.get_str(offset).ok(), + AttributeValue::DebugStrRefSup(offset) => { + debug_str_sup.and_then(|s| s.get_str(offset).ok()) + } + _ => None, + } + } +} + +fn length_u8_value(input: &mut R) -> Result { + let len = input.read_u8().map(R::Offset::from_u8)?; + input.split(len) +} + +fn length_u16_value(input: &mut R) -> Result { + let len = input.read_u16().map(R::Offset::from_u16)?; + input.split(len) +} + +fn length_u32_value(input: &mut R) -> Result { + let len = input.read_u32().map(R::Offset::from_u32)?; + input.split(len) +} + +fn length_uleb128_value(input: &mut R) -> Result { + let len = input.read_uleb128().and_then(R::Offset::from_u64)?; + input.split(len) +} + +// Return true if the given `name` can be a section offset in DWARF version 2/3. +// This is required to correctly handle relocations. +fn allow_section_offset(name: constants::DwAt, version: u16) -> bool { + match name { + constants::DW_AT_location + | constants::DW_AT_stmt_list + | constants::DW_AT_string_length + | constants::DW_AT_return_addr + | constants::DW_AT_start_scope + | constants::DW_AT_frame_base + | constants::DW_AT_macro_info + | constants::DW_AT_macros + | constants::DW_AT_segment + | constants::DW_AT_static_link + | constants::DW_AT_use_location + | constants::DW_AT_vtable_elem_location + | constants::DW_AT_ranges => true, + constants::DW_AT_data_member_location => version == 2 || version == 3, + _ => false, + } +} + +pub(crate) fn parse_attribute<'unit, R: Reader>( + input: &mut R, + encoding: Encoding, + spec: AttributeSpecification, +) -> Result> { + let mut form = spec.form(); + loop { + let value = match form { + constants::DW_FORM_indirect => { + let dynamic_form = input.read_uleb128_u16()?; + form = constants::DwForm(dynamic_form); + continue; + } + constants::DW_FORM_addr => { + let addr = input.read_address(encoding.address_size)?; + AttributeValue::Addr(addr) + } + constants::DW_FORM_block1 => { + let block = length_u8_value(input)?; + AttributeValue::Block(block) + } + constants::DW_FORM_block2 => { + let block = length_u16_value(input)?; + AttributeValue::Block(block) + } + constants::DW_FORM_block4 => { + let block = length_u32_value(input)?; + AttributeValue::Block(block) + } + constants::DW_FORM_block => { + let block = length_uleb128_value(input)?; + AttributeValue::Block(block) + } + constants::DW_FORM_data1 => { + let data = input.read_u8()?; + AttributeValue::Data1(data) + } + constants::DW_FORM_data2 => { + let data = input.read_u16()?; + AttributeValue::Data2(data) + } + constants::DW_FORM_data4 => { + // DWARF version 2/3 may use DW_FORM_data4/8 for section offsets. + // Ensure we handle relocations here. + if encoding.format == Format::Dwarf32 + && allow_section_offset(spec.name(), encoding.version) + { + let offset = input.read_offset(Format::Dwarf32)?; + AttributeValue::SecOffset(offset) + } else { + let data = input.read_u32()?; + AttributeValue::Data4(data) + } + } + constants::DW_FORM_data8 => { + // DWARF version 2/3 may use DW_FORM_data4/8 for section offsets. + // Ensure we handle relocations here. + if encoding.format == Format::Dwarf64 + && allow_section_offset(spec.name(), encoding.version) + { + let offset = input.read_offset(Format::Dwarf64)?; + AttributeValue::SecOffset(offset) + } else { + let data = input.read_u64()?; + AttributeValue::Data8(data) + } + } + constants::DW_FORM_data16 => { + let block = input.split(R::Offset::from_u8(16))?; + AttributeValue::Block(block) + } + constants::DW_FORM_udata => { + let data = input.read_uleb128()?; + AttributeValue::Udata(data) + } + constants::DW_FORM_sdata => { + let data = input.read_sleb128()?; + AttributeValue::Sdata(data) + } + constants::DW_FORM_exprloc => { + let block = length_uleb128_value(input)?; + AttributeValue::Exprloc(Expression(block)) + } + constants::DW_FORM_flag => { + let present = input.read_u8()?; + AttributeValue::Flag(present != 0) + } + constants::DW_FORM_flag_present => { + // FlagPresent is this weird compile time always true thing that + // isn't actually present in the serialized DIEs, only in the abbreviation. + AttributeValue::Flag(true) + } + constants::DW_FORM_sec_offset => { + let offset = input.read_offset(encoding.format)?; + AttributeValue::SecOffset(offset) + } + constants::DW_FORM_ref1 => { + let reference = input.read_u8().map(R::Offset::from_u8)?; + AttributeValue::UnitRef(UnitOffset(reference)) + } + constants::DW_FORM_ref2 => { + let reference = input.read_u16().map(R::Offset::from_u16)?; + AttributeValue::UnitRef(UnitOffset(reference)) + } + constants::DW_FORM_ref4 => { + let reference = input.read_u32().map(R::Offset::from_u32)?; + AttributeValue::UnitRef(UnitOffset(reference)) + } + constants::DW_FORM_ref8 => { + let reference = input.read_u64().and_then(R::Offset::from_u64)?; + AttributeValue::UnitRef(UnitOffset(reference)) + } + constants::DW_FORM_ref_udata => { + let reference = input.read_uleb128().and_then(R::Offset::from_u64)?; + AttributeValue::UnitRef(UnitOffset(reference)) + } + constants::DW_FORM_ref_addr => { + // This is an offset, but DWARF version 2 specifies that DW_FORM_ref_addr + // has the same size as an address on the target system. This was changed + // in DWARF version 3. + let offset = if encoding.version == 2 { + input.read_sized_offset(encoding.address_size)? + } else { + input.read_offset(encoding.format)? + }; + AttributeValue::DebugInfoRef(DebugInfoOffset(offset)) + } + constants::DW_FORM_ref_sig8 => { + let signature = input.read_u64()?; + AttributeValue::DebugTypesRef(DebugTypeSignature(signature)) + } + constants::DW_FORM_ref_sup4 => { + let offset = input.read_u32().map(R::Offset::from_u32)?; + AttributeValue::DebugInfoRefSup(DebugInfoOffset(offset)) + } + constants::DW_FORM_ref_sup8 => { + let offset = input.read_u64().and_then(R::Offset::from_u64)?; + AttributeValue::DebugInfoRefSup(DebugInfoOffset(offset)) + } + constants::DW_FORM_GNU_ref_alt => { + let offset = input.read_offset(encoding.format)?; + AttributeValue::DebugInfoRefSup(DebugInfoOffset(offset)) + } + constants::DW_FORM_string => { + let string = input.read_null_terminated_slice()?; + AttributeValue::String(string) + } + constants::DW_FORM_strp => { + let offset = input.read_offset(encoding.format)?; + AttributeValue::DebugStrRef(DebugStrOffset(offset)) + } + constants::DW_FORM_strp_sup | constants::DW_FORM_GNU_strp_alt => { + let offset = input.read_offset(encoding.format)?; + AttributeValue::DebugStrRefSup(DebugStrOffset(offset)) + } + constants::DW_FORM_line_strp => { + let offset = input.read_offset(encoding.format)?; + AttributeValue::DebugLineStrRef(DebugLineStrOffset(offset)) + } + constants::DW_FORM_implicit_const => { + let data = spec + .implicit_const_value() + .ok_or(Error::InvalidImplicitConst)?; + AttributeValue::Sdata(data) + } + constants::DW_FORM_strx | constants::DW_FORM_GNU_str_index => { + let index = input.read_uleb128().and_then(R::Offset::from_u64)?; + AttributeValue::DebugStrOffsetsIndex(DebugStrOffsetsIndex(index)) + } + constants::DW_FORM_strx1 => { + let index = input.read_u8().map(R::Offset::from_u8)?; + AttributeValue::DebugStrOffsetsIndex(DebugStrOffsetsIndex(index)) + } + constants::DW_FORM_strx2 => { + let index = input.read_u16().map(R::Offset::from_u16)?; + AttributeValue::DebugStrOffsetsIndex(DebugStrOffsetsIndex(index)) + } + constants::DW_FORM_strx3 => { + let index = input.read_uint(3).and_then(R::Offset::from_u64)?; + AttributeValue::DebugStrOffsetsIndex(DebugStrOffsetsIndex(index)) + } + constants::DW_FORM_strx4 => { + let index = input.read_u32().map(R::Offset::from_u32)?; + AttributeValue::DebugStrOffsetsIndex(DebugStrOffsetsIndex(index)) + } + constants::DW_FORM_addrx | constants::DW_FORM_GNU_addr_index => { + let index = input.read_uleb128().and_then(R::Offset::from_u64)?; + AttributeValue::DebugAddrIndex(DebugAddrIndex(index)) + } + constants::DW_FORM_addrx1 => { + let index = input.read_u8().map(R::Offset::from_u8)?; + AttributeValue::DebugAddrIndex(DebugAddrIndex(index)) + } + constants::DW_FORM_addrx2 => { + let index = input.read_u16().map(R::Offset::from_u16)?; + AttributeValue::DebugAddrIndex(DebugAddrIndex(index)) + } + constants::DW_FORM_addrx3 => { + let index = input.read_uint(3).and_then(R::Offset::from_u64)?; + AttributeValue::DebugAddrIndex(DebugAddrIndex(index)) + } + constants::DW_FORM_addrx4 => { + let index = input.read_u32().map(R::Offset::from_u32)?; + AttributeValue::DebugAddrIndex(DebugAddrIndex(index)) + } + constants::DW_FORM_loclistx => { + let index = input.read_uleb128().and_then(R::Offset::from_u64)?; + AttributeValue::DebugLocListsIndex(DebugLocListsIndex(index)) + } + constants::DW_FORM_rnglistx => { + let index = input.read_uleb128().and_then(R::Offset::from_u64)?; + AttributeValue::DebugRngListsIndex(DebugRngListsIndex(index)) + } + _ => { + return Err(Error::UnknownForm); + } + }; + let attr = Attribute { + name: spec.name(), + value, + }; + return Ok(attr); + } +} + +pub(crate) fn skip_attributes<'unit, R: Reader>( + input: &mut R, + encoding: Encoding, + specs: &[AttributeSpecification], +) -> Result<()> { + let mut skip_bytes = R::Offset::from_u8(0); + for spec in specs { + let mut form = spec.form(); + loop { + if let Some(len) = get_attribute_size(form, encoding) { + // We know the length of this attribute. Accumulate that length. + skip_bytes += R::Offset::from_u8(len); + break; + } + + // We have encountered a variable-length attribute. + if skip_bytes != R::Offset::from_u8(0) { + // Skip the accumulated skip bytes and then read the attribute normally. + input.skip(skip_bytes)?; + skip_bytes = R::Offset::from_u8(0); + } + + match form { + constants::DW_FORM_indirect => { + let dynamic_form = input.read_uleb128_u16()?; + form = constants::DwForm(dynamic_form); + continue; + } + constants::DW_FORM_block1 => { + skip_bytes = input.read_u8().map(R::Offset::from_u8)?; + } + constants::DW_FORM_block2 => { + skip_bytes = input.read_u16().map(R::Offset::from_u16)?; + } + constants::DW_FORM_block4 => { + skip_bytes = input.read_u32().map(R::Offset::from_u32)?; + } + constants::DW_FORM_block | constants::DW_FORM_exprloc => { + skip_bytes = input.read_uleb128().and_then(R::Offset::from_u64)?; + } + constants::DW_FORM_string => { + let _ = input.read_null_terminated_slice()?; + } + constants::DW_FORM_udata + | constants::DW_FORM_sdata + | constants::DW_FORM_ref_udata + | constants::DW_FORM_strx + | constants::DW_FORM_GNU_str_index + | constants::DW_FORM_addrx + | constants::DW_FORM_GNU_addr_index + | constants::DW_FORM_loclistx + | constants::DW_FORM_rnglistx => { + input.skip_leb128()?; + } + _ => { + return Err(Error::UnknownForm); + } + }; + break; + } + } + if skip_bytes != R::Offset::from_u8(0) { + // Skip the remaining accumulated skip bytes. + input.skip(skip_bytes)?; + } + Ok(()) +} + +/// An iterator over a particular entry's attributes. +/// +/// See [the documentation for +/// `DebuggingInformationEntry::attrs()`](./struct.DebuggingInformationEntry.html#method.attrs) +/// for details. +/// +/// Can be [used with +/// `FallibleIterator`](./index.html#using-with-fallibleiterator). +#[derive(Clone, Copy, Debug)] +pub struct AttrsIter<'abbrev, 'entry, 'unit, R: Reader> { + input: R, + attributes: &'abbrev [AttributeSpecification], + entry: &'entry DebuggingInformationEntry<'abbrev, 'unit, R>, +} + +impl<'abbrev, 'entry, 'unit, R: Reader> AttrsIter<'abbrev, 'entry, 'unit, R> { + /// Advance the iterator and return the next attribute. + /// + /// Returns `None` when iteration is finished. If an error + /// occurs while parsing the next attribute, then this error + /// is returned, and all subsequent calls return `None`. + #[allow(clippy::inline_always)] + #[inline(always)] + pub fn next(&mut self) -> Result>> { + if self.attributes.is_empty() { + // Now that we have parsed all of the attributes, we know where + // either (1) this entry's children start, if the abbreviation says + // this entry has children; or (2) where this entry's siblings + // begin. + if let Some(end) = self.entry.attrs_len.get() { + debug_assert_eq!(end, self.input.offset_from(&self.entry.attrs_slice)); + } else { + self.entry + .attrs_len + .set(Some(self.input.offset_from(&self.entry.attrs_slice))); + } + + return Ok(None); + } + + let spec = self.attributes[0]; + let rest_spec = &self.attributes[1..]; + match parse_attribute(&mut self.input, self.entry.unit.encoding(), spec) { + Ok(attr) => { + self.attributes = rest_spec; + Ok(Some(attr)) + } + Err(e) => { + self.input.empty(); + Err(e) + } + } + } +} + +#[cfg(feature = "fallible-iterator")] +impl<'abbrev, 'entry, 'unit, R: Reader> fallible_iterator::FallibleIterator + for AttrsIter<'abbrev, 'entry, 'unit, R> +{ + type Item = Attribute; + type Error = Error; + + fn next(&mut self) -> ::core::result::Result, Self::Error> { + AttrsIter::next(self) + } +} + +/// A raw reader of the data that defines the Debugging Information Entries. +/// +/// `EntriesRaw` provides primitives to read the components of Debugging Information +/// Entries (DIEs). A DIE consists of an abbreviation code (read with `read_abbreviation`) +/// followed by a number of attributes (read with `read_attribute`). +/// The user must provide the control flow to read these correctly. +/// In particular, all attributes must always be read before reading another +/// abbreviation code. +/// +/// `EntriesRaw` lacks some features of `EntriesCursor`, such as the ability to skip +/// to the next sibling DIE. However, this also allows it to optimize better, since it +/// does not need to perform the extra bookkeeping required to support these features, +/// and thus it is suitable for cases where performance is important. +/// +/// ## Example Usage +/// ```rust,no_run +/// # fn example() -> Result<(), gimli::Error> { +/// # let debug_info = gimli::DebugInfo::new(&[], gimli::LittleEndian); +/// # let get_some_unit = || debug_info.units().next().unwrap().unwrap(); +/// let unit = get_some_unit(); +/// # let debug_abbrev = gimli::DebugAbbrev::new(&[], gimli::LittleEndian); +/// # let get_abbrevs_for_unit = |_| unit.abbreviations(&debug_abbrev).unwrap(); +/// let abbrevs = get_abbrevs_for_unit(&unit); +/// +/// let mut entries = unit.entries_raw(&abbrevs, None)?; +/// while !entries.is_empty() { +/// let abbrev = if let Some(abbrev) = entries.read_abbreviation()? { +/// abbrev +/// } else { +/// // Null entry with no attributes. +/// continue +/// }; +/// match abbrev.tag() { +/// gimli::DW_TAG_subprogram => { +/// // Loop over attributes for DIEs we care about. +/// for spec in abbrev.attributes() { +/// let attr = entries.read_attribute(*spec)?; +/// match attr.name() { +/// // Handle attributes. +/// _ => {} +/// } +/// } +/// } +/// _ => { +/// // Skip attributes for DIEs we don't care about. +/// entries.skip_attributes(abbrev.attributes()); +/// } +/// } +/// } +/// # unreachable!() +/// # } +/// ``` +#[derive(Clone, Debug)] +pub struct EntriesRaw<'abbrev, 'unit, R> +where + R: Reader, +{ + input: R, + unit: &'unit UnitHeader, + abbreviations: &'abbrev Abbreviations, + depth: isize, +} + +impl<'abbrev, 'unit, R: Reader> EntriesRaw<'abbrev, 'unit, R> { + /// Return true if there is no more input. + #[inline] + pub fn is_empty(&self) -> bool { + self.input.is_empty() + } + + /// Return the unit offset at which the reader will read next. + /// + /// If you want the offset of the next entry, then this must be called prior to reading + /// the next entry. + pub fn next_offset(&self) -> UnitOffset { + UnitOffset(self.unit.header_size() + self.input.offset_from(&self.unit.entries_buf)) + } + + /// Return the depth of the next entry. + /// + /// This depth is updated when `read_abbreviation` is called, and is updated + /// based on null entries and the `has_children` field in the abbreviation. + #[inline] + pub fn next_depth(&self) -> isize { + self.depth + } + + /// Read an abbreviation code and lookup the corresponding `Abbreviation`. + /// + /// Returns `Ok(None)` for null entries. + #[inline] + pub fn read_abbreviation(&mut self) -> Result> { + let code = self.input.read_uleb128()?; + if code == 0 { + self.depth -= 1; + return Ok(None); + }; + let abbrev = self + .abbreviations + .get(code) + .ok_or(Error::UnknownAbbreviation)?; + if abbrev.has_children() { + self.depth += 1; + } + Ok(Some(abbrev)) + } + + /// Read an attribute. + #[inline] + pub fn read_attribute(&mut self, spec: AttributeSpecification) -> Result> { + parse_attribute(&mut self.input, self.unit.encoding(), spec) + } + + /// Skip all the attributes of an abbreviation. + #[inline] + pub fn skip_attributes(&mut self, specs: &[AttributeSpecification]) -> Result<()> { + skip_attributes(&mut self.input, self.unit.encoding(), specs) + } +} + +/// A cursor into the Debugging Information Entries tree for a compilation unit. +/// +/// The `EntriesCursor` can traverse the DIE tree in DFS order using `next_dfs()`, +/// or skip to the next sibling of the entry the cursor is currently pointing to +/// using `next_sibling()`. +/// +/// It is also possible to traverse the DIE tree at a lower abstraction level +/// using `next_entry()`. This method does not skip over null entries, or provide +/// any indication of the current tree depth. In this case, you must use `current()` +/// to obtain the current entry, and `current().has_children()` to determine if +/// the entry following the current entry will be a sibling or child. `current()` +/// will return `None` if the current entry is a null entry, which signifies the +/// end of the current tree depth. +#[derive(Clone, Debug)] +pub struct EntriesCursor<'abbrev, 'unit, R> +where + R: Reader, +{ + input: R, + unit: &'unit UnitHeader, + abbreviations: &'abbrev Abbreviations, + cached_current: Option>, + delta_depth: isize, +} + +impl<'abbrev, 'unit, R: Reader> EntriesCursor<'abbrev, 'unit, R> { + /// Get a reference to the entry that the cursor is currently pointing to. + /// + /// If the cursor is not pointing at an entry, or if the current entry is a + /// null entry, then `None` is returned. + #[inline] + pub fn current(&self) -> Option<&DebuggingInformationEntry<'abbrev, 'unit, R>> { + self.cached_current.as_ref() + } + + /// Move the cursor to the next DIE in the tree. + /// + /// Returns `Some` if there is a next entry, even if this entry is null. + /// If there is no next entry, then `None` is returned. + pub fn next_entry(&mut self) -> Result> { + if let Some(ref current) = self.cached_current { + self.input = current.after_attrs()?; + } + + if self.input.is_empty() { + self.cached_current = None; + self.delta_depth = 0; + return Ok(None); + } + + match DebuggingInformationEntry::parse(&mut self.input, self.unit, self.abbreviations) { + Ok(Some(entry)) => { + self.delta_depth = entry.has_children() as isize; + self.cached_current = Some(entry); + Ok(Some(())) + } + Ok(None) => { + self.delta_depth = -1; + self.cached_current = None; + Ok(Some(())) + } + Err(e) => { + self.input.empty(); + self.delta_depth = 0; + self.cached_current = None; + Err(e) + } + } + } + + /// Move the cursor to the next DIE in the tree in DFS order. + /// + /// Upon successful movement of the cursor, return the delta traversal + /// depth and the entry: + /// + /// * If we moved down into the previous current entry's children, we get + /// `Some((1, entry))`. + /// + /// * If we moved to the previous current entry's sibling, we get + /// `Some((0, entry))`. + /// + /// * If the previous entry does not have any siblings and we move up to + /// its parent's next sibling, then we get `Some((-1, entry))`. Note that + /// if the parent doesn't have a next sibling, then it could go up to the + /// parent's parent's next sibling and return `Some((-2, entry))`, etc. + /// + /// If there is no next entry, then `None` is returned. + /// + /// Here is an example that finds the first entry in a compilation unit that + /// does not have any children. + /// + /// ``` + /// # use gimli::{DebugAbbrev, DebugInfo, LittleEndian}; + /// # let info_buf = [ + /// # // Comilation unit header + /// # + /// # // 32-bit unit length = 25 + /// # 0x19, 0x00, 0x00, 0x00, + /// # // Version 4 + /// # 0x04, 0x00, + /// # // debug_abbrev_offset + /// # 0x00, 0x00, 0x00, 0x00, + /// # // Address size + /// # 0x04, + /// # + /// # // DIEs + /// # + /// # // Abbreviation code + /// # 0x01, + /// # // Attribute of form DW_FORM_string = "foo\0" + /// # 0x66, 0x6f, 0x6f, 0x00, + /// # + /// # // Children + /// # + /// # // Abbreviation code + /// # 0x01, + /// # // Attribute of form DW_FORM_string = "foo\0" + /// # 0x66, 0x6f, 0x6f, 0x00, + /// # + /// # // Children + /// # + /// # // Abbreviation code + /// # 0x01, + /// # // Attribute of form DW_FORM_string = "foo\0" + /// # 0x66, 0x6f, 0x6f, 0x00, + /// # + /// # // Children + /// # + /// # // End of children + /// # 0x00, + /// # + /// # // End of children + /// # 0x00, + /// # + /// # // End of children + /// # 0x00, + /// # ]; + /// # let debug_info = DebugInfo::new(&info_buf, LittleEndian); + /// # + /// # let abbrev_buf = [ + /// # // Code + /// # 0x01, + /// # // DW_TAG_subprogram + /// # 0x2e, + /// # // DW_CHILDREN_yes + /// # 0x01, + /// # // Begin attributes + /// # // Attribute name = DW_AT_name + /// # 0x03, + /// # // Attribute form = DW_FORM_string + /// # 0x08, + /// # // End attributes + /// # 0x00, + /// # 0x00, + /// # // Null terminator + /// # 0x00 + /// # ]; + /// # let debug_abbrev = DebugAbbrev::new(&abbrev_buf, LittleEndian); + /// # + /// # let get_some_unit = || debug_info.units().next().unwrap().unwrap(); + /// + /// let unit = get_some_unit(); + /// # let get_abbrevs_for_unit = |_| unit.abbreviations(&debug_abbrev).unwrap(); + /// let abbrevs = get_abbrevs_for_unit(&unit); + /// + /// let mut first_entry_with_no_children = None; + /// let mut cursor = unit.entries(&abbrevs); + /// + /// // Move the cursor to the root. + /// assert!(cursor.next_dfs().unwrap().is_some()); + /// + /// // Traverse the DIE tree in depth-first search order. + /// let mut depth = 0; + /// while let Some((delta_depth, current)) = cursor.next_dfs().expect("Should parse next dfs") { + /// // Update depth value, and break out of the loop when we + /// // return to the original starting position. + /// depth += delta_depth; + /// if depth <= 0 { + /// break; + /// } + /// + /// first_entry_with_no_children = Some(current.clone()); + /// } + /// + /// println!("The first entry with no children is {:?}", + /// first_entry_with_no_children.unwrap()); + /// ``` + #[allow(clippy::type_complexity)] + pub fn next_dfs( + &mut self, + ) -> Result)>> { + let mut delta_depth = self.delta_depth; + loop { + // The next entry should be the one we want. + if self.next_entry()?.is_some() { + if let Some(ref entry) = self.cached_current { + return Ok(Some((delta_depth, entry))); + } + + // next_entry() read a null entry. + delta_depth += self.delta_depth; + } else { + return Ok(None); + } + } + } + + /// Move the cursor to the next sibling DIE of the current one. + /// + /// Returns `Ok(Some(entry))` when the cursor has been moved to + /// the next sibling, `Ok(None)` when there is no next sibling. + /// + /// The depth of the cursor is never changed if this method returns `Ok`. + /// Once `Ok(None)` is returned, this method will continue to return + /// `Ok(None)` until either `next_entry` or `next_dfs` is called. + /// + /// Here is an example that iterates over all of the direct children of the + /// root entry: + /// + /// ``` + /// # use gimli::{DebugAbbrev, DebugInfo, LittleEndian}; + /// # let info_buf = [ + /// # // Comilation unit header + /// # + /// # // 32-bit unit length = 25 + /// # 0x19, 0x00, 0x00, 0x00, + /// # // Version 4 + /// # 0x04, 0x00, + /// # // debug_abbrev_offset + /// # 0x00, 0x00, 0x00, 0x00, + /// # // Address size + /// # 0x04, + /// # + /// # // DIEs + /// # + /// # // Abbreviation code + /// # 0x01, + /// # // Attribute of form DW_FORM_string = "foo\0" + /// # 0x66, 0x6f, 0x6f, 0x00, + /// # + /// # // Children + /// # + /// # // Abbreviation code + /// # 0x01, + /// # // Attribute of form DW_FORM_string = "foo\0" + /// # 0x66, 0x6f, 0x6f, 0x00, + /// # + /// # // Children + /// # + /// # // Abbreviation code + /// # 0x01, + /// # // Attribute of form DW_FORM_string = "foo\0" + /// # 0x66, 0x6f, 0x6f, 0x00, + /// # + /// # // Children + /// # + /// # // End of children + /// # 0x00, + /// # + /// # // End of children + /// # 0x00, + /// # + /// # // End of children + /// # 0x00, + /// # ]; + /// # let debug_info = DebugInfo::new(&info_buf, LittleEndian); + /// # + /// # let get_some_unit = || debug_info.units().next().unwrap().unwrap(); + /// + /// # let abbrev_buf = [ + /// # // Code + /// # 0x01, + /// # // DW_TAG_subprogram + /// # 0x2e, + /// # // DW_CHILDREN_yes + /// # 0x01, + /// # // Begin attributes + /// # // Attribute name = DW_AT_name + /// # 0x03, + /// # // Attribute form = DW_FORM_string + /// # 0x08, + /// # // End attributes + /// # 0x00, + /// # 0x00, + /// # // Null terminator + /// # 0x00 + /// # ]; + /// # let debug_abbrev = DebugAbbrev::new(&abbrev_buf, LittleEndian); + /// # + /// let unit = get_some_unit(); + /// # let get_abbrevs_for_unit = |_| unit.abbreviations(&debug_abbrev).unwrap(); + /// let abbrevs = get_abbrevs_for_unit(&unit); + /// + /// let mut cursor = unit.entries(&abbrevs); + /// + /// // Move the cursor to the root. + /// assert!(cursor.next_dfs().unwrap().is_some()); + /// + /// // Move the cursor to the root's first child. + /// assert!(cursor.next_dfs().unwrap().is_some()); + /// + /// // Iterate the root's children. + /// loop { + /// { + /// let current = cursor.current().expect("Should be at an entry"); + /// println!("{:?} is a child of the root", current); + /// } + /// + /// if cursor.next_sibling().expect("Should parse next sibling").is_none() { + /// break; + /// } + /// } + /// ``` + pub fn next_sibling( + &mut self, + ) -> Result>> { + if self.current().is_none() { + // We're already at the null for the end of the sibling list. + return Ok(None); + } + + // Loop until we find an entry at the current level. + let mut depth = 0; + loop { + // Use is_some() and unwrap() to keep borrow checker happy. + if self.current().is_some() && self.current().unwrap().has_children() { + if let Some(sibling_input) = self.current().unwrap().sibling() { + // Fast path: this entry has a DW_AT_sibling + // attribute pointing to its sibling, so jump + // to it (which keeps us at the same depth). + self.input = sibling_input; + self.cached_current = None; + } else { + // This entry has children, so the next entry is + // down one level. + depth += 1; + } + } + + if self.next_entry()?.is_none() { + // End of input. + return Ok(None); + } + + if depth == 0 { + // Found an entry at the current level. + return Ok(self.current()); + } + + if self.current().is_none() { + // A null entry means the end of a child list, so we're + // back up a level. + depth -= 1; + } + } + } +} + +/// The state information for a tree view of the Debugging Information Entries. +/// +/// The `EntriesTree` can be used to recursively iterate through the DIE +/// tree, following the parent/child relationships. The `EntriesTree` contains +/// shared state for all nodes in the tree, avoiding any duplicate parsing of +/// entries during the traversal. +/// +/// ## Example Usage +/// ```rust,no_run +/// # fn example() -> Result<(), gimli::Error> { +/// # let debug_info = gimli::DebugInfo::new(&[], gimli::LittleEndian); +/// # let get_some_unit = || debug_info.units().next().unwrap().unwrap(); +/// let unit = get_some_unit(); +/// # let debug_abbrev = gimli::DebugAbbrev::new(&[], gimli::LittleEndian); +/// # let get_abbrevs_for_unit = |_| unit.abbreviations(&debug_abbrev).unwrap(); +/// let abbrevs = get_abbrevs_for_unit(&unit); +/// +/// let mut tree = unit.entries_tree(&abbrevs, None)?; +/// let root = tree.root()?; +/// process_tree(root)?; +/// # unreachable!() +/// # } +/// +/// fn process_tree(mut node: gimli::EntriesTreeNode) -> gimli::Result<()> +/// where R: gimli::Reader +/// { +/// { +/// // Examine the entry attributes. +/// let mut attrs = node.entry().attrs(); +/// while let Some(attr) = attrs.next()? { +/// } +/// } +/// let mut children = node.children(); +/// while let Some(child) = children.next()? { +/// // Recursively process a child. +/// process_tree(child); +/// } +/// Ok(()) +/// } +/// ``` +#[derive(Clone, Debug)] +pub struct EntriesTree<'abbrev, 'unit, R> +where + R: Reader, +{ + root: R, + unit: &'unit UnitHeader, + abbreviations: &'abbrev Abbreviations, + input: R, + entry: Option>, + depth: isize, +} + +impl<'abbrev, 'unit, R: Reader> EntriesTree<'abbrev, 'unit, R> { + fn new(root: R, unit: &'unit UnitHeader, abbreviations: &'abbrev Abbreviations) -> Self { + let input = root.clone(); + EntriesTree { + root, + unit, + abbreviations, + input, + entry: None, + depth: 0, + } + } + + /// Returns the root node of the tree. + pub fn root<'me>(&'me mut self) -> Result> { + self.input = self.root.clone(); + self.entry = + DebuggingInformationEntry::parse(&mut self.input, self.unit, self.abbreviations)?; + if self.entry.is_none() { + return Err(Error::UnexpectedNull); + } + self.depth = 0; + Ok(EntriesTreeNode::new(self, 1)) + } + + /// Move the cursor to the next entry at the specified depth. + /// + /// Requires `depth <= self.depth + 1`. + /// + /// Returns `true` if successful. + fn next(&mut self, depth: isize) -> Result { + if self.depth < depth { + debug_assert_eq!(self.depth + 1, depth); + + match self.entry { + Some(ref entry) => { + if !entry.has_children() { + return Ok(false); + } + self.depth += 1; + self.input = entry.after_attrs()?; + } + None => return Ok(false), + } + + if self.input.is_empty() { + self.entry = None; + return Ok(false); + } + + return match DebuggingInformationEntry::parse( + &mut self.input, + self.unit, + self.abbreviations, + ) { + Ok(entry) => { + self.entry = entry; + Ok(self.entry.is_some()) + } + Err(e) => { + self.input.empty(); + self.entry = None; + Err(e) + } + }; + } + + loop { + match self.entry { + Some(ref entry) => { + if entry.has_children() { + if let Some(sibling_input) = entry.sibling() { + // Fast path: this entry has a DW_AT_sibling + // attribute pointing to its sibling, so jump + // to it (which keeps us at the same depth). + self.input = sibling_input; + } else { + // This entry has children, so the next entry is + // down one level. + self.depth += 1; + self.input = entry.after_attrs()?; + } + } else { + // This entry has no children, so next entry is at same depth. + self.input = entry.after_attrs()?; + } + } + None => { + // This entry is a null, so next entry is up one level. + self.depth -= 1; + } + } + + if self.input.is_empty() { + self.entry = None; + return Ok(false); + } + + match DebuggingInformationEntry::parse(&mut self.input, self.unit, self.abbreviations) { + Ok(entry) => { + self.entry = entry; + if self.depth == depth { + return Ok(self.entry.is_some()); + } + } + Err(e) => { + self.input.empty(); + self.entry = None; + return Err(e); + } + } + } + } +} + +/// A node in the Debugging Information Entry tree. +/// +/// The root node of a tree can be obtained +/// via [`EntriesTree::root`](./struct.EntriesTree.html#method.root). +#[derive(Debug)] +pub struct EntriesTreeNode<'abbrev, 'unit, 'tree, R: Reader> { + tree: &'tree mut EntriesTree<'abbrev, 'unit, R>, + depth: isize, +} + +impl<'abbrev, 'unit, 'tree, R: Reader> EntriesTreeNode<'abbrev, 'unit, 'tree, R> { + fn new( + tree: &'tree mut EntriesTree<'abbrev, 'unit, R>, + depth: isize, + ) -> EntriesTreeNode<'abbrev, 'unit, 'tree, R> { + debug_assert!(tree.entry.is_some()); + EntriesTreeNode { tree, depth } + } + + /// Returns the current entry in the tree. + pub fn entry(&self) -> &DebuggingInformationEntry<'abbrev, 'unit, R> { + // We never create a node without an entry. + self.tree.entry.as_ref().unwrap() + } + + /// Create an iterator for the children of the current entry. + /// + /// The current entry can no longer be accessed after creating the + /// iterator. + pub fn children(self) -> EntriesTreeIter<'abbrev, 'unit, 'tree, R> { + EntriesTreeIter::new(self.tree, self.depth) + } +} + +/// An iterator that allows traversal of the children of an +/// `EntriesTreeNode`. +/// +/// The items returned by this iterator are also `EntriesTreeNode`s, +/// which allow recursive traversal of grandchildren, etc. +#[derive(Debug)] +pub struct EntriesTreeIter<'abbrev, 'unit, 'tree, R: Reader> { + tree: &'tree mut EntriesTree<'abbrev, 'unit, R>, + depth: isize, + empty: bool, +} + +impl<'abbrev, 'unit, 'tree, R: Reader> EntriesTreeIter<'abbrev, 'unit, 'tree, R> { + fn new( + tree: &'tree mut EntriesTree<'abbrev, 'unit, R>, + depth: isize, + ) -> EntriesTreeIter<'abbrev, 'unit, 'tree, R> { + EntriesTreeIter { + tree, + depth, + empty: false, + } + } + + /// Returns an `EntriesTreeNode` for the next child entry. + /// + /// Returns `None` if there are no more children. + pub fn next<'me>(&'me mut self) -> Result>> { + if self.empty { + Ok(None) + } else if self.tree.next(self.depth)? { + Ok(Some(EntriesTreeNode::new(self.tree, self.depth + 1))) + } else { + self.empty = true; + Ok(None) + } + } +} + +/// Parse a type unit header's unique type signature. Callers should handle +/// unique-ness checking. +fn parse_type_signature(input: &mut R) -> Result { + input.read_u64().map(DebugTypeSignature) +} + +/// Parse a type unit header's type offset. +fn parse_type_offset(input: &mut R, format: Format) -> Result> { + input.read_offset(format).map(UnitOffset) +} + +/// The `DebugTypes` struct represents the DWARF type information +/// found in the `.debug_types` section. +#[derive(Debug, Default, Clone, Copy)] +pub struct DebugTypes { + debug_types_section: R, +} + +impl<'input, Endian> DebugTypes> +where + Endian: Endianity, +{ + /// Construct a new `DebugTypes` instance from the data in the `.debug_types` + /// section. + /// + /// It is the caller's responsibility to read the `.debug_types` section and + /// present it as a `&[u8]` slice. That means using some ELF loader on + /// Linux, a Mach-O loader on macOS, etc. + /// + /// ``` + /// use gimli::{DebugTypes, LittleEndian}; + /// + /// # let buf = [0x00, 0x01, 0x02, 0x03]; + /// # let read_debug_types_section_somehow = || &buf; + /// let debug_types = DebugTypes::new(read_debug_types_section_somehow(), LittleEndian); + /// ``` + pub fn new(debug_types_section: &'input [u8], endian: Endian) -> Self { + Self::from(EndianSlice::new(debug_types_section, endian)) + } +} + +impl DebugTypes { + /// Create a `DebugTypes` section that references the data in `self`. + /// + /// This is useful when `R` implements `Reader` but `T` does not. + /// + /// ## Example Usage + /// + /// ```rust,no_run + /// # let load_section = || unimplemented!(); + /// // Read the DWARF section into a `Vec` with whatever object loader you're using. + /// let owned_section: gimli::DebugTypes> = load_section(); + /// // Create a reference to the DWARF section. + /// let section = owned_section.borrow(|section| { + /// gimli::EndianSlice::new(§ion, gimli::LittleEndian) + /// }); + /// ``` + pub fn borrow<'a, F, R>(&'a self, mut borrow: F) -> DebugTypes + where + F: FnMut(&'a T) -> R, + { + borrow(&self.debug_types_section).into() + } +} + +impl Section for DebugTypes { + fn id() -> SectionId { + SectionId::DebugTypes + } + + fn reader(&self) -> &R { + &self.debug_types_section + } +} + +impl From for DebugTypes { + fn from(debug_types_section: R) -> Self { + DebugTypes { + debug_types_section, + } + } +} + +impl DebugTypes { + /// Iterate the type-units in this `.debug_types` section. + /// + /// ``` + /// use gimli::{DebugTypes, LittleEndian}; + /// + /// # let buf = []; + /// # let read_debug_types_section_somehow = || &buf; + /// let debug_types = DebugTypes::new(read_debug_types_section_somehow(), LittleEndian); + /// + /// let mut iter = debug_types.units(); + /// while let Some(unit) = iter.next().unwrap() { + /// println!("unit's length is {}", unit.unit_length()); + /// } + /// ``` + /// + /// Can be [used with + /// `FallibleIterator`](./index.html#using-with-fallibleiterator). + pub fn units(&self) -> DebugTypesUnitHeadersIter { + DebugTypesUnitHeadersIter { + input: self.debug_types_section.clone(), + offset: DebugTypesOffset(R::Offset::from_u8(0)), + } + } +} + +/// An iterator over the type-units of this `.debug_types` section. +/// +/// See the [documentation on +/// `DebugTypes::units`](./struct.DebugTypes.html#method.units) for +/// more detail. +#[derive(Clone, Debug)] +pub struct DebugTypesUnitHeadersIter { + input: R, + offset: DebugTypesOffset, +} + +impl DebugTypesUnitHeadersIter { + /// Advance the iterator to the next type unit header. + pub fn next(&mut self) -> Result>> { + if self.input.is_empty() { + Ok(None) + } else { + let len = self.input.len(); + match parse_unit_header(&mut self.input, self.offset.into()) { + Ok(header) => { + self.offset.0 += len - self.input.len(); + Ok(Some(header)) + } + Err(e) => { + self.input.empty(); + Err(e) + } + } + } + } +} + +#[cfg(feature = "fallible-iterator")] +impl fallible_iterator::FallibleIterator for DebugTypesUnitHeadersIter { + type Item = UnitHeader; + type Error = Error; + + fn next(&mut self) -> ::core::result::Result, Self::Error> { + DebugTypesUnitHeadersIter::next(self) + } +} + +#[cfg(test)] +// Tests require leb128::write. +#[cfg(feature = "write")] +mod tests { + use super::*; + use crate::constants; + use crate::constants::*; + use crate::endianity::{Endianity, LittleEndian}; + use crate::leb128; + use crate::read::abbrev::tests::AbbrevSectionMethods; + use crate::read::{ + Abbreviation, AttributeSpecification, DebugAbbrev, EndianSlice, Error, Result, + }; + use crate::test_util::GimliSectionMethods; + use alloc::vec::Vec; + use core::cell::Cell; + use test_assembler::{Endian, Label, LabelMaker, Section}; + + // Mixin methods for `Section` to help define binary test data. + + trait UnitSectionMethods { + fn unit<'input, E>(self, unit: &mut UnitHeader>) -> Self + where + E: Endianity; + fn die(self, code: u64, attr: F) -> Self + where + F: Fn(Section) -> Section; + fn die_null(self) -> Self; + fn attr_string(self, s: &str) -> Self; + fn attr_ref1(self, o: u8) -> Self; + fn offset(self, offset: usize, format: Format) -> Self; + } + + impl UnitSectionMethods for Section { + fn unit<'input, E>(self, unit: &mut UnitHeader>) -> Self + where + E: Endianity, + { + let size = self.size(); + let length = Label::new(); + let start = Label::new(); + let end = Label::new(); + + let section = match unit.format() { + Format::Dwarf32 => self.L32(&length), + Format::Dwarf64 => self.L32(0xffff_ffff).L64(&length), + }; + + let section = match unit.version() { + 2 | 3 | 4 => section + .mark(&start) + .L16(unit.version()) + .offset(unit.debug_abbrev_offset.0, unit.format()) + .D8(unit.address_size()), + 5 => section + .mark(&start) + .L16(unit.version()) + .D8(unit.type_().dw_ut().0) + .D8(unit.address_size()) + .offset(unit.debug_abbrev_offset.0, unit.format()), + _ => unreachable!(), + }; + + let section = match unit.type_() { + UnitType::Compilation | UnitType::Partial => { + unit.unit_offset = DebugInfoOffset(size as usize).into(); + section + } + UnitType::Type { + type_signature, + type_offset, + } + | UnitType::SplitType { + type_signature, + type_offset, + } => { + if unit.version() == 5 { + unit.unit_offset = DebugInfoOffset(size as usize).into(); + } else { + unit.unit_offset = DebugTypesOffset(size as usize).into(); + } + section + .L64(type_signature.0) + .offset(type_offset.0, unit.format()) + } + UnitType::Skeleton(dwo_id) | UnitType::SplitCompilation(dwo_id) => { + unit.unit_offset = DebugInfoOffset(size as usize).into(); + section.L64(dwo_id.0) + } + }; + + let section = section.append_bytes(unit.entries_buf.into()).mark(&end); + + unit.unit_length = (&end - &start) as usize; + length.set_const(unit.unit_length as u64); + + section + } + + fn die(self, code: u64, attr: F) -> Self + where + F: Fn(Section) -> Section, + { + let section = self.uleb(code); + attr(section) + } + + fn die_null(self) -> Self { + self.D8(0) + } + + fn attr_string(self, attr: &str) -> Self { + self.append_bytes(attr.as_bytes()).D8(0) + } + + fn attr_ref1(self, attr: u8) -> Self { + self.D8(attr) + } + + fn offset(self, offset: usize, format: Format) -> Self { + match format { + Format::Dwarf32 => self.L32(offset as u32), + Format::Dwarf64 => self.L64(offset as u64), + } + } + } + + /// Ensure that `UnitHeader` is covariant wrt R. + #[test] + fn test_unit_header_variance() { + /// This only needs to compile. + fn _f<'a: 'b, 'b, E: Endianity>( + x: UnitHeader>, + ) -> UnitHeader> { + x + } + } + + #[test] + fn test_parse_debug_abbrev_offset_32() { + let section = Section::with_endian(Endian::Little).L32(0x0403_0201); + let buf = section.get_contents().unwrap(); + let buf = &mut EndianSlice::new(&buf, LittleEndian); + + match parse_debug_abbrev_offset(buf, Format::Dwarf32) { + Ok(val) => assert_eq!(val, DebugAbbrevOffset(0x0403_0201)), + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + fn test_parse_debug_abbrev_offset_32_incomplete() { + let buf = [0x01, 0x02]; + let buf = &mut EndianSlice::new(&buf, LittleEndian); + + match parse_debug_abbrev_offset(buf, Format::Dwarf32) { + Err(Error::UnexpectedEof(_)) => assert!(true), + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_parse_debug_abbrev_offset_64() { + let section = Section::with_endian(Endian::Little).L64(0x0807_0605_0403_0201); + let buf = section.get_contents().unwrap(); + let buf = &mut EndianSlice::new(&buf, LittleEndian); + + match parse_debug_abbrev_offset(buf, Format::Dwarf64) { + Ok(val) => assert_eq!(val, DebugAbbrevOffset(0x0807_0605_0403_0201)), + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + fn test_parse_debug_abbrev_offset_64_incomplete() { + let buf = [0x01, 0x02]; + let buf = &mut EndianSlice::new(&buf, LittleEndian); + + match parse_debug_abbrev_offset(buf, Format::Dwarf64) { + Err(Error::UnexpectedEof(_)) => assert!(true), + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + fn test_parse_debug_info_offset_32() { + let section = Section::with_endian(Endian::Little).L32(0x0403_0201); + let buf = section.get_contents().unwrap(); + let buf = &mut EndianSlice::new(&buf, LittleEndian); + + match parse_debug_info_offset(buf, Format::Dwarf32) { + Ok(val) => assert_eq!(val, DebugInfoOffset(0x0403_0201)), + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + fn test_parse_debug_info_offset_32_incomplete() { + let buf = [0x01, 0x02]; + let buf = &mut EndianSlice::new(&buf, LittleEndian); + + match parse_debug_info_offset(buf, Format::Dwarf32) { + Err(Error::UnexpectedEof(_)) => assert!(true), + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_parse_debug_info_offset_64() { + let section = Section::with_endian(Endian::Little).L64(0x0807_0605_0403_0201); + let buf = section.get_contents().unwrap(); + let buf = &mut EndianSlice::new(&buf, LittleEndian); + + match parse_debug_info_offset(buf, Format::Dwarf64) { + Ok(val) => assert_eq!(val, DebugInfoOffset(0x0807_0605_0403_0201)), + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + fn test_parse_debug_info_offset_64_incomplete() { + let buf = [0x01, 0x02]; + let buf = &mut EndianSlice::new(&buf, LittleEndian); + + match parse_debug_info_offset(buf, Format::Dwarf64) { + Err(Error::UnexpectedEof(_)) => assert!(true), + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_units() { + let expected_rest = &[1, 2, 3, 4, 5, 6, 7, 8, 9]; + let mut unit64 = UnitHeader { + encoding: Encoding { + format: Format::Dwarf64, + version: 4, + address_size: 8, + }, + unit_length: 0, + unit_type: UnitType::Compilation, + debug_abbrev_offset: DebugAbbrevOffset(0x0102_0304_0506_0708), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(expected_rest, LittleEndian), + }; + let mut unit32 = UnitHeader { + encoding: Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 4, + }, + unit_length: 0, + unit_type: UnitType::Compilation, + debug_abbrev_offset: DebugAbbrevOffset(0x0807_0605), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(expected_rest, LittleEndian), + }; + let section = Section::with_endian(Endian::Little) + .unit(&mut unit64) + .unit(&mut unit32); + let buf = section.get_contents().unwrap(); + + let debug_info = DebugInfo::new(&buf, LittleEndian); + let mut units = debug_info.units(); + + assert_eq!(units.next(), Ok(Some(unit64))); + assert_eq!(units.next(), Ok(Some(unit32))); + assert_eq!(units.next(), Ok(None)); + } + + #[test] + fn test_unit_version_unknown_version() { + let buf = [0x02, 0x00, 0x00, 0x00, 0xab, 0xcd]; + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + match parse_unit_header(rest, DebugInfoOffset(0).into()) { + Err(Error::UnknownVersion(0xcdab)) => assert!(true), + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + + let buf = [0x02, 0x00, 0x00, 0x00, 0x1, 0x0]; + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + match parse_unit_header(rest, DebugInfoOffset(0).into()) { + Err(Error::UnknownVersion(1)) => assert!(true), + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + fn test_unit_version_incomplete() { + let buf = [0x01, 0x00, 0x00, 0x00, 0x04]; + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + match parse_unit_header(rest, DebugInfoOffset(0).into()) { + Err(Error::UnexpectedEof(_)) => assert!(true), + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + fn test_parse_unit_header_32_ok() { + let expected_rest = &[1, 2, 3, 4, 5, 6, 7, 8, 9]; + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 4, + }; + let mut expected_unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::Compilation, + debug_abbrev_offset: DebugAbbrevOffset(0x0807_0605), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(expected_rest, LittleEndian), + }; + let section = Section::with_endian(Endian::Little) + .unit(&mut expected_unit) + .append_bytes(expected_rest); + let buf = section.get_contents().unwrap(); + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + assert_eq!( + parse_unit_header(rest, DebugInfoOffset(0).into()), + Ok(expected_unit) + ); + assert_eq!(*rest, EndianSlice::new(expected_rest, LittleEndian)); + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_parse_unit_header_64_ok() { + let expected_rest = &[1, 2, 3, 4, 5, 6, 7, 8, 9]; + let encoding = Encoding { + format: Format::Dwarf64, + version: 4, + address_size: 8, + }; + let mut expected_unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::Compilation, + debug_abbrev_offset: DebugAbbrevOffset(0x0102_0304_0506_0708), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(expected_rest, LittleEndian), + }; + let section = Section::with_endian(Endian::Little) + .unit(&mut expected_unit) + .append_bytes(expected_rest); + let buf = section.get_contents().unwrap(); + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + assert_eq!( + parse_unit_header(rest, DebugInfoOffset(0).into()), + Ok(expected_unit) + ); + assert_eq!(*rest, EndianSlice::new(expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_v5_unit_header_32_ok() { + let expected_rest = &[1, 2, 3, 4, 5, 6, 7, 8, 9]; + let encoding = Encoding { + format: Format::Dwarf32, + version: 5, + address_size: 4, + }; + let mut expected_unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::Compilation, + debug_abbrev_offset: DebugAbbrevOffset(0x0807_0605), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(expected_rest, LittleEndian), + }; + let section = Section::with_endian(Endian::Little) + .unit(&mut expected_unit) + .append_bytes(expected_rest); + let buf = section.get_contents().unwrap(); + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + assert_eq!( + parse_unit_header(rest, DebugInfoOffset(0).into()), + Ok(expected_unit) + ); + assert_eq!(*rest, EndianSlice::new(expected_rest, LittleEndian)); + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_parse_v5_unit_header_64_ok() { + let expected_rest = &[1, 2, 3, 4, 5, 6, 7, 8, 9]; + let encoding = Encoding { + format: Format::Dwarf64, + version: 5, + address_size: 8, + }; + let mut expected_unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::Compilation, + debug_abbrev_offset: DebugAbbrevOffset(0x0102_0304_0506_0708), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(expected_rest, LittleEndian), + }; + let section = Section::with_endian(Endian::Little) + .unit(&mut expected_unit) + .append_bytes(expected_rest); + let buf = section.get_contents().unwrap(); + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + assert_eq!( + parse_unit_header(rest, DebugInfoOffset(0).into()), + Ok(expected_unit) + ); + assert_eq!(*rest, EndianSlice::new(expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_v5_partial_unit_header_32_ok() { + let expected_rest = &[1, 2, 3, 4, 5, 6, 7, 8, 9]; + let encoding = Encoding { + format: Format::Dwarf32, + version: 5, + address_size: 4, + }; + let mut expected_unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::Partial, + debug_abbrev_offset: DebugAbbrevOffset(0x0807_0605), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(expected_rest, LittleEndian), + }; + let section = Section::with_endian(Endian::Little) + .unit(&mut expected_unit) + .append_bytes(expected_rest); + let buf = section.get_contents().unwrap(); + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + assert_eq!( + parse_unit_header(rest, DebugInfoOffset(0).into()), + Ok(expected_unit) + ); + assert_eq!(*rest, EndianSlice::new(expected_rest, LittleEndian)); + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_parse_v5_partial_unit_header_64_ok() { + let expected_rest = &[1, 2, 3, 4, 5, 6, 7, 8, 9]; + let encoding = Encoding { + format: Format::Dwarf64, + version: 5, + address_size: 8, + }; + let mut expected_unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::Partial, + debug_abbrev_offset: DebugAbbrevOffset(0x0102_0304_0506_0708), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(expected_rest, LittleEndian), + }; + let section = Section::with_endian(Endian::Little) + .unit(&mut expected_unit) + .append_bytes(expected_rest); + let buf = section.get_contents().unwrap(); + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + assert_eq!( + parse_unit_header(rest, DebugInfoOffset(0).into()), + Ok(expected_unit) + ); + assert_eq!(*rest, EndianSlice::new(expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_v5_skeleton_unit_header_32_ok() { + let expected_rest = &[1, 2, 3, 4, 5, 6, 7, 8, 9]; + let encoding = Encoding { + format: Format::Dwarf32, + version: 5, + address_size: 4, + }; + let mut expected_unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::Skeleton(DwoId(0x0706_5040_0302_1000)), + debug_abbrev_offset: DebugAbbrevOffset(0x0807_0605), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(expected_rest, LittleEndian), + }; + let section = Section::with_endian(Endian::Little) + .unit(&mut expected_unit) + .append_bytes(expected_rest); + let buf = section.get_contents().unwrap(); + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + assert_eq!( + parse_unit_header(rest, DebugInfoOffset(0).into()), + Ok(expected_unit) + ); + assert_eq!(*rest, EndianSlice::new(expected_rest, LittleEndian)); + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_parse_v5_skeleton_unit_header_64_ok() { + let expected_rest = &[1, 2, 3, 4, 5, 6, 7, 8, 9]; + let encoding = Encoding { + format: Format::Dwarf64, + version: 5, + address_size: 8, + }; + let mut expected_unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::Skeleton(DwoId(0x0706_5040_0302_1000)), + debug_abbrev_offset: DebugAbbrevOffset(0x0102_0304_0506_0708), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(expected_rest, LittleEndian), + }; + let section = Section::with_endian(Endian::Little) + .unit(&mut expected_unit) + .append_bytes(expected_rest); + let buf = section.get_contents().unwrap(); + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + assert_eq!( + parse_unit_header(rest, DebugInfoOffset(0).into()), + Ok(expected_unit) + ); + assert_eq!(*rest, EndianSlice::new(expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_v5_split_compilation_unit_header_32_ok() { + let expected_rest = &[1, 2, 3, 4, 5, 6, 7, 8, 9]; + let encoding = Encoding { + format: Format::Dwarf32, + version: 5, + address_size: 4, + }; + let mut expected_unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::SplitCompilation(DwoId(0x0706_5040_0302_1000)), + debug_abbrev_offset: DebugAbbrevOffset(0x0807_0605), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(expected_rest, LittleEndian), + }; + let section = Section::with_endian(Endian::Little) + .unit(&mut expected_unit) + .append_bytes(expected_rest); + let buf = section.get_contents().unwrap(); + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + assert_eq!( + parse_unit_header(rest, DebugInfoOffset(0).into()), + Ok(expected_unit) + ); + assert_eq!(*rest, EndianSlice::new(expected_rest, LittleEndian)); + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_parse_v5_split_compilation_unit_header_64_ok() { + let expected_rest = &[1, 2, 3, 4, 5, 6, 7, 8, 9]; + let encoding = Encoding { + format: Format::Dwarf64, + version: 5, + address_size: 8, + }; + let mut expected_unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::SplitCompilation(DwoId(0x0706_5040_0302_1000)), + debug_abbrev_offset: DebugAbbrevOffset(0x0102_0304_0506_0708), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(expected_rest, LittleEndian), + }; + let section = Section::with_endian(Endian::Little) + .unit(&mut expected_unit) + .append_bytes(expected_rest); + let buf = section.get_contents().unwrap(); + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + assert_eq!( + parse_unit_header(rest, DebugInfoOffset(0).into()), + Ok(expected_unit) + ); + assert_eq!(*rest, EndianSlice::new(expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_type_offset_32_ok() { + let buf = [0x12, 0x34, 0x56, 0x78, 0x00]; + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + match parse_type_offset(rest, Format::Dwarf32) { + Ok(offset) => { + assert_eq!(rest.len(), 1); + assert_eq!(UnitOffset(0x7856_3412), offset); + } + otherwise => panic!("Unexpected result: {:?}", otherwise), + } + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_parse_type_offset_64_ok() { + let buf = [0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xff, 0x00]; + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + match parse_type_offset(rest, Format::Dwarf64) { + Ok(offset) => { + assert_eq!(rest.len(), 1); + assert_eq!(UnitOffset(0xffde_bc9a_7856_3412), offset); + } + otherwise => panic!("Unexpected result: {:?}", otherwise), + } + } + + #[test] + fn test_parse_type_offset_incomplete() { + // Need at least 4 bytes. + let buf = [0xff, 0xff, 0xff]; + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + match parse_type_offset(rest, Format::Dwarf32) { + Err(Error::UnexpectedEof(_)) => assert!(true), + otherwise => panic!("Unexpected result: {:?}", otherwise), + }; + } + + #[test] + fn test_parse_type_unit_header_32_ok() { + let expected_rest = &[1, 2, 3, 4, 5, 6, 7, 8, 9]; + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 8, + }; + let mut expected_unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::Type { + type_signature: DebugTypeSignature(0xdead_beef_dead_beef), + type_offset: UnitOffset(0x7856_3412), + }, + debug_abbrev_offset: DebugAbbrevOffset(0x0807_0605), + unit_offset: DebugTypesOffset(0).into(), + entries_buf: EndianSlice::new(expected_rest, LittleEndian), + }; + let section = Section::with_endian(Endian::Little) + .unit(&mut expected_unit) + .append_bytes(expected_rest); + let buf = section.get_contents().unwrap(); + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + assert_eq!( + parse_unit_header(rest, DebugTypesOffset(0).into()), + Ok(expected_unit) + ); + assert_eq!(*rest, EndianSlice::new(expected_rest, LittleEndian)); + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_parse_type_unit_header_64_ok() { + let expected_rest = &[1, 2, 3, 4, 5, 6, 7, 8, 9]; + let encoding = Encoding { + format: Format::Dwarf64, + version: 4, + address_size: 8, + }; + let mut expected_unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::Type { + type_signature: DebugTypeSignature(0xdead_beef_dead_beef), + type_offset: UnitOffset(0x7856_3412_7856_3412), + }, + debug_abbrev_offset: DebugAbbrevOffset(0x0807_0605), + unit_offset: DebugTypesOffset(0).into(), + entries_buf: EndianSlice::new(expected_rest, LittleEndian), + }; + let section = Section::with_endian(Endian::Little) + .unit(&mut expected_unit) + .append_bytes(expected_rest); + let buf = section.get_contents().unwrap(); + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + assert_eq!( + parse_unit_header(rest, DebugTypesOffset(0).into()), + Ok(expected_unit) + ); + assert_eq!(*rest, EndianSlice::new(expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_v5_type_unit_header_32_ok() { + let expected_rest = &[1, 2, 3, 4, 5, 6, 7, 8, 9]; + let encoding = Encoding { + format: Format::Dwarf32, + version: 5, + address_size: 8, + }; + let mut expected_unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::Type { + type_signature: DebugTypeSignature(0xdead_beef_dead_beef), + type_offset: UnitOffset(0x7856_3412), + }, + debug_abbrev_offset: DebugAbbrevOffset(0x0807_0605), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(expected_rest, LittleEndian), + }; + let section = Section::with_endian(Endian::Little) + .unit(&mut expected_unit) + .append_bytes(expected_rest); + let buf = section.get_contents().unwrap(); + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + assert_eq!( + parse_unit_header(rest, DebugInfoOffset(0).into()), + Ok(expected_unit) + ); + assert_eq!(*rest, EndianSlice::new(expected_rest, LittleEndian)); + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_parse_v5_type_unit_header_64_ok() { + let expected_rest = &[1, 2, 3, 4, 5, 6, 7, 8, 9]; + let encoding = Encoding { + format: Format::Dwarf64, + version: 5, + address_size: 8, + }; + let mut expected_unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::Type { + type_signature: DebugTypeSignature(0xdead_beef_dead_beef), + type_offset: UnitOffset(0x7856_3412_7856_3412), + }, + debug_abbrev_offset: DebugAbbrevOffset(0x0807_0605), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(expected_rest, LittleEndian), + }; + let section = Section::with_endian(Endian::Little) + .unit(&mut expected_unit) + .append_bytes(expected_rest); + let buf = section.get_contents().unwrap(); + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + assert_eq!( + parse_unit_header(rest, DebugInfoOffset(0).into()), + Ok(expected_unit) + ); + assert_eq!(*rest, EndianSlice::new(expected_rest, LittleEndian)); + } + + #[test] + fn test_parse_v5_split_type_unit_header_32_ok() { + let expected_rest = &[1, 2, 3, 4, 5, 6, 7, 8, 9]; + let encoding = Encoding { + format: Format::Dwarf32, + version: 5, + address_size: 8, + }; + let mut expected_unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::SplitType { + type_signature: DebugTypeSignature(0xdead_beef_dead_beef), + type_offset: UnitOffset(0x7856_3412), + }, + debug_abbrev_offset: DebugAbbrevOffset(0x0807_0605), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(expected_rest, LittleEndian), + }; + let section = Section::with_endian(Endian::Little) + .unit(&mut expected_unit) + .append_bytes(expected_rest); + let buf = section.get_contents().unwrap(); + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + assert_eq!( + parse_unit_header(rest, DebugInfoOffset(0).into()), + Ok(expected_unit) + ); + assert_eq!(*rest, EndianSlice::new(expected_rest, LittleEndian)); + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_parse_v5_split_type_unit_header_64_ok() { + let expected_rest = &[1, 2, 3, 4, 5, 6, 7, 8, 9]; + let encoding = Encoding { + format: Format::Dwarf64, + version: 5, + address_size: 8, + }; + let mut expected_unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::SplitType { + type_signature: DebugTypeSignature(0xdead_beef_dead_beef), + type_offset: UnitOffset(0x7856_3412_7856_3412), + }, + debug_abbrev_offset: DebugAbbrevOffset(0x0807_0605), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(expected_rest, LittleEndian), + }; + let section = Section::with_endian(Endian::Little) + .unit(&mut expected_unit) + .append_bytes(expected_rest); + let buf = section.get_contents().unwrap(); + let rest = &mut EndianSlice::new(&buf, LittleEndian); + + assert_eq!( + parse_unit_header(rest, DebugInfoOffset(0).into()), + Ok(expected_unit) + ); + assert_eq!(*rest, EndianSlice::new(expected_rest, LittleEndian)); + } + + fn section_contents(f: F) -> Vec + where + F: Fn(Section) -> Section, + { + f(Section::with_endian(Endian::Little)) + .get_contents() + .unwrap() + } + + #[test] + fn test_attribute_value() { + let mut unit = test_parse_attribute_unit_default(); + let endian = unit.entries_buf.endian(); + + let block_data = &[1, 2, 3, 4]; + let buf = section_contents(|s| s.uleb(block_data.len() as u64).append_bytes(block_data)); + let block = EndianSlice::new(&buf, endian); + + let buf = section_contents(|s| s.L32(0x0102_0304)); + let data4 = EndianSlice::new(&buf, endian); + + let buf = section_contents(|s| s.L64(0x0102_0304_0506_0708)); + let data8 = EndianSlice::new(&buf, endian); + + let tests = [ + ( + Format::Dwarf32, + 2, + constants::DW_AT_data_member_location, + constants::DW_FORM_block, + block, + AttributeValue::Block(EndianSlice::new(block_data, endian)), + AttributeValue::Exprloc(Expression(EndianSlice::new(block_data, endian))), + ), + ( + Format::Dwarf32, + 2, + constants::DW_AT_data_member_location, + constants::DW_FORM_data4, + data4, + AttributeValue::SecOffset(0x0102_0304), + AttributeValue::LocationListsRef(LocationListsOffset(0x0102_0304)), + ), + ( + Format::Dwarf64, + 2, + constants::DW_AT_data_member_location, + constants::DW_FORM_data4, + data4, + AttributeValue::Data4(0x0102_0304), + AttributeValue::Udata(0x0102_0304), + ), + ( + Format::Dwarf32, + 4, + constants::DW_AT_data_member_location, + constants::DW_FORM_data4, + data4, + AttributeValue::Data4(0x0102_0304), + AttributeValue::Udata(0x0102_0304), + ), + ( + Format::Dwarf32, + 2, + constants::DW_AT_data_member_location, + constants::DW_FORM_data8, + data8, + AttributeValue::Data8(0x0102_0304_0506_0708), + AttributeValue::Udata(0x0102_0304_0506_0708), + ), + #[cfg(target_pointer_width = "64")] + ( + Format::Dwarf64, + 2, + constants::DW_AT_data_member_location, + constants::DW_FORM_data8, + data8, + AttributeValue::SecOffset(0x0102_0304_0506_0708), + AttributeValue::LocationListsRef(LocationListsOffset(0x0102_0304_0506_0708)), + ), + ( + Format::Dwarf64, + 4, + constants::DW_AT_data_member_location, + constants::DW_FORM_data8, + data8, + AttributeValue::Data8(0x0102_0304_0506_0708), + AttributeValue::Udata(0x0102_0304_0506_0708), + ), + ( + Format::Dwarf32, + 4, + constants::DW_AT_location, + constants::DW_FORM_data4, + data4, + AttributeValue::SecOffset(0x0102_0304), + AttributeValue::LocationListsRef(LocationListsOffset(0x0102_0304)), + ), + #[cfg(target_pointer_width = "64")] + ( + Format::Dwarf64, + 4, + constants::DW_AT_location, + constants::DW_FORM_data8, + data8, + AttributeValue::SecOffset(0x0102_0304_0506_0708), + AttributeValue::LocationListsRef(LocationListsOffset(0x0102_0304_0506_0708)), + ), + ( + Format::Dwarf32, + 4, + constants::DW_AT_str_offsets_base, + constants::DW_FORM_sec_offset, + data4, + AttributeValue::SecOffset(0x0102_0304), + AttributeValue::DebugStrOffsetsBase(DebugStrOffsetsBase(0x0102_0304)), + ), + ( + Format::Dwarf32, + 4, + constants::DW_AT_stmt_list, + constants::DW_FORM_sec_offset, + data4, + AttributeValue::SecOffset(0x0102_0304), + AttributeValue::DebugLineRef(DebugLineOffset(0x0102_0304)), + ), + ( + Format::Dwarf32, + 4, + constants::DW_AT_addr_base, + constants::DW_FORM_sec_offset, + data4, + AttributeValue::SecOffset(0x0102_0304), + AttributeValue::DebugAddrBase(DebugAddrBase(0x0102_0304)), + ), + ( + Format::Dwarf32, + 4, + constants::DW_AT_rnglists_base, + constants::DW_FORM_sec_offset, + data4, + AttributeValue::SecOffset(0x0102_0304), + AttributeValue::DebugRngListsBase(DebugRngListsBase(0x0102_0304)), + ), + ( + Format::Dwarf32, + 4, + constants::DW_AT_loclists_base, + constants::DW_FORM_sec_offset, + data4, + AttributeValue::SecOffset(0x0102_0304), + AttributeValue::DebugLocListsBase(DebugLocListsBase(0x0102_0304)), + ), + ]; + + for test in tests.iter() { + let (format, version, name, form, mut input, expect_raw, expect_value) = *test; + unit.encoding.format = format; + unit.encoding.version = version; + let spec = AttributeSpecification::new(name, form, None); + let attribute = + parse_attribute(&mut input, unit.encoding(), spec).expect("Should parse attribute"); + assert_eq!(attribute.raw_value(), expect_raw); + assert_eq!(attribute.value(), expect_value); + } + } + + #[test] + fn test_attribute_udata_sdata_value() { + #[allow(clippy::type_complexity)] + let tests: &[( + AttributeValue>, + Option, + Option, + )] = &[ + (AttributeValue::Data1(1), Some(1), Some(1)), + ( + AttributeValue::Data1(core::u8::MAX), + Some(u64::from(std::u8::MAX)), + Some(-1), + ), + (AttributeValue::Data2(1), Some(1), Some(1)), + ( + AttributeValue::Data2(core::u16::MAX), + Some(u64::from(std::u16::MAX)), + Some(-1), + ), + (AttributeValue::Data4(1), Some(1), Some(1)), + ( + AttributeValue::Data4(core::u32::MAX), + Some(u64::from(std::u32::MAX)), + Some(-1), + ), + (AttributeValue::Data8(1), Some(1), Some(1)), + ( + AttributeValue::Data8(core::u64::MAX), + Some(core::u64::MAX), + Some(-1), + ), + (AttributeValue::Sdata(1), Some(1), Some(1)), + (AttributeValue::Sdata(-1), None, Some(-1)), + (AttributeValue::Udata(1), Some(1), Some(1)), + (AttributeValue::Udata(1u64 << 63), Some(1u64 << 63), None), + ]; + for test in tests.iter() { + let (value, expect_udata, expect_sdata) = *test; + let attribute = Attribute { + name: DW_AT_data_member_location, + value, + }; + assert_eq!(attribute.udata_value(), expect_udata); + assert_eq!(attribute.sdata_value(), expect_sdata); + } + } + + fn test_parse_attribute_unit( + address_size: u8, + format: Format, + endian: Endian, + ) -> UnitHeader> + where + Endian: Endianity, + { + let encoding = Encoding { + format, + version: 4, + address_size, + }; + UnitHeader::new( + encoding, + 7, + UnitType::Compilation, + DebugAbbrevOffset(0x0807_0605), + DebugInfoOffset(0).into(), + EndianSlice::new(&[], endian), + ) + } + + fn test_parse_attribute_unit_default() -> UnitHeader> { + test_parse_attribute_unit(4, Format::Dwarf32, LittleEndian) + } + + fn test_parse_attribute<'input, Endian>( + buf: &'input [u8], + len: usize, + unit: &UnitHeader>, + form: constants::DwForm, + value: AttributeValue>, + ) where + Endian: Endianity, + { + let spec = AttributeSpecification::new(constants::DW_AT_low_pc, form, None); + + let expect = Attribute { + name: constants::DW_AT_low_pc, + value, + }; + + let rest = &mut EndianSlice::new(buf, Endian::default()); + match parse_attribute(rest, unit.encoding(), spec) { + Ok(attr) => { + assert_eq!(attr, expect); + assert_eq!(*rest, EndianSlice::new(&buf[len..], Endian::default())); + if let Some(size) = spec.size(unit) { + assert_eq!(rest.len() + size, buf.len()); + } + } + otherwise => { + assert!(false, "Unexpected parse result = {:#?}", otherwise); + } + }; + } + + #[test] + fn test_parse_attribute_addr() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; + let unit = test_parse_attribute_unit(4, Format::Dwarf32, LittleEndian); + let form = constants::DW_FORM_addr; + let value = AttributeValue::Addr(0x0403_0201); + test_parse_attribute(&buf, 4, &unit, form, value); + } + + #[test] + fn test_parse_attribute_addr8() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; + let unit = test_parse_attribute_unit(8, Format::Dwarf32, LittleEndian); + let form = constants::DW_FORM_addr; + let value = AttributeValue::Addr(0x0807_0605_0403_0201); + test_parse_attribute(&buf, 8, &unit, form, value); + } + + #[test] + fn test_parse_attribute_block1() { + // Length of data (3), three bytes of data, two bytes of left over input. + let buf = [0x03, 0x09, 0x09, 0x09, 0x00, 0x00]; + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_block1; + let value = AttributeValue::Block(EndianSlice::new(&buf[1..4], LittleEndian)); + test_parse_attribute(&buf, 4, &unit, form, value); + } + + #[test] + fn test_parse_attribute_block2() { + // Two byte length of data (2), two bytes of data, two bytes of left over input. + let buf = [0x02, 0x00, 0x09, 0x09, 0x00, 0x00]; + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_block2; + let value = AttributeValue::Block(EndianSlice::new(&buf[2..4], LittleEndian)); + test_parse_attribute(&buf, 4, &unit, form, value); + } + + #[test] + fn test_parse_attribute_block4() { + // Four byte length of data (2), two bytes of data, no left over input. + let buf = [0x02, 0x00, 0x00, 0x00, 0x99, 0x99]; + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_block4; + let value = AttributeValue::Block(EndianSlice::new(&buf[4..], LittleEndian)); + test_parse_attribute(&buf, 6, &unit, form, value); + } + + #[test] + fn test_parse_attribute_block() { + // LEB length of data (2, one byte), two bytes of data, no left over input. + let buf = [0x02, 0x99, 0x99]; + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_block; + let value = AttributeValue::Block(EndianSlice::new(&buf[1..], LittleEndian)); + test_parse_attribute(&buf, 3, &unit, form, value); + } + + #[test] + fn test_parse_attribute_data1() { + let buf = [0x03]; + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_data1; + let value = AttributeValue::Data1(0x03); + test_parse_attribute(&buf, 1, &unit, form, value); + } + + #[test] + fn test_parse_attribute_data2() { + let buf = [0x02, 0x01, 0x0]; + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_data2; + let value = AttributeValue::Data2(0x0102); + test_parse_attribute(&buf, 2, &unit, form, value); + } + + #[test] + fn test_parse_attribute_data4() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x99, 0x99]; + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_data4; + let value = AttributeValue::Data4(0x0403_0201); + test_parse_attribute(&buf, 4, &unit, form, value); + } + + #[test] + fn test_parse_attribute_data8() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x99, 0x99]; + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_data8; + let value = AttributeValue::Data8(0x0807_0605_0403_0201); + test_parse_attribute(&buf, 8, &unit, form, value); + } + + #[test] + fn test_parse_attribute_udata() { + let mut buf = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + + let bytes_written = { + let mut writable = &mut buf[..]; + leb128::write::unsigned(&mut writable, 4097).expect("should write ok") + }; + + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_udata; + let value = AttributeValue::Udata(4097); + test_parse_attribute(&buf, bytes_written, &unit, form, value); + } + + #[test] + fn test_parse_attribute_sdata() { + let mut buf = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + + let bytes_written = { + let mut writable = &mut buf[..]; + leb128::write::signed(&mut writable, -4097).expect("should write ok") + }; + + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_sdata; + let value = AttributeValue::Sdata(-4097); + test_parse_attribute(&buf, bytes_written, &unit, form, value); + } + + #[test] + fn test_parse_attribute_exprloc() { + // LEB length of data (2, one byte), two bytes of data, one byte left over input. + let buf = [0x02, 0x99, 0x99, 0x11]; + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_exprloc; + let value = AttributeValue::Exprloc(Expression(EndianSlice::new(&buf[1..3], LittleEndian))); + test_parse_attribute(&buf, 3, &unit, form, value); + } + + #[test] + fn test_parse_attribute_flag_true() { + let buf = [0x42]; + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_flag; + let value = AttributeValue::Flag(true); + test_parse_attribute(&buf, 1, &unit, form, value); + } + + #[test] + fn test_parse_attribute_flag_false() { + let buf = [0x00]; + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_flag; + let value = AttributeValue::Flag(false); + test_parse_attribute(&buf, 1, &unit, form, value); + } + + #[test] + fn test_parse_attribute_flag_present() { + let buf = [0x01, 0x02, 0x03, 0x04]; + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_flag_present; + let value = AttributeValue::Flag(true); + // DW_FORM_flag_present does not consume any bytes of the input stream. + test_parse_attribute(&buf, 0, &unit, form, value); + } + + #[test] + fn test_parse_attribute_sec_offset_32() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10]; + let unit = test_parse_attribute_unit(4, Format::Dwarf32, LittleEndian); + let form = constants::DW_FORM_sec_offset; + let value = AttributeValue::SecOffset(0x0403_0201); + test_parse_attribute(&buf, 4, &unit, form, value); + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_parse_attribute_sec_offset_64() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10]; + let unit = test_parse_attribute_unit(4, Format::Dwarf64, LittleEndian); + let form = constants::DW_FORM_sec_offset; + let value = AttributeValue::SecOffset(0x0807_0605_0403_0201); + test_parse_attribute(&buf, 8, &unit, form, value); + } + + #[test] + fn test_parse_attribute_ref1() { + let buf = [0x03]; + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_ref1; + let value = AttributeValue::UnitRef(UnitOffset(3)); + test_parse_attribute(&buf, 1, &unit, form, value); + } + + #[test] + fn test_parse_attribute_ref2() { + let buf = [0x02, 0x01, 0x0]; + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_ref2; + let value = AttributeValue::UnitRef(UnitOffset(258)); + test_parse_attribute(&buf, 2, &unit, form, value); + } + + #[test] + fn test_parse_attribute_ref4() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x99, 0x99]; + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_ref4; + let value = AttributeValue::UnitRef(UnitOffset(0x0403_0201)); + test_parse_attribute(&buf, 4, &unit, form, value); + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_parse_attribute_ref8() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x99, 0x99]; + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_ref8; + let value = AttributeValue::UnitRef(UnitOffset(0x0807_0605_0403_0201)); + test_parse_attribute(&buf, 8, &unit, form, value); + } + + #[test] + fn test_parse_attribute_ref_sup4() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x99, 0x99]; + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_ref_sup4; + let value = AttributeValue::DebugInfoRefSup(DebugInfoOffset(0x0403_0201)); + test_parse_attribute(&buf, 4, &unit, form, value); + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_parse_attribute_ref_sup8() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x99, 0x99]; + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_ref_sup8; + let value = AttributeValue::DebugInfoRefSup(DebugInfoOffset(0x0807_0605_0403_0201)); + test_parse_attribute(&buf, 8, &unit, form, value); + } + + #[test] + fn test_parse_attribute_refudata() { + let mut buf = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + + let bytes_written = { + let mut writable = &mut buf[..]; + leb128::write::unsigned(&mut writable, 4097).expect("should write ok") + }; + + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_ref_udata; + let value = AttributeValue::UnitRef(UnitOffset(4097)); + test_parse_attribute(&buf, bytes_written, &unit, form, value); + } + + #[test] + fn test_parse_attribute_refaddr_32() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x99, 0x99]; + let unit = test_parse_attribute_unit(4, Format::Dwarf32, LittleEndian); + let form = constants::DW_FORM_ref_addr; + let value = AttributeValue::DebugInfoRef(DebugInfoOffset(0x0403_0201)); + test_parse_attribute(&buf, 4, &unit, form, value); + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_parse_attribute_refaddr_64() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x99, 0x99]; + let unit = test_parse_attribute_unit(4, Format::Dwarf64, LittleEndian); + let form = constants::DW_FORM_ref_addr; + let value = AttributeValue::DebugInfoRef(DebugInfoOffset(0x0807_0605_0403_0201)); + test_parse_attribute(&buf, 8, &unit, form, value); + } + + #[test] + fn test_parse_attribute_refaddr_version2() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x99, 0x99]; + let mut unit = test_parse_attribute_unit(4, Format::Dwarf32, LittleEndian); + unit.encoding.version = 2; + let form = constants::DW_FORM_ref_addr; + let value = AttributeValue::DebugInfoRef(DebugInfoOffset(0x0403_0201)); + test_parse_attribute(&buf, 4, &unit, form, value); + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_parse_attribute_refaddr8_version2() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x99, 0x99]; + let mut unit = test_parse_attribute_unit(8, Format::Dwarf32, LittleEndian); + unit.encoding.version = 2; + let form = constants::DW_FORM_ref_addr; + let value = AttributeValue::DebugInfoRef(DebugInfoOffset(0x0807_0605_0403_0201)); + test_parse_attribute(&buf, 8, &unit, form, value); + } + + #[test] + fn test_parse_attribute_gnu_ref_alt_32() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x99, 0x99]; + let unit = test_parse_attribute_unit(4, Format::Dwarf32, LittleEndian); + let form = constants::DW_FORM_GNU_ref_alt; + let value = AttributeValue::DebugInfoRefSup(DebugInfoOffset(0x0403_0201)); + test_parse_attribute(&buf, 4, &unit, form, value); + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_parse_attribute_gnu_ref_alt_64() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x99, 0x99]; + let unit = test_parse_attribute_unit(4, Format::Dwarf64, LittleEndian); + let form = constants::DW_FORM_GNU_ref_alt; + let value = AttributeValue::DebugInfoRefSup(DebugInfoOffset(0x0807_0605_0403_0201)); + test_parse_attribute(&buf, 8, &unit, form, value); + } + + #[test] + fn test_parse_attribute_refsig8() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x99, 0x99]; + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_ref_sig8; + let value = AttributeValue::DebugTypesRef(DebugTypeSignature(0x0807_0605_0403_0201)); + test_parse_attribute(&buf, 8, &unit, form, value); + } + + #[test] + fn test_parse_attribute_string() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x0, 0x99, 0x99]; + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_string; + let value = AttributeValue::String(EndianSlice::new(&buf[..5], LittleEndian)); + test_parse_attribute(&buf, 6, &unit, form, value); + } + + #[test] + fn test_parse_attribute_strp_32() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x99, 0x99]; + let unit = test_parse_attribute_unit(4, Format::Dwarf32, LittleEndian); + let form = constants::DW_FORM_strp; + let value = AttributeValue::DebugStrRef(DebugStrOffset(0x0403_0201)); + test_parse_attribute(&buf, 4, &unit, form, value); + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_parse_attribute_strp_64() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x99, 0x99]; + let unit = test_parse_attribute_unit(4, Format::Dwarf64, LittleEndian); + let form = constants::DW_FORM_strp; + let value = AttributeValue::DebugStrRef(DebugStrOffset(0x0807_0605_0403_0201)); + test_parse_attribute(&buf, 8, &unit, form, value); + } + + #[test] + fn test_parse_attribute_strp_sup_32() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x99, 0x99]; + let unit = test_parse_attribute_unit(4, Format::Dwarf32, LittleEndian); + let form = constants::DW_FORM_strp_sup; + let value = AttributeValue::DebugStrRefSup(DebugStrOffset(0x0403_0201)); + test_parse_attribute(&buf, 4, &unit, form, value); + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_parse_attribute_strp_sup_64() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x99, 0x99]; + let unit = test_parse_attribute_unit(4, Format::Dwarf64, LittleEndian); + let form = constants::DW_FORM_strp_sup; + let value = AttributeValue::DebugStrRefSup(DebugStrOffset(0x0807_0605_0403_0201)); + test_parse_attribute(&buf, 8, &unit, form, value); + } + + #[test] + fn test_parse_attribute_gnu_strp_alt_32() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x99, 0x99]; + let unit = test_parse_attribute_unit(4, Format::Dwarf32, LittleEndian); + let form = constants::DW_FORM_GNU_strp_alt; + let value = AttributeValue::DebugStrRefSup(DebugStrOffset(0x0403_0201)); + test_parse_attribute(&buf, 4, &unit, form, value); + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_parse_attribute_gnu_strp_alt_64() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x99, 0x99]; + let unit = test_parse_attribute_unit(4, Format::Dwarf64, LittleEndian); + let form = constants::DW_FORM_GNU_strp_alt; + let value = AttributeValue::DebugStrRefSup(DebugStrOffset(0x0807_0605_0403_0201)); + test_parse_attribute(&buf, 8, &unit, form, value); + } + + #[test] + fn test_parse_attribute_strx() { + let mut buf = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + + let bytes_written = { + let mut writable = &mut buf[..]; + leb128::write::unsigned(&mut writable, 4097).expect("should write ok") + }; + + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_strx; + let value = AttributeValue::DebugStrOffsetsIndex(DebugStrOffsetsIndex(4097)); + test_parse_attribute(&buf, bytes_written, &unit, form, value); + } + + #[test] + fn test_parse_attribute_strx1() { + let buf = [0x01, 0x99, 0x99]; + let unit = test_parse_attribute_unit(4, Format::Dwarf64, LittleEndian); + let form = constants::DW_FORM_strx1; + let value = AttributeValue::DebugStrOffsetsIndex(DebugStrOffsetsIndex(0x01)); + test_parse_attribute(&buf, 1, &unit, form, value); + } + + #[test] + fn test_parse_attribute_strx2() { + let buf = [0x01, 0x02, 0x99, 0x99]; + let unit = test_parse_attribute_unit(4, Format::Dwarf64, LittleEndian); + let form = constants::DW_FORM_strx2; + let value = AttributeValue::DebugStrOffsetsIndex(DebugStrOffsetsIndex(0x0201)); + test_parse_attribute(&buf, 2, &unit, form, value); + } + + #[test] + fn test_parse_attribute_strx3() { + let buf = [0x01, 0x02, 0x03, 0x99, 0x99]; + let unit = test_parse_attribute_unit(4, Format::Dwarf64, LittleEndian); + let form = constants::DW_FORM_strx3; + let value = AttributeValue::DebugStrOffsetsIndex(DebugStrOffsetsIndex(0x03_0201)); + test_parse_attribute(&buf, 3, &unit, form, value); + } + + #[test] + fn test_parse_attribute_strx4() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x99, 0x99]; + let unit = test_parse_attribute_unit(4, Format::Dwarf64, LittleEndian); + let form = constants::DW_FORM_strx4; + let value = AttributeValue::DebugStrOffsetsIndex(DebugStrOffsetsIndex(0x0403_0201)); + test_parse_attribute(&buf, 4, &unit, form, value); + } + + #[test] + fn test_parse_attribute_addrx() { + let mut buf = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + + let bytes_written = { + let mut writable = &mut buf[..]; + leb128::write::unsigned(&mut writable, 4097).expect("should write ok") + }; + + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_addrx; + let value = AttributeValue::DebugAddrIndex(DebugAddrIndex(4097)); + test_parse_attribute(&buf, bytes_written, &unit, form, value); + } + + #[test] + fn test_parse_attribute_addrx1() { + let buf = [0x01, 0x99, 0x99]; + let unit = test_parse_attribute_unit(4, Format::Dwarf64, LittleEndian); + let form = constants::DW_FORM_addrx1; + let value = AttributeValue::DebugAddrIndex(DebugAddrIndex(0x01)); + test_parse_attribute(&buf, 1, &unit, form, value); + } + + #[test] + fn test_parse_attribute_addrx2() { + let buf = [0x01, 0x02, 0x99, 0x99]; + let unit = test_parse_attribute_unit(4, Format::Dwarf64, LittleEndian); + let form = constants::DW_FORM_addrx2; + let value = AttributeValue::DebugAddrIndex(DebugAddrIndex(0x0201)); + test_parse_attribute(&buf, 2, &unit, form, value); + } + + #[test] + fn test_parse_attribute_addrx3() { + let buf = [0x01, 0x02, 0x03, 0x99, 0x99]; + let unit = test_parse_attribute_unit(4, Format::Dwarf64, LittleEndian); + let form = constants::DW_FORM_addrx3; + let value = AttributeValue::DebugAddrIndex(DebugAddrIndex(0x03_0201)); + test_parse_attribute(&buf, 3, &unit, form, value); + } + + #[test] + fn test_parse_attribute_addrx4() { + let buf = [0x01, 0x02, 0x03, 0x04, 0x99, 0x99]; + let unit = test_parse_attribute_unit(4, Format::Dwarf64, LittleEndian); + let form = constants::DW_FORM_addrx4; + let value = AttributeValue::DebugAddrIndex(DebugAddrIndex(0x0403_0201)); + test_parse_attribute(&buf, 4, &unit, form, value); + } + + #[test] + fn test_parse_attribute_loclistx() { + let mut buf = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + + let bytes_written = { + let mut writable = &mut buf[..]; + leb128::write::unsigned(&mut writable, 4097).expect("should write ok") + }; + + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_loclistx; + let value = AttributeValue::DebugLocListsIndex(DebugLocListsIndex(4097)); + test_parse_attribute(&buf, bytes_written, &unit, form, value); + } + + #[test] + fn test_parse_attribute_rnglistx() { + let mut buf = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + + let bytes_written = { + let mut writable = &mut buf[..]; + leb128::write::unsigned(&mut writable, 4097).expect("should write ok") + }; + + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_rnglistx; + let value = AttributeValue::DebugRngListsIndex(DebugRngListsIndex(4097)); + test_parse_attribute(&buf, bytes_written, &unit, form, value); + } + + #[test] + fn test_parse_attribute_indirect() { + let mut buf = [0; 100]; + + let bytes_written = { + let mut writable = &mut buf[..]; + leb128::write::unsigned(&mut writable, constants::DW_FORM_udata.0.into()) + .expect("should write udata") + + leb128::write::unsigned(&mut writable, 9_999_999).expect("should write value") + }; + + let unit = test_parse_attribute_unit_default(); + let form = constants::DW_FORM_indirect; + let value = AttributeValue::Udata(9_999_999); + test_parse_attribute(&buf, bytes_written, &unit, form, value); + } + + #[test] + fn test_parse_attribute_indirect_implicit_const() { + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 4, + }; + let mut buf = [0; 100]; + let mut writable = &mut buf[..]; + leb128::write::unsigned(&mut writable, constants::DW_FORM_implicit_const.0.into()) + .expect("should write implicit_const"); + + let input = &mut EndianSlice::new(&buf, LittleEndian); + let spec = + AttributeSpecification::new(constants::DW_AT_low_pc, constants::DW_FORM_indirect, None); + assert_eq!( + parse_attribute(input, encoding, spec), + Err(Error::InvalidImplicitConst) + ); + } + + #[test] + fn test_attrs_iter() { + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 4, + }; + let unit = UnitHeader::new( + encoding, + 7, + UnitType::Compilation, + DebugAbbrevOffset(0x0807_0605), + DebugInfoOffset(0).into(), + EndianSlice::new(&[], LittleEndian), + ); + + let abbrev = Abbreviation::new( + 42, + constants::DW_TAG_subprogram, + constants::DW_CHILDREN_yes, + vec![ + AttributeSpecification::new(constants::DW_AT_name, constants::DW_FORM_string, None), + AttributeSpecification::new(constants::DW_AT_low_pc, constants::DW_FORM_addr, None), + AttributeSpecification::new( + constants::DW_AT_high_pc, + constants::DW_FORM_addr, + None, + ), + ] + .into(), + ); + + // "foo", 42, 1337, 4 dangling bytes of 0xaa where children would be + let buf = [ + 0x66, 0x6f, 0x6f, 0x00, 0x2a, 0x00, 0x00, 0x00, 0x39, 0x05, 0x00, 0x00, 0xaa, 0xaa, + 0xaa, 0xaa, + ]; + + let entry = DebuggingInformationEntry { + offset: UnitOffset(0), + attrs_slice: EndianSlice::new(&buf, LittleEndian), + attrs_len: Cell::new(None), + abbrev: &abbrev, + unit: &unit, + }; + + let mut attrs = AttrsIter { + input: EndianSlice::new(&buf, LittleEndian), + attributes: abbrev.attributes(), + entry: &entry, + }; + + match attrs.next() { + Ok(Some(attr)) => { + assert_eq!( + attr, + Attribute { + name: constants::DW_AT_name, + value: AttributeValue::String(EndianSlice::new(b"foo", LittleEndian)), + } + ); + } + otherwise => { + assert!(false, "Unexpected parse result = {:#?}", otherwise); + } + } + + assert!(entry.attrs_len.get().is_none()); + + match attrs.next() { + Ok(Some(attr)) => { + assert_eq!( + attr, + Attribute { + name: constants::DW_AT_low_pc, + value: AttributeValue::Addr(0x2a), + } + ); + } + otherwise => { + assert!(false, "Unexpected parse result = {:#?}", otherwise); + } + } + + assert!(entry.attrs_len.get().is_none()); + + match attrs.next() { + Ok(Some(attr)) => { + assert_eq!( + attr, + Attribute { + name: constants::DW_AT_high_pc, + value: AttributeValue::Addr(0x539), + } + ); + } + otherwise => { + assert!(false, "Unexpected parse result = {:#?}", otherwise); + } + } + + assert!(entry.attrs_len.get().is_none()); + + assert!(attrs.next().expect("should parse next").is_none()); + assert!(entry.attrs_len.get().is_some()); + assert_eq!( + entry.attrs_len.get().expect("should have entry.attrs_len"), + buf.len() - 4 + ) + } + + #[test] + fn test_attrs_iter_incomplete() { + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 4, + }; + let unit = UnitHeader::new( + encoding, + 7, + UnitType::Compilation, + DebugAbbrevOffset(0x0807_0605), + DebugInfoOffset(0).into(), + EndianSlice::new(&[], LittleEndian), + ); + + let abbrev = Abbreviation::new( + 42, + constants::DW_TAG_subprogram, + constants::DW_CHILDREN_yes, + vec![ + AttributeSpecification::new(constants::DW_AT_name, constants::DW_FORM_string, None), + AttributeSpecification::new(constants::DW_AT_low_pc, constants::DW_FORM_addr, None), + AttributeSpecification::new( + constants::DW_AT_high_pc, + constants::DW_FORM_addr, + None, + ), + ] + .into(), + ); + + // "foo" + let buf = [0x66, 0x6f, 0x6f, 0x00]; + + let entry = DebuggingInformationEntry { + offset: UnitOffset(0), + attrs_slice: EndianSlice::new(&buf, LittleEndian), + attrs_len: Cell::new(None), + abbrev: &abbrev, + unit: &unit, + }; + + let mut attrs = AttrsIter { + input: EndianSlice::new(&buf, LittleEndian), + attributes: abbrev.attributes(), + entry: &entry, + }; + + match attrs.next() { + Ok(Some(attr)) => { + assert_eq!( + attr, + Attribute { + name: constants::DW_AT_name, + value: AttributeValue::String(EndianSlice::new(b"foo", LittleEndian)), + } + ); + } + otherwise => { + assert!(false, "Unexpected parse result = {:#?}", otherwise); + } + } + + assert!(entry.attrs_len.get().is_none()); + + // Return error for incomplete attribute. + assert!(attrs.next().is_err()); + assert!(entry.attrs_len.get().is_none()); + + // Return error for all subsequent calls. + assert!(attrs.next().is_err()); + assert!(attrs.next().is_err()); + assert!(attrs.next().is_err()); + assert!(attrs.next().is_err()); + assert!(entry.attrs_len.get().is_none()); + } + + fn assert_entry_name(entry: &DebuggingInformationEntry>, name: &str) + where + Endian: Endianity, + { + let value = entry + .attr_value(constants::DW_AT_name) + .expect("Should have parsed the name attribute") + .expect("Should have found the name attribute"); + + assert_eq!( + value, + AttributeValue::String(EndianSlice::new(name.as_bytes(), Endian::default())) + ); + } + + fn assert_current_name(cursor: &EntriesCursor>, name: &str) + where + Endian: Endianity, + { + let entry = cursor.current().expect("Should have an entry result"); + assert_entry_name(entry, name); + } + + fn assert_next_entry(cursor: &mut EntriesCursor>, name: &str) + where + Endian: Endianity, + { + cursor + .next_entry() + .expect("Should parse next entry") + .expect("Should have an entry"); + assert_current_name(cursor, name); + } + + fn assert_next_entry_null(cursor: &mut EntriesCursor>) + where + Endian: Endianity, + { + cursor + .next_entry() + .expect("Should parse next entry") + .expect("Should have an entry"); + assert!(cursor.current().is_none()); + } + + fn assert_next_dfs( + cursor: &mut EntriesCursor>, + name: &str, + depth: isize, + ) where + Endian: Endianity, + { + { + let (val, entry) = cursor + .next_dfs() + .expect("Should parse next dfs") + .expect("Should not be done with traversal"); + assert_eq!(val, depth); + assert_entry_name(entry, name); + } + assert_current_name(cursor, name); + } + + fn assert_next_sibling(cursor: &mut EntriesCursor>, name: &str) + where + Endian: Endianity, + { + { + let entry = cursor + .next_sibling() + .expect("Should parse next sibling") + .expect("Should not be done with traversal"); + assert_entry_name(entry, name); + } + assert_current_name(cursor, name); + } + + fn assert_valid_sibling_ptr(cursor: &EntriesCursor>) + where + Endian: Endianity, + { + let sibling_ptr = cursor + .current() + .expect("Should have current entry") + .attr_value(constants::DW_AT_sibling); + match sibling_ptr { + Ok(Some(AttributeValue::UnitRef(offset))) => { + cursor + .unit + .range_from(offset..) + .expect("Sibling offset should be valid"); + } + _ => panic!("Invalid sibling pointer {:?}", sibling_ptr), + } + } + + fn entries_cursor_tests_abbrev_buf() -> Vec { + #[rustfmt::skip] + let section = Section::with_endian(Endian::Little) + .abbrev(1, DW_TAG_subprogram, DW_CHILDREN_yes) + .abbrev_attr(DW_AT_name, DW_FORM_string) + .abbrev_attr_null() + .abbrev_null(); + section.get_contents().unwrap() + } + + fn entries_cursor_tests_debug_info_buf() -> Vec { + #[rustfmt::skip] + let section = Section::with_endian(Endian::Little) + .die(1, |s| s.attr_string("001")) + .die(1, |s| s.attr_string("002")) + .die(1, |s| s.attr_string("003")) + .die_null() + .die_null() + .die(1, |s| s.attr_string("004")) + .die(1, |s| s.attr_string("005")) + .die_null() + .die(1, |s| s.attr_string("006")) + .die_null() + .die_null() + .die(1, |s| s.attr_string("007")) + .die(1, |s| s.attr_string("008")) + .die(1, |s| s.attr_string("009")) + .die_null() + .die_null() + .die_null() + .die(1, |s| s.attr_string("010")) + .die_null() + .die_null(); + let entries_buf = section.get_contents().unwrap(); + + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 4, + }; + let mut unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::Compilation, + debug_abbrev_offset: DebugAbbrevOffset(0), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(&entries_buf, LittleEndian), + }; + let section = Section::with_endian(Endian::Little).unit(&mut unit); + section.get_contents().unwrap() + } + + #[test] + fn test_cursor_next_entry_incomplete() { + #[rustfmt::skip] + let section = Section::with_endian(Endian::Little) + .die(1, |s| s.attr_string("001")) + .die(1, |s| s.attr_string("002")) + .die(1, |s| s); + let entries_buf = section.get_contents().unwrap(); + + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 4, + }; + let mut unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::Compilation, + debug_abbrev_offset: DebugAbbrevOffset(0), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(&entries_buf, LittleEndian), + }; + let section = Section::with_endian(Endian::Little).unit(&mut unit); + let info_buf = §ion.get_contents().unwrap(); + let debug_info = DebugInfo::new(info_buf, LittleEndian); + + let unit = debug_info + .units() + .next() + .expect("should have a unit result") + .expect("and it should be ok"); + + let abbrevs_buf = &entries_cursor_tests_abbrev_buf(); + let debug_abbrev = DebugAbbrev::new(abbrevs_buf, LittleEndian); + + let abbrevs = unit + .abbreviations(&debug_abbrev) + .expect("Should parse abbreviations"); + + let mut cursor = unit.entries(&abbrevs); + + assert_next_entry(&mut cursor, "001"); + assert_next_entry(&mut cursor, "002"); + + { + // Entry code is present, but none of the attributes. + cursor + .next_entry() + .expect("Should parse next entry") + .expect("Should have an entry"); + let entry = cursor.current().expect("Should have an entry result"); + assert!(entry.attrs().next().is_err()); + } + + assert!(cursor.next_entry().is_err()); + assert!(cursor.next_entry().is_err()); + } + + #[test] + fn test_cursor_next_entry() { + let info_buf = &entries_cursor_tests_debug_info_buf(); + let debug_info = DebugInfo::new(info_buf, LittleEndian); + + let unit = debug_info + .units() + .next() + .expect("should have a unit result") + .expect("and it should be ok"); + + let abbrevs_buf = &entries_cursor_tests_abbrev_buf(); + let debug_abbrev = DebugAbbrev::new(abbrevs_buf, LittleEndian); + + let abbrevs = unit + .abbreviations(&debug_abbrev) + .expect("Should parse abbreviations"); + + let mut cursor = unit.entries(&abbrevs); + + assert_next_entry(&mut cursor, "001"); + assert_next_entry(&mut cursor, "002"); + assert_next_entry(&mut cursor, "003"); + assert_next_entry_null(&mut cursor); + assert_next_entry_null(&mut cursor); + assert_next_entry(&mut cursor, "004"); + assert_next_entry(&mut cursor, "005"); + assert_next_entry_null(&mut cursor); + assert_next_entry(&mut cursor, "006"); + assert_next_entry_null(&mut cursor); + assert_next_entry_null(&mut cursor); + assert_next_entry(&mut cursor, "007"); + assert_next_entry(&mut cursor, "008"); + assert_next_entry(&mut cursor, "009"); + assert_next_entry_null(&mut cursor); + assert_next_entry_null(&mut cursor); + assert_next_entry_null(&mut cursor); + assert_next_entry(&mut cursor, "010"); + assert_next_entry_null(&mut cursor); + assert_next_entry_null(&mut cursor); + + assert!(cursor + .next_entry() + .expect("Should parse next entry") + .is_none()); + assert!(cursor.current().is_none()); + } + + #[test] + fn test_cursor_next_dfs() { + let info_buf = &entries_cursor_tests_debug_info_buf(); + let debug_info = DebugInfo::new(info_buf, LittleEndian); + + let unit = debug_info + .units() + .next() + .expect("should have a unit result") + .expect("and it should be ok"); + + let abbrevs_buf = &entries_cursor_tests_abbrev_buf(); + let debug_abbrev = DebugAbbrev::new(abbrevs_buf, LittleEndian); + + let abbrevs = unit + .abbreviations(&debug_abbrev) + .expect("Should parse abbreviations"); + + let mut cursor = unit.entries(&abbrevs); + + assert_next_dfs(&mut cursor, "001", 0); + assert_next_dfs(&mut cursor, "002", 1); + assert_next_dfs(&mut cursor, "003", 1); + assert_next_dfs(&mut cursor, "004", -1); + assert_next_dfs(&mut cursor, "005", 1); + assert_next_dfs(&mut cursor, "006", 0); + assert_next_dfs(&mut cursor, "007", -1); + assert_next_dfs(&mut cursor, "008", 1); + assert_next_dfs(&mut cursor, "009", 1); + assert_next_dfs(&mut cursor, "010", -2); + + assert!(cursor.next_dfs().expect("Should parse next dfs").is_none()); + assert!(cursor.current().is_none()); + } + + #[test] + fn test_cursor_next_sibling_no_sibling_ptr() { + let info_buf = &entries_cursor_tests_debug_info_buf(); + let debug_info = DebugInfo::new(info_buf, LittleEndian); + + let unit = debug_info + .units() + .next() + .expect("should have a unit result") + .expect("and it should be ok"); + + let abbrevs_buf = &entries_cursor_tests_abbrev_buf(); + let debug_abbrev = DebugAbbrev::new(abbrevs_buf, LittleEndian); + + let abbrevs = unit + .abbreviations(&debug_abbrev) + .expect("Should parse abbreviations"); + + let mut cursor = unit.entries(&abbrevs); + + assert_next_dfs(&mut cursor, "001", 0); + + // Down to the first child of the root entry. + + assert_next_dfs(&mut cursor, "002", 1); + + // Now iterate all children of the root via `next_sibling`. + + assert_next_sibling(&mut cursor, "004"); + assert_next_sibling(&mut cursor, "007"); + assert_next_sibling(&mut cursor, "010"); + + // There should be no more siblings. + + assert!(cursor + .next_sibling() + .expect("Should parse next sibling") + .is_none()); + assert!(cursor.current().is_none()); + } + + #[test] + fn test_cursor_next_sibling_continuation() { + let info_buf = &entries_cursor_tests_debug_info_buf(); + let debug_info = DebugInfo::new(info_buf, LittleEndian); + + let unit = debug_info + .units() + .next() + .expect("should have a unit result") + .expect("and it should be ok"); + + let abbrevs_buf = &entries_cursor_tests_abbrev_buf(); + let debug_abbrev = DebugAbbrev::new(abbrevs_buf, LittleEndian); + + let abbrevs = unit + .abbreviations(&debug_abbrev) + .expect("Should parse abbreviations"); + + let mut cursor = unit.entries(&abbrevs); + + assert_next_dfs(&mut cursor, "001", 0); + + // Down to the first child of the root entry. + + assert_next_dfs(&mut cursor, "002", 1); + + // Get the next sibling, then iterate its children + + assert_next_sibling(&mut cursor, "004"); + assert_next_dfs(&mut cursor, "005", 1); + assert_next_sibling(&mut cursor, "006"); + assert!(cursor + .next_sibling() + .expect("Should parse next sibling") + .is_none()); + assert!(cursor + .next_sibling() + .expect("Should parse next sibling") + .is_none()); + assert!(cursor + .next_sibling() + .expect("Should parse next sibling") + .is_none()); + assert!(cursor + .next_sibling() + .expect("Should parse next sibling") + .is_none()); + + // And we should be able to continue with the children of the root entry. + + assert_next_dfs(&mut cursor, "007", -1); + assert_next_sibling(&mut cursor, "010"); + + // There should be no more siblings. + + assert!(cursor + .next_sibling() + .expect("Should parse next sibling") + .is_none()); + assert!(cursor.current().is_none()); + } + + fn entries_cursor_sibling_abbrev_buf() -> Vec { + #[rustfmt::skip] + let section = Section::with_endian(Endian::Little) + .abbrev(1, DW_TAG_subprogram, DW_CHILDREN_yes) + .abbrev_attr(DW_AT_name, DW_FORM_string) + .abbrev_attr(DW_AT_sibling, DW_FORM_ref1) + .abbrev_attr_null() + .abbrev(2, DW_TAG_subprogram, DW_CHILDREN_yes) + .abbrev_attr(DW_AT_name, DW_FORM_string) + .abbrev_attr_null() + .abbrev_null(); + section.get_contents().unwrap() + } + + fn entries_cursor_sibling_entries_buf(header_size: usize) -> Vec { + let start = Label::new(); + let sibling004_ref = Label::new(); + let sibling004 = Label::new(); + let sibling009_ref = Label::new(); + let sibling009 = Label::new(); + + #[rustfmt::skip] + let section = Section::with_endian(Endian::Little) + .mark(&start) + .die(2, |s| s.attr_string("001")) + // Valid sibling attribute. + .die(1, |s| s.attr_string("002").D8(&sibling004_ref)) + // Invalid code to ensure the sibling attribute was used. + .die(10, |s| s.attr_string("003")) + .die_null() + .die_null() + .mark(&sibling004) + // Invalid sibling attribute. + .die(1, |s| s.attr_string("004").attr_ref1(255)) + .die(2, |s| s.attr_string("005")) + .die_null() + .die_null() + // Sibling attribute in child only. + .die(2, |s| s.attr_string("006")) + // Valid sibling attribute. + .die(1, |s| s.attr_string("007").D8(&sibling009_ref)) + // Invalid code to ensure the sibling attribute was used. + .die(10, |s| s.attr_string("008")) + .die_null() + .die_null() + .mark(&sibling009) + .die(2, |s| s.attr_string("009")) + .die_null() + .die_null() + // No sibling attribute. + .die(2, |s| s.attr_string("010")) + .die(2, |s| s.attr_string("011")) + .die_null() + .die_null() + .die_null(); + + let offset = header_size as u64 + (&sibling004 - &start) as u64; + sibling004_ref.set_const(offset); + + let offset = header_size as u64 + (&sibling009 - &start) as u64; + sibling009_ref.set_const(offset); + + section.get_contents().unwrap() + } + + fn test_cursor_next_sibling_with_ptr(cursor: &mut EntriesCursor>) { + assert_next_dfs(cursor, "001", 0); + + // Down to the first child of the root. + + assert_next_dfs(cursor, "002", 1); + + // Now iterate all children of the root via `next_sibling`. + + assert_valid_sibling_ptr(&cursor); + assert_next_sibling(cursor, "004"); + assert_next_sibling(cursor, "006"); + assert_next_sibling(cursor, "010"); + + // There should be no more siblings. + + assert!(cursor + .next_sibling() + .expect("Should parse next sibling") + .is_none()); + assert!(cursor.current().is_none()); + } + + #[test] + fn test_debug_info_next_sibling_with_ptr() { + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 4, + }; + + let mut unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::Compilation, + debug_abbrev_offset: DebugAbbrevOffset(0), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(&[], LittleEndian), + }; + let header_size = unit.size_of_header(); + let entries_buf = entries_cursor_sibling_entries_buf(header_size); + unit.entries_buf = EndianSlice::new(&entries_buf, LittleEndian); + let section = Section::with_endian(Endian::Little).unit(&mut unit); + let info_buf = section.get_contents().unwrap(); + let debug_info = DebugInfo::new(&info_buf, LittleEndian); + + let unit = debug_info + .units() + .next() + .expect("should have a unit result") + .expect("and it should be ok"); + + let abbrev_buf = entries_cursor_sibling_abbrev_buf(); + let debug_abbrev = DebugAbbrev::new(&abbrev_buf, LittleEndian); + + let abbrevs = unit + .abbreviations(&debug_abbrev) + .expect("Should parse abbreviations"); + + let mut cursor = unit.entries(&abbrevs); + test_cursor_next_sibling_with_ptr(&mut cursor); + } + + #[test] + fn test_debug_types_next_sibling_with_ptr() { + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 4, + }; + let mut unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::Type { + type_signature: DebugTypeSignature(0), + type_offset: UnitOffset(0), + }, + debug_abbrev_offset: DebugAbbrevOffset(0), + unit_offset: DebugTypesOffset(0).into(), + entries_buf: EndianSlice::new(&[], LittleEndian), + }; + let header_size = unit.size_of_header(); + let entries_buf = entries_cursor_sibling_entries_buf(header_size); + unit.entries_buf = EndianSlice::new(&entries_buf, LittleEndian); + let section = Section::with_endian(Endian::Little).unit(&mut unit); + let info_buf = section.get_contents().unwrap(); + let debug_types = DebugTypes::new(&info_buf, LittleEndian); + + let unit = debug_types + .units() + .next() + .expect("should have a unit result") + .expect("and it should be ok"); + + let abbrev_buf = entries_cursor_sibling_abbrev_buf(); + let debug_abbrev = DebugAbbrev::new(&abbrev_buf, LittleEndian); + + let abbrevs = unit + .abbreviations(&debug_abbrev) + .expect("Should parse abbreviations"); + + let mut cursor = unit.entries(&abbrevs); + test_cursor_next_sibling_with_ptr(&mut cursor); + } + + #[test] + fn test_entries_at_offset() { + let info_buf = &entries_cursor_tests_debug_info_buf(); + let debug_info = DebugInfo::new(info_buf, LittleEndian); + + let unit = debug_info + .units() + .next() + .expect("should have a unit result") + .expect("and it should be ok"); + + let abbrevs_buf = &entries_cursor_tests_abbrev_buf(); + let debug_abbrev = DebugAbbrev::new(abbrevs_buf, LittleEndian); + + let abbrevs = unit + .abbreviations(&debug_abbrev) + .expect("Should parse abbreviations"); + + let mut cursor = unit + .entries_at_offset(&abbrevs, UnitOffset(unit.header_size())) + .unwrap(); + assert_next_entry(&mut cursor, "001"); + + let cursor = unit.entries_at_offset(&abbrevs, UnitOffset(0)); + match cursor { + Err(Error::OffsetOutOfBounds) => {} + otherwise => { + assert!(false, "Unexpected parse result = {:#?}", otherwise); + } + } + } + + fn entries_tree_tests_debug_abbrevs_buf() -> Vec { + #[rustfmt::skip] + let section = Section::with_endian(Endian::Little) + .abbrev(1, DW_TAG_subprogram, DW_CHILDREN_yes) + .abbrev_attr(DW_AT_name, DW_FORM_string) + .abbrev_attr_null() + .abbrev(2, DW_TAG_subprogram, DW_CHILDREN_no) + .abbrev_attr(DW_AT_name, DW_FORM_string) + .abbrev_attr_null() + .abbrev_null() + .get_contents() + .unwrap(); + section + } + + fn entries_tree_tests_debug_info_buf(header_size: usize) -> (Vec, UnitOffset) { + let start = Label::new(); + let entry2 = Label::new(); + #[rustfmt::skip] + let section = Section::with_endian(Endian::Little) + .mark(&start) + .die(1, |s| s.attr_string("root")) + .die(1, |s| s.attr_string("1")) + .die(1, |s| s.attr_string("1a")) + .die_null() + .die(2, |s| s.attr_string("1b")) + .die_null() + .mark(&entry2) + .die(1, |s| s.attr_string("2")) + .die(1, |s| s.attr_string("2a")) + .die(1, |s| s.attr_string("2a1")) + .die_null() + .die_null() + .die(1, |s| s.attr_string("2b")) + .die(2, |s| s.attr_string("2b1")) + .die_null() + .die_null() + .die(1, |s| s.attr_string("3")) + .die(1, |s| s.attr_string("3a")) + .die(2, |s| s.attr_string("3a1")) + .die(2, |s| s.attr_string("3a2")) + .die_null() + .die(2, |s| s.attr_string("3b")) + .die_null() + .die(2, |s| s.attr_string("final")) + .die_null() + .get_contents() + .unwrap(); + let entry2 = UnitOffset(header_size + (&entry2 - &start) as usize); + (section, entry2) + } + + #[test] + fn test_entries_tree() { + fn assert_entry<'input, 'abbrev, 'unit, 'tree, Endian>( + node: Result< + Option>>, + >, + name: &str, + ) -> EntriesTreeIter<'abbrev, 'unit, 'tree, EndianSlice<'input, Endian>> + where + Endian: Endianity, + { + let node = node + .expect("Should parse entry") + .expect("Should have entry"); + assert_entry_name(node.entry(), name); + node.children() + } + + fn assert_null(node: Result>>>) { + match node { + Ok(None) => {} + otherwise => { + assert!(false, "Unexpected parse result = {:#?}", otherwise); + } + } + } + + let abbrevs_buf = entries_tree_tests_debug_abbrevs_buf(); + let debug_abbrev = DebugAbbrev::new(&abbrevs_buf, LittleEndian); + + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 4, + }; + let mut unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::Compilation, + debug_abbrev_offset: DebugAbbrevOffset(0), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(&[], LittleEndian), + }; + let header_size = unit.size_of_header(); + let (entries_buf, entry2) = entries_tree_tests_debug_info_buf(header_size); + unit.entries_buf = EndianSlice::new(&entries_buf, LittleEndian); + let info_buf = Section::with_endian(Endian::Little) + .unit(&mut unit) + .get_contents() + .unwrap(); + let debug_info = DebugInfo::new(&info_buf, LittleEndian); + + let unit = debug_info + .units() + .next() + .expect("Should parse unit") + .expect("and it should be some"); + let abbrevs = unit + .abbreviations(&debug_abbrev) + .expect("Should parse abbreviations"); + let mut tree = unit + .entries_tree(&abbrevs, None) + .expect("Should have entries tree"); + + // Test we can restart iteration of the tree. + { + let mut iter = assert_entry(tree.root().map(Some), "root"); + assert_entry(iter.next(), "1"); + } + { + let mut iter = assert_entry(tree.root().map(Some), "root"); + assert_entry(iter.next(), "1"); + } + + let mut iter = assert_entry(tree.root().map(Some), "root"); + { + // Test iteration with children. + let mut iter = assert_entry(iter.next(), "1"); + { + // Test iteration with children flag, but no children. + let mut iter = assert_entry(iter.next(), "1a"); + assert_null(iter.next()); + assert_null(iter.next()); + } + { + // Test iteration without children flag. + let mut iter = assert_entry(iter.next(), "1b"); + assert_null(iter.next()); + assert_null(iter.next()); + } + assert_null(iter.next()); + assert_null(iter.next()); + } + { + // Test skipping over children. + let mut iter = assert_entry(iter.next(), "2"); + assert_entry(iter.next(), "2a"); + assert_entry(iter.next(), "2b"); + assert_null(iter.next()); + } + { + // Test skipping after partial iteration. + let mut iter = assert_entry(iter.next(), "3"); + { + let mut iter = assert_entry(iter.next(), "3a"); + assert_entry(iter.next(), "3a1"); + // Parent iter should be able to skip over "3a2". + } + assert_entry(iter.next(), "3b"); + assert_null(iter.next()); + } + assert_entry(iter.next(), "final"); + assert_null(iter.next()); + + // Test starting at an offset. + let mut tree = unit + .entries_tree(&abbrevs, Some(entry2)) + .expect("Should have entries tree"); + let mut iter = assert_entry(tree.root().map(Some), "2"); + assert_entry(iter.next(), "2a"); + assert_entry(iter.next(), "2b"); + assert_null(iter.next()); + } + + #[test] + fn test_entries_raw() { + fn assert_abbrev<'input, 'abbrev, 'unit, Endian>( + entries: &mut EntriesRaw<'abbrev, 'unit, EndianSlice<'input, Endian>>, + tag: DwTag, + ) -> &'abbrev Abbreviation + where + Endian: Endianity, + { + let abbrev = entries + .read_abbreviation() + .expect("Should parse abbrev") + .expect("Should have abbrev"); + assert_eq!(abbrev.tag(), tag); + abbrev + } + + fn assert_null<'input, 'abbrev, 'unit, Endian>( + entries: &mut EntriesRaw<'abbrev, 'unit, EndianSlice<'input, Endian>>, + ) where + Endian: Endianity, + { + match entries.read_abbreviation() { + Ok(None) => {} + otherwise => { + assert!(false, "Unexpected parse result = {:#?}", otherwise); + } + } + } + + fn assert_attr<'input, 'abbrev, 'unit, Endian>( + entries: &mut EntriesRaw<'abbrev, 'unit, EndianSlice<'input, Endian>>, + spec: Option, + name: DwAt, + value: &str, + ) where + Endian: Endianity, + { + let spec = spec.expect("Should have attribute specification"); + let attr = entries + .read_attribute(spec) + .expect("Should parse attribute"); + assert_eq!(attr.name(), name); + assert_eq!( + attr.value(), + AttributeValue::String(EndianSlice::new(value.as_bytes(), Endian::default())) + ); + } + + #[rustfmt::skip] + let section = Section::with_endian(Endian::Little) + .abbrev(1, DW_TAG_subprogram, DW_CHILDREN_yes) + .abbrev_attr(DW_AT_name, DW_FORM_string) + .abbrev_attr(DW_AT_linkage_name, DW_FORM_string) + .abbrev_attr_null() + .abbrev(2, DW_TAG_variable, DW_CHILDREN_no) + .abbrev_attr(DW_AT_name, DW_FORM_string) + .abbrev_attr_null() + .abbrev_null(); + let abbrevs_buf = section.get_contents().unwrap(); + let debug_abbrev = DebugAbbrev::new(&abbrevs_buf, LittleEndian); + + #[rustfmt::skip] + let section = Section::with_endian(Endian::Little) + .die(1, |s| s.attr_string("f1").attr_string("l1")) + .die(2, |s| s.attr_string("v1")) + .die(2, |s| s.attr_string("v2")) + .die(1, |s| s.attr_string("f2").attr_string("l2")) + .die_null() + .die_null(); + let entries_buf = section.get_contents().unwrap(); + + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 4, + }; + let mut unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::Compilation, + debug_abbrev_offset: DebugAbbrevOffset(0), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(&entries_buf, LittleEndian), + }; + let section = Section::with_endian(Endian::Little).unit(&mut unit); + let info_buf = section.get_contents().unwrap(); + let debug_info = DebugInfo::new(&info_buf, LittleEndian); + + let unit = debug_info + .units() + .next() + .expect("should have a unit result") + .expect("and it should be ok"); + + let abbrevs = unit + .abbreviations(&debug_abbrev) + .expect("Should parse abbreviations"); + + let mut entries = unit + .entries_raw(&abbrevs, None) + .expect("Should have entries"); + + assert_eq!(entries.next_depth(), 0); + let abbrev = assert_abbrev(&mut entries, DW_TAG_subprogram); + let mut attrs = abbrev.attributes().iter().copied(); + assert_attr(&mut entries, attrs.next(), DW_AT_name, "f1"); + assert_attr(&mut entries, attrs.next(), DW_AT_linkage_name, "l1"); + assert!(attrs.next().is_none()); + + assert_eq!(entries.next_depth(), 1); + let abbrev = assert_abbrev(&mut entries, DW_TAG_variable); + let mut attrs = abbrev.attributes().iter().copied(); + assert_attr(&mut entries, attrs.next(), DW_AT_name, "v1"); + assert!(attrs.next().is_none()); + + assert_eq!(entries.next_depth(), 1); + let abbrev = assert_abbrev(&mut entries, DW_TAG_variable); + let mut attrs = abbrev.attributes().iter().copied(); + assert_attr(&mut entries, attrs.next(), DW_AT_name, "v2"); + assert!(attrs.next().is_none()); + + assert_eq!(entries.next_depth(), 1); + let abbrev = assert_abbrev(&mut entries, DW_TAG_subprogram); + let mut attrs = abbrev.attributes().iter().copied(); + assert_attr(&mut entries, attrs.next(), DW_AT_name, "f2"); + assert_attr(&mut entries, attrs.next(), DW_AT_linkage_name, "l2"); + assert!(attrs.next().is_none()); + + assert_eq!(entries.next_depth(), 2); + assert_null(&mut entries); + + assert_eq!(entries.next_depth(), 1); + assert_null(&mut entries); + + assert_eq!(entries.next_depth(), 0); + assert!(entries.is_empty()); + } + + #[test] + fn test_debug_info_offset() { + let padding = &[0; 10]; + let entries = &[0; 20]; + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 4, + }; + let mut unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::Compilation, + debug_abbrev_offset: DebugAbbrevOffset(0), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(entries, LittleEndian), + }; + Section::with_endian(Endian::Little) + .append_bytes(padding) + .unit(&mut unit); + let offset = padding.len(); + let header_length = unit.size_of_header(); + let length = unit.length_including_self(); + assert_eq!(DebugInfoOffset(0).to_unit_offset(&unit), None); + assert_eq!(DebugInfoOffset(offset - 1).to_unit_offset(&unit), None); + assert_eq!(DebugInfoOffset(offset).to_unit_offset(&unit), None); + assert_eq!( + DebugInfoOffset(offset + header_length - 1).to_unit_offset(&unit), + None + ); + assert_eq!( + DebugInfoOffset(offset + header_length).to_unit_offset(&unit), + Some(UnitOffset(header_length)) + ); + assert_eq!( + DebugInfoOffset(offset + length - 1).to_unit_offset(&unit), + Some(UnitOffset(length - 1)) + ); + assert_eq!(DebugInfoOffset(offset + length).to_unit_offset(&unit), None); + assert_eq!( + UnitOffset(header_length).to_debug_info_offset(&unit), + Some(DebugInfoOffset(offset + header_length)) + ); + assert_eq!( + UnitOffset(length - 1).to_debug_info_offset(&unit), + Some(DebugInfoOffset(offset + length - 1)) + ); + } + + #[test] + fn test_debug_types_offset() { + let padding = &[0; 10]; + let entries = &[0; 20]; + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 4, + }; + let mut unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::Type { + type_signature: DebugTypeSignature(0), + type_offset: UnitOffset(0), + }, + debug_abbrev_offset: DebugAbbrevOffset(0), + unit_offset: DebugTypesOffset(0).into(), + entries_buf: EndianSlice::new(entries, LittleEndian), + }; + Section::with_endian(Endian::Little) + .append_bytes(padding) + .unit(&mut unit); + let offset = padding.len(); + let header_length = unit.size_of_header(); + let length = unit.length_including_self(); + assert_eq!(DebugTypesOffset(0).to_unit_offset(&unit), None); + assert_eq!(DebugTypesOffset(offset - 1).to_unit_offset(&unit), None); + assert_eq!(DebugTypesOffset(offset).to_unit_offset(&unit), None); + assert_eq!( + DebugTypesOffset(offset + header_length - 1).to_unit_offset(&unit), + None + ); + assert_eq!( + DebugTypesOffset(offset + header_length).to_unit_offset(&unit), + Some(UnitOffset(header_length)) + ); + assert_eq!( + DebugTypesOffset(offset + length - 1).to_unit_offset(&unit), + Some(UnitOffset(length - 1)) + ); + assert_eq!( + DebugTypesOffset(offset + length).to_unit_offset(&unit), + None + ); + assert_eq!( + UnitOffset(header_length).to_debug_types_offset(&unit), + Some(DebugTypesOffset(offset + header_length)) + ); + assert_eq!( + UnitOffset(length - 1).to_debug_types_offset(&unit), + Some(DebugTypesOffset(offset + length - 1)) + ); + } + + #[test] + fn test_length_including_self() { + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 4, + }; + let mut unit = UnitHeader { + encoding, + unit_length: 0, + unit_type: UnitType::Compilation, + debug_abbrev_offset: DebugAbbrevOffset(0), + unit_offset: DebugInfoOffset(0).into(), + entries_buf: EndianSlice::new(&[], LittleEndian), + }; + unit.encoding.format = Format::Dwarf32; + assert_eq!(unit.length_including_self(), 4); + unit.encoding.format = Format::Dwarf64; + assert_eq!(unit.length_including_self(), 12); + unit.unit_length = 10; + assert_eq!(unit.length_including_self(), 22); + } + + #[test] + fn test_parse_type_unit_abbrevs() { + let types_buf = [ + // Type unit header + 0x25, 0x00, 0x00, 0x00, // 32-bit unit length = 37 + 0x04, 0x00, // Version 4 + 0x00, 0x00, 0x00, 0x00, // debug_abbrev_offset + 0x04, // Address size + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // Type signature + 0x01, 0x02, 0x03, 0x04, // Type offset + // DIEs + // Abbreviation code + 0x01, // Attribute of form DW_FORM_string = "foo\0" + 0x66, 0x6f, 0x6f, 0x00, // Children + // Abbreviation code + 0x01, // Attribute of form DW_FORM_string = "foo\0" + 0x66, 0x6f, 0x6f, 0x00, // Children + // Abbreviation code + 0x01, // Attribute of form DW_FORM_string = "foo\0" + 0x66, 0x6f, 0x6f, 0x00, // Children + 0x00, // End of children + 0x00, // End of children + 0x00, // End of children + ]; + let debug_types = DebugTypes::new(&types_buf, LittleEndian); + + let abbrev_buf = [ + // Code + 0x01, // DW_TAG_subprogram + 0x2e, // DW_CHILDREN_yes + 0x01, // Begin attributes + 0x03, // Attribute name = DW_AT_name + 0x08, // Attribute form = DW_FORM_string + 0x00, 0x00, // End attributes + 0x00, // Null terminator + ]; + + let get_some_type_unit = || debug_types.units().next().unwrap().unwrap(); + + let unit = get_some_type_unit(); + + let read_debug_abbrev_section_somehow = || &abbrev_buf; + let debug_abbrev = DebugAbbrev::new(read_debug_abbrev_section_somehow(), LittleEndian); + let _abbrevs_for_unit = unit.abbreviations(&debug_abbrev).unwrap(); + } +} diff --git a/crux-mir/lib/gimli/src/read/util.rs b/crux-mir/lib/gimli/src/read/util.rs new file mode 100644 index 000000000..16eafdde4 --- /dev/null +++ b/crux-mir/lib/gimli/src/read/util.rs @@ -0,0 +1,250 @@ +#[cfg(feature = "read")] +use alloc::boxed::Box; +#[cfg(feature = "read")] +use alloc::vec::Vec; +use core::fmt; +use core::mem::MaybeUninit; +use core::ops; +use core::ptr; +use core::slice; + +mod sealed { + // SAFETY: Implementer must not modify the content in storage. + pub unsafe trait Sealed { + type Storage; + + fn new_storage() -> Self::Storage; + + fn grow(_storage: &mut Self::Storage, _additional: usize) -> Result<(), CapacityFull> { + Err(CapacityFull) + } + } + + #[derive(Clone, Copy, Debug)] + pub struct CapacityFull; +} + +use sealed::*; + +/// Marker trait for types that can be used as backing storage when a growable array type is needed. +/// +/// This trait is sealed and cannot be implemented for types outside this crate. +pub trait ArrayLike: Sealed { + /// Type of the elements being stored. + type Item; + + #[doc(hidden)] + fn as_slice(storage: &Self::Storage) -> &[MaybeUninit]; + + #[doc(hidden)] + fn as_mut_slice(storage: &mut Self::Storage) -> &mut [MaybeUninit]; +} + +// Use macro since const generics can't be used due to MSRV. +macro_rules! impl_array { + () => {}; + ($n:literal $($rest:tt)*) => { + // SAFETY: does not modify the content in storage. + unsafe impl Sealed for [T; $n] { + type Storage = [MaybeUninit; $n]; + + fn new_storage() -> Self::Storage { + // SAFETY: An uninitialized `[MaybeUninit<_>; _]` is valid. + unsafe { MaybeUninit::uninit().assume_init() } + } + } + + impl ArrayLike for [T; $n] { + type Item = T; + + fn as_slice(storage: &Self::Storage) -> &[MaybeUninit] { + storage + } + + fn as_mut_slice(storage: &mut Self::Storage) -> &mut [MaybeUninit] { + storage + } + } + + impl_array!($($rest)*); + } +} + +impl_array!(0 1 2 3 4 8 16 32 64 128 192); + +#[cfg(feature = "read")] +unsafe impl Sealed for Vec { + type Storage = Box<[MaybeUninit]>; + + fn new_storage() -> Self::Storage { + Box::new([]) + } + + fn grow(storage: &mut Self::Storage, additional: usize) -> Result<(), CapacityFull> { + let mut vec: Vec<_> = core::mem::replace(storage, Box::new([])).into(); + vec.reserve(additional); + // SAFETY: This is a `Vec` of `MaybeUninit`. + unsafe { vec.set_len(vec.capacity()) }; + *storage = vec.into_boxed_slice(); + Ok(()) + } +} + +#[cfg(feature = "read")] +impl ArrayLike for Vec { + type Item = T; + + fn as_slice(storage: &Self::Storage) -> &[MaybeUninit] { + storage + } + + fn as_mut_slice(storage: &mut Self::Storage) -> &mut [MaybeUninit] { + storage + } +} + +pub(crate) struct ArrayVec { + storage: A::Storage, + len: usize, +} + +impl ArrayVec { + pub fn new() -> Self { + Self { + storage: A::new_storage(), + len: 0, + } + } + + pub fn clear(&mut self) { + let ptr: *mut [A::Item] = &mut **self; + // Set length first so the type invariant is upheld even if `drop_in_place` panicks. + self.len = 0; + // SAFETY: `ptr` contains valid elements only and we "forget" them by setting the length. + unsafe { ptr::drop_in_place(ptr) }; + } + + pub fn try_push(&mut self, value: A::Item) -> Result<(), CapacityFull> { + let mut storage = A::as_mut_slice(&mut self.storage); + if self.len >= storage.len() { + A::grow(&mut self.storage, 1)?; + storage = A::as_mut_slice(&mut self.storage); + } + + storage[self.len] = MaybeUninit::new(value); + self.len += 1; + Ok(()) + } + + pub fn try_insert(&mut self, index: usize, element: A::Item) -> Result<(), CapacityFull> { + assert!(index <= self.len); + + let mut storage = A::as_mut_slice(&mut self.storage); + if self.len >= storage.len() { + A::grow(&mut self.storage, 1)?; + storage = A::as_mut_slice(&mut self.storage); + } + + // SAFETY: storage[index] is filled later. + unsafe { + let p = storage.as_mut_ptr().add(index); + core::ptr::copy(p as *const _, p.add(1), self.len - index); + } + storage[index] = MaybeUninit::new(element); + self.len += 1; + Ok(()) + } + + pub fn pop(&mut self) -> Option { + if self.len == 0 { + None + } else { + self.len -= 1; + // SAFETY: this element is valid and we "forget" it by setting the length. + Some(unsafe { A::as_slice(&mut self.storage)[self.len].as_ptr().read() }) + } + } + + pub fn swap_remove(&mut self, index: usize) -> A::Item { + assert!(self.len > 0); + A::as_mut_slice(&mut self.storage).swap(index, self.len - 1); + self.pop().unwrap() + } +} + +#[cfg(feature = "read")] +impl ArrayVec> { + pub fn into_vec(mut self) -> Vec { + let len = core::mem::replace(&mut self.len, 0); + let storage = core::mem::replace(&mut self.storage, Box::new([])); + let slice = Box::leak(storage); + debug_assert!(len <= slice.len()); + // SAFETY: valid elements. + unsafe { Vec::from_raw_parts(slice.as_mut_ptr() as *mut T, len, slice.len()) } + } +} + +impl Drop for ArrayVec { + fn drop(&mut self) { + self.clear(); + } +} + +impl Default for ArrayVec { + fn default() -> Self { + Self::new() + } +} + +impl ops::Deref for ArrayVec { + type Target = [A::Item]; + + fn deref(&self) -> &[A::Item] { + let slice = &A::as_slice(&self.storage); + debug_assert!(self.len <= slice.len()); + // SAFETY: valid elements. + unsafe { slice::from_raw_parts(slice.as_ptr() as _, self.len) } + } +} + +impl ops::DerefMut for ArrayVec { + fn deref_mut(&mut self) -> &mut [A::Item] { + let slice = &mut A::as_mut_slice(&mut self.storage); + debug_assert!(self.len <= slice.len()); + // SAFETY: valid elements. + unsafe { slice::from_raw_parts_mut(slice.as_mut_ptr() as _, self.len) } + } +} + +impl Clone for ArrayVec +where + A::Item: Clone, +{ + fn clone(&self) -> Self { + let mut new = Self::default(); + for value in &**self { + new.try_push(value.clone()).unwrap(); + } + new + } +} + +impl PartialEq for ArrayVec +where + A::Item: PartialEq, +{ + fn eq(&self, other: &Self) -> bool { + **self == **other + } +} + +impl Eq for ArrayVec where A::Item: Eq {} + +impl fmt::Debug for ArrayVec +where + A::Item: fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} diff --git a/crux-mir/lib/gimli/src/read/value.rs b/crux-mir/lib/gimli/src/read/value.rs new file mode 100644 index 000000000..6f43ebb26 --- /dev/null +++ b/crux-mir/lib/gimli/src/read/value.rs @@ -0,0 +1,1621 @@ +//! Definitions for values used in DWARF expressions. + +use crate::constants; +#[cfg(feature = "read")] +use crate::read::{AttributeValue, DebuggingInformationEntry}; +use crate::read::{Error, Reader, Result}; + +/// Convert a u64 to an i64, with sign extension if required. +/// +/// This is primarily used when needing to treat `Value::Generic` +/// as a signed value. +#[inline] +fn sign_extend(value: u64, mask: u64) -> i64 { + let value = (value & mask) as i64; + let sign = ((mask >> 1) + 1) as i64; + (value ^ sign).wrapping_sub(sign) +} + +#[inline] +fn mask_bit_size(addr_mask: u64) -> u32 { + 64 - addr_mask.leading_zeros() +} + +/// The type of an entry on the DWARF stack. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ValueType { + /// The generic type, which is address-sized and of unspecified sign, + /// as specified in the DWARF 5 standard, section 2.5.1. + /// This type is also used to represent address base types. + Generic, + /// Signed 8-bit integer type. + I8, + /// Unsigned 8-bit integer type. + U8, + /// Signed 16-bit integer type. + I16, + /// Unsigned 16-bit integer type. + U16, + /// Signed 32-bit integer type. + I32, + /// Unsigned 32-bit integer type. + U32, + /// Signed 64-bit integer type. + I64, + /// Unsigned 64-bit integer type. + U64, + /// 32-bit floating point type. + F32, + /// 64-bit floating point type. + F64, +} + +/// The value of an entry on the DWARF stack. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Value { + /// A generic value, which is address-sized and of unspecified sign. + Generic(u64), + /// A signed 8-bit integer value. + I8(i8), + /// An unsigned 8-bit integer value. + U8(u8), + /// A signed 16-bit integer value. + I16(i16), + /// An unsigned 16-bit integer value. + U16(u16), + /// A signed 32-bit integer value. + I32(i32), + /// An unsigned 32-bit integer value. + U32(u32), + /// A signed 64-bit integer value. + I64(i64), + /// An unsigned 64-bit integer value. + U64(u64), + /// A 32-bit floating point value. + F32(f32), + /// A 64-bit floating point value. + F64(f64), +} + +impl ValueType { + /// The size in bits of a value for this type. + pub fn bit_size(self, addr_mask: u64) -> u32 { + match self { + ValueType::Generic => mask_bit_size(addr_mask), + ValueType::I8 | ValueType::U8 => 8, + ValueType::I16 | ValueType::U16 => 16, + ValueType::I32 | ValueType::U32 | ValueType::F32 => 32, + ValueType::I64 | ValueType::U64 | ValueType::F64 => 64, + } + } + + /// Construct a `ValueType` from the attributes of a base type DIE. + pub fn from_encoding(encoding: constants::DwAte, byte_size: u64) -> Option { + Some(match (encoding, byte_size) { + (constants::DW_ATE_signed, 1) => ValueType::I8, + (constants::DW_ATE_signed, 2) => ValueType::I16, + (constants::DW_ATE_signed, 4) => ValueType::I32, + (constants::DW_ATE_signed, 8) => ValueType::I64, + (constants::DW_ATE_unsigned, 1) => ValueType::U8, + (constants::DW_ATE_unsigned, 2) => ValueType::U16, + (constants::DW_ATE_unsigned, 4) => ValueType::U32, + (constants::DW_ATE_unsigned, 8) => ValueType::U64, + (constants::DW_ATE_float, 4) => ValueType::F32, + (constants::DW_ATE_float, 8) => ValueType::F64, + _ => return None, + }) + } + + /// Construct a `ValueType` from a base type DIE. + #[cfg(feature = "read")] + pub fn from_entry( + entry: &DebuggingInformationEntry, + ) -> Result> { + if entry.tag() != constants::DW_TAG_base_type { + return Ok(None); + } + let mut encoding = None; + let mut byte_size = None; + let mut endianity = constants::DW_END_default; + let mut attrs = entry.attrs(); + while let Some(attr) = attrs.next()? { + match attr.name() { + constants::DW_AT_byte_size => byte_size = attr.udata_value(), + constants::DW_AT_encoding => { + if let AttributeValue::Encoding(x) = attr.value() { + encoding = Some(x); + } + } + constants::DW_AT_endianity => { + if let AttributeValue::Endianity(x) = attr.value() { + endianity = x; + } + } + _ => {} + } + } + + if endianity != constants::DW_END_default { + // TODO: we could check if it matches the reader endianity, + // but normally it would use DW_END_default in that case. + return Ok(None); + } + + if let (Some(encoding), Some(byte_size)) = (encoding, byte_size) { + Ok(ValueType::from_encoding(encoding, byte_size)) + } else { + Ok(None) + } + } +} + +impl Value { + /// Return the `ValueType` corresponding to this `Value`. + pub fn value_type(&self) -> ValueType { + match *self { + Value::Generic(_) => ValueType::Generic, + Value::I8(_) => ValueType::I8, + Value::U8(_) => ValueType::U8, + Value::I16(_) => ValueType::I16, + Value::U16(_) => ValueType::U16, + Value::I32(_) => ValueType::I32, + Value::U32(_) => ValueType::U32, + Value::I64(_) => ValueType::I64, + Value::U64(_) => ValueType::U64, + Value::F32(_) => ValueType::F32, + Value::F64(_) => ValueType::F64, + } + } + + /// Read a `Value` with the given `value_type` from a `Reader`. + pub fn parse(value_type: ValueType, mut bytes: R) -> Result { + let value = match value_type { + ValueType::I8 => Value::I8(bytes.read_i8()?), + ValueType::U8 => Value::U8(bytes.read_u8()?), + ValueType::I16 => Value::I16(bytes.read_i16()?), + ValueType::U16 => Value::U16(bytes.read_u16()?), + ValueType::I32 => Value::I32(bytes.read_i32()?), + ValueType::U32 => Value::U32(bytes.read_u32()?), + ValueType::I64 => Value::I64(bytes.read_i64()?), + ValueType::U64 => Value::U64(bytes.read_u64()?), + ValueType::F32 => Value::F32(bytes.read_f32()?), + ValueType::F64 => Value::F64(bytes.read_f64()?), + _ => return Err(Error::UnsupportedTypeOperation), + }; + Ok(value) + } + + /// Convert a `Value` to a `u64`. + /// + /// The `ValueType` of `self` must be integral. + /// Values are sign extended if the source value is signed. + pub fn to_u64(self, addr_mask: u64) -> Result { + let value = match self { + Value::Generic(value) => value & addr_mask, + Value::I8(value) => value as u64, + Value::U8(value) => u64::from(value), + Value::I16(value) => value as u64, + Value::U16(value) => u64::from(value), + Value::I32(value) => value as u64, + Value::U32(value) => u64::from(value), + Value::I64(value) => value as u64, + Value::U64(value) => value as u64, + _ => return Err(Error::IntegralTypeRequired), + }; + Ok(value) + } + + /// Create a `Value` with the given `value_type` from a `u64` value. + /// + /// The `value_type` may be integral or floating point. + /// The result is truncated if the `u64` value does + /// not fit the bounds of the `value_type`. + pub fn from_u64(value_type: ValueType, value: u64) -> Result { + let value = match value_type { + ValueType::Generic => Value::Generic(value), + ValueType::I8 => Value::I8(value as i8), + ValueType::U8 => Value::U8(value as u8), + ValueType::I16 => Value::I16(value as i16), + ValueType::U16 => Value::U16(value as u16), + ValueType::I32 => Value::I32(value as i32), + ValueType::U32 => Value::U32(value as u32), + ValueType::I64 => Value::I64(value as i64), + ValueType::U64 => Value::U64(value), + ValueType::F32 => Value::F32(value as f32), + ValueType::F64 => Value::F64(value as f64), + }; + Ok(value) + } + + /// Create a `Value` with the given `value_type` from a `f32` value. + /// + /// The `value_type` may be integral or floating point. + /// The result is not defined if the `f32` value does + /// not fit the bounds of the `value_type`. + fn from_f32(value_type: ValueType, value: f32) -> Result { + let value = match value_type { + ValueType::Generic => Value::Generic(value as u64), + ValueType::I8 => Value::I8(value as i8), + ValueType::U8 => Value::U8(value as u8), + ValueType::I16 => Value::I16(value as i16), + ValueType::U16 => Value::U16(value as u16), + ValueType::I32 => Value::I32(value as i32), + ValueType::U32 => Value::U32(value as u32), + ValueType::I64 => Value::I64(value as i64), + ValueType::U64 => Value::U64(value as u64), + ValueType::F32 => Value::F32(value), + ValueType::F64 => Value::F64(f64::from(value)), + }; + Ok(value) + } + + /// Create a `Value` with the given `value_type` from a `f64` value. + /// + /// The `value_type` may be integral or floating point. + /// The result is not defined if the `f64` value does + /// not fit the bounds of the `value_type`. + fn from_f64(value_type: ValueType, value: f64) -> Result { + let value = match value_type { + ValueType::Generic => Value::Generic(value as u64), + ValueType::I8 => Value::I8(value as i8), + ValueType::U8 => Value::U8(value as u8), + ValueType::I16 => Value::I16(value as i16), + ValueType::U16 => Value::U16(value as u16), + ValueType::I32 => Value::I32(value as i32), + ValueType::U32 => Value::U32(value as u32), + ValueType::I64 => Value::I64(value as i64), + ValueType::U64 => Value::U64(value as u64), + ValueType::F32 => Value::F32(value as f32), + ValueType::F64 => Value::F64(value), + }; + Ok(value) + } + + /// Convert a `Value` to the given `value_type`. + /// + /// When converting between integral types, the result is truncated + /// if the source value does not fit the bounds of the `value_type`. + /// When converting from floating point types, the result is not defined + /// if the source value does not fit the bounds of the `value_type`. + /// + /// This corresponds to the DWARF `DW_OP_convert` operation. + pub fn convert(self, value_type: ValueType, addr_mask: u64) -> Result { + match self { + Value::F32(value) => Value::from_f32(value_type, value), + Value::F64(value) => Value::from_f64(value_type, value), + _ => Value::from_u64(value_type, self.to_u64(addr_mask)?), + } + } + + /// Reinterpret the bits in a `Value` as the given `value_type`. + /// + /// The source and result value types must have equal sizes. + /// + /// This corresponds to the DWARF `DW_OP_reinterpret` operation. + pub fn reinterpret(self, value_type: ValueType, addr_mask: u64) -> Result { + if self.value_type().bit_size(addr_mask) != value_type.bit_size(addr_mask) { + return Err(Error::TypeMismatch); + } + let bits = match self { + Value::Generic(value) => value, + Value::I8(value) => value as u64, + Value::U8(value) => u64::from(value), + Value::I16(value) => value as u64, + Value::U16(value) => u64::from(value), + Value::I32(value) => value as u64, + Value::U32(value) => u64::from(value), + Value::I64(value) => value as u64, + Value::U64(value) => value, + Value::F32(value) => u64::from(f32::to_bits(value)), + Value::F64(value) => f64::to_bits(value), + }; + let value = match value_type { + ValueType::Generic => Value::Generic(bits), + ValueType::I8 => Value::I8(bits as i8), + ValueType::U8 => Value::U8(bits as u8), + ValueType::I16 => Value::I16(bits as i16), + ValueType::U16 => Value::U16(bits as u16), + ValueType::I32 => Value::I32(bits as i32), + ValueType::U32 => Value::U32(bits as u32), + ValueType::I64 => Value::I64(bits as i64), + ValueType::U64 => Value::U64(bits), + ValueType::F32 => Value::F32(f32::from_bits(bits as u32)), + ValueType::F64 => Value::F64(f64::from_bits(bits)), + }; + Ok(value) + } + + /// Perform an absolute value operation. + /// + /// If the value type is `Generic`, then it is interpreted as a signed value. + /// + /// This corresponds to the DWARF `DW_OP_abs` operation. + pub fn abs(self, addr_mask: u64) -> Result { + // wrapping_abs() can be used because DWARF specifies that the result is undefined + // for negative minimal values. + let value = match self { + Value::Generic(value) => { + Value::Generic(sign_extend(value, addr_mask).wrapping_abs() as u64) + } + Value::I8(value) => Value::I8(value.wrapping_abs()), + Value::I16(value) => Value::I16(value.wrapping_abs()), + Value::I32(value) => Value::I32(value.wrapping_abs()), + Value::I64(value) => Value::I64(value.wrapping_abs()), + // f32/f64::abs() is not available in libcore + Value::F32(value) => Value::F32(if value < 0. { -value } else { value }), + Value::F64(value) => Value::F64(if value < 0. { -value } else { value }), + Value::U8(_) | Value::U16(_) | Value::U32(_) | Value::U64(_) => self, + }; + Ok(value) + } + + /// Perform a negation operation. + /// + /// If the value type is `Generic`, then it is interpreted as a signed value. + /// + /// This corresponds to the DWARF `DW_OP_neg` operation. + pub fn neg(self, addr_mask: u64) -> Result { + // wrapping_neg() can be used because DWARF specifies that the result is undefined + // for negative minimal values. + let value = match self { + Value::Generic(value) => { + Value::Generic(sign_extend(value, addr_mask).wrapping_neg() as u64) + } + Value::I8(value) => Value::I8(value.wrapping_neg()), + Value::I16(value) => Value::I16(value.wrapping_neg()), + Value::I32(value) => Value::I32(value.wrapping_neg()), + Value::I64(value) => Value::I64(value.wrapping_neg()), + Value::F32(value) => Value::F32(-value), + Value::F64(value) => Value::F64(-value), + // It's unclear if these should implicity convert to a signed value. + // For now, we don't support them. + Value::U8(_) | Value::U16(_) | Value::U32(_) | Value::U64(_) => { + return Err(Error::UnsupportedTypeOperation); + } + }; + Ok(value) + } + + /// Perform an addition operation. + /// + /// This operation requires matching types. + /// + /// This corresponds to the DWARF `DW_OP_plus` operation. + pub fn add(self, rhs: Value, addr_mask: u64) -> Result { + let value = match (self, rhs) { + (Value::Generic(v1), Value::Generic(v2)) => { + Value::Generic(v1.wrapping_add(v2) & addr_mask) + } + (Value::I8(v1), Value::I8(v2)) => Value::I8(v1.wrapping_add(v2)), + (Value::U8(v1), Value::U8(v2)) => Value::U8(v1.wrapping_add(v2)), + (Value::I16(v1), Value::I16(v2)) => Value::I16(v1.wrapping_add(v2)), + (Value::U16(v1), Value::U16(v2)) => Value::U16(v1.wrapping_add(v2)), + (Value::I32(v1), Value::I32(v2)) => Value::I32(v1.wrapping_add(v2)), + (Value::U32(v1), Value::U32(v2)) => Value::U32(v1.wrapping_add(v2)), + (Value::I64(v1), Value::I64(v2)) => Value::I64(v1.wrapping_add(v2)), + (Value::U64(v1), Value::U64(v2)) => Value::U64(v1.wrapping_add(v2)), + (Value::F32(v1), Value::F32(v2)) => Value::F32(v1 + v2), + (Value::F64(v1), Value::F64(v2)) => Value::F64(v1 + v2), + _ => return Err(Error::TypeMismatch), + }; + Ok(value) + } + + /// Perform a subtraction operation. + /// + /// This operation requires matching types. + /// + /// This corresponds to the DWARF `DW_OP_minus` operation. + pub fn sub(self, rhs: Value, addr_mask: u64) -> Result { + let value = match (self, rhs) { + (Value::Generic(v1), Value::Generic(v2)) => { + Value::Generic(v1.wrapping_sub(v2) & addr_mask) + } + (Value::I8(v1), Value::I8(v2)) => Value::I8(v1.wrapping_sub(v2)), + (Value::U8(v1), Value::U8(v2)) => Value::U8(v1.wrapping_sub(v2)), + (Value::I16(v1), Value::I16(v2)) => Value::I16(v1.wrapping_sub(v2)), + (Value::U16(v1), Value::U16(v2)) => Value::U16(v1.wrapping_sub(v2)), + (Value::I32(v1), Value::I32(v2)) => Value::I32(v1.wrapping_sub(v2)), + (Value::U32(v1), Value::U32(v2)) => Value::U32(v1.wrapping_sub(v2)), + (Value::I64(v1), Value::I64(v2)) => Value::I64(v1.wrapping_sub(v2)), + (Value::U64(v1), Value::U64(v2)) => Value::U64(v1.wrapping_sub(v2)), + (Value::F32(v1), Value::F32(v2)) => Value::F32(v1 - v2), + (Value::F64(v1), Value::F64(v2)) => Value::F64(v1 - v2), + _ => return Err(Error::TypeMismatch), + }; + Ok(value) + } + + /// Perform a multiplication operation. + /// + /// This operation requires matching types. + /// + /// This corresponds to the DWARF `DW_OP_mul` operation. + pub fn mul(self, rhs: Value, addr_mask: u64) -> Result { + let value = match (self, rhs) { + (Value::Generic(v1), Value::Generic(v2)) => { + Value::Generic(v1.wrapping_mul(v2) & addr_mask) + } + (Value::I8(v1), Value::I8(v2)) => Value::I8(v1.wrapping_mul(v2)), + (Value::U8(v1), Value::U8(v2)) => Value::U8(v1.wrapping_mul(v2)), + (Value::I16(v1), Value::I16(v2)) => Value::I16(v1.wrapping_mul(v2)), + (Value::U16(v1), Value::U16(v2)) => Value::U16(v1.wrapping_mul(v2)), + (Value::I32(v1), Value::I32(v2)) => Value::I32(v1.wrapping_mul(v2)), + (Value::U32(v1), Value::U32(v2)) => Value::U32(v1.wrapping_mul(v2)), + (Value::I64(v1), Value::I64(v2)) => Value::I64(v1.wrapping_mul(v2)), + (Value::U64(v1), Value::U64(v2)) => Value::U64(v1.wrapping_mul(v2)), + (Value::F32(v1), Value::F32(v2)) => Value::F32(v1 * v2), + (Value::F64(v1), Value::F64(v2)) => Value::F64(v1 * v2), + _ => return Err(Error::TypeMismatch), + }; + Ok(value) + } + + /// Perform a division operation. + /// + /// This operation requires matching types. + /// If the value type is `Generic`, then it is interpreted as a signed value. + /// + /// This corresponds to the DWARF `DW_OP_div` operation. + pub fn div(self, rhs: Value, addr_mask: u64) -> Result { + match rhs { + Value::Generic(v2) if sign_extend(v2, addr_mask) == 0 => { + return Err(Error::DivisionByZero); + } + Value::I8(0) + | Value::U8(0) + | Value::I16(0) + | Value::U16(0) + | Value::I32(0) + | Value::U32(0) + | Value::I64(0) + | Value::U64(0) => { + return Err(Error::DivisionByZero); + } + _ => {} + } + let value = match (self, rhs) { + (Value::Generic(v1), Value::Generic(v2)) => { + // Signed division + Value::Generic( + sign_extend(v1, addr_mask).wrapping_div(sign_extend(v2, addr_mask)) as u64, + ) + } + (Value::I8(v1), Value::I8(v2)) => Value::I8(v1.wrapping_div(v2)), + (Value::U8(v1), Value::U8(v2)) => Value::U8(v1.wrapping_div(v2)), + (Value::I16(v1), Value::I16(v2)) => Value::I16(v1.wrapping_div(v2)), + (Value::U16(v1), Value::U16(v2)) => Value::U16(v1.wrapping_div(v2)), + (Value::I32(v1), Value::I32(v2)) => Value::I32(v1.wrapping_div(v2)), + (Value::U32(v1), Value::U32(v2)) => Value::U32(v1.wrapping_div(v2)), + (Value::I64(v1), Value::I64(v2)) => Value::I64(v1.wrapping_div(v2)), + (Value::U64(v1), Value::U64(v2)) => Value::U64(v1.wrapping_div(v2)), + (Value::F32(v1), Value::F32(v2)) => Value::F32(v1 / v2), + (Value::F64(v1), Value::F64(v2)) => Value::F64(v1 / v2), + _ => return Err(Error::TypeMismatch), + }; + Ok(value) + } + + /// Perform a remainder operation. + /// + /// This operation requires matching integral types. + /// If the value type is `Generic`, then it is interpreted as an unsigned value. + /// + /// This corresponds to the DWARF `DW_OP_mod` operation. + pub fn rem(self, rhs: Value, addr_mask: u64) -> Result { + match rhs { + Value::Generic(rhs) if (rhs & addr_mask) == 0 => { + return Err(Error::DivisionByZero); + } + Value::I8(0) + | Value::U8(0) + | Value::I16(0) + | Value::U16(0) + | Value::I32(0) + | Value::U32(0) + | Value::I64(0) + | Value::U64(0) => { + return Err(Error::DivisionByZero); + } + _ => {} + } + let value = match (self, rhs) { + (Value::Generic(v1), Value::Generic(v2)) => { + // Unsigned modulus + Value::Generic((v1 & addr_mask).wrapping_rem(v2 & addr_mask)) + } + (Value::I8(v1), Value::I8(v2)) => Value::I8(v1.wrapping_rem(v2)), + (Value::U8(v1), Value::U8(v2)) => Value::U8(v1.wrapping_rem(v2)), + (Value::I16(v1), Value::I16(v2)) => Value::I16(v1.wrapping_rem(v2)), + (Value::U16(v1), Value::U16(v2)) => Value::U16(v1.wrapping_rem(v2)), + (Value::I32(v1), Value::I32(v2)) => Value::I32(v1.wrapping_rem(v2)), + (Value::U32(v1), Value::U32(v2)) => Value::U32(v1.wrapping_rem(v2)), + (Value::I64(v1), Value::I64(v2)) => Value::I64(v1.wrapping_rem(v2)), + (Value::U64(v1), Value::U64(v2)) => Value::U64(v1.wrapping_rem(v2)), + (Value::F32(_), Value::F32(_)) => return Err(Error::IntegralTypeRequired), + (Value::F64(_), Value::F64(_)) => return Err(Error::IntegralTypeRequired), + _ => return Err(Error::TypeMismatch), + }; + Ok(value) + } + + /// Perform a bitwise not operation. + /// + /// This operation requires matching integral types. + /// + /// This corresponds to the DWARF `DW_OP_not` operation. + pub fn not(self, addr_mask: u64) -> Result { + let value_type = self.value_type(); + let v = self.to_u64(addr_mask)?; + Value::from_u64(value_type, !v) + } + + /// Perform a bitwise and operation. + /// + /// This operation requires matching integral types. + /// + /// This corresponds to the DWARF `DW_OP_and` operation. + pub fn and(self, rhs: Value, addr_mask: u64) -> Result { + let value_type = self.value_type(); + if value_type != rhs.value_type() { + return Err(Error::TypeMismatch); + } + let v1 = self.to_u64(addr_mask)?; + let v2 = rhs.to_u64(addr_mask)?; + Value::from_u64(value_type, v1 & v2) + } + + /// Perform a bitwise or operation. + /// + /// This operation requires matching integral types. + /// + /// This corresponds to the DWARF `DW_OP_or` operation. + pub fn or(self, rhs: Value, addr_mask: u64) -> Result { + let value_type = self.value_type(); + if value_type != rhs.value_type() { + return Err(Error::TypeMismatch); + } + let v1 = self.to_u64(addr_mask)?; + let v2 = rhs.to_u64(addr_mask)?; + Value::from_u64(value_type, v1 | v2) + } + + /// Perform a bitwise exclusive-or operation. + /// + /// This operation requires matching integral types. + /// + /// This corresponds to the DWARF `DW_OP_xor` operation. + pub fn xor(self, rhs: Value, addr_mask: u64) -> Result { + let value_type = self.value_type(); + if value_type != rhs.value_type() { + return Err(Error::TypeMismatch); + } + let v1 = self.to_u64(addr_mask)?; + let v2 = rhs.to_u64(addr_mask)?; + Value::from_u64(value_type, v1 ^ v2) + } + + /// Convert value to bit length suitable for a shift operation. + /// + /// If the value is negative then an error is returned. + fn shift_length(self) -> Result { + let value = match self { + Value::Generic(value) => value, + Value::I8(value) if value >= 0 => value as u64, + Value::U8(value) => u64::from(value), + Value::I16(value) if value >= 0 => value as u64, + Value::U16(value) => u64::from(value), + Value::I32(value) if value >= 0 => value as u64, + Value::U32(value) => u64::from(value), + Value::I64(value) if value >= 0 => value as u64, + Value::U64(value) => value, + _ => return Err(Error::InvalidShiftExpression), + }; + Ok(value) + } + + /// Perform a shift left operation. + /// + /// This operation requires integral types. + /// If the shift length exceeds the type size, then 0 is returned. + /// If the shift length is negative then an error is returned. + /// + /// This corresponds to the DWARF `DW_OP_shl` operation. + pub fn shl(self, rhs: Value, addr_mask: u64) -> Result { + let v2 = rhs.shift_length()?; + let value = match self { + Value::Generic(v1) => Value::Generic(if v2 >= u64::from(mask_bit_size(addr_mask)) { + 0 + } else { + (v1 & addr_mask) << v2 + }), + Value::I8(v1) => Value::I8(if v2 >= 8 { 0 } else { v1 << v2 }), + Value::U8(v1) => Value::U8(if v2 >= 8 { 0 } else { v1 << v2 }), + Value::I16(v1) => Value::I16(if v2 >= 16 { 0 } else { v1 << v2 }), + Value::U16(v1) => Value::U16(if v2 >= 16 { 0 } else { v1 << v2 }), + Value::I32(v1) => Value::I32(if v2 >= 32 { 0 } else { v1 << v2 }), + Value::U32(v1) => Value::U32(if v2 >= 32 { 0 } else { v1 << v2 }), + Value::I64(v1) => Value::I64(if v2 >= 64 { 0 } else { v1 << v2 }), + Value::U64(v1) => Value::U64(if v2 >= 64 { 0 } else { v1 << v2 }), + _ => return Err(Error::IntegralTypeRequired), + }; + Ok(value) + } + + /// Perform a logical shift right operation. + /// + /// This operation requires an unsigned integral type for the value. + /// If the value type is `Generic`, then it is interpreted as an unsigned value. + /// + /// This operation requires an integral type for the shift length. + /// If the shift length exceeds the type size, then 0 is returned. + /// If the shift length is negative then an error is returned. + /// + /// This corresponds to the DWARF `DW_OP_shr` operation. + pub fn shr(self, rhs: Value, addr_mask: u64) -> Result { + let v2 = rhs.shift_length()?; + let value = match self { + Value::Generic(v1) => Value::Generic(if v2 >= u64::from(mask_bit_size(addr_mask)) { + 0 + } else { + (v1 & addr_mask) >> v2 + }), + Value::U8(v1) => Value::U8(if v2 >= 8 { 0 } else { v1 >> v2 }), + Value::U16(v1) => Value::U16(if v2 >= 16 { 0 } else { v1 >> v2 }), + Value::U32(v1) => Value::U32(if v2 >= 32 { 0 } else { v1 >> v2 }), + Value::U64(v1) => Value::U64(if v2 >= 64 { 0 } else { v1 >> v2 }), + // It's unclear if signed values should implicity convert to an unsigned value. + // For now, we don't support them. + Value::I8(_) | Value::I16(_) | Value::I32(_) | Value::I64(_) => { + return Err(Error::UnsupportedTypeOperation); + } + _ => return Err(Error::IntegralTypeRequired), + }; + Ok(value) + } + + /// Perform an arithmetic shift right operation. + /// + /// This operation requires a signed integral type for the value. + /// If the value type is `Generic`, then it is interpreted as a signed value. + /// + /// This operation requires an integral type for the shift length. + /// If the shift length exceeds the type size, then 0 is returned for positive values, + /// and -1 is returned for negative values. + /// If the shift length is negative then an error is returned. + /// + /// This corresponds to the DWARF `DW_OP_shra` operation. + pub fn shra(self, rhs: Value, addr_mask: u64) -> Result { + let v2 = rhs.shift_length()?; + let value = match self { + Value::Generic(v1) => { + let v1 = sign_extend(v1, addr_mask); + let value = if v2 >= u64::from(mask_bit_size(addr_mask)) { + if v1 < 0 { + !0 + } else { + 0 + } + } else { + (v1 >> v2) as u64 + }; + Value::Generic(value) + } + Value::I8(v1) => Value::I8(if v2 >= 8 { + if v1 < 0 { + !0 + } else { + 0 + } + } else { + v1 >> v2 + }), + Value::I16(v1) => Value::I16(if v2 >= 16 { + if v1 < 0 { + !0 + } else { + 0 + } + } else { + v1 >> v2 + }), + Value::I32(v1) => Value::I32(if v2 >= 32 { + if v1 < 0 { + !0 + } else { + 0 + } + } else { + v1 >> v2 + }), + Value::I64(v1) => Value::I64(if v2 >= 64 { + if v1 < 0 { + !0 + } else { + 0 + } + } else { + v1 >> v2 + }), + // It's unclear if unsigned values should implicity convert to a signed value. + // For now, we don't support them. + Value::U8(_) | Value::U16(_) | Value::U32(_) | Value::U64(_) => { + return Err(Error::UnsupportedTypeOperation); + } + _ => return Err(Error::IntegralTypeRequired), + }; + Ok(value) + } + + /// Perform the `==` relational operation. + /// + /// This operation requires matching integral types. + /// If the value type is `Generic`, then it is interpreted as a signed value. + /// + /// This corresponds to the DWARF `DW_OP_eq` operation. + pub fn eq(self, rhs: Value, addr_mask: u64) -> Result { + let value = match (self, rhs) { + (Value::Generic(v1), Value::Generic(v2)) => { + sign_extend(v1, addr_mask) == sign_extend(v2, addr_mask) + } + (Value::I8(v1), Value::I8(v2)) => v1 == v2, + (Value::U8(v1), Value::U8(v2)) => v1 == v2, + (Value::I16(v1), Value::I16(v2)) => v1 == v2, + (Value::U16(v1), Value::U16(v2)) => v1 == v2, + (Value::I32(v1), Value::I32(v2)) => v1 == v2, + (Value::U32(v1), Value::U32(v2)) => v1 == v2, + (Value::I64(v1), Value::I64(v2)) => v1 == v2, + (Value::U64(v1), Value::U64(v2)) => v1 == v2, + (Value::F32(v1), Value::F32(v2)) => v1 == v2, + (Value::F64(v1), Value::F64(v2)) => v1 == v2, + _ => return Err(Error::TypeMismatch), + }; + Ok(Value::Generic(value as u64)) + } + + /// Perform the `>=` relational operation. + /// + /// This operation requires matching integral types. + /// If the value type is `Generic`, then it is interpreted as a signed value. + /// + /// This corresponds to the DWARF `DW_OP_ge` operation. + pub fn ge(self, rhs: Value, addr_mask: u64) -> Result { + let value = match (self, rhs) { + (Value::Generic(v1), Value::Generic(v2)) => { + sign_extend(v1, addr_mask) >= sign_extend(v2, addr_mask) + } + (Value::I8(v1), Value::I8(v2)) => v1 >= v2, + (Value::U8(v1), Value::U8(v2)) => v1 >= v2, + (Value::I16(v1), Value::I16(v2)) => v1 >= v2, + (Value::U16(v1), Value::U16(v2)) => v1 >= v2, + (Value::I32(v1), Value::I32(v2)) => v1 >= v2, + (Value::U32(v1), Value::U32(v2)) => v1 >= v2, + (Value::I64(v1), Value::I64(v2)) => v1 >= v2, + (Value::U64(v1), Value::U64(v2)) => v1 >= v2, + (Value::F32(v1), Value::F32(v2)) => v1 >= v2, + (Value::F64(v1), Value::F64(v2)) => v1 >= v2, + _ => return Err(Error::TypeMismatch), + }; + Ok(Value::Generic(value as u64)) + } + + /// Perform the `>` relational operation. + /// + /// This operation requires matching integral types. + /// If the value type is `Generic`, then it is interpreted as a signed value. + /// + /// This corresponds to the DWARF `DW_OP_gt` operation. + pub fn gt(self, rhs: Value, addr_mask: u64) -> Result { + let value = match (self, rhs) { + (Value::Generic(v1), Value::Generic(v2)) => { + sign_extend(v1, addr_mask) > sign_extend(v2, addr_mask) + } + (Value::I8(v1), Value::I8(v2)) => v1 > v2, + (Value::U8(v1), Value::U8(v2)) => v1 > v2, + (Value::I16(v1), Value::I16(v2)) => v1 > v2, + (Value::U16(v1), Value::U16(v2)) => v1 > v2, + (Value::I32(v1), Value::I32(v2)) => v1 > v2, + (Value::U32(v1), Value::U32(v2)) => v1 > v2, + (Value::I64(v1), Value::I64(v2)) => v1 > v2, + (Value::U64(v1), Value::U64(v2)) => v1 > v2, + (Value::F32(v1), Value::F32(v2)) => v1 > v2, + (Value::F64(v1), Value::F64(v2)) => v1 > v2, + _ => return Err(Error::TypeMismatch), + }; + Ok(Value::Generic(value as u64)) + } + + /// Perform the `<= relational operation. + /// + /// This operation requires matching integral types. + /// If the value type is `Generic`, then it is interpreted as a signed value. + /// + /// This corresponds to the DWARF `DW_OP_le` operation. + pub fn le(self, rhs: Value, addr_mask: u64) -> Result { + let value = match (self, rhs) { + (Value::Generic(v1), Value::Generic(v2)) => { + sign_extend(v1, addr_mask) <= sign_extend(v2, addr_mask) + } + (Value::I8(v1), Value::I8(v2)) => v1 <= v2, + (Value::U8(v1), Value::U8(v2)) => v1 <= v2, + (Value::I16(v1), Value::I16(v2)) => v1 <= v2, + (Value::U16(v1), Value::U16(v2)) => v1 <= v2, + (Value::I32(v1), Value::I32(v2)) => v1 <= v2, + (Value::U32(v1), Value::U32(v2)) => v1 <= v2, + (Value::I64(v1), Value::I64(v2)) => v1 <= v2, + (Value::U64(v1), Value::U64(v2)) => v1 <= v2, + (Value::F32(v1), Value::F32(v2)) => v1 <= v2, + (Value::F64(v1), Value::F64(v2)) => v1 <= v2, + _ => return Err(Error::TypeMismatch), + }; + Ok(Value::Generic(value as u64)) + } + + /// Perform the `< relational operation. + /// + /// This operation requires matching integral types. + /// If the value type is `Generic`, then it is interpreted as a signed value. + /// + /// This corresponds to the DWARF `DW_OP_lt` operation. + pub fn lt(self, rhs: Value, addr_mask: u64) -> Result { + let value = match (self, rhs) { + (Value::Generic(v1), Value::Generic(v2)) => { + sign_extend(v1, addr_mask) < sign_extend(v2, addr_mask) + } + (Value::I8(v1), Value::I8(v2)) => v1 < v2, + (Value::U8(v1), Value::U8(v2)) => v1 < v2, + (Value::I16(v1), Value::I16(v2)) => v1 < v2, + (Value::U16(v1), Value::U16(v2)) => v1 < v2, + (Value::I32(v1), Value::I32(v2)) => v1 < v2, + (Value::U32(v1), Value::U32(v2)) => v1 < v2, + (Value::I64(v1), Value::I64(v2)) => v1 < v2, + (Value::U64(v1), Value::U64(v2)) => v1 < v2, + (Value::F32(v1), Value::F32(v2)) => v1 < v2, + (Value::F64(v1), Value::F64(v2)) => v1 < v2, + _ => return Err(Error::TypeMismatch), + }; + Ok(Value::Generic(value as u64)) + } + + /// Perform the `!= relational operation. + /// + /// This operation requires matching integral types. + /// If the value type is `Generic`, then it is interpreted as a signed value. + /// + /// This corresponds to the DWARF `DW_OP_ne` operation. + pub fn ne(self, rhs: Value, addr_mask: u64) -> Result { + let value = match (self, rhs) { + (Value::Generic(v1), Value::Generic(v2)) => { + sign_extend(v1, addr_mask) != sign_extend(v2, addr_mask) + } + (Value::I8(v1), Value::I8(v2)) => v1 != v2, + (Value::U8(v1), Value::U8(v2)) => v1 != v2, + (Value::I16(v1), Value::I16(v2)) => v1 != v2, + (Value::U16(v1), Value::U16(v2)) => v1 != v2, + (Value::I32(v1), Value::I32(v2)) => v1 != v2, + (Value::U32(v1), Value::U32(v2)) => v1 != v2, + (Value::I64(v1), Value::I64(v2)) => v1 != v2, + (Value::U64(v1), Value::U64(v2)) => v1 != v2, + (Value::F32(v1), Value::F32(v2)) => v1 != v2, + (Value::F64(v1), Value::F64(v2)) => v1 != v2, + _ => return Err(Error::TypeMismatch), + }; + Ok(Value::Generic(value as u64)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::common::{DebugAbbrevOffset, DebugInfoOffset, Encoding, Format}; + use crate::endianity::LittleEndian; + use crate::read::{ + Abbreviation, AttributeSpecification, DebuggingInformationEntry, EndianSlice, UnitHeader, + UnitOffset, UnitType, + }; + + #[test] + #[rustfmt::skip] + fn valuetype_from_encoding() { + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 4, + }; + let unit = UnitHeader::new( + encoding, + 7, + UnitType::Compilation, + DebugAbbrevOffset(0), + DebugInfoOffset(0).into(), + EndianSlice::new(&[], LittleEndian), + ); + + let abbrev = Abbreviation::new( + 42, + constants::DW_TAG_base_type, + constants::DW_CHILDREN_no, + vec![ + AttributeSpecification::new( + constants::DW_AT_byte_size, + constants::DW_FORM_udata, + None, + ), + AttributeSpecification::new( + constants::DW_AT_encoding, + constants::DW_FORM_udata, + None, + ), + AttributeSpecification::new( + constants::DW_AT_endianity, + constants::DW_FORM_udata, + None, + ), + ].into(), + ); + + for &(attrs, result) in &[ + ([0x01, constants::DW_ATE_signed.0, constants::DW_END_default.0], ValueType::I8), + ([0x02, constants::DW_ATE_signed.0, constants::DW_END_default.0], ValueType::I16), + ([0x04, constants::DW_ATE_signed.0, constants::DW_END_default.0], ValueType::I32), + ([0x08, constants::DW_ATE_signed.0, constants::DW_END_default.0], ValueType::I64), + ([0x01, constants::DW_ATE_unsigned.0, constants::DW_END_default.0], ValueType::U8), + ([0x02, constants::DW_ATE_unsigned.0, constants::DW_END_default.0], ValueType::U16), + ([0x04, constants::DW_ATE_unsigned.0, constants::DW_END_default.0], ValueType::U32), + ([0x08, constants::DW_ATE_unsigned.0, constants::DW_END_default.0], ValueType::U64), + ([0x04, constants::DW_ATE_float.0, constants::DW_END_default.0], ValueType::F32), + ([0x08, constants::DW_ATE_float.0, constants::DW_END_default.0], ValueType::F64), + ] { + let entry = DebuggingInformationEntry::new( + UnitOffset(0), + EndianSlice::new(&attrs, LittleEndian), + &abbrev, + &unit, + ); + assert_eq!(ValueType::from_entry(&entry), Ok(Some(result))); + } + + for attrs in &[ + [0x03, constants::DW_ATE_signed.0, constants::DW_END_default.0], + [0x02, constants::DW_ATE_signed.0, constants::DW_END_big.0], + ] { + let entry = DebuggingInformationEntry::new( + UnitOffset(0), + EndianSlice::new(attrs, LittleEndian), + &abbrev, + &unit, + ); + assert_eq!(ValueType::from_entry(&entry), Ok(None)); + } + } + + #[test] + fn value_convert() { + let addr_mask = !0 >> 32; + for &(v, t, result) in &[ + (Value::Generic(1), ValueType::I8, Ok(Value::I8(1))), + (Value::I8(1), ValueType::U8, Ok(Value::U8(1))), + (Value::U8(1), ValueType::I16, Ok(Value::I16(1))), + (Value::I16(1), ValueType::U16, Ok(Value::U16(1))), + (Value::U16(1), ValueType::I32, Ok(Value::I32(1))), + (Value::I32(1), ValueType::U32, Ok(Value::U32(1))), + (Value::U32(1), ValueType::F32, Ok(Value::F32(1.))), + (Value::F32(1.), ValueType::I64, Ok(Value::I64(1))), + (Value::I64(1), ValueType::U64, Ok(Value::U64(1))), + (Value::U64(1), ValueType::F64, Ok(Value::F64(1.))), + (Value::F64(1.), ValueType::Generic, Ok(Value::Generic(1))), + ] { + assert_eq!(v.convert(t, addr_mask), result); + } + } + + #[test] + #[rustfmt::skip] + fn value_reinterpret() { + let addr_mask = !0 >> 32; + for &(v, t, result) in &[ + // 8-bit + (Value::I8(-1), ValueType::U8, Ok(Value::U8(0xff))), + (Value::U8(0xff), ValueType::I8, Ok(Value::I8(-1))), + // 16-bit + (Value::I16(1), ValueType::U16, Ok(Value::U16(1))), + (Value::U16(1), ValueType::I16, Ok(Value::I16(1))), + // 32-bit + (Value::Generic(1), ValueType::I32, Ok(Value::I32(1))), + (Value::I32(1), ValueType::U32, Ok(Value::U32(1))), + (Value::U32(0x3f80_0000), ValueType::F32, Ok(Value::F32(1.0))), + (Value::F32(1.0), ValueType::Generic, Ok(Value::Generic(0x3f80_0000))), + // Type mismatches + (Value::Generic(1), ValueType::U8, Err(Error::TypeMismatch)), + (Value::U8(1), ValueType::U16, Err(Error::TypeMismatch)), + (Value::U16(1), ValueType::U32, Err(Error::TypeMismatch)), + (Value::U32(1), ValueType::U64, Err(Error::TypeMismatch)), + (Value::U64(1), ValueType::Generic, Err(Error::TypeMismatch)), + ] { + assert_eq!(v.reinterpret(t, addr_mask), result); + } + + let addr_mask = !0; + for &(v, t, result) in &[ + // 64-bit + (Value::Generic(1), ValueType::I64, Ok(Value::I64(1))), + (Value::I64(1), ValueType::U64, Ok(Value::U64(1))), + (Value::U64(0x3ff0_0000_0000_0000), ValueType::F64, Ok(Value::F64(1.0))), + (Value::F64(1.0), ValueType::Generic, Ok(Value::Generic(0x3ff0_0000_0000_0000))), + ] { + assert_eq!(v.reinterpret(t, addr_mask), result); + } + } + + #[test] + #[rustfmt::skip] + fn value_abs() { + let addr_mask = 0xffff_ffff; + for &(v, result) in &[ + (Value::Generic(0xffff_ffff), Ok(Value::Generic(1))), + (Value::I8(-1), Ok(Value::I8(1))), + (Value::U8(1), Ok(Value::U8(1))), + (Value::I16(-1), Ok(Value::I16(1))), + (Value::U16(1), Ok(Value::U16(1))), + (Value::I32(-1), Ok(Value::I32(1))), + (Value::U32(1), Ok(Value::U32(1))), + (Value::I64(-1), Ok(Value::I64(1))), + (Value::U64(1), Ok(Value::U64(1))), + (Value::F32(-1.), Ok(Value::F32(1.))), + (Value::F64(-1.), Ok(Value::F64(1.))), + ] { + assert_eq!(v.abs(addr_mask), result); + } + } + + #[test] + #[rustfmt::skip] + fn value_neg() { + let addr_mask = 0xffff_ffff; + for &(v, result) in &[ + (Value::Generic(0xffff_ffff), Ok(Value::Generic(1))), + (Value::I8(1), Ok(Value::I8(-1))), + (Value::U8(1), Err(Error::UnsupportedTypeOperation)), + (Value::I16(1), Ok(Value::I16(-1))), + (Value::U16(1), Err(Error::UnsupportedTypeOperation)), + (Value::I32(1), Ok(Value::I32(-1))), + (Value::U32(1), Err(Error::UnsupportedTypeOperation)), + (Value::I64(1), Ok(Value::I64(-1))), + (Value::U64(1), Err(Error::UnsupportedTypeOperation)), + (Value::F32(1.), Ok(Value::F32(-1.))), + (Value::F64(1.), Ok(Value::F64(-1.))), + ] { + assert_eq!(v.neg(addr_mask), result); + } + } + + #[test] + #[rustfmt::skip] + fn value_add() { + let addr_mask = 0xffff_ffff; + for &(v1, v2, result) in &[ + (Value::Generic(1), Value::Generic(2), Ok(Value::Generic(3))), + (Value::I8(-1), Value::I8(2), Ok(Value::I8(1))), + (Value::U8(1), Value::U8(2), Ok(Value::U8(3))), + (Value::I16(-1), Value::I16(2), Ok(Value::I16(1))), + (Value::U16(1), Value::U16(2), Ok(Value::U16(3))), + (Value::I32(-1), Value::I32(2), Ok(Value::I32(1))), + (Value::U32(1), Value::U32(2), Ok(Value::U32(3))), + (Value::I64(-1), Value::I64(2), Ok(Value::I64(1))), + (Value::U64(1), Value::U64(2), Ok(Value::U64(3))), + (Value::F32(-1.), Value::F32(2.), Ok(Value::F32(1.))), + (Value::F64(-1.), Value::F64(2.), Ok(Value::F64(1.))), + (Value::Generic(1), Value::U32(2), Err(Error::TypeMismatch)), + ] { + assert_eq!(v1.add(v2, addr_mask), result); + } + } + + #[test] + #[rustfmt::skip] + fn value_sub() { + let addr_mask = 0xffff_ffff; + for &(v1, v2, result) in &[ + (Value::Generic(3), Value::Generic(2), Ok(Value::Generic(1))), + (Value::I8(-1), Value::I8(2), Ok(Value::I8(-3))), + (Value::U8(3), Value::U8(2), Ok(Value::U8(1))), + (Value::I16(-1), Value::I16(2), Ok(Value::I16(-3))), + (Value::U16(3), Value::U16(2), Ok(Value::U16(1))), + (Value::I32(-1), Value::I32(2), Ok(Value::I32(-3))), + (Value::U32(3), Value::U32(2), Ok(Value::U32(1))), + (Value::I64(-1), Value::I64(2), Ok(Value::I64(-3))), + (Value::U64(3), Value::U64(2), Ok(Value::U64(1))), + (Value::F32(-1.), Value::F32(2.), Ok(Value::F32(-3.))), + (Value::F64(-1.), Value::F64(2.), Ok(Value::F64(-3.))), + (Value::Generic(3), Value::U32(2), Err(Error::TypeMismatch)), + ] { + assert_eq!(v1.sub(v2, addr_mask), result); + } + } + + #[test] + #[rustfmt::skip] + fn value_mul() { + let addr_mask = 0xffff_ffff; + for &(v1, v2, result) in &[ + (Value::Generic(2), Value::Generic(3), Ok(Value::Generic(6))), + (Value::I8(-2), Value::I8(3), Ok(Value::I8(-6))), + (Value::U8(2), Value::U8(3), Ok(Value::U8(6))), + (Value::I16(-2), Value::I16(3), Ok(Value::I16(-6))), + (Value::U16(2), Value::U16(3), Ok(Value::U16(6))), + (Value::I32(-2), Value::I32(3), Ok(Value::I32(-6))), + (Value::U32(2), Value::U32(3), Ok(Value::U32(6))), + (Value::I64(-2), Value::I64(3), Ok(Value::I64(-6))), + (Value::U64(2), Value::U64(3), Ok(Value::U64(6))), + (Value::F32(-2.), Value::F32(3.), Ok(Value::F32(-6.))), + (Value::F64(-2.), Value::F64(3.), Ok(Value::F64(-6.))), + (Value::Generic(2), Value::U32(3), Err(Error::TypeMismatch)), + ] { + assert_eq!(v1.mul(v2, addr_mask), result); + } + } + + #[test] + #[rustfmt::skip] + fn value_div() { + let addr_mask = 0xffff_ffff; + for &(v1, v2, result) in &[ + (Value::Generic(6), Value::Generic(3), Ok(Value::Generic(2))), + (Value::I8(-6), Value::I8(3), Ok(Value::I8(-2))), + (Value::U8(6), Value::U8(3), Ok(Value::U8(2))), + (Value::I16(-6), Value::I16(3), Ok(Value::I16(-2))), + (Value::U16(6), Value::U16(3), Ok(Value::U16(2))), + (Value::I32(-6), Value::I32(3), Ok(Value::I32(-2))), + (Value::U32(6), Value::U32(3), Ok(Value::U32(2))), + (Value::I64(-6), Value::I64(3), Ok(Value::I64(-2))), + (Value::U64(6), Value::U64(3), Ok(Value::U64(2))), + (Value::F32(-6.), Value::F32(3.), Ok(Value::F32(-2.))), + (Value::F64(-6.), Value::F64(3.), Ok(Value::F64(-2.))), + (Value::Generic(6), Value::U32(3), Err(Error::TypeMismatch)), + ] { + assert_eq!(v1.div(v2, addr_mask), result); + } + for &(v1, v2, result) in &[ + (Value::Generic(6), Value::Generic(0), Err(Error::DivisionByZero)), + (Value::I8(-6), Value::I8(0), Err(Error::DivisionByZero)), + (Value::U8(6), Value::U8(0), Err(Error::DivisionByZero)), + (Value::I16(-6), Value::I16(0), Err(Error::DivisionByZero)), + (Value::U16(6), Value::U16(0), Err(Error::DivisionByZero)), + (Value::I32(-6), Value::I32(0), Err(Error::DivisionByZero)), + (Value::U32(6), Value::U32(0), Err(Error::DivisionByZero)), + (Value::I64(-6), Value::I64(0), Err(Error::DivisionByZero)), + (Value::U64(6), Value::U64(0), Err(Error::DivisionByZero)), + (Value::F32(-6.), Value::F32(0.), Ok(Value::F32(-6. / 0.))), + (Value::F64(-6.), Value::F64(0.), Ok(Value::F64(-6. / 0.))), + ] { + assert_eq!(v1.div(v2, addr_mask), result); + } + } + + #[test] + #[rustfmt::skip] + fn value_rem() { + let addr_mask = 0xffff_ffff; + for &(v1, v2, result) in &[ + (Value::Generic(3), Value::Generic(2), Ok(Value::Generic(1))), + (Value::I8(-3), Value::I8(2), Ok(Value::I8(-1))), + (Value::U8(3), Value::U8(2), Ok(Value::U8(1))), + (Value::I16(-3), Value::I16(2), Ok(Value::I16(-1))), + (Value::U16(3), Value::U16(2), Ok(Value::U16(1))), + (Value::I32(-3), Value::I32(2), Ok(Value::I32(-1))), + (Value::U32(3), Value::U32(2), Ok(Value::U32(1))), + (Value::I64(-3), Value::I64(2), Ok(Value::I64(-1))), + (Value::U64(3), Value::U64(2), Ok(Value::U64(1))), + (Value::F32(-3.), Value::F32(2.), Err(Error::IntegralTypeRequired)), + (Value::F64(-3.), Value::F64(2.), Err(Error::IntegralTypeRequired)), + (Value::Generic(3), Value::U32(2), Err(Error::TypeMismatch)), + ] { + assert_eq!(v1.rem(v2, addr_mask), result); + } + for &(v1, v2, result) in &[ + (Value::Generic(3), Value::Generic(0), Err(Error::DivisionByZero)), + (Value::I8(-3), Value::I8(0), Err(Error::DivisionByZero)), + (Value::U8(3), Value::U8(0), Err(Error::DivisionByZero)), + (Value::I16(-3), Value::I16(0), Err(Error::DivisionByZero)), + (Value::U16(3), Value::U16(0), Err(Error::DivisionByZero)), + (Value::I32(-3), Value::I32(0), Err(Error::DivisionByZero)), + (Value::U32(3), Value::U32(0), Err(Error::DivisionByZero)), + (Value::I64(-3), Value::I64(0), Err(Error::DivisionByZero)), + (Value::U64(3), Value::U64(0), Err(Error::DivisionByZero)), + ] { + assert_eq!(v1.rem(v2, addr_mask), result); + } + } + + #[test] + #[rustfmt::skip] + fn value_not() { + let addr_mask = 0xffff_ffff; + for &(v, result) in &[ + (Value::Generic(1), Ok(Value::Generic(!1))), + (Value::I8(1), Ok(Value::I8(!1))), + (Value::U8(1), Ok(Value::U8(!1))), + (Value::I16(1), Ok(Value::I16(!1))), + (Value::U16(1), Ok(Value::U16(!1))), + (Value::I32(1), Ok(Value::I32(!1))), + (Value::U32(1), Ok(Value::U32(!1))), + (Value::I64(1), Ok(Value::I64(!1))), + (Value::U64(1), Ok(Value::U64(!1))), + (Value::F32(1.), Err(Error::IntegralTypeRequired)), + (Value::F64(1.), Err(Error::IntegralTypeRequired)), + ] { + assert_eq!(v.not(addr_mask), result); + } + } + + #[test] + #[rustfmt::skip] + fn value_and() { + let addr_mask = 0xffff_ffff; + for &(v1, v2, result) in &[ + (Value::Generic(3), Value::Generic(5), Ok(Value::Generic(1))), + (Value::I8(3), Value::I8(5), Ok(Value::I8(1))), + (Value::U8(3), Value::U8(5), Ok(Value::U8(1))), + (Value::I16(3), Value::I16(5), Ok(Value::I16(1))), + (Value::U16(3), Value::U16(5), Ok(Value::U16(1))), + (Value::I32(3), Value::I32(5), Ok(Value::I32(1))), + (Value::U32(3), Value::U32(5), Ok(Value::U32(1))), + (Value::I64(3), Value::I64(5), Ok(Value::I64(1))), + (Value::U64(3), Value::U64(5), Ok(Value::U64(1))), + (Value::F32(3.), Value::F32(5.), Err(Error::IntegralTypeRequired)), + (Value::F64(3.), Value::F64(5.), Err(Error::IntegralTypeRequired)), + (Value::Generic(3), Value::U32(5), Err(Error::TypeMismatch)), + ] { + assert_eq!(v1.and(v2, addr_mask), result); + } + } + + #[test] + #[rustfmt::skip] + fn value_or() { + let addr_mask = 0xffff_ffff; + for &(v1, v2, result) in &[ + (Value::Generic(3), Value::Generic(5), Ok(Value::Generic(7))), + (Value::I8(3), Value::I8(5), Ok(Value::I8(7))), + (Value::U8(3), Value::U8(5), Ok(Value::U8(7))), + (Value::I16(3), Value::I16(5), Ok(Value::I16(7))), + (Value::U16(3), Value::U16(5), Ok(Value::U16(7))), + (Value::I32(3), Value::I32(5), Ok(Value::I32(7))), + (Value::U32(3), Value::U32(5), Ok(Value::U32(7))), + (Value::I64(3), Value::I64(5), Ok(Value::I64(7))), + (Value::U64(3), Value::U64(5), Ok(Value::U64(7))), + (Value::F32(3.), Value::F32(5.), Err(Error::IntegralTypeRequired)), + (Value::F64(3.), Value::F64(5.), Err(Error::IntegralTypeRequired)), + (Value::Generic(3), Value::U32(5), Err(Error::TypeMismatch)), + ] { + assert_eq!(v1.or(v2, addr_mask), result); + } + } + + #[test] + #[rustfmt::skip] + fn value_xor() { + let addr_mask = 0xffff_ffff; + for &(v1, v2, result) in &[ + (Value::Generic(3), Value::Generic(5), Ok(Value::Generic(6))), + (Value::I8(3), Value::I8(5), Ok(Value::I8(6))), + (Value::U8(3), Value::U8(5), Ok(Value::U8(6))), + (Value::I16(3), Value::I16(5), Ok(Value::I16(6))), + (Value::U16(3), Value::U16(5), Ok(Value::U16(6))), + (Value::I32(3), Value::I32(5), Ok(Value::I32(6))), + (Value::U32(3), Value::U32(5), Ok(Value::U32(6))), + (Value::I64(3), Value::I64(5), Ok(Value::I64(6))), + (Value::U64(3), Value::U64(5), Ok(Value::U64(6))), + (Value::F32(3.), Value::F32(5.), Err(Error::IntegralTypeRequired)), + (Value::F64(3.), Value::F64(5.), Err(Error::IntegralTypeRequired)), + (Value::Generic(3), Value::U32(5), Err(Error::TypeMismatch)), + ] { + assert_eq!(v1.xor(v2, addr_mask), result); + } + } + + #[test] + #[rustfmt::skip] + fn value_shl() { + let addr_mask = 0xffff_ffff; + for &(v1, v2, result) in &[ + // One of each type + (Value::Generic(3), Value::Generic(5), Ok(Value::Generic(96))), + (Value::I8(3), Value::U8(5), Ok(Value::I8(96))), + (Value::U8(3), Value::I8(5), Ok(Value::U8(96))), + (Value::I16(3), Value::U16(5), Ok(Value::I16(96))), + (Value::U16(3), Value::I16(5), Ok(Value::U16(96))), + (Value::I32(3), Value::U32(5), Ok(Value::I32(96))), + (Value::U32(3), Value::I32(5), Ok(Value::U32(96))), + (Value::I64(3), Value::U64(5), Ok(Value::I64(96))), + (Value::U64(3), Value::I64(5), Ok(Value::U64(96))), + (Value::F32(3.), Value::U8(5), Err(Error::IntegralTypeRequired)), + (Value::F64(3.), Value::U8(5), Err(Error::IntegralTypeRequired)), + // Invalid shifts + (Value::U8(3), Value::I8(-5), Err(Error::InvalidShiftExpression)), + (Value::U8(3), Value::I16(-5), Err(Error::InvalidShiftExpression)), + (Value::U8(3), Value::I32(-5), Err(Error::InvalidShiftExpression)), + (Value::U8(3), Value::I64(-5), Err(Error::InvalidShiftExpression)), + (Value::U8(3), Value::F32(5.), Err(Error::InvalidShiftExpression)), + (Value::U8(3), Value::F64(5.), Err(Error::InvalidShiftExpression)), + // Large shifts + (Value::Generic(3), Value::Generic(32), Ok(Value::Generic(0))), + (Value::I8(3), Value::U8(8), Ok(Value::I8(0))), + (Value::U8(3), Value::I8(9), Ok(Value::U8(0))), + (Value::I16(3), Value::U16(17), Ok(Value::I16(0))), + (Value::U16(3), Value::I16(16), Ok(Value::U16(0))), + (Value::I32(3), Value::U32(32), Ok(Value::I32(0))), + (Value::U32(3), Value::I32(33), Ok(Value::U32(0))), + (Value::I64(3), Value::U64(65), Ok(Value::I64(0))), + (Value::U64(3), Value::I64(64), Ok(Value::U64(0))), + ] { + assert_eq!(v1.shl(v2, addr_mask), result); + } + } + + #[test] + #[rustfmt::skip] + fn value_shr() { + let addr_mask = 0xffff_ffff; + for &(v1, v2, result) in &[ + // One of each type + (Value::Generic(96), Value::Generic(5), Ok(Value::Generic(3))), + (Value::I8(96), Value::U8(5), Err(Error::UnsupportedTypeOperation)), + (Value::U8(96), Value::I8(5), Ok(Value::U8(3))), + (Value::I16(96), Value::U16(5), Err(Error::UnsupportedTypeOperation)), + (Value::U16(96), Value::I16(5), Ok(Value::U16(3))), + (Value::I32(96), Value::U32(5), Err(Error::UnsupportedTypeOperation)), + (Value::U32(96), Value::I32(5), Ok(Value::U32(3))), + (Value::I64(96), Value::U64(5), Err(Error::UnsupportedTypeOperation)), + (Value::U64(96), Value::I64(5), Ok(Value::U64(3))), + (Value::F32(96.), Value::U8(5), Err(Error::IntegralTypeRequired)), + (Value::F64(96.), Value::U8(5), Err(Error::IntegralTypeRequired)), + // Invalid shifts + (Value::U8(96), Value::I8(-5), Err(Error::InvalidShiftExpression)), + (Value::U8(96), Value::I16(-5), Err(Error::InvalidShiftExpression)), + (Value::U8(96), Value::I32(-5), Err(Error::InvalidShiftExpression)), + (Value::U8(96), Value::I64(-5), Err(Error::InvalidShiftExpression)), + (Value::U8(96), Value::F32(5.), Err(Error::InvalidShiftExpression)), + (Value::U8(96), Value::F64(5.), Err(Error::InvalidShiftExpression)), + // Large shifts + (Value::Generic(96), Value::Generic(32), Ok(Value::Generic(0))), + (Value::U8(96), Value::I8(9), Ok(Value::U8(0))), + (Value::U16(96), Value::I16(16), Ok(Value::U16(0))), + (Value::U32(96), Value::I32(33), Ok(Value::U32(0))), + (Value::U64(96), Value::I64(64), Ok(Value::U64(0))), + ] { + assert_eq!(v1.shr(v2, addr_mask), result); + } + } + + #[test] + #[rustfmt::skip] + fn value_shra() { + let addr_mask = 0xffff_ffff; + for &(v1, v2, result) in &[ + // One of each type + (Value::Generic(u64::from(-96i32 as u32)), Value::Generic(5), Ok(Value::Generic(-3i64 as u64))), + (Value::I8(-96), Value::U8(5), Ok(Value::I8(-3))), + (Value::U8(96), Value::I8(5), Err(Error::UnsupportedTypeOperation)), + (Value::I16(-96), Value::U16(5), Ok(Value::I16(-3))), + (Value::U16(96), Value::I16(5), Err(Error::UnsupportedTypeOperation)), + (Value::I32(-96), Value::U32(5), Ok(Value::I32(-3))), + (Value::U32(96), Value::I32(5), Err(Error::UnsupportedTypeOperation)), + (Value::I64(-96), Value::U64(5), Ok(Value::I64(-3))), + (Value::U64(96), Value::I64(5), Err(Error::UnsupportedTypeOperation)), + (Value::F32(96.), Value::U8(5), Err(Error::IntegralTypeRequired)), + (Value::F64(96.), Value::U8(5), Err(Error::IntegralTypeRequired)), + // Invalid shifts + (Value::U8(96), Value::I8(-5), Err(Error::InvalidShiftExpression)), + (Value::U8(96), Value::I16(-5), Err(Error::InvalidShiftExpression)), + (Value::U8(96), Value::I32(-5), Err(Error::InvalidShiftExpression)), + (Value::U8(96), Value::I64(-5), Err(Error::InvalidShiftExpression)), + (Value::U8(96), Value::F32(5.), Err(Error::InvalidShiftExpression)), + (Value::U8(96), Value::F64(5.), Err(Error::InvalidShiftExpression)), + // Large shifts + (Value::Generic(96), Value::Generic(32), Ok(Value::Generic(0))), + (Value::I8(96), Value::U8(8), Ok(Value::I8(0))), + (Value::I8(-96), Value::U8(8), Ok(Value::I8(-1))), + (Value::I16(96), Value::U16(17), Ok(Value::I16(0))), + (Value::I16(-96), Value::U16(17), Ok(Value::I16(-1))), + (Value::I32(96), Value::U32(32), Ok(Value::I32(0))), + (Value::I32(-96), Value::U32(32), Ok(Value::I32(-1))), + (Value::I64(96), Value::U64(65), Ok(Value::I64(0))), + (Value::I64(-96), Value::U64(65), Ok(Value::I64(-1))), + ] { + assert_eq!(v1.shra(v2, addr_mask), result); + } + } + + #[test] + fn value_eq() { + let addr_mask = 0xffff_ffff; + for &(v1, v2, result) in &[ + (Value::Generic(3), Value::Generic(3), Ok(Value::Generic(1))), + (Value::Generic(!3), Value::Generic(3), Ok(Value::Generic(0))), + (Value::I8(3), Value::I8(3), Ok(Value::Generic(1))), + (Value::I8(!3), Value::I8(3), Ok(Value::Generic(0))), + (Value::U8(3), Value::U8(3), Ok(Value::Generic(1))), + (Value::U8(!3), Value::U8(3), Ok(Value::Generic(0))), + (Value::I16(3), Value::I16(3), Ok(Value::Generic(1))), + (Value::I16(!3), Value::I16(3), Ok(Value::Generic(0))), + (Value::U16(3), Value::U16(3), Ok(Value::Generic(1))), + (Value::U16(!3), Value::U16(3), Ok(Value::Generic(0))), + (Value::I32(3), Value::I32(3), Ok(Value::Generic(1))), + (Value::I32(!3), Value::I32(3), Ok(Value::Generic(0))), + (Value::U32(3), Value::U32(3), Ok(Value::Generic(1))), + (Value::U32(!3), Value::U32(3), Ok(Value::Generic(0))), + (Value::I64(3), Value::I64(3), Ok(Value::Generic(1))), + (Value::I64(!3), Value::I64(3), Ok(Value::Generic(0))), + (Value::U64(3), Value::U64(3), Ok(Value::Generic(1))), + (Value::U64(!3), Value::U64(3), Ok(Value::Generic(0))), + (Value::F32(3.), Value::F32(3.), Ok(Value::Generic(1))), + (Value::F32(-3.), Value::F32(3.), Ok(Value::Generic(0))), + (Value::F64(3.), Value::F64(3.), Ok(Value::Generic(1))), + (Value::F64(-3.), Value::F64(3.), Ok(Value::Generic(0))), + (Value::Generic(3), Value::U32(3), Err(Error::TypeMismatch)), + ] { + assert_eq!(v1.eq(v2, addr_mask), result); + } + } + + #[test] + fn value_ne() { + let addr_mask = 0xffff_ffff; + for &(v1, v2, result) in &[ + (Value::Generic(3), Value::Generic(3), Ok(Value::Generic(0))), + (Value::Generic(!3), Value::Generic(3), Ok(Value::Generic(1))), + (Value::I8(3), Value::I8(3), Ok(Value::Generic(0))), + (Value::I8(!3), Value::I8(3), Ok(Value::Generic(1))), + (Value::U8(3), Value::U8(3), Ok(Value::Generic(0))), + (Value::U8(!3), Value::U8(3), Ok(Value::Generic(1))), + (Value::I16(3), Value::I16(3), Ok(Value::Generic(0))), + (Value::I16(!3), Value::I16(3), Ok(Value::Generic(1))), + (Value::U16(3), Value::U16(3), Ok(Value::Generic(0))), + (Value::U16(!3), Value::U16(3), Ok(Value::Generic(1))), + (Value::I32(3), Value::I32(3), Ok(Value::Generic(0))), + (Value::I32(!3), Value::I32(3), Ok(Value::Generic(1))), + (Value::U32(3), Value::U32(3), Ok(Value::Generic(0))), + (Value::U32(!3), Value::U32(3), Ok(Value::Generic(1))), + (Value::I64(3), Value::I64(3), Ok(Value::Generic(0))), + (Value::I64(!3), Value::I64(3), Ok(Value::Generic(1))), + (Value::U64(3), Value::U64(3), Ok(Value::Generic(0))), + (Value::U64(!3), Value::U64(3), Ok(Value::Generic(1))), + (Value::F32(3.), Value::F32(3.), Ok(Value::Generic(0))), + (Value::F32(-3.), Value::F32(3.), Ok(Value::Generic(1))), + (Value::F64(3.), Value::F64(3.), Ok(Value::Generic(0))), + (Value::F64(-3.), Value::F64(3.), Ok(Value::Generic(1))), + (Value::Generic(3), Value::U32(3), Err(Error::TypeMismatch)), + ] { + assert_eq!(v1.ne(v2, addr_mask), result); + } + } + + #[test] + fn value_ge() { + let addr_mask = 0xffff_ffff; + for &(v1, v2, result) in &[ + (Value::Generic(3), Value::Generic(!3), Ok(Value::Generic(1))), + (Value::Generic(!3), Value::Generic(3), Ok(Value::Generic(0))), + (Value::I8(3), Value::I8(!3), Ok(Value::Generic(1))), + (Value::I8(!3), Value::I8(3), Ok(Value::Generic(0))), + (Value::U8(3), Value::U8(!3), Ok(Value::Generic(0))), + (Value::U8(!3), Value::U8(3), Ok(Value::Generic(1))), + (Value::I16(3), Value::I16(!3), Ok(Value::Generic(1))), + (Value::I16(!3), Value::I16(3), Ok(Value::Generic(0))), + (Value::U16(3), Value::U16(!3), Ok(Value::Generic(0))), + (Value::U16(!3), Value::U16(3), Ok(Value::Generic(1))), + (Value::I32(3), Value::I32(!3), Ok(Value::Generic(1))), + (Value::I32(!3), Value::I32(3), Ok(Value::Generic(0))), + (Value::U32(3), Value::U32(!3), Ok(Value::Generic(0))), + (Value::U32(!3), Value::U32(3), Ok(Value::Generic(1))), + (Value::I64(3), Value::I64(!3), Ok(Value::Generic(1))), + (Value::I64(!3), Value::I64(3), Ok(Value::Generic(0))), + (Value::U64(3), Value::U64(!3), Ok(Value::Generic(0))), + (Value::U64(!3), Value::U64(3), Ok(Value::Generic(1))), + (Value::F32(3.), Value::F32(-3.), Ok(Value::Generic(1))), + (Value::F32(-3.), Value::F32(3.), Ok(Value::Generic(0))), + (Value::F64(3.), Value::F64(-3.), Ok(Value::Generic(1))), + (Value::F64(-3.), Value::F64(3.), Ok(Value::Generic(0))), + (Value::Generic(3), Value::U32(3), Err(Error::TypeMismatch)), + ] { + assert_eq!(v1.ge(v2, addr_mask), result); + } + } + + #[test] + fn value_gt() { + let addr_mask = 0xffff_ffff; + for &(v1, v2, result) in &[ + (Value::Generic(3), Value::Generic(!3), Ok(Value::Generic(1))), + (Value::Generic(!3), Value::Generic(3), Ok(Value::Generic(0))), + (Value::I8(3), Value::I8(!3), Ok(Value::Generic(1))), + (Value::I8(!3), Value::I8(3), Ok(Value::Generic(0))), + (Value::U8(3), Value::U8(!3), Ok(Value::Generic(0))), + (Value::U8(!3), Value::U8(3), Ok(Value::Generic(1))), + (Value::I16(3), Value::I16(!3), Ok(Value::Generic(1))), + (Value::I16(!3), Value::I16(3), Ok(Value::Generic(0))), + (Value::U16(3), Value::U16(!3), Ok(Value::Generic(0))), + (Value::U16(!3), Value::U16(3), Ok(Value::Generic(1))), + (Value::I32(3), Value::I32(!3), Ok(Value::Generic(1))), + (Value::I32(!3), Value::I32(3), Ok(Value::Generic(0))), + (Value::U32(3), Value::U32(!3), Ok(Value::Generic(0))), + (Value::U32(!3), Value::U32(3), Ok(Value::Generic(1))), + (Value::I64(3), Value::I64(!3), Ok(Value::Generic(1))), + (Value::I64(!3), Value::I64(3), Ok(Value::Generic(0))), + (Value::U64(3), Value::U64(!3), Ok(Value::Generic(0))), + (Value::U64(!3), Value::U64(3), Ok(Value::Generic(1))), + (Value::F32(3.), Value::F32(-3.), Ok(Value::Generic(1))), + (Value::F32(-3.), Value::F32(3.), Ok(Value::Generic(0))), + (Value::F64(3.), Value::F64(-3.), Ok(Value::Generic(1))), + (Value::F64(-3.), Value::F64(3.), Ok(Value::Generic(0))), + (Value::Generic(3), Value::U32(3), Err(Error::TypeMismatch)), + ] { + assert_eq!(v1.gt(v2, addr_mask), result); + } + } + + #[test] + fn value_le() { + let addr_mask = 0xffff_ffff; + for &(v1, v2, result) in &[ + (Value::Generic(3), Value::Generic(!3), Ok(Value::Generic(0))), + (Value::Generic(!3), Value::Generic(3), Ok(Value::Generic(1))), + (Value::I8(3), Value::I8(!3), Ok(Value::Generic(0))), + (Value::I8(!3), Value::I8(3), Ok(Value::Generic(1))), + (Value::U8(3), Value::U8(!3), Ok(Value::Generic(1))), + (Value::U8(!3), Value::U8(3), Ok(Value::Generic(0))), + (Value::I16(3), Value::I16(!3), Ok(Value::Generic(0))), + (Value::I16(!3), Value::I16(3), Ok(Value::Generic(1))), + (Value::U16(3), Value::U16(!3), Ok(Value::Generic(1))), + (Value::U16(!3), Value::U16(3), Ok(Value::Generic(0))), + (Value::I32(3), Value::I32(!3), Ok(Value::Generic(0))), + (Value::I32(!3), Value::I32(3), Ok(Value::Generic(1))), + (Value::U32(3), Value::U32(!3), Ok(Value::Generic(1))), + (Value::U32(!3), Value::U32(3), Ok(Value::Generic(0))), + (Value::I64(3), Value::I64(!3), Ok(Value::Generic(0))), + (Value::I64(!3), Value::I64(3), Ok(Value::Generic(1))), + (Value::U64(3), Value::U64(!3), Ok(Value::Generic(1))), + (Value::U64(!3), Value::U64(3), Ok(Value::Generic(0))), + (Value::F32(3.), Value::F32(-3.), Ok(Value::Generic(0))), + (Value::F32(-3.), Value::F32(3.), Ok(Value::Generic(1))), + (Value::F64(3.), Value::F64(-3.), Ok(Value::Generic(0))), + (Value::F64(-3.), Value::F64(3.), Ok(Value::Generic(1))), + (Value::Generic(3), Value::U32(3), Err(Error::TypeMismatch)), + ] { + assert_eq!(v1.le(v2, addr_mask), result); + } + } + + #[test] + fn value_lt() { + let addr_mask = 0xffff_ffff; + for &(v1, v2, result) in &[ + (Value::Generic(3), Value::Generic(!3), Ok(Value::Generic(0))), + (Value::Generic(!3), Value::Generic(3), Ok(Value::Generic(1))), + (Value::I8(3), Value::I8(!3), Ok(Value::Generic(0))), + (Value::I8(!3), Value::I8(3), Ok(Value::Generic(1))), + (Value::U8(3), Value::U8(!3), Ok(Value::Generic(1))), + (Value::U8(!3), Value::U8(3), Ok(Value::Generic(0))), + (Value::I16(3), Value::I16(!3), Ok(Value::Generic(0))), + (Value::I16(!3), Value::I16(3), Ok(Value::Generic(1))), + (Value::U16(3), Value::U16(!3), Ok(Value::Generic(1))), + (Value::U16(!3), Value::U16(3), Ok(Value::Generic(0))), + (Value::I32(3), Value::I32(!3), Ok(Value::Generic(0))), + (Value::I32(!3), Value::I32(3), Ok(Value::Generic(1))), + (Value::U32(3), Value::U32(!3), Ok(Value::Generic(1))), + (Value::U32(!3), Value::U32(3), Ok(Value::Generic(0))), + (Value::I64(3), Value::I64(!3), Ok(Value::Generic(0))), + (Value::I64(!3), Value::I64(3), Ok(Value::Generic(1))), + (Value::U64(3), Value::U64(!3), Ok(Value::Generic(1))), + (Value::U64(!3), Value::U64(3), Ok(Value::Generic(0))), + (Value::F32(3.), Value::F32(-3.), Ok(Value::Generic(0))), + (Value::F32(-3.), Value::F32(3.), Ok(Value::Generic(1))), + (Value::F64(3.), Value::F64(-3.), Ok(Value::Generic(0))), + (Value::F64(-3.), Value::F64(3.), Ok(Value::Generic(1))), + (Value::Generic(3), Value::U32(3), Err(Error::TypeMismatch)), + ] { + assert_eq!(v1.lt(v2, addr_mask), result); + } + } +} diff --git a/crux-mir/lib/gimli/src/test_util.rs b/crux-mir/lib/gimli/src/test_util.rs new file mode 100644 index 000000000..706aaf934 --- /dev/null +++ b/crux-mir/lib/gimli/src/test_util.rs @@ -0,0 +1,53 @@ +#![allow(missing_docs)] + +use crate::Format; +use test_assembler::{Label, Section}; + +pub trait GimliSectionMethods { + fn sleb(self, val: i64) -> Self; + fn uleb(self, val: u64) -> Self; + fn initial_length(self, format: Format, length: &Label, start: &Label) -> Self; + fn word(self, size: u8, val: u64) -> Self; + fn word_label(self, size: u8, val: &Label) -> Self; +} + +impl GimliSectionMethods for Section { + fn sleb(mut self, mut val: i64) -> Self { + while val & !0x3f != 0 && val | 0x3f != -1 { + self = self.D8(val as u8 | 0x80); + val >>= 7; + } + self.D8(val as u8 & 0x7f) + } + + fn uleb(mut self, mut val: u64) -> Self { + while val & !0x7f != 0 { + self = self.D8(val as u8 | 0x80); + val >>= 7; + } + self.D8(val as u8) + } + + fn initial_length(self, format: Format, length: &Label, start: &Label) -> Self { + match format { + Format::Dwarf32 => self.D32(length).mark(start), + Format::Dwarf64 => self.D32(0xffff_ffff).D64(length).mark(start), + } + } + + fn word(self, size: u8, val: u64) -> Self { + match size { + 4 => self.D32(val as u32), + 8 => self.D64(val), + _ => panic!("unsupported word size"), + } + } + + fn word_label(self, size: u8, val: &Label) -> Self { + match size { + 4 => self.D32(val), + 8 => self.D64(val), + _ => panic!("unsupported word size"), + } + } +} diff --git a/crux-mir/lib/gimli/src/write/abbrev.rs b/crux-mir/lib/gimli/src/write/abbrev.rs new file mode 100644 index 000000000..7cdfa969c --- /dev/null +++ b/crux-mir/lib/gimli/src/write/abbrev.rs @@ -0,0 +1,188 @@ +use alloc::vec::Vec; +use indexmap::IndexSet; +use std::ops::{Deref, DerefMut}; + +use crate::common::{DebugAbbrevOffset, SectionId}; +use crate::constants; +use crate::write::{Result, Section, Writer}; + +/// A table of abbreviations that will be stored in a `.debug_abbrev` section. +// Requirements: +// - values are `Abbreviation` +// - insertion returns an abbreviation code for use in writing a DIE +// - inserting a duplicate returns the code of the existing value +#[derive(Debug, Default)] +pub(crate) struct AbbreviationTable { + abbrevs: IndexSet, +} + +impl AbbreviationTable { + /// Add an abbreviation to the table and return its code. + pub fn add(&mut self, abbrev: Abbreviation) -> u64 { + let (code, _) = self.abbrevs.insert_full(abbrev); + // Code must be non-zero + (code + 1) as u64 + } + + /// Write the abbreviation table to the `.debug_abbrev` section. + pub fn write(&self, w: &mut DebugAbbrev) -> Result<()> { + for (code, abbrev) in self.abbrevs.iter().enumerate() { + w.write_uleb128((code + 1) as u64)?; + abbrev.write(w)?; + } + // Null abbreviation code + w.write_u8(0) + } +} + +/// An abbreviation describes the shape of a `DebuggingInformationEntry`'s type: +/// its tag type, whether it has children, and its set of attributes. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub(crate) struct Abbreviation { + tag: constants::DwTag, + has_children: bool, + attributes: Vec, +} + +impl Abbreviation { + /// Construct a new `Abbreviation`. + #[inline] + pub fn new( + tag: constants::DwTag, + has_children: bool, + attributes: Vec, + ) -> Abbreviation { + Abbreviation { + tag, + has_children, + attributes, + } + } + + /// Write the abbreviation to the `.debug_abbrev` section. + pub fn write(&self, w: &mut DebugAbbrev) -> Result<()> { + w.write_uleb128(self.tag.0.into())?; + w.write_u8(if self.has_children { + constants::DW_CHILDREN_yes.0 + } else { + constants::DW_CHILDREN_no.0 + })?; + for attr in &self.attributes { + attr.write(w)?; + } + // Null name and form + w.write_u8(0)?; + w.write_u8(0) + } +} + +/// The description of an attribute in an abbreviated type. +// TODO: support implicit const +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct AttributeSpecification { + name: constants::DwAt, + form: constants::DwForm, +} + +impl AttributeSpecification { + /// Construct a new `AttributeSpecification`. + #[inline] + pub fn new(name: constants::DwAt, form: constants::DwForm) -> AttributeSpecification { + AttributeSpecification { name, form } + } + + /// Write the attribute specification to the `.debug_abbrev` section. + #[inline] + pub fn write(&self, w: &mut DebugAbbrev) -> Result<()> { + w.write_uleb128(self.name.0.into())?; + w.write_uleb128(self.form.0.into()) + } +} + +define_section!( + DebugAbbrev, + DebugAbbrevOffset, + "A writable `.debug_abbrev` section." +); + +#[cfg(test)] +#[cfg(feature = "read")] +mod tests { + use super::*; + use crate::constants; + use crate::read; + use crate::write::EndianVec; + use crate::LittleEndian; + + #[test] + fn test_abbreviation_table() { + let mut abbrevs = AbbreviationTable::default(); + let abbrev1 = Abbreviation::new( + constants::DW_TAG_subprogram, + false, + vec![AttributeSpecification::new( + constants::DW_AT_name, + constants::DW_FORM_string, + )], + ); + let abbrev2 = Abbreviation::new( + constants::DW_TAG_compile_unit, + true, + vec![ + AttributeSpecification::new(constants::DW_AT_producer, constants::DW_FORM_strp), + AttributeSpecification::new(constants::DW_AT_language, constants::DW_FORM_data2), + ], + ); + let code1 = abbrevs.add(abbrev1.clone()); + assert_eq!(code1, 1); + let code2 = abbrevs.add(abbrev2.clone()); + assert_eq!(code2, 2); + assert_eq!(abbrevs.add(abbrev1.clone()), code1); + assert_eq!(abbrevs.add(abbrev2.clone()), code2); + + let mut debug_abbrev = DebugAbbrev::from(EndianVec::new(LittleEndian)); + let debug_abbrev_offset = debug_abbrev.offset(); + assert_eq!(debug_abbrev_offset, DebugAbbrevOffset(0)); + abbrevs.write(&mut debug_abbrev).unwrap(); + assert_eq!(debug_abbrev.offset(), DebugAbbrevOffset(17)); + + let read_debug_abbrev = read::DebugAbbrev::new(debug_abbrev.slice(), LittleEndian); + let read_abbrevs = read_debug_abbrev + .abbreviations(debug_abbrev_offset) + .unwrap(); + + let read_abbrev1 = read_abbrevs.get(code1).unwrap(); + assert_eq!(abbrev1.tag, read_abbrev1.tag()); + assert_eq!(abbrev1.has_children, read_abbrev1.has_children()); + assert_eq!(abbrev1.attributes.len(), read_abbrev1.attributes().len()); + assert_eq!( + abbrev1.attributes[0].name, + read_abbrev1.attributes()[0].name() + ); + assert_eq!( + abbrev1.attributes[0].form, + read_abbrev1.attributes()[0].form() + ); + + let read_abbrev2 = read_abbrevs.get(code2).unwrap(); + assert_eq!(abbrev2.tag, read_abbrev2.tag()); + assert_eq!(abbrev2.has_children, read_abbrev2.has_children()); + assert_eq!(abbrev2.attributes.len(), read_abbrev2.attributes().len()); + assert_eq!( + abbrev2.attributes[0].name, + read_abbrev2.attributes()[0].name() + ); + assert_eq!( + abbrev2.attributes[0].form, + read_abbrev2.attributes()[0].form() + ); + assert_eq!( + abbrev2.attributes[1].name, + read_abbrev2.attributes()[1].name() + ); + assert_eq!( + abbrev2.attributes[1].form, + read_abbrev2.attributes()[1].form() + ); + } +} diff --git a/crux-mir/lib/gimli/src/write/cfi.rs b/crux-mir/lib/gimli/src/write/cfi.rs new file mode 100644 index 000000000..718cb69ad --- /dev/null +++ b/crux-mir/lib/gimli/src/write/cfi.rs @@ -0,0 +1,1025 @@ +use alloc::vec::Vec; +use indexmap::IndexSet; +use std::ops::{Deref, DerefMut}; + +use crate::common::{DebugFrameOffset, EhFrameOffset, Encoding, Format, Register, SectionId}; +use crate::constants; +use crate::write::{Address, BaseId, Error, Expression, Result, Section, Writer}; + +define_section!( + DebugFrame, + DebugFrameOffset, + "A writable `.debug_frame` section." +); + +define_section!(EhFrame, EhFrameOffset, "A writable `.eh_frame` section."); + +define_id!(CieId, "An identifier for a CIE in a `FrameTable`."); + +/// A table of frame description entries. +#[derive(Debug, Default)] +pub struct FrameTable { + /// Base id for CIEs. + base_id: BaseId, + /// The common information entries. + cies: IndexSet, + /// The frame description entries. + fdes: Vec<(CieId, FrameDescriptionEntry)>, +} + +impl FrameTable { + /// Add a CIE and return its id. + /// + /// If the CIE already exists, then return the id of the existing CIE. + pub fn add_cie(&mut self, cie: CommonInformationEntry) -> CieId { + let (index, _) = self.cies.insert_full(cie); + CieId::new(self.base_id, index) + } + + /// The number of CIEs. + pub fn cie_count(&self) -> usize { + self.cies.len() + } + + /// Add a FDE. + /// + /// Does not check for duplicates. + /// + /// # Panics + /// + /// Panics if the CIE id is invalid. + pub fn add_fde(&mut self, cie: CieId, fde: FrameDescriptionEntry) { + debug_assert_eq!(self.base_id, cie.base_id); + self.fdes.push((cie, fde)); + } + + /// The number of FDEs. + pub fn fde_count(&self) -> usize { + self.fdes.len() + } + + /// Write the frame table entries to the given `.debug_frame` section. + pub fn write_debug_frame(&self, w: &mut DebugFrame) -> Result<()> { + self.write(&mut w.0, false) + } + + /// Write the frame table entries to the given `.eh_frame` section. + pub fn write_eh_frame(&self, w: &mut EhFrame) -> Result<()> { + self.write(&mut w.0, true) + } + + fn write(&self, w: &mut W, eh_frame: bool) -> Result<()> { + let mut cie_offsets = vec![None; self.cies.len()]; + for (cie_id, fde) in &self.fdes { + let cie_index = cie_id.index; + let cie = self.cies.get_index(cie_index).unwrap(); + let cie_offset = match cie_offsets[cie_index] { + Some(offset) => offset, + None => { + // Only write CIEs as they are referenced. + let offset = cie.write(w, eh_frame)?; + cie_offsets[cie_index] = Some(offset); + offset + } + }; + + fde.write(w, eh_frame, cie_offset, cie)?; + } + // TODO: write length 0 terminator for eh_frame? + Ok(()) + } +} + +/// A common information entry. This contains information that is shared between FDEs. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CommonInformationEntry { + encoding: Encoding, + + /// A constant that is factored out of code offsets. + /// + /// This should be set to the minimum instruction length. + /// Writing a code offset that is not a multiple of this factor will generate an error. + code_alignment_factor: u8, + + /// A constant that is factored out of data offsets. + /// + /// This should be set to the minimum data alignment for the frame. + /// Writing a data offset that is not a multiple of this factor will generate an error. + data_alignment_factor: i8, + + /// The return address register. This might not correspond to an actual machine register. + return_address_register: Register, + + /// The address of the personality function and its encoding. + pub personality: Option<(constants::DwEhPe, Address)>, + + /// The encoding to use for the LSDA address in FDEs. + /// + /// If set then all FDEs which use this CIE must have a LSDA address. + pub lsda_encoding: Option, + + /// The encoding to use for addresses in FDEs. + pub fde_address_encoding: constants::DwEhPe, + + /// True for signal trampolines. + pub signal_trampoline: bool, + + /// The initial instructions upon entry to this function. + instructions: Vec, +} + +impl CommonInformationEntry { + /// Create a new common information entry. + /// + /// The encoding version must be a CFI version, not a DWARF version. + pub fn new( + encoding: Encoding, + code_alignment_factor: u8, + data_alignment_factor: i8, + return_address_register: Register, + ) -> Self { + CommonInformationEntry { + encoding, + code_alignment_factor, + data_alignment_factor, + return_address_register, + personality: None, + lsda_encoding: None, + fde_address_encoding: constants::DW_EH_PE_absptr, + signal_trampoline: false, + instructions: Vec::new(), + } + } + + /// Add an initial instruction. + pub fn add_instruction(&mut self, instruction: CallFrameInstruction) { + self.instructions.push(instruction); + } + + fn has_augmentation(&self) -> bool { + self.personality.is_some() + || self.lsda_encoding.is_some() + || self.signal_trampoline + || self.fde_address_encoding != constants::DW_EH_PE_absptr + } + + /// Returns the section offset of the CIE. + fn write(&self, w: &mut W, eh_frame: bool) -> Result { + let encoding = self.encoding; + let offset = w.len(); + + let length_offset = w.write_initial_length(encoding.format)?; + let length_base = w.len(); + + if eh_frame { + w.write_u32(0)?; + } else { + match encoding.format { + Format::Dwarf32 => w.write_u32(0xffff_ffff)?, + Format::Dwarf64 => w.write_u64(0xffff_ffff_ffff_ffff)?, + } + } + + if eh_frame { + if encoding.version != 1 { + return Err(Error::UnsupportedVersion(encoding.version)); + }; + } else { + match encoding.version { + 1 | 3 | 4 => {} + _ => return Err(Error::UnsupportedVersion(encoding.version)), + }; + } + w.write_u8(encoding.version as u8)?; + + let augmentation = self.has_augmentation(); + if augmentation { + w.write_u8(b'z')?; + if self.lsda_encoding.is_some() { + w.write_u8(b'L')?; + } + if self.personality.is_some() { + w.write_u8(b'P')?; + } + if self.fde_address_encoding != constants::DW_EH_PE_absptr { + w.write_u8(b'R')?; + } + if self.signal_trampoline { + w.write_u8(b'S')?; + } + } + w.write_u8(0)?; + + if encoding.version >= 4 { + w.write_u8(encoding.address_size)?; + // TODO: segment_selector_size + w.write_u8(0)?; + } + + w.write_uleb128(self.code_alignment_factor.into())?; + w.write_sleb128(self.data_alignment_factor.into())?; + + if !eh_frame && encoding.version == 1 { + let register = self.return_address_register.0 as u8; + if u16::from(register) != self.return_address_register.0 { + return Err(Error::ValueTooLarge); + } + w.write_u8(register)?; + } else { + w.write_uleb128(self.return_address_register.0.into())?; + } + + if augmentation { + let augmentation_length_offset = w.len(); + w.write_u8(0)?; + let augmentation_length_base = w.len(); + + if let Some(eh_pe) = self.lsda_encoding { + w.write_u8(eh_pe.0)?; + } + if let Some((eh_pe, address)) = self.personality { + w.write_u8(eh_pe.0)?; + w.write_eh_pointer(address, eh_pe, encoding.address_size)?; + } + if self.fde_address_encoding != constants::DW_EH_PE_absptr { + w.write_u8(self.fde_address_encoding.0)?; + } + + let augmentation_length = (w.len() - augmentation_length_base) as u64; + debug_assert!(augmentation_length < 0x80); + w.write_udata_at(augmentation_length_offset, augmentation_length, 1)?; + } + + for instruction in &self.instructions { + instruction.write(w, encoding, self)?; + } + + write_nop( + w, + encoding.format.word_size() as usize + w.len() - length_base, + encoding.address_size, + )?; + + let length = (w.len() - length_base) as u64; + w.write_initial_length_at(length_offset, length, encoding.format)?; + + Ok(offset) + } +} + +/// A frame description entry. There should be one FDE per function. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FrameDescriptionEntry { + /// The initial address of the function. + address: Address, + + /// The length in bytes of the function. + length: u32, + + /// The address of the LSDA. + pub lsda: Option
, + + /// The instructions for this function, ordered by offset. + instructions: Vec<(u32, CallFrameInstruction)>, +} + +impl FrameDescriptionEntry { + /// Create a new frame description entry for a function. + pub fn new(address: Address, length: u32) -> Self { + FrameDescriptionEntry { + address, + length, + lsda: None, + instructions: Vec::new(), + } + } + + /// Add an instruction. + /// + /// Instructions must be added in increasing order of offset, or writing will fail. + pub fn add_instruction(&mut self, offset: u32, instruction: CallFrameInstruction) { + debug_assert!(self.instructions.last().map(|x| x.0).unwrap_or(0) <= offset); + self.instructions.push((offset, instruction)); + } + + fn write( + &self, + w: &mut W, + eh_frame: bool, + cie_offset: usize, + cie: &CommonInformationEntry, + ) -> Result<()> { + let encoding = cie.encoding; + let length_offset = w.write_initial_length(encoding.format)?; + let length_base = w.len(); + + if eh_frame { + // .eh_frame uses a relative offset which doesn't need relocation. + w.write_udata((w.len() - cie_offset) as u64, 4)?; + } else { + w.write_offset( + cie_offset, + SectionId::DebugFrame, + encoding.format.word_size(), + )?; + } + + if cie.fde_address_encoding != constants::DW_EH_PE_absptr { + w.write_eh_pointer( + self.address, + cie.fde_address_encoding, + encoding.address_size, + )?; + w.write_eh_pointer_data( + self.length.into(), + cie.fde_address_encoding.format(), + encoding.address_size, + )?; + } else { + w.write_address(self.address, encoding.address_size)?; + w.write_udata(self.length.into(), encoding.address_size)?; + } + + if cie.has_augmentation() { + let mut augmentation_length = 0u64; + if self.lsda.is_some() { + augmentation_length += u64::from(encoding.address_size); + } + w.write_uleb128(augmentation_length)?; + + debug_assert_eq!(self.lsda.is_some(), cie.lsda_encoding.is_some()); + if let (Some(lsda), Some(lsda_encoding)) = (self.lsda, cie.lsda_encoding) { + w.write_eh_pointer(lsda, lsda_encoding, encoding.address_size)?; + } + } + + let mut prev_offset = 0; + for (offset, instruction) in &self.instructions { + write_advance_loc(w, cie.code_alignment_factor, prev_offset, *offset)?; + prev_offset = *offset; + instruction.write(w, encoding, cie)?; + } + + write_nop( + w, + encoding.format.word_size() as usize + w.len() - length_base, + encoding.address_size, + )?; + + let length = (w.len() - length_base) as u64; + w.write_initial_length_at(length_offset, length, encoding.format)?; + + Ok(()) + } +} + +/// An instruction in a frame description entry. +/// +/// This may be a CFA definition, a register rule, or some other directive. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum CallFrameInstruction { + /// Define the CFA rule to use the provided register and offset. + Cfa(Register, i32), + /// Update the CFA rule to use the provided register. The offset is unchanged. + CfaRegister(Register), + /// Update the CFA rule to use the provided offset. The register is unchanged. + CfaOffset(i32), + /// Define the CFA rule to use the provided expression. + CfaExpression(Expression), + + /// Restore the initial rule for the register. + Restore(Register), + /// The previous value of the register is not recoverable. + Undefined(Register), + /// The register has not been modified. + SameValue(Register), + /// The previous value of the register is saved at address CFA + offset. + Offset(Register, i32), + /// The previous value of the register is CFA + offset. + ValOffset(Register, i32), + /// The previous value of the register is stored in another register. + Register(Register, Register), + /// The previous value of the register is saved at address given by the expression. + Expression(Register, Expression), + /// The previous value of the register is given by the expression. + ValExpression(Register, Expression), + + /// Push all register rules onto a stack. + RememberState, + /// Pop all register rules off the stack. + RestoreState, + /// The size of the arguments that have been pushed onto the stack. + ArgsSize(u32), +} + +impl CallFrameInstruction { + fn write( + &self, + w: &mut W, + encoding: Encoding, + cie: &CommonInformationEntry, + ) -> Result<()> { + match *self { + CallFrameInstruction::Cfa(register, offset) => { + if offset < 0 { + let offset = factored_data_offset(offset, cie.data_alignment_factor)?; + w.write_u8(constants::DW_CFA_def_cfa_sf.0)?; + w.write_uleb128(register.0.into())?; + w.write_sleb128(offset.into())?; + } else { + // Unfactored offset. + w.write_u8(constants::DW_CFA_def_cfa.0)?; + w.write_uleb128(register.0.into())?; + w.write_uleb128(offset as u64)?; + } + } + CallFrameInstruction::CfaRegister(register) => { + w.write_u8(constants::DW_CFA_def_cfa_register.0)?; + w.write_uleb128(register.0.into())?; + } + CallFrameInstruction::CfaOffset(offset) => { + if offset < 0 { + let offset = factored_data_offset(offset, cie.data_alignment_factor)?; + w.write_u8(constants::DW_CFA_def_cfa_offset_sf.0)?; + w.write_sleb128(offset.into())?; + } else { + // Unfactored offset. + w.write_u8(constants::DW_CFA_def_cfa_offset.0)?; + w.write_uleb128(offset as u64)?; + } + } + CallFrameInstruction::CfaExpression(ref expression) => { + w.write_u8(constants::DW_CFA_def_cfa_expression.0)?; + w.write_uleb128(expression.size(encoding, None) as u64)?; + expression.write(w, None, encoding, None)?; + } + CallFrameInstruction::Restore(register) => { + if register.0 < 0x40 { + w.write_u8(constants::DW_CFA_restore.0 | register.0 as u8)?; + } else { + w.write_u8(constants::DW_CFA_restore_extended.0)?; + w.write_uleb128(register.0.into())?; + } + } + CallFrameInstruction::Undefined(register) => { + w.write_u8(constants::DW_CFA_undefined.0)?; + w.write_uleb128(register.0.into())?; + } + CallFrameInstruction::SameValue(register) => { + w.write_u8(constants::DW_CFA_same_value.0)?; + w.write_uleb128(register.0.into())?; + } + CallFrameInstruction::Offset(register, offset) => { + let offset = factored_data_offset(offset, cie.data_alignment_factor)?; + if offset < 0 { + w.write_u8(constants::DW_CFA_offset_extended_sf.0)?; + w.write_uleb128(register.0.into())?; + w.write_sleb128(offset.into())?; + } else if register.0 < 0x40 { + w.write_u8(constants::DW_CFA_offset.0 | register.0 as u8)?; + w.write_uleb128(offset as u64)?; + } else { + w.write_u8(constants::DW_CFA_offset_extended.0)?; + w.write_uleb128(register.0.into())?; + w.write_uleb128(offset as u64)?; + } + } + CallFrameInstruction::ValOffset(register, offset) => { + let offset = factored_data_offset(offset, cie.data_alignment_factor)?; + if offset < 0 { + w.write_u8(constants::DW_CFA_val_offset_sf.0)?; + w.write_uleb128(register.0.into())?; + w.write_sleb128(offset.into())?; + } else { + w.write_u8(constants::DW_CFA_val_offset.0)?; + w.write_uleb128(register.0.into())?; + w.write_uleb128(offset as u64)?; + } + } + CallFrameInstruction::Register(register1, register2) => { + w.write_u8(constants::DW_CFA_register.0)?; + w.write_uleb128(register1.0.into())?; + w.write_uleb128(register2.0.into())?; + } + CallFrameInstruction::Expression(register, ref expression) => { + w.write_u8(constants::DW_CFA_expression.0)?; + w.write_uleb128(register.0.into())?; + w.write_uleb128(expression.size(encoding, None) as u64)?; + expression.write(w, None, encoding, None)?; + } + CallFrameInstruction::ValExpression(register, ref expression) => { + w.write_u8(constants::DW_CFA_val_expression.0)?; + w.write_uleb128(register.0.into())?; + w.write_uleb128(expression.size(encoding, None) as u64)?; + expression.write(w, None, encoding, None)?; + } + CallFrameInstruction::RememberState => { + w.write_u8(constants::DW_CFA_remember_state.0)?; + } + CallFrameInstruction::RestoreState => { + w.write_u8(constants::DW_CFA_restore_state.0)?; + } + CallFrameInstruction::ArgsSize(size) => { + w.write_u8(constants::DW_CFA_GNU_args_size.0)?; + w.write_uleb128(size.into())?; + } + } + Ok(()) + } +} + +fn write_advance_loc( + w: &mut W, + code_alignment_factor: u8, + prev_offset: u32, + offset: u32, +) -> Result<()> { + if offset == prev_offset { + return Ok(()); + } + let delta = factored_code_delta(prev_offset, offset, code_alignment_factor)?; + if delta < 0x40 { + w.write_u8(constants::DW_CFA_advance_loc.0 | delta as u8)?; + } else if delta < 0x100 { + w.write_u8(constants::DW_CFA_advance_loc1.0)?; + w.write_u8(delta as u8)?; + } else if delta < 0x10000 { + w.write_u8(constants::DW_CFA_advance_loc2.0)?; + w.write_u16(delta as u16)?; + } else { + w.write_u8(constants::DW_CFA_advance_loc4.0)?; + w.write_u32(delta)?; + } + Ok(()) +} + +fn write_nop(w: &mut W, len: usize, align: u8) -> Result<()> { + debug_assert_eq!(align & (align - 1), 0); + let tail_len = (!len + 1) & (align as usize - 1); + for _ in 0..tail_len { + w.write_u8(constants::DW_CFA_nop.0)?; + } + Ok(()) +} + +fn factored_code_delta(prev_offset: u32, offset: u32, factor: u8) -> Result { + if offset < prev_offset { + return Err(Error::InvalidFrameCodeOffset(offset)); + } + let delta = offset - prev_offset; + let factor = u32::from(factor); + let factored_delta = delta / factor; + if delta != factored_delta * factor { + return Err(Error::InvalidFrameCodeOffset(offset)); + } + Ok(factored_delta) +} + +fn factored_data_offset(offset: i32, factor: i8) -> Result { + let factor = i32::from(factor); + let factored_offset = offset / factor; + if offset != factored_offset * factor { + return Err(Error::InvalidFrameDataOffset(offset)); + } + Ok(factored_offset) +} + +#[cfg(feature = "read")] +pub(crate) mod convert { + use super::*; + use crate::read::{self, Reader}; + use crate::write::{ConvertError, ConvertResult}; + use std::collections::{hash_map, HashMap}; + + impl FrameTable { + /// Create a frame table by reading the data in the given section. + /// + /// `convert_address` is a function to convert read addresses into the `Address` + /// type. For non-relocatable addresses, this function may simply return + /// `Address::Constant(address)`. For relocatable addresses, it is the caller's + /// responsibility to determine the symbol and addend corresponding to the address + /// and return `Address::Symbol { symbol, addend }`. + pub fn from( + frame: &Section, + convert_address: &dyn Fn(u64) -> Option
, + ) -> ConvertResult + where + R: Reader, + Section: read::UnwindSection, + Section::Offset: read::UnwindOffset, + { + let bases = read::BaseAddresses::default().set_eh_frame(0); + + let mut frame_table = FrameTable::default(); + + let mut cie_ids = HashMap::new(); + let mut entries = frame.entries(&bases); + while let Some(entry) = entries.next()? { + let partial = match entry { + read::CieOrFde::Cie(_) => continue, + read::CieOrFde::Fde(partial) => partial, + }; + + // TODO: is it worth caching the parsed CIEs? It would be better if FDEs only + // stored a reference. + let from_fde = partial.parse(Section::cie_from_offset)?; + let from_cie = from_fde.cie(); + let cie_id = match cie_ids.entry(from_cie.offset()) { + hash_map::Entry::Occupied(o) => *o.get(), + hash_map::Entry::Vacant(e) => { + let cie = + CommonInformationEntry::from(from_cie, frame, &bases, convert_address)?; + let cie_id = frame_table.add_cie(cie); + e.insert(cie_id); + cie_id + } + }; + let fde = FrameDescriptionEntry::from(&from_fde, frame, &bases, convert_address)?; + frame_table.add_fde(cie_id, fde); + } + + Ok(frame_table) + } + } + + impl CommonInformationEntry { + fn from( + from_cie: &read::CommonInformationEntry, + frame: &Section, + bases: &read::BaseAddresses, + convert_address: &dyn Fn(u64) -> Option
, + ) -> ConvertResult + where + R: Reader, + Section: read::UnwindSection, + Section::Offset: read::UnwindOffset, + { + let mut cie = CommonInformationEntry::new( + from_cie.encoding(), + from_cie.code_alignment_factor() as u8, + from_cie.data_alignment_factor() as i8, + from_cie.return_address_register(), + ); + + cie.personality = match from_cie.personality_with_encoding() { + // We treat these the same because the encoding already determines + // whether it is indirect. + Some((eh_pe, read::Pointer::Direct(p))) + | Some((eh_pe, read::Pointer::Indirect(p))) => { + let address = convert_address(p).ok_or(ConvertError::InvalidAddress)?; + Some((eh_pe, address)) + } + _ => None, + }; + cie.lsda_encoding = from_cie.lsda_encoding(); + cie.fde_address_encoding = from_cie + .fde_address_encoding() + .unwrap_or(constants::DW_EH_PE_absptr); + cie.signal_trampoline = from_cie.is_signal_trampoline(); + + let mut offset = 0; + let mut from_instructions = from_cie.instructions(frame, bases); + while let Some(from_instruction) = from_instructions.next()? { + if let Some(instruction) = CallFrameInstruction::from( + from_instruction, + from_cie, + convert_address, + &mut offset, + )? { + cie.instructions.push(instruction); + } + } + Ok(cie) + } + } + + impl FrameDescriptionEntry { + fn from( + from_fde: &read::FrameDescriptionEntry, + frame: &Section, + bases: &read::BaseAddresses, + convert_address: &dyn Fn(u64) -> Option
, + ) -> ConvertResult + where + R: Reader, + Section: read::UnwindSection, + Section::Offset: read::UnwindOffset, + { + let address = + convert_address(from_fde.initial_address()).ok_or(ConvertError::InvalidAddress)?; + let length = from_fde.len() as u32; + let mut fde = FrameDescriptionEntry::new(address, length); + + match from_fde.lsda() { + // We treat these the same because the encoding already determines + // whether it is indirect. + Some(read::Pointer::Direct(p)) | Some(read::Pointer::Indirect(p)) => { + let address = convert_address(p).ok_or(ConvertError::InvalidAddress)?; + fde.lsda = Some(address); + } + None => {} + } + + let from_cie = from_fde.cie(); + let mut offset = 0; + let mut from_instructions = from_fde.instructions(frame, bases); + while let Some(from_instruction) = from_instructions.next()? { + if let Some(instruction) = CallFrameInstruction::from( + from_instruction, + from_cie, + convert_address, + &mut offset, + )? { + fde.instructions.push((offset, instruction)); + } + } + + Ok(fde) + } + } + + impl CallFrameInstruction { + fn from>( + from_instruction: read::CallFrameInstruction, + from_cie: &read::CommonInformationEntry, + convert_address: &dyn Fn(u64) -> Option
, + offset: &mut u32, + ) -> ConvertResult> { + let convert_expression = + |x| Expression::from(x, from_cie.encoding(), None, None, None, convert_address); + // TODO: validate integer type conversions + Ok(Some(match from_instruction { + read::CallFrameInstruction::SetLoc { .. } => { + return Err(ConvertError::UnsupportedCfiInstruction); + } + read::CallFrameInstruction::AdvanceLoc { delta } => { + *offset += delta * from_cie.code_alignment_factor() as u32; + return Ok(None); + } + read::CallFrameInstruction::DefCfa { register, offset } => { + CallFrameInstruction::Cfa(register, offset as i32) + } + read::CallFrameInstruction::DefCfaSf { + register, + factored_offset, + } => { + let offset = factored_offset * from_cie.data_alignment_factor(); + CallFrameInstruction::Cfa(register, offset as i32) + } + read::CallFrameInstruction::DefCfaRegister { register } => { + CallFrameInstruction::CfaRegister(register) + } + + read::CallFrameInstruction::DefCfaOffset { offset } => { + CallFrameInstruction::CfaOffset(offset as i32) + } + read::CallFrameInstruction::DefCfaOffsetSf { factored_offset } => { + let offset = factored_offset * from_cie.data_alignment_factor(); + CallFrameInstruction::CfaOffset(offset as i32) + } + read::CallFrameInstruction::DefCfaExpression { expression } => { + CallFrameInstruction::CfaExpression(convert_expression(expression)?) + } + read::CallFrameInstruction::Undefined { register } => { + CallFrameInstruction::Undefined(register) + } + read::CallFrameInstruction::SameValue { register } => { + CallFrameInstruction::SameValue(register) + } + read::CallFrameInstruction::Offset { + register, + factored_offset, + } => { + let offset = factored_offset as i64 * from_cie.data_alignment_factor(); + CallFrameInstruction::Offset(register, offset as i32) + } + read::CallFrameInstruction::OffsetExtendedSf { + register, + factored_offset, + } => { + let offset = factored_offset * from_cie.data_alignment_factor(); + CallFrameInstruction::Offset(register, offset as i32) + } + read::CallFrameInstruction::ValOffset { + register, + factored_offset, + } => { + let offset = factored_offset as i64 * from_cie.data_alignment_factor(); + CallFrameInstruction::ValOffset(register, offset as i32) + } + read::CallFrameInstruction::ValOffsetSf { + register, + factored_offset, + } => { + let offset = factored_offset * from_cie.data_alignment_factor(); + CallFrameInstruction::ValOffset(register, offset as i32) + } + read::CallFrameInstruction::Register { + dest_register, + src_register, + } => CallFrameInstruction::Register(dest_register, src_register), + read::CallFrameInstruction::Expression { + register, + expression, + } => CallFrameInstruction::Expression(register, convert_expression(expression)?), + read::CallFrameInstruction::ValExpression { + register, + expression, + } => CallFrameInstruction::ValExpression(register, convert_expression(expression)?), + read::CallFrameInstruction::Restore { register } => { + CallFrameInstruction::Restore(register) + } + read::CallFrameInstruction::RememberState => CallFrameInstruction::RememberState, + read::CallFrameInstruction::RestoreState => CallFrameInstruction::RestoreState, + read::CallFrameInstruction::ArgsSize { size } => { + CallFrameInstruction::ArgsSize(size as u32) + } + read::CallFrameInstruction::Nop => return Ok(None), + })) + } + } +} + +#[cfg(test)] +#[cfg(feature = "read")] +mod tests { + use super::*; + use crate::arch::X86_64; + use crate::read; + use crate::write::EndianVec; + use crate::LittleEndian; + + #[test] + fn test_frame_table() { + for &version in &[1, 3, 4] { + for &address_size in &[4, 8] { + for &format in &[Format::Dwarf32, Format::Dwarf64] { + let encoding = Encoding { + format, + version, + address_size, + }; + let mut frames = FrameTable::default(); + + let cie1 = CommonInformationEntry::new(encoding, 1, 8, X86_64::RA); + let cie1_id = frames.add_cie(cie1.clone()); + assert_eq!(cie1_id, frames.add_cie(cie1.clone())); + + let mut cie2 = CommonInformationEntry::new(encoding, 1, 8, X86_64::RA); + cie2.lsda_encoding = Some(constants::DW_EH_PE_absptr); + cie2.personality = + Some((constants::DW_EH_PE_absptr, Address::Constant(0x1234))); + cie2.signal_trampoline = true; + let cie2_id = frames.add_cie(cie2.clone()); + assert_ne!(cie1_id, cie2_id); + assert_eq!(cie2_id, frames.add_cie(cie2.clone())); + + let fde1 = FrameDescriptionEntry::new(Address::Constant(0x1000), 0x10); + frames.add_fde(cie1_id, fde1.clone()); + + let fde2 = FrameDescriptionEntry::new(Address::Constant(0x2000), 0x20); + frames.add_fde(cie1_id, fde2.clone()); + + let mut fde3 = FrameDescriptionEntry::new(Address::Constant(0x3000), 0x30); + fde3.lsda = Some(Address::Constant(0x3300)); + frames.add_fde(cie2_id, fde3.clone()); + + let mut fde4 = FrameDescriptionEntry::new(Address::Constant(0x4000), 0x40); + fde4.lsda = Some(Address::Constant(0x4400)); + frames.add_fde(cie2_id, fde4.clone()); + + let mut cie3 = CommonInformationEntry::new(encoding, 1, 8, X86_64::RA); + cie3.fde_address_encoding = constants::DW_EH_PE_pcrel; + cie3.lsda_encoding = Some(constants::DW_EH_PE_pcrel); + cie3.personality = Some((constants::DW_EH_PE_pcrel, Address::Constant(0x1235))); + cie3.signal_trampoline = true; + let cie3_id = frames.add_cie(cie3.clone()); + assert_ne!(cie2_id, cie3_id); + assert_eq!(cie3_id, frames.add_cie(cie3.clone())); + + let mut fde5 = FrameDescriptionEntry::new(Address::Constant(0x5000), 0x50); + fde5.lsda = Some(Address::Constant(0x5500)); + frames.add_fde(cie3_id, fde5.clone()); + + // Test writing `.debug_frame`. + let mut debug_frame = DebugFrame::from(EndianVec::new(LittleEndian)); + frames.write_debug_frame(&mut debug_frame).unwrap(); + + let mut read_debug_frame = + read::DebugFrame::new(debug_frame.slice(), LittleEndian); + read_debug_frame.set_address_size(address_size); + let convert_frames = FrameTable::from(&read_debug_frame, &|address| { + Some(Address::Constant(address)) + }) + .unwrap(); + assert_eq!(frames.cies, convert_frames.cies); + assert_eq!(frames.fdes.len(), convert_frames.fdes.len()); + for (a, b) in frames.fdes.iter().zip(convert_frames.fdes.iter()) { + assert_eq!(a.1, b.1); + } + + if version == 1 { + // Test writing `.eh_frame`. + let mut eh_frame = EhFrame::from(EndianVec::new(LittleEndian)); + frames.write_eh_frame(&mut eh_frame).unwrap(); + + let mut read_eh_frame = read::EhFrame::new(eh_frame.slice(), LittleEndian); + read_eh_frame.set_address_size(address_size); + let convert_frames = FrameTable::from(&read_eh_frame, &|address| { + Some(Address::Constant(address)) + }) + .unwrap(); + assert_eq!(frames.cies, convert_frames.cies); + assert_eq!(frames.fdes.len(), convert_frames.fdes.len()); + for (a, b) in frames.fdes.iter().zip(convert_frames.fdes.iter()) { + assert_eq!(a.1, b.1); + } + } + } + } + } + } + + #[test] + fn test_frame_instruction() { + let mut expression = Expression::new(); + expression.op_constu(0); + + let cie_instructions = [ + CallFrameInstruction::Cfa(X86_64::RSP, 8), + CallFrameInstruction::Offset(X86_64::RA, -8), + ]; + + let fde_instructions = [ + (0, CallFrameInstruction::Cfa(X86_64::RSP, 0)), + (0, CallFrameInstruction::Cfa(X86_64::RSP, -8)), + (2, CallFrameInstruction::CfaRegister(X86_64::RBP)), + (4, CallFrameInstruction::CfaOffset(8)), + (4, CallFrameInstruction::CfaOffset(0)), + (4, CallFrameInstruction::CfaOffset(-8)), + (6, CallFrameInstruction::CfaExpression(expression.clone())), + (8, CallFrameInstruction::Restore(Register(1))), + (8, CallFrameInstruction::Restore(Register(101))), + (10, CallFrameInstruction::Undefined(Register(2))), + (12, CallFrameInstruction::SameValue(Register(3))), + (14, CallFrameInstruction::Offset(Register(4), 16)), + (14, CallFrameInstruction::Offset(Register(104), 16)), + (16, CallFrameInstruction::ValOffset(Register(5), -24)), + (16, CallFrameInstruction::ValOffset(Register(5), 24)), + (18, CallFrameInstruction::Register(Register(6), Register(7))), + ( + 20, + CallFrameInstruction::Expression(Register(8), expression.clone()), + ), + ( + 22, + CallFrameInstruction::ValExpression(Register(9), expression.clone()), + ), + (24 + 0x80, CallFrameInstruction::RememberState), + (26 + 0x280, CallFrameInstruction::RestoreState), + (28 + 0x20280, CallFrameInstruction::ArgsSize(23)), + ]; + + for &version in &[1, 3, 4] { + for &address_size in &[4, 8] { + for &format in &[Format::Dwarf32, Format::Dwarf64] { + let encoding = Encoding { + format, + version, + address_size, + }; + let mut frames = FrameTable::default(); + + let mut cie = CommonInformationEntry::new(encoding, 2, 8, X86_64::RA); + for i in &cie_instructions { + cie.add_instruction(i.clone()); + } + let cie_id = frames.add_cie(cie); + + let mut fde = FrameDescriptionEntry::new(Address::Constant(0x1000), 0x10); + for (o, i) in &fde_instructions { + fde.add_instruction(*o, i.clone()); + } + frames.add_fde(cie_id, fde); + + let mut debug_frame = DebugFrame::from(EndianVec::new(LittleEndian)); + frames.write_debug_frame(&mut debug_frame).unwrap(); + + let mut read_debug_frame = + read::DebugFrame::new(debug_frame.slice(), LittleEndian); + read_debug_frame.set_address_size(address_size); + let frames = FrameTable::from(&read_debug_frame, &|address| { + Some(Address::Constant(address)) + }) + .unwrap(); + + assert_eq!( + &frames.cies.get_index(0).unwrap().instructions, + &cie_instructions + ); + assert_eq!(&frames.fdes[0].1.instructions, &fde_instructions); + } + } + } + } +} diff --git a/crux-mir/lib/gimli/src/write/dwarf.rs b/crux-mir/lib/gimli/src/write/dwarf.rs new file mode 100644 index 000000000..ea507126a --- /dev/null +++ b/crux-mir/lib/gimli/src/write/dwarf.rs @@ -0,0 +1,138 @@ +use alloc::vec::Vec; + +use crate::common::Encoding; +use crate::write::{ + AbbreviationTable, LineProgram, LineStringTable, Result, Sections, StringTable, Unit, + UnitTable, Writer, +}; + +/// Writable DWARF information for more than one unit. +#[derive(Debug, Default)] +pub struct Dwarf { + /// A table of units. These are primarily stored in the `.debug_info` section, + /// but they also contain information that is stored in other sections. + pub units: UnitTable, + + /// Extra line number programs that are not associated with a unit. + /// + /// These should only be used when generating DWARF5 line-only debug + /// information. + pub line_programs: Vec, + + /// A table of strings that will be stored in the `.debug_line_str` section. + pub line_strings: LineStringTable, + + /// A table of strings that will be stored in the `.debug_str` section. + pub strings: StringTable, +} + +impl Dwarf { + /// Create a new `Dwarf` instance. + #[inline] + pub fn new() -> Self { + Self::default() + } + + /// Write the DWARF information to the given sections. + pub fn write(&mut self, sections: &mut Sections) -> Result<()> { + let line_strings = self.line_strings.write(&mut sections.debug_line_str)?; + let strings = self.strings.write(&mut sections.debug_str)?; + self.units.write(sections, &line_strings, &strings)?; + for line_program in &self.line_programs { + line_program.write( + &mut sections.debug_line, + line_program.encoding(), + &line_strings, + &strings, + )?; + } + Ok(()) + } +} + +/// Writable DWARF information for a single unit. +#[derive(Debug)] +pub struct DwarfUnit { + /// A unit. This is primarily stored in the `.debug_info` section, + /// but also contains information that is stored in other sections. + pub unit: Unit, + + /// A table of strings that will be stored in the `.debug_line_str` section. + pub line_strings: LineStringTable, + + /// A table of strings that will be stored in the `.debug_str` section. + pub strings: StringTable, +} + +impl DwarfUnit { + /// Create a new `DwarfUnit`. + /// + /// Note: you should set `self.unit.line_program` after creation. + /// This cannot be done earlier because it may need to reference + /// `self.line_strings`. + pub fn new(encoding: Encoding) -> Self { + let unit = Unit::new(encoding, LineProgram::none()); + DwarfUnit { + unit, + line_strings: LineStringTable::default(), + strings: StringTable::default(), + } + } + + /// Write the DWARf information to the given sections. + pub fn write(&mut self, sections: &mut Sections) -> Result<()> { + let line_strings = self.line_strings.write(&mut sections.debug_line_str)?; + let strings = self.strings.write(&mut sections.debug_str)?; + + let abbrev_offset = sections.debug_abbrev.offset(); + let mut abbrevs = AbbreviationTable::default(); + + self.unit.write( + sections, + abbrev_offset, + &mut abbrevs, + &line_strings, + &strings, + )?; + // None should exist because we didn't give out any UnitId. + assert!(sections.debug_info_refs.is_empty()); + assert!(sections.debug_loc_refs.is_empty()); + assert!(sections.debug_loclists_refs.is_empty()); + + abbrevs.write(&mut sections.debug_abbrev)?; + Ok(()) + } +} + +#[cfg(feature = "read")] +pub(crate) mod convert { + use super::*; + use crate::read::{self, Reader}; + use crate::write::{Address, ConvertResult}; + + impl Dwarf { + /// Create a `write::Dwarf` by converting a `read::Dwarf`. + /// + /// `convert_address` is a function to convert read addresses into the `Address` + /// type. For non-relocatable addresses, this function may simply return + /// `Address::Constant(address)`. For relocatable addresses, it is the caller's + /// responsibility to determine the symbol and addend corresponding to the address + /// and return `Address::Symbol { symbol, addend }`. + pub fn from>( + dwarf: &read::Dwarf, + convert_address: &dyn Fn(u64) -> Option
, + ) -> ConvertResult { + let mut line_strings = LineStringTable::default(); + let mut strings = StringTable::default(); + let units = UnitTable::from(dwarf, &mut line_strings, &mut strings, convert_address)?; + // TODO: convert the line programs that were not referenced by a unit. + let line_programs = Vec::new(); + Ok(Dwarf { + units, + line_programs, + line_strings, + strings, + }) + } + } +} diff --git a/crux-mir/lib/gimli/src/write/endian_vec.rs b/crux-mir/lib/gimli/src/write/endian_vec.rs new file mode 100644 index 000000000..7b040606a --- /dev/null +++ b/crux-mir/lib/gimli/src/write/endian_vec.rs @@ -0,0 +1,117 @@ +use alloc::vec::Vec; +use std::mem; + +use crate::endianity::Endianity; +use crate::write::{Error, Result, Writer}; + +/// A `Vec` with endianity metadata. +/// +/// This implements the `Writer` trait, which is used for all writing of DWARF sections. +#[derive(Debug, Clone)] +pub struct EndianVec +where + Endian: Endianity, +{ + vec: Vec, + endian: Endian, +} + +impl EndianVec +where + Endian: Endianity, +{ + /// Construct an empty `EndianVec` with the given endianity. + pub fn new(endian: Endian) -> EndianVec { + EndianVec { + vec: Vec::new(), + endian, + } + } + + /// Return a reference to the raw slice. + pub fn slice(&self) -> &[u8] { + &self.vec + } + + /// Convert into a `Vec`. + pub fn into_vec(self) -> Vec { + self.vec + } + + /// Take any written data out of the `EndianVec`, leaving an empty `Vec` in its place. + pub fn take(&mut self) -> Vec { + let mut vec = Vec::new(); + mem::swap(&mut self.vec, &mut vec); + vec + } +} + +impl Writer for EndianVec +where + Endian: Endianity, +{ + type Endian = Endian; + + #[inline] + fn endian(&self) -> Self::Endian { + self.endian + } + + #[inline] + fn len(&self) -> usize { + self.vec.len() + } + + fn write(&mut self, bytes: &[u8]) -> Result<()> { + self.vec.extend(bytes); + Ok(()) + } + + fn write_at(&mut self, offset: usize, bytes: &[u8]) -> Result<()> { + if offset > self.vec.len() { + return Err(Error::OffsetOutOfBounds); + } + let to = &mut self.vec[offset..]; + if bytes.len() > to.len() { + return Err(Error::LengthOutOfBounds); + } + let to = &mut to[..bytes.len()]; + to.copy_from_slice(bytes); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::LittleEndian; + + #[test] + fn test_endian_vec() { + let mut w = EndianVec::new(LittleEndian); + assert_eq!(w.endian(), LittleEndian); + assert_eq!(w.len(), 0); + + w.write(&[1, 2]).unwrap(); + assert_eq!(w.slice(), &[1, 2]); + assert_eq!(w.len(), 2); + + w.write(&[3, 4, 5]).unwrap(); + assert_eq!(w.slice(), &[1, 2, 3, 4, 5]); + assert_eq!(w.len(), 5); + + w.write_at(0, &[6, 7]).unwrap(); + assert_eq!(w.slice(), &[6, 7, 3, 4, 5]); + assert_eq!(w.len(), 5); + + w.write_at(3, &[8, 9]).unwrap(); + assert_eq!(w.slice(), &[6, 7, 3, 8, 9]); + assert_eq!(w.len(), 5); + + assert_eq!(w.write_at(4, &[6, 7]), Err(Error::LengthOutOfBounds)); + assert_eq!(w.write_at(5, &[6, 7]), Err(Error::LengthOutOfBounds)); + assert_eq!(w.write_at(6, &[6, 7]), Err(Error::OffsetOutOfBounds)); + + assert_eq!(w.into_vec(), vec![6, 7, 3, 8, 9]); + } +} diff --git a/crux-mir/lib/gimli/src/write/line.rs b/crux-mir/lib/gimli/src/write/line.rs new file mode 100644 index 000000000..310170d9a --- /dev/null +++ b/crux-mir/lib/gimli/src/write/line.rs @@ -0,0 +1,1960 @@ +use alloc::vec::Vec; +use indexmap::{IndexMap, IndexSet}; +use std::ops::{Deref, DerefMut}; + +use crate::common::{DebugLineOffset, Encoding, Format, LineEncoding, SectionId}; +use crate::constants; +use crate::leb128; +use crate::write::{ + Address, DebugLineStrOffsets, DebugStrOffsets, Error, LineStringId, LineStringTable, Result, + Section, StringId, Writer, +}; + +/// The number assigned to the first special opcode. +// +// We output all instructions for all DWARF versions, since readers +// should be able to ignore instructions they don't support. +const OPCODE_BASE: u8 = 13; + +/// A line number program. +#[derive(Debug, Clone)] +pub struct LineProgram { + /// True if this line program was created with `LineProgram::none()`. + none: bool, + encoding: Encoding, + line_encoding: LineEncoding, + + /// A list of source directory path names. + /// + /// If a path is relative, then the directory is located relative to the working + /// directory of the compilation unit. + /// + /// The first entry is for the working directory of the compilation unit. + directories: IndexSet, + + /// A list of source file entries. + /// + /// Each entry has a path name and a directory. + /// + /// If a path is a relative, then the file is located relative to the + /// directory. Otherwise the directory is meaningless. + /// + /// Does not include comp_file, even for version >= 5. + files: IndexMap<(LineString, DirectoryId), FileInfo>, + + /// The primary source file of the compilation unit. + /// This is required for version >= 5, but we never reference it elsewhere + /// because DWARF defines DW_AT_decl_file=0 to mean not specified. + comp_file: (LineString, FileInfo), + + /// True if the file entries may have valid timestamps. + /// + /// Entries may still have a timestamp of 0 even if this is set. + /// For version <= 4, this is ignored. + /// For version 5, this controls whether to emit `DW_LNCT_timestamp`. + pub file_has_timestamp: bool, + + /// True if the file entries may have valid sizes. + /// + /// Entries may still have a size of 0 even if this is set. + /// For version <= 4, this is ignored. + /// For version 5, this controls whether to emit `DW_LNCT_size`. + pub file_has_size: bool, + + /// True if the file entries have valid MD5 checksums. + /// + /// For version <= 4, this is ignored. + /// For version 5, this controls whether to emit `DW_LNCT_MD5`. + pub file_has_md5: bool, + + prev_row: LineRow, + row: LineRow, + // TODO: this probably should be either rows or sequences instead + instructions: Vec, + in_sequence: bool, +} + +impl LineProgram { + /// Create a new `LineProgram`. + /// + /// `comp_dir` defines the working directory of the compilation unit, + /// and must be the same as the `DW_AT_comp_dir` attribute + /// of the compilation unit DIE. + /// + /// `comp_file` and `comp_file_info` define the primary source file + /// of the compilation unit and must be the same as the `DW_AT_name` + /// attribute of the compilation unit DIE. + /// + /// # Panics + /// + /// Panics if `line_encoding.line_base` > 0. + /// + /// Panics if `line_encoding.line_base` + `line_encoding.line_range` <= 0. + /// + /// Panics if `comp_dir` is empty or contains a null byte. + /// + /// Panics if `comp_file` is empty or contains a null byte. + #[allow(clippy::too_many_arguments)] + #[allow(clippy::new_ret_no_self)] + pub fn new( + encoding: Encoding, + line_encoding: LineEncoding, + comp_dir: LineString, + comp_file: LineString, + comp_file_info: Option, + ) -> LineProgram { + // We require a special opcode for a line advance of 0. + // See the debug_asserts in generate_row(). + assert!(line_encoding.line_base <= 0); + assert!(line_encoding.line_base + line_encoding.line_range as i8 > 0); + let mut program = LineProgram { + none: false, + encoding, + line_encoding, + directories: IndexSet::new(), + files: IndexMap::new(), + comp_file: (comp_file, comp_file_info.unwrap_or_default()), + prev_row: LineRow::initial_state(line_encoding), + row: LineRow::initial_state(line_encoding), + instructions: Vec::new(), + in_sequence: false, + file_has_timestamp: false, + file_has_size: false, + file_has_md5: false, + }; + // For all DWARF versions, directory index 0 is comp_dir. + // For version <= 4, the entry is implicit. We still add + // it here so that we use it, but we don't emit it. + program.add_directory(comp_dir); + program + } + + /// Create a new `LineProgram` with no fields set. + /// + /// This can be used when the `LineProgram` will not be used. + /// + /// You should not attempt to add files or line instructions to + /// this line program, or write it to the `.debug_line` section. + pub fn none() -> Self { + let line_encoding = LineEncoding::default(); + LineProgram { + none: true, + encoding: Encoding { + format: Format::Dwarf32, + version: 2, + address_size: 0, + }, + line_encoding, + directories: IndexSet::new(), + files: IndexMap::new(), + comp_file: (LineString::String(Vec::new()), FileInfo::default()), + prev_row: LineRow::initial_state(line_encoding), + row: LineRow::initial_state(line_encoding), + instructions: Vec::new(), + in_sequence: false, + file_has_timestamp: false, + file_has_size: false, + file_has_md5: false, + } + } + + /// Return true if this line program was created with `LineProgram::none()`. + #[inline] + pub fn is_none(&self) -> bool { + self.none + } + + /// Return the encoding parameters for this line program. + #[inline] + pub fn encoding(&self) -> Encoding { + self.encoding + } + + /// Return the DWARF version for this line program. + #[inline] + pub fn version(&self) -> u16 { + self.encoding.version + } + + /// Return the address size in bytes for this line program. + #[inline] + pub fn address_size(&self) -> u8 { + self.encoding.address_size + } + + /// Return the DWARF format for this line program. + #[inline] + pub fn format(&self) -> Format { + self.encoding.format + } + + /// Return the id for the working directory of the compilation unit. + #[inline] + pub fn default_directory(&self) -> DirectoryId { + DirectoryId(0) + } + + /// Add a directory entry and return its id. + /// + /// If the directory already exists, then return the id of the existing entry. + /// + /// If the path is relative, then the directory is located relative to the working + /// directory of the compilation unit. + /// + /// # Panics + /// + /// Panics if `directory` is empty or contains a null byte. + pub fn add_directory(&mut self, directory: LineString) -> DirectoryId { + if let LineString::String(ref val) = directory { + // For DWARF version <= 4, directories must not be empty. + // The first directory isn't emitted so skip the check for it. + if self.encoding.version <= 4 && !self.directories.is_empty() { + assert!(!val.is_empty()); + } + assert!(!val.contains(&0)); + } + let (index, _) = self.directories.insert_full(directory); + DirectoryId(index) + } + + /// Get a reference to a directory entry. + /// + /// # Panics + /// + /// Panics if `id` is invalid. + pub fn get_directory(&self, id: DirectoryId) -> &LineString { + self.directories.get_index(id.0).unwrap() + } + + /// Add a file entry and return its id. + /// + /// If the file already exists, then return the id of the existing entry. + /// + /// If the file path is relative, then the file is located relative + /// to the directory. Otherwise the directory is meaningless, but it + /// is still used as a key for file entries. + /// + /// If `info` is `None`, then new entries are assigned + /// default information, and existing entries are unmodified. + /// + /// If `info` is not `None`, then it is always assigned to the + /// entry, even if the entry already exists. + /// + /// # Panics + /// + /// Panics if 'file' is empty or contains a null byte. + pub fn add_file( + &mut self, + file: LineString, + directory: DirectoryId, + info: Option, + ) -> FileId { + if let LineString::String(ref val) = file { + assert!(!val.is_empty()); + assert!(!val.contains(&0)); + } + + let key = (file, directory); + let index = if let Some(info) = info { + let (index, _) = self.files.insert_full(key, info); + index + } else { + let entry = self.files.entry(key); + let index = entry.index(); + entry.or_insert(FileInfo::default()); + index + }; + FileId::new(index) + } + + /// Get a reference to a file entry. + /// + /// # Panics + /// + /// Panics if `id` is invalid. + pub fn get_file(&self, id: FileId) -> (&LineString, DirectoryId) { + match id.index() { + None => (&self.comp_file.0, DirectoryId(0)), + Some(index) => self + .files + .get_index(index) + .map(|entry| (&(entry.0).0, (entry.0).1)) + .unwrap(), + } + } + + /// Get a reference to the info for a file entry. + /// + /// # Panics + /// + /// Panics if `id` is invalid. + pub fn get_file_info(&self, id: FileId) -> &FileInfo { + match id.index() { + None => &self.comp_file.1, + Some(index) => self.files.get_index(index).map(|entry| entry.1).unwrap(), + } + } + + /// Get a mutable reference to the info for a file entry. + /// + /// # Panics + /// + /// Panics if `id` is invalid. + pub fn get_file_info_mut(&mut self, id: FileId) -> &mut FileInfo { + match id.index() { + None => &mut self.comp_file.1, + Some(index) => self + .files + .get_index_mut(index) + .map(|entry| entry.1) + .unwrap(), + } + } + + /// Begin a new sequence and set its base address. + /// + /// # Panics + /// + /// Panics if a sequence has already begun. + pub fn begin_sequence(&mut self, address: Option
) { + assert!(!self.in_sequence); + self.in_sequence = true; + if let Some(address) = address { + self.instructions.push(LineInstruction::SetAddress(address)); + } + } + + /// End the sequence, and reset the row to its default values. + /// + /// Only the `address_offset` and op_index` fields of the current row are used. + /// + /// # Panics + /// + /// Panics if a sequence has not begun. + pub fn end_sequence(&mut self, address_offset: u64) { + assert!(self.in_sequence); + self.in_sequence = false; + self.row.address_offset = address_offset; + let op_advance = self.op_advance(); + if op_advance != 0 { + self.instructions + .push(LineInstruction::AdvancePc(op_advance)); + } + self.instructions.push(LineInstruction::EndSequence); + self.prev_row = LineRow::initial_state(self.line_encoding); + self.row = LineRow::initial_state(self.line_encoding); + } + + /// Return true if a sequence has begun. + #[inline] + pub fn in_sequence(&self) -> bool { + self.in_sequence + } + + /// Returns a reference to the data for the current row. + #[inline] + pub fn row(&mut self) -> &mut LineRow { + &mut self.row + } + + /// Generates the line number information instructions for the current row. + /// + /// After the instructions are generated, it sets `discriminator` to 0, and sets + /// `basic_block`, `prologue_end`, and `epilogue_begin` to false. + /// + /// # Panics + /// + /// Panics if a sequence has not begun. + /// Panics if the address_offset decreases. + pub fn generate_row(&mut self) { + assert!(self.in_sequence); + + // Output fields that are reset on every row. + if self.row.discriminator != 0 { + self.instructions + .push(LineInstruction::SetDiscriminator(self.row.discriminator)); + self.row.discriminator = 0; + } + if self.row.basic_block { + self.instructions.push(LineInstruction::SetBasicBlock); + self.row.basic_block = false; + } + if self.row.prologue_end { + self.instructions.push(LineInstruction::SetPrologueEnd); + self.row.prologue_end = false; + } + if self.row.epilogue_begin { + self.instructions.push(LineInstruction::SetEpilogueBegin); + self.row.epilogue_begin = false; + } + + // Output fields that are not reset on every row. + if self.row.is_statement != self.prev_row.is_statement { + self.instructions.push(LineInstruction::NegateStatement); + } + if self.row.file != self.prev_row.file { + self.instructions + .push(LineInstruction::SetFile(self.row.file)); + } + if self.row.column != self.prev_row.column { + self.instructions + .push(LineInstruction::SetColumn(self.row.column)); + } + if self.row.isa != self.prev_row.isa { + self.instructions + .push(LineInstruction::SetIsa(self.row.isa)); + } + + // Advance the line, address, and operation index. + let line_base = i64::from(self.line_encoding.line_base) as u64; + let line_range = u64::from(self.line_encoding.line_range); + let line_advance = self.row.line as i64 - self.prev_row.line as i64; + let op_advance = self.op_advance(); + + // Default to special advances of 0. + let special_base = u64::from(OPCODE_BASE); + // TODO: handle lack of special opcodes for 0 line advance + debug_assert!(self.line_encoding.line_base <= 0); + debug_assert!(self.line_encoding.line_base + self.line_encoding.line_range as i8 >= 0); + let special_default = special_base.wrapping_sub(line_base); + let mut special = special_default; + let mut use_special = false; + + if line_advance != 0 { + let special_line = (line_advance as u64).wrapping_sub(line_base); + if special_line < line_range { + special = special_base + special_line; + use_special = true; + } else { + self.instructions + .push(LineInstruction::AdvanceLine(line_advance)); + } + } + + if op_advance != 0 { + // Using ConstAddPc can save a byte. + let (special_op_advance, const_add_pc) = if special + op_advance * line_range <= 255 { + (op_advance, false) + } else { + let op_range = (255 - special_base) / line_range; + (op_advance - op_range, true) + }; + + let special_op = special_op_advance * line_range; + if special + special_op <= 255 { + special += special_op; + use_special = true; + if const_add_pc { + self.instructions.push(LineInstruction::ConstAddPc); + } + } else { + self.instructions + .push(LineInstruction::AdvancePc(op_advance)); + } + } + + if use_special && special != special_default { + debug_assert!(special >= special_base); + debug_assert!(special <= 255); + self.instructions + .push(LineInstruction::Special(special as u8)); + } else { + self.instructions.push(LineInstruction::Copy); + } + + self.prev_row = self.row; + } + + fn op_advance(&self) -> u64 { + debug_assert!(self.row.address_offset >= self.prev_row.address_offset); + let mut address_advance = self.row.address_offset - self.prev_row.address_offset; + if self.line_encoding.minimum_instruction_length != 1 { + debug_assert_eq!( + self.row.address_offset % u64::from(self.line_encoding.minimum_instruction_length), + 0 + ); + address_advance /= u64::from(self.line_encoding.minimum_instruction_length); + } + address_advance * u64::from(self.line_encoding.maximum_operations_per_instruction) + + self.row.op_index + - self.prev_row.op_index + } + + /// Returns true if the line number program has no instructions. + /// + /// Does not check the file or directory entries. + #[inline] + pub fn is_empty(&self) -> bool { + self.instructions.is_empty() + } + + /// Write the line number program to the given section. + /// + /// # Panics + /// + /// Panics if `self.is_none()`. + pub fn write( + &self, + w: &mut DebugLine, + encoding: Encoding, + debug_line_str_offsets: &DebugLineStrOffsets, + debug_str_offsets: &DebugStrOffsets, + ) -> Result { + assert!(!self.is_none()); + + if encoding.version < self.version() + || encoding.format != self.format() + || encoding.address_size != self.address_size() + { + return Err(Error::IncompatibleLineProgramEncoding); + } + + let offset = w.offset(); + + let length_offset = w.write_initial_length(self.format())?; + let length_base = w.len(); + + if self.version() < 2 || self.version() > 5 { + return Err(Error::UnsupportedVersion(self.version())); + } + w.write_u16(self.version())?; + + if self.version() >= 5 { + w.write_u8(self.address_size())?; + // Segment selector size. + w.write_u8(0)?; + } + + let header_length_offset = w.len(); + w.write_udata(0, self.format().word_size())?; + let header_length_base = w.len(); + + w.write_u8(self.line_encoding.minimum_instruction_length)?; + if self.version() >= 4 { + w.write_u8(self.line_encoding.maximum_operations_per_instruction)?; + } else if self.line_encoding.maximum_operations_per_instruction != 1 { + return Err(Error::NeedVersion(4)); + }; + w.write_u8(if self.line_encoding.default_is_stmt { + 1 + } else { + 0 + })?; + w.write_u8(self.line_encoding.line_base as u8)?; + w.write_u8(self.line_encoding.line_range)?; + w.write_u8(OPCODE_BASE)?; + w.write(&[0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1])?; + + if self.version() <= 4 { + // The first directory is stored as DW_AT_comp_dir. + for dir in self.directories.iter().skip(1) { + dir.write( + w, + constants::DW_FORM_string, + self.encoding, + debug_line_str_offsets, + debug_str_offsets, + )?; + } + w.write_u8(0)?; + + for ((file, dir), info) in self.files.iter() { + file.write( + w, + constants::DW_FORM_string, + self.encoding, + debug_line_str_offsets, + debug_str_offsets, + )?; + w.write_uleb128(dir.0 as u64)?; + w.write_uleb128(info.timestamp)?; + w.write_uleb128(info.size)?; + } + w.write_u8(0)?; + } else { + // Directory entry formats (only ever 1). + w.write_u8(1)?; + w.write_uleb128(u64::from(constants::DW_LNCT_path.0))?; + let dir_form = self.directories.get_index(0).unwrap().form(); + w.write_uleb128(dir_form.0.into())?; + + // Directory entries. + w.write_uleb128(self.directories.len() as u64)?; + for dir in self.directories.iter() { + dir.write( + w, + dir_form, + self.encoding, + debug_line_str_offsets, + debug_str_offsets, + )?; + } + + // File name entry formats. + let count = 2 + + if self.file_has_timestamp { 1 } else { 0 } + + if self.file_has_size { 1 } else { 0 } + + if self.file_has_md5 { 1 } else { 0 }; + w.write_u8(count)?; + w.write_uleb128(u64::from(constants::DW_LNCT_path.0))?; + let file_form = self.comp_file.0.form(); + w.write_uleb128(file_form.0.into())?; + w.write_uleb128(u64::from(constants::DW_LNCT_directory_index.0))?; + w.write_uleb128(constants::DW_FORM_udata.0.into())?; + if self.file_has_timestamp { + w.write_uleb128(u64::from(constants::DW_LNCT_timestamp.0))?; + w.write_uleb128(constants::DW_FORM_udata.0.into())?; + } + if self.file_has_size { + w.write_uleb128(u64::from(constants::DW_LNCT_size.0))?; + w.write_uleb128(constants::DW_FORM_udata.0.into())?; + } + if self.file_has_md5 { + w.write_uleb128(u64::from(constants::DW_LNCT_MD5.0))?; + w.write_uleb128(constants::DW_FORM_data16.0.into())?; + } + + // File name entries. + w.write_uleb128(self.files.len() as u64 + 1)?; + let mut write_file = |file: &LineString, dir: DirectoryId, info: &FileInfo| { + file.write( + w, + file_form, + self.encoding, + debug_line_str_offsets, + debug_str_offsets, + )?; + w.write_uleb128(dir.0 as u64)?; + if self.file_has_timestamp { + w.write_uleb128(info.timestamp)?; + } + if self.file_has_size { + w.write_uleb128(info.size)?; + } + if self.file_has_md5 { + w.write(&info.md5)?; + } + Ok(()) + }; + write_file(&self.comp_file.0, DirectoryId(0), &self.comp_file.1)?; + for ((file, dir), info) in self.files.iter() { + write_file(file, *dir, info)?; + } + } + + let header_length = (w.len() - header_length_base) as u64; + w.write_udata_at( + header_length_offset, + header_length, + self.format().word_size(), + )?; + + for instruction in &self.instructions { + instruction.write(w, self.address_size())?; + } + + let length = (w.len() - length_base) as u64; + w.write_initial_length_at(length_offset, length, self.format())?; + + Ok(offset) + } +} + +/// A row in the line number table that corresponds to a machine instruction. +#[derive(Debug, Clone, Copy)] +pub struct LineRow { + /// The offset of the instruction from the start address of the sequence. + pub address_offset: u64, + /// The index of an operation within a VLIW instruction. + /// + /// The index of the first operation is 0. + /// Set to 0 for non-VLIW instructions. + pub op_index: u64, + + /// The source file corresponding to the instruction. + pub file: FileId, + /// The line number within the source file. + /// + /// Lines are numbered beginning at 1. Set to 0 if there is no source line. + pub line: u64, + /// The column number within the source line. + /// + /// Columns are numbered beginning at 1. Set to 0 for the "left edge" of the line. + pub column: u64, + /// An additional discriminator used to distinguish between source locations. + /// This value is assigned arbitrarily by the DWARF producer. + pub discriminator: u64, + + /// Set to true if the instruction is a recommended breakpoint for a statement. + pub is_statement: bool, + /// Set to true if the instruction is the beginning of a basic block. + pub basic_block: bool, + /// Set to true if the instruction is a recommended breakpoint at the entry of a + /// function. + pub prologue_end: bool, + /// Set to true if the instruction is a recommended breakpoint prior to the exit of + /// a function. + pub epilogue_begin: bool, + + /// The instruction set architecture of the instruction. + /// + /// Set to 0 for the default ISA. Other values are defined by the architecture ABI. + pub isa: u64, +} + +impl LineRow { + /// Return the initial state as specified in the DWARF standard. + fn initial_state(line_encoding: LineEncoding) -> Self { + LineRow { + address_offset: 0, + op_index: 0, + + file: FileId::initial_state(), + line: 1, + column: 0, + discriminator: 0, + + is_statement: line_encoding.default_is_stmt, + basic_block: false, + prologue_end: false, + epilogue_begin: false, + + isa: 0, + } + } +} + +/// An instruction in a line number program. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LineInstruction { + // Special opcodes + Special(u8), + + // Standard opcodes + Copy, + AdvancePc(u64), + AdvanceLine(i64), + SetFile(FileId), + SetColumn(u64), + NegateStatement, + SetBasicBlock, + ConstAddPc, + // DW_LNS_fixed_advance_pc is not supported. + SetPrologueEnd, + SetEpilogueBegin, + SetIsa(u64), + + // Extended opcodes + EndSequence, + // TODO: this doubles the size of this enum. + SetAddress(Address), + // DW_LNE_define_file is not supported. + SetDiscriminator(u64), +} + +impl LineInstruction { + /// Write the line number instruction to the given section. + fn write(self, w: &mut DebugLine, address_size: u8) -> Result<()> { + use self::LineInstruction::*; + match self { + Special(val) => w.write_u8(val)?, + Copy => w.write_u8(constants::DW_LNS_copy.0)?, + AdvancePc(val) => { + w.write_u8(constants::DW_LNS_advance_pc.0)?; + w.write_uleb128(val)?; + } + AdvanceLine(val) => { + w.write_u8(constants::DW_LNS_advance_line.0)?; + w.write_sleb128(val)?; + } + SetFile(val) => { + w.write_u8(constants::DW_LNS_set_file.0)?; + w.write_uleb128(val.raw())?; + } + SetColumn(val) => { + w.write_u8(constants::DW_LNS_set_column.0)?; + w.write_uleb128(val)?; + } + NegateStatement => w.write_u8(constants::DW_LNS_negate_stmt.0)?, + SetBasicBlock => w.write_u8(constants::DW_LNS_set_basic_block.0)?, + ConstAddPc => w.write_u8(constants::DW_LNS_const_add_pc.0)?, + SetPrologueEnd => w.write_u8(constants::DW_LNS_set_prologue_end.0)?, + SetEpilogueBegin => w.write_u8(constants::DW_LNS_set_epilogue_begin.0)?, + SetIsa(val) => { + w.write_u8(constants::DW_LNS_set_isa.0)?; + w.write_uleb128(val)?; + } + EndSequence => { + w.write_u8(0)?; + w.write_uleb128(1)?; + w.write_u8(constants::DW_LNE_end_sequence.0)?; + } + SetAddress(address) => { + w.write_u8(0)?; + w.write_uleb128(1 + u64::from(address_size))?; + w.write_u8(constants::DW_LNE_set_address.0)?; + w.write_address(address, address_size)?; + } + SetDiscriminator(val) => { + let mut bytes = [0u8; 10]; + // bytes is long enough so this will never fail. + let len = leb128::write::unsigned(&mut { &mut bytes[..] }, val).unwrap(); + w.write_u8(0)?; + w.write_uleb128(1 + len as u64)?; + w.write_u8(constants::DW_LNE_set_discriminator.0)?; + w.write(&bytes[..len])?; + } + } + Ok(()) + } +} + +/// A string value for use in defining paths in line number programs. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum LineString { + /// A slice of bytes representing a string. Must not include null bytes. + /// Not guaranteed to be UTF-8 or anything like that. + String(Vec), + + /// A reference to a string in the `.debug_str` section. + StringRef(StringId), + + /// A reference to a string in the `.debug_line_str` section. + LineStringRef(LineStringId), +} + +impl LineString { + /// Create a `LineString` using the normal form for the given encoding. + pub fn new(val: T, encoding: Encoding, line_strings: &mut LineStringTable) -> Self + where + T: Into>, + { + let val = val.into(); + if encoding.version <= 4 { + LineString::String(val) + } else { + LineString::LineStringRef(line_strings.add(val)) + } + } + + fn form(&self) -> constants::DwForm { + match *self { + LineString::String(..) => constants::DW_FORM_string, + LineString::StringRef(..) => constants::DW_FORM_strp, + LineString::LineStringRef(..) => constants::DW_FORM_line_strp, + } + } + + fn write( + &self, + w: &mut DebugLine, + form: constants::DwForm, + encoding: Encoding, + debug_line_str_offsets: &DebugLineStrOffsets, + debug_str_offsets: &DebugStrOffsets, + ) -> Result<()> { + if form != self.form() { + return Err(Error::LineStringFormMismatch); + } + + match *self { + LineString::String(ref val) => { + if encoding.version <= 4 { + debug_assert!(!val.is_empty()); + } + w.write(val)?; + w.write_u8(0)?; + } + LineString::StringRef(val) => { + if encoding.version < 5 { + return Err(Error::NeedVersion(5)); + } + w.write_offset( + debug_str_offsets.get(val).0, + SectionId::DebugStr, + encoding.format.word_size(), + )?; + } + LineString::LineStringRef(val) => { + if encoding.version < 5 { + return Err(Error::NeedVersion(5)); + } + w.write_offset( + debug_line_str_offsets.get(val).0, + SectionId::DebugLineStr, + encoding.format.word_size(), + )?; + } + } + Ok(()) + } +} + +/// An identifier for a directory in a `LineProgram`. +/// +/// Defaults to the working directory of the compilation unit. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct DirectoryId(usize); + +// Force FileId access via the methods. +mod id { + /// An identifier for a file in a `LineProgram`. + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + pub struct FileId(usize); + + impl FileId { + /// Create a FileId given an index into `LineProgram::files`. + pub(crate) fn new(index: usize) -> Self { + FileId(index + 1) + } + + /// The index of the file in `LineProgram::files`. + pub(super) fn index(self) -> Option { + if self.0 == 0 { + None + } else { + Some(self.0 - 1) + } + } + + /// The initial state of the file register. + pub(super) fn initial_state() -> Self { + FileId(1) + } + + /// The raw value used when writing. + pub(crate) fn raw(self) -> u64 { + self.0 as u64 + } + + /// The id for file index 0 in DWARF version 5. + /// Only used when converting. + // Used for tests only. + #[allow(unused)] + pub(super) fn zero() -> Self { + FileId(0) + } + } +} +pub use self::id::*; + +/// Extra information for file in a `LineProgram`. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct FileInfo { + /// The implementation defined timestamp of the last modification of the file, + /// or 0 if not available. + pub timestamp: u64, + + /// The size of the file in bytes, or 0 if not available. + pub size: u64, + + /// A 16-byte MD5 digest of the file contents. + /// + /// Only used if version >= 5 and `LineProgram::file_has_md5` is `true`. + pub md5: [u8; 16], +} + +define_section!( + DebugLine, + DebugLineOffset, + "A writable `.debug_line` section." +); + +#[cfg(feature = "read")] +mod convert { + use super::*; + use crate::read::{self, Reader}; + use crate::write::{self, ConvertError, ConvertResult}; + + impl LineProgram { + /// Create a line number program by reading the data from the given program. + /// + /// Return the program and a mapping from file index to `FileId`. + pub fn from>( + mut from_program: read::IncompleteLineProgram, + dwarf: &read::Dwarf, + line_strings: &mut write::LineStringTable, + strings: &mut write::StringTable, + convert_address: &dyn Fn(u64) -> Option
, + ) -> ConvertResult<(LineProgram, Vec)> { + // Create mappings in case the source has duplicate files or directories. + let mut dirs = Vec::new(); + let mut files = Vec::new(); + + let mut program = { + let from_header = from_program.header(); + let encoding = from_header.encoding(); + + let comp_dir = match from_header.directory(0) { + Some(comp_dir) => LineString::from(comp_dir, dwarf, line_strings, strings)?, + None => LineString::new(&[][..], encoding, line_strings), + }; + + let (comp_name, comp_file_info) = match from_header.file(0) { + Some(comp_file) => { + if comp_file.directory_index() != 0 { + return Err(ConvertError::InvalidDirectoryIndex); + } + ( + LineString::from(comp_file.path_name(), dwarf, line_strings, strings)?, + Some(FileInfo { + timestamp: comp_file.timestamp(), + size: comp_file.size(), + md5: *comp_file.md5(), + }), + ) + } + None => (LineString::new(&[][..], encoding, line_strings), None), + }; + + if from_header.line_base() > 0 { + return Err(ConvertError::InvalidLineBase); + } + let mut program = LineProgram::new( + encoding, + from_header.line_encoding(), + comp_dir, + comp_name, + comp_file_info, + ); + + let file_skip; + if from_header.version() <= 4 { + // The first directory is implicit. + dirs.push(DirectoryId(0)); + // A file index of 0 is invalid for version <= 4, but putting + // something there makes the indexing easier. + file_skip = 0; + files.push(FileId::zero()); + } else { + // We don't add the first file to `files`, but still allow + // it to be referenced from converted instructions. + file_skip = 1; + files.push(FileId::zero()); + } + + for from_dir in from_header.include_directories() { + let from_dir = + LineString::from(from_dir.clone(), dwarf, line_strings, strings)?; + dirs.push(program.add_directory(from_dir)); + } + + program.file_has_timestamp = from_header.file_has_timestamp(); + program.file_has_size = from_header.file_has_size(); + program.file_has_md5 = from_header.file_has_md5(); + for from_file in from_header.file_names().iter().skip(file_skip) { + let from_name = + LineString::from(from_file.path_name(), dwarf, line_strings, strings)?; + let from_dir = from_file.directory_index(); + if from_dir >= dirs.len() as u64 { + return Err(ConvertError::InvalidDirectoryIndex); + } + let from_dir = dirs[from_dir as usize]; + let from_info = Some(FileInfo { + timestamp: from_file.timestamp(), + size: from_file.size(), + md5: *from_file.md5(), + }); + files.push(program.add_file(from_name, from_dir, from_info)); + } + + program + }; + + // We can't use the `from_program.rows()` because that wouldn't let + // us preserve address relocations. + let mut from_row = read::LineRow::new(from_program.header()); + let mut instructions = from_program.header().instructions(); + let mut address = None; + while let Some(instruction) = instructions.next_instruction(from_program.header())? { + match instruction { + read::LineInstruction::SetAddress(val) => { + if program.in_sequence() { + return Err(ConvertError::UnsupportedLineInstruction); + } + match convert_address(val) { + Some(val) => address = Some(val), + None => return Err(ConvertError::InvalidAddress), + } + from_row.execute(read::LineInstruction::SetAddress(0), &mut from_program); + } + read::LineInstruction::DefineFile(_) => { + return Err(ConvertError::UnsupportedLineInstruction); + } + _ => { + if from_row.execute(instruction, &mut from_program) { + if !program.in_sequence() { + program.begin_sequence(address); + address = None; + } + if from_row.end_sequence() { + program.end_sequence(from_row.address()); + } else { + program.row().address_offset = from_row.address(); + program.row().op_index = from_row.op_index(); + program.row().file = { + let file = from_row.file_index(); + if file >= files.len() as u64 { + return Err(ConvertError::InvalidFileIndex); + } + if file == 0 && program.version() <= 4 { + return Err(ConvertError::InvalidFileIndex); + } + files[file as usize] + }; + program.row().line = match from_row.line() { + Some(line) => line.get(), + None => 0, + }; + program.row().column = match from_row.column() { + read::ColumnType::LeftEdge => 0, + read::ColumnType::Column(val) => val.get(), + }; + program.row().discriminator = from_row.discriminator(); + program.row().is_statement = from_row.is_stmt(); + program.row().basic_block = from_row.basic_block(); + program.row().prologue_end = from_row.prologue_end(); + program.row().epilogue_begin = from_row.epilogue_begin(); + program.row().isa = from_row.isa(); + program.generate_row(); + } + from_row.reset(from_program.header()); + } + } + }; + } + Ok((program, files)) + } + } + + impl LineString { + fn from>( + from_attr: read::AttributeValue, + dwarf: &read::Dwarf, + line_strings: &mut write::LineStringTable, + strings: &mut write::StringTable, + ) -> ConvertResult { + Ok(match from_attr { + read::AttributeValue::String(r) => LineString::String(r.to_slice()?.to_vec()), + read::AttributeValue::DebugStrRef(offset) => { + let r = dwarf.debug_str.get_str(offset)?; + let id = strings.add(r.to_slice()?); + LineString::StringRef(id) + } + read::AttributeValue::DebugLineStrRef(offset) => { + let r = dwarf.debug_line_str.get_str(offset)?; + let id = line_strings.add(r.to_slice()?); + LineString::LineStringRef(id) + } + _ => return Err(ConvertError::UnsupportedLineStringForm), + }) + } + } +} + +#[cfg(test)] +#[cfg(feature = "read")] +mod tests { + use super::*; + use crate::read; + use crate::write::{DebugLineStr, DebugStr, EndianVec, StringTable}; + use crate::LittleEndian; + + #[test] + fn test_line_program_table() { + let dir1 = LineString::String(b"dir1".to_vec()); + let file1 = LineString::String(b"file1".to_vec()); + let dir2 = LineString::String(b"dir2".to_vec()); + let file2 = LineString::String(b"file2".to_vec()); + + let mut programs = Vec::new(); + for &version in &[2, 3, 4, 5] { + for &address_size in &[4, 8] { + for &format in &[Format::Dwarf32, Format::Dwarf64] { + let encoding = Encoding { + format, + version, + address_size, + }; + let mut program = LineProgram::new( + encoding, + LineEncoding::default(), + dir1.clone(), + file1.clone(), + None, + ); + + { + assert_eq!(&dir1, program.get_directory(program.default_directory())); + program.file_has_timestamp = true; + program.file_has_size = true; + if encoding.version >= 5 { + program.file_has_md5 = true; + } + + let dir_id = program.add_directory(dir2.clone()); + assert_eq!(&dir2, program.get_directory(dir_id)); + assert_eq!(dir_id, program.add_directory(dir2.clone())); + + let file_info = FileInfo { + timestamp: 1, + size: 2, + md5: if encoding.version >= 5 { + [3; 16] + } else { + [0; 16] + }, + }; + let file_id = program.add_file(file2.clone(), dir_id, Some(file_info)); + assert_eq!((&file2, dir_id), program.get_file(file_id)); + assert_eq!(file_info, *program.get_file_info(file_id)); + + program.get_file_info_mut(file_id).size = 3; + assert_ne!(file_info, *program.get_file_info(file_id)); + assert_eq!(file_id, program.add_file(file2.clone(), dir_id, None)); + assert_ne!(file_info, *program.get_file_info(file_id)); + assert_eq!( + file_id, + program.add_file(file2.clone(), dir_id, Some(file_info)) + ); + assert_eq!(file_info, *program.get_file_info(file_id)); + + programs.push((program, file_id, encoding)); + } + } + } + } + + let debug_line_str_offsets = DebugLineStrOffsets::none(); + let debug_str_offsets = DebugStrOffsets::none(); + let mut debug_line = DebugLine::from(EndianVec::new(LittleEndian)); + let mut debug_line_offsets = Vec::new(); + for (program, _, encoding) in &programs { + debug_line_offsets.push( + program + .write( + &mut debug_line, + *encoding, + &debug_line_str_offsets, + &debug_str_offsets, + ) + .unwrap(), + ); + } + + let read_debug_line = read::DebugLine::new(debug_line.slice(), LittleEndian); + + let convert_address = &|address| Some(Address::Constant(address)); + for ((program, file_id, encoding), offset) in programs.iter().zip(debug_line_offsets.iter()) + { + let read_program = read_debug_line + .program( + *offset, + encoding.address_size, + Some(read::EndianSlice::new(b"dir1", LittleEndian)), + Some(read::EndianSlice::new(b"file1", LittleEndian)), + ) + .unwrap(); + + let dwarf = read::Dwarf::default(); + let mut convert_line_strings = LineStringTable::default(); + let mut convert_strings = StringTable::default(); + let (convert_program, convert_files) = LineProgram::from( + read_program, + &dwarf, + &mut convert_line_strings, + &mut convert_strings, + convert_address, + ) + .unwrap(); + assert_eq!(convert_program.version(), program.version()); + assert_eq!(convert_program.address_size(), program.address_size()); + assert_eq!(convert_program.format(), program.format()); + + let convert_file_id = convert_files[file_id.raw() as usize]; + let (file, dir) = program.get_file(*file_id); + let (convert_file, convert_dir) = convert_program.get_file(convert_file_id); + assert_eq!(file, convert_file); + assert_eq!( + program.get_directory(dir), + convert_program.get_directory(convert_dir) + ); + assert_eq!( + program.get_file_info(*file_id), + convert_program.get_file_info(convert_file_id) + ); + } + } + + #[test] + fn test_line_row() { + let dir1 = &b"dir1"[..]; + let file1 = &b"file1"[..]; + let file2 = &b"file2"[..]; + let convert_address = &|address| Some(Address::Constant(address)); + + let debug_line_str_offsets = DebugLineStrOffsets::none(); + let debug_str_offsets = DebugStrOffsets::none(); + + for &version in &[2, 3, 4, 5] { + for &address_size in &[4, 8] { + for &format in &[Format::Dwarf32, Format::Dwarf64] { + let encoding = Encoding { + format, + version, + address_size, + }; + let line_base = -5; + let line_range = 14; + let neg_line_base = (-line_base) as u8; + let mut program = LineProgram::new( + encoding, + LineEncoding { + line_base, + line_range, + ..Default::default() + }, + LineString::String(dir1.to_vec()), + LineString::String(file1.to_vec()), + None, + ); + let dir_id = program.default_directory(); + program.add_file(LineString::String(file1.to_vec()), dir_id, None); + let file_id = + program.add_file(LineString::String(file2.to_vec()), dir_id, None); + + // Test sequences. + { + let mut program = program.clone(); + let address = Address::Constant(0x12); + program.begin_sequence(Some(address)); + assert_eq!( + program.instructions, + vec![LineInstruction::SetAddress(address)] + ); + } + + { + let mut program = program.clone(); + program.begin_sequence(None); + assert_eq!(program.instructions, Vec::new()); + } + + { + let mut program = program.clone(); + program.begin_sequence(None); + program.end_sequence(0x1234); + assert_eq!( + program.instructions, + vec![ + LineInstruction::AdvancePc(0x1234), + LineInstruction::EndSequence + ] + ); + } + + // Create a base program. + program.begin_sequence(None); + program.row.line = 0x1000; + program.generate_row(); + let base_row = program.row; + let base_instructions = program.instructions.clone(); + + // Create test cases. + let mut tests = Vec::new(); + + let row = base_row; + tests.push((row, vec![LineInstruction::Copy])); + + let mut row = base_row; + row.line -= u64::from(neg_line_base); + tests.push((row, vec![LineInstruction::Special(OPCODE_BASE)])); + + let mut row = base_row; + row.line += u64::from(line_range) - 1; + row.line -= u64::from(neg_line_base); + tests.push(( + row, + vec![LineInstruction::Special(OPCODE_BASE + line_range - 1)], + )); + + let mut row = base_row; + row.line += u64::from(line_range); + row.line -= u64::from(neg_line_base); + tests.push(( + row, + vec![ + LineInstruction::AdvanceLine(i64::from(line_range - neg_line_base)), + LineInstruction::Copy, + ], + )); + + let mut row = base_row; + row.address_offset = 1; + row.line -= u64::from(neg_line_base); + tests.push(( + row, + vec![LineInstruction::Special(OPCODE_BASE + line_range)], + )); + + let op_range = (255 - OPCODE_BASE) / line_range; + let mut row = base_row; + row.address_offset = u64::from(op_range); + row.line -= u64::from(neg_line_base); + tests.push(( + row, + vec![LineInstruction::Special( + OPCODE_BASE + op_range * line_range, + )], + )); + + let mut row = base_row; + row.address_offset = u64::from(op_range); + row.line += u64::from(255 - OPCODE_BASE - op_range * line_range); + row.line -= u64::from(neg_line_base); + tests.push((row, vec![LineInstruction::Special(255)])); + + let mut row = base_row; + row.address_offset = u64::from(op_range); + row.line += u64::from(255 - OPCODE_BASE - op_range * line_range) + 1; + row.line -= u64::from(neg_line_base); + tests.push(( + row, + vec![LineInstruction::ConstAddPc, LineInstruction::Copy], + )); + + let mut row = base_row; + row.address_offset = u64::from(op_range); + row.line += u64::from(255 - OPCODE_BASE - op_range * line_range) + 2; + row.line -= u64::from(neg_line_base); + tests.push(( + row, + vec![ + LineInstruction::ConstAddPc, + LineInstruction::Special(OPCODE_BASE + 6), + ], + )); + + let mut row = base_row; + row.address_offset = u64::from(op_range) * 2; + row.line += u64::from(255 - OPCODE_BASE - op_range * line_range); + row.line -= u64::from(neg_line_base); + tests.push(( + row, + vec![LineInstruction::ConstAddPc, LineInstruction::Special(255)], + )); + + let mut row = base_row; + row.address_offset = u64::from(op_range) * 2; + row.line += u64::from(255 - OPCODE_BASE - op_range * line_range) + 1; + row.line -= u64::from(neg_line_base); + tests.push(( + row, + vec![ + LineInstruction::AdvancePc(row.address_offset), + LineInstruction::Copy, + ], + )); + + let mut row = base_row; + row.address_offset = u64::from(op_range) * 2; + row.line += u64::from(255 - OPCODE_BASE - op_range * line_range) + 2; + row.line -= u64::from(neg_line_base); + tests.push(( + row, + vec![ + LineInstruction::AdvancePc(row.address_offset), + LineInstruction::Special(OPCODE_BASE + 6), + ], + )); + + let mut row = base_row; + row.address_offset = 0x1234; + tests.push(( + row, + vec![LineInstruction::AdvancePc(0x1234), LineInstruction::Copy], + )); + + let mut row = base_row; + row.line += 0x1234; + tests.push(( + row, + vec![LineInstruction::AdvanceLine(0x1234), LineInstruction::Copy], + )); + + let mut row = base_row; + row.file = file_id; + tests.push(( + row, + vec![LineInstruction::SetFile(file_id), LineInstruction::Copy], + )); + + let mut row = base_row; + row.column = 0x1234; + tests.push(( + row, + vec![LineInstruction::SetColumn(0x1234), LineInstruction::Copy], + )); + + let mut row = base_row; + row.discriminator = 0x1234; + tests.push(( + row, + vec![ + LineInstruction::SetDiscriminator(0x1234), + LineInstruction::Copy, + ], + )); + + let mut row = base_row; + row.is_statement = !row.is_statement; + tests.push(( + row, + vec![LineInstruction::NegateStatement, LineInstruction::Copy], + )); + + let mut row = base_row; + row.basic_block = true; + tests.push(( + row, + vec![LineInstruction::SetBasicBlock, LineInstruction::Copy], + )); + + let mut row = base_row; + row.prologue_end = true; + tests.push(( + row, + vec![LineInstruction::SetPrologueEnd, LineInstruction::Copy], + )); + + let mut row = base_row; + row.epilogue_begin = true; + tests.push(( + row, + vec![LineInstruction::SetEpilogueBegin, LineInstruction::Copy], + )); + + let mut row = base_row; + row.isa = 0x1234; + tests.push(( + row, + vec![LineInstruction::SetIsa(0x1234), LineInstruction::Copy], + )); + + for test in tests { + // Test generate_row(). + let mut program = program.clone(); + program.row = test.0; + program.generate_row(); + assert_eq!( + &program.instructions[base_instructions.len()..], + &test.1[..] + ); + + // Test LineProgram::from(). + let mut debug_line = DebugLine::from(EndianVec::new(LittleEndian)); + let debug_line_offset = program + .write( + &mut debug_line, + encoding, + &debug_line_str_offsets, + &debug_str_offsets, + ) + .unwrap(); + + let read_debug_line = + read::DebugLine::new(debug_line.slice(), LittleEndian); + let read_program = read_debug_line + .program( + debug_line_offset, + address_size, + Some(read::EndianSlice::new(dir1, LittleEndian)), + Some(read::EndianSlice::new(file1, LittleEndian)), + ) + .unwrap(); + + let dwarf = read::Dwarf::default(); + let mut convert_line_strings = LineStringTable::default(); + let mut convert_strings = StringTable::default(); + let (convert_program, _convert_files) = LineProgram::from( + read_program, + &dwarf, + &mut convert_line_strings, + &mut convert_strings, + convert_address, + ) + .unwrap(); + assert_eq!( + &convert_program.instructions[base_instructions.len()..], + &test.1[..] + ); + } + } + } + } + } + + #[test] + fn test_line_instruction() { + let dir1 = &b"dir1"[..]; + let file1 = &b"file1"[..]; + + let debug_line_str_offsets = DebugLineStrOffsets::none(); + let debug_str_offsets = DebugStrOffsets::none(); + + for &version in &[2, 3, 4, 5] { + for &address_size in &[4, 8] { + for &format in &[Format::Dwarf32, Format::Dwarf64] { + let encoding = Encoding { + format, + version, + address_size, + }; + let mut program = LineProgram::new( + encoding, + LineEncoding::default(), + LineString::String(dir1.to_vec()), + LineString::String(file1.to_vec()), + None, + ); + let dir_id = program.default_directory(); + let file_id = + program.add_file(LineString::String(file1.to_vec()), dir_id, None); + + for &(ref inst, ref expect_inst) in &[ + ( + LineInstruction::Special(OPCODE_BASE), + read::LineInstruction::Special(OPCODE_BASE), + ), + ( + LineInstruction::Special(255), + read::LineInstruction::Special(255), + ), + (LineInstruction::Copy, read::LineInstruction::Copy), + ( + LineInstruction::AdvancePc(0x12), + read::LineInstruction::AdvancePc(0x12), + ), + ( + LineInstruction::AdvanceLine(0x12), + read::LineInstruction::AdvanceLine(0x12), + ), + ( + LineInstruction::SetFile(file_id), + read::LineInstruction::SetFile(file_id.raw()), + ), + ( + LineInstruction::SetColumn(0x12), + read::LineInstruction::SetColumn(0x12), + ), + ( + LineInstruction::NegateStatement, + read::LineInstruction::NegateStatement, + ), + ( + LineInstruction::SetBasicBlock, + read::LineInstruction::SetBasicBlock, + ), + ( + LineInstruction::ConstAddPc, + read::LineInstruction::ConstAddPc, + ), + ( + LineInstruction::SetPrologueEnd, + read::LineInstruction::SetPrologueEnd, + ), + ( + LineInstruction::SetEpilogueBegin, + read::LineInstruction::SetEpilogueBegin, + ), + ( + LineInstruction::SetIsa(0x12), + read::LineInstruction::SetIsa(0x12), + ), + ( + LineInstruction::EndSequence, + read::LineInstruction::EndSequence, + ), + ( + LineInstruction::SetAddress(Address::Constant(0x12)), + read::LineInstruction::SetAddress(0x12), + ), + ( + LineInstruction::SetDiscriminator(0x12), + read::LineInstruction::SetDiscriminator(0x12), + ), + ][..] + { + let mut program = program.clone(); + program.instructions.push(*inst); + + let mut debug_line = DebugLine::from(EndianVec::new(LittleEndian)); + let debug_line_offset = program + .write( + &mut debug_line, + encoding, + &debug_line_str_offsets, + &debug_str_offsets, + ) + .unwrap(); + + let read_debug_line = + read::DebugLine::new(debug_line.slice(), LittleEndian); + let read_program = read_debug_line + .program( + debug_line_offset, + address_size, + Some(read::EndianSlice::new(dir1, LittleEndian)), + Some(read::EndianSlice::new(file1, LittleEndian)), + ) + .unwrap(); + let read_header = read_program.header(); + let mut read_insts = read_header.instructions(); + assert_eq!( + *expect_inst, + read_insts.next_instruction(read_header).unwrap().unwrap() + ); + assert_eq!(None, read_insts.next_instruction(read_header).unwrap()); + } + } + } + } + } + + // Test that the address/line advance is correct. We don't test for optimality. + #[test] + #[allow(clippy::useless_vec)] + fn test_advance() { + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 8, + }; + + let dir1 = &b"dir1"[..]; + let file1 = &b"file1"[..]; + + let addresses = 0..50; + let lines = -10..25i64; + + let debug_line_str_offsets = DebugLineStrOffsets::none(); + let debug_str_offsets = DebugStrOffsets::none(); + + for minimum_instruction_length in vec![1, 4] { + for maximum_operations_per_instruction in vec![1, 3] { + for line_base in vec![-5, 0] { + for line_range in vec![10, 20] { + let line_encoding = LineEncoding { + minimum_instruction_length, + maximum_operations_per_instruction, + line_base, + line_range, + default_is_stmt: true, + }; + let mut program = LineProgram::new( + encoding, + line_encoding, + LineString::String(dir1.to_vec()), + LineString::String(file1.to_vec()), + None, + ); + for address_advance in addresses.clone() { + program.begin_sequence(Some(Address::Constant(0x1000))); + program.row().line = 0x10000; + program.generate_row(); + for line_advance in lines.clone() { + { + let row = program.row(); + row.address_offset += + address_advance * u64::from(minimum_instruction_length); + row.line = row.line.wrapping_add(line_advance as u64); + } + program.generate_row(); + } + let address_offset = program.row().address_offset + + u64::from(minimum_instruction_length); + program.end_sequence(address_offset); + } + + let mut debug_line = DebugLine::from(EndianVec::new(LittleEndian)); + let debug_line_offset = program + .write( + &mut debug_line, + encoding, + &debug_line_str_offsets, + &debug_str_offsets, + ) + .unwrap(); + + let read_debug_line = + read::DebugLine::new(debug_line.slice(), LittleEndian); + let read_program = read_debug_line + .program( + debug_line_offset, + 8, + Some(read::EndianSlice::new(dir1, LittleEndian)), + Some(read::EndianSlice::new(file1, LittleEndian)), + ) + .unwrap(); + + let mut rows = read_program.rows(); + for address_advance in addresses.clone() { + let mut address; + let mut line; + { + let row = rows.next_row().unwrap().unwrap().1; + address = row.address(); + line = row.line().unwrap().get(); + } + assert_eq!(address, 0x1000); + assert_eq!(line, 0x10000); + for line_advance in lines.clone() { + let row = rows.next_row().unwrap().unwrap().1; + assert_eq!( + row.address() - address, + address_advance * u64::from(minimum_instruction_length) + ); + assert_eq!( + (row.line().unwrap().get() as i64) - (line as i64), + line_advance + ); + address = row.address(); + line = row.line().unwrap().get(); + } + let row = rows.next_row().unwrap().unwrap().1; + assert!(row.end_sequence()); + } + } + } + } + } + } + + #[test] + fn test_line_string() { + let version = 5; + + let file = b"file1"; + + let mut strings = StringTable::default(); + let string_id = strings.add("file2"); + let mut debug_str = DebugStr::from(EndianVec::new(LittleEndian)); + let debug_str_offsets = strings.write(&mut debug_str).unwrap(); + + let mut line_strings = LineStringTable::default(); + let line_string_id = line_strings.add("file3"); + let mut debug_line_str = DebugLineStr::from(EndianVec::new(LittleEndian)); + let debug_line_str_offsets = line_strings.write(&mut debug_line_str).unwrap(); + + for &address_size in &[4, 8] { + for &format in &[Format::Dwarf32, Format::Dwarf64] { + let encoding = Encoding { + format, + version, + address_size, + }; + + for (file, expect_file) in vec![ + ( + LineString::String(file.to_vec()), + read::AttributeValue::String(read::EndianSlice::new(file, LittleEndian)), + ), + ( + LineString::StringRef(string_id), + read::AttributeValue::DebugStrRef(debug_str_offsets.get(string_id)), + ), + ( + LineString::LineStringRef(line_string_id), + read::AttributeValue::DebugLineStrRef( + debug_line_str_offsets.get(line_string_id), + ), + ), + ] { + let program = LineProgram::new( + encoding, + LineEncoding::default(), + LineString::String(b"dir".to_vec()), + file, + None, + ); + + let mut debug_line = DebugLine::from(EndianVec::new(LittleEndian)); + let debug_line_offset = program + .write( + &mut debug_line, + encoding, + &debug_line_str_offsets, + &debug_str_offsets, + ) + .unwrap(); + + let read_debug_line = read::DebugLine::new(debug_line.slice(), LittleEndian); + let read_program = read_debug_line + .program(debug_line_offset, address_size, None, None) + .unwrap(); + let read_header = read_program.header(); + assert_eq!(read_header.file(0).unwrap().path_name(), expect_file); + } + } + } + } + + #[test] + fn test_missing_comp_dir() { + let debug_line_str_offsets = DebugLineStrOffsets::none(); + let debug_str_offsets = DebugStrOffsets::none(); + + for &version in &[2, 3, 4, 5] { + for &address_size in &[4, 8] { + for &format in &[Format::Dwarf32, Format::Dwarf64] { + let encoding = Encoding { + format, + version, + address_size, + }; + let program = LineProgram::new( + encoding, + LineEncoding::default(), + LineString::String(Vec::new()), + LineString::String(Vec::new()), + None, + ); + + let mut debug_line = DebugLine::from(EndianVec::new(LittleEndian)); + let debug_line_offset = program + .write( + &mut debug_line, + encoding, + &debug_line_str_offsets, + &debug_str_offsets, + ) + .unwrap(); + + let read_debug_line = read::DebugLine::new(debug_line.slice(), LittleEndian); + let read_program = read_debug_line + .program( + debug_line_offset, + address_size, + // Testing missing comp_dir/comp_name. + None, + None, + ) + .unwrap(); + + let dwarf = read::Dwarf::default(); + let mut convert_line_strings = LineStringTable::default(); + let mut convert_strings = StringTable::default(); + let convert_address = &|address| Some(Address::Constant(address)); + LineProgram::from( + read_program, + &dwarf, + &mut convert_line_strings, + &mut convert_strings, + convert_address, + ) + .unwrap(); + } + } + } + } +} diff --git a/crux-mir/lib/gimli/src/write/loc.rs b/crux-mir/lib/gimli/src/write/loc.rs new file mode 100644 index 000000000..ea0ecb1cf --- /dev/null +++ b/crux-mir/lib/gimli/src/write/loc.rs @@ -0,0 +1,549 @@ +use alloc::vec::Vec; +use indexmap::IndexSet; +use std::ops::{Deref, DerefMut}; + +use crate::common::{Encoding, LocationListsOffset, SectionId}; +use crate::write::{ + Address, BaseId, DebugInfoReference, Error, Expression, Result, Section, Sections, UnitOffsets, + Writer, +}; + +define_section!( + DebugLoc, + LocationListsOffset, + "A writable `.debug_loc` section." +); +define_section!( + DebugLocLists, + LocationListsOffset, + "A writable `.debug_loclists` section." +); + +define_offsets!( + LocationListOffsets: LocationListId => LocationListsOffset, + "The section offsets of a series of location lists within the `.debug_loc` or `.debug_loclists` sections." +); + +define_id!( + LocationListId, + "An identifier for a location list in a `LocationListTable`." +); + +/// A table of location lists that will be stored in a `.debug_loc` or `.debug_loclists` section. +#[derive(Debug, Default)] +pub struct LocationListTable { + base_id: BaseId, + locations: IndexSet, +} + +impl LocationListTable { + /// Add a location list to the table. + pub fn add(&mut self, loc_list: LocationList) -> LocationListId { + let (index, _) = self.locations.insert_full(loc_list); + LocationListId::new(self.base_id, index) + } + + /// Write the location list table to the appropriate section for the given DWARF version. + pub(crate) fn write( + &self, + sections: &mut Sections, + encoding: Encoding, + unit_offsets: Option<&UnitOffsets>, + ) -> Result { + if self.locations.is_empty() { + return Ok(LocationListOffsets::none()); + } + + match encoding.version { + 2..=4 => self.write_loc( + &mut sections.debug_loc, + &mut sections.debug_loc_refs, + encoding, + unit_offsets, + ), + 5 => self.write_loclists( + &mut sections.debug_loclists, + &mut sections.debug_loclists_refs, + encoding, + unit_offsets, + ), + _ => Err(Error::UnsupportedVersion(encoding.version)), + } + } + + /// Write the location list table to the `.debug_loc` section. + fn write_loc( + &self, + w: &mut DebugLoc, + refs: &mut Vec, + encoding: Encoding, + unit_offsets: Option<&UnitOffsets>, + ) -> Result { + let address_size = encoding.address_size; + let mut offsets = Vec::new(); + for loc_list in self.locations.iter() { + offsets.push(w.offset()); + for loc in &loc_list.0 { + // Note that we must ensure none of the ranges have both begin == 0 and end == 0. + // We do this by ensuring that begin != end, which is a bit more restrictive + // than required, but still seems reasonable. + match *loc { + Location::BaseAddress { address } => { + let marker = !0 >> (64 - address_size * 8); + w.write_udata(marker, address_size)?; + w.write_address(address, address_size)?; + } + Location::OffsetPair { + begin, + end, + ref data, + } => { + if begin == end { + return Err(Error::InvalidRange); + } + w.write_udata(begin, address_size)?; + w.write_udata(end, address_size)?; + write_expression(&mut w.0, refs, encoding, unit_offsets, data)?; + } + Location::StartEnd { + begin, + end, + ref data, + } => { + if begin == end { + return Err(Error::InvalidRange); + } + w.write_address(begin, address_size)?; + w.write_address(end, address_size)?; + write_expression(&mut w.0, refs, encoding, unit_offsets, data)?; + } + Location::StartLength { + begin, + length, + ref data, + } => { + let end = match begin { + Address::Constant(begin) => Address::Constant(begin + length), + Address::Symbol { symbol, addend } => Address::Symbol { + symbol, + addend: addend + length as i64, + }, + }; + if begin == end { + return Err(Error::InvalidRange); + } + w.write_address(begin, address_size)?; + w.write_address(end, address_size)?; + write_expression(&mut w.0, refs, encoding, unit_offsets, data)?; + } + Location::DefaultLocation { .. } => { + return Err(Error::InvalidRange); + } + } + } + w.write_udata(0, address_size)?; + w.write_udata(0, address_size)?; + } + Ok(LocationListOffsets { + base_id: self.base_id, + offsets, + }) + } + + /// Write the location list table to the `.debug_loclists` section. + fn write_loclists( + &self, + w: &mut DebugLocLists, + refs: &mut Vec, + encoding: Encoding, + unit_offsets: Option<&UnitOffsets>, + ) -> Result { + let mut offsets = Vec::new(); + + if encoding.version != 5 { + return Err(Error::NeedVersion(5)); + } + + let length_offset = w.write_initial_length(encoding.format)?; + let length_base = w.len(); + + w.write_u16(encoding.version)?; + w.write_u8(encoding.address_size)?; + w.write_u8(0)?; // segment_selector_size + w.write_u32(0)?; // offset_entry_count (when set to zero DW_FORM_rnglistx can't be used, see section 7.28) + // FIXME implement DW_FORM_rnglistx writing and implement the offset entry list + + for loc_list in self.locations.iter() { + offsets.push(w.offset()); + for loc in &loc_list.0 { + match *loc { + Location::BaseAddress { address } => { + w.write_u8(crate::constants::DW_LLE_base_address.0)?; + w.write_address(address, encoding.address_size)?; + } + Location::OffsetPair { + begin, + end, + ref data, + } => { + w.write_u8(crate::constants::DW_LLE_offset_pair.0)?; + w.write_uleb128(begin)?; + w.write_uleb128(end)?; + write_expression(&mut w.0, refs, encoding, unit_offsets, data)?; + } + Location::StartEnd { + begin, + end, + ref data, + } => { + w.write_u8(crate::constants::DW_LLE_start_end.0)?; + w.write_address(begin, encoding.address_size)?; + w.write_address(end, encoding.address_size)?; + write_expression(&mut w.0, refs, encoding, unit_offsets, data)?; + } + Location::StartLength { + begin, + length, + ref data, + } => { + w.write_u8(crate::constants::DW_LLE_start_length.0)?; + w.write_address(begin, encoding.address_size)?; + w.write_uleb128(length)?; + write_expression(&mut w.0, refs, encoding, unit_offsets, data)?; + } + Location::DefaultLocation { ref data } => { + w.write_u8(crate::constants::DW_LLE_default_location.0)?; + write_expression(&mut w.0, refs, encoding, unit_offsets, data)?; + } + } + } + + w.write_u8(crate::constants::DW_LLE_end_of_list.0)?; + } + + let length = (w.len() - length_base) as u64; + w.write_initial_length_at(length_offset, length, encoding.format)?; + + Ok(LocationListOffsets { + base_id: self.base_id, + offsets, + }) + } +} + +/// A locations list that will be stored in a `.debug_loc` or `.debug_loclists` section. +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +pub struct LocationList(pub Vec); + +/// A single location. +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +pub enum Location { + /// DW_LLE_base_address + BaseAddress { + /// Base address. + address: Address, + }, + /// DW_LLE_offset_pair + OffsetPair { + /// Start of range relative to base address. + begin: u64, + /// End of range relative to base address. + end: u64, + /// Location description. + data: Expression, + }, + /// DW_LLE_start_end + StartEnd { + /// Start of range. + begin: Address, + /// End of range. + end: Address, + /// Location description. + data: Expression, + }, + /// DW_LLE_start_length + StartLength { + /// Start of range. + begin: Address, + /// Length of range. + length: u64, + /// Location description. + data: Expression, + }, + /// DW_LLE_default_location + DefaultLocation { + /// Location description. + data: Expression, + }, +} + +fn write_expression( + w: &mut W, + refs: &mut Vec, + encoding: Encoding, + unit_offsets: Option<&UnitOffsets>, + val: &Expression, +) -> Result<()> { + let size = val.size(encoding, unit_offsets) as u64; + if encoding.version <= 4 { + w.write_udata(size, 2)?; + } else { + w.write_uleb128(size)?; + } + val.write(w, Some(refs), encoding, unit_offsets)?; + Ok(()) +} + +#[cfg(feature = "read")] +mod convert { + use super::*; + + use crate::read::{self, Reader}; + use crate::write::{ConvertError, ConvertResult, ConvertUnitContext}; + + impl LocationList { + /// Create a location list by reading the data from the give location list iter. + pub(crate) fn from>( + mut from: read::RawLocListIter, + context: &ConvertUnitContext, + ) -> ConvertResult { + let mut have_base_address = context.base_address != Address::Constant(0); + let convert_address = + |x| (context.convert_address)(x).ok_or(ConvertError::InvalidAddress); + let convert_expression = |x| { + Expression::from( + x, + context.unit.encoding(), + Some(context.dwarf), + Some(context.unit), + Some(context.entry_ids), + context.convert_address, + ) + }; + let mut loc_list = Vec::new(); + while let Some(from_loc) = from.next()? { + let loc = match from_loc { + read::RawLocListEntry::AddressOrOffsetPair { begin, end, data } => { + // These were parsed as addresses, even if they are offsets. + let begin = convert_address(begin)?; + let end = convert_address(end)?; + let data = convert_expression(data)?; + match (begin, end) { + (Address::Constant(begin_offset), Address::Constant(end_offset)) => { + if have_base_address { + Location::OffsetPair { + begin: begin_offset, + end: end_offset, + data, + } + } else { + Location::StartEnd { begin, end, data } + } + } + _ => { + if have_base_address { + // At least one of begin/end is an address, but we also have + // a base address. Adding addresses is undefined. + return Err(ConvertError::InvalidRangeRelativeAddress); + } + Location::StartEnd { begin, end, data } + } + } + } + read::RawLocListEntry::BaseAddress { addr } => { + have_base_address = true; + let address = convert_address(addr)?; + Location::BaseAddress { address } + } + read::RawLocListEntry::BaseAddressx { addr } => { + have_base_address = true; + let address = convert_address(context.dwarf.address(context.unit, addr)?)?; + Location::BaseAddress { address } + } + read::RawLocListEntry::StartxEndx { begin, end, data } => { + let begin = convert_address(context.dwarf.address(context.unit, begin)?)?; + let end = convert_address(context.dwarf.address(context.unit, end)?)?; + let data = convert_expression(data)?; + Location::StartEnd { begin, end, data } + } + read::RawLocListEntry::StartxLength { + begin, + length, + data, + } => { + let begin = convert_address(context.dwarf.address(context.unit, begin)?)?; + let data = convert_expression(data)?; + Location::StartLength { + begin, + length, + data, + } + } + read::RawLocListEntry::OffsetPair { begin, end, data } => { + let data = convert_expression(data)?; + Location::OffsetPair { begin, end, data } + } + read::RawLocListEntry::StartEnd { begin, end, data } => { + let begin = convert_address(begin)?; + let end = convert_address(end)?; + let data = convert_expression(data)?; + Location::StartEnd { begin, end, data } + } + read::RawLocListEntry::StartLength { + begin, + length, + data, + } => { + let begin = convert_address(begin)?; + let data = convert_expression(data)?; + Location::StartLength { + begin, + length, + data, + } + } + read::RawLocListEntry::DefaultLocation { data } => { + let data = convert_expression(data)?; + Location::DefaultLocation { data } + } + }; + // In some cases, existing data may contain begin == end, filtering + // these out. + match loc { + Location::StartLength { length, .. } if length == 0 => continue, + Location::StartEnd { begin, end, .. } if begin == end => continue, + Location::OffsetPair { begin, end, .. } if begin == end => continue, + _ => (), + } + loc_list.push(loc); + } + Ok(LocationList(loc_list)) + } + } +} + +#[cfg(test)] +#[cfg(feature = "read")] +mod tests { + use super::*; + use crate::common::{ + DebugAbbrevOffset, DebugAddrBase, DebugInfoOffset, DebugLocListsBase, DebugRngListsBase, + DebugStrOffsetsBase, Format, + }; + use crate::read; + use crate::write::{ + ConvertUnitContext, EndianVec, LineStringTable, RangeListTable, StringTable, + }; + use crate::LittleEndian; + use std::collections::HashMap; + + #[test] + fn test_loc_list() { + let mut line_strings = LineStringTable::default(); + let mut strings = StringTable::default(); + let mut expression = Expression::new(); + expression.op_constu(0); + + for &version in &[2, 3, 4, 5] { + for &address_size in &[4, 8] { + for &format in &[Format::Dwarf32, Format::Dwarf64] { + let encoding = Encoding { + format, + version, + address_size, + }; + + let mut loc_list = LocationList(vec![ + Location::StartLength { + begin: Address::Constant(6666), + length: 7777, + data: expression.clone(), + }, + Location::StartEnd { + begin: Address::Constant(4444), + end: Address::Constant(5555), + data: expression.clone(), + }, + Location::BaseAddress { + address: Address::Constant(1111), + }, + Location::OffsetPair { + begin: 2222, + end: 3333, + data: expression.clone(), + }, + ]); + if version >= 5 { + loc_list.0.push(Location::DefaultLocation { + data: expression.clone(), + }); + } + + let mut locations = LocationListTable::default(); + let loc_list_id = locations.add(loc_list.clone()); + + let mut sections = Sections::new(EndianVec::new(LittleEndian)); + let loc_list_offsets = locations.write(&mut sections, encoding, None).unwrap(); + assert!(sections.debug_loc_refs.is_empty()); + assert!(sections.debug_loclists_refs.is_empty()); + + let read_debug_loc = + read::DebugLoc::new(sections.debug_loc.slice(), LittleEndian); + let read_debug_loclists = + read::DebugLocLists::new(sections.debug_loclists.slice(), LittleEndian); + let read_loc = read::LocationLists::new(read_debug_loc, read_debug_loclists); + let offset = loc_list_offsets.get(loc_list_id); + let read_loc_list = read_loc.raw_locations(offset, encoding).unwrap(); + + let dwarf = read::Dwarf { + locations: read_loc, + ..Default::default() + }; + let unit = read::Unit { + header: read::UnitHeader::new( + encoding, + 0, + read::UnitType::Compilation, + DebugAbbrevOffset(0), + DebugInfoOffset(0).into(), + read::EndianSlice::default(), + ), + abbreviations: read::Abbreviations::default(), + name: None, + comp_dir: None, + low_pc: 0, + str_offsets_base: DebugStrOffsetsBase(0), + addr_base: DebugAddrBase(0), + loclists_base: DebugLocListsBase(0), + rnglists_base: DebugRngListsBase(0), + line_program: None, + dwo_id: None, + }; + let context = ConvertUnitContext { + dwarf: &dwarf, + unit: &unit, + line_strings: &mut line_strings, + strings: &mut strings, + ranges: &mut RangeListTable::default(), + locations: &mut locations, + convert_address: &|address| Some(Address::Constant(address)), + base_address: Address::Constant(0), + line_program_offset: None, + line_program_files: Vec::new(), + entry_ids: &HashMap::new(), + }; + let convert_loc_list = LocationList::from(read_loc_list, &context).unwrap(); + + if version <= 4 { + loc_list.0[0] = Location::StartEnd { + begin: Address::Constant(6666), + end: Address::Constant(6666 + 7777), + data: expression.clone(), + }; + } + assert_eq!(loc_list, convert_loc_list); + } + } + } + } +} diff --git a/crux-mir/lib/gimli/src/write/mod.rs b/crux-mir/lib/gimli/src/write/mod.rs new file mode 100644 index 000000000..47ba6319d --- /dev/null +++ b/crux-mir/lib/gimli/src/write/mod.rs @@ -0,0 +1,425 @@ +//! Write DWARF debugging information. +//! +//! ## API Structure +//! +//! This module works by building up a representation of the debugging information +//! in memory, and then writing it all at once. It supports two major use cases: +//! +//! * Use the [`DwarfUnit`](./struct.DwarfUnit.html) type when writing DWARF +//! for a single compilation unit. +//! +//! * Use the [`Dwarf`](./struct.Dwarf.html) type when writing DWARF for multiple +//! compilation units. +//! +//! The module also supports reading in DWARF debugging information and writing it out +//! again, possibly after modifying it. Create a [`read::Dwarf`](../read/struct.Dwarf.html) +//! instance, and then use [`Dwarf::from`](./struct.Dwarf.html#method.from) to convert +//! it to a writable instance. +//! +//! ## Example Usage +//! +//! Write a compilation unit containing only the top level DIE. +//! +//! ```rust +//! use gimli::write::{ +//! Address, AttributeValue, DwarfUnit, EndianVec, Error, Range, RangeList, Sections, +//! }; +//! +//! fn example() -> Result<(), Error> { +//! // Choose the encoding parameters. +//! let encoding = gimli::Encoding { +//! format: gimli::Format::Dwarf32, +//! version: 5, +//! address_size: 8, +//! }; +//! // Create a container for a single compilation unit. +//! let mut dwarf = DwarfUnit::new(encoding); +//! // Set a range attribute on the root DIE. +//! let range_list = RangeList(vec![Range::StartLength { +//! begin: Address::Constant(0x100), +//! length: 42, +//! }]); +//! let range_list_id = dwarf.unit.ranges.add(range_list); +//! let root = dwarf.unit.root(); +//! dwarf.unit.get_mut(root).set( +//! gimli::DW_AT_ranges, +//! AttributeValue::RangeListRef(range_list_id), +//! ); +//! // Create a `Vec` for each DWARF section. +//! let mut sections = Sections::new(EndianVec::new(gimli::LittleEndian)); +//! // Finally, write the DWARF data to the sections. +//! dwarf.write(&mut sections)?; +//! sections.for_each(|id, data| { +//! // Here you can add the data to the output object file. +//! Ok(()) +//! }) +//! } +//! # fn main() { +//! # example().unwrap(); +//! # } + +use std::error; +use std::fmt; +use std::result; + +use crate::constants; + +mod endian_vec; +pub use self::endian_vec::*; + +mod writer; +pub use self::writer::*; + +#[macro_use] +mod section; +pub use self::section::*; + +macro_rules! define_id { + ($name:ident, $docs:expr) => { + #[doc=$docs] + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + pub struct $name { + base_id: BaseId, + index: usize, + } + + impl $name { + #[inline] + fn new(base_id: BaseId, index: usize) -> Self { + $name { base_id, index } + } + } + }; +} + +macro_rules! define_offsets { + ($offsets:ident: $id:ident => $offset:ident, $off_doc:expr) => { + #[doc=$off_doc] + #[derive(Debug)] + pub struct $offsets { + base_id: BaseId, + // We know ids start at 0. + offsets: Vec<$offset>, + } + + impl $offsets { + /// Return an empty list of offsets. + #[inline] + pub fn none() -> Self { + $offsets { + base_id: BaseId::default(), + offsets: Vec::new(), + } + } + + /// Get the offset + /// + /// # Panics + /// + /// Panics if `id` is invalid. + #[inline] + pub fn get(&self, id: $id) -> $offset { + debug_assert_eq!(self.base_id, id.base_id); + self.offsets[id.index] + } + + /// Return the number of offsets. + #[inline] + pub fn count(&self) -> usize { + self.offsets.len() + } + } + }; +} + +mod abbrev; +pub use self::abbrev::*; + +mod cfi; +pub use self::cfi::*; + +mod dwarf; +pub use self::dwarf::*; + +mod line; +pub use self::line::*; + +mod loc; +pub use self::loc::*; + +mod op; +pub use self::op::*; + +mod range; +pub use self::range::*; + +mod str; +pub use self::str::*; + +mod unit; +pub use self::unit::*; + +/// An error that occurred when writing. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Error { + /// The given offset is out of bounds. + OffsetOutOfBounds, + /// The given length is out of bounds. + LengthOutOfBounds, + /// The attribute value is an invalid for writing. + InvalidAttributeValue, + /// The value is too large for the encoding form. + ValueTooLarge, + /// Unsupported word size. + UnsupportedWordSize(u8), + /// Unsupported DWARF version. + UnsupportedVersion(u16), + /// The unit length is too large for the requested DWARF format. + InitialLengthOverflow, + /// The address is invalid. + InvalidAddress, + /// The reference is invalid. + InvalidReference, + /// A requested feature requires a different DWARF version. + NeedVersion(u16), + /// Strings in line number program have mismatched forms. + LineStringFormMismatch, + /// The range is empty or otherwise invalid. + InvalidRange, + /// The line number program encoding is incompatible with the unit encoding. + IncompatibleLineProgramEncoding, + /// Could not encode code offset for a frame instruction. + InvalidFrameCodeOffset(u32), + /// Could not encode data offset for a frame instruction. + InvalidFrameDataOffset(i32), + /// Unsupported eh_frame pointer encoding. + UnsupportedPointerEncoding(constants::DwEhPe), + /// Unsupported reference in CFI expression. + UnsupportedCfiExpressionReference, + /// Unsupported forward reference in expression. + UnsupportedExpressionForwardReference, +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> result::Result<(), fmt::Error> { + match *self { + Error::OffsetOutOfBounds => write!(f, "The given offset is out of bounds."), + Error::LengthOutOfBounds => write!(f, "The given length is out of bounds."), + Error::InvalidAttributeValue => { + write!(f, "The attribute value is an invalid for writing.") + } + Error::ValueTooLarge => write!(f, "The value is too large for the encoding form."), + Error::UnsupportedWordSize(size) => write!(f, "Unsupported word size: {}", size), + Error::UnsupportedVersion(version) => { + write!(f, "Unsupported DWARF version: {}", version) + } + Error::InitialLengthOverflow => write!( + f, + "The unit length is too large for the requested DWARF format." + ), + Error::InvalidAddress => write!(f, "The address is invalid."), + Error::InvalidReference => write!(f, "The reference is invalid."), + Error::NeedVersion(version) => write!( + f, + "A requested feature requires a DWARF version {}.", + version + ), + Error::LineStringFormMismatch => { + write!(f, "Strings in line number program have mismatched forms.") + } + Error::InvalidRange => write!(f, "The range is empty or otherwise invalid."), + Error::IncompatibleLineProgramEncoding => write!( + f, + "The line number program encoding is incompatible with the unit encoding." + ), + Error::InvalidFrameCodeOffset(offset) => write!( + f, + "Could not encode code offset ({}) for a frame instruction.", + offset, + ), + Error::InvalidFrameDataOffset(offset) => write!( + f, + "Could not encode data offset ({}) for a frame instruction.", + offset, + ), + Error::UnsupportedPointerEncoding(eh_pe) => { + write!(f, "Unsupported eh_frame pointer encoding ({}).", eh_pe) + } + Error::UnsupportedCfiExpressionReference => { + write!(f, "Unsupported reference in CFI expression.") + } + Error::UnsupportedExpressionForwardReference => { + write!(f, "Unsupported forward reference in expression.") + } + } + } +} + +impl error::Error for Error {} + +/// The result of a write. +pub type Result = result::Result; + +/// An address. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Address { + /// A fixed address that does not require relocation. + Constant(u64), + /// An address that is relative to a symbol which may be relocated. + Symbol { + /// The symbol that the address is relative to. + /// + /// The meaning of this value is decided by the writer, but + /// will typically be an index into a symbol table. + symbol: usize, + /// The offset of the address relative to the symbol. + /// + /// This will typically be used as the addend in a relocation. + addend: i64, + }, +} + +/// A reference to a `.debug_info` entry. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Reference { + /// An external symbol. + /// + /// The meaning of this value is decided by the writer, but + /// will typically be an index into a symbol table. + Symbol(usize), + /// An entry in the same section. + /// + /// This only supports references in units that are emitted together. + Entry(UnitId, UnitEntryId), +} + +// This type is only used in debug assertions. +#[cfg(not(debug_assertions))] +type BaseId = (); + +#[cfg(debug_assertions)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct BaseId(usize); + +#[cfg(debug_assertions)] +impl Default for BaseId { + fn default() -> Self { + use std::sync::atomic; + static BASE_ID: atomic::AtomicUsize = atomic::AtomicUsize::new(0); + BaseId(BASE_ID.fetch_add(1, atomic::Ordering::Relaxed)) + } +} + +#[cfg(feature = "read")] +mod convert { + use super::*; + use crate::read; + + pub(crate) use super::unit::convert::*; + + /// An error that occurred when converting a read value into a write value. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum ConvertError { + /// An error occurred when reading. + Read(read::Error), + /// Writing of this attribute value is not implemented yet. + UnsupportedAttributeValue, + /// This attribute value is an invalid name/form combination. + InvalidAttributeValue, + /// A `.debug_info` reference does not refer to a valid entry. + InvalidDebugInfoOffset, + /// An address could not be converted. + InvalidAddress, + /// Writing this line number instruction is not implemented yet. + UnsupportedLineInstruction, + /// Writing this form of line string is not implemented yet. + UnsupportedLineStringForm, + /// A `.debug_line` file index is invalid. + InvalidFileIndex, + /// A `.debug_line` directory index is invalid. + InvalidDirectoryIndex, + /// A `.debug_line` line base is invalid. + InvalidLineBase, + /// A `.debug_line` reference is invalid. + InvalidLineRef, + /// A `.debug_info` unit entry reference is invalid. + InvalidUnitRef, + /// A `.debug_info` reference is invalid. + InvalidDebugInfoRef, + /// Invalid relative address in a range list. + InvalidRangeRelativeAddress, + /// Writing this CFI instruction is not implemented yet. + UnsupportedCfiInstruction, + /// Writing indirect pointers is not implemented yet. + UnsupportedIndirectAddress, + /// Writing this expression operation is not implemented yet. + UnsupportedOperation, + /// Operation branch target is invalid. + InvalidBranchTarget, + /// Writing this unit type is not supported yet. + UnsupportedUnitType, + } + + impl fmt::Display for ConvertError { + fn fmt(&self, f: &mut fmt::Formatter) -> result::Result<(), fmt::Error> { + use self::ConvertError::*; + match *self { + Read(ref e) => e.fmt(f), + UnsupportedAttributeValue => { + write!(f, "Writing of this attribute value is not implemented yet.") + } + InvalidAttributeValue => write!( + f, + "This attribute value is an invalid name/form combination." + ), + InvalidDebugInfoOffset => write!( + f, + "A `.debug_info` reference does not refer to a valid entry." + ), + InvalidAddress => write!(f, "An address could not be converted."), + UnsupportedLineInstruction => write!( + f, + "Writing this line number instruction is not implemented yet." + ), + UnsupportedLineStringForm => write!( + f, + "Writing this form of line string is not implemented yet." + ), + InvalidFileIndex => write!(f, "A `.debug_line` file index is invalid."), + InvalidDirectoryIndex => write!(f, "A `.debug_line` directory index is invalid."), + InvalidLineBase => write!(f, "A `.debug_line` line base is invalid."), + InvalidLineRef => write!(f, "A `.debug_line` reference is invalid."), + InvalidUnitRef => write!(f, "A `.debug_info` unit entry reference is invalid."), + InvalidDebugInfoRef => write!(f, "A `.debug_info` reference is invalid."), + InvalidRangeRelativeAddress => { + write!(f, "Invalid relative address in a range list.") + } + UnsupportedCfiInstruction => { + write!(f, "Writing this CFI instruction is not implemented yet.") + } + UnsupportedIndirectAddress => { + write!(f, "Writing indirect pointers is not implemented yet.") + } + UnsupportedOperation => write!( + f, + "Writing this expression operation is not implemented yet." + ), + InvalidBranchTarget => write!(f, "Operation branch target is invalid."), + UnsupportedUnitType => write!(f, "Writing this unit type is not supported yet."), + } + } + } + + impl error::Error for ConvertError {} + + impl From for ConvertError { + fn from(e: read::Error) -> Self { + ConvertError::Read(e) + } + } + + /// The result of a conversion. + pub type ConvertResult = result::Result; +} +#[cfg(feature = "read")] +pub use self::convert::*; diff --git a/crux-mir/lib/gimli/src/write/op.rs b/crux-mir/lib/gimli/src/write/op.rs new file mode 100644 index 000000000..c70eec2dd --- /dev/null +++ b/crux-mir/lib/gimli/src/write/op.rs @@ -0,0 +1,1621 @@ +use alloc::boxed::Box; +use alloc::vec::Vec; + +use crate::common::{Encoding, Register}; +use crate::constants::{self, DwOp}; +use crate::leb128::write::{sleb128_size, uleb128_size}; +use crate::write::{ + Address, DebugInfoReference, Error, Reference, Result, UnitEntryId, UnitOffsets, Writer, +}; + +/// The bytecode for a DWARF expression or location description. +#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] +pub struct Expression { + operations: Vec, +} + +impl Expression { + /// Create an empty expression. + #[inline] + pub fn new() -> Self { + Self::default() + } + + /// Create an expression from raw bytecode. + /// + /// This does not support operations that require references, such as `DW_OP_addr`. + #[inline] + pub fn raw(bytecode: Vec) -> Self { + Expression { + operations: vec![Operation::Raw(bytecode)], + } + } + + /// Add an operation to the expression. + /// + /// This should only be used for operations that have no explicit operands. + pub fn op(&mut self, opcode: DwOp) { + self.operations.push(Operation::Simple(opcode)); + } + + /// Add a `DW_OP_addr` operation to the expression. + pub fn op_addr(&mut self, address: Address) { + self.operations.push(Operation::Address(address)); + } + + /// Add a `DW_OP_constu` operation to the expression. + /// + /// This may be emitted as a smaller equivalent operation. + pub fn op_constu(&mut self, value: u64) { + self.operations.push(Operation::UnsignedConstant(value)); + } + + /// Add a `DW_OP_consts` operation to the expression. + /// + /// This may be emitted as a smaller equivalent operation. + pub fn op_consts(&mut self, value: i64) { + self.operations.push(Operation::SignedConstant(value)); + } + + /// Add a `DW_OP_const_type` or `DW_OP_GNU_const_type` operation to the expression. + pub fn op_const_type(&mut self, base: UnitEntryId, value: Box<[u8]>) { + self.operations.push(Operation::ConstantType(base, value)); + } + + /// Add a `DW_OP_fbreg` operation to the expression. + pub fn op_fbreg(&mut self, offset: i64) { + self.operations.push(Operation::FrameOffset(offset)); + } + + /// Add a `DW_OP_bregx` operation to the expression. + /// + /// This may be emitted as a smaller equivalent operation. + pub fn op_breg(&mut self, register: Register, offset: i64) { + self.operations + .push(Operation::RegisterOffset(register, offset)); + } + + /// Add a `DW_OP_regval_type` or `DW_OP_GNU_regval_type` operation to the expression. + /// + /// This may be emitted as a smaller equivalent operation. + pub fn op_regval_type(&mut self, register: Register, base: UnitEntryId) { + self.operations + .push(Operation::RegisterType(register, base)); + } + + /// Add a `DW_OP_pick` operation to the expression. + /// + /// This may be emitted as a `DW_OP_dup` or `DW_OP_over` operation. + pub fn op_pick(&mut self, index: u8) { + self.operations.push(Operation::Pick(index)); + } + + /// Add a `DW_OP_deref` operation to the expression. + pub fn op_deref(&mut self) { + self.operations.push(Operation::Deref { space: false }); + } + + /// Add a `DW_OP_xderef` operation to the expression. + pub fn op_xderef(&mut self) { + self.operations.push(Operation::Deref { space: true }); + } + + /// Add a `DW_OP_deref_size` operation to the expression. + pub fn op_deref_size(&mut self, size: u8) { + self.operations + .push(Operation::DerefSize { size, space: false }); + } + + /// Add a `DW_OP_xderef_size` operation to the expression. + pub fn op_xderef_size(&mut self, size: u8) { + self.operations + .push(Operation::DerefSize { size, space: true }); + } + + /// Add a `DW_OP_deref_type` or `DW_OP_GNU_deref_type` operation to the expression. + pub fn op_deref_type(&mut self, size: u8, base: UnitEntryId) { + self.operations.push(Operation::DerefType { + size, + base, + space: false, + }); + } + + /// Add a `DW_OP_xderef_type` operation to the expression. + pub fn op_xderef_type(&mut self, size: u8, base: UnitEntryId) { + self.operations.push(Operation::DerefType { + size, + base, + space: true, + }); + } + + /// Add a `DW_OP_plus_uconst` operation to the expression. + pub fn op_plus_uconst(&mut self, value: u64) { + self.operations.push(Operation::PlusConstant(value)); + } + + /// Add a `DW_OP_skip` operation to the expression. + /// + /// Returns the index of the operation. The caller must call `set_target` with + /// this index to set the target of the branch. + pub fn op_skip(&mut self) -> usize { + let index = self.next_index(); + self.operations.push(Operation::Skip(!0)); + index + } + + /// Add a `DW_OP_bra` operation to the expression. + /// + /// Returns the index of the operation. The caller must call `set_target` with + /// this index to set the target of the branch. + pub fn op_bra(&mut self) -> usize { + let index = self.next_index(); + self.operations.push(Operation::Branch(!0)); + index + } + + /// Return the index that will be assigned to the next operation. + /// + /// This can be passed to `set_target`. + #[inline] + pub fn next_index(&self) -> usize { + self.operations.len() + } + + /// Set the target of a `DW_OP_skip` or `DW_OP_bra` operation . + pub fn set_target(&mut self, operation: usize, new_target: usize) { + debug_assert!(new_target <= self.next_index()); + debug_assert_ne!(operation, new_target); + match self.operations[operation] { + Operation::Skip(ref mut target) | Operation::Branch(ref mut target) => { + *target = new_target; + } + _ => unimplemented!(), + } + } + + /// Add a `DW_OP_call4` operation to the expression. + pub fn op_call(&mut self, entry: UnitEntryId) { + self.operations.push(Operation::Call(entry)); + } + + /// Add a `DW_OP_call_ref` operation to the expression. + pub fn op_call_ref(&mut self, entry: Reference) { + self.operations.push(Operation::CallRef(entry)); + } + + /// Add a `DW_OP_convert` or `DW_OP_GNU_convert` operation to the expression. + /// + /// `base` is the DIE of the base type, or `None` for the generic type. + pub fn op_convert(&mut self, base: Option) { + self.operations.push(Operation::Convert(base)); + } + + /// Add a `DW_OP_reinterpret` or `DW_OP_GNU_reinterpret` operation to the expression. + /// + /// `base` is the DIE of the base type, or `None` for the generic type. + pub fn op_reinterpret(&mut self, base: Option) { + self.operations.push(Operation::Reinterpret(base)); + } + + /// Add a `DW_OP_entry_value` or `DW_OP_GNU_entry_value` operation to the expression. + pub fn op_entry_value(&mut self, expression: Expression) { + self.operations.push(Operation::EntryValue(expression)); + } + + /// Add a `DW_OP_regx` operation to the expression. + /// + /// This may be emitted as a smaller equivalent operation. + pub fn op_reg(&mut self, register: Register) { + self.operations.push(Operation::Register(register)); + } + + /// Add a `DW_OP_implicit_value` operation to the expression. + pub fn op_implicit_value(&mut self, data: Box<[u8]>) { + self.operations.push(Operation::ImplicitValue(data)); + } + + /// Add a `DW_OP_implicit_pointer` or `DW_OP_GNU_implicit_pointer` operation to the expression. + pub fn op_implicit_pointer(&mut self, entry: Reference, byte_offset: i64) { + self.operations + .push(Operation::ImplicitPointer { entry, byte_offset }); + } + + /// Add a `DW_OP_piece` operation to the expression. + pub fn op_piece(&mut self, size_in_bytes: u64) { + self.operations.push(Operation::Piece { size_in_bytes }); + } + + /// Add a `DW_OP_bit_piece` operation to the expression. + pub fn op_bit_piece(&mut self, size_in_bits: u64, bit_offset: u64) { + self.operations.push(Operation::BitPiece { + size_in_bits, + bit_offset, + }); + } + + /// Add a `DW_OP_GNU_parameter_ref` operation to the expression. + pub fn op_gnu_parameter_ref(&mut self, entry: UnitEntryId) { + self.operations.push(Operation::ParameterRef(entry)); + } + + /// Add a `DW_OP_WASM_location 0x0` operation to the expression. + pub fn op_wasm_local(&mut self, index: u32) { + self.operations.push(Operation::WasmLocal(index)); + } + + /// Add a `DW_OP_WASM_location 0x1` operation to the expression. + pub fn op_wasm_global(&mut self, index: u32) { + self.operations.push(Operation::WasmGlobal(index)); + } + + /// Add a `DW_OP_WASM_location 0x2` operation to the expression. + pub fn op_wasm_stack(&mut self, index: u32) { + self.operations.push(Operation::WasmStack(index)); + } + + pub(crate) fn size(&self, encoding: Encoding, unit_offsets: Option<&UnitOffsets>) -> usize { + let mut size = 0; + for operation in &self.operations { + size += operation.size(encoding, unit_offsets); + } + size + } + + pub(crate) fn write( + &self, + w: &mut W, + mut refs: Option<&mut Vec>, + encoding: Encoding, + unit_offsets: Option<&UnitOffsets>, + ) -> Result<()> { + // TODO: only calculate offsets if needed? + let mut offsets = Vec::with_capacity(self.operations.len()); + let mut offset = w.len(); + for operation in &self.operations { + offsets.push(offset); + offset += operation.size(encoding, unit_offsets); + } + offsets.push(offset); + for (operation, offset) in self.operations.iter().zip(offsets.iter().copied()) { + let refs = match refs { + Some(ref mut refs) => Some(&mut **refs), + None => None, + }; + debug_assert_eq!(w.len(), offset); + operation.write(w, refs, encoding, unit_offsets, &offsets)?; + } + Ok(()) + } +} + +/// A single DWARF operation. +// +// This type is intentionally not public so that we can change the +// representation of expressions as needed. +// +// Variants are listed in the order they appear in Section 2.5. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +enum Operation { + /// Raw bytecode. + /// + /// Does not support references. + Raw(Vec), + /// An operation that has no explicit operands. + /// + /// Represents: + /// - `DW_OP_drop`, `DW_OP_swap`, `DW_OP_rot` + /// - `DW_OP_push_object_address`, `DW_OP_form_tls_address`, `DW_OP_call_frame_cfa` + /// - `DW_OP_abs`, `DW_OP_and`, `DW_OP_div`, `DW_OP_minus`, `DW_OP_mod`, `DW_OP_mul`, + /// `DW_OP_neg`, `DW_OP_not`, `DW_OP_or`, `DW_OP_plus`, `DW_OP_shl`, `DW_OP_shr`, + /// `DW_OP_shra`, `DW_OP_xor` + /// - `DW_OP_le`, `DW_OP_ge`, `DW_OP_eq`, `DW_OP_lt`, `DW_OP_gt`, `DW_OP_ne` + /// - `DW_OP_nop` + /// - `DW_OP_stack_value` + Simple(DwOp), + /// Relocate the address if needed, and push it on the stack. + /// + /// Represents `DW_OP_addr`. + Address(Address), + /// Push an unsigned constant value on the stack. + /// + /// Represents `DW_OP_constu`. + UnsignedConstant(u64), + /// Push a signed constant value on the stack. + /// + /// Represents `DW_OP_consts`. + SignedConstant(i64), + /* TODO: requires .debug_addr write support + /// Read the address at the given index in `.debug_addr, relocate the address if needed, + /// and push it on the stack. + /// + /// Represents `DW_OP_addrx`. + AddressIndex(DebugAddrIndex), + /// Read the address at the given index in `.debug_addr, and push it on the stack. + /// Do not relocate the address. + /// + /// Represents `DW_OP_constx`. + ConstantIndex(DebugAddrIndex), + */ + /// Interpret the value bytes as a constant of a given type, and push it on the stack. + /// + /// Represents `DW_OP_const_type`. + ConstantType(UnitEntryId, Box<[u8]>), + /// Compute the frame base (using `DW_AT_frame_base`), add the + /// given offset, and then push the resulting sum on the stack. + /// + /// Represents `DW_OP_fbreg`. + FrameOffset(i64), + /// Find the contents of the given register, add the offset, and then + /// push the resulting sum on the stack. + /// + /// Represents `DW_OP_bregx`. + RegisterOffset(Register, i64), + /// Interpret the contents of the given register as a value of the given type, + /// and push it on the stack. + /// + /// Represents `DW_OP_regval_type`. + RegisterType(Register, UnitEntryId), + /// Copy the item at a stack index and push it on top of the stack. + /// + /// Represents `DW_OP_pick`, `DW_OP_dup`, and `DW_OP_over`. + Pick(u8), + /// Pop the topmost value of the stack, dereference it, and push the + /// resulting value. + /// + /// Represents `DW_OP_deref` and `DW_OP_xderef`. + Deref { + /// True if the dereference operation takes an address space + /// argument from the stack; false otherwise. + space: bool, + }, + /// Pop the topmost value of the stack, dereference it to obtain a value + /// of the given size, and push the resulting value. + /// + /// Represents `DW_OP_deref_size` and `DW_OP_xderef_size`. + DerefSize { + /// True if the dereference operation takes an address space + /// argument from the stack; false otherwise. + space: bool, + /// The size of the data to dereference. + size: u8, + }, + /// Pop the topmost value of the stack, dereference it to obtain a value + /// of the given type, and push the resulting value. + /// + /// Represents `DW_OP_deref_type` and `DW_OP_xderef_type`. + DerefType { + /// True if the dereference operation takes an address space + /// argument from the stack; false otherwise. + space: bool, + /// The size of the data to dereference. + size: u8, + /// The DIE of the base type, or `None` for the generic type. + base: UnitEntryId, + }, + /// Add an unsigned constant to the topmost value on the stack. + /// + /// Represents `DW_OP_plus_uconst`. + PlusConstant(u64), + /// Unconditional branch to the target location. + /// + /// The value is the index within the expression of the operation to branch to. + /// This will be converted to a relative offset when writing. + /// + /// Represents `DW_OP_skip`. + Skip(usize), + /// Branch to the target location if the top of stack is nonzero. + /// + /// The value is the index within the expression of the operation to branch to. + /// This will be converted to a relative offset when writing. + /// + /// Represents `DW_OP_bra`. + Branch(usize), + /// Evaluate a DWARF expression as a subroutine. + /// + /// The expression comes from the `DW_AT_location` attribute of the indicated DIE. + /// + /// Represents `DW_OP_call4`. + Call(UnitEntryId), + /// Evaluate an external DWARF expression as a subroutine. + /// + /// The expression comes from the `DW_AT_location` attribute of the indicated DIE, + /// which may be in another compilation unit or shared object. + /// + /// Represents `DW_OP_call_ref`. + CallRef(Reference), + /// Pop the top stack entry, convert it to a different type, and push it on the stack. + /// + /// Represents `DW_OP_convert`. + Convert(Option), + /// Pop the top stack entry, reinterpret the bits in its value as a different type, + /// and push it on the stack. + /// + /// Represents `DW_OP_reinterpret`. + Reinterpret(Option), + /// Evaluate an expression at the entry to the current subprogram, and push it on the stack. + /// + /// Represents `DW_OP_entry_value`. + EntryValue(Expression), + // FIXME: EntryRegister + /// Indicate that this piece's location is in the given register. + /// + /// Completes the piece or expression. + /// + /// Represents `DW_OP_regx`. + Register(Register), + /// The object has no location, but has a known constant value. + /// + /// Completes the piece or expression. + /// + /// Represents `DW_OP_implicit_value`. + ImplicitValue(Box<[u8]>), + /// The object is a pointer to a value which has no actual location, such as + /// an implicit value or a stack value. + /// + /// Completes the piece or expression. + /// + /// Represents `DW_OP_implicit_pointer`. + ImplicitPointer { + /// The DIE of the value that this is an implicit pointer into. + entry: Reference, + /// The byte offset into the value that the implicit pointer points to. + byte_offset: i64, + }, + /// Terminate a piece. + /// + /// Represents `DW_OP_piece`. + Piece { + /// The size of this piece in bytes. + size_in_bytes: u64, + }, + /// Terminate a piece with a size in bits. + /// + /// Represents `DW_OP_bit_piece`. + BitPiece { + /// The size of this piece in bits. + size_in_bits: u64, + /// The bit offset of this piece. + bit_offset: u64, + }, + /// This represents a parameter that was optimized out. + /// + /// The entry is the definition of the parameter, and is matched to + /// the `DW_TAG_GNU_call_site_parameter` in the caller that also + /// points to the same definition of the parameter. + /// + /// Represents `DW_OP_GNU_parameter_ref`. + ParameterRef(UnitEntryId), + /// The index of a local in the currently executing function. + /// + /// Represents `DW_OP_WASM_location 0x00`. + WasmLocal(u32), + /// The index of a global. + /// + /// Represents `DW_OP_WASM_location 0x01`. + WasmGlobal(u32), + /// The index of an item on the operand stack. + /// + /// Represents `DW_OP_WASM_location 0x02`. + WasmStack(u32), +} + +impl Operation { + fn size(&self, encoding: Encoding, unit_offsets: Option<&UnitOffsets>) -> usize { + let base_size = |base| { + // Errors are handled during writes. + match unit_offsets { + Some(offsets) => uleb128_size(offsets.unit_offset(base)), + None => 0, + } + }; + 1 + match *self { + Operation::Raw(ref bytecode) => return bytecode.len(), + Operation::Simple(_) => 0, + Operation::Address(_) => encoding.address_size as usize, + Operation::UnsignedConstant(value) => { + if value < 32 { + 0 + } else { + uleb128_size(value) + } + } + Operation::SignedConstant(value) => sleb128_size(value), + Operation::ConstantType(base, ref value) => base_size(base) + 1 + value.len(), + Operation::FrameOffset(offset) => sleb128_size(offset), + Operation::RegisterOffset(register, offset) => { + if register.0 < 32 { + sleb128_size(offset) + } else { + uleb128_size(register.0.into()) + sleb128_size(offset) + } + } + Operation::RegisterType(register, base) => { + uleb128_size(register.0.into()) + base_size(base) + } + Operation::Pick(index) => { + if index > 1 { + 1 + } else { + 0 + } + } + Operation::Deref { .. } => 0, + Operation::DerefSize { .. } => 1, + Operation::DerefType { base, .. } => 1 + base_size(base), + Operation::PlusConstant(value) => uleb128_size(value), + Operation::Skip(_) => 2, + Operation::Branch(_) => 2, + Operation::Call(_) => 4, + Operation::CallRef(_) => encoding.format.word_size() as usize, + Operation::Convert(base) => match base { + Some(base) => base_size(base), + None => 1, + }, + Operation::Reinterpret(base) => match base { + Some(base) => base_size(base), + None => 1, + }, + Operation::EntryValue(ref expression) => { + let length = expression.size(encoding, unit_offsets); + uleb128_size(length as u64) + length + } + Operation::Register(register) => { + if register.0 < 32 { + 0 + } else { + uleb128_size(register.0.into()) + } + } + Operation::ImplicitValue(ref data) => uleb128_size(data.len() as u64) + data.len(), + Operation::ImplicitPointer { byte_offset, .. } => { + encoding.format.word_size() as usize + sleb128_size(byte_offset) + } + Operation::Piece { size_in_bytes } => uleb128_size(size_in_bytes), + Operation::BitPiece { + size_in_bits, + bit_offset, + } => uleb128_size(size_in_bits) + uleb128_size(bit_offset), + Operation::ParameterRef(_) => 4, + Operation::WasmLocal(index) + | Operation::WasmGlobal(index) + | Operation::WasmStack(index) => 1 + uleb128_size(index.into()), + } + } + + pub(crate) fn write( + &self, + w: &mut W, + refs: Option<&mut Vec>, + encoding: Encoding, + unit_offsets: Option<&UnitOffsets>, + offsets: &[usize], + ) -> Result<()> { + let entry_offset = |entry| match unit_offsets { + Some(offsets) => { + let offset = offsets.unit_offset(entry); + if offset == 0 { + Err(Error::UnsupportedExpressionForwardReference) + } else { + Ok(offset) + } + } + None => Err(Error::UnsupportedCfiExpressionReference), + }; + match *self { + Operation::Raw(ref bytecode) => w.write(bytecode)?, + Operation::Simple(opcode) => w.write_u8(opcode.0)?, + Operation::Address(address) => { + w.write_u8(constants::DW_OP_addr.0)?; + w.write_address(address, encoding.address_size)?; + } + Operation::UnsignedConstant(value) => { + if value < 32 { + w.write_u8(constants::DW_OP_lit0.0 + value as u8)?; + } else { + w.write_u8(constants::DW_OP_constu.0)?; + w.write_uleb128(value)?; + } + } + Operation::SignedConstant(value) => { + w.write_u8(constants::DW_OP_consts.0)?; + w.write_sleb128(value)?; + } + Operation::ConstantType(base, ref value) => { + if encoding.version >= 5 { + w.write_u8(constants::DW_OP_const_type.0)?; + } else { + w.write_u8(constants::DW_OP_GNU_const_type.0)?; + } + w.write_uleb128(entry_offset(base)?)?; + w.write_udata(value.len() as u64, 1)?; + w.write(&value)?; + } + Operation::FrameOffset(offset) => { + w.write_u8(constants::DW_OP_fbreg.0)?; + w.write_sleb128(offset)?; + } + Operation::RegisterOffset(register, offset) => { + if register.0 < 32 { + w.write_u8(constants::DW_OP_breg0.0 + register.0 as u8)?; + } else { + w.write_u8(constants::DW_OP_bregx.0)?; + w.write_uleb128(register.0.into())?; + } + w.write_sleb128(offset)?; + } + Operation::RegisterType(register, base) => { + if encoding.version >= 5 { + w.write_u8(constants::DW_OP_regval_type.0)?; + } else { + w.write_u8(constants::DW_OP_GNU_regval_type.0)?; + } + w.write_uleb128(register.0.into())?; + w.write_uleb128(entry_offset(base)?)?; + } + Operation::Pick(index) => match index { + 0 => w.write_u8(constants::DW_OP_dup.0)?, + 1 => w.write_u8(constants::DW_OP_over.0)?, + _ => { + w.write_u8(constants::DW_OP_pick.0)?; + w.write_u8(index)?; + } + }, + Operation::Deref { space } => { + if space { + w.write_u8(constants::DW_OP_xderef.0)?; + } else { + w.write_u8(constants::DW_OP_deref.0)?; + } + } + Operation::DerefSize { space, size } => { + if space { + w.write_u8(constants::DW_OP_xderef_size.0)?; + } else { + w.write_u8(constants::DW_OP_deref_size.0)?; + } + w.write_u8(size)?; + } + Operation::DerefType { space, size, base } => { + if space { + w.write_u8(constants::DW_OP_xderef_type.0)?; + } else { + if encoding.version >= 5 { + w.write_u8(constants::DW_OP_deref_type.0)?; + } else { + w.write_u8(constants::DW_OP_GNU_deref_type.0)?; + } + } + w.write_u8(size)?; + w.write_uleb128(entry_offset(base)?)?; + } + Operation::PlusConstant(value) => { + w.write_u8(constants::DW_OP_plus_uconst.0)?; + w.write_uleb128(value)?; + } + Operation::Skip(target) => { + w.write_u8(constants::DW_OP_skip.0)?; + let offset = offsets[target] as i64 - (w.len() as i64 + 2); + w.write_sdata(offset, 2)?; + } + Operation::Branch(target) => { + w.write_u8(constants::DW_OP_bra.0)?; + let offset = offsets[target] as i64 - (w.len() as i64 + 2); + w.write_sdata(offset, 2)?; + } + Operation::Call(entry) => { + w.write_u8(constants::DW_OP_call4.0)?; + // TODO: this probably won't work in practice, because we may + // only know the offsets of base type DIEs at this point. + w.write_udata(entry_offset(entry)?, 4)?; + } + Operation::CallRef(entry) => { + w.write_u8(constants::DW_OP_call_ref.0)?; + let size = encoding.format.word_size(); + match entry { + Reference::Symbol(symbol) => w.write_reference(symbol, size)?, + Reference::Entry(unit, entry) => { + let refs = refs.ok_or(Error::InvalidReference)?; + refs.push(DebugInfoReference { + offset: w.len(), + unit, + entry, + size, + }); + w.write_udata(0, size)?; + } + } + } + Operation::Convert(base) => { + if encoding.version >= 5 { + w.write_u8(constants::DW_OP_convert.0)?; + } else { + w.write_u8(constants::DW_OP_GNU_convert.0)?; + } + match base { + Some(base) => w.write_uleb128(entry_offset(base)?)?, + None => w.write_u8(0)?, + } + } + Operation::Reinterpret(base) => { + if encoding.version >= 5 { + w.write_u8(constants::DW_OP_reinterpret.0)?; + } else { + w.write_u8(constants::DW_OP_GNU_reinterpret.0)?; + } + match base { + Some(base) => w.write_uleb128(entry_offset(base)?)?, + None => w.write_u8(0)?, + } + } + Operation::EntryValue(ref expression) => { + if encoding.version >= 5 { + w.write_u8(constants::DW_OP_entry_value.0)?; + } else { + w.write_u8(constants::DW_OP_GNU_entry_value.0)?; + } + let length = expression.size(encoding, unit_offsets); + w.write_uleb128(length as u64)?; + expression.write(w, refs, encoding, unit_offsets)?; + } + Operation::Register(register) => { + if register.0 < 32 { + w.write_u8(constants::DW_OP_reg0.0 + register.0 as u8)?; + } else { + w.write_u8(constants::DW_OP_regx.0)?; + w.write_uleb128(register.0.into())?; + } + } + Operation::ImplicitValue(ref data) => { + w.write_u8(constants::DW_OP_implicit_value.0)?; + w.write_uleb128(data.len() as u64)?; + w.write(&data)?; + } + Operation::ImplicitPointer { entry, byte_offset } => { + if encoding.version >= 5 { + w.write_u8(constants::DW_OP_implicit_pointer.0)?; + } else { + w.write_u8(constants::DW_OP_GNU_implicit_pointer.0)?; + } + let size = if encoding.version == 2 { + encoding.address_size + } else { + encoding.format.word_size() + }; + match entry { + Reference::Symbol(symbol) => { + w.write_reference(symbol, size)?; + } + Reference::Entry(unit, entry) => { + let refs = refs.ok_or(Error::InvalidReference)?; + refs.push(DebugInfoReference { + offset: w.len(), + unit, + entry, + size, + }); + w.write_udata(0, size)?; + } + } + w.write_sleb128(byte_offset)?; + } + Operation::Piece { size_in_bytes } => { + w.write_u8(constants::DW_OP_piece.0)?; + w.write_uleb128(size_in_bytes)?; + } + Operation::BitPiece { + size_in_bits, + bit_offset, + } => { + w.write_u8(constants::DW_OP_bit_piece.0)?; + w.write_uleb128(size_in_bits)?; + w.write_uleb128(bit_offset)?; + } + Operation::ParameterRef(entry) => { + w.write_u8(constants::DW_OP_GNU_parameter_ref.0)?; + w.write_udata(entry_offset(entry)?, 4)?; + } + Operation::WasmLocal(index) => { + w.write(&[constants::DW_OP_WASM_location.0, 0])?; + w.write_uleb128(index.into())?; + } + Operation::WasmGlobal(index) => { + w.write(&[constants::DW_OP_WASM_location.0, 1])?; + w.write_uleb128(index.into())?; + } + Operation::WasmStack(index) => { + w.write(&[constants::DW_OP_WASM_location.0, 2])?; + w.write_uleb128(index.into())?; + } + } + Ok(()) + } +} + +#[cfg(feature = "read")] +pub(crate) mod convert { + use super::*; + use crate::common::UnitSectionOffset; + use crate::read::{self, Reader}; + use crate::write::{ConvertError, ConvertResult, UnitEntryId, UnitId}; + use std::collections::HashMap; + + impl Expression { + /// Create an expression from the input expression. + pub fn from>( + from_expression: read::Expression, + encoding: Encoding, + dwarf: Option<&read::Dwarf>, + unit: Option<&read::Unit>, + entry_ids: Option<&HashMap>, + convert_address: &dyn Fn(u64) -> Option
, + ) -> ConvertResult { + let convert_unit_offset = |offset: read::UnitOffset| -> ConvertResult<_> { + let entry_ids = entry_ids.ok_or(ConvertError::UnsupportedOperation)?; + let unit = unit.ok_or(ConvertError::UnsupportedOperation)?; + let id = entry_ids + .get(&offset.to_unit_section_offset(unit)) + .ok_or(ConvertError::InvalidUnitRef)?; + Ok(id.1) + }; + let convert_debug_info_offset = |offset| -> ConvertResult<_> { + // TODO: support relocations + let entry_ids = entry_ids.ok_or(ConvertError::UnsupportedOperation)?; + let id = entry_ids + .get(&UnitSectionOffset::DebugInfoOffset(offset)) + .ok_or(ConvertError::InvalidDebugInfoRef)?; + Ok(Reference::Entry(id.0, id.1)) + }; + + // Calculate offsets for use in branch/skip operations. + let mut offsets = Vec::new(); + let mut offset = 0; + let mut from_operations = from_expression.clone().operations(encoding); + while let Some(_) = from_operations.next()? { + offsets.push(offset); + offset = from_operations.offset_from(&from_expression); + } + offsets.push(from_expression.0.len()); + + let mut from_operations = from_expression.clone().operations(encoding); + let mut operations = Vec::new(); + while let Some(from_operation) = from_operations.next()? { + let operation = match from_operation { + read::Operation::Deref { + base_type, + size, + space, + } => { + if base_type.0 != 0 { + let base = convert_unit_offset(base_type)?; + Operation::DerefType { space, size, base } + } else if size != encoding.address_size { + Operation::DerefSize { space, size } + } else { + Operation::Deref { space } + } + } + read::Operation::Drop => Operation::Simple(constants::DW_OP_drop), + read::Operation::Pick { index } => Operation::Pick(index), + read::Operation::Swap => Operation::Simple(constants::DW_OP_swap), + read::Operation::Rot => Operation::Simple(constants::DW_OP_rot), + read::Operation::Abs => Operation::Simple(constants::DW_OP_abs), + read::Operation::And => Operation::Simple(constants::DW_OP_and), + read::Operation::Div => Operation::Simple(constants::DW_OP_div), + read::Operation::Minus => Operation::Simple(constants::DW_OP_minus), + read::Operation::Mod => Operation::Simple(constants::DW_OP_mod), + read::Operation::Mul => Operation::Simple(constants::DW_OP_mul), + read::Operation::Neg => Operation::Simple(constants::DW_OP_neg), + read::Operation::Not => Operation::Simple(constants::DW_OP_not), + read::Operation::Or => Operation::Simple(constants::DW_OP_or), + read::Operation::Plus => Operation::Simple(constants::DW_OP_plus), + read::Operation::PlusConstant { value } => Operation::PlusConstant(value), + read::Operation::Shl => Operation::Simple(constants::DW_OP_shl), + read::Operation::Shr => Operation::Simple(constants::DW_OP_shr), + read::Operation::Shra => Operation::Simple(constants::DW_OP_shra), + read::Operation::Xor => Operation::Simple(constants::DW_OP_xor), + read::Operation::Eq => Operation::Simple(constants::DW_OP_eq), + read::Operation::Ge => Operation::Simple(constants::DW_OP_ge), + read::Operation::Gt => Operation::Simple(constants::DW_OP_gt), + read::Operation::Le => Operation::Simple(constants::DW_OP_le), + read::Operation::Lt => Operation::Simple(constants::DW_OP_lt), + read::Operation::Ne => Operation::Simple(constants::DW_OP_ne), + read::Operation::Bra { target } => { + let offset = from_operations + .offset_from(&from_expression) + .wrapping_add(i64::from(target) as usize); + let index = offsets + .binary_search(&offset) + .map_err(|_| ConvertError::InvalidBranchTarget)?; + Operation::Branch(index) + } + read::Operation::Skip { target } => { + let offset = from_operations + .offset_from(&from_expression) + .wrapping_add(i64::from(target) as usize); + let index = offsets + .binary_search(&offset) + .map_err(|_| ConvertError::InvalidBranchTarget)?; + Operation::Skip(index) + } + read::Operation::UnsignedConstant { value } => { + Operation::UnsignedConstant(value) + } + read::Operation::SignedConstant { value } => Operation::SignedConstant(value), + read::Operation::Register { register } => Operation::Register(register), + read::Operation::RegisterOffset { + register, + offset, + base_type, + } => { + if base_type.0 != 0 { + Operation::RegisterType(register, convert_unit_offset(base_type)?) + } else { + Operation::RegisterOffset(register, offset) + } + } + read::Operation::FrameOffset { offset } => Operation::FrameOffset(offset), + read::Operation::Nop => Operation::Simple(constants::DW_OP_nop), + read::Operation::PushObjectAddress => { + Operation::Simple(constants::DW_OP_push_object_address) + } + read::Operation::Call { offset } => match offset { + read::DieReference::UnitRef(offset) => { + Operation::Call(convert_unit_offset(offset)?) + } + read::DieReference::DebugInfoRef(offset) => { + Operation::CallRef(convert_debug_info_offset(offset)?) + } + }, + read::Operation::TLS => Operation::Simple(constants::DW_OP_form_tls_address), + read::Operation::CallFrameCFA => { + Operation::Simple(constants::DW_OP_call_frame_cfa) + } + read::Operation::Piece { + size_in_bits, + bit_offset: None, + } => Operation::Piece { + size_in_bytes: size_in_bits / 8, + }, + read::Operation::Piece { + size_in_bits, + bit_offset: Some(bit_offset), + } => Operation::BitPiece { + size_in_bits, + bit_offset, + }, + read::Operation::ImplicitValue { data } => { + Operation::ImplicitValue(data.to_slice()?.into_owned().into()) + } + read::Operation::StackValue => Operation::Simple(constants::DW_OP_stack_value), + read::Operation::ImplicitPointer { value, byte_offset } => { + let entry = convert_debug_info_offset(value)?; + Operation::ImplicitPointer { entry, byte_offset } + } + read::Operation::EntryValue { expression } => { + let expression = Expression::from( + read::Expression(expression), + encoding, + dwarf, + unit, + entry_ids, + convert_address, + )?; + Operation::EntryValue(expression) + } + read::Operation::ParameterRef { offset } => { + let entry = convert_unit_offset(offset)?; + Operation::ParameterRef(entry) + } + read::Operation::Address { address } => { + let address = + convert_address(address).ok_or(ConvertError::InvalidAddress)?; + Operation::Address(address) + } + read::Operation::AddressIndex { index } => { + let dwarf = dwarf.ok_or(ConvertError::UnsupportedOperation)?; + let unit = unit.ok_or(ConvertError::UnsupportedOperation)?; + let val = dwarf.address(unit, index)?; + let address = convert_address(val).ok_or(ConvertError::InvalidAddress)?; + Operation::Address(address) + } + read::Operation::ConstantIndex { index } => { + let dwarf = dwarf.ok_or(ConvertError::UnsupportedOperation)?; + let unit = unit.ok_or(ConvertError::UnsupportedOperation)?; + let val = dwarf.address(unit, index)?; + Operation::UnsignedConstant(val) + } + read::Operation::TypedLiteral { base_type, value } => { + let entry = convert_unit_offset(base_type)?; + Operation::ConstantType(entry, value.to_slice()?.into_owned().into()) + } + read::Operation::Convert { base_type } => { + if base_type.0 == 0 { + Operation::Convert(None) + } else { + let entry = convert_unit_offset(base_type)?; + Operation::Convert(Some(entry)) + } + } + read::Operation::Reinterpret { base_type } => { + if base_type.0 == 0 { + Operation::Reinterpret(None) + } else { + let entry = convert_unit_offset(base_type)?; + Operation::Reinterpret(Some(entry)) + } + } + read::Operation::WasmLocal { index } => Operation::WasmLocal(index), + read::Operation::WasmGlobal { index } => Operation::WasmGlobal(index), + read::Operation::WasmStack { index } => Operation::WasmStack(index), + }; + operations.push(operation); + } + Ok(Expression { operations }) + } + } +} + +#[cfg(test)] +#[cfg(feature = "read")] +mod tests { + use super::*; + use crate::common::{ + DebugAbbrevOffset, DebugAddrBase, DebugInfoOffset, DebugLocListsBase, DebugRngListsBase, + DebugStrOffsetsBase, Format, SectionId, + }; + use crate::read; + use crate::write::{ + DebugLineStrOffsets, DebugStrOffsets, EndianVec, LineProgram, Sections, Unit, UnitTable, + }; + use crate::LittleEndian; + use std::collections::HashMap; + + #[test] + fn test_operation() { + for &version in &[3, 4, 5] { + for &address_size in &[4, 8] { + for &format in &[Format::Dwarf32, Format::Dwarf64] { + let encoding = Encoding { + format, + version, + address_size, + }; + + let mut units = UnitTable::default(); + let unit_id = units.add(Unit::new(encoding, LineProgram::none())); + let unit = units.get_mut(unit_id); + let entry_id = unit.add(unit.root(), constants::DW_TAG_base_type); + let reference = Reference::Entry(unit_id, entry_id); + + let mut sections = Sections::new(EndianVec::new(LittleEndian)); + let debug_line_str_offsets = DebugLineStrOffsets::none(); + let debug_str_offsets = DebugStrOffsets::none(); + let debug_info_offsets = units + .write(&mut sections, &debug_line_str_offsets, &debug_str_offsets) + .unwrap(); + let unit_offsets = debug_info_offsets.unit_offsets(unit_id); + let debug_info_offset = unit_offsets.debug_info_offset(entry_id); + let entry_offset = + read::UnitOffset(unit_offsets.unit_offset(entry_id) as usize); + + let mut reg_expression = Expression::new(); + reg_expression.op_reg(Register(23)); + + let operations: &[(&dyn Fn(&mut Expression), Operation, read::Operation<_>)] = + &[ + ( + &|x| x.op_deref(), + Operation::Deref { space: false }, + read::Operation::Deref { + base_type: read::UnitOffset(0), + size: address_size, + space: false, + }, + ), + ( + &|x| x.op_xderef(), + Operation::Deref { space: true }, + read::Operation::Deref { + base_type: read::UnitOffset(0), + size: address_size, + space: true, + }, + ), + ( + &|x| x.op_deref_size(2), + Operation::DerefSize { + space: false, + size: 2, + }, + read::Operation::Deref { + base_type: read::UnitOffset(0), + size: 2, + space: false, + }, + ), + ( + &|x| x.op_xderef_size(2), + Operation::DerefSize { + space: true, + size: 2, + }, + read::Operation::Deref { + base_type: read::UnitOffset(0), + size: 2, + space: true, + }, + ), + ( + &|x| x.op_deref_type(2, entry_id), + Operation::DerefType { + space: false, + size: 2, + base: entry_id, + }, + read::Operation::Deref { + base_type: entry_offset, + size: 2, + space: false, + }, + ), + ( + &|x| x.op_xderef_type(2, entry_id), + Operation::DerefType { + space: true, + size: 2, + base: entry_id, + }, + read::Operation::Deref { + base_type: entry_offset, + size: 2, + space: true, + }, + ), + ( + &|x| x.op(constants::DW_OP_drop), + Operation::Simple(constants::DW_OP_drop), + read::Operation::Drop, + ), + ( + &|x| x.op_pick(0), + Operation::Pick(0), + read::Operation::Pick { index: 0 }, + ), + ( + &|x| x.op_pick(1), + Operation::Pick(1), + read::Operation::Pick { index: 1 }, + ), + ( + &|x| x.op_pick(2), + Operation::Pick(2), + read::Operation::Pick { index: 2 }, + ), + ( + &|x| x.op(constants::DW_OP_swap), + Operation::Simple(constants::DW_OP_swap), + read::Operation::Swap, + ), + ( + &|x| x.op(constants::DW_OP_rot), + Operation::Simple(constants::DW_OP_rot), + read::Operation::Rot, + ), + ( + &|x| x.op(constants::DW_OP_abs), + Operation::Simple(constants::DW_OP_abs), + read::Operation::Abs, + ), + ( + &|x| x.op(constants::DW_OP_and), + Operation::Simple(constants::DW_OP_and), + read::Operation::And, + ), + ( + &|x| x.op(constants::DW_OP_div), + Operation::Simple(constants::DW_OP_div), + read::Operation::Div, + ), + ( + &|x| x.op(constants::DW_OP_minus), + Operation::Simple(constants::DW_OP_minus), + read::Operation::Minus, + ), + ( + &|x| x.op(constants::DW_OP_mod), + Operation::Simple(constants::DW_OP_mod), + read::Operation::Mod, + ), + ( + &|x| x.op(constants::DW_OP_mul), + Operation::Simple(constants::DW_OP_mul), + read::Operation::Mul, + ), + ( + &|x| x.op(constants::DW_OP_neg), + Operation::Simple(constants::DW_OP_neg), + read::Operation::Neg, + ), + ( + &|x| x.op(constants::DW_OP_not), + Operation::Simple(constants::DW_OP_not), + read::Operation::Not, + ), + ( + &|x| x.op(constants::DW_OP_or), + Operation::Simple(constants::DW_OP_or), + read::Operation::Or, + ), + ( + &|x| x.op(constants::DW_OP_plus), + Operation::Simple(constants::DW_OP_plus), + read::Operation::Plus, + ), + ( + &|x| x.op_plus_uconst(23), + Operation::PlusConstant(23), + read::Operation::PlusConstant { value: 23 }, + ), + ( + &|x| x.op(constants::DW_OP_shl), + Operation::Simple(constants::DW_OP_shl), + read::Operation::Shl, + ), + ( + &|x| x.op(constants::DW_OP_shr), + Operation::Simple(constants::DW_OP_shr), + read::Operation::Shr, + ), + ( + &|x| x.op(constants::DW_OP_shra), + Operation::Simple(constants::DW_OP_shra), + read::Operation::Shra, + ), + ( + &|x| x.op(constants::DW_OP_xor), + Operation::Simple(constants::DW_OP_xor), + read::Operation::Xor, + ), + ( + &|x| x.op(constants::DW_OP_eq), + Operation::Simple(constants::DW_OP_eq), + read::Operation::Eq, + ), + ( + &|x| x.op(constants::DW_OP_ge), + Operation::Simple(constants::DW_OP_ge), + read::Operation::Ge, + ), + ( + &|x| x.op(constants::DW_OP_gt), + Operation::Simple(constants::DW_OP_gt), + read::Operation::Gt, + ), + ( + &|x| x.op(constants::DW_OP_le), + Operation::Simple(constants::DW_OP_le), + read::Operation::Le, + ), + ( + &|x| x.op(constants::DW_OP_lt), + Operation::Simple(constants::DW_OP_lt), + read::Operation::Lt, + ), + ( + &|x| x.op(constants::DW_OP_ne), + Operation::Simple(constants::DW_OP_ne), + read::Operation::Ne, + ), + ( + &|x| x.op_constu(23), + Operation::UnsignedConstant(23), + read::Operation::UnsignedConstant { value: 23 }, + ), + ( + &|x| x.op_consts(-23), + Operation::SignedConstant(-23), + read::Operation::SignedConstant { value: -23 }, + ), + ( + &|x| x.op_reg(Register(23)), + Operation::Register(Register(23)), + read::Operation::Register { + register: Register(23), + }, + ), + ( + &|x| x.op_reg(Register(123)), + Operation::Register(Register(123)), + read::Operation::Register { + register: Register(123), + }, + ), + ( + &|x| x.op_breg(Register(23), 34), + Operation::RegisterOffset(Register(23), 34), + read::Operation::RegisterOffset { + register: Register(23), + offset: 34, + base_type: read::UnitOffset(0), + }, + ), + ( + &|x| x.op_breg(Register(123), 34), + Operation::RegisterOffset(Register(123), 34), + read::Operation::RegisterOffset { + register: Register(123), + offset: 34, + base_type: read::UnitOffset(0), + }, + ), + ( + &|x| x.op_regval_type(Register(23), entry_id), + Operation::RegisterType(Register(23), entry_id), + read::Operation::RegisterOffset { + register: Register(23), + offset: 0, + base_type: entry_offset, + }, + ), + ( + &|x| x.op_fbreg(34), + Operation::FrameOffset(34), + read::Operation::FrameOffset { offset: 34 }, + ), + ( + &|x| x.op(constants::DW_OP_nop), + Operation::Simple(constants::DW_OP_nop), + read::Operation::Nop, + ), + ( + &|x| x.op(constants::DW_OP_push_object_address), + Operation::Simple(constants::DW_OP_push_object_address), + read::Operation::PushObjectAddress, + ), + ( + &|x| x.op_call(entry_id), + Operation::Call(entry_id), + read::Operation::Call { + offset: read::DieReference::UnitRef(entry_offset), + }, + ), + ( + &|x| x.op_call_ref(reference), + Operation::CallRef(reference), + read::Operation::Call { + offset: read::DieReference::DebugInfoRef(debug_info_offset), + }, + ), + ( + &|x| x.op(constants::DW_OP_form_tls_address), + Operation::Simple(constants::DW_OP_form_tls_address), + read::Operation::TLS, + ), + ( + &|x| x.op(constants::DW_OP_call_frame_cfa), + Operation::Simple(constants::DW_OP_call_frame_cfa), + read::Operation::CallFrameCFA, + ), + ( + &|x| x.op_piece(23), + Operation::Piece { size_in_bytes: 23 }, + read::Operation::Piece { + size_in_bits: 23 * 8, + bit_offset: None, + }, + ), + ( + &|x| x.op_bit_piece(23, 34), + Operation::BitPiece { + size_in_bits: 23, + bit_offset: 34, + }, + read::Operation::Piece { + size_in_bits: 23, + bit_offset: Some(34), + }, + ), + ( + &|x| x.op_implicit_value(vec![23].into()), + Operation::ImplicitValue(vec![23].into()), + read::Operation::ImplicitValue { + data: read::EndianSlice::new(&[23], LittleEndian), + }, + ), + ( + &|x| x.op(constants::DW_OP_stack_value), + Operation::Simple(constants::DW_OP_stack_value), + read::Operation::StackValue, + ), + ( + &|x| x.op_implicit_pointer(reference, 23), + Operation::ImplicitPointer { + entry: reference, + byte_offset: 23, + }, + read::Operation::ImplicitPointer { + value: debug_info_offset, + byte_offset: 23, + }, + ), + ( + &|x| x.op_entry_value(reg_expression.clone()), + Operation::EntryValue(reg_expression.clone()), + read::Operation::EntryValue { + expression: read::EndianSlice::new( + &[constants::DW_OP_reg23.0], + LittleEndian, + ), + }, + ), + ( + &|x| x.op_gnu_parameter_ref(entry_id), + Operation::ParameterRef(entry_id), + read::Operation::ParameterRef { + offset: entry_offset, + }, + ), + ( + &|x| x.op_addr(Address::Constant(23)), + Operation::Address(Address::Constant(23)), + read::Operation::Address { address: 23 }, + ), + ( + &|x| x.op_const_type(entry_id, vec![23].into()), + Operation::ConstantType(entry_id, vec![23].into()), + read::Operation::TypedLiteral { + base_type: entry_offset, + value: read::EndianSlice::new(&[23], LittleEndian), + }, + ), + ( + &|x| x.op_convert(None), + Operation::Convert(None), + read::Operation::Convert { + base_type: read::UnitOffset(0), + }, + ), + ( + &|x| x.op_convert(Some(entry_id)), + Operation::Convert(Some(entry_id)), + read::Operation::Convert { + base_type: entry_offset, + }, + ), + ( + &|x| x.op_reinterpret(None), + Operation::Reinterpret(None), + read::Operation::Reinterpret { + base_type: read::UnitOffset(0), + }, + ), + ( + &|x| x.op_reinterpret(Some(entry_id)), + Operation::Reinterpret(Some(entry_id)), + read::Operation::Reinterpret { + base_type: entry_offset, + }, + ), + ( + &|x| x.op_wasm_local(1000), + Operation::WasmLocal(1000), + read::Operation::WasmLocal { index: 1000 }, + ), + ( + &|x| x.op_wasm_global(1000), + Operation::WasmGlobal(1000), + read::Operation::WasmGlobal { index: 1000 }, + ), + ( + &|x| x.op_wasm_stack(1000), + Operation::WasmStack(1000), + read::Operation::WasmStack { index: 1000 }, + ), + ]; + + let mut expression = Expression::new(); + let start_index = expression.next_index(); + for (f, o, _) in operations { + f(&mut expression); + assert_eq!(expression.operations.last(), Some(o)); + } + + let bra_index = expression.op_bra(); + let skip_index = expression.op_skip(); + expression.op(constants::DW_OP_nop); + let end_index = expression.next_index(); + expression.set_target(bra_index, start_index); + expression.set_target(skip_index, end_index); + + let mut w = EndianVec::new(LittleEndian); + let mut refs = Vec::new(); + expression + .write(&mut w, Some(&mut refs), encoding, Some(&unit_offsets)) + .unwrap(); + for r in &refs { + assert_eq!(r.unit, unit_id); + assert_eq!(r.entry, entry_id); + w.write_offset_at( + r.offset, + debug_info_offset.0, + SectionId::DebugInfo, + r.size, + ) + .unwrap(); + } + + let read_expression = + read::Expression(read::EndianSlice::new(w.slice(), LittleEndian)); + let mut read_operations = read_expression.operations(encoding); + for (_, _, operation) in operations { + assert_eq!(read_operations.next(), Ok(Some(*operation))); + } + + // 4 = DW_OP_skip + i16 + DW_OP_nop + assert_eq!( + read_operations.next(), + Ok(Some(read::Operation::Bra { + target: -(w.len() as i16) + 4 + })) + ); + // 1 = DW_OP_nop + assert_eq!( + read_operations.next(), + Ok(Some(read::Operation::Skip { target: 1 })) + ); + assert_eq!(read_operations.next(), Ok(Some(read::Operation::Nop))); + assert_eq!(read_operations.next(), Ok(None)); + + // Fake the unit. + let unit = read::Unit { + header: read::UnitHeader::new( + encoding, + 0, + read::UnitType::Compilation, + DebugAbbrevOffset(0), + DebugInfoOffset(0).into(), + read::EndianSlice::new(&[], LittleEndian), + ), + abbreviations: read::Abbreviations::default(), + name: None, + comp_dir: None, + low_pc: 0, + str_offsets_base: DebugStrOffsetsBase(0), + addr_base: DebugAddrBase(0), + loclists_base: DebugLocListsBase(0), + rnglists_base: DebugRngListsBase(0), + line_program: None, + dwo_id: None, + }; + + let mut entry_ids = HashMap::new(); + entry_ids.insert(debug_info_offset.into(), (unit_id, entry_id)); + let convert_expression = Expression::from( + read_expression, + encoding, + None, /* dwarf */ + Some(&unit), + Some(&entry_ids), + &|address| Some(Address::Constant(address)), + ) + .unwrap(); + let mut convert_operations = convert_expression.operations.iter(); + for (_, operation, _) in operations { + assert_eq!(convert_operations.next(), Some(operation)); + } + assert_eq!( + convert_operations.next(), + Some(&Operation::Branch(start_index)) + ); + assert_eq!(convert_operations.next(), Some(&Operation::Skip(end_index))); + assert_eq!( + convert_operations.next(), + Some(&Operation::Simple(constants::DW_OP_nop)) + ); + } + } + } + } +} diff --git a/crux-mir/lib/gimli/src/write/range.rs b/crux-mir/lib/gimli/src/write/range.rs new file mode 100644 index 000000000..b44ce1b7b --- /dev/null +++ b/crux-mir/lib/gimli/src/write/range.rs @@ -0,0 +1,415 @@ +use alloc::vec::Vec; +use indexmap::IndexSet; +use std::ops::{Deref, DerefMut}; + +use crate::common::{Encoding, RangeListsOffset, SectionId}; +use crate::write::{Address, BaseId, Error, Result, Section, Sections, Writer}; + +define_section!( + DebugRanges, + RangeListsOffset, + "A writable `.debug_ranges` section." +); +define_section!( + DebugRngLists, + RangeListsOffset, + "A writable `.debug_rnglists` section." +); + +define_offsets!( + RangeListOffsets: RangeListId => RangeListsOffset, + "The section offsets of a series of range lists within the `.debug_ranges` or `.debug_rnglists` sections." +); + +define_id!( + RangeListId, + "An identifier for a range list in a `RangeListTable`." +); + +/// A table of range lists that will be stored in a `.debug_ranges` or `.debug_rnglists` section. +#[derive(Debug, Default)] +pub struct RangeListTable { + base_id: BaseId, + ranges: IndexSet, +} + +impl RangeListTable { + /// Add a range list to the table. + pub fn add(&mut self, range_list: RangeList) -> RangeListId { + let (index, _) = self.ranges.insert_full(range_list); + RangeListId::new(self.base_id, index) + } + + /// Write the range list table to the appropriate section for the given DWARF version. + pub(crate) fn write( + &self, + sections: &mut Sections, + encoding: Encoding, + ) -> Result { + if self.ranges.is_empty() { + return Ok(RangeListOffsets::none()); + } + + match encoding.version { + 2..=4 => self.write_ranges(&mut sections.debug_ranges, encoding.address_size), + 5 => self.write_rnglists(&mut sections.debug_rnglists, encoding), + _ => Err(Error::UnsupportedVersion(encoding.version)), + } + } + + /// Write the range list table to the `.debug_ranges` section. + fn write_ranges( + &self, + w: &mut DebugRanges, + address_size: u8, + ) -> Result { + let mut offsets = Vec::new(); + for range_list in self.ranges.iter() { + offsets.push(w.offset()); + for range in &range_list.0 { + // Note that we must ensure none of the ranges have both begin == 0 and end == 0. + // We do this by ensuring that begin != end, which is a bit more restrictive + // than required, but still seems reasonable. + match *range { + Range::BaseAddress { address } => { + let marker = !0 >> (64 - address_size * 8); + w.write_udata(marker, address_size)?; + w.write_address(address, address_size)?; + } + Range::OffsetPair { begin, end } => { + if begin == end { + return Err(Error::InvalidRange); + } + w.write_udata(begin, address_size)?; + w.write_udata(end, address_size)?; + } + Range::StartEnd { begin, end } => { + if begin == end { + return Err(Error::InvalidRange); + } + w.write_address(begin, address_size)?; + w.write_address(end, address_size)?; + } + Range::StartLength { begin, length } => { + let end = match begin { + Address::Constant(begin) => Address::Constant(begin + length), + Address::Symbol { symbol, addend } => Address::Symbol { + symbol, + addend: addend + length as i64, + }, + }; + if begin == end { + return Err(Error::InvalidRange); + } + w.write_address(begin, address_size)?; + w.write_address(end, address_size)?; + } + } + } + w.write_udata(0, address_size)?; + w.write_udata(0, address_size)?; + } + Ok(RangeListOffsets { + base_id: self.base_id, + offsets, + }) + } + + /// Write the range list table to the `.debug_rnglists` section. + fn write_rnglists( + &self, + w: &mut DebugRngLists, + encoding: Encoding, + ) -> Result { + let mut offsets = Vec::new(); + + if encoding.version != 5 { + return Err(Error::NeedVersion(5)); + } + + let length_offset = w.write_initial_length(encoding.format)?; + let length_base = w.len(); + + w.write_u16(encoding.version)?; + w.write_u8(encoding.address_size)?; + w.write_u8(0)?; // segment_selector_size + w.write_u32(0)?; // offset_entry_count (when set to zero DW_FORM_rnglistx can't be used, see section 7.28) + // FIXME implement DW_FORM_rnglistx writing and implement the offset entry list + + for range_list in self.ranges.iter() { + offsets.push(w.offset()); + for range in &range_list.0 { + match *range { + Range::BaseAddress { address } => { + w.write_u8(crate::constants::DW_RLE_base_address.0)?; + w.write_address(address, encoding.address_size)?; + } + Range::OffsetPair { begin, end } => { + w.write_u8(crate::constants::DW_RLE_offset_pair.0)?; + w.write_uleb128(begin)?; + w.write_uleb128(end)?; + } + Range::StartEnd { begin, end } => { + w.write_u8(crate::constants::DW_RLE_start_end.0)?; + w.write_address(begin, encoding.address_size)?; + w.write_address(end, encoding.address_size)?; + } + Range::StartLength { begin, length } => { + w.write_u8(crate::constants::DW_RLE_start_length.0)?; + w.write_address(begin, encoding.address_size)?; + w.write_uleb128(length)?; + } + } + } + + w.write_u8(crate::constants::DW_RLE_end_of_list.0)?; + } + + let length = (w.len() - length_base) as u64; + w.write_initial_length_at(length_offset, length, encoding.format)?; + + Ok(RangeListOffsets { + base_id: self.base_id, + offsets, + }) + } +} + +/// A range list that will be stored in a `.debug_ranges` or `.debug_rnglists` section. +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +pub struct RangeList(pub Vec); + +/// A single range. +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +pub enum Range { + /// DW_RLE_base_address + BaseAddress { + /// Base address. + address: Address, + }, + /// DW_RLE_offset_pair + OffsetPair { + /// Start of range relative to base address. + begin: u64, + /// End of range relative to base address. + end: u64, + }, + /// DW_RLE_start_end + StartEnd { + /// Start of range. + begin: Address, + /// End of range. + end: Address, + }, + /// DW_RLE_start_length + StartLength { + /// Start of range. + begin: Address, + /// Length of range. + length: u64, + }, +} + +#[cfg(feature = "read")] +mod convert { + use super::*; + + use crate::read::{self, Reader}; + use crate::write::{ConvertError, ConvertResult, ConvertUnitContext}; + + impl RangeList { + /// Create a range list by reading the data from the give range list iter. + pub(crate) fn from>( + mut from: read::RawRngListIter, + context: &ConvertUnitContext, + ) -> ConvertResult { + let mut have_base_address = context.base_address != Address::Constant(0); + let convert_address = + |x| (context.convert_address)(x).ok_or(ConvertError::InvalidAddress); + let mut ranges = Vec::new(); + while let Some(from_range) = from.next()? { + let range = match from_range { + read::RawRngListEntry::AddressOrOffsetPair { begin, end } => { + // These were parsed as addresses, even if they are offsets. + let begin = convert_address(begin)?; + let end = convert_address(end)?; + match (begin, end) { + (Address::Constant(begin_offset), Address::Constant(end_offset)) => { + if have_base_address { + Range::OffsetPair { + begin: begin_offset, + end: end_offset, + } + } else { + Range::StartEnd { begin, end } + } + } + _ => { + if have_base_address { + // At least one of begin/end is an address, but we also have + // a base address. Adding addresses is undefined. + return Err(ConvertError::InvalidRangeRelativeAddress); + } + Range::StartEnd { begin, end } + } + } + } + read::RawRngListEntry::BaseAddress { addr } => { + have_base_address = true; + let address = convert_address(addr)?; + Range::BaseAddress { address } + } + read::RawRngListEntry::BaseAddressx { addr } => { + have_base_address = true; + let address = convert_address(context.dwarf.address(context.unit, addr)?)?; + Range::BaseAddress { address } + } + read::RawRngListEntry::StartxEndx { begin, end } => { + let begin = convert_address(context.dwarf.address(context.unit, begin)?)?; + let end = convert_address(context.dwarf.address(context.unit, end)?)?; + Range::StartEnd { begin, end } + } + read::RawRngListEntry::StartxLength { begin, length } => { + let begin = convert_address(context.dwarf.address(context.unit, begin)?)?; + Range::StartLength { begin, length } + } + read::RawRngListEntry::OffsetPair { begin, end } => { + Range::OffsetPair { begin, end } + } + read::RawRngListEntry::StartEnd { begin, end } => { + let begin = convert_address(begin)?; + let end = convert_address(end)?; + Range::StartEnd { begin, end } + } + read::RawRngListEntry::StartLength { begin, length } => { + let begin = convert_address(begin)?; + Range::StartLength { begin, length } + } + }; + // Filtering empty ranges out. + match range { + Range::StartLength { length, .. } if length == 0 => continue, + Range::StartEnd { begin, end, .. } if begin == end => continue, + Range::OffsetPair { begin, end, .. } if begin == end => continue, + _ => (), + } + ranges.push(range); + } + Ok(RangeList(ranges)) + } + } +} + +#[cfg(test)] +#[cfg(feature = "read")] +mod tests { + use super::*; + use crate::common::{ + DebugAbbrevOffset, DebugAddrBase, DebugInfoOffset, DebugLocListsBase, DebugRngListsBase, + DebugStrOffsetsBase, Format, + }; + use crate::read; + use crate::write::{ + ConvertUnitContext, EndianVec, LineStringTable, LocationListTable, Range, RangeListTable, + StringTable, + }; + use crate::LittleEndian; + use std::collections::HashMap; + + #[test] + fn test_range() { + let mut line_strings = LineStringTable::default(); + let mut strings = StringTable::default(); + + for &version in &[2, 3, 4, 5] { + for &address_size in &[4, 8] { + for &format in &[Format::Dwarf32, Format::Dwarf64] { + let encoding = Encoding { + format, + version, + address_size, + }; + + let mut range_list = RangeList(vec![ + Range::StartLength { + begin: Address::Constant(6666), + length: 7777, + }, + Range::StartEnd { + begin: Address::Constant(4444), + end: Address::Constant(5555), + }, + Range::BaseAddress { + address: Address::Constant(1111), + }, + Range::OffsetPair { + begin: 2222, + end: 3333, + }, + ]); + + let mut ranges = RangeListTable::default(); + let range_list_id = ranges.add(range_list.clone()); + + let mut sections = Sections::new(EndianVec::new(LittleEndian)); + let range_list_offsets = ranges.write(&mut sections, encoding).unwrap(); + + let read_debug_ranges = + read::DebugRanges::new(sections.debug_ranges.slice(), LittleEndian); + let read_debug_rnglists = + read::DebugRngLists::new(sections.debug_rnglists.slice(), LittleEndian); + let read_ranges = read::RangeLists::new(read_debug_ranges, read_debug_rnglists); + let offset = range_list_offsets.get(range_list_id); + let read_range_list = read_ranges.raw_ranges(offset, encoding).unwrap(); + + let dwarf = read::Dwarf { + ranges: read_ranges, + ..Default::default() + }; + let unit = read::Unit { + header: read::UnitHeader::new( + encoding, + 0, + read::UnitType::Compilation, + DebugAbbrevOffset(0), + DebugInfoOffset(0).into(), + read::EndianSlice::default(), + ), + abbreviations: read::Abbreviations::default(), + name: None, + comp_dir: None, + low_pc: 0, + str_offsets_base: DebugStrOffsetsBase(0), + addr_base: DebugAddrBase(0), + loclists_base: DebugLocListsBase(0), + rnglists_base: DebugRngListsBase(0), + line_program: None, + dwo_id: None, + }; + let context = ConvertUnitContext { + dwarf: &dwarf, + unit: &unit, + line_strings: &mut line_strings, + strings: &mut strings, + ranges: &mut ranges, + locations: &mut LocationListTable::default(), + convert_address: &|address| Some(Address::Constant(address)), + base_address: Address::Constant(0), + line_program_offset: None, + line_program_files: Vec::new(), + entry_ids: &HashMap::new(), + }; + let convert_range_list = RangeList::from(read_range_list, &context).unwrap(); + + if version <= 4 { + range_list.0[0] = Range::StartEnd { + begin: Address::Constant(6666), + end: Address::Constant(6666 + 7777), + }; + } + assert_eq!(range_list, convert_range_list); + } + } + } + } +} diff --git a/crux-mir/lib/gimli/src/write/section.rs b/crux-mir/lib/gimli/src/write/section.rs new file mode 100644 index 000000000..e8f3378cd --- /dev/null +++ b/crux-mir/lib/gimli/src/write/section.rs @@ -0,0 +1,172 @@ +use std::ops::DerefMut; +use std::result; +use std::vec::Vec; + +use crate::common::SectionId; +use crate::write::{ + DebugAbbrev, DebugFrame, DebugInfo, DebugInfoReference, DebugLine, DebugLineStr, DebugLoc, + DebugLocLists, DebugRanges, DebugRngLists, DebugStr, EhFrame, Writer, +}; + +macro_rules! define_section { + ($name:ident, $offset:ident, $docs:expr) => { + #[doc=$docs] + #[derive(Debug, Default)] + pub struct $name(pub W); + + impl $name { + /// Return the offset of the next write. + pub fn offset(&self) -> $offset { + $offset(self.len()) + } + } + + impl From for $name { + #[inline] + fn from(w: W) -> Self { + $name(w) + } + } + + impl Deref for $name { + type Target = W; + + #[inline] + fn deref(&self) -> &W { + &self.0 + } + } + + impl DerefMut for $name { + #[inline] + fn deref_mut(&mut self) -> &mut W { + &mut self.0 + } + } + + impl Section for $name { + #[inline] + fn id(&self) -> SectionId { + SectionId::$name + } + } + }; +} + +/// Functionality common to all writable DWARF sections. +pub trait Section: DerefMut { + /// Returns the DWARF section kind for this type. + fn id(&self) -> SectionId; + + /// Returns the ELF section name for this type. + fn name(&self) -> &'static str { + self.id().name() + } +} + +/// All of the writable DWARF sections. +#[derive(Debug, Default)] +pub struct Sections { + /// The `.debug_abbrev` section. + pub debug_abbrev: DebugAbbrev, + /// The `.debug_info` section. + pub debug_info: DebugInfo, + /// The `.debug_line` section. + pub debug_line: DebugLine, + /// The `.debug_line_str` section. + pub debug_line_str: DebugLineStr, + /// The `.debug_ranges` section. + pub debug_ranges: DebugRanges, + /// The `.debug_rnglists` section. + pub debug_rnglists: DebugRngLists, + /// The `.debug_loc` section. + pub debug_loc: DebugLoc, + /// The `.debug_loclists` section. + pub debug_loclists: DebugLocLists, + /// The `.debug_str` section. + pub debug_str: DebugStr, + /// The `.debug_frame` section. + pub debug_frame: DebugFrame, + /// The `.eh_frame` section. + pub eh_frame: EhFrame, + /// Unresolved references in the `.debug_info` section. + pub(crate) debug_info_refs: Vec, + /// Unresolved references in the `.debug_loc` section. + pub(crate) debug_loc_refs: Vec, + /// Unresolved references in the `.debug_loclists` section. + pub(crate) debug_loclists_refs: Vec, +} + +impl Sections { + /// Create a new `Sections` using clones of the given `section`. + pub fn new(section: W) -> Self { + Sections { + debug_abbrev: DebugAbbrev(section.clone()), + debug_info: DebugInfo(section.clone()), + debug_line: DebugLine(section.clone()), + debug_line_str: DebugLineStr(section.clone()), + debug_ranges: DebugRanges(section.clone()), + debug_rnglists: DebugRngLists(section.clone()), + debug_loc: DebugLoc(section.clone()), + debug_loclists: DebugLocLists(section.clone()), + debug_str: DebugStr(section.clone()), + debug_frame: DebugFrame(section.clone()), + eh_frame: EhFrame(section.clone()), + debug_info_refs: Vec::new(), + debug_loc_refs: Vec::new(), + debug_loclists_refs: Vec::new(), + } + } +} + +impl Sections { + /// For each section, call `f` once with a shared reference. + pub fn for_each(&self, mut f: F) -> result::Result<(), E> + where + F: FnMut(SectionId, &W) -> result::Result<(), E>, + { + macro_rules! f { + ($s:expr) => { + f($s.id(), &$s) + }; + } + // Ordered so that earlier sections do not reference later sections. + f!(self.debug_abbrev)?; + f!(self.debug_str)?; + f!(self.debug_line_str)?; + f!(self.debug_line)?; + f!(self.debug_ranges)?; + f!(self.debug_rnglists)?; + f!(self.debug_loc)?; + f!(self.debug_loclists)?; + f!(self.debug_info)?; + f!(self.debug_frame)?; + f!(self.eh_frame)?; + Ok(()) + } + + /// For each section, call `f` once with a mutable reference. + pub fn for_each_mut(&mut self, mut f: F) -> result::Result<(), E> + where + F: FnMut(SectionId, &mut W) -> result::Result<(), E>, + { + macro_rules! f { + ($s:expr) => { + f($s.id(), &mut $s) + }; + } + // Ordered so that earlier sections do not reference later sections. + f!(self.debug_abbrev)?; + f!(self.debug_str)?; + f!(self.debug_line_str)?; + f!(self.debug_line)?; + f!(self.debug_ranges)?; + f!(self.debug_rnglists)?; + f!(self.debug_loc)?; + f!(self.debug_loclists)?; + f!(self.debug_info)?; + f!(self.debug_frame)?; + f!(self.eh_frame)?; + Ok(()) + } +} diff --git a/crux-mir/lib/gimli/src/write/str.rs b/crux-mir/lib/gimli/src/write/str.rs new file mode 100644 index 000000000..83285c035 --- /dev/null +++ b/crux-mir/lib/gimli/src/write/str.rs @@ -0,0 +1,172 @@ +use alloc::vec::Vec; +use indexmap::IndexSet; +use std::ops::{Deref, DerefMut}; + +use crate::common::{DebugLineStrOffset, DebugStrOffset, SectionId}; +use crate::write::{BaseId, Result, Section, Writer}; + +// Requirements: +// - values are `[u8]`, null bytes are not allowed +// - insertion returns a fixed id +// - inserting a duplicate returns the id of the existing value +// - able to convert an id to a section offset +// Optional? +// - able to get an existing value given an id +// +// Limitations of current implementation (using IndexSet): +// - inserting requires either an allocation for duplicates, +// or a double lookup for non-duplicates +// - doesn't preserve offsets when updating an existing `.debug_str` section +// +// Possible changes: +// - calculate offsets as we add values, and use that as the id. +// This would avoid the need for DebugStrOffsets but would make it +// hard to implement `get`. +macro_rules! define_string_table { + ($name:ident, $id:ident, $section:ident, $offsets:ident, $docs:expr) => { + #[doc=$docs] + #[derive(Debug, Default)] + pub struct $name { + base_id: BaseId, + strings: IndexSet>, + } + + impl $name { + /// Add a string to the string table and return its id. + /// + /// If the string already exists, then return the id of the existing string. + /// + /// # Panics + /// + /// Panics if `bytes` contains a null byte. + pub fn add(&mut self, bytes: T) -> $id + where + T: Into>, + { + let bytes = bytes.into(); + assert!(!bytes.contains(&0)); + let (index, _) = self.strings.insert_full(bytes); + $id::new(self.base_id, index) + } + + /// Return the number of strings in the table. + #[inline] + pub fn count(&self) -> usize { + self.strings.len() + } + + /// Get a reference to a string in the table. + /// + /// # Panics + /// + /// Panics if `id` is invalid. + pub fn get(&self, id: $id) -> &[u8] { + debug_assert_eq!(self.base_id, id.base_id); + self.strings.get_index(id.index).map(Vec::as_slice).unwrap() + } + + /// Write the string table to the `.debug_str` section. + /// + /// Returns the offsets at which the strings are written. + pub fn write(&self, w: &mut $section) -> Result<$offsets> { + let mut offsets = Vec::new(); + for bytes in self.strings.iter() { + offsets.push(w.offset()); + w.write(bytes)?; + w.write_u8(0)?; + } + + Ok($offsets { + base_id: self.base_id, + offsets, + }) + } + } + }; +} + +define_id!(StringId, "An identifier for a string in a `StringTable`."); + +define_string_table!( + StringTable, + StringId, + DebugStr, + DebugStrOffsets, + "A table of strings that will be stored in a `.debug_str` section." +); + +define_section!(DebugStr, DebugStrOffset, "A writable `.debug_str` section."); + +define_offsets!( + DebugStrOffsets: StringId => DebugStrOffset, + "The section offsets of all strings within a `.debug_str` section." +); + +define_id!( + LineStringId, + "An identifier for a string in a `LineStringTable`." +); + +define_string_table!( + LineStringTable, + LineStringId, + DebugLineStr, + DebugLineStrOffsets, + "A table of strings that will be stored in a `.debug_line_str` section." +); + +define_section!( + DebugLineStr, + DebugLineStrOffset, + "A writable `.debug_line_str` section." +); + +define_offsets!( + DebugLineStrOffsets: LineStringId => DebugLineStrOffset, + "The section offsets of all strings within a `.debug_line_str` section." +); + +#[cfg(test)] +#[cfg(feature = "read")] +mod tests { + use super::*; + use crate::read; + use crate::write::EndianVec; + use crate::LittleEndian; + + #[test] + fn test_string_table() { + let mut strings = StringTable::default(); + assert_eq!(strings.count(), 0); + let id1 = strings.add(&b"one"[..]); + let id2 = strings.add(&b"two"[..]); + assert_eq!(strings.add(&b"one"[..]), id1); + assert_eq!(strings.add(&b"two"[..]), id2); + assert_eq!(strings.get(id1), &b"one"[..]); + assert_eq!(strings.get(id2), &b"two"[..]); + assert_eq!(strings.count(), 2); + + let mut debug_str = DebugStr::from(EndianVec::new(LittleEndian)); + let offsets = strings.write(&mut debug_str).unwrap(); + assert_eq!(debug_str.slice(), b"one\0two\0"); + assert_eq!(offsets.get(id1), DebugStrOffset(0)); + assert_eq!(offsets.get(id2), DebugStrOffset(4)); + assert_eq!(offsets.count(), 2); + } + + #[test] + fn test_string_table_read() { + let mut strings = StringTable::default(); + let id1 = strings.add(&b"one"[..]); + let id2 = strings.add(&b"two"[..]); + + let mut debug_str = DebugStr::from(EndianVec::new(LittleEndian)); + let offsets = strings.write(&mut debug_str).unwrap(); + + let read_debug_str = read::DebugStr::new(debug_str.slice(), LittleEndian); + let str1 = read_debug_str.get_str(offsets.get(id1)).unwrap(); + let str2 = read_debug_str.get_str(offsets.get(id2)).unwrap(); + assert_eq!(str1.slice(), &b"one"[..]); + assert_eq!(str2.slice(), &b"two"[..]); + } +} diff --git a/crux-mir/lib/gimli/src/write/unit.rs b/crux-mir/lib/gimli/src/write/unit.rs new file mode 100644 index 000000000..bf85ff421 --- /dev/null +++ b/crux-mir/lib/gimli/src/write/unit.rs @@ -0,0 +1,3157 @@ +use alloc::vec::Vec; +use std::ops::{Deref, DerefMut}; +use std::{slice, usize}; + +use crate::common::{ + DebugAbbrevOffset, DebugInfoOffset, DebugLineOffset, DebugMacinfoOffset, DebugMacroOffset, + DebugStrOffset, DebugTypeSignature, DwoId, Encoding, Format, SectionId, +}; +use crate::constants; +use crate::leb128::write::{sleb128_size, uleb128_size}; +use crate::write::{ + Abbreviation, AbbreviationTable, Address, AttributeSpecification, BaseId, DebugLineStrOffsets, + DebugStrOffsets, Error, Expression, FileId, LineProgram, LineStringId, LocationListId, + LocationListOffsets, LocationListTable, RangeListId, RangeListOffsets, RangeListTable, + Reference, Result, Section, Sections, StringId, Writer, +}; + +define_id!(UnitId, "An identifier for a unit in a `UnitTable`."); + +define_id!(UnitEntryId, "An identifier for an entry in a `Unit`."); + +/// A table of units that will be stored in the `.debug_info` section. +#[derive(Debug, Default)] +pub struct UnitTable { + base_id: BaseId, + units: Vec, +} + +impl UnitTable { + /// Create a new unit and add it to the table. + /// + /// `address_size` must be in bytes. + /// + /// Returns the `UnitId` of the new unit. + #[inline] + pub fn add(&mut self, unit: Unit) -> UnitId { + let id = UnitId::new(self.base_id, self.units.len()); + self.units.push(unit); + id + } + + /// Return the number of units. + #[inline] + pub fn count(&self) -> usize { + self.units.len() + } + + /// Return the id of a unit. + /// + /// # Panics + /// + /// Panics if `index >= self.count()`. + #[inline] + pub fn id(&self, index: usize) -> UnitId { + assert!(index < self.count()); + UnitId::new(self.base_id, index) + } + + /// Get a reference to a unit. + /// + /// # Panics + /// + /// Panics if `id` is invalid. + #[inline] + pub fn get(&self, id: UnitId) -> &Unit { + debug_assert_eq!(self.base_id, id.base_id); + &self.units[id.index] + } + + /// Get a mutable reference to a unit. + /// + /// # Panics + /// + /// Panics if `id` is invalid. + #[inline] + pub fn get_mut(&mut self, id: UnitId) -> &mut Unit { + debug_assert_eq!(self.base_id, id.base_id); + &mut self.units[id.index] + } + + /// Write the units to the given sections. + /// + /// `strings` must contain the `.debug_str` offsets of the corresponding + /// `StringTable`. + pub fn write( + &mut self, + sections: &mut Sections, + line_strings: &DebugLineStrOffsets, + strings: &DebugStrOffsets, + ) -> Result { + let mut offsets = DebugInfoOffsets { + base_id: self.base_id, + units: Vec::new(), + }; + for unit in &mut self.units { + // TODO: maybe share abbreviation tables + let abbrev_offset = sections.debug_abbrev.offset(); + let mut abbrevs = AbbreviationTable::default(); + + offsets.units.push(unit.write( + sections, + abbrev_offset, + &mut abbrevs, + line_strings, + strings, + )?); + + abbrevs.write(&mut sections.debug_abbrev)?; + } + + write_section_refs( + &mut sections.debug_info_refs, + &mut sections.debug_info.0, + &offsets, + )?; + write_section_refs( + &mut sections.debug_loc_refs, + &mut sections.debug_loc.0, + &offsets, + )?; + write_section_refs( + &mut sections.debug_loclists_refs, + &mut sections.debug_loclists.0, + &offsets, + )?; + + Ok(offsets) + } +} + +fn write_section_refs( + references: &mut Vec, + w: &mut W, + offsets: &DebugInfoOffsets, +) -> Result<()> { + for r in references.drain(..) { + let entry_offset = offsets.entry(r.unit, r.entry).0; + debug_assert_ne!(entry_offset, 0); + w.write_offset_at(r.offset, entry_offset, SectionId::DebugInfo, r.size)?; + } + Ok(()) +} + +/// A unit's debugging information. +#[derive(Debug)] +pub struct Unit { + base_id: BaseId, + /// The encoding parameters for this unit. + encoding: Encoding, + /// The line number program for this unit. + pub line_program: LineProgram, + /// A table of range lists used by this unit. + pub ranges: RangeListTable, + /// A table of location lists used by this unit. + pub locations: LocationListTable, + /// All entries in this unit. The order is unrelated to the tree order. + // Requirements: + // - entries form a tree + // - entries can be added in any order + // - entries have a fixed id + // - able to quickly lookup an entry from its id + // Limitations of current implemention: + // - mutable iteration of children is messy due to borrow checker + entries: Vec, + /// The index of the root entry in entries. + root: UnitEntryId, +} + +impl Unit { + /// Create a new `Unit`. + pub fn new(encoding: Encoding, line_program: LineProgram) -> Self { + let base_id = BaseId::default(); + let ranges = RangeListTable::default(); + let locations = LocationListTable::default(); + let mut entries = Vec::new(); + let root = DebuggingInformationEntry::new( + base_id, + &mut entries, + None, + constants::DW_TAG_compile_unit, + ); + Unit { + base_id, + encoding, + line_program, + ranges, + locations, + entries, + root, + } + } + + /// Return the encoding parameters for this unit. + #[inline] + pub fn encoding(&self) -> Encoding { + self.encoding + } + + /// Return the DWARF version for this unit. + #[inline] + pub fn version(&self) -> u16 { + self.encoding.version + } + + /// Return the address size in bytes for this unit. + #[inline] + pub fn address_size(&self) -> u8 { + self.encoding.address_size + } + + /// Return the DWARF format for this unit. + #[inline] + pub fn format(&self) -> Format { + self.encoding.format + } + + /// Return the number of `DebuggingInformationEntry`s created for this unit. + /// + /// This includes entries that no longer have a parent. + #[inline] + pub fn count(&self) -> usize { + self.entries.len() + } + + /// Return the id of the root entry. + #[inline] + pub fn root(&self) -> UnitEntryId { + self.root + } + + /// Add a new `DebuggingInformationEntry` to this unit and return its id. + /// + /// The `parent` must be within the same unit. + /// + /// # Panics + /// + /// Panics if `parent` is invalid. + #[inline] + pub fn add(&mut self, parent: UnitEntryId, tag: constants::DwTag) -> UnitEntryId { + debug_assert_eq!(self.base_id, parent.base_id); + DebuggingInformationEntry::new(self.base_id, &mut self.entries, Some(parent), tag) + } + + /// Get a reference to an entry. + /// + /// # Panics + /// + /// Panics if `id` is invalid. + #[inline] + pub fn get(&self, id: UnitEntryId) -> &DebuggingInformationEntry { + debug_assert_eq!(self.base_id, id.base_id); + &self.entries[id.index] + } + + /// Get a mutable reference to an entry. + /// + /// # Panics + /// + /// Panics if `id` is invalid. + #[inline] + pub fn get_mut(&mut self, id: UnitEntryId) -> &mut DebuggingInformationEntry { + debug_assert_eq!(self.base_id, id.base_id); + &mut self.entries[id.index] + } + + /// Return true if `self.line_program` is used by a DIE. + fn line_program_in_use(&self) -> bool { + if self.line_program.is_none() { + return false; + } + if !self.line_program.is_empty() { + return true; + } + + for entry in &self.entries { + for attr in &entry.attrs { + if let AttributeValue::FileIndex(Some(_)) = attr.value { + return true; + } + } + } + + false + } + + /// Write the unit to the given sections. + pub(crate) fn write( + &mut self, + sections: &mut Sections, + abbrev_offset: DebugAbbrevOffset, + abbrevs: &mut AbbreviationTable, + line_strings: &DebugLineStrOffsets, + strings: &DebugStrOffsets, + ) -> Result { + let line_program = if self.line_program_in_use() { + self.entries[self.root.index] + .set(constants::DW_AT_stmt_list, AttributeValue::LineProgramRef); + Some(self.line_program.write( + &mut sections.debug_line, + self.encoding, + line_strings, + strings, + )?) + } else { + self.entries[self.root.index].delete(constants::DW_AT_stmt_list); + None + }; + + // TODO: use .debug_types for type units in DWARF v4. + let w = &mut sections.debug_info; + + let mut offsets = UnitOffsets { + base_id: self.base_id, + unit: w.offset(), + // Entries can be written in any order, so create the complete vec now. + entries: vec![EntryOffset::none(); self.entries.len()], + }; + + let length_offset = w.write_initial_length(self.format())?; + let length_base = w.len(); + + w.write_u16(self.version())?; + if 2 <= self.version() && self.version() <= 4 { + w.write_offset( + abbrev_offset.0, + SectionId::DebugAbbrev, + self.format().word_size(), + )?; + w.write_u8(self.address_size())?; + } else if self.version() == 5 { + w.write_u8(constants::DW_UT_compile.0)?; + w.write_u8(self.address_size())?; + w.write_offset( + abbrev_offset.0, + SectionId::DebugAbbrev, + self.format().word_size(), + )?; + } else { + return Err(Error::UnsupportedVersion(self.version())); + } + + // Calculate all DIE offsets, so that we are able to output references to them. + // However, references to base types in expressions use ULEB128, so base types + // must be moved to the front before we can calculate offsets. + self.reorder_base_types(); + let mut offset = w.len(); + self.entries[self.root.index].calculate_offsets( + self, + &mut offset, + &mut offsets, + abbrevs, + )?; + + let range_lists = self.ranges.write(sections, self.encoding)?; + // Location lists can't be written until we have DIE offsets. + let loc_lists = self + .locations + .write(sections, self.encoding, Some(&offsets))?; + + let w = &mut sections.debug_info; + let mut unit_refs = Vec::new(); + self.entries[self.root.index].write( + w, + &mut sections.debug_info_refs, + &mut unit_refs, + self, + &mut offsets, + abbrevs, + line_program, + line_strings, + strings, + &range_lists, + &loc_lists, + )?; + + let length = (w.len() - length_base) as u64; + w.write_initial_length_at(length_offset, length, self.format())?; + + for (offset, entry) in unit_refs { + // This does not need relocation. + w.write_udata_at( + offset.0, + offsets.unit_offset(entry), + self.format().word_size(), + )?; + } + + Ok(offsets) + } + + /// Reorder base types to come first so that typed stack operations + /// can get their offset. + fn reorder_base_types(&mut self) { + let root = &self.entries[self.root.index]; + let mut root_children = Vec::with_capacity(root.children.len()); + for entry in &root.children { + if self.entries[entry.index].tag == constants::DW_TAG_base_type { + root_children.push(*entry); + } + } + for entry in &root.children { + if self.entries[entry.index].tag != constants::DW_TAG_base_type { + root_children.push(*entry); + } + } + self.entries[self.root.index].children = root_children; + } +} + +/// A Debugging Information Entry (DIE). +/// +/// DIEs have a set of attributes and optionally have children DIEs as well. +/// +/// DIEs form a tree without any cycles. This is enforced by specifying the +/// parent when creating a DIE, and disallowing changes of parent. +#[derive(Debug)] +pub struct DebuggingInformationEntry { + id: UnitEntryId, + parent: Option, + tag: constants::DwTag, + /// Whether to emit `DW_AT_sibling`. + sibling: bool, + attrs: Vec, + children: Vec, +} + +impl DebuggingInformationEntry { + /// Create a new `DebuggingInformationEntry`. + /// + /// # Panics + /// + /// Panics if `parent` is invalid. + #[allow(clippy::new_ret_no_self)] + fn new( + base_id: BaseId, + entries: &mut Vec, + parent: Option, + tag: constants::DwTag, + ) -> UnitEntryId { + let id = UnitEntryId::new(base_id, entries.len()); + entries.push(DebuggingInformationEntry { + id, + parent, + tag, + sibling: false, + attrs: Vec::new(), + children: Vec::new(), + }); + if let Some(parent) = parent { + debug_assert_eq!(base_id, parent.base_id); + assert_ne!(parent, id); + entries[parent.index].children.push(id); + } + id + } + + /// Return the id of this entry. + #[inline] + pub fn id(&self) -> UnitEntryId { + self.id + } + + /// Return the parent of this entry. + #[inline] + pub fn parent(&self) -> Option { + self.parent + } + + /// Return the tag of this entry. + #[inline] + pub fn tag(&self) -> constants::DwTag { + self.tag + } + + /// Return `true` if a `DW_AT_sibling` attribute will be emitted. + #[inline] + pub fn sibling(&self) -> bool { + self.sibling + } + + /// Set whether a `DW_AT_sibling` attribute will be emitted. + /// + /// The attribute will only be emitted if the DIE has children. + #[inline] + pub fn set_sibling(&mut self, sibling: bool) { + self.sibling = sibling; + } + + /// Iterate over the attributes of this entry. + #[inline] + pub fn attrs(&self) -> slice::Iter { + self.attrs.iter() + } + + /// Iterate over the attributes of this entry for modification. + #[inline] + pub fn attrs_mut(&mut self) -> slice::IterMut { + self.attrs.iter_mut() + } + + /// Get an attribute. + pub fn get(&self, name: constants::DwAt) -> Option<&AttributeValue> { + self.attrs + .iter() + .find(|attr| attr.name == name) + .map(|attr| &attr.value) + } + + /// Get an attribute for modification. + pub fn get_mut(&mut self, name: constants::DwAt) -> Option<&mut AttributeValue> { + self.attrs + .iter_mut() + .find(|attr| attr.name == name) + .map(|attr| &mut attr.value) + } + + /// Set an attribute. + /// + /// Replaces any existing attribute with the same name. + /// + /// # Panics + /// + /// Panics if `name` is `DW_AT_sibling`. Use `set_sibling` instead. + pub fn set(&mut self, name: constants::DwAt, value: AttributeValue) { + assert_ne!(name, constants::DW_AT_sibling); + if let Some(attr) = self.attrs.iter_mut().find(|attr| attr.name == name) { + attr.value = value; + return; + } + self.attrs.push(Attribute { name, value }); + } + + /// Delete an attribute. + /// + /// Replaces any existing attribute with the same name. + pub fn delete(&mut self, name: constants::DwAt) { + self.attrs.retain(|x| x.name != name); + } + + /// Iterate over the children of this entry. + /// + /// Note: use `Unit::add` to add a new child to this entry. + #[inline] + pub fn children(&self) -> slice::Iter { + self.children.iter() + } + + /// Delete a child entry and all of its children. + pub fn delete_child(&mut self, id: UnitEntryId) { + self.children.retain(|&child| child != id); + } + + /// Return the type abbreviation for this DIE. + fn abbreviation(&self, encoding: Encoding) -> Result { + let mut attrs = Vec::new(); + + if self.sibling && !self.children.is_empty() { + let form = match encoding.format { + Format::Dwarf32 => constants::DW_FORM_ref4, + Format::Dwarf64 => constants::DW_FORM_ref8, + }; + attrs.push(AttributeSpecification::new(constants::DW_AT_sibling, form)); + } + + for attr in &self.attrs { + attrs.push(attr.specification(encoding)?); + } + + Ok(Abbreviation::new( + self.tag, + !self.children.is_empty(), + attrs, + )) + } + + fn calculate_offsets( + &self, + unit: &Unit, + offset: &mut usize, + offsets: &mut UnitOffsets, + abbrevs: &mut AbbreviationTable, + ) -> Result<()> { + offsets.entries[self.id.index].offset = DebugInfoOffset(*offset); + offsets.entries[self.id.index].abbrev = abbrevs.add(self.abbreviation(unit.encoding())?); + *offset += self.size(unit, offsets); + if !self.children.is_empty() { + for child in &self.children { + unit.entries[child.index].calculate_offsets(unit, offset, offsets, abbrevs)?; + } + // Null child + *offset += 1; + } + Ok(()) + } + + fn size(&self, unit: &Unit, offsets: &UnitOffsets) -> usize { + let mut size = uleb128_size(offsets.abbrev(self.id)); + if self.sibling && !self.children.is_empty() { + size += unit.format().word_size() as usize; + } + for attr in &self.attrs { + size += attr.value.size(unit, offsets); + } + size + } + + /// Write the entry to the given sections. + #[allow(clippy::too_many_arguments)] + fn write( + &self, + w: &mut DebugInfo, + debug_info_refs: &mut Vec, + unit_refs: &mut Vec<(DebugInfoOffset, UnitEntryId)>, + unit: &Unit, + offsets: &mut UnitOffsets, + abbrevs: &mut AbbreviationTable, + line_program: Option, + line_strings: &DebugLineStrOffsets, + strings: &DebugStrOffsets, + range_lists: &RangeListOffsets, + loc_lists: &LocationListOffsets, + ) -> Result<()> { + debug_assert_eq!(offsets.debug_info_offset(self.id), w.offset()); + w.write_uleb128(offsets.abbrev(self.id))?; + + let sibling_offset = if self.sibling && !self.children.is_empty() { + let offset = w.offset(); + w.write_udata(0, unit.format().word_size())?; + Some(offset) + } else { + None + }; + + for attr in &self.attrs { + attr.value.write( + w, + debug_info_refs, + unit_refs, + unit, + offsets, + line_program, + line_strings, + strings, + range_lists, + loc_lists, + )?; + } + + if !self.children.is_empty() { + for child in &self.children { + unit.entries[child.index].write( + w, + debug_info_refs, + unit_refs, + unit, + offsets, + abbrevs, + line_program, + line_strings, + strings, + range_lists, + loc_lists, + )?; + } + // Null child + w.write_u8(0)?; + } + + if let Some(offset) = sibling_offset { + let next_offset = (w.offset().0 - offsets.unit.0) as u64; + // This does not need relocation. + w.write_udata_at(offset.0, next_offset, unit.format().word_size())?; + } + Ok(()) + } +} + +/// An attribute in a `DebuggingInformationEntry`, consisting of a name and +/// associated value. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Attribute { + name: constants::DwAt, + value: AttributeValue, +} + +impl Attribute { + /// Get the name of this attribute. + #[inline] + pub fn name(&self) -> constants::DwAt { + self.name + } + + /// Get the value of this attribute. + #[inline] + pub fn get(&self) -> &AttributeValue { + &self.value + } + + /// Set the value of this attribute. + #[inline] + pub fn set(&mut self, value: AttributeValue) { + self.value = value; + } + + /// Return the type specification for this attribute. + fn specification(&self, encoding: Encoding) -> Result { + Ok(AttributeSpecification::new( + self.name, + self.value.form(encoding)?, + )) + } +} + +/// The value of an attribute in a `DebuggingInformationEntry`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AttributeValue { + /// "Refers to some location in the address space of the described program." + Address(Address), + + /// A slice of an arbitrary number of bytes. + Block(Vec), + + /// A one byte constant data value. How to interpret the byte depends on context. + /// + /// From section 7 of the standard: "Depending on context, it may be a + /// signed integer, an unsigned integer, a floating-point constant, or + /// anything else." + Data1(u8), + + /// A two byte constant data value. How to interpret the bytes depends on context. + /// + /// This value will be converted to the target endian before writing. + /// + /// From section 7 of the standard: "Depending on context, it may be a + /// signed integer, an unsigned integer, a floating-point constant, or + /// anything else." + Data2(u16), + + /// A four byte constant data value. How to interpret the bytes depends on context. + /// + /// This value will be converted to the target endian before writing. + /// + /// From section 7 of the standard: "Depending on context, it may be a + /// signed integer, an unsigned integer, a floating-point constant, or + /// anything else." + Data4(u32), + + /// An eight byte constant data value. How to interpret the bytes depends on context. + /// + /// This value will be converted to the target endian before writing. + /// + /// From section 7 of the standard: "Depending on context, it may be a + /// signed integer, an unsigned integer, a floating-point constant, or + /// anything else." + Data8(u64), + + /// A signed integer constant. + Sdata(i64), + + /// An unsigned integer constant. + Udata(u64), + + /// "The information bytes contain a DWARF expression (see Section 2.5) or + /// location description (see Section 2.6)." + Exprloc(Expression), + + /// A boolean that indicates presence or absence of the attribute. + Flag(bool), + + /// An attribute that is always present. + FlagPresent, + + /// A reference to a `DebuggingInformationEntry` in this unit. + UnitRef(UnitEntryId), + + /// A reference to a `DebuggingInformationEntry` in a potentially different unit. + DebugInfoRef(Reference), + + /// An offset into the `.debug_info` section of the supplementary object file. + /// + /// The API does not currently assist with generating this offset. + /// This variant will be removed from the API once support for writing + /// supplementary object files is implemented. + DebugInfoRefSup(DebugInfoOffset), + + /// A reference to a line number program. + LineProgramRef, + + /// A reference to a location list. + LocationListRef(LocationListId), + + /// An offset into the `.debug_macinfo` section. + /// + /// The API does not currently assist with generating this offset. + /// This variant will be removed from the API once support for writing + /// `.debug_macinfo` sections is implemented. + DebugMacinfoRef(DebugMacinfoOffset), + + /// An offset into the `.debug_macro` section. + /// + /// The API does not currently assist with generating this offset. + /// This variant will be removed from the API once support for writing + /// `.debug_macro` sections is implemented. + DebugMacroRef(DebugMacroOffset), + + /// A reference to a range list. + RangeListRef(RangeListId), + + /// A type signature. + /// + /// The API does not currently assist with generating this signature. + /// This variant will be removed from the API once support for writing + /// `.debug_types` sections is implemented. + DebugTypesRef(DebugTypeSignature), + + /// A reference to a string in the `.debug_str` section. + StringRef(StringId), + + /// An offset into the `.debug_str` section of the supplementary object file. + /// + /// The API does not currently assist with generating this offset. + /// This variant will be removed from the API once support for writing + /// supplementary object files is implemented. + DebugStrRefSup(DebugStrOffset), + + /// A reference to a string in the `.debug_line_str` section. + LineStringRef(LineStringId), + + /// A slice of bytes representing a string. Must not include null bytes. + /// Not guaranteed to be UTF-8 or anything like that. + String(Vec), + + /// The value of a `DW_AT_encoding` attribute. + Encoding(constants::DwAte), + + /// The value of a `DW_AT_decimal_sign` attribute. + DecimalSign(constants::DwDs), + + /// The value of a `DW_AT_endianity` attribute. + Endianity(constants::DwEnd), + + /// The value of a `DW_AT_accessibility` attribute. + Accessibility(constants::DwAccess), + + /// The value of a `DW_AT_visibility` attribute. + Visibility(constants::DwVis), + + /// The value of a `DW_AT_virtuality` attribute. + Virtuality(constants::DwVirtuality), + + /// The value of a `DW_AT_language` attribute. + Language(constants::DwLang), + + /// The value of a `DW_AT_address_class` attribute. + AddressClass(constants::DwAddr), + + /// The value of a `DW_AT_identifier_case` attribute. + IdentifierCase(constants::DwId), + + /// The value of a `DW_AT_calling_convention` attribute. + CallingConvention(constants::DwCc), + + /// The value of a `DW_AT_inline` attribute. + Inline(constants::DwInl), + + /// The value of a `DW_AT_ordering` attribute. + Ordering(constants::DwOrd), + + /// An index into the filename entries from the line number information + /// table for the unit containing this value. + FileIndex(Option), +} + +impl AttributeValue { + /// Return the form that will be used to encode this value. + pub fn form(&self, encoding: Encoding) -> Result { + // TODO: missing forms: + // - DW_FORM_indirect + // - DW_FORM_implicit_const + // - FW_FORM_block1/block2/block4 + // - DW_FORM_str/strx1/strx2/strx3/strx4 + // - DW_FORM_addrx/addrx1/addrx2/addrx3/addrx4 + // - DW_FORM_data16 + // - DW_FORM_line_strp + // - DW_FORM_loclistx + // - DW_FORM_rnglistx + let form = match *self { + AttributeValue::Address(_) => constants::DW_FORM_addr, + AttributeValue::Block(_) => constants::DW_FORM_block, + AttributeValue::Data1(_) => constants::DW_FORM_data1, + AttributeValue::Data2(_) => constants::DW_FORM_data2, + AttributeValue::Data4(_) => constants::DW_FORM_data4, + AttributeValue::Data8(_) => constants::DW_FORM_data8, + AttributeValue::Exprloc(_) => constants::DW_FORM_exprloc, + AttributeValue::Flag(_) => constants::DW_FORM_flag, + AttributeValue::FlagPresent => constants::DW_FORM_flag_present, + AttributeValue::UnitRef(_) => { + // Using a fixed size format lets us write a placeholder before we know + // the value. + match encoding.format { + Format::Dwarf32 => constants::DW_FORM_ref4, + Format::Dwarf64 => constants::DW_FORM_ref8, + } + } + AttributeValue::DebugInfoRef(_) => constants::DW_FORM_ref_addr, + AttributeValue::DebugInfoRefSup(_) => { + // TODO: should this depend on the size of supplementary section? + match encoding.format { + Format::Dwarf32 => constants::DW_FORM_ref_sup4, + Format::Dwarf64 => constants::DW_FORM_ref_sup8, + } + } + AttributeValue::LineProgramRef + | AttributeValue::LocationListRef(_) + | AttributeValue::DebugMacinfoRef(_) + | AttributeValue::DebugMacroRef(_) + | AttributeValue::RangeListRef(_) => { + if encoding.version == 2 || encoding.version == 3 { + match encoding.format { + Format::Dwarf32 => constants::DW_FORM_data4, + Format::Dwarf64 => constants::DW_FORM_data8, + } + } else { + constants::DW_FORM_sec_offset + } + } + AttributeValue::DebugTypesRef(_) => constants::DW_FORM_ref_sig8, + AttributeValue::StringRef(_) => constants::DW_FORM_strp, + AttributeValue::DebugStrRefSup(_) => constants::DW_FORM_strp_sup, + AttributeValue::LineStringRef(_) => constants::DW_FORM_line_strp, + AttributeValue::String(_) => constants::DW_FORM_string, + AttributeValue::Encoding(_) + | AttributeValue::DecimalSign(_) + | AttributeValue::Endianity(_) + | AttributeValue::Accessibility(_) + | AttributeValue::Visibility(_) + | AttributeValue::Virtuality(_) + | AttributeValue::Language(_) + | AttributeValue::AddressClass(_) + | AttributeValue::IdentifierCase(_) + | AttributeValue::CallingConvention(_) + | AttributeValue::Inline(_) + | AttributeValue::Ordering(_) + | AttributeValue::FileIndex(_) + | AttributeValue::Udata(_) => constants::DW_FORM_udata, + AttributeValue::Sdata(_) => constants::DW_FORM_sdata, + }; + Ok(form) + } + + fn size(&self, unit: &Unit, offsets: &UnitOffsets) -> usize { + macro_rules! debug_assert_form { + ($form:expr) => { + debug_assert_eq!(self.form(unit.encoding()).unwrap(), $form) + }; + } + match *self { + AttributeValue::Address(_) => { + debug_assert_form!(constants::DW_FORM_addr); + unit.address_size() as usize + } + AttributeValue::Block(ref val) => { + debug_assert_form!(constants::DW_FORM_block); + uleb128_size(val.len() as u64) + val.len() + } + AttributeValue::Data1(_) => { + debug_assert_form!(constants::DW_FORM_data1); + 1 + } + AttributeValue::Data2(_) => { + debug_assert_form!(constants::DW_FORM_data2); + 2 + } + AttributeValue::Data4(_) => { + debug_assert_form!(constants::DW_FORM_data4); + 4 + } + AttributeValue::Data8(_) => { + debug_assert_form!(constants::DW_FORM_data8); + 8 + } + AttributeValue::Sdata(val) => { + debug_assert_form!(constants::DW_FORM_sdata); + sleb128_size(val) + } + AttributeValue::Udata(val) => { + debug_assert_form!(constants::DW_FORM_udata); + uleb128_size(val) + } + AttributeValue::Exprloc(ref val) => { + debug_assert_form!(constants::DW_FORM_exprloc); + let size = val.size(unit.encoding(), Some(offsets)); + uleb128_size(size as u64) + size + } + AttributeValue::Flag(_) => { + debug_assert_form!(constants::DW_FORM_flag); + 1 + } + AttributeValue::FlagPresent => { + debug_assert_form!(constants::DW_FORM_flag_present); + 0 + } + AttributeValue::UnitRef(_) => { + match unit.format() { + Format::Dwarf32 => debug_assert_form!(constants::DW_FORM_ref4), + Format::Dwarf64 => debug_assert_form!(constants::DW_FORM_ref8), + } + unit.format().word_size() as usize + } + AttributeValue::DebugInfoRef(_) => { + debug_assert_form!(constants::DW_FORM_ref_addr); + if unit.version() == 2 { + unit.address_size() as usize + } else { + unit.format().word_size() as usize + } + } + AttributeValue::DebugInfoRefSup(_) => { + match unit.format() { + Format::Dwarf32 => debug_assert_form!(constants::DW_FORM_ref_sup4), + Format::Dwarf64 => debug_assert_form!(constants::DW_FORM_ref_sup8), + } + unit.format().word_size() as usize + } + AttributeValue::LineProgramRef => { + if unit.version() >= 4 { + debug_assert_form!(constants::DW_FORM_sec_offset); + } + unit.format().word_size() as usize + } + AttributeValue::LocationListRef(_) => { + if unit.version() >= 4 { + debug_assert_form!(constants::DW_FORM_sec_offset); + } + unit.format().word_size() as usize + } + AttributeValue::DebugMacinfoRef(_) => { + if unit.version() >= 4 { + debug_assert_form!(constants::DW_FORM_sec_offset); + } + unit.format().word_size() as usize + } + AttributeValue::DebugMacroRef(_) => { + if unit.version() >= 4 { + debug_assert_form!(constants::DW_FORM_sec_offset); + } + unit.format().word_size() as usize + } + AttributeValue::RangeListRef(_) => { + if unit.version() >= 4 { + debug_assert_form!(constants::DW_FORM_sec_offset); + } + unit.format().word_size() as usize + } + AttributeValue::DebugTypesRef(_) => { + debug_assert_form!(constants::DW_FORM_ref_sig8); + 8 + } + AttributeValue::StringRef(_) => { + debug_assert_form!(constants::DW_FORM_strp); + unit.format().word_size() as usize + } + AttributeValue::DebugStrRefSup(_) => { + debug_assert_form!(constants::DW_FORM_strp_sup); + unit.format().word_size() as usize + } + AttributeValue::LineStringRef(_) => { + debug_assert_form!(constants::DW_FORM_line_strp); + unit.format().word_size() as usize + } + AttributeValue::String(ref val) => { + debug_assert_form!(constants::DW_FORM_string); + val.len() + 1 + } + AttributeValue::Encoding(val) => { + debug_assert_form!(constants::DW_FORM_udata); + uleb128_size(val.0 as u64) + } + AttributeValue::DecimalSign(val) => { + debug_assert_form!(constants::DW_FORM_udata); + uleb128_size(val.0 as u64) + } + AttributeValue::Endianity(val) => { + debug_assert_form!(constants::DW_FORM_udata); + uleb128_size(val.0 as u64) + } + AttributeValue::Accessibility(val) => { + debug_assert_form!(constants::DW_FORM_udata); + uleb128_size(val.0 as u64) + } + AttributeValue::Visibility(val) => { + debug_assert_form!(constants::DW_FORM_udata); + uleb128_size(val.0 as u64) + } + AttributeValue::Virtuality(val) => { + debug_assert_form!(constants::DW_FORM_udata); + uleb128_size(val.0 as u64) + } + AttributeValue::Language(val) => { + debug_assert_form!(constants::DW_FORM_udata); + uleb128_size(val.0 as u64) + } + AttributeValue::AddressClass(val) => { + debug_assert_form!(constants::DW_FORM_udata); + uleb128_size(val.0 as u64) + } + AttributeValue::IdentifierCase(val) => { + debug_assert_form!(constants::DW_FORM_udata); + uleb128_size(val.0 as u64) + } + AttributeValue::CallingConvention(val) => { + debug_assert_form!(constants::DW_FORM_udata); + uleb128_size(val.0 as u64) + } + AttributeValue::Inline(val) => { + debug_assert_form!(constants::DW_FORM_udata); + uleb128_size(val.0 as u64) + } + AttributeValue::Ordering(val) => { + debug_assert_form!(constants::DW_FORM_udata); + uleb128_size(val.0 as u64) + } + AttributeValue::FileIndex(val) => { + debug_assert_form!(constants::DW_FORM_udata); + uleb128_size(val.map(FileId::raw).unwrap_or(0)) + } + } + } + + /// Write the attribute value to the given sections. + #[allow(clippy::cyclomatic_complexity, clippy::too_many_arguments)] + fn write( + &self, + w: &mut DebugInfo, + debug_info_refs: &mut Vec, + unit_refs: &mut Vec<(DebugInfoOffset, UnitEntryId)>, + unit: &Unit, + offsets: &UnitOffsets, + line_program: Option, + line_strings: &DebugLineStrOffsets, + strings: &DebugStrOffsets, + range_lists: &RangeListOffsets, + loc_lists: &LocationListOffsets, + ) -> Result<()> { + macro_rules! debug_assert_form { + ($form:expr) => { + debug_assert_eq!(self.form(unit.encoding()).unwrap(), $form) + }; + } + match *self { + AttributeValue::Address(val) => { + debug_assert_form!(constants::DW_FORM_addr); + w.write_address(val, unit.address_size())?; + } + AttributeValue::Block(ref val) => { + debug_assert_form!(constants::DW_FORM_block); + w.write_uleb128(val.len() as u64)?; + w.write(&val)?; + } + AttributeValue::Data1(val) => { + debug_assert_form!(constants::DW_FORM_data1); + w.write_u8(val)?; + } + AttributeValue::Data2(val) => { + debug_assert_form!(constants::DW_FORM_data2); + w.write_u16(val)?; + } + AttributeValue::Data4(val) => { + debug_assert_form!(constants::DW_FORM_data4); + w.write_u32(val)?; + } + AttributeValue::Data8(val) => { + debug_assert_form!(constants::DW_FORM_data8); + w.write_u64(val)?; + } + AttributeValue::Sdata(val) => { + debug_assert_form!(constants::DW_FORM_sdata); + w.write_sleb128(val)?; + } + AttributeValue::Udata(val) => { + debug_assert_form!(constants::DW_FORM_udata); + w.write_uleb128(val)?; + } + AttributeValue::Exprloc(ref val) => { + debug_assert_form!(constants::DW_FORM_exprloc); + w.write_uleb128(val.size(unit.encoding(), Some(offsets)) as u64)?; + val.write( + &mut w.0, + Some(debug_info_refs), + unit.encoding(), + Some(offsets), + )?; + } + AttributeValue::Flag(val) => { + debug_assert_form!(constants::DW_FORM_flag); + w.write_u8(val as u8)?; + } + AttributeValue::FlagPresent => { + debug_assert_form!(constants::DW_FORM_flag_present); + } + AttributeValue::UnitRef(id) => { + match unit.format() { + Format::Dwarf32 => debug_assert_form!(constants::DW_FORM_ref4), + Format::Dwarf64 => debug_assert_form!(constants::DW_FORM_ref8), + } + unit_refs.push((w.offset(), id)); + w.write_udata(0, unit.format().word_size())?; + } + AttributeValue::DebugInfoRef(reference) => { + debug_assert_form!(constants::DW_FORM_ref_addr); + let size = if unit.version() == 2 { + unit.address_size() + } else { + unit.format().word_size() + }; + match reference { + Reference::Symbol(symbol) => w.write_reference(symbol, size)?, + Reference::Entry(unit, entry) => { + debug_info_refs.push(DebugInfoReference { + offset: w.len(), + unit, + entry, + size, + }); + w.write_udata(0, size)?; + } + } + } + AttributeValue::DebugInfoRefSup(val) => { + match unit.format() { + Format::Dwarf32 => debug_assert_form!(constants::DW_FORM_ref_sup4), + Format::Dwarf64 => debug_assert_form!(constants::DW_FORM_ref_sup8), + } + w.write_udata(val.0 as u64, unit.format().word_size())?; + } + AttributeValue::LineProgramRef => { + if unit.version() >= 4 { + debug_assert_form!(constants::DW_FORM_sec_offset); + } + match line_program { + Some(line_program) => { + w.write_offset( + line_program.0, + SectionId::DebugLine, + unit.format().word_size(), + )?; + } + None => return Err(Error::InvalidAttributeValue), + } + } + AttributeValue::LocationListRef(val) => { + if unit.version() >= 4 { + debug_assert_form!(constants::DW_FORM_sec_offset); + } + let section = if unit.version() <= 4 { + SectionId::DebugLoc + } else { + SectionId::DebugLocLists + }; + w.write_offset(loc_lists.get(val).0, section, unit.format().word_size())?; + } + AttributeValue::DebugMacinfoRef(val) => { + if unit.version() >= 4 { + debug_assert_form!(constants::DW_FORM_sec_offset); + } + w.write_offset(val.0, SectionId::DebugMacinfo, unit.format().word_size())?; + } + AttributeValue::DebugMacroRef(val) => { + if unit.version() >= 4 { + debug_assert_form!(constants::DW_FORM_sec_offset); + } + w.write_offset(val.0, SectionId::DebugMacro, unit.format().word_size())?; + } + AttributeValue::RangeListRef(val) => { + if unit.version() >= 4 { + debug_assert_form!(constants::DW_FORM_sec_offset); + } + let section = if unit.version() <= 4 { + SectionId::DebugRanges + } else { + SectionId::DebugRngLists + }; + w.write_offset(range_lists.get(val).0, section, unit.format().word_size())?; + } + AttributeValue::DebugTypesRef(val) => { + debug_assert_form!(constants::DW_FORM_ref_sig8); + w.write_u64(val.0)?; + } + AttributeValue::StringRef(val) => { + debug_assert_form!(constants::DW_FORM_strp); + w.write_offset( + strings.get(val).0, + SectionId::DebugStr, + unit.format().word_size(), + )?; + } + AttributeValue::DebugStrRefSup(val) => { + debug_assert_form!(constants::DW_FORM_strp_sup); + w.write_udata(val.0 as u64, unit.format().word_size())?; + } + AttributeValue::LineStringRef(val) => { + debug_assert_form!(constants::DW_FORM_line_strp); + w.write_offset( + line_strings.get(val).0, + SectionId::DebugLineStr, + unit.format().word_size(), + )?; + } + AttributeValue::String(ref val) => { + debug_assert_form!(constants::DW_FORM_string); + w.write(&val)?; + w.write_u8(0)?; + } + AttributeValue::Encoding(val) => { + debug_assert_form!(constants::DW_FORM_udata); + w.write_uleb128(u64::from(val.0))?; + } + AttributeValue::DecimalSign(val) => { + debug_assert_form!(constants::DW_FORM_udata); + w.write_uleb128(u64::from(val.0))?; + } + AttributeValue::Endianity(val) => { + debug_assert_form!(constants::DW_FORM_udata); + w.write_uleb128(u64::from(val.0))?; + } + AttributeValue::Accessibility(val) => { + debug_assert_form!(constants::DW_FORM_udata); + w.write_uleb128(u64::from(val.0))?; + } + AttributeValue::Visibility(val) => { + debug_assert_form!(constants::DW_FORM_udata); + w.write_uleb128(u64::from(val.0))?; + } + AttributeValue::Virtuality(val) => { + debug_assert_form!(constants::DW_FORM_udata); + w.write_uleb128(u64::from(val.0))?; + } + AttributeValue::Language(val) => { + debug_assert_form!(constants::DW_FORM_udata); + w.write_uleb128(u64::from(val.0))?; + } + AttributeValue::AddressClass(val) => { + debug_assert_form!(constants::DW_FORM_udata); + w.write_uleb128(val.0)?; + } + AttributeValue::IdentifierCase(val) => { + debug_assert_form!(constants::DW_FORM_udata); + w.write_uleb128(u64::from(val.0))?; + } + AttributeValue::CallingConvention(val) => { + debug_assert_form!(constants::DW_FORM_udata); + w.write_uleb128(u64::from(val.0))?; + } + AttributeValue::Inline(val) => { + debug_assert_form!(constants::DW_FORM_udata); + w.write_uleb128(u64::from(val.0))?; + } + AttributeValue::Ordering(val) => { + debug_assert_form!(constants::DW_FORM_udata); + w.write_uleb128(u64::from(val.0))?; + } + AttributeValue::FileIndex(val) => { + debug_assert_form!(constants::DW_FORM_udata); + w.write_uleb128(val.map(FileId::raw).unwrap_or(0))?; + } + } + Ok(()) + } +} + +define_section!( + DebugInfo, + DebugInfoOffset, + "A writable `.debug_info` section." +); + +/// The section offsets of all elements within a `.debug_info` section. +#[derive(Debug, Default)] +pub struct DebugInfoOffsets { + base_id: BaseId, + units: Vec, +} + +impl DebugInfoOffsets { + #[cfg(test)] + pub(crate) fn unit_offsets(&self, unit: UnitId) -> &UnitOffsets { + debug_assert_eq!(self.base_id, unit.base_id); + &self.units[unit.index] + } + + /// Get the `.debug_info` section offset for the given unit. + #[inline] + pub fn unit(&self, unit: UnitId) -> DebugInfoOffset { + debug_assert_eq!(self.base_id, unit.base_id); + self.units[unit.index].unit + } + + /// Get the `.debug_info` section offset for the given entry. + #[inline] + pub fn entry(&self, unit: UnitId, entry: UnitEntryId) -> DebugInfoOffset { + debug_assert_eq!(self.base_id, unit.base_id); + self.units[unit.index].debug_info_offset(entry) + } +} + +/// The section offsets of all elements of a unit within a `.debug_info` section. +#[derive(Debug)] +pub(crate) struct UnitOffsets { + base_id: BaseId, + unit: DebugInfoOffset, + entries: Vec, +} + +impl UnitOffsets { + #[cfg(test)] + fn none() -> Self { + UnitOffsets { + base_id: BaseId::default(), + unit: DebugInfoOffset(0), + entries: Vec::new(), + } + } + + /// Get the .debug_info offset for the given entry. + #[inline] + pub(crate) fn debug_info_offset(&self, entry: UnitEntryId) -> DebugInfoOffset { + debug_assert_eq!(self.base_id, entry.base_id); + let offset = self.entries[entry.index].offset; + debug_assert_ne!(offset.0, 0); + offset + } + + /// Get the unit offset for the given entry. + #[inline] + pub(crate) fn unit_offset(&self, entry: UnitEntryId) -> u64 { + let offset = self.debug_info_offset(entry); + (offset.0 - self.unit.0) as u64 + } + + /// Get the abbreviation code for the given entry. + #[inline] + pub(crate) fn abbrev(&self, entry: UnitEntryId) -> u64 { + debug_assert_eq!(self.base_id, entry.base_id); + self.entries[entry.index].abbrev + } +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct EntryOffset { + offset: DebugInfoOffset, + abbrev: u64, +} + +impl EntryOffset { + fn none() -> Self { + EntryOffset { + offset: DebugInfoOffset(0), + abbrev: 0, + } + } +} + +/// A reference to a `.debug_info` entry that has yet to be resolved. +#[derive(Debug, Clone, Copy)] +pub(crate) struct DebugInfoReference { + /// The offset within the section of the reference. + pub offset: usize, + /// The size of the reference. + pub size: u8, + /// The unit containing the entry. + pub unit: UnitId, + /// The entry being referenced. + pub entry: UnitEntryId, +} + +#[cfg(feature = "read")] +pub(crate) mod convert { + use super::*; + use crate::common::UnitSectionOffset; + use crate::read::{self, Reader}; + use crate::write::{self, ConvertError, ConvertResult, LocationList, RangeList}; + use std::collections::HashMap; + + pub(crate) struct ConvertUnit> { + from_unit: read::Unit, + base_id: BaseId, + encoding: Encoding, + entries: Vec, + entry_offsets: Vec, + root: UnitEntryId, + } + + pub(crate) struct ConvertUnitContext<'a, R: Reader> { + pub dwarf: &'a read::Dwarf, + pub unit: &'a read::Unit, + pub line_strings: &'a mut write::LineStringTable, + pub strings: &'a mut write::StringTable, + pub ranges: &'a mut write::RangeListTable, + pub locations: &'a mut write::LocationListTable, + pub convert_address: &'a dyn Fn(u64) -> Option
, + pub base_address: Address, + pub line_program_offset: Option, + pub line_program_files: Vec, + pub entry_ids: &'a HashMap, + } + + impl UnitTable { + /// Create a unit table by reading the data in the given sections. + /// + /// This also updates the given tables with the values that are referenced from + /// attributes in this section. + /// + /// `convert_address` is a function to convert read addresses into the `Address` + /// type. For non-relocatable addresses, this function may simply return + /// `Address::Constant(address)`. For relocatable addresses, it is the caller's + /// responsibility to determine the symbol and addend corresponding to the address + /// and return `Address::Symbol { symbol, addend }`. + pub fn from>( + dwarf: &read::Dwarf, + line_strings: &mut write::LineStringTable, + strings: &mut write::StringTable, + convert_address: &dyn Fn(u64) -> Option
, + ) -> ConvertResult { + let base_id = BaseId::default(); + let mut unit_entries = Vec::new(); + let mut entry_ids = HashMap::new(); + + let mut from_units = dwarf.units(); + while let Some(from_unit) = from_units.next()? { + let unit_id = UnitId::new(base_id, unit_entries.len()); + unit_entries.push(Unit::convert_entries( + from_unit, + unit_id, + &mut entry_ids, + dwarf, + )?); + } + + // Attributes must be converted in a separate pass so that we can handle + // references to other compilation units. + let mut units = Vec::new(); + for unit_entries in unit_entries.drain(..) { + units.push(Unit::convert_attributes( + unit_entries, + &entry_ids, + dwarf, + line_strings, + strings, + convert_address, + )?); + } + + Ok(UnitTable { base_id, units }) + } + } + + impl Unit { + /// Create a unit by reading the data in the input sections. + /// + /// Does not add entry attributes. + #[allow(clippy::too_many_arguments)] + pub(crate) fn convert_entries>( + from_header: read::UnitHeader, + unit_id: UnitId, + entry_ids: &mut HashMap, + dwarf: &read::Dwarf, + ) -> ConvertResult> { + match from_header.type_() { + read::UnitType::Compilation => (), + _ => return Err(ConvertError::UnsupportedUnitType), + } + let base_id = BaseId::default(); + + let from_unit = dwarf.unit(from_header)?; + let encoding = from_unit.encoding(); + + let mut entries = Vec::new(); + let mut entry_offsets = Vec::new(); + + let mut from_tree = from_unit.entries_tree(None)?; + let from_root = from_tree.root()?; + let root = DebuggingInformationEntry::convert_entry( + from_root, + &from_unit, + base_id, + &mut entries, + &mut entry_offsets, + entry_ids, + None, + unit_id, + )?; + + Ok(ConvertUnit { + from_unit, + base_id, + encoding, + entries, + entry_offsets, + root, + }) + } + + /// Create entry attributes by reading the data in the input sections. + fn convert_attributes>( + unit: ConvertUnit, + entry_ids: &HashMap, + dwarf: &read::Dwarf, + line_strings: &mut write::LineStringTable, + strings: &mut write::StringTable, + convert_address: &dyn Fn(u64) -> Option
, + ) -> ConvertResult { + let from_unit = unit.from_unit; + let base_address = + convert_address(from_unit.low_pc).ok_or(ConvertError::InvalidAddress)?; + + let (line_program_offset, line_program, line_program_files) = + match from_unit.line_program { + Some(ref from_program) => { + let from_program = from_program.clone(); + let line_program_offset = from_program.header().offset(); + let (line_program, line_program_files) = LineProgram::from( + from_program, + dwarf, + line_strings, + strings, + convert_address, + )?; + (Some(line_program_offset), line_program, line_program_files) + } + None => (None, LineProgram::none(), Vec::new()), + }; + + let mut ranges = RangeListTable::default(); + let mut locations = LocationListTable::default(); + + let mut context = ConvertUnitContext { + entry_ids, + dwarf, + unit: &from_unit, + line_strings, + strings, + ranges: &mut ranges, + locations: &mut locations, + convert_address, + base_address, + line_program_offset, + line_program_files, + }; + + let mut entries = unit.entries; + for entry in &mut entries { + entry.convert_attributes(&mut context, &unit.entry_offsets)?; + } + + Ok(Unit { + base_id: unit.base_id, + encoding: unit.encoding, + line_program, + ranges, + locations, + entries, + root: unit.root, + }) + } + } + + impl DebuggingInformationEntry { + /// Create an entry by reading the data in the input sections. + /// + /// Does not add the entry attributes. + fn convert_entry>( + from: read::EntriesTreeNode, + from_unit: &read::Unit, + base_id: BaseId, + entries: &mut Vec, + entry_offsets: &mut Vec, + entry_ids: &mut HashMap, + parent: Option, + unit_id: UnitId, + ) -> ConvertResult { + let from_entry = from.entry(); + let id = DebuggingInformationEntry::new(base_id, entries, parent, from_entry.tag()); + let offset = from_entry.offset(); + entry_offsets.push(offset); + entry_ids.insert(offset.to_unit_section_offset(from_unit), (unit_id, id)); + + let mut from_children = from.children(); + while let Some(from_child) = from_children.next()? { + DebuggingInformationEntry::convert_entry( + from_child, + from_unit, + base_id, + entries, + entry_offsets, + entry_ids, + Some(id), + unit_id, + )?; + } + Ok(id) + } + + /// Create an entry's attributes by reading the data in the input sections. + fn convert_attributes>( + &mut self, + context: &mut ConvertUnitContext, + entry_offsets: &[read::UnitOffset], + ) -> ConvertResult<()> { + let offset = entry_offsets[self.id.index]; + let from = context.unit.entry(offset)?; + let mut from_attrs = from.attrs(); + while let Some(from_attr) = from_attrs.next()? { + if from_attr.name() == constants::DW_AT_sibling { + // This may point to a null entry, so we have to treat it differently. + self.set_sibling(true); + } else if let Some(attr) = Attribute::from(context, &from_attr)? { + self.set(attr.name, attr.value); + } + } + Ok(()) + } + } + + impl Attribute { + /// Create an attribute by reading the data in the given sections. + pub(crate) fn from>( + context: &mut ConvertUnitContext, + from: &read::Attribute, + ) -> ConvertResult> { + let value = AttributeValue::from(context, from.value())?; + Ok(value.map(|value| Attribute { + name: from.name(), + value, + })) + } + } + + impl AttributeValue { + /// Create an attribute value by reading the data in the given sections. + pub(crate) fn from>( + context: &mut ConvertUnitContext, + from: read::AttributeValue, + ) -> ConvertResult> { + let to = match from { + read::AttributeValue::Addr(val) => match (context.convert_address)(val) { + Some(val) => AttributeValue::Address(val), + None => return Err(ConvertError::InvalidAddress), + }, + read::AttributeValue::Block(r) => AttributeValue::Block(r.to_slice()?.into()), + read::AttributeValue::Data1(val) => AttributeValue::Data1(val), + read::AttributeValue::Data2(val) => AttributeValue::Data2(val), + read::AttributeValue::Data4(val) => AttributeValue::Data4(val), + read::AttributeValue::Data8(val) => AttributeValue::Data8(val), + read::AttributeValue::Sdata(val) => AttributeValue::Sdata(val), + read::AttributeValue::Udata(val) => AttributeValue::Udata(val), + read::AttributeValue::Exprloc(expression) => { + let expression = Expression::from( + expression, + context.unit.encoding(), + Some(context.dwarf), + Some(context.unit), + Some(context.entry_ids), + context.convert_address, + )?; + AttributeValue::Exprloc(expression) + } + // TODO: it would be nice to preserve the flag form. + read::AttributeValue::Flag(val) => AttributeValue::Flag(val), + read::AttributeValue::DebugAddrBase(_base) => { + // We convert all address indices to addresses, + // so this is unneeded. + return Ok(None); + } + read::AttributeValue::DebugAddrIndex(index) => { + let val = context.dwarf.address(context.unit, index)?; + match (context.convert_address)(val) { + Some(val) => AttributeValue::Address(val), + None => return Err(ConvertError::InvalidAddress), + } + } + read::AttributeValue::UnitRef(val) => { + if !context.unit.header.is_valid_offset(val) { + return Err(ConvertError::InvalidUnitRef); + } + let id = context + .entry_ids + .get(&val.to_unit_section_offset(context.unit)) + .ok_or(ConvertError::InvalidUnitRef)?; + AttributeValue::UnitRef(id.1) + } + read::AttributeValue::DebugInfoRef(val) => { + // TODO: support relocation of this value + let id = context + .entry_ids + .get(&UnitSectionOffset::DebugInfoOffset(val)) + .ok_or(ConvertError::InvalidDebugInfoRef)?; + AttributeValue::DebugInfoRef(Reference::Entry(id.0, id.1)) + } + read::AttributeValue::DebugInfoRefSup(val) => AttributeValue::DebugInfoRefSup(val), + read::AttributeValue::DebugLineRef(val) => { + // There should only be the line program in the CU DIE which we've already + // converted, so check if it matches that. + if Some(val) == context.line_program_offset { + AttributeValue::LineProgramRef + } else { + return Err(ConvertError::InvalidLineRef); + } + } + read::AttributeValue::DebugMacinfoRef(val) => AttributeValue::DebugMacinfoRef(val), + read::AttributeValue::DebugMacroRef(val) => AttributeValue::DebugMacroRef(val), + read::AttributeValue::LocationListsRef(val) => { + let iter = context + .dwarf + .locations + .raw_locations(val, context.unit.encoding())?; + let loc_list = LocationList::from(iter, context)?; + let loc_id = context.locations.add(loc_list); + AttributeValue::LocationListRef(loc_id) + } + read::AttributeValue::DebugLocListsBase(_base) => { + // We convert all location list indices to offsets, + // so this is unneeded. + return Ok(None); + } + read::AttributeValue::DebugLocListsIndex(index) => { + let offset = context.dwarf.locations_offset(context.unit, index)?; + let iter = context + .dwarf + .locations + .raw_locations(offset, context.unit.encoding())?; + let loc_list = LocationList::from(iter, context)?; + let loc_id = context.locations.add(loc_list); + AttributeValue::LocationListRef(loc_id) + } + read::AttributeValue::RangeListsRef(offset) => { + let offset = context.dwarf.ranges_offset_from_raw(context.unit, offset); + let iter = context.dwarf.raw_ranges(context.unit, offset)?; + let range_list = RangeList::from(iter, context)?; + let range_id = context.ranges.add(range_list); + AttributeValue::RangeListRef(range_id) + } + read::AttributeValue::DebugRngListsBase(_base) => { + // We convert all range list indices to offsets, + // so this is unneeded. + return Ok(None); + } + read::AttributeValue::DebugRngListsIndex(index) => { + let offset = context.dwarf.ranges_offset(context.unit, index)?; + let iter = context + .dwarf + .ranges + .raw_ranges(offset, context.unit.encoding())?; + let range_list = RangeList::from(iter, context)?; + let range_id = context.ranges.add(range_list); + AttributeValue::RangeListRef(range_id) + } + read::AttributeValue::DebugTypesRef(val) => AttributeValue::DebugTypesRef(val), + read::AttributeValue::DebugStrRef(offset) => { + let r = context.dwarf.string(offset)?; + let id = context.strings.add(r.to_slice()?); + AttributeValue::StringRef(id) + } + read::AttributeValue::DebugStrRefSup(val) => AttributeValue::DebugStrRefSup(val), + read::AttributeValue::DebugStrOffsetsBase(_base) => { + // We convert all string offsets to `.debug_str` references, + // so this is unneeded. + return Ok(None); + } + read::AttributeValue::DebugStrOffsetsIndex(index) => { + let offset = context.dwarf.string_offset(context.unit, index)?; + let r = context.dwarf.string(offset)?; + let id = context.strings.add(r.to_slice()?); + AttributeValue::StringRef(id) + } + read::AttributeValue::DebugLineStrRef(offset) => { + let r = context.dwarf.line_string(offset)?; + let id = context.line_strings.add(r.to_slice()?); + AttributeValue::LineStringRef(id) + } + read::AttributeValue::String(r) => AttributeValue::String(r.to_slice()?.into()), + read::AttributeValue::Encoding(val) => AttributeValue::Encoding(val), + read::AttributeValue::DecimalSign(val) => AttributeValue::DecimalSign(val), + read::AttributeValue::Endianity(val) => AttributeValue::Endianity(val), + read::AttributeValue::Accessibility(val) => AttributeValue::Accessibility(val), + read::AttributeValue::Visibility(val) => AttributeValue::Visibility(val), + read::AttributeValue::Virtuality(val) => AttributeValue::Virtuality(val), + read::AttributeValue::Language(val) => AttributeValue::Language(val), + read::AttributeValue::AddressClass(val) => AttributeValue::AddressClass(val), + read::AttributeValue::IdentifierCase(val) => AttributeValue::IdentifierCase(val), + read::AttributeValue::CallingConvention(val) => { + AttributeValue::CallingConvention(val) + } + read::AttributeValue::Inline(val) => AttributeValue::Inline(val), + read::AttributeValue::Ordering(val) => AttributeValue::Ordering(val), + read::AttributeValue::FileIndex(val) => { + if val == 0 { + // 0 means not specified, even for version 5. + AttributeValue::FileIndex(None) + } else { + match context.line_program_files.get(val as usize) { + Some(id) => AttributeValue::FileIndex(Some(*id)), + None => return Err(ConvertError::InvalidFileIndex), + } + } + } + // Should always be a more specific section reference. + read::AttributeValue::SecOffset(_) => { + return Err(ConvertError::InvalidAttributeValue); + } + read::AttributeValue::DwoId(DwoId(val)) => AttributeValue::Udata(val), + }; + Ok(Some(to)) + } + } +} + +#[cfg(test)] +#[cfg(feature = "read")] +mod tests { + use super::*; + use crate::common::{ + DebugAddrBase, DebugLocListsBase, DebugRngListsBase, DebugStrOffsetsBase, LineEncoding, + }; + use crate::constants; + use crate::read; + use crate::write::{ + DebugLine, DebugLineStr, DebugStr, DwarfUnit, EndianVec, LineString, LineStringTable, + Location, LocationList, LocationListTable, Range, RangeList, RangeListOffsets, + RangeListTable, StringTable, + }; + use crate::LittleEndian; + use std::collections::HashMap; + use std::mem; + + #[test] + #[allow(clippy::cyclomatic_complexity)] + fn test_unit_table() { + let mut strings = StringTable::default(); + + let mut units = UnitTable::default(); + let unit_id1 = units.add(Unit::new( + Encoding { + version: 4, + address_size: 8, + format: Format::Dwarf32, + }, + LineProgram::none(), + )); + let unit2 = units.add(Unit::new( + Encoding { + version: 2, + address_size: 4, + format: Format::Dwarf64, + }, + LineProgram::none(), + )); + let unit3 = units.add(Unit::new( + Encoding { + version: 5, + address_size: 4, + format: Format::Dwarf32, + }, + LineProgram::none(), + )); + assert_eq!(units.count(), 3); + { + let unit1 = units.get_mut(unit_id1); + assert_eq!(unit1.version(), 4); + assert_eq!(unit1.address_size(), 8); + assert_eq!(unit1.format(), Format::Dwarf32); + assert_eq!(unit1.count(), 1); + + let root_id = unit1.root(); + assert_eq!(root_id, UnitEntryId::new(unit1.base_id, 0)); + { + let root = unit1.get_mut(root_id); + assert_eq!(root.id(), root_id); + assert!(root.parent().is_none()); + assert_eq!(root.tag(), constants::DW_TAG_compile_unit); + + // Test get/get_mut + assert!(root.get(constants::DW_AT_producer).is_none()); + assert!(root.get_mut(constants::DW_AT_producer).is_none()); + let mut producer = AttributeValue::String(b"root"[..].into()); + root.set(constants::DW_AT_producer, producer.clone()); + assert_eq!(root.get(constants::DW_AT_producer), Some(&producer)); + assert_eq!(root.get_mut(constants::DW_AT_producer), Some(&mut producer)); + + // Test attrs + let mut attrs = root.attrs(); + let attr = attrs.next().unwrap(); + assert_eq!(attr.name(), constants::DW_AT_producer); + assert_eq!(attr.get(), &producer); + assert!(attrs.next().is_none()); + } + + let child1 = unit1.add(root_id, constants::DW_TAG_subprogram); + assert_eq!(child1, UnitEntryId::new(unit1.base_id, 1)); + { + let child1 = unit1.get_mut(child1); + assert_eq!(child1.parent(), Some(root_id)); + + let tmp = AttributeValue::String(b"tmp"[..].into()); + child1.set(constants::DW_AT_name, tmp.clone()); + assert_eq!(child1.get(constants::DW_AT_name), Some(&tmp)); + + // Test attrs_mut + let name = AttributeValue::StringRef(strings.add(&b"child1"[..])); + { + let attr = child1.attrs_mut().next().unwrap(); + assert_eq!(attr.name(), constants::DW_AT_name); + attr.set(name.clone()); + } + assert_eq!(child1.get(constants::DW_AT_name), Some(&name)); + } + + let child2 = unit1.add(root_id, constants::DW_TAG_subprogram); + assert_eq!(child2, UnitEntryId::new(unit1.base_id, 2)); + { + let child2 = unit1.get_mut(child2); + assert_eq!(child2.parent(), Some(root_id)); + + let tmp = AttributeValue::String(b"tmp"[..].into()); + child2.set(constants::DW_AT_name, tmp.clone()); + assert_eq!(child2.get(constants::DW_AT_name), Some(&tmp)); + + // Test replace + let name = AttributeValue::StringRef(strings.add(&b"child2"[..])); + child2.set(constants::DW_AT_name, name.clone()); + assert_eq!(child2.get(constants::DW_AT_name), Some(&name)); + } + + { + let root = unit1.get(root_id); + assert_eq!( + root.children().cloned().collect::>(), + vec![child1, child2] + ); + } + } + { + let unit2 = units.get(unit2); + assert_eq!(unit2.version(), 2); + assert_eq!(unit2.address_size(), 4); + assert_eq!(unit2.format(), Format::Dwarf64); + assert_eq!(unit2.count(), 1); + + let root = unit2.root(); + assert_eq!(root, UnitEntryId::new(unit2.base_id, 0)); + let root = unit2.get(root); + assert_eq!(root.id(), UnitEntryId::new(unit2.base_id, 0)); + assert!(root.parent().is_none()); + assert_eq!(root.tag(), constants::DW_TAG_compile_unit); + } + + let mut sections = Sections::new(EndianVec::new(LittleEndian)); + let debug_line_str_offsets = DebugLineStrOffsets::none(); + let debug_str_offsets = strings.write(&mut sections.debug_str).unwrap(); + units + .write(&mut sections, &debug_line_str_offsets, &debug_str_offsets) + .unwrap(); + + println!("{:?}", sections.debug_str); + println!("{:?}", sections.debug_info); + println!("{:?}", sections.debug_abbrev); + + let dwarf = read::Dwarf { + debug_abbrev: read::DebugAbbrev::new(sections.debug_abbrev.slice(), LittleEndian), + debug_info: read::DebugInfo::new(sections.debug_info.slice(), LittleEndian), + debug_str: read::DebugStr::new(sections.debug_str.slice(), LittleEndian), + ..Default::default() + }; + let mut read_units = dwarf.units(); + + { + let read_unit1 = read_units.next().unwrap().unwrap(); + let unit1 = units.get(unit_id1); + assert_eq!(unit1.version(), read_unit1.version()); + assert_eq!(unit1.address_size(), read_unit1.address_size()); + assert_eq!(unit1.format(), read_unit1.format()); + + let read_unit1 = dwarf.unit(read_unit1).unwrap(); + let mut read_entries = read_unit1.entries(); + + let root = unit1.get(unit1.root()); + { + let (depth, read_root) = read_entries.next_dfs().unwrap().unwrap(); + assert_eq!(depth, 0); + assert_eq!(root.tag(), read_root.tag()); + assert!(read_root.has_children()); + + let producer = match root.get(constants::DW_AT_producer).unwrap() { + AttributeValue::String(ref producer) => &**producer, + otherwise => panic!("unexpected {:?}", otherwise), + }; + assert_eq!(producer, b"root"); + let read_producer = read_root + .attr_value(constants::DW_AT_producer) + .unwrap() + .unwrap(); + assert_eq!( + dwarf + .attr_string(&read_unit1, read_producer) + .unwrap() + .slice(), + producer + ); + } + + let mut children = root.children().cloned(); + + { + let child = children.next().unwrap(); + assert_eq!(child, UnitEntryId::new(unit1.base_id, 1)); + let child = unit1.get(child); + let (depth, read_child) = read_entries.next_dfs().unwrap().unwrap(); + assert_eq!(depth, 1); + assert_eq!(child.tag(), read_child.tag()); + assert!(!read_child.has_children()); + + let name = match child.get(constants::DW_AT_name).unwrap() { + AttributeValue::StringRef(name) => *name, + otherwise => panic!("unexpected {:?}", otherwise), + }; + let name = strings.get(name); + assert_eq!(name, b"child1"); + let read_name = read_child + .attr_value(constants::DW_AT_name) + .unwrap() + .unwrap(); + assert_eq!( + dwarf.attr_string(&read_unit1, read_name).unwrap().slice(), + name + ); + } + + { + let child = children.next().unwrap(); + assert_eq!(child, UnitEntryId::new(unit1.base_id, 2)); + let child = unit1.get(child); + let (depth, read_child) = read_entries.next_dfs().unwrap().unwrap(); + assert_eq!(depth, 0); + assert_eq!(child.tag(), read_child.tag()); + assert!(!read_child.has_children()); + + let name = match child.get(constants::DW_AT_name).unwrap() { + AttributeValue::StringRef(name) => *name, + otherwise => panic!("unexpected {:?}", otherwise), + }; + let name = strings.get(name); + assert_eq!(name, b"child2"); + let read_name = read_child + .attr_value(constants::DW_AT_name) + .unwrap() + .unwrap(); + assert_eq!( + dwarf.attr_string(&read_unit1, read_name).unwrap().slice(), + name + ); + } + + assert!(read_entries.next_dfs().unwrap().is_none()); + } + + { + let read_unit2 = read_units.next().unwrap().unwrap(); + let unit2 = units.get(unit2); + assert_eq!(unit2.version(), read_unit2.version()); + assert_eq!(unit2.address_size(), read_unit2.address_size()); + assert_eq!(unit2.format(), read_unit2.format()); + + let abbrevs = dwarf.abbreviations(&read_unit2).unwrap(); + let mut read_entries = read_unit2.entries(&abbrevs); + + { + let root = unit2.get(unit2.root()); + let (depth, read_root) = read_entries.next_dfs().unwrap().unwrap(); + assert_eq!(depth, 0); + assert_eq!(root.tag(), read_root.tag()); + assert!(!read_root.has_children()); + } + + assert!(read_entries.next_dfs().unwrap().is_none()); + } + + { + let read_unit3 = read_units.next().unwrap().unwrap(); + let unit3 = units.get(unit3); + assert_eq!(unit3.version(), read_unit3.version()); + assert_eq!(unit3.address_size(), read_unit3.address_size()); + assert_eq!(unit3.format(), read_unit3.format()); + + let abbrevs = dwarf.abbreviations(&read_unit3).unwrap(); + let mut read_entries = read_unit3.entries(&abbrevs); + + { + let root = unit3.get(unit3.root()); + let (depth, read_root) = read_entries.next_dfs().unwrap().unwrap(); + assert_eq!(depth, 0); + assert_eq!(root.tag(), read_root.tag()); + assert!(!read_root.has_children()); + } + + assert!(read_entries.next_dfs().unwrap().is_none()); + } + + assert!(read_units.next().unwrap().is_none()); + + let mut convert_line_strings = LineStringTable::default(); + let mut convert_strings = StringTable::default(); + let convert_units = UnitTable::from( + &dwarf, + &mut convert_line_strings, + &mut convert_strings, + &|address| Some(Address::Constant(address)), + ) + .unwrap(); + assert_eq!(convert_units.count(), units.count()); + + for i in 0..convert_units.count() { + let unit_id = units.id(i); + let unit = units.get(unit_id); + let convert_unit_id = convert_units.id(i); + let convert_unit = convert_units.get(convert_unit_id); + assert_eq!(convert_unit.version(), unit.version()); + assert_eq!(convert_unit.address_size(), unit.address_size()); + assert_eq!(convert_unit.format(), unit.format()); + assert_eq!(convert_unit.count(), unit.count()); + + let root = unit.get(unit.root()); + let convert_root = convert_unit.get(convert_unit.root()); + assert_eq!(convert_root.tag(), root.tag()); + for (convert_attr, attr) in convert_root.attrs().zip(root.attrs()) { + assert_eq!(convert_attr, attr); + } + } + } + + #[test] + fn test_attribute_value() { + // Create a string table and a string with a non-zero id/offset. + let mut strings = StringTable::default(); + strings.add("string one"); + let string_id = strings.add("string two"); + let mut debug_str = DebugStr::from(EndianVec::new(LittleEndian)); + let debug_str_offsets = strings.write(&mut debug_str).unwrap(); + let read_debug_str = read::DebugStr::new(debug_str.slice(), LittleEndian); + + let mut line_strings = LineStringTable::default(); + line_strings.add("line string one"); + let line_string_id = line_strings.add("line string two"); + let mut debug_line_str = DebugLineStr::from(EndianVec::new(LittleEndian)); + let debug_line_str_offsets = line_strings.write(&mut debug_line_str).unwrap(); + let read_debug_line_str = + read::DebugLineStr::from(read::EndianSlice::new(debug_line_str.slice(), LittleEndian)); + + let data = vec![1, 2, 3, 4]; + let read_data = read::EndianSlice::new(&[1, 2, 3, 4], LittleEndian); + + let mut expression = Expression::new(); + expression.op_constu(57); + let read_expression = read::Expression(read::EndianSlice::new( + &[constants::DW_OP_constu.0, 57], + LittleEndian, + )); + + let mut ranges = RangeListTable::default(); + let range_id = ranges.add(RangeList(vec![Range::StartEnd { + begin: Address::Constant(0x1234), + end: Address::Constant(0x2345), + }])); + + let mut locations = LocationListTable::default(); + let loc_id = locations.add(LocationList(vec![Location::StartEnd { + begin: Address::Constant(0x1234), + end: Address::Constant(0x2345), + data: expression.clone(), + }])); + + for &version in &[2, 3, 4, 5] { + for &address_size in &[4, 8] { + for &format in &[Format::Dwarf32, Format::Dwarf64] { + let encoding = Encoding { + format, + version, + address_size, + }; + + let mut sections = Sections::new(EndianVec::new(LittleEndian)); + let range_list_offsets = ranges.write(&mut sections, encoding).unwrap(); + let loc_list_offsets = locations.write(&mut sections, encoding, None).unwrap(); + + let read_debug_ranges = + read::DebugRanges::new(sections.debug_ranges.slice(), LittleEndian); + let read_debug_rnglists = + read::DebugRngLists::new(sections.debug_rnglists.slice(), LittleEndian); + + let read_debug_loc = + read::DebugLoc::new(sections.debug_loc.slice(), LittleEndian); + let read_debug_loclists = + read::DebugLocLists::new(sections.debug_loclists.slice(), LittleEndian); + + let mut units = UnitTable::default(); + let unit = units.add(Unit::new(encoding, LineProgram::none())); + let unit = units.get(unit); + let encoding = Encoding { + format, + version, + address_size, + }; + let from_unit = read::UnitHeader::new( + encoding, + 0, + read::UnitType::Compilation, + DebugAbbrevOffset(0), + DebugInfoOffset(0).into(), + read::EndianSlice::new(&[], LittleEndian), + ); + + for &(ref name, ref value, ref expect_value) in &[ + ( + constants::DW_AT_name, + AttributeValue::Address(Address::Constant(0x1234)), + read::AttributeValue::Addr(0x1234), + ), + ( + constants::DW_AT_name, + AttributeValue::Block(data.clone()), + read::AttributeValue::Block(read_data), + ), + ( + constants::DW_AT_name, + AttributeValue::Data1(0x12), + read::AttributeValue::Data1(0x12), + ), + ( + constants::DW_AT_name, + AttributeValue::Data2(0x1234), + read::AttributeValue::Data2(0x1234), + ), + ( + constants::DW_AT_name, + AttributeValue::Data4(0x1234), + read::AttributeValue::Data4(0x1234), + ), + ( + constants::DW_AT_name, + AttributeValue::Data8(0x1234), + read::AttributeValue::Data8(0x1234), + ), + ( + constants::DW_AT_name, + AttributeValue::Sdata(0x1234), + read::AttributeValue::Sdata(0x1234), + ), + ( + constants::DW_AT_name, + AttributeValue::Udata(0x1234), + read::AttributeValue::Udata(0x1234), + ), + ( + constants::DW_AT_name, + AttributeValue::Exprloc(expression.clone()), + read::AttributeValue::Exprloc(read_expression), + ), + ( + constants::DW_AT_name, + AttributeValue::Flag(false), + read::AttributeValue::Flag(false), + ), + /* + ( + constants::DW_AT_name, + AttributeValue::FlagPresent, + read::AttributeValue::Flag(true), + ), + */ + ( + constants::DW_AT_name, + AttributeValue::DebugInfoRefSup(DebugInfoOffset(0x1234)), + read::AttributeValue::DebugInfoRefSup(DebugInfoOffset(0x1234)), + ), + ( + constants::DW_AT_location, + AttributeValue::LocationListRef(loc_id), + read::AttributeValue::SecOffset(loc_list_offsets.get(loc_id).0), + ), + ( + constants::DW_AT_macro_info, + AttributeValue::DebugMacinfoRef(DebugMacinfoOffset(0x1234)), + read::AttributeValue::SecOffset(0x1234), + ), + ( + constants::DW_AT_macros, + AttributeValue::DebugMacroRef(DebugMacroOffset(0x1234)), + read::AttributeValue::SecOffset(0x1234), + ), + ( + constants::DW_AT_ranges, + AttributeValue::RangeListRef(range_id), + read::AttributeValue::SecOffset(range_list_offsets.get(range_id).0), + ), + ( + constants::DW_AT_name, + AttributeValue::DebugTypesRef(DebugTypeSignature(0x1234)), + read::AttributeValue::DebugTypesRef(DebugTypeSignature(0x1234)), + ), + ( + constants::DW_AT_name, + AttributeValue::StringRef(string_id), + read::AttributeValue::DebugStrRef(debug_str_offsets.get(string_id)), + ), + ( + constants::DW_AT_name, + AttributeValue::DebugStrRefSup(DebugStrOffset(0x1234)), + read::AttributeValue::DebugStrRefSup(DebugStrOffset(0x1234)), + ), + ( + constants::DW_AT_name, + AttributeValue::LineStringRef(line_string_id), + read::AttributeValue::DebugLineStrRef( + debug_line_str_offsets.get(line_string_id), + ), + ), + ( + constants::DW_AT_name, + AttributeValue::String(data.clone()), + read::AttributeValue::String(read_data), + ), + ( + constants::DW_AT_encoding, + AttributeValue::Encoding(constants::DwAte(0x12)), + read::AttributeValue::Udata(0x12), + ), + ( + constants::DW_AT_decimal_sign, + AttributeValue::DecimalSign(constants::DwDs(0x12)), + read::AttributeValue::Udata(0x12), + ), + ( + constants::DW_AT_endianity, + AttributeValue::Endianity(constants::DwEnd(0x12)), + read::AttributeValue::Udata(0x12), + ), + ( + constants::DW_AT_accessibility, + AttributeValue::Accessibility(constants::DwAccess(0x12)), + read::AttributeValue::Udata(0x12), + ), + ( + constants::DW_AT_visibility, + AttributeValue::Visibility(constants::DwVis(0x12)), + read::AttributeValue::Udata(0x12), + ), + ( + constants::DW_AT_virtuality, + AttributeValue::Virtuality(constants::DwVirtuality(0x12)), + read::AttributeValue::Udata(0x12), + ), + ( + constants::DW_AT_language, + AttributeValue::Language(constants::DwLang(0x12)), + read::AttributeValue::Udata(0x12), + ), + ( + constants::DW_AT_address_class, + AttributeValue::AddressClass(constants::DwAddr(0x12)), + read::AttributeValue::Udata(0x12), + ), + ( + constants::DW_AT_identifier_case, + AttributeValue::IdentifierCase(constants::DwId(0x12)), + read::AttributeValue::Udata(0x12), + ), + ( + constants::DW_AT_calling_convention, + AttributeValue::CallingConvention(constants::DwCc(0x12)), + read::AttributeValue::Udata(0x12), + ), + ( + constants::DW_AT_ordering, + AttributeValue::Ordering(constants::DwOrd(0x12)), + read::AttributeValue::Udata(0x12), + ), + ( + constants::DW_AT_inline, + AttributeValue::Inline(constants::DwInl(0x12)), + read::AttributeValue::Udata(0x12), + ), + ][..] + { + let form = value.form(encoding).unwrap(); + let attr = Attribute { + name: *name, + value: value.clone(), + }; + + let offsets = UnitOffsets::none(); + let line_program_offset = None; + let mut debug_info_refs = Vec::new(); + let mut unit_refs = Vec::new(); + let mut debug_info = DebugInfo::from(EndianVec::new(LittleEndian)); + attr.value + .write( + &mut debug_info, + &mut debug_info_refs, + &mut unit_refs, + &unit, + &offsets, + line_program_offset, + &debug_line_str_offsets, + &debug_str_offsets, + &range_list_offsets, + &loc_list_offsets, + ) + .unwrap(); + + let spec = read::AttributeSpecification::new(*name, form, None); + let mut r = read::EndianSlice::new(debug_info.slice(), LittleEndian); + let read_attr = read::parse_attribute(&mut r, encoding, spec).unwrap(); + let read_value = &read_attr.raw_value(); + // read::AttributeValue is invariant in the lifetime of R. + // The lifetimes here are all okay, so transmute it. + let read_value = unsafe { + mem::transmute::< + &read::AttributeValue>, + &read::AttributeValue>, + >(read_value) + }; + assert_eq!(read_value, expect_value); + + let dwarf = read::Dwarf { + debug_str: read_debug_str.clone(), + debug_line_str: read_debug_line_str.clone(), + ranges: read::RangeLists::new(read_debug_ranges, read_debug_rnglists), + locations: read::LocationLists::new( + read_debug_loc, + read_debug_loclists, + ), + ..Default::default() + }; + + let unit = read::Unit { + header: from_unit, + abbreviations: read::Abbreviations::default(), + name: None, + comp_dir: None, + low_pc: 0, + str_offsets_base: DebugStrOffsetsBase(0), + addr_base: DebugAddrBase(0), + loclists_base: DebugLocListsBase(0), + rnglists_base: DebugRngListsBase(0), + line_program: None, + dwo_id: None, + }; + + let mut context = convert::ConvertUnitContext { + dwarf: &dwarf, + unit: &unit, + line_strings: &mut line_strings, + strings: &mut strings, + ranges: &mut ranges, + locations: &mut locations, + convert_address: &|address| Some(Address::Constant(address)), + base_address: Address::Constant(0), + line_program_offset: None, + line_program_files: Vec::new(), + entry_ids: &HashMap::new(), + }; + + let convert_attr = + Attribute::from(&mut context, &read_attr).unwrap().unwrap(); + assert_eq!(convert_attr, attr); + } + } + } + } + } + + #[test] + #[allow(clippy::cyclomatic_complexity)] + fn test_unit_ref() { + let mut units = UnitTable::default(); + let unit_id1 = units.add(Unit::new( + Encoding { + version: 4, + address_size: 8, + format: Format::Dwarf32, + }, + LineProgram::none(), + )); + assert_eq!(unit_id1, units.id(0)); + let unit_id2 = units.add(Unit::new( + Encoding { + version: 2, + address_size: 4, + format: Format::Dwarf64, + }, + LineProgram::none(), + )); + assert_eq!(unit_id2, units.id(1)); + let unit1_child1 = UnitEntryId::new(units.get(unit_id1).base_id, 1); + let unit1_child2 = UnitEntryId::new(units.get(unit_id1).base_id, 2); + let unit2_child1 = UnitEntryId::new(units.get(unit_id2).base_id, 1); + let unit2_child2 = UnitEntryId::new(units.get(unit_id2).base_id, 2); + { + let unit1 = units.get_mut(unit_id1); + let root = unit1.root(); + let child_id1 = unit1.add(root, constants::DW_TAG_subprogram); + assert_eq!(child_id1, unit1_child1); + let child_id2 = unit1.add(root, constants::DW_TAG_subprogram); + assert_eq!(child_id2, unit1_child2); + { + let child1 = unit1.get_mut(child_id1); + child1.set(constants::DW_AT_type, AttributeValue::UnitRef(child_id2)); + } + { + let child2 = unit1.get_mut(child_id2); + child2.set( + constants::DW_AT_type, + AttributeValue::DebugInfoRef(Reference::Entry(unit_id2, unit2_child1)), + ); + } + } + { + let unit2 = units.get_mut(unit_id2); + let root = unit2.root(); + let child_id1 = unit2.add(root, constants::DW_TAG_subprogram); + assert_eq!(child_id1, unit2_child1); + let child_id2 = unit2.add(root, constants::DW_TAG_subprogram); + assert_eq!(child_id2, unit2_child2); + { + let child1 = unit2.get_mut(child_id1); + child1.set(constants::DW_AT_type, AttributeValue::UnitRef(child_id2)); + } + { + let child2 = unit2.get_mut(child_id2); + child2.set( + constants::DW_AT_type, + AttributeValue::DebugInfoRef(Reference::Entry(unit_id1, unit1_child1)), + ); + } + } + + let debug_line_str_offsets = DebugLineStrOffsets::none(); + let debug_str_offsets = DebugStrOffsets::none(); + let mut sections = Sections::new(EndianVec::new(LittleEndian)); + let debug_info_offsets = units + .write(&mut sections, &debug_line_str_offsets, &debug_str_offsets) + .unwrap(); + + println!("{:?}", sections.debug_info); + println!("{:?}", sections.debug_abbrev); + + let dwarf = read::Dwarf { + debug_abbrev: read::DebugAbbrev::new(sections.debug_abbrev.slice(), LittleEndian), + debug_info: read::DebugInfo::new(sections.debug_info.slice(), LittleEndian), + ..Default::default() + }; + + let mut read_units = dwarf.units(); + { + let read_unit1 = read_units.next().unwrap().unwrap(); + assert_eq!( + read_unit1.offset(), + debug_info_offsets.unit(unit_id1).into() + ); + + let abbrevs = dwarf.abbreviations(&read_unit1).unwrap(); + let mut read_entries = read_unit1.entries(&abbrevs); + { + let (_, _read_root) = read_entries.next_dfs().unwrap().unwrap(); + } + { + let (_, read_child1) = read_entries.next_dfs().unwrap().unwrap(); + let offset = debug_info_offsets + .entry(unit_id1, unit1_child2) + .to_unit_offset(&read_unit1) + .unwrap(); + assert_eq!( + read_child1.attr_value(constants::DW_AT_type).unwrap(), + Some(read::AttributeValue::UnitRef(offset)) + ); + } + { + let (_, read_child2) = read_entries.next_dfs().unwrap().unwrap(); + let offset = debug_info_offsets.entry(unit_id2, unit2_child1); + assert_eq!( + read_child2.attr_value(constants::DW_AT_type).unwrap(), + Some(read::AttributeValue::DebugInfoRef(offset)) + ); + } + } + { + let read_unit2 = read_units.next().unwrap().unwrap(); + assert_eq!( + read_unit2.offset(), + debug_info_offsets.unit(unit_id2).into() + ); + + let abbrevs = dwarf.abbreviations(&read_unit2).unwrap(); + let mut read_entries = read_unit2.entries(&abbrevs); + { + let (_, _read_root) = read_entries.next_dfs().unwrap().unwrap(); + } + { + let (_, read_child1) = read_entries.next_dfs().unwrap().unwrap(); + let offset = debug_info_offsets + .entry(unit_id2, unit2_child2) + .to_unit_offset(&read_unit2) + .unwrap(); + assert_eq!( + read_child1.attr_value(constants::DW_AT_type).unwrap(), + Some(read::AttributeValue::UnitRef(offset)) + ); + } + { + let (_, read_child2) = read_entries.next_dfs().unwrap().unwrap(); + let offset = debug_info_offsets.entry(unit_id1, unit1_child1); + assert_eq!( + read_child2.attr_value(constants::DW_AT_type).unwrap(), + Some(read::AttributeValue::DebugInfoRef(offset)) + ); + } + } + + let mut convert_line_strings = LineStringTable::default(); + let mut convert_strings = StringTable::default(); + let convert_units = UnitTable::from( + &dwarf, + &mut convert_line_strings, + &mut convert_strings, + &|address| Some(Address::Constant(address)), + ) + .unwrap(); + assert_eq!(convert_units.count(), units.count()); + + for i in 0..convert_units.count() { + let unit = units.get(units.id(i)); + let convert_unit = convert_units.get(convert_units.id(i)); + assert_eq!(convert_unit.version(), unit.version()); + assert_eq!(convert_unit.address_size(), unit.address_size()); + assert_eq!(convert_unit.format(), unit.format()); + assert_eq!(convert_unit.count(), unit.count()); + + let root = unit.get(unit.root()); + let convert_root = convert_unit.get(convert_unit.root()); + assert_eq!(convert_root.tag(), root.tag()); + for (convert_attr, attr) in convert_root.attrs().zip(root.attrs()) { + assert_eq!(convert_attr, attr); + } + + let child1 = unit.get(UnitEntryId::new(unit.base_id, 1)); + let convert_child1 = convert_unit.get(UnitEntryId::new(convert_unit.base_id, 1)); + assert_eq!(convert_child1.tag(), child1.tag()); + for (convert_attr, attr) in convert_child1.attrs().zip(child1.attrs()) { + assert_eq!(convert_attr.name, attr.name); + match (convert_attr.value.clone(), attr.value.clone()) { + ( + AttributeValue::DebugInfoRef(Reference::Entry(convert_unit, convert_entry)), + AttributeValue::DebugInfoRef(Reference::Entry(unit, entry)), + ) => { + assert_eq!(convert_unit.index, unit.index); + assert_eq!(convert_entry.index, entry.index); + } + (AttributeValue::UnitRef(convert_id), AttributeValue::UnitRef(id)) => { + assert_eq!(convert_id.index, id.index); + } + (convert_value, value) => assert_eq!(convert_value, value), + } + } + + let child2 = unit.get(UnitEntryId::new(unit.base_id, 2)); + let convert_child2 = convert_unit.get(UnitEntryId::new(convert_unit.base_id, 2)); + assert_eq!(convert_child2.tag(), child2.tag()); + for (convert_attr, attr) in convert_child2.attrs().zip(child2.attrs()) { + assert_eq!(convert_attr.name, attr.name); + match (convert_attr.value.clone(), attr.value.clone()) { + ( + AttributeValue::DebugInfoRef(Reference::Entry(convert_unit, convert_entry)), + AttributeValue::DebugInfoRef(Reference::Entry(unit, entry)), + ) => { + assert_eq!(convert_unit.index, unit.index); + assert_eq!(convert_entry.index, entry.index); + } + (AttributeValue::UnitRef(convert_id), AttributeValue::UnitRef(id)) => { + assert_eq!(convert_id.index, id.index); + } + (convert_value, value) => assert_eq!(convert_value, value), + } + } + } + } + + #[test] + fn test_sibling() { + fn add_child( + unit: &mut Unit, + parent: UnitEntryId, + tag: constants::DwTag, + name: &str, + ) -> UnitEntryId { + let id = unit.add(parent, tag); + let child = unit.get_mut(id); + child.set(constants::DW_AT_name, AttributeValue::String(name.into())); + child.set_sibling(true); + id + } + + fn add_children(units: &mut UnitTable, unit_id: UnitId) { + let unit = units.get_mut(unit_id); + let root = unit.root(); + let child1 = add_child(unit, root, constants::DW_TAG_subprogram, "child1"); + add_child(unit, child1, constants::DW_TAG_variable, "grandchild1"); + add_child(unit, root, constants::DW_TAG_subprogram, "child2"); + add_child(unit, root, constants::DW_TAG_subprogram, "child3"); + } + + fn next_child>( + entries: &mut read::EntriesCursor, + ) -> (read::UnitOffset, Option) { + let (_, entry) = entries.next_dfs().unwrap().unwrap(); + let offset = entry.offset(); + let sibling = + entry + .attr_value(constants::DW_AT_sibling) + .unwrap() + .map(|attr| match attr { + read::AttributeValue::UnitRef(offset) => offset, + _ => panic!("bad sibling value"), + }); + (offset, sibling) + } + + fn check_sibling>( + unit: &read::UnitHeader, + debug_abbrev: &read::DebugAbbrev, + ) { + let abbrevs = unit.abbreviations(debug_abbrev).unwrap(); + let mut entries = unit.entries(&abbrevs); + // root + entries.next_dfs().unwrap().unwrap(); + // child1 + let (_, sibling1) = next_child(&mut entries); + // grandchild1 + entries.next_dfs().unwrap().unwrap(); + // child2 + let (offset2, sibling2) = next_child(&mut entries); + // child3 + let (_, _) = next_child(&mut entries); + assert_eq!(sibling1, Some(offset2)); + assert_eq!(sibling2, None); + } + + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 8, + }; + let mut units = UnitTable::default(); + let unit_id1 = units.add(Unit::new(encoding, LineProgram::none())); + add_children(&mut units, unit_id1); + let unit_id2 = units.add(Unit::new(encoding, LineProgram::none())); + add_children(&mut units, unit_id2); + + let debug_line_str_offsets = DebugLineStrOffsets::none(); + let debug_str_offsets = DebugStrOffsets::none(); + let mut sections = Sections::new(EndianVec::new(LittleEndian)); + units + .write(&mut sections, &debug_line_str_offsets, &debug_str_offsets) + .unwrap(); + + println!("{:?}", sections.debug_info); + println!("{:?}", sections.debug_abbrev); + + let read_debug_info = read::DebugInfo::new(sections.debug_info.slice(), LittleEndian); + let read_debug_abbrev = read::DebugAbbrev::new(sections.debug_abbrev.slice(), LittleEndian); + let mut read_units = read_debug_info.units(); + check_sibling(&read_units.next().unwrap().unwrap(), &read_debug_abbrev); + check_sibling(&read_units.next().unwrap().unwrap(), &read_debug_abbrev); + } + + #[test] + fn test_line_ref() { + for &version in &[2, 3, 4, 5] { + for &address_size in &[4, 8] { + for &format in &[Format::Dwarf32, Format::Dwarf64] { + let encoding = Encoding { + format, + version, + address_size, + }; + + // The line program we'll be referencing. + let mut line_program = LineProgram::new( + encoding, + LineEncoding::default(), + LineString::String(b"comp_dir".to_vec()), + LineString::String(b"comp_name".to_vec()), + None, + ); + let dir = line_program.default_directory(); + let file1 = + line_program.add_file(LineString::String(b"file1".to_vec()), dir, None); + let file2 = + line_program.add_file(LineString::String(b"file2".to_vec()), dir, None); + + // Write, read, and convert the line program, so that we have the info + // required to convert the attributes. + let line_strings = DebugLineStrOffsets::none(); + let strings = DebugStrOffsets::none(); + let mut debug_line = DebugLine::from(EndianVec::new(LittleEndian)); + let line_program_offset = line_program + .write(&mut debug_line, encoding, &line_strings, &strings) + .unwrap(); + let read_debug_line = read::DebugLine::new(debug_line.slice(), LittleEndian); + let read_line_program = read_debug_line + .program( + line_program_offset, + address_size, + Some(read::EndianSlice::new(b"comp_dir", LittleEndian)), + Some(read::EndianSlice::new(b"comp_name", LittleEndian)), + ) + .unwrap(); + let dwarf = read::Dwarf::default(); + let mut convert_line_strings = LineStringTable::default(); + let mut convert_strings = StringTable::default(); + let (_, line_program_files) = LineProgram::from( + read_line_program, + &dwarf, + &mut convert_line_strings, + &mut convert_strings, + &|address| Some(Address::Constant(address)), + ) + .unwrap(); + + // Fake the unit. + let mut units = UnitTable::default(); + let unit = units.add(Unit::new(encoding, LineProgram::none())); + let unit = units.get(unit); + let from_unit = read::UnitHeader::new( + encoding, + 0, + read::UnitType::Compilation, + DebugAbbrevOffset(0), + DebugInfoOffset(0).into(), + read::EndianSlice::new(&[], LittleEndian), + ); + + for &(ref name, ref value, ref expect_value) in &[ + ( + constants::DW_AT_stmt_list, + AttributeValue::LineProgramRef, + read::AttributeValue::SecOffset(line_program_offset.0), + ), + ( + constants::DW_AT_decl_file, + AttributeValue::FileIndex(Some(file1)), + read::AttributeValue::Udata(file1.raw()), + ), + ( + constants::DW_AT_decl_file, + AttributeValue::FileIndex(Some(file2)), + read::AttributeValue::Udata(file2.raw()), + ), + ][..] + { + let mut ranges = RangeListTable::default(); + let mut locations = LocationListTable::default(); + let mut strings = StringTable::default(); + let mut line_strings = LineStringTable::default(); + + let form = value.form(encoding).unwrap(); + let attr = Attribute { + name: *name, + value: value.clone(), + }; + + let mut debug_info_refs = Vec::new(); + let mut unit_refs = Vec::new(); + let mut debug_info = DebugInfo::from(EndianVec::new(LittleEndian)); + let offsets = UnitOffsets::none(); + let debug_line_str_offsets = DebugLineStrOffsets::none(); + let debug_str_offsets = DebugStrOffsets::none(); + let range_list_offsets = RangeListOffsets::none(); + let loc_list_offsets = LocationListOffsets::none(); + attr.value + .write( + &mut debug_info, + &mut debug_info_refs, + &mut unit_refs, + &unit, + &offsets, + Some(line_program_offset), + &debug_line_str_offsets, + &debug_str_offsets, + &range_list_offsets, + &loc_list_offsets, + ) + .unwrap(); + + let spec = read::AttributeSpecification::new(*name, form, None); + let mut r = read::EndianSlice::new(debug_info.slice(), LittleEndian); + let read_attr = read::parse_attribute(&mut r, encoding, spec).unwrap(); + let read_value = &read_attr.raw_value(); + // read::AttributeValue is invariant in the lifetime of R. + // The lifetimes here are all okay, so transmute it. + let read_value = unsafe { + mem::transmute::< + &read::AttributeValue>, + &read::AttributeValue>, + >(read_value) + }; + assert_eq!(read_value, expect_value); + + let unit = read::Unit { + header: from_unit, + abbreviations: read::Abbreviations::default(), + name: None, + comp_dir: None, + low_pc: 0, + str_offsets_base: DebugStrOffsetsBase(0), + addr_base: DebugAddrBase(0), + loclists_base: DebugLocListsBase(0), + rnglists_base: DebugRngListsBase(0), + line_program: None, + dwo_id: None, + }; + + let mut context = convert::ConvertUnitContext { + dwarf: &dwarf, + unit: &unit, + line_strings: &mut line_strings, + strings: &mut strings, + ranges: &mut ranges, + locations: &mut locations, + convert_address: &|address| Some(Address::Constant(address)), + base_address: Address::Constant(0), + line_program_offset: Some(line_program_offset), + line_program_files: line_program_files.clone(), + entry_ids: &HashMap::new(), + }; + + let convert_attr = + Attribute::from(&mut context, &read_attr).unwrap().unwrap(); + assert_eq!(convert_attr, attr); + } + } + } + } + } + + #[test] + fn test_line_program_used() { + for used in vec![false, true] { + let encoding = Encoding { + format: Format::Dwarf32, + version: 5, + address_size: 8, + }; + + let line_program = LineProgram::new( + encoding, + LineEncoding::default(), + LineString::String(b"comp_dir".to_vec()), + LineString::String(b"comp_name".to_vec()), + None, + ); + + let mut unit = Unit::new(encoding, line_program); + let file_id = if used { Some(FileId::new(0)) } else { None }; + let root = unit.root(); + unit.get_mut(root).set( + constants::DW_AT_decl_file, + AttributeValue::FileIndex(file_id), + ); + + let mut units = UnitTable::default(); + units.add(unit); + + let debug_line_str_offsets = DebugLineStrOffsets::none(); + let debug_str_offsets = DebugStrOffsets::none(); + let mut sections = Sections::new(EndianVec::new(LittleEndian)); + units + .write(&mut sections, &debug_line_str_offsets, &debug_str_offsets) + .unwrap(); + assert_eq!(!used, sections.debug_line.slice().is_empty()); + } + } + + #[test] + fn test_delete_child() { + fn set_name(unit: &mut Unit, id: UnitEntryId, name: &str) { + let entry = unit.get_mut(id); + entry.set(constants::DW_AT_name, AttributeValue::String(name.into())); + } + fn check_name( + entry: &read::DebuggingInformationEntry, + debug_str: &read::DebugStr, + name: &str, + ) { + let name_attr = entry.attr(constants::DW_AT_name).unwrap().unwrap(); + let entry_name = name_attr.string_value(debug_str).unwrap(); + let entry_name_str = entry_name.to_string().unwrap(); + assert_eq!(entry_name_str, name); + } + let encoding = Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 8, + }; + let mut dwarf = DwarfUnit::new(encoding); + let root = dwarf.unit.root(); + + // Add and delete entries in the root unit + let child1 = dwarf.unit.add(root, constants::DW_TAG_subprogram); + set_name(&mut dwarf.unit, child1, "child1"); + let grandchild1 = dwarf.unit.add(child1, constants::DW_TAG_variable); + set_name(&mut dwarf.unit, grandchild1, "grandchild1"); + let child2 = dwarf.unit.add(root, constants::DW_TAG_subprogram); + set_name(&mut dwarf.unit, child2, "child2"); + // This deletes both `child1` and its child `grandchild1` + dwarf.unit.get_mut(root).delete_child(child1); + let child3 = dwarf.unit.add(root, constants::DW_TAG_subprogram); + set_name(&mut dwarf.unit, child3, "child3"); + let child4 = dwarf.unit.add(root, constants::DW_TAG_subprogram); + set_name(&mut dwarf.unit, child4, "child4"); + let grandchild4 = dwarf.unit.add(child4, constants::DW_TAG_variable); + set_name(&mut dwarf.unit, grandchild4, "grandchild4"); + dwarf.unit.get_mut(child4).delete_child(grandchild4); + + let mut sections = Sections::new(EndianVec::new(LittleEndian)); + + // Write DWARF data which should only include `child2`, `child3` and `child4` + dwarf.write(&mut sections).unwrap(); + + let read_debug_info = read::DebugInfo::new(sections.debug_info.slice(), LittleEndian); + let read_debug_abbrev = read::DebugAbbrev::new(sections.debug_abbrev.slice(), LittleEndian); + let read_debug_str = read::DebugStr::new(sections.debug_str.slice(), LittleEndian); + let read_unit = read_debug_info.units().next().unwrap().unwrap(); + let abbrevs = read_unit.abbreviations(&read_debug_abbrev).unwrap(); + let mut entries = read_unit.entries(&abbrevs); + // root + entries.next_dfs().unwrap().unwrap(); + // child2 + let (_, read_child2) = entries.next_dfs().unwrap().unwrap(); + check_name(read_child2, &read_debug_str, "child2"); + // child3 + let (_, read_child3) = entries.next_dfs().unwrap().unwrap(); + check_name(read_child3, &read_debug_str, "child3"); + // child4 + let (_, read_child4) = entries.next_dfs().unwrap().unwrap(); + check_name(read_child4, &read_debug_str, "child4"); + // There should be no more entries + assert!(entries.next_dfs().unwrap().is_none()); + } +} diff --git a/crux-mir/lib/gimli/src/write/writer.rs b/crux-mir/lib/gimli/src/write/writer.rs new file mode 100644 index 000000000..0785d1686 --- /dev/null +++ b/crux-mir/lib/gimli/src/write/writer.rs @@ -0,0 +1,497 @@ +use crate::common::{Format, SectionId}; +use crate::constants; +use crate::endianity::Endianity; +use crate::leb128; +use crate::write::{Address, Error, Result}; + +/// A trait for writing the data to a DWARF section. +/// +/// All write operations append to the section unless otherwise specified. +#[allow(clippy::len_without_is_empty)] +pub trait Writer { + /// The endianity of bytes that are written. + type Endian: Endianity; + + /// Return the endianity of bytes that are written. + fn endian(&self) -> Self::Endian; + + /// Return the current section length. + /// + /// This may be used as an offset for future `write_at` calls. + fn len(&self) -> usize; + + /// Write a slice. + fn write(&mut self, bytes: &[u8]) -> Result<()>; + + /// Write a slice at a given offset. + /// + /// The write must not extend past the current section length. + fn write_at(&mut self, offset: usize, bytes: &[u8]) -> Result<()>; + + /// Write an address. + /// + /// If the writer supports relocations, then it must provide its own implementation + /// of this method. + // TODO: use write_reference instead? + fn write_address(&mut self, address: Address, size: u8) -> Result<()> { + match address { + Address::Constant(val) => self.write_udata(val, size), + Address::Symbol { .. } => Err(Error::InvalidAddress), + } + } + + /// Write an address with a `.eh_frame` pointer encoding. + /// + /// The given size is only used for `DW_EH_PE_absptr` formats. + /// + /// If the writer supports relocations, then it must provide its own implementation + /// of this method. + fn write_eh_pointer( + &mut self, + address: Address, + eh_pe: constants::DwEhPe, + size: u8, + ) -> Result<()> { + match address { + Address::Constant(val) => { + // Indirect doesn't matter here. + let val = match eh_pe.application() { + constants::DW_EH_PE_absptr => val, + constants::DW_EH_PE_pcrel => { + // TODO: better handling of sign + let offset = self.len() as u64; + val.wrapping_sub(offset) + } + _ => { + return Err(Error::UnsupportedPointerEncoding(eh_pe)); + } + }; + self.write_eh_pointer_data(val, eh_pe.format(), size) + } + Address::Symbol { .. } => Err(Error::InvalidAddress), + } + } + + /// Write a value with a `.eh_frame` pointer format. + /// + /// The given size is only used for `DW_EH_PE_absptr` formats. + /// + /// This must not be used directly for values that may require relocation. + fn write_eh_pointer_data( + &mut self, + val: u64, + format: constants::DwEhPe, + size: u8, + ) -> Result<()> { + match format { + constants::DW_EH_PE_absptr => self.write_udata(val, size), + constants::DW_EH_PE_uleb128 => self.write_uleb128(val), + constants::DW_EH_PE_udata2 => self.write_udata(val, 2), + constants::DW_EH_PE_udata4 => self.write_udata(val, 4), + constants::DW_EH_PE_udata8 => self.write_udata(val, 8), + constants::DW_EH_PE_sleb128 => self.write_sleb128(val as i64), + constants::DW_EH_PE_sdata2 => self.write_sdata(val as i64, 2), + constants::DW_EH_PE_sdata4 => self.write_sdata(val as i64, 4), + constants::DW_EH_PE_sdata8 => self.write_sdata(val as i64, 8), + _ => { + return Err(Error::UnsupportedPointerEncoding(format)); + } + } + } + + /// Write an offset that is relative to the start of the given section. + /// + /// If the writer supports relocations, then it must provide its own implementation + /// of this method. + fn write_offset(&mut self, val: usize, _section: SectionId, size: u8) -> Result<()> { + self.write_udata(val as u64, size) + } + + /// Write an offset that is relative to the start of the given section. + /// + /// If the writer supports relocations, then it must provide its own implementation + /// of this method. + fn write_offset_at( + &mut self, + offset: usize, + val: usize, + _section: SectionId, + size: u8, + ) -> Result<()> { + self.write_udata_at(offset, val as u64, size) + } + + /// Write a reference to a symbol. + /// + /// If the writer supports symbols, then it must provide its own implementation + /// of this method. + fn write_reference(&mut self, _symbol: usize, _size: u8) -> Result<()> { + Err(Error::InvalidReference) + } + + /// Write a u8. + fn write_u8(&mut self, val: u8) -> Result<()> { + let bytes = [val]; + self.write(&bytes) + } + + /// Write a u16. + fn write_u16(&mut self, val: u16) -> Result<()> { + let mut bytes = [0; 2]; + self.endian().write_u16(&mut bytes, val); + self.write(&bytes) + } + + /// Write a u32. + fn write_u32(&mut self, val: u32) -> Result<()> { + let mut bytes = [0; 4]; + self.endian().write_u32(&mut bytes, val); + self.write(&bytes) + } + + /// Write a u64. + fn write_u64(&mut self, val: u64) -> Result<()> { + let mut bytes = [0; 8]; + self.endian().write_u64(&mut bytes, val); + self.write(&bytes) + } + + /// Write a u8 at the given offset. + fn write_u8_at(&mut self, offset: usize, val: u8) -> Result<()> { + let bytes = [val]; + self.write_at(offset, &bytes) + } + + /// Write a u16 at the given offset. + fn write_u16_at(&mut self, offset: usize, val: u16) -> Result<()> { + let mut bytes = [0; 2]; + self.endian().write_u16(&mut bytes, val); + self.write_at(offset, &bytes) + } + + /// Write a u32 at the given offset. + fn write_u32_at(&mut self, offset: usize, val: u32) -> Result<()> { + let mut bytes = [0; 4]; + self.endian().write_u32(&mut bytes, val); + self.write_at(offset, &bytes) + } + + /// Write a u64 at the given offset. + fn write_u64_at(&mut self, offset: usize, val: u64) -> Result<()> { + let mut bytes = [0; 8]; + self.endian().write_u64(&mut bytes, val); + self.write_at(offset, &bytes) + } + + /// Write unsigned data of the given size. + /// + /// Returns an error if the value is too large for the size. + /// This must not be used directly for values that may require relocation. + fn write_udata(&mut self, val: u64, size: u8) -> Result<()> { + match size { + 1 => { + let write_val = val as u8; + if val != u64::from(write_val) { + return Err(Error::ValueTooLarge); + } + self.write_u8(write_val) + } + 2 => { + let write_val = val as u16; + if val != u64::from(write_val) { + return Err(Error::ValueTooLarge); + } + self.write_u16(write_val) + } + 4 => { + let write_val = val as u32; + if val != u64::from(write_val) { + return Err(Error::ValueTooLarge); + } + self.write_u32(write_val) + } + 8 => self.write_u64(val), + otherwise => Err(Error::UnsupportedWordSize(otherwise)), + } + } + + /// Write signed data of the given size. + /// + /// Returns an error if the value is too large for the size. + /// This must not be used directly for values that may require relocation. + fn write_sdata(&mut self, val: i64, size: u8) -> Result<()> { + match size { + 1 => { + let write_val = val as i8; + if val != i64::from(write_val) { + return Err(Error::ValueTooLarge); + } + self.write_u8(write_val as u8) + } + 2 => { + let write_val = val as i16; + if val != i64::from(write_val) { + return Err(Error::ValueTooLarge); + } + self.write_u16(write_val as u16) + } + 4 => { + let write_val = val as i32; + if val != i64::from(write_val) { + return Err(Error::ValueTooLarge); + } + self.write_u32(write_val as u32) + } + 8 => self.write_u64(val as u64), + otherwise => Err(Error::UnsupportedWordSize(otherwise)), + } + } + + /// Write a word of the given size at the given offset. + /// + /// Returns an error if the value is too large for the size. + /// This must not be used directly for values that may require relocation. + fn write_udata_at(&mut self, offset: usize, val: u64, size: u8) -> Result<()> { + match size { + 1 => { + let write_val = val as u8; + if val != u64::from(write_val) { + return Err(Error::ValueTooLarge); + } + self.write_u8_at(offset, write_val) + } + 2 => { + let write_val = val as u16; + if val != u64::from(write_val) { + return Err(Error::ValueTooLarge); + } + self.write_u16_at(offset, write_val) + } + 4 => { + let write_val = val as u32; + if val != u64::from(write_val) { + return Err(Error::ValueTooLarge); + } + self.write_u32_at(offset, write_val) + } + 8 => self.write_u64_at(offset, val), + otherwise => Err(Error::UnsupportedWordSize(otherwise)), + } + } + + /// Write an unsigned LEB128 encoded integer. + fn write_uleb128(&mut self, val: u64) -> Result<()> { + let mut bytes = [0u8; 10]; + // bytes is long enough so this will never fail. + let len = leb128::write::unsigned(&mut { &mut bytes[..] }, val).unwrap(); + self.write(&bytes[..len]) + } + + /// Read an unsigned LEB128 encoded integer. + fn write_sleb128(&mut self, val: i64) -> Result<()> { + let mut bytes = [0u8; 10]; + // bytes is long enough so this will never fail. + let len = leb128::write::signed(&mut { &mut bytes[..] }, val).unwrap(); + self.write(&bytes[..len]) + } + + /// Write an initial length according to the given DWARF format. + /// + /// This will only write a length of zero, since the length isn't + /// known yet, and a subsequent call to `write_initial_length_at` + /// will write the actual length. + fn write_initial_length(&mut self, format: Format) -> Result { + if format == Format::Dwarf64 { + self.write_u32(0xffff_ffff)?; + } + let offset = InitialLengthOffset(self.len()); + self.write_udata(0, format.word_size())?; + Ok(offset) + } + + /// Write an initial length at the given offset according to the given DWARF format. + /// + /// `write_initial_length` must have previously returned the offset. + fn write_initial_length_at( + &mut self, + offset: InitialLengthOffset, + length: u64, + format: Format, + ) -> Result<()> { + self.write_udata_at(offset.0, length, format.word_size()) + } +} + +/// The offset at which an initial length should be written. +#[derive(Debug, Clone, Copy)] +pub struct InitialLengthOffset(usize); + +#[cfg(test)] +mod tests { + use super::*; + use crate::write; + use crate::{BigEndian, LittleEndian}; + use std::{i64, u64}; + + #[test] + #[allow(clippy::cyclomatic_complexity)] + fn test_writer() { + let mut w = write::EndianVec::new(LittleEndian); + w.write_address(Address::Constant(0x1122_3344), 4).unwrap(); + assert_eq!(w.slice(), &[0x44, 0x33, 0x22, 0x11]); + assert_eq!( + w.write_address( + Address::Symbol { + symbol: 0, + addend: 0 + }, + 4 + ), + Err(Error::InvalidAddress) + ); + + let mut w = write::EndianVec::new(LittleEndian); + w.write_offset(0x1122_3344, SectionId::DebugInfo, 4) + .unwrap(); + assert_eq!(w.slice(), &[0x44, 0x33, 0x22, 0x11]); + w.write_offset_at(1, 0x5566, SectionId::DebugInfo, 2) + .unwrap(); + assert_eq!(w.slice(), &[0x44, 0x66, 0x55, 0x11]); + + let mut w = write::EndianVec::new(LittleEndian); + w.write_u8(0x11).unwrap(); + w.write_u16(0x2233).unwrap(); + w.write_u32(0x4455_6677).unwrap(); + w.write_u64(0x8081_8283_8485_8687).unwrap(); + #[rustfmt::skip] + assert_eq!(w.slice(), &[ + 0x11, + 0x33, 0x22, + 0x77, 0x66, 0x55, 0x44, + 0x87, 0x86, 0x85, 0x84, 0x83, 0x82, 0x81, 0x80, + ]); + w.write_u8_at(14, 0x11).unwrap(); + w.write_u16_at(12, 0x2233).unwrap(); + w.write_u32_at(8, 0x4455_6677).unwrap(); + w.write_u64_at(0, 0x8081_8283_8485_8687).unwrap(); + #[rustfmt::skip] + assert_eq!(w.slice(), &[ + 0x87, 0x86, 0x85, 0x84, 0x83, 0x82, 0x81, 0x80, + 0x77, 0x66, 0x55, 0x44, + 0x33, 0x22, + 0x11, + ]); + + let mut w = write::EndianVec::new(BigEndian); + w.write_u8(0x11).unwrap(); + w.write_u16(0x2233).unwrap(); + w.write_u32(0x4455_6677).unwrap(); + w.write_u64(0x8081_8283_8485_8687).unwrap(); + #[rustfmt::skip] + assert_eq!(w.slice(), &[ + 0x11, + 0x22, 0x33, + 0x44, 0x55, 0x66, 0x77, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + ]); + w.write_u8_at(14, 0x11).unwrap(); + w.write_u16_at(12, 0x2233).unwrap(); + w.write_u32_at(8, 0x4455_6677).unwrap(); + w.write_u64_at(0, 0x8081_8283_8485_8687).unwrap(); + #[rustfmt::skip] + assert_eq!(w.slice(), &[ + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x44, 0x55, 0x66, 0x77, + 0x22, 0x33, + 0x11, + ]); + + let mut w = write::EndianVec::new(LittleEndian); + w.write_udata(0x11, 1).unwrap(); + w.write_udata(0x2233, 2).unwrap(); + w.write_udata(0x4455_6677, 4).unwrap(); + w.write_udata(0x8081_8283_8485_8687, 8).unwrap(); + #[rustfmt::skip] + assert_eq!(w.slice(), &[ + 0x11, + 0x33, 0x22, + 0x77, 0x66, 0x55, 0x44, + 0x87, 0x86, 0x85, 0x84, 0x83, 0x82, 0x81, 0x80, + ]); + assert_eq!(w.write_udata(0x100, 1), Err(Error::ValueTooLarge)); + assert_eq!(w.write_udata(0x1_0000, 2), Err(Error::ValueTooLarge)); + assert_eq!(w.write_udata(0x1_0000_0000, 4), Err(Error::ValueTooLarge)); + assert_eq!(w.write_udata(0x00, 3), Err(Error::UnsupportedWordSize(3))); + w.write_udata_at(14, 0x11, 1).unwrap(); + w.write_udata_at(12, 0x2233, 2).unwrap(); + w.write_udata_at(8, 0x4455_6677, 4).unwrap(); + w.write_udata_at(0, 0x8081_8283_8485_8687, 8).unwrap(); + #[rustfmt::skip] + assert_eq!(w.slice(), &[ + 0x87, 0x86, 0x85, 0x84, 0x83, 0x82, 0x81, 0x80, + 0x77, 0x66, 0x55, 0x44, + 0x33, 0x22, + 0x11, + ]); + assert_eq!(w.write_udata_at(0, 0x100, 1), Err(Error::ValueTooLarge)); + assert_eq!(w.write_udata_at(0, 0x1_0000, 2), Err(Error::ValueTooLarge)); + assert_eq!( + w.write_udata_at(0, 0x1_0000_0000, 4), + Err(Error::ValueTooLarge) + ); + assert_eq!( + w.write_udata_at(0, 0x00, 3), + Err(Error::UnsupportedWordSize(3)) + ); + + let mut w = write::EndianVec::new(LittleEndian); + w.write_uleb128(0).unwrap(); + assert_eq!(w.slice(), &[0]); + + let mut w = write::EndianVec::new(LittleEndian); + w.write_uleb128(u64::MAX).unwrap(); + assert_eq!( + w.slice(), + &[0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 1] + ); + + let mut w = write::EndianVec::new(LittleEndian); + w.write_sleb128(0).unwrap(); + assert_eq!(w.slice(), &[0]); + + let mut w = write::EndianVec::new(LittleEndian); + w.write_sleb128(i64::MAX).unwrap(); + assert_eq!( + w.slice(), + &[0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0] + ); + + let mut w = write::EndianVec::new(LittleEndian); + w.write_sleb128(i64::MIN).unwrap(); + assert_eq!( + w.slice(), + &[0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x7f] + ); + + let mut w = write::EndianVec::new(LittleEndian); + let offset = w.write_initial_length(Format::Dwarf32).unwrap(); + assert_eq!(w.slice(), &[0, 0, 0, 0]); + w.write_initial_length_at(offset, 0x1122_3344, Format::Dwarf32) + .unwrap(); + assert_eq!(w.slice(), &[0x44, 0x33, 0x22, 0x11]); + assert_eq!( + w.write_initial_length_at(offset, 0x1_0000_0000, Format::Dwarf32), + Err(Error::ValueTooLarge) + ); + + let mut w = write::EndianVec::new(LittleEndian); + let offset = w.write_initial_length(Format::Dwarf64).unwrap(); + assert_eq!(w.slice(), &[0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0]); + w.write_initial_length_at(offset, 0x1122_3344_5566_7788, Format::Dwarf64) + .unwrap(); + assert_eq!( + w.slice(), + &[0xff, 0xff, 0xff, 0xff, 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11] + ); + } +} diff --git a/crux-mir/lib/gimli/tests/convert_self.rs b/crux-mir/lib/gimli/tests/convert_self.rs new file mode 100644 index 000000000..7c069ebd6 --- /dev/null +++ b/crux-mir/lib/gimli/tests/convert_self.rs @@ -0,0 +1,158 @@ +#![cfg(all(feature = "read", feature = "write"))] + +use std::env; +use std::fs::File; +use std::io::Read; +use std::path::PathBuf; + +use gimli::read; +use gimli::write::{self, Address, EndianVec}; +use gimli::LittleEndian; + +fn read_section(section: &str) -> Vec { + let mut path = PathBuf::new(); + if let Ok(dir) = env::var("CARGO_MANIFEST_DIR") { + path.push(dir); + } + path.push("fixtures/self"); + path.push(section); + + println!("Reading section \"{}\" at path {:?}", section, path); + assert!(path.is_file()); + let mut file = File::open(path).unwrap(); + + let mut buf = Vec::new(); + file.read_to_end(&mut buf).unwrap(); + buf +} + +#[test] +fn test_convert_debug_info() { + // Convert existing sections + let debug_abbrev = read_section("debug_abbrev"); + let debug_abbrev = read::DebugAbbrev::new(&debug_abbrev, LittleEndian); + + let debug_info = read_section("debug_info"); + let debug_info = read::DebugInfo::new(&debug_info, LittleEndian); + + let debug_line = read_section("debug_line"); + let debug_line = read::DebugLine::new(&debug_line, LittleEndian); + + let debug_str = read_section("debug_str"); + let debug_str = read::DebugStr::new(&debug_str, LittleEndian); + + let debug_ranges = read_section("debug_ranges"); + let debug_ranges = read::DebugRanges::new(&debug_ranges, LittleEndian); + + let debug_rnglists = read::DebugRngLists::new(&[], LittleEndian); + + let ranges = gimli::RangeLists::new(debug_ranges, debug_rnglists); + + let debug_loc = read_section("debug_loc"); + let debug_loc = read::DebugLoc::new(&debug_loc, LittleEndian); + + let debug_loclists = read::DebugLocLists::new(&[], LittleEndian); + + let locations = gimli::LocationLists::new(debug_loc, debug_loclists); + + let dwarf = read::Dwarf { + debug_abbrev, + debug_info, + debug_line, + debug_str, + ranges, + locations, + ..Default::default() + }; + + let mut dwarf = write::Dwarf::from(&dwarf, &|address| Some(Address::Constant(address))) + .expect("Should convert DWARF information"); + + assert_eq!(dwarf.units.count(), 23); + let entries: usize = (0..dwarf.units.count()) + .map(|i| dwarf.units.get(dwarf.units.id(i)).count()) + .sum(); + assert_eq!(entries, 29_560); + assert_eq!(dwarf.line_strings.count(), 0); + assert_eq!(dwarf.strings.count(), 3921); + + // Write to new sections + let mut write_sections = write::Sections::new(EndianVec::new(LittleEndian)); + dwarf + .write(&mut write_sections) + .expect("Should write DWARF information"); + let debug_info_data = write_sections.debug_info.slice(); + let debug_abbrev_data = write_sections.debug_abbrev.slice(); + let debug_line_data = write_sections.debug_line.slice(); + let debug_ranges_data = write_sections.debug_ranges.slice(); + let debug_loc_data = write_sections.debug_loc.slice(); + let debug_str_data = write_sections.debug_str.slice(); + assert_eq!(debug_info_data.len(), 394_930); + assert_eq!(debug_abbrev_data.len(), 9701); + assert_eq!(debug_line_data.len(), 105_797); + assert_eq!(debug_ranges_data.len(), 155_712); + assert_eq!(debug_loc_data.len(), 245_168); + assert_eq!(debug_str_data.len(), 144_731); + + // Convert new sections + let debug_abbrev = read::DebugAbbrev::new(debug_abbrev_data, LittleEndian); + let debug_info = read::DebugInfo::new(debug_info_data, LittleEndian); + let debug_line = read::DebugLine::new(debug_line_data, LittleEndian); + let debug_str = read::DebugStr::new(debug_str_data, LittleEndian); + let debug_ranges = read::DebugRanges::new(debug_ranges_data, LittleEndian); + let debug_rnglists = read::DebugRngLists::new(&[], LittleEndian); + let debug_loc = read::DebugLoc::new(debug_loc_data, LittleEndian); + let debug_loclists = read::DebugLocLists::new(&[], LittleEndian); + + let ranges = gimli::RangeLists::new(debug_ranges, debug_rnglists); + let locations = gimli::LocationLists::new(debug_loc, debug_loclists); + + let dwarf = read::Dwarf { + debug_abbrev, + debug_info, + debug_line, + debug_str, + ranges, + locations, + ..Default::default() + }; + + let dwarf = write::Dwarf::from(&dwarf, &|address| Some(Address::Constant(address))) + .expect("Should convert DWARF information"); + + assert_eq!(dwarf.units.count(), 23); + let entries: usize = (0..dwarf.units.count()) + .map(|i| dwarf.units.get(dwarf.units.id(i)).count()) + .sum(); + assert_eq!(entries, 29_560); + assert_eq!(dwarf.strings.count(), 3921); +} + +#[test] +fn test_convert_eh_frame() { + // Convert existing section + let eh_frame = read_section("eh_frame"); + let mut eh_frame = read::EhFrame::new(&eh_frame, LittleEndian); + // The `.eh_frame` fixture data was created on a 64-bit machine. + eh_frame.set_address_size(8); + let frames = write::FrameTable::from(&eh_frame, &|address| Some(Address::Constant(address))) + .expect("Should convert eh_frame information"); + assert_eq!(frames.cie_count(), 2); + assert_eq!(frames.fde_count(), 3482); + + // Write to new section + let mut write_eh_frame = write::EhFrame(EndianVec::new(LittleEndian)); + frames + .write_eh_frame(&mut write_eh_frame) + .expect("Should write eh_frame information"); + let eh_frame = write_eh_frame.slice(); + assert_eq!(eh_frame.len(), 147144); + + // Convert new section + let mut eh_frame = read::EhFrame::new(&eh_frame, LittleEndian); + eh_frame.set_address_size(8); + let frames = write::FrameTable::from(&eh_frame, &|address| Some(Address::Constant(address))) + .expect("Should convert eh_frame information"); + assert_eq!(frames.cie_count(), 2); + assert_eq!(frames.fde_count(), 3482); +} diff --git a/crux-mir/lib/gimli/tests/parse_self.rs b/crux-mir/lib/gimli/tests/parse_self.rs new file mode 100755 index 000000000..fb316314e --- /dev/null +++ b/crux-mir/lib/gimli/tests/parse_self.rs @@ -0,0 +1,431 @@ +#![cfg(all(feature = "read", feature = "std", feature = "endian-reader"))] + +use gimli::{ + AttributeValue, DebugAbbrev, DebugAddr, DebugAddrBase, DebugAranges, DebugInfo, DebugLine, + DebugLoc, DebugLocLists, DebugPubNames, DebugPubTypes, DebugRanges, DebugRngLists, DebugStr, + Encoding, EndianSlice, Expression, LittleEndian, LocationLists, Operation, RangeLists, + RangeListsOffset, Reader, +}; +use std::collections::hash_map::HashMap; +use std::env; +use std::fs::File; +use std::io::Read; +use std::path::PathBuf; +use std::rc::Rc; + +fn read_section(section: &str) -> Vec { + let mut path = PathBuf::new(); + if let Ok(dir) = env::var("CARGO_MANIFEST_DIR") { + path.push(dir); + } + path.push("fixtures/self"); + path.push(section); + + println!("Reading section \"{}\" at path {:?}", section, path); + assert!(path.is_file()); + let mut file = File::open(path).unwrap(); + + let mut buf = Vec::new(); + file.read_to_end(&mut buf).unwrap(); + buf +} + +fn parse_expression(expr: Expression, encoding: Encoding) { + let mut pc = expr.0.clone(); + while !pc.is_empty() { + Operation::parse(&mut pc, encoding).expect("Should parse operation"); + } + + // Also attempt to evaluate some of it. + let mut eval = expr.evaluation(encoding); + eval.set_initial_value(0); + eval.evaluate().expect("Should evaluate expression"); +} + +fn impl_parse_self_debug_info( + debug_info: &DebugInfo, + debug_abbrev: &DebugAbbrev, +) { + let mut iter = debug_info.units(); + while let Some(unit) = iter.next().expect("Should parse compilation unit") { + let abbrevs = unit + .abbreviations(&debug_abbrev) + .expect("Should parse abbreviations"); + + let mut cursor = unit.entries(&abbrevs); + + while cursor.next_dfs().expect("Should parse next dfs").is_some() { + let entry = cursor.current().expect("Should have a current entry"); + + let mut attrs = entry.attrs(); + while let Some(attr) = attrs.next().expect("Should parse entry's attribute") { + if let AttributeValue::Exprloc(expression) = attr.value() { + parse_expression(expression, unit.encoding()); + } + } + } + } +} + +#[test] +fn test_parse_self_debug_info() { + let debug_info = read_section("debug_info"); + let debug_info = DebugInfo::new(&debug_info, LittleEndian); + + let debug_abbrev = read_section("debug_abbrev"); + let debug_abbrev = DebugAbbrev::new(&debug_abbrev, LittleEndian); + + impl_parse_self_debug_info(&debug_info, &debug_abbrev); +} + +#[test] +fn test_parse_self_debug_info_with_endian_rc_slice() { + let debug_info = read_section("debug_info"); + let debug_info = Rc::from(&debug_info[..]); + let debug_info = gimli::EndianRcSlice::new(debug_info, LittleEndian); + let debug_info = DebugInfo::from(debug_info); + + let debug_abbrev = read_section("debug_abbrev"); + let debug_abbrev = Rc::from(&debug_abbrev[..]); + let debug_abbrev = gimli::EndianRcSlice::new(debug_abbrev, LittleEndian); + let debug_abbrev = DebugAbbrev::from(debug_abbrev); + + impl_parse_self_debug_info(&debug_info, &debug_abbrev); +} + +#[test] +fn test_parse_self_debug_line() { + let debug_info = read_section("debug_info"); + let debug_info = DebugInfo::new(&debug_info, LittleEndian); + + let debug_abbrev = read_section("debug_abbrev"); + let debug_abbrev = DebugAbbrev::new(&debug_abbrev, LittleEndian); + + let debug_line = read_section("debug_line"); + let debug_line = DebugLine::new(&debug_line, LittleEndian); + + let debug_str = read_section("debug_str"); + let debug_str = DebugStr::new(&debug_str, LittleEndian); + + let mut iter = debug_info.units(); + while let Some(unit) = iter.next().expect("Should parse compilation unit") { + let abbrevs = unit + .abbreviations(&debug_abbrev) + .expect("Should parse abbreviations"); + + let mut cursor = unit.entries(&abbrevs); + cursor.next_dfs().expect("Should parse next dfs"); + + let unit_entry = cursor.current().expect("Should have a root entry"); + + let comp_dir = unit_entry + .attr_value(gimli::DW_AT_comp_dir) + .expect("Should parse comp_dir attribute") + .and_then(|val| val.string_value(&debug_str)); + let comp_name = unit_entry + .attr_value(gimli::DW_AT_name) + .expect("Should parse name attribute") + .and_then(|val| val.string_value(&debug_str)); + + if let Some(AttributeValue::DebugLineRef(offset)) = unit_entry + .attr_value(gimli::DW_AT_stmt_list) + .expect("Should parse stmt_list") + { + let program = debug_line + .program(offset, unit.address_size(), comp_dir, comp_name) + .expect("should parse line number program header"); + + let mut results = Vec::new(); + let mut rows = program.rows(); + while let Some((_, row)) = rows + .next_row() + .expect("Should parse and execute all rows in the line number program") + { + results.push(*row); + } + results.reverse(); + + let program = debug_line + .program(offset, unit.address_size(), comp_dir, comp_name) + .expect("should parse line number program header"); + let (program, sequences) = program + .sequences() + .expect("should parse and execute the entire line number program"); + assert!(!sequences.is_empty()); // Should be at least one sequence. + for sequence in sequences { + let mut rows = program.resume_from(&sequence); + while let Some((_, row)) = rows + .next_row() + .expect("Should parse and execute all rows after resuming") + { + let other_row = results.pop().unwrap(); + assert!(row.address() >= sequence.start); + assert!(row.address() <= sequence.end); + assert_eq!(row.address(), other_row.address()); + assert_eq!(row.line(), other_row.line()); + } + } + assert!(results.is_empty()); + } + } +} + +#[test] +fn test_parse_self_debug_loc() { + let debug_info = read_section("debug_info"); + let debug_info = DebugInfo::new(&debug_info, LittleEndian); + + let debug_abbrev = read_section("debug_abbrev"); + let debug_abbrev = DebugAbbrev::new(&debug_abbrev, LittleEndian); + + let debug_addr = DebugAddr::from(EndianSlice::new(&[], LittleEndian)); + let debug_addr_base = DebugAddrBase(0); + + let debug_loc = read_section("debug_loc"); + let debug_loc = DebugLoc::new(&debug_loc, LittleEndian); + let debug_loclists = DebugLocLists::new(&[], LittleEndian); + let loclists = LocationLists::new(debug_loc, debug_loclists); + + let mut iter = debug_info.units(); + while let Some(unit) = iter.next().expect("Should parse compilation unit") { + let abbrevs = unit + .abbreviations(&debug_abbrev) + .expect("Should parse abbreviations"); + + let mut cursor = unit.entries(&abbrevs); + cursor.next_dfs().expect("Should parse next dfs"); + + let mut low_pc = 0; + + { + let unit_entry = cursor.current().expect("Should have a root entry"); + let low_pc_attr = unit_entry + .attr_value(gimli::DW_AT_low_pc) + .expect("Should parse low_pc"); + if let Some(gimli::AttributeValue::Addr(address)) = low_pc_attr { + low_pc = address; + } + } + + while cursor.next_dfs().expect("Should parse next dfs").is_some() { + let entry = cursor.current().expect("Should have a current entry"); + let mut attrs = entry.attrs(); + while let Some(attr) = attrs.next().expect("Should parse entry's attribute") { + if let AttributeValue::LocationListsRef(offset) = attr.value() { + let mut locs = loclists + .locations( + offset, + unit.encoding(), + low_pc, + &debug_addr, + debug_addr_base, + ) + .expect("Should parse locations OK"); + while let Some(loc) = locs.next().expect("Should parse next location") { + assert!(loc.range.begin <= loc.range.end); + parse_expression(loc.data, unit.encoding()); + } + } + } + } + } +} + +#[test] +fn test_parse_self_debug_ranges() { + let debug_info = read_section("debug_info"); + let debug_info = DebugInfo::new(&debug_info, LittleEndian); + + let debug_abbrev = read_section("debug_abbrev"); + let debug_abbrev = DebugAbbrev::new(&debug_abbrev, LittleEndian); + + let debug_addr = DebugAddr::from(EndianSlice::new(&[], LittleEndian)); + let debug_addr_base = DebugAddrBase(0); + + let debug_ranges = read_section("debug_ranges"); + let debug_ranges = DebugRanges::new(&debug_ranges, LittleEndian); + let debug_rnglists = DebugRngLists::new(&[], LittleEndian); + let rnglists = RangeLists::new(debug_ranges, debug_rnglists); + + let mut iter = debug_info.units(); + while let Some(unit) = iter.next().expect("Should parse compilation unit") { + let abbrevs = unit + .abbreviations(&debug_abbrev) + .expect("Should parse abbreviations"); + + let mut cursor = unit.entries(&abbrevs); + cursor.next_dfs().expect("Should parse next dfs"); + + let mut low_pc = 0; + + { + let unit_entry = cursor.current().expect("Should have a root entry"); + let low_pc_attr = unit_entry + .attr_value(gimli::DW_AT_low_pc) + .expect("Should parse low_pc"); + if let Some(gimli::AttributeValue::Addr(address)) = low_pc_attr { + low_pc = address; + } + } + + while cursor.next_dfs().expect("Should parse next dfs").is_some() { + let entry = cursor.current().expect("Should have a current entry"); + let mut attrs = entry.attrs(); + while let Some(attr) = attrs.next().expect("Should parse entry's attribute") { + if let AttributeValue::RangeListsRef(offset) = attr.value() { + let mut ranges = rnglists + .ranges( + RangeListsOffset(offset.0), + unit.encoding(), + low_pc, + &debug_addr, + debug_addr_base, + ) + .expect("Should parse ranges OK"); + while let Some(range) = ranges.next().expect("Should parse next range") { + assert!(range.begin <= range.end); + } + } + } + } + } +} + +#[test] +fn test_parse_self_debug_aranges() { + let debug_aranges = read_section("debug_aranges"); + let debug_aranges = DebugAranges::new(&debug_aranges, LittleEndian); + + let mut headers = debug_aranges.headers(); + while let Some(header) = headers.next().expect("Should parse arange header OK") { + let mut entries = header.entries(); + while let Some(_) = entries.next().expect("Should parse arange entry OK") { + // Not really anything else we can check right now. + } + } +} + +#[test] +fn test_parse_self_debug_pubnames() { + let debug_info = read_section("debug_info"); + let debug_info = DebugInfo::new(&debug_info, LittleEndian); + + let debug_abbrev = read_section("debug_abbrev"); + let debug_abbrev = DebugAbbrev::new(&debug_abbrev, LittleEndian); + + let debug_pubnames = read_section("debug_pubnames"); + let debug_pubnames = DebugPubNames::new(&debug_pubnames, LittleEndian); + + let mut units = HashMap::new(); + let mut abbrevs = HashMap::new(); + let mut pubnames = debug_pubnames.items(); + while let Some(entry) = pubnames.next().expect("Should parse pubname OK") { + let unit_offset = entry.unit_header_offset(); + let unit = units.entry(unit_offset).or_insert_with(|| { + debug_info + .header_from_offset(unit_offset) + .expect("Should parse unit header OK") + }); + let abbrev_offset = unit.debug_abbrev_offset(); + let abbrevs = abbrevs.entry(abbrev_offset).or_insert_with(|| { + debug_abbrev + .abbreviations(abbrev_offset) + .expect("Should parse abbreviations OK") + }); + let mut cursor = unit + .entries_at_offset(abbrevs, entry.die_offset()) + .expect("DIE offset should be valid"); + assert!(cursor.next_dfs().expect("Should parse DIE").is_some()); + } +} + +#[test] +fn test_parse_self_debug_pubtypes() { + let debug_info = read_section("debug_info"); + let debug_info = DebugInfo::new(&debug_info, LittleEndian); + + let debug_abbrev = read_section("debug_abbrev"); + let debug_abbrev = DebugAbbrev::new(&debug_abbrev, LittleEndian); + + let debug_pubtypes = read_section("debug_pubtypes"); + let debug_pubtypes = DebugPubTypes::new(&debug_pubtypes, LittleEndian); + + let mut units = HashMap::new(); + let mut abbrevs = HashMap::new(); + let mut pubtypes = debug_pubtypes.items(); + while let Some(entry) = pubtypes.next().expect("Should parse pubtype OK") { + let unit_offset = entry.unit_header_offset(); + let unit = units.entry(unit_offset).or_insert_with(|| { + debug_info + .header_from_offset(unit_offset) + .expect("Should parse unit header OK") + }); + let abbrev_offset = unit.debug_abbrev_offset(); + let abbrevs = abbrevs.entry(abbrev_offset).or_insert_with(|| { + debug_abbrev + .abbreviations(abbrev_offset) + .expect("Should parse abbreviations OK") + }); + let mut cursor = unit + .entries_at_offset(abbrevs, entry.die_offset()) + .expect("DIE offset should be valid"); + assert!(cursor.next_dfs().expect("Should parse DIE").is_some()); + } +} + +#[test] +fn test_parse_self_eh_frame() { + use gimli::{BaseAddresses, CieOrFde, EhFrame, UnwindSection}; + + let eh_frame = read_section("eh_frame"); + let mut eh_frame = EhFrame::new(&eh_frame, LittleEndian); + // The `.eh_frame` fixture data was created on a 64-bit machine. + eh_frame.set_address_size(8); + + let bases = BaseAddresses::default() + .set_eh_frame(0) + .set_text(0) + .set_got(0); + let mut entries = eh_frame.entries(&bases); + while let Some(entry) = entries.next().expect("Should parse CFI entry OK") { + match entry { + CieOrFde::Cie(cie) => { + let mut instrs = cie.instructions(&eh_frame, &bases); + while let Some(_) = instrs.next().expect("Can parse next CFI instruction OK") { + // TODO FITZGEN + } + } + CieOrFde::Fde(partial) => { + let fde = partial + .parse(UnwindSection::cie_from_offset) + .expect("Should be able to get CIE for FDE"); + + let mut instrs = fde.instructions(&eh_frame, &bases); + while let Some(_) = instrs.next().expect("Can parse next CFI instruction OK") { + // TODO FITZGEN + } + } + } + } +} + +#[test] +fn test_parse_self_eh_frame_hdr() { + use gimli::{BaseAddresses, EhFrameHdr}; + + let eh_frame_hdr = read_section("eh_frame_hdr"); + let eh_frame_hdr = EhFrameHdr::new(&eh_frame_hdr, LittleEndian); + + let bases = BaseAddresses::default() + .set_eh_frame(0) + .set_eh_frame_hdr(0) + .set_text(0) + .set_got(0); + + // `.eh_frame_hdr` was generated on a 64 bit machine. + let address_size = 8; + + let _parsed_header = eh_frame_hdr + .parse(&bases, address_size) + .expect("we can parse the `.eh_frame_hdr` section OK"); +} diff --git a/crux-mir/lib/hashbrown/.cargo-ok b/crux-mir/lib/hashbrown/.cargo-ok new file mode 100644 index 000000000..b5754e203 --- /dev/null +++ b/crux-mir/lib/hashbrown/.cargo-ok @@ -0,0 +1 @@ +ok \ No newline at end of file diff --git a/crux-mir/lib/hashbrown/.cargo_vcs_info.json b/crux-mir/lib/hashbrown/.cargo_vcs_info.json new file mode 100644 index 000000000..7e8a5bbd1 --- /dev/null +++ b/crux-mir/lib/hashbrown/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "1d2c1a81d1b53285decbd64410a21a90112613d7" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/crux-mir/lib/hashbrown/.gitignore b/crux-mir/lib/hashbrown/.gitignore new file mode 100644 index 000000000..693699042 --- /dev/null +++ b/crux-mir/lib/hashbrown/.gitignore @@ -0,0 +1,3 @@ +/target +**/*.rs.bk +Cargo.lock diff --git a/crux-mir/lib/hashbrown/CHANGELOG.md b/crux-mir/lib/hashbrown/CHANGELOG.md index 13493707a..3354b54bb 100644 --- a/crux-mir/lib/hashbrown/CHANGELOG.md +++ b/crux-mir/lib/hashbrown/CHANGELOG.md @@ -2,11 +2,191 @@ All notable changes to this project will be documented in this file. -The format is based on [Keep a Changelog](http://keepachangelog.com/) -and this project adheres to [Semantic Versioning](http://semver.org/). +The format is based on [Keep a Changelog](https://keepachangelog.com/) +and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +## [v0.12.3] - 2022-07-17 + +## Fixed + +- Fixed double-drop in `RawTable::clone_from`. (#348) + +## [v0.12.2] - 2022-07-09 + +## Added + +- Added `Entry` API for `HashSet`. (#342) +- Added `Extend<&'a (K, V)> for HashMap`. (#340) +- Added length-based short-circuiting for hash table iteration. (#338) +- Added a function to access the `RawTable` of a `HashMap`. (#335) + +## Changed + +- Edited `do_alloc` to reduce LLVM IR generated. (#341) + +## [v0.12.1] - 2022-05-02 + +## Fixed + +- Fixed underflow in `RawIterRange::size_hint`. (#325) +- Fixed the implementation of `Debug` for `ValuesMut` and `IntoValues`. (#325) + +## [v0.12.0] - 2022-01-17 + +## Added + +- Added `From<[T; N]>` and `From<[(K, V); N]>` for `HashSet` and `HashMap` respectively. (#297) +- Added an `allocator()` getter to HashMap and HashSet. (#257) +- Added `insert_unique_unchecked` to `HashMap` and `HashSet`. (#293) +- Added `into_keys` and `into_values` to HashMap. (#295) +- Implement `From` on `HashSet` and `HashMap`. (#298) +- Added `entry_ref` API to `HashMap`. (#201) + +## Changed + +- Bumped minimum Rust version to 1.56.1 and edition to 2021. +- Use u64 for the GroupWord on WebAssembly. (#271) +- Optimized `find`. (#279) +- Made rehashing and resizing less generic to reduce compilation time. (#282) +- Inlined small functions. (#283) +- Use `BuildHasher::hash_one` when `feature = "nightly"` is enabled. (#292) +- Relaxed the bounds on `Debug` for `HashSet`. (#296) +- Rename `get_each_mut` to `get_many_mut` and align API with the stdlib. (#291) +- Don't hash the key when searching in an empty table. (#305) + +## Fixed + +- Guard against allocations exceeding isize::MAX. (#268) +- Made `RawTable::insert_no_grow` unsafe. (#254) +- Inline `static_empty`. (#280) +- Fixed trait bounds on Send/Sync impls. (#303) + +## [v0.11.2] - 2021-03-25 + +## Fixed + +- Added missing allocator type parameter to `HashMap`'s and `HashSet`'s `Clone` impls. (#252) + +## [v0.11.1] - 2021-03-20 + +## Fixed + +- Added missing `pub` modifier to `BumpWrapper`. (#251) + +## [v0.11.0] - 2021-03-14 + +## Added +- Added safe `try_insert_no_grow` method to `RawTable`. (#229) +- Added support for `bumpalo` as an allocator without the `nightly` feature. (#231) +- Implemented `Default` for `RawTable`. (#237) +- Added new safe methods `RawTable::get_each_mut`, `HashMap::get_each_mut`, and + `HashMap::get_each_key_value_mut`. (#239) +- Added `From>` for `HashSet`. (#235) +- Added `try_insert` method to `HashMap`. (#247) + +## Changed +- The minimum Rust version has been bumped to 1.49.0. (#230) +- Significantly improved compilation times by reducing the amount of generated IR. (#205) + +## Removed +- We no longer re-export the unstable allocator items from the standard library, nor the stable shims approximating the same. (#227) +- Removed hasher specialization support from `aHash`, which was resulting in inconsistent hashes being generated for a key. (#248) + +## Fixed +- Fixed union length comparison. (#228) + +## ~~[v0.10.0] - 2021-01-16~~ + +This release was _yanked_ due to inconsistent hashes being generated with the `nightly` feature. (#248) + +## Changed +- Parametrized `RawTable`, `HashSet` and `HashMap` over an allocator. (#133) +- Improved branch prediction hints on stable. (#209) +- Optimized hashing of primitive types with AHash using specialization. (#207) +- Only instantiate `RawTable`'s reserve functions once per key-value. (#204) + +## [v0.9.1] - 2020-09-28 + +## Added +- Added safe methods to `RawTable` (#202): + - `get`: `find` and `as_ref` + - `get_mut`: `find` and `as_mut` + - `insert_entry`: `insert` and `as_mut` + - `remove_entry`: `find` and `remove` + - `erase_entry`: `find` and `erase` + +## Changed +- Removed `from_key_hashed_nocheck`'s `Q: Hash`. (#200) +- Made `RawTable::drain` safe. (#201) + +## [v0.9.0] - 2020-09-03 + +### Fixed +- `drain_filter` now removes and yields items that do match the predicate, + rather than items that don't. This is a **breaking change** to match the + behavior of the `drain_filter` methods in `std`. (#187) + +### Added +- Added `replace_entry_with` to `OccupiedEntry`, and `and_replace_entry_with` to `Entry`. (#190) +- Implemented `FusedIterator` and `size_hint` for `DrainFilter`. (#188) + +### Changed +- The minimum Rust version has been bumped to 1.36 (due to `crossbeam` dependency). (#193) +- Updated `ahash` dependency to 0.4. (#198) +- `HashMap::with_hasher` and `HashSet::with_hasher` are now `const fn`. (#195) +- Removed `T: Hash + Eq` and `S: BuildHasher` bounds on `HashSet::new`, + `with_capacity`, `with_hasher`, and `with_capacity_and_hasher`. (#185) + +## [v0.8.2] - 2020-08-08 + +### Changed +- Avoid closures to improve compile times. (#183) +- Do not iterate to drop if empty. (#182) + +## [v0.8.1] - 2020-07-16 + +### Added +- Added `erase` and `remove` to `RawTable`. (#171) +- Added `try_with_capacity` to `RawTable`. (#174) +- Added methods that allow re-using a `RawIter` for `RawDrain`, + `RawIntoIter`, and `RawParIter`. (#175) +- Added `reflect_remove` and `reflect_insert` to `RawIter`. (#175) +- Added a `drain_filter` function to `HashSet`. (#179) + +### Changed +- Deprecated `RawTable::erase_no_drop` in favor of `erase` and `remove`. (#176) +- `insert_no_grow` is now exposed under the `"raw"` feature. (#180) + +## [v0.8.0] - 2020-06-18 + +### Fixed +- Marked `RawTable::par_iter` as `unsafe`. (#157) + +### Changed +- Reduced the size of `HashMap`. (#159) +- No longer create tables with a capacity of 1 element. (#162) +- Removed `K: Eq + Hash` bounds on `retain`. (#163) +- Pulled in `HashMap` changes from rust-lang/rust (#164): + - `extend_one` support on nightly. + - `CollectionAllocErr` renamed to `TryReserveError`. + - Added `HashSet::get_or_insert_owned`. + - `Default` for `HashSet` no longer requires `T: Eq + Hash` and `S: BuildHasher`. + +## [v0.7.2] - 2020-04-27 + +### Added +- Added `or_insert_with_key` to `Entry`. (#152) + +### Fixed +- Partially reverted `Clone` optimization which was unsound. (#154) + +### Changed +- Disabled use of `const-random` by default, which prevented reproducible builds. (#155) +- Optimized `repeat` function. (#150) +- Use `NonNull` for buckets, which improves codegen for iterators. (#148) + ## [v0.7.1] - 2020-03-16 ### Added @@ -183,7 +363,21 @@ This release was _yanked_ due to a breaking change for users of `no-default-feat - Initial release -[Unreleased]: https://github.com/rust-lang/hashbrown/compare/v0.7.1...HEAD +[Unreleased]: https://github.com/rust-lang/hashbrown/compare/v0.12.3...HEAD +[v0.12.3]: https://github.com/rust-lang/hashbrown/compare/v0.12.2...v0.12.3 +[v0.12.2]: https://github.com/rust-lang/hashbrown/compare/v0.12.1...v0.12.2 +[v0.12.1]: https://github.com/rust-lang/hashbrown/compare/v0.12.0...v0.12.1 +[v0.12.0]: https://github.com/rust-lang/hashbrown/compare/v0.11.2...v0.12.0 +[v0.11.2]: https://github.com/rust-lang/hashbrown/compare/v0.11.1...v0.11.2 +[v0.11.1]: https://github.com/rust-lang/hashbrown/compare/v0.11.0...v0.11.1 +[v0.11.0]: https://github.com/rust-lang/hashbrown/compare/v0.10.0...v0.11.0 +[v0.10.0]: https://github.com/rust-lang/hashbrown/compare/v0.9.1...v0.10.0 +[v0.9.1]: https://github.com/rust-lang/hashbrown/compare/v0.9.0...v0.9.1 +[v0.9.0]: https://github.com/rust-lang/hashbrown/compare/v0.8.2...v0.9.0 +[v0.8.2]: https://github.com/rust-lang/hashbrown/compare/v0.8.1...v0.8.2 +[v0.8.1]: https://github.com/rust-lang/hashbrown/compare/v0.8.0...v0.8.1 +[v0.8.0]: https://github.com/rust-lang/hashbrown/compare/v0.7.2...v0.8.0 +[v0.7.2]: https://github.com/rust-lang/hashbrown/compare/v0.7.1...v0.7.2 [v0.7.1]: https://github.com/rust-lang/hashbrown/compare/v0.7.0...v0.7.1 [v0.7.0]: https://github.com/rust-lang/hashbrown/compare/v0.6.3...v0.7.0 [v0.6.3]: https://github.com/rust-lang/hashbrown/compare/v0.6.2...v0.6.3 diff --git a/crux-mir/lib/hashbrown/Cargo.toml b/crux-mir/lib/hashbrown/Cargo.toml index 4d622df0d..fb130d24d 100644 --- a/crux-mir/lib/hashbrown/Cargo.toml +++ b/crux-mir/lib/hashbrown/Cargo.toml @@ -1,58 +1,113 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + [package] +edition = "2021" +rust-version = "1.56.0" name = "hashbrown" -version = "0.7.1" +version = "0.12.3" authors = ["Amanieu d'Antras "] +exclude = [ + ".github", + "/ci/*", +] description = "A Rust port of Google's SwissTable hash map" -license = "Apache-2.0/MIT" -repository = "https://github.com/rust-lang/hashbrown" readme = "README.md" -keywords = ["hash", "no_std", "hashmap", "swisstable"] -categories = ["data-structures", "no-std"] -exclude = [".travis.yml", "bors.toml", "/ci/*"] -edition = "2018" -build = "build.rs" - -[dependencies] -# For the default hasher -ahash = { version = "0.3.2", optional = true, default-features = false } - -# For external trait impls -rayon = { version = "1.0", optional = true } -serde = { version = "1.0.25", default-features = false, optional = true } - -# When built as part of libstd -core = { version = "1.0.0", optional = true, package = "rustc-std-workspace-core" } -compiler_builtins = { version = "0.1.2", optional = true } -alloc = { version = "1.0.0", optional = true, package = "rustc-std-workspace-alloc" } - -[build-dependencies] -autocfg = "1" - -[dev-dependencies] -lazy_static = "1.2" -rand = { version = "0.7.3", features = ["small_rng"] } -rayon = "1.0" -rustc-hash = "=1.0" -serde_test = "1.0" -doc-comment = "0.3.1" +keywords = [ + "hash", + "no_std", + "hashmap", + "swisstable", +] +categories = [ + "data-structures", + "no-std", +] +license = "MIT OR Apache-2.0" +repository = "https://github.com/rust-lang/hashbrown" +resolver = "2" + +[package.metadata.docs.rs] +features = [ + "nightly", + "rayon", + "serde", + "raw", +] + +[dependencies.ahash] +version = "0.7.0" +optional = true +default-features = false + +[dependencies.alloc] +version = "1.0.0" +optional = true +package = "rustc-std-workspace-alloc" + +[dependencies.bumpalo] +version = "3.5.0" +optional = true + +[dependencies.compiler_builtins] +version = "0.1.2" +optional = true + +[dependencies.core] +version = "1.0.0" +optional = true +package = "rustc-std-workspace-core" + +[dependencies.rayon] +version = "1.0" +optional = true + +[dependencies.serde] +version = "1.0.25" +optional = true +default-features = false + +[dev-dependencies.doc-comment] +version = "0.3.1" + +[dev-dependencies.fnv] +version = "1.0.7" + +[dev-dependencies.lazy_static] +version = "1.4" + +[dev-dependencies.rand] +version = "0.8.3" +features = ["small_rng"] + +[dev-dependencies.rayon] +version = "1.0" + +[dev-dependencies.serde_test] +version = "1.0" [features] +ahash-compile-time-rng = ["ahash/compile-time-rng"] default = [ "ahash", - "ahash-compile-time-rng", "inline-more", ] - -ahash-compile-time-rng = [ "ahash/compile-time-rng" ] +inline-more = [] nightly = [] -rustc-internal-api = [] -rustc-dep-of-std = ["nightly", "core", "compiler_builtins", "alloc", "rustc-internal-api"] raw = [] - -# Enables usage of `#[inline]` on far more functions than by default in this -# crate. This may lead to a performance increase but often comes at a compile -# time cost. -inline-more = [] - -[package.metadata.docs.rs] -features = ["nightly", "rayon", "serde", "raw"] +rustc-dep-of-std = [ + "nightly", + "core", + "compiler_builtins", + "alloc", + "rustc-internal-api", +] +rustc-internal-api = [] diff --git a/crux-mir/lib/hashbrown/Cargo.toml.orig b/crux-mir/lib/hashbrown/Cargo.toml.orig new file mode 100644 index 000000000..7c1625ed5 --- /dev/null +++ b/crux-mir/lib/hashbrown/Cargo.toml.orig @@ -0,0 +1,60 @@ +[package] +name = "hashbrown" +version = "0.12.3" +authors = ["Amanieu d'Antras "] +description = "A Rust port of Google's SwissTable hash map" +license = "MIT OR Apache-2.0" +repository = "https://github.com/rust-lang/hashbrown" +readme = "README.md" +keywords = ["hash", "no_std", "hashmap", "swisstable"] +categories = ["data-structures", "no-std"] +exclude = [".github", "/ci/*"] +edition = "2021" +rust-version = "1.56.0" + +[dependencies] +# For the default hasher +ahash = { version = "0.7.0", default-features = false, optional = true } + +# For external trait impls +rayon = { version = "1.0", optional = true } +serde = { version = "1.0.25", default-features = false, optional = true } + +# When built as part of libstd +core = { version = "1.0.0", optional = true, package = "rustc-std-workspace-core" } +compiler_builtins = { version = "0.1.2", optional = true } +alloc = { version = "1.0.0", optional = true, package = "rustc-std-workspace-alloc" } + +# Optional support for bumpalo +bumpalo = { version = "3.5.0", optional = true } + +[dev-dependencies] +lazy_static = "1.4" +rand = { version = "0.8.3", features = ["small_rng"] } +rayon = "1.0" +fnv = "1.0.7" +serde_test = "1.0" +doc-comment = "0.3.1" + +[features] +default = ["ahash", "inline-more"] + +ahash-compile-time-rng = ["ahash/compile-time-rng"] +nightly = [] +rustc-internal-api = [] +rustc-dep-of-std = [ + "nightly", + "core", + "compiler_builtins", + "alloc", + "rustc-internal-api", +] +raw = [] + +# Enables usage of `#[inline]` on far more functions than by default in this +# crate. This may lead to a performance increase but often comes at a compile +# time cost. +inline-more = [] + +[package.metadata.docs.rs] +features = ["nightly", "rayon", "serde", "raw"] diff --git a/crux-mir/lib/hashbrown/README.md b/crux-mir/lib/hashbrown/README.md index bbf6e6cff..2eddcf3e2 100644 --- a/crux-mir/lib/hashbrown/README.md +++ b/crux-mir/lib/hashbrown/README.md @@ -1,9 +1,10 @@ hashbrown ========= -[![Build Status](https://travis-ci.com/rust-lang/hashbrown.svg?branch=master)](https://travis-ci.com/rust-lang/hashbrown) +[![Build Status](https://github.com/rust-lang/hashbrown/actions/workflows/rust.yml/badge.svg)](https://github.com/rust-lang/hashbrown/actions) [![Crates.io](https://img.shields.io/crates/v/hashbrown.svg)](https://crates.io/crates/hashbrown) [![Documentation](https://docs.rs/hashbrown/badge.svg)](https://docs.rs/hashbrown) +[![Rust](https://img.shields.io/badge/rust-1.56.1%2B-blue.svg?maxAge=3600)](https://github.com/rust-lang/hashbrown) This crate is a Rust port of Google's high-performance [SwissTable] hash map, adapted to make it a drop-in replacement for Rust's standard `HashMap` @@ -25,7 +26,8 @@ in environments without `std`, such as embedded systems and kernels. ## Features - Drop-in replacement for the standard library `HashMap` and `HashSet` types. -- Uses `AHash` as the default hasher, which is much faster than SipHash. +- Uses [AHash](https://github.com/tkaitchuck/aHash) as the default hasher, which is much faster than SipHash. + However, AHash does *not provide the same level of HashDoS resistance* as SipHash, so if that is important to you, you might want to consider using a different hasher. - Around 2x faster than the previous standard library `HashMap`. - Lower memory usage: only 1 byte of overhead per entry instead of 8. - Compatible with `#[no_std]` (but requires a global allocator with the `alloc` crate). @@ -36,47 +38,46 @@ in environments without `std`, such as embedded systems and kernels. Compared to the previous implementation of `std::collections::HashMap` (Rust 1.35). -With the hashbrown default AHash hasher (not HashDoS-resistant): - -```text - name oldstdhash ns/iter hashbrown ns/iter diff ns/iter diff % speedup - insert_ahash_highbits 20,846 7,397 -13,449 -64.52% x 2.82 - insert_ahash_random 20,515 7,796 -12,719 -62.00% x 2.63 - insert_ahash_serial 21,668 7,264 -14,404 -66.48% x 2.98 - insert_erase_ahash_highbits 29,570 17,498 -12,072 -40.83% x 1.69 - insert_erase_ahash_random 39,569 17,474 -22,095 -55.84% x 2.26 - insert_erase_ahash_serial 32,073 17,332 -14,741 -45.96% x 1.85 - iter_ahash_highbits 1,572 2,087 515 32.76% x 0.75 - iter_ahash_random 1,609 2,074 465 28.90% x 0.78 - iter_ahash_serial 2,293 2,120 -173 -7.54% x 1.08 - lookup_ahash_highbits 3,460 4,403 943 27.25% x 0.79 - lookup_ahash_random 6,377 3,911 -2,466 -38.67% x 1.63 - lookup_ahash_serial 3,629 3,586 -43 -1.18% x 1.01 - lookup_fail_ahash_highbits 5,286 3,411 -1,875 -35.47% x 1.55 - lookup_fail_ahash_random 12,365 4,171 -8,194 -66.27% x 2.96 - lookup_fail_ahash_serial 4,902 3,240 -1,662 -33.90% x 1.51 -``` - -With the libstd default SipHash hasher (HashDoS-resistant): - -```text - name oldstdhash ns/iter hashbrown ns/iter diff ns/iter diff % speedup - insert_std_highbits 32,598 20,199 -12,399 -38.04% x 1.61 - insert_std_random 29,824 20,760 -9,064 -30.39% x 1.44 - insert_std_serial 33,151 17,256 -15,895 -47.95% x 1.92 - insert_erase_std_highbits 74,731 48,735 -25,996 -34.79% x 1.53 - insert_erase_std_random 73,828 47,649 -26,179 -35.46% x 1.55 - insert_erase_std_serial 73,864 40,147 -33,717 -45.65% x 1.84 - iter_std_highbits 1,518 2,264 746 49.14% x 0.67 - iter_std_random 1,502 2,414 912 60.72% x 0.62 - iter_std_serial 6,361 2,118 -4,243 -66.70% x 3.00 - lookup_std_highbits 21,705 16,962 -4,743 -21.85% x 1.28 - lookup_std_random 21,654 17,158 -4,496 -20.76% x 1.26 - lookup_std_serial 18,726 14,509 -4,217 -22.52% x 1.29 - lookup_fail_std_highbits 25,852 17,323 -8,529 -32.99% x 1.49 - lookup_fail_std_random 25,913 17,760 -8,153 -31.46% x 1.46 - lookup_fail_std_serial 22,648 14,839 -7,809 -34.48% x 1.53 -``` +With the hashbrown default AHash hasher: + +| name | oldstdhash ns/iter | hashbrown ns/iter | diff ns/iter | diff % | speedup | +|:------------------------|:-------------------:|------------------:|:------------:|---------:|---------| +| insert_ahash_highbits | 18,865 | 8,020 | -10,845 | -57.49% | x 2.35 | +| insert_ahash_random | 19,711 | 8,019 | -11,692 | -59.32% | x 2.46 | +| insert_ahash_serial | 19,365 | 6,463 | -12,902 | -66.63% | x 3.00 | +| insert_erase_ahash_highbits | 51,136 | 17,916 | -33,220 | -64.96% | x 2.85 | +| insert_erase_ahash_random | 51,157 | 17,688 | -33,469 | -65.42% | x 2.89 | +| insert_erase_ahash_serial | 45,479 | 14,895 | -30,584 | -67.25% | x 3.05 | +| iter_ahash_highbits | 1,399 | 1,092 | -307 | -21.94% | x 1.28 | +| iter_ahash_random | 1,586 | 1,059 | -527 | -33.23% | x 1.50 | +| iter_ahash_serial | 3,168 | 1,079 | -2,089 | -65.94% | x 2.94 | +| lookup_ahash_highbits | 32,351 | 4,792 | -27,559 | -85.19% | x 6.75 | +| lookup_ahash_random | 17,419 | 4,817 | -12,602 | -72.35% | x 3.62 | +| lookup_ahash_serial | 15,254 | 3,606 | -11,648 | -76.36% | x 4.23 | +| lookup_fail_ahash_highbits | 21,187 | 4,369 | -16,818 | -79.38% | x 4.85 | +| lookup_fail_ahash_random | 21,550 | 4,395 | -17,155 | -79.61% | x 4.90 | +| lookup_fail_ahash_serial | 19,450 | 3,176 | -16,274 | -83.67% | x 6.12 | + + +With the libstd default SipHash hasher: + +|name | oldstdhash ns/iter | hashbrown ns/iter | diff ns/iter | diff % | speedup | +|:------------------------|:-------------------:|------------------:|:------------:|---------:|---------| +|insert_std_highbits |19,216 |16,885 | -2,331 | -12.13% | x 1.14 | +|insert_std_random |19,179 |17,034 | -2,145 | -11.18% | x 1.13 | +|insert_std_serial |19,462 |17,493 | -1,969 | -10.12% | x 1.11 | +|insert_erase_std_highbits |50,825 |35,847 | -14,978 | -29.47% | x 1.42 | +|insert_erase_std_random |51,448 |35,392 | -16,056 | -31.21% | x 1.45 | +|insert_erase_std_serial |87,711 |38,091 | -49,620 | -56.57% | x 2.30 | +|iter_std_highbits |1,378 |1,159 | -219 | -15.89% | x 1.19 | +|iter_std_random |1,395 |1,132 | -263 | -18.85% | x 1.23 | +|iter_std_serial |1,704 |1,105 | -599 | -35.15% | x 1.54 | +|lookup_std_highbits |17,195 |13,642 | -3,553 | -20.66% | x 1.26 | +|lookup_std_random |17,181 |13,773 | -3,408 | -19.84% | x 1.25 | +|lookup_std_serial |15,483 |13,651 | -1,832 | -11.83% | x 1.13 | +|lookup_fail_std_highbits |20,926 |13,474 | -7,452 | -35.61% | x 1.55 | +|lookup_fail_std_random |21,766 |13,505 | -8,261 | -37.95% | x 1.61 | +|lookup_fail_std_serial |19,336 |13,519 | -5,817 | -30.08% | x 1.43 | ## Usage @@ -84,7 +85,7 @@ Add this to your `Cargo.toml`: ```toml [dependencies] -hashbrown = "0.7" +hashbrown = "0.12" ``` Then: @@ -95,26 +96,26 @@ use hashbrown::HashMap; let mut map = HashMap::new(); map.insert(1, "one"); ``` - +## Flags This crate has the following Cargo features: -- `nightly`: Enables nightly-only features: `#[may_dangle]`. +- `nightly`: Enables nightly-only features including: `#[may_dangle]`. - `serde`: Enables serde serialization support. - `rayon`: Enables rayon parallel iterator support. - `raw`: Enables access to the experimental and unsafe `RawTable` API. - `inline-more`: Adds inline hints to most functions, improving run-time performance at the cost of compilation time. (enabled by default) +- `bumpalo`: Provides a `BumpWrapper` type which allows `bumpalo` to be used for memory allocation. - `ahash`: Compiles with ahash as default hasher. (enabled by default) -- `ahash-compile-time-rng`: Activates the `compile-time-rng` feature of ahash, to increase the - DOS-resistance, but can result in issues for `no_std` builds. More details in - [issue#124](https://github.com/rust-lang/hashbrown/issues/124). (enabled by default) +- `ahash-compile-time-rng`: Activates the `compile-time-rng` feature of ahash. For targets with no random number generator +this pre-generates seeds at compile time and embeds them as constants. See [aHash's documentation](https://github.com/tkaitchuck/aHash#flags) (disabled by default) ## License Licensed under either of: - * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) - * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) + * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or https://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([LICENSE-MIT](LICENSE-MIT) or https://opensource.org/licenses/MIT) at your option. diff --git a/crux-mir/lib/hashbrown/benches/bench.rs b/crux-mir/lib/hashbrown/benches/bench.rs index 771e7169a..c393b9a70 100644 --- a/crux-mir/lib/hashbrown/benches/bench.rs +++ b/crux-mir/lib/hashbrown/benches/bench.rs @@ -9,8 +9,11 @@ extern crate test; use test::{black_box, Bencher}; use hashbrown::hash_map::DefaultHashBuilder; -use hashbrown::HashMap; -use std::collections::hash_map::RandomState; +use hashbrown::{HashMap, HashSet}; +use std::{ + collections::hash_map::RandomState, + sync::atomic::{self, AtomicUsize}, +}; const SIZE: usize = 1000; @@ -35,11 +38,25 @@ impl Iterator for RandomKeys { type Item = usize; fn next(&mut self) -> Option { // Add 1 then multiply by some 32 bit prime. - self.state = self.state.wrapping_add(1).wrapping_mul(3787392781); + self.state = self.state.wrapping_add(1).wrapping_mul(3_787_392_781); Some(self.state) } } +// Just an arbitrary side effect to make the maps not shortcircuit to the non-dropping path +// when dropping maps/entries (most real world usages likely have drop in the key or value) +lazy_static::lazy_static! { + static ref SIDE_EFFECT: AtomicUsize = AtomicUsize::new(0); +} + +#[derive(Clone)] +struct DropType(usize); +impl Drop for DropType { + fn drop(&mut self) { + SIDE_EFFECT.fetch_add(self.0, atomic::Ordering::SeqCst); + } +} + macro_rules! bench_suite { ($bench_macro:ident, $bench_ahash_serial:ident, $bench_std_serial:ident, $bench_ahash_highbits:ident, $bench_std_highbits:ident, @@ -69,10 +86,11 @@ macro_rules! bench_insert { b.iter(|| { m.clear(); for i in ($keydist).take(SIZE) { - m.insert(i, i); + m.insert(i, (DropType(i), [i; 20])); } black_box(&mut m); - }) + }); + eprintln!("{}", SIDE_EFFECT.load(atomic::Ordering::SeqCst)); } }; } @@ -87,13 +105,38 @@ bench_suite!( insert_std_random ); +macro_rules! bench_grow_insert { + ($name:ident, $maptype:ident, $keydist:expr) => { + #[bench] + fn $name(b: &mut Bencher) { + b.iter(|| { + let mut m = $maptype::default(); + for i in ($keydist).take(SIZE) { + m.insert(i, DropType(i)); + } + black_box(&mut m); + }) + } + }; +} + +bench_suite!( + bench_grow_insert, + grow_insert_ahash_serial, + grow_insert_std_serial, + grow_insert_ahash_highbits, + grow_insert_std_highbits, + grow_insert_ahash_random, + grow_insert_std_random +); + macro_rules! bench_insert_erase { ($name:ident, $maptype:ident, $keydist:expr) => { #[bench] fn $name(b: &mut Bencher) { let mut base = $maptype::default(); for i in ($keydist).take(SIZE) { - base.insert(i, i); + base.insert(i, DropType(i)); } let skip = $keydist.skip(SIZE); b.iter(|| { @@ -103,11 +146,12 @@ macro_rules! bench_insert_erase { // While keeping the size constant, // replace the first keydist with the second. for (add, remove) in (&mut add_iter).zip(&mut remove_iter).take(SIZE) { - m.insert(add, add); + m.insert(add, DropType(add)); black_box(m.remove(&remove)); } black_box(m); - }) + }); + eprintln!("{}", SIDE_EFFECT.load(atomic::Ordering::SeqCst)); } }; } @@ -128,14 +172,15 @@ macro_rules! bench_lookup { fn $name(b: &mut Bencher) { let mut m = $maptype::default(); for i in $keydist.take(SIZE) { - m.insert(i, i); + m.insert(i, DropType(i)); } b.iter(|| { for i in $keydist.take(SIZE) { black_box(m.get(&i)); } - }) + }); + eprintln!("{}", SIDE_EFFECT.load(atomic::Ordering::SeqCst)); } }; } @@ -157,7 +202,7 @@ macro_rules! bench_lookup_fail { let mut m = $maptype::default(); let mut iter = $keydist; for i in (&mut iter).take(SIZE) { - m.insert(i, i); + m.insert(i, DropType(i)); } b.iter(|| { @@ -185,7 +230,7 @@ macro_rules! bench_iter { fn $name(b: &mut Bencher) { let mut m = $maptype::default(); for i in ($keydist).take(SIZE) { - m.insert(i, i); + m.insert(i, DropType(i)); } b.iter(|| { @@ -211,7 +256,7 @@ bench_suite!( fn clone_small(b: &mut Bencher) { let mut m = HashMap::new(); for i in 0..10 { - m.insert(i, i); + m.insert(i, DropType(i)); } b.iter(|| { @@ -224,7 +269,7 @@ fn clone_from_small(b: &mut Bencher) { let mut m = HashMap::new(); let mut m2 = HashMap::new(); for i in 0..10 { - m.insert(i, i); + m.insert(i, DropType(i)); } b.iter(|| { @@ -237,7 +282,7 @@ fn clone_from_small(b: &mut Bencher) { fn clone_large(b: &mut Bencher) { let mut m = HashMap::new(); for i in 0..1000 { - m.insert(i, i); + m.insert(i, DropType(i)); } b.iter(|| { @@ -250,7 +295,7 @@ fn clone_from_large(b: &mut Bencher) { let mut m = HashMap::new(); let mut m2 = HashMap::new(); for i in 0..1000 { - m.insert(i, i); + m.insert(i, DropType(i)); } b.iter(|| { @@ -258,3 +303,29 @@ fn clone_from_large(b: &mut Bencher) { black_box(&mut m2); }) } + +#[bench] +fn rehash_in_place(b: &mut Bencher) { + b.iter(|| { + let mut set = HashSet::new(); + + // Each loop triggers one rehash + for _ in 0..10 { + for i in 0..224 { + set.insert(i); + } + + assert_eq!( + set.capacity(), + 224, + "The set must be at or close to capacity to trigger a re hashing" + ); + + for i in 100..1400 { + set.remove(&(i - 100)); + set.insert(i); + } + set.clear(); + } + }); +} diff --git a/crux-mir/lib/hashbrown/benches/insert_unique_unchecked.rs b/crux-mir/lib/hashbrown/benches/insert_unique_unchecked.rs new file mode 100644 index 000000000..857ad18e5 --- /dev/null +++ b/crux-mir/lib/hashbrown/benches/insert_unique_unchecked.rs @@ -0,0 +1,32 @@ +//! Compare `insert` and `insert_unique_unchecked` operations performance. + +#![feature(test)] + +extern crate test; + +use hashbrown::HashMap; +use test::Bencher; + +#[bench] +fn insert(b: &mut Bencher) { + let keys: Vec = (0..1000).map(|i| format!("xxxx{}yyyy", i)).collect(); + b.iter(|| { + let mut m = HashMap::with_capacity(1000); + for k in &keys { + m.insert(k, k); + } + m + }); +} + +#[bench] +fn insert_unique_unchecked(b: &mut Bencher) { + let keys: Vec = (0..1000).map(|i| format!("xxxx{}yyyy", i)).collect(); + b.iter(|| { + let mut m = HashMap::with_capacity(1000); + for k in &keys { + m.insert_unique_unchecked(k, k); + } + m + }); +} diff --git a/crux-mir/lib/hashbrown/bors.toml b/crux-mir/lib/hashbrown/bors.toml deleted file mode 100644 index ca08e818b..000000000 --- a/crux-mir/lib/hashbrown/bors.toml +++ /dev/null @@ -1,3 +0,0 @@ -status = [ - "continuous-integration/travis-ci/push", -] diff --git a/crux-mir/lib/hashbrown/build.rs b/crux-mir/lib/hashbrown/build.rs deleted file mode 100644 index 818b20182..000000000 --- a/crux-mir/lib/hashbrown/build.rs +++ /dev/null @@ -1,9 +0,0 @@ -fn main() { - println!("cargo:rerun-if-changed=build.rs"); - let nightly = std::env::var_os("CARGO_FEATURE_NIGHTLY").is_some(); - let has_stable_alloc = || autocfg::new().probe_rustc_version(1, 36); - - if nightly || has_stable_alloc() { - autocfg::emit("has_extern_crate_alloc") - } -} diff --git a/crux-mir/lib/hashbrown/ci/miri.sh b/crux-mir/lib/hashbrown/ci/miri.sh deleted file mode 100644 index 6b95c2d97..000000000 --- a/crux-mir/lib/hashbrown/ci/miri.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env sh - -set -ex - -export CARGO_NET_RETRY=5 -export CARGO_NET_TIMEOUT=10 - -MIRI_NIGHTLY=nightly-$(curl -s https://rust-lang.github.io/rustup-components-history/x86_64-unknown-linux-gnu/miri) -echo "Installing latest nightly with Miri: $MIRI_NIGHTLY" -rustup default "$MIRI_NIGHTLY" - -rustup component add miri -cargo miri setup - -cargo miri test diff --git a/crux-mir/lib/hashbrown/ci/run.sh b/crux-mir/lib/hashbrown/ci/run.sh deleted file mode 100644 index 9bf0b0b60..000000000 --- a/crux-mir/lib/hashbrown/ci/run.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env sh - -set -ex - -: "${TARGET?The TARGET environment variable must be set.}" - -if [ "${NO_STD}" = "1" ]; then - # Unfortunately serde currently doesn't work without std due to a cargo bug. - FEATURES="rustc-internal-api" - OP="build" -else - FEATURES="rustc-internal-api,serde,rayon" - OP="test" -fi -if [ "${TRAVIS_RUST_VERSION}" = "nightly" ]; then - FEATURES="${FEATURES},nightly" - export RUSTFLAGS="$RUSTFLAGS -D warnings" -fi - -CARGO=cargo -if [ "${CROSS}" = "1" ]; then - export CARGO_NET_RETRY=5 - export CARGO_NET_TIMEOUT=10 - - cargo install --git https://github.com/rust-embedded/cross.git - CARGO=cross -fi - -# Make sure we can compile without the default hasher -"${CARGO}" -vv build --target="${TARGET}" --no-default-features -"${CARGO}" -vv build --target="${TARGET}" --release --no-default-features - -"${CARGO}" -vv ${OP} --target="${TARGET}" -"${CARGO}" -vv ${OP} --target="${TARGET}" --features "${FEATURES}" - -"${CARGO}" -vv ${OP} --target="${TARGET}" --release -"${CARGO}" -vv ${OP} --target="${TARGET}" --release --features "${FEATURES}" - -if [ "${TRAVIS_RUST_VERSION}" = "nightly" ] && [ "${NO_STD}" != 1 ]; then - # Run benchmark on native targets, build them on non-native ones: - NO_RUN="" - if [ "${CROSS}" = "1" ]; then - NO_RUN="--no-run" - fi - - "${CARGO}" -vv bench "${NO_RUN}" --features "${FEATURES}" -fi diff --git a/crux-mir/lib/hashbrown/ci/tools.sh b/crux-mir/lib/hashbrown/ci/tools.sh deleted file mode 100644 index 421958e24..000000000 --- a/crux-mir/lib/hashbrown/ci/tools.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env sh - -set -ex - -retry() { - result=0 - count=1 - max=5 - while [ "$count" -le 3 ]; do - [ "$result" -ne 0 ] && { - printf "\nRetrying, %d of %d\n" $count $max >&2 - } - "$@" - result=$? - [ $result -eq 0 ] && break - count=$((count + 1)) - sleep 1 - done - - [ "$count" -gt 3 ] && { - printf "\nFailed %d times.\n" $max >&2 - } - - return $result -} - - -if retry rustup component add rustfmt ; then - cargo fmt --all -- --check -fi - -if retry rustup component add clippy ; then - cargo clippy --all -- -D clippy::pedantic -fi - -if [ "${TRAVIS_OS_NAME}" = "linux" ]; then - if retry rustup component add clippy ; then - cargo clippy --all --target=i586-unknown-linux-gnu -- -D clippy::all -D clippy::pedantic - fi -fi - -if command -v shellcheck ; then - shellcheck --version - shellcheck ci/*.sh -fi diff --git a/crux-mir/lib/hashbrown/src/external_trait_impls/rayon/helpers.rs b/crux-mir/lib/hashbrown/src/external_trait_impls/rayon/helpers.rs index 9382007ea..070b08cd5 100644 --- a/crux-mir/lib/hashbrown/src/external_trait_impls/rayon/helpers.rs +++ b/crux-mir/lib/hashbrown/src/external_trait_impls/rayon/helpers.rs @@ -4,6 +4,7 @@ use alloc::vec::Vec; use rayon::iter::{IntoParallelIterator, ParallelIterator}; /// Helper for collecting parallel iterators to an intermediary +#[allow(clippy::linkedlist)] // yes, we need linked list here for efficient appending! pub(super) fn collect(iter: I) -> (LinkedList>, usize) { let list = iter .into_par_iter() diff --git a/crux-mir/lib/hashbrown/src/external_trait_impls/rayon/map.rs b/crux-mir/lib/hashbrown/src/external_trait_impls/rayon/map.rs index 022e23e0f..14d91c220 100644 --- a/crux-mir/lib/hashbrown/src/external_trait_impls/rayon/map.rs +++ b/crux-mir/lib/hashbrown/src/external_trait_impls/rayon/map.rs @@ -1,8 +1,11 @@ //! Rayon extensions for `HashMap`. +use super::raw::{RawIntoParIter, RawParDrain, RawParIter}; use crate::hash_map::HashMap; +use crate::raw::{Allocator, Global}; use core::fmt; use core::hash::{BuildHasher, Hash}; +use core::marker::PhantomData; use rayon::iter::plumbing::UnindexedConsumer; use rayon::iter::{FromParallelIterator, IntoParallelIterator, ParallelExtend, ParallelIterator}; @@ -15,11 +18,12 @@ use rayon::iter::{FromParallelIterator, IntoParallelIterator, ParallelExtend, Pa /// [`par_iter`]: /hashbrown/struct.HashMap.html#method.par_iter /// [`HashMap`]: /hashbrown/struct.HashMap.html /// [`IntoParallelRefIterator`]: https://docs.rs/rayon/1.0/rayon/iter/trait.IntoParallelRefIterator.html -pub struct ParIter<'a, K, V, S> { - map: &'a HashMap, +pub struct ParIter<'a, K, V> { + inner: RawParIter<(K, V)>, + marker: PhantomData<(&'a K, &'a V)>, } -impl<'a, K: Sync, V: Sync, S: Sync> ParallelIterator for ParIter<'a, K, V, S> { +impl<'a, K: Sync, V: Sync> ParallelIterator for ParIter<'a, K, V> { type Item = (&'a K, &'a V); #[cfg_attr(feature = "inline-more", inline)] @@ -27,9 +31,7 @@ impl<'a, K: Sync, V: Sync, S: Sync> ParallelIterator for ParIter<'a, K, V, S> { where C: UnindexedConsumer, { - self.map - .table - .par_iter() + self.inner .map(|x| unsafe { let r = x.as_ref(); (&r.0, &r.1) @@ -38,16 +40,23 @@ impl<'a, K: Sync, V: Sync, S: Sync> ParallelIterator for ParIter<'a, K, V, S> { } } -impl Clone for ParIter<'_, K, V, S> { +impl Clone for ParIter<'_, K, V> { #[cfg_attr(feature = "inline-more", inline)] fn clone(&self) -> Self { - ParIter { map: self.map } + Self { + inner: self.inner.clone(), + marker: PhantomData, + } } } -impl fmt::Debug for ParIter<'_, K, V, S> { +impl fmt::Debug for ParIter<'_, K, V> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.map.iter().fmt(f) + let iter = unsafe { self.inner.iter() }.map(|x| unsafe { + let r = x.as_ref(); + (&r.0, &r.1) + }); + f.debug_list().entries(iter).finish() } } @@ -58,11 +67,12 @@ impl fmt::Debug for Pa /// /// [`par_keys`]: /hashbrown/struct.HashMap.html#method.par_keys /// [`HashMap`]: /hashbrown/struct.HashMap.html -pub struct ParKeys<'a, K, V, S> { - map: &'a HashMap, +pub struct ParKeys<'a, K, V> { + inner: RawParIter<(K, V)>, + marker: PhantomData<(&'a K, &'a V)>, } -impl<'a, K: Sync, V: Sync, S: Sync> ParallelIterator for ParKeys<'a, K, V, S> { +impl<'a, K: Sync, V: Sync> ParallelIterator for ParKeys<'a, K, V> { type Item = &'a K; #[cfg_attr(feature = "inline-more", inline)] @@ -70,24 +80,26 @@ impl<'a, K: Sync, V: Sync, S: Sync> ParallelIterator for ParKeys<'a, K, V, S> { where C: UnindexedConsumer, { - self.map - .table - .par_iter() + self.inner .map(|x| unsafe { &x.as_ref().0 }) .drive_unindexed(consumer) } } -impl Clone for ParKeys<'_, K, V, S> { +impl Clone for ParKeys<'_, K, V> { #[cfg_attr(feature = "inline-more", inline)] fn clone(&self) -> Self { - ParKeys { map: self.map } + Self { + inner: self.inner.clone(), + marker: PhantomData, + } } } -impl fmt::Debug for ParKeys<'_, K, V, S> { +impl fmt::Debug for ParKeys<'_, K, V> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.map.keys().fmt(f) + let iter = unsafe { self.inner.iter() }.map(|x| unsafe { &x.as_ref().0 }); + f.debug_list().entries(iter).finish() } } @@ -98,11 +110,12 @@ impl fmt::Debug for ParKeys<'_, K, /// /// [`par_values`]: /hashbrown/struct.HashMap.html#method.par_values /// [`HashMap`]: /hashbrown/struct.HashMap.html -pub struct ParValues<'a, K, V, S> { - map: &'a HashMap, +pub struct ParValues<'a, K, V> { + inner: RawParIter<(K, V)>, + marker: PhantomData<(&'a K, &'a V)>, } -impl<'a, K: Sync, V: Sync, S: Sync> ParallelIterator for ParValues<'a, K, V, S> { +impl<'a, K: Sync, V: Sync> ParallelIterator for ParValues<'a, K, V> { type Item = &'a V; #[cfg_attr(feature = "inline-more", inline)] @@ -110,24 +123,26 @@ impl<'a, K: Sync, V: Sync, S: Sync> ParallelIterator for ParValues<'a, K, V, S> where C: UnindexedConsumer, { - self.map - .table - .par_iter() + self.inner .map(|x| unsafe { &x.as_ref().1 }) .drive_unindexed(consumer) } } -impl Clone for ParValues<'_, K, V, S> { +impl Clone for ParValues<'_, K, V> { #[cfg_attr(feature = "inline-more", inline)] fn clone(&self) -> Self { - ParValues { map: self.map } + Self { + inner: self.inner.clone(), + marker: PhantomData, + } } } -impl fmt::Debug for ParValues<'_, K, V, S> { +impl fmt::Debug for ParValues<'_, K, V> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.map.values().fmt(f) + let iter = unsafe { self.inner.iter() }.map(|x| unsafe { &x.as_ref().1 }); + f.debug_list().entries(iter).finish() } } @@ -140,11 +155,12 @@ impl fmt::Debug for ParValues<'_, K /// [`par_iter_mut`]: /hashbrown/struct.HashMap.html#method.par_iter_mut /// [`HashMap`]: /hashbrown/struct.HashMap.html /// [`IntoParallelRefMutIterator`]: https://docs.rs/rayon/1.0/rayon/iter/trait.IntoParallelRefMutIterator.html -pub struct ParIterMut<'a, K, V, S> { - map: &'a mut HashMap, +pub struct ParIterMut<'a, K, V> { + inner: RawParIter<(K, V)>, + marker: PhantomData<(&'a K, &'a mut V)>, } -impl<'a, K: Send + Sync, V: Send, S: Send> ParallelIterator for ParIterMut<'a, K, V, S> { +impl<'a, K: Sync, V: Send> ParallelIterator for ParIterMut<'a, K, V> { type Item = (&'a K, &'a mut V); #[cfg_attr(feature = "inline-more", inline)] @@ -152,9 +168,7 @@ impl<'a, K: Send + Sync, V: Send, S: Send> ParallelIterator for ParIterMut<'a, K where C: UnindexedConsumer, { - self.map - .table - .par_iter() + self.inner .map(|x| unsafe { let r = x.as_mut(); (&r.0, &mut r.1) @@ -163,11 +177,13 @@ impl<'a, K: Send + Sync, V: Send, S: Send> ParallelIterator for ParIterMut<'a, K } } -impl fmt::Debug - for ParIterMut<'_, K, V, S> -{ +impl fmt::Debug for ParIterMut<'_, K, V> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.map.iter().fmt(f) + ParIter { + inner: self.inner.clone(), + marker: PhantomData, + } + .fmt(f) } } @@ -178,11 +194,12 @@ impl fmt::Debug /// /// [`par_values_mut`]: /hashbrown/struct.HashMap.html#method.par_values_mut /// [`HashMap`]: /hashbrown/struct.HashMap.html -pub struct ParValuesMut<'a, K, V, S> { - map: &'a mut HashMap, +pub struct ParValuesMut<'a, K, V> { + inner: RawParIter<(K, V)>, + marker: PhantomData<(&'a K, &'a mut V)>, } -impl<'a, K: Send, V: Send, S: Send> ParallelIterator for ParValuesMut<'a, K, V, S> { +impl<'a, K: Sync, V: Send> ParallelIterator for ParValuesMut<'a, K, V> { type Item = &'a mut V; #[cfg_attr(feature = "inline-more", inline)] @@ -190,17 +207,19 @@ impl<'a, K: Send, V: Send, S: Send> ParallelIterator for ParValuesMut<'a, K, V, where C: UnindexedConsumer, { - self.map - .table - .par_iter() + self.inner .map(|x| unsafe { &mut x.as_mut().1 }) .drive_unindexed(consumer) } } -impl fmt::Debug for ParValuesMut<'_, K, V, S> { +impl fmt::Debug for ParValuesMut<'_, K, V> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.map.values().fmt(f) + ParValues { + inner: self.inner.clone(), + marker: PhantomData, + } + .fmt(f) } } @@ -213,11 +232,11 @@ impl fmt::Debug for ParValuesMut<'_ /// [`into_par_iter`]: /hashbrown/struct.HashMap.html#method.into_par_iter /// [`HashMap`]: /hashbrown/struct.HashMap.html /// [`IntoParallelIterator`]: https://docs.rs/rayon/1.0/rayon/iter/trait.IntoParallelIterator.html -pub struct IntoParIter { - map: HashMap, +pub struct IntoParIter { + inner: RawIntoParIter<(K, V), A>, } -impl ParallelIterator for IntoParIter { +impl ParallelIterator for IntoParIter { type Item = (K, V); #[cfg_attr(feature = "inline-more", inline)] @@ -225,13 +244,19 @@ impl ParallelIterator for IntoParIter { where C: UnindexedConsumer, { - self.map.table.into_par_iter().drive_unindexed(consumer) + self.inner.drive_unindexed(consumer) } } -impl fmt::Debug for IntoParIter { +impl fmt::Debug + for IntoParIter +{ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.map.iter().fmt(f) + ParIter { + inner: unsafe { self.inner.par_iter() }, + marker: PhantomData, + } + .fmt(f) } } @@ -242,11 +267,11 @@ impl fmt::Debug for In /// /// [`par_drain`]: /hashbrown/struct.HashMap.html#method.par_drain /// [`HashMap`]: /hashbrown/struct.HashMap.html -pub struct ParDrain<'a, K, V, S> { - map: &'a mut HashMap, +pub struct ParDrain<'a, K, V, A: Allocator + Clone = Global> { + inner: RawParDrain<'a, (K, V), A>, } -impl ParallelIterator for ParDrain<'_, K, V, S> { +impl ParallelIterator for ParDrain<'_, K, V, A> { type Item = (K, V); #[cfg_attr(feature = "inline-more", inline)] @@ -254,52 +279,68 @@ impl ParallelIterator for ParDrain<'_, K, V, S> { where C: UnindexedConsumer, { - self.map.table.par_drain().drive_unindexed(consumer) + self.inner.drive_unindexed(consumer) } } -impl fmt::Debug - for ParDrain<'_, K, V, S> +impl fmt::Debug + for ParDrain<'_, K, V, A> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.map.iter().fmt(f) + ParIter { + inner: unsafe { self.inner.par_iter() }, + marker: PhantomData, + } + .fmt(f) } } -impl HashMap { +impl HashMap { /// Visits (potentially in parallel) immutably borrowed keys in an arbitrary order. #[cfg_attr(feature = "inline-more", inline)] - pub fn par_keys(&self) -> ParKeys<'_, K, V, S> { - ParKeys { map: self } + pub fn par_keys(&self) -> ParKeys<'_, K, V> { + ParKeys { + inner: unsafe { self.table.par_iter() }, + marker: PhantomData, + } } /// Visits (potentially in parallel) immutably borrowed values in an arbitrary order. #[cfg_attr(feature = "inline-more", inline)] - pub fn par_values(&self) -> ParValues<'_, K, V, S> { - ParValues { map: self } + pub fn par_values(&self) -> ParValues<'_, K, V> { + ParValues { + inner: unsafe { self.table.par_iter() }, + marker: PhantomData, + } } } -impl HashMap { +impl HashMap { /// Visits (potentially in parallel) mutably borrowed values in an arbitrary order. #[cfg_attr(feature = "inline-more", inline)] - pub fn par_values_mut(&mut self) -> ParValuesMut<'_, K, V, S> { - ParValuesMut { map: self } + pub fn par_values_mut(&mut self) -> ParValuesMut<'_, K, V> { + ParValuesMut { + inner: unsafe { self.table.par_iter() }, + marker: PhantomData, + } } /// Consumes (potentially in parallel) all values in an arbitrary order, /// while preserving the map's allocated memory for reuse. #[cfg_attr(feature = "inline-more", inline)] - pub fn par_drain(&mut self) -> ParDrain<'_, K, V, S> { - ParDrain { map: self } + pub fn par_drain(&mut self) -> ParDrain<'_, K, V, A> { + ParDrain { + inner: self.table.par_drain(), + } } } -impl HashMap +impl HashMap where K: Eq + Hash + Sync, V: PartialEq + Sync, S: BuildHasher + Sync, + A: Allocator + Clone + Sync, { /// Returns `true` if the map is equal to another, /// i.e. both maps contain the same keys mapped to the same values. @@ -313,33 +354,47 @@ where } } -impl IntoParallelIterator for HashMap { +impl IntoParallelIterator + for HashMap +{ type Item = (K, V); - type Iter = IntoParIter; + type Iter = IntoParIter; #[cfg_attr(feature = "inline-more", inline)] fn into_par_iter(self) -> Self::Iter { - IntoParIter { map: self } + IntoParIter { + inner: self.table.into_par_iter(), + } } } -impl<'a, K: Sync, V: Sync, S: Sync> IntoParallelIterator for &'a HashMap { +impl<'a, K: Sync, V: Sync, S, A: Allocator + Clone> IntoParallelIterator + for &'a HashMap +{ type Item = (&'a K, &'a V); - type Iter = ParIter<'a, K, V, S>; + type Iter = ParIter<'a, K, V>; #[cfg_attr(feature = "inline-more", inline)] fn into_par_iter(self) -> Self::Iter { - ParIter { map: self } + ParIter { + inner: unsafe { self.table.par_iter() }, + marker: PhantomData, + } } } -impl<'a, K: Send + Sync, V: Send, S: Send> IntoParallelIterator for &'a mut HashMap { +impl<'a, K: Sync, V: Send, S, A: Allocator + Clone> IntoParallelIterator + for &'a mut HashMap +{ type Item = (&'a K, &'a mut V); - type Iter = ParIterMut<'a, K, V, S>; + type Iter = ParIterMut<'a, K, V>; #[cfg_attr(feature = "inline-more", inline)] fn into_par_iter(self) -> Self::Iter { - ParIterMut { map: self } + ParIterMut { + inner: unsafe { self.table.par_iter() }, + marker: PhantomData, + } } } @@ -347,7 +402,7 @@ impl<'a, K: Send + Sync, V: Send, S: Send> IntoParallelIterator for &'a mut Hash /// hashmap. If multiple pairs correspond to the same key, then the /// ones produced earlier in the parallel iterator will be /// overwritten, just as with a sequential iterator. -impl FromParallelIterator<(K, V)> for HashMap +impl FromParallelIterator<(K, V)> for HashMap where K: Eq + Hash + Send, V: Send, @@ -364,11 +419,12 @@ where } /// Extend a hash map with items from a parallel iterator. -impl ParallelExtend<(K, V)> for HashMap +impl ParallelExtend<(K, V)> for HashMap where K: Eq + Hash + Send, V: Send, S: BuildHasher, + A: Allocator + Clone, { fn par_extend(&mut self, par_iter: I) where @@ -379,11 +435,12 @@ where } /// Extend a hash map with copied items from a parallel iterator. -impl<'a, K, V, S> ParallelExtend<(&'a K, &'a V)> for HashMap +impl<'a, K, V, S, A> ParallelExtend<(&'a K, &'a V)> for HashMap where K: Copy + Eq + Hash + Sync, V: Copy + Sync, S: BuildHasher, + A: Allocator + Clone, { fn par_extend(&mut self, par_iter: I) where @@ -394,12 +451,13 @@ where } // This is equal to the normal `HashMap` -- no custom advantage. -fn extend(map: &mut HashMap, par_iter: I) +fn extend(map: &mut HashMap, par_iter: I) where K: Eq + Hash, S: BuildHasher, I: IntoParallelIterator, - HashMap: Extend, + A: Allocator + Clone, + HashMap: Extend, { let (list, len) = super::helpers::collect(par_iter); @@ -454,7 +512,7 @@ mod test_par_map { where H: Hasher, { - self.k.hash(state) + self.k.hash(state); } } @@ -621,7 +679,7 @@ mod test_par_map { fn test_values_mut() { let vec = vec![(1, 1), (2, 2), (3, 3)]; let mut map: HashMap<_, _> = vec.into_par_iter().collect(); - map.par_values_mut().for_each(|value| *value = (*value) * 2); + map.par_values_mut().for_each(|value| *value *= 2); let values: Vec<_> = map.par_values().cloned().collect(); assert_eq!(values.len(), 3); assert!(values.contains(&2)); diff --git a/crux-mir/lib/hashbrown/src/external_trait_impls/rayon/raw.rs b/crux-mir/lib/hashbrown/src/external_trait_impls/rayon/raw.rs index c6fe09dff..883303e27 100644 --- a/crux-mir/lib/hashbrown/src/external_trait_impls/rayon/raw.rs +++ b/crux-mir/lib/hashbrown/src/external_trait_impls/rayon/raw.rs @@ -1,5 +1,5 @@ use crate::raw::Bucket; -use crate::raw::{RawIterRange, RawTable}; +use crate::raw::{Allocator, Global, RawIter, RawIterRange, RawTable}; use crate::scopeguard::guard; use alloc::alloc::dealloc; use core::marker::PhantomData; @@ -15,6 +15,28 @@ pub struct RawParIter { iter: RawIterRange, } +impl RawParIter { + #[cfg_attr(feature = "inline-more", inline)] + pub(super) unsafe fn iter(&self) -> RawIterRange { + self.iter.clone() + } +} + +impl Clone for RawParIter { + #[cfg_attr(feature = "inline-more", inline)] + fn clone(&self) -> Self { + Self { + iter: self.iter.clone(), + } + } +} + +impl From> for RawParIter { + fn from(it: RawIter) -> Self { + RawParIter { iter: it.iter } + } +} + impl ParallelIterator for RawParIter { type Item = Bucket; @@ -54,11 +76,18 @@ impl UnindexedProducer for ParIterProducer { } /// Parallel iterator which consumes a table and returns elements. -pub struct RawIntoParIter { - table: RawTable, +pub struct RawIntoParIter { + table: RawTable, +} + +impl RawIntoParIter { + #[cfg_attr(feature = "inline-more", inline)] + pub(super) unsafe fn par_iter(&self) -> RawParIter { + self.table.par_iter() + } } -impl ParallelIterator for RawIntoParIter { +impl ParallelIterator for RawIntoParIter { type Item = T; #[cfg_attr(feature = "inline-more", inline)] @@ -67,7 +96,7 @@ impl ParallelIterator for RawIntoParIter { C: UnindexedConsumer, { let iter = unsafe { self.table.iter().iter }; - let _guard = guard(self.table.into_alloc(), |alloc| { + let _guard = guard(self.table.into_allocation(), |alloc| { if let Some((ptr, layout)) = *alloc { unsafe { dealloc(ptr.as_ptr(), layout); @@ -80,16 +109,23 @@ impl ParallelIterator for RawIntoParIter { } /// Parallel iterator which consumes elements without freeing the table storage. -pub struct RawParDrain<'a, T> { +pub struct RawParDrain<'a, T, A: Allocator + Clone = Global> { // We don't use a &'a mut RawTable because we want RawParDrain to be // covariant over T. - table: NonNull>, - marker: PhantomData<&'a RawTable>, + table: NonNull>, + marker: PhantomData<&'a RawTable>, } -unsafe impl Send for RawParDrain<'_, T> {} +unsafe impl Send for RawParDrain<'_, T, A> {} -impl ParallelIterator for RawParDrain<'_, T> { +impl RawParDrain<'_, T, A> { + #[cfg_attr(feature = "inline-more", inline)] + pub(super) unsafe fn par_iter(&self) -> RawParIter { + self.table.as_ref().par_iter() + } +} + +impl ParallelIterator for RawParDrain<'_, T, A> { type Item = T; #[cfg_attr(feature = "inline-more", inline)] @@ -98,7 +134,7 @@ impl ParallelIterator for RawParDrain<'_, T> { C: UnindexedConsumer, { let _guard = guard(self.table, |table| unsafe { - table.as_mut().clear_no_drop() + table.as_mut().clear_no_drop(); }); let iter = unsafe { self.table.as_ref().iter().iter }; mem::forget(self); @@ -107,10 +143,12 @@ impl ParallelIterator for RawParDrain<'_, T> { } } -impl Drop for RawParDrain<'_, T> { +impl Drop for RawParDrain<'_, T, A> { fn drop(&mut self) { // If drive_unindexed is not called then simply clear the table. - unsafe { self.table.as_mut().clear() } + unsafe { + self.table.as_mut().clear(); + } } } @@ -139,7 +177,7 @@ impl UnindexedProducer for ParDrainProducer { { // Make sure to modify the iterator in-place so that any remaining // elements are processed in our Drop impl. - while let Some(item) = self.iter.next() { + for item in &mut self.iter { folder = folder.consume(unsafe { item.read() }); if folder.full() { return folder; @@ -157,7 +195,7 @@ impl Drop for ParDrainProducer { fn drop(&mut self) { // Drop all remaining elements if mem::needs_drop::() { - while let Some(item) = self.iter.next() { + for item in &mut self.iter { unsafe { item.drop(); } @@ -166,25 +204,25 @@ impl Drop for ParDrainProducer { } } -impl RawTable { +impl RawTable { /// Returns a parallel iterator over the elements in a `RawTable`. #[cfg_attr(feature = "inline-more", inline)] - pub fn par_iter(&self) -> RawParIter { + pub unsafe fn par_iter(&self) -> RawParIter { RawParIter { - iter: unsafe { self.iter().iter }, + iter: self.iter().iter, } } /// Returns a parallel iterator over the elements in a `RawTable`. #[cfg_attr(feature = "inline-more", inline)] - pub fn into_par_iter(self) -> RawIntoParIter { + pub fn into_par_iter(self) -> RawIntoParIter { RawIntoParIter { table: self } } /// Returns a parallel iterator which consumes all elements of a `RawTable` /// without freeing its memory allocation. #[cfg_attr(feature = "inline-more", inline)] - pub fn par_drain(&mut self) -> RawParDrain<'_, T> { + pub fn par_drain(&mut self) -> RawParDrain<'_, T, A> { RawParDrain { table: NonNull::from(self), marker: PhantomData, diff --git a/crux-mir/lib/hashbrown/src/external_trait_impls/rayon/set.rs b/crux-mir/lib/hashbrown/src/external_trait_impls/rayon/set.rs index 53d2660d5..ee4f6e669 100644 --- a/crux-mir/lib/hashbrown/src/external_trait_impls/rayon/set.rs +++ b/crux-mir/lib/hashbrown/src/external_trait_impls/rayon/set.rs @@ -1,6 +1,8 @@ //! Rayon extensions for `HashSet`. +use super::map; use crate::hash_set::HashSet; +use crate::raw::{Allocator, Global}; use core::hash::{BuildHasher, Hash}; use rayon::iter::plumbing::UnindexedConsumer; use rayon::iter::{FromParallelIterator, IntoParallelIterator, ParallelExtend, ParallelIterator}; @@ -14,22 +16,18 @@ use rayon::iter::{FromParallelIterator, IntoParallelIterator, ParallelExtend, Pa /// [`into_par_iter`]: /hashbrown/struct.HashSet.html#method.into_par_iter /// [`HashSet`]: /hashbrown/struct.HashSet.html /// [`IntoParallelIterator`]: https://docs.rs/rayon/1.0/rayon/iter/trait.IntoParallelIterator.html -pub struct IntoParIter { - set: HashSet, +pub struct IntoParIter { + inner: map::IntoParIter, } -impl ParallelIterator for IntoParIter { +impl ParallelIterator for IntoParIter { type Item = T; fn drive_unindexed(self, consumer: C) -> C::Result where C: UnindexedConsumer, { - self.set - .map - .into_par_iter() - .map(|(k, _)| k) - .drive_unindexed(consumer) + self.inner.map(|(k, _)| k).drive_unindexed(consumer) } } @@ -40,22 +38,18 @@ impl ParallelIterator for IntoParIter { /// /// [`par_drain`]: /hashbrown/struct.HashSet.html#method.par_drain /// [`HashSet`]: /hashbrown/struct.HashSet.html -pub struct ParDrain<'a, T, S> { - set: &'a mut HashSet, +pub struct ParDrain<'a, T, A: Allocator + Clone = Global> { + inner: map::ParDrain<'a, T, (), A>, } -impl ParallelIterator for ParDrain<'_, T, S> { +impl ParallelIterator for ParDrain<'_, T, A> { type Item = T; fn drive_unindexed(self, consumer: C) -> C::Result where C: UnindexedConsumer, { - self.set - .map - .par_drain() - .map(|(k, _)| k) - .drive_unindexed(consumer) + self.inner.map(|(k, _)| k).drive_unindexed(consumer) } } @@ -68,18 +62,18 @@ impl ParallelIterator for ParDrain<'_, T, S> { /// [`par_iter`]: /hashbrown/struct.HashSet.html#method.par_iter /// [`HashSet`]: /hashbrown/struct.HashSet.html /// [`IntoParallelRefIterator`]: https://docs.rs/rayon/1.0/rayon/iter/trait.IntoParallelRefIterator.html -pub struct ParIter<'a, T, S> { - set: &'a HashSet, +pub struct ParIter<'a, T> { + inner: map::ParKeys<'a, T, ()>, } -impl<'a, T: Sync, S: Sync> ParallelIterator for ParIter<'a, T, S> { +impl<'a, T: Sync> ParallelIterator for ParIter<'a, T> { type Item = &'a T; fn drive_unindexed(self, consumer: C) -> C::Result where C: UnindexedConsumer, { - self.set.map.par_keys().drive_unindexed(consumer) + self.inner.drive_unindexed(consumer) } } @@ -91,15 +85,16 @@ impl<'a, T: Sync, S: Sync> ParallelIterator for ParIter<'a, T, S> { /// /// [`par_difference`]: /hashbrown/struct.HashSet.html#method.par_difference /// [`HashSet`]: /hashbrown/struct.HashSet.html -pub struct ParDifference<'a, T, S> { - a: &'a HashSet, - b: &'a HashSet, +pub struct ParDifference<'a, T, S, A: Allocator + Clone = Global> { + a: &'a HashSet, + b: &'a HashSet, } -impl<'a, T, S> ParallelIterator for ParDifference<'a, T, S> +impl<'a, T, S, A> ParallelIterator for ParDifference<'a, T, S, A> where T: Eq + Hash + Sync, S: BuildHasher + Sync, + A: Allocator + Clone + Sync, { type Item = &'a T; @@ -123,15 +118,16 @@ where /// /// [`par_symmetric_difference`]: /hashbrown/struct.HashSet.html#method.par_symmetric_difference /// [`HashSet`]: /hashbrown/struct.HashSet.html -pub struct ParSymmetricDifference<'a, T, S> { - a: &'a HashSet, - b: &'a HashSet, +pub struct ParSymmetricDifference<'a, T, S, A: Allocator + Clone = Global> { + a: &'a HashSet, + b: &'a HashSet, } -impl<'a, T, S> ParallelIterator for ParSymmetricDifference<'a, T, S> +impl<'a, T, S, A> ParallelIterator for ParSymmetricDifference<'a, T, S, A> where T: Eq + Hash + Sync, S: BuildHasher + Sync, + A: Allocator + Clone + Sync, { type Item = &'a T; @@ -154,15 +150,16 @@ where /// /// [`par_intersection`]: /hashbrown/struct.HashSet.html#method.par_intersection /// [`HashSet`]: /hashbrown/struct.HashSet.html -pub struct ParIntersection<'a, T, S> { - a: &'a HashSet, - b: &'a HashSet, +pub struct ParIntersection<'a, T, S, A: Allocator + Clone = Global> { + a: &'a HashSet, + b: &'a HashSet, } -impl<'a, T, S> ParallelIterator for ParIntersection<'a, T, S> +impl<'a, T, S, A> ParallelIterator for ParIntersection<'a, T, S, A> where T: Eq + Hash + Sync, S: BuildHasher + Sync, + A: Allocator + Clone + Sync, { type Item = &'a T; @@ -184,15 +181,16 @@ where /// /// [`par_union`]: /hashbrown/struct.HashSet.html#method.par_union /// [`HashSet`]: /hashbrown/struct.HashSet.html -pub struct ParUnion<'a, T, S> { - a: &'a HashSet, - b: &'a HashSet, +pub struct ParUnion<'a, T, S, A: Allocator + Clone = Global> { + a: &'a HashSet, + b: &'a HashSet, } -impl<'a, T, S> ParallelIterator for ParUnion<'a, T, S> +impl<'a, T, S, A> ParallelIterator for ParUnion<'a, T, S, A> where T: Eq + Hash + Sync, S: BuildHasher + Sync, + A: Allocator + Clone + Sync, { type Item = &'a T; @@ -200,22 +198,37 @@ where where C: UnindexedConsumer, { - self.a + // We'll iterate one set in full, and only the remaining difference from the other. + // Use the smaller set for the difference in order to reduce hash lookups. + let (smaller, larger) = if self.a.len() <= self.b.len() { + (self.a, self.b) + } else { + (self.b, self.a) + }; + larger .into_par_iter() - .chain(self.b.par_difference(self.a)) + .chain(smaller.par_difference(larger)) .drive_unindexed(consumer) } } -impl HashSet +impl HashSet where T: Eq + Hash + Sync, S: BuildHasher + Sync, + A: Allocator + Clone + Sync, { + /// Visits (potentially in parallel) the values representing the union, + /// i.e. all the values in `self` or `other`, without duplicates. + #[cfg_attr(feature = "inline-more", inline)] + pub fn par_union<'a>(&'a self, other: &'a Self) -> ParUnion<'a, T, S, A> { + ParUnion { a: self, b: other } + } + /// Visits (potentially in parallel) the values representing the difference, /// i.e. the values that are in `self` but not in `other`. #[cfg_attr(feature = "inline-more", inline)] - pub fn par_difference<'a>(&'a self, other: &'a Self) -> ParDifference<'a, T, S> { + pub fn par_difference<'a>(&'a self, other: &'a Self) -> ParDifference<'a, T, S, A> { ParDifference { a: self, b: other } } @@ -225,24 +238,17 @@ where pub fn par_symmetric_difference<'a>( &'a self, other: &'a Self, - ) -> ParSymmetricDifference<'a, T, S> { + ) -> ParSymmetricDifference<'a, T, S, A> { ParSymmetricDifference { a: self, b: other } } /// Visits (potentially in parallel) the values representing the /// intersection, i.e. the values that are both in `self` and `other`. #[cfg_attr(feature = "inline-more", inline)] - pub fn par_intersection<'a>(&'a self, other: &'a Self) -> ParIntersection<'a, T, S> { + pub fn par_intersection<'a>(&'a self, other: &'a Self) -> ParIntersection<'a, T, S, A> { ParIntersection { a: self, b: other } } - /// Visits (potentially in parallel) the values representing the union, - /// i.e. all the values in `self` or `other`, without duplicates. - #[cfg_attr(feature = "inline-more", inline)] - pub fn par_union<'a>(&'a self, other: &'a Self) -> ParUnion<'a, T, S> { - ParUnion { a: self, b: other } - } - /// Returns `true` if `self` has no elements in common with `other`. /// This is equivalent to checking for an empty intersection. /// @@ -280,41 +286,47 @@ where } } -impl HashSet +impl HashSet where T: Eq + Hash + Send, - S: BuildHasher + Send, + A: Allocator + Clone + Send, { /// Consumes (potentially in parallel) all values in an arbitrary order, /// while preserving the set's allocated memory for reuse. #[cfg_attr(feature = "inline-more", inline)] - pub fn par_drain(&mut self) -> ParDrain<'_, T, S> { - ParDrain { set: self } + pub fn par_drain(&mut self) -> ParDrain<'_, T, A> { + ParDrain { + inner: self.map.par_drain(), + } } } -impl IntoParallelIterator for HashSet { +impl IntoParallelIterator for HashSet { type Item = T; - type Iter = IntoParIter; + type Iter = IntoParIter; #[cfg_attr(feature = "inline-more", inline)] fn into_par_iter(self) -> Self::Iter { - IntoParIter { set: self } + IntoParIter { + inner: self.map.into_par_iter(), + } } } -impl<'a, T: Sync, S: Sync> IntoParallelIterator for &'a HashSet { +impl<'a, T: Sync, S, A: Allocator + Clone> IntoParallelIterator for &'a HashSet { type Item = &'a T; - type Iter = ParIter<'a, T, S>; + type Iter = ParIter<'a, T>; #[cfg_attr(feature = "inline-more", inline)] fn into_par_iter(self) -> Self::Iter { - ParIter { set: self } + ParIter { + inner: self.map.par_keys(), + } } } /// Collect values from a parallel iterator into a hashset. -impl FromParallelIterator for HashSet +impl FromParallelIterator for HashSet where T: Eq + Hash + Send, S: BuildHasher + Default, @@ -330,7 +342,7 @@ where } /// Extend a hash set with items from a parallel iterator. -impl ParallelExtend for HashSet +impl ParallelExtend for HashSet where T: Eq + Hash + Send, S: BuildHasher, @@ -344,7 +356,7 @@ where } /// Extend a hash set with copied items from a parallel iterator. -impl<'a, T, S> ParallelExtend<&'a T> for HashSet +impl<'a, T, S> ParallelExtend<&'a T> for HashSet where T: 'a + Copy + Eq + Hash + Sync, S: BuildHasher, @@ -358,12 +370,13 @@ where } // This is equal to the normal `HashSet` -- no custom advantage. -fn extend(set: &mut HashSet, par_iter: I) +fn extend(set: &mut HashSet, par_iter: I) where T: Eq + Hash, S: BuildHasher, + A: Allocator + Clone, I: IntoParallelIterator, - HashSet: Extend, + HashSet: Extend, { let (list, len) = super::helpers::collect(par_iter); diff --git a/crux-mir/lib/hashbrown/src/external_trait_impls/serde.rs b/crux-mir/lib/hashbrown/src/external_trait_impls/serde.rs index 7816e7803..4d62deeb7 100644 --- a/crux-mir/lib/hashbrown/src/external_trait_impls/serde.rs +++ b/crux-mir/lib/hashbrown/src/external_trait_impls/serde.rs @@ -161,6 +161,7 @@ mod set { deserializer.deserialize_seq(visitor) } + #[allow(clippy::missing_errors_doc)] fn deserialize_in_place(deserializer: D, place: &mut Self) -> Result<(), D::Error> where D: Deserializer<'de>, diff --git a/crux-mir/lib/hashbrown/src/lib.rs b/crux-mir/lib/hashbrown/src/lib.rs index 63b180c1a..bc1c97130 100644 --- a/crux-mir/lib/hashbrown/src/lib.rs +++ b/crux-mir/lib/hashbrown/src/lib.rs @@ -13,19 +13,27 @@ #![cfg_attr( feature = "nightly", feature( - alloc_layout_extra, - allocator_api, - ptr_offset_from, test, core_intrinsics, dropck_eyepatch, - specialization, + min_specialization, + extend_one, + allocator_api, + slice_ptr_get, + nonnull_slice_from_raw_parts, + maybe_uninit_array_assume_init, + build_hasher_simple_hash_one ) )] #![allow( clippy::doc_markdown, clippy::module_name_repetitions, - clippy::must_use_candidate + clippy::must_use_candidate, + clippy::option_if_let_else, + clippy::redundant_else, + clippy::manual_map, + clippy::missing_safety_doc, + clippy::missing_errors_doc )] #![warn(missing_docs)] #![warn(rust_2018_idioms)] @@ -34,11 +42,8 @@ #[macro_use] extern crate std; -#[cfg(has_extern_crate_alloc)] #[cfg_attr(test, macro_use)] extern crate alloc; -#[cfg(not(has_extern_crate_alloc))] -extern crate std as alloc; #[cfg(feature = "nightly")] #[cfg(doctest)] @@ -58,6 +63,11 @@ pub mod raw { pub use inner::*; #[cfg(feature = "rayon")] + /// [rayon]-based parallel iterator types for hash maps. + /// You will rarely need to interact with it directly unless you have need + /// to name one of the iterator types. + /// + /// [rayon]: https://docs.rs/rayon/1.0/rayon pub mod rayon { pub use crate::external_trait_impls::rayon::raw::*; } @@ -107,15 +117,34 @@ pub mod hash_set { pub use crate::map::HashMap; pub use crate::set::HashSet; -/// Augments `AllocErr` with a `CapacityOverflow` variant. +/// The error type for `try_reserve` methods. #[derive(Clone, PartialEq, Eq, Debug)] -pub enum CollectionAllocErr { +pub enum TryReserveError { /// Error due to the computed capacity exceeding the collection's maximum /// (usually `isize::MAX` bytes). CapacityOverflow, - /// Error due to the allocator. - AllocErr { + + /// The memory allocator returned an error + AllocError { /// The layout of the allocation request that failed. layout: alloc::alloc::Layout, }, } + +/// Wrapper around `Bump` which allows it to be used as an allocator for +/// `HashMap`, `HashSet` and `RawTable`. +/// +/// `Bump` can be used directly without this wrapper on nightly if you enable +/// the `allocator-api` feature of the `bumpalo` crate. +#[cfg(feature = "bumpalo")] +#[derive(Clone, Copy, Debug)] +pub struct BumpWrapper<'a>(pub &'a bumpalo::Bump); + +#[cfg(feature = "bumpalo")] +#[test] +fn test_bumpalo() { + use bumpalo::Bump; + let bump = Bump::new(); + let mut map = HashMap::new_in(BumpWrapper(&bump)); + map.insert(0, 1); +} diff --git a/crux-mir/lib/hashbrown/src/macros.rs b/crux-mir/lib/hashbrown/src/macros.rs index e74309476..f8ef917b1 100644 --- a/crux-mir/lib/hashbrown/src/macros.rs +++ b/crux-mir/lib/hashbrown/src/macros.rs @@ -1,4 +1,5 @@ // See the cfg-if crate. +#[allow(unused_macro_rules)] macro_rules! cfg_if { // match if/else chains with a final `else` ($( @@ -52,3 +53,18 @@ macro_rules! cfg_if { $(#[$m] $it)* }; } + +// Helper macro for specialization. This also helps avoid parse errors if the +// default fn syntax for specialization changes in the future. +#[cfg(feature = "nightly")] +macro_rules! default_fn { + (#[$($a:tt)*] $($tt:tt)*) => { + #[$($a)*] default $($tt)* + } +} +#[cfg(not(feature = "nightly"))] +macro_rules! default_fn { + ($($tt:tt)*) => { + $($tt)* + } +} diff --git a/crux-mir/lib/hashbrown/src/map.rs b/crux-mir/lib/hashbrown/src/map.rs index edb940150..a5d3ccb97 100644 --- a/crux-mir/lib/hashbrown/src/map.rs +++ b/crux-mir/lib/hashbrown/src/map.rs @@ -1,8 +1,8 @@ -use crate::raw::{Bucket, RawDrain, RawIntoIter, RawIter, RawTable}; -use crate::CollectionAllocErr; +use crate::raw::{Allocator, Bucket, Global, RawDrain, RawIntoIter, RawIter, RawTable}; +use crate::TryReserveError; use core::borrow::Borrow; use core::fmt::{self, Debug}; -use core::hash::{BuildHasher, Hash, Hasher}; +use core::hash::{BuildHasher, Hash}; use core::iter::{FromIterator, FusedIterator}; use core::marker::PhantomData; use core::mem; @@ -181,19 +181,16 @@ pub enum DefaultHashBuilder {} /// ``` /// use hashbrown::HashMap; /// -/// let timber_resources: HashMap<&str, i32> = -/// [("Norway", 100), -/// ("Denmark", 50), -/// ("Iceland", 10)] -/// .iter().cloned().collect(); +/// let timber_resources: HashMap<&str, i32> = [("Norway", 100), ("Denmark", 50), ("Iceland", 10)] +/// .iter().cloned().collect(); /// // use the values stored in map /// ``` -pub struct HashMap { +pub struct HashMap { pub(crate) hash_builder: S, - pub(crate) table: RawTable<(K, V)>, + pub(crate) table: RawTable<(K, V), A>, } -impl Clone for HashMap { +impl Clone for HashMap { fn clone(&self) -> Self { HashMap { hash_builder: self.hash_builder.clone(), @@ -202,49 +199,95 @@ impl Clone for HashMap { } fn clone_from(&mut self, source: &Self) { - // We clone the hash_builder first since this might panic and we don't - // want the table to have elements hashed with the wrong hash_builder. - let hash_builder = source.hash_builder.clone(); - - #[cfg(not(feature = "nightly"))] - { - self.table.clone_from(&source.table); - } - #[cfg(feature = "nightly")] - { - trait HashClone { - fn clone_from(&mut self, source: &Self, hash_builder: &S); - } - impl HashClone for HashMap { - default fn clone_from(&mut self, source: &Self, _hash_builder: &S) { - self.table.clone_from(&source.table); - } - } - impl HashClone for HashMap - where - K: Eq + Hash, - S: BuildHasher, - { - fn clone_from(&mut self, source: &Self, hash_builder: &S) { - self.table - .clone_from_with_hasher(&source.table, |x| make_hash(hash_builder, &x.0)); - } - } - HashClone::clone_from(self, source, &hash_builder); - } + self.table.clone_from(&source.table); // Update hash_builder only if we successfully cloned all elements. - self.hash_builder = hash_builder; + self.hash_builder.clone_from(&source.hash_builder); } } +/// Ensures that a single closure type across uses of this which, in turn prevents multiple +/// instances of any functions like RawTable::reserve from being generated +#[cfg_attr(feature = "inline-more", inline)] +pub(crate) fn make_hasher(hash_builder: &S) -> impl Fn(&(Q, V)) -> u64 + '_ +where + K: Borrow, + Q: Hash, + S: BuildHasher, +{ + move |val| make_hash::(hash_builder, &val.0) +} + +/// Ensures that a single closure type across uses of this which, in turn prevents multiple +/// instances of any functions like RawTable::reserve from being generated +#[cfg_attr(feature = "inline-more", inline)] +fn equivalent_key(k: &Q) -> impl Fn(&(K, V)) -> bool + '_ +where + K: Borrow, + Q: ?Sized + Eq, +{ + move |x| k.eq(x.0.borrow()) +} + +/// Ensures that a single closure type across uses of this which, in turn prevents multiple +/// instances of any functions like RawTable::reserve from being generated +#[cfg_attr(feature = "inline-more", inline)] +fn equivalent(k: &Q) -> impl Fn(&K) -> bool + '_ +where + K: Borrow, + Q: ?Sized + Eq, +{ + move |x| k.eq(x.borrow()) +} + +#[cfg(not(feature = "nightly"))] +#[cfg_attr(feature = "inline-more", inline)] +pub(crate) fn make_hash(hash_builder: &S, val: &Q) -> u64 +where + K: Borrow, + Q: Hash + ?Sized, + S: BuildHasher, +{ + use core::hash::Hasher; + let mut state = hash_builder.build_hasher(); + val.hash(&mut state); + state.finish() +} + +#[cfg(feature = "nightly")] +#[cfg_attr(feature = "inline-more", inline)] +pub(crate) fn make_hash(hash_builder: &S, val: &Q) -> u64 +where + K: Borrow, + Q: Hash + ?Sized, + S: BuildHasher, +{ + hash_builder.hash_one(val) +} + +#[cfg(not(feature = "nightly"))] #[cfg_attr(feature = "inline-more", inline)] -pub(crate) fn make_hash(hash_builder: &impl BuildHasher, val: &K) -> u64 { +pub(crate) fn make_insert_hash(hash_builder: &S, val: &K) -> u64 +where + K: Hash, + S: BuildHasher, +{ + use core::hash::Hasher; let mut state = hash_builder.build_hasher(); val.hash(&mut state); state.finish() } +#[cfg(feature = "nightly")] +#[cfg_attr(feature = "inline-more", inline)] +pub(crate) fn make_insert_hash(hash_builder: &S, val: &K) -> u64 +where + K: Hash, + S: BuildHasher, +{ + hash_builder.hash_one(val) +} + #[cfg(feature = "ahash")] impl HashMap { /// Creates an empty `HashMap`. @@ -257,6 +300,8 @@ impl HashMap { /// ``` /// use hashbrown::HashMap; /// let mut map: HashMap<&str, i32> = HashMap::new(); + /// assert_eq!(map.len(), 0); + /// assert_eq!(map.capacity(), 0); /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn new() -> Self { @@ -273,6 +318,8 @@ impl HashMap { /// ``` /// use hashbrown::HashMap; /// let mut map: HashMap<&str, i32> = HashMap::with_capacity(10); + /// assert_eq!(map.len(), 0); + /// assert!(map.capacity() >= 10); /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn with_capacity(capacity: usize) -> Self { @@ -280,17 +327,42 @@ impl HashMap { } } +#[cfg(feature = "ahash")] +impl HashMap { + /// Creates an empty `HashMap` using the given allocator. + /// + /// The hash map is initially created with a capacity of 0, so it will not allocate until it + /// is first inserted into. + #[cfg_attr(feature = "inline-more", inline)] + pub fn new_in(alloc: A) -> Self { + Self::with_hasher_in(DefaultHashBuilder::default(), alloc) + } + + /// Creates an empty `HashMap` with the specified capacity using the given allocator. + /// + /// The hash map will be able to hold at least `capacity` elements without + /// reallocating. If `capacity` is 0, the hash map will not allocate. + #[cfg_attr(feature = "inline-more", inline)] + pub fn with_capacity_in(capacity: usize, alloc: A) -> Self { + Self::with_capacity_and_hasher_in(capacity, DefaultHashBuilder::default(), alloc) + } +} + impl HashMap { /// Creates an empty `HashMap` which will use the given hash builder to hash /// keys. /// - /// The created map has the default initial capacity. + /// The hash map is initially created with a capacity of 0, so it will not + /// allocate until it is first inserted into. /// /// Warning: `hash_builder` is normally randomly generated, and /// is designed to allow HashMaps to be resistant to attacks that /// cause many collisions and very poor performance. Setting it /// manually using this function can expose a DoS attack vector. /// + /// The `hash_builder` passed should implement the [`BuildHasher`] trait for + /// the HashMap to be useful, see its documentation for details. + /// /// # Examples /// /// ``` @@ -299,10 +371,15 @@ impl HashMap { /// /// let s = DefaultHashBuilder::default(); /// let mut map = HashMap::with_hasher(s); + /// assert_eq!(map.len(), 0); + /// assert_eq!(map.capacity(), 0); + /// /// map.insert(1, 2); /// ``` + /// + /// [`BuildHasher`]: https://doc.rust-lang.org/std/hash/trait.BuildHasher.html #[cfg_attr(feature = "inline-more", inline)] - pub fn with_hasher(hash_builder: S) -> Self { + pub const fn with_hasher(hash_builder: S) -> Self { Self { hash_builder, table: RawTable::new(), @@ -320,6 +397,9 @@ impl HashMap { /// cause many collisions and very poor performance. Setting it /// manually using this function can expose a DoS attack vector. /// + /// The `hash_builder` passed should implement the [`BuildHasher`] trait for + /// the HashMap to be useful, see its documentation for details. + /// /// # Examples /// /// ``` @@ -328,8 +408,13 @@ impl HashMap { /// /// let s = DefaultHashBuilder::default(); /// let mut map = HashMap::with_capacity_and_hasher(10, s); + /// assert_eq!(map.len(), 0); + /// assert!(map.capacity() >= 10); + /// /// map.insert(1, 2); /// ``` + /// + /// [`BuildHasher`]: https://doc.rust-lang.org/std/hash/trait.BuildHasher.html #[cfg_attr(feature = "inline-more", inline)] pub fn with_capacity_and_hasher(capacity: usize, hash_builder: S) -> Self { Self { @@ -337,6 +422,71 @@ impl HashMap { table: RawTable::with_capacity(capacity), } } +} + +impl HashMap { + /// Returns a reference to the underlying allocator. + #[inline] + pub fn allocator(&self) -> &A { + self.table.allocator() + } + + /// Creates an empty `HashMap` which will use the given hash builder to hash + /// keys. It will be allocated with the given allocator. + /// + /// The created map has the default initial capacity. + /// + /// Warning: `hash_builder` is normally randomly generated, and + /// is designed to allow HashMaps to be resistant to attacks that + /// cause many collisions and very poor performance. Setting it + /// manually using this function can expose a DoS attack vector. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// use hashbrown::hash_map::DefaultHashBuilder; + /// + /// let s = DefaultHashBuilder::default(); + /// let mut map = HashMap::with_hasher(s); + /// map.insert(1, 2); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn with_hasher_in(hash_builder: S, alloc: A) -> Self { + Self { + hash_builder, + table: RawTable::new_in(alloc), + } + } + + /// Creates an empty `HashMap` with the specified capacity, using `hash_builder` + /// to hash the keys. It will be allocated with the given allocator. + /// + /// The hash map will be able to hold at least `capacity` elements without + /// reallocating. If `capacity` is 0, the hash map will not allocate. + /// + /// Warning: `hash_builder` is normally randomly generated, and + /// is designed to allow HashMaps to be resistant to attacks that + /// cause many collisions and very poor performance. Setting it + /// manually using this function can expose a DoS attack vector. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// use hashbrown::hash_map::DefaultHashBuilder; + /// + /// let s = DefaultHashBuilder::default(); + /// let mut map = HashMap::with_capacity_and_hasher(10, s); + /// map.insert(1, 2); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn with_capacity_and_hasher_in(capacity: usize, hash_builder: S, alloc: A) -> Self { + Self { + hash_builder, + table: RawTable::with_capacity_in(capacity, alloc), + } + } /// Returns a reference to the map's [`BuildHasher`]. /// @@ -367,6 +517,7 @@ impl HashMap { /// ``` /// use hashbrown::HashMap; /// let map: HashMap = HashMap::with_capacity(100); + /// assert_eq!(map.len(), 0); /// assert!(map.capacity() >= 100); /// ``` #[cfg_attr(feature = "inline-more", inline)] @@ -386,10 +537,20 @@ impl HashMap { /// map.insert("a", 1); /// map.insert("b", 2); /// map.insert("c", 3); + /// assert_eq!(map.len(), 3); + /// let mut vec: Vec<&str> = Vec::new(); /// /// for key in map.keys() { /// println!("{}", key); + /// vec.push(*key); /// } + /// + /// // The `Keys` iterator produces keys in arbitrary order, so the + /// // keys must be sorted to test them against a sorted array. + /// vec.sort_unstable(); + /// assert_eq!(vec, ["a", "b", "c"]); + /// + /// assert_eq!(map.len(), 3); /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn keys(&self) -> Keys<'_, K, V> { @@ -408,10 +569,20 @@ impl HashMap { /// map.insert("a", 1); /// map.insert("b", 2); /// map.insert("c", 3); + /// assert_eq!(map.len(), 3); + /// let mut vec: Vec = Vec::new(); /// /// for val in map.values() { /// println!("{}", val); + /// vec.push(*val); /// } + /// + /// // The `Values` iterator produces values in arbitrary order, so the + /// // values must be sorted to test them against a sorted array. + /// vec.sort_unstable(); + /// assert_eq!(vec, [1, 2, 3]); + /// + /// assert_eq!(map.len(), 3); /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn values(&self) -> Values<'_, K, V> { @@ -436,9 +607,20 @@ impl HashMap { /// *val = *val + 10; /// } /// + /// assert_eq!(map.len(), 3); + /// let mut vec: Vec = Vec::new(); + /// /// for val in map.values() { /// println!("{}", val); + /// vec.push(*val); /// } + /// + /// // The `Values` iterator produces values in arbitrary order, so the + /// // values must be sorted to test them against a sorted array. + /// vec.sort_unstable(); + /// assert_eq!(vec, [11, 12, 13]); + /// + /// assert_eq!(map.len(), 3); /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn values_mut(&mut self) -> ValuesMut<'_, K, V> { @@ -459,10 +641,20 @@ impl HashMap { /// map.insert("a", 1); /// map.insert("b", 2); /// map.insert("c", 3); + /// assert_eq!(map.len(), 3); + /// let mut vec: Vec<(&str, i32)> = Vec::new(); /// /// for (key, val) in map.iter() { /// println!("key: {} val: {}", key, val); + /// vec.push((*key, *val)); /// } + /// + /// // The `Iter` iterator produces items in arbitrary order, so the + /// // items must be sorted to test them against a sorted array. + /// vec.sort_unstable(); + /// assert_eq!(vec, [("a", 1), ("b", 2), ("c", 3)]); + /// + /// assert_eq!(map.len(), 3); /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn iter(&self) -> Iter<'_, K, V> { @@ -494,9 +686,20 @@ impl HashMap { /// *val *= 2; /// } /// + /// assert_eq!(map.len(), 3); + /// let mut vec: Vec<(&str, i32)> = Vec::new(); + /// /// for (key, val) in &map { /// println!("key: {} val: {}", key, val); + /// vec.push((*key, *val)); /// } + /// + /// // The `Iter` iterator produces items in arbitrary order, so the + /// // items must be sorted to test them against a sorted array. + /// vec.sort_unstable(); + /// assert_eq!(vec, [("a", 2), ("b", 4), ("c", 6)]); + /// + /// assert_eq!(map.len(), 3); /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn iter_mut(&mut self) -> IterMut<'_, K, V> { @@ -552,6 +755,10 @@ impl HashMap { /// Clears the map, returning all key-value pairs as an iterator. Keeps the /// allocated memory for reuse. /// + /// If the returned iterator is dropped before being fully consumed, it + /// drops the remaining key-value pairs. The returned iterator keeps a + /// mutable borrow on the vector to optimize its implementation. + /// /// # Examples /// /// ``` @@ -560,24 +767,138 @@ impl HashMap { /// let mut a = HashMap::new(); /// a.insert(1, "a"); /// a.insert(2, "b"); + /// let capacity_before_drain = a.capacity(); /// /// for (k, v) in a.drain().take(1) { /// assert!(k == 1 || k == 2); /// assert!(v == "a" || v == "b"); /// } /// + /// // As we can see, the map is empty and contains no element. + /// assert!(a.is_empty() && a.len() == 0); + /// // But map capacity is equal to old one. + /// assert_eq!(a.capacity(), capacity_before_drain); + /// + /// let mut a = HashMap::new(); + /// a.insert(1, "a"); + /// a.insert(2, "b"); + /// + /// { // Iterator is dropped without being consumed. + /// let d = a.drain(); + /// } + /// + /// // But the map is empty even if we do not use Drain iterator. /// assert!(a.is_empty()); /// ``` #[cfg_attr(feature = "inline-more", inline)] - pub fn drain(&mut self) -> Drain<'_, K, V> { - // Here we tie the lifetime of self to the iter. + pub fn drain(&mut self) -> Drain<'_, K, V, A> { + Drain { + inner: self.table.drain(), + } + } + + /// Retains only the elements specified by the predicate. Keeps the + /// allocated memory for reuse. + /// + /// In other words, remove all pairs `(k, v)` such that `f(&k, &mut v)` returns `false`. + /// The elements are visited in unsorted (and unspecified) order. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let mut map: HashMap = (0..8).map(|x|(x, x*10)).collect(); + /// assert_eq!(map.len(), 8); + /// let capacity_before_retain = map.capacity(); + /// + /// map.retain(|&k, _| k % 2 == 0); + /// + /// // We can see, that the number of elements inside map is changed. + /// assert_eq!(map.len(), 4); + /// // But map capacity is equal to old one. + /// assert_eq!(map.capacity(), capacity_before_retain); + /// + /// let mut vec: Vec<(i32, i32)> = map.iter().map(|(&k, &v)| (k, v)).collect(); + /// vec.sort_unstable(); + /// assert_eq!(vec, [(0, 0), (2, 20), (4, 40), (6, 60)]); + /// ``` + pub fn retain(&mut self, mut f: F) + where + F: FnMut(&K, &mut V) -> bool, + { + // Here we only use `iter` as a temporary, preventing use-after-free unsafe { - Drain { - inner: self.table.drain(), + for item in self.table.iter() { + let &mut (ref key, ref mut value) = item.as_mut(); + if !f(key, value) { + self.table.erase(item); + } } } } + /// Drains elements which are true under the given predicate, + /// and returns an iterator over the removed items. + /// + /// In other words, move all pairs `(k, v)` such that `f(&k, &mut v)` returns `true` out + /// into another iterator. + /// + /// Note that `drain_filter` lets you mutate every value in the filter closure, regardless of + /// whether you choose to keep or remove it. + /// + /// When the returned DrainedFilter is dropped, any remaining elements that satisfy + /// the predicate are dropped from the table. + /// + /// It is unspecified how many more elements will be subjected to the closure + /// if a panic occurs in the closure, or a panic occurs while dropping an element, + /// or if the `DrainFilter` value is leaked. + /// + /// Keeps the allocated memory for reuse. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let mut map: HashMap = (0..8).map(|x| (x, x)).collect(); + /// let capacity_before_drain_filter = map.capacity(); + /// let drained: HashMap = map.drain_filter(|k, _v| k % 2 == 0).collect(); + /// + /// let mut evens = drained.keys().cloned().collect::>(); + /// let mut odds = map.keys().cloned().collect::>(); + /// evens.sort(); + /// odds.sort(); + /// + /// assert_eq!(evens, vec![0, 2, 4, 6]); + /// assert_eq!(odds, vec![1, 3, 5, 7]); + /// // Map capacity is equal to old one. + /// assert_eq!(map.capacity(), capacity_before_drain_filter); + /// + /// let mut map: HashMap = (0..8).map(|x| (x, x)).collect(); + /// + /// { // Iterator is dropped without being consumed. + /// let d = map.drain_filter(|k, _v| k % 2 != 0); + /// } + /// + /// // But the map lens have been reduced by half + /// // even if we do not use DrainFilter iterator. + /// assert_eq!(map.len(), 4); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn drain_filter(&mut self, f: F) -> DrainFilter<'_, K, V, F, A> + where + F: FnMut(&K, &mut V) -> bool, + { + DrainFilter { + f, + inner: DrainFilterInner { + iter: unsafe { self.table.iter() }, + table: &mut self.table, + }, + } + } + /// Clears the map, removing all key-value pairs. Keeps the allocated memory /// for reuse. /// @@ -588,19 +909,82 @@ impl HashMap { /// /// let mut a = HashMap::new(); /// a.insert(1, "a"); + /// let capacity_before_clear = a.capacity(); + /// /// a.clear(); + /// + /// // Map is empty. /// assert!(a.is_empty()); + /// // But map capacity is equal to old one. + /// assert_eq!(a.capacity(), capacity_before_clear); /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn clear(&mut self) { self.table.clear(); } + + /// Creates a consuming iterator visiting all the keys in arbitrary order. + /// The map cannot be used after calling this. + /// The iterator element type is `K`. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let mut map = HashMap::new(); + /// map.insert("a", 1); + /// map.insert("b", 2); + /// map.insert("c", 3); + /// + /// let mut vec: Vec<&str> = map.into_keys().collect(); + /// + /// // The `IntoKeys` iterator produces keys in arbitrary order, so the + /// // keys must be sorted to test them against a sorted array. + /// vec.sort_unstable(); + /// assert_eq!(vec, ["a", "b", "c"]); + /// ``` + #[inline] + pub fn into_keys(self) -> IntoKeys { + IntoKeys { + inner: self.into_iter(), + } + } + + /// Creates a consuming iterator visiting all the values in arbitrary order. + /// The map cannot be used after calling this. + /// The iterator element type is `V`. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let mut map = HashMap::new(); + /// map.insert("a", 1); + /// map.insert("b", 2); + /// map.insert("c", 3); + /// + /// let mut vec: Vec = map.into_values().collect(); + /// + /// // The `IntoValues` iterator produces values in arbitrary order, so + /// // the values must be sorted to test them against a sorted array. + /// vec.sort_unstable(); + /// assert_eq!(vec, [1, 2, 3]); + /// ``` + #[inline] + pub fn into_values(self) -> IntoValues { + IntoValues { + inner: self.into_iter(), + } + } } -impl HashMap +impl HashMap where K: Eq + Hash, S: BuildHasher, + A: Allocator + Clone, { /// Reserves capacity for at least `additional` more elements to be inserted /// in the `HashMap`. The collection may reserve more space to avoid @@ -617,13 +1001,18 @@ where /// ``` /// use hashbrown::HashMap; /// let mut map: HashMap<&str, i32> = HashMap::new(); + /// // Map is empty and doesn't allocate memory + /// assert_eq!(map.capacity(), 0); + /// /// map.reserve(10); + /// + /// // And now map can hold at least 10 elements + /// assert!(map.capacity() >= 10); /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn reserve(&mut self, additional: usize) { - let hash_builder = &self.hash_builder; self.table - .reserve(additional, |x| make_hash(hash_builder, &x.0)); + .reserve(additional, make_hasher::(&self.hash_builder)); } /// Tries to reserve capacity for at least `additional` more elements to be inserted @@ -639,16 +1028,43 @@ where /// /// ``` /// use hashbrown::HashMap; + /// /// let mut map: HashMap<&str, isize> = HashMap::new(); + /// // Map is empty and doesn't allocate memory + /// assert_eq!(map.capacity(), 0); + /// /// map.try_reserve(10).expect("why is the test harness OOMing on 10 bytes?"); + /// + /// // And now map can hold at least 10 elements + /// assert!(map.capacity() >= 10); /// ``` - #[cfg_attr(feature = "inline-more", inline)] - pub fn try_reserve(&mut self, additional: usize) -> Result<(), CollectionAllocErr> { - let hash_builder = &self.hash_builder; - self.table - .try_reserve(additional, |x| make_hash(hash_builder, &x.0)) - } - + /// If the capacity overflows, or the allocator reports a failure, then an error + /// is returned: + /// ``` + /// # fn test() { + /// use hashbrown::HashMap; + /// use hashbrown::TryReserveError; + /// let mut map: HashMap = HashMap::new(); + /// + /// match map.try_reserve(usize::MAX) { + /// Err(error) => match error { + /// TryReserveError::CapacityOverflow => {} + /// _ => panic!("TryReserveError::AllocError ?"), + /// }, + /// _ => panic!(), + /// } + /// # } + /// # fn main() { + /// # #[cfg(not(miri))] + /// # test() + /// # } + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.table + .try_reserve(additional, make_hasher::(&self.hash_builder)) + } + /// Shrinks the capacity of the map as much as possible. It will drop /// down as much as possible while maintaining the internal rules /// and possibly leaving some space in accordance with the resize policy. @@ -667,8 +1083,8 @@ where /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn shrink_to_fit(&mut self) { - let hash_builder = &self.hash_builder; - self.table.shrink_to(0, |x| make_hash(hash_builder, &x.0)); + self.table + .shrink_to(0, make_hasher::(&self.hash_builder)); } /// Shrinks the capacity of the map with a lower limit. It will drop @@ -696,9 +1112,8 @@ where /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn shrink_to(&mut self, min_capacity: usize) { - let hash_builder = &self.hash_builder; self.table - .shrink_to(min_capacity, |x| make_hash(hash_builder, &x.0)); + .shrink_to(min_capacity, make_hasher::(&self.hash_builder)); } /// Gets the given key's corresponding entry in the map for in-place manipulation. @@ -721,10 +1136,11 @@ where /// assert_eq!(letters.get(&'y'), None); /// ``` #[cfg_attr(feature = "inline-more", inline)] - pub fn entry(&mut self, key: K) -> Entry<'_, K, V, S> { - let hash = make_hash(&self.hash_builder, &key); - if let Some(elem) = self.table.find(hash, |q| q.0.eq(&key)) { + pub fn entry(&mut self, key: K) -> Entry<'_, K, V, S, A> { + let hash = make_insert_hash::(&self.hash_builder, &key); + if let Some(elem) = self.table.find(hash, equivalent_key(&key)) { Entry::Occupied(OccupiedEntry { + hash, key: Some(key), elem, table: self, @@ -738,6 +1154,46 @@ where } } + /// Gets the given key's corresponding entry by reference in the map for in-place manipulation. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let mut words: HashMap = HashMap::new(); + /// let source = ["poneyland", "horseyland", "poneyland", "poneyland"]; + /// for (i, &s) in source.iter().enumerate() { + /// let counter = words.entry_ref(s).or_insert(0); + /// *counter += 1; + /// } + /// + /// assert_eq!(words["poneyland"], 3); + /// assert_eq!(words["horseyland"], 1); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn entry_ref<'a, 'b, Q: ?Sized>(&'a mut self, key: &'b Q) -> EntryRef<'a, 'b, K, Q, V, S, A> + where + K: Borrow, + Q: Hash + Eq, + { + let hash = make_hash::(&self.hash_builder, key); + if let Some(elem) = self.table.find(hash, equivalent_key(key)) { + EntryRef::Occupied(OccupiedEntryRef { + hash, + key: Some(KeyOrRef::Borrowed(key)), + elem, + table: self, + }) + } else { + EntryRef::Vacant(VacantEntryRef { + hash, + key: KeyOrRef::Borrowed(key), + table: self, + }) + } + } + /// Returns a reference to the value corresponding to the key. /// /// The key may be any borrowed form of the map's key type, but @@ -763,7 +1219,11 @@ where K: Borrow, Q: Hash + Eq, { - self.get_key_value(k).map(|(_, v)| v) + // Avoid `Option::map` because it bloats LLVM IR. + match self.get_inner(k) { + Some(&(_, ref v)) => Some(v), + None => None, + } } /// Returns the key-value pair corresponding to the supplied key. @@ -791,13 +1251,25 @@ where K: Borrow, Q: Hash + Eq, { - let hash = make_hash(&self.hash_builder, k); - self.table - .find(hash, |x| k.eq(x.0.borrow())) - .map(|item| unsafe { - let &(ref key, ref value) = item.as_ref(); - (key, value) - }) + // Avoid `Option::map` because it bloats LLVM IR. + match self.get_inner(k) { + Some(&(ref key, ref value)) => Some((key, value)), + None => None, + } + } + + #[inline] + fn get_inner(&self, k: &Q) -> Option<&(K, V)> + where + K: Borrow, + Q: Hash + Eq, + { + if self.table.is_empty() { + None + } else { + let hash = make_hash::(&self.hash_builder, k); + self.table.get(hash, equivalent_key(k)) + } } /// Returns the key-value pair corresponding to the supplied key, with a mutable reference to value. @@ -829,13 +1301,11 @@ where K: Borrow, Q: Hash + Eq, { - let hash = make_hash(&self.hash_builder, k); - self.table - .find(hash, |x| k.eq(x.0.borrow())) - .map(|item| unsafe { - let &mut (ref key, ref mut value) = item.as_mut(); - (key, value) - }) + // Avoid `Option::map` because it bloats LLVM IR. + match self.get_inner_mut(k) { + Some(&mut (ref key, ref mut value)) => Some((key, value)), + None => None, + } } /// Returns `true` if the map contains a value for the specified key. @@ -863,7 +1333,7 @@ where K: Borrow, Q: Hash + Eq, { - self.get(k).is_some() + self.get_inner(k).is_some() } /// Returns a mutable reference to the value corresponding to the key. @@ -886,6 +1356,8 @@ where /// *x = "b"; /// } /// assert_eq!(map[&1], "b"); + /// + /// assert_eq!(map.get_mut(&2), None); /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn get_mut(&mut self, k: &Q) -> Option<&mut V> @@ -893,10 +1365,283 @@ where K: Borrow, Q: Hash + Eq, { - let hash = make_hash(&self.hash_builder, k); + // Avoid `Option::map` because it bloats LLVM IR. + match self.get_inner_mut(k) { + Some(&mut (_, ref mut v)) => Some(v), + None => None, + } + } + + #[inline] + fn get_inner_mut(&mut self, k: &Q) -> Option<&mut (K, V)> + where + K: Borrow, + Q: Hash + Eq, + { + if self.table.is_empty() { + None + } else { + let hash = make_hash::(&self.hash_builder, k); + self.table.get_mut(hash, equivalent_key(k)) + } + } + + /// Attempts to get mutable references to `N` values in the map at once. + /// + /// Returns an array of length `N` with the results of each query. For soundness, at most one + /// mutable reference will be returned to any value. `None` will be returned if any of the + /// keys are duplicates or missing. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let mut libraries = HashMap::new(); + /// libraries.insert("Bodleian Library".to_string(), 1602); + /// libraries.insert("Athenæum".to_string(), 1807); + /// libraries.insert("Herzogin-Anna-Amalia-Bibliothek".to_string(), 1691); + /// libraries.insert("Library of Congress".to_string(), 1800); + /// + /// let got = libraries.get_many_mut([ + /// "Athenæum", + /// "Library of Congress", + /// ]); + /// assert_eq!( + /// got, + /// Some([ + /// &mut 1807, + /// &mut 1800, + /// ]), + /// ); + /// + /// // Missing keys result in None + /// let got = libraries.get_many_mut([ + /// "Athenæum", + /// "New York Public Library", + /// ]); + /// assert_eq!(got, None); + /// + /// // Duplicate keys result in None + /// let got = libraries.get_many_mut([ + /// "Athenæum", + /// "Athenæum", + /// ]); + /// assert_eq!(got, None); + /// ``` + pub fn get_many_mut(&mut self, ks: [&Q; N]) -> Option<[&'_ mut V; N]> + where + K: Borrow, + Q: Hash + Eq, + { + self.get_many_mut_inner(ks).map(|res| res.map(|(_, v)| v)) + } + + /// Attempts to get mutable references to `N` values in the map at once, without validating that + /// the values are unique. + /// + /// Returns an array of length `N` with the results of each query. `None` will be returned if + /// any of the keys are missing. + /// + /// For a safe alternative see [`get_many_mut`](`HashMap::get_many_mut`). + /// + /// # Safety + /// + /// Calling this method with overlapping keys is *[undefined behavior]* even if the resulting + /// references are not used. + /// + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let mut libraries = HashMap::new(); + /// libraries.insert("Bodleian Library".to_string(), 1602); + /// libraries.insert("Athenæum".to_string(), 1807); + /// libraries.insert("Herzogin-Anna-Amalia-Bibliothek".to_string(), 1691); + /// libraries.insert("Library of Congress".to_string(), 1800); + /// + /// let got = libraries.get_many_mut([ + /// "Athenæum", + /// "Library of Congress", + /// ]); + /// assert_eq!( + /// got, + /// Some([ + /// &mut 1807, + /// &mut 1800, + /// ]), + /// ); + /// + /// // Missing keys result in None + /// let got = libraries.get_many_mut([ + /// "Athenæum", + /// "New York Public Library", + /// ]); + /// assert_eq!(got, None); + /// ``` + pub unsafe fn get_many_unchecked_mut( + &mut self, + ks: [&Q; N], + ) -> Option<[&'_ mut V; N]> + where + K: Borrow, + Q: Hash + Eq, + { + self.get_many_unchecked_mut_inner(ks) + .map(|res| res.map(|(_, v)| v)) + } + + /// Attempts to get mutable references to `N` values in the map at once, with immutable + /// references to the corresponding keys. + /// + /// Returns an array of length `N` with the results of each query. For soundness, at most one + /// mutable reference will be returned to any value. `None` will be returned if any of the keys + /// are duplicates or missing. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let mut libraries = HashMap::new(); + /// libraries.insert("Bodleian Library".to_string(), 1602); + /// libraries.insert("Athenæum".to_string(), 1807); + /// libraries.insert("Herzogin-Anna-Amalia-Bibliothek".to_string(), 1691); + /// libraries.insert("Library of Congress".to_string(), 1800); + /// + /// let got = libraries.get_many_key_value_mut([ + /// "Bodleian Library", + /// "Herzogin-Anna-Amalia-Bibliothek", + /// ]); + /// assert_eq!( + /// got, + /// Some([ + /// (&"Bodleian Library".to_string(), &mut 1602), + /// (&"Herzogin-Anna-Amalia-Bibliothek".to_string(), &mut 1691), + /// ]), + /// ); + /// // Missing keys result in None + /// let got = libraries.get_many_key_value_mut([ + /// "Bodleian Library", + /// "Gewandhaus", + /// ]); + /// assert_eq!(got, None); + /// + /// // Duplicate keys result in None + /// let got = libraries.get_many_key_value_mut([ + /// "Bodleian Library", + /// "Herzogin-Anna-Amalia-Bibliothek", + /// "Herzogin-Anna-Amalia-Bibliothek", + /// ]); + /// assert_eq!(got, None); + /// ``` + pub fn get_many_key_value_mut( + &mut self, + ks: [&Q; N], + ) -> Option<[(&'_ K, &'_ mut V); N]> + where + K: Borrow, + Q: Hash + Eq, + { + self.get_many_mut_inner(ks) + .map(|res| res.map(|(k, v)| (&*k, v))) + } + + /// Attempts to get mutable references to `N` values in the map at once, with immutable + /// references to the corresponding keys, without validating that the values are unique. + /// + /// Returns an array of length `N` with the results of each query. `None` will be returned if + /// any of the keys are missing. + /// + /// For a safe alternative see [`get_many_key_value_mut`](`HashMap::get_many_key_value_mut`). + /// + /// # Safety + /// + /// Calling this method with overlapping keys is *[undefined behavior]* even if the resulting + /// references are not used. + /// + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let mut libraries = HashMap::new(); + /// libraries.insert("Bodleian Library".to_string(), 1602); + /// libraries.insert("Athenæum".to_string(), 1807); + /// libraries.insert("Herzogin-Anna-Amalia-Bibliothek".to_string(), 1691); + /// libraries.insert("Library of Congress".to_string(), 1800); + /// + /// let got = libraries.get_many_key_value_mut([ + /// "Bodleian Library", + /// "Herzogin-Anna-Amalia-Bibliothek", + /// ]); + /// assert_eq!( + /// got, + /// Some([ + /// (&"Bodleian Library".to_string(), &mut 1602), + /// (&"Herzogin-Anna-Amalia-Bibliothek".to_string(), &mut 1691), + /// ]), + /// ); + /// // Missing keys result in None + /// let got = libraries.get_many_key_value_mut([ + /// "Bodleian Library", + /// "Gewandhaus", + /// ]); + /// assert_eq!(got, None); + /// ``` + pub unsafe fn get_many_key_value_unchecked_mut( + &mut self, + ks: [&Q; N], + ) -> Option<[(&'_ K, &'_ mut V); N]> + where + K: Borrow, + Q: Hash + Eq, + { + self.get_many_unchecked_mut_inner(ks) + .map(|res| res.map(|(k, v)| (&*k, v))) + } + + fn get_many_mut_inner( + &mut self, + ks: [&Q; N], + ) -> Option<[&'_ mut (K, V); N]> + where + K: Borrow, + Q: Hash + Eq, + { + let hashes = self.build_hashes_inner(ks); + self.table + .get_many_mut(hashes, |i, (k, _)| ks[i].eq(k.borrow())) + } + + unsafe fn get_many_unchecked_mut_inner( + &mut self, + ks: [&Q; N], + ) -> Option<[&'_ mut (K, V); N]> + where + K: Borrow, + Q: Hash + Eq, + { + let hashes = self.build_hashes_inner(ks); self.table - .find(hash, |x| k.eq(x.0.borrow())) - .map(|item| unsafe { &mut item.as_mut().1 }) + .get_many_unchecked_mut(hashes, |i, (k, _)| ks[i].eq(k.borrow())) + } + + fn build_hashes_inner(&self, ks: [&Q; N]) -> [u64; N] + where + K: Borrow, + Q: Hash + Eq, + { + let mut hashes = [0_u64; N]; + for i in 0..N { + hashes[i] = make_hash::(&self.hash_builder, ks[i]); + } + hashes } /// Inserts a key-value pair into the map. @@ -905,11 +1650,12 @@ where /// /// If the map did have this key present, the value is updated, and the old /// value is returned. The key is not updated, though; this matters for - /// types that can be `==` without being identical. See the [module-level - /// documentation] for more. + /// types that can be `==` without being identical. See the [`std::collections`] + /// [module-level documentation] for more. /// /// [`None`]: https://doc.rust-lang.org/std/option/enum.Option.html#variant.None - /// [module-level documentation]: index.html#insert-and-complex-keys + /// [`std::collections`]: https://doc.rust-lang.org/std/collections/index.html + /// [module-level documentation]: https://doc.rust-lang.org/std/collections/index.html#insert-and-complex-keys /// /// # Examples /// @@ -926,21 +1672,117 @@ where /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn insert(&mut self, k: K, v: V) -> Option { - unsafe { - let hash = make_hash(&self.hash_builder, &k); - if let Some(item) = self.table.find(hash, |x| k.eq(&x.0)) { - Some(mem::replace(&mut item.as_mut().1, v)) - } else { - let hash_builder = &self.hash_builder; - self.table - .insert(hash, (k, v), |x| make_hash(hash_builder, &x.0)); - None - } + let hash = make_insert_hash::(&self.hash_builder, &k); + if let Some((_, item)) = self.table.get_mut(hash, equivalent_key(&k)) { + Some(mem::replace(item, v)) + } else { + self.table + .insert(hash, (k, v), make_hasher::(&self.hash_builder)); + None + } + } + + /// Insert a key-value pair into the map without checking + /// if the key already exists in the map. + /// + /// Returns a reference to the key and value just inserted. + /// + /// This operation is safe if a key does not exist in the map. + /// + /// However, if a key exists in the map already, the behavior is unspecified: + /// this operation may panic, loop forever, or any following operation with the map + /// may panic, loop forever or return arbitrary result. + /// + /// That said, this operation (and following operations) are guaranteed to + /// not violate memory safety. + /// + /// This operation is faster than regular insert, because it does not perform + /// lookup before insertion. + /// + /// This operation is useful during initial population of the map. + /// For example, when constructing a map from another map, we know + /// that keys are unique. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let mut map1 = HashMap::new(); + /// assert_eq!(map1.insert(1, "a"), None); + /// assert_eq!(map1.insert(2, "b"), None); + /// assert_eq!(map1.insert(3, "c"), None); + /// assert_eq!(map1.len(), 3); + /// + /// let mut map2 = HashMap::new(); + /// + /// for (key, value) in map1.into_iter() { + /// map2.insert_unique_unchecked(key, value); + /// } + /// + /// let (key, value) = map2.insert_unique_unchecked(4, "d"); + /// assert_eq!(key, &4); + /// assert_eq!(value, &mut "d"); + /// *value = "e"; + /// + /// assert_eq!(map2[&1], "a"); + /// assert_eq!(map2[&2], "b"); + /// assert_eq!(map2[&3], "c"); + /// assert_eq!(map2[&4], "e"); + /// assert_eq!(map2.len(), 4); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn insert_unique_unchecked(&mut self, k: K, v: V) -> (&K, &mut V) { + let hash = make_insert_hash::(&self.hash_builder, &k); + let bucket = self + .table + .insert(hash, (k, v), make_hasher::(&self.hash_builder)); + let (k_ref, v_ref) = unsafe { bucket.as_mut() }; + (k_ref, v_ref) + } + + /// Tries to insert a key-value pair into the map, and returns + /// a mutable reference to the value in the entry. + /// + /// # Errors + /// + /// If the map already had this key present, nothing is updated, and + /// an error containing the occupied entry and the value is returned. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use hashbrown::HashMap; + /// use hashbrown::hash_map::OccupiedError; + /// + /// let mut map = HashMap::new(); + /// assert_eq!(map.try_insert(37, "a").unwrap(), &"a"); + /// + /// match map.try_insert(37, "b") { + /// Err(OccupiedError { entry, value }) => { + /// assert_eq!(entry.key(), &37); + /// assert_eq!(entry.get(), &"a"); + /// assert_eq!(value, "b"); + /// } + /// _ => panic!() + /// } + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn try_insert( + &mut self, + key: K, + value: V, + ) -> Result<&mut V, OccupiedError<'_, K, V, S, A>> { + match self.entry(key) { + Entry::Occupied(entry) => Err(OccupiedError { entry, value }), + Entry::Vacant(entry) => Ok(entry.insert(value)), } } /// Removes a key from the map, returning the value at the key if the key - /// was previously in the map. + /// was previously in the map. Keeps the allocated memory for reuse. /// /// The key may be any borrowed form of the map's key type, but /// [`Hash`] and [`Eq`] on the borrowed form *must* match those for @@ -955,9 +1797,17 @@ where /// use hashbrown::HashMap; /// /// let mut map = HashMap::new(); + /// // The map is empty + /// assert!(map.is_empty() && map.capacity() == 0); + /// /// map.insert(1, "a"); + /// let capacity_before_remove = map.capacity(); + /// /// assert_eq!(map.remove(&1), Some("a")); /// assert_eq!(map.remove(&1), None); + /// + /// // Now map holds none elements but capacity is equal to the old one + /// assert!(map.len() == 0 && map.capacity() == capacity_before_remove); /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn remove(&mut self, k: &Q) -> Option @@ -965,11 +1815,15 @@ where K: Borrow, Q: Hash + Eq, { - self.remove_entry(k).map(|(_, v)| v) + // Avoid `Option::map` because it bloats LLVM IR. + match self.remove_entry(k) { + Some((_, v)) => Some(v), + None => None, + } } /// Removes a key from the map, returning the stored key and value if the - /// key was previously in the map. + /// key was previously in the map. Keeps the allocated memory for reuse. /// /// The key may be any borrowed form of the map's key type, but /// [`Hash`] and [`Eq`] on the borrowed form *must* match those for @@ -984,9 +1838,17 @@ where /// use hashbrown::HashMap; /// /// let mut map = HashMap::new(); + /// // The map is empty + /// assert!(map.is_empty() && map.capacity() == 0); + /// /// map.insert(1, "a"); + /// let capacity_before_remove = map.capacity(); + /// /// assert_eq!(map.remove_entry(&1), Some((1, "a"))); /// assert_eq!(map.remove(&1), None); + /// + /// // Now map hold none elements but capacity is equal to the old one + /// assert!(map.len() == 0 && map.capacity() == capacity_before_remove); /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn remove_entry(&mut self, k: &Q) -> Option<(K, V)> @@ -994,111 +1856,111 @@ where K: Borrow, Q: Hash + Eq, { - unsafe { - let hash = make_hash(&self.hash_builder, &k); - if let Some(item) = self.table.find(hash, |x| k.eq(x.0.borrow())) { - self.table.erase_no_drop(&item); - Some(item.read()) - } else { - None - } - } + let hash = make_hash::(&self.hash_builder, k); + self.table.remove_entry(hash, equivalent_key(k)) } +} - /// Retains only the elements specified by the predicate. - /// - /// In other words, remove all pairs `(k, v)` such that `f(&k,&mut v)` returns `false`. - /// - /// # Examples +impl HashMap { + /// Creates a raw entry builder for the HashMap. /// - /// ``` - /// use hashbrown::HashMap; + /// Raw entries provide the lowest level of control for searching and + /// manipulating a map. They must be manually initialized with a hash and + /// then manually searched. After this, insertions into a vacant entry + /// still require an owned key to be provided. /// - /// let mut map: HashMap = (0..8).map(|x|(x, x*10)).collect(); - /// map.retain(|&k, _| k % 2 == 0); - /// assert_eq!(map.len(), 4); - /// ``` - pub fn retain(&mut self, mut f: F) - where - F: FnMut(&K, &mut V) -> bool, - { - // Here we only use `iter` as a temporary, preventing use-after-free - unsafe { - for item in self.table.iter() { - let &mut (ref key, ref mut value) = item.as_mut(); - if !f(key, value) { - // Erase the element from the table first since drop might panic. - self.table.erase_no_drop(&item); - item.drop(); - } - } - } - } - /// Drains elements which are false under the given predicate, - /// and returns an iterator over the removed items. + /// Raw entries are useful for such exotic situations as: /// - /// In other words, move all pairs `(k, v)` such that `f(&k,&mut v)` returns `false` out - /// into another iterator. + /// * Hash memoization + /// * Deferring the creation of an owned key until it is known to be required + /// * Using a search key that doesn't work with the Borrow trait + /// * Using custom comparison logic without newtype wrappers /// - /// When the returned DrainedFilter is dropped, the elements that don't satisfy - /// the predicate are dropped from the table. + /// Because raw entries provide much more low-level control, it's much easier + /// to put the HashMap into an inconsistent state which, while memory-safe, + /// will cause the map to produce seemingly random results. Higher-level and + /// more foolproof APIs like `entry` should be preferred when possible. /// - /// # Examples + /// In particular, the hash used to initialized the raw entry must still be + /// consistent with the hash of the key that is ultimately stored in the entry. + /// This is because implementations of HashMap may need to recompute hashes + /// when resizing, at which point only the keys are available. /// - /// ``` - /// use hashbrown::HashMap; + /// Raw entries give mutable access to the keys. This must not be used + /// to modify how the key would compare or hash, as the map will not re-evaluate + /// where the key should go, meaning the keys may become "lost" if their + /// location does not reflect their state. For instance, if you change a key + /// so that the map now contains keys which compare equal, search may start + /// acting erratically, with two keys randomly masking each other. Implementations + /// are free to assume this doesn't happen (within the limits of memory-safety). + /// + /// # Examples /// - /// let mut map: HashMap = (0..8).map(|x|(x, x*10)).collect(); - /// let drained = map.drain_filter(|&k, _| k % 2 == 0); - /// assert_eq!(drained.count(), 4); - /// assert_eq!(map.len(), 4); /// ``` - pub fn drain_filter(&mut self, f: F) -> DrainFilter<'_, K, V, F> - where - F: FnMut(&K, &mut V) -> bool, - { - DrainFilter { - f, - iter: unsafe { self.table.iter() }, - table: &mut self.table, - } - } -} - -impl HashMap { - /// Creates a raw entry builder for the HashMap. + /// use core::hash::{BuildHasher, Hash}; + /// use hashbrown::hash_map::{HashMap, RawEntryMut}; /// - /// Raw entries provide the lowest level of control for searching and - /// manipulating a map. They must be manually initialized with a hash and - /// then manually searched. After this, insertions into a vacant entry - /// still require an owned key to be provided. + /// let mut map = HashMap::new(); + /// map.extend([("a", 100), ("b", 200), ("c", 300)]); /// - /// Raw entries are useful for such exotic situations as: + /// fn compute_hash(hash_builder: &S, key: &K) -> u64 { + /// use core::hash::Hasher; + /// let mut state = hash_builder.build_hasher(); + /// key.hash(&mut state); + /// state.finish() + /// } /// - /// * Hash memoization - /// * Deferring the creation of an owned key until it is known to be required - /// * Using a search key that doesn't work with the Borrow trait - /// * Using custom comparison logic without newtype wrappers + /// // Existing key (insert and update) + /// match map.raw_entry_mut().from_key(&"a") { + /// RawEntryMut::Vacant(_) => unreachable!(), + /// RawEntryMut::Occupied(mut view) => { + /// assert_eq!(view.get(), &100); + /// let v = view.get_mut(); + /// let new_v = (*v) * 10; + /// *v = new_v; + /// assert_eq!(view.insert(1111), 1000); + /// } + /// } /// - /// Because raw entries provide much more low-level control, it's much easier - /// to put the HashMap into an inconsistent state which, while memory-safe, - /// will cause the map to produce seemingly random results. Higher-level and - /// more foolproof APIs like `entry` should be preferred when possible. + /// assert_eq!(map[&"a"], 1111); + /// assert_eq!(map.len(), 3); /// - /// In particular, the hash used to initialized the raw entry must still be - /// consistent with the hash of the key that is ultimately stored in the entry. - /// This is because implementations of HashMap may need to recompute hashes - /// when resizing, at which point only the keys are available. + /// // Existing key (take) + /// let hash = compute_hash(map.hasher(), &"c"); + /// match map.raw_entry_mut().from_key_hashed_nocheck(hash, &"c") { + /// RawEntryMut::Vacant(_) => unreachable!(), + /// RawEntryMut::Occupied(view) => { + /// assert_eq!(view.remove_entry(), ("c", 300)); + /// } + /// } + /// assert_eq!(map.raw_entry().from_key(&"c"), None); + /// assert_eq!(map.len(), 2); + /// + /// // Nonexistent key (insert and update) + /// let key = "d"; + /// let hash = compute_hash(map.hasher(), &key); + /// match map.raw_entry_mut().from_hash(hash, |q| *q == key) { + /// RawEntryMut::Occupied(_) => unreachable!(), + /// RawEntryMut::Vacant(view) => { + /// let (k, value) = view.insert("d", 4000); + /// assert_eq!((*k, *value), ("d", 4000)); + /// *value = 40000; + /// } + /// } + /// assert_eq!(map[&"d"], 40000); + /// assert_eq!(map.len(), 3); /// - /// Raw entries give mutable access to the keys. This must not be used - /// to modify how the key would compare or hash, as the map will not re-evaluate - /// where the key should go, meaning the keys may become "lost" if their - /// location does not reflect their state. For instance, if you change a key - /// so that the map now contains keys which compare equal, search may start - /// acting erratically, with two keys randomly masking each other. Implementations - /// are free to assume this doesn't happen (within the limits of memory-safety). + /// match map.raw_entry_mut().from_hash(hash, |q| *q == key) { + /// RawEntryMut::Vacant(_) => unreachable!(), + /// RawEntryMut::Occupied(view) => { + /// assert_eq!(view.remove_entry(), ("d", 40000)); + /// } + /// } + /// assert_eq!(map.get(&"d"), None); + /// assert_eq!(map.len(), 2); + /// ``` #[cfg_attr(feature = "inline-more", inline)] - pub fn raw_entry_mut(&mut self) -> RawEntryBuilderMut<'_, K, V, S> { + pub fn raw_entry_mut(&mut self) -> RawEntryBuilderMut<'_, K, V, S, A> { RawEntryBuilderMut { map: self } } @@ -1117,17 +1979,108 @@ impl HashMap { /// `get` should be preferred. /// /// Immutable raw entries have very limited use; you might instead want `raw_entry_mut`. + /// + /// # Examples + /// + /// ``` + /// use core::hash::{BuildHasher, Hash}; + /// use hashbrown::HashMap; + /// + /// let mut map = HashMap::new(); + /// map.extend([("a", 100), ("b", 200), ("c", 300)]); + /// + /// fn compute_hash(hash_builder: &S, key: &K) -> u64 { + /// use core::hash::Hasher; + /// let mut state = hash_builder.build_hasher(); + /// key.hash(&mut state); + /// state.finish() + /// } + /// + /// for k in ["a", "b", "c", "d", "e", "f"] { + /// let hash = compute_hash(map.hasher(), k); + /// let v = map.get(&k).cloned(); + /// let kv = v.as_ref().map(|v| (&k, v)); + /// + /// println!("Key: {} and value: {:?}", k, v); + /// + /// assert_eq!(map.raw_entry().from_key(&k), kv); + /// assert_eq!(map.raw_entry().from_hash(hash, |q| *q == k), kv); + /// assert_eq!(map.raw_entry().from_key_hashed_nocheck(hash, &k), kv); + /// } + /// ``` #[cfg_attr(feature = "inline-more", inline)] - pub fn raw_entry(&self) -> RawEntryBuilder<'_, K, V, S> { + pub fn raw_entry(&self) -> RawEntryBuilder<'_, K, V, S, A> { RawEntryBuilder { map: self } } + + /// Returns a mutable reference to the [`RawTable`] used underneath [`HashMap`]. + /// This function is only available if the `raw` feature of the crate is enabled. + /// + /// # Note + /// + /// Calling the function safe, but using raw hash table API's may require + /// unsafe functions or blocks. + /// + /// `RawTable` API gives the lowest level of control under the map that can be useful + /// for extending the HashMap's API, but may lead to *[undefined behavior]*. + /// + /// [`HashMap`]: struct.HashMap.html + /// [`RawTable`]: raw/struct.RawTable.html + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html + /// + /// # Examples + /// + /// ``` + /// use core::hash::{BuildHasher, Hash}; + /// use hashbrown::HashMap; + /// + /// let mut map = HashMap::new(); + /// map.extend([("a", 10), ("b", 20), ("c", 30)]); + /// assert_eq!(map.len(), 3); + /// + /// // Let's imagine that we have a value and a hash of the key, but not the key itself. + /// // However, if you want to remove the value from the map by hash and value, and you + /// // know exactly that the value is unique, then you can create a function like this: + /// fn remove_by_hash( + /// map: &mut HashMap, + /// hash: u64, + /// is_match: F, + /// ) -> Option<(K, V)> + /// where + /// F: Fn(&(K, V)) -> bool, + /// { + /// let raw_table = map.raw_table(); + /// match raw_table.find(hash, is_match) { + /// Some(bucket) => Some(unsafe { raw_table.remove(bucket) }), + /// None => None, + /// } + /// } + /// + /// fn compute_hash(hash_builder: &S, key: &K) -> u64 { + /// use core::hash::Hasher; + /// let mut state = hash_builder.build_hasher(); + /// key.hash(&mut state); + /// state.finish() + /// } + /// + /// let hash = compute_hash(map.hasher(), "a"); + /// assert_eq!(remove_by_hash(&mut map, hash, |(_, v)| *v == 10), Some(("a", 10))); + /// assert_eq!(map.get(&"a"), None); + /// assert_eq!(map.len(), 2); + /// ``` + #[cfg(feature = "raw")] + #[cfg_attr(feature = "inline-more", inline)] + pub fn raw_table(&mut self) -> &mut RawTable<(K, V), A> { + &mut self.table + } } -impl PartialEq for HashMap +impl PartialEq for HashMap where K: Eq + Hash, V: PartialEq, S: BuildHasher, + A: Allocator + Clone, { fn eq(&self, other: &Self) -> bool { if self.len() != other.len() { @@ -1139,40 +2092,58 @@ where } } -impl Eq for HashMap +impl Eq for HashMap where K: Eq + Hash, V: Eq, S: BuildHasher, + A: Allocator + Clone, { } -impl Debug for HashMap +impl Debug for HashMap where K: Debug, V: Debug, + A: Allocator + Clone, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_map().entries(self.iter()).finish() } } -impl Default for HashMap +impl Default for HashMap where S: Default, + A: Default + Allocator + Clone, { - /// Creates an empty `HashMap`, with the `Default` value for the hasher. + /// Creates an empty `HashMap`, with the `Default` value for the hasher and allocator. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// use std::collections::hash_map::RandomState; + /// + /// // You can specify all types of HashMap, including hasher and allocator. + /// // Created map is empty and don't allocate memory + /// let map: HashMap = Default::default(); + /// assert_eq!(map.capacity(), 0); + /// let map: HashMap = HashMap::default(); + /// assert_eq!(map.capacity(), 0); + /// ``` #[cfg_attr(feature = "inline-more", inline)] fn default() -> Self { - Self::with_hasher(Default::default()) + Self::with_hasher_in(Default::default(), Default::default()) } } -impl Index<&Q> for HashMap +impl Index<&Q> for HashMap where K: Eq + Hash + Borrow, Q: Eq + Hash, S: BuildHasher, + A: Allocator + Clone, { type Output = V; @@ -1181,19 +2152,72 @@ where /// # Panics /// /// Panics if the key is not present in the `HashMap`. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let map: HashMap<_, _> = [("a", "One"), ("b", "Two")].into(); + /// + /// assert_eq!(map[&"a"], "One"); + /// assert_eq!(map[&"b"], "Two"); + /// ``` #[cfg_attr(feature = "inline-more", inline)] fn index(&self, key: &Q) -> &V { self.get(key).expect("no entry found for key") } } -/// An iterator over the entries of a `HashMap`. +// The default hasher is used to match the std implementation signature +#[cfg(feature = "ahash")] +impl From<[(K, V); N]> for HashMap +where + K: Eq + Hash, + A: Default + Allocator + Clone, +{ + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let map1 = HashMap::from([(1, 2), (3, 4)]); + /// let map2: HashMap<_, _> = [(1, 2), (3, 4)].into(); + /// assert_eq!(map1, map2); + /// ``` + fn from(arr: [(K, V); N]) -> Self { + arr.into_iter().collect() + } +} + +/// An iterator over the entries of a `HashMap` in arbitrary order. +/// The iterator element type is `(&'a K, &'a V)`. /// /// This `struct` is created by the [`iter`] method on [`HashMap`]. See its /// documentation for more. /// /// [`iter`]: struct.HashMap.html#method.iter /// [`HashMap`]: struct.HashMap.html +/// +/// # Examples +/// +/// ``` +/// use hashbrown::HashMap; +/// +/// let map: HashMap<_, _> = [(1, "a"), (2, "b"), (3, "c")].into(); +/// +/// let mut iter = map.iter(); +/// let mut vec = vec![iter.next(), iter.next(), iter.next()]; +/// +/// // The `Iter` iterator produces items in arbitrary order, so the +/// // items must be sorted to test them against a sorted array. +/// vec.sort_unstable(); +/// assert_eq!(vec, [Some((&1, &"a")), Some((&2, &"b")), Some((&3, &"c"))]); +/// +/// // It is fused iterator +/// assert_eq!(iter.next(), None); +/// assert_eq!(iter.next(), None); +/// ``` pub struct Iter<'a, K, V> { inner: RawIter<(K, V)>, marker: PhantomData<(&'a K, &'a V)>, @@ -1216,13 +2240,33 @@ impl fmt::Debug for Iter<'_, K, V> { } } -/// A mutable iterator over the entries of a `HashMap`. +/// A mutable iterator over the entries of a `HashMap` in arbitrary order. +/// The iterator element type is `(&'a K, &'a mut V)`. /// /// This `struct` is created by the [`iter_mut`] method on [`HashMap`]. See its /// documentation for more. /// /// [`iter_mut`]: struct.HashMap.html#method.iter_mut /// [`HashMap`]: struct.HashMap.html +/// +/// # Examples +/// +/// ``` +/// use hashbrown::HashMap; +/// +/// let mut map: HashMap<_, _> = [(1, "One".to_owned()), (2, "Two".into())].into(); +/// +/// let mut iter = map.iter_mut(); +/// iter.next().map(|(_, v)| v.push_str(" Mississippi")); +/// iter.next().map(|(_, v)| v.push_str(" Mississippi")); +/// +/// // It is fused iterator +/// assert_eq!(iter.next(), None); +/// assert_eq!(iter.next(), None); +/// +/// assert_eq!(map.get(&1).unwrap(), &"One Mississippi".to_owned()); +/// assert_eq!(map.get(&2).unwrap(), &"Two Mississippi".to_owned()); +/// ``` pub struct IterMut<'a, K, V> { inner: RawIter<(K, V)>, // To ensure invariance with respect to V @@ -1245,18 +2289,41 @@ impl IterMut<'_, K, V> { } } -/// An owning iterator over the entries of a `HashMap`. +/// An owning iterator over the entries of a `HashMap` in arbitrary order. +/// The iterator element type is `(K, V)`. /// -/// This `struct` is created by the [`into_iter`] method on [`HashMap`][`HashMap`] -/// (provided by the `IntoIterator` trait). See its documentation for more. +/// This `struct` is created by the [`into_iter`] method on [`HashMap`] +/// (provided by the [`IntoIterator`] trait). See its documentation for more. +/// The map cannot be used after calling that method. /// /// [`into_iter`]: struct.HashMap.html#method.into_iter /// [`HashMap`]: struct.HashMap.html -pub struct IntoIter { - inner: RawIntoIter<(K, V)>, +/// [`IntoIterator`]: https://doc.rust-lang.org/core/iter/trait.IntoIterator.html +/// +/// # Examples +/// +/// ``` +/// use hashbrown::HashMap; +/// +/// let map: HashMap<_, _> = [(1, "a"), (2, "b"), (3, "c")].into(); +/// +/// let mut iter = map.into_iter(); +/// let mut vec = vec![iter.next(), iter.next(), iter.next()]; +/// +/// // The `IntoIter` iterator produces items in arbitrary order, so the +/// // items must be sorted to test them against a sorted array. +/// vec.sort_unstable(); +/// assert_eq!(vec, [Some((1, "a")), Some((2, "b")), Some((3, "c"))]); +/// +/// // It is fused iterator +/// assert_eq!(iter.next(), None); +/// assert_eq!(iter.next(), None); +/// ``` +pub struct IntoIter { + inner: RawIntoIter<(K, V), A>, } -impl IntoIter { +impl IntoIter { /// Returns a iterator of references over the remaining items. #[cfg_attr(feature = "inline-more", inline)] pub(super) fn iter(&self) -> Iter<'_, K, V> { @@ -1267,13 +2334,159 @@ impl IntoIter { } } -/// An iterator over the keys of a `HashMap`. +/// An owning iterator over the keys of a `HashMap` in arbitrary order. +/// The iterator element type is `K`. +/// +/// This `struct` is created by the [`into_keys`] method on [`HashMap`]. +/// See its documentation for more. +/// The map cannot be used after calling that method. +/// +/// [`into_keys`]: struct.HashMap.html#method.into_keys +/// [`HashMap`]: struct.HashMap.html +/// +/// # Examples +/// +/// ``` +/// use hashbrown::HashMap; +/// +/// let map: HashMap<_, _> = [(1, "a"), (2, "b"), (3, "c")].into(); +/// +/// let mut keys = map.into_keys(); +/// let mut vec = vec![keys.next(), keys.next(), keys.next()]; +/// +/// // The `IntoKeys` iterator produces keys in arbitrary order, so the +/// // keys must be sorted to test them against a sorted array. +/// vec.sort_unstable(); +/// assert_eq!(vec, [Some(1), Some(2), Some(3)]); +/// +/// // It is fused iterator +/// assert_eq!(keys.next(), None); +/// assert_eq!(keys.next(), None); +/// ``` +pub struct IntoKeys { + inner: IntoIter, +} + +impl Iterator for IntoKeys { + type Item = K; + + #[inline] + fn next(&mut self) -> Option { + self.inner.next().map(|(k, _)| k) + } + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} + +impl ExactSizeIterator for IntoKeys { + #[inline] + fn len(&self) -> usize { + self.inner.len() + } +} + +impl FusedIterator for IntoKeys {} + +impl fmt::Debug for IntoKeys { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list() + .entries(self.inner.iter().map(|(k, _)| k)) + .finish() + } +} + +/// An owning iterator over the values of a `HashMap` in arbitrary order. +/// The iterator element type is `V`. +/// +/// This `struct` is created by the [`into_values`] method on [`HashMap`]. +/// See its documentation for more. The map cannot be used after calling that method. +/// +/// [`into_values`]: struct.HashMap.html#method.into_values +/// [`HashMap`]: struct.HashMap.html +/// +/// # Examples +/// +/// ``` +/// use hashbrown::HashMap; +/// +/// let map: HashMap<_, _> = [(1, "a"), (2, "b"), (3, "c")].into(); +/// +/// let mut values = map.into_values(); +/// let mut vec = vec![values.next(), values.next(), values.next()]; +/// +/// // The `IntoValues` iterator produces values in arbitrary order, so +/// // the values must be sorted to test them against a sorted array. +/// vec.sort_unstable(); +/// assert_eq!(vec, [Some("a"), Some("b"), Some("c")]); +/// +/// // It is fused iterator +/// assert_eq!(values.next(), None); +/// assert_eq!(values.next(), None); +/// ``` +pub struct IntoValues { + inner: IntoIter, +} + +impl Iterator for IntoValues { + type Item = V; + + #[inline] + fn next(&mut self) -> Option { + self.inner.next().map(|(_, v)| v) + } + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} + +impl ExactSizeIterator for IntoValues { + #[inline] + fn len(&self) -> usize { + self.inner.len() + } +} + +impl FusedIterator for IntoValues {} + +impl fmt::Debug for IntoValues { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list() + .entries(self.inner.iter().map(|(_, v)| v)) + .finish() + } +} + +/// An iterator over the keys of a `HashMap` in arbitrary order. +/// The iterator element type is `&'a K`. /// /// This `struct` is created by the [`keys`] method on [`HashMap`]. See its /// documentation for more. /// /// [`keys`]: struct.HashMap.html#method.keys /// [`HashMap`]: struct.HashMap.html +/// +/// # Examples +/// +/// ``` +/// use hashbrown::HashMap; +/// +/// let map: HashMap<_, _> = [(1, "a"), (2, "b"), (3, "c")].into(); +/// +/// let mut keys = map.keys(); +/// let mut vec = vec![keys.next(), keys.next(), keys.next()]; +/// +/// // The `Keys` iterator produces keys in arbitrary order, so the +/// // keys must be sorted to test them against a sorted array. +/// vec.sort_unstable(); +/// assert_eq!(vec, [Some(&1), Some(&2), Some(&3)]); +/// +/// // It is fused iterator +/// assert_eq!(keys.next(), None); +/// assert_eq!(keys.next(), None); +/// ``` pub struct Keys<'a, K, V> { inner: Iter<'a, K, V>, } @@ -1294,13 +2507,34 @@ impl fmt::Debug for Keys<'_, K, V> { } } -/// An iterator over the values of a `HashMap`. +/// An iterator over the values of a `HashMap` in arbitrary order. +/// The iterator element type is `&'a V`. /// /// This `struct` is created by the [`values`] method on [`HashMap`]. See its /// documentation for more. /// /// [`values`]: struct.HashMap.html#method.values /// [`HashMap`]: struct.HashMap.html +/// +/// # Examples +/// +/// ``` +/// use hashbrown::HashMap; +/// +/// let map: HashMap<_, _> = [(1, "a"), (2, "b"), (3, "c")].into(); +/// +/// let mut values = map.values(); +/// let mut vec = vec![values.next(), values.next(), values.next()]; +/// +/// // The `Values` iterator produces values in arbitrary order, so the +/// // values must be sorted to test them against a sorted array. +/// vec.sort_unstable(); +/// assert_eq!(vec, [Some(&"a"), Some(&"b"), Some(&"c")]); +/// +/// // It is fused iterator +/// assert_eq!(values.next(), None); +/// assert_eq!(values.next(), None); +/// ``` pub struct Values<'a, K, V> { inner: Iter<'a, K, V>, } @@ -1321,18 +2555,39 @@ impl fmt::Debug for Values<'_, K, V> { } } -/// A draining iterator over the entries of a `HashMap`. +/// A draining iterator over the entries of a `HashMap` in arbitrary +/// order. The iterator element type is `(K, V)`. /// /// This `struct` is created by the [`drain`] method on [`HashMap`]. See its /// documentation for more. /// /// [`drain`]: struct.HashMap.html#method.drain /// [`HashMap`]: struct.HashMap.html -pub struct Drain<'a, K, V> { - inner: RawDrain<'a, (K, V)>, +/// +/// # Examples +/// +/// ``` +/// use hashbrown::HashMap; +/// +/// let mut map: HashMap<_, _> = [(1, "a"), (2, "b"), (3, "c")].into(); +/// +/// let mut drain_iter = map.drain(); +/// let mut vec = vec![drain_iter.next(), drain_iter.next(), drain_iter.next()]; +/// +/// // The `Drain` iterator produces items in arbitrary order, so the +/// // items must be sorted to test them against a sorted array. +/// vec.sort_unstable(); +/// assert_eq!(vec, [Some((1, "a")), Some((2, "b")), Some((3, "c"))]); +/// +/// // It is fused iterator +/// assert_eq!(drain_iter.next(), None); +/// assert_eq!(drain_iter.next(), None); +/// ``` +pub struct Drain<'a, K, V, A: Allocator + Clone = Global> { + inner: RawDrain<'a, (K, V), A>, } -impl Drain<'_, K, V> { +impl Drain<'_, K, V, A> { /// Returns a iterator of references over the remaining items. #[cfg_attr(feature = "inline-more", inline)] pub(super) fn iter(&self) -> Iter<'_, K, V> { @@ -1343,59 +2598,106 @@ impl Drain<'_, K, V> { } } -/// A draining iterator over entries of a `HashMap` which don't satisfy the predicate `f`. +/// A draining iterator over entries of a `HashMap` which don't satisfy the predicate +/// `f(&k, &mut v)` in arbitrary order. The iterator element type is `(K, V)`. /// /// This `struct` is created by the [`drain_filter`] method on [`HashMap`]. See its /// documentation for more. /// /// [`drain_filter`]: struct.HashMap.html#method.drain_filter /// [`HashMap`]: struct.HashMap.html -pub struct DrainFilter<'a, K, V, F> -where - F: FnMut(&K, &mut V) -> bool, -{ +/// +/// # Examples +/// +/// ``` +/// use hashbrown::HashMap; +/// +/// let mut map: HashMap = [(1, "a"), (2, "b"), (3, "c")].into(); +/// +/// let mut drain_filter = map.drain_filter(|k, _v| k % 2 != 0); +/// let mut vec = vec![drain_filter.next(), drain_filter.next()]; +/// +/// // The `DrainFilter` iterator produces items in arbitrary order, so the +/// // items must be sorted to test them against a sorted array. +/// vec.sort_unstable(); +/// assert_eq!(vec, [Some((1, "a")),Some((3, "c"))]); +/// +/// // It is fused iterator +/// assert_eq!(drain_filter.next(), None); +/// assert_eq!(drain_filter.next(), None); +/// drop(drain_filter); +/// +/// assert_eq!(map.len(), 1); +/// ``` +pub struct DrainFilter<'a, K, V, F, A: Allocator + Clone = Global> +where + F: FnMut(&K, &mut V) -> bool, +{ f: F, - iter: RawIter<(K, V)>, - table: &'a mut RawTable<(K, V)>, + inner: DrainFilterInner<'a, K, V, A>, } -impl<'a, K, V, F> Drop for DrainFilter<'a, K, V, F> +impl<'a, K, V, F, A> Drop for DrainFilter<'a, K, V, F, A> where F: FnMut(&K, &mut V) -> bool, + A: Allocator + Clone, { + #[cfg_attr(feature = "inline-more", inline)] fn drop(&mut self) { - struct DropGuard<'r, 'a, K, V, F>(&'r mut DrainFilter<'a, K, V, F>) - where - F: FnMut(&K, &mut V) -> bool; - - impl<'r, 'a, K, V, F> Drop for DropGuard<'r, 'a, K, V, F> - where - F: FnMut(&K, &mut V) -> bool, - { - fn drop(&mut self) { - while let Some(_) = self.0.next() {} - } - } while let Some(item) = self.next() { - let guard = DropGuard(self); + let guard = ConsumeAllOnDrop(self); drop(item); mem::forget(guard); } } } -impl Iterator for DrainFilter<'_, K, V, F> +pub(super) struct ConsumeAllOnDrop<'a, T: Iterator>(pub &'a mut T); + +impl Drop for ConsumeAllOnDrop<'_, T> { + #[cfg_attr(feature = "inline-more", inline)] + fn drop(&mut self) { + self.0.for_each(drop); + } +} + +impl Iterator for DrainFilter<'_, K, V, F, A> where F: FnMut(&K, &mut V) -> bool, + A: Allocator + Clone, { type Item = (K, V); + + #[cfg_attr(feature = "inline-more", inline)] fn next(&mut self) -> Option { + self.inner.next(&mut self.f) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + (0, self.inner.iter.size_hint().1) + } +} + +impl FusedIterator for DrainFilter<'_, K, V, F> where F: FnMut(&K, &mut V) -> bool {} + +/// Portions of `DrainFilter` shared with `set::DrainFilter` +pub(super) struct DrainFilterInner<'a, K, V, A: Allocator + Clone> { + pub iter: RawIter<(K, V)>, + pub table: &'a mut RawTable<(K, V), A>, +} + +impl DrainFilterInner<'_, K, V, A> { + #[cfg_attr(feature = "inline-more", inline)] + pub(super) fn next(&mut self, f: &mut F) -> Option<(K, V)> + where + F: FnMut(&K, &mut V) -> bool, + { unsafe { - while let Some(item) = self.iter.next() { + for item in &mut self.iter { let &mut (ref key, ref mut value) = item.as_mut(); - if !(self.f)(key, value) { - self.table.erase_no_drop(&item); - return Some(item.read()); + if f(key, value) { + return Some(self.table.remove(item)); } } } @@ -1403,13 +2705,33 @@ where } } -/// A mutable iterator over the values of a `HashMap`. +/// A mutable iterator over the values of a `HashMap` in arbitrary order. +/// The iterator element type is `&'a mut V`. /// /// This `struct` is created by the [`values_mut`] method on [`HashMap`]. See its /// documentation for more. /// /// [`values_mut`]: struct.HashMap.html#method.values_mut /// [`HashMap`]: struct.HashMap.html +/// +/// # Examples +/// +/// ``` +/// use hashbrown::HashMap; +/// +/// let mut map: HashMap<_, _> = [(1, "One".to_owned()), (2, "Two".into())].into(); +/// +/// let mut values = map.values_mut(); +/// values.next().map(|v| v.push_str(" Mississippi")); +/// values.next().map(|v| v.push_str(" Mississippi")); +/// +/// // It is fused iterator +/// assert_eq!(values.next(), None); +/// assert_eq!(values.next(), None); +/// +/// assert_eq!(map.get(&1).unwrap(), &"One Mississippi".to_owned()); +/// assert_eq!(map.get(&2).unwrap(), &"Two Mississippi".to_owned()); +/// ``` pub struct ValuesMut<'a, K, V> { inner: IterMut<'a, K, V>, } @@ -1419,8 +2741,58 @@ pub struct ValuesMut<'a, K, V> { /// See the [`HashMap::raw_entry_mut`] docs for usage examples. /// /// [`HashMap::raw_entry_mut`]: struct.HashMap.html#method.raw_entry_mut -pub struct RawEntryBuilderMut<'a, K, V, S> { - map: &'a mut HashMap, +/// +/// # Examples +/// +/// ``` +/// use hashbrown::hash_map::{RawEntryBuilderMut, RawEntryMut::Vacant, RawEntryMut::Occupied}; +/// use hashbrown::HashMap; +/// use core::hash::{BuildHasher, Hash}; +/// +/// let mut map = HashMap::new(); +/// map.extend([(1, 11), (2, 12), (3, 13), (4, 14), (5, 15), (6, 16)]); +/// assert_eq!(map.len(), 6); +/// +/// fn compute_hash(hash_builder: &S, key: &K) -> u64 { +/// use core::hash::Hasher; +/// let mut state = hash_builder.build_hasher(); +/// key.hash(&mut state); +/// state.finish() +/// } +/// +/// let builder: RawEntryBuilderMut<_, _, _> = map.raw_entry_mut(); +/// +/// // Existing key +/// match builder.from_key(&6) { +/// Vacant(_) => unreachable!(), +/// Occupied(view) => assert_eq!(view.get(), &16), +/// } +/// +/// for key in 0..12 { +/// let hash = compute_hash(map.hasher(), &key); +/// let value = map.get(&key).cloned(); +/// let key_value = value.as_ref().map(|v| (&key, v)); +/// +/// println!("Key: {} and value: {:?}", key, value); +/// +/// match map.raw_entry_mut().from_key(&key) { +/// Occupied(mut o) => assert_eq!(Some(o.get_key_value()), key_value), +/// Vacant(_) => assert_eq!(value, None), +/// } +/// match map.raw_entry_mut().from_key_hashed_nocheck(hash, &key) { +/// Occupied(mut o) => assert_eq!(Some(o.get_key_value()), key_value), +/// Vacant(_) => assert_eq!(value, None), +/// } +/// match map.raw_entry_mut().from_hash(hash, |q| *q == key) { +/// Occupied(mut o) => assert_eq!(Some(o.get_key_value()), key_value), +/// Vacant(_) => assert_eq!(value, None), +/// } +/// } +/// +/// assert_eq!(map.len(), 6); +/// ``` +pub struct RawEntryBuilderMut<'a, K, V, S, A: Allocator + Clone = Global> { + map: &'a mut HashMap, } /// A view into a single entry in a map, which may either be vacant or occupied. @@ -1434,32 +2806,190 @@ pub struct RawEntryBuilderMut<'a, K, V, S> { /// [`Entry`]: enum.Entry.html /// [`raw_entry_mut`]: struct.HashMap.html#method.raw_entry_mut /// [`RawEntryBuilderMut`]: struct.RawEntryBuilderMut.html -pub enum RawEntryMut<'a, K, V, S> { +/// +/// # Examples +/// +/// ``` +/// use core::hash::{BuildHasher, Hash}; +/// use hashbrown::hash_map::{HashMap, RawEntryMut, RawOccupiedEntryMut}; +/// +/// let mut map = HashMap::new(); +/// map.extend([('a', 1), ('b', 2), ('c', 3)]); +/// assert_eq!(map.len(), 3); +/// +/// fn compute_hash(hash_builder: &S, key: &K) -> u64 { +/// use core::hash::Hasher; +/// let mut state = hash_builder.build_hasher(); +/// key.hash(&mut state); +/// state.finish() +/// } +/// +/// // Existing key (insert) +/// let raw: RawEntryMut<_, _, _> = map.raw_entry_mut().from_key(&'a'); +/// let _raw_o: RawOccupiedEntryMut<_, _, _> = raw.insert('a', 10); +/// assert_eq!(map.len(), 3); +/// +/// // Nonexistent key (insert) +/// map.raw_entry_mut().from_key(&'d').insert('d', 40); +/// assert_eq!(map.len(), 4); +/// +/// // Existing key (or_insert) +/// let hash = compute_hash(map.hasher(), &'b'); +/// let kv = map +/// .raw_entry_mut() +/// .from_key_hashed_nocheck(hash, &'b') +/// .or_insert('b', 20); +/// assert_eq!(kv, (&mut 'b', &mut 2)); +/// *kv.1 = 20; +/// assert_eq!(map.len(), 4); +/// +/// // Nonexistent key (or_insert) +/// let hash = compute_hash(map.hasher(), &'e'); +/// let kv = map +/// .raw_entry_mut() +/// .from_key_hashed_nocheck(hash, &'e') +/// .or_insert('e', 50); +/// assert_eq!(kv, (&mut 'e', &mut 50)); +/// assert_eq!(map.len(), 5); +/// +/// // Existing key (or_insert_with) +/// let hash = compute_hash(map.hasher(), &'c'); +/// let kv = map +/// .raw_entry_mut() +/// .from_hash(hash, |q| q == &'c') +/// .or_insert_with(|| ('c', 30)); +/// assert_eq!(kv, (&mut 'c', &mut 3)); +/// *kv.1 = 30; +/// assert_eq!(map.len(), 5); +/// +/// // Nonexistent key (or_insert_with) +/// let hash = compute_hash(map.hasher(), &'f'); +/// let kv = map +/// .raw_entry_mut() +/// .from_hash(hash, |q| q == &'f') +/// .or_insert_with(|| ('f', 60)); +/// assert_eq!(kv, (&mut 'f', &mut 60)); +/// assert_eq!(map.len(), 6); +/// +/// println!("Our HashMap: {:?}", map); +/// +/// let mut vec: Vec<_> = map.iter().map(|(&k, &v)| (k, v)).collect(); +/// // The `Iter` iterator produces items in arbitrary order, so the +/// // items must be sorted to test them against a sorted array. +/// vec.sort_unstable(); +/// assert_eq!(vec, [('a', 10), ('b', 20), ('c', 30), ('d', 40), ('e', 50), ('f', 60)]); +/// ``` +pub enum RawEntryMut<'a, K, V, S, A: Allocator + Clone = Global> { /// An occupied entry. - Occupied(RawOccupiedEntryMut<'a, K, V>), + /// + /// # Examples + /// + /// ``` + /// use hashbrown::{hash_map::RawEntryMut, HashMap}; + /// let mut map: HashMap<_, _> = [("a", 100), ("b", 200)].into(); + /// + /// match map.raw_entry_mut().from_key(&"a") { + /// RawEntryMut::Vacant(_) => unreachable!(), + /// RawEntryMut::Occupied(_) => { } + /// } + /// ``` + Occupied(RawOccupiedEntryMut<'a, K, V, S, A>), /// A vacant entry. - Vacant(RawVacantEntryMut<'a, K, V, S>), + /// + /// # Examples + /// + /// ``` + /// use hashbrown::{hash_map::RawEntryMut, HashMap}; + /// let mut map: HashMap<&str, i32> = HashMap::new(); + /// + /// match map.raw_entry_mut().from_key("a") { + /// RawEntryMut::Occupied(_) => unreachable!(), + /// RawEntryMut::Vacant(_) => { } + /// } + /// ``` + Vacant(RawVacantEntryMut<'a, K, V, S, A>), } /// A view into an occupied entry in a `HashMap`. /// It is part of the [`RawEntryMut`] enum. /// /// [`RawEntryMut`]: enum.RawEntryMut.html -pub struct RawOccupiedEntryMut<'a, K, V> { +/// +/// # Examples +/// +/// ``` +/// use core::hash::{BuildHasher, Hash}; +/// use hashbrown::hash_map::{HashMap, RawEntryMut, RawOccupiedEntryMut}; +/// +/// let mut map = HashMap::new(); +/// map.extend([("a", 10), ("b", 20), ("c", 30)]); +/// +/// fn compute_hash(hash_builder: &S, key: &K) -> u64 { +/// use core::hash::Hasher; +/// let mut state = hash_builder.build_hasher(); +/// key.hash(&mut state); +/// state.finish() +/// } +/// +/// let _raw_o: RawOccupiedEntryMut<_, _, _> = map.raw_entry_mut().from_key(&"a").insert("a", 100); +/// assert_eq!(map.len(), 3); +/// +/// // Existing key (insert and update) +/// match map.raw_entry_mut().from_key(&"a") { +/// RawEntryMut::Vacant(_) => unreachable!(), +/// RawEntryMut::Occupied(mut view) => { +/// assert_eq!(view.get(), &100); +/// let v = view.get_mut(); +/// let new_v = (*v) * 10; +/// *v = new_v; +/// assert_eq!(view.insert(1111), 1000); +/// } +/// } +/// +/// assert_eq!(map[&"a"], 1111); +/// assert_eq!(map.len(), 3); +/// +/// // Existing key (take) +/// let hash = compute_hash(map.hasher(), &"c"); +/// match map.raw_entry_mut().from_key_hashed_nocheck(hash, &"c") { +/// RawEntryMut::Vacant(_) => unreachable!(), +/// RawEntryMut::Occupied(view) => { +/// assert_eq!(view.remove_entry(), ("c", 30)); +/// } +/// } +/// assert_eq!(map.raw_entry().from_key(&"c"), None); +/// assert_eq!(map.len(), 2); +/// +/// let hash = compute_hash(map.hasher(), &"b"); +/// match map.raw_entry_mut().from_hash(hash, |q| *q == "b") { +/// RawEntryMut::Vacant(_) => unreachable!(), +/// RawEntryMut::Occupied(view) => { +/// assert_eq!(view.remove_entry(), ("b", 20)); +/// } +/// } +/// assert_eq!(map.get(&"b"), None); +/// assert_eq!(map.len(), 1); +/// ``` +pub struct RawOccupiedEntryMut<'a, K, V, S, A: Allocator + Clone = Global> { elem: Bucket<(K, V)>, - table: &'a mut RawTable<(K, V)>, + table: &'a mut RawTable<(K, V), A>, + hash_builder: &'a S, } -unsafe impl Send for RawOccupiedEntryMut<'_, K, V> +unsafe impl Send for RawOccupiedEntryMut<'_, K, V, S, A> where K: Send, V: Send, + S: Send, + A: Send + Allocator + Clone, { } -unsafe impl Sync for RawOccupiedEntryMut<'_, K, V> +unsafe impl Sync for RawOccupiedEntryMut<'_, K, V, S, A> where K: Sync, V: Sync, + S: Sync, + A: Sync + Allocator + Clone, { } @@ -1467,8 +2997,52 @@ where /// It is part of the [`RawEntryMut`] enum. /// /// [`RawEntryMut`]: enum.RawEntryMut.html -pub struct RawVacantEntryMut<'a, K, V, S> { - table: &'a mut RawTable<(K, V)>, +/// +/// # Examples +/// +/// ``` +/// use core::hash::{BuildHasher, Hash}; +/// use hashbrown::hash_map::{HashMap, RawEntryMut, RawVacantEntryMut}; +/// +/// let mut map = HashMap::<&str, i32>::new(); +/// +/// fn compute_hash(hash_builder: &S, key: &K) -> u64 { +/// use core::hash::Hasher; +/// let mut state = hash_builder.build_hasher(); +/// key.hash(&mut state); +/// state.finish() +/// } +/// +/// let raw_v: RawVacantEntryMut<_, _, _> = match map.raw_entry_mut().from_key(&"a") { +/// RawEntryMut::Vacant(view) => view, +/// RawEntryMut::Occupied(_) => unreachable!(), +/// }; +/// raw_v.insert("a", 10); +/// assert!(map[&"a"] == 10 && map.len() == 1); +/// +/// // Nonexistent key (insert and update) +/// let hash = compute_hash(map.hasher(), &"b"); +/// match map.raw_entry_mut().from_key_hashed_nocheck(hash, &"b") { +/// RawEntryMut::Occupied(_) => unreachable!(), +/// RawEntryMut::Vacant(view) => { +/// let (k, value) = view.insert("b", 2); +/// assert_eq!((*k, *value), ("b", 2)); +/// *value = 20; +/// } +/// } +/// assert!(map[&"b"] == 20 && map.len() == 2); +/// +/// let hash = compute_hash(map.hasher(), &"c"); +/// match map.raw_entry_mut().from_hash(hash, |q| *q == "c") { +/// RawEntryMut::Occupied(_) => unreachable!(), +/// RawEntryMut::Vacant(view) => { +/// assert_eq!(view.insert("c", 30), (&mut "c", &mut 30)); +/// } +/// } +/// assert!(map[&"c"] == 30 && map.len() == 3); +/// ``` +pub struct RawVacantEntryMut<'a, K, V, S, A: Allocator + Clone = Global> { + table: &'a mut RawTable<(K, V), A>, hash_builder: &'a S, } @@ -1477,42 +3051,124 @@ pub struct RawVacantEntryMut<'a, K, V, S> { /// See the [`HashMap::raw_entry`] docs for usage examples. /// /// [`HashMap::raw_entry`]: struct.HashMap.html#method.raw_entry -pub struct RawEntryBuilder<'a, K, V, S> { - map: &'a HashMap, +/// +/// # Examples +/// +/// ``` +/// use hashbrown::hash_map::{HashMap, RawEntryBuilder}; +/// use core::hash::{BuildHasher, Hash}; +/// +/// let mut map = HashMap::new(); +/// map.extend([(1, 10), (2, 20), (3, 30)]); +/// +/// fn compute_hash(hash_builder: &S, key: &K) -> u64 { +/// use core::hash::Hasher; +/// let mut state = hash_builder.build_hasher(); +/// key.hash(&mut state); +/// state.finish() +/// } +/// +/// for k in 0..6 { +/// let hash = compute_hash(map.hasher(), &k); +/// let v = map.get(&k).cloned(); +/// let kv = v.as_ref().map(|v| (&k, v)); +/// +/// println!("Key: {} and value: {:?}", k, v); +/// let builder: RawEntryBuilder<_, _, _> = map.raw_entry(); +/// assert_eq!(builder.from_key(&k), kv); +/// assert_eq!(map.raw_entry().from_hash(hash, |q| *q == k), kv); +/// assert_eq!(map.raw_entry().from_key_hashed_nocheck(hash, &k), kv); +/// } +/// ``` +pub struct RawEntryBuilder<'a, K, V, S, A: Allocator + Clone = Global> { + map: &'a HashMap, } -impl<'a, K, V, S> RawEntryBuilderMut<'a, K, V, S> { +impl<'a, K, V, S, A: Allocator + Clone> RawEntryBuilderMut<'a, K, V, S, A> { /// Creates a `RawEntryMut` from the given key. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{HashMap, RawEntryMut}; + /// + /// let mut map: HashMap<&str, u32> = HashMap::new(); + /// let key = "a"; + /// let entry: RawEntryMut<&str, u32, _> = map.raw_entry_mut().from_key(&key); + /// entry.insert(key, 100); + /// assert_eq!(map[&"a"], 100); + /// ``` #[cfg_attr(feature = "inline-more", inline)] #[allow(clippy::wrong_self_convention)] - pub fn from_key(self, k: &Q) -> RawEntryMut<'a, K, V, S> + pub fn from_key(self, k: &Q) -> RawEntryMut<'a, K, V, S, A> where S: BuildHasher, K: Borrow, Q: Hash + Eq, { - let mut hasher = self.map.hash_builder.build_hasher(); - k.hash(&mut hasher); - self.from_key_hashed_nocheck(hasher.finish(), k) + let hash = make_hash::(&self.map.hash_builder, k); + self.from_key_hashed_nocheck(hash, k) } /// Creates a `RawEntryMut` from the given key and its hash. + /// + /// # Examples + /// + /// ``` + /// use core::hash::{BuildHasher, Hash}; + /// use hashbrown::hash_map::{HashMap, RawEntryMut}; + /// + /// fn compute_hash(hash_builder: &S, key: &K) -> u64 { + /// use core::hash::Hasher; + /// let mut state = hash_builder.build_hasher(); + /// key.hash(&mut state); + /// state.finish() + /// } + /// + /// let mut map: HashMap<&str, u32> = HashMap::new(); + /// let key = "a"; + /// let hash = compute_hash(map.hasher(), &key); + /// let entry: RawEntryMut<&str, u32, _> = map.raw_entry_mut().from_key_hashed_nocheck(hash, &key); + /// entry.insert(key, 100); + /// assert_eq!(map[&"a"], 100); + /// ``` #[inline] #[allow(clippy::wrong_self_convention)] - pub fn from_key_hashed_nocheck(self, hash: u64, k: &Q) -> RawEntryMut<'a, K, V, S> + pub fn from_key_hashed_nocheck(self, hash: u64, k: &Q) -> RawEntryMut<'a, K, V, S, A> where K: Borrow, Q: Eq, { - self.from_hash(hash, |q| q.borrow().eq(k)) + self.from_hash(hash, equivalent(k)) } } -impl<'a, K, V, S> RawEntryBuilderMut<'a, K, V, S> { - /// Creates a `RawEntryMut` from the given hash. +impl<'a, K, V, S, A: Allocator + Clone> RawEntryBuilderMut<'a, K, V, S, A> { + /// Creates a `RawEntryMut` from the given hash and matching function. + /// + /// # Examples + /// + /// ``` + /// use core::hash::{BuildHasher, Hash}; + /// use hashbrown::hash_map::{HashMap, RawEntryMut}; + /// + /// fn compute_hash(hash_builder: &S, key: &K) -> u64 { + /// use core::hash::Hasher; + /// let mut state = hash_builder.build_hasher(); + /// key.hash(&mut state); + /// state.finish() + /// } + /// + /// let mut map: HashMap<&str, u32> = HashMap::new(); + /// let key = "a"; + /// let hash = compute_hash(map.hasher(), &key); + /// let entry: RawEntryMut<&str, u32, _> = map.raw_entry_mut().from_hash(hash, |k| k == &key); + /// entry.insert(key, 100); + /// assert_eq!(map[&"a"], 100); + /// ``` #[cfg_attr(feature = "inline-more", inline)] #[allow(clippy::wrong_self_convention)] - pub fn from_hash(self, hash: u64, is_match: F) -> RawEntryMut<'a, K, V, S> + pub fn from_hash(self, hash: u64, is_match: F) -> RawEntryMut<'a, K, V, S, A> where for<'b> F: FnMut(&'b K) -> bool, { @@ -1520,7 +3176,7 @@ impl<'a, K, V, S> RawEntryBuilderMut<'a, K, V, S> { } #[cfg_attr(feature = "inline-more", inline)] - fn search(self, hash: u64, mut is_match: F) -> RawEntryMut<'a, K, V, S> + fn search(self, hash: u64, mut is_match: F) -> RawEntryMut<'a, K, V, S, A> where for<'b> F: FnMut(&'b K) -> bool, { @@ -1528,6 +3184,7 @@ impl<'a, K, V, S> RawEntryBuilderMut<'a, K, V, S> { Some(elem) => RawEntryMut::Occupied(RawOccupiedEntryMut { elem, table: &mut self.map.table, + hash_builder: &self.map.hash_builder, }), None => RawEntryMut::Vacant(RawVacantEntryMut { table: &mut self.map.table, @@ -1537,8 +3194,18 @@ impl<'a, K, V, S> RawEntryBuilderMut<'a, K, V, S> { } } -impl<'a, K, V, S> RawEntryBuilder<'a, K, V, S> { - /// Access an entry by key. +impl<'a, K, V, S, A: Allocator + Clone> RawEntryBuilder<'a, K, V, S, A> { + /// Access an immutable entry by key. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let map: HashMap<&str, u32> = [("a", 100), ("b", 200)].into(); + /// let key = "a"; + /// assert_eq!(map.raw_entry().from_key(&key), Some((&"a", &100))); + /// ``` #[cfg_attr(feature = "inline-more", inline)] #[allow(clippy::wrong_self_convention)] pub fn from_key(self, k: &Q) -> Option<(&'a K, &'a V)> @@ -1547,20 +3214,38 @@ impl<'a, K, V, S> RawEntryBuilder<'a, K, V, S> { K: Borrow, Q: Hash + Eq, { - let mut hasher = self.map.hash_builder.build_hasher(); - k.hash(&mut hasher); - self.from_key_hashed_nocheck(hasher.finish(), k) + let hash = make_hash::(&self.map.hash_builder, k); + self.from_key_hashed_nocheck(hash, k) } - /// Access an entry by a key and its hash. + /// Access an immutable entry by a key and its hash. + /// + /// # Examples + /// + /// ``` + /// use core::hash::{BuildHasher, Hash}; + /// use hashbrown::HashMap; + /// + /// fn compute_hash(hash_builder: &S, key: &K) -> u64 { + /// use core::hash::Hasher; + /// let mut state = hash_builder.build_hasher(); + /// key.hash(&mut state); + /// state.finish() + /// } + /// + /// let map: HashMap<&str, u32> = [("a", 100), ("b", 200)].into(); + /// let key = "a"; + /// let hash = compute_hash(map.hasher(), &key); + /// assert_eq!(map.raw_entry().from_key_hashed_nocheck(hash, &key), Some((&"a", &100))); + /// ``` #[cfg_attr(feature = "inline-more", inline)] #[allow(clippy::wrong_self_convention)] pub fn from_key_hashed_nocheck(self, hash: u64, k: &Q) -> Option<(&'a K, &'a V)> where K: Borrow, - Q: Hash + Eq, + Q: Eq, { - self.from_hash(hash, |q| q.borrow().eq(k)) + self.from_hash(hash, equivalent(k)) } #[cfg_attr(feature = "inline-more", inline)] @@ -1568,16 +3253,32 @@ impl<'a, K, V, S> RawEntryBuilder<'a, K, V, S> { where F: FnMut(&K) -> bool, { - self.map - .table - .find(hash, |(k, _)| is_match(k)) - .map(|item| unsafe { - let &(ref key, ref value) = item.as_ref(); - (key, value) - }) + match self.map.table.get(hash, |(k, _)| is_match(k)) { + Some(&(ref key, ref value)) => Some((key, value)), + None => None, + } } - /// Access an entry by hash. + /// Access an immutable entry by hash and matching function. + /// + /// # Examples + /// + /// ``` + /// use core::hash::{BuildHasher, Hash}; + /// use hashbrown::HashMap; + /// + /// fn compute_hash(hash_builder: &S, key: &K) -> u64 { + /// use core::hash::Hasher; + /// let mut state = hash_builder.build_hasher(); + /// key.hash(&mut state); + /// state.finish() + /// } + /// + /// let map: HashMap<&str, u32> = [("a", 100), ("b", 200)].into(); + /// let key = "a"; + /// let hash = compute_hash(map.hasher(), &key); + /// assert_eq!(map.raw_entry().from_hash(hash, |k| k == &key), Some((&"a", &100))); + /// ``` #[cfg_attr(feature = "inline-more", inline)] #[allow(clippy::wrong_self_convention)] pub fn from_hash(self, hash: u64, is_match: F) -> Option<(&'a K, &'a V)> @@ -1588,7 +3289,7 @@ impl<'a, K, V, S> RawEntryBuilder<'a, K, V, S> { } } -impl<'a, K, V, S> RawEntryMut<'a, K, V, S> { +impl<'a, K, V, S, A: Allocator + Clone> RawEntryMut<'a, K, V, S, A> { /// Sets the value of the entry, and returns a RawOccupiedEntryMut. /// /// # Examples @@ -1602,7 +3303,7 @@ impl<'a, K, V, S> RawEntryMut<'a, K, V, S> { /// assert_eq!(entry.remove_entry(), ("horseyland", 37)); /// ``` #[cfg_attr(feature = "inline-more", inline)] - pub fn insert(self, key: K, value: V) -> RawOccupiedEntryMut<'a, K, V> + pub fn insert(self, key: K, value: V) -> RawOccupiedEntryMut<'a, K, V, S, A> where K: Hash, S: BuildHasher, @@ -1714,16 +3415,120 @@ impl<'a, K, V, S> RawEntryMut<'a, K, V, S> { RawEntryMut::Vacant(entry) => RawEntryMut::Vacant(entry), } } + + /// Provides shared access to the key and owned access to the value of + /// an occupied entry and allows to replace or remove it based on the + /// value of the returned option. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// use hashbrown::hash_map::RawEntryMut; + /// + /// let mut map: HashMap<&str, u32> = HashMap::new(); + /// + /// let entry = map + /// .raw_entry_mut() + /// .from_key("poneyland") + /// .and_replace_entry_with(|_k, _v| panic!()); + /// + /// match entry { + /// RawEntryMut::Vacant(_) => {}, + /// RawEntryMut::Occupied(_) => panic!(), + /// } + /// + /// map.insert("poneyland", 42); + /// + /// let entry = map + /// .raw_entry_mut() + /// .from_key("poneyland") + /// .and_replace_entry_with(|k, v| { + /// assert_eq!(k, &"poneyland"); + /// assert_eq!(v, 42); + /// Some(v + 1) + /// }); + /// + /// match entry { + /// RawEntryMut::Occupied(e) => { + /// assert_eq!(e.key(), &"poneyland"); + /// assert_eq!(e.get(), &43); + /// }, + /// RawEntryMut::Vacant(_) => panic!(), + /// } + /// + /// assert_eq!(map["poneyland"], 43); + /// + /// let entry = map + /// .raw_entry_mut() + /// .from_key("poneyland") + /// .and_replace_entry_with(|_k, _v| None); + /// + /// match entry { + /// RawEntryMut::Vacant(_) => {}, + /// RawEntryMut::Occupied(_) => panic!(), + /// } + /// + /// assert!(!map.contains_key("poneyland")); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn and_replace_entry_with(self, f: F) -> Self + where + F: FnOnce(&K, V) -> Option, + { + match self { + RawEntryMut::Occupied(entry) => entry.replace_entry_with(f), + RawEntryMut::Vacant(_) => self, + } + } } -impl<'a, K, V> RawOccupiedEntryMut<'a, K, V> { +impl<'a, K, V, S, A: Allocator + Clone> RawOccupiedEntryMut<'a, K, V, S, A> { /// Gets a reference to the key in the entry. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{HashMap, RawEntryMut}; + /// + /// let mut map: HashMap<&str, u32> = [("a", 100), ("b", 200)].into(); + /// + /// match map.raw_entry_mut().from_key(&"a") { + /// RawEntryMut::Vacant(_) => panic!(), + /// RawEntryMut::Occupied(o) => assert_eq!(o.key(), &"a") + /// } + /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn key(&self) -> &K { unsafe { &self.elem.as_ref().0 } } /// Gets a mutable reference to the key in the entry. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{HashMap, RawEntryMut}; + /// use std::rc::Rc; + /// + /// let key_one = Rc::new("a"); + /// let key_two = Rc::new("a"); + /// + /// let mut map: HashMap, u32> = HashMap::new(); + /// map.insert(key_one.clone(), 10); + /// + /// assert_eq!(map[&key_one], 10); + /// assert!(Rc::strong_count(&key_one) == 2 && Rc::strong_count(&key_two) == 1); + /// + /// match map.raw_entry_mut().from_key(&key_one) { + /// RawEntryMut::Vacant(_) => panic!(), + /// RawEntryMut::Occupied(mut o) => { + /// *o.key_mut() = key_two.clone(); + /// } + /// } + /// assert_eq!(map[&key_two], 10); + /// assert!(Rc::strong_count(&key_one) == 1 && Rc::strong_count(&key_two) == 2); + /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn key_mut(&mut self) -> &mut K { unsafe { &mut self.elem.as_mut().0 } @@ -1731,12 +3536,52 @@ impl<'a, K, V> RawOccupiedEntryMut<'a, K, V> { /// Converts the entry into a mutable reference to the key in the entry /// with a lifetime bound to the map itself. - #[cfg_attr(feature = "inline-more", inline)] - pub fn into_key(self) -> &'a mut K { - unsafe { &mut self.elem.as_mut().0 } - } - + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{HashMap, RawEntryMut}; + /// use std::rc::Rc; + /// + /// let key_one = Rc::new("a"); + /// let key_two = Rc::new("a"); + /// + /// let mut map: HashMap, u32> = HashMap::new(); + /// map.insert(key_one.clone(), 10); + /// + /// assert_eq!(map[&key_one], 10); + /// assert!(Rc::strong_count(&key_one) == 2 && Rc::strong_count(&key_two) == 1); + /// + /// let inside_key: &mut Rc<&str>; + /// + /// match map.raw_entry_mut().from_key(&key_one) { + /// RawEntryMut::Vacant(_) => panic!(), + /// RawEntryMut::Occupied(o) => inside_key = o.into_key(), + /// } + /// *inside_key = key_two.clone(); + /// + /// assert_eq!(map[&key_two], 10); + /// assert!(Rc::strong_count(&key_one) == 1 && Rc::strong_count(&key_two) == 2); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn into_key(self) -> &'a mut K { + unsafe { &mut self.elem.as_mut().0 } + } + /// Gets a reference to the value in the entry. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{HashMap, RawEntryMut}; + /// + /// let mut map: HashMap<&str, u32> = [("a", 100), ("b", 200)].into(); + /// + /// match map.raw_entry_mut().from_key(&"a") { + /// RawEntryMut::Vacant(_) => panic!(), + /// RawEntryMut::Occupied(o) => assert_eq!(o.get(), &100), + /// } + /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn get(&self) -> &V { unsafe { &self.elem.as_ref().1 } @@ -1744,20 +3589,66 @@ impl<'a, K, V> RawOccupiedEntryMut<'a, K, V> { /// Converts the OccupiedEntry into a mutable reference to the value in the entry /// with a lifetime bound to the map itself. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{HashMap, RawEntryMut}; + /// + /// let mut map: HashMap<&str, u32> = [("a", 100), ("b", 200)].into(); + /// + /// let value: &mut u32; + /// + /// match map.raw_entry_mut().from_key(&"a") { + /// RawEntryMut::Vacant(_) => panic!(), + /// RawEntryMut::Occupied(o) => value = o.into_mut(), + /// } + /// *value += 900; + /// + /// assert_eq!(map[&"a"], 1000); + /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn into_mut(self) -> &'a mut V { unsafe { &mut self.elem.as_mut().1 } } /// Gets a mutable reference to the value in the entry. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{HashMap, RawEntryMut}; + /// + /// let mut map: HashMap<&str, u32> = [("a", 100), ("b", 200)].into(); + /// + /// match map.raw_entry_mut().from_key(&"a") { + /// RawEntryMut::Vacant(_) => panic!(), + /// RawEntryMut::Occupied(mut o) => *o.get_mut() += 900, + /// } + /// + /// assert_eq!(map[&"a"], 1000); + /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn get_mut(&mut self) -> &mut V { unsafe { &mut self.elem.as_mut().1 } } /// Gets a reference to the key and value in the entry. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{HashMap, RawEntryMut}; + /// + /// let mut map: HashMap<&str, u32> = [("a", 100), ("b", 200)].into(); + /// + /// match map.raw_entry_mut().from_key(&"a") { + /// RawEntryMut::Vacant(_) => panic!(), + /// RawEntryMut::Occupied(o) => assert_eq!(o.get_key_value(), (&"a", &100)), + /// } + /// ``` #[cfg_attr(feature = "inline-more", inline)] - pub fn get_key_value(&mut self) -> (&K, &V) { + pub fn get_key_value(&self) -> (&K, &V) { unsafe { let &(ref key, ref value) = self.elem.as_ref(); (key, value) @@ -1765,6 +3656,33 @@ impl<'a, K, V> RawOccupiedEntryMut<'a, K, V> { } /// Gets a mutable reference to the key and value in the entry. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{HashMap, RawEntryMut}; + /// use std::rc::Rc; + /// + /// let key_one = Rc::new("a"); + /// let key_two = Rc::new("a"); + /// + /// let mut map: HashMap, u32> = HashMap::new(); + /// map.insert(key_one.clone(), 10); + /// + /// assert_eq!(map[&key_one], 10); + /// assert!(Rc::strong_count(&key_one) == 2 && Rc::strong_count(&key_two) == 1); + /// + /// match map.raw_entry_mut().from_key(&key_one) { + /// RawEntryMut::Vacant(_) => panic!(), + /// RawEntryMut::Occupied(mut o) => { + /// let (inside_key, inside_value) = o.get_key_value_mut(); + /// *inside_key = key_two.clone(); + /// *inside_value = 100; + /// } + /// } + /// assert_eq!(map[&key_two], 100); + /// assert!(Rc::strong_count(&key_one) == 1 && Rc::strong_count(&key_two) == 2); + /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn get_key_value_mut(&mut self) -> (&mut K, &mut V) { unsafe { @@ -1775,6 +3693,37 @@ impl<'a, K, V> RawOccupiedEntryMut<'a, K, V> { /// Converts the OccupiedEntry into a mutable reference to the key and value in the entry /// with a lifetime bound to the map itself. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{HashMap, RawEntryMut}; + /// use std::rc::Rc; + /// + /// let key_one = Rc::new("a"); + /// let key_two = Rc::new("a"); + /// + /// let mut map: HashMap, u32> = HashMap::new(); + /// map.insert(key_one.clone(), 10); + /// + /// assert_eq!(map[&key_one], 10); + /// assert!(Rc::strong_count(&key_one) == 2 && Rc::strong_count(&key_two) == 1); + /// + /// let inside_key: &mut Rc<&str>; + /// let inside_value: &mut u32; + /// match map.raw_entry_mut().from_key(&key_one) { + /// RawEntryMut::Vacant(_) => panic!(), + /// RawEntryMut::Occupied(o) => { + /// let tuple = o.into_key_value(); + /// inside_key = tuple.0; + /// inside_value = tuple.1; + /// } + /// } + /// *inside_key = key_two.clone(); + /// *inside_value = 100; + /// assert_eq!(map[&key_two], 100); + /// assert!(Rc::strong_count(&key_one) == 1 && Rc::strong_count(&key_two) == 2); + /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn into_key_value(self) -> (&'a mut K, &'a mut V) { unsafe { @@ -1784,49 +3733,213 @@ impl<'a, K, V> RawOccupiedEntryMut<'a, K, V> { } /// Sets the value of the entry, and returns the entry's old value. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{HashMap, RawEntryMut}; + /// + /// let mut map: HashMap<&str, u32> = [("a", 100), ("b", 200)].into(); + /// + /// match map.raw_entry_mut().from_key(&"a") { + /// RawEntryMut::Vacant(_) => panic!(), + /// RawEntryMut::Occupied(mut o) => assert_eq!(o.insert(1000), 100), + /// } + /// + /// assert_eq!(map[&"a"], 1000); + /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn insert(&mut self, value: V) -> V { mem::replace(self.get_mut(), value) } /// Sets the value of the entry, and returns the entry's old value. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{HashMap, RawEntryMut}; + /// use std::rc::Rc; + /// + /// let key_one = Rc::new("a"); + /// let key_two = Rc::new("a"); + /// + /// let mut map: HashMap, u32> = HashMap::new(); + /// map.insert(key_one.clone(), 10); + /// + /// assert_eq!(map[&key_one], 10); + /// assert!(Rc::strong_count(&key_one) == 2 && Rc::strong_count(&key_two) == 1); + /// + /// match map.raw_entry_mut().from_key(&key_one) { + /// RawEntryMut::Vacant(_) => panic!(), + /// RawEntryMut::Occupied(mut o) => { + /// let old_key = o.insert_key(key_two.clone()); + /// assert!(Rc::ptr_eq(&old_key, &key_one)); + /// } + /// } + /// assert_eq!(map[&key_two], 10); + /// assert!(Rc::strong_count(&key_one) == 1 && Rc::strong_count(&key_two) == 2); + /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn insert_key(&mut self, key: K) -> K { mem::replace(self.key_mut(), key) } /// Takes the value out of the entry, and returns it. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{HashMap, RawEntryMut}; + /// + /// let mut map: HashMap<&str, u32> = [("a", 100), ("b", 200)].into(); + /// + /// match map.raw_entry_mut().from_key(&"a") { + /// RawEntryMut::Vacant(_) => panic!(), + /// RawEntryMut::Occupied(o) => assert_eq!(o.remove(), 100), + /// } + /// assert_eq!(map.get(&"a"), None); + /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn remove(self) -> V { self.remove_entry().1 } /// Take the ownership of the key and value from the map. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{HashMap, RawEntryMut}; + /// + /// let mut map: HashMap<&str, u32> = [("a", 100), ("b", 200)].into(); + /// + /// match map.raw_entry_mut().from_key(&"a") { + /// RawEntryMut::Vacant(_) => panic!(), + /// RawEntryMut::Occupied(o) => assert_eq!(o.remove_entry(), ("a", 100)), + /// } + /// assert_eq!(map.get(&"a"), None); + /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn remove_entry(self) -> (K, V) { + unsafe { self.table.remove(self.elem) } + } + + /// Provides shared access to the key and owned access to the value of + /// the entry and allows to replace or remove it based on the + /// value of the returned option. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{HashMap, RawEntryMut}; + /// + /// let mut map: HashMap<&str, u32> = [("a", 100), ("b", 200)].into(); + /// + /// let raw_entry = match map.raw_entry_mut().from_key(&"a") { + /// RawEntryMut::Vacant(_) => panic!(), + /// RawEntryMut::Occupied(o) => o.replace_entry_with(|k, v| { + /// assert_eq!(k, &"a"); + /// assert_eq!(v, 100); + /// Some(v + 900) + /// }), + /// }; + /// let raw_entry = match raw_entry { + /// RawEntryMut::Vacant(_) => panic!(), + /// RawEntryMut::Occupied(o) => o.replace_entry_with(|k, v| { + /// assert_eq!(k, &"a"); + /// assert_eq!(v, 1000); + /// None + /// }), + /// }; + /// match raw_entry { + /// RawEntryMut::Vacant(_) => { }, + /// RawEntryMut::Occupied(_) => panic!(), + /// }; + /// assert_eq!(map.get(&"a"), None); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn replace_entry_with(self, f: F) -> RawEntryMut<'a, K, V, S, A> + where + F: FnOnce(&K, V) -> Option, + { unsafe { - self.table.erase_no_drop(&self.elem); - self.elem.read() + let still_occupied = self + .table + .replace_bucket_with(self.elem.clone(), |(key, value)| { + f(&key, value).map(|new_value| (key, new_value)) + }); + + if still_occupied { + RawEntryMut::Occupied(self) + } else { + RawEntryMut::Vacant(RawVacantEntryMut { + table: self.table, + hash_builder: self.hash_builder, + }) + } } } } -impl<'a, K, V, S> RawVacantEntryMut<'a, K, V, S> { +impl<'a, K, V, S, A: Allocator + Clone> RawVacantEntryMut<'a, K, V, S, A> { /// Sets the value of the entry with the VacantEntry's key, /// and returns a mutable reference to it. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{HashMap, RawEntryMut}; + /// + /// let mut map: HashMap<&str, u32> = [("a", 100), ("b", 200)].into(); + /// + /// match map.raw_entry_mut().from_key(&"c") { + /// RawEntryMut::Occupied(_) => panic!(), + /// RawEntryMut::Vacant(v) => assert_eq!(v.insert("c", 300), (&mut "c", &mut 300)), + /// } + /// + /// assert_eq!(map[&"c"], 300); + /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn insert(self, key: K, value: V) -> (&'a mut K, &'a mut V) where K: Hash, S: BuildHasher, { - let mut hasher = self.hash_builder.build_hasher(); - key.hash(&mut hasher); - self.insert_hashed_nocheck(hasher.finish(), key, value) + let hash = make_insert_hash::(self.hash_builder, &key); + self.insert_hashed_nocheck(hash, key, value) } /// Sets the value of the entry with the VacantEntry's key, /// and returns a mutable reference to it. + /// + /// # Examples + /// + /// ``` + /// use core::hash::{BuildHasher, Hash}; + /// use hashbrown::hash_map::{HashMap, RawEntryMut}; + /// + /// fn compute_hash(hash_builder: &S, key: &K) -> u64 { + /// use core::hash::Hasher; + /// let mut state = hash_builder.build_hasher(); + /// key.hash(&mut state); + /// state.finish() + /// } + /// + /// let mut map: HashMap<&str, u32> = [("a", 100), ("b", 200)].into(); + /// let key = "c"; + /// let hash = compute_hash(map.hasher(), &key); + /// + /// match map.raw_entry_mut().from_key_hashed_nocheck(hash, &key) { + /// RawEntryMut::Occupied(_) => panic!(), + /// RawEntryMut::Vacant(v) => assert_eq!( + /// v.insert_hashed_nocheck(hash, key, 300), + /// (&mut "c", &mut 300) + /// ), + /// } + /// + /// assert_eq!(map[&"c"], 300); + /// ``` #[cfg_attr(feature = "inline-more", inline)] #[allow(clippy::shadow_unrelated)] pub fn insert_hashed_nocheck(self, hash: u64, key: K, value: V) -> (&'a mut K, &'a mut V) @@ -1834,11 +3947,50 @@ impl<'a, K, V, S> RawVacantEntryMut<'a, K, V, S> { K: Hash, S: BuildHasher, { - let hash_builder = self.hash_builder; - self.insert_with_hasher(hash, key, value, |k| make_hash(hash_builder, k)) + let &mut (ref mut k, ref mut v) = self.table.insert_entry( + hash, + (key, value), + make_hasher::(self.hash_builder), + ); + (k, v) } /// Set the value of an entry with a custom hasher function. + /// + /// # Examples + /// + /// ``` + /// use core::hash::{BuildHasher, Hash}; + /// use hashbrown::hash_map::{HashMap, RawEntryMut}; + /// + /// fn make_hasher(hash_builder: &S) -> impl Fn(&K) -> u64 + '_ + /// where + /// K: Hash + ?Sized, + /// S: BuildHasher, + /// { + /// move |key: &K| { + /// use core::hash::Hasher; + /// let mut state = hash_builder.build_hasher(); + /// key.hash(&mut state); + /// state.finish() + /// } + /// } + /// + /// let mut map: HashMap<&str, u32> = HashMap::new(); + /// let key = "a"; + /// let hash_builder = map.hasher().clone(); + /// let hash = make_hasher(&hash_builder)(&key); + /// + /// match map.raw_entry_mut().from_hash(hash, |q| q == &key) { + /// RawEntryMut::Occupied(_) => panic!(), + /// RawEntryMut::Vacant(v) => assert_eq!( + /// v.insert_with_hasher(hash, key, 100, make_hasher(&hash_builder)), + /// (&mut "a", &mut 100) + /// ), + /// } + /// map.extend([("b", 200), ("c", 300), ("d", 400), ("e", 500), ("f", 600)]); + /// assert_eq!(map[&"a"], 100); + /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn insert_with_hasher( self, @@ -1850,40 +4002,39 @@ impl<'a, K, V, S> RawVacantEntryMut<'a, K, V, S> { where H: Fn(&K) -> u64, { - unsafe { - let elem = self.table.insert(hash, (key, value), |x| hasher(&x.0)); - let &mut (ref mut k, ref mut v) = elem.as_mut(); - (k, v) - } + let &mut (ref mut k, ref mut v) = self + .table + .insert_entry(hash, (key, value), |x| hasher(&x.0)); + (k, v) } #[cfg_attr(feature = "inline-more", inline)] - fn insert_entry(self, key: K, value: V) -> RawOccupiedEntryMut<'a, K, V> + fn insert_entry(self, key: K, value: V) -> RawOccupiedEntryMut<'a, K, V, S, A> where K: Hash, S: BuildHasher, { - let hash_builder = self.hash_builder; - let mut hasher = self.hash_builder.build_hasher(); - key.hash(&mut hasher); - - let elem = self.table.insert(hasher.finish(), (key, value), |k| { - make_hash(hash_builder, &k.0) - }); + let hash = make_insert_hash::(self.hash_builder, &key); + let elem = self.table.insert( + hash, + (key, value), + make_hasher::(self.hash_builder), + ); RawOccupiedEntryMut { elem, table: self.table, + hash_builder: self.hash_builder, } } } -impl Debug for RawEntryBuilderMut<'_, K, V, S> { +impl Debug for RawEntryBuilderMut<'_, K, V, S, A> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("RawEntryBuilder").finish() } } -impl Debug for RawEntryMut<'_, K, V, S> { +impl Debug for RawEntryMut<'_, K, V, S, A> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match *self { RawEntryMut::Vacant(ref v) => f.debug_tuple("RawEntry").field(v).finish(), @@ -1892,7 +4043,7 @@ impl Debug for RawEntryMut<'_, K, V, S> { } } -impl Debug for RawOccupiedEntryMut<'_, K, V> { +impl Debug for RawOccupiedEntryMut<'_, K, V, S, A> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("RawOccupiedEntryMut") .field("key", self.key()) @@ -1901,13 +4052,13 @@ impl Debug for RawOccupiedEntryMut<'_, K, V> { } } -impl Debug for RawVacantEntryMut<'_, K, V, S> { +impl Debug for RawVacantEntryMut<'_, K, V, S, A> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("RawVacantEntryMut").finish() } } -impl Debug for RawEntryBuilder<'_, K, V, S> { +impl Debug for RawEntryBuilder<'_, K, V, S, A> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("RawEntryBuilder").finish() } @@ -1919,15 +4070,79 @@ impl Debug for RawEntryBuilder<'_, K, V, S> { /// /// [`HashMap`]: struct.HashMap.html /// [`entry`]: struct.HashMap.html#method.entry -pub enum Entry<'a, K, V, S> { +/// +/// # Examples +/// +/// ``` +/// use hashbrown::hash_map::{Entry, HashMap, OccupiedEntry}; +/// +/// let mut map = HashMap::new(); +/// map.extend([("a", 10), ("b", 20), ("c", 30)]); +/// assert_eq!(map.len(), 3); +/// +/// // Existing key (insert) +/// let entry: Entry<_, _, _> = map.entry("a"); +/// let _raw_o: OccupiedEntry<_, _, _> = entry.insert(1); +/// assert_eq!(map.len(), 3); +/// // Nonexistent key (insert) +/// map.entry("d").insert(4); +/// +/// // Existing key (or_insert) +/// let v = map.entry("b").or_insert(2); +/// assert_eq!(std::mem::replace(v, 2), 20); +/// // Nonexistent key (or_insert) +/// map.entry("e").or_insert(5); +/// +/// // Existing key (or_insert_with) +/// let v = map.entry("c").or_insert_with(|| 3); +/// assert_eq!(std::mem::replace(v, 3), 30); +/// // Nonexistent key (or_insert_with) +/// map.entry("f").or_insert_with(|| 6); +/// +/// println!("Our HashMap: {:?}", map); +/// +/// let mut vec: Vec<_> = map.iter().map(|(&k, &v)| (k, v)).collect(); +/// // The `Iter` iterator produces items in arbitrary order, so the +/// // items must be sorted to test them against a sorted array. +/// vec.sort_unstable(); +/// assert_eq!(vec, [("a", 1), ("b", 2), ("c", 3), ("d", 4), ("e", 5), ("f", 6)]); +/// ``` +pub enum Entry<'a, K, V, S, A = Global> +where + A: Allocator + Clone, +{ /// An occupied entry. - Occupied(OccupiedEntry<'a, K, V, S>), + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{Entry, HashMap}; + /// let mut map: HashMap<_, _> = [("a", 100), ("b", 200)].into(); + /// + /// match map.entry("a") { + /// Entry::Vacant(_) => unreachable!(), + /// Entry::Occupied(_) => { } + /// } + /// ``` + Occupied(OccupiedEntry<'a, K, V, S, A>), /// A vacant entry. - Vacant(VacantEntry<'a, K, V, S>), + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{Entry, HashMap}; + /// let mut map: HashMap<&str, i32> = HashMap::new(); + /// + /// match map.entry("a") { + /// Entry::Occupied(_) => unreachable!(), + /// Entry::Vacant(_) => { } + /// } + /// ``` + Vacant(VacantEntry<'a, K, V, S, A>), } -impl Debug for Entry<'_, K, V, S> { +impl Debug for Entry<'_, K, V, S, A> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match *self { Entry::Vacant(ref v) => f.debug_tuple("Entry").field(v).finish(), @@ -1940,28 +4155,67 @@ impl Debug for Entry<'_, K, V, S> { /// It is part of the [`Entry`] enum. /// /// [`Entry`]: enum.Entry.html -pub struct OccupiedEntry<'a, K, V, S> { +/// +/// # Examples +/// +/// ``` +/// use hashbrown::hash_map::{Entry, HashMap, OccupiedEntry}; +/// +/// let mut map = HashMap::new(); +/// map.extend([("a", 10), ("b", 20), ("c", 30)]); +/// +/// let _entry_o: OccupiedEntry<_, _, _> = map.entry("a").insert(100); +/// assert_eq!(map.len(), 3); +/// +/// // Existing key (insert and update) +/// match map.entry("a") { +/// Entry::Vacant(_) => unreachable!(), +/// Entry::Occupied(mut view) => { +/// assert_eq!(view.get(), &100); +/// let v = view.get_mut(); +/// *v *= 10; +/// assert_eq!(view.insert(1111), 1000); +/// } +/// } +/// +/// assert_eq!(map[&"a"], 1111); +/// assert_eq!(map.len(), 3); +/// +/// // Existing key (take) +/// match map.entry("c") { +/// Entry::Vacant(_) => unreachable!(), +/// Entry::Occupied(view) => { +/// assert_eq!(view.remove_entry(), ("c", 30)); +/// } +/// } +/// assert_eq!(map.get(&"c"), None); +/// assert_eq!(map.len(), 2); +/// ``` +pub struct OccupiedEntry<'a, K, V, S, A: Allocator + Clone = Global> { + hash: u64, key: Option, elem: Bucket<(K, V)>, - table: &'a mut HashMap, + table: &'a mut HashMap, } -unsafe impl Send for OccupiedEntry<'_, K, V, S> +unsafe impl Send for OccupiedEntry<'_, K, V, S, A> where K: Send, V: Send, S: Send, + A: Send + Allocator + Clone, { } -unsafe impl Sync for OccupiedEntry<'_, K, V, S> +unsafe impl Sync for OccupiedEntry<'_, K, V, S, A> where K: Sync, V: Sync, S: Sync, + A: Sync + Allocator + Clone, { } -impl Debug for OccupiedEntry<'_, K, V, S> { +impl Debug for OccupiedEntry<'_, K, V, S, A> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("OccupiedEntry") .field("key", self.key()) @@ -1974,41 +4228,417 @@ impl Debug for OccupiedEntry<'_, K, V, S> { /// It is part of the [`Entry`] enum. /// /// [`Entry`]: enum.Entry.html -pub struct VacantEntry<'a, K, V, S> { +/// +/// # Examples +/// +/// ``` +/// use hashbrown::hash_map::{Entry, HashMap, VacantEntry}; +/// +/// let mut map = HashMap::<&str, i32>::new(); +/// +/// let entry_v: VacantEntry<_, _, _> = match map.entry("a") { +/// Entry::Vacant(view) => view, +/// Entry::Occupied(_) => unreachable!(), +/// }; +/// entry_v.insert(10); +/// assert!(map[&"a"] == 10 && map.len() == 1); +/// +/// // Nonexistent key (insert and update) +/// match map.entry("b") { +/// Entry::Occupied(_) => unreachable!(), +/// Entry::Vacant(view) => { +/// let value = view.insert(2); +/// assert_eq!(*value, 2); +/// *value = 20; +/// } +/// } +/// assert!(map[&"b"] == 20 && map.len() == 2); +/// ``` +pub struct VacantEntry<'a, K, V, S, A: Allocator + Clone = Global> { hash: u64, key: K, - table: &'a mut HashMap, + table: &'a mut HashMap, } -impl Debug for VacantEntry<'_, K, V, S> { +impl Debug for VacantEntry<'_, K, V, S, A> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("VacantEntry").field(self.key()).finish() } } -impl<'a, K, V, S> IntoIterator for &'a HashMap { - type Item = (&'a K, &'a V); - type IntoIter = Iter<'a, K, V>; - - #[cfg_attr(feature = "inline-more", inline)] - fn into_iter(self) -> Iter<'a, K, V> { - self.iter() - } -} +/// A view into a single entry in a map, which may either be vacant or occupied, +/// with any borrowed form of the map's key type. +/// +/// +/// This `enum` is constructed from the [`entry_ref`] method on [`HashMap`]. +/// +/// [`Hash`] and [`Eq`] on the borrowed form of the map's key type *must* match those +/// for the key type. It also require that key may be constructed from the borrowed +/// form through the [`From`] trait. +/// +/// [`HashMap`]: struct.HashMap.html +/// [`entry_ref`]: struct.HashMap.html#method.entry_ref +/// [`Eq`]: https://doc.rust-lang.org/std/cmp/trait.Eq.html +/// [`Hash`]: https://doc.rust-lang.org/std/hash/trait.Hash.html +/// [`From`]: https://doc.rust-lang.org/std/convert/trait.From.html +/// +/// # Examples +/// +/// ``` +/// use hashbrown::hash_map::{EntryRef, HashMap, OccupiedEntryRef}; +/// +/// let mut map = HashMap::new(); +/// map.extend([("a".to_owned(), 10), ("b".into(), 20), ("c".into(), 30)]); +/// assert_eq!(map.len(), 3); +/// +/// // Existing key (insert) +/// let key = String::from("a"); +/// let entry: EntryRef<_, _, _, _> = map.entry_ref(&key); +/// let _raw_o: OccupiedEntryRef<_, _, _, _> = entry.insert(1); +/// assert_eq!(map.len(), 3); +/// // Nonexistent key (insert) +/// map.entry_ref("d").insert(4); +/// +/// // Existing key (or_insert) +/// let v = map.entry_ref("b").or_insert(2); +/// assert_eq!(std::mem::replace(v, 2), 20); +/// // Nonexistent key (or_insert) +/// map.entry_ref("e").or_insert(5); +/// +/// // Existing key (or_insert_with) +/// let v = map.entry_ref("c").or_insert_with(|| 3); +/// assert_eq!(std::mem::replace(v, 3), 30); +/// // Nonexistent key (or_insert_with) +/// map.entry_ref("f").or_insert_with(|| 6); +/// +/// println!("Our HashMap: {:?}", map); +/// +/// for (key, value) in ["a", "b", "c", "d", "e", "f"].into_iter().zip(1..=6) { +/// assert_eq!(map[key], value) +/// } +/// assert_eq!(map.len(), 6); +/// ``` +pub enum EntryRef<'a, 'b, K, Q: ?Sized, V, S, A = Global> +where + A: Allocator + Clone, +{ + /// An occupied entry. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{EntryRef, HashMap}; + /// let mut map: HashMap<_, _> = [("a".to_owned(), 100), ("b".into(), 200)].into(); + /// + /// match map.entry_ref("a") { + /// EntryRef::Vacant(_) => unreachable!(), + /// EntryRef::Occupied(_) => { } + /// } + /// ``` + Occupied(OccupiedEntryRef<'a, 'b, K, Q, V, S, A>), -impl<'a, K, V, S> IntoIterator for &'a mut HashMap { + /// A vacant entry. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{EntryRef, HashMap}; + /// let mut map: HashMap = HashMap::new(); + /// + /// match map.entry_ref("a") { + /// EntryRef::Occupied(_) => unreachable!(), + /// EntryRef::Vacant(_) => { } + /// } + /// ``` + Vacant(VacantEntryRef<'a, 'b, K, Q, V, S, A>), +} + +impl, Q: ?Sized + Debug, V: Debug, S, A: Allocator + Clone> Debug + for EntryRef<'_, '_, K, Q, V, S, A> +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + EntryRef::Vacant(ref v) => f.debug_tuple("EntryRef").field(v).finish(), + EntryRef::Occupied(ref o) => f.debug_tuple("EntryRef").field(o).finish(), + } + } +} + +enum KeyOrRef<'a, K, Q: ?Sized> { + Borrowed(&'a Q), + Owned(K), +} + +impl<'a, K, Q: ?Sized> KeyOrRef<'a, K, Q> { + fn into_owned(self) -> K + where + K: From<&'a Q>, + { + match self { + Self::Borrowed(borrowed) => borrowed.into(), + Self::Owned(owned) => owned, + } + } +} + +impl<'a, K: Borrow, Q: ?Sized> AsRef for KeyOrRef<'a, K, Q> { + fn as_ref(&self) -> &Q { + match self { + Self::Borrowed(borrowed) => borrowed, + Self::Owned(owned) => owned.borrow(), + } + } +} + +/// A view into an occupied entry in a `HashMap`. +/// It is part of the [`EntryRef`] enum. +/// +/// [`EntryRef`]: enum.EntryRef.html +/// +/// # Examples +/// +/// ``` +/// use hashbrown::hash_map::{EntryRef, HashMap, OccupiedEntryRef}; +/// +/// let mut map = HashMap::new(); +/// map.extend([("a".to_owned(), 10), ("b".into(), 20), ("c".into(), 30)]); +/// +/// let key = String::from("a"); +/// let _entry_o: OccupiedEntryRef<_, _, _, _> = map.entry_ref(&key).insert(100); +/// assert_eq!(map.len(), 3); +/// +/// // Existing key (insert and update) +/// match map.entry_ref("a") { +/// EntryRef::Vacant(_) => unreachable!(), +/// EntryRef::Occupied(mut view) => { +/// assert_eq!(view.get(), &100); +/// let v = view.get_mut(); +/// *v *= 10; +/// assert_eq!(view.insert(1111), 1000); +/// } +/// } +/// +/// assert_eq!(map["a"], 1111); +/// assert_eq!(map.len(), 3); +/// +/// // Existing key (take) +/// match map.entry_ref("c") { +/// EntryRef::Vacant(_) => unreachable!(), +/// EntryRef::Occupied(view) => { +/// assert_eq!(view.remove_entry(), ("c".to_owned(), 30)); +/// } +/// } +/// assert_eq!(map.get("c"), None); +/// assert_eq!(map.len(), 2); +/// ``` +pub struct OccupiedEntryRef<'a, 'b, K, Q: ?Sized, V, S, A: Allocator + Clone = Global> { + hash: u64, + key: Option>, + elem: Bucket<(K, V)>, + table: &'a mut HashMap, +} + +unsafe impl<'a, 'b, K, Q, V, S, A> Send for OccupiedEntryRef<'a, 'b, K, Q, V, S, A> +where + K: Send, + Q: Sync + ?Sized, + V: Send, + S: Send, + A: Send + Allocator + Clone, +{ +} +unsafe impl<'a, 'b, K, Q, V, S, A> Sync for OccupiedEntryRef<'a, 'b, K, Q, V, S, A> +where + K: Sync, + Q: Sync + ?Sized, + V: Sync, + S: Sync, + A: Sync + Allocator + Clone, +{ +} + +impl, Q: ?Sized + Debug, V: Debug, S, A: Allocator + Clone> Debug + for OccupiedEntryRef<'_, '_, K, Q, V, S, A> +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OccupiedEntryRef") + .field("key", &self.key()) + .field("value", &self.get()) + .finish() + } +} + +/// A view into a vacant entry in a `HashMap`. +/// It is part of the [`EntryRef`] enum. +/// +/// [`EntryRef`]: enum.EntryRef.html +/// +/// # Examples +/// +/// ``` +/// use hashbrown::hash_map::{EntryRef, HashMap, VacantEntryRef}; +/// +/// let mut map = HashMap::::new(); +/// +/// let entry_v: VacantEntryRef<_, _, _, _> = match map.entry_ref("a") { +/// EntryRef::Vacant(view) => view, +/// EntryRef::Occupied(_) => unreachable!(), +/// }; +/// entry_v.insert(10); +/// assert!(map["a"] == 10 && map.len() == 1); +/// +/// // Nonexistent key (insert and update) +/// match map.entry_ref("b") { +/// EntryRef::Occupied(_) => unreachable!(), +/// EntryRef::Vacant(view) => { +/// let value = view.insert(2); +/// assert_eq!(*value, 2); +/// *value = 20; +/// } +/// } +/// assert!(map["b"] == 20 && map.len() == 2); +/// ``` +pub struct VacantEntryRef<'a, 'b, K, Q: ?Sized, V, S, A: Allocator + Clone = Global> { + hash: u64, + key: KeyOrRef<'b, K, Q>, + table: &'a mut HashMap, +} + +impl, Q: ?Sized + Debug, V, S, A: Allocator + Clone> Debug + for VacantEntryRef<'_, '_, K, Q, V, S, A> +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("VacantEntryRef").field(&self.key()).finish() + } +} + +/// The error returned by [`try_insert`](HashMap::try_insert) when the key already exists. +/// +/// Contains the occupied entry, and the value that was not inserted. +/// +/// # Examples +/// +/// ``` +/// use hashbrown::hash_map::{HashMap, OccupiedError}; +/// +/// let mut map: HashMap<_, _> = [("a", 10), ("b", 20)].into(); +/// +/// // try_insert method returns mutable reference to the value if keys are vacant, +/// // but if the map did have key present, nothing is updated, and the provided +/// // value is returned inside `Err(_)` variant +/// match map.try_insert("a", 100) { +/// Err(OccupiedError { mut entry, value }) => { +/// assert_eq!(entry.key(), &"a"); +/// assert_eq!(value, 100); +/// assert_eq!(entry.insert(100), 10) +/// } +/// _ => unreachable!(), +/// } +/// assert_eq!(map[&"a"], 100); +/// ``` +pub struct OccupiedError<'a, K, V, S, A: Allocator + Clone = Global> { + /// The entry in the map that was already occupied. + pub entry: OccupiedEntry<'a, K, V, S, A>, + /// The value which was not inserted, because the entry was already occupied. + pub value: V, +} + +impl Debug for OccupiedError<'_, K, V, S, A> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OccupiedError") + .field("key", self.entry.key()) + .field("old_value", self.entry.get()) + .field("new_value", &self.value) + .finish() + } +} + +impl<'a, K: Debug, V: Debug, S, A: Allocator + Clone> fmt::Display + for OccupiedError<'a, K, V, S, A> +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "failed to insert {:?}, key {:?} already exists with value {:?}", + self.value, + self.entry.key(), + self.entry.get(), + ) + } +} + +impl<'a, K, V, S, A: Allocator + Clone> IntoIterator for &'a HashMap { + type Item = (&'a K, &'a V); + type IntoIter = Iter<'a, K, V>; + + /// Creates an iterator over the entries of a `HashMap` in arbitrary order. + /// The iterator element type is `(&'a K, &'a V)`. + /// + /// Return the same `Iter` struct as by the [`iter`] method on [`HashMap`]. + /// + /// [`iter`]: struct.HashMap.html#method.iter + /// [`HashMap`]: struct.HashMap.html + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// let map_one: HashMap<_, _> = [(1, "a"), (2, "b"), (3, "c")].into(); + /// let mut map_two = HashMap::new(); + /// + /// for (key, value) in &map_one { + /// println!("Key: {}, Value: {}", key, value); + /// map_two.insert_unique_unchecked(*key, *value); + /// } + /// + /// assert_eq!(map_one, map_two); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + fn into_iter(self) -> Iter<'a, K, V> { + self.iter() + } +} + +impl<'a, K, V, S, A: Allocator + Clone> IntoIterator for &'a mut HashMap { type Item = (&'a K, &'a mut V); type IntoIter = IterMut<'a, K, V>; + /// Creates an iterator over the entries of a `HashMap` in arbitrary order + /// with mutable references to the values. The iterator element type is + /// `(&'a K, &'a mut V)`. + /// + /// Return the same `IterMut` struct as by the [`iter_mut`] method on + /// [`HashMap`]. + /// + /// [`iter_mut`]: struct.HashMap.html#method.iter_mut + /// [`HashMap`]: struct.HashMap.html + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// let mut map: HashMap<_, _> = [("a", 1), ("b", 2), ("c", 3)].into(); + /// + /// for (key, value) in &mut map { + /// println!("Key: {}, Value: {}", key, value); + /// *value *= 2; + /// } + /// + /// let mut vec = map.iter().collect::>(); + /// // The `Iter` iterator produces items in arbitrary order, so the + /// // items must be sorted to test them against a sorted array. + /// vec.sort_unstable(); + /// assert_eq!(vec, [(&"a", &2), (&"b", &4), (&"c", &6)]); + /// ``` #[cfg_attr(feature = "inline-more", inline)] fn into_iter(self) -> IterMut<'a, K, V> { self.iter_mut() } } -impl IntoIterator for HashMap { +impl IntoIterator for HashMap { type Item = (K, V); - type IntoIter = IntoIter; + type IntoIter = IntoIter; /// Creates a consuming iterator, that is, one that moves each key-value /// pair out of the map in arbitrary order. The map cannot be used after @@ -2019,16 +4649,17 @@ impl IntoIterator for HashMap { /// ``` /// use hashbrown::HashMap; /// - /// let mut map = HashMap::new(); - /// map.insert("a", 1); - /// map.insert("b", 2); - /// map.insert("c", 3); + /// let map: HashMap<_, _> = [("a", 1), ("b", 2), ("c", 3)].into(); /// /// // Not possible with .iter() - /// let vec: Vec<(&str, i32)> = map.into_iter().collect(); + /// let mut vec: Vec<(&str, i32)> = map.into_iter().collect(); + /// // The `IntoIter` iterator produces items in arbitrary order, so + /// // the items must be sorted to test them against a sorted array. + /// vec.sort_unstable(); + /// assert_eq!(vec, [("a", 1), ("b", 2), ("c", 3)]); /// ``` #[cfg_attr(feature = "inline-more", inline)] - fn into_iter(self) -> IntoIter { + fn into_iter(self) -> IntoIter { IntoIter { inner: self.table.into_iter(), } @@ -2040,10 +4671,14 @@ impl<'a, K, V> Iterator for Iter<'a, K, V> { #[cfg_attr(feature = "inline-more", inline)] fn next(&mut self) -> Option<(&'a K, &'a V)> { - self.inner.next().map(|x| unsafe { - let r = x.as_ref(); - (&r.0, &r.1) - }) + // Avoid `Option::map` because it bloats LLVM IR. + match self.inner.next() { + Some(x) => unsafe { + let r = x.as_ref(); + Some((&r.0, &r.1)) + }, + None => None, + } } #[cfg_attr(feature = "inline-more", inline)] fn size_hint(&self) -> (usize, Option) { @@ -2064,10 +4699,14 @@ impl<'a, K, V> Iterator for IterMut<'a, K, V> { #[cfg_attr(feature = "inline-more", inline)] fn next(&mut self) -> Option<(&'a K, &'a mut V)> { - self.inner.next().map(|x| unsafe { - let r = x.as_mut(); - (&r.0, &mut r.1) - }) + // Avoid `Option::map` because it bloats LLVM IR. + match self.inner.next() { + Some(x) => unsafe { + let r = x.as_mut(); + Some((&r.0, &mut r.1)) + }, + None => None, + } } #[cfg_attr(feature = "inline-more", inline)] fn size_hint(&self) -> (usize, Option) { @@ -2092,7 +4731,7 @@ where } } -impl Iterator for IntoIter { +impl Iterator for IntoIter { type Item = (K, V); #[cfg_attr(feature = "inline-more", inline)] @@ -2104,15 +4743,15 @@ impl Iterator for IntoIter { self.inner.size_hint() } } -impl ExactSizeIterator for IntoIter { +impl ExactSizeIterator for IntoIter { #[cfg_attr(feature = "inline-more", inline)] fn len(&self) -> usize { self.inner.len() } } -impl FusedIterator for IntoIter {} +impl FusedIterator for IntoIter {} -impl fmt::Debug for IntoIter { +impl fmt::Debug for IntoIter { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_list().entries(self.iter()).finish() } @@ -2123,7 +4762,11 @@ impl<'a, K, V> Iterator for Keys<'a, K, V> { #[cfg_attr(feature = "inline-more", inline)] fn next(&mut self) -> Option<&'a K> { - self.inner.next().map(|(k, _)| k) + // Avoid `Option::map` because it bloats LLVM IR. + match self.inner.next() { + Some((k, _)) => Some(k), + None => None, + } } #[cfg_attr(feature = "inline-more", inline)] fn size_hint(&self) -> (usize, Option) { @@ -2143,7 +4786,11 @@ impl<'a, K, V> Iterator for Values<'a, K, V> { #[cfg_attr(feature = "inline-more", inline)] fn next(&mut self) -> Option<&'a V> { - self.inner.next().map(|(_, v)| v) + // Avoid `Option::map` because it bloats LLVM IR. + match self.inner.next() { + Some((_, v)) => Some(v), + None => None, + } } #[cfg_attr(feature = "inline-more", inline)] fn size_hint(&self) -> (usize, Option) { @@ -2163,7 +4810,11 @@ impl<'a, K, V> Iterator for ValuesMut<'a, K, V> { #[cfg_attr(feature = "inline-more", inline)] fn next(&mut self) -> Option<&'a mut V> { - self.inner.next().map(|(_, v)| v) + // Avoid `Option::map` because it bloats LLVM IR. + match self.inner.next() { + Some((_, v)) => Some(v), + None => None, + } } #[cfg_attr(feature = "inline-more", inline)] fn size_hint(&self) -> (usize, Option) { @@ -2178,17 +4829,15 @@ impl ExactSizeIterator for ValuesMut<'_, K, V> { } impl FusedIterator for ValuesMut<'_, K, V> {} -impl fmt::Debug for ValuesMut<'_, K, V> -where - K: fmt::Debug, - V: fmt::Debug, -{ +impl fmt::Debug for ValuesMut<'_, K, V> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_list().entries(self.inner.iter()).finish() + f.debug_list() + .entries(self.inner.iter().map(|(_, val)| val)) + .finish() } } -impl<'a, K, V> Iterator for Drain<'a, K, V> { +impl<'a, K, V, A: Allocator + Clone> Iterator for Drain<'a, K, V, A> { type Item = (K, V); #[cfg_attr(feature = "inline-more", inline)] @@ -2200,25 +4849,26 @@ impl<'a, K, V> Iterator for Drain<'a, K, V> { self.inner.size_hint() } } -impl ExactSizeIterator for Drain<'_, K, V> { +impl ExactSizeIterator for Drain<'_, K, V, A> { #[cfg_attr(feature = "inline-more", inline)] fn len(&self) -> usize { self.inner.len() } } -impl FusedIterator for Drain<'_, K, V> {} +impl FusedIterator for Drain<'_, K, V, A> {} -impl fmt::Debug for Drain<'_, K, V> +impl fmt::Debug for Drain<'_, K, V, A> where K: fmt::Debug, V: fmt::Debug, + A: Allocator + Clone, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_list().entries(self.iter()).finish() } } -impl<'a, K, V, S> Entry<'a, K, V, S> { +impl<'a, K, V, S, A: Allocator + Clone> Entry<'a, K, V, S, A> { /// Sets the value of the entry, and returns an OccupiedEntry. /// /// # Examples @@ -2232,7 +4882,7 @@ impl<'a, K, V, S> Entry<'a, K, V, S> { /// assert_eq!(entry.key(), &"horseyland"); /// ``` #[cfg_attr(feature = "inline-more", inline)] - pub fn insert(self, value: V) -> OccupiedEntry<'a, K, V, S> + pub fn insert(self, value: V) -> OccupiedEntry<'a, K, V, S, A> where K: Hash, S: BuildHasher, @@ -2256,9 +4906,11 @@ impl<'a, K, V, S> Entry<'a, K, V, S> { /// /// let mut map: HashMap<&str, u32> = HashMap::new(); /// + /// // nonexistent key /// map.entry("poneyland").or_insert(3); /// assert_eq!(map["poneyland"], 3); /// + /// // existing key /// *map.entry("poneyland").or_insert(10) *= 2; /// assert_eq!(map["poneyland"], 6); /// ``` @@ -2282,12 +4934,15 @@ impl<'a, K, V, S> Entry<'a, K, V, S> { /// ``` /// use hashbrown::HashMap; /// - /// let mut map: HashMap<&str, String> = HashMap::new(); - /// let s = "hoho".to_string(); + /// let mut map: HashMap<&str, u32> = HashMap::new(); /// - /// map.entry("poneyland").or_insert_with(|| s); + /// // nonexistent key + /// map.entry("poneyland").or_insert_with(|| 3); + /// assert_eq!(map["poneyland"], 3); /// - /// assert_eq!(map["poneyland"], "hoho".to_string()); + /// // existing key + /// *map.entry("poneyland").or_insert_with(|| 10) *= 2; + /// assert_eq!(map["poneyland"], 6); /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn or_insert_with V>(self, default: F) -> &'a mut V @@ -2301,6 +4956,43 @@ impl<'a, K, V, S> Entry<'a, K, V, S> { } } + /// Ensures a value is in the entry by inserting, if empty, the result of the default function. + /// This method allows for generating key-derived values for insertion by providing the default + /// function a reference to the key that was moved during the `.entry(key)` method call. + /// + /// The reference to the moved key is provided so that cloning or copying the key is + /// unnecessary, unlike with `.or_insert_with(|| ... )`. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let mut map: HashMap<&str, usize> = HashMap::new(); + /// + /// // nonexistent key + /// map.entry("poneyland").or_insert_with_key(|key| key.chars().count()); + /// assert_eq!(map["poneyland"], 9); + /// + /// // existing key + /// *map.entry("poneyland").or_insert_with_key(|key| key.chars().count() * 10) *= 2; + /// assert_eq!(map["poneyland"], 18); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn or_insert_with_key V>(self, default: F) -> &'a mut V + where + K: Hash, + S: BuildHasher, + { + match self { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(entry) => { + let value = default(entry.key()); + entry.insert(value) + } + } + } + /// Returns a reference to this entry's key. /// /// # Examples @@ -2309,7 +5001,11 @@ impl<'a, K, V, S> Entry<'a, K, V, S> { /// use hashbrown::HashMap; /// /// let mut map: HashMap<&str, u32> = HashMap::new(); + /// map.entry("poneyland").or_insert(3); + /// // existing key /// assert_eq!(map.entry("poneyland").key(), &"poneyland"); + /// // nonexistent key + /// assert_eq!(map.entry("horseland").key(), &"horseland"); /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn key(&self) -> &K { @@ -2352,9 +5048,74 @@ impl<'a, K, V, S> Entry<'a, K, V, S> { Entry::Vacant(entry) => Entry::Vacant(entry), } } + + /// Provides shared access to the key and owned access to the value of + /// an occupied entry and allows to replace or remove it based on the + /// value of the returned option. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// use hashbrown::hash_map::Entry; + /// + /// let mut map: HashMap<&str, u32> = HashMap::new(); + /// + /// let entry = map + /// .entry("poneyland") + /// .and_replace_entry_with(|_k, _v| panic!()); + /// + /// match entry { + /// Entry::Vacant(e) => { + /// assert_eq!(e.key(), &"poneyland"); + /// } + /// Entry::Occupied(_) => panic!(), + /// } + /// + /// map.insert("poneyland", 42); + /// + /// let entry = map + /// .entry("poneyland") + /// .and_replace_entry_with(|k, v| { + /// assert_eq!(k, &"poneyland"); + /// assert_eq!(v, 42); + /// Some(v + 1) + /// }); + /// + /// match entry { + /// Entry::Occupied(e) => { + /// assert_eq!(e.key(), &"poneyland"); + /// assert_eq!(e.get(), &43); + /// } + /// Entry::Vacant(_) => panic!(), + /// } + /// + /// assert_eq!(map["poneyland"], 43); + /// + /// let entry = map + /// .entry("poneyland") + /// .and_replace_entry_with(|_k, _v| None); + /// + /// match entry { + /// Entry::Vacant(e) => assert_eq!(e.key(), &"poneyland"), + /// Entry::Occupied(_) => panic!(), + /// } + /// + /// assert!(!map.contains_key("poneyland")); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn and_replace_entry_with(self, f: F) -> Self + where + F: FnOnce(&K, V) -> Option, + { + match self { + Entry::Occupied(entry) => entry.replace_entry_with(f), + Entry::Vacant(_) => self, + } + } } -impl<'a, K, V: Default, S> Entry<'a, K, V, S> { +impl<'a, K, V: Default, S, A: Allocator + Clone> Entry<'a, K, V, S, A> { /// Ensures a value is in the entry by inserting the default value if empty, /// and returns a mutable reference to the value in the entry. /// @@ -2364,9 +5125,15 @@ impl<'a, K, V: Default, S> Entry<'a, K, V, S> { /// use hashbrown::HashMap; /// /// let mut map: HashMap<&str, Option> = HashMap::new(); - /// map.entry("poneyland").or_default(); /// + /// // nonexistent key + /// map.entry("poneyland").or_default(); /// assert_eq!(map["poneyland"], None); + /// + /// map.insert("horseland", Some(3)); + /// + /// // existing key + /// assert_eq!(map.entry("horseland").or_default(), &mut Some(3)); /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn or_default(self) -> &'a mut V @@ -2381,17 +5148,21 @@ impl<'a, K, V: Default, S> Entry<'a, K, V, S> { } } -impl<'a, K, V, S> OccupiedEntry<'a, K, V, S> { +impl<'a, K, V, S, A: Allocator + Clone> OccupiedEntry<'a, K, V, S, A> { /// Gets a reference to the key in the entry. /// /// # Examples /// /// ``` - /// use hashbrown::HashMap; + /// use hashbrown::hash_map::{Entry, HashMap}; /// /// let mut map: HashMap<&str, u32> = HashMap::new(); /// map.entry("poneyland").or_insert(12); - /// assert_eq!(map.entry("poneyland").key(), &"poneyland"); + /// + /// match map.entry("poneyland") { + /// Entry::Vacant(_) => panic!(), + /// Entry::Occupied(entry) => assert_eq!(entry.key(), &"poneyland"), + /// } /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn key(&self) -> &K { @@ -2399,6 +5170,7 @@ impl<'a, K, V, S> OccupiedEntry<'a, K, V, S> { } /// Take the ownership of the key and value from the map. + /// Keeps the allocated memory for reuse. /// /// # Examples /// @@ -2407,21 +5179,24 @@ impl<'a, K, V, S> OccupiedEntry<'a, K, V, S> { /// use hashbrown::hash_map::Entry; /// /// let mut map: HashMap<&str, u32> = HashMap::new(); + /// // The map is empty + /// assert!(map.is_empty() && map.capacity() == 0); + /// /// map.entry("poneyland").or_insert(12); + /// let capacity_before_remove = map.capacity(); /// /// if let Entry::Occupied(o) = map.entry("poneyland") { /// // We delete the entry from the map. - /// o.remove_entry(); + /// assert_eq!(o.remove_entry(), ("poneyland", 12)); /// } /// /// assert_eq!(map.contains_key("poneyland"), false); + /// // Now map hold none elements but capacity is equal to the old one + /// assert!(map.len() == 0 && map.capacity() == capacity_before_remove); /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn remove_entry(self) -> (K, V) { - unsafe { - self.table.table.erase_no_drop(&self.elem); - self.elem.read() - } + unsafe { self.table.table.remove(self.elem) } } /// Gets a reference to the value in the entry. @@ -2435,8 +5210,9 @@ impl<'a, K, V, S> OccupiedEntry<'a, K, V, S> { /// let mut map: HashMap<&str, u32> = HashMap::new(); /// map.entry("poneyland").or_insert(12); /// - /// if let Entry::Occupied(o) = map.entry("poneyland") { - /// assert_eq!(o.get(), &12); + /// match map.entry("poneyland") { + /// Entry::Vacant(_) => panic!(), + /// Entry::Occupied(entry) => assert_eq!(entry.get(), &12), /// } /// ``` #[cfg_attr(feature = "inline-more", inline)] @@ -2486,16 +5262,19 @@ impl<'a, K, V, S> OccupiedEntry<'a, K, V, S> { /// # Examples /// /// ``` - /// use hashbrown::HashMap; - /// use hashbrown::hash_map::Entry; + /// use hashbrown::hash_map::{Entry, HashMap}; /// /// let mut map: HashMap<&str, u32> = HashMap::new(); /// map.entry("poneyland").or_insert(12); /// /// assert_eq!(map["poneyland"], 12); - /// if let Entry::Occupied(o) = map.entry("poneyland") { - /// *o.into_mut() += 10; + /// + /// let value: &mut u32; + /// match map.entry("poneyland") { + /// Entry::Occupied(entry) => value = entry.into_mut(), + /// Entry::Vacant(_) => panic!(), /// } + /// *value += 10; /// /// assert_eq!(map["poneyland"], 22); /// ``` @@ -2522,13 +5301,12 @@ impl<'a, K, V, S> OccupiedEntry<'a, K, V, S> { /// assert_eq!(map["poneyland"], 15); /// ``` #[cfg_attr(feature = "inline-more", inline)] - pub fn insert(&mut self, mut value: V) -> V { - let old_value = self.get_mut(); - mem::swap(&mut value, old_value); - value + pub fn insert(&mut self, value: V) -> V { + mem::replace(self.get_mut(), value) } /// Takes the value out of the entry, and returns it. + /// Keeps the allocated memory for reuse. /// /// # Examples /// @@ -2537,13 +5315,19 @@ impl<'a, K, V, S> OccupiedEntry<'a, K, V, S> { /// use hashbrown::hash_map::Entry; /// /// let mut map: HashMap<&str, u32> = HashMap::new(); + /// // The map is empty + /// assert!(map.is_empty() && map.capacity() == 0); + /// /// map.entry("poneyland").or_insert(12); + /// let capacity_before_remove = map.capacity(); /// /// if let Entry::Occupied(o) = map.entry("poneyland") { /// assert_eq!(o.remove(), 12); /// } /// /// assert_eq!(map.contains_key("poneyland"), false); + /// // Now map hold none elements but capacity is equal to the old one + /// assert!(map.len() == 0 && map.capacity() == capacity_before_remove); /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn remove(self) -> V { @@ -2553,23 +5337,34 @@ impl<'a, K, V, S> OccupiedEntry<'a, K, V, S> { /// Replaces the entry, returning the old key and value. The new key in the hash map will be /// the key used to create this entry. /// - /// # Examples - /// - /// ``` - /// use hashbrown::hash_map::{Entry, HashMap}; - /// use std::rc::Rc; - /// - /// let mut map: HashMap, u32> = HashMap::new(); - /// map.insert(Rc::new("Stringthing".to_string()), 15); + /// # Panics /// - /// let my_key = Rc::new("Stringthing".to_string()); + /// Will panic if this OccupiedEntry was created through [`Entry::insert`]. /// - /// if let Entry::Occupied(entry) = map.entry(my_key) { - /// // Also replace the key with a handle to our other key. - /// let (old_key, old_value): (Rc, u32) = entry.replace_entry(16); - /// } + /// # Examples /// /// ``` + /// use hashbrown::hash_map::{Entry, HashMap}; + /// use std::rc::Rc; + /// + /// let mut map: HashMap, u32> = HashMap::new(); + /// let key_one = Rc::new("Stringthing".to_string()); + /// let key_two = Rc::new("Stringthing".to_string()); + /// + /// map.insert(key_one.clone(), 15); + /// assert!(Rc::strong_count(&key_one) == 2 && Rc::strong_count(&key_two) == 1); + /// + /// match map.entry(key_two.clone()) { + /// Entry::Occupied(entry) => { + /// let (old_key, old_value): (Rc, u32) = entry.replace_entry(16); + /// assert!(Rc::ptr_eq(&key_one, &old_key) && old_value == 15); + /// } + /// Entry::Vacant(_) => panic!(), + /// } + /// + /// assert!(Rc::strong_count(&key_one) == 1 && Rc::strong_count(&key_two) == 2); + /// assert_eq!(map[&"Stringthing".to_owned()], 16); + /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn replace_entry(self, value: V) -> (K, V) { let entry = unsafe { self.elem.as_mut() }; @@ -2582,24 +5377,44 @@ impl<'a, K, V, S> OccupiedEntry<'a, K, V, S> { /// Replaces the key in the hash map with the key used to create this entry. /// + /// # Panics + /// + /// Will panic if this OccupiedEntry was created through [`Entry::insert`]. + /// /// # Examples /// /// ``` /// use hashbrown::hash_map::{Entry, HashMap}; /// use std::rc::Rc; /// - /// let mut map: HashMap, u32> = HashMap::new(); - /// let mut known_strings: Vec> = Vec::new(); + /// let mut map: HashMap, usize> = HashMap::with_capacity(6); + /// let mut keys_one: Vec> = Vec::with_capacity(6); + /// let mut keys_two: Vec> = Vec::with_capacity(6); /// - /// // Initialise known strings, run program, etc. + /// for (value, key) in ["a", "b", "c", "d", "e", "f"].into_iter().enumerate() { + /// let rc_key = Rc::new(key.to_owned()); + /// keys_one.push(rc_key.clone()); + /// map.insert(rc_key.clone(), value); + /// keys_two.push(Rc::new(key.to_owned())); + /// } /// - /// reclaim_memory(&mut map, &known_strings); + /// assert!( + /// keys_one.iter().all(|key| Rc::strong_count(key) == 2) + /// && keys_two.iter().all(|key| Rc::strong_count(key) == 1) + /// ); /// - /// fn reclaim_memory(map: &mut HashMap, u32>, known_strings: &[Rc] ) { - /// for s in known_strings { - /// if let Entry::Occupied(entry) = map.entry(s.clone()) { - /// // Replaces the entry's key with our version of it in `known_strings`. - /// entry.replace_key(); + /// reclaim_memory(&mut map, &keys_two); + /// + /// assert!( + /// keys_one.iter().all(|key| Rc::strong_count(key) == 1) + /// && keys_two.iter().all(|key| Rc::strong_count(key) == 2) + /// ); + /// + /// fn reclaim_memory(map: &mut HashMap, usize>, keys: &[Rc]) { + /// for key in keys { + /// if let Entry::Occupied(entry) = map.entry(key.clone()) { + /// // Replaces the entry's key with our version of it in `keys`. + /// entry.replace_key(); /// } /// } /// } @@ -2609,9 +5424,88 @@ impl<'a, K, V, S> OccupiedEntry<'a, K, V, S> { let entry = unsafe { self.elem.as_mut() }; mem::replace(&mut entry.0, self.key.unwrap()) } + + /// Provides shared access to the key and owned access to the value of + /// the entry and allows to replace or remove it based on the + /// value of the returned option. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// use hashbrown::hash_map::Entry; + /// + /// let mut map: HashMap<&str, u32> = HashMap::new(); + /// map.insert("poneyland", 42); + /// + /// let entry = match map.entry("poneyland") { + /// Entry::Occupied(e) => { + /// e.replace_entry_with(|k, v| { + /// assert_eq!(k, &"poneyland"); + /// assert_eq!(v, 42); + /// Some(v + 1) + /// }) + /// } + /// Entry::Vacant(_) => panic!(), + /// }; + /// + /// match entry { + /// Entry::Occupied(e) => { + /// assert_eq!(e.key(), &"poneyland"); + /// assert_eq!(e.get(), &43); + /// } + /// Entry::Vacant(_) => panic!(), + /// } + /// + /// assert_eq!(map["poneyland"], 43); + /// + /// let entry = match map.entry("poneyland") { + /// Entry::Occupied(e) => e.replace_entry_with(|_k, _v| None), + /// Entry::Vacant(_) => panic!(), + /// }; + /// + /// match entry { + /// Entry::Vacant(e) => { + /// assert_eq!(e.key(), &"poneyland"); + /// } + /// Entry::Occupied(_) => panic!(), + /// } + /// + /// assert!(!map.contains_key("poneyland")); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn replace_entry_with(self, f: F) -> Entry<'a, K, V, S, A> + where + F: FnOnce(&K, V) -> Option, + { + unsafe { + let mut spare_key = None; + + self.table + .table + .replace_bucket_with(self.elem.clone(), |(key, value)| { + if let Some(new_value) = f(&key, value) { + Some((key, new_value)) + } else { + spare_key = Some(key); + None + } + }); + + if let Some(key) = spare_key { + Entry::Vacant(VacantEntry { + hash: self.hash, + key, + table: self.table, + }) + } else { + Entry::Occupied(self) + } + } + } } -impl<'a, K, V, S> VacantEntry<'a, K, V, S> { +impl<'a, K, V, S, A: Allocator + Clone> VacantEntry<'a, K, V, S, A> { /// Gets a reference to the key that would be used when inserting a value /// through the `VacantEntry`. /// @@ -2633,13 +5527,13 @@ impl<'a, K, V, S> VacantEntry<'a, K, V, S> { /// # Examples /// /// ``` - /// use hashbrown::HashMap; - /// use hashbrown::hash_map::Entry; + /// use hashbrown::hash_map::{Entry, HashMap}; /// /// let mut map: HashMap<&str, u32> = HashMap::new(); /// - /// if let Entry::Vacant(v) = map.entry("poneyland") { - /// v.into_key(); + /// match map.entry("poneyland") { + /// Entry::Occupied(_) => panic!(), + /// Entry::Vacant(v) => assert_eq!(v.into_key(), "poneyland"), /// } /// ``` #[cfg_attr(feature = "inline-more", inline)] @@ -2669,24 +5563,28 @@ impl<'a, K, V, S> VacantEntry<'a, K, V, S> { K: Hash, S: BuildHasher, { - let hash_builder = &self.table.hash_builder; - let bucket = self.table.table.insert(self.hash, (self.key, value), |x| { - make_hash(hash_builder, &x.0) - }); - unsafe { &mut bucket.as_mut().1 } + let table = &mut self.table.table; + let entry = table.insert_entry( + self.hash, + (self.key, value), + make_hasher::(&self.table.hash_builder), + ); + &mut entry.1 } #[cfg_attr(feature = "inline-more", inline)] - fn insert_entry(self, value: V) -> OccupiedEntry<'a, K, V, S> + pub(crate) fn insert_entry(self, value: V) -> OccupiedEntry<'a, K, V, S, A> where K: Hash, S: BuildHasher, { - let hash_builder = &self.table.hash_builder; - let elem = self.table.table.insert(self.hash, (self.key, value), |x| { - make_hash(hash_builder, &x.0) - }); + let elem = self.table.table.insert( + self.hash, + (self.key, value), + make_hasher::(&self.table.hash_builder), + ); OccupiedEntry { + hash: self.hash, key: None, elem, table: self.table, @@ -2694,858 +5592,2391 @@ impl<'a, K, V, S> VacantEntry<'a, K, V, S> { } } -impl FromIterator<(K, V)> for HashMap -where - K: Eq + Hash, - S: BuildHasher + Default, -{ +impl<'a, 'b, K, Q: ?Sized, V, S, A: Allocator + Clone> EntryRef<'a, 'b, K, Q, V, S, A> { + /// Sets the value of the entry, and returns an OccupiedEntryRef. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let mut map: HashMap = HashMap::new(); + /// let entry = map.entry_ref("horseyland").insert(37); + /// + /// assert_eq!(entry.key(), "horseyland"); + /// ``` #[cfg_attr(feature = "inline-more", inline)] - fn from_iter>(iter: T) -> Self { - let iter = iter.into_iter(); - let mut map = Self::with_capacity_and_hasher(iter.size_hint().0, S::default()); - iter.for_each(|(k, v)| { - map.insert(k, v); - }); - map + pub fn insert(self, value: V) -> OccupiedEntryRef<'a, 'b, K, Q, V, S, A> + where + K: Hash + From<&'b Q>, + S: BuildHasher, + { + match self { + EntryRef::Occupied(mut entry) => { + entry.insert(value); + entry + } + EntryRef::Vacant(entry) => entry.insert_entry(value), + } } -} -impl Extend<(K, V)> for HashMap -where - K: Eq + Hash, - S: BuildHasher, -{ + /// Ensures a value is in the entry by inserting the default if empty, and returns + /// a mutable reference to the value in the entry. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let mut map: HashMap = HashMap::new(); + /// + /// // nonexistent key + /// map.entry_ref("poneyland").or_insert(3); + /// assert_eq!(map["poneyland"], 3); + /// + /// // existing key + /// *map.entry_ref("poneyland").or_insert(10) *= 2; + /// assert_eq!(map["poneyland"], 6); + /// ``` #[cfg_attr(feature = "inline-more", inline)] - fn extend>(&mut self, iter: T) { - // Keys may be already present or show multiple times in the iterator. - // Reserve the entire hint lower bound if the map is empty. - // Otherwise reserve half the hint (rounded up), so the map - // will only resize twice in the worst case. - let iter = iter.into_iter(); - let reserve = if self.is_empty() { - iter.size_hint().0 - } else { - (iter.size_hint().0 + 1) / 2 - }; - self.reserve(reserve); - iter.for_each(move |(k, v)| { - self.insert(k, v); - }); + pub fn or_insert(self, default: V) -> &'a mut V + where + K: Hash + From<&'b Q>, + S: BuildHasher, + { + match self { + EntryRef::Occupied(entry) => entry.into_mut(), + EntryRef::Vacant(entry) => entry.insert(default), + } } -} -impl<'a, K, V, S> Extend<(&'a K, &'a V)> for HashMap -where - K: Eq + Hash + Copy, - V: Copy, - S: BuildHasher, -{ + /// Ensures a value is in the entry by inserting the result of the default function if empty, + /// and returns a mutable reference to the value in the entry. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let mut map: HashMap = HashMap::new(); + /// + /// // nonexistent key + /// map.entry_ref("poneyland").or_insert_with(|| 3); + /// assert_eq!(map["poneyland"], 3); + /// + /// // existing key + /// *map.entry_ref("poneyland").or_insert_with(|| 10) *= 2; + /// assert_eq!(map["poneyland"], 6); + /// ``` #[cfg_attr(feature = "inline-more", inline)] - fn extend>(&mut self, iter: T) { - self.extend(iter.into_iter().map(|(&key, &value)| (key, value))); + pub fn or_insert_with V>(self, default: F) -> &'a mut V + where + K: Hash + From<&'b Q>, + S: BuildHasher, + { + match self { + EntryRef::Occupied(entry) => entry.into_mut(), + EntryRef::Vacant(entry) => entry.insert(default()), + } } -} -#[allow(dead_code)] -fn assert_covariance() { - fn map_key<'new>(v: HashMap<&'static str, u8>) -> HashMap<&'new str, u8> { - v - } - fn map_val<'new>(v: HashMap) -> HashMap { - v - } - fn iter_key<'a, 'new>(v: Iter<'a, &'static str, u8>) -> Iter<'a, &'new str, u8> { - v - } - fn iter_val<'a, 'new>(v: Iter<'a, u8, &'static str>) -> Iter<'a, u8, &'new str> { - v - } - fn into_iter_key<'new>(v: IntoIter<&'static str, u8>) -> IntoIter<&'new str, u8> { - v - } - fn into_iter_val<'new>(v: IntoIter) -> IntoIter { - v - } - fn keys_key<'a, 'new>(v: Keys<'a, &'static str, u8>) -> Keys<'a, &'new str, u8> { - v - } - fn keys_val<'a, 'new>(v: Keys<'a, u8, &'static str>) -> Keys<'a, u8, &'new str> { - v + /// Ensures a value is in the entry by inserting, if empty, the result of the default function. + /// This method allows for generating key-derived values for insertion by providing the default + /// function a reference to the key that was moved during the `.entry_ref(key)` method call. + /// + /// The reference to the moved key is provided so that cloning or copying the key is + /// unnecessary, unlike with `.or_insert_with(|| ... )`. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let mut map: HashMap = HashMap::new(); + /// + /// // nonexistent key + /// map.entry_ref("poneyland").or_insert_with_key(|key| key.chars().count()); + /// assert_eq!(map["poneyland"], 9); + /// + /// // existing key + /// *map.entry_ref("poneyland").or_insert_with_key(|key| key.chars().count() * 10) *= 2; + /// assert_eq!(map["poneyland"], 18); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn or_insert_with_key V>(self, default: F) -> &'a mut V + where + K: Hash + Borrow + From<&'b Q>, + S: BuildHasher, + { + match self { + EntryRef::Occupied(entry) => entry.into_mut(), + EntryRef::Vacant(entry) => { + let value = default(entry.key.as_ref()); + entry.insert(value) + } + } } - fn values_key<'a, 'new>(v: Values<'a, &'static str, u8>) -> Values<'a, &'new str, u8> { - v + + /// Returns a reference to this entry's key. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let mut map: HashMap = HashMap::new(); + /// map.entry_ref("poneyland").or_insert(3); + /// // existing key + /// assert_eq!(map.entry_ref("poneyland").key(), "poneyland"); + /// // nonexistent key + /// assert_eq!(map.entry_ref("horseland").key(), "horseland"); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn key(&self) -> &Q + where + K: Borrow, + { + match *self { + EntryRef::Occupied(ref entry) => entry.key(), + EntryRef::Vacant(ref entry) => entry.key(), + } } - fn values_val<'a, 'new>(v: Values<'a, u8, &'static str>) -> Values<'a, u8, &'new str> { - v + + /// Provides in-place mutable access to an occupied entry before any + /// potential inserts into the map. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let mut map: HashMap = HashMap::new(); + /// + /// map.entry_ref("poneyland") + /// .and_modify(|e| { *e += 1 }) + /// .or_insert(42); + /// assert_eq!(map["poneyland"], 42); + /// + /// map.entry_ref("poneyland") + /// .and_modify(|e| { *e += 1 }) + /// .or_insert(42); + /// assert_eq!(map["poneyland"], 43); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn and_modify(self, f: F) -> Self + where + F: FnOnce(&mut V), + { + match self { + EntryRef::Occupied(mut entry) => { + f(entry.get_mut()); + EntryRef::Occupied(entry) + } + EntryRef::Vacant(entry) => EntryRef::Vacant(entry), + } } - fn drain<'new>( - d: Drain<'static, &'static str, &'static str>, - ) -> Drain<'new, &'new str, &'new str> { - d + + /// Provides shared access to the key and owned access to the value of + /// an occupied entry and allows to replace or remove it based on the + /// value of the returned option. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// use hashbrown::hash_map::EntryRef; + /// + /// let mut map: HashMap = HashMap::new(); + /// + /// let entry = map + /// .entry_ref("poneyland") + /// .and_replace_entry_with(|_k, _v| panic!()); + /// + /// match entry { + /// EntryRef::Vacant(e) => { + /// assert_eq!(e.key(), "poneyland"); + /// } + /// EntryRef::Occupied(_) => panic!(), + /// } + /// + /// map.insert("poneyland".to_string(), 42); + /// + /// let entry = map + /// .entry_ref("poneyland") + /// .and_replace_entry_with(|k, v| { + /// assert_eq!(k, "poneyland"); + /// assert_eq!(v, 42); + /// Some(v + 1) + /// }); + /// + /// match entry { + /// EntryRef::Occupied(e) => { + /// assert_eq!(e.key(), "poneyland"); + /// assert_eq!(e.get(), &43); + /// } + /// EntryRef::Vacant(_) => panic!(), + /// } + /// + /// assert_eq!(map["poneyland"], 43); + /// + /// let entry = map + /// .entry_ref("poneyland") + /// .and_replace_entry_with(|_k, _v| None); + /// + /// match entry { + /// EntryRef::Vacant(e) => assert_eq!(e.key(), "poneyland"), + /// EntryRef::Occupied(_) => panic!(), + /// } + /// + /// assert!(!map.contains_key("poneyland")); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn and_replace_entry_with(self, f: F) -> Self + where + F: FnOnce(&Q, V) -> Option, + K: Borrow, + { + match self { + EntryRef::Occupied(entry) => entry.replace_entry_with(f), + EntryRef::Vacant(_) => self, + } } } -#[cfg(test)] -mod test_map { - use super::DefaultHashBuilder; +impl<'a, 'b, K, Q: ?Sized, V: Default, S, A: Allocator + Clone> EntryRef<'a, 'b, K, Q, V, S, A> { + /// Ensures a value is in the entry by inserting the default value if empty, + /// and returns a mutable reference to the value in the entry. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let mut map: HashMap> = HashMap::new(); + /// + /// // nonexistent key + /// map.entry_ref("poneyland").or_default(); + /// assert_eq!(map["poneyland"], None); + /// + /// map.insert("horseland".to_string(), Some(3)); + /// + /// // existing key + /// assert_eq!(map.entry_ref("horseland").or_default(), &mut Some(3)); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn or_default(self) -> &'a mut V + where + K: Hash + From<&'b Q>, + S: BuildHasher, + { + match self { + EntryRef::Occupied(entry) => entry.into_mut(), + EntryRef::Vacant(entry) => entry.insert(Default::default()), + } + } +} + +impl<'a, 'b, K, Q: ?Sized, V, S, A: Allocator + Clone> OccupiedEntryRef<'a, 'b, K, Q, V, S, A> { + /// Gets a reference to the key in the entry. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{EntryRef, HashMap}; + /// + /// let mut map: HashMap = HashMap::new(); + /// map.entry_ref("poneyland").or_insert(12); + /// + /// match map.entry_ref("poneyland") { + /// EntryRef::Vacant(_) => panic!(), + /// EntryRef::Occupied(entry) => assert_eq!(entry.key(), "poneyland"), + /// } + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn key(&self) -> &Q + where + K: Borrow, + { + unsafe { &self.elem.as_ref().0 }.borrow() + } + + /// Take the ownership of the key and value from the map. + /// Keeps the allocated memory for reuse. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// use hashbrown::hash_map::EntryRef; + /// + /// let mut map: HashMap = HashMap::new(); + /// // The map is empty + /// assert!(map.is_empty() && map.capacity() == 0); + /// + /// map.entry_ref("poneyland").or_insert(12); + /// let capacity_before_remove = map.capacity(); + /// + /// if let EntryRef::Occupied(o) = map.entry_ref("poneyland") { + /// // We delete the entry from the map. + /// assert_eq!(o.remove_entry(), ("poneyland".to_owned(), 12)); + /// } + /// + /// assert_eq!(map.contains_key("poneyland"), false); + /// // Now map hold none elements but capacity is equal to the old one + /// assert!(map.len() == 0 && map.capacity() == capacity_before_remove); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn remove_entry(self) -> (K, V) { + unsafe { self.table.table.remove(self.elem) } + } + + /// Gets a reference to the value in the entry. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// use hashbrown::hash_map::EntryRef; + /// + /// let mut map: HashMap = HashMap::new(); + /// map.entry_ref("poneyland").or_insert(12); + /// + /// match map.entry_ref("poneyland") { + /// EntryRef::Vacant(_) => panic!(), + /// EntryRef::Occupied(entry) => assert_eq!(entry.get(), &12), + /// } + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn get(&self) -> &V { + unsafe { &self.elem.as_ref().1 } + } + + /// Gets a mutable reference to the value in the entry. + /// + /// If you need a reference to the `OccupiedEntryRef` which may outlive the + /// destruction of the `EntryRef` value, see [`into_mut`]. + /// + /// [`into_mut`]: #method.into_mut + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// use hashbrown::hash_map::EntryRef; + /// + /// let mut map: HashMap = HashMap::new(); + /// map.entry_ref("poneyland").or_insert(12); + /// + /// assert_eq!(map["poneyland"], 12); + /// if let EntryRef::Occupied(mut o) = map.entry_ref("poneyland") { + /// *o.get_mut() += 10; + /// assert_eq!(*o.get(), 22); + /// + /// // We can use the same Entry multiple times. + /// *o.get_mut() += 2; + /// } + /// + /// assert_eq!(map["poneyland"], 24); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn get_mut(&mut self) -> &mut V { + unsafe { &mut self.elem.as_mut().1 } + } + + /// Converts the OccupiedEntryRef into a mutable reference to the value in the entry + /// with a lifetime bound to the map itself. + /// + /// If you need multiple references to the `OccupiedEntryRef`, see [`get_mut`]. + /// + /// [`get_mut`]: #method.get_mut + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{EntryRef, HashMap}; + /// + /// let mut map: HashMap = HashMap::new(); + /// map.entry_ref("poneyland").or_insert(12); + /// + /// let value: &mut u32; + /// match map.entry_ref("poneyland") { + /// EntryRef::Occupied(entry) => value = entry.into_mut(), + /// EntryRef::Vacant(_) => panic!(), + /// } + /// *value += 10; + /// + /// assert_eq!(map["poneyland"], 22); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn into_mut(self) -> &'a mut V { + unsafe { &mut self.elem.as_mut().1 } + } + + /// Sets the value of the entry, and returns the entry's old value. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// use hashbrown::hash_map::EntryRef; + /// + /// let mut map: HashMap = HashMap::new(); + /// map.entry_ref("poneyland").or_insert(12); + /// + /// if let EntryRef::Occupied(mut o) = map.entry_ref("poneyland") { + /// assert_eq!(o.insert(15), 12); + /// } + /// + /// assert_eq!(map["poneyland"], 15); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn insert(&mut self, value: V) -> V { + mem::replace(self.get_mut(), value) + } + + /// Takes the value out of the entry, and returns it. + /// Keeps the allocated memory for reuse. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// use hashbrown::hash_map::EntryRef; + /// + /// let mut map: HashMap = HashMap::new(); + /// // The map is empty + /// assert!(map.is_empty() && map.capacity() == 0); + /// + /// map.entry_ref("poneyland").or_insert(12); + /// let capacity_before_remove = map.capacity(); + /// + /// if let EntryRef::Occupied(o) = map.entry_ref("poneyland") { + /// assert_eq!(o.remove(), 12); + /// } + /// + /// assert_eq!(map.contains_key("poneyland"), false); + /// // Now map hold none elements but capacity is equal to the old one + /// assert!(map.len() == 0 && map.capacity() == capacity_before_remove); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn remove(self) -> V { + self.remove_entry().1 + } + + /// Replaces the entry, returning the old key and value. The new key in the hash map will be + /// the key used to create this entry. + /// + /// # Panics + /// + /// Will panic if this OccupiedEntry was created through [`EntryRef::insert`]. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{EntryRef, HashMap}; + /// use std::rc::Rc; + /// + /// let mut map: HashMap, u32> = HashMap::new(); + /// let key: Rc = Rc::from("Stringthing"); + /// + /// map.insert(key.clone(), 15); + /// assert_eq!(Rc::strong_count(&key), 2); + /// + /// match map.entry_ref("Stringthing") { + /// EntryRef::Occupied(entry) => { + /// let (old_key, old_value): (Rc, u32) = entry.replace_entry(16); + /// assert!(Rc::ptr_eq(&key, &old_key) && old_value == 15); + /// } + /// EntryRef::Vacant(_) => panic!(), + /// } + /// + /// assert_eq!(Rc::strong_count(&key), 1); + /// assert_eq!(map["Stringthing"], 16); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn replace_entry(self, value: V) -> (K, V) + where + K: From<&'b Q>, + { + let entry = unsafe { self.elem.as_mut() }; + + let old_key = mem::replace(&mut entry.0, self.key.unwrap().into_owned()); + let old_value = mem::replace(&mut entry.1, value); + + (old_key, old_value) + } + + /// Replaces the key in the hash map with the key used to create this entry. + /// + /// # Panics + /// + /// Will panic if this OccupiedEntry was created through [`Entry::insert`]. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{EntryRef, HashMap}; + /// use std::rc::Rc; + /// + /// let mut map: HashMap, usize> = HashMap::with_capacity(6); + /// let mut keys: Vec> = Vec::with_capacity(6); + /// + /// for (value, key) in ["a", "b", "c", "d", "e", "f"].into_iter().enumerate() { + /// let rc_key: Rc = Rc::from(key); + /// keys.push(rc_key.clone()); + /// map.insert(rc_key.clone(), value); + /// } + /// + /// assert!(keys.iter().all(|key| Rc::strong_count(key) == 2)); + /// + /// // It doesn't matter that we kind of use a vector with the same keys, + /// // because all keys will be newly created from the references + /// reclaim_memory(&mut map, &keys); + /// + /// assert!(keys.iter().all(|key| Rc::strong_count(key) == 1)); + /// + /// fn reclaim_memory(map: &mut HashMap, usize>, keys: &[Rc]) { + /// for key in keys { + /// if let EntryRef::Occupied(entry) = map.entry_ref(key.as_ref()) { + /// /// Replaces the entry's key with our version of it in `keys`. + /// entry.replace_key(); + /// } + /// } + /// } + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn replace_key(self) -> K + where + K: From<&'b Q>, + { + let entry = unsafe { self.elem.as_mut() }; + mem::replace(&mut entry.0, self.key.unwrap().into_owned()) + } + + /// Provides shared access to the key and owned access to the value of + /// the entry and allows to replace or remove it based on the + /// value of the returned option. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// use hashbrown::hash_map::EntryRef; + /// + /// let mut map: HashMap = HashMap::new(); + /// map.insert("poneyland".to_string(), 42); + /// + /// let entry = match map.entry_ref("poneyland") { + /// EntryRef::Occupied(e) => { + /// e.replace_entry_with(|k, v| { + /// assert_eq!(k, "poneyland"); + /// assert_eq!(v, 42); + /// Some(v + 1) + /// }) + /// } + /// EntryRef::Vacant(_) => panic!(), + /// }; + /// + /// match entry { + /// EntryRef::Occupied(e) => { + /// assert_eq!(e.key(), "poneyland"); + /// assert_eq!(e.get(), &43); + /// } + /// EntryRef::Vacant(_) => panic!(), + /// } + /// + /// assert_eq!(map["poneyland"], 43); + /// + /// let entry = match map.entry_ref("poneyland") { + /// EntryRef::Occupied(e) => e.replace_entry_with(|_k, _v| None), + /// EntryRef::Vacant(_) => panic!(), + /// }; + /// + /// match entry { + /// EntryRef::Vacant(e) => { + /// assert_eq!(e.key(), "poneyland"); + /// } + /// EntryRef::Occupied(_) => panic!(), + /// } + /// + /// assert!(!map.contains_key("poneyland")); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn replace_entry_with(self, f: F) -> EntryRef<'a, 'b, K, Q, V, S, A> + where + F: FnOnce(&Q, V) -> Option, + K: Borrow, + { + unsafe { + let mut spare_key = None; + + self.table + .table + .replace_bucket_with(self.elem.clone(), |(key, value)| { + if let Some(new_value) = f(key.borrow(), value) { + Some((key, new_value)) + } else { + spare_key = Some(KeyOrRef::Owned(key)); + None + } + }); + + if let Some(key) = spare_key { + EntryRef::Vacant(VacantEntryRef { + hash: self.hash, + key, + table: self.table, + }) + } else { + EntryRef::Occupied(self) + } + } + } +} + +impl<'a, 'b, K, Q: ?Sized, V, S, A: Allocator + Clone> VacantEntryRef<'a, 'b, K, Q, V, S, A> { + /// Gets a reference to the key that would be used when inserting a value + /// through the `VacantEntryRef`. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// + /// let mut map: HashMap = HashMap::new(); + /// let key: &str = "poneyland"; + /// assert_eq!(map.entry_ref(key).key(), "poneyland"); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn key(&self) -> &Q + where + K: Borrow, + { + self.key.as_ref() + } + + /// Take ownership of the key. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::{EntryRef, HashMap}; + /// + /// let mut map: HashMap = HashMap::new(); + /// let key: &str = "poneyland"; + /// + /// match map.entry_ref(key) { + /// EntryRef::Occupied(_) => panic!(), + /// EntryRef::Vacant(v) => assert_eq!(v.into_key(), "poneyland".to_owned()), + /// } + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn into_key(self) -> K + where + K: From<&'b Q>, + { + self.key.into_owned() + } + + /// Sets the value of the entry with the VacantEntryRef's key, + /// and returns a mutable reference to it. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashMap; + /// use hashbrown::hash_map::EntryRef; + /// + /// let mut map: HashMap = HashMap::new(); + /// let key: &str = "poneyland"; + /// + /// if let EntryRef::Vacant(o) = map.entry_ref(key) { + /// o.insert(37); + /// } + /// assert_eq!(map["poneyland"], 37); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn insert(self, value: V) -> &'a mut V + where + K: Hash + From<&'b Q>, + S: BuildHasher, + { + let table = &mut self.table.table; + let entry = table.insert_entry( + self.hash, + (self.key.into_owned(), value), + make_hasher::(&self.table.hash_builder), + ); + &mut entry.1 + } + + #[cfg_attr(feature = "inline-more", inline)] + fn insert_entry(self, value: V) -> OccupiedEntryRef<'a, 'b, K, Q, V, S, A> + where + K: Hash + From<&'b Q>, + S: BuildHasher, + { + let elem = self.table.table.insert( + self.hash, + (self.key.into_owned(), value), + make_hasher::(&self.table.hash_builder), + ); + OccupiedEntryRef { + hash: self.hash, + key: None, + elem, + table: self.table, + } + } +} + +impl FromIterator<(K, V)> for HashMap +where + K: Eq + Hash, + S: BuildHasher + Default, + A: Default + Allocator + Clone, +{ + #[cfg_attr(feature = "inline-more", inline)] + fn from_iter>(iter: T) -> Self { + let iter = iter.into_iter(); + let mut map = + Self::with_capacity_and_hasher_in(iter.size_hint().0, S::default(), A::default()); + iter.for_each(|(k, v)| { + map.insert(k, v); + }); + map + } +} + +/// Inserts all new key-values from the iterator and replaces values with existing +/// keys with new values returned from the iterator. +impl Extend<(K, V)> for HashMap +where + K: Eq + Hash, + S: BuildHasher, + A: Allocator + Clone, +{ + /// Inserts all new key-values from the iterator to existing `HashMap`. + /// Replace values with existing keys with new values returned from the iterator. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::HashMap; + /// + /// let mut map = HashMap::new(); + /// map.insert(1, 100); + /// + /// let some_iter = [(1, 1), (2, 2)].into_iter(); + /// map.extend(some_iter); + /// // Replace values with existing keys with new values returned from the iterator. + /// // So that the map.get(&1) doesn't return Some(&100). + /// assert_eq!(map.get(&1), Some(&1)); + /// + /// let some_vec: Vec<_> = vec![(3, 3), (4, 4)]; + /// map.extend(some_vec); + /// + /// let some_arr = [(5, 5), (6, 6)]; + /// map.extend(some_arr); + /// let old_map_len = map.len(); + /// + /// // You can also extend from another HashMap + /// let mut new_map = HashMap::new(); + /// new_map.extend(map); + /// assert_eq!(new_map.len(), old_map_len); + /// + /// let mut vec: Vec<_> = new_map.into_iter().collect(); + /// // The `IntoIter` iterator produces items in arbitrary order, so the + /// // items must be sorted to test them against a sorted array. + /// vec.sort_unstable(); + /// assert_eq!(vec, [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + fn extend>(&mut self, iter: T) { + // Keys may be already present or show multiple times in the iterator. + // Reserve the entire hint lower bound if the map is empty. + // Otherwise reserve half the hint (rounded up), so the map + // will only resize twice in the worst case. + let iter = iter.into_iter(); + let reserve = if self.is_empty() { + iter.size_hint().0 + } else { + (iter.size_hint().0 + 1) / 2 + }; + self.reserve(reserve); + iter.for_each(move |(k, v)| { + self.insert(k, v); + }); + } + + #[inline] + #[cfg(feature = "nightly")] + fn extend_one(&mut self, (k, v): (K, V)) { + self.insert(k, v); + } + + #[inline] + #[cfg(feature = "nightly")] + fn extend_reserve(&mut self, additional: usize) { + // Keys may be already present or show multiple times in the iterator. + // Reserve the entire hint lower bound if the map is empty. + // Otherwise reserve half the hint (rounded up), so the map + // will only resize twice in the worst case. + let reserve = if self.is_empty() { + additional + } else { + (additional + 1) / 2 + }; + self.reserve(reserve); + } +} + +/// Inserts all new key-values from the iterator and replaces values with existing +/// keys with new values returned from the iterator. +impl<'a, K, V, S, A> Extend<(&'a K, &'a V)> for HashMap +where + K: Eq + Hash + Copy, + V: Copy, + S: BuildHasher, + A: Allocator + Clone, +{ + /// Inserts all new key-values from the iterator to existing `HashMap`. + /// Replace values with existing keys with new values returned from the iterator. + /// The keys and values must implement [`Copy`] trait. + /// + /// [`Copy`]: https://doc.rust-lang.org/core/marker/trait.Copy.html + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::HashMap; + /// + /// let mut map = HashMap::new(); + /// map.insert(1, 100); + /// + /// let arr = [(1, 1), (2, 2)]; + /// let some_iter = arr.iter().map(|&(k, v)| (k, v)); + /// map.extend(some_iter); + /// // Replace values with existing keys with new values returned from the iterator. + /// // So that the map.get(&1) doesn't return Some(&100). + /// assert_eq!(map.get(&1), Some(&1)); + /// + /// let some_vec: Vec<_> = vec![(3, 3), (4, 4)]; + /// map.extend(some_vec.iter().map(|&(k, v)| (k, v))); + /// + /// let some_arr = [(5, 5), (6, 6)]; + /// map.extend(some_arr.iter().map(|&(k, v)| (k, v))); + /// + /// // You can also extend from another HashMap + /// let mut new_map = HashMap::new(); + /// new_map.extend(&map); + /// assert_eq!(new_map, map); + /// + /// let mut vec: Vec<_> = new_map.into_iter().collect(); + /// // The `IntoIter` iterator produces items in arbitrary order, so the + /// // items must be sorted to test them against a sorted array. + /// vec.sort_unstable(); + /// assert_eq!(vec, [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + fn extend>(&mut self, iter: T) { + self.extend(iter.into_iter().map(|(&key, &value)| (key, value))); + } + + #[inline] + #[cfg(feature = "nightly")] + fn extend_one(&mut self, (k, v): (&'a K, &'a V)) { + self.insert(*k, *v); + } + + #[inline] + #[cfg(feature = "nightly")] + fn extend_reserve(&mut self, additional: usize) { + Extend::<(K, V)>::extend_reserve(self, additional); + } +} + +/// Inserts all new key-values from the iterator and replaces values with existing +/// keys with new values returned from the iterator. +impl<'a, K, V, S, A> Extend<&'a (K, V)> for HashMap +where + K: Eq + Hash + Copy, + V: Copy, + S: BuildHasher, + A: Allocator + Clone, +{ + /// Inserts all new key-values from the iterator to existing `HashMap`. + /// Replace values with existing keys with new values returned from the iterator. + /// The keys and values must implement [`Copy`] trait. + /// + /// [`Copy`]: https://doc.rust-lang.org/core/marker/trait.Copy.html + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_map::HashMap; + /// + /// let mut map = HashMap::new(); + /// map.insert(1, 100); + /// + /// let arr = [(1, 1), (2, 2)]; + /// let some_iter = arr.iter(); + /// map.extend(some_iter); + /// // Replace values with existing keys with new values returned from the iterator. + /// // So that the map.get(&1) doesn't return Some(&100). + /// assert_eq!(map.get(&1), Some(&1)); + /// + /// let some_vec: Vec<_> = vec![(3, 3), (4, 4)]; + /// map.extend(&some_vec); + /// + /// let some_arr = [(5, 5), (6, 6)]; + /// map.extend(&some_arr); + /// + /// let mut vec: Vec<_> = map.into_iter().collect(); + /// // The `IntoIter` iterator produces items in arbitrary order, so the + /// // items must be sorted to test them against a sorted array. + /// vec.sort_unstable(); + /// assert_eq!(vec, [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + fn extend>(&mut self, iter: T) { + self.extend(iter.into_iter().map(|&(key, value)| (key, value))); + } + + #[inline] + #[cfg(feature = "nightly")] + fn extend_one(&mut self, &(k, v): &'a (K, V)) { + self.insert(k, v); + } + + #[inline] + #[cfg(feature = "nightly")] + fn extend_reserve(&mut self, additional: usize) { + Extend::<(K, V)>::extend_reserve(self, additional); + } +} + +#[allow(dead_code)] +fn assert_covariance() { + fn map_key<'new>(v: HashMap<&'static str, u8>) -> HashMap<&'new str, u8> { + v + } + fn map_val<'new>(v: HashMap) -> HashMap { + v + } + fn iter_key<'a, 'new>(v: Iter<'a, &'static str, u8>) -> Iter<'a, &'new str, u8> { + v + } + fn iter_val<'a, 'new>(v: Iter<'a, u8, &'static str>) -> Iter<'a, u8, &'new str> { + v + } + fn into_iter_key<'new, A: Allocator + Clone>( + v: IntoIter<&'static str, u8, A>, + ) -> IntoIter<&'new str, u8, A> { + v + } + fn into_iter_val<'new, A: Allocator + Clone>( + v: IntoIter, + ) -> IntoIter { + v + } + fn keys_key<'a, 'new>(v: Keys<'a, &'static str, u8>) -> Keys<'a, &'new str, u8> { + v + } + fn keys_val<'a, 'new>(v: Keys<'a, u8, &'static str>) -> Keys<'a, u8, &'new str> { + v + } + fn values_key<'a, 'new>(v: Values<'a, &'static str, u8>) -> Values<'a, &'new str, u8> { + v + } + fn values_val<'a, 'new>(v: Values<'a, u8, &'static str>) -> Values<'a, u8, &'new str> { + v + } + fn drain<'new>( + d: Drain<'static, &'static str, &'static str>, + ) -> Drain<'new, &'new str, &'new str> { + d + } +} + +#[cfg(test)] +mod test_map { + use super::DefaultHashBuilder; use super::Entry::{Occupied, Vacant}; + use super::EntryRef; use super::{HashMap, RawEntryMut}; - use crate::CollectionAllocErr::*; use rand::{rngs::SmallRng, Rng, SeedableRng}; + use std::borrow::ToOwned; use std::cell::RefCell; use std::usize; use std::vec::Vec; #[test] - fn test_zero_capacities() { - type HM = HashMap; + fn test_zero_capacities() { + type HM = HashMap; + + let m = HM::new(); + assert_eq!(m.capacity(), 0); + + let m = HM::default(); + assert_eq!(m.capacity(), 0); + + let m = HM::with_hasher(DefaultHashBuilder::default()); + assert_eq!(m.capacity(), 0); + + let m = HM::with_capacity(0); + assert_eq!(m.capacity(), 0); + + let m = HM::with_capacity_and_hasher(0, DefaultHashBuilder::default()); + assert_eq!(m.capacity(), 0); + + let mut m = HM::new(); + m.insert(1, 1); + m.insert(2, 2); + m.remove(&1); + m.remove(&2); + m.shrink_to_fit(); + assert_eq!(m.capacity(), 0); + + let mut m = HM::new(); + m.reserve(0); + assert_eq!(m.capacity(), 0); + } + + #[test] + fn test_create_capacity_zero() { + let mut m = HashMap::with_capacity(0); + + assert!(m.insert(1, 1).is_none()); + + assert!(m.contains_key(&1)); + assert!(!m.contains_key(&0)); + } + + #[test] + fn test_insert() { + let mut m = HashMap::new(); + assert_eq!(m.len(), 0); + assert!(m.insert(1, 2).is_none()); + assert_eq!(m.len(), 1); + assert!(m.insert(2, 4).is_none()); + assert_eq!(m.len(), 2); + assert_eq!(*m.get(&1).unwrap(), 2); + assert_eq!(*m.get(&2).unwrap(), 4); + } + + #[test] + fn test_clone() { + let mut m = HashMap::new(); + assert_eq!(m.len(), 0); + assert!(m.insert(1, 2).is_none()); + assert_eq!(m.len(), 1); + assert!(m.insert(2, 4).is_none()); + assert_eq!(m.len(), 2); + #[allow(clippy::redundant_clone)] + let m2 = m.clone(); + assert_eq!(*m2.get(&1).unwrap(), 2); + assert_eq!(*m2.get(&2).unwrap(), 4); + assert_eq!(m2.len(), 2); + } + + #[test] + fn test_clone_from() { + let mut m = HashMap::new(); + let mut m2 = HashMap::new(); + assert_eq!(m.len(), 0); + assert!(m.insert(1, 2).is_none()); + assert_eq!(m.len(), 1); + assert!(m.insert(2, 4).is_none()); + assert_eq!(m.len(), 2); + m2.clone_from(&m); + assert_eq!(*m2.get(&1).unwrap(), 2); + assert_eq!(*m2.get(&2).unwrap(), 4); + assert_eq!(m2.len(), 2); + } + + thread_local! { static DROP_VECTOR: RefCell> = RefCell::new(Vec::new()) } + + #[derive(Hash, PartialEq, Eq)] + struct Droppable { + k: usize, + } + + impl Droppable { + fn new(k: usize) -> Droppable { + DROP_VECTOR.with(|slot| { + slot.borrow_mut()[k] += 1; + }); + + Droppable { k } + } + } + + impl Drop for Droppable { + fn drop(&mut self) { + DROP_VECTOR.with(|slot| { + slot.borrow_mut()[self.k] -= 1; + }); + } + } + + impl Clone for Droppable { + fn clone(&self) -> Self { + Droppable::new(self.k) + } + } + + #[test] + fn test_drops() { + DROP_VECTOR.with(|slot| { + *slot.borrow_mut() = vec![0; 200]; + }); + + { + let mut m = HashMap::new(); + + DROP_VECTOR.with(|v| { + for i in 0..200 { + assert_eq!(v.borrow()[i], 0); + } + }); + + for i in 0..100 { + let d1 = Droppable::new(i); + let d2 = Droppable::new(i + 100); + m.insert(d1, d2); + } + + DROP_VECTOR.with(|v| { + for i in 0..200 { + assert_eq!(v.borrow()[i], 1); + } + }); + + for i in 0..50 { + let k = Droppable::new(i); + let v = m.remove(&k); + + assert!(v.is_some()); + + DROP_VECTOR.with(|v| { + assert_eq!(v.borrow()[i], 1); + assert_eq!(v.borrow()[i + 100], 1); + }); + } + + DROP_VECTOR.with(|v| { + for i in 0..50 { + assert_eq!(v.borrow()[i], 0); + assert_eq!(v.borrow()[i + 100], 0); + } + + for i in 50..100 { + assert_eq!(v.borrow()[i], 1); + assert_eq!(v.borrow()[i + 100], 1); + } + }); + } + + DROP_VECTOR.with(|v| { + for i in 0..200 { + assert_eq!(v.borrow()[i], 0); + } + }); + } + + #[test] + fn test_into_iter_drops() { + DROP_VECTOR.with(|v| { + *v.borrow_mut() = vec![0; 200]; + }); + + let hm = { + let mut hm = HashMap::new(); + + DROP_VECTOR.with(|v| { + for i in 0..200 { + assert_eq!(v.borrow()[i], 0); + } + }); + + for i in 0..100 { + let d1 = Droppable::new(i); + let d2 = Droppable::new(i + 100); + hm.insert(d1, d2); + } + + DROP_VECTOR.with(|v| { + for i in 0..200 { + assert_eq!(v.borrow()[i], 1); + } + }); + + hm + }; + + // By the way, ensure that cloning doesn't screw up the dropping. + drop(hm.clone()); + + { + let mut half = hm.into_iter().take(50); + + DROP_VECTOR.with(|v| { + for i in 0..200 { + assert_eq!(v.borrow()[i], 1); + } + }); + + #[allow(clippy::let_underscore_drop)] // kind-of a false positive + for _ in half.by_ref() {} + + DROP_VECTOR.with(|v| { + let nk = (0..100).filter(|&i| v.borrow()[i] == 1).count(); + + let nv = (0..100).filter(|&i| v.borrow()[i + 100] == 1).count(); + + assert_eq!(nk, 50); + assert_eq!(nv, 50); + }); + }; + + DROP_VECTOR.with(|v| { + for i in 0..200 { + assert_eq!(v.borrow()[i], 0); + } + }); + } + + #[test] + fn test_empty_remove() { + let mut m: HashMap = HashMap::new(); + assert_eq!(m.remove(&0), None); + } + + #[test] + fn test_empty_entry() { + let mut m: HashMap = HashMap::new(); + match m.entry(0) { + Occupied(_) => panic!(), + Vacant(_) => {} + } + assert!(*m.entry(0).or_insert(true)); + assert_eq!(m.len(), 1); + } + + #[test] + fn test_empty_entry_ref() { + let mut m: HashMap = HashMap::new(); + match m.entry_ref("poneyland") { + EntryRef::Occupied(_) => panic!(), + EntryRef::Vacant(_) => {} + } + assert!(*m.entry_ref("poneyland").or_insert(true)); + assert_eq!(m.len(), 1); + } + + #[test] + fn test_empty_iter() { + let mut m: HashMap = HashMap::new(); + assert_eq!(m.drain().next(), None); + assert_eq!(m.keys().next(), None); + assert_eq!(m.values().next(), None); + assert_eq!(m.values_mut().next(), None); + assert_eq!(m.iter().next(), None); + assert_eq!(m.iter_mut().next(), None); + assert_eq!(m.len(), 0); + assert!(m.is_empty()); + assert_eq!(m.into_iter().next(), None); + } + + #[test] + #[cfg_attr(miri, ignore)] // FIXME: takes too long + fn test_lots_of_insertions() { + let mut m = HashMap::new(); - let m = HM::new(); - assert_eq!(m.capacity(), 0); + // Try this a few times to make sure we never screw up the hashmap's + // internal state. + for _ in 0..10 { + assert!(m.is_empty()); - let m = HM::default(); - assert_eq!(m.capacity(), 0); + for i in 1..1001 { + assert!(m.insert(i, i).is_none()); - let m = HM::with_hasher(DefaultHashBuilder::default()); - assert_eq!(m.capacity(), 0); + for j in 1..=i { + let r = m.get(&j); + assert_eq!(r, Some(&j)); + } - let m = HM::with_capacity(0); - assert_eq!(m.capacity(), 0); + for j in i + 1..1001 { + let r = m.get(&j); + assert_eq!(r, None); + } + } - let m = HM::with_capacity_and_hasher(0, DefaultHashBuilder::default()); - assert_eq!(m.capacity(), 0); + for i in 1001..2001 { + assert!(!m.contains_key(&i)); + } - let mut m = HM::new(); - m.insert(1, 1); - m.insert(2, 2); - m.remove(&1); - m.remove(&2); - m.shrink_to_fit(); - assert_eq!(m.capacity(), 0); + // remove forwards + for i in 1..1001 { + assert!(m.remove(&i).is_some()); - let mut m = HM::new(); - m.reserve(0); - assert_eq!(m.capacity(), 0); + for j in 1..=i { + assert!(!m.contains_key(&j)); + } + + for j in i + 1..1001 { + assert!(m.contains_key(&j)); + } + } + + for i in 1..1001 { + assert!(!m.contains_key(&i)); + } + + for i in 1..1001 { + assert!(m.insert(i, i).is_none()); + } + + // remove backwards + for i in (1..1001).rev() { + assert!(m.remove(&i).is_some()); + + for j in i..1001 { + assert!(!m.contains_key(&j)); + } + + for j in 1..i { + assert!(m.contains_key(&j)); + } + } + } + } + + #[test] + fn test_find_mut() { + let mut m = HashMap::new(); + assert!(m.insert(1, 12).is_none()); + assert!(m.insert(2, 8).is_none()); + assert!(m.insert(5, 14).is_none()); + let new = 100; + match m.get_mut(&5) { + None => panic!(), + Some(x) => *x = new, + } + assert_eq!(m.get(&5), Some(&new)); + } + + #[test] + fn test_insert_overwrite() { + let mut m = HashMap::new(); + assert!(m.insert(1, 2).is_none()); + assert_eq!(*m.get(&1).unwrap(), 2); + assert!(m.insert(1, 3).is_some()); + assert_eq!(*m.get(&1).unwrap(), 3); + } + + #[test] + fn test_insert_conflicts() { + let mut m = HashMap::with_capacity(4); + assert!(m.insert(1, 2).is_none()); + assert!(m.insert(5, 3).is_none()); + assert!(m.insert(9, 4).is_none()); + assert_eq!(*m.get(&9).unwrap(), 4); + assert_eq!(*m.get(&5).unwrap(), 3); + assert_eq!(*m.get(&1).unwrap(), 2); + } + + #[test] + fn test_conflict_remove() { + let mut m = HashMap::with_capacity(4); + assert!(m.insert(1, 2).is_none()); + assert_eq!(*m.get(&1).unwrap(), 2); + assert!(m.insert(5, 3).is_none()); + assert_eq!(*m.get(&1).unwrap(), 2); + assert_eq!(*m.get(&5).unwrap(), 3); + assert!(m.insert(9, 4).is_none()); + assert_eq!(*m.get(&1).unwrap(), 2); + assert_eq!(*m.get(&5).unwrap(), 3); + assert_eq!(*m.get(&9).unwrap(), 4); + assert!(m.remove(&1).is_some()); + assert_eq!(*m.get(&9).unwrap(), 4); + assert_eq!(*m.get(&5).unwrap(), 3); + } + + #[test] + fn test_insert_unique_unchecked() { + let mut map = HashMap::new(); + let (k1, v1) = map.insert_unique_unchecked(10, 11); + assert_eq!((&10, &mut 11), (k1, v1)); + let (k2, v2) = map.insert_unique_unchecked(20, 21); + assert_eq!((&20, &mut 21), (k2, v2)); + assert_eq!(Some(&11), map.get(&10)); + assert_eq!(Some(&21), map.get(&20)); + assert_eq!(None, map.get(&30)); + } + + #[test] + fn test_is_empty() { + let mut m = HashMap::with_capacity(4); + assert!(m.insert(1, 2).is_none()); + assert!(!m.is_empty()); + assert!(m.remove(&1).is_some()); + assert!(m.is_empty()); + } + + #[test] + fn test_remove() { + let mut m = HashMap::new(); + m.insert(1, 2); + assert_eq!(m.remove(&1), Some(2)); + assert_eq!(m.remove(&1), None); + } + + #[test] + fn test_remove_entry() { + let mut m = HashMap::new(); + m.insert(1, 2); + assert_eq!(m.remove_entry(&1), Some((1, 2))); + assert_eq!(m.remove(&1), None); + } + + #[test] + fn test_iterate() { + let mut m = HashMap::with_capacity(4); + for i in 0..32 { + assert!(m.insert(i, i * 2).is_none()); + } + assert_eq!(m.len(), 32); + + let mut observed: u32 = 0; + + for (k, v) in &m { + assert_eq!(*v, *k * 2); + observed |= 1 << *k; + } + assert_eq!(observed, 0xFFFF_FFFF); + } + + #[test] + fn test_keys() { + let vec = vec![(1, 'a'), (2, 'b'), (3, 'c')]; + let map: HashMap<_, _> = vec.into_iter().collect(); + let keys: Vec<_> = map.keys().copied().collect(); + assert_eq!(keys.len(), 3); + assert!(keys.contains(&1)); + assert!(keys.contains(&2)); + assert!(keys.contains(&3)); + } + + #[test] + fn test_values() { + let vec = vec![(1, 'a'), (2, 'b'), (3, 'c')]; + let map: HashMap<_, _> = vec.into_iter().collect(); + let values: Vec<_> = map.values().copied().collect(); + assert_eq!(values.len(), 3); + assert!(values.contains(&'a')); + assert!(values.contains(&'b')); + assert!(values.contains(&'c')); + } + + #[test] + fn test_values_mut() { + let vec = vec![(1, 1), (2, 2), (3, 3)]; + let mut map: HashMap<_, _> = vec.into_iter().collect(); + for value in map.values_mut() { + *value *= 2; + } + let values: Vec<_> = map.values().copied().collect(); + assert_eq!(values.len(), 3); + assert!(values.contains(&2)); + assert!(values.contains(&4)); + assert!(values.contains(&6)); + } + + #[test] + fn test_into_keys() { + let vec = vec![(1, 'a'), (2, 'b'), (3, 'c')]; + let map: HashMap<_, _> = vec.into_iter().collect(); + let keys: Vec<_> = map.into_keys().collect(); + + assert_eq!(keys.len(), 3); + assert!(keys.contains(&1)); + assert!(keys.contains(&2)); + assert!(keys.contains(&3)); + } + + #[test] + fn test_into_values() { + let vec = vec![(1, 'a'), (2, 'b'), (3, 'c')]; + let map: HashMap<_, _> = vec.into_iter().collect(); + let values: Vec<_> = map.into_values().collect(); + + assert_eq!(values.len(), 3); + assert!(values.contains(&'a')); + assert!(values.contains(&'b')); + assert!(values.contains(&'c')); + } + + #[test] + fn test_find() { + let mut m = HashMap::new(); + assert!(m.get(&1).is_none()); + m.insert(1, 2); + match m.get(&1) { + None => panic!(), + Some(v) => assert_eq!(*v, 2), + } + } + + #[test] + fn test_eq() { + let mut m1 = HashMap::new(); + m1.insert(1, 2); + m1.insert(2, 3); + m1.insert(3, 4); + + let mut m2 = HashMap::new(); + m2.insert(1, 2); + m2.insert(2, 3); + + assert!(m1 != m2); + + m2.insert(3, 4); + + assert_eq!(m1, m2); } #[test] - fn test_create_capacity_zero() { - let mut m = HashMap::with_capacity(0); + fn test_show() { + let mut map = HashMap::new(); + let empty: HashMap = HashMap::new(); - assert!(m.insert(1, 1).is_none()); + map.insert(1, 2); + map.insert(3, 4); - assert!(m.contains_key(&1)); - assert!(!m.contains_key(&0)); - } + let map_str = format!("{:?}", map); - #[test] - fn test_insert() { - let mut m = HashMap::new(); - assert_eq!(m.len(), 0); - assert!(m.insert(1, 2).is_none()); - assert_eq!(m.len(), 1); - assert!(m.insert(2, 4).is_none()); - assert_eq!(m.len(), 2); - assert_eq!(*m.get(&1).unwrap(), 2); - assert_eq!(*m.get(&2).unwrap(), 4); + assert!(map_str == "{1: 2, 3: 4}" || map_str == "{3: 4, 1: 2}"); + assert_eq!(format!("{:?}", empty), "{}"); } #[test] - fn test_clone() { + fn test_expand() { let mut m = HashMap::new(); + assert_eq!(m.len(), 0); - assert!(m.insert(1, 2).is_none()); - assert_eq!(m.len(), 1); - assert!(m.insert(2, 4).is_none()); - assert_eq!(m.len(), 2); - let m2 = m.clone(); - assert_eq!(*m2.get(&1).unwrap(), 2); - assert_eq!(*m2.get(&2).unwrap(), 4); - assert_eq!(m2.len(), 2); + assert!(m.is_empty()); + + let mut i = 0; + let old_raw_cap = m.raw_capacity(); + while old_raw_cap == m.raw_capacity() { + m.insert(i, i); + i += 1; + } + + assert_eq!(m.len(), i); + assert!(!m.is_empty()); } #[test] - fn test_clone_from() { + fn test_behavior_resize_policy() { let mut m = HashMap::new(); - let mut m2 = HashMap::new(); - assert_eq!(m.len(), 0); - assert!(m.insert(1, 2).is_none()); - assert_eq!(m.len(), 1); - assert!(m.insert(2, 4).is_none()); - assert_eq!(m.len(), 2); - m2.clone_from(&m); - assert_eq!(*m2.get(&1).unwrap(), 2); - assert_eq!(*m2.get(&2).unwrap(), 4); - assert_eq!(m2.len(), 2); - } - thread_local! { static DROP_VECTOR: RefCell> = RefCell::new(Vec::new()) } + assert_eq!(m.len(), 0); + assert_eq!(m.raw_capacity(), 1); + assert!(m.is_empty()); - #[derive(Hash, PartialEq, Eq)] - struct Droppable { - k: usize, - } + m.insert(0, 0); + m.remove(&0); + assert!(m.is_empty()); + let initial_raw_cap = m.raw_capacity(); + m.reserve(initial_raw_cap); + let raw_cap = m.raw_capacity(); - impl Droppable { - fn new(k: usize) -> Droppable { - DROP_VECTOR.with(|slot| { - slot.borrow_mut()[k] += 1; - }); + assert_eq!(raw_cap, initial_raw_cap * 2); - Droppable { k } + let mut i = 0; + for _ in 0..raw_cap * 3 / 4 { + m.insert(i, i); + i += 1; } - } + // three quarters full - impl Drop for Droppable { - fn drop(&mut self) { - DROP_VECTOR.with(|slot| { - slot.borrow_mut()[self.k] -= 1; - }); + assert_eq!(m.len(), i); + assert_eq!(m.raw_capacity(), raw_cap); + + for _ in 0..raw_cap / 4 { + m.insert(i, i); + i += 1; } - } + // half full - impl Clone for Droppable { - fn clone(&self) -> Self { - Droppable::new(self.k) + let new_raw_cap = m.raw_capacity(); + assert_eq!(new_raw_cap, raw_cap * 2); + + for _ in 0..raw_cap / 2 - 1 { + i -= 1; + m.remove(&i); + assert_eq!(m.raw_capacity(), new_raw_cap); + } + // A little more than one quarter full. + m.shrink_to_fit(); + assert_eq!(m.raw_capacity(), raw_cap); + // again, a little more than half full + for _ in 0..raw_cap / 2 { + i -= 1; + m.remove(&i); } + m.shrink_to_fit(); + + assert_eq!(m.len(), i); + assert!(!m.is_empty()); + assert_eq!(m.raw_capacity(), initial_raw_cap); } #[test] - fn test_drops() { - DROP_VECTOR.with(|slot| { - *slot.borrow_mut() = vec![0; 200]; - }); - - { - let mut m = HashMap::new(); + fn test_reserve_shrink_to_fit() { + let mut m = HashMap::new(); + m.insert(0, 0); + m.remove(&0); + assert!(m.capacity() >= m.len()); + for i in 0..128 { + m.insert(i, i); + } + m.reserve(256); - DROP_VECTOR.with(|v| { - for i in 0..200 { - assert_eq!(v.borrow()[i], 0); - } - }); + let usable_cap = m.capacity(); + for i in 128..(128 + 256) { + m.insert(i, i); + assert_eq!(m.capacity(), usable_cap); + } - for i in 0..100 { - let d1 = Droppable::new(i); - let d2 = Droppable::new(i + 100); - m.insert(d1, d2); - } + for i in 100..(128 + 256) { + assert_eq!(m.remove(&i), Some(i)); + } + m.shrink_to_fit(); - DROP_VECTOR.with(|v| { - for i in 0..200 { - assert_eq!(v.borrow()[i], 1); - } - }); + assert_eq!(m.len(), 100); + assert!(!m.is_empty()); + assert!(m.capacity() >= m.len()); - for i in 0..50 { - let k = Droppable::new(i); - let v = m.remove(&k); + for i in 0..100 { + assert_eq!(m.remove(&i), Some(i)); + } + m.shrink_to_fit(); + m.insert(0, 0); - assert!(v.is_some()); + assert_eq!(m.len(), 1); + assert!(m.capacity() >= m.len()); + assert_eq!(m.remove(&0), Some(0)); + } - DROP_VECTOR.with(|v| { - assert_eq!(v.borrow()[i], 1); - assert_eq!(v.borrow()[i + 100], 1); - }); - } + #[test] + fn test_from_iter() { + let xs = [(1, 1), (2, 2), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]; - DROP_VECTOR.with(|v| { - for i in 0..50 { - assert_eq!(v.borrow()[i], 0); - assert_eq!(v.borrow()[i + 100], 0); - } + let map: HashMap<_, _> = xs.iter().copied().collect(); - for i in 50..100 { - assert_eq!(v.borrow()[i], 1); - assert_eq!(v.borrow()[i + 100], 1); - } - }); + for &(k, v) in &xs { + assert_eq!(map.get(&k), Some(&v)); } - DROP_VECTOR.with(|v| { - for i in 0..200 { - assert_eq!(v.borrow()[i], 0); - } - }); + assert_eq!(map.iter().len(), xs.len() - 1); } #[test] - fn test_into_iter_drops() { - DROP_VECTOR.with(|v| { - *v.borrow_mut() = vec![0; 200]; - }); + fn test_size_hint() { + let xs = [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]; - let hm = { - let mut hm = HashMap::new(); + let map: HashMap<_, _> = xs.iter().copied().collect(); - DROP_VECTOR.with(|v| { - for i in 0..200 { - assert_eq!(v.borrow()[i], 0); - } - }); + let mut iter = map.iter(); - for i in 0..100 { - let d1 = Droppable::new(i); - let d2 = Droppable::new(i + 100); - hm.insert(d1, d2); - } + for _ in iter.by_ref().take(3) {} - DROP_VECTOR.with(|v| { - for i in 0..200 { - assert_eq!(v.borrow()[i], 1); - } - }); + assert_eq!(iter.size_hint(), (3, Some(3))); + } - hm - }; + #[test] + fn test_iter_len() { + let xs = [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]; - // By the way, ensure that cloning doesn't screw up the dropping. - drop(hm.clone()); + let map: HashMap<_, _> = xs.iter().copied().collect(); - { - let mut half = hm.into_iter().take(50); + let mut iter = map.iter(); - DROP_VECTOR.with(|v| { - for i in 0..200 { - assert_eq!(v.borrow()[i], 1); - } - }); + for _ in iter.by_ref().take(3) {} - for _ in half.by_ref() {} + assert_eq!(iter.len(), 3); + } - DROP_VECTOR.with(|v| { - let nk = (0..100).filter(|&i| v.borrow()[i] == 1).count(); + #[test] + fn test_mut_size_hint() { + let xs = [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]; + + let mut map: HashMap<_, _> = xs.iter().copied().collect(); - let nv = (0..100).filter(|&i| v.borrow()[i + 100] == 1).count(); + let mut iter = map.iter_mut(); - assert_eq!(nk, 50); - assert_eq!(nv, 50); - }); - }; + for _ in iter.by_ref().take(3) {} - DROP_VECTOR.with(|v| { - for i in 0..200 { - assert_eq!(v.borrow()[i], 0); - } - }); + assert_eq!(iter.size_hint(), (3, Some(3))); } #[test] - fn test_empty_remove() { - let mut m: HashMap = HashMap::new(); - assert_eq!(m.remove(&0), None); + fn test_iter_mut_len() { + let xs = [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]; + + let mut map: HashMap<_, _> = xs.iter().copied().collect(); + + let mut iter = map.iter_mut(); + + for _ in iter.by_ref().take(3) {} + + assert_eq!(iter.len(), 3); } #[test] - fn test_empty_entry() { - let mut m: HashMap = HashMap::new(); - match m.entry(0) { - Occupied(_) => panic!(), - Vacant(_) => {} - } - assert!(*m.entry(0).or_insert(true)); - assert_eq!(m.len(), 1); + fn test_index() { + let mut map = HashMap::new(); + + map.insert(1, 2); + map.insert(2, 1); + map.insert(3, 4); + + assert_eq!(map[&2], 1); } #[test] - fn test_empty_iter() { - let mut m: HashMap = HashMap::new(); - assert_eq!(m.drain().next(), None); - assert_eq!(m.keys().next(), None); - assert_eq!(m.values().next(), None); - assert_eq!(m.values_mut().next(), None); - assert_eq!(m.iter().next(), None); - assert_eq!(m.iter_mut().next(), None); - assert_eq!(m.len(), 0); - assert!(m.is_empty()); - assert_eq!(m.into_iter().next(), None); + #[should_panic] + fn test_index_nonexistent() { + let mut map = HashMap::new(); + + map.insert(1, 2); + map.insert(2, 1); + map.insert(3, 4); + + #[allow(clippy::no_effect)] // false positive lint + map[&4]; } #[test] - #[cfg_attr(miri, ignore)] // FIXME: takes too long - fn test_lots_of_insertions() { - let mut m = HashMap::new(); + fn test_entry() { + let xs = [(1, 10), (2, 20), (3, 30), (4, 40), (5, 50), (6, 60)]; - // Try this a few times to make sure we never screw up the hashmap's - // internal state. - for _ in 0..10 { - assert!(m.is_empty()); + let mut map: HashMap<_, _> = xs.iter().copied().collect(); - for i in 1..1001 { - assert!(m.insert(i, i).is_none()); + // Existing key (insert) + match map.entry(1) { + Vacant(_) => unreachable!(), + Occupied(mut view) => { + assert_eq!(view.get(), &10); + assert_eq!(view.insert(100), 10); + } + } + assert_eq!(map.get(&1).unwrap(), &100); + assert_eq!(map.len(), 6); - for j in 1..=i { - let r = m.get(&j); - assert_eq!(r, Some(&j)); - } + // Existing key (update) + match map.entry(2) { + Vacant(_) => unreachable!(), + Occupied(mut view) => { + let v = view.get_mut(); + let new_v = (*v) * 10; + *v = new_v; + } + } + assert_eq!(map.get(&2).unwrap(), &200); + assert_eq!(map.len(), 6); - for j in i + 1..1001 { - let r = m.get(&j); - assert_eq!(r, None); - } + // Existing key (take) + match map.entry(3) { + Vacant(_) => unreachable!(), + Occupied(view) => { + assert_eq!(view.remove(), 30); } + } + assert_eq!(map.get(&3), None); + assert_eq!(map.len(), 5); - for i in 1001..2001 { - assert!(!m.contains_key(&i)); + // Inexistent key (insert) + match map.entry(10) { + Occupied(_) => unreachable!(), + Vacant(view) => { + assert_eq!(*view.insert(1000), 1000); } + } + assert_eq!(map.get(&10).unwrap(), &1000); + assert_eq!(map.len(), 6); + } - // remove forwards - for i in 1..1001 { - assert!(m.remove(&i).is_some()); + #[test] + fn test_entry_ref() { + let xs = [ + ("One".to_owned(), 10), + ("Two".to_owned(), 20), + ("Three".to_owned(), 30), + ("Four".to_owned(), 40), + ("Five".to_owned(), 50), + ("Six".to_owned(), 60), + ]; - for j in 1..=i { - assert!(!m.contains_key(&j)); - } + let mut map: HashMap<_, _> = xs.iter().cloned().collect(); - for j in i + 1..1001 { - assert!(m.contains_key(&j)); - } + // Existing key (insert) + match map.entry_ref("One") { + EntryRef::Vacant(_) => unreachable!(), + EntryRef::Occupied(mut view) => { + assert_eq!(view.get(), &10); + assert_eq!(view.insert(100), 10); } + } + assert_eq!(map.get("One").unwrap(), &100); + assert_eq!(map.len(), 6); - for i in 1..1001 { - assert!(!m.contains_key(&i)); + // Existing key (update) + match map.entry_ref("Two") { + EntryRef::Vacant(_) => unreachable!(), + EntryRef::Occupied(mut view) => { + let v = view.get_mut(); + let new_v = (*v) * 10; + *v = new_v; } + } + assert_eq!(map.get("Two").unwrap(), &200); + assert_eq!(map.len(), 6); - for i in 1..1001 { - assert!(m.insert(i, i).is_none()); + // Existing key (take) + match map.entry_ref("Three") { + EntryRef::Vacant(_) => unreachable!(), + EntryRef::Occupied(view) => { + assert_eq!(view.remove(), 30); } + } + assert_eq!(map.get("Three"), None); + assert_eq!(map.len(), 5); - // remove backwards - for i in (1..1001).rev() { - assert!(m.remove(&i).is_some()); - - for j in i..1001 { - assert!(!m.contains_key(&j)); - } - - for j in 1..i { - assert!(m.contains_key(&j)); - } + // Inexistent key (insert) + match map.entry_ref("Ten") { + EntryRef::Occupied(_) => unreachable!(), + EntryRef::Vacant(view) => { + assert_eq!(*view.insert(1000), 1000); } } + assert_eq!(map.get("Ten").unwrap(), &1000); + assert_eq!(map.len(), 6); } #[test] - fn test_find_mut() { - let mut m = HashMap::new(); - assert!(m.insert(1, 12).is_none()); - assert!(m.insert(2, 8).is_none()); - assert!(m.insert(5, 14).is_none()); - let new = 100; - match m.get_mut(&5) { - None => panic!(), - Some(x) => *x = new, + fn test_entry_take_doesnt_corrupt() { + #![allow(deprecated)] //rand + // Test for #19292 + fn check(m: &HashMap) { + for k in m.keys() { + assert!(m.contains_key(k), "{} is in keys() but not in the map?", k); + } } - assert_eq!(m.get(&5), Some(&new)); - } - #[test] - fn test_insert_overwrite() { let mut m = HashMap::new(); - assert!(m.insert(1, 2).is_none()); - assert_eq!(*m.get(&1).unwrap(), 2); - assert!(!m.insert(1, 3).is_none()); - assert_eq!(*m.get(&1).unwrap(), 3); - } - #[test] - fn test_insert_conflicts() { - let mut m = HashMap::with_capacity(4); - assert!(m.insert(1, 2).is_none()); - assert!(m.insert(5, 3).is_none()); - assert!(m.insert(9, 4).is_none()); - assert_eq!(*m.get(&9).unwrap(), 4); - assert_eq!(*m.get(&5).unwrap(), 3); - assert_eq!(*m.get(&1).unwrap(), 2); - } + let mut rng = { + let seed = u64::from_le_bytes(*b"testseed"); + SmallRng::seed_from_u64(seed) + }; - #[test] - fn test_conflict_remove() { - let mut m = HashMap::with_capacity(4); - assert!(m.insert(1, 2).is_none()); - assert_eq!(*m.get(&1).unwrap(), 2); - assert!(m.insert(5, 3).is_none()); - assert_eq!(*m.get(&1).unwrap(), 2); - assert_eq!(*m.get(&5).unwrap(), 3); - assert!(m.insert(9, 4).is_none()); - assert_eq!(*m.get(&1).unwrap(), 2); - assert_eq!(*m.get(&5).unwrap(), 3); - assert_eq!(*m.get(&9).unwrap(), 4); - assert!(m.remove(&1).is_some()); - assert_eq!(*m.get(&9).unwrap(), 4); - assert_eq!(*m.get(&5).unwrap(), 3); - } + // Populate the map with some items. + for _ in 0..50 { + let x = rng.gen_range(-10..10); + m.insert(x, ()); + } - #[test] - fn test_is_empty() { - let mut m = HashMap::with_capacity(4); - assert!(m.insert(1, 2).is_none()); - assert!(!m.is_empty()); - assert!(m.remove(&1).is_some()); - assert!(m.is_empty()); - } + for _ in 0..1000 { + let x = rng.gen_range(-10..10); + match m.entry(x) { + Vacant(_) => {} + Occupied(e) => { + e.remove(); + } + } - #[test] - fn test_remove() { - let mut m = HashMap::new(); - m.insert(1, 2); - assert_eq!(m.remove(&1), Some(2)); - assert_eq!(m.remove(&1), None); + check(&m); + } } #[test] - fn test_remove_entry() { + fn test_entry_ref_take_doesnt_corrupt() { + #![allow(deprecated)] //rand + // Test for #19292 + fn check(m: &HashMap) { + for k in m.keys() { + assert!(m.contains_key(k), "{} is in keys() but not in the map?", k); + } + } + let mut m = HashMap::new(); - m.insert(1, 2); - assert_eq!(m.remove_entry(&1), Some((1, 2))); - assert_eq!(m.remove(&1), None); - } - #[test] - fn test_iterate() { - let mut m = HashMap::with_capacity(4); - for i in 0..32 { - assert!(m.insert(i, i * 2).is_none()); + let mut rng = { + let seed = u64::from_le_bytes(*b"testseed"); + SmallRng::seed_from_u64(seed) + }; + + // Populate the map with some items. + for _ in 0..50 { + let mut x = std::string::String::with_capacity(1); + x.push(rng.gen_range('a'..='z')); + m.insert(x, ()); } - assert_eq!(m.len(), 32); - let mut observed: u32 = 0; + for _ in 0..1000 { + let mut x = std::string::String::with_capacity(1); + x.push(rng.gen_range('a'..='z')); + match m.entry_ref(x.as_str()) { + EntryRef::Vacant(_) => {} + EntryRef::Occupied(e) => { + e.remove(); + } + } - for (k, v) in &m { - assert_eq!(*v, *k * 2); - observed |= 1 << *k; + check(&m); } - assert_eq!(observed, 0xFFFF_FFFF); } #[test] - fn test_keys() { - let vec = vec![(1, 'a'), (2, 'b'), (3, 'c')]; - let map: HashMap<_, _> = vec.into_iter().collect(); - let keys: Vec<_> = map.keys().cloned().collect(); - assert_eq!(keys.len(), 3); - assert!(keys.contains(&1)); - assert!(keys.contains(&2)); - assert!(keys.contains(&3)); - } + fn test_extend_ref_k_ref_v() { + let mut a = HashMap::new(); + a.insert(1, "one"); + let mut b = HashMap::new(); + b.insert(2, "two"); + b.insert(3, "three"); - #[test] - fn test_values() { - let vec = vec![(1, 'a'), (2, 'b'), (3, 'c')]; - let map: HashMap<_, _> = vec.into_iter().collect(); - let values: Vec<_> = map.values().cloned().collect(); - assert_eq!(values.len(), 3); - assert!(values.contains(&'a')); - assert!(values.contains(&'b')); - assert!(values.contains(&'c')); + a.extend(&b); + + assert_eq!(a.len(), 3); + assert_eq!(a[&1], "one"); + assert_eq!(a[&2], "two"); + assert_eq!(a[&3], "three"); } #[test] - fn test_values_mut() { - let vec = vec![(1, 1), (2, 2), (3, 3)]; - let mut map: HashMap<_, _> = vec.into_iter().collect(); - for value in map.values_mut() { - *value = (*value) * 2 + fn test_extend_ref_kv_tuple() { + use std::ops::AddAssign; + let mut a = HashMap::new(); + a.insert(0, 0); + + fn create_arr + Copy, const N: usize>(start: T, step: T) -> [(T, T); N] { + let mut outs: [(T, T); N] = [(start, start); N]; + let mut element = step; + outs.iter_mut().skip(1).for_each(|(k, v)| { + *k += element; + *v += element; + element += step; + }); + outs } - let values: Vec<_> = map.values().cloned().collect(); - assert_eq!(values.len(), 3); - assert!(values.contains(&2)); - assert!(values.contains(&4)); - assert!(values.contains(&6)); - } - #[test] - fn test_find() { - let mut m = HashMap::new(); - assert!(m.get(&1).is_none()); - m.insert(1, 2); - match m.get(&1) { - None => panic!(), - Some(v) => assert_eq!(*v, 2), + let for_iter: Vec<_> = (0..100).map(|i| (i, i)).collect(); + let iter = for_iter.iter(); + let vec: Vec<_> = (100..200).map(|i| (i, i)).collect(); + a.extend(iter); + a.extend(&vec); + a.extend(&create_arr::(200, 1)); + + assert_eq!(a.len(), 300); + + for item in 0..300 { + assert_eq!(a[&item], item); } } #[test] - fn test_eq() { - let mut m1 = HashMap::new(); - m1.insert(1, 2); - m1.insert(2, 3); - m1.insert(3, 4); + fn test_capacity_not_less_than_len() { + let mut a = HashMap::new(); + let mut item = 0; - let mut m2 = HashMap::new(); - m2.insert(1, 2); - m2.insert(2, 3); + for _ in 0..116 { + a.insert(item, 0); + item += 1; + } - assert!(m1 != m2); + assert!(a.capacity() > a.len()); - m2.insert(3, 4); + let free = a.capacity() - a.len(); + for _ in 0..free { + a.insert(item, 0); + item += 1; + } - assert_eq!(m1, m2); + assert_eq!(a.len(), a.capacity()); + + // Insert at capacity should cause allocation. + a.insert(item, 0); + assert!(a.capacity() > a.len()); } #[test] - fn test_show() { - let mut map = HashMap::new(); - let empty: HashMap = HashMap::new(); + fn test_occupied_entry_key() { + let mut a = HashMap::new(); + let key = "hello there"; + let value = "value goes here"; + assert!(a.is_empty()); + a.insert(key, value); + assert_eq!(a.len(), 1); + assert_eq!(a[key], value); - map.insert(1, 2); - map.insert(3, 4); + match a.entry(key) { + Vacant(_) => panic!(), + Occupied(e) => assert_eq!(key, *e.key()), + } + assert_eq!(a.len(), 1); + assert_eq!(a[key], value); + } - let map_str = format!("{:?}", map); + #[test] + fn test_occupied_entry_ref_key() { + let mut a = HashMap::new(); + let key = "hello there"; + let value = "value goes here"; + assert!(a.is_empty()); + a.insert(key.to_owned(), value); + assert_eq!(a.len(), 1); + assert_eq!(a[key], value); - assert!(map_str == "{1: 2, 3: 4}" || map_str == "{3: 4, 1: 2}"); - assert_eq!(format!("{:?}", empty), "{}"); + match a.entry_ref(key) { + EntryRef::Vacant(_) => panic!(), + EntryRef::Occupied(e) => assert_eq!(key, e.key()), + } + assert_eq!(a.len(), 1); + assert_eq!(a[key], value); } #[test] - fn test_expand() { - let mut m = HashMap::new(); - - assert_eq!(m.len(), 0); - assert!(m.is_empty()); + fn test_vacant_entry_key() { + let mut a = HashMap::new(); + let key = "hello there"; + let value = "value goes here"; - let mut i = 0; - let old_raw_cap = m.raw_capacity(); - while old_raw_cap == m.raw_capacity() { - m.insert(i, i); - i += 1; + assert!(a.is_empty()); + match a.entry(key) { + Occupied(_) => panic!(), + Vacant(e) => { + assert_eq!(key, *e.key()); + e.insert(value); + } } - - assert_eq!(m.len(), i); - assert!(!m.is_empty()); + assert_eq!(a.len(), 1); + assert_eq!(a[key], value); } #[test] - fn test_behavior_resize_policy() { - let mut m = HashMap::new(); - - assert_eq!(m.len(), 0); - assert_eq!(m.raw_capacity(), 1); - assert!(m.is_empty()); + fn test_vacant_entry_ref_key() { + let mut a: HashMap = HashMap::new(); + let key = "hello there"; + let value = "value goes here"; - m.insert(0, 0); - m.remove(&0); - assert!(m.is_empty()); - let initial_raw_cap = m.raw_capacity(); - m.reserve(initial_raw_cap); - let raw_cap = m.raw_capacity(); + assert!(a.is_empty()); + match a.entry_ref(key) { + EntryRef::Occupied(_) => panic!(), + EntryRef::Vacant(e) => { + assert_eq!(key, e.key()); + e.insert(value); + } + } + assert_eq!(a.len(), 1); + assert_eq!(a[key], value); + } - assert_eq!(raw_cap, initial_raw_cap * 2); + #[test] + fn test_occupied_entry_replace_entry_with() { + let mut a = HashMap::new(); - let mut i = 0; - for _ in 0..raw_cap * 3 / 4 { - m.insert(i, i); - i += 1; - } - // three quarters full + let key = "a key"; + let value = "an initial value"; + let new_value = "a new value"; - assert_eq!(m.len(), i); - assert_eq!(m.raw_capacity(), raw_cap); + let entry = a.entry(key).insert(value).replace_entry_with(|k, v| { + assert_eq!(k, &key); + assert_eq!(v, value); + Some(new_value) + }); - for _ in 0..raw_cap / 4 { - m.insert(i, i); - i += 1; + match entry { + Occupied(e) => { + assert_eq!(e.key(), &key); + assert_eq!(e.get(), &new_value); + } + Vacant(_) => panic!(), } - // half full - let new_raw_cap = m.raw_capacity(); - assert_eq!(new_raw_cap, raw_cap * 2); + assert_eq!(a[key], new_value); + assert_eq!(a.len(), 1); - for _ in 0..raw_cap / 2 - 1 { - i -= 1; - m.remove(&i); - assert_eq!(m.raw_capacity(), new_raw_cap); - } - // A little more than one quarter full. - m.shrink_to_fit(); - assert_eq!(m.raw_capacity(), raw_cap); - // again, a little more than half full - for _ in 0..raw_cap / 2 { - i -= 1; - m.remove(&i); + let entry = match a.entry(key) { + Occupied(e) => e.replace_entry_with(|k, v| { + assert_eq!(k, &key); + assert_eq!(v, new_value); + None + }), + Vacant(_) => panic!(), + }; + + match entry { + Vacant(e) => assert_eq!(e.key(), &key), + Occupied(_) => panic!(), } - m.shrink_to_fit(); - assert_eq!(m.len(), i); - assert!(!m.is_empty()); - assert_eq!(m.raw_capacity(), initial_raw_cap); + assert!(!a.contains_key(key)); + assert_eq!(a.len(), 0); } #[test] - fn test_reserve_shrink_to_fit() { - let mut m = HashMap::new(); - m.insert(0, 0); - m.remove(&0); - assert!(m.capacity() >= m.len()); - for i in 0..128 { - m.insert(i, i); - } - m.reserve(256); - - let usable_cap = m.capacity(); - for i in 128..(128 + 256) { - m.insert(i, i); - assert_eq!(m.capacity(), usable_cap); - } + fn test_occupied_entry_ref_replace_entry_with() { + let mut a: HashMap = HashMap::new(); - for i in 100..(128 + 256) { - assert_eq!(m.remove(&i), Some(i)); - } - m.shrink_to_fit(); + let key = "a key"; + let value = "an initial value"; + let new_value = "a new value"; - assert_eq!(m.len(), 100); - assert!(!m.is_empty()); - assert!(m.capacity() >= m.len()); + let entry = a.entry_ref(key).insert(value).replace_entry_with(|k, v| { + assert_eq!(k, key); + assert_eq!(v, value); + Some(new_value) + }); - for i in 0..100 { - assert_eq!(m.remove(&i), Some(i)); + match entry { + EntryRef::Occupied(e) => { + assert_eq!(e.key(), key); + assert_eq!(e.get(), &new_value); + } + EntryRef::Vacant(_) => panic!(), } - m.shrink_to_fit(); - m.insert(0, 0); - - assert_eq!(m.len(), 1); - assert!(m.capacity() >= m.len()); - assert_eq!(m.remove(&0), Some(0)); - } - #[test] - fn test_from_iter() { - let xs = [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]; + assert_eq!(a[key], new_value); + assert_eq!(a.len(), 1); - let map: HashMap<_, _> = xs.iter().cloned().collect(); + let entry = match a.entry_ref(key) { + EntryRef::Occupied(e) => e.replace_entry_with(|k, v| { + assert_eq!(k, key); + assert_eq!(v, new_value); + None + }), + EntryRef::Vacant(_) => panic!(), + }; - for &(k, v) in &xs { - assert_eq!(map.get(&k), Some(&v)); + match entry { + EntryRef::Vacant(e) => assert_eq!(e.key(), key), + EntryRef::Occupied(_) => panic!(), } + + assert!(!a.contains_key(key)); + assert_eq!(a.len(), 0); } #[test] - fn test_size_hint() { - let xs = [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]; + fn test_entry_and_replace_entry_with() { + let mut a = HashMap::new(); - let map: HashMap<_, _> = xs.iter().cloned().collect(); + let key = "a key"; + let value = "an initial value"; + let new_value = "a new value"; - let mut iter = map.iter(); + let entry = a.entry(key).and_replace_entry_with(|_, _| panic!()); - for _ in iter.by_ref().take(3) {} + match entry { + Vacant(e) => assert_eq!(e.key(), &key), + Occupied(_) => panic!(), + } - assert_eq!(iter.size_hint(), (3, Some(3))); - } + a.insert(key, value); - #[test] - fn test_iter_len() { - let xs = [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]; + let entry = a.entry(key).and_replace_entry_with(|k, v| { + assert_eq!(k, &key); + assert_eq!(v, value); + Some(new_value) + }); - let map: HashMap<_, _> = xs.iter().cloned().collect(); + match entry { + Occupied(e) => { + assert_eq!(e.key(), &key); + assert_eq!(e.get(), &new_value); + } + Vacant(_) => panic!(), + } - let mut iter = map.iter(); + assert_eq!(a[key], new_value); + assert_eq!(a.len(), 1); - for _ in iter.by_ref().take(3) {} + let entry = a.entry(key).and_replace_entry_with(|k, v| { + assert_eq!(k, &key); + assert_eq!(v, new_value); + None + }); - assert_eq!(iter.len(), 3); + match entry { + Vacant(e) => assert_eq!(e.key(), &key), + Occupied(_) => panic!(), + } + + assert!(!a.contains_key(key)); + assert_eq!(a.len(), 0); } #[test] - fn test_mut_size_hint() { - let xs = [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]; + fn test_entry_ref_and_replace_entry_with() { + let mut a = HashMap::new(); - let mut map: HashMap<_, _> = xs.iter().cloned().collect(); + let key = "a key"; + let value = "an initial value"; + let new_value = "a new value"; - let mut iter = map.iter_mut(); + let entry = a.entry_ref(key).and_replace_entry_with(|_, _| panic!()); - for _ in iter.by_ref().take(3) {} + match entry { + EntryRef::Vacant(e) => assert_eq!(e.key(), key), + EntryRef::Occupied(_) => panic!(), + } - assert_eq!(iter.size_hint(), (3, Some(3))); - } + a.insert(key.to_owned(), value); - #[test] - fn test_iter_mut_len() { - let xs = [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]; + let entry = a.entry_ref(key).and_replace_entry_with(|k, v| { + assert_eq!(k, key); + assert_eq!(v, value); + Some(new_value) + }); - let mut map: HashMap<_, _> = xs.iter().cloned().collect(); + match entry { + EntryRef::Occupied(e) => { + assert_eq!(e.key(), key); + assert_eq!(e.get(), &new_value); + } + EntryRef::Vacant(_) => panic!(), + } - let mut iter = map.iter_mut(); + assert_eq!(a[key], new_value); + assert_eq!(a.len(), 1); - for _ in iter.by_ref().take(3) {} + let entry = a.entry_ref(key).and_replace_entry_with(|k, v| { + assert_eq!(k, key); + assert_eq!(v, new_value); + None + }); - assert_eq!(iter.len(), 3); + match entry { + EntryRef::Vacant(e) => assert_eq!(e.key(), key), + EntryRef::Occupied(_) => panic!(), + } + + assert!(!a.contains_key(key)); + assert_eq!(a.len(), 0); } #[test] - fn test_index() { - let mut map = HashMap::new(); + fn test_raw_occupied_entry_replace_entry_with() { + let mut a = HashMap::new(); - map.insert(1, 2); - map.insert(2, 1); - map.insert(3, 4); + let key = "a key"; + let value = "an initial value"; + let new_value = "a new value"; + + let entry = a + .raw_entry_mut() + .from_key(&key) + .insert(key, value) + .replace_entry_with(|k, v| { + assert_eq!(k, &key); + assert_eq!(v, value); + Some(new_value) + }); - assert_eq!(map[&2], 1); - } + match entry { + RawEntryMut::Occupied(e) => { + assert_eq!(e.key(), &key); + assert_eq!(e.get(), &new_value); + } + RawEntryMut::Vacant(_) => panic!(), + } - #[test] - #[should_panic] - fn test_index_nonexistent() { - let mut map = HashMap::new(); + assert_eq!(a[key], new_value); + assert_eq!(a.len(), 1); - map.insert(1, 2); - map.insert(2, 1); - map.insert(3, 4); + let entry = match a.raw_entry_mut().from_key(&key) { + RawEntryMut::Occupied(e) => e.replace_entry_with(|k, v| { + assert_eq!(k, &key); + assert_eq!(v, new_value); + None + }), + RawEntryMut::Vacant(_) => panic!(), + }; - map[&4]; + match entry { + RawEntryMut::Vacant(_) => {} + RawEntryMut::Occupied(_) => panic!(), + } + + assert!(!a.contains_key(key)); + assert_eq!(a.len(), 0); } #[test] - fn test_entry() { - let xs = [(1, 10), (2, 20), (3, 30), (4, 40), (5, 50), (6, 60)]; + fn test_raw_entry_and_replace_entry_with() { + let mut a = HashMap::new(); - let mut map: HashMap<_, _> = xs.iter().cloned().collect(); + let key = "a key"; + let value = "an initial value"; + let new_value = "a new value"; - // Existing key (insert) - match map.entry(1) { - Vacant(_) => unreachable!(), - Occupied(mut view) => { - assert_eq!(view.get(), &10); - assert_eq!(view.insert(100), 10); - } - } - assert_eq!(map.get(&1).unwrap(), &100); - assert_eq!(map.len(), 6); + let entry = a + .raw_entry_mut() + .from_key(&key) + .and_replace_entry_with(|_, _| panic!()); - // Existing key (update) - match map.entry(2) { - Vacant(_) => unreachable!(), - Occupied(mut view) => { - let v = view.get_mut(); - let new_v = (*v) * 10; - *v = new_v; - } + match entry { + RawEntryMut::Vacant(_) => {} + RawEntryMut::Occupied(_) => panic!(), } - assert_eq!(map.get(&2).unwrap(), &200); - assert_eq!(map.len(), 6); - // Existing key (take) - match map.entry(3) { - Vacant(_) => unreachable!(), - Occupied(view) => { - assert_eq!(view.remove(), 30); + a.insert(key, value); + + let entry = a + .raw_entry_mut() + .from_key(&key) + .and_replace_entry_with(|k, v| { + assert_eq!(k, &key); + assert_eq!(v, value); + Some(new_value) + }); + + match entry { + RawEntryMut::Occupied(e) => { + assert_eq!(e.key(), &key); + assert_eq!(e.get(), &new_value); } + RawEntryMut::Vacant(_) => panic!(), } - assert_eq!(map.get(&3), None); - assert_eq!(map.len(), 5); - // Inexistent key (insert) - match map.entry(10) { - Occupied(_) => unreachable!(), - Vacant(view) => { - assert_eq!(*view.insert(1000), 1000); - } + assert_eq!(a[key], new_value); + assert_eq!(a.len(), 1); + + let entry = a + .raw_entry_mut() + .from_key(&key) + .and_replace_entry_with(|k, v| { + assert_eq!(k, &key); + assert_eq!(v, new_value); + None + }); + + match entry { + RawEntryMut::Vacant(_) => {} + RawEntryMut::Occupied(_) => panic!(), } - assert_eq!(map.get(&10).unwrap(), &1000); - assert_eq!(map.len(), 6); + + assert!(!a.contains_key(key)); + assert_eq!(a.len(), 0); } #[test] - fn test_entry_take_doesnt_corrupt() { + fn test_replace_entry_with_doesnt_corrupt() { #![allow(deprecated)] //rand // Test for #19292 fn check(m: &HashMap) { @@ -3557,104 +7988,53 @@ mod test_map { let mut m = HashMap::new(); let mut rng = { - let seed = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; - SmallRng::from_seed(seed) + let seed = u64::from_le_bytes(*b"testseed"); + SmallRng::seed_from_u64(seed) }; // Populate the map with some items. for _ in 0..50 { - let x = rng.gen_range(-10, 10); + let x = rng.gen_range(-10..10); m.insert(x, ()); } for _ in 0..1000 { - let x = rng.gen_range(-10, 10); - match m.entry(x) { - Vacant(_) => {} - Occupied(e) => { - e.remove(); - } - } - + let x = rng.gen_range(-10..10); + m.entry(x).and_replace_entry_with(|_, _| None); check(&m); } } #[test] - fn test_extend_ref() { - let mut a = HashMap::new(); - a.insert(1, "one"); - let mut b = HashMap::new(); - b.insert(2, "two"); - b.insert(3, "three"); - - a.extend(&b); - - assert_eq!(a.len(), 3); - assert_eq!(a[&1], "one"); - assert_eq!(a[&2], "two"); - assert_eq!(a[&3], "three"); - } - - #[test] - fn test_capacity_not_less_than_len() { - let mut a = HashMap::new(); - let mut item = 0; - - for _ in 0..116 { - a.insert(item, 0); - item += 1; - } - - assert!(a.capacity() > a.len()); - - let free = a.capacity() - a.len(); - for _ in 0..free { - a.insert(item, 0); - item += 1; + fn test_replace_entry_ref_with_doesnt_corrupt() { + #![allow(deprecated)] //rand + // Test for #19292 + fn check(m: &HashMap) { + for k in m.keys() { + assert!(m.contains_key(k), "{} is in keys() but not in the map?", k); + } } - assert_eq!(a.len(), a.capacity()); - - // Insert at capacity should cause allocation. - a.insert(item, 0); - assert!(a.capacity() > a.len()); - } + let mut m = HashMap::new(); - #[test] - fn test_occupied_entry_key() { - let mut a = HashMap::new(); - let key = "hello there"; - let value = "value goes here"; - assert!(a.is_empty()); - a.insert(key.clone(), value.clone()); - assert_eq!(a.len(), 1); - assert_eq!(a[key], value); + let mut rng = { + let seed = u64::from_le_bytes(*b"testseed"); + SmallRng::seed_from_u64(seed) + }; - match a.entry(key.clone()) { - Vacant(_) => panic!(), - Occupied(e) => assert_eq!(key, *e.key()), + // Populate the map with some items. + for _ in 0..50 { + let mut x = std::string::String::with_capacity(1); + x.push(rng.gen_range('a'..='z')); + m.insert(x, ()); } - assert_eq!(a.len(), 1); - assert_eq!(a[key], value); - } - - #[test] - fn test_vacant_entry_key() { - let mut a = HashMap::new(); - let key = "hello there"; - let value = "value goes here"; - assert!(a.is_empty()); - match a.entry(key.clone()) { - Occupied(_) => panic!(), - Vacant(e) => { - assert_eq!(key, *e.key()); - e.insert(value.clone()); - } + for _ in 0..1000 { + let mut x = std::string::String::with_capacity(1); + x.push(rng.gen_range('a'..='z')); + m.entry_ref(x.as_str()).and_replace_entry_with(|_, _| None); + check(&m); } - assert_eq!(a.len(), 1); - assert_eq!(a[key], value); } #[test] @@ -3675,7 +8055,7 @@ mod test_map { let drained = map.drain_filter(|&k, _| k % 2 == 0); let mut out = drained.collect::>(); out.sort_unstable(); - assert_eq!(vec![(1, 10), (3, 30), (5, 50), (7, 70)], out); + assert_eq!(vec![(0, 0), (2, 20), (4, 40), (6, 60)], out); assert_eq!(map.len(), 4); } { @@ -3688,21 +8068,27 @@ mod test_map { #[test] #[cfg_attr(miri, ignore)] // FIXME: no OOM signalling (https://github.com/rust-lang/miri/issues/613) fn test_try_reserve() { - let mut empty_bytes: HashMap = HashMap::new(); + use crate::TryReserveError::{AllocError, CapacityOverflow}; const MAX_USIZE: usize = usize::MAX; + let mut empty_bytes: HashMap = HashMap::new(); + if let Err(CapacityOverflow) = empty_bytes.try_reserve(MAX_USIZE) { } else { panic!("usize::MAX should trigger an overflow!"); } - if let Err(AllocErr { .. }) = empty_bytes.try_reserve(MAX_USIZE / 8) { + if let Err(AllocError { .. }) = empty_bytes.try_reserve(MAX_USIZE / 16) { } else { // This may succeed if there is enough free memory. Attempt to - // allocate a second hashmap to ensure the allocation will fail. + // allocate a few more hashmaps to ensure the allocation will fail. let mut empty_bytes2: HashMap = HashMap::new(); - if let Err(AllocErr { .. }) = empty_bytes2.try_reserve(MAX_USIZE / 8) { + let _ = empty_bytes2.try_reserve(MAX_USIZE / 16); + let mut empty_bytes3: HashMap = HashMap::new(); + let _ = empty_bytes3.try_reserve(MAX_USIZE / 16); + let mut empty_bytes4: HashMap = HashMap::new(); + if let Err(AllocError { .. }) = empty_bytes4.try_reserve(MAX_USIZE / 16) { } else { panic!("usize::MAX / 8 should trigger an OOM!"); } @@ -3713,16 +8099,12 @@ mod test_map { fn test_raw_entry() { use super::RawEntryMut::{Occupied, Vacant}; - let xs = [(1i32, 10i32), (2, 20), (3, 30), (4, 40), (5, 50), (6, 60)]; + let xs = [(1_i32, 10_i32), (2, 20), (3, 30), (4, 40), (5, 50), (6, 60)]; - let mut map: HashMap<_, _> = xs.iter().cloned().collect(); + let mut map: HashMap<_, _> = xs.iter().copied().collect(); let compute_hash = |map: &HashMap, k: i32| -> u64 { - use core::hash::{BuildHasher, Hash, Hasher}; - - let mut hasher = map.hasher().build_hasher(); - k.hash(&mut hasher); - hasher.finish() + super::make_insert_hash::(map.hasher(), &k) }; // Existing key (insert) @@ -3792,7 +8174,7 @@ mod test_map { // Ensure all lookup methods produce equivalent results. for k in 0..12 { let hash = compute_hash(&map, k); - let v = map.get(&k).cloned(); + let v = map.get(&k).copied(); let kv = v.as_ref().map(|v| (&k, v)); assert_eq!(map.raw_entry().from_key(&k), kv); @@ -3800,15 +8182,15 @@ mod test_map { assert_eq!(map.raw_entry().from_key_hashed_nocheck(hash, &k), kv); match map.raw_entry_mut().from_key(&k) { - Occupied(mut o) => assert_eq!(Some(o.get_key_value()), kv), + Occupied(o) => assert_eq!(Some(o.get_key_value()), kv), Vacant(_) => assert_eq!(v, None), } match map.raw_entry_mut().from_key_hashed_nocheck(hash, &k) { - Occupied(mut o) => assert_eq!(Some(o.get_key_value()), kv), + Occupied(o) => assert_eq!(Some(o.get_key_value()), kv), Vacant(_) => assert_eq!(v, None), } match map.raw_entry_mut().from_hash(hash, |q| *q == k) { - Occupied(mut o) => assert_eq!(Some(o.get_key_value()), kv), + Occupied(o) => assert_eq!(Some(o.get_key_value()), kv), Vacant(_) => assert_eq!(v, None), } } @@ -3858,4 +8240,169 @@ mod test_map { assert!(m.raw_entry().from_hash(1, |k| k.0 == 1).is_some()); assert!(m.raw_entry().from_hash(2, |k| k.0 == 2).is_none()); } + + #[test] + #[cfg(feature = "raw")] + fn test_into_iter_refresh() { + #[cfg(miri)] + const N: usize = 32; + #[cfg(not(miri))] + const N: usize = 128; + + let mut rng = rand::thread_rng(); + for n in 0..N { + let mut map = HashMap::new(); + for i in 0..n { + assert!(map.insert(i, 2 * i).is_none()); + } + let hash_builder = map.hasher().clone(); + + let mut it = unsafe { map.table.iter() }; + assert_eq!(it.len(), n); + + let mut i = 0; + let mut left = n; + let mut removed = Vec::new(); + loop { + // occasionally remove some elements + if i < n && rng.gen_bool(0.1) { + let hash_value = super::make_insert_hash(&hash_builder, &i); + + unsafe { + let e = map.table.find(hash_value, |q| q.0.eq(&i)); + if let Some(e) = e { + it.reflect_remove(&e); + let t = map.table.remove(e); + removed.push(t); + left -= 1; + } else { + assert!(removed.contains(&(i, 2 * i)), "{} not in {:?}", i, removed); + let e = map.table.insert( + hash_value, + (i, 2 * i), + super::make_hasher::(&hash_builder), + ); + it.reflect_insert(&e); + if let Some(p) = removed.iter().position(|e| e == &(i, 2 * i)) { + removed.swap_remove(p); + } + left += 1; + } + } + } + + let e = it.next(); + if e.is_none() { + break; + } + assert!(i < n); + let t = unsafe { e.unwrap().as_ref() }; + assert!(!removed.contains(t)); + let (key, value) = t; + assert_eq!(*value, 2 * key); + i += 1; + } + assert!(i <= n); + + // just for safety: + assert_eq!(map.table.len(), left); + } + } + + #[test] + fn test_const_with_hasher() { + use core::hash::BuildHasher; + use std::collections::hash_map::DefaultHasher; + + #[derive(Clone)] + struct MyHasher; + impl BuildHasher for MyHasher { + type Hasher = DefaultHasher; + + fn build_hasher(&self) -> DefaultHasher { + DefaultHasher::new() + } + } + + const EMPTY_MAP: HashMap = + HashMap::with_hasher(MyHasher); + + let mut map = EMPTY_MAP; + map.insert(17, "seventeen".to_owned()); + assert_eq!("seventeen", map[&17]); + } + + #[test] + fn test_get_each_mut() { + let mut map = HashMap::new(); + map.insert("foo".to_owned(), 0); + map.insert("bar".to_owned(), 10); + map.insert("baz".to_owned(), 20); + map.insert("qux".to_owned(), 30); + + let xs = map.get_many_mut(["foo", "qux"]); + assert_eq!(xs, Some([&mut 0, &mut 30])); + + let xs = map.get_many_mut(["foo", "dud"]); + assert_eq!(xs, None); + + let xs = map.get_many_mut(["foo", "foo"]); + assert_eq!(xs, None); + + let ys = map.get_many_key_value_mut(["bar", "baz"]); + assert_eq!( + ys, + Some([(&"bar".to_owned(), &mut 10), (&"baz".to_owned(), &mut 20),]), + ); + + let ys = map.get_many_key_value_mut(["bar", "dip"]); + assert_eq!(ys, None); + + let ys = map.get_many_key_value_mut(["baz", "baz"]); + assert_eq!(ys, None); + } + + #[test] + #[should_panic = "panic in drop"] + fn test_clone_from_double_drop() { + #[derive(Clone)] + struct CheckedDrop { + panic_in_drop: bool, + dropped: bool, + } + impl Drop for CheckedDrop { + fn drop(&mut self) { + if self.panic_in_drop { + self.dropped = true; + panic!("panic in drop"); + } + if self.dropped { + panic!("double drop"); + } + self.dropped = true; + } + } + const DISARMED: CheckedDrop = CheckedDrop { + panic_in_drop: false, + dropped: false, + }; + const ARMED: CheckedDrop = CheckedDrop { + panic_in_drop: true, + dropped: false, + }; + + let mut map1 = HashMap::new(); + map1.insert(1, DISARMED); + map1.insert(2, DISARMED); + map1.insert(3, DISARMED); + map1.insert(4, DISARMED); + + let mut map2 = HashMap::new(); + map2.insert(1, DISARMED); + map2.insert(2, ARMED); + map2.insert(3, DISARMED); + map2.insert(4, DISARMED); + + map2.clone_from(&map1); + } } diff --git a/crux-mir/lib/hashbrown/src/raw/alloc.rs b/crux-mir/lib/hashbrown/src/raw/alloc.rs new file mode 100644 index 000000000..ba09ea9de --- /dev/null +++ b/crux-mir/lib/hashbrown/src/raw/alloc.rs @@ -0,0 +1,73 @@ +pub(crate) use self::inner::{do_alloc, Allocator, Global}; + +#[cfg(feature = "nightly")] +mod inner { + use crate::alloc::alloc::Layout; + pub use crate::alloc::alloc::{Allocator, Global}; + use core::ptr::NonNull; + + #[allow(clippy::map_err_ignore)] + pub fn do_alloc(alloc: &A, layout: Layout) -> Result, ()> { + match alloc.allocate(layout) { + Ok(ptr) => Ok(ptr.as_non_null_ptr()), + Err(_) => Err(()), + } + } + + #[cfg(feature = "bumpalo")] + unsafe impl Allocator for crate::BumpWrapper<'_> { + #[inline] + fn allocate(&self, layout: Layout) -> Result, core::alloc::AllocError> { + match self.0.try_alloc_layout(layout) { + Ok(ptr) => Ok(NonNull::slice_from_raw_parts(ptr, layout.size())), + Err(_) => Err(core::alloc::AllocError), + } + } + #[inline] + unsafe fn deallocate(&self, _ptr: NonNull, _layout: Layout) {} + } +} + +#[cfg(not(feature = "nightly"))] +mod inner { + use crate::alloc::alloc::{alloc, dealloc, Layout}; + use core::ptr::NonNull; + + #[allow(clippy::missing_safety_doc)] // not exposed outside of this crate + pub unsafe trait Allocator { + fn allocate(&self, layout: Layout) -> Result, ()>; + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout); + } + + #[derive(Copy, Clone)] + pub struct Global; + unsafe impl Allocator for Global { + #[inline] + fn allocate(&self, layout: Layout) -> Result, ()> { + unsafe { NonNull::new(alloc(layout)).ok_or(()) } + } + #[inline] + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + dealloc(ptr.as_ptr(), layout); + } + } + impl Default for Global { + #[inline] + fn default() -> Self { + Global + } + } + + pub fn do_alloc(alloc: &A, layout: Layout) -> Result, ()> { + alloc.allocate(layout) + } + + #[cfg(feature = "bumpalo")] + unsafe impl Allocator for crate::BumpWrapper<'_> { + #[allow(clippy::map_err_ignore)] + fn allocate(&self, layout: Layout) -> Result, ()> { + self.0.try_alloc_layout(layout).map_err(|_| ()) + } + unsafe fn deallocate(&self, _ptr: NonNull, _layout: Layout) {} + } +} diff --git a/crux-mir/lib/hashbrown/src/raw/bitmask.rs b/crux-mir/lib/hashbrown/src/raw/bitmask.rs index 6e61d4311..7d4f9fc38 100644 --- a/crux-mir/lib/hashbrown/src/raw/bitmask.rs +++ b/crux-mir/lib/hashbrown/src/raw/bitmask.rs @@ -25,6 +25,20 @@ impl BitMask { BitMask(self.0 ^ BITMASK_MASK) } + /// Flip the bit in the mask for the entry at the given index. + /// + /// Returns the bit's previous state. + #[inline] + #[allow(clippy::cast_ptr_alignment)] + #[cfg(feature = "raw")] + pub unsafe fn flip(&mut self, index: usize) -> bool { + // NOTE: The + BITMASK_STRIDE - 1 is to set the high bit. + let mask = 1 << (index * BITMASK_STRIDE + BITMASK_STRIDE - 1); + self.0 ^= mask; + // The bit was set if the bit is now 0. + self.0 & mask == 0 + } + /// Returns a new `BitMask` with the lowest bit removed. #[inline] #[must_use] @@ -92,7 +106,7 @@ impl IntoIterator for BitMask { } } -/// Iterator over the contents of a `BitMask`, returning the indicies of set +/// Iterator over the contents of a `BitMask`, returning the indices of set /// bits. pub struct BitMaskIter(BitMask); diff --git a/crux-mir/lib/hashbrown/src/raw/generic.rs b/crux-mir/lib/hashbrown/src/raw/generic.rs index dc5bdc190..b4d31e62c 100644 --- a/crux-mir/lib/hashbrown/src/raw/generic.rs +++ b/crux-mir/lib/hashbrown/src/raw/generic.rs @@ -9,12 +9,14 @@ use core::{mem, ptr}; target_pointer_width = "64", target_arch = "aarch64", target_arch = "x86_64", + target_arch = "wasm32", ))] type GroupWord = u64; #[cfg(all( target_pointer_width = "32", not(target_arch = "aarch64"), not(target_arch = "x86_64"), + not(target_arch = "wasm32"), ))] type GroupWord = u32; @@ -27,11 +29,7 @@ pub const BITMASK_MASK: BitMaskWord = 0x8080_8080_8080_8080_u64 as GroupWord; /// Helper function to replicate a byte across a `GroupWord`. #[inline] fn repeat(byte: u8) -> GroupWord { - let repeat = GroupWord::from(byte); - let repeat = repeat | repeat.wrapping_shl(8); - let repeat = repeat | repeat.wrapping_shl(16); - // This last line is a no-op with a 32-bit GroupWord - repeat | repeat.wrapping_shl(32) + GroupWord::from_ne_bytes([byte; Group::WIDTH]) } /// Abstraction over a group of control bytes which can be scanned in @@ -41,7 +39,7 @@ fn repeat(byte: u8) -> GroupWord { #[derive(Copy, Clone)] pub struct Group(GroupWord); -// We perform all operations in the native endianess, and convert to +// We perform all operations in the native endianness, and convert to // little-endian just before creating a BitMask. The can potentially // enable the compiler to eliminate unnecessary byte swaps if we are // only checking whether a BitMask is empty. @@ -51,27 +49,28 @@ impl Group { pub const WIDTH: usize = mem::size_of::(); /// Returns a full group of empty bytes, suitable for use as the initial - /// value for an empty hash table. This value is explicitly declared as - /// a static variable to ensure the address is consistent across dylibs. + /// value for an empty hash table. /// /// This is guaranteed to be aligned to the group size. #[inline] - pub fn static_empty() -> &'static [u8] { - union AlignedBytes { - _align: Group, + pub const fn static_empty() -> &'static [u8; Group::WIDTH] { + #[repr(C)] + struct AlignedBytes { + _align: [Group; 0], bytes: [u8; Group::WIDTH], - }; - static ALIGNED_BYTES: AlignedBytes = AlignedBytes { + } + const ALIGNED_BYTES: AlignedBytes = AlignedBytes { + _align: [], bytes: [EMPTY; Group::WIDTH], }; - unsafe { &ALIGNED_BYTES.bytes } + &ALIGNED_BYTES.bytes } /// Loads a group of bytes starting at the given address. #[inline] #[allow(clippy::cast_ptr_alignment)] // unaligned load pub unsafe fn load(ptr: *const u8) -> Self { - Group(ptr::read_unaligned(ptr as *const _)) + Group(ptr::read_unaligned(ptr.cast())) } /// Loads a group of bytes starting at the given address, which must be @@ -81,7 +80,7 @@ impl Group { pub unsafe fn load_aligned(ptr: *const u8) -> Self { // FIXME: use align_offset once it stabilizes debug_assert_eq!(ptr as usize & (mem::align_of::() - 1), 0); - Group(ptr::read(ptr as *const _)) + Group(ptr::read(ptr.cast())) } /// Stores the group of bytes to the given address, which must be @@ -91,7 +90,7 @@ impl Group { pub unsafe fn store_aligned(self, ptr: *mut u8) { // FIXME: use align_offset once it stabilizes debug_assert_eq!(ptr as usize & (mem::align_of::() - 1), 0); - ptr::write(ptr as *mut _, self.0); + ptr::write(ptr.cast(), self.0); } /// Returns a `BitMask` indicating all bytes in the group which *may* @@ -107,7 +106,7 @@ impl Group { #[inline] pub fn match_byte(self, byte: u8) -> BitMask { // This algorithm is derived from - // http://graphics.stanford.edu/~seander/bithacks.html##ValueInWord + // https://graphics.stanford.edu/~seander/bithacks.html##ValueInWord let cmp = self.0 ^ repeat(byte); BitMask((cmp.wrapping_sub(repeat(0x01)) & !cmp & repeat(0x80)).to_le()) } diff --git a/crux-mir/lib/hashbrown/src/raw/mod.rs b/crux-mir/lib/hashbrown/src/raw/mod.rs index f22a63d4e..211b818a5 100644 --- a/crux-mir/lib/hashbrown/src/raw/mod.rs +++ b/crux-mir/lib/hashbrown/src/raw/mod.rs @@ -1,13 +1,13 @@ -use crate::alloc::alloc::{alloc, dealloc, handle_alloc_error}; -use crate::scopeguard::guard; -use crate::CollectionAllocErr; -use core::alloc::Layout; -use core::hint; +use crate::alloc::alloc::{handle_alloc_error, Layout}; +use crate::scopeguard::{guard, ScopeGuard}; +use crate::TryReserveError; use core::iter::FusedIterator; use core::marker::PhantomData; use core::mem; use core::mem::ManuallyDrop; +use core::mem::MaybeUninit; use core::ptr::NonNull; +use core::{hint, ptr}; cfg_if! { // Use the SSE2 implementation if possible: it allows us to scan 16 buckets @@ -32,36 +32,47 @@ cfg_if! { } } +mod alloc; +pub(crate) use self::alloc::{do_alloc, Allocator, Global}; + mod bitmask; -use self::bitmask::BitMask; +use self::bitmask::{BitMask, BitMaskIter}; use self::imp::Group; // Branch prediction hint. This is currently only available on nightly but it // consistently improves performance by 10-15%. #[cfg(feature = "nightly")] use core::intrinsics::{likely, unlikely}; + +// On stable we can use #[cold] to get a equivalent effect: this attributes +// suggests that the function is unlikely to be called +#[cfg(not(feature = "nightly"))] +#[inline] +#[cold] +fn cold() {} + #[cfg(not(feature = "nightly"))] #[inline] fn likely(b: bool) -> bool { + if !b { + cold(); + } b } #[cfg(not(feature = "nightly"))] #[inline] fn unlikely(b: bool) -> bool { + if b { + cold(); + } b } -#[cfg(feature = "nightly")] -#[cfg_attr(feature = "inline-more", inline)] +#[inline] unsafe fn offset_from(to: *const T, from: *const T) -> usize { to.offset_from(from) as usize } -#[cfg(not(feature = "nightly"))] -#[cfg_attr(feature = "inline-more", inline)] -unsafe fn offset_from(to: *const T, from: *const T) -> usize { - (to as usize - from as usize) / mem::size_of::() -} /// Whether memory allocation errors should return an error or abort. #[derive(Copy, Clone)] @@ -73,18 +84,18 @@ enum Fallibility { impl Fallibility { /// Error to return on capacity overflow. #[cfg_attr(feature = "inline-more", inline)] - fn capacity_overflow(self) -> CollectionAllocErr { + fn capacity_overflow(self) -> TryReserveError { match self { - Fallibility::Fallible => CollectionAllocErr::CapacityOverflow, + Fallibility::Fallible => TryReserveError::CapacityOverflow, Fallibility::Infallible => panic!("Hash table capacity overflow"), } } /// Error to return on allocation error. #[cfg_attr(feature = "inline-more", inline)] - fn alloc_err(self, layout: Layout) -> CollectionAllocErr { + fn alloc_err(self, layout: Layout) -> TryReserveError { match self { - Fallibility::Fallible => CollectionAllocErr::AllocErr { layout }, + Fallibility::Fallible => TryReserveError::AllocError { layout }, Fallibility::Infallible => handle_alloc_error(layout), } } @@ -145,27 +156,22 @@ fn h2(hash: u64) -> u8 { /// Proof that the probe will visit every group in the table: /// struct ProbeSeq { - bucket_mask: usize, pos: usize, stride: usize, } -impl Iterator for ProbeSeq { - type Item = usize; - +impl ProbeSeq { #[inline] - fn next(&mut self) -> Option { + fn move_next(&mut self, bucket_mask: usize) { // We should have found an empty bucket by now and ended the probe. debug_assert!( - self.stride <= self.bucket_mask, + self.stride <= bucket_mask, "Went past end of probe sequence" ); - let result = self.pos; self.stride += Group::WIDTH; self.pos += self.stride; - self.pos &= self.bucket_mask; - Some(result) + self.pos &= bucket_mask; } } @@ -173,30 +179,36 @@ impl Iterator for ProbeSeq { /// taking the maximum load factor into account. /// /// Returns `None` if an overflow occurs. -#[cfg_attr(feature = "inline-more", inline)] // Workaround for emscripten bug emscripten-core/emscripten-fastcomp#258 #[cfg_attr(target_os = "emscripten", inline(never))] +#[cfg_attr(not(target_os = "emscripten"), inline)] fn capacity_to_buckets(cap: usize) -> Option { - let adjusted_cap = if cap < 8 { - // Need at least 1 free bucket on small tables - cap + 1 - } else { - // Otherwise require 1/8 buckets to be empty (87.5% load) - // - // Be careful when modifying this, calculate_layout relies on the - // overflow check here. - cap.checked_mul(8)? / 7 - }; + debug_assert_ne!(cap, 0); + + // For small tables we require at least 1 empty bucket so that lookups are + // guaranteed to terminate if an element doesn't exist in the table. + if cap < 8 { + // We don't bother with a table size of 2 buckets since that can only + // hold a single element. Instead we skip directly to a 4 bucket table + // which can hold 3 elements. + return Some(if cap < 4 { 4 } else { 8 }); + } + + // Otherwise require 1/8 buckets to be empty (87.5% load) + // + // Be careful when modifying this, calculate_layout relies on the + // overflow check here. + let adjusted_cap = cap.checked_mul(8)? / 7; // Any overflows will have been caught by the checked_mul. Also, any // rounding errors from the division above will be cleaned up by - // next_power_of_two (which can't overflow because of the previous divison). + // next_power_of_two (which can't overflow because of the previous division). Some(adjusted_cap.next_power_of_two()) } /// Returns the maximum effective capacity for the given bucket mask, taking /// the maximum load factor into account. -#[cfg_attr(feature = "inline-more", inline)] +#[inline] fn bucket_mask_to_capacity(bucket_mask: usize) -> usize { if bucket_mask < 8 { // For tables with 1/2/4/8 buckets, we always reserve one empty slot. @@ -208,47 +220,49 @@ fn bucket_mask_to_capacity(bucket_mask: usize) -> usize { } } -// Returns a Layout which describes the allocation required for a hash table, -// and the offset of the buckets in the allocation. -/// -/// Returns `None` if an overflow occurs. -#[cfg_attr(feature = "inline-more", inline)] -#[cfg(feature = "nightly")] -fn calculate_layout(buckets: usize) -> Option<(Layout, usize)> { - debug_assert!(buckets.is_power_of_two()); +/// Helper which allows the max calculation for ctrl_align to be statically computed for each T +/// while keeping the rest of `calculate_layout_for` independent of `T` +#[derive(Copy, Clone)] +struct TableLayout { + size: usize, + ctrl_align: usize, +} - // Array of buckets - let data = Layout::array::(buckets).ok()?; +impl TableLayout { + #[inline] + fn new() -> Self { + let layout = Layout::new::(); + Self { + size: layout.size(), + ctrl_align: usize::max(layout.align(), Group::WIDTH), + } + } - // Array of control bytes. This must be aligned to the group size. - // - // We add `Group::WIDTH` control bytes at the end of the array which - // replicate the bytes at the start of the array and thus avoids the need to - // perform bounds-checking while probing. - // - // There is no possible overflow here since buckets is a power of two and - // Group::WIDTH is a small number. - let ctrl = unsafe { Layout::from_size_align_unchecked(buckets + Group::WIDTH, Group::WIDTH) }; + #[inline] + fn calculate_layout_for(self, buckets: usize) -> Option<(Layout, usize)> { + debug_assert!(buckets.is_power_of_two()); + + let TableLayout { size, ctrl_align } = self; + // Manual layout calculation since Layout methods are not yet stable. + let ctrl_offset = + size.checked_mul(buckets)?.checked_add(ctrl_align - 1)? & !(ctrl_align - 1); + let len = ctrl_offset.checked_add(buckets + Group::WIDTH)?; - ctrl.extend(data).ok() + Some(( + unsafe { Layout::from_size_align_unchecked(len, ctrl_align) }, + ctrl_offset, + )) + } } -// Returns a Layout which describes the allocation required for a hash table, -// and the offset of the buckets in the allocation. +/// Returns a Layout which describes the allocation required for a hash table, +/// and the offset of the control bytes in the allocation. +/// (the offset is also one past last element of buckets) +/// +/// Returns `None` if an overflow occurs. #[cfg_attr(feature = "inline-more", inline)] -#[cfg(not(feature = "nightly"))] fn calculate_layout(buckets: usize) -> Option<(Layout, usize)> { - debug_assert!(buckets.is_power_of_two()); - - // Manual layout calculation since Layout methods are not yet stable. - let data_align = usize::max(mem::align_of::(), Group::WIDTH); - let data_offset = (buckets + Group::WIDTH).checked_add(data_align - 1)? & !(data_align - 1); - let len = data_offset.checked_add(mem::size_of::().checked_mul(buckets)?)?; - - Some(( - unsafe { Layout::from_size_align_unchecked(len, data_align) }, - data_offset, - )) + TableLayout::new::().calculate_layout_for(buckets) } /// A reference to a hash table bucket containing a `T`. @@ -257,8 +271,11 @@ fn calculate_layout(buckets: usize) -> Option<(Layout, usize)> { /// is a ZST, then we instead track the index of the element in the table so /// that `erase` works properly. pub struct Bucket { - // Using *const for variance - ptr: *const T, + // Actually it is pointer to next element than element itself + // this is needed to maintain pointer arithmetic invariants + // keeping direct pointer to element introduces difficulty. + // Using `NonNull` for variance and niche layout + ptr: NonNull, } // This Send impl is needed for rayon support. This is safe since Bucket is @@ -266,103 +283,146 @@ pub struct Bucket { unsafe impl Send for Bucket {} impl Clone for Bucket { - #[cfg_attr(feature = "inline-more", inline)] + #[inline] fn clone(&self) -> Self { Self { ptr: self.ptr } } } impl Bucket { - #[cfg_attr(feature = "inline-more", inline)] - unsafe fn from_base_index(base: *const T, index: usize) -> Self { + #[inline] + unsafe fn from_base_index(base: NonNull, index: usize) -> Self { let ptr = if mem::size_of::() == 0 { - index as *const T + // won't overflow because index must be less than length + (index + 1) as *mut T } else { - base.add(index) + base.as_ptr().sub(index) }; - Self { ptr } + Self { + ptr: NonNull::new_unchecked(ptr), + } } - #[cfg_attr(feature = "inline-more", inline)] - pub unsafe fn as_ptr(&self) -> *mut T { + #[inline] + unsafe fn to_base_index(&self, base: NonNull) -> usize { + if mem::size_of::() == 0 { + self.ptr.as_ptr() as usize - 1 + } else { + offset_from(base.as_ptr(), self.ptr.as_ptr()) + } + } + #[inline] + pub fn as_ptr(&self) -> *mut T { if mem::size_of::() == 0 { // Just return an arbitrary ZST pointer which is properly aligned mem::align_of::() as *mut T } else { - self.ptr as *mut T + unsafe { self.ptr.as_ptr().sub(1) } } } - #[cfg_attr(feature = "inline-more", inline)] - unsafe fn add(&self, offset: usize) -> Self { + #[inline] + unsafe fn next_n(&self, offset: usize) -> Self { let ptr = if mem::size_of::() == 0 { - (self.ptr as usize + offset) as *const T + (self.ptr.as_ptr() as usize + offset) as *mut T } else { - self.ptr.add(offset) + self.ptr.as_ptr().sub(offset) }; - Self { ptr } + Self { + ptr: NonNull::new_unchecked(ptr), + } } #[cfg_attr(feature = "inline-more", inline)] pub unsafe fn drop(&self) { self.as_ptr().drop_in_place(); } - #[cfg_attr(feature = "inline-more", inline)] + #[inline] pub unsafe fn read(&self) -> T { self.as_ptr().read() } - #[cfg_attr(feature = "inline-more", inline)] + #[inline] pub unsafe fn write(&self, val: T) { self.as_ptr().write(val); } - #[cfg_attr(feature = "inline-more", inline)] + #[inline] pub unsafe fn as_ref<'a>(&self) -> &'a T { &*self.as_ptr() } - #[cfg_attr(feature = "inline-more", inline)] + #[inline] pub unsafe fn as_mut<'a>(&self) -> &'a mut T { &mut *self.as_ptr() } - #[cfg_attr(feature = "inline-more", inline)] + #[cfg(feature = "raw")] + #[inline] pub unsafe fn copy_from_nonoverlapping(&self, other: &Self) { self.as_ptr().copy_from_nonoverlapping(other.as_ptr(), 1); } } /// A raw hash table with an unsafe API. -pub struct RawTable { +pub struct RawTable { + table: RawTableInner, + // Tell dropck that we own instances of T. + marker: PhantomData, +} + +/// Non-generic part of `RawTable` which allows functions to be instantiated only once regardless +/// of how many different key-value types are used. +struct RawTableInner { // Mask to get an index from a hash value. The value is one less than the // number of buckets in the table. bucket_mask: usize, - // Pointer to the array of control bytes + // [Padding], T1, T2, ..., Tlast, C1, C2, ... + // ^ points here ctrl: NonNull, - // Pointer to the array of buckets - data: NonNull, - // Number of elements that can be inserted before we need to grow the table growth_left: usize, // Number of elements in the table, only really used by len() items: usize, - // Tell dropck that we own instances of T. - marker: PhantomData, + alloc: A, } -impl RawTable { +impl RawTable { /// Creates a new empty hash table without allocating any memory. /// /// In effect this returns a table with exactly 1 bucket. However we can /// leave the data pointer dangling since that bucket is never written to /// due to our load factor forcing us to always have at least 1 free bucket. - #[cfg_attr(feature = "inline-more", inline)] - pub fn new() -> Self { + #[inline] + pub const fn new() -> Self { Self { - data: NonNull::dangling(), - // Be careful to cast the entire slice to a raw pointer. - ctrl: unsafe { NonNull::new_unchecked(Group::static_empty().as_ptr() as *mut u8) }, - bucket_mask: 0, - items: 0, - growth_left: 0, + table: RawTableInner::new_in(Global), + marker: PhantomData, + } + } + + /// Attempts to allocate a new hash table with at least enough capacity + /// for inserting the given number of elements without reallocating. + #[cfg(feature = "raw")] + pub fn try_with_capacity(capacity: usize) -> Result { + Self::try_with_capacity_in(capacity, Global) + } + + /// Allocates a new hash table with at least enough capacity for inserting + /// the given number of elements without reallocating. + pub fn with_capacity(capacity: usize) -> Self { + Self::with_capacity_in(capacity, Global) + } +} + +impl RawTable { + /// Creates a new empty hash table without allocating any memory, using the + /// given allocator. + /// + /// In effect this returns a table with exactly 1 bucket. However we can + /// leave the data pointer dangling since that bucket is never written to + /// due to our load factor forcing us to always have at least 1 free bucket. + #[inline] + pub fn new_in(alloc: A) -> Self { + Self { + table: RawTableInner::new_in(alloc), marker: PhantomData, } } @@ -372,214 +432,170 @@ impl RawTable { /// The control bytes are left uninitialized. #[cfg_attr(feature = "inline-more", inline)] unsafe fn new_uninitialized( + alloc: A, buckets: usize, - fallability: Fallibility, - ) -> Result { + fallibility: Fallibility, + ) -> Result { debug_assert!(buckets.is_power_of_two()); - let (layout, data_offset) = - calculate_layout::(buckets).ok_or_else(|| fallability.capacity_overflow())?; - let ctrl = NonNull::new(alloc(layout)).ok_or_else(|| fallability.alloc_err(layout))?; - let data = NonNull::new_unchecked(ctrl.as_ptr().add(data_offset) as *mut T); + Ok(Self { - data, - ctrl, - bucket_mask: buckets - 1, - items: 0, - growth_left: bucket_mask_to_capacity(buckets - 1), + table: RawTableInner::new_uninitialized( + alloc, + TableLayout::new::(), + buckets, + fallibility, + )?, marker: PhantomData, }) } /// Attempts to allocate a new hash table with at least enough capacity /// for inserting the given number of elements without reallocating. - fn try_with_capacity( + fn fallible_with_capacity( + alloc: A, capacity: usize, - fallability: Fallibility, - ) -> Result { - if capacity == 0 { - Ok(Self::new()) - } else { - unsafe { - let buckets = - capacity_to_buckets(capacity).ok_or_else(|| fallability.capacity_overflow())?; - let result = Self::new_uninitialized(buckets, fallability)?; - result.ctrl(0).write_bytes(EMPTY, result.num_ctrl_bytes()); + fallibility: Fallibility, + ) -> Result { + Ok(Self { + table: RawTableInner::fallible_with_capacity( + alloc, + TableLayout::new::(), + capacity, + fallibility, + )?, + marker: PhantomData, + }) + } - Ok(result) - } + /// Attempts to allocate a new hash table using the given allocator, with at least enough + /// capacity for inserting the given number of elements without reallocating. + #[cfg(feature = "raw")] + pub fn try_with_capacity_in(capacity: usize, alloc: A) -> Result { + Self::fallible_with_capacity(alloc, capacity, Fallibility::Fallible) + } + + /// Allocates a new hash table using the given allocator, with at least enough capacity for + /// inserting the given number of elements without reallocating. + pub fn with_capacity_in(capacity: usize, alloc: A) -> Self { + // Avoid `Result::unwrap_or_else` because it bloats LLVM IR. + match Self::fallible_with_capacity(alloc, capacity, Fallibility::Infallible) { + Ok(capacity) => capacity, + Err(_) => unsafe { hint::unreachable_unchecked() }, } } - /// Allocates a new hash table with at least enough capacity for inserting - /// the given number of elements without reallocating. - pub fn with_capacity(capacity: usize) -> Self { - Self::try_with_capacity(capacity, Fallibility::Infallible) - .unwrap_or_else(|_| unsafe { hint::unreachable_unchecked() }) + /// Returns a reference to the underlying allocator. + #[inline] + pub fn allocator(&self) -> &A { + &self.table.alloc } /// Deallocates the table without dropping any entries. #[cfg_attr(feature = "inline-more", inline)] unsafe fn free_buckets(&mut self) { - let (layout, _) = - calculate_layout::(self.buckets()).unwrap_or_else(|| hint::unreachable_unchecked()); - dealloc(self.ctrl.as_ptr(), layout); + self.table.free_buckets(TableLayout::new::()); } - /// Returns the index of a bucket from a `Bucket`. - #[cfg_attr(feature = "inline-more", inline)] - pub unsafe fn bucket_index(&self, bucket: &Bucket) -> usize { - if mem::size_of::() == 0 { - bucket.ptr as usize - } else { - offset_from(bucket.ptr, self.data.as_ptr()) - } + /// Returns pointer to one past last element of data table. + #[inline] + pub unsafe fn data_end(&self) -> NonNull { + NonNull::new_unchecked(self.table.ctrl.as_ptr().cast()) } - /// Returns a pointer to a control byte. - #[cfg_attr(feature = "inline-more", inline)] - unsafe fn ctrl(&self, index: usize) -> *mut u8 { - debug_assert!(index < self.num_ctrl_bytes()); - self.ctrl.as_ptr().add(index) + /// Returns pointer to start of data table. + #[inline] + #[cfg(feature = "nightly")] + pub unsafe fn data_start(&self) -> *mut T { + self.data_end().as_ptr().wrapping_sub(self.buckets()) + } + + /// Returns the index of a bucket from a `Bucket`. + #[inline] + pub unsafe fn bucket_index(&self, bucket: &Bucket) -> usize { + bucket.to_base_index(self.data_end()) } /// Returns a pointer to an element in the table. - #[cfg_attr(feature = "inline-more", inline)] + #[inline] pub unsafe fn bucket(&self, index: usize) -> Bucket { - debug_assert_ne!(self.bucket_mask, 0); + debug_assert_ne!(self.table.bucket_mask, 0); debug_assert!(index < self.buckets()); - Bucket::from_base_index(self.data.as_ptr(), index) + Bucket::from_base_index(self.data_end(), index) } /// Erases an element from the table without dropping it. #[cfg_attr(feature = "inline-more", inline)] + #[deprecated(since = "0.8.1", note = "use erase or remove instead")] pub unsafe fn erase_no_drop(&mut self, item: &Bucket) { let index = self.bucket_index(item); - debug_assert!(is_full(*self.ctrl(index))); - let index_before = index.wrapping_sub(Group::WIDTH) & self.bucket_mask; - let empty_before = Group::load(self.ctrl(index_before)).match_empty(); - let empty_after = Group::load(self.ctrl(index)).match_empty(); + self.table.erase(index); + } - // If we are inside a continuous block of Group::WIDTH full or deleted - // cells then a probe window may have seen a full block when trying to - // insert. We therefore need to keep that block non-empty so that - // lookups will continue searching to the next probe window. - // - // Note that in this context `leading_zeros` refers to the bytes at the - // end of a group, while `trailing_zeros` refers to the bytes at the - // begining of a group. - let ctrl = if empty_before.leading_zeros() + empty_after.trailing_zeros() >= Group::WIDTH { - DELETED - } else { - self.growth_left += 1; - EMPTY - }; - self.set_ctrl(index, ctrl); - self.items -= 1; + /// Erases an element from the table, dropping it in place. + #[cfg_attr(feature = "inline-more", inline)] + #[allow(clippy::needless_pass_by_value)] + #[allow(deprecated)] + pub unsafe fn erase(&mut self, item: Bucket) { + // Erase the element from the table first since drop might panic. + self.erase_no_drop(&item); + item.drop(); } - /// Returns an iterator for a probe sequence on the table. - /// - /// This iterator never terminates, but is guaranteed to visit each bucket - /// group exactly once. The loop using `probe_seq` must terminate upon - /// reaching a group containing an empty bucket. + /// Finds and erases an element from the table, dropping it in place. + /// Returns true if an element was found. + #[cfg(feature = "raw")] #[cfg_attr(feature = "inline-more", inline)] - fn probe_seq(&self, hash: u64) -> ProbeSeq { - ProbeSeq { - bucket_mask: self.bucket_mask, - pos: h1(hash) & self.bucket_mask, - stride: 0, + pub fn erase_entry(&mut self, hash: u64, eq: impl FnMut(&T) -> bool) -> bool { + // Avoid `Option::map` because it bloats LLVM IR. + if let Some(bucket) = self.find(hash, eq) { + unsafe { + self.erase(bucket); + } + true + } else { + false } } - /// Sets a control byte, and possibly also the replicated control byte at - /// the end of the array. + /// Removes an element from the table, returning it. #[cfg_attr(feature = "inline-more", inline)] - unsafe fn set_ctrl(&self, index: usize, ctrl: u8) { - // Replicate the first Group::WIDTH control bytes at the end of - // the array without using a branch: - // - If index >= Group::WIDTH then index == index2. - // - Otherwise index2 == self.bucket_mask + 1 + index. - // - // The very last replicated control byte is never actually read because - // we mask the initial index for unaligned loads, but we write it - // anyways because it makes the set_ctrl implementation simpler. - // - // If there are fewer buckets than Group::WIDTH then this code will - // replicate the buckets at the end of the trailing group. For example - // with 2 buckets and a group size of 4, the control bytes will look - // like this: - // - // Real | Replicated - // --------------------------------------------- - // | [A] | [B] | [EMPTY] | [EMPTY] | [A] | [B] | - // --------------------------------------------- - let index2 = ((index.wrapping_sub(Group::WIDTH)) & self.bucket_mask) + Group::WIDTH; - - *self.ctrl(index) = ctrl; - *self.ctrl(index2) = ctrl; + #[allow(clippy::needless_pass_by_value)] + #[allow(deprecated)] + pub unsafe fn remove(&mut self, item: Bucket) -> T { + self.erase_no_drop(&item); + item.read() } - /// Searches for an empty or deleted bucket which is suitable for inserting - /// a new element. - /// - /// There must be at least 1 empty bucket in the table. + /// Finds and removes an element from the table, returning it. #[cfg_attr(feature = "inline-more", inline)] - fn find_insert_slot(&self, hash: u64) -> usize { - for pos in self.probe_seq(hash) { - unsafe { - let group = Group::load(self.ctrl(pos)); - if let Some(bit) = group.match_empty_or_deleted().lowest_set_bit() { - let result = (pos + bit) & self.bucket_mask; - - // In tables smaller than the group width, trailing control - // bytes outside the range of the table are filled with - // EMPTY entries. These will unfortunately trigger a - // match, but once masked may point to a full bucket that - // is already occupied. We detect this situation here and - // perform a second scan starting at the begining of the - // table. This second scan is guaranteed to find an empty - // slot (due to the load factor) before hitting the trailing - // control bytes (containing EMPTY). - if unlikely(is_full(*self.ctrl(result))) { - debug_assert!(self.bucket_mask < Group::WIDTH); - debug_assert_ne!(pos, 0); - return Group::load_aligned(self.ctrl(0)) - .match_empty_or_deleted() - .lowest_set_bit_nonzero(); - } else { - return result; - } - } - } + pub fn remove_entry(&mut self, hash: u64, eq: impl FnMut(&T) -> bool) -> Option { + // Avoid `Option::map` because it bloats LLVM IR. + match self.find(hash, eq) { + Some(bucket) => Some(unsafe { self.remove(bucket) }), + None => None, } - - // probe_seq never returns. - unreachable!(); } /// Marks all table buckets as empty without dropping their contents. #[cfg_attr(feature = "inline-more", inline)] pub fn clear_no_drop(&mut self) { - if !self.is_empty_singleton() { - unsafe { - self.ctrl(0).write_bytes(EMPTY, self.num_ctrl_bytes()); - } - } - self.items = 0; - self.growth_left = bucket_mask_to_capacity(self.bucket_mask); + self.table.clear_no_drop(); } /// Removes all elements from the table without freeing the backing memory. #[cfg_attr(feature = "inline-more", inline)] pub fn clear(&mut self) { // Ensure that the table is reset even if one of the drops panic - let self_ = guard(self, |self_| self_.clear_no_drop()); + let mut self_ = guard(self, |self_| self_.clear_no_drop()); + unsafe { + self_.drop_elements(); + } + } - if mem::needs_drop::() { - unsafe { - for item in self_.iter() { - item.drop(); - } + unsafe fn drop_elements(&mut self) { + if mem::needs_drop::() && !self.is_empty() { + for item in self.iter() { + item.drop(); } } } @@ -589,9 +605,9 @@ impl RawTable { pub fn shrink_to(&mut self, min_size: usize, hasher: impl Fn(&T) -> u64) { // Calculate the minimal number of elements that we need to reserve // space for. - let min_size = usize::max(self.items, min_size); + let min_size = usize::max(self.table.items, min_size); if min_size == 0 { - *self = Self::new(); + *self = Self::new_in(self.table.alloc.clone()); return; } @@ -607,11 +623,16 @@ impl RawTable { // If we have more buckets than we need, shrink the table. if min_buckets < self.buckets() { // Fast path if the table is empty - if self.items == 0 { - *self = Self::with_capacity(min_size) + if self.table.items == 0 { + *self = Self::with_capacity_in(min_size, self.table.alloc.clone()); } else { - self.resize(min_size, hasher, Fallibility::Infallible) - .unwrap_or_else(|_| unsafe { hint::unreachable_unchecked() }); + // Avoid `Result::unwrap_or_else` because it bloats LLVM IR. + if self + .resize(min_size, hasher, Fallibility::Infallible) + .is_err() + { + unsafe { hint::unreachable_unchecked() } + } } } } @@ -620,9 +641,14 @@ impl RawTable { /// without reallocation. #[cfg_attr(feature = "inline-more", inline)] pub fn reserve(&mut self, additional: usize, hasher: impl Fn(&T) -> u64) { - if additional > self.growth_left { - self.reserve_rehash(additional, hasher, Fallibility::Infallible) - .unwrap_or_else(|_| unsafe { hint::unreachable_unchecked() }); + if additional > self.table.growth_left { + // Avoid `Result::unwrap_or_else` because it bloats LLVM IR. + if self + .reserve_rehash(additional, hasher, Fallibility::Infallible) + .is_err() + { + unsafe { hint::unreachable_unchecked() } + } } } @@ -633,8 +659,8 @@ impl RawTable { &mut self, additional: usize, hasher: impl Fn(&T) -> u64, - ) -> Result<(), CollectionAllocErr> { - if additional > self.growth_left { + ) -> Result<(), TryReserveError> { + if additional > self.table.growth_left { self.reserve_rehash(additional, hasher, Fallibility::Fallible) } else { Ok(()) @@ -648,126 +674,20 @@ impl RawTable { &mut self, additional: usize, hasher: impl Fn(&T) -> u64, - fallability: Fallibility, - ) -> Result<(), CollectionAllocErr> { - let new_items = self - .items - .checked_add(additional) - .ok_or_else(|| fallability.capacity_overflow())?; - - let full_capacity = bucket_mask_to_capacity(self.bucket_mask); - if new_items <= full_capacity / 2 { - // Rehash in-place without re-allocating if we have plenty of spare - // capacity that is locked up due to DELETED entries. - self.rehash_in_place(hasher); - Ok(()) - } else { - // Otherwise, conservatively resize to at least the next size up - // to avoid churning deletes into frequent rehashes. - self.resize( - usize::max(new_items, full_capacity + 1), - hasher, - fallability, - ) - } - } - - /// Rehashes the contents of the table in place (i.e. without changing the - /// allocation). - /// - /// If `hasher` panics then some the table's contents may be lost. - fn rehash_in_place(&mut self, hasher: impl Fn(&T) -> u64) { + fallibility: Fallibility, + ) -> Result<(), TryReserveError> { unsafe { - // Bulk convert all full control bytes to DELETED, and all DELETED - // control bytes to EMPTY. This effectively frees up all buckets - // containing a DELETED entry. - for i in (0..self.buckets()).step_by(Group::WIDTH) { - let group = Group::load_aligned(self.ctrl(i)); - let group = group.convert_special_to_empty_and_full_to_deleted(); - group.store_aligned(self.ctrl(i)); - } - - // Fix up the trailing control bytes. See the comments in set_ctrl - // for the handling of tables smaller than the group width. - if self.buckets() < Group::WIDTH { - self.ctrl(0) - .copy_to(self.ctrl(Group::WIDTH), self.buckets()); - } else { - self.ctrl(0) - .copy_to(self.ctrl(self.buckets()), Group::WIDTH); - } - - // If the hash function panics then properly clean up any elements - // that we haven't rehashed yet. We unfortunately can't preserve the - // element since we lost their hash and have no way of recovering it - // without risking another panic. - let mut guard = guard(self, |self_| { + self.table.reserve_rehash_inner( + additional, + &|table, index| hasher(table.bucket::(index).as_ref()), + fallibility, + TableLayout::new::(), if mem::needs_drop::() { - for i in 0..self_.buckets() { - if *self_.ctrl(i) == DELETED { - self_.set_ctrl(i, EMPTY); - self_.bucket(i).drop(); - self_.items -= 1; - } - } - } - self_.growth_left = bucket_mask_to_capacity(self_.bucket_mask) - self_.items; - }); - - // At this point, DELETED elements are elements that we haven't - // rehashed yet. Find them and re-insert them at their ideal - // position. - 'outer: for i in 0..guard.buckets() { - if *guard.ctrl(i) != DELETED { - continue; - } - 'inner: loop { - // Hash the current item - let item = guard.bucket(i); - let hash = hasher(item.as_ref()); - - // Search for a suitable place to put it - let new_i = guard.find_insert_slot(hash); - - // Probing works by scanning through all of the control - // bytes in groups, which may not be aligned to the group - // size. If both the new and old position fall within the - // same unaligned group, then there is no benefit in moving - // it and we can just continue to the next item. - let probe_index = |pos: usize| { - (pos.wrapping_sub(guard.probe_seq(hash).pos) & guard.bucket_mask) - / Group::WIDTH - }; - if likely(probe_index(i) == probe_index(new_i)) { - guard.set_ctrl(i, h2(hash)); - continue 'outer; - } - - // We are moving the current item to a new position. Write - // our H2 to the control byte of the new position. - let prev_ctrl = *guard.ctrl(new_i); - guard.set_ctrl(new_i, h2(hash)); - - if prev_ctrl == EMPTY { - // If the target slot is empty, simply move the current - // element into the new slot and clear the old control - // byte. - guard.set_ctrl(i, EMPTY); - guard.bucket(new_i).copy_from_nonoverlapping(&item); - continue 'outer; - } else { - // If the target slot is occupied, swap the two elements - // and then continue processing the element that we just - // swapped into the old slot. - debug_assert_eq!(prev_ctrl, DELETED); - mem::swap(guard.bucket(new_i).as_mut(), item.as_mut()); - continue 'inner; - } - } - } - - guard.growth_left = bucket_mask_to_capacity(guard.bucket_mask) - guard.items; - mem::forget(guard); + Some(mem::transmute(ptr::drop_in_place:: as unsafe fn(*mut T))) + } else { + None + }, + ) } } @@ -777,255 +697,978 @@ impl RawTable { &mut self, capacity: usize, hasher: impl Fn(&T) -> u64, - fallability: Fallibility, - ) -> Result<(), CollectionAllocErr> { + fallibility: Fallibility, + ) -> Result<(), TryReserveError> { unsafe { - debug_assert!(self.items <= capacity); - - // Allocate and initialize the new table. - let mut new_table = Self::try_with_capacity(capacity, fallability)?; - new_table.growth_left -= self.items; - new_table.items = self.items; - - // The hash function may panic, in which case we simply free the new - // table without dropping any elements that may have been copied into - // it. - // - // This guard is also used to free the old table on success, see - // the comment at the bottom of this function. - let mut new_table = guard(ManuallyDrop::new(new_table), |new_table| { - if !new_table.is_empty_singleton() { - new_table.free_buckets(); - } - }); - - // Copy all elements to the new table. - for item in self.iter() { - // This may panic. - let hash = hasher(item.as_ref()); - - // We can use a simpler version of insert() here since: - // - there are no DELETED entries. - // - we know there is enough space in the table. - // - all elements are unique. - let index = new_table.find_insert_slot(hash); - new_table.set_ctrl(index, h2(hash)); - new_table.bucket(index).copy_from_nonoverlapping(&item); - } - - // We successfully copied all elements without panicking. Now replace - // self with the new table. The old table will have its memory freed but - // the items will not be dropped (since they have been moved into the - // new table). - mem::swap(self, &mut new_table); - - Ok(()) + self.table.resize_inner( + capacity, + &|table, index| hasher(table.bucket::(index).as_ref()), + fallibility, + TableLayout::new::(), + ) } } - /// Inserts a new element into the table. + /// Inserts a new element into the table, and returns its raw bucket. /// /// This does not check if the given element already exists in the table. #[cfg_attr(feature = "inline-more", inline)] pub fn insert(&mut self, hash: u64, value: T, hasher: impl Fn(&T) -> u64) -> Bucket { unsafe { - let mut index = self.find_insert_slot(hash); + let mut index = self.table.find_insert_slot(hash); // We can avoid growing the table once we have reached our load // factor if we are replacing a tombstone. This works since the // number of EMPTY slots does not change in this case. - let old_ctrl = *self.ctrl(index); - if unlikely(self.growth_left == 0 && special_is_empty(old_ctrl)) { + let old_ctrl = *self.table.ctrl(index); + if unlikely(self.table.growth_left == 0 && special_is_empty(old_ctrl)) { self.reserve(1, hasher); - index = self.find_insert_slot(hash); + index = self.table.find_insert_slot(hash); } + self.table.record_item_insert_at(index, old_ctrl, hash); + let bucket = self.bucket(index); - self.growth_left -= special_is_empty(old_ctrl) as usize; - self.set_ctrl(index, h2(hash)); bucket.write(value); - self.items += 1; bucket } } + /// Attempts to insert a new element without growing the table and return its raw bucket. + /// + /// Returns an `Err` containing the given element if inserting it would require growing the + /// table. + /// + /// This does not check if the given element already exists in the table. + #[cfg(feature = "raw")] + #[cfg_attr(feature = "inline-more", inline)] + pub fn try_insert_no_grow(&mut self, hash: u64, value: T) -> Result, T> { + unsafe { + match self.table.prepare_insert_no_grow(hash) { + Ok(index) => { + let bucket = self.bucket(index); + bucket.write(value); + Ok(bucket) + } + Err(()) => Err(value), + } + } + } + + /// Inserts a new element into the table, and returns a mutable reference to it. + /// + /// This does not check if the given element already exists in the table. + #[cfg_attr(feature = "inline-more", inline)] + pub fn insert_entry(&mut self, hash: u64, value: T, hasher: impl Fn(&T) -> u64) -> &mut T { + unsafe { self.insert(hash, value, hasher).as_mut() } + } + /// Inserts a new element into the table, without growing the table. /// /// There must be enough space in the table to insert the new element. /// /// This does not check if the given element already exists in the table. #[cfg_attr(feature = "inline-more", inline)] - #[cfg(feature = "rustc-internal-api")] - pub fn insert_no_grow(&mut self, hash: u64, value: T) -> Bucket { - unsafe { - let index = self.find_insert_slot(hash); - let bucket = self.bucket(index); + #[cfg(any(feature = "raw", feature = "rustc-internal-api"))] + pub unsafe fn insert_no_grow(&mut self, hash: u64, value: T) -> Bucket { + let (index, old_ctrl) = self.table.prepare_insert_slot(hash); + let bucket = self.table.bucket(index); - // If we are replacing a DELETED entry then we don't need to update - // the load counter. - let old_ctrl = *self.ctrl(index); - self.growth_left -= special_is_empty(old_ctrl) as usize; + // If we are replacing a DELETED entry then we don't need to update + // the load counter. + self.table.growth_left -= special_is_empty(old_ctrl) as usize; - self.set_ctrl(index, h2(hash)); - bucket.write(value); - self.items += 1; - bucket + bucket.write(value); + self.table.items += 1; + bucket + } + + /// Temporary removes a bucket, applying the given function to the removed + /// element and optionally put back the returned value in the same bucket. + /// + /// Returns `true` if the bucket still contains an element + /// + /// This does not check if the given bucket is actually occupied. + #[cfg_attr(feature = "inline-more", inline)] + pub unsafe fn replace_bucket_with(&mut self, bucket: Bucket, f: F) -> bool + where + F: FnOnce(T) -> Option, + { + let index = self.bucket_index(&bucket); + let old_ctrl = *self.table.ctrl(index); + debug_assert!(is_full(old_ctrl)); + let old_growth_left = self.table.growth_left; + let item = self.remove(bucket); + if let Some(new_item) = f(item) { + self.table.growth_left = old_growth_left; + self.table.set_ctrl(index, old_ctrl); + self.table.items += 1; + self.bucket(index).write(new_item); + true + } else { + false } } /// Searches for an element in the table. #[inline] pub fn find(&self, hash: u64, mut eq: impl FnMut(&T) -> bool) -> Option> { + let result = self.table.find_inner(hash, &mut |index| unsafe { + eq(self.bucket(index).as_ref()) + }); + + // Avoid `Option::map` because it bloats LLVM IR. + match result { + Some(index) => Some(unsafe { self.bucket(index) }), + None => None, + } + } + + /// Gets a reference to an element in the table. + #[inline] + pub fn get(&self, hash: u64, eq: impl FnMut(&T) -> bool) -> Option<&T> { + // Avoid `Option::map` because it bloats LLVM IR. + match self.find(hash, eq) { + Some(bucket) => Some(unsafe { bucket.as_ref() }), + None => None, + } + } + + /// Gets a mutable reference to an element in the table. + #[inline] + pub fn get_mut(&mut self, hash: u64, eq: impl FnMut(&T) -> bool) -> Option<&mut T> { + // Avoid `Option::map` because it bloats LLVM IR. + match self.find(hash, eq) { + Some(bucket) => Some(unsafe { bucket.as_mut() }), + None => None, + } + } + + /// Attempts to get mutable references to `N` entries in the table at once. + /// + /// Returns an array of length `N` with the results of each query. + /// + /// At most one mutable reference will be returned to any entry. `None` will be returned if any + /// of the hashes are duplicates. `None` will be returned if the hash is not found. + /// + /// The `eq` argument should be a closure such that `eq(i, k)` returns true if `k` is equal to + /// the `i`th key to be looked up. + pub fn get_many_mut( + &mut self, + hashes: [u64; N], + eq: impl FnMut(usize, &T) -> bool, + ) -> Option<[&'_ mut T; N]> { unsafe { - for pos in self.probe_seq(hash) { - let group = Group::load(self.ctrl(pos)); - for bit in group.match_byte(h2(hash)) { - let index = (pos + bit) & self.bucket_mask; - let bucket = self.bucket(index); - if likely(eq(bucket.as_ref())) { - return Some(bucket); + let ptrs = self.get_many_mut_pointers(hashes, eq)?; + + for (i, &cur) in ptrs.iter().enumerate() { + if ptrs[..i].iter().any(|&prev| ptr::eq::(prev, cur)) { + return None; + } + } + // All bucket are distinct from all previous buckets so we're clear to return the result + // of the lookup. + + // TODO use `MaybeUninit::array_assume_init` here instead once that's stable. + Some(mem::transmute_copy(&ptrs)) + } + } + + pub unsafe fn get_many_unchecked_mut( + &mut self, + hashes: [u64; N], + eq: impl FnMut(usize, &T) -> bool, + ) -> Option<[&'_ mut T; N]> { + let ptrs = self.get_many_mut_pointers(hashes, eq)?; + Some(mem::transmute_copy(&ptrs)) + } + + unsafe fn get_many_mut_pointers( + &mut self, + hashes: [u64; N], + mut eq: impl FnMut(usize, &T) -> bool, + ) -> Option<[*mut T; N]> { + // TODO use `MaybeUninit::uninit_array` here instead once that's stable. + let mut outs: MaybeUninit<[*mut T; N]> = MaybeUninit::uninit(); + let outs_ptr = outs.as_mut_ptr(); + + for (i, &hash) in hashes.iter().enumerate() { + let cur = self.find(hash, |k| eq(i, k))?; + *(*outs_ptr).get_unchecked_mut(i) = cur.as_mut(); + } + + // TODO use `MaybeUninit::array_assume_init` here instead once that's stable. + Some(outs.assume_init()) + } + + /// Returns the number of elements the map can hold without reallocating. + /// + /// This number is a lower bound; the table might be able to hold + /// more, but is guaranteed to be able to hold at least this many. + #[inline] + pub fn capacity(&self) -> usize { + self.table.items + self.table.growth_left + } + + /// Returns the number of elements in the table. + #[inline] + pub fn len(&self) -> usize { + self.table.items + } + + /// Returns `true` if the table contains no elements. + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns the number of buckets in the table. + #[inline] + pub fn buckets(&self) -> usize { + self.table.bucket_mask + 1 + } + + /// Returns an iterator over every element in the table. It is up to + /// the caller to ensure that the `RawTable` outlives the `RawIter`. + /// Because we cannot make the `next` method unsafe on the `RawIter` + /// struct, we have to make the `iter` method unsafe. + #[inline] + pub unsafe fn iter(&self) -> RawIter { + let data = Bucket::from_base_index(self.data_end(), 0); + RawIter { + iter: RawIterRange::new(self.table.ctrl.as_ptr(), data, self.table.buckets()), + items: self.table.items, + } + } + + /// Returns an iterator over occupied buckets that could match a given hash. + /// + /// `RawTable` only stores 7 bits of the hash value, so this iterator may + /// return items that have a hash value different than the one provided. You + /// should always validate the returned values before using them. + /// + /// It is up to the caller to ensure that the `RawTable` outlives the + /// `RawIterHash`. Because we cannot make the `next` method unsafe on the + /// `RawIterHash` struct, we have to make the `iter_hash` method unsafe. + #[cfg_attr(feature = "inline-more", inline)] + #[cfg(feature = "raw")] + pub unsafe fn iter_hash(&self, hash: u64) -> RawIterHash<'_, T, A> { + RawIterHash::new(self, hash) + } + + /// Returns an iterator which removes all elements from the table without + /// freeing the memory. + #[cfg_attr(feature = "inline-more", inline)] + pub fn drain(&mut self) -> RawDrain<'_, T, A> { + unsafe { + let iter = self.iter(); + self.drain_iter_from(iter) + } + } + + /// Returns an iterator which removes all elements from the table without + /// freeing the memory. + /// + /// Iteration starts at the provided iterator's current location. + /// + /// It is up to the caller to ensure that the iterator is valid for this + /// `RawTable` and covers all items that remain in the table. + #[cfg_attr(feature = "inline-more", inline)] + pub unsafe fn drain_iter_from(&mut self, iter: RawIter) -> RawDrain<'_, T, A> { + debug_assert_eq!(iter.len(), self.len()); + RawDrain { + iter, + table: ManuallyDrop::new(mem::replace(self, Self::new_in(self.table.alloc.clone()))), + orig_table: NonNull::from(self), + marker: PhantomData, + } + } + + /// Returns an iterator which consumes all elements from the table. + /// + /// Iteration starts at the provided iterator's current location. + /// + /// It is up to the caller to ensure that the iterator is valid for this + /// `RawTable` and covers all items that remain in the table. + pub unsafe fn into_iter_from(self, iter: RawIter) -> RawIntoIter { + debug_assert_eq!(iter.len(), self.len()); + + let alloc = self.table.alloc.clone(); + let allocation = self.into_allocation(); + RawIntoIter { + iter, + allocation, + marker: PhantomData, + alloc, + } + } + + /// Converts the table into a raw allocation. The contents of the table + /// should be dropped using a `RawIter` before freeing the allocation. + #[cfg_attr(feature = "inline-more", inline)] + pub(crate) fn into_allocation(self) -> Option<(NonNull, Layout)> { + let alloc = if self.table.is_empty_singleton() { + None + } else { + // Avoid `Option::unwrap_or_else` because it bloats LLVM IR. + let (layout, ctrl_offset) = match calculate_layout::(self.table.buckets()) { + Some(lco) => lco, + None => unsafe { hint::unreachable_unchecked() }, + }; + Some(( + unsafe { NonNull::new_unchecked(self.table.ctrl.as_ptr().sub(ctrl_offset)) }, + layout, + )) + }; + mem::forget(self); + alloc + } +} + +unsafe impl Send for RawTable +where + T: Send, + A: Send, +{ +} +unsafe impl Sync for RawTable +where + T: Sync, + A: Sync, +{ +} + +impl RawTableInner { + #[inline] + const fn new_in(alloc: A) -> Self { + Self { + // Be careful to cast the entire slice to a raw pointer. + ctrl: unsafe { NonNull::new_unchecked(Group::static_empty() as *const _ as *mut u8) }, + bucket_mask: 0, + items: 0, + growth_left: 0, + alloc, + } + } +} + +impl RawTableInner { + #[cfg_attr(feature = "inline-more", inline)] + unsafe fn new_uninitialized( + alloc: A, + table_layout: TableLayout, + buckets: usize, + fallibility: Fallibility, + ) -> Result { + debug_assert!(buckets.is_power_of_two()); + + // Avoid `Option::ok_or_else` because it bloats LLVM IR. + let (layout, ctrl_offset) = match table_layout.calculate_layout_for(buckets) { + Some(lco) => lco, + None => return Err(fallibility.capacity_overflow()), + }; + + // We need an additional check to ensure that the allocation doesn't + // exceed `isize::MAX`. We can skip this check on 64-bit systems since + // such allocations will never succeed anyways. + // + // This mirrors what Vec does in the standard library. + if mem::size_of::() < 8 && layout.size() > isize::MAX as usize { + return Err(fallibility.capacity_overflow()); + } + + let ptr: NonNull = match do_alloc(&alloc, layout) { + Ok(block) => block.cast(), + Err(_) => return Err(fallibility.alloc_err(layout)), + }; + + let ctrl = NonNull::new_unchecked(ptr.as_ptr().add(ctrl_offset)); + Ok(Self { + ctrl, + bucket_mask: buckets - 1, + items: 0, + growth_left: bucket_mask_to_capacity(buckets - 1), + alloc, + }) + } + + #[inline] + fn fallible_with_capacity( + alloc: A, + table_layout: TableLayout, + capacity: usize, + fallibility: Fallibility, + ) -> Result { + if capacity == 0 { + Ok(Self::new_in(alloc)) + } else { + unsafe { + let buckets = + capacity_to_buckets(capacity).ok_or_else(|| fallibility.capacity_overflow())?; + + let result = Self::new_uninitialized(alloc, table_layout, buckets, fallibility)?; + result.ctrl(0).write_bytes(EMPTY, result.num_ctrl_bytes()); + + Ok(result) + } + } + } + + /// Searches for an empty or deleted bucket which is suitable for inserting + /// a new element and sets the hash for that slot. + /// + /// There must be at least 1 empty bucket in the table. + #[inline] + unsafe fn prepare_insert_slot(&self, hash: u64) -> (usize, u8) { + let index = self.find_insert_slot(hash); + let old_ctrl = *self.ctrl(index); + self.set_ctrl_h2(index, hash); + (index, old_ctrl) + } + + /// Searches for an empty or deleted bucket which is suitable for inserting + /// a new element. + /// + /// There must be at least 1 empty bucket in the table. + #[inline] + fn find_insert_slot(&self, hash: u64) -> usize { + let mut probe_seq = self.probe_seq(hash); + loop { + unsafe { + let group = Group::load(self.ctrl(probe_seq.pos)); + if let Some(bit) = group.match_empty_or_deleted().lowest_set_bit() { + let result = (probe_seq.pos + bit) & self.bucket_mask; + + // In tables smaller than the group width, trailing control + // bytes outside the range of the table are filled with + // EMPTY entries. These will unfortunately trigger a + // match, but once masked may point to a full bucket that + // is already occupied. We detect this situation here and + // perform a second scan starting at the beginning of the + // table. This second scan is guaranteed to find an empty + // slot (due to the load factor) before hitting the trailing + // control bytes (containing EMPTY). + if unlikely(is_full(*self.ctrl(result))) { + debug_assert!(self.bucket_mask < Group::WIDTH); + debug_assert_ne!(probe_seq.pos, 0); + return Group::load_aligned(self.ctrl(0)) + .match_empty_or_deleted() + .lowest_set_bit_nonzero(); } + + return result; } - if likely(group.match_empty().any_bit_set()) { - return None; + } + probe_seq.move_next(self.bucket_mask); + } + } + + /// Searches for an element in the table. This uses dynamic dispatch to reduce the amount of + /// code generated, but it is eliminated by LLVM optimizations. + #[inline] + fn find_inner(&self, hash: u64, eq: &mut dyn FnMut(usize) -> bool) -> Option { + let h2_hash = h2(hash); + let mut probe_seq = self.probe_seq(hash); + + loop { + let group = unsafe { Group::load(self.ctrl(probe_seq.pos)) }; + + for bit in group.match_byte(h2_hash) { + let index = (probe_seq.pos + bit) & self.bucket_mask; + + if likely(eq(index)) { + return Some(index); + } + } + + if likely(group.match_empty().any_bit_set()) { + return None; + } + + probe_seq.move_next(self.bucket_mask); + } + } + + #[allow(clippy::mut_mut)] + #[inline] + unsafe fn prepare_rehash_in_place(&mut self) { + // Bulk convert all full control bytes to DELETED, and all DELETED + // control bytes to EMPTY. This effectively frees up all buckets + // containing a DELETED entry. + for i in (0..self.buckets()).step_by(Group::WIDTH) { + let group = Group::load_aligned(self.ctrl(i)); + let group = group.convert_special_to_empty_and_full_to_deleted(); + group.store_aligned(self.ctrl(i)); + } + + // Fix up the trailing control bytes. See the comments in set_ctrl + // for the handling of tables smaller than the group width. + if self.buckets() < Group::WIDTH { + self.ctrl(0) + .copy_to(self.ctrl(Group::WIDTH), self.buckets()); + } else { + self.ctrl(0) + .copy_to(self.ctrl(self.buckets()), Group::WIDTH); + } + } + + #[inline] + unsafe fn bucket(&self, index: usize) -> Bucket { + debug_assert_ne!(self.bucket_mask, 0); + debug_assert!(index < self.buckets()); + Bucket::from_base_index(self.data_end(), index) + } + + #[inline] + unsafe fn bucket_ptr(&self, index: usize, size_of: usize) -> *mut u8 { + debug_assert_ne!(self.bucket_mask, 0); + debug_assert!(index < self.buckets()); + let base: *mut u8 = self.data_end().as_ptr(); + base.sub((index + 1) * size_of) + } + + #[inline] + unsafe fn data_end(&self) -> NonNull { + NonNull::new_unchecked(self.ctrl.as_ptr().cast()) + } + + /// Returns an iterator-like object for a probe sequence on the table. + /// + /// This iterator never terminates, but is guaranteed to visit each bucket + /// group exactly once. The loop using `probe_seq` must terminate upon + /// reaching a group containing an empty bucket. + #[inline] + fn probe_seq(&self, hash: u64) -> ProbeSeq { + ProbeSeq { + pos: h1(hash) & self.bucket_mask, + stride: 0, + } + } + + /// Returns the index of a bucket for which a value must be inserted if there is enough rooom + /// in the table, otherwise returns error + #[cfg(feature = "raw")] + #[inline] + unsafe fn prepare_insert_no_grow(&mut self, hash: u64) -> Result { + let index = self.find_insert_slot(hash); + let old_ctrl = *self.ctrl(index); + if unlikely(self.growth_left == 0 && special_is_empty(old_ctrl)) { + Err(()) + } else { + self.record_item_insert_at(index, old_ctrl, hash); + Ok(index) + } + } + + #[inline] + unsafe fn record_item_insert_at(&mut self, index: usize, old_ctrl: u8, hash: u64) { + self.growth_left -= usize::from(special_is_empty(old_ctrl)); + self.set_ctrl_h2(index, hash); + self.items += 1; + } + + #[inline] + fn is_in_same_group(&self, i: usize, new_i: usize, hash: u64) -> bool { + let probe_seq_pos = self.probe_seq(hash).pos; + let probe_index = + |pos: usize| (pos.wrapping_sub(probe_seq_pos) & self.bucket_mask) / Group::WIDTH; + probe_index(i) == probe_index(new_i) + } + + /// Sets a control byte to the hash, and possibly also the replicated control byte at + /// the end of the array. + #[inline] + unsafe fn set_ctrl_h2(&self, index: usize, hash: u64) { + self.set_ctrl(index, h2(hash)); + } + + #[inline] + unsafe fn replace_ctrl_h2(&self, index: usize, hash: u64) -> u8 { + let prev_ctrl = *self.ctrl(index); + self.set_ctrl_h2(index, hash); + prev_ctrl + } + + /// Sets a control byte, and possibly also the replicated control byte at + /// the end of the array. + #[inline] + unsafe fn set_ctrl(&self, index: usize, ctrl: u8) { + // Replicate the first Group::WIDTH control bytes at the end of + // the array without using a branch: + // - If index >= Group::WIDTH then index == index2. + // - Otherwise index2 == self.bucket_mask + 1 + index. + // + // The very last replicated control byte is never actually read because + // we mask the initial index for unaligned loads, but we write it + // anyways because it makes the set_ctrl implementation simpler. + // + // If there are fewer buckets than Group::WIDTH then this code will + // replicate the buckets at the end of the trailing group. For example + // with 2 buckets and a group size of 4, the control bytes will look + // like this: + // + // Real | Replicated + // --------------------------------------------- + // | [A] | [B] | [EMPTY] | [EMPTY] | [A] | [B] | + // --------------------------------------------- + let index2 = ((index.wrapping_sub(Group::WIDTH)) & self.bucket_mask) + Group::WIDTH; + + *self.ctrl(index) = ctrl; + *self.ctrl(index2) = ctrl; + } + + /// Returns a pointer to a control byte. + #[inline] + unsafe fn ctrl(&self, index: usize) -> *mut u8 { + debug_assert!(index < self.num_ctrl_bytes()); + self.ctrl.as_ptr().add(index) + } + + #[inline] + fn buckets(&self) -> usize { + self.bucket_mask + 1 + } + + #[inline] + fn num_ctrl_bytes(&self) -> usize { + self.bucket_mask + 1 + Group::WIDTH + } + + #[inline] + fn is_empty_singleton(&self) -> bool { + self.bucket_mask == 0 + } + + #[allow(clippy::mut_mut)] + #[inline] + unsafe fn prepare_resize( + &self, + table_layout: TableLayout, + capacity: usize, + fallibility: Fallibility, + ) -> Result, TryReserveError> { + debug_assert!(self.items <= capacity); + + // Allocate and initialize the new table. + let mut new_table = RawTableInner::fallible_with_capacity( + self.alloc.clone(), + table_layout, + capacity, + fallibility, + )?; + new_table.growth_left -= self.items; + new_table.items = self.items; + + // The hash function may panic, in which case we simply free the new + // table without dropping any elements that may have been copied into + // it. + // + // This guard is also used to free the old table on success, see + // the comment at the bottom of this function. + Ok(guard(new_table, move |self_| { + if !self_.is_empty_singleton() { + self_.free_buckets(table_layout); + } + })) + } + + /// Reserves or rehashes to make room for `additional` more elements. + /// + /// This uses dynamic dispatch to reduce the amount of + /// code generated, but it is eliminated by LLVM optimizations when inlined. + #[allow(clippy::inline_always)] + #[inline(always)] + unsafe fn reserve_rehash_inner( + &mut self, + additional: usize, + hasher: &dyn Fn(&mut Self, usize) -> u64, + fallibility: Fallibility, + layout: TableLayout, + drop: Option, + ) -> Result<(), TryReserveError> { + // Avoid `Option::ok_or_else` because it bloats LLVM IR. + let new_items = match self.items.checked_add(additional) { + Some(new_items) => new_items, + None => return Err(fallibility.capacity_overflow()), + }; + let full_capacity = bucket_mask_to_capacity(self.bucket_mask); + if new_items <= full_capacity / 2 { + // Rehash in-place without re-allocating if we have plenty of spare + // capacity that is locked up due to DELETED entries. + self.rehash_in_place(hasher, layout.size, drop); + Ok(()) + } else { + // Otherwise, conservatively resize to at least the next size up + // to avoid churning deletes into frequent rehashes. + self.resize_inner( + usize::max(new_items, full_capacity + 1), + hasher, + fallibility, + layout, + ) + } + } + + /// Allocates a new table of a different size and moves the contents of the + /// current table into it. + /// + /// This uses dynamic dispatch to reduce the amount of + /// code generated, but it is eliminated by LLVM optimizations when inlined. + #[allow(clippy::inline_always)] + #[inline(always)] + unsafe fn resize_inner( + &mut self, + capacity: usize, + hasher: &dyn Fn(&mut Self, usize) -> u64, + fallibility: Fallibility, + layout: TableLayout, + ) -> Result<(), TryReserveError> { + let mut new_table = self.prepare_resize(layout, capacity, fallibility)?; + + // Copy all elements to the new table. + for i in 0..self.buckets() { + if !is_full(*self.ctrl(i)) { + continue; + } + + // This may panic. + let hash = hasher(self, i); + + // We can use a simpler version of insert() here since: + // - there are no DELETED entries. + // - we know there is enough space in the table. + // - all elements are unique. + let (index, _) = new_table.prepare_insert_slot(hash); + + ptr::copy_nonoverlapping( + self.bucket_ptr(i, layout.size), + new_table.bucket_ptr(index, layout.size), + layout.size, + ); + } + + // We successfully copied all elements without panicking. Now replace + // self with the new table. The old table will have its memory freed but + // the items will not be dropped (since they have been moved into the + // new table). + mem::swap(self, &mut new_table); + + Ok(()) + } + + /// Rehashes the contents of the table in place (i.e. without changing the + /// allocation). + /// + /// If `hasher` panics then some the table's contents may be lost. + /// + /// This uses dynamic dispatch to reduce the amount of + /// code generated, but it is eliminated by LLVM optimizations when inlined. + #[allow(clippy::inline_always)] + #[cfg_attr(feature = "inline-more", inline(always))] + #[cfg_attr(not(feature = "inline-more"), inline)] + unsafe fn rehash_in_place( + &mut self, + hasher: &dyn Fn(&mut Self, usize) -> u64, + size_of: usize, + drop: Option, + ) { + // If the hash function panics then properly clean up any elements + // that we haven't rehashed yet. We unfortunately can't preserve the + // element since we lost their hash and have no way of recovering it + // without risking another panic. + self.prepare_rehash_in_place(); + + let mut guard = guard(self, move |self_| { + if let Some(drop) = drop { + for i in 0..self_.buckets() { + if *self_.ctrl(i) == DELETED { + self_.set_ctrl(i, EMPTY); + drop(self_.bucket_ptr(i, size_of)); + self_.items -= 1; + } } } - } + self_.growth_left = bucket_mask_to_capacity(self_.bucket_mask) - self_.items; + }); - // probe_seq never returns. - unreachable!(); - } + // At this point, DELETED elements are elements that we haven't + // rehashed yet. Find them and re-insert them at their ideal + // position. + 'outer: for i in 0..guard.buckets() { + if *guard.ctrl(i) != DELETED { + continue; + } - /// Returns the number of elements the map can hold without reallocating. - /// - /// This number is a lower bound; the table might be able to hold - /// more, but is guaranteed to be able to hold at least this many. - #[cfg_attr(feature = "inline-more", inline)] - pub fn capacity(&self) -> usize { - self.items + self.growth_left - } + let i_p = guard.bucket_ptr(i, size_of); - /// Returns the number of elements in the table. - #[cfg_attr(feature = "inline-more", inline)] - pub fn len(&self) -> usize { - self.items - } + 'inner: loop { + // Hash the current item + let hash = hasher(*guard, i); - /// Returns the number of buckets in the table. - #[cfg_attr(feature = "inline-more", inline)] - pub fn buckets(&self) -> usize { - self.bucket_mask + 1 - } + // Search for a suitable place to put it + let new_i = guard.find_insert_slot(hash); + let new_i_p = guard.bucket_ptr(new_i, size_of); - /// Returns the number of control bytes in the table. - #[cfg_attr(feature = "inline-more", inline)] - fn num_ctrl_bytes(&self) -> usize { - self.bucket_mask + 1 + Group::WIDTH - } + // Probing works by scanning through all of the control + // bytes in groups, which may not be aligned to the group + // size. If both the new and old position fall within the + // same unaligned group, then there is no benefit in moving + // it and we can just continue to the next item. + if likely(guard.is_in_same_group(i, new_i, hash)) { + guard.set_ctrl_h2(i, hash); + continue 'outer; + } - /// Returns whether this table points to the empty singleton with a capacity - /// of 0. - #[cfg_attr(feature = "inline-more", inline)] - fn is_empty_singleton(&self) -> bool { - self.bucket_mask == 0 + // We are moving the current item to a new position. Write + // our H2 to the control byte of the new position. + let prev_ctrl = guard.replace_ctrl_h2(new_i, hash); + if prev_ctrl == EMPTY { + guard.set_ctrl(i, EMPTY); + // If the target slot is empty, simply move the current + // element into the new slot and clear the old control + // byte. + ptr::copy_nonoverlapping(i_p, new_i_p, size_of); + continue 'outer; + } else { + // If the target slot is occupied, swap the two elements + // and then continue processing the element that we just + // swapped into the old slot. + debug_assert_eq!(prev_ctrl, DELETED); + ptr::swap_nonoverlapping(i_p, new_i_p, size_of); + continue 'inner; + } + } + } + + guard.growth_left = bucket_mask_to_capacity(guard.bucket_mask) - guard.items; + + mem::forget(guard); } - /// Returns an iterator over every element in the table. It is up to - /// the caller to ensure that the `RawTable` outlives the `RawIter`. - /// Because we cannot make the `next` method unsafe on the `RawIter` - /// struct, we have to make the `iter` method unsafe. - #[cfg_attr(feature = "inline-more", inline)] - pub unsafe fn iter(&self) -> RawIter { - let data = Bucket::from_base_index(self.data.as_ptr(), 0); - RawIter { - iter: RawIterRange::new(self.ctrl.as_ptr(), data, self.buckets()), - items: self.items, - } + #[inline] + unsafe fn free_buckets(&mut self, table_layout: TableLayout) { + // Avoid `Option::unwrap_or_else` because it bloats LLVM IR. + let (layout, ctrl_offset) = match table_layout.calculate_layout_for(self.buckets()) { + Some(lco) => lco, + None => hint::unreachable_unchecked(), + }; + self.alloc.deallocate( + NonNull::new_unchecked(self.ctrl.as_ptr().sub(ctrl_offset)), + layout, + ); } - /// Returns an iterator which removes all elements from the table without - /// freeing the memory. It is up to the caller to ensure that the `RawTable` - /// outlives the `RawDrain`. Because we cannot make the `next` method unsafe - /// on the `RawDrain`, we have to make the `drain` method unsafe. - #[cfg_attr(feature = "inline-more", inline)] - pub unsafe fn drain(&mut self) -> RawDrain<'_, T> { - RawDrain { - iter: self.iter(), - table: ManuallyDrop::new(mem::replace(self, Self::new())), - orig_table: NonNull::from(self), - marker: PhantomData, + /// Marks all table buckets as empty without dropping their contents. + #[inline] + fn clear_no_drop(&mut self) { + if !self.is_empty_singleton() { + unsafe { + self.ctrl(0).write_bytes(EMPTY, self.num_ctrl_bytes()); + } } + self.items = 0; + self.growth_left = bucket_mask_to_capacity(self.bucket_mask); } - /// Converts the table into a raw allocation. The contents of the table - /// should be dropped using a `RawIter` before freeing the allocation. - #[cfg_attr(feature = "inline-more", inline)] - pub(crate) fn into_alloc(self) -> Option<(NonNull, Layout)> { - let alloc = if self.is_empty_singleton() { - None + #[inline] + unsafe fn erase(&mut self, index: usize) { + debug_assert!(is_full(*self.ctrl(index))); + let index_before = index.wrapping_sub(Group::WIDTH) & self.bucket_mask; + let empty_before = Group::load(self.ctrl(index_before)).match_empty(); + let empty_after = Group::load(self.ctrl(index)).match_empty(); + + // If we are inside a continuous block of Group::WIDTH full or deleted + // cells then a probe window may have seen a full block when trying to + // insert. We therefore need to keep that block non-empty so that + // lookups will continue searching to the next probe window. + // + // Note that in this context `leading_zeros` refers to the bytes at the + // end of a group, while `trailing_zeros` refers to the bytes at the + // beginning of a group. + let ctrl = if empty_before.leading_zeros() + empty_after.trailing_zeros() >= Group::WIDTH { + DELETED } else { - let (layout, _) = calculate_layout::(self.buckets()) - .unwrap_or_else(|| unsafe { hint::unreachable_unchecked() }); - Some((self.ctrl.cast(), layout)) + self.growth_left += 1; + EMPTY }; - mem::forget(self); - alloc + self.set_ctrl(index, ctrl); + self.items -= 1; } } -unsafe impl Send for RawTable where T: Send {} -unsafe impl Sync for RawTable where T: Sync {} - -impl Clone for RawTable { +impl Clone for RawTable { fn clone(&self) -> Self { - if self.is_empty_singleton() { - Self::new() + if self.table.is_empty_singleton() { + Self::new_in(self.table.alloc.clone()) } else { unsafe { - let mut new_table = ManuallyDrop::new( - Self::new_uninitialized(self.buckets(), Fallibility::Infallible) - .unwrap_or_else(|_| hint::unreachable_unchecked()), - ); - - new_table.clone_from_spec(self, |new_table| { - // We need to free the memory allocated for the new table. + // Avoid `Result::ok_or_else` because it bloats LLVM IR. + let new_table = match Self::new_uninitialized( + self.table.alloc.clone(), + self.table.buckets(), + Fallibility::Infallible, + ) { + Ok(table) => table, + Err(_) => hint::unreachable_unchecked(), + }; + + // If cloning fails then we need to free the allocation for the + // new table. However we don't run its drop since its control + // bytes are not initialized yet. + let mut guard = guard(ManuallyDrop::new(new_table), |new_table| { new_table.free_buckets(); }); - // Return the newly created table. - ManuallyDrop::into_inner(new_table) + guard.clone_from_spec(self); + + // Disarm the scope guard and return the newly created table. + ManuallyDrop::into_inner(ScopeGuard::into_inner(guard)) } } } fn clone_from(&mut self, source: &Self) { - if source.is_empty_singleton() { - *self = Self::new(); + if source.table.is_empty_singleton() { + *self = Self::new_in(self.table.alloc.clone()); } else { unsafe { - // First, drop all our elements without clearing the control bytes. - if mem::needs_drop::() { - for item in self.iter() { - item.drop(); - } - } + // Make sure that if any panics occurs, we clear the table and + // leave it in an empty state. + let mut self_ = guard(self, |self_| { + self_.clear_no_drop(); + }); + + // First, drop all our elements without clearing the control + // bytes. If this panics then the scope guard will clear the + // table, leaking any elements that were not dropped yet. + // + // This leak is unavoidable: we can't try dropping more elements + // since this could lead to another panic and abort the process. + self_.drop_elements(); // If necessary, resize our table to match the source. - if self.buckets() != source.buckets() { + if self_.buckets() != source.buckets() { // Skip our drop by using ptr::write. - if !self.is_empty_singleton() { - self.free_buckets(); + if !self_.table.is_empty_singleton() { + self_.free_buckets(); } - (self as *mut Self).write( - Self::new_uninitialized(source.buckets(), Fallibility::Infallible) - .unwrap_or_else(|_| hint::unreachable_unchecked()), + (&mut **self_ as *mut Self).write( + // Avoid `Result::unwrap_or_else` because it bloats LLVM IR. + match Self::new_uninitialized( + self_.table.alloc.clone(), + source.buckets(), + Fallibility::Infallible, + ) { + Ok(table) => table, + Err(_) => hint::unreachable_unchecked(), + }, ); } - self.clone_from_spec(source, |self_| { - // We need to leave the table in an empty state. - self_.clear_no_drop() - }); + self_.clone_from_spec(source); + + // Disarm the scope guard if cloning was successful. + ScopeGuard::into_inner(self_); } } } @@ -1033,63 +1676,57 @@ impl Clone for RawTable { /// Specialization of `clone_from` for `Copy` types trait RawTableClone { - unsafe fn clone_from_spec(&mut self, source: &Self, on_panic: impl FnMut(&mut Self)); + unsafe fn clone_from_spec(&mut self, source: &Self); } -impl RawTableClone for RawTable { - #[cfg(feature = "nightly")] - #[cfg_attr(feature = "inline-more", inline)] - default unsafe fn clone_from_spec(&mut self, source: &Self, on_panic: impl FnMut(&mut Self)) { - self.clone_from_impl(source, on_panic); - } - - #[cfg(not(feature = "nightly"))] - #[cfg_attr(feature = "inline-more", inline)] - unsafe fn clone_from_spec(&mut self, source: &Self, on_panic: impl FnMut(&mut Self)) { - self.clone_from_impl(source, on_panic); +impl RawTableClone for RawTable { + default_fn! { + #[cfg_attr(feature = "inline-more", inline)] + unsafe fn clone_from_spec(&mut self, source: &Self) { + self.clone_from_impl(source); + } } } #[cfg(feature = "nightly")] -impl RawTableClone for RawTable { +impl RawTableClone for RawTable { #[cfg_attr(feature = "inline-more", inline)] - unsafe fn clone_from_spec(&mut self, source: &Self, _on_panic: impl FnMut(&mut Self)) { + unsafe fn clone_from_spec(&mut self, source: &Self) { source + .table .ctrl(0) - .copy_to_nonoverlapping(self.ctrl(0), self.num_ctrl_bytes()); + .copy_to_nonoverlapping(self.table.ctrl(0), self.table.num_ctrl_bytes()); source - .data - .as_ptr() - .copy_to_nonoverlapping(self.data.as_ptr(), self.buckets()); + .data_start() + .copy_to_nonoverlapping(self.data_start(), self.table.buckets()); - self.items = source.items; - self.growth_left = source.growth_left; + self.table.items = source.table.items; + self.table.growth_left = source.table.growth_left; } } -impl RawTable { - /// Common code for clone and clone_from. Assumes `self.buckets() == source.buckets()`. +impl RawTable { + /// Common code for clone and clone_from. Assumes: + /// - `self.buckets() == source.buckets()`. + /// - Any existing elements have been dropped. + /// - The control bytes are not initialized yet. #[cfg_attr(feature = "inline-more", inline)] - unsafe fn clone_from_impl(&mut self, source: &Self, mut on_panic: impl FnMut(&mut Self)) { + unsafe fn clone_from_impl(&mut self, source: &Self) { // Copy the control bytes unchanged. We do this in a single pass source + .table .ctrl(0) - .copy_to_nonoverlapping(self.ctrl(0), self.num_ctrl_bytes()); + .copy_to_nonoverlapping(self.table.ctrl(0), self.table.num_ctrl_bytes()); // The cloning of elements may panic, in which case we need // to make sure we drop only the elements that have been // cloned so far. let mut guard = guard((0, &mut *self), |(index, self_)| { - if mem::needs_drop::() { + if mem::needs_drop::() && !self_.is_empty() { for i in 0..=*index { - if is_full(*self_.ctrl(i)) { + if is_full(*self_.table.ctrl(i)) { self_.bucket(i).drop(); } } } - - // Depending on whether we were called from clone or clone_from, we - // either need to free the memory for the destination table or just - // clear the control bytes. - on_panic(self_); }); for from in source.iter() { @@ -1104,19 +1741,19 @@ impl RawTable { // Successfully cloned all items, no need to clean up. mem::forget(guard); - self.items = source.items; - self.growth_left = source.growth_left; + self.table.items = source.table.items; + self.table.growth_left = source.table.growth_left; } /// Variant of `clone_from` to use when a hasher is available. - #[cfg(any(feature = "nightly", feature = "raw"))] + #[cfg(feature = "raw")] pub fn clone_from_with_hasher(&mut self, source: &Self, hasher: impl Fn(&T) -> u64) { // If we have enough capacity in the table, just clear it and insert // elements one by one. We don't do this if we have the same number of // buckets as the source since we can just copy the contents directly // in that case. - if self.buckets() != source.buckets() - && bucket_mask_to_capacity(self.bucket_mask) >= source.len() + if self.table.buckets() != source.table.buckets() + && bucket_mask_to_capacity(self.table.bucket_mask) >= source.len() { self.clear(); @@ -1137,8 +1774,7 @@ impl RawTable { // - there are no DELETED entries. // - we know there is enough space in the table. // - all elements are unique. - let index = guard_self.find_insert_slot(hash); - guard_self.set_ctrl(index, h2(hash)); + let (index, _) = guard_self.table.prepare_insert_slot(hash); guard_self.bucket(index).write(item); } } @@ -1146,61 +1782,55 @@ impl RawTable { // Successfully cloned all items, no need to clean up. mem::forget(guard_self); - self.items = source.items; - self.growth_left -= source.items; + self.table.items = source.table.items; + self.table.growth_left -= source.table.items; } else { self.clone_from(source); } } } +impl Default for RawTable { + #[inline] + fn default() -> Self { + Self::new_in(Default::default()) + } +} + #[cfg(feature = "nightly")] -unsafe impl<#[may_dangle] T> Drop for RawTable { +unsafe impl<#[may_dangle] T, A: Allocator + Clone> Drop for RawTable { #[cfg_attr(feature = "inline-more", inline)] fn drop(&mut self) { - if !self.is_empty_singleton() { + if !self.table.is_empty_singleton() { unsafe { - if mem::needs_drop::() { - for item in self.iter() { - item.drop(); - } - } + self.drop_elements(); self.free_buckets(); } } } } #[cfg(not(feature = "nightly"))] -impl Drop for RawTable { +impl Drop for RawTable { #[cfg_attr(feature = "inline-more", inline)] fn drop(&mut self) { - if !self.is_empty_singleton() { + if !self.table.is_empty_singleton() { unsafe { - if mem::needs_drop::() { - for item in self.iter() { - item.drop(); - } - } + self.drop_elements(); self.free_buckets(); } } } } -impl IntoIterator for RawTable { +impl IntoIterator for RawTable { type Item = T; - type IntoIter = RawIntoIter; + type IntoIter = RawIntoIter; #[cfg_attr(feature = "inline-more", inline)] - fn into_iter(self) -> RawIntoIter { + fn into_iter(self) -> RawIntoIter { unsafe { let iter = self.iter(); - let alloc = self.into_alloc(); - RawIntoIter { - iter, - alloc, - marker: PhantomData, - } + self.into_iter_from(iter) } } } @@ -1274,10 +1904,13 @@ impl RawIterRange { let tail = Self::new( self.next_ctrl.add(mid), - self.data.add(Group::WIDTH).add(mid), + self.data.next_n(Group::WIDTH).next_n(mid), len - mid, ); - debug_assert_eq!(self.data.add(Group::WIDTH).add(mid).ptr, tail.data.ptr); + debug_assert_eq!( + self.data.next_n(Group::WIDTH).next_n(mid).ptr, + tail.data.ptr + ); debug_assert_eq!(self.end, tail.end); self.end = self.next_ctrl.add(mid); debug_assert_eq!(self.end.add(Group::WIDTH), tail.next_ctrl); @@ -1285,6 +1918,32 @@ impl RawIterRange { } } } + + /// # Safety + /// If DO_CHECK_PTR_RANGE is false, caller must ensure that we never try to iterate + /// after yielding all elements. + #[cfg_attr(feature = "inline-more", inline)] + unsafe fn next_impl(&mut self) -> Option> { + loop { + if let Some(index) = self.current_group.lowest_set_bit() { + self.current_group = self.current_group.remove_lowest_bit(); + return Some(self.data.next_n(index)); + } + + if DO_CHECK_PTR_RANGE && self.next_ctrl >= self.end { + return None; + } + + // We might read past self.end up to the next group boundary, + // but this is fine because it only occurs on tables smaller + // than the group size where the trailing control bytes are all + // EMPTY. On larger tables self.end is guaranteed to be aligned + // to the group size (since tables are power-of-two sized). + self.current_group = Group::load_aligned(self.next_ctrl).match_full(); + self.data = self.data.next_n(Group::WIDTH); + self.next_ctrl = self.next_ctrl.add(Group::WIDTH); + } + } } // We make raw iterators unconditionally Send and Sync, and let the PhantomData @@ -1310,46 +1969,170 @@ impl Iterator for RawIterRange { #[cfg_attr(feature = "inline-more", inline)] fn next(&mut self) -> Option> { unsafe { - loop { - if let Some(index) = self.current_group.lowest_set_bit() { - self.current_group = self.current_group.remove_lowest_bit(); - return Some(self.data.add(index)); - } - - if self.next_ctrl >= self.end { - return None; - } - - // We might read past self.end up to the next group boundary, - // but this is fine because it only occurs on tables smaller - // than the group size where the trailing control bytes are all - // EMPTY. On larger tables self.end is guaranteed to be aligned - // to the group size (since tables are power-of-two sized). - self.current_group = Group::load_aligned(self.next_ctrl).match_full(); - self.data = self.data.add(Group::WIDTH); - self.next_ctrl = self.next_ctrl.add(Group::WIDTH); - } + // SAFETY: We set checker flag to true. + self.next_impl::() } } - #[cfg_attr(feature = "inline-more", inline)] + #[inline] fn size_hint(&self) -> (usize, Option) { // We don't have an item count, so just guess based on the range size. - ( - 0, - Some(unsafe { offset_from(self.end, self.next_ctrl) + Group::WIDTH }), - ) + let remaining_buckets = if self.end > self.next_ctrl { + unsafe { offset_from(self.end, self.next_ctrl) } + } else { + 0 + }; + + // Add a group width to include the group we are currently processing. + (0, Some(Group::WIDTH + remaining_buckets)) } } impl FusedIterator for RawIterRange {} /// Iterator which returns a raw pointer to every full bucket in the table. +/// +/// For maximum flexibility this iterator is not bound by a lifetime, but you +/// must observe several rules when using it: +/// - You must not free the hash table while iterating (including via growing/shrinking). +/// - It is fine to erase a bucket that has been yielded by the iterator. +/// - Erasing a bucket that has not yet been yielded by the iterator may still +/// result in the iterator yielding that bucket (unless `reflect_remove` is called). +/// - It is unspecified whether an element inserted after the iterator was +/// created will be yielded by that iterator (unless `reflect_insert` is called). +/// - The order in which the iterator yields bucket is unspecified and may +/// change in the future. pub struct RawIter { pub(crate) iter: RawIterRange, items: usize, } +impl RawIter { + /// Refresh the iterator so that it reflects a removal from the given bucket. + /// + /// For the iterator to remain valid, this method must be called once + /// for each removed bucket before `next` is called again. + /// + /// This method should be called _before_ the removal is made. It is not necessary to call this + /// method if you are removing an item that this iterator yielded in the past. + #[cfg(feature = "raw")] + pub fn reflect_remove(&mut self, b: &Bucket) { + self.reflect_toggle_full(b, false); + } + + /// Refresh the iterator so that it reflects an insertion into the given bucket. + /// + /// For the iterator to remain valid, this method must be called once + /// for each insert before `next` is called again. + /// + /// This method does not guarantee that an insertion of a bucket with a greater + /// index than the last one yielded will be reflected in the iterator. + /// + /// This method should be called _after_ the given insert is made. + #[cfg(feature = "raw")] + pub fn reflect_insert(&mut self, b: &Bucket) { + self.reflect_toggle_full(b, true); + } + + /// Refresh the iterator so that it reflects a change to the state of the given bucket. + #[cfg(feature = "raw")] + fn reflect_toggle_full(&mut self, b: &Bucket, is_insert: bool) { + unsafe { + if b.as_ptr() > self.iter.data.as_ptr() { + // The iterator has already passed the bucket's group. + // So the toggle isn't relevant to this iterator. + return; + } + + if self.iter.next_ctrl < self.iter.end + && b.as_ptr() <= self.iter.data.next_n(Group::WIDTH).as_ptr() + { + // The iterator has not yet reached the bucket's group. + // We don't need to reload anything, but we do need to adjust the item count. + + if cfg!(debug_assertions) { + // Double-check that the user isn't lying to us by checking the bucket state. + // To do that, we need to find its control byte. We know that self.iter.data is + // at self.iter.next_ctrl - Group::WIDTH, so we work from there: + let offset = offset_from(self.iter.data.as_ptr(), b.as_ptr()); + let ctrl = self.iter.next_ctrl.sub(Group::WIDTH).add(offset); + // This method should be called _before_ a removal, or _after_ an insert, + // so in both cases the ctrl byte should indicate that the bucket is full. + assert!(is_full(*ctrl)); + } + + if is_insert { + self.items += 1; + } else { + self.items -= 1; + } + + return; + } + + // The iterator is at the bucket group that the toggled bucket is in. + // We need to do two things: + // + // - Determine if the iterator already yielded the toggled bucket. + // If it did, we're done. + // - Otherwise, update the iterator cached group so that it won't + // yield a to-be-removed bucket, or _will_ yield a to-be-added bucket. + // We'll also need to update the item count accordingly. + if let Some(index) = self.iter.current_group.lowest_set_bit() { + let next_bucket = self.iter.data.next_n(index); + if b.as_ptr() > next_bucket.as_ptr() { + // The toggled bucket is "before" the bucket the iterator would yield next. We + // therefore don't need to do anything --- the iterator has already passed the + // bucket in question. + // + // The item count must already be correct, since a removal or insert "prior" to + // the iterator's position wouldn't affect the item count. + } else { + // The removed bucket is an upcoming bucket. We need to make sure it does _not_ + // get yielded, and also that it's no longer included in the item count. + // + // NOTE: We can't just reload the group here, both since that might reflect + // inserts we've already passed, and because that might inadvertently unset the + // bits for _other_ removals. If we do that, we'd have to also decrement the + // item count for those other bits that we unset. But the presumably subsequent + // call to reflect for those buckets might _also_ decrement the item count. + // Instead, we _just_ flip the bit for the particular bucket the caller asked + // us to reflect. + let our_bit = offset_from(self.iter.data.as_ptr(), b.as_ptr()); + let was_full = self.iter.current_group.flip(our_bit); + debug_assert_ne!(was_full, is_insert); + + if is_insert { + self.items += 1; + } else { + self.items -= 1; + } + + if cfg!(debug_assertions) { + if b.as_ptr() == next_bucket.as_ptr() { + // The removed bucket should no longer be next + debug_assert_ne!(self.iter.current_group.lowest_set_bit(), Some(index)); + } else { + // We should not have changed what bucket comes next. + debug_assert_eq!(self.iter.current_group.lowest_set_bit(), Some(index)); + } + } + } + } else { + // We must have already iterated past the removed item. + } + } + } + + unsafe fn drop_elements(&mut self) { + if mem::needs_drop::() && self.len() != 0 { + for item in self { + item.drop(); + } + } + } +} + impl Clone for RawIter { #[cfg_attr(feature = "inline-more", inline)] fn clone(&self) -> Self { @@ -1365,19 +2148,25 @@ impl Iterator for RawIter { #[cfg_attr(feature = "inline-more", inline)] fn next(&mut self) -> Option> { - if let Some(b) = self.iter.next() { + // Inner iterator iterates over buckets + // so it can do unnecessary work if we already yielded all items. + if self.items == 0 { + return None; + } + + let nxt = unsafe { + // SAFETY: We check number of items to yield using `items` field. + self.iter.next_impl::() + }; + + if nxt.is_some() { self.items -= 1; - Some(b) - } else { - // We don't check against items == 0 here to allow the - // compiler to optimize away the item count entirely if the - // iterator length is never queried. - debug_assert_eq!(self.items, 0); - None } + + nxt } - #[cfg_attr(feature = "inline-more", inline)] + #[inline] fn size_hint(&self) -> (usize, Option) { (self.items, Some(self.items)) } @@ -1387,62 +2176,65 @@ impl ExactSizeIterator for RawIter {} impl FusedIterator for RawIter {} /// Iterator which consumes a table and returns elements. -pub struct RawIntoIter { +pub struct RawIntoIter { iter: RawIter, - alloc: Option<(NonNull, Layout)>, + allocation: Option<(NonNull, Layout)>, marker: PhantomData, + alloc: A, } -impl RawIntoIter { +impl RawIntoIter { #[cfg_attr(feature = "inline-more", inline)] pub fn iter(&self) -> RawIter { self.iter.clone() } } -unsafe impl Send for RawIntoIter where T: Send {} -unsafe impl Sync for RawIntoIter where T: Sync {} +unsafe impl Send for RawIntoIter +where + T: Send, + A: Send, +{ +} +unsafe impl Sync for RawIntoIter +where + T: Sync, + A: Sync, +{ +} #[cfg(feature = "nightly")] -unsafe impl<#[may_dangle] T> Drop for RawIntoIter { +unsafe impl<#[may_dangle] T, A: Allocator + Clone> Drop for RawIntoIter { #[cfg_attr(feature = "inline-more", inline)] fn drop(&mut self) { unsafe { // Drop all remaining elements - if mem::needs_drop::() { - while let Some(item) = self.iter.next() { - item.drop(); - } - } + self.iter.drop_elements(); // Free the table - if let Some((ptr, layout)) = self.alloc { - dealloc(ptr.as_ptr(), layout); + if let Some((ptr, layout)) = self.allocation { + self.alloc.deallocate(ptr, layout); } } } } #[cfg(not(feature = "nightly"))] -impl Drop for RawIntoIter { +impl Drop for RawIntoIter { #[cfg_attr(feature = "inline-more", inline)] fn drop(&mut self) { unsafe { // Drop all remaining elements - if mem::needs_drop::() { - while let Some(item) = self.iter.next() { - item.drop(); - } - } + self.iter.drop_elements(); // Free the table - if let Some((ptr, layout)) = self.alloc { - dealloc(ptr.as_ptr(), layout); + if let Some((ptr, layout)) = self.allocation { + self.alloc.deallocate(ptr, layout); } } } } -impl Iterator for RawIntoIter { +impl Iterator for RawIntoIter { type Item = T; #[cfg_attr(feature = "inline-more", inline)] @@ -1450,50 +2242,56 @@ impl Iterator for RawIntoIter { unsafe { Some(self.iter.next()?.read()) } } - #[cfg_attr(feature = "inline-more", inline)] + #[inline] fn size_hint(&self) -> (usize, Option) { self.iter.size_hint() } } -impl ExactSizeIterator for RawIntoIter {} -impl FusedIterator for RawIntoIter {} +impl ExactSizeIterator for RawIntoIter {} +impl FusedIterator for RawIntoIter {} /// Iterator which consumes elements without freeing the table storage. -pub struct RawDrain<'a, T> { +pub struct RawDrain<'a, T, A: Allocator + Clone = Global> { iter: RawIter, // The table is moved into the iterator for the duration of the drain. This // ensures that an empty table is left if the drain iterator is leaked // without dropping. - table: ManuallyDrop>, - orig_table: NonNull>, + table: ManuallyDrop>, + orig_table: NonNull>, // We don't use a &'a mut RawTable because we want RawDrain to be // covariant over T. - marker: PhantomData<&'a RawTable>, + marker: PhantomData<&'a RawTable>, } -impl RawDrain<'_, T> { +impl RawDrain<'_, T, A> { #[cfg_attr(feature = "inline-more", inline)] pub fn iter(&self) -> RawIter { self.iter.clone() } } -unsafe impl Send for RawDrain<'_, T> where T: Send {} -unsafe impl Sync for RawDrain<'_, T> where T: Sync {} +unsafe impl Send for RawDrain<'_, T, A> +where + T: Send, + A: Send, +{ +} +unsafe impl Sync for RawDrain<'_, T, A> +where + T: Sync, + A: Sync, +{ +} -impl Drop for RawDrain<'_, T> { +impl Drop for RawDrain<'_, T, A> { #[cfg_attr(feature = "inline-more", inline)] fn drop(&mut self) { unsafe { // Drop all remaining elements. Note that this may panic. - if mem::needs_drop::() { - while let Some(item) = self.iter.next() { - item.drop(); - } - } + self.iter.drop_elements(); // Reset the contents of the table now that all elements have been // dropped. @@ -1507,7 +2305,7 @@ impl Drop for RawDrain<'_, T> { } } -impl Iterator for RawDrain<'_, T> { +impl Iterator for RawDrain<'_, T, A> { type Item = T; #[cfg_attr(feature = "inline-more", inline)] @@ -1518,11 +2316,145 @@ impl Iterator for RawDrain<'_, T> { } } - #[cfg_attr(feature = "inline-more", inline)] + #[inline] fn size_hint(&self) -> (usize, Option) { self.iter.size_hint() } } -impl ExactSizeIterator for RawDrain<'_, T> {} -impl FusedIterator for RawDrain<'_, T> {} +impl ExactSizeIterator for RawDrain<'_, T, A> {} +impl FusedIterator for RawDrain<'_, T, A> {} + +/// Iterator over occupied buckets that could match a given hash. +/// +/// `RawTable` only stores 7 bits of the hash value, so this iterator may return +/// items that have a hash value different than the one provided. You should +/// always validate the returned values before using them. +pub struct RawIterHash<'a, T, A: Allocator + Clone = Global> { + inner: RawIterHashInner<'a, A>, + _marker: PhantomData, +} + +struct RawIterHashInner<'a, A: Allocator + Clone> { + table: &'a RawTableInner, + + // The top 7 bits of the hash. + h2_hash: u8, + + // The sequence of groups to probe in the search. + probe_seq: ProbeSeq, + + group: Group, + + // The elements within the group with a matching h2-hash. + bitmask: BitMaskIter, +} + +impl<'a, T, A: Allocator + Clone> RawIterHash<'a, T, A> { + #[cfg_attr(feature = "inline-more", inline)] + #[cfg(feature = "raw")] + fn new(table: &'a RawTable, hash: u64) -> Self { + RawIterHash { + inner: RawIterHashInner::new(&table.table, hash), + _marker: PhantomData, + } + } +} +impl<'a, A: Allocator + Clone> RawIterHashInner<'a, A> { + #[cfg_attr(feature = "inline-more", inline)] + #[cfg(feature = "raw")] + fn new(table: &'a RawTableInner, hash: u64) -> Self { + unsafe { + let h2_hash = h2(hash); + let probe_seq = table.probe_seq(hash); + let group = Group::load(table.ctrl(probe_seq.pos)); + let bitmask = group.match_byte(h2_hash).into_iter(); + + RawIterHashInner { + table, + h2_hash, + probe_seq, + group, + bitmask, + } + } + } +} + +impl<'a, T, A: Allocator + Clone> Iterator for RawIterHash<'a, T, A> { + type Item = Bucket; + + fn next(&mut self) -> Option> { + unsafe { + match self.inner.next() { + Some(index) => Some(self.inner.table.bucket(index)), + None => None, + } + } + } +} + +impl<'a, A: Allocator + Clone> Iterator for RawIterHashInner<'a, A> { + type Item = usize; + + fn next(&mut self) -> Option { + unsafe { + loop { + if let Some(bit) = self.bitmask.next() { + let index = (self.probe_seq.pos + bit) & self.table.bucket_mask; + return Some(index); + } + if likely(self.group.match_empty().any_bit_set()) { + return None; + } + self.probe_seq.move_next(self.table.bucket_mask); + self.group = Group::load(self.table.ctrl(self.probe_seq.pos)); + self.bitmask = self.group.match_byte(self.h2_hash).into_iter(); + } + } + } +} + +#[cfg(test)] +mod test_map { + use super::*; + + fn rehash_in_place(table: &mut RawTable, hasher: impl Fn(&T) -> u64) { + unsafe { + table.table.rehash_in_place( + &|table, index| hasher(table.bucket::(index).as_ref()), + mem::size_of::(), + if mem::needs_drop::() { + Some(mem::transmute(ptr::drop_in_place:: as unsafe fn(*mut T))) + } else { + None + }, + ); + } + } + + #[test] + fn rehash() { + let mut table = RawTable::new(); + let hasher = |i: &u64| *i; + for i in 0..100 { + table.insert(i, i, hasher); + } + + for i in 0..100 { + unsafe { + assert_eq!(table.find(i, |x| *x == i).map(|b| b.read()), Some(i)); + } + assert!(table.find(i + 100, |x| *x == i + 100).is_none()); + } + + rehash_in_place(&mut table, hasher); + + for i in 0..100 { + unsafe { + assert_eq!(table.find(i, |x| *x == i).map(|b| b.read()), Some(i)); + } + assert!(table.find(i + 100, |x| *x == i + 100).is_none()); + } + } +} diff --git a/crux-mir/lib/hashbrown/src/raw/sse2.rs b/crux-mir/lib/hashbrown/src/raw/sse2.rs index 79b0aad42..a0bf6da80 100644 --- a/crux-mir/lib/hashbrown/src/raw/sse2.rs +++ b/crux-mir/lib/hashbrown/src/raw/sse2.rs @@ -25,26 +25,29 @@ impl Group { pub const WIDTH: usize = mem::size_of::(); /// Returns a full group of empty bytes, suitable for use as the initial - /// value for an empty hash table. This value is explicitly declared as - /// a static variable to ensure the address is consistent across dylibs. + /// value for an empty hash table. /// /// This is guaranteed to be aligned to the group size. - pub fn static_empty() -> &'static [u8] { - union AlignedBytes { - _align: Group, + #[inline] + #[allow(clippy::items_after_statements)] + pub const fn static_empty() -> &'static [u8; Group::WIDTH] { + #[repr(C)] + struct AlignedBytes { + _align: [Group; 0], bytes: [u8; Group::WIDTH], - }; - static ALIGNED_BYTES: AlignedBytes = AlignedBytes { + } + const ALIGNED_BYTES: AlignedBytes = AlignedBytes { + _align: [], bytes: [EMPTY; Group::WIDTH], }; - unsafe { &ALIGNED_BYTES.bytes } + &ALIGNED_BYTES.bytes } /// Loads a group of bytes starting at the given address. #[inline] #[allow(clippy::cast_ptr_alignment)] // unaligned load pub unsafe fn load(ptr: *const u8) -> Self { - Group(x86::_mm_loadu_si128(ptr as *const _)) + Group(x86::_mm_loadu_si128(ptr.cast())) } /// Loads a group of bytes starting at the given address, which must be @@ -54,7 +57,7 @@ impl Group { pub unsafe fn load_aligned(ptr: *const u8) -> Self { // FIXME: use align_offset once it stabilizes debug_assert_eq!(ptr as usize & (mem::align_of::() - 1), 0); - Group(x86::_mm_load_si128(ptr as *const _)) + Group(x86::_mm_load_si128(ptr.cast())) } /// Stores the group of bytes to the given address, which must be @@ -64,7 +67,7 @@ impl Group { pub unsafe fn store_aligned(self, ptr: *mut u8) { // FIXME: use align_offset once it stabilizes debug_assert_eq!(ptr as usize & (mem::align_of::() - 1), 0); - x86::_mm_store_si128(ptr as *mut _, self.0); + x86::_mm_store_si128(ptr.cast(), self.0); } /// Returns a `BitMask` indicating all bytes in the group which have diff --git a/crux-mir/lib/hashbrown/src/rustc_entry.rs b/crux-mir/lib/hashbrown/src/rustc_entry.rs index 39dc51aa2..2e8459526 100644 --- a/crux-mir/lib/hashbrown/src/rustc_entry.rs +++ b/crux-mir/lib/hashbrown/src/rustc_entry.rs @@ -1,14 +1,15 @@ use self::RustcEntry::*; -use crate::map::{make_hash, Drain, HashMap, IntoIter, Iter, IterMut}; -use crate::raw::{Bucket, RawTable}; +use crate::map::{make_insert_hash, Drain, HashMap, IntoIter, Iter, IterMut}; +use crate::raw::{Allocator, Bucket, Global, RawTable}; use core::fmt::{self, Debug}; use core::hash::{BuildHasher, Hash}; use core::mem; -impl HashMap +impl HashMap where K: Eq + Hash, S: BuildHasher, + A: Allocator + Clone, { /// Gets the given key's corresponding entry in the map for in-place manipulation. /// @@ -30,8 +31,8 @@ where /// assert_eq!(letters.get(&'y'), None); /// ``` #[cfg_attr(feature = "inline-more", inline)] - pub fn rustc_entry(&mut self, key: K) -> RustcEntry<'_, K, V> { - let hash = make_hash(&self.hash_builder, &key); + pub fn rustc_entry(&mut self, key: K) -> RustcEntry<'_, K, V, A> { + let hash = make_insert_hash(&self.hash_builder, &key); if let Some(elem) = self.table.find(hash, |q| q.0.eq(&key)) { RustcEntry::Occupied(RustcOccupiedEntry { key: Some(key), @@ -55,19 +56,22 @@ where /// A view into a single entry in a map, which may either be vacant or occupied. /// -/// This `enum` is constructed from the [`entry`] method on [`HashMap`]. +/// This `enum` is constructed from the [`rustc_entry`] method on [`HashMap`]. /// /// [`HashMap`]: struct.HashMap.html -/// [`entry`]: struct.HashMap.html#method.rustc_entry -pub enum RustcEntry<'a, K, V> { +/// [`rustc_entry`]: struct.HashMap.html#method.rustc_entry +pub enum RustcEntry<'a, K, V, A = Global> +where + A: Allocator + Clone, +{ /// An occupied entry. - Occupied(RustcOccupiedEntry<'a, K, V>), + Occupied(RustcOccupiedEntry<'a, K, V, A>), /// A vacant entry. - Vacant(RustcVacantEntry<'a, K, V>), + Vacant(RustcVacantEntry<'a, K, V, A>), } -impl Debug for RustcEntry<'_, K, V> { +impl Debug for RustcEntry<'_, K, V, A> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match *self { Vacant(ref v) => f.debug_tuple("Entry").field(v).finish(), @@ -80,26 +84,31 @@ impl Debug for RustcEntry<'_, K, V> { /// It is part of the [`RustcEntry`] enum. /// /// [`RustcEntry`]: enum.RustcEntry.html -pub struct RustcOccupiedEntry<'a, K, V> { +pub struct RustcOccupiedEntry<'a, K, V, A = Global> +where + A: Allocator + Clone, +{ key: Option, elem: Bucket<(K, V)>, - table: &'a mut RawTable<(K, V)>, + table: &'a mut RawTable<(K, V), A>, } -unsafe impl Send for RustcOccupiedEntry<'_, K, V> +unsafe impl Send for RustcOccupiedEntry<'_, K, V, A> where K: Send, V: Send, + A: Allocator + Clone + Send, { } -unsafe impl Sync for RustcOccupiedEntry<'_, K, V> +unsafe impl Sync for RustcOccupiedEntry<'_, K, V, A> where K: Sync, V: Sync, + A: Allocator + Clone + Sync, { } -impl Debug for RustcOccupiedEntry<'_, K, V> { +impl Debug for RustcOccupiedEntry<'_, K, V, A> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("OccupiedEntry") .field("key", self.key()) @@ -112,19 +121,22 @@ impl Debug for RustcOccupiedEntry<'_, K, V> { /// It is part of the [`RustcEntry`] enum. /// /// [`RustcEntry`]: enum.RustcEntry.html -pub struct RustcVacantEntry<'a, K, V> { +pub struct RustcVacantEntry<'a, K, V, A = Global> +where + A: Allocator + Clone, +{ hash: u64, key: K, - table: &'a mut RawTable<(K, V)>, + table: &'a mut RawTable<(K, V), A>, } -impl Debug for RustcVacantEntry<'_, K, V> { +impl Debug for RustcVacantEntry<'_, K, V, A> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("VacantEntry").field(self.key()).finish() } } -impl<'a, K, V> RustcEntry<'a, K, V> { +impl<'a, K, V, A: Allocator + Clone> RustcEntry<'a, K, V, A> { /// Sets the value of the entry, and returns a RustcOccupiedEntry. /// /// # Examples @@ -133,11 +145,11 @@ impl<'a, K, V> RustcEntry<'a, K, V> { /// use hashbrown::HashMap; /// /// let mut map: HashMap<&str, u32> = HashMap::new(); - /// let entry = map.entry("horseyland").insert(37); + /// let entry = map.rustc_entry("horseyland").insert(37); /// /// assert_eq!(entry.key(), &"horseyland"); /// ``` - pub fn insert(self, value: V) -> RustcOccupiedEntry<'a, K, V> { + pub fn insert(self, value: V) -> RustcOccupiedEntry<'a, K, V, A> { match self { Vacant(entry) => entry.insert_entry(value), Occupied(mut entry) => { @@ -253,7 +265,7 @@ impl<'a, K, V> RustcEntry<'a, K, V> { } } -impl<'a, K, V: Default> RustcEntry<'a, K, V> { +impl<'a, K, V: Default, A: Allocator + Clone> RustcEntry<'a, K, V, A> { /// Ensures a value is in the entry by inserting the default value if empty, /// and returns a mutable reference to the value in the entry. /// @@ -281,7 +293,7 @@ impl<'a, K, V: Default> RustcEntry<'a, K, V> { } } -impl<'a, K, V> RustcOccupiedEntry<'a, K, V> { +impl<'a, K, V, A: Allocator + Clone> RustcOccupiedEntry<'a, K, V, A> { /// Gets a reference to the key in the entry. /// /// # Examples @@ -318,10 +330,7 @@ impl<'a, K, V> RustcOccupiedEntry<'a, K, V> { /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn remove_entry(self) -> (K, V) { - unsafe { - self.table.erase_no_drop(&self.elem); - self.elem.read() - } + unsafe { self.table.remove(self.elem) } } /// Gets a reference to the value in the entry. @@ -422,10 +431,8 @@ impl<'a, K, V> RustcOccupiedEntry<'a, K, V> { /// assert_eq!(map["poneyland"], 15); /// ``` #[cfg_attr(feature = "inline-more", inline)] - pub fn insert(&mut self, mut value: V) -> V { - let old_value = self.get_mut(); - mem::swap(&mut value, old_value); - value + pub fn insert(&mut self, value: V) -> V { + mem::replace(self.get_mut(), value) } /// Takes the value out of the entry, and returns it. @@ -511,7 +518,7 @@ impl<'a, K, V> RustcOccupiedEntry<'a, K, V> { } } -impl<'a, K, V> RustcVacantEntry<'a, K, V> { +impl<'a, K, V, A: Allocator + Clone> RustcVacantEntry<'a, K, V, A> { /// Gets a reference to the key that would be used when inserting a value /// through the `RustcVacantEntry`. /// @@ -565,8 +572,10 @@ impl<'a, K, V> RustcVacantEntry<'a, K, V> { /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn insert(self, value: V) -> &'a mut V { - let bucket = self.table.insert_no_grow(self.hash, (self.key, value)); - unsafe { &mut bucket.as_mut().1 } + unsafe { + let bucket = self.table.insert_no_grow(self.hash, (self.key, value)); + &mut bucket.as_mut().1 + } } /// Sets the value of the entry with the RustcVacantEntry's key, @@ -586,8 +595,8 @@ impl<'a, K, V> RustcVacantEntry<'a, K, V> { /// } /// ``` #[cfg_attr(feature = "inline-more", inline)] - pub fn insert_entry(self, value: V) -> RustcOccupiedEntry<'a, K, V> { - let bucket = self.table.insert_no_grow(self.hash, (self.key, value)); + pub fn insert_entry(self, value: V) -> RustcOccupiedEntry<'a, K, V, A> { + let bucket = unsafe { self.table.insert_no_grow(self.hash, (self.key, value)) }; RustcOccupiedEntry { key: None, elem: bucket, diff --git a/crux-mir/lib/hashbrown/src/scopeguard.rs b/crux-mir/lib/hashbrown/src/scopeguard.rs index 32c969437..f85e6ab0e 100644 --- a/crux-mir/lib/hashbrown/src/scopeguard.rs +++ b/crux-mir/lib/hashbrown/src/scopeguard.rs @@ -1,5 +1,9 @@ // Extracted from the scopeguard crate -use core::ops::{Deref, DerefMut}; +use core::{ + mem, + ops::{Deref, DerefMut}, + ptr, +}; pub struct ScopeGuard where @@ -9,7 +13,7 @@ where value: T, } -#[cfg_attr(feature = "inline-more", inline)] +#[inline] pub fn guard(value: T, dropfn: F) -> ScopeGuard where F: FnMut(&mut T), @@ -17,12 +21,33 @@ where ScopeGuard { dropfn, value } } +impl ScopeGuard +where + F: FnMut(&mut T), +{ + #[inline] + pub fn into_inner(guard: Self) -> T { + // Cannot move out of Drop-implementing types, so + // ptr::read the value and forget the guard. + unsafe { + let value = ptr::read(&guard.value); + // read the closure so that it is dropped, and assign it to a local + // variable to ensure that it is only dropped after the guard has + // been forgotten. (In case the Drop impl of the closure, or that + // of any consumed captured variable, panics). + let _dropfn = ptr::read(&guard.dropfn); + mem::forget(guard); + value + } + } +} + impl Deref for ScopeGuard where F: FnMut(&mut T), { type Target = T; - #[cfg_attr(feature = "inline-more", inline)] + #[inline] fn deref(&self) -> &T { &self.value } @@ -32,7 +57,7 @@ impl DerefMut for ScopeGuard where F: FnMut(&mut T), { - #[cfg_attr(feature = "inline-more", inline)] + #[inline] fn deref_mut(&mut self) -> &mut T { &mut self.value } @@ -42,8 +67,8 @@ impl Drop for ScopeGuard where F: FnMut(&mut T), { - #[cfg_attr(feature = "inline-more", inline)] + #[inline] fn drop(&mut self) { - (self.dropfn)(&mut self.value) + (self.dropfn)(&mut self.value); } } diff --git a/crux-mir/lib/hashbrown/src/set.rs b/crux-mir/lib/hashbrown/src/set.rs index e5a94c2de..2a4dcea52 100644 --- a/crux-mir/lib/hashbrown/src/set.rs +++ b/crux-mir/lib/hashbrown/src/set.rs @@ -1,11 +1,14 @@ -use crate::CollectionAllocErr; +use crate::TryReserveError; +use alloc::borrow::ToOwned; use core::borrow::Borrow; use core::fmt; use core::hash::{BuildHasher, Hash}; use core::iter::{Chain, FromIterator, FusedIterator}; +use core::mem; use core::ops::{BitAnd, BitOr, BitXor, Sub}; -use super::map::{self, DefaultHashBuilder, HashMap, Keys}; +use super::map::{self, ConsumeAllOnDrop, DefaultHashBuilder, DrainFilterInner, HashMap, Keys}; +use crate::raw::{Allocator, Global}; // Future Optimization (FIXME!) // ============================= @@ -69,7 +72,7 @@ use super::map::{self, DefaultHashBuilder, HashMap, Keys}; /// ``` /// /// The easiest way to use `HashSet` with a custom type is to derive -/// [`Eq`] and [`Hash`]. We must also derive [`PartialEq`], this will in the +/// [`Eq`] and [`Hash`]. We must also derive [`PartialEq`]. This will in the /// future be implied by [`Eq`]. /// /// ``` @@ -98,11 +101,9 @@ use super::map::{self, DefaultHashBuilder, HashMap, Keys}; /// ``` /// use hashbrown::HashSet; /// -/// fn main() { -/// let viking_names: HashSet<&'static str> = -/// [ "Einar", "Olaf", "Harald" ].iter().cloned().collect(); -/// // use the values stored in the set -/// } +/// let viking_names: HashSet<&'static str> = +/// [ "Einar", "Olaf", "Harald" ].iter().cloned().collect(); +/// // use the values stored in the set /// ``` /// /// [`Cell`]: https://doc.rust-lang.org/std/cell/struct.Cell.html @@ -111,11 +112,11 @@ use super::map::{self, DefaultHashBuilder, HashMap, Keys}; /// [`HashMap`]: struct.HashMap.html /// [`PartialEq`]: https://doc.rust-lang.org/std/cmp/trait.PartialEq.html /// [`RefCell`]: https://doc.rust-lang.org/std/cell/struct.RefCell.html -pub struct HashSet { - pub(crate) map: HashMap, +pub struct HashSet { + pub(crate) map: HashMap, } -impl Clone for HashSet { +impl Clone for HashSet { fn clone(&self) -> Self { HashSet { map: self.map.clone(), @@ -128,7 +129,7 @@ impl Clone for HashSet { } #[cfg(feature = "ahash")] -impl HashSet { +impl HashSet { /// Creates an empty `HashSet`. /// /// The hash set is initially created with a capacity of 0, so it will not allocate until it @@ -167,7 +168,47 @@ impl HashSet { } } -impl HashSet { +#[cfg(feature = "ahash")] +impl HashSet { + /// Creates an empty `HashSet`. + /// + /// The hash set is initially created with a capacity of 0, so it will not allocate until it + /// is first inserted into. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashSet; + /// let set: HashSet = HashSet::new(); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn new_in(alloc: A) -> Self { + Self { + map: HashMap::new_in(alloc), + } + } + + /// Creates an empty `HashSet` with the specified capacity. + /// + /// The hash set will be able to hold at least `capacity` elements without + /// reallocating. If `capacity` is 0, the hash set will not allocate. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashSet; + /// let set: HashSet = HashSet::with_capacity(10); + /// assert!(set.capacity() >= 10); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn with_capacity_in(capacity: usize, alloc: A) -> Self { + Self { + map: HashMap::with_capacity_in(capacity, alloc), + } + } +} + +impl HashSet { /// Returns the number of elements the set can hold without reallocating. /// /// # Examples @@ -257,12 +298,72 @@ impl HashSet { /// assert!(set.is_empty()); /// ``` #[cfg_attr(feature = "inline-more", inline)] - pub fn drain(&mut self) -> Drain<'_, T> { + pub fn drain(&mut self) -> Drain<'_, T, A> { Drain { iter: self.map.drain(), } } + /// Retains only the elements specified by the predicate. + /// + /// In other words, remove all elements `e` such that `f(&e)` returns `false`. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashSet; + /// + /// let xs = [1,2,3,4,5,6]; + /// let mut set: HashSet = xs.iter().cloned().collect(); + /// set.retain(|&k| k % 2 == 0); + /// assert_eq!(set.len(), 3); + /// ``` + pub fn retain(&mut self, mut f: F) + where + F: FnMut(&T) -> bool, + { + self.map.retain(|k, _| f(k)); + } + + /// Drains elements which are true under the given predicate, + /// and returns an iterator over the removed items. + /// + /// In other words, move all elements `e` such that `f(&e)` returns `true` out + /// into another iterator. + /// + /// When the returned DrainedFilter is dropped, any remaining elements that satisfy + /// the predicate are dropped from the set. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashSet; + /// + /// let mut set: HashSet = (0..8).collect(); + /// let drained: HashSet = set.drain_filter(|v| v % 2 == 0).collect(); + /// + /// let mut evens = drained.into_iter().collect::>(); + /// let mut odds = set.into_iter().collect::>(); + /// evens.sort(); + /// odds.sort(); + /// + /// assert_eq!(evens, vec![0, 2, 4, 6]); + /// assert_eq!(odds, vec![1, 3, 5, 7]); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn drain_filter(&mut self, f: F) -> DrainFilter<'_, T, F, A> + where + F: FnMut(&T) -> bool, + { + DrainFilter { + f, + inner: DrainFilterInner { + iter: unsafe { self.map.table.iter() }, + table: &mut self.map.table, + }, + } + } + /// Clears the set, removing all values. /// /// # Examples @@ -277,15 +378,11 @@ impl HashSet { /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn clear(&mut self) { - self.map.clear() + self.map.clear(); } } -impl HashSet -where - T: Eq + Hash, - S: BuildHasher, -{ +impl HashSet { /// Creates a new empty hash set which will use the given hasher to hash /// keys. /// @@ -296,6 +393,10 @@ where /// cause many collisions and very poor performance. Setting it /// manually using this function can expose a DoS attack vector. /// + /// The `hash_builder` passed should implement the [`BuildHasher`] trait for + /// the HashMap to be useful, see its documentation for details. + /// + /// /// # Examples /// /// ``` @@ -306,8 +407,10 @@ where /// let mut set = HashSet::with_hasher(s); /// set.insert(2); /// ``` + /// + /// [`BuildHasher`]: ../../std/hash/trait.BuildHasher.html #[cfg_attr(feature = "inline-more", inline)] - pub fn with_hasher(hasher: S) -> Self { + pub const fn with_hasher(hasher: S) -> Self { Self { map: HashMap::with_hasher(hasher), } @@ -324,6 +427,9 @@ where /// cause many collisions and very poor performance. Setting it /// manually using this function can expose a DoS attack vector. /// + /// The `hash_builder` passed should implement the [`BuildHasher`] trait for + /// the HashMap to be useful, see its documentation for details. + /// /// # Examples /// /// ``` @@ -334,12 +440,80 @@ where /// let mut set = HashSet::with_capacity_and_hasher(10, s); /// set.insert(1); /// ``` + /// + /// [`BuildHasher`]: ../../std/hash/trait.BuildHasher.html #[cfg_attr(feature = "inline-more", inline)] pub fn with_capacity_and_hasher(capacity: usize, hasher: S) -> Self { Self { map: HashMap::with_capacity_and_hasher(capacity, hasher), } } +} + +impl HashSet +where + A: Allocator + Clone, +{ + /// Returns a reference to the underlying allocator. + #[inline] + pub fn allocator(&self) -> &A { + self.map.allocator() + } + + /// Creates a new empty hash set which will use the given hasher to hash + /// keys. + /// + /// The hash set is also created with the default initial capacity. + /// + /// Warning: `hasher` is normally randomly generated, and + /// is designed to allow `HashSet`s to be resistant to attacks that + /// cause many collisions and very poor performance. Setting it + /// manually using this function can expose a DoS attack vector. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashSet; + /// use hashbrown::hash_map::DefaultHashBuilder; + /// + /// let s = DefaultHashBuilder::default(); + /// let mut set = HashSet::with_hasher(s); + /// set.insert(2); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn with_hasher_in(hasher: S, alloc: A) -> Self { + Self { + map: HashMap::with_hasher_in(hasher, alloc), + } + } + + /// Creates an empty `HashSet` with the specified capacity, using + /// `hasher` to hash the keys. + /// + /// The hash set will be able to hold at least `capacity` elements without + /// reallocating. If `capacity` is 0, the hash set will not allocate. + /// + /// Warning: `hasher` is normally randomly generated, and + /// is designed to allow `HashSet`s to be resistant to attacks that + /// cause many collisions and very poor performance. Setting it + /// manually using this function can expose a DoS attack vector. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashSet; + /// use hashbrown::hash_map::DefaultHashBuilder; + /// + /// let s = DefaultHashBuilder::default(); + /// let mut set = HashSet::with_capacity_and_hasher(10, s); + /// set.insert(1); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn with_capacity_and_hasher_in(capacity: usize, hasher: S, alloc: A) -> Self { + Self { + map: HashMap::with_capacity_and_hasher_in(capacity, hasher, alloc), + } + } /// Returns a reference to the set's [`BuildHasher`]. /// @@ -359,7 +533,14 @@ where pub fn hasher(&self) -> &S { self.map.hasher() } +} +impl HashSet +where + T: Eq + Hash, + S: BuildHasher, + A: Allocator + Clone, +{ /// Reserves capacity for at least `additional` more elements to be inserted /// in the `HashSet`. The collection may reserve more space to avoid /// frequent reallocations. @@ -378,7 +559,7 @@ where /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn reserve(&mut self, additional: usize) { - self.map.reserve(additional) + self.map.reserve(additional); } /// Tries to reserve capacity for at least `additional` more elements to be inserted @@ -398,7 +579,7 @@ where /// set.try_reserve(10).expect("why is the test harness OOMing on 10 bytes?"); /// ``` #[cfg_attr(feature = "inline-more", inline)] - pub fn try_reserve(&mut self, additional: usize) -> Result<(), CollectionAllocErr> { + pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { self.map.try_reserve(additional) } @@ -420,7 +601,7 @@ where /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn shrink_to_fit(&mut self) { - self.map.shrink_to_fit() + self.map.shrink_to_fit(); } /// Shrinks the capacity of the set with a lower limit. It will drop @@ -446,7 +627,7 @@ where /// ``` #[cfg_attr(feature = "inline-more", inline)] pub fn shrink_to(&mut self, min_capacity: usize) { - self.map.shrink_to(min_capacity) + self.map.shrink_to(min_capacity); } /// Visits the values representing the difference, @@ -473,7 +654,7 @@ where /// assert_eq!(diff, [4].iter().collect()); /// ``` #[cfg_attr(feature = "inline-more", inline)] - pub fn difference<'a>(&'a self, other: &'a Self) -> Difference<'a, T, S> { + pub fn difference<'a>(&'a self, other: &'a Self) -> Difference<'a, T, S, A> { Difference { iter: self.iter(), other, @@ -502,7 +683,7 @@ where /// assert_eq!(diff1, [1, 4].iter().collect()); /// ``` #[cfg_attr(feature = "inline-more", inline)] - pub fn symmetric_difference<'a>(&'a self, other: &'a Self) -> SymmetricDifference<'a, T, S> { + pub fn symmetric_difference<'a>(&'a self, other: &'a Self) -> SymmetricDifference<'a, T, S, A> { SymmetricDifference { iter: self.difference(other).chain(other.difference(self)), } @@ -527,7 +708,7 @@ where /// assert_eq!(intersection, [2, 3].iter().collect()); /// ``` #[cfg_attr(feature = "inline-more", inline)] - pub fn intersection<'a>(&'a self, other: &'a Self) -> Intersection<'a, T, S> { + pub fn intersection<'a>(&'a self, other: &'a Self) -> Intersection<'a, T, S, A> { let (smaller, larger) = if self.len() <= other.len() { (self, other) } else { @@ -558,7 +739,9 @@ where /// assert_eq!(union, [1, 2, 3, 4].iter().collect()); /// ``` #[cfg_attr(feature = "inline-more", inline)] - pub fn union<'a>(&'a self, other: &'a Self) -> Union<'a, T, S> { + pub fn union<'a>(&'a self, other: &'a Self) -> Union<'a, T, S, A> { + // We'll iterate one set in full, and only the remaining difference from the other. + // Use the smaller set for the difference in order to reduce hash lookups. let (smaller, larger) = if self.len() <= other.len() { (self, other) } else { @@ -620,7 +803,11 @@ where T: Borrow, Q: Hash + Eq, { - self.map.get_key_value(value).map(|(k, _)| k) + // Avoid `Option::map` because it bloats LLVM IR. + match self.map.get_key_value(value) { + Some((k, _)) => Some(k), + None => None, + } } /// Inserts the given `value` into the set if it is not present, then @@ -648,6 +835,39 @@ where .0 } + /// Inserts an owned copy of the given `value` into the set if it is not + /// present, then returns a reference to the value in the set. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashSet; + /// + /// let mut set: HashSet = ["cat", "dog", "horse"] + /// .iter().map(|&pet| pet.to_owned()).collect(); + /// + /// assert_eq!(set.len(), 3); + /// for &pet in &["cat", "dog", "fish"] { + /// let value = set.get_or_insert_owned(pet); + /// assert_eq!(value, pet); + /// } + /// assert_eq!(set.len(), 4); // a new "fish" was inserted + /// ``` + #[inline] + pub fn get_or_insert_owned(&mut self, value: &Q) -> &T + where + T: Borrow, + Q: Hash + Eq + ToOwned, + { + // Although the raw entry gives us `&mut T`, we only return `&T` to be consistent with + // `get`. Key mutation is "raw" because you're not supposed to affect `Eq` or `Hash`. + self.map + .raw_entry_mut() + .from_key(value) + .or_insert_with(|| (value.to_owned(), ())) + .0 + } + /// Inserts a value computed from `f` into the set if the given `value` is /// not present, then returns a reference to the value in the set. /// @@ -682,6 +902,47 @@ where .0 } + /// Gets the given value's corresponding entry in the set for in-place manipulation. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashSet; + /// use hashbrown::hash_set::Entry::*; + /// + /// let mut singles = HashSet::new(); + /// let mut dupes = HashSet::new(); + /// + /// for ch in "a short treatise on fungi".chars() { + /// if let Vacant(dupe_entry) = dupes.entry(ch) { + /// // We haven't already seen a duplicate, so + /// // check if we've at least seen it once. + /// match singles.entry(ch) { + /// Vacant(single_entry) => { + /// // We found a new character for the first time. + /// single_entry.insert() + /// } + /// Occupied(single_entry) => { + /// // We've already seen this once, "move" it to dupes. + /// single_entry.remove(); + /// dupe_entry.insert(); + /// } + /// } + /// } + /// } + /// + /// assert!(!singles.contains(&'t') && dupes.contains(&'t')); + /// assert!(singles.contains(&'u') && !dupes.contains(&'u')); + /// assert!(!singles.contains(&'v') && !dupes.contains(&'v')); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn entry(&mut self, value: T) -> Entry<'_, T, S, A> { + match self.map.entry(value) { + map::Entry::Occupied(entry) => Entry::Occupied(OccupiedEntry { inner: entry }), + map::Entry::Vacant(entry) => Entry::Vacant(VacantEntry { inner: entry }), + } + } + /// Returns `true` if `self` has no elements in common with `other`. /// This is equivalent to checking for an empty intersection. /// @@ -771,6 +1032,30 @@ where self.map.insert(value, ()).is_none() } + /// Insert a value the set without checking if the value already exists in the set. + /// + /// Returns a reference to the value just inserted. + /// + /// This operation is safe if a value does not exist in the set. + /// + /// However, if a value exists in the set already, the behavior is unspecified: + /// this operation may panic, loop forever, or any following operation with the set + /// may panic, loop forever or return arbitrary result. + /// + /// That said, this operation (and following operations) are guaranteed to + /// not violate memory safety. + /// + /// This operation is faster than regular insert, because it does not perform + /// lookup before insertion. + /// + /// This operation is useful during initial population of the set. + /// For example, when constructing a set from another set, we know + /// that values are unique. + #[cfg_attr(feature = "inline-more", inline)] + pub fn insert_unique_unchecked(&mut self, value: T) -> &T { + self.map.insert_unique_unchecked(value, ()).0 + } + /// Adds a value to the set, replacing the existing value, if any, that is equal to the given /// one. Returns the replaced value. /// @@ -851,35 +1136,19 @@ where T: Borrow, Q: Hash + Eq, { - self.map.remove_entry(value).map(|(k, _)| k) - } - - /// Retains only the elements specified by the predicate. - /// - /// In other words, remove all elements `e` such that `f(&e)` returns `false`. - /// - /// # Examples - /// - /// ``` - /// use hashbrown::HashSet; - /// - /// let xs = [1,2,3,4,5,6]; - /// let mut set: HashSet = xs.iter().cloned().collect(); - /// set.retain(|&k| k % 2 == 0); - /// assert_eq!(set.len(), 3); - /// ``` - pub fn retain(&mut self, mut f: F) - where - F: FnMut(&T) -> bool, - { - self.map.retain(|k, _| f(k)); + // Avoid `Option::map` because it bloats LLVM IR. + match self.map.remove_entry(value) { + Some((k, _)) => Some(k), + None => None, + } } } -impl PartialEq for HashSet +impl PartialEq for HashSet where T: Eq + Hash, S: BuildHasher, + A: Allocator + Clone, { fn eq(&self, other: &Self) -> bool { if self.len() != other.len() { @@ -890,62 +1159,120 @@ where } } -impl Eq for HashSet +impl Eq for HashSet where T: Eq + Hash, S: BuildHasher, + A: Allocator + Clone, { } -impl fmt::Debug for HashSet +impl fmt::Debug for HashSet where - T: Eq + Hash + fmt::Debug, - S: BuildHasher, + T: fmt::Debug, + A: Allocator + Clone, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_set().entries(self.iter()).finish() } } -impl FromIterator for HashSet +impl From> for HashSet +where + A: Allocator + Clone, +{ + fn from(map: HashMap) -> Self { + Self { map } + } +} + +impl FromIterator for HashSet where T: Eq + Hash, S: BuildHasher + Default, + A: Default + Allocator + Clone, { #[cfg_attr(feature = "inline-more", inline)] fn from_iter>(iter: I) -> Self { - let mut set = Self::with_hasher(Default::default()); + let mut set = Self::with_hasher_in(Default::default(), Default::default()); set.extend(iter); set } } -impl Extend for HashSet +// The default hasher is used to match the std implementation signature +#[cfg(feature = "ahash")] +impl From<[T; N]> for HashSet +where + T: Eq + Hash, + A: Default + Allocator + Clone, +{ + /// # Examples + /// + /// ``` + /// use hashbrown::HashSet; + /// + /// let set1 = HashSet::from([1, 2, 3, 4]); + /// let set2: HashSet<_> = [1, 2, 3, 4].into(); + /// assert_eq!(set1, set2); + /// ``` + fn from(arr: [T; N]) -> Self { + arr.into_iter().collect() + } +} + +impl Extend for HashSet where T: Eq + Hash, S: BuildHasher, + A: Allocator + Clone, { #[cfg_attr(feature = "inline-more", inline)] fn extend>(&mut self, iter: I) { self.map.extend(iter.into_iter().map(|k| (k, ()))); } + + #[inline] + #[cfg(feature = "nightly")] + fn extend_one(&mut self, k: T) { + self.map.insert(k, ()); + } + + #[inline] + #[cfg(feature = "nightly")] + fn extend_reserve(&mut self, additional: usize) { + Extend::<(T, ())>::extend_reserve(&mut self.map, additional); + } } -impl<'a, T, S> Extend<&'a T> for HashSet +impl<'a, T, S, A> Extend<&'a T> for HashSet where T: 'a + Eq + Hash + Copy, S: BuildHasher, + A: Allocator + Clone, { #[cfg_attr(feature = "inline-more", inline)] fn extend>(&mut self, iter: I) { - self.extend(iter.into_iter().cloned()); + self.extend(iter.into_iter().copied()); + } + + #[inline] + #[cfg(feature = "nightly")] + fn extend_one(&mut self, k: &'a T) { + self.map.insert(*k, ()); + } + + #[inline] + #[cfg(feature = "nightly")] + fn extend_reserve(&mut self, additional: usize) { + Extend::<(T, ())>::extend_reserve(&mut self.map, additional); } } -impl Default for HashSet +impl Default for HashSet where - T: Eq + Hash, - S: BuildHasher + Default, + S: Default, + A: Default + Allocator + Clone, { /// Creates an empty `HashSet` with the `Default` value for the hasher. #[cfg_attr(feature = "inline-more", inline)] @@ -956,10 +1283,11 @@ where } } -impl BitOr<&HashSet> for &HashSet +impl BitOr<&HashSet> for &HashSet where T: Eq + Hash + Clone, S: BuildHasher + Default, + A: Allocator + Clone, { type Output = HashSet; @@ -983,15 +1311,16 @@ where /// } /// assert_eq!(i, expected.len()); /// ``` - fn bitor(self, rhs: &HashSet) -> HashSet { + fn bitor(self, rhs: &HashSet) -> HashSet { self.union(rhs).cloned().collect() } } -impl BitAnd<&HashSet> for &HashSet +impl BitAnd<&HashSet> for &HashSet where T: Eq + Hash + Clone, S: BuildHasher + Default, + A: Allocator + Clone, { type Output = HashSet; @@ -1015,7 +1344,7 @@ where /// } /// assert_eq!(i, expected.len()); /// ``` - fn bitand(self, rhs: &HashSet) -> HashSet { + fn bitand(self, rhs: &HashSet) -> HashSet { self.intersection(rhs).cloned().collect() } } @@ -1097,13 +1426,13 @@ pub struct Iter<'a, K> { /// An owning iterator over the items of a `HashSet`. /// -/// This `struct` is created by the [`into_iter`] method on [`HashSet`][`HashSet`] +/// This `struct` is created by the [`into_iter`] method on [`HashSet`] /// (provided by the `IntoIterator` trait). See its documentation for more. /// /// [`HashSet`]: struct.HashSet.html /// [`into_iter`]: struct.HashSet.html#method.into_iter -pub struct IntoIter { - iter: map::IntoIter, +pub struct IntoIter { + iter: map::IntoIter, } /// A draining iterator over the items of a `HashSet`. @@ -1113,8 +1442,23 @@ pub struct IntoIter { /// /// [`HashSet`]: struct.HashSet.html /// [`drain`]: struct.HashSet.html#method.drain -pub struct Drain<'a, K> { - iter: map::Drain<'a, K, ()>, +pub struct Drain<'a, K, A: Allocator + Clone = Global> { + iter: map::Drain<'a, K, (), A>, +} + +/// A draining iterator over entries of a `HashSet` which don't satisfy the predicate `f`. +/// +/// This `struct` is created by the [`drain_filter`] method on [`HashSet`]. See its +/// documentation for more. +/// +/// [`drain_filter`]: struct.HashSet.html#method.drain_filter +/// [`HashSet`]: struct.HashSet.html +pub struct DrainFilter<'a, K, F, A: Allocator + Clone = Global> +where + F: FnMut(&K) -> bool, +{ + f: F, + inner: DrainFilterInner<'a, K, (), A>, } /// A lazy iterator producing elements in the intersection of `HashSet`s. @@ -1124,11 +1468,11 @@ pub struct Drain<'a, K> { /// /// [`HashSet`]: struct.HashSet.html /// [`intersection`]: struct.HashSet.html#method.intersection -pub struct Intersection<'a, T, S> { +pub struct Intersection<'a, T, S, A: Allocator + Clone = Global> { // iterator of the first set iter: Iter<'a, T>, // the second set - other: &'a HashSet, + other: &'a HashSet, } /// A lazy iterator producing elements in the difference of `HashSet`s. @@ -1138,11 +1482,11 @@ pub struct Intersection<'a, T, S> { /// /// [`HashSet`]: struct.HashSet.html /// [`difference`]: struct.HashSet.html#method.difference -pub struct Difference<'a, T, S> { +pub struct Difference<'a, T, S, A: Allocator + Clone = Global> { // iterator of the first set iter: Iter<'a, T>, // the second set - other: &'a HashSet, + other: &'a HashSet, } /// A lazy iterator producing elements in the symmetric difference of `HashSet`s. @@ -1152,8 +1496,8 @@ pub struct Difference<'a, T, S> { /// /// [`HashSet`]: struct.HashSet.html /// [`symmetric_difference`]: struct.HashSet.html#method.symmetric_difference -pub struct SymmetricDifference<'a, T, S> { - iter: Chain, Difference<'a, T, S>>, +pub struct SymmetricDifference<'a, T, S, A: Allocator + Clone = Global> { + iter: Chain, Difference<'a, T, S, A>>, } /// A lazy iterator producing elements in the union of `HashSet`s. @@ -1163,11 +1507,11 @@ pub struct SymmetricDifference<'a, T, S> { /// /// [`HashSet`]: struct.HashSet.html /// [`union`]: struct.HashSet.html#method.union -pub struct Union<'a, T, S> { - iter: Chain, Difference<'a, T, S>>, +pub struct Union<'a, T, S, A: Allocator + Clone = Global> { + iter: Chain, Difference<'a, T, S, A>>, } -impl<'a, T, S> IntoIterator for &'a HashSet { +impl<'a, T, S, A: Allocator + Clone> IntoIterator for &'a HashSet { type Item = &'a T; type IntoIter = Iter<'a, T>; @@ -1177,9 +1521,9 @@ impl<'a, T, S> IntoIterator for &'a HashSet { } } -impl IntoIterator for HashSet { +impl IntoIterator for HashSet { type Item = T; - type IntoIter = IntoIter; + type IntoIter = IntoIter; /// Creates a consuming iterator, that is, one that moves each value out /// of the set in arbitrary order. The set cannot be used after calling @@ -1202,7 +1546,7 @@ impl IntoIterator for HashSet { /// } /// ``` #[cfg_attr(feature = "inline-more", inline)] - fn into_iter(self) -> IntoIter { + fn into_iter(self) -> IntoIter { IntoIter { iter: self.map.into_iter(), } @@ -1243,61 +1587,107 @@ impl fmt::Debug for Iter<'_, K> { } } -impl Iterator for IntoIter { +impl Iterator for IntoIter { type Item = K; #[cfg_attr(feature = "inline-more", inline)] fn next(&mut self) -> Option { - self.iter.next().map(|(k, _)| k) + // Avoid `Option::map` because it bloats LLVM IR. + match self.iter.next() { + Some((k, _)) => Some(k), + None => None, + } } #[cfg_attr(feature = "inline-more", inline)] fn size_hint(&self) -> (usize, Option) { self.iter.size_hint() } } -impl ExactSizeIterator for IntoIter { +impl ExactSizeIterator for IntoIter { #[cfg_attr(feature = "inline-more", inline)] fn len(&self) -> usize { self.iter.len() } } -impl FusedIterator for IntoIter {} +impl FusedIterator for IntoIter {} -impl fmt::Debug for IntoIter { +impl fmt::Debug for IntoIter { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let entries_iter = self.iter.iter().map(|(k, _)| k); f.debug_list().entries(entries_iter).finish() } } -impl Iterator for Drain<'_, K> { +impl Iterator for Drain<'_, K, A> { type Item = K; #[cfg_attr(feature = "inline-more", inline)] fn next(&mut self) -> Option { - self.iter.next().map(|(k, _)| k) + // Avoid `Option::map` because it bloats LLVM IR. + match self.iter.next() { + Some((k, _)) => Some(k), + None => None, + } } #[cfg_attr(feature = "inline-more", inline)] fn size_hint(&self) -> (usize, Option) { self.iter.size_hint() } } -impl ExactSizeIterator for Drain<'_, K> { +impl ExactSizeIterator for Drain<'_, K, A> { #[cfg_attr(feature = "inline-more", inline)] fn len(&self) -> usize { self.iter.len() } } -impl FusedIterator for Drain<'_, K> {} +impl FusedIterator for Drain<'_, K, A> {} -impl fmt::Debug for Drain<'_, K> { +impl fmt::Debug for Drain<'_, K, A> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let entries_iter = self.iter.iter().map(|(k, _)| k); f.debug_list().entries(entries_iter).finish() } } -impl Clone for Intersection<'_, T, S> { +impl<'a, K, F, A: Allocator + Clone> Drop for DrainFilter<'a, K, F, A> +where + F: FnMut(&K) -> bool, +{ + #[cfg_attr(feature = "inline-more", inline)] + fn drop(&mut self) { + while let Some(item) = self.next() { + let guard = ConsumeAllOnDrop(self); + drop(item); + mem::forget(guard); + } + } +} + +impl Iterator for DrainFilter<'_, K, F, A> +where + F: FnMut(&K) -> bool, +{ + type Item = K; + + #[cfg_attr(feature = "inline-more", inline)] + fn next(&mut self) -> Option { + let f = &mut self.f; + let (k, _) = self.inner.next(&mut |k, _| f(k))?; + Some(k) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + (0, self.inner.iter.size_hint().1) + } +} + +impl FusedIterator for DrainFilter<'_, K, F, A> where + F: FnMut(&K) -> bool +{ +} + +impl Clone for Intersection<'_, T, S, A> { #[cfg_attr(feature = "inline-more", inline)] fn clone(&self) -> Self { Intersection { @@ -1307,10 +1697,11 @@ impl Clone for Intersection<'_, T, S> { } } -impl<'a, T, S> Iterator for Intersection<'a, T, S> +impl<'a, T, S, A> Iterator for Intersection<'a, T, S, A> where T: Eq + Hash, S: BuildHasher, + A: Allocator + Clone, { type Item = &'a T; @@ -1331,24 +1722,26 @@ where } } -impl fmt::Debug for Intersection<'_, T, S> +impl fmt::Debug for Intersection<'_, T, S, A> where T: fmt::Debug + Eq + Hash, S: BuildHasher, + A: Allocator + Clone, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_list().entries(self.clone()).finish() } } -impl FusedIterator for Intersection<'_, T, S> +impl FusedIterator for Intersection<'_, T, S, A> where T: Eq + Hash, S: BuildHasher, + A: Allocator + Clone, { } -impl Clone for Difference<'_, T, S> { +impl Clone for Difference<'_, T, S, A> { #[cfg_attr(feature = "inline-more", inline)] fn clone(&self) -> Self { Difference { @@ -1358,10 +1751,11 @@ impl Clone for Difference<'_, T, S> { } } -impl<'a, T, S> Iterator for Difference<'a, T, S> +impl<'a, T, S, A> Iterator for Difference<'a, T, S, A> where T: Eq + Hash, S: BuildHasher, + A: Allocator + Clone, { type Item = &'a T; @@ -1382,24 +1776,26 @@ where } } -impl FusedIterator for Difference<'_, T, S> +impl FusedIterator for Difference<'_, T, S, A> where T: Eq + Hash, S: BuildHasher, + A: Allocator + Clone, { } -impl fmt::Debug for Difference<'_, T, S> +impl fmt::Debug for Difference<'_, T, S, A> where T: fmt::Debug + Eq + Hash, S: BuildHasher, + A: Allocator + Clone, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_list().entries(self.clone()).finish() } } -impl Clone for SymmetricDifference<'_, T, S> { +impl Clone for SymmetricDifference<'_, T, S, A> { #[cfg_attr(feature = "inline-more", inline)] fn clone(&self) -> Self { SymmetricDifference { @@ -1408,10 +1804,11 @@ impl Clone for SymmetricDifference<'_, T, S> { } } -impl<'a, T, S> Iterator for SymmetricDifference<'a, T, S> +impl<'a, T, S, A> Iterator for SymmetricDifference<'a, T, S, A> where T: Eq + Hash, S: BuildHasher, + A: Allocator + Clone, { type Item = &'a T; @@ -1425,24 +1822,26 @@ where } } -impl FusedIterator for SymmetricDifference<'_, T, S> +impl FusedIterator for SymmetricDifference<'_, T, S, A> where T: Eq + Hash, S: BuildHasher, + A: Allocator + Clone, { } -impl fmt::Debug for SymmetricDifference<'_, T, S> +impl fmt::Debug for SymmetricDifference<'_, T, S, A> where T: fmt::Debug + Eq + Hash, S: BuildHasher, + A: Allocator + Clone, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_list().entries(self.clone()).finish() } } -impl Clone for Union<'_, T, S> { +impl Clone for Union<'_, T, S, A> { #[cfg_attr(feature = "inline-more", inline)] fn clone(&self) -> Self { Union { @@ -1451,27 +1850,30 @@ impl Clone for Union<'_, T, S> { } } -impl FusedIterator for Union<'_, T, S> +impl FusedIterator for Union<'_, T, S, A> where T: Eq + Hash, S: BuildHasher, + A: Allocator + Clone, { } -impl fmt::Debug for Union<'_, T, S> +impl fmt::Debug for Union<'_, T, S, A> where T: fmt::Debug + Eq + Hash, S: BuildHasher, + A: Allocator + Clone, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_list().entries(self.clone()).finish() } } -impl<'a, T, S> Iterator for Union<'a, T, S> +impl<'a, T, S, A> Iterator for Union<'a, T, S, A> where T: Eq + Hash, S: BuildHasher, + A: Allocator + Clone, { type Item = &'a T; @@ -1485,6 +1887,406 @@ where } } +/// A view into a single entry in a set, which may either be vacant or occupied. +/// +/// This `enum` is constructed from the [`entry`] method on [`HashSet`]. +/// +/// [`HashSet`]: struct.HashSet.html +/// [`entry`]: struct.HashSet.html#method.entry +/// +/// # Examples +/// +/// ``` +/// use hashbrown::hash_set::{Entry, HashSet, OccupiedEntry}; +/// +/// let mut set = HashSet::new(); +/// set.extend(["a", "b", "c"]); +/// assert_eq!(set.len(), 3); +/// +/// // Existing value (insert) +/// let entry: Entry<_, _> = set.entry("a"); +/// let _raw_o: OccupiedEntry<_, _> = entry.insert(); +/// assert_eq!(set.len(), 3); +/// // Nonexistent value (insert) +/// set.entry("d").insert(); +/// +/// // Existing value (or_insert) +/// set.entry("b").or_insert(); +/// // Nonexistent value (or_insert) +/// set.entry("e").or_insert(); +/// +/// println!("Our HashSet: {:?}", set); +/// +/// let mut vec: Vec<_> = set.iter().copied().collect(); +/// // The `Iter` iterator produces items in arbitrary order, so the +/// // items must be sorted to test them against a sorted array. +/// vec.sort_unstable(); +/// assert_eq!(vec, ["a", "b", "c", "d", "e"]); +/// ``` +pub enum Entry<'a, T, S, A = Global> +where + A: Allocator + Clone, +{ + /// An occupied entry. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_set::{Entry, HashSet}; + /// let mut set: HashSet<_> = ["a", "b"].into(); + /// + /// match set.entry("a") { + /// Entry::Vacant(_) => unreachable!(), + /// Entry::Occupied(_) => { } + /// } + /// ``` + Occupied(OccupiedEntry<'a, T, S, A>), + + /// A vacant entry. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_set::{Entry, HashSet}; + /// let mut set: HashSet<&str> = HashSet::new(); + /// + /// match set.entry("a") { + /// Entry::Occupied(_) => unreachable!(), + /// Entry::Vacant(_) => { } + /// } + /// ``` + Vacant(VacantEntry<'a, T, S, A>), +} + +impl fmt::Debug for Entry<'_, T, S, A> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + Entry::Vacant(ref v) => f.debug_tuple("Entry").field(v).finish(), + Entry::Occupied(ref o) => f.debug_tuple("Entry").field(o).finish(), + } + } +} + +/// A view into an occupied entry in a `HashSet`. +/// It is part of the [`Entry`] enum. +/// +/// [`Entry`]: enum.Entry.html +/// +/// # Examples +/// +/// ``` +/// use hashbrown::hash_set::{Entry, HashSet, OccupiedEntry}; +/// +/// let mut set = HashSet::new(); +/// set.extend(["a", "b", "c"]); +/// +/// let _entry_o: OccupiedEntry<_, _> = set.entry("a").insert(); +/// assert_eq!(set.len(), 3); +/// +/// // Existing key +/// match set.entry("a") { +/// Entry::Vacant(_) => unreachable!(), +/// Entry::Occupied(view) => { +/// assert_eq!(view.get(), &"a"); +/// } +/// } +/// +/// assert_eq!(set.len(), 3); +/// +/// // Existing key (take) +/// match set.entry("c") { +/// Entry::Vacant(_) => unreachable!(), +/// Entry::Occupied(view) => { +/// assert_eq!(view.remove(), "c"); +/// } +/// } +/// assert_eq!(set.get(&"c"), None); +/// assert_eq!(set.len(), 2); +/// ``` +pub struct OccupiedEntry<'a, T, S, A: Allocator + Clone = Global> { + inner: map::OccupiedEntry<'a, T, (), S, A>, +} + +impl fmt::Debug for OccupiedEntry<'_, T, S, A> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OccupiedEntry") + .field("value", self.get()) + .finish() + } +} + +/// A view into a vacant entry in a `HashSet`. +/// It is part of the [`Entry`] enum. +/// +/// [`Entry`]: enum.Entry.html +/// +/// # Examples +/// +/// ``` +/// use hashbrown::hash_set::{Entry, HashSet, VacantEntry}; +/// +/// let mut set = HashSet::<&str>::new(); +/// +/// let entry_v: VacantEntry<_, _> = match set.entry("a") { +/// Entry::Vacant(view) => view, +/// Entry::Occupied(_) => unreachable!(), +/// }; +/// entry_v.insert(); +/// assert!(set.contains("a") && set.len() == 1); +/// +/// // Nonexistent key (insert) +/// match set.entry("b") { +/// Entry::Vacant(view) => view.insert(), +/// Entry::Occupied(_) => unreachable!(), +/// } +/// assert!(set.contains("b") && set.len() == 2); +/// ``` +pub struct VacantEntry<'a, T, S, A: Allocator + Clone = Global> { + inner: map::VacantEntry<'a, T, (), S, A>, +} + +impl fmt::Debug for VacantEntry<'_, T, S, A> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("VacantEntry").field(self.get()).finish() + } +} + +impl<'a, T, S, A: Allocator + Clone> Entry<'a, T, S, A> { + /// Sets the value of the entry, and returns an OccupiedEntry. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashSet; + /// + /// let mut set: HashSet<&str> = HashSet::new(); + /// let entry = set.entry("horseyland").insert(); + /// + /// assert_eq!(entry.get(), &"horseyland"); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn insert(self) -> OccupiedEntry<'a, T, S, A> + where + T: Hash, + S: BuildHasher, + { + match self { + Entry::Occupied(entry) => entry, + Entry::Vacant(entry) => entry.insert_entry(), + } + } + + /// Ensures a value is in the entry by inserting if it was vacant. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashSet; + /// + /// let mut set: HashSet<&str> = HashSet::new(); + /// + /// // nonexistent key + /// set.entry("poneyland").or_insert(); + /// assert!(set.contains("poneyland")); + /// + /// // existing key + /// set.entry("poneyland").or_insert(); + /// assert!(set.contains("poneyland")); + /// assert_eq!(set.len(), 1); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn or_insert(self) + where + T: Hash, + S: BuildHasher, + { + if let Entry::Vacant(entry) = self { + entry.insert(); + } + } + + /// Returns a reference to this entry's value. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashSet; + /// + /// let mut set: HashSet<&str> = HashSet::new(); + /// set.entry("poneyland").or_insert(); + /// // existing key + /// assert_eq!(set.entry("poneyland").get(), &"poneyland"); + /// // nonexistent key + /// assert_eq!(set.entry("horseland").get(), &"horseland"); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn get(&self) -> &T { + match *self { + Entry::Occupied(ref entry) => entry.get(), + Entry::Vacant(ref entry) => entry.get(), + } + } +} + +impl OccupiedEntry<'_, T, S, A> { + /// Gets a reference to the value in the entry. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_set::{Entry, HashSet}; + /// + /// let mut set: HashSet<&str> = HashSet::new(); + /// set.entry("poneyland").or_insert(); + /// + /// match set.entry("poneyland") { + /// Entry::Vacant(_) => panic!(), + /// Entry::Occupied(entry) => assert_eq!(entry.get(), &"poneyland"), + /// } + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn get(&self) -> &T { + self.inner.key() + } + + /// Takes the value out of the entry, and returns it. + /// Keeps the allocated memory for reuse. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashSet; + /// use hashbrown::hash_set::Entry; + /// + /// let mut set: HashSet<&str> = HashSet::new(); + /// // The set is empty + /// assert!(set.is_empty() && set.capacity() == 0); + /// + /// set.entry("poneyland").or_insert(); + /// let capacity_before_remove = set.capacity(); + /// + /// if let Entry::Occupied(o) = set.entry("poneyland") { + /// assert_eq!(o.remove(), "poneyland"); + /// } + /// + /// assert_eq!(set.contains("poneyland"), false); + /// // Now set hold none elements but capacity is equal to the old one + /// assert!(set.len() == 0 && set.capacity() == capacity_before_remove); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn remove(self) -> T { + self.inner.remove_entry().0 + } + + /// Replaces the entry, returning the old value. The new value in the hash map will be + /// the value used to create this entry. + /// + /// # Panics + /// + /// Will panic if this OccupiedEntry was created through [`Entry::insert`]. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_set::{Entry, HashSet}; + /// use std::rc::Rc; + /// + /// let mut set: HashSet> = HashSet::new(); + /// let key_one = Rc::new("Stringthing".to_string()); + /// let key_two = Rc::new("Stringthing".to_string()); + /// + /// set.insert(key_one.clone()); + /// assert!(Rc::strong_count(&key_one) == 2 && Rc::strong_count(&key_two) == 1); + /// + /// match set.entry(key_two.clone()) { + /// Entry::Occupied(entry) => { + /// let old_key: Rc = entry.replace(); + /// assert!(Rc::ptr_eq(&key_one, &old_key)); + /// } + /// Entry::Vacant(_) => panic!(), + /// } + /// + /// assert!(Rc::strong_count(&key_one) == 1 && Rc::strong_count(&key_two) == 2); + /// assert!(set.contains(&"Stringthing".to_owned())); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn replace(self) -> T { + self.inner.replace_key() + } +} + +impl<'a, T, S, A: Allocator + Clone> VacantEntry<'a, T, S, A> { + /// Gets a reference to the value that would be used when inserting + /// through the `VacantEntry`. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashSet; + /// + /// let mut set: HashSet<&str> = HashSet::new(); + /// assert_eq!(set.entry("poneyland").get(), &"poneyland"); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn get(&self) -> &T { + self.inner.key() + } + + /// Take ownership of the value. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::hash_set::{Entry, HashSet}; + /// + /// let mut set: HashSet<&str> = HashSet::new(); + /// + /// match set.entry("poneyland") { + /// Entry::Occupied(_) => panic!(), + /// Entry::Vacant(v) => assert_eq!(v.into_value(), "poneyland"), + /// } + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn into_value(self) -> T { + self.inner.into_key() + } + + /// Sets the value of the entry with the VacantEntry's value. + /// + /// # Examples + /// + /// ``` + /// use hashbrown::HashSet; + /// use hashbrown::hash_set::Entry; + /// + /// let mut set: HashSet<&str> = HashSet::new(); + /// + /// if let Entry::Vacant(o) = set.entry("poneyland") { + /// o.insert(); + /// } + /// assert!(set.contains("poneyland")); + /// ``` + #[cfg_attr(feature = "inline-more", inline)] + pub fn insert(self) + where + T: Hash, + S: BuildHasher, + { + self.inner.insert(()); + } + + #[cfg_attr(feature = "inline-more", inline)] + fn insert_entry(self) -> OccupiedEntry<'a, T, S, A> + where + T: Hash, + S: BuildHasher, + { + OccupiedEntry { + inner: self.inner.insert_entry(()), + } + } +} + #[allow(dead_code)] fn assert_covariance() { fn set<'new>(v: HashSet<&'static str>) -> HashSet<&'new str> { @@ -1493,30 +2295,34 @@ fn assert_covariance() { fn iter<'a, 'new>(v: Iter<'a, &'static str>) -> Iter<'a, &'new str> { v } - fn into_iter<'new>(v: IntoIter<&'static str>) -> IntoIter<&'new str> { + fn into_iter<'new, A: Allocator + Clone>( + v: IntoIter<&'static str, A>, + ) -> IntoIter<&'new str, A> { v } - fn difference<'a, 'new>( - v: Difference<'a, &'static str, DefaultHashBuilder>, - ) -> Difference<'a, &'new str, DefaultHashBuilder> { + fn difference<'a, 'new, A: Allocator + Clone>( + v: Difference<'a, &'static str, DefaultHashBuilder, A>, + ) -> Difference<'a, &'new str, DefaultHashBuilder, A> { v } - fn symmetric_difference<'a, 'new>( - v: SymmetricDifference<'a, &'static str, DefaultHashBuilder>, - ) -> SymmetricDifference<'a, &'new str, DefaultHashBuilder> { + fn symmetric_difference<'a, 'new, A: Allocator + Clone>( + v: SymmetricDifference<'a, &'static str, DefaultHashBuilder, A>, + ) -> SymmetricDifference<'a, &'new str, DefaultHashBuilder, A> { v } - fn intersection<'a, 'new>( - v: Intersection<'a, &'static str, DefaultHashBuilder>, - ) -> Intersection<'a, &'new str, DefaultHashBuilder> { + fn intersection<'a, 'new, A: Allocator + Clone>( + v: Intersection<'a, &'static str, DefaultHashBuilder, A>, + ) -> Intersection<'a, &'new str, DefaultHashBuilder, A> { v } - fn union<'a, 'new>( - v: Union<'a, &'static str, DefaultHashBuilder>, - ) -> Union<'a, &'new str, DefaultHashBuilder> { + fn union<'a, 'new, A: Allocator + Clone>( + v: Union<'a, &'static str, DefaultHashBuilder, A>, + ) -> Union<'a, &'new str, DefaultHashBuilder, A> { v } - fn drain<'new>(d: Drain<'static, &'static str>) -> Drain<'new, &'new str> { + fn drain<'new, A: Allocator + Clone>( + d: Drain<'static, &'static str, A>, + ) -> Drain<'new, &'new str, A> { d } } @@ -1648,7 +2454,7 @@ mod test_set { let expected = [3, 5, 11, 77]; for x in a.intersection(&b) { assert!(expected.contains(x)); - i += 1 + i += 1; } assert_eq!(i, expected.len()); } @@ -1671,7 +2477,7 @@ mod test_set { let expected = [1, 5, 11]; for x in a.difference(&b) { assert!(expected.contains(x)); - i += 1 + i += 1; } assert_eq!(i, expected.len()); } @@ -1697,7 +2503,7 @@ mod test_set { let expected = [-2, 1, 5, 11, 14, 22]; for x in a.symmetric_difference(&b) { assert!(expected.contains(x)); - i += 1 + i += 1; } assert_eq!(i, expected.len()); } @@ -1727,20 +2533,39 @@ mod test_set { let expected = [-2, 1, 3, 5, 9, 11, 13, 16, 19, 24]; for x in a.union(&b) { assert!(expected.contains(x)); - i += 1 + i += 1; } assert_eq!(i, expected.len()); } + #[test] + fn test_from_map() { + let mut a = crate::HashMap::new(); + a.insert(1, ()); + a.insert(2, ()); + a.insert(3, ()); + a.insert(4, ()); + + let a: HashSet<_> = a.into(); + + assert_eq!(a.len(), 4); + assert!(a.contains(&1)); + assert!(a.contains(&2)); + assert!(a.contains(&3)); + assert!(a.contains(&4)); + } + #[test] fn test_from_iter() { - let xs = [1, 2, 3, 4, 5, 6, 7, 8, 9]; + let xs = [1, 2, 2, 3, 4, 5, 6, 7, 8, 9]; - let set: HashSet<_> = xs.iter().cloned().collect(); + let set: HashSet<_> = xs.iter().copied().collect(); for x in &xs { assert!(set.contains(x)); } + + assert_eq!(set.iter().len(), xs.len() - 1); } #[test] @@ -1896,11 +2721,70 @@ mod test_set { #[test] fn test_retain() { let xs = [1, 2, 3, 4, 5, 6]; - let mut set: HashSet = xs.iter().cloned().collect(); + let mut set: HashSet = xs.iter().copied().collect(); set.retain(|&k| k % 2 == 0); assert_eq!(set.len(), 3); assert!(set.contains(&2)); assert!(set.contains(&4)); assert!(set.contains(&6)); } + + #[test] + fn test_drain_filter() { + { + let mut set: HashSet = (0..8).collect(); + let drained = set.drain_filter(|&k| k % 2 == 0); + let mut out = drained.collect::>(); + out.sort_unstable(); + assert_eq!(vec![0, 2, 4, 6], out); + assert_eq!(set.len(), 4); + } + { + let mut set: HashSet = (0..8).collect(); + drop(set.drain_filter(|&k| k % 2 == 0)); + assert_eq!(set.len(), 4, "Removes non-matching items on drop"); + } + } + + #[test] + fn test_const_with_hasher() { + use core::hash::BuildHasher; + use std::collections::hash_map::DefaultHasher; + + #[derive(Clone)] + struct MyHasher; + impl BuildHasher for MyHasher { + type Hasher = DefaultHasher; + + fn build_hasher(&self) -> DefaultHasher { + DefaultHasher::new() + } + } + + const EMPTY_SET: HashSet = HashSet::with_hasher(MyHasher); + + let mut set = EMPTY_SET; + set.insert(19); + assert!(set.contains(&19)); + } + + #[test] + fn rehash_in_place() { + let mut set = HashSet::new(); + + for i in 0..224 { + set.insert(i); + } + + assert_eq!( + set.capacity(), + 224, + "The set must be at or close to capacity to trigger a re hashing" + ); + + for i in 100..1400 { + set.remove(&(i - 100)); + set.insert(i); + } + } } diff --git a/crux-mir/lib/hashbrown/tests/rayon.rs b/crux-mir/lib/hashbrown/tests/rayon.rs index 39b47708d..8c603c5c4 100644 --- a/crux-mir/lib/hashbrown/tests/rayon.rs +++ b/crux-mir/lib/hashbrown/tests/rayon.rs @@ -269,20 +269,20 @@ fn map_seq_par_equivalence_existing_empty_extend_empty() { let mut map_seq = MAP_EXISTING_EMPTY.clone(); let mut map_par = MAP_EXISTING_EMPTY.clone(); - map_seq.extend(MAP_EXTENSION_EMPTY.iter().cloned()); - map_par.par_extend(MAP_EXTENSION_EMPTY.par_iter().cloned()); + map_seq.extend(MAP_EXTENSION_EMPTY.iter().copied()); + map_par.par_extend(MAP_EXTENSION_EMPTY.par_iter().copied()); assert_eq3!(map_seq, map_par, expected); } #[test] fn map_seq_par_equivalence_existing_empty_extend() { - let expected = MAP_EXTENSION.iter().cloned().collect::>(); + let expected = MAP_EXTENSION.iter().copied().collect::>(); let mut map_seq = MAP_EXISTING_EMPTY.clone(); let mut map_par = MAP_EXISTING_EMPTY.clone(); - map_seq.extend(MAP_EXTENSION.iter().cloned()); - map_par.par_extend(MAP_EXTENSION.par_iter().cloned()); + map_seq.extend(MAP_EXTENSION.iter().copied()); + map_par.par_extend(MAP_EXTENSION.par_iter().copied()); assert_eq3!(map_seq, map_par, expected); } @@ -293,8 +293,8 @@ fn map_seq_par_equivalence_existing_extend_empty() { let mut map_seq = MAP_EXISTING.clone(); let mut map_par = MAP_EXISTING.clone(); - map_seq.extend(MAP_EXTENSION_EMPTY.iter().cloned()); - map_par.par_extend(MAP_EXTENSION_EMPTY.par_iter().cloned()); + map_seq.extend(MAP_EXTENSION_EMPTY.iter().copied()); + map_par.par_extend(MAP_EXTENSION_EMPTY.par_iter().copied()); assert_eq3!(map_seq, map_par, expected); } @@ -305,8 +305,8 @@ fn map_seq_par_equivalence_existing_extend() { let mut map_seq = MAP_EXISTING.clone(); let mut map_par = MAP_EXISTING.clone(); - map_seq.extend(MAP_EXTENSION.iter().cloned()); - map_par.par_extend(MAP_EXTENSION.par_iter().cloned()); + map_seq.extend(MAP_EXTENSION.iter().copied()); + map_par.par_extend(MAP_EXTENSION.par_iter().copied()); assert_eq3!(map_seq, map_par, expected); } @@ -423,20 +423,20 @@ fn set_seq_par_equivalence_existing_empty_extend_empty() { let mut set_seq = SET_EXISTING_EMPTY.clone(); let mut set_par = SET_EXISTING_EMPTY.clone(); - set_seq.extend(SET_EXTENSION_EMPTY.iter().cloned()); - set_par.par_extend(SET_EXTENSION_EMPTY.par_iter().cloned()); + set_seq.extend(SET_EXTENSION_EMPTY.iter().copied()); + set_par.par_extend(SET_EXTENSION_EMPTY.par_iter().copied()); assert_eq3!(set_seq, set_par, expected); } #[test] fn set_seq_par_equivalence_existing_empty_extend() { - let expected = SET_EXTENSION.iter().cloned().collect::>(); + let expected = SET_EXTENSION.iter().copied().collect::>(); let mut set_seq = SET_EXISTING_EMPTY.clone(); let mut set_par = SET_EXISTING_EMPTY.clone(); - set_seq.extend(SET_EXTENSION.iter().cloned()); - set_par.par_extend(SET_EXTENSION.par_iter().cloned()); + set_seq.extend(SET_EXTENSION.iter().copied()); + set_par.par_extend(SET_EXTENSION.par_iter().copied()); assert_eq3!(set_seq, set_par, expected); } @@ -447,8 +447,8 @@ fn set_seq_par_equivalence_existing_extend_empty() { let mut set_seq = SET_EXISTING.clone(); let mut set_par = SET_EXISTING.clone(); - set_seq.extend(SET_EXTENSION_EMPTY.iter().cloned()); - set_par.par_extend(SET_EXTENSION_EMPTY.par_iter().cloned()); + set_seq.extend(SET_EXTENSION_EMPTY.iter().copied()); + set_par.par_extend(SET_EXTENSION_EMPTY.par_iter().copied()); assert_eq3!(set_seq, set_par, expected); } @@ -459,37 +459,37 @@ fn set_seq_par_equivalence_existing_extend() { let mut set_seq = SET_EXISTING.clone(); let mut set_par = SET_EXISTING.clone(); - set_seq.extend(SET_EXTENSION.iter().cloned()); - set_par.par_extend(SET_EXTENSION.par_iter().cloned()); + set_seq.extend(SET_EXTENSION.iter().copied()); + set_par.par_extend(SET_EXTENSION.par_iter().copied()); assert_eq3!(set_seq, set_par, expected); } lazy_static! { - static ref SET_A: HashSet = ['a', 'b', 'c', 'd'].iter().cloned().collect(); - static ref SET_B: HashSet = ['a', 'b', 'e', 'f'].iter().cloned().collect(); - static ref SET_DIFF_AB: HashSet = ['c', 'd'].iter().cloned().collect(); - static ref SET_DIFF_BA: HashSet = ['e', 'f'].iter().cloned().collect(); - static ref SET_SYMM_DIFF_AB: HashSet = ['c', 'd', 'e', 'f'].iter().cloned().collect(); - static ref SET_INTERSECTION_AB: HashSet = ['a', 'b'].iter().cloned().collect(); + static ref SET_A: HashSet = ['a', 'b', 'c', 'd'].iter().copied().collect(); + static ref SET_B: HashSet = ['a', 'b', 'e', 'f'].iter().copied().collect(); + static ref SET_DIFF_AB: HashSet = ['c', 'd'].iter().copied().collect(); + static ref SET_DIFF_BA: HashSet = ['e', 'f'].iter().copied().collect(); + static ref SET_SYMM_DIFF_AB: HashSet = ['c', 'd', 'e', 'f'].iter().copied().collect(); + static ref SET_INTERSECTION_AB: HashSet = ['a', 'b'].iter().copied().collect(); static ref SET_UNION_AB: HashSet = - ['a', 'b', 'c', 'd', 'e', 'f'].iter().cloned().collect(); + ['a', 'b', 'c', 'd', 'e', 'f'].iter().copied().collect(); } #[test] fn set_seq_par_equivalence_difference() { - let diff_ab_seq = SET_A.difference(&*SET_B).cloned().collect::>(); + let diff_ab_seq = SET_A.difference(&*SET_B).copied().collect::>(); let diff_ab_par = SET_A .par_difference(&*SET_B) - .cloned() + .copied() .collect::>(); assert_eq3!(diff_ab_seq, diff_ab_par, *SET_DIFF_AB); - let diff_ba_seq = SET_B.difference(&*SET_A).cloned().collect::>(); + let diff_ba_seq = SET_B.difference(&*SET_A).copied().collect::>(); let diff_ba_par = SET_B .par_difference(&*SET_A) - .cloned() + .copied() .collect::>(); assert_eq3!(diff_ba_seq, diff_ba_par, *SET_DIFF_BA); @@ -499,11 +499,11 @@ fn set_seq_par_equivalence_difference() { fn set_seq_par_equivalence_symmetric_difference() { let symm_diff_ab_seq = SET_A .symmetric_difference(&*SET_B) - .cloned() + .copied() .collect::>(); let symm_diff_ab_par = SET_A .par_symmetric_difference(&*SET_B) - .cloned() + .copied() .collect::>(); assert_eq3!(symm_diff_ab_seq, symm_diff_ab_par, *SET_SYMM_DIFF_AB); @@ -511,10 +511,10 @@ fn set_seq_par_equivalence_symmetric_difference() { #[test] fn set_seq_par_equivalence_intersection() { - let intersection_ab_seq = SET_A.intersection(&*SET_B).cloned().collect::>(); + let intersection_ab_seq = SET_A.intersection(&*SET_B).copied().collect::>(); let intersection_ab_par = SET_A .par_intersection(&*SET_B) - .cloned() + .copied() .collect::>(); assert_eq3!( @@ -526,8 +526,8 @@ fn set_seq_par_equivalence_intersection() { #[test] fn set_seq_par_equivalence_union() { - let union_ab_seq = SET_A.union(&*SET_B).cloned().collect::>(); - let union_ab_par = SET_A.par_union(&*SET_B).cloned().collect::>(); + let union_ab_seq = SET_A.union(&*SET_B).copied().collect::>(); + let union_ab_par = SET_A.par_union(&*SET_B).copied().collect::>(); assert_eq3!(union_ab_seq, union_ab_par, *SET_UNION_AB); } diff --git a/crux-mir/lib/hashbrown/tests/serde.rs b/crux-mir/lib/hashbrown/tests/serde.rs index 570bf70da..a642348b3 100644 --- a/crux-mir/lib/hashbrown/tests/serde.rs +++ b/crux-mir/lib/hashbrown/tests/serde.rs @@ -1,24 +1,24 @@ #![cfg(feature = "serde")] use core::hash::BuildHasherDefault; +use fnv::FnvHasher; use hashbrown::{HashMap, HashSet}; -use rustc_hash::FxHasher; use serde_test::{assert_tokens, Token}; -// We use FxHash for this test because we rely on the ordering -type FxHashMap = HashMap>; -type FxHashSet = HashSet>; +// We use FnvHash for this test because we rely on the ordering +type FnvHashMap = HashMap>; +type FnvHashSet = HashSet>; #[test] fn map_serde_tokens_empty() { - let map = FxHashMap::::default(); + let map = FnvHashMap::::default(); assert_tokens(&map, &[Token::Map { len: Some(0) }, Token::MapEnd]); } #[test] fn map_serde_tokens() { - let mut map = FxHashMap::default(); + let mut map = FnvHashMap::default(); map.insert('b', 20); map.insert('a', 10); map.insert('c', 30); @@ -29,10 +29,10 @@ fn map_serde_tokens() { Token::Map { len: Some(3) }, Token::Char('a'), Token::I32(10), - Token::Char('b'), - Token::I32(20), Token::Char('c'), Token::I32(30), + Token::Char('b'), + Token::I32(20), Token::MapEnd, ], ); @@ -40,14 +40,14 @@ fn map_serde_tokens() { #[test] fn set_serde_tokens_empty() { - let set = FxHashSet::::default(); + let set = FnvHashSet::::default(); assert_tokens(&set, &[Token::Seq { len: Some(0) }, Token::SeqEnd]); } #[test] fn set_serde_tokens() { - let mut set = FxHashSet::default(); + let mut set = FnvHashSet::default(); set.insert(20); set.insert(10); set.insert(30); @@ -56,9 +56,9 @@ fn set_serde_tokens() { &set, &[ Token::Seq { len: Some(3) }, + Token::I32(30), Token::I32(20), Token::I32(10), - Token::I32(30), Token::SeqEnd, ], ); diff --git a/crux-mir/lib/hashbrown/tests/set.rs b/crux-mir/lib/hashbrown/tests/set.rs index 3fc071705..5ae1ec98e 100644 --- a/crux-mir/lib/hashbrown/tests/set.rs +++ b/crux-mir/lib/hashbrown/tests/set.rs @@ -2,29 +2,33 @@ use hashbrown::HashSet; use rand::{distributions::Alphanumeric, rngs::SmallRng, Rng, SeedableRng}; +use std::iter; #[test] fn test_hashset_insert_remove() { let mut m: HashSet> = HashSet::new(); - //let num: u32 = 4096; - //let tx: Vec> = (0..num).map(|i| (i..(16 + i)).collect()).collect(); - let seed: [u8; 16] = [ - 130, 220, 246, 217, 111, 124, 221, 189, 190, 234, 121, 93, 67, 95, 100, 43, - ]; + let seed = u64::from_le_bytes(*b"testseed"); - let rng = &mut SmallRng::from_seed(seed); - let tx: Vec> = (0..4096) - .map(|_| (rng.sample_iter(&Alphanumeric).take(32).collect())) - .collect(); + let rng = &mut SmallRng::seed_from_u64(seed); + let tx: Vec> = iter::repeat_with(|| { + rng.sample_iter(&Alphanumeric) + .take(32) + .map(char::from) + .collect() + }) + .take(4096) + .collect(); + // more readable with explicit `true` / `false` + #[allow(clippy::bool_assert_comparison)] for _ in 0..32 { - for i in 0..4096 { - assert_eq!(m.contains(&tx[i].clone()), false); - assert_eq!(m.insert(tx[i].clone()), true); + for x in &tx { + assert_eq!(m.contains(x), false); + assert_eq!(m.insert(x.clone()), true); } - for i in 0..4096 { - println!("removing {} {:?}", i, tx[i]); - assert_eq!(m.remove(&tx[i]), true); + for (i, x) in tx.iter().enumerate() { + println!("removing {} {:?}", i, x); + assert_eq!(m.remove(x), true); } } } diff --git a/crux-mir/lib/int512.rs b/crux-mir/lib/int512.rs deleted file mode 100644 index dd3fb4b38..000000000 --- a/crux-mir/lib/int512.rs +++ /dev/null @@ -1,135 +0,0 @@ -#![no_std] - -extern crate core; -use core::ops::{Add, Sub, Mul, Div, Rem, BitAnd, BitOr, BitXor, Shl, Shr}; -use core::cmp::{Ord, PartialOrd, Ordering}; - -#[derive(Copy)] -pub struct Int512 { _dummy: u8 } - -pub fn clone(_i: &Int512) -> Int512 { unimplemented!() } -impl Clone for Int512 { - fn clone(&self) -> Int512 { clone(self) } -} - -pub fn symbolic(_x: &'static str) -> Int512 { unimplemented!() } -impl Int512 { - pub fn symbolic(x: &'static str) -> Int512 { symbolic(x) } -} - -macro_rules! binop { - ($Op:ident, $op:ident) => { - pub fn $op(_x: Int512, _y: Int512) -> Int512 { unimplemented!() } - - impl Int512 { - pub fn $op(self, other: Int512) -> Int512 { $op(self, other) } - } - - /* - impl $Op for Int512 { - type Output = Int512; - fn $op(self, other: Int512) -> Int512 { $op(self, other) } - } - */ - }; -} -binop!(Add, add); -binop!(Sub, sub); -binop!(Mul, mul); -binop!(Div, div); -binop!(Rem, rem); -binop!(BitAnd, bitand); -binop!(BitOr, bitor); -binop!(BitXor, bitxor); - -macro_rules! shift_op { - ($Op:ident, $op:ident) => { - pub fn $op(_x: Int512, _bits: u32) -> Int512 { unimplemented!() } - - impl Int512 { - pub fn $op(self, bits: u32) -> Int512 { $op(self, bits) } - } - - /* - impl $Op for Int512 { - type Output = Int512; - fn $op(self, bits: u32) -> Int512 { $op(self, bits) } - } - */ - }; -} -shift_op!(Shl, shl); -shift_op!(Shr, shr); - -pub fn eq(_x: Int512, _y: Int512) -> bool { unimplemented!() } -impl PartialEq for Int512 { - fn eq(&self, other: &Int512) -> bool { eq(*self, *other) } - fn ne(&self, other: &Int512) -> bool { !eq(*self, *other) } -} -impl Eq for Int512 {} - -pub fn lt(_x: Int512, _y: Int512) -> bool { unimplemented!() } -impl PartialOrd for Int512 { - fn partial_cmp(&self, other: &Int512) -> Option { - Some(self.cmp(other)) - } -} -impl Ord for Int512 { - fn cmp(&self, other: &Int512) -> Ordering { - if eq(*self, *other) { Ordering::Equal } - else if lt(*self, *other) { Ordering::Less } - else { Ordering::Greater } - } -} - -macro_rules! prim_cast { - ($Prim:ident) => { - pub mod $Prim { - use super::Int512; - pub fn from_prim(_x: $Prim) -> Int512 { unimplemented!() } - pub fn as_prim(_x: Int512) -> $Prim { unimplemented!() } - } - - impl From<$Prim> for Int512 { - fn from(x: $Prim) -> Int512 { self::$Prim::from_prim(x) } - } - - impl From for $Prim { - fn from(x: Int512) -> $Prim { self::$Prim::as_prim(x) } - } - }; -} -prim_cast!(u8); -prim_cast!(u16); -prim_cast!(u32); -prim_cast!(u64); -prim_cast!(u128); -prim_cast!(usize); -prim_cast!(i8); -prim_cast!(i16); -prim_cast!(i32); -prim_cast!(i64); -prim_cast!(i128); -prim_cast!(isize); - - -impl From<[u8; 32]> for Int512 { - fn from(x: [u8; 32]) -> Int512 { - let mut acc = Int512::from(0_i32); - for i in 0..5 { - acc = acc.bitor(Int512::from(x[i]).shl(8 * i as u32)); - } - acc - } -} - -impl From for [u8; 32] { - fn from(x: Int512) -> [u8; 32] { - let mut acc = [0; 32]; - let mask: Int512 = Int512::from((1_u64 << 8) - 1); - for i in 0..32 { - acc[i] = u8::from(x.shr(8 * i as u32).bitand(mask)); - } - acc - } -} diff --git a/crux-mir/lib/libc/.cargo-ok b/crux-mir/lib/libc/.cargo-ok new file mode 100644 index 000000000..b5754e203 --- /dev/null +++ b/crux-mir/lib/libc/.cargo-ok @@ -0,0 +1 @@ +ok \ No newline at end of file diff --git a/crux-mir/lib/libc/.cargo_vcs_info.json b/crux-mir/lib/libc/.cargo_vcs_info.json new file mode 100644 index 000000000..52c4e52dd --- /dev/null +++ b/crux-mir/lib/libc/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "ce5afa7b523fd114ea7baadb26d5b318017cde58" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/crux-mir/lib/libc/.gitignore b/crux-mir/lib/libc/.gitignore new file mode 100644 index 000000000..bbbad4bc5 --- /dev/null +++ b/crux-mir/lib/libc/.gitignore @@ -0,0 +1,4 @@ +target +Cargo.lock +*~ +style diff --git a/crux-mir/lib/libc/CONTRIBUTING.md b/crux-mir/lib/libc/CONTRIBUTING.md index 052777f41..8c551dbdb 100644 --- a/crux-mir/lib/libc/CONTRIBUTING.md +++ b/crux-mir/lib/libc/CONTRIBUTING.md @@ -28,17 +28,27 @@ at, fear not! This crate has CI support which tests any binding against all platforms supported, so you'll see failures if an API is added at the wrong level or has different signatures across platforms. +New symbol(s) (i.e. functions, constants etc.) should also be added to the +symbols list(s) found in the `libc-test/semver` directory. These lists keep +track of what symbols are public in the libc crate and ensures they remain +available between changes to the crate. If the new symbol(s) are available on +all supported Unixes it should be added to `unix.txt` list1, +otherwise they should be added to the OS specific list(s). + With that in mind, the steps for adding a new API are: 1. Determine where in the module hierarchy your API should be added. -2. Add the API. +2. Add the API, including adding new symbol(s) to the semver lists. 3. Send a PR to this repo. 4. Wait for CI to pass, fixing errors. 5. Wait for a merge! -### Test before you commit +1: Note that this list has nothing to do with any Unix or Posix +standard, it's just a list shared between all OSs that declare `#[cfg(unix)]`. + +## Test before you commit -We have two automated tests running on [Azure Pipelines](https://dev.azure.com/rust-lang2/libc/_build?definitionId=1&_a=summary): +We have two automated tests running on [GitHub Actions](https://github.com/rust-lang/libc/actions): 1. [`libc-test`](https://github.com/gnzlbg/ctest) - `cd libc-test && cargo test` @@ -46,21 +56,38 @@ We have two automated tests running on [Azure Pipelines](https://dev.azure.com/r 2. Style checker - `rustc ci/style.rs && ./style src` -### Releasing your change to crates.io +## Breaking change policy + +Sometimes an upstream adds a breaking change to their API e.g. removing outdated items, +changing the type signature, etc. And we probably should follow that change to build the +`libc` crate successfully. It's annoying to do the equivalent of semver-major versioning +for each such change. Instead, we mark the item as deprecated and do the actual change +after a certain period. The steps are: + +1. Add `#[deprecated(since = "", note="")]` attribute to the item. + - The `since` field should have a next version of `libc` + (e.g., if the current version is `0.2.1`, it should be `0.2.2`). + - The `note` field should have a reason to deprecate and a tracking issue to call for comments + (e.g., "We consider removing this as the upstream removed it. + If you're using it, please comment on #XXX"). +2. If we don't see any concerns for a while, do the change actually. + +## Supported target policy + +When Rust removes a support for a target, the libc crate also may remove the support anytime. + +## Releasing your change to crates.io Now that you've done the amazing job of landing your new API or your new platform in this crate, the next step is to get that sweet, sweet usage from crates.io! The only next step is to bump the version of libc and then publish it. If you'd like to get a release out ASAP you can follow these steps: -1. Update the version number in `Cargo.toml`, you'll just be bumping the patch - version number. -2. Run `cargo update` to regenerate the lockfile to encode your version bump in - the lock file. You may pull in some other updated dependencies, that's ok. -3. Send a PR to this repository. It should [look like this][example], but it'd +1. Increment the patch version number in `Cargo.toml` and `libc-test/Cargo.toml`. +1. Send a PR to this repository. It should [look like this][example-pr], but it'd also be nice to fill out the description with a small rationale for the - release (any rationale is ok though!) -4. Once merged the release will be tagged and published by one of the libc crate + release (any rationale is ok though!). +1. Once merged, the release will be tagged and published by one of the libc crate maintainers. -[example]: https://github.com/rust-lang/libc/pull/583 +[example-pr]: https://github.com/rust-lang/libc/pull/2120 diff --git a/crux-mir/lib/libc/Cargo.toml b/crux-mir/lib/libc/Cargo.toml index c415c9b4f..5c6208353 100644 --- a/crux-mir/lib/libc/Cargo.toml +++ b/crux-mir/lib/libc/Cargo.toml @@ -1,36 +1,64 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + [package] name = "libc" -version = "0.2.68" +version = "0.2.138" authors = ["The Rust Project Developers"] -license = "MIT OR Apache-2.0" -readme = "README.md" -repository = "https://github.com/rust-lang/libc" -homepage = "https://github.com/rust-lang/libc" -documentation = "http://doc.rust-lang.org/libc" -keywords = ["libc", "ffi", "bindings", "operating", "system" ] -categories = ["external-ffi-bindings", "no-std", "os"] build = "build.rs" -exclude = ["/ci/*", "/azure-pipelines.yml"] +exclude = [ + "/ci/*", + "/.github/*", + "/.cirrus.yml", + "/triagebot.toml", +] description = """ Raw FFI bindings to platform libraries like libc. """ +homepage = "https://github.com/rust-lang/libc" +documentation = "https://docs.rs/libc/" +readme = "README.md" +keywords = [ + "libc", + "ffi", + "bindings", + "operating", + "system", +] +categories = [ + "external-ffi-bindings", + "no-std", + "os", +] +license = "MIT OR Apache-2.0" +repository = "https://github.com/rust-lang/libc" -[badges] -cirrus-ci = { repository = "rust-lang/libc", branch = "master" } -azure-devops = { project = "rust-lang2/libc", pipeline = "rust-lang.libc%20(1)" } +[package.metadata.docs.rs] +features = [ + "const-extern-fn", + "extra_traits", +] -[dependencies] -rustc-std-workspace-core = { version = "1.0.0", optional = true } +[dependencies.rustc-std-workspace-core] +version = "1.0.0" +optional = true [features] -default = ["std"] -std = [] align = [] -rustc-dep-of-std = ['align', 'rustc-std-workspace-core'] -extra_traits = [] const-extern-fn = [] -# use_std is deprecated, use `std` instead -use_std = [ 'std' ] - -[workspace] -members = ["libc-test"] +default = ["std"] +extra_traits = [] +rustc-dep-of-std = [ + "align", + "rustc-std-workspace-core", +] +std = [] +use_std = ["std"] diff --git a/crux-mir/lib/libc/Cargo.toml.orig b/crux-mir/lib/libc/Cargo.toml.orig new file mode 100644 index 000000000..62b218a54 --- /dev/null +++ b/crux-mir/lib/libc/Cargo.toml.orig @@ -0,0 +1,35 @@ +[package] +name = "libc" +version = "0.2.138" +authors = ["The Rust Project Developers"] +license = "MIT OR Apache-2.0" +readme = "README.md" +repository = "https://github.com/rust-lang/libc" +homepage = "https://github.com/rust-lang/libc" +documentation = "https://docs.rs/libc/" +keywords = ["libc", "ffi", "bindings", "operating", "system" ] +categories = ["external-ffi-bindings", "no-std", "os"] +build = "build.rs" +exclude = ["/ci/*", "/.github/*", "/.cirrus.yml", "/triagebot.toml"] +description = """ +Raw FFI bindings to platform libraries like libc. +""" + +[package.metadata.docs.rs] +features = ["const-extern-fn", "extra_traits"] + +[dependencies] +rustc-std-workspace-core = { version = "1.0.0", optional = true } + +[features] +default = ["std"] +std = [] +align = [] +rustc-dep-of-std = ['align', 'rustc-std-workspace-core'] +extra_traits = [] +const-extern-fn = [] +# use_std is deprecated, use `std` instead +use_std = [ 'std' ] + +[workspace] +members = ["libc-test"] diff --git a/crux-mir/lib/libc/LICENSE-APACHE b/crux-mir/lib/libc/LICENSE-APACHE index 16fe87b06..1b5ec8b78 100644 --- a/crux-mir/lib/libc/LICENSE-APACHE +++ b/crux-mir/lib/libc/LICENSE-APACHE @@ -174,28 +174,3 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - -Copyright [yyyy] [name of copyright owner] - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/crux-mir/lib/libc/LICENSE-MIT b/crux-mir/lib/libc/LICENSE-MIT index 39d4bdb5a..78061811c 100644 --- a/crux-mir/lib/libc/LICENSE-MIT +++ b/crux-mir/lib/libc/LICENSE-MIT @@ -1,4 +1,4 @@ -Copyright (c) 2014 The Rust Project Developers +Copyright (c) 2014-2020 The Rust Project Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated diff --git a/crux-mir/lib/libc/README.md b/crux-mir/lib/libc/README.md index ae2cfd860..bc5ad18f6 100644 --- a/crux-mir/lib/libc/README.md +++ b/crux-mir/lib/libc/README.md @@ -1,7 +1,6 @@ -[![Azure Status]][Azure] [![Cirrus CI Status]][Cirrus CI] [![Latest Version]][crates.io] [![Documentation]][docs.rs] ![License] +# libc - Raw FFI bindings to platforms' system libraries -libc - Raw FFI bindings to platforms' system libraries -==== +[![GHA Status]][GitHub Actions] [![Cirrus CI Status]][Cirrus CI] [![Latest Version]][crates.io] [![Documentation]][docs.rs] ![License] `libc` provides all of the definitions necessary to easily interoperate with C code (or "C-like" code) on each of the platforms that Rust supports. This @@ -29,20 +28,23 @@ libc = "0.2" ## Features * `std`: by default `libc` links to the standard library. Disable this - feature remove this dependency and be able to use `libc` in `#![no_std]` + feature to remove this dependency and be able to use `libc` in `#![no_std]` crates. * `extra_traits`: all `struct`s implemented in `libc` are `Copy` and `Clone`. This feature derives `Debug`, `Eq`, `Hash`, and `PartialEq`. * `const-extern-fn`: Changes some `extern fn`s into `const extern fn`s. - This features requires a nightly rustc + If you use Rust >= 1.62, this feature is implicitly enabled. + Otherwise it requires a nightly rustc. * **deprecated**: `use_std` is deprecated, and is equivalent to `std`. ## Rust version support -The minimum supported Rust toolchain version is **Rust 1.13.0** . APIs requiring +The minimum supported Rust toolchain version is currently **Rust 1.13.0**. +(libc does not currently have any policy regarding changes to the minimum +supported Rust version; such policy is a work in progress.) APIs requiring newer Rust features are only available on newer Rust toolchains: | Feature | Version | @@ -53,6 +55,8 @@ newer Rust features are only available on newer Rust toolchains: | `extra_traits` | 1.25.0 | | `core::ffi::c_void` | 1.30.0 | | `repr(packed(N))` | 1.33.0 | +| `cfg(target_vendor)` | 1.33.0 | +| `const-extern-fn` | 1.62.0 | ## Platform support @@ -61,7 +65,7 @@ newer Rust features are only available on newer Rust toolchains: See [`ci/build.sh`](https://github.com/rust-lang/libc/blob/master/ci/build.sh) for the platforms on which `libc` is guaranteed to build for each Rust -toolchain. The test-matrix at [Azure] and [Cirrus CI] show the +toolchain. The test-matrix at [GitHub Actions] and [Cirrus CI] show the platforms in which `libc` tests are run.
@@ -71,10 +75,10 @@ platforms in which `libc` tests are run. This project is licensed under either of * [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0) - ([LICENSE-APACHE](LICENSE-APACHE)) + ([LICENSE-APACHE](https://github.com/rust-lang/libc/blob/master/LICENSE-APACHE)) * [MIT License](https://opensource.org/licenses/MIT) - ([LICENSE-MIT](LICENSE-MIT)) + ([LICENSE-MIT](https://github.com/rust-lang/libc/blob/master/LICENSE-MIT)) at your option. @@ -83,7 +87,7 @@ at your option. We welcome all people who want to contribute. Please see the [contributing instructions] for more information. -[contributing instructions]: CONTRIBUTING.md +[contributing instructions]: https://github.com/rust-lang/libc/blob/master/CONTRIBUTING.md Contributions in any form (issues, pull requests, etc.) to this project must adhere to Rust's [Code of Conduct]. @@ -94,8 +98,8 @@ Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in `libc` by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. -[Azure Status]: https://dev.azure.com/rust-lang2/libc/_apis/build/status/rust-lang.libc%20(1)?branchName=master -[Azure]: https://dev.azure.com/rust-lang2/libc/_build/latest?definitionId=1&branchName=master +[GitHub Actions]: https://github.com/rust-lang/libc/actions +[GHA Status]: https://github.com/rust-lang/libc/workflows/CI/badge.svg [Cirrus CI]: https://cirrus-ci.com/github/rust-lang/libc [Cirrus CI Status]: https://api.cirrus-ci.com/github/rust-lang/libc.svg [crates.io]: https://crates.io/crates/libc diff --git a/crux-mir/lib/libc/build.rs b/crux-mir/lib/libc/build.rs index f447c0ef9..bbee2d28a 100644 --- a/crux-mir/lib/libc/build.rs +++ b/crux-mir/lib/libc/build.rs @@ -3,12 +3,13 @@ use std::process::Command; use std::str; fn main() { - let (rustc_minor_ver, is_nightly) = - rustc_minor_nightly().expect("Failed to get rustc version"); + // Avoid unnecessary re-building. + println!("cargo:rerun-if-changed=build.rs"); + + let (rustc_minor_ver, is_nightly) = rustc_minor_nightly(); let rustc_dep_of_std = env::var("CARGO_FEATURE_RUSTC_DEP_OF_STD").is_ok(); let align_cargo_feature = env::var("CARGO_FEATURE_ALIGN").is_ok(); - let const_extern_fn_cargo_feature = - env::var("CARGO_FEATURE_CONST_EXTERN_FN").is_ok(); + let const_extern_fn_cargo_feature = env::var("CARGO_FEATURE_CONST_EXTERN_FN").is_ok(); let libc_ci = env::var("LIBC_CI").is_ok(); if env::var("CARGO_FEATURE_USE_STD").is_ok() { @@ -30,6 +31,7 @@ fn main() { Some(11) if libc_ci => println!("cargo:rustc-cfg=freebsd11"), Some(12) if libc_ci => println!("cargo:rustc-cfg=freebsd12"), Some(13) if libc_ci => println!("cargo:rustc-cfg=freebsd13"), + Some(14) if libc_ci => println!("cargo:rustc-cfg=freebsd14"), Some(_) | None => println!("cargo:rustc-cfg=freebsd11"), } @@ -58,6 +60,11 @@ fn main() { println!("cargo:rustc-cfg=libc_align"); } + // Rust >= 1.26 supports i128 and u128: + if rustc_minor_ver >= 26 || rustc_dep_of_std { + println!("cargo:rustc-cfg=libc_int128"); + } + // Rust >= 1.30 supports `core::ffi::c_void`, so libc can just re-export it. // Otherwise, it defines an incompatible type to retaining // backwards-compatibility. @@ -65,9 +72,24 @@ fn main() { println!("cargo:rustc-cfg=libc_core_cvoid"); } - // Rust >= 1.33 supports repr(packed(N)) + // Rust >= 1.33 supports repr(packed(N)) and cfg(target_vendor). if rustc_minor_ver >= 33 || rustc_dep_of_std { println!("cargo:rustc-cfg=libc_packedN"); + println!("cargo:rustc-cfg=libc_cfg_target_vendor"); + } + + // Rust >= 1.40 supports #[non_exhaustive]. + if rustc_minor_ver >= 40 || rustc_dep_of_std { + println!("cargo:rustc-cfg=libc_non_exhaustive"); + } + + if rustc_minor_ver >= 51 || rustc_dep_of_std { + println!("cargo:rustc-cfg=libc_ptr_addr_of"); + } + + // Rust >= 1.37.0 allows underscores as anonymous constant names. + if rustc_minor_ver >= 37 || rustc_dep_of_std { + println!("cargo:rustc-cfg=libc_underscore_const_names"); } // #[thread_local] is currently unstable @@ -75,31 +97,42 @@ fn main() { println!("cargo:rustc-cfg=libc_thread_local"); } - if const_extern_fn_cargo_feature { - if !is_nightly || rustc_minor_ver < 40 { - panic!("const-extern-fn requires a nightly compiler >= 1.40") - } + // Rust >= 1.62.0 allows to use `const_extern_fn` for "Rust" and "C". + if rustc_minor_ver >= 62 { println!("cargo:rustc-cfg=libc_const_extern_fn"); + } else { + // Rust < 1.62.0 requires a crate feature and feature gate. + if const_extern_fn_cargo_feature { + if !is_nightly || rustc_minor_ver < 40 { + panic!("const-extern-fn requires a nightly compiler >= 1.40"); + } + println!("cargo:rustc-cfg=libc_const_extern_fn_unstable"); + println!("cargo:rustc-cfg=libc_const_extern_fn"); + } } } -fn rustc_minor_nightly() -> Option<(u32, bool)> { +fn rustc_minor_nightly() -> (u32, bool) { macro_rules! otry { ($e:expr) => { match $e { Some(e) => e, - None => return None, + None => panic!("Failed to get rustc version"), } }; } let rustc = otry!(env::var_os("RUSTC")); - let output = otry!(Command::new(rustc).arg("--version").output().ok()); + let output = Command::new(rustc) + .arg("--version") + .output() + .ok() + .expect("Failed to get rustc version"); let version = otry!(str::from_utf8(&output.stdout).ok()); let mut pieces = version.split('.'); if pieces.next() != Some("rustc 1") { - return None; + panic!("Failed to get rustc version"); } let minor = pieces.next(); @@ -115,7 +148,7 @@ fn rustc_minor_nightly() -> Option<(u32, bool)> { .unwrap_or(false); let minor = otry!(otry!(minor).parse().ok()); - Some((minor, nightly)) + (minor, nightly) } fn which_freebsd() -> Option { @@ -139,6 +172,7 @@ fn which_freebsd() -> Option { s if s.starts_with("11") => Some(11), s if s.starts_with("12") => Some(12), s if s.starts_with("13") => Some(13), + s if s.starts_with("14") => Some(14), _ => None, } } diff --git a/crux-mir/lib/libc/ci/README.md b/crux-mir/lib/libc/ci/README.md deleted file mode 100644 index 312355380..000000000 --- a/crux-mir/lib/libc/ci/README.md +++ /dev/null @@ -1,236 +0,0 @@ -The goal of the libc crate is to have CI running everywhere to have the -strongest guarantees about the definitions that this library contains, and as a -result the CI is pretty complicated and also pretty large! Hopefully this can -serve as a guide through the sea of scripts in this directory and elsewhere in -this project. - -# Files - -First up, let's talk about the files in this directory: - -* `run-docker.sh` - a shell script run by most builders, it will execute - `run.sh` inside a Docker container configured for the target. - -* `run.sh` - the actual script which runs tests for a particular architecture. - -* `dox.sh` - build the documentation of the crate and publish it to gh-pages. - -* `landing-page-*.html` - used by `dox.sh` to generate a landing page for all - architectures' documentation. - -* `run-qemu.sh` - see discussion about QEMU below - -* `mips`, `rumprun` - instructions to build the docker image for each respective - CI target - -# CI Systems - -Currently this repository leverages a combination of Travis CI and AppVeyor for -running tests. The triples tested are: - -* AppVeyor - * `{i686,x86_64}-pc-windows-{msvc,gnu}` -* Travis - * `{i686,x86_64,mips,aarch64}-unknown-linux-gnu` - * `{x86_64,aarch64}-unknown-linux-musl` - * `arm-unknown-linux-gnueabihf` - * `arm-linux-androideabi` - * `{i686,x86_64}-apple-{darwin,ios}` - * `x86_64-rumprun-netbsd` - * `x86_64-unknown-freebsd` - * `x86_64-unknown-openbsd` - -The Windows triples are all pretty standard, they just set up their environment -then run tests, no need for downloading any extra target libs (we just download -the right installer). The Intel Linux/OSX builds are similar in that we just -download the right target libs and run tests. Note that the Intel Linux/OSX -builds are run on stable/beta/nightly, but are the only ones that do so. - -The remaining architectures look like: - -* Android runs in a [docker image][android-docker] with an emulator, the NDK, - and the SDK already set up. The entire build happens within the docker image. -* The MIPS, ARM, and AArch64 builds all use the QEMU userspace emulator to run - the generated binary to actually verify the tests pass. -* The MUSL build just has to download a MUSL compiler and target libraries and - then otherwise runs tests normally. -* iOS builds need an extra linker flag currently, but beyond that they're built - as standard as everything else. -* The rumprun target builds an entire kernel from the test suite and then runs - it inside QEMU using the serial console to test whether it succeeded or - failed. -* The BSD builds, currently OpenBSD and FreeBSD, use QEMU to boot up a system - and compile/run tests. More information on that below. - -[android-docker]: https://github.com/rust-lang/rust-buildbot/blob/master/slaves/android/Dockerfile - -## QEMU - -Lots of the architectures tested here use QEMU in the tests, so it's worth going -over all the crazy capabilities QEMU has and the various flavors in which we use -it! - -First up, QEMU has userspace emulation where it doesn't boot a full kernel, it -just runs a binary from another architecture (using the `qemu-` wrappers). -We provide it the runtime path for the dynamically loaded system libraries, -however. This strategy is used for all Linux architectures that aren't intel. -Note that one downside of this QEMU system is that threads are barely -implemented, so we're careful to not spawn many threads. - -For the rumprun target the only output is a kernel image, so we just use that -plus the `rumpbake` command to create a full kernel image which is then run from -within QEMU. - -Finally, the fun part, the BSDs. Quite a few hoops are jumped through to get CI -working for these platforms, but the gist of it looks like: - -* Cross compiling from Linux to any of the BSDs seems to be quite non-standard. - We may be able to get it working but it might be difficult at that point to - ensure that the libc definitions align with what you'd get on the BSD itself. - As a result, we try to do compiles within the BSD distro. -* On Travis we can't run a VM-in-a-VM, so we resort to userspace emulation - (QEMU). -* Unfortunately on Travis we also can't use KVM, so the emulation is super slow. - -With all that in mind, the way BSD is tested looks like: - -1. Download a pre-prepared image for the OS being tested. -2. Generate the tests for the OS being tested. This involves running the `ctest` - library over libc to generate a Rust file and a C file which will then be - compiled into the final test. -3. Generate a disk image which will later be mounted by the OS being tested. - This image is mostly just the libc directory, but some modifications are made - to compile the generated files from step 2. -4. The kernel is booted in QEMU, and it is configured to detect the libc-test - image being available, run the test script, and then shut down afterwards. -5. Look for whether the tests passed in the serial console output of the kernel. - -There's some pretty specific instructions for setting up each image (detailed -below), but the main gist of this is that we must avoid a vanilla `cargo run` -inside of the `libc-test` directory (which is what it's intended for) because -that would compile `syntex_syntax`, a large library, with userspace emulation. -This invariably times out on Travis, so we can't do that. - -Once all those hoops are jumped through, however, we can be happy that we're -testing almost everything! - -Below are some details of how to set up the initial OS images which are -downloaded. Each image must be enabled have input/output over the serial -console, log in automatically at the serial console, detect if a second drive in -QEMU is available, and if so mount it, run a script (it'll specifically be -`run-qemu.sh` in this folder which is copied into the generated image talked -about above), and then shut down. - -### QEMU Setup - FreeBSD - -1. [Download the latest stable amd64-bootonly release ISO](https://www.freebsd.org/where.html). - E.g. FreeBSD-11.1-RELEASE-amd64-bootonly.iso -2. Create the disk image: `qemu-img create -f qcow2 FreeBSD-11.1-RELEASE-amd64.qcow2 2G` -3. Boot the machine: `qemu-system-x86_64 -cdrom FreeBSD-11.1-RELEASE-amd64-bootonly.iso -drive if=virtio,file=FreeBSD-11.1-RELEASE-amd64.qcow2 -net nic,model=virtio -net user` -4. Run the installer, and install FreeBSD: - 1. Install - 1. Continue with default keymap - 1. Set Hostname: freebsd-ci - 1. Distribution Select: - 1. Uncheck lib32 - 1. Uncheck ports - 1. Network Configuration: vtnet0 - 1. Configure IPv4? Yes - 1. DHCP? Yes - 1. Configure IPv6? No - 1. Resolver Configuration: Ok - 1. Mirror Selection: Main Site - 1. Partitioning: Auto (UFS) - 1. Partition: Entire Disk - 1. Partition Scheme: MBR - 1. App Partition: Ok - 1. Partition Editor: Finish - 1. Confirmation: Commit - 1. Wait for sets to install - 1. Set the root password to nothing (press enter twice) - 1. Set time zone to UTC - 1. Set Date: Skip - 1. Set Time: Skip - 1. System Configuration: - 1. Disable sshd - 1. Disable dumpdev - 1. System Hardening - 1. Disable Sendmail service - 1. Add User Accounts: No - 1. Final Configuration: Exit - 1. Manual Configuration: Yes - 1. `echo 'console="comconsole"' >> /boot/loader.conf` - 1. `echo 'autoboot_delay="0"' >> /boot/loader.conf` - 1. `echo 'ext2fs_load="YES"' >> /boot/loader.conf` - 1. Look at `/etc/ttys`, see what getty argument is for `ttyu0` (E.g. `3wire`) - 1. Edit `/etc/gettytab` (with `vi` for example), look for `ttyu0` argument, - prepend `:al=root` to the line beneath to have the machine auto-login as - root. E.g. - - 3wire:\ - :np:nc:sp#0: - becomes: - - 3wire:\ - :al=root:np:nc:sp#0: - - 1. Edit `/root/.login` and put this in it: - - [ -e /dev/vtbd1 ] || exit 0 - mount -t ext2fs /dev/vtbd1 /mnt - sh /mnt/run.sh /mnt - poweroff - - 1. Exit the post install shell: `exit` - 1. Back in in the installer choose Reboot - 1. If all went well the machine should reboot and show a login prompt. - If you switch to the serial console by choosing View > serial0 in - the qemu menu, you should be logged in as root. - 1. Shutdown the machine: `shutdown -p now` - -Helpful links - -* https://en.wikibooks.org/wiki/QEMU/Images -* https://blog.nekoconeko.nl/blog/2015/06/04/creating-an-openstack-freebsd-image.html -* https://www.freebsd.org/doc/handbook/serialconsole-setup.html - - -### QEMU setup - OpenBSD - -1. Download CD installer -2. `qemu-img create -f qcow2 foo.qcow2 2G` -3. `qemu -cdrom foo.iso -drive if=virtio,file=foo.qcow2 -net nic,model=virtio -net user` -4. run installer -5. `echo 'set tty com0' >> /etc/boot.conf` -6. `echo 'boot' >> /etc/boot.conf` -7. Modify /etc/ttys, change the `tty00` at the end from 'unknown off' to - 'vt220 on secure' -8. Modify same line in /etc/ttys to have `"/root/foo.sh"` as the shell -9. Add this script to `/root/foo.sh` - -``` -#!/bin/sh -exec 1>/dev/tty00 -exec 2>&1 - -if mount -t ext2fs /dev/sd1c /mnt; then - sh /mnt/run.sh /mnt - shutdown -ph now -fi - -# limited shell... -exec /bin/sh < /dev/tty00 -``` - -10. `chmod +x /root/foo.sh` - -Helpful links: - -* https://en.wikibooks.org/wiki/QEMU/Images -* http://www.openbsd.org/faq/faq7.html#SerCon - -# Questions? - -Hopefully that's at least somewhat of an introduction to everything going on -here, and feel free to ping @alexcrichton with questions! - diff --git a/crux-mir/lib/libc/ci/android-install-ndk.sh b/crux-mir/lib/libc/ci/android-install-ndk.sh deleted file mode 100644 index 723e71905..000000000 --- a/crux-mir/lib/libc/ci/android-install-ndk.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env sh -# Copyright 2016 The Rust Project Developers. See the COPYRIGHT -# file at the top-level directory of this distribution and at -# http://rust-lang.org/COPYRIGHT. -# -# Licensed under the Apache License, Version 2.0 or the MIT license -# , at your -# option. This file may not be copied, modified, or distributed -# except according to those terms. - -set -ex - -NDK=android-ndk-r19c -curl --retry 20 -O https://dl.google.com/android/repository/${NDK}-linux-x86_64.zip -unzip -q ${NDK}-linux-x86_64.zip - -case "$1" in - arm) - arch=arm - api=24 - ;; - armv7) - arch=arm - api=24 - ;; - aarch64) - arch=arm64 - api=24 - ;; - i686) - arch=x86 - api=28 - ;; - x86_64) - arch=x86_64 - api=28 - ;; - *) - echo "invalid arch: $1" - exit 1 - ;; -esac; - -${NDK}/build/tools/make_standalone_toolchain.py \ - --install-dir "/android/ndk-${1}" \ - --arch "${arch}" \ - --api ${api} - -rm -rf ./${NDK}-linux-x86_64.zip ./${NDK} diff --git a/crux-mir/lib/libc/ci/android-install-sdk.sh b/crux-mir/lib/libc/ci/android-install-sdk.sh deleted file mode 100644 index 7f2104000..000000000 --- a/crux-mir/lib/libc/ci/android-install-sdk.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env sh -# Copyright 2016 The Rust Project Developers. See the COPYRIGHT -# file at the top-level directory of this distribution and at -# http://rust-lang.org/COPYRIGHT. -# -# Licensed under the Apache License, Version 2.0 or the MIT license -# , at your -# option. This file may not be copied, modified, or distributed -# except according to those terms. - -set -ex - -# Prep the SDK and emulator -# -# Note that the update process requires that we accept a bunch of licenses, and -# we can't just pipe `yes` into it for some reason, so we take the same strategy -# located in https://github.com/appunite/docker by just wrapping it in a script -# which apparently magically accepts the licenses. - -SDK=4333796 -mkdir sdk -curl --retry 20 https://dl.google.com/android/repository/sdk-tools-linux-${SDK}.zip -O -unzip -q -d sdk sdk-tools-linux-${SDK}.zip - -case "$1" in - arm | armv7) - api=24 - image="system-images;android-${api};google_apis;armeabi-v7a" - ;; - aarch64) - api=24 - image="system-images;android-${api};google_apis;arm64-v8a" - ;; - i686) - api=28 - image="system-images;android-${api};default;x86" - ;; - x86_64) - api=28 - image="system-images;android-${api};default;x86_64" - ;; - *) - echo "invalid arch: $1" - exit 1 - ;; -esac; - -# Try to fix warning about missing file. -# See https://askubuntu.com/a/1078784 -mkdir -p /root/.android/ -echo '### User Sources for Android SDK Manager' >> /root/.android/repositories.cfg -echo '#Fri Nov 03 10:11:27 CET 2017 count=0' >> /root/.android/repositories.cfg - -# Print all available packages -# yes | ./sdk/tools/bin/sdkmanager --list --verbose - -# --no_https avoids -# javax.net.ssl.SSLHandshakeException: sun.security.validator.ValidatorException: No trusted certificate found -# -# | grep -v = || true removes the progress bar output from the sdkmanager -# which produces an insane amount of output. -yes | ./sdk/tools/bin/sdkmanager --licenses --no_https | grep -v = || true -yes | ./sdk/tools/bin/sdkmanager --no_https \ - "emulator" \ - "platform-tools" \ - "platforms;android-${api}" \ - "${image}" | grep -v = || true - -echo "no" | - ./sdk/tools/bin/avdmanager create avd \ - --name "${1}" \ - --package "${image}" | grep -v = || true diff --git a/crux-mir/lib/libc/ci/android-sysimage.sh b/crux-mir/lib/libc/ci/android-sysimage.sh deleted file mode 100644 index 9eabd7c8d..000000000 --- a/crux-mir/lib/libc/ci/android-sysimage.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2017 The Rust Project Developers. See the COPYRIGHT -# file at the top-level directory of this distribution and at -# http://rust-lang.org/COPYRIGHT. -# -# Licensed under the Apache License, Version 2.0 or the MIT license -# , at your -# option. This file may not be copied, modified, or distributed -# except according to those terms. - -set -ex - -URL=https://dl.google.com/android/repository/sys-img/android - -main() { - local arch="${1}" - local name="${2}" - local dest=/system - local td - td="$(mktemp -d)" - - apt-get install --no-install-recommends e2tools - - pushd "${td}" - curl --retry 5 -O "${URL}/${name}" - unzip -q "${name}" - - local system - system="$(find . -name system.img)" - mkdir -p ${dest}/{bin,lib,lib64} - - # Extract android linker and libraries to /system - # This allows android executables to be run directly (or with qemu) - if [ "${arch}" = "x86_64" ] || [ "${arch}" = "arm64" ]; then - e2cp -p "${system}:/bin/linker64" "${dest}/bin/" - e2cp -p "${system}:/lib64/libdl.so" "${dest}/lib64/" - e2cp -p "${system}:/lib64/libc.so" "${dest}/lib64/" - e2cp -p "${system}:/lib64/libm.so" "${dest}/lib64/" - else - e2cp -p "${system}:/bin/linker" "${dest}/bin/" - e2cp -p "${system}:/lib/libdl.so" "${dest}/lib/" - e2cp -p "${system}:/lib/libc.so" "${dest}/lib/" - e2cp -p "${system}:/lib/libm.so" "${dest}/lib/" - fi - - # clean up - apt-get purge --auto-remove -y e2tools - - popd - - rm -rf "${td}" -} - -main "${@}" diff --git a/crux-mir/lib/libc/ci/azure-install-rust.yml b/crux-mir/lib/libc/ci/azure-install-rust.yml deleted file mode 100644 index ec13ff9ea..000000000 --- a/crux-mir/lib/libc/ci/azure-install-rust.yml +++ /dev/null @@ -1,82 +0,0 @@ -steps: - - bash: | - set -ex - toolchain=$TOOLCHAIN - if [ "$toolchain" = "" ]; then - toolchain=nightly - fi - if command -v rustup; then - # Uncomment when rustup on Azure is updated - #rustup set profile minimal - rustup update --force $toolchain - rustup default $toolchain - else - curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain $toolchain --profile=minimal - echo "##vso[task.prependpath]$HOME/.cargo/bin" - fi - displayName: Install rust (unix) - condition: ne( variables['Agent.OS'], 'Windows_NT' ) - - script: | - @echo on - if not defined TOOLCHAIN set TOOLCHAIN=nightly - :: Uncomment when rustup on Azure is updated - ::rustup set profile minimal - rustup update --no-self-update %TOOLCHAIN%-%TARGET% - rustup default %TOOLCHAIN%-%TARGET% - displayName: Install rust (windows) - condition: eq( variables['Agent.OS'], 'Windows_NT' ) - - script: | - set -ex - if [ -n "${TARGET}" ]; then - rustup target add $TARGET - fi - condition: ne( variables['Agent.OS'], 'Windows_NT' ) - displayName: Install target (unix) - - script: | - @echo on - if defined TARGET rustup target add %TARGET% - condition: eq( variables['Agent.OS'], 'Windows_NT' ) - displayName: Install target (windows) - - script: | - @echo on - if "%ARCH%" == "i686" choco install mingw --x86 --force - condition: eq( variables['Agent.OS'], 'Windows_NT' ) - displayName: Install MinGW32 (windows) - - bash: | - set -ex - gcc -print-search-dirs - /usr/bin/find "C:\ProgramData\Chocolatey" -name "crt2*" - /usr/bin/find "C:\ProgramData\Chocolatey" -name "dllcrt2*" - /usr/bin/find "C:\ProgramData\Chocolatey" -name "libmsvcrt*" - condition: eq( variables['Agent.OS'], 'Windows_NT' ) - displayName: Find GCC libraries (windows) - - bash: | - set -ex - if [[ -n ${ARCH_BITS} ]]; then - for i in crt2.o dllcrt2.o libmingwex.a libmsvcrt.a ; do - cp -f "/C/ProgramData/Chocolatey/lib/mingw/tools/install/mingw${ARCH_BITS}/${ARCH}-w64-mingw32/lib/$i" "`rustc --print sysroot`/lib/rustlib/${TARGET}/lib" - done - fi - condition: eq( variables['Agent.OS'], 'Windows_NT' ) - displayName: Fix MinGW (windows) - - bash: | - set -ex - rustc -Vv - cargo -V - rustup -Vv - rustup show - which rustc - which cargo - which rustup - displayName: Query rust and cargo versions - - script: | - @echo on - where gcc - condition: eq( variables['Agent.OS'], 'Windows_NT' ) - displayName: Query gcc path - - bash: | - set -ex - cargo generate-lockfile - cargo generate-lockfile --manifest-path libc-test/Cargo.toml - displayName: Generate lockfiles - diff --git a/crux-mir/lib/libc/ci/azure-master.yml b/crux-mir/lib/libc/ci/azure-master.yml deleted file mode 100644 index c61e2b4c2..000000000 --- a/crux-mir/lib/libc/ci/azure-master.yml +++ /dev/null @@ -1,22 +0,0 @@ -variables: - - group: secrets -resources: - repositories: - - repository: rustinfra - type: github - name: rust-lang/simpleinfra - endpoint: gnzlbg -trigger: ["master"] -pr: ["master"] - -jobs: - - job: StyleAndDocs - pool: - vmImage: ubuntu-18.04 - steps: - - template: azure-install-rust.yml - - script: LIBC_CI=1 sh ci/dox.sh - displayName: Generate documentation - - template: azure-configs/static-websites.yml@rustinfra - parameters: - deploy_dir: target/doc diff --git a/crux-mir/lib/libc/ci/azure.yml b/crux-mir/lib/libc/ci/azure.yml deleted file mode 100644 index 6dd0fcecb..000000000 --- a/crux-mir/lib/libc/ci/azure.yml +++ /dev/null @@ -1,215 +0,0 @@ -variables: - - group: secrets -resources: - repositories: - - repository: rustinfra - type: github - name: rust-lang/simpleinfra - endpoint: gnzlbg -trigger: ["auto-libc","try"] -pr: ["master"] - -jobs: - - job: DockerLinuxTier1 - pool: - vmImage: ubuntu-18.04 - steps: - - template: azure-install-rust.yml - - bash: LIBC_CI=1 sh ./ci/run-docker.sh $TARGET - displayName: Execute run-docker.sh - strategy: - matrix: - i686-unknown-linux-gnu: - TARGET: i686-unknown-linux-gnu - x86_64-unknown-linux-gnu: - TARGET: x86_64-unknown-linux-gnu - - - job: DockerLinuxTier2 - #dependsOn: DockerLinuxTier1 - pool: - vmImage: ubuntu-18.04 - steps: - - template: azure-install-rust.yml - - bash: LIBC_CI=1 sh ./ci/run-docker.sh $TARGET - displayName: Execute run-docker.sh - strategy: - matrix: - aarch64-unknown-linux-android: - TARGET: aarch64-linux-android - aarch64-unknown-linux-gnu: - TARGET: aarch64-unknown-linux-gnu - aarch64-unknown-linux-musl: - TARGET: aarch64-unknown-linux-musl - arm-linux-androideabi: - TARGET: arm-linux-androideabi - arm-unknown-linux-gnueabihf: - TARGET: arm-unknown-linux-gnueabihf - arm-unknown-linux-musleabihf: - TARGET: arm-unknown-linux-musleabihf - # Disabled because currently broken, see: - # https://github.com/rust-lang/libc/issues/1591 - # asmjs-unknown-emscripten: - # TARGET: asmjs-unknown-emscripten - i686-linux-android: - TARGET: i686-linux-android - i686-unknown-linux-musl: - TARGET: i686-unknown-linux-musl - mips-unknown-linux-gnu: - TARGET: mips-unknown-linux-gnu - mips-unknown-linux-musl: - TARGET: mips-unknown-linux-musl - mips64-unknown-linux-gnuabi64: - TARGET: mips64-unknown-linux-gnuabi64 - mips64el-unknown-linux-gnuabi64: - TARGET: mips64el-unknown-linux-gnuabi64 - mipsel-unknown-linux-musl: - TARGET: mipsel-unknown-linux-musl - #powerpc-unknown-linux-gnu: - # TARGET: powerpc-unknown-linux-gnu - powerpc64-unknown-linux-gnu: - TARGET: powerpc64-unknown-linux-gnu - powerpc64le-unknown-linux-gnu: - TARGET: powerpc64le-unknown-linux-gnu - s390x-unknown-linux-gnu: - TARGET: s390x-unknown-linux-gnu - #wasm32-wasi - # TARGET: wasm32-wasi - sparc64-unknown-linux-gnu: - TARGET: sparc64-unknown-linux-gnu - # Disabled because currently broken, see: - # https://github.com/rust-lang/libc/issues/1591 - # wasm32-unknown-emscripten: - # TARGET: wasm32-unknown-emscripten - x86_64-linux-android: - TARGET: x86_64-linux-android - x86_64-unknown-linux-gnux32: - TARGET: x86_64-unknown-linux-gnux32 - x86_64-unknown-linux-musl: - TARGET: x86_64-unknown-linux-musl - - - job: DockerOSX64 - pool: - vmImage: macos-10.15 - steps: - - template: azure-install-rust.yml - - bash: LIBC_CI=1 sh ./ci/run.sh $TARGET - displayName: Execute run.sh - strategy: - matrix: - x86_64-apple-darwin: - TARGET: x86_64-apple-darwin - - - job: Windows - pool: - vmImage: vs2017-win2016 - steps: - - template: azure-install-rust.yml - - bash: LIBC_CI=1 sh ./ci/run.sh $TARGET - displayName: Execute run.sh - strategy: - matrix: - x86_64-pc-windows-gnu: - TARGET: x86_64-pc-windows-gnu - ARCH_BITS: 64 - ARCH: x86_64 - x86_64-pc-windows-msvc: - TARGET: x86_64-pc-windows-msvc - # Disabled because broken: - # https://github.com/rust-lang/libc/issues/1592 - #i686-pc-windows-gnu: - # TARGET: i686-pc-windows-gnu - # ARCH_BITS: 32 - # ARCH: i686 - i686-pc-windows-msvc: - TARGET: i686-pc-windows-msvc - - - job: StyleAndDocs - pool: - vmImage: ubuntu-18.04 - steps: - - template: azure-install-rust.yml - - script: sh ci/style.sh - displayName: Check style - - script: LIBC_CI=1 sh ci/dox.sh - displayName: Generate documentation - - template: azure-configs/static-websites.yml@rustinfra - parameters: - deploy_dir: target/doc - - - job: SemverLinux - dependsOn: BuildChannelsLinux - continueOnError: true - pool: - vmImage: ubuntu-18.04 - steps: - - template: azure-install-rust.yml - - script: sh ci/semver.sh linux - displayName: Check breaking changes - - - job: SemverOSX - dependsOn: BuildChannelsOSX - continueOnError: true - pool: - vmImage: macos-10.15 - steps: - - template: azure-install-rust.yml - - script: sh ci/semver.sh osx - displayName: Check breaking changes - - - job: BuildChannelsLinux - dependsOn: StyleAndDocs - pool: - vmImage: ubuntu-18.04 - steps: - - template: azure-install-rust.yml - - script: LIBC_CI=1 sh ./ci/build.sh - displayName: Execute build.sh - strategy: - matrix: - stable: - TOOLCHAIN: stable - beta: - TOOLCHAIN: beta - nightly: - TOOLCHAIN: nightly - 1.13.0: - TOOLCHAIN: 1.13.0 - 1.19.0: - TOOLCHAIN: 1.19.0 - 1.24.0: - TOOLCHAIN: 1.24.0 - 1.25.0: - TOOLCHAIN: 1.25.0 - 1.30.0: - TOOLCHAIN: 1.30.0 - variables: - OS: linux - - - job: BuildChannelsOSX - dependsOn: StyleAndDocs - pool: - vmImage: macos-10.15 - steps: - - template: azure-install-rust.yml - - script: LIBC_CI=1 sh ./ci/build.sh - displayName: Execute build.sh - strategy: - matrix: - stable: - TOOLCHAIN: stable - beta: - TOOLCHAIN: beta - nightly: - TOOLCHAIN: nightly - 1.13.0: - TOOLCHAIN: 1.13.0 - 1.19.0: - TOOLCHAIN: 1.19.0 - 1.24.0: - TOOLCHAIN: 1.24.0 - 1.25.0: - TOOLCHAIN: 1.25.0 - 1.30.0: - TOOLCHAIN: 1.30.0 - variables: - OS: osx diff --git a/crux-mir/lib/libc/ci/build.sh b/crux-mir/lib/libc/ci/build.sh deleted file mode 100644 index 80ed7c678..000000000 --- a/crux-mir/lib/libc/ci/build.sh +++ /dev/null @@ -1,267 +0,0 @@ -#!/usr/bin/env sh - -# Checks that libc builds properly for all supported targets on a particular -# Rust version: -# The FILTER environment variable can be used to select which target(s) to build. -# For example: set FILTER to vxworks to select the targets that has vxworks in name - -set -ex - -: "${TOOLCHAIN?The TOOLCHAIN environment variable must be set.}" -: "${OS?The OS environment variable must be set.}" - -RUST=${TOOLCHAIN} - -echo "Testing Rust ${RUST} on ${OS}" - -if [ "${TOOLCHAIN}" = "nightly" ] ; then - cargo +nightly install cargo-xbuild - rustup component add rust-src -fi - -test_target() { - BUILD_CMD="${1}" - TARGET="${2}" - NO_STD="${3}" - - # If there is a std component, fetch it: - if [ "${NO_STD}" != "1" ]; then - # FIXME: rustup often fails to download some artifacts due to network - # issues, so we retry this N times. - N=5 - n=0 - until [ $n -ge $N ] - do - if rustup target add "${TARGET}" --toolchain "${RUST}" ; then - break - fi - n=$((n+1)) - sleep 1 - done - fi - - # Test that libc builds without any default features (no libstd) - cargo "+${RUST}" "${BUILD_CMD}" -vv --no-default-features --target "${TARGET}" - - # Test that libc builds with default features (e.g. libstd) - # if the target supports libstd - if [ "$NO_STD" != "1" ]; then - cargo "+${RUST}" "${BUILD_CMD}" -vv --target "${TARGET}" - fi - - # Test that libc builds with the `extra_traits` feature - cargo "+${RUST}" "${BUILD_CMD}" -vv --no-default-features --target "${TARGET}" \ - --features extra_traits - - # Test the 'const-extern-fn' feature on nightly - if [ "${RUST}" = "nightly" ]; then - cargo "+${RUST}" "${BUILD_CMD}" -vv --no-default-features --target "${TARGET}" \ - --features const-extern-fn - fi - - - # Also test that it builds with `extra_traits` and default features: - if [ "$NO_STD" != "1" ]; then - cargo "+${RUST}" "${BUILD_CMD}" -vv --target "${TARGET}" \ - --features extra_traits - fi -} - -RUST_LINUX_TARGETS="\ -aarch64-linux-android \ -aarch64-unknown-linux-gnu \ -arm-linux-androideabi \ -arm-unknown-linux-gnueabi \ -arm-unknown-linux-gnueabihf \ -armv7-linux-androideabi \ -armv7-unknown-linux-gnueabihf \ -i586-unknown-linux-gnu \ -i686-linux-android \ -i686-unknown-freebsd \ -i686-unknown-linux-gnu \ -i686-unknown-linux-musl \ -mips-unknown-linux-gnu \ -mips-unknown-linux-musl \ -mips64-unknown-linux-gnuabi64 \ -mips64el-unknown-linux-gnuabi64 \ -mipsel-unknown-linux-gnu \ -mipsel-unknown-linux-musl \ -powerpc-unknown-linux-gnu \ -powerpc64-unknown-linux-gnu \ -powerpc64le-unknown-linux-gnu \ -s390x-unknown-linux-gnu \ -x86_64-unknown-freebsd \ -x86_64-unknown-linux-gnu \ -x86_64-unknown-linux-musl \ -x86_64-unknown-netbsd \ -" - -RUST_GT_1_13_LINUX_TARGETS="\ -arm-unknown-linux-musleabi \ -arm-unknown-linux-musleabihf \ -armv7-unknown-linux-musleabihf \ -sparc64-unknown-linux-gnu \ -wasm32-unknown-emscripten \ -x86_64-linux-android \ -x86_64-rumprun-netbsd \ -" -RUST_GT_1_19_LINUX_TARGETS="\ -aarch64-unknown-linux-musl \ -sparcv9-sun-solaris \ -wasm32-unknown-unknown \ -x86_64-sun-solaris \ -" -RUST_GT_1_24_LINUX_TARGETS="\ -i586-unknown-linux-musl \ -x86_64-unknown-cloudabi \ -" - -# FIXME: temporarirly disable the redox target -# https://github.com/rust-lang/libc/issues/1457 -# x86_64-unknown-redox -RUST_NIGHTLY_LINUX_TARGETS="\ -aarch64-fuchsia \ -armv5te-unknown-linux-gnueabi \ -armv5te-unknown-linux-musleabi \ -i686-pc-windows-gnu \ -wasm32-wasi \ -x86_64-fortanix-unknown-sgx \ -x86_64-fuchsia \ -x86_64-pc-windows-gnu \ -x86_64-unknown-linux-gnux32 \ -" - -RUST_OSX_TARGETS="\ -aarch64-apple-ios \ -x86_64-apple-darwin \ -x86_64-apple-ios \ -" - -# The targets are listed here alphabetically -TARGETS="" -case "${OS}" in - linux*) - TARGETS="${RUST_LINUX_TARGETS}" - - if [ "${RUST}" != "1.13.0" ]; then - TARGETS="${TARGETS} ${RUST_GT_1_13_LINUX_TARGETS}" - if [ "${RUST}" != "1.19.0" ]; then - TARGETS="${TARGETS} ${RUST_GT_1_19_LINUX_TARGETS}" - if [ "${RUST}" != "1.24.0" ]; then - TARGETS="${TARGETS} ${RUST_GT_1_24_LINUX_TARGETS}" - fi - fi - fi - - if [ "${RUST}" = "nightly" ]; then - TARGETS="${TARGETS} ${RUST_NIGHTLY_LINUX_TARGETS}" - fi - - ;; - osx*) - TARGETS="${RUST_OSX_TARGETS}" - ;; - *) - ;; -esac - -for TARGET in $TARGETS; do - if echo "$TARGET"|grep -q "$FILTER";then - test_target build "$TARGET" - fi -done - -# FIXME: https://github.com/rust-lang/rust/issues/58564 -# sparc-unknown-linux-gnu -RUST_LINUX_NO_CORE_TARGETS="\ -aarch64-pc-windows-msvc \ -aarch64-unknown-cloudabi \ -aarch64-unknown-freebsd \ -aarch64-unknown-hermit \ -aarch64-unknown-netbsd \ -aarch64-unknown-openbsd \ -armebv7r-none-eabi \ -armebv7r-none-eabihf \ -armv7-unknown-cloudabi-eabihf \ -armv7r-none-eabi \ -armv7r-none-eabihf \ -hexagon-unknown-linux-musl \ -i586-pc-windows-msvc \ -i686-pc-windows-msvc \ -i686-unknown-cloudabi \ -i686-unknown-haiku \ -i686-unknown-netbsd \ -i686-unknown-openbsd \ -mips-unknown-linux-uclibc \ -mipsel-unknown-linux-uclibc \ -mips64-unknown-linux-muslabi64 \ -mips64el-unknown-linux-muslabi64 \ -nvptx64-nvidia-cuda \ -powerpc-unknown-linux-gnuspe \ -powerpc-unknown-netbsd \ -powerpc64-unknown-freebsd \ -riscv64gc-unknown-linux-gnu \ -riscv32imac-unknown-none-elf \ -riscv32imc-unknown-none-elf \ -sparc64-unknown-netbsd \ - -thumbv6m-none-eabi \ -thumbv7em-none-eabi \ -thumbv7em-none-eabihf \ -thumbv7m-none-eabi \ -thumbv7neon-linux-androideabi \ -thumbv7neon-unknown-linux-gnueabihf \ -thumbv8m.main-none-eabi \ -x86_64-pc-windows-msvc -x86_64-unknown-dragonfly \ -x86_64-unknown-haiku \ -x86_64-unknown-hermit \ -x86_64-unknown-l4re-uclibc \ -x86_64-unknown-openbsd \ -armv7-wrs-vxworks-eabihf \ -aarch64-wrs-vxworks \ -i686-wrs-vxworks \ -x86_64-wrs-vxworks \ -powerpc-wrs-vxworks \ -powerpc-wrs-vxworks-spe \ -powerpc64-wrs-vxworks \ -" - -if [ "${RUST}" = "nightly" ] && [ "${OS}" = "linux" ]; then - for TARGET in $RUST_LINUX_NO_CORE_TARGETS; do - if echo "$TARGET"|grep -q "$FILTER";then - test_target xbuild "$TARGET" 1 - fi - done - - # Nintendo switch - cargo clean - mkdir -p target - ( - cd target - wget https://github.com/devkitPro/pacman/releases/download/devkitpro-pacman-1.0.1/devkitpro-pacman.deb - sudo dpkg -i devkitpro-pacman.deb - sudo dkp-pacman -Sy - sudo dkp-pacman -Syu - sudo dkp-pacman -S -v --noconfirm switch-dev devkitA64 - ) - cp ci/switch.json switch.json - PATH="$PATH:/opt/devkitpro/devkitA64/bin" - PATH="$PATH:/opt/devkitpro/tools/bin" - cargo xbuild --target switch.json -fi - -RUST_OSX_NO_CORE_TARGETS="\ -armv7-apple-ios \ -armv7s-apple-ios \ -i386-apple-ios \ -i686-apple-darwin \ -" - -if [ "${RUST}" = "nightly" ] && [ "${OS}" = "osx" ]; then - for TARGET in $RUST_OSX_NO_CORE_TARGETS; do - if echo "$TARGET" | grep -q "$FILTER"; then - test_target xbuild "$TARGET" 1 - fi - done -fi diff --git a/crux-mir/lib/libc/ci/docker/aarch64-linux-android/Dockerfile b/crux-mir/lib/libc/ci/docker/aarch64-linux-android/Dockerfile deleted file mode 100644 index 06b0d6dbf..000000000 --- a/crux-mir/lib/libc/ci/docker/aarch64-linux-android/Dockerfile +++ /dev/null @@ -1,45 +0,0 @@ -FROM ubuntu:19.10 - -RUN dpkg --add-architecture i386 && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - file \ - curl \ - ca-certificates \ - python \ - unzip \ - expect \ - openjdk-8-jre \ - libstdc++6:i386 \ - libpulse0 \ - gcc \ - libc6-dev - -WORKDIR /android/ -COPY android* /android/ - -ENV ANDROID_ARCH=aarch64 -ENV PATH=$PATH:/android/ndk-$ANDROID_ARCH/bin:/android/sdk/tools:/android/sdk/platform-tools - -RUN sh /android/android-install-ndk.sh $ANDROID_ARCH -RUN sh /android/android-install-sdk.sh $ANDROID_ARCH -RUN mv /root/.android /tmp -RUN chmod 777 -R /tmp/.android -RUN chmod 755 /android/sdk/tools/* /android/sdk/emulator/qemu/linux-x86_64/* - -ENV PATH=$PATH:/rust/bin \ - CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER=aarch64-linux-android-gcc \ - CARGO_TARGET_AARCH64_LINUX_ANDROID_RUNNER=/tmp/runtest \ - HOME=/tmp - -ADD runtest-android.rs /tmp/runtest.rs -ENTRYPOINT [ \ - "bash", \ - "-c", \ - # set SHELL so android can detect a 64bits system, see - # http://stackoverflow.com/a/41789144 - "SHELL=/bin/dash /android/sdk/emulator/emulator @aarch64 -no-window & \ - rustc /tmp/runtest.rs -o /tmp/runtest && \ - exec \"$@\"", \ - "--" \ -] diff --git a/crux-mir/lib/libc/ci/docker/aarch64-unknown-linux-gnu/Dockerfile b/crux-mir/lib/libc/ci/docker/aarch64-unknown-linux-gnu/Dockerfile deleted file mode 100644 index eb2bb4de8..000000000 --- a/crux-mir/lib/libc/ci/docker/aarch64-unknown-linux-gnu/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM ubuntu:19.10 -RUN apt-get update && apt-get install -y --no-install-recommends \ - gcc libc6-dev ca-certificates \ - gcc-aarch64-linux-gnu libc6-dev-arm64-cross qemu-user -ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \ - CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER="qemu-aarch64 -L /usr/aarch64-linux-gnu" \ - PATH=$PATH:/rust/bin diff --git a/crux-mir/lib/libc/ci/docker/aarch64-unknown-linux-musl/Dockerfile b/crux-mir/lib/libc/ci/docker/aarch64-unknown-linux-musl/Dockerfile deleted file mode 100644 index 7cd35d4f0..000000000 --- a/crux-mir/lib/libc/ci/docker/aarch64-unknown-linux-musl/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM ubuntu:19.10 - -RUN apt-get update && apt-get install -y --no-install-recommends \ - gcc make libc6-dev git curl ca-certificates \ - gcc-aarch64-linux-gnu qemu-user - -COPY install-musl.sh / -RUN sh /install-musl.sh aarch64 - -# FIXME: shouldn't need the `-lgcc` here, shouldn't that be in libstd? -ENV PATH=$PATH:/musl-aarch64/bin:/rust/bin \ - CC_aarch64_unknown_linux_musl=musl-gcc \ - RUSTFLAGS='-Clink-args=-lgcc' \ - CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=musl-gcc \ - CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER="qemu-aarch64 -L /musl-aarch64" diff --git a/crux-mir/lib/libc/ci/docker/arm-linux-androideabi/Dockerfile b/crux-mir/lib/libc/ci/docker/arm-linux-androideabi/Dockerfile deleted file mode 100644 index 6da8f25b3..000000000 --- a/crux-mir/lib/libc/ci/docker/arm-linux-androideabi/Dockerfile +++ /dev/null @@ -1,45 +0,0 @@ -FROM ubuntu:19.10 - -RUN dpkg --add-architecture i386 && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - file \ - curl \ - ca-certificates \ - python \ - unzip \ - expect \ - openjdk-8-jre \ - libstdc++6:i386 \ - libpulse0 \ - gcc \ - libc6-dev - -WORKDIR /android/ -COPY android* /android/ - -ENV ANDROID_ARCH=arm -ENV PATH=$PATH:/android/ndk-$ANDROID_ARCH/bin:/android/sdk/tools:/android/sdk/platform-tools - -RUN sh /android/android-install-ndk.sh $ANDROID_ARCH -RUN sh /android/android-install-sdk.sh $ANDROID_ARCH -RUN mv /root/.android /tmp -RUN chmod 777 -R /tmp/.android -RUN chmod 755 /android/sdk/tools/* /android/sdk/emulator/qemu/linux-x86_64/* - -ENV PATH=$PATH:/rust/bin \ - CARGO_TARGET_ARM_LINUX_ANDROIDEABI_LINKER=arm-linux-androideabi-gcc \ - CARGO_TARGET_ARM_LINUX_ANDROIDEABI_RUNNER=/tmp/runtest \ - HOME=/tmp - -ADD runtest-android.rs /tmp/runtest.rs -ENTRYPOINT [ \ - "bash", \ - "-c", \ - # set SHELL so android can detect a 64bits system, see - # http://stackoverflow.com/a/41789144 - "SHELL=/bin/dash /android/sdk/emulator/emulator @arm -no-window & \ - rustc /tmp/runtest.rs -o /tmp/runtest && \ - exec \"$@\"", \ - "--" \ -] diff --git a/crux-mir/lib/libc/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile b/crux-mir/lib/libc/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile deleted file mode 100644 index b6980d81d..000000000 --- a/crux-mir/lib/libc/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM ubuntu:19.10 -RUN apt-get update && apt-get install -y --no-install-recommends \ - gcc libc6-dev ca-certificates \ - gcc-arm-linux-gnueabihf libc6-dev-armhf-cross qemu-user -ENV CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_LINKER=arm-linux-gnueabihf-gcc \ - CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_RUNNER="qemu-arm -L /usr/arm-linux-gnueabihf" \ - PATH=$PATH:/rust/bin diff --git a/crux-mir/lib/libc/ci/docker/arm-unknown-linux-musleabihf/Dockerfile b/crux-mir/lib/libc/ci/docker/arm-unknown-linux-musleabihf/Dockerfile deleted file mode 100644 index 8315721dc..000000000 --- a/crux-mir/lib/libc/ci/docker/arm-unknown-linux-musleabihf/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM ubuntu:19.10 - -RUN apt-get update && apt-get install -y --no-install-recommends \ - gcc make libc6-dev git curl ca-certificates \ - gcc-arm-linux-gnueabihf qemu-user - -COPY install-musl.sh / -RUN sh /install-musl.sh arm - -ENV PATH=$PATH:/musl-arm/bin:/rust/bin \ - CC_arm_unknown_linux_musleabihf=musl-gcc \ - CARGO_TARGET_ARM_UNKNOWN_LINUX_MUSLEABIHF_LINKER=musl-gcc \ - CARGO_TARGET_ARM_UNKNOWN_LINUX_MUSLEABIHF_RUNNER="qemu-arm -L /musl-arm" diff --git a/crux-mir/lib/libc/ci/docker/asmjs-unknown-emscripten/Dockerfile b/crux-mir/lib/libc/ci/docker/asmjs-unknown-emscripten/Dockerfile deleted file mode 100644 index 76cd02ef3..000000000 --- a/crux-mir/lib/libc/ci/docker/asmjs-unknown-emscripten/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM ubuntu:19.10 - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - ca-certificates \ - curl \ - gcc \ - git \ - libc6-dev \ - libxml2 \ - python \ - xz-utils - -COPY emscripten.sh / -RUN bash /emscripten.sh - -ENV PATH=$PATH:/rust/bin \ - CARGO_TARGET_ASMJS_UNKNOWN_EMSCRIPTEN_RUNNER=node - -COPY emscripten-entry.sh / -ENTRYPOINT ["/emscripten-entry.sh"] diff --git a/crux-mir/lib/libc/ci/docker/i686-linux-android/Dockerfile b/crux-mir/lib/libc/ci/docker/i686-linux-android/Dockerfile deleted file mode 100644 index 4703736d9..000000000 --- a/crux-mir/lib/libc/ci/docker/i686-linux-android/Dockerfile +++ /dev/null @@ -1,45 +0,0 @@ -FROM ubuntu:19.10 - -RUN dpkg --add-architecture i386 && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - file \ - curl \ - ca-certificates \ - python \ - unzip \ - expect \ - openjdk-8-jre \ - libstdc++6:i386 \ - libpulse0 \ - gcc \ - libc6-dev - -WORKDIR /android/ -COPY android* /android/ - -ENV ANDROID_ARCH=i686 -ENV PATH=$PATH:/android/ndk-$ANDROID_ARCH/bin:/android/sdk/tools:/android/sdk/platform-tools - -RUN sh /android/android-install-ndk.sh $ANDROID_ARCH -RUN sh /android/android-install-sdk.sh $ANDROID_ARCH -RUN mv /root/.android /tmp -RUN chmod 777 -R /tmp/.android -RUN chmod 755 /android/sdk/tools/* /android/sdk/emulator/qemu/linux-x86_64/* - -ENV PATH=$PATH:/rust/bin \ - CARGO_TARGET_I686_LINUX_ANDROID_LINKER=i686-linux-android-gcc \ - CARGO_TARGET_I686_LINUX_ANDROID_RUNNER=/tmp/runtest \ - HOME=/tmp - -ADD runtest-android.rs /tmp/runtest.rs -ENTRYPOINT [ \ - "bash", \ - "-c", \ - # set SHELL so android can detect a 64bits system, see - # http://stackoverflow.com/a/41789144 - "SHELL=/bin/dash /android/sdk/emulator/emulator @i686 -no-window -no-accel & \ - rustc /tmp/runtest.rs -o /tmp/runtest && \ - exec \"$@\"", \ - "--" \ -] diff --git a/crux-mir/lib/libc/ci/docker/i686-unknown-linux-gnu/Dockerfile b/crux-mir/lib/libc/ci/docker/i686-unknown-linux-gnu/Dockerfile deleted file mode 100644 index 3f90923a0..000000000 --- a/crux-mir/lib/libc/ci/docker/i686-unknown-linux-gnu/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM ubuntu:19.10 -RUN apt-get update -RUN apt-get install -y --no-install-recommends \ - gcc-multilib libc6-dev ca-certificates -ENV PATH=$PATH:/rust/bin diff --git a/crux-mir/lib/libc/ci/docker/i686-unknown-linux-musl/Dockerfile b/crux-mir/lib/libc/ci/docker/i686-unknown-linux-musl/Dockerfile deleted file mode 100644 index a8834366e..000000000 --- a/crux-mir/lib/libc/ci/docker/i686-unknown-linux-musl/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM ubuntu:19.10 - -RUN dpkg --add-architecture i386 -RUN apt-get update -RUN apt-get install -y --no-install-recommends \ - gcc-multilib make libc6-dev git curl ca-certificates libc6:i386 - -COPY install-musl.sh / -RUN sh /install-musl.sh i686 - -ENV PATH=$PATH:/musl-i686/bin:/rust/bin \ - CC_i686_unknown_linux_musl=musl-gcc diff --git a/crux-mir/lib/libc/ci/docker/mips-unknown-linux-gnu/Dockerfile b/crux-mir/lib/libc/ci/docker/mips-unknown-linux-gnu/Dockerfile deleted file mode 100644 index a45075a78..000000000 --- a/crux-mir/lib/libc/ci/docker/mips-unknown-linux-gnu/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -# FIXME: Ubuntu 19.10 is missing gcc-mips-linux-gnu. -FROM ubuntu:19.04 - -RUN apt-get update && apt-get install -y --no-install-recommends \ - gcc libc6-dev qemu-user ca-certificates \ - gcc-mips-linux-gnu libc6-dev-mips-cross \ - qemu-system-mips linux-headers-generic - -ENV CARGO_TARGET_MIPS_UNKNOWN_LINUX_GNU_LINKER=mips-linux-gnu-gcc \ - CARGO_TARGET_MIPS_UNKNOWN_LINUX_GNU_RUNNER="qemu-mips -L /usr/mips-linux-gnu" \ - PATH=$PATH:/rust/bin diff --git a/crux-mir/lib/libc/ci/docker/mips-unknown-linux-musl/Dockerfile b/crux-mir/lib/libc/ci/docker/mips-unknown-linux-musl/Dockerfile deleted file mode 100644 index f34f3f1b4..000000000 --- a/crux-mir/lib/libc/ci/docker/mips-unknown-linux-musl/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM ubuntu:19.10 - -RUN apt-get update && apt-get install -y --no-install-recommends \ - gcc libc6-dev qemu-user ca-certificates qemu-system-mips curl \ - xz-utils patch - -RUN mkdir /toolchain - -# Linux kernel version: 4.14.151 -# See build_dir/target-mips_24kc_musl/linux-ar71xx_generic/linux-4.14.151 -# Musl version: 1.1.24 -# See staging_dir/toolchain-mips_24kc_gcc-7.4.0_musl/info.mk -RUN curl --retry 5 -L https://downloads.openwrt.org/releases/19.07.0-rc1/targets/ar71xx/generic/openwrt-sdk-19.07.0-rc1-ar71xx-generic_gcc-7.4.0_musl.Linux-x86_64.tar.xz | \ - tar xJf - -C /toolchain --strip-components=1 - -# See https://lkml.org/lkml/2014/3/14/269 -COPY sysinfo_guard.patch /toolchain -RUN patch /toolchain/staging_dir/toolchain-mips_24kc_gcc-7.4.0_musl/include/linux/kernel.h ) { - for line in BufReader::new(input).lines() { - let line = line.unwrap(); - println!("{}", line); - if (line.starts_with("PASSED ") && line.contains(" tests")) || - line.starts_with("test result: ok"){ - tx.send(()).unwrap(); - } - } -} diff --git a/crux-mir/lib/libc/ci/docker/x86_64-unknown-linux-gnu/Dockerfile b/crux-mir/lib/libc/ci/docker/x86_64-unknown-linux-gnu/Dockerfile deleted file mode 100644 index e73d2882a..000000000 --- a/crux-mir/lib/libc/ci/docker/x86_64-unknown-linux-gnu/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM ubuntu:19.10 -RUN apt-get update -RUN apt-get install -y --no-install-recommends \ - gcc libc6-dev ca-certificates linux-headers-generic - -RUN apt search linux-headers -RUN ls /usr/src - -ENV PATH=$PATH:/rust/bin diff --git a/crux-mir/lib/libc/ci/docker/x86_64-unknown-linux-gnux32/Dockerfile b/crux-mir/lib/libc/ci/docker/x86_64-unknown-linux-gnux32/Dockerfile deleted file mode 100644 index 3f90923a0..000000000 --- a/crux-mir/lib/libc/ci/docker/x86_64-unknown-linux-gnux32/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM ubuntu:19.10 -RUN apt-get update -RUN apt-get install -y --no-install-recommends \ - gcc-multilib libc6-dev ca-certificates -ENV PATH=$PATH:/rust/bin diff --git a/crux-mir/lib/libc/ci/docker/x86_64-unknown-linux-musl/Dockerfile b/crux-mir/lib/libc/ci/docker/x86_64-unknown-linux-musl/Dockerfile deleted file mode 100644 index 5c51b7a1a..000000000 --- a/crux-mir/lib/libc/ci/docker/x86_64-unknown-linux-musl/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM ubuntu:19.10 - -RUN apt-get update -RUN apt-get install -y --no-install-recommends \ - gcc make libc6-dev git curl ca-certificates - -COPY install-musl.sh / -RUN sh /install-musl.sh x86_64 - -ENV PATH=$PATH:/musl-x86_64/bin:/rust/bin diff --git a/crux-mir/lib/libc/ci/dox.sh b/crux-mir/lib/libc/ci/dox.sh deleted file mode 100644 index 271b54b4f..000000000 --- a/crux-mir/lib/libc/ci/dox.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env sh - -# Builds documentation for all target triples that we have a registered URL for -# in liblibc. This scrapes the list of triples to document from `src/lib.rs` -# which has a bunch of `html_root_url` directives we pick up. - -set -ex - -TARGET_DOC_DIR=target/doc -README=README.md -PLATFORM_SUPPORT=platform-support.md - -rm -rf $TARGET_DOC_DIR -mkdir -p $TARGET_DOC_DIR - -if ! rustc --version | grep -E "nightly" ; then - echo "Building the documentation requires a nightly Rust toolchain" - exit 1 -fi - -rustup component add rust-src -cargo +nightly install cargo-xbuild - -# List all targets that do currently build successfully: -# shellcheck disable=SC1003 -grep '[\d|\w|-]* \\' ci/build.sh > targets -sed -i.bak 's/ \\//g' targets -grep '^[_a-zA-Z0-9-]*$' targets | sort > tmp && mv tmp targets - -# Create a markdown list of supported platforms in $PLATFORM_SUPPORT -rm $PLATFORM_SUPPORT || true - -printf '### Platform-specific documentation\n' >> $PLATFORM_SUPPORT - -while read -r target; do - echo "documenting ${target}" - - case "${target}" in - *apple*) - # FIXME: - # We can't build docs of apple targets from Linux yet. - continue - ;; - *) - ;; - esac - - rustup target add "${target}" || true - - # Enable extra configuration flags: - export RUSTDOCFLAGS="--cfg freebsd11" - - # If cargo doc fails, then try xargo: - if ! cargo doc --target "${target}" \ - --no-default-features --features extra_traits ; then - cargo xdoc --target "${target}" \ - --no-default-features --features extra_traits - fi - - cp -r "target/${target}/doc" "${TARGET_DOC_DIR}/${target}" - - echo "* [${target}](${target}/libc/index.html)" >> $PLATFORM_SUPPORT -done < targets - -# Replace
with the contents of $PLATFORM_SUPPORT -cp $README $TARGET_DOC_DIR -line=$(grep -n '
' $README | cut -d ":" -f 1) - -set +x -{ head -n "$((line-1))" $README; cat $PLATFORM_SUPPORT; tail -n "+$((line+1))" $README; } > $TARGET_DOC_DIR/$README -set -x - -# Copy the licenses -cp LICENSE-* $TARGET_DOC_DIR/ diff --git a/crux-mir/lib/libc/ci/emscripten-entry.sh b/crux-mir/lib/libc/ci/emscripten-entry.sh deleted file mode 100755 index 0016f5660..000000000 --- a/crux-mir/lib/libc/ci/emscripten-entry.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2017 The Rust Project Developers. See the COPYRIGHT -# file at the top-level directory of this distribution and at -# http://rust-lang.org/COPYRIGHT. -# -# Licensed under the Apache License, Version 2.0 or the MIT license -# , at your -# option. This file may not be copied, modified, or distributed -# except according to those terms. - -set -ex - -# shellcheck disable=SC1091 -source /emsdk-portable/emsdk_env.sh &> /dev/null - -# emsdk-portable provides a node binary, but we need version 8 to run wasm -export PATH="/node-v12.3.1-linux-x64/bin:$PATH" - -exec "$@" diff --git a/crux-mir/lib/libc/ci/emscripten.sh b/crux-mir/lib/libc/ci/emscripten.sh deleted file mode 100644 index 3c2650c31..000000000 --- a/crux-mir/lib/libc/ci/emscripten.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2017 The Rust Project Developers. See the COPYRIGHT -# file at the top-level directory of this distribution and at -# http://rust-lang.org/COPYRIGHT. -# -# Licensed under the Apache License, Version 2.0 or the MIT license -# , at your -# option. This file may not be copied, modified, or distributed -# except according to those terms. - -set -ex - -hide_output() { - set +x - on_err=" -echo ERROR: An error was encountered with the build. -cat /tmp/build.log -exit 1 -" - trap '$on_err' ERR - bash -c "while true; do sleep 30; echo \$(date) - building ...; done" & - PING_LOOP_PID=$! - "${@}" &> /tmp/build.log - trap - ERR - kill $PING_LOOP_PID - rm -f /tmp/build.log - set -x -} - -git clone https://github.com/emscripten-core/emsdk.git /emsdk-portable -cd /emsdk-portable -# FIXME: switch to an upstream install once -# https://github.com/rust-lang/rust/pull/63649 lands -hide_output ./emsdk install 1.38.42 -./emsdk activate 1.38.42 - -# Compile and cache libc -# shellcheck disable=SC1091 -source ./emsdk_env.sh -echo "main(){}" > a.c -HOME=/emsdk-portable/ emcc a.c -rm -f a.* - -# Make emsdk usable by any user -cp /root/.emscripten /emsdk-portable -chmod a+rxw -R /emsdk-portable - -# node 8 is required to run wasm -cd / -curl --retry 5 -L https://nodejs.org/dist/v12.3.1/node-v12.3.1-linux-x64.tar.xz | \ - tar -xJ diff --git a/crux-mir/lib/libc/ci/install-musl.sh b/crux-mir/lib/libc/ci/install-musl.sh deleted file mode 100644 index 88cb4f0b1..000000000 --- a/crux-mir/lib/libc/ci/install-musl.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env sh -# -# Install musl and musl-sanitized linux kernel headers -# to musl-{$1} directory - -set -ex - -MUSL_VERSION=1.1.24 -MUSL="musl-${MUSL_VERSION}" - -# Download, configure, build, and install musl: -curl --retry 5 https://www.musl-libc.org/releases/${MUSL}.tar.gz | tar xzf - - -cd $MUSL -case ${1} in - aarch64) - musl_arch=aarch64 - kernel_arch=arm64 - CC=aarch64-linux-gnu-gcc \ - ./configure --prefix="/musl-${musl_arch}" --enable-wrapper=yes - make install -j4 - ;; - arm) - musl_arch=arm - kernel_arch=arm - CC=arm-linux-gnueabihf-gcc CFLAGS="-march=armv6 -marm -mfpu=vfp" \ - ./configure --prefix="/musl-${musl_arch}" --enable-wrapper=yes - make install -j4 - ;; - i686) - # cross-compile musl for i686 using the system compiler on an x86_64 - # system. - musl_arch=i686 - kernel_arch=i386 - # Specifically pass -m32 in CFLAGS and override CC when running - # ./configure, since otherwise the script will fail to find a compiler. - CC=gcc CFLAGS="-m32" \ - ./configure --prefix="/musl-${musl_arch}" --disable-shared --target=i686 - # unset CROSS_COMPILE when running make; otherwise the makefile will - # call the non-existent binary 'i686-ar'. - make CROSS_COMPILE= install -j4 - ;; - x86_64) - musl_arch=x86_64 - kernel_arch=x86_64 - ./configure --prefix="/musl-${musl_arch}" - make install -j4 - ;; - mips64) - musl_arch=mips64 - kernel_arch=mips - CC=mips64-linux-gnuabi64-gcc CFLAGS="-march=mips64r2 -mabi=64" \ - ./configure --prefix="/musl-${musl_arch}" --enable-wrapper=yes - make install -j4 - ;; - mips64el) - musl_arch=mips64el - kernel_arch=mips - CC=mips64el-linux-gnuabi64-gcc CFLAGS="-march=mips64r2 -mabi=64" \ - ./configure --prefix="/musl-${musl_arch}" --enable-wrapper=yes - make install -j4 - ;; - *) - echo "Unknown target arch: \"${1}\"" - exit 1 - ;; -esac - - -# shellcheck disable=SC2103 -cd .. -rm -rf $MUSL - -# Download, configure, build, and install musl-sanitized kernel headers: -KERNEL_HEADER_VER="4.4.2-2" -curl --retry 5 -L \ - "https://github.com/sabotage-linux/kernel-headers/archive/v${KERNEL_HEADER_VER}.tar.gz" | \ - tar xzf - -( - cd kernel-headers-${KERNEL_HEADER_VER} - make ARCH="${kernel_arch}" prefix="/musl-${musl_arch}" install -j4 -) -rm -rf kernel-headers-${KERNEL_HEADER_VER} diff --git a/crux-mir/lib/libc/ci/ios/deploy_and_run_on_ios_simulator.rs b/crux-mir/lib/libc/ci/ios/deploy_and_run_on_ios_simulator.rs deleted file mode 100644 index 2075be6d6..000000000 --- a/crux-mir/lib/libc/ci/ios/deploy_and_run_on_ios_simulator.rs +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright 2017 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -// This is a script to deploy and execute a binary on an iOS simulator. -// The primary use of this is to be able to run unit tests on the simulator and -// retrieve the results. -// -// To do this through Cargo instead, use Dinghy -// (https://github.com/snipsco/dinghy): cargo dinghy install, then cargo dinghy -// test. - -use std::env; -use std::fs::{self, File}; -use std::io::Write; -use std::path::Path; -use std::process; -use std::process::Command; - -macro_rules! t { - ($e:expr) => (match $e { - Ok(e) => e, - Err(e) => panic!("{} failed with: {}", stringify!($e), e), - }) -} - -// Step one: Wrap as an app -fn package_as_simulator_app(crate_name: &str, test_binary_path: &Path) { - println!("Packaging simulator app"); - drop(fs::remove_dir_all("ios_simulator_app")); - t!(fs::create_dir("ios_simulator_app")); - t!(fs::copy(test_binary_path, - Path::new("ios_simulator_app").join(crate_name))); - - let mut f = t!(File::create("ios_simulator_app/Info.plist")); - t!(f.write_all(format!(r#" - - - - - CFBundleExecutable - {} - CFBundleIdentifier - com.rust.unittests - - - "#, crate_name).as_bytes())); -} - -// Step two: Start the iOS simulator -fn start_simulator() { - println!("Looking for iOS simulator"); - let output = t!(Command::new("xcrun").arg("simctl").arg("list").output()); - assert!(output.status.success()); - let mut simulator_exists = false; - let mut simulator_booted = false; - let mut found_rust_sim = false; - let stdout = t!(String::from_utf8(output.stdout)); - for line in stdout.lines() { - if line.contains("rust_ios") { - if found_rust_sim { - panic!("Duplicate rust_ios simulators found. Please \ - double-check xcrun simctl list."); - } - simulator_exists = true; - simulator_booted = line.contains("(Booted)"); - found_rust_sim = true; - } - } - - if simulator_exists == false { - println!("Creating iOS simulator"); - Command::new("xcrun") - .arg("simctl") - .arg("create") - .arg("rust_ios") - .arg("com.apple.CoreSimulator.SimDeviceType.iPhone-SE") - .arg("com.apple.CoreSimulator.SimRuntime.iOS-10-2") - .check_status(); - } else if simulator_booted == true { - println!("Shutting down already-booted simulator"); - Command::new("xcrun") - .arg("simctl") - .arg("shutdown") - .arg("rust_ios") - .check_status(); - } - - println!("Starting iOS simulator"); - // We can't uninstall the app (if present) as that will hang if the - // simulator isn't completely booted; just erase the simulator instead. - Command::new("xcrun").arg("simctl").arg("erase").arg("rust_ios").check_status(); - Command::new("xcrun").arg("simctl").arg("boot").arg("rust_ios").check_status(); -} - -// Step three: Install the app -fn install_app_to_simulator() { - println!("Installing app to simulator"); - Command::new("xcrun") - .arg("simctl") - .arg("install") - .arg("booted") - .arg("ios_simulator_app/") - .check_status(); -} - -// Step four: Run the app -fn run_app_on_simulator() { - println!("Running app"); - let output = t!(Command::new("xcrun") - .arg("simctl") - .arg("launch") - .arg("--console") - .arg("booted") - .arg("com.rust.unittests") - .output()); - - println!("status: {}", output.status); - println!("stdout --\n{}\n", String::from_utf8_lossy(&output.stdout)); - println!("stderr --\n{}\n", String::from_utf8_lossy(&output.stderr)); - - let stdout = String::from_utf8_lossy(&output.stdout); - let passed = stdout.lines() - .find(|l| - (l.contains("PASSED") && - l.contains("tests")) || - l.contains("test result: ok") - ) - .unwrap_or(false); - - println!("Shutting down simulator"); - Command::new("xcrun") - .arg("simctl") - .arg("shutdown") - .arg("rust_ios") - .check_status(); - if !passed { - panic!("tests didn't pass"); - } -} - -trait CheckStatus { - fn check_status(&mut self); -} - -impl CheckStatus for Command { - fn check_status(&mut self) { - println!("\trunning: {:?}", self); - assert!(t!(self.status()).success()); - } -} - -fn main() { - let args: Vec = env::args().collect(); - if args.len() != 2 { - println!("Usage: {} ", args[0]); - process::exit(-1); - } - - let test_binary_path = Path::new(&args[1]); - let crate_name = test_binary_path.file_name().unwrap(); - - package_as_simulator_app(crate_name.to_str().unwrap(), test_binary_path); - start_simulator(); - install_app_to_simulator(); - run_app_on_simulator(); -} diff --git a/crux-mir/lib/libc/ci/linux-s390x.sh b/crux-mir/lib/libc/ci/linux-s390x.sh deleted file mode 100644 index 18c71f45c..000000000 --- a/crux-mir/lib/libc/ci/linux-s390x.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env sh - -set -ex - -mkdir -m 777 /qemu -cd /qemu - -curl --retry 5 -LO https://github.com/qemu/qemu/raw/master/pc-bios/s390-ccw.img -curl --retry 5 -LO http://ftp.debian.org/debian/dists/testing/main/installer-s390x/20191129/images/generic/kernel.debian -curl --retry 5 -LO http://ftp.debian.org/debian/dists/testing/main/installer-s390x/20191129/images/generic/initrd.debian - -mv kernel.debian kernel -mv initrd.debian initrd.gz - -mkdir init -cd init -gunzip -c ../initrd.gz | cpio -id -rm ../initrd.gz -cp /usr/s390x-linux-gnu/lib/libgcc_s.so.1 usr/lib/ -chmod a+w . diff --git a/crux-mir/lib/libc/ci/linux-sparc64.sh b/crux-mir/lib/libc/ci/linux-sparc64.sh deleted file mode 100644 index 5580a0e3c..000000000 --- a/crux-mir/lib/libc/ci/linux-sparc64.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env sh - -set -ex - -mkdir -m 777 /qemu -cd /qemu - -curl --retry 5 -LO https://cdimage.debian.org/cdimage/ports/10.0/sparc64/iso-cd/debian-10.0-sparc64-NETINST-1.iso -7z e debian-10.0-sparc64-NETINST-1.iso boot/initrd.gz -7z e debian-10.0-sparc64-NETINST-1.iso boot/sparc64 -mv sparc64 kernel -rm debian-10.0-sparc64-NETINST-1.iso - -mkdir init -cd init -gunzip -c ../initrd.gz | cpio -id -rm ../initrd.gz -cp /usr/sparc64-linux-gnu/lib/libgcc_s.so.1 usr/lib/ -chmod a+w . diff --git a/crux-mir/lib/libc/ci/run-docker.sh b/crux-mir/lib/libc/ci/run-docker.sh deleted file mode 100755 index 3c0736a26..000000000 --- a/crux-mir/lib/libc/ci/run-docker.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env sh - -# Small script to run tests for a target (or all targets) inside all the -# respective docker images. - -set -ex - -echo "${HOME}" -pwd - -run() { - echo "Building docker container for target ${1}" - - # use -f so we can use ci/ as build context - docker build -t libc -f "ci/docker/${1}/Dockerfile" ci/ - mkdir -p target - if [ -w /dev/kvm ]; then - kvm="--volume /dev/kvm:/dev/kvm" - else - kvm="" - fi - - docker run \ - --rm \ - --user "$(id -u)":"$(id -g)" \ - --env LIBC_CI \ - --env CARGO_HOME=/cargo \ - --env CARGO_TARGET_DIR=/checkout/target \ - --volume "$(dirname "$(dirname "$(command -v cargo)")")":/cargo \ - --volume "$(rustc --print sysroot)":/rust:ro \ - --volume "$(pwd)":/checkout:ro \ - --volume "$(pwd)"/target:/checkout/target \ - $kvm \ - --init \ - --workdir /checkout \ - libc \ - sh -c "HOME=/tmp PATH=\$PATH:/rust/bin exec ci/run.sh ${1}" -} - -if [ -z "${1}" ]; then - for d in ci/docker/*; do - run "${d}" - done -else - run "${1}" -fi diff --git a/crux-mir/lib/libc/ci/run-qemu.sh b/crux-mir/lib/libc/ci/run-qemu.sh deleted file mode 100644 index 6fba62987..000000000 --- a/crux-mir/lib/libc/ci/run-qemu.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env sh - -# Initial script which is run inside of all qemu images. The first argument to -# this script (as arranged by the qemu image itself) is the path to where the -# libc crate is mounted. -# -# For qemu images we currently need to install Rust manually as this wasn't done -# by the initial run-travis.sh script -# -# FIXME: feels like run-travis.sh should be responsible for downloading the -# compiler. - -set -ex - -ROOT="${1}" -cp -r "${ROOT}/libc" /tmp/libc -cd /tmp/libc - -TARGET="$(cat "${ROOT}/TARGET")" -export CARGO_TARGET_DIR=/tmp - -case $TARGET in - *-openbsd) - pkg_add cargo gcc%4.9 rust - export CC=egcc - ;; - - *) - echo "Unknown target: ${TARGET}" - exit 1 - ;; -esac - -exec sh ci/run.sh "${TARGET}" diff --git a/crux-mir/lib/libc/ci/run.sh b/crux-mir/lib/libc/ci/run.sh deleted file mode 100755 index d0938121c..000000000 --- a/crux-mir/lib/libc/ci/run.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env sh - -# Builds and runs tests for a particular target passed as an argument to this -# script. - -set -ex - -MIRRORS_URL="https://ci-mirrors.rust-lang.org/libc" - -TARGET="${1}" - -# If we're going to run tests inside of a qemu image, then we don't need any of -# the scripts below. Instead, download the image, prepare a filesystem which has -# the current state of this repository, and then run the image. -# -# It's assume that all images, when run with two disks, will run the `run.sh` -# script from the second which we place inside. -if [ "$QEMU" != "" ]; then - tmpdir=/tmp/qemu-img-creation - mkdir -p "${tmpdir}" - - if [ -z "${QEMU#*.gz}" ]; then - # image is .gz : download and uncompress it - qemufile="$(echo "${QEMU%.gz}" | sed 's/\//__/g')" - if [ ! -f "${tmpdir}/${qemufile}" ]; then - curl --retry 5 "${MIRRORS_URL}/${QEMU}" | \ - gunzip -d > "${tmpdir}/${qemufile}" - fi - elif [ -z "${QEMU#*.xz}" ]; then - # image is .xz : download and uncompress it - qemufile="$(echo "${QEMU%.xz}" | sed 's/\//__/g')" - if [ ! -f "${tmpdir}/${qemufile}" ]; then - curl --retry 5 "${MIRRORS_URL}/${QEMU}" | \ - unxz > "${tmpdir}/${qemufile}" - fi - else - # plain qcow2 image: just download it - qemufile="$(echo "${QEMU}" | sed 's/\//__/g')" - if [ ! -f "${tmpdir}/${qemufile}" ]; then - curl --retry 5 "${MIRRORS_URL}/${QEMU}" \ - > "${tmpdir}/${qemufile}" - fi - fi - - # Create a mount a fresh new filesystem image that we'll later pass to QEMU. - # This will have a `run.sh` script will which use the artifacts inside to run - # on the host. - rm -f "${tmpdir}/libc-test.img" - mkdir "${tmpdir}/mount" - - # Do the standard rigamarole of cross-compiling an executable and then the - # script to run just executes the binary. - cargo build \ - --manifest-path libc-test/Cargo.toml \ - --target "${TARGET}" \ - --test main - rm "${CARGO_TARGET_DIR}/${TARGET}"/debug/main-*.d - cp "${CARGO_TARGET_DIR}/${TARGET}"/debug/main-* "${tmpdir}"/mount/libc-test - # shellcheck disable=SC2016 - echo 'exec $1/libc-test' > "${tmpdir}/mount/run.sh" - - du -sh "${tmpdir}/mount" - genext2fs \ - --root "${tmpdir}/mount" \ - --size-in-blocks 100000 \ - "${tmpdir}/libc-test.img" - - # Pass -snapshot to prevent tampering with the disk images, this helps when - # running this script in development. The two drives are then passed next, - # first is the OS and second is the one we just made. Next the network is - # configured to work (I'm not entirely sure how), and then finally we turn off - # graphics and redirect the serial console output to out.log. - qemu-system-x86_64 \ - -m 1024 \ - -snapshot \ - -drive if=virtio,file="${tmpdir}/${qemufile}" \ - -drive if=virtio,file="${tmpdir}/libc-test.img" \ - -net nic,model=virtio \ - -net user \ - -nographic \ - -vga none 2>&1 | tee "${CARGO_TARGET_DIR}/out.log" - exec egrep "^(PASSED)|(test result: ok)" "${CARGO_TARGET_DIR}/out.log" -fi - -# FIXME: x86_64-unknown-linux-gnux32 fail to compile without --release -# See https://github.com/rust-lang/rust/issues/59220 -opt= -if [ "$TARGET" = "x86_64-unknown-linux-gnux32" ]; then - opt="--release" -fi - -cargo test $opt --no-default-features --manifest-path libc-test/Cargo.toml \ - --target "${TARGET}" - -cargo test $opt --manifest-path libc-test/Cargo.toml --target "${TARGET}" - -cargo test $opt --features extra_traits --manifest-path libc-test/Cargo.toml \ - --target "${TARGET}" diff --git a/crux-mir/lib/libc/ci/runtest-android.rs b/crux-mir/lib/libc/ci/runtest-android.rs deleted file mode 100644 index b8030c41a..000000000 --- a/crux-mir/lib/libc/ci/runtest-android.rs +++ /dev/null @@ -1,47 +0,0 @@ -use std::env; -use std::process::Command; -use std::path::{Path, PathBuf}; - -fn main() { - let args = env::args_os() - .skip(1) - .filter(|arg| arg != "--quiet") - .collect::>(); - assert_eq!(args.len(), 1); - let test = PathBuf::from(&args[0]); - let dst = Path::new("/data/local/tmp").join(test.file_name().unwrap()); - - let status = Command::new("adb") - .arg("wait-for-device") - .status() - .expect("failed to run: adb wait-for-device"); - assert!(status.success()); - - let status = Command::new("adb") - .arg("push") - .arg(&test) - .arg(&dst) - .status() - .expect("failed to run: adb pushr"); - assert!(status.success()); - - let output = Command::new("adb") - .arg("shell") - .arg(&dst) - .output() - .expect("failed to run: adb shell"); - assert!(status.success()); - - println!("status: {}\nstdout ---\n{}\nstderr ---\n{}", - output.status, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr)); - - let stdout = String::from_utf8_lossy(&output.stdout); - stdout.lines().find(|l| - (l.starts_with("PASSED ") && l.contains(" tests")) || - l.starts_with("test result: ok") - ).unwrap_or_else(|| { - panic!("failed to find successful test run"); - }); -} diff --git a/crux-mir/lib/libc/ci/semver.sh b/crux-mir/lib/libc/ci/semver.sh deleted file mode 100644 index fb58415e3..000000000 --- a/crux-mir/lib/libc/ci/semver.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env sh - -# Checks that libc does not contain breaking changes for the following targets. - -set -ex - -OS=${1} - -echo "Testing Semver on ${OS}" - -if ! rustc --version | grep -E "nightly" ; then - echo "Building semverver requires a nightly Rust toolchain" - exit 1 -fi - -rustup component add rustc-dev - -# FIXME: Use upstream once it gets rustup. -cargo +nightly install semververfork - -TARGETS= -case "${OS}" in - *linux*) - TARGETS="\ -aarch64-fuchsia \ -aarch64-linux-android \ -aarch64-unknown-linux-gnu \ -aarch64-unknown-linux-musl \ -armv7-linux-androideabi \ -armv7-unknown-linux-gnueabihf \ -i586-unknown-linux-gnu \ -i586-unknown-linux-musl \ -i686-linux-android \ -i686-unknown-freebsd \ -i686-unknown-linux-gnu \ -i686-unknown-linux-musl \ -i686-pc-windows-gnu \ -x86_64-unknown-freebsd \ -x86_64-unknown-linux-gnu \ -x86_64-unknown-linux-musl \ -x86_64-unknown-netbsd \ -x86_64-unknown-cloudabi \ -x86_64-sun-solaris \ -x86_64-fuchsia \ -x86_64-pc-windows-gnu \ -x86_64-unknown-linux-gnux32 \ -x86_64-unknown-redox \ -x86_64-fortanix-unknown-sgx \ -wasm32-unknown-unknown \ -" - ;; - *osx*) - TARGETS="\ -aarch64-apple-ios \ -x86_64-apple-darwin \ -x86_64-apple-ios \ -" - ;; -esac - -for TARGET in $TARGETS; do - # FIXME: rustup often fails to download some artifacts due to network - # issues, so we retry this N times. - N=5 - n=0 - until [ $n -ge $N ] - do - if rustup target add "${TARGET}" ; then - break - fi - n=$((n+1)) - sleep 1 - done - - # FIXME: Use upstream once it gets rustup. - cargo +nightly semverfork --api-guidelines --target="${TARGET}" -done diff --git a/crux-mir/lib/libc/ci/style.rs b/crux-mir/lib/libc/ci/style.rs deleted file mode 100644 index dcb3536ec..000000000 --- a/crux-mir/lib/libc/ci/style.rs +++ /dev/null @@ -1,208 +0,0 @@ -//! Simple script to verify the coding style of this library -//! -//! ## How to run -//! -//! The first argument to this script is the directory to run on, so running -//! this script should be as simple as: -//! -//! ```notrust -//! rustc ci/style.rs -//! ./style src -//! ``` -//! -//! ## Guidelines -//! -//! The current style is: -//! -//! * No trailing whitespace -//! * No tabs -//! * 80-character lines -//! * Specific module layout: -//! 1. use directives -//! 2. typedefs -//! 3. structs -//! 4. constants -//! 5. f! { ... } functions -//! 6. extern functions -//! 7. modules + pub use -//! -//! Things not verified: -//! -//! * alignment -//! * 4-space tabs -//! * leading colons on paths - -use std::env; -use std::fs; -use std::io::prelude::*; -use std::path::Path; - -macro_rules! t { - ($e:expr) => (match $e { - Ok(e) => e, - Err(e) => panic!("{} failed with {}", stringify!($e), e), - }) -} - -fn main() { - let arg = env::args().skip(1).next().unwrap_or(".".to_string()); - - let mut errors = Errors { errs: false }; - walk(Path::new(&arg), &mut errors); - - if errors.errs { - panic!("found some lint errors"); - } else { - println!("good style!"); - } -} - -fn walk(path: &Path, err: &mut Errors) { - for entry in t!(path.read_dir()).map(|e| t!(e)) { - let path = entry.path(); - if t!(entry.file_type()).is_dir() { - walk(&path, err); - continue - } - - let name = entry.file_name().into_string().unwrap(); - match &name[..] { - n if !n.ends_with(".rs") => continue, - - "dox.rs" | - "lib.rs" | - "ctypes.rs" | - "libc.rs" | - "macros.rs" => continue, - - _ => {} - } - - let mut contents = String::new(); - t!(t!(fs::File::open(&path)).read_to_string(&mut contents)); - - check_style(&contents, &path, err); - } -} - -struct Errors { - errs: bool, -} - -#[derive(Clone, Copy, PartialEq)] -enum State { - Start, - Imports, - Typedefs, - Structs, - Constants, - FunctionDefinitions, - Functions, - Modules, -} - -fn check_style(file: &str, path: &Path, err: &mut Errors) { - let mut state = State::Start; - let mut s_macros = 0; - let mut f_macros = 0; - let mut prev_blank = false; - - for (i, line) in file.lines().enumerate() { - if line == "" { - if prev_blank { - err.error(path, i, "double blank line"); - } - prev_blank = true; - } else { - prev_blank = false; - } - if line != line.trim_end() { - err.error(path, i, "trailing whitespace"); - } - if line.contains("\t") { - err.error(path, i, "tab character"); - } - if line.len() > 80 { - err.error(path, i, "line longer than 80 chars"); - } - if line.contains("#[cfg(") && !line.contains(" if ") - && !(line.contains("target_endian") || - line.contains("target_arch")) - { - if state != State::Structs { - err.error(path, i, "use cfg_if! and submodules \ - instead of #[cfg]"); - } - } - - let line = line.trim_start(); - let is_pub = line.starts_with("pub "); - let line = if is_pub {&line[4..]} else {line}; - - let line_state = if line.starts_with("use ") { - if line.contains("c_void") { - continue; - } - if is_pub { - State::Modules - } else { - State::Imports - } - } else if line.starts_with("const ") { - State::Constants - } else if line.starts_with("type ") { - State::Typedefs - } else if line.starts_with("s! {") { - s_macros += 1; - State::Structs - } else if line.starts_with("f! {") { - f_macros += 1; - State::FunctionDefinitions - } else if line.starts_with("extern ") { - State::Functions - } else if line.starts_with("mod ") { - State::Modules - } else { - continue - }; - - if state as usize > line_state as usize { - err.error(path, i, &format!("{} found after {} when \ - it belongs before", - line_state.desc(), state.desc())); - } - - if f_macros == 2 { - f_macros += 1; - err.error(path, i, "multiple f! macros in one module"); - } - if s_macros == 2 { - s_macros += 1; - err.error(path, i, "multiple s! macros in one module"); - } - - state = line_state; - } -} - -impl State { - fn desc(&self) -> &str { - match *self { - State::Start => "start", - State::Imports => "import", - State::Typedefs => "typedef", - State::Structs => "struct", - State::Constants => "constant", - State::FunctionDefinitions => "function definition", - State::Functions => "extern function", - State::Modules => "module", - } - } -} - -impl Errors { - fn error(&mut self, path: &Path, line: usize, msg: &str) { - self.errs = true; - println!("{}:{} - {}", path.display(), line + 1, msg); - } -} diff --git a/crux-mir/lib/libc/ci/style.sh b/crux-mir/lib/libc/ci/style.sh deleted file mode 100644 index 7acd128de..000000000 --- a/crux-mir/lib/libc/ci/style.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env sh - -set -ex - -rustc ci/style.rs && ./style src - -if rustup component add rustfmt-preview ; then - command -v rustfmt - rustfmt -V - cargo fmt --all -- --check -fi - -if shellcheck --version ; then - shellcheck -e SC2103 ci/*.sh -else - echo "shellcheck not found" - exit 1 -fi - diff --git a/crux-mir/lib/libc/ci/switch.json b/crux-mir/lib/libc/ci/switch.json deleted file mode 100644 index bc1894879..000000000 --- a/crux-mir/lib/libc/ci/switch.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "family": "unix", - "env": "newlib", - "target-env": "newlib", - "target-family": "unix", - "target-c-int-width": "32", - "target-endian": "little", - "target-pointer-width": "64", - "os": "horizon", - "arch": "aarch64", - "panic-strategy": "unwind", - "abi-blacklist": [ - "stdcall", - "fastcall", - "vectorcall", - "thiscall", - "win64", - "sysv64" - ], - "dynamic-linking" : false, - "features": "+a53,+strict-align", - "data-layout": "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128", - "executables": true, - "position-independent-executables" : true, - "linker-flavor": "gcc", - "llvm-target": "aarch64-unknown-none", - "has-elf-tls" : false, - "linker-is-gnu" : true, - "disable-redzone" : true, - "relocation-model" : "pic", - "max-atomic-width": 128, - "exe-suffix": ".elf", - "staticlib-suffix" : ".a", - "trap-unreachable" : true, - "emit-debug-gdb-scripts" : true, - "requires-uwtable" : true -} \ No newline at end of file diff --git a/crux-mir/lib/libc/ci/sysinfo_guard.patch b/crux-mir/lib/libc/ci/sysinfo_guard.patch deleted file mode 100644 index 69fb795c0..000000000 --- a/crux-mir/lib/libc/ci/sysinfo_guard.patch +++ /dev/null @@ -1,10 +0,0 @@ -@@ -2,7 +2,9 @@ - #ifndef _LINUX_KERNEL_H - #define _LINUX_KERNEL_H - -+#ifdef __GLIBC__ - #include -+#endif - - /* - * 'kernel.h' contains some often-used function prototypes etc diff --git a/crux-mir/lib/libc/ci/test-runner-linux b/crux-mir/lib/libc/ci/test-runner-linux deleted file mode 100755 index cad31ec4c..000000000 --- a/crux-mir/lib/libc/ci/test-runner-linux +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh - -set -e - -arch=$1 -prog=$2 - -cd /qemu/init -echo "#!/bin/sh\n/prog --color=never" > run_prog.sh -chmod +x run_prog.sh -cp -f $2 prog -find . | cpio --create --format='newc' --quiet | gzip > ../initrd.gz -cd .. - -timeout 30s qemu-system-$arch \ - -m 1024 \ - -nographic \ - -kernel kernel \ - -initrd initrd.gz \ - -append init=/run_prog.sh > output || true - -# remove kernel messages -tr -d '\r' < output | egrep -v '^\[' - -egrep "(PASSED)|(test result: ok)" output > /dev/null diff --git a/crux-mir/lib/libc/libc-test/Cargo.toml b/crux-mir/lib/libc/libc-test/Cargo.toml deleted file mode 100644 index 0b6866d49..000000000 --- a/crux-mir/lib/libc/libc-test/Cargo.toml +++ /dev/null @@ -1,59 +0,0 @@ -[package] -name = "libc-test" -version = "0.1.0" -authors = ["Alex Crichton "] -build = "build.rs" - -[dependencies.libc] -path = ".." -default-features = false - -[build-dependencies] -cc = "1.0" -ctest = "0.2" - -[features] -default = [ "std" ] -std = [ "libc/std" ] -align = [ "libc/align" ] -extra_traits = [ "libc/extra_traits" ] - -[[test]] -name = "main" -path = "test/main.rs" -harness = false - -[[test]] -name = "linux-fcntl" -path = "test/linux_fcntl.rs" -harness = false - -[[test]] -name = "linux-ipv6" -path = "test/linux_ipv6.rs" -harness = false - -[[test]] -name = "linux-elf" -path = "test/linux_elf.rs" -harness = false - -[[test]] -name = "linux-strerror_r" -path = "test/linux_strerror_r.rs" -harness = false - -[[test]] -name = "linux-termios" -path = "test/linux_termios.rs" -harness = false - -[[test]] -name = "cmsg" -path = "test/cmsg.rs" -harness = true - -[[test]] -name = "errqueue" -path = "test/errqueue.rs" -harness = true diff --git a/crux-mir/lib/libc/libc-test/build.rs b/crux-mir/lib/libc/libc-test/build.rs deleted file mode 100644 index 1f7152e42..000000000 --- a/crux-mir/lib/libc/libc-test/build.rs +++ /dev/null @@ -1,2717 +0,0 @@ -#![deny(warnings)] - -extern crate cc; -extern crate ctest; - -use std::env; - -fn do_cc() { - let target = env::var("TARGET").unwrap(); - if cfg!(unix) && !target.contains("wasi") { - cc::Build::new().file("src/cmsg.c").compile("cmsg"); - } - if target.contains("android") || target.contains("linux") { - cc::Build::new().file("src/errqueue.c").compile("errqueue"); - } -} - -fn do_ctest() { - match &env::var("TARGET").unwrap() { - t if t.contains("android") => return test_android(t), - t if t.contains("apple") => return test_apple(t), - t if t.contains("cloudabi") => return test_cloudabi(t), - t if t.contains("dragonfly") => return test_dragonflybsd(t), - t if t.contains("emscripten") => return test_emscripten(t), - t if t.contains("freebsd") => return test_freebsd(t), - t if t.contains("linux") => return test_linux(t), - t if t.contains("netbsd") => return test_netbsd(t), - t if t.contains("openbsd") => return test_openbsd(t), - t if t.contains("redox") => return test_redox(t), - t if t.contains("solaris") => return test_solaris(t), - t if t.contains("wasi") => return test_wasi(t), - t if t.contains("windows") => return test_windows(t), - t if t.contains("vxworks") => return test_vxworks(t), - t => panic!("unknown target {}", t), - } -} - -fn ctest_cfg() -> ctest::TestGenerator { - let mut cfg = ctest::TestGenerator::new(); - let libc_cfgs = [ - "libc_priv_mod_use", - "libc_union", - "libc_const_size_of", - "libc_align", - "libc_core_cvoid", - "libc_packedN", - "libc_thread_local", - ]; - for f in &libc_cfgs { - cfg.cfg(f, None); - } - cfg -} - -fn main() { - do_cc(); - do_ctest(); -} - -macro_rules! headers { - ($cfg:ident: [$m:expr]: $header:literal) => { - if $m { - $cfg.header($header); - } - }; - ($cfg:ident: $header:literal) => { - $cfg.header($header); - }; - ($($cfg:ident: $([$c:expr]:)* $header:literal,)*) => { - $(headers!($cfg: $([$c]:)* $header);)* - }; - ($cfg:ident: $( $([$c:expr]:)* $header:literal,)*) => { - headers!($($cfg: $([$c]:)* $header,)*); - }; - ($cfg:ident: $( $([$c:expr]:)* $header:literal),*) => { - headers!($($cfg: $([$c]:)* $header,)*); - }; -} - -fn test_apple(target: &str) { - assert!(target.contains("apple")); - let x86_64 = target.contains("x86_64"); - let i686 = target.contains("i686"); - - let mut cfg = ctest_cfg(); - cfg.flag("-Wno-deprecated-declarations"); - cfg.define("__APPLE_USE_RFC_3542", None); - - headers! { cfg: - "aio.h", - "ctype.h", - "dirent.h", - "dlfcn.h", - "errno.h", - "execinfo.h", - "fcntl.h", - "glob.h", - "grp.h", - "ifaddrs.h", - "langinfo.h", - "limits.h", - "locale.h", - "mach-o/dyld.h", - "mach/mach_time.h", - "malloc/malloc.h", - "net/bpf.h", - "net/if.h", - "net/if_arp.h", - "net/if_dl.h", - "net/if_utun.h", - "net/route.h", - "net/route.h", - "netdb.h", - "netinet/if_ether.h", - "netinet/in.h", - "netinet/in.h", - "netinet/ip.h", - "netinet/tcp.h", - "netinet/udp.h", - "poll.h", - "pthread.h", - "pwd.h", - "resolv.h", - "sched.h", - "semaphore.h", - "signal.h", - "spawn.h", - "stddef.h", - "stdint.h", - "stdio.h", - "stdlib.h", - "string.h", - "sys/event.h", - "sys/file.h", - "sys/ioctl.h", - "sys/ipc.h", - "sys/kern_control.h", - "sys/mman.h", - "sys/mount.h", - "sys/proc_info.h", - "sys/ptrace.h", - "sys/quota.h", - "sys/resource.h", - "sys/sem.h", - "sys/shm.h", - "sys/socket.h", - "sys/stat.h", - "sys/statvfs.h", - "sys/sys_domain.h", - "sys/sysctl.h", - "sys/time.h", - "sys/times.h", - "sys/timex.h", - "sys/types.h", - "sys/uio.h", - "sys/un.h", - "sys/utsname.h", - "sys/wait.h", - "sys/xattr.h", - "syslog.h", - "termios.h", - "time.h", - "unistd.h", - "util.h", - "utime.h", - "utmpx.h", - "wchar.h", - "xlocale.h", - [x86_64]: "crt_externs.h", - } - - cfg.skip_struct(move |ty| { - match ty { - // FIXME: actually a union - "sigval" => true, - - _ => false, - } - }); - - cfg.skip_const(move |name| { - match name { - // These OSX constants are removed in Sierra. - // https://developer.apple.com/library/content/releasenotes/General/APIDiffsMacOS10_12/Swift/Darwin.html - "KERN_KDENABLE_BG_TRACE" | "KERN_KDDISABLE_BG_TRACE" => true, - // FIXME: the value has been changed since Catalina (0xffff0000 -> 0x3fff0000). - "SF_SETTABLE" => true, - // FIXME: the value has been changed since Catalina (VM_FLAGS_RESILIENT_MEDIA is also contained now). - "VM_FLAGS_USER_REMAP" => true, - _ => false, - } - }); - - cfg.skip_fn(move |name| { - // skip those that are manually verified - match name { - // FIXME: https://github.com/rust-lang/libc/issues/1272 - "execv" | "execve" | "execvp" => true, - - // close calls the close_nocancel system call - "close" => true, - - _ => false, - } - }); - - cfg.skip_field(move |struct_, field| { - match (struct_, field) { - // FIXME: the array size has been changed since macOS 10.15 ([8] -> [7]). - ("statfs", "f_reserved") => true, - _ => false, - } - }); - - cfg.skip_field_type(move |struct_, field| { - match (struct_, field) { - // FIXME: actually a union - ("sigevent", "sigev_value") => true, - _ => false, - } - }); - - cfg.volatile_item(|i| { - use ctest::VolatileItemKind::*; - match i { - StructField(ref n, ref f) if n == "aiocb" && f == "aio_buf" => { - true - } - _ => false, - } - }); - - cfg.type_name(move |ty, is_struct, is_union| { - match ty { - // Just pass all these through, no need for a "struct" prefix - "FILE" | "DIR" | "Dl_info" => ty.to_string(), - - // OSX calls this something else - "sighandler_t" => "sig_t".to_string(), - - t if is_union => format!("union {}", t), - t if t.ends_with("_t") => t.to_string(), - t if is_struct => format!("struct {}", t), - t => t.to_string(), - } - }); - - cfg.field_name(move |struct_, field| { - match field { - s if s.ends_with("_nsec") && struct_.starts_with("stat") => { - s.replace("e_nsec", "espec.tv_nsec") - } - // FIXME: sigaction actually contains a union with two variants: - // a sa_sigaction with type: (*)(int, struct __siginfo *, void *) - // a sa_handler with type sig_t - "sa_sigaction" if struct_ == "sigaction" => { - "sa_handler".to_string() - } - s => s.to_string(), - } - }); - - cfg.skip_roundtrip(move |s| match s { - // FIXME: this type has the wrong ABI - "max_align_t" if i686 => true, - _ => false, - }); - cfg.generate("../src/lib.rs", "main.rs"); -} - -fn test_openbsd(target: &str) { - assert!(target.contains("openbsd")); - - let mut cfg = ctest_cfg(); - cfg.flag("-Wno-deprecated-declarations"); - - headers! { cfg: - "errno.h", - "fcntl.h", - "limits.h", - "locale.h", - "stddef.h", - "stdint.h", - "stdio.h", - "stdlib.h", - "sys/stat.h", - "sys/types.h", - "time.h", - "wchar.h", - "ctype.h", - "dirent.h", - "sys/socket.h", - "net/if.h", - "net/route.h", - "net/if_arp.h", - "netdb.h", - "netinet/in.h", - "netinet/ip.h", - "netinet/tcp.h", - "netinet/udp.h", - "net/bpf.h", - "resolv.h", - "pthread.h", - "dlfcn.h", - "signal.h", - "string.h", - "sys/file.h", - "sys/ioctl.h", - "sys/mman.h", - "sys/resource.h", - "sys/socket.h", - "sys/time.h", - "sys/un.h", - "sys/wait.h", - "unistd.h", - "utime.h", - "pwd.h", - "grp.h", - "sys/utsname.h", - "sys/ptrace.h", - "sys/mount.h", - "sys/uio.h", - "sched.h", - "termios.h", - "poll.h", - "syslog.h", - "semaphore.h", - "sys/statvfs.h", - "sys/times.h", - "glob.h", - "ifaddrs.h", - "langinfo.h", - "sys/sysctl.h", - "utmp.h", - "sys/event.h", - "net/if_dl.h", - "util.h", - "ufs/ufs/quota.h", - "pthread_np.h", - "sys/syscall.h", - "sys/shm.h", - } - - cfg.skip_struct(move |ty| { - match ty { - // FIXME: actually a union - "sigval" => true, - - _ => false, - } - }); - - cfg.skip_const(move |name| { - match name { - // Removed in OpenBSD 6.0 - "KERN_USERMOUNT" | "KERN_ARND" => true, - _ => false, - } - }); - - cfg.skip_fn(move |name| { - match name { - // FIXME: https://github.com/rust-lang/libc/issues/1272 - "execv" | "execve" | "execvp" | "execvpe" => true, - - // Removed in OpenBSD 6.5 - // https://marc.info/?l=openbsd-cvs&m=154723400730318 - "mincore" => true, - - _ => false, - } - }); - - cfg.type_name(move |ty, is_struct, is_union| { - match ty { - // Just pass all these through, no need for a "struct" prefix - "FILE" | "DIR" | "Dl_info" => ty.to_string(), - - // OSX calls this something else - "sighandler_t" => "sig_t".to_string(), - - t if is_union => format!("union {}", t), - t if t.ends_with("_t") => t.to_string(), - t if is_struct => format!("struct {}", t), - t => t.to_string(), - } - }); - - cfg.field_name(move |struct_, field| match field { - "st_birthtime" if struct_.starts_with("stat") => { - "__st_birthtime".to_string() - } - "st_birthtime_nsec" if struct_.starts_with("stat") => { - "__st_birthtimensec".to_string() - } - s if s.ends_with("_nsec") && struct_.starts_with("stat") => { - s.replace("e_nsec", ".tv_nsec") - } - "sa_sigaction" if struct_ == "sigaction" => "sa_handler".to_string(), - s => s.to_string(), - }); - - cfg.skip_field_type(move |struct_, field| { - // type siginfo_t.si_addr changed from OpenBSD 6.0 to 6.1 - struct_ == "siginfo_t" && field == "si_addr" - }); - - cfg.generate("../src/lib.rs", "main.rs"); -} - -fn test_windows(target: &str) { - assert!(target.contains("windows")); - let gnu = target.contains("gnu"); - - let mut cfg = ctest_cfg(); - cfg.define("_WIN32_WINNT", Some("0x8000")); - - headers! { cfg: - "direct.h", - "errno.h", - "fcntl.h", - "io.h", - "limits.h", - "locale.h", - "process.h", - "signal.h", - "stddef.h", - "stdint.h", - "stdio.h", - "stdlib.h", - "sys/stat.h", - "sys/types.h", - "sys/utime.h", - "time.h", - "wchar.h", - [gnu]: "ws2tcpip.h", - [!gnu]: "Winsock2.h", - } - - cfg.type_name(move |ty, is_struct, is_union| { - match ty { - // Just pass all these through, no need for a "struct" prefix - "FILE" | "DIR" | "Dl_info" => ty.to_string(), - - // FIXME: these don't exist: - "time64_t" => "__time64_t".to_string(), - "ssize_t" => "SSIZE_T".to_string(), - - "sighandler_t" if !gnu => "_crt_signal_t".to_string(), - "sighandler_t" if gnu => "__p_sig_fn_t".to_string(), - - t if is_union => format!("union {}", t), - t if t.ends_with("_t") => t.to_string(), - - // Windows uppercase structs don't have `struct` in front: - t if is_struct => { - if ty.clone().chars().next().unwrap().is_uppercase() { - t.to_string() - } else if t == "stat" { - "struct __stat64".to_string() - } else if t == "utimbuf" { - "struct __utimbuf64".to_string() - } else { - // put `struct` in front of all structs: - format!("struct {}", t) - } - } - t => t.to_string(), - } - }); - - cfg.fn_cname(move |name, cname| cname.unwrap_or(name).to_string()); - - cfg.skip_type(move |name| match name { - "SSIZE_T" if !gnu => true, - "ssize_t" if !gnu => true, - _ => false, - }); - - cfg.skip_const(move |name| { - match name { - // FIXME: API error: - // SIG_ERR type is "void (*)(int)", not "int" - "SIG_ERR" => true, - _ => false, - } - }); - - // FIXME: All functions point to the wrong addresses? - cfg.skip_fn_ptrcheck(|_| true); - - cfg.skip_signededness(move |c| { - match c { - // windows-isms - n if n.starts_with("P") => true, - n if n.starts_with("H") => true, - n if n.starts_with("LP") => true, - "sighandler_t" if gnu => true, - _ => false, - } - }); - - cfg.skip_fn(move |name| { - match name { - // FIXME: https://github.com/rust-lang/libc/issues/1272 - "execv" | "execve" | "execvp" | "execvpe" => true, - - _ => false, - } - }); - - cfg.generate("../src/lib.rs", "main.rs"); -} - -fn test_redox(target: &str) { - assert!(target.contains("redox")); - - let mut cfg = ctest_cfg(); - cfg.flag("-Wno-deprecated-declarations"); - - headers! { - cfg: - "ctype.h", - "dirent.h", - "dlfcn.h", - "errno.h", - "execinfo.h", - "fcntl.h", - "glob.h", - "grp.h", - "ifaddrs.h", - "langinfo.h", - "limits.h", - "locale.h", - "net/if.h", - "net/if_arp.h", - "net/route.h", - "netdb.h", - "netinet/in.h", - "netinet/ip.h", - "netinet/tcp.h", - "netinet/udp.h", - "poll.h", - "pthread.h", - "pwd.h", - "resolv.h", - "sched.h", - "semaphore.h", - "string.h", - "strings.h", - "sys/file.h", - "sys/ioctl.h", - "sys/mman.h", - "sys/mount.h", - "sys/ptrace.h", - "sys/quota.h", - "sys/resource.h", - "sys/socket.h", - "sys/stat.h", - "sys/statvfs.h", - "sys/sysctl.h", - "sys/time.h", - "sys/times.h", - "sys/types.h", - "sys/uio.h", - "sys/un.h", - "sys/utsname.h", - "sys/wait.h", - "syslog.h", - "termios.h", - "time.h", - "unistd.h", - "utime.h", - "utmpx.h", - "wchar.h", - } - - cfg.generate("../src/lib.rs", "main.rs"); -} - -fn test_cloudabi(target: &str) { - assert!(target.contains("cloudabi")); - - let mut cfg = ctest_cfg(); - cfg.flag("-Wno-deprecated-declarations"); - - headers! { - cfg: - "execinfo.h", - "glob.h", - "ifaddrs.h", - "langinfo.h", - "sys/ptrace.h", - "sys/quota.h", - "sys/sysctl.h", - "utmpx.h", - "ctype.h", - "dirent.h", - "dlfcn.h", - "errno.h", - "fcntl.h", - "grp.h", - "limits.h", - "locale.h", - "net/if.h", - "net/if_arp.h", - "net/route.h", - "netdb.h", - "netinet/in.h", - "netinet/ip.h", - "netinet/tcp.h", - "netinet/udp.h", - "poll.h", - "pthread.h", - "pwd.h", - "resolv.h", - "sched.h", - "semaphore.h", - "signal.h", - "stddef.h", - "stdint.h", - "stdio.h", - "stdlib.h", - "string.h", - "strings.h", - "sys/file.h", - "sys/ioctl.h", - "sys/mman.h", - "sys/mount.h", - "sys/resource.h", - "sys/socket.h", - "sys/stat.h", - "sys/statvfs.h", - "sys/time.h", - "sys/times.h", - "sys/types.h", - "sys/uio.h", - "sys/un.h", - "sys/utsname.h", - "sys/wait.h", - "syslog.h", - "termios.h", - "time.h", - "unistd.h", - "utime.h", - "wchar.h", - } - - cfg.generate("../src/lib.rs", "main.rs"); -} - -fn test_solaris(target: &str) { - assert!(target.contains("solaris")); - - let mut cfg = ctest_cfg(); - cfg.flag("-Wno-deprecated-declarations"); - - cfg.define("_XOPEN_SOURCE", Some("700")); - cfg.define("__EXTENSIONS__", None); - cfg.define("_LCONV_C99", None); - - headers! { - cfg: - "ctype.h", - "dirent.h", - "dlfcn.h", - "errno.h", - "execinfo.h", - "fcntl.h", - "glob.h", - "grp.h", - "ifaddrs.h", - "langinfo.h", - "limits.h", - "locale.h", - "net/if.h", - "net/if_arp.h", - "net/route.h", - "netdb.h", - "netinet/in.h", - "netinet/ip.h", - "netinet/tcp.h", - "netinet/udp.h", - "poll.h", - "port.h", - "pthread.h", - "pwd.h", - "resolv.h", - "sched.h", - "semaphore.h", - "signal.h", - "stddef.h", - "stdint.h", - "stdio.h", - "stdlib.h", - "string.h", - "sys/epoll.h", - "sys/file.h", - "sys/filio.h", - "sys/ioctl.h", - "sys/loadavg.h", - "sys/mman.h", - "sys/mount.h", - "sys/resource.h", - "sys/socket.h", - "sys/stat.h", - "sys/statvfs.h", - "sys/time.h", - "sys/times.h", - "sys/timex.h", - "sys/types.h", - "sys/uio.h", - "sys/un.h", - "sys/utsname.h", - "sys/wait.h", - "syslog.h", - "termios.h", - "time.h", - "ucontext.h", - "unistd.h", - "utime.h", - "utmpx.h", - "wchar.h", - } - - cfg.skip_const(move |name| match name { - "DT_FIFO" | "DT_CHR" | "DT_DIR" | "DT_BLK" | "DT_REG" | "DT_LNK" - | "DT_SOCK" | "USRQUOTA" | "GRPQUOTA" | "PRIO_MIN" | "PRIO_MAX" => { - true - } - - _ => false, - }); - - cfg.skip_fn(move |name| { - // skip those that are manually verified - match name { - // const-ness only added recently - "dladdr" => true, - - // Definition of those functions as changed since unified headers - // from NDK r14b These changes imply some API breaking changes but - // are still ABI compatible. We can wait for the next major release - // to be compliant with the new API. - // - // FIXME: unskip these for next major release - "setpriority" | "personality" => true, - - // signal is defined with sighandler_t, so ignore - "signal" => true, - - "cfmakeraw" | "cfsetspeed" => true, - - // FIXME: mincore is defined with caddr_t on Solaris. - "mincore" => true, - - _ => false, - } - }); - - cfg.generate("../src/lib.rs", "main.rs"); -} - -fn test_netbsd(target: &str) { - assert!(target.contains("netbsd")); - let rumprun = target.contains("rumprun"); - let mut cfg = ctest_cfg(); - - cfg.flag("-Wno-deprecated-declarations"); - cfg.define("_NETBSD_SOURCE", Some("1")); - - headers! { - cfg: - "errno.h", - "fcntl.h", - "limits.h", - "locale.h", - "stddef.h", - "stdint.h", - "stdio.h", - "stdlib.h", - "sys/stat.h", - "sys/types.h", - "time.h", - "wchar.h", - "aio.h", - "ctype.h", - "dirent.h", - "dlfcn.h", - "glob.h", - "grp.h", - "ifaddrs.h", - "langinfo.h", - "net/if.h", - "net/if_arp.h", - "net/if_dl.h", - "net/route.h", - "netdb.h", - "netinet/in.h", - "netinet/ip.h", - "netinet/tcp.h", - "netinet/udp.h", - "poll.h", - "pthread.h", - "pwd.h", - "resolv.h", - "sched.h", - "semaphore.h", - "signal.h", - "string.h", - "sys/extattr.h", - "sys/file.h", - "sys/ioctl.h", - "sys/ioctl_compat.h", - "sys/mman.h", - "sys/mount.h", - "sys/ptrace.h", - "sys/resource.h", - "sys/socket.h", - "sys/statvfs.h", - "sys/sysctl.h", - "sys/time.h", - "sys/times.h", - "sys/timex.h", - "sys/uio.h", - "sys/un.h", - "sys/utsname.h", - "sys/wait.h", - "syslog.h", - "termios.h", - "ufs/ufs/quota.h", - "ufs/ufs/quota1.h", - "unistd.h", - "util.h", - "utime.h", - "mqueue.h", - "netinet/dccp.h", - "sys/event.h", - "sys/quota.h", - "sys/shm.h", - } - - cfg.type_name(move |ty, is_struct, is_union| { - match ty { - // Just pass all these through, no need for a "struct" prefix - "FILE" | "fd_set" | "Dl_info" | "DIR" | "Elf32_Phdr" - | "Elf64_Phdr" | "Elf32_Shdr" | "Elf64_Shdr" | "Elf32_Sym" - | "Elf64_Sym" | "Elf32_Ehdr" | "Elf64_Ehdr" | "Elf32_Chdr" - | "Elf64_Chdr" => ty.to_string(), - - // OSX calls this something else - "sighandler_t" => "sig_t".to_string(), - - t if is_union => format!("union {}", t), - - t if t.ends_with("_t") => t.to_string(), - - // put `struct` in front of all structs:. - t if is_struct => format!("struct {}", t), - - t => t.to_string(), - } - }); - - cfg.field_name(move |struct_, field| { - match field { - // Our stat *_nsec fields normally don't actually exist but are part - // of a timeval struct - s if s.ends_with("_nsec") && struct_.starts_with("stat") => { - s.replace("e_nsec", ".tv_nsec") - } - "u64" if struct_ == "epoll_event" => "data.u64".to_string(), - s => s.to_string(), - } - }); - - cfg.skip_type(move |ty| { - match ty { - // FIXME: sighandler_t is crazy across platforms - "sighandler_t" => true, - _ => false, - } - }); - - cfg.skip_struct(move |ty| { - match ty { - // This is actually a union, not a struct - "sigval" => true, - // These are tested as part of the linux_fcntl tests since there are - // header conflicts when including them with all the other structs. - "termios2" => true, - _ => false, - } - }); - - cfg.skip_signededness(move |c| { - match c { - "LARGE_INTEGER" | "float" | "double" => true, - n if n.starts_with("pthread") => true, - // sem_t is a struct or pointer - "sem_t" => true, - _ => false, - } - }); - - cfg.skip_const(move |name| { - match name { - "SIG_DFL" | "SIG_ERR" | "SIG_IGN" => true, // sighandler_t weirdness - "SIGUNUSED" => true, // removed in glibc 2.26 - - // weird signed extension or something like that? - "MS_NOUSER" => true, - "MS_RMT_MASK" => true, // updated in glibc 2.22 and musl 1.1.13 - "BOTHER" => true, - - _ => false, - } - }); - - cfg.skip_fn(move |name| { - match name { - // FIXME: https://github.com/rust-lang/libc/issues/1272 - "execv" | "execve" | "execvp" => true, - - "getrlimit" | "getrlimit64" | // non-int in 1st arg - "setrlimit" | "setrlimit64" | // non-int in 1st arg - "prlimit" | "prlimit64" | // non-int in 2nd arg - - // These functions presumably exist on netbsd but don't look like - // they're implemented on rumprun yet, just let them slide for now. - // Some of them look like they have headers but then don't have - // corresponding actual definitions either... - "shm_open" | - "shm_unlink" | - "syscall" | - "mq_open" | - "mq_close" | - "mq_getattr" | - "mq_notify" | - "mq_receive" | - "mq_send" | - "mq_setattr" | - "mq_timedreceive" | - "mq_timedsend" | - "mq_unlink" | - "ptrace" | - "sigaltstack" if rumprun => true, - - _ => false, - } - }); - - cfg.skip_field_type(move |struct_, field| { - // This is a weird union, don't check the type. - (struct_ == "ifaddrs" && field == "ifa_ifu") || - // sighandler_t type is super weird - (struct_ == "sigaction" && field == "sa_sigaction") || - // sigval is actually a union, but we pretend it's a struct - (struct_ == "sigevent" && field == "sigev_value") || - // aio_buf is "volatile void*" and Rust doesn't understand volatile - (struct_ == "aiocb" && field == "aio_buf") - }); - - cfg.generate("../src/lib.rs", "main.rs"); -} - -fn test_dragonflybsd(target: &str) { - assert!(target.contains("dragonfly")); - let mut cfg = ctest_cfg(); - cfg.flag("-Wno-deprecated-declarations"); - - headers! { - cfg: - "aio.h", - "ctype.h", - "dirent.h", - "dlfcn.h", - "errno.h", - "execinfo.h", - "fcntl.h", - "glob.h", - "grp.h", - "ifaddrs.h", - "langinfo.h", - "limits.h", - "locale.h", - "mqueue.h", - "net/if.h", - "net/if_arp.h", - "net/if_dl.h", - "net/route.h", - "netdb.h", - "netinet/in.h", - "netinet/ip.h", - "netinet/tcp.h", - "netinet/udp.h", - "poll.h", - "pthread.h", - "pthread_np.h", - "pwd.h", - "resolv.h", - "sched.h", - "semaphore.h", - "signal.h", - "stddef.h", - "stdint.h", - "stdio.h", - "stdlib.h", - "string.h", - "sys/event.h", - "sys/file.h", - "sys/ioctl.h", - "sys/mman.h", - "sys/mount.h", - "sys/ptrace.h", - "sys/resource.h", - "sys/rtprio.h", - "sys/socket.h", - "sys/stat.h", - "sys/statvfs.h", - "sys/sysctl.h", - "sys/time.h", - "sys/times.h", - "sys/types.h", - "sys/uio.h", - "sys/un.h", - "sys/utsname.h", - "sys/wait.h", - "syslog.h", - "termios.h", - "time.h", - "ufs/ufs/quota.h", - "unistd.h", - "util.h", - "utime.h", - "utmpx.h", - "wchar.h", - } - - cfg.type_name(move |ty, is_struct, is_union| { - match ty { - // Just pass all these through, no need for a "struct" prefix - "FILE" | "fd_set" | "Dl_info" | "DIR" | "Elf32_Phdr" - | "Elf64_Phdr" | "Elf32_Shdr" | "Elf64_Shdr" | "Elf32_Sym" - | "Elf64_Sym" | "Elf32_Ehdr" | "Elf64_Ehdr" | "Elf32_Chdr" - | "Elf64_Chdr" => ty.to_string(), - - // FIXME: OSX calls this something else - "sighandler_t" => "sig_t".to_string(), - - t if is_union => format!("union {}", t), - - t if t.ends_with("_t") => t.to_string(), - - // put `struct` in front of all structs:. - t if is_struct => format!("struct {}", t), - - t => t.to_string(), - } - }); - - cfg.field_name(move |struct_, field| { - match field { - // Our stat *_nsec fields normally don't actually exist but are part - // of a timeval struct - s if s.ends_with("_nsec") && struct_.starts_with("stat") => { - s.replace("e_nsec", ".tv_nsec") - } - "u64" if struct_ == "epoll_event" => "data.u64".to_string(), - // Field is named `type` in C but that is a Rust keyword, - // so these fields are translated to `type_` in the bindings. - "type_" if struct_ == "rtprio" => "type".to_string(), - s => s.to_string(), - } - }); - - cfg.skip_type(move |ty| { - match ty { - // sighandler_t is crazy across platforms - "sighandler_t" => true, - - _ => false, - } - }); - - cfg.skip_struct(move |ty| { - match ty { - // This is actually a union, not a struct - "sigval" => true, - - // FIXME: These are tested as part of the linux_fcntl tests since - // there are header conflicts when including them with all the other - // structs. - "termios2" => true, - - _ => false, - } - }); - - cfg.skip_signededness(move |c| { - match c { - "LARGE_INTEGER" | "float" | "double" => true, - // uuid_t is a struct, not an integer. - "uuid_t" => true, - n if n.starts_with("pthread") => true, - // sem_t is a struct or pointer - "sem_t" => true, - // mqd_t is a pointer on DragonFly - "mqd_t" => true, - - _ => false, - } - }); - - cfg.skip_const(move |name| { - match name { - "SIG_DFL" | "SIG_ERR" | "SIG_IGN" => true, // sighandler_t weirdness - - // weird signed extension or something like that? - "MS_NOUSER" => true, - "MS_RMT_MASK" => true, // updated in glibc 2.22 and musl 1.1.13 - - // These are defined for Solaris 11, but the crate is tested on - // illumos, where they are currently not defined - "EADI" - | "PORT_SOURCE_POSTWAIT" - | "PORT_SOURCE_SIGNAL" - | "PTHREAD_STACK_MIN" => true, - - _ => false, - } - }); - - cfg.skip_fn(move |name| { - // skip those that are manually verified - match name { - // FIXME: https://github.com/rust-lang/libc/issues/1272 - "execv" | "execve" | "execvp" => true, - - "getrlimit" | "getrlimit64" | // non-int in 1st arg - "setrlimit" | "setrlimit64" | // non-int in 1st arg - "prlimit" | "prlimit64" // non-int in 2nd arg - => true, - - _ => false, - } - }); - - cfg.skip_field_type(move |struct_, field| { - // This is a weird union, don't check the type. - (struct_ == "ifaddrs" && field == "ifa_ifu") || - // sighandler_t type is super weird - (struct_ == "sigaction" && field == "sa_sigaction") || - // sigval is actually a union, but we pretend it's a struct - (struct_ == "sigevent" && field == "sigev_value") || - // aio_buf is "volatile void*" and Rust doesn't understand volatile - (struct_ == "aiocb" && field == "aio_buf") - }); - - cfg.skip_field(move |struct_, field| { - // this is actually a union on linux, so we can't represent it well and - // just insert some padding. - (struct_ == "siginfo_t" && field == "_pad") || - // sigev_notify_thread_id is actually part of a sigev_un union - (struct_ == "sigevent" && field == "sigev_notify_thread_id") - }); - - cfg.generate("../src/lib.rs", "main.rs"); -} - -fn test_wasi(target: &str) { - assert!(target.contains("wasi")); - - let mut cfg = ctest_cfg(); - cfg.define("_GNU_SOURCE", None); - - headers! { cfg: - "ctype.h", - "dirent.h", - "errno.h", - "fcntl.h", - "limits.h", - "locale.h", - "malloc.h", - "poll.h", - "sched.h", - "stdbool.h", - "stddef.h", - "stdint.h", - "stdio.h", - "stdlib.h", - "string.h", - "sys/resource.h", - "sys/select.h", - "sys/socket.h", - "sys/stat.h", - "sys/times.h", - "sys/types.h", - "sys/uio.h", - "sys/utsname.h", - "time.h", - "unistd.h", - "wasi/api.h", - "wasi/libc.h", - "wasi/libc-find-relpath.h", - "wchar.h", - } - - cfg.type_name(move |ty, is_struct, is_union| match ty { - "FILE" | "fd_set" | "DIR" => ty.to_string(), - t if is_union => format!("union {}", t), - t if t.starts_with("__wasi") && t.ends_with("_u") => { - format!("union {}", t) - } - t if t.starts_with("__wasi") && is_struct => format!("struct {}", t), - t if t.ends_with("_t") => t.to_string(), - t if is_struct => format!("struct {}", t), - t => t.to_string(), - }); - - cfg.field_name(move |_struct, field| { - match field { - // deal with fields as rust keywords - "type_" => "type".to_string(), - s => s.to_string(), - } - }); - - // Looks like LLD doesn't merge duplicate imports, so if the Rust - // code imports from a module and the C code also imports from a - // module we end up with two imports of function pointers which - // import the same thing but have different function pointers - cfg.skip_fn_ptrcheck(|f| f.starts_with("__wasi")); - - // d_name is declared as a flexible array in WASI libc, so it - // doesn't support sizeof. - cfg.skip_field(|s, field| s == "dirent" && field == "d_name"); - - // Currently Rust/clang disagree on function argument ABI, so skip these - // tests. For more info see WebAssembly/tool-conventions#88 - cfg.skip_roundtrip(|_| true); - - cfg.generate("../src/lib.rs", "main.rs"); -} - -fn test_android(target: &str) { - assert!(target.contains("android")); - let target_pointer_width = match target { - t if t.contains("aarch64") || t.contains("x86_64") => 64, - t if t.contains("i686") || t.contains("arm") => 32, - t => panic!("unsupported target: {}", t), - }; - let x86 = target.contains("i686") || target.contains("x86_64"); - - let mut cfg = ctest_cfg(); - cfg.define("_GNU_SOURCE", None); - - headers! { cfg: - "arpa/inet.h", - "ctype.h", - "dirent.h", - "dlfcn.h", - "errno.h", - "fcntl.h", - "grp.h", - "ifaddrs.h", - "limits.h", - "locale.h", - "malloc.h", - "net/ethernet.h", - "net/if.h", - "net/if_arp.h", - "net/route.h", - "netdb.h", - "netinet/in.h", - "netinet/ip.h", - "netinet/tcp.h", - "netinet/udp.h", - "netpacket/packet.h", - "poll.h", - "pthread.h", - "pty.h", - "pwd.h", - "resolv.h", - "sched.h", - "semaphore.h", - "signal.h", - "stddef.h", - "stdint.h", - "stdio.h", - "stdlib.h", - "string.h", - "sys/epoll.h", - "sys/eventfd.h", - "sys/file.h", - "sys/fsuid.h", - "sys/inotify.h", - "sys/ioctl.h", - "sys/mman.h", - "sys/mount.h", - "sys/personality.h", - "sys/prctl.h", - "sys/ptrace.h", - "sys/random.h", - "sys/reboot.h", - "sys/resource.h", - "sys/sendfile.h", - "sys/signalfd.h", - "sys/socket.h", - "sys/stat.h", - "sys/statvfs.h", - "sys/swap.h", - "sys/syscall.h", - "sys/sysinfo.h", - "sys/time.h", - "sys/times.h", - "sys/types.h", - "sys/ucontext.h", - "sys/uio.h", - "sys/un.h", - "sys/utsname.h", - "sys/vfs.h", - "sys/xattr.h", - "sys/wait.h", - "syslog.h", - "termios.h", - "time.h", - "unistd.h", - "utime.h", - "utmp.h", - "wchar.h", - "xlocale.h", - // time64_t is not defined for 64-bit targets If included it will - // generate the error 'Your time_t is already 64-bit' - [target_pointer_width == 32]: "time64.h", - [x86]: "sys/reg.h", - } - - // Include linux headers at the end: - headers! { cfg: - "asm/mman.h", - "linux/dccp.h", - "linux/errqueue.h", - "linux/futex.h", - "linux/fs.h", - "linux/genetlink.h", - "linux/if_alg.h", - "linux/if_ether.h", - "linux/if_tun.h", - "linux/magic.h", - "linux/memfd.h", - "linux/module.h", - "linux/net_tstamp.h", - "linux/netfilter/nfnetlink.h", - "linux/netfilter/nfnetlink_log.h", - "linux/netfilter/nfnetlink_queue.h", - "linux/netfilter/nf_tables.h", - "linux/netfilter_ipv4.h", - "linux/netfilter_ipv6.h", - "linux/netfilter_ipv6/ip6_tables.h", - "linux/netlink.h", - "linux/quota.h", - "linux/reboot.h", - "linux/seccomp.h", - "linux/sockios.h", - - } - - cfg.type_name(move |ty, is_struct, is_union| { - match ty { - // Just pass all these through, no need for a "struct" prefix - "FILE" | "fd_set" | "Dl_info" => ty.to_string(), - - t if is_union => format!("union {}", t), - - t if t.ends_with("_t") => t.to_string(), - - // sigval is a struct in Rust, but a union in C: - "sigval" => format!("union sigval"), - - // put `struct` in front of all structs:. - t if is_struct => format!("struct {}", t), - - t => t.to_string(), - } - }); - - cfg.field_name(move |struct_, field| { - match field { - // Our stat *_nsec fields normally don't actually exist but are part - // of a timeval struct - s if s.ends_with("_nsec") && struct_.starts_with("stat") => { - s.to_string() - } - // FIXME: appears that `epoll_event.data` is an union - "u64" if struct_ == "epoll_event" => "data.u64".to_string(), - s => s.to_string(), - } - }); - - cfg.skip_type(move |ty| { - match ty { - // FIXME: `sighandler_t` type is incorrect, see: - // https://github.com/rust-lang/libc/issues/1359 - "sighandler_t" => true, - _ => false, - } - }); - - cfg.skip_struct(move |ty| { - if ty.starts_with("__c_anonymous_") { - return true; - } - match ty { - // These are tested as part of the linux_fcntl tests since there are - // header conflicts when including them with all the other structs. - "termios2" => true, - // uc_sigmask and uc_sigmask64 of ucontext_t are an anonymous union - "ucontext_t" => true, - - _ => false, - } - }); - - cfg.skip_const(move |name| { - match name { - // FIXME: deprecated: not available in any header - // See: https://github.com/rust-lang/libc/issues/1356 - "ENOATTR" => true, - - // FIXME: still necessary? - "SIG_DFL" | "SIG_ERR" | "SIG_IGN" => true, // sighandler_t weirdness - // FIXME: deprecated - removed in glibc 2.26 - "SIGUNUSED" => true, - - _ => false, - } - }); - - cfg.skip_fn(move |name| { - // skip those that are manually verified - match name { - // FIXME: https://github.com/rust-lang/libc/issues/1272 - "execv" | "execve" | "execvp" | "execvpe" | "fexecve" => true, - - // There are two versions of the sterror_r function, see - // - // https://linux.die.net/man/3/strerror_r - // - // An XSI-compliant version provided if: - // - // (_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && ! _GNU_SOURCE - // - // and a GNU specific version provided if _GNU_SOURCE is defined. - // - // libc provides bindings for the XSI-compliant version, which is - // preferred for portable applications. - // - // We skip the test here since here _GNU_SOURCE is defined, and - // test the XSI version below. - "strerror_r" => true, - - _ => false, - } - }); - - cfg.skip_field_type(move |struct_, field| { - // This is a weird union, don't check the type. - (struct_ == "ifaddrs" && field == "ifa_ifu") || - // sigval is actually a union, but we pretend it's a struct - (struct_ == "sigevent" && field == "sigev_value") - }); - - cfg.skip_field(move |struct_, field| { - // this is actually a union on linux, so we can't represent it well and - // just insert some padding. - (struct_ == "siginfo_t" && field == "_pad") || - // FIXME: `sa_sigaction` has type `sighandler_t` but that type is - // incorrect, see: https://github.com/rust-lang/libc/issues/1359 - (struct_ == "sigaction" && field == "sa_sigaction") || - // sigev_notify_thread_id is actually part of a sigev_un union - (struct_ == "sigevent" && field == "sigev_notify_thread_id") || - // signalfd had SIGSYS fields added in Android 4.19, but CI does not have that version yet. - (struct_ == "signalfd_siginfo" && (field == "ssi_syscall" || - field == "ssi_call_addr" || - field == "ssi_arch")) - }); - - cfg.generate("../src/lib.rs", "main.rs"); - - test_linux_like_apis(target); -} - -fn test_freebsd(target: &str) { - assert!(target.contains("freebsd")); - let mut cfg = ctest_cfg(); - - let freebsd_ver = which_freebsd(); - - match freebsd_ver { - Some(10) => cfg.cfg("freebsd10", None), - Some(11) => cfg.cfg("freebsd11", None), - Some(12) => cfg.cfg("freebsd12", None), - Some(13) => cfg.cfg("freebsd13", None), - _ => &mut cfg, - }; - - // Required for `getline`: - cfg.define("_WITH_GETLINE", None); - // Required for making freebsd11_stat available in the headers - match freebsd_ver { - Some(10) => &mut cfg, - _ => cfg.define("_WANT_FREEBSD11_STAT", None), - }; - - headers! { cfg: - "aio.h", - "arpa/inet.h", - "ctype.h", - "dirent.h", - "dlfcn.h", - "errno.h", - "fcntl.h", - "glob.h", - "grp.h", - "ifaddrs.h", - "langinfo.h", - "libutil.h", - "limits.h", - "locale.h", - "mqueue.h", - "net/bpf.h", - "net/if.h", - "net/if_arp.h", - "net/if_dl.h", - "net/route.h", - "netdb.h", - "netinet/ip.h", - "netinet/in.h", - "netinet/tcp.h", - "netinet/udp.h", - "poll.h", - "pthread.h", - "pthread_np.h", - "pwd.h", - "resolv.h", - "sched.h", - "semaphore.h", - "signal.h", - "spawn.h", - "stddef.h", - "stdint.h", - "stdio.h", - "stdlib.h", - "string.h", - "sys/event.h", - "sys/extattr.h", - "sys/file.h", - "sys/ioctl.h", - "sys/ipc.h", - "sys/jail.h", - "sys/mman.h", - "sys/mount.h", - "sys/msg.h", - "sys/procdesc.h", - "sys/ptrace.h", - "sys/resource.h", - "sys/rtprio.h", - "sys/shm.h", - "sys/socket.h", - "sys/stat.h", - "sys/statvfs.h", - "sys/sysctl.h", - "sys/time.h", - "sys/times.h", - "sys/timex.h", - "sys/types.h", - "sys/uio.h", - "sys/un.h", - "sys/utsname.h", - "sys/wait.h", - "syslog.h", - "termios.h", - "time.h", - "ufs/ufs/quota.h", - "unistd.h", - "utime.h", - "utmpx.h", - "wchar.h", - } - - cfg.type_name(move |ty, is_struct, is_union| { - match ty { - // Just pass all these through, no need for a "struct" prefix - "FILE" | "fd_set" | "Dl_info" | "DIR" => ty.to_string(), - - // FIXME: https://github.com/rust-lang/libc/issues/1273 - "sighandler_t" => "sig_t".to_string(), - - t if is_union => format!("union {}", t), - - t if t.ends_with("_t") => t.to_string(), - - // sigval is a struct in Rust, but a union in C: - "sigval" => format!("union sigval"), - - // put `struct` in front of all structs:. - t if is_struct => format!("struct {}", t), - - t => t.to_string(), - } - }); - - cfg.field_name(move |struct_, field| { - match field { - // Our stat *_nsec fields normally don't actually exist but are part - // of a timeval struct - s if s.ends_with("_nsec") && struct_.starts_with("stat") => { - s.replace("e_nsec", ".tv_nsec") - } - // Field is named `type` in C but that is a Rust keyword, - // so these fields are translated to `type_` in the bindings. - "type_" if struct_ == "rtprio" => "type".to_string(), - s => s.to_string(), - } - }); - - cfg.skip_const(move |name| { - match name { - // These constants were introduced in FreeBSD 12: - "SF_USER_READAHEAD" - | "EVFILT_EMPTY" - | "SO_REUSEPORT_LB" - | "IP_ORIGDSTADDR" - | "IP_RECVORIGDSTADDR" - | "IPV6_ORIGDSTADDR" - | "IPV6_RECVORIGDSTADDR" - if Some(11) == freebsd_ver => - { - true - } - - // These constants were introduced in FreeBSD 11: - "SF_USER_READAHEAD" - | "SF_NOCACHE" - | "RLIMIT_KQUEUES" - | "RLIMIT_UMTXP" - | "EVFILT_PROCDESC" - | "EVFILT_SENDFILE" - | "EVFILT_EMPTY" - | "SO_REUSEPORT_LB" - | "TCP_CCALGOOPT" - | "TCP_PCAP_OUT" - | "TCP_PCAP_IN" - | "IP_BINDMULTI" - | "IP_ORIGDSTADDR" - | "IP_RECVORIGDSTADDR" - | "IPV6_ORIGDSTADDR" - | "IPV6_RECVORIGDSTADDR" - | "PD_CLOEXEC" - | "PD_ALLOWED_AT_FORK" - | "IP_RSS_LISTEN_BUCKET" - if Some(10) == freebsd_ver => - { - true - } - - // FIXME: This constant has a different value in FreeBSD 10: - "RLIM_NLIMITS" if Some(10) == freebsd_ver => true, - - // FIXME: There are deprecated - remove in a couple of releases. - // These constants were removed in FreeBSD 11 (svn r273250) but will - // still be accepted and ignored at runtime. - "MAP_RENAME" | "MAP_NORESERVE" if Some(10) != freebsd_ver => true, - - // FIXME: There are deprecated - remove in a couple of releases. - // These constants were removed in FreeBSD 11 (svn r262489), - // and they've never had any legitimate use outside of the - // base system anyway. - "CTL_MAXID" | "KERN_MAXID" | "HW_MAXID" | "USER_MAXID" => true, - - _ => false, - } - }); - - cfg.skip_struct(move |ty| { - match ty { - // `mmsghdr` is not available in FreeBSD 10 - "mmsghdr" if Some(10) == freebsd_ver => true, - - // `max_align_t` is not available in FreeBSD 10 - "max_align_t" if Some(10) == freebsd_ver => true, - - _ => false, - } - }); - - cfg.skip_fn(move |name| { - // skip those that are manually verified - match name { - // FIXME: https://github.com/rust-lang/libc/issues/1272 - "execv" | "execve" | "execvp" | "execvpe" | "fexecve" => true, - - // These functions were added in FreeBSD 11: - "fdatasync" | "mq_getfd_np" | "sendmmsg" | "recvmmsg" - if Some(10) == freebsd_ver => - { - true - } - - // This function changed its return type from `int` in FreeBSD10 to - // `ssize_t` in FreeBSD11: - "aio_waitcomplete" if Some(10) == freebsd_ver => true, - - // The `uname` function in the `utsname.h` FreeBSD header is a C - // inline function (has no symbol) that calls the `__xuname` symbol. - // Therefore the function pointer comparison does not make sense for it. - "uname" => true, - - // FIXME: Our API is unsound. The Rust API allows aliasing - // pointers, but the C API requires pointers not to alias. - // We should probably be at least using `&`/`&mut` here, see: - // https://github.com/gnzlbg/ctest/issues/68 - "lio_listio" => true, - - _ => false, - } - }); - - cfg.skip_signededness(move |c| { - match c { - // FIXME: has a different sign in FreeBSD10 - "blksize_t" if Some(10) == freebsd_ver => true, - _ => false, - } - }); - - cfg.volatile_item(|i| { - use ctest::VolatileItemKind::*; - match i { - // aio_buf is a volatile void** but since we cannot express that in - // Rust types, we have to explicitly tell the checker about it here: - StructField(ref n, ref f) if n == "aiocb" && f == "aio_buf" => { - true - } - _ => false, - } - }); - - cfg.skip_field(move |struct_, field| { - match (struct_, field) { - // FIXME: `sa_sigaction` has type `sighandler_t` but that type is - // incorrect, see: https://github.com/rust-lang/libc/issues/1359 - ("sigaction", "sa_sigaction") => true, - - // FIXME: in FreeBSD10 this field has type `char*` instead of - // `void*`: - ("stack_t", "ss_sp") if Some(10) == freebsd_ver => true, - - _ => false, - } - }); - - cfg.generate("../src/lib.rs", "main.rs"); -} - -fn test_emscripten(target: &str) { - assert!(target.contains("emscripten")); - - let mut cfg = ctest_cfg(); - cfg.define("_GNU_SOURCE", None); // FIXME: ?? - - headers! { cfg: - "aio.h", - "ctype.h", - "dirent.h", - "dlfcn.h", - "errno.h", - "fcntl.h", - "glob.h", - "grp.h", - "ifaddrs.h", - "langinfo.h", - "limits.h", - "locale.h", - "malloc.h", - "mntent.h", - "mqueue.h", - "net/ethernet.h", - "net/if.h", - "net/if_arp.h", - "net/route.h", - "netdb.h", - "netinet/in.h", - "netinet/ip.h", - "netinet/tcp.h", - "netinet/udp.h", - "netpacket/packet.h", - "poll.h", - "pthread.h", - "pty.h", - "pwd.h", - "resolv.h", - "sched.h", - "sched.h", - "semaphore.h", - "shadow.h", - "signal.h", - "stddef.h", - "stdint.h", - "stdio.h", - "stdlib.h", - "string.h", - "sys/epoll.h", - "sys/eventfd.h", - "sys/file.h", - "sys/ioctl.h", - "sys/ipc.h", - "sys/mman.h", - "sys/mount.h", - "sys/msg.h", - "sys/personality.h", - "sys/prctl.h", - "sys/ptrace.h", - "sys/quota.h", - "sys/reboot.h", - "sys/resource.h", - "sys/sem.h", - "sys/sendfile.h", - "sys/shm.h", - "sys/signalfd.h", - "sys/socket.h", - "sys/stat.h", - "sys/statvfs.h", - "sys/swap.h", - "sys/syscall.h", - "sys/sysctl.h", - "sys/sysinfo.h", - "sys/time.h", - "sys/timerfd.h", - "sys/times.h", - "sys/types.h", - "sys/uio.h", - "sys/un.h", - "sys/user.h", - "sys/utsname.h", - "sys/vfs.h", - "sys/wait.h", - "sys/xattr.h", - "syslog.h", - "termios.h", - "time.h", - "ucontext.h", - "unistd.h", - "utime.h", - "utmp.h", - "utmpx.h", - "wchar.h", - } - - cfg.type_name(move |ty, is_struct, is_union| { - match ty { - // Just pass all these through, no need for a "struct" prefix - "FILE" | "fd_set" | "Dl_info" | "DIR" => ty.to_string(), - - t if is_union => format!("union {}", t), - - t if t.ends_with("_t") => t.to_string(), - - // put `struct` in front of all structs:. - t if is_struct => format!("struct {}", t), - - t => t.to_string(), - } - }); - - cfg.field_name(move |struct_, field| { - match field { - // Our stat *_nsec fields normally don't actually exist but are part - // of a timeval struct - s if s.ends_with("_nsec") && struct_.starts_with("stat") => { - s.replace("e_nsec", ".tv_nsec") - } - // FIXME: appears that `epoll_event.data` is an union - "u64" if struct_ == "epoll_event" => "data.u64".to_string(), - s => s.to_string(), - } - }); - - cfg.skip_type(move |ty| { - match ty { - // sighandler_t is crazy across platforms - // FIXME: is this necessary? - "sighandler_t" => true, - - _ => false, - } - }); - - cfg.skip_struct(move |ty| { - match ty { - // This is actually a union, not a struct - // FIXME: is this necessary? - "sigval" => true, - - _ => false, - } - }); - - cfg.skip_fn(move |name| { - match name { - // FIXME: https://github.com/rust-lang/libc/issues/1272 - "execv" | "execve" | "execvp" | "execvpe" | "fexecve" => true, - - _ => false, - } - }); - - cfg.skip_const(move |name| { - match name { - // FIXME: deprecated - SIGNUNUSED was removed in glibc 2.26 - // users should use SIGSYS instead - "SIGUNUSED" => true, - - // FIXME: emscripten uses different constants to constructs these - n if n.contains("__SIZEOF_PTHREAD") => true, - - _ => false, - } - }); - - cfg.skip_field_type(move |struct_, field| { - // This is a weird union, don't check the type. - // FIXME: is this necessary? - (struct_ == "ifaddrs" && field == "ifa_ifu") || - // sighandler_t type is super weird - // FIXME: is this necessary? - (struct_ == "sigaction" && field == "sa_sigaction") || - // sigval is actually a union, but we pretend it's a struct - // FIXME: is this necessary? - (struct_ == "sigevent" && field == "sigev_value") || - // aio_buf is "volatile void*" and Rust doesn't understand volatile - // FIXME: is this necessary? - (struct_ == "aiocb" && field == "aio_buf") - }); - - cfg.skip_field(move |struct_, field| { - // this is actually a union on linux, so we can't represent it well and - // just insert some padding. - // FIXME: is this necessary? - (struct_ == "siginfo_t" && field == "_pad") || - // musl names this __dummy1 but it's still there - // FIXME: is this necessary? - (struct_ == "glob_t" && field == "gl_flags") || - // musl seems to define this as an *anonymous* bitfield - // FIXME: is this necessary? - (struct_ == "statvfs" && field == "__f_unused") || - // sigev_notify_thread_id is actually part of a sigev_un union - (struct_ == "sigevent" && field == "sigev_notify_thread_id") || - // signalfd had SIGSYS fields added in Linux 4.18, but no libc release has them yet. - (struct_ == "signalfd_siginfo" && (field == "ssi_addr_lsb" || - field == "_pad2" || - field == "ssi_syscall" || - field == "ssi_call_addr" || - field == "ssi_arch")) - }); - - // FIXME: test linux like - cfg.generate("../src/lib.rs", "main.rs"); -} - -fn test_vxworks(target: &str) { - assert!(target.contains("vxworks")); - - let mut cfg = ctest::TestGenerator::new(); - headers! { cfg: - "vxWorks.h", - "yvals.h", - "nfs/nfsCommon.h", - "rtpLibCommon.h", - "randomNumGen.h", - "taskLib.h", - "sysLib.h", - "ioLib.h", - "inetLib.h", - "socket.h", - "errnoLib.h", - "ctype.h", - "dirent.h", - "dlfcn.h", - "elf.h", - "fcntl.h", - "grp.h", - "sys/poll.h", - "ifaddrs.h", - "langinfo.h", - "limits.h", - "link.h", - "locale.h", - "sys/stat.h", - "netdb.h", - "pthread.h", - "pwd.h", - "sched.h", - "semaphore.h", - "signal.h", - "stddef.h", - "stdint.h", - "stdio.h", - "stdlib.h", - "string.h", - "sys/file.h", - "sys/ioctl.h", - "sys/socket.h", - "sys/time.h", - "sys/times.h", - "sys/types.h", - "sys/uio.h", - "sys/un.h", - "sys/utsname.h", - "sys/wait.h", - "netinet/tcp.h", - "syslog.h", - "termios.h", - "time.h", - "ucontext.h", - "unistd.h", - "utime.h", - "wchar.h", - "errno.h", - "sys/mman.h", - "pathLib.h", - "mqueue.h", - } - /* Fix me */ - cfg.skip_const(move |name| match name { - // sighandler_t weirdness - "SIG_DFL" | "SIG_ERR" | "SIG_IGN" - // This is not defined in vxWorks - | "RTLD_DEFAULT" => true, - _ => false, - }); - /* Fix me */ - cfg.skip_type(move |ty| match ty { - "stat64" | "sighandler_t" | "off64_t" => true, - _ => false, - }); - - cfg.skip_field_type(move |struct_, field| match (struct_, field) { - ("siginfo_t", "si_value") - | ("stat", "st_size") - | ("sigaction", "sa_u") => true, - _ => false, - }); - - cfg.skip_roundtrip(move |s| match s { - _ => false, - }); - - cfg.type_name(move |ty, is_struct, is_union| match ty { - "DIR" | "FILE" | "Dl_info" | "RTP_DESC" => ty.to_string(), - t if is_union => format!("union {}", t), - t if t.ends_with("_t") => t.to_string(), - t if is_struct => format!("struct {}", t), - t => t.to_string(), - }); - - /* Fix me */ - cfg.skip_fn(move |name| match name { - /* sigval */ - "sigqueue" | "_sigqueue" - /* sighandler_t*/ - | "signal" - /* not used in static linking by default */ - | "dlerror" => true, - _ => false, - }); - - cfg.generate("../src/lib.rs", "main.rs"); -} - -fn test_linux(target: &str) { - assert!(target.contains("linux")); - - // target_env - let gnu = target.contains("gnu"); - let musl = target.contains("musl"); - let uclibc = target.contains("uclibc"); - - match (gnu, musl, uclibc) { - (true, false, false) => (), - (false, true, false) => (), - (false, false, true) => (), - (_, _, _) => panic!( - "linux target lib is gnu: {}, musl: {}, uclibc: {}", - gnu, musl, uclibc - ), - } - - let arm = target.contains("arm"); - let i686 = target.contains("i686"); - let mips = target.contains("mips"); - let mips32 = mips && !target.contains("64"); - let mips64 = mips && target.contains("64"); - let ppc64 = target.contains("powerpc64"); - let s390x = target.contains("s390x"); - let sparc64 = target.contains("sparc64"); - let x32 = target.contains("x32"); - let x86_32 = target.contains("i686"); - let x86_64 = target.contains("x86_64"); - let aarch64_musl = target.contains("aarch64") && musl; - let gnuabihf = target.contains("gnueabihf"); - - let mut cfg = ctest_cfg(); - cfg.define("_GNU_SOURCE", None); - // This macro re-deifnes fscanf,scanf,sscanf to link to the symbols that are - // deprecated since glibc >= 2.29. This allows Rust binaries to link against - // glibc versions older than 2.29. - cfg.define("__GLIBC_USE_DEPRECATED_SCANF", None); - - headers! { cfg: - "ctype.h", - "dirent.h", - "dlfcn.h", - "elf.h", - "fcntl.h", - "glob.h", - "grp.h", - "ifaddrs.h", - "langinfo.h", - "limits.h", - "link.h", - "locale.h", - "malloc.h", - "mntent.h", - "mqueue.h", - "net/ethernet.h", - "net/if.h", - "net/if_arp.h", - "net/route.h", - "netdb.h", - "netinet/in.h", - "netinet/ip.h", - "netinet/tcp.h", - "netinet/udp.h", - "netpacket/packet.h", - "poll.h", - "pthread.h", - "pty.h", - "pwd.h", - "resolv.h", - "sched.h", - "semaphore.h", - "shadow.h", - "signal.h", - "spawn.h", - "stddef.h", - "stdint.h", - "stdio.h", - "stdlib.h", - "string.h", - "sys/epoll.h", - "sys/eventfd.h", - "sys/file.h", - "sys/fsuid.h", - "sys/inotify.h", - "sys/ioctl.h", - "sys/ipc.h", - "sys/mman.h", - "sys/mount.h", - "sys/msg.h", - "sys/personality.h", - "sys/prctl.h", - "sys/ptrace.h", - "sys/quota.h", - "sys/random.h", - "sys/reboot.h", - "sys/resource.h", - "sys/sem.h", - "sys/sendfile.h", - "sys/shm.h", - "sys/signalfd.h", - "sys/socket.h", - "sys/stat.h", - "sys/statvfs.h", - "sys/swap.h", - "sys/syscall.h", - "sys/time.h", - "sys/timerfd.h", - "sys/times.h", - "sys/timex.h", - "sys/types.h", - "sys/uio.h", - "sys/un.h", - "sys/user.h", - "sys/utsname.h", - "sys/vfs.h", - "sys/wait.h", - "syslog.h", - "termios.h", - "time.h", - "ucontext.h", - "unistd.h", - "utime.h", - "utmp.h", - "utmpx.h", - "wchar.h", - "errno.h", - // `sys/io.h` is only available on x86*, Alpha, IA64, and 32-bit - // ARM: https://bugzilla.redhat.com/show_bug.cgi?id=1116162 - // Also unavailable on gnuabihf with glibc 2.30. - // https://sourceware.org/git/?p=glibc.git;a=commitdiff;h=6b33f373c7b9199e00ba5fbafd94ac9bfb4337b1 - [(x86_64 || x86_32 || arm) && !gnuabihf]: "sys/io.h", - // `sys/reg.h` is only available on x86 and x86_64 - [x86_64 || x86_32]: "sys/reg.h", - // sysctl system call is deprecated and not available on musl - // It is also unsupported in x32, deprecated since glibc 2.30: - [!(x32 || musl || gnu)]: "sys/sysctl.h", - // is not supported by musl: - // https://www.openwall.com/lists/musl/2015/04/09/3 - [!musl]: "execinfo.h", - } - - // Include linux headers at the end: - headers! { - cfg: - "asm/mman.h", - "linux/dccp.h", - "linux/errqueue.h", - "linux/falloc.h", - "linux/fs.h", - "linux/futex.h", - "linux/genetlink.h", - "linux/if.h", - "linux/if_addr.h", - "linux/if_alg.h", - "linux/if_ether.h", - "linux/if_tun.h", - "linux/input.h", - "linux/keyctl.h", - "linux/magic.h", - "linux/memfd.h", - "linux/module.h", - "linux/net_tstamp.h", - "linux/netfilter/nfnetlink.h", - "linux/netfilter/nfnetlink_log.h", - "linux/netfilter/nfnetlink_queue.h", - "linux/netfilter/nf_tables.h", - "linux/netfilter_ipv4.h", - "linux/netfilter_ipv6.h", - "linux/netfilter_ipv6/ip6_tables.h", - "linux/netlink.h", - "linux/quota.h", - "linux/random.h", - "linux/reboot.h", - "linux/rtnetlink.h", - "linux/seccomp.h", - "linux/sockios.h", - "linux/vm_sockets.h", - "sys/auxv.h", - } - - // note: aio.h must be included before sys/mount.h - headers! { - cfg: - "sys/xattr.h", - "sys/sysinfo.h", - "aio.h", - } - - cfg.type_name(move |ty, is_struct, is_union| { - match ty { - // Just pass all these through, no need for a "struct" prefix - "FILE" | "fd_set" | "Dl_info" | "DIR" | "Elf32_Phdr" - | "Elf64_Phdr" | "Elf32_Shdr" | "Elf64_Shdr" | "Elf32_Sym" - | "Elf64_Sym" | "Elf32_Ehdr" | "Elf64_Ehdr" | "Elf32_Chdr" - | "Elf64_Chdr" => ty.to_string(), - - t if is_union => format!("union {}", t), - - t if t.ends_with("_t") => t.to_string(), - - // In MUSL `flock64` is a typedef to `flock`. - "flock64" if musl => format!("struct {}", ty), - - // put `struct` in front of all structs:. - t if is_struct => format!("struct {}", t), - - t => t.to_string(), - } - }); - - cfg.field_name(move |struct_, field| { - match field { - // Our stat *_nsec fields normally don't actually exist but are part - // of a timeval struct - s if s.ends_with("_nsec") && struct_.starts_with("stat") => { - s.replace("e_nsec", ".tv_nsec") - } - // FIXME: epoll_event.data is actuall a union in C, but in Rust - // it is only a u64 because we only expose one field - // http://man7.org/linux/man-pages/man2/epoll_wait.2.html - "u64" if struct_ == "epoll_event" => "data.u64".to_string(), - // The following structs have a field called `type` in C, - // but `type` is a Rust keyword, so these fields are translated - // to `type_` in Rust. - "type_" - if struct_ == "input_event" - || struct_ == "input_mask" - || struct_ == "ff_effect" => - { - "type".to_string() - } - - s => s.to_string(), - } - }); - - cfg.skip_type(move |ty| { - match ty { - // FIXME: `sighandler_t` type is incorrect, see: - // https://github.com/rust-lang/libc/issues/1359 - "sighandler_t" => true, - - // These cannot be tested when "resolv.h" is included and are tested - // in the `linux_elf.rs` file. - "Elf64_Phdr" | "Elf32_Phdr" => true, - - // This type is private on Linux. It is implemented as a C `enum` - // (`c_uint`) and this clashes with the type of the `rlimit` APIs - // which expect a `c_int` even though both are ABI compatible. - "__rlimit_resource_t" => true, - - _ => false, - } - }); - - cfg.skip_struct(move |ty| { - match ty { - // These cannot be tested when "resolv.h" is included and are tested - // in the `linux_elf.rs` file. - "Elf64_Phdr" | "Elf32_Phdr" => true, - - // On Linux, the type of `ut_tv` field of `struct utmpx` - // can be an anonymous struct, so an extra struct, - // which is absent in glibc, has to be defined. - "__timeval" => true, - - // FIXME: This is actually a union, not a struct - "sigval" => true, - - // This type is tested in the `linux_termios.rs` file since there - // are header conflicts when including them with all the other - // structs. - "termios2" => true, - - // FIXME: remove once Ubuntu 20.04 LTS is released, somewhere in 2020. - // ucontext_t added a new field as of glibc 2.28; our struct definition is - // conservative and omits the field, but that means the size doesn't match for newer - // glibcs (see https://github.com/rust-lang/libc/issues/1410) - "ucontext_t" if gnu => true, - - // FIXME: Somehow we cannot include headers correctly in glibc 2.30. - // So let's ignore for now and re-visit later. - // Probably related: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=91085 - "statx" => true, - "statx_timestamp" => true, - - _ => false, - } - }); - - cfg.skip_const(move |name| { - match name { - // These constants are not available if gnu headers have been included - // and can therefore not be tested here - // - // The IPV6 constants are tested in the `linux_ipv6.rs` tests: - | "IPV6_FLOWINFO" - | "IPV6_FLOWLABEL_MGR" - | "IPV6_FLOWINFO_SEND" - | "IPV6_FLOWINFO_FLOWLABEL" - | "IPV6_FLOWINFO_PRIORITY" - // The F_ fnctl constants are tested in the `linux_fnctl.rs` tests: - | "F_CANCELLK" - | "F_ADD_SEALS" - | "F_GET_SEALS" - | "F_SEAL_SEAL" - | "F_SEAL_SHRINK" - | "F_SEAL_GROW" - | "F_SEAL_WRITE" => true, - - // The musl-sanitized kernel headers used in CI - // target the Linux kernel 4.4 and do not contain the - // following constants: - // - // Requires Linux kernel 4.9 - | "FALLOC_FL_UNSHARE_RANGE" - // - // Require Linux kernel 5.x: - | "MSG_COPY" - if musl => true, - // Require Linux kernel 5.1: - "F_SEAL_FUTURE_WRITE" => true, - - // The musl version 1.1.24 used in CI does not - // contain these glibc constants yet: - | "RLIMIT_RTTIME" // should be in `resource.h` - | "TCP_COOKIE_TRANSACTIONS" // should be in the `netinet/tcp.h` header - if musl => true, - - // FIXME: deprecated: not available in any header - // See: https://github.com/rust-lang/libc/issues/1356 - "ENOATTR" => true, - - // FIXME: SIGUNUSED was removed in glibc 2.26 - // Users should use SIGSYS instead. - "SIGUNUSED" => true, - - // FIXME: conflicts with glibc headers and is tested in - // `linux_termios.rs` below: - "BOTHER" => true, - - // FIXME: on musl the pthread types are defined a little differently - // - these constants are used by the glibc implementation. - n if musl && n.contains("__SIZEOF_PTHREAD") => true, - - _ => false, - } - }); - - cfg.skip_fn(move |name| { - // skip those that are manually verified - match name { - // FIXME: https://github.com/rust-lang/libc/issues/1272 - "execv" | "execve" | "execvp" | "execvpe" | "fexecve" => true, - - // There are two versions of the sterror_r function, see - // - // https://linux.die.net/man/3/strerror_r - // - // An XSI-compliant version provided if: - // - // (_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) - // && ! _GNU_SOURCE - // - // and a GNU specific version provided if _GNU_SOURCE is defined. - // - // libc provides bindings for the XSI-compliant version, which is - // preferred for portable applications. - // - // We skip the test here since here _GNU_SOURCE is defined, and - // test the XSI version below. - "strerror_r" => true, - - // FIXME: Our API is unsound. The Rust API allows aliasing - // pointers, but the C API requires pointers not to alias. - // We should probably be at least using `&`/`&mut` here, see: - // https://github.com/gnzlbg/ctest/issues/68 - "lio_listio" if musl => true, - - // FIXME: the glibc version used by the Sparc64 build jobs - // which use Debian 10.0 is too old. - "statx" if sparc64 => true, - - // FIXME: Deprecated since glibc 2.30. Remove fn once upstream does. - "sysctl" if gnu => true, - - _ => false, - } - }); - - cfg.skip_field_type(move |struct_, field| { - // This is a weird union, don't check the type. - (struct_ == "ifaddrs" && field == "ifa_ifu") || - // sighandler_t type is super weird - (struct_ == "sigaction" && field == "sa_sigaction") || - // __timeval type is a patch which doesn't exist in glibc - (struct_ == "utmpx" && field == "ut_tv") || - // sigval is actually a union, but we pretend it's a struct - (struct_ == "sigevent" && field == "sigev_value") || - // this one is an anonymous union - (struct_ == "ff_effect" && field == "u") - }); - - cfg.volatile_item(|i| { - use ctest::VolatileItemKind::*; - match i { - // aio_buf is a volatile void** but since we cannot express that in - // Rust types, we have to explicitly tell the checker about it here: - StructField(ref n, ref f) if n == "aiocb" && f == "aio_buf" => { - true - } - _ => false, - } - }); - - cfg.skip_field(move |struct_, field| { - // this is actually a union on linux, so we can't represent it well and - // just insert some padding. - (struct_ == "siginfo_t" && field == "_pad") || - // musl names this __dummy1 but it's still there - (musl && struct_ == "glob_t" && field == "gl_flags") || - // musl seems to define this as an *anonymous* bitfield - (musl && struct_ == "statvfs" && field == "__f_unused") || - // sigev_notify_thread_id is actually part of a sigev_un union - (struct_ == "sigevent" && field == "sigev_notify_thread_id") || - // signalfd had SIGSYS fields added in Linux 4.18, but no libc release - // has them yet. - (struct_ == "signalfd_siginfo" && (field == "ssi_addr_lsb" || - field == "_pad2" || - field == "ssi_syscall" || - field == "ssi_call_addr" || - field == "ssi_arch")) || - // FIXME: After musl 1.1.24, it have only one field `sched_priority`, - // while other fields become reserved. - (struct_ == "sched_param" && [ - "sched_ss_low_priority", - "sched_ss_repl_period", - "sched_ss_init_budget", - "sched_ss_max_repl", - ].contains(&field) && musl) || - // FIXME: After musl 1.1.24, the type becomes `int` instead of `unsigned short`. - (struct_ == "ipc_perm" && field == "__seq" && aarch64_musl) || - // glibc uses unnamed fields here and Rust doesn't support that yet - (struct_ == "timex" && field.starts_with("__unused")) - }); - - cfg.skip_roundtrip(move |s| match s { - // FIXME: - "utsname" if mips32 || mips64 => true, - // FIXME: - "mcontext_t" if s390x => true, - // FIXME: This is actually a union. - "fpreg_t" if s390x => true, - - "sockaddr_un" | "sembuf" | "ff_constant_effect" - if mips32 && (gnu || musl) => - { - true - } - "ipv6_mreq" - | "ip_mreq_source" - | "sockaddr_in6" - | "sockaddr_ll" - | "in_pktinfo" - | "arpreq" - | "arpreq_old" - | "sockaddr_un" - | "ff_constant_effect" - | "ff_ramp_effect" - | "ff_condition_effect" - | "Elf32_Ehdr" - | "Elf32_Chdr" - | "ucred" - | "in6_pktinfo" - | "sockaddr_nl" - | "termios" - | "nlmsgerr" - if (mips64 || sparc64) && gnu => - { - true - } - - // FIXME: the call ABI of max_align_t is incorrect on these platforms: - "max_align_t" if i686 || mips64 || ppc64 => true, - - _ => false, - }); - - cfg.generate("../src/lib.rs", "main.rs"); - - test_linux_like_apis(target); -} - -// This function tests APIs that are incompatible to test when other APIs -// are included (e.g. because including both sets of headers clashes) -fn test_linux_like_apis(target: &str) { - let musl = target.contains("musl"); - let linux = target.contains("linux"); - let emscripten = target.contains("emscripten"); - let android = target.contains("android"); - assert!(linux || android || emscripten); - - if linux || android || emscripten { - // test strerror_r from the `string.h` header - let mut cfg = ctest_cfg(); - cfg.skip_type(|_| true).skip_static(|_| true); - - headers! { cfg: "string.h" } - cfg.skip_fn(|f| match f { - "strerror_r" => false, - _ => true, - }) - .skip_const(|_| true) - .skip_struct(|_| true); - cfg.generate("../src/lib.rs", "linux_strerror_r.rs"); - } - - if linux || android || emscripten { - // test fcntl - see: - // http://man7.org/linux/man-pages/man2/fcntl.2.html - let mut cfg = ctest_cfg(); - - if musl { - cfg.header("fcntl.h"); - } else { - cfg.header("linux/fcntl.h"); - } - - cfg.skip_type(|_| true) - .skip_static(|_| true) - .skip_struct(|_| true) - .skip_fn(|_| true) - .skip_const(move |name| match name { - // test fcntl constants: - "F_CANCELLK" | "F_ADD_SEALS" | "F_GET_SEALS" - | "F_SEAL_SEAL" | "F_SEAL_SHRINK" | "F_SEAL_GROW" - | "F_SEAL_WRITE" => false, - _ => true, - }) - .type_name(move |ty, is_struct, is_union| match ty { - t if is_struct => format!("struct {}", t), - t if is_union => format!("union {}", t), - t => t.to_string(), - }); - - cfg.generate("../src/lib.rs", "linux_fcntl.rs"); - } - - if linux || android { - // test termios - let mut cfg = ctest_cfg(); - cfg.header("asm/termbits.h"); - cfg.skip_type(|_| true) - .skip_static(|_| true) - .skip_fn(|_| true) - .skip_const(|c| c != "BOTHER") - .skip_struct(|s| s != "termios2") - .type_name(move |ty, is_struct, is_union| match ty { - t if is_struct => format!("struct {}", t), - t if is_union => format!("union {}", t), - t => t.to_string(), - }); - cfg.generate("../src/lib.rs", "linux_termios.rs"); - } - - if linux || android { - // test IPV6_ constants: - let mut cfg = ctest_cfg(); - headers! { - cfg: - "linux/in6.h" - } - cfg.skip_type(|_| true) - .skip_static(|_| true) - .skip_fn(|_| true) - .skip_const(|_| true) - .skip_struct(|_| true) - .skip_const(move |name| match name { - "IPV6_FLOWINFO" - | "IPV6_FLOWLABEL_MGR" - | "IPV6_FLOWINFO_SEND" - | "IPV6_FLOWINFO_FLOWLABEL" - | "IPV6_FLOWINFO_PRIORITY" => false, - _ => true, - }) - .type_name(move |ty, is_struct, is_union| match ty { - t if is_struct => format!("struct {}", t), - t if is_union => format!("union {}", t), - t => t.to_string(), - }); - cfg.generate("../src/lib.rs", "linux_ipv6.rs"); - } - - if linux || android { - // Test Elf64_Phdr and Elf32_Phdr - // These types have a field called `p_type`, but including - // "resolve.h" defines a `p_type` macro that expands to `__p_type` - // making the tests for these fails when both are included. - let mut cfg = ctest_cfg(); - cfg.header("elf.h"); - cfg.skip_fn(|_| true) - .skip_static(|_| true) - .skip_fn(|_| true) - .skip_const(|_| true) - .type_name(move |ty, _is_struct, _is_union| ty.to_string()) - .skip_struct(move |ty| match ty { - "Elf64_Phdr" | "Elf32_Phdr" => false, - _ => true, - }) - .skip_type(move |ty| match ty { - "Elf64_Phdr" | "Elf32_Phdr" => false, - _ => true, - }); - cfg.generate("../src/lib.rs", "linux_elf.rs"); - } -} - -fn which_freebsd() -> Option { - let output = std::process::Command::new("freebsd-version") - .output() - .ok()?; - if !output.status.success() { - return None; - } - - let stdout = String::from_utf8(output.stdout).ok()?; - - match &stdout { - s if s.starts_with("10") => Some(10), - s if s.starts_with("11") => Some(11), - s if s.starts_with("12") => Some(12), - s if s.starts_with("13") => Some(13), - _ => None, - } -} diff --git a/crux-mir/lib/libc/libc-test/src/cmsg.c b/crux-mir/lib/libc/libc-test/src/cmsg.c deleted file mode 100644 index a8b1c3717..000000000 --- a/crux-mir/lib/libc/libc-test/src/cmsg.c +++ /dev/null @@ -1,28 +0,0 @@ -#include -#include - -// Since the cmsg(3) macros are macros instead of functions, they aren't -// available to FFI. libc must reimplement them, which is error-prone. This -// file provides FFI access to the actual macros so they can be tested against -// the Rust reimplementations. - -struct cmsghdr *cmsg_firsthdr(struct msghdr *msgh) { - return CMSG_FIRSTHDR(msgh); -} - -struct cmsghdr *cmsg_nxthdr(struct msghdr *msgh, struct cmsghdr *cmsg) { - return CMSG_NXTHDR(msgh, cmsg); -} - -size_t cmsg_space(size_t length) { - return CMSG_SPACE(length); -} - -size_t cmsg_len(size_t length) { - return CMSG_LEN(length); -} - -unsigned char *cmsg_data(struct cmsghdr *cmsg) { - return CMSG_DATA(cmsg); -} - diff --git a/crux-mir/lib/libc/libc-test/src/errqueue.c b/crux-mir/lib/libc/libc-test/src/errqueue.c deleted file mode 100644 index 931ca91bb..000000000 --- a/crux-mir/lib/libc/libc-test/src/errqueue.c +++ /dev/null @@ -1,10 +0,0 @@ -#include -#include - -// SO_EE_OFFENDER is defined as a macro in linux/errqueue.h. This file wraps -// that macro in a function so we can test the reimplementation in this package -// is equivalent. - -struct sockaddr *so_ee_offender(struct sock_extended_err *ee) { - return SO_EE_OFFENDER(ee); -} diff --git a/crux-mir/lib/libc/libc-test/test/cmsg.rs b/crux-mir/lib/libc/libc-test/test/cmsg.rs deleted file mode 100644 index 38a8ce150..000000000 --- a/crux-mir/lib/libc/libc-test/test/cmsg.rs +++ /dev/null @@ -1,100 +0,0 @@ -//! Compare libc's CMSG(3) family of functions against the actual C macros, for -//! various inputs. - -extern crate libc; - -#[cfg(unix)] -mod t { - - use libc::{self, c_uchar, c_uint, c_void, cmsghdr, msghdr}; - use std::mem; - - extern "C" { - pub fn cmsg_firsthdr(msgh: *const msghdr) -> *mut cmsghdr; - pub fn cmsg_nxthdr( - mhdr: *const msghdr, - cmsg: *const cmsghdr, - ) -> *mut cmsghdr; - pub fn cmsg_space(length: c_uint) -> usize; - pub fn cmsg_len(length: c_uint) -> usize; - pub fn cmsg_data(cmsg: *const cmsghdr) -> *mut c_uchar; - } - - #[test] - fn test_cmsg_data() { - for l in 0..128 { - let pcmsghdr = l as *const cmsghdr; - unsafe { - assert_eq!(libc::CMSG_DATA(pcmsghdr), cmsg_data(pcmsghdr)); - } - } - } - - #[test] - fn test_cmsg_firsthdr() { - let mut mhdr: msghdr = unsafe { mem::zeroed() }; - mhdr.msg_control = 0xdeadbeef as *mut c_void; - let pmhdr = &mhdr as *const msghdr; - for l in 0..128 { - mhdr.msg_controllen = l; - unsafe { - assert_eq!(libc::CMSG_FIRSTHDR(pmhdr), cmsg_firsthdr(pmhdr)); - } - } - } - - #[test] - fn test_cmsg_len() { - for l in 0..128 { - unsafe { - assert_eq!(libc::CMSG_LEN(l) as usize, cmsg_len(l)); - } - } - } - - // Skip on sparc64 - // https://github.com/rust-lang/libc/issues/1239 - #[cfg(not(target_arch = "sparc64"))] - #[test] - fn test_cmsg_nxthdr() { - use std::ptr; - - let mut buffer = [0u8; 256]; - let mut mhdr: msghdr = unsafe { mem::zeroed() }; - let pmhdr = &mhdr as *const msghdr; - for start_ofs in 0..64 { - let pcmsghdr = &mut buffer[start_ofs] as *mut u8 as *mut cmsghdr; - mhdr.msg_control = pcmsghdr as *mut c_void; - mhdr.msg_controllen = (160 - start_ofs) as _; - for cmsg_len in 0..64 { - for next_cmsg_len in 0..32 { - for i in buffer[start_ofs..].iter_mut() { - *i = 0; - } - unsafe { - (*pcmsghdr).cmsg_len = cmsg_len; - let libc_next = libc::CMSG_NXTHDR(pmhdr, pcmsghdr); - let next = cmsg_nxthdr(pmhdr, pcmsghdr); - assert_eq!(libc_next, next); - - if libc_next != ptr::null_mut() { - (*libc_next).cmsg_len = next_cmsg_len; - let libc_next = libc::CMSG_NXTHDR(pmhdr, pcmsghdr); - let next = cmsg_nxthdr(pmhdr, pcmsghdr); - assert_eq!(libc_next, next); - } - } - } - } - } - } - - #[test] - fn test_cmsg_space() { - unsafe { - for l in 0..128 { - assert_eq!(libc::CMSG_SPACE(l) as usize, cmsg_space(l)); - } - } - } -} diff --git a/crux-mir/lib/libc/libc-test/test/errqueue.rs b/crux-mir/lib/libc/libc-test/test/errqueue.rs deleted file mode 100644 index 8d0c7bb74..000000000 --- a/crux-mir/lib/libc/libc-test/test/errqueue.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! Compare libc's SO_EE_OFFENDER function against the actual C macro - -extern crate libc; - -#[cfg(any(target_os = "linux", target_os = "android"))] -mod t { - use libc::{self, sock_extended_err, sockaddr}; - - extern "C" { - pub fn so_ee_offender(ee: *const sock_extended_err) -> *mut sockaddr; - } - - #[test] - fn test_cmsg_data() { - for l in 0..128 { - let ee = l as *const sock_extended_err; - unsafe { - assert_eq!(libc::SO_EE_OFFENDER(ee), so_ee_offender(ee)); - } - } - } -} diff --git a/crux-mir/lib/libc/libc-test/test/linux_elf.rs b/crux-mir/lib/libc/libc-test/test/linux_elf.rs deleted file mode 100644 index d149c9aaf..000000000 --- a/crux-mir/lib/libc/libc-test/test/linux_elf.rs +++ /dev/null @@ -1,12 +0,0 @@ -#![allow(bad_style, improper_ctypes, unused, deprecated)] - -extern crate libc; -use libc::*; - -#[cfg(target_os = "linux")] -include!(concat!(env!("OUT_DIR"), "/linux_elf.rs")); - -#[cfg(not(target_os = "linux"))] -fn main() { - println!("PASSED 0 tests"); -} diff --git a/crux-mir/lib/libc/libc-test/test/linux_fcntl.rs b/crux-mir/lib/libc/libc-test/test/linux_fcntl.rs deleted file mode 100644 index 49c06cc4f..000000000 --- a/crux-mir/lib/libc/libc-test/test/linux_fcntl.rs +++ /dev/null @@ -1,12 +0,0 @@ -#![allow(bad_style, improper_ctypes, unused, deprecated)] - -extern crate libc; -use libc::*; - -#[cfg(any(target_os = "linux", target_os = "android"))] -include!(concat!(env!("OUT_DIR"), "/linux_fcntl.rs")); - -#[cfg(not(any(target_os = "linux", target_os = "android")))] -fn main() { - println!("PASSED 0 tests"); -} diff --git a/crux-mir/lib/libc/libc-test/test/linux_ipv6.rs b/crux-mir/lib/libc/libc-test/test/linux_ipv6.rs deleted file mode 100644 index 83c389ce1..000000000 --- a/crux-mir/lib/libc/libc-test/test/linux_ipv6.rs +++ /dev/null @@ -1,12 +0,0 @@ -#![allow(bad_style, improper_ctypes, unused, deprecated)] - -extern crate libc; -use libc::*; - -#[cfg(target_os = "linux")] -include!(concat!(env!("OUT_DIR"), "/linux_ipv6.rs")); - -#[cfg(not(target_os = "linux"))] -fn main() { - println!("PASSED 0 tests"); -} diff --git a/crux-mir/lib/libc/libc-test/test/linux_strerror_r.rs b/crux-mir/lib/libc/libc-test/test/linux_strerror_r.rs deleted file mode 100644 index 17db959d8..000000000 --- a/crux-mir/lib/libc/libc-test/test/linux_strerror_r.rs +++ /dev/null @@ -1,12 +0,0 @@ -#![allow(bad_style, improper_ctypes, unused, deprecated)] - -extern crate libc; -use libc::*; - -#[cfg(any(target_os = "linux", target_os = "android"))] -include!(concat!(env!("OUT_DIR"), "/linux_strerror_r.rs")); - -#[cfg(not(any(target_os = "linux", target_os = "android")))] -fn main() { - println!("PASSED 0 tests"); -} diff --git a/crux-mir/lib/libc/libc-test/test/linux_termios.rs b/crux-mir/lib/libc/libc-test/test/linux_termios.rs deleted file mode 100644 index 703a9b9b2..000000000 --- a/crux-mir/lib/libc/libc-test/test/linux_termios.rs +++ /dev/null @@ -1,12 +0,0 @@ -#![allow(bad_style, improper_ctypes, unused, deprecated)] - -extern crate libc; -use libc::*; - -#[cfg(any(target_os = "linux", target_os = "android"))] -include!(concat!(env!("OUT_DIR"), "/linux_termios.rs")); - -#[cfg(not(any(target_os = "linux", target_os = "android")))] -fn main() { - println!("PASSED 0 tests"); -} diff --git a/crux-mir/lib/libc/libc-test/test/main.rs b/crux-mir/lib/libc/libc-test/test/main.rs deleted file mode 100644 index 62a587cf5..000000000 --- a/crux-mir/lib/libc/libc-test/test/main.rs +++ /dev/null @@ -1,6 +0,0 @@ -#![allow(bad_style, improper_ctypes, deprecated)] -extern crate libc; - -use libc::*; - -include!(concat!(env!("OUT_DIR"), "/main.rs")); diff --git a/crux-mir/lib/libc/rustfmt.toml b/crux-mir/lib/libc/rustfmt.toml index 7ecc610f3..dc85c9946 100644 --- a/crux-mir/lib/libc/rustfmt.toml +++ b/crux-mir/lib/libc/rustfmt.toml @@ -1,3 +1 @@ -max_width = 79 -comment_width = 79 -error_on_line_overflow = true \ No newline at end of file +error_on_line_overflow = true diff --git a/crux-mir/lib/libc/src/cloudabi/mod.rs b/crux-mir/lib/libc/src/cloudabi/mod.rs deleted file mode 100644 index 77817121d..000000000 --- a/crux-mir/lib/libc/src/cloudabi/mod.rs +++ /dev/null @@ -1,375 +0,0 @@ -pub type c_schar = i8; -pub type c_uchar = u8; -pub type c_short = i16; -pub type c_ushort = u16; -pub type c_int = i32; -pub type c_uint = u32; -pub type c_float = f32; -pub type c_double = f64; -pub type c_longlong = i64; -pub type c_ulonglong = u64; -pub type intmax_t = i64; -pub type uintmax_t = u64; - -pub type size_t = usize; -pub type ptrdiff_t = isize; -pub type intptr_t = isize; -pub type uintptr_t = usize; -pub type ssize_t = isize; - -pub type in_addr_t = u32; -pub type in_port_t = u16; -pub type pthread_key_t = usize; -pub type pthread_t = usize; -pub type sa_family_t = u8; -pub type socklen_t = usize; -pub type time_t = i64; - -s! { - pub struct addrinfo { - pub ai_flags: ::c_int, - pub ai_family: ::c_int, - pub ai_socktype: ::c_int, - pub ai_protocol: ::c_int, - pub ai_addrlen: ::socklen_t, - pub ai_addr: *mut ::sockaddr, - pub ai_canonname: *mut ::c_char, - pub ai_next: *mut addrinfo, - } - - pub struct in_addr { - pub s_addr: in_addr_t, - } - - pub struct in6_addr { - pub s6_addr: [u8; 16], - } - - pub struct pthread_attr_t { - __detachstate: ::c_int, - __stacksize: usize, - } - - pub struct sockaddr { - pub sa_family: sa_family_t, - pub sa_data: [::c_char; 0], - } - - pub struct sockaddr_in { - pub sin_family: ::sa_family_t, - pub sin_port: ::in_port_t, - pub sin_addr: ::in_addr, - } - - pub struct sockaddr_in6 { - pub sin6_family: sa_family_t, - pub sin6_port: ::in_port_t, - pub sin6_flowinfo: u32, - pub sin6_addr: ::in6_addr, - pub sin6_scope_id: u32, - } - - pub struct sockaddr_storage { - pub ss_family: ::sa_family_t, - __ss_data: [u8; 32], - } -} - -pub const INT_MIN: c_int = -2147483648; -pub const INT_MAX: c_int = 2147483647; - -pub const _SC_NPROCESSORS_ONLN: ::c_int = 52; -pub const _SC_PAGESIZE: ::c_int = 54; - -pub const AF_INET: ::c_int = 1; -pub const AF_INET6: ::c_int = 2; - -pub const EACCES: ::c_int = 2; -pub const EADDRINUSE: ::c_int = 3; -pub const EADDRNOTAVAIL: ::c_int = 4; -pub const EAGAIN: ::c_int = 6; -pub const ECONNABORTED: ::c_int = 13; -pub const ECONNREFUSED: ::c_int = 14; -pub const ECONNRESET: ::c_int = 15; -pub const EEXIST: ::c_int = 20; -pub const EINTR: ::c_int = 27; -pub const EINVAL: ::c_int = 28; -pub const ENOENT: ::c_int = 44; -pub const ENOTCONN: ::c_int = 53; -pub const EPERM: ::c_int = 63; -pub const EPIPE: ::c_int = 64; -pub const ETIMEDOUT: ::c_int = 73; -pub const EWOULDBLOCK: ::c_int = EAGAIN; - -pub const EAI_SYSTEM: ::c_int = 9; - -pub const EXIT_FAILURE: ::c_int = 1; -pub const EXIT_SUCCESS: ::c_int = 0; - -pub const PTHREAD_STACK_MIN: ::size_t = 1024; - -pub const SOCK_DGRAM: ::c_int = 128; -pub const SOCK_STREAM: ::c_int = 130; - -#[cfg_attr(feature = "extra_traits", derive(Debug))] -pub enum FILE {} -impl ::Copy for FILE {} -impl ::Clone for FILE { - fn clone(&self) -> FILE { - *self - } -} -#[cfg_attr(feature = "extra_traits", derive(Debug))] -pub enum fpos_t {} // FIXME: fill this out with a struct -impl ::Copy for fpos_t {} -impl ::Clone for fpos_t { - fn clone(&self) -> fpos_t { - *self - } -} - -extern "C" { - pub fn isalnum(c: c_int) -> c_int; - pub fn isalpha(c: c_int) -> c_int; - pub fn iscntrl(c: c_int) -> c_int; - pub fn isdigit(c: c_int) -> c_int; - pub fn isgraph(c: c_int) -> c_int; - pub fn islower(c: c_int) -> c_int; - pub fn isprint(c: c_int) -> c_int; - pub fn ispunct(c: c_int) -> c_int; - pub fn isspace(c: c_int) -> c_int; - pub fn isupper(c: c_int) -> c_int; - pub fn isxdigit(c: c_int) -> c_int; - pub fn isblank(c: c_int) -> c_int; - pub fn tolower(c: c_int) -> c_int; - pub fn toupper(c: c_int) -> c_int; - pub fn fopen(filename: *const c_char, mode: *const c_char) -> *mut FILE; - pub fn freopen( - filename: *const c_char, - mode: *const c_char, - file: *mut FILE, - ) -> *mut FILE; - pub fn fflush(file: *mut FILE) -> c_int; - pub fn fclose(file: *mut FILE) -> c_int; - pub fn remove(filename: *const c_char) -> c_int; - pub fn rename(oldname: *const c_char, newname: *const c_char) -> c_int; - pub fn tmpfile() -> *mut FILE; - pub fn setvbuf( - stream: *mut FILE, - buffer: *mut c_char, - mode: c_int, - size: size_t, - ) -> c_int; - pub fn setbuf(stream: *mut FILE, buf: *mut c_char); - pub fn getchar() -> c_int; - pub fn putchar(c: c_int) -> c_int; - pub fn fgetc(stream: *mut FILE) -> c_int; - pub fn fgets(buf: *mut c_char, n: c_int, stream: *mut FILE) - -> *mut c_char; - pub fn fputc(c: c_int, stream: *mut FILE) -> c_int; - pub fn fputs(s: *const c_char, stream: *mut FILE) -> c_int; - pub fn puts(s: *const c_char) -> c_int; - pub fn ungetc(c: c_int, stream: *mut FILE) -> c_int; - pub fn fread( - ptr: *mut c_void, - size: size_t, - nobj: size_t, - stream: *mut FILE, - ) -> size_t; - pub fn fwrite( - ptr: *const c_void, - size: size_t, - nobj: size_t, - stream: *mut FILE, - ) -> size_t; - pub fn fseek(stream: *mut FILE, offset: c_long, whence: c_int) -> c_int; - pub fn ftell(stream: *mut FILE) -> c_long; - pub fn rewind(stream: *mut FILE); - pub fn fgetpos(stream: *mut FILE, ptr: *mut fpos_t) -> c_int; - pub fn fsetpos(stream: *mut FILE, ptr: *const fpos_t) -> c_int; - pub fn feof(stream: *mut FILE) -> c_int; - pub fn ferror(stream: *mut FILE) -> c_int; - pub fn perror(s: *const c_char); - pub fn atoi(s: *const c_char) -> c_int; - pub fn strtod(s: *const c_char, endp: *mut *mut c_char) -> c_double; - pub fn strtol( - s: *const c_char, - endp: *mut *mut c_char, - base: c_int, - ) -> c_long; - pub fn strtoul( - s: *const c_char, - endp: *mut *mut c_char, - base: c_int, - ) -> c_ulong; - pub fn calloc(nobj: size_t, size: size_t) -> *mut c_void; - pub fn malloc(size: size_t) -> *mut c_void; - pub fn realloc(p: *mut c_void, size: size_t) -> *mut c_void; - pub fn free(p: *mut c_void); - pub fn abort() -> !; - pub fn exit(status: c_int) -> !; - pub fn _exit(status: c_int) -> !; - pub fn atexit(cb: extern "C" fn()) -> c_int; - pub fn system(s: *const c_char) -> c_int; - pub fn getenv(s: *const c_char) -> *mut c_char; - pub fn getline( - lineptr: *mut *mut c_char, - n: *mut size_t, - stream: *mut FILE, - ) -> ssize_t; - - pub fn strcpy(dst: *mut c_char, src: *const c_char) -> *mut c_char; - pub fn strncpy( - dst: *mut c_char, - src: *const c_char, - n: size_t, - ) -> *mut c_char; - pub fn strcat(s: *mut c_char, ct: *const c_char) -> *mut c_char; - pub fn strncat( - s: *mut c_char, - ct: *const c_char, - n: size_t, - ) -> *mut c_char; - pub fn strcmp(cs: *const c_char, ct: *const c_char) -> c_int; - pub fn strncmp(cs: *const c_char, ct: *const c_char, n: size_t) -> c_int; - pub fn strcoll(cs: *const c_char, ct: *const c_char) -> c_int; - pub fn strchr(cs: *const c_char, c: c_int) -> *mut c_char; - pub fn strrchr(cs: *const c_char, c: c_int) -> *mut c_char; - pub fn strspn(cs: *const c_char, ct: *const c_char) -> size_t; - pub fn strcspn(cs: *const c_char, ct: *const c_char) -> size_t; - pub fn strdup(cs: *const c_char) -> *mut c_char; - pub fn strpbrk(cs: *const c_char, ct: *const c_char) -> *mut c_char; - pub fn strstr(cs: *const c_char, ct: *const c_char) -> *mut c_char; - pub fn strcasecmp(s1: *const c_char, s2: *const c_char) -> c_int; - pub fn strncasecmp( - s1: *const c_char, - s2: *const c_char, - n: size_t, - ) -> c_int; - pub fn strlen(cs: *const c_char) -> size_t; - pub fn strnlen(cs: *const c_char, maxlen: size_t) -> size_t; - pub fn strerror(n: c_int) -> *mut c_char; - pub fn strtok(s: *mut c_char, t: *const c_char) -> *mut c_char; - pub fn strxfrm(s: *mut c_char, ct: *const c_char, n: size_t) -> size_t; - pub fn wcslen(buf: *const wchar_t) -> size_t; - pub fn wcstombs( - dest: *mut c_char, - src: *const wchar_t, - n: size_t, - ) -> ::size_t; - - pub fn memchr(cx: *const c_void, c: c_int, n: size_t) -> *mut c_void; - pub fn wmemchr(cx: *const wchar_t, c: wchar_t, n: size_t) -> *mut wchar_t; - pub fn memcmp(cx: *const c_void, ct: *const c_void, n: size_t) -> c_int; - pub fn memcpy( - dest: *mut c_void, - src: *const c_void, - n: size_t, - ) -> *mut c_void; - pub fn memmove( - dest: *mut c_void, - src: *const c_void, - n: size_t, - ) -> *mut c_void; - pub fn memset(dest: *mut c_void, c: c_int, n: size_t) -> *mut c_void; - - pub fn abs(i: c_int) -> c_int; - pub fn atof(s: *const c_char) -> c_double; - pub fn labs(i: c_long) -> c_long; - pub fn rand() -> c_int; - pub fn srand(seed: c_uint); - - pub fn arc4random_buf(buf: *const ::c_void, len: ::size_t); - pub fn freeaddrinfo(res: *mut addrinfo); - pub fn gai_strerror(errcode: ::c_int) -> *const ::c_char; - pub fn getaddrinfo( - node: *const c_char, - service: *const c_char, - hints: *const addrinfo, - res: *mut *mut addrinfo, - ) -> ::c_int; - pub fn getsockopt( - sockfd: ::c_int, - level: ::c_int, - optname: ::c_int, - optval: *mut ::c_void, - optlen: *mut ::socklen_t, - ) -> ::c_int; - pub fn posix_memalign( - memptr: *mut *mut ::c_void, - align: ::size_t, - size: ::size_t, - ) -> ::c_int; - pub fn pthread_attr_destroy(attr: *mut ::pthread_attr_t) -> ::c_int; - pub fn pthread_attr_init(attr: *mut ::pthread_attr_t) -> ::c_int; - pub fn pthread_attr_setstacksize( - attr: *mut ::pthread_attr_t, - stack_size: ::size_t, - ) -> ::c_int; - pub fn pthread_create( - native: *mut ::pthread_t, - attr: *const ::pthread_attr_t, - f: extern "C" fn(*mut ::c_void) -> *mut ::c_void, - value: *mut ::c_void, - ) -> ::c_int; - pub fn pthread_detach(thread: ::pthread_t) -> ::c_int; - pub fn pthread_getspecific(key: pthread_key_t) -> *mut ::c_void; - pub fn pthread_join( - native: ::pthread_t, - value: *mut *mut ::c_void, - ) -> ::c_int; - pub fn pthread_key_create( - key: *mut pthread_key_t, - dtor: ::Option, - ) -> ::c_int; - pub fn pthread_key_delete(key: pthread_key_t) -> ::c_int; - pub fn pthread_setspecific( - key: pthread_key_t, - value: *const ::c_void, - ) -> ::c_int; - pub fn send( - socket: ::c_int, - buf: *const ::c_void, - len: ::size_t, - flags: ::c_int, - ) -> ::ssize_t; - pub fn sysconf(name: ::c_int) -> ::c_long; -} - -cfg_if! { - if #[cfg(target_arch = "aarch64")] { - mod aarch64; - pub use self::aarch64::*; - } else if #[cfg(any(target_arch = "arm"))] { - mod arm; - pub use self::arm::*; - } else if #[cfg(any(target_arch = "x86"))] { - mod x86; - pub use self::x86::*; - } else if #[cfg(any(target_arch = "x86_64"))] { - mod x86_64; - pub use self::x86_64::*; - } else { - // Unknown target_arch - } -} - -cfg_if! { - if #[cfg(libc_core_cvoid)] { - pub use ::ffi::c_void; - } else { - // Use repr(u8) as LLVM expects `void*` to be the same as `i8*` to help - // enable more optimization opportunities around it recognizing things - // like malloc/free. - #[repr(u8)] - #[allow(missing_copy_implementations)] - #[allow(missing_debug_implementations)] - pub enum c_void { - // Two dummy variants so the #[repr] attribute can be used. - #[doc(hidden)] - __variant1, - #[doc(hidden)] - __variant2, - } - } -} diff --git a/crux-mir/lib/libc/src/cloudabi/x86.rs b/crux-mir/lib/libc/src/cloudabi/x86.rs deleted file mode 100644 index 2f9f39c9b..000000000 --- a/crux-mir/lib/libc/src/cloudabi/x86.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub type c_char = i8; -pub type c_long = i32; -pub type c_ulong = u32; -pub type wchar_t = i32; diff --git a/crux-mir/lib/libc/src/cloudabi/x86_64.rs b/crux-mir/lib/libc/src/cloudabi/x86_64.rs deleted file mode 100644 index bb17624b1..000000000 --- a/crux-mir/lib/libc/src/cloudabi/x86_64.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub type c_char = i8; -pub type c_long = i64; -pub type c_ulong = u64; -pub type wchar_t = i32; diff --git a/crux-mir/lib/libc/src/fixed_width_ints.rs b/crux-mir/lib/libc/src/fixed_width_ints.rs index 014640855..999de8f54 100644 --- a/crux-mir/lib/libc/src/fixed_width_ints.rs +++ b/crux-mir/lib/libc/src/fixed_width_ints.rs @@ -18,3 +18,82 @@ pub type uint16_t = u16; pub type uint32_t = u32; #[deprecated(since = "0.2.55", note = "Use u64 instead.")] pub type uint64_t = u64; + +cfg_if! { + if #[cfg(all(libc_int128, target_arch = "aarch64", not(target_os = "windows")))] { + // This introduces partial support for FFI with __int128 and + // equivalent types on platforms where Rust's definition is validated + // to match the standard C ABI of that platform. + // + // Rust does not guarantee u128/i128 are sound for FFI, and its + // definitions are in fact known to be incompatible. [0] + // + // However these problems aren't fundamental, and are just platform + // inconsistencies. Specifically at the time of this writing: + // + // * For x64 SysV ABIs (everything but Windows), the types are underaligned. + // * For all Windows ABIs, Microsoft doesn't actually officially define __int128, + // and as a result different implementations don't actually agree on its ABI. + // + // But on the other major aarch64 platforms (android, linux, ios, macos) we have + // validated that rustc has the right ABI for these types. This is important because + // aarch64 uses these types in some fundamental OS types like user_fpsimd_struct, + // which represents saved simd registers. + // + // Any API which uses these types will need to `#[ignore(improper_ctypes)]` + // until the upstream rust issue is resolved, but this at least lets us make + // progress on platforms where this type is important. + // + // The list of supported architectures and OSes is intentionally very restricted, + // as careful work needs to be done to verify that a particular platform + // has a conformant ABI. + // + // [0]: https://github.com/rust-lang/rust/issues/54341 + + /// C `__int128` (a GCC extension that's part of many ABIs) + pub type __int128 = i128; + /// C `unsigned __int128` (a GCC extension that's part of many ABIs) + pub type __uint128 = u128; + /// C __int128_t (alternate name for [__int128][]) + pub type __int128_t = i128; + /// C __uint128_t (alternate name for [__uint128][]) + pub type __uint128_t = u128; + + cfg_if! { + if #[cfg(libc_underscore_const_names)] { + macro_rules! static_assert_eq { + ($a:expr, $b:expr) => { + const _: [(); $a] = [(); $b]; + }; + } + + // NOTE: if you add more platforms to here, you may need to cfg + // these consts. They should always match the platform's values + // for `sizeof(__int128)` and `_Alignof(__int128)`. + const _SIZE_128: usize = 16; + const _ALIGN_128: usize = 16; + + // Since Rust doesn't officially guarantee that these types + // have compatible ABIs, we const assert that these values have the + // known size/align of the target platform's libc. If rustc ever + // tries to regress things, it will cause a compilation error. + // + // This isn't a bullet-proof solution because e.g. it doesn't + // catch the fact that llvm and gcc disagree on how x64 __int128 + // is actually *passed* on the stack (clang underaligns it for + // the same reason that rustc *never* properly aligns it). + static_assert_eq!(core::mem::size_of::<__int128>(), _SIZE_128); + static_assert_eq!(core::mem::align_of::<__int128>(), _ALIGN_128); + + static_assert_eq!(core::mem::size_of::<__uint128>(), _SIZE_128); + static_assert_eq!(core::mem::align_of::<__uint128>(), _ALIGN_128); + + static_assert_eq!(core::mem::size_of::<__int128_t>(), _SIZE_128); + static_assert_eq!(core::mem::align_of::<__int128_t>(), _ALIGN_128); + + static_assert_eq!(core::mem::size_of::<__uint128_t>(), _SIZE_128); + static_assert_eq!(core::mem::align_of::<__uint128_t>(), _ALIGN_128); + } + } + } +} diff --git a/crux-mir/lib/libc/src/fuchsia/mod.rs b/crux-mir/lib/libc/src/fuchsia/mod.rs index 471708949..7a9edada1 100644 --- a/crux-mir/lib/libc/src/fuchsia/mod.rs +++ b/crux-mir/lib/libc/src/fuchsia/mod.rs @@ -209,6 +209,12 @@ s! { pub imr_interface: in_addr, } + pub struct ip_mreqn { + pub imr_multiaddr: in_addr, + pub imr_address: in_addr, + pub imr_ifindex: ::c_int, + } + pub struct ipv6_mreq { pub ipv6mr_multiaddr: in6_addr, pub ipv6mr_interface: ::c_uint, @@ -871,6 +877,11 @@ s! { pub c_ispeed: ::speed_t, pub c_ospeed: ::speed_t, } + + pub struct in6_pktinfo { + pub ipi6_addr: ::in6_addr, + pub ipi6_ifindex: ::c_uint, + } } s_no_extra_traits! { @@ -898,8 +909,8 @@ s_no_extra_traits! { pub struct sockaddr_storage { pub ss_family: sa_family_t, + __ss_pad2: [u8; 128 - 2 - 8], __ss_align: ::size_t, - __ss_pad2: [u8; 128 - 2 * 8], } pub struct utsname { @@ -1307,6 +1318,7 @@ pub const SIG_DFL: sighandler_t = 0 as sighandler_t; pub const SIG_IGN: sighandler_t = 1 as sighandler_t; pub const SIG_ERR: sighandler_t = !0 as sighandler_t; +pub const DT_UNKNOWN: u8 = 0; pub const DT_FIFO: u8 = 1; pub const DT_CHR: u8 = 2; pub const DT_DIR: u8 = 4; @@ -1327,6 +1339,7 @@ pub const S_ISGID: ::c_int = 0x400; pub const S_ISVTX: ::c_int = 0x200; pub const IF_NAMESIZE: ::size_t = 16; +pub const IFNAMSIZ: ::size_t = IF_NAMESIZE; pub const LOG_EMERG: ::c_int = 0; pub const LOG_ALERT: ::c_int = 1; @@ -1454,8 +1467,6 @@ pub const O_RDONLY: ::c_int = 0; pub const O_WRONLY: ::c_int = 1; pub const O_RDWR: ::c_int = 2; -pub const SOCK_CLOEXEC: ::c_int = O_CLOEXEC; - pub const S_IFIFO: ::mode_t = 4096; pub const S_IFCHR: ::mode_t = 8192; pub const S_IFBLK: ::mode_t = 24576; @@ -1762,14 +1773,19 @@ pub const SCM_TIMESTAMP: ::c_int = SO_TIMESTAMP; pub const SOCK_RAW: ::c_int = 3; pub const SOCK_RDM: ::c_int = 4; + +pub const IP_TOS: ::c_int = 1; +pub const IP_TTL: ::c_int = 2; +pub const IP_HDRINCL: ::c_int = 3; +pub const IP_RECVTOS: ::c_int = 13; +pub const IP_FREEBIND: ::c_int = 15; +pub const IP_TRANSPARENT: ::c_int = 19; pub const IP_MULTICAST_IF: ::c_int = 32; pub const IP_MULTICAST_TTL: ::c_int = 33; pub const IP_MULTICAST_LOOP: ::c_int = 34; -pub const IP_TTL: ::c_int = 2; -pub const IP_HDRINCL: ::c_int = 3; pub const IP_ADD_MEMBERSHIP: ::c_int = 35; pub const IP_DROP_MEMBERSHIP: ::c_int = 36; -pub const IP_TRANSPARENT: ::c_int = 19; + pub const IPV6_UNICAST_HOPS: ::c_int = 16; pub const IPV6_MULTICAST_IF: ::c_int = 17; pub const IPV6_MULTICAST_HOPS: ::c_int = 18; @@ -1777,6 +1793,9 @@ pub const IPV6_MULTICAST_LOOP: ::c_int = 19; pub const IPV6_ADD_MEMBERSHIP: ::c_int = 20; pub const IPV6_DROP_MEMBERSHIP: ::c_int = 21; pub const IPV6_V6ONLY: ::c_int = 26; +pub const IPV6_RECVPKTINFO: ::c_int = 49; +pub const IPV6_RECVTCLASS: ::c_int = 66; +pub const IPV6_TCLASS: ::c_int = 67; pub const TCP_NODELAY: ::c_int = 1; pub const TCP_MAXSEG: ::c_int = 2; @@ -1980,6 +1999,13 @@ pub const PIPE_BUF: usize = 4096; pub const SI_LOAD_SHIFT: ::c_uint = 16; +pub const CLD_EXITED: ::c_int = 1; +pub const CLD_KILLED: ::c_int = 2; +pub const CLD_DUMPED: ::c_int = 3; +pub const CLD_TRAPPED: ::c_int = 4; +pub const CLD_STOPPED: ::c_int = 5; +pub const CLD_CONTINUED: ::c_int = 6; + pub const SIGEV_SIGNAL: ::c_int = 0; pub const SIGEV_NONE: ::c_int = 1; pub const SIGEV_THREAD: ::c_int = 2; @@ -2651,6 +2677,9 @@ pub const PT_GNU_EH_FRAME: u32 = 0x6474e550; pub const PT_GNU_STACK: u32 = 0x6474e551; pub const PT_GNU_RELRO: u32 = 0x6474e552; +// Ethernet protocol IDs. +pub const ETH_P_IP: ::c_int = 0x0800; + pub const SFD_CLOEXEC: ::c_int = 0x080000; pub const NCCS: usize = 32; @@ -2702,6 +2731,7 @@ pub const POSIX_MADV_DONTNEED: ::c_int = 4; pub const RLIM_INFINITY: ::rlim_t = !0; pub const RLIMIT_RTTIME: ::c_int = 15; pub const RLIMIT_NLIMITS: ::c_int = 16; +pub const RLIM_NLIMITS: ::c_int = RLIMIT_NLIMITS; pub const MAP_ANONYMOUS: ::c_int = MAP_ANON; @@ -2891,7 +2921,8 @@ pub const O_SYNC: ::c_int = 0x00000040 | O_DSYNC; pub const O_RSYNC: ::c_int = O_SYNC; pub const O_DSYNC: ::c_int = 0x00000020; -pub const SOCK_NONBLOCK: ::c_int = 2048; +pub const SOCK_CLOEXEC: ::c_int = 0o2000000; +pub const SOCK_NONBLOCK: ::c_int = 0o4000; pub const MAP_ANON: ::c_int = 0x0020; pub const MAP_GROWSDOWN: ::c_int = 0x0100; @@ -3139,7 +3170,7 @@ f! { return } - pub fn FD_ISSET(fd: ::c_int, set: *mut fd_set) -> bool { + pub fn FD_ISSET(fd: ::c_int, set: *const fd_set) -> bool { let fd = fd as usize; let size = ::mem::size_of_val(&(*set).fds_bits[0]) * 8; return ((*set).fds_bits[fd / size] & (1 << (fd % size))) != 0 @@ -3158,42 +3189,6 @@ f! { } } - pub fn WIFSTOPPED(status: ::c_int) -> bool { - (status & 0xff) == 0x7f - } - - pub fn WSTOPSIG(status: ::c_int) -> ::c_int { - (status >> 8) & 0xff - } - - pub fn WIFCONTINUED(status: ::c_int) -> bool { - status == 0xffff - } - - pub fn WIFSIGNALED(status: ::c_int) -> bool { - ((status & 0x7f) + 1) as i8 >= 2 - } - - pub fn WTERMSIG(status: ::c_int) -> ::c_int { - status & 0x7f - } - - pub fn WIFEXITED(status: ::c_int) -> bool { - (status & 0x7f) == 0 - } - - pub fn WEXITSTATUS(status: ::c_int) -> ::c_int { - (status >> 8) & 0xff - } - - pub fn WCOREDUMP(status: ::c_int) -> bool { - (status & 0x80) != 0 - } - - pub fn QCMD(cmd: ::c_int, type_: ::c_int) -> ::c_int { - (cmd << 8) | (type_ & 0x00ff) - } - pub fn CPU_ZERO(cpuset: &mut cpu_set_t) -> () { for slot in cpuset.bits.iter_mut() { *slot = 0; @@ -3240,7 +3235,84 @@ f! { minor as ::c_uint } - pub fn makedev(major: ::c_uint, minor: ::c_uint) -> ::dev_t { + pub fn CMSG_DATA(cmsg: *const cmsghdr) -> *mut c_uchar { + cmsg.offset(1) as *mut c_uchar + } + + pub fn CMSG_NXTHDR(mhdr: *const msghdr, cmsg: *const cmsghdr) + -> *mut cmsghdr + { + if ((*cmsg).cmsg_len as ::size_t) < ::mem::size_of::() { + 0 as *mut cmsghdr + } else if __CMSG_NEXT(cmsg).add(::mem::size_of::()) + >= __MHDR_END(mhdr) { + 0 as *mut cmsghdr + } else { + __CMSG_NEXT(cmsg).cast() + } + } + + pub fn CMSG_FIRSTHDR(mhdr: *const msghdr) -> *mut cmsghdr { + if (*mhdr).msg_controllen as ::size_t >= ::mem::size_of::() { + (*mhdr).msg_control.cast() + } else { + 0 as *mut cmsghdr + } + } + + pub {const} fn CMSG_ALIGN(len: ::size_t) -> ::size_t { + (len + ::mem::size_of::<::size_t>() - 1) + & !(::mem::size_of::<::size_t>() - 1) + } + + pub {const} fn CMSG_SPACE(len: ::c_uint) -> ::c_uint { + (CMSG_ALIGN(len as ::size_t) + CMSG_ALIGN(::mem::size_of::())) + as ::c_uint + } + + pub fn CMSG_LEN(len: ::c_uint) -> ::c_uint { + (CMSG_ALIGN(::mem::size_of::()) + len as ::size_t) as ::c_uint + } +} + +safe_f! { + pub {const} fn WIFSTOPPED(status: ::c_int) -> bool { + (status & 0xff) == 0x7f + } + + pub {const} fn WSTOPSIG(status: ::c_int) -> ::c_int { + (status >> 8) & 0xff + } + + pub {const} fn WIFCONTINUED(status: ::c_int) -> bool { + status == 0xffff + } + + pub {const} fn WIFSIGNALED(status: ::c_int) -> bool { + ((status & 0x7f) + 1) as i8 >= 2 + } + + pub {const} fn WTERMSIG(status: ::c_int) -> ::c_int { + status & 0x7f + } + + pub {const} fn WIFEXITED(status: ::c_int) -> bool { + (status & 0x7f) == 0 + } + + pub {const} fn WEXITSTATUS(status: ::c_int) -> ::c_int { + (status >> 8) & 0xff + } + + pub {const} fn WCOREDUMP(status: ::c_int) -> bool { + (status & 0x80) != 0 + } + + pub {const} fn QCMD(cmd: ::c_int, type_: ::c_int) -> ::c_int { + (cmd << 8) | (type_ & 0x00ff) + } + + pub {const} fn makedev(major: ::c_uint, minor: ::c_uint) -> ::dev_t { let major = major as ::dev_t; let minor = minor as ::dev_t; let mut dev = 0; @@ -3252,6 +3324,19 @@ f! { } } +fn __CMSG_LEN(cmsg: *const cmsghdr) -> ::ssize_t { + ((unsafe { (*cmsg).cmsg_len as ::size_t } + ::mem::size_of::<::c_long>() - 1) + & !(::mem::size_of::<::c_long>() - 1)) as ::ssize_t +} + +fn __CMSG_NEXT(cmsg: *const cmsghdr) -> *mut c_uchar { + (unsafe { cmsg.offset(__CMSG_LEN(cmsg)) }) as *mut c_uchar +} + +fn __MHDR_END(mhdr: *const msghdr) -> *mut c_uchar { + unsafe { (*mhdr).msg_control.offset((*mhdr).msg_controllen as isize) }.cast() +} + // EXTERN_FN #[link(name = "c")] @@ -3291,44 +3376,24 @@ extern "C" { pub fn tolower(c: c_int) -> c_int; pub fn toupper(c: c_int) -> c_int; pub fn fopen(filename: *const c_char, mode: *const c_char) -> *mut FILE; - pub fn freopen( - filename: *const c_char, - mode: *const c_char, - file: *mut FILE, - ) -> *mut FILE; + pub fn freopen(filename: *const c_char, mode: *const c_char, file: *mut FILE) -> *mut FILE; pub fn fflush(file: *mut FILE) -> c_int; pub fn fclose(file: *mut FILE) -> c_int; pub fn remove(filename: *const c_char) -> c_int; pub fn rename(oldname: *const c_char, newname: *const c_char) -> c_int; pub fn tmpfile() -> *mut FILE; - pub fn setvbuf( - stream: *mut FILE, - buffer: *mut c_char, - mode: c_int, - size: size_t, - ) -> c_int; + pub fn setvbuf(stream: *mut FILE, buffer: *mut c_char, mode: c_int, size: size_t) -> c_int; pub fn setbuf(stream: *mut FILE, buf: *mut c_char); pub fn getchar() -> c_int; pub fn putchar(c: c_int) -> c_int; pub fn fgetc(stream: *mut FILE) -> c_int; - pub fn fgets(buf: *mut c_char, n: c_int, stream: *mut FILE) - -> *mut c_char; + pub fn fgets(buf: *mut c_char, n: c_int, stream: *mut FILE) -> *mut c_char; pub fn fputc(c: c_int, stream: *mut FILE) -> c_int; pub fn fputs(s: *const c_char, stream: *mut FILE) -> c_int; pub fn puts(s: *const c_char) -> c_int; pub fn ungetc(c: c_int, stream: *mut FILE) -> c_int; - pub fn fread( - ptr: *mut c_void, - size: size_t, - nobj: size_t, - stream: *mut FILE, - ) -> size_t; - pub fn fwrite( - ptr: *const c_void, - size: size_t, - nobj: size_t, - stream: *mut FILE, - ) -> size_t; + pub fn fread(ptr: *mut c_void, size: size_t, nobj: size_t, stream: *mut FILE) -> size_t; + pub fn fwrite(ptr: *const c_void, size: size_t, nobj: size_t, stream: *mut FILE) -> size_t; pub fn fseek(stream: *mut FILE, offset: c_long, whence: c_int) -> c_int; pub fn ftell(stream: *mut FILE) -> c_long; pub fn rewind(stream: *mut FILE); @@ -3339,16 +3404,9 @@ extern "C" { pub fn perror(s: *const c_char); pub fn atoi(s: *const c_char) -> c_int; pub fn strtod(s: *const c_char, endp: *mut *mut c_char) -> c_double; - pub fn strtol( - s: *const c_char, - endp: *mut *mut c_char, - base: c_int, - ) -> c_long; - pub fn strtoul( - s: *const c_char, - endp: *mut *mut c_char, - base: c_int, - ) -> c_ulong; + pub fn strtof(s: *const c_char, endp: *mut *mut c_char) -> c_float; + pub fn strtol(s: *const c_char, endp: *mut *mut c_char, base: c_int) -> c_long; + pub fn strtoul(s: *const c_char, endp: *mut *mut c_char, base: c_int) -> c_ulong; pub fn calloc(nobj: size_t, size: size_t) -> *mut c_void; pub fn malloc(size: size_t) -> *mut c_void; pub fn realloc(p: *mut c_void, size: size_t) -> *mut c_void; @@ -3361,17 +3419,9 @@ extern "C" { pub fn getenv(s: *const c_char) -> *mut c_char; pub fn strcpy(dst: *mut c_char, src: *const c_char) -> *mut c_char; - pub fn strncpy( - dst: *mut c_char, - src: *const c_char, - n: size_t, - ) -> *mut c_char; + pub fn strncpy(dst: *mut c_char, src: *const c_char, n: size_t) -> *mut c_char; pub fn strcat(s: *mut c_char, ct: *const c_char) -> *mut c_char; - pub fn strncat( - s: *mut c_char, - ct: *const c_char, - n: size_t, - ) -> *mut c_char; + pub fn strncat(s: *mut c_char, ct: *const c_char, n: size_t) -> *mut c_char; pub fn strcmp(cs: *const c_char, ct: *const c_char) -> c_int; pub fn strncmp(cs: *const c_char, ct: *const c_char, n: size_t) -> c_int; pub fn strcoll(cs: *const c_char, ct: *const c_char) -> c_int; @@ -3388,25 +3438,13 @@ extern "C" { pub fn strtok(s: *mut c_char, t: *const c_char) -> *mut c_char; pub fn strxfrm(s: *mut c_char, ct: *const c_char, n: size_t) -> size_t; pub fn wcslen(buf: *const wchar_t) -> size_t; - pub fn wcstombs( - dest: *mut c_char, - src: *const wchar_t, - n: size_t, - ) -> ::size_t; + pub fn wcstombs(dest: *mut c_char, src: *const wchar_t, n: size_t) -> ::size_t; pub fn memchr(cx: *const c_void, c: c_int, n: size_t) -> *mut c_void; pub fn wmemchr(cx: *const wchar_t, c: wchar_t, n: size_t) -> *mut wchar_t; pub fn memcmp(cx: *const c_void, ct: *const c_void, n: size_t) -> c_int; - pub fn memcpy( - dest: *mut c_void, - src: *const c_void, - n: size_t, - ) -> *mut c_void; - pub fn memmove( - dest: *mut c_void, - src: *const c_void, - n: size_t, - ) -> *mut c_void; + pub fn memcpy(dest: *mut c_void, src: *const c_void, n: size_t) -> *mut c_void; + pub fn memmove(dest: *mut c_void, src: *const c_void, n: size_t) -> *mut c_void; pub fn memset(dest: *mut c_void, c: c_int, n: size_t) -> *mut c_void; pub fn abs(i: c_int) -> c_int; @@ -3418,42 +3456,20 @@ extern "C" { pub fn getpwnam(name: *const ::c_char) -> *mut passwd; pub fn getpwuid(uid: ::uid_t) -> *mut passwd; - pub fn fprintf( - stream: *mut ::FILE, - format: *const ::c_char, - ... - ) -> ::c_int; + pub fn fprintf(stream: *mut ::FILE, format: *const ::c_char, ...) -> ::c_int; pub fn printf(format: *const ::c_char, ...) -> ::c_int; - pub fn snprintf( - s: *mut ::c_char, - n: ::size_t, - format: *const ::c_char, - ... - ) -> ::c_int; + pub fn snprintf(s: *mut ::c_char, n: ::size_t, format: *const ::c_char, ...) -> ::c_int; pub fn sprintf(s: *mut ::c_char, format: *const ::c_char, ...) -> ::c_int; - pub fn fscanf( - stream: *mut ::FILE, - format: *const ::c_char, - ... - ) -> ::c_int; + pub fn fscanf(stream: *mut ::FILE, format: *const ::c_char, ...) -> ::c_int; pub fn scanf(format: *const ::c_char, ...) -> ::c_int; - pub fn sscanf(s: *const ::c_char, format: *const ::c_char, ...) - -> ::c_int; + pub fn sscanf(s: *const ::c_char, format: *const ::c_char, ...) -> ::c_int; pub fn getchar_unlocked() -> ::c_int; pub fn putchar_unlocked(c: ::c_int) -> ::c_int; pub fn socket(domain: ::c_int, ty: ::c_int, protocol: ::c_int) -> ::c_int; - pub fn connect( - socket: ::c_int, - address: *const sockaddr, - len: socklen_t, - ) -> ::c_int; + pub fn connect(socket: ::c_int, address: *const sockaddr, len: socklen_t) -> ::c_int; pub fn listen(socket: ::c_int, backlog: ::c_int) -> ::c_int; - pub fn accept( - socket: ::c_int, - address: *mut sockaddr, - address_len: *mut socklen_t, - ) -> ::c_int; + pub fn accept(socket: ::c_int, address: *mut sockaddr, address_len: *mut socklen_t) -> ::c_int; pub fn getpeername( socket: ::c_int, address: *mut sockaddr, @@ -3506,20 +3522,12 @@ extern "C" { pub fn opendir(dirname: *const c_char) -> *mut ::DIR; pub fn readdir(dirp: *mut ::DIR) -> *mut ::dirent; - pub fn readdir_r( - dirp: *mut ::DIR, - entry: *mut ::dirent, - result: *mut *mut ::dirent, - ) -> ::c_int; + pub fn readdir_r(dirp: *mut ::DIR, entry: *mut ::dirent, result: *mut *mut ::dirent) + -> ::c_int; pub fn closedir(dirp: *mut ::DIR) -> ::c_int; pub fn rewinddir(dirp: *mut ::DIR); - pub fn openat( - dirfd: ::c_int, - pathname: *const ::c_char, - flags: ::c_int, - ... - ) -> ::c_int; + pub fn openat(dirfd: ::c_int, pathname: *const ::c_char, flags: ::c_int, ...) -> ::c_int; pub fn fchmodat( dirfd: ::c_int, pathname: *const ::c_char, @@ -3547,11 +3555,7 @@ extern "C" { newpath: *const ::c_char, flags: ::c_int, ) -> ::c_int; - pub fn mkdirat( - dirfd: ::c_int, - pathname: *const ::c_char, - mode: ::mode_t, - ) -> ::c_int; + pub fn mkdirat(dirfd: ::c_int, pathname: *const ::c_char, mode: ::mode_t) -> ::c_int; pub fn readlinkat( dirfd: ::c_int, pathname: *const ::c_char, @@ -3569,11 +3573,7 @@ extern "C" { newdirfd: ::c_int, linkpath: *const ::c_char, ) -> ::c_int; - pub fn unlinkat( - dirfd: ::c_int, - pathname: *const ::c_char, - flags: ::c_int, - ) -> ::c_int; + pub fn unlinkat(dirfd: ::c_int, pathname: *const ::c_char, flags: ::c_int) -> ::c_int; pub fn access(path: *const c_char, amode: ::c_int) -> ::c_int; pub fn alarm(seconds: ::c_uint) -> ::c_uint; @@ -3584,16 +3584,8 @@ extern "C" { pub fn dup(fd: ::c_int) -> ::c_int; pub fn dup2(src: ::c_int, dst: ::c_int) -> ::c_int; pub fn execl(path: *const c_char, arg0: *const c_char, ...) -> ::c_int; - pub fn execle( - path: *const ::c_char, - arg0: *const ::c_char, - ... - ) -> ::c_int; - pub fn execlp( - file: *const ::c_char, - arg0: *const ::c_char, - ... - ) -> ::c_int; + pub fn execle(path: *const ::c_char, arg0: *const ::c_char, ...) -> ::c_int; + pub fn execlp(file: *const ::c_char, arg0: *const ::c_char, ...) -> ::c_int; pub fn execv(prog: *const c_char, argv: *const *const c_char) -> ::c_int; pub fn execve( prog: *const c_char, @@ -3609,11 +3601,7 @@ extern "C" { pub fn getgid() -> gid_t; pub fn getgroups(ngroups_max: ::c_int, groups: *mut gid_t) -> ::c_int; pub fn getlogin() -> *mut c_char; - pub fn getopt( - argc: ::c_int, - argv: *const *mut c_char, - optstr: *const c_char, - ) -> ::c_int; + pub fn getopt(argc: ::c_int, argv: *const *mut c_char, optstr: *const c_char) -> ::c_int; pub fn getpgid(pid: pid_t) -> pid_t; pub fn getpgrp() -> pid_t; pub fn getpid() -> pid_t; @@ -3625,13 +3613,8 @@ extern "C" { pub fn pathconf(path: *const c_char, name: ::c_int) -> c_long; pub fn pause() -> ::c_int; pub fn pipe(fds: *mut ::c_int) -> ::c_int; - pub fn posix_memalign( - memptr: *mut *mut ::c_void, - align: ::size_t, - size: ::size_t, - ) -> ::c_int; - pub fn read(fd: ::c_int, buf: *mut ::c_void, count: ::size_t) - -> ::ssize_t; + pub fn posix_memalign(memptr: *mut *mut ::c_void, align: ::size_t, size: ::size_t) -> ::c_int; + pub fn read(fd: ::c_int, buf: *mut ::c_void, count: ::size_t) -> ::ssize_t; pub fn rmdir(path: *const c_char) -> ::c_int; pub fn seteuid(uid: uid_t) -> ::c_int; pub fn setegid(gid: gid_t) -> ::c_int; @@ -3646,28 +3629,10 @@ extern "C" { pub fn ttyname(fd: ::c_int) -> *mut c_char; pub fn unlink(c: *const c_char) -> ::c_int; pub fn wait(status: *mut ::c_int) -> pid_t; - pub fn waitpid( - pid: pid_t, - status: *mut ::c_int, - options: ::c_int, - ) -> pid_t; - pub fn write( - fd: ::c_int, - buf: *const ::c_void, - count: ::size_t, - ) -> ::ssize_t; - pub fn pread( - fd: ::c_int, - buf: *mut ::c_void, - count: ::size_t, - offset: off_t, - ) -> ::ssize_t; - pub fn pwrite( - fd: ::c_int, - buf: *const ::c_void, - count: ::size_t, - offset: off_t, - ) -> ::ssize_t; + pub fn waitpid(pid: pid_t, status: *mut ::c_int, options: ::c_int) -> pid_t; + pub fn write(fd: ::c_int, buf: *const ::c_void, count: ::size_t) -> ::ssize_t; + pub fn pread(fd: ::c_int, buf: *mut ::c_void, count: ::size_t, offset: off_t) -> ::ssize_t; + pub fn pwrite(fd: ::c_int, buf: *const ::c_void, count: ::size_t, offset: off_t) -> ::ssize_t; pub fn umask(mask: mode_t) -> mode_t; pub fn utime(file: *const c_char, buf: *const utimbuf) -> ::c_int; @@ -3690,20 +3655,13 @@ extern "C" { pub fn munmap(addr: *mut ::c_void, len: ::size_t) -> ::c_int; pub fn if_nametoindex(ifname: *const c_char) -> ::c_uint; - pub fn if_indextoname( - ifindex: ::c_uint, - ifname: *mut ::c_char, - ) -> *mut ::c_char; + pub fn if_indextoname(ifindex: ::c_uint, ifname: *mut ::c_char) -> *mut ::c_char; pub fn lstat(path: *const c_char, buf: *mut stat) -> ::c_int; pub fn fsync(fd: ::c_int) -> ::c_int; - pub fn setenv( - name: *const c_char, - val: *const c_char, - overwrite: ::c_int, - ) -> ::c_int; + pub fn setenv(name: *const c_char, val: *const c_char, overwrite: ::c_int) -> ::c_int; pub fn unsetenv(name: *const c_char) -> ::c_int; pub fn symlink(path1: *const c_char, path2: *const c_char) -> ::c_int; @@ -3712,10 +3670,7 @@ extern "C" { pub fn signal(signum: ::c_int, handler: sighandler_t) -> sighandler_t; - pub fn realpath( - pathname: *const ::c_char, - resolved: *mut ::c_char, - ) -> *mut ::c_char; + pub fn realpath(pathname: *const ::c_char, resolved: *mut ::c_char) -> *mut ::c_char; pub fn flock(fd: ::c_int, operation: ::c_int) -> ::c_int; @@ -3723,21 +3678,12 @@ extern "C" { pub fn times(buf: *mut ::tms) -> ::clock_t; pub fn pthread_self() -> ::pthread_t; - pub fn pthread_join( - native: ::pthread_t, - value: *mut *mut ::c_void, - ) -> ::c_int; + pub fn pthread_join(native: ::pthread_t, value: *mut *mut ::c_void) -> ::c_int; pub fn pthread_exit(value: *mut ::c_void) -> !; pub fn pthread_attr_init(attr: *mut ::pthread_attr_t) -> ::c_int; pub fn pthread_attr_destroy(attr: *mut ::pthread_attr_t) -> ::c_int; - pub fn pthread_attr_setstacksize( - attr: *mut ::pthread_attr_t, - stack_size: ::size_t, - ) -> ::c_int; - pub fn pthread_attr_setdetachstate( - attr: *mut ::pthread_attr_t, - state: ::c_int, - ) -> ::c_int; + pub fn pthread_attr_setstacksize(attr: *mut ::pthread_attr_t, stack_size: ::size_t) -> ::c_int; + pub fn pthread_attr_setdetachstate(attr: *mut ::pthread_attr_t, state: ::c_int) -> ::c_int; pub fn pthread_detach(thread: ::pthread_t) -> ::c_int; pub fn sched_yield() -> ::c_int; pub fn pthread_key_create( @@ -3746,10 +3692,7 @@ extern "C" { ) -> ::c_int; pub fn pthread_key_delete(key: pthread_key_t) -> ::c_int; pub fn pthread_getspecific(key: pthread_key_t) -> *mut ::c_void; - pub fn pthread_setspecific( - key: pthread_key_t, - value: *const ::c_void, - ) -> ::c_int; + pub fn pthread_setspecific(key: pthread_key_t, value: *const ::c_void) -> ::c_int; pub fn pthread_mutex_init( lock: *mut pthread_mutex_t, attr: *const pthread_mutexattr_t, @@ -3760,22 +3703,12 @@ extern "C" { pub fn pthread_mutex_unlock(lock: *mut pthread_mutex_t) -> ::c_int; pub fn pthread_mutexattr_init(attr: *mut pthread_mutexattr_t) -> ::c_int; - pub fn pthread_mutexattr_destroy( - attr: *mut pthread_mutexattr_t, - ) -> ::c_int; - pub fn pthread_mutexattr_settype( - attr: *mut pthread_mutexattr_t, - _type: ::c_int, - ) -> ::c_int; + pub fn pthread_mutexattr_destroy(attr: *mut pthread_mutexattr_t) -> ::c_int; + pub fn pthread_mutexattr_settype(attr: *mut pthread_mutexattr_t, _type: ::c_int) -> ::c_int; - pub fn pthread_cond_init( - cond: *mut pthread_cond_t, - attr: *const pthread_condattr_t, - ) -> ::c_int; - pub fn pthread_cond_wait( - cond: *mut pthread_cond_t, - lock: *mut pthread_mutex_t, - ) -> ::c_int; + pub fn pthread_cond_init(cond: *mut pthread_cond_t, attr: *const pthread_condattr_t) + -> ::c_int; + pub fn pthread_cond_wait(cond: *mut pthread_cond_t, lock: *mut pthread_mutex_t) -> ::c_int; pub fn pthread_cond_timedwait( cond: *mut pthread_cond_t, lock: *mut pthread_mutex_t, @@ -3796,16 +3729,9 @@ extern "C" { pub fn pthread_rwlock_wrlock(lock: *mut pthread_rwlock_t) -> ::c_int; pub fn pthread_rwlock_trywrlock(lock: *mut pthread_rwlock_t) -> ::c_int; pub fn pthread_rwlock_unlock(lock: *mut pthread_rwlock_t) -> ::c_int; - pub fn pthread_rwlockattr_init(attr: *mut pthread_rwlockattr_t) - -> ::c_int; - pub fn pthread_rwlockattr_destroy( - attr: *mut pthread_rwlockattr_t, - ) -> ::c_int; - pub fn strerror_r( - errnum: ::c_int, - buf: *mut c_char, - buflen: ::size_t, - ) -> ::c_int; + pub fn pthread_rwlockattr_init(attr: *mut pthread_rwlockattr_t) -> ::c_int; + pub fn pthread_rwlockattr_destroy(attr: *mut pthread_rwlockattr_t) -> ::c_int; + pub fn strerror_r(errnum: ::c_int, buf: *mut c_char, buflen: ::size_t) -> ::c_int; pub fn getsockopt( sockfd: ::c_int, @@ -3815,22 +3741,12 @@ extern "C" { optlen: *mut ::socklen_t, ) -> ::c_int; pub fn raise(signum: ::c_int) -> ::c_int; - pub fn sigaction( - signum: ::c_int, - act: *const sigaction, - oldact: *mut sigaction, - ) -> ::c_int; + pub fn sigaction(signum: ::c_int, act: *const sigaction, oldact: *mut sigaction) -> ::c_int; - pub fn utimes( - filename: *const ::c_char, - times: *const ::timeval, - ) -> ::c_int; + pub fn utimes(filename: *const ::c_char, times: *const ::timeval) -> ::c_int; pub fn dlopen(filename: *const ::c_char, flag: ::c_int) -> *mut ::c_void; pub fn dlerror() -> *mut ::c_char; - pub fn dlsym( - handle: *mut ::c_void, - symbol: *const ::c_char, - ) -> *mut ::c_void; + pub fn dlsym(handle: *mut ::c_void, symbol: *const ::c_char) -> *mut ::c_void; pub fn dlclose(handle: *mut ::c_void) -> ::c_int; pub fn dladdr(addr: *const ::c_void, info: *mut Dl_info) -> ::c_int; @@ -3851,64 +3767,36 @@ extern "C" { pub fn gmtime(time_p: *const time_t) -> *mut tm; pub fn localtime(time_p: *const time_t) -> *mut tm; - pub fn mknod( - pathname: *const ::c_char, - mode: ::mode_t, - dev: ::dev_t, - ) -> ::c_int; + pub fn mknod(pathname: *const ::c_char, mode: ::mode_t, dev: ::dev_t) -> ::c_int; pub fn uname(buf: *mut ::utsname) -> ::c_int; pub fn gethostname(name: *mut ::c_char, len: ::size_t) -> ::c_int; - pub fn getservbyname( - name: *const ::c_char, - proto: *const ::c_char, - ) -> *mut servent; + pub fn getservbyname(name: *const ::c_char, proto: *const ::c_char) -> *mut servent; pub fn getprotobyname(name: *const ::c_char) -> *mut protoent; pub fn getprotobynumber(proto: ::c_int) -> *mut protoent; pub fn usleep(secs: ::c_uint) -> ::c_int; - pub fn send( - socket: ::c_int, - buf: *const ::c_void, - len: ::size_t, - flags: ::c_int, - ) -> ::ssize_t; - pub fn recv( - socket: ::c_int, - buf: *mut ::c_void, - len: ::size_t, - flags: ::c_int, - ) -> ::ssize_t; + pub fn send(socket: ::c_int, buf: *const ::c_void, len: ::size_t, flags: ::c_int) -> ::ssize_t; + pub fn recv(socket: ::c_int, buf: *mut ::c_void, len: ::size_t, flags: ::c_int) -> ::ssize_t; pub fn putenv(string: *mut c_char) -> ::c_int; pub fn poll(fds: *mut pollfd, nfds: nfds_t, timeout: ::c_int) -> ::c_int; pub fn select( nfds: ::c_int, - readfs: *mut fd_set, + readfds: *mut fd_set, writefds: *mut fd_set, errorfds: *mut fd_set, timeout: *mut timeval, ) -> ::c_int; - pub fn setlocale( - category: ::c_int, - locale: *const ::c_char, - ) -> *mut ::c_char; + pub fn setlocale(category: ::c_int, locale: *const ::c_char) -> *mut ::c_char; pub fn localeconv() -> *mut lconv; pub fn sem_destroy(sem: *mut sem_t) -> ::c_int; pub fn sem_wait(sem: *mut sem_t) -> ::c_int; pub fn sem_trywait(sem: *mut sem_t) -> ::c_int; pub fn sem_post(sem: *mut sem_t) -> ::c_int; - pub fn sem_init( - sem: *mut sem_t, - pshared: ::c_int, - value: ::c_uint, - ) -> ::c_int; + pub fn sem_init(sem: *mut sem_t, pshared: ::c_int, value: ::c_uint) -> ::c_int; pub fn statvfs(path: *const c_char, buf: *mut statvfs) -> ::c_int; pub fn fstatvfs(fd: ::c_int, buf: *mut statvfs) -> ::c_int; - pub fn readlink( - path: *const c_char, - buf: *mut c_char, - bufsz: ::size_t, - ) -> ::ssize_t; + pub fn readlink(path: *const c_char, buf: *mut c_char, bufsz: ::size_t) -> ::ssize_t; pub fn sigemptyset(set: *mut sigset_t) -> ::c_int; pub fn sigaddset(set: *mut sigset_t, signum: ::c_int) -> ::c_int; @@ -3916,11 +3804,7 @@ extern "C" { pub fn sigdelset(set: *mut sigset_t, signum: ::c_int) -> ::c_int; pub fn sigismember(set: *const sigset_t, signum: ::c_int) -> ::c_int; - pub fn sigprocmask( - how: ::c_int, - set: *const sigset_t, - oldset: *mut sigset_t, - ) -> ::c_int; + pub fn sigprocmask(how: ::c_int, set: *const sigset_t, oldset: *mut sigset_t) -> ::c_int; pub fn sigpending(set: *mut sigset_t) -> ::c_int; pub fn timegm(tm: *mut ::tm) -> time_t; @@ -3933,17 +3817,13 @@ extern "C" { pub fn pselect( nfds: ::c_int, - readfs: *mut fd_set, + readfds: *mut fd_set, writefds: *mut fd_set, errorfds: *mut fd_set, timeout: *const timespec, sigmask: *const sigset_t, ) -> ::c_int; - pub fn fseeko( - stream: *mut ::FILE, - offset: ::off_t, - whence: ::c_int, - ) -> ::c_int; + pub fn fseeko(stream: *mut ::FILE, offset: ::off_t, whence: ::c_int) -> ::c_int; pub fn ftello(stream: *mut ::FILE) -> ::off_t; pub fn tcdrain(fd: ::c_int) -> ::c_int; pub fn cfgetispeed(termios: *const ::termios) -> ::speed_t; @@ -3953,11 +3833,7 @@ extern "C" { pub fn cfsetospeed(termios: *mut ::termios, speed: ::speed_t) -> ::c_int; pub fn cfsetspeed(termios: *mut ::termios, speed: ::speed_t) -> ::c_int; pub fn tcgetattr(fd: ::c_int, termios: *mut ::termios) -> ::c_int; - pub fn tcsetattr( - fd: ::c_int, - optional_actions: ::c_int, - termios: *const ::termios, - ) -> ::c_int; + pub fn tcsetattr(fd: ::c_int, optional_actions: ::c_int, termios: *const ::termios) -> ::c_int; pub fn tcflow(fd: ::c_int, action: ::c_int) -> ::c_int; pub fn tcflush(fd: ::c_int, action: ::c_int) -> ::c_int; pub fn tcgetsid(fd: ::c_int) -> ::pid_t; @@ -3980,16 +3856,10 @@ extern "C" { pub fn fdatasync(fd: ::c_int) -> ::c_int; pub fn clock_getres(clk_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; pub fn clock_gettime(clk_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; - pub fn clock_settime( - clk_id: ::clockid_t, - tp: *const ::timespec, - ) -> ::c_int; + pub fn clock_settime(clk_id: ::clockid_t, tp: *const ::timespec) -> ::c_int; pub fn dirfd(dirp: *mut ::DIR) -> ::c_int; - pub fn pthread_getattr_np( - native: ::pthread_t, - attr: *mut ::pthread_attr_t, - ) -> ::c_int; + pub fn pthread_getattr_np(native: ::pthread_t, attr: *mut ::pthread_attr_t) -> ::c_int; pub fn pthread_attr_getstack( attr: *const ::pthread_attr_t, stackaddr: *mut *mut ::c_void, @@ -4000,18 +3870,9 @@ extern "C" { pub fn pipe2(fds: *mut ::c_int, flags: ::c_int) -> ::c_int; pub fn statfs(path: *const ::c_char, buf: *mut statfs) -> ::c_int; pub fn fstatfs(fd: ::c_int, buf: *mut statfs) -> ::c_int; - pub fn memrchr( - cx: *const ::c_void, - c: ::c_int, - n: ::size_t, - ) -> *mut ::c_void; + pub fn memrchr(cx: *const ::c_void, c: ::c_int, n: ::size_t) -> *mut ::c_void; - pub fn posix_fadvise( - fd: ::c_int, - offset: ::off_t, - len: ::off_t, - advise: ::c_int, - ) -> ::c_int; + pub fn posix_fadvise(fd: ::c_int, offset: ::off_t, len: ::off_t, advise: ::c_int) -> ::c_int; pub fn futimens(fd: ::c_int, times: *const ::timespec) -> ::c_int; pub fn utimensat( dirfd: ::c_int, @@ -4021,11 +3882,7 @@ extern "C" { ) -> ::c_int; pub fn duplocale(base: ::locale_t) -> ::locale_t; pub fn freelocale(loc: ::locale_t); - pub fn newlocale( - mask: ::c_int, - locale: *const ::c_char, - base: ::locale_t, - ) -> ::locale_t; + pub fn newlocale(mask: ::c_int, locale: *const ::c_char, base: ::locale_t) -> ::locale_t; pub fn uselocale(loc: ::locale_t) -> ::locale_t; pub fn fdopendir(fd: ::c_int) -> *mut ::DIR; @@ -4050,30 +3907,14 @@ extern "C" { len: *mut ::socklen_t, flg: ::c_int, ) -> ::c_int; - pub fn ptsname_r( - fd: ::c_int, - buf: *mut ::c_char, - buflen: ::size_t, - ) -> ::c_int; + pub fn ptsname_r(fd: ::c_int, buf: *mut ::c_char, buflen: ::size_t) -> ::c_int; pub fn clearenv() -> ::c_int; - pub fn waitid( - idtype: idtype_t, - id: id_t, - infop: *mut ::siginfo_t, - options: ::c_int, - ) -> ::c_int; + pub fn waitid(idtype: idtype_t, id: id_t, infop: *mut ::siginfo_t, options: ::c_int) + -> ::c_int; pub fn setreuid(ruid: ::uid_t, euid: ::uid_t) -> ::c_int; pub fn setregid(rgid: ::gid_t, egid: ::gid_t) -> ::c_int; - pub fn getresuid( - ruid: *mut ::uid_t, - euid: *mut ::uid_t, - suid: *mut ::uid_t, - ) -> ::c_int; - pub fn getresgid( - rgid: *mut ::gid_t, - egid: *mut ::gid_t, - sgid: *mut ::gid_t, - ) -> ::c_int; + pub fn getresuid(ruid: *mut ::uid_t, euid: *mut ::uid_t, suid: *mut ::uid_t) -> ::c_int; + pub fn getresgid(rgid: *mut ::gid_t, egid: *mut ::gid_t, sgid: *mut ::gid_t) -> ::c_int; pub fn acct(filename: *const ::c_char) -> ::c_int; pub fn brk(addr: *mut ::c_void) -> ::c_int; pub fn setresgid(rgid: ::gid_t, egid: ::gid_t, sgid: ::gid_t) -> ::c_int; @@ -4104,40 +3945,18 @@ extern "C" { pub fn endpwent(); pub fn getpwent() -> *mut passwd; - pub fn shm_open( - name: *const c_char, - oflag: ::c_int, - mode: mode_t, - ) -> ::c_int; + pub fn shm_open(name: *const c_char, oflag: ::c_int, mode: mode_t) -> ::c_int; // System V IPC pub fn shmget(key: ::key_t, size: ::size_t, shmflg: ::c_int) -> ::c_int; - pub fn shmat( - shmid: ::c_int, - shmaddr: *const ::c_void, - shmflg: ::c_int, - ) -> *mut ::c_void; + pub fn shmat(shmid: ::c_int, shmaddr: *const ::c_void, shmflg: ::c_int) -> *mut ::c_void; pub fn shmdt(shmaddr: *const ::c_void) -> ::c_int; - pub fn shmctl( - shmid: ::c_int, - cmd: ::c_int, - buf: *mut ::shmid_ds, - ) -> ::c_int; + pub fn shmctl(shmid: ::c_int, cmd: ::c_int, buf: *mut ::shmid_ds) -> ::c_int; pub fn ftok(pathname: *const ::c_char, proj_id: ::c_int) -> ::key_t; pub fn semget(key: ::key_t, nsems: ::c_int, semflag: ::c_int) -> ::c_int; - pub fn semop( - semid: ::c_int, - sops: *mut ::sembuf, - nsops: ::size_t, - ) -> ::c_int; - pub fn semctl( - semid: ::c_int, - semnum: ::c_int, - cmd: ::c_int, - ... - ) -> ::c_int; - pub fn msgctl(msqid: ::c_int, cmd: ::c_int, buf: *mut msqid_ds) - -> ::c_int; + pub fn semop(semid: ::c_int, sops: *mut ::sembuf, nsops: ::size_t) -> ::c_int; + pub fn semctl(semid: ::c_int, semnum: ::c_int, cmd: ::c_int, ...) -> ::c_int; + pub fn msgctl(msqid: ::c_int, cmd: ::c_int, buf: *mut msqid_ds) -> ::c_int; pub fn msgget(key: ::key_t, msgflg: ::c_int) -> ::c_int; pub fn msgrcv( msqid: ::c_int, @@ -4153,57 +3972,24 @@ extern "C" { msgflg: ::c_int, ) -> ::c_int; - pub fn mprotect( - addr: *mut ::c_void, - len: ::size_t, - prot: ::c_int, - ) -> ::c_int; + pub fn mprotect(addr: *mut ::c_void, len: ::size_t, prot: ::c_int) -> ::c_int; pub fn __errno_location() -> *mut ::c_int; - pub fn fallocate( - fd: ::c_int, - mode: ::c_int, - offset: ::off_t, - len: ::off_t, - ) -> ::c_int; - pub fn posix_fallocate( - fd: ::c_int, - offset: ::off_t, - len: ::off_t, - ) -> ::c_int; - pub fn readahead( - fd: ::c_int, - offset: ::off64_t, - count: ::size_t, - ) -> ::ssize_t; - pub fn signalfd( - fd: ::c_int, - mask: *const ::sigset_t, - flags: ::c_int, - ) -> ::c_int; + pub fn fallocate(fd: ::c_int, mode: ::c_int, offset: ::off_t, len: ::off_t) -> ::c_int; + pub fn posix_fallocate(fd: ::c_int, offset: ::off_t, len: ::off_t) -> ::c_int; + pub fn readahead(fd: ::c_int, offset: ::off64_t, count: ::size_t) -> ::ssize_t; + pub fn signalfd(fd: ::c_int, mask: *const ::sigset_t, flags: ::c_int) -> ::c_int; pub fn timerfd_create(clockid: ::c_int, flags: ::c_int) -> ::c_int; - pub fn timerfd_gettime( - fd: ::c_int, - curr_value: *mut itimerspec, - ) -> ::c_int; + pub fn timerfd_gettime(fd: ::c_int, curr_value: *mut itimerspec) -> ::c_int; pub fn timerfd_settime( fd: ::c_int, flags: ::c_int, new_value: *const itimerspec, old_value: *mut itimerspec, ) -> ::c_int; - pub fn pwritev( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - offset: ::off_t, - ) -> ::ssize_t; - pub fn preadv( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - offset: ::off_t, - ) -> ::ssize_t; + pub fn pwritev(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int, offset: ::off_t) + -> ::ssize_t; + pub fn preadv(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int, offset: ::off_t) -> ::ssize_t; pub fn quotactl( cmd: ::c_int, special: *const ::c_char, @@ -4212,19 +3998,14 @@ extern "C" { ) -> ::c_int; pub fn dup3(oldfd: ::c_int, newfd: ::c_int, flags: ::c_int) -> ::c_int; pub fn mkostemp(template: *mut ::c_char, flags: ::c_int) -> ::c_int; - pub fn mkostemps( - template: *mut ::c_char, - suffixlen: ::c_int, - flags: ::c_int, - ) -> ::c_int; + pub fn mkostemps(template: *mut ::c_char, suffixlen: ::c_int, flags: ::c_int) -> ::c_int; pub fn sigtimedwait( set: *const sigset_t, info: *mut siginfo_t, timeout: *const ::timespec, ) -> ::c_int; pub fn sigwaitinfo(set: *const sigset_t, info: *mut siginfo_t) -> ::c_int; - pub fn nl_langinfo_l(item: ::nl_item, locale: ::locale_t) - -> *mut ::c_char; + pub fn nl_langinfo_l(item: ::nl_item, locale: ::locale_t) -> *mut ::c_char; pub fn getnameinfo( sa: *const ::sockaddr, salen: ::socklen_t, @@ -4239,11 +4020,7 @@ extern "C" { pub fn setfsuid(uid: ::uid_t) -> ::c_int; // Not available now on Android - pub fn mkfifoat( - dirfd: ::c_int, - pathname: *const ::c_char, - mode: ::mode_t, - ) -> ::c_int; + pub fn mkfifoat(dirfd: ::c_int, pathname: *const ::c_char, mode: ::mode_t) -> ::c_int; pub fn if_nameindex() -> *mut if_nameindex; pub fn if_freenameindex(ptr: *mut if_nameindex); pub fn sync_file_range( @@ -4258,35 +4035,21 @@ extern "C" { pub fn glob( pattern: *const c_char, flags: ::c_int, - errfunc: ::Option< - extern "C" fn(epath: *const c_char, errno: ::c_int) -> ::c_int, - >, + errfunc: ::Option ::c_int>, pglob: *mut ::glob_t, ) -> ::c_int; pub fn globfree(pglob: *mut ::glob_t); - pub fn posix_madvise( - addr: *mut ::c_void, - len: ::size_t, - advice: ::c_int, - ) -> ::c_int; + pub fn posix_madvise(addr: *mut ::c_void, len: ::size_t, advice: ::c_int) -> ::c_int; pub fn shm_unlink(name: *const ::c_char) -> ::c_int; pub fn seekdir(dirp: *mut ::DIR, loc: ::c_long); pub fn telldir(dirp: *mut ::DIR) -> ::c_long; - pub fn madvise( - addr: *mut ::c_void, - len: ::size_t, - advice: ::c_int, - ) -> ::c_int; + pub fn madvise(addr: *mut ::c_void, len: ::size_t, advice: ::c_int) -> ::c_int; - pub fn msync( - addr: *mut ::c_void, - len: ::size_t, - flags: ::c_int, - ) -> ::c_int; + pub fn msync(addr: *mut ::c_void, len: ::size_t, flags: ::c_int) -> ::c_int; pub fn recvfrom( socket: ::c_int, @@ -4300,33 +4063,13 @@ extern "C" { pub fn futimes(fd: ::c_int, times: *const ::timeval) -> ::c_int; pub fn nl_langinfo(item: ::nl_item) -> *mut ::c_char; - pub fn bind( - socket: ::c_int, - address: *const ::sockaddr, - address_len: ::socklen_t, - ) -> ::c_int; + pub fn bind(socket: ::c_int, address: *const ::sockaddr, address_len: ::socklen_t) -> ::c_int; - pub fn writev( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - ) -> ::ssize_t; - pub fn readv( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - ) -> ::ssize_t; + pub fn writev(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int) -> ::ssize_t; + pub fn readv(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int) -> ::ssize_t; - pub fn sendmsg( - fd: ::c_int, - msg: *const ::msghdr, - flags: ::c_int, - ) -> ::ssize_t; - pub fn recvmsg( - fd: ::c_int, - msg: *mut ::msghdr, - flags: ::c_int, - ) -> ::ssize_t; + pub fn sendmsg(fd: ::c_int, msg: *const ::msghdr, flags: ::c_int) -> ::ssize_t; + pub fn recvmsg(fd: ::c_int, msg: *mut ::msghdr, flags: ::c_int) -> ::ssize_t; pub fn getdomainname(name: *mut ::c_char, len: ::size_t) -> ::c_int; pub fn setdomainname(name: *const ::c_char, len: ::size_t) -> ::c_int; pub fn vhangup() -> ::c_int; @@ -4345,11 +4088,8 @@ extern "C" { ) -> ::c_int; pub fn sync(); pub fn syscall(num: ::c_long, ...) -> ::c_long; - pub fn sched_getaffinity( - pid: ::pid_t, - cpusetsize: ::size_t, - cpuset: *mut cpu_set_t, - ) -> ::c_int; + pub fn sched_getaffinity(pid: ::pid_t, cpusetsize: ::size_t, cpuset: *mut cpu_set_t) + -> ::c_int; pub fn sched_setaffinity( pid: ::pid_t, cpusetsize: ::size_t, @@ -4357,16 +4097,8 @@ extern "C" { ) -> ::c_int; pub fn umount(target: *const ::c_char) -> ::c_int; pub fn sched_get_priority_max(policy: ::c_int) -> ::c_int; - pub fn tee( - fd_in: ::c_int, - fd_out: ::c_int, - len: ::size_t, - flags: ::c_uint, - ) -> ::ssize_t; - pub fn settimeofday( - tv: *const ::timeval, - tz: *const ::timezone, - ) -> ::c_int; + pub fn tee(fd_in: ::c_int, fd_out: ::c_int, len: ::size_t, flags: ::c_uint) -> ::ssize_t; + pub fn settimeofday(tv: *const ::timeval, tz: *const ::timezone) -> ::c_int; pub fn splice( fd_in: ::c_int, off_in: *mut ::loff_t, @@ -4376,17 +4108,10 @@ extern "C" { flags: ::c_uint, ) -> ::ssize_t; pub fn eventfd(init: ::c_uint, flags: ::c_int) -> ::c_int; - pub fn sched_rr_get_interval(pid: ::pid_t, tp: *mut ::timespec) - -> ::c_int; - pub fn sem_timedwait( - sem: *mut sem_t, - abstime: *const ::timespec, - ) -> ::c_int; + pub fn sched_rr_get_interval(pid: ::pid_t, tp: *mut ::timespec) -> ::c_int; + pub fn sem_timedwait(sem: *mut sem_t, abstime: *const ::timespec) -> ::c_int; pub fn sem_getvalue(sem: *mut sem_t, sval: *mut ::c_int) -> ::c_int; - pub fn sched_setparam( - pid: ::pid_t, - param: *const ::sched_param, - ) -> ::c_int; + pub fn sched_setparam(pid: ::pid_t, param: *const ::sched_param) -> ::c_int; pub fn swapoff(puath: *const ::c_char) -> ::c_int; pub fn vmsplice( fd: ::c_int, @@ -4459,11 +4184,7 @@ extern "C" { result: *mut *mut ::group, ) -> ::c_int; pub fn initgroups(user: *const ::c_char, group: ::gid_t) -> ::c_int; - pub fn pthread_sigmask( - how: ::c_int, - set: *const sigset_t, - oldset: *mut sigset_t, - ) -> ::c_int; + pub fn pthread_sigmask(how: ::c_int, set: *const sigset_t, oldset: *mut sigset_t) -> ::c_int; pub fn sem_open(name: *const ::c_char, oflag: ::c_int, ...) -> *mut sem_t; pub fn getgrnam(name: *const ::c_char) -> *mut ::group; pub fn pthread_cancel(thread: ::pthread_t) -> ::c_int; @@ -4491,6 +4212,11 @@ extern "C" { child: ::Option, ) -> ::c_int; pub fn getgrgid(gid: ::gid_t) -> *mut ::group; + + pub fn setgrent(); + pub fn endgrent(); + pub fn getgrent() -> *mut ::group; + pub fn getgrouplist( user: *const ::c_char, group: ::gid_t, diff --git a/crux-mir/lib/libc/src/hermit/mod.rs b/crux-mir/lib/libc/src/hermit/mod.rs index 9880b5072..bffcefdd8 100644 --- a/crux-mir/lib/libc/src/hermit/mod.rs +++ b/crux-mir/lib/libc/src/hermit/mod.rs @@ -1,13 +1,3 @@ -// Copyright 2018 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - // libc port for HermitCore (https://hermitcore.org) // // Ported by Colin Fink diff --git a/crux-mir/lib/libc/src/lib.rs b/crux-mir/lib/libc/src/lib.rs index d5fba02da..acda09159 100644 --- a/crux-mir/lib/libc/src/lib.rs +++ b/crux-mir/lib/libc/src/lib.rs @@ -1,12 +1,3 @@ -// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. //! libc - Raw FFI bindings to platforms' system libraries //! //! [Documentation for other platforms][pd]. @@ -14,30 +5,28 @@ //! [pd]: https://rust-lang.github.io/libc/#platform-specific-documentation #![crate_name = "libc"] #![crate_type = "rlib"] -// FIXME: Remove this and redundant_semicolon once renamed lint reaches stable. -#![allow(renamed_and_removed_lints)] #![allow( + renamed_and_removed_lints, // Keep this order. + unknown_lints, // Keep this order. bad_style, overflowing_literals, improper_ctypes, - unknown_lints, + // This lint is renamed but we run CI for old stable rustc so should be here. redundant_semicolon, - redundant_semicolons + redundant_semicolons, + unused_macros, + unused_macro_rules, )] #![cfg_attr(libc_deny_warnings, deny(warnings))] // Attributes needed when building as part of the standard library -#![cfg_attr( - feature = "rustc-dep-of-std", - feature(cfg_target_vendor, link_cfg, no_core) -)] +#![cfg_attr(feature = "rustc-dep-of-std", feature(link_cfg, no_core))] #![cfg_attr(libc_thread_local, feature(thread_local))] // Enable extra lints: #![cfg_attr(feature = "extra_traits", deny(missing_debug_implementations))] #![deny(missing_copy_implementations, safe_packed_borrows)] -#![no_std] +#![cfg_attr(not(feature = "rustc-dep-of-std"), no_std)] #![cfg_attr(feature = "rustc-dep-of-std", no_core)] -#![cfg_attr(target_os = "redox", feature(static_nobundle))] -#![cfg_attr(libc_const_extern_fn, feature(const_extern_fn))] +#![cfg_attr(libc_const_extern_fn_unstable, feature(const_extern_fn))] #[macro_use] mod macros; @@ -48,6 +37,8 @@ cfg_if! { #[allow(unused_imports)] use core::iter; #[allow(unused_imports)] + use core::ops; + #[allow(unused_imports)] use core::option; } } @@ -70,7 +61,7 @@ cfg_if! { use core::clone::Clone; #[doc(hidden)] #[allow(unused_imports)] - use core::marker::Copy; + use core::marker::{Copy, Send, Sync}; #[doc(hidden)] #[allow(unused_imports)] use core::option::Option; @@ -92,7 +83,7 @@ cfg_if! { pub use core::clone::Clone; #[doc(hidden)] #[allow(unused_imports)] - pub use core::marker::Copy; + pub use core::marker::{Copy, Send, Sync}; #[doc(hidden)] #[allow(unused_imports)] pub use core::option::Option; @@ -106,12 +97,6 @@ cfg_if! { mod windows; pub use windows::*; - } else if #[cfg(target_os = "cloudabi")] { - mod fixed_width_ints; - pub use fixed_width_ints::*; - - mod cloudabi; - pub use cloudabi::*; } else if #[cfg(target_os = "fuchsia")] { mod fixed_width_ints; pub use fixed_width_ints::*; @@ -124,12 +109,24 @@ cfg_if! { mod switch; pub use switch::*; + } else if #[cfg(target_os = "psp")] { + mod fixed_width_ints; + pub use fixed_width_ints::*; + + mod psp; + pub use psp::*; } else if #[cfg(target_os = "vxworks")] { mod fixed_width_ints; pub use fixed_width_ints::*; mod vxworks; pub use vxworks::*; + } else if #[cfg(target_os = "solid_asp3")] { + mod fixed_width_ints; + pub use fixed_width_ints::*; + + mod solid; + pub use solid::*; } else if #[cfg(unix)] { mod fixed_width_ints; pub use fixed_width_ints::*; diff --git a/crux-mir/lib/libc/src/macros.rs b/crux-mir/lib/libc/src/macros.rs index f14bbf552..fd473702f 100644 --- a/crux-mir/lib/libc/src/macros.rs +++ b/crux-mir/lib/libc/src/macros.rs @@ -6,7 +6,6 @@ /// /// This allows you to conveniently provide a long list #[cfg]'d blocks of code /// without having to rewrite each clause multiple times. -#[allow(unused_macros)] macro_rules! cfg_if { // match if/else chains with a final `else` ($( @@ -40,12 +39,12 @@ macro_rules! cfg_if { // Internal and recursive macro to emit all the items // - // Collects all the negated cfgs in a list at the beginning and after the + // Collects all the negated `cfg`s in a list at the beginning and after the // semicolon is all the remaining items (@__items ($($not:meta,)*) ; ) => {}; (@__items ($($not:meta,)*) ; ( ($($m:meta),*) ($($it:item)*) ), $($rest:tt)*) => { - // Emit all items within one block, applying an approprate #[cfg]. The + // Emit all items within one block, applying an appropriate #[cfg]. The // #[cfg] will require all `$m` matchers specified and must also negate // all previous matchers. cfg_if! { @__apply cfg(all($($m,)* not(any($($not),*)))), $($it)* } @@ -62,7 +61,6 @@ macro_rules! cfg_if { }; } -#[allow(unused_macros)] macro_rules! s { ($($(#[$attr:meta])* pub $t:ident $i:ident { $($field:tt)* })*) => ($( s!(it: $(#[$attr])* pub $t $i { $($field)* }); @@ -87,7 +85,6 @@ macro_rules! s { ); } -#[allow(unused_macros)] macro_rules! s_no_extra_traits { ($($(#[$attr:meta])* pub $t:ident $i:ident { $($field:tt)* })*) => ($( s_no_extra_traits!(it: $(#[$attr])* pub $t $i { $($field)* }); @@ -114,13 +111,43 @@ macro_rules! s_no_extra_traits { $(#[$attr])* pub struct $i { $($field)* } } + #[allow(deprecated)] impl ::Copy for $i {} + #[allow(deprecated)] impl ::Clone for $i { fn clone(&self) -> $i { *self } } ); } +macro_rules! e { + ($($(#[$attr:meta])* pub enum $i:ident { $($field:tt)* })*) => ($( + __item! { + #[cfg_attr(feature = "extra_traits", derive(Debug, Eq, Hash, PartialEq))] + $(#[$attr])* + pub enum $i { $($field)* } + } + impl ::Copy for $i {} + impl ::Clone for $i { + fn clone(&self) -> $i { *self } + } + )*); +} + +macro_rules! s_paren { + ($($(#[$attr:meta])* pub struct $i:ident ( $($field:tt)* ); )* ) => ($( + __item! { + #[cfg_attr(feature = "extra_traits", derive(Debug, Eq, Hash, PartialEq))] + $(#[$attr])* + pub struct $i ( $($field)* ); + } + impl ::Copy for $i {} + impl ::Clone for $i { + fn clone(&self) -> $i { *self } + } + )*); +} + // This is a pretty horrible hack to allow us to conditionally mark // some functions as 'const', without requiring users of this macro // to care about the "const-extern-fn" feature. @@ -137,7 +164,7 @@ macro_rules! s_no_extra_traits { // so we need to avoid emitting it at all of 'const-extern-fn'. // // Specifically, moving the 'cfg_if' into the macro body will *not* work. -// Doing so would cause the '#[cfg(feature = "const-extern-fn")]' to be emiited +// Doing so would cause the '#[cfg(feature = "const-extern-fn")]' to be emitted // into user code. The 'cfg' gate will not stop Rust from trying to parse the // 'pub const unsafe extern fn', so users would get a compiler error even when // the 'const-extern-fn' feature is disabled @@ -145,19 +172,19 @@ macro_rules! s_no_extra_traits { // Note that users of this macro need to place 'const' in a weird position // (after the closing ')' for the arguments, but before the return type). // This was the only way I could satisfy the following two requirements: -// 1. Avoid ambuguity errors from 'macro_rules!' (which happen when writing '$foo:ident fn' +// 1. Avoid ambiguity errors from 'macro_rules!' (which happen when writing '$foo:ident fn' // 2. Allow users of this macro to mix 'pub fn foo' and 'pub const fn bar' within the same // 'f!' block cfg_if! { if #[cfg(libc_const_extern_fn)] { - #[allow(unused_macros)] macro_rules! f { - ($(pub $({$constness:ident})* fn $i:ident( + ($($(#[$attr:meta])* pub $({$constness:ident})* fn $i:ident( $($arg:ident: $argty:ty),* ) -> $ret:ty { $($body:stmt);* })*) => ($( #[inline] + $(#[$attr])* pub $($constness)* unsafe extern fn $i($($arg: $argty),* ) -> $ret { $($body);* @@ -165,14 +192,29 @@ cfg_if! { )*) } - #[allow(unused_macros)] + macro_rules! safe_f { + ($($(#[$attr:meta])* pub $({$constness:ident})* fn $i:ident( + $($arg:ident: $argty:ty),* + ) -> $ret:ty { + $($body:stmt);* + })*) => ($( + #[inline] + $(#[$attr])* + pub $($constness)* extern fn $i($($arg: $argty),* + ) -> $ret { + $($body);* + } + )*) + } + macro_rules! const_fn { - ($($({$constness:ident})* fn $i:ident( + ($($(#[$attr:meta])* $({$constness:ident})* fn $i:ident( $($arg:ident: $argty:ty),* ) -> $ret:ty { $($body:stmt);* })*) => ($( #[inline] + $(#[$attr])* $($constness)* fn $i($($arg: $argty),* ) -> $ret { $($body);* @@ -181,14 +223,14 @@ cfg_if! { } } else { - #[allow(unused_macros)] macro_rules! f { - ($(pub $({$constness:ident})* fn $i:ident( + ($($(#[$attr:meta])* pub $({$constness:ident})* fn $i:ident( $($arg:ident: $argty:ty),* ) -> $ret:ty { $($body:stmt);* })*) => ($( #[inline] + $(#[$attr])* pub unsafe extern fn $i($($arg: $argty),* ) -> $ret { $($body);* @@ -196,14 +238,29 @@ cfg_if! { )*) } - #[allow(unused_macros)] + macro_rules! safe_f { + ($($(#[$attr:meta])* pub $({$constness:ident})* fn $i:ident( + $($arg:ident: $argty:ty),* + ) -> $ret:ty { + $($body:stmt);* + })*) => ($( + #[inline] + $(#[$attr])* + pub extern fn $i($($arg: $argty),* + ) -> $ret { + $($body);* + } + )*) + } + macro_rules! const_fn { - ($($({$constness:ident})* fn $i:ident( + ($($(#[$attr:meta])* $({$constness:ident})* fn $i:ident( $($arg:ident: $argty:ty),* ) -> $ret:ty { $($body:stmt);* })*) => ($( #[inline] + $(#[$attr])* fn $i($($arg: $argty),* ) -> $ret { $($body);* @@ -213,14 +270,12 @@ cfg_if! { } } -#[allow(unused_macros)] macro_rules! __item { ($i:item) => { $i }; } -#[allow(unused_macros)] macro_rules! align_const { ($($(#[$attr:meta])* pub const $name:ident : $t1:ty @@ -239,13 +294,12 @@ macro_rules! align_const { )*) } -// This macro is used to deprecate items that should be accessed via the mach crate -#[allow(unused_macros)] +// This macro is used to deprecate items that should be accessed via the mach2 crate macro_rules! deprecated_mach { (pub const $id:ident: $ty:ty = $expr:expr;) => { #[deprecated( since = "0.2.55", - note = "Use the `mach` crate instead", + note = "Use the `mach2` crate instead", )] #[allow(deprecated)] pub const $id: $ty = $expr; @@ -260,7 +314,7 @@ macro_rules! deprecated_mach { (pub type $id:ident = $ty:ty;) => { #[deprecated( since = "0.2.55", - note = "Use the `mach` crate instead", + note = "Use the `mach2` crate instead", )] #[allow(deprecated)] pub type $id = $ty; @@ -273,3 +327,17 @@ macro_rules! deprecated_mach { )* } } + +#[cfg(not(libc_ptr_addr_of))] +macro_rules! ptr_addr_of { + ($place:expr) => { + &$place + }; +} + +#[cfg(libc_ptr_addr_of)] +macro_rules! ptr_addr_of { + ($place:expr) => { + ::core::ptr::addr_of!($place) + }; +} diff --git a/crux-mir/lib/libc/src/psp.rs b/crux-mir/lib/libc/src/psp.rs new file mode 100644 index 000000000..575232dad --- /dev/null +++ b/crux-mir/lib/libc/src/psp.rs @@ -0,0 +1,4174 @@ +//! PSP C type definitions +//! +//! These type declarations are not enough, as they must be ultimately resolved +//! by the linker. Crates that use these definitions must, somewhere in the +//! crate graph, include a stub provider crate such as the `psp` crate. + +pub type c_schar = i8; +pub type c_uchar = u8; +pub type c_short = i16; +pub type c_ushort = u16; +pub type c_int = i32; +pub type c_uint = u32; +pub type c_float = f32; +pub type c_double = f64; +pub type c_longlong = i64; +pub type c_ulonglong = u64; +pub type intmax_t = i64; +pub type uintmax_t = u64; + +pub type size_t = usize; +pub type ptrdiff_t = isize; +pub type intptr_t = isize; +pub type uintptr_t = usize; +pub type ssize_t = isize; + +pub type c_char = u8; +pub type c_long = i64; +pub type c_ulong = u64; + +cfg_if! { + if #[cfg(libc_core_cvoid)] { + pub use ::ffi::c_void; + } else { + // Use repr(u8) as LLVM expects `void*` to be the same as `i8*` to help + // enable more optimization opportunities around it recognizing things + // like malloc/free. + #[repr(u8)] + #[allow(missing_copy_implementations)] + #[allow(missing_debug_implementations)] + pub enum c_void { + // Two dummy variants so the #[repr] attribute can be used. + #[doc(hidden)] + __variant1, + #[doc(hidden)] + __variant2, + } + } +} + +pub type SceKernelVTimerHandler = unsafe extern "C" fn( + uid: SceUid, + arg1: *mut SceKernelSysClock, + arg2: *mut SceKernelSysClock, + arg3: *mut c_void, +) -> u32; + +pub type SceKernelVTimerHandlerWide = + unsafe extern "C" fn(uid: SceUid, arg1: i64, arg2: i64, arg3: *mut c_void) -> u32; + +pub type SceKernelThreadEventHandler = + unsafe extern "C" fn(mask: i32, thid: SceUid, common: *mut c_void) -> i32; + +pub type SceKernelAlarmHandler = unsafe extern "C" fn(common: *mut c_void) -> u32; + +pub type SceKernelCallbackFunction = + unsafe extern "C" fn(arg1: i32, arg2: i32, arg: *mut c_void) -> i32; + +pub type SceKernelThreadEntry = unsafe extern "C" fn(args: usize, argp: *mut c_void) -> i32; + +pub type PowerCallback = extern "C" fn(unknown: i32, power_info: i32); + +pub type IoPermissions = i32; + +pub type UmdCallback = fn(unknown: i32, event: i32) -> i32; + +pub type SceMpegRingbufferCb = + ::Option i32>; + +pub type GuCallback = ::Option; +pub type GuSwapBuffersCallback = + ::Option; + +pub type SceNetAdhocctlHandler = + ::Option; + +pub type AdhocMatchingCallback = ::Option< + unsafe extern "C" fn( + matching_id: i32, + event: i32, + mac: *mut u8, + opt_len: i32, + opt_data: *mut c_void, + ), +>; + +pub type SceNetApctlHandler = ::Option< + unsafe extern "C" fn(oldState: i32, newState: i32, event: i32, error: i32, pArg: *mut c_void), +>; + +pub type HttpMallocFunction = ::Option *mut c_void>; +pub type HttpReallocFunction = + ::Option *mut c_void>; +pub type HttpFreeFunction = ::Option; +pub type HttpPasswordCB = ::Option< + unsafe extern "C" fn( + request: i32, + auth_type: HttpAuthType, + realm: *const u8, + username: *mut u8, + password: *mut u8, + need_entity: i32, + entity_body: *mut *mut u8, + entity_size: *mut usize, + save: *mut i32, + ) -> i32, +>; + +pub type socklen_t = u32; + +e! { + #[repr(u32)] + pub enum AudioFormat { + Stereo = 0, + Mono = 0x10, + } + + #[repr(u32)] + pub enum DisplayMode { + Lcd = 0, + } + + #[repr(u32)] + pub enum DisplayPixelFormat { + Psm5650 = 0, + Psm5551 = 1, + Psm4444 = 2, + Psm8888 = 3, + } + + #[repr(u32)] + pub enum DisplaySetBufSync { + Immediate = 0, + NextFrame = 1, + } + + #[repr(i32)] + pub enum AudioOutputFrequency { + Khz48 = 48000, + Khz44_1 = 44100, + Khz32 = 32000, + Khz24 = 24000, + Khz22_05 = 22050, + Khz16 = 16000, + Khz12 = 12000, + Khz11_025 = 11025, + Khz8 = 8000, + } + + #[repr(i32)] + pub enum AudioInputFrequency { + Khz44_1 = 44100, + Khz22_05 = 22050, + Khz11_025 = 11025, + } + + #[repr(u32)] + pub enum CtrlMode { + Digital = 0, + Analog, + } + + #[repr(i32)] + pub enum GeMatrixType { + Bone0 = 0, + Bone1, + Bone2, + Bone3, + Bone4, + Bone5, + Bone6, + Bone7, + World, + View, + Projection, + TexGen, + } + + #[repr(i32)] + pub enum GeListState { + Done = 0, + Queued, + DrawingDone, + StallReached, + CancelDone, + } + + #[repr(u8)] + pub enum GeCommand { + Nop = 0, + Vaddr = 0x1, + Iaddr = 0x2, + Prim = 0x4, + Bezier = 0x5, + Spline = 0x6, + BoundingBox = 0x7, + Jump = 0x8, + BJump = 0x9, + Call = 0xa, + Ret = 0xb, + End = 0xc, + Signal = 0xe, + Finish = 0xf, + Base = 0x10, + VertexType = 0x12, + OffsetAddr = 0x13, + Origin = 0x14, + Region1 = 0x15, + Region2 = 0x16, + LightingEnable = 0x17, + LightEnable0 = 0x18, + LightEnable1 = 0x19, + LightEnable2 = 0x1a, + LightEnable3 = 0x1b, + DepthClampEnable = 0x1c, + CullFaceEnable = 0x1d, + TextureMapEnable = 0x1e, + FogEnable = 0x1f, + DitherEnable = 0x20, + AlphaBlendEnable = 0x21, + AlphaTestEnable = 0x22, + ZTestEnable = 0x23, + StencilTestEnable = 0x24, + AntiAliasEnable = 0x25, + PatchCullEnable = 0x26, + ColorTestEnable = 0x27, + LogicOpEnable = 0x28, + BoneMatrixNumber = 0x2a, + BoneMatrixData = 0x2b, + MorphWeight0 = 0x2c, + MorphWeight1 = 0x2d, + MorphWeight2 = 0x2e, + MorphWeight3 = 0x2f, + MorphWeight4 = 0x30, + MorphWeight5 = 0x31, + MorphWeight6 = 0x32, + MorphWeight7 = 0x33, + PatchDivision = 0x36, + PatchPrimitive = 0x37, + PatchFacing = 0x38, + WorldMatrixNumber = 0x3a, + WorldMatrixData = 0x3b, + ViewMatrixNumber = 0x3c, + ViewMatrixData = 0x3d, + ProjMatrixNumber = 0x3e, + ProjMatrixData = 0x3f, + TGenMatrixNumber = 0x40, + TGenMatrixData = 0x41, + ViewportXScale = 0x42, + ViewportYScale = 0x43, + ViewportZScale = 0x44, + ViewportXCenter = 0x45, + ViewportYCenter = 0x46, + ViewportZCenter = 0x47, + TexScaleU = 0x48, + TexScaleV = 0x49, + TexOffsetU = 0x4a, + TexOffsetV = 0x4b, + OffsetX = 0x4c, + OffsetY = 0x4d, + ShadeMode = 0x50, + ReverseNormal = 0x51, + MaterialUpdate = 0x53, + MaterialEmissive = 0x54, + MaterialAmbient = 0x55, + MaterialDiffuse = 0x56, + MaterialSpecular = 0x57, + MaterialAlpha = 0x58, + MaterialSpecularCoef = 0x5b, + AmbientColor = 0x5c, + AmbientAlpha = 0x5d, + LightMode = 0x5e, + LightType0 = 0x5f, + LightType1 = 0x60, + LightType2 = 0x61, + LightType3 = 0x62, + Light0X = 0x63, + Light0Y, + Light0Z, + Light1X, + Light1Y, + Light1Z, + Light2X, + Light2Y, + Light2Z, + Light3X, + Light3Y, + Light3Z, + Light0DirectionX = 0x6f, + Light0DirectionY, + Light0DirectionZ, + Light1DirectionX, + Light1DirectionY, + Light1DirectionZ, + Light2DirectionX, + Light2DirectionY, + Light2DirectionZ, + Light3DirectionX, + Light3DirectionY, + Light3DirectionZ, + Light0ConstantAtten = 0x7b, + Light0LinearAtten, + Light0QuadtraticAtten, + Light1ConstantAtten, + Light1LinearAtten, + Light1QuadtraticAtten, + Light2ConstantAtten, + Light2LinearAtten, + Light2QuadtraticAtten, + Light3ConstantAtten, + Light3LinearAtten, + Light3QuadtraticAtten, + Light0ExponentAtten = 0x87, + Light1ExponentAtten, + Light2ExponentAtten, + Light3ExponentAtten, + Light0CutoffAtten = 0x8b, + Light1CutoffAtten, + Light2CutoffAtten, + Light3CutoffAtten, + Light0Ambient = 0x8f, + Light0Diffuse, + Light0Specular, + Light1Ambient, + Light1Diffuse, + Light1Specular, + Light2Ambient, + Light2Diffuse, + Light2Specular, + Light3Ambient, + Light3Diffuse, + Light3Specular, + Cull = 0x9b, + FrameBufPtr = 0x9c, + FrameBufWidth = 0x9d, + ZBufPtr = 0x9e, + ZBufWidth = 0x9f, + TexAddr0 = 0xa0, + TexAddr1, + TexAddr2, + TexAddr3, + TexAddr4, + TexAddr5, + TexAddr6, + TexAddr7, + TexBufWidth0 = 0xa8, + TexBufWidth1, + TexBufWidth2, + TexBufWidth3, + TexBufWidth4, + TexBufWidth5, + TexBufWidth6, + TexBufWidth7, + ClutAddr = 0xb0, + ClutAddrUpper = 0xb1, + TransferSrc, + TransferSrcW, + TransferDst, + TransferDstW, + TexSize0 = 0xb8, + TexSize1, + TexSize2, + TexSize3, + TexSize4, + TexSize5, + TexSize6, + TexSize7, + TexMapMode = 0xc0, + TexShadeLs = 0xc1, + TexMode = 0xc2, + TexFormat = 0xc3, + LoadClut = 0xc4, + ClutFormat = 0xc5, + TexFilter = 0xc6, + TexWrap = 0xc7, + TexLevel = 0xc8, + TexFunc = 0xc9, + TexEnvColor = 0xca, + TexFlush = 0xcb, + TexSync = 0xcc, + Fog1 = 0xcd, + Fog2 = 0xce, + FogColor = 0xcf, + TexLodSlope = 0xd0, + FramebufPixFormat = 0xd2, + ClearMode = 0xd3, + Scissor1 = 0xd4, + Scissor2 = 0xd5, + MinZ = 0xd6, + MaxZ = 0xd7, + ColorTest = 0xd8, + ColorRef = 0xd9, + ColorTestmask = 0xda, + AlphaTest = 0xdb, + StencilTest = 0xdc, + StencilOp = 0xdd, + ZTest = 0xde, + BlendMode = 0xdf, + BlendFixedA = 0xe0, + BlendFixedB = 0xe1, + Dith0 = 0xe2, + Dith1, + Dith2, + Dith3, + LogicOp = 0xe6, + ZWriteDisable = 0xe7, + MaskRgb = 0xe8, + MaskAlpha = 0xe9, + TransferStart = 0xea, + TransferSrcPos = 0xeb, + TransferDstPos = 0xec, + TransferSize = 0xee, + Vscx = 0xf0, + Vscy = 0xf1, + Vscz = 0xf2, + Vtcs = 0xf3, + Vtct = 0xf4, + Vtcq = 0xf5, + Vcv = 0xf6, + Vap = 0xf7, + Vfc = 0xf8, + Vscv = 0xf9, + + Unknown03 = 0x03, + Unknown0D = 0x0d, + Unknown11 = 0x11, + Unknown29 = 0x29, + Unknown34 = 0x34, + Unknown35 = 0x35, + Unknown39 = 0x39, + Unknown4E = 0x4e, + Unknown4F = 0x4f, + Unknown52 = 0x52, + Unknown59 = 0x59, + Unknown5A = 0x5a, + UnknownB6 = 0xb6, + UnknownB7 = 0xb7, + UnknownD1 = 0xd1, + UnknownED = 0xed, + UnknownEF = 0xef, + UnknownFA = 0xfa, + UnknownFB = 0xfb, + UnknownFC = 0xfc, + UnknownFD = 0xfd, + UnknownFE = 0xfe, + NopFF = 0xff, + } + + #[repr(i32)] + pub enum SceSysMemPartitionId { + SceKernelUnknownPartition = 0, + SceKernelPrimaryKernelPartition = 1, + SceKernelPrimaryUserPartition = 2, + SceKernelOtherKernelPartition1 = 3, + SceKernelOtherKernelPartition2 = 4, + SceKernelVshellPARTITION = 5, + SceKernelScUserPartition = 6, + SceKernelMeUserPartition = 7, + SceKernelExtendedScKernelPartition = 8, + SceKernelExtendedSc2KernelPartition = 9, + SceKernelExtendedMeKernelPartition = 10, + SceKernelVshellKernelPartition = 11, + SceKernelExtendedKernelPartition = 12, + } + + #[repr(i32)] + pub enum SceSysMemBlockTypes { + Low = 0, + High, + Addr, + } + + #[repr(u32)] + pub enum Interrupt { + Gpio = 4, + Ata = 5, + Umd = 6, + Mscm0 = 7, + Wlan = 8, + Audio = 10, + I2c = 12, + Sircs = 14, + Systimer0 = 15, + Systimer1 = 16, + Systimer2 = 17, + Systimer3 = 18, + Thread0 = 19, + Nand = 20, + Dmacplus = 21, + Dma0 = 22, + Dma1 = 23, + Memlmd = 24, + Ge = 25, + Vblank = 30, + Mecodec = 31, + Hpremote = 36, + Mscm1 = 60, + Mscm2 = 61, + Thread1 = 65, + Interrupt = 66, + } + + #[repr(u32)] + pub enum SubInterrupt { + Gpio = Interrupt::Gpio as u32, + Ata = Interrupt::Ata as u32, + Umd = Interrupt::Umd as u32, + Dmacplus = Interrupt::Dmacplus as u32, + Ge = Interrupt::Ge as u32, + Display = Interrupt::Vblank as u32, + } + + #[repr(u32)] + pub enum SceKernelIdListType { + Thread = 1, + Semaphore = 2, + EventFlag = 3, + Mbox = 4, + Vpl = 5, + Fpl = 6, + Mpipe = 7, + Callback = 8, + ThreadEventHandler = 9, + Alarm = 10, + VTimer = 11, + SleepThread = 64, + DelayThread = 65, + SuspendThread = 66, + DormantThread = 67, + } + + #[repr(i32)] + pub enum UsbCamResolution { + Px160_120 = 0, + Px176_144 = 1, + Px320_240 = 2, + Px352_288 = 3, + Px640_480 = 4, + Px1024_768 = 5, + Px1280_960 = 6, + Px480_272 = 7, + Px360_272 = 8, + } + + #[repr(i32)] + pub enum UsbCamResolutionEx { + Px160_120 = 0, + Px176_144 = 1, + Px320_240 = 2, + Px352_288 = 3, + Px360_272 = 4, + Px480_272 = 5, + Px640_480 = 6, + Px1024_768 = 7, + Px1280_960 = 8, + } + + #[repr(i32)] + pub enum UsbCamDelay { + NoDelay = 0, + Delay10Sec = 1, + Delay20Sec = 2, + Delay30Sec = 3, + } + + #[repr(i32)] + pub enum UsbCamFrameRate { + Fps3_75 = 0, + Fps5 = 1, + Fps7_5 = 2, + Fps10 = 3, + Fps15 = 4, + Fps20 = 5, + Fps30 = 6, + Fps60 = 7, + } + + #[repr(i32)] + pub enum UsbCamWb { + Auto = 0, + Daylight = 1, + Fluorescent = 2, + Incadescent = 3, + } + + #[repr(i32)] + pub enum UsbCamEffectMode { + Normal = 0, + Negative = 1, + Blackwhite = 2, + Sepia = 3, + Blue = 4, + Red = 5, + Green = 6, + } + + #[repr(i32)] + pub enum UsbCamEvLevel { + Pos2_0 = 0, + Pos1_7 = 1, + Pos1_5 = 2, + Pos1_3 = 3, + Pos1_0 = 4, + Pos0_7 = 5, + Pos0_5 = 6, + Pos0_3 = 7, + Zero = 8, + Neg0_3, + Neg0_5, + Neg0_7, + Neg1_0, + Neg1_3, + Neg1_5, + Neg1_7, + Neg2_0, + } + + #[repr(i32)] + pub enum RtcCheckValidError { + InvalidYear = -1, + InvalidMonth = -2, + InvalidDay = -3, + InvalidHour = -4, + InvalidMinutes = -5, + InvalidSeconds = -6, + InvalidMicroseconds = -7, + } + + #[repr(u32)] + pub enum PowerTick { + All = 0, + Suspend = 1, + Display = 6, + } + + #[repr(u32)] + pub enum IoAssignPerms { + RdWr = 0, + RdOnly = 1, + } + + #[repr(u32)] + pub enum IoWhence { + Set = 0, + Cur = 1, + End = 2, + } + + #[repr(u32)] + pub enum UmdType { + Game = 0x10, + Video = 0x20, + Audio = 0x40, + } + + #[repr(u32)] + pub enum GuPrimitive { + Points = 0, + Lines = 1, + LineStrip = 2, + Triangles = 3, + TriangleStrip = 4, + TriangleFan = 5, + Sprites = 6, + } + + #[repr(u32)] + pub enum PatchPrimitive { + Points = 0, + LineStrip = 2, + TriangleStrip = 4, + } + + #[repr(u32)] + pub enum GuState { + AlphaTest = 0, + DepthTest = 1, + ScissorTest = 2, + StencilTest = 3, + Blend = 4, + CullFace = 5, + Dither = 6, + Fog = 7, + ClipPlanes = 8, + Texture2D = 9, + Lighting = 10, + Light0 = 11, + Light1 = 12, + Light2 = 13, + Light3 = 14, + LineSmooth = 15, + PatchCullFace = 16, + ColorTest = 17, + ColorLogicOp = 18, + FaceNormalReverse = 19, + PatchFace = 20, + Fragment2X = 21, + } + + #[repr(u32)] + pub enum MatrixMode { + Projection = 0, + View = 1, + Model = 2, + Texture = 3, + } + + #[repr(u32)] + pub enum TexturePixelFormat { + Psm5650 = 0, + Psm5551 = 1, + Psm4444 = 2, + Psm8888 = 3, + PsmT4 = 4, + PsmT8 = 5, + PsmT16 = 6, + PsmT32 = 7, + PsmDxt1 = 8, + PsmDxt3 = 9, + PsmDxt5 = 10, + } + + #[repr(u32)] + pub enum SplineMode { + FillFill = 0, + OpenFill = 1, + FillOpen = 2, + OpenOpen = 3, + } + + #[repr(u32)] + pub enum ShadingModel { + Flat = 0, + Smooth = 1, + } + + #[repr(u32)] + pub enum LogicalOperation { + Clear = 0, + And = 1, + AndReverse = 2, + Copy = 3, + AndInverted = 4, + Noop = 5, + Xor = 6, + Or = 7, + Nor = 8, + Equiv = 9, + Inverted = 10, + OrReverse = 11, + CopyInverted = 12, + OrInverted = 13, + Nand = 14, + Set = 15, + } + + #[repr(u32)] + pub enum TextureFilter { + Nearest = 0, + Linear = 1, + NearestMipmapNearest = 4, + LinearMipmapNearest = 5, + NearestMipmapLinear = 6, + LinearMipmapLinear = 7, + } + + #[repr(u32)] + pub enum TextureMapMode { + TextureCoords = 0, + TextureMatrix = 1, + EnvironmentMap = 2, + } + + #[repr(u32)] + pub enum TextureLevelMode { + Auto = 0, + Const = 1, + Slope = 2, + } + + #[repr(u32)] + pub enum TextureProjectionMapMode { + Position = 0, + Uv = 1, + NormalizedNormal = 2, + Normal = 3, + } + + #[repr(u32)] + pub enum GuTexWrapMode { + Repeat = 0, + Clamp = 1, + } + + #[repr(u32)] + pub enum FrontFaceDirection { + Clockwise = 0, + CounterClockwise = 1, + } + + #[repr(u32)] + pub enum AlphaFunc { + Never = 0, + Always, + Equal, + NotEqual, + Less, + LessOrEqual, + Greater, + GreaterOrEqual, + } + + #[repr(u32)] + pub enum StencilFunc { + Never = 0, + Always, + Equal, + NotEqual, + Less, + LessOrEqual, + Greater, + GreaterOrEqual, + } + + #[repr(u32)] + pub enum ColorFunc { + Never = 0, + Always, + Equal, + NotEqual, + } + + #[repr(u32)] + pub enum DepthFunc { + Never = 0, + Always, + Equal, + NotEqual, + Less, + LessOrEqual, + Greater, + GreaterOrEqual, + } + + #[repr(u32)] + pub enum TextureEffect { + Modulate = 0, + Decal = 1, + Blend = 2, + Replace = 3, + Add = 4, + } + + #[repr(u32)] + pub enum TextureColorComponent { + Rgb = 0, + Rgba = 1, + } + + #[repr(u32)] + pub enum MipmapLevel { + None = 0, + Level1, + Level2, + Level3, + Level4, + Level5, + Level6, + Level7, + } + + #[repr(u32)] + pub enum BlendOp { + Add = 0, + Subtract = 1, + ReverseSubtract = 2, + Min = 3, + Max = 4, + Abs = 5, + } + + #[repr(u32)] + pub enum BlendSrc { + SrcColor = 0, + OneMinusSrcColor = 1, + SrcAlpha = 2, + OneMinusSrcAlpha = 3, + Fix = 10, + } + + #[repr(u32)] + pub enum BlendDst { + DstColor = 0, + OneMinusDstColor = 1, + DstAlpha = 4, + OneMinusDstAlpha = 5, + Fix = 10, + } + + #[repr(u32)] + pub enum StencilOperation { + Keep = 0, + Zero = 1, + Replace = 2, + Invert = 3, + Incr = 4, + Decr = 5, + } + + #[repr(u32)] + pub enum LightMode { + SingleColor = 0, + SeparateSpecularColor = 1, + } + + #[repr(u32)] + pub enum LightType { + Directional = 0, + Pointlight = 1, + Spotlight = 2, + } + + #[repr(u32)] + pub enum GuContextType { + Direct = 0, + Call = 1, + Send = 2, + } + + #[repr(u32)] + pub enum GuQueueMode { + Tail = 0, + Head = 1, + } + + #[repr(u32)] + pub enum GuSyncMode { + Finish = 0, + Signal = 1, + Done = 2, + List = 3, + Send = 4, + } + + #[repr(u32)] + pub enum GuSyncBehavior { + Wait = 0, + NoWait = 1, + } + + #[repr(u32)] + pub enum GuCallbackId { + Signal = 1, + Finish = 4, + } + + #[repr(u32)] + pub enum SignalBehavior { + Suspend = 1, + Continue = 2, + } + + #[repr(u32)] + pub enum ClutPixelFormat { + Psm5650 = 0, + Psm5551 = 1, + Psm4444 = 2, + Psm8888 = 3, + } + + #[repr(C)] + pub enum KeyType { + Directory = 1, + Integer = 2, + String = 3, + Bytes = 4, + } + + #[repr(u32)] + pub enum UtilityMsgDialogMode { + Error, + Text, + } + + #[repr(u32)] + pub enum UtilityMsgDialogPressed { + Unknown1, + Yes, + No, + Back, + } + + #[repr(u32)] + pub enum UtilityDialogButtonAccept { + Circle, + Cross, + } + + #[repr(u32)] + pub enum SceUtilityOskInputLanguage { + Default, + Japanese, + English, + French, + Spanish, + German, + Italian, + Dutch, + Portugese, + Russian, + Korean, + } + + #[repr(u32)] + pub enum SceUtilityOskInputType { + All, + LatinDigit, + LatinSymbol, + LatinLowercase = 4, + LatinUppercase = 8, + JapaneseDigit = 0x100, + JapaneseSymbol = 0x200, + JapaneseLowercase = 0x400, + JapaneseUppercase = 0x800, + JapaneseHiragana = 0x1000, + JapaneseHalfWidthKatakana = 0x2000, + JapaneseKatakana = 0x4000, + JapaneseKanji = 0x8000, + RussianLowercase = 0x10000, + RussianUppercase = 0x20000, + Korean = 0x40000, + Url = 0x80000, + } + + #[repr(u32)] + pub enum SceUtilityOskState { + None, + Initializing, + Initialized, + Visible, + Quit, + Finished, + } + + #[repr(u32)] + pub enum SceUtilityOskResult { + Unchanged, + Cancelled, + Changed, + } + + #[repr(u32)] + pub enum SystemParamLanguage { + Japanese, + English, + French, + Spanish, + German, + Italian, + Dutch, + Portugese, + Russian, + Korean, + ChineseTraditional, + ChineseSimplified, + } + + #[repr(u32)] + pub enum SystemParamId { + StringNickname = 1, + AdhocChannel, + WlanPowerSave, + DateFormat, + TimeFormat, + Timezone, + DaylightSavings, + Language, + Unknown, + } + + #[repr(u32)] + pub enum SystemParamAdhocChannel { + ChannelAutomatic = 0, + Channel1 = 1, + Channel6 = 6, + Channel11 = 11, + } + + #[repr(u32)] + pub enum SystemParamWlanPowerSaveState { + Off, + On, + } + + #[repr(u32)] + pub enum SystemParamDateFormat { + YYYYMMDD, + MMDDYYYY, + DDMMYYYY, + } + + #[repr(u32)] + pub enum SystemParamTimeFormat { + Hour24, + Hour12, + } + + #[repr(u32)] + pub enum SystemParamDaylightSavings { + Std, + Dst, + } + + #[repr(u32)] + pub enum AvModule { + AvCodec, + SasCore, + Atrac3Plus, + MpegBase, + Mp3, + Vaudio, + Aac, + G729, + } + + #[repr(u32)] + pub enum Module { + NetCommon = 0x100, + NetAdhoc, + NetInet, + NetParseUri, + NetHttp, + NetSsl, + + UsbPspCm = 0x200, + UsbMic, + UsbCam, + UsbGps, + + AvCodec = 0x300, + AvSascore, + AvAtrac3Plus, + AvMpegBase, + AvMp3, + AvVaudio, + AvAac, + AvG729, + + NpCommon = 0x400, + NpService, + NpMatching2, + NpDrm = 0x500, + + Irda = 0x600, + } + + #[repr(u32)] + pub enum NetModule { + NetCommon = 1, + NetAdhoc, + NetInet, + NetParseUri, + NetHttp, + NetSsl, + } + + #[repr(u32)] + pub enum UsbModule { + UsbPspCm = 1, + UsbAcc, + UsbMic, + UsbCam, + UsbGps, + } + + #[repr(u32)] + pub enum NetParam { + Name, + Ssid, + Secure, + WepKey, + IsStaticIp, + Ip, + NetMask, + Route, + ManualDns, + PrimaryDns, + SecondaryDns, + ProxyUser, + ProxyPass, + UseProxy, + ProxyServer, + ProxyPort, + Unknown1, + Unknown2, + } + + #[repr(u32)] + pub enum UtilityNetconfAction { + ConnectAP, + DisplayStatus, + ConnectAdhoc, + } + + #[repr(u32)] + pub enum UtilitySavedataMode { + AutoLoad, + AutoSave, + Load, + Save, + ListLoad, + ListSave, + ListDelete, + Delete, + } + + #[repr(u32)] + pub enum UtilitySavedataFocus { + Unknown1, + FirstList, + LastList, + Latest, + Oldest, + Unknown2, + Unknown3, + FirstEmpty, + LastEmpty, + } + + #[repr(u32)] + pub enum UtilityGameSharingMode { + Single = 1, + Multiple, + } + + #[repr(u32)] + pub enum UtilityGameSharingDataType { + File = 1, + Memory, + } + + #[repr(u32)] + pub enum UtilityHtmlViewerInterfaceMode { + Full, + Limited, + None, + } + + #[repr(u32)] + pub enum UtilityHtmlViewerCookieMode { + Disabled = 0, + Enabled, + Confirm, + Default, + } + + #[repr(u32)] + pub enum UtilityHtmlViewerTextSize { + Large, + Normal, + Small, + } + + #[repr(u32)] + pub enum UtilityHtmlViewerDisplayMode { + Normal, + Fit, + SmartFit, + } + + #[repr(u32)] + pub enum UtilityHtmlViewerConnectMode { + Last, + ManualOnce, + ManualAll, + } + + #[repr(u32)] + pub enum UtilityHtmlViewerDisconnectMode { + Enable, + Disable, + Confirm, + } + + #[repr(u32)] + pub enum ScePspnetAdhocPtpState { + Closed, + Listen, + SynSent, + SynReceived, + Established, + } + + #[repr(u32)] + pub enum AdhocMatchingMode { + Host = 1, + Client, + Ptp, + } + + #[repr(u32)] + pub enum ApctlState { + Disconnected, + Scanning, + Joining, + GettingIp, + GotIp, + EapAuth, + KeyExchange, + } + + #[repr(u32)] + pub enum ApctlEvent { + ConnectRequest, + ScanRequest, + ScanComplete, + Established, + GetIp, + DisconnectRequest, + Error, + Info, + EapAuth, + KeyExchange, + Reconnect, + } + + #[repr(u32)] + pub enum ApctlInfo { + ProfileName, + Bssid, + Ssid, + SsidLength, + SecurityType, + Strength, + Channel, + PowerSave, + Ip, + SubnetMask, + Gateway, + PrimaryDns, + SecondaryDns, + UseProxy, + ProxyUrl, + ProxyPort, + EapType, + StartBrowser, + Wifisp, + } + + #[repr(u32)] + pub enum ApctlInfoSecurityType { + None, + Wep, + Wpa, + } + + #[repr(u32)] + pub enum HttpMethod { + Get, + Post, + Head, + } + + #[repr(u32)] + pub enum HttpAuthType { + Basic, + Digest, + } +} + +s_paren! { + #[repr(transparent)] + pub struct SceUid(pub i32); + + #[repr(transparent)] + pub struct SceMpeg(*mut *mut c_void); + + #[repr(transparent)] + pub struct SceMpegStream(*mut c_void); + + #[repr(transparent)] + pub struct Mp3Handle(pub i32); + + #[repr(transparent)] + pub struct RegHandle(u32); +} + +s! { + pub struct sockaddr { + pub sa_len: u8, + pub sa_family: u8, + pub sa_data: [u8;14], + } + + pub struct in_addr { + pub s_addr: u32, + } + + pub struct AudioInputParams { + pub unknown1: i32, + pub gain: i32, + pub unknown2: i32, + pub unknown3: i32, + pub unknown4: i32, + pub unknown5: i32, + } + + pub struct Atrac3BufferInfo { + pub puc_write_position_first_buf: *mut u8, + pub ui_writable_byte_first_buf: u32, + pub ui_min_write_byte_first_buf: u32, + pub ui_read_position_first_buf: u32, + pub puc_write_position_second_buf: *mut u8, + pub ui_writable_byte_second_buf: u32, + pub ui_min_write_byte_second_buf: u32, + pub ui_read_position_second_buf: u32, + } + + pub struct SceCtrlData { + pub timestamp: u32, + pub buttons: i32, + pub lx: u8, + pub ly: u8, + pub rsrv: [u8; 6], + } + + pub struct SceCtrlLatch { + pub ui_make: u32, + pub ui_break: u32, + pub ui_press: u32, + pub ui_release: u32, + } + + pub struct GeStack { + pub stack: [u32; 8], + } + + pub struct GeCallbackData { + pub signal_func: ::Option, + pub signal_arg: *mut c_void, + pub finish_func: ::Option, + pub finish_arg: *mut c_void, + } + + pub struct GeListArgs { + pub size: u32, + pub context: *mut GeContext, + pub num_stacks: u32, + pub stacks: *mut GeStack, + } + + pub struct GeBreakParam { + pub buf: [u32; 4], + } + + pub struct SceKernelLoadExecParam { + pub size: usize, + pub args: usize, + pub argp: *mut c_void, + pub key: *const u8, + } + + pub struct timeval { + pub tv_sec: i32, + pub tv_usec: i32, + } + + pub struct timezone { + pub tz_minutes_west: i32, + pub tz_dst_time: i32, + } + + pub struct IntrHandlerOptionParam { + size: i32, + entry: u32, + common: u32, + gp: u32, + intr_code: u16, + sub_count: u16, + intr_level: u16, + enabled: u16, + calls: u32, + field_1c: u32, + total_clock_lo: u32, + total_clock_hi: u32, + min_clock_lo: u32, + min_clock_hi: u32, + max_clock_lo: u32, + max_clock_hi: u32, + } + + pub struct SceKernelLMOption { + pub size: usize, + pub m_pid_text: SceUid, + pub m_pid_data: SceUid, + pub flags: u32, + pub position: u8, + pub access: u8, + pub c_reserved: [u8; 2usize], + } + + pub struct SceKernelSMOption { + pub size: usize, + pub m_pid_stack: SceUid, + pub stack_size: usize, + pub priority: i32, + pub attribute: u32, + } + + pub struct SceKernelModuleInfo { + pub size: usize, + pub n_segment: u8, + pub reserved: [u8; 3usize], + pub segment_addr: [i32; 4usize], + pub segment_size: [i32; 4usize], + pub entry_addr: u32, + pub gp_value: u32, + pub text_addr: u32, + pub text_size: u32, + pub data_size: u32, + pub bss_size: u32, + pub attribute: u16, + pub version: [u8; 2usize], + pub name: [u8; 28usize], + } + + pub struct DebugProfilerRegs { + pub enable: u32, + pub systemck: u32, + pub cpuck: u32, + pub internal: u32, + pub memory: u32, + pub copz: u32, + pub vfpu: u32, + pub sleep: u32, + pub bus_access: u32, + pub uncached_load: u32, + pub uncached_store: u32, + pub cached_load: u32, + pub cached_store: u32, + pub i_miss: u32, + pub d_miss: u32, + pub d_writeback: u32, + pub cop0_inst: u32, + pub fpu_inst: u32, + pub vfpu_inst: u32, + pub local_bus: u32, + } + + pub struct SceKernelSysClock { + pub low: u32, + pub hi: u32, + } + + pub struct SceKernelThreadOptParam { + pub size: usize, + pub stack_mpid: SceUid, + } + + pub struct SceKernelThreadInfo { + pub size: usize, + pub name: [u8; 32], + pub attr: u32, + pub status: i32, + pub entry: SceKernelThreadEntry, + pub stack: *mut c_void, + pub stack_size: i32, + pub gp_reg: *mut c_void, + pub init_priority: i32, + pub current_priority: i32, + pub wait_type: i32, + pub wait_id: SceUid, + pub wakeup_count: i32, + pub exit_status: i32, + pub run_clocks: SceKernelSysClock, + pub intr_preempt_count: u32, + pub thread_preempt_count: u32, + pub release_count: u32, + } + + pub struct SceKernelThreadRunStatus { + pub size: usize, + pub status: i32, + pub current_priority: i32, + pub wait_type: i32, + pub wait_id: i32, + pub wakeup_count: i32, + pub run_clocks: SceKernelSysClock, + pub intr_preempt_count: u32, + pub thread_preempt_count: u32, + pub release_count: u32, + } + + pub struct SceKernelSemaOptParam { + pub size: usize, + } + + pub struct SceKernelSemaInfo { + pub size: usize, + pub name: [u8; 32], + pub attr: u32, + pub init_count: i32, + pub current_count: i32, + pub max_count: i32, + pub num_wait_threads: i32, + } + + pub struct SceKernelEventFlagInfo { + pub size: usize, + pub name: [u8; 32], + pub attr: u32, + pub init_pattern: u32, + pub current_pattern: u32, + pub num_wait_threads: i32, + } + + pub struct SceKernelEventFlagOptParam { + pub size: usize, + } + + pub struct SceKernelMbxOptParam { + pub size: usize, + } + + pub struct SceKernelMbxInfo { + pub size: usize, + pub name: [u8; 32usize], + pub attr: u32, + pub num_wait_threads: i32, + pub num_messages: i32, + pub first_message: *mut c_void, + } + + pub struct SceKernelVTimerInfo { + pub size: usize, + pub name: [u8; 32], + pub active: i32, + pub base: SceKernelSysClock, + pub current: SceKernelSysClock, + pub schedule: SceKernelSysClock, + pub handler: SceKernelVTimerHandler, + pub common: *mut c_void, + } + + pub struct SceKernelThreadEventHandlerInfo { + pub size: usize, + pub name: [u8; 32], + pub thread_id: SceUid, + pub mask: i32, + pub handler: SceKernelThreadEventHandler, + pub common: *mut c_void, + } + + pub struct SceKernelAlarmInfo { + pub size: usize, + pub schedule: SceKernelSysClock, + pub handler: SceKernelAlarmHandler, + pub common: *mut c_void, + } + + pub struct SceKernelSystemStatus { + pub size: usize, + pub status: u32, + pub idle_clocks: SceKernelSysClock, + pub comes_out_of_idle_count: u32, + pub thread_switch_count: u32, + pub vfpu_switch_count: u32, + } + + pub struct SceKernelMppInfo { + pub size: usize, + pub name: [u8; 32], + pub attr: u32, + pub buf_size: i32, + pub free_size: i32, + pub num_send_wait_threads: i32, + pub num_receive_wait_threads: i32, + } + + pub struct SceKernelVplOptParam { + pub size: usize, + } + + pub struct SceKernelVplInfo { + pub size: usize, + pub name: [u8; 32], + pub attr: u32, + pub pool_size: i32, + pub free_size: i32, + pub num_wait_threads: i32, + } + + pub struct SceKernelFplOptParam { + pub size: usize, + } + + pub struct SceKernelFplInfo { + pub size: usize, + pub name: [u8; 32usize], + pub attr: u32, + pub block_size: i32, + pub num_blocks: i32, + pub free_blocks: i32, + pub num_wait_threads: i32, + } + + pub struct SceKernelVTimerOptParam { + pub size: usize, + } + + pub struct SceKernelCallbackInfo { + pub size: usize, + pub name: [u8; 32usize], + pub thread_id: SceUid, + pub callback: SceKernelCallbackFunction, + pub common: *mut c_void, + pub notify_count: i32, + pub notify_arg: i32, + } + + pub struct UsbCamSetupStillParam { + pub size: i32, + pub resolution: UsbCamResolution, + pub jpeg_size: i32, + pub reverse_flags: i32, + pub delay: UsbCamDelay, + pub comp_level: i32, + } + + pub struct UsbCamSetupStillExParam { + pub size: i32, + pub unk: u32, + pub resolution: UsbCamResolutionEx, + pub jpeg_size: i32, + pub comp_level: i32, + pub unk2: u32, + pub unk3: u32, + pub flip: i32, + pub mirror: i32, + pub delay: UsbCamDelay, + pub unk4: [u32; 5usize], + } + + pub struct UsbCamSetupVideoParam { + pub size: i32, + pub resolution: UsbCamResolution, + pub framerate: UsbCamFrameRate, + pub white_balance: UsbCamWb, + pub saturation: i32, + pub brightness: i32, + pub contrast: i32, + pub sharpness: i32, + pub effect_mode: UsbCamEffectMode, + pub frame_size: i32, + pub unk: u32, + pub evl_evel: UsbCamEvLevel, + } + + pub struct UsbCamSetupVideoExParam { + pub size: i32, + pub unk: u32, + pub resolution: UsbCamResolutionEx, + pub framerate: UsbCamFrameRate, + pub unk2: u32, + pub unk3: u32, + pub white_balance: UsbCamWb, + pub saturation: i32, + pub brightness: i32, + pub contrast: i32, + pub sharpness: i32, + pub unk4: u32, + pub unk5: u32, + pub unk6: [u32; 3usize], + pub effect_mode: UsbCamEffectMode, + pub unk7: u32, + pub unk8: u32, + pub unk9: u32, + pub unk10: u32, + pub unk11: u32, + pub frame_size: i32, + pub unk12: u32, + pub ev_level: UsbCamEvLevel, + } + + pub struct ScePspDateTime { + pub year: u16, + pub month: u16, + pub day: u16, + pub hour: u16, + pub minutes: u16, + pub seconds: u16, + pub microseconds: u32, + } + + pub struct SceIoStat { + pub st_mode: i32, + pub st_attr: i32, + pub st_size: i64, + pub st_ctime: ScePspDateTime, + pub st_atime: ScePspDateTime, + pub st_mtime: ScePspDateTime, + pub st_private: [u32; 6usize], + } + + pub struct UmdInfo { + pub size: u32, + pub type_: UmdType, + } + + pub struct SceMpegRingbuffer { + pub packets: i32, + pub unk0: u32, + pub unk1: u32, + pub unk2: u32, + pub unk3: u32, + pub data: *mut c_void, + pub callback: SceMpegRingbufferCb, + pub cb_param: *mut c_void, + pub unk4: u32, + pub unk5: u32, + pub sce_mpeg: *mut c_void, + } + + pub struct SceMpegAu { + pub pts_msb: u32, + pub pts: u32, + pub dts_msb: u32, + pub dts: u32, + pub es_buffer: u32, + pub au_size: u32, + } + + pub struct SceMpegAvcMode { + pub unk0: i32, + pub pixel_format: super::DisplayPixelFormat, + } + + #[repr(align(64))] + pub struct SceMpegLLI { + pub src: *mut c_void, + pub dst: *mut c_void, + pub next: *mut c_void, + pub size: i32, + } + + #[repr(align(64))] + pub struct SceMpegYCrCbBuffer { + pub frame_buffer_height16: i32, + pub frame_buffer_width16: i32, + pub unknown: i32, + pub unknown2: i32, + pub y_buffer: *mut c_void, + pub y_buffer2: *mut c_void, + pub cr_buffer: *mut c_void, + pub cb_buffer: *mut c_void, + pub cr_buffer2: *mut c_void, + pub cb_buffer2: *mut c_void, + + pub frame_height: i32, + pub frame_width: i32, + pub frame_buffer_width: i32, + pub unknown3: [i32; 11usize], + } + + pub struct ScePspSRect { + pub x: i16, + pub y: i16, + pub w: i16, + pub h: i16, + } + + pub struct ScePspIRect { + pub x: i32, + pub y: i32, + pub w: i32, + pub h: i32, + } + + pub struct ScePspL64Rect { + pub x: u64, + pub y: u64, + pub w: u64, + pub h: u64, + } + + pub struct ScePspSVector2 { + pub x: i16, + pub y: i16, + } + + pub struct ScePspIVector2 { + pub x: i32, + pub y: i32, + } + + pub struct ScePspL64Vector2 { + pub x: u64, + pub y: u64, + } + + pub struct ScePspSVector3 { + pub x: i16, + pub y: i16, + pub z: i16, + } + + pub struct ScePspIVector3 { + pub x: i32, + pub y: i32, + pub z: i32, + } + + pub struct ScePspL64Vector3 { + pub x: u64, + pub y: u64, + pub z: u64, + } + + pub struct ScePspSVector4 { + pub x: i16, + pub y: i16, + pub z: i16, + pub w: i16, + } + + pub struct ScePspIVector4 { + pub x: i32, + pub y: i32, + pub z: i32, + pub w: i32, + } + + pub struct ScePspL64Vector4 { + pub x: u64, + pub y: u64, + pub z: u64, + pub w: u64, + } + + pub struct ScePspIMatrix2 { + pub x: ScePspIVector2, + pub y: ScePspIVector2, + } + + pub struct ScePspIMatrix3 { + pub x: ScePspIVector3, + pub y: ScePspIVector3, + pub z: ScePspIVector3, + } + + #[repr(align(16))] + pub struct ScePspIMatrix4 { + pub x: ScePspIVector4, + pub y: ScePspIVector4, + pub z: ScePspIVector4, + pub w: ScePspIVector4, + } + + pub struct ScePspIMatrix4Unaligned { + pub x: ScePspIVector4, + pub y: ScePspIVector4, + pub z: ScePspIVector4, + pub w: ScePspIVector4, + } + + pub struct SceMp3InitArg { + pub mp3_stream_start: u32, + pub unk1: u32, + pub mp3_stream_end: u32, + pub unk2: u32, + pub mp3_buf: *mut c_void, + pub mp3_buf_size: i32, + pub pcm_buf: *mut c_void, + pub pcm_buf_size: i32, + } + + pub struct OpenPSID { + pub data: [u8; 16usize], + } + + pub struct UtilityDialogCommon { + pub size: u32, + pub language: SystemParamLanguage, + pub button_accept: UtilityDialogButtonAccept, + pub graphics_thread: i32, + pub access_thread: i32, + pub font_thread: i32, + pub sound_thread: i32, + pub result: i32, + pub reserved: [i32; 4usize], + } + + pub struct UtilityNetconfAdhoc { + pub name: [u8; 8usize], + pub timeout: u32, + } + + pub struct UtilityNetconfData { + pub base: UtilityDialogCommon, + pub action: UtilityNetconfAction, + pub adhocparam: *mut UtilityNetconfAdhoc, + pub hotspot: i32, + pub hotspot_connected: i32, + pub wifisp: i32, + } + + pub struct UtilitySavedataFileData { + pub buf: *mut c_void, + pub buf_size: usize, + pub size: usize, + pub unknown: i32, + } + + pub struct UtilitySavedataListSaveNewData { + pub icon0: UtilitySavedataFileData, + pub title: *mut u8, + } + + pub struct UtilityGameSharingParams { + pub base: UtilityDialogCommon, + pub unknown1: i32, + pub unknown2: i32, + pub name: [u8; 8usize], + pub unknown3: i32, + pub unknown4: i32, + pub unknown5: i32, + pub result: i32, + pub filepath: *mut u8, + pub mode: UtilityGameSharingMode, + pub datatype: UtilityGameSharingDataType, + pub data: *mut c_void, + pub datasize: u32, + } + + pub struct UtilityHtmlViewerParam { + pub base: UtilityDialogCommon, + pub memaddr: *mut c_void, + pub memsize: u32, + pub unknown1: i32, + pub unknown2: i32, + pub initialurl: *mut u8, + pub numtabs: u32, + pub interfacemode: UtilityHtmlViewerInterfaceMode, + pub options: i32, + pub dldirname: *mut u8, + pub dlfilename: *mut u8, + pub uldirname: *mut u8, + pub ulfilename: *mut u8, + pub cookiemode: UtilityHtmlViewerCookieMode, + pub unknown3: u32, + pub homeurl: *mut u8, + pub textsize: UtilityHtmlViewerTextSize, + pub displaymode: UtilityHtmlViewerDisplayMode, + pub connectmode: UtilityHtmlViewerConnectMode, + pub disconnectmode: UtilityHtmlViewerDisconnectMode, + pub memused: u32, + pub unknown4: [i32; 10usize], + } + + pub struct SceUtilityOskData { + pub unk_00: i32, + pub unk_04: i32, + pub language: SceUtilityOskInputLanguage, + pub unk_12: i32, + pub inputtype: SceUtilityOskInputType, + pub lines: i32, + pub unk_24: i32, + pub desc: *mut u16, + pub intext: *mut u16, + pub outtextlength: i32, + pub outtext: *mut u16, + pub result: SceUtilityOskResult, + pub outtextlimit: i32, + } + + pub struct SceUtilityOskParams { + pub base: UtilityDialogCommon, + pub datacount: i32, + pub data: *mut SceUtilityOskData, + pub state: SceUtilityOskState, + pub unk_60: i32, + } + + pub struct SceNetMallocStat { + pub pool: i32, + pub maximum: i32, + pub free: i32, + } + + pub struct SceNetAdhocctlAdhocId { + pub unknown: i32, + pub adhoc_id: [u8; 9usize], + pub unk: [u8; 3usize], + } + + pub struct SceNetAdhocctlScanInfo { + pub next: *mut SceNetAdhocctlScanInfo, + pub channel: i32, + pub name: [u8; 8usize], + pub bssid: [u8; 6usize], + pub unknown: [u8; 2usize], + pub unknown2: i32, + } + + pub struct SceNetAdhocctlGameModeInfo { + pub count: i32, + pub macs: [[u8; 6usize]; 16usize], + } + + pub struct SceNetAdhocPtpStat { + pub next: *mut SceNetAdhocPtpStat, + pub ptp_id: i32, + pub mac: [u8; 6usize], + pub peermac: [u8; 6usize], + pub port: u16, + pub peerport: u16, + pub sent_data: u32, + pub rcvd_data: u32, + pub state: ScePspnetAdhocPtpState, + } + + pub struct SceNetAdhocPdpStat { + pub next: *mut SceNetAdhocPdpStat, + pub pdp_id: i32, + pub mac: [u8; 6usize], + pub port: u16, + pub rcvd_data: u32, + } + + pub struct AdhocPoolStat { + pub size: i32, + pub maxsize: i32, + pub freesize: i32, + } +} + +s_no_extra_traits! { + #[allow(missing_debug_implementations)] + pub struct GeContext { + pub context: [u32; 512], + } + + #[allow(missing_debug_implementations)] + pub struct SceKernelUtilsSha1Context { + pub h: [u32; 5usize], + pub us_remains: u16, + pub us_computed: u16, + pub ull_total_len: u64, + pub buf: [u8; 64usize], + } + + #[allow(missing_debug_implementations)] + pub struct SceKernelUtilsMt19937Context { + pub count: u32, + pub state: [u32; 624usize], + } + + #[allow(missing_debug_implementations)] + pub struct SceKernelUtilsMd5Context { + pub h: [u32; 4usize], + pub pad: u32, + pub us_remains: u16, + pub us_computed: u16, + pub ull_total_len: u64, + pub buf: [u8; 64usize], + } + + #[allow(missing_debug_implementations)] + pub struct SceIoDirent { + pub d_stat: SceIoStat, + pub d_name: [u8; 256usize], + pub d_private: *mut c_void, + pub dummy: i32, + } + + #[cfg_attr(feature = "extra_traits", derive(Debug))] + pub struct ScePspFRect { + pub x: f32, + pub y: f32, + pub w: f32, + pub h: f32, + } + + #[repr(align(16))] + #[cfg_attr(feature = "extra_traits", derive(Debug))] + pub struct ScePspFVector3 { + pub x: f32, + pub y: f32, + pub z: f32, + } + + #[repr(align(16))] + #[cfg_attr(feature = "extra_traits", derive(Debug))] + pub struct ScePspFVector4 { + pub x: f32, + pub y: f32, + pub z: f32, + pub w: f32, + } + + #[cfg_attr(feature = "extra_traits", derive(Debug))] + pub struct ScePspFVector4Unaligned { + pub x: f32, + pub y: f32, + pub z: f32, + pub w: f32, + } + + #[cfg_attr(feature = "extra_traits", derive(Debug))] + pub struct ScePspFVector2 { + pub x: f32, + pub y: f32, + } + + #[cfg_attr(feature = "extra_traits", derive(Debug))] + pub struct ScePspFMatrix2 { + pub x: ScePspFVector2, + pub y: ScePspFVector2, + } + + #[cfg_attr(feature = "extra_traits", derive(Debug))] + pub struct ScePspFMatrix3 { + pub x: ScePspFVector3, + pub y: ScePspFVector3, + pub z: ScePspFVector3, + } + + #[cfg_attr(feature = "extra_traits", derive(Debug))] + #[repr(align(16))] + pub struct ScePspFMatrix4 { + pub x: ScePspFVector4, + pub y: ScePspFVector4, + pub z: ScePspFVector4, + pub w: ScePspFVector4, + } + + #[allow(missing_debug_implementations)] + pub struct ScePspFMatrix4Unaligned { + pub x: ScePspFVector4, + pub y: ScePspFVector4, + pub z: ScePspFVector4, + pub w: ScePspFVector4, + } + + #[allow(missing_debug_implementations)] + pub union ScePspVector3 { + pub fv: ScePspFVector3, + pub iv: ScePspIVector3, + pub f: [f32; 3usize], + pub i: [i32; 3usize], + } + + #[allow(missing_debug_implementations)] + pub union ScePspVector4 { + pub fv: ScePspFVector4, + pub iv: ScePspIVector4, + pub qw: u128, + pub f: [f32; 4usize], + pub i: [i32; 4usize], + } + + #[allow(missing_debug_implementations)] + pub union ScePspMatrix2 { + pub fm: ScePspFMatrix2, + pub im: ScePspIMatrix2, + pub fv: [ScePspFVector2; 2usize], + pub iv: [ScePspIVector2; 2usize], + pub v: [ScePspVector2; 2usize], + pub f: [[f32; 2usize]; 2usize], + pub i: [[i32; 2usize]; 2usize], + } + + #[allow(missing_debug_implementations)] + pub union ScePspMatrix3 { + pub fm: ScePspFMatrix3, + pub im: ScePspIMatrix3, + pub fv: [ScePspFVector3; 3usize], + pub iv: [ScePspIVector3; 3usize], + pub v: [ScePspVector3; 3usize], + pub f: [[f32; 3usize]; 3usize], + pub i: [[i32; 3usize]; 3usize], + } + + #[allow(missing_debug_implementations)] + pub union ScePspVector2 { + pub fv: ScePspFVector2, + pub iv: ScePspIVector2, + pub f: [f32; 2usize], + pub i: [i32; 2usize], + } + + #[allow(missing_debug_implementations)] + pub union ScePspMatrix4 { + pub fm: ScePspFMatrix4, + pub im: ScePspIMatrix4, + pub fv: [ScePspFVector4; 4usize], + pub iv: [ScePspIVector4; 4usize], + pub v: [ScePspVector4; 4usize], + pub f: [[f32; 4usize]; 4usize], + pub i: [[i32; 4usize]; 4usize], + } + + #[allow(missing_debug_implementations)] + pub struct Key { + pub key_type: KeyType, + pub name: [u8; 256usize], + pub name_len: u32, + pub unk2: u32, + pub unk3: u32, + } + + #[allow(missing_debug_implementations)] + pub struct UtilityMsgDialogParams { + pub base: UtilityDialogCommon, + pub unknown: i32, + pub mode: UtilityMsgDialogMode, + pub error_value: u32, + pub message: [u8; 512usize], + pub options: i32, + pub button_pressed: UtilityMsgDialogPressed, + } + + #[allow(missing_debug_implementations)] + pub union UtilityNetData { + pub as_uint: u32, + pub as_string: [u8; 128usize], + } + + #[allow(missing_debug_implementations)] + pub struct UtilitySavedataSFOParam { + pub title: [u8; 128usize], + pub savedata_title: [u8; 128usize], + pub detail: [u8; 1024usize], + pub parental_level: u8, + pub unknown: [u8; 3usize], + } + + #[allow(missing_debug_implementations)] + pub struct SceUtilitySavedataParam { + pub base: UtilityDialogCommon, + pub mode: UtilitySavedataMode, + pub unknown1: i32, + pub overwrite: i32, + pub game_name: [u8; 13usize], + pub reserved: [u8; 3usize], + pub save_name: [u8; 20usize], + pub save_name_list: *mut [u8; 20usize], + pub file_name: [u8; 13usize], + pub reserved1: [u8; 3usize], + pub data_buf: *mut c_void, + pub data_buf_size: usize, + pub data_size: usize, + pub sfo_param: UtilitySavedataSFOParam, + pub icon0_file_data: UtilitySavedataFileData, + pub icon1_file_data: UtilitySavedataFileData, + pub pic1_file_data: UtilitySavedataFileData, + pub snd0_file_data: UtilitySavedataFileData, + pub new_data: *mut UtilitySavedataListSaveNewData, + pub focus: UtilitySavedataFocus, + pub unknown2: [i32; 4usize], + pub key: [u8; 16], + pub unknown3: [u8; 20], + } + + #[allow(missing_debug_implementations)] + pub struct SceNetAdhocctlPeerInfo { + pub next: *mut SceNetAdhocctlPeerInfo, + pub nickname: [u8; 128usize], + pub mac: [u8; 6usize], + pub unknown: [u8; 6usize], + pub timestamp: u32, + } + + #[allow(missing_debug_implementations)] + pub struct SceNetAdhocctlParams { + pub channel: i32, + pub name: [u8; 8usize], + pub bssid: [u8; 6usize], + pub nickname: [u8; 128usize], + } + + #[cfg_attr(feature = "extra_traits", allow(missing_debug_implementations))] + pub union SceNetApctlInfo { + pub name: [u8; 64usize], + pub bssid: [u8; 6usize], + pub ssid: [u8; 32usize], + pub ssid_length: u32, + pub security_type: u32, + pub strength: u8, + pub channel: u8, + pub power_save: u8, + pub ip: [u8; 16usize], + pub sub_net_mask: [u8; 16usize], + pub gateway: [u8; 16usize], + pub primary_dns: [u8; 16usize], + pub secondary_dns: [u8; 16usize], + pub use_proxy: u32, + pub proxy_url: [u8; 128usize], + pub proxy_port: u16, + pub eap_type: u32, + pub start_browser: u32, + pub wifisp: u32, + } +} + +pub const INT_MIN: c_int = -2147483648; +pub const INT_MAX: c_int = 2147483647; + +pub const AUDIO_VOLUME_MAX: u32 = 0x8000; +pub const AUDIO_CHANNEL_MAX: u32 = 8; +pub const AUDIO_NEXT_CHANNEL: i32 = -1; +pub const AUDIO_SAMPLE_MIN: u32 = 64; +pub const AUDIO_SAMPLE_MAX: u32 = 65472; + +pub const PSP_CTRL_SELECT: i32 = 0x000001; +pub const PSP_CTRL_START: i32 = 0x000008; +pub const PSP_CTRL_UP: i32 = 0x000010; +pub const PSP_CTRL_RIGHT: i32 = 0x000020; +pub const PSP_CTRL_DOWN: i32 = 0x000040; +pub const PSP_CTRL_LEFT: i32 = 0x000080; +pub const PSP_CTRL_LTRIGGER: i32 = 0x000100; +pub const PSP_CTRL_RTRIGGER: i32 = 0x000200; +pub const PSP_CTRL_TRIANGLE: i32 = 0x001000; +pub const PSP_CTRL_CIRCLE: i32 = 0x002000; +pub const PSP_CTRL_CROSS: i32 = 0x004000; +pub const PSP_CTRL_SQUARE: i32 = 0x008000; +pub const PSP_CTRL_HOME: i32 = 0x010000; +pub const PSP_CTRL_HOLD: i32 = 0x020000; +pub const PSP_CTRL_NOTE: i32 = 0x800000; +pub const PSP_CTRL_SCREEN: i32 = 0x400000; +pub const PSP_CTRL_VOLUP: i32 = 0x100000; +pub const PSP_CTRL_VOLDOWN: i32 = 0x200000; +pub const PSP_CTRL_WLAN_UP: i32 = 0x040000; +pub const PSP_CTRL_REMOTE: i32 = 0x080000; +pub const PSP_CTRL_DISC: i32 = 0x1000000; +pub const PSP_CTRL_MS: i32 = 0x2000000; + +pub const USB_CAM_PID: i32 = 0x282; +pub const USB_BUS_DRIVER_NAME: &str = "USBBusDriver"; +pub const USB_CAM_DRIVER_NAME: &str = "USBCamDriver"; +pub const USB_CAM_MIC_DRIVER_NAME: &str = "USBCamMicDriver"; +pub const USB_STOR_DRIVER_NAME: &str = "USBStor_Driver"; + +pub const ACTIVATED: i32 = 0x200; +pub const CONNECTED: i32 = 0x020; +pub const ESTABLISHED: i32 = 0x002; + +pub const USB_CAM_FLIP: i32 = 1; +pub const USB_CAM_MIRROR: i32 = 0x100; + +pub const THREAD_ATTR_VFPU: i32 = 0x00004000; +pub const THREAD_ATTR_USER: i32 = 0x80000000; +pub const THREAD_ATTR_USBWLAN: i32 = 0xa0000000; +pub const THREAD_ATTR_VSH: i32 = 0xc0000000; +pub const THREAD_ATTR_SCRATCH_SRAM: i32 = 0x00008000; +pub const THREAD_ATTR_NO_FILLSTACK: i32 = 0x00100000; +pub const THREAD_ATTR_CLEAR_STACK: i32 = 0x00200000; + +pub const EVENT_WAIT_MULTIPLE: i32 = 0x200; + +pub const EVENT_WAIT_AND: i32 = 0; +pub const EVENT_WAIT_OR: i32 = 1; +pub const EVENT_WAIT_CLEAR: i32 = 0x20; + +pub const POWER_INFO_POWER_SWITCH: i32 = 0x80000000; +pub const POWER_INFO_HOLD_SWITCH: i32 = 0x40000000; +pub const POWER_INFO_STANDBY: i32 = 0x00080000; +pub const POWER_INFO_RESUME_COMPLETE: i32 = 0x00040000; +pub const POWER_INFO_RESUMING: i32 = 0x00020000; +pub const POWER_INFO_SUSPENDING: i32 = 0x00010000; +pub const POWER_INFO_AC_POWER: i32 = 0x00001000; +pub const POWER_INFO_BATTERY_LOW: i32 = 0x00000100; +pub const POWER_INFO_BATTERY_EXIST: i32 = 0x00000080; +pub const POWER_INFO_BATTERY_POWER: i32 = 0x0000007; + +pub const FIO_S_IFLNK: i32 = 0x4000; +pub const FIO_S_IFDIR: i32 = 0x1000; +pub const FIO_S_IFREG: i32 = 0x2000; +pub const FIO_S_ISUID: i32 = 0x0800; +pub const FIO_S_ISGID: i32 = 0x0400; +pub const FIO_S_ISVTX: i32 = 0x0200; +pub const FIO_S_IRUSR: i32 = 0x0100; +pub const FIO_S_IWUSR: i32 = 0x0080; +pub const FIO_S_IXUSR: i32 = 0x0040; +pub const FIO_S_IRGRP: i32 = 0x0020; +pub const FIO_S_IWGRP: i32 = 0x0010; +pub const FIO_S_IXGRP: i32 = 0x0008; +pub const FIO_S_IROTH: i32 = 0x0004; +pub const FIO_S_IWOTH: i32 = 0x0002; +pub const FIO_S_IXOTH: i32 = 0x0001; + +pub const FIO_SO_IFLNK: i32 = 0x0008; +pub const FIO_SO_IFDIR: i32 = 0x0010; +pub const FIO_SO_IFREG: i32 = 0x0020; +pub const FIO_SO_IROTH: i32 = 0x0004; +pub const FIO_SO_IWOTH: i32 = 0x0002; +pub const FIO_SO_IXOTH: i32 = 0x0001; + +pub const PSP_O_RD_ONLY: i32 = 0x0001; +pub const PSP_O_WR_ONLY: i32 = 0x0002; +pub const PSP_O_RD_WR: i32 = 0x0003; +pub const PSP_O_NBLOCK: i32 = 0x0004; +pub const PSP_O_DIR: i32 = 0x0008; +pub const PSP_O_APPEND: i32 = 0x0100; +pub const PSP_O_CREAT: i32 = 0x0200; +pub const PSP_O_TRUNC: i32 = 0x0400; +pub const PSP_O_EXCL: i32 = 0x0800; +pub const PSP_O_NO_WAIT: i32 = 0x8000; + +pub const UMD_NOT_PRESENT: i32 = 0x01; +pub const UMD_PRESENT: i32 = 0x02; +pub const UMD_CHANGED: i32 = 0x04; +pub const UMD_INITING: i32 = 0x08; +pub const UMD_INITED: i32 = 0x10; +pub const UMD_READY: i32 = 0x20; + +pub const PLAY_PAUSE: i32 = 0x1; +pub const FORWARD: i32 = 0x4; +pub const BACK: i32 = 0x8; +pub const VOL_UP: i32 = 0x10; +pub const VOL_DOWN: i32 = 0x20; +pub const HOLD: i32 = 0x80; + +pub const GU_PI: f32 = 3.141593; + +pub const GU_TEXTURE_8BIT: i32 = 1; +pub const GU_TEXTURE_16BIT: i32 = 2; +pub const GU_TEXTURE_32BITF: i32 = 3; +pub const GU_COLOR_5650: i32 = 4 << 2; +pub const GU_COLOR_5551: i32 = 5 << 2; +pub const GU_COLOR_4444: i32 = 6 << 2; +pub const GU_COLOR_8888: i32 = 7 << 2; +pub const GU_NORMAL_8BIT: i32 = 1 << 5; +pub const GU_NORMAL_16BIT: i32 = 2 << 5; +pub const GU_NORMAL_32BITF: i32 = 3 << 5; +pub const GU_VERTEX_8BIT: i32 = 1 << 7; +pub const GU_VERTEX_16BIT: i32 = 2 << 7; +pub const GU_VERTEX_32BITF: i32 = 3 << 7; +pub const GU_WEIGHT_8BIT: i32 = 1 << 9; +pub const GU_WEIGHT_16BIT: i32 = 2 << 9; +pub const GU_WEIGHT_32BITF: i32 = 3 << 9; +pub const GU_INDEX_8BIT: i32 = 1 << 11; +pub const GU_INDEX_16BIT: i32 = 2 << 11; +pub const GU_WEIGHTS1: i32 = (((1 - 1) & 7) << 14) as i32; +pub const GU_WEIGHTS2: i32 = (((2 - 1) & 7) << 14) as i32; +pub const GU_WEIGHTS3: i32 = (((3 - 1) & 7) << 14) as i32; +pub const GU_WEIGHTS4: i32 = (((4 - 1) & 7) << 14) as i32; +pub const GU_WEIGHTS5: i32 = (((5 - 1) & 7) << 14) as i32; +pub const GU_WEIGHTS6: i32 = (((6 - 1) & 7) << 14) as i32; +pub const GU_WEIGHTS7: i32 = (((7 - 1) & 7) << 14) as i32; +pub const GU_WEIGHTS8: i32 = (((8 - 1) & 7) << 14) as i32; +pub const GU_VERTICES1: i32 = (((1 - 1) & 7) << 18) as i32; +pub const GU_VERTICES2: i32 = (((2 - 1) & 7) << 18) as i32; +pub const GU_VERTICES3: i32 = (((3 - 1) & 7) << 18) as i32; +pub const GU_VERTICES4: i32 = (((4 - 1) & 7) << 18) as i32; +pub const GU_VERTICES5: i32 = (((5 - 1) & 7) << 18) as i32; +pub const GU_VERTICES6: i32 = (((6 - 1) & 7) << 18) as i32; +pub const GU_VERTICES7: i32 = (((7 - 1) & 7) << 18) as i32; +pub const GU_VERTICES8: i32 = (((8 - 1) & 7) << 18) as i32; +pub const GU_TRANSFORM_2D: i32 = 1 << 23; +pub const GU_TRANSFORM_3D: i32 = 0; + +pub const GU_COLOR_BUFFER_BIT: i32 = 1; +pub const GU_STENCIL_BUFFER_BIT: i32 = 2; +pub const GU_DEPTH_BUFFER_BIT: i32 = 4; +pub const GU_FAST_CLEAR_BIT: i32 = 16; + +pub const GU_AMBIENT: i32 = 1; +pub const GU_DIFFUSE: i32 = 2; +pub const GU_SPECULAR: i32 = 4; +pub const GU_UNKNOWN_LIGHT_COMPONENT: i32 = 8; + +pub const SYSTEM_REGISTRY: [u8; 7] = *b"/system"; +pub const REG_KEYNAME_SIZE: u32 = 27; + +pub const UTILITY_MSGDIALOG_ERROR: i32 = 0; +pub const UTILITY_MSGDIALOG_TEXT: i32 = 1; +pub const UTILITY_MSGDIALOG_YES_NO_BUTTONS: i32 = 0x10; +pub const UTILITY_MSGDIALOG_DEFAULT_NO: i32 = 0x100; + +pub const UTILITY_HTMLVIEWER_OPEN_SCE_START_PAGE: i32 = 0x000001; +pub const UTILITY_HTMLVIEWER_DISABLE_STARTUP_LIMITS: i32 = 0x000002; +pub const UTILITY_HTMLVIEWER_DISABLE_EXIT_DIALOG: i32 = 0x000004; +pub const UTILITY_HTMLVIEWER_DISABLE_CURSOR: i32 = 0x000008; +pub const UTILITY_HTMLVIEWER_DISABLE_DOWNLOAD_COMPLETE_DIALOG: i32 = 0x000010; +pub const UTILITY_HTMLVIEWER_DISABLE_DOWNLOAD_START_DIALOG: i32 = 0x000020; +pub const UTILITY_HTMLVIEWER_DISABLE_DOWNLOAD_DESTINATION_DIALOG: i32 = 0x000040; +pub const UTILITY_HTMLVIEWER_LOCK_DOWNLOAD_DESTINATION_DIALOG: i32 = 0x000080; +pub const UTILITY_HTMLVIEWER_DISABLE_TAB_DISPLAY: i32 = 0x000100; +pub const UTILITY_HTMLVIEWER_ENABLE_ANALOG_HOLD: i32 = 0x000200; +pub const UTILITY_HTMLVIEWER_ENABLE_FLASH: i32 = 0x000400; +pub const UTILITY_HTMLVIEWER_DISABLE_LRTRIGGER: i32 = 0x000800; + +extern "C" { + pub fn sceAudioChReserve(channel: i32, sample_count: i32, format: AudioFormat) -> i32; + pub fn sceAudioChRelease(channel: i32) -> i32; + pub fn sceAudioOutput(channel: i32, vol: i32, buf: *mut c_void) -> i32; + pub fn sceAudioOutputBlocking(channel: i32, vol: i32, buf: *mut c_void) -> i32; + pub fn sceAudioOutputPanned( + channel: i32, + left_vol: i32, + right_vol: i32, + buf: *mut c_void, + ) -> i32; + pub fn sceAudioOutputPannedBlocking( + channel: i32, + left_vol: i32, + right_vol: i32, + buf: *mut c_void, + ) -> i32; + pub fn sceAudioGetChannelRestLen(channel: i32) -> i32; + pub fn sceAudioGetChannelRestLength(channel: i32) -> i32; + pub fn sceAudioSetChannelDataLen(channel: i32, sample_count: i32) -> i32; + pub fn sceAudioChangeChannelConfig(channel: i32, format: AudioFormat) -> i32; + pub fn sceAudioChangeChannelVolume(channel: i32, left_vol: i32, right_vol: i32) -> i32; + pub fn sceAudioOutput2Reserve(sample_count: i32) -> i32; + pub fn sceAudioOutput2Release() -> i32; + pub fn sceAudioOutput2ChangeLength(sample_count: i32) -> i32; + pub fn sceAudioOutput2OutputBlocking(vol: i32, buf: *mut c_void) -> i32; + pub fn sceAudioOutput2GetRestSample() -> i32; + pub fn sceAudioSRCChReserve( + sample_count: i32, + freq: AudioOutputFrequency, + channels: i32, + ) -> i32; + pub fn sceAudioSRCChRelease() -> i32; + pub fn sceAudioSRCOutputBlocking(vol: i32, buf: *mut c_void) -> i32; + pub fn sceAudioInputInit(unknown1: i32, gain: i32, unknown2: i32) -> i32; + pub fn sceAudioInputInitEx(params: *mut AudioInputParams) -> i32; + pub fn sceAudioInputBlocking(sample_count: i32, freq: AudioInputFrequency, buf: *mut c_void); + pub fn sceAudioInput(sample_count: i32, freq: AudioInputFrequency, buf: *mut c_void); + pub fn sceAudioGetInputLength() -> i32; + pub fn sceAudioWaitInputEnd() -> i32; + pub fn sceAudioPollInputEnd() -> i32; + + pub fn sceAtracGetAtracID(ui_codec_type: u32) -> i32; + pub fn sceAtracSetDataAndGetID(buf: *mut c_void, bufsize: usize) -> i32; + pub fn sceAtracDecodeData( + atrac_id: i32, + out_samples: *mut u16, + out_n: *mut i32, + out_end: *mut i32, + out_remain_frame: *mut i32, + ) -> i32; + pub fn sceAtracGetRemainFrame(atrac_id: i32, out_remain_frame: *mut i32) -> i32; + pub fn sceAtracGetStreamDataInfo( + atrac_id: i32, + write_pointer: *mut *mut u8, + available_bytes: *mut u32, + read_offset: *mut u32, + ) -> i32; + pub fn sceAtracAddStreamData(atrac_id: i32, bytes_to_add: u32) -> i32; + pub fn sceAtracGetBitrate(atrac_id: i32, out_bitrate: *mut i32) -> i32; + pub fn sceAtracSetLoopNum(atrac_id: i32, nloops: i32) -> i32; + pub fn sceAtracReleaseAtracID(atrac_id: i32) -> i32; + pub fn sceAtracGetNextSample(atrac_id: i32, out_n: *mut i32) -> i32; + pub fn sceAtracGetMaxSample(atrac_id: i32, out_max: *mut i32) -> i32; + pub fn sceAtracGetBufferInfoForReseting( + atrac_id: i32, + ui_sample: u32, + pbuffer_info: *mut Atrac3BufferInfo, + ) -> i32; + pub fn sceAtracGetChannel(atrac_id: i32, pui_channel: *mut u32) -> i32; + pub fn sceAtracGetInternalErrorInfo(atrac_id: i32, pi_result: *mut i32) -> i32; + pub fn sceAtracGetLoopStatus( + atrac_id: i32, + pi_loop_num: *mut i32, + pui_loop_status: *mut u32, + ) -> i32; + pub fn sceAtracGetNextDecodePosition(atrac_id: i32, pui_sample_position: *mut u32) -> i32; + pub fn sceAtracGetSecondBufferInfo( + atrac_id: i32, + pui_position: *mut u32, + pui_data_byte: *mut u32, + ) -> i32; + pub fn sceAtracGetSoundSample( + atrac_id: i32, + pi_end_sample: *mut i32, + pi_loop_start_sample: *mut i32, + pi_loop_end_sample: *mut i32, + ) -> i32; + pub fn sceAtracResetPlayPosition( + atrac_id: i32, + ui_sample: u32, + ui_write_byte_first_buf: u32, + ui_write_byte_second_buf: u32, + ) -> i32; + pub fn sceAtracSetData(atrac_id: i32, puc_buffer_addr: *mut u8, ui_buffer_byte: u32) -> i32; + pub fn sceAtracSetHalfwayBuffer( + atrac_id: i32, + puc_buffer_addr: *mut u8, + ui_read_byte: u32, + ui_buffer_byte: u32, + ) -> i32; + pub fn sceAtracSetHalfwayBufferAndGetID( + puc_buffer_addr: *mut u8, + ui_read_byte: u32, + ui_buffer_byte: u32, + ) -> i32; + pub fn sceAtracSetSecondBuffer( + atrac_id: i32, + puc_second_buffer_addr: *mut u8, + ui_second_buffer_byte: u32, + ) -> i32; + + pub fn sceCtrlSetSamplingCycle(cycle: i32) -> i32; + pub fn sceCtrlGetSamplingCycle(pcycle: *mut i32) -> i32; + pub fn sceCtrlSetSamplingMode(mode: CtrlMode) -> i32; + pub fn sceCtrlGetSamplingMode(pmode: *mut i32) -> i32; + pub fn sceCtrlPeekBufferPositive(pad_data: *mut SceCtrlData, count: i32) -> i32; + pub fn sceCtrlPeekBufferNegative(pad_data: *mut SceCtrlData, count: i32) -> i32; + pub fn sceCtrlReadBufferPositive(pad_data: *mut SceCtrlData, count: i32) -> i32; + pub fn sceCtrlReadBufferNegative(pad_data: *mut SceCtrlData, count: i32) -> i32; + pub fn sceCtrlPeekLatch(latch_data: *mut SceCtrlLatch) -> i32; + pub fn sceCtrlReadLatch(latch_data: *mut SceCtrlLatch) -> i32; + pub fn sceCtrlSetIdleCancelThreshold(idlereset: i32, idleback: i32) -> i32; + pub fn sceCtrlGetIdleCancelThreshold(idlereset: *mut i32, idleback: *mut i32) -> i32; + + pub fn sceDisplaySetMode(mode: DisplayMode, width: usize, height: usize) -> u32; + pub fn sceDisplayGetMode(pmode: *mut i32, pwidth: *mut i32, pheight: *mut i32) -> i32; + pub fn sceDisplaySetFrameBuf( + top_addr: *const u8, + buffer_width: usize, + pixel_format: DisplayPixelFormat, + sync: DisplaySetBufSync, + ) -> u32; + pub fn sceDisplayGetFrameBuf( + top_addr: *mut *mut c_void, + buffer_width: *mut usize, + pixel_format: *mut DisplayPixelFormat, + sync: DisplaySetBufSync, + ) -> i32; + pub fn sceDisplayGetVcount() -> u32; + pub fn sceDisplayWaitVblank() -> i32; + pub fn sceDisplayWaitVblankCB() -> i32; + pub fn sceDisplayWaitVblankStart() -> i32; + pub fn sceDisplayWaitVblankStartCB() -> i32; + pub fn sceDisplayGetAccumulatedHcount() -> i32; + pub fn sceDisplayGetCurrentHcount() -> i32; + pub fn sceDisplayGetFramePerSec() -> f32; + pub fn sceDisplayIsForeground() -> i32; + pub fn sceDisplayIsVblank() -> i32; + + pub fn sceGeEdramGetSize() -> u32; + pub fn sceGeEdramGetAddr() -> *mut u8; + pub fn sceGeEdramSetAddrTranslation(width: i32) -> i32; + pub fn sceGeGetCmd(cmd: i32) -> u32; + pub fn sceGeGetMtx(type_: GeMatrixType, matrix: *mut c_void) -> i32; + pub fn sceGeGetStack(stack_id: i32, stack: *mut GeStack) -> i32; + pub fn sceGeSaveContext(context: *mut GeContext) -> i32; + pub fn sceGeRestoreContext(context: *const GeContext) -> i32; + pub fn sceGeListEnQueue( + list: *const c_void, + stall: *mut c_void, + cbid: i32, + arg: *mut GeListArgs, + ) -> i32; + pub fn sceGeListEnQueueHead( + list: *const c_void, + stall: *mut c_void, + cbid: i32, + arg: *mut GeListArgs, + ) -> i32; + pub fn sceGeListDeQueue(qid: i32) -> i32; + pub fn sceGeListUpdateStallAddr(qid: i32, stall: *mut c_void) -> i32; + pub fn sceGeListSync(qid: i32, sync_type: i32) -> GeListState; + pub fn sceGeDrawSync(sync_type: i32) -> GeListState; + pub fn sceGeBreak(mode: i32, p_param: *mut GeBreakParam) -> i32; + pub fn sceGeContinue() -> i32; + pub fn sceGeSetCallback(cb: *mut GeCallbackData) -> i32; + pub fn sceGeUnsetCallback(cbid: i32) -> i32; + + pub fn sceKernelExitGame(); + pub fn sceKernelRegisterExitCallback(id: SceUid) -> i32; + pub fn sceKernelLoadExec(file: *const u8, param: *mut SceKernelLoadExecParam) -> i32; + + pub fn sceKernelAllocPartitionMemory( + partition: SceSysMemPartitionId, + name: *const u8, + type_: SceSysMemBlockTypes, + size: u32, + addr: *mut c_void, + ) -> SceUid; + pub fn sceKernelGetBlockHeadAddr(blockid: SceUid) -> *mut c_void; + pub fn sceKernelFreePartitionMemory(blockid: SceUid) -> i32; + pub fn sceKernelTotalFreeMemSize() -> usize; + pub fn sceKernelMaxFreeMemSize() -> usize; + pub fn sceKernelDevkitVersion() -> u32; + pub fn sceKernelSetCompiledSdkVersion(version: u32) -> i32; + pub fn sceKernelGetCompiledSdkVersion() -> u32; + + pub fn sceKernelLibcTime(t: *mut i32) -> i32; + pub fn sceKernelLibcClock() -> u32; + pub fn sceKernelLibcGettimeofday(tp: *mut timeval, tzp: *mut timezone) -> i32; + pub fn sceKernelDcacheWritebackAll(); + pub fn sceKernelDcacheWritebackInvalidateAll(); + pub fn sceKernelDcacheWritebackRange(p: *const c_void, size: u32); + pub fn sceKernelDcacheWritebackInvalidateRange(p: *const c_void, size: u32); + pub fn sceKernelDcacheInvalidateRange(p: *const c_void, size: u32); + pub fn sceKernelIcacheInvalidateAll(); + pub fn sceKernelIcacheInvalidateRange(p: *const c_void, size: u32); + pub fn sceKernelUtilsMt19937Init(ctx: *mut SceKernelUtilsMt19937Context, seed: u32) -> i32; + pub fn sceKernelUtilsMt19937UInt(ctx: *mut SceKernelUtilsMt19937Context) -> u32; + pub fn sceKernelUtilsMd5Digest(data: *mut u8, size: u32, digest: *mut u8) -> i32; + pub fn sceKernelUtilsMd5BlockInit(ctx: *mut SceKernelUtilsMd5Context) -> i32; + pub fn sceKernelUtilsMd5BlockUpdate( + ctx: *mut SceKernelUtilsMd5Context, + data: *mut u8, + size: u32, + ) -> i32; + pub fn sceKernelUtilsMd5BlockResult(ctx: *mut SceKernelUtilsMd5Context, digest: *mut u8) + -> i32; + pub fn sceKernelUtilsSha1Digest(data: *mut u8, size: u32, digest: *mut u8) -> i32; + pub fn sceKernelUtilsSha1BlockInit(ctx: *mut SceKernelUtilsSha1Context) -> i32; + pub fn sceKernelUtilsSha1BlockUpdate( + ctx: *mut SceKernelUtilsSha1Context, + data: *mut u8, + size: u32, + ) -> i32; + pub fn sceKernelUtilsSha1BlockResult( + ctx: *mut SceKernelUtilsSha1Context, + digest: *mut u8, + ) -> i32; + + pub fn sceKernelRegisterSubIntrHandler( + int_no: i32, + no: i32, + handler: *mut c_void, + arg: *mut c_void, + ) -> i32; + pub fn sceKernelReleaseSubIntrHandler(int_no: i32, no: i32) -> i32; + pub fn sceKernelEnableSubIntr(int_no: i32, no: i32) -> i32; + pub fn sceKernelDisableSubIntr(int_no: i32, no: i32) -> i32; + pub fn QueryIntrHandlerInfo( + intr_code: SceUid, + sub_intr_code: SceUid, + data: *mut IntrHandlerOptionParam, + ) -> i32; + + pub fn sceKernelCpuSuspendIntr() -> u32; + pub fn sceKernelCpuResumeIntr(flags: u32); + pub fn sceKernelCpuResumeIntrWithSync(flags: u32); + pub fn sceKernelIsCpuIntrSuspended(flags: u32) -> i32; + pub fn sceKernelIsCpuIntrEnable() -> i32; + + pub fn sceKernelLoadModule( + path: *const u8, + flags: i32, + option: *mut SceKernelLMOption, + ) -> SceUid; + pub fn sceKernelLoadModuleMs( + path: *const u8, + flags: i32, + option: *mut SceKernelLMOption, + ) -> SceUid; + pub fn sceKernelLoadModuleByID( + fid: SceUid, + flags: i32, + option: *mut SceKernelLMOption, + ) -> SceUid; + pub fn sceKernelLoadModuleBufferUsbWlan( + buf_size: usize, + buf: *mut c_void, + flags: i32, + option: *mut SceKernelLMOption, + ) -> SceUid; + pub fn sceKernelStartModule( + mod_id: SceUid, + arg_size: usize, + argp: *mut c_void, + status: *mut i32, + option: *mut SceKernelSMOption, + ) -> i32; + pub fn sceKernelStopModule( + mod_id: SceUid, + arg_size: usize, + argp: *mut c_void, + status: *mut i32, + option: *mut SceKernelSMOption, + ) -> i32; + pub fn sceKernelUnloadModule(mod_id: SceUid) -> i32; + pub fn sceKernelSelfStopUnloadModule(unknown: i32, arg_size: usize, argp: *mut c_void) -> i32; + pub fn sceKernelStopUnloadSelfModule( + arg_size: usize, + argp: *mut c_void, + status: *mut i32, + option: *mut SceKernelSMOption, + ) -> i32; + pub fn sceKernelQueryModuleInfo(mod_id: SceUid, info: *mut SceKernelModuleInfo) -> i32; + pub fn sceKernelGetModuleIdList( + read_buf: *mut SceUid, + read_buf_size: i32, + id_count: *mut i32, + ) -> i32; + + pub fn sceKernelVolatileMemLock(unk: i32, ptr: *mut *mut c_void, size: *mut i32) -> i32; + pub fn sceKernelVolatileMemTryLock(unk: i32, ptr: *mut *mut c_void, size: *mut i32) -> i32; + pub fn sceKernelVolatileMemUnlock(unk: i32) -> i32; + + pub fn sceKernelStdin() -> SceUid; + pub fn sceKernelStdout() -> SceUid; + pub fn sceKernelStderr() -> SceUid; + + pub fn sceKernelGetThreadmanIdType(uid: SceUid) -> SceKernelIdListType; + pub fn sceKernelCreateThread( + name: *const u8, + entry: SceKernelThreadEntry, + init_priority: i32, + stack_size: i32, + attr: i32, + option: *mut SceKernelThreadOptParam, + ) -> SceUid; + pub fn sceKernelDeleteThread(thid: SceUid) -> i32; + pub fn sceKernelStartThread(id: SceUid, arg_len: usize, arg_p: *mut c_void) -> i32; + pub fn sceKernelExitThread(status: i32) -> i32; + pub fn sceKernelExitDeleteThread(status: i32) -> i32; + pub fn sceKernelTerminateThread(thid: SceUid) -> i32; + pub fn sceKernelTerminateDeleteThread(thid: SceUid) -> i32; + pub fn sceKernelSuspendDispatchThread() -> i32; + pub fn sceKernelResumeDispatchThread(state: i32) -> i32; + pub fn sceKernelSleepThread() -> i32; + pub fn sceKernelSleepThreadCB() -> i32; + pub fn sceKernelWakeupThread(thid: SceUid) -> i32; + pub fn sceKernelCancelWakeupThread(thid: SceUid) -> i32; + pub fn sceKernelSuspendThread(thid: SceUid) -> i32; + pub fn sceKernelResumeThread(thid: SceUid) -> i32; + pub fn sceKernelWaitThreadEnd(thid: SceUid, timeout: *mut u32) -> i32; + pub fn sceKernelWaitThreadEndCB(thid: SceUid, timeout: *mut u32) -> i32; + pub fn sceKernelDelayThread(delay: u32) -> i32; + pub fn sceKernelDelayThreadCB(delay: u32) -> i32; + pub fn sceKernelDelaySysClockThread(delay: *mut SceKernelSysClock) -> i32; + pub fn sceKernelDelaySysClockThreadCB(delay: *mut SceKernelSysClock) -> i32; + pub fn sceKernelChangeCurrentThreadAttr(unknown: i32, attr: i32) -> i32; + pub fn sceKernelChangeThreadPriority(thid: SceUid, priority: i32) -> i32; + pub fn sceKernelRotateThreadReadyQueue(priority: i32) -> i32; + pub fn sceKernelReleaseWaitThread(thid: SceUid) -> i32; + pub fn sceKernelGetThreadId() -> i32; + pub fn sceKernelGetThreadCurrentPriority() -> i32; + pub fn sceKernelGetThreadExitStatus(thid: SceUid) -> i32; + pub fn sceKernelCheckThreadStack() -> i32; + pub fn sceKernelGetThreadStackFreeSize(thid: SceUid) -> i32; + pub fn sceKernelReferThreadStatus(thid: SceUid, info: *mut SceKernelThreadInfo) -> i32; + pub fn sceKernelReferThreadRunStatus( + thid: SceUid, + status: *mut SceKernelThreadRunStatus, + ) -> i32; + pub fn sceKernelCreateSema( + name: *const u8, + attr: u32, + init_val: i32, + max_val: i32, + option: *mut SceKernelSemaOptParam, + ) -> SceUid; + pub fn sceKernelDeleteSema(sema_id: SceUid) -> i32; + pub fn sceKernelSignalSema(sema_id: SceUid, signal: i32) -> i32; + pub fn sceKernelWaitSema(sema_id: SceUid, signal: i32, timeout: *mut u32) -> i32; + pub fn sceKernelWaitSemaCB(sema_id: SceUid, signal: i32, timeout: *mut u32) -> i32; + pub fn sceKernelPollSema(sema_id: SceUid, signal: i32) -> i32; + pub fn sceKernelReferSemaStatus(sema_id: SceUid, info: *mut SceKernelSemaInfo) -> i32; + pub fn sceKernelCreateEventFlag( + name: *const u8, + attr: i32, + bits: i32, + opt: *mut SceKernelEventFlagOptParam, + ) -> SceUid; + pub fn sceKernelSetEventFlag(ev_id: SceUid, bits: u32) -> i32; + pub fn sceKernelClearEventFlag(ev_id: SceUid, bits: u32) -> i32; + pub fn sceKernelPollEventFlag(ev_id: SceUid, bits: u32, wait: i32, out_bits: *mut u32) -> i32; + pub fn sceKernelWaitEventFlag( + ev_id: SceUid, + bits: u32, + wait: i32, + out_bits: *mut u32, + timeout: *mut u32, + ) -> i32; + pub fn sceKernelWaitEventFlagCB( + ev_id: SceUid, + bits: u32, + wait: i32, + out_bits: *mut u32, + timeout: *mut u32, + ) -> i32; + pub fn sceKernelDeleteEventFlag(ev_id: SceUid) -> i32; + pub fn sceKernelReferEventFlagStatus(event: SceUid, status: *mut SceKernelEventFlagInfo) + -> i32; + pub fn sceKernelCreateMbx( + name: *const u8, + attr: u32, + option: *mut SceKernelMbxOptParam, + ) -> SceUid; + pub fn sceKernelDeleteMbx(mbx_id: SceUid) -> i32; + pub fn sceKernelSendMbx(mbx_id: SceUid, message: *mut c_void) -> i32; + pub fn sceKernelReceiveMbx(mbx_id: SceUid, message: *mut *mut c_void, timeout: *mut u32) + -> i32; + pub fn sceKernelReceiveMbxCB( + mbx_id: SceUid, + message: *mut *mut c_void, + timeout: *mut u32, + ) -> i32; + pub fn sceKernelPollMbx(mbx_id: SceUid, pmessage: *mut *mut c_void) -> i32; + pub fn sceKernelCancelReceiveMbx(mbx_id: SceUid, num: *mut i32) -> i32; + pub fn sceKernelReferMbxStatus(mbx_id: SceUid, info: *mut SceKernelMbxInfo) -> i32; + pub fn sceKernelSetAlarm( + clock: u32, + handler: SceKernelAlarmHandler, + common: *mut c_void, + ) -> SceUid; + pub fn sceKernelSetSysClockAlarm( + clock: *mut SceKernelSysClock, + handler: *mut SceKernelAlarmHandler, + common: *mut c_void, + ) -> SceUid; + pub fn sceKernelCancelAlarm(alarm_id: SceUid) -> i32; + pub fn sceKernelReferAlarmStatus(alarm_id: SceUid, info: *mut SceKernelAlarmInfo) -> i32; + pub fn sceKernelCreateCallback( + name: *const u8, + func: SceKernelCallbackFunction, + arg: *mut c_void, + ) -> SceUid; + pub fn sceKernelReferCallbackStatus(cb: SceUid, status: *mut SceKernelCallbackInfo) -> i32; + pub fn sceKernelDeleteCallback(cb: SceUid) -> i32; + pub fn sceKernelNotifyCallback(cb: SceUid, arg2: i32) -> i32; + pub fn sceKernelCancelCallback(cb: SceUid) -> i32; + pub fn sceKernelGetCallbackCount(cb: SceUid) -> i32; + pub fn sceKernelCheckCallback() -> i32; + pub fn sceKernelGetThreadmanIdList( + type_: SceKernelIdListType, + read_buf: *mut SceUid, + read_buf_size: i32, + id_count: *mut i32, + ) -> i32; + pub fn sceKernelReferSystemStatus(status: *mut SceKernelSystemStatus) -> i32; + pub fn sceKernelCreateMsgPipe( + name: *const u8, + part: i32, + attr: i32, + unk1: *mut c_void, + opt: *mut c_void, + ) -> SceUid; + pub fn sceKernelDeleteMsgPipe(uid: SceUid) -> i32; + pub fn sceKernelSendMsgPipe( + uid: SceUid, + message: *mut c_void, + size: u32, + unk1: i32, + unk2: *mut c_void, + timeout: *mut u32, + ) -> i32; + pub fn sceKernelSendMsgPipeCB( + uid: SceUid, + message: *mut c_void, + size: u32, + unk1: i32, + unk2: *mut c_void, + timeout: *mut u32, + ) -> i32; + pub fn sceKernelTrySendMsgPipe( + uid: SceUid, + message: *mut c_void, + size: u32, + unk1: i32, + unk2: *mut c_void, + ) -> i32; + pub fn sceKernelReceiveMsgPipe( + uid: SceUid, + message: *mut c_void, + size: u32, + unk1: i32, + unk2: *mut c_void, + timeout: *mut u32, + ) -> i32; + pub fn sceKernelReceiveMsgPipeCB( + uid: SceUid, + message: *mut c_void, + size: u32, + unk1: i32, + unk2: *mut c_void, + timeout: *mut u32, + ) -> i32; + pub fn sceKernelTryReceiveMsgPipe( + uid: SceUid, + message: *mut c_void, + size: u32, + unk1: i32, + unk2: *mut c_void, + ) -> i32; + pub fn sceKernelCancelMsgPipe(uid: SceUid, send: *mut i32, recv: *mut i32) -> i32; + pub fn sceKernelReferMsgPipeStatus(uid: SceUid, info: *mut SceKernelMppInfo) -> i32; + pub fn sceKernelCreateVpl( + name: *const u8, + part: i32, + attr: i32, + size: u32, + opt: *mut SceKernelVplOptParam, + ) -> SceUid; + pub fn sceKernelDeleteVpl(uid: SceUid) -> i32; + pub fn sceKernelAllocateVpl( + uid: SceUid, + size: u32, + data: *mut *mut c_void, + timeout: *mut u32, + ) -> i32; + pub fn sceKernelAllocateVplCB( + uid: SceUid, + size: u32, + data: *mut *mut c_void, + timeout: *mut u32, + ) -> i32; + pub fn sceKernelTryAllocateVpl(uid: SceUid, size: u32, data: *mut *mut c_void) -> i32; + pub fn sceKernelFreeVpl(uid: SceUid, data: *mut c_void) -> i32; + pub fn sceKernelCancelVpl(uid: SceUid, num: *mut i32) -> i32; + pub fn sceKernelReferVplStatus(uid: SceUid, info: *mut SceKernelVplInfo) -> i32; + pub fn sceKernelCreateFpl( + name: *const u8, + part: i32, + attr: i32, + size: u32, + blocks: u32, + opt: *mut SceKernelFplOptParam, + ) -> i32; + pub fn sceKernelDeleteFpl(uid: SceUid) -> i32; + pub fn sceKernelAllocateFpl(uid: SceUid, data: *mut *mut c_void, timeout: *mut u32) -> i32; + pub fn sceKernelAllocateFplCB(uid: SceUid, data: *mut *mut c_void, timeout: *mut u32) -> i32; + pub fn sceKernelTryAllocateFpl(uid: SceUid, data: *mut *mut c_void) -> i32; + pub fn sceKernelFreeFpl(uid: SceUid, data: *mut c_void) -> i32; + pub fn sceKernelCancelFpl(uid: SceUid, pnum: *mut i32) -> i32; + pub fn sceKernelReferFplStatus(uid: SceUid, info: *mut SceKernelFplInfo) -> i32; + pub fn sceKernelUSec2SysClock(usec: u32, clock: *mut SceKernelSysClock) -> i32; + pub fn sceKernelUSec2SysClockWide(usec: u32) -> i64; + pub fn sceKernelSysClock2USec( + clock: *mut SceKernelSysClock, + low: *mut u32, + high: *mut u32, + ) -> i32; + pub fn sceKernelSysClock2USecWide(clock: i64, low: *mut u32, high: *mut u32) -> i32; + pub fn sceKernelGetSystemTime(time: *mut SceKernelSysClock) -> i32; + pub fn sceKernelGetSystemTimeWide() -> i64; + pub fn sceKernelGetSystemTimeLow() -> u32; + pub fn sceKernelCreateVTimer(name: *const u8, opt: *mut SceKernelVTimerOptParam) -> SceUid; + pub fn sceKernelDeleteVTimer(uid: SceUid) -> i32; + pub fn sceKernelGetVTimerBase(uid: SceUid, base: *mut SceKernelSysClock) -> i32; + pub fn sceKernelGetVTimerBaseWide(uid: SceUid) -> i64; + pub fn sceKernelGetVTimerTime(uid: SceUid, time: *mut SceKernelSysClock) -> i32; + pub fn sceKernelGetVTimerTimeWide(uid: SceUid) -> i64; + pub fn sceKernelSetVTimerTime(uid: SceUid, time: *mut SceKernelSysClock) -> i32; + pub fn sceKernelSetVTimerTimeWide(uid: SceUid, time: i64) -> i64; + pub fn sceKernelStartVTimer(uid: SceUid) -> i32; + pub fn sceKernelStopVTimer(uid: SceUid) -> i32; + pub fn sceKernelSetVTimerHandler( + uid: SceUid, + time: *mut SceKernelSysClock, + handler: SceKernelVTimerHandler, + common: *mut c_void, + ) -> i32; + pub fn sceKernelSetVTimerHandlerWide( + uid: SceUid, + time: i64, + handler: SceKernelVTimerHandlerWide, + common: *mut c_void, + ) -> i32; + pub fn sceKernelCancelVTimerHandler(uid: SceUid) -> i32; + pub fn sceKernelReferVTimerStatus(uid: SceUid, info: *mut SceKernelVTimerInfo) -> i32; + pub fn sceKernelRegisterThreadEventHandler( + name: *const u8, + thread_id: SceUid, + mask: i32, + handler: SceKernelThreadEventHandler, + common: *mut c_void, + ) -> SceUid; + pub fn sceKernelReleaseThreadEventHandler(uid: SceUid) -> i32; + pub fn sceKernelReferThreadEventHandlerStatus( + uid: SceUid, + info: *mut SceKernelThreadEventHandlerInfo, + ) -> i32; + pub fn sceKernelReferThreadProfiler() -> *mut DebugProfilerRegs; + pub fn sceKernelReferGlobalProfiler() -> *mut DebugProfilerRegs; + + pub fn sceUsbStart(driver_name: *const u8, size: i32, args: *mut c_void) -> i32; + pub fn sceUsbStop(driver_name: *const u8, size: i32, args: *mut c_void) -> i32; + pub fn sceUsbActivate(pid: u32) -> i32; + pub fn sceUsbDeactivate(pid: u32) -> i32; + pub fn sceUsbGetState() -> i32; + pub fn sceUsbGetDrvState(driver_name: *const u8) -> i32; +} + +extern "C" { + pub fn sceUsbCamSetupStill(param: *mut UsbCamSetupStillParam) -> i32; + pub fn sceUsbCamSetupStillEx(param: *mut UsbCamSetupStillExParam) -> i32; + pub fn sceUsbCamStillInputBlocking(buf: *mut u8, size: usize) -> i32; + pub fn sceUsbCamStillInput(buf: *mut u8, size: usize) -> i32; + pub fn sceUsbCamStillWaitInputEnd() -> i32; + pub fn sceUsbCamStillPollInputEnd() -> i32; + pub fn sceUsbCamStillCancelInput() -> i32; + pub fn sceUsbCamStillGetInputLength() -> i32; + pub fn sceUsbCamSetupVideo( + param: *mut UsbCamSetupVideoParam, + work_area: *mut c_void, + work_area_size: i32, + ) -> i32; + pub fn sceUsbCamSetupVideoEx( + param: *mut UsbCamSetupVideoExParam, + work_area: *mut c_void, + work_area_size: i32, + ) -> i32; + pub fn sceUsbCamStartVideo() -> i32; + pub fn sceUsbCamStopVideo() -> i32; + pub fn sceUsbCamReadVideoFrameBlocking(buf: *mut u8, size: usize) -> i32; + pub fn sceUsbCamReadVideoFrame(buf: *mut u8, size: usize) -> i32; + pub fn sceUsbCamWaitReadVideoFrameEnd() -> i32; + pub fn sceUsbCamPollReadVideoFrameEnd() -> i32; + pub fn sceUsbCamGetReadVideoFrameSize() -> i32; + pub fn sceUsbCamSetSaturation(saturation: i32) -> i32; + pub fn sceUsbCamSetBrightness(brightness: i32) -> i32; + pub fn sceUsbCamSetContrast(contrast: i32) -> i32; + pub fn sceUsbCamSetSharpness(sharpness: i32) -> i32; + pub fn sceUsbCamSetImageEffectMode(effect_mode: UsbCamEffectMode) -> i32; + pub fn sceUsbCamSetEvLevel(exposure_level: UsbCamEvLevel) -> i32; + pub fn sceUsbCamSetReverseMode(reverse_flags: i32) -> i32; + pub fn sceUsbCamSetZoom(zoom: i32) -> i32; + pub fn sceUsbCamGetSaturation(saturation: *mut i32) -> i32; + pub fn sceUsbCamGetBrightness(brightness: *mut i32) -> i32; + pub fn sceUsbCamGetContrast(contrast: *mut i32) -> i32; + pub fn sceUsbCamGetSharpness(sharpness: *mut i32) -> i32; + pub fn sceUsbCamGetImageEffectMode(effect_mode: *mut UsbCamEffectMode) -> i32; + pub fn sceUsbCamGetEvLevel(exposure_level: *mut UsbCamEvLevel) -> i32; + pub fn sceUsbCamGetReverseMode(reverse_flags: *mut i32) -> i32; + pub fn sceUsbCamGetZoom(zoom: *mut i32) -> i32; + pub fn sceUsbCamAutoImageReverseSW(on: i32) -> i32; + pub fn sceUsbCamGetAutoImageReverseState() -> i32; + pub fn sceUsbCamGetLensDirection() -> i32; + + pub fn sceUsbstorBootRegisterNotify(event_flag: SceUid) -> i32; + pub fn sceUsbstorBootUnregisterNotify(event_flag: u32) -> i32; + pub fn sceUsbstorBootSetCapacity(size: u32) -> i32; + + pub fn scePowerRegisterCallback(slot: i32, cbid: SceUid) -> i32; + pub fn scePowerUnregisterCallback(slot: i32) -> i32; + pub fn scePowerIsPowerOnline() -> i32; + pub fn scePowerIsBatteryExist() -> i32; + pub fn scePowerIsBatteryCharging() -> i32; + pub fn scePowerGetBatteryChargingStatus() -> i32; + pub fn scePowerIsLowBattery() -> i32; + pub fn scePowerGetBatteryLifePercent() -> i32; + pub fn scePowerGetBatteryLifeTime() -> i32; + pub fn scePowerGetBatteryTemp() -> i32; + pub fn scePowerGetBatteryElec() -> i32; + pub fn scePowerGetBatteryVolt() -> i32; + pub fn scePowerSetCpuClockFrequency(cpufreq: i32) -> i32; + pub fn scePowerSetBusClockFrequency(busfreq: i32) -> i32; + pub fn scePowerGetCpuClockFrequency() -> i32; + pub fn scePowerGetCpuClockFrequencyInt() -> i32; + pub fn scePowerGetCpuClockFrequencyFloat() -> f32; + pub fn scePowerGetBusClockFrequency() -> i32; + pub fn scePowerGetBusClockFrequencyInt() -> i32; + pub fn scePowerGetBusClockFrequencyFloat() -> f32; + pub fn scePowerSetClockFrequency(pllfreq: i32, cpufreq: i32, busfreq: i32) -> i32; + pub fn scePowerLock(unknown: i32) -> i32; + pub fn scePowerUnlock(unknown: i32) -> i32; + pub fn scePowerTick(t: PowerTick) -> i32; + pub fn scePowerGetIdleTimer() -> i32; + pub fn scePowerIdleTimerEnable(unknown: i32) -> i32; + pub fn scePowerIdleTimerDisable(unknown: i32) -> i32; + pub fn scePowerRequestStandby() -> i32; + pub fn scePowerRequestSuspend() -> i32; + + pub fn sceWlanDevIsPowerOn() -> i32; + pub fn sceWlanGetSwitchState() -> i32; + pub fn sceWlanGetEtherAddr(ether_addr: *mut u8) -> i32; + + pub fn sceWlanDevAttach() -> i32; + pub fn sceWlanDevDetach() -> i32; + + pub fn sceRtcGetTickResolution() -> u32; + pub fn sceRtcGetCurrentTick(tick: *mut u64) -> i32; + pub fn sceRtcGetCurrentClock(tm: *mut ScePspDateTime, tz: i32) -> i32; + pub fn sceRtcGetCurrentClockLocalTime(tm: *mut ScePspDateTime) -> i32; + pub fn sceRtcConvertUtcToLocalTime(tick_utc: *const u64, tick_local: *mut u64) -> i32; + pub fn sceRtcConvertLocalTimeToUTC(tick_local: *const u64, tick_utc: *mut u64) -> i32; + pub fn sceRtcIsLeapYear(year: i32) -> i32; + pub fn sceRtcGetDaysInMonth(year: i32, month: i32) -> i32; + pub fn sceRtcGetDayOfWeek(year: i32, month: i32, day: i32) -> i32; + pub fn sceRtcCheckValid(date: *const ScePspDateTime) -> i32; + pub fn sceRtcSetTick(date: *mut ScePspDateTime, tick: *const u64) -> i32; + pub fn sceRtcGetTick(date: *const ScePspDateTime, tick: *mut u64) -> i32; + pub fn sceRtcCompareTick(tick1: *const u64, tick2: *const u64) -> i32; + pub fn sceRtcTickAddTicks(dest_tick: *mut u64, src_tick: *const u64, num_ticks: u64) -> i32; + pub fn sceRtcTickAddMicroseconds(dest_tick: *mut u64, src_tick: *const u64, num_ms: u64) + -> i32; + pub fn sceRtcTickAddSeconds(dest_tick: *mut u64, src_tick: *const u64, num_seconds: u64) + -> i32; + pub fn sceRtcTickAddMinutes(dest_tick: *mut u64, src_tick: *const u64, num_minutes: u64) + -> i32; + pub fn sceRtcTickAddHours(dest_tick: *mut u64, src_tick: *const u64, num_hours: u64) -> i32; + pub fn sceRtcTickAddDays(dest_tick: *mut u64, src_tick: *const u64, num_days: u64) -> i32; + pub fn sceRtcTickAddWeeks(dest_tick: *mut u64, src_tick: *const u64, num_weeks: u64) -> i32; + pub fn sceRtcTickAddMonths(dest_tick: *mut u64, src_tick: *const u64, num_months: u64) -> i32; + pub fn sceRtcTickAddYears(dest_tick: *mut u64, src_tick: *const u64, num_years: u64) -> i32; + pub fn sceRtcSetTime_t(date: *mut ScePspDateTime, time: u32) -> i32; + pub fn sceRtcGetTime_t(date: *const ScePspDateTime, time: *mut u32) -> i32; + pub fn sceRtcSetTime64_t(date: *mut ScePspDateTime, time: u64) -> i32; + pub fn sceRtcGetTime64_t(date: *const ScePspDateTime, time: *mut u64) -> i32; + pub fn sceRtcSetDosTime(date: *mut ScePspDateTime, dos_time: u32) -> i32; + pub fn sceRtcGetDosTime(date: *mut ScePspDateTime, dos_time: u32) -> i32; + pub fn sceRtcSetWin32FileTime(date: *mut ScePspDateTime, time: *mut u64) -> i32; + pub fn sceRtcGetWin32FileTime(date: *mut ScePspDateTime, time: *mut u64) -> i32; + pub fn sceRtcParseDateTime(dest_tick: *mut u64, date_string: *const u8) -> i32; + pub fn sceRtcFormatRFC3339( + psz_date_time: *mut char, + p_utc: *const u64, + time_zone_minutes: i32, + ) -> i32; + pub fn sceRtcFormatRFC3339LocalTime(psz_date_time: *mut char, p_utc: *const u64) -> i32; + pub fn sceRtcParseRFC3339(p_utc: *mut u64, psz_date_time: *const u8) -> i32; + pub fn sceRtcFormatRFC2822( + psz_date_time: *mut char, + p_utc: *const u64, + time_zone_minutes: i32, + ) -> i32; + pub fn sceRtcFormatRFC2822LocalTime(psz_date_time: *mut char, p_utc: *const u64) -> i32; + + pub fn sceIoOpen(file: *const u8, flags: i32, permissions: IoPermissions) -> SceUid; + pub fn sceIoOpenAsync(file: *const u8, flags: i32, permissions: IoPermissions) -> SceUid; + pub fn sceIoClose(fd: SceUid) -> i32; + pub fn sceIoCloseAsync(fd: SceUid) -> i32; + pub fn sceIoRead(fd: SceUid, data: *mut c_void, size: u32) -> i32; + pub fn sceIoReadAsync(fd: SceUid, data: *mut c_void, size: u32) -> i32; + pub fn sceIoWrite(fd: SceUid, data: *const c_void, size: usize) -> i32; + pub fn sceIoWriteAsync(fd: SceUid, data: *const c_void, size: u32) -> i32; + pub fn sceIoLseek(fd: SceUid, offset: i64, whence: IoWhence) -> i64; + pub fn sceIoLseekAsync(fd: SceUid, offset: i64, whence: IoWhence) -> i32; + pub fn sceIoLseek32(fd: SceUid, offset: i32, whence: IoWhence) -> i32; + pub fn sceIoLseek32Async(fd: SceUid, offset: i32, whence: IoWhence) -> i32; + pub fn sceIoRemove(file: *const u8) -> i32; + pub fn sceIoMkdir(dir: *const u8, mode: IoPermissions) -> i32; + pub fn sceIoRmdir(path: *const u8) -> i32; + pub fn sceIoChdir(path: *const u8) -> i32; + pub fn sceIoRename(oldname: *const u8, newname: *const u8) -> i32; + pub fn sceIoDopen(dirname: *const u8) -> SceUid; + pub fn sceIoDread(fd: SceUid, dir: *mut SceIoDirent) -> i32; + pub fn sceIoDclose(fd: SceUid) -> i32; + pub fn sceIoDevctl( + dev: *const u8, + cmd: u32, + indata: *mut c_void, + inlen: i32, + outdata: *mut c_void, + outlen: i32, + ) -> i32; + pub fn sceIoAssign( + dev1: *const u8, + dev2: *const u8, + dev3: *const u8, + mode: IoAssignPerms, + unk1: *mut c_void, + unk2: i32, + ) -> i32; + pub fn sceIoUnassign(dev: *const u8) -> i32; + pub fn sceIoGetstat(file: *const u8, stat: *mut SceIoStat) -> i32; + pub fn sceIoChstat(file: *const u8, stat: *mut SceIoStat, bits: i32) -> i32; + pub fn sceIoIoctl( + fd: SceUid, + cmd: u32, + indata: *mut c_void, + inlen: i32, + outdata: *mut c_void, + outlen: i32, + ) -> i32; + pub fn sceIoIoctlAsync( + fd: SceUid, + cmd: u32, + indata: *mut c_void, + inlen: i32, + outdata: *mut c_void, + outlen: i32, + ) -> i32; + pub fn sceIoSync(device: *const u8, unk: u32) -> i32; + pub fn sceIoWaitAsync(fd: SceUid, res: *mut i64) -> i32; + pub fn sceIoWaitAsyncCB(fd: SceUid, res: *mut i64) -> i32; + pub fn sceIoPollAsync(fd: SceUid, res: *mut i64) -> i32; + pub fn sceIoGetAsyncStat(fd: SceUid, poll: i32, res: *mut i64) -> i32; + pub fn sceIoCancel(fd: SceUid) -> i32; + pub fn sceIoGetDevType(fd: SceUid) -> i32; + pub fn sceIoChangeAsyncPriority(fd: SceUid, pri: i32) -> i32; + pub fn sceIoSetAsyncCallback(fd: SceUid, cb: SceUid, argp: *mut c_void) -> i32; + + pub fn sceJpegInitMJpeg() -> i32; + pub fn sceJpegFinishMJpeg() -> i32; + pub fn sceJpegCreateMJpeg(width: i32, height: i32) -> i32; + pub fn sceJpegDeleteMJpeg() -> i32; + pub fn sceJpegDecodeMJpeg(jpeg_buf: *mut u8, size: usize, rgba: *mut c_void, unk: u32) -> i32; + + pub fn sceUmdCheckMedium() -> i32; + pub fn sceUmdGetDiscInfo(info: *mut UmdInfo) -> i32; + pub fn sceUmdActivate(unit: i32, drive: *const u8) -> i32; + pub fn sceUmdDeactivate(unit: i32, drive: *const u8) -> i32; + pub fn sceUmdWaitDriveStat(state: i32) -> i32; + pub fn sceUmdWaitDriveStatWithTimer(state: i32, timeout: u32) -> i32; + pub fn sceUmdWaitDriveStatCB(state: i32, timeout: u32) -> i32; + pub fn sceUmdCancelWaitDriveStat() -> i32; + pub fn sceUmdGetDriveStat() -> i32; + pub fn sceUmdGetErrorStat() -> i32; + pub fn sceUmdRegisterUMDCallBack(cbid: i32) -> i32; + pub fn sceUmdUnRegisterUMDCallBack(cbid: i32) -> i32; + pub fn sceUmdReplacePermit() -> i32; + pub fn sceUmdReplaceProhibit() -> i32; + + pub fn sceMpegInit() -> i32; + pub fn sceMpegFinish(); + pub fn sceMpegRingbufferQueryMemSize(packets: i32) -> i32; + pub fn sceMpegRingbufferConstruct( + ringbuffer: *mut SceMpegRingbuffer, + packets: i32, + data: *mut c_void, + size: i32, + callback: SceMpegRingbufferCb, + cb_param: *mut c_void, + ) -> i32; + pub fn sceMpegRingbufferDestruct(ringbuffer: *mut SceMpegRingbuffer); + pub fn sceMpegRingbufferAvailableSize(ringbuffer: *mut SceMpegRingbuffer) -> i32; + pub fn sceMpegRingbufferPut( + ringbuffer: *mut SceMpegRingbuffer, + num_packets: i32, + available: i32, + ) -> i32; + pub fn sceMpegQueryMemSize(unk: i32) -> i32; + pub fn sceMpegCreate( + handle: SceMpeg, + data: *mut c_void, + size: i32, + ringbuffer: *mut SceMpegRingbuffer, + frame_width: i32, + unk1: i32, + unk2: i32, + ) -> i32; + pub fn sceMpegDelete(handle: SceMpeg); + pub fn sceMpegQueryStreamOffset(handle: SceMpeg, buffer: *mut c_void, offset: *mut i32) -> i32; + pub fn sceMpegQueryStreamSize(buffer: *mut c_void, size: *mut i32) -> i32; + pub fn sceMpegRegistStream(handle: SceMpeg, stream_id: i32, unk: i32) -> SceMpegStream; + pub fn sceMpegUnRegistStream(handle: SceMpeg, stream: SceMpegStream); + pub fn sceMpegFlushAllStream(handle: SceMpeg) -> i32; + pub fn sceMpegMallocAvcEsBuf(handle: SceMpeg) -> *mut c_void; + pub fn sceMpegFreeAvcEsBuf(handle: SceMpeg, buf: *mut c_void); + pub fn sceMpegQueryAtracEsSize(handle: SceMpeg, es_size: *mut i32, out_size: *mut i32) -> i32; + pub fn sceMpegInitAu(handle: SceMpeg, es_buffer: *mut c_void, au: *mut SceMpegAu) -> i32; + pub fn sceMpegGetAvcAu( + handle: SceMpeg, + stream: SceMpegStream, + au: *mut SceMpegAu, + unk: *mut i32, + ) -> i32; + pub fn sceMpegAvcDecodeMode(handle: SceMpeg, mode: *mut SceMpegAvcMode) -> i32; + pub fn sceMpegAvcDecode( + handle: SceMpeg, + au: *mut SceMpegAu, + iframe_width: i32, + buffer: *mut c_void, + init: *mut i32, + ) -> i32; + pub fn sceMpegAvcDecodeStop( + handle: SceMpeg, + frame_width: i32, + buffer: *mut c_void, + status: *mut i32, + ) -> i32; + pub fn sceMpegGetAtracAu( + handle: SceMpeg, + stream: SceMpegStream, + au: *mut SceMpegAu, + unk: *mut c_void, + ) -> i32; + pub fn sceMpegAtracDecode( + handle: SceMpeg, + au: *mut SceMpegAu, + buffer: *mut c_void, + init: i32, + ) -> i32; + + pub fn sceMpegBaseYCrCbCopyVme(yuv_buffer: *mut c_void, buffer: *mut i32, type_: i32) -> i32; + pub fn sceMpegBaseCscInit(width: i32) -> i32; + pub fn sceMpegBaseCscVme( + rgb_buffer: *mut c_void, + rgb_buffer2: *mut c_void, + width: i32, + y_cr_cb_buffer: *mut SceMpegYCrCbBuffer, + ) -> i32; + pub fn sceMpegbase_BEA18F91(lli: *mut SceMpegLLI) -> i32; + + pub fn sceHprmPeekCurrentKey(key: *mut i32) -> i32; + pub fn sceHprmPeekLatch(latch: *mut [u32; 4]) -> i32; + pub fn sceHprmReadLatch(latch: *mut [u32; 4]) -> i32; + pub fn sceHprmIsHeadphoneExist() -> i32; + pub fn sceHprmIsRemoteExist() -> i32; + pub fn sceHprmIsMicrophoneExist() -> i32; + + pub fn sceGuDepthBuffer(zbp: *mut c_void, zbw: i32); + pub fn sceGuDispBuffer(width: i32, height: i32, dispbp: *mut c_void, dispbw: i32); + pub fn sceGuDrawBuffer(psm: DisplayPixelFormat, fbp: *mut c_void, fbw: i32); + pub fn sceGuDrawBufferList(psm: DisplayPixelFormat, fbp: *mut c_void, fbw: i32); + pub fn sceGuDisplay(state: bool) -> bool; + pub fn sceGuDepthFunc(function: DepthFunc); + pub fn sceGuDepthMask(mask: i32); + pub fn sceGuDepthOffset(offset: i32); + pub fn sceGuDepthRange(near: i32, far: i32); + pub fn sceGuFog(near: f32, far: f32, color: u32); + pub fn sceGuInit(); + pub fn sceGuTerm(); + pub fn sceGuBreak(mode: i32); + pub fn sceGuContinue(); + pub fn sceGuSetCallback(signal: GuCallbackId, callback: GuCallback) -> GuCallback; + pub fn sceGuSignal(behavior: SignalBehavior, signal: i32); + pub fn sceGuSendCommandf(cmd: GeCommand, argument: f32); + pub fn sceGuSendCommandi(cmd: GeCommand, argument: i32); + pub fn sceGuGetMemory(size: i32) -> *mut c_void; + pub fn sceGuStart(context_type: GuContextType, list: *mut c_void); + pub fn sceGuFinish() -> i32; + pub fn sceGuFinishId(id: u32) -> i32; + pub fn sceGuCallList(list: *const c_void); + pub fn sceGuCallMode(mode: i32); + pub fn sceGuCheckList() -> i32; + pub fn sceGuSendList(mode: GuQueueMode, list: *const c_void, context: *mut GeContext); + pub fn sceGuSwapBuffers() -> *mut c_void; + pub fn sceGuSync(mode: GuSyncMode, behavior: GuSyncBehavior) -> GeListState; + pub fn sceGuDrawArray( + prim: GuPrimitive, + vtype: i32, + count: i32, + indices: *const c_void, + vertices: *const c_void, + ); + pub fn sceGuBeginObject( + vtype: i32, + count: i32, + indices: *const c_void, + vertices: *const c_void, + ); + pub fn sceGuEndObject(); + pub fn sceGuSetStatus(state: GuState, status: i32); + pub fn sceGuGetStatus(state: GuState) -> bool; + pub fn sceGuSetAllStatus(status: i32); + pub fn sceGuGetAllStatus() -> i32; + pub fn sceGuEnable(state: GuState); + pub fn sceGuDisable(state: GuState); + pub fn sceGuLight(light: i32, type_: LightType, components: i32, position: &ScePspFVector3); + pub fn sceGuLightAtt(light: i32, atten0: f32, atten1: f32, atten2: f32); + pub fn sceGuLightColor(light: i32, component: i32, color: u32); + pub fn sceGuLightMode(mode: LightMode); + pub fn sceGuLightSpot(light: i32, direction: &ScePspFVector3, exponent: f32, cutoff: f32); + pub fn sceGuClear(flags: i32); + pub fn sceGuClearColor(color: u32); + pub fn sceGuClearDepth(depth: u32); + pub fn sceGuClearStencil(stencil: u32); + pub fn sceGuPixelMask(mask: u32); + pub fn sceGuColor(color: u32); + pub fn sceGuColorFunc(func: ColorFunc, color: u32, mask: u32); + pub fn sceGuColorMaterial(components: i32); + pub fn sceGuAlphaFunc(func: AlphaFunc, value: i32, mask: i32); + pub fn sceGuAmbient(color: u32); + pub fn sceGuAmbientColor(color: u32); + pub fn sceGuBlendFunc(op: BlendOp, src: BlendSrc, dest: BlendDst, src_fix: u32, dest_fix: u32); + pub fn sceGuMaterial(components: i32, color: u32); + pub fn sceGuModelColor(emissive: u32, ambient: u32, diffuse: u32, specular: u32); + pub fn sceGuStencilFunc(func: StencilFunc, ref_: i32, mask: i32); + pub fn sceGuStencilOp(fail: StencilOperation, zfail: StencilOperation, zpass: StencilOperation); + pub fn sceGuSpecular(power: f32); + pub fn sceGuFrontFace(order: FrontFaceDirection); + pub fn sceGuLogicalOp(op: LogicalOperation); + pub fn sceGuSetDither(matrix: &ScePspIMatrix4); + pub fn sceGuShadeModel(mode: ShadingModel); + pub fn sceGuCopyImage( + psm: DisplayPixelFormat, + sx: i32, + sy: i32, + width: i32, + height: i32, + srcw: i32, + src: *mut c_void, + dx: i32, + dy: i32, + destw: i32, + dest: *mut c_void, + ); + pub fn sceGuTexEnvColor(color: u32); + pub fn sceGuTexFilter(min: TextureFilter, mag: TextureFilter); + pub fn sceGuTexFlush(); + pub fn sceGuTexFunc(tfx: TextureEffect, tcc: TextureColorComponent); + pub fn sceGuTexImage( + mipmap: MipmapLevel, + width: i32, + height: i32, + tbw: i32, + tbp: *const c_void, + ); + pub fn sceGuTexLevelMode(mode: TextureLevelMode, bias: f32); + pub fn sceGuTexMapMode(mode: TextureMapMode, a1: u32, a2: u32); + pub fn sceGuTexMode(tpsm: TexturePixelFormat, maxmips: i32, a2: i32, swizzle: i32); + pub fn sceGuTexOffset(u: f32, v: f32); + pub fn sceGuTexProjMapMode(mode: TextureProjectionMapMode); + pub fn sceGuTexScale(u: f32, v: f32); + pub fn sceGuTexSlope(slope: f32); + pub fn sceGuTexSync(); + pub fn sceGuTexWrap(u: GuTexWrapMode, v: GuTexWrapMode); + pub fn sceGuClutLoad(num_blocks: i32, cbp: *const c_void); + pub fn sceGuClutMode(cpsm: ClutPixelFormat, shift: u32, mask: u32, a3: u32); + pub fn sceGuOffset(x: u32, y: u32); + pub fn sceGuScissor(x: i32, y: i32, w: i32, h: i32); + pub fn sceGuViewport(cx: i32, cy: i32, width: i32, height: i32); + pub fn sceGuDrawBezier( + v_type: i32, + u_count: i32, + v_count: i32, + indices: *const c_void, + vertices: *const c_void, + ); + pub fn sceGuPatchDivide(ulevel: u32, vlevel: u32); + pub fn sceGuPatchFrontFace(a0: u32); + pub fn sceGuPatchPrim(prim: PatchPrimitive); + pub fn sceGuDrawSpline( + v_type: i32, + u_count: i32, + v_count: i32, + u_edge: i32, + v_edge: i32, + indices: *const c_void, + vertices: *const c_void, + ); + pub fn sceGuSetMatrix(type_: MatrixMode, matrix: &ScePspFMatrix4); + pub fn sceGuBoneMatrix(index: u32, matrix: &ScePspFMatrix4); + pub fn sceGuMorphWeight(index: i32, weight: f32); + pub fn sceGuDrawArrayN( + primitive_type: GuPrimitive, + v_type: i32, + count: i32, + a3: i32, + indices: *const c_void, + vertices: *const c_void, + ); + + pub fn sceGumDrawArray( + prim: GuPrimitive, + v_type: i32, + count: i32, + indices: *const c_void, + vertices: *const c_void, + ); + pub fn sceGumDrawArrayN( + prim: GuPrimitive, + v_type: i32, + count: i32, + a3: i32, + indices: *const c_void, + vertices: *const c_void, + ); + pub fn sceGumDrawBezier( + v_type: i32, + u_count: i32, + v_count: i32, + indices: *const c_void, + vertices: *const c_void, + ); + pub fn sceGumDrawSpline( + v_type: i32, + u_count: i32, + v_count: i32, + u_edge: i32, + v_edge: i32, + indices: *const c_void, + vertices: *const c_void, + ); + pub fn sceGumFastInverse(); + pub fn sceGumFullInverse(); + pub fn sceGumLoadIdentity(); + pub fn sceGumLoadMatrix(m: &ScePspFMatrix4); + pub fn sceGumLookAt(eye: &ScePspFVector3, center: &ScePspFVector3, up: &ScePspFVector3); + pub fn sceGumMatrixMode(mode: MatrixMode); + pub fn sceGumMultMatrix(m: &ScePspFMatrix4); + pub fn sceGumOrtho(left: f32, right: f32, bottom: f32, top: f32, near: f32, far: f32); + pub fn sceGumPerspective(fovy: f32, aspect: f32, near: f32, far: f32); + pub fn sceGumPopMatrix(); + pub fn sceGumPushMatrix(); + pub fn sceGumRotateX(angle: f32); + pub fn sceGumRotateY(angle: f32); + pub fn sceGumRotateZ(angle: f32); + pub fn sceGumRotateXYZ(v: &ScePspFVector3); + pub fn sceGumRotateZYX(v: &ScePspFVector3); + pub fn sceGumScale(v: &ScePspFVector3); + pub fn sceGumStoreMatrix(m: &mut ScePspFMatrix4); + pub fn sceGumTranslate(v: &ScePspFVector3); + pub fn sceGumUpdateMatrix(); + + pub fn sceMp3ReserveMp3Handle(args: *mut SceMp3InitArg) -> i32; + pub fn sceMp3ReleaseMp3Handle(handle: Mp3Handle) -> i32; + pub fn sceMp3InitResource() -> i32; + pub fn sceMp3TermResource() -> i32; + pub fn sceMp3Init(handle: Mp3Handle) -> i32; + pub fn sceMp3Decode(handle: Mp3Handle, dst: *mut *mut i16) -> i32; + pub fn sceMp3GetInfoToAddStreamData( + handle: Mp3Handle, + dst: *mut *mut u8, + to_write: *mut i32, + src_pos: *mut i32, + ) -> i32; + pub fn sceMp3NotifyAddStreamData(handle: Mp3Handle, size: i32) -> i32; + pub fn sceMp3CheckStreamDataNeeded(handle: Mp3Handle) -> i32; + pub fn sceMp3SetLoopNum(handle: Mp3Handle, loop_: i32) -> i32; + pub fn sceMp3GetLoopNum(handle: Mp3Handle) -> i32; + pub fn sceMp3GetSumDecodedSample(handle: Mp3Handle) -> i32; + pub fn sceMp3GetMaxOutputSample(handle: Mp3Handle) -> i32; + pub fn sceMp3GetSamplingRate(handle: Mp3Handle) -> i32; + pub fn sceMp3GetBitRate(handle: Mp3Handle) -> i32; + pub fn sceMp3GetMp3ChannelNum(handle: Mp3Handle) -> i32; + pub fn sceMp3ResetPlayPosition(handle: Mp3Handle) -> i32; + + pub fn sceRegOpenRegistry(reg: *mut Key, mode: i32, handle: *mut RegHandle) -> i32; + pub fn sceRegFlushRegistry(handle: RegHandle) -> i32; + pub fn sceRegCloseRegistry(handle: RegHandle) -> i32; + pub fn sceRegOpenCategory( + handle: RegHandle, + name: *const u8, + mode: i32, + dir_handle: *mut RegHandle, + ) -> i32; + pub fn sceRegRemoveCategory(handle: RegHandle, name: *const u8) -> i32; + pub fn sceRegCloseCategory(dir_handle: RegHandle) -> i32; + pub fn sceRegFlushCategory(dir_handle: RegHandle) -> i32; + pub fn sceRegGetKeyInfo( + dir_handle: RegHandle, + name: *const u8, + key_handle: *mut RegHandle, + type_: *mut KeyType, + size: *mut usize, + ) -> i32; + pub fn sceRegGetKeyInfoByName( + dir_handle: RegHandle, + name: *const u8, + type_: *mut KeyType, + size: *mut usize, + ) -> i32; + pub fn sceRegGetKeyValue( + dir_handle: RegHandle, + key_handle: RegHandle, + buf: *mut c_void, + size: usize, + ) -> i32; + pub fn sceRegGetKeyValueByName( + dir_handle: RegHandle, + name: *const u8, + buf: *mut c_void, + size: usize, + ) -> i32; + pub fn sceRegSetKeyValue( + dir_handle: RegHandle, + name: *const u8, + buf: *const c_void, + size: usize, + ) -> i32; + pub fn sceRegGetKeysNum(dir_handle: RegHandle, num: *mut i32) -> i32; + pub fn sceRegGetKeys(dir_handle: RegHandle, buf: *mut u8, num: i32) -> i32; + pub fn sceRegCreateKey(dir_handle: RegHandle, name: *const u8, type_: i32, size: usize) -> i32; + pub fn sceRegRemoveRegistry(key: *mut Key) -> i32; + + pub fn sceOpenPSIDGetOpenPSID(openpsid: *mut OpenPSID) -> i32; + + pub fn sceUtilityMsgDialogInitStart(params: *mut UtilityMsgDialogParams) -> i32; + pub fn sceUtilityMsgDialogShutdownStart(); + pub fn sceUtilityMsgDialogGetStatus() -> i32; + pub fn sceUtilityMsgDialogUpdate(n: i32); + pub fn sceUtilityMsgDialogAbort() -> i32; + pub fn sceUtilityNetconfInitStart(data: *mut UtilityNetconfData) -> i32; + pub fn sceUtilityNetconfShutdownStart() -> i32; + pub fn sceUtilityNetconfUpdate(unknown: i32) -> i32; + pub fn sceUtilityNetconfGetStatus() -> i32; + pub fn sceUtilityCheckNetParam(id: i32) -> i32; + pub fn sceUtilityGetNetParam(conf: i32, param: NetParam, data: *mut UtilityNetData) -> i32; + pub fn sceUtilitySavedataInitStart(params: *mut SceUtilitySavedataParam) -> i32; + pub fn sceUtilitySavedataGetStatus() -> i32; + pub fn sceUtilitySavedataShutdownStart() -> i32; + pub fn sceUtilitySavedataUpdate(unknown: i32); + pub fn sceUtilityGameSharingInitStart(params: *mut UtilityGameSharingParams) -> i32; + pub fn sceUtilityGameSharingShutdownStart(); + pub fn sceUtilityGameSharingGetStatus() -> i32; + pub fn sceUtilityGameSharingUpdate(n: i32); + pub fn sceUtilityHtmlViewerInitStart(params: *mut UtilityHtmlViewerParam) -> i32; + pub fn sceUtilityHtmlViewerShutdownStart() -> i32; + pub fn sceUtilityHtmlViewerUpdate(n: i32) -> i32; + pub fn sceUtilityHtmlViewerGetStatus() -> i32; + pub fn sceUtilitySetSystemParamInt(id: SystemParamId, value: i32) -> i32; + pub fn sceUtilitySetSystemParamString(id: SystemParamId, str: *const u8) -> i32; + pub fn sceUtilityGetSystemParamInt(id: SystemParamId, value: *mut i32) -> i32; + pub fn sceUtilityGetSystemParamString(id: SystemParamId, str: *mut u8, len: i32) -> i32; + pub fn sceUtilityOskInitStart(params: *mut SceUtilityOskParams) -> i32; + pub fn sceUtilityOskShutdownStart() -> i32; + pub fn sceUtilityOskUpdate(n: i32) -> i32; + pub fn sceUtilityOskGetStatus() -> i32; + pub fn sceUtilityLoadNetModule(module: NetModule) -> i32; + pub fn sceUtilityUnloadNetModule(module: NetModule) -> i32; + pub fn sceUtilityLoadAvModule(module: AvModule) -> i32; + pub fn sceUtilityUnloadAvModule(module: AvModule) -> i32; + pub fn sceUtilityLoadUsbModule(module: UsbModule) -> i32; + pub fn sceUtilityUnloadUsbModule(module: UsbModule) -> i32; + pub fn sceUtilityLoadModule(module: Module) -> i32; + pub fn sceUtilityUnloadModule(module: Module) -> i32; + pub fn sceUtilityCreateNetParam(conf: i32) -> i32; + pub fn sceUtilitySetNetParam(param: NetParam, val: *const c_void) -> i32; + pub fn sceUtilityCopyNetParam(src: i32, dest: i32) -> i32; + pub fn sceUtilityDeleteNetParam(conf: i32) -> i32; + + pub fn sceNetInit( + poolsize: i32, + calloutprio: i32, + calloutstack: i32, + netintrprio: i32, + netintrstack: i32, + ) -> i32; + pub fn sceNetTerm() -> i32; + pub fn sceNetFreeThreadinfo(thid: i32) -> i32; + pub fn sceNetThreadAbort(thid: i32) -> i32; + pub fn sceNetEtherStrton(name: *mut u8, mac: *mut u8); + pub fn sceNetEtherNtostr(mac: *mut u8, name: *mut u8); + pub fn sceNetGetLocalEtherAddr(mac: *mut u8) -> i32; + pub fn sceNetGetMallocStat(stat: *mut SceNetMallocStat) -> i32; + + pub fn sceNetAdhocctlInit( + stacksize: i32, + priority: i32, + adhoc_id: *mut SceNetAdhocctlAdhocId, + ) -> i32; + pub fn sceNetAdhocctlTerm() -> i32; + pub fn sceNetAdhocctlConnect(name: *const u8) -> i32; + pub fn sceNetAdhocctlDisconnect() -> i32; + pub fn sceNetAdhocctlGetState(event: *mut i32) -> i32; + pub fn sceNetAdhocctlCreate(name: *const u8) -> i32; + pub fn sceNetAdhocctlJoin(scaninfo: *mut SceNetAdhocctlScanInfo) -> i32; + pub fn sceNetAdhocctlGetAdhocId(id: *mut SceNetAdhocctlAdhocId) -> i32; + pub fn sceNetAdhocctlCreateEnterGameMode( + name: *const u8, + unknown: i32, + num: i32, + macs: *mut u8, + timeout: u32, + unknown2: i32, + ) -> i32; + pub fn sceNetAdhocctlJoinEnterGameMode( + name: *const u8, + hostmac: *mut u8, + timeout: u32, + unknown: i32, + ) -> i32; + pub fn sceNetAdhocctlGetGameModeInfo(gamemodeinfo: *mut SceNetAdhocctlGameModeInfo) -> i32; + pub fn sceNetAdhocctlExitGameMode() -> i32; + pub fn sceNetAdhocctlGetPeerList(length: *mut i32, buf: *mut c_void) -> i32; + pub fn sceNetAdhocctlGetPeerInfo( + mac: *mut u8, + size: i32, + peerinfo: *mut SceNetAdhocctlPeerInfo, + ) -> i32; + pub fn sceNetAdhocctlScan() -> i32; + pub fn sceNetAdhocctlGetScanInfo(length: *mut i32, buf: *mut c_void) -> i32; + pub fn sceNetAdhocctlAddHandler(handler: SceNetAdhocctlHandler, unknown: *mut c_void) -> i32; + pub fn sceNetAdhocctlDelHandler(id: i32) -> i32; + pub fn sceNetAdhocctlGetNameByAddr(mac: *mut u8, nickname: *mut u8) -> i32; + pub fn sceNetAdhocctlGetAddrByName( + nickname: *mut u8, + length: *mut i32, + buf: *mut c_void, + ) -> i32; + pub fn sceNetAdhocctlGetParameter(params: *mut SceNetAdhocctlParams) -> i32; + + pub fn sceNetAdhocInit() -> i32; + pub fn sceNetAdhocTerm() -> i32; + pub fn sceNetAdhocPdpCreate(mac: *mut u8, port: u16, buf_size: u32, unk1: i32) -> i32; + pub fn sceNetAdhocPdpDelete(id: i32, unk1: i32) -> i32; + pub fn sceNetAdhocPdpSend( + id: i32, + dest_mac_addr: *mut u8, + port: u16, + data: *mut c_void, + len: u32, + timeout: u32, + nonblock: i32, + ) -> i32; + pub fn sceNetAdhocPdpRecv( + id: i32, + src_mac_addr: *mut u8, + port: *mut u16, + data: *mut c_void, + data_length: *mut c_void, + timeout: u32, + nonblock: i32, + ) -> i32; + pub fn sceNetAdhocGetPdpStat(size: *mut i32, stat: *mut SceNetAdhocPdpStat) -> i32; + pub fn sceNetAdhocGameModeCreateMaster(data: *mut c_void, size: i32) -> i32; + pub fn sceNetAdhocGameModeCreateReplica(mac: *mut u8, data: *mut c_void, size: i32) -> i32; + pub fn sceNetAdhocGameModeUpdateMaster() -> i32; + pub fn sceNetAdhocGameModeUpdateReplica(id: i32, unk1: i32) -> i32; + pub fn sceNetAdhocGameModeDeleteMaster() -> i32; + pub fn sceNetAdhocGameModeDeleteReplica(id: i32) -> i32; + pub fn sceNetAdhocPtpOpen( + srcmac: *mut u8, + srcport: u16, + destmac: *mut u8, + destport: u16, + buf_size: u32, + delay: u32, + count: i32, + unk1: i32, + ) -> i32; + pub fn sceNetAdhocPtpConnect(id: i32, timeout: u32, nonblock: i32) -> i32; + pub fn sceNetAdhocPtpListen( + srcmac: *mut u8, + srcport: u16, + buf_size: u32, + delay: u32, + count: i32, + queue: i32, + unk1: i32, + ) -> i32; + pub fn sceNetAdhocPtpAccept( + id: i32, + mac: *mut u8, + port: *mut u16, + timeout: u32, + nonblock: i32, + ) -> i32; + pub fn sceNetAdhocPtpSend( + id: i32, + data: *mut c_void, + data_size: *mut i32, + timeout: u32, + nonblock: i32, + ) -> i32; + pub fn sceNetAdhocPtpRecv( + id: i32, + data: *mut c_void, + data_size: *mut i32, + timeout: u32, + nonblock: i32, + ) -> i32; + pub fn sceNetAdhocPtpFlush(id: i32, timeout: u32, nonblock: i32) -> i32; + pub fn sceNetAdhocPtpClose(id: i32, unk1: i32) -> i32; + pub fn sceNetAdhocGetPtpStat(size: *mut i32, stat: *mut SceNetAdhocPtpStat) -> i32; +} + +extern "C" { + pub fn sceNetAdhocMatchingInit(memsize: i32) -> i32; + pub fn sceNetAdhocMatchingTerm() -> i32; + pub fn sceNetAdhocMatchingCreate( + mode: AdhocMatchingMode, + max_peers: i32, + port: u16, + buf_size: i32, + hello_delay: u32, + ping_delay: u32, + init_count: i32, + msg_delay: u32, + callback: AdhocMatchingCallback, + ) -> i32; + pub fn sceNetAdhocMatchingDelete(matching_id: i32) -> i32; + pub fn sceNetAdhocMatchingStart( + matching_id: i32, + evth_pri: i32, + evth_stack: i32, + inth_pri: i32, + inth_stack: i32, + opt_len: i32, + opt_data: *mut c_void, + ) -> i32; + pub fn sceNetAdhocMatchingStop(matching_id: i32) -> i32; + pub fn sceNetAdhocMatchingSelectTarget( + matching_id: i32, + mac: *mut u8, + opt_len: i32, + opt_data: *mut c_void, + ) -> i32; + pub fn sceNetAdhocMatchingCancelTarget(matching_id: i32, mac: *mut u8) -> i32; + pub fn sceNetAdhocMatchingCancelTargetWithOpt( + matching_id: i32, + mac: *mut u8, + opt_len: i32, + opt_data: *mut c_void, + ) -> i32; + pub fn sceNetAdhocMatchingSendData( + matching_id: i32, + mac: *mut u8, + data_len: i32, + data: *mut c_void, + ) -> i32; + pub fn sceNetAdhocMatchingAbortSendData(matching_id: i32, mac: *mut u8) -> i32; + pub fn sceNetAdhocMatchingSetHelloOpt( + matching_id: i32, + opt_len: i32, + opt_data: *mut c_void, + ) -> i32; + pub fn sceNetAdhocMatchingGetHelloOpt( + matching_id: i32, + opt_len: *mut i32, + opt_data: *mut c_void, + ) -> i32; + pub fn sceNetAdhocMatchingGetMembers( + matching_id: i32, + length: *mut i32, + buf: *mut c_void, + ) -> i32; + pub fn sceNetAdhocMatchingGetPoolMaxAlloc() -> i32; + pub fn sceNetAdhocMatchingGetPoolStat(poolstat: *mut AdhocPoolStat) -> i32; +} + +extern "C" { + pub fn sceNetApctlInit(stack_size: i32, init_priority: i32) -> i32; + pub fn sceNetApctlTerm() -> i32; + pub fn sceNetApctlGetInfo(code: ApctlInfo, pinfo: *mut SceNetApctlInfo) -> i32; + pub fn sceNetApctlAddHandler(handler: SceNetApctlHandler, parg: *mut c_void) -> i32; + pub fn sceNetApctlDelHandler(handler_id: i32) -> i32; + pub fn sceNetApctlConnect(conn_index: i32) -> i32; + pub fn sceNetApctlDisconnect() -> i32; + pub fn sceNetApctlGetState(pstate: *mut ApctlState) -> i32; + + pub fn sceNetInetInit() -> i32; + pub fn sceNetInetTerm() -> i32; + pub fn sceNetInetAccept(s: i32, addr: *mut sockaddr, addr_len: *mut socklen_t) -> i32; + pub fn sceNetInetBind(s: i32, my_addr: *const sockaddr, addr_len: socklen_t) -> i32; + pub fn sceNetInetConnect(s: i32, serv_addr: *const sockaddr, addr_len: socklen_t) -> i32; + pub fn sceNetInetGetsockopt( + s: i32, + level: i32, + opt_name: i32, + opt_val: *mut c_void, + optl_en: *mut socklen_t, + ) -> i32; + pub fn sceNetInetListen(s: i32, backlog: i32) -> i32; + pub fn sceNetInetRecv(s: i32, buf: *mut c_void, len: usize, flags: i32) -> usize; + pub fn sceNetInetRecvfrom( + s: i32, + buf: *mut c_void, + flags: usize, + arg1: i32, + from: *mut sockaddr, + from_len: *mut socklen_t, + ) -> usize; + pub fn sceNetInetSend(s: i32, buf: *const c_void, len: usize, flags: i32) -> usize; + pub fn sceNetInetSendto( + s: i32, + buf: *const c_void, + len: usize, + flags: i32, + to: *const sockaddr, + to_len: socklen_t, + ) -> usize; + pub fn sceNetInetSetsockopt( + s: i32, + level: i32, + opt_name: i32, + opt_val: *const c_void, + opt_len: socklen_t, + ) -> i32; + pub fn sceNetInetShutdown(s: i32, how: i32) -> i32; + pub fn sceNetInetSocket(domain: i32, type_: i32, protocol: i32) -> i32; + pub fn sceNetInetClose(s: i32) -> i32; + pub fn sceNetInetGetErrno() -> i32; + + pub fn sceSslInit(unknown1: i32) -> i32; + pub fn sceSslEnd() -> i32; + pub fn sceSslGetUsedMemoryMax(memory: *mut u32) -> i32; + pub fn sceSslGetUsedMemoryCurrent(memory: *mut u32) -> i32; + + pub fn sceHttpInit(unknown1: u32) -> i32; + pub fn sceHttpEnd() -> i32; + pub fn sceHttpCreateTemplate(agent: *mut u8, unknown1: i32, unknown2: i32) -> i32; + pub fn sceHttpDeleteTemplate(templateid: i32) -> i32; + pub fn sceHttpCreateConnection( + templateid: i32, + host: *mut u8, + unknown1: *mut u8, + port: u16, + unknown2: i32, + ) -> i32; + pub fn sceHttpCreateConnectionWithURL(templateid: i32, url: *const u8, unknown1: i32) -> i32; + pub fn sceHttpDeleteConnection(connection_id: i32) -> i32; + pub fn sceHttpCreateRequest( + connection_id: i32, + method: HttpMethod, + path: *mut u8, + content_length: u64, + ) -> i32; + pub fn sceHttpCreateRequestWithURL( + connection_id: i32, + method: HttpMethod, + url: *mut u8, + content_length: u64, + ) -> i32; + pub fn sceHttpDeleteRequest(request_id: i32) -> i32; + pub fn sceHttpSendRequest(request_id: i32, data: *mut c_void, data_size: u32) -> i32; + pub fn sceHttpAbortRequest(request_id: i32) -> i32; + pub fn sceHttpReadData(request_id: i32, data: *mut c_void, data_size: u32) -> i32; + pub fn sceHttpGetContentLength(request_id: i32, content_length: *mut u64) -> i32; + pub fn sceHttpGetStatusCode(request_id: i32, status_code: *mut i32) -> i32; + pub fn sceHttpSetResolveTimeOut(id: i32, timeout: u32) -> i32; + pub fn sceHttpSetResolveRetry(id: i32, count: i32) -> i32; + pub fn sceHttpSetConnectTimeOut(id: i32, timeout: u32) -> i32; + pub fn sceHttpSetSendTimeOut(id: i32, timeout: u32) -> i32; + pub fn sceHttpSetRecvTimeOut(id: i32, timeout: u32) -> i32; + pub fn sceHttpEnableKeepAlive(id: i32) -> i32; + pub fn sceHttpDisableKeepAlive(id: i32) -> i32; + pub fn sceHttpEnableRedirect(id: i32) -> i32; + pub fn sceHttpDisableRedirect(id: i32) -> i32; + pub fn sceHttpEnableCookie(id: i32) -> i32; + pub fn sceHttpDisableCookie(id: i32) -> i32; + pub fn sceHttpSaveSystemCookie() -> i32; + pub fn sceHttpLoadSystemCookie() -> i32; + pub fn sceHttpAddExtraHeader(id: i32, name: *mut u8, value: *mut u8, unknown1: i32) -> i32; + pub fn sceHttpDeleteHeader(id: i32, name: *const u8) -> i32; + pub fn sceHttpsInit(unknown1: i32, unknown2: i32, unknown3: i32, unknown4: i32) -> i32; + pub fn sceHttpsEnd() -> i32; + pub fn sceHttpsLoadDefaultCert(unknown1: i32, unknown2: i32) -> i32; + pub fn sceHttpDisableAuth(id: i32) -> i32; + pub fn sceHttpDisableCache(id: i32) -> i32; + pub fn sceHttpEnableAuth(id: i32) -> i32; + pub fn sceHttpEnableCache(id: i32) -> i32; + pub fn sceHttpEndCache() -> i32; + pub fn sceHttpGetAllHeader(request: i32, header: *mut *mut u8, header_size: *mut u32) -> i32; + pub fn sceHttpGetNetworkErrno(request: i32, err_num: *mut i32) -> i32; + pub fn sceHttpGetProxy( + id: i32, + activate_flag: *mut i32, + mode: *mut i32, + proxy_host: *mut u8, + len: usize, + proxy_port: *mut u16, + ) -> i32; + pub fn sceHttpInitCache(max_size: usize) -> i32; + pub fn sceHttpSetAuthInfoCB(id: i32, cbfunc: HttpPasswordCB) -> i32; + pub fn sceHttpSetProxy( + id: i32, + activate_flag: i32, + mode: i32, + new_proxy_host: *const u8, + new_proxy_port: u16, + ) -> i32; + pub fn sceHttpSetResHeaderMaxSize(id: i32, header_size: u32) -> i32; + pub fn sceHttpSetMallocFunction( + malloc_func: HttpMallocFunction, + free_func: HttpFreeFunction, + realloc_func: HttpReallocFunction, + ) -> i32; + + pub fn sceNetResolverInit() -> i32; + pub fn sceNetResolverCreate(rid: *mut i32, buf: *mut c_void, buf_length: u32) -> i32; + pub fn sceNetResolverDelete(rid: i32) -> i32; + pub fn sceNetResolverStartNtoA( + rid: i32, + hostname: *const u8, + addr: *mut in_addr, + timeout: u32, + retry: i32, + ) -> i32; + pub fn sceNetResolverStartAtoN( + rid: i32, + addr: *const in_addr, + hostname: *mut u8, + hostname_len: u32, + timeout: u32, + retry: i32, + ) -> i32; + pub fn sceNetResolverStop(rid: i32) -> i32; + pub fn sceNetResolverTerm() -> i32; +} diff --git a/crux-mir/lib/libc/src/cloudabi/aarch64.rs b/crux-mir/lib/libc/src/solid/aarch64.rs similarity index 76% rename from crux-mir/lib/libc/src/cloudabi/aarch64.rs rename to crux-mir/lib/libc/src/solid/aarch64.rs index 4caa6d7bb..ceabea397 100644 --- a/crux-mir/lib/libc/src/cloudabi/aarch64.rs +++ b/crux-mir/lib/libc/src/solid/aarch64.rs @@ -1,4 +1,4 @@ -pub type c_char = u8; +pub type c_char = i8; +pub type wchar_t = u32; pub type c_long = i64; pub type c_ulong = u64; -pub type wchar_t = u32; diff --git a/crux-mir/lib/libc/src/cloudabi/arm.rs b/crux-mir/lib/libc/src/solid/arm.rs similarity index 76% rename from crux-mir/lib/libc/src/cloudabi/arm.rs rename to crux-mir/lib/libc/src/solid/arm.rs index eca536074..04cc1542d 100644 --- a/crux-mir/lib/libc/src/cloudabi/arm.rs +++ b/crux-mir/lib/libc/src/solid/arm.rs @@ -1,4 +1,4 @@ -pub type c_char = u8; +pub type c_char = i8; +pub type wchar_t = u32; pub type c_long = i32; pub type c_ulong = u32; -pub type wchar_t = u32; diff --git a/crux-mir/lib/libc/src/solid/mod.rs b/crux-mir/lib/libc/src/solid/mod.rs new file mode 100644 index 000000000..f0f2ae89b --- /dev/null +++ b/crux-mir/lib/libc/src/solid/mod.rs @@ -0,0 +1,904 @@ +//! Interface to the [SOLID] C library +//! +//! [SOLID]: https://solid.kmckk.com/ + +pub type c_schar = i8; +pub type c_uchar = u8; +pub type c_short = i16; +pub type c_ushort = u16; +pub type c_int = i32; +pub type c_uint = u32; +pub type c_float = f32; +pub type c_double = f64; +pub type c_longlong = i64; +pub type c_ulonglong = u64; +pub type intmax_t = i64; +pub type uintmax_t = u64; + +pub type uintptr_t = usize; +pub type intptr_t = isize; +pub type ptrdiff_t = isize; +pub type size_t = ::uintptr_t; +pub type ssize_t = ::intptr_t; + +pub type clock_t = c_uint; +pub type time_t = i64; +pub type clockid_t = c_int; +pub type timer_t = c_int; +pub type suseconds_t = c_int; +pub type useconds_t = c_uint; + +pub type sighandler_t = size_t; + +// sys/ansi.h +pub type __caddr_t = *mut c_char; +pub type __gid_t = u32; +pub type __in_addr_t = u32; +pub type __in_port_t = u16; +pub type __mode_t = u32; +pub type __off_t = i64; +pub type __pid_t = i32; +pub type __sa_family_t = u8; +pub type __socklen_t = c_uint; +pub type __uid_t = u32; +pub type __fsblkcnt_t = u64; +pub type __fsfilcnt_t = u64; + +// locale.h +pub type locale_t = usize; + +// nl_types.h +pub type nl_item = c_long; + +// sys/types.h +pub type __va_list = *mut c_char; +pub type u_int8_t = u8; +pub type u_int16_t = u16; +pub type u_int32_t = u32; +pub type u_int64_t = u64; +pub type u_char = c_uchar; +pub type u_short = c_ushort; +pub type u_int = c_uint; +pub type u_long = c_ulong; +pub type unchar = c_uchar; +pub type ushort = c_ushort; +pub type uint = c_uint; +pub type ulong = c_ulong; +pub type u_quad_t = u64; +pub type quad_t = i64; +pub type qaddr_t = *mut quad_t; +pub type longlong_t = i64; +pub type u_longlong_t = u64; +pub type blkcnt_t = i64; +pub type blksize_t = i32; +pub type fsblkcnt_t = __fsblkcnt_t; +pub type fsfilcnt_t = __fsfilcnt_t; +pub type caddr_t = __caddr_t; +pub type daddr_t = i64; +pub type dev_t = u64; +pub type fixpt_t = u32; +pub type gid_t = __gid_t; +pub type idtype_t = c_int; +pub type id_t = u32; +pub type ino_t = u64; +pub type key_t = c_long; +pub type mode_t = __mode_t; +pub type nlink_t = u32; +pub type off_t = __off_t; +pub type pid_t = __pid_t; +pub type lwpid_t = i32; +pub type rlim_t = u64; +pub type segsz_t = i32; +pub type swblk_t = i32; +pub type mqd_t = c_int; +pub type cpuid_t = c_ulong; +pub type psetid_t = c_int; + +s! { + // stat.h + pub struct stat { + pub st_dev: dev_t, + pub st_ino: ino_t, + pub st_mode: c_short, + pub st_nlink: c_short, + pub st_uid: c_short, + pub st_gid: c_short, + pub st_rdev: dev_t, + pub st_size: off_t, + pub st_atime: time_t, + pub st_mtime: time_t, + pub st_ctime: time_t, + pub st_blksize: blksize_t, + } + + // time.h + pub struct tm { + pub tm_sec: c_int, + pub tm_min: c_int, + pub tm_hour: c_int, + pub tm_mday: c_int, + pub tm_mon: c_int, + pub tm_year: c_int, + pub tm_wday: c_int, + pub tm_yday: c_int, + pub tm_isdst: c_int, + pub tm_gmtoff: c_long, + pub tm_zone: *mut c_char, + } + + // stdlib.h + pub struct qdiv_t { + pub quot: quad_t, + pub rem: quad_t, + } + pub struct lldiv_t { + pub quot: c_longlong, + pub rem: c_longlong, + } + pub struct div_t { + pub quot: c_int, + pub rem: c_int, + } + pub struct ldiv_t { + pub quot: c_long, + pub rem: c_long, + } + + // locale.h + pub struct lconv { + pub decimal_point: *mut c_char, + pub thousands_sep: *mut c_char, + pub grouping: *mut c_char, + pub int_curr_symbol: *mut c_char, + pub currency_symbol: *mut c_char, + pub mon_decimal_point: *mut c_char, + pub mon_thousands_sep: *mut c_char, + pub mon_grouping: *mut c_char, + pub positive_sign: *mut c_char, + pub negative_sign: *mut c_char, + pub int_frac_digits: c_char, + pub frac_digits: c_char, + pub p_cs_precedes: c_char, + pub p_sep_by_space: c_char, + pub n_cs_precedes: c_char, + pub n_sep_by_space: c_char, + pub p_sign_posn: c_char, + pub n_sign_posn: c_char, + pub int_p_cs_precedes: c_char, + pub int_n_cs_precedes: c_char, + pub int_p_sep_by_space: c_char, + pub int_n_sep_by_space: c_char, + pub int_p_sign_posn: c_char, + pub int_n_sign_posn: c_char, + } + + pub struct iovec { + pub iov_base: *mut c_void, + pub iov_len: size_t, + } + + pub struct timeval { + pub tv_sec: c_long, + pub tv_usec: c_long, + } +} + +pub const INT_MIN: c_int = -2147483648; +pub const INT_MAX: c_int = 2147483647; + +pub const EXIT_FAILURE: c_int = 1; +pub const EXIT_SUCCESS: c_int = 0; +pub const RAND_MAX: c_int = 0x7fffffff; +pub const EOF: c_int = -1; +pub const SEEK_SET: c_int = 0; +pub const SEEK_CUR: c_int = 1; +pub const SEEK_END: c_int = 2; +pub const _IOFBF: c_int = 0; +pub const _IONBF: c_int = 2; +pub const _IOLBF: c_int = 1; +pub const BUFSIZ: c_uint = 1024; +pub const FOPEN_MAX: c_uint = 20; +pub const FILENAME_MAX: c_uint = 1024; + +pub const O_RDONLY: c_int = 1; +pub const O_WRONLY: c_int = 2; +pub const O_RDWR: c_int = 4; +pub const O_APPEND: c_int = 8; +pub const O_CREAT: c_int = 0x10; +pub const O_EXCL: c_int = 0x400; +pub const O_TEXT: c_int = 0x100; +pub const O_BINARY: c_int = 0x200; +pub const O_TRUNC: c_int = 0x20; +pub const S_IEXEC: c_short = 0x0040; +pub const S_IWRITE: c_short = 0x0080; +pub const S_IREAD: c_short = 0x0100; +pub const S_IFCHR: c_short = 0x2000; +pub const S_IFDIR: c_short = 0x4000; +pub const S_IFMT: c_short = 0o160000; +pub const S_IFIFO: c_short = 0o0010000; +pub const S_IFBLK: c_short = 0o0060000; +pub const S_IFREG: c_short = 0o0100000; + +pub const LC_ALL: c_int = 0; +pub const LC_COLLATE: c_int = 1; +pub const LC_CTYPE: c_int = 2; +pub const LC_MONETARY: c_int = 3; +pub const LC_NUMERIC: c_int = 4; +pub const LC_TIME: c_int = 5; +pub const LC_MESSAGES: c_int = 6; +pub const _LC_LAST: c_int = 7; + +pub const EPERM: c_int = 1; +pub const ENOENT: c_int = 2; +pub const ESRCH: c_int = 3; +pub const EINTR: c_int = 4; +pub const EIO: c_int = 5; +pub const ENXIO: c_int = 6; +pub const E2BIG: c_int = 7; +pub const ENOEXEC: c_int = 8; +pub const EBADF: c_int = 9; +pub const ECHILD: c_int = 10; +pub const EAGAIN: c_int = 11; +pub const ENOMEM: c_int = 12; +pub const EACCES: c_int = 13; +pub const EFAULT: c_int = 14; +pub const ENOTBLK: c_int = 15; +pub const EBUSY: c_int = 16; +pub const EEXIST: c_int = 17; +pub const EXDEV: c_int = 18; +pub const ENODEV: c_int = 19; +pub const ENOTDIR: c_int = 20; +pub const EISDIR: c_int = 21; +pub const EINVAL: c_int = 22; +pub const ENFILE: c_int = 23; +pub const EMFILE: c_int = 24; +pub const ENOTTY: c_int = 25; +pub const ETXTBSY: c_int = 26; +pub const EFBIG: c_int = 27; +pub const ENOSPC: c_int = 28; +pub const ESPIPE: c_int = 29; +pub const EROFS: c_int = 30; +pub const EMLINK: c_int = 31; +pub const EPIPE: c_int = 32; +pub const EDOM: c_int = 33; +pub const ERANGE: c_int = 34; + +pub const EDEADLK: c_int = 35; +pub const ENAMETOOLONG: c_int = 36; +pub const ENOLCK: c_int = 37; +pub const ENOSYS: c_int = 38; +pub const ENOTEMPTY: c_int = 39; +pub const ELOOP: c_int = 40; +pub const EWOULDBLOCK: c_int = EAGAIN; +pub const ENOMSG: c_int = 42; +pub const EIDRM: c_int = 43; +pub const ECHRNG: c_int = 44; +pub const EL2NSYNC: c_int = 45; +pub const EL3HLT: c_int = 46; +pub const EL3RST: c_int = 47; +pub const ELNRNG: c_int = 48; +pub const EUNATCH: c_int = 49; +pub const ENOCSI: c_int = 50; +pub const EL2HLT: c_int = 51; +pub const EBADE: c_int = 52; +pub const EBADR: c_int = 53; +pub const EXFULL: c_int = 54; +pub const ENOANO: c_int = 55; +pub const EBADRQC: c_int = 56; +pub const EBADSLT: c_int = 57; + +pub const EDEADLOCK: c_int = EDEADLK; + +pub const EBFONT: c_int = 59; +pub const ENOSTR: c_int = 60; +pub const ENODATA: c_int = 61; +pub const ETIME: c_int = 62; +pub const ENOSR: c_int = 63; +pub const ENONET: c_int = 64; +pub const ENOPKG: c_int = 65; +pub const EREMOTE: c_int = 66; +pub const ENOLINK: c_int = 67; +pub const EADV: c_int = 68; +pub const ESRMNT: c_int = 69; +pub const ECOMM: c_int = 70; +pub const EPROTO: c_int = 71; +pub const EMULTIHOP: c_int = 72; +pub const EDOTDOT: c_int = 73; +pub const EBADMSG: c_int = 74; +pub const EOVERFLOW: c_int = 75; +pub const ENOTUNIQ: c_int = 76; +pub const EBADFD: c_int = 77; +pub const EREMCHG: c_int = 78; +pub const ELIBACC: c_int = 79; +pub const ELIBBAD: c_int = 80; +pub const ELIBSCN: c_int = 81; +pub const ELIBMAX: c_int = 82; +pub const ELIBEXEC: c_int = 83; +pub const EILSEQ: c_int = 84; +pub const ERESTART: c_int = 85; +pub const ESTRPIPE: c_int = 86; +pub const EUSERS: c_int = 87; +pub const ENOTSOCK: c_int = 88; +pub const EDESTADDRREQ: c_int = 89; +pub const EMSGSIZE: c_int = 90; +pub const EPROTOTYPE: c_int = 91; +pub const ENOPROTOOPT: c_int = 92; +pub const EPROTONOSUPPORT: c_int = 93; +pub const ESOCKTNOSUPPORT: c_int = 94; +pub const EOPNOTSUPP: c_int = 95; +pub const EPFNOSUPPORT: c_int = 96; +pub const EAFNOSUPPORT: c_int = 97; +pub const EADDRINUSE: c_int = 98; +pub const EADDRNOTAVAIL: c_int = 99; +pub const ENETDOWN: c_int = 100; +pub const ENETUNREACH: c_int = 101; +pub const ENETRESET: c_int = 102; +pub const ECONNABORTED: c_int = 103; +pub const ECONNRESET: c_int = 104; +pub const ENOBUFS: c_int = 105; +pub const EISCONN: c_int = 106; +pub const ENOTCONN: c_int = 107; +pub const ESHUTDOWN: c_int = 108; +pub const ETOOMANYREFS: c_int = 109; +pub const ETIMEDOUT: c_int = 110; +pub const ECONNREFUSED: c_int = 111; +pub const EHOSTDOWN: c_int = 112; +pub const EHOSTUNREACH: c_int = 113; +pub const EALREADY: c_int = 114; +pub const EINPROGRESS: c_int = 115; +pub const ESTALE: c_int = 116; +pub const EUCLEAN: c_int = 117; +pub const ENOTNAM: c_int = 118; +pub const ENAVAIL: c_int = 119; +pub const EISNAM: c_int = 120; +pub const EREMOTEIO: c_int = 121; +pub const EDQUOT: c_int = 122; + +pub const ENOMEDIUM: c_int = 123; +pub const EMEDIUMTYPE: c_int = 124; +pub const ECANCELED: c_int = 125; +pub const ENOKEY: c_int = 126; +pub const EKEYEXPIRED: c_int = 127; +pub const EKEYREVOKED: c_int = 128; +pub const EKEYREJECTED: c_int = 129; + +pub const EOWNERDEAD: c_int = 130; +pub const ENOTRECOVERABLE: c_int = 131; + +pub const ENOTSUP: c_int = 132; +pub const EFTYPE: c_int = 133; + +// signal codes +pub const SIGHUP: c_int = 1; +pub const SIGINT: c_int = 2; +pub const SIGQUIT: c_int = 3; +pub const SIGILL: c_int = 4; +pub const SIGTRAP: c_int = 5; +pub const SIGABRT: c_int = 6; +pub const SIGIOT: c_int = SIGABRT; +pub const SIGEMT: c_int = 7; +pub const SIGFPE: c_int = 8; +pub const SIGKILL: c_int = 9; +pub const SIGBUS: c_int = 10; +pub const SIGSEGV: c_int = 11; +pub const SIGSYS: c_int = 12; +pub const SIGPIPE: c_int = 13; +pub const SIGALRM: c_int = 14; +pub const SIGTERM: c_int = 15; +pub const SIGURG: c_int = 16; +pub const SIGSTOP: c_int = 17; +pub const SIGTSTP: c_int = 18; +pub const SIGCONT: c_int = 19; +pub const SIGCHLD: c_int = 20; +pub const SIGTTIN: c_int = 21; +pub const SIGTTOU: c_int = 22; +pub const SIGIO: c_int = 23; +pub const SIGXCPU: c_int = 24; +pub const SIGXFSZ: c_int = 25; +pub const SIGVTALRM: c_int = 26; +pub const SIGPROF: c_int = 27; +pub const SIGWINCH: c_int = 28; +pub const SIGINFO: c_int = 29; +pub const SIGUSR1: c_int = 30; +pub const SIGUSR2: c_int = 31; +pub const SIGPWR: c_int = 32; + +#[cfg_attr(feature = "extra_traits", derive(Debug))] +pub enum FILE {} +impl ::Copy for FILE {} +impl ::Clone for FILE { + fn clone(&self) -> FILE { + *self + } +} +#[cfg_attr(feature = "extra_traits", derive(Debug))] +pub enum fpos_t {} +impl ::Copy for fpos_t {} +impl ::Clone for fpos_t { + fn clone(&self) -> fpos_t { + *self + } +} + +extern "C" { + // ctype.h + pub fn isalnum(c: c_int) -> c_int; + pub fn isalpha(c: c_int) -> c_int; + pub fn iscntrl(c: c_int) -> c_int; + pub fn isdigit(c: c_int) -> c_int; + pub fn isgraph(c: c_int) -> c_int; + pub fn islower(c: c_int) -> c_int; + pub fn isprint(c: c_int) -> c_int; + pub fn ispunct(c: c_int) -> c_int; + pub fn isspace(c: c_int) -> c_int; + pub fn isupper(c: c_int) -> c_int; + pub fn isxdigit(c: c_int) -> c_int; + pub fn isblank(c: c_int) -> c_int; + pub fn tolower(c: c_int) -> c_int; + pub fn toupper(c: c_int) -> c_int; + + // stdio.h + pub fn __get_stdio_file(fileno: c_int) -> *mut FILE; + pub fn clearerr(arg1: *mut FILE); + pub fn fclose(arg1: *mut FILE) -> c_int; + pub fn feof(arg1: *mut FILE) -> c_int; + pub fn ferror(arg1: *mut FILE) -> c_int; + pub fn fflush(arg1: *mut FILE) -> c_int; + pub fn fgetc(arg1: *mut FILE) -> c_int; + pub fn fgets(arg1: *mut c_char, arg2: c_int, arg3: *mut FILE) -> *mut c_char; + pub fn fopen(arg1: *const c_char, arg2: *const c_char) -> *mut FILE; + pub fn fprintf(arg1: *mut FILE, arg2: *const c_char, ...) -> c_int; + pub fn fputc(arg1: c_int, arg2: *mut FILE) -> c_int; + pub fn fputs(arg1: *const c_char, arg2: *mut FILE) -> c_int; + pub fn fread(arg1: *mut c_void, arg2: size_t, arg3: size_t, arg4: *mut FILE) -> size_t; + pub fn freopen(arg1: *const c_char, arg2: *const c_char, arg3: *mut FILE) -> *mut FILE; + pub fn fscanf(arg1: *mut FILE, arg2: *const c_char, ...) -> c_int; + pub fn fseek(arg1: *mut FILE, arg2: c_long, arg3: c_int) -> c_int; + pub fn ftell(arg1: *mut FILE) -> c_long; + pub fn fwrite(arg1: *const c_void, arg2: size_t, arg3: size_t, arg4: *mut FILE) -> size_t; + pub fn getc(arg1: *mut FILE) -> c_int; + pub fn getchar() -> c_int; + pub fn perror(arg1: *const c_char); + pub fn printf(arg1: *const c_char, ...) -> c_int; + pub fn putc(arg1: c_int, arg2: *mut FILE) -> c_int; + pub fn putchar(arg1: c_int) -> c_int; + pub fn puts(arg1: *const c_char) -> c_int; + pub fn remove(arg1: *const c_char) -> c_int; + pub fn rewind(arg1: *mut FILE); + pub fn scanf(arg1: *const c_char, ...) -> c_int; + pub fn setbuf(arg1: *mut FILE, arg2: *mut c_char); + pub fn setvbuf(arg1: *mut FILE, arg2: *mut c_char, arg3: c_int, arg4: size_t) -> c_int; + pub fn sscanf(arg1: *const c_char, arg2: *const c_char, ...) -> c_int; + pub fn tmpfile() -> *mut FILE; + pub fn ungetc(arg1: c_int, arg2: *mut FILE) -> c_int; + pub fn vfprintf(arg1: *mut FILE, arg2: *const c_char, arg3: __va_list) -> c_int; + pub fn vprintf(arg1: *const c_char, arg2: __va_list) -> c_int; + pub fn gets(arg1: *mut c_char) -> *mut c_char; + pub fn sprintf(arg1: *mut c_char, arg2: *const c_char, ...) -> c_int; + pub fn tmpnam(arg1: *const c_char) -> *mut c_char; + pub fn vsprintf(arg1: *mut c_char, arg2: *const c_char, arg3: __va_list) -> c_int; + pub fn rename(arg1: *const c_char, arg2: *const c_char) -> c_int; + pub fn asiprintf(arg1: *mut *mut c_char, arg2: *const c_char, ...) -> c_int; + pub fn fiprintf(arg1: *mut FILE, arg2: *const c_char, ...) -> c_int; + pub fn fiscanf(arg1: *mut FILE, arg2: *const c_char, ...) -> c_int; + pub fn iprintf(arg1: *const c_char, ...) -> c_int; + pub fn iscanf(arg1: *const c_char, ...) -> c_int; + pub fn siprintf(arg1: *mut c_char, arg2: *const c_char, ...) -> c_int; + pub fn siscanf(arg1: *mut c_char, arg2: *const c_char, ...) -> c_int; + pub fn sniprintf(arg1: *mut c_char, arg2: size_t, arg3: *const c_char, ...) -> c_int; + pub fn vasiprintf(arg1: *mut *mut c_char, arg2: *const c_char, arg3: __va_list) -> c_int; + pub fn vfiprintf(arg1: *mut FILE, arg2: *const c_char, arg3: __va_list) -> c_int; + pub fn vfiscanf(arg1: *mut FILE, arg2: *const c_char, arg3: __va_list) -> c_int; + pub fn viprintf(arg1: *const c_char, arg2: __va_list) -> c_int; + pub fn viscanf(arg1: *const c_char, arg2: __va_list) -> c_int; + pub fn vsiprintf(arg1: *mut c_char, arg2: *const c_char, arg3: __va_list) -> c_int; + pub fn vsiscanf(arg1: *const c_char, arg2: *const c_char, arg3: __va_list) -> c_int; + pub fn vsniprintf( + arg1: *mut c_char, + arg2: size_t, + arg3: *const c_char, + arg4: __va_list, + ) -> c_int; + pub fn vdiprintf(arg1: c_int, arg2: *const c_char, arg3: __va_list) -> c_int; + pub fn diprintf(arg1: c_int, arg2: *const c_char, ...) -> c_int; + pub fn fgetpos(arg1: *mut FILE, arg2: *mut fpos_t) -> c_int; + pub fn fsetpos(arg1: *mut FILE, arg2: *const fpos_t) -> c_int; + pub fn fdopen(arg1: c_int, arg2: *const c_char) -> *mut FILE; + pub fn fileno(arg1: *mut FILE) -> c_int; + pub fn flockfile(arg1: *mut FILE); + pub fn ftrylockfile(arg1: *mut FILE) -> c_int; + pub fn funlockfile(arg1: *mut FILE); + pub fn getc_unlocked(arg1: *mut FILE) -> c_int; + pub fn getchar_unlocked() -> c_int; + pub fn putc_unlocked(arg1: c_int, arg2: *mut FILE) -> c_int; + pub fn putchar_unlocked(arg1: c_int) -> c_int; + pub fn snprintf(arg1: *mut c_char, arg2: size_t, arg3: *const c_char, ...) -> c_int; + pub fn vsnprintf( + arg1: *mut c_char, + arg2: size_t, + arg3: *const c_char, + arg4: __va_list, + ) -> c_int; + pub fn getw(arg1: *mut FILE) -> c_int; + pub fn putw(arg1: c_int, arg2: *mut FILE) -> c_int; + pub fn tempnam(arg1: *const c_char, arg2: *const c_char) -> *mut c_char; + pub fn fseeko(stream: *mut FILE, offset: off_t, whence: c_int) -> c_int; + pub fn ftello(stream: *mut FILE) -> off_t; + + // stdlib.h + pub fn atof(arg1: *const c_char) -> f64; + pub fn strtod(arg1: *const c_char, arg2: *mut *mut c_char) -> f64; + pub fn drand48() -> f64; + pub fn erand48(arg1: *mut c_ushort) -> f64; + pub fn strtof(arg1: *const c_char, arg2: *mut *mut c_char) -> f32; + pub fn strtold(arg1: *const c_char, arg2: *mut *mut c_char) -> f64; + pub fn strtod_l(arg1: *const c_char, arg2: *mut *mut c_char, arg3: locale_t) -> f64; + pub fn strtof_l(arg1: *const c_char, arg2: *mut *mut c_char, arg3: locale_t) -> f32; + pub fn strtold_l(arg1: *const c_char, arg2: *mut *mut c_char, arg3: locale_t) -> f64; + pub fn _Exit(arg1: c_int) -> !; + pub fn abort() -> !; + pub fn abs(arg1: c_int) -> c_int; + pub fn atexit(arg1: ::Option) -> c_int; + pub fn atoi(arg1: *const c_char) -> c_int; + pub fn atol(arg1: *const c_char) -> c_long; + pub fn itoa(arg1: c_int, arg2: *mut c_char, arg3: c_int) -> *mut c_char; + pub fn ltoa(arg1: c_long, arg2: *mut c_char, arg3: c_int) -> *mut c_char; + pub fn ultoa(arg1: c_ulong, arg2: *mut c_char, arg3: c_int) -> *mut c_char; + pub fn bsearch( + arg1: *const c_void, + arg2: *const c_void, + arg3: size_t, + arg4: size_t, + arg5: ::Option c_int>, + ) -> *mut c_void; + pub fn calloc(arg1: size_t, arg2: size_t) -> *mut c_void; + pub fn div(arg1: c_int, arg2: c_int) -> div_t; + pub fn exit(arg1: c_int) -> !; + pub fn free(arg1: *mut c_void); + pub fn getenv(arg1: *const c_char) -> *mut c_char; + pub fn labs(arg1: c_long) -> c_long; + pub fn ldiv(arg1: c_long, arg2: c_long) -> ldiv_t; + pub fn malloc(arg1: size_t) -> *mut c_void; + pub fn qsort( + arg1: *mut c_void, + arg2: size_t, + arg3: size_t, + arg4: ::Option c_int>, + ); + pub fn rand() -> c_int; + pub fn realloc(arg1: *mut c_void, arg2: size_t) -> *mut c_void; + pub fn srand(arg1: c_uint); + pub fn strtol(arg1: *const c_char, arg2: *mut *mut c_char, arg3: c_int) -> c_long; + pub fn strtoul(arg1: *const c_char, arg2: *mut *mut c_char, arg3: c_int) -> c_ulong; + pub fn mblen(arg1: *const c_char, arg2: size_t) -> c_int; + pub fn mbstowcs(arg1: *mut wchar_t, arg2: *const c_char, arg3: size_t) -> size_t; + pub fn wctomb(arg1: *mut c_char, arg2: wchar_t) -> c_int; + pub fn mbtowc(arg1: *mut wchar_t, arg2: *const c_char, arg3: size_t) -> c_int; + pub fn wcstombs(arg1: *mut c_char, arg2: *const wchar_t, arg3: size_t) -> size_t; + pub fn rand_r(arg1: *mut c_uint) -> c_int; + pub fn jrand48(arg1: *mut c_ushort) -> c_long; + pub fn lcong48(arg1: *mut c_ushort); + pub fn lrand48() -> c_long; + pub fn mrand48() -> c_long; + pub fn nrand48(arg1: *mut c_ushort) -> c_long; + pub fn seed48(arg1: *mut c_ushort) -> *mut c_ushort; + pub fn srand48(arg1: c_long); + pub fn putenv(arg1: *mut c_char) -> c_int; + pub fn a64l(arg1: *const c_char) -> c_long; + pub fn l64a(arg1: c_long) -> *mut c_char; + pub fn random() -> c_long; + pub fn setstate(arg1: *mut c_char) -> *mut c_char; + pub fn initstate(arg1: c_uint, arg2: *mut c_char, arg3: size_t) -> *mut c_char; + pub fn srandom(arg1: c_uint); + pub fn mkostemp(arg1: *mut c_char, arg2: c_int) -> c_int; + pub fn mkostemps(arg1: *mut c_char, arg2: c_int, arg3: c_int) -> c_int; + pub fn mkdtemp(arg1: *mut c_char) -> *mut c_char; + pub fn mkstemp(arg1: *mut c_char) -> c_int; + pub fn mktemp(arg1: *mut c_char) -> *mut c_char; + pub fn atoll(arg1: *const c_char) -> c_longlong; + pub fn llabs(arg1: c_longlong) -> c_longlong; + pub fn lldiv(arg1: c_longlong, arg2: c_longlong) -> lldiv_t; + pub fn strtoll(arg1: *const c_char, arg2: *mut *mut c_char, arg3: c_int) -> c_longlong; + pub fn strtoull(arg1: *const c_char, arg2: *mut *mut c_char, arg3: c_int) -> c_ulonglong; + pub fn aligned_alloc(arg1: size_t, arg2: size_t) -> *mut c_void; + pub fn at_quick_exit(arg1: ::Option) -> c_int; + pub fn quick_exit(arg1: c_int); + pub fn setenv(arg1: *const c_char, arg2: *const c_char, arg3: c_int) -> c_int; + pub fn unsetenv(arg1: *const c_char) -> c_int; + pub fn humanize_number( + arg1: *mut c_char, + arg2: size_t, + arg3: i64, + arg4: *const c_char, + arg5: c_int, + arg6: c_int, + ) -> c_int; + pub fn dehumanize_number(arg1: *const c_char, arg2: *mut i64) -> c_int; + pub fn getenv_r(arg1: *const c_char, arg2: *mut c_char, arg3: size_t) -> c_int; + pub fn heapsort( + arg1: *mut c_void, + arg2: size_t, + arg3: size_t, + arg4: ::Option c_int>, + ) -> c_int; + pub fn mergesort( + arg1: *mut c_void, + arg2: size_t, + arg3: size_t, + arg4: ::Option c_int>, + ) -> c_int; + pub fn radixsort( + arg1: *mut *const c_uchar, + arg2: c_int, + arg3: *const c_uchar, + arg4: c_uint, + ) -> c_int; + pub fn sradixsort( + arg1: *mut *const c_uchar, + arg2: c_int, + arg3: *const c_uchar, + arg4: c_uint, + ) -> c_int; + pub fn getprogname() -> *const c_char; + pub fn setprogname(arg1: *const c_char); + pub fn qabs(arg1: quad_t) -> quad_t; + pub fn strtoq(arg1: *const c_char, arg2: *mut *mut c_char, arg3: c_int) -> quad_t; + pub fn strtouq(arg1: *const c_char, arg2: *mut *mut c_char, arg3: c_int) -> u_quad_t; + pub fn strsuftoll( + arg1: *const c_char, + arg2: *const c_char, + arg3: c_longlong, + arg4: c_longlong, + ) -> c_longlong; + pub fn strsuftollx( + arg1: *const c_char, + arg2: *const c_char, + arg3: c_longlong, + arg4: c_longlong, + arg5: *mut c_char, + arg6: size_t, + ) -> c_longlong; + pub fn l64a_r(arg1: c_long, arg2: *mut c_char, arg3: c_int) -> c_int; + pub fn qdiv(arg1: quad_t, arg2: quad_t) -> qdiv_t; + pub fn strtol_l( + arg1: *const c_char, + arg2: *mut *mut c_char, + arg3: c_int, + arg4: locale_t, + ) -> c_long; + pub fn strtoul_l( + arg1: *const c_char, + arg2: *mut *mut c_char, + arg3: c_int, + arg4: locale_t, + ) -> c_ulong; + pub fn strtoll_l( + arg1: *const c_char, + arg2: *mut *mut c_char, + arg3: c_int, + arg4: locale_t, + ) -> c_longlong; + pub fn strtoull_l( + arg1: *const c_char, + arg2: *mut *mut c_char, + arg3: c_int, + arg4: locale_t, + ) -> c_ulonglong; + pub fn strtoq_l( + arg1: *const c_char, + arg2: *mut *mut c_char, + arg3: c_int, + arg4: locale_t, + ) -> quad_t; + pub fn strtouq_l( + arg1: *const c_char, + arg2: *mut *mut c_char, + arg3: c_int, + arg4: locale_t, + ) -> u_quad_t; + pub fn _mb_cur_max_l(arg1: locale_t) -> size_t; + pub fn mblen_l(arg1: *const c_char, arg2: size_t, arg3: locale_t) -> c_int; + pub fn mbstowcs_l( + arg1: *mut wchar_t, + arg2: *const c_char, + arg3: size_t, + arg4: locale_t, + ) -> size_t; + pub fn wctomb_l(arg1: *mut c_char, arg2: wchar_t, arg3: locale_t) -> c_int; + pub fn mbtowc_l(arg1: *mut wchar_t, arg2: *const c_char, arg3: size_t, arg4: locale_t) + -> c_int; + pub fn wcstombs_l( + arg1: *mut c_char, + arg2: *const wchar_t, + arg3: size_t, + arg4: locale_t, + ) -> size_t; + + // string.h + pub fn memchr(arg1: *const c_void, arg2: c_int, arg3: size_t) -> *mut c_void; + pub fn memcmp(arg1: *const c_void, arg2: *const c_void, arg3: size_t) -> c_int; + pub fn memcpy(arg1: *mut c_void, arg2: *const c_void, arg3: size_t) -> *mut c_void; + pub fn memmove(arg1: *mut c_void, arg2: *const c_void, arg3: size_t) -> *mut c_void; + pub fn memset(arg1: *mut c_void, arg2: c_int, arg3: size_t) -> *mut c_void; + pub fn strcat(arg1: *mut c_char, arg2: *const c_char) -> *mut c_char; + pub fn strchr(arg1: *const c_char, arg2: c_int) -> *mut c_char; + pub fn strcmp(arg1: *const c_char, arg2: *const c_char) -> c_int; + pub fn strcoll(arg1: *const c_char, arg2: *const c_char) -> c_int; + pub fn strcpy(arg1: *mut c_char, arg2: *const c_char) -> *mut c_char; + pub fn strcspn(arg1: *const c_char, arg2: *const c_char) -> size_t; + pub fn strerror(arg1: c_int) -> *mut c_char; + pub fn strlen(arg1: *const c_char) -> size_t; + pub fn strncat(arg1: *mut c_char, arg2: *const c_char, arg3: size_t) -> *mut c_char; + pub fn strncmp(arg1: *const c_char, arg2: *const c_char, arg3: size_t) -> c_int; + pub fn strncpy(arg1: *mut c_char, arg2: *const c_char, arg3: size_t) -> *mut c_char; + pub fn strpbrk(arg1: *const c_char, arg2: *const c_char) -> *mut c_char; + pub fn strrchr(arg1: *const c_char, arg2: c_int) -> *mut c_char; + pub fn strspn(arg1: *const c_char, arg2: *const c_char) -> size_t; + pub fn strstr(arg1: *const c_char, arg2: *const c_char) -> *mut c_char; + pub fn strtok(arg1: *mut c_char, arg2: *const c_char) -> *mut c_char; + pub fn strtok_r(arg1: *mut c_char, arg2: *const c_char, arg3: *mut *mut c_char) -> *mut c_char; + pub fn strerror_r(arg1: c_int, arg2: *mut c_char, arg3: size_t) -> c_int; + pub fn strxfrm(arg1: *mut c_char, arg2: *const c_char, arg3: size_t) -> size_t; + pub fn memccpy( + arg1: *mut c_void, + arg2: *const c_void, + arg3: c_int, + arg4: size_t, + ) -> *mut c_void; + pub fn strdup(arg1: *const c_char) -> *mut c_char; + pub fn stpcpy(arg1: *mut c_char, arg2: *const c_char) -> *mut c_char; + pub fn stpncpy(arg1: *mut c_char, arg2: *const c_char, arg3: size_t) -> *mut c_char; + pub fn strnlen(arg1: *const c_char, arg2: size_t) -> size_t; + pub fn memmem( + arg1: *const c_void, + arg2: size_t, + arg3: *const c_void, + arg4: size_t, + ) -> *mut c_void; + pub fn strcasestr(arg1: *const c_char, arg2: *const c_char) -> *mut c_char; + pub fn strlcat(arg1: *mut c_char, arg2: *const c_char, arg3: size_t) -> size_t; + pub fn strlcpy(arg1: *mut c_char, arg2: *const c_char, arg3: size_t) -> size_t; + pub fn strsep(arg1: *mut *mut c_char, arg2: *const c_char) -> *mut c_char; + pub fn stresep(arg1: *mut *mut c_char, arg2: *const c_char, arg3: c_int) -> *mut c_char; + pub fn strndup(arg1: *const c_char, arg2: size_t) -> *mut c_char; + pub fn memrchr(arg1: *const c_void, arg2: c_int, arg3: size_t) -> *mut c_void; + pub fn explicit_memset(arg1: *mut c_void, arg2: c_int, arg3: size_t) -> *mut c_void; + pub fn consttime_memequal(arg1: *const c_void, arg2: *const c_void, arg3: size_t) -> c_int; + pub fn strcoll_l(arg1: *const c_char, arg2: *const c_char, arg3: locale_t) -> c_int; + pub fn strxfrm_l( + arg1: *mut c_char, + arg2: *const c_char, + arg3: size_t, + arg4: locale_t, + ) -> size_t; + pub fn strerror_l(arg1: c_int, arg2: locale_t) -> *mut c_char; + + // strings.h + pub fn bcmp(arg1: *const c_void, arg2: *const c_void, arg3: size_t) -> c_int; + pub fn bcopy(arg1: *const c_void, arg2: *mut c_void, arg3: size_t); + pub fn bzero(arg1: *mut c_void, arg2: size_t); + pub fn ffs(arg1: c_int) -> c_int; + pub fn popcount(arg1: c_uint) -> c_uint; + pub fn popcountl(arg1: c_ulong) -> c_uint; + pub fn popcountll(arg1: c_ulonglong) -> c_uint; + pub fn popcount32(arg1: u32) -> c_uint; + pub fn popcount64(arg1: u64) -> c_uint; + pub fn rindex(arg1: *const c_char, arg2: c_int) -> *mut c_char; + pub fn strcasecmp(arg1: *const c_char, arg2: *const c_char) -> c_int; + pub fn strncasecmp(arg1: *const c_char, arg2: *const c_char, arg3: size_t) -> c_int; + + // signal.h + pub fn signal(arg1: c_int, arg2: sighandler_t) -> sighandler_t; + pub fn raise(arg1: c_int) -> c_int; + + // time.h + pub fn asctime(arg1: *const tm) -> *mut c_char; + pub fn clock() -> clock_t; + pub fn ctime(arg1: *const time_t) -> *mut c_char; + pub fn difftime(arg1: time_t, arg2: time_t) -> f64; + pub fn gmtime(arg1: *const time_t) -> *mut tm; + pub fn localtime(arg1: *const time_t) -> *mut tm; + pub fn time(arg1: *mut time_t) -> time_t; + pub fn mktime(arg1: *mut tm) -> time_t; + pub fn strftime( + arg1: *mut c_char, + arg2: size_t, + arg3: *const c_char, + arg4: *const tm, + ) -> size_t; + pub fn utime(arg1: *const c_char, arg2: *mut time_t) -> c_int; + pub fn asctime_r(arg1: *const tm, arg2: *mut c_char) -> *mut c_char; + pub fn ctime_r(arg1: *const time_t, arg2: *mut c_char) -> *mut c_char; + pub fn gmtime_r(arg1: *const time_t, arg2: *mut tm) -> *mut tm; + pub fn localtime_r(arg1: *const time_t, arg2: *mut tm) -> *mut tm; + + // sys/stat.h + pub fn stat(arg1: *const c_char, arg2: *mut stat) -> c_int; + pub fn lstat(arg1: *const c_char, arg2: *mut stat) -> c_int; + pub fn fstat(arg1: c_int, arg2: *mut stat) -> c_int; + pub fn chmod(arg1: *const c_char, arg2: __mode_t) -> c_int; + pub fn mkdir(arg1: *const c_char, arg2: __mode_t) -> c_int; + + // fcntl.h + pub fn open(arg1: *const c_char, arg2: c_int, ...) -> c_int; + pub fn creat(arg1: *const c_char, arg2: c_int) -> c_int; + pub fn close(arg1: c_int) -> c_int; + pub fn read(arg1: c_int, arg2: *mut c_void, arg3: c_int) -> c_int; + pub fn write(arg1: c_int, arg2: *const c_void, arg3: c_int) -> c_int; + pub fn unlink(arg1: *const c_char) -> c_int; + pub fn tell(arg1: c_int) -> c_long; + pub fn dup(arg1: c_int) -> c_int; + pub fn dup2(arg1: c_int, arg2: c_int) -> c_int; + pub fn access(arg1: *const c_char, arg2: c_int) -> c_int; + pub fn rmdir(arg1: *const c_char) -> c_int; + pub fn chdir(arg1: *const c_char) -> c_int; + pub fn _exit(arg1: c_int); + pub fn getwd(arg1: *mut c_char) -> *mut c_char; + pub fn getcwd(arg1: *mut c_char, arg2: size_t) -> *mut c_char; + pub static mut optarg: *mut c_char; + pub static mut opterr: c_int; + pub static mut optind: c_int; + pub static mut optopt: c_int; + pub static mut optreset: c_int; + pub fn getopt(arg1: c_int, arg2: *mut *mut c_char, arg3: *const c_char) -> c_int; + pub static mut suboptarg: *mut c_char; + pub fn getsubopt( + arg1: *mut *mut c_char, + arg2: *const *mut c_char, + arg3: *mut *mut c_char, + ) -> c_int; + pub fn fcntl(arg1: c_int, arg2: c_int, ...) -> c_int; + pub fn getpid() -> pid_t; + pub fn sleep(arg1: c_uint) -> c_uint; + pub fn usleep(arg1: useconds_t) -> c_int; + + // locale.h + pub fn localeconv() -> *mut lconv; + pub fn setlocale(arg1: c_int, arg2: *const c_char) -> *mut c_char; + pub fn duplocale(arg1: locale_t) -> locale_t; + pub fn freelocale(arg1: locale_t); + pub fn localeconv_l(arg1: locale_t) -> *mut lconv; + pub fn newlocale(arg1: c_int, arg2: *const c_char, arg3: locale_t) -> locale_t; + + // langinfo.h + pub fn nl_langinfo(item: ::nl_item) -> *mut ::c_char; + pub fn nl_langinfo_l(item: ::nl_item, locale: locale_t) -> *mut ::c_char; + + // malloc.h + pub fn memalign(align: ::size_t, size: ::size_t) -> *mut ::c_void; + + // sys/types.h + pub fn lseek(arg1: c_int, arg2: __off_t, arg3: c_int) -> __off_t; +} + +cfg_if! { + if #[cfg(libc_core_cvoid)] { + pub use ::ffi::c_void; + } else { + // Use repr(u8) as LLVM expects `void*` to be the same as `i8*` to help + // enable more optimization opportunities around it recognizing things + // like malloc/free. + #[repr(u8)] + #[allow(missing_copy_implementations)] + #[allow(missing_debug_implementations)] + pub enum c_void { + // Two dummy variants so the #[repr] attribute can be used. + #[doc(hidden)] + __variant1, + #[doc(hidden)] + __variant2, + } + } +} + +cfg_if! { + if #[cfg(target_arch = "aarch64")] { + mod aarch64; + pub use self::aarch64::*; + } else if #[cfg(any(target_arch = "arm"))] { + mod arm; + pub use self::arm::*; + } else { + // Unknown target_arch + } +} diff --git a/crux-mir/lib/libc/src/unix/bsd/apple/b32/mod.rs b/crux-mir/lib/libc/src/unix/bsd/apple/b32/mod.rs index 9248e3adf..0f1722f97 100644 --- a/crux-mir/lib/libc/src/unix/bsd/apple/b32/mod.rs +++ b/crux-mir/lib/libc/src/unix/bsd/apple/b32/mod.rs @@ -43,6 +43,10 @@ s! { pub bh_datalen: u32, pub bh_hdrlen: ::c_ushort, } + + pub struct malloc_zone_t { + __private: [::uintptr_t; 18], // FIXME: keeping private for now + } } s_no_extra_traits! { diff --git a/crux-mir/lib/libc/src/unix/bsd/apple/b64/aarch64/align.rs b/crux-mir/lib/libc/src/unix/bsd/apple/b64/aarch64/align.rs new file mode 100644 index 000000000..29db97ec7 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/bsd/apple/b64/aarch64/align.rs @@ -0,0 +1,56 @@ +pub type mcontext_t = *mut __darwin_mcontext64; + +s_no_extra_traits! { + #[allow(missing_debug_implementations)] + pub struct max_align_t { + priv_: f64 + } +} + +s! { + pub struct ucontext_t { + pub uc_onstack: ::c_int, + pub uc_sigmask: ::sigset_t, + pub uc_stack: ::stack_t, + pub uc_link: *mut ::ucontext_t, + pub uc_mcsize: usize, + pub uc_mcontext: mcontext_t, + __mcontext_data: __darwin_mcontext64, + } + + pub struct __darwin_mcontext64 { + pub __es: __darwin_arm_exception_state64, + pub __ss: __darwin_arm_thread_state64, + pub __ns: __darwin_arm_neon_state64, + } + + pub struct __darwin_arm_exception_state64 { + pub __far: u64, + pub __esr: u32, + pub __exception: u32, + } + + pub struct __darwin_arm_thread_state64 { + pub __x: [u64; 29], + pub __fp: u64, + pub __lr: u64, + pub __sp: u64, + pub __pc: u64, + pub __cpsr: u32, + pub __pad: u32, + } + + // This type natively uses a uint128, but for a while we hacked + // it in with repr(align) and `[u64; 2]`. uint128 isn't available + // all the way back to our earliest supported versions so we + // preserver the old shim. + #[cfg_attr(not(libc_int128), repr(align(16)))] + pub struct __darwin_arm_neon_state64 { + #[cfg(libc_int128)] + pub __v: [::__uint128_t; 32], + #[cfg(not(libc_int128))] + pub __v: [[u64; 2]; 32], + pub __fpsr: u32, + pub __fpcr: u32, + } +} diff --git a/crux-mir/lib/libc/src/unix/bsd/apple/b64/aarch64/mod.rs b/crux-mir/lib/libc/src/unix/bsd/apple/b64/aarch64/mod.rs new file mode 100644 index 000000000..79e9ac842 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/bsd/apple/b64/aarch64/mod.rs @@ -0,0 +1,14 @@ +pub type boolean_t = ::c_int; + +s! { + pub struct malloc_zone_t { + __private: [::uintptr_t; 18], // FIXME: needs arm64 auth pointers support + } +} + +cfg_if! { + if #[cfg(libc_align)] { + mod align; + pub use self::align::*; + } +} diff --git a/crux-mir/lib/libc/src/unix/bsd/apple/b64/mod.rs b/crux-mir/lib/libc/src/unix/bsd/apple/b64/mod.rs index 7f7008387..48d94bcd6 100644 --- a/crux-mir/lib/libc/src/unix/bsd/apple/b64/mod.rs +++ b/crux-mir/lib/libc/src/unix/bsd/apple/b64/mod.rs @@ -2,7 +2,6 @@ pub type c_long = i64; pub type c_ulong = u64; -pub type boolean_t = ::c_uint; s! { pub struct timeval32 { @@ -113,8 +112,13 @@ extern "C" { } cfg_if! { - if #[cfg(libc_align)] { - mod align; - pub use self::align::*; + if #[cfg(target_arch = "x86_64")] { + mod x86_64; + pub use self::x86_64::*; + } else if #[cfg(target_arch = "aarch64")] { + mod aarch64; + pub use self::aarch64::*; + } else { + // Unknown target_arch } } diff --git a/crux-mir/lib/libc/src/unix/bsd/apple/b64/x86_64/align.rs b/crux-mir/lib/libc/src/unix/bsd/apple/b64/x86_64/align.rs new file mode 100644 index 000000000..ca1fe1ce2 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/bsd/apple/b64/x86_64/align.rs @@ -0,0 +1,7 @@ +s_no_extra_traits! { + #[allow(missing_debug_implementations)] + #[repr(align(16))] + pub struct max_align_t { + priv_: [f64; 2] + } +} diff --git a/crux-mir/lib/libc/src/unix/bsd/apple/b64/x86_64/mod.rs b/crux-mir/lib/libc/src/unix/bsd/apple/b64/x86_64/mod.rs new file mode 100644 index 000000000..653650c26 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/bsd/apple/b64/x86_64/mod.rs @@ -0,0 +1,180 @@ +pub type boolean_t = ::c_uint; +pub type mcontext_t = *mut __darwin_mcontext64; + +s! { + pub struct ucontext_t { + pub uc_onstack: ::c_int, + pub uc_sigmask: ::sigset_t, + pub uc_stack: ::stack_t, + pub uc_link: *mut ::ucontext_t, + pub uc_mcsize: usize, + pub uc_mcontext: mcontext_t, + } + + pub struct __darwin_mcontext64 { + pub __es: __darwin_x86_exception_state64, + pub __ss: __darwin_x86_thread_state64, + pub __fs: __darwin_x86_float_state64, + } + + pub struct __darwin_x86_exception_state64 { + pub __trapno: u16, + pub __cpu: u16, + pub __err: u32, + pub __faultvaddr: u64, + } + + pub struct __darwin_x86_thread_state64 { + pub __rax: u64, + pub __rbx: u64, + pub __rcx: u64, + pub __rdx: u64, + pub __rdi: u64, + pub __rsi: u64, + pub __rbp: u64, + pub __rsp: u64, + pub __r8: u64, + pub __r9: u64, + pub __r10: u64, + pub __r11: u64, + pub __r12: u64, + pub __r13: u64, + pub __r14: u64, + pub __r15: u64, + pub __rip: u64, + pub __rflags: u64, + pub __cs: u64, + pub __fs: u64, + pub __gs: u64, + } + + pub struct __darwin_x86_float_state64 { + pub __fpu_reserved: [::c_int; 2], + __fpu_fcw: ::c_short, + __fpu_fsw: ::c_short, + pub __fpu_ftw: u8, + pub __fpu_rsrv1: u8, + pub __fpu_fop: u16, + pub __fpu_ip: u32, + pub __fpu_cs: u16, + pub __fpu_rsrv2: u16, + pub __fpu_dp: u32, + pub __fpu_ds: u16, + pub __fpu_rsrv3: u16, + pub __fpu_mxcsr: u32, + pub __fpu_mxcsrmask: u32, + pub __fpu_stmm0: __darwin_mmst_reg, + pub __fpu_stmm1: __darwin_mmst_reg, + pub __fpu_stmm2: __darwin_mmst_reg, + pub __fpu_stmm3: __darwin_mmst_reg, + pub __fpu_stmm4: __darwin_mmst_reg, + pub __fpu_stmm5: __darwin_mmst_reg, + pub __fpu_stmm6: __darwin_mmst_reg, + pub __fpu_stmm7: __darwin_mmst_reg, + pub __fpu_xmm0: __darwin_xmm_reg, + pub __fpu_xmm1: __darwin_xmm_reg, + pub __fpu_xmm2: __darwin_xmm_reg, + pub __fpu_xmm3: __darwin_xmm_reg, + pub __fpu_xmm4: __darwin_xmm_reg, + pub __fpu_xmm5: __darwin_xmm_reg, + pub __fpu_xmm6: __darwin_xmm_reg, + pub __fpu_xmm7: __darwin_xmm_reg, + pub __fpu_xmm8: __darwin_xmm_reg, + pub __fpu_xmm9: __darwin_xmm_reg, + pub __fpu_xmm10: __darwin_xmm_reg, + pub __fpu_xmm11: __darwin_xmm_reg, + pub __fpu_xmm12: __darwin_xmm_reg, + pub __fpu_xmm13: __darwin_xmm_reg, + pub __fpu_xmm14: __darwin_xmm_reg, + pub __fpu_xmm15: __darwin_xmm_reg, + // this field is actually [u8; 96], but defining it with a bigger type + // allows us to auto-implement traits for it since the length of the + // array is less than 32 + __fpu_rsrv4: [u32; 24], + pub __fpu_reserved1: ::c_int, + } + + pub struct __darwin_mmst_reg { + pub __mmst_reg: [::c_char; 10], + pub __mmst_rsrv: [::c_char; 6], + } + + pub struct __darwin_xmm_reg { + pub __xmm_reg: [::c_char; 16], + } + + pub struct malloc_introspection_t { + _private: [::uintptr_t; 16], // FIXME: keeping private for now + } + + pub struct malloc_zone_t { + _reserved1: *mut ::c_void, + _reserved2: *mut ::c_void, + pub size: ::Option ::size_t>, + pub malloc: ::Option *mut ::c_void>, + pub calloc: ::Option *mut ::c_void>, + pub valloc: ::Option *mut ::c_void>, + pub free: ::Option, + pub realloc: ::Option *mut ::c_void>, + pub destroy: ::Option, + pub zone_name: *const ::c_char, + pub batch_malloc: ::Option ::c_uint>, + pub batch_free: ::Option, + pub introspect: *mut malloc_introspection_t, + pub version: ::c_uint, + pub memalign: ::Option *mut ::c_void>, + pub free_definite_size: ::Option, + pub pressure_relief: ::Option ::size_t>, + pub claimed_address: ::Option ::boolean_t>, + } +} + +cfg_if! { + if #[cfg(libc_align)] { + mod align; + pub use self::align::*; + } +} diff --git a/crux-mir/lib/libc/src/unix/bsd/apple/mod.rs b/crux-mir/lib/libc/src/unix/bsd/apple/mod.rs index 485cf17f7..6b391893a 100644 --- a/crux-mir/lib/libc/src/unix/bsd/apple/mod.rs +++ b/crux-mir/lib/libc/src/unix/bsd/apple/mod.rs @@ -2,6 +2,7 @@ //! //! This covers *-apple-* triples currently pub type c_char = i8; +pub type wchar_t = i32; pub type clock_t = c_ulong; pub type time_t = c_long; pub type suseconds_t = i32; @@ -25,6 +26,17 @@ pub type idtype_t = ::c_uint; pub type integer_t = ::c_int; pub type cpu_type_t = integer_t; pub type cpu_subtype_t = integer_t; +pub type natural_t = u32; +pub type mach_msg_type_number_t = natural_t; +pub type kern_return_t = ::c_int; +pub type uuid_t = [u8; 16]; +pub type task_info_t = *mut integer_t; +pub type host_info_t = *mut integer_t; +pub type task_flavor_t = natural_t; +pub type rusage_info_t = *mut ::c_void; +pub type vm_offset_t = ::uintptr_t; +pub type vm_size_t = ::uintptr_t; +pub type vm_address_t = vm_offset_t; pub type posix_spawnattr_t = *mut ::c_void; pub type posix_spawn_file_actions_t = *mut ::c_void; @@ -34,9 +46,110 @@ pub type shmatt_t = ::c_ushort; pub type sae_associd_t = u32; pub type sae_connid_t = u32; +pub type mach_port_t = ::c_uint; +pub type host_t = ::c_uint; +pub type host_flavor_t = integer_t; +pub type host_info64_t = *mut integer_t; +pub type processor_flavor_t = ::c_int; +pub type thread_flavor_t = natural_t; +pub type thread_inspect_t = ::mach_port_t; +pub type thread_act_t = ::mach_port_t; +pub type thread_act_array_t = *mut ::thread_act_t; +pub type policy_t = ::c_int; +pub type mach_vm_address_t = u64; +pub type mach_vm_offset_t = u64; +pub type mach_vm_size_t = u64; +pub type vm_map_t = ::mach_port_t; +pub type mem_entry_name_port_t = ::mach_port_t; +pub type memory_object_t = ::mach_port_t; +pub type memory_object_offset_t = ::c_ulonglong; +pub type vm_inherit_t = ::c_uint; +pub type vm_prot_t = ::c_int; + +pub type ledger_t = ::mach_port_t; +pub type ledger_array_t = *mut ::ledger_t; + +pub type iconv_t = *mut ::c_void; + +pub type processor_cpu_load_info_t = *mut processor_cpu_load_info; +pub type processor_cpu_load_info_data_t = processor_cpu_load_info; +pub type processor_basic_info_t = *mut processor_basic_info; +pub type processor_basic_info_data_t = processor_basic_info; +pub type processor_set_basic_info_data_t = processor_set_basic_info; +pub type processor_set_basic_info_t = *mut processor_set_basic_info; +pub type processor_set_load_info_data_t = processor_set_load_info; +pub type processor_set_load_info_t = *mut processor_set_load_info; +pub type processor_info_t = *mut integer_t; +pub type processor_info_array_t = *mut integer_t; + +pub type mach_task_basic_info_data_t = mach_task_basic_info; +pub type mach_task_basic_info_t = *mut mach_task_basic_info; +pub type task_thread_times_info_data_t = task_thread_times_info; +pub type task_thread_times_info_t = *mut task_thread_times_info; + +pub type thread_info_t = *mut integer_t; +pub type thread_basic_info_t = *mut thread_basic_info; +pub type thread_basic_info_data_t = thread_basic_info; +pub type thread_identifier_info_t = *mut thread_identifier_info; +pub type thread_identifier_info_data_t = thread_identifier_info; +pub type thread_extended_info_t = *mut thread_extended_info; +pub type thread_extended_info_data_t = thread_extended_info; + +pub type thread_t = ::mach_port_t; +pub type thread_policy_flavor_t = natural_t; +pub type thread_policy_t = *mut integer_t; +pub type thread_latency_qos_t = integer_t; +pub type thread_throughput_qos_t = integer_t; +pub type thread_standard_policy_data_t = thread_standard_policy; +pub type thread_standard_policy_t = *mut thread_standard_policy; +pub type thread_extended_policy_data_t = thread_extended_policy; +pub type thread_extended_policy_t = *mut thread_extended_policy; +pub type thread_time_constraint_policy_data_t = thread_time_constraint_policy; +pub type thread_time_constraint_policy_t = *mut thread_time_constraint_policy; +pub type thread_precedence_policy_data_t = thread_precedence_policy; +pub type thread_precedence_policy_t = *mut thread_precedence_policy; +pub type thread_affinity_policy_data_t = thread_affinity_policy; +pub type thread_affinity_policy_t = *mut thread_affinity_policy; +pub type thread_background_policy_data_t = thread_background_policy; +pub type thread_background_policy_t = *mut thread_background_policy; +pub type thread_latency_qos_policy_data_t = thread_latency_qos_policy; +pub type thread_latency_qos_policy_t = *mut thread_latency_qos_policy; +pub type thread_throughput_qos_policy_data_t = thread_throughput_qos_policy; +pub type thread_throughput_qos_policy_t = *mut thread_throughput_qos_policy; + +pub type pthread_introspection_hook_t = + extern "C" fn(event: ::c_uint, thread: ::pthread_t, addr: *mut ::c_void, size: ::size_t); +pub type pthread_jit_write_callback_t = ::Option ::c_int>; + +pub type os_unfair_lock = os_unfair_lock_s; +pub type os_unfair_lock_t = *mut os_unfair_lock; + +pub type os_log_t = *mut ::c_void; +pub type os_log_type_t = u8; +pub type os_signpost_id_t = u64; +pub type os_signpost_type_t = u8; + +pub type vm_statistics_t = *mut vm_statistics; +pub type vm_statistics_data_t = vm_statistics; +pub type vm_statistics64_t = *mut vm_statistics64; +pub type vm_statistics64_data_t = vm_statistics64; + +pub type task_t = ::mach_port_t; +pub type task_inspect_t = ::mach_port_t; + +pub type sysdir_search_path_enumeration_state = ::c_uint; + +pub type CCStatus = i32; +pub type CCCryptorStatus = i32; +pub type CCRNGStatus = ::CCCryptorStatus; + +pub type copyfile_state_t = *mut ::c_void; +pub type copyfile_flags_t = u32; + +pub type attrgroup_t = u32; +pub type vol_capabilities_set_t = [u32; 4]; + deprecated_mach! { - pub type vm_prot_t = ::c_int; - pub type vm_size_t = ::uintptr_t; pub type mach_timebase_info_data_t = mach_timebase_info; } @@ -49,12 +162,92 @@ impl ::Clone for timezone { } } +#[cfg_attr(feature = "extra_traits", derive(Debug))] +#[repr(u32)] +pub enum qos_class_t { + QOS_CLASS_USER_INTERACTIVE = 0x21, + QOS_CLASS_USER_INITIATED = 0x19, + QOS_CLASS_DEFAULT = 0x15, + QOS_CLASS_UTILITY = 0x11, + QOS_CLASS_BACKGROUND = 0x09, + QOS_CLASS_UNSPECIFIED = 0x00, +} +impl ::Copy for qos_class_t {} +impl ::Clone for qos_class_t { + fn clone(&self) -> qos_class_t { + *self + } +} + +#[cfg_attr(feature = "extra_traits", derive(Debug))] +#[repr(u32)] +pub enum sysdir_search_path_directory_t { + SYSDIR_DIRECTORY_APPLICATION = 1, + SYSDIR_DIRECTORY_DEMO_APPLICATION = 2, + SYSDIR_DIRECTORY_DEVELOPER_APPLICATION = 3, + SYSDIR_DIRECTORY_ADMIN_APPLICATION = 4, + SYSDIR_DIRECTORY_LIBRARY = 5, + SYSDIR_DIRECTORY_DEVELOPER = 6, + SYSDIR_DIRECTORY_USER = 7, + SYSDIR_DIRECTORY_DOCUMENTATION = 8, + SYSDIR_DIRECTORY_DOCUMENT = 9, + SYSDIR_DIRECTORY_CORESERVICE = 10, + SYSDIR_DIRECTORY_AUTOSAVED_INFORMATION = 11, + SYSDIR_DIRECTORY_DESKTOP = 12, + SYSDIR_DIRECTORY_CACHES = 13, + SYSDIR_DIRECTORY_APPLICATION_SUPPORT = 14, + SYSDIR_DIRECTORY_DOWNLOADS = 15, + SYSDIR_DIRECTORY_INPUT_METHODS = 16, + SYSDIR_DIRECTORY_MOVIES = 17, + SYSDIR_DIRECTORY_MUSIC = 18, + SYSDIR_DIRECTORY_PICTURES = 19, + SYSDIR_DIRECTORY_PRINTER_DESCRIPTION = 20, + SYSDIR_DIRECTORY_SHARED_PUBLIC = 21, + SYSDIR_DIRECTORY_PREFERENCE_PANES = 22, + SYSDIR_DIRECTORY_ALL_APPLICATIONS = 100, + SYSDIR_DIRECTORY_ALL_LIBRARIES = 101, +} +impl ::Copy for sysdir_search_path_directory_t {} +impl ::Clone for sysdir_search_path_directory_t { + fn clone(&self) -> sysdir_search_path_directory_t { + *self + } +} + +#[cfg_attr(feature = "extra_traits", derive(Debug))] +#[repr(u32)] +pub enum sysdir_search_path_domain_mask_t { + SYSDIR_DOMAIN_MASK_USER = (1 << 0), + SYSDIR_DOMAIN_MASK_LOCAL = (1 << 1), + SYSDIR_DOMAIN_MASK_NETWORK = (1 << 2), + SYSDIR_DOMAIN_MASK_SYSTEM = (1 << 3), + SYSDIR_DOMAIN_MASK_ALL = 0x0ffff, +} +impl ::Copy for sysdir_search_path_domain_mask_t {} +impl ::Clone for sysdir_search_path_domain_mask_t { + fn clone(&self) -> sysdir_search_path_domain_mask_t { + *self + } +} + s! { pub struct ip_mreq { pub imr_multiaddr: in_addr, pub imr_interface: in_addr, } + pub struct ip_mreqn { + pub imr_multiaddr: in_addr, + pub imr_address: in_addr, + pub imr_ifindex: ::c_int, + } + + pub struct ip_mreq_source { + pub imr_multiaddr: in_addr, + pub imr_sourceaddr: in_addr, + pub imr_interface: in_addr, + } + pub struct aiocb { pub aio_fildes: ::c_int, pub aio_offset: ::off_t, @@ -94,7 +287,7 @@ s! { #[deprecated( since = "0.2.55", - note = "Use the `mach` crate instead", + note = "Use the `mach2` crate instead", )] pub struct mach_timebase_info { pub numer: u32, @@ -362,7 +555,7 @@ s! { #[deprecated( since = "0.2.55", - note = "Use the `mach` crate instead", + note = "Use the `mach2` crate instead", )] pub struct mach_header { pub magic: u32, @@ -376,7 +569,7 @@ s! { #[deprecated( since = "0.2.55", - note = "Use the `mach` crate instead", + note = "Use the `mach2` crate instead", )] pub struct mach_header_64 { pub magic: u32, @@ -497,6 +690,13 @@ s! { pub s_addr: ::in_addr_t, } + // net/ndrv.h + pub struct sockaddr_ndrv { + pub snd_len: ::c_uchar, + pub snd_family: ::c_uchar, + pub snd_name: [::c_uchar; 16] // IFNAMSIZ from if.h + } + // sys/socket.h pub struct sa_endpoints_t { @@ -534,6 +734,298 @@ s! { pub tai: ::c_long, pub time_state: ::c_int, } + + pub struct thread_standard_policy { + pub no_data: natural_t, + } + + pub struct thread_extended_policy { + pub timeshare: boolean_t, + } + + pub struct thread_time_constraint_policy { + pub period: u32, + pub computation: u32, + pub constraint: u32, + pub preemptible: boolean_t, + } + + pub struct thread_precedence_policy { + pub importance: integer_t, + } + + pub struct thread_affinity_policy { + pub affinity_tag: integer_t, + } + + pub struct thread_background_policy { + pub priority: integer_t, + } + + pub struct thread_latency_qos_policy { + pub thread_latency_qos_tier: thread_latency_qos_t, + } + + pub struct thread_throughput_qos_policy { + pub thread_throughput_qos_tier: thread_throughput_qos_t, + } + + // malloc/malloc.h + pub struct malloc_statistics_t { + pub blocks_in_use: ::c_uint, + pub size_in_use: ::size_t, + pub max_size_in_use: ::size_t, + pub size_allocated: ::size_t, + } + + pub struct mstats { + pub bytes_total: ::size_t, + pub chunks_used: ::size_t, + pub bytes_used: ::size_t, + pub chunks_free: ::size_t, + pub bytes_free: ::size_t, + } + + pub struct vm_range_t { + pub address: ::vm_address_t, + pub size: ::vm_size_t, + } + + // sched.h + pub struct sched_param { + pub sched_priority: ::c_int, + __opaque: [::c_char; 4], + } + + pub struct vinfo_stat { + pub vst_dev: u32, + pub vst_mode: u16, + pub vst_nlink: u16, + pub vst_ino: u64, + pub vst_uid: ::uid_t, + pub vst_gid: ::gid_t, + pub vst_atime: i64, + pub vst_atimensec: i64, + pub vst_mtime: i64, + pub vst_mtimensec: i64, + pub vst_ctime: i64, + pub vst_ctimensec: i64, + pub vst_birthtime: i64, + pub vst_birthtimensec: i64, + pub vst_size: ::off_t, + pub vst_blocks: i64, + pub vst_blksize: i32, + pub vst_flags: u32, + pub vst_gen: u32, + pub vst_rdev: u32, + pub vst_qspare: [i64; 2], + } + + pub struct vnode_info { + pub vi_stat: vinfo_stat, + pub vi_type: ::c_int, + pub vi_pad: ::c_int, + pub vi_fsid: ::fsid_t, + } + + pub struct vnode_info_path { + pub vip_vi: vnode_info, + // Normally it's `vip_path: [::c_char; MAXPATHLEN]` but because libc supports an old rustc + // version, we go around this limitation like this. + pub vip_path: [[::c_char; 32]; 32], + } + + pub struct proc_vnodepathinfo { + pub pvi_cdir: vnode_info_path, + pub pvi_rdir: vnode_info_path, + } + + pub struct vm_statistics { + pub free_count: natural_t, + pub active_count: natural_t, + pub inactive_count: natural_t, + pub wire_count: natural_t, + pub zero_fill_count: natural_t, + pub reactivations: natural_t, + pub pageins: natural_t, + pub pageouts: natural_t, + pub faults: natural_t, + pub cow_faults: natural_t, + pub lookups: natural_t, + pub hits: natural_t, + pub purgeable_count: natural_t, + pub purges: natural_t, + pub speculative_count: natural_t, + } + + pub struct task_thread_times_info { + pub user_time: time_value_t, + pub system_time: time_value_t, + } + + pub struct rusage_info_v0 { + pub ri_uuid: [u8; 16], + pub ri_user_time: u64, + pub ri_system_time: u64, + pub ri_pkg_idle_wkups: u64, + pub ri_interrupt_wkups: u64, + pub ri_pageins: u64, + pub ri_wired_size: u64, + pub ri_resident_size: u64, + pub ri_phys_footprint: u64, + pub ri_proc_start_abstime: u64, + pub ri_proc_exit_abstime: u64, + } + + pub struct rusage_info_v1 { + pub ri_uuid: [u8; 16], + pub ri_user_time: u64, + pub ri_system_time: u64, + pub ri_pkg_idle_wkups: u64, + pub ri_interrupt_wkups: u64, + pub ri_pageins: u64, + pub ri_wired_size: u64, + pub ri_resident_size: u64, + pub ri_phys_footprint: u64, + pub ri_proc_start_abstime: u64, + pub ri_proc_exit_abstime: u64, + pub ri_child_user_time: u64, + pub ri_child_system_time: u64, + pub ri_child_pkg_idle_wkups: u64, + pub ri_child_interrupt_wkups: u64, + pub ri_child_pageins: u64, + pub ri_child_elapsed_abstime: u64, + } + + pub struct rusage_info_v2 { + pub ri_uuid: [u8; 16], + pub ri_user_time: u64, + pub ri_system_time: u64, + pub ri_pkg_idle_wkups: u64, + pub ri_interrupt_wkups: u64, + pub ri_pageins: u64, + pub ri_wired_size: u64, + pub ri_resident_size: u64, + pub ri_phys_footprint: u64, + pub ri_proc_start_abstime: u64, + pub ri_proc_exit_abstime: u64, + pub ri_child_user_time: u64, + pub ri_child_system_time: u64, + pub ri_child_pkg_idle_wkups: u64, + pub ri_child_interrupt_wkups: u64, + pub ri_child_pageins: u64, + pub ri_child_elapsed_abstime: u64, + pub ri_diskio_bytesread: u64, + pub ri_diskio_byteswritten: u64, + } + + pub struct rusage_info_v3 { + pub ri_uuid: [u8; 16], + pub ri_user_time: u64, + pub ri_system_time: u64, + pub ri_pkg_idle_wkups: u64, + pub ri_interrupt_wkups: u64, + pub ri_pageins: u64, + pub ri_wired_size: u64, + pub ri_resident_size: u64, + pub ri_phys_footprint: u64, + pub ri_proc_start_abstime: u64, + pub ri_proc_exit_abstime: u64, + pub ri_child_user_time: u64, + pub ri_child_system_time: u64, + pub ri_child_pkg_idle_wkups: u64, + pub ri_child_interrupt_wkups: u64, + pub ri_child_pageins: u64, + pub ri_child_elapsed_abstime: u64, + pub ri_diskio_bytesread: u64, + pub ri_diskio_byteswritten: u64, + pub ri_cpu_time_qos_default: u64, + pub ri_cpu_time_qos_maintenance: u64, + pub ri_cpu_time_qos_background: u64, + pub ri_cpu_time_qos_utility: u64, + pub ri_cpu_time_qos_legacy: u64, + pub ri_cpu_time_qos_user_initiated: u64, + pub ri_cpu_time_qos_user_interactive: u64, + pub ri_billed_system_time: u64, + pub ri_serviced_system_time: u64, + } + + pub struct rusage_info_v4 { + pub ri_uuid: [u8; 16], + pub ri_user_time: u64, + pub ri_system_time: u64, + pub ri_pkg_idle_wkups: u64, + pub ri_interrupt_wkups: u64, + pub ri_pageins: u64, + pub ri_wired_size: u64, + pub ri_resident_size: u64, + pub ri_phys_footprint: u64, + pub ri_proc_start_abstime: u64, + pub ri_proc_exit_abstime: u64, + pub ri_child_user_time: u64, + pub ri_child_system_time: u64, + pub ri_child_pkg_idle_wkups: u64, + pub ri_child_interrupt_wkups: u64, + pub ri_child_pageins: u64, + pub ri_child_elapsed_abstime: u64, + pub ri_diskio_bytesread: u64, + pub ri_diskio_byteswritten: u64, + pub ri_cpu_time_qos_default: u64, + pub ri_cpu_time_qos_maintenance: u64, + pub ri_cpu_time_qos_background: u64, + pub ri_cpu_time_qos_utility: u64, + pub ri_cpu_time_qos_legacy: u64, + pub ri_cpu_time_qos_user_initiated: u64, + pub ri_cpu_time_qos_user_interactive: u64, + pub ri_billed_system_time: u64, + pub ri_serviced_system_time: u64, + pub ri_logical_writes: u64, + pub ri_lifetime_max_phys_footprint: u64, + pub ri_instructions: u64, + pub ri_cycles: u64, + pub ri_billed_energy: u64, + pub ri_serviced_energy: u64, + pub ri_interval_max_phys_footprint: u64, + pub ri_runnable_time: u64, + } + + pub struct image_offset { + pub uuid: ::uuid_t, + pub offset: u32, + } + + pub struct attrlist { + pub bitmapcount: ::c_ushort, + pub reserved: u16, + pub commonattr: attrgroup_t, + pub volattr: attrgroup_t, + pub dirattr: attrgroup_t, + pub fileattr: attrgroup_t, + pub forkattr: attrgroup_t, + } + + pub struct attrreference_t { + pub attr_dataoffset: i32, + pub attr_length: u32, + } + + pub struct vol_capabilities_attr_t { + pub capabilities: vol_capabilities_set_t, + pub valid: vol_capabilities_set_t, + } + + pub struct attribute_set_t { + pub commonattr: attrgroup_t, + pub volattr: attrgroup_t, + pub dirattr: attrgroup_t, + pub fileattr: attrgroup_t, + pub forkattr: attrgroup_t, + } + + pub struct vol_attributes_attr_t { + pub validattr: attribute_set_t, + pub nativeattr: attribute_set_t, + } } s_no_extra_traits! { @@ -604,7 +1096,8 @@ s_no_extra_traits! { pub f_fstypename: [::c_char; 16], pub f_mntonname: [::c_char; 1024], pub f_mntfromname: [::c_char; 1024], - pub f_reserved: [u32; 8], + pub f_flags_ext: u32, + pub f_reserved: [u32; 7], } pub struct dirent { @@ -657,6 +1150,163 @@ s_no_extra_traits! { __unused1: *mut ::c_void, //actually a function pointer pub sigev_notify_attributes: *mut ::pthread_attr_t } + + pub struct processor_cpu_load_info { + pub cpu_ticks: [::c_uint; CPU_STATE_MAX as usize], + } + + pub struct processor_basic_info { + pub cpu_type: cpu_type_t, + pub cpu_subtype: cpu_subtype_t, + pub running: ::boolean_t, + pub slot_num: ::c_int, + pub is_master: ::boolean_t, + } + + pub struct processor_set_basic_info { + pub processor_count: ::c_int, + pub default_policy: ::c_int, + } + + pub struct processor_set_load_info { + pub task_count: ::c_int, + pub thread_count: ::c_int, + pub load_average: integer_t, + pub mach_factor: integer_t, + } + + pub struct time_value_t { + pub seconds: integer_t, + pub microseconds: integer_t, + } + + pub struct thread_basic_info { + pub user_time: time_value_t, + pub system_time: time_value_t, + pub cpu_usage: ::integer_t, + pub policy: ::policy_t, + pub run_state: ::integer_t, + pub flags: ::integer_t, + pub suspend_count: ::integer_t, + pub sleep_time: ::integer_t, + } + + pub struct thread_identifier_info { + pub thread_id: u64, + pub thread_handle: u64, + pub dispatch_qaddr: u64, + } + + pub struct thread_extended_info { + pub pth_user_time: u64, + pub pth_system_time: u64, + pub pth_cpu_usage: i32, + pub pth_policy: i32, + pub pth_run_state: i32, + pub pth_flags: i32, + pub pth_sleep_time: i32, + pub pth_curpri: i32, + pub pth_priority: i32, + pub pth_maxpriority: i32, + pub pth_name: [::c_char; MAXTHREADNAMESIZE], + } + + #[cfg_attr(libc_packedN, repr(packed(4)))] + pub struct if_data64 { + pub ifi_type: ::c_uchar, + pub ifi_typelen: ::c_uchar, + pub ifi_physical: ::c_uchar, + pub ifi_addrlen: ::c_uchar, + pub ifi_hdrlen: ::c_uchar, + pub ifi_recvquota: ::c_uchar, + pub ifi_xmitquota: ::c_uchar, + pub ifi_unused1: ::c_uchar, + pub ifi_mtu: u32, + pub ifi_metric: u32, + pub ifi_baudrate: u64, + pub ifi_ipackets: u64, + pub ifi_ierrors: u64, + pub ifi_opackets: u64, + pub ifi_oerrors: u64, + pub ifi_collisions: u64, + pub ifi_ibytes: u64, + pub ifi_obytes: u64, + pub ifi_imcasts: u64, + pub ifi_omcasts: u64, + pub ifi_iqdrops: u64, + pub ifi_noproto: u64, + pub ifi_recvtiming: u32, + pub ifi_xmittiming: u32, + #[cfg(target_pointer_width = "32")] + pub ifi_lastchange: ::timeval, + #[cfg(not(target_pointer_width = "32"))] + pub ifi_lastchange: timeval32, + } + + #[cfg_attr(libc_packedN, repr(packed(4)))] + pub struct if_msghdr2 { + pub ifm_msglen: ::c_ushort, + pub ifm_version: ::c_uchar, + pub ifm_type: ::c_uchar, + pub ifm_addrs: ::c_int, + pub ifm_flags: ::c_int, + pub ifm_index: ::c_ushort, + pub ifm_snd_len: ::c_int, + pub ifm_snd_maxlen: ::c_int, + pub ifm_snd_drops: ::c_int, + pub ifm_timer: ::c_int, + pub ifm_data: if_data64, + } + + #[cfg_attr(libc_packedN, repr(packed(8)))] + pub struct vm_statistics64 { + pub free_count: natural_t, + pub active_count: natural_t, + pub inactive_count: natural_t, + pub wire_count: natural_t, + pub zero_fill_count: u64, + pub reactivations: u64, + pub pageins: u64, + pub pageouts: u64, + pub faults: u64, + pub cow_faults: u64, + pub lookups: u64, + pub hits: u64, + pub purges: u64, + pub purgeable_count: natural_t, + pub speculative_count: natural_t, + pub decompressions: u64, + pub compressions: u64, + pub swapins: u64, + pub swapouts: u64, + pub compressor_page_count: natural_t, + pub throttled_count: natural_t, + pub external_page_count: natural_t, + pub internal_page_count: natural_t, + pub total_uncompressed_pages_in_compressor: u64, + } + + #[cfg_attr(libc_packedN, repr(packed(4)))] + pub struct mach_task_basic_info { + pub virtual_size: mach_vm_size_t, + pub resident_size: mach_vm_size_t, + pub resident_size_max: mach_vm_size_t, + pub user_time: time_value_t, + pub system_time: time_value_t, + pub policy: ::policy_t, + pub suspend_count: integer_t, + } + + #[cfg_attr(libc_packedN, repr(packed(4)))] + pub struct log2phys { + pub l2p_flags: ::c_uint, + pub l2p_contigbytes: ::off_t, + pub l2p_devoffset: ::off_t, + } + + pub struct os_unfair_lock_s { + _os_unfair_lock_opaque: u32, + } } impl siginfo_t { @@ -679,6 +1329,18 @@ impl siginfo_t { (*(self as *const siginfo_t as *const siginfo_timer)).si_value } + + pub unsafe fn si_pid(&self) -> ::pid_t { + self.si_pid + } + + pub unsafe fn si_uid(&self) -> ::uid_t { + self.si_uid + } + + pub unsafe fn si_status(&self) -> ::c_int { + self.si_status + } } cfg_if! { @@ -1259,6 +1921,694 @@ cfg_if! { self.sigev_notify_attributes.hash(state); } } + + impl PartialEq for processor_cpu_load_info { + fn eq(&self, other: &processor_cpu_load_info) -> bool { + self.cpu_ticks == other.cpu_ticks + } + } + impl Eq for processor_cpu_load_info {} + impl ::fmt::Debug for processor_cpu_load_info { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("processor_cpu_load_info") + .field("cpu_ticks", &self.cpu_ticks) + .finish() + } + } + impl ::hash::Hash for processor_cpu_load_info { + fn hash(&self, state: &mut H) { + self.cpu_ticks.hash(state); + } + } + + impl PartialEq for processor_basic_info { + fn eq(&self, other: &processor_basic_info) -> bool { + self.cpu_type == other.cpu_type + && self.cpu_subtype == other.cpu_subtype + && self.running == other.running + && self.slot_num == other.slot_num + && self.is_master == other.is_master + } + } + impl Eq for processor_basic_info {} + impl ::fmt::Debug for processor_basic_info { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("processor_basic_info") + .field("cpu_type", &self.cpu_type) + .field("cpu_subtype", &self.cpu_subtype) + .field("running", &self.running) + .field("slot_num", &self.slot_num) + .field("is_master", &self.is_master) + .finish() + } + } + impl ::hash::Hash for processor_basic_info { + fn hash(&self, state: &mut H) { + self.cpu_type.hash(state); + self.cpu_subtype.hash(state); + self.running.hash(state); + self.slot_num.hash(state); + self.is_master.hash(state); + } + } + + impl PartialEq for processor_set_basic_info { + fn eq(&self, other: &processor_set_basic_info) -> bool { + self.processor_count == other.processor_count + && self.default_policy == other.default_policy + } + } + impl Eq for processor_set_basic_info {} + impl ::fmt::Debug for processor_set_basic_info { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("processor_set_basic_info") + .field("processor_count", &self.processor_count) + .field("default_policy", &self.default_policy) + .finish() + } + } + impl ::hash::Hash for processor_set_basic_info { + fn hash(&self, state: &mut H) { + self.processor_count.hash(state); + self.default_policy.hash(state); + } + } + + impl PartialEq for processor_set_load_info { + fn eq(&self, other: &processor_set_load_info) -> bool { + self.task_count == other.task_count + && self.thread_count == other.thread_count + && self.load_average == other.load_average + && self.mach_factor == other.mach_factor + } + } + impl Eq for processor_set_load_info {} + impl ::fmt::Debug for processor_set_load_info { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("processor_set_load_info") + .field("task_count", &self.task_count) + .field("thread_count", &self.thread_count) + .field("load_average", &self.load_average) + .field("mach_factor", &self.mach_factor) + .finish() + } + } + impl ::hash::Hash for processor_set_load_info { + fn hash(&self, state: &mut H) { + self.task_count.hash(state); + self.thread_count.hash(state); + self.load_average.hash(state); + self.mach_factor.hash(state); + } + } + + impl PartialEq for time_value_t { + fn eq(&self, other: &time_value_t) -> bool { + self.seconds == other.seconds + && self.microseconds == other.microseconds + } + } + impl Eq for time_value_t {} + impl ::fmt::Debug for time_value_t { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("time_value_t") + .field("seconds", &self.seconds) + .field("microseconds", &self.microseconds) + .finish() + } + } + impl ::hash::Hash for time_value_t { + fn hash(&self, state: &mut H) { + self.seconds.hash(state); + self.microseconds.hash(state); + } + } + impl PartialEq for thread_basic_info { + fn eq(&self, other: &thread_basic_info) -> bool { + self.user_time == other.user_time + && self.system_time == other.system_time + && self.cpu_usage == other.cpu_usage + && self.policy == other.policy + && self.run_state == other.run_state + && self.flags == other.flags + && self.suspend_count == other.suspend_count + && self.sleep_time == other.sleep_time + } + } + impl Eq for thread_basic_info {} + impl ::fmt::Debug for thread_basic_info { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("thread_basic_info") + .field("user_time", &self.user_time) + .field("system_time", &self.system_time) + .field("cpu_usage", &self.cpu_usage) + .field("policy", &self.policy) + .field("run_state", &self.run_state) + .field("flags", &self.flags) + .field("suspend_count", &self.suspend_count) + .field("sleep_time", &self.sleep_time) + .finish() + } + } + impl ::hash::Hash for thread_basic_info { + fn hash(&self, state: &mut H) { + self.user_time.hash(state); + self.system_time.hash(state); + self.cpu_usage.hash(state); + self.policy.hash(state); + self.run_state.hash(state); + self.flags.hash(state); + self.suspend_count.hash(state); + self.sleep_time.hash(state); + } + } + impl PartialEq for thread_extended_info { + fn eq(&self, other: &thread_extended_info) -> bool { + self.pth_user_time == other.pth_user_time + && self.pth_system_time == other.pth_system_time + && self.pth_cpu_usage == other.pth_cpu_usage + && self.pth_policy == other.pth_policy + && self.pth_run_state == other.pth_run_state + && self.pth_flags == other.pth_flags + && self.pth_sleep_time == other.pth_sleep_time + && self.pth_curpri == other.pth_curpri + && self.pth_priority == other.pth_priority + && self.pth_maxpriority == other.pth_maxpriority + && self.pth_name + .iter() + .zip(other.pth_name.iter()) + .all(|(a,b)| a == b) + } + } + impl Eq for thread_extended_info {} + impl ::fmt::Debug for thread_extended_info { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("proc_threadinfo") + .field("pth_user_time", &self.pth_user_time) + .field("pth_system_time", &self.pth_system_time) + .field("pth_cpu_usage", &self.pth_cpu_usage) + .field("pth_policy", &self.pth_policy) + .field("pth_run_state", &self.pth_run_state) + .field("pth_flags", &self.pth_flags) + .field("pth_sleep_time", &self.pth_sleep_time) + .field("pth_curpri", &self.pth_curpri) + .field("pth_priority", &self.pth_priority) + .field("pth_maxpriority", &self.pth_maxpriority) + // FIXME: .field("pth_name", &self.pth_name) + .finish() + } + } + impl ::hash::Hash for thread_extended_info { + fn hash(&self, state: &mut H) { + self.pth_user_time.hash(state); + self.pth_system_time.hash(state); + self.pth_cpu_usage.hash(state); + self.pth_policy.hash(state); + self.pth_run_state.hash(state); + self.pth_flags.hash(state); + self.pth_sleep_time.hash(state); + self.pth_curpri.hash(state); + self.pth_priority.hash(state); + self.pth_maxpriority.hash(state); + self.pth_name.hash(state); + } + } + impl PartialEq for thread_identifier_info { + fn eq(&self, other: &thread_identifier_info) -> bool { + self.thread_id == other.thread_id + && self.thread_handle == other.thread_handle + && self.dispatch_qaddr == other.dispatch_qaddr + } + } + impl Eq for thread_identifier_info {} + impl ::fmt::Debug for thread_identifier_info { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("thread_identifier_info") + .field("thread_id", &self.thread_id) + .field("thread_handle", &self.thread_handle) + .field("dispatch_qaddr", &self.dispatch_qaddr) + .finish() + } + } + impl ::hash::Hash for thread_identifier_info { + fn hash(&self, state: &mut H) { + self.thread_id.hash(state); + self.thread_handle.hash(state); + self.dispatch_qaddr.hash(state); + } + } + impl PartialEq for if_data64 { + fn eq(&self, other: &if_data64) -> bool { + self.ifi_type == other.ifi_type && + self.ifi_typelen == other.ifi_typelen && + self.ifi_physical == other.ifi_physical && + self.ifi_addrlen == other.ifi_addrlen && + self.ifi_hdrlen == other.ifi_hdrlen && + self.ifi_recvquota == other.ifi_recvquota && + self.ifi_xmitquota == other.ifi_xmitquota && + self.ifi_unused1 == other.ifi_unused1 && + self.ifi_mtu == other.ifi_mtu && + self.ifi_metric == other.ifi_metric && + self.ifi_baudrate == other.ifi_baudrate && + self.ifi_ipackets == other.ifi_ipackets && + self.ifi_ierrors == other.ifi_ierrors && + self.ifi_opackets == other.ifi_opackets && + self.ifi_oerrors == other.ifi_oerrors && + self.ifi_collisions == other.ifi_collisions && + self.ifi_ibytes == other.ifi_ibytes && + self.ifi_obytes == other.ifi_obytes && + self.ifi_imcasts == other.ifi_imcasts && + self.ifi_omcasts == other.ifi_omcasts && + self.ifi_iqdrops == other.ifi_iqdrops && + self.ifi_noproto == other.ifi_noproto && + self.ifi_recvtiming == other.ifi_recvtiming && + self.ifi_xmittiming == other.ifi_xmittiming && + self.ifi_lastchange == other.ifi_lastchange + } + } + impl Eq for if_data64 {} + impl ::fmt::Debug for if_data64 { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + let ifi_type = self.ifi_type; + let ifi_typelen = self.ifi_typelen; + let ifi_physical = self.ifi_physical; + let ifi_addrlen = self.ifi_addrlen; + let ifi_hdrlen = self.ifi_hdrlen; + let ifi_recvquota = self.ifi_recvquota; + let ifi_xmitquota = self.ifi_xmitquota; + let ifi_unused1 = self.ifi_unused1; + let ifi_mtu = self.ifi_mtu; + let ifi_metric = self.ifi_metric; + let ifi_baudrate = self.ifi_baudrate; + let ifi_ipackets = self.ifi_ipackets; + let ifi_ierrors = self.ifi_ierrors; + let ifi_opackets = self.ifi_opackets; + let ifi_oerrors = self.ifi_oerrors; + let ifi_collisions = self.ifi_collisions; + let ifi_ibytes = self.ifi_ibytes; + let ifi_obytes = self.ifi_obytes; + let ifi_imcasts = self.ifi_imcasts; + let ifi_omcasts = self.ifi_omcasts; + let ifi_iqdrops = self.ifi_iqdrops; + let ifi_noproto = self.ifi_noproto; + let ifi_recvtiming = self.ifi_recvtiming; + let ifi_xmittiming = self.ifi_xmittiming; + let ifi_lastchange = self.ifi_lastchange; + f.debug_struct("if_data64") + .field("ifi_type", &ifi_type) + .field("ifi_typelen", &ifi_typelen) + .field("ifi_physical", &ifi_physical) + .field("ifi_addrlen", &ifi_addrlen) + .field("ifi_hdrlen", &ifi_hdrlen) + .field("ifi_recvquota", &ifi_recvquota) + .field("ifi_xmitquota", &ifi_xmitquota) + .field("ifi_unused1", &ifi_unused1) + .field("ifi_mtu", &ifi_mtu) + .field("ifi_metric", &ifi_metric) + .field("ifi_baudrate", &ifi_baudrate) + .field("ifi_ipackets", &ifi_ipackets) + .field("ifi_ierrors", &ifi_ierrors) + .field("ifi_opackets", &ifi_opackets) + .field("ifi_oerrors", &ifi_oerrors) + .field("ifi_collisions", &ifi_collisions) + .field("ifi_ibytes", &ifi_ibytes) + .field("ifi_obytes", &ifi_obytes) + .field("ifi_imcasts", &ifi_imcasts) + .field("ifi_omcasts", &ifi_omcasts) + .field("ifi_iqdrops", &ifi_iqdrops) + .field("ifi_noproto", &ifi_noproto) + .field("ifi_recvtiming", &ifi_recvtiming) + .field("ifi_xmittiming", &ifi_xmittiming) + .field("ifi_lastchange", &ifi_lastchange) + .finish() + } + } + impl ::hash::Hash for if_data64 { + fn hash(&self, state: &mut H) { + let ifi_type = self.ifi_type; + let ifi_typelen = self.ifi_typelen; + let ifi_physical = self.ifi_physical; + let ifi_addrlen = self.ifi_addrlen; + let ifi_hdrlen = self.ifi_hdrlen; + let ifi_recvquota = self.ifi_recvquota; + let ifi_xmitquota = self.ifi_xmitquota; + let ifi_unused1 = self.ifi_unused1; + let ifi_mtu = self.ifi_mtu; + let ifi_metric = self.ifi_metric; + let ifi_baudrate = self.ifi_baudrate; + let ifi_ipackets = self.ifi_ipackets; + let ifi_ierrors = self.ifi_ierrors; + let ifi_opackets = self.ifi_opackets; + let ifi_oerrors = self.ifi_oerrors; + let ifi_collisions = self.ifi_collisions; + let ifi_ibytes = self.ifi_ibytes; + let ifi_obytes = self.ifi_obytes; + let ifi_imcasts = self.ifi_imcasts; + let ifi_omcasts = self.ifi_omcasts; + let ifi_iqdrops = self.ifi_iqdrops; + let ifi_noproto = self.ifi_noproto; + let ifi_recvtiming = self.ifi_recvtiming; + let ifi_xmittiming = self.ifi_xmittiming; + let ifi_lastchange = self.ifi_lastchange; + ifi_type.hash(state); + ifi_typelen.hash(state); + ifi_physical.hash(state); + ifi_addrlen.hash(state); + ifi_hdrlen.hash(state); + ifi_recvquota.hash(state); + ifi_xmitquota.hash(state); + ifi_unused1.hash(state); + ifi_mtu.hash(state); + ifi_metric.hash(state); + ifi_baudrate.hash(state); + ifi_ipackets.hash(state); + ifi_ierrors.hash(state); + ifi_opackets.hash(state); + ifi_oerrors.hash(state); + ifi_collisions.hash(state); + ifi_ibytes.hash(state); + ifi_obytes.hash(state); + ifi_imcasts.hash(state); + ifi_omcasts.hash(state); + ifi_iqdrops.hash(state); + ifi_noproto.hash(state); + ifi_recvtiming.hash(state); + ifi_xmittiming.hash(state); + ifi_lastchange.hash(state); + } + } + impl PartialEq for if_msghdr2 { + fn eq(&self, other: &if_msghdr2) -> bool { + self.ifm_msglen == other.ifm_msglen && + self.ifm_version == other.ifm_version && + self.ifm_type == other.ifm_type && + self.ifm_addrs == other.ifm_addrs && + self.ifm_flags == other.ifm_flags && + self.ifm_index == other.ifm_index && + self.ifm_snd_len == other.ifm_snd_len && + self.ifm_snd_maxlen == other.ifm_snd_maxlen && + self.ifm_snd_drops == other.ifm_snd_drops && + self.ifm_timer == other.ifm_timer && + self.ifm_data == other.ifm_data + } + } + impl Eq for if_msghdr2 {} + impl ::fmt::Debug for if_msghdr2 { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + let ifm_msglen = self.ifm_msglen; + let ifm_version = self.ifm_version; + let ifm_type = self.ifm_type; + let ifm_addrs = self.ifm_addrs; + let ifm_flags = self.ifm_flags; + let ifm_index = self.ifm_index; + let ifm_snd_len = self.ifm_snd_len; + let ifm_snd_maxlen = self.ifm_snd_maxlen; + let ifm_snd_drops = self.ifm_snd_drops; + let ifm_timer = self.ifm_timer; + let ifm_data = self.ifm_data; + f.debug_struct("if_msghdr2") + .field("ifm_msglen", &ifm_msglen) + .field("ifm_version", &ifm_version) + .field("ifm_type", &ifm_type) + .field("ifm_addrs", &ifm_addrs) + .field("ifm_flags", &ifm_flags) + .field("ifm_index", &ifm_index) + .field("ifm_snd_len", &ifm_snd_len) + .field("ifm_snd_maxlen", &ifm_snd_maxlen) + .field("ifm_snd_drops", &ifm_snd_drops) + .field("ifm_timer", &ifm_timer) + .field("ifm_data", &ifm_data) + .finish() + } + } + impl ::hash::Hash for if_msghdr2 { + fn hash(&self, state: &mut H) { + let ifm_msglen = self.ifm_msglen; + let ifm_version = self.ifm_version; + let ifm_type = self.ifm_type; + let ifm_addrs = self.ifm_addrs; + let ifm_flags = self.ifm_flags; + let ifm_index = self.ifm_index; + let ifm_snd_len = self.ifm_snd_len; + let ifm_snd_maxlen = self.ifm_snd_maxlen; + let ifm_snd_drops = self.ifm_snd_drops; + let ifm_timer = self.ifm_timer; + let ifm_data = self.ifm_data; + ifm_msglen.hash(state); + ifm_version.hash(state); + ifm_type.hash(state); + ifm_addrs.hash(state); + ifm_flags.hash(state); + ifm_index.hash(state); + ifm_snd_len.hash(state); + ifm_snd_maxlen.hash(state); + ifm_snd_drops.hash(state); + ifm_timer.hash(state); + ifm_data.hash(state); + } + } + impl PartialEq for vm_statistics64 { + fn eq(&self, other: &vm_statistics64) -> bool { + // Otherwise rustfmt crashes... + let total_uncompressed = self.total_uncompressed_pages_in_compressor; + self.free_count == other.free_count && + self.active_count == other.active_count && + self.inactive_count == other.inactive_count && + self.wire_count == other.wire_count && + self.zero_fill_count == other.zero_fill_count && + self.reactivations == other.reactivations && + self.pageins == other.pageins && + self.pageouts == other.pageouts && + self.faults == other.faults && + self.cow_faults == other.cow_faults && + self.lookups == other.lookups && + self.hits == other.hits && + self.purges == other.purges && + self.purgeable_count == other.purgeable_count && + self.speculative_count == other.speculative_count && + self.decompressions == other.decompressions && + self.compressions == other.compressions && + self.swapins == other.swapins && + self.swapouts == other.swapouts && + self.compressor_page_count == other.compressor_page_count && + self.throttled_count == other.throttled_count && + self.external_page_count == other.external_page_count && + self.internal_page_count == other.internal_page_count && + total_uncompressed == other.total_uncompressed_pages_in_compressor + } + } + impl Eq for vm_statistics64 {} + impl ::fmt::Debug for vm_statistics64 { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + let free_count = self.free_count; + let active_count = self.active_count; + let inactive_count = self.inactive_count; + let wire_count = self.wire_count; + let zero_fill_count = self.zero_fill_count; + let reactivations = self.reactivations; + let pageins = self.pageins; + let pageouts = self.pageouts; + let faults = self.faults; + let cow_faults = self.cow_faults; + let lookups = self.lookups; + let hits = self.hits; + let purges = self.purges; + let purgeable_count = self.purgeable_count; + let speculative_count = self.speculative_count; + let decompressions = self.decompressions; + let compressions = self.compressions; + let swapins = self.swapins; + let swapouts = self.swapouts; + let compressor_page_count = self.compressor_page_count; + let throttled_count = self.throttled_count; + let external_page_count = self.external_page_count; + let internal_page_count = self.internal_page_count; + // Otherwise rustfmt crashes... + let total_uncompressed = self.total_uncompressed_pages_in_compressor; + f.debug_struct("vm_statistics64") + .field("free_count", &free_count) + .field("active_count", &active_count) + .field("inactive_count", &inactive_count) + .field("wire_count", &wire_count) + .field("zero_fill_count", &zero_fill_count) + .field("reactivations", &reactivations) + .field("pageins", &pageins) + .field("pageouts", &pageouts) + .field("faults", &faults) + .field("cow_faults", &cow_faults) + .field("lookups", &lookups) + .field("hits", &hits) + .field("purges", &purges) + .field("purgeable_count", &purgeable_count) + .field("speculative_count", &speculative_count) + .field("decompressions", &decompressions) + .field("compressions", &compressions) + .field("swapins", &swapins) + .field("swapouts", &swapouts) + .field("compressor_page_count", &compressor_page_count) + .field("throttled_count", &throttled_count) + .field("external_page_count", &external_page_count) + .field("internal_page_count", &internal_page_count) + .field("total_uncompressed_pages_in_compressor", &total_uncompressed) + .finish() + } + } + impl ::hash::Hash for vm_statistics64 { + fn hash(&self, state: &mut H) { + let free_count = self.free_count; + let active_count = self.active_count; + let inactive_count = self.inactive_count; + let wire_count = self.wire_count; + let zero_fill_count = self.zero_fill_count; + let reactivations = self.reactivations; + let pageins = self.pageins; + let pageouts = self.pageouts; + let faults = self.faults; + let cow_faults = self.cow_faults; + let lookups = self.lookups; + let hits = self.hits; + let purges = self.purges; + let purgeable_count = self.purgeable_count; + let speculative_count = self.speculative_count; + let decompressions = self.decompressions; + let compressions = self.compressions; + let swapins = self.swapins; + let swapouts = self.swapouts; + let compressor_page_count = self.compressor_page_count; + let throttled_count = self.throttled_count; + let external_page_count = self.external_page_count; + let internal_page_count = self.internal_page_count; + // Otherwise rustfmt crashes... + let total_uncompressed = self.total_uncompressed_pages_in_compressor; + free_count.hash(state); + active_count.hash(state); + inactive_count.hash(state); + wire_count.hash(state); + zero_fill_count.hash(state); + reactivations.hash(state); + pageins.hash(state); + pageouts.hash(state); + faults.hash(state); + cow_faults.hash(state); + lookups.hash(state); + hits.hash(state); + purges.hash(state); + purgeable_count.hash(state); + speculative_count.hash(state); + decompressions.hash(state); + compressions.hash(state); + swapins.hash(state); + swapouts.hash(state); + compressor_page_count.hash(state); + throttled_count.hash(state); + external_page_count.hash(state); + internal_page_count.hash(state); + total_uncompressed.hash(state); + } + } + + impl PartialEq for mach_task_basic_info { + fn eq(&self, other: &mach_task_basic_info) -> bool { + self.virtual_size == other.virtual_size + && self.resident_size == other.resident_size + && self.resident_size_max == other.resident_size_max + && self.user_time == other.user_time + && self.system_time == other.system_time + && self.policy == other.policy + && self.suspend_count == other.suspend_count + } + } + impl Eq for mach_task_basic_info {} + impl ::fmt::Debug for mach_task_basic_info { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + let virtual_size = self.virtual_size; + let resident_size = self.resident_size; + let resident_size_max = self.resident_size_max; + let user_time = self.user_time; + let system_time = self.system_time; + let policy = self.policy; + let suspend_count = self.suspend_count; + f.debug_struct("mach_task_basic_info") + .field("virtual_size", &virtual_size) + .field("resident_size", &resident_size) + .field("resident_size_max", &resident_size_max) + .field("user_time", &user_time) + .field("system_time", &system_time) + .field("policy", &policy) + .field("suspend_count", &suspend_count) + .finish() + } + } + impl ::hash::Hash for mach_task_basic_info { + fn hash(&self, state: &mut H) { + let virtual_size = self.virtual_size; + let resident_size = self.resident_size; + let resident_size_max = self.resident_size_max; + let user_time = self.user_time; + let system_time = self.system_time; + let policy = self.policy; + let suspend_count = self.suspend_count; + virtual_size.hash(state); + resident_size.hash(state); + resident_size_max.hash(state); + user_time.hash(state); + system_time.hash(state); + policy.hash(state); + suspend_count.hash(state); + } + } + + impl PartialEq for log2phys { + fn eq(&self, other: &log2phys) -> bool { + self.l2p_flags == other.l2p_flags + && self.l2p_contigbytes == other.l2p_contigbytes + && self.l2p_devoffset == other.l2p_devoffset + } + } + impl Eq for log2phys {} + impl ::fmt::Debug for log2phys { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + let l2p_flags = self.l2p_flags; + let l2p_contigbytes = self.l2p_contigbytes; + let l2p_devoffset = self.l2p_devoffset; + f.debug_struct("log2phys") + .field("l2p_flags", &l2p_flags) + .field("l2p_contigbytes", &l2p_contigbytes) + .field("l2p_devoffset", &l2p_devoffset) + .finish() + } + } + impl ::hash::Hash for log2phys { + fn hash(&self, state: &mut H) { + let l2p_flags = self.l2p_flags; + let l2p_contigbytes = self.l2p_contigbytes; + let l2p_devoffset = self.l2p_devoffset; + l2p_flags.hash(state); + l2p_contigbytes.hash(state); + l2p_devoffset.hash(state); + } + } + impl PartialEq for os_unfair_lock { + fn eq(&self, other: &os_unfair_lock) -> bool { + self._os_unfair_lock_opaque == other._os_unfair_lock_opaque + } + } + + impl Eq for os_unfair_lock {} + + impl ::fmt::Debug for os_unfair_lock { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("os_unfair_lock") + .field("_os_unfair_lock_opaque", &self._os_unfair_lock_opaque) + .finish() + } + } + + impl ::hash::Hash for os_unfair_lock { + fn hash(&self, state: &mut H) { + self._os_unfair_lock_opaque.hash(state); + } + } } } @@ -1344,7 +2694,11 @@ pub const ABMON_11: ::nl_item = 43; pub const ABMON_12: ::nl_item = 44; pub const CLOCK_REALTIME: ::clockid_t = 0; +pub const CLOCK_MONOTONIC_RAW: ::clockid_t = 4; +pub const CLOCK_MONOTONIC_RAW_APPROX: ::clockid_t = 5; pub const CLOCK_MONOTONIC: ::clockid_t = 6; +pub const CLOCK_UPTIME_RAW: ::clockid_t = 8; +pub const CLOCK_UPTIME_RAW_APPROX: ::clockid_t = 9; pub const CLOCK_PROCESS_CPUTIME_ID: ::clockid_t = 12; pub const CLOCK_THREAD_CPUTIME_ID: ::clockid_t = 16; @@ -1374,6 +2728,8 @@ pub const EOF: ::c_int = -1; pub const SEEK_SET: ::c_int = 0; pub const SEEK_CUR: ::c_int = 1; pub const SEEK_END: ::c_int = 2; +pub const SEEK_HOLE: ::c_int = 3; +pub const SEEK_DATA: ::c_int = 4; pub const _IOFBF: ::c_int = 0; pub const _IONBF: ::c_int = 2; pub const _IOLBF: ::c_int = 1; @@ -1391,10 +2747,13 @@ pub const _PC_PIPE_BUF: ::c_int = 6; pub const _PC_CHOWN_RESTRICTED: ::c_int = 7; pub const _PC_NO_TRUNC: ::c_int = 8; pub const _PC_VDISABLE: ::c_int = 9; -pub const O_DSYNC: ::c_int = 0x400000; -pub const O_NOCTTY: ::c_int = 0x20000; -pub const O_CLOEXEC: ::c_int = 0x1000000; -pub const O_DIRECTORY: ::c_int = 0x100000; +pub const O_EVTONLY: ::c_int = 0x00008000; +pub const O_NOCTTY: ::c_int = 0x00020000; +pub const O_DIRECTORY: ::c_int = 0x00100000; +pub const O_SYMLINK: ::c_int = 0x00200000; +pub const O_DSYNC: ::c_int = 0x00400000; +pub const O_CLOEXEC: ::c_int = 0x01000000; +pub const O_NOFOLLOW_ANY: ::c_int = 0x20000000; pub const S_IFIFO: mode_t = 4096; pub const S_IFCHR: mode_t = 8192; pub const S_IFBLK: mode_t = 24576; @@ -1477,6 +2836,19 @@ pub const MAP_FIXED: ::c_int = 0x0010; pub const MAP_ANON: ::c_int = 0x1000; pub const MAP_ANONYMOUS: ::c_int = MAP_ANON; +pub const CPU_STATE_USER: ::c_int = 0; +pub const CPU_STATE_SYSTEM: ::c_int = 1; +pub const CPU_STATE_IDLE: ::c_int = 2; +pub const CPU_STATE_NICE: ::c_int = 3; +pub const CPU_STATE_MAX: ::c_int = 4; + +pub const PROCESSOR_BASIC_INFO: ::c_int = 1; +pub const PROCESSOR_CPU_LOAD_INFO: ::c_int = 2; +pub const PROCESSOR_PM_REGS_INFO: ::c_int = 0x10000001; +pub const PROCESSOR_TEMPERATURE: ::c_int = 0x10000002; +pub const PROCESSOR_SET_LOAD_INFO: ::c_int = 4; +pub const PROCESSOR_SET_BASIC_INFO: ::c_int = 5; + deprecated_mach! { pub const VM_FLAGS_FIXED: ::c_int = 0x0000; pub const VM_FLAGS_ANYWHERE: ::c_int = 0x0001; @@ -1721,12 +3093,16 @@ pub const F_PREALLOCATE: ::c_int = 42; pub const F_RDADVISE: ::c_int = 44; pub const F_RDAHEAD: ::c_int = 45; pub const F_NOCACHE: ::c_int = 48; +pub const F_LOG2PHYS: ::c_int = 49; pub const F_GETPATH: ::c_int = 50; pub const F_FULLFSYNC: ::c_int = 51; pub const F_FREEZE_FS: ::c_int = 53; pub const F_THAW_FS: ::c_int = 54; pub const F_GLOBAL_NOCACHE: ::c_int = 55; pub const F_NODIRECT: ::c_int = 62; +pub const F_LOG2PHYS_EXT: ::c_int = 65; +pub const F_BARRIERFSYNC: ::c_int = 85; +pub const F_GETPATH_NOFIRMLINK: ::c_int = 102; pub const F_ALLOCATECONTIG: ::c_uint = 0x02; pub const F_ALLOCATEALL: ::c_uint = 0x04; @@ -1740,6 +3116,11 @@ pub const AT_SYMLINK_NOFOLLOW: ::c_int = 0x0020; pub const AT_SYMLINK_FOLLOW: ::c_int = 0x0040; pub const AT_REMOVEDIR: ::c_int = 0x0080; +pub const PTHREAD_INTROSPECTION_THREAD_CREATE: ::c_uint = 1; +pub const PTHREAD_INTROSPECTION_THREAD_START: ::c_uint = 2; +pub const PTHREAD_INTROSPECTION_THREAD_TERMINATE: ::c_uint = 3; +pub const PTHREAD_INTROSPECTION_THREAD_DESTROY: ::c_uint = 4; + pub const TIOCMODG: ::c_ulong = 0x40047403; pub const TIOCMODS: ::c_ulong = 0x80047404; pub const TIOCM_LE: ::c_int = 0x1; @@ -1760,8 +3141,6 @@ pub const TIOCGETD: ::c_ulong = 0x4004741a; pub const TIOCSETD: ::c_ulong = 0x8004741b; pub const TIOCIXON: ::c_uint = 0x20007481; pub const TIOCIXOFF: ::c_uint = 0x20007480; -pub const TIOCSBRK: ::c_uint = 0x2000747b; -pub const TIOCCBRK: ::c_uint = 0x2000747a; pub const TIOCSDTR: ::c_uint = 0x20007479; pub const TIOCCDTR: ::c_uint = 0x20007478; pub const TIOCGPGRP: ::c_ulong = 0x40047477; @@ -1893,6 +3272,9 @@ pub const PTHREAD_PROCESS_PRIVATE: ::c_int = 2; pub const PTHREAD_PROCESS_SHARED: ::c_int = 1; pub const PTHREAD_CREATE_JOINABLE: ::c_int = 1; pub const PTHREAD_CREATE_DETACHED: ::c_int = 2; +#[cfg(target_arch = "aarch64")] +pub const PTHREAD_STACK_MIN: ::size_t = 16384; +#[cfg(not(target_arch = "aarch64"))] pub const PTHREAD_STACK_MIN: ::size_t = 8192; pub const RLIMIT_CPU: ::c_int = 0; @@ -2176,6 +3558,7 @@ pub const pseudo_AF_RTIP: ::c_int = 22; pub const AF_IPX: ::c_int = 23; pub const AF_SIP: ::c_int = 24; pub const pseudo_AF_PIP: ::c_int = 25; +pub const AF_NDRV: ::c_int = 27; pub const AF_ISDN: ::c_int = 28; pub const AF_E164: ::c_int = AF_ISDN; pub const pseudo_AF_KEY: ::c_int = 29; @@ -2218,6 +3601,7 @@ pub const PF_SIP: ::c_int = AF_SIP; pub const PF_IPX: ::c_int = AF_IPX; pub const PF_RTIP: ::c_int = pseudo_AF_RTIP; pub const PF_PIP: ::c_int = pseudo_AF_PIP; +pub const PF_NDRV: ::c_int = AF_NDRV; pub const PF_ISDN: ::c_int = AF_ISDN; pub const PF_KEY: ::c_int = pseudo_AF_KEY; pub const PF_INET6: ::c_int = AF_INET6; @@ -2245,18 +3629,30 @@ pub const IP_RECVDSTADDR: ::c_int = 7; pub const IP_ADD_MEMBERSHIP: ::c_int = 12; pub const IP_DROP_MEMBERSHIP: ::c_int = 13; pub const IP_RECVIF: ::c_int = 20; +pub const IP_BOUND_IF: ::c_int = 25; pub const IP_PKTINFO: ::c_int = 26; pub const IP_RECVTOS: ::c_int = 27; +pub const IP_DONTFRAG: ::c_int = 28; pub const IPV6_JOIN_GROUP: ::c_int = 12; pub const IPV6_LEAVE_GROUP: ::c_int = 13; +pub const IPV6_CHECKSUM: ::c_int = 26; pub const IPV6_RECVTCLASS: ::c_int = 35; pub const IPV6_TCLASS: ::c_int = 36; pub const IPV6_PKTINFO: ::c_int = 46; +pub const IPV6_HOPLIMIT: ::c_int = 47; pub const IPV6_RECVPKTINFO: ::c_int = 61; +pub const IPV6_DONTFRAG: ::c_int = 62; +pub const IP_ADD_SOURCE_MEMBERSHIP: ::c_int = 70; +pub const IP_DROP_SOURCE_MEMBERSHIP: ::c_int = 71; +pub const IP_BLOCK_SOURCE: ::c_int = 72; +pub const IP_UNBLOCK_SOURCE: ::c_int = 73; +pub const IPV6_BOUND_IF: ::c_int = 125; pub const TCP_NOPUSH: ::c_int = 4; pub const TCP_NOOPT: ::c_int = 8; pub const TCP_KEEPALIVE: ::c_int = 0x10; +pub const TCP_KEEPINTVL: ::c_int = 0x101; +pub const TCP_KEEPCNT: ::c_int = 0x102; /// Enable/Disable TCP Fastopen on this socket pub const TCP_FASTOPEN: ::c_int = 0x105; @@ -2302,6 +3698,7 @@ pub const SO_NOADDRERR: ::c_int = 0x1023; pub const SO_NWRITE: ::c_int = 0x1024; pub const SO_REUSESHAREUID: ::c_int = 0x1025; pub const SO_NOTIFYCONFLICT: ::c_int = 0x1026; +pub const SO_LINGER_SEC: ::c_int = 0x1080; pub const SO_RANDOMPORT: ::c_int = 0x1082; pub const SO_NP_EXTENSIONS: ::c_int = 0x1083; @@ -2477,6 +3874,11 @@ pub const _SC_TRACE_NAME_MAX: ::c_int = 128; pub const _SC_TRACE_SYS_MAX: ::c_int = 129; pub const _SC_TRACE_USER_EVENT_MAX: ::c_int = 130; pub const _SC_PASS_MAX: ::c_int = 131; +// `confstr` keys (only the values guaranteed by `man confstr`). +pub const _CS_PATH: ::c_int = 1; +pub const _CS_DARWIN_USER_DIR: ::c_int = 65536; +pub const _CS_DARWIN_USER_TEMP_DIR: ::c_int = 65537; +pub const _CS_DARWIN_USER_CACHE_DIR: ::c_int = 65538; pub const PTHREAD_MUTEX_NORMAL: ::c_int = 0; pub const PTHREAD_MUTEX_ERRORCHECK: ::c_int = 1; @@ -2498,6 +3900,20 @@ pub const PTHREAD_RWLOCK_INITIALIZER: pthread_rwlock_t = pthread_rwlock_t { __opaque: [0; __PTHREAD_RWLOCK_SIZE__], }; +pub const OS_UNFAIR_LOCK_INIT: os_unfair_lock = os_unfair_lock { + _os_unfair_lock_opaque: 0, +}; + +pub const OS_LOG_TYPE_DEFAULT: ::os_log_type_t = 0x00; +pub const OS_LOG_TYPE_INFO: ::os_log_type_t = 0x01; +pub const OS_LOG_TYPE_DEBUG: ::os_log_type_t = 0x02; +pub const OS_LOG_TYPE_ERROR: ::os_log_type_t = 0x10; +pub const OS_LOG_TYPE_FAULT: ::os_log_type_t = 0x11; + +pub const OS_SIGNPOST_EVENT: ::os_signpost_type_t = 0x00; +pub const OS_SIGNPOST_INTERVAL_BEGIN: ::os_signpost_type_t = 0x01; +pub const OS_SIGNPOST_INTERVAL_END: ::os_signpost_type_t = 0x02; + pub const MINSIGSTKSZ: ::size_t = 32768; pub const SIGSTKSZ: ::size_t = 131072; @@ -2505,6 +3921,10 @@ pub const FD_SETSIZE: usize = 1024; pub const ST_NOSUID: ::c_ulong = 2; +pub const SCHED_OTHER: ::c_int = 1; +pub const SCHED_FIFO: ::c_int = 4; +pub const SCHED_RR: ::c_int = 2; + pub const EVFILT_READ: i16 = -1; pub const EVFILT_WRITE: i16 = -2; pub const EVFILT_AIO: i16 = -3; @@ -2785,6 +4205,59 @@ pub const KERN_PROC_TTY: ::c_int = 4; pub const KERN_PROC_UID: ::c_int = 5; pub const KERN_PROC_RUID: ::c_int = 6; pub const KERN_PROC_LCID: ::c_int = 7; +pub const KERN_SUCCESS: ::c_int = 0; +pub const KERN_INVALID_ADDRESS: ::c_int = 1; +pub const KERN_PROTECTION_FAILURE: ::c_int = 2; +pub const KERN_NO_SPACE: ::c_int = 3; +pub const KERN_INVALID_ARGUMENT: ::c_int = 4; +pub const KERN_FAILURE: ::c_int = 5; +pub const KERN_RESOURCE_SHORTAGE: ::c_int = 6; +pub const KERN_NOT_RECEIVER: ::c_int = 7; +pub const KERN_NO_ACCESS: ::c_int = 8; +pub const KERN_MEMORY_FAILURE: ::c_int = 9; +pub const KERN_MEMORY_ERROR: ::c_int = 10; +pub const KERN_ALREADY_IN_SET: ::c_int = 11; +pub const KERN_NOT_IN_SET: ::c_int = 12; +pub const KERN_NAME_EXISTS: ::c_int = 13; +pub const KERN_ABORTED: ::c_int = 14; +pub const KERN_INVALID_NAME: ::c_int = 15; +pub const KERN_INVALID_TASK: ::c_int = 16; +pub const KERN_INVALID_RIGHT: ::c_int = 17; +pub const KERN_INVALID_VALUE: ::c_int = 18; +pub const KERN_UREFS_OVERFLOW: ::c_int = 19; +pub const KERN_INVALID_CAPABILITY: ::c_int = 20; +pub const KERN_RIGHT_EXISTS: ::c_int = 21; +pub const KERN_INVALID_HOST: ::c_int = 22; +pub const KERN_MEMORY_PRESENT: ::c_int = 23; +pub const KERN_MEMORY_DATA_MOVED: ::c_int = 24; +pub const KERN_MEMORY_RESTART_COPY: ::c_int = 25; +pub const KERN_INVALID_PROCESSOR_SET: ::c_int = 26; +pub const KERN_POLICY_LIMIT: ::c_int = 27; +pub const KERN_INVALID_POLICY: ::c_int = 28; +pub const KERN_INVALID_OBJECT: ::c_int = 29; +pub const KERN_ALREADY_WAITING: ::c_int = 30; +pub const KERN_DEFAULT_SET: ::c_int = 31; +pub const KERN_EXCEPTION_PROTECTED: ::c_int = 32; +pub const KERN_INVALID_LEDGER: ::c_int = 33; +pub const KERN_INVALID_MEMORY_CONTROL: ::c_int = 34; +pub const KERN_INVALID_SECURITY: ::c_int = 35; +pub const KERN_NOT_DEPRESSED: ::c_int = 36; +pub const KERN_TERMINATED: ::c_int = 37; +pub const KERN_LOCK_SET_DESTROYED: ::c_int = 38; +pub const KERN_LOCK_UNSTABLE: ::c_int = 39; +pub const KERN_LOCK_OWNED: ::c_int = 40; +pub const KERN_LOCK_OWNED_SELF: ::c_int = 41; +pub const KERN_SEMAPHORE_DESTROYED: ::c_int = 42; +pub const KERN_RPC_SERVER_TERMINATED: ::c_int = 43; +pub const KERN_RPC_TERMINATE_ORPHAN: ::c_int = 44; +pub const KERN_RPC_CONTINUE_ORPHAN: ::c_int = 45; +pub const KERN_NOT_SUPPORTED: ::c_int = 46; +pub const KERN_NODE_DOWN: ::c_int = 47; +pub const KERN_NOT_WAITING: ::c_int = 48; +pub const KERN_OPERATION_TIMED_OUT: ::c_int = 49; +pub const KERN_CODESIGN_ERROR: ::c_int = 50; +pub const KERN_POLICY_STATIC: ::c_int = 51; +pub const KERN_INSUFFICIENT_BUFFER_SIZE: ::c_int = 52; pub const KIPC_MAXSOCKBUF: ::c_int = 1; pub const KIPC_SOCKBUF_WASTE: ::c_int = 2; pub const KIPC_SOMAXCONN: ::c_int = 3; @@ -2800,6 +4273,11 @@ pub const VM_LOADAVG: ::c_int = 2; pub const VM_MACHFACTOR: ::c_int = 4; pub const VM_SWAPUSAGE: ::c_int = 5; pub const VM_MAXID: ::c_int = 6; +pub const VM_PROT_NONE: ::vm_prot_t = 0x00; +pub const VM_PROT_READ: ::vm_prot_t = 0x01; +pub const VM_PROT_WRITE: ::vm_prot_t = 0x02; +pub const VM_PROT_EXECUTE: ::vm_prot_t = 0x04; +pub const MEMORY_OBJECT_NULL: ::memory_object_t = 0; pub const HW_MACHINE: ::c_int = 1; pub const HW_MODEL: ::c_int = 2; pub const HW_NCPU: ::c_int = 3; @@ -2825,7 +4303,9 @@ pub const HW_L3CACHESIZE: ::c_int = 22; pub const HW_TB_FREQ: ::c_int = 23; pub const HW_MEMSIZE: ::c_int = 24; pub const HW_AVAILCPU: ::c_int = 25; -pub const HW_MAXID: ::c_int = 26; +pub const HW_TARGET: ::c_int = 26; +pub const HW_PRODUCT: ::c_int = 27; +pub const HW_MAXID: ::c_int = 28; pub const USER_CS_PATH: ::c_int = 1; pub const USER_BC_BASE_MAX: ::c_int = 2; pub const USER_BC_DIM_MAX: ::c_int = 3; @@ -2862,11 +4342,8 @@ pub const AI_PASSIVE: ::c_int = 0x00000001; pub const AI_CANONNAME: ::c_int = 0x00000002; pub const AI_NUMERICHOST: ::c_int = 0x00000004; pub const AI_NUMERICSERV: ::c_int = 0x00001000; -pub const AI_MASK: ::c_int = AI_PASSIVE - | AI_CANONNAME - | AI_NUMERICHOST - | AI_NUMERICSERV - | AI_ADDRCONFIG; +pub const AI_MASK: ::c_int = + AI_PASSIVE | AI_CANONNAME | AI_NUMERICHOST | AI_NUMERICSERV | AI_ADDRCONFIG; pub const AI_ALL: ::c_int = 0x00000100; pub const AI_V4MAPPED_CFG: ::c_int = 0x00000200; pub const AI_ADDRCONFIG: ::c_int = 0x00000400; @@ -2942,6 +4419,8 @@ pub const RTF_CONDEMNED: ::c_int = 0x2000000; pub const RTF_IFREF: ::c_int = 0x4000000; pub const RTF_PROXY: ::c_int = 0x8000000; pub const RTF_ROUTER: ::c_int = 0x10000000; +pub const RTF_DEAD: ::c_int = 0x20000000; +pub const RTF_GLOBAL: ::c_int = 0x40000000; pub const RTM_VERSION: ::c_int = 5; @@ -3000,8 +4479,14 @@ pub const RTAX_MAX: ::c_int = 8; pub const KERN_PROCARGS2: ::c_int = 49; pub const PROC_PIDTASKALLINFO: ::c_int = 2; +pub const PROC_PIDTBSDINFO: ::c_int = 3; pub const PROC_PIDTASKINFO: ::c_int = 4; pub const PROC_PIDTHREADINFO: ::c_int = 5; +pub const PROC_PIDVNODEPATHINFO: ::c_int = 9; +pub const PROC_PIDPATHINFO_MAXSIZE: ::c_int = 4096; +pub const PROC_CSM_ALL: ::c_uint = 0x0001; +pub const PROC_CSM_NOSMT: ::c_uint = 0x0002; +pub const PROC_CSM_TECS: ::c_uint = 0x0004; pub const MAXCOMLEN: usize = 16; pub const MAXTHREADNAMESIZE: usize = 64; @@ -3038,18 +4523,10 @@ pub const DLT_LOOP: ::c_uint = 108; pub const BPF_ALIGNMENT: ::c_int = 4; // sys/mount.h -pub const MNT_RDONLY: ::c_int = 0x00000001; -pub const MNT_SYNCHRONOUS: ::c_int = 0x00000002; -pub const MNT_NOEXEC: ::c_int = 0x00000004; -pub const MNT_NOSUID: ::c_int = 0x00000008; pub const MNT_NODEV: ::c_int = 0x00000010; pub const MNT_UNION: ::c_int = 0x00000020; -pub const MNT_ASYNC: ::c_int = 0x00000040; pub const MNT_CPROTECT: ::c_int = 0x00000080; -// NFS export related mount flags. -pub const MNT_EXPORTED: ::c_int = 0x00000100; - // MAC labeled / "quarantined" flag pub const MNT_QUARANTINE: ::c_int = 0x00000400; @@ -3070,9 +4547,7 @@ pub const MNT_NOATIME: ::c_int = 0x10000000; pub const MNT_SNAPSHOT: ::c_int = 0x40000000; // External filesystem command modifier flags. -pub const MNT_UPDATE: ::c_int = 0x00010000; pub const MNT_NOBLOCK: ::c_int = 0x00020000; -pub const MNT_RELOAD: ::c_int = 0x00040000; // sys/spawn.h: pub const POSIX_SPAWN_RESETIDS: ::c_int = 0x01; @@ -3111,6 +4586,9 @@ pub const SETALL: ::c_int = 9; // sys/shm.h pub const SHM_RDONLY: ::c_int = 0x1000; pub const SHM_RND: ::c_int = 0x2000; +#[cfg(target_arch = "aarch64")] +pub const SHMLBA: ::c_int = 16 * 1024; +#[cfg(not(target_arch = "aarch64"))] pub const SHMLBA: ::c_int = 4096; pub const SHM_R: ::c_int = IPC_R; pub const SHM_W: ::c_int = IPC_W; @@ -3181,8 +4659,268 @@ pub const TIME_OOP: ::c_int = 3; pub const TIME_WAIT: ::c_int = 4; pub const TIME_ERROR: ::c_int = 5; +// +pub const MNT_WAIT: ::c_int = 1; +pub const MNT_NOWAIT: ::c_int = 2; + +// +pub const THREAD_STANDARD_POLICY: ::c_int = 1; +pub const THREAD_STANDARD_POLICY_COUNT: ::c_int = 0; +pub const THREAD_EXTENDED_POLICY: ::c_int = 1; +pub const THREAD_TIME_CONSTRAINT_POLICY: ::c_int = 2; +pub const THREAD_PRECEDENCE_POLICY: ::c_int = 3; +pub const THREAD_AFFINITY_POLICY: ::c_int = 4; +pub const THREAD_AFFINITY_TAG_NULL: ::c_int = 0; +pub const THREAD_BACKGROUND_POLICY: ::c_int = 5; +pub const THREAD_BACKGROUND_POLICY_DARWIN_BG: ::c_int = 0x1000; +pub const THREAD_LATENCY_QOS_POLICY: ::c_int = 7; +pub const THREAD_THROUGHPUT_QOS_POLICY: ::c_int = 8; + +// +pub const TH_STATE_RUNNING: ::c_int = 1; +pub const TH_STATE_STOPPED: ::c_int = 2; +pub const TH_STATE_WAITING: ::c_int = 3; +pub const TH_STATE_UNINTERRUPTIBLE: ::c_int = 4; +pub const TH_STATE_HALTED: ::c_int = 5; +pub const TH_FLAGS_SWAPPED: ::c_int = 0x1; +pub const TH_FLAGS_IDLE: ::c_int = 0x2; +pub const TH_FLAGS_GLOBAL_FORCED_IDLE: ::c_int = 0x4; +pub const THREAD_BASIC_INFO: ::c_int = 3; +pub const THREAD_IDENTIFIER_INFO: ::c_int = 4; +pub const THREAD_EXTENDED_INFO: ::c_int = 5; + +// CommonCrypto/CommonCryptoError.h +pub const kCCSuccess: i32 = 0; +pub const kCCParamError: i32 = -4300; +pub const kCCBufferTooSmall: i32 = -4301; +pub const kCCMemoryFailure: i32 = -4302; +pub const kCCAlignmentError: i32 = -4303; +pub const kCCDecodeError: i32 = -4304; +pub const kCCUnimplemented: i32 = -4305; +pub const kCCOverflow: i32 = -4306; +pub const kCCRNGFailure: i32 = -4307; +pub const kCCUnspecifiedError: i32 = -4308; +pub const kCCCallSequenceError: i32 = -4309; +pub const kCCKeySizeError: i32 = -4310; +pub const kCCInvalidKey: i32 = -4311; + +// mach/host_info.h +pub const HOST_LOAD_INFO: i32 = 1; +pub const HOST_VM_INFO: i32 = 2; +pub const HOST_CPU_LOAD_INFO: i32 = 3; +pub const HOST_VM_INFO64: i32 = 4; +pub const HOST_EXTMOD_INFO64: i32 = 5; +pub const HOST_EXPIRED_TASK_INFO: i32 = 6; + +// mach/vm_statistics.h +pub const VM_PAGE_QUERY_PAGE_PRESENT: i32 = 0x1; +pub const VM_PAGE_QUERY_PAGE_FICTITIOUS: i32 = 0x2; +pub const VM_PAGE_QUERY_PAGE_REF: i32 = 0x4; +pub const VM_PAGE_QUERY_PAGE_DIRTY: i32 = 0x8; +pub const VM_PAGE_QUERY_PAGE_PAGED_OUT: i32 = 0x10; +pub const VM_PAGE_QUERY_PAGE_COPIED: i32 = 0x20; +pub const VM_PAGE_QUERY_PAGE_SPECULATIVE: i32 = 0x40; +pub const VM_PAGE_QUERY_PAGE_EXTERNAL: i32 = 0x80; +pub const VM_PAGE_QUERY_PAGE_CS_VALIDATED: i32 = 0x100; +pub const VM_PAGE_QUERY_PAGE_CS_TAINTED: i32 = 0x200; +pub const VM_PAGE_QUERY_PAGE_CS_NX: i32 = 0x400; + +// mach/task_info.h +pub const TASK_THREAD_TIMES_INFO: u32 = 3; +pub const HOST_CPU_LOAD_INFO_COUNT: u32 = 4; +pub const MACH_TASK_BASIC_INFO: u32 = 20; + +pub const MACH_PORT_NULL: i32 = 0; + +pub const RUSAGE_INFO_V0: ::c_int = 0; +pub const RUSAGE_INFO_V1: ::c_int = 1; +pub const RUSAGE_INFO_V2: ::c_int = 2; +pub const RUSAGE_INFO_V3: ::c_int = 3; +pub const RUSAGE_INFO_V4: ::c_int = 4; + +// copyfile.h +pub const COPYFILE_ACL: ::copyfile_flags_t = 1 << 0; +pub const COPYFILE_STAT: ::copyfile_flags_t = 1 << 1; +pub const COPYFILE_XATTR: ::copyfile_flags_t = 1 << 2; +pub const COPYFILE_DATA: ::copyfile_flags_t = 1 << 3; +pub const COPYFILE_SECURITY: ::copyfile_flags_t = COPYFILE_STAT | COPYFILE_ACL; +pub const COPYFILE_METADATA: ::copyfile_flags_t = COPYFILE_SECURITY | COPYFILE_XATTR; +pub const COPYFILE_RECURSIVE: ::copyfile_flags_t = 1 << 15; +pub const COPYFILE_CHECK: ::copyfile_flags_t = 1 << 16; +pub const COPYFILE_EXCL: ::copyfile_flags_t = 1 << 17; +pub const COPYFILE_NOFOLLOW_SRC: ::copyfile_flags_t = 1 << 18; +pub const COPYFILE_NOFOLLOW_DST: ::copyfile_flags_t = 1 << 19; +pub const COPYFILE_MOVE: ::copyfile_flags_t = 1 << 20; +pub const COPYFILE_UNLINK: ::copyfile_flags_t = 1 << 21; +pub const COPYFILE_NOFOLLOW: ::copyfile_flags_t = COPYFILE_NOFOLLOW_SRC | COPYFILE_NOFOLLOW_DST; +pub const COPYFILE_PACK: ::copyfile_flags_t = 1 << 22; +pub const COPYFILE_UNPACK: ::copyfile_flags_t = 1 << 23; +pub const COPYFILE_CLONE: ::copyfile_flags_t = 1 << 24; +pub const COPYFILE_CLONE_FORCE: ::copyfile_flags_t = 1 << 25; +pub const COPYFILE_RUN_IN_PLACE: ::copyfile_flags_t = 1 << 26; +pub const COPYFILE_DATA_SPARSE: ::copyfile_flags_t = 1 << 27; +pub const COPYFILE_PRESERVE_DST_TRACKED: ::copyfile_flags_t = 1 << 28; +pub const COPYFILE_VERBOSE: ::copyfile_flags_t = 1 << 30; +pub const COPYFILE_RECURSE_ERROR: ::c_int = 0; +pub const COPYFILE_RECURSE_FILE: ::c_int = 1; +pub const COPYFILE_RECURSE_DIR: ::c_int = 2; +pub const COPYFILE_RECURSE_DIR_CLEANUP: ::c_int = 3; +pub const COPYFILE_COPY_DATA: ::c_int = 4; +pub const COPYFILE_COPY_XATTR: ::c_int = 5; +pub const COPYFILE_START: ::c_int = 1; +pub const COPYFILE_FINISH: ::c_int = 2; +pub const COPYFILE_ERR: ::c_int = 3; +pub const COPYFILE_PROGRESS: ::c_int = 4; +pub const COPYFILE_CONTINUE: ::c_int = 0; +pub const COPYFILE_SKIP: ::c_int = 1; +pub const COPYFILE_QUIT: ::c_int = 2; + +// +pub const ATTR_BIT_MAP_COUNT: ::c_ushort = 5; +pub const FSOPT_NOFOLLOW: u32 = 0x1; +pub const FSOPT_NOFOLLOW_ANY: u32 = 0x800; +pub const FSOPT_REPORT_FULLSIZE: u32 = 0x4; +pub const FSOPT_PACK_INVAL_ATTRS: u32 = 0x8; +pub const FSOPT_ATTR_CMN_EXTENDED: u32 = 0x20; +pub const FSOPT_RETURN_REALDEV: u32 = 0x200; +pub const ATTR_CMN_NAME: attrgroup_t = 0x00000001; +pub const ATTR_CMN_DEVID: attrgroup_t = 0x00000002; +pub const ATTR_CMN_FSID: attrgroup_t = 0x00000004; +pub const ATTR_CMN_OBJTYPE: attrgroup_t = 0x00000008; +pub const ATTR_CMN_OBJTAG: attrgroup_t = 0x00000010; +pub const ATTR_CMN_OBJID: attrgroup_t = 0x00000020; +pub const ATTR_CMN_OBJPERMANENTID: attrgroup_t = 0x00000040; +pub const ATTR_CMN_PAROBJID: attrgroup_t = 0x00000080; +pub const ATTR_CMN_SCRIPT: attrgroup_t = 0x00000100; +pub const ATTR_CMN_CRTIME: attrgroup_t = 0x00000200; +pub const ATTR_CMN_MODTIME: attrgroup_t = 0x00000400; +pub const ATTR_CMN_CHGTIME: attrgroup_t = 0x00000800; +pub const ATTR_CMN_ACCTIME: attrgroup_t = 0x00001000; +pub const ATTR_CMN_BKUPTIME: attrgroup_t = 0x00002000; +pub const ATTR_CMN_FNDRINFO: attrgroup_t = 0x00004000; +pub const ATTR_CMN_OWNERID: attrgroup_t = 0x00008000; +pub const ATTR_CMN_GRPID: attrgroup_t = 0x00010000; +pub const ATTR_CMN_ACCESSMASK: attrgroup_t = 0x00020000; +pub const ATTR_CMN_FLAGS: attrgroup_t = 0x00040000; +pub const ATTR_CMN_GEN_COUNT: attrgroup_t = 0x00080000; +pub const ATTR_CMN_DOCUMENT_ID: attrgroup_t = 0x00100000; +pub const ATTR_CMN_USERACCESS: attrgroup_t = 0x00200000; +pub const ATTR_CMN_EXTENDED_SECURITY: attrgroup_t = 0x00400000; +pub const ATTR_CMN_UUID: attrgroup_t = 0x00800000; +pub const ATTR_CMN_GRPUUID: attrgroup_t = 0x01000000; +pub const ATTR_CMN_FILEID: attrgroup_t = 0x02000000; +pub const ATTR_CMN_PARENTID: attrgroup_t = 0x04000000; +pub const ATTR_CMN_FULLPATH: attrgroup_t = 0x08000000; +pub const ATTR_CMN_ADDEDTIME: attrgroup_t = 0x10000000; +pub const ATTR_CMN_DATA_PROTECT_FLAGS: attrgroup_t = 0x40000000; +pub const ATTR_CMN_RETURNED_ATTRS: attrgroup_t = 0x80000000; +pub const ATTR_VOL_FSTYPE: attrgroup_t = 0x00000001; +pub const ATTR_VOL_SIGNATURE: attrgroup_t = 0x00000002; +pub const ATTR_VOL_SIZE: attrgroup_t = 0x00000004; +pub const ATTR_VOL_SPACEFREE: attrgroup_t = 0x00000008; +pub const ATTR_VOL_SPACEAVAIL: attrgroup_t = 0x00000010; +pub const ATTR_VOL_MINALLOCATION: attrgroup_t = 0x00000020; +pub const ATTR_VOL_ALLOCATIONCLUMP: attrgroup_t = 0x00000040; +pub const ATTR_VOL_IOBLOCKSIZE: attrgroup_t = 0x00000080; +pub const ATTR_VOL_OBJCOUNT: attrgroup_t = 0x00000100; +pub const ATTR_VOL_FILECOUNT: attrgroup_t = 0x00000200; +pub const ATTR_VOL_DIRCOUNT: attrgroup_t = 0x00000400; +pub const ATTR_VOL_MAXOBJCOUNT: attrgroup_t = 0x00000800; +pub const ATTR_VOL_MOUNTPOINT: attrgroup_t = 0x00001000; +pub const ATTR_VOL_NAME: attrgroup_t = 0x00002000; +pub const ATTR_VOL_MOUNTFLAGS: attrgroup_t = 0x00004000; +pub const ATTR_VOL_MOUNTEDDEVICE: attrgroup_t = 0x00008000; +pub const ATTR_VOL_ENCODINGSUSED: attrgroup_t = 0x00010000; +pub const ATTR_VOL_CAPABILITIES: attrgroup_t = 0x00020000; +pub const ATTR_VOL_UUID: attrgroup_t = 0x00040000; +pub const ATTR_VOL_SPACEUSED: attrgroup_t = 0x00800000; +pub const ATTR_VOL_QUOTA_SIZE: attrgroup_t = 0x10000000; +pub const ATTR_VOL_RESERVED_SIZE: attrgroup_t = 0x20000000; +pub const ATTR_VOL_ATTRIBUTES: attrgroup_t = 0x40000000; +pub const ATTR_VOL_INFO: attrgroup_t = 0x80000000; +pub const ATTR_DIR_LINKCOUNT: attrgroup_t = 0x00000001; +pub const ATTR_DIR_ENTRYCOUNT: attrgroup_t = 0x00000002; +pub const ATTR_DIR_MOUNTSTATUS: attrgroup_t = 0x00000004; +pub const ATTR_DIR_ALLOCSIZE: attrgroup_t = 0x00000008; +pub const ATTR_DIR_IOBLOCKSIZE: attrgroup_t = 0x00000010; +pub const ATTR_DIR_DATALENGTH: attrgroup_t = 0x00000020; +pub const ATTR_FILE_LINKCOUNT: attrgroup_t = 0x00000001; +pub const ATTR_FILE_TOTALSIZE: attrgroup_t = 0x00000002; +pub const ATTR_FILE_ALLOCSIZE: attrgroup_t = 0x00000004; +pub const ATTR_FILE_IOBLOCKSIZE: attrgroup_t = 0x00000008; +pub const ATTR_FILE_DEVTYPE: attrgroup_t = 0x00000020; +pub const ATTR_FILE_FORKCOUNT: attrgroup_t = 0x00000080; +pub const ATTR_FILE_FORKLIST: attrgroup_t = 0x00000100; +pub const ATTR_FILE_DATALENGTH: attrgroup_t = 0x00000200; +pub const ATTR_FILE_DATAALLOCSIZE: attrgroup_t = 0x00000400; +pub const ATTR_FILE_RSRCLENGTH: attrgroup_t = 0x00001000; +pub const ATTR_FILE_RSRCALLOCSIZE: attrgroup_t = 0x00002000; +pub const ATTR_CMNEXT_RELPATH: attrgroup_t = 0x00000004; +pub const ATTR_CMNEXT_PRIVATESIZE: attrgroup_t = 0x00000008; +pub const ATTR_CMNEXT_LINKID: attrgroup_t = 0x00000010; +pub const ATTR_CMNEXT_NOFIRMLINKPATH: attrgroup_t = 0x00000020; +pub const ATTR_CMNEXT_REALDEVID: attrgroup_t = 0x00000040; +pub const ATTR_CMNEXT_REALFSID: attrgroup_t = 0x00000080; +pub const ATTR_CMNEXT_CLONEID: attrgroup_t = 0x00000100; +pub const ATTR_CMNEXT_EXT_FLAGS: attrgroup_t = 0x00000200; +pub const ATTR_CMNEXT_RECURSIVE_GENCOUNT: attrgroup_t = 0x00000400; +pub const DIR_MNTSTATUS_MNTPOINT: u32 = 0x1; +pub const VOL_CAPABILITIES_FORMAT: usize = 0; +pub const VOL_CAPABILITIES_INTERFACES: usize = 1; +pub const VOL_CAP_FMT_PERSISTENTOBJECTIDS: attrgroup_t = 0x00000001; +pub const VOL_CAP_FMT_SYMBOLICLINKS: attrgroup_t = 0x00000002; +pub const VOL_CAP_FMT_HARDLINKS: attrgroup_t = 0x00000004; +pub const VOL_CAP_FMT_JOURNAL: attrgroup_t = 0x00000008; +pub const VOL_CAP_FMT_JOURNAL_ACTIVE: attrgroup_t = 0x00000010; +pub const VOL_CAP_FMT_NO_ROOT_TIMES: attrgroup_t = 0x00000020; +pub const VOL_CAP_FMT_SPARSE_FILES: attrgroup_t = 0x00000040; +pub const VOL_CAP_FMT_ZERO_RUNS: attrgroup_t = 0x00000080; +pub const VOL_CAP_FMT_CASE_SENSITIVE: attrgroup_t = 0x00000100; +pub const VOL_CAP_FMT_CASE_PRESERVING: attrgroup_t = 0x00000200; +pub const VOL_CAP_FMT_FAST_STATFS: attrgroup_t = 0x00000400; +pub const VOL_CAP_FMT_2TB_FILESIZE: attrgroup_t = 0x00000800; +pub const VOL_CAP_FMT_OPENDENYMODES: attrgroup_t = 0x00001000; +pub const VOL_CAP_FMT_HIDDEN_FILES: attrgroup_t = 0x00002000; +pub const VOL_CAP_FMT_PATH_FROM_ID: attrgroup_t = 0x00004000; +pub const VOL_CAP_FMT_NO_VOLUME_SIZES: attrgroup_t = 0x00008000; +pub const VOL_CAP_FMT_DECMPFS_COMPRESSION: attrgroup_t = 0x00010000; +pub const VOL_CAP_FMT_64BIT_OBJECT_IDS: attrgroup_t = 0x00020000; +pub const VOL_CAP_FMT_DIR_HARDLINKS: attrgroup_t = 0x00040000; +pub const VOL_CAP_FMT_DOCUMENT_ID: attrgroup_t = 0x00080000; +pub const VOL_CAP_FMT_WRITE_GENERATION_COUNT: attrgroup_t = 0x00100000; +pub const VOL_CAP_FMT_NO_IMMUTABLE_FILES: attrgroup_t = 0x00200000; +pub const VOL_CAP_FMT_NO_PERMISSIONS: attrgroup_t = 0x00400000; +pub const VOL_CAP_FMT_SHARED_SPACE: attrgroup_t = 0x00800000; +pub const VOL_CAP_FMT_VOL_GROUPS: attrgroup_t = 0x01000000; +pub const VOL_CAP_FMT_SEALED: attrgroup_t = 0x02000000; +pub const VOL_CAP_INT_SEARCHFS: attrgroup_t = 0x00000001; +pub const VOL_CAP_INT_ATTRLIST: attrgroup_t = 0x00000002; +pub const VOL_CAP_INT_NFSEXPORT: attrgroup_t = 0x00000004; +pub const VOL_CAP_INT_READDIRATTR: attrgroup_t = 0x00000008; +pub const VOL_CAP_INT_EXCHANGEDATA: attrgroup_t = 0x00000010; +pub const VOL_CAP_INT_COPYFILE: attrgroup_t = 0x00000020; +pub const VOL_CAP_INT_ALLOCATE: attrgroup_t = 0x00000040; +pub const VOL_CAP_INT_VOL_RENAME: attrgroup_t = 0x00000080; +pub const VOL_CAP_INT_ADVLOCK: attrgroup_t = 0x00000100; +pub const VOL_CAP_INT_FLOCK: attrgroup_t = 0x00000200; +pub const VOL_CAP_INT_EXTENDED_SECURITY: attrgroup_t = 0x00000400; +pub const VOL_CAP_INT_USERACCESS: attrgroup_t = 0x00000800; +pub const VOL_CAP_INT_MANLOCK: attrgroup_t = 0x00001000; +pub const VOL_CAP_INT_NAMEDSTREAMS: attrgroup_t = 0x00002000; +pub const VOL_CAP_INT_EXTENDED_ATTR: attrgroup_t = 0x00004000; +pub const VOL_CAP_INT_CLONE: attrgroup_t = 0x00010000; +pub const VOL_CAP_INT_SNAPSHOT: attrgroup_t = 0x00020000; +pub const VOL_CAP_INT_RENAME_SWAP: attrgroup_t = 0x00040000; +pub const VOL_CAP_INT_RENAME_EXCL: attrgroup_t = 0x00080000; +pub const VOL_CAP_INT_RENAME_OPENFAIL: attrgroup_t = 0x00100000; + cfg_if! { - if #[cfg(libc_const_size_of)] { + if #[cfg(libc_const_extern_fn)] { + const fn __DARWIN_ALIGN32(p: usize) -> usize { + const __DARWIN_ALIGNBYTES32: usize = ::mem::size_of::() - 1; + p + __DARWIN_ALIGNBYTES32 & !__DARWIN_ALIGNBYTES32 + } + } else if #[cfg(libc_const_size_of)] { fn __DARWIN_ALIGN32(p: usize) -> usize { const __DARWIN_ALIGNBYTES32: usize = ::mem::size_of::() - 1; p + __DARWIN_ALIGNBYTES32 & !__DARWIN_ALIGNBYTES32 @@ -3195,6 +4933,64 @@ cfg_if! { } } +cfg_if! { + if #[cfg(libc_const_size_of)] { + pub const THREAD_EXTENDED_POLICY_COUNT: mach_msg_type_number_t = + (::mem::size_of::() / ::mem::size_of::()) + as mach_msg_type_number_t; + pub const THREAD_TIME_CONSTRAINT_POLICY_COUNT: mach_msg_type_number_t = + (::mem::size_of::() / + ::mem::size_of::()) as mach_msg_type_number_t; + pub const THREAD_PRECEDENCE_POLICY_COUNT: mach_msg_type_number_t = + (::mem::size_of::() / ::mem::size_of::()) + as mach_msg_type_number_t; + pub const THREAD_AFFINITY_POLICY_COUNT: mach_msg_type_number_t = + (::mem::size_of::() / ::mem::size_of::()) + as mach_msg_type_number_t; + pub const THREAD_BACKGROUND_POLICY_COUNT: mach_msg_type_number_t = + (::mem::size_of::() / ::mem::size_of::()) + as mach_msg_type_number_t; + pub const THREAD_LATENCY_QOS_POLICY_COUNT: mach_msg_type_number_t = + (::mem::size_of::() / ::mem::size_of::()) + as mach_msg_type_number_t; + pub const THREAD_THROUGHPUT_QOS_POLICY_COUNT: mach_msg_type_number_t = + (::mem::size_of::() / + ::mem::size_of::()) as mach_msg_type_number_t; + pub const THREAD_BASIC_INFO_COUNT: mach_msg_type_number_t = + (::mem::size_of::() / ::mem::size_of::()) + as mach_msg_type_number_t; + pub const THREAD_IDENTIFIER_INFO_COUNT: mach_msg_type_number_t = + (::mem::size_of::() / ::mem::size_of::()) + as mach_msg_type_number_t; + pub const THREAD_EXTENDED_INFO_COUNT: mach_msg_type_number_t = + (::mem::size_of::() / ::mem::size_of::()) + as mach_msg_type_number_t; + + pub const TASK_THREAD_TIMES_INFO_COUNT: u32 = + (::mem::size_of::() + / ::mem::size_of::()) as u32; + pub const MACH_TASK_BASIC_INFO_COUNT: u32 = (::mem::size_of::() + / ::mem::size_of::()) as u32; + pub const HOST_VM_INFO64_COUNT: mach_msg_type_number_t = + (::mem::size_of::() / ::mem::size_of::()) + as mach_msg_type_number_t; + } else { + pub const THREAD_EXTENDED_POLICY_COUNT: mach_msg_type_number_t = 1; + pub const THREAD_TIME_CONSTRAINT_POLICY_COUNT: mach_msg_type_number_t = 4; + pub const THREAD_PRECEDENCE_POLICY_COUNT: mach_msg_type_number_t = 1; + pub const THREAD_AFFINITY_POLICY_COUNT: mach_msg_type_number_t = 1; + pub const THREAD_BACKGROUND_POLICY_COUNT: mach_msg_type_number_t = 1; + pub const THREAD_LATENCY_QOS_POLICY_COUNT: mach_msg_type_number_t = 1; + pub const THREAD_THROUGHPUT_QOS_POLICY_COUNT: mach_msg_type_number_t = 1; + pub const THREAD_BASIC_INFO_COUNT: mach_msg_type_number_t = 10; + pub const THREAD_IDENTIFIER_INFO_COUNT: mach_msg_type_number_t = 6; + pub const THREAD_EXTENDED_INFO_COUNT: mach_msg_type_number_t = 28; + pub const TASK_THREAD_TIMES_INFO_COUNT: u32 = 4; + pub const MACH_TASK_BASIC_INFO_COUNT: u32 = 12; + pub const HOST_VM_INFO64_COUNT: mach_msg_type_number_t = 38; + } +} + f! { pub fn CMSG_NXTHDR(mhdr: *const ::msghdr, cmsg: *const ::cmsghdr) -> *mut ::cmsghdr { @@ -3204,7 +5000,7 @@ f! { let cmsg_len = (*cmsg).cmsg_len as usize; let next = cmsg as usize + __DARWIN_ALIGN32(cmsg_len as usize); let max = (*mhdr).msg_control as usize - + (*mhdr).msg_controllen as usize; + + (*mhdr).msg_controllen as usize; if next + __DARWIN_ALIGN32(::mem::size_of::<::cmsghdr>()) > max { 0 as *mut ::cmsghdr } else { @@ -3217,7 +5013,7 @@ f! { .offset(__DARWIN_ALIGN32(::mem::size_of::<::cmsghdr>()) as isize) } - pub fn CMSG_SPACE(length: ::c_uint) -> ::c_uint { + pub {const} fn CMSG_SPACE(length: ::c_uint) -> ::c_uint { (__DARWIN_ALIGN32(::mem::size_of::<::cmsghdr>()) + __DARWIN_ALIGN32(length as usize)) as ::c_uint @@ -3228,23 +5024,41 @@ f! { as ::c_uint } - pub fn WSTOPSIG(status: ::c_int) -> ::c_int { + pub {const} fn VM_MAKE_TAG(id: u8) -> u32 { + (id as u32) << 24u32 + } + + pub fn major(dev: dev_t) -> i32 { + (dev >> 24) & 0xff + } + + pub fn minor(dev: dev_t) -> i32 { + dev & 0xffffff + } + + pub fn makedev(major: i32, minor: i32) -> dev_t { + (major << 24) | minor + } +} + +safe_f! { + pub {const} fn WSTOPSIG(status: ::c_int) -> ::c_int { status >> 8 } - pub fn _WSTATUS(status: ::c_int) -> ::c_int { + pub {const} fn _WSTATUS(status: ::c_int) -> ::c_int { status & 0x7f } - pub fn WIFCONTINUED(status: ::c_int) -> bool { + pub {const} fn WIFCONTINUED(status: ::c_int) -> bool { _WSTATUS(status) == _WSTOPPED && WSTOPSIG(status) == 0x13 } - pub fn WIFSIGNALED(status: ::c_int) -> bool { + pub {const} fn WIFSIGNALED(status: ::c_int) -> bool { _WSTATUS(status) != _WSTOPPED && _WSTATUS(status) != 0 } - pub fn WIFSTOPPED(status: ::c_int) -> bool { + pub {const} fn WIFSTOPPED(status: ::c_int) -> bool { _WSTATUS(status) == _WSTOPPED && WSTOPSIG(status) != 0x13 } } @@ -3253,18 +5067,14 @@ extern "C" { pub fn setgrent(); #[doc(hidden)] #[deprecated(since = "0.2.49", note = "Deprecated in MacOSX 10.5")] - #[link_name = "daemon$1050"] + #[cfg_attr(not(target_arch = "aarch64"), link_name = "daemon$1050")] pub fn daemon(nochdir: ::c_int, noclose: ::c_int) -> ::c_int; #[doc(hidden)] #[deprecated(since = "0.2.49", note = "Deprecated in MacOSX 10.10")] pub fn sem_destroy(sem: *mut sem_t) -> ::c_int; #[doc(hidden)] #[deprecated(since = "0.2.49", note = "Deprecated in MacOSX 10.10")] - pub fn sem_init( - sem: *mut sem_t, - pshared: ::c_int, - value: ::c_uint, - ) -> ::c_int; + pub fn sem_init(sem: *mut sem_t, pshared: ::c_int, value: ::c_uint) -> ::c_int; pub fn aio_read(aiocbp: *mut aiocb) -> ::c_int; pub fn aio_write(aiocbp: *mut aiocb) -> ::c_int; pub fn aio_fsync(op: ::c_int, aiocbp: *mut aiocb) -> ::c_int; @@ -3284,6 +5094,11 @@ extern "C" { pub fn fchflags(fd: ::c_int, flags: ::c_uint) -> ::c_int; pub fn clock_getres(clk_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; pub fn clock_gettime(clk_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; + #[cfg_attr( + all(target_os = "macos", target_arch = "x86"), + link_name = "confstr$UNIX2003" + )] + pub fn confstr(name: ::c_int, buf: *mut ::c_char, len: ::size_t) -> ::size_t; pub fn lio_listio( mode: ::c_int, aiocb_list: *const *mut aiocb, @@ -3313,11 +5128,7 @@ extern "C" { sevlen: ::socklen_t, flags: ::c_int, ) -> ::c_int; - pub fn mincore( - addr: *const ::c_void, - len: ::size_t, - vec: *mut ::c_char, - ) -> ::c_int; + pub fn mincore(addr: *const ::c_void, len: ::size_t, vec: *mut ::c_char) -> ::c_int; pub fn sysctlnametomib( name: *const ::c_char, mibp: *mut ::c_int, @@ -3327,44 +5138,23 @@ extern "C" { all(target_os = "macos", target_arch = "x86"), link_name = "mprotect$UNIX2003" )] - pub fn mprotect( - addr: *mut ::c_void, - len: ::size_t, - prot: ::c_int, - ) -> ::c_int; + pub fn mprotect(addr: *mut ::c_void, len: ::size_t, prot: ::c_int) -> ::c_int; pub fn semget(key: key_t, nsems: ::c_int, semflg: ::c_int) -> ::c_int; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "semctl$UNIX2003" )] - pub fn semctl( - semid: ::c_int, - semnum: ::c_int, - cmd: ::c_int, - ... - ) -> ::c_int; - pub fn semop( - semid: ::c_int, - sops: *mut sembuf, - nsops: ::size_t, - ) -> ::c_int; + pub fn semctl(semid: ::c_int, semnum: ::c_int, cmd: ::c_int, ...) -> ::c_int; + pub fn semop(semid: ::c_int, sops: *mut sembuf, nsops: ::size_t) -> ::c_int; pub fn shm_open(name: *const ::c_char, oflag: ::c_int, ...) -> ::c_int; pub fn ftok(pathname: *const c_char, proj_id: ::c_int) -> key_t; - pub fn shmat( - shmid: ::c_int, - shmaddr: *const ::c_void, - shmflg: ::c_int, - ) -> *mut ::c_void; + pub fn shmat(shmid: ::c_int, shmaddr: *const ::c_void, shmflg: ::c_int) -> *mut ::c_void; pub fn shmdt(shmaddr: *const ::c_void) -> ::c_int; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "shmctl$UNIX2003" )] - pub fn shmctl( - shmid: ::c_int, - cmd: ::c_int, - buf: *mut ::shmid_ds, - ) -> ::c_int; + pub fn shmctl(shmid: ::c_int, cmd: ::c_int, buf: *mut ::shmid_ds) -> ::c_int; pub fn shmget(key: key_t, size: ::size_t, shmflg: ::c_int) -> ::c_int; pub fn sysctl( name: *mut ::c_int, @@ -3381,23 +5171,26 @@ extern "C" { newp: *mut ::c_void, newlen: ::size_t, ) -> ::c_int; - #[deprecated(since = "0.2.55", note = "Use the mach crate")] + #[deprecated(since = "0.2.55", note = "Use the `mach2` crate instead")] pub fn mach_absolute_time() -> u64; - #[deprecated(since = "0.2.55", note = "Use the mach crate")] + #[deprecated(since = "0.2.55", note = "Use the `mach2` crate instead")] #[allow(deprecated)] pub fn mach_timebase_info(info: *mut ::mach_timebase_info) -> ::c_int; + pub fn mach_host_self() -> mach_port_t; + pub fn mach_thread_self() -> mach_port_t; pub fn pthread_setname_np(name: *const ::c_char) -> ::c_int; - pub fn pthread_getname_np( - thread: ::pthread_t, - name: *mut ::c_char, - len: ::size_t, + pub fn pthread_getname_np(thread: ::pthread_t, name: *mut ::c_char, len: ::size_t) -> ::c_int; + pub fn pthread_mach_thread_np(thread: ::pthread_t) -> ::mach_port_t; + pub fn pthread_from_mach_thread_np(port: ::mach_port_t) -> ::pthread_t; + pub fn pthread_create_from_mach_thread( + thread: *mut ::pthread_t, + attr: *const ::pthread_attr_t, + f: extern "C" fn(*mut ::c_void) -> *mut ::c_void, + value: *mut ::c_void, ) -> ::c_int; pub fn pthread_get_stackaddr_np(thread: ::pthread_t) -> *mut ::c_void; pub fn pthread_get_stacksize_np(thread: ::pthread_t) -> ::size_t; - pub fn pthread_condattr_setpshared( - attr: *mut pthread_condattr_t, - pshared: ::c_int, - ) -> ::c_int; + pub fn pthread_condattr_setpshared(attr: *mut pthread_condattr_t, pshared: ::c_int) -> ::c_int; pub fn pthread_condattr_getpshared( attr: *const pthread_condattr_t, pshared: *mut ::c_int, @@ -3414,15 +5207,135 @@ extern "C" { attr: *const pthread_rwlockattr_t, val: *mut ::c_int, ) -> ::c_int; - pub fn pthread_rwlockattr_setpshared( - attr: *mut pthread_rwlockattr_t, - val: ::c_int, + pub fn pthread_rwlockattr_setpshared(attr: *mut pthread_rwlockattr_t, val: ::c_int) -> ::c_int; + pub fn pthread_threadid_np(thread: ::pthread_t, thread_id: *mut u64) -> ::c_int; + pub fn pthread_attr_set_qos_class_np( + attr: *mut pthread_attr_t, + class: qos_class_t, + priority: ::c_int, + ) -> ::c_int; + pub fn pthread_attr_get_qos_class_np( + attr: *mut pthread_attr_t, + class: *mut qos_class_t, + priority: *mut ::c_int, + ) -> ::c_int; + pub fn pthread_set_qos_class_self_np(class: qos_class_t, priority: ::c_int) -> ::c_int; + pub fn pthread_get_qos_class_np( + thread: ::pthread_t, + class: *mut qos_class_t, + priority: *mut ::c_int, + ) -> ::c_int; + pub fn pthread_attr_getschedparam( + attr: *const ::pthread_attr_t, + param: *mut sched_param, + ) -> ::c_int; + pub fn pthread_attr_setschedparam( + attr: *mut ::pthread_attr_t, + param: *const sched_param, + ) -> ::c_int; + pub fn pthread_getschedparam( + thread: ::pthread_t, + policy: *mut ::c_int, + param: *mut sched_param, + ) -> ::c_int; + pub fn pthread_setschedparam( + thread: ::pthread_t, + policy: ::c_int, + param: *const sched_param, + ) -> ::c_int; + + // Available from Big Sur + pub fn pthread_introspection_hook_install( + hook: ::pthread_introspection_hook_t, + ) -> ::pthread_introspection_hook_t; + pub fn pthread_introspection_setspecific_np( + thread: ::pthread_t, + key: ::pthread_key_t, + value: *const ::c_void, ) -> ::c_int; + pub fn pthread_introspection_getspecific_np( + thread: ::pthread_t, + key: ::pthread_key_t, + ) -> *mut ::c_void; + pub fn pthread_jit_write_protect_np(enabled: ::c_int); + pub fn pthread_jit_write_protect_supported_np() -> ::c_int; + // An array of pthread_jit_write_with_callback_np must declare + // the list of callbacks e.g. + // #[link_section = "__DATA_CONST,__pth_jit_func"] + // static callbacks: [libc::pthread_jit_write_callback_t; 2] = [native_jit_write_cb, + // std::mem::transmute::(std::ptr::null())]; + // (a handy PTHREAD_JIT_WRITE_CALLBACK_NP macro for other languages). + pub fn pthread_jit_write_with_callback_np( + callback: ::pthread_jit_write_callback_t, + ctx: *mut ::c_void, + ) -> ::c_int; + pub fn pthread_jit_write_freeze_callbacks_np(); + pub fn pthread_cpu_number_np(cpu_number_out: *mut ::size_t) -> ::c_int; + + pub fn os_unfair_lock_lock(lock: os_unfair_lock_t); + pub fn os_unfair_lock_trylock(lock: os_unfair_lock_t) -> bool; + pub fn os_unfair_lock_unlock(lock: os_unfair_lock_t); + pub fn os_unfair_lock_assert_owner(lock: os_unfair_lock_t); + pub fn os_unfair_lock_assert_not_owner(lock: os_unfair_lock_t); + + pub fn os_log_create(subsystem: *const ::c_char, category: *const ::c_char) -> ::os_log_t; + pub fn os_log_type_enabled(oslog: ::os_log_t, tpe: ::os_log_type_t) -> bool; + pub fn os_signpost_id_make_with_pointer( + log: ::os_log_t, + ptr: *const ::c_void, + ) -> ::os_signpost_id_t; + pub fn os_signpost_id_generate(log: ::os_log_t) -> ::os_signpost_id_t; + pub fn os_signpost_enabled(log: ::os_log_t) -> bool; + + pub fn thread_policy_set( + thread: thread_t, + flavor: thread_policy_flavor_t, + policy_info: thread_policy_t, + count: mach_msg_type_number_t, + ) -> kern_return_t; + pub fn thread_policy_get( + thread: thread_t, + flavor: thread_policy_flavor_t, + policy_info: thread_policy_t, + count: *mut mach_msg_type_number_t, + get_default: *mut boolean_t, + ) -> kern_return_t; + pub fn thread_info( + target_act: thread_inspect_t, + flavor: thread_flavor_t, + thread_info_out: thread_info_t, + thread_info_outCnt: *mut mach_msg_type_number_t, + ) -> kern_return_t; + #[cfg_attr(doc, doc(alias = "__errno_location"))] + #[cfg_attr(doc, doc(alias = "errno"))] pub fn __error() -> *mut ::c_int; pub fn backtrace(buf: *mut *mut ::c_void, sz: ::c_int) -> ::c_int; - #[cfg_attr(target_os = "macos", link_name = "statfs$INODE64")] + pub fn backtrace_symbols(addrs: *const *mut ::c_void, sz: ::c_int) -> *mut *mut ::c_char; + pub fn backtrace_symbols_fd(addrs: *const *mut ::c_void, sz: ::c_int, fd: ::c_int); + pub fn backtrace_from_fp( + startfp: *mut ::c_void, + array: *mut *mut ::c_void, + size: ::c_int, + ) -> ::c_int; + pub fn backtrace_image_offsets( + array: *const *mut ::c_void, + image_offsets: *mut image_offset, + size: ::c_int, + ); + pub fn backtrace_async( + array: *mut *mut ::c_void, + length: ::size_t, + task_id: *mut u32, + ) -> ::size_t; + #[cfg_attr( + all(target_os = "macos", not(target_arch = "aarch64")), + link_name = "statfs$INODE64" + )] pub fn statfs(path: *const ::c_char, buf: *mut statfs) -> ::c_int; - #[cfg_attr(target_os = "macos", link_name = "fstatfs$INODE64")] + #[cfg_attr( + all(target_os = "macos", not(target_arch = "aarch64")), + link_name = "fstatfs$INODE64" + )] pub fn fstatfs(fd: ::c_int, buf: *mut statfs) -> ::c_int; pub fn kevent( kq: ::c_int, @@ -3447,12 +5360,13 @@ extern "C" { flags: ::c_int, data: *mut ::c_void, ) -> ::c_int; - pub fn ptrace( - request: ::c_int, - pid: ::pid_t, - addr: *mut ::c_char, - data: ::c_int, + pub fn fmount( + src: *const ::c_char, + fd: ::c_int, + flags: ::c_int, + data: *mut ::c_void, ) -> ::c_int; + pub fn ptrace(request: ::c_int, pid: ::pid_t, addr: *mut ::c_char, data: ::c_int) -> ::c_int; pub fn quotactl( special: *const ::c_char, cmd: ::c_int, @@ -3492,17 +5406,16 @@ extern "C" { pub fn duplocale(base: ::locale_t) -> ::locale_t; pub fn freelocale(loc: ::locale_t) -> ::c_int; pub fn localeconv_l(loc: ::locale_t) -> *mut lconv; - pub fn newlocale( - mask: ::c_int, - locale: *const ::c_char, - base: ::locale_t, - ) -> ::locale_t; + pub fn newlocale(mask: ::c_int, locale: *const ::c_char, base: ::locale_t) -> ::locale_t; pub fn uselocale(loc: ::locale_t) -> ::locale_t; pub fn querylocale(mask: ::c_int, loc: ::locale_t) -> *const ::c_char; pub fn getpriority(which: ::c_int, who: ::id_t) -> ::c_int; pub fn setpriority(which: ::c_int, who: ::id_t, prio: ::c_int) -> ::c_int; pub fn getdomainname(name: *mut ::c_char, len: ::c_int) -> ::c_int; pub fn setdomainname(name: *const ::c_char, len: ::c_int) -> ::c_int; + pub fn preadv(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int, offset: ::off_t) -> ::ssize_t; + pub fn pwritev(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int, offset: ::off_t) + -> ::ssize_t; pub fn getxattr( path: *const ::c_char, name: *const ::c_char, @@ -3547,16 +5460,8 @@ extern "C" { size: ::size_t, flags: ::c_int, ) -> ::ssize_t; - pub fn removexattr( - path: *const ::c_char, - name: *const ::c_char, - flags: ::c_int, - ) -> ::c_int; - pub fn renamex_np( - from: *const ::c_char, - to: *const ::c_char, - flags: ::c_uint, - ) -> ::c_int; + pub fn removexattr(path: *const ::c_char, name: *const ::c_char, flags: ::c_int) -> ::c_int; + pub fn renamex_np(from: *const ::c_char, to: *const ::c_char, flags: ::c_uint) -> ::c_int; pub fn renameatx_np( fromfd: ::c_int, from: *const ::c_char, @@ -3564,11 +5469,7 @@ extern "C" { to: *const ::c_char, flags: ::c_uint, ) -> ::c_int; - pub fn fremovexattr( - filedes: ::c_int, - name: *const ::c_char, - flags: ::c_int, - ) -> ::c_int; + pub fn fremovexattr(filedes: ::c_int, name: *const ::c_char, flags: ::c_int) -> ::c_int; pub fn getgrouplist( name: *const ::c_char, @@ -3582,26 +5483,19 @@ extern "C" { all(target_os = "macos", target_arch = "x86"), link_name = "waitid$UNIX2003" )] - pub fn waitid( - idtype: idtype_t, - id: id_t, - infop: *mut ::siginfo_t, - options: ::c_int, - ) -> ::c_int; + pub fn waitid(idtype: idtype_t, id: id_t, infop: *mut ::siginfo_t, options: ::c_int) + -> ::c_int; pub fn brk(addr: *const ::c_void) -> *mut ::c_void; pub fn sbrk(increment: ::c_int) -> *mut ::c_void; - pub fn settimeofday( - tv: *const ::timeval, - tz: *const ::timezone, - ) -> ::c_int; - #[deprecated(since = "0.2.55", note = "Use the mach crate")] + pub fn settimeofday(tv: *const ::timeval, tz: *const ::timezone) -> ::c_int; + #[deprecated(since = "0.2.55", note = "Use the `mach2` crate instead")] pub fn _dyld_image_count() -> u32; - #[deprecated(since = "0.2.55", note = "Use the mach crate")] + #[deprecated(since = "0.2.55", note = "Use the `mach2` crate instead")] #[allow(deprecated)] pub fn _dyld_get_image_header(image_index: u32) -> *const mach_header; - #[deprecated(since = "0.2.55", note = "Use the mach crate")] + #[deprecated(since = "0.2.55", note = "Use the `mach2` crate instead")] pub fn _dyld_get_image_vmaddr_slide(image_index: u32) -> ::intptr_t; - #[deprecated(since = "0.2.55", note = "Use the mach crate")] + #[deprecated(since = "0.2.55", note = "Use the `mach2` crate instead")] pub fn _dyld_get_image_name(image_index: u32) -> *const ::c_char; pub fn posix_spawn( @@ -3642,25 +5536,29 @@ extern "C" { attr: *const posix_spawnattr_t, flags: *mut ::c_short, ) -> ::c_int; - pub fn posix_spawnattr_setflags( - attr: *mut posix_spawnattr_t, - flags: ::c_short, - ) -> ::c_int; + pub fn posix_spawnattr_setflags(attr: *mut posix_spawnattr_t, flags: ::c_short) -> ::c_int; pub fn posix_spawnattr_getpgroup( attr: *const posix_spawnattr_t, flags: *mut ::pid_t, ) -> ::c_int; - pub fn posix_spawnattr_setpgroup( + pub fn posix_spawnattr_setpgroup(attr: *mut posix_spawnattr_t, flags: ::pid_t) -> ::c_int; + pub fn posix_spawnattr_setarchpref_np( attr: *mut posix_spawnattr_t, - flags: ::pid_t, - ) -> ::c_int; - - pub fn posix_spawn_file_actions_init( - actions: *mut posix_spawn_file_actions_t, + count: ::size_t, + pref: *mut ::cpu_type_t, + subpref: *mut ::cpu_subtype_t, + ocount: *mut ::size_t, ) -> ::c_int; - pub fn posix_spawn_file_actions_destroy( - actions: *mut posix_spawn_file_actions_t, + pub fn posix_spawnattr_getarchpref_np( + attr: *const posix_spawnattr_t, + count: ::size_t, + pref: *mut ::cpu_type_t, + subpref: *mut ::cpu_subtype_t, + ocount: *mut ::size_t, ) -> ::c_int; + + pub fn posix_spawn_file_actions_init(actions: *mut posix_spawn_file_actions_t) -> ::c_int; + pub fn posix_spawn_file_actions_destroy(actions: *mut posix_spawn_file_actions_t) -> ::c_int; pub fn posix_spawn_file_actions_addopen( actions: *mut posix_spawn_file_actions_t, fd: ::c_int, @@ -3689,21 +5587,344 @@ extern "C" { len: *mut ::size_t, connid: *mut sae_connid_t, ) -> ::c_int; - pub fn disconnectx( - socket: ::c_int, - associd: sae_associd_t, - connid: sae_connid_t, - ) -> ::c_int; + pub fn disconnectx(socket: ::c_int, associd: sae_associd_t, connid: sae_connid_t) -> ::c_int; pub fn ntp_adjtime(buf: *mut timex) -> ::c_int; pub fn ntp_gettime(buf: *mut ntptimeval) -> ::c_int; + + #[cfg_attr( + all(target_os = "macos", not(target_arch = "aarch64")), + link_name = "getmntinfo$INODE64" + )] + pub fn getmntinfo(mntbufp: *mut *mut statfs, flags: ::c_int) -> ::c_int; + #[cfg_attr( + all(target_os = "macos", not(target_arch = "aarch64")), + link_name = "getfsstat$INODE64" + )] + pub fn getfsstat(mntbufp: *mut statfs, bufsize: ::c_int, flags: ::c_int) -> ::c_int; + + // Copy-on-write functions. + // According to the man page `flags` is an `int` but in the header + // this is a `uint32_t`. + pub fn clonefile(src: *const ::c_char, dst: *const ::c_char, flags: u32) -> ::c_int; + pub fn clonefileat( + src_dirfd: ::c_int, + src: *const ::c_char, + dst_dirfd: ::c_int, + dst: *const ::c_char, + flags: u32, + ) -> ::c_int; + pub fn fclonefileat( + srcfd: ::c_int, + dst_dirfd: ::c_int, + dst: *const ::c_char, + flags: u32, + ) -> ::c_int; + + pub fn copyfile( + from: *const ::c_char, + to: *const ::c_char, + state: copyfile_state_t, + flags: copyfile_flags_t, + ) -> ::c_int; + pub fn fcopyfile( + from: ::c_int, + to: ::c_int, + state: copyfile_state_t, + flags: copyfile_flags_t, + ) -> ::c_int; + + // Added in macOS 10.13 + // ISO/IEC 9899:2011 ("ISO C11") K.3.7.4.1 + pub fn memset_s(s: *mut ::c_void, smax: ::size_t, c: ::c_int, n: ::size_t) -> ::c_int; + // Added in macOS 10.5 + pub fn memset_pattern4(b: *mut ::c_void, pattern4: *const ::c_void, len: ::size_t); + pub fn memset_pattern8(b: *mut ::c_void, pattern8: *const ::c_void, len: ::size_t); + pub fn memset_pattern16(b: *mut ::c_void, pattern16: *const ::c_void, len: ::size_t); + + // Inherited from BSD but available from Big Sur only + pub fn strtonum( + __numstr: *const ::c_char, + __minval: ::c_longlong, + __maxval: ::c_longlong, + errstrp: *mut *const ::c_char, + ) -> ::c_longlong; + + pub fn mstats() -> mstats; + pub fn malloc_printf(format: *const ::c_char, ...); + pub fn malloc_zone_check(zone: *mut ::malloc_zone_t) -> ::boolean_t; + pub fn malloc_zone_print(zone: *mut ::malloc_zone_t, verbose: ::boolean_t); + pub fn malloc_zone_statistics(zone: *mut ::malloc_zone_t, stats: *mut malloc_statistics_t); + pub fn malloc_zone_log(zone: *mut ::malloc_zone_t, address: *mut ::c_void); + pub fn malloc_zone_print_ptr_info(ptr: *mut ::c_void); + pub fn malloc_default_zone() -> *mut ::malloc_zone_t; + pub fn malloc_zone_from_ptr(ptr: *const ::c_void) -> *mut ::malloc_zone_t; + pub fn malloc_zone_malloc(zone: *mut ::malloc_zone_t, size: ::size_t) -> *mut ::c_void; + pub fn malloc_zone_valloc(zone: *mut ::malloc_zone_t, size: ::size_t) -> *mut ::c_void; + pub fn malloc_zone_calloc( + zone: *mut ::malloc_zone_t, + num_items: ::size_t, + size: ::size_t, + ) -> *mut ::c_void; + pub fn malloc_zone_realloc( + zone: *mut ::malloc_zone_t, + ptr: *mut ::c_void, + size: ::size_t, + ) -> *mut ::c_void; + pub fn malloc_zone_free(zone: *mut ::malloc_zone_t, ptr: *mut ::c_void); + + pub fn proc_listpids( + t: u32, + typeinfo: u32, + buffer: *mut ::c_void, + buffersize: ::c_int, + ) -> ::c_int; + pub fn proc_listallpids(buffer: *mut ::c_void, buffersize: ::c_int) -> ::c_int; + pub fn proc_listpgrppids( + pgrpid: ::pid_t, + buffer: *mut ::c_void, + buffersize: ::c_int, + ) -> ::c_int; + pub fn proc_listchildpids(ppid: ::pid_t, buffer: *mut ::c_void, buffersize: ::c_int) + -> ::c_int; + pub fn proc_pidinfo( + pid: ::c_int, + flavor: ::c_int, + arg: u64, + buffer: *mut ::c_void, + buffersize: ::c_int, + ) -> ::c_int; + pub fn proc_pidfdinfo( + pid: ::c_int, + fd: ::c_int, + flavor: ::c_int, + buffer: *mut ::c_void, + buffersize: ::c_int, + ) -> ::c_int; + pub fn proc_pidfileportinfo( + pid: ::c_int, + fileport: u32, + flavor: ::c_int, + buffer: *mut ::c_void, + buffersize: ::c_int, + ) -> ::c_int; + pub fn proc_pidpath(pid: ::c_int, buffer: *mut ::c_void, buffersize: u32) -> ::c_int; + pub fn proc_name(pid: ::c_int, buffer: *mut ::c_void, buffersize: u32) -> ::c_int; + pub fn proc_regionfilename( + pid: ::c_int, + address: u64, + buffer: *mut ::c_void, + buffersize: u32, + ) -> ::c_int; + pub fn proc_kmsgbuf(buffer: *mut ::c_void, buffersize: u32) -> ::c_int; + pub fn proc_libversion(major: *mut ::c_int, mintor: *mut ::c_int) -> ::c_int; + pub fn proc_pid_rusage(pid: ::c_int, flavor: ::c_int, buffer: *mut rusage_info_t) -> ::c_int; + + // Available from Big Sur + pub fn proc_set_no_smt() -> ::c_int; + pub fn proc_setthread_no_smt() -> ::c_int; + pub fn proc_set_csm(flags: u32) -> ::c_int; + pub fn proc_setthread_csm(flags: u32) -> ::c_int; + /// # Notes + /// + /// `id` is of type [`uuid_t`]. + pub fn gethostuuid(id: *mut u8, timeout: *const ::timespec) -> ::c_int; + + pub fn gethostid() -> ::c_long; + pub fn sethostid(hostid: ::c_long); + + pub fn CCRandomGenerateBytes(bytes: *mut ::c_void, size: ::size_t) -> ::CCRNGStatus; + + pub fn _NSGetExecutablePath(buf: *mut ::c_char, bufsize: *mut u32) -> ::c_int; + pub fn _NSGetEnviron() -> *mut *mut *mut ::c_char; + + pub fn mach_vm_map( + target_task: ::vm_map_t, + address: *mut ::mach_vm_address_t, + size: ::mach_vm_size_t, + mask: ::mach_vm_offset_t, + flags: ::c_int, + object: ::mem_entry_name_port_t, + offset: ::memory_object_offset_t, + copy: ::boolean_t, + cur_protection: ::vm_prot_t, + max_protection: ::vm_prot_t, + inheritance: ::vm_inherit_t, + ) -> ::kern_return_t; + + pub fn vm_deallocate( + target_task: vm_map_t, + address: vm_address_t, + size: vm_size_t, + ) -> ::kern_return_t; + + pub fn host_statistics64( + host_priv: host_t, + flavor: host_flavor_t, + host_info64_out: host_info64_t, + host_info64_outCnt: *mut mach_msg_type_number_t, + ) -> ::kern_return_t; + pub fn host_processor_info( + host: host_t, + flavor: processor_flavor_t, + out_processor_count: *mut natural_t, + out_processor_info: *mut processor_info_array_t, + out_processor_infoCnt: *mut mach_msg_type_number_t, + ) -> ::kern_return_t; + + pub static mut mach_task_self_: ::mach_port_t; + pub fn task_for_pid( + host: ::mach_port_t, + pid: ::pid_t, + task: *mut ::mach_port_t, + ) -> ::kern_return_t; + pub fn task_info( + host: ::mach_port_t, + flavor: task_flavor_t, + task_info_out: task_info_t, + task_info_count: *mut mach_msg_type_number_t, + ) -> ::kern_return_t; + pub fn task_create( + target_task: ::task_t, + ledgers: ::ledger_array_t, + ledgersCnt: ::mach_msg_type_number_t, + inherit_memory: ::boolean_t, + child_task: *mut ::task_t, + ) -> ::kern_return_t; + pub fn task_terminate(target_task: ::task_t) -> ::kern_return_t; + pub fn task_threads( + target_task: ::task_inspect_t, + act_list: *mut ::thread_act_array_t, + act_listCnt: *mut ::mach_msg_type_number_t, + ) -> ::kern_return_t; + pub fn host_statistics( + host_priv: host_t, + flavor: host_flavor_t, + host_info_out: host_info_t, + host_info_outCnt: *mut mach_msg_type_number_t, + ) -> ::kern_return_t; + + // sysdir.h + pub fn sysdir_start_search_path_enumeration( + dir: sysdir_search_path_directory_t, + domainMask: sysdir_search_path_domain_mask_t, + ) -> ::sysdir_search_path_enumeration_state; + pub fn sysdir_get_next_search_path_enumeration( + state: ::sysdir_search_path_enumeration_state, + path: *mut ::c_char, + ) -> ::sysdir_search_path_enumeration_state; + + pub static vm_page_size: vm_size_t; + + pub fn getattrlist( + path: *const ::c_char, + attrList: *mut ::c_void, + attrBuf: *mut ::c_void, + attrBufSize: ::size_t, + options: u32, + ) -> ::c_int; + pub fn fgetattrlist( + fd: ::c_int, + attrList: *mut ::c_void, + attrBuf: *mut ::c_void, + attrBufSize: ::size_t, + options: u32, + ) -> ::c_int; + pub fn getattrlistat( + fd: ::c_int, + path: *const ::c_char, + attrList: *mut ::c_void, + attrBuf: *mut ::c_void, + attrBufSize: ::size_t, + options: ::c_ulong, + ) -> ::c_int; + pub fn setattrlist( + path: *const ::c_char, + attrList: *mut ::c_void, + attrBuf: *mut ::c_void, + attrBufSize: ::size_t, + options: u32, + ) -> ::c_int; + pub fn fsetattrlist( + fd: ::c_int, + attrList: *mut ::c_void, + attrBuf: *mut ::c_void, + attrBufSize: ::size_t, + options: u32, + ) -> ::c_int; + pub fn setattrlistat( + dir_fd: ::c_int, + path: *const ::c_char, + attrList: *mut ::c_void, + attrBuf: *mut ::c_void, + attrBufSize: ::size_t, + options: u32, + ) -> ::c_int; + pub fn getattrlistbulk( + dirfd: ::c_int, + attrList: *mut ::c_void, + attrBuf: *mut ::c_void, + attrBufSize: ::size_t, + options: u64, + ) -> ::c_int; + + pub fn malloc_size(ptr: *const ::c_void) -> ::size_t; + pub fn malloc_good_size(size: ::size_t) -> ::size_t; + + pub fn dirname(path: *mut ::c_char) -> *mut ::c_char; + pub fn basename(path: *mut ::c_char) -> *mut ::c_char; +} + +pub unsafe fn mach_task_self() -> ::mach_port_t { + mach_task_self_ +} + +cfg_if! { + if #[cfg(target_os = "macos")] { + extern "C" { + pub fn clock_settime(clock_id: ::clockid_t, tp: *const ::timespec) -> ::c_int; + } + } +} +cfg_if! { + if #[cfg(any(target_os = "macos", target_os = "ios"))] { + extern "C" { + pub fn memmem( + haystack: *const ::c_void, + haystacklen: ::size_t, + needle: *const ::c_void, + needlelen: ::size_t, + ) -> *mut ::c_void; + pub fn task_set_info(target_task: ::task_t, + flavor: ::task_flavor_t, + task_info_in: ::task_info_t, + task_info_inCnt: ::mach_msg_type_number_t + ) -> ::kern_return_t; + } + } +} + +// These require a dependency on `libiconv`, and including this when built as +// part of `std` means every Rust program gets it. Ideally we would have a link +// modifier to only include these if they are used, but we do not. +#[cfg_attr(not(feature = "rustc-dep-of-std"), link(name = "iconv"))] +extern "C" { + pub fn iconv_open(tocode: *const ::c_char, fromcode: *const ::c_char) -> iconv_t; + pub fn iconv( + cd: iconv_t, + inbuf: *mut *mut ::c_char, + inbytesleft: *mut ::size_t, + outbuf: *mut *mut ::c_char, + outbytesleft: *mut ::size_t, + ) -> ::size_t; + pub fn iconv_close(cd: iconv_t) -> ::c_int; } cfg_if! { - if #[cfg(any(target_arch = "arm", target_arch = "x86"))] { + if #[cfg(target_pointer_width = "32")] { mod b32; pub use self::b32::*; - } else if #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] { + } else if #[cfg(target_pointer_width = "64")] { mod b64; pub use self::b64::*; } else { diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/dragonfly/errno.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/dragonfly/errno.rs index e9ad63b86..5fe6bb89c 100644 --- a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/dragonfly/errno.rs +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/dragonfly/errno.rs @@ -1,6 +1,7 @@ // DragonFlyBSD's __error function is declared with "static inline", so it must // be implemented in the libc crate, as a pointer to a static thread_local. f! { + #[deprecated(since = "0.2.77", note = "Use `__errno_location()` instead")] pub fn __error() -> *mut ::c_int { &mut errno } diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/dragonfly/mod.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/dragonfly/mod.rs index 014636312..70fe6e2ed 100644 --- a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/dragonfly/mod.rs +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/dragonfly/mod.rs @@ -1,5 +1,6 @@ pub type dev_t = u32; pub type c_char = i8; +pub type wchar_t = i32; pub type clock_t = u64; pub type ino_t = u64; pub type lwpid_t = i32; @@ -16,10 +17,34 @@ pub type uuid_t = ::uuid; pub type fsblkcnt_t = u64; pub type fsfilcnt_t = u64; +pub type idtype_t = ::c_uint; +pub type shmatt_t = ::c_uint; pub type mqd_t = ::c_int; pub type sem_t = *mut sem; +pub type cpuset_t = cpumask_t; +pub type cpu_set_t = cpumask_t; + +pub type register_t = ::c_long; +pub type umtx_t = ::c_int; +pub type pthread_barrierattr_t = ::c_int; +pub type pthread_barrier_t = ::uintptr_t; +pub type pthread_spinlock_t = ::uintptr_t; + +pub type segsz_t = usize; + +pub type vm_prot_t = u8; +pub type vm_maptype_t = u8; +pub type vm_inherit_t = i8; +pub type vm_subsys_t = ::c_int; +pub type vm_eflags_t = ::c_uint; + +pub type vm_map_t = *mut __c_anonymous_vm_map; +pub type vm_map_entry_t = *mut vm_map_entry; + +pub type pmap = __c_anonymous_pmap; + #[cfg_attr(feature = "extra_traits", derive(Debug))] pub enum sem {} impl ::Copy for sem {} @@ -29,6 +54,24 @@ impl ::Clone for sem { } } +e! { + #[repr(u32)] + pub enum lwpstat { + LSRUN = 1, + LSSTOP = 2, + LSSLEEP = 3, + } + + #[repr(u32)] + pub enum procstat { + SIDL = 1, + SACTIVE = 2, + SSTOP = 3, + SZOMB = 4, + SCORE = 5, + } +} + s! { pub struct kevent { pub ident: ::uintptr_t, @@ -111,11 +154,11 @@ s! { pub st_ctime_nsec: ::c_long, pub st_size: ::off_t, pub st_blocks: i64, - pub st_blksize: u32, + pub __old_st_blksize: u32, pub st_flags: u32, pub st_gen: u32, pub st_lspare: i32, - pub st_qspare1: i64, + pub st_blksize: i64, pub st_qspare2: i64, } @@ -169,11 +212,205 @@ s! { pub sdl_route: [::c_ushort; 16], } + pub struct xucred { + pub cr_version: ::c_uint, + pub cr_uid: ::uid_t, + pub cr_ngroups: ::c_short, + pub cr_groups: [::gid_t; 16], + __cr_unused1: *mut ::c_void, + } + pub struct stack_t { - pub ss_sp: *mut ::c_char, + pub ss_sp: *mut ::c_void, pub ss_size: ::size_t, pub ss_flags: ::c_int, } + + pub struct cpumask_t { + ary: [u64; 4], + } + + pub struct shmid_ds { + pub shm_perm: ::ipc_perm, + pub shm_segsz: ::size_t, + pub shm_lpid: ::pid_t, + pub shm_cpid: ::pid_t, + pub shm_nattch: ::shmatt_t, + pub shm_atime: ::time_t, + pub shm_dtime: ::time_t, + pub shm_ctime: ::time_t, + shm_internal: *mut ::c_void, + } + + pub struct kinfo_file { + pub f_size: ::size_t, + pub f_pid: ::pid_t, + pub f_uid: ::uid_t, + pub f_fd: ::c_int, + pub f_file: *mut ::c_void, + pub f_type: ::c_short, + pub f_count: ::c_int, + pub f_msgcount: ::c_int, + pub f_offset: ::off_t, + pub f_data: *mut ::c_void, + pub f_flag: ::c_uint, + } + + pub struct kinfo_cputime { + pub cp_user: u64, + pub cp_nice: u64, + pub cp_sys: u64, + pub cp_intr: u64, + pub cp_idel: u64, + cp_unused01: u64, + cp_unused02: u64, + pub cp_sample_pc: u64, + pub cp_sample_sp: u64, + pub cp_msg: [::c_char; 32], + } + + pub struct kinfo_lwp { + pub kl_pid: ::pid_t, + pub kl_tid: ::lwpid_t, + pub kl_flags: ::c_int, + pub kl_stat: ::lwpstat, + pub kl_lock: ::c_int, + pub kl_tdflags: ::c_int, + pub kl_mpcount: ::c_int, + pub kl_prio: ::c_int, + pub kl_tdprio: ::c_int, + pub kl_rtprio: ::rtprio, + pub kl_uticks: u64, + pub kl_sticks: u64, + pub kl_iticks: u64, + pub kl_cpticks: u64, + pub kl_pctcpu: ::c_uint, + pub kl_slptime: ::c_uint, + pub kl_origcpu: ::c_int, + pub kl_estcpu: ::c_int, + pub kl_cpuid: ::c_int, + pub kl_ru: ::rusage, + pub kl_siglist: ::sigset_t, + pub kl_sigmask: ::sigset_t, + pub kl_wchan: ::uintptr_t, + pub kl_wmesg: [::c_char; 9], + pub kl_comm: [::c_char; MAXCOMLEN+1], + } + + pub struct kinfo_proc { + pub kp_paddr: ::uintptr_t, + pub kp_flags: ::c_int, + pub kp_stat: ::procstat, + pub kp_lock: ::c_int, + pub kp_acflag: ::c_int, + pub kp_traceflag: ::c_int, + pub kp_fd: ::uintptr_t, + pub kp_siglist: ::sigset_t, + pub kp_sigignore: ::sigset_t, + pub kp_sigcatch: ::sigset_t, + pub kp_sigflag: ::c_int, + pub kp_start: ::timeval, + pub kp_comm: [::c_char; MAXCOMLEN+1], + pub kp_uid: ::uid_t, + pub kp_ngroups: ::c_short, + pub kp_groups: [::gid_t; NGROUPS], + pub kp_ruid: ::uid_t, + pub kp_svuid: ::uid_t, + pub kp_rgid: ::gid_t, + pub kp_svgid: ::gid_t, + pub kp_pid: ::pid_t, + pub kp_ppid: ::pid_t, + pub kp_pgid: ::pid_t, + pub kp_jobc: ::c_int, + pub kp_sid: ::pid_t, + pub kp_login: [::c_char; 40], // MAXNAMELEN rounded up to the nearest sizeof(long) + pub kp_tdev: ::dev_t, + pub kp_tpgid: ::pid_t, + pub kp_tsid: ::pid_t, + pub kp_exitstat: ::c_ushort, + pub kp_nthreads: ::c_int, + pub kp_nice: ::c_int, + pub kp_swtime: ::c_uint, + pub kp_vm_map_size: ::size_t, + pub kp_vm_rssize: ::segsz_t, + pub kp_vm_swrss: ::segsz_t, + pub kp_vm_tsize: ::segsz_t, + pub kp_vm_dsize: ::segsz_t, + pub kp_vm_ssize: ::segsz_t, + pub kp_vm_prssize: ::c_uint, + pub kp_jailid: ::c_int, + pub kp_ru: ::rusage, + pub kp_cru: ::rusage, + pub kp_auxflags: ::c_int, + pub kp_lwp: ::kinfo_lwp, + pub kp_ktaddr: ::uintptr_t, + kp_spare: [::c_int; 2], + } + + pub struct __c_anonymous_vm_map { + _priv: [::uintptr_t; 36], + } + + pub struct vm_map_entry { + _priv: [::uintptr_t; 15], + pub eflags: ::vm_eflags_t, + pub maptype: ::vm_maptype_t, + pub protection: ::vm_prot_t, + pub max_protection: ::vm_prot_t, + pub inheritance: ::vm_inherit_t, + pub wired_count: ::c_int, + pub id: ::vm_subsys_t, + } + + pub struct __c_anonymous_pmap { + _priv1: [::uintptr_t; 32], + _priv2: [::uintptr_t; 32], + _priv3: [::uintptr_t; 32], + _priv4: [::uintptr_t; 32], + _priv5: [::uintptr_t; 8], + } + + pub struct vmspace { + vm_map: __c_anonymous_vm_map, + vm_pmap: __c_anonymous_pmap, + pub vm_flags: ::c_int, + pub vm_shm: *mut ::c_char, + pub vm_rssize: ::segsz_t, + pub vm_swrss: ::segsz_t, + pub vm_tsize: ::segsz_t, + pub vm_dsize: ::segsz_t, + pub vm_ssize: ::segsz_t, + pub vm_taddr: *mut ::c_char, + pub vm_daddr: *mut ::c_char, + pub vm_maxsaddr: *mut ::c_char, + pub vm_minsaddr: *mut ::c_char, + _unused1: ::c_int, + _unused2: ::c_int, + pub vm_pagesupply: ::c_int, + pub vm_holdcnt: ::c_uint, + pub vm_refcnt: ::c_uint, + } + + pub struct cpuctl_msr_args_t { + pub msr: ::c_int, + pub data: u64, + } + + pub struct cpuctl_cpuid_args_t { + pub level: ::c_int, + pub data: [u32; 4], + } + + pub struct cpuctl_cpuid_count_args_t { + pub level: ::c_int, + pub level_type: ::c_int, + pub data: [u32; 4], + } + + pub struct cpuctl_update_args_t { + pub data: *mut ::c_void, + pub size: ::size_t, + } } s_no_extra_traits! { @@ -194,6 +431,13 @@ s_no_extra_traits! { pub ut_unused2: [u8; 16], } + pub struct lastlogx { + pub ll_tv: ::timeval, + pub ll_line: [::c_char; _UTX_LINESIZE], + pub ll_host: [::c_char; _UTX_HOSTSIZE], + pub ll_ss: ::sockaddr_storage, + } + pub struct dirent { pub d_fileno: ::ino_t, pub d_namlen: u16, @@ -204,6 +448,7 @@ s_no_extra_traits! { } pub struct statfs { + __spare2: ::c_long, pub f_bsize: ::c_long, pub f_iosize: ::c_long, pub f_blocks: ::c_long, @@ -213,15 +458,18 @@ s_no_extra_traits! { pub f_ffree: ::c_long, pub f_fsid: ::fsid_t, pub f_owner: ::uid_t, - pub f_type: i32, - pub f_flags: i32, + pub f_type: ::c_int, + pub f_flags: ::c_int, pub f_syncwrites: ::c_long, pub f_asyncwrites: ::c_long, pub f_fstypename: [::c_char; 16], - pub f_mntonname: [::c_char; 90], + pub f_mntonname: [::c_char; 80], pub f_syncreads: ::c_long, pub f_asyncreads: ::c_long, - pub f_mntfromname: [::c_char; 90], + __spares1: ::c_short, + pub f_mntfromname: [::c_char; 80], + __spares2: ::c_short, + __spare: [::c_long; 2], } pub struct sigevent { @@ -237,6 +485,50 @@ s_no_extra_traits! { __unused3: *mut ::c_void //actually a function pointer } + pub struct mcontext_t { + pub mc_onstack: register_t, + pub mc_rdi: register_t, + pub mc_rsi: register_t, + pub mc_rdx: register_t, + pub mc_rcx: register_t, + pub mc_r8: register_t, + pub mc_r9: register_t, + pub mc_rax: register_t, + pub mc_rbx: register_t, + pub mc_rbp: register_t, + pub mc_r10: register_t, + pub mc_r11: register_t, + pub mc_r12: register_t, + pub mc_r13: register_t, + pub mc_r14: register_t, + pub mc_r15: register_t, + pub mc_xflags: register_t, + pub mc_trapno: register_t, + pub mc_addr: register_t, + pub mc_flags: register_t, + pub mc_err: register_t, + pub mc_rip: register_t, + pub mc_cs: register_t, + pub mc_rflags: register_t, + pub mc_rsp: register_t, + pub mc_ss: register_t, + pub mc_len: ::c_uint, + pub mc_fpformat: ::c_uint, + pub mc_ownedfp: ::c_uint, + __reserved: ::c_uint, + __unused: [::c_uint; 8], + pub mc_fpregs: [[::c_uint; 8]; 32], + } + + pub struct ucontext_t { + pub uc_sigmask: ::sigset_t, + pub uc_mcontext: mcontext_t, + pub uc_link: *mut ucontext_t, + pub uc_stack: stack_t, + pub uc_cofunc: ::Option, + pub uc_arg: *mut ::c_void, + __pad: [::c_int; 4], + } } cfg_if! { @@ -296,6 +588,33 @@ cfg_if! { self.ut_unused2.hash(state); } } + impl PartialEq for lastlogx { + fn eq(&self, other: &lastlogx) -> bool { + self.ll_tv == other.ll_tv + && self.ll_line == other.ll_line + && self.ll_host == other.ll_host + && self.ll_ss == other.ll_ss + } + } + impl Eq for lastlogx {} + impl ::fmt::Debug for lastlogx { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("lastlogx") + .field("ll_tv", &self.ll_tv) + .field("ll_line", &self.ll_line) + .field("ll_host", &self.ll_host) + .field("ll_ss", &self.ll_ss) + .finish() + } + } + impl ::hash::Hash for lastlogx { + fn hash(&self, state: &mut H) { + self.ll_tv.hash(state); + self.ll_line.hash(state); + self.ll_host.hash(state); + self.ll_ss.hash(state); + } + } impl PartialEq for dirent { fn eq(&self, other: &dirent) -> bool { @@ -436,12 +755,157 @@ cfg_if! { self.sigev_value.hash(state); } } + impl PartialEq for mcontext_t { + fn eq(&self, other: &mcontext_t) -> bool { + self.mc_onstack == other.mc_onstack && + self.mc_rdi == other.mc_rdi && + self.mc_rsi == other.mc_rsi && + self.mc_rdx == other.mc_rdx && + self.mc_rcx == other.mc_rcx && + self.mc_r8 == other.mc_r8 && + self.mc_r9 == other.mc_r9 && + self.mc_rax == other.mc_rax && + self.mc_rbx == other.mc_rbx && + self.mc_rbp == other.mc_rbp && + self.mc_r10 == other.mc_r10 && + self.mc_r11 == other.mc_r11 && + self.mc_r12 == other.mc_r12 && + self.mc_r13 == other.mc_r13 && + self.mc_r14 == other.mc_r14 && + self.mc_r15 == other.mc_r15 && + self.mc_xflags == other.mc_xflags && + self.mc_trapno == other.mc_trapno && + self.mc_addr == other.mc_addr && + self.mc_flags == other.mc_flags && + self.mc_err == other.mc_err && + self.mc_rip == other.mc_rip && + self.mc_cs == other.mc_cs && + self.mc_rflags == other.mc_rflags && + self.mc_rsp == other.mc_rsp && + self.mc_ss == other.mc_ss && + self.mc_len == other.mc_len && + self.mc_fpformat == other.mc_fpformat && + self.mc_ownedfp == other.mc_ownedfp && + self.mc_fpregs.iter().zip(other.mc_fpregs.iter()). + all(|(a, b)| a == b) + } + } + impl Eq for mcontext_t {} + impl ::fmt::Debug for mcontext_t { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("mcontext_t") + .field("mc_onstack", &self.mc_onstack) + .field("mc_rdi", &self.mc_rdi) + .field("mc_rsi", &self.mc_rsi) + .field("mc_rdx", &self.mc_rdx) + .field("mc_rcx", &self.mc_rcx) + .field("mc_r8", &self.mc_r8) + .field("mc_r9", &self.mc_r9) + .field("mc_rax", &self.mc_rax) + .field("mc_rbx", &self.mc_rbx) + .field("mc_rbp", &self.mc_rbp) + .field("mc_r10", &self.mc_r10) + .field("mc_r11", &self.mc_r11) + .field("mc_r12", &self.mc_r12) + .field("mc_r13", &self.mc_r13) + .field("mc_r14", &self.mc_r14) + .field("mc_r15", &self.mc_r15) + .field("mc_xflags", &self.mc_xflags) + .field("mc_trapno", &self.mc_trapno) + .field("mc_addr", &self.mc_addr) + .field("mc_flags", &self.mc_flags) + .field("mc_err", &self.mc_err) + .field("mc_rip", &self.mc_rip) + .field("mc_cs", &self.mc_cs) + .field("mc_rflags", &self.mc_rflags) + .field("mc_rsp", &self.mc_rsp) + .field("mc_ss", &self.mc_ss) + .field("mc_len", &self.mc_len) + .field("mc_fpformat", &self.mc_fpformat) + .field("mc_ownedfp", &self.mc_ownedfp) + .field("mc_fpregs", &self.mc_fpregs) + .finish() + } + } + impl ::hash::Hash for mcontext_t { + fn hash(&self, state: &mut H) { + self.mc_onstack.hash(state); + self.mc_rdi.hash(state); + self.mc_rsi.hash(state); + self.mc_rdx.hash(state); + self.mc_rcx.hash(state); + self.mc_r8.hash(state); + self.mc_r9.hash(state); + self.mc_rax.hash(state); + self.mc_rbx.hash(state); + self.mc_rbp.hash(state); + self.mc_r10.hash(state); + self.mc_r11.hash(state); + self.mc_r10.hash(state); + self.mc_r11.hash(state); + self.mc_r12.hash(state); + self.mc_r13.hash(state); + self.mc_r14.hash(state); + self.mc_r15.hash(state); + self.mc_xflags.hash(state); + self.mc_trapno.hash(state); + self.mc_addr.hash(state); + self.mc_flags.hash(state); + self.mc_err.hash(state); + self.mc_rip.hash(state); + self.mc_cs.hash(state); + self.mc_rflags.hash(state); + self.mc_rsp.hash(state); + self.mc_ss.hash(state); + self.mc_len.hash(state); + self.mc_fpformat.hash(state); + self.mc_ownedfp.hash(state); + self.mc_fpregs.hash(state); + } + } + impl PartialEq for ucontext_t { + fn eq(&self, other: &ucontext_t) -> bool { + self.uc_sigmask == other.uc_sigmask + && self.uc_mcontext == other.uc_mcontext + && self.uc_link == other.uc_link + && self.uc_stack == other.uc_stack + && self.uc_cofunc == other.uc_cofunc + && self.uc_arg == other.uc_arg + } + } + impl Eq for ucontext_t {} + impl ::fmt::Debug for ucontext_t { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("ucontext_t") + .field("uc_sigmask", &self.uc_sigmask) + .field("uc_mcontext", &self.uc_mcontext) + .field("uc_link", &self.uc_link) + .field("uc_stack", &self.uc_stack) + .field("uc_cofunc", &self.uc_cofunc) + .field("uc_arg", &self.uc_arg) + .finish() + } + } + impl ::hash::Hash for ucontext_t { + fn hash(&self, state: &mut H) { + self.uc_sigmask.hash(state); + self.uc_mcontext.hash(state); + self.uc_link.hash(state); + self.uc_stack.hash(state); + self.uc_cofunc.hash(state); + self.uc_arg.hash(state); + } + } } } pub const RAND_MAX: ::c_int = 0x7fff_ffff; pub const PTHREAD_STACK_MIN: ::size_t = 16384; pub const SIGSTKSZ: ::size_t = 40960; +pub const SIGCKPT: ::c_int = 33; +pub const SIGCKPTEXIT: ::c_int = 34; +pub const CKPT_FREEZE: ::c_int = 0x1; +pub const CKPT_THAW: ::c_int = 0x2; pub const MADV_INVAL: ::c_int = 10; pub const MADV_SETMAP: ::c_int = 11; pub const O_CLOEXEC: ::c_int = 0x00020000; @@ -449,7 +913,10 @@ pub const O_DIRECTORY: ::c_int = 0x08000000; pub const F_GETLK: ::c_int = 7; pub const F_SETLK: ::c_int = 8; pub const F_SETLKW: ::c_int = 9; +pub const F_GETPATH: ::c_int = 19; pub const ENOMEDIUM: ::c_int = 93; +pub const ENOTRECOVERABLE: ::c_int = 94; +pub const EOWNERDEAD: ::c_int = 95; pub const EASYNC: ::c_int = 99; pub const ELAST: ::c_int = 99; pub const RLIMIT_POSIXLOCKS: ::c_int = 11; @@ -459,21 +926,6 @@ pub const RLIM_NLIMITS: ::rlim_t = 12; pub const Q_GETQUOTA: ::c_int = 0x300; pub const Q_SETQUOTA: ::c_int = 0x400; -pub const CLOCK_REALTIME: ::clockid_t = 0; -pub const CLOCK_VIRTUAL: ::clockid_t = 1; -pub const CLOCK_PROF: ::clockid_t = 2; -pub const CLOCK_MONOTONIC: ::clockid_t = 4; -pub const CLOCK_UPTIME: ::clockid_t = 5; -pub const CLOCK_UPTIME_PRECISE: ::clockid_t = 7; -pub const CLOCK_UPTIME_FAST: ::clockid_t = 8; -pub const CLOCK_REALTIME_PRECISE: ::clockid_t = 9; -pub const CLOCK_REALTIME_FAST: ::clockid_t = 10; -pub const CLOCK_MONOTONIC_PRECISE: ::clockid_t = 11; -pub const CLOCK_MONOTONIC_FAST: ::clockid_t = 12; -pub const CLOCK_SECOND: ::clockid_t = 13; -pub const CLOCK_THREAD_CPUTIME_ID: ::clockid_t = 14; -pub const CLOCK_PROCESS_CPUTIME_ID: ::clockid_t = 15; - pub const CTL_UNSPEC: ::c_int = 0; pub const CTL_KERN: ::c_int = 1; pub const CTL_VM: ::c_int = 2; @@ -606,6 +1058,16 @@ pub const CTL_P1003_1B_SIGQUEUE_MAX: ::c_int = 24; pub const CTL_P1003_1B_TIMER_MAX: ::c_int = 25; pub const CTL_P1003_1B_MAXID: ::c_int = 26; +pub const CPUCTL_RSMSR: ::c_int = 0xc0106301; +pub const CPUCTL_WRMSR: ::c_int = 0xc0106302; +pub const CPUCTL_CPUID: ::c_int = 0xc0106303; +pub const CPUCTL_UPDATE: ::c_int = 0xc0106304; +pub const CPUCTL_MSRSBIT: ::c_int = 0xc0106305; +pub const CPUCTL_MSRCBIT: ::c_int = 0xc0106306; +pub const CPUCTL_CPUID_COUNT: ::c_int = 0xc0106307; + +pub const CPU_SETSIZE: ::size_t = ::mem::size_of::<::cpumask_t>() * 8; + pub const EVFILT_READ: i16 = -1; pub const EVFILT_WRITE: i16 = -2; pub const EVFILT_AIO: i16 = -3; @@ -631,6 +1093,8 @@ pub const EV_ERROR: u16 = 0x4000; pub const EV_EOF: u16 = 0x8000; pub const EV_SYSFLAGS: u16 = 0xf000; +pub const FIODNAME: ::c_ulong = 0x80106678; + pub const NOTE_TRIGGER: u32 = 0x01000000; pub const NOTE_FFNOP: u32 = 0x00000000; pub const NOTE_FFAND: u32 = 0x40000000; @@ -658,9 +1122,16 @@ pub const NOTE_CHILD: u32 = 0x00000004; pub const SO_SNDSPACE: ::c_int = 0x100a; pub const SO_CPUHINT: ::c_int = 0x1030; +pub const SO_PASSCRED: ::c_int = 0x4000; pub const PT_FIRSTMACH: ::c_int = 32; +pub const PROC_REAP_ACQUIRE: ::c_int = 0x0001; +pub const PROC_REAP_RELEASE: ::c_int = 0x0002; +pub const PROC_REAP_STATUS: ::c_int = 0x0003; +pub const PROC_PDEATHSIG_CTL: ::c_int = 0x0004; +pub const PROC_PDEATHSIG_STATUS: ::c_int = 0x0005; + // https://github.com/DragonFlyBSD/DragonFlyBSD/blob/master/sys/net/if.h#L101 pub const IFF_UP: ::c_int = 0x1; // interface is up pub const IFF_BROADCAST: ::c_int = 0x2; // broadcast address valid @@ -938,6 +1409,16 @@ pub const MSG_FBLOCKING: ::c_int = 0x00010000; pub const MSG_FNONBLOCKING: ::c_int = 0x00020000; pub const MSG_FMASK: ::c_int = 0xFFFF0000; +// sys/mount.h +pub const MNT_NODEV: ::c_int = 0x00000010; +pub const MNT_AUTOMOUNTED: ::c_int = 0x00000020; +pub const MNT_TRIM: ::c_int = 0x01000000; +pub const MNT_LOCAL: ::c_int = 0x00001000; +pub const MNT_QUOTA: ::c_int = 0x00002000; +pub const MNT_ROOTFS: ::c_int = 0x00004000; +pub const MNT_USER: ::c_int = 0x00008000; +pub const MNT_IGNORE: ::c_int = 0x00800000; + // utmpx entry types pub const EMPTY: ::c_short = 0; pub const RUN_LVL: ::c_short = 1; @@ -955,6 +1436,10 @@ pub const DOWNTIME: ::c_short = 11; pub const UTX_DB_UTMPX: ::c_uint = 0; pub const UTX_DB_WTMPX: ::c_uint = 1; pub const UTX_DB_LASTLOG: ::c_uint = 2; +pub const _UTX_LINESIZE: usize = 32; +pub const _UTX_USERSIZE: usize = 32; +pub const _UTX_IDSIZE: usize = 4; +pub const _UTX_HOSTSIZE: usize = 256; pub const LC_COLLATE_MASK: ::c_int = 1 << 0; pub const LC_CTYPE_MASK: ::c_int = 1 << 1; @@ -969,11 +1454,11 @@ pub const LC_ALL_MASK: ::c_int = LC_COLLATE_MASK | LC_NUMERIC_MASK | LC_TIME_MASK; -pub const TIOCSIG: ::c_uint = 0x2000745f; +pub const TIOCSIG: ::c_ulong = 0x2000745f; pub const BTUARTDISC: ::c_int = 0x7; -pub const TIOCDCDTIMESTAMP: ::c_uint = 0x40107458; -pub const TIOCISPTMASTER: ::c_uint = 0x20007455; -pub const TIOCMODG: ::c_uint = 0x40047403; +pub const TIOCDCDTIMESTAMP: ::c_ulong = 0x40107458; +pub const TIOCISPTMASTER: ::c_ulong = 0x20007455; +pub const TIOCMODG: ::c_ulong = 0x40047403; pub const TIOCMODS: ::c_ulong = 0x80047404; pub const TIOCREMOTE: ::c_ulong = 0x80047469; @@ -996,8 +1481,17 @@ pub const _SC_V7_LPBIG_OFFBIG: ::c_int = 125; pub const _SC_THREAD_ROBUST_PRIO_INHERIT: ::c_int = 126; pub const _SC_THREAD_ROBUST_PRIO_PROTECT: ::c_int = 127; -pub const WCONTINUED: ::c_int = 4; -pub const WSTOPPED: ::c_int = 0o177; +pub const WCONTINUED: ::c_int = 0x4; +pub const WSTOPPED: ::c_int = 0x2; +pub const WNOWAIT: ::c_int = 0x8; +pub const WEXITED: ::c_int = 0x10; +pub const WTRAPPED: ::c_int = 0x20; + +// Similar to FreeBSD, only the standardized ones are exposed. +// There are more. +pub const P_PID: idtype_t = 0; +pub const P_PGID: idtype_t = 2; +pub const P_ALL: idtype_t = 7; // Values for struct rtprio (type_ field) pub const RTP_PRIO_REALTIME: ::c_ushort = 0; @@ -1017,8 +1511,17 @@ pub const SF_XLINK: ::c_ulong = 0x01000000; pub const UTIME_OMIT: c_long = -2; pub const UTIME_NOW: c_long = -1; -fn _CMSG_ALIGN(n: usize) -> usize { - (n + 3) & !3 +pub const MINCORE_SUPER: ::c_int = 0x20; + +// kinfo_proc constants +pub const MAXCOMLEN: usize = 16; +pub const MAXLOGNAME: usize = 33; +pub const NGROUPS: usize = 16; + +const_fn! { + {const} fn _CMSG_ALIGN(n: usize) -> usize { + (n + (::mem::size_of::<::c_long>() - 1)) & !(::mem::size_of::<::c_long>() - 1) + } } f! { @@ -1047,31 +1550,71 @@ f! { } } - pub fn CMSG_SPACE(length: ::c_uint) -> ::c_uint { + pub {const} fn CMSG_SPACE(length: ::c_uint) -> ::c_uint { (_CMSG_ALIGN(::mem::size_of::<::cmsghdr>()) + _CMSG_ALIGN(length as usize)) as ::c_uint } + + pub fn CPU_ZERO(cpuset: &mut cpu_set_t) -> () { + for slot in cpuset.ary.iter_mut() { + *slot = 0; + } + } + + pub fn CPU_SET(cpu: usize, cpuset: &mut cpu_set_t) -> () { + let (idx, offset) = ((cpu >> 6) & 3, cpu & 63); + cpuset.ary[idx] |= 1 << offset; + () + } + + pub fn CPU_CLR(cpu: usize, cpuset: &mut cpu_set_t) -> () { + let (idx, offset) = ((cpu >> 6) & 3, cpu & 63); + cpuset.ary[idx] &= !(1 << offset); + () + } + + pub fn CPU_ISSET(cpu: usize, cpuset: &cpu_set_t) -> bool { + let (idx, offset) = ((cpu >> 6) & 3, cpu & 63); + 0 != cpuset.ary[idx] & (1 << offset) + } +} + +safe_f! { + pub {const} fn WIFSIGNALED(status: ::c_int) -> bool { + (status & 0o177) != 0o177 && (status & 0o177) != 0 + } + + pub {const} fn makedev(major: ::c_uint, minor: ::c_uint) -> ::dev_t { + let major = major as ::dev_t; + let minor = minor as ::dev_t; + let mut dev = 0; + dev |= major << 8; + dev |= minor; + dev + } } extern "C" { + pub fn __errno_location() -> *mut ::c_int; pub fn setgrent(); - pub fn mprotect( - addr: *mut ::c_void, - len: ::size_t, - prot: ::c_int, - ) -> ::c_int; - pub fn clock_getres(clk_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; - pub fn clock_gettime(clk_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; - pub fn clock_settime( - clk_id: ::clockid_t, - tp: *const ::timespec, - ) -> ::c_int; + pub fn mprotect(addr: *mut ::c_void, len: ::size_t, prot: ::c_int) -> ::c_int; pub fn setutxdb(_type: ::c_uint, file: *mut ::c_char) -> ::c_int; - pub fn aio_waitcomplete( - iocbp: *mut *mut aiocb, - timeout: *mut ::timespec, + pub fn aio_waitcomplete(iocbp: *mut *mut aiocb, timeout: *mut ::timespec) -> ::c_int; + + pub fn devname_r( + dev: ::dev_t, + mode: ::mode_t, + buf: *mut ::c_char, + len: ::size_t, + ) -> *mut ::c_char; + + pub fn waitid( + idtype: idtype_t, + id: ::id_t, + infop: *mut ::siginfo_t, + options: ::c_int, ) -> ::c_int; pub fn freelocale(loc: ::locale_t); @@ -1092,6 +1635,75 @@ extern "C" { needle: *const ::c_void, needlelen: ::size_t, ) -> *mut ::c_void; + pub fn pthread_spin_init(lock: *mut pthread_spinlock_t, pshared: ::c_int) -> ::c_int; + pub fn pthread_spin_destroy(lock: *mut pthread_spinlock_t) -> ::c_int; + pub fn pthread_spin_lock(lock: *mut pthread_spinlock_t) -> ::c_int; + pub fn pthread_spin_trylock(lock: *mut pthread_spinlock_t) -> ::c_int; + pub fn pthread_spin_unlock(lock: *mut pthread_spinlock_t) -> ::c_int; + + pub fn sched_getaffinity(pid: ::pid_t, cpusetsize: ::size_t, mask: *mut cpu_set_t) -> ::c_int; + pub fn sched_setaffinity(pid: ::pid_t, cpusetsize: ::size_t, mask: *const cpu_set_t) + -> ::c_int; + pub fn sched_getcpu() -> ::c_int; + pub fn setproctitle(fmt: *const ::c_char, ...); + + pub fn shmget(key: ::key_t, size: ::size_t, shmflg: ::c_int) -> ::c_int; + pub fn shmat(shmid: ::c_int, shmaddr: *const ::c_void, shmflg: ::c_int) -> *mut ::c_void; + pub fn shmdt(shmaddr: *const ::c_void) -> ::c_int; + pub fn shmctl(shmid: ::c_int, cmd: ::c_int, buf: *mut ::shmid_ds) -> ::c_int; + pub fn procctl(idtype: ::idtype_t, id: ::id_t, cmd: ::c_int, data: *mut ::c_void) -> ::c_int; + + pub fn updwtmpx(file: *const ::c_char, ut: *const utmpx) -> ::c_int; + pub fn getlastlogx(fname: *const ::c_char, uid: ::uid_t, ll: *mut lastlogx) -> *mut lastlogx; + pub fn updlastlogx(fname: *const ::c_char, uid: ::uid_t, ll: *mut lastlogx) -> ::c_int; + pub fn getutxuser(name: *const ::c_char) -> utmpx; + pub fn utmpxname(file: *const ::c_char) -> ::c_int; + + pub fn sys_checkpoint(tpe: ::c_int, fd: ::c_int, pid: ::pid_t, retval: ::c_int) -> ::c_int; + + pub fn umtx_sleep(ptr: *const ::c_int, value: ::c_int, timeout: ::c_int) -> ::c_int; + pub fn umtx_wakeup(ptr: *const ::c_int, count: ::c_int) -> ::c_int; + + pub fn dirname(path: *mut ::c_char) -> *mut ::c_char; + pub fn basename(path: *mut ::c_char) -> *mut ::c_char; +} + +#[link(name = "rt")] +extern "C" { + pub fn aio_cancel(fd: ::c_int, aiocbp: *mut aiocb) -> ::c_int; + pub fn aio_error(aiocbp: *const aiocb) -> ::c_int; + pub fn aio_fsync(op: ::c_int, aiocbp: *mut aiocb) -> ::c_int; + pub fn aio_read(aiocbp: *mut aiocb) -> ::c_int; + pub fn aio_return(aiocbp: *mut aiocb) -> ::ssize_t; + pub fn aio_suspend( + aiocb_list: *const *const aiocb, + nitems: ::c_int, + timeout: *const ::timespec, + ) -> ::c_int; + pub fn aio_write(aiocbp: *mut aiocb) -> ::c_int; + pub fn lio_listio( + mode: ::c_int, + aiocb_list: *const *mut aiocb, + nitems: ::c_int, + sevp: *mut sigevent, + ) -> ::c_int; + + pub fn reallocf(ptr: *mut ::c_void, size: ::size_t) -> *mut ::c_void; + pub fn freezero(ptr: *mut ::c_void, size: ::size_t); +} + +#[link(name = "kvm")] +extern "C" { + pub fn kvm_vm_map_entry_first( + kvm: *mut ::kvm_t, + map: vm_map_t, + entry: vm_map_entry_t, + ) -> vm_map_entry_t; + pub fn kvm_vm_map_entry_next( + kvm: *mut ::kvm_t, + map: vm_map_entry_t, + entry: vm_map_entry_t, + ) -> vm_map_entry_t; } cfg_if! { diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/aarch64.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/aarch64.rs index 22fd2b84f..e8be8815c 100644 --- a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/aarch64.rs +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/aarch64.rs @@ -1,32 +1,35 @@ pub type c_char = u8; pub type c_long = i64; pub type c_ulong = u64; +pub type wchar_t = u32; pub type time_t = i64; pub type suseconds_t = i64; +pub type register_t = i64; -s! { - pub struct stat { - pub st_dev: ::dev_t, - pub st_ino: ::ino_t, - pub st_mode: ::mode_t, - pub st_nlink: ::nlink_t, - pub st_uid: ::uid_t, - pub st_gid: ::gid_t, - pub st_rdev: ::dev_t, - pub st_atime: ::time_t, - pub st_atime_nsec: ::c_long, - pub st_mtime: ::time_t, - pub st_mtime_nsec: ::c_long, - pub st_ctime: ::time_t, - pub st_ctime_nsec: ::c_long, - pub st_size: ::off_t, - pub st_blocks: ::blkcnt_t, - pub st_blksize: ::blksize_t, - pub st_flags: ::fflags_t, - pub st_gen: u32, - pub st_lspare: i32, - pub st_birthtime: ::time_t, - pub st_birthtime_nsec: ::c_long, +s_no_extra_traits! { + pub struct gpregs { + pub gp_x: [::register_t; 30], + pub gp_lr: ::register_t, + pub gp_sp: ::register_t, + pub gp_elr: ::register_t, + pub gp_spsr: u32, + pub gp_pad: ::c_int, + } + + pub struct fpregs { + pub fp_q: u128, + pub fp_sr: u32, + pub fp_cr: u32, + pub fp_flags: ::c_int, + pub fp_pad: ::c_int, + } + + pub struct mcontext_t { + pub mc_gpregs: gpregs, + pub mc_fpregs: fpregs, + pub mc_flags: ::c_int, + pub mc_pad: ::c_int, + pub mc_spare: [u64; 8], } } @@ -41,4 +44,103 @@ cfg_if! { } } +cfg_if! { + if #[cfg(feature = "extra_traits")] { + impl PartialEq for gpregs { + fn eq(&self, other: &gpregs) -> bool { + self.gp_x.iter().zip(other.gp_x.iter()).all(|(a, b)| a == b) && + self.gp_lr == other.gp_lr && + self.gp_sp == other.gp_sp && + self.gp_elr == other.gp_elr && + self.gp_spsr == other.gp_spsr && + self.gp_pad == other.gp_pad + } + } + impl Eq for gpregs {} + impl ::fmt::Debug for gpregs { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("gpregs") + .field("gp_x", &self.gp_x) + .field("gp_lr", &self.gp_lr) + .field("gp_sp", &self.gp_sp) + .field("gp_elr", &self.gp_elr) + .field("gp_spsr", &self.gp_spsr) + .field("gp_pad", &self.gp_pad) + .finish() + } + } + impl ::hash::Hash for gpregs { + fn hash(&self, state: &mut H) { + self.gp_x.hash(state); + self.gp_lr.hash(state); + self.gp_sp.hash(state); + self.gp_elr.hash(state); + self.gp_spsr.hash(state); + self.gp_pad.hash(state); + } + } + impl PartialEq for fpregs { + fn eq(&self, other: &fpregs) -> bool { + self.fp_q == other.fp_q && + self.fp_sr == other.fp_sr && + self.fp_cr == other.fp_cr && + self.fp_flags == other.fp_flags && + self.fp_pad == other.fp_pad + } + } + impl Eq for fpregs {} + impl ::fmt::Debug for fpregs { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("fpregs") + .field("fp_q", &self.fp_q) + .field("fp_sr", &self.fp_sr) + .field("fp_cr", &self.fp_cr) + .field("fp_flags", &self.fp_flags) + .field("fp_pad", &self.fp_pad) + .finish() + } + } + impl ::hash::Hash for fpregs { + fn hash(&self, state: &mut H) { + self.fp_q.hash(state); + self.fp_sr.hash(state); + self.fp_cr.hash(state); + self.fp_flags.hash(state); + self.fp_pad.hash(state); + } + } + impl PartialEq for mcontext_t { + fn eq(&self, other: &mcontext_t) -> bool { + self.mc_gpregs == other.mc_gpregs && + self.mc_fpregs == other.mc_fpregs && + self.mc_flags == other.mc_flags && + self.mc_pad == other.mc_pad && + self.mc_spare.iter().zip(other.mc_spare.iter()).all(|(a, b)| a == b) + } + } + impl Eq for mcontext_t {} + impl ::fmt::Debug for mcontext_t { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("mcontext_t") + .field("mc_gpregs", &self.mc_gpregs) + .field("mc_fpregs", &self.mc_fpregs) + .field("mc_flags", &self.mc_flags) + .field("mc_pad", &self.mc_pad) + .field("mc_spare", &self.mc_spare) + .finish() + } + } + impl ::hash::Hash for mcontext_t { + fn hash(&self, state: &mut H) { + self.mc_gpregs.hash(state); + self.mc_fpregs.hash(state); + self.mc_flags.hash(state); + self.mc_pad.hash(state); + self.mc_spare.hash(state); + } + } + } +} + pub const MAP_32BIT: ::c_int = 0x00080000; +pub const MINSIGSTKSZ: ::size_t = 4096; // 1024 * 4 diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/arm.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/arm.rs index b7480aa78..300b3dd45 100644 --- a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/arm.rs +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/arm.rs @@ -1,8 +1,10 @@ pub type c_char = u8; pub type c_long = i32; pub type c_ulong = u32; +pub type wchar_t = u32; pub type time_t = i64; pub type suseconds_t = i32; +pub type register_t = i32; s! { pub struct stat { @@ -45,3 +47,4 @@ cfg_if! { } } pub const MAP_32BIT: ::c_int = 0x00080000; +pub const MINSIGSTKSZ: ::size_t = 4096; // 1024 * 4 diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd11/x86_64.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd11/b64.rs similarity index 100% rename from crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd11/x86_64.rs rename to crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd11/b64.rs diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd11/mod.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd11/mod.rs index 79a152fc8..563c0f936 100644 --- a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd11/mod.rs +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd11/mod.rs @@ -1,4 +1,4 @@ -// APIs that had breaking changes after FreeBSD 11 +// APIs that were changed after FreeBSD 11 // The type of `nlink_t` changed from `u16` to `u64` in FreeBSD 12: pub type nlink_t = u16; @@ -29,6 +29,186 @@ s! { pub shm_dtime: ::time_t, pub shm_ctime: ::time_t, } + + pub struct kinfo_proc { + /// Size of this structure. + pub ki_structsize: ::c_int, + /// Reserved: layout identifier. + pub ki_layout: ::c_int, + /// Address of command arguments. + pub ki_args: *mut ::pargs, + // This is normally "struct proc". + /// Address of proc. + pub ki_paddr: *mut ::c_void, + // This is normally "struct user". + /// Kernel virtual address of u-area. + pub ki_addr: *mut ::c_void, + // This is normally "struct vnode". + /// Pointer to trace file. + pub ki_tracep: *mut ::c_void, + // This is normally "struct vnode". + /// Pointer to executable file. + pub ki_textvp: *mut ::c_void, + // This is normally "struct filedesc". + /// Pointer to open file info. + pub ki_fd: *mut ::c_void, + // This is normally "struct vmspace". + /// Pointer to kernel vmspace struct. + pub ki_vmspace: *mut ::c_void, + /// Sleep address. + pub ki_wchan: *mut ::c_void, + /// Process identifier. + pub ki_pid: ::pid_t, + /// Parent process ID. + pub ki_ppid: ::pid_t, + /// Process group ID. + pub ki_pgid: ::pid_t, + /// tty process group ID. + pub ki_tpgid: ::pid_t, + /// Process session ID. + pub ki_sid: ::pid_t, + /// Terminal session ID. + pub ki_tsid: ::pid_t, + /// Job control counter. + pub ki_jobc: ::c_short, + /// Unused (just here for alignment). + pub ki_spare_short1: ::c_short, + /// Controlling tty dev. + pub ki_tdev: ::dev_t, + /// Signals arrived but not delivered. + pub ki_siglist: ::sigset_t, + /// Current signal mask. + pub ki_sigmask: ::sigset_t, + /// Signals being ignored. + pub ki_sigignore: ::sigset_t, + /// Signals being caught by user. + pub ki_sigcatch: ::sigset_t, + /// Effective user ID. + pub ki_uid: ::uid_t, + /// Real user ID. + pub ki_ruid: ::uid_t, + /// Saved effective user ID. + pub ki_svuid: ::uid_t, + /// Real group ID. + pub ki_rgid: ::gid_t, + /// Saved effective group ID. + pub ki_svgid: ::gid_t, + /// Number of groups. + pub ki_ngroups: ::c_short, + /// Unused (just here for alignment). + pub ki_spare_short2: ::c_short, + /// Groups. + pub ki_groups: [::gid_t; ::KI_NGROUPS], + /// Virtual size. + pub ki_size: ::vm_size_t, + /// Current resident set size in pages. + pub ki_rssize: ::segsz_t, + /// Resident set size before last swap. + pub ki_swrss: ::segsz_t, + /// Text size (pages) XXX. + pub ki_tsize: ::segsz_t, + /// Data size (pages) XXX. + pub ki_dsize: ::segsz_t, + /// Stack size (pages). + pub ki_ssize: ::segsz_t, + /// Exit status for wait & stop signal. + pub ki_xstat: ::u_short, + /// Accounting flags. + pub ki_acflag: ::u_short, + /// %cpu for process during `ki_swtime`. + pub ki_pctcpu: ::fixpt_t, + /// Time averaged value of `ki_cpticks`. + pub ki_estcpu: ::u_int, + /// Time since last blocked. + pub ki_slptime: ::u_int, + /// Time swapped in or out. + pub ki_swtime: ::u_int, + /// Number of copy-on-write faults. + pub ki_cow: ::u_int, + /// Real time in microsec. + pub ki_runtime: u64, + /// Starting time. + pub ki_start: ::timeval, + /// Time used by process children. + pub ki_childtime: ::timeval, + /// P_* flags. + pub ki_flag: ::c_long, + /// KI_* flags (below). + pub ki_kiflag: ::c_long, + /// Kernel trace points. + pub ki_traceflag: ::c_int, + /// S* process status. + pub ki_stat: ::c_char, + /// Process "nice" value. + pub ki_nice: i8, // signed char + /// Process lock (prevent swap) count. + pub ki_lock: ::c_char, + /// Run queue index. + pub ki_rqindex: ::c_char, + /// Which cpu we are on. + pub ki_oncpu_old: ::c_uchar, + /// Last cpu we were on. + pub ki_lastcpu_old: ::c_uchar, + /// Thread name. + pub ki_tdname: [::c_char; ::TDNAMLEN + 1], + /// Wchan message. + pub ki_wmesg: [::c_char; ::WMESGLEN + 1], + /// Setlogin name. + pub ki_login: [::c_char; ::LOGNAMELEN + 1], + /// Lock name. + pub ki_lockname: [::c_char; ::LOCKNAMELEN + 1], + /// Command name. + pub ki_comm: [::c_char; ::COMMLEN + 1], + /// Emulation name. + pub ki_emul: [::c_char; ::KI_EMULNAMELEN + 1], + /// Login class. + pub ki_loginclass: [::c_char; ::LOGINCLASSLEN + 1], + /// More thread name. + pub ki_moretdname: [::c_char; ::MAXCOMLEN - ::TDNAMLEN + 1], + /// Spare string space. + pub ki_sparestrings: [[::c_char; 23]; 2], // little hack to allow PartialEq + /// Spare room for growth. + pub ki_spareints: [::c_int; ::KI_NSPARE_INT], + /// Which cpu we are on. + pub ki_oncpu: ::c_int, + /// Last cpu we were on. + pub ki_lastcpu: ::c_int, + /// PID of tracing process. + pub ki_tracer: ::c_int, + /// P2_* flags. + pub ki_flag2: ::c_int, + /// Default FIB number. + pub ki_fibnum: ::c_int, + /// Credential flags. + pub ki_cr_flags: ::u_int, + /// Process jail ID. + pub ki_jid: ::c_int, + /// Number of threads in total. + pub ki_numthreads: ::c_int, + /// Thread ID. + pub ki_tid: ::lwpid_t, + /// Process priority. + pub ki_pri: ::priority, + /// Process rusage statistics. + pub ki_rusage: ::rusage, + /// rusage of children processes. + pub ki_rusage_ch: ::rusage, + // This is normally "struct pcb". + /// Kernel virtual addr of pcb. + pub ki_pcb: *mut ::c_void, + /// Kernel virtual addr of stack. + pub ki_kstack: *mut ::c_void, + /// User convenience pointer. + pub ki_udata: *mut ::c_void, + // This is normally "struct thread". + pub ki_tdaddr: *mut ::c_void, + pub ki_spareptrs: [*mut ::c_void; ::KI_NSPARE_PTR], + pub ki_sparelongs: [::c_long; ::KI_NSPARE_LONG], + /// PS_* flags. + pub ki_sflag: ::c_long, + /// kthread flag. + pub ki_tdflags: ::c_long, + } } s_no_extra_traits! { @@ -67,6 +247,17 @@ s_no_extra_traits! { // Array length changed from 88 to 1024 in FreeBSD 12: pub f_mntonname: [::c_char; 88], } + + pub struct vnstat { + pub vn_fileid: u64, + pub vn_size: u64, + pub vn_mntdir: *mut ::c_char, + pub vn_dev: u32, + pub vn_fsid: u32, + pub vn_type: ::c_int, + pub vn_mode: u16, + pub vn_devname: [::c_char; ::SPECNAMELEN as usize + 1], + } } cfg_if! { @@ -186,10 +377,70 @@ cfg_if! { self.d_name[..self.d_namlen as _].hash(state); } } + + impl PartialEq for vnstat { + fn eq(&self, other: &vnstat) -> bool { + let self_vn_devname: &[::c_char] = &self.vn_devname; + let other_vn_devname: &[::c_char] = &other.vn_devname; + + self.vn_fileid == other.vn_fileid && + self.vn_size == other.vn_size && + self.vn_mntdir == other.vn_mntdir && + self.vn_dev == other.vn_dev && + self.vn_fsid == other.vn_fsid && + self.vn_type == other.vn_type && + self.vn_mode == other.vn_mode && + self_vn_devname == other_vn_devname + } + } + impl Eq for vnstat {} + impl ::fmt::Debug for vnstat { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + let self_vn_devname: &[::c_char] = &self.vn_devname; + + f.debug_struct("vnstat") + .field("vn_fileid", &self.vn_fileid) + .field("vn_size", &self.vn_size) + .field("vn_mntdir", &self.vn_mntdir) + .field("vn_dev", &self.vn_dev) + .field("vn_fsid", &self.vn_fsid) + .field("vn_type", &self.vn_type) + .field("vn_mode", &self.vn_mode) + .field("vn_devname", &self_vn_devname) + .finish() + } + } + impl ::hash::Hash for vnstat { + fn hash(&self, state: &mut H) { + let self_vn_devname: &[::c_char] = &self.vn_devname; + + self.vn_fileid.hash(state); + self.vn_size.hash(state); + self.vn_mntdir.hash(state); + self.vn_dev.hash(state); + self.vn_fsid.hash(state); + self.vn_type.hash(state); + self.vn_mode.hash(state); + self_vn_devname.hash(state); + } + } } } pub const ELAST: ::c_int = 96; +pub const RAND_MAX: ::c_int = 0x7fff_fffd; +pub const KI_NSPARE_PTR: usize = 6; +pub const MINCORE_SUPER: ::c_int = 0x20; +/// max length of devicename +pub const SPECNAMELEN: ::c_int = 63; + +safe_f! { + pub {const} fn makedev(major: ::c_uint, minor: ::c_uint) -> ::dev_t { + let major = major as ::dev_t; + let minor = minor as ::dev_t; + (major << 8) | minor + } +} extern "C" { // Return type ::c_int was removed in FreeBSD 12 @@ -197,11 +448,7 @@ extern "C" { // Type of `addr` argument changed from `const void*` to `void*` // in FreeBSD 12 - pub fn mprotect( - addr: *const ::c_void, - len: ::size_t, - prot: ::c_int, - ) -> ::c_int; + pub fn mprotect(addr: *const ::c_void, len: ::size_t, prot: ::c_int) -> ::c_int; // Return type ::c_int was removed in FreeBSD 12 pub fn freelocale(loc: ::locale_t) -> ::c_int; @@ -214,11 +461,18 @@ extern "C" { msgtyp: ::c_long, msgflg: ::c_int, ) -> ::c_int; + + pub fn fdatasync(fd: ::c_int) -> ::c_int; + + pub fn dirname(path: *const ::c_char) -> *mut ::c_char; + pub fn basename(path: *const ::c_char) -> *mut ::c_char; } cfg_if! { - if #[cfg(target_arch = "x86_64")] { - mod x86_64; - pub use self::x86_64::*; + if #[cfg(any(target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64"))] { + mod b64; + pub use self::b64::*; } } diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd12/b64.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd12/b64.rs new file mode 100644 index 000000000..80c6fa168 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd12/b64.rs @@ -0,0 +1,34 @@ +#[repr(C)] +#[cfg_attr(feature = "extra_traits", derive(Debug, Eq, Hash, PartialEq))] +pub struct stat { + pub st_dev: ::dev_t, + pub st_ino: ::ino_t, + pub st_nlink: ::nlink_t, + pub st_mode: ::mode_t, + st_padding0: i16, + pub st_uid: ::uid_t, + pub st_gid: ::gid_t, + st_padding1: i32, + pub st_rdev: ::dev_t, + pub st_atime: ::time_t, + pub st_atime_nsec: ::c_long, + pub st_mtime: ::time_t, + pub st_mtime_nsec: ::c_long, + pub st_ctime: ::time_t, + pub st_ctime_nsec: ::c_long, + pub st_birthtime: ::time_t, + pub st_birthtime_nsec: ::c_long, + pub st_size: ::off_t, + pub st_blocks: ::blkcnt_t, + pub st_blksize: ::blksize_t, + pub st_flags: ::fflags_t, + pub st_gen: u64, + pub st_spare: [u64; 10], +} + +impl ::Copy for ::stat {} +impl ::Clone for ::stat { + fn clone(&self) -> ::stat { + *self + } +} diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd12/mod.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd12/mod.rs index 6bf7f957e..df00b6c1d 100644 --- a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd12/mod.rs +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd12/mod.rs @@ -1,4 +1,4 @@ -// APIs that changed in FreeBSD12 +// APIs in FreeBSD 12 that have changed since 11. pub type nlink_t = u64; pub type dev_t = u64; @@ -22,10 +22,202 @@ s! { pub filter: ::c_short, pub flags: ::c_ushort, pub fflags: ::c_uint, - pub data: ::intptr_t, + pub data: i64, pub udata: *mut ::c_void, pub ext: [u64; 4], } + + pub struct kvm_page { + pub version: ::c_uint, + pub paddr: ::c_ulong, + pub kmap_vaddr: ::c_ulong, + pub dmap_vaddr: ::c_ulong, + pub prot: ::vm_prot_t, + pub offset: ::u_long, + pub len: ::size_t, + } + + pub struct kinfo_proc { + /// Size of this structure. + pub ki_structsize: ::c_int, + /// Reserved: layout identifier. + pub ki_layout: ::c_int, + /// Address of command arguments. + pub ki_args: *mut ::pargs, + // This is normally "struct proc". + /// Address of proc. + pub ki_paddr: *mut ::c_void, + // This is normally "struct user". + /// Kernel virtual address of u-area. + pub ki_addr: *mut ::c_void, + // This is normally "struct vnode". + /// Pointer to trace file. + pub ki_tracep: *mut ::c_void, + // This is normally "struct vnode". + /// Pointer to executable file. + pub ki_textvp: *mut ::c_void, + // This is normally "struct filedesc". + /// Pointer to open file info. + pub ki_fd: *mut ::c_void, + // This is normally "struct vmspace". + /// Pointer to kernel vmspace struct. + pub ki_vmspace: *mut ::c_void, + /// Sleep address. + pub ki_wchan: *mut ::c_void, + /// Process identifier. + pub ki_pid: ::pid_t, + /// Parent process ID. + pub ki_ppid: ::pid_t, + /// Process group ID. + pub ki_pgid: ::pid_t, + /// tty process group ID. + pub ki_tpgid: ::pid_t, + /// Process session ID. + pub ki_sid: ::pid_t, + /// Terminal session ID. + pub ki_tsid: ::pid_t, + /// Job control counter. + pub ki_jobc: ::c_short, + /// Unused (just here for alignment). + pub ki_spare_short1: ::c_short, + /// Controlling tty dev. + pub ki_tdev_freebsd11: u32, + /// Signals arrived but not delivered. + pub ki_siglist: ::sigset_t, + /// Current signal mask. + pub ki_sigmask: ::sigset_t, + /// Signals being ignored. + pub ki_sigignore: ::sigset_t, + /// Signals being caught by user. + pub ki_sigcatch: ::sigset_t, + /// Effective user ID. + pub ki_uid: ::uid_t, + /// Real user ID. + pub ki_ruid: ::uid_t, + /// Saved effective user ID. + pub ki_svuid: ::uid_t, + /// Real group ID. + pub ki_rgid: ::gid_t, + /// Saved effective group ID. + pub ki_svgid: ::gid_t, + /// Number of groups. + pub ki_ngroups: ::c_short, + /// Unused (just here for alignment). + pub ki_spare_short2: ::c_short, + /// Groups. + pub ki_groups: [::gid_t; ::KI_NGROUPS], + /// Virtual size. + pub ki_size: ::vm_size_t, + /// Current resident set size in pages. + pub ki_rssize: ::segsz_t, + /// Resident set size before last swap. + pub ki_swrss: ::segsz_t, + /// Text size (pages) XXX. + pub ki_tsize: ::segsz_t, + /// Data size (pages) XXX. + pub ki_dsize: ::segsz_t, + /// Stack size (pages). + pub ki_ssize: ::segsz_t, + /// Exit status for wait & stop signal. + pub ki_xstat: ::u_short, + /// Accounting flags. + pub ki_acflag: ::u_short, + /// %cpu for process during `ki_swtime`. + pub ki_pctcpu: ::fixpt_t, + /// Time averaged value of `ki_cpticks`. + pub ki_estcpu: ::u_int, + /// Time since last blocked. + pub ki_slptime: ::u_int, + /// Time swapped in or out. + pub ki_swtime: ::u_int, + /// Number of copy-on-write faults. + pub ki_cow: ::u_int, + /// Real time in microsec. + pub ki_runtime: u64, + /// Starting time. + pub ki_start: ::timeval, + /// Time used by process children. + pub ki_childtime: ::timeval, + /// P_* flags. + pub ki_flag: ::c_long, + /// KI_* flags (below). + pub ki_kiflag: ::c_long, + /// Kernel trace points. + pub ki_traceflag: ::c_int, + /// S* process status. + pub ki_stat: ::c_char, + /// Process "nice" value. + pub ki_nice: i8, // signed char + /// Process lock (prevent swap) count. + pub ki_lock: ::c_char, + /// Run queue index. + pub ki_rqindex: ::c_char, + /// Which cpu we are on. + pub ki_oncpu_old: ::c_uchar, + /// Last cpu we were on. + pub ki_lastcpu_old: ::c_uchar, + /// Thread name. + pub ki_tdname: [::c_char; ::TDNAMLEN + 1], + /// Wchan message. + pub ki_wmesg: [::c_char; ::WMESGLEN + 1], + /// Setlogin name. + pub ki_login: [::c_char; ::LOGNAMELEN + 1], + /// Lock name. + pub ki_lockname: [::c_char; ::LOCKNAMELEN + 1], + /// Command name. + pub ki_comm: [::c_char; ::COMMLEN + 1], + /// Emulation name. + pub ki_emul: [::c_char; ::KI_EMULNAMELEN + 1], + /// Login class. + pub ki_loginclass: [::c_char; ::LOGINCLASSLEN + 1], + /// More thread name. + pub ki_moretdname: [::c_char; ::MAXCOMLEN - ::TDNAMLEN + 1], + /// Spare string space. + pub ki_sparestrings: [[::c_char; 23]; 2], // little hack to allow PartialEq + /// Spare room for growth. + pub ki_spareints: [::c_int; ::KI_NSPARE_INT], + /// Controlling tty dev. + pub ki_tdev: ::dev_t, + /// Which cpu we are on. + pub ki_oncpu: ::c_int, + /// Last cpu we were on. + pub ki_lastcpu: ::c_int, + /// PID of tracing process. + pub ki_tracer: ::c_int, + /// P2_* flags. + pub ki_flag2: ::c_int, + /// Default FIB number. + pub ki_fibnum: ::c_int, + /// Credential flags. + pub ki_cr_flags: ::u_int, + /// Process jail ID. + pub ki_jid: ::c_int, + /// Number of threads in total. + pub ki_numthreads: ::c_int, + /// Thread ID. + pub ki_tid: ::lwpid_t, + /// Process priority. + pub ki_pri: ::priority, + /// Process rusage statistics. + pub ki_rusage: ::rusage, + /// rusage of children processes. + pub ki_rusage_ch: ::rusage, + // This is normally "struct pcb". + /// Kernel virtual addr of pcb. + pub ki_pcb: *mut ::c_void, + /// Kernel virtual addr of stack. + pub ki_kstack: *mut ::c_void, + /// User convenience pointer. + pub ki_udata: *mut ::c_void, + // This is normally "struct thread". + pub ki_tdaddr: *mut ::c_void, + pub ki_spareptrs: [*mut ::c_void; ::KI_NSPARE_PTR], + pub ki_sparelongs: [::c_long; ::KI_NSPARE_LONG], + /// PS_* flags. + pub ki_sflag: ::c_long, + /// kthread flag. + pub ki_tdflags: ::c_long, + } } s_no_extra_traits! { @@ -64,6 +256,17 @@ s_no_extra_traits! { pub f_mntfromname: [::c_char; 1024], pub f_mntonname: [::c_char; 1024], } + + pub struct vnstat { + pub vn_fileid: u64, + pub vn_size: u64, + pub vn_dev: u64, + pub vn_fsid: u64, + pub vn_mntdir: *mut ::c_char, + pub vn_type: ::c_int, + pub vn_mode: u16, + pub vn_devname: [::c_char; ::SPECNAMELEN as usize + 1], + } } cfg_if! { @@ -187,25 +390,81 @@ cfg_if! { self.d_name[..self.d_namlen as _].hash(state); } } + + impl PartialEq for vnstat { + fn eq(&self, other: &vnstat) -> bool { + let self_vn_devname: &[::c_char] = &self.vn_devname; + let other_vn_devname: &[::c_char] = &other.vn_devname; + + self.vn_fileid == other.vn_fileid && + self.vn_size == other.vn_size && + self.vn_dev == other.vn_dev && + self.vn_fsid == other.vn_fsid && + self.vn_mntdir == other.vn_mntdir && + self.vn_type == other.vn_type && + self.vn_mode == other.vn_mode && + self_vn_devname == other_vn_devname + } + } + impl Eq for vnstat {} + impl ::fmt::Debug for vnstat { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + let self_vn_devname: &[::c_char] = &self.vn_devname; + + f.debug_struct("vnstat") + .field("vn_fileid", &self.vn_fileid) + .field("vn_size", &self.vn_size) + .field("vn_dev", &self.vn_dev) + .field("vn_fsid", &self.vn_fsid) + .field("vn_mntdir", &self.vn_mntdir) + .field("vn_type", &self.vn_type) + .field("vn_mode", &self.vn_mode) + .field("vn_devname", &self_vn_devname) + .finish() + } + } + impl ::hash::Hash for vnstat { + fn hash(&self, state: &mut H) { + let self_vn_devname: &[::c_char] = &self.vn_devname; + + self.vn_fileid.hash(state); + self.vn_size.hash(state); + self.vn_dev.hash(state); + self.vn_fsid.hash(state); + self.vn_mntdir.hash(state); + self.vn_type.hash(state); + self.vn_mode.hash(state); + self_vn_devname.hash(state); + } + } } } -cfg_if! { - if #[cfg(not(freebsd13))] { - pub const ELAST: ::c_int = 96; - } else { - pub const EINTEGRITY: ::c_int = 97; - pub const ELAST: ::c_int = 97; +pub const RAND_MAX: ::c_int = 0x7fff_fffd; +pub const ELAST: ::c_int = 97; + +/// max length of devicename +pub const SPECNAMELEN: ::c_int = 63; +pub const KI_NSPARE_PTR: usize = 6; + +pub const MINCORE_SUPER: ::c_int = 0x20; + +safe_f! { + pub {const} fn makedev(major: ::c_uint, minor: ::c_uint) -> ::dev_t { + let major = major as ::dev_t; + let minor = minor as ::dev_t; + let mut dev = 0; + dev |= ((major & 0xffffff00) as dev_t) << 32; + dev |= ((major & 0x000000ff) as dev_t) << 8; + dev |= ((minor & 0x0000ff00) as dev_t) << 24; + dev |= ((minor & 0xffff00ff) as dev_t) << 0; + dev } } extern "C" { pub fn setgrent(); - pub fn mprotect( - addr: *mut ::c_void, - len: ::size_t, - prot: ::c_int, - ) -> ::c_int; + pub fn mprotect(addr: *mut ::c_void, len: ::size_t, prot: ::c_int) -> ::c_int; pub fn freelocale(loc: ::locale_t); pub fn msgrcv( msqid: ::c_int, @@ -214,6 +473,32 @@ extern "C" { msgtyp: ::c_long, msgflg: ::c_int, ) -> ::ssize_t; + pub fn clock_nanosleep( + clk_id: ::clockid_t, + flags: ::c_int, + rqtp: *const ::timespec, + rmtp: *mut ::timespec, + ) -> ::c_int; + + pub fn fdatasync(fd: ::c_int) -> ::c_int; + + pub fn getrandom(buf: *mut ::c_void, buflen: ::size_t, flags: ::c_uint) -> ::ssize_t; + pub fn elf_aux_info(aux: ::c_int, buf: *mut ::c_void, buflen: ::c_int) -> ::c_int; + pub fn setproctitle_fast(fmt: *const ::c_char, ...); + pub fn timingsafe_bcmp(a: *const ::c_void, b: *const ::c_void, len: ::size_t) -> ::c_int; + pub fn timingsafe_memcmp(a: *const ::c_void, b: *const ::c_void, len: ::size_t) -> ::c_int; + + pub fn dirname(path: *mut ::c_char) -> *mut ::c_char; + pub fn basename(path: *mut ::c_char) -> *mut ::c_char; +} + +cfg_if! { + if #[cfg(any(target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64"))] { + mod b64; + pub use self::b64::*; + } } cfg_if! { diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd12/x86_64.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd12/x86_64.rs index 80c6fa168..7bf253445 100644 --- a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd12/x86_64.rs +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd12/x86_64.rs @@ -1,34 +1,5 @@ -#[repr(C)] -#[cfg_attr(feature = "extra_traits", derive(Debug, Eq, Hash, PartialEq))] -pub struct stat { - pub st_dev: ::dev_t, - pub st_ino: ::ino_t, - pub st_nlink: ::nlink_t, - pub st_mode: ::mode_t, - st_padding0: i16, - pub st_uid: ::uid_t, - pub st_gid: ::gid_t, - st_padding1: i32, - pub st_rdev: ::dev_t, - pub st_atime: ::time_t, - pub st_atime_nsec: ::c_long, - pub st_mtime: ::time_t, - pub st_mtime_nsec: ::c_long, - pub st_ctime: ::time_t, - pub st_ctime_nsec: ::c_long, - pub st_birthtime: ::time_t, - pub st_birthtime_nsec: ::c_long, - pub st_size: ::off_t, - pub st_blocks: ::blkcnt_t, - pub st_blksize: ::blksize_t, - pub st_flags: ::fflags_t, - pub st_gen: u64, - pub st_spare: [u64; 10], -} - -impl ::Copy for ::stat {} -impl ::Clone for ::stat { - fn clone(&self) -> ::stat { - *self - } -} +pub const PROC_KPTI_CTL: ::c_int = ::PROC_PROCCTL_MD_MIN; +pub const PROC_KPTI_CTL_ENABLE_ON_EXEC: ::c_int = 1; +pub const PROC_KPTI_CTL_DISABLE_ON_EXEC: ::c_int = 2; +pub const PROC_KPTI_STATUS: ::c_int = ::PROC_PROCCTL_MD_MIN + 1; +pub const PROC_KPTI_STATUS_ACTIVE: ::c_int = 0x80000000; diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd13/b64.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd13/b64.rs new file mode 100644 index 000000000..80c6fa168 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd13/b64.rs @@ -0,0 +1,34 @@ +#[repr(C)] +#[cfg_attr(feature = "extra_traits", derive(Debug, Eq, Hash, PartialEq))] +pub struct stat { + pub st_dev: ::dev_t, + pub st_ino: ::ino_t, + pub st_nlink: ::nlink_t, + pub st_mode: ::mode_t, + st_padding0: i16, + pub st_uid: ::uid_t, + pub st_gid: ::gid_t, + st_padding1: i32, + pub st_rdev: ::dev_t, + pub st_atime: ::time_t, + pub st_atime_nsec: ::c_long, + pub st_mtime: ::time_t, + pub st_mtime_nsec: ::c_long, + pub st_ctime: ::time_t, + pub st_ctime_nsec: ::c_long, + pub st_birthtime: ::time_t, + pub st_birthtime_nsec: ::c_long, + pub st_size: ::off_t, + pub st_blocks: ::blkcnt_t, + pub st_blksize: ::blksize_t, + pub st_flags: ::fflags_t, + pub st_gen: u64, + pub st_spare: [u64; 10], +} + +impl ::Copy for ::stat {} +impl ::Clone for ::stat { + fn clone(&self) -> ::stat { + *self + } +} diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd13/mod.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd13/mod.rs new file mode 100644 index 000000000..72a38dc22 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd13/mod.rs @@ -0,0 +1,553 @@ +// APIs in FreeBSD 14 that have changed since 11. + +pub type nlink_t = u64; +pub type dev_t = u64; +pub type ino_t = ::c_ulong; +pub type shmatt_t = ::c_uint; +pub type kpaddr_t = u64; +pub type kssize_t = i64; +pub type domainset_t = __c_anonymous_domainset; + +s! { + pub struct shmid_ds { + pub shm_perm: ::ipc_perm, + pub shm_segsz: ::size_t, + pub shm_lpid: ::pid_t, + pub shm_cpid: ::pid_t, + pub shm_nattch: ::shmatt_t, + pub shm_atime: ::time_t, + pub shm_dtime: ::time_t, + pub shm_ctime: ::time_t, + } + + pub struct kevent { + pub ident: ::uintptr_t, + pub filter: ::c_short, + pub flags: ::c_ushort, + pub fflags: ::c_uint, + pub data: i64, + pub udata: *mut ::c_void, + pub ext: [u64; 4], + } + + pub struct kvm_page { + pub kp_version: ::u_int, + pub kp_paddr: ::kpaddr_t, + pub kp_kmap_vaddr: ::kvaddr_t, + pub kp_dmap_vaddr: ::kvaddr_t, + pub kp_prot: ::vm_prot_t, + pub kp_offset: ::off_t, + pub kp_len: ::size_t, + } + + pub struct __c_anonymous_domainset { + _priv: [::uintptr_t; 4], + } + + pub struct kinfo_proc { + /// Size of this structure. + pub ki_structsize: ::c_int, + /// Reserved: layout identifier. + pub ki_layout: ::c_int, + /// Address of command arguments. + pub ki_args: *mut ::pargs, + // This is normally "struct proc". + /// Address of proc. + pub ki_paddr: *mut ::c_void, + // This is normally "struct user". + /// Kernel virtual address of u-area. + pub ki_addr: *mut ::c_void, + // This is normally "struct vnode". + /// Pointer to trace file. + pub ki_tracep: *mut ::c_void, + // This is normally "struct vnode". + /// Pointer to executable file. + pub ki_textvp: *mut ::c_void, + // This is normally "struct filedesc". + /// Pointer to open file info. + pub ki_fd: *mut ::c_void, + // This is normally "struct vmspace". + /// Pointer to kernel vmspace struct. + pub ki_vmspace: *mut ::c_void, + /// Sleep address. + pub ki_wchan: *const ::c_void, + /// Process identifier. + pub ki_pid: ::pid_t, + /// Parent process ID. + pub ki_ppid: ::pid_t, + /// Process group ID. + pub ki_pgid: ::pid_t, + /// tty process group ID. + pub ki_tpgid: ::pid_t, + /// Process session ID. + pub ki_sid: ::pid_t, + /// Terminal session ID. + pub ki_tsid: ::pid_t, + /// Job control counter. + pub ki_jobc: ::c_short, + /// Unused (just here for alignment). + pub ki_spare_short1: ::c_short, + /// Controlling tty dev. + pub ki_tdev_freebsd11: u32, + /// Signals arrived but not delivered. + pub ki_siglist: ::sigset_t, + /// Current signal mask. + pub ki_sigmask: ::sigset_t, + /// Signals being ignored. + pub ki_sigignore: ::sigset_t, + /// Signals being caught by user. + pub ki_sigcatch: ::sigset_t, + /// Effective user ID. + pub ki_uid: ::uid_t, + /// Real user ID. + pub ki_ruid: ::uid_t, + /// Saved effective user ID. + pub ki_svuid: ::uid_t, + /// Real group ID. + pub ki_rgid: ::gid_t, + /// Saved effective group ID. + pub ki_svgid: ::gid_t, + /// Number of groups. + pub ki_ngroups: ::c_short, + /// Unused (just here for alignment). + pub ki_spare_short2: ::c_short, + /// Groups. + pub ki_groups: [::gid_t; ::KI_NGROUPS], + /// Virtual size. + pub ki_size: ::vm_size_t, + /// Current resident set size in pages. + pub ki_rssize: ::segsz_t, + /// Resident set size before last swap. + pub ki_swrss: ::segsz_t, + /// Text size (pages) XXX. + pub ki_tsize: ::segsz_t, + /// Data size (pages) XXX. + pub ki_dsize: ::segsz_t, + /// Stack size (pages). + pub ki_ssize: ::segsz_t, + /// Exit status for wait & stop signal. + pub ki_xstat: ::u_short, + /// Accounting flags. + pub ki_acflag: ::u_short, + /// %cpu for process during `ki_swtime`. + pub ki_pctcpu: ::fixpt_t, + /// Time averaged value of `ki_cpticks`. + pub ki_estcpu: ::u_int, + /// Time since last blocked. + pub ki_slptime: ::u_int, + /// Time swapped in or out. + pub ki_swtime: ::u_int, + /// Number of copy-on-write faults. + pub ki_cow: ::u_int, + /// Real time in microsec. + pub ki_runtime: u64, + /// Starting time. + pub ki_start: ::timeval, + /// Time used by process children. + pub ki_childtime: ::timeval, + /// P_* flags. + pub ki_flag: ::c_long, + /// KI_* flags (below). + pub ki_kiflag: ::c_long, + /// Kernel trace points. + pub ki_traceflag: ::c_int, + /// S* process status. + pub ki_stat: ::c_char, + /// Process "nice" value. + pub ki_nice: i8, // signed char + /// Process lock (prevent swap) count. + pub ki_lock: ::c_char, + /// Run queue index. + pub ki_rqindex: ::c_char, + /// Which cpu we are on. + pub ki_oncpu_old: ::c_uchar, + /// Last cpu we were on. + pub ki_lastcpu_old: ::c_uchar, + /// Thread name. + pub ki_tdname: [::c_char; ::TDNAMLEN + 1], + /// Wchan message. + pub ki_wmesg: [::c_char; ::WMESGLEN + 1], + /// Setlogin name. + pub ki_login: [::c_char; ::LOGNAMELEN + 1], + /// Lock name. + pub ki_lockname: [::c_char; ::LOCKNAMELEN + 1], + /// Command name. + pub ki_comm: [::c_char; ::COMMLEN + 1], + /// Emulation name. + pub ki_emul: [::c_char; ::KI_EMULNAMELEN + 1], + /// Login class. + pub ki_loginclass: [::c_char; ::LOGINCLASSLEN + 1], + /// More thread name. + pub ki_moretdname: [::c_char; ::MAXCOMLEN - ::TDNAMLEN + 1], + /// Spare string space. + pub ki_sparestrings: [[::c_char; 23]; 2], // little hack to allow PartialEq + /// Spare room for growth. + pub ki_spareints: [::c_int; ::KI_NSPARE_INT], + /// Controlling tty dev. + pub ki_tdev: u64, + /// Which cpu we are on. + pub ki_oncpu: ::c_int, + /// Last cpu we were on. + pub ki_lastcpu: ::c_int, + /// PID of tracing process. + pub ki_tracer: ::c_int, + /// P2_* flags. + pub ki_flag2: ::c_int, + /// Default FIB number. + pub ki_fibnum: ::c_int, + /// Credential flags. + pub ki_cr_flags: ::u_int, + /// Process jail ID. + pub ki_jid: ::c_int, + /// Number of threads in total. + pub ki_numthreads: ::c_int, + /// Thread ID. + pub ki_tid: ::lwpid_t, + /// Process priority. + pub ki_pri: ::priority, + /// Process rusage statistics. + pub ki_rusage: ::rusage, + /// rusage of children processes. + pub ki_rusage_ch: ::rusage, + // This is normally "struct pcb". + /// Kernel virtual addr of pcb. + pub ki_pcb: *mut ::c_void, + /// Kernel virtual addr of stack. + pub ki_kstack: *mut ::c_void, + /// User convenience pointer. + pub ki_udata: *mut ::c_void, + // This is normally "struct thread". + pub ki_tdaddr: *mut ::c_void, + // This is normally "struct pwddesc". + /// Pointer to process paths info. + pub ki_pd: *mut ::c_void, + pub ki_spareptrs: [*mut ::c_void; ::KI_NSPARE_PTR], + pub ki_sparelongs: [::c_long; ::KI_NSPARE_LONG], + /// PS_* flags. + pub ki_sflag: ::c_long, + /// kthread flag. + pub ki_tdflags: ::c_long, + } +} + +s_no_extra_traits! { + pub struct dirent { + pub d_fileno: ::ino_t, + pub d_off: ::off_t, + pub d_reclen: u16, + pub d_type: u8, + d_pad0: u8, + pub d_namlen: u16, + d_pad1: u16, + pub d_name: [::c_char; 256], + } + + pub struct statfs { + pub f_version: u32, + pub f_type: u32, + pub f_flags: u64, + pub f_bsize: u64, + pub f_iosize: u64, + pub f_blocks: u64, + pub f_bfree: u64, + pub f_bavail: i64, + pub f_files: u64, + pub f_ffree: i64, + pub f_syncwrites: u64, + pub f_asyncwrites: u64, + pub f_syncreads: u64, + pub f_asyncreads: u64, + f_spare: [u64; 10], + pub f_namemax: u32, + pub f_owner: ::uid_t, + pub f_fsid: ::fsid_t, + f_charspare: [::c_char; 80], + pub f_fstypename: [::c_char; 16], + pub f_mntfromname: [::c_char; 1024], + pub f_mntonname: [::c_char; 1024], + } + + pub struct vnstat { + pub vn_fileid: u64, + pub vn_size: u64, + pub vn_dev: u64, + pub vn_fsid: u64, + pub vn_mntdir: *mut ::c_char, + pub vn_type: ::c_int, + pub vn_mode: u16, + pub vn_devname: [::c_char; ::SPECNAMELEN as usize + 1], + } +} + +cfg_if! { + if #[cfg(feature = "extra_traits")] { + impl PartialEq for statfs { + fn eq(&self, other: &statfs) -> bool { + self.f_version == other.f_version + && self.f_type == other.f_type + && self.f_flags == other.f_flags + && self.f_bsize == other.f_bsize + && self.f_iosize == other.f_iosize + && self.f_blocks == other.f_blocks + && self.f_bfree == other.f_bfree + && self.f_bavail == other.f_bavail + && self.f_files == other.f_files + && self.f_ffree == other.f_ffree + && self.f_syncwrites == other.f_syncwrites + && self.f_asyncwrites == other.f_asyncwrites + && self.f_syncreads == other.f_syncreads + && self.f_asyncreads == other.f_asyncreads + && self.f_namemax == other.f_namemax + && self.f_owner == other.f_owner + && self.f_fsid == other.f_fsid + && self.f_fstypename == other.f_fstypename + && self + .f_mntfromname + .iter() + .zip(other.f_mntfromname.iter()) + .all(|(a,b)| a == b) + && self + .f_mntonname + .iter() + .zip(other.f_mntonname.iter()) + .all(|(a,b)| a == b) + } + } + impl Eq for statfs {} + impl ::fmt::Debug for statfs { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("statfs") + .field("f_bsize", &self.f_bsize) + .field("f_iosize", &self.f_iosize) + .field("f_blocks", &self.f_blocks) + .field("f_bfree", &self.f_bfree) + .field("f_bavail", &self.f_bavail) + .field("f_files", &self.f_files) + .field("f_ffree", &self.f_ffree) + .field("f_syncwrites", &self.f_syncwrites) + .field("f_asyncwrites", &self.f_asyncwrites) + .field("f_syncreads", &self.f_syncreads) + .field("f_asyncreads", &self.f_asyncreads) + .field("f_namemax", &self.f_namemax) + .field("f_owner", &self.f_owner) + .field("f_fsid", &self.f_fsid) + .field("f_fstypename", &self.f_fstypename) + .field("f_mntfromname", &&self.f_mntfromname[..]) + .field("f_mntonname", &&self.f_mntonname[..]) + .finish() + } + } + impl ::hash::Hash for statfs { + fn hash(&self, state: &mut H) { + self.f_version.hash(state); + self.f_type.hash(state); + self.f_flags.hash(state); + self.f_bsize.hash(state); + self.f_iosize.hash(state); + self.f_blocks.hash(state); + self.f_bfree.hash(state); + self.f_bavail.hash(state); + self.f_files.hash(state); + self.f_ffree.hash(state); + self.f_syncwrites.hash(state); + self.f_asyncwrites.hash(state); + self.f_syncreads.hash(state); + self.f_asyncreads.hash(state); + self.f_namemax.hash(state); + self.f_owner.hash(state); + self.f_fsid.hash(state); + self.f_charspare.hash(state); + self.f_fstypename.hash(state); + self.f_mntfromname.hash(state); + self.f_mntonname.hash(state); + } + } + + impl PartialEq for dirent { + fn eq(&self, other: &dirent) -> bool { + self.d_fileno == other.d_fileno + && self.d_off == other.d_off + && self.d_reclen == other.d_reclen + && self.d_type == other.d_type + && self.d_namlen == other.d_namlen + && self + .d_name[..self.d_namlen as _] + .iter() + .zip(other.d_name.iter()) + .all(|(a,b)| a == b) + } + } + impl Eq for dirent {} + impl ::fmt::Debug for dirent { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("dirent") + .field("d_fileno", &self.d_fileno) + .field("d_off", &self.d_off) + .field("d_reclen", &self.d_reclen) + .field("d_type", &self.d_type) + .field("d_namlen", &self.d_namlen) + .field("d_name", &&self.d_name[..self.d_namlen as _]) + .finish() + } + } + impl ::hash::Hash for dirent { + fn hash(&self, state: &mut H) { + self.d_fileno.hash(state); + self.d_off.hash(state); + self.d_reclen.hash(state); + self.d_type.hash(state); + self.d_namlen.hash(state); + self.d_name[..self.d_namlen as _].hash(state); + } + } + + impl PartialEq for vnstat { + fn eq(&self, other: &vnstat) -> bool { + let self_vn_devname: &[::c_char] = &self.vn_devname; + let other_vn_devname: &[::c_char] = &other.vn_devname; + + self.vn_fileid == other.vn_fileid && + self.vn_size == other.vn_size && + self.vn_dev == other.vn_dev && + self.vn_fsid == other.vn_fsid && + self.vn_mntdir == other.vn_mntdir && + self.vn_type == other.vn_type && + self.vn_mode == other.vn_mode && + self_vn_devname == other_vn_devname + } + } + impl Eq for vnstat {} + impl ::fmt::Debug for vnstat { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + let self_vn_devname: &[::c_char] = &self.vn_devname; + + f.debug_struct("vnstat") + .field("vn_fileid", &self.vn_fileid) + .field("vn_size", &self.vn_size) + .field("vn_dev", &self.vn_dev) + .field("vn_fsid", &self.vn_fsid) + .field("vn_mntdir", &self.vn_mntdir) + .field("vn_type", &self.vn_type) + .field("vn_mode", &self.vn_mode) + .field("vn_devname", &self_vn_devname) + .finish() + } + } + impl ::hash::Hash for vnstat { + fn hash(&self, state: &mut H) { + let self_vn_devname: &[::c_char] = &self.vn_devname; + + self.vn_fileid.hash(state); + self.vn_size.hash(state); + self.vn_dev.hash(state); + self.vn_fsid.hash(state); + self.vn_mntdir.hash(state); + self.vn_type.hash(state); + self.vn_mode.hash(state); + self_vn_devname.hash(state); + } + } + } +} + +pub const RAND_MAX: ::c_int = 0x7fff_ffff; +pub const ELAST: ::c_int = 97; + +pub const KF_TYPE_EVENTFD: ::c_int = 13; + +/// max length of devicename +pub const SPECNAMELEN: ::c_int = 255; +pub const KI_NSPARE_PTR: usize = 5; + +/// domainset policies +pub const DOMAINSET_POLICY_INVALID: ::c_int = 0; +pub const DOMAINSET_POLICY_ROUNDROBIN: ::c_int = 1; +pub const DOMAINSET_POLICY_FIRSTTOUCH: ::c_int = 2; +pub const DOMAINSET_POLICY_PREFER: ::c_int = 3; +pub const DOMAINSET_POLICY_INTERLEAVE: ::c_int = 4; + +pub const MINCORE_SUPER: ::c_int = 0x20; + +safe_f! { + pub {const} fn makedev(major: ::c_uint, minor: ::c_uint) -> ::dev_t { + let major = major as ::dev_t; + let minor = minor as ::dev_t; + let mut dev = 0; + dev |= ((major & 0xffffff00) as dev_t) << 32; + dev |= ((major & 0x000000ff) as dev_t) << 8; + dev |= ((minor & 0x0000ff00) as dev_t) << 24; + dev |= ((minor & 0xffff00ff) as dev_t) << 0; + dev + } +} + +extern "C" { + pub fn setgrent(); + pub fn mprotect(addr: *mut ::c_void, len: ::size_t, prot: ::c_int) -> ::c_int; + pub fn freelocale(loc: ::locale_t); + pub fn msgrcv( + msqid: ::c_int, + msgp: *mut ::c_void, + msgsz: ::size_t, + msgtyp: ::c_long, + msgflg: ::c_int, + ) -> ::ssize_t; + pub fn clock_nanosleep( + clk_id: ::clockid_t, + flags: ::c_int, + rqtp: *const ::timespec, + rmtp: *mut ::timespec, + ) -> ::c_int; + + pub fn eventfd(init: ::c_uint, flags: ::c_int) -> ::c_int; + + pub fn fdatasync(fd: ::c_int) -> ::c_int; + + pub fn getrandom(buf: *mut ::c_void, buflen: ::size_t, flags: ::c_uint) -> ::ssize_t; + pub fn getentropy(buf: *mut ::c_void, buflen: ::size_t) -> ::c_int; + pub fn elf_aux_info(aux: ::c_int, buf: *mut ::c_void, buflen: ::c_int) -> ::c_int; + pub fn setproctitle_fast(fmt: *const ::c_char, ...); + pub fn timingsafe_bcmp(a: *const ::c_void, b: *const ::c_void, len: ::size_t) -> ::c_int; + pub fn timingsafe_memcmp(a: *const ::c_void, b: *const ::c_void, len: ::size_t) -> ::c_int; + + pub fn cpuset_getdomain( + level: ::cpulevel_t, + which: ::cpuwhich_t, + id: ::id_t, + setsize: ::size_t, + mask: *mut ::domainset_t, + policy: *mut ::c_int, + ) -> ::c_int; + pub fn cpuset_setdomain( + level: ::cpulevel_t, + which: ::cpuwhich_t, + id: ::id_t, + setsize: ::size_t, + mask: *const ::domainset_t, + policy: ::c_int, + ) -> ::c_int; + + pub fn dirname(path: *mut ::c_char) -> *mut ::c_char; + pub fn basename(path: *mut ::c_char) -> *mut ::c_char; +} + +#[link(name = "kvm")] +extern "C" { + pub fn kvm_kerndisp(kd: *mut ::kvm_t) -> ::kssize_t; +} + +cfg_if! { + if #[cfg(any(target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64"))] { + mod b64; + pub use self::b64::*; + } +} + +cfg_if! { + if #[cfg(target_arch = "x86_64")] { + mod x86_64; + pub use self::x86_64::*; + } +} diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd13/x86_64.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd13/x86_64.rs new file mode 100644 index 000000000..7bf253445 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd13/x86_64.rs @@ -0,0 +1,5 @@ +pub const PROC_KPTI_CTL: ::c_int = ::PROC_PROCCTL_MD_MIN; +pub const PROC_KPTI_CTL_ENABLE_ON_EXEC: ::c_int = 1; +pub const PROC_KPTI_CTL_DISABLE_ON_EXEC: ::c_int = 2; +pub const PROC_KPTI_STATUS: ::c_int = ::PROC_PROCCTL_MD_MIN + 1; +pub const PROC_KPTI_STATUS_ACTIVE: ::c_int = 0x80000000; diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd14/b64.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd14/b64.rs new file mode 100644 index 000000000..80c6fa168 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd14/b64.rs @@ -0,0 +1,34 @@ +#[repr(C)] +#[cfg_attr(feature = "extra_traits", derive(Debug, Eq, Hash, PartialEq))] +pub struct stat { + pub st_dev: ::dev_t, + pub st_ino: ::ino_t, + pub st_nlink: ::nlink_t, + pub st_mode: ::mode_t, + st_padding0: i16, + pub st_uid: ::uid_t, + pub st_gid: ::gid_t, + st_padding1: i32, + pub st_rdev: ::dev_t, + pub st_atime: ::time_t, + pub st_atime_nsec: ::c_long, + pub st_mtime: ::time_t, + pub st_mtime_nsec: ::c_long, + pub st_ctime: ::time_t, + pub st_ctime_nsec: ::c_long, + pub st_birthtime: ::time_t, + pub st_birthtime_nsec: ::c_long, + pub st_size: ::off_t, + pub st_blocks: ::blkcnt_t, + pub st_blksize: ::blksize_t, + pub st_flags: ::fflags_t, + pub st_gen: u64, + pub st_spare: [u64; 10], +} + +impl ::Copy for ::stat {} +impl ::Clone for ::stat { + fn clone(&self) -> ::stat { + *self + } +} diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd14/mod.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd14/mod.rs new file mode 100644 index 000000000..115b47764 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd14/mod.rs @@ -0,0 +1,553 @@ +// APIs in FreeBSD 13 that have changed since 11. + +pub type nlink_t = u64; +pub type dev_t = u64; +pub type ino_t = ::c_ulong; +pub type shmatt_t = ::c_uint; +pub type kpaddr_t = u64; +pub type kssize_t = i64; +pub type domainset_t = __c_anonymous_domainset; + +s! { + pub struct shmid_ds { + pub shm_perm: ::ipc_perm, + pub shm_segsz: ::size_t, + pub shm_lpid: ::pid_t, + pub shm_cpid: ::pid_t, + pub shm_nattch: ::shmatt_t, + pub shm_atime: ::time_t, + pub shm_dtime: ::time_t, + pub shm_ctime: ::time_t, + } + + pub struct kevent { + pub ident: ::uintptr_t, + pub filter: ::c_short, + pub flags: ::c_ushort, + pub fflags: ::c_uint, + pub data: i64, + pub udata: *mut ::c_void, + pub ext: [u64; 4], + } + + pub struct kvm_page { + pub kp_version: ::u_int, + pub kp_paddr: ::kpaddr_t, + pub kp_kmap_vaddr: ::kvaddr_t, + pub kp_dmap_vaddr: ::kvaddr_t, + pub kp_prot: ::vm_prot_t, + pub kp_offset: ::off_t, + pub kp_len: ::size_t, + } + + pub struct __c_anonymous_domainset { + _priv: [::uintptr_t; 4], + } + + pub struct kinfo_proc { + /// Size of this structure. + pub ki_structsize: ::c_int, + /// Reserved: layout identifier. + pub ki_layout: ::c_int, + /// Address of command arguments. + pub ki_args: *mut ::pargs, + // This is normally "struct proc". + /// Address of proc. + pub ki_paddr: *mut ::c_void, + // This is normally "struct user". + /// Kernel virtual address of u-area. + pub ki_addr: *mut ::c_void, + // This is normally "struct vnode". + /// Pointer to trace file. + pub ki_tracep: *mut ::c_void, + // This is normally "struct vnode". + /// Pointer to executable file. + pub ki_textvp: *mut ::c_void, + // This is normally "struct filedesc". + /// Pointer to open file info. + pub ki_fd: *mut ::c_void, + // This is normally "struct vmspace". + /// Pointer to kernel vmspace struct. + pub ki_vmspace: *mut ::c_void, + /// Sleep address. + pub ki_wchan: *const ::c_void, + /// Process identifier. + pub ki_pid: ::pid_t, + /// Parent process ID. + pub ki_ppid: ::pid_t, + /// Process group ID. + pub ki_pgid: ::pid_t, + /// tty process group ID. + pub ki_tpgid: ::pid_t, + /// Process session ID. + pub ki_sid: ::pid_t, + /// Terminal session ID. + pub ki_tsid: ::pid_t, + /// Job control counter. + pub ki_jobc: ::c_short, + /// Unused (just here for alignment). + pub ki_spare_short1: ::c_short, + /// Controlling tty dev. + pub ki_tdev_freebsd11: u32, + /// Signals arrived but not delivered. + pub ki_siglist: ::sigset_t, + /// Current signal mask. + pub ki_sigmask: ::sigset_t, + /// Signals being ignored. + pub ki_sigignore: ::sigset_t, + /// Signals being caught by user. + pub ki_sigcatch: ::sigset_t, + /// Effective user ID. + pub ki_uid: ::uid_t, + /// Real user ID. + pub ki_ruid: ::uid_t, + /// Saved effective user ID. + pub ki_svuid: ::uid_t, + /// Real group ID. + pub ki_rgid: ::gid_t, + /// Saved effective group ID. + pub ki_svgid: ::gid_t, + /// Number of groups. + pub ki_ngroups: ::c_short, + /// Unused (just here for alignment). + pub ki_spare_short2: ::c_short, + /// Groups. + pub ki_groups: [::gid_t; ::KI_NGROUPS], + /// Virtual size. + pub ki_size: ::vm_size_t, + /// Current resident set size in pages. + pub ki_rssize: ::segsz_t, + /// Resident set size before last swap. + pub ki_swrss: ::segsz_t, + /// Text size (pages) XXX. + pub ki_tsize: ::segsz_t, + /// Data size (pages) XXX. + pub ki_dsize: ::segsz_t, + /// Stack size (pages). + pub ki_ssize: ::segsz_t, + /// Exit status for wait & stop signal. + pub ki_xstat: ::u_short, + /// Accounting flags. + pub ki_acflag: ::u_short, + /// %cpu for process during `ki_swtime`. + pub ki_pctcpu: ::fixpt_t, + /// Time averaged value of `ki_cpticks`. + pub ki_estcpu: ::u_int, + /// Time since last blocked. + pub ki_slptime: ::u_int, + /// Time swapped in or out. + pub ki_swtime: ::u_int, + /// Number of copy-on-write faults. + pub ki_cow: ::u_int, + /// Real time in microsec. + pub ki_runtime: u64, + /// Starting time. + pub ki_start: ::timeval, + /// Time used by process children. + pub ki_childtime: ::timeval, + /// P_* flags. + pub ki_flag: ::c_long, + /// KI_* flags (below). + pub ki_kiflag: ::c_long, + /// Kernel trace points. + pub ki_traceflag: ::c_int, + /// S* process status. + pub ki_stat: ::c_char, + /// Process "nice" value. + pub ki_nice: i8, // signed char + /// Process lock (prevent swap) count. + pub ki_lock: ::c_char, + /// Run queue index. + pub ki_rqindex: ::c_char, + /// Which cpu we are on. + pub ki_oncpu_old: ::c_uchar, + /// Last cpu we were on. + pub ki_lastcpu_old: ::c_uchar, + /// Thread name. + pub ki_tdname: [::c_char; ::TDNAMLEN + 1], + /// Wchan message. + pub ki_wmesg: [::c_char; ::WMESGLEN + 1], + /// Setlogin name. + pub ki_login: [::c_char; ::LOGNAMELEN + 1], + /// Lock name. + pub ki_lockname: [::c_char; ::LOCKNAMELEN + 1], + /// Command name. + pub ki_comm: [::c_char; ::COMMLEN + 1], + /// Emulation name. + pub ki_emul: [::c_char; ::KI_EMULNAMELEN + 1], + /// Login class. + pub ki_loginclass: [::c_char; ::LOGINCLASSLEN + 1], + /// More thread name. + pub ki_moretdname: [::c_char; ::MAXCOMLEN - ::TDNAMLEN + 1], + /// Spare string space. + pub ki_sparestrings: [[::c_char; 23]; 2], // little hack to allow PartialEq + /// Spare room for growth. + pub ki_spareints: [::c_int; ::KI_NSPARE_INT], + /// Controlling tty dev. + pub ki_tdev: u64, + /// Which cpu we are on. + pub ki_oncpu: ::c_int, + /// Last cpu we were on. + pub ki_lastcpu: ::c_int, + /// PID of tracing process. + pub ki_tracer: ::c_int, + /// P2_* flags. + pub ki_flag2: ::c_int, + /// Default FIB number. + pub ki_fibnum: ::c_int, + /// Credential flags. + pub ki_cr_flags: ::u_int, + /// Process jail ID. + pub ki_jid: ::c_int, + /// Number of threads in total. + pub ki_numthreads: ::c_int, + /// Thread ID. + pub ki_tid: ::lwpid_t, + /// Process priority. + pub ki_pri: ::priority, + /// Process rusage statistics. + pub ki_rusage: ::rusage, + /// rusage of children processes. + pub ki_rusage_ch: ::rusage, + // This is normally "struct pcb". + /// Kernel virtual addr of pcb. + pub ki_pcb: *mut ::c_void, + /// Kernel virtual addr of stack. + pub ki_kstack: *mut ::c_void, + /// User convenience pointer. + pub ki_udata: *mut ::c_void, + // This is normally "struct thread". + pub ki_tdaddr: *mut ::c_void, + // This is normally "struct pwddesc". + /// Pointer to process paths info. + pub ki_pd: *mut ::c_void, + pub ki_spareptrs: [*mut ::c_void; ::KI_NSPARE_PTR], + pub ki_sparelongs: [::c_long; ::KI_NSPARE_LONG], + /// PS_* flags. + pub ki_sflag: ::c_long, + /// kthread flag. + pub ki_tdflags: ::c_long, + } +} + +s_no_extra_traits! { + pub struct dirent { + pub d_fileno: ::ino_t, + pub d_off: ::off_t, + pub d_reclen: u16, + pub d_type: u8, + d_pad0: u8, + pub d_namlen: u16, + d_pad1: u16, + pub d_name: [::c_char; 256], + } + + pub struct statfs { + pub f_version: u32, + pub f_type: u32, + pub f_flags: u64, + pub f_bsize: u64, + pub f_iosize: u64, + pub f_blocks: u64, + pub f_bfree: u64, + pub f_bavail: i64, + pub f_files: u64, + pub f_ffree: i64, + pub f_syncwrites: u64, + pub f_asyncwrites: u64, + pub f_syncreads: u64, + pub f_asyncreads: u64, + f_spare: [u64; 10], + pub f_namemax: u32, + pub f_owner: ::uid_t, + pub f_fsid: ::fsid_t, + f_charspare: [::c_char; 80], + pub f_fstypename: [::c_char; 16], + pub f_mntfromname: [::c_char; 1024], + pub f_mntonname: [::c_char; 1024], + } + + pub struct vnstat { + pub vn_fileid: u64, + pub vn_size: u64, + pub vn_dev: u64, + pub vn_fsid: u64, + pub vn_mntdir: *mut ::c_char, + pub vn_type: ::c_int, + pub vn_mode: u16, + pub vn_devname: [::c_char; ::SPECNAMELEN as usize + 1], + } +} + +cfg_if! { + if #[cfg(feature = "extra_traits")] { + impl PartialEq for statfs { + fn eq(&self, other: &statfs) -> bool { + self.f_version == other.f_version + && self.f_type == other.f_type + && self.f_flags == other.f_flags + && self.f_bsize == other.f_bsize + && self.f_iosize == other.f_iosize + && self.f_blocks == other.f_blocks + && self.f_bfree == other.f_bfree + && self.f_bavail == other.f_bavail + && self.f_files == other.f_files + && self.f_ffree == other.f_ffree + && self.f_syncwrites == other.f_syncwrites + && self.f_asyncwrites == other.f_asyncwrites + && self.f_syncreads == other.f_syncreads + && self.f_asyncreads == other.f_asyncreads + && self.f_namemax == other.f_namemax + && self.f_owner == other.f_owner + && self.f_fsid == other.f_fsid + && self.f_fstypename == other.f_fstypename + && self + .f_mntfromname + .iter() + .zip(other.f_mntfromname.iter()) + .all(|(a,b)| a == b) + && self + .f_mntonname + .iter() + .zip(other.f_mntonname.iter()) + .all(|(a,b)| a == b) + } + } + impl Eq for statfs {} + impl ::fmt::Debug for statfs { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("statfs") + .field("f_bsize", &self.f_bsize) + .field("f_iosize", &self.f_iosize) + .field("f_blocks", &self.f_blocks) + .field("f_bfree", &self.f_bfree) + .field("f_bavail", &self.f_bavail) + .field("f_files", &self.f_files) + .field("f_ffree", &self.f_ffree) + .field("f_syncwrites", &self.f_syncwrites) + .field("f_asyncwrites", &self.f_asyncwrites) + .field("f_syncreads", &self.f_syncreads) + .field("f_asyncreads", &self.f_asyncreads) + .field("f_namemax", &self.f_namemax) + .field("f_owner", &self.f_owner) + .field("f_fsid", &self.f_fsid) + .field("f_fstypename", &self.f_fstypename) + .field("f_mntfromname", &&self.f_mntfromname[..]) + .field("f_mntonname", &&self.f_mntonname[..]) + .finish() + } + } + impl ::hash::Hash for statfs { + fn hash(&self, state: &mut H) { + self.f_version.hash(state); + self.f_type.hash(state); + self.f_flags.hash(state); + self.f_bsize.hash(state); + self.f_iosize.hash(state); + self.f_blocks.hash(state); + self.f_bfree.hash(state); + self.f_bavail.hash(state); + self.f_files.hash(state); + self.f_ffree.hash(state); + self.f_syncwrites.hash(state); + self.f_asyncwrites.hash(state); + self.f_syncreads.hash(state); + self.f_asyncreads.hash(state); + self.f_namemax.hash(state); + self.f_owner.hash(state); + self.f_fsid.hash(state); + self.f_charspare.hash(state); + self.f_fstypename.hash(state); + self.f_mntfromname.hash(state); + self.f_mntonname.hash(state); + } + } + + impl PartialEq for dirent { + fn eq(&self, other: &dirent) -> bool { + self.d_fileno == other.d_fileno + && self.d_off == other.d_off + && self.d_reclen == other.d_reclen + && self.d_type == other.d_type + && self.d_namlen == other.d_namlen + && self + .d_name[..self.d_namlen as _] + .iter() + .zip(other.d_name.iter()) + .all(|(a,b)| a == b) + } + } + impl Eq for dirent {} + impl ::fmt::Debug for dirent { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("dirent") + .field("d_fileno", &self.d_fileno) + .field("d_off", &self.d_off) + .field("d_reclen", &self.d_reclen) + .field("d_type", &self.d_type) + .field("d_namlen", &self.d_namlen) + .field("d_name", &&self.d_name[..self.d_namlen as _]) + .finish() + } + } + impl ::hash::Hash for dirent { + fn hash(&self, state: &mut H) { + self.d_fileno.hash(state); + self.d_off.hash(state); + self.d_reclen.hash(state); + self.d_type.hash(state); + self.d_namlen.hash(state); + self.d_name[..self.d_namlen as _].hash(state); + } + } + + impl PartialEq for vnstat { + fn eq(&self, other: &vnstat) -> bool { + let self_vn_devname: &[::c_char] = &self.vn_devname; + let other_vn_devname: &[::c_char] = &other.vn_devname; + + self.vn_fileid == other.vn_fileid && + self.vn_size == other.vn_size && + self.vn_dev == other.vn_dev && + self.vn_fsid == other.vn_fsid && + self.vn_mntdir == other.vn_mntdir && + self.vn_type == other.vn_type && + self.vn_mode == other.vn_mode && + self_vn_devname == other_vn_devname + } + } + impl Eq for vnstat {} + impl ::fmt::Debug for vnstat { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + let self_vn_devname: &[::c_char] = &self.vn_devname; + + f.debug_struct("vnstat") + .field("vn_fileid", &self.vn_fileid) + .field("vn_size", &self.vn_size) + .field("vn_dev", &self.vn_dev) + .field("vn_fsid", &self.vn_fsid) + .field("vn_mntdir", &self.vn_mntdir) + .field("vn_type", &self.vn_type) + .field("vn_mode", &self.vn_mode) + .field("vn_devname", &self_vn_devname) + .finish() + } + } + impl ::hash::Hash for vnstat { + fn hash(&self, state: &mut H) { + let self_vn_devname: &[::c_char] = &self.vn_devname; + + self.vn_fileid.hash(state); + self.vn_size.hash(state); + self.vn_dev.hash(state); + self.vn_fsid.hash(state); + self.vn_mntdir.hash(state); + self.vn_type.hash(state); + self.vn_mode.hash(state); + self_vn_devname.hash(state); + } + } + } +} + +pub const RAND_MAX: ::c_int = 0x7fff_ffff; +pub const ELAST: ::c_int = 97; + +pub const KF_TYPE_EVENTFD: ::c_int = 13; + +/// max length of devicename +pub const SPECNAMELEN: ::c_int = 255; +pub const KI_NSPARE_PTR: usize = 5; + +/// domainset policies +pub const DOMAINSET_POLICY_INVALID: ::c_int = 0; +pub const DOMAINSET_POLICY_ROUNDROBIN: ::c_int = 1; +pub const DOMAINSET_POLICY_FIRSTTOUCH: ::c_int = 2; +pub const DOMAINSET_POLICY_PREFER: ::c_int = 3; +pub const DOMAINSET_POLICY_INTERLEAVE: ::c_int = 4; + +pub const MINCORE_SUPER: ::c_int = 0x60; + +safe_f! { + pub {const} fn makedev(major: ::c_uint, minor: ::c_uint) -> ::dev_t { + let major = major as ::dev_t; + let minor = minor as ::dev_t; + let mut dev = 0; + dev |= ((major & 0xffffff00) as dev_t) << 32; + dev |= ((major & 0x000000ff) as dev_t) << 8; + dev |= ((minor & 0x0000ff00) as dev_t) << 24; + dev |= ((minor & 0xffff00ff) as dev_t) << 0; + dev + } +} + +extern "C" { + pub fn setgrent(); + pub fn mprotect(addr: *mut ::c_void, len: ::size_t, prot: ::c_int) -> ::c_int; + pub fn freelocale(loc: ::locale_t); + pub fn msgrcv( + msqid: ::c_int, + msgp: *mut ::c_void, + msgsz: ::size_t, + msgtyp: ::c_long, + msgflg: ::c_int, + ) -> ::ssize_t; + pub fn clock_nanosleep( + clk_id: ::clockid_t, + flags: ::c_int, + rqtp: *const ::timespec, + rmtp: *mut ::timespec, + ) -> ::c_int; + + pub fn eventfd(init: ::c_uint, flags: ::c_int) -> ::c_int; + + pub fn fdatasync(fd: ::c_int) -> ::c_int; + + pub fn getrandom(buf: *mut ::c_void, buflen: ::size_t, flags: ::c_uint) -> ::ssize_t; + pub fn getentropy(buf: *mut ::c_void, buflen: ::size_t) -> ::c_int; + pub fn elf_aux_info(aux: ::c_int, buf: *mut ::c_void, buflen: ::c_int) -> ::c_int; + pub fn setproctitle_fast(fmt: *const ::c_char, ...); + pub fn timingsafe_bcmp(a: *const ::c_void, b: *const ::c_void, len: ::size_t) -> ::c_int; + pub fn timingsafe_memcmp(a: *const ::c_void, b: *const ::c_void, len: ::size_t) -> ::c_int; + + pub fn cpuset_getdomain( + level: ::cpulevel_t, + which: ::cpuwhich_t, + id: ::id_t, + setsize: ::size_t, + mask: *mut ::domainset_t, + policy: *mut ::c_int, + ) -> ::c_int; + pub fn cpuset_setdomain( + level: ::cpulevel_t, + which: ::cpuwhich_t, + id: ::id_t, + setsize: ::size_t, + mask: *const ::domainset_t, + policy: ::c_int, + ) -> ::c_int; + + pub fn dirname(path: *mut ::c_char) -> *mut ::c_char; + pub fn basename(path: *mut ::c_char) -> *mut ::c_char; +} + +#[link(name = "kvm")] +extern "C" { + pub fn kvm_kerndisp(kd: *mut ::kvm_t) -> ::kssize_t; +} + +cfg_if! { + if #[cfg(any(target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64"))] { + mod b64; + pub use self::b64::*; + } +} + +cfg_if! { + if #[cfg(target_arch = "x86_64")] { + mod x86_64; + pub use self::x86_64::*; + } +} diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd14/x86_64.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd14/x86_64.rs new file mode 100644 index 000000000..7bf253445 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/freebsd14/x86_64.rs @@ -0,0 +1,5 @@ +pub const PROC_KPTI_CTL: ::c_int = ::PROC_PROCCTL_MD_MIN; +pub const PROC_KPTI_CTL_ENABLE_ON_EXEC: ::c_int = 1; +pub const PROC_KPTI_CTL_DISABLE_ON_EXEC: ::c_int = 2; +pub const PROC_KPTI_STATUS: ::c_int = ::PROC_PROCCTL_MD_MIN + 1; +pub const PROC_KPTI_STATUS_ACTIVE: ::c_int = 0x80000000; diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/mod.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/mod.rs index 1b714b775..2a4fbbf05 100644 --- a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/mod.rs +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/mod.rs @@ -1,23 +1,237 @@ pub type fflags_t = u32; pub type clock_t = i32; -pub type lwpid_t = i32; +pub type vm_prot_t = u_char; +pub type kvaddr_t = u64; +pub type segsz_t = isize; +pub type __fixpt_t = u32; +pub type fixpt_t = __fixpt_t; +pub type __lwpid_t = i32; +pub type lwpid_t = __lwpid_t; pub type blksize_t = i32; pub type clockid_t = ::c_int; pub type sem_t = _sem; +pub type timer_t = *mut __c_anonymous__timer; pub type fsblkcnt_t = u64; pub type fsfilcnt_t = u64; pub type idtype_t = ::c_uint; -pub type key_t = ::c_long; pub type msglen_t = ::c_ulong; pub type msgqnum_t = ::c_ulong; +pub type cpulevel_t = ::c_int; +pub type cpuwhich_t = ::c_int; + pub type mqd_t = *mut ::c_void; pub type posix_spawnattr_t = *mut ::c_void; pub type posix_spawn_file_actions_t = *mut ::c_void; +pub type pthread_spinlock_t = *mut __c_anonymous_pthread_spinlock; +pub type pthread_barrierattr_t = *mut __c_anonymous_pthread_barrierattr; +pub type pthread_barrier_t = *mut __c_anonymous_pthread_barrier; + +pub type uuid_t = ::uuid; +pub type u_int = ::c_uint; +pub type u_char = ::c_uchar; +pub type u_long = ::c_ulong; +pub type u_short = ::c_ushort; + +pub type caddr_t = *mut ::c_char; + +pub type fhandle_t = fhandle; + +pub type au_id_t = ::uid_t; +pub type au_asid_t = ::pid_t; + +pub type cpusetid_t = ::c_int; + +#[cfg_attr(feature = "extra_traits", derive(Debug, Hash, PartialEq, Eq))] +#[repr(u32)] +pub enum devstat_support_flags { + DEVSTAT_ALL_SUPPORTED = 0x00, + DEVSTAT_NO_BLOCKSIZE = 0x01, + DEVSTAT_NO_ORDERED_TAGS = 0x02, + DEVSTAT_BS_UNAVAILABLE = 0x04, +} +impl ::Copy for devstat_support_flags {} +impl ::Clone for devstat_support_flags { + fn clone(&self) -> devstat_support_flags { + *self + } +} + +#[cfg_attr(feature = "extra_traits", derive(Debug, Hash, PartialEq, Eq))] +#[repr(u32)] +pub enum devstat_trans_flags { + DEVSTAT_NO_DATA = 0x00, + DEVSTAT_READ = 0x01, + DEVSTAT_WRITE = 0x02, + DEVSTAT_FREE = 0x03, +} + +impl ::Copy for devstat_trans_flags {} +impl ::Clone for devstat_trans_flags { + fn clone(&self) -> devstat_trans_flags { + *self + } +} + +#[cfg_attr(feature = "extra_traits", derive(Debug, Hash, PartialEq, Eq))] +#[repr(u32)] +pub enum devstat_tag_type { + DEVSTAT_TAG_SIMPLE = 0x00, + DEVSTAT_TAG_HEAD = 0x01, + DEVSTAT_TAG_ORDERED = 0x02, + DEVSTAT_TAG_NONE = 0x03, +} +impl ::Copy for devstat_tag_type {} +impl ::Clone for devstat_tag_type { + fn clone(&self) -> devstat_tag_type { + *self + } +} + +#[cfg_attr(feature = "extra_traits", derive(Debug, Hash, PartialEq, Eq))] +#[repr(u32)] +pub enum devstat_match_flags { + DEVSTAT_MATCH_NONE = 0x00, + DEVSTAT_MATCH_TYPE = 0x01, + DEVSTAT_MATCH_IF = 0x02, + DEVSTAT_MATCH_PASS = 0x04, +} +impl ::Copy for devstat_match_flags {} +impl ::Clone for devstat_match_flags { + fn clone(&self) -> devstat_match_flags { + *self + } +} + +#[cfg_attr(feature = "extra_traits", derive(Debug, Hash, PartialEq, Eq))] +#[repr(u32)] +pub enum devstat_priority { + DEVSTAT_PRIORITY_MIN = 0x000, + DEVSTAT_PRIORITY_OTHER = 0x020, + DEVSTAT_PRIORITY_PASS = 0x030, + DEVSTAT_PRIORITY_FD = 0x040, + DEVSTAT_PRIORITY_WFD = 0x050, + DEVSTAT_PRIORITY_TAPE = 0x060, + DEVSTAT_PRIORITY_CD = 0x090, + DEVSTAT_PRIORITY_DISK = 0x110, + DEVSTAT_PRIORITY_ARRAY = 0x120, + DEVSTAT_PRIORITY_MAX = 0xfff, +} +impl ::Copy for devstat_priority {} +impl ::Clone for devstat_priority { + fn clone(&self) -> devstat_priority { + *self + } +} + +#[cfg_attr(feature = "extra_traits", derive(Debug, Hash, PartialEq, Eq))] +#[repr(u32)] +pub enum devstat_type_flags { + DEVSTAT_TYPE_DIRECT = 0x000, + DEVSTAT_TYPE_SEQUENTIAL = 0x001, + DEVSTAT_TYPE_PRINTER = 0x002, + DEVSTAT_TYPE_PROCESSOR = 0x003, + DEVSTAT_TYPE_WORM = 0x004, + DEVSTAT_TYPE_CDROM = 0x005, + DEVSTAT_TYPE_SCANNER = 0x006, + DEVSTAT_TYPE_OPTICAL = 0x007, + DEVSTAT_TYPE_CHANGER = 0x008, + DEVSTAT_TYPE_COMM = 0x009, + DEVSTAT_TYPE_ASC0 = 0x00a, + DEVSTAT_TYPE_ASC1 = 0x00b, + DEVSTAT_TYPE_STORARRAY = 0x00c, + DEVSTAT_TYPE_ENCLOSURE = 0x00d, + DEVSTAT_TYPE_FLOPPY = 0x00e, + DEVSTAT_TYPE_MASK = 0x00f, + DEVSTAT_TYPE_IF_SCSI = 0x010, + DEVSTAT_TYPE_IF_IDE = 0x020, + DEVSTAT_TYPE_IF_OTHER = 0x030, + DEVSTAT_TYPE_IF_MASK = 0x0f0, + DEVSTAT_TYPE_PASS = 0x100, +} +impl ::Copy for devstat_type_flags {} +impl ::Clone for devstat_type_flags { + fn clone(&self) -> devstat_type_flags { + *self + } +} + +#[cfg_attr(feature = "extra_traits", derive(Debug, Hash, PartialEq, Eq))] +#[repr(u32)] +pub enum devstat_metric { + DSM_NONE, + DSM_TOTAL_BYTES, + DSM_TOTAL_BYTES_READ, + DSM_TOTAL_BYTES_WRITE, + DSM_TOTAL_TRANSFERS, + DSM_TOTAL_TRANSFERS_READ, + DSM_TOTAL_TRANSFERS_WRITE, + DSM_TOTAL_TRANSFERS_OTHER, + DSM_TOTAL_BLOCKS, + DSM_TOTAL_BLOCKS_READ, + DSM_TOTAL_BLOCKS_WRITE, + DSM_KB_PER_TRANSFER, + DSM_KB_PER_TRANSFER_READ, + DSM_KB_PER_TRANSFER_WRITE, + DSM_TRANSFERS_PER_SECOND, + DSM_TRANSFERS_PER_SECOND_READ, + DSM_TRANSFERS_PER_SECOND_WRITE, + DSM_TRANSFERS_PER_SECOND_OTHER, + DSM_MB_PER_SECOND, + DSM_MB_PER_SECOND_READ, + DSM_MB_PER_SECOND_WRITE, + DSM_BLOCKS_PER_SECOND, + DSM_BLOCKS_PER_SECOND_READ, + DSM_BLOCKS_PER_SECOND_WRITE, + DSM_MS_PER_TRANSACTION, + DSM_MS_PER_TRANSACTION_READ, + DSM_MS_PER_TRANSACTION_WRITE, + DSM_SKIP, + DSM_TOTAL_BYTES_FREE, + DSM_TOTAL_TRANSFERS_FREE, + DSM_TOTAL_BLOCKS_FREE, + DSM_KB_PER_TRANSFER_FREE, + DSM_MB_PER_SECOND_FREE, + DSM_TRANSFERS_PER_SECOND_FREE, + DSM_BLOCKS_PER_SECOND_FREE, + DSM_MS_PER_TRANSACTION_OTHER, + DSM_MS_PER_TRANSACTION_FREE, + DSM_BUSY_PCT, + DSM_QUEUE_LENGTH, + DSM_TOTAL_DURATION, + DSM_TOTAL_DURATION_READ, + DSM_TOTAL_DURATION_WRITE, + DSM_TOTAL_DURATION_FREE, + DSM_TOTAL_DURATION_OTHER, + DSM_TOTAL_BUSY_TIME, + DSM_MAX, +} +impl ::Copy for devstat_metric {} +impl ::Clone for devstat_metric { + fn clone(&self) -> devstat_metric { + *self + } +} + +#[cfg_attr(feature = "extra_traits", derive(Debug, Hash, PartialEq, Eq))] +#[repr(u32)] +pub enum devstat_select_mode { + DS_SELECT_ADD, + DS_SELECT_ONLY, + DS_SELECT_REMOVE, + DS_SELECT_ADDONLY, +} +impl ::Copy for devstat_select_mode {} +impl ::Clone for devstat_select_mode { + fn clone(&self) -> devstat_select_mode { + *self + } +} + s! { pub struct aiocb { pub aio_fildes: ::c_int, @@ -64,15 +278,10 @@ s! { pub struct _sem { data: [u32; 4], } - - pub struct ipc_perm { - pub cuid: ::uid_t, - pub cgid: ::gid_t, - pub uid: ::uid_t, - pub gid: ::gid_t, - pub mode: ::mode_t, - pub seq: ::c_ushort, - pub key: ::key_t, + pub struct sembuf { + pub sem_num: ::c_ushort, + pub sem_op: ::c_short, + pub sem_flg: ::c_short, } pub struct msqid_ds { @@ -89,14 +298,6 @@ s! { pub msg_ctime: ::time_t, } - pub struct xucred { - pub cr_version: ::c_uint, - pub cr_uid: ::uid_t, - pub cr_ngroups: ::c_short, - pub cr_groups: [::gid_t;16], - __cr_unused1: *mut ::c_void, - } - pub struct stack_t { pub ss_sp: *mut ::c_void, pub ss_size: ::size_t, @@ -107,6 +308,705 @@ s! { pub msg_hdr: ::msghdr, pub msg_len: ::ssize_t, } + + pub struct sockcred { + pub sc_uid: ::uid_t, + pub sc_euid: ::uid_t, + pub sc_gid: ::gid_t, + pub sc_egid: ::gid_t, + pub sc_ngroups: ::c_int, + pub sc_groups: [::gid_t; 1], + } + + pub struct ptrace_vm_entry { + pub pve_entry: ::c_int, + pub pve_timestamp: ::c_int, + pub pve_start: ::c_ulong, + pub pve_end: ::c_ulong, + pub pve_offset: ::c_ulong, + pub pve_prot: ::c_uint, + pub pve_pathlen: ::c_uint, + pub pve_fileid: ::c_long, + pub pve_fsid: u32, + pub pve_path: *mut ::c_char, + } + + pub struct ptrace_lwpinfo { + pub pl_lwpid: lwpid_t, + pub pl_event: ::c_int, + pub pl_flags: ::c_int, + pub pl_sigmask: ::sigset_t, + pub pl_siglist: ::sigset_t, + pub pl_siginfo: ::siginfo_t, + pub pl_tdname: [::c_char; ::MAXCOMLEN as usize + 1], + pub pl_child_pid: ::pid_t, + pub pl_syscall_code: ::c_uint, + pub pl_syscall_narg: ::c_uint, + } + + pub struct ptrace_sc_ret { + pub sr_retval: [::register_t; 2], + pub sr_error: ::c_int, + } + + pub struct ptrace_coredump { + pub pc_fd: ::c_int, + pub pc_flags: u32, + pub pc_limit: ::off_t, + } + + pub struct cpuset_t { + #[cfg(target_pointer_width = "64")] + __bits: [::c_long; 4], + #[cfg(target_pointer_width = "32")] + __bits: [::c_long; 8], + } + + pub struct cap_rights_t { + cr_rights: [u64; 2], + } + + pub struct umutex { + m_owner: ::lwpid_t, + m_flags: u32, + m_ceilings: [u32; 2], + m_rb_link: ::uintptr_t, + #[cfg(target_pointer_width = "32")] + m_pad: u32, + m_spare: [u32; 2], + + } + + pub struct ucond { + c_has_waiters: u32, + c_flags: u32, + c_clockid: u32, + c_spare: [u32; 1], + } + + pub struct uuid { + pub time_low: u32, + pub time_mid: u16, + pub time_hi_and_version: u16, + pub clock_seq_hi_and_reserved: u8, + pub clock_seq_low: u8, + pub node: [u8; _UUID_NODE_LEN], + } + + pub struct __c_anonymous_pthread_spinlock { + s_clock: umutex, + } + + pub struct __c_anonymous_pthread_barrierattr { + pshared: ::c_int, + } + + pub struct __c_anonymous_pthread_barrier { + b_lock: umutex, + b_cv: ucond, + b_cycle: i64, + b_count: ::c_int, + b_waiters: ::c_int, + b_refcount: ::c_int, + b_destroying: ::c_int, + } + + pub struct kinfo_vmentry { + pub kve_structsize: ::c_int, + pub kve_type: ::c_int, + pub kve_start: u64, + pub kve_end: u64, + pub kve_offset: u64, + pub kve_vn_fileid: u64, + #[cfg(not(freebsd11))] + pub kve_vn_fsid_freebsd11: u32, + #[cfg(freebsd11)] + pub kve_vn_fsid: u32, + pub kve_flags: ::c_int, + pub kve_resident: ::c_int, + pub kve_private_resident: ::c_int, + pub kve_protection: ::c_int, + pub kve_ref_count: ::c_int, + pub kve_shadow_count: ::c_int, + pub kve_vn_type: ::c_int, + pub kve_vn_size: u64, + #[cfg(not(freebsd11))] + pub kve_vn_rdev_freebsd11: u32, + #[cfg(freebsd11)] + pub kve_vn_rdev: u32, + pub kve_vn_mode: u16, + pub kve_status: u16, + #[cfg(not(freebsd11))] + pub kve_vn_fsid: u64, + #[cfg(not(freebsd11))] + pub kve_vn_rdev: u64, + #[cfg(not(freebsd11))] + _kve_is_spare: [::c_int; 8], + #[cfg(freebsd11)] + _kve_is_spare: [::c_int; 12], + pub kve_path: [[::c_char; 32]; 32], + } + + pub struct __c_anonymous_filestat { + pub stqe_next: *mut filestat, + } + + pub struct filestat { + pub fs_type: ::c_int, + pub fs_flags: ::c_int, + pub fs_fflags: ::c_int, + pub fs_uflags: ::c_int, + pub fs_fd: ::c_int, + pub fs_ref_count: ::c_int, + pub fs_offset: ::off_t, + pub fs_typedep: *mut ::c_void, + pub fs_path: *mut ::c_char, + pub next: __c_anonymous_filestat, + pub fs_cap_rights: cap_rights_t, + } + + pub struct filestat_list { + pub stqh_first: *mut filestat, + pub stqh_last: *mut *mut filestat, + } + + pub struct procstat { + pub tpe: ::c_int, + pub kd: ::uintptr_t, + pub vmentries: *mut ::c_void, + pub files: *mut ::c_void, + pub argv: *mut ::c_void, + pub envv: *mut ::c_void, + pub core: ::uintptr_t, + } + + pub struct itimerspec { + pub it_interval: ::timespec, + pub it_value: ::timespec, + } + + pub struct __c_anonymous__timer { + _priv: [::c_int; 3], + } + + /// Used to hold a copy of the command line, if it had a sane length. + pub struct pargs { + /// Reference count. + pub ar_ref: u_int, + /// Length. + pub ar_length: u_int, + /// Arguments. + pub ar_args: [::c_uchar; 1], + } + + pub struct priority { + /// Scheduling class. + pub pri_class: u_char, + /// Normal priority level. + pub pri_level: u_char, + /// Priority before propagation. + pub pri_native: u_char, + /// User priority based on p_cpu and p_nice. + pub pri_user: u_char, + } + + pub struct kvm_swap { + pub ksw_devname: [::c_char; 32], + pub ksw_used: u_int, + pub ksw_total: u_int, + pub ksw_flags: ::c_int, + pub ksw_reserved1: u_int, + pub ksw_reserved2: u_int, + } + + pub struct nlist { + /// symbol name (in memory) + pub n_name: *const ::c_char, + /// type defines + pub n_type: ::c_uchar, + /// "type" and binding information + pub n_other: ::c_char, + /// used by stab entries + pub n_desc: ::c_short, + pub n_value: ::c_ulong, + } + + pub struct kvm_nlist { + pub n_name: *const ::c_char, + pub n_type: ::c_uchar, + pub n_value: ::kvaddr_t, + } + + pub struct __c_anonymous_sem { + _priv: ::uintptr_t, + } + + pub struct semid_ds { + pub sem_perm: ::ipc_perm, + pub __sem_base: *mut __c_anonymous_sem, + pub sem_nsems: ::c_ushort, + pub sem_otime: ::time_t, + pub sem_ctime: ::time_t, + } + + pub struct vmtotal { + pub t_vm: u64, + pub t_avm: u64, + pub t_rm: u64, + pub t_arm: u64, + pub t_vmshr: u64, + pub t_avmshr: u64, + pub t_rmshr: u64, + pub t_armshr: u64, + pub t_free: u64, + pub t_rq: i16, + pub t_dw: i16, + pub t_pw: i16, + pub t_sl: i16, + pub t_sw: i16, + pub t_pad: [u16; 3], + } + + pub struct sockstat { + pub inp_ppcb: u64, + pub so_addr: u64, + pub so_pcb: u64, + pub unp_conn: u64, + pub dom_family: ::c_int, + pub proto: ::c_int, + pub so_rcv_sb_state: ::c_int, + pub so_snd_sb_state: ::c_int, + /// Socket address. + pub sa_local: ::sockaddr_storage, + /// Peer address. + pub sa_peer: ::sockaddr_storage, + pub type_: ::c_int, + pub dname: [::c_char; 32], + #[cfg(any(freebsd12, freebsd13, freebsd14))] + pub sendq: ::c_uint, + #[cfg(any(freebsd12, freebsd13, freebsd14))] + pub recvq: ::c_uint, + } + + pub struct shmstat { + pub size: u64, + pub mode: u16, + } + + pub struct spacectl_range { + pub r_offset: ::off_t, + pub r_len: ::off_t + } + + pub struct rusage_ext { + pub rux_runtime: u64, + pub rux_uticks: u64, + pub rux_sticks: u64, + pub rux_iticks: u64, + pub rux_uu: u64, + pub rux_su: u64, + pub rux_tu: u64, + } + + pub struct if_clonereq { + pub ifcr_total: ::c_int, + pub ifcr_count: ::c_int, + pub ifcr_buffer: *mut ::c_char, + } + + pub struct if_msghdr { + /// to skip over non-understood messages + pub ifm_msglen: ::c_ushort, + /// future binary compatibility + pub ifm_version: ::c_uchar, + /// message type + pub ifm_type: ::c_uchar, + /// like rtm_addrs + pub ifm_addrs: ::c_int, + /// value of if_flags + pub ifm_flags: ::c_int, + /// index for associated ifp + pub ifm_index: ::c_ushort, + pub _ifm_spare1: ::c_ushort, + /// statistics and other data about if + pub ifm_data: if_data, + } + + pub struct if_msghdrl { + /// to skip over non-understood messages + pub ifm_msglen: ::c_ushort, + /// future binary compatibility + pub ifm_version: ::c_uchar, + /// message type + pub ifm_type: ::c_uchar, + /// like rtm_addrs + pub ifm_addrs: ::c_int, + /// value of if_flags + pub ifm_flags: ::c_int, + /// index for associated ifp + pub ifm_index: ::c_ushort, + /// spare space to grow if_index, see if_var.h + pub _ifm_spare1: ::c_ushort, + /// length of if_msghdrl incl. if_data + pub ifm_len: ::c_ushort, + /// offset of if_data from beginning + pub ifm_data_off: ::c_ushort, + pub _ifm_spare2: ::c_int, + /// statistics and other data about if + pub ifm_data: if_data, + } + + pub struct ifa_msghdr { + /// to skip over non-understood messages + pub ifam_msglen: ::c_ushort, + /// future binary compatibility + pub ifam_version: ::c_uchar, + /// message type + pub ifam_type: ::c_uchar, + /// like rtm_addrs + pub ifam_addrs: ::c_int, + /// value of ifa_flags + pub ifam_flags: ::c_int, + /// index for associated ifp + pub ifam_index: ::c_ushort, + pub _ifam_spare1: ::c_ushort, + /// value of ifa_ifp->if_metric + pub ifam_metric: ::c_int, + } + + pub struct ifa_msghdrl { + /// to skip over non-understood messages + pub ifam_msglen: ::c_ushort, + /// future binary compatibility + pub ifam_version: ::c_uchar, + /// message type + pub ifam_type: ::c_uchar, + /// like rtm_addrs + pub ifam_addrs: ::c_int, + /// value of ifa_flags + pub ifam_flags: ::c_int, + /// index for associated ifp + pub ifam_index: ::c_ushort, + /// spare space to grow if_index, see if_var.h + pub _ifam_spare1: ::c_ushort, + /// length of ifa_msghdrl incl. if_data + pub ifam_len: ::c_ushort, + /// offset of if_data from beginning + pub ifam_data_off: ::c_ushort, + /// value of ifa_ifp->if_metric + pub ifam_metric: ::c_int, + /// statistics and other data about if or address + pub ifam_data: if_data, + } + + pub struct ifma_msghdr { + /// to skip over non-understood messages + pub ifmam_msglen: ::c_ushort, + /// future binary compatibility + pub ifmam_version: ::c_uchar, + /// message type + pub ifmam_type: ::c_uchar, + /// like rtm_addrs + pub ifmam_addrs: ::c_int, + /// value of ifa_flags + pub ifmam_flags: ::c_int, + /// index for associated ifp + pub ifmam_index: ::c_ushort, + pub _ifmam_spare1: ::c_ushort, + } + + pub struct if_announcemsghdr { + /// to skip over non-understood messages + pub ifan_msglen: ::c_ushort, + /// future binary compatibility + pub ifan_version: ::c_uchar, + /// message type + pub ifan_type: ::c_uchar, + /// index for associated ifp + pub ifan_index: ::c_ushort, + /// if name, e.g. "en0" + pub ifan_name: [::c_char; ::IFNAMSIZ as usize], + /// what type of announcement + pub ifan_what: ::c_ushort, + } + + pub struct ifreq_buffer { + pub length: ::size_t, + pub buffer: *mut ::c_void, + } + + pub struct ifaliasreq { + /// if name, e.g. "en0" + pub ifra_name: [::c_char; ::IFNAMSIZ as usize], + pub ifra_addr: ::sockaddr, + pub ifra_broadaddr: ::sockaddr, + pub ifra_mask: ::sockaddr, + pub ifra_vhid: ::c_int, + } + + /// 9.x compat + pub struct oifaliasreq { + /// if name, e.g. "en0" + pub ifra_name: [::c_char; ::IFNAMSIZ as usize], + pub ifra_addr: ::sockaddr, + pub ifra_broadaddr: ::sockaddr, + pub ifra_mask: ::sockaddr, + } + + pub struct ifmediareq { + /// if name, e.g. "en0" + pub ifm_name: [::c_char; ::IFNAMSIZ as usize], + /// current media options + pub ifm_current: ::c_int, + /// don't care mask + pub ifm_mask: ::c_int, + /// media status + pub ifm_status: ::c_int, + /// active options + pub ifm_active: ::c_int, + /// # entries in ifm_ulist array + pub ifm_count: ::c_int, + /// media words + pub ifm_ulist: *mut ::c_int, + } + + pub struct ifdrv { + /// if name, e.g. "en0" + pub ifd_name: [::c_char; ::IFNAMSIZ as usize], + pub ifd_cmd: ::c_ulong, + pub ifd_len: ::size_t, + pub ifd_data: *mut ::c_void, + } + + pub struct ifi2creq { + /// i2c address (0xA0, 0xA2) + pub dev_addr: u8, + /// read offset + pub offset: u8, + /// read length + pub len: u8, + pub spare0: u8, + pub spare1: u32, + /// read buffer + pub data: [u8; 8], + } + + pub struct ifrsshash { + /// if name, e.g. "en0" + pub ifrh_name: [::c_char; ::IFNAMSIZ as usize], + /// RSS_FUNC_ + pub ifrh_func: u8, + pub ifrh_spare0: u8, + pub ifrh_spare1: u16, + /// RSS_TYPE_ + pub ifrh_types: u32, + } + + pub struct ifmibdata { + /// name of interface + pub ifmd_name: [::c_char; ::IFNAMSIZ as usize], + /// number of promiscuous listeners + pub ifmd_pcount: ::c_int, + /// interface flags + pub ifmd_flags: ::c_int, + /// instantaneous length of send queue + pub ifmd_snd_len: ::c_int, + /// maximum length of send queue + pub ifmd_snd_maxlen: ::c_int, + /// number of drops in send queue + pub ifmd_snd_drops: ::c_int, + /// for future expansion + pub ifmd_filler: [::c_int; 4], + /// generic information and statistics + pub ifmd_data: if_data, + } + + pub struct ifmib_iso_8802_3 { + pub dot3StatsAlignmentErrors: u32, + pub dot3StatsFCSErrors: u32, + pub dot3StatsSingleCollisionFrames: u32, + pub dot3StatsMultipleCollisionFrames: u32, + pub dot3StatsSQETestErrors: u32, + pub dot3StatsDeferredTransmissions: u32, + pub dot3StatsLateCollisions: u32, + pub dot3StatsExcessiveCollisions: u32, + pub dot3StatsInternalMacTransmitErrors: u32, + pub dot3StatsCarrierSenseErrors: u32, + pub dot3StatsFrameTooLongs: u32, + pub dot3StatsInternalMacReceiveErrors: u32, + pub dot3StatsEtherChipSet: u32, + pub dot3StatsMissedFrames: u32, + pub dot3StatsCollFrequencies: [u32; 16], + pub dot3Compliance: u32, + } + + pub struct __c_anonymous_ph { + pub ph1: u64, + pub ph2: u64, + } + + pub struct fid { + pub fid_len: ::c_ushort, + pub fid_data0: ::c_ushort, + pub fid_data: [::c_char; ::MAXFIDSZ as usize], + } + + pub struct fhandle { + pub fh_fsid: ::fsid_t, + pub fh_fid: fid, + } + + pub struct bintime { + pub sec: ::time_t, + pub frac: u64, + } + + pub struct clockinfo { + /// clock frequency + pub hz: ::c_int, + /// micro-seconds per hz tick + pub tick: ::c_int, + pub spare: ::c_int, + /// statistics clock frequency + pub stathz: ::c_int, + /// profiling clock frequency + pub profhz: ::c_int, + } + + pub struct __c_anonymous_stailq_entry_devstat { + pub stqe_next: *mut devstat, + } + + pub struct devstat { + /// Update sequence + pub sequence0: ::u_int, + /// Allocated entry + pub allocated: ::c_int, + /// started ops + pub start_count: ::u_int, + /// completed ops + pub end_count: ::u_int, + /// busy time unaccounted for since this time + pub busy_from: bintime, + pub dev_links: __c_anonymous_stailq_entry_devstat, + /// Devstat device number. + pub device_number: u32, + pub device_name: [::c_char; DEVSTAT_NAME_LEN as usize], + pub unit_number: ::c_int, + pub bytes: [u64; DEVSTAT_N_TRANS_FLAGS as usize], + pub operations: [u64; DEVSTAT_N_TRANS_FLAGS as usize], + pub duration: [bintime; DEVSTAT_N_TRANS_FLAGS as usize], + pub busy_time: bintime, + /// Time the device was created. + pub creation_time: bintime, + /// Block size, bytes + pub block_size: u32, + /// The number of simple, ordered, and head of queue tags sent. + pub tag_types: [u64; 3], + /// Which statistics are supported by a given device. + pub flags: devstat_support_flags, + /// Device type + pub device_type: devstat_type_flags, + /// Controls list pos. + pub priority: devstat_priority, + /// Identification for GEOM nodes + pub id: *const ::c_void, + /// Update sequence + pub sequence1: ::u_int, + } + + pub struct devstat_match { + pub match_fields: devstat_match_flags, + pub device_type: devstat_type_flags, + pub num_match_categories: ::c_int, + } + + pub struct devstat_match_table { + pub match_str: *const ::c_char, + pub type_: devstat_type_flags, + pub match_field: devstat_match_flags, + } + + pub struct device_selection { + pub device_number: u32, + pub device_name: [::c_char; DEVSTAT_NAME_LEN as usize], + pub unit_number: ::c_int, + pub selected: ::c_int, + pub bytes: u64, + pub position: ::c_int, + } + + pub struct devinfo { + pub devices: *mut devstat, + pub mem_ptr: *mut u8, + pub generation: ::c_long, + pub numdevs: ::c_int, + } + + pub struct sockcred2 { + pub sc_version: ::c_int, + pub sc_pid: ::pid_t, + pub sc_uid: ::uid_t, + pub sc_euid: ::uid_t, + pub sc_gid: ::gid_t, + pub sc_egid: ::gid_t, + pub sc_ngroups: ::c_int, + pub sc_groups: [::gid_t; 1], + } + + pub struct ifconf { + pub ifc_len: ::c_int, + #[cfg(libc_union)] + pub ifc_ifcu: __c_anonymous_ifc_ifcu, + } + + pub struct au_mask_t { + pub am_success: ::c_uint, + pub am_failure: ::c_uint, + } + + pub struct au_tid_t { + pub port: u32, + pub machine: u32, + } + + pub struct auditinfo_t { + pub ai_auid: ::au_id_t, + pub ai_mask: ::au_mask_t, + pub ai_termid: au_tid_t, + pub ai_asid: ::au_asid_t, + } + + pub struct tcp_fastopen { + pub enable: ::c_int, + pub psk: [u8; ::TCP_FASTOPEN_PSK_LEN as usize], + } + + pub struct tcp_function_set { + pub function_set_name: [::c_char; ::TCP_FUNCTION_NAME_LEN_MAX as usize], + pub pcbcnt: u32, + } + + pub struct _umtx_time { + pub _timeout: ::timespec, + pub _flags: u32, + pub _clockid: u32, + } + + pub struct shm_largepage_conf { + pub psind: ::c_int, + pub alloc_policy: ::c_int, + __pad: [::c_int; 10], + } + + pub struct memory_type { + __priva: [::uintptr_t; 32], + __privb: [::uintptr_t; 26], + } + + pub struct memory_type_list { + __priv: [::uintptr_t; 2], + } } s_no_extra_traits! { @@ -121,6 +1021,23 @@ s_no_extra_traits! { pub __ut_spare: [::c_char; 64], } + #[cfg(libc_union)] + pub union __c_anonymous_cr_pid { + __cr_unused: *mut ::c_void, + pub cr_pid: ::pid_t, + } + + pub struct xucred { + pub cr_version: ::c_uint, + pub cr_uid: ::uid_t, + pub cr_ngroups: ::c_short, + pub cr_groups: [::gid_t; 16], + #[cfg(libc_union)] + pub cr_pid__c_anonymous_union: __c_anonymous_cr_pid, + #[cfg(not(libc_union))] + __cr_unused1: *mut ::c_void, + } + pub struct sockaddr_dl { pub sdl_len: ::c_uchar, pub sdl_family: ::c_uchar, @@ -151,6 +1068,155 @@ s_no_extra_traits! { __unused1: ::c_int, __unused2: [::c_long; 7] } + + pub struct ptsstat { + #[cfg(any(freebsd12, freebsd13, freebsd14))] + pub dev: u64, + #[cfg(not(any(freebsd12, freebsd13, freebsd14)))] + pub dev: u32, + pub devname: [::c_char; SPECNAMELEN as usize + 1], + } + + #[cfg(libc_union)] + pub union __c_anonymous_elf32_auxv_union { + pub a_val: ::c_int, + } + + pub struct Elf32_Auxinfo { + pub a_type: ::c_int, + #[cfg(libc_union)] + pub a_un: __c_anonymous_elf32_auxv_union, + } + + #[cfg(libc_union)] + pub union __c_anonymous_ifi_epoch { + pub tt: ::time_t, + pub ph: u64, + } + + #[cfg(libc_union)] + pub union __c_anonymous_ifi_lastchange { + pub tv: ::timeval, + pub ph: __c_anonymous_ph, + } + + pub struct if_data { + /// ethernet, tokenring, etc + pub ifi_type: u8, + /// e.g., AUI, Thinnet, 10base-T, etc + pub ifi_physical: u8, + /// media address length + pub ifi_addrlen: u8, + /// media header length + pub ifi_hdrlen: u8, + /// current link state + pub ifi_link_state: u8, + /// carp vhid + pub ifi_vhid: u8, + /// length of this data struct + pub ifi_datalen: u16, + /// maximum transmission unit + pub ifi_mtu: u32, + /// routing metric (external only) + pub ifi_metric: u32, + /// linespeed + pub ifi_baudrate: u64, + /// packets received on interface + pub ifi_ipackets: u64, + /// input errors on interface + pub ifi_ierrors: u64, + /// packets sent on interface + pub ifi_opackets: u64, + /// output errors on interface + pub ifi_oerrors: u64, + /// collisions on csma interfaces + pub ifi_collisions: u64, + /// total number of octets received + pub ifi_ibytes: u64, + /// total number of octets sent + pub ifi_obytes: u64, + /// packets received via multicast + pub ifi_imcasts: u64, + /// packets sent via multicast + pub ifi_omcasts: u64, + /// dropped on input + pub ifi_iqdrops: u64, + /// dropped on output + pub ifi_oqdrops: u64, + /// destined for unsupported protocol + pub ifi_noproto: u64, + /// HW offload capabilities, see IFCAP + pub ifi_hwassist: u64, + /// uptime at attach or stat reset + #[cfg(libc_union)] + pub __ifi_epoch: __c_anonymous_ifi_epoch, + /// uptime at attach or stat reset + #[cfg(not(libc_union))] + pub __ifi_epoch: u64, + /// time of last administrative change + #[cfg(libc_union)] + pub __ifi_lastchange: __c_anonymous_ifi_lastchange, + /// time of last administrative change + #[cfg(not(libc_union))] + pub __ifi_lastchange: ::timeval, + } + + #[cfg(libc_union)] + pub union __c_anonymous_ifr_ifru { + pub ifru_addr: ::sockaddr, + pub ifru_dstaddr: ::sockaddr, + pub ifru_broadaddr: ::sockaddr, + pub ifru_buffer: ifreq_buffer, + pub ifru_flags: [::c_short; 2], + pub ifru_index: ::c_short, + pub ifru_jid: ::c_int, + pub ifru_metric: ::c_int, + pub ifru_mtu: ::c_int, + pub ifru_phys: ::c_int, + pub ifru_media: ::c_int, + pub ifru_data: ::caddr_t, + pub ifru_cap: [::c_int; 2], + pub ifru_fib: ::c_uint, + pub ifru_vlan_pcp: ::c_uchar, + } + + pub struct ifreq { + /// if name, e.g. "en0" + pub ifr_name: [::c_char; ::IFNAMSIZ], + #[cfg(libc_union)] + pub ifr_ifru: __c_anonymous_ifr_ifru, + #[cfg(not(libc_union))] + pub ifr_ifru: ::sockaddr, + } + + #[cfg(libc_union)] + pub union __c_anonymous_ifc_ifcu { + pub ifcu_buf: ::caddr_t, + pub ifcu_req: *mut ifreq, + } + + pub struct ifstat { + /// if name, e.g. "en0" + pub ifs_name: [::c_char; ::IFNAMSIZ as usize], + pub ascii: [::c_char; ::IFSTATMAX as usize + 1], + } + + pub struct ifrsskey { + /// if name, e.g. "en0" + pub ifrk_name: [::c_char; ::IFNAMSIZ as usize], + /// RSS_FUNC_ + pub ifrk_func: u8, + pub ifrk_spare0: u8, + pub ifrk_keylen: u16, + pub ifrk_key: [u8; ::RSS_KEYLEN as usize], + } + + pub struct ifdownreason { + pub ifdr_name: [::c_char; ::IFNAMSIZ as usize], + pub ifdr_reason: u32, + pub ifdr_vendor: u32, + pub ifdr_msg: [::c_char; ::IFDR_MSG_SIZE as usize], + } } cfg_if! { @@ -203,6 +1269,73 @@ cfg_if! { } } + #[cfg(libc_union)] + impl PartialEq for __c_anonymous_cr_pid { + fn eq(&self, other: &__c_anonymous_cr_pid) -> bool { + unsafe { self.cr_pid == other.cr_pid} + } + } + #[cfg(libc_union)] + impl Eq for __c_anonymous_cr_pid {} + #[cfg(libc_union)] + impl ::fmt::Debug for __c_anonymous_cr_pid { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("cr_pid") + .field("cr_pid", unsafe { &self.cr_pid }) + .finish() + } + } + #[cfg(libc_union)] + impl ::hash::Hash for __c_anonymous_cr_pid { + fn hash(&self, state: &mut H) { + unsafe { self.cr_pid.hash(state) }; + } + } + + impl PartialEq for xucred { + fn eq(&self, other: &xucred) -> bool { + #[cfg(libc_union)] + let equal_cr_pid = self.cr_pid__c_anonymous_union + == other.cr_pid__c_anonymous_union; + #[cfg(not(libc_union))] + let equal_cr_pid = self.__cr_unused1 == other.__cr_unused1; + + self.cr_version == other.cr_version + && self.cr_uid == other.cr_uid + && self.cr_ngroups == other.cr_ngroups + && self.cr_groups == other.cr_groups + && equal_cr_pid + } + } + impl Eq for xucred {} + impl ::fmt::Debug for xucred { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + let mut struct_formatter = f.debug_struct("xucred"); + struct_formatter.field("cr_version", &self.cr_version); + struct_formatter.field("cr_uid", &self.cr_uid); + struct_formatter.field("cr_ngroups", &self.cr_ngroups); + struct_formatter.field("cr_groups", &self.cr_groups); + #[cfg(libc_union)] + struct_formatter.field( + "cr_pid__c_anonymous_union", + &self.cr_pid__c_anonymous_union + ); + struct_formatter.finish() + } + } + impl ::hash::Hash for xucred { + fn hash(&self, state: &mut H) { + self.cr_version.hash(state); + self.cr_uid.hash(state); + self.cr_ngroups.hash(state); + self.cr_groups.hash(state); + #[cfg(libc_union)] + self.cr_pid__c_anonymous_union.hash(state); + #[cfg(not(libc_union))] + self.__cr_unused1.hash(state); + } + } + impl PartialEq for sockaddr_dl { fn eq(&self, other: &sockaddr_dl) -> bool { self.sdl_len == other.sdl_len @@ -304,19 +1437,490 @@ cfg_if! { self.sigev_notify_thread_id.hash(state); } } + + impl PartialEq for ptsstat { + fn eq(&self, other: &ptsstat) -> bool { + let self_devname: &[::c_char] = &self.devname; + let other_devname: &[::c_char] = &other.devname; + + self.dev == other.dev && self_devname == other_devname + } + } + impl Eq for ptsstat {} + impl ::fmt::Debug for ptsstat { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + let self_devname: &[::c_char] = &self.devname; + + f.debug_struct("ptsstat") + .field("dev", &self.dev) + .field("devname", &self_devname) + .finish() + } + } + impl ::hash::Hash for ptsstat { + fn hash(&self, state: &mut H) { + let self_devname: &[::c_char] = &self.devname; + + self.dev.hash(state); + self_devname.hash(state); + } + } + + #[cfg(libc_union)] + impl PartialEq for __c_anonymous_elf32_auxv_union { + fn eq(&self, other: &__c_anonymous_elf32_auxv_union) -> bool { + unsafe { self.a_val == other.a_val} + } + } + #[cfg(libc_union)] + impl Eq for __c_anonymous_elf32_auxv_union {} + #[cfg(libc_union)] + impl ::fmt::Debug for __c_anonymous_elf32_auxv_union { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("a_val") + .field("a_val", unsafe { &self.a_val }) + .finish() + } + } + #[cfg(not(libc_union))] + impl PartialEq for Elf32_Auxinfo { + fn eq(&self, other: &Elf32_Auxinfo) -> bool { + self.a_type == other.a_type + } + } + #[cfg(libc_union)] + impl PartialEq for Elf32_Auxinfo { + fn eq(&self, other: &Elf32_Auxinfo) -> bool { + self.a_type == other.a_type + && self.a_un == other.a_un + } + } + impl Eq for Elf32_Auxinfo {} + #[cfg(not(libc_union))] + impl ::fmt::Debug for Elf32_Auxinfo { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("Elf32_Auxinfo") + .field("a_type", &self.a_type) + .finish() + } + } + #[cfg(libc_union)] + impl ::fmt::Debug for Elf32_Auxinfo { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("Elf32_Auxinfo") + .field("a_type", &self.a_type) + .field("a_un", &self.a_un) + .finish() + } + } + + #[cfg(libc_union)] + impl PartialEq for __c_anonymous_ifr_ifru { + fn eq(&self, other: &__c_anonymous_ifr_ifru) -> bool { + unsafe { + self.ifru_addr == other.ifru_addr && + self.ifru_dstaddr == other.ifru_dstaddr && + self.ifru_broadaddr == other.ifru_broadaddr && + self.ifru_buffer == other.ifru_buffer && + self.ifru_flags == other.ifru_flags && + self.ifru_index == other.ifru_index && + self.ifru_jid == other.ifru_jid && + self.ifru_metric == other.ifru_metric && + self.ifru_mtu == other.ifru_mtu && + self.ifru_phys == other.ifru_phys && + self.ifru_media == other.ifru_media && + self.ifru_data == other.ifru_data && + self.ifru_cap == other.ifru_cap && + self.ifru_fib == other.ifru_fib && + self.ifru_vlan_pcp == other.ifru_vlan_pcp + } + } + } + #[cfg(libc_union)] + impl Eq for __c_anonymous_ifr_ifru {} + #[cfg(libc_union)] + impl ::fmt::Debug for __c_anonymous_ifr_ifru { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("ifr_ifru") + .field("ifru_addr", unsafe { &self.ifru_addr }) + .field("ifru_dstaddr", unsafe { &self.ifru_dstaddr }) + .field("ifru_broadaddr", unsafe { &self.ifru_broadaddr }) + .field("ifru_buffer", unsafe { &self.ifru_buffer }) + .field("ifru_flags", unsafe { &self.ifru_flags }) + .field("ifru_index", unsafe { &self.ifru_index }) + .field("ifru_jid", unsafe { &self.ifru_jid }) + .field("ifru_metric", unsafe { &self.ifru_metric }) + .field("ifru_mtu", unsafe { &self.ifru_mtu }) + .field("ifru_phys", unsafe { &self.ifru_phys }) + .field("ifru_media", unsafe { &self.ifru_media }) + .field("ifru_data", unsafe { &self.ifru_data }) + .field("ifru_cap", unsafe { &self.ifru_cap }) + .field("ifru_fib", unsafe { &self.ifru_fib }) + .field("ifru_vlan_pcp", unsafe { &self.ifru_vlan_pcp }) + .finish() + } + } + #[cfg(libc_union)] + impl ::hash::Hash for __c_anonymous_ifr_ifru { + fn hash(&self, state: &mut H) { + unsafe { self.ifru_addr.hash(state) }; + unsafe { self.ifru_dstaddr.hash(state) }; + unsafe { self.ifru_broadaddr.hash(state) }; + unsafe { self.ifru_buffer.hash(state) }; + unsafe { self.ifru_flags.hash(state) }; + unsafe { self.ifru_index.hash(state) }; + unsafe { self.ifru_jid.hash(state) }; + unsafe { self.ifru_metric.hash(state) }; + unsafe { self.ifru_mtu.hash(state) }; + unsafe { self.ifru_phys.hash(state) }; + unsafe { self.ifru_media.hash(state) }; + unsafe { self.ifru_data.hash(state) }; + unsafe { self.ifru_cap.hash(state) }; + unsafe { self.ifru_fib.hash(state) }; + unsafe { self.ifru_vlan_pcp.hash(state) }; + } + } + + impl PartialEq for ifreq { + fn eq(&self, other: &ifreq) -> bool { + self.ifr_name == other.ifr_name && self.ifr_ifru == other.ifr_ifru + } + } + impl Eq for ifreq {} + impl ::fmt::Debug for ifreq { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("ifreq") + .field("ifr_name", &self.ifr_name) + .field("ifr_ifru", &self.ifr_ifru) + .finish() + } + } + impl ::hash::Hash for ifreq { + fn hash(&self, state: &mut H) { + self.ifr_name.hash(state); + self.ifr_ifru.hash(state); + } + } + + #[cfg(libc_union)] + impl Eq for __c_anonymous_ifc_ifcu {} + + #[cfg(libc_union)] + impl PartialEq for __c_anonymous_ifc_ifcu { + fn eq(&self, other: &__c_anonymous_ifc_ifcu) -> bool { + unsafe { + self.ifcu_buf == other.ifcu_buf && + self.ifcu_req == other.ifcu_req + } + } + } + + #[cfg(libc_union)] + impl ::fmt::Debug for __c_anonymous_ifc_ifcu { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("ifc_ifcu") + .field("ifcu_buf", unsafe { &self.ifcu_buf }) + .field("ifcu_req", unsafe { &self.ifcu_req }) + .finish() + } + } + + #[cfg(libc_union)] + impl ::hash::Hash for __c_anonymous_ifc_ifcu { + fn hash(&self, state: &mut H) { + unsafe { self.ifcu_buf.hash(state) }; + unsafe { self.ifcu_req.hash(state) }; + } + } + + impl PartialEq for ifstat { + fn eq(&self, other: &ifstat) -> bool { + let self_ascii: &[::c_char] = &self.ascii; + let other_ascii: &[::c_char] = &other.ascii; + + self.ifs_name == other.ifs_name && self_ascii == other_ascii + } + } + impl Eq for ifstat {} + impl ::fmt::Debug for ifstat { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + let ascii: &[::c_char] = &self.ascii; + + f.debug_struct("ifstat") + .field("ifs_name", &self.ifs_name) + .field("ascii", &ascii) + .finish() + } + } + impl ::hash::Hash for ifstat { + fn hash(&self, state: &mut H) { + self.ifs_name.hash(state); + self.ascii.hash(state); + } + } + + impl PartialEq for ifrsskey { + fn eq(&self, other: &ifrsskey) -> bool { + let self_ifrk_key: &[u8] = &self.ifrk_key; + let other_ifrk_key: &[u8] = &other.ifrk_key; + + self.ifrk_name == other.ifrk_name && + self.ifrk_func == other.ifrk_func && + self.ifrk_spare0 == other.ifrk_spare0 && + self.ifrk_keylen == other.ifrk_keylen && + self_ifrk_key == other_ifrk_key + } + } + impl Eq for ifrsskey {} + impl ::fmt::Debug for ifrsskey { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + let ifrk_key: &[u8] = &self.ifrk_key; + + f.debug_struct("ifrsskey") + .field("ifrk_name", &self.ifrk_name) + .field("ifrk_func", &self.ifrk_func) + .field("ifrk_spare0", &self.ifrk_spare0) + .field("ifrk_keylen", &self.ifrk_keylen) + .field("ifrk_key", &ifrk_key) + .finish() + } + } + impl ::hash::Hash for ifrsskey { + fn hash(&self, state: &mut H) { + self.ifrk_name.hash(state); + self.ifrk_func.hash(state); + self.ifrk_spare0.hash(state); + self.ifrk_keylen.hash(state); + self.ifrk_key.hash(state); + } + } + + impl PartialEq for ifdownreason { + fn eq(&self, other: &ifdownreason) -> bool { + let self_ifdr_msg: &[::c_char] = &self.ifdr_msg; + let other_ifdr_msg: &[::c_char] = &other.ifdr_msg; + + self.ifdr_name == other.ifdr_name && + self.ifdr_reason == other.ifdr_reason && + self.ifdr_vendor == other.ifdr_vendor && + self_ifdr_msg == other_ifdr_msg + } + } + impl Eq for ifdownreason {} + impl ::fmt::Debug for ifdownreason { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + let ifdr_msg: &[::c_char] = &self.ifdr_msg; + + f.debug_struct("ifdownreason") + .field("ifdr_name", &self.ifdr_name) + .field("ifdr_reason", &self.ifdr_reason) + .field("ifdr_vendor", &self.ifdr_vendor) + .field("ifdr_msg", &ifdr_msg) + .finish() + } + } + impl ::hash::Hash for ifdownreason { + fn hash(&self, state: &mut H) { + self.ifdr_name.hash(state); + self.ifdr_reason.hash(state); + self.ifdr_vendor.hash(state); + self.ifdr_msg.hash(state); + } + } + + #[cfg(libc_union)] + impl PartialEq for __c_anonymous_ifi_epoch { + fn eq(&self, other: &__c_anonymous_ifi_epoch) -> bool { + unsafe { + self.tt == other.tt && + self.ph == other.ph + } + } + } + #[cfg(libc_union)] + impl Eq for __c_anonymous_ifi_epoch {} + #[cfg(libc_union)] + impl ::fmt::Debug for __c_anonymous_ifi_epoch { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("__c_anonymous_ifi_epoch") + .field("tt", unsafe { &self.tt }) + .field("ph", unsafe { &self.ph }) + .finish() + } + } + #[cfg(libc_union)] + impl ::hash::Hash for __c_anonymous_ifi_epoch { + fn hash(&self, state: &mut H) { + unsafe { + self.tt.hash(state); + self.ph.hash(state); + } + } + } + + #[cfg(libc_union)] + impl PartialEq for __c_anonymous_ifi_lastchange { + fn eq(&self, other: &__c_anonymous_ifi_lastchange) -> bool { + unsafe { + self.tv == other.tv && + self.ph == other.ph + } + } + } + #[cfg(libc_union)] + impl Eq for __c_anonymous_ifi_lastchange {} + #[cfg(libc_union)] + impl ::fmt::Debug for __c_anonymous_ifi_lastchange { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("__c_anonymous_ifi_lastchange") + .field("tv", unsafe { &self.tv }) + .field("ph", unsafe { &self.ph }) + .finish() + } + } + #[cfg(libc_union)] + impl ::hash::Hash for __c_anonymous_ifi_lastchange { + fn hash(&self, state: &mut H) { + unsafe { + self.tv.hash(state); + self.ph.hash(state); + } + } + } + + impl PartialEq for if_data { + fn eq(&self, other: &if_data) -> bool { + self.ifi_type == other.ifi_type && + self.ifi_physical == other.ifi_physical && + self.ifi_addrlen == other.ifi_addrlen && + self.ifi_hdrlen == other.ifi_hdrlen && + self.ifi_link_state == other.ifi_link_state && + self.ifi_vhid == other.ifi_vhid && + self.ifi_datalen == other.ifi_datalen && + self.ifi_mtu == other.ifi_mtu && + self.ifi_metric == other.ifi_metric && + self.ifi_baudrate == other.ifi_baudrate && + self.ifi_ipackets == other.ifi_ipackets && + self.ifi_ierrors == other.ifi_ierrors && + self.ifi_opackets == other.ifi_opackets && + self.ifi_oerrors == other.ifi_oerrors && + self.ifi_collisions == other.ifi_collisions && + self.ifi_ibytes == other.ifi_ibytes && + self.ifi_obytes == other.ifi_obytes && + self.ifi_imcasts == other.ifi_imcasts && + self.ifi_omcasts == other.ifi_omcasts && + self.ifi_iqdrops == other.ifi_iqdrops && + self.ifi_oqdrops == other.ifi_oqdrops && + self.ifi_noproto == other.ifi_noproto && + self.ifi_hwassist == other.ifi_hwassist && + self.__ifi_epoch == other.__ifi_epoch && + self.__ifi_lastchange == other.__ifi_lastchange + } + } + impl Eq for if_data {} + impl ::fmt::Debug for if_data { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("if_data") + .field("ifi_type", &self.ifi_type) + .field("ifi_physical", &self.ifi_physical) + .field("ifi_addrlen", &self.ifi_addrlen) + .field("ifi_hdrlen", &self.ifi_hdrlen) + .field("ifi_link_state", &self.ifi_link_state) + .field("ifi_vhid", &self.ifi_vhid) + .field("ifi_datalen", &self.ifi_datalen) + .field("ifi_mtu", &self.ifi_mtu) + .field("ifi_metric", &self.ifi_metric) + .field("ifi_baudrate", &self.ifi_baudrate) + .field("ifi_ipackets", &self.ifi_ipackets) + .field("ifi_ierrors", &self.ifi_ierrors) + .field("ifi_opackets", &self.ifi_opackets) + .field("ifi_oerrors", &self.ifi_oerrors) + .field("ifi_collisions", &self.ifi_collisions) + .field("ifi_ibytes", &self.ifi_ibytes) + .field("ifi_obytes", &self.ifi_obytes) + .field("ifi_imcasts", &self.ifi_imcasts) + .field("ifi_omcasts", &self.ifi_omcasts) + .field("ifi_iqdrops", &self.ifi_iqdrops) + .field("ifi_oqdrops", &self.ifi_oqdrops) + .field("ifi_noproto", &self.ifi_noproto) + .field("ifi_hwassist", &self.ifi_hwassist) + .field("__ifi_epoch", &self.__ifi_epoch) + .field("__ifi_lastchange", &self.__ifi_lastchange) + .finish() + } + } + impl ::hash::Hash for if_data { + fn hash(&self, state: &mut H) { + self.ifi_type.hash(state); + self.ifi_physical.hash(state); + self.ifi_addrlen.hash(state); + self.ifi_hdrlen.hash(state); + self.ifi_link_state.hash(state); + self.ifi_vhid.hash(state); + self.ifi_datalen.hash(state); + self.ifi_mtu.hash(state); + self.ifi_metric.hash(state); + self.ifi_baudrate.hash(state); + self.ifi_ipackets.hash(state); + self.ifi_ierrors.hash(state); + self.ifi_opackets.hash(state); + self.ifi_oerrors.hash(state); + self.ifi_collisions.hash(state); + self.ifi_ibytes.hash(state); + self.ifi_obytes.hash(state); + self.ifi_imcasts.hash(state); + self.ifi_omcasts.hash(state); + self.ifi_iqdrops.hash(state); + self.ifi_oqdrops.hash(state); + self.ifi_noproto.hash(state); + self.ifi_hwassist.hash(state); + self.__ifi_epoch.hash(state); + self.__ifi_lastchange.hash(state); + } + } + } +} + +#[cfg_attr(feature = "extra_traits", derive(Debug))] +#[repr(u32)] +pub enum dot3Vendors { + dot3VendorAMD = 1, + dot3VendorIntel = 2, + dot3VendorNational = 4, + dot3VendorFujitsu = 5, + dot3VendorDigital = 6, + dot3VendorWesternDigital = 7, +} +impl ::Copy for dot3Vendors {} +impl ::Clone for dot3Vendors { + fn clone(&self) -> dot3Vendors { + *self } } +// aio.h +pub const LIO_VECTORED: ::c_int = 4; +pub const LIO_WRITEV: ::c_int = 5; +pub const LIO_READV: ::c_int = 6; + +// sys/devicestat.h +pub const DEVSTAT_N_TRANS_FLAGS: ::c_int = 4; +pub const DEVSTAT_NAME_LEN: ::c_int = 16; + +// sys/cpuset.h +pub const CPU_SETSIZE: ::c_int = 256; + pub const SIGEV_THREAD_ID: ::c_int = 4; pub const EXTATTR_NAMESPACE_EMPTY: ::c_int = 0; pub const EXTATTR_NAMESPACE_USER: ::c_int = 1; pub const EXTATTR_NAMESPACE_SYSTEM: ::c_int = 2; -pub const RAND_MAX: ::c_int = 0x7fff_fffd; -pub const PTHREAD_STACK_MIN: ::size_t = 2048; +pub const PTHREAD_STACK_MIN: ::size_t = MINSIGSTKSZ; pub const PTHREAD_MUTEX_ADAPTIVE_NP: ::c_int = 4; -pub const SIGSTKSZ: ::size_t = 34816; +pub const PTHREAD_MUTEX_STALLED: ::c_int = 0; +pub const PTHREAD_MUTEX_ROBUST: ::c_int = 1; +pub const SIGSTKSZ: ::size_t = MINSIGSTKSZ + 32768; pub const SF_NODISKIO: ::c_int = 0x00000001; pub const SF_MNOWAIT: ::c_int = 0x00000002; pub const SF_SYNC: ::c_int = 0x00000004; @@ -333,16 +1937,40 @@ pub const ENOTCAPABLE: ::c_int = 93; pub const ECAPMODE: ::c_int = 94; pub const ENOTRECOVERABLE: ::c_int = 95; pub const EOWNERDEAD: ::c_int = 96; +pub const EINTEGRITY: ::c_int = 97; pub const RLIMIT_NPTS: ::c_int = 11; pub const RLIMIT_SWAP: ::c_int = 12; pub const RLIMIT_KQUEUES: ::c_int = 13; pub const RLIMIT_UMTXP: ::c_int = 14; #[deprecated(since = "0.2.64", note = "Not stable across OS versions")] pub const RLIM_NLIMITS: ::rlim_t = 15; +pub const RLIM_SAVED_MAX: ::rlim_t = ::RLIM_INFINITY; +pub const RLIM_SAVED_CUR: ::rlim_t = ::RLIM_INFINITY; + +pub const CP_USER: ::c_int = 0; +pub const CP_NICE: ::c_int = 1; +pub const CP_SYS: ::c_int = 2; +pub const CP_INTR: ::c_int = 3; +pub const CP_IDLE: ::c_int = 4; +pub const CPUSTATES: ::c_int = 5; + +pub const NI_NOFQDN: ::c_int = 0x00000001; +pub const NI_NUMERICHOST: ::c_int = 0x00000002; +pub const NI_NAMEREQD: ::c_int = 0x00000004; +pub const NI_NUMERICSERV: ::c_int = 0x00000008; +pub const NI_DGRAM: ::c_int = 0x00000010; +pub const NI_NUMERICSCOPE: ::c_int = 0x00000020; + +pub const XU_NGROUPS: ::c_int = 16; pub const Q_GETQUOTA: ::c_int = 0x700; pub const Q_SETQUOTA: ::c_int = 0x800; +pub const MAP_GUARD: ::c_int = 0x00002000; +pub const MAP_EXCL: ::c_int = 0x00004000; +pub const MAP_PREFAULT_READ: ::c_int = 0x00040000; +pub const MAP_ALIGNED_SUPER: ::c_int = 1 << 24; + pub const POSIX_FADV_NORMAL: ::c_int = 0; pub const POSIX_FADV_RANDOM: ::c_int = 1; pub const POSIX_FADV_SEQUENTIAL: ::c_int = 2; @@ -409,24 +2037,14 @@ pub const NOTE_USECONDS: u32 = 0x00000004; pub const NOTE_NSECONDS: u32 = 0x00000008; pub const MADV_PROTECT: ::c_int = 10; -pub const RUSAGE_THREAD: ::c_int = 1; - -pub const CLOCK_REALTIME: ::clockid_t = 0; -pub const CLOCK_VIRTUAL: ::clockid_t = 1; -pub const CLOCK_PROF: ::clockid_t = 2; -pub const CLOCK_MONOTONIC: ::clockid_t = 4; -pub const CLOCK_UPTIME: ::clockid_t = 5; -pub const CLOCK_UPTIME_PRECISE: ::clockid_t = 7; -pub const CLOCK_UPTIME_FAST: ::clockid_t = 8; -pub const CLOCK_REALTIME_PRECISE: ::clockid_t = 9; -pub const CLOCK_REALTIME_FAST: ::clockid_t = 10; -pub const CLOCK_MONOTONIC_PRECISE: ::clockid_t = 11; -pub const CLOCK_MONOTONIC_FAST: ::clockid_t = 12; -pub const CLOCK_SECOND: ::clockid_t = 13; -pub const CLOCK_THREAD_CPUTIME_ID: ::clockid_t = 14; -pub const CLOCK_PROCESS_CPUTIME_ID: ::clockid_t = 15; +#[doc(hidden)] +#[deprecated( + since = "0.2.72", + note = "CTL_UNSPEC is deprecated. Use CTL_SYSCTL instead" +)] pub const CTL_UNSPEC: ::c_int = 0; +pub const CTL_SYSCTL: ::c_int = 0; pub const CTL_KERN: ::c_int = 1; pub const CTL_VM: ::c_int = 2; pub const CTL_VFS: ::c_int = 3; @@ -436,6 +2054,67 @@ pub const CTL_HW: ::c_int = 6; pub const CTL_MACHDEP: ::c_int = 7; pub const CTL_USER: ::c_int = 8; pub const CTL_P1003_1B: ::c_int = 9; + +// sys/sysctl.h +pub const CTL_MAXNAME: ::c_int = 24; + +pub const CTLTYPE: ::c_int = 0xf; +pub const CTLTYPE_NODE: ::c_int = 1; +pub const CTLTYPE_INT: ::c_int = 2; +pub const CTLTYPE_STRING: ::c_int = 3; +pub const CTLTYPE_S64: ::c_int = 4; +pub const CTLTYPE_OPAQUE: ::c_int = 5; +pub const CTLTYPE_STRUCT: ::c_int = CTLTYPE_OPAQUE; +pub const CTLTYPE_UINT: ::c_int = 6; +pub const CTLTYPE_LONG: ::c_int = 7; +pub const CTLTYPE_ULONG: ::c_int = 8; +pub const CTLTYPE_U64: ::c_int = 9; +pub const CTLTYPE_U8: ::c_int = 0xa; +pub const CTLTYPE_U16: ::c_int = 0xb; +pub const CTLTYPE_S8: ::c_int = 0xc; +pub const CTLTYPE_S16: ::c_int = 0xd; +pub const CTLTYPE_S32: ::c_int = 0xe; +pub const CTLTYPE_U32: ::c_int = 0xf; + +pub const CTLFLAG_RD: ::c_int = 0x80000000; +pub const CTLFLAG_WR: ::c_int = 0x40000000; +pub const CTLFLAG_RW: ::c_int = CTLFLAG_RD | CTLFLAG_WR; +pub const CTLFLAG_DORMANT: ::c_int = 0x20000000; +pub const CTLFLAG_ANYBODY: ::c_int = 0x10000000; +pub const CTLFLAG_SECURE: ::c_int = 0x08000000; +pub const CTLFLAG_PRISON: ::c_int = 0x04000000; +pub const CTLFLAG_DYN: ::c_int = 0x02000000; +pub const CTLFLAG_SKIP: ::c_int = 0x01000000; +pub const CTLMASK_SECURE: ::c_int = 0x00F00000; +pub const CTLFLAG_TUN: ::c_int = 0x00080000; +pub const CTLFLAG_RDTUN: ::c_int = CTLFLAG_RD | CTLFLAG_TUN; +pub const CTLFLAG_RWTUN: ::c_int = CTLFLAG_RW | CTLFLAG_TUN; +pub const CTLFLAG_MPSAFE: ::c_int = 0x00040000; +pub const CTLFLAG_VNET: ::c_int = 0x00020000; +pub const CTLFLAG_DYING: ::c_int = 0x00010000; +pub const CTLFLAG_CAPRD: ::c_int = 0x00008000; +pub const CTLFLAG_CAPWR: ::c_int = 0x00004000; +pub const CTLFLAG_STATS: ::c_int = 0x00002000; +pub const CTLFLAG_NOFETCH: ::c_int = 0x00001000; +pub const CTLFLAG_CAPRW: ::c_int = CTLFLAG_CAPRD | CTLFLAG_CAPWR; +pub const CTLFLAG_NEEDGIANT: ::c_int = 0x00000800; + +pub const CTLSHIFT_SECURE: ::c_int = 20; +pub const CTLFLAG_SECURE1: ::c_int = CTLFLAG_SECURE | (0 << CTLSHIFT_SECURE); +pub const CTLFLAG_SECURE2: ::c_int = CTLFLAG_SECURE | (1 << CTLSHIFT_SECURE); +pub const CTLFLAG_SECURE3: ::c_int = CTLFLAG_SECURE | (2 << CTLSHIFT_SECURE); + +pub const OID_AUTO: ::c_int = -1; + +pub const CTL_SYSCTL_DEBUG: ::c_int = 0; +pub const CTL_SYSCTL_NAME: ::c_int = 1; +pub const CTL_SYSCTL_NEXT: ::c_int = 2; +pub const CTL_SYSCTL_NAME2OID: ::c_int = 3; +pub const CTL_SYSCTL_OIDFMT: ::c_int = 4; +pub const CTL_SYSCTL_OIDDESCR: ::c_int = 5; +pub const CTL_SYSCTL_OIDLABEL: ::c_int = 6; +pub const CTL_SYSCTL_NEXTNOSKIP: ::c_int = 7; + pub const KERN_OSTYPE: ::c_int = 1; pub const KERN_OSRELEASE: ::c_int = 2; pub const KERN_OSREV: ::c_int = 3; @@ -473,6 +2152,8 @@ pub const KERN_LOGSIGEXIT: ::c_int = 34; pub const KERN_IOV_MAX: ::c_int = 35; pub const KERN_HOSTUUID: ::c_int = 36; pub const KERN_ARND: ::c_int = 37; +pub const KERN_MAXPHYS: ::c_int = 38; + pub const KERN_PROC_ALL: ::c_int = 0; pub const KERN_PROC_PID: ::c_int = 1; pub const KERN_PROC_PGRP: ::c_int = 2; @@ -500,6 +2181,10 @@ pub const KERN_PROC_PS_STRINGS: ::c_int = 38; pub const KERN_PROC_UMASK: ::c_int = 39; pub const KERN_PROC_OSREL: ::c_int = 40; pub const KERN_PROC_SIGTRAMP: ::c_int = 41; +pub const KERN_PROC_CWD: ::c_int = 42; +pub const KERN_PROC_NFDS: ::c_int = 43; +pub const KERN_PROC_SIGFASTBLK: ::c_int = 44; + pub const KIPC_MAXSOCKBUF: ::c_int = 1; pub const KIPC_SOCKBUF_WASTE: ::c_int = 2; pub const KIPC_SOMAXCONN: ::c_int = 3; @@ -507,6 +2192,7 @@ pub const KIPC_MAX_LINKHDR: ::c_int = 4; pub const KIPC_MAX_PROTOHDR: ::c_int = 5; pub const KIPC_MAX_HDR: ::c_int = 6; pub const KIPC_MAX_DATALEN: ::c_int = 7; + pub const HW_MACHINE: ::c_int = 1; pub const HW_MODEL: ::c_int = 2; pub const HW_NCPU: ::c_int = 3; @@ -519,6 +2205,7 @@ pub const HW_DISKSTATS: ::c_int = 9; pub const HW_FLOATINGPT: ::c_int = 10; pub const HW_MACHINE_ARCH: ::c_int = 11; pub const HW_REALMEM: ::c_int = 12; + pub const USER_CS_PATH: ::c_int = 1; pub const USER_BC_BASE_MAX: ::c_int = 2; pub const USER_BC_DIM_MAX: ::c_int = 3; @@ -539,6 +2226,8 @@ pub const USER_POSIX2_SW_DEV: ::c_int = 17; pub const USER_POSIX2_UPE: ::c_int = 18; pub const USER_STREAM_MAX: ::c_int = 19; pub const USER_TZNAME_MAX: ::c_int = 20; +pub const USER_LOCALBASE: ::c_int = 21; + pub const CTL_P1003_1B_ASYNCHRONOUS_IO: ::c_int = 1; pub const CTL_P1003_1B_MAPPED_FILES: ::c_int = 2; pub const CTL_P1003_1B_MEMLOCK: ::c_int = 3; @@ -564,18 +2253,23 @@ pub const CTL_P1003_1B_SEM_NSEMS_MAX: ::c_int = 22; pub const CTL_P1003_1B_SEM_VALUE_MAX: ::c_int = 23; pub const CTL_P1003_1B_SIGQUEUE_MAX: ::c_int = 24; pub const CTL_P1003_1B_TIMER_MAX: ::c_int = 25; -pub const TIOCGPTN: ::c_uint = 0x4004740f; -pub const TIOCPTMASTER: ::c_uint = 0x2000741c; -pub const TIOCSIG: ::c_uint = 0x2004745f; + +pub const TIOCGPTN: ::c_ulong = 0x4004740f; +pub const TIOCPTMASTER: ::c_ulong = 0x2000741c; +pub const TIOCSIG: ::c_ulong = 0x2004745f; pub const TIOCM_DCD: ::c_int = 0x40; pub const H4DISC: ::c_int = 0x7; +pub const VM_TOTAL: ::c_int = 1; + pub const BIOCSETFNR: ::c_ulong = 0x80104282; +pub const FIODGNAME: ::c_ulong = 0x80106678; pub const FIONWRITE: ::c_ulong = 0x40046677; pub const FIONSPACE: ::c_ulong = 0x40046676; pub const FIOSEEKDATA: ::c_ulong = 0xc0086661; pub const FIOSEEKHOLE: ::c_ulong = 0xc0086662; +pub const FIOSSHMLPGCNF: ::c_ulong = 0x80306664; pub const JAIL_API_VERSION: u32 = 2; pub const JAIL_CREATE: ::c_int = 0x01; @@ -588,6 +2282,17 @@ pub const JAIL_SYS_DISABLE: ::c_int = 0; pub const JAIL_SYS_NEW: ::c_int = 1; pub const JAIL_SYS_INHERIT: ::c_int = 2; +pub const MNT_ACLS: ::c_int = 0x08000000; +pub const MNT_BYFSID: ::c_int = 0x08000000; +pub const MNT_GJOURNAL: ::c_int = 0x02000000; +pub const MNT_MULTILABEL: ::c_int = 0x04000000; +pub const MNT_NFS4ACLS: ::c_int = 0x00000010; +pub const MNT_SNAPSHOT: ::c_int = 0x01000000; +pub const MNT_UNION: ::c_int = 0x00000020; +pub const MNT_NONBUSY: ::c_int = 0x04000000; + +pub const SCM_CREDS2: ::c_int = 0x08; + pub const SO_BINTIME: ::c_int = 0x2000; pub const SO_NO_OFFLOAD: ::c_int = 0x4000; pub const SO_NO_DDP: ::c_int = 0x8000; @@ -601,13 +2306,29 @@ pub const SO_SETFIB: ::c_int = 0x1014; pub const SO_USER_COOKIE: ::c_int = 0x1015; pub const SO_PROTOCOL: ::c_int = 0x1016; pub const SO_PROTOTYPE: ::c_int = SO_PROTOCOL; +pub const SO_DOMAIN: ::c_int = 0x1019; pub const SO_VENDOR: ::c_int = 0x80000000; -pub const LOCAL_PEERCRED: ::c_int = 1; pub const LOCAL_CREDS: ::c_int = 2; +pub const LOCAL_CREDS_PERSISTENT: ::c_int = 3; pub const LOCAL_CONNWAIT: ::c_int = 4; pub const LOCAL_VENDOR: ::c_int = SO_VENDOR; +pub const PL_EVENT_NONE: ::c_int = 0; +pub const PL_EVENT_SIGNAL: ::c_int = 1; +pub const PL_FLAG_SA: ::c_int = 0x01; +pub const PL_FLAG_BOUND: ::c_int = 0x02; +pub const PL_FLAG_SCE: ::c_int = 0x04; +pub const PL_FLAG_SCX: ::c_int = 0x08; +pub const PL_FLAG_EXEC: ::c_int = 0x10; +pub const PL_FLAG_SI: ::c_int = 0x20; +pub const PL_FLAG_FORKED: ::c_int = 0x40; +pub const PL_FLAG_CHILD: ::c_int = 0x80; +pub const PL_FLAG_BORN: ::c_int = 0x100; +pub const PL_FLAG_EXITED: ::c_int = 0x200; +pub const PL_FLAG_VFORKED: ::c_int = 0x400; +pub const PL_FLAG_VFORK_DONE: ::c_int = 0x800; + pub const PT_LWPINFO: ::c_int = 13; pub const PT_GETNUMLWPS: ::c_int = 14; pub const PT_GETLWPLIST: ::c_int = 15; @@ -622,6 +2343,9 @@ pub const PT_FOLLOW_FORK: ::c_int = 23; pub const PT_LWP_EVENTS: ::c_int = 24; pub const PT_GET_EVENT_MASK: ::c_int = 25; pub const PT_SET_EVENT_MASK: ::c_int = 26; +pub const PT_GET_SC_ARGS: ::c_int = 27; +pub const PT_GET_SC_RET: ::c_int = 28; +pub const PT_COREDUMP: ::c_int = 29; pub const PT_GETREGS: ::c_int = 33; pub const PT_SETREGS: ::c_int = 34; pub const PT_GETFPREGS: ::c_int = 35; @@ -630,6 +2354,8 @@ pub const PT_GETDBREGS: ::c_int = 37; pub const PT_SETDBREGS: ::c_int = 38; pub const PT_VM_TIMESTAMP: ::c_int = 40; pub const PT_VM_ENTRY: ::c_int = 41; +pub const PT_GETREGSET: ::c_int = 42; +pub const PT_SETREGSET: ::c_int = 43; pub const PT_FIRSTMACH: ::c_int = 64; pub const PTRACE_EXEC: ::c_int = 0x0001; @@ -641,6 +2367,67 @@ pub const PTRACE_LWP: ::c_int = 0x0010; pub const PTRACE_VFORK: ::c_int = 0x0020; pub const PTRACE_DEFAULT: ::c_int = PTRACE_EXEC; +pub const PC_COMPRESS: u32 = 0x00000001; +pub const PC_ALL: u32 = 0x00000002; + +pub const PROC_SPROTECT: ::c_int = 1; +pub const PROC_REAP_ACQUIRE: ::c_int = 2; +pub const PROC_REAP_RELEASE: ::c_int = 3; +pub const PROC_REAP_STATUS: ::c_int = 4; +pub const PROC_REAP_GETPIDS: ::c_int = 5; +pub const PROC_REAP_KILL: ::c_int = 6; +pub const PROC_TRACE_CTL: ::c_int = 7; +pub const PROC_TRACE_STATUS: ::c_int = 8; +pub const PROC_TRAPCAP_CTL: ::c_int = 9; +pub const PROC_TRAPCAP_STATUS: ::c_int = 10; +pub const PROC_PDEATHSIG_CTL: ::c_int = 11; +pub const PROC_PDEATHSIG_STATUS: ::c_int = 12; +pub const PROC_ASLR_CTL: ::c_int = 13; +pub const PROC_ASLR_STATUS: ::c_int = 14; +pub const PROC_PROTMAX_CTL: ::c_int = 15; +pub const PROC_PROTMAX_STATUS: ::c_int = 16; +pub const PROC_STACKGAP_CTL: ::c_int = 17; +pub const PROC_STACKGAP_STATUS: ::c_int = 18; +pub const PROC_NO_NEW_PRIVS_CTL: ::c_int = 19; +pub const PROC_NO_NEW_PRIVS_STATUS: ::c_int = 20; +pub const PROC_WXMAP_CTL: ::c_int = 21; +pub const PROC_WXMAP_STATUS: ::c_int = 22; +pub const PROC_PROCCTL_MD_MIN: ::c_int = 0x10000000; + +pub const PPROT_SET: ::c_int = 1; +pub const PPROT_CLEAR: ::c_int = 2; +pub const PPROT_DESCEND: ::c_int = 0x10; +pub const PPROT_INHERIT: ::c_int = 0x20; + +pub const PROC_TRACE_CTL_ENABLE: ::c_int = 1; +pub const PROC_TRACE_CTL_DISABLE: ::c_int = 2; +pub const PROC_TRACE_CTL_DISABLE_EXEC: ::c_int = 3; + +pub const PROC_TRAPCAP_CTL_ENABLE: ::c_int = 1; +pub const PROC_TRAPCAP_CTL_DISABLE: ::c_int = 2; + +pub const PROC_ASLR_FORCE_ENABLE: ::c_int = 1; +pub const PROC_ASLR_FORCE_DISABLE: ::c_int = 2; +pub const PROC_ASLR_NOFORCE: ::c_int = 3; +pub const PROC_ASLR_ACTIVE: ::c_int = 0x80000000; + +pub const PROC_PROTMAX_FORCE_ENABLE: ::c_int = 1; +pub const PROC_PROTMAX_FORCE_DISABLE: ::c_int = 2; +pub const PROC_PROTMAX_NOFORCE: ::c_int = 3; +pub const PROC_PROTMAX_ACTIVE: ::c_int = 0x80000000; + +pub const PROC_STACKGAP_ENABLE: ::c_int = 0x0001; +pub const PROC_STACKGAP_DISABLE: ::c_int = 0x0002; +pub const PROC_STACKGAP_ENABLE_EXEC: ::c_int = 0x0004; +pub const PROC_STACKGAP_DISABLE_EXEC: ::c_int = 0x0008; + +pub const PROC_NO_NEW_PRIVS_ENABLE: ::c_int = 1; +pub const PROC_NO_NEW_PRIVS_DISABLE: ::c_int = 2; + +pub const PROC_WX_MAPPINGS_PERMIT: ::c_int = 0x0001; +pub const PROC_WX_MAPPINGS_DISALLOW_EXEC: ::c_int = 0x0002; +pub const PROC_WXORX_ENFORCE: ::c_int = 0x80000000; + pub const AF_SLOW: ::c_int = 33; pub const AF_SCLUSTER: ::c_int = 34; pub const AF_ARP: ::c_int = 35; @@ -649,43 +2436,220 @@ pub const AF_IEEE80211: ::c_int = 37; pub const AF_INET_SDP: ::c_int = 40; pub const AF_INET6_SDP: ::c_int = 42; -// https://github.com/freebsd/freebsd/blob/master/sys/net/if.h#L140 -pub const IFF_UP: ::c_int = 0x1; // (n) interface is up -pub const IFF_BROADCAST: ::c_int = 0x2; // (i) broadcast address valid -pub const IFF_DEBUG: ::c_int = 0x4; // (n) turn on debugging -pub const IFF_LOOPBACK: ::c_int = 0x8; // (i) is a loopback net -pub const IFF_POINTOPOINT: ::c_int = 0x10; // (i) is a point-to-point link - // 0x20 was IFF_SMART -pub const IFF_RUNNING: ::c_int = 0x40; // (d) resources allocated -#[doc(hidden)] -#[deprecated( - since = "0.2.54", - note = "IFF_DRV_RUNNING is deprecated. Use the portable IFF_RUNNING instead" -)] -pub const IFF_DRV_RUNNING: ::c_int = 0x40; -pub const IFF_NOARP: ::c_int = 0x80; // (n) no address resolution protocol -pub const IFF_PROMISC: ::c_int = 0x100; // (n) receive all packets -pub const IFF_ALLMULTI: ::c_int = 0x200; // (n) receive all multicast packets -pub const IFF_OACTIVE: ::c_int = 0x400; // (d) tx hardware queue is full -#[doc(hidden)] -#[deprecated( - since = "0.2.54", - note = "Use the portable `IFF_OACTIVE` instead" -)] -pub const IFF_DRV_OACTIVE: ::c_int = 0x400; -pub const IFF_SIMPLEX: ::c_int = 0x800; // (i) can't hear own transmissions -pub const IFF_LINK0: ::c_int = 0x1000; // per link layer defined bit -pub const IFF_LINK1: ::c_int = 0x2000; // per link layer defined bit -pub const IFF_LINK2: ::c_int = 0x4000; // per link layer defined bit -pub const IFF_ALTPHYS: ::c_int = IFF_LINK2; // use alternate physical connection -pub const IFF_MULTICAST: ::c_int = 0x8000; // (i) supports multicast - // (i) unconfigurable using ioctl(2) -pub const IFF_CANTCONFIG: ::c_int = 0x10000; -pub const IFF_PPROMISC: ::c_int = 0x20000; // (n) user-requested promisc mode -pub const IFF_MONITOR: ::c_int = 0x40000; // (n) user-requested monitor mode -pub const IFF_STATICARP: ::c_int = 0x80000; // (n) static ARP -pub const IFF_DYING: ::c_int = 0x200000; // (n) interface is winding down -pub const IFF_RENAMING: ::c_int = 0x400000; // (n) interface is being renamed +// sys/net/if.h +pub const IF_MAXUNIT: ::c_int = 0x7fff; +/// (n) interface is up +pub const IFF_UP: ::c_int = 0x1; +/// (i) broadcast address valid +pub const IFF_BROADCAST: ::c_int = 0x2; +/// (n) turn on debugging +pub const IFF_DEBUG: ::c_int = 0x4; +/// (i) is a loopback net +pub const IFF_LOOPBACK: ::c_int = 0x8; +/// (i) is a point-to-point link +pub const IFF_POINTOPOINT: ::c_int = 0x10; +/// (i) calls if_input in net epoch +pub const IFF_KNOWSEPOCH: ::c_int = 0x20; +/// (d) resources allocated +pub const IFF_RUNNING: ::c_int = 0x40; +#[doc(hidden)] +#[deprecated( + since = "0.2.54", + note = "IFF_DRV_RUNNING is deprecated. Use the portable IFF_RUNNING instead" +)] +/// (d) resources allocate +pub const IFF_DRV_RUNNING: ::c_int = 0x40; +/// (n) no address resolution protocol +pub const IFF_NOARP: ::c_int = 0x80; +/// (n) receive all packets +pub const IFF_PROMISC: ::c_int = 0x100; +/// (n) receive all multicast packets +pub const IFF_ALLMULTI: ::c_int = 0x200; +/// (d) tx hardware queue is full +pub const IFF_OACTIVE: ::c_int = 0x400; +#[doc(hidden)] +#[deprecated(since = "0.2.54", note = "Use the portable `IFF_OACTIVE` instead")] +/// (d) tx hardware queue is full +pub const IFF_DRV_OACTIVE: ::c_int = 0x400; +/// (i) can't hear own transmissions +pub const IFF_SIMPLEX: ::c_int = 0x800; +/// per link layer defined bit +pub const IFF_LINK0: ::c_int = 0x1000; +/// per link layer defined bit +pub const IFF_LINK1: ::c_int = 0x2000; +/// per link layer defined bit +pub const IFF_LINK2: ::c_int = 0x4000; +/// use alternate physical connection +pub const IFF_ALTPHYS: ::c_int = IFF_LINK2; +/// (i) supports multicast +pub const IFF_MULTICAST: ::c_int = 0x8000; +/// (i) unconfigurable using ioctl(2) +pub const IFF_CANTCONFIG: ::c_int = 0x10000; +/// (n) user-requested promisc mode +pub const IFF_PPROMISC: ::c_int = 0x20000; +/// (n) user-requested monitor mode +pub const IFF_MONITOR: ::c_int = 0x40000; +/// (n) static ARP +pub const IFF_STATICARP: ::c_int = 0x80000; +/// (n) interface is winding down +pub const IFF_DYING: ::c_int = 0x200000; +/// (n) interface is being renamed +pub const IFF_RENAMING: ::c_int = 0x400000; +/// interface is not part of any groups +pub const IFF_NOGROUP: ::c_int = 0x800000; + +/// link invalid/unknown +pub const LINK_STATE_UNKNOWN: ::c_int = 0; +/// link is down +pub const LINK_STATE_DOWN: ::c_int = 1; +/// link is up +pub const LINK_STATE_UP: ::c_int = 2; + +/// can offload checksum on RX +pub const IFCAP_RXCSUM: ::c_int = 0x00001; +/// can offload checksum on TX +pub const IFCAP_TXCSUM: ::c_int = 0x00002; +/// can be a network console +pub const IFCAP_NETCONS: ::c_int = 0x00004; +/// VLAN-compatible MTU +pub const IFCAP_VLAN_MTU: ::c_int = 0x00008; +/// hardware VLAN tag support +pub const IFCAP_VLAN_HWTAGGING: ::c_int = 0x00010; +/// 9000 byte MTU supported +pub const IFCAP_JUMBO_MTU: ::c_int = 0x00020; +/// driver supports polling +pub const IFCAP_POLLING: ::c_int = 0x00040; +/// can do IFCAP_HWCSUM on VLANs +pub const IFCAP_VLAN_HWCSUM: ::c_int = 0x00080; +/// can do TCP Segmentation Offload +pub const IFCAP_TSO4: ::c_int = 0x00100; +/// can do TCP6 Segmentation Offload +pub const IFCAP_TSO6: ::c_int = 0x00200; +/// can do Large Receive Offload +pub const IFCAP_LRO: ::c_int = 0x00400; +/// wake on any unicast frame +pub const IFCAP_WOL_UCAST: ::c_int = 0x00800; +/// wake on any multicast frame +pub const IFCAP_WOL_MCAST: ::c_int = 0x01000; +/// wake on any Magic Packet +pub const IFCAP_WOL_MAGIC: ::c_int = 0x02000; +/// interface can offload TCP +pub const IFCAP_TOE4: ::c_int = 0x04000; +/// interface can offload TCP6 +pub const IFCAP_TOE6: ::c_int = 0x08000; +/// interface hw can filter vlan tag +pub const IFCAP_VLAN_HWFILTER: ::c_int = 0x10000; +/// can do SIOCGIFCAPNV/SIOCSIFCAPNV +pub const IFCAP_NV: ::c_int = 0x20000; +/// can do IFCAP_TSO on VLANs +pub const IFCAP_VLAN_HWTSO: ::c_int = 0x40000; +/// the runtime link state is dynamic +pub const IFCAP_LINKSTATE: ::c_int = 0x80000; +/// netmap mode supported/enabled +pub const IFCAP_NETMAP: ::c_int = 0x100000; +/// can offload checksum on IPv6 RX +pub const IFCAP_RXCSUM_IPV6: ::c_int = 0x200000; +/// can offload checksum on IPv6 TX +pub const IFCAP_TXCSUM_IPV6: ::c_int = 0x400000; +/// manages counters internally +pub const IFCAP_HWSTATS: ::c_int = 0x800000; +/// hardware supports TX rate limiting +pub const IFCAP_TXRTLMT: ::c_int = 0x1000000; +/// hardware rx timestamping +pub const IFCAP_HWRXTSTMP: ::c_int = 0x2000000; +/// understands M_EXTPG mbufs +pub const IFCAP_MEXTPG: ::c_int = 0x4000000; +/// can do TLS encryption and segmentation for TCP +pub const IFCAP_TXTLS4: ::c_int = 0x8000000; +/// can do TLS encryption and segmentation for TCP6 +pub const IFCAP_TXTLS6: ::c_int = 0x10000000; +/// can do IFCAN_HWCSUM on VXLANs +pub const IFCAP_VXLAN_HWCSUM: ::c_int = 0x20000000; +/// can do IFCAP_TSO on VXLANs +pub const IFCAP_VXLAN_HWTSO: ::c_int = 0x40000000; +/// can do TLS with rate limiting +pub const IFCAP_TXTLS_RTLMT: ::c_int = 0x80000000; + +pub const IFCAP_HWCSUM_IPV6: ::c_int = IFCAP_RXCSUM_IPV6 | IFCAP_TXCSUM_IPV6; +pub const IFCAP_HWCSUM: ::c_int = IFCAP_RXCSUM | IFCAP_TXCSUM; +pub const IFCAP_TSO: ::c_int = IFCAP_TSO4 | IFCAP_TSO6; +pub const IFCAP_WOL: ::c_int = IFCAP_WOL_UCAST | IFCAP_WOL_MCAST | IFCAP_WOL_MAGIC; +pub const IFCAP_TOE: ::c_int = IFCAP_TOE4 | IFCAP_TOE6; +pub const IFCAP_TXTLS: ::c_int = IFCAP_TXTLS4 | IFCAP_TXTLS6; +pub const IFCAP_CANTCHANGE: ::c_int = IFCAP_NETMAP | IFCAP_NV; + +pub const IFQ_MAXLEN: ::c_int = 50; +pub const IFNET_SLOWHZ: ::c_int = 1; + +pub const IFAN_ARRIVAL: ::c_int = 0; +pub const IFAN_DEPARTURE: ::c_int = 1; + +pub const IFSTATMAX: ::c_int = 800; + +pub const RSS_FUNC_NONE: ::c_int = 0; +pub const RSS_FUNC_PRIVATE: ::c_int = 1; +pub const RSS_FUNC_TOEPLITZ: ::c_int = 2; + +pub const RSS_TYPE_IPV4: ::c_int = 0x00000001; +pub const RSS_TYPE_TCP_IPV4: ::c_int = 0x00000002; +pub const RSS_TYPE_IPV6: ::c_int = 0x00000004; +pub const RSS_TYPE_IPV6_EX: ::c_int = 0x00000008; +pub const RSS_TYPE_TCP_IPV6: ::c_int = 0x00000010; +pub const RSS_TYPE_TCP_IPV6_EX: ::c_int = 0x00000020; +pub const RSS_TYPE_UDP_IPV4: ::c_int = 0x00000040; +pub const RSS_TYPE_UDP_IPV6: ::c_int = 0x00000080; +pub const RSS_TYPE_UDP_IPV6_EX: ::c_int = 0x00000100; +pub const RSS_KEYLEN: ::c_int = 128; + +pub const IFNET_PCP_NONE: ::c_int = 0xff; +pub const IFDR_MSG_SIZE: ::c_int = 64; +pub const IFDR_REASON_MSG: ::c_int = 1; +pub const IFDR_REASON_VENDOR: ::c_int = 2; + +// sys/net/if_mib.h + +/// non-interface-specific +pub const IFMIB_SYSTEM: ::c_int = 1; +/// per-interface data table +pub const IFMIB_IFDATA: ::c_int = 2; + +/// generic stats for all kinds of ifaces +pub const IFDATA_GENERAL: ::c_int = 1; +/// specific to the type of interface +pub const IFDATA_LINKSPECIFIC: ::c_int = 2; +/// driver name and unit +pub const IFDATA_DRIVERNAME: ::c_int = 3; + +/// number of interfaces configured +pub const IFMIB_IFCOUNT: ::c_int = 1; + +/// functions not specific to a type of iface +pub const NETLINK_GENERIC: ::c_int = 0; + +pub const DOT3COMPLIANCE_STATS: ::c_int = 1; +pub const DOT3COMPLIANCE_COLLS: ::c_int = 2; + +pub const dot3ChipSetAMD7990: ::c_int = 1; +pub const dot3ChipSetAMD79900: ::c_int = 2; +pub const dot3ChipSetAMD79C940: ::c_int = 3; + +pub const dot3ChipSetIntel82586: ::c_int = 1; +pub const dot3ChipSetIntel82596: ::c_int = 2; +pub const dot3ChipSetIntel82557: ::c_int = 3; + +pub const dot3ChipSetNational8390: ::c_int = 1; +pub const dot3ChipSetNationalSonic: ::c_int = 2; + +pub const dot3ChipSetFujitsu86950: ::c_int = 1; + +pub const dot3ChipSetDigitalDC21040: ::c_int = 1; +pub const dot3ChipSetDigitalDC21140: ::c_int = 2; +pub const dot3ChipSetDigitalDC21041: ::c_int = 3; +pub const dot3ChipSetDigitalDC21140A: ::c_int = 4; +pub const dot3ChipSetDigitalDC21142: ::c_int = 5; + +pub const dot3ChipSetWesternDigital83C690: ::c_int = 1; +pub const dot3ChipSetWesternDigital83C790: ::c_int = 2; // sys/netinet/in.h // Protocols (RFC 1700) @@ -753,8 +2717,14 @@ pub const IPPROTO_BLT: ::c_int = 30; pub const IPPROTO_NSP: ::c_int = 31; /// Merit Internodal pub const IPPROTO_INP: ::c_int = 32; -/// Sequential Exchange +#[doc(hidden)] +#[deprecated( + since = "0.2.72", + note = "IPPROTO_SEP is deprecated. Use IPPROTO_DCCP instead" +)] pub const IPPROTO_SEP: ::c_int = 33; +/// Datagram Congestion Control Protocol +pub const IPPROTO_DCCP: ::c_int = 33; /// Third Party Connect pub const IPPROTO_3PC: ::c_int = 34; /// InterDomain Policy Routing @@ -931,10 +2901,24 @@ pub const TCP_MD5SIG: ::c_int = 16; pub const TCP_INFO: ::c_int = 32; pub const TCP_CONGESTION: ::c_int = 64; pub const TCP_CCALGOOPT: ::c_int = 65; +pub const TCP_MAXUNACKTIME: ::c_int = 68; +pub const TCP_MAXPEAKRATE: ::c_int = 69; +pub const TCP_IDLE_REDUCE: ::c_int = 70; +pub const TCP_REMOTE_UDP_ENCAPS_PORT: ::c_int = 71; +pub const TCP_DELACK: ::c_int = 72; +pub const TCP_FIN_IS_RST: ::c_int = 73; +pub const TCP_LOG_LIMIT: ::c_int = 74; +pub const TCP_SHARED_CWND_ALLOWED: ::c_int = 75; +pub const TCP_PROC_ACCOUNTING: ::c_int = 76; +pub const TCP_USE_CMP_ACKS: ::c_int = 77; +pub const TCP_PERF_INFO: ::c_int = 78; +pub const TCP_LRD: ::c_int = 79; pub const TCP_KEEPINIT: ::c_int = 128; pub const TCP_FASTOPEN: ::c_int = 1025; pub const TCP_PCAP_OUT: ::c_int = 2048; pub const TCP_PCAP_IN: ::c_int = 4096; +pub const TCP_FASTOPEN_PSK_LEN: ::c_int = 16; +pub const TCP_FUNCTION_NAME_LEN_MAX: ::c_int = 32; pub const IP_BINDANY: ::c_int = 24; pub const IP_BINDMULTI: ::c_int = 25; @@ -942,8 +2926,10 @@ pub const IP_RSS_LISTEN_BUCKET: ::c_int = 26; pub const IP_ORIGDSTADDR: ::c_int = 27; pub const IP_RECVORIGDSTADDR: ::c_int = IP_ORIGDSTADDR; +pub const IP_DONTFRAG: ::c_int = 67; pub const IP_RECVTOS: ::c_int = 68; +pub const IPV6_BINDANY: ::c_int = 64; pub const IPV6_ORIGDSTADDR: ::c_int = 72; pub const IPV6_RECVORIGDSTADDR: ::c_int = IPV6_ORIGDSTADDR; @@ -962,22 +2948,8 @@ pub const NET_RT_IFMALIST: ::c_int = 4; pub const NET_RT_IFLISTL: ::c_int = 5; // System V IPC -pub const IPC_PRIVATE: ::key_t = 0; -pub const IPC_CREAT: ::c_int = 0o1000; -pub const IPC_EXCL: ::c_int = 0o2000; -pub const IPC_NOWAIT: ::c_int = 0o4000; -pub const IPC_RMID: ::c_int = 0; -pub const IPC_SET: ::c_int = 1; -pub const IPC_STAT: ::c_int = 2; pub const IPC_INFO: ::c_int = 3; -pub const IPC_R: ::c_int = 0o400; -pub const IPC_W: ::c_int = 0o200; -pub const IPC_M: ::c_int = 0o10000; pub const MSG_NOERROR: ::c_int = 0o10000; -pub const SHM_RDONLY: ::c_int = 0o10000; -pub const SHM_RND: ::c_int = 0o20000; -pub const SHM_R: ::c_int = 0o400; -pub const SHM_W: ::c_int = 0o200; pub const SHM_LOCK: ::c_int = 11; pub const SHM_UNLOCK: ::c_int = 12; pub const SHM_STAT: ::c_int = 13; @@ -1001,6 +2973,7 @@ pub const HW_MAXID: ::c_int = 13; #[deprecated(since = "0.2.54", note = "Removed in FreeBSD 11")] pub const USER_MAXID: ::c_int = 21; #[doc(hidden)] +#[deprecated(since = "0.2.74", note = "Removed in FreeBSD 13")] pub const CTL_P1003_1B_MAXID: ::c_int = 26; pub const MSG_NOTIFICATION: ::c_int = 0x00002000; @@ -1061,6 +3034,23 @@ pub const AT_SYMLINK_NOFOLLOW: ::c_int = 0x200; pub const AT_SYMLINK_FOLLOW: ::c_int = 0x400; pub const AT_REMOVEDIR: ::c_int = 0x800; +pub const AT_NULL: ::c_int = 0; +pub const AT_IGNORE: ::c_int = 1; +pub const AT_EXECFD: ::c_int = 2; +pub const AT_PHDR: ::c_int = 3; +pub const AT_PHENT: ::c_int = 4; +pub const AT_PHNUM: ::c_int = 5; +pub const AT_PAGESZ: ::c_int = 6; +pub const AT_BASE: ::c_int = 7; +pub const AT_FLAGS: ::c_int = 8; +pub const AT_ENTRY: ::c_int = 9; +pub const AT_NOTELF: ::c_int = 10; +pub const AT_UID: ::c_int = 11; +pub const AT_EUID: ::c_int = 12; +pub const AT_GID: ::c_int = 13; +pub const AT_EGID: ::c_int = 14; +pub const AT_EXECPATH: ::c_int = 15; + pub const TABDLY: ::tcflag_t = 0x00000004; pub const TAB0: ::tcflag_t = 0x00000000; pub const TAB3: ::tcflag_t = 0x00000004; @@ -1069,8 +3059,7 @@ pub const _PC_ACL_NFS4: ::c_int = 64; pub const _SC_CPUSET_SIZE: ::c_int = 122; -pub const XU_NGROUPS: ::c_int = 16; -pub const XUCRED_VERSION: ::c_uint = 0; +pub const _UUID_NODE_LEN: usize = 6; // Flags which can be passed to pdfork(2) pub const PD_DAEMON: ::c_int = 0x00000001; @@ -1099,8 +3088,714 @@ pub const UF_READONLY: ::c_ulong = 0x00001000; pub const UF_HIDDEN: ::c_ulong = 0x00008000; pub const SF_SNAPSHOT: ::c_ulong = 0x00200000; -fn _ALIGN(p: usize) -> usize { - (p + _ALIGNBYTES) & !_ALIGNBYTES +// fcntl commands +pub const F_ADD_SEALS: ::c_int = 19; +pub const F_GET_SEALS: ::c_int = 20; +pub const F_OGETLK: ::c_int = 7; +pub const F_OSETLK: ::c_int = 8; +pub const F_OSETLKW: ::c_int = 9; +pub const F_RDAHEAD: ::c_int = 16; +pub const F_READAHEAD: ::c_int = 15; +pub const F_SETLK_REMOTE: ::c_int = 14; +pub const F_KINFO: ::c_int = 22; + +// for use with F_ADD_SEALS +pub const F_SEAL_GROW: ::c_int = 4; +pub const F_SEAL_SEAL: ::c_int = 1; +pub const F_SEAL_SHRINK: ::c_int = 2; +pub const F_SEAL_WRITE: ::c_int = 8; + +// for use with fspacectl +pub const SPACECTL_DEALLOC: ::c_int = 1; + +// For getrandom() +pub const GRND_NONBLOCK: ::c_uint = 0x1; +pub const GRND_RANDOM: ::c_uint = 0x2; +pub const GRND_INSECURE: ::c_uint = 0x4; + +// For realhostname* api +pub const HOSTNAME_FOUND: ::c_int = 0; +pub const HOSTNAME_INCORRECTNAME: ::c_int = 1; +pub const HOSTNAME_INVALIDADDR: ::c_int = 2; +pub const HOSTNAME_INVALIDNAME: ::c_int = 3; + +// For rfork +pub const RFFDG: ::c_int = 4; +pub const RFPROC: ::c_int = 16; +pub const RFMEM: ::c_int = 32; +pub const RFNOWAIT: ::c_int = 64; +pub const RFCFDG: ::c_int = 4096; +pub const RFTHREAD: ::c_int = 8192; +pub const RFLINUXTHPN: ::c_int = 65536; +pub const RFTSIGZMB: ::c_int = 524288; +pub const RFSPAWN: ::c_int = 2147483648; + +// For eventfd +pub const EFD_SEMAPHORE: ::c_int = 0x1; +pub const EFD_NONBLOCK: ::c_int = 0x4; +pub const EFD_CLOEXEC: ::c_int = 0x100000; + +pub const MALLOCX_ZERO: ::c_int = 0x40; + +/// size of returned wchan message +pub const WMESGLEN: usize = 8; +/// size of returned lock name +pub const LOCKNAMELEN: usize = 8; +/// size of returned thread name +pub const TDNAMLEN: usize = 16; +/// size of returned ki_comm name +pub const COMMLEN: usize = 19; +/// size of returned ki_emul +pub const KI_EMULNAMELEN: usize = 16; +/// number of groups in ki_groups +pub const KI_NGROUPS: usize = 16; +cfg_if! { + if #[cfg(freebsd11)] { + pub const KI_NSPARE_INT: usize = 4; + } else { + pub const KI_NSPARE_INT: usize = 2; + } +} +pub const KI_NSPARE_LONG: usize = 12; +/// Flags for the process credential. +pub const KI_CRF_CAPABILITY_MODE: usize = 0x00000001; +/// Steal a bit from ki_cr_flags to indicate that the cred had more than +/// KI_NGROUPS groups. +pub const KI_CRF_GRP_OVERFLOW: usize = 0x80000000; +/// controlling tty vnode active +pub const KI_CTTY: usize = 0x00000001; +/// session leader +pub const KI_SLEADER: usize = 0x00000002; +/// proc blocked on lock ki_lockname +pub const KI_LOCKBLOCK: usize = 0x00000004; +/// size of returned ki_login +pub const LOGNAMELEN: usize = 17; +/// size of returned ki_loginclass +pub const LOGINCLASSLEN: usize = 17; + +pub const KF_ATTR_VALID: ::c_int = 0x0001; +pub const KF_TYPE_NONE: ::c_int = 0; +pub const KF_TYPE_VNODE: ::c_int = 1; +pub const KF_TYPE_SOCKET: ::c_int = 2; +pub const KF_TYPE_PIPE: ::c_int = 3; +pub const KF_TYPE_FIFO: ::c_int = 4; +pub const KF_TYPE_KQUEUE: ::c_int = 5; +pub const KF_TYPE_MQUEUE: ::c_int = 7; +pub const KF_TYPE_SHM: ::c_int = 8; +pub const KF_TYPE_SEM: ::c_int = 9; +pub const KF_TYPE_PTS: ::c_int = 10; +pub const KF_TYPE_PROCDESC: ::c_int = 11; +pub const KF_TYPE_DEV: ::c_int = 12; +pub const KF_TYPE_UNKNOWN: ::c_int = 255; + +pub const KF_VTYPE_VNON: ::c_int = 0; +pub const KF_VTYPE_VREG: ::c_int = 1; +pub const KF_VTYPE_VDIR: ::c_int = 2; +pub const KF_VTYPE_VBLK: ::c_int = 3; +pub const KF_VTYPE_VCHR: ::c_int = 4; +pub const KF_VTYPE_VLNK: ::c_int = 5; +pub const KF_VTYPE_VSOCK: ::c_int = 6; +pub const KF_VTYPE_VFIFO: ::c_int = 7; +pub const KF_VTYPE_VBAD: ::c_int = 8; +pub const KF_VTYPE_UNKNOWN: ::c_int = 255; + +/// Current working directory +pub const KF_FD_TYPE_CWD: ::c_int = -1; +/// Root directory +pub const KF_FD_TYPE_ROOT: ::c_int = -2; +/// Jail directory +pub const KF_FD_TYPE_JAIL: ::c_int = -3; +/// Ktrace vnode +pub const KF_FD_TYPE_TRACE: ::c_int = -4; +pub const KF_FD_TYPE_TEXT: ::c_int = -5; +/// Controlling terminal +pub const KF_FD_TYPE_CTTY: ::c_int = -6; +pub const KF_FLAG_READ: ::c_int = 0x00000001; +pub const KF_FLAG_WRITE: ::c_int = 0x00000002; +pub const KF_FLAG_APPEND: ::c_int = 0x00000004; +pub const KF_FLAG_ASYNC: ::c_int = 0x00000008; +pub const KF_FLAG_FSYNC: ::c_int = 0x00000010; +pub const KF_FLAG_NONBLOCK: ::c_int = 0x00000020; +pub const KF_FLAG_DIRECT: ::c_int = 0x00000040; +pub const KF_FLAG_HASLOCK: ::c_int = 0x00000080; +pub const KF_FLAG_SHLOCK: ::c_int = 0x00000100; +pub const KF_FLAG_EXLOCK: ::c_int = 0x00000200; +pub const KF_FLAG_NOFOLLOW: ::c_int = 0x00000400; +pub const KF_FLAG_CREAT: ::c_int = 0x00000800; +pub const KF_FLAG_TRUNC: ::c_int = 0x00001000; +pub const KF_FLAG_EXCL: ::c_int = 0x00002000; +pub const KF_FLAG_EXEC: ::c_int = 0x00004000; + +pub const KVME_TYPE_NONE: ::c_int = 0; +pub const KVME_TYPE_DEFAULT: ::c_int = 1; +pub const KVME_TYPE_VNODE: ::c_int = 2; +pub const KVME_TYPE_SWAP: ::c_int = 3; +pub const KVME_TYPE_DEVICE: ::c_int = 4; +pub const KVME_TYPE_PHYS: ::c_int = 5; +pub const KVME_TYPE_DEAD: ::c_int = 6; +pub const KVME_TYPE_SG: ::c_int = 7; +pub const KVME_TYPE_MGTDEVICE: ::c_int = 8; +// Present in `sys/user.h` but is undefined for whatever reason... +// pub const KVME_TYPE_GUARD: ::c_int = 9; +pub const KVME_TYPE_UNKNOWN: ::c_int = 255; +pub const KVME_PROT_READ: ::c_int = 0x00000001; +pub const KVME_PROT_WRITE: ::c_int = 0x00000002; +pub const KVME_PROT_EXEC: ::c_int = 0x00000004; +pub const KVME_FLAG_COW: ::c_int = 0x00000001; +pub const KVME_FLAG_NEEDS_COPY: ::c_int = 0x00000002; +pub const KVME_FLAG_NOCOREDUMP: ::c_int = 0x00000004; +pub const KVME_FLAG_SUPER: ::c_int = 0x00000008; +pub const KVME_FLAG_GROWS_UP: ::c_int = 0x00000010; +pub const KVME_FLAG_GROWS_DOWN: ::c_int = 0x00000020; +pub const KVME_FLAG_USER_WIRED: ::c_int = 0x00000040; + +pub const KKST_MAXLEN: ::c_int = 1024; +/// Stack is valid. +pub const KKST_STATE_STACKOK: ::c_int = 0; +/// Stack swapped out. +pub const KKST_STATE_SWAPPED: ::c_int = 1; +pub const KKST_STATE_RUNNING: ::c_int = 2; + +// Constants about priority. +pub const PRI_MIN: ::c_int = 0; +pub const PRI_MAX: ::c_int = 255; +pub const PRI_MIN_ITHD: ::c_int = PRI_MIN; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +#[allow(deprecated)] +pub const PRI_MAX_ITHD: ::c_int = PRI_MIN_REALTIME - 1; +pub const PI_REALTIME: ::c_int = PRI_MIN_ITHD + 0; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +pub const PI_AV: ::c_int = PRI_MIN_ITHD + 4; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +pub const PI_NET: ::c_int = PRI_MIN_ITHD + 8; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +pub const PI_DISK: ::c_int = PRI_MIN_ITHD + 12; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +pub const PI_TTY: ::c_int = PRI_MIN_ITHD + 16; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +pub const PI_DULL: ::c_int = PRI_MIN_ITHD + 20; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +pub const PI_SOFT: ::c_int = PRI_MIN_ITHD + 24; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +pub const PRI_MIN_REALTIME: ::c_int = 48; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +#[allow(deprecated)] +pub const PRI_MAX_REALTIME: ::c_int = PRI_MIN_KERN - 1; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +pub const PRI_MIN_KERN: ::c_int = 80; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +#[allow(deprecated)] +pub const PRI_MAX_KERN: ::c_int = PRI_MIN_TIMESHARE - 1; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +#[allow(deprecated)] +pub const PSWP: ::c_int = PRI_MIN_KERN + 0; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +#[allow(deprecated)] +pub const PVM: ::c_int = PRI_MIN_KERN + 4; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +#[allow(deprecated)] +pub const PINOD: ::c_int = PRI_MIN_KERN + 8; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +#[allow(deprecated)] +pub const PRIBIO: ::c_int = PRI_MIN_KERN + 12; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +#[allow(deprecated)] +pub const PVFS: ::c_int = PRI_MIN_KERN + 16; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +#[allow(deprecated)] +pub const PZERO: ::c_int = PRI_MIN_KERN + 20; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +#[allow(deprecated)] +pub const PSOCK: ::c_int = PRI_MIN_KERN + 24; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +#[allow(deprecated)] +pub const PWAIT: ::c_int = PRI_MIN_KERN + 28; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +#[allow(deprecated)] +pub const PLOCK: ::c_int = PRI_MIN_KERN + 32; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +#[allow(deprecated)] +pub const PPAUSE: ::c_int = PRI_MIN_KERN + 36; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +pub const PRI_MIN_TIMESHARE: ::c_int = 120; +pub const PRI_MAX_TIMESHARE: ::c_int = PRI_MIN_IDLE - 1; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +#[allow(deprecated)] +pub const PUSER: ::c_int = PRI_MIN_TIMESHARE; +pub const PRI_MIN_IDLE: ::c_int = 224; +pub const PRI_MAX_IDLE: ::c_int = PRI_MAX; + +pub const NZERO: ::c_int = 0; + +// Resource utilization information. +pub const RUSAGE_THREAD: ::c_int = 1; + +cfg_if! { + if #[cfg(any(freebsd11, target_pointer_width = "32"))] { + pub const ARG_MAX: ::c_int = 256 * 1024; + } else { + pub const ARG_MAX: ::c_int = 2 * 256 * 1024; + } +} +pub const CHILD_MAX: ::c_int = 40; +/// max command name remembered +pub const MAXCOMLEN: usize = 19; +/// max interpreter file name length +pub const MAXINTERP: ::c_int = ::PATH_MAX; +/// max login name length (incl. NUL) +pub const MAXLOGNAME: ::c_int = 33; +/// max simultaneous processes +pub const MAXUPRC: ::c_int = CHILD_MAX; +/// max bytes for an exec function +pub const NCARGS: ::c_int = ARG_MAX; +/// /* max number groups +pub const NGROUPS: ::c_int = NGROUPS_MAX + 1; +/// max open files per process +pub const NOFILE: ::c_int = OPEN_MAX; +/// marker for empty group set member +pub const NOGROUP: ::c_int = 65535; +/// max hostname size +pub const MAXHOSTNAMELEN: ::c_int = 256; +/// max bytes in term canon input line +pub const MAX_CANON: ::c_int = 255; +/// max bytes in terminal input +pub const MAX_INPUT: ::c_int = 255; +/// max bytes in a file name +pub const NAME_MAX: ::c_int = 255; +pub const MAXSYMLINKS: ::c_int = 32; +/// max supplemental group id's +pub const NGROUPS_MAX: ::c_int = 1023; +/// max open files per process +pub const OPEN_MAX: ::c_int = 64; + +pub const _POSIX_ARG_MAX: ::c_int = 4096; +pub const _POSIX_LINK_MAX: ::c_int = 8; +pub const _POSIX_MAX_CANON: ::c_int = 255; +pub const _POSIX_MAX_INPUT: ::c_int = 255; +pub const _POSIX_NAME_MAX: ::c_int = 14; +pub const _POSIX_PIPE_BUF: ::c_int = 512; +pub const _POSIX_SSIZE_MAX: ::c_int = 32767; +pub const _POSIX_STREAM_MAX: ::c_int = 8; + +/// max ibase/obase values in bc(1) +pub const BC_BASE_MAX: ::c_int = 99; +/// max array elements in bc(1) +pub const BC_DIM_MAX: ::c_int = 2048; +/// max scale value in bc(1) +pub const BC_SCALE_MAX: ::c_int = 99; +/// max const string length in bc(1) +pub const BC_STRING_MAX: ::c_int = 1000; +/// max character class name size +pub const CHARCLASS_NAME_MAX: ::c_int = 14; +/// max weights for order keyword +pub const COLL_WEIGHTS_MAX: ::c_int = 10; +/// max expressions nested in expr(1) +pub const EXPR_NEST_MAX: ::c_int = 32; +/// max bytes in an input line +pub const LINE_MAX: ::c_int = 2048; +/// max RE's in interval notation +pub const RE_DUP_MAX: ::c_int = 255; + +pub const _POSIX2_BC_BASE_MAX: ::c_int = 99; +pub const _POSIX2_BC_DIM_MAX: ::c_int = 2048; +pub const _POSIX2_BC_SCALE_MAX: ::c_int = 99; +pub const _POSIX2_BC_STRING_MAX: ::c_int = 1000; +pub const _POSIX2_CHARCLASS_NAME_MAX: ::c_int = 14; +pub const _POSIX2_COLL_WEIGHTS_MAX: ::c_int = 2; +pub const _POSIX2_EQUIV_CLASS_MAX: ::c_int = 2; +pub const _POSIX2_EXPR_NEST_MAX: ::c_int = 32; +pub const _POSIX2_LINE_MAX: ::c_int = 2048; +pub const _POSIX2_RE_DUP_MAX: ::c_int = 255; + +// sys/proc.h +pub const TDF_BORROWING: ::c_int = 0x00000001; +pub const TDF_INPANIC: ::c_int = 0x00000002; +pub const TDF_INMEM: ::c_int = 0x00000004; +pub const TDF_SINTR: ::c_int = 0x00000008; +pub const TDF_TIMEOUT: ::c_int = 0x00000010; +pub const TDF_IDLETD: ::c_int = 0x00000020; +pub const TDF_CANSWAP: ::c_int = 0x00000040; +pub const TDF_KTH_SUSP: ::c_int = 0x00000100; +pub const TDF_ALLPROCSUSP: ::c_int = 0x00000200; +pub const TDF_BOUNDARY: ::c_int = 0x00000400; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +pub const TDF_ASTPENDING: ::c_int = 0x00000800; +pub const TDF_SBDRY: ::c_int = 0x00002000; +pub const TDF_UPIBLOCKED: ::c_int = 0x00004000; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +pub const TDF_NEEDSUSPCHK: ::c_int = 0x00008000; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +pub const TDF_NEEDRESCHED: ::c_int = 0x00010000; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +pub const TDF_NEEDSIGCHK: ::c_int = 0x00020000; +pub const TDF_NOLOAD: ::c_int = 0x00040000; +pub const TDF_SERESTART: ::c_int = 0x00080000; +pub const TDF_THRWAKEUP: ::c_int = 0x00100000; +pub const TDF_SEINTR: ::c_int = 0x00200000; +pub const TDF_SWAPINREQ: ::c_int = 0x00400000; +#[deprecated(since = "0.2.133", note = "Removed in FreeBSD 14")] +pub const TDF_UNUSED23: ::c_int = 0x00800000; +pub const TDF_SCHED0: ::c_int = 0x01000000; +pub const TDF_SCHED1: ::c_int = 0x02000000; +pub const TDF_SCHED2: ::c_int = 0x04000000; +pub const TDF_SCHED3: ::c_int = 0x08000000; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +pub const TDF_ALRMPEND: ::c_int = 0x10000000; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +pub const TDF_PROFPEND: ::c_int = 0x20000000; +#[deprecated(since = "0.2.133", note = "Not stable across OS versions")] +pub const TDF_MACPEND: ::c_int = 0x40000000; + +pub const TDB_SUSPEND: ::c_int = 0x00000001; +pub const TDB_XSIG: ::c_int = 0x00000002; +pub const TDB_USERWR: ::c_int = 0x00000004; +pub const TDB_SCE: ::c_int = 0x00000008; +pub const TDB_SCX: ::c_int = 0x00000010; +pub const TDB_EXEC: ::c_int = 0x00000020; +pub const TDB_FORK: ::c_int = 0x00000040; +pub const TDB_STOPATFORK: ::c_int = 0x00000080; +pub const TDB_CHILD: ::c_int = 0x00000100; +pub const TDB_BORN: ::c_int = 0x00000200; +pub const TDB_EXIT: ::c_int = 0x00000400; +pub const TDB_VFORK: ::c_int = 0x00000800; +pub const TDB_FSTP: ::c_int = 0x00001000; +pub const TDB_STEP: ::c_int = 0x00002000; + +pub const TDP_OLDMASK: ::c_int = 0x00000001; +pub const TDP_INKTR: ::c_int = 0x00000002; +pub const TDP_INKTRACE: ::c_int = 0x00000004; +pub const TDP_BUFNEED: ::c_int = 0x00000008; +pub const TDP_COWINPROGRESS: ::c_int = 0x00000010; +pub const TDP_ALTSTACK: ::c_int = 0x00000020; +pub const TDP_DEADLKTREAT: ::c_int = 0x00000040; +pub const TDP_NOFAULTING: ::c_int = 0x00000080; +pub const TDP_OWEUPC: ::c_int = 0x00000200; +pub const TDP_ITHREAD: ::c_int = 0x00000400; +pub const TDP_SYNCIO: ::c_int = 0x00000800; +pub const TDP_SCHED1: ::c_int = 0x00001000; +pub const TDP_SCHED2: ::c_int = 0x00002000; +pub const TDP_SCHED3: ::c_int = 0x00004000; +pub const TDP_SCHED4: ::c_int = 0x00008000; +pub const TDP_GEOM: ::c_int = 0x00010000; +pub const TDP_SOFTDEP: ::c_int = 0x00020000; +pub const TDP_NORUNNINGBUF: ::c_int = 0x00040000; +pub const TDP_WAKEUP: ::c_int = 0x00080000; +pub const TDP_INBDFLUSH: ::c_int = 0x00100000; +pub const TDP_KTHREAD: ::c_int = 0x00200000; +pub const TDP_CALLCHAIN: ::c_int = 0x00400000; +pub const TDP_IGNSUSP: ::c_int = 0x00800000; +pub const TDP_AUDITREC: ::c_int = 0x01000000; +pub const TDP_RFPPWAIT: ::c_int = 0x02000000; +pub const TDP_RESETSPUR: ::c_int = 0x04000000; +pub const TDP_NERRNO: ::c_int = 0x08000000; +pub const TDP_EXECVMSPC: ::c_int = 0x40000000; + +pub const TDI_SUSPENDED: ::c_int = 0x0001; +pub const TDI_SLEEPING: ::c_int = 0x0002; +pub const TDI_SWAPPED: ::c_int = 0x0004; +pub const TDI_LOCK: ::c_int = 0x0008; +pub const TDI_IWAIT: ::c_int = 0x0010; + +pub const P_ADVLOCK: ::c_int = 0x00000001; +pub const P_CONTROLT: ::c_int = 0x00000002; +pub const P_KPROC: ::c_int = 0x00000004; +pub const P_UNUSED3: ::c_int = 0x00000008; +pub const P_PPWAIT: ::c_int = 0x00000010; +pub const P_PROFIL: ::c_int = 0x00000020; +pub const P_STOPPROF: ::c_int = 0x00000040; +pub const P_HADTHREADS: ::c_int = 0x00000080; +pub const P_SUGID: ::c_int = 0x00000100; +pub const P_SYSTEM: ::c_int = 0x00000200; +pub const P_SINGLE_EXIT: ::c_int = 0x00000400; +pub const P_TRACED: ::c_int = 0x00000800; +pub const P_WAITED: ::c_int = 0x00001000; +pub const P_WEXIT: ::c_int = 0x00002000; +pub const P_EXEC: ::c_int = 0x00004000; +pub const P_WKILLED: ::c_int = 0x00008000; +pub const P_CONTINUED: ::c_int = 0x00010000; +pub const P_STOPPED_SIG: ::c_int = 0x00020000; +pub const P_STOPPED_TRACE: ::c_int = 0x00040000; +pub const P_STOPPED_SINGLE: ::c_int = 0x00080000; +pub const P_PROTECTED: ::c_int = 0x00100000; +pub const P_SIGEVENT: ::c_int = 0x00200000; +pub const P_SINGLE_BOUNDARY: ::c_int = 0x00400000; +pub const P_HWPMC: ::c_int = 0x00800000; +pub const P_JAILED: ::c_int = 0x01000000; +pub const P_TOTAL_STOP: ::c_int = 0x02000000; +pub const P_INEXEC: ::c_int = 0x04000000; +pub const P_STATCHILD: ::c_int = 0x08000000; +pub const P_INMEM: ::c_int = 0x10000000; +pub const P_SWAPPINGOUT: ::c_int = 0x20000000; +pub const P_SWAPPINGIN: ::c_int = 0x40000000; +pub const P_PPTRACE: ::c_int = 0x80000000; +pub const P_STOPPED: ::c_int = P_STOPPED_SIG | P_STOPPED_SINGLE | P_STOPPED_TRACE; + +pub const P2_INHERIT_PROTECTED: ::c_int = 0x00000001; +pub const P2_NOTRACE: ::c_int = 0x00000002; +pub const P2_NOTRACE_EXEC: ::c_int = 0x00000004; +pub const P2_AST_SU: ::c_int = 0x00000008; +pub const P2_PTRACE_FSTP: ::c_int = 0x00000010; +pub const P2_TRAPCAP: ::c_int = 0x00000020; +pub const P2_STKGAP_DISABLE: ::c_int = 0x00000800; +pub const P2_STKGAP_DISABLE_EXEC: ::c_int = 0x00001000; + +pub const P_TREE_ORPHANED: ::c_int = 0x00000001; +pub const P_TREE_FIRST_ORPHAN: ::c_int = 0x00000002; +pub const P_TREE_REAPER: ::c_int = 0x00000004; + +pub const SIDL: ::c_char = 1; +pub const SRUN: ::c_char = 2; +pub const SSLEEP: ::c_char = 3; +pub const SSTOP: ::c_char = 4; +pub const SZOMB: ::c_char = 5; +pub const SWAIT: ::c_char = 6; +pub const SLOCK: ::c_char = 7; + +pub const P_MAGIC: ::c_int = 0xbeefface; + +pub const TDP_SIGFASTBLOCK: ::c_int = 0x00000100; +pub const TDP_UIOHELD: ::c_int = 0x10000000; +pub const TDP_SIGFASTPENDING: ::c_int = 0x80000000; +pub const TDP2_COMPAT32RB: ::c_int = 0x00000002; +pub const P2_PROTMAX_ENABLE: ::c_int = 0x00000200; +pub const P2_PROTMAX_DISABLE: ::c_int = 0x00000400; +pub const TDP2_SBPAGES: ::c_int = 0x00000001; +pub const P2_ASLR_ENABLE: ::c_int = 0x00000040; +pub const P2_ASLR_DISABLE: ::c_int = 0x00000080; +pub const P2_ASLR_IGNSTART: ::c_int = 0x00000100; +pub const P_TREE_GRPEXITED: ::c_int = 0x00000008; + +// libprocstat.h +pub const PS_FST_VTYPE_VNON: ::c_int = 1; +pub const PS_FST_VTYPE_VREG: ::c_int = 2; +pub const PS_FST_VTYPE_VDIR: ::c_int = 3; +pub const PS_FST_VTYPE_VBLK: ::c_int = 4; +pub const PS_FST_VTYPE_VCHR: ::c_int = 5; +pub const PS_FST_VTYPE_VLNK: ::c_int = 6; +pub const PS_FST_VTYPE_VSOCK: ::c_int = 7; +pub const PS_FST_VTYPE_VFIFO: ::c_int = 8; +pub const PS_FST_VTYPE_VBAD: ::c_int = 9; +pub const PS_FST_VTYPE_UNKNOWN: ::c_int = 255; + +pub const PS_FST_TYPE_VNODE: ::c_int = 1; +pub const PS_FST_TYPE_FIFO: ::c_int = 2; +pub const PS_FST_TYPE_SOCKET: ::c_int = 3; +pub const PS_FST_TYPE_PIPE: ::c_int = 4; +pub const PS_FST_TYPE_PTS: ::c_int = 5; +pub const PS_FST_TYPE_KQUEUE: ::c_int = 6; +pub const PS_FST_TYPE_MQUEUE: ::c_int = 8; +pub const PS_FST_TYPE_SHM: ::c_int = 9; +pub const PS_FST_TYPE_SEM: ::c_int = 10; +pub const PS_FST_TYPE_UNKNOWN: ::c_int = 11; +pub const PS_FST_TYPE_NONE: ::c_int = 12; +pub const PS_FST_TYPE_PROCDESC: ::c_int = 13; +pub const PS_FST_TYPE_DEV: ::c_int = 14; +pub const PS_FST_TYPE_EVENTFD: ::c_int = 15; + +pub const PS_FST_UFLAG_RDIR: ::c_int = 0x0001; +pub const PS_FST_UFLAG_CDIR: ::c_int = 0x0002; +pub const PS_FST_UFLAG_JAIL: ::c_int = 0x0004; +pub const PS_FST_UFLAG_TRACE: ::c_int = 0x0008; +pub const PS_FST_UFLAG_TEXT: ::c_int = 0x0010; +pub const PS_FST_UFLAG_MMAP: ::c_int = 0x0020; +pub const PS_FST_UFLAG_CTTY: ::c_int = 0x0040; + +pub const PS_FST_FFLAG_READ: ::c_int = 0x0001; +pub const PS_FST_FFLAG_WRITE: ::c_int = 0x0002; +pub const PS_FST_FFLAG_NONBLOCK: ::c_int = 0x0004; +pub const PS_FST_FFLAG_APPEND: ::c_int = 0x0008; +pub const PS_FST_FFLAG_SHLOCK: ::c_int = 0x0010; +pub const PS_FST_FFLAG_EXLOCK: ::c_int = 0x0020; +pub const PS_FST_FFLAG_ASYNC: ::c_int = 0x0040; +pub const PS_FST_FFLAG_SYNC: ::c_int = 0x0080; +pub const PS_FST_FFLAG_NOFOLLOW: ::c_int = 0x0100; +pub const PS_FST_FFLAG_CREAT: ::c_int = 0x0200; +pub const PS_FST_FFLAG_TRUNC: ::c_int = 0x0400; +pub const PS_FST_FFLAG_EXCL: ::c_int = 0x0800; +pub const PS_FST_FFLAG_DIRECT: ::c_int = 0x1000; +pub const PS_FST_FFLAG_EXEC: ::c_int = 0x2000; +pub const PS_FST_FFLAG_HASLOCK: ::c_int = 0x4000; + +// sys/mount.h + +/// File identifier. +/// These are unique per filesystem on a single machine. +/// +/// Note that the offset of fid_data is 4 bytes, so care must be taken to avoid +/// undefined behavior accessing unaligned fields within an embedded struct. +pub const MAXFIDSZ: ::c_int = 16; +/// Length of type name including null. +pub const MFSNAMELEN: ::c_int = 16; +cfg_if! { + if #[cfg(any(freebsd10, freebsd11))] { + /// Size of on/from name bufs. + pub const MNAMELEN: ::c_int = 88; + } else { + /// Size of on/from name bufs. + pub const MNAMELEN: ::c_int = 1024; + } +} + +/// Using journaled soft updates. +pub const MNT_SUJ: u64 = 0x100000000; +/// Mounted by automountd(8). +pub const MNT_AUTOMOUNTED: u64 = 0x200000000; +/// Filesys metadata untrusted. +pub const MNT_UNTRUSTED: u64 = 0x800000000; + +/// Require TLS. +pub const MNT_EXTLS: u64 = 0x4000000000; +/// Require TLS with client cert. +pub const MNT_EXTLSCERT: u64 = 0x8000000000; +/// Require TLS with user cert. +pub const MNT_EXTLSCERTUSER: u64 = 0x10000000000; + +/// Filesystem is stored locally. +pub const MNT_LOCAL: u64 = 0x000001000; +/// Quotas are enabled on fs. +pub const MNT_QUOTA: u64 = 0x000002000; +/// Identifies the root fs. +pub const MNT_ROOTFS: u64 = 0x000004000; +/// Mounted by a user. +pub const MNT_USER: u64 = 0x000008000; +/// Do not show entry in df. +pub const MNT_IGNORE: u64 = 0x000800000; +/// Filesystem is verified. +pub const MNT_VERIFIED: u64 = 0x400000000; + +/// Do not cover a mount point. +pub const MNT_NOCOVER: u64 = 0x001000000000; +/// Only mount on empty dir. +pub const MNT_EMPTYDIR: u64 = 0x002000000000; +/// Recursively unmount uppers. +pub const MNT_RECURSE: u64 = 0x100000000000; +/// Unmount in async context. +pub const MNT_DEFERRED: u64 = 0x200000000000; + +/// Get configured filesystems. +pub const VFS_VFSCONF: ::c_int = 0; +/// Generic filesystem information. +pub const VFS_GENERIC: ::c_int = 0; + +/// int: highest defined filesystem type. +pub const VFS_MAXTYPENUM: ::c_int = 1; +/// struct: vfsconf for filesystem given as next argument. +pub const VFS_CONF: ::c_int = 2; + +/// Synchronously wait for I/O to complete. +pub const MNT_WAIT: ::c_int = 1; +/// Start all I/O, but do not wait for it. +pub const MNT_NOWAIT: ::c_int = 2; +/// Push data not written by filesystem syncer. +pub const MNT_LAZY: ::c_int = 3; +/// Suspend file system after sync. +pub const MNT_SUSPEND: ::c_int = 4; + +pub const MAXSECFLAVORS: ::c_int = 5; + +/// Statically compiled into kernel. +pub const VFCF_STATIC: ::c_int = 0x00010000; +/// May get data over the network. +pub const VFCF_NETWORK: ::c_int = 0x00020000; +/// Writes are not implemented. +pub const VFCF_READONLY: ::c_int = 0x00040000; +/// Data does not represent real files. +pub const VFCF_SYNTHETIC: ::c_int = 0x00080000; +/// Aliases some other mounted FS. +pub const VFCF_LOOPBACK: ::c_int = 0x00100000; +/// Stores file names as Unicode. +pub const VFCF_UNICODE: ::c_int = 0x00200000; +/// Can be mounted from within a jail. +pub const VFCF_JAIL: ::c_int = 0x00400000; +/// Supports delegated administration. +pub const VFCF_DELEGADMIN: ::c_int = 0x00800000; +/// Stop at Boundary: defer stop requests to kernel->user (AST) transition. +pub const VFCF_SBDRY: ::c_int = 0x01000000; + +// time.h + +/// not on dst +pub const DST_NONE: ::c_int = 0; +/// USA style dst +pub const DST_USA: ::c_int = 1; +/// Australian style dst +pub const DST_AUST: ::c_int = 2; +/// Western European dst +pub const DST_WET: ::c_int = 3; +/// Middle European dst +pub const DST_MET: ::c_int = 4; +/// Eastern European dst +pub const DST_EET: ::c_int = 5; +/// Canada +pub const DST_CAN: ::c_int = 6; + +pub const CPUCLOCK_WHICH_PID: ::c_int = 0; +pub const CPUCLOCK_WHICH_TID: ::c_int = 1; + +pub const MFD_CLOEXEC: ::c_uint = 0x00000001; +pub const MFD_ALLOW_SEALING: ::c_uint = 0x00000002; +pub const MFD_HUGETLB: ::c_uint = 0x00000004; +pub const MFD_HUGE_MASK: ::c_uint = 0xFC000000; +pub const MFD_HUGE_64KB: ::c_uint = 16 << 26; +pub const MFD_HUGE_512KB: ::c_uint = 19 << 26; +pub const MFD_HUGE_1MB: ::c_uint = 20 << 26; +pub const MFD_HUGE_2MB: ::c_uint = 21 << 26; +pub const MFD_HUGE_8MB: ::c_uint = 23 << 26; +pub const MFD_HUGE_16MB: ::c_uint = 24 << 26; +pub const MFD_HUGE_32MB: ::c_uint = 25 << 26; +pub const MFD_HUGE_256MB: ::c_uint = 28 << 26; +pub const MFD_HUGE_512MB: ::c_uint = 29 << 26; +pub const MFD_HUGE_1GB: ::c_uint = 30 << 26; +pub const MFD_HUGE_2GB: ::c_uint = 31 << 26; +pub const MFD_HUGE_16GB: ::c_uint = 34 << 26; + +pub const SHM_LARGEPAGE_ALLOC_DEFAULT: ::c_int = 0; +pub const SHM_LARGEPAGE_ALLOC_NOWAIT: ::c_int = 1; +pub const SHM_LARGEPAGE_ALLOC_HARD: ::c_int = 2; +pub const SHM_RENAME_NOREPLACE: ::c_int = 1 << 0; +pub const SHM_RENAME_EXCHANGE: ::c_int = 1 << 1; + +// sys/umtx.h + +pub const UMTX_OP_WAIT: ::c_int = 2; +pub const UMTX_OP_WAKE: ::c_int = 3; +pub const UMTX_OP_MUTEX_TRYLOCK: ::c_int = 4; +pub const UMTX_OP_MUTEX_LOCK: ::c_int = 5; +pub const UMTX_OP_MUTEX_UNLOCK: ::c_int = 6; +pub const UMTX_OP_SET_CEILING: ::c_int = 7; +pub const UMTX_OP_CV_WAIT: ::c_int = 8; +pub const UMTX_OP_CV_SIGNAL: ::c_int = 9; +pub const UMTX_OP_CV_BROADCAST: ::c_int = 10; +pub const UMTX_OP_WAIT_UINT: ::c_int = 11; +pub const UMTX_OP_RW_RDLOCK: ::c_int = 12; +pub const UMTX_OP_RW_WRLOCK: ::c_int = 13; +pub const UMTX_OP_RW_UNLOCK: ::c_int = 14; +pub const UMTX_OP_WAIT_UINT_PRIVATE: ::c_int = 15; +pub const UMTX_OP_WAKE_PRIVATE: ::c_int = 16; +pub const UMTX_OP_MUTEX_WAIT: ::c_int = 17; +pub const UMTX_OP_NWAKE_PRIVATE: ::c_int = 21; +pub const UMTX_OP_MUTEX_WAKE2: ::c_int = 22; +pub const UMTX_OP_SEM2_WAIT: ::c_int = 23; +pub const UMTX_OP_SEM2_WAKE: ::c_int = 24; +pub const UMTX_OP_SHM: ::c_int = 25; +pub const UMTX_OP_ROBUST_LISTS: ::c_int = 26; + +pub const UMTX_ABSTIME: u32 = 1; + +pub const CPU_LEVEL_ROOT: ::c_int = 1; +pub const CPU_LEVEL_CPUSET: ::c_int = 2; +pub const CPU_LEVEL_WHICH: ::c_int = 3; + +pub const CPU_WHICH_TID: ::c_int = 1; +pub const CPU_WHICH_PID: ::c_int = 2; +pub const CPU_WHICH_CPUSET: ::c_int = 3; +pub const CPU_WHICH_IRQ: ::c_int = 4; +pub const CPU_WHICH_JAIL: ::c_int = 5; + +const_fn! { + {const} fn _ALIGN(p: usize) -> usize { + (p + _ALIGNBYTES) & !_ALIGNBYTES + } } f! { @@ -1131,25 +3826,149 @@ f! { } } - pub fn CMSG_SPACE(length: ::c_uint) -> ::c_uint { + pub {const} fn CMSG_SPACE(length: ::c_uint) -> ::c_uint { (_ALIGN(::mem::size_of::<::cmsghdr>()) + _ALIGN(length as usize)) as ::c_uint } + pub fn MALLOCX_ALIGN(lg: ::c_uint) -> ::c_int { + ffsl(lg as ::c_long - 1) + } + + pub {const} fn MALLOCX_TCACHE(tc: ::c_int) -> ::c_int { + (tc + 2) << 8 as ::c_int + } + + pub {const} fn MALLOCX_ARENA(a: ::c_int) -> ::c_int { + (a + 1) << 20 as ::c_int + } + + pub fn SOCKCREDSIZE(ngrps: usize) -> usize { + let ngrps = if ngrps > 0 { + ngrps - 1 + } else { + 0 + }; + ::mem::size_of::() + ::mem::size_of::<::gid_t>() * ngrps + } + pub fn uname(buf: *mut ::utsname) -> ::c_int { __xuname(256, buf as *mut ::c_void) } + + pub fn CPU_ZERO(cpuset: &mut cpuset_t) -> () { + for slot in cpuset.__bits.iter_mut() { + *slot = 0; + } + } + + pub fn CPU_FILL(cpuset: &mut cpuset_t) -> () { + for slot in cpuset.__bits.iter_mut() { + *slot = !0; + } + } + + pub fn CPU_SET(cpu: usize, cpuset: &mut cpuset_t) -> () { + let bitset_bits = 8 * ::mem::size_of::<::c_long>(); + let (idx, offset) = (cpu / bitset_bits, cpu % bitset_bits); + cpuset.__bits[idx] |= 1 << offset; + () + } + + pub fn CPU_CLR(cpu: usize, cpuset: &mut cpuset_t) -> () { + let bitset_bits = 8 * ::mem::size_of::<::c_long>(); + let (idx, offset) = (cpu / bitset_bits, cpu % bitset_bits); + cpuset.__bits[idx] &= !(1 << offset); + () + } + + pub fn CPU_ISSET(cpu: usize, cpuset: &cpuset_t) -> bool { + let bitset_bits = 8 * ::mem::size_of::<::c_long>(); + let (idx, offset) = (cpu / bitset_bits, cpu % bitset_bits); + 0 != cpuset.__bits[idx] & (1 << offset) + } + + pub fn CPU_COUNT(cpuset: &cpuset_t) -> ::c_int { + let mut s: u32 = 0; + let cpuset_size = ::mem::size_of::(); + let bitset_size = ::mem::size_of::<::c_long>(); + + for i in cpuset.__bits[..(cpuset_size / bitset_size)].iter() { + s += i.count_ones(); + }; + s as ::c_int + } + + pub fn SOCKCRED2SIZE(ngrps: usize) -> usize { + let ngrps = if ngrps > 0 { + ngrps - 1 + } else { + 0 + }; + ::mem::size_of::() + ::mem::size_of::<::gid_t>() * ngrps + } +} + +safe_f! { + pub {const} fn WIFSIGNALED(status: ::c_int) -> bool { + (status & 0o177) != 0o177 && (status & 0o177) != 0 && status != 0x13 + } +} + +cfg_if! { + if #[cfg(not(any(freebsd10, freebsd11)))] { + extern "C" { + pub fn fhlink(fhp: *mut fhandle_t, to: *const ::c_char) -> ::c_int; + pub fn fhlinkat(fhp: *mut fhandle_t, tofd: ::c_int, to: *const ::c_char) -> ::c_int; + pub fn fhreadlink( + fhp: *mut fhandle_t, + buf: *mut ::c_char, + bufsize: ::size_t, + ) -> ::c_int; + pub fn getfhat( + fd: ::c_int, + path: *mut ::c_char, + fhp: *mut fhandle, + flag: ::c_int, + ) -> ::c_int; + } + } } extern "C" { + #[cfg_attr(doc, doc(alias = "__errno_location"))] + #[cfg_attr(doc, doc(alias = "errno"))] pub fn __error() -> *mut ::c_int; - pub fn clock_getres(clk_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; - pub fn clock_gettime(clk_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; - pub fn clock_settime( - clk_id: ::clockid_t, - tp: *const ::timespec, + pub fn aio_cancel(fd: ::c_int, aiocbp: *mut aiocb) -> ::c_int; + pub fn aio_error(aiocbp: *const aiocb) -> ::c_int; + pub fn aio_fsync(op: ::c_int, aiocbp: *mut aiocb) -> ::c_int; + pub fn aio_read(aiocbp: *mut aiocb) -> ::c_int; + pub fn aio_readv(aiocbp: *mut ::aiocb) -> ::c_int; + pub fn aio_return(aiocbp: *mut aiocb) -> ::ssize_t; + pub fn aio_suspend( + aiocb_list: *const *const aiocb, + nitems: ::c_int, + timeout: *const ::timespec, ) -> ::c_int; + pub fn aio_write(aiocbp: *mut aiocb) -> ::c_int; + pub fn aio_writev(aiocbp: *mut ::aiocb) -> ::c_int; + + pub fn copy_file_range( + infd: ::c_int, + inoffp: *mut ::off_t, + outfd: ::c_int, + outoffp: *mut ::off_t, + len: ::size_t, + flags: ::c_uint, + ) -> ::ssize_t; + + pub fn devname_r( + dev: ::dev_t, + mode: ::mode_t, + buf: *mut ::c_char, + len: ::c_int, + ) -> *mut ::c_char; pub fn extattr_delete_fd( fd: ::c_int, @@ -1227,46 +4046,34 @@ extern "C" { nbytes: ::size_t, ) -> ::ssize_t; + pub fn fspacectl( + fd: ::c_int, + cmd: ::c_int, + rqsr: *const spacectl_range, + flags: ::c_int, + rmsr: *mut spacectl_range, + ) -> ::c_int; + pub fn jail(jail: *mut ::jail) -> ::c_int; pub fn jail_attach(jid: ::c_int) -> ::c_int; pub fn jail_remove(jid: ::c_int) -> ::c_int; - pub fn jail_get( - iov: *mut ::iovec, - niov: ::c_uint, - flags: ::c_int, - ) -> ::c_int; - pub fn jail_set( - iov: *mut ::iovec, - niov: ::c_uint, - flags: ::c_int, - ) -> ::c_int; + pub fn jail_get(iov: *mut ::iovec, niov: ::c_uint, flags: ::c_int) -> ::c_int; + pub fn jail_set(iov: *mut ::iovec, niov: ::c_uint, flags: ::c_int) -> ::c_int; - pub fn fdatasync(fd: ::c_int) -> ::c_int; - pub fn posix_fallocate( - fd: ::c_int, - offset: ::off_t, - len: ::off_t, - ) -> ::c_int; - pub fn posix_fadvise( - fd: ::c_int, - offset: ::off_t, - len: ::off_t, - advise: ::c_int, + pub fn lio_listio( + mode: ::c_int, + aiocb_list: *const *mut aiocb, + nitems: ::c_int, + sevp: *mut sigevent, ) -> ::c_int; + pub fn mkostemp(template: *mut ::c_char, flags: ::c_int) -> ::c_int; - pub fn mkostemps( - template: *mut ::c_char, - suffixlen: ::c_int, - flags: ::c_int, - ) -> ::c_int; + pub fn mkostemps(template: *mut ::c_char, suffixlen: ::c_int, flags: ::c_int) -> ::c_int; pub fn getutxuser(user: *const ::c_char) -> *mut utmpx; pub fn setutxdb(_type: ::c_int, file: *const ::c_char) -> ::c_int; - pub fn aio_waitcomplete( - iocbp: *mut *mut aiocb, - timeout: *mut ::timespec, - ) -> ::ssize_t; + pub fn aio_waitcomplete(iocbp: *mut *mut aiocb, timeout: *mut ::timespec) -> ::ssize_t; pub fn mq_getfd_np(mqd: ::mqd_t) -> ::c_int; pub fn waitid( @@ -1275,25 +4082,17 @@ extern "C" { infop: *mut ::siginfo_t, options: ::c_int, ) -> ::c_int; + pub fn ptsname_r(fd: ::c_int, buf: *mut ::c_char, buflen: ::size_t) -> ::c_int; pub fn ftok(pathname: *const ::c_char, proj_id: ::c_int) -> ::key_t; pub fn shmget(key: ::key_t, size: ::size_t, shmflg: ::c_int) -> ::c_int; - pub fn shmat( - shmid: ::c_int, - shmaddr: *const ::c_void, - shmflg: ::c_int, - ) -> *mut ::c_void; + pub fn shmat(shmid: ::c_int, shmaddr: *const ::c_void, shmflg: ::c_int) -> *mut ::c_void; pub fn shmdt(shmaddr: *const ::c_void) -> ::c_int; - pub fn shmctl( - shmid: ::c_int, - cmd: ::c_int, - buf: *mut ::shmid_ds, - ) -> ::c_int; - pub fn msgctl( - msqid: ::c_int, - cmd: ::c_int, - buf: *mut ::msqid_ds, - ) -> ::c_int; + pub fn shmctl(shmid: ::c_int, cmd: ::c_int, buf: *mut ::shmid_ds) -> ::c_int; + pub fn semget(key: ::key_t, nsems: ::c_int, semflg: ::c_int) -> ::c_int; + pub fn semctl(semid: ::c_int, semnum: ::c_int, cmd: ::c_int, ...) -> ::c_int; + pub fn semop(semid: ::c_int, sops: *mut sembuf, nsops: ::size_t) -> ::c_int; + pub fn msgctl(msqid: ::c_int, cmd: ::c_int, buf: *mut ::msqid_ds) -> ::c_int; pub fn msgget(key: ::key_t, msgflg: ::c_int) -> ::c_int; pub fn msgsnd( msqid: ::c_int, @@ -1302,21 +4101,12 @@ extern "C" { msgflg: ::c_int, ) -> ::c_int; pub fn cfmakesane(termios: *mut ::termios); - pub fn fexecve( - fd: ::c_int, - argv: *const *const ::c_char, - envp: *const *const ::c_char, - ) -> ::c_int; pub fn pdfork(fdp: *mut ::c_int, flags: ::c_int) -> ::pid_t; pub fn pdgetpid(fd: ::c_int, pidp: *mut ::pid_t) -> ::c_int; pub fn pdkill(fd: ::c_int, signum: ::c_int) -> ::c_int; - pub fn rtprio_thread( - function: ::c_int, - lwpid: ::lwpid_t, - rtp: *mut super::rtprio, - ) -> ::c_int; + pub fn rtprio_thread(function: ::c_int, lwpid: ::lwpid_t, rtp: *mut super::rtprio) -> ::c_int; pub fn posix_spawn( pid: *mut ::pid_t, @@ -1356,26 +4146,17 @@ extern "C" { attr: *const posix_spawnattr_t, flags: *mut ::c_short, ) -> ::c_int; - pub fn posix_spawnattr_setflags( - attr: *mut posix_spawnattr_t, - flags: ::c_short, - ) -> ::c_int; + pub fn posix_spawnattr_setflags(attr: *mut posix_spawnattr_t, flags: ::c_short) -> ::c_int; pub fn posix_spawnattr_getpgroup( attr: *const posix_spawnattr_t, flags: *mut ::pid_t, ) -> ::c_int; - pub fn posix_spawnattr_setpgroup( - attr: *mut posix_spawnattr_t, - flags: ::pid_t, - ) -> ::c_int; + pub fn posix_spawnattr_setpgroup(attr: *mut posix_spawnattr_t, flags: ::pid_t) -> ::c_int; pub fn posix_spawnattr_getschedpolicy( attr: *const posix_spawnattr_t, flags: *mut ::c_int, ) -> ::c_int; - pub fn posix_spawnattr_setschedpolicy( - attr: *mut posix_spawnattr_t, - flags: ::c_int, - ) -> ::c_int; + pub fn posix_spawnattr_setschedpolicy(attr: *mut posix_spawnattr_t, flags: ::c_int) -> ::c_int; pub fn posix_spawnattr_getschedparam( attr: *const posix_spawnattr_t, param: *mut ::sched_param, @@ -1385,12 +4166,8 @@ extern "C" { param: *const ::sched_param, ) -> ::c_int; - pub fn posix_spawn_file_actions_init( - actions: *mut posix_spawn_file_actions_t, - ) -> ::c_int; - pub fn posix_spawn_file_actions_destroy( - actions: *mut posix_spawn_file_actions_t, - ) -> ::c_int; + pub fn posix_spawn_file_actions_init(actions: *mut posix_spawn_file_actions_t) -> ::c_int; + pub fn posix_spawn_file_actions_destroy(actions: *mut posix_spawn_file_actions_t) -> ::c_int; pub fn posix_spawn_file_actions_addopen( actions: *mut posix_spawn_file_actions_t, fd: ::c_int, @@ -1408,15 +4185,52 @@ extern "C" { newfd: ::c_int, ) -> ::c_int; - #[cfg_attr( - all(target_os = "freebsd", freebsd11), - link_name = "statfs@FBSD_1.0" - )] + pub fn uuidgen(store: *mut uuid, count: ::c_int) -> ::c_int; + + pub fn thr_kill(id: ::c_long, sig: ::c_int) -> ::c_int; + pub fn thr_kill2(pid: ::pid_t, id: ::c_long, sig: ::c_int) -> ::c_int; + pub fn thr_self(tid: *mut ::c_long) -> ::c_int; + pub fn pthread_getthreadid_np() -> ::c_int; + pub fn pthread_getaffinity_np( + td: ::pthread_t, + cpusetsize: ::size_t, + cpusetp: *mut cpuset_t, + ) -> ::c_int; + pub fn pthread_setaffinity_np( + td: ::pthread_t, + cpusetsize: ::size_t, + cpusetp: *const cpuset_t, + ) -> ::c_int; + + // sched.h linux compatibility api + pub fn sched_getaffinity(pid: ::pid_t, cpusetsz: ::size_t, cpuset: *mut ::cpuset_t) -> ::c_int; + pub fn sched_setaffinity( + pid: ::pid_t, + cpusetsz: ::size_t, + cpuset: *const ::cpuset_t, + ) -> ::c_int; + pub fn sched_getcpu() -> ::c_int; + + pub fn pthread_mutex_consistent(mutex: *mut ::pthread_mutex_t) -> ::c_int; + + pub fn pthread_mutexattr_getrobust( + attr: *mut ::pthread_mutexattr_t, + robust: *mut ::c_int, + ) -> ::c_int; + pub fn pthread_mutexattr_setrobust( + attr: *mut ::pthread_mutexattr_t, + robust: ::c_int, + ) -> ::c_int; + + pub fn pthread_spin_init(lock: *mut pthread_spinlock_t, pshared: ::c_int) -> ::c_int; + pub fn pthread_spin_destroy(lock: *mut pthread_spinlock_t) -> ::c_int; + pub fn pthread_spin_lock(lock: *mut pthread_spinlock_t) -> ::c_int; + pub fn pthread_spin_trylock(lock: *mut pthread_spinlock_t) -> ::c_int; + pub fn pthread_spin_unlock(lock: *mut pthread_spinlock_t) -> ::c_int; + + #[cfg_attr(all(target_os = "freebsd", freebsd11), link_name = "statfs@FBSD_1.0")] pub fn statfs(path: *const ::c_char, buf: *mut statfs) -> ::c_int; - #[cfg_attr( - all(target_os = "freebsd", freebsd11), - link_name = "fstatfs@FBSD_1.0" - )] + #[cfg_attr(all(target_os = "freebsd", freebsd11), link_name = "fstatfs@FBSD_1.0")] pub fn fstatfs(fd: ::c_int, buf: *mut statfs) -> ::c_int; pub fn dup3(src: ::c_int, dst: ::c_int, flags: ::c_int) -> ::c_int; @@ -1441,6 +4255,195 @@ extern "C" { needle: *const ::c_void, needlelen: ::size_t, ) -> *mut ::c_void; + + pub fn fhopen(fhp: *const fhandle_t, flags: ::c_int) -> ::c_int; + pub fn fhstat(fhp: *const fhandle, buf: *mut ::stat) -> ::c_int; + pub fn fhstatfs(fhp: *const fhandle_t, buf: *mut ::statfs) -> ::c_int; + pub fn getfh(path: *const ::c_char, fhp: *mut fhandle_t) -> ::c_int; + pub fn lgetfh(path: *const ::c_char, fhp: *mut fhandle_t) -> ::c_int; + pub fn getfsstat(buf: *mut ::statfs, bufsize: ::c_long, mode: ::c_int) -> ::c_int; + #[cfg_attr( + all(target_os = "freebsd", freebsd11), + link_name = "getmntinfo@FBSD_1.0" + )] + pub fn getmntinfo(mntbufp: *mut *mut ::statfs, mode: ::c_int) -> ::c_int; + pub fn mount( + type_: *const ::c_char, + dir: *const ::c_char, + flags: ::c_int, + data: *mut ::c_void, + ) -> ::c_int; + pub fn nmount(iov: *mut ::iovec, niov: ::c_uint, flags: ::c_int) -> ::c_int; + + pub fn setproctitle(fmt: *const ::c_char, ...); + pub fn rfork(flags: ::c_int) -> ::c_int; + pub fn cpuset_getaffinity( + level: cpulevel_t, + which: cpuwhich_t, + id: ::id_t, + setsize: ::size_t, + mask: *mut cpuset_t, + ) -> ::c_int; + pub fn cpuset_setaffinity( + level: cpulevel_t, + which: cpuwhich_t, + id: ::id_t, + setsize: ::size_t, + mask: *const cpuset_t, + ) -> ::c_int; + pub fn cpuset(setid: *mut ::cpusetid_t) -> ::c_int; + pub fn cpuset_getid( + level: cpulevel_t, + which: cpuwhich_t, + id: ::id_t, + setid: *mut ::cpusetid_t, + ) -> ::c_int; + pub fn cpuset_setid(which: cpuwhich_t, id: ::id_t, setid: ::cpusetid_t) -> ::c_int; + pub fn cap_enter() -> ::c_int; + pub fn cap_getmode(modep: *mut ::c_uint) -> ::c_int; + pub fn cap_fcntls_get(fd: ::c_int, fcntlrightsp: *mut u32) -> ::c_int; + pub fn cap_fcntls_limit(fd: ::c_int, fcntlrights: u32) -> ::c_int; + pub fn cap_ioctls_get(fd: ::c_int, cmds: *mut u_long, maxcmds: usize) -> isize; + pub fn cap_ioctls_limit(fd: ::c_int, cmds: *const u_long, ncmds: usize) -> ::c_int; + pub fn __cap_rights_init(version: ::c_int, rights: *mut cap_rights_t, ...) + -> *mut cap_rights_t; + pub fn __cap_rights_get(version: ::c_int, fd: ::c_int, rightsp: *mut cap_rights_t) -> ::c_int; + pub fn __cap_rights_set(rights: *mut cap_rights_t, ...) -> *mut cap_rights_t; + pub fn __cap_rights_clear(rights: *mut cap_rights_t, ...) -> *mut cap_rights_t; + pub fn __cap_rights_is_set(rights: *const cap_rights_t, ...) -> bool; + pub fn cap_rights_is_valid(rights: *const cap_rights_t) -> bool; + pub fn cap_rights_limit(fd: ::c_int, rights: *const cap_rights_t) -> ::c_int; + pub fn cap_rights_merge(dst: *mut cap_rights_t, src: *const cap_rights_t) -> *mut cap_rights_t; + pub fn cap_rights_remove(dst: *mut cap_rights_t, src: *const cap_rights_t) + -> *mut cap_rights_t; + pub fn cap_rights_contains(big: *const cap_rights_t, little: *const cap_rights_t) -> bool; + pub fn cap_sandboxed() -> bool; + + pub fn reallocarray(ptr: *mut ::c_void, nmemb: ::size_t, size: ::size_t) -> *mut ::c_void; + + pub fn ffs(value: ::c_int) -> ::c_int; + pub fn ffsl(value: ::c_long) -> ::c_int; + pub fn ffsll(value: ::c_longlong) -> ::c_int; + pub fn fls(value: ::c_int) -> ::c_int; + pub fn flsl(value: ::c_long) -> ::c_int; + pub fn flsll(value: ::c_longlong) -> ::c_int; + pub fn malloc_usable_size(ptr: *const ::c_void) -> ::size_t; + pub fn malloc_stats_print( + write_cb: unsafe extern "C" fn(*mut ::c_void, *const ::c_char), + cbopaque: *mut ::c_void, + opt: *const ::c_char, + ); + pub fn mallctl( + name: *const ::c_char, + oldp: *mut ::c_void, + oldlenp: *mut ::size_t, + newp: *mut ::c_void, + newlen: ::size_t, + ) -> ::c_int; + pub fn mallctlnametomib( + name: *const ::c_char, + mibp: *mut ::size_t, + miplen: *mut ::size_t, + ) -> ::c_int; + pub fn mallctlbymib( + mib: *const ::size_t, + mible: ::size_t, + oldp: *mut ::c_void, + oldlenp: *mut ::size_t, + newp: *mut ::c_void, + newlen: ::size_t, + ) -> ::c_int; + pub fn mallocx(size: ::size_t, flags: ::c_int) -> *mut ::c_void; + pub fn rallocx(ptr: *mut ::c_void, size: ::size_t, flags: ::c_int) -> *mut ::c_void; + pub fn xallocx(ptr: *mut ::c_void, size: ::size_t, extra: ::size_t, flags: ::c_int) + -> ::size_t; + pub fn sallocx(ptr: *const ::c_void, flags: ::c_int) -> ::size_t; + pub fn dallocx(ptr: *mut ::c_void, flags: ::c_int); + pub fn sdallocx(ptr: *mut ::c_void, size: ::size_t, flags: ::c_int); + pub fn nallocx(size: ::size_t, flags: ::c_int) -> ::size_t; + + pub fn procctl(idtype: ::idtype_t, id: ::id_t, cmd: ::c_int, data: *mut ::c_void) -> ::c_int; + + pub fn getpagesize() -> ::c_int; + pub fn getpagesizes(pagesize: *mut ::size_t, nelem: ::c_int) -> ::c_int; + + pub fn clock_getcpuclockid2(arg1: ::id_t, arg2: ::c_int, arg3: *mut clockid_t) -> ::c_int; + + pub fn shm_create_largepage( + path: *const ::c_char, + flags: ::c_int, + psind: ::c_int, + alloc_policy: ::c_int, + mode: ::mode_t, + ) -> ::c_int; + pub fn shm_rename( + path_from: *const ::c_char, + path_to: *const ::c_char, + flags: ::c_int, + ) -> ::c_int; + pub fn memfd_create(name: *const ::c_char, flags: ::c_uint) -> ::c_int; + pub fn setaudit(auditinfo: *const auditinfo_t) -> ::c_int; + + pub fn _umtx_op( + obj: *mut ::c_void, + op: ::c_int, + val: ::c_ulong, + uaddr: *mut ::c_void, + uaddr2: *mut ::c_void, + ) -> ::c_int; +} + +#[link(name = "memstat")] +extern "C" { + pub fn memstat_strerror(error: ::c_int) -> *const ::c_char; + pub fn memstat_mtl_alloc() -> *mut memory_type_list; + pub fn memstat_mtl_first(list: *mut memory_type_list) -> *mut memory_type; + pub fn memstat_mtl_next(mtp: *mut memory_type) -> *mut memory_type; + pub fn memstat_mtl_find( + list: *mut memory_type_list, + allocator: ::c_int, + name: *const ::c_char, + ) -> *mut memory_type; + pub fn memstat_mtl_free(list: *mut memory_type_list); + pub fn memstat_mtl_geterror(list: *mut memory_type_list) -> ::c_int; + pub fn memstat_get_name(mtp: *const memory_type) -> *const ::c_char; +} + +#[link(name = "kvm")] +extern "C" { + pub fn kvm_dpcpu_setcpu(kd: *mut ::kvm_t, cpu: ::c_uint) -> ::c_int; + pub fn kvm_getargv(kd: *mut ::kvm_t, p: *const kinfo_proc, nchr: ::c_int) + -> *mut *mut ::c_char; + pub fn kvm_getcptime(kd: *mut ::kvm_t, cp_time: *mut ::c_long) -> ::c_int; + pub fn kvm_getenvv(kd: *mut ::kvm_t, p: *const kinfo_proc, nchr: ::c_int) + -> *mut *mut ::c_char; + pub fn kvm_geterr(kd: *mut ::kvm_t) -> *mut ::c_char; + pub fn kvm_getmaxcpu(kd: *mut ::kvm_t) -> ::c_int; + pub fn kvm_getncpus(kd: *mut ::kvm_t) -> ::c_int; + pub fn kvm_getpcpu(kd: *mut ::kvm_t, cpu: ::c_int) -> *mut ::c_void; + pub fn kvm_counter_u64_fetch(kd: *mut ::kvm_t, base: ::c_ulong) -> u64; + pub fn kvm_getswapinfo( + kd: *mut ::kvm_t, + info: *mut kvm_swap, + maxswap: ::c_int, + flags: ::c_int, + ) -> ::c_int; + pub fn kvm_native(kd: *mut ::kvm_t) -> ::c_int; + pub fn kvm_nlist(kd: *mut ::kvm_t, nl: *mut nlist) -> ::c_int; + pub fn kvm_nlist2(kd: *mut ::kvm_t, nl: *mut kvm_nlist) -> ::c_int; + pub fn kvm_read_zpcpu( + kd: *mut ::kvm_t, + base: ::c_ulong, + buf: *mut ::c_void, + size: ::size_t, + cpu: ::c_int, + ) -> ::ssize_t; + pub fn kvm_read2( + kd: *mut ::kvm_t, + addr: kvaddr_t, + buf: *mut ::c_void, + nbytes: ::size_t, + ) -> ::ssize_t; } #[link(name = "util")] @@ -1453,13 +4456,178 @@ extern "C" { string: *const ::c_char, attrnamespace: *mut ::c_int, ) -> ::c_int; + pub fn realhostname(host: *mut ::c_char, hsize: ::size_t, ip: *const ::in_addr) -> ::c_int; + pub fn realhostname_sa( + host: *mut ::c_char, + hsize: ::size_t, + addr: *mut ::sockaddr, + addrlen: ::c_int, + ) -> ::c_int; + + pub fn kld_isloaded(name: *const ::c_char) -> ::c_int; + pub fn kld_load(name: *const ::c_char) -> ::c_int; + + pub fn kinfo_getvmmap(pid: ::pid_t, cntp: *mut ::c_int) -> *mut kinfo_vmentry; + + pub fn hexdump(ptr: *const ::c_void, length: ::c_int, hdr: *const ::c_char, flags: ::c_int); + pub fn humanize_number( + buf: *mut ::c_char, + len: ::size_t, + number: i64, + suffix: *const ::c_char, + scale: ::c_int, + flags: ::c_int, + ) -> ::c_int; + + pub fn flopen(path: *const ::c_char, flags: ::c_int, ...) -> ::c_int; + pub fn flopenat(fd: ::c_int, path: *const ::c_char, flags: ::c_int, ...) -> ::c_int; + + pub fn getlocalbase() -> *const ::c_char; +} + +#[link(name = "procstat")] +extern "C" { + pub fn procstat_open_sysctl() -> *mut procstat; + pub fn procstat_getfiles( + procstat: *mut procstat, + kp: *mut kinfo_proc, + mmapped: ::c_int, + ) -> *mut filestat_list; + pub fn procstat_freefiles(procstat: *mut procstat, head: *mut filestat_list); + pub fn procstat_getprocs( + procstat: *mut procstat, + what: ::c_int, + arg: ::c_int, + count: *mut ::c_uint, + ) -> *mut kinfo_proc; + pub fn procstat_freeprocs(procstat: *mut procstat, p: *mut kinfo_proc); + pub fn procstat_getvmmap( + procstat: *mut procstat, + kp: *mut kinfo_proc, + count: *mut ::c_uint, + ) -> *mut kinfo_vmentry; + pub fn procstat_freevmmap(procstat: *mut procstat, vmmap: *mut kinfo_vmentry); + pub fn procstat_close(procstat: *mut procstat); + pub fn procstat_freeargv(procstat: *mut procstat); + pub fn procstat_freeenvv(procstat: *mut procstat); + pub fn procstat_freegroups(procstat: *mut procstat, groups: *mut ::gid_t); + pub fn procstat_freeptlwpinfo(procstat: *mut procstat, pl: *mut ptrace_lwpinfo); + pub fn procstat_getargv( + procstat: *mut procstat, + kp: *mut kinfo_proc, + nchr: ::size_t, + ) -> *mut *mut ::c_char; + pub fn procstat_getenvv( + procstat: *mut procstat, + kp: *mut kinfo_proc, + nchr: ::size_t, + ) -> *mut *mut ::c_char; + pub fn procstat_getgroups( + procstat: *mut procstat, + kp: *mut kinfo_proc, + count: *mut ::c_uint, + ) -> *mut ::gid_t; + pub fn procstat_getosrel( + procstat: *mut procstat, + kp: *mut kinfo_proc, + osrelp: *mut ::c_int, + ) -> ::c_int; + pub fn procstat_getpathname( + procstat: *mut procstat, + kp: *mut kinfo_proc, + pathname: *mut ::c_char, + maxlen: ::size_t, + ) -> ::c_int; + pub fn procstat_getrlimit( + procstat: *mut procstat, + kp: *mut kinfo_proc, + which: ::c_int, + rlimit: *mut ::rlimit, + ) -> ::c_int; + pub fn procstat_getumask( + procstat: *mut procstat, + kp: *mut kinfo_proc, + maskp: *mut ::c_ushort, + ) -> ::c_int; + pub fn procstat_open_core(filename: *const ::c_char) -> *mut procstat; + pub fn procstat_open_kvm(nlistf: *const ::c_char, memf: *const ::c_char) -> *mut procstat; + pub fn procstat_get_socket_info( + proc_: *mut procstat, + fst: *mut filestat, + sock: *mut sockstat, + errbuf: *mut ::c_char, + ) -> ::c_int; + pub fn procstat_get_vnode_info( + proc_: *mut procstat, + fst: *mut filestat, + vn: *mut vnstat, + errbuf: *mut ::c_char, + ) -> ::c_int; + pub fn procstat_get_pts_info( + proc_: *mut procstat, + fst: *mut filestat, + pts: *mut ptsstat, + errbuf: *mut ::c_char, + ) -> ::c_int; + pub fn procstat_get_shm_info( + proc_: *mut procstat, + fst: *mut filestat, + shm: *mut shmstat, + errbuf: *mut ::c_char, + ) -> ::c_int; +} + +#[link(name = "rt")] +extern "C" { + pub fn timer_create(clock_id: clockid_t, evp: *mut sigevent, timerid: *mut timer_t) -> ::c_int; + pub fn timer_delete(timerid: timer_t) -> ::c_int; + pub fn timer_getoverrun(timerid: timer_t) -> ::c_int; + pub fn timer_gettime(timerid: timer_t, value: *mut itimerspec) -> ::c_int; + pub fn timer_settime( + timerid: timer_t, + flags: ::c_int, + value: *const itimerspec, + ovalue: *mut itimerspec, + ) -> ::c_int; +} + +#[link(name = "devstat")] +extern "C" { + pub fn devstat_getnumdevs(kd: *mut ::kvm_t) -> ::c_int; + pub fn devstat_getgeneration(kd: *mut ::kvm_t) -> ::c_long; + pub fn devstat_getversion(kd: *mut ::kvm_t) -> ::c_int; + pub fn devstat_checkversion(kd: *mut ::kvm_t) -> ::c_int; + pub fn devstat_selectdevs( + dev_select: *mut *mut device_selection, + num_selected: *mut ::c_int, + num_selections: *mut ::c_int, + select_generation: *mut ::c_long, + current_generation: ::c_long, + devices: *mut devstat, + numdevs: ::c_int, + matches: *mut devstat_match, + num_matches: ::c_int, + dev_selections: *mut *mut ::c_char, + num_dev_selections: ::c_int, + select_mode: devstat_select_mode, + maxshowdevs: ::c_int, + perf_select: ::c_int, + ) -> ::c_int; + pub fn devstat_buildmatch( + match_str: *mut ::c_char, + matches: *mut *mut devstat_match, + num_matches: *mut ::c_int, + ) -> ::c_int; } cfg_if! { - if #[cfg(freebsd12)] { - mod freebsd12; - pub use self::freebsd12::*; + if #[cfg(freebsd14)] { + mod freebsd14; + pub use self::freebsd14::*; } else if #[cfg(freebsd13)] { + mod freebsd13; + pub use self::freebsd13::*; + } else if #[cfg(freebsd12)] { mod freebsd12; pub use self::freebsd12::*; } else if #[cfg(any(freebsd10, freebsd11))] { @@ -1486,6 +4654,12 @@ cfg_if! { } else if #[cfg(target_arch = "powerpc64")] { mod powerpc64; pub use self::powerpc64::*; + } else if #[cfg(target_arch = "powerpc")] { + mod powerpc; + pub use self::powerpc::*; + } else if #[cfg(target_arch = "riscv64")] { + mod riscv64; + pub use self::riscv64::*; } else { // Unknown target_arch } diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/powerpc.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/powerpc.rs new file mode 100644 index 000000000..a0120c337 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/powerpc.rs @@ -0,0 +1,47 @@ +pub type c_char = u8; +pub type c_long = i32; +pub type c_ulong = u32; +pub type wchar_t = i32; +pub type time_t = i64; +pub type suseconds_t = i32; +pub type register_t = i32; + +s! { + pub struct stat { + pub st_dev: ::dev_t, + pub st_ino: ::ino_t, + pub st_mode: ::mode_t, + pub st_nlink: ::nlink_t, + pub st_uid: ::uid_t, + pub st_gid: ::gid_t, + pub st_rdev: ::dev_t, + pub st_atime: ::time_t, + pub st_atime_nsec: ::c_long, + pub st_mtime: ::time_t, + pub st_mtime_nsec: ::c_long, + pub st_ctime: ::time_t, + pub st_ctime_nsec: ::c_long, + pub st_size: ::off_t, + pub st_blocks: ::blkcnt_t, + pub st_blksize: ::blksize_t, + pub st_flags: ::fflags_t, + pub st_gen: u32, + pub st_lspare: i32, + pub st_birthtime: ::time_t, + pub st_birthtime_nsec: ::c_long, + } +} + +// should be pub(crate), but that requires Rust 1.18.0 +cfg_if! { + if #[cfg(libc_const_size_of)] { + #[doc(hidden)] + pub const _ALIGNBYTES: usize = ::mem::size_of::<::c_int>() - 1; + } else { + #[doc(hidden)] + pub const _ALIGNBYTES: usize = 4 - 1; + } +} + +pub const MAP_32BIT: ::c_int = 0x00080000; +pub const MINSIGSTKSZ: ::size_t = 2048; // 512 * 4 diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/powerpc64.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/powerpc64.rs index 5c0c6e7f3..7f5b97522 100644 --- a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/powerpc64.rs +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/powerpc64.rs @@ -1,8 +1,10 @@ pub type c_char = u8; pub type c_long = i64; pub type c_ulong = u64; +pub type wchar_t = i32; pub type time_t = i64; pub type suseconds_t = i64; +pub type register_t = i64; s! { pub struct stat { @@ -42,3 +44,4 @@ cfg_if! { } pub const MAP_32BIT: ::c_int = 0x00080000; +pub const MINSIGSTKSZ: ::size_t = 2048; // 512 * 4 diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/riscv64.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/riscv64.rs new file mode 100644 index 000000000..f9fa1c275 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/riscv64.rs @@ -0,0 +1,154 @@ +pub type c_char = u8; +pub type c_long = i64; +pub type c_ulong = u64; +pub type wchar_t = ::c_int; +pub type time_t = i64; +pub type suseconds_t = ::c_long; +pub type register_t = i64; + +s_no_extra_traits! { + pub struct gpregs { + pub gp_ra: ::register_t, + pub gp_sp: ::register_t, + pub gp_gp: ::register_t, + pub gp_tp: ::register_t, + pub gp_t: [::register_t; 7], + pub gp_s: [::register_t; 12], + pub gp_a: [::register_t; 8], + pub gp_sepc: ::register_t, + pub gp_sstatus: ::register_t, + } + + pub struct fpregs { + pub fp_x: [[::register_t; 2]; 32], + pub fp_fcsr: ::register_t, + pub fp_flags: ::c_int, + pub fp_pad: ::c_int, + } + + pub struct mcontext_t { + pub mc_gpregs: gpregs, + pub mc_fpregs: fpregs, + pub mc_flags: ::c_int, + pub mc_pad: ::c_int, + pub mc_spare: [u64; 8], + } +} + +// should be pub(crate), but that requires Rust 1.18.0 +cfg_if! { + if #[cfg(libc_const_size_of)] { + #[doc(hidden)] + pub const _ALIGNBYTES: usize = ::mem::size_of::<::c_longlong>() - 1; + } else { + #[doc(hidden)] + pub const _ALIGNBYTES: usize = 8 - 1; + } +} + +cfg_if! { + if #[cfg(feature = "extra_traits")] { + impl PartialEq for gpregs { + fn eq(&self, other: &gpregs) -> bool { + self.gp_ra == other.gp_ra && + self.gp_sp == other.gp_sp && + self.gp_gp == other.gp_gp && + self.gp_tp == other.gp_tp && + self.gp_t.iter().zip(other.gp_t.iter()).all(|(a, b)| a == b) && + self.gp_s.iter().zip(other.gp_s.iter()).all(|(a, b)| a == b) && + self.gp_a.iter().zip(other.gp_a.iter()).all(|(a, b)| a == b) && + self.gp_sepc == other.gp_sepc && + self.gp_sstatus == other.gp_sstatus + } + } + impl Eq for gpregs {} + impl ::fmt::Debug for gpregs { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("gpregs") + .field("gp_ra", &self.gp_ra) + .field("gp_sp", &self.gp_sp) + .field("gp_gp", &self.gp_gp) + .field("gp_tp", &self.gp_tp) + .field("gp_t", &self.gp_t) + .field("gp_s", &self.gp_s) + .field("gp_a", &self.gp_a) + .field("gp_sepc", &self.gp_sepc) + .field("gp_sstatus", &self.gp_sstatus) + .finish() + } + } + impl ::hash::Hash for gpregs { + fn hash(&self, state: &mut H) { + self.gp_ra.hash(state); + self.gp_sp.hash(state); + self.gp_gp.hash(state); + self.gp_tp.hash(state); + self.gp_t.hash(state); + self.gp_s.hash(state); + self.gp_a.hash(state); + self.gp_sepc.hash(state); + self.gp_sstatus.hash(state); + } + } + impl PartialEq for fpregs { + fn eq(&self, other: &fpregs) -> bool { + self.fp_x == other.fp_x && + self.fp_fcsr == other.fp_fcsr && + self.fp_flags == other.fp_flags && + self.fp_pad == other.fp_pad + } + } + impl Eq for fpregs {} + impl ::fmt::Debug for fpregs { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("fpregs") + .field("fp_x", &self.fp_x) + .field("fp_fcsr", &self.fp_fcsr) + .field("fp_flags", &self.fp_flags) + .field("fp_pad", &self.fp_pad) + .finish() + } + } + impl ::hash::Hash for fpregs { + fn hash(&self, state: &mut H) { + self.fp_x.hash(state); + self.fp_fcsr.hash(state); + self.fp_flags.hash(state); + self.fp_pad.hash(state); + } + } + impl PartialEq for mcontext_t { + fn eq(&self, other: &mcontext_t) -> bool { + self.mc_gpregs == other.mc_gpregs && + self.mc_fpregs == other.mc_fpregs && + self.mc_flags == other.mc_flags && + self.mc_pad == other.mc_pad && + self.mc_spare.iter().zip(other.mc_spare.iter()).all(|(a, b)| a == b) + } + } + impl Eq for mcontext_t {} + impl ::fmt::Debug for mcontext_t { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("mcontext_t") + .field("mc_gpregs", &self.mc_gpregs) + .field("mc_fpregs", &self.mc_fpregs) + .field("mc_flags", &self.mc_flags) + .field("mc_pad", &self.mc_pad) + .field("mc_spare", &self.mc_spare) + .finish() + } + } + impl ::hash::Hash for mcontext_t { + fn hash(&self, state: &mut H) { + self.mc_gpregs.hash(state); + self.mc_fpregs.hash(state); + self.mc_flags.hash(state); + self.mc_pad.hash(state); + self.mc_spare.hash(state); + } + } + } +} + +pub const MAP_32BIT: ::c_int = 0x00080000; +pub const MINSIGSTKSZ: ::size_t = 4096; // 1024 * 4 diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/x86.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/x86.rs index adec88cb5..d3a3f34b0 100644 --- a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/x86.rs +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/x86.rs @@ -1,8 +1,10 @@ pub type c_char = i8; pub type c_long = i32; pub type c_ulong = u32; +pub type wchar_t = i32; pub type time_t = i32; pub type suseconds_t = i32; +pub type register_t = i32; s! { pub struct stat { @@ -38,6 +40,7 @@ cfg_if! { pub const _ALIGNBYTES: usize = ::mem::size_of::<::c_long>() - 1; } else { #[doc(hidden)] - pub const _ALIGNBYTES: usize = 8 - 1; + pub const _ALIGNBYTES: usize = 4 - 1; } } +pub const MINSIGSTKSZ: ::size_t = 2048; // 512 * 4 diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/x86_64/align.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/x86_64/align.rs index 7ca870fd0..3a016a051 100644 --- a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/x86_64/align.rs +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/x86_64/align.rs @@ -1,7 +1,197 @@ +use {c_long, register_t}; + s_no_extra_traits! { #[allow(missing_debug_implementations)] #[repr(align(16))] pub struct max_align_t { priv_: [f64; 4] } + + #[repr(align(16))] + pub struct mcontext_t { + pub mc_onstack: register_t, + pub mc_rdi: register_t, + pub mc_rsi: register_t, + pub mc_rdx: register_t, + pub mc_rcx: register_t, + pub mc_r8: register_t, + pub mc_r9: register_t, + pub mc_rax: register_t, + pub mc_rbx: register_t, + pub mc_rbp: register_t, + pub mc_r10: register_t, + pub mc_r11: register_t, + pub mc_r12: register_t, + pub mc_r13: register_t, + pub mc_r14: register_t, + pub mc_r15: register_t, + pub mc_trapno: u32, + pub mc_fs: u16, + pub mc_gs: u16, + pub mc_addr: register_t, + pub mc_flags: u32, + pub mc_es: u16, + pub mc_ds: u16, + pub mc_err: register_t, + pub mc_rip: register_t, + pub mc_cs: register_t, + pub mc_rflags: register_t, + pub mc_rsp: register_t, + pub mc_ss: register_t, + pub mc_len: c_long, + pub mc_fpformat: c_long, + pub mc_ownedfp: c_long, + pub mc_fpstate: [c_long; 64], + pub mc_fsbase: register_t, + pub mc_gsbase: register_t, + pub mc_xfpustate: register_t, + pub mc_xfpustate_len: register_t, + pub mc_spare: [c_long; 4], + } +} + +cfg_if! { + if #[cfg(feature = "extra_traits")] { + impl PartialEq for mcontext_t { + fn eq(&self, other: &mcontext_t) -> bool { + self.mc_onstack == other.mc_onstack && + self.mc_rdi == other.mc_rdi && + self.mc_rsi == other.mc_rsi && + self.mc_rdx == other.mc_rdx && + self.mc_rcx == other.mc_rcx && + self.mc_r8 == other.mc_r8 && + self.mc_r9 == other.mc_r9 && + self.mc_rax == other.mc_rax && + self.mc_rbx == other.mc_rbx && + self.mc_rbp == other.mc_rbp && + self.mc_r10 == other.mc_r10 && + self.mc_r11 == other.mc_r11 && + self.mc_r12 == other.mc_r12 && + self.mc_r13 == other.mc_r13 && + self.mc_r14 == other.mc_r14 && + self.mc_r15 == other.mc_r15 && + self.mc_trapno == other.mc_trapno && + self.mc_fs == other.mc_fs && + self.mc_gs == other.mc_gs && + self.mc_addr == other.mc_addr && + self.mc_flags == other.mc_flags && + self.mc_es == other.mc_es && + self.mc_ds == other.mc_ds && + self.mc_err == other.mc_err && + self.mc_rip == other.mc_rip && + self.mc_cs == other.mc_cs && + self.mc_rflags == other.mc_rflags && + self.mc_rsp == other.mc_rsp && + self.mc_ss == other.mc_ss && + self.mc_len == other.mc_len && + self.mc_fpformat == other.mc_fpformat && + self.mc_ownedfp == other.mc_ownedfp && + self.mc_fpstate.iter().zip(other.mc_fpstate.iter()) + .all(|(a, b)| a == b) && + self.mc_fsbase == other.mc_fsbase && + self.mc_gsbase == other.mc_gsbase && + self.mc_xfpustate == other.mc_xfpustate && + self.mc_xfpustate_len == other.mc_xfpustate_len && + self.mc_spare == other.mc_spare + } + } + impl Eq for mcontext_t {} + impl ::fmt::Debug for mcontext_t { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("mcontext_t") + .field("mc_onstack", &self.mc_onstack) + .field("mc_rdi", &self.mc_rdi) + .field("mc_rsi", &self.mc_rsi) + .field("mc_rdx", &self.mc_rdx) + .field("mc_rcx", &self.mc_rcx) + .field("mc_r8", &self.mc_r8) + .field("mc_r9", &self.mc_r9) + .field("mc_rax", &self.mc_rax) + .field("mc_rbx", &self.mc_rbx) + .field("mc_rbp", &self.mc_rbp) + .field("mc_r10", &self.mc_r10) + .field("mc_r11", &self.mc_r11) + .field("mc_r12", &self.mc_r12) + .field("mc_r13", &self.mc_r13) + .field("mc_r14", &self.mc_r14) + .field("mc_r15", &self.mc_r15) + .field("mc_trapno", &self.mc_trapno) + .field("mc_fs", &self.mc_fs) + .field("mc_gs", &self.mc_gs) + .field("mc_addr", &self.mc_addr) + .field("mc_flags", &self.mc_flags) + .field("mc_es", &self.mc_es) + .field("mc_ds", &self.mc_ds) + .field("mc_err", &self.mc_err) + .field("mc_rip", &self.mc_rip) + .field("mc_cs", &self.mc_cs) + .field("mc_rflags", &self.mc_rflags) + .field("mc_rsp", &self.mc_rsp) + .field("mc_ss", &self.mc_ss) + .field("mc_len", &self.mc_len) + .field("mc_fpformat", &self.mc_fpformat) + .field("mc_ownedfp", &self.mc_ownedfp) + // FIXME: .field("mc_fpstate", &self.mc_fpstate) + .field("mc_fsbase", &self.mc_fsbase) + .field("mc_gsbase", &self.mc_gsbase) + .field("mc_xfpustate", &self.mc_xfpustate) + .field("mc_xfpustate_len", &self.mc_xfpustate_len) + .field("mc_spare", &self.mc_spare) + .finish() + } + } + impl ::hash::Hash for mcontext_t { + fn hash(&self, state: &mut H) { + self.mc_onstack.hash(state); + self.mc_rdi.hash(state); + self.mc_rsi.hash(state); + self.mc_rdx.hash(state); + self.mc_rcx.hash(state); + self.mc_r8.hash(state); + self.mc_r9.hash(state); + self.mc_rax.hash(state); + self.mc_rbx.hash(state); + self.mc_rbp.hash(state); + self.mc_r10.hash(state); + self.mc_r11.hash(state); + self.mc_r12.hash(state); + self.mc_r13.hash(state); + self.mc_r14.hash(state); + self.mc_r15.hash(state); + self.mc_trapno.hash(state); + self.mc_fs.hash(state); + self.mc_gs.hash(state); + self.mc_addr.hash(state); + self.mc_flags.hash(state); + self.mc_es.hash(state); + self.mc_ds.hash(state); + self.mc_err.hash(state); + self.mc_rip.hash(state); + self.mc_cs.hash(state); + self.mc_rflags.hash(state); + self.mc_rsp.hash(state); + self.mc_ss.hash(state); + self.mc_len.hash(state); + self.mc_fpformat.hash(state); + self.mc_ownedfp.hash(state); + self.mc_fpstate.hash(state); + self.mc_fsbase.hash(state); + self.mc_gsbase.hash(state); + self.mc_xfpustate.hash(state); + self.mc_xfpustate_len.hash(state); + self.mc_spare.hash(state); + } + } + } +} + +s! { + pub struct ucontext_t { + pub uc_sigmask: ::sigset_t, + pub uc_mcontext: ::mcontext_t, + pub uc_link: *mut ::ucontext_t, + pub uc_stack: ::stack_t, + pub uc_flags: ::c_int, + __spare__: [::c_int; 4], + } } diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/x86_64/mod.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/x86_64/mod.rs index 0769a22f8..ae1fcf781 100644 --- a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/x86_64/mod.rs +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/freebsd/x86_64/mod.rs @@ -1,8 +1,306 @@ pub type c_char = i8; pub type c_long = i64; pub type c_ulong = u64; +pub type wchar_t = i32; pub type time_t = i64; pub type suseconds_t = i64; +pub type register_t = i64; + +s! { + pub struct reg32 { + pub r_fs: u32, + pub r_es: u32, + pub r_ds: u32, + pub r_edi: u32, + pub r_esi: u32, + pub r_ebp: u32, + pub r_isp: u32, + pub r_ebx: u32, + pub r_edx: u32, + pub r_ecx: u32, + pub r_eax: u32, + pub r_trapno: u32, + pub r_err: u32, + pub r_eip: u32, + pub r_cs: u32, + pub r_eflags: u32, + pub r_esp: u32, + pub r_ss: u32, + pub r_gs: u32, + } + + pub struct reg { + pub r_r15: i64, + pub r_r14: i64, + pub r_r13: i64, + pub r_r12: i64, + pub r_r11: i64, + pub r_r10: i64, + pub r_r9: i64, + pub r_r8: i64, + pub r_rdi: i64, + pub r_rsi: i64, + pub r_rbp: i64, + pub r_rbx: i64, + pub r_rdx: i64, + pub r_rcx: i64, + pub r_rax: i64, + pub r_trapno: u32, + pub r_fs: u16, + pub r_gs: u16, + pub r_err: u32, + pub r_es: u16, + pub r_ds: u16, + pub r_rip: i64, + pub r_cs: i64, + pub r_rflags: i64, + pub r_rsp: i64, + pub r_ss: i64, + } +} + +s_no_extra_traits! { + pub struct fpreg32 { + pub fpr_env: [u32; 7], + pub fpr_acc: [[u8; 10]; 8], + pub fpr_ex_sw: u32, + pub fpr_pad: [u8; 64], + } + + pub struct fpreg { + pub fpr_env: [u64; 4], + pub fpr_acc: [[u8; 16]; 8], + pub fpr_xacc: [[u8; 16]; 16], + pub fpr_spare: [u64; 12], + } + + pub struct xmmreg { + pub xmm_env: [u32; 8], + pub xmm_acc: [[u8; 16]; 8], + pub xmm_reg: [[u8; 16]; 8], + pub xmm_pad: [u8; 224], + } + + #[cfg(libc_union)] + pub union __c_anonymous_elf64_auxv_union { + pub a_val: ::c_long, + pub a_ptr: *mut ::c_void, + pub a_fcn: extern "C" fn(), + } + + pub struct Elf64_Auxinfo { + pub a_type: ::c_long, + #[cfg(libc_union)] + pub a_un: __c_anonymous_elf64_auxv_union, + } + + pub struct kinfo_file { + pub kf_structsize: ::c_int, + pub kf_type: ::c_int, + pub kf_fd: ::c_int, + pub kf_ref_count: ::c_int, + pub kf_flags: ::c_int, + _kf_pad0: ::c_int, + pub kf_offset: i64, + _priv: [::uintptr_t; 38], // FIXME if needed + pub kf_status: u16, + _kf_pad1: u16, + _kf_ispare0: ::c_int, + pub kf_cap_rights: ::cap_rights_t, + _kf_cap_spare: u64, + pub kf_path: [::c_char; ::PATH_MAX as usize], + } +} + +cfg_if! { + if #[cfg(feature = "extra_traits")] { + impl PartialEq for fpreg32 { + fn eq(&self, other: &fpreg32) -> bool { + self.fpr_env == other.fpr_env && + self.fpr_acc == other.fpr_acc && + self.fpr_ex_sw == other.fpr_ex_sw && + self.fpr_pad + .iter() + .zip(other.fpr_pad.iter()) + .all(|(a,b)| a == b) + } + } + impl Eq for fpreg32 {} + impl ::fmt::Debug for fpreg32 { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("fpreg32") + .field("fpr_env", &&self.fpr_env[..]) + .field("fpr_acc", &self.fpr_acc) + .field("fpr_ex_sw", &self.fpr_ex_sw) + .field("fpr_pad", &&self.fpr_pad[..]) + .finish() + } + } + impl ::hash::Hash for fpreg32 { + fn hash(&self, state: &mut H) { + self.fpr_env.hash(state); + self.fpr_acc.hash(state); + self.fpr_ex_sw.hash(state); + self.fpr_pad.hash(state); + } + } + + impl PartialEq for fpreg { + fn eq(&self, other: &fpreg) -> bool { + self.fpr_env == other.fpr_env && + self.fpr_acc == other.fpr_acc && + self.fpr_xacc == other.fpr_xacc && + self.fpr_spare == other.fpr_spare + } + } + impl Eq for fpreg {} + impl ::fmt::Debug for fpreg { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("fpreg") + .field("fpr_env", &self.fpr_env) + .field("fpr_acc", &self.fpr_acc) + .field("fpr_xacc", &self.fpr_xacc) + .field("fpr_spare", &self.fpr_spare) + .finish() + } + } + impl ::hash::Hash for fpreg { + fn hash(&self, state: &mut H) { + self.fpr_env.hash(state); + self.fpr_acc.hash(state); + self.fpr_xacc.hash(state); + self.fpr_spare.hash(state); + } + } + + impl PartialEq for xmmreg { + fn eq(&self, other: &xmmreg) -> bool { + self.xmm_env == other.xmm_env && + self.xmm_acc == other.xmm_acc && + self.xmm_reg == other.xmm_reg && + self.xmm_pad + .iter() + .zip(other.xmm_pad.iter()) + .all(|(a,b)| a == b) + } + } + impl Eq for xmmreg {} + impl ::fmt::Debug for xmmreg { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("xmmreg") + .field("xmm_env", &self.xmm_env) + .field("xmm_acc", &self.xmm_acc) + .field("xmm_reg", &self.xmm_reg) + .field("xmm_pad", &&self.xmm_pad[..]) + .finish() + } + } + impl ::hash::Hash for xmmreg { + fn hash(&self, state: &mut H) { + self.xmm_env.hash(state); + self.xmm_acc.hash(state); + self.xmm_reg.hash(state); + self.xmm_pad.hash(state); + } + } + + #[cfg(libc_union)] + impl PartialEq for __c_anonymous_elf64_auxv_union { + fn eq(&self, other: &__c_anonymous_elf64_auxv_union) -> bool { + unsafe { self.a_val == other.a_val + || self.a_ptr == other.a_ptr + || self.a_fcn == other.a_fcn } + } + } + #[cfg(libc_union)] + impl Eq for __c_anonymous_elf64_auxv_union {} + #[cfg(libc_union)] + impl ::fmt::Debug for __c_anonymous_elf64_auxv_union { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("a_val") + .field("a_val", unsafe { &self.a_val }) + .finish() + } + } + #[cfg(not(libc_union))] + impl PartialEq for Elf64_Auxinfo { + fn eq(&self, other: &Elf64_Auxinfo) -> bool { + self.a_type == other.a_type + } + } + #[cfg(libc_union)] + impl PartialEq for Elf64_Auxinfo { + fn eq(&self, other: &Elf64_Auxinfo) -> bool { + self.a_type == other.a_type + && self.a_un == other.a_un + } + } + impl Eq for Elf64_Auxinfo {} + #[cfg(not(libc_union))] + impl ::fmt::Debug for Elf64_Auxinfo { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("Elf64_Auxinfo") + .field("a_type", &self.a_type) + .finish() + } + } + #[cfg(libc_union)] + impl ::fmt::Debug for Elf64_Auxinfo { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("Elf64_Auxinfo") + .field("a_type", &self.a_type) + .field("a_un", &self.a_un) + .finish() + } + } + + impl PartialEq for kinfo_file { + fn eq(&self, other: &kinfo_file) -> bool { + self.kf_structsize == other.kf_structsize && + self.kf_type == other.kf_type && + self.kf_fd == other.kf_fd && + self.kf_ref_count == other.kf_ref_count && + self.kf_flags == other.kf_flags && + self.kf_offset == other.kf_offset && + self.kf_status == other.kf_status && + self.kf_cap_rights == other.kf_cap_rights && + self.kf_path + .iter() + .zip(other.kf_path.iter()) + .all(|(a,b)| a == b) + } + } + impl Eq for kinfo_file {} + impl ::fmt::Debug for kinfo_file { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("kinfo_file") + .field("kf_structsize", &self.kf_structsize) + .field("kf_type", &self.kf_type) + .field("kf_fd", &self.kf_fd) + .field("kf_ref_count", &self.kf_ref_count) + .field("kf_flags", &self.kf_flags) + .field("kf_offset", &self.kf_offset) + .field("kf_status", &self.kf_status) + .field("kf_cap_rights", &self.kf_cap_rights) + .field("kf_path", &&self.kf_path[..]) + .finish() + } + } + impl ::hash::Hash for kinfo_file { + fn hash(&self, state: &mut H) { + self.kf_structsize.hash(state); + self.kf_type.hash(state); + self.kf_fd.hash(state); + self.kf_ref_count.hash(state); + self.kf_flags.hash(state); + self.kf_offset.hash(state); + self.kf_status.hash(state); + self.kf_cap_rights.hash(state); + self.kf_path.hash(state); + } + } + } +} // should be pub(crate), but that requires Rust 1.18.0 cfg_if! { @@ -15,6 +313,18 @@ cfg_if! { } } pub const MAP_32BIT: ::c_int = 0x00080000; +pub const MINSIGSTKSZ: ::size_t = 2048; // 512 * 4 + +pub const _MC_HASSEGS: u32 = 0x1; +pub const _MC_HASBASES: u32 = 0x2; +pub const _MC_HASFPXSTATE: u32 = 0x4; +pub const _MC_FLAG_MASK: u32 = _MC_HASSEGS | _MC_HASBASES | _MC_HASFPXSTATE; + +pub const _MC_FPFMT_NODEV: c_long = 0x10000; +pub const _MC_FPFMT_XMM: c_long = 0x10002; +pub const _MC_FPOWNED_NONE: c_long = 0x20000; +pub const _MC_FPOWNED_FPU: c_long = 0x20001; +pub const _MC_FPOWNED_PCB: c_long = 0x20002; cfg_if! { if #[cfg(libc_align)] { diff --git a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/mod.rs b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/mod.rs index d0510466d..9aefb36e4 100644 --- a/crux-mir/lib/libc/src/unix/bsd/freebsdlike/mod.rs +++ b/crux-mir/lib/libc/src/unix/bsd/freebsdlike/mod.rs @@ -13,6 +13,45 @@ pub type speed_t = ::c_uint; pub type nl_item = ::c_int; pub type id_t = i64; pub type vm_size_t = ::uintptr_t; +pub type key_t = ::c_long; + +// elf.h + +pub type Elf32_Addr = u32; +pub type Elf32_Half = u16; +pub type Elf32_Lword = u64; +pub type Elf32_Off = u32; +pub type Elf32_Sword = i32; +pub type Elf32_Word = u32; + +pub type Elf64_Addr = u64; +pub type Elf64_Half = u16; +pub type Elf64_Lword = u64; +pub type Elf64_Off = u64; +pub type Elf64_Sword = i32; +pub type Elf64_Sxword = i64; +pub type Elf64_Word = u32; +pub type Elf64_Xword = u64; + +pub type iconv_t = *mut ::c_void; + +// It's an alias over "struct __kvm_t". However, its fields aren't supposed to be used directly, +// making the type definition system dependent. Better not bind it exactly. +pub type kvm_t = ::c_void; + +cfg_if! { + if #[cfg(target_pointer_width = "64")] { + type Elf_Addr = Elf64_Addr; + type Elf_Half = Elf64_Half; + type Elf_Phdr = Elf64_Phdr; + } else if #[cfg(target_pointer_width = "32")] { + type Elf_Addr = Elf32_Addr; + type Elf_Half = Elf32_Half; + type Elf_Phdr = Elf32_Phdr; + } +} + +// link.h #[cfg_attr(feature = "extra_traits", derive(Debug))] pub enum timezone {} @@ -31,6 +70,18 @@ impl siginfo_t { pub unsafe fn si_value(&self) -> ::sigval { self.si_value } + + pub unsafe fn si_pid(&self) -> ::pid_t { + self.si_pid + } + + pub unsafe fn si_uid(&self) -> ::uid_t { + self.si_uid + } + + pub unsafe fn si_status(&self) -> ::c_int { + self.si_status + } } s! { @@ -43,6 +94,18 @@ s! { pub imr_interface: in_addr, } + pub struct ip_mreqn { + pub imr_multiaddr: in_addr, + pub imr_address: in_addr, + pub imr_ifindex: ::c_int, + } + + pub struct ip_mreq_source { + pub imr_multiaddr: in_addr, + pub imr_sourceaddr: in_addr, + pub imr_interface: in_addr, + } + pub struct glob_t { pub gl_pathc: ::size_t, pub gl_matchc: ::size_t, @@ -218,6 +281,105 @@ s! { pub tai: ::c_long, pub time_state: ::c_int, } + + pub struct accept_filter_arg { + pub af_name: [::c_char; 16], + af_arg: [[::c_char; 10]; 24], + } + + pub struct ptrace_io_desc { + pub piod_op: ::c_int, + pub piod_offs: *mut ::c_void, + pub piod_addr: *mut ::c_void, + pub piod_len: ::size_t, + } + + // bpf.h + + pub struct bpf_program { + pub bf_len: ::c_uint, + pub bf_insns: *mut bpf_insn, + } + + pub struct bpf_stat { + pub bs_recv: ::c_uint, + pub bs_drop: ::c_uint, + } + + pub struct bpf_version { + pub bv_major: ::c_ushort, + pub bv_minor: ::c_ushort, + } + + pub struct bpf_hdr { + pub bh_tstamp: ::timeval, + pub bh_caplen: u32, + pub bh_datalen: u32, + pub bh_hdrlen: ::c_ushort, + } + + pub struct bpf_insn { + pub code: ::c_ushort, + pub jt: ::c_uchar, + pub jf: ::c_uchar, + pub k: u32, + } + + pub struct bpf_dltlist { + bfl_len: ::c_uint, + bfl_list: *mut ::c_uint, + } + + // elf.h + + pub struct Elf32_Phdr { + pub p_type: Elf32_Word, + pub p_offset: Elf32_Off, + pub p_vaddr: Elf32_Addr, + pub p_paddr: Elf32_Addr, + pub p_filesz: Elf32_Word, + pub p_memsz: Elf32_Word, + pub p_flags: Elf32_Word, + pub p_align: Elf32_Word, + } + + pub struct Elf64_Phdr { + pub p_type: Elf64_Word, + pub p_flags: Elf64_Word, + pub p_offset: Elf64_Off, + pub p_vaddr: Elf64_Addr, + pub p_paddr: Elf64_Addr, + pub p_filesz: Elf64_Xword, + pub p_memsz: Elf64_Xword, + pub p_align: Elf64_Xword, + } + + // link.h + + pub struct dl_phdr_info { + pub dlpi_addr: Elf_Addr, + pub dlpi_name: *const ::c_char, + pub dlpi_phdr: *const Elf_Phdr, + pub dlpi_phnum: Elf_Half, + pub dlpi_adds: ::c_ulonglong, + pub dlpi_subs: ::c_ulonglong, + pub dlpi_tls_modid: usize, + pub dlpi_tls_data: *mut ::c_void, + } + + pub struct ipc_perm { + pub cuid: ::uid_t, + pub cgid: ::gid_t, + pub uid: ::uid_t, + pub gid: ::gid_t, + pub mode: ::mode_t, + pub seq: ::c_ushort, + pub key: ::key_t, + } + + pub struct eui64 { + pub octet: [u8; EUI64_LEN], + } } s_no_extra_traits! { @@ -269,6 +431,17 @@ cfg_if! { } } +// Non-public helper constant +cfg_if! { + if #[cfg(all(not(libc_const_size_of), target_pointer_width = "32"))] { + const SIZEOF_LONG: usize = 4; + } else if #[cfg(all(not(libc_const_size_of), target_pointer_width = "64"))] { + const SIZEOF_LONG: usize = 8; + } else if #[cfg(libc_const_size_of)] { + const SIZEOF_LONG: usize = ::mem::size_of::<::c_long>(); + } +} + #[deprecated( since = "0.2.64", note = "Can vary at runtime. Use sysconf(3) instead" @@ -425,6 +598,8 @@ pub const F_TEST: ::c_int = 3; pub const F_TLOCK: ::c_int = 2; pub const F_ULOCK: ::c_int = 0; pub const F_DUPFD_CLOEXEC: ::c_int = 17; +pub const F_DUP2FD: ::c_int = 10; +pub const F_DUP2FD_CLOEXEC: ::c_int = 18; pub const SIGHUP: ::c_int = 1; pub const SIGINT: ::c_int = 2; pub const SIGQUIT: ::c_int = 3; @@ -455,6 +630,19 @@ pub const MAP_FAILED: *mut ::c_void = !0 as *mut ::c_void; pub const MCL_CURRENT: ::c_int = 0x0001; pub const MCL_FUTURE: ::c_int = 0x0002; +pub const MNT_EXPUBLIC: ::c_int = 0x20000000; +pub const MNT_NOATIME: ::c_int = 0x10000000; +pub const MNT_NOCLUSTERR: ::c_int = 0x40000000; +pub const MNT_NOCLUSTERW: ::c_int = 0x80000000; +pub const MNT_NOSYMFOLLOW: ::c_int = 0x00400000; +pub const MNT_SOFTDEP: ::c_int = 0x00200000; +pub const MNT_SUIDDIR: ::c_int = 0x00100000; +pub const MNT_EXRDONLY: ::c_int = 0x00000080; +pub const MNT_DEFEXPORTED: ::c_int = 0x00000200; +pub const MNT_EXPORTANON: ::c_int = 0x00000400; +pub const MNT_EXKERB: ::c_int = 0x00000800; +pub const MNT_DELEXPORT: ::c_int = 0x00020000; + pub const MS_SYNC: ::c_int = 0x0000; pub const MS_ASYNC: ::c_int = 0x0001; pub const MS_INVALIDATE: ::c_int = 0x0002; @@ -564,6 +752,14 @@ pub const POLLSTANDARD: ::c_short = ::POLLIN | ::POLLHUP | ::POLLNVAL; +pub const AI_PASSIVE: ::c_int = 0x00000001; +pub const AI_CANONNAME: ::c_int = 0x00000002; +pub const AI_NUMERICHOST: ::c_int = 0x00000004; +pub const AI_NUMERICSERV: ::c_int = 0x00000008; +pub const AI_ALL: ::c_int = 0x00000100; +pub const AI_ADDRCONFIG: ::c_int = 0x00000400; +pub const AI_V4MAPPED: ::c_int = 0x00000800; + pub const EAI_AGAIN: ::c_int = 2; pub const EAI_BADFLAGS: ::c_int = 3; pub const EAI_FAIL: ::c_int = 4; @@ -623,6 +819,21 @@ pub const RLIM_INFINITY: rlim_t = 0x7fff_ffff_ffff_ffff; pub const RUSAGE_SELF: ::c_int = 0; pub const RUSAGE_CHILDREN: ::c_int = -1; +pub const CLOCK_REALTIME: ::clockid_t = 0; +pub const CLOCK_VIRTUAL: ::clockid_t = 1; +pub const CLOCK_PROF: ::clockid_t = 2; +pub const CLOCK_MONOTONIC: ::clockid_t = 4; +pub const CLOCK_UPTIME: ::clockid_t = 5; +pub const CLOCK_UPTIME_PRECISE: ::clockid_t = 7; +pub const CLOCK_UPTIME_FAST: ::clockid_t = 8; +pub const CLOCK_REALTIME_PRECISE: ::clockid_t = 9; +pub const CLOCK_REALTIME_FAST: ::clockid_t = 10; +pub const CLOCK_MONOTONIC_PRECISE: ::clockid_t = 11; +pub const CLOCK_MONOTONIC_FAST: ::clockid_t = 12; +pub const CLOCK_SECOND: ::clockid_t = 13; +pub const CLOCK_THREAD_CPUTIME_ID: ::clockid_t = 14; +pub const CLOCK_PROCESS_CPUTIME_ID: ::clockid_t = 15; + pub const MADV_NORMAL: ::c_int = 0; pub const MADV_RANDOM: ::c_int = 1; pub const MADV_SEQUENTIAL: ::c_int = 2; @@ -639,7 +850,6 @@ pub const MINCORE_REFERENCED: ::c_int = 0x2; pub const MINCORE_MODIFIED: ::c_int = 0x4; pub const MINCORE_REFERENCED_OTHER: ::c_int = 0x8; pub const MINCORE_MODIFIED_OTHER: ::c_int = 0x10; -pub const MINCORE_SUPER: ::c_int = 0x20; pub const AF_UNSPEC: ::c_int = 0; pub const AF_LOCAL: ::c_int = 1; @@ -713,6 +923,11 @@ pub const PF_NATM: ::c_int = AF_NATM; pub const PF_ATM: ::c_int = AF_ATM; pub const PF_NETGRAPH: ::c_int = AF_NETGRAPH; +pub const PIOD_READ_D: ::c_int = 1; +pub const PIOD_WRITE_D: ::c_int = 2; +pub const PIOD_READ_I: ::c_int = 3; +pub const PIOD_WRITE_I: ::c_int = 4; + pub const PT_TRACE_ME: ::c_int = 0; pub const PT_READ_I: ::c_int = 1; pub const PT_READ_D: ::c_int = 2; @@ -738,6 +953,7 @@ pub const MSG_DONTWAIT: ::c_int = 0x00000080; pub const MSG_EOF: ::c_int = 0x00000100; pub const SCM_TIMESTAMP: ::c_int = 0x02; +pub const SCM_CREDS: ::c_int = 0x03; pub const SOCK_STREAM: ::c_int = 1; pub const SOCK_DGRAM: ::c_int = 2; @@ -756,10 +972,17 @@ pub const IP_DROP_MEMBERSHIP: ::c_int = 13; pub const IP_RECVIF: ::c_int = 20; pub const IPV6_JOIN_GROUP: ::c_int = 12; pub const IPV6_LEAVE_GROUP: ::c_int = 13; +pub const IPV6_CHECKSUM: ::c_int = 26; pub const IPV6_RECVPKTINFO: ::c_int = 36; pub const IPV6_PKTINFO: ::c_int = 46; +pub const IPV6_HOPLIMIT: ::c_int = 47; pub const IPV6_RECVTCLASS: ::c_int = 57; pub const IPV6_TCLASS: ::c_int = 61; +pub const IPV6_DONTFRAG: ::c_int = 62; +pub const IP_ADD_SOURCE_MEMBERSHIP: ::c_int = 70; +pub const IP_DROP_SOURCE_MEMBERSHIP: ::c_int = 71; +pub const IP_BLOCK_SOURCE: ::c_int = 72; +pub const IP_UNBLOCK_SOURCE: ::c_int = 73; pub const TCP_NOPUSH: ::c_int = 4; pub const TCP_NOOPT: ::c_int = 8; @@ -790,6 +1013,8 @@ pub const SO_RCVTIMEO: ::c_int = 0x1006; pub const SO_ERROR: ::c_int = 0x1007; pub const SO_TYPE: ::c_int = 0x1008; +pub const LOCAL_PEERCRED: ::c_int = 1; + pub const SHUT_RD: ::c_int = 0; pub const SHUT_WR: ::c_int = 1; pub const SHUT_RDWR: ::c_int = 2; @@ -801,10 +1026,16 @@ pub const LOCK_UN: ::c_int = 8; pub const MAP_COPY: ::c_int = 0x0002; #[doc(hidden)] -#[deprecated(since = "0.2.54", note = "Removed in FreeBSD 11")] +#[deprecated( + since = "0.2.54", + note = "Removed in FreeBSD 11, unused in DragonFlyBSD" +)] pub const MAP_RENAME: ::c_int = 0x0020; #[doc(hidden)] -#[deprecated(since = "0.2.54", note = "Removed in FreeBSD 11")] +#[deprecated( + since = "0.2.54", + note = "Removed in FreeBSD 11, unused in DragonFlyBSD" +)] pub const MAP_NORESERVE: ::c_int = 0x0040; pub const MAP_HASSEMAPHORE: ::c_int = 0x0200; pub const MAP_STACK: ::c_int = 0x0400; @@ -976,6 +1207,8 @@ pub const ST_NOSUID: ::c_ulong = 2; pub const NI_MAXHOST: ::size_t = 1025; +pub const XUCRED_VERSION: ::c_uint = 0; + pub const RTLD_LOCAL: ::c_int = 0; pub const RTLD_NODELETE: ::c_int = 0x1000; pub const RTLD_NOLOAD: ::c_int = 0x2000; @@ -986,30 +1219,30 @@ pub const LOG_SECURITY: ::c_int = 13 << 3; pub const LOG_CONSOLE: ::c_int = 14 << 3; pub const LOG_NFACILITIES: ::c_int = 24; -pub const TIOCEXCL: ::c_uint = 0x2000740d; -pub const TIOCNXCL: ::c_uint = 0x2000740e; +pub const TIOCEXCL: ::c_ulong = 0x2000740d; +pub const TIOCNXCL: ::c_ulong = 0x2000740e; pub const TIOCFLUSH: ::c_ulong = 0x80047410; -pub const TIOCGETA: ::c_uint = 0x402c7413; +pub const TIOCGETA: ::c_ulong = 0x402c7413; pub const TIOCSETA: ::c_ulong = 0x802c7414; pub const TIOCSETAW: ::c_ulong = 0x802c7415; pub const TIOCSETAF: ::c_ulong = 0x802c7416; -pub const TIOCGETD: ::c_uint = 0x4004741a; +pub const TIOCGETD: ::c_ulong = 0x4004741a; pub const TIOCSETD: ::c_ulong = 0x8004741b; -pub const TIOCGDRAINWAIT: ::c_uint = 0x40047456; +pub const TIOCGDRAINWAIT: ::c_ulong = 0x40047456; pub const TIOCSDRAINWAIT: ::c_ulong = 0x80047457; -pub const TIOCTIMESTAMP: ::c_uint = 0x40107459; -pub const TIOCMGDTRWAIT: ::c_uint = 0x4004745a; +pub const TIOCTIMESTAMP: ::c_ulong = 0x40107459; +pub const TIOCMGDTRWAIT: ::c_ulong = 0x4004745a; pub const TIOCMSDTRWAIT: ::c_ulong = 0x8004745b; -pub const TIOCDRAIN: ::c_uint = 0x2000745e; +pub const TIOCDRAIN: ::c_ulong = 0x2000745e; pub const TIOCEXT: ::c_ulong = 0x80047460; -pub const TIOCSCTTY: ::c_uint = 0x20007461; +pub const TIOCSCTTY: ::c_ulong = 0x20007461; pub const TIOCCONS: ::c_ulong = 0x80047462; -pub const TIOCGSID: ::c_uint = 0x40047463; -pub const TIOCSTAT: ::c_uint = 0x20007465; +pub const TIOCGSID: ::c_ulong = 0x40047463; +pub const TIOCSTAT: ::c_ulong = 0x20007465; pub const TIOCUCNTL: ::c_ulong = 0x80047466; pub const TIOCSWINSZ: ::c_ulong = 0x80087467; -pub const TIOCGWINSZ: ::c_uint = 0x40087468; -pub const TIOCMGET: ::c_uint = 0x4004746a; +pub const TIOCGWINSZ: ::c_ulong = 0x40087468; +pub const TIOCMGET: ::c_ulong = 0x4004746a; pub const TIOCM_LE: ::c_int = 0x1; pub const TIOCM_DTR: ::c_int = 0x2; pub const TIOCM_RTS: ::c_int = 0x4; @@ -1024,8 +1257,8 @@ pub const TIOCM_RNG: ::c_int = 0x80; pub const TIOCMBIC: ::c_ulong = 0x8004746b; pub const TIOCMBIS: ::c_ulong = 0x8004746c; pub const TIOCMSET: ::c_ulong = 0x8004746d; -pub const TIOCSTART: ::c_uint = 0x2000746e; -pub const TIOCSTOP: ::c_uint = 0x2000746f; +pub const TIOCSTART: ::c_ulong = 0x2000746e; +pub const TIOCSTOP: ::c_ulong = 0x2000746f; pub const TIOCPKT: ::c_ulong = 0x80047470; pub const TIOCPKT_DATA: ::c_int = 0x0; pub const TIOCPKT_FLUSHREAD: ::c_int = 0x1; @@ -1035,15 +1268,13 @@ pub const TIOCPKT_START: ::c_int = 0x8; pub const TIOCPKT_NOSTOP: ::c_int = 0x10; pub const TIOCPKT_DOSTOP: ::c_int = 0x20; pub const TIOCPKT_IOCTL: ::c_int = 0x40; -pub const TIOCNOTTY: ::c_uint = 0x20007471; +pub const TIOCNOTTY: ::c_ulong = 0x20007471; pub const TIOCSTI: ::c_ulong = 0x80017472; -pub const TIOCOUTQ: ::c_uint = 0x40047473; +pub const TIOCOUTQ: ::c_ulong = 0x40047473; pub const TIOCSPGRP: ::c_ulong = 0x80047476; -pub const TIOCGPGRP: ::c_uint = 0x40047477; -pub const TIOCCDTR: ::c_uint = 0x20007478; -pub const TIOCSDTR: ::c_uint = 0x20007479; -pub const TIOCCBRK: ::c_uint = 0x2000747a; -pub const TIOCSBRK: ::c_uint = 0x2000747b; +pub const TIOCGPGRP: ::c_ulong = 0x40047477; +pub const TIOCCDTR: ::c_ulong = 0x20007478; +pub const TIOCSDTR: ::c_ulong = 0x20007479; pub const TTYDISC: ::c_int = 0x0; pub const SLIPDISC: ::c_int = 0x4; pub const PPPDISC: ::c_int = 0x5; @@ -1061,7 +1292,6 @@ pub const BIOCGRTIMEOUT: ::c_ulong = 0x4010426e; pub const FIODTYPE: ::c_ulong = 0x4004667a; pub const FIOGETLBA: ::c_ulong = 0x40046679; -pub const FIODGNAME: ::c_ulong = 0x80106678; pub const B0: speed_t = 0; pub const B50: speed_t = 50; @@ -1104,9 +1334,10 @@ pub const ONLRET: ::tcflag_t = 0x40; pub const CMGROUP_MAX: usize = 16; +pub const EUI64_LEN: usize = 8; + // https://github.com/freebsd/freebsd/blob/master/sys/net/bpf.h -// sizeof(long) -pub const BPF_ALIGNMENT: ::c_int = 8; +pub const BPF_ALIGNMENT: usize = SIZEOF_LONG; // Values for rtprio struct (prio field) and syscall (function argument) pub const RTP_PRIO_MIN: ::c_ushort = 0; @@ -1181,31 +1412,42 @@ pub const TIME_OOP: ::c_int = 3; pub const TIME_WAIT: ::c_int = 4; pub const TIME_ERROR: ::c_int = 5; -f! { - pub fn WIFCONTINUED(status: ::c_int) -> bool { +pub const REG_ENOSYS: ::c_int = -1; +pub const REG_ILLSEQ: ::c_int = 17; + +pub const IPC_PRIVATE: ::key_t = 0; +pub const IPC_CREAT: ::c_int = 0o1000; +pub const IPC_EXCL: ::c_int = 0o2000; +pub const IPC_NOWAIT: ::c_int = 0o4000; +pub const IPC_RMID: ::c_int = 0; +pub const IPC_SET: ::c_int = 1; +pub const IPC_STAT: ::c_int = 2; +pub const IPC_R: ::c_int = 0o400; +pub const IPC_W: ::c_int = 0o200; +pub const IPC_M: ::c_int = 0o10000; + +pub const SHM_RDONLY: ::c_int = 0o10000; +pub const SHM_RND: ::c_int = 0o20000; +pub const SHM_R: ::c_int = 0o400; +pub const SHM_W: ::c_int = 0o200; + +safe_f! { + pub {const} fn WIFCONTINUED(status: ::c_int) -> bool { status == 0x13 } - pub fn WSTOPSIG(status: ::c_int) -> ::c_int { + pub {const} fn WSTOPSIG(status: ::c_int) -> ::c_int { status >> 8 } - pub fn WIFSIGNALED(status: ::c_int) -> bool { - (status & 0o177) != 0o177 && (status & 0o177) != 0 - } - - pub fn WIFSTOPPED(status: ::c_int) -> bool { + pub {const} fn WIFSTOPPED(status: ::c_int) -> bool { (status & 0o177) == 0o177 } } extern "C" { pub fn sem_destroy(sem: *mut sem_t) -> ::c_int; - pub fn sem_init( - sem: *mut sem_t, - pshared: ::c_int, - value: ::c_uint, - ) -> ::c_int; + pub fn sem_init(sem: *mut sem_t, pshared: ::c_int, value: ::c_uint) -> ::c_int; pub fn daemon(nochdir: ::c_int, noclose: ::c_int) -> ::c_int; pub fn gettimeofday(tp: *mut ::timeval, tz: *mut ::timezone) -> ::c_int; @@ -1215,17 +1457,6 @@ extern "C" { addrlen: *mut ::socklen_t, flags: ::c_int, ) -> ::c_int; - pub fn aio_read(aiocbp: *mut aiocb) -> ::c_int; - pub fn aio_write(aiocbp: *mut aiocb) -> ::c_int; - pub fn aio_fsync(op: ::c_int, aiocbp: *mut aiocb) -> ::c_int; - pub fn aio_error(aiocbp: *const aiocb) -> ::c_int; - pub fn aio_return(aiocbp: *mut aiocb) -> ::ssize_t; - pub fn aio_suspend( - aiocb_list: *const *const aiocb, - nitems: ::c_int, - timeout: *const ::timespec, - ) -> ::c_int; - pub fn aio_cancel(fd: ::c_int, aiocbp: *mut aiocb) -> ::c_int; pub fn chflags(path: *const ::c_char, flags: ::c_ulong) -> ::c_int; pub fn chflagsat( fd: ::c_int, @@ -1233,10 +1464,23 @@ extern "C" { flags: ::c_ulong, atflag: ::c_int, ) -> ::c_int; + + pub fn clock_getres(clk_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; + pub fn clock_gettime(clk_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; + pub fn clock_settime(clk_id: ::clockid_t, tp: *const ::timespec) -> ::c_int; + pub fn clock_getcpuclockid(pid: ::pid_t, clk_id: *mut ::clockid_t) -> ::c_int; + + pub fn pthread_getcpuclockid(thread: ::pthread_t, clk_id: *mut ::clockid_t) -> ::c_int; + pub fn dirfd(dirp: *mut ::DIR) -> ::c_int; pub fn duplocale(base: ::locale_t) -> ::locale_t; pub fn endutxent(); pub fn fchflags(fd: ::c_int, flags: ::c_ulong) -> ::c_int; + pub fn fexecve( + fd: ::c_int, + argv: *const *const ::c_char, + envp: *const *const ::c_char, + ) -> ::c_int; pub fn futimens(fd: ::c_int, times: *const ::timespec) -> ::c_int; pub fn getdomainname(name: *mut ::c_char, len: ::c_int) -> ::c_int; pub fn getgrent_r( @@ -1245,7 +1489,6 @@ extern "C" { buflen: ::size_t, result: *mut *mut ::group, ) -> ::c_int; - #[cfg_attr(target_os = "netbsd", link_name = "__getpwent_r50")] pub fn getpwent_r( pwd: *mut ::passwd, buf: *mut ::c_char, @@ -1268,6 +1511,8 @@ extern "C" { flags: ::c_int, ) -> ::c_int; pub fn getpriority(which: ::c_int, who: ::c_int) -> ::c_int; + pub fn getresgid(rgid: *mut ::gid_t, egid: *mut ::gid_t, sgid: *mut ::gid_t) -> ::c_int; + pub fn getresuid(ruid: *mut ::uid_t, euid: *mut ::uid_t, suid: *mut ::uid_t) -> ::c_int; pub fn getutxent() -> *mut utmpx; pub fn getutxid(ut: *const utmpx) -> *mut utmpx; pub fn getutxline(ut: *const utmpx) -> *mut utmpx; @@ -1285,23 +1530,9 @@ extern "C" { timeout: *const ::timespec, ) -> ::c_int; pub fn lchflags(path: *const ::c_char, flags: ::c_ulong) -> ::c_int; - pub fn lio_listio( - mode: ::c_int, - aiocb_list: *const *mut aiocb, - nitems: ::c_int, - sevp: *mut sigevent, - ) -> ::c_int; pub fn lutimes(file: *const ::c_char, times: *const ::timeval) -> ::c_int; - pub fn memrchr( - cx: *const ::c_void, - c: ::c_int, - n: ::size_t, - ) -> *mut ::c_void; - pub fn mkfifoat( - dirfd: ::c_int, - pathname: *const ::c_char, - mode: ::mode_t, - ) -> ::c_int; + pub fn memrchr(cx: *const ::c_void, c: ::c_int, n: ::size_t) -> *mut ::c_void; + pub fn mkfifoat(dirfd: ::c_int, pathname: *const ::c_char, mode: ::mode_t) -> ::c_int; #[cfg_attr( all(target_os = "freebsd", any(freebsd11, freebsd10)), link_name = "mknodat@FBSD_1.1" @@ -1312,72 +1543,20 @@ extern "C" { mode: ::mode_t, dev: dev_t, ) -> ::c_int; - pub fn mq_close(mqd: ::mqd_t) -> ::c_int; - pub fn mq_getattr(mqd: ::mqd_t, attr: *mut ::mq_attr) -> ::c_int; - pub fn mq_notify(mqd: ::mqd_t, notification: *const ::sigevent) - -> ::c_int; - pub fn mq_open(name: *const ::c_char, oflag: ::c_int, ...) -> ::mqd_t; - pub fn mq_receive( - mqd: ::mqd_t, - msg_ptr: *mut ::c_char, - msg_len: ::size_t, - msg_prio: *mut ::c_uint, - ) -> ::ssize_t; - pub fn mq_send( - mqd: ::mqd_t, - msg_ptr: *const ::c_char, - msg_len: ::size_t, - msg_prio: ::c_uint, - ) -> ::c_int; - pub fn mq_setattr( - mqd: ::mqd_t, - newattr: *const ::mq_attr, - oldattr: *mut ::mq_attr, - ) -> ::c_int; - pub fn mq_timedreceive( - mqd: ::mqd_t, - msg_ptr: *mut ::c_char, - msg_len: ::size_t, - msg_prio: *mut ::c_uint, - abs_timeout: *const ::timespec, - ) -> ::ssize_t; - pub fn mq_timedsend( - mqd: ::mqd_t, - msg_ptr: *const ::c_char, - msg_len: ::size_t, - msg_prio: ::c_uint, - abs_timeout: *const ::timespec, - ) -> ::c_int; - pub fn mq_unlink(name: *const ::c_char) -> ::c_int; - pub fn mincore( - addr: *const ::c_void, - len: ::size_t, - vec: *mut ::c_char, - ) -> ::c_int; - pub fn newlocale( - mask: ::c_int, - locale: *const ::c_char, - base: ::locale_t, - ) -> ::locale_t; - pub fn nl_langinfo_l(item: ::nl_item, locale: ::locale_t) - -> *mut ::c_char; + pub fn mincore(addr: *const ::c_void, len: ::size_t, vec: *mut ::c_char) -> ::c_int; + pub fn newlocale(mask: ::c_int, locale: *const ::c_char, base: ::locale_t) -> ::locale_t; + pub fn nl_langinfo_l(item: ::nl_item, locale: ::locale_t) -> *mut ::c_char; pub fn pipe2(fds: *mut ::c_int, flags: ::c_int) -> ::c_int; + pub fn posix_fallocate(fd: ::c_int, offset: ::off_t, len: ::off_t) -> ::c_int; + pub fn posix_fadvise(fd: ::c_int, offset: ::off_t, len: ::off_t, advise: ::c_int) -> ::c_int; pub fn ppoll( fds: *mut ::pollfd, nfds: ::nfds_t, timeout: *const ::timespec, sigmask: *const sigset_t, ) -> ::c_int; - pub fn preadv( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - offset: ::off_t, - ) -> ::ssize_t; - pub fn pthread_attr_get_np( - tid: ::pthread_t, - attr: *mut ::pthread_attr_t, - ) -> ::c_int; + pub fn preadv(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int, offset: ::off_t) -> ::ssize_t; + pub fn pthread_attr_get_np(tid: ::pthread_t, attr: *mut ::pthread_attr_t) -> ::c_int; pub fn pthread_attr_getguardsize( attr: *const ::pthread_attr_t, guardsize: *mut ::size_t, @@ -1399,10 +1578,7 @@ extern "C" { attr: *mut pthread_condattr_t, clock_id: ::clockid_t, ) -> ::c_int; - pub fn pthread_condattr_setpshared( - attr: *mut pthread_condattr_t, - pshared: ::c_int, - ) -> ::c_int; + pub fn pthread_condattr_setpshared(attr: *mut pthread_condattr_t, pshared: ::c_int) -> ::c_int; pub fn pthread_main_np() -> ::c_int; pub fn pthread_mutex_timedlock( lock: *mut pthread_mutex_t, @@ -1420,30 +1596,46 @@ extern "C" { attr: *const pthread_rwlockattr_t, val: *mut ::c_int, ) -> ::c_int; - pub fn pthread_rwlockattr_setpshared( - attr: *mut pthread_rwlockattr_t, - val: ::c_int, + pub fn pthread_rwlockattr_setpshared(attr: *mut pthread_rwlockattr_t, val: ::c_int) -> ::c_int; + pub fn pthread_barrierattr_init(attr: *mut ::pthread_barrierattr_t) -> ::c_int; + pub fn pthread_barrierattr_destroy(attr: *mut ::pthread_barrierattr_t) -> ::c_int; + pub fn pthread_barrierattr_getpshared( + attr: *const ::pthread_barrierattr_t, + shared: *mut ::c_int, ) -> ::c_int; + pub fn pthread_barrierattr_setpshared( + attr: *mut ::pthread_barrierattr_t, + shared: ::c_int, + ) -> ::c_int; + pub fn pthread_barrier_init( + barrier: *mut pthread_barrier_t, + attr: *const ::pthread_barrierattr_t, + count: ::c_uint, + ) -> ::c_int; + pub fn pthread_barrier_destroy(barrier: *mut pthread_barrier_t) -> ::c_int; + pub fn pthread_barrier_wait(barrier: *mut pthread_barrier_t) -> ::c_int; + pub fn pthread_get_name_np(tid: ::pthread_t, name: *mut ::c_char, len: ::size_t); pub fn pthread_set_name_np(tid: ::pthread_t, name: *const ::c_char); - pub fn ptrace( - request: ::c_int, - pid: ::pid_t, - addr: *mut ::c_char, - data: ::c_int, + pub fn pthread_setschedparam( + native: ::pthread_t, + policy: ::c_int, + param: *const sched_param, + ) -> ::c_int; + pub fn pthread_getschedparam( + native: ::pthread_t, + policy: *mut ::c_int, + param: *mut sched_param, ) -> ::c_int; + pub fn ptrace(request: ::c_int, pid: ::pid_t, addr: *mut ::c_char, data: ::c_int) -> ::c_int; + pub fn utrace(addr: *const ::c_void, len: ::size_t) -> ::c_int; pub fn pututxline(ut: *const utmpx) -> *mut utmpx; - pub fn pwritev( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - offset: ::off_t, - ) -> ::ssize_t; + pub fn pwritev(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int, offset: ::off_t) + -> ::ssize_t; pub fn querylocale(mask: ::c_int, loc: ::locale_t) -> *const ::c_char; - pub fn rtprio( - function: ::c_int, - pid: ::pid_t, - rtp: *mut rtprio, - ) -> ::c_int; + pub fn rtprio(function: ::c_int, pid: ::pid_t, rtp: *mut rtprio) -> ::c_int; + pub fn sched_rr_get_interval(pid: ::pid_t, t: *mut ::timespec) -> ::c_int; + pub fn sched_getparam(pid: ::pid_t, param: *mut sched_param) -> ::c_int; + pub fn sched_setparam(pid: ::pid_t, param: *const sched_param) -> ::c_int; pub fn sched_getscheduler(pid: ::pid_t) -> ::c_int; pub fn sched_setscheduler( pid: ::pid_t, @@ -1451,10 +1643,7 @@ extern "C" { param: *const ::sched_param, ) -> ::c_int; pub fn sem_getvalue(sem: *mut sem_t, sval: *mut ::c_int) -> ::c_int; - pub fn sem_timedwait( - sem: *mut sem_t, - abstime: *const ::timespec, - ) -> ::c_int; + pub fn sem_timedwait(sem: *mut sem_t, abstime: *const ::timespec) -> ::c_int; pub fn sendfile( fd: ::c_int, s: ::c_int, @@ -1466,20 +1655,12 @@ extern "C" { ) -> ::c_int; pub fn setdomainname(name: *const ::c_char, len: ::c_int) -> ::c_int; pub fn sethostname(name: *const ::c_char, len: ::c_int) -> ::c_int; - pub fn setpriority(which: ::c_int, who: ::c_int, prio: ::c_int) - -> ::c_int; + pub fn setpriority(which: ::c_int, who: ::c_int, prio: ::c_int) -> ::c_int; pub fn setresgid(rgid: ::gid_t, egid: ::gid_t, sgid: ::gid_t) -> ::c_int; pub fn setresuid(ruid: ::uid_t, euid: ::uid_t, suid: ::uid_t) -> ::c_int; - pub fn settimeofday( - tv: *const ::timeval, - tz: *const ::timezone, - ) -> ::c_int; + pub fn settimeofday(tv: *const ::timeval, tz: *const ::timezone) -> ::c_int; pub fn setutxent(); - pub fn shm_open( - name: *const ::c_char, - oflag: ::c_int, - mode: ::mode_t, - ) -> ::c_int; + pub fn shm_open(name: *const ::c_char, oflag: ::c_int, mode: ::mode_t) -> ::c_int; pub fn sigtimedwait( set: *const sigset_t, info: *mut siginfo_t, @@ -1516,6 +1697,79 @@ extern "C" { pub fn ntp_adjtime(buf: *mut timex) -> ::c_int; pub fn ntp_gettime(buf: *mut ntptimeval) -> ::c_int; + + // #include + pub fn dl_iterate_phdr( + callback: ::Option< + unsafe extern "C" fn( + info: *mut dl_phdr_info, + size: usize, + data: *mut ::c_void, + ) -> ::c_int, + >, + data: *mut ::c_void, + ) -> ::c_int; + + pub fn iconv_open(tocode: *const ::c_char, fromcode: *const ::c_char) -> iconv_t; + pub fn iconv( + cd: iconv_t, + inbuf: *mut *mut ::c_char, + inbytesleft: *mut ::size_t, + outbuf: *mut *mut ::c_char, + outbytesleft: *mut ::size_t, + ) -> ::size_t; + pub fn iconv_close(cd: iconv_t) -> ::c_int; + + // Added in `FreeBSD` 11.0 + // Added in `DragonFly BSD` 5.4 + pub fn explicit_bzero(s: *mut ::c_void, len: ::size_t); + // ISO/IEC 9899:2011 ("ISO C11") K.3.7.4.1 + pub fn memset_s(s: *mut ::c_void, smax: ::size_t, c: ::c_int, n: ::size_t) -> ::c_int; + pub fn gethostid() -> ::c_long; + pub fn sethostid(hostid: ::c_long); + + pub fn eui64_aton(a: *const ::c_char, e: *mut eui64) -> ::c_int; + pub fn eui64_ntoa(id: *const eui64, a: *mut ::c_char, len: ::size_t) -> ::c_int; + pub fn eui64_ntohost(hostname: *mut ::c_char, len: ::size_t, id: *const eui64) -> ::c_int; + pub fn eui64_hostton(hostname: *const ::c_char, id: *mut eui64) -> ::c_int; + + pub fn eaccess(path: *const ::c_char, mode: ::c_int) -> ::c_int; +} + +#[link(name = "rt")] +extern "C" { + pub fn mq_close(mqd: ::mqd_t) -> ::c_int; + pub fn mq_getattr(mqd: ::mqd_t, attr: *mut ::mq_attr) -> ::c_int; + pub fn mq_notify(mqd: ::mqd_t, notification: *const ::sigevent) -> ::c_int; + pub fn mq_open(name: *const ::c_char, oflag: ::c_int, ...) -> ::mqd_t; + pub fn mq_receive( + mqd: ::mqd_t, + msg_ptr: *mut ::c_char, + msg_len: ::size_t, + msg_prio: *mut ::c_uint, + ) -> ::ssize_t; + pub fn mq_send( + mqd: ::mqd_t, + msg_ptr: *const ::c_char, + msg_len: ::size_t, + msg_prio: ::c_uint, + ) -> ::c_int; + pub fn mq_setattr(mqd: ::mqd_t, newattr: *const ::mq_attr, oldattr: *mut ::mq_attr) -> ::c_int; + pub fn mq_timedreceive( + mqd: ::mqd_t, + msg_ptr: *mut ::c_char, + msg_len: ::size_t, + msg_prio: *mut ::c_uint, + abs_timeout: *const ::timespec, + ) -> ::ssize_t; + pub fn mq_timedsend( + mqd: ::mqd_t, + msg_ptr: *const ::c_char, + msg_len: ::size_t, + msg_prio: ::c_uint, + abs_timeout: *const ::timespec, + ) -> ::c_int; + pub fn mq_unlink(name: *const ::c_char) -> ::c_int; } #[link(name = "util")] @@ -1534,6 +1788,62 @@ extern "C" { winp: *mut ::winsize, ) -> ::pid_t; pub fn login_tty(fd: ::c_int) -> ::c_int; + pub fn fparseln( + stream: *mut ::FILE, + len: *mut ::size_t, + lineno: *mut ::size_t, + delim: *const ::c_char, + flags: ::c_int, + ) -> *mut ::c_char; +} + +#[link(name = "execinfo")] +extern "C" { + pub fn backtrace(addrlist: *mut *mut ::c_void, len: ::size_t) -> ::size_t; + pub fn backtrace_symbols(addrlist: *const *mut ::c_void, len: ::size_t) -> *mut *mut ::c_char; + pub fn backtrace_symbols_fd( + addrlist: *const *mut ::c_void, + len: ::size_t, + fd: ::c_int, + ) -> ::c_int; +} + +#[link(name = "kvm")] +extern "C" { + pub fn kvm_open( + execfile: *const ::c_char, + corefile: *const ::c_char, + swapfile: *const ::c_char, + flags: ::c_int, + errstr: *const ::c_char, + ) -> *mut ::kvm_t; + pub fn kvm_close(kd: *mut ::kvm_t) -> ::c_int; + pub fn kvm_getprocs( + kd: *mut ::kvm_t, + op: ::c_int, + arg: ::c_int, + cnt: *mut ::c_int, + ) -> *mut ::kinfo_proc; + pub fn kvm_getloadavg(kd: *mut kvm_t, loadavg: *mut ::c_double, nelem: ::c_int) -> ::c_int; + pub fn kvm_openfiles( + execfile: *const ::c_char, + corefile: *const ::c_char, + swapfile: *const ::c_char, + flags: ::c_int, + errbuf: *mut ::c_char, + ) -> *mut ::kvm_t; + pub fn kvm_read( + kd: *mut ::kvm_t, + addr: ::c_ulong, + buf: *mut ::c_void, + nbytes: ::size_t, + ) -> ::ssize_t; + pub fn kvm_write( + kd: *mut ::kvm_t, + addr: ::c_ulong, + buf: *const ::c_void, + nbytes: ::size_t, + ) -> ::ssize_t; } cfg_if! { diff --git a/crux-mir/lib/libc/src/unix/bsd/mod.rs b/crux-mir/lib/libc/src/unix/bsd/mod.rs index d129f1ee0..d49e3c440 100644 --- a/crux-mir/lib/libc/src/unix/bsd/mod.rs +++ b/crux-mir/lib/libc/src/unix/bsd/mod.rs @@ -1,4 +1,3 @@ -pub type wchar_t = i32; pub type off_t = i64; pub type useconds_t = u32; pub type blkcnt_t = i64; @@ -6,6 +5,7 @@ pub type socklen_t = u32; pub type sa_family_t = u8; pub type pthread_t = ::uintptr_t; pub type nfds_t = ::c_uint; +pub type regoff_t = off_t; s! { pub struct sockaddr { @@ -37,6 +37,8 @@ s! { #[cfg(not(any(target_os = "macos", target_os = "ios", + target_os = "tvos", + target_os = "watchos", target_os = "netbsd", target_os = "openbsd")))] pub pw_fields: ::c_int, @@ -101,6 +103,25 @@ s! { pub if_index: ::c_uint, pub if_name: *mut ::c_char, } + + pub struct regex_t { + __re_magic: ::c_int, + __re_nsub: ::size_t, + __re_endp: *const ::c_char, + __re_g: *mut ::c_void, + } + + pub struct regmatch_t { + pub rm_so: regoff_t, + pub rm_eo: regoff_t, + } + + pub struct option { + pub name: *const ::c_char, + pub has_arg: ::c_int, + pub flag: *mut ::c_int, + pub val: ::c_int, + } } s_no_extra_traits! { @@ -241,6 +262,9 @@ pub const FIOSETOWN: ::c_ulong = 0x8004667c; pub const FIOGETOWN: ::c_ulong = 0x4004667b; pub const PATH_MAX: ::c_int = 1024; +pub const MAXPATHLEN: ::c_int = PATH_MAX; + +pub const IOV_MAX: ::c_int = 1024; pub const SA_ONSTACK: ::c_int = 0x0001; pub const SA_SIGINFO: ::c_int = 0x0040; @@ -323,7 +347,15 @@ pub const F_RDLCK: ::c_short = 1; pub const F_UNLCK: ::c_short = 2; pub const F_WRLCK: ::c_short = 3; -pub const MNT_FORCE: ::c_int = 0x80000; +pub const MNT_RDONLY: ::c_int = 0x00000001; +pub const MNT_SYNCHRONOUS: ::c_int = 0x00000002; +pub const MNT_NOEXEC: ::c_int = 0x00000004; +pub const MNT_NOSUID: ::c_int = 0x00000008; +pub const MNT_ASYNC: ::c_int = 0x00000040; +pub const MNT_EXPORTED: ::c_int = 0x00000100; +pub const MNT_UPDATE: ::c_int = 0x00010000; +pub const MNT_RELOAD: ::c_int = 0x00040000; +pub const MNT_FORCE: ::c_int = 0x00080000; pub const Q_SYNC: ::c_int = 0x600; pub const Q_QUOTAON: ::c_int = 0x100; @@ -425,6 +457,19 @@ pub const TCP_MAXSEG: ::c_int = 2; pub const PIPE_BUF: usize = 512; +// si_code values for SIGBUS signal +pub const BUS_ADRALN: ::c_int = 1; +pub const BUS_ADRERR: ::c_int = 2; +pub const BUS_OBJERR: ::c_int = 3; + +// si_code values for SIGCHLD signal +pub const CLD_EXITED: ::c_int = 1; +pub const CLD_KILLED: ::c_int = 2; +pub const CLD_DUMPED: ::c_int = 3; +pub const CLD_TRAPPED: ::c_int = 4; +pub const CLD_STOPPED: ::c_int = 5; +pub const CLD_CONTINUED: ::c_int = 6; + pub const POLLIN: ::c_short = 0x1; pub const POLLPRI: ::c_short = 0x2; pub const POLLOUT: ::c_short = 0x4; @@ -450,6 +495,52 @@ pub const BIOCGHDRCMPLT: ::c_ulong = 0x40044274; pub const BIOCSHDRCMPLT: ::c_ulong = 0x80044275; pub const SIOCGIFADDR: ::c_ulong = 0xc0206921; +pub const REG_BASIC: ::c_int = 0o0000; +pub const REG_EXTENDED: ::c_int = 0o0001; +pub const REG_ICASE: ::c_int = 0o0002; +pub const REG_NOSUB: ::c_int = 0o0004; +pub const REG_NEWLINE: ::c_int = 0o0010; +pub const REG_NOSPEC: ::c_int = 0o0020; +pub const REG_PEND: ::c_int = 0o0040; +pub const REG_DUMP: ::c_int = 0o0200; + +pub const REG_NOMATCH: ::c_int = 1; +pub const REG_BADPAT: ::c_int = 2; +pub const REG_ECOLLATE: ::c_int = 3; +pub const REG_ECTYPE: ::c_int = 4; +pub const REG_EESCAPE: ::c_int = 5; +pub const REG_ESUBREG: ::c_int = 6; +pub const REG_EBRACK: ::c_int = 7; +pub const REG_EPAREN: ::c_int = 8; +pub const REG_EBRACE: ::c_int = 9; +pub const REG_BADBR: ::c_int = 10; +pub const REG_ERANGE: ::c_int = 11; +pub const REG_ESPACE: ::c_int = 12; +pub const REG_BADRPT: ::c_int = 13; +pub const REG_EMPTY: ::c_int = 14; +pub const REG_ASSERT: ::c_int = 15; +pub const REG_INVARG: ::c_int = 16; +pub const REG_ATOI: ::c_int = 255; +pub const REG_ITOA: ::c_int = 0o0400; + +pub const REG_NOTBOL: ::c_int = 0o00001; +pub const REG_NOTEOL: ::c_int = 0o00002; +pub const REG_STARTEND: ::c_int = 0o00004; +pub const REG_TRACE: ::c_int = 0o00400; +pub const REG_LARGE: ::c_int = 0o01000; +pub const REG_BACKR: ::c_int = 0o02000; + +pub const TIOCCBRK: ::c_uint = 0x2000747a; +pub const TIOCSBRK: ::c_uint = 0x2000747b; + +pub const PRIO_PROCESS: ::c_int = 0; +pub const PRIO_PGRP: ::c_int = 1; +pub const PRIO_USER: ::c_int = 2; + +pub const ITIMER_REAL: ::c_int = 0; +pub const ITIMER_VIRTUAL: ::c_int = 1; +pub const ITIMER_PROF: ::c_int = 2; + f! { pub fn CMSG_FIRSTHDR(mhdr: *const ::msghdr) -> *mut ::cmsghdr { if (*mhdr).msg_controllen as usize >= ::mem::size_of::<::cmsghdr>() { @@ -466,7 +557,7 @@ f! { return } - pub fn FD_ISSET(fd: ::c_int, set: *mut fd_set) -> bool { + pub fn FD_ISSET(fd: ::c_int, set: *const fd_set) -> bool { let bits = ::mem::size_of_val(&(*set).fds_bits[0]) * 8; let fd = fd as usize; return ((*set).fds_bits[fd / bits] & (1 << (fd % bits))) != 0 @@ -484,24 +575,26 @@ f! { *slot = 0; } } +} - pub fn WTERMSIG(status: ::c_int) -> ::c_int { +safe_f! { + pub {const} fn WTERMSIG(status: ::c_int) -> ::c_int { status & 0o177 } - pub fn WIFEXITED(status: ::c_int) -> bool { + pub {const} fn WIFEXITED(status: ::c_int) -> bool { (status & 0o177) == 0 } - pub fn WEXITSTATUS(status: ::c_int) -> ::c_int { + pub {const} fn WEXITSTATUS(status: ::c_int) -> ::c_int { status >> 8 } - pub fn WCOREDUMP(status: ::c_int) -> bool { + pub {const} fn WCOREDUMP(status: ::c_int) -> bool { (status & 0o200) != 0 } - pub fn QCMD(cmd: ::c_int, type_: ::c_int) -> ::c_int { + pub {const} fn QCMD(cmd: ::c_int, type_: ::c_int) -> ::c_int { (cmd << 8) | (type_ & 0x00ff) } } @@ -518,15 +611,19 @@ extern "C" { )] pub fn setrlimit(resource: ::c_int, rlim: *const ::rlimit) -> ::c_int; - pub fn strerror_r( - errnum: ::c_int, - buf: *mut c_char, - buflen: ::size_t, - ) -> ::c_int; + pub fn strerror_r(errnum: ::c_int, buf: *mut c_char, buflen: ::size_t) -> ::c_int; pub fn abs(i: ::c_int) -> ::c_int; pub fn atof(s: *const ::c_char) -> ::c_double; pub fn labs(i: ::c_long) -> ::c_long; + #[cfg_attr( + all(target_os = "freebsd", any(freebsd12, freebsd11, freebsd10)), + link_name = "rand@FBSD_1.0" + )] pub fn rand() -> ::c_int; + #[cfg_attr( + all(target_os = "freebsd", any(freebsd12, freebsd11, freebsd10)), + link_name = "srand@FBSD_1.0" + )] pub fn srand(seed: ::c_uint); pub fn getifaddrs(ifap: *mut *mut ::ifaddrs) -> ::c_int; @@ -549,13 +646,12 @@ extern "C" { pub fn if_nameindex() -> *mut if_nameindex; pub fn if_freenameindex(ptr: *mut if_nameindex); - pub fn getpeereid( - socket: ::c_int, - euid: *mut ::uid_t, - egid: *mut ::gid_t, - ) -> ::c_int; + pub fn getpeereid(socket: ::c_int, euid: *mut ::uid_t, egid: *mut ::gid_t) -> ::c_int; - #[cfg_attr(target_os = "macos", link_name = "glob$INODE64")] + #[cfg_attr( + all(target_os = "macos", not(target_arch = "aarch64")), + link_name = "glob$INODE64" + )] #[cfg_attr(target_os = "netbsd", link_name = "__glob30")] #[cfg_attr( all(target_os = "freebsd", any(freebsd11, freebsd10)), @@ -564,9 +660,7 @@ extern "C" { pub fn glob( pattern: *const ::c_char, flags: ::c_int, - errfunc: ::Option< - extern "C" fn(epath: *const ::c_char, errno: ::c_int) -> ::c_int, - >, + errfunc: ::Option ::c_int>, pglob: *mut ::glob_t, ) -> ::c_int; #[cfg_attr(target_os = "netbsd", link_name = "__globfree30")] @@ -576,11 +670,7 @@ extern "C" { )] pub fn globfree(pglob: *mut ::glob_t); - pub fn posix_madvise( - addr: *mut ::c_void, - len: ::size_t, - advice: ::c_int, - ) -> ::c_int; + pub fn posix_madvise(addr: *mut ::c_void, len: ::size_t, advice: ::c_int) -> ::c_int; pub fn shm_unlink(name: *const ::c_char) -> ::c_int; @@ -603,22 +693,14 @@ extern "C" { link_name = "telldir$INODE64$UNIX2003" )] pub fn telldir(dirp: *mut ::DIR) -> ::c_long; - pub fn madvise( - addr: *mut ::c_void, - len: ::size_t, - advice: ::c_int, - ) -> ::c_int; + pub fn madvise(addr: *mut ::c_void, len: ::size_t, advice: ::c_int) -> ::c_int; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "msync$UNIX2003" )] #[cfg_attr(target_os = "netbsd", link_name = "__msync13")] - pub fn msync( - addr: *mut ::c_void, - len: ::size_t, - flags: ::c_int, - ) -> ::c_int; + pub fn msync(addr: *mut ::c_void, len: ::size_t, flags: ::c_int) -> ::c_int; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), @@ -641,52 +723,31 @@ extern "C" { all(target_os = "macos", target_arch = "x86"), link_name = "bind$UNIX2003" )] - pub fn bind( - socket: ::c_int, - address: *const ::sockaddr, - address_len: ::socklen_t, - ) -> ::c_int; + pub fn bind(socket: ::c_int, address: *const ::sockaddr, address_len: ::socklen_t) -> ::c_int; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "writev$UNIX2003" )] - pub fn writev( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - ) -> ::ssize_t; + pub fn writev(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int) -> ::ssize_t; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "readv$UNIX2003" )] - pub fn readv( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - ) -> ::ssize_t; + pub fn readv(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int) -> ::ssize_t; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "sendmsg$UNIX2003" )] - pub fn sendmsg( - fd: ::c_int, - msg: *const ::msghdr, - flags: ::c_int, - ) -> ::ssize_t; + pub fn sendmsg(fd: ::c_int, msg: *const ::msghdr, flags: ::c_int) -> ::ssize_t; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "recvmsg$UNIX2003" )] - pub fn recvmsg( - fd: ::c_int, - msg: *mut ::msghdr, - flags: ::c_int, - ) -> ::ssize_t; + pub fn recvmsg(fd: ::c_int, msg: *mut ::msghdr, flags: ::c_int) -> ::ssize_t; pub fn sync(); - #[cfg_attr(target_os = "solaris", link_name = "__posix_getgrgid_r")] pub fn getgrgid_r( gid: ::gid_t, grp: *mut ::group, @@ -702,7 +763,6 @@ extern "C" { pub fn sigaltstack(ss: *const stack_t, oss: *mut stack_t) -> ::c_int; pub fn sem_close(sem: *mut sem_t) -> ::c_int; pub fn getdtablesize() -> ::c_int; - #[cfg_attr(target_os = "solaris", link_name = "__posix_getgrnam_r")] pub fn getgrnam_r( name: *const ::c_char, grp: *mut ::group, @@ -714,11 +774,7 @@ extern "C" { all(target_os = "macos", target_arch = "x86"), link_name = "pthread_sigmask$UNIX2003" )] - pub fn pthread_sigmask( - how: ::c_int, - set: *const sigset_t, - oldset: *mut sigset_t, - ) -> ::c_int; + pub fn pthread_sigmask(how: ::c_int, set: *const sigset_t, oldset: *mut sigset_t) -> ::c_int; pub fn sem_open(name: *const ::c_char, oflag: ::c_int, ...) -> *mut sem_t; pub fn getgrnam(name: *const ::c_char) -> *mut ::group; #[cfg_attr( @@ -727,9 +783,10 @@ extern "C" { )] pub fn pthread_cancel(thread: ::pthread_t) -> ::c_int; pub fn pthread_kill(thread: ::pthread_t, sig: ::c_int) -> ::c_int; + pub fn sched_get_priority_min(policy: ::c_int) -> ::c_int; + pub fn sched_get_priority_max(policy: ::c_int) -> ::c_int; pub fn sem_unlink(name: *const ::c_char) -> ::c_int; #[cfg_attr(target_os = "netbsd", link_name = "__getpwnam_r50")] - #[cfg_attr(target_os = "solaris", link_name = "__posix_getpwnam_r")] pub fn getpwnam_r( name: *const ::c_char, pwd: *mut passwd, @@ -738,7 +795,6 @@ extern "C" { result: *mut *mut passwd, ) -> ::c_int; #[cfg_attr(target_os = "netbsd", link_name = "__getpwuid_r50")] - #[cfg_attr(target_os = "solaris", link_name = "__posix_getpwuid_r")] pub fn getpwuid_r( uid: ::uid_t, pwd: *mut passwd, @@ -750,7 +806,6 @@ extern "C" { all(target_os = "macos", target_arch = "x86"), link_name = "sigwait$UNIX2003" )] - #[cfg_attr(target_os = "solaris", link_name = "__posix_sigwait")] pub fn sigwait(set: *const sigset_t, sig: *mut ::c_int) -> ::c_int; pub fn pthread_atfork( prepare: ::Option, @@ -790,10 +845,64 @@ extern "C" { options: ::c_int, rusage: *mut ::rusage, ) -> ::pid_t; + #[cfg_attr( + all(target_os = "macos", target_arch = "x86"), + link_name = "getitimer$UNIX2003" + )] + pub fn getitimer(which: ::c_int, curr_value: *mut ::itimerval) -> ::c_int; + #[cfg_attr( + all(target_os = "macos", target_arch = "x86"), + link_name = "setitimer$UNIX2003" + )] + pub fn setitimer( + which: ::c_int, + new_value: *const ::itimerval, + old_value: *mut ::itimerval, + ) -> ::c_int; + + pub fn regcomp(preg: *mut regex_t, pattern: *const ::c_char, cflags: ::c_int) -> ::c_int; + + pub fn regexec( + preg: *const regex_t, + input: *const ::c_char, + nmatch: ::size_t, + pmatch: *mut regmatch_t, + eflags: ::c_int, + ) -> ::c_int; + + pub fn regerror( + errcode: ::c_int, + preg: *const regex_t, + errbuf: *mut ::c_char, + errbuf_size: ::size_t, + ) -> ::size_t; + + pub fn regfree(preg: *mut regex_t); + + pub fn arc4random() -> u32; + pub fn arc4random_buf(buf: *mut ::c_void, size: ::size_t); + pub fn arc4random_uniform(l: u32) -> u32; + + pub fn drand48() -> ::c_double; + pub fn erand48(xseed: *mut ::c_ushort) -> ::c_double; + pub fn lrand48() -> ::c_long; + pub fn nrand48(xseed: *mut ::c_ushort) -> ::c_long; + pub fn mrand48() -> ::c_long; + pub fn jrand48(xseed: *mut ::c_ushort) -> ::c_long; + pub fn srand48(seed: ::c_long); + pub fn seed48(xseed: *mut ::c_ushort) -> *mut ::c_ushort; + pub fn lcong48(p: *mut ::c_ushort); + pub fn getopt_long( + argc: ::c_int, + argv: *const *mut c_char, + optstring: *const c_char, + longopts: *const option, + longindex: *mut ::c_int, + ) -> ::c_int; } cfg_if! { - if #[cfg(any(target_os = "macos", target_os = "ios"))] { + if #[cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos"))] { mod apple; pub use self::apple::*; } else if #[cfg(any(target_os = "openbsd", target_os = "netbsd"))] { diff --git a/crux-mir/lib/libc/src/unix/bsd/netbsdlike/mod.rs b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/mod.rs index 970cb233a..b71531c25 100644 --- a/crux-mir/lib/libc/src/unix/bsd/netbsdlike/mod.rs +++ b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/mod.rs @@ -1,3 +1,4 @@ +pub type wchar_t = i32; pub type time_t = i64; pub type mode_t = u32; pub type nlink_t = u32; @@ -30,6 +31,10 @@ impl ::Clone for sem { } s! { + pub struct sched_param { + pub sched_priority: ::c_int, + } + pub struct sigaction { pub sa_sigaction: ::sighandler_t, pub sa_mask: ::sigset_t, @@ -71,8 +76,21 @@ s! { pub uid: ::uid_t, pub gid: ::gid_t, pub mode: ::mode_t, + #[cfg(target_os = "openbsd")] pub seq: ::c_ushort, + #[cfg(target_os = "netbsd")] + pub _seq: ::c_ushort, + #[cfg(target_os = "openbsd")] pub key: ::key_t, + #[cfg(target_os = "netbsd")] + pub _key: ::key_t, + } + + pub struct ptrace_io_desc { + pub piod_op: ::c_int, + pub piod_offs: *mut ::c_void, + pub piod_addr: *mut ::c_void, + pub piod_len: ::size_t, } } @@ -224,6 +242,13 @@ pub const IPC_RMID: ::c_int = 0; pub const IPC_SET: ::c_int = 1; pub const IPC_STAT: ::c_int = 2; +pub const IPC_R: ::c_int = 0o000400; +pub const IPC_W: ::c_int = 0o000200; +pub const IPC_M: ::c_int = 0o010000; + +pub const SHM_R: ::c_int = IPC_R; +pub const SHM_W: ::c_int = IPC_W; + pub const MCL_CURRENT: ::c_int = 0x0001; pub const MCL_FUTURE: ::c_int = 0x0002; @@ -342,6 +367,12 @@ pub const POSIX_MADV_DONTNEED: ::c_int = 4; pub const PTHREAD_CREATE_JOINABLE: ::c_int = 0; pub const PTHREAD_CREATE_DETACHED: ::c_int = 1; +pub const PIOD_READ_D: ::c_int = 1; +pub const PIOD_WRITE_D: ::c_int = 2; +pub const PIOD_READ_I: ::c_int = 3; +pub const PIOD_WRITE_I: ::c_int = 4; +pub const PIOD_READ_AUXV: ::c_int = 5; + pub const PT_TRACE_ME: ::c_int = 0; pub const PT_READ_I: ::c_int = 1; pub const PT_READ_D: ::c_int = 2; @@ -388,6 +419,11 @@ pub const MADV_WILLNEED: ::c_int = 3; pub const MADV_DONTNEED: ::c_int = 4; pub const MADV_FREE: ::c_int = 6; +// sys/fstypes.h in NetBSD, or sys/mount.h in OpenBSD +pub const MNT_NODEV: ::c_int = 0x00000010; +pub const MNT_LOCAL: ::c_int = 0x00001000; +pub const MNT_QUOTA: ::c_int = 0x00002000; + pub const AF_UNSPEC: ::c_int = 0; pub const AF_LOCAL: ::c_int = 1; pub const AF_UNIX: ::c_int = AF_LOCAL; @@ -608,73 +644,36 @@ pub const TIOCM_DSR: ::c_int = 0o0400; pub const TIOCM_CD: ::c_int = TIOCM_CAR; pub const TIOCM_RI: ::c_int = TIOCM_RNG; -// Flags for chflags(2) -pub const UF_SETTABLE: ::c_ulong = 0x0000ffff; -pub const UF_NODUMP: ::c_ulong = 0x00000001; -pub const UF_IMMUTABLE: ::c_ulong = 0x00000002; -pub const UF_APPEND: ::c_ulong = 0x00000004; -pub const UF_OPAQUE: ::c_ulong = 0x00000008; -pub const SF_SETTABLE: ::c_ulong = 0xffff0000; -pub const SF_ARCHIVED: ::c_ulong = 0x00010000; -pub const SF_IMMUTABLE: ::c_ulong = 0x00020000; -pub const SF_APPEND: ::c_ulong = 0x00040000; - pub const TIMER_ABSTIME: ::c_int = 1; #[link(name = "util")] extern "C" { pub fn setgrent(); pub fn sem_destroy(sem: *mut sem_t) -> ::c_int; - pub fn sem_init( - sem: *mut sem_t, - pshared: ::c_int, - value: ::c_uint, - ) -> ::c_int; + pub fn sem_init(sem: *mut sem_t, pshared: ::c_int, value: ::c_uint) -> ::c_int; pub fn daemon(nochdir: ::c_int, noclose: ::c_int) -> ::c_int; - pub fn mincore( - addr: *mut ::c_void, - len: ::size_t, - vec: *mut ::c_char, + pub fn accept4( + s: ::c_int, + addr: *mut ::sockaddr, + addrlen: *mut ::socklen_t, + flags: ::c_int, ) -> ::c_int; + pub fn mincore(addr: *mut ::c_void, len: ::size_t, vec: *mut ::c_char) -> ::c_int; #[cfg_attr(target_os = "netbsd", link_name = "__clock_getres50")] pub fn clock_getres(clk_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; #[cfg_attr(target_os = "netbsd", link_name = "__clock_gettime50")] pub fn clock_gettime(clk_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; #[cfg_attr(target_os = "netbsd", link_name = "__clock_settime50")] - pub fn clock_settime( - clk_id: ::clockid_t, - tp: *const ::timespec, - ) -> ::c_int; + pub fn clock_settime(clk_id: ::clockid_t, tp: *const ::timespec) -> ::c_int; pub fn __errno() -> *mut ::c_int; - pub fn shm_open( - name: *const ::c_char, - oflag: ::c_int, - mode: ::mode_t, - ) -> ::c_int; - pub fn memrchr( - cx: *const ::c_void, - c: ::c_int, - n: ::size_t, - ) -> *mut ::c_void; + pub fn shm_open(name: *const ::c_char, oflag: ::c_int, mode: ::mode_t) -> ::c_int; + pub fn memrchr(cx: *const ::c_void, c: ::c_int, n: ::size_t) -> *mut ::c_void; pub fn mkostemp(template: *mut ::c_char, flags: ::c_int) -> ::c_int; - pub fn mkostemps( - template: *mut ::c_char, - suffixlen: ::c_int, - flags: ::c_int, - ) -> ::c_int; - pub fn pwritev( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - offset: ::off_t, - ) -> ::ssize_t; - pub fn preadv( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - offset: ::off_t, - ) -> ::ssize_t; + pub fn mkostemps(template: *mut ::c_char, suffixlen: ::c_int, flags: ::c_int) -> ::c_int; + pub fn pwritev(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int, offset: ::off_t) + -> ::ssize_t; + pub fn preadv(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int, offset: ::off_t) -> ::ssize_t; pub fn futimens(fd: ::c_int, times: *const ::timespec) -> ::c_int; pub fn utimensat( dirfd: ::c_int, @@ -683,19 +682,6 @@ extern "C" { flag: ::c_int, ) -> ::c_int; pub fn fdatasync(fd: ::c_int) -> ::c_int; - pub fn openpty( - amaster: *mut ::c_int, - aslave: *mut ::c_int, - name: *mut ::c_char, - termp: *mut termios, - winp: *mut ::winsize, - ) -> ::c_int; - pub fn forkpty( - amaster: *mut ::c_int, - name: *mut ::c_char, - termp: *mut termios, - winp: *mut ::winsize, - ) -> ::pid_t; pub fn login_tty(fd: ::c_int) -> ::c_int; pub fn getpriority(which: ::c_int, who: ::id_t) -> ::c_int; pub fn setpriority(which: ::c_int, who: ::id_t, prio: ::c_int) -> ::c_int; @@ -706,15 +692,8 @@ extern "C" { mode: ::mode_t, dev: dev_t, ) -> ::c_int; - pub fn mkfifoat( - dirfd: ::c_int, - pathname: *const ::c_char, - mode: ::mode_t, - ) -> ::c_int; - pub fn sem_timedwait( - sem: *mut sem_t, - abstime: *const ::timespec, - ) -> ::c_int; + pub fn mkfifoat(dirfd: ::c_int, pathname: *const ::c_char, mode: ::mode_t) -> ::c_int; + pub fn sem_timedwait(sem: *mut sem_t, abstime: *const ::timespec) -> ::c_int; pub fn sem_getvalue(sem: *mut sem_t, sval: *mut ::c_int) -> ::c_int; pub fn pthread_condattr_setclock( attr: *mut pthread_condattr_t, @@ -725,6 +704,21 @@ extern "C" { lock: *mut pthread_mutex_t, abstime: *const ::timespec, ) -> ::c_int; + pub fn pthread_spin_init(lock: *mut pthread_spinlock_t, pshared: ::c_int) -> ::c_int; + pub fn pthread_spin_destroy(lock: *mut pthread_spinlock_t) -> ::c_int; + pub fn pthread_spin_lock(lock: *mut pthread_spinlock_t) -> ::c_int; + pub fn pthread_spin_trylock(lock: *mut pthread_spinlock_t) -> ::c_int; + pub fn pthread_spin_unlock(lock: *mut pthread_spinlock_t) -> ::c_int; + pub fn pthread_setschedparam( + native: ::pthread_t, + policy: ::c_int, + param: *const sched_param, + ) -> ::c_int; + pub fn pthread_getschedparam( + native: ::pthread_t, + policy: *mut ::c_int, + param: *mut sched_param, + ) -> ::c_int; pub fn pipe2(fds: *mut ::c_int, flags: ::c_int) -> ::c_int; pub fn getgrouplist( @@ -739,17 +733,19 @@ extern "C" { pub fn uname(buf: *mut ::utsname) -> ::c_int; pub fn shmget(key: ::key_t, size: ::size_t, shmflg: ::c_int) -> ::c_int; - pub fn shmat( - shmid: ::c_int, - shmaddr: *const ::c_void, - shmflg: ::c_int, - ) -> *mut ::c_void; + pub fn shmat(shmid: ::c_int, shmaddr: *const ::c_void, shmflg: ::c_int) -> *mut ::c_void; pub fn shmdt(shmaddr: *const ::c_void) -> ::c_int; - pub fn shmctl( - shmid: ::c_int, - cmd: ::c_int, - buf: *mut ::shmid_ds, - ) -> ::c_int; + pub fn shmctl(shmid: ::c_int, cmd: ::c_int, buf: *mut ::shmid_ds) -> ::c_int; +} + +extern "C" { + pub fn reallocarray(ptr: *mut ::c_void, nmemb: ::size_t, size: ::size_t) -> *mut ::c_void; + pub fn gethostid() -> ::c_long; + pub fn sethostid(hostid: ::c_long) -> ::c_int; + pub fn ftok(path: *const ::c_char, id: ::c_int) -> ::key_t; + + pub fn dirname(path: *mut ::c_char) -> *mut ::c_char; + pub fn basename(path: *mut ::c_char) -> *mut ::c_char; } cfg_if! { diff --git a/crux-mir/lib/libc/src/unix/bsd/netbsdlike/netbsd/aarch64.rs b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/netbsd/aarch64.rs index 58c4cf7c4..7b895f632 100644 --- a/crux-mir/lib/libc/src/unix/bsd/netbsdlike/netbsd/aarch64.rs +++ b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/netbsd/aarch64.rs @@ -3,8 +3,89 @@ use PT_FIRSTMACH; pub type c_long = i64; pub type c_ulong = u64; pub type c_char = u8; +pub type greg_t = u64; pub type __cpu_simple_lock_nv_t = ::c_uchar; +s! { + pub struct __fregset { + #[cfg(libc_union)] + pub __qregs: [__c_anonymous__freg; 32], + pub __fpcr: u32, + pub __fpsr: u32, + } + + pub struct mcontext_t { + pub __gregs: [::greg_t; 32], + pub __fregs: __fregset, + __spare: [::greg_t; 8], + } + + pub struct ucontext_t { + pub uc_flags: ::c_uint, + pub uc_link: *mut ucontext_t, + pub uc_sigmask: ::sigset_t, + pub uc_stack: ::stack_t, + pub uc_mcontext: mcontext_t, + } +} + +s_no_extra_traits! { + #[cfg(libc_union)] + #[repr(align(16))] + pub union __c_anonymous__freg { + pub __b8: [u8; 16], + pub __h16: [u16; 8], + pub __s32: [u32; 4], + pub __d64: [u64; 2], + pub __q128: [u128; 1], + } +} + +cfg_if! { + if #[cfg(feature = "extra_traits")] { + #[cfg(libc_union)] + impl PartialEq for __c_anonymous__freg { + fn eq(&self, other: &__c_anonymous__freg) -> bool { + unsafe { + self.__b8 == other.__b8 + || self.__h16 == other.__h16 + || self.__s32 == other.__s32 + || self.__d64 == other.__d64 + || self.__q128 == other.__q128 + } + } + } + #[cfg(libc_union)] + impl Eq for __c_anonymous__freg {} + #[cfg(libc_union)] + impl ::fmt::Debug for __c_anonymous__freg { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + unsafe { + f.debug_struct("__c_anonymous__freg") + .field("__b8", &self.__b8) + .field("__h16", &self.__h16) + .field("__s32", &self.__s32) + .field("__d64", &self.__d64) + .field("__q128", &self.__q128) + .finish() + } + } + } + #[cfg(libc_union)] + impl ::hash::Hash for __c_anonymous__freg { + fn hash(&self, state: &mut H) { + unsafe { + self.__b8.hash(state); + self.__h16.hash(state); + self.__s32.hash(state); + self.__d64.hash(state); + self.__q128.hash(state); + } + } + } + } +} + // should be pub(crate), but that requires Rust 1.18.0 cfg_if! { if #[cfg(libc_const_size_of)] { diff --git a/crux-mir/lib/libc/src/unix/bsd/netbsdlike/netbsd/mod.rs b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/netbsd/mod.rs index 09dfb22db..41f6b23d1 100644 --- a/crux-mir/lib/libc/src/unix/bsd/netbsdlike/netbsd/mod.rs +++ b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/netbsd/mod.rs @@ -7,11 +7,59 @@ pub type fsfilcnt_t = u64; pub type idtype_t = ::c_int; pub type mqd_t = ::c_int; type __pthread_spin_t = __cpu_simple_lock_nv_t; -pub type vm_size_t = ::uintptr_t; +pub type vm_size_t = ::uintptr_t; // FIXME: deprecated since long time pub type lwpid_t = ::c_uint; pub type shmatt_t = ::c_uint; +pub type cpuid_t = u64; +pub type cpuset_t = _cpuset; +pub type pthread_spin_t = ::c_uchar; +pub type timer_t = ::c_int; + +// elf.h + +pub type Elf32_Addr = u32; +pub type Elf32_Half = u16; +pub type Elf32_Lword = u64; +pub type Elf32_Off = u32; +pub type Elf32_Sword = i32; +pub type Elf32_Word = u32; + +pub type Elf64_Addr = u64; +pub type Elf64_Half = u16; +pub type Elf64_Lword = u64; +pub type Elf64_Off = u64; +pub type Elf64_Sword = i32; +pub type Elf64_Sxword = i64; +pub type Elf64_Word = u32; +pub type Elf64_Xword = u64; + +pub type iconv_t = *mut ::c_void; + +e! { + pub enum fae_action { + FAE_OPEN, + FAE_DUP2, + FAE_CLOSE, + } +} + +cfg_if! { + if #[cfg(target_pointer_width = "64")] { + type Elf_Addr = Elf64_Addr; + type Elf_Half = Elf64_Half; + type Elf_Phdr = Elf64_Phdr; + } else if #[cfg(target_pointer_width = "32")] { + type Elf_Addr = Elf32_Addr; + type Elf_Half = Elf32_Half; + type Elf_Phdr = Elf32_Phdr; + } +} impl siginfo_t { + pub unsafe fn si_addr(&self) -> *mut ::c_void { + self.si_addr + } + pub unsafe fn si_value(&self) -> ::sigval { #[repr(C)] struct siginfo_timer { @@ -25,6 +73,23 @@ impl siginfo_t { } (*(self as *const siginfo_t as *const siginfo_timer)).value } + + pub unsafe fn si_status(&self) -> ::c_int { + #[repr(C)] + struct siginfo_timer { + _si_signo: ::c_int, + _si_errno: ::c_int, + _si_code: ::c_int, + __pad1: ::c_int, + _pid: ::pid_t, + _uid: ::uid_t, + _value: ::sigval, + _cpid: ::pid_t, + _cuid: ::uid_t, + status: ::c_int, + } + (*(self as *const siginfo_t as *const siginfo_timer)).status + } } s! { @@ -64,6 +129,11 @@ s! { pub mq_curmsgs: ::c_long, } + pub struct itimerspec { + pub it_interval: ::timespec, + pub it_value: ::timespec, + } + pub struct sigset_t { __bits: [u32; 4], } @@ -92,7 +162,7 @@ s! { pub st_spare: [u32; 2], } - pub struct addrinfo { + pub struct addrinfo { pub ai_flags: ::c_int, pub ai_family: ::c_int, pub ai_socktype: ::c_int, @@ -171,13 +241,19 @@ s! { ptr_private: *mut ::c_void, } + pub struct pthread_spinlock_t { + pts_magic: ::c_uint, + pts_spin: ::pthread_spin_t, + pts_flags: ::c_int, + } + pub struct kevent { pub ident: ::uintptr_t, pub filter: u32, pub flags: u32, pub fflags: u32, pub data: i64, - pub udata: ::intptr_t, + pub udata: ::intptr_t, /* FIXME: NetBSD 10.0 will finally have same layout as other BSD */ } pub struct dqblk { @@ -267,6 +343,20 @@ s! { pub sc_groups: [::gid_t; 1], } + pub struct uucred { + pub cr_unused: ::c_ushort, + pub cr_uid: ::uid_t, + pub cr_gid: ::gid_t, + pub cr_ngroups: ::c_int, + pub cr_groups: [::gid_t; NGROUPS_MAX as usize], + } + + pub struct unpcbid { + pub unp_pid: ::pid_t, + pub unp_euid: ::uid_t, + pub unp_egid: ::gid_t, + } + pub struct sockaddr_dl { pub sdl_len: ::c_uchar, pub sdl_family: ::c_uchar, @@ -286,7 +376,7 @@ s! { pub struct __exit_status { pub e_termination: u16, pub e_exit: u16, - } + } pub struct shmid_ds { pub shm_perm: ::ipc_perm, @@ -341,6 +431,284 @@ s! { pub time_state: ::c_int, } + // elf.h + + pub struct Elf32_Phdr { + pub p_type: Elf32_Word, + pub p_offset: Elf32_Off, + pub p_vaddr: Elf32_Addr, + pub p_paddr: Elf32_Addr, + pub p_filesz: Elf32_Word, + pub p_memsz: Elf32_Word, + pub p_flags: Elf32_Word, + pub p_align: Elf32_Word, + } + + pub struct Elf64_Phdr { + pub p_type: Elf64_Word, + pub p_flags: Elf64_Word, + pub p_offset: Elf64_Off, + pub p_vaddr: Elf64_Addr, + pub p_paddr: Elf64_Addr, + pub p_filesz: Elf64_Xword, + pub p_memsz: Elf64_Xword, + pub p_align: Elf64_Xword, + } + + pub struct Aux32Info { + pub a_type: Elf32_Word, + pub a_v: Elf32_Word, + } + + pub struct Aux64Info { + pub a_type: Elf64_Word, + pub a_v: Elf64_Xword, + } + + // link.h + + pub struct dl_phdr_info { + pub dlpi_addr: Elf_Addr, + pub dlpi_name: *const ::c_char, + pub dlpi_phdr: *const Elf_Phdr, + pub dlpi_phnum: Elf_Half, + pub dlpi_adds: ::c_ulonglong, + pub dlpi_subs: ::c_ulonglong, + pub dlpi_tls_modid: usize, + pub dlpi_tls_data: *mut ::c_void, + } + + pub struct _cpuset { + bits: [u32; 0] + } + + pub struct accept_filter_arg { + pub af_name: [::c_char; 16], + af_arg: [[::c_char; 10]; 24], + } + + pub struct ki_sigset_t { + pub __bits: [u32; 4], + } + + pub struct kinfo_proc2 { + pub p_forw: u64, + pub p_back: u64, + pub p_paddr: u64, + pub p_addr: u64, + pub p_fd: u64, + pub p_cwdi: u64, + pub p_stats: u64, + pub p_limit: u64, + pub p_vmspace: u64, + pub p_sigacts: u64, + pub p_sess: u64, + pub p_tsess: u64, + pub p_ru: u64, + pub p_eflag: i32, + pub p_exitsig: i32, + pub p_flag: i32, + pub p_pid: i32, + pub p_ppid: i32, + pub p_sid: i32, + pub p__pgid: i32, + pub p_tpgid: i32, + pub p_uid: u32, + pub p_ruid: u32, + pub p_gid: u32, + pub p_rgid: u32, + pub p_groups: [u32; KI_NGROUPS as usize], + pub p_ngroups: i16, + pub p_jobc: i16, + pub p_tdev: u32, + pub p_estcpu: u32, + pub p_rtime_sec: u32, + pub p_rtime_usec: u32, + pub p_cpticks: i32, + pub p_pctcpu: u32, + pub p_swtime: u32, + pub p_slptime: u32, + pub p_schedflags: i32, + pub p_uticks: u64, + pub p_sticks: u64, + pub p_iticks: u64, + pub p_tracep: u64, + pub p_traceflag: i32, + pub p_holdcnt: i32, + pub p_siglist: ki_sigset_t, + pub p_sigmask: ki_sigset_t, + pub p_sigignore: ki_sigset_t, + pub p_sigcatch: ki_sigset_t, + pub p_stat: i8, + pub p_priority: u8, + pub p_usrpri: u8, + pub p_nice: u8, + pub p_xstat: u16, + pub p_acflag: u16, + pub p_comm: [::c_char; KI_MAXCOMLEN as usize], + pub p_wmesg: [::c_char; KI_WMESGLEN as usize], + pub p_wchan: u64, + pub p_login: [::c_char; KI_MAXLOGNAME as usize], + pub p_vm_rssize: i32, + pub p_vm_tsize: i32, + pub p_vm_dsize: i32, + pub p_vm_ssize: i32, + pub p_uvalid: i64, + pub p_ustart_sec: u32, + pub p_ustart_usec: u32, + pub p_uutime_sec: u32, + pub p_uutime_usec: u32, + pub p_ustime_sec: u32, + pub p_ustime_usec: u32, + pub p_uru_maxrss: u64, + pub p_uru_ixrss: u64, + pub p_uru_idrss: u64, + pub p_uru_isrss: u64, + pub p_uru_minflt: u64, + pub p_uru_majflt: u64, + pub p_uru_nswap: u64, + pub p_uru_inblock: u64, + pub p_uru_oublock: u64, + pub p_uru_msgsnd: u64, + pub p_uru_msgrcv: u64, + pub p_uru_nsignals: u64, + pub p_uru_nvcsw: u64, + pub p_uru_nivcsw: u64, + pub p_uctime_sec: u32, + pub p_uctime_usec: u32, + pub p_cpuid: u64, + pub p_realflag: u64, + pub p_nlwps: u64, + pub p_nrlwps: u64, + pub p_realstat: u64, + pub p_svuid: u32, + pub p_svgid: u32, + pub p_ename: [::c_char; KI_MAXEMULLEN as usize], + pub p_vm_vsize: i64, + pub p_vm_msize: i64, + } + + pub struct kinfo_lwp { + pub l_forw: u64, + pub l_back: u64, + pub l_laddr: u64, + pub l_addr: u64, + pub l_lid: i32, + pub l_flag: i32, + pub l_swtime: u32, + pub l_slptime: u32, + pub l_schedflags: i32, + pub l_holdcnt: i32, + pub l_priority: u8, + pub l_usrpri: u8, + pub l_stat: i8, + l_pad1: i8, + l_pad2: i32, + pub l_wmesg: [::c_char; KI_WMESGLEN as usize], + pub l_wchan: u64, + pub l_cpuid: u64, + pub l_rtime_sec: u32, + pub l_rtime_usec: u32, + pub l_cpticks: u32, + pub l_pctcpu: u32, + pub l_pid: u32, + pub l_name: [::c_char; KI_LNAMELEN as usize], + } + + pub struct kinfo_vmentry { + pub kve_start: u64, + pub kve_end: u64, + pub kve_offset: u64, + pub kve_type: u32, + pub kve_flags: u32, + pub kve_count: u32, + pub kve_wired_count: u32, + pub kve_advice: u32, + pub kve_attributes: u32, + pub kve_protection: u32, + pub kve_max_protection: u32, + pub kve_ref_count: u32, + pub kve_inheritance: u32, + pub kve_vn_fileid: u64, + pub kve_vn_size: u64, + pub kve_vn_fsid: u64, + pub kve_vn_rdev: u64, + pub kve_vn_type: u32, + pub kve_vn_mode: u32, + pub kve_path: [[::c_char; 32]; 32], + } + + pub struct __c_anonymous_posix_spawn_fae_open { + pub path: *mut ::c_char, + pub oflag: ::c_int, + pub mode: ::mode_t, + } + + pub struct __c_anonymous_posix_spawn_fae_dup2 { + pub newfildes: ::c_int, + } + + pub struct posix_spawnattr_t { + pub sa_flags: ::c_short, + pub sa_pgroup: ::pid_t, + pub sa_schedparam: ::sched_param, + pub sa_schedpolicy: ::c_int, + pub sa_sigdefault: sigset_t, + pub sa_sigmask: sigset_t, + } + + pub struct posix_spawn_file_actions_entry_t { + pub fae_action: fae_action, + pub fae_fildes: ::c_int, + #[cfg(libc_union)] + pub fae_data: __c_anonymous_posix_spawn_fae, + } + + pub struct posix_spawn_file_actions_t { + pub size: ::c_uint, + pub len: ::c_uint, + #[cfg(libc_union)] + pub fae: *mut posix_spawn_file_actions_entry_t, + } + + pub struct ptrace_lwpinfo { + pub pl_lwpid: lwpid_t, + pub pl_event: ::c_int, + } + + pub struct ptrace_lwpstatus { + pub pl_lwpid: lwpid_t, + pub pl_sigpend: sigset_t, + pub pl_sigmask: sigset_t, + pub pl_name: [::c_char; 20], + pub pl_private: *mut ::c_void, + } + + pub struct ptrace_siginfo { + pub psi_siginfo: siginfo_t, + pub psi_lwpid: lwpid_t, + } + + pub struct ptrace_event { + pub pe_set_event: ::c_int, + } + + pub struct sysctldesc { + pub descr_num: i32, + pub descr_ver: u32, + pub descr_len: u32, + pub descr_str: [::c_char; 1], + } + + pub struct ifreq { + pub _priv: [[::c_char; 6]; 24], + } + + pub struct ifconf { + pub ifc_len: ::c_int, + #[cfg(libc_union)] + pub ifc_ifcu: __c_anonymous_ifc_ifcu, + } } s_no_extra_traits! { @@ -353,7 +721,7 @@ s_no_extra_traits! { pub ut_session: u16, pub ut_type: u16, pub ut_pid: ::pid_t, - pub ut_exit: __exit_status, + pub ut_exit: __exit_status, // FIXME: when anonymous struct are supported pub ut_ss: sockaddr_storage, pub ut_tv: ::timeval, pub ut_pad: [u8; _UTX_PADSIZE], @@ -371,7 +739,6 @@ s_no_extra_traits! { pub ipi_ifindex: ::c_uint, } - #[repr(packed)] pub struct arphdr { pub ar_hrd: u16, pub ar_pro: u16, @@ -380,7 +747,6 @@ s_no_extra_traits! { pub ar_op: u16, } - #[repr(packed)] pub struct in_addr { pub s_addr: ::in_addr_t, } @@ -455,6 +821,18 @@ s_no_extra_traits! { __unused1: *mut ::c_void, //actually a function pointer pub sigev_notify_attributes: *mut ::c_void } + + #[cfg(libc_union)] + pub union __c_anonymous_posix_spawn_fae { + pub open: __c_anonymous_posix_spawn_fae_open, + pub dup2: __c_anonymous_posix_spawn_fae_dup2, + } + + #[cfg(libc_union)] + pub union __c_anonymous_ifc_ifcu { + pub ifcu_buf: *mut ::c_void, + pub ifcu_req: *mut ifreq, + } } cfg_if! { @@ -882,6 +1260,76 @@ cfg_if! { self.sigev_notify_attributes.hash(state); } } + + #[cfg(libc_union)] + impl Eq for __c_anonymous_posix_spawn_fae {} + + #[cfg(libc_union)] + impl PartialEq for __c_anonymous_posix_spawn_fae { + fn eq(&self, other: &__c_anonymous_posix_spawn_fae) -> bool { + unsafe { + self.open == other.open + || self.dup2 == other.dup2 + } + } + } + + #[cfg(libc_union)] + impl ::fmt::Debug for __c_anonymous_posix_spawn_fae { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + unsafe { + f.debug_struct("__c_anonymous_posix_fae") + .field("open", &self.open) + .field("dup2", &self.dup2) + .finish() + } + } + } + + #[cfg(libc_union)] + impl ::hash::Hash for __c_anonymous_posix_spawn_fae { + fn hash(&self, state: &mut H) { + unsafe { + self.open.hash(state); + self.dup2.hash(state); + } + } + } + + #[cfg(libc_union)] + impl Eq for __c_anonymous_ifc_ifcu {} + + #[cfg(libc_union)] + impl PartialEq for __c_anonymous_ifc_ifcu { + fn eq(&self, other: &__c_anonymous_ifc_ifcu) -> bool { + unsafe { + self.ifcu_buf == other.ifcu_buf + || self.ifcu_req == other.ifcu_req + } + } + } + + #[cfg(libc_union)] + impl ::fmt::Debug for __c_anonymous_ifc_ifcu { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + unsafe { + f.debug_struct("__c_anonymous_ifc_ifcu") + .field("ifcu_buf", &self.ifcu_buf) + .field("ifcu_req", &self.ifcu_req) + .finish() + } + } + } + + #[cfg(libc_union)] + impl ::hash::Hash for __c_anonymous_ifc_ifcu { + fn hash(&self, state: &mut H) { + unsafe { + self.ifcu_buf.hash(state); + self.ifcu_req.hash(state); + } + } + } } } @@ -891,6 +1339,36 @@ pub const AT_SYMLINK_NOFOLLOW: ::c_int = 0x200; pub const AT_SYMLINK_FOLLOW: ::c_int = 0x400; pub const AT_REMOVEDIR: ::c_int = 0x800; +pub const AT_NULL: ::c_int = 0; +pub const AT_IGNORE: ::c_int = 1; +pub const AT_EXECFD: ::c_int = 2; +pub const AT_PHDR: ::c_int = 3; +pub const AT_PHENT: ::c_int = 4; +pub const AT_PHNUM: ::c_int = 5; +pub const AT_PAGESZ: ::c_int = 6; +pub const AT_BASE: ::c_int = 7; +pub const AT_FLAGS: ::c_int = 8; +pub const AT_ENTRY: ::c_int = 9; +pub const AT_DCACHEBSIZE: ::c_int = 10; +pub const AT_ICACHEBSIZE: ::c_int = 11; +pub const AT_UCACHEBSIZE: ::c_int = 12; +pub const AT_STACKBASE: ::c_int = 13; +pub const AT_EUID: ::c_int = 2000; +pub const AT_RUID: ::c_int = 2001; +pub const AT_EGID: ::c_int = 2002; +pub const AT_RGID: ::c_int = 2003; +pub const AT_SUN_LDELF: ::c_int = 2004; +pub const AT_SUN_LDSHDR: ::c_int = 2005; +pub const AT_SUN_LDNAME: ::c_int = 2006; +pub const AT_SUN_LDPGSIZE: ::c_int = 2007; +pub const AT_SUN_PLATFORM: ::c_int = 2008; +pub const AT_SUN_HWCAP: ::c_int = 2009; +pub const AT_SUN_IFLUSH: ::c_int = 2010; +pub const AT_SUN_CPU: ::c_int = 2011; +pub const AT_SUN_EMUL_ENTRY: ::c_int = 2012; +pub const AT_SUN_EMUL_EXECFD: ::c_int = 2013; +pub const AT_SUN_EXECNAME: ::c_int = 2014; + pub const EXTATTR_NAMESPACE_USER: ::c_int = 1; pub const EXTATTR_NAMESPACE_SYSTEM: ::c_int = 2; @@ -919,6 +1397,12 @@ pub const O_RSYNC: ::c_int = 0x00020000; pub const MS_SYNC: ::c_int = 0x4; pub const MS_INVALIDATE: ::c_int = 0x2; +// Here because they are not present on OpenBSD +// (https://github.com/openbsd/src/blob/master/sys/sys/resource.h) +pub const RLIMIT_SBSIZE: ::c_int = 9; +pub const RLIMIT_AS: ::c_int = 10; +pub const RLIMIT_NTHR: ::c_int = 11; + #[deprecated(since = "0.2.64", note = "Not stable across OS versions")] pub const RLIM_NLIMITS: ::c_int = 12; @@ -944,6 +1428,30 @@ pub const F_CLOSEM: ::c_int = 10; pub const F_GETNOSIGPIPE: ::c_int = 13; pub const F_SETNOSIGPIPE: ::c_int = 14; pub const F_MAXFD: ::c_int = 11; +pub const F_GETPATH: ::c_int = 15; + +pub const FUTEX_WAIT: ::c_int = 0; +pub const FUTEX_WAKE: ::c_int = 1; +pub const FUTEX_FD: ::c_int = 2; +pub const FUTEX_REQUEUE: ::c_int = 3; +pub const FUTEX_CMP_REQUEUE: ::c_int = 4; +pub const FUTEX_WAKE_OP: ::c_int = 5; +pub const FUTEX_LOCK_PI: ::c_int = 6; +pub const FUTEX_UNLOCK_PI: ::c_int = 7; +pub const FUTEX_TRYLOCK_PI: ::c_int = 8; +pub const FUTEX_WAIT_BITSET: ::c_int = 9; +pub const FUTEX_WAKE_BITSET: ::c_int = 10; +pub const FUTEX_WAIT_REQUEUE_PI: ::c_int = 11; +pub const FUTEX_CMP_REQUEUE_PI: ::c_int = 12; +pub const FUTEX_PRIVATE_FLAG: ::c_int = 1 << 7; +pub const FUTEX_CLOCK_REALTIME: ::c_int = 1 << 8; +pub const FUTEX_CMD_MASK: ::c_int = !(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME); +pub const FUTEX_WAITERS: u32 = 1 << 31; +pub const FUTEX_OWNER_DIED: u32 = 1 << 30; +pub const FUTEX_SYNCOBJ_1: u32 = 1 << 29; +pub const FUTEX_SYNCOBJ_0: u32 = 1 << 28; +pub const FUTEX_TID_MASK: u32 = (1 << 28) - 1; +pub const FUTEX_BITSET_MATCH_ANY: u32 = !0; pub const IP_RECVDSTADDR: ::c_int = 7; pub const IP_SENDSRCADDR: ::c_int = IP_RECVDSTADDR; @@ -973,6 +1481,12 @@ pub const SO_TIMESTAMP: ::c_int = 0x2000; pub const SO_OVERFLOWED: ::c_int = 0x1009; pub const SO_NOHEADER: ::c_int = 0x100a; +// http://cvsweb.netbsd.org/bsdweb.cgi/src/sys/sys/un.h?annotate +pub const LOCAL_OCREDS: ::c_int = 0x0001; // pass credentials to receiver +pub const LOCAL_CONNWAIT: ::c_int = 0x0002; // connects block until accepted +pub const LOCAL_PEEREID: ::c_int = 0x0003; // get peer identification +pub const LOCAL_CREDS: ::c_int = 0x0004; // pass credentials to receiver + // https://github.com/NetBSD/src/blob/trunk/sys/net/if.h#L373 pub const IFF_UP: ::c_int = 0x0001; // interface is up pub const IFF_BROADCAST: ::c_int = 0x0002; // broadcast address valid @@ -1054,10 +1568,7 @@ pub const IPPROTO_VRRP: ::c_int = 112; /// Common Address Resolution Protocol pub const IPPROTO_CARP: ::c_int = 112; /// L2TPv3 -// TEMP: Disabled for now; this constant was added to NetBSD on 2017-02-16, -// but isn't yet supported by the NetBSD rumprun kernel image used for -// libc testing. -//pub const IPPROTO_L2TP: ::c_int = 115; +pub const IPPROTO_L2TP: ::c_int = 115; /// SCTP pub const IPPROTO_SCTP: ::c_int = 132; /// PFSYNC @@ -1106,6 +1617,18 @@ pub const MAP_RENAME: ::c_int = 0x20; pub const MAP_NORESERVE: ::c_int = 0x40; pub const MAP_HASSEMAPHORE: ::c_int = 0x200; pub const MAP_WIRED: ::c_int = 0x800; +pub const MAP_STACK: ::c_int = 0x2000; +// map alignment aliases for MAP_ALIGNED +pub const MAP_ALIGNMENT_SHIFT: ::c_int = 24; +pub const MAP_ALIGNMENT_MASK: ::c_int = 0xff << MAP_ALIGNMENT_SHIFT; +pub const MAP_ALIGNMENT_64KB: ::c_int = 16 << MAP_ALIGNMENT_SHIFT; +pub const MAP_ALIGNMENT_16MB: ::c_int = 24 << MAP_ALIGNMENT_SHIFT; +pub const MAP_ALIGNMENT_4GB: ::c_int = 32 << MAP_ALIGNMENT_SHIFT; +pub const MAP_ALIGNMENT_1TB: ::c_int = 40 << MAP_ALIGNMENT_SHIFT; +pub const MAP_ALIGNMENT_256TB: ::c_int = 48 << MAP_ALIGNMENT_SHIFT; +pub const MAP_ALIGNMENT_64PB: ::c_int = 56 << MAP_ALIGNMENT_SHIFT; +// mremap flag +pub const MAP_REMAPDUP: ::c_int = 0x004; pub const DCCP_TYPE_REQUEST: ::c_int = 0; pub const DCCP_TYPE_RESPONSE: ::c_int = 1; @@ -1262,6 +1785,23 @@ pub const BIOCSDLT: ::c_ulong = 0x80044278; pub const BIOCGSEESENT: ::c_ulong = 0x40044276; pub const BIOCSSEESENT: ::c_ulong = 0x80044277; +// +pub const MNT_UNION: ::c_int = 0x00000020; +pub const MNT_NOCOREDUMP: ::c_int = 0x00008000; +pub const MNT_RELATIME: ::c_int = 0x00020000; +pub const MNT_IGNORE: ::c_int = 0x00100000; +pub const MNT_NFS4ACLS: ::c_int = 0x00200000; +pub const MNT_DISCARD: ::c_int = 0x00800000; +pub const MNT_EXTATTR: ::c_int = 0x01000000; +pub const MNT_LOG: ::c_int = 0x02000000; +pub const MNT_NOATIME: ::c_int = 0x04000000; +pub const MNT_AUTOMOUNTED: ::c_int = 0x10000000; +pub const MNT_SYMPERM: ::c_int = 0x20000000; +pub const MNT_NODEVMTIME: ::c_int = 0x40000000; +pub const MNT_SOFTDEP: ::c_int = 0x80000000; +pub const MNT_POSIX1EACLS: ::c_int = 0x00000800; +pub const MNT_ACLS: ::c_int = MNT_POSIX1EACLS; + // pub const NTP_API: ::c_int = 4; pub const MAXPHASE: ::c_long = 500000000; @@ -1314,6 +1854,13 @@ pub const TIME_OOP: ::c_int = 3; pub const TIME_WAIT: ::c_int = 4; pub const TIME_ERROR: ::c_int = 5; +pub const LITTLE_ENDIAN: ::c_int = 1234; +pub const BIG_ENDIAN: ::c_int = 4321; + +pub const PL_EVENT_NONE: ::c_int = 0; +pub const PL_EVENT_SIGNAL: ::c_int = 1; +pub const PL_EVENT_SUSPENDED: ::c_int = 2; + cfg_if! { if #[cfg(any(target_arch = "sparc", target_arch = "sparc64", target_arch = "x86", target_arch = "x86_64"))] { @@ -1367,6 +1914,11 @@ pub const PTHREAD_MUTEX_ERRORCHECK: ::c_int = 1; pub const PTHREAD_MUTEX_RECURSIVE: ::c_int = 2; pub const PTHREAD_MUTEX_DEFAULT: ::c_int = PTHREAD_MUTEX_NORMAL; +pub const SCHED_NONE: ::c_int = -1; +pub const SCHED_OTHER: ::c_int = 0; +pub const SCHED_FIFO: ::c_int = 1; +pub const SCHED_RR: ::c_int = 2; + pub const EVFILT_AIO: u32 = 2; pub const EVFILT_PROC: u32 = 4; pub const EVFILT_READ: u32 = 0; @@ -1407,7 +1959,23 @@ pub const NOTE_CHILD: u32 = 0x00000004; pub const TMP_MAX: ::c_uint = 308915776; +pub const AI_PASSIVE: ::c_int = 0x00000001; +pub const AI_CANONNAME: ::c_int = 0x00000002; +pub const AI_NUMERICHOST: ::c_int = 0x00000004; +pub const AI_NUMERICSERV: ::c_int = 0x00000008; +pub const AI_ADDRCONFIG: ::c_int = 0x00000400; +pub const AI_SRV: ::c_int = 0x00000800; + pub const NI_MAXHOST: ::socklen_t = 1025; +pub const NI_MAXSERV: ::socklen_t = 32; + +pub const NI_NOFQDN: ::c_int = 0x00000001; +pub const NI_NUMERICHOST: ::c_int = 0x000000002; +pub const NI_NAMEREQD: ::c_int = 0x000000004; +pub const NI_NUMERICSERV: ::c_int = 0x000000008; +pub const NI_DGRAM: ::c_int = 0x00000010; +pub const NI_WITHSCOPEID: ::c_int = 0x00000020; +pub const NI_NUMERICSCOPE: ::c_int = 0x00000040; pub const RTLD_NOLOAD: ::c_int = 0x2000; pub const RTLD_LOCAL: ::c_int = 0x200; @@ -1561,6 +2129,8 @@ pub const KERN_PROC_NARGV: ::c_int = 2; pub const KERN_PROC_ENV: ::c_int = 3; pub const KERN_PROC_NENV: ::c_int = 4; pub const KERN_PROC_PATHNAME: ::c_int = 5; +pub const VM_PROC: ::c_int = 16; +pub const VM_PROC_MAP: ::c_int = 1; pub const EAI_AGAIN: ::c_int = 2; pub const EAI_BADFLAGS: ::c_int = 3; @@ -1646,6 +2216,8 @@ pub const FIBMAP: ::c_ulong = 0xc008667a; pub const SIGSTKSZ: ::size_t = 40960; +pub const REG_ENOSYS: ::c_int = 17; + pub const PT_DUMPCORE: ::c_int = 12; pub const PT_LWPINFO: ::c_int = 13; pub const PT_SYSCALL: ::c_int = 14; @@ -1653,15 +2225,106 @@ pub const PT_SYSCALLEMU: ::c_int = 15; pub const PT_SET_EVENT_MASK: ::c_int = 16; pub const PT_GET_EVENT_MASK: ::c_int = 17; pub const PT_GET_PROCESS_STATE: ::c_int = 18; +pub const PT_SET_SIGINFO: ::c_int = 19; +pub const PT_GET_SIGINFO: ::c_int = 20; +pub const PT_RESUME: ::c_int = 21; +pub const PT_SUSPEND: ::c_int = 23; +pub const PT_STOP: ::c_int = 23; +pub const PT_LWPSTATUS: ::c_int = 24; +pub const PT_LWPNEXT: ::c_int = 25; +pub const PT_SET_SIGPASS: ::c_int = 26; +pub const PT_GET_SIGPASS: ::c_int = 27; pub const PT_FIRSTMACH: ::c_int = 32; +pub const POSIX_SPAWN_RESETIDS: ::c_int = 0x01; +pub const POSIX_SPAWN_SETPGROUP: ::c_int = 0x02; +pub const POSIX_SPAWN_SETSCHEDPARAM: ::c_int = 0x04; +pub const POSIX_SPAWN_SETSCHEDULER: ::c_int = 0x08; +pub const POSIX_SPAWN_SETSIGDEF: ::c_int = 0x10; +pub const POSIX_SPAWN_SETSIGMASK: ::c_int = 0x20; +pub const POSIX_SPAWN_RETURNERROR: ::c_int = 0x40; + // Flags for chflags(2) -pub const SF_SNAPSHOT: ::c_ulong = 0x00200000; +pub const SF_APPEND: ::c_ulong = 0x00040000; +pub const SF_ARCHIVED: ::c_ulong = 0x00010000; +pub const SF_IMMUTABLE: ::c_ulong = 0x00020000; pub const SF_LOG: ::c_ulong = 0x00400000; +pub const SF_SETTABLE: ::c_ulong = 0xffff0000; pub const SF_SNAPINVAL: ::c_ulong = 0x00800000; - -fn _ALIGN(p: usize) -> usize { - (p + _ALIGNBYTES) & !_ALIGNBYTES +pub const SF_SNAPSHOT: ::c_ulong = 0x00200000; +pub const UF_APPEND: ::c_ulong = 0x00000004; +pub const UF_IMMUTABLE: ::c_ulong = 0x00000002; +pub const UF_NODUMP: ::c_ulong = 0x00000001; +pub const UF_OPAQUE: ::c_ulong = 0x00000008; +pub const UF_SETTABLE: ::c_ulong = 0x0000ffff; + +// sys/sysctl.h +pub const KVME_PROT_READ: ::c_int = 0x00000001; +pub const KVME_PROT_WRITE: ::c_int = 0x00000002; +pub const KVME_PROT_EXEC: ::c_int = 0x00000004; + +pub const KVME_FLAG_COW: ::c_int = 0x00000001; +pub const KVME_FLAG_NEEDS_COPY: ::c_int = 0x00000002; +pub const KVME_FLAG_NOCOREDUMP: ::c_int = 0x000000004; +pub const KVME_FLAG_PAGEABLE: ::c_int = 0x000000008; +pub const KVME_FLAG_GROWS_UP: ::c_int = 0x000000010; +pub const KVME_FLAG_GROWS_DOWN: ::c_int = 0x000000020; + +pub const NGROUPS_MAX: ::c_int = 16; + +pub const KI_NGROUPS: ::c_int = 16; +pub const KI_MAXCOMLEN: ::c_int = 24; +pub const KI_WMESGLEN: ::c_int = 8; +pub const KI_MAXLOGNAME: ::c_int = 24; +pub const KI_MAXEMULLEN: ::c_int = 16; +pub const KI_LNAMELEN: ::c_int = 20; + +// sys/lwp.h +pub const LSIDL: ::c_int = 1; +pub const LSRUN: ::c_int = 2; +pub const LSSLEEP: ::c_int = 3; +pub const LSSTOP: ::c_int = 4; +pub const LSZOMB: ::c_int = 5; +pub const LSONPROC: ::c_int = 7; +pub const LSSUSPENDED: ::c_int = 8; + +pub const _REG_RDI: ::c_int = 0; +pub const _REG_RSI: ::c_int = 1; +pub const _REG_RDX: ::c_int = 2; +pub const _REG_RCX: ::c_int = 3; +pub const _REG_R8: ::c_int = 4; +pub const _REG_R9: ::c_int = 5; +pub const _REG_R10: ::c_int = 6; +pub const _REG_R11: ::c_int = 7; +pub const _REG_R12: ::c_int = 8; +pub const _REG_R13: ::c_int = 9; +pub const _REG_R14: ::c_int = 10; +pub const _REG_R15: ::c_int = 11; +pub const _REG_RBP: ::c_int = 12; +pub const _REG_RBX: ::c_int = 13; +pub const _REG_RAX: ::c_int = 14; +pub const _REG_GS: ::c_int = 15; +pub const _REG_FS: ::c_int = 16; +pub const _REG_ES: ::c_int = 17; +pub const _REG_DS: ::c_int = 18; +pub const _REG_TRAPNO: ::c_int = 19; +pub const _REG_ERR: ::c_int = 20; +pub const _REG_RIP: ::c_int = 21; +pub const _REG_CS: ::c_int = 22; +pub const _REG_RFLAGS: ::c_int = 23; +pub const _REG_RSP: ::c_int = 24; +pub const _REG_SS: ::c_int = 25; + +// sys/xattr.h +pub const XATTR_CREATE: ::c_int = 0x01; +pub const XATTR_REPLACE: ::c_int = 0x02; +// sys/extattr.h +pub const EXTATTR_NAMESPACE_EMPTY: ::c_int = 0; + +const_fn! { + {const} fn _ALIGN(p: usize) -> usize { + (p + _ALIGNBYTES) & !_ALIGNBYTES + } } f! { @@ -1692,23 +2355,11 @@ f! { } } - pub fn CMSG_SPACE(length: ::c_uint) -> ::c_uint { + pub {const} fn CMSG_SPACE(length: ::c_uint) -> ::c_uint { (_ALIGN(::mem::size_of::<::cmsghdr>()) + _ALIGN(length as usize)) as ::c_uint } - pub fn WSTOPSIG(status: ::c_int) -> ::c_int { - status >> 8 - } - - pub fn WIFSIGNALED(status: ::c_int) -> bool { - (status & 0o177) != 0o177 && (status & 0o177) != 0 - } - - pub fn WIFSTOPPED(status: ::c_int) -> bool { - (status & 0o177) == 0o177 - } - // dirfd() is a macro on netbsd to access // the first field of the struct where dirp points to: // http://cvsweb.netbsd.org/bsdweb.cgi/src/include/dirent.h?rev=1.36 @@ -1716,10 +2367,6 @@ f! { *(dirp as *const ::c_int) } - pub fn WIFCONTINUED(status: ::c_int) -> bool { - status == 0xffff - } - pub fn SOCKCREDSIZE(ngrps: usize) -> usize { let ngrps = if ngrps > 0 { ngrps - 1 @@ -1728,11 +2375,55 @@ f! { }; ::mem::size_of::() + ::mem::size_of::<::gid_t>() * ngrps } + + pub fn PROT_MPROTECT(x: ::c_int) -> ::c_int { + x << 3 + } + + pub fn PROT_MPROTECT_EXTRACT(x: ::c_int) -> ::c_int { + (x >> 3) & 0x7 + } +} + +safe_f! { + pub {const} fn WSTOPSIG(status: ::c_int) -> ::c_int { + status >> 8 + } + + pub {const} fn WIFSIGNALED(status: ::c_int) -> bool { + (status & 0o177) != 0o177 && (status & 0o177) != 0 + } + + pub {const} fn WIFSTOPPED(status: ::c_int) -> bool { + (status & 0o177) == 0o177 + } + + pub {const} fn WIFCONTINUED(status: ::c_int) -> bool { + status == 0xffff + } + + pub {const} fn makedev(major: ::c_uint, minor: ::c_uint) -> ::dev_t { + let major = major as ::dev_t; + let minor = minor as ::dev_t; + let mut dev = 0; + dev |= (major << 8) & 0x000ff00; + dev |= (minor << 12) & 0xfff00000; + dev |= minor & 0xff; + dev + } } extern "C" { pub fn ntp_adjtime(buf: *mut timex) -> ::c_int; pub fn ntp_gettime(buf: *mut ntptimeval) -> ::c_int; + pub fn clock_nanosleep( + clk_id: ::clockid_t, + flags: ::c_int, + rqtp: *const ::timespec, + rmtp: *mut ::timespec, + ) -> ::c_int; + + pub fn reallocarr(ptr: *mut ::c_void, number: ::size_t, size: ::size_t) -> ::c_int; } #[link(name = "rt")] @@ -1762,6 +2453,30 @@ extern "C" { pub fn fchflags(fd: ::c_int, flags: ::c_ulong) -> ::c_int; pub fn lchflags(path: *const ::c_char, flags: ::c_ulong) -> ::c_int; + pub fn execvpe( + file: *const ::c_char, + argv: *const *const ::c_char, + envp: *const *const ::c_char, + ) -> ::c_int; + + pub fn extattr_list_fd( + fd: ::c_int, + attrnamespace: ::c_int, + data: *mut ::c_void, + nbytes: ::size_t, + ) -> ::ssize_t; + pub fn extattr_list_file( + path: *const ::c_char, + attrnamespace: ::c_int, + data: *mut ::c_void, + nbytes: ::size_t, + ) -> ::ssize_t; + pub fn extattr_list_link( + path: *const ::c_char, + attrnamespace: ::c_int, + data: *mut ::c_void, + nbytes: ::size_t, + ) -> ::ssize_t; pub fn extattr_delete_fd( fd: ::c_int, attrnamespace: ::c_int, @@ -1828,6 +2543,20 @@ extern "C" { attrnamespace: *mut ::c_int, ) -> ::c_int; + pub fn openpty( + amaster: *mut ::c_int, + aslave: *mut ::c_int, + name: *mut ::c_char, + termp: *mut ::termios, + winp: *mut ::winsize, + ) -> ::c_int; + pub fn forkpty( + amaster: *mut ::c_int, + name: *mut ::c_char, + termp: *mut ::termios, + winp: *mut ::winsize, + ) -> ::pid_t; + #[link_name = "__lutimes50"] pub fn lutimes(file: *const ::c_char, times: *const ::timeval) -> ::c_int; #[link_name = "__gettimeofday50"] @@ -1841,11 +2570,7 @@ extern "C" { sevlen: ::socklen_t, flags: ::c_int, ) -> ::c_int; - pub fn mprotect( - addr: *mut ::c_void, - len: ::size_t, - prot: ::c_int, - ) -> ::c_int; + pub fn mprotect(addr: *mut ::c_void, len: ::size_t, prot: ::c_int) -> ::c_int; pub fn sysctl( name: *const ::c_int, namelen: ::c_uint, @@ -1881,8 +2606,7 @@ extern "C" { pub fn mq_open(name: *const ::c_char, oflag: ::c_int, ...) -> ::mqd_t; pub fn mq_close(mqd: ::mqd_t) -> ::c_int; pub fn mq_getattr(mqd: ::mqd_t, attr: *mut ::mq_attr) -> ::c_int; - pub fn mq_notify(mqd: ::mqd_t, notification: *const ::sigevent) - -> ::c_int; + pub fn mq_notify(mqd: ::mqd_t, notification: *const ::sigevent) -> ::c_int; pub fn mq_receive( mqd: ::mqd_t, msg_ptr: *mut ::c_char, @@ -1895,11 +2619,7 @@ extern "C" { msg_len: ::size_t, msg_prio: ::c_uint, ) -> ::c_int; - pub fn mq_setattr( - mqd: ::mqd_t, - newattr: *const ::mq_attr, - oldattr: *mut ::mq_attr, - ) -> ::c_int; + pub fn mq_setattr(mqd: ::mqd_t, newattr: *const ::mq_attr, oldattr: *mut ::mq_attr) -> ::c_int; #[link_name = "__mq_timedreceive50"] pub fn mq_timedreceive( mqd: ::mqd_t, @@ -1917,25 +2637,16 @@ extern "C" { abs_timeout: *const ::timespec, ) -> ::c_int; pub fn mq_unlink(name: *const ::c_char) -> ::c_int; - pub fn ptrace( - request: ::c_int, - pid: ::pid_t, - addr: *mut ::c_void, - data: ::c_int, - ) -> ::c_int; + pub fn ptrace(request: ::c_int, pid: ::pid_t, addr: *mut ::c_void, data: ::c_int) -> ::c_int; + pub fn utrace(label: *const ::c_char, addr: *mut ::c_void, len: ::size_t) -> ::c_int; + pub fn pthread_getname_np(t: ::pthread_t, name: *mut ::c_char, len: ::size_t) -> ::c_int; pub fn pthread_setname_np( t: ::pthread_t, name: *const ::c_char, arg: *const ::c_void, ) -> ::c_int; - pub fn pthread_attr_get_np( - thread: ::pthread_t, - attr: *mut ::pthread_attr_t, - ) -> ::c_int; - pub fn pthread_getattr_np( - native: ::pthread_t, - attr: *mut ::pthread_attr_t, - ) -> ::c_int; + pub fn pthread_attr_get_np(thread: ::pthread_t, attr: *mut ::pthread_attr_t) -> ::c_int; + pub fn pthread_getattr_np(native: ::pthread_t, attr: *mut ::pthread_attr_t) -> ::c_int; pub fn pthread_attr_getguardsize( attr: *const ::pthread_attr_t, guardsize: *mut ::size_t, @@ -1945,6 +2656,24 @@ extern "C" { stackaddr: *mut *mut ::c_void, stacksize: *mut ::size_t, ) -> ::c_int; + pub fn pthread_getaffinity_np( + thread: ::pthread_t, + size: ::size_t, + set: *mut cpuset_t, + ) -> ::c_int; + pub fn pthread_setaffinity_np( + thread: ::pthread_t, + size: ::size_t, + set: *mut cpuset_t, + ) -> ::c_int; + + pub fn _cpuset_create() -> *mut cpuset_t; + pub fn _cpuset_destroy(set: *mut cpuset_t); + pub fn _cpuset_clr(cpu: cpuid_t, set: *mut cpuset_t) -> ::c_int; + pub fn _cpuset_set(cpu: cpuid_t, set: *mut cpuset_t) -> ::c_int; + pub fn _cpuset_isset(cpu: cpuid_t, set: *const cpuset_t) -> ::c_int; + pub fn _cpuset_size(set: *const cpuset_t) -> ::size_t; + pub fn _cpuset_zero(set: *mut cpuset_t); #[link_name = "__sigtimedwait50"] pub fn sigtimedwait( set: *const sigset_t, @@ -1952,19 +2681,24 @@ extern "C" { timeout: *const ::timespec, ) -> ::c_int; pub fn sigwaitinfo(set: *const sigset_t, info: *mut siginfo_t) -> ::c_int; + pub fn waitid( + idtype: idtype_t, + id: ::id_t, + infop: *mut ::siginfo_t, + options: ::c_int, + ) -> ::c_int; + pub fn duplocale(base: ::locale_t) -> ::locale_t; pub fn freelocale(loc: ::locale_t); pub fn localeconv_l(loc: ::locale_t) -> *mut lconv; - pub fn newlocale( - mask: ::c_int, - locale: *const ::c_char, - base: ::locale_t, - ) -> ::locale_t; + pub fn newlocale(mask: ::c_int, locale: *const ::c_char, base: ::locale_t) -> ::locale_t; #[link_name = "__settimeofday50"] pub fn settimeofday(tv: *const ::timeval, tz: *const ::c_void) -> ::c_int; pub fn dup3(src: ::c_int, dst: ::c_int, flags: ::c_int) -> ::c_int; + pub fn kqueue1(flags: ::c_int) -> ::c_int; + pub fn sendmmsg( sockfd: ::c_int, msgvec: *mut ::mmsghdr, @@ -1986,6 +2720,156 @@ extern "C" { needle: *const ::c_void, needlelen: ::size_t, ) -> *mut ::c_void; + + // link.h + + pub fn dl_iterate_phdr( + callback: ::Option< + unsafe extern "C" fn( + info: *mut dl_phdr_info, + size: usize, + data: *mut ::c_void, + ) -> ::c_int, + >, + data: *mut ::c_void, + ) -> ::c_int; + + // dlfcn.h + + pub fn _dlauxinfo() -> *mut ::c_void; + + pub fn iconv_open(tocode: *const ::c_char, fromcode: *const ::c_char) -> iconv_t; + pub fn iconv( + cd: iconv_t, + inbuf: *mut *mut ::c_char, + inbytesleft: *mut ::size_t, + outbuf: *mut *mut ::c_char, + outbytesleft: *mut ::size_t, + ) -> ::size_t; + pub fn iconv_close(cd: iconv_t) -> ::c_int; + + pub fn timer_create( + clockid: ::clockid_t, + sevp: *mut ::sigevent, + timerid: *mut ::timer_t, + ) -> ::c_int; + pub fn timer_delete(timerid: ::timer_t) -> ::c_int; + pub fn timer_getoverrun(timerid: ::timer_t) -> ::c_int; + pub fn timer_gettime(timerid: ::timer_t, curr_value: *mut ::itimerspec) -> ::c_int; + pub fn timer_settime( + timerid: ::timer_t, + flags: ::c_int, + new_value: *const ::itimerspec, + old_value: *mut ::itimerspec, + ) -> ::c_int; + + // Added in `NetBSD` 7.0 + pub fn explicit_memset(b: *mut ::c_void, c: ::c_int, len: ::size_t); + pub fn consttime_memequal(a: *const ::c_void, b: *const ::c_void, len: ::size_t) -> ::c_int; + + pub fn setproctitle(fmt: *const ::c_char, ...); + pub fn mremap( + oldp: *mut ::c_void, + oldsize: ::size_t, + newp: *mut ::c_void, + newsize: ::size_t, + flags: ::c_int, + ) -> *mut ::c_void; + + pub fn sched_rr_get_interval(pid: ::pid_t, t: *mut ::timespec) -> ::c_int; + pub fn sched_setparam(pid: ::pid_t, param: *const ::sched_param) -> ::c_int; + pub fn sched_getparam(pid: ::pid_t, param: *mut ::sched_param) -> ::c_int; + pub fn sched_getscheduler(pid: ::pid_t) -> ::c_int; + pub fn sched_setscheduler( + pid: ::pid_t, + policy: ::c_int, + param: *const ::sched_param, + ) -> ::c_int; + + #[link_name = "__pollts50"] + pub fn pollts( + fds: *mut ::pollfd, + nfds: ::nfds_t, + ts: *const ::timespec, + sigmask: *const ::sigset_t, + ) -> ::c_int; + + pub fn posix_spawn( + pid: *mut ::pid_t, + path: *const ::c_char, + file_actions: *const ::posix_spawn_file_actions_t, + attrp: *const ::posix_spawnattr_t, + argv: *const *mut ::c_char, + envp: *const *mut ::c_char, + ) -> ::c_int; + pub fn posix_spawnp( + pid: *mut ::pid_t, + file: *const ::c_char, + file_actions: *const ::posix_spawn_file_actions_t, + attrp: *const ::posix_spawnattr_t, + argv: *const *mut ::c_char, + envp: *const *mut ::c_char, + ) -> ::c_int; + pub fn posix_spawnattr_init(attr: *mut posix_spawnattr_t) -> ::c_int; + pub fn posix_spawnattr_destroy(attr: *mut posix_spawnattr_t) -> ::c_int; + pub fn posix_spawnattr_getsigdefault( + attr: *const posix_spawnattr_t, + default: *mut ::sigset_t, + ) -> ::c_int; + pub fn posix_spawnattr_setsigdefault( + attr: *mut posix_spawnattr_t, + default: *const ::sigset_t, + ) -> ::c_int; + pub fn posix_spawnattr_getsigmask( + attr: *const posix_spawnattr_t, + default: *mut ::sigset_t, + ) -> ::c_int; + pub fn posix_spawnattr_setsigmask( + attr: *mut posix_spawnattr_t, + default: *const ::sigset_t, + ) -> ::c_int; + pub fn posix_spawnattr_getflags( + attr: *const posix_spawnattr_t, + flags: *mut ::c_short, + ) -> ::c_int; + pub fn posix_spawnattr_setflags(attr: *mut posix_spawnattr_t, flags: ::c_short) -> ::c_int; + pub fn posix_spawnattr_getpgroup( + attr: *const posix_spawnattr_t, + flags: *mut ::pid_t, + ) -> ::c_int; + pub fn posix_spawnattr_setpgroup(attr: *mut posix_spawnattr_t, flags: ::pid_t) -> ::c_int; + pub fn posix_spawnattr_getschedpolicy( + attr: *const posix_spawnattr_t, + flags: *mut ::c_int, + ) -> ::c_int; + pub fn posix_spawnattr_setschedpolicy(attr: *mut posix_spawnattr_t, flags: ::c_int) -> ::c_int; + pub fn posix_spawnattr_getschedparam( + attr: *const posix_spawnattr_t, + param: *mut ::sched_param, + ) -> ::c_int; + pub fn posix_spawnattr_setschedparam( + attr: *mut posix_spawnattr_t, + param: *const ::sched_param, + ) -> ::c_int; + + pub fn posix_spawn_file_actions_init(actions: *mut posix_spawn_file_actions_t) -> ::c_int; + pub fn posix_spawn_file_actions_destroy(actions: *mut posix_spawn_file_actions_t) -> ::c_int; + pub fn posix_spawn_file_actions_addopen( + actions: *mut posix_spawn_file_actions_t, + fd: ::c_int, + path: *const ::c_char, + oflag: ::c_int, + mode: ::mode_t, + ) -> ::c_int; + pub fn posix_spawn_file_actions_addclose( + actions: *mut posix_spawn_file_actions_t, + fd: ::c_int, + ) -> ::c_int; + pub fn posix_spawn_file_actions_adddup2( + actions: *mut posix_spawn_file_actions_t, + fd: ::c_int, + newfd: ::c_int, + ) -> ::c_int; } #[link(name = "util")] @@ -2005,16 +2889,8 @@ extern "C" { ) -> ::c_int; pub fn updwtmpx(file: *const ::c_char, ut: *const utmpx) -> ::c_int; - pub fn getlastlogx( - fname: *const ::c_char, - uid: ::uid_t, - ll: *mut lastlogx, - ) -> *mut lastlogx; - pub fn updlastlogx( - fname: *const ::c_char, - uid: ::uid_t, - ll: *mut lastlogx, - ) -> ::c_int; + pub fn getlastlogx(fname: *const ::c_char, uid: ::uid_t, ll: *mut lastlogx) -> *mut lastlogx; + pub fn updlastlogx(fname: *const ::c_char, uid: ::uid_t, ll: *mut lastlogx) -> ::c_int; pub fn utmpxname(file: *const ::c_char) -> ::c_int; pub fn getutxent() -> *mut utmpx; pub fn getutxid(ut: *const utmpx) -> *mut utmpx; @@ -2030,6 +2906,147 @@ extern "C" { pub fn setutent(); pub fn endutent(); pub fn getutent() -> *mut utmp; + + pub fn efopen(p: *const ::c_char, m: *const ::c_char) -> ::FILE; + pub fn emalloc(n: ::size_t) -> *mut ::c_void; + pub fn ecalloc(n: ::size_t, c: ::size_t) -> *mut ::c_void; + pub fn erealloc(p: *mut ::c_void, n: ::size_t) -> *mut ::c_void; + pub fn ereallocarr(p: *mut ::c_void, n: ::size_t, s: ::size_t); + pub fn estrdup(s: *const ::c_char) -> *mut ::c_char; + pub fn estrndup(s: *const ::c_char, len: ::size_t) -> *mut ::c_char; + pub fn estrlcpy(dst: *mut ::c_char, src: *const ::c_char, len: ::size_t) -> ::size_t; + pub fn estrlcat(dst: *mut ::c_char, src: *const ::c_char, len: ::size_t) -> ::size_t; + pub fn estrtoi( + nptr: *const ::c_char, + base: ::c_int, + lo: ::intmax_t, + hi: ::intmax_t, + ) -> ::intmax_t; + pub fn estrtou( + nptr: *const ::c_char, + base: ::c_int, + lo: ::uintmax_t, + hi: ::uintmax_t, + ) -> ::uintmax_t; + pub fn easprintf(string: *mut *mut ::c_char, fmt: *const ::c_char, ...) -> ::c_int; + pub fn evasprintf(string: *mut *mut ::c_char, fmt: *const ::c_char, ...) -> ::c_int; + pub fn esetfunc( + cb: ::Option, + ) -> ::Option; + pub fn secure_path(path: *const ::c_char) -> ::c_int; + pub fn snprintb( + buf: *mut ::c_char, + buflen: ::size_t, + fmt: *const ::c_char, + val: u64, + ) -> ::c_int; + pub fn snprintb_m( + buf: *mut ::c_char, + buflen: ::size_t, + fmt: *const ::c_char, + val: u64, + max: ::size_t, + ) -> ::c_int; + + pub fn getbootfile() -> *const ::c_char; + pub fn getbyteorder() -> ::c_int; + pub fn getdiskrawname( + buf: *mut ::c_char, + buflen: ::size_t, + name: *const ::c_char, + ) -> *const ::c_char; + pub fn getdiskcookedname( + buf: *mut ::c_char, + buflen: ::size_t, + name: *const ::c_char, + ) -> *const ::c_char; + pub fn getfsspecname( + buf: *mut ::c_char, + buflen: ::size_t, + spec: *const ::c_char, + ) -> *const ::c_char; + + pub fn strpct( + buf: *mut ::c_char, + bufsiz: ::size_t, + numerator: ::uintmax_t, + denominator: ::uintmax_t, + precision: ::size_t, + ) -> *mut ::c_char; + pub fn strspct( + buf: *mut ::c_char, + bufsiz: ::size_t, + numerator: ::intmax_t, + denominator: ::intmax_t, + precision: ::size_t, + ) -> *mut ::c_char; + #[link_name = "__login50"] + pub fn login(ut: *const utmp); + #[link_name = "__loginx50"] + pub fn loginx(ut: *const utmpx); + pub fn logout(line: *const ::c_char); + pub fn logoutx(line: *const ::c_char, status: ::c_int, tpe: ::c_int); + pub fn logwtmp(line: *const ::c_char, name: *const ::c_char, host: *const ::c_char); + pub fn logwtmpx( + line: *const ::c_char, + name: *const ::c_char, + host: *const ::c_char, + status: ::c_int, + tpe: ::c_int, + ); + + pub fn getxattr( + path: *const ::c_char, + name: *const ::c_char, + value: *mut ::c_void, + size: ::size_t, + ) -> ::ssize_t; + pub fn lgetxattr( + path: *const ::c_char, + name: *const ::c_char, + value: *mut ::c_void, + size: ::size_t, + ) -> ::ssize_t; + pub fn fgetxattr( + filedes: ::c_int, + name: *const ::c_char, + value: *mut ::c_void, + size: ::size_t, + ) -> ::ssize_t; + pub fn setxattr( + path: *const ::c_char, + name: *const ::c_char, + value: *const ::c_void, + size: ::size_t, + ) -> ::c_int; + pub fn lsetxattr( + path: *const ::c_char, + name: *const ::c_char, + value: *const ::c_void, + size: ::size_t, + ) -> ::c_int; + pub fn fsetxattr( + filedes: ::c_int, + name: *const ::c_char, + value: *const ::c_void, + size: ::size_t, + flags: ::c_int, + ) -> ::c_int; + pub fn listxattr(path: *const ::c_char, list: *mut ::c_char, size: ::size_t) -> ::ssize_t; + pub fn llistxattr(path: *const ::c_char, list: *mut ::c_char, size: ::size_t) -> ::ssize_t; + pub fn flistxattr(filedes: ::c_int, list: *mut ::c_char, size: ::size_t) -> ::ssize_t; + pub fn removexattr(path: *const ::c_char, name: *const ::c_char) -> ::c_int; + pub fn lremovexattr(path: *const ::c_char, name: *const ::c_char) -> ::c_int; + pub fn fremovexattr(fd: ::c_int, path: *const ::c_char, name: *const ::c_char) -> ::c_int; + + pub fn string_to_flags( + string_p: *mut *mut ::c_char, + setp: *mut ::c_ulong, + clrp: *mut ::c_ulong, + ) -> ::c_int; + pub fn flags_to_string(flags: ::c_ulong, def: *const ::c_char) -> ::c_int; + + pub fn kinfo_getvmmap(pid: ::pid_t, cntp: *mut ::size_t) -> *mut kinfo_vmentry; } cfg_if! { diff --git a/crux-mir/lib/libc/src/unix/bsd/netbsdlike/netbsd/x86_64.rs b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/netbsd/x86_64.rs index 0860d4f6c..2f6e44545 100644 --- a/crux-mir/lib/libc/src/unix/bsd/netbsdlike/netbsd/x86_64.rs +++ b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/netbsd/x86_64.rs @@ -3,8 +3,25 @@ use PT_FIRSTMACH; pub type c_long = i64; pub type c_ulong = u64; pub type c_char = i8; +pub type c___greg_t = u64; pub type __cpu_simple_lock_nv_t = ::c_uchar; +s! { + pub struct mcontext_t { + pub __gregs: [c___greg_t; 26], + pub _mc_tlsbase: c___greg_t, + pub __fpregs: [[::c_char;32]; 16], + } + + pub struct ucontext_t { + pub uc_flags: ::c_uint, + pub uc_link: *mut ::ucontext_t, + pub uc_sigmask: ::sigset_t, + pub uc_stack: ::stack_t, + pub uc_mcontext: ::mcontext_t, + } +} + // should be pub(crate), but that requires Rust 1.18.0 cfg_if! { if #[cfg(libc_const_size_of)] { diff --git a/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/aarch64.rs b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/aarch64.rs index 99350ec8d..2bc82e486 100644 --- a/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/aarch64.rs +++ b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/aarch64.rs @@ -1,6 +1,20 @@ pub type c_long = i64; pub type c_ulong = u64; pub type c_char = u8; +pub type ucontext_t = sigcontext; + +s! { + pub struct sigcontext { + __sc_unused: ::c_int, + pub sc_mask: ::c_int, + pub sc_sp: ::c_ulong, + pub sc_lr: ::c_ulong, + pub sc_elr: ::c_ulong, + pub sc_spsr: ::c_ulong, + pub sc_x: [::c_ulong; 30], + pub sc_cookie: ::c_long, + } +} // should be pub(crate), but that requires Rust 1.18.0 cfg_if! { diff --git a/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/arm.rs b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/arm.rs new file mode 100644 index 000000000..f1ab365d1 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/arm.rs @@ -0,0 +1,16 @@ +pub type c_long = i32; +pub type c_ulong = u32; +pub type c_char = u8; + +// should be pub(crate), but that requires Rust 1.18.0 +cfg_if! { + if #[cfg(libc_const_size_of)] { + #[doc(hidden)] + pub const _ALIGNBYTES: usize = ::mem::size_of::<::c_double>() - 1; + } else { + #[doc(hidden)] + pub const _ALIGNBYTES: usize = 8 - 1; + } +} + +pub const _MAX_PAGE_SHIFT: u32 = 12; diff --git a/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/mips64.rs b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/mips64.rs new file mode 100644 index 000000000..15803ced0 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/mips64.rs @@ -0,0 +1,8 @@ +pub type c_long = i64; +pub type c_ulong = u64; +pub type c_char = i8; + +#[doc(hidden)] +pub const _ALIGNBYTES: usize = 7; + +pub const _MAX_PAGE_SHIFT: u32 = 14; diff --git a/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/mod.rs b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/mod.rs index 8e1bfcb57..3c966990a 100644 --- a/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/mod.rs +++ b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/mod.rs @@ -14,9 +14,51 @@ pub type pthread_cond_t = *mut ::c_void; pub type pthread_condattr_t = *mut ::c_void; pub type pthread_rwlock_t = *mut ::c_void; pub type pthread_rwlockattr_t = *mut ::c_void; +pub type pthread_spinlock_t = ::uintptr_t; pub type caddr_t = *mut ::c_char; +// elf.h + +pub type Elf32_Addr = u32; +pub type Elf32_Half = u16; +pub type Elf32_Lword = u64; +pub type Elf32_Off = u32; +pub type Elf32_Sword = i32; +pub type Elf32_Word = u32; + +pub type Elf64_Addr = u64; +pub type Elf64_Half = u16; +pub type Elf64_Lword = u64; +pub type Elf64_Off = u64; +pub type Elf64_Sword = i32; +pub type Elf64_Sxword = i64; +pub type Elf64_Word = u32; +pub type Elf64_Xword = u64; + +// search.h + +pub type ENTRY = entry; +pub type ACTION = ::c_uint; + +cfg_if! { + if #[cfg(target_pointer_width = "64")] { + type Elf_Addr = Elf64_Addr; + type Elf_Half = Elf64_Half; + type Elf_Phdr = Elf64_Phdr; + } else if #[cfg(target_pointer_width = "32")] { + type Elf_Addr = Elf32_Addr; + type Elf_Half = Elf32_Half; + type Elf_Phdr = Elf32_Phdr; + } +} + s! { + pub struct ip_mreqn { + pub imr_multiaddr: in_addr, + pub imr_address: in_addr, + pub imr_ifindex: ::c_int, + } + pub struct glob_t { pub gl_pathc: ::size_t, pub gl_matchc: ::size_t, @@ -178,6 +220,12 @@ s! { pub sin_zero: [i8; 8], } + pub struct splice { + pub sp_fd: ::c_int, + pub sp_max: ::off_t, + pub sp_idle: ::timeval, + } + pub struct kevent { pub ident: ::uintptr_t, pub filter: ::c_short, @@ -321,9 +369,175 @@ s! { __shm_ctimensec: c_long, pub shm_internal: *mut ::c_void, } + + // elf.h + pub struct Elf32_Phdr { + pub p_type: Elf32_Word, + pub p_offset: Elf32_Off, + pub p_vaddr: Elf32_Addr, + pub p_paddr: Elf32_Addr, + pub p_filesz: Elf32_Word, + pub p_memsz: Elf32_Word, + pub p_flags: Elf32_Word, + pub p_align: Elf32_Word, + } + + pub struct Elf64_Phdr { + pub p_type: Elf64_Word, + pub p_flags: Elf64_Word, + pub p_offset: Elf64_Off, + pub p_vaddr: Elf64_Addr, + pub p_paddr: Elf64_Addr, + pub p_filesz: Elf64_Xword, + pub p_memsz: Elf64_Xword, + pub p_align: Elf64_Xword, + } + + // link.h + + pub struct dl_phdr_info { + pub dlpi_addr: Elf_Addr, + pub dlpi_name: *const ::c_char, + pub dlpi_phdr: *const Elf_Phdr, + pub dlpi_phnum: Elf_Half, + } + + // sys/sysctl.h + pub struct kinfo_proc { + pub p_forw: u64, + pub p_back: u64, + pub p_paddr: u64, + pub p_addr: u64, + pub p_fd: u64, + pub p_stats: u64, + pub p_limit: u64, + pub p_vmspace: u64, + pub p_sigacts: u64, + pub p_sess: u64, + pub p_tsess: u64, + pub p_ru: u64, + pub p_eflag: i32, + pub p_exitsig: i32, + pub p_flag: i32, + pub p_pid: i32, + pub p_ppid: i32, + pub p_sid: i32, + pub p__pgid: i32, + pub p_tpgid: i32, + pub p_uid: u32, + pub p_ruid: u32, + pub p_gid: u32, + pub p_rgid: u32, + pub p_groups: [u32; KI_NGROUPS as usize], + pub p_ngroups: i16, + pub p_jobc: i16, + pub p_tdev: u32, + pub p_estcpu: u32, + pub p_rtime_sec: u32, + pub p_rtime_usec: u32, + pub p_cpticks: i32, + pub p_pctcpu: u32, + pub p_swtime: u32, + pub p_slptime: u32, + pub p_schedflags: i32, + pub p_uticks: u64, + pub p_sticks: u64, + pub p_iticks: u64, + pub p_tracep: u64, + pub p_traceflag: i32, + pub p_holdcnt: i32, + pub p_siglist: i32, + pub p_sigmask: u32, + pub p_sigignore: u32, + pub p_sigcatch: u32, + pub p_stat: i8, + pub p_priority: u8, + pub p_usrpri: u8, + pub p_nice: u8, + pub p_xstat: u16, + pub p_spare: u16, + pub p_comm: [::c_char; KI_MAXCOMLEN as usize], + pub p_wmesg: [::c_char; KI_WMESGLEN as usize], + pub p_wchan: u64, + pub p_login: [::c_char; KI_MAXLOGNAME as usize], + pub p_vm_rssize: i32, + pub p_vm_tsize: i32, + pub p_vm_dsize: i32, + pub p_vm_ssize: i32, + pub p_uvalid: i64, + pub p_ustart_sec: u64, + pub p_ustart_usec: u32, + pub p_uutime_sec: u32, + pub p_uutime_usec: u32, + pub p_ustime_sec: u32, + pub p_ustime_usec: u32, + pub p_uru_maxrss: u64, + pub p_uru_ixrss: u64, + pub p_uru_idrss: u64, + pub p_uru_isrss: u64, + pub p_uru_minflt: u64, + pub p_uru_majflt: u64, + pub p_uru_nswap: u64, + pub p_uru_inblock: u64, + pub p_uru_oublock: u64, + pub p_uru_msgsnd: u64, + pub p_uru_msgrcv: u64, + pub p_uru_nsignals: u64, + pub p_uru_nvcsw: u64, + pub p_uru_nivcsw: u64, + pub p_uctime_sec: u32, + pub p_uctime_usec: u32, + pub p_psflags: u32, + pub p_acflag: u32, + pub p_svuid: u32, + pub p_svgid: u32, + pub p_emul: [::c_char; KI_EMULNAMELEN as usize], + pub p_rlim_rss_cur: u64, + pub p_cpuid: u64, + pub p_vm_map_size: u64, + pub p_tid: i32, + pub p_rtableid: u32, + pub p_pledge: u64, + } + + pub struct kinfo_vmentry { + pub kve_start: ::c_ulong, + pub kve_end: ::c_ulong, + pub kve_guard: ::c_ulong, + pub kve_fspace: ::c_ulong, + pub kve_fspace_augment: ::c_ulong, + pub kve_offset: u64, + pub kve_wired_count: ::c_int, + pub kve_etype: ::c_int, + pub kve_protection: ::c_int, + pub kve_max_protection: ::c_int, + pub kve_advice: ::c_int, + pub kve_inheritance: ::c_int, + pub kve_flags: u8, + } + + pub struct ptrace_state { + pub pe_report_event: ::c_int, + pub pe_other_pid: ::pid_t, + pub pe_tid: ::pid_t, + } + + pub struct ptrace_thread_state { + pub pts_tid: ::pid_t, + } + + // search.h + pub struct entry { + pub key: *mut ::c_char, + pub data: *mut ::c_void, + } } impl siginfo_t { + pub unsafe fn si_addr(&self) -> *mut ::c_char { + self.si_addr + } + pub unsafe fn si_value(&self) -> ::sigval { #[repr(C)] struct siginfo_timer { @@ -874,6 +1088,10 @@ pub const NET_RT_STATS: ::c_int = 4; pub const NET_RT_TABLE: ::c_int = 5; pub const NET_RT_IFNAMES: ::c_int = 6; #[doc(hidden)] +#[deprecated( + since = "0.2.95", + note = "Possibly increasing over the releases and might not be so used in the field" +)] pub const NET_RT_MAXID: ::c_int = 7; pub const IPV6_JOIN_GROUP: ::c_int = 12; @@ -1044,6 +1262,10 @@ pub const _SC_NPROCESSORS_ONLN: ::c_int = 503; pub const FD_SETSIZE: usize = 1024; +pub const SCHED_FIFO: ::c_int = 1; +pub const SCHED_OTHER: ::c_int = 2; +pub const SCHED_RR: ::c_int = 3; + pub const ST_NOSUID: ::c_ulong = 2; pub const PTHREAD_MUTEX_INITIALIZER: pthread_mutex_t = 0 as *mut _; @@ -1075,7 +1297,9 @@ pub const EV_DISPATCH: u16 = 0x80; pub const EV_FLAG1: u16 = 0x2000; pub const EV_ERROR: u16 = 0x4000; pub const EV_EOF: u16 = 0x8000; -pub const EV_SYSFLAGS: u16 = 0xf000; + +#[deprecated(since = "0.2.113", note = "Not stable across OS versions")] +pub const EV_SYSFLAGS: u16 = 0xf800; pub const NOTE_LOWAT: u32 = 0x00000001; pub const NOTE_EOF: u32 = 0x00000002; @@ -1098,6 +1322,20 @@ pub const NOTE_CHILD: u32 = 0x00000004; pub const TMP_MAX: ::c_uint = 0x7fffffff; +pub const AI_PASSIVE: ::c_int = 1; +pub const AI_CANONNAME: ::c_int = 2; +pub const AI_NUMERICHOST: ::c_int = 4; +pub const AI_EXT: ::c_int = 8; +pub const AI_NUMERICSERV: ::c_int = 16; +pub const AI_FQDN: ::c_int = 32; +pub const AI_ADDRCONFIG: ::c_int = 64; + +pub const NI_NUMERICHOST: ::c_int = 1; +pub const NI_NUMERICSERV: ::c_int = 2; +pub const NI_NOFQDN: ::c_int = 4; +pub const NI_NAMEREQD: ::c_int = 8; +pub const NI_DGRAM: ::c_int = 16; + pub const NI_MAXHOST: ::size_t = 256; pub const RTLD_LOCAL: ::c_int = 0; @@ -1150,12 +1388,14 @@ pub const KERN_NTHREADS: ::c_int = 26; pub const KERN_OSVERSION: ::c_int = 27; pub const KERN_SOMAXCONN: ::c_int = 28; pub const KERN_SOMINCONN: ::c_int = 29; +#[deprecated(since = "0.2.71", note = "Removed in OpenBSD 6.0")] pub const KERN_USERMOUNT: ::c_int = 30; pub const KERN_NOSUIDCOREDUMP: ::c_int = 32; pub const KERN_FSYNC: ::c_int = 33; pub const KERN_SYSVMSG: ::c_int = 34; pub const KERN_SYSVSEM: ::c_int = 35; pub const KERN_SYSVSHM: ::c_int = 36; +#[deprecated(since = "0.2.71", note = "Removed in OpenBSD 6.0")] pub const KERN_ARND: ::c_int = 37; pub const KERN_MSGBUFSIZE: ::c_int = 38; pub const KERN_MALLOCSTATS: ::c_int = 39; @@ -1202,6 +1442,10 @@ pub const KERN_AUDIO: ::c_int = 84; pub const KERN_CPUSTATS: ::c_int = 85; pub const KERN_PFSTATUS: ::c_int = 86; pub const KERN_TIMEOUT_STATS: ::c_int = 87; +#[deprecated( + since = "0.2.95", + note = "Possibly increasing over the releases and might not be so used in the field" +)] pub const KERN_MAXID: ::c_int = 88; pub const KERN_PROC_ALL: ::c_int = 0; @@ -1229,6 +1473,35 @@ pub const KI_WMESGLEN: ::c_int = 8; pub const KI_MAXLOGNAME: ::c_int = 32; pub const KI_EMULNAMELEN: ::c_int = 8; +pub const KVE_ET_OBJ: ::c_int = 0x00000001; +pub const KVE_ET_SUBMAP: ::c_int = 0x00000002; +pub const KVE_ET_COPYONWRITE: ::c_int = 0x00000004; +pub const KVE_ET_NEEDSCOPY: ::c_int = 0x00000008; +pub const KVE_ET_HOLE: ::c_int = 0x00000010; +pub const KVE_ET_NOFAULT: ::c_int = 0x00000020; +pub const KVE_ET_STACK: ::c_int = 0x00000040; +pub const KVE_ET_WC: ::c_int = 0x000000080; +pub const KVE_ET_CONCEAL: ::c_int = 0x000000100; +pub const KVE_ET_SYSCALL: ::c_int = 0x000000200; +pub const KVE_ET_FREEMAPPED: ::c_int = 0x000000800; + +pub const KVE_PROT_NONE: ::c_int = 0x00000000; +pub const KVE_PROT_READ: ::c_int = 0x00000001; +pub const KVE_PROT_WRITE: ::c_int = 0x00000002; +pub const KVE_PROT_EXEC: ::c_int = 0x00000004; + +pub const KVE_ADV_NORMAL: ::c_int = 0x00000000; +pub const KVE_ADV_RANDOM: ::c_int = 0x00000001; +pub const KVE_ADV_SEQUENTIAL: ::c_int = 0x00000002; + +pub const KVE_INH_SHARE: ::c_int = 0x00000000; +pub const KVE_INH_COPY: ::c_int = 0x00000010; +pub const KVE_INH_NONE: ::c_int = 0x00000020; +pub const KVE_INH_ZERO: ::c_int = 0x00000030; + +pub const KVE_F_STATIC: ::c_int = 0x1; +pub const KVE_F_KMEM: ::c_int = 0x2; + pub const CHWFLOW: ::tcflag_t = ::MDMBUF | ::CRTSCTS; pub const OLCUC: ::tcflag_t = 0x20; pub const ONOCR: ::tcflag_t = 0x40; @@ -1296,6 +1569,7 @@ pub const NTFS_MFLAG_ALLNAMES: ::c_int = 0x2; pub const TMPFS_ARGS_VERSION: ::c_int = 1; pub const MAP_STACK: ::c_int = 0x4000; +pub const MAP_CONCEAL: ::c_int = 0x8000; // https://github.com/openbsd/src/blob/master/sys/net/if.h#L187 pub const IFF_UP: ::c_int = 0x1; // interface is up @@ -1319,6 +1593,11 @@ pub const PTHREAD_STACK_MIN: ::size_t = 1_usize << _MAX_PAGE_SHIFT; pub const MINSIGSTKSZ: ::size_t = 3_usize << _MAX_PAGE_SHIFT; pub const SIGSTKSZ: ::size_t = MINSIGSTKSZ + (1_usize << _MAX_PAGE_SHIFT) * 4; +pub const PT_SET_EVENT_MASK: ::c_int = 12; +pub const PT_GET_EVENT_MASK: ::c_int = 13; +pub const PT_GET_PROCESS_STATE: ::c_int = 14; +pub const PT_GET_THREAD_FIRST: ::c_int = 15; +pub const PT_GET_THREAD_NEXT: ::c_int = 16; pub const PT_FIRSTMACH: ::c_int = 32; pub const SOCK_CLOEXEC: ::c_int = 0x8000; @@ -1333,16 +1612,73 @@ pub const PTRACE_FORK: ::c_int = 0x0002; pub const WCONTINUED: ::c_int = 8; -fn _ALIGN(p: usize) -> usize { - (p + _ALIGNBYTES) & !_ALIGNBYTES +// search.h +pub const FIND: ::ACTION = 0; +pub const ENTER: ::ACTION = 1; + +// futex.h +pub const FUTEX_WAIT: ::c_int = 1; +pub const FUTEX_WAKE: ::c_int = 2; +pub const FUTEX_REQUEUE: ::c_int = 3; +pub const FUTEX_PRIVATE_FLAG: ::c_int = 128; + +// sysctl.h, kinfo_proc p_eflag constants +pub const EPROC_CTTY: i32 = 0x01; // controlling tty vnode active +pub const EPROC_SLEADER: i32 = 0x02; // session leader +pub const EPROC_UNVEIL: i32 = 0x04; // has unveil settings +pub const EPROC_LKUNVEIL: i32 = 0x08; // unveil is locked + +// Flags for chflags(2) +pub const UF_SETTABLE: ::c_uint = 0x0000ffff; +pub const UF_NODUMP: ::c_uint = 0x00000001; +pub const UF_IMMUTABLE: ::c_uint = 0x00000002; +pub const UF_APPEND: ::c_uint = 0x00000004; +pub const UF_OPAQUE: ::c_uint = 0x00000008; +pub const SF_SETTABLE: ::c_uint = 0xffff0000; +pub const SF_ARCHIVED: ::c_uint = 0x00010000; +pub const SF_IMMUTABLE: ::c_uint = 0x00020000; +pub const SF_APPEND: ::c_uint = 0x00040000; + +// sys/mount.h +pub const MNT_NOPERM: ::c_int = 0x00000020; +pub const MNT_WXALLOWED: ::c_int = 0x00000800; +pub const MNT_EXRDONLY: ::c_int = 0x00000080; +pub const MNT_DEFEXPORTED: ::c_int = 0x00000200; +pub const MNT_EXPORTANON: ::c_int = 0x00000400; +pub const MNT_ROOTFS: ::c_int = 0x00004000; +pub const MNT_NOATIME: ::c_int = 0x00008000; +pub const MNT_DELEXPORT: ::c_int = 0x00020000; +pub const MNT_STALLED: ::c_int = 0x00100000; +pub const MNT_SWAPPABLE: ::c_int = 0x00200000; +pub const MNT_WANTRDWR: ::c_int = 0x02000000; +pub const MNT_SOFTDEP: ::c_int = 0x04000000; +pub const MNT_DOOMED: ::c_int = 0x08000000; + +// For use with vfs_fsync and getfsstat +pub const MNT_WAIT: ::c_int = 1; +pub const MNT_NOWAIT: ::c_int = 2; +pub const MNT_LAZY: ::c_int = 3; + +pub const LC_COLLATE_MASK: ::c_int = 1 << ::LC_COLLATE; +pub const LC_CTYPE_MASK: ::c_int = 1 << ::LC_CTYPE; +pub const LC_MONETARY_MASK: ::c_int = 1 << ::LC_MONETARY; +pub const LC_NUMERIC_MASK: ::c_int = 1 << ::LC_NUMERIC; +pub const LC_TIME_MASK: ::c_int = 1 << ::LC_TIME; +pub const LC_MESSAGES_MASK: ::c_int = 1 << ::LC_MESSAGES; + +const _LC_LAST: ::c_int = 7; +pub const LC_ALL_MASK: ::c_int = (1 << _LC_LAST) - 2; + +pub const LC_GLOBAL_LOCALE: ::locale_t = -1isize as ::locale_t; + +const_fn! { + {const} fn _ALIGN(p: usize) -> usize { + (p + _ALIGNBYTES) & !_ALIGNBYTES + } } f! { - pub fn WIFCONTINUED(status: ::c_int) -> bool { - status & 0o177777 == 0o177777 - } - - pub fn CMSG_DATA(cmsg: *const ::cmsghdr) -> *mut ::c_uchar { + pub fn CMSG_DATA(cmsg: *const ::cmsghdr) -> *mut ::c_uchar { (cmsg as *mut ::c_uchar) .offset(_ALIGN(::mem::size_of::<::cmsghdr>()) as isize) } @@ -1369,45 +1705,50 @@ f! { } } - pub fn CMSG_SPACE(length: ::c_uint) -> ::c_uint { + pub {const} fn CMSG_SPACE(length: ::c_uint) -> ::c_uint { (_ALIGN(::mem::size_of::<::cmsghdr>()) + _ALIGN(length as usize)) as ::c_uint } +} - pub fn WSTOPSIG(status: ::c_int) -> ::c_int { +safe_f! { + pub {const} fn WSTOPSIG(status: ::c_int) -> ::c_int { status >> 8 } - pub fn WIFSIGNALED(status: ::c_int) -> bool { + pub {const} fn WIFSIGNALED(status: ::c_int) -> bool { (status & 0o177) != 0o177 && (status & 0o177) != 0 } - pub fn WIFSTOPPED(status: ::c_int) -> bool { - (status & 0o177) == 0o177 + pub {const} fn WIFSTOPPED(status: ::c_int) -> bool { + (status & 0xff) == 0o177 + } + + pub {const} fn WIFCONTINUED(status: ::c_int) -> bool { + (status & 0o177777) == 0o177777 + } + + pub {const} fn makedev(major: ::c_uint, minor: ::c_uint) -> ::dev_t { + let major = major as ::dev_t; + let minor = minor as ::dev_t; + let mut dev = 0; + dev |= (major & 0xff) << 8; + dev |= minor & 0xff; + dev |= (minor & 0xffff00) << 8; + dev } } extern "C" { pub fn gettimeofday(tp: *mut ::timeval, tz: *mut ::timezone) -> ::c_int; - pub fn settimeofday( - tp: *const ::timeval, - tz: *const ::timezone, - ) -> ::c_int; - pub fn accept4( - s: ::c_int, - addr: *mut ::sockaddr, - addrlen: *mut ::socklen_t, - flags: ::c_int, - ) -> ::c_int; + pub fn settimeofday(tp: *const ::timeval, tz: *const ::timezone) -> ::c_int; pub fn execvpe( file: *const ::c_char, argv: *const *const ::c_char, envp: *const *const ::c_char, ) -> ::c_int; - pub fn pledge( - promises: *const ::c_char, - execpromises: *const ::c_char, - ) -> ::c_int; + pub fn pledge(promises: *const ::c_char, execpromises: *const ::c_char) -> ::c_int; + pub fn unveil(path: *const ::c_char, permissions: *const ::c_char) -> ::c_int; pub fn strtonum( nptr: *const ::c_char, minval: ::c_longlong, @@ -1433,6 +1774,8 @@ extern "C" { servlen: ::size_t, flags: ::c_int, ) -> ::c_int; + pub fn getresgid(rgid: *mut ::gid_t, egid: *mut ::gid_t, sgid: *mut ::gid_t) -> ::c_int; + pub fn getresuid(ruid: *mut ::uid_t, euid: *mut ::uid_t, suid: *mut ::uid_t) -> ::c_int; pub fn kevent( kq: ::c_int, changelist: *const ::kevent, @@ -1441,11 +1784,8 @@ extern "C" { nevents: ::c_int, timeout: *const ::timespec, ) -> ::c_int; - pub fn mprotect( - addr: *mut ::c_void, - len: ::size_t, - prot: ::c_int, - ) -> ::c_int; + pub fn mprotect(addr: *mut ::c_void, len: ::size_t, prot: ::c_int) -> ::c_int; + pub fn getthrid() -> ::pid_t; pub fn pthread_attr_getguardsize( attr: *const ::pthread_attr_t, guardsize: *mut ::size_t, @@ -1456,11 +1796,24 @@ extern "C" { stacksize: *mut ::size_t, ) -> ::c_int; pub fn pthread_main_np() -> ::c_int; + pub fn pthread_get_name_np(tid: ::pthread_t, name: *mut ::c_char, len: ::size_t); pub fn pthread_set_name_np(tid: ::pthread_t, name: *const ::c_char); - pub fn pthread_stackseg_np( - thread: ::pthread_t, - sinfo: *mut ::stack_t, + pub fn pthread_stackseg_np(thread: ::pthread_t, sinfo: *mut ::stack_t) -> ::c_int; + + pub fn openpty( + amaster: *mut ::c_int, + aslave: *mut ::c_int, + name: *mut ::c_char, + termp: *const ::termios, + winp: *const ::winsize, ) -> ::c_int; + pub fn forkpty( + amaster: *mut ::c_int, + name: *mut ::c_char, + termp: *const ::termios, + winp: *const ::winsize, + ) -> ::pid_t; + pub fn sysctl( name: *const ::c_int, namelen: ::c_uint, @@ -1472,18 +1825,85 @@ extern "C" { pub fn getentropy(buf: *mut ::c_void, buflen: ::size_t) -> ::c_int; pub fn setresgid(rgid: ::gid_t, egid: ::gid_t, sgid: ::gid_t) -> ::c_int; pub fn setresuid(ruid: ::uid_t, euid: ::uid_t, suid: ::uid_t) -> ::c_int; - pub fn ptrace( - request: ::c_int, - pid: ::pid_t, - addr: caddr_t, - data: ::c_int, - ) -> ::c_int; + pub fn ptrace(request: ::c_int, pid: ::pid_t, addr: caddr_t, data: ::c_int) -> ::c_int; + pub fn utrace(label: *const ::c_char, addr: *const ::c_void, len: ::size_t) -> ::c_int; pub fn memmem( haystack: *const ::c_void, haystacklen: ::size_t, needle: *const ::c_void, needlelen: ::size_t, ) -> *mut ::c_void; + // #include + pub fn dl_iterate_phdr( + callback: ::Option< + unsafe extern "C" fn( + info: *mut dl_phdr_info, + size: usize, + data: *mut ::c_void, + ) -> ::c_int, + >, + data: *mut ::c_void, + ) -> ::c_int; + pub fn uselocale(loc: ::locale_t) -> ::locale_t; + pub fn freelocale(loc: ::locale_t); + pub fn newlocale(mask: ::c_int, locale: *const ::c_char, base: ::locale_t) -> ::locale_t; + pub fn duplocale(base: ::locale_t) -> ::locale_t; + + // Added in `OpenBSD` 5.5 + pub fn explicit_bzero(s: *mut ::c_void, len: ::size_t); + + pub fn setproctitle(fmt: *const ::c_char, ...); + + pub fn freezero(ptr: *mut ::c_void, size: ::size_t); + pub fn malloc_conceal(size: ::size_t) -> *mut ::c_void; + pub fn calloc_conceal(nmemb: ::size_t, size: ::size_t) -> *mut ::c_void; + + pub fn srand48_deterministic(seed: ::c_long); + pub fn seed48_deterministic(xseed: *mut ::c_ushort) -> *mut ::c_ushort; + pub fn lcong48_deterministic(p: *mut ::c_ushort); + + pub fn lsearch( + key: *const ::c_void, + base: *mut ::c_void, + nelp: *mut ::size_t, + width: ::size_t, + compar: ::Option ::c_int>, + ) -> *mut ::c_void; + pub fn lfind( + key: *const ::c_void, + base: *const ::c_void, + nelp: *mut ::size_t, + width: ::size_t, + compar: ::Option ::c_int>, + ) -> *mut ::c_void; + pub fn hcreate(nelt: ::size_t) -> ::c_int; + pub fn hdestroy(); + pub fn hsearch(entry: ::ENTRY, action: ::ACTION) -> *mut ::ENTRY; + + // futex.h + pub fn futex( + uaddr: *mut u32, + op: ::c_int, + val: ::c_int, + timeout: *const ::timespec, + uaddr2: *mut u32, + ) -> ::c_int; +} + +#[link(name = "execinfo")] +extern "C" { + pub fn backtrace(addrlist: *mut *mut ::c_void, len: ::size_t) -> ::size_t; + pub fn backtrace_symbols(addrlist: *const *mut ::c_void, len: ::size_t) -> *mut *mut ::c_char; + pub fn backtrace_symbols_fd( + addrlist: *const *mut ::c_void, + len: ::size_t, + fd: ::c_int, + ) -> ::c_int; + pub fn backtrace_symbols_fmt( + addrlist: *const *mut ::c_void, + len: ::size_t, + fmt: *const ::c_char, + ) -> *mut *mut ::c_char; } cfg_if! { @@ -1492,23 +1912,40 @@ cfg_if! { // these functions use statfs which uses the union mount_info: pub fn statfs(path: *const ::c_char, buf: *mut statfs) -> ::c_int; pub fn fstatfs(fd: ::c_int, buf: *mut statfs) -> ::c_int; + pub fn getmntinfo(mntbufp: *mut *mut ::statfs, flags: ::c_int) -> ::c_int; + pub fn getfsstat(buf: *mut statfs, bufsize: ::size_t, flags: ::c_int) -> ::c_int; } } } cfg_if! { - if #[cfg(target_arch = "x86")] { - mod x86; - pub use self::x86::*; - } else if #[cfg(target_arch = "x86_64")] { - mod x86_64; - pub use self::x86_64::*; - } else if #[cfg(target_arch = "aarch64")] { + if #[cfg(target_arch = "aarch64")] { mod aarch64; pub use self::aarch64::*; + } else if #[cfg(target_arch = "arm")] { + mod arm; + pub use self::arm::*; + } else if #[cfg(target_arch = "mips64")] { + mod mips64; + pub use self::mips64::*; + } else if #[cfg(target_arch = "powerpc")] { + mod powerpc; + pub use self::powerpc::*; + } else if #[cfg(target_arch = "powerpc64")] { + mod powerpc64; + pub use self::powerpc64::*; + } else if #[cfg(target_arch = "riscv64")] { + mod riscv64; + pub use self::riscv64::*; } else if #[cfg(target_arch = "sparc64")] { mod sparc64; pub use self::sparc64::*; + } else if #[cfg(target_arch = "x86")] { + mod x86; + pub use self::x86::*; + } else if #[cfg(target_arch = "x86_64")] { + mod x86_64; + pub use self::x86_64::*; } else { // Unknown target_arch } diff --git a/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/powerpc.rs b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/powerpc.rs new file mode 100644 index 000000000..f1ab365d1 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/powerpc.rs @@ -0,0 +1,16 @@ +pub type c_long = i32; +pub type c_ulong = u32; +pub type c_char = u8; + +// should be pub(crate), but that requires Rust 1.18.0 +cfg_if! { + if #[cfg(libc_const_size_of)] { + #[doc(hidden)] + pub const _ALIGNBYTES: usize = ::mem::size_of::<::c_double>() - 1; + } else { + #[doc(hidden)] + pub const _ALIGNBYTES: usize = 8 - 1; + } +} + +pub const _MAX_PAGE_SHIFT: u32 = 12; diff --git a/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/powerpc64.rs b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/powerpc64.rs new file mode 100644 index 000000000..99350ec8d --- /dev/null +++ b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/powerpc64.rs @@ -0,0 +1,16 @@ +pub type c_long = i64; +pub type c_ulong = u64; +pub type c_char = u8; + +// should be pub(crate), but that requires Rust 1.18.0 +cfg_if! { + if #[cfg(libc_const_size_of)] { + #[doc(hidden)] + pub const _ALIGNBYTES: usize = ::mem::size_of::<::c_long>() - 1; + } else { + #[doc(hidden)] + pub const _ALIGNBYTES: usize = 8 - 1; + } +} + +pub const _MAX_PAGE_SHIFT: u32 = 12; diff --git a/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/riscv64.rs b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/riscv64.rs new file mode 100644 index 000000000..99350ec8d --- /dev/null +++ b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/riscv64.rs @@ -0,0 +1,16 @@ +pub type c_long = i64; +pub type c_ulong = u64; +pub type c_char = u8; + +// should be pub(crate), but that requires Rust 1.18.0 +cfg_if! { + if #[cfg(libc_const_size_of)] { + #[doc(hidden)] + pub const _ALIGNBYTES: usize = ::mem::size_of::<::c_long>() - 1; + } else { + #[doc(hidden)] + pub const _ALIGNBYTES: usize = 8 - 1; + } +} + +pub const _MAX_PAGE_SHIFT: u32 = 12; diff --git a/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/x86_64.rs b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/x86_64.rs index 263b6e13a..60dab0044 100644 --- a/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/x86_64.rs +++ b/crux-mir/lib/libc/src/unix/bsd/netbsdlike/openbsd/x86_64.rs @@ -3,6 +3,112 @@ use PT_FIRSTMACH; pub type c_long = i64; pub type c_ulong = u64; pub type c_char = i8; +pub type ucontext_t = sigcontext; + +s! { + pub struct sigcontext { + pub sc_rdi: ::c_long, + pub sc_rsi: ::c_long, + pub sc_rdx: ::c_long, + pub sc_rcx: ::c_long, + pub sc_r8: ::c_long, + pub sc_r9: ::c_long, + pub sc_r10: ::c_long, + pub sc_r11: ::c_long, + pub sc_r12: ::c_long, + pub sc_r13: ::c_long, + pub sc_r14: ::c_long, + pub sc_r15: ::c_long, + pub sc_rbp: ::c_long, + pub sc_rbx: ::c_long, + pub sc_rax: ::c_long, + pub sc_gs: ::c_long, + pub sc_fs: ::c_long, + pub sc_es: ::c_long, + pub sc_ds: ::c_long, + pub sc_trapno: ::c_long, + pub sc_err: ::c_long, + pub sc_rip: ::c_long, + pub sc_cs: ::c_long, + pub sc_rflags: ::c_long, + pub sc_rsp: ::c_long, + pub sc_ss: ::c_long, + pub sc_fpstate: *mut fxsave64, + __sc_unused: ::c_int, + pub sc_mask: ::c_int, + pub sc_cookie: ::c_long, + } +} + +s_no_extra_traits! { + #[repr(packed)] + pub struct fxsave64 { + pub fx_fcw: u16, + pub fx_fsw: u16, + pub fx_ftw: u8, + __fx_unused1: u8, + pub fx_fop: u16, + pub fx_rip: u64, + pub fx_rdp: u64, + pub fx_mxcsr: u32, + pub fx_mxcsr_mask: u32, + pub fx_st: [[u64; 2]; 8], + pub fx_xmm: [[u64; 2]; 16], + __fx_unused3: [u8; 96], + } +} + +cfg_if! { + if #[cfg(feature = "extra_traits")] { + // `fxsave64` is packed, so field access is unaligned. + // use {x} to create temporary storage, copy field to it, and do aligned access. + impl PartialEq for fxsave64 { + fn eq(&self, other: &fxsave64) -> bool { + return {self.fx_fcw} == {other.fx_fcw} && + {self.fx_fsw} == {other.fx_fsw} && + {self.fx_ftw} == {other.fx_ftw} && + {self.fx_fop} == {other.fx_fop} && + {self.fx_rip} == {other.fx_rip} && + {self.fx_rdp} == {other.fx_rdp} && + {self.fx_mxcsr} == {other.fx_mxcsr} && + {self.fx_mxcsr_mask} == {other.fx_mxcsr_mask} && + {self.fx_st}.iter().zip({other.fx_st}.iter()).all(|(a,b)| a == b) && + {self.fx_xmm}.iter().zip({other.fx_xmm}.iter()).all(|(a,b)| a == b) + } + } + impl Eq for fxsave64 {} + impl ::fmt::Debug for fxsave64 { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("fxsave64") + .field("fx_fcw", &{self.fx_fcw}) + .field("fx_fsw", &{self.fx_fsw}) + .field("fx_ftw", &{self.fx_ftw}) + .field("fx_fop", &{self.fx_fop}) + .field("fx_rip", &{self.fx_rip}) + .field("fx_rdp", &{self.fx_rdp}) + .field("fx_mxcsr", &{self.fx_mxcsr}) + .field("fx_mxcsr_mask", &{self.fx_mxcsr_mask}) + // FIXME: .field("fx_st", &{self.fx_st}) + // FIXME: .field("fx_xmm", &{self.fx_xmm}) + .finish() + } + } + impl ::hash::Hash for fxsave64 { + fn hash(&self, state: &mut H) { + {self.fx_fcw}.hash(state); + {self.fx_fsw}.hash(state); + {self.fx_ftw}.hash(state); + {self.fx_fop}.hash(state); + {self.fx_rip}.hash(state); + {self.fx_rdp}.hash(state); + {self.fx_mxcsr}.hash(state); + {self.fx_mxcsr_mask}.hash(state); + {self.fx_st}.hash(state); + {self.fx_xmm}.hash(state); + } + } + } +} // should be pub(crate), but that requires Rust 1.18.0 cfg_if! { diff --git a/crux-mir/lib/libc/src/unix/haiku/b32.rs b/crux-mir/lib/libc/src/unix/haiku/b32.rs index cce886488..073ae9d4b 100644 --- a/crux-mir/lib/libc/src/unix/haiku/b32.rs +++ b/crux-mir/lib/libc/src/unix/haiku/b32.rs @@ -1,3 +1,20 @@ pub type c_long = i32; pub type c_ulong = u32; pub type time_t = i32; + +pub type Elf_Addr = ::Elf32_Addr; +pub type Elf_Half = ::Elf32_Half; +pub type Elf_Phdr = ::Elf32_Phdr; + +s! { + pub struct Elf32_Phdr { + pub p_type: ::Elf32_Word, + pub p_offset: ::Elf32_Off, + pub p_vaddr: ::Elf32_Addr, + pub p_paddr: ::Elf32_Addr, + pub p_filesz: ::Elf32_Word, + pub p_memsz: ::Elf32_Word, + pub p_flags: ::Elf32_Word, + pub p_align: ::Elf32_Word, + } +} diff --git a/crux-mir/lib/libc/src/unix/haiku/b64.rs b/crux-mir/lib/libc/src/unix/haiku/b64.rs index 3e66f14c9..456918052 100644 --- a/crux-mir/lib/libc/src/unix/haiku/b64.rs +++ b/crux-mir/lib/libc/src/unix/haiku/b64.rs @@ -1,3 +1,20 @@ pub type c_ulong = u64; pub type c_long = i64; pub type time_t = i64; + +pub type Elf_Addr = ::Elf64_Addr; +pub type Elf_Half = ::Elf64_Half; +pub type Elf_Phdr = ::Elf64_Phdr; + +s! { + pub struct Elf64_Phdr { + pub p_type: ::Elf64_Word, + pub p_flags: ::Elf64_Word, + pub p_offset: ::Elf64_Off, + pub p_vaddr: ::Elf64_Addr, + pub p_paddr: ::Elf64_Addr, + pub p_filesz: ::Elf64_Xword, + pub p_memsz: ::Elf64_Xword, + pub p_align: ::Elf64_Xword, + } +} diff --git a/crux-mir/lib/libc/src/unix/haiku/mod.rs b/crux-mir/lib/libc/src/unix/haiku/mod.rs index 95adabdf1..005b1d9df 100644 --- a/crux-mir/lib/libc/src/unix/haiku/mod.rs +++ b/crux-mir/lib/libc/src/unix/haiku/mod.rs @@ -1,9 +1,9 @@ pub type rlim_t = ::uintptr_t; pub type sa_family_t = u8; pub type pthread_key_t = ::c_int; -pub type nfds_t = ::c_long; +pub type nfds_t = ::c_ulong; pub type tcflag_t = ::c_uint; -pub type speed_t = ::c_uint; +pub type speed_t = ::c_uchar; pub type c_char = i8; pub type clock_t = i32; pub type clockid_t = i32; @@ -19,6 +19,7 @@ pub type nlink_t = i32; pub type useconds_t = u32; pub type socklen_t = u32; pub type pthread_t = ::uintptr_t; +pub type pthread_condattr_t = ::uintptr_t; pub type pthread_mutexattr_t = ::uintptr_t; pub type pthread_rwlockattr_t = ::uintptr_t; pub type sigset_t = u64; @@ -27,7 +28,32 @@ pub type fsfilcnt_t = i64; pub type pthread_attr_t = *mut ::c_void; pub type nl_item = ::c_int; pub type id_t = i32; -pub type idtype_t = ::c_uint; +pub type idtype_t = ::c_int; +pub type fd_mask = u32; +pub type regoff_t = ::c_int; +pub type key_t = i32; +pub type msgqnum_t = u32; +pub type msglen_t = u32; + +pub type Elf32_Addr = u32; +pub type Elf32_Half = u16; +pub type Elf32_Off = u32; +pub type Elf32_Sword = i32; +pub type Elf32_Word = u32; + +pub type Elf64_Addr = u64; +pub type Elf64_Half = u16; +pub type Elf64_Off = u64; +pub type Elf64_Sword = i32; +pub type Elf64_Sxword = i64; +pub type Elf64_Word = u32; +pub type Elf64_Xword = u64; + +pub type ENTRY = entry; +pub type ACTION = ::c_int; + +pub type posix_spawnattr_t = *mut ::c_void; +pub type posix_spawn_file_actions_t = *mut ::c_void; #[cfg_attr(feature = "extra_traits", derive(Debug))] pub enum timezone {} @@ -38,6 +64,24 @@ impl ::Clone for timezone { } } +impl siginfo_t { + pub unsafe fn si_addr(&self) -> *mut ::c_void { + self.si_addr + } + + pub unsafe fn si_pid(&self) -> ::pid_t { + self.si_pid + } + + pub unsafe fn si_uid(&self) -> ::uid_t { + self.si_uid + } + + pub unsafe fn si_status(&self) -> ::c_int { + self.si_status + } +} + s! { pub struct in_addr { pub s_addr: ::in_addr_t, @@ -51,7 +95,7 @@ s! { pub struct sockaddr { pub sa_len: u8, pub sa_family: sa_family_t, - pub sa_data: [::c_char; 30], + pub sa_data: [u8; 30], } pub struct sockaddr_in { @@ -59,13 +103,13 @@ s! { pub sin_family: sa_family_t, pub sin_port: ::in_port_t, pub sin_addr: ::in_addr, - pub sin_zero: [u8; 24], + pub sin_zero: [i8; 24], } pub struct sockaddr_in6 { pub sin6_len: u8, - pub sin6_family: sa_family_t, - pub sin6_port: ::in_port_t, + pub sin6_family: u8, + pub sin6_port: u16, pub sin6_flowinfo: u32, pub sin6_addr: ::in6_addr, pub sin6_scope_id: u32, @@ -82,8 +126,19 @@ s! { pub ai_next: *mut addrinfo, } + pub struct ifaddrs { + pub ifa_next: *mut ifaddrs, + pub ifa_name: *const ::c_char, + pub ifa_flags: ::c_uint, + pub ifa_addr: *mut ::sockaddr, + pub ifa_netmask: *mut ::sockaddr, + pub ifa_dstaddr: *mut ::sockaddr, + pub ifa_data: *mut ::c_void, + } + pub struct fd_set { - fds_bits: [c_ulong; FD_SETSIZE / ULONG_SIZE], + // size for 1024 bits, and a fd_mask with size u32 + fds_bits: [fd_mask; 32], } pub struct tm { @@ -96,8 +151,8 @@ s! { pub tm_wday: ::c_int, pub tm_yday: ::c_int, pub tm_isdst: ::c_int, - pub tm_gmtoff: ::c_long, - pub tm_zone: *const ::c_char, + pub tm_gmtoff: ::c_int, + pub tm_zone: *mut ::c_char, } pub struct utsname { @@ -137,16 +192,16 @@ s! { pub struct msghdr { pub msg_name: *mut ::c_void, - pub msg_namelen: ::socklen_t, + pub msg_namelen: socklen_t, pub msg_iov: *mut ::iovec, pub msg_iovlen: ::c_int, pub msg_control: *mut ::c_void, - pub msg_controllen: ::socklen_t, + pub msg_controllen: socklen_t, pub msg_flags: ::c_int, } pub struct cmsghdr { - pub cmsg_len: ::size_t, + pub cmsg_len: ::socklen_t, pub cmsg_level: ::c_int, pub cmsg_type: ::c_int, } @@ -240,6 +295,10 @@ s! { waiters: [*mut ::c_void; 2], } + pub struct pthread_spinlock_t { + lock: u32, + } + pub struct passwd { pub pw_name: *mut ::c_char, pub pw_passwd: *mut ::c_char, @@ -283,22 +342,100 @@ s! { } pub struct sigaction { - pub sa_sigaction: ::sighandler_t, + pub sa_sigaction: ::sighandler_t, //actually a union with sa_handler pub sa_mask: ::sigset_t, pub sa_flags: ::c_int, sa_userdata: *mut ::c_void, } pub struct sem_t { - pub se_type: i32, - pub se_named_id: i32, // this is actually a union - pub se_unnamed: i32, - pub se_padding: [i32; 4], + pub type_: i32, + pub named_sem_id: i32, // actually a union with unnamed_sem (i32) + pub padding: [i32; 2], + } + + pub struct ucred { + pub pid: ::pid_t, + pub uid: ::uid_t, + pub gid: ::gid_t, + } + + pub struct sockaddr_dl { + pub sdl_len: u8, + pub sdl_family: u8, + pub sdl_e_type: u16, + pub sdl_index: u32, + pub sdl_type: u8, + pub sdl_nlen: u8, + pub sdl_alen: u8, + pub sdl_slen: u8, + pub sdl_data: [u8; 46], + } + + pub struct spwd { + pub sp_namp: *mut ::c_char, + pub sp_pwdp: *mut ::c_char, + pub sp_lstchg: ::c_int, + pub sp_min: ::c_int, + pub sp_max: ::c_int, + pub sp_warn: ::c_int, + pub sp_inact: ::c_int, + pub sp_expire: ::c_int, + pub sp_flag: ::c_int, + } + + pub struct regex_t { + __buffer: *mut ::c_void, + __allocated: ::size_t, + __used: ::size_t, + __syntax: ::c_ulong, + __fastmap: *mut ::c_char, + __translate: *mut ::c_char, + __re_nsub: ::size_t, + __bitfield: u8, + } + + pub struct regmatch_t { + pub rm_so: regoff_t, + pub rm_eo: regoff_t, } - pub struct pthread_condattr_t { - pub process_shared: bool, - pub clock_id: i32, + pub struct msqid_ds { + pub msg_perm: ::ipc_perm, + pub msg_qnum: ::msgqnum_t, + pub msg_qbytes: ::msglen_t, + pub msg_lspid: ::pid_t, + pub msg_lrpid: ::pid_t, + pub msg_stime: ::time_t, + pub msg_rtime: ::time_t, + pub msg_ctime: ::time_t, + } + + pub struct ipc_perm { + pub key: ::key_t, + pub uid: ::uid_t, + pub gid: ::gid_t, + pub cuid: ::uid_t, + pub cgid: ::gid_t, + pub mode: ::mode_t, + } + + pub struct sembuf { + pub sem_num: ::c_ushort, + pub sem_op: ::c_short, + pub sem_flg: ::c_short, + } + + pub struct entry { + pub key: *mut ::c_char, + pub data: *mut ::c_void, + } + + pub struct option { + pub name: *const ::c_char, + pub has_arg: ::c_int, + pub flag: *mut ::c_int, + pub val: ::c_int, } } @@ -331,10 +468,63 @@ s_no_extra_traits! { __unused1: *mut ::c_void, // actually a function pointer pub sigev_notify_attributes: *mut ::pthread_attr_t, } + + pub struct utmpx { + pub ut_type: ::c_short, + pub ut_tv: ::timeval, + pub ut_id: [::c_char; 8], + pub ut_pid: ::pid_t, + pub ut_user: [::c_char; 32], + pub ut_line: [::c_char; 16], + pub ut_host: [::c_char; 128], + __ut_reserved: [::c_char; 64], + } } cfg_if! { if #[cfg(feature = "extra_traits")] { + impl PartialEq for utmpx { + fn eq(&self, other: &utmpx) -> bool { + self.ut_type == other.ut_type + && self.ut_tv == other.ut_tv + && self.ut_id == other.ut_id + && self.ut_pid == other.ut_pid + && self.ut_user == other.ut_user + && self.ut_line == other.ut_line + && self.ut_host.iter().zip(other.ut_host.iter()).all(|(a,b)| a == b) + && self.__ut_reserved == other.__ut_reserved + } + } + + impl Eq for utmpx {} + + impl ::fmt::Debug for utmpx { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("utmpx") + .field("ut_type", &self.ut_type) + .field("ut_tv", &self.ut_tv) + .field("ut_id", &self.ut_id) + .field("ut_pid", &self.ut_pid) + .field("ut_user", &self.ut_user) + .field("ut_line", &self.ut_line) + .field("ut_host", &self.ut_host) + .field("__ut_reserved", &self.__ut_reserved) + .finish() + } + } + + impl ::hash::Hash for utmpx { + fn hash(&self, state: &mut H) { + self.ut_type.hash(state); + self.ut_tv.hash(state); + self.ut_id.hash(state); + self.ut_pid.hash(state); + self.ut_user.hash(state); + self.ut_line.hash(state); + self.ut_host.hash(state); + self.__ut_reserved.hash(state); + } + } impl PartialEq for sockaddr_un { fn eq(&self, other: &sockaddr_un) -> bool { self.sun_len == other.sun_len @@ -473,17 +663,6 @@ cfg_if! { } } -// intentionally not public, only used for fd_set -cfg_if! { - if #[cfg(target_pointer_width = "32")] { - const ULONG_SIZE: usize = 32; - } else if #[cfg(target_pointer_width = "64")] { - const ULONG_SIZE: usize = 64; - } else { - // Unknown target_pointer_width - } -} - pub const EXIT_FAILURE: ::c_int = 1; pub const EXIT_SUCCESS: ::c_int = 0; pub const RAND_MAX: ::c_int = 2147483647; @@ -531,6 +710,8 @@ pub const PTHREAD_CREATE_DETACHED: ::c_int = 1; pub const CLOCK_REALTIME: ::c_int = -1; pub const CLOCK_MONOTONIC: ::c_int = 0; +pub const CLOCK_PROCESS_CPUTIME_ID: ::c_int = -2; +pub const CLOCK_THREAD_CPUTIME_ID: ::c_int = -3; pub const RLIMIT_CORE: ::c_int = 0; pub const RLIMIT_CPU: ::c_int = 1; @@ -539,9 +720,10 @@ pub const RLIMIT_FSIZE: ::c_int = 3; pub const RLIMIT_NOFILE: ::c_int = 4; pub const RLIMIT_STACK: ::c_int = 5; pub const RLIMIT_AS: ::c_int = 6; +pub const RLIM_INFINITY: ::rlim_t = 0xffffffff; // Haiku specific pub const RLIMIT_NOVMON: ::c_int = 7; -pub const RLIMIT_NLIMITS: ::c_int = 8; +pub const RLIM_NLIMITS: ::c_int = 8; pub const RUSAGE_SELF: ::c_int = 0; @@ -578,18 +760,20 @@ pub const S_IFREG: ::mode_t = 32768; pub const S_IFLNK: ::mode_t = 40960; pub const S_IFSOCK: ::mode_t = 49152; pub const S_IFMT: ::mode_t = 61440; -pub const S_IRWXU: ::mode_t = 448; -pub const S_IXUSR: ::mode_t = 64; -pub const S_IWUSR: ::mode_t = 128; -pub const S_IRUSR: ::mode_t = 256; -pub const S_IRWXG: ::mode_t = 70; -pub const S_IXGRP: ::mode_t = 10; -pub const S_IWGRP: ::mode_t = 20; -pub const S_IRGRP: ::mode_t = 40; -pub const S_IRWXO: ::mode_t = 7; -pub const S_IXOTH: ::mode_t = 1; -pub const S_IWOTH: ::mode_t = 2; -pub const S_IROTH: ::mode_t = 4; + +pub const S_IRWXU: ::mode_t = 0o00700; +pub const S_IRUSR: ::mode_t = 0o00400; +pub const S_IWUSR: ::mode_t = 0o00200; +pub const S_IXUSR: ::mode_t = 0o00100; +pub const S_IRWXG: ::mode_t = 0o00070; +pub const S_IRGRP: ::mode_t = 0o00040; +pub const S_IWGRP: ::mode_t = 0o00020; +pub const S_IXGRP: ::mode_t = 0o00010; +pub const S_IRWXO: ::mode_t = 0o00007; +pub const S_IROTH: ::mode_t = 0o00004; +pub const S_IWOTH: ::mode_t = 0o00002; +pub const S_IXOTH: ::mode_t = 0o00001; + pub const F_OK: ::c_int = 0; pub const R_OK: ::c_int = 4; pub const W_OK: ::c_int = 2; @@ -668,6 +852,7 @@ pub const MAP_SHARED: ::c_int = 0x01; pub const MAP_PRIVATE: ::c_int = 0x02; pub const MAP_FIXED: ::c_int = 0x04; pub const MAP_ANONYMOUS: ::c_int = 0x08; +pub const MAP_NORESERVE: ::c_int = 0x10; pub const MAP_ANON: ::c_int = MAP_ANONYMOUS; pub const MAP_FAILED: *mut ::c_void = !0 as *mut ::c_void; @@ -689,8 +874,8 @@ pub const ENOTTY: ::c_int = -2147454966; pub const ENXIO: ::c_int = -2147454965; pub const ESPIPE: ::c_int = -2147454964; pub const ESRCH: ::c_int = -2147454963; -pub const EFPOS: ::c_int = -2147457962; -pub const ESIGPARM: ::c_int = -2147457961; +pub const EFPOS: ::c_int = -2147454962; +pub const ESIGPARM: ::c_int = -2147454961; pub const EDOM: ::c_int = -2147454960; pub const ERANGE: ::c_int = -2147454959; pub const EPROTOTYPE: ::c_int = -2147454958; @@ -738,7 +923,7 @@ pub const ETXTBSY: ::c_int = -2147454917; pub const ENOATTR: ::c_int = -2147454916; // INT_MIN -pub const ENOMEM: ::c_int = -2147454976; +pub const ENOMEM: ::c_int = -2147483648; // POSIX errors that can be mapped to BeOS error codes pub const EACCES: ::c_int = -2147483646; @@ -774,6 +959,7 @@ pub const MADV_SEQUENTIAL: ::c_int = 2; pub const MADV_RANDOM: ::c_int = 3; pub const MADV_WILLNEED: ::c_int = 4; pub const MADV_DONTNEED: ::c_int = 5; +pub const MADV_FREE: ::c_int = 6; // https://github.com/haiku/haiku/blob/master/headers/posix/net/if.h#L80 pub const IFF_UP: ::c_int = 0x0001; @@ -790,7 +976,7 @@ pub const IFF_AUTO_CONFIGURED: ::c_int = 0x2000; pub const IFF_CONFIGURING: ::c_int = 0x4000; pub const IFF_MULTICAST: ::c_int = 0x8000; // supports multicast -pub const AF_UNSEC: ::c_int = 0; +pub const AF_UNSPEC: ::c_int = 0; pub const AF_INET: ::c_int = 1; pub const AF_APPLETALK: ::c_int = 2; pub const AF_ROUTE: ::c_int = 3; @@ -803,6 +989,15 @@ pub const AF_LOCAL: ::c_int = 9; pub const AF_UNIX: ::c_int = AF_LOCAL; pub const AF_BLUETOOTH: ::c_int = 10; +pub const PF_UNSPEC: ::c_int = AF_UNSPEC; +pub const PF_INET: ::c_int = AF_INET; +pub const PF_ROUTE: ::c_int = AF_ROUTE; +pub const PF_LINK: ::c_int = AF_LINK; +pub const PF_INET6: ::c_int = AF_INET6; +pub const PF_LOCAL: ::c_int = AF_LOCAL; +pub const PF_UNIX: ::c_int = AF_UNIX; +pub const PF_BLUETOOTH: ::c_int = AF_BLUETOOTH; + pub const IP_OPTIONS: ::c_int = 1; pub const IP_HDRINCL: ::c_int = 2; pub const IP_TOS: ::c_int = 3; @@ -826,6 +1021,9 @@ pub const TCP_MAXSEG: ::c_int = 0x02; pub const TCP_NOPUSH: ::c_int = 0x04; pub const TCP_NOOPT: ::c_int = 0x08; +pub const IF_NAMESIZE: ::size_t = 32; +pub const IFNAMSIZ: ::size_t = IF_NAMESIZE; + pub const IPV6_MULTICAST_IF: ::c_int = 24; pub const IPV6_MULTICAST_HOPS: ::c_int = 25; pub const IPV6_MULTICAST_LOOP: ::c_int = 26; @@ -836,7 +1034,7 @@ pub const IPV6_V6ONLY: ::c_int = 30; pub const IPV6_PKTINFO: ::c_int = 31; pub const IPV6_RECVPKTINFO: ::c_int = 32; pub const IPV6_HOPLIMIT: ::c_int = 33; -pub const IPV6_REVCHOPLIMIT: ::c_int = 34; +pub const IPV6_RECVHOPLIMIT: ::c_int = 34; pub const IPV6_HOPOPTS: ::c_int = 35; pub const IPV6_DSTOPTS: ::c_int = 36; pub const IPV6_RTHDR: ::c_int = 37; @@ -863,8 +1061,10 @@ pub const LOCK_EX: ::c_int = 0x02; pub const LOCK_NB: ::c_int = 0x04; pub const LOCK_UN: ::c_int = 0x08; +pub const MINSIGSTKSZ: ::size_t = 8192; pub const SIGSTKSZ: ::size_t = 16384; +pub const IOV_MAX: ::c_int = 1024; pub const PATH_MAX: ::c_int = 1024; pub const SA_NOCLDSTOP: ::c_int = 0x01; @@ -878,6 +1078,9 @@ pub const SA_NOMASK: ::c_int = SA_NODEFER; pub const SA_STACK: ::c_int = SA_ONSTACK; pub const SA_ONESHOT: ::c_int = SA_RESETHAND; +pub const SS_ONSTACK: ::c_int = 0x1; +pub const SS_DISABLE: ::c_int = 0x2; + pub const FD_SETSIZE: usize = 1024; pub const RTLD_LOCAL: ::c_int = 0x0; @@ -915,7 +1118,10 @@ pub const _PC_2_SYMLINKS: ::c_int = 37; pub const _PC_XATTR_EXISTS: ::c_int = 38; pub const _PC_XATTR_ENABLED: ::c_int = 39; -pub const FIONBIO: ::c_int = 0xbe000000; +pub const FIONBIO: ::c_ulong = 0xbe000000; +pub const FIONREAD: ::c_ulong = 0xbe000001; +pub const FIOSEEKDATA: ::c_ulong = 0xbe000002; +pub const FIOSEEKHOLE: ::c_ulong = 0xbe000003; pub const _SC_ARG_MAX: ::c_int = 15; pub const _SC_CHILD_MAX: ::c_int = 16; @@ -962,6 +1168,11 @@ pub const _SC_TIMER_MAX: ::c_int = 57; pub const _SC_TIMERS: ::c_int = 58; pub const _SC_CPUTIME: ::c_int = 59; pub const _SC_THREAD_CPUTIME: ::c_int = 60; +pub const _SC_HOST_NAME_MAX: ::c_int = 61; +pub const _SC_REGEXP: ::c_int = 62; +pub const _SC_SYMLOOP_MAX: ::c_int = 63; +pub const _SC_SHELL: ::c_int = 64; +pub const _SC_TTY_NAME_MAX: ::c_int = 65; pub const PTHREAD_STACK_MIN: ::size_t = 8192; @@ -1028,6 +1239,8 @@ pub const SO_PEERCRED: ::c_int = 0x4000000b; pub const SCM_RIGHTS: ::c_int = 0x01; +pub const SOMAXCONN: ::c_int = 32; + pub const NI_MAXHOST: ::size_t = 1025; pub const WNOHANG: ::c_int = 0x01; @@ -1037,6 +1250,19 @@ pub const WEXITED: ::c_int = 0x08; pub const WSTOPPED: ::c_int = 0x10; pub const WNOWAIT: ::c_int = 0x20; +// si_code values for SIGBUS signal +pub const BUS_ADRALN: ::c_int = 40; +pub const BUS_ADRERR: ::c_int = 41; +pub const BUS_OBJERR: ::c_int = 42; + +// si_code values for SIGCHLD signal +pub const CLD_EXITED: ::c_int = 60; +pub const CLD_KILLED: ::c_int = 61; +pub const CLD_DUMPED: ::c_int = 62; +pub const CLD_TRAPPED: ::c_int = 63; +pub const CLD_STOPPED: ::c_int = 64; +pub const CLD_CONTINUED: ::c_int = 65; + pub const P_ALL: idtype_t = 0; pub const P_PID: idtype_t = 1; pub const P_PGID: idtype_t = 2; @@ -1181,32 +1407,106 @@ pub const TCIFLUSH: ::c_int = 0x01; pub const TCOFLUSH: ::c_int = 0x02; pub const TCIOFLUSH: ::c_int = 0x03; -pub const TCGETA: ::c_int = 0x8000; -pub const TCSETA: ::c_int = TCGETA + 1; -pub const TCSETAF: ::c_int = TCGETA + 2; -pub const TCSETAW: ::c_int = TCGETA + 3; -pub const TCWAITEVENT: ::c_int = TCGETA + 4; -pub const TCSBRK: ::c_int = TCGETA + 5; -pub const TCFLSH: ::c_int = TCGETA + 6; -pub const TCXONC: ::c_int = TCGETA + 7; -pub const TCQUERYCONNECTED: ::c_int = TCGETA + 8; -pub const TCGETBITS: ::c_int = TCGETA + 9; -pub const TCSETDTR: ::c_int = TCGETA + 10; -pub const TCSETRTS: ::c_int = TCGETA + 11; -pub const TIOCGWINSZ: ::c_int = TCGETA + 12; -pub const TIOCSWINSZ: ::c_int = TCGETA + 13; -pub const TCVTIME: ::c_int = TCGETA + 14; -pub const TIOCGPGRP: ::c_int = TCGETA + 15; -pub const TIOCSPGRP: ::c_int = TCGETA + 16; -pub const TIOCSCTTY: ::c_int = TCGETA + 17; -pub const TIOCMGET: ::c_int = TCGETA + 18; -pub const TIOCMSET: ::c_int = TCGETA + 19; -pub const TIOCSBRK: ::c_int = TCGETA + 20; -pub const TIOCCBRK: ::c_int = TCGETA + 21; -pub const TIOCMBIS: ::c_int = TCGETA + 22; -pub const TIOCMBIC: ::c_int = TCGETA + 23; +pub const TCGETA: ::c_ulong = 0x8000; +pub const TCSETA: ::c_ulong = TCGETA + 1; +pub const TCSETAF: ::c_ulong = TCGETA + 2; +pub const TCSETAW: ::c_ulong = TCGETA + 3; +pub const TCWAITEVENT: ::c_ulong = TCGETA + 4; +pub const TCSBRK: ::c_ulong = TCGETA + 5; +pub const TCFLSH: ::c_ulong = TCGETA + 6; +pub const TCXONC: ::c_ulong = TCGETA + 7; +pub const TCQUERYCONNECTED: ::c_ulong = TCGETA + 8; +pub const TCGETBITS: ::c_ulong = TCGETA + 9; +pub const TCSETDTR: ::c_ulong = TCGETA + 10; +pub const TCSETRTS: ::c_ulong = TCGETA + 11; +pub const TIOCGWINSZ: ::c_ulong = TCGETA + 12; +pub const TIOCSWINSZ: ::c_ulong = TCGETA + 13; +pub const TCVTIME: ::c_ulong = TCGETA + 14; +pub const TIOCGPGRP: ::c_ulong = TCGETA + 15; +pub const TIOCSPGRP: ::c_ulong = TCGETA + 16; +pub const TIOCSCTTY: ::c_ulong = TCGETA + 17; +pub const TIOCMGET: ::c_ulong = TCGETA + 18; +pub const TIOCMSET: ::c_ulong = TCGETA + 19; +pub const TIOCSBRK: ::c_ulong = TCGETA + 20; +pub const TIOCCBRK: ::c_ulong = TCGETA + 21; +pub const TIOCMBIS: ::c_ulong = TCGETA + 22; +pub const TIOCMBIC: ::c_ulong = TCGETA + 23; + +pub const PRIO_PROCESS: ::c_int = 0; +pub const PRIO_PGRP: ::c_int = 1; +pub const PRIO_USER: ::c_int = 2; + +// utmpx entry types +pub const EMPTY: ::c_short = 0; +pub const BOOT_TIME: ::c_short = 1; +pub const OLD_TIME: ::c_short = 2; +pub const NEW_TIME: ::c_short = 3; +pub const USER_PROCESS: ::c_short = 4; +pub const INIT_PROCESS: ::c_short = 5; +pub const LOGIN_PROCESS: ::c_short = 6; +pub const DEAD_PROCESS: ::c_short = 7; + +pub const LOG_PID: ::c_int = 1 << 12; +pub const LOG_CONS: ::c_int = 2 << 12; +pub const LOG_ODELAY: ::c_int = 4 << 12; +pub const LOG_NDELAY: ::c_int = 8 << 12; +pub const LOG_SERIAL: ::c_int = 16 << 12; +pub const LOG_PERROR: ::c_int = 32 << 12; +pub const LOG_NOWAIT: ::c_int = 64 << 12; + +// spawn.h +pub const POSIX_SPAWN_RESETIDS: ::c_int = 0x01; +pub const POSIX_SPAWN_SETPGROUP: ::c_int = 0x02; +pub const POSIX_SPAWN_SETSIGDEF: ::c_int = 0x10; +pub const POSIX_SPAWN_SETSIGMASK: ::c_int = 0x20; +pub const POSIX_SPAWN_SETSID: ::c_int = 0x40; + +const_fn! { + {const} fn CMSG_ALIGN(len: usize) -> usize { + len + ::mem::size_of::() - 1 & !(::mem::size_of::() - 1) + } +} f! { + pub fn CMSG_FIRSTHDR(mhdr: *const msghdr) -> *mut cmsghdr { + if (*mhdr).msg_controllen as usize >= ::mem::size_of::() { + (*mhdr).msg_control as *mut cmsghdr + } else { + 0 as *mut cmsghdr + } + } + + pub fn CMSG_DATA(cmsg: *const ::cmsghdr) -> *mut ::c_uchar { + (cmsg as *mut ::c_uchar) + .offset(CMSG_ALIGN(::mem::size_of::<::cmsghdr>()) as isize) + } + + pub {const} fn CMSG_SPACE(length: ::c_uint) -> ::c_uint { + (CMSG_ALIGN(length as usize) + CMSG_ALIGN(::mem::size_of::())) + as ::c_uint + } + + pub fn CMSG_LEN(length: ::c_uint) -> ::c_uint { + CMSG_ALIGN(::mem::size_of::()) as ::c_uint + length + } + + pub fn CMSG_NXTHDR(mhdr: *const msghdr, + cmsg: *const cmsghdr) -> *mut cmsghdr { + if cmsg.is_null() { + return ::CMSG_FIRSTHDR(mhdr); + }; + let next = cmsg as usize + CMSG_ALIGN((*cmsg).cmsg_len as usize) + + CMSG_ALIGN(::mem::size_of::<::cmsghdr>()); + let max = (*mhdr).msg_control as usize + + (*mhdr).msg_controllen as usize; + if next > max { + 0 as *mut ::cmsghdr + } else { + (cmsg as usize + CMSG_ALIGN((*cmsg).cmsg_len as usize)) + as *mut ::cmsghdr + } + } + pub fn FD_CLR(fd: ::c_int, set: *mut fd_set) -> () { let fd = fd as usize; let size = ::mem::size_of_val(&(*set).fds_bits[0]) * 8; @@ -1214,7 +1514,7 @@ f! { return } - pub fn FD_ISSET(fd: ::c_int, set: *mut fd_set) -> bool { + pub fn FD_ISSET(fd: ::c_int, set: *const fd_set) -> bool { let fd = fd as usize; let size = ::mem::size_of_val(&(*set).fds_bits[0]) * 8; return ((*set).fds_bits[fd / size] & (1 << (fd % size))) != 0 @@ -1232,37 +1532,39 @@ f! { *slot = 0; } } +} - pub fn WIFEXITED(status: ::c_int) -> bool { +safe_f! { + pub {const} fn WIFEXITED(status: ::c_int) -> bool { (status & !0xff) == 0 } - pub fn WEXITSTATUS(status: ::c_int) -> ::c_int { + pub {const} fn WEXITSTATUS(status: ::c_int) -> ::c_int { status & 0xff } - pub fn WIFSIGNALED(status: ::c_int) -> bool { + pub {const} fn WIFSIGNALED(status: ::c_int) -> bool { ((status >> 8) & 0xff) != 0 } - pub fn WTERMSIG(status: ::c_int) -> ::c_int { + pub {const} fn WTERMSIG(status: ::c_int) -> ::c_int { (status >> 8) & 0xff } - pub fn WIFSTOPPED(status: ::c_int) -> bool { + pub {const} fn WIFSTOPPED(status: ::c_int) -> bool { ((status >> 16) & 0xff) != 0 } - pub fn WSTOPSIG(status: ::c_int) -> ::c_int { + pub {const} fn WSTOPSIG(status: ::c_int) -> ::c_int { (status >> 16) & 0xff } // actually WIFCORED, but this is used everywhere else - pub fn WCOREDUMP(status: ::c_int) -> bool { + pub {const} fn WCOREDUMP(status: ::c_int) -> bool { (status & 0x10000) != 0 } - pub fn WIFCONTINUED(status: ::c_int) -> bool { + pub {const} fn WIFCONTINUED(status: ::c_int) -> bool { (status & 0x20000) != 0 } } @@ -1270,11 +1572,23 @@ f! { extern "C" { pub fn getrlimit(resource: ::c_int, rlim: *mut ::rlimit) -> ::c_int; pub fn setrlimit(resource: ::c_int, rlim: *const ::rlimit) -> ::c_int; - pub fn strerror_r( - errnum: ::c_int, - buf: *mut c_char, - buflen: ::size_t, + pub fn getpriority(which: ::c_int, who: id_t) -> ::c_int; + pub fn setpriority(which: ::c_int, who: id_t, priority: ::c_int) -> ::c_int; + + pub fn endusershell(); + pub fn getpass(prompt: *const ::c_char) -> *mut ::c_char; + pub fn getusershell() -> *mut ::c_char; + pub fn issetugid() -> ::c_int; + pub fn setusershell(); + + pub fn utimensat( + fd: ::c_int, + path: *const ::c_char, + times: *const ::timespec, + flag: ::c_int, ) -> ::c_int; + pub fn futimens(fd: ::c_int, times: *const ::timespec) -> ::c_int; + pub fn strerror_r(errnum: ::c_int, buf: *mut c_char, buflen: ::size_t) -> ::c_int; pub fn _errnop() -> *mut ::c_int; pub fn abs(i: ::c_int) -> ::c_int; @@ -1282,19 +1596,62 @@ extern "C" { pub fn labs(i: ::c_long) -> ::c_long; pub fn rand() -> ::c_int; pub fn srand(seed: ::c_uint); -} + pub fn getifaddrs(ifap: *mut *mut ::ifaddrs) -> ::c_int; + pub fn freeifaddrs(ifa: *mut ::ifaddrs); + pub fn ppoll( + fds: *mut ::pollfd, + numfds: ::nfds_t, + timeout: *const ::timespec, + sigMask: *const sigset_t, + ) -> ::c_int; -#[link(name = "bsd")] -extern "C" { - pub fn sem_destroy(sem: *mut sem_t) -> ::c_int; - pub fn sem_init( - sem: *mut sem_t, - pshared: ::c_int, - value: ::c_uint, + pub fn getspent() -> *mut spwd; + pub fn getspent_r( + pwd: *mut spwd, + buf: *mut ::c_char, + bufferSize: ::size_t, + res: *mut *mut spwd, + ) -> ::c_int; + pub fn setspent(); + pub fn endspent(); + pub fn getspnam(name: *const ::c_char) -> *mut spwd; + pub fn getspnam_r( + name: *const ::c_char, + spwd: *mut spwd, + buffer: *mut ::c_char, + bufferSize: ::size_t, + res: *mut *mut spwd, + ) -> ::c_int; + pub fn sgetspent(line: *const ::c_char) -> *mut spwd; + pub fn sgetspent_r( + line: *const ::c_char, + spwd: *mut spwd, + buffer: *mut ::c_char, + bufferSize: ::size_t, + res: *mut *mut spwd, + ) -> ::c_int; + pub fn fgetspent(file: *mut ::FILE) -> *mut spwd; + pub fn fgetspent_r( + file: *mut ::FILE, + spwd: *mut spwd, + buffer: *mut ::c_char, + bufferSize: ::size_t, + res: *mut *mut spwd, + ) -> ::c_int; + pub fn mkfifoat(dirfd: ::c_int, pathname: *const ::c_char, mode: ::mode_t) -> ::c_int; + pub fn mknodat( + dirfd: ::c_int, + pathname: *const ::c_char, + mode: ::mode_t, + dev: dev_t, ) -> ::c_int; + pub fn sem_destroy(sem: *mut sem_t) -> ::c_int; + pub fn sem_init(sem: *mut sem_t, pshared: ::c_int, value: ::c_uint) -> ::c_int; - pub fn clock_gettime(clk_id: ::c_int, tp: *mut ::timespec) -> ::c_int; - pub fn clock_settime(clk_id: ::c_int, tp: *const ::timespec) -> ::c_int; + pub fn clock_getres(clk_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; + pub fn clock_gettime(clk_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; + pub fn clock_settime(clk_id: ::clockid_t, tp: *const ::timespec) -> ::c_int; + pub fn clock_getcpuclockid(pid: ::pid_t, clk_id: *mut ::clockid_t) -> ::c_int; pub fn pthread_create( thread: *mut ::pthread_t, attr: *const ::pthread_attr_t, @@ -1318,72 +1675,56 @@ extern "C" { attr: *mut pthread_condattr_t, clock_id: ::clockid_t, ) -> ::c_int; + pub fn valloc(numBytes: ::size_t) -> *mut ::c_void; + pub fn malloc_usable_size(ptr: *mut ::c_void) -> ::size_t; pub fn memalign(align: ::size_t, size: ::size_t) -> *mut ::c_void; - pub fn setgroups(ngroups: ::size_t, ptr: *const ::gid_t) -> ::c_int; - pub fn ioctl(fd: ::c_int, request: ::c_int, ...) -> ::c_int; - pub fn mprotect( - addr: *const ::c_void, - len: ::size_t, - prot: ::c_int, - ) -> ::c_int; + pub fn setgroups(ngroups: ::c_int, ptr: *const ::gid_t) -> ::c_int; + pub fn initgroups(name: *const ::c_char, basegid: ::gid_t) -> ::c_int; + pub fn ioctl(fd: ::c_int, request: ::c_ulong, ...) -> ::c_int; + pub fn mprotect(addr: *mut ::c_void, len: ::size_t, prot: ::c_int) -> ::c_int; pub fn dirfd(dirp: *mut ::DIR) -> ::c_int; pub fn getnameinfo( sa: *const ::sockaddr, salen: ::socklen_t, host: *mut ::c_char, - hostlen: ::size_t, + hostlen: ::socklen_t, serv: *mut ::c_char, - sevlen: ::size_t, + sevlen: ::socklen_t, flags: ::c_int, ) -> ::c_int; pub fn pthread_mutex_timedlock( lock: *mut pthread_mutex_t, abstime: *const ::timespec, ) -> ::c_int; - pub fn waitid( - idtype: idtype_t, - id: id_t, - infop: *mut ::siginfo_t, - options: ::c_int, - ) -> ::c_int; + pub fn pthread_spin_init(lock: *mut ::pthread_spinlock_t, pshared: ::c_int) -> ::c_int; + pub fn pthread_spin_destroy(lock: *mut ::pthread_spinlock_t) -> ::c_int; + pub fn pthread_spin_lock(lock: *mut ::pthread_spinlock_t) -> ::c_int; + pub fn pthread_spin_trylock(lock: *mut ::pthread_spinlock_t) -> ::c_int; + pub fn pthread_spin_unlock(lock: *mut ::pthread_spinlock_t) -> ::c_int; + pub fn waitid(idtype: idtype_t, id: id_t, infop: *mut ::siginfo_t, options: ::c_int) + -> ::c_int; pub fn glob( pattern: *const ::c_char, flags: ::c_int, - errfunc: ::Option< - extern "C" fn(epath: *const ::c_char, errno: ::c_int) -> ::c_int, - >, + errfunc: ::Option ::c_int>, pglob: *mut ::glob_t, ) -> ::c_int; pub fn globfree(pglob: *mut ::glob_t); pub fn gettimeofday(tp: *mut ::timeval, tz: *mut ::c_void) -> ::c_int; - pub fn posix_madvise( - addr: *mut ::c_void, - len: ::size_t, - advice: ::c_int, - ) -> ::c_int; + pub fn posix_madvise(addr: *mut ::c_void, len: ::size_t, advice: ::c_int) -> ::c_int; + pub fn posix_fadvise(fd: ::c_int, offset: ::off_t, len: ::off_t, advice: ::c_int) -> ::c_int; + pub fn posix_fallocate(fd: ::c_int, offset: ::off_t, len: ::off_t) -> ::c_int; - pub fn shm_open( - name: *const ::c_char, - oflag: ::c_int, - mode: ::mode_t, - ) -> ::c_int; + pub fn shm_open(name: *const ::c_char, oflag: ::c_int, mode: ::mode_t) -> ::c_int; pub fn shm_unlink(name: *const ::c_char) -> ::c_int; pub fn seekdir(dirp: *mut ::DIR, loc: ::c_long); pub fn telldir(dirp: *mut ::DIR) -> ::c_long; - pub fn madvise( - addr: *mut ::c_void, - len: ::size_t, - advice: ::c_int, - ) -> ::c_int; + pub fn madvise(addr: *mut ::c_void, len: ::size_t, advice: ::c_int) -> ::c_int; - pub fn msync( - addr: *mut ::c_void, - len: ::size_t, - flags: ::c_int, - ) -> ::c_int; + pub fn msync(addr: *mut ::c_void, len: ::size_t, flags: ::c_int) -> ::c_int; pub fn recvfrom( socket: ::c_int, @@ -1394,43 +1735,20 @@ extern "C" { addrlen: *mut ::socklen_t, ) -> ::ssize_t; pub fn mkstemps(template: *mut ::c_char, suffixlen: ::c_int) -> ::c_int; - pub fn futimes(fd: ::c_int, times: *const ::timeval) -> ::c_int; - pub fn lutimes(file: *const ::c_char, times: *const ::timeval) -> ::c_int; pub fn nl_langinfo(item: ::nl_item) -> *mut ::c_char; - pub fn bind( - socket: ::c_int, - address: *const ::sockaddr, - address_len: ::socklen_t, - ) -> ::c_int; + pub fn bind(socket: ::c_int, address: *const ::sockaddr, address_len: ::socklen_t) -> ::c_int; - pub fn writev( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - ) -> ::ssize_t; - pub fn readv( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - ) -> ::ssize_t; + pub fn writev(fd: ::c_int, iov: *const ::iovec, count: ::c_int) -> ::ssize_t; + pub fn readv(fd: ::c_int, iov: *const ::iovec, count: ::c_int) -> ::ssize_t; - pub fn sendmsg( - fd: ::c_int, - msg: *const ::msghdr, - flags: ::c_int, - ) -> ::ssize_t; - pub fn recvmsg( - fd: ::c_int, - msg: *mut ::msghdr, - flags: ::c_int, - ) -> ::ssize_t; + pub fn sendmsg(fd: ::c_int, msg: *const ::msghdr, flags: ::c_int) -> ::ssize_t; + pub fn recvmsg(fd: ::c_int, msg: *mut ::msghdr, flags: ::c_int) -> ::ssize_t; pub fn execvpe( file: *const ::c_char, argv: *const *const ::c_char, environment: *const *const ::c_char, ) -> ::c_int; - #[cfg_attr(target_os = "solaris", link_name = "__posix_getgrgid_r")] pub fn getgrgid_r( gid: ::gid_t, grp: *mut ::group, @@ -1438,15 +1756,15 @@ extern "C" { buflen: ::size_t, result: *mut *mut ::group, ) -> ::c_int; - #[cfg_attr( - all(target_os = "macos", target_arch = "x86"), - link_name = "sigaltstack$UNIX2003" - )] - #[cfg_attr(target_os = "netbsd", link_name = "__sigaltstack14")] + pub fn getgrouplist( + user: *const ::c_char, + basegroup: ::gid_t, + grouplist: *mut ::gid_t, + groupcount: *mut ::c_int, + ) -> ::c_int; pub fn sigaltstack(ss: *const stack_t, oss: *mut stack_t) -> ::c_int; pub fn sem_close(sem: *mut sem_t) -> ::c_int; pub fn getdtablesize() -> ::c_int; - #[cfg_attr(target_os = "solaris", link_name = "__posix_getgrnam_r")] pub fn getgrnam_r( name: *const ::c_char, grp: *mut ::group, @@ -1454,22 +1772,11 @@ extern "C" { buflen: ::size_t, result: *mut *mut ::group, ) -> ::c_int; - #[cfg_attr( - all(target_os = "macos", target_arch = "x86"), - link_name = "pthread_sigmask$UNIX2003" - )] - pub fn pthread_sigmask( - how: ::c_int, - set: *const sigset_t, - oldset: *mut sigset_t, - ) -> ::c_int; + pub fn pthread_sigmask(how: ::c_int, set: *const sigset_t, oldset: *mut sigset_t) -> ::c_int; pub fn sem_open(name: *const ::c_char, oflag: ::c_int, ...) -> *mut sem_t; pub fn getgrnam(name: *const ::c_char) -> *mut ::group; pub fn pthread_kill(thread: ::pthread_t, sig: ::c_int) -> ::c_int; pub fn sem_unlink(name: *const ::c_char) -> ::c_int; - pub fn daemon(nochdir: ::c_int, noclose: ::c_int) -> ::c_int; - #[cfg_attr(target_os = "netbsd", link_name = "__getpwnam_r50")] - #[cfg_attr(target_os = "solaris", link_name = "__posix_getpwnam_r")] pub fn getpwnam_r( name: *const ::c_char, pwd: *mut passwd, @@ -1477,8 +1784,6 @@ extern "C" { buflen: ::size_t, result: *mut *mut passwd, ) -> ::c_int; - #[cfg_attr(target_os = "netbsd", link_name = "__getpwuid_r50")] - #[cfg_attr(target_os = "solaris", link_name = "__posix_getpwuid_r")] pub fn getpwuid_r( uid: ::uid_t, pwd: *mut passwd, @@ -1486,11 +1791,12 @@ extern "C" { buflen: ::size_t, result: *mut *mut passwd, ) -> ::c_int; - #[cfg_attr( - all(target_os = "macos", target_arch = "x86"), - link_name = "sigwait$UNIX2003" - )] - #[cfg_attr(target_os = "solaris", link_name = "__posix_sigwait")] + pub fn getpwent() -> *mut passwd; + pub fn setpwent(); + pub fn endpwent(); + pub fn endgrent(); + pub fn getgrent() -> *mut ::group; + pub fn setgrent(); pub fn sigwait(set: *const sigset_t, sig: *mut ::c_int) -> ::c_int; pub fn pthread_atfork( prepare: ::Option, @@ -1498,26 +1804,209 @@ extern "C" { child: ::Option, ) -> ::c_int; pub fn getgrgid(gid: ::gid_t) -> *mut ::group; - #[cfg_attr( - all(target_os = "macos", target_arch = "x86"), - link_name = "popen$UNIX2003" - )] pub fn popen(command: *const c_char, mode: *const c_char) -> *mut ::FILE; - pub fn openpty( + pub fn sethostname(name: *const ::c_char, len: ::size_t) -> ::c_int; + pub fn uname(buf: *mut ::utsname) -> ::c_int; + pub fn getutxent() -> *mut utmpx; + pub fn getutxid(ut: *const utmpx) -> *mut utmpx; + pub fn getutxline(ut: *const utmpx) -> *mut utmpx; + pub fn pututxline(ut: *const utmpx) -> *mut utmpx; + pub fn setutxent(); + pub fn endutxent(); + pub fn faccessat( + dirfd: ::c_int, + pathname: *const ::c_char, + mode: ::c_int, + flags: ::c_int, + ) -> ::c_int; + + pub fn sigtimedwait( + set: *const sigset_t, + info: *mut siginfo_t, + timeout: *const ::timespec, + ) -> ::c_int; + pub fn sigwaitinfo(set: *const sigset_t, info: *mut siginfo_t) -> ::c_int; + + pub fn getitimer(which: ::c_int, curr_value: *mut ::itimerval) -> ::c_int; + pub fn setitimer( + which: ::c_int, + new_value: *const ::itimerval, + old_value: *mut ::itimerval, + ) -> ::c_int; + + pub fn regcomp(preg: *mut regex_t, pattern: *const ::c_char, cflags: ::c_int) -> ::c_int; + + pub fn regexec( + preg: *const regex_t, + input: *const ::c_char, + nmatch: ::size_t, + pmatch: *mut regmatch_t, + eflags: ::c_int, + ) -> ::c_int; + + pub fn regerror( + errcode: ::c_int, + preg: *const regex_t, + errbuf: *mut ::c_char, + errbuf_size: ::size_t, + ) -> ::size_t; + + pub fn regfree(preg: *mut regex_t); + + pub fn msgctl(msqid: ::c_int, cmd: ::c_int, buf: *mut msqid_ds) -> ::c_int; + pub fn msgget(key: ::key_t, msgflg: ::c_int) -> ::c_int; + pub fn msgrcv( + msqid: ::c_int, + msgp: *mut ::c_void, + msgsz: ::size_t, + msgtype: ::c_long, + msgflg: ::c_int, + ) -> ::ssize_t; + pub fn msgsnd( + msqid: ::c_int, + msgp: *const ::c_void, + msgsz: ::size_t, + msgflg: ::c_int, + ) -> ::c_int; + pub fn semget(key: ::key_t, nsems: ::c_int, semflg: ::c_int) -> ::c_int; + pub fn semctl(semid: ::c_int, semnum: ::c_int, cmd: ::c_int, ...) -> ::c_int; + pub fn semop(semid: ::c_int, sops: *mut sembuf, nsops: ::size_t) -> ::c_int; + pub fn ftok(pathname: *const ::c_char, proj_id: ::c_int) -> ::key_t; + + pub fn memrchr(cx: *const ::c_void, c: ::c_int, n: ::size_t) -> *mut ::c_void; + + pub fn lsearch( + key: *const ::c_void, + base: *mut ::c_void, + nelp: *mut ::size_t, + width: ::size_t, + compar: ::Option ::c_int>, + ) -> *mut ::c_void; + pub fn lfind( + key: *const ::c_void, + base: *const ::c_void, + nelp: *mut ::size_t, + width: ::size_t, + compar: ::Option ::c_int>, + ) -> *mut ::c_void; + pub fn hcreate(nelt: ::size_t) -> ::c_int; + pub fn hdestroy(); + pub fn hsearch(entry: ::ENTRY, action: ::ACTION) -> *mut ::ENTRY; + + pub fn drand48() -> ::c_double; + pub fn erand48(xseed: *mut ::c_ushort) -> ::c_double; + pub fn lrand48() -> ::c_long; + pub fn nrand48(xseed: *mut ::c_ushort) -> ::c_long; + pub fn mrand48() -> ::c_long; + pub fn jrand48(xseed: *mut ::c_ushort) -> ::c_long; + pub fn srand48(seed: ::c_long); + pub fn seed48(xseed: *mut ::c_ushort) -> *mut ::c_ushort; + pub fn lcong48(p: *mut ::c_ushort); + + pub fn clearenv() -> ::c_int; + pub fn ctermid(s: *mut ::c_char) -> *mut ::c_char; + + pub fn sync(); + pub fn getpagesize() -> ::c_int; + + pub fn brk(addr: *mut ::c_void) -> ::c_int; + pub fn sbrk(increment: ::intptr_t) -> *mut ::c_void; + + pub fn posix_spawn( + pid: *mut ::pid_t, + path: *const ::c_char, + file_actions: *const ::posix_spawn_file_actions_t, + attrp: *const ::posix_spawnattr_t, + argv: *const *mut ::c_char, + envp: *const *mut ::c_char, + ) -> ::c_int; + pub fn posix_spawnp( + pid: *mut ::pid_t, + file: *const ::c_char, + file_actions: *const ::posix_spawn_file_actions_t, + attrp: *const ::posix_spawnattr_t, + argv: *const *mut ::c_char, + envp: *const *mut ::c_char, + ) -> ::c_int; + + pub fn posix_spawn_file_actions_init(file_actions: *mut posix_spawn_file_actions_t) -> ::c_int; + pub fn posix_spawn_file_actions_destroy( + file_actions: *mut posix_spawn_file_actions_t, + ) -> ::c_int; + pub fn posix_spawn_file_actions_addopen( + file_actions: *mut posix_spawn_file_actions_t, + fildes: ::c_int, + path: *const ::c_char, + oflag: ::c_int, + mode: ::mode_t, + ) -> ::c_int; + pub fn posix_spawn_file_actions_addclose( + file_actions: *mut posix_spawn_file_actions_t, + fildes: ::c_int, + ) -> ::c_int; + pub fn posix_spawn_file_actions_adddup2( + file_actions: *mut posix_spawn_file_actions_t, + fildes: ::c_int, + newfildes: ::c_int, + ) -> ::c_int; + + pub fn posix_spawnattr_init(attr: *mut posix_spawnattr_t) -> ::c_int; + pub fn posix_spawnattr_destroy(attr: *mut posix_spawnattr_t) -> ::c_int; + pub fn posix_spawnattr_getflags( + attr: *const posix_spawnattr_t, + _flags: *mut ::c_short, + ) -> ::c_int; + pub fn posix_spawnattr_setflags(attr: *mut posix_spawnattr_t, flags: ::c_short) -> ::c_int; + pub fn posix_spawnattr_getpgroup( + attr: *const posix_spawnattr_t, + _pgroup: *mut ::pid_t, + ) -> ::c_int; + pub fn posix_spawnattr_setpgroup(attr: *mut posix_spawnattr_t, pgroup: ::pid_t) -> ::c_int; + pub fn posix_spawnattr_getsigdefault( + attr: *const posix_spawnattr_t, + sigdefault: *mut ::sigset_t, + ) -> ::c_int; + pub fn posix_spawnattr_setsigdefault( + attr: *mut posix_spawnattr_t, + sigdefault: *const ::sigset_t, + ) -> ::c_int; + pub fn posix_spawnattr_getsigmask( + attr: *const posix_spawnattr_t, + _sigmask: *mut ::sigset_t, + ) -> ::c_int; + pub fn posix_spawnattr_setsigmask( + attr: *mut posix_spawnattr_t, + sigmask: *const ::sigset_t, + ) -> ::c_int; + pub fn getopt_long( + argc: ::c_int, + argv: *const *mut c_char, + optstring: *const c_char, + longopts: *const option, + longindex: *mut ::c_int, + ) -> ::c_int; +} + +#[link(name = "bsd")] +extern "C" { + pub fn lutimes(file: *const ::c_char, times: *const ::timeval) -> ::c_int; + pub fn daemon(nochdir: ::c_int, noclose: ::c_int) -> ::c_int; + pub fn forkpty( amaster: *mut ::c_int, - aslave: *mut ::c_int, name: *mut ::c_char, termp: *mut termios, winp: *mut ::winsize, - ) -> ::c_int; - pub fn forkpty( + ) -> ::pid_t; + pub fn openpty( amaster: *mut ::c_int, + aslave: *mut ::c_int, name: *mut ::c_char, termp: *mut termios, winp: *mut ::winsize, - ) -> ::pid_t; - pub fn sethostname(name: *const ::c_char, len: ::size_t) -> ::c_int; - pub fn uname(buf: *mut ::utsname) -> ::c_int; + ) -> ::c_int; + pub fn strsep(string: *mut *mut ::c_char, delimiters: *const ::c_char) -> *mut ::c_char; + pub fn explicit_bzero(buf: *mut ::c_void, len: ::size_t); + pub fn login_tty(_fd: ::c_int) -> ::c_int; } cfg_if! { @@ -1529,3 +2018,21 @@ cfg_if! { pub use self::b32::*; } } + +cfg_if! { + if #[cfg(target_arch = "x86")] { + // TODO + // mod x86; + // pub use self::x86::*; + } else if #[cfg(target_arch = "x86_64")] { + mod x86_64; + pub use self::x86_64::*; + } else if #[cfg(target_arch = "aarch64")] { + // TODO + // mod aarch64; + // pub use self::aarch64::*; + } +} + +mod native; +pub use self::native::*; diff --git a/crux-mir/lib/libc/src/unix/haiku/native.rs b/crux-mir/lib/libc/src/unix/haiku/native.rs new file mode 100644 index 000000000..44bcc1e3b --- /dev/null +++ b/crux-mir/lib/libc/src/unix/haiku/native.rs @@ -0,0 +1,1366 @@ +// This module contains bindings to the native Haiku API. The Haiku API +// originates from BeOS, and it was the original way to perform low level +// system and IO operations. The POSIX API was in that era was like a +// compatibility layer. In current Haiku development, both the POSIX API and +// the Haiku API are considered to be co-equal status. However, they are not +// integrated like they are on other UNIX platforms, which means that for many +// low level concepts there are two versions, like processes (POSIX) and +// teams (Haiku), or pthreads and native threads. +// +// Both the POSIX API and the Haiku API live in libroot.so, the library that is +// linked to any binary by default. +// +// This file follows the Haiku API for Haiku R1 beta 2. It is organized by the +// C/C++ header files in which the concepts can be found, while adhering to the +// style guide for this crate. + +// Helper macro to generate u32 constants. The Haiku API uses (non-standard) +// multi-character constants (like 'UPDA' or 'MSGM') to represent 32 bit +// integer constants. + +macro_rules! haiku_constant { + ($a:tt, $b:tt, $c:tt, $d:tt) => { + (($a as u32) << 24) + (($b as u32) << 16) + (($c as u32) << 8) + ($d as u32) + }; +} + +// support/SupportDefs.h +pub type status_t = i32; +pub type bigtime_t = i64; +pub type nanotime_t = i64; +pub type type_code = u32; +pub type perform_code = u32; + +// kernel/OS.h +pub type area_id = i32; +pub type port_id = i32; +pub type sem_id = i32; +pub type team_id = i32; +pub type thread_id = i32; + +pub type thread_func = extern "C" fn(*mut ::c_void) -> status_t; + +// kernel/image.h +pub type image_id = i32; + +e! { + // kernel/OS.h + pub enum thread_state { + B_THREAD_RUNNING = 1, + B_THREAD_READY, + B_THREAD_RECEIVING, + B_THREAD_ASLEEP, + B_THREAD_SUSPENDED, + B_THREAD_WAITING + } + + // kernel/image.h + pub enum image_type { + B_APP_IMAGE = 1, + B_LIBRARY_IMAGE, + B_ADD_ON_IMAGE, + B_SYSTEM_IMAGE + } + + // kernel/scheduler.h + + pub enum be_task_flags { + B_DEFAULT_MEDIA_PRIORITY = 0x000, + B_OFFLINE_PROCESSING = 0x001, + B_STATUS_RENDERING = 0x002, + B_USER_INPUT_HANDLING = 0x004, + B_LIVE_VIDEO_MANIPULATION = 0x008, + B_VIDEO_PLAYBACK = 0x010, + B_VIDEO_RECORDING = 0x020, + B_LIVE_AUDIO_MANIPULATION = 0x040, + B_AUDIO_PLAYBACK = 0x080, + B_AUDIO_RECORDING = 0x100, + B_LIVE_3D_RENDERING = 0x200, + B_NUMBER_CRUNCHING = 0x400, + B_MIDI_PROCESSING = 0x800, + } + + pub enum schduler_mode { + SCHEDULER_MODE_LOW_LATENCY, + SCHEDULER_MODE_POWER_SAVING, + } + + // FindDirectory.h + pub enum path_base_directory { + B_FIND_PATH_INSTALLATION_LOCATION_DIRECTORY, + B_FIND_PATH_ADD_ONS_DIRECTORY, + B_FIND_PATH_APPS_DIRECTORY, + B_FIND_PATH_BIN_DIRECTORY, + B_FIND_PATH_BOOT_DIRECTORY, + B_FIND_PATH_CACHE_DIRECTORY, + B_FIND_PATH_DATA_DIRECTORY, + B_FIND_PATH_DEVELOP_DIRECTORY, + B_FIND_PATH_DEVELOP_LIB_DIRECTORY, + B_FIND_PATH_DOCUMENTATION_DIRECTORY, + B_FIND_PATH_ETC_DIRECTORY, + B_FIND_PATH_FONTS_DIRECTORY, + B_FIND_PATH_HEADERS_DIRECTORY, + B_FIND_PATH_LIB_DIRECTORY, + B_FIND_PATH_LOG_DIRECTORY, + B_FIND_PATH_MEDIA_NODES_DIRECTORY, + B_FIND_PATH_PACKAGES_DIRECTORY, + B_FIND_PATH_PREFERENCES_DIRECTORY, + B_FIND_PATH_SERVERS_DIRECTORY, + B_FIND_PATH_SETTINGS_DIRECTORY, + B_FIND_PATH_SOUNDS_DIRECTORY, + B_FIND_PATH_SPOOL_DIRECTORY, + B_FIND_PATH_TRANSLATORS_DIRECTORY, + B_FIND_PATH_VAR_DIRECTORY, + B_FIND_PATH_IMAGE_PATH = 1000, + B_FIND_PATH_PACKAGE_PATH, + } + + pub enum directory_which { + B_DESKTOP_DIRECTORY = 0, + B_TRASH_DIRECTORY, + B_SYSTEM_DIRECTORY = 1000, + B_SYSTEM_ADDONS_DIRECTORY = 1002, + B_SYSTEM_BOOT_DIRECTORY, + B_SYSTEM_FONTS_DIRECTORY, + B_SYSTEM_LIB_DIRECTORY, + B_SYSTEM_SERVERS_DIRECTORY, + B_SYSTEM_APPS_DIRECTORY, + B_SYSTEM_BIN_DIRECTORY, + B_SYSTEM_DOCUMENTATION_DIRECTORY = 1010, + B_SYSTEM_PREFERENCES_DIRECTORY, + B_SYSTEM_TRANSLATORS_DIRECTORY, + B_SYSTEM_MEDIA_NODES_DIRECTORY, + B_SYSTEM_SOUNDS_DIRECTORY, + B_SYSTEM_DATA_DIRECTORY, + B_SYSTEM_DEVELOP_DIRECTORY, + B_SYSTEM_PACKAGES_DIRECTORY, + B_SYSTEM_HEADERS_DIRECTORY, + B_SYSTEM_ETC_DIRECTORY = 2008, + B_SYSTEM_SETTINGS_DIRECTORY = 2010, + B_SYSTEM_LOG_DIRECTORY = 2012, + B_SYSTEM_SPOOL_DIRECTORY, + B_SYSTEM_TEMP_DIRECTORY, + B_SYSTEM_VAR_DIRECTORY, + B_SYSTEM_CACHE_DIRECTORY = 2020, + B_SYSTEM_NONPACKAGED_DIRECTORY = 2023, + B_SYSTEM_NONPACKAGED_ADDONS_DIRECTORY, + B_SYSTEM_NONPACKAGED_TRANSLATORS_DIRECTORY, + B_SYSTEM_NONPACKAGED_MEDIA_NODES_DIRECTORY, + B_SYSTEM_NONPACKAGED_BIN_DIRECTORY, + B_SYSTEM_NONPACKAGED_DATA_DIRECTORY, + B_SYSTEM_NONPACKAGED_FONTS_DIRECTORY, + B_SYSTEM_NONPACKAGED_SOUNDS_DIRECTORY, + B_SYSTEM_NONPACKAGED_DOCUMENTATION_DIRECTORY, + B_SYSTEM_NONPACKAGED_LIB_DIRECTORY, + B_SYSTEM_NONPACKAGED_HEADERS_DIRECTORY, + B_SYSTEM_NONPACKAGED_DEVELOP_DIRECTORY, + B_USER_DIRECTORY = 3000, + B_USER_CONFIG_DIRECTORY, + B_USER_ADDONS_DIRECTORY, + B_USER_BOOT_DIRECTORY, + B_USER_FONTS_DIRECTORY, + B_USER_LIB_DIRECTORY, + B_USER_SETTINGS_DIRECTORY, + B_USER_DESKBAR_DIRECTORY, + B_USER_PRINTERS_DIRECTORY, + B_USER_TRANSLATORS_DIRECTORY, + B_USER_MEDIA_NODES_DIRECTORY, + B_USER_SOUNDS_DIRECTORY, + B_USER_DATA_DIRECTORY, + B_USER_CACHE_DIRECTORY, + B_USER_PACKAGES_DIRECTORY, + B_USER_HEADERS_DIRECTORY, + B_USER_NONPACKAGED_DIRECTORY, + B_USER_NONPACKAGED_ADDONS_DIRECTORY, + B_USER_NONPACKAGED_TRANSLATORS_DIRECTORY, + B_USER_NONPACKAGED_MEDIA_NODES_DIRECTORY, + B_USER_NONPACKAGED_BIN_DIRECTORY, + B_USER_NONPACKAGED_DATA_DIRECTORY, + B_USER_NONPACKAGED_FONTS_DIRECTORY, + B_USER_NONPACKAGED_SOUNDS_DIRECTORY, + B_USER_NONPACKAGED_DOCUMENTATION_DIRECTORY, + B_USER_NONPACKAGED_LIB_DIRECTORY, + B_USER_NONPACKAGED_HEADERS_DIRECTORY, + B_USER_NONPACKAGED_DEVELOP_DIRECTORY, + B_USER_DEVELOP_DIRECTORY, + B_USER_DOCUMENTATION_DIRECTORY, + B_USER_SERVERS_DIRECTORY, + B_USER_APPS_DIRECTORY, + B_USER_BIN_DIRECTORY, + B_USER_PREFERENCES_DIRECTORY, + B_USER_ETC_DIRECTORY, + B_USER_LOG_DIRECTORY, + B_USER_SPOOL_DIRECTORY, + B_USER_VAR_DIRECTORY, + B_APPS_DIRECTORY = 4000, + B_PREFERENCES_DIRECTORY, + B_UTILITIES_DIRECTORY, + B_PACKAGE_LINKS_DIRECTORY, + } +} + +s! { + // kernel/OS.h + pub struct area_info { + pub area: area_id, + pub name: [::c_char; B_OS_NAME_LENGTH], + pub size: usize, + pub lock: u32, + pub protection: u32, + pub team: team_id, + pub ram_size: u32, + pub copy_count: u32, + pub in_count: u32, + pub out_count: u32, + pub address: *mut ::c_void + } + + pub struct port_info { + pub port: port_id, + pub team: team_id, + pub name: [::c_char; B_OS_NAME_LENGTH], + pub capacity: i32, + pub queue_count: i32, + pub total_count: i32, + } + + pub struct port_message_info { + pub size: ::size_t, + pub sender: ::uid_t, + pub sender_group: ::gid_t, + pub sender_team: ::team_id + } + + pub struct team_info { + pub team: team_id, + pub thread_count: i32, + pub image_count: i32, + pub area_count: i32, + pub debugger_nub_thread: thread_id, + pub debugger_nub_port: port_id, + pub argc: i32, + pub args: [::c_char; 64], + pub uid: ::uid_t, + pub gid: ::gid_t + } + + pub struct sem_info { + pub sem: sem_id, + pub team: team_id, + pub name: [::c_char; B_OS_NAME_LENGTH], + pub count: i32, + pub latest_holder: thread_id + } + + pub struct team_usage_info { + pub user_time: bigtime_t, + pub kernel_time: bigtime_t + } + + pub struct thread_info { + pub thread: thread_id, + pub team: team_id, + pub name: [::c_char; B_OS_NAME_LENGTH], + pub state: thread_state, + pub priority: i32, + pub sem: sem_id, + pub user_time: bigtime_t, + pub kernel_time: bigtime_t, + pub stack_base: *mut ::c_void, + pub stack_end: *mut ::c_void + } + + pub struct cpu_info { + pub active_time: bigtime_t, + pub enabled: bool, + pub current_frequency: u64 + } + + pub struct system_info { + pub boot_time: bigtime_t, + pub cpu_count: u32, + pub max_pages: u64, + pub used_pages: u64, + pub cached_pages: u64, + pub block_cache_pages: u64, + pub ignored_pages: u64, + pub needed_memory: u64, + pub free_memory: u64, + pub max_swap_pages: u64, + pub free_swap_pages: u64, + pub page_faults: u32, + pub max_sems: u32, + pub used_sems: u32, + pub max_ports: u32, + pub used_ports: u32, + pub max_threads: u32, + pub used_threads: u32, + pub max_teams: u32, + pub used_teams: u32, + pub kernel_name: [::c_char; B_FILE_NAME_LENGTH], + pub kernel_build_date: [::c_char; B_OS_NAME_LENGTH], + pub kernel_build_time: [::c_char; B_OS_NAME_LENGTH], + pub kernel_version: i64, + pub abi: u32 + } + + pub struct object_wait_info { + pub object: i32, + pub type_: u16, + pub events: u16 + } + + // kernel/fs_attr.h + pub struct attr_info { + pub type_: u32, + pub size: ::off_t + } + + // kernel/fs_index.h + pub struct index_info { + pub type_: u32, + pub size: ::off_t, + pub modification_time: ::time_t, + pub creation_time: ::time_t, + pub uid: ::uid_t, + pub gid: ::gid_t + } + + //kernel/fs_info.h + pub struct fs_info { + pub dev: ::dev_t, + pub root: ::ino_t, + pub flags: u32, + pub block_size: ::off_t, + pub io_size: ::off_t, + pub total_blocks: ::off_t, + pub free_blocks: ::off_t, + pub total_nodes: ::off_t, + pub free_nodes: ::off_t, + pub device_name: [::c_char; 128], + pub volume_name: [::c_char; B_FILE_NAME_LENGTH], + pub fsh_name: [::c_char; B_OS_NAME_LENGTH] + } + + // kernel/image.h + pub struct image_info { + pub id: image_id, + pub image_type: ::c_int, + pub sequence: i32, + pub init_order: i32, + pub init_routine: extern "C" fn(), + pub term_routine: extern "C" fn(), + pub device: ::dev_t, + pub node: ::ino_t, + pub name: [::c_char; ::PATH_MAX as usize], + pub text: *mut ::c_void, + pub data: *mut ::c_void, + pub text_size: i32, + pub data_size: i32, + pub api_version: i32, + pub abi: i32 + } + + pub struct __c_anonymous_eax_0 { + pub max_eax: u32, + pub vendor_id: [::c_char; 12], + } + + pub struct __c_anonymous_eax_1 { + pub stepping: u32, + pub model: u32, + pub family: u32, + pub tpe: u32, + __reserved_0: u32, + pub extended_model: u32, + pub extended_family: u32, + __reserved_1: u32, + pub brand_index: u32, + pub clflush: u32, + pub logical_cpus: u32, + pub apic_id: u32, + pub features: u32, + pub extended_features: u32, + } + + pub struct __c_anonymous_eax_2 { + pub call_num: u8, + pub cache_descriptors: [u8; 15], + } + + pub struct __c_anonymous_eax_3 { + __reserved: [u32; 2], + pub serial_number_high: u32, + pub serial_number_low: u32, + } + + pub struct __c_anonymous_regs { + pub eax: u32, + pub ebx: u32, + pub edx: u32, + pub ecx: u32, + } +} + +s_no_extra_traits! { + #[cfg(libc_union)] + pub union cpuid_info { + pub eax_0: __c_anonymous_eax_0, + pub eax_1: __c_anonymous_eax_1, + pub eax_2: __c_anonymous_eax_2, + pub eax_3: __c_anonymous_eax_3, + pub as_chars: [::c_char; 16], + pub regs: __c_anonymous_regs, + } +} + +cfg_if! { + if #[cfg(feature = "extra_traits")] { + #[cfg(libc_union)] + impl PartialEq for cpuid_info { + fn eq(&self, other: &cpuid_info) -> bool { + unsafe { + self.eax_0 == other.eax_0 + || self.eax_1 == other.eax_1 + || self.eax_2 == other.eax_2 + || self.eax_3 == other.eax_3 + || self.as_chars == other.as_chars + || self.regs == other.regs + } + } + } + #[cfg(libc_union)] + impl Eq for cpuid_info {} + #[cfg(libc_union)] + impl ::fmt::Debug for cpuid_info { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + unsafe { + f.debug_struct("cpuid_info") + .field("eax_0", &self.eax_0) + .field("eax_1", &self.eax_1) + .field("eax_2", &self.eax_2) + .field("eax_3", &self.eax_3) + .field("as_chars", &self.as_chars) + .field("regs", &self.regs) + .finish() + } + } + } + } +} + +// kernel/OS.h +pub const B_OS_NAME_LENGTH: usize = 32; +pub const B_PAGE_SIZE: usize = 4096; +pub const B_INFINITE_TIMEOUT: usize = 9223372036854775807; + +pub const B_RELATIVE_TIMEOUT: u32 = 0x8; +pub const B_ABSOLUTE_TIMEOUT: u32 = 0x10; +pub const B_TIMEOUT_REAL_TIME_BASE: u32 = 0x40; +pub const B_ABSOLUTE_REAL_TIME_TIMEOUT: u32 = B_ABSOLUTE_TIMEOUT | B_TIMEOUT_REAL_TIME_BASE; + +pub const B_NO_LOCK: u32 = 0; +pub const B_LAZY_LOCK: u32 = 1; +pub const B_FULL_LOCK: u32 = 2; +pub const B_CONTIGUOUS: u32 = 3; +pub const B_LOMEM: u32 = 4; +pub const B_32_BIT_FULL_LOCK: u32 = 5; +pub const B_32_BIT_CONTIGUOUS: u32 = 6; + +pub const B_ANY_ADDRESS: u32 = 0; +pub const B_EXACT_ADDRESS: u32 = 1; +pub const B_BASE_ADDRESS: u32 = 2; +pub const B_CLONE_ADDRESS: u32 = 3; +pub const B_ANY_KERNEL_ADDRESS: u32 = 4; +pub const B_RANDOMIZED_ANY_ADDRESS: u32 = 6; +pub const B_RANDOMIZED_BASE_ADDRESS: u32 = 7; + +pub const B_READ_AREA: u32 = 1 << 0; +pub const B_WRITE_AREA: u32 = 1 << 1; +pub const B_EXECUTE_AREA: u32 = 1 << 2; +pub const B_STACK_AREA: u32 = 1 << 3; +pub const B_CLONEABLE_AREA: u32 = 1 << 8; + +pub const B_CAN_INTERRUPT: u32 = 0x01; +pub const B_CHECK_PERMISSION: u32 = 0x04; +pub const B_KILL_CAN_INTERRUPT: u32 = 0x20; +pub const B_DO_NOT_RESCHEDULE: u32 = 0x02; +pub const B_RELEASE_ALL: u32 = 0x08; +pub const B_RELEASE_IF_WAITING_ONLY: u32 = 0x10; + +pub const B_CURRENT_TEAM: team_id = 0; +pub const B_SYSTEM_TEAM: team_id = 1; + +pub const B_TEAM_USAGE_SELF: i32 = 0; +pub const B_TEAM_USAGE_CHILDREN: i32 = -1; + +pub const B_IDLE_PRIORITY: i32 = 0; +pub const B_LOWEST_ACTIVE_PRIORITY: i32 = 1; +pub const B_LOW_PRIORITY: i32 = 5; +pub const B_NORMAL_PRIORITY: i32 = 10; +pub const B_DISPLAY_PRIORITY: i32 = 15; +pub const B_URGENT_DISPLAY_PRIORITY: i32 = 20; +pub const B_REAL_TIME_DISPLAY_PRIORITY: i32 = 100; +pub const B_URGENT_PRIORITY: i32 = 110; +pub const B_REAL_TIME_PRIORITY: i32 = 120; + +pub const B_SYSTEM_TIMEBASE: i32 = 0; +pub const B_FIRST_REAL_TIME_PRIORITY: i32 = B_REAL_TIME_DISPLAY_PRIORITY; + +pub const B_ONE_SHOT_ABSOLUTE_ALARM: u32 = 1; +pub const B_ONE_SHOT_RELATIVE_ALARM: u32 = 2; +pub const B_PERIODIC_ALARM: u32 = 3; + +pub const B_OBJECT_TYPE_FD: u16 = 0; +pub const B_OBJECT_TYPE_SEMAPHORE: u16 = 1; +pub const B_OBJECT_TYPE_PORT: u16 = 2; +pub const B_OBJECT_TYPE_THREAD: u16 = 3; + +pub const B_EVENT_READ: u16 = 0x0001; +pub const B_EVENT_WRITE: u16 = 0x0002; +pub const B_EVENT_ERROR: u16 = 0x0004; +pub const B_EVENT_PRIORITY_READ: u16 = 0x0008; +pub const B_EVENT_PRIORITY_WRITE: u16 = 0x0010; +pub const B_EVENT_HIGH_PRIORITY_READ: u16 = 0x0020; +pub const B_EVENT_HIGH_PRIORITY_WRITE: u16 = 0x0040; +pub const B_EVENT_DISCONNECTED: u16 = 0x0080; +pub const B_EVENT_ACQUIRE_SEMAPHORE: u16 = 0x0001; +pub const B_EVENT_INVALID: u16 = 0x1000; + +// kernel/fs_info.h +pub const B_FS_IS_READONLY: u32 = 0x00000001; +pub const B_FS_IS_REMOVABLE: u32 = 0x00000002; +pub const B_FS_IS_PERSISTENT: u32 = 0x00000004; +pub const B_FS_IS_SHARED: u32 = 0x00000008; +pub const B_FS_HAS_MIME: u32 = 0x00010000; +pub const B_FS_HAS_ATTR: u32 = 0x00020000; +pub const B_FS_HAS_QUERY: u32 = 0x00040000; +pub const B_FS_HAS_SELF_HEALING_LINKS: u32 = 0x00080000; +pub const B_FS_HAS_ALIASES: u32 = 0x00100000; +pub const B_FS_SUPPORTS_NODE_MONITORING: u32 = 0x00200000; +pub const B_FS_SUPPORTS_MONITOR_CHILDREN: u32 = 0x00400000; + +// kernel/fs_query.h +pub const B_LIVE_QUERY: u32 = 0x00000001; +pub const B_QUERY_NON_INDEXED: u32 = 0x00000002; + +// kernel/fs_volume.h +pub const B_MOUNT_READ_ONLY: u32 = 1; +pub const B_MOUNT_VIRTUAL_DEVICE: u32 = 2; +pub const B_FORCE_UNMOUNT: u32 = 1; + +// kernel/image.h +pub const B_FLUSH_DCACHE: u32 = 0x0001; +pub const B_FLUSH_ICACHE: u32 = 0x0004; +pub const B_INVALIDATE_DCACHE: u32 = 0x0002; +pub const B_INVALIDATE_ICACHE: u32 = 0x0008; + +pub const B_SYMBOL_TYPE_DATA: i32 = 0x1; +pub const B_SYMBOL_TYPE_TEXT: i32 = 0x2; +pub const B_SYMBOL_TYPE_ANY: i32 = 0x5; + +// storage/StorageDefs.h +pub const B_DEV_NAME_LENGTH: usize = 128; +pub const B_FILE_NAME_LENGTH: usize = ::FILENAME_MAX as usize; +pub const B_PATH_NAME_LENGTH: usize = ::PATH_MAX as usize; +pub const B_ATTR_NAME_LENGTH: usize = B_FILE_NAME_LENGTH - 1; +pub const B_MIME_TYPE_LENGTH: usize = B_ATTR_NAME_LENGTH - 15; +pub const B_MAX_SYMLINKS: usize = 16; + +// Haiku open modes in BFile are passed as u32 +pub const B_READ_ONLY: u32 = ::O_RDONLY as u32; +pub const B_WRITE_ONLY: u32 = ::O_WRONLY as u32; +pub const B_READ_WRITE: u32 = ::O_RDWR as u32; + +pub const B_FAIL_IF_EXISTS: u32 = ::O_EXCL as u32; +pub const B_CREATE_FILE: u32 = ::O_CREAT as u32; +pub const B_ERASE_FILE: u32 = ::O_TRUNC as u32; +pub const B_OPEN_AT_END: u32 = ::O_APPEND as u32; + +pub const B_FILE_NODE: u32 = 0x01; +pub const B_SYMLINK_NODE: u32 = 0x02; +pub const B_DIRECTORY_NODE: u32 = 0x04; +pub const B_ANY_NODE: u32 = 0x07; + +// support/Errors.h +pub const B_GENERAL_ERROR_BASE: status_t = core::i32::MIN; +pub const B_OS_ERROR_BASE: status_t = B_GENERAL_ERROR_BASE + 0x1000; +pub const B_APP_ERROR_BASE: status_t = B_GENERAL_ERROR_BASE + 0x2000; +pub const B_INTERFACE_ERROR_BASE: status_t = B_GENERAL_ERROR_BASE + 0x3000; +pub const B_MEDIA_ERROR_BASE: status_t = B_GENERAL_ERROR_BASE + 0x4000; +pub const B_TRANSLATION_ERROR_BASE: status_t = B_GENERAL_ERROR_BASE + 0x4800; +pub const B_MIDI_ERROR_BASE: status_t = B_GENERAL_ERROR_BASE + 0x5000; +pub const B_STORAGE_ERROR_BASE: status_t = B_GENERAL_ERROR_BASE + 0x6000; +pub const B_POSIX_ERROR_BASE: status_t = B_GENERAL_ERROR_BASE + 0x7000; +pub const B_MAIL_ERROR_BASE: status_t = B_GENERAL_ERROR_BASE + 0x8000; +pub const B_PRINT_ERROR_BASE: status_t = B_GENERAL_ERROR_BASE + 0x9000; +pub const B_DEVICE_ERROR_BASE: status_t = B_GENERAL_ERROR_BASE + 0xa000; +pub const B_ERRORS_END: status_t = B_GENERAL_ERROR_BASE + 0xffff; + +// General errors +pub const B_NO_MEMORY: status_t = B_GENERAL_ERROR_BASE + 0; +pub const B_IO_ERROR: status_t = B_GENERAL_ERROR_BASE + 1; +pub const B_PERMISSION_DENIED: status_t = B_GENERAL_ERROR_BASE + 2; +pub const B_BAD_INDEX: status_t = B_GENERAL_ERROR_BASE + 3; +pub const B_BAD_TYPE: status_t = B_GENERAL_ERROR_BASE + 4; +pub const B_BAD_VALUE: status_t = B_GENERAL_ERROR_BASE + 5; +pub const B_MISMATCHED_VALUES: status_t = B_GENERAL_ERROR_BASE + 6; +pub const B_NAME_NOT_FOUND: status_t = B_GENERAL_ERROR_BASE + 7; +pub const B_NAME_IN_USE: status_t = B_GENERAL_ERROR_BASE + 8; +pub const B_TIMED_OUT: status_t = B_GENERAL_ERROR_BASE + 9; +pub const B_INTERRUPTED: status_t = B_GENERAL_ERROR_BASE + 10; +pub const B_WOULD_BLOCK: status_t = B_GENERAL_ERROR_BASE + 11; +pub const B_CANCELED: status_t = B_GENERAL_ERROR_BASE + 12; +pub const B_NO_INIT: status_t = B_GENERAL_ERROR_BASE + 13; +pub const B_NOT_INITIALIZED: status_t = B_GENERAL_ERROR_BASE + 13; +pub const B_BUSY: status_t = B_GENERAL_ERROR_BASE + 14; +pub const B_NOT_ALLOWED: status_t = B_GENERAL_ERROR_BASE + 15; +pub const B_BAD_DATA: status_t = B_GENERAL_ERROR_BASE + 16; +pub const B_DONT_DO_THAT: status_t = B_GENERAL_ERROR_BASE + 17; + +pub const B_ERROR: status_t = -1; +pub const B_OK: status_t = 0; +pub const B_NO_ERROR: status_t = 0; + +// Kernel kit errors +pub const B_BAD_SEM_ID: status_t = B_OS_ERROR_BASE + 0; +pub const B_NO_MORE_SEMS: status_t = B_OS_ERROR_BASE + 1; + +pub const B_BAD_THREAD_ID: status_t = B_OS_ERROR_BASE + 0x100; +pub const B_NO_MORE_THREADS: status_t = B_OS_ERROR_BASE + 0x101; +pub const B_BAD_THREAD_STATE: status_t = B_OS_ERROR_BASE + 0x102; +pub const B_BAD_TEAM_ID: status_t = B_OS_ERROR_BASE + 0x103; +pub const B_NO_MORE_TEAMS: status_t = B_OS_ERROR_BASE + 0x104; + +pub const B_BAD_PORT_ID: status_t = B_OS_ERROR_BASE + 0x200; +pub const B_NO_MORE_PORTS: status_t = B_OS_ERROR_BASE + 0x201; + +pub const B_BAD_IMAGE_ID: status_t = B_OS_ERROR_BASE + 0x300; +pub const B_BAD_ADDRESS: status_t = B_OS_ERROR_BASE + 0x301; +pub const B_NOT_AN_EXECUTABLE: status_t = B_OS_ERROR_BASE + 0x302; +pub const B_MISSING_LIBRARY: status_t = B_OS_ERROR_BASE + 0x303; +pub const B_MISSING_SYMBOL: status_t = B_OS_ERROR_BASE + 0x304; +pub const B_UNKNOWN_EXECUTABLE: status_t = B_OS_ERROR_BASE + 0x305; +pub const B_LEGACY_EXECUTABLE: status_t = B_OS_ERROR_BASE + 0x306; + +pub const B_DEBUGGER_ALREADY_INSTALLED: status_t = B_OS_ERROR_BASE + 0x400; + +// Application kit errors +pub const B_BAD_REPLY: status_t = B_APP_ERROR_BASE + 0; +pub const B_DUPLICATE_REPLY: status_t = B_APP_ERROR_BASE + 1; +pub const B_MESSAGE_TO_SELF: status_t = B_APP_ERROR_BASE + 2; +pub const B_BAD_HANDLER: status_t = B_APP_ERROR_BASE + 3; +pub const B_ALREADY_RUNNING: status_t = B_APP_ERROR_BASE + 4; +pub const B_LAUNCH_FAILED: status_t = B_APP_ERROR_BASE + 5; +pub const B_AMBIGUOUS_APP_LAUNCH: status_t = B_APP_ERROR_BASE + 6; +pub const B_UNKNOWN_MIME_TYPE: status_t = B_APP_ERROR_BASE + 7; +pub const B_BAD_SCRIPT_SYNTAX: status_t = B_APP_ERROR_BASE + 8; +pub const B_LAUNCH_FAILED_NO_RESOLVE_LINK: status_t = B_APP_ERROR_BASE + 9; +pub const B_LAUNCH_FAILED_EXECUTABLE: status_t = B_APP_ERROR_BASE + 10; +pub const B_LAUNCH_FAILED_APP_NOT_FOUND: status_t = B_APP_ERROR_BASE + 11; +pub const B_LAUNCH_FAILED_APP_IN_TRASH: status_t = B_APP_ERROR_BASE + 12; +pub const B_LAUNCH_FAILED_NO_PREFERRED_APP: status_t = B_APP_ERROR_BASE + 13; +pub const B_LAUNCH_FAILED_FILES_APP_NOT_FOUND: status_t = B_APP_ERROR_BASE + 14; +pub const B_BAD_MIME_SNIFFER_RULE: status_t = B_APP_ERROR_BASE + 15; +pub const B_NOT_A_MESSAGE: status_t = B_APP_ERROR_BASE + 16; +pub const B_SHUTDOWN_CANCELLED: status_t = B_APP_ERROR_BASE + 17; +pub const B_SHUTTING_DOWN: status_t = B_APP_ERROR_BASE + 18; + +// Storage kit errors +pub const B_FILE_ERROR: status_t = B_STORAGE_ERROR_BASE + 0; +pub const B_FILE_EXISTS: status_t = B_STORAGE_ERROR_BASE + 2; +pub const B_ENTRY_NOT_FOUND: status_t = B_STORAGE_ERROR_BASE + 3; +pub const B_NAME_TOO_LONG: status_t = B_STORAGE_ERROR_BASE + 4; +pub const B_NOT_A_DIRECTORY: status_t = B_STORAGE_ERROR_BASE + 5; +pub const B_DIRECTORY_NOT_EMPTY: status_t = B_STORAGE_ERROR_BASE + 6; +pub const B_DEVICE_FULL: status_t = B_STORAGE_ERROR_BASE + 7; +pub const B_READ_ONLY_DEVICE: status_t = B_STORAGE_ERROR_BASE + 8; +pub const B_IS_A_DIRECTORY: status_t = B_STORAGE_ERROR_BASE + 9; +pub const B_NO_MORE_FDS: status_t = B_STORAGE_ERROR_BASE + 10; +pub const B_CROSS_DEVICE_LINK: status_t = B_STORAGE_ERROR_BASE + 11; +pub const B_LINK_LIMIT: status_t = B_STORAGE_ERROR_BASE + 12; +pub const B_BUSTED_PIPE: status_t = B_STORAGE_ERROR_BASE + 13; +pub const B_UNSUPPORTED: status_t = B_STORAGE_ERROR_BASE + 14; +pub const B_PARTITION_TOO_SMALL: status_t = B_STORAGE_ERROR_BASE + 15; +pub const B_PARTIAL_READ: status_t = B_STORAGE_ERROR_BASE + 16; +pub const B_PARTIAL_WRITE: status_t = B_STORAGE_ERROR_BASE + 17; + +// Mapped posix errors +pub const B_BUFFER_OVERFLOW: status_t = ::EOVERFLOW; +pub const B_TOO_MANY_ARGS: status_t = ::E2BIG; +pub const B_FILE_TOO_LARGE: status_t = ::EFBIG; +pub const B_RESULT_NOT_REPRESENTABLE: status_t = ::ERANGE; +pub const B_DEVICE_NOT_FOUND: status_t = ::ENODEV; +pub const B_NOT_SUPPORTED: status_t = ::EOPNOTSUPP; + +// Media kit errors +pub const B_STREAM_NOT_FOUND: status_t = B_MEDIA_ERROR_BASE + 0; +pub const B_SERVER_NOT_FOUND: status_t = B_MEDIA_ERROR_BASE + 1; +pub const B_RESOURCE_NOT_FOUND: status_t = B_MEDIA_ERROR_BASE + 2; +pub const B_RESOURCE_UNAVAILABLE: status_t = B_MEDIA_ERROR_BASE + 3; +pub const B_BAD_SUBSCRIBER: status_t = B_MEDIA_ERROR_BASE + 4; +pub const B_SUBSCRIBER_NOT_ENTERED: status_t = B_MEDIA_ERROR_BASE + 5; +pub const B_BUFFER_NOT_AVAILABLE: status_t = B_MEDIA_ERROR_BASE + 6; +pub const B_LAST_BUFFER_ERROR: status_t = B_MEDIA_ERROR_BASE + 7; + +pub const B_MEDIA_SYSTEM_FAILURE: status_t = B_MEDIA_ERROR_BASE + 100; +pub const B_MEDIA_BAD_NODE: status_t = B_MEDIA_ERROR_BASE + 101; +pub const B_MEDIA_NODE_BUSY: status_t = B_MEDIA_ERROR_BASE + 102; +pub const B_MEDIA_BAD_FORMAT: status_t = B_MEDIA_ERROR_BASE + 103; +pub const B_MEDIA_BAD_BUFFER: status_t = B_MEDIA_ERROR_BASE + 104; +pub const B_MEDIA_TOO_MANY_NODES: status_t = B_MEDIA_ERROR_BASE + 105; +pub const B_MEDIA_TOO_MANY_BUFFERS: status_t = B_MEDIA_ERROR_BASE + 106; +pub const B_MEDIA_NODE_ALREADY_EXISTS: status_t = B_MEDIA_ERROR_BASE + 107; +pub const B_MEDIA_BUFFER_ALREADY_EXISTS: status_t = B_MEDIA_ERROR_BASE + 108; +pub const B_MEDIA_CANNOT_SEEK: status_t = B_MEDIA_ERROR_BASE + 109; +pub const B_MEDIA_CANNOT_CHANGE_RUN_MODE: status_t = B_MEDIA_ERROR_BASE + 110; +pub const B_MEDIA_APP_ALREADY_REGISTERED: status_t = B_MEDIA_ERROR_BASE + 111; +pub const B_MEDIA_APP_NOT_REGISTERED: status_t = B_MEDIA_ERROR_BASE + 112; +pub const B_MEDIA_CANNOT_RECLAIM_BUFFERS: status_t = B_MEDIA_ERROR_BASE + 113; +pub const B_MEDIA_BUFFERS_NOT_RECLAIMED: status_t = B_MEDIA_ERROR_BASE + 114; +pub const B_MEDIA_TIME_SOURCE_STOPPED: status_t = B_MEDIA_ERROR_BASE + 115; +pub const B_MEDIA_TIME_SOURCE_BUSY: status_t = B_MEDIA_ERROR_BASE + 116; +pub const B_MEDIA_BAD_SOURCE: status_t = B_MEDIA_ERROR_BASE + 117; +pub const B_MEDIA_BAD_DESTINATION: status_t = B_MEDIA_ERROR_BASE + 118; +pub const B_MEDIA_ALREADY_CONNECTED: status_t = B_MEDIA_ERROR_BASE + 119; +pub const B_MEDIA_NOT_CONNECTED: status_t = B_MEDIA_ERROR_BASE + 120; +pub const B_MEDIA_BAD_CLIP_FORMAT: status_t = B_MEDIA_ERROR_BASE + 121; +pub const B_MEDIA_ADDON_FAILED: status_t = B_MEDIA_ERROR_BASE + 122; +pub const B_MEDIA_ADDON_DISABLED: status_t = B_MEDIA_ERROR_BASE + 123; +pub const B_MEDIA_CHANGE_IN_PROGRESS: status_t = B_MEDIA_ERROR_BASE + 124; +pub const B_MEDIA_STALE_CHANGE_COUNT: status_t = B_MEDIA_ERROR_BASE + 125; +pub const B_MEDIA_ADDON_RESTRICTED: status_t = B_MEDIA_ERROR_BASE + 126; +pub const B_MEDIA_NO_HANDLER: status_t = B_MEDIA_ERROR_BASE + 127; +pub const B_MEDIA_DUPLICATE_FORMAT: status_t = B_MEDIA_ERROR_BASE + 128; +pub const B_MEDIA_REALTIME_DISABLED: status_t = B_MEDIA_ERROR_BASE + 129; +pub const B_MEDIA_REALTIME_UNAVAILABLE: status_t = B_MEDIA_ERROR_BASE + 130; + +// Mail kit errors +pub const B_MAIL_NO_DAEMON: status_t = B_MAIL_ERROR_BASE + 0; +pub const B_MAIL_UNKNOWN_USER: status_t = B_MAIL_ERROR_BASE + 1; +pub const B_MAIL_WRONG_PASSWORD: status_t = B_MAIL_ERROR_BASE + 2; +pub const B_MAIL_UNKNOWN_HOST: status_t = B_MAIL_ERROR_BASE + 3; +pub const B_MAIL_ACCESS_ERROR: status_t = B_MAIL_ERROR_BASE + 4; +pub const B_MAIL_UNKNOWN_FIELD: status_t = B_MAIL_ERROR_BASE + 5; +pub const B_MAIL_NO_RECIPIENT: status_t = B_MAIL_ERROR_BASE + 6; +pub const B_MAIL_INVALID_MAIL: status_t = B_MAIL_ERROR_BASE + 7; + +// Print kit errors +pub const B_NO_PRINT_SERVER: status_t = B_PRINT_ERROR_BASE + 0; + +// Device kit errors +pub const B_DEV_INVALID_IOCTL: status_t = B_DEVICE_ERROR_BASE + 0; +pub const B_DEV_NO_MEMORY: status_t = B_DEVICE_ERROR_BASE + 1; +pub const B_DEV_BAD_DRIVE_NUM: status_t = B_DEVICE_ERROR_BASE + 2; +pub const B_DEV_NO_MEDIA: status_t = B_DEVICE_ERROR_BASE + 3; +pub const B_DEV_UNREADABLE: status_t = B_DEVICE_ERROR_BASE + 4; +pub const B_DEV_FORMAT_ERROR: status_t = B_DEVICE_ERROR_BASE + 5; +pub const B_DEV_TIMEOUT: status_t = B_DEVICE_ERROR_BASE + 6; +pub const B_DEV_RECALIBRATE_ERROR: status_t = B_DEVICE_ERROR_BASE + 7; +pub const B_DEV_SEEK_ERROR: status_t = B_DEVICE_ERROR_BASE + 8; +pub const B_DEV_ID_ERROR: status_t = B_DEVICE_ERROR_BASE + 9; +pub const B_DEV_READ_ERROR: status_t = B_DEVICE_ERROR_BASE + 10; +pub const B_DEV_WRITE_ERROR: status_t = B_DEVICE_ERROR_BASE + 11; +pub const B_DEV_NOT_READY: status_t = B_DEVICE_ERROR_BASE + 12; +pub const B_DEV_MEDIA_CHANGED: status_t = B_DEVICE_ERROR_BASE + 13; +pub const B_DEV_MEDIA_CHANGE_REQUESTED: status_t = B_DEVICE_ERROR_BASE + 14; +pub const B_DEV_RESOURCE_CONFLICT: status_t = B_DEVICE_ERROR_BASE + 15; +pub const B_DEV_CONFIGURATION_ERROR: status_t = B_DEVICE_ERROR_BASE + 16; +pub const B_DEV_DISABLED_BY_USER: status_t = B_DEVICE_ERROR_BASE + 17; +pub const B_DEV_DOOR_OPEN: status_t = B_DEVICE_ERROR_BASE + 18; + +pub const B_DEV_INVALID_PIPE: status_t = B_DEVICE_ERROR_BASE + 19; +pub const B_DEV_CRC_ERROR: status_t = B_DEVICE_ERROR_BASE + 20; +pub const B_DEV_STALLED: status_t = B_DEVICE_ERROR_BASE + 21; +pub const B_DEV_BAD_PID: status_t = B_DEVICE_ERROR_BASE + 22; +pub const B_DEV_UNEXPECTED_PID: status_t = B_DEVICE_ERROR_BASE + 23; +pub const B_DEV_DATA_OVERRUN: status_t = B_DEVICE_ERROR_BASE + 24; +pub const B_DEV_DATA_UNDERRUN: status_t = B_DEVICE_ERROR_BASE + 25; +pub const B_DEV_FIFO_OVERRUN: status_t = B_DEVICE_ERROR_BASE + 26; +pub const B_DEV_FIFO_UNDERRUN: status_t = B_DEVICE_ERROR_BASE + 27; +pub const B_DEV_PENDING: status_t = B_DEVICE_ERROR_BASE + 28; +pub const B_DEV_MULTIPLE_ERRORS: status_t = B_DEVICE_ERROR_BASE + 29; +pub const B_DEV_TOO_LATE: status_t = B_DEVICE_ERROR_BASE + 30; + +// translation kit errors +pub const B_TRANSLATION_BASE_ERROR: status_t = B_TRANSLATION_ERROR_BASE + 0; +pub const B_NO_TRANSLATOR: status_t = B_TRANSLATION_ERROR_BASE + 1; +pub const B_ILLEGAL_DATA: status_t = B_TRANSLATION_ERROR_BASE + 2; + +// support/TypeConstants.h +pub const B_AFFINE_TRANSFORM_TYPE: u32 = haiku_constant!('A', 'M', 'T', 'X'); +pub const B_ALIGNMENT_TYPE: u32 = haiku_constant!('A', 'L', 'G', 'N'); +pub const B_ANY_TYPE: u32 = haiku_constant!('A', 'N', 'Y', 'T'); +pub const B_ATOM_TYPE: u32 = haiku_constant!('A', 'T', 'O', 'M'); +pub const B_ATOMREF_TYPE: u32 = haiku_constant!('A', 'T', 'M', 'R'); +pub const B_BOOL_TYPE: u32 = haiku_constant!('B', 'O', 'O', 'L'); +pub const B_CHAR_TYPE: u32 = haiku_constant!('C', 'H', 'A', 'R'); +pub const B_COLOR_8_BIT_TYPE: u32 = haiku_constant!('C', 'L', 'R', 'B'); +pub const B_DOUBLE_TYPE: u32 = haiku_constant!('D', 'B', 'L', 'E'); +pub const B_FLOAT_TYPE: u32 = haiku_constant!('F', 'L', 'O', 'T'); +pub const B_GRAYSCALE_8_BIT_TYPE: u32 = haiku_constant!('G', 'R', 'Y', 'B'); +pub const B_INT16_TYPE: u32 = haiku_constant!('S', 'H', 'R', 'T'); +pub const B_INT32_TYPE: u32 = haiku_constant!('L', 'O', 'N', 'G'); +pub const B_INT64_TYPE: u32 = haiku_constant!('L', 'L', 'N', 'G'); +pub const B_INT8_TYPE: u32 = haiku_constant!('B', 'Y', 'T', 'E'); +pub const B_LARGE_ICON_TYPE: u32 = haiku_constant!('I', 'C', 'O', 'N'); +pub const B_MEDIA_PARAMETER_GROUP_TYPE: u32 = haiku_constant!('B', 'M', 'C', 'G'); +pub const B_MEDIA_PARAMETER_TYPE: u32 = haiku_constant!('B', 'M', 'C', 'T'); +pub const B_MEDIA_PARAMETER_WEB_TYPE: u32 = haiku_constant!('B', 'M', 'C', 'W'); +pub const B_MESSAGE_TYPE: u32 = haiku_constant!('M', 'S', 'G', 'G'); +pub const B_MESSENGER_TYPE: u32 = haiku_constant!('M', 'S', 'N', 'G'); +pub const B_MIME_TYPE: u32 = haiku_constant!('M', 'I', 'M', 'E'); +pub const B_MINI_ICON_TYPE: u32 = haiku_constant!('M', 'I', 'C', 'N'); +pub const B_MONOCHROME_1_BIT_TYPE: u32 = haiku_constant!('M', 'N', 'O', 'B'); +pub const B_OBJECT_TYPE: u32 = haiku_constant!('O', 'P', 'T', 'R'); +pub const B_OFF_T_TYPE: u32 = haiku_constant!('O', 'F', 'F', 'T'); +pub const B_PATTERN_TYPE: u32 = haiku_constant!('P', 'A', 'T', 'N'); +pub const B_POINTER_TYPE: u32 = haiku_constant!('P', 'N', 'T', 'R'); +pub const B_POINT_TYPE: u32 = haiku_constant!('B', 'P', 'N', 'T'); +pub const B_PROPERTY_INFO_TYPE: u32 = haiku_constant!('S', 'C', 'T', 'D'); +pub const B_RAW_TYPE: u32 = haiku_constant!('R', 'A', 'W', 'T'); +pub const B_RECT_TYPE: u32 = haiku_constant!('R', 'E', 'C', 'T'); +pub const B_REF_TYPE: u32 = haiku_constant!('R', 'R', 'E', 'F'); +pub const B_RGB_32_BIT_TYPE: u32 = haiku_constant!('R', 'G', 'B', 'B'); +pub const B_RGB_COLOR_TYPE: u32 = haiku_constant!('R', 'G', 'B', 'C'); +pub const B_SIZE_TYPE: u32 = haiku_constant!('S', 'I', 'Z', 'E'); +pub const B_SIZE_T_TYPE: u32 = haiku_constant!('S', 'I', 'Z', 'T'); +pub const B_SSIZE_T_TYPE: u32 = haiku_constant!('S', 'S', 'Z', 'T'); +pub const B_STRING_TYPE: u32 = haiku_constant!('C', 'S', 'T', 'R'); +pub const B_STRING_LIST_TYPE: u32 = haiku_constant!('S', 'T', 'R', 'L'); +pub const B_TIME_TYPE: u32 = haiku_constant!('T', 'I', 'M', 'E'); +pub const B_UINT16_TYPE: u32 = haiku_constant!('U', 'S', 'H', 'T'); +pub const B_UINT32_TYPE: u32 = haiku_constant!('U', 'L', 'N', 'G'); +pub const B_UINT64_TYPE: u32 = haiku_constant!('U', 'L', 'L', 'G'); +pub const B_UINT8_TYPE: u32 = haiku_constant!('U', 'B', 'Y', 'T'); +pub const B_VECTOR_ICON_TYPE: u32 = haiku_constant!('V', 'I', 'C', 'N'); +pub const B_XATTR_TYPE: u32 = haiku_constant!('X', 'A', 'T', 'R'); +pub const B_NETWORK_ADDRESS_TYPE: u32 = haiku_constant!('N', 'W', 'A', 'D'); +pub const B_MIME_STRING_TYPE: u32 = haiku_constant!('M', 'I', 'M', 'S'); +pub const B_ASCII_TYPE: u32 = haiku_constant!('T', 'E', 'X', 'T'); + +extern "C" { + // kernel/OS.h + pub fn create_area( + name: *const ::c_char, + startAddress: *mut *mut ::c_void, + addressSpec: u32, + size: usize, + lock: u32, + protection: u32, + ) -> area_id; + pub fn clone_area( + name: *const ::c_char, + destAddress: *mut *mut ::c_void, + addressSpec: u32, + protection: u32, + source: area_id, + ) -> area_id; + pub fn find_area(name: *const ::c_char) -> area_id; + pub fn area_for(address: *mut ::c_void) -> area_id; + pub fn delete_area(id: area_id) -> status_t; + pub fn resize_area(id: area_id, newSize: usize) -> status_t; + pub fn set_area_protection(id: area_id, newProtection: u32) -> status_t; + pub fn _get_area_info(id: area_id, areaInfo: *mut area_info, size: usize) -> status_t; + pub fn _get_next_area_info( + team: team_id, + cookie: *mut isize, + areaInfo: *mut area_info, + size: usize, + ) -> status_t; + + pub fn create_port(capacity: i32, name: *const ::c_char) -> port_id; + pub fn find_port(name: *const ::c_char) -> port_id; + pub fn read_port( + port: port_id, + code: *mut i32, + buffer: *mut ::c_void, + bufferSize: ::size_t, + ) -> ::ssize_t; + pub fn read_port_etc( + port: port_id, + code: *mut i32, + buffer: *mut ::c_void, + bufferSize: ::size_t, + flags: u32, + timeout: bigtime_t, + ) -> ::ssize_t; + pub fn write_port( + port: port_id, + code: i32, + buffer: *const ::c_void, + bufferSize: ::size_t, + ) -> status_t; + pub fn write_port_etc( + port: port_id, + code: i32, + buffer: *const ::c_void, + bufferSize: ::size_t, + flags: u32, + timeout: bigtime_t, + ) -> status_t; + pub fn close_port(port: port_id) -> status_t; + pub fn delete_port(port: port_id) -> status_t; + pub fn port_buffer_size(port: port_id) -> ::ssize_t; + pub fn port_buffer_size_etc(port: port_id, flags: u32, timeout: bigtime_t) -> ::ssize_t; + pub fn port_count(port: port_id) -> ::ssize_t; + pub fn set_port_owner(port: port_id, team: team_id) -> status_t; + + pub fn _get_port_info(port: port_id, buf: *mut port_info, portInfoSize: ::size_t) -> status_t; + pub fn _get_next_port_info( + port: port_id, + cookie: *mut i32, + portInfo: *mut port_info, + portInfoSize: ::size_t, + ) -> status_t; + pub fn _get_port_message_info_etc( + port: port_id, + info: *mut port_message_info, + infoSize: ::size_t, + flags: u32, + timeout: bigtime_t, + ) -> status_t; + + pub fn create_sem(count: i32, name: *const ::c_char) -> sem_id; + pub fn delete_sem(id: sem_id) -> status_t; + pub fn acquire_sem(id: sem_id) -> status_t; + pub fn acquire_sem_etc(id: sem_id, count: i32, flags: u32, timeout: bigtime_t) -> status_t; + pub fn release_sem(id: sem_id) -> status_t; + pub fn release_sem_etc(id: sem_id, count: i32, flags: u32) -> status_t; + pub fn switch_sem(semToBeReleased: sem_id, id: sem_id) -> status_t; + pub fn switch_sem_etc( + semToBeReleased: sem_id, + id: sem_id, + count: i32, + flags: u32, + timeout: bigtime_t, + ) -> status_t; + pub fn get_sem_count(id: sem_id, threadCount: *mut i32) -> status_t; + pub fn set_sem_owner(id: sem_id, team: team_id) -> status_t; + pub fn _get_sem_info(id: sem_id, info: *mut sem_info, infoSize: ::size_t) -> status_t; + pub fn _get_next_sem_info( + team: team_id, + cookie: *mut i32, + info: *mut sem_info, + infoSize: ::size_t, + ) -> status_t; + + pub fn kill_team(team: team_id) -> status_t; + pub fn _get_team_info(team: team_id, info: *mut team_info, size: ::size_t) -> status_t; + pub fn _get_next_team_info(cookie: *mut i32, info: *mut team_info, size: ::size_t) -> status_t; + + pub fn spawn_thread( + func: thread_func, + name: *const ::c_char, + priority: i32, + data: *mut ::c_void, + ) -> thread_id; + pub fn kill_thread(thread: thread_id) -> status_t; + pub fn resume_thread(thread: thread_id) -> status_t; + pub fn suspend_thread(thread: thread_id) -> status_t; + + pub fn rename_thread(thread: thread_id, newName: *const ::c_char) -> status_t; + pub fn set_thread_priority(thread: thread_id, newPriority: i32) -> status_t; + pub fn suggest_thread_priority( + what: u32, + period: i32, + jitter: ::bigtime_t, + length: ::bigtime_t, + ) -> i32; + pub fn estimate_max_scheduling_latency(th: ::thread_id) -> ::bigtime_t; + pub fn exit_thread(status: status_t); + pub fn wait_for_thread(thread: thread_id, returnValue: *mut status_t) -> status_t; + pub fn on_exit_thread(callback: extern "C" fn(*mut ::c_void), data: *mut ::c_void) -> status_t; + + pub fn find_thread(name: *const ::c_char) -> thread_id; + + pub fn get_scheduler_mode() -> i32; + pub fn set_scheduler_mode(mode: i32) -> status_t; + + pub fn send_data( + thread: thread_id, + code: i32, + buffer: *const ::c_void, + bufferSize: ::size_t, + ) -> status_t; + pub fn receive_data(sender: *mut thread_id, buffer: *mut ::c_void, bufferSize: ::size_t) + -> i32; + pub fn has_data(thread: thread_id) -> bool; + + pub fn snooze(amount: bigtime_t) -> status_t; + pub fn snooze_etc(amount: bigtime_t, timeBase: ::c_int, flags: u32) -> status_t; + pub fn snooze_until(time: bigtime_t, timeBase: ::c_int) -> status_t; + + pub fn _get_thread_info(id: thread_id, info: *mut thread_info, size: ::size_t) -> status_t; + pub fn _get_next_thread_info( + team: team_id, + cookie: *mut i32, + info: *mut thread_info, + size: ::size_t, + ) -> status_t; + + pub fn get_pthread_thread_id(thread: ::pthread_t) -> thread_id; + + pub fn _get_team_usage_info( + team: team_id, + who: i32, + info: *mut team_usage_info, + size: ::size_t, + ) -> status_t; + + pub fn real_time_clock() -> ::c_ulong; + pub fn set_real_time_clock(secsSinceJan1st1970: ::c_ulong); + pub fn real_time_clock_usecs() -> bigtime_t; + pub fn system_time() -> bigtime_t; + pub fn system_time_nsecs() -> nanotime_t; + // set_timezone() is deprecated and a no-op + + pub fn set_alarm(when: bigtime_t, flags: u32) -> bigtime_t; + pub fn debugger(message: *const ::c_char); + pub fn disable_debugger(state: ::c_int) -> ::c_int; + + pub fn get_system_info(info: *mut system_info) -> status_t; + pub fn _get_cpu_info_etc( + firstCPU: u32, + cpuCount: u32, + info: *mut cpu_info, + size: ::size_t, + ) -> status_t; + pub fn is_computer_on() -> i32; + pub fn is_computer_on_fire() -> ::c_double; + pub fn send_signal(threadID: thread_id, signal: ::c_uint) -> ::c_int; + pub fn set_signal_stack(base: *mut ::c_void, size: ::size_t); + + pub fn wait_for_objects(infos: *mut object_wait_info, numInfos: ::c_int) -> ::ssize_t; + pub fn wait_for_objects_etc( + infos: *mut object_wait_info, + numInfos: ::c_int, + flags: u32, + timeout: bigtime_t, + ) -> ::ssize_t; + + // kernel/fs_attr.h + pub fn fs_read_attr( + fd: ::c_int, + attribute: *const ::c_char, + type_: u32, + pos: ::off_t, + buffer: *mut ::c_void, + readBytes: ::size_t, + ) -> ::ssize_t; + pub fn fs_write_attr( + fd: ::c_int, + attribute: *const ::c_char, + type_: u32, + pos: ::off_t, + buffer: *const ::c_void, + writeBytes: ::size_t, + ) -> ::ssize_t; + pub fn fs_remove_attr(fd: ::c_int, attribute: *const ::c_char) -> ::c_int; + pub fn fs_stat_attr( + fd: ::c_int, + attribute: *const ::c_char, + attrInfo: *mut attr_info, + ) -> ::c_int; + + pub fn fs_open_attr( + path: *const ::c_char, + attribute: *const ::c_char, + type_: u32, + openMode: ::c_int, + ) -> ::c_int; + pub fn fs_fopen_attr( + fd: ::c_int, + attribute: *const ::c_char, + type_: u32, + openMode: ::c_int, + ) -> ::c_int; + pub fn fs_close_attr(fd: ::c_int) -> ::c_int; + + pub fn fs_open_attr_dir(path: *const ::c_char) -> *mut ::DIR; + pub fn fs_lopen_attr_dir(path: *const ::c_char) -> *mut ::DIR; + pub fn fs_fopen_attr_dir(fd: ::c_int) -> *mut ::DIR; + pub fn fs_close_attr_dir(dir: *mut ::DIR) -> ::c_int; + pub fn fs_read_attr_dir(dir: *mut ::DIR) -> *mut ::dirent; + pub fn fs_rewind_attr_dir(dir: *mut ::DIR); + + // kernel/fs_image.h + pub fn fs_create_index( + device: ::dev_t, + name: *const ::c_char, + type_: u32, + flags: u32, + ) -> ::c_int; + pub fn fs_remove_index(device: ::dev_t, name: *const ::c_char) -> ::c_int; + pub fn fs_stat_index( + device: ::dev_t, + name: *const ::c_char, + indexInfo: *mut index_info, + ) -> ::c_int; + + pub fn fs_open_index_dir(device: ::dev_t) -> *mut ::DIR; + pub fn fs_close_index_dir(indexDirectory: *mut ::DIR) -> ::c_int; + pub fn fs_read_index_dir(indexDirectory: *mut ::DIR) -> *mut ::dirent; + pub fn fs_rewind_index_dir(indexDirectory: *mut ::DIR); + + // kernel/fs_info.h + pub fn dev_for_path(path: *const ::c_char) -> ::dev_t; + pub fn next_dev(pos: *mut i32) -> ::dev_t; + pub fn fs_stat_dev(dev: ::dev_t, info: *mut fs_info) -> ::c_int; + + // kernel/fs_query.h + pub fn fs_open_query(device: ::dev_t, query: *const ::c_char, flags: u32) -> *mut ::DIR; + pub fn fs_open_live_query( + device: ::dev_t, + query: *const ::c_char, + flags: u32, + port: port_id, + token: i32, + ) -> *mut ::DIR; + pub fn fs_close_query(d: *mut ::DIR) -> ::c_int; + pub fn fs_read_query(d: *mut ::DIR) -> *mut ::dirent; + pub fn get_path_for_dirent(dent: *mut ::dirent, buf: *mut ::c_char, len: ::size_t) -> status_t; + + // kernel/fs_volume.h + pub fn fs_mount_volume( + where_: *const ::c_char, + device: *const ::c_char, + filesystem: *const ::c_char, + flags: u32, + parameters: *const ::c_char, + ) -> ::dev_t; + pub fn fs_unmount_volume(path: *const ::c_char, flags: u32) -> status_t; + + // kernel/image.h + pub fn load_image( + argc: i32, + argv: *mut *const ::c_char, + environ: *mut *const ::c_char, + ) -> thread_id; + pub fn load_add_on(path: *const ::c_char) -> image_id; + pub fn unload_add_on(image: image_id) -> status_t; + pub fn get_image_symbol( + image: image_id, + name: *const ::c_char, + symbolType: i32, + symbolLocation: *mut *mut ::c_void, + ) -> status_t; + pub fn get_nth_image_symbol( + image: image_id, + n: i32, + nameBuffer: *mut ::c_char, + nameLength: *mut i32, + symbolType: *mut i32, + symbolLocation: *mut *mut ::c_void, + ) -> status_t; + pub fn clear_caches(address: *mut ::c_void, length: ::size_t, flags: u32); + pub fn _get_image_info(image: image_id, info: *mut image_info, size: ::size_t) -> status_t; + pub fn _get_next_image_info( + team: team_id, + cookie: *mut i32, + info: *mut image_info, + size: ::size_t, + ) -> status_t; + pub fn find_path( + codePointer: *const ::c_void, + baseDirectory: path_base_directory, + subPath: *const ::c_char, + pathBuffer: *mut ::c_char, + bufferSize: usize, + ) -> status_t; + pub fn find_path_etc( + codePointer: *const ::c_void, + dependency: *const ::c_char, + architecture: *const ::c_char, + baseDirectory: path_base_directory, + subPath: *const ::c_char, + flags: u32, + pathBuffer: *mut ::c_char, + bufferSize: ::size_t, + ) -> status_t; + pub fn find_path_for_path( + path: *const ::c_char, + baseDirectory: path_base_directory, + subPath: *const ::c_char, + pathBuffer: *mut ::c_char, + bufferSize: ::size_t, + ) -> status_t; + pub fn find_path_for_path_etc( + path: *const ::c_char, + dependency: *const ::c_char, + architectur: *const ::c_char, + baseDirectory: path_base_directory, + subPath: *const ::c_char, + flags: u32, + pathBuffer: *mut ::c_char, + bufferSize: ::size_t, + ) -> status_t; + pub fn find_paths( + baseDirectory: path_base_directory, + subPath: *const ::c_char, + _paths: *mut *mut *mut ::c_char, + pathCount: *mut ::size_t, + ) -> status_t; + pub fn find_paths_etc( + architecture: *const ::c_char, + baseDirectory: path_base_directory, + subPath: *const ::c_char, + flags: u32, + _paths: *mut *mut *mut ::c_char, + pathCount: *mut ::size_t, + ) -> status_t; + pub fn find_directory( + which: directory_which, + volume: ::dev_t, + createIt: bool, + pathString: *mut ::c_char, + length: i32, + ) -> status_t; +} + +cfg_if! { + if #[cfg(libc_union)] { + extern "C" { + pub fn get_cpuid(info: *mut cpuid_info, eaxRegister: u32, cpuNum: u32) -> status_t; + } + } +} + +// The following functions are defined as macros in C/C++ +#[inline] +pub unsafe fn get_cpu_info(firstCPU: u32, cpuCount: u32, info: *mut cpu_info) -> status_t { + _get_cpu_info_etc( + firstCPU, + cpuCount, + info, + core::mem::size_of::() as ::size_t, + ) +} + +#[inline] +pub unsafe fn get_area_info(id: area_id, info: *mut area_info) -> status_t { + _get_area_info(id, info, core::mem::size_of::() as usize) +} + +#[inline] +pub unsafe fn get_next_area_info( + team: team_id, + cookie: *mut isize, + info: *mut area_info, +) -> status_t { + _get_next_area_info( + team, + cookie, + info, + core::mem::size_of::() as usize, + ) +} + +#[inline] +pub unsafe fn get_port_info(port: port_id, buf: *mut port_info) -> status_t { + _get_port_info(port, buf, core::mem::size_of::() as ::size_t) +} + +#[inline] +pub unsafe fn get_next_port_info( + port: port_id, + cookie: *mut i32, + portInfo: *mut port_info, +) -> status_t { + _get_next_port_info( + port, + cookie, + portInfo, + core::mem::size_of::() as ::size_t, + ) +} + +#[inline] +pub unsafe fn get_port_message_info_etc( + port: port_id, + info: *mut port_message_info, + flags: u32, + timeout: bigtime_t, +) -> status_t { + _get_port_message_info_etc( + port, + info, + core::mem::size_of::() as ::size_t, + flags, + timeout, + ) +} + +#[inline] +pub unsafe fn get_sem_info(id: sem_id, info: *mut sem_info) -> status_t { + _get_sem_info(id, info, core::mem::size_of::() as ::size_t) +} + +#[inline] +pub unsafe fn get_next_sem_info(team: team_id, cookie: *mut i32, info: *mut sem_info) -> status_t { + _get_next_sem_info( + team, + cookie, + info, + core::mem::size_of::() as ::size_t, + ) +} + +#[inline] +pub unsafe fn get_team_info(team: team_id, info: *mut team_info) -> status_t { + _get_team_info(team, info, core::mem::size_of::() as ::size_t) +} + +#[inline] +pub unsafe fn get_next_team_info(cookie: *mut i32, info: *mut team_info) -> status_t { + _get_next_team_info(cookie, info, core::mem::size_of::() as ::size_t) +} + +#[inline] +pub unsafe fn get_team_usage_info(team: team_id, who: i32, info: *mut team_usage_info) -> status_t { + _get_team_usage_info( + team, + who, + info, + core::mem::size_of::() as ::size_t, + ) +} + +#[inline] +pub unsafe fn get_thread_info(id: thread_id, info: *mut thread_info) -> status_t { + _get_thread_info(id, info, core::mem::size_of::() as ::size_t) +} + +#[inline] +pub unsafe fn get_next_thread_info( + team: team_id, + cookie: *mut i32, + info: *mut thread_info, +) -> status_t { + _get_next_thread_info( + team, + cookie, + info, + core::mem::size_of::() as ::size_t, + ) +} + +// kernel/image.h +#[inline] +pub unsafe fn get_image_info(image: image_id, info: *mut image_info) -> status_t { + _get_image_info(image, info, core::mem::size_of::() as ::size_t) +} + +#[inline] +pub unsafe fn get_next_image_info( + team: team_id, + cookie: *mut i32, + info: *mut image_info, +) -> status_t { + _get_next_image_info( + team, + cookie, + info, + core::mem::size_of::() as ::size_t, + ) +} diff --git a/crux-mir/lib/libc/src/unix/haiku/x86_64.rs b/crux-mir/lib/libc/src/unix/haiku/x86_64.rs new file mode 100644 index 000000000..1b0462f20 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/haiku/x86_64.rs @@ -0,0 +1,264 @@ +s_no_extra_traits! { + pub struct fpu_state { + pub control: ::c_ushort, + pub status: ::c_ushort, + pub tag: ::c_ushort, + pub opcode: ::c_ushort, + pub rip: ::c_ulong, + pub rdp: ::c_ulong, + pub mxcsr: ::c_uint, + pub mscsr_mask: ::c_uint, + pub _fpreg: [[::c_uchar; 8]; 16], + pub _xmm: [[::c_uchar; 16]; 16], + pub _reserved_416_511: [::c_uchar; 96], + } + + pub struct xstate_hdr { + pub bv: ::c_ulong, + pub xcomp_bv: ::c_ulong, + pub _reserved: [::c_uchar; 48], + } + + pub struct savefpu { + pub fp_fxsave: fpu_state, + pub fp_xstate: xstate_hdr, + pub _fp_ymm: [[::c_uchar; 16]; 16], + } + + pub struct mcontext_t { + pub rax: ::c_ulong, + pub rbx: ::c_ulong, + pub rcx: ::c_ulong, + pub rdx: ::c_ulong, + pub rdi: ::c_ulong, + pub rsi: ::c_ulong, + pub rbp: ::c_ulong, + pub r8: ::c_ulong, + pub r9: ::c_ulong, + pub r10: ::c_ulong, + pub r11: ::c_ulong, + pub r12: ::c_ulong, + pub r13: ::c_ulong, + pub r14: ::c_ulong, + pub r15: ::c_ulong, + pub rsp: ::c_ulong, + pub rip: ::c_ulong, + pub rflags: ::c_ulong, + pub fpu: savefpu, + } + + pub struct ucontext_t { + pub uc_link: *mut ucontext_t, + pub uc_sigmask: ::sigset_t, + pub uc_stack: ::stack_t, + pub uc_mcontext: mcontext_t, + } +} + +cfg_if! { + if #[cfg(feature = "extra_traits")] { + impl PartialEq for fpu_state { + fn eq(&self, other: &fpu_state) -> bool { + self.control == other.control + && self.status == other.status + && self.tag == other.tag + && self.opcode == other.opcode + && self.rip == other.rip + && self.rdp == other.rdp + && self.mxcsr == other.mxcsr + && self.mscsr_mask == other.mscsr_mask + && self._fpreg.iter().zip(other._fpreg.iter()).all(|(a, b)| a == b) + && self._xmm.iter().zip(other._xmm.iter()).all(|(a, b)| a == b) + && self._reserved_416_511. + iter(). + zip(other._reserved_416_511.iter()). + all(|(a, b)| a == b) + } + } + impl Eq for fpu_state {} + impl ::fmt::Debug for fpu_state { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("fpu_state") + .field("control", &self.control) + .field("status", &self.status) + .field("tag", &self.tag) + .field("opcode", &self.opcode) + .field("rip", &self.rip) + .field("rdp", &self.rdp) + .field("mxcsr", &self.mxcsr) + .field("mscsr_mask", &self.mscsr_mask) + // FIXME: .field("_fpreg", &self._fpreg) + // FIXME: .field("_xmm", &self._xmm) + // FIXME: .field("_reserved_416_511", &self._reserved_416_511) + .finish() + } + } + impl ::hash::Hash for fpu_state { + fn hash(&self, state: &mut H) { + self.control.hash(state); + self.status.hash(state); + self.tag.hash(state); + self.opcode.hash(state); + self.rip.hash(state); + self.rdp.hash(state); + self.mxcsr.hash(state); + self.mscsr_mask.hash(state); + self._fpreg.hash(state); + self._xmm.hash(state); + self._reserved_416_511.hash(state); + } + } + + impl PartialEq for xstate_hdr { + fn eq(&self, other: &xstate_hdr) -> bool { + self.bv == other.bv + && self.xcomp_bv == other.xcomp_bv + && self._reserved.iter().zip(other._reserved.iter()).all(|(a, b)| a == b) + } + } + impl Eq for xstate_hdr {} + impl ::fmt::Debug for xstate_hdr { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("xstate_hdr") + .field("bv", &self.bv) + .field("xcomp_bv", &self.xcomp_bv) + // FIXME: .field("_reserved", &field._reserved) + .finish() + } + } + impl ::hash::Hash for xstate_hdr { + fn hash(&self, state: &mut H) { + self.bv.hash(state); + self.xcomp_bv.hash(state); + self._reserved.hash(state); + } + } + + impl PartialEq for savefpu { + fn eq(&self, other: &savefpu) -> bool { + self.fp_fxsave == other.fp_fxsave + && self.fp_xstate == other.fp_xstate + && self._fp_ymm.iter().zip(other._fp_ymm.iter()).all(|(a, b)| a == b) + } + } + impl Eq for savefpu {} + impl ::fmt::Debug for savefpu { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("savefpu") + .field("fp_fxsave", &self.fp_fxsave) + .field("fp_xstate", &self.fp_xstate) + // FIXME: .field("_fp_ymm", &field._fp_ymm) + .finish() + } + } + impl ::hash::Hash for savefpu { + fn hash(&self, state: &mut H) { + self.fp_fxsave.hash(state); + self.fp_xstate.hash(state); + self._fp_ymm.hash(state); + } + } + + impl PartialEq for mcontext_t { + fn eq(&self, other: &mcontext_t) -> bool { + self.rax == other.rax + && self.rbx == other.rbx + && self.rbx == other.rbx + && self.rcx == other.rcx + && self.rdx == other.rdx + && self.rdi == other.rdi + && self.rsi == other.rsi + && self.r8 == other.r8 + && self.r9 == other.r9 + && self.r10 == other.r10 + && self.r11 == other.r11 + && self.r12 == other.r12 + && self.r13 == other.r13 + && self.r14 == other.r14 + && self.r15 == other.r15 + && self.rsp == other.rsp + && self.rip == other.rip + && self.rflags == other.rflags + && self.fpu == other.fpu + } + } + impl Eq for mcontext_t {} + impl ::fmt::Debug for mcontext_t { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("mcontext_t") + .field("rax", &self.rax) + .field("rbx", &self.rbx) + .field("rcx", &self.rcx) + .field("rdx", &self.rdx) + .field("rdi", &self.rdi) + .field("rsi", &self.rsi) + .field("rbp", &self.rbp) + .field("r8", &self.r8) + .field("r9", &self.r9) + .field("r10", &self.r10) + .field("r11", &self.r11) + .field("r12", &self.r12) + .field("r13", &self.r13) + .field("r14", &self.r14) + .field("r15", &self.r15) + .field("rsp", &self.rsp) + .field("rip", &self.rip) + .field("rflags", &self.rflags) + .field("fpu", &self.fpu) + .finish() + + } + } + impl ::hash::Hash for mcontext_t { + fn hash(&self, state: &mut H) { + self.rax.hash(state); + self.rbx.hash(state); + self.rcx.hash(state); + self.rdx.hash(state); + self.rdi.hash(state); + self.rsi.hash(state); + self.rbp.hash(state); + self.r8.hash(state); + self.r9.hash(state); + self.r10.hash(state); + self.r11.hash(state); + self.r12.hash(state); + self.r13.hash(state); + self.r14.hash(state); + self.r15.hash(state); + self.rsp.hash(state); + self.rip.hash(state); + self.rflags.hash(state); + self.fpu.hash(state); + } + } + + impl PartialEq for ucontext_t { + fn eq(&self, other: &ucontext_t) -> bool { + self.uc_link == other.uc_link + && self.uc_sigmask == other.uc_sigmask + && self.uc_stack == other.uc_stack + && self.uc_mcontext == other.uc_mcontext + } + } + impl Eq for ucontext_t {} + impl ::fmt::Debug for ucontext_t { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("ucontext_t") + .field("uc_link", &self.uc_link) + .field("uc_sigmask", &self.uc_sigmask) + .field("uc_stack", &self.uc_stack) + .field("uc_mcontext", &self.uc_mcontext) + .finish() + } + } + impl ::hash::Hash for ucontext_t { + fn hash(&self, state: &mut H) { + self.uc_link.hash(state); + self.uc_sigmask.hash(state); + self.uc_stack.hash(state); + self.uc_mcontext.hash(state); + } + } + } +} diff --git a/crux-mir/lib/libc/src/unix/hermit/mod.rs b/crux-mir/lib/libc/src/unix/hermit/mod.rs index 83e064e71..eedfd28a4 100644 --- a/crux-mir/lib/libc/src/unix/hermit/mod.rs +++ b/crux-mir/lib/libc/src/unix/hermit/mod.rs @@ -1,13 +1,3 @@ -// Copyright 2018 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - // liblibc port for HermitCore (https://hermitcore.org) // HermitCore is a unikernel based on lwIP, newlib, and // pthread-embedded. @@ -949,16 +939,20 @@ const ULONG_SIZE: usize = 64; pub const WNOHANG: ::c_int = 0x00000001; -f! { - pub fn WEXITSTATUS(status: ::c_int) -> ::c_int { +pub const PRIO_PROCESS: ::c_int = 0; +pub const PRIO_PGRP: ::c_int = 1; +pub const PRIO_USER: ::c_int = 2; + +safe_f! { + pub {const} fn WEXITSTATUS(status: ::c_int) -> ::c_int { (status >> 8) & 0xff } - pub fn WIFEXITED(status: ::c_int) -> bool { + pub {const} fn WIFEXITED(status: ::c_int) -> bool { (status & 0xff) == 0 } - pub fn WTERMSIG(status: ::c_int) -> ::c_int { + pub {const} fn WTERMSIG(status: ::c_int) -> ::c_int { status & 0x7f } } @@ -966,18 +960,10 @@ f! { extern "C" { pub fn getrlimit(resource: ::c_int, rlim: *mut ::rlimit) -> ::c_int; pub fn setrlimit(resource: ::c_int, rlim: *const ::rlimit) -> ::c_int; - pub fn strerror_r( - errnum: ::c_int, - buf: *mut c_char, - buflen: ::size_t, - ) -> ::c_int; + pub fn strerror_r(errnum: ::c_int, buf: *mut c_char, buflen: ::size_t) -> ::c_int; pub fn sem_destroy(sem: *mut sem_t) -> ::c_int; - pub fn sem_init( - sem: *mut sem_t, - pshared: ::c_int, - value: ::c_uint, - ) -> ::c_int; + pub fn sem_init(sem: *mut sem_t, pshared: ::c_int, value: ::c_uint) -> ::c_int; pub fn abs(i: ::c_int) -> ::c_int; pub fn atof(s: *const ::c_char) -> ::c_double; @@ -985,16 +971,9 @@ extern "C" { pub fn rand() -> ::c_int; pub fn srand(seed: ::c_uint); - pub fn bind( - s: ::c_int, - name: *const ::sockaddr, - namelen: ::socklen_t, - ) -> ::c_int; + pub fn bind(s: ::c_int, name: *const ::sockaddr, namelen: ::socklen_t) -> ::c_int; - pub fn clock_gettime( - clock_id: ::clockid_t, - tp: *mut ::timespec, - ) -> ::c_int; + pub fn clock_gettime(clock_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; pub fn gettimeofday(tp: *mut ::timeval, tz: *mut ::c_void) -> ::c_int; pub fn getpwuid_r( @@ -1017,11 +996,7 @@ extern "C" { arg: *mut ::c_void, ) -> ::c_int; - pub fn pthread_sigmask( - how: ::c_int, - set: *const ::sigset_t, - oset: *mut ::sigset_t, - ) -> ::c_int; + pub fn pthread_sigmask(how: ::c_int, set: *const ::sigset_t, oset: *mut ::sigset_t) -> ::c_int; pub fn recvfrom( s: ::c_int, diff --git a/crux-mir/lib/libc/src/unix/linux_like/android/b32/arm.rs b/crux-mir/lib/libc/src/unix/linux_like/android/b32/arm.rs index aa9beb765..33d2fa070 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/android/b32/arm.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/android/b32/arm.rs @@ -138,7 +138,7 @@ cfg_if! { self.uc_stack.hash(state); self.uc_mcontext.hash(state); self.uc_sigmask__c_anonymous_union.hash(state); - &self.uc_regspace[..].hash(state); + self.uc_regspace[..].hash(state); // Ignore padding field } } @@ -521,3 +521,19 @@ pub const REG_R14: ::c_int = 14; pub const REG_R15: ::c_int = 15; pub const NGREG: ::c_int = 18; + +f! { + // Sadly, Android before 5.0 (API level 21), the accept4 syscall is not + // exposed by the libc. As work-around, we implement it through `syscall` + // directly. This workaround can be removed if the minimum version of + // Android is bumped. When the workaround is removed, `accept4` can be + // moved back to `linux_like/mod.rs` + pub fn accept4( + fd: ::c_int, + addr: *mut ::sockaddr, + len: *mut ::socklen_t, + flg: ::c_int + ) -> ::c_int { + ::syscall(SYS_accept4, fd, addr, len, flg) as ::c_int + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/android/b32/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/android/b32/mod.rs index 5c4f03234..1f4f796f2 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/android/b32/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/android/b32/mod.rs @@ -8,6 +8,7 @@ pub type off64_t = ::c_longlong; pub type sigset_t = ::c_ulong; pub type socklen_t = i32; pub type time64_t = i64; +pub type __u64 = ::c_ulonglong; s! { pub struct sigaction { @@ -27,7 +28,7 @@ s! { __pad0: [::c_uchar; 4], __st_ino: ::ino_t, pub st_mode: ::c_uint, - pub st_nlink: ::c_uint, + pub st_nlink: ::nlink_t, pub st_uid: ::uid_t, pub st_gid: ::gid_t, pub st_rdev: ::c_ulonglong, @@ -49,7 +50,7 @@ s! { __pad0: [::c_uchar; 4], __st_ino: ::ino_t, pub st_mode: ::c_uint, - pub st_nlink: ::c_uint, + pub st_nlink: ::nlink_t, pub st_uid: ::uid_t, pub st_gid: ::gid_t, pub st_rdev: ::c_ulonglong, @@ -74,7 +75,7 @@ s! { pub f_bavail: u64, pub f_files: u64, pub f_ffree: u64, - f_fsid: [u32; 2], + pub f_fsid: ::__fsid_t, pub f_namelen: u32, pub f_frsize: u32, pub f_flags: u32, @@ -119,6 +120,14 @@ s! { __reserved: [::c_char; 12], } + pub struct pthread_barrier_t { + __private: [i32; 8], + } + + pub struct pthread_spinlock_t { + __private: [i32; 2], + } + pub struct passwd { pub pw_name: *mut ::c_char, pub pw_passwd: *mut ::c_char, @@ -194,13 +203,9 @@ pub const RTLD_DEFAULT: *mut ::c_void = -1isize as *mut ::c_void; pub const PTRACE_GETFPREGS: ::c_int = 14; pub const PTRACE_SETFPREGS: ::c_int = 15; -pub const PTRACE_GETREGS: ::c_int = 12; -pub const PTRACE_SETREGS: ::c_int = 13; -pub const PTHREAD_MUTEX_INITIALIZER: pthread_mutex_t = - pthread_mutex_t { value: 0 }; -pub const PTHREAD_COND_INITIALIZER: pthread_cond_t = - pthread_cond_t { value: 0 }; +pub const PTHREAD_MUTEX_INITIALIZER: pthread_mutex_t = pthread_mutex_t { value: 0 }; +pub const PTHREAD_COND_INITIALIZER: pthread_cond_t = pthread_cond_t { value: 0 }; pub const PTHREAD_RWLOCK_INITIALIZER: pthread_rwlock_t = pthread_rwlock_t { lock: PTHREAD_MUTEX_INITIALIZER, cond: PTHREAD_COND_INITIALIZER, diff --git a/crux-mir/lib/libc/src/unix/linux_like/android/b32/x86/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/android/b32/x86/mod.rs index 879ea1a17..6507cb4e0 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/android/b32/x86/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/android/b32/x86/mod.rs @@ -574,6 +574,35 @@ pub const REG_EFL: ::c_int = 16; pub const REG_UESP: ::c_int = 17; pub const REG_SS: ::c_int = 18; +// socketcall values from linux/net.h (only the needed ones, and not public) +const SYS_ACCEPT4: ::c_int = 18; + +f! { + // Sadly, Android before 5.0 (API level 21), the accept4 syscall is not + // exposed by the libc. As work-around, we implement it as raw syscall. + // Note that for x86, the `accept4` syscall is not available either, + // and we must use the `socketcall` syscall instead. + // This workaround can be removed if the minimum Android version is bumped. + // When the workaround is removed, `accept4` can be moved back + // to `linux_like/mod.rs` + pub fn accept4( + fd: ::c_int, + addr: *mut ::sockaddr, + len: *mut ::socklen_t, + flg: ::c_int + ) -> ::c_int { + // Arguments are passed as array of `long int` + // (which is big enough on x86 for a pointer). + let mut args = [ + fd as ::c_long, + addr as ::c_long, + len as ::c_long, + flg as ::c_long, + ]; + ::syscall(SYS_socketcall, SYS_ACCEPT4, args[..].as_mut_ptr()) + } +} + cfg_if! { if #[cfg(libc_align)] { mod align; diff --git a/crux-mir/lib/libc/src/unix/linux_like/android/b64/aarch64/align.rs b/crux-mir/lib/libc/src/unix/linux_like/android/b64/aarch64/align.rs index 8e949963a..154c2c54c 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/android/b64/aarch64/align.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/android/b64/aarch64/align.rs @@ -5,3 +5,25 @@ s_no_extra_traits! { priv_: [f32; 8] } } + +s! { + pub struct ucontext_t { + pub uc_flags: ::c_ulong, + pub uc_link: *mut ucontext_t, + pub uc_stack: ::stack_t, + pub uc_sigmask: ::sigset_t, + pub uc_mcontext: mcontext_t, + } + + #[repr(align(16))] + pub struct mcontext_t { + pub fault_address: ::c_ulonglong, + pub regs: [::c_ulonglong; 31], + pub sp: ::c_ulonglong, + pub pc: ::c_ulonglong, + pub pstate: ::c_ulonglong, + // nested arrays to get the right size/length while being able to + // auto-derive traits like Debug + __reserved: [[u64; 32]; 16], + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/android/b64/aarch64/int128.rs b/crux-mir/lib/libc/src/unix/linux_like/android/b64/aarch64/int128.rs new file mode 100644 index 000000000..4535e73ee --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/android/b64/aarch64/int128.rs @@ -0,0 +1,7 @@ +s! { + pub struct user_fpsimd_struct { + pub vregs: [::__uint128_t; 32], + pub fpsr: u32, + pub fpcr: u32, + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/android/b64/aarch64/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/android/b64/aarch64/mod.rs index b2b91889a..c4d442060 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/android/b64/aarch64/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/android/b64/aarch64/mod.rs @@ -1,12 +1,13 @@ pub type c_char = u8; pub type wchar_t = u32; +pub type __u64 = ::c_ulonglong; s! { pub struct stat { pub st_dev: ::dev_t, pub st_ino: ::ino_t, pub st_mode: ::c_uint, - pub st_nlink: ::c_uint, + pub st_nlink: ::nlink_t, pub st_uid: ::uid_t, pub st_gid: ::gid_t, pub st_rdev: ::dev_t, @@ -29,7 +30,7 @@ s! { pub st_dev: ::dev_t, pub st_ino: ::ino_t, pub st_mode: ::c_uint, - pub st_nlink: ::c_uint, + pub st_nlink: ::nlink_t, pub st_uid: ::uid_t, pub st_gid: ::gid_t, pub st_rdev: ::dev_t, @@ -47,6 +48,13 @@ s! { __unused4: ::c_uint, __unused5: ::c_uint, } + + pub struct user_regs_struct { + pub regs: [u64; 31], + pub sp: u64, + pub pc: u64, + pub pstate: u64, + } } pub const O_DIRECT: ::c_int = 0x10000; @@ -57,6 +65,49 @@ pub const O_LARGEFILE: ::c_int = 0o400000; pub const SIGSTKSZ: ::size_t = 16384; pub const MINSIGSTKSZ: ::size_t = 5120; +// From NDK's asm/hwcap.h +pub const HWCAP_FP: ::c_ulong = 1 << 0; +pub const HWCAP_ASIMD: ::c_ulong = 1 << 1; +pub const HWCAP_EVTSTRM: ::c_ulong = 1 << 2; +pub const HWCAP_AES: ::c_ulong = 1 << 3; +pub const HWCAP_PMULL: ::c_ulong = 1 << 4; +pub const HWCAP_SHA1: ::c_ulong = 1 << 5; +pub const HWCAP_SHA2: ::c_ulong = 1 << 6; +pub const HWCAP_CRC32: ::c_ulong = 1 << 7; +pub const HWCAP_ATOMICS: ::c_ulong = 1 << 8; +pub const HWCAP_FPHP: ::c_ulong = 1 << 9; +pub const HWCAP_ASIMDHP: ::c_ulong = 1 << 10; +pub const HWCAP_CPUID: ::c_ulong = 1 << 11; +pub const HWCAP_ASIMDRDM: ::c_ulong = 1 << 12; +pub const HWCAP_JSCVT: ::c_ulong = 1 << 13; +pub const HWCAP_FCMA: ::c_ulong = 1 << 14; +pub const HWCAP_LRCPC: ::c_ulong = 1 << 15; +pub const HWCAP_DCPOP: ::c_ulong = 1 << 16; +pub const HWCAP_SHA3: ::c_ulong = 1 << 17; +pub const HWCAP_SM3: ::c_ulong = 1 << 18; +pub const HWCAP_SM4: ::c_ulong = 1 << 19; +pub const HWCAP_ASIMDDP: ::c_ulong = 1 << 20; +pub const HWCAP_SHA512: ::c_ulong = 1 << 21; +pub const HWCAP_SVE: ::c_ulong = 1 << 22; +pub const HWCAP_ASIMDFHM: ::c_ulong = 1 << 23; +pub const HWCAP_DIT: ::c_ulong = 1 << 24; +pub const HWCAP_USCAT: ::c_ulong = 1 << 25; +pub const HWCAP_ILRCPC: ::c_ulong = 1 << 26; +pub const HWCAP_FLAGM: ::c_ulong = 1 << 27; +pub const HWCAP_SSBS: ::c_ulong = 1 << 28; +pub const HWCAP_SB: ::c_ulong = 1 << 29; +pub const HWCAP_PACA: ::c_ulong = 1 << 30; +pub const HWCAP_PACG: ::c_ulong = 1 << 31; +pub const HWCAP2_DCPODP: ::c_ulong = 1 << 0; +pub const HWCAP2_SVE2: ::c_ulong = 1 << 1; +pub const HWCAP2_SVEAES: ::c_ulong = 1 << 2; +pub const HWCAP2_SVEPMULL: ::c_ulong = 1 << 3; +pub const HWCAP2_SVEBITPERM: ::c_ulong = 1 << 4; +pub const HWCAP2_SVESHA3: ::c_ulong = 1 << 5; +pub const HWCAP2_SVESM4: ::c_ulong = 1 << 6; +pub const HWCAP2_FLAGM2: ::c_ulong = 1 << 7; +pub const HWCAP2_FRINT: ::c_ulong = 1 << 8; + pub const SYS_io_setup: ::c_long = 0; pub const SYS_io_destroy: ::c_long = 1; pub const SYS_io_submit: ::c_long = 2; @@ -82,6 +133,7 @@ pub const SYS_epoll_ctl: ::c_long = 21; pub const SYS_epoll_pwait: ::c_long = 22; pub const SYS_dup: ::c_long = 23; pub const SYS_dup3: ::c_long = 24; +pub const SYS_fcntl: ::c_long = 25; pub const SYS_inotify_init1: ::c_long = 26; pub const SYS_inotify_add_watch: ::c_long = 27; pub const SYS_inotify_rm_watch: ::c_long = 28; @@ -322,7 +374,7 @@ pub const SYS_pwritev2: ::c_long = 287; pub const SYS_pkey_mprotect: ::c_long = 288; pub const SYS_pkey_alloc: ::c_long = 289; pub const SYS_pkey_free: ::c_long = 290; -pub const SYS_syscalls: ::c_long = 292; +pub const SYS_syscalls: ::c_long = 436; cfg_if! { if #[cfg(libc_align)] { @@ -330,3 +382,10 @@ cfg_if! { pub use self::align::*; } } + +cfg_if! { + if #[cfg(libc_int128)] { + mod int128; + pub use self::int128::*; + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/android/b64/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/android/b64/mod.rs index 0f9443f10..67d0dacf1 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/android/b64/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/android/b64/mod.rs @@ -84,7 +84,7 @@ s! { pub f_bavail: u64, pub f_files: u64, pub f_ffree: u64, - f_fsid: [u32; 2], + pub f_fsid: ::__fsid_t, pub f_namelen: u64, pub f_frsize: u64, pub f_flags: u64, @@ -105,6 +105,14 @@ s! { pub f_namemax: ::c_ulong, __f_spare: [::c_int; 6], } + + pub struct pthread_barrier_t { + __private: [i64; 4], + } + + pub struct pthread_spinlock_t { + __private: i64, + } } s_no_extra_traits! { @@ -256,6 +264,31 @@ pub const RTLD_GLOBAL: ::c_int = 0x00100; pub const RTLD_NOW: ::c_int = 2; pub const RTLD_DEFAULT: *mut ::c_void = 0i64 as *mut ::c_void; +// From NDK's linux/auxvec.h +pub const AT_NULL: ::c_ulong = 0; +pub const AT_IGNORE: ::c_ulong = 1; +pub const AT_EXECFD: ::c_ulong = 2; +pub const AT_PHDR: ::c_ulong = 3; +pub const AT_PHENT: ::c_ulong = 4; +pub const AT_PHNUM: ::c_ulong = 5; +pub const AT_PAGESZ: ::c_ulong = 6; +pub const AT_BASE: ::c_ulong = 7; +pub const AT_FLAGS: ::c_ulong = 8; +pub const AT_ENTRY: ::c_ulong = 9; +pub const AT_NOTELF: ::c_ulong = 10; +pub const AT_UID: ::c_ulong = 11; +pub const AT_EUID: ::c_ulong = 12; +pub const AT_GID: ::c_ulong = 13; +pub const AT_EGID: ::c_ulong = 14; +pub const AT_PLATFORM: ::c_ulong = 15; +pub const AT_HWCAP: ::c_ulong = 16; +pub const AT_CLKTCK: ::c_ulong = 17; +pub const AT_SECURE: ::c_ulong = 23; +pub const AT_BASE_PLATFORM: ::c_ulong = 24; +pub const AT_RANDOM: ::c_ulong = 25; +pub const AT_HWCAP2: ::c_ulong = 26; +pub const AT_EXECFN: ::c_ulong = 31; + pub const PTHREAD_MUTEX_INITIALIZER: pthread_mutex_t = pthread_mutex_t { value: 0, __reserved: [0; 36], @@ -280,6 +313,32 @@ pub const UT_LINESIZE: usize = 32; pub const UT_NAMESIZE: usize = 32; pub const UT_HOSTSIZE: usize = 256; +f! { + // Sadly, Android before 5.0 (API level 21), the accept4 syscall is not + // exposed by the libc. As work-around, we implement it through `syscall` + // directly. This workaround can be removed if the minimum version of + // Android is bumped. When the workaround is removed, `accept4` can be + // moved back to `linux_like/mod.rs` + pub fn accept4( + fd: ::c_int, + addr: *mut ::sockaddr, + len: *mut ::socklen_t, + flg: ::c_int + ) -> ::c_int { + ::syscall(SYS_accept4, fd, addr, len, flg) as ::c_int + } +} + +extern "C" { + pub fn getauxval(type_: ::c_ulong) -> ::c_ulong; + pub fn __system_property_wait( + pi: *const ::prop_info, + __old_serial: u32, + __new_serial_ptr: *mut u32, + __relative_timeout: *const ::timespec, + ) -> bool; +} + cfg_if! { if #[cfg(target_arch = "x86_64")] { mod x86_64; @@ -287,6 +346,9 @@ cfg_if! { } else if #[cfg(target_arch = "aarch64")] { mod aarch64; pub use self::aarch64::*; + } else if #[cfg(target_arch = "riscv64")] { + mod riscv64; + pub use self::riscv64::*; } else { // Unknown target_arch } diff --git a/crux-mir/lib/libc/src/unix/linux_like/android/b64/riscv64/align.rs b/crux-mir/lib/libc/src/unix/linux_like/android/b64/riscv64/align.rs new file mode 100644 index 000000000..8e949963a --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/android/b64/riscv64/align.rs @@ -0,0 +1,7 @@ +s_no_extra_traits! { + #[allow(missing_debug_implementations)] + #[repr(align(16))] + pub struct max_align_t { + priv_: [f32; 8] + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/android/b64/riscv64/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/android/b64/riscv64/mod.rs new file mode 100644 index 000000000..2421792cf --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/android/b64/riscv64/mod.rs @@ -0,0 +1,342 @@ +pub type c_char = i8; +pub type wchar_t = u32; +pub type greg_t = i64; +pub type __u64 = ::c_ulonglong; + +s! { + pub struct stat { + pub st_dev: ::dev_t, + pub st_ino: ::ino_t, + pub st_mode: ::c_uint, + pub st_nlink: ::c_uint, + pub st_uid: ::uid_t, + pub st_gid: ::gid_t, + pub st_rdev: ::dev_t, + __pad1: ::c_ulong, + pub st_size: ::off64_t, + pub st_blksize: ::c_int, + __pad2: ::c_int, + pub st_blocks: ::c_long, + pub st_atime: ::time_t, + pub st_atime_nsec: ::c_long, + pub st_mtime: ::time_t, + pub st_mtime_nsec: ::c_long, + pub st_ctime: ::time_t, + pub st_ctime_nsec: ::c_long, + __unused4: ::c_uint, + __unused5: ::c_uint, + } + + pub struct stat64 { + pub st_dev: ::dev_t, + pub st_ino: ::ino_t, + pub st_mode: ::c_uint, + pub st_nlink: ::c_uint, + pub st_uid: ::uid_t, + pub st_gid: ::gid_t, + pub st_rdev: ::dev_t, + __pad1: ::c_ulong, + pub st_size: ::off64_t, + pub st_blksize: ::c_int, + __pad2: ::c_int, + pub st_blocks: ::c_long, + pub st_atime: ::time_t, + pub st_atime_nsec: ::c_long, + pub st_mtime: ::time_t, + pub st_mtime_nsec: ::c_long, + pub st_ctime: ::time_t, + pub st_ctime_nsec: ::c_long, + __unused4: ::c_uint, + __unused5: ::c_uint, + } +} + +pub const O_DIRECT: ::c_int = 0x40000; +pub const O_DIRECTORY: ::c_int = 0x200000; +pub const O_NOFOLLOW: ::c_int = 0x400000; +pub const O_LARGEFILE: ::c_int = 0x100000; + +pub const SIGSTKSZ: ::size_t = 8192; +pub const MINSIGSTKSZ: ::size_t = 2048; + +// From NDK's asm/hwcap.h +pub const COMPAT_HWCAP_ISA_I: ::c_ulong = 1 << ('I' - 'A'); +pub const COMPAT_HWCAP_ISA_M: ::c_ulong = 1 << ('M' - 'A'); +pub const COMPAT_HWCAP_ISA_A: ::c_ulong = 1 << ('A' - 'A'); +pub const COMPAT_HWCAP_ISA_F: ::c_ulong = 1 << ('F' - 'A'); +pub const COMPAT_HWCAP_ISA_D: ::c_ulong = 1 << ('D' - 'A'); +pub const COMPAT_HWCAP_ISA_C: ::c_ulong = 1 << ('C' - 'A'); + +pub const SYS_io_setup: ::c_long = 0; +pub const SYS_io_destroy: ::c_long = 1; +pub const SYS_io_submit: ::c_long = 2; +pub const SYS_io_cancel: ::c_long = 3; +pub const SYS_io_getevents: ::c_long = 4; +pub const SYS_setxattr: ::c_long = 5; +pub const SYS_lsetxattr: ::c_long = 6; +pub const SYS_fsetxattr: ::c_long = 7; +pub const SYS_getxattr: ::c_long = 8; +pub const SYS_lgetxattr: ::c_long = 9; +pub const SYS_fgetxattr: ::c_long = 10; +pub const SYS_listxattr: ::c_long = 11; +pub const SYS_llistxattr: ::c_long = 12; +pub const SYS_flistxattr: ::c_long = 13; +pub const SYS_removexattr: ::c_long = 14; +pub const SYS_lremovexattr: ::c_long = 15; +pub const SYS_fremovexattr: ::c_long = 16; +pub const SYS_getcwd: ::c_long = 17; +pub const SYS_lookup_dcookie: ::c_long = 18; +pub const SYS_eventfd2: ::c_long = 19; +pub const SYS_epoll_create1: ::c_long = 20; +pub const SYS_epoll_ctl: ::c_long = 21; +pub const SYS_epoll_pwait: ::c_long = 22; +pub const SYS_dup: ::c_long = 23; +pub const SYS_dup3: ::c_long = 24; +pub const SYS_inotify_init1: ::c_long = 26; +pub const SYS_inotify_add_watch: ::c_long = 27; +pub const SYS_inotify_rm_watch: ::c_long = 28; +pub const SYS_ioctl: ::c_long = 29; +pub const SYS_ioprio_set: ::c_long = 30; +pub const SYS_ioprio_get: ::c_long = 31; +pub const SYS_flock: ::c_long = 32; +pub const SYS_mknodat: ::c_long = 33; +pub const SYS_mkdirat: ::c_long = 34; +pub const SYS_unlinkat: ::c_long = 35; +pub const SYS_symlinkat: ::c_long = 36; +pub const SYS_linkat: ::c_long = 37; +pub const SYS_renameat: ::c_long = 38; +pub const SYS_umount2: ::c_long = 39; +pub const SYS_mount: ::c_long = 40; +pub const SYS_pivot_root: ::c_long = 41; +pub const SYS_nfsservctl: ::c_long = 42; +pub const SYS_fallocate: ::c_long = 47; +pub const SYS_faccessat: ::c_long = 48; +pub const SYS_chdir: ::c_long = 49; +pub const SYS_fchdir: ::c_long = 50; +pub const SYS_chroot: ::c_long = 51; +pub const SYS_fchmod: ::c_long = 52; +pub const SYS_fchmodat: ::c_long = 53; +pub const SYS_fchownat: ::c_long = 54; +pub const SYS_fchown: ::c_long = 55; +pub const SYS_openat: ::c_long = 56; +pub const SYS_close: ::c_long = 57; +pub const SYS_vhangup: ::c_long = 58; +pub const SYS_pipe2: ::c_long = 59; +pub const SYS_quotactl: ::c_long = 60; +pub const SYS_getdents64: ::c_long = 61; +pub const SYS_read: ::c_long = 63; +pub const SYS_write: ::c_long = 64; +pub const SYS_readv: ::c_long = 65; +pub const SYS_writev: ::c_long = 66; +pub const SYS_pread64: ::c_long = 67; +pub const SYS_pwrite64: ::c_long = 68; +pub const SYS_preadv: ::c_long = 69; +pub const SYS_pwritev: ::c_long = 70; +pub const SYS_pselect6: ::c_long = 72; +pub const SYS_ppoll: ::c_long = 73; +pub const SYS_signalfd4: ::c_long = 74; +pub const SYS_vmsplice: ::c_long = 75; +pub const SYS_splice: ::c_long = 76; +pub const SYS_tee: ::c_long = 77; +pub const SYS_readlinkat: ::c_long = 78; +pub const SYS_sync: ::c_long = 81; +pub const SYS_fsync: ::c_long = 82; +pub const SYS_fdatasync: ::c_long = 83; +pub const SYS_sync_file_range: ::c_long = 84; +pub const SYS_timerfd_create: ::c_long = 85; +pub const SYS_timerfd_settime: ::c_long = 86; +pub const SYS_timerfd_gettime: ::c_long = 87; +pub const SYS_utimensat: ::c_long = 88; +pub const SYS_acct: ::c_long = 89; +pub const SYS_capget: ::c_long = 90; +pub const SYS_capset: ::c_long = 91; +pub const SYS_personality: ::c_long = 92; +pub const SYS_exit: ::c_long = 93; +pub const SYS_exit_group: ::c_long = 94; +pub const SYS_waitid: ::c_long = 95; +pub const SYS_set_tid_address: ::c_long = 96; +pub const SYS_unshare: ::c_long = 97; +pub const SYS_futex: ::c_long = 98; +pub const SYS_set_robust_list: ::c_long = 99; +pub const SYS_get_robust_list: ::c_long = 100; +pub const SYS_nanosleep: ::c_long = 101; +pub const SYS_getitimer: ::c_long = 102; +pub const SYS_setitimer: ::c_long = 103; +pub const SYS_kexec_load: ::c_long = 104; +pub const SYS_init_module: ::c_long = 105; +pub const SYS_delete_module: ::c_long = 106; +pub const SYS_timer_create: ::c_long = 107; +pub const SYS_timer_gettime: ::c_long = 108; +pub const SYS_timer_getoverrun: ::c_long = 109; +pub const SYS_timer_settime: ::c_long = 110; +pub const SYS_timer_delete: ::c_long = 111; +pub const SYS_clock_settime: ::c_long = 112; +pub const SYS_clock_gettime: ::c_long = 113; +pub const SYS_clock_getres: ::c_long = 114; +pub const SYS_clock_nanosleep: ::c_long = 115; +pub const SYS_syslog: ::c_long = 116; +pub const SYS_ptrace: ::c_long = 117; +pub const SYS_sched_setparam: ::c_long = 118; +pub const SYS_sched_setscheduler: ::c_long = 119; +pub const SYS_sched_getscheduler: ::c_long = 120; +pub const SYS_sched_getparam: ::c_long = 121; +pub const SYS_sched_setaffinity: ::c_long = 122; +pub const SYS_sched_getaffinity: ::c_long = 123; +pub const SYS_sched_yield: ::c_long = 124; +pub const SYS_sched_get_priority_max: ::c_long = 125; +pub const SYS_sched_get_priority_min: ::c_long = 126; +pub const SYS_sched_rr_get_interval: ::c_long = 127; +pub const SYS_restart_syscall: ::c_long = 128; +pub const SYS_kill: ::c_long = 129; +pub const SYS_tkill: ::c_long = 130; +pub const SYS_tgkill: ::c_long = 131; +pub const SYS_sigaltstack: ::c_long = 132; +pub const SYS_rt_sigsuspend: ::c_long = 133; +pub const SYS_rt_sigaction: ::c_long = 134; +pub const SYS_rt_sigprocmask: ::c_long = 135; +pub const SYS_rt_sigpending: ::c_long = 136; +pub const SYS_rt_sigtimedwait: ::c_long = 137; +pub const SYS_rt_sigqueueinfo: ::c_long = 138; +pub const SYS_rt_sigreturn: ::c_long = 139; +pub const SYS_setpriority: ::c_long = 140; +pub const SYS_getpriority: ::c_long = 141; +pub const SYS_reboot: ::c_long = 142; +pub const SYS_setregid: ::c_long = 143; +pub const SYS_setgid: ::c_long = 144; +pub const SYS_setreuid: ::c_long = 145; +pub const SYS_setuid: ::c_long = 146; +pub const SYS_setresuid: ::c_long = 147; +pub const SYS_getresuid: ::c_long = 148; +pub const SYS_setresgid: ::c_long = 149; +pub const SYS_getresgid: ::c_long = 150; +pub const SYS_setfsuid: ::c_long = 151; +pub const SYS_setfsgid: ::c_long = 152; +pub const SYS_times: ::c_long = 153; +pub const SYS_setpgid: ::c_long = 154; +pub const SYS_getpgid: ::c_long = 155; +pub const SYS_getsid: ::c_long = 156; +pub const SYS_setsid: ::c_long = 157; +pub const SYS_getgroups: ::c_long = 158; +pub const SYS_setgroups: ::c_long = 159; +pub const SYS_uname: ::c_long = 160; +pub const SYS_sethostname: ::c_long = 161; +pub const SYS_setdomainname: ::c_long = 162; +pub const SYS_getrlimit: ::c_long = 163; +pub const SYS_setrlimit: ::c_long = 164; +pub const SYS_getrusage: ::c_long = 165; +pub const SYS_umask: ::c_long = 166; +pub const SYS_prctl: ::c_long = 167; +pub const SYS_getcpu: ::c_long = 168; +pub const SYS_gettimeofday: ::c_long = 169; +pub const SYS_settimeofday: ::c_long = 170; +pub const SYS_adjtimex: ::c_long = 171; +pub const SYS_getpid: ::c_long = 172; +pub const SYS_getppid: ::c_long = 173; +pub const SYS_getuid: ::c_long = 174; +pub const SYS_geteuid: ::c_long = 175; +pub const SYS_getgid: ::c_long = 176; +pub const SYS_getegid: ::c_long = 177; +pub const SYS_gettid: ::c_long = 178; +pub const SYS_sysinfo: ::c_long = 179; +pub const SYS_mq_open: ::c_long = 180; +pub const SYS_mq_unlink: ::c_long = 181; +pub const SYS_mq_timedsend: ::c_long = 182; +pub const SYS_mq_timedreceive: ::c_long = 183; +pub const SYS_mq_notify: ::c_long = 184; +pub const SYS_mq_getsetattr: ::c_long = 185; +pub const SYS_msgget: ::c_long = 186; +pub const SYS_msgctl: ::c_long = 187; +pub const SYS_msgrcv: ::c_long = 188; +pub const SYS_msgsnd: ::c_long = 189; +pub const SYS_semget: ::c_long = 190; +pub const SYS_semctl: ::c_long = 191; +pub const SYS_semtimedop: ::c_long = 192; +pub const SYS_semop: ::c_long = 193; +pub const SYS_shmget: ::c_long = 194; +pub const SYS_shmctl: ::c_long = 195; +pub const SYS_shmat: ::c_long = 196; +pub const SYS_shmdt: ::c_long = 197; +pub const SYS_socket: ::c_long = 198; +pub const SYS_socketpair: ::c_long = 199; +pub const SYS_bind: ::c_long = 200; +pub const SYS_listen: ::c_long = 201; +pub const SYS_accept: ::c_long = 202; +pub const SYS_connect: ::c_long = 203; +pub const SYS_getsockname: ::c_long = 204; +pub const SYS_getpeername: ::c_long = 205; +pub const SYS_sendto: ::c_long = 206; +pub const SYS_recvfrom: ::c_long = 207; +pub const SYS_setsockopt: ::c_long = 208; +pub const SYS_getsockopt: ::c_long = 209; +pub const SYS_shutdown: ::c_long = 210; +pub const SYS_sendmsg: ::c_long = 211; +pub const SYS_recvmsg: ::c_long = 212; +pub const SYS_readahead: ::c_long = 213; +pub const SYS_brk: ::c_long = 214; +pub const SYS_munmap: ::c_long = 215; +pub const SYS_mremap: ::c_long = 216; +pub const SYS_add_key: ::c_long = 217; +pub const SYS_request_key: ::c_long = 218; +pub const SYS_keyctl: ::c_long = 219; +pub const SYS_clone: ::c_long = 220; +pub const SYS_execve: ::c_long = 221; +pub const SYS_swapon: ::c_long = 224; +pub const SYS_swapoff: ::c_long = 225; +pub const SYS_mprotect: ::c_long = 226; +pub const SYS_msync: ::c_long = 227; +pub const SYS_mlock: ::c_long = 228; +pub const SYS_munlock: ::c_long = 229; +pub const SYS_mlockall: ::c_long = 230; +pub const SYS_munlockall: ::c_long = 231; +pub const SYS_mincore: ::c_long = 232; +pub const SYS_madvise: ::c_long = 233; +pub const SYS_remap_file_pages: ::c_long = 234; +pub const SYS_mbind: ::c_long = 235; +pub const SYS_get_mempolicy: ::c_long = 236; +pub const SYS_set_mempolicy: ::c_long = 237; +pub const SYS_migrate_pages: ::c_long = 238; +pub const SYS_move_pages: ::c_long = 239; +pub const SYS_rt_tgsigqueueinfo: ::c_long = 240; +pub const SYS_perf_event_open: ::c_long = 241; +pub const SYS_accept4: ::c_long = 242; +pub const SYS_recvmmsg: ::c_long = 243; +pub const SYS_arch_specific_syscall: ::c_long = 244; +pub const SYS_wait4: ::c_long = 260; +pub const SYS_prlimit64: ::c_long = 261; +pub const SYS_fanotify_init: ::c_long = 262; +pub const SYS_fanotify_mark: ::c_long = 263; +pub const SYS_name_to_handle_at: ::c_long = 264; +pub const SYS_open_by_handle_at: ::c_long = 265; +pub const SYS_clock_adjtime: ::c_long = 266; +pub const SYS_syncfs: ::c_long = 267; +pub const SYS_setns: ::c_long = 268; +pub const SYS_sendmmsg: ::c_long = 269; +pub const SYS_process_vm_readv: ::c_long = 270; +pub const SYS_process_vm_writev: ::c_long = 271; +pub const SYS_kcmp: ::c_long = 272; +pub const SYS_finit_module: ::c_long = 273; +pub const SYS_sched_setattr: ::c_long = 274; +pub const SYS_sched_getattr: ::c_long = 275; +pub const SYS_renameat2: ::c_long = 276; +pub const SYS_seccomp: ::c_long = 277; +pub const SYS_getrandom: ::c_long = 278; +pub const SYS_memfd_create: ::c_long = 279; +pub const SYS_bpf: ::c_long = 280; +pub const SYS_execveat: ::c_long = 281; +pub const SYS_userfaultfd: ::c_long = 282; +pub const SYS_membarrier: ::c_long = 283; +pub const SYS_mlock2: ::c_long = 284; +pub const SYS_copy_file_range: ::c_long = 285; +pub const SYS_preadv2: ::c_long = 286; +pub const SYS_pwritev2: ::c_long = 287; +pub const SYS_pkey_mprotect: ::c_long = 288; +pub const SYS_pkey_alloc: ::c_long = 289; +pub const SYS_pkey_free: ::c_long = 290; +pub const SYS_syscalls: ::c_long = 436; + +cfg_if! { + if #[cfg(libc_align)] { + mod align; + pub use self::align::*; + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/android/b64/x86_64/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/android/b64/x86_64/mod.rs index 27fd17b3f..d25b50775 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/android/b64/x86_64/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/android/b64/x86_64/mod.rs @@ -1,6 +1,7 @@ pub type c_char = i8; pub type wchar_t = i32; pub type greg_t = i64; +pub type __u64 = ::c_ulonglong; s! { pub struct stat { @@ -46,6 +47,61 @@ s! { pub struct _libc_xmmreg { pub element: [u32; 4], } + + pub struct user_regs_struct { + pub r15: ::c_ulong, + pub r14: ::c_ulong, + pub r13: ::c_ulong, + pub r12: ::c_ulong, + pub rbp: ::c_ulong, + pub rbx: ::c_ulong, + pub r11: ::c_ulong, + pub r10: ::c_ulong, + pub r9: ::c_ulong, + pub r8: ::c_ulong, + pub rax: ::c_ulong, + pub rcx: ::c_ulong, + pub rdx: ::c_ulong, + pub rsi: ::c_ulong, + pub rdi: ::c_ulong, + pub orig_rax: ::c_ulong, + pub rip: ::c_ulong, + pub cs: ::c_ulong, + pub eflags: ::c_ulong, + pub rsp: ::c_ulong, + pub ss: ::c_ulong, + pub fs_base: ::c_ulong, + pub gs_base: ::c_ulong, + pub ds: ::c_ulong, + pub es: ::c_ulong, + pub fs: ::c_ulong, + pub gs: ::c_ulong, + } + + pub struct user { + pub regs: user_regs_struct, + pub u_fpvalid: ::c_int, + pub i387: user_fpregs_struct, + pub u_tsize: ::c_ulong, + pub u_dsize: ::c_ulong, + pub u_ssize: ::c_ulong, + pub start_code: ::c_ulong, + pub start_stack: ::c_ulong, + pub signal: ::c_long, + __reserved: ::c_int, + #[cfg(target_pointer_width = "32")] + __pad1: u32, + pub u_ar0: *mut user_regs_struct, + #[cfg(target_pointer_width = "32")] + __pad2: u32, + pub u_fpstate: *mut user_fpregs_struct, + pub magic: ::c_ulong, + pub u_comm: [::c_char; 32], + pub u_debugreg: [::c_ulong; 8], + pub error_code: ::c_ulong, + pub fault_address: ::c_ulong, + } + } cfg_if! { @@ -117,6 +173,20 @@ s_no_extra_traits! { pub uc_sigmask64: __c_anonymous_uc_sigmask, __fpregs_mem: _libc_fpstate, } + + pub struct user_fpregs_struct { + pub cwd: ::c_ushort, + pub swd: ::c_ushort, + pub ftw: ::c_ushort, + pub fop: ::c_ushort, + pub rip: ::c_ulong, + pub rdp: ::c_ulong, + pub mxcsr: ::c_uint, + pub mxcr_mask: ::c_uint, + pub st_space: [::c_uint; 32], + pub xmm_space: [::c_uint; 64], + padding: [::c_uint; 24], + } } cfg_if! { @@ -253,6 +323,62 @@ cfg_if! { // Ignore padding field } } + + impl PartialEq for user_fpregs_struct { + fn eq(&self, other: &user_fpregs_struct) -> bool { + self.cwd == other.cwd + && self.swd == other.swd + && self.ftw == other.ftw + && self.fop == other.fop + && self.rip == other.rip + && self.rdp == other.rdp + && self.mxcsr == other.mxcsr + && self.mxcr_mask == other.mxcr_mask + && self.st_space == other.st_space + && self + .xmm_space + .iter() + .zip(other.xmm_space.iter()) + .all(|(a,b)| a == b) + // Ignore padding field + } + } + + impl Eq for user_fpregs_struct {} + + impl ::fmt::Debug for user_fpregs_struct { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("user_fpregs_struct") + .field("cwd", &self.cwd) + .field("swd", &self.swd) + .field("ftw", &self.ftw) + .field("fop", &self.fop) + .field("rip", &self.rip) + .field("rdp", &self.rdp) + .field("mxcsr", &self.mxcsr) + .field("mxcr_mask", &self.mxcr_mask) + .field("st_space", &self.st_space) + // FIXME: .field("xmm_space", &self.xmm_space) + // Ignore padding field + .finish() + } + } + + impl ::hash::Hash for user_fpregs_struct { + fn hash(&self, state: &mut H) { + self.cwd.hash(state); + self.swd.hash(state); + self.ftw.hash(state); + self.fop.hash(state); + self.rip.hash(state); + self.rdp.hash(state); + self.mxcsr.hash(state); + self.mxcr_mask.hash(state); + self.st_space.hash(state); + self.xmm_space.hash(state); + // Ignore padding field + } + } } } diff --git a/crux-mir/lib/libc/src/unix/linux_like/android/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/android/mod.rs index 8135750a7..61885582a 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/android/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/android/mod.rs @@ -11,6 +11,7 @@ pub type useconds_t = u32; pub type pthread_t = ::c_long; pub type pthread_mutexattr_t = ::c_long; pub type pthread_rwlockattr_t = ::c_long; +pub type pthread_barrierattr_t = ::c_int; pub type pthread_condattr_t = ::c_long; pub type pthread_key_t = ::c_int; pub type fsfilcnt_t = ::c_ulong; @@ -19,12 +20,32 @@ pub type nfds_t = ::c_uint; pub type rlim_t = ::c_ulong; pub type dev_t = ::c_ulong; pub type ino_t = ::c_ulong; +pub type ino64_t = u64; pub type __CPU_BITTYPE = ::c_ulong; pub type idtype_t = ::c_int; pub type loff_t = ::c_longlong; pub type __kernel_loff_t = ::c_longlong; pub type __kernel_pid_t = ::c_int; +pub type __u8 = ::c_uchar; +pub type __u16 = ::c_ushort; +pub type __s16 = ::c_short; +pub type __u32 = ::c_uint; +pub type __s32 = ::c_int; + +// linux/elf.h + +pub type Elf32_Addr = u32; +pub type Elf32_Half = u16; +pub type Elf32_Off = u32; +pub type Elf32_Word = u32; + +pub type Elf64_Addr = u64; +pub type Elf64_Half = u16; +pub type Elf64_Off = u64; +pub type Elf64_Word = u32; +pub type Elf64_Xword = u64; + s! { pub struct stack_t { pub ss_sp: *mut ::c_void, @@ -72,6 +93,19 @@ s! { pub c_ospeed: ::speed_t, } + pub struct mallinfo { + pub arena: ::size_t, + pub ordblks: ::size_t, + pub smblks: ::size_t, + pub hblks: ::size_t, + pub hblkhd: ::size_t, + pub usmblks: ::size_t, + pub fsmblks: ::size_t, + pub uordblks: ::size_t, + pub fordblks: ::size_t, + pub keepcost: ::size_t, + } + pub struct flock { pub l_type: ::c_short, pub l_whence: ::c_short, @@ -147,6 +181,11 @@ s! { _pad: [u8; 28], } + pub struct itimerspec { + pub it_interval: ::timespec, + pub it_value: ::timespec, + } + pub struct ucred { pub pid: ::pid_t, pub uid: ::uid_t, @@ -208,6 +247,248 @@ s! { pub cookie: u32, pub len: u32 } + + pub struct sock_extended_err { + pub ee_errno: u32, + pub ee_origin: u8, + pub ee_type: u8, + pub ee_code: u8, + pub ee_pad: u8, + pub ee_info: u32, + pub ee_data: u32, + } + + pub struct regex_t { + re_magic: ::c_int, + re_nsub: ::size_t, + re_endp: *const ::c_char, + re_guts: *mut ::c_void, + } + + pub struct regmatch_t { + pub rm_so: ::ssize_t, + pub rm_eo: ::ssize_t, + } + + pub struct sockaddr_vm { + pub svm_family: ::sa_family_t, + pub svm_reserved1: ::c_ushort, + pub svm_port: ::c_uint, + pub svm_cid: ::c_uint, + pub svm_zero: [u8; 4] + } + + // linux/elf.h + + pub struct Elf32_Phdr { + pub p_type: Elf32_Word, + pub p_offset: Elf32_Off, + pub p_vaddr: Elf32_Addr, + pub p_paddr: Elf32_Addr, + pub p_filesz: Elf32_Word, + pub p_memsz: Elf32_Word, + pub p_flags: Elf32_Word, + pub p_align: Elf32_Word, + } + + pub struct Elf64_Phdr { + pub p_type: Elf64_Word, + pub p_flags: Elf64_Word, + pub p_offset: Elf64_Off, + pub p_vaddr: Elf64_Addr, + pub p_paddr: Elf64_Addr, + pub p_filesz: Elf64_Xword, + pub p_memsz: Elf64_Xword, + pub p_align: Elf64_Xword, + } + + // link.h + + pub struct dl_phdr_info { + #[cfg(target_pointer_width = "64")] + pub dlpi_addr: Elf64_Addr, + #[cfg(target_pointer_width = "32")] + pub dlpi_addr: Elf32_Addr, + + pub dlpi_name: *const ::c_char, + + #[cfg(target_pointer_width = "64")] + pub dlpi_phdr: *const Elf64_Phdr, + #[cfg(target_pointer_width = "32")] + pub dlpi_phdr: *const Elf32_Phdr, + + #[cfg(target_pointer_width = "64")] + pub dlpi_phnum: Elf64_Half, + #[cfg(target_pointer_width = "32")] + pub dlpi_phnum: Elf32_Half, + + // These fields were added in Android R + pub dlpi_adds: ::c_ulonglong, + pub dlpi_subs: ::c_ulonglong, + pub dlpi_tls_modid: ::size_t, + pub dlpi_tls_data: *mut ::c_void, + } + + // linux/filter.h + pub struct sock_filter { + pub code: ::__u16, + pub jt: ::__u8, + pub jf: ::__u8, + pub k: ::__u32, + } + + pub struct sock_fprog { + pub len: ::c_ushort, + pub filter: *mut sock_filter, + } + + // linux/seccomp.h + pub struct seccomp_data { + pub nr: ::c_int, + pub arch: ::__u32, + pub instruction_pointer: ::__u64, + pub args: [::__u64; 6], + } + + pub struct ptrace_peeksiginfo_args { + pub off: ::__u64, + pub flags: ::__u32, + pub nr: ::__s32, + } + + // linux/input.h + pub struct input_event { + pub time: ::timeval, + pub type_: ::__u16, + pub code: ::__u16, + pub value: ::__s32, + } + + pub struct input_id { + pub bustype: ::__u16, + pub vendor: ::__u16, + pub product: ::__u16, + pub version: ::__u16, + } + + pub struct input_absinfo { + pub value: ::__s32, + pub minimum: ::__s32, + pub maximum: ::__s32, + pub fuzz: ::__s32, + pub flat: ::__s32, + pub resolution: ::__s32, + } + + pub struct input_keymap_entry { + pub flags: ::__u8, + pub len: ::__u8, + pub index: ::__u16, + pub keycode: ::__u32, + pub scancode: [::__u8; 32], + } + + pub struct input_mask { + pub type_: ::__u32, + pub codes_size: ::__u32, + pub codes_ptr: ::__u64, + } + + pub struct ff_replay { + pub length: ::__u16, + pub delay: ::__u16, + } + + pub struct ff_trigger { + pub button: ::__u16, + pub interval: ::__u16, + } + + pub struct ff_envelope { + pub attack_length: ::__u16, + pub attack_level: ::__u16, + pub fade_length: ::__u16, + pub fade_level: ::__u16, + } + + pub struct ff_constant_effect { + pub level: ::__s16, + pub envelope: ff_envelope, + } + + pub struct ff_ramp_effect { + pub start_level: ::__s16, + pub end_level: ::__s16, + pub envelope: ff_envelope, + } + + pub struct ff_condition_effect { + pub right_saturation: ::__u16, + pub left_saturation: ::__u16, + + pub right_coeff: ::__s16, + pub left_coeff: ::__s16, + + pub deadband: ::__u16, + pub center: ::__s16, + } + + pub struct ff_periodic_effect { + pub waveform: ::__u16, + pub period: ::__u16, + pub magnitude: ::__s16, + pub offset: ::__s16, + pub phase: ::__u16, + + pub envelope: ff_envelope, + + pub custom_len: ::__u32, + pub custom_data: *mut ::__s16, + } + + pub struct ff_rumble_effect { + pub strong_magnitude: ::__u16, + pub weak_magnitude: ::__u16, + } + + pub struct ff_effect { + pub type_: ::__u16, + pub id: ::__s16, + pub direction: ::__u16, + pub trigger: ff_trigger, + pub replay: ff_replay, + // FIXME this is actually a union + #[cfg(target_pointer_width = "64")] + pub u: [u64; 4], + #[cfg(target_pointer_width = "32")] + pub u: [u32; 7], + } + + // linux/uinput.h + pub struct uinput_ff_upload { + pub request_id: ::__u32, + pub retval: ::__s32, + pub effect: ff_effect, + pub old: ff_effect, + } + + pub struct uinput_ff_erase { + pub request_id: ::__u32, + pub retval: ::__s32, + pub effect_id: ::__u32, + } + + pub struct uinput_abs_setup { + pub code: ::__u16, + pub absinfo: input_absinfo, + } + + pub struct option { + pub name: *const ::c_char, + pub has_arg: ::c_int, + pub flag: *mut ::c_int, + pub val: ::c_int, + } } s_no_extra_traits! { @@ -270,10 +551,39 @@ s_no_extra_traits! { pub salg_name: [::c_uchar; 64], } + pub struct uinput_setup { + pub id: input_id, + pub name: [::c_char; UINPUT_MAX_NAME_SIZE], + pub ff_effects_max: ::__u32, + } + + pub struct uinput_user_dev { + pub name: [::c_char; UINPUT_MAX_NAME_SIZE], + pub id: input_id, + pub ff_effects_max: ::__u32, + pub absmax: [::__s32; ABS_CNT], + pub absmin: [::__s32; ABS_CNT], + pub absfuzz: [::__s32; ABS_CNT], + pub absflat: [::__s32; ABS_CNT], + } + + /// WARNING: The `PartialEq`, `Eq` and `Hash` implementations of this + /// type are unsound and will be removed in the future. + #[deprecated( + note = "this struct has unsafe trait implementations that will be \ + removed in the future", + since = "0.2.80" + )] pub struct af_alg_iv { pub ivlen: u32, pub iv: [::c_uchar; 0], } + + pub struct prop_info { + __name: [::c_char; 32], + __serial: ::c_uint, + __value: [[::c_char; 4]; 23], + } } cfg_if! { @@ -555,6 +865,73 @@ cfg_if! { } } + impl PartialEq for uinput_setup { + fn eq(&self, other: &uinput_setup) -> bool { + self.id == other.id + && self.name[..] == other.name[..] + && self.ff_effects_max == other.ff_effects_max + } + } + impl Eq for uinput_setup {} + + impl ::fmt::Debug for uinput_setup { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("uinput_setup") + .field("id", &self.id) + .field("name", &&self.name[..]) + .field("ff_effects_max", &self.ff_effects_max) + .finish() + } + } + + impl ::hash::Hash for uinput_setup { + fn hash(&self, state: &mut H) { + self.id.hash(state); + self.name.hash(state); + self.ff_effects_max.hash(state); + } + } + + impl PartialEq for uinput_user_dev { + fn eq(&self, other: &uinput_user_dev) -> bool { + self.name[..] == other.name[..] + && self.id == other.id + && self.ff_effects_max == other.ff_effects_max + && self.absmax[..] == other.absmax[..] + && self.absmin[..] == other.absmin[..] + && self.absfuzz[..] == other.absfuzz[..] + && self.absflat[..] == other.absflat[..] + } + } + impl Eq for uinput_user_dev {} + + impl ::fmt::Debug for uinput_user_dev { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("uinput_setup") + .field("name", &&self.name[..]) + .field("id", &self.id) + .field("ff_effects_max", &self.ff_effects_max) + .field("absmax", &&self.absmax[..]) + .field("absmin", &&self.absmin[..]) + .field("absfuzz", &&self.absfuzz[..]) + .field("absflat", &&self.absflat[..]) + .finish() + } + } + + impl ::hash::Hash for uinput_user_dev { + fn hash(&self, state: &mut H) { + self.name.hash(state); + self.id.hash(state); + self.ff_effects_max.hash(state); + self.absmax.hash(state); + self.absmin.hash(state); + self.absfuzz.hash(state); + self.absflat.hash(state); + } + } + + #[allow(deprecated)] impl af_alg_iv { fn as_slice(&self) -> &[u8] { unsafe { @@ -566,27 +943,49 @@ cfg_if! { } } + #[allow(deprecated)] impl PartialEq for af_alg_iv { fn eq(&self, other: &af_alg_iv) -> bool { *self.as_slice() == *other.as_slice() } } + #[allow(deprecated)] impl Eq for af_alg_iv {} + #[allow(deprecated)] impl ::fmt::Debug for af_alg_iv { fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { f.debug_struct("af_alg_iv") - .field("iv", &self.as_slice()) + .field("ivlen", &self.ivlen) .finish() } } + #[allow(deprecated)] impl ::hash::Hash for af_alg_iv { fn hash(&self, state: &mut H) { self.as_slice().hash(state); } } + + impl PartialEq for prop_info { + fn eq(&self, other: &prop_info) -> bool { + self.__name == other.__name && + self.__serial == other.__serial && + self.__value == other.__value + } + } + impl Eq for prop_info {} + impl ::fmt::Debug for prop_info { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("prop_info") + .field("__name", &self.__name) + .field("__serial", &self.__serial) + .field("__value", &self.__value) + .finish() + } + } } } @@ -615,15 +1014,30 @@ pub const EPROTO: ::c_int = 71; pub const EDOTDOT: ::c_int = 73; pub const EPOLL_CLOEXEC: ::c_int = 0x80000; -pub const EPOLLONESHOT: ::c_int = 0x40000000; -pub const EPOLLRDHUP: ::c_int = 0x00002000; -pub const EPOLLWAKEUP: ::c_int = 0x20000000; -pub const EFD_CLOEXEC: ::c_int = 0x80000; +// sys/eventfd.h +pub const EFD_SEMAPHORE: ::c_int = 0x1; +pub const EFD_CLOEXEC: ::c_int = O_CLOEXEC; +pub const EFD_NONBLOCK: ::c_int = O_NONBLOCK; + +// sys/timerfd.h +pub const TFD_CLOEXEC: ::c_int = O_CLOEXEC; +pub const TFD_NONBLOCK: ::c_int = O_NONBLOCK; +pub const TFD_TIMER_ABSTIME: ::c_int = 1; +pub const TFD_TIMER_CANCEL_ON_SET: ::c_int = 2; pub const USER_PROCESS: ::c_short = 7; +pub const _POSIX_VDISABLE: ::cc_t = 0; + +// linux/falloc.h +pub const FALLOC_FL_KEEP_SIZE: ::c_int = 0x01; +pub const FALLOC_FL_PUNCH_HOLE: ::c_int = 0x02; +pub const FALLOC_FL_NO_HIDE_STALE: ::c_int = 0x04; pub const FALLOC_FL_COLLAPSE_RANGE: ::c_int = 0x08; +pub const FALLOC_FL_ZERO_RANGE: ::c_int = 0x10; +pub const FALLOC_FL_INSERT_RANGE: ::c_int = 0x20; +pub const FALLOC_FL_UNSHARE_RANGE: ::c_int = 0x40; pub const BUFSIZ: ::c_uint = 1024; pub const FILENAME_MAX: ::c_uint = 4096; @@ -789,11 +1203,27 @@ pub const _SC_V7_LPBIG_OFFBIG: ::c_int = 140; pub const _SC_XOPEN_STREAMS: ::c_int = 141; pub const _SC_XOPEN_UUCP: ::c_int = 142; +pub const F_LOCK: ::c_int = 1; +pub const F_TEST: ::c_int = 3; +pub const F_TLOCK: ::c_int = 2; +pub const F_ULOCK: ::c_int = 0; + +pub const F_SEAL_FUTURE_WRITE: ::c_int = 0x0010; + +pub const IFF_LOWER_UP: ::c_int = 0x10000; +pub const IFF_DORMANT: ::c_int = 0x20000; +pub const IFF_ECHO: ::c_int = 0x40000; + pub const PTHREAD_MUTEX_NORMAL: ::c_int = 0; pub const PTHREAD_MUTEX_RECURSIVE: ::c_int = 1; pub const PTHREAD_MUTEX_ERRORCHECK: ::c_int = 2; pub const PTHREAD_MUTEX_DEFAULT: ::c_int = PTHREAD_MUTEX_NORMAL; +// stdio.h +pub const RENAME_NOREPLACE: ::c_int = 1; +pub const RENAME_EXCHANGE: ::c_int = 2; +pub const RENAME_WHITEOUT: ::c_int = 4; + pub const FIOCLEX: ::c_int = 0x5451; pub const FIONCLEX: ::c_int = 0x5450; @@ -951,6 +1381,8 @@ pub const SOCK_SEQPACKET: ::c_int = 5; pub const SOCK_DCCP: ::c_int = 6; pub const SOCK_PACKET: ::c_int = 10; +pub const IPPROTO_MAX: ::c_int = 256; + pub const SOL_SOCKET: ::c_int = 1; pub const SOL_SCTP: ::c_int = 132; pub const SOL_IPX: ::c_int = 256; @@ -1000,10 +1432,15 @@ pub const SO_SNDLOWAT: ::c_int = 19; pub const SO_RCVTIMEO: ::c_int = 20; pub const SO_SNDTIMEO: ::c_int = 21; pub const SO_BINDTODEVICE: ::c_int = 25; +pub const SO_ATTACH_FILTER: ::c_int = 26; +pub const SO_DETACH_FILTER: ::c_int = 27; +pub const SO_GET_FILTER: ::c_int = SO_ATTACH_FILTER; pub const SO_TIMESTAMP: ::c_int = 29; pub const SO_ACCEPTCONN: ::c_int = 30; +pub const SO_PEERSEC: ::c_int = 31; pub const SO_SNDBUFFORCE: ::c_int = 32; pub const SO_RCVBUFFORCE: ::c_int = 33; +pub const SO_PASSSEC: ::c_int = 34; pub const SO_MARK: ::c_int = 36; pub const SO_PROTOCOL: ::c_int = 38; pub const SO_DOMAIN: ::c_int = 39; @@ -1023,8 +1460,16 @@ pub const O_SYNC: ::c_int = 0x101000; pub const O_ASYNC: ::c_int = 0x2000; pub const O_NDELAY: ::c_int = 0x800; pub const O_DSYNC: ::c_int = 4096; +pub const O_RSYNC: ::c_int = O_SYNC; pub const NI_MAXHOST: ::size_t = 1025; +pub const NI_MAXSERV: ::size_t = 32; + +pub const NI_NOFQDN: ::c_int = 0x00000001; +pub const NI_NUMERICHOST: ::c_int = 0x00000002; +pub const NI_NAMEREQD: ::c_int = 0x00000004; +pub const NI_NUMERICSERV: ::c_int = 0x00000008; +pub const NI_DGRAM: ::c_int = 0x00000010; pub const NCCS: usize = 19; pub const TCSBRKP: ::c_int = 0x5425; @@ -1040,33 +1485,6 @@ pub const TOSTOP: ::tcflag_t = 0x00000100; pub const FLUSHO: ::tcflag_t = 0x00001000; pub const EXTPROC: ::tcflag_t = 0o200000; -pub const ADFS_SUPER_MAGIC: ::c_long = 0x0000adf5; -pub const AFFS_SUPER_MAGIC: ::c_long = 0x0000adff; -pub const CODA_SUPER_MAGIC: ::c_long = 0x73757245; -pub const CRAMFS_MAGIC: ::c_long = 0x28cd3d45; -pub const EFS_SUPER_MAGIC: ::c_long = 0x00414a53; -pub const EXT2_SUPER_MAGIC: ::c_long = 0x0000ef53; -pub const EXT3_SUPER_MAGIC: ::c_long = 0x0000ef53; -pub const EXT4_SUPER_MAGIC: ::c_long = 0x0000ef53; -pub const HPFS_SUPER_MAGIC: ::c_long = 0xf995e849; -pub const HUGETLBFS_MAGIC: ::c_long = 0x958458f6; -pub const ISOFS_SUPER_MAGIC: ::c_long = 0x00009660; -pub const JFFS2_SUPER_MAGIC: ::c_long = 0x000072b6; -pub const MINIX_SUPER_MAGIC: ::c_long = 0x0000137f; -pub const MINIX_SUPER_MAGIC2: ::c_long = 0x0000138f; -pub const MINIX2_SUPER_MAGIC: ::c_long = 0x00002468; -pub const MINIX2_SUPER_MAGIC2: ::c_long = 0x00002478; -pub const MSDOS_SUPER_MAGIC: ::c_long = 0x00004d44; -pub const NCP_SUPER_MAGIC: ::c_long = 0x0000564c; -pub const NFS_SUPER_MAGIC: ::c_long = 0x00006969; -pub const OPENPROM_SUPER_MAGIC: ::c_long = 0x00009fa1; -pub const PROC_SUPER_MAGIC: ::c_long = 0x00009fa0; -pub const QNX4_SUPER_MAGIC: ::c_long = 0x0000002f; -pub const REISERFS_SUPER_MAGIC: ::c_long = 0x52654973; -pub const SMB_SUPER_MAGIC: ::c_long = 0x0000517b; -pub const TMPFS_MAGIC: ::c_long = 0x01021994; -pub const USBDEVICE_SUPER_MAGIC: ::c_long = 0x00009fa2; - pub const MAP_HUGETLB: ::c_int = 0x040000; pub const PTRACE_TRACEME: ::c_int = 0; @@ -1079,6 +1497,8 @@ pub const PTRACE_POKEUSER: ::c_int = 6; pub const PTRACE_CONT: ::c_int = 7; pub const PTRACE_KILL: ::c_int = 8; pub const PTRACE_SINGLESTEP: ::c_int = 9; +pub const PTRACE_GETREGS: ::c_int = 12; +pub const PTRACE_SETREGS: ::c_int = 13; pub const PTRACE_ATTACH: ::c_int = 16; pub const PTRACE_DETACH: ::c_int = 17; pub const PTRACE_SYSCALL: ::c_int = 24; @@ -1086,8 +1506,10 @@ pub const PTRACE_SETOPTIONS: ::c_int = 0x4200; pub const PTRACE_GETEVENTMSG: ::c_int = 0x4201; pub const PTRACE_GETSIGINFO: ::c_int = 0x4202; pub const PTRACE_SETSIGINFO: ::c_int = 0x4203; +pub const PTRACE_GETREGSET: ::c_int = 0x4204; +pub const PTRACE_SETREGSET: ::c_int = 0x4205; -pub const EFD_NONBLOCK: ::c_int = 0x800; +pub const PTRACE_EVENT_STOP: ::c_int = 128; pub const F_GETLK: ::c_int = 5; pub const F_GETOWN: ::c_int = 9; @@ -1106,18 +1528,28 @@ pub const RLIMIT_FSIZE: ::c_int = 1; pub const RLIMIT_DATA: ::c_int = 2; pub const RLIMIT_STACK: ::c_int = 3; pub const RLIMIT_CORE: ::c_int = 4; +pub const RLIMIT_RSS: ::c_int = 5; +pub const RLIMIT_NPROC: ::c_int = 6; +pub const RLIMIT_NOFILE: ::c_int = 7; +pub const RLIMIT_MEMLOCK: ::c_int = 8; +pub const RLIMIT_AS: ::c_int = 9; pub const RLIMIT_LOCKS: ::c_int = 10; pub const RLIMIT_SIGPENDING: ::c_int = 11; pub const RLIMIT_MSGQUEUE: ::c_int = 12; pub const RLIMIT_NICE: ::c_int = 13; pub const RLIMIT_RTPRIO: ::c_int = 14; +pub const RLIM_NLIMITS: ::c_int = 16; pub const RLIM_INFINITY: ::rlim_t = !0; pub const TCGETS: ::c_int = 0x5401; pub const TCSETS: ::c_int = 0x5402; pub const TCSETSW: ::c_int = 0x5403; pub const TCSETSF: ::c_int = 0x5404; +pub const TCGETS2: ::c_int = 0x802c542a; +pub const TCSETS2: ::c_int = 0x402c542b; +pub const TCSETSW2: ::c_int = 0x402c542c; +pub const TCSETSF2: ::c_int = 0x402c542d; pub const TCGETA: ::c_int = 0x5405; pub const TCSETA: ::c_int = 0x5406; pub const TCSETAW: ::c_int = 0x5407; @@ -1145,6 +1577,8 @@ pub const TIOCMBIC: ::c_int = 0x5417; pub const TIOCMSET: ::c_int = 0x5418; pub const FIONREAD: ::c_int = 0x541B; pub const TIOCCONS: ::c_int = 0x541D; +pub const TIOCSBRK: ::c_int = 0x5427; +pub const TIOCCBRK: ::c_int = 0x5428; pub const ST_RDONLY: ::c_ulong = 1; pub const ST_NOSUID: ::c_ulong = 2; @@ -1164,11 +1598,8 @@ pub const AI_PASSIVE: ::c_int = 0x00000001; pub const AI_CANONNAME: ::c_int = 0x00000002; pub const AI_NUMERICHOST: ::c_int = 0x00000004; pub const AI_NUMERICSERV: ::c_int = 0x00000008; -pub const AI_MASK: ::c_int = AI_PASSIVE - | AI_CANONNAME - | AI_NUMERICHOST - | AI_NUMERICSERV - | AI_ADDRCONFIG; +pub const AI_MASK: ::c_int = + AI_PASSIVE | AI_CANONNAME | AI_NUMERICHOST | AI_NUMERICSERV | AI_ADDRCONFIG; pub const AI_ALL: ::c_int = 0x00000100; pub const AI_V4MAPPED_CFG: ::c_int = 0x00000200; pub const AI_ADDRCONFIG: ::c_int = 0x00000400; @@ -1190,6 +1621,41 @@ pub const LINUX_REBOOT_CMD_RESTART2: ::c_int = 0xA1B2C3D4; pub const LINUX_REBOOT_CMD_SW_SUSPEND: ::c_int = 0xD000FCE2; pub const LINUX_REBOOT_CMD_KEXEC: ::c_int = 0x45584543; +pub const REG_BASIC: ::c_int = 0; +pub const REG_EXTENDED: ::c_int = 1; +pub const REG_ICASE: ::c_int = 2; +pub const REG_NOSUB: ::c_int = 4; +pub const REG_NEWLINE: ::c_int = 8; +pub const REG_NOSPEC: ::c_int = 16; +pub const REG_PEND: ::c_int = 32; +pub const REG_DUMP: ::c_int = 128; + +pub const REG_NOMATCH: ::c_int = 1; +pub const REG_BADPAT: ::c_int = 2; +pub const REG_ECOLLATE: ::c_int = 3; +pub const REG_ECTYPE: ::c_int = 4; +pub const REG_EESCAPE: ::c_int = 5; +pub const REG_ESUBREG: ::c_int = 6; +pub const REG_EBRACK: ::c_int = 7; +pub const REG_EPAREN: ::c_int = 8; +pub const REG_EBRACE: ::c_int = 9; +pub const REG_BADBR: ::c_int = 10; +pub const REG_ERANGE: ::c_int = 11; +pub const REG_ESPACE: ::c_int = 12; +pub const REG_BADRPT: ::c_int = 13; +pub const REG_EMPTY: ::c_int = 14; +pub const REG_ASSERT: ::c_int = 15; +pub const REG_INVARG: ::c_int = 16; +pub const REG_ATOI: ::c_int = 255; +pub const REG_ITOA: ::c_int = 256; + +pub const REG_NOTBOL: ::c_int = 1; +pub const REG_NOTEOL: ::c_int = 2; +pub const REG_STARTEND: ::c_int = 4; +pub const REG_TRACE: ::c_int = 256; +pub const REG_LARGE: ::c_int = 512; +pub const REG_BACKR: ::c_int = 1024; + pub const MCL_CURRENT: ::c_int = 0x0001; pub const MCL_FUTURE: ::c_int = 0x0002; @@ -1277,6 +1743,12 @@ pub const B2500000: ::speed_t = 0o010014; pub const B3000000: ::speed_t = 0o010015; pub const B3500000: ::speed_t = 0o010016; pub const B4000000: ::speed_t = 0o010017; +pub const IBSHIFT: ::tcflag_t = 16; + +pub const BLKIOMIN: ::c_int = 0x1278; +pub const BLKIOOPT: ::c_int = 0x1279; +pub const BLKSSZGET: ::c_int = 0x1268; +pub const BLKPBSZGET: ::c_int = 0x127B; pub const EAI_AGAIN: ::c_int = 2; pub const EAI_BADFLAGS: ::c_int = 3; @@ -1530,14 +2002,39 @@ pub const NETLINK_BROADCAST_ERROR: ::c_int = 4; pub const NETLINK_NO_ENOBUFS: ::c_int = 5; pub const NETLINK_RX_RING: ::c_int = 6; pub const NETLINK_TX_RING: ::c_int = 7; +pub const NETLINK_LISTEN_ALL_NSID: ::c_int = 8; +pub const NETLINK_LIST_MEMBERSHIPS: ::c_int = 9; +pub const NETLINK_CAP_ACK: ::c_int = 10; +pub const NETLINK_EXT_ACK: ::c_int = 11; +pub const NETLINK_GET_STRICT_CHK: ::c_int = 12; pub const GRND_NONBLOCK: ::c_uint = 0x0001; pub const GRND_RANDOM: ::c_uint = 0x0002; +pub const GRND_INSECURE: ::c_uint = 0x0004; pub const SECCOMP_MODE_DISABLED: ::c_uint = 0; pub const SECCOMP_MODE_STRICT: ::c_uint = 1; pub const SECCOMP_MODE_FILTER: ::c_uint = 2; +pub const SECCOMP_FILTER_FLAG_TSYNC: ::c_ulong = 1; +pub const SECCOMP_FILTER_FLAG_LOG: ::c_ulong = 2; +pub const SECCOMP_FILTER_FLAG_SPEC_ALLOW: ::c_ulong = 4; +pub const SECCOMP_FILTER_FLAG_NEW_LISTENER: ::c_ulong = 8; + +pub const SECCOMP_RET_ACTION_FULL: ::c_uint = 0xffff0000; +pub const SECCOMP_RET_ACTION: ::c_uint = 0x7fff0000; +pub const SECCOMP_RET_DATA: ::c_uint = 0x0000ffff; + +pub const SECCOMP_RET_KILL_PROCESS: ::c_uint = 0x80000000; +pub const SECCOMP_RET_KILL_THREAD: ::c_uint = 0x00000000; +pub const SECCOMP_RET_KILL: ::c_uint = SECCOMP_RET_KILL_THREAD; +pub const SECCOMP_RET_TRAP: ::c_uint = 0x00030000; +pub const SECCOMP_RET_ERRNO: ::c_uint = 0x00050000; +pub const SECCOMP_RET_USER_NOTIF: ::c_uint = 0x7fc00000; +pub const SECCOMP_RET_TRACE: ::c_uint = 0x7ff00000; +pub const SECCOMP_RET_LOG: ::c_uint = 0x7ffc0000; +pub const SECCOMP_RET_ALLOW: ::c_uint = 0x7fff0000; + pub const NLA_F_NESTED: ::c_int = 1 << 15; pub const NLA_F_NET_BYTEORDER: ::c_int = 1 << 14; pub const NLA_TYPE_MASK: ::c_int = !(NLA_F_NESTED | NLA_F_NET_BYTEORDER); @@ -1570,12 +2067,16 @@ pub const SFD_NONBLOCK: ::c_int = O_NONBLOCK; pub const SOCK_NONBLOCK: ::c_int = O_NONBLOCK; pub const SO_ORIGINAL_DST: ::c_int = 80; -pub const IP_ORIGDSTADDR: ::c_int = 20; -pub const IP_RECVORIGDSTADDR: ::c_int = IP_ORIGDSTADDR; -pub const IPV6_ORIGDSTADDR: ::c_int = 74; -pub const IPV6_RECVORIGDSTADDR: ::c_int = IPV6_ORIGDSTADDR; + +pub const IP_RECVFRAGSIZE: ::c_int = 25; + +pub const IPV6_FLOWINFO: ::c_int = 11; +pub const IPV6_MULTICAST_ALL: ::c_int = 29; +pub const IPV6_ROUTER_ALERT_ISOLATE: ::c_int = 30; pub const IPV6_FLOWLABEL_MGR: ::c_int = 32; pub const IPV6_FLOWINFO_SEND: ::c_int = 33; +pub const IPV6_RECVFRAGSIZE: ::c_int = 77; +pub const IPV6_FREEBIND: ::c_int = 78; pub const IPV6_FLOWINFO_FLOWLABEL: ::c_int = 0x000fffff; pub const IPV6_FLOWINFO_PRIORITY: ::c_int = 0x0ff00000; @@ -1586,6 +2087,39 @@ pub const O_TMPFILE: ::c_int = 0o20000000 | O_DIRECTORY; pub const MFD_CLOEXEC: ::c_uint = 0x0001; pub const MFD_ALLOW_SEALING: ::c_uint = 0x0002; pub const MFD_HUGETLB: ::c_uint = 0x0004; +pub const MFD_HUGE_64KB: ::c_uint = 0x40000000; +pub const MFD_HUGE_512KB: ::c_uint = 0x4c000000; +pub const MFD_HUGE_1MB: ::c_uint = 0x50000000; +pub const MFD_HUGE_2MB: ::c_uint = 0x54000000; +pub const MFD_HUGE_8MB: ::c_uint = 0x5c000000; +pub const MFD_HUGE_16MB: ::c_uint = 0x60000000; +pub const MFD_HUGE_32MB: ::c_uint = 0x64000000; +pub const MFD_HUGE_256MB: ::c_uint = 0x70000000; +pub const MFD_HUGE_512MB: ::c_uint = 0x74000000; +pub const MFD_HUGE_1GB: ::c_uint = 0x78000000; +pub const MFD_HUGE_2GB: ::c_uint = 0x7c000000; +pub const MFD_HUGE_16GB: ::c_uint = 0x88000000; +pub const MFD_HUGE_MASK: ::c_uint = 63; +pub const MFD_HUGE_SHIFT: ::c_uint = 26; + +// these are used in the p_type field of Elf32_Phdr and Elf64_Phdr, which has +// the type Elf32Word and Elf64Word respectively. Luckily, both of those are u32 +// so we can use that type here to avoid having to cast. +pub const PT_NULL: u32 = 0; +pub const PT_LOAD: u32 = 1; +pub const PT_DYNAMIC: u32 = 2; +pub const PT_INTERP: u32 = 3; +pub const PT_NOTE: u32 = 4; +pub const PT_SHLIB: u32 = 5; +pub const PT_PHDR: u32 = 6; +pub const PT_TLS: u32 = 7; +pub const PT_LOOS: u32 = 0x60000000; +pub const PT_GNU_EH_FRAME: u32 = 0x6474e550; +pub const PT_GNU_STACK: u32 = 0x6474e551; +pub const PT_GNU_RELRO: u32 = 0x6474e552; +pub const PT_HIOS: u32 = 0x6fffffff; +pub const PT_LOPROC: u32 = 0x70000000; +pub const PT_HIPROC: u32 = 0x7fffffff; // linux/netfilter.h pub const NF_DROP: ::c_int = 0; @@ -1855,6 +2389,38 @@ pub const NFT_TRACETYPE_RULE: ::c_int = 3; pub const NFT_NG_INCREMENTAL: ::c_int = 0; pub const NFT_NG_RANDOM: ::c_int = 1; +// linux/input.h +pub const FF_MAX: ::__u16 = 0x7f; +pub const FF_CNT: usize = FF_MAX as usize + 1; + +// linux/input-event-codes.h +pub const INPUT_PROP_MAX: ::__u16 = 0x1f; +pub const INPUT_PROP_CNT: usize = INPUT_PROP_MAX as usize + 1; +pub const EV_MAX: ::__u16 = 0x1f; +pub const EV_CNT: usize = EV_MAX as usize + 1; +pub const SYN_MAX: ::__u16 = 0xf; +pub const SYN_CNT: usize = SYN_MAX as usize + 1; +pub const KEY_MAX: ::__u16 = 0x2ff; +pub const KEY_CNT: usize = KEY_MAX as usize + 1; +pub const REL_MAX: ::__u16 = 0x0f; +pub const REL_CNT: usize = REL_MAX as usize + 1; +pub const ABS_MAX: ::__u16 = 0x3f; +pub const ABS_CNT: usize = ABS_MAX as usize + 1; +pub const SW_MAX: ::__u16 = 0x0f; +pub const SW_CNT: usize = SW_MAX as usize + 1; +pub const MSC_MAX: ::__u16 = 0x07; +pub const MSC_CNT: usize = MSC_MAX as usize + 1; +pub const LED_MAX: ::__u16 = 0x0f; +pub const LED_CNT: usize = LED_MAX as usize + 1; +pub const REP_MAX: ::__u16 = 0x01; +pub const REP_CNT: usize = REP_MAX as usize + 1; +pub const SND_MAX: ::__u16 = 0x07; +pub const SND_CNT: usize = SND_MAX as usize + 1; + +// linux/uinput.h +pub const UINPUT_VERSION: ::c_uint = 5; +pub const UINPUT_MAX_NAME_SIZE: usize = 80; + pub const IFF_TUN: ::c_int = 0x0001; pub const IFF_TAP: ::c_int = 0x0002; pub const IFF_NO_PI: ::c_int = 0x1000; @@ -2019,6 +2585,16 @@ pub const ALG_SET_AEAD_AUTHSIZE: ::c_int = 5; pub const ALG_OP_DECRYPT: ::c_int = 0; pub const ALG_OP_ENCRYPT: ::c_int = 1; +// sys/mman.h +pub const MLOCK_ONFAULT: ::c_int = 0x01; + +// uapi/linux/vm_sockets.h +pub const VMADDR_CID_ANY: ::c_uint = 0xFFFFFFFF; +pub const VMADDR_CID_HYPERVISOR: ::c_uint = 0; +pub const VMADDR_CID_LOCAL: ::c_uint = 1; +pub const VMADDR_CID_HOST: ::c_uint = 2; +pub const VMADDR_PORT_ANY: ::c_uint = 0xFFFFFFFF; + // uapi/linux/inotify.h pub const IN_ACCESS: u32 = 0x0000_0001; pub const IN_MODIFY: u32 = 0x0000_0002; @@ -2078,8 +2654,319 @@ pub const FUTEX_CMP_REQUEUE_PI: ::c_int = 12; pub const FUTEX_PRIVATE_FLAG: ::c_int = 128; pub const FUTEX_CLOCK_REALTIME: ::c_int = 256; -pub const FUTEX_CMD_MASK: ::c_int = - !(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME); +pub const FUTEX_CMD_MASK: ::c_int = !(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME); + +// linux/errqueue.h +pub const SO_EE_ORIGIN_NONE: u8 = 0; +pub const SO_EE_ORIGIN_LOCAL: u8 = 1; +pub const SO_EE_ORIGIN_ICMP: u8 = 2; +pub const SO_EE_ORIGIN_ICMP6: u8 = 3; +pub const SO_EE_ORIGIN_TXSTATUS: u8 = 4; +pub const SO_EE_ORIGIN_TIMESTAMPING: u8 = SO_EE_ORIGIN_TXSTATUS; + +// errno.h +pub const EPERM: ::c_int = 1; +pub const ENOENT: ::c_int = 2; +pub const ESRCH: ::c_int = 3; +pub const EINTR: ::c_int = 4; +pub const EIO: ::c_int = 5; +pub const ENXIO: ::c_int = 6; +pub const E2BIG: ::c_int = 7; +pub const ENOEXEC: ::c_int = 8; +pub const EBADF: ::c_int = 9; +pub const ECHILD: ::c_int = 10; +pub const EAGAIN: ::c_int = 11; +pub const ENOMEM: ::c_int = 12; +pub const EACCES: ::c_int = 13; +pub const EFAULT: ::c_int = 14; +pub const ENOTBLK: ::c_int = 15; +pub const EBUSY: ::c_int = 16; +pub const EEXIST: ::c_int = 17; +pub const EXDEV: ::c_int = 18; +pub const ENODEV: ::c_int = 19; +pub const ENOTDIR: ::c_int = 20; +pub const EISDIR: ::c_int = 21; +pub const EINVAL: ::c_int = 22; +pub const ENFILE: ::c_int = 23; +pub const EMFILE: ::c_int = 24; +pub const ENOTTY: ::c_int = 25; +pub const ETXTBSY: ::c_int = 26; +pub const EFBIG: ::c_int = 27; +pub const ENOSPC: ::c_int = 28; +pub const ESPIPE: ::c_int = 29; +pub const EROFS: ::c_int = 30; +pub const EMLINK: ::c_int = 31; +pub const EPIPE: ::c_int = 32; +pub const EDOM: ::c_int = 33; +pub const ERANGE: ::c_int = 34; +pub const EWOULDBLOCK: ::c_int = EAGAIN; + +pub const PRIO_PROCESS: ::c_int = 0; +pub const PRIO_PGRP: ::c_int = 1; +pub const PRIO_USER: ::c_int = 2; + +// linux/sched.h +pub const SCHED_NORMAL: ::c_int = 0; +pub const SCHED_FIFO: ::c_int = 1; +pub const SCHED_RR: ::c_int = 2; +pub const SCHED_BATCH: ::c_int = 3; +pub const SCHED_IDLE: ::c_int = 5; +pub const SCHED_DEADLINE: ::c_int = 6; + +pub const SCHED_RESET_ON_FORK: ::c_int = 0x40000000; + +pub const CLONE_PIDFD: ::c_int = 0x1000; + +// linux/mempolicy.h +pub const MPOL_DEFAULT: ::c_int = 0; +pub const MPOL_PREFERRED: ::c_int = 1; +pub const MPOL_BIND: ::c_int = 2; +pub const MPOL_INTERLEAVE: ::c_int = 3; +pub const MPOL_LOCAL: ::c_int = 4; +pub const MPOL_F_NUMA_BALANCING: ::c_int = 1 << 13; +pub const MPOL_F_RELATIVE_NODES: ::c_int = 1 << 14; +pub const MPOL_F_STATIC_NODES: ::c_int = 1 << 15; + +// bits/seek_constants.h +pub const SEEK_DATA: ::c_int = 3; +pub const SEEK_HOLE: ::c_int = 4; + +// sys/socket.h +pub const AF_NFC: ::c_int = 39; +pub const AF_VSOCK: ::c_int = 40; +pub const PF_NFC: ::c_int = AF_NFC; +pub const PF_VSOCK: ::c_int = AF_VSOCK; + +pub const SOMAXCONN: ::c_int = 128; + +// sys/system_properties.h +pub const PROP_VALUE_MAX: ::c_int = 92; +pub const PROP_NAME_MAX: ::c_int = 32; + +// sys/prctl.h +pub const PR_SET_VMA: ::c_int = 0x53564d41; +pub const PR_SET_VMA_ANON_NAME: ::c_int = 0; +pub const PR_SET_NO_NEW_PRIVS: ::c_int = 38; +pub const PR_GET_NO_NEW_PRIVS: ::c_int = 39; +pub const PR_GET_SECCOMP: ::c_int = 21; +pub const PR_SET_SECCOMP: ::c_int = 22; +pub const PR_GET_TIMING: ::c_int = 13; +pub const PR_SET_TIMING: ::c_int = 14; +pub const PR_TIMING_STATISTICAL: ::c_int = 0; +pub const PR_TIMING_TIMESTAMP: ::c_int = 1; + +// linux/if_addr.h +pub const IFA_UNSPEC: ::c_ushort = 0; +pub const IFA_ADDRESS: ::c_ushort = 1; +pub const IFA_LOCAL: ::c_ushort = 2; +pub const IFA_LABEL: ::c_ushort = 3; +pub const IFA_BROADCAST: ::c_ushort = 4; +pub const IFA_ANYCAST: ::c_ushort = 5; +pub const IFA_CACHEINFO: ::c_ushort = 6; +pub const IFA_MULTICAST: ::c_ushort = 7; + +pub const IFA_F_SECONDARY: u32 = 0x01; +pub const IFA_F_TEMPORARY: u32 = 0x01; +pub const IFA_F_NODAD: u32 = 0x02; +pub const IFA_F_OPTIMISTIC: u32 = 0x04; +pub const IFA_F_DADFAILED: u32 = 0x08; +pub const IFA_F_HOMEADDRESS: u32 = 0x10; +pub const IFA_F_DEPRECATED: u32 = 0x20; +pub const IFA_F_TENTATIVE: u32 = 0x40; +pub const IFA_F_PERMANENT: u32 = 0x80; + +// linux/if_link.h +pub const IFLA_UNSPEC: ::c_ushort = 0; +pub const IFLA_ADDRESS: ::c_ushort = 1; +pub const IFLA_BROADCAST: ::c_ushort = 2; +pub const IFLA_IFNAME: ::c_ushort = 3; +pub const IFLA_MTU: ::c_ushort = 4; +pub const IFLA_LINK: ::c_ushort = 5; +pub const IFLA_QDISC: ::c_ushort = 6; +pub const IFLA_STATS: ::c_ushort = 7; +pub const IFLA_COST: ::c_ushort = 8; +pub const IFLA_PRIORITY: ::c_ushort = 9; +pub const IFLA_MASTER: ::c_ushort = 10; +pub const IFLA_WIRELESS: ::c_ushort = 11; +pub const IFLA_PROTINFO: ::c_ushort = 12; +pub const IFLA_TXQLEN: ::c_ushort = 13; +pub const IFLA_MAP: ::c_ushort = 14; +pub const IFLA_WEIGHT: ::c_ushort = 15; +pub const IFLA_OPERSTATE: ::c_ushort = 16; +pub const IFLA_LINKMODE: ::c_ushort = 17; +pub const IFLA_LINKINFO: ::c_ushort = 18; +pub const IFLA_NET_NS_PID: ::c_ushort = 19; +pub const IFLA_IFALIAS: ::c_ushort = 20; +pub const IFLA_NUM_VF: ::c_ushort = 21; +pub const IFLA_VFINFO_LIST: ::c_ushort = 22; +pub const IFLA_STATS64: ::c_ushort = 23; +pub const IFLA_VF_PORTS: ::c_ushort = 24; +pub const IFLA_PORT_SELF: ::c_ushort = 25; +pub const IFLA_AF_SPEC: ::c_ushort = 26; +pub const IFLA_GROUP: ::c_ushort = 27; +pub const IFLA_NET_NS_FD: ::c_ushort = 28; +pub const IFLA_EXT_MASK: ::c_ushort = 29; +pub const IFLA_PROMISCUITY: ::c_ushort = 30; +pub const IFLA_NUM_TX_QUEUES: ::c_ushort = 31; +pub const IFLA_NUM_RX_QUEUES: ::c_ushort = 32; +pub const IFLA_CARRIER: ::c_ushort = 33; +pub const IFLA_PHYS_PORT_ID: ::c_ushort = 34; +pub const IFLA_CARRIER_CHANGES: ::c_ushort = 35; +pub const IFLA_PHYS_SWITCH_ID: ::c_ushort = 36; +pub const IFLA_LINK_NETNSID: ::c_ushort = 37; +pub const IFLA_PHYS_PORT_NAME: ::c_ushort = 38; +pub const IFLA_PROTO_DOWN: ::c_ushort = 39; +pub const IFLA_GSO_MAX_SEGS: ::c_ushort = 40; +pub const IFLA_GSO_MAX_SIZE: ::c_ushort = 41; +pub const IFLA_PAD: ::c_ushort = 42; +pub const IFLA_XDP: ::c_ushort = 43; +pub const IFLA_EVENT: ::c_ushort = 44; +pub const IFLA_NEW_NETNSID: ::c_ushort = 45; +pub const IFLA_IF_NETNSID: ::c_ushort = 46; +pub const IFLA_TARGET_NETNSID: ::c_ushort = IFLA_IF_NETNSID; +pub const IFLA_CARRIER_UP_COUNT: ::c_ushort = 47; +pub const IFLA_CARRIER_DOWN_COUNT: ::c_ushort = 48; +pub const IFLA_NEW_IFINDEX: ::c_ushort = 49; +pub const IFLA_MIN_MTU: ::c_ushort = 50; +pub const IFLA_MAX_MTU: ::c_ushort = 51; + +pub const IFLA_INFO_UNSPEC: ::c_ushort = 0; +pub const IFLA_INFO_KIND: ::c_ushort = 1; +pub const IFLA_INFO_DATA: ::c_ushort = 2; +pub const IFLA_INFO_XSTATS: ::c_ushort = 3; +pub const IFLA_INFO_SLAVE_KIND: ::c_ushort = 4; +pub const IFLA_INFO_SLAVE_DATA: ::c_ushort = 5; + +// linux/rtnetlink.h +pub const TCA_UNSPEC: ::c_ushort = 0; +pub const TCA_KIND: ::c_ushort = 1; +pub const TCA_OPTIONS: ::c_ushort = 2; +pub const TCA_STATS: ::c_ushort = 3; +pub const TCA_XSTATS: ::c_ushort = 4; +pub const TCA_RATE: ::c_ushort = 5; +pub const TCA_FCNT: ::c_ushort = 6; +pub const TCA_STATS2: ::c_ushort = 7; +pub const TCA_STAB: ::c_ushort = 8; + +pub const RTM_NEWLINK: u16 = 16; +pub const RTM_DELLINK: u16 = 17; +pub const RTM_GETLINK: u16 = 18; +pub const RTM_SETLINK: u16 = 19; +pub const RTM_NEWADDR: u16 = 20; +pub const RTM_DELADDR: u16 = 21; +pub const RTM_GETADDR: u16 = 22; +pub const RTM_NEWROUTE: u16 = 24; +pub const RTM_DELROUTE: u16 = 25; +pub const RTM_GETROUTE: u16 = 26; +pub const RTM_NEWNEIGH: u16 = 28; +pub const RTM_DELNEIGH: u16 = 29; +pub const RTM_GETNEIGH: u16 = 30; +pub const RTM_NEWRULE: u16 = 32; +pub const RTM_DELRULE: u16 = 33; +pub const RTM_GETRULE: u16 = 34; +pub const RTM_NEWQDISC: u16 = 36; +pub const RTM_DELQDISC: u16 = 37; +pub const RTM_GETQDISC: u16 = 38; +pub const RTM_NEWTCLASS: u16 = 40; +pub const RTM_DELTCLASS: u16 = 41; +pub const RTM_GETTCLASS: u16 = 42; +pub const RTM_NEWTFILTER: u16 = 44; +pub const RTM_DELTFILTER: u16 = 45; +pub const RTM_GETTFILTER: u16 = 46; +pub const RTM_NEWACTION: u16 = 48; +pub const RTM_DELACTION: u16 = 49; +pub const RTM_GETACTION: u16 = 50; +pub const RTM_NEWPREFIX: u16 = 52; +pub const RTM_GETMULTICAST: u16 = 58; +pub const RTM_GETANYCAST: u16 = 62; +pub const RTM_NEWNEIGHTBL: u16 = 64; +pub const RTM_GETNEIGHTBL: u16 = 66; +pub const RTM_SETNEIGHTBL: u16 = 67; +pub const RTM_NEWNDUSEROPT: u16 = 68; +pub const RTM_NEWADDRLABEL: u16 = 72; +pub const RTM_DELADDRLABEL: u16 = 73; +pub const RTM_GETADDRLABEL: u16 = 74; +pub const RTM_GETDCB: u16 = 78; +pub const RTM_SETDCB: u16 = 79; +pub const RTM_NEWNETCONF: u16 = 80; +pub const RTM_GETNETCONF: u16 = 82; +pub const RTM_NEWMDB: u16 = 84; +pub const RTM_DELMDB: u16 = 85; +pub const RTM_GETMDB: u16 = 86; +pub const RTM_NEWNSID: u16 = 88; +pub const RTM_DELNSID: u16 = 89; +pub const RTM_GETNSID: u16 = 90; + +pub const RTM_F_NOTIFY: ::c_uint = 0x100; +pub const RTM_F_CLONED: ::c_uint = 0x200; +pub const RTM_F_EQUALIZE: ::c_uint = 0x400; +pub const RTM_F_PREFIX: ::c_uint = 0x800; + +pub const RTA_UNSPEC: ::c_ushort = 0; +pub const RTA_DST: ::c_ushort = 1; +pub const RTA_SRC: ::c_ushort = 2; +pub const RTA_IIF: ::c_ushort = 3; +pub const RTA_OIF: ::c_ushort = 4; +pub const RTA_GATEWAY: ::c_ushort = 5; +pub const RTA_PRIORITY: ::c_ushort = 6; +pub const RTA_PREFSRC: ::c_ushort = 7; +pub const RTA_METRICS: ::c_ushort = 8; +pub const RTA_MULTIPATH: ::c_ushort = 9; +pub const RTA_PROTOINFO: ::c_ushort = 10; // No longer used +pub const RTA_FLOW: ::c_ushort = 11; +pub const RTA_CACHEINFO: ::c_ushort = 12; +pub const RTA_SESSION: ::c_ushort = 13; // No longer used +pub const RTA_MP_ALGO: ::c_ushort = 14; // No longer used +pub const RTA_TABLE: ::c_ushort = 15; +pub const RTA_MARK: ::c_ushort = 16; +pub const RTA_MFC_STATS: ::c_ushort = 17; + +pub const RTN_UNSPEC: ::c_uchar = 0; +pub const RTN_UNICAST: ::c_uchar = 1; +pub const RTN_LOCAL: ::c_uchar = 2; +pub const RTN_BROADCAST: ::c_uchar = 3; +pub const RTN_ANYCAST: ::c_uchar = 4; +pub const RTN_MULTICAST: ::c_uchar = 5; +pub const RTN_BLACKHOLE: ::c_uchar = 6; +pub const RTN_UNREACHABLE: ::c_uchar = 7; +pub const RTN_PROHIBIT: ::c_uchar = 8; +pub const RTN_THROW: ::c_uchar = 9; +pub const RTN_NAT: ::c_uchar = 10; +pub const RTN_XRESOLVE: ::c_uchar = 11; + +pub const RTPROT_UNSPEC: ::c_uchar = 0; +pub const RTPROT_REDIRECT: ::c_uchar = 1; +pub const RTPROT_KERNEL: ::c_uchar = 2; +pub const RTPROT_BOOT: ::c_uchar = 3; +pub const RTPROT_STATIC: ::c_uchar = 4; + +pub const RT_SCOPE_UNIVERSE: ::c_uchar = 0; +pub const RT_SCOPE_SITE: ::c_uchar = 200; +pub const RT_SCOPE_LINK: ::c_uchar = 253; +pub const RT_SCOPE_HOST: ::c_uchar = 254; +pub const RT_SCOPE_NOWHERE: ::c_uchar = 255; + +pub const RT_TABLE_UNSPEC: ::c_uchar = 0; +pub const RT_TABLE_COMPAT: ::c_uchar = 252; +pub const RT_TABLE_DEFAULT: ::c_uchar = 253; +pub const RT_TABLE_MAIN: ::c_uchar = 254; +pub const RT_TABLE_LOCAL: ::c_uchar = 255; + +pub const RTMSG_NEWDEVICE: u32 = 0x11; +pub const RTMSG_DELDEVICE: u32 = 0x12; +pub const RTMSG_NEWROUTE: u32 = 0x21; +pub const RTMSG_DELROUTE: u32 = 0x22; + +// Most `*_SUPER_MAGIC` constants are defined at the `linux_like` level; the +// following are only available on newer Linux versions than the versions +// currently used in CI in some configurations, so we define them here. +cfg_if! { + if #[cfg(not(target_arch = "s390x"))] { + pub const XFS_SUPER_MAGIC: ::c_long = 0x58465342; + } else if #[cfg(target_arch = "s390x")] { + pub const XFS_SUPER_MAGIC: ::c_uint = 0x58465342; + } +} f! { pub fn CMSG_NXTHDR(mhdr: *const msghdr, @@ -2096,6 +2983,12 @@ f! { } } + pub fn CPU_ALLOC_SIZE(count: ::c_int) -> ::size_t { + let _dummy: cpu_set_t = ::mem::zeroed(); + let size_in_bits = 8 * ::mem::size_of_val(&_dummy.__bits[0]); + ((count as ::size_t + size_in_bits - 1) / 8) as ::size_t + } + pub fn CPU_ZERO(cpuset: &mut cpu_set_t) -> () { for slot in cpuset.__bits.iter_mut() { *slot = 0; @@ -2103,43 +2996,66 @@ f! { } pub fn CPU_SET(cpu: usize, cpuset: &mut cpu_set_t) -> () { - let size_in___bits = 8 * ::mem::size_of_val(&cpuset.__bits[0]); - let (idx, offset) = (cpu / size_in___bits, cpu % size_in___bits); + let size_in_bits + = 8 * ::mem::size_of_val(&cpuset.__bits[0]); // 32, 64 etc + let (idx, offset) = (cpu / size_in_bits, cpu % size_in_bits); cpuset.__bits[idx] |= 1 << offset; () } pub fn CPU_CLR(cpu: usize, cpuset: &mut cpu_set_t) -> () { - let size_in___bits = 8 * ::mem::size_of_val(&cpuset.__bits[0]); - let (idx, offset) = (cpu / size_in___bits, cpu % size_in___bits); + let size_in_bits + = 8 * ::mem::size_of_val(&cpuset.__bits[0]); // 32, 64 etc + let (idx, offset) = (cpu / size_in_bits, cpu % size_in_bits); cpuset.__bits[idx] &= !(1 << offset); () } pub fn CPU_ISSET(cpu: usize, cpuset: &cpu_set_t) -> bool { - let size_in___bits = 8 * ::mem::size_of_val(&cpuset.__bits[0]); - let (idx, offset) = (cpu / size_in___bits, cpu % size_in___bits); + let size_in_bits = 8 * ::mem::size_of_val(&cpuset.__bits[0]); + let (idx, offset) = (cpu / size_in_bits, cpu % size_in_bits); 0 != (cpuset.__bits[idx] & (1 << offset)) } + pub fn CPU_COUNT_S(size: usize, cpuset: &cpu_set_t) -> ::c_int { + let mut s: u32 = 0; + let size_of_mask = ::mem::size_of_val(&cpuset.__bits[0]); + for i in cpuset.__bits[..(size / size_of_mask)].iter() { + s += i.count_ones(); + }; + s as ::c_int + } + + pub fn CPU_COUNT(cpuset: &cpu_set_t) -> ::c_int { + CPU_COUNT_S(::mem::size_of::(), cpuset) + } + pub fn CPU_EQUAL(set1: &cpu_set_t, set2: &cpu_set_t) -> bool { set1.__bits == set2.__bits } + pub fn major(dev: ::dev_t) -> ::c_int { ((dev >> 8) & 0xfff) as ::c_int } pub fn minor(dev: ::dev_t) -> ::c_int { ((dev & 0xff) | ((dev >> 12) & 0xfff00)) as ::c_int } - pub fn makedev(ma: ::c_int, mi: ::c_int) -> ::dev_t { + pub fn NLA_ALIGN(len: ::c_int) -> ::c_int { + return ((len) + NLA_ALIGNTO - 1) & !(NLA_ALIGNTO - 1) + } + + pub fn SO_EE_OFFENDER(ee: *const ::sock_extended_err) -> *mut ::sockaddr { + ee.offset(1) as *mut ::sockaddr + } +} + +safe_f! { + pub {const} fn makedev(ma: ::c_uint, mi: ::c_uint) -> ::dev_t { let ma = ma as ::dev_t; let mi = mi as ::dev_t; ((ma & 0xfff) << 8) | (mi & 0xff) | ((mi & 0xfff00) << 12) } - pub fn NLA_ALIGN(len: ::c_int) -> ::c_int { - return ((len) + NLA_ALIGNTO - 1) & !(NLA_ALIGNTO - 1) - } } extern "C" { @@ -2147,29 +3063,26 @@ extern "C" { pub fn setrlimit64(resource: ::c_int, rlim: *const rlimit64) -> ::c_int; pub fn getrlimit(resource: ::c_int, rlim: *mut ::rlimit) -> ::c_int; pub fn setrlimit(resource: ::c_int, rlim: *const ::rlimit) -> ::c_int; - pub fn strerror_r( - errnum: ::c_int, - buf: *mut c_char, - buflen: ::size_t, + pub fn prlimit( + pid: ::pid_t, + resource: ::c_int, + new_limit: *const ::rlimit, + old_limit: *mut ::rlimit, + ) -> ::c_int; + pub fn prlimit64( + pid: ::pid_t, + resource: ::c_int, + new_limit: *const ::rlimit64, + old_limit: *mut ::rlimit64, ) -> ::c_int; + pub fn strerror_r(errnum: ::c_int, buf: *mut c_char, buflen: ::size_t) -> ::c_int; pub fn gettimeofday(tp: *mut ::timeval, tz: *mut ::timezone) -> ::c_int; - pub fn madvise( - addr: *mut ::c_void, - len: ::size_t, - advice: ::c_int, - ) -> ::c_int; + pub fn mlock2(addr: *const ::c_void, len: ::size_t, flags: ::c_int) -> ::c_int; + pub fn madvise(addr: *mut ::c_void, len: ::size_t, advice: ::c_int) -> ::c_int; pub fn ioctl(fd: ::c_int, request: ::c_int, ...) -> ::c_int; - pub fn msync( - addr: *mut ::c_void, - len: ::size_t, - flags: ::c_int, - ) -> ::c_int; - pub fn mprotect( - addr: *mut ::c_void, - len: ::size_t, - prot: ::c_int, - ) -> ::c_int; + pub fn msync(addr: *mut ::c_void, len: ::size_t, flags: ::c_int) -> ::c_int; + pub fn mprotect(addr: *mut ::c_void, len: ::size_t, prot: ::c_int) -> ::c_int; pub fn recvfrom( socket: ::c_int, buf: *mut ::c_void, @@ -2187,37 +3100,104 @@ extern "C" { sevlen: ::size_t, flags: ::c_int, ) -> ::c_int; + pub fn preadv(fd: ::c_int, iov: *const ::iovec, count: ::c_int, offset: ::off_t) -> ::ssize_t; + pub fn pwritev(fd: ::c_int, iov: *const ::iovec, count: ::c_int, offset: ::off_t) -> ::ssize_t; + pub fn process_vm_readv( + pid: ::pid_t, + local_iov: *const ::iovec, + local_iov_count: ::c_ulong, + remote_iov: *const ::iovec, + remote_iov_count: ::c_ulong, + flags: ::c_ulong, + ) -> ::ssize_t; + pub fn process_vm_writev( + pid: ::pid_t, + local_iov: *const ::iovec, + local_iov_count: ::c_ulong, + remote_iov: *const ::iovec, + remote_iov_count: ::c_ulong, + flags: ::c_ulong, + ) -> ::ssize_t; pub fn ptrace(request: ::c_int, ...) -> ::c_long; pub fn getpriority(which: ::c_int, who: ::id_t) -> ::c_int; pub fn setpriority(which: ::c_int, who: ::id_t, prio: ::c_int) -> ::c_int; pub fn __sched_cpualloc(count: ::size_t) -> *mut ::cpu_set_t; pub fn __sched_cpufree(set: *mut ::cpu_set_t); - pub fn __sched_cpucount( - setsize: ::size_t, - set: *const cpu_set_t, - ) -> ::c_int; + pub fn __sched_cpucount(setsize: ::size_t, set: *const cpu_set_t) -> ::c_int; pub fn sched_getcpu() -> ::c_int; + pub fn mallinfo() -> ::mallinfo; + // available from API 23 + pub fn malloc_info(options: ::c_int, stream: *mut ::FILE) -> ::c_int; + + pub fn malloc_usable_size(ptr: *const ::c_void) -> ::size_t; pub fn utmpname(name: *const ::c_char) -> ::c_int; pub fn setutent(); pub fn getutent() -> *mut utmp; - pub fn posix_fallocate( - fd: ::c_int, - offset: ::off_t, - len: ::off_t, + pub fn seekdir(dirp: *mut ::DIR, loc: ::c_long); + pub fn telldir(dirp: *mut ::DIR) -> ::c_long; + pub fn fallocate(fd: ::c_int, mode: ::c_int, offset: ::off_t, len: ::off_t) -> ::c_int; + pub fn fallocate64(fd: ::c_int, mode: ::c_int, offset: ::off64_t, len: ::off64_t) -> ::c_int; + pub fn posix_fallocate(fd: ::c_int, offset: ::off_t, len: ::off_t) -> ::c_int; + pub fn posix_fallocate64(fd: ::c_int, offset: ::off64_t, len: ::off64_t) -> ::c_int; + pub fn getxattr( + path: *const c_char, + name: *const c_char, + value: *mut ::c_void, + size: ::size_t, + ) -> ::ssize_t; + pub fn lgetxattr( + path: *const c_char, + name: *const c_char, + value: *mut ::c_void, + size: ::size_t, + ) -> ::ssize_t; + pub fn fgetxattr( + filedes: ::c_int, + name: *const c_char, + value: *mut ::c_void, + size: ::size_t, + ) -> ::ssize_t; + pub fn setxattr( + path: *const c_char, + name: *const c_char, + value: *const ::c_void, + size: ::size_t, + flags: ::c_int, + ) -> ::c_int; + pub fn lsetxattr( + path: *const c_char, + name: *const c_char, + value: *const ::c_void, + size: ::size_t, + flags: ::c_int, ) -> ::c_int; - pub fn signalfd( + pub fn fsetxattr( + filedes: ::c_int, + name: *const c_char, + value: *const ::c_void, + size: ::size_t, + flags: ::c_int, + ) -> ::c_int; + pub fn listxattr(path: *const c_char, list: *mut c_char, size: ::size_t) -> ::ssize_t; + pub fn llistxattr(path: *const c_char, list: *mut c_char, size: ::size_t) -> ::ssize_t; + pub fn flistxattr(filedes: ::c_int, list: *mut c_char, size: ::size_t) -> ::ssize_t; + pub fn removexattr(path: *const c_char, name: *const c_char) -> ::c_int; + pub fn lremovexattr(path: *const c_char, name: *const c_char) -> ::c_int; + pub fn fremovexattr(filedes: ::c_int, name: *const c_char) -> ::c_int; + pub fn signalfd(fd: ::c_int, mask: *const ::sigset_t, flags: ::c_int) -> ::c_int; + pub fn timerfd_create(clock: ::clockid_t, flags: ::c_int) -> ::c_int; + pub fn timerfd_gettime(fd: ::c_int, current_value: *mut itimerspec) -> ::c_int; + pub fn timerfd_settime( fd: ::c_int, - mask: *const ::sigset_t, flags: ::c_int, + new_value: *const itimerspec, + old_value: *mut itimerspec, ) -> ::c_int; pub fn syscall(num: ::c_long, ...) -> ::c_long; - pub fn sched_getaffinity( - pid: ::pid_t, - cpusetsize: ::size_t, - cpuset: *mut cpu_set_t, - ) -> ::c_int; + pub fn sched_getaffinity(pid: ::pid_t, cpusetsize: ::size_t, cpuset: *mut cpu_set_t) + -> ::c_int; pub fn sched_setaffinity( pid: ::pid_t, cpusetsize: ::size_t, @@ -2231,12 +3211,8 @@ extern "C" { maxevents: ::c_int, timeout: ::c_int, ) -> ::c_int; - pub fn epoll_ctl( - epfd: ::c_int, - op: ::c_int, - fd: ::c_int, - event: *mut ::epoll_event, - ) -> ::c_int; + pub fn epoll_ctl(epfd: ::c_int, op: ::c_int, fd: ::c_int, event: *mut ::epoll_event) + -> ::c_int; pub fn pthread_getschedparam( native: ::pthread_t, policy: *mut ::c_int, @@ -2245,16 +3221,8 @@ extern "C" { pub fn unshare(flags: ::c_int) -> ::c_int; pub fn umount(target: *const ::c_char) -> ::c_int; pub fn sched_get_priority_max(policy: ::c_int) -> ::c_int; - pub fn tee( - fd_in: ::c_int, - fd_out: ::c_int, - len: ::size_t, - flags: ::c_uint, - ) -> ::ssize_t; - pub fn settimeofday( - tv: *const ::timeval, - tz: *const ::timezone, - ) -> ::c_int; + pub fn tee(fd_in: ::c_int, fd_out: ::c_int, len: ::size_t, flags: ::c_uint) -> ::ssize_t; + pub fn settimeofday(tv: *const ::timeval, tz: *const ::timezone) -> ::c_int; pub fn splice( fd_in: ::c_int, off_in: *mut ::loff_t, @@ -2264,17 +3232,10 @@ extern "C" { flags: ::c_uint, ) -> ::ssize_t; pub fn eventfd(init: ::c_uint, flags: ::c_int) -> ::c_int; - pub fn sched_rr_get_interval(pid: ::pid_t, tp: *mut ::timespec) - -> ::c_int; - pub fn sem_timedwait( - sem: *mut sem_t, - abstime: *const ::timespec, - ) -> ::c_int; + pub fn sched_rr_get_interval(pid: ::pid_t, tp: *mut ::timespec) -> ::c_int; + pub fn sem_timedwait(sem: *mut sem_t, abstime: *const ::timespec) -> ::c_int; pub fn sem_getvalue(sem: *mut sem_t, sval: *mut ::c_int) -> ::c_int; - pub fn sched_setparam( - pid: ::pid_t, - param: *const ::sched_param, - ) -> ::c_int; + pub fn sched_setparam(pid: ::pid_t, param: *const ::sched_param) -> ::c_int; pub fn setns(fd: ::c_int, nstype: ::c_int) -> ::c_int; pub fn swapoff(puath: *const ::c_char) -> ::c_int; pub fn vmsplice( @@ -2303,6 +3264,28 @@ extern "C" { lock: *mut pthread_mutex_t, abstime: *const ::timespec, ) -> ::c_int; + pub fn pthread_barrierattr_init(attr: *mut ::pthread_barrierattr_t) -> ::c_int; + pub fn pthread_barrierattr_destroy(attr: *mut ::pthread_barrierattr_t) -> ::c_int; + pub fn pthread_barrierattr_getpshared( + attr: *const ::pthread_barrierattr_t, + shared: *mut ::c_int, + ) -> ::c_int; + pub fn pthread_barrierattr_setpshared( + attr: *mut ::pthread_barrierattr_t, + shared: ::c_int, + ) -> ::c_int; + pub fn pthread_barrier_init( + barrier: *mut pthread_barrier_t, + attr: *const ::pthread_barrierattr_t, + count: ::c_uint, + ) -> ::c_int; + pub fn pthread_barrier_destroy(barrier: *mut pthread_barrier_t) -> ::c_int; + pub fn pthread_barrier_wait(barrier: *mut pthread_barrier_t) -> ::c_int; + pub fn pthread_spin_init(lock: *mut ::pthread_spinlock_t, pshared: ::c_int) -> ::c_int; + pub fn pthread_spin_destroy(lock: *mut ::pthread_spinlock_t) -> ::c_int; + pub fn pthread_spin_lock(lock: *mut ::pthread_spinlock_t) -> ::c_int; + pub fn pthread_spin_trylock(lock: *mut ::pthread_spinlock_t) -> ::c_int; + pub fn pthread_spin_unlock(lock: *mut ::pthread_spinlock_t) -> ::c_int; pub fn clone( cb: extern "C" fn(*mut ::c_void) -> ::c_int, child_stack: *mut ::c_void, @@ -2349,7 +3332,6 @@ extern "C" { pub fn setfsgid(gid: ::gid_t) -> ::c_int; pub fn setfsuid(uid: ::uid_t) -> ::c_int; pub fn sigsuspend(mask: *const ::sigset_t) -> ::c_int; - #[cfg_attr(target_os = "solaris", link_name = "__posix_getgrgid_r")] pub fn getgrgid_r( gid: ::gid_t, grp: *mut ::group, @@ -2357,14 +3339,8 @@ extern "C" { buflen: ::size_t, result: *mut *mut ::group, ) -> ::c_int; - #[cfg_attr( - all(target_os = "macos", target_arch = "x86"), - link_name = "sigaltstack$UNIX2003" - )] - #[cfg_attr(target_os = "netbsd", link_name = "__sigaltstack14")] pub fn sigaltstack(ss: *const stack_t, oss: *mut stack_t) -> ::c_int; pub fn sem_close(sem: *mut sem_t) -> ::c_int; - #[cfg_attr(target_os = "solaris", link_name = "__posix_getgrnam_r")] pub fn getgrnam_r( name: *const ::c_char, grp: *mut ::group, @@ -2372,22 +3348,12 @@ extern "C" { buflen: ::size_t, result: *mut *mut ::group, ) -> ::c_int; - #[cfg_attr( - all(target_os = "macos", target_arch = "x86"), - link_name = "pthread_sigmask$UNIX2003" - )] - pub fn pthread_sigmask( - how: ::c_int, - set: *const sigset_t, - oldset: *mut sigset_t, - ) -> ::c_int; + pub fn pthread_sigmask(how: ::c_int, set: *const sigset_t, oldset: *mut sigset_t) -> ::c_int; pub fn sem_open(name: *const ::c_char, oflag: ::c_int, ...) -> *mut sem_t; pub fn getgrnam(name: *const ::c_char) -> *mut ::group; pub fn pthread_kill(thread: ::pthread_t, sig: ::c_int) -> ::c_int; pub fn sem_unlink(name: *const ::c_char) -> ::c_int; pub fn daemon(nochdir: ::c_int, noclose: ::c_int) -> ::c_int; - #[cfg_attr(target_os = "netbsd", link_name = "__getpwnam_r50")] - #[cfg_attr(target_os = "solaris", link_name = "__posix_getpwnam_r")] pub fn getpwnam_r( name: *const ::c_char, pwd: *mut passwd, @@ -2395,8 +3361,6 @@ extern "C" { buflen: ::size_t, result: *mut *mut passwd, ) -> ::c_int; - #[cfg_attr(target_os = "netbsd", link_name = "__getpwuid_r50")] - #[cfg_attr(target_os = "solaris", link_name = "__posix_getpwuid_r")] pub fn getpwuid_r( uid: ::uid_t, pwd: *mut passwd, @@ -2404,11 +3368,11 @@ extern "C" { buflen: ::size_t, result: *mut *mut passwd, ) -> ::c_int; - #[cfg_attr( - all(target_os = "macos", target_arch = "x86"), - link_name = "sigwait$UNIX2003" - )] - #[cfg_attr(target_os = "solaris", link_name = "__posix_sigwait")] + pub fn sigtimedwait( + set: *const sigset_t, + info: *mut siginfo_t, + timeout: *const ::timespec, + ) -> ::c_int; pub fn sigwait(set: *const sigset_t, sig: *mut ::c_int) -> ::c_int; pub fn pthread_atfork( prepare: ::Option, @@ -2427,10 +3391,6 @@ extern "C" { attr: *const pthread_mutexattr_t, pshared: *mut ::c_int, ) -> ::c_int; - #[cfg_attr( - all(target_os = "macos", target_arch = "x86"), - link_name = "popen$UNIX2003" - )] pub fn popen(command: *const c_char, mode: *const c_char) -> *mut ::FILE; pub fn faccessat( dirfd: ::c_int, @@ -2461,10 +3421,69 @@ extern "C" { ) -> ::c_int; pub fn inotify_init() -> ::c_int; pub fn inotify_init1(flags: ::c_int) -> ::c_int; - pub fn inotify_add_watch( - fd: ::c_int, - path: *const ::c_char, - mask: u32, + pub fn inotify_add_watch(fd: ::c_int, path: *const ::c_char, mask: u32) -> ::c_int; + + pub fn regcomp(preg: *mut ::regex_t, pattern: *const ::c_char, cflags: ::c_int) -> ::c_int; + + pub fn regexec( + preg: *const ::regex_t, + input: *const ::c_char, + nmatch: ::size_t, + pmatch: *mut regmatch_t, + eflags: ::c_int, + ) -> ::c_int; + + pub fn regerror( + errcode: ::c_int, + preg: *const ::regex_t, + errbuf: *mut ::c_char, + errbuf_size: ::size_t, + ) -> ::size_t; + + pub fn regfree(preg: *mut ::regex_t); + + pub fn android_set_abort_message(msg: *const ::c_char); + + pub fn gettid() -> ::pid_t; + + pub fn __system_property_set(__name: *const ::c_char, __value: *const ::c_char) -> ::c_int; + pub fn __system_property_get(__name: *const ::c_char, __value: *mut ::c_char) -> ::c_int; + pub fn __system_property_find(__name: *const ::c_char) -> *const prop_info; + pub fn __system_property_find_nth(__n: ::c_uint) -> *const prop_info; + pub fn __system_property_foreach( + __callback: unsafe extern "C" fn(__pi: *const prop_info, __cookie: *mut ::c_void), + __cookie: *mut ::c_void, + ) -> ::c_int; + + // #include + /// Only available in API Version 21+ + pub fn dl_iterate_phdr( + callback: ::Option< + unsafe extern "C" fn( + info: *mut dl_phdr_info, + size: usize, + data: *mut ::c_void, + ) -> ::c_int, + >, + data: *mut ::c_void, + ) -> ::c_int; + + pub fn arc4random() -> u32; + pub fn arc4random_uniform(__upper_bound: u32) -> u32; + pub fn arc4random_buf(__buf: *mut ::c_void, __n: ::size_t); + + pub fn reallocarray(ptr: *mut ::c_void, nmemb: ::size_t, size: ::size_t) -> *mut ::c_void; + + pub fn pthread_getcpuclockid(thread: ::pthread_t, clk_id: *mut ::clockid_t) -> ::c_int; + + pub fn dirname(path: *const ::c_char) -> *mut ::c_char; + pub fn basename(path: *const ::c_char) -> *mut ::c_char; + pub fn getopt_long( + argc: ::c_int, + argv: *const *mut c_char, + optstring: *const c_char, + longopts: *const option, + longindex: *mut ::c_int, ) -> ::c_int; } @@ -2481,6 +3500,17 @@ cfg_if! { } impl siginfo_t { + pub unsafe fn si_addr(&self) -> *mut ::c_void { + #[repr(C)] + struct siginfo_sigfault { + _si_signo: ::c_int, + _si_errno: ::c_int, + _si_code: ::c_int, + si_addr: *mut ::c_void, + } + (*(self as *const siginfo_t as *const siginfo_sigfault)).si_addr + } + pub unsafe fn si_value(&self) -> ::sigval { #[repr(C)] struct siginfo_timer { @@ -2494,3 +3524,65 @@ impl siginfo_t { (*(self as *const siginfo_t as *const siginfo_timer)).si_sigval } } + +cfg_if! { + if #[cfg(libc_union)] { + // Internal, for casts to access union fields + #[repr(C)] + struct sifields_sigchld { + si_pid: ::pid_t, + si_uid: ::uid_t, + si_status: ::c_int, + si_utime: ::c_long, + si_stime: ::c_long, + } + impl ::Copy for sifields_sigchld {} + impl ::Clone for sifields_sigchld { + fn clone(&self) -> sifields_sigchld { + *self + } + } + + // Internal, for casts to access union fields + #[repr(C)] + union sifields { + _align_pointer: *mut ::c_void, + sigchld: sifields_sigchld, + } + + // Internal, for casts to access union fields. Note that some variants + // of sifields start with a pointer, which makes the alignment of + // sifields vary on 32-bit and 64-bit architectures. + #[repr(C)] + struct siginfo_f { + _siginfo_base: [::c_int; 3], + sifields: sifields, + } + + impl siginfo_t { + unsafe fn sifields(&self) -> &sifields { + &(*(self as *const siginfo_t as *const siginfo_f)).sifields + } + + pub unsafe fn si_pid(&self) -> ::pid_t { + self.sifields().sigchld.si_pid + } + + pub unsafe fn si_uid(&self) -> ::uid_t { + self.sifields().sigchld.si_uid + } + + pub unsafe fn si_status(&self) -> ::c_int { + self.sifields().sigchld.si_status + } + + pub unsafe fn si_utime(&self) -> ::c_long { + self.sifields().sigchld.si_utime + } + + pub unsafe fn si_stime(&self) -> ::c_long { + self.sifields().sigchld.si_stime + } + } + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/emscripten/align.rs b/crux-mir/lib/libc/src/unix/linux_like/emscripten/align.rs index 141570f88..b9ea3f39e 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/emscripten/align.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/emscripten/align.rs @@ -38,9 +38,9 @@ macro_rules! expand_align { } #[allow(missing_debug_implementations)] - #[repr(align(8))] + #[repr(align(16))] pub struct max_align_t { - priv_: [f64; 2] + priv_: [f64; 4] } } diff --git a/crux-mir/lib/libc/src/unix/linux_like/emscripten/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/emscripten/mod.rs index a7a81ae20..11fbb31c3 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/emscripten/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/emscripten/mod.rs @@ -899,8 +899,6 @@ pub const RTLD_DEFAULT: *mut ::c_void = 0i64 as *mut ::c_void; pub const RTLD_NODELETE: ::c_int = 0x1000; pub const RTLD_NOW: ::c_int = 0x2; -pub const TCP_MD5SIG: ::c_int = 14; - align_const! { pub const PTHREAD_MUTEX_INITIALIZER: pthread_mutex_t = pthread_mutex_t { size: [0; __SIZEOF_PTHREAD_MUTEX_T], @@ -967,10 +965,6 @@ pub const SHM_UNLOCK: ::c_int = 12; pub const SHM_HUGETLB: ::c_int = 0o4000; pub const SHM_NORESERVE: ::c_int = 0o10000; -pub const EPOLLRDHUP: ::c_int = 0x2000; -pub const EPOLLEXCLUSIVE: ::c_int = 0x10000000; -pub const EPOLLONESHOT: ::c_int = 0x40000000; - pub const QFMT_VFS_OLD: ::c_int = 1; pub const QFMT_VFS_V0: ::c_int = 2; @@ -1155,9 +1149,6 @@ pub const ITIMER_REAL: ::c_int = 0; pub const ITIMER_VIRTUAL: ::c_int = 1; pub const ITIMER_PROF: ::c_int = 2; -pub const XATTR_CREATE: ::c_int = 0x1; -pub const XATTR_REPLACE: ::c_int = 0x2; - pub const _POSIX_VDISABLE: ::cc_t = 0; pub const FALLOC_FL_KEEP_SIZE: ::c_int = 0x01; @@ -1174,20 +1165,142 @@ pub const O_TRUNC: ::c_int = 512; pub const O_NOATIME: ::c_int = 0o1000000; pub const O_CLOEXEC: ::c_int = 0x80000; -pub const EBFONT: ::c_int = 59; -pub const ENOSTR: ::c_int = 60; -pub const ENODATA: ::c_int = 61; -pub const ETIME: ::c_int = 62; -pub const ENOSR: ::c_int = 63; -pub const ENONET: ::c_int = 64; -pub const ENOPKG: ::c_int = 65; -pub const EREMOTE: ::c_int = 66; -pub const ENOLINK: ::c_int = 67; -pub const EADV: ::c_int = 68; -pub const ESRMNT: ::c_int = 69; -pub const ECOMM: ::c_int = 70; -pub const EPROTO: ::c_int = 71; -pub const EDOTDOT: ::c_int = 73; +// Defined as wasi value. +pub const EPERM: ::c_int = 63; +pub const ENOENT: ::c_int = 44; +pub const ESRCH: ::c_int = 71; +pub const EINTR: ::c_int = 27; +pub const EIO: ::c_int = 29; +pub const ENXIO: ::c_int = 60; +pub const E2BIG: ::c_int = 1; +pub const ENOEXEC: ::c_int = 45; +pub const EBADF: ::c_int = 8; +pub const ECHILD: ::c_int = 12; +pub const EAGAIN: ::c_int = 6; +pub const ENOMEM: ::c_int = 48; +pub const EACCES: ::c_int = 2; +pub const EFAULT: ::c_int = 21; +pub const ENOTBLK: ::c_int = 105; +pub const EBUSY: ::c_int = 10; +pub const EEXIST: ::c_int = 20; +pub const EXDEV: ::c_int = 75; +pub const ENODEV: ::c_int = 43; +pub const ENOTDIR: ::c_int = 54; +pub const EISDIR: ::c_int = 31; +pub const EINVAL: ::c_int = 28; +pub const ENFILE: ::c_int = 41; +pub const EMFILE: ::c_int = 33; +pub const ENOTTY: ::c_int = 59; +pub const ETXTBSY: ::c_int = 74; +pub const EFBIG: ::c_int = 22; +pub const ENOSPC: ::c_int = 51; +pub const ESPIPE: ::c_int = 70; +pub const EROFS: ::c_int = 69; +pub const EMLINK: ::c_int = 34; +pub const EPIPE: ::c_int = 64; +pub const EDOM: ::c_int = 18; +pub const ERANGE: ::c_int = 68; +pub const EWOULDBLOCK: ::c_int = EAGAIN; +pub const ENOLINK: ::c_int = 47; +pub const EPROTO: ::c_int = 65; +pub const EDEADLK: ::c_int = 16; +pub const EDEADLOCK: ::c_int = EDEADLK; +pub const ENAMETOOLONG: ::c_int = 37; +pub const ENOLCK: ::c_int = 46; +pub const ENOSYS: ::c_int = 52; +pub const ENOTEMPTY: ::c_int = 55; +pub const ELOOP: ::c_int = 32; +pub const ENOMSG: ::c_int = 49; +pub const EIDRM: ::c_int = 24; +pub const EMULTIHOP: ::c_int = 36; +pub const EBADMSG: ::c_int = 9; +pub const EOVERFLOW: ::c_int = 61; +pub const EILSEQ: ::c_int = 25; +pub const ENOTSOCK: ::c_int = 57; +pub const EDESTADDRREQ: ::c_int = 17; +pub const EMSGSIZE: ::c_int = 35; +pub const EPROTOTYPE: ::c_int = 67; +pub const ENOPROTOOPT: ::c_int = 50; +pub const EPROTONOSUPPORT: ::c_int = 66; +pub const EAFNOSUPPORT: ::c_int = 5; +pub const EADDRINUSE: ::c_int = 3; +pub const EADDRNOTAVAIL: ::c_int = 4; +pub const ENETDOWN: ::c_int = 38; +pub const ENETUNREACH: ::c_int = 40; +pub const ENETRESET: ::c_int = 39; +pub const ECONNABORTED: ::c_int = 13; +pub const ECONNRESET: ::c_int = 15; +pub const ENOBUFS: ::c_int = 42; +pub const EISCONN: ::c_int = 30; +pub const ENOTCONN: ::c_int = 53; +pub const ETIMEDOUT: ::c_int = 73; +pub const ECONNREFUSED: ::c_int = 14; +pub const EHOSTUNREACH: ::c_int = 23; +pub const EALREADY: ::c_int = 7; +pub const EINPROGRESS: ::c_int = 26; +pub const ESTALE: ::c_int = 72; +pub const EDQUOT: ::c_int = 19; +pub const ECANCELED: ::c_int = 11; +pub const EOWNERDEAD: ::c_int = 62; +pub const ENOTRECOVERABLE: ::c_int = 56; + +pub const ENOSTR: ::c_int = 100; +pub const EBFONT: ::c_int = 101; +pub const EBADSLT: ::c_int = 102; +pub const EBADRQC: ::c_int = 103; +pub const ENOANO: ::c_int = 104; +pub const ECHRNG: ::c_int = 106; +pub const EL3HLT: ::c_int = 107; +pub const EL3RST: ::c_int = 108; +pub const ELNRNG: ::c_int = 109; +pub const EUNATCH: ::c_int = 110; +pub const ENOCSI: ::c_int = 111; +pub const EL2HLT: ::c_int = 112; +pub const EBADE: ::c_int = 113; +pub const EBADR: ::c_int = 114; +pub const EXFULL: ::c_int = 115; +pub const ENODATA: ::c_int = 116; +pub const ETIME: ::c_int = 117; +pub const ENOSR: ::c_int = 118; +pub const ENONET: ::c_int = 119; +pub const ENOPKG: ::c_int = 120; +pub const EREMOTE: ::c_int = 121; +pub const EADV: ::c_int = 122; +pub const ESRMNT: ::c_int = 123; +pub const ECOMM: ::c_int = 124; +pub const EDOTDOT: ::c_int = 125; +pub const ENOTUNIQ: ::c_int = 126; +pub const EBADFD: ::c_int = 127; +pub const EREMCHG: ::c_int = 128; +pub const ELIBACC: ::c_int = 129; +pub const ELIBBAD: ::c_int = 130; +pub const ELIBSCN: ::c_int = 131; +pub const ELIBMAX: ::c_int = 132; +pub const ELIBEXEC: ::c_int = 133; +pub const ERESTART: ::c_int = 134; +pub const ESTRPIPE: ::c_int = 135; +pub const EUSERS: ::c_int = 136; +pub const ESOCKTNOSUPPORT: ::c_int = 137; +pub const EOPNOTSUPP: ::c_int = 138; +pub const ENOTSUP: ::c_int = EOPNOTSUPP; +pub const EPFNOSUPPORT: ::c_int = 139; +pub const ESHUTDOWN: ::c_int = 140; +pub const ETOOMANYREFS: ::c_int = 141; +pub const EHOSTDOWN: ::c_int = 142; +pub const EUCLEAN: ::c_int = 143; +pub const ENOTNAM: ::c_int = 144; +pub const ENAVAIL: ::c_int = 145; +pub const EISNAM: ::c_int = 146; +pub const EREMOTEIO: ::c_int = 147; +pub const ENOMEDIUM: ::c_int = 148; +pub const EMEDIUMTYPE: ::c_int = 149; +pub const ENOKEY: ::c_int = 150; +pub const EKEYEXPIRED: ::c_int = 151; +pub const EKEYREVOKED: ::c_int = 152; +pub const EKEYREJECTED: ::c_int = 153; +pub const ERFKILL: ::c_int = 154; +pub const EHWPOISON: ::c_int = 155; +pub const EL2NSYNC: ::c_int = 156; pub const SA_NODEFER: ::c_int = 0x40000000; pub const SA_RESETHAND: ::c_int = 0x80000000; @@ -1215,20 +1328,10 @@ pub const POSIX_MADV_DONTNEED: ::c_int = 0; pub const RLIM_INFINITY: ::rlim_t = !0; pub const RLIMIT_NLIMITS: ::c_int = 15; +pub const RLIM_NLIMITS: ::c_int = RLIMIT_NLIMITS; pub const MAP_ANONYMOUS: ::c_int = MAP_ANON; -pub const TCP_THIN_LINEAR_TIMEOUTS: ::c_int = 16; -pub const TCP_THIN_DUPACK: ::c_int = 17; -pub const TCP_USER_TIMEOUT: ::c_int = 18; -pub const TCP_REPAIR: ::c_int = 19; -pub const TCP_REPAIR_QUEUE: ::c_int = 20; -pub const TCP_QUEUE_SEQ: ::c_int = 21; -pub const TCP_REPAIR_OPTIONS: ::c_int = 22; -pub const TCP_FASTOPEN: ::c_int = 23; -pub const TCP_TIMESTAMP: ::c_int = 24; -pub const TCP_FASTOPEN_CONNECT: ::c_int = 30; - #[doc(hidden)] #[deprecated(since = "0.2.55", note = "Use SIGSYS instead")] pub const SIGUNUSED: ::c_int = ::SIGSYS; @@ -1265,8 +1368,6 @@ pub const PTRACE_INTERRUPT: ::c_int = 0x4207; pub const PTRACE_LISTEN: ::c_int = 0x4208; pub const PTRACE_PEEKSIGINFO: ::c_int = 0x4209; -pub const EPOLLWAKEUP: ::c_int = 0x20000000; - pub const PTRACE_GETFPREGS: ::c_uint = 14; pub const PTRACE_SETFPREGS: ::c_uint = 15; pub const PTRACE_GETFPXREGS: ::c_uint = 18; @@ -1436,93 +1537,9 @@ pub const SOCK_STREAM: ::c_int = 1; pub const SOCK_DGRAM: ::c_int = 2; pub const SOCK_SEQPACKET: ::c_int = 5; -pub const SOL_SOCKET: ::c_int = 1; +pub const IPPROTO_MAX: ::c_int = 256; -pub const EDEADLK: ::c_int = 35; -pub const ENAMETOOLONG: ::c_int = 36; -pub const ENOLCK: ::c_int = 37; -pub const ENOSYS: ::c_int = 38; -pub const ENOTEMPTY: ::c_int = 39; -pub const ELOOP: ::c_int = 40; -pub const ENOMSG: ::c_int = 42; -pub const EIDRM: ::c_int = 43; -pub const ECHRNG: ::c_int = 44; -pub const EL2NSYNC: ::c_int = 45; -pub const EL3HLT: ::c_int = 46; -pub const EL3RST: ::c_int = 47; -pub const ELNRNG: ::c_int = 48; -pub const EUNATCH: ::c_int = 49; -pub const ENOCSI: ::c_int = 50; -pub const EL2HLT: ::c_int = 51; -pub const EBADE: ::c_int = 52; -pub const EBADR: ::c_int = 53; -pub const EXFULL: ::c_int = 54; -pub const ENOANO: ::c_int = 55; -pub const EBADRQC: ::c_int = 56; -pub const EBADSLT: ::c_int = 57; -pub const EDEADLOCK: ::c_int = EDEADLK; -pub const EMULTIHOP: ::c_int = 72; -pub const EBADMSG: ::c_int = 74; -pub const EOVERFLOW: ::c_int = 75; -pub const ENOTUNIQ: ::c_int = 76; -pub const EBADFD: ::c_int = 77; -pub const EREMCHG: ::c_int = 78; -pub const ELIBACC: ::c_int = 79; -pub const ELIBBAD: ::c_int = 80; -pub const ELIBSCN: ::c_int = 81; -pub const ELIBMAX: ::c_int = 82; -pub const ELIBEXEC: ::c_int = 83; -pub const EILSEQ: ::c_int = 84; -pub const ERESTART: ::c_int = 85; -pub const ESTRPIPE: ::c_int = 86; -pub const EUSERS: ::c_int = 87; -pub const ENOTSOCK: ::c_int = 88; -pub const EDESTADDRREQ: ::c_int = 89; -pub const EMSGSIZE: ::c_int = 90; -pub const EPROTOTYPE: ::c_int = 91; -pub const ENOPROTOOPT: ::c_int = 92; -pub const EPROTONOSUPPORT: ::c_int = 93; -pub const ESOCKTNOSUPPORT: ::c_int = 94; -pub const EOPNOTSUPP: ::c_int = 95; -pub const ENOTSUP: ::c_int = EOPNOTSUPP; -pub const EPFNOSUPPORT: ::c_int = 96; -pub const EAFNOSUPPORT: ::c_int = 97; -pub const EADDRINUSE: ::c_int = 98; -pub const EADDRNOTAVAIL: ::c_int = 99; -pub const ENETDOWN: ::c_int = 100; -pub const ENETUNREACH: ::c_int = 101; -pub const ENETRESET: ::c_int = 102; -pub const ECONNABORTED: ::c_int = 103; -pub const ECONNRESET: ::c_int = 104; -pub const ENOBUFS: ::c_int = 105; -pub const EISCONN: ::c_int = 106; -pub const ENOTCONN: ::c_int = 107; -pub const ESHUTDOWN: ::c_int = 108; -pub const ETOOMANYREFS: ::c_int = 109; -pub const ETIMEDOUT: ::c_int = 110; -pub const ECONNREFUSED: ::c_int = 111; -pub const EHOSTDOWN: ::c_int = 112; -pub const EHOSTUNREACH: ::c_int = 113; -pub const EALREADY: ::c_int = 114; -pub const EINPROGRESS: ::c_int = 115; -pub const ESTALE: ::c_int = 116; -pub const EUCLEAN: ::c_int = 117; -pub const ENOTNAM: ::c_int = 118; -pub const ENAVAIL: ::c_int = 119; -pub const EISNAM: ::c_int = 120; -pub const EREMOTEIO: ::c_int = 121; -pub const EDQUOT: ::c_int = 122; -pub const ENOMEDIUM: ::c_int = 123; -pub const EMEDIUMTYPE: ::c_int = 124; -pub const ECANCELED: ::c_int = 125; -pub const ENOKEY: ::c_int = 126; -pub const EKEYEXPIRED: ::c_int = 127; -pub const EKEYREVOKED: ::c_int = 128; -pub const EKEYREJECTED: ::c_int = 129; -pub const EOWNERDEAD: ::c_int = 130; -pub const ENOTRECOVERABLE: ::c_int = 131; -pub const ERFKILL: ::c_int = 132; -pub const EHWPOISON: ::c_int = 133; +pub const SOL_SOCKET: ::c_int = 1; pub const SO_REUSEADDR: ::c_int = 2; pub const SO_TYPE: ::c_int = 3; @@ -1541,6 +1558,9 @@ pub const SO_RCVTIMEO: ::c_int = 20; pub const SO_SNDTIMEO: ::c_int = 21; pub const SO_ACCEPTCONN: ::c_int = 30; +pub const IPV6_RTHDR_LOOSE: ::c_int = 0; +pub const IPV6_RTHDR_STRICT: ::c_int = 1; + pub const SA_ONSTACK: ::c_int = 0x08000000; pub const SA_SIGINFO: ::c_int = 0x00000004; pub const SA_NOCLDWAIT: ::c_int = 0x00000002; @@ -1645,6 +1665,12 @@ pub const ARPD_LOOKUP: ::c_ushort = 0x02; pub const ARPD_FLUSH: ::c_ushort = 0x03; pub const ATF_MAGIC: ::c_int = 0x80; +pub const PRIO_PROCESS: ::c_int = 0; +pub const PRIO_PGRP: ::c_int = 1; +pub const PRIO_USER: ::c_int = 2; + +pub const SOMAXCONN: ::c_int = 128; + f! { pub fn CMSG_NXTHDR(mhdr: *const msghdr, cmsg: *const cmsghdr) -> *mut cmsghdr { @@ -1697,7 +1723,7 @@ f! { pub fn major(dev: ::dev_t) -> ::c_uint { // see - // https://github.com/kripken/emscripten/blob/ + // https://github.com/emscripten-core/emscripten/blob/ // master/system/include/libc/sys/sysmacros.h let mut major = 0; major |= (dev & 0x00000fff) >> 8; @@ -1707,15 +1733,17 @@ f! { pub fn minor(dev: ::dev_t) -> ::c_uint { // see - // https://github.com/kripken/emscripten/blob/ + // https://github.com/emscripten-core/emscripten/blob/ // master/system/include/libc/sys/sysmacros.h let mut minor = 0; minor |= (dev & 0x000000ff) >> 0; minor |= (dev & 0xffffff00) >> 12; minor as ::c_uint } +} - pub fn makedev(major: ::c_uint, minor: ::c_uint) -> ::dev_t { +safe_f! { + pub {const} fn makedev(major: ::c_uint, minor: ::c_uint) -> ::dev_t { let major = major as ::dev_t; let minor = minor as ::dev_t; let mut dev = 0; @@ -1732,11 +1760,7 @@ extern "C" { pub fn setrlimit64(resource: ::c_int, rlim: *const rlimit64) -> ::c_int; pub fn getrlimit(resource: ::c_int, rlim: *mut ::rlimit) -> ::c_int; pub fn setrlimit(resource: ::c_int, rlim: *const ::rlimit) -> ::c_int; - pub fn strerror_r( - errnum: ::c_int, - buf: *mut c_char, - buflen: ::size_t, - ) -> ::c_int; + pub fn strerror_r(errnum: ::c_int, buf: *mut c_char, buflen: ::size_t) -> ::c_int; pub fn abs(i: ::c_int) -> ::c_int; pub fn atof(s: *const ::c_char) -> ::c_double; @@ -1750,23 +1774,12 @@ extern "C" { pub fn endpwent(); pub fn getpwent() -> *mut passwd; - pub fn shm_open( - name: *const c_char, - oflag: ::c_int, - mode: mode_t, - ) -> ::c_int; + pub fn shm_open(name: *const c_char, oflag: ::c_int, mode: mode_t) -> ::c_int; - pub fn mprotect( - addr: *mut ::c_void, - len: ::size_t, - prot: ::c_int, - ) -> ::c_int; + pub fn mprotect(addr: *mut ::c_void, len: ::size_t, prot: ::c_int) -> ::c_int; pub fn __errno_location() -> *mut ::c_int; - pub fn fopen64( - filename: *const c_char, - mode: *const c_char, - ) -> *mut ::FILE; + pub fn fopen64(filename: *const c_char, mode: *const c_char) -> *mut ::FILE; pub fn freopen64( filename: *const c_char, mode: *const c_char, @@ -1775,38 +1788,22 @@ extern "C" { pub fn tmpfile64() -> *mut ::FILE; pub fn fgetpos64(stream: *mut ::FILE, ptr: *mut fpos64_t) -> ::c_int; pub fn fsetpos64(stream: *mut ::FILE, ptr: *const fpos64_t) -> ::c_int; - pub fn fseeko64( - stream: *mut ::FILE, - offset: ::off64_t, - whence: ::c_int, - ) -> ::c_int; + pub fn fseeko64(stream: *mut ::FILE, offset: ::off64_t, whence: ::c_int) -> ::c_int; pub fn ftello64(stream: *mut ::FILE) -> ::off64_t; - pub fn posix_fallocate( - fd: ::c_int, - offset: ::off_t, - len: ::off_t, - ) -> ::c_int; - pub fn pwritev( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - offset: ::off_t, - ) -> ::ssize_t; - pub fn preadv( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - offset: ::off_t, - ) -> ::ssize_t; + pub fn posix_fallocate(fd: ::c_int, offset: ::off_t, len: ::off_t) -> ::c_int; + pub fn pwritev(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int, offset: ::off_t) + -> ::ssize_t; + pub fn preadv(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int, offset: ::off_t) -> ::ssize_t; pub fn dup3(oldfd: ::c_int, newfd: ::c_int, flags: ::c_int) -> ::c_int; pub fn mkostemp(template: *mut ::c_char, flags: ::c_int) -> ::c_int; - pub fn mkostemps( - template: *mut ::c_char, - suffixlen: ::c_int, - flags: ::c_int, + pub fn mkostemps(template: *mut ::c_char, suffixlen: ::c_int, flags: ::c_int) -> ::c_int; + pub fn nl_langinfo_l(item: ::nl_item, locale: ::locale_t) -> *mut ::c_char; + pub fn accept4( + fd: ::c_int, + addr: *mut ::sockaddr, + len: *mut ::socklen_t, + flg: ::c_int, ) -> ::c_int; - pub fn nl_langinfo_l(item: ::nl_item, locale: ::locale_t) - -> *mut ::c_char; pub fn getnameinfo( sa: *const ::sockaddr, salen: ::socklen_t, @@ -1819,11 +1816,7 @@ extern "C" { pub fn getloadavg(loadavg: *mut ::c_double, nelem: ::c_int) -> ::c_int; // Not available now on Android - pub fn mkfifoat( - dirfd: ::c_int, - pathname: *const ::c_char, - mode: ::mode_t, - ) -> ::c_int; + pub fn mkfifoat(dirfd: ::c_int, pathname: *const ::c_char, mode: ::mode_t) -> ::c_int; pub fn if_nameindex() -> *mut if_nameindex; pub fn if_freenameindex(ptr: *mut if_nameindex); @@ -1838,35 +1831,21 @@ extern "C" { pub fn glob( pattern: *const c_char, flags: ::c_int, - errfunc: ::Option< - extern "C" fn(epath: *const c_char, errno: ::c_int) -> ::c_int, - >, + errfunc: ::Option ::c_int>, pglob: *mut ::glob_t, ) -> ::c_int; pub fn globfree(pglob: *mut ::glob_t); - pub fn posix_madvise( - addr: *mut ::c_void, - len: ::size_t, - advice: ::c_int, - ) -> ::c_int; + pub fn posix_madvise(addr: *mut ::c_void, len: ::size_t, advice: ::c_int) -> ::c_int; pub fn shm_unlink(name: *const ::c_char) -> ::c_int; pub fn seekdir(dirp: *mut ::DIR, loc: ::c_long); pub fn telldir(dirp: *mut ::DIR) -> ::c_long; - pub fn madvise( - addr: *mut ::c_void, - len: ::size_t, - advice: ::c_int, - ) -> ::c_int; + pub fn madvise(addr: *mut ::c_void, len: ::size_t, advice: ::c_int) -> ::c_int; - pub fn msync( - addr: *mut ::c_void, - len: ::size_t, - flags: ::c_int, - ) -> ::c_int; + pub fn msync(addr: *mut ::c_void, len: ::size_t, flags: ::c_int) -> ::c_int; pub fn recvfrom( socket: ::c_int, diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/align.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/align.rs index 09bf8c853..3a3277f29 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/align.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/align.rs @@ -8,7 +8,9 @@ macro_rules! expand_align { target_arch = "s390x", target_arch = "sparc64", target_arch = "aarch64", - target_arch = "riscv64"), + target_arch = "riscv64", + target_arch = "riscv32", + target_arch = "loongarch64"), repr(align(4)))] #[cfg_attr(not(any(target_pointer_width = "32", target_arch = "x86_64", @@ -17,7 +19,9 @@ macro_rules! expand_align { target_arch = "s390x", target_arch = "sparc64", target_arch = "aarch64", - target_arch = "riscv64")), + target_arch = "riscv64", + target_arch = "riscv32", + target_arch = "loongarch64")), repr(align(8)))] pub struct pthread_mutexattr_t { #[doc(hidden)] @@ -39,6 +43,17 @@ macro_rules! expand_align { #[doc(hidden)] size: [u8; ::__SIZEOF_PTHREAD_CONDATTR_T], } + + #[repr(align(8))] + pub struct fanotify_event_metadata { + pub event_len: __u32, + pub vers: __u8, + pub reserved: __u8, + pub metadata_len: __u16, + pub mask: __u64, + pub fd: ::c_int, + pub pid: ::c_int, + } } s_no_extra_traits! { @@ -63,6 +78,7 @@ macro_rules! expand_align { any(target_arch = "mips", target_arch = "arm", target_arch = "hexagon", + target_arch = "m68k", target_arch = "powerpc", target_arch = "sparc", target_arch = "x86_64", @@ -72,6 +88,7 @@ macro_rules! expand_align { not(any(target_arch = "mips", target_arch = "arm", target_arch = "hexagon", + target_arch = "m68k", target_arch = "powerpc", target_arch = "sparc", target_arch = "x86_64", @@ -86,6 +103,7 @@ macro_rules! expand_align { any(target_arch = "mips", target_arch = "arm", target_arch = "hexagon", + target_arch = "m68k", target_arch = "powerpc", target_arch = "sparc", target_arch = "x86_64", @@ -95,6 +113,7 @@ macro_rules! expand_align { not(any(target_arch = "mips", target_arch = "arm", target_arch = "hexagon", + target_arch = "m68k", target_arch = "powerpc", target_arch = "sparc", target_arch = "x86_64", @@ -103,6 +122,29 @@ macro_rules! expand_align { pub struct pthread_rwlock_t { size: [u8; ::__SIZEOF_PTHREAD_RWLOCK_T], } + + // linux/can.h + #[repr(align(8))] + #[allow(missing_debug_implementations)] + pub struct can_frame { + pub can_id: canid_t, + pub can_dlc: u8, + __pad: u8, + __res0: u8, + __res1: u8, + pub data: [u8; CAN_MAX_DLEN], + } + + #[repr(align(8))] + #[allow(missing_debug_implementations)] + pub struct canfd_frame { + pub can_id: canid_t, + pub len: u8, + pub flags: u8, + __res0: u8, + __res1: u8, + pub data: [u8; CANFD_MAX_DLEN], + } } }; } diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/arch/generic/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/arch/generic/mod.rs new file mode 100644 index 000000000..40bc30a4f --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/arch/generic/mod.rs @@ -0,0 +1,287 @@ +s! { + pub struct termios2 { + pub c_iflag: ::tcflag_t, + pub c_oflag: ::tcflag_t, + pub c_cflag: ::tcflag_t, + pub c_lflag: ::tcflag_t, + pub c_line: ::cc_t, + pub c_cc: [::cc_t; 19], + pub c_ispeed: ::speed_t, + pub c_ospeed: ::speed_t, + } +} + +// include/uapi/asm-generic/socket.h +// arch/alpha/include/uapi/asm/socket.h +// tools/include/uapi/asm-generic/socket.h +// arch/mips/include/uapi/asm/socket.h +pub const SOL_SOCKET: ::c_int = 1; + +// Defined in unix/linux_like/mod.rs +// pub const SO_DEBUG: ::c_int = 1; +pub const SO_REUSEADDR: ::c_int = 2; +pub const SO_TYPE: ::c_int = 3; +pub const SO_ERROR: ::c_int = 4; +pub const SO_DONTROUTE: ::c_int = 5; +pub const SO_BROADCAST: ::c_int = 6; +pub const SO_SNDBUF: ::c_int = 7; +pub const SO_RCVBUF: ::c_int = 8; +pub const SO_KEEPALIVE: ::c_int = 9; +pub const SO_OOBINLINE: ::c_int = 10; +pub const SO_NO_CHECK: ::c_int = 11; +pub const SO_PRIORITY: ::c_int = 12; +pub const SO_LINGER: ::c_int = 13; +pub const SO_BSDCOMPAT: ::c_int = 14; +pub const SO_REUSEPORT: ::c_int = 15; +pub const SO_PASSCRED: ::c_int = 16; +pub const SO_PEERCRED: ::c_int = 17; +pub const SO_RCVLOWAT: ::c_int = 18; +pub const SO_SNDLOWAT: ::c_int = 19; +pub const SO_RCVTIMEO: ::c_int = 20; +pub const SO_SNDTIMEO: ::c_int = 21; +// pub const SO_RCVTIMEO_OLD: ::c_int = 20; +// pub const SO_SNDTIMEO_OLD: ::c_int = 21; +pub const SO_SECURITY_AUTHENTICATION: ::c_int = 22; +pub const SO_SECURITY_ENCRYPTION_TRANSPORT: ::c_int = 23; +pub const SO_SECURITY_ENCRYPTION_NETWORK: ::c_int = 24; +pub const SO_BINDTODEVICE: ::c_int = 25; +pub const SO_ATTACH_FILTER: ::c_int = 26; +pub const SO_DETACH_FILTER: ::c_int = 27; +pub const SO_GET_FILTER: ::c_int = SO_ATTACH_FILTER; +pub const SO_PEERNAME: ::c_int = 28; +pub const SO_TIMESTAMP: ::c_int = 29; +// pub const SO_TIMESTAMP_OLD: ::c_int = 29; +pub const SO_ACCEPTCONN: ::c_int = 30; +pub const SO_PEERSEC: ::c_int = 31; +pub const SO_SNDBUFFORCE: ::c_int = 32; +pub const SO_RCVBUFFORCE: ::c_int = 33; +pub const SO_PASSSEC: ::c_int = 34; +pub const SO_TIMESTAMPNS: ::c_int = 35; +// pub const SO_TIMESTAMPNS_OLD: ::c_int = 35; +pub const SO_MARK: ::c_int = 36; +pub const SO_TIMESTAMPING: ::c_int = 37; +// pub const SO_TIMESTAMPING_OLD: ::c_int = 37; +pub const SO_PROTOCOL: ::c_int = 38; +pub const SO_DOMAIN: ::c_int = 39; +pub const SO_RXQ_OVFL: ::c_int = 40; +pub const SO_WIFI_STATUS: ::c_int = 41; +pub const SCM_WIFI_STATUS: ::c_int = SO_WIFI_STATUS; +pub const SO_PEEK_OFF: ::c_int = 42; +pub const SO_NOFCS: ::c_int = 43; +pub const SO_LOCK_FILTER: ::c_int = 44; +pub const SO_SELECT_ERR_QUEUE: ::c_int = 45; +pub const SO_BUSY_POLL: ::c_int = 46; +pub const SO_MAX_PACING_RATE: ::c_int = 47; +pub const SO_BPF_EXTENSIONS: ::c_int = 48; +pub const SO_INCOMING_CPU: ::c_int = 49; +pub const SO_ATTACH_BPF: ::c_int = 50; +pub const SO_DETACH_BPF: ::c_int = SO_DETACH_FILTER; +pub const SO_ATTACH_REUSEPORT_CBPF: ::c_int = 51; +pub const SO_ATTACH_REUSEPORT_EBPF: ::c_int = 52; +pub const SO_CNX_ADVICE: ::c_int = 53; +pub const SCM_TIMESTAMPING_OPT_STATS: ::c_int = 54; +pub const SO_MEMINFO: ::c_int = 55; +pub const SO_INCOMING_NAPI_ID: ::c_int = 56; +pub const SO_COOKIE: ::c_int = 57; +pub const SCM_TIMESTAMPING_PKTINFO: ::c_int = 58; +pub const SO_PEERGROUPS: ::c_int = 59; +pub const SO_ZEROCOPY: ::c_int = 60; +pub const SO_TXTIME: ::c_int = 61; +pub const SCM_TXTIME: ::c_int = SO_TXTIME; +pub const SO_BINDTOIFINDEX: ::c_int = 62; +cfg_if! { + // Some of these platforms in CI already have these constants. + // But they may still not have those _OLD ones. + if #[cfg(all(any(target_arch = "x86", + target_arch = "x86_64", + target_arch = "aarch64"), + not(target_env = "musl")))] { + pub const SO_TIMESTAMP_NEW: ::c_int = 63; + pub const SO_TIMESTAMPNS_NEW: ::c_int = 64; + pub const SO_TIMESTAMPING_NEW: ::c_int = 65; + pub const SO_RCVTIMEO_NEW: ::c_int = 66; + pub const SO_SNDTIMEO_NEW: ::c_int = 67; + pub const SO_DETACH_REUSEPORT_BPF: ::c_int = 68; + } +} + +cfg_if! { + if #[cfg(any(target_arch = "x86", + target_arch = "x86_64", + target_arch = "aarch64"))] { + pub const FICLONE: ::c_ulong = 0x40049409; + pub const FICLONERANGE: ::c_ulong = 0x4020940D; + } +} +// pub const SO_PREFER_BUSY_POLL: ::c_int = 69; +// pub const SO_BUSY_POLL_BUDGET: ::c_int = 70; + +// Defined in unix/linux_like/mod.rs +// pub const SCM_TIMESTAMP: ::c_int = SO_TIMESTAMP; +pub const SCM_TIMESTAMPNS: ::c_int = SO_TIMESTAMPNS; +pub const SCM_TIMESTAMPING: ::c_int = SO_TIMESTAMPING; + +// Ioctl Constants + +pub const TCGETS: ::Ioctl = 0x5401; +pub const TCSETS: ::Ioctl = 0x5402; +pub const TCSETSW: ::Ioctl = 0x5403; +pub const TCSETSF: ::Ioctl = 0x5404; +pub const TCGETA: ::Ioctl = 0x5405; +pub const TCSETA: ::Ioctl = 0x5406; +pub const TCSETAW: ::Ioctl = 0x5407; +pub const TCSETAF: ::Ioctl = 0x5408; +pub const TCSBRK: ::Ioctl = 0x5409; +pub const TCXONC: ::Ioctl = 0x540A; +pub const TCFLSH: ::Ioctl = 0x540B; +pub const TIOCEXCL: ::Ioctl = 0x540C; +pub const TIOCNXCL: ::Ioctl = 0x540D; +pub const TIOCSCTTY: ::Ioctl = 0x540E; +pub const TIOCGPGRP: ::Ioctl = 0x540F; +pub const TIOCSPGRP: ::Ioctl = 0x5410; +pub const TIOCOUTQ: ::Ioctl = 0x5411; +pub const TIOCSTI: ::Ioctl = 0x5412; +pub const TIOCGWINSZ: ::Ioctl = 0x5413; +pub const TIOCSWINSZ: ::Ioctl = 0x5414; +pub const TIOCMGET: ::Ioctl = 0x5415; +pub const TIOCMBIS: ::Ioctl = 0x5416; +pub const TIOCMBIC: ::Ioctl = 0x5417; +pub const TIOCMSET: ::Ioctl = 0x5418; +pub const TIOCGSOFTCAR: ::Ioctl = 0x5419; +pub const TIOCSSOFTCAR: ::Ioctl = 0x541A; +pub const FIONREAD: ::Ioctl = 0x541B; +pub const TIOCINQ: ::Ioctl = FIONREAD; +pub const TIOCLINUX: ::Ioctl = 0x541C; +pub const TIOCCONS: ::Ioctl = 0x541D; +pub const TIOCGSERIAL: ::Ioctl = 0x541E; +pub const TIOCSSERIAL: ::Ioctl = 0x541F; +pub const TIOCPKT: ::Ioctl = 0x5420; +pub const FIONBIO: ::Ioctl = 0x5421; +pub const TIOCNOTTY: ::Ioctl = 0x5422; +pub const TIOCSETD: ::Ioctl = 0x5423; +pub const TIOCGETD: ::Ioctl = 0x5424; +pub const TCSBRKP: ::Ioctl = 0x5425; +pub const TIOCSBRK: ::Ioctl = 0x5427; +pub const TIOCCBRK: ::Ioctl = 0x5428; +pub const TIOCGSID: ::Ioctl = 0x5429; +pub const TCGETS2: ::Ioctl = 0x802c542a; +pub const TCSETS2: ::Ioctl = 0x402c542b; +pub const TCSETSW2: ::Ioctl = 0x402c542c; +pub const TCSETSF2: ::Ioctl = 0x402c542d; +pub const TIOCGRS485: ::Ioctl = 0x542E; +pub const TIOCSRS485: ::Ioctl = 0x542F; +pub const TIOCGPTN: ::Ioctl = 0x80045430; +pub const TIOCSPTLCK: ::Ioctl = 0x40045431; +pub const TIOCGDEV: ::Ioctl = 0x80045432; +pub const TCGETX: ::Ioctl = 0x5432; +pub const TCSETX: ::Ioctl = 0x5433; +pub const TCSETXF: ::Ioctl = 0x5434; +pub const TCSETXW: ::Ioctl = 0x5435; +pub const TIOCSIG: ::Ioctl = 0x40045436; +pub const TIOCVHANGUP: ::Ioctl = 0x5437; +pub const TIOCGPKT: ::Ioctl = 0x80045438; +pub const TIOCGPTLCK: ::Ioctl = 0x80045439; +pub const TIOCGEXCL: ::Ioctl = 0x80045440; +pub const TIOCGPTPEER: ::Ioctl = 0x5441; +// pub const TIOCGISO7816: ::Ioctl = 0x80285442; +// pub const TIOCSISO7816: ::Ioctl = 0xc0285443; +pub const FIONCLEX: ::Ioctl = 0x5450; +pub const FIOCLEX: ::Ioctl = 0x5451; +pub const FIOASYNC: ::Ioctl = 0x5452; +pub const TIOCSERCONFIG: ::Ioctl = 0x5453; +pub const TIOCSERGWILD: ::Ioctl = 0x5454; +pub const TIOCSERSWILD: ::Ioctl = 0x5455; +pub const TIOCGLCKTRMIOS: ::Ioctl = 0x5456; +pub const TIOCSLCKTRMIOS: ::Ioctl = 0x5457; +pub const TIOCSERGSTRUCT: ::Ioctl = 0x5458; +pub const TIOCSERGETLSR: ::Ioctl = 0x5459; +pub const TIOCSERGETMULTI: ::Ioctl = 0x545A; +pub const TIOCSERSETMULTI: ::Ioctl = 0x545B; +pub const TIOCMIWAIT: ::Ioctl = 0x545C; +pub const TIOCGICOUNT: ::Ioctl = 0x545D; +pub const BLKIOMIN: ::Ioctl = 0x1278; +pub const BLKIOOPT: ::Ioctl = 0x1279; +pub const BLKSSZGET: ::Ioctl = 0x1268; +pub const BLKPBSZGET: ::Ioctl = 0x127B; + +cfg_if! { + if #[cfg(any(target_arch = "arm", + target_arch = "s390x"))] { + pub const FIOQSIZE: ::Ioctl = 0x545E; + } else { + pub const FIOQSIZE: ::Ioctl = 0x5460; + } +} + +pub const TIOCM_LE: ::c_int = 0x001; +pub const TIOCM_DTR: ::c_int = 0x002; +pub const TIOCM_RTS: ::c_int = 0x004; +pub const TIOCM_ST: ::c_int = 0x008; +pub const TIOCM_SR: ::c_int = 0x010; +pub const TIOCM_CTS: ::c_int = 0x020; +pub const TIOCM_CAR: ::c_int = 0x040; +pub const TIOCM_CD: ::c_int = TIOCM_CAR; +pub const TIOCM_RNG: ::c_int = 0x080; +pub const TIOCM_RI: ::c_int = TIOCM_RNG; +pub const TIOCM_DSR: ::c_int = 0x100; + +pub const BOTHER: ::speed_t = 0o010000; +pub const IBSHIFT: ::tcflag_t = 16; + +// RLIMIT Constants + +cfg_if! { + if #[cfg(any(target_env = "gnu", + target_env = "uclibc"))] { + + pub const RLIMIT_CPU: ::__rlimit_resource_t = 0; + pub const RLIMIT_FSIZE: ::__rlimit_resource_t = 1; + pub const RLIMIT_DATA: ::__rlimit_resource_t = 2; + pub const RLIMIT_STACK: ::__rlimit_resource_t = 3; + pub const RLIMIT_CORE: ::__rlimit_resource_t = 4; + pub const RLIMIT_RSS: ::__rlimit_resource_t = 5; + pub const RLIMIT_NPROC: ::__rlimit_resource_t = 6; + pub const RLIMIT_NOFILE: ::__rlimit_resource_t = 7; + pub const RLIMIT_MEMLOCK: ::__rlimit_resource_t = 8; + pub const RLIMIT_AS: ::__rlimit_resource_t = 9; + pub const RLIMIT_LOCKS: ::__rlimit_resource_t = 10; + pub const RLIMIT_SIGPENDING: ::__rlimit_resource_t = 11; + pub const RLIMIT_MSGQUEUE: ::__rlimit_resource_t = 12; + pub const RLIMIT_NICE: ::__rlimit_resource_t = 13; + pub const RLIMIT_RTPRIO: ::__rlimit_resource_t = 14; + pub const RLIMIT_RTTIME: ::__rlimit_resource_t = 15; + pub const RLIMIT_NLIMITS: ::__rlimit_resource_t = RLIM_NLIMITS; + + } else if #[cfg(target_env = "musl")] { + + pub const RLIMIT_CPU: ::c_int = 0; + pub const RLIMIT_FSIZE: ::c_int = 1; + pub const RLIMIT_DATA: ::c_int = 2; + pub const RLIMIT_STACK: ::c_int = 3; + pub const RLIMIT_CORE: ::c_int = 4; + pub const RLIMIT_RSS: ::c_int = 5; + pub const RLIMIT_NPROC: ::c_int = 6; + pub const RLIMIT_NOFILE: ::c_int = 7; + pub const RLIMIT_MEMLOCK: ::c_int = 8; + pub const RLIMIT_AS: ::c_int = 9; + pub const RLIMIT_LOCKS: ::c_int = 10; + pub const RLIMIT_SIGPENDING: ::c_int = 11; + pub const RLIMIT_MSGQUEUE: ::c_int = 12; + pub const RLIMIT_NICE: ::c_int = 13; + pub const RLIMIT_RTPRIO: ::c_int = 14; + pub const RLIMIT_RTTIME: ::c_int = 15; + pub const RLIM_NLIMITS: ::c_int = 15; + pub const RLIMIT_NLIMITS: ::c_int = RLIM_NLIMITS; + } +} + +cfg_if! { + if #[cfg(target_env = "gnu")] { + pub const RLIM_NLIMITS: ::__rlimit_resource_t = 16; + } + else if #[cfg(target_env = "uclibc")] { + pub const RLIM_NLIMITS: ::__rlimit_resource_t = 15; + } +} + +pub const RLIM_INFINITY: ::rlim_t = !0; diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/arch/mips/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/arch/mips/mod.rs new file mode 100644 index 000000000..077417de5 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/arch/mips/mod.rs @@ -0,0 +1,285 @@ +s! { + pub struct termios2 { + pub c_iflag: ::tcflag_t, + pub c_oflag: ::tcflag_t, + pub c_cflag: ::tcflag_t, + pub c_lflag: ::tcflag_t, + pub c_line: ::cc_t, + pub c_cc: [::cc_t; 23], + pub c_ispeed: ::speed_t, + pub c_ospeed: ::speed_t, + } +} + +// arch/mips/include/uapi/asm/socket.h +pub const SOL_SOCKET: ::c_int = 0xffff; + +// Defined in unix/linux_like/mod.rs +// pub const SO_DEBUG: ::c_int = 0x0001; +pub const SO_REUSEADDR: ::c_int = 0x0004; +pub const SO_KEEPALIVE: ::c_int = 0x0008; +pub const SO_DONTROUTE: ::c_int = 0x0010; +pub const SO_BROADCAST: ::c_int = 0x0020; +pub const SO_LINGER: ::c_int = 0x0080; +pub const SO_OOBINLINE: ::c_int = 0x0100; +pub const SO_REUSEPORT: ::c_int = 0x0200; +pub const SO_TYPE: ::c_int = 0x1008; +// pub const SO_STYLE: ::c_int = SO_TYPE; +pub const SO_ERROR: ::c_int = 0x1007; +pub const SO_SNDBUF: ::c_int = 0x1001; +pub const SO_RCVBUF: ::c_int = 0x1002; +pub const SO_SNDLOWAT: ::c_int = 0x1003; +pub const SO_RCVLOWAT: ::c_int = 0x1004; +// NOTE: These definitions are now being renamed with _OLD postfix, +// but CI haven't support them yet. +// Some related consts could be found in b32.rs and b64.rs +pub const SO_SNDTIMEO: ::c_int = 0x1005; +pub const SO_RCVTIMEO: ::c_int = 0x1006; +// pub const SO_SNDTIMEO_OLD: ::c_int = 0x1005; +// pub const SO_RCVTIMEO_OLD: ::c_int = 0x1006; +pub const SO_ACCEPTCONN: ::c_int = 0x1009; +pub const SO_PROTOCOL: ::c_int = 0x1028; +pub const SO_DOMAIN: ::c_int = 0x1029; + +pub const SO_NO_CHECK: ::c_int = 11; +pub const SO_PRIORITY: ::c_int = 12; +pub const SO_BSDCOMPAT: ::c_int = 14; +pub const SO_PASSCRED: ::c_int = 17; +pub const SO_PEERCRED: ::c_int = 18; +pub const SO_SECURITY_AUTHENTICATION: ::c_int = 22; +pub const SO_SECURITY_ENCRYPTION_TRANSPORT: ::c_int = 23; +pub const SO_SECURITY_ENCRYPTION_NETWORK: ::c_int = 24; +pub const SO_BINDTODEVICE: ::c_int = 25; +pub const SO_ATTACH_FILTER: ::c_int = 26; +pub const SO_DETACH_FILTER: ::c_int = 27; +pub const SO_GET_FILTER: ::c_int = SO_ATTACH_FILTER; +pub const SO_PEERNAME: ::c_int = 28; +pub const SO_PEERSEC: ::c_int = 30; +pub const SO_SNDBUFFORCE: ::c_int = 31; +pub const SO_RCVBUFFORCE: ::c_int = 33; +pub const SO_PASSSEC: ::c_int = 34; +pub const SO_MARK: ::c_int = 36; +pub const SO_RXQ_OVFL: ::c_int = 40; +pub const SO_WIFI_STATUS: ::c_int = 41; +pub const SCM_WIFI_STATUS: ::c_int = SO_WIFI_STATUS; +pub const SO_PEEK_OFF: ::c_int = 42; +pub const SO_NOFCS: ::c_int = 43; +pub const SO_LOCK_FILTER: ::c_int = 44; +pub const SO_SELECT_ERR_QUEUE: ::c_int = 45; +pub const SO_BUSY_POLL: ::c_int = 46; +pub const SO_MAX_PACING_RATE: ::c_int = 47; +pub const SO_BPF_EXTENSIONS: ::c_int = 48; +pub const SO_INCOMING_CPU: ::c_int = 49; +pub const SO_ATTACH_BPF: ::c_int = 50; +pub const SO_DETACH_BPF: ::c_int = SO_DETACH_FILTER; +pub const SO_ATTACH_REUSEPORT_CBPF: ::c_int = 51; +pub const SO_ATTACH_REUSEPORT_EBPF: ::c_int = 52; +pub const SO_CNX_ADVICE: ::c_int = 53; +pub const SCM_TIMESTAMPING_OPT_STATS: ::c_int = 54; +pub const SO_MEMINFO: ::c_int = 55; +pub const SO_INCOMING_NAPI_ID: ::c_int = 56; +pub const SO_COOKIE: ::c_int = 57; +pub const SCM_TIMESTAMPING_PKTINFO: ::c_int = 58; +pub const SO_PEERGROUPS: ::c_int = 59; +pub const SO_ZEROCOPY: ::c_int = 60; +pub const SO_TXTIME: ::c_int = 61; +pub const SCM_TXTIME: ::c_int = SO_TXTIME; +pub const SO_BINDTOIFINDEX: ::c_int = 62; +// NOTE: These definitions are now being renamed with _OLD postfix, +// but CI haven't support them yet. +// Some related consts could be found in b32.rs and b64.rs +pub const SO_TIMESTAMP: ::c_int = 29; +pub const SO_TIMESTAMPNS: ::c_int = 35; +pub const SO_TIMESTAMPING: ::c_int = 37; +// pub const SO_TIMESTAMP_OLD: ::c_int = 29; +// pub const SO_TIMESTAMPNS_OLD: ::c_int = 35; +// pub const SO_TIMESTAMPING_OLD: ::c_int = 37; +// pub const SO_TIMESTAMP_NEW: ::c_int = 63; +// pub const SO_TIMESTAMPNS_NEW: ::c_int = 64; +// pub const SO_TIMESTAMPING_NEW: ::c_int = 65; +// pub const SO_RCVTIMEO_NEW: ::c_int = 66; +// pub const SO_SNDTIMEO_NEW: ::c_int = 67; +// pub const SO_DETACH_REUSEPORT_BPF: ::c_int = 68; +// pub const SO_PREFER_BUSY_POLL: ::c_int = 69; +// pub const SO_BUSY_POLL_BUDGET: ::c_int = 70; + +// Defined in unix/linux_like/mod.rs +// pub const SCM_TIMESTAMP: ::c_int = SO_TIMESTAMP; +pub const SCM_TIMESTAMPNS: ::c_int = SO_TIMESTAMPNS; +pub const SCM_TIMESTAMPING: ::c_int = SO_TIMESTAMPING; + +// Ioctl Constants + +pub const TCGETS: ::Ioctl = 0x540d; +pub const TCSETS: ::Ioctl = 0x540e; +pub const TCSETSW: ::Ioctl = 0x540f; +pub const TCSETSF: ::Ioctl = 0x5410; +pub const TCGETA: ::Ioctl = 0x5401; +pub const TCSETA: ::Ioctl = 0x5402; +pub const TCSETAW: ::Ioctl = 0x5403; +pub const TCSETAF: ::Ioctl = 0x5404; +pub const TCSBRK: ::Ioctl = 0x5405; +pub const TCXONC: ::Ioctl = 0x5406; +pub const TCFLSH: ::Ioctl = 0x5407; +pub const TIOCEXCL: ::Ioctl = 0x740d; +pub const TIOCNXCL: ::Ioctl = 0x740e; +pub const TIOCSCTTY: ::Ioctl = 0x5480; +pub const TIOCGPGRP: ::Ioctl = 0x40047477; +pub const TIOCSPGRP: ::Ioctl = 0x80047476; +pub const TIOCOUTQ: ::Ioctl = 0x7472; +pub const TIOCSTI: ::Ioctl = 0x5472; +pub const TIOCGWINSZ: ::Ioctl = 0x40087468; +pub const TIOCSWINSZ: ::Ioctl = 0x80087467; +pub const TIOCMGET: ::Ioctl = 0x741d; +pub const TIOCMBIS: ::Ioctl = 0x741b; +pub const TIOCMBIC: ::Ioctl = 0x741c; +pub const TIOCMSET: ::Ioctl = 0x741a; +pub const TIOCGSOFTCAR: ::Ioctl = 0x5481; +pub const TIOCSSOFTCAR: ::Ioctl = 0x5482; +pub const FIONREAD: ::Ioctl = 0x467f; +pub const TIOCINQ: ::Ioctl = FIONREAD; +pub const TIOCLINUX: ::Ioctl = 0x5483; +pub const TIOCCONS: ::Ioctl = 0x80047478; +pub const TIOCGSERIAL: ::Ioctl = 0x5484; +pub const TIOCSSERIAL: ::Ioctl = 0x5485; +pub const TIOCPKT: ::Ioctl = 0x5470; +pub const FIONBIO: ::Ioctl = 0x667e; +pub const TIOCNOTTY: ::Ioctl = 0x5471; +pub const TIOCSETD: ::Ioctl = 0x7401; +pub const TIOCGETD: ::Ioctl = 0x7400; +pub const TCSBRKP: ::Ioctl = 0x5486; +pub const TIOCSBRK: ::Ioctl = 0x5427; +pub const TIOCCBRK: ::Ioctl = 0x5428; +pub const TIOCGSID: ::Ioctl = 0x7416; +pub const TCGETS2: ::Ioctl = 0x4030542a; +pub const TCSETS2: ::Ioctl = 0x8030542b; +pub const TCSETSW2: ::Ioctl = 0x8030542c; +pub const TCSETSF2: ::Ioctl = 0x8030542d; +pub const TIOCGPTN: ::Ioctl = 0x40045430; +pub const TIOCSPTLCK: ::Ioctl = 0x80045431; +pub const TIOCGDEV: ::Ioctl = 0x40045432; +pub const TIOCSIG: ::Ioctl = 0x80045436; +pub const TIOCVHANGUP: ::Ioctl = 0x5437; +pub const TIOCGPKT: ::Ioctl = 0x40045438; +pub const TIOCGPTLCK: ::Ioctl = 0x40045439; +pub const TIOCGEXCL: ::Ioctl = 0x40045440; +pub const TIOCGPTPEER: ::Ioctl = 0x20005441; +//pub const TIOCGISO7816: ::Ioctl = 0x40285442; +//pub const TIOCSISO7816: ::Ioctl = 0xc0285443; +pub const FIONCLEX: ::Ioctl = 0x6602; +pub const FIOCLEX: ::Ioctl = 0x6601; +pub const FIOASYNC: ::Ioctl = 0x667d; +pub const TIOCSERCONFIG: ::Ioctl = 0x5488; +pub const TIOCSERGWILD: ::Ioctl = 0x5489; +pub const TIOCSERSWILD: ::Ioctl = 0x548a; +pub const TIOCGLCKTRMIOS: ::Ioctl = 0x548b; +pub const TIOCSLCKTRMIOS: ::Ioctl = 0x548c; +pub const TIOCSERGSTRUCT: ::Ioctl = 0x548d; +pub const TIOCSERGETLSR: ::Ioctl = 0x548e; +pub const TIOCSERGETMULTI: ::Ioctl = 0x548f; +pub const TIOCSERSETMULTI: ::Ioctl = 0x5490; +pub const TIOCMIWAIT: ::Ioctl = 0x5491; +pub const TIOCGICOUNT: ::Ioctl = 0x5492; +pub const FIOQSIZE: ::Ioctl = 0x667f; +pub const TIOCSLTC: ::Ioctl = 0x7475; +pub const TIOCGETP: ::Ioctl = 0x7408; +pub const TIOCSETP: ::Ioctl = 0x7409; +pub const TIOCSETN: ::Ioctl = 0x740a; +pub const BLKIOMIN: ::Ioctl = 0x20001278; +pub const BLKIOOPT: ::Ioctl = 0x20001279; +pub const BLKSSZGET: ::Ioctl = 0x20001268; +pub const BLKPBSZGET: ::Ioctl = 0x2000127B; + +cfg_if! { + if #[cfg(target_env = "musl")] { + pub const TIOCGRS485: ::Ioctl = 0x4020542e; + pub const TIOCSRS485: ::Ioctl = 0xc020542f; + } +} + +pub const TIOCM_LE: ::c_int = 0x001; +pub const TIOCM_DTR: ::c_int = 0x002; +pub const TIOCM_RTS: ::c_int = 0x004; +pub const TIOCM_ST: ::c_int = 0x010; +pub const TIOCM_SR: ::c_int = 0x020; +pub const TIOCM_CTS: ::c_int = 0x040; +pub const TIOCM_CAR: ::c_int = 0x100; +pub const TIOCM_CD: ::c_int = TIOCM_CAR; +pub const TIOCM_RNG: ::c_int = 0x200; +pub const TIOCM_RI: ::c_int = TIOCM_RNG; +pub const TIOCM_DSR: ::c_int = 0x400; + +pub const BOTHER: ::speed_t = 0o010000; +pub const IBSHIFT: ::tcflag_t = 16; + +// RLIMIT Constants + +cfg_if! { + if #[cfg(any(target_env = "gnu", + target_env = "uclibc"))] { + + pub const RLIMIT_CPU: ::__rlimit_resource_t = 0; + pub const RLIMIT_FSIZE: ::__rlimit_resource_t = 1; + pub const RLIMIT_DATA: ::__rlimit_resource_t = 2; + pub const RLIMIT_STACK: ::__rlimit_resource_t = 3; + pub const RLIMIT_CORE: ::__rlimit_resource_t = 4; + pub const RLIMIT_NOFILE: ::__rlimit_resource_t = 5; + pub const RLIMIT_AS: ::__rlimit_resource_t = 6; + pub const RLIMIT_RSS: ::__rlimit_resource_t = 7; + pub const RLIMIT_NPROC: ::__rlimit_resource_t = 8; + pub const RLIMIT_MEMLOCK: ::__rlimit_resource_t = 9; + pub const RLIMIT_LOCKS: ::__rlimit_resource_t = 10; + pub const RLIMIT_SIGPENDING: ::__rlimit_resource_t = 11; + pub const RLIMIT_MSGQUEUE: ::__rlimit_resource_t = 12; + pub const RLIMIT_NICE: ::__rlimit_resource_t = 13; + pub const RLIMIT_RTPRIO: ::__rlimit_resource_t = 14; + pub const RLIMIT_RTTIME: ::__rlimit_resource_t = 15; + pub const RLIMIT_NLIMITS: ::__rlimit_resource_t = RLIM_NLIMITS; + + } else if #[cfg(target_env = "musl")] { + + pub const RLIMIT_CPU: ::c_int = 0; + pub const RLIMIT_FSIZE: ::c_int = 1; + pub const RLIMIT_DATA: ::c_int = 2; + pub const RLIMIT_STACK: ::c_int = 3; + pub const RLIMIT_CORE: ::c_int = 4; + pub const RLIMIT_NOFILE: ::c_int = 5; + pub const RLIMIT_AS: ::c_int = 6; + pub const RLIMIT_RSS: ::c_int = 7; + pub const RLIMIT_NPROC: ::c_int = 8; + pub const RLIMIT_MEMLOCK: ::c_int = 9; + pub const RLIMIT_LOCKS: ::c_int = 10; + pub const RLIMIT_SIGPENDING: ::c_int = 11; + pub const RLIMIT_MSGQUEUE: ::c_int = 12; + pub const RLIMIT_NICE: ::c_int = 13; + pub const RLIMIT_RTPRIO: ::c_int = 14; + pub const RLIMIT_RTTIME: ::c_int = 15; + pub const RLIM_NLIMITS: ::c_int = 15; + pub const RLIMIT_NLIMITS: ::c_int = RLIM_NLIMITS; + pub const RLIM_INFINITY: ::rlim_t = !0; + } +} + +cfg_if! { + if #[cfg(target_env = "gnu")] { + pub const RLIM_NLIMITS: ::__rlimit_resource_t = 16; + } else if #[cfg(target_env = "uclibc")] { + pub const RLIM_NLIMITS: ::__rlimit_resource_t = 15; + } +} + +cfg_if! { + if #[cfg(target_arch = "mips64", + any(target_env = "gnu", + target_env = "uclibc"))] { + pub const RLIM_INFINITY: ::rlim_t = !0; + } +} + +cfg_if! { + if #[cfg(target_arch = "mips", + any(target_env = "gnu", + target_env = "uclibc"))] { + pub const RLIM_INFINITY: ::rlim_t = 0x7fffffff; + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/arch/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/arch/mod.rs new file mode 100644 index 000000000..c1528f593 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/arch/mod.rs @@ -0,0 +1,15 @@ +cfg_if! { + if #[cfg(any(target_arch = "mips", target_arch = "mips64"))] { + mod mips; + pub use self::mips::*; + } else if #[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] { + mod powerpc; + pub use self::powerpc::*; + } else if #[cfg(any(target_arch = "sparc", target_arch = "sparc64"))] { + mod sparc; + pub use self::sparc::*; + } else { + mod generic; + pub use self::generic::*; + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/arch/powerpc/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/arch/powerpc/mod.rs new file mode 100644 index 000000000..637b7a1e3 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/arch/powerpc/mod.rs @@ -0,0 +1,240 @@ +// arch/powerpc/include/uapi/asm/socket.h + +pub const SOL_SOCKET: ::c_int = 1; + +// Defined in unix/linux_like/mod.rs +// pub const SO_DEBUG: ::c_int = 1; +pub const SO_REUSEADDR: ::c_int = 2; +pub const SO_TYPE: ::c_int = 3; +pub const SO_ERROR: ::c_int = 4; +pub const SO_DONTROUTE: ::c_int = 5; +pub const SO_BROADCAST: ::c_int = 6; +pub const SO_SNDBUF: ::c_int = 7; +pub const SO_RCVBUF: ::c_int = 8; +pub const SO_KEEPALIVE: ::c_int = 9; +pub const SO_OOBINLINE: ::c_int = 10; +pub const SO_NO_CHECK: ::c_int = 11; +pub const SO_PRIORITY: ::c_int = 12; +pub const SO_LINGER: ::c_int = 13; +pub const SO_BSDCOMPAT: ::c_int = 14; +pub const SO_REUSEPORT: ::c_int = 15; +// powerpc only differs in these +pub const SO_RCVLOWAT: ::c_int = 16; +pub const SO_SNDLOWAT: ::c_int = 17; +pub const SO_RCVTIMEO: ::c_int = 18; +pub const SO_SNDTIMEO: ::c_int = 19; +// pub const SO_RCVTIMEO_OLD: ::c_int = 18; +// pub const SO_SNDTIMEO_OLD: ::c_int = 19; +pub const SO_PASSCRED: ::c_int = 20; +pub const SO_PEERCRED: ::c_int = 21; +// end +pub const SO_SECURITY_AUTHENTICATION: ::c_int = 22; +pub const SO_SECURITY_ENCRYPTION_TRANSPORT: ::c_int = 23; +pub const SO_SECURITY_ENCRYPTION_NETWORK: ::c_int = 24; +pub const SO_BINDTODEVICE: ::c_int = 25; +pub const SO_ATTACH_FILTER: ::c_int = 26; +pub const SO_DETACH_FILTER: ::c_int = 27; +pub const SO_GET_FILTER: ::c_int = SO_ATTACH_FILTER; +pub const SO_PEERNAME: ::c_int = 28; +pub const SO_TIMESTAMP: ::c_int = 29; +// pub const SO_TIMESTAMP_OLD: ::c_int = 29; +pub const SO_ACCEPTCONN: ::c_int = 30; +pub const SO_PEERSEC: ::c_int = 31; +pub const SO_SNDBUFFORCE: ::c_int = 32; +pub const SO_RCVBUFFORCE: ::c_int = 33; +pub const SO_PASSSEC: ::c_int = 34; +pub const SO_TIMESTAMPNS: ::c_int = 35; +// pub const SO_TIMESTAMPNS_OLD: ::c_int = 35; +pub const SO_MARK: ::c_int = 36; +pub const SO_TIMESTAMPING: ::c_int = 37; +// pub const SO_TIMESTAMPING_OLD: ::c_int = 37; +pub const SO_PROTOCOL: ::c_int = 38; +pub const SO_DOMAIN: ::c_int = 39; +pub const SO_RXQ_OVFL: ::c_int = 40; +pub const SO_WIFI_STATUS: ::c_int = 41; +pub const SCM_WIFI_STATUS: ::c_int = SO_WIFI_STATUS; +pub const SO_PEEK_OFF: ::c_int = 42; +pub const SO_NOFCS: ::c_int = 43; +pub const SO_LOCK_FILTER: ::c_int = 44; +pub const SO_SELECT_ERR_QUEUE: ::c_int = 45; +pub const SO_BUSY_POLL: ::c_int = 46; +pub const SO_MAX_PACING_RATE: ::c_int = 47; +pub const SO_BPF_EXTENSIONS: ::c_int = 48; +pub const SO_INCOMING_CPU: ::c_int = 49; +pub const SO_ATTACH_BPF: ::c_int = 50; +pub const SO_DETACH_BPF: ::c_int = SO_DETACH_FILTER; +pub const SO_ATTACH_REUSEPORT_CBPF: ::c_int = 51; +pub const SO_ATTACH_REUSEPORT_EBPF: ::c_int = 52; +pub const SO_CNX_ADVICE: ::c_int = 53; +pub const SCM_TIMESTAMPING_OPT_STATS: ::c_int = 54; +pub const SO_MEMINFO: ::c_int = 55; +pub const SO_INCOMING_NAPI_ID: ::c_int = 56; +pub const SO_COOKIE: ::c_int = 57; +pub const SCM_TIMESTAMPING_PKTINFO: ::c_int = 58; +pub const SO_PEERGROUPS: ::c_int = 59; +pub const SO_ZEROCOPY: ::c_int = 60; +pub const SO_TXTIME: ::c_int = 61; +pub const SCM_TXTIME: ::c_int = SO_TXTIME; +pub const SO_BINDTOIFINDEX: ::c_int = 62; +// pub const SO_TIMESTAMP_NEW: ::c_int = 63; +// pub const SO_TIMESTAMPNS_NEW: ::c_int = 64; +// pub const SO_TIMESTAMPING_NEW: ::c_int = 65; +// pub const SO_RCVTIMEO_NEW: ::c_int = 66; +// pub const SO_SNDTIMEO_NEW: ::c_int = 67; +// pub const SO_DETACH_REUSEPORT_BPF: ::c_int = 68; +// pub const SO_PREFER_BUSY_POLL: ::c_int = 69; +// pub const SO_BUSY_POLL_BUDGET: ::c_int = 70; + +// Defined in unix/linux_like/mod.rs +// pub const SCM_TIMESTAMP: ::c_int = SO_TIMESTAMP; +pub const SCM_TIMESTAMPNS: ::c_int = SO_TIMESTAMPNS; +pub const SCM_TIMESTAMPING: ::c_int = SO_TIMESTAMPING; + +// Ioctl Constants + +cfg_if! { + if #[cfg(target_env = "gnu")] { + pub const TCGETS: ::Ioctl = 0x403c7413; + pub const TCSETS: ::Ioctl = 0x803c7414; + pub const TCSETSW: ::Ioctl = 0x803c7415; + pub const TCSETSF: ::Ioctl = 0x803c7416; + } else if #[cfg(target_env = "musl")] { + pub const TCGETS: ::Ioctl = 0x402c7413; + pub const TCSETS: ::Ioctl = 0x802c7414; + pub const TCSETSW: ::Ioctl = 0x802c7415; + pub const TCSETSF: ::Ioctl = 0x802c7416; + } +} + +pub const TCGETA: ::Ioctl = 0x40147417; +pub const TCSETA: ::Ioctl = 0x80147418; +pub const TCSETAW: ::Ioctl = 0x80147419; +pub const TCSETAF: ::Ioctl = 0x8014741C; +pub const TCSBRK: ::Ioctl = 0x2000741D; +pub const TCXONC: ::Ioctl = 0x2000741E; +pub const TCFLSH: ::Ioctl = 0x2000741F; +pub const TIOCEXCL: ::Ioctl = 0x540C; +pub const TIOCNXCL: ::Ioctl = 0x540D; +pub const TIOCSCTTY: ::Ioctl = 0x540E; +pub const TIOCGPGRP: ::Ioctl = 0x40047477; +pub const TIOCSPGRP: ::Ioctl = 0x80047476; +pub const TIOCOUTQ: ::Ioctl = 0x40047473; +pub const TIOCSTI: ::Ioctl = 0x5412; +pub const TIOCGWINSZ: ::Ioctl = 0x40087468; +pub const TIOCSWINSZ: ::Ioctl = 0x80087467; +pub const TIOCMGET: ::Ioctl = 0x5415; +pub const TIOCMBIS: ::Ioctl = 0x5416; +pub const TIOCMBIC: ::Ioctl = 0x5417; +pub const TIOCMSET: ::Ioctl = 0x5418; +pub const TIOCGSOFTCAR: ::Ioctl = 0x5419; +pub const TIOCSSOFTCAR: ::Ioctl = 0x541A; +pub const FIONREAD: ::Ioctl = 0x4004667F; +pub const TIOCINQ: ::Ioctl = FIONREAD; +pub const TIOCLINUX: ::Ioctl = 0x541C; +pub const TIOCCONS: ::Ioctl = 0x541D; +pub const TIOCGSERIAL: ::Ioctl = 0x541E; +pub const TIOCSSERIAL: ::Ioctl = 0x541F; +pub const TIOCPKT: ::Ioctl = 0x5420; +pub const FIONBIO: ::Ioctl = 0x8004667e; +pub const TIOCNOTTY: ::Ioctl = 0x5422; +pub const TIOCSETD: ::Ioctl = 0x5423; +pub const TIOCGETD: ::Ioctl = 0x5424; +pub const TCSBRKP: ::Ioctl = 0x5425; +pub const TIOCSBRK: ::Ioctl = 0x5427; +pub const TIOCCBRK: ::Ioctl = 0x5428; +pub const TIOCGSID: ::Ioctl = 0x5429; +pub const TIOCGRS485: ::Ioctl = 0x542e; +pub const TIOCSRS485: ::Ioctl = 0x542f; +pub const TIOCGPTN: ::Ioctl = 0x40045430; +pub const TIOCSPTLCK: ::Ioctl = 0x80045431; +pub const TIOCGDEV: ::Ioctl = 0x40045432; +pub const TIOCSIG: ::Ioctl = 0x80045436; +pub const TIOCVHANGUP: ::Ioctl = 0x5437; +pub const TIOCGPKT: ::Ioctl = 0x40045438; +pub const TIOCGPTLCK: ::Ioctl = 0x40045439; +pub const TIOCGEXCL: ::Ioctl = 0x40045440; +pub const TIOCGPTPEER: ::Ioctl = 0x20005441; +//pub const TIOCGISO7816: ::Ioctl = 0x40285442; +//pub const TIOCSISO7816: ::Ioctl = 0xc0285443; +pub const FIONCLEX: ::Ioctl = 0x20006602; +pub const FIOCLEX: ::Ioctl = 0x20006601; +pub const FIOASYNC: ::Ioctl = 0x8004667d; +pub const TIOCSERCONFIG: ::Ioctl = 0x5453; +pub const TIOCSERGWILD: ::Ioctl = 0x5454; +pub const TIOCSERSWILD: ::Ioctl = 0x5455; +pub const TIOCGLCKTRMIOS: ::Ioctl = 0x5456; +pub const TIOCSLCKTRMIOS: ::Ioctl = 0x5457; +pub const TIOCSERGSTRUCT: ::Ioctl = 0x5458; +pub const TIOCSERGETLSR: ::Ioctl = 0x5459; +pub const TIOCSERGETMULTI: ::Ioctl = 0x545A; +pub const TIOCSERSETMULTI: ::Ioctl = 0x545B; +pub const TIOCMIWAIT: ::Ioctl = 0x545C; +pub const TIOCGICOUNT: ::Ioctl = 0x545D; +pub const BLKIOMIN: ::Ioctl = 0x20001278; +pub const BLKIOOPT: ::Ioctl = 0x20001279; +pub const BLKSSZGET: ::Ioctl = 0x20001268; +pub const BLKPBSZGET: ::Ioctl = 0x2000127B; +//pub const FIOQSIZE: ::Ioctl = 0x40086680; + +pub const TIOCM_LE: ::c_int = 0x001; +pub const TIOCM_DTR: ::c_int = 0x002; +pub const TIOCM_RTS: ::c_int = 0x004; +pub const TIOCM_ST: ::c_int = 0x008; +pub const TIOCM_SR: ::c_int = 0x010; +pub const TIOCM_CTS: ::c_int = 0x020; +pub const TIOCM_CAR: ::c_int = 0x040; +pub const TIOCM_CD: ::c_int = TIOCM_CAR; +pub const TIOCM_RNG: ::c_int = 0x080; +pub const TIOCM_RI: ::c_int = TIOCM_RNG; +pub const TIOCM_DSR: ::c_int = 0x100; + +pub const BOTHER: ::speed_t = 0o0037; +pub const IBSHIFT: ::tcflag_t = 16; + +// RLIMIT Constants + +cfg_if! { + if #[cfg(target_env = "gnu")] { + + pub const RLIMIT_CPU: ::__rlimit_resource_t = 0; + pub const RLIMIT_FSIZE: ::__rlimit_resource_t = 1; + pub const RLIMIT_DATA: ::__rlimit_resource_t = 2; + pub const RLIMIT_STACK: ::__rlimit_resource_t = 3; + pub const RLIMIT_CORE: ::__rlimit_resource_t = 4; + pub const RLIMIT_RSS: ::__rlimit_resource_t = 5; + pub const RLIMIT_NPROC: ::__rlimit_resource_t = 6; + pub const RLIMIT_NOFILE: ::__rlimit_resource_t = 7; + pub const RLIMIT_MEMLOCK: ::__rlimit_resource_t = 8; + pub const RLIMIT_AS: ::__rlimit_resource_t = 9; + pub const RLIMIT_LOCKS: ::__rlimit_resource_t = 10; + pub const RLIMIT_SIGPENDING: ::__rlimit_resource_t = 11; + pub const RLIMIT_MSGQUEUE: ::__rlimit_resource_t = 12; + pub const RLIMIT_NICE: ::__rlimit_resource_t = 13; + pub const RLIMIT_RTPRIO: ::__rlimit_resource_t = 14; + pub const RLIMIT_RTTIME: ::__rlimit_resource_t = 15; + pub const RLIM_NLIMITS: ::__rlimit_resource_t = 16; + pub const RLIMIT_NLIMITS: ::__rlimit_resource_t = RLIM_NLIMITS; + + } else if #[cfg(target_env = "musl")] { + + pub const RLIMIT_CPU: ::c_int = 0; + pub const RLIMIT_FSIZE: ::c_int = 1; + pub const RLIMIT_DATA: ::c_int = 2; + pub const RLIMIT_STACK: ::c_int = 3; + pub const RLIMIT_CORE: ::c_int = 4; + pub const RLIMIT_RSS: ::c_int = 5; + pub const RLIMIT_NPROC: ::c_int = 6; + pub const RLIMIT_NOFILE: ::c_int = 7; + pub const RLIMIT_MEMLOCK: ::c_int = 8; + pub const RLIMIT_AS: ::c_int = 9; + pub const RLIMIT_LOCKS: ::c_int = 10; + pub const RLIMIT_SIGPENDING: ::c_int = 11; + pub const RLIMIT_MSGQUEUE: ::c_int = 12; + pub const RLIMIT_NICE: ::c_int = 13; + pub const RLIMIT_RTPRIO: ::c_int = 14; + pub const RLIMIT_RTTIME: ::c_int = 15; + pub const RLIM_NLIMITS: ::c_int = 15; + pub const RLIMIT_NLIMITS: ::c_int = RLIM_NLIMITS; + } +} +pub const RLIM_INFINITY: ::rlim_t = !0; diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/arch/sparc/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/arch/sparc/mod.rs new file mode 100644 index 000000000..da3e388e3 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/arch/sparc/mod.rs @@ -0,0 +1,228 @@ +s! { + pub struct termios2 { + pub c_iflag: ::tcflag_t, + pub c_oflag: ::tcflag_t, + pub c_cflag: ::tcflag_t, + pub c_lflag: ::tcflag_t, + pub c_line: ::cc_t, + pub c_cc: [::cc_t; 19], + pub c_ispeed: ::speed_t, + pub c_ospeed: ::speed_t, + } +} + +// arch/sparc/include/uapi/asm/socket.h +pub const SOL_SOCKET: ::c_int = 0xffff; + +// Defined in unix/linux_like/mod.rs +// pub const SO_DEBUG: ::c_int = 0x0001; +pub const SO_PASSCRED: ::c_int = 0x0002; +pub const SO_REUSEADDR: ::c_int = 0x0004; +pub const SO_KEEPALIVE: ::c_int = 0x0008; +pub const SO_DONTROUTE: ::c_int = 0x0010; +pub const SO_BROADCAST: ::c_int = 0x0020; +pub const SO_PEERCRED: ::c_int = 0x0040; +pub const SO_LINGER: ::c_int = 0x0080; +pub const SO_OOBINLINE: ::c_int = 0x0100; +pub const SO_REUSEPORT: ::c_int = 0x0200; +pub const SO_BSDCOMPAT: ::c_int = 0x0400; +pub const SO_RCVLOWAT: ::c_int = 0x0800; +pub const SO_SNDLOWAT: ::c_int = 0x1000; +pub const SO_RCVTIMEO: ::c_int = 0x2000; +pub const SO_SNDTIMEO: ::c_int = 0x4000; +// pub const SO_RCVTIMEO_OLD: ::c_int = 0x2000; +// pub const SO_SNDTIMEO_OLD: ::c_int = 0x4000; +pub const SO_ACCEPTCONN: ::c_int = 0x8000; +pub const SO_SNDBUF: ::c_int = 0x1001; +pub const SO_RCVBUF: ::c_int = 0x1002; +pub const SO_SNDBUFFORCE: ::c_int = 0x100a; +pub const SO_RCVBUFFORCE: ::c_int = 0x100b; +pub const SO_ERROR: ::c_int = 0x1007; +pub const SO_TYPE: ::c_int = 0x1008; +pub const SO_PROTOCOL: ::c_int = 0x1028; +pub const SO_DOMAIN: ::c_int = 0x1029; +pub const SO_NO_CHECK: ::c_int = 0x000b; +pub const SO_PRIORITY: ::c_int = 0x000c; +pub const SO_BINDTODEVICE: ::c_int = 0x000d; +pub const SO_ATTACH_FILTER: ::c_int = 0x001a; +pub const SO_DETACH_FILTER: ::c_int = 0x001b; +pub const SO_GET_FILTER: ::c_int = SO_ATTACH_FILTER; +pub const SO_PEERNAME: ::c_int = 0x001c; +pub const SO_PEERSEC: ::c_int = 0x001e; +pub const SO_PASSSEC: ::c_int = 0x001f; +pub const SO_MARK: ::c_int = 0x0022; +pub const SO_RXQ_OVFL: ::c_int = 0x0024; +pub const SO_WIFI_STATUS: ::c_int = 0x0025; +pub const SCM_WIFI_STATUS: ::c_int = SO_WIFI_STATUS; +pub const SO_PEEK_OFF: ::c_int = 0x0026; +pub const SO_NOFCS: ::c_int = 0x0027; +pub const SO_LOCK_FILTER: ::c_int = 0x0028; +pub const SO_SELECT_ERR_QUEUE: ::c_int = 0x0029; +pub const SO_BUSY_POLL: ::c_int = 0x0030; +pub const SO_MAX_PACING_RATE: ::c_int = 0x0031; +pub const SO_BPF_EXTENSIONS: ::c_int = 0x0032; +pub const SO_INCOMING_CPU: ::c_int = 0x0033; +pub const SO_ATTACH_BPF: ::c_int = 0x0034; +pub const SO_DETACH_BPF: ::c_int = SO_DETACH_FILTER; +pub const SO_ATTACH_REUSEPORT_CBPF: ::c_int = 0x0035; +pub const SO_ATTACH_REUSEPORT_EBPF: ::c_int = 0x0036; +pub const SO_CNX_ADVICE: ::c_int = 0x0037; +pub const SCM_TIMESTAMPING_OPT_STATS: ::c_int = 0x0038; +pub const SO_MEMINFO: ::c_int = 0x0039; +pub const SO_INCOMING_NAPI_ID: ::c_int = 0x003a; +pub const SO_COOKIE: ::c_int = 0x003b; +pub const SCM_TIMESTAMPING_PKTINFO: ::c_int = 0x003c; +pub const SO_PEERGROUPS: ::c_int = 0x003d; +pub const SO_ZEROCOPY: ::c_int = 0x003e; +pub const SO_TXTIME: ::c_int = 0x003f; +pub const SCM_TXTIME: ::c_int = SO_TXTIME; +pub const SO_BINDTOIFINDEX: ::c_int = 0x0041; +pub const SO_SECURITY_AUTHENTICATION: ::c_int = 0x5001; +pub const SO_SECURITY_ENCRYPTION_TRANSPORT: ::c_int = 0x5002; +pub const SO_SECURITY_ENCRYPTION_NETWORK: ::c_int = 0x5004; +pub const SO_TIMESTAMP: ::c_int = 0x001d; +pub const SO_TIMESTAMPNS: ::c_int = 0x0021; +pub const SO_TIMESTAMPING: ::c_int = 0x0023; +// pub const SO_TIMESTAMP_OLD: ::c_int = 0x001d; +// pub const SO_TIMESTAMPNS_OLD: ::c_int = 0x0021; +// pub const SO_TIMESTAMPING_OLD: ::c_int = 0x0023; +// pub const SO_TIMESTAMP_NEW: ::c_int = 0x0046; +// pub const SO_TIMESTAMPNS_NEW: ::c_int = 0x0042; +// pub const SO_TIMESTAMPING_NEW: ::c_int = 0x0043; +// pub const SO_RCVTIMEO_NEW: ::c_int = 0x0044; +// pub const SO_SNDTIMEO_NEW: ::c_int = 0x0045; +// pub const SO_DETACH_REUSEPORT_BPF: ::c_int = 0x0047; +// pub const SO_PREFER_BUSY_POLL: ::c_int = 0x0048; +// pub const SO_BUSY_POLL_BUDGET: ::c_int = 0x0049; + +// Defined in unix/linux_like/mod.rs +// pub const SCM_TIMESTAMP: ::c_int = SO_TIMESTAMP; +pub const SCM_TIMESTAMPNS: ::c_int = SO_TIMESTAMPNS; +pub const SCM_TIMESTAMPING: ::c_int = SO_TIMESTAMPING; + +// Ioctl Constants + +pub const TCGETS: ::Ioctl = 0x40245408; +pub const TCSETS: ::Ioctl = 0x80245409; +pub const TCSETSW: ::Ioctl = 0x8024540a; +pub const TCSETSF: ::Ioctl = 0x8024540b; +pub const TCGETA: ::Ioctl = 0x40125401; +pub const TCSETA: ::Ioctl = 0x80125402; +pub const TCSETAW: ::Ioctl = 0x80125403; +pub const TCSETAF: ::Ioctl = 0x80125404; +pub const TCSBRK: ::Ioctl = 0x20005405; +pub const TCXONC: ::Ioctl = 0x20005406; +pub const TCFLSH: ::Ioctl = 0x20005407; +pub const TIOCEXCL: ::Ioctl = 0x2000740d; +pub const TIOCNXCL: ::Ioctl = 0x2000740e; +pub const TIOCSCTTY: ::Ioctl = 0x20007484; +pub const TIOCGPGRP: ::Ioctl = 0x40047483; +pub const TIOCSPGRP: ::Ioctl = 0x80047482; +pub const TIOCOUTQ: ::Ioctl = 0x40047473; +pub const TIOCSTI: ::Ioctl = 0x80017472; +pub const TIOCGWINSZ: ::Ioctl = 0x40087468; +pub const TIOCSWINSZ: ::Ioctl = 0x80087467; +pub const TIOCMGET: ::Ioctl = 0x4004746a; +pub const TIOCMBIS: ::Ioctl = 0x8004746c; +pub const TIOCMBIC: ::Ioctl = 0x8004746b; +pub const TIOCMSET: ::Ioctl = 0x8004746d; +pub const TIOCGSOFTCAR: ::Ioctl = 0x40047464; +pub const TIOCSSOFTCAR: ::Ioctl = 0x80047465; +pub const FIONREAD: ::Ioctl = 0x4004667f; +pub const TIOCINQ: ::Ioctl = FIONREAD; +pub const TIOCLINUX: ::Ioctl = 0x541C; +pub const TIOCCONS: ::Ioctl = 0x20007424; +pub const TIOCGSERIAL: ::Ioctl = 0x541E; +pub const TIOCSSERIAL: ::Ioctl = 0x541F; +pub const TIOCPKT: ::Ioctl = 0x80047470; +pub const FIONBIO: ::Ioctl = 0x8004667e; +pub const TIOCNOTTY: ::Ioctl = 0x20007471; +pub const TIOCSETD: ::Ioctl = 0x80047401; +pub const TIOCGETD: ::Ioctl = 0x40047400; +pub const TCSBRKP: ::Ioctl = 0x5425; +pub const TIOCSBRK: ::Ioctl = 0x2000747b; +pub const TIOCCBRK: ::Ioctl = 0x2000747a; +pub const TIOCGSID: ::Ioctl = 0x40047485; +pub const TCGETS2: ::Ioctl = 0x402c540c; +pub const TCSETS2: ::Ioctl = 0x802c540d; +pub const TCSETSW2: ::Ioctl = 0x802c540e; +pub const TCSETSF2: ::Ioctl = 0x802c540f; +pub const TIOCGPTN: ::Ioctl = 0x40047486; +pub const TIOCSPTLCK: ::Ioctl = 0x80047487; +pub const TIOCGDEV: ::Ioctl = 0x40045432; +pub const TIOCSIG: ::Ioctl = 0x80047488; +pub const TIOCVHANGUP: ::Ioctl = 0x20005437; +pub const TIOCGPKT: ::Ioctl = 0x40045438; +pub const TIOCGPTLCK: ::Ioctl = 0x40045439; +pub const TIOCGEXCL: ::Ioctl = 0x40045440; +pub const TIOCGPTPEER: ::Ioctl = 0x20007489; +pub const FIONCLEX: ::Ioctl = 0x20006602; +pub const FIOCLEX: ::Ioctl = 0x20006601; +pub const TIOCSERCONFIG: ::Ioctl = 0x5453; +pub const TIOCSERGWILD: ::Ioctl = 0x5454; +pub const TIOCSERSWILD: ::Ioctl = 0x5455; +pub const TIOCGLCKTRMIOS: ::Ioctl = 0x5456; +pub const TIOCSLCKTRMIOS: ::Ioctl = 0x5457; +pub const TIOCSERGSTRUCT: ::Ioctl = 0x5458; +pub const TIOCSERGETLSR: ::Ioctl = 0x5459; +pub const TIOCSERGETMULTI: ::Ioctl = 0x545A; +pub const TIOCSERSETMULTI: ::Ioctl = 0x545B; +pub const TIOCMIWAIT: ::Ioctl = 0x545C; +pub const TIOCGICOUNT: ::Ioctl = 0x545D; +pub const TIOCSTART: ::Ioctl = 0x2000746e; +pub const TIOCSTOP: ::Ioctl = 0x2000746f; +pub const BLKIOMIN: ::Ioctl = 0x20001278; +pub const BLKIOOPT: ::Ioctl = 0x20001279; +pub const BLKSSZGET: ::Ioctl = 0x20001268; +pub const BLKPBSZGET: ::Ioctl = 0x2000127B; + +//pub const FIOASYNC: ::Ioctl = 0x4004667d; +//pub const FIOQSIZE: ::Ioctl = ; +//pub const TIOCGISO7816: ::Ioctl = 0x40285443; +//pub const TIOCSISO7816: ::Ioctl = 0xc0285444; +//pub const TIOCGRS485: ::Ioctl = 0x40205441; +//pub const TIOCSRS485: ::Ioctl = 0xc0205442; + +pub const TIOCM_LE: ::c_int = 0x001; +pub const TIOCM_DTR: ::c_int = 0x002; +pub const TIOCM_RTS: ::c_int = 0x004; +pub const TIOCM_ST: ::c_int = 0x008; +pub const TIOCM_SR: ::c_int = 0x010; +pub const TIOCM_CTS: ::c_int = 0x020; +pub const TIOCM_CAR: ::c_int = 0x040; +pub const TIOCM_CD: ::c_int = TIOCM_CAR; +pub const TIOCM_RNG: ::c_int = 0x080; +pub const TIOCM_RI: ::c_int = TIOCM_RNG; +pub const TIOCM_DSR: ::c_int = 0x100; + +pub const BOTHER: ::speed_t = 0x1000; +pub const IBSHIFT: ::tcflag_t = 16; + +// RLIMIT Constants + +pub const RLIMIT_CPU: ::__rlimit_resource_t = 0; +pub const RLIMIT_FSIZE: ::__rlimit_resource_t = 1; +pub const RLIMIT_DATA: ::__rlimit_resource_t = 2; +pub const RLIMIT_STACK: ::__rlimit_resource_t = 3; +pub const RLIMIT_CORE: ::__rlimit_resource_t = 4; +pub const RLIMIT_RSS: ::__rlimit_resource_t = 5; +pub const RLIMIT_NOFILE: ::__rlimit_resource_t = 6; +pub const RLIMIT_NPROC: ::__rlimit_resource_t = 7; +pub const RLIMIT_MEMLOCK: ::__rlimit_resource_t = 8; +pub const RLIMIT_AS: ::__rlimit_resource_t = 9; +pub const RLIMIT_LOCKS: ::__rlimit_resource_t = 10; +pub const RLIMIT_SIGPENDING: ::__rlimit_resource_t = 11; +pub const RLIMIT_MSGQUEUE: ::__rlimit_resource_t = 12; +pub const RLIMIT_NICE: ::__rlimit_resource_t = 13; +pub const RLIMIT_RTPRIO: ::__rlimit_resource_t = 14; +pub const RLIMIT_RTTIME: ::__rlimit_resource_t = 15; +pub const RLIM_NLIMITS: ::__rlimit_resource_t = 16; +pub const RLIMIT_NLIMITS: ::__rlimit_resource_t = RLIM_NLIMITS; + +cfg_if! { + if #[cfg(target_arch = "sparc64")] { + pub const RLIM_INFINITY: ::rlim_t = !0; + } else if #[cfg(target_arch = "sparc")] { + pub const RLIM_INFINITY: ::rlim_t = 0x7fffffff; + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/arm/align.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/arm/align.rs index 825546be9..2645ec4c3 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/arm/align.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/arm/align.rs @@ -4,4 +4,50 @@ s_no_extra_traits! { pub struct max_align_t { priv_: [i64; 2] } + + #[allow(missing_debug_implementations)] + #[repr(align(8))] + pub struct ucontext_t { + pub uc_flags: ::c_ulong, + pub uc_link: *mut ucontext_t, + pub uc_stack: ::stack_t, + pub uc_mcontext: ::mcontext_t, + pub uc_sigmask: ::sigset_t, + pub uc_regspace: [::c_ulong; 128], + } +} + +cfg_if! { + if #[cfg(feature = "extra_traits")] { + impl PartialEq for ucontext_t { + fn eq(&self, other: &ucontext_t) -> bool { + self.uc_flags == other.uc_flags + && self.uc_link == other.uc_link + && self.uc_stack == other.uc_stack + && self.uc_mcontext == other.uc_mcontext + && self.uc_sigmask == other.uc_sigmask + } + } + impl Eq for ucontext_t {} + impl ::fmt::Debug for ucontext_t { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("ucontext_t") + .field("uc_flags", &self.uc_link) + .field("uc_link", &self.uc_link) + .field("uc_stack", &self.uc_stack) + .field("uc_mcontext", &self.uc_mcontext) + .field("uc_sigmask", &self.uc_sigmask) + .finish() + } + } + impl ::hash::Hash for ucontext_t { + fn hash(&self, state: &mut H) { + self.uc_flags.hash(state); + self.uc_link.hash(state); + self.uc_stack.hash(state); + self.uc_mcontext.hash(state); + self.uc_sigmask.hash(state); + } + } + } } diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/arm/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/arm/mod.rs index 22746ee94..e0ac0dfc3 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/arm/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/arm/mod.rs @@ -141,17 +141,6 @@ s! { __glibc_reserved5: ::c_ulong, } - pub struct termios2 { - pub c_iflag: ::tcflag_t, - pub c_oflag: ::tcflag_t, - pub c_cflag: ::tcflag_t, - pub c_lflag: ::tcflag_t, - pub c_line: ::cc_t, - pub c_cc: [::cc_t; 19], - pub c_ispeed: ::speed_t, - pub c_ospeed: ::speed_t, - } - pub struct siginfo_t { pub si_signo: ::c_int, pub si_errno: ::c_int, @@ -172,9 +161,38 @@ s! { pub ss_flags: ::c_int, pub ss_size: ::size_t } + + pub struct seccomp_notif_sizes { + pub seccomp_notif: ::__u16, + pub seccomp_notif_resp: ::__u16, + pub seccomp_data: ::__u16, + } + + pub struct mcontext_t { + pub trap_no: ::c_ulong, + pub error_code: ::c_ulong, + pub oldmask: ::c_ulong, + pub arm_r0: ::c_ulong, + pub arm_r1: ::c_ulong, + pub arm_r2: ::c_ulong, + pub arm_r3: ::c_ulong, + pub arm_r4: ::c_ulong, + pub arm_r5: ::c_ulong, + pub arm_r6: ::c_ulong, + pub arm_r7: ::c_ulong, + pub arm_r8: ::c_ulong, + pub arm_r9: ::c_ulong, + pub arm_r10: ::c_ulong, + pub arm_fp: ::c_ulong, + pub arm_ip: ::c_ulong, + pub arm_sp: ::c_ulong, + pub arm_lr: ::c_ulong, + pub arm_pc: ::c_ulong, + pub arm_cpsr: ::c_ulong, + pub fault_address: ::c_ulong, + } } -pub const RLIM_INFINITY: ::rlim_t = !0; pub const VEOF: usize = 4; pub const RTLD_DEEPBIND: ::c_int = 0x8; pub const RTLD_GLOBAL: ::c_int = 0x100; @@ -194,11 +212,6 @@ pub const O_DSYNC: ::c_int = 4096; pub const O_FSYNC: ::c_int = 0x101000; pub const O_ASYNC: ::c_int = 0x2000; pub const O_NDELAY: ::c_int = 0x800; -pub const RLIMIT_NOFILE: ::__rlimit_resource_t = 7; -pub const RLIMIT_NPROC: ::__rlimit_resource_t = 6; -pub const RLIMIT_RSS: ::__rlimit_resource_t = 5; -pub const RLIMIT_AS: ::__rlimit_resource_t = 9; -pub const RLIMIT_MEMLOCK: ::__rlimit_resource_t = 8; pub const MADV_SOFT_OFFLINE: ::c_int = 101; pub const MAP_LOCKED: ::c_int = 0x02000; @@ -212,9 +225,7 @@ pub const MAP_NONBLOCK: ::c_int = 0x010000; pub const MAP_STACK: ::c_int = 0x020000; pub const MAP_HUGETLB: ::c_int = 0x040000; pub const MAP_GROWSDOWN: ::c_int = 0x0100; -pub const MAP_SYNC : ::c_int = 0x080000; - -pub const SOL_SOCKET: ::c_int = 1; +pub const MAP_SYNC: ::c_int = 0x080000; pub const EDEADLOCK: ::c_int = 35; pub const EUCLEAN: ::c_int = 117; @@ -301,39 +312,12 @@ pub const ENOTRECOVERABLE: ::c_int = 131; pub const EHWPOISON: ::c_int = 133; pub const ERFKILL: ::c_int = 132; -pub const SO_REUSEADDR: ::c_int = 2; -pub const SO_TYPE: ::c_int = 3; -pub const SO_ERROR: ::c_int = 4; -pub const SO_DONTROUTE: ::c_int = 5; -pub const SO_BROADCAST: ::c_int = 6; -pub const SO_SNDBUF: ::c_int = 7; -pub const SO_RCVBUF: ::c_int = 8; -pub const SO_KEEPALIVE: ::c_int = 9; -pub const SO_OOBINLINE: ::c_int = 10; -pub const SO_LINGER: ::c_int = 13; -pub const SO_REUSEPORT: ::c_int = 15; -pub const SO_ACCEPTCONN: ::c_int = 30; -pub const SO_PROTOCOL: ::c_int = 38; -pub const SO_DOMAIN: ::c_int = 39; -pub const SO_PASSCRED: ::c_int = 16; -pub const SO_PEERCRED: ::c_int = 17; -pub const SO_RCVLOWAT: ::c_int = 18; -pub const SO_SNDLOWAT: ::c_int = 19; -pub const SO_RCVTIMEO: ::c_int = 20; -pub const SO_SNDTIMEO: ::c_int = 21; -pub const SO_SNDBUFFORCE: ::c_int = 32; -pub const SO_RCVBUFFORCE: ::c_int = 33; - pub const SA_SIGINFO: ::c_int = 0x00000004; pub const SA_NOCLDWAIT: ::c_int = 0x00000002; pub const SOCK_STREAM: ::c_int = 1; pub const SOCK_DGRAM: ::c_int = 2; -pub const FIOCLEX: ::c_ulong = 0x5451; -pub const FIONCLEX: ::c_ulong = 0x5450; -pub const FIONBIO: ::c_ulong = 0x5421; - pub const MCL_CURRENT: ::c_int = 0x0001; pub const MCL_FUTURE: ::c_int = 0x0002; @@ -444,7 +428,6 @@ pub const B19200: ::speed_t = 0o000016; pub const B38400: ::speed_t = 0o000017; pub const EXTA: ::speed_t = B19200; pub const EXTB: ::speed_t = B38400; -pub const BOTHER: ::speed_t = 0o010000; pub const B57600: ::speed_t = 0o010001; pub const B115200: ::speed_t = 0o010002; pub const B230400: ::speed_t = 0o010003; @@ -461,6 +444,11 @@ pub const B3000000: ::speed_t = 0o010015; pub const B3500000: ::speed_t = 0o010016; pub const B4000000: ::speed_t = 0o010017; +pub const SECCOMP_SET_MODE_STRICT: ::c_uint = 0; +pub const SECCOMP_SET_MODE_FILTER: ::c_uint = 1; +pub const SECCOMP_GET_ACTION_AVAIL: ::c_uint = 2; +pub const SECCOMP_GET_NOTIF_SIZES: ::c_uint = 3; + pub const VEOL: usize = 11; pub const VEOL2: usize = 16; pub const VMIN: usize = 6; @@ -468,52 +456,11 @@ pub const IEXTEN: ::tcflag_t = 0x00008000; pub const TOSTOP: ::tcflag_t = 0x00000100; pub const FLUSHO: ::tcflag_t = 0x00001000; pub const EXTPROC: ::tcflag_t = 0x00010000; -pub const TCGETS: ::c_ulong = 0x5401; -pub const TCSETS: ::c_ulong = 0x5402; -pub const TCSETSW: ::c_ulong = 0x5403; -pub const TCSETSF: ::c_ulong = 0x5404; -pub const TCGETA: ::c_ulong = 0x5405; -pub const TCSETA: ::c_ulong = 0x5406; -pub const TCSETAW: ::c_ulong = 0x5407; -pub const TCSETAF: ::c_ulong = 0x5408; -pub const TCSBRK: ::c_ulong = 0x5409; -pub const TCXONC: ::c_ulong = 0x540A; -pub const TCFLSH: ::c_ulong = 0x540B; -pub const TIOCINQ: ::c_ulong = 0x541B; -pub const TIOCGPGRP: ::c_ulong = 0x540F; -pub const TIOCSPGRP: ::c_ulong = 0x5410; -pub const TIOCOUTQ: ::c_ulong = 0x5411; -pub const TIOCGWINSZ: ::c_ulong = 0x5413; -pub const TIOCSWINSZ: ::c_ulong = 0x5414; -pub const TIOCGRS485: ::c_int = 0x542E; -pub const TIOCSRS485: ::c_int = 0x542F; -pub const FIONREAD: ::c_ulong = 0x541B; - -pub const TIOCGSOFTCAR: ::c_ulong = 0x5419; -pub const TIOCSSOFTCAR: ::c_ulong = 0x541A; -pub const TIOCEXCL: ::c_ulong = 0x540C; -pub const TIOCNXCL: ::c_ulong = 0x540D; -pub const TIOCSCTTY: ::c_ulong = 0x540E; -pub const TIOCSTI: ::c_ulong = 0x5412; -pub const TIOCMGET: ::c_ulong = 0x5415; -pub const TIOCMBIS: ::c_ulong = 0x5416; -pub const TIOCMBIC: ::c_ulong = 0x5417; -pub const TIOCMSET: ::c_ulong = 0x5418; -pub const TIOCCONS: ::c_ulong = 0x541D; pub const TCSANOW: ::c_int = 0; pub const TCSADRAIN: ::c_int = 1; pub const TCSAFLUSH: ::c_int = 2; -pub const TIOCLINUX: ::c_ulong = 0x541C; -pub const TIOCGSERIAL: ::c_ulong = 0x541E; -pub const TIOCM_ST: ::c_int = 0x008; -pub const TIOCM_SR: ::c_int = 0x010; -pub const TIOCM_CTS: ::c_int = 0x020; -pub const TIOCM_CAR: ::c_int = 0x040; -pub const TIOCM_RNG: ::c_int = 0x080; -pub const TIOCM_DSR: ::c_int = 0x100; - // Syscall table pub const SYS_restart_syscall: ::c_long = 0; pub const SYS_exit: ::c_long = 1; @@ -868,6 +815,34 @@ pub const SYS_pkey_mprotect: ::c_long = 394; pub const SYS_pkey_alloc: ::c_long = 395; pub const SYS_pkey_free: ::c_long = 396; pub const SYS_statx: ::c_long = 397; +pub const SYS_rseq: ::c_long = 398; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; +pub const SYS_quotactl_fd: ::c_long = 443; +pub const SYS_landlock_create_ruleset: ::c_long = 444; +pub const SYS_landlock_add_rule: ::c_long = 445; +pub const SYS_landlock_restrict_self: ::c_long = 446; +pub const SYS_memfd_secret: ::c_long = 447; +pub const SYS_process_mrelease: ::c_long = 448; +pub const SYS_futex_waitv: ::c_long = 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 450; cfg_if! { if #[cfg(libc_align)] { diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/m68k/align.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/m68k/align.rs new file mode 100644 index 000000000..639394a30 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/m68k/align.rs @@ -0,0 +1,7 @@ +s_no_extra_traits! { + #[allow(missing_debug_implementations)] + #[repr(align(2))] + pub struct max_align_t { + priv_: [i8; 20] + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/m68k/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/m68k/mod.rs new file mode 100644 index 000000000..3b78f181c --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/m68k/mod.rs @@ -0,0 +1,849 @@ +pub type c_char = i8; +pub type wchar_t = i32; + +s! { + pub struct sigaction { + pub sa_sigaction: ::sighandler_t, + pub sa_mask: ::sigset_t, + pub sa_flags: ::c_ulong, + pub sa_restorer: ::Option, + } + + pub struct statfs { + pub f_type: ::__fsword_t, + pub f_bsize: ::__fsword_t, + pub f_blocks: ::fsblkcnt_t, + pub f_bfree: ::fsblkcnt_t, + pub f_bavail: ::fsblkcnt_t, + + pub f_files: ::fsfilcnt_t, + pub f_ffree: ::fsfilcnt_t, + pub f_fsid: ::fsid_t, + + pub f_namelen: ::__fsword_t, + pub f_frsize: ::__fsword_t, + pub f_flags: ::__fsword_t, + f_spare: [::__fsword_t; 4], + } + + pub struct flock { + pub l_type: ::c_short, + pub l_whence: ::c_short, + pub l_start: ::off_t, + pub l_len: ::off_t, + pub l_pid: ::pid_t, + } + + pub struct flock64 { + pub l_type: ::c_short, + pub l_whence: ::c_short, + pub l_start: ::off64_t, + pub l_len: ::off64_t, + pub l_pid: ::pid_t, + } + + pub struct ipc_perm { + __key: ::key_t, + pub uid: ::uid_t, + pub gid: ::gid_t, + pub cuid: ::uid_t, + pub cgid: ::gid_t, + pub mode: ::mode_t, + __seq: ::c_ushort, + __pad1: ::c_ushort, + __glibc_reserved1: ::c_ulong, + __glibc_reserved2: ::c_ulong, + } + + pub struct stat64 { + pub st_dev: ::dev_t, + __pad1: ::c_ushort, + pub __st_ino: ::ino_t, + pub st_mode: ::mode_t, + pub st_nlink: ::nlink_t, + pub st_uid: ::uid_t, + pub st_gid: ::gid_t, + pub st_rdev: ::dev_t, + __pad2: ::c_ushort, + pub st_size: ::off64_t, + pub st_blksize: ::blksize_t, + pub st_blocks: ::blkcnt64_t, + pub st_atime: ::time_t, + pub st_atime_nsec: ::c_ulong, + pub st_mtime: ::time_t, + pub st_mtime_nsec: ::c_ulong, + pub st_ctime: ::time_t, + pub st_ctime_nsec: ::c_ulong, + pub st_ino: ::ino64_t, + } + + pub struct statfs64 { + pub f_type: ::__fsword_t, + pub f_bsize: ::__fsword_t, + pub f_blocks: ::fsblkcnt64_t, + pub f_bfree: ::fsblkcnt64_t, + pub f_bavail: ::fsblkcnt64_t, + pub f_files: ::fsblkcnt64_t, + pub f_ffree: ::fsblkcnt64_t, + pub f_fsid: ::fsid_t, + pub f_namelen: ::__fsword_t, + pub f_frsize: ::__fsword_t, + pub f_flags: ::__fsword_t, + pub f_spare: [::__fsword_t; 4], + } + + pub struct statvfs64 { + pub f_bsize: ::c_ulong, + pub f_frsize: ::c_ulong, + pub f_blocks: ::fsblkcnt64_t, + pub f_bfree: ::fsblkcnt64_t, + pub f_bavail: ::fsblkcnt64_t, + pub f_files: ::fsblkcnt64_t, + pub f_ffree: ::fsblkcnt64_t, + pub f_favail: ::fsblkcnt64_t, + pub f_fsid: ::c_ulong, + __f_unused: ::c_int, + pub f_flag: ::c_ulong, + pub f_namemax: ::c_ulong, + __f_spare: [::c_int; 6], + } + + pub struct shmid_ds { + pub shm_perm: ::ipc_perm, + pub shm_segsz: ::size_t, + pub shm_atime: ::time_t, + __glibc_reserved1: ::c_long, + pub shm_dtime: ::time_t, + __glibc_reserved2: ::c_long, + pub shm_ctime: ::time_t, + __glibc_reserved3: ::c_long, + pub shm_cpid: ::pid_t, + pub shm_lpid: ::pid_t, + pub shm_nattch: ::shmatt_t, + __glibc_reserved5: ::c_ulong, + __glibc_reserved6: ::c_ulong, + } + + pub struct msqid_ds { + pub msg_perm: ::ipc_perm, + pub msg_stime: ::time_t, + __glibc_reserved1: ::c_uint, + pub msg_rtime: ::time_t, + __glibc_reserved2: ::c_uint, + pub msg_ctime: ::time_t, + __glibc_reserved3: ::c_uint, + __msg_cbytes: ::c_ulong, + pub msg_qnum: ::msgqnum_t, + pub msg_qbytes: ::msglen_t, + pub msg_lspid: ::pid_t, + pub msg_lrpid: ::pid_t, + __glibc_reserved4: ::c_ulong, + __glibc_reserved5: ::c_ulong, + } + + pub struct siginfo_t { + pub si_signo: ::c_int, + pub si_code: ::c_int, + pub si_errno: ::c_int, + _pad: [::c_int; 29], + _align: [usize; 0], + } + + pub struct stack_t { + pub ss_sp: *mut ::c_void, + pub ss_flags: ::c_int, + pub ss_size: ::size_t + } +} + +pub const VEOF: usize = 4; +pub const RTLD_DEEPBIND: ::c_int = 0x8; +pub const RTLD_GLOBAL: ::c_int = 0x100; +pub const RTLD_NOLOAD: ::c_int = 0x4; +pub const O_DIRECT: ::c_int = 0x10000; +pub const O_DIRECTORY: ::c_int = 0x4000; +pub const O_NOFOLLOW: ::c_int = 0x8000; +pub const O_LARGEFILE: ::c_int = 0x20000; +pub const O_APPEND: ::c_int = 1024; +pub const O_CREAT: ::c_int = 64; +pub const O_EXCL: ::c_int = 128; +pub const O_NOCTTY: ::c_int = 256; +pub const O_NONBLOCK: ::c_int = 2048; +pub const O_SYNC: ::c_int = 1052672; +pub const O_RSYNC: ::c_int = 1052672; +pub const O_DSYNC: ::c_int = 4096; +pub const O_FSYNC: ::c_int = 0x101000; +pub const O_ASYNC: ::c_int = 0x2000; +pub const O_NDELAY: ::c_int = 0x800; + +pub const MADV_SOFT_OFFLINE: ::c_int = 101; +pub const MAP_LOCKED: ::c_int = 0x02000; +pub const MAP_NORESERVE: ::c_int = 0x04000; +pub const MAP_32BIT: ::c_int = 0x0040; +pub const MAP_ANON: ::c_int = 0x0020; +pub const MAP_ANONYMOUS: ::c_int = 0x0020; +pub const MAP_DENYWRITE: ::c_int = 0x0800; +pub const MAP_EXECUTABLE: ::c_int = 0x01000; +pub const MAP_POPULATE: ::c_int = 0x08000; +pub const MAP_NONBLOCK: ::c_int = 0x010000; +pub const MAP_STACK: ::c_int = 0x020000; +pub const MAP_HUGETLB: ::c_int = 0x040000; +pub const MAP_GROWSDOWN: ::c_int = 0x0100; +pub const MAP_SYNC: ::c_int = 0x080000; + +pub const EDEADLOCK: ::c_int = 35; +pub const EUCLEAN: ::c_int = 117; +pub const ENOTNAM: ::c_int = 118; +pub const ENAVAIL: ::c_int = 119; +pub const EISNAM: ::c_int = 120; +pub const EREMOTEIO: ::c_int = 121; +pub const EDEADLK: ::c_int = 35; +pub const ENAMETOOLONG: ::c_int = 36; +pub const ENOLCK: ::c_int = 37; +pub const ENOSYS: ::c_int = 38; +pub const ENOTEMPTY: ::c_int = 39; +pub const ELOOP: ::c_int = 40; +pub const ENOMSG: ::c_int = 42; +pub const EIDRM: ::c_int = 43; +pub const ECHRNG: ::c_int = 44; +pub const EL2NSYNC: ::c_int = 45; +pub const EL3HLT: ::c_int = 46; +pub const EL3RST: ::c_int = 47; +pub const ELNRNG: ::c_int = 48; +pub const EUNATCH: ::c_int = 49; +pub const ENOCSI: ::c_int = 50; +pub const EL2HLT: ::c_int = 51; +pub const EBADE: ::c_int = 52; +pub const EBADR: ::c_int = 53; +pub const EXFULL: ::c_int = 54; +pub const ENOANO: ::c_int = 55; +pub const EBADRQC: ::c_int = 56; +pub const EBADSLT: ::c_int = 57; +pub const EMULTIHOP: ::c_int = 72; +pub const EOVERFLOW: ::c_int = 75; +pub const ENOTUNIQ: ::c_int = 76; +pub const EBADFD: ::c_int = 77; +pub const EBADMSG: ::c_int = 74; +pub const EREMCHG: ::c_int = 78; +pub const ELIBACC: ::c_int = 79; +pub const ELIBBAD: ::c_int = 80; +pub const ELIBSCN: ::c_int = 81; +pub const ELIBMAX: ::c_int = 82; +pub const ELIBEXEC: ::c_int = 83; +pub const EILSEQ: ::c_int = 84; +pub const ERESTART: ::c_int = 85; +pub const ESTRPIPE: ::c_int = 86; +pub const EUSERS: ::c_int = 87; +pub const ENOTSOCK: ::c_int = 88; +pub const EDESTADDRREQ: ::c_int = 89; +pub const EMSGSIZE: ::c_int = 90; +pub const EPROTOTYPE: ::c_int = 91; +pub const ENOPROTOOPT: ::c_int = 92; +pub const EPROTONOSUPPORT: ::c_int = 93; +pub const ESOCKTNOSUPPORT: ::c_int = 94; +pub const EOPNOTSUPP: ::c_int = 95; +pub const EPFNOSUPPORT: ::c_int = 96; +pub const EAFNOSUPPORT: ::c_int = 97; +pub const EADDRINUSE: ::c_int = 98; +pub const EADDRNOTAVAIL: ::c_int = 99; +pub const ENETDOWN: ::c_int = 100; +pub const ENETUNREACH: ::c_int = 101; +pub const ENETRESET: ::c_int = 102; +pub const ECONNABORTED: ::c_int = 103; +pub const ECONNRESET: ::c_int = 104; +pub const ENOBUFS: ::c_int = 105; +pub const EISCONN: ::c_int = 106; +pub const ENOTCONN: ::c_int = 107; +pub const ESHUTDOWN: ::c_int = 108; +pub const ETOOMANYREFS: ::c_int = 109; +pub const ETIMEDOUT: ::c_int = 110; +pub const ECONNREFUSED: ::c_int = 111; +pub const EHOSTDOWN: ::c_int = 112; +pub const EHOSTUNREACH: ::c_int = 113; +pub const EALREADY: ::c_int = 114; +pub const EINPROGRESS: ::c_int = 115; +pub const ESTALE: ::c_int = 116; +pub const EDQUOT: ::c_int = 122; +pub const ENOMEDIUM: ::c_int = 123; +pub const EMEDIUMTYPE: ::c_int = 124; +pub const ECANCELED: ::c_int = 125; +pub const ENOKEY: ::c_int = 126; +pub const EKEYEXPIRED: ::c_int = 127; +pub const EKEYREVOKED: ::c_int = 128; +pub const EKEYREJECTED: ::c_int = 129; +pub const EOWNERDEAD: ::c_int = 130; +pub const ENOTRECOVERABLE: ::c_int = 131; +pub const EHWPOISON: ::c_int = 133; +pub const ERFKILL: ::c_int = 132; + +pub const SA_SIGINFO: ::c_int = 0x00000004; +pub const SA_NOCLDWAIT: ::c_int = 0x00000002; + +pub const SOCK_STREAM: ::c_int = 1; +pub const SOCK_DGRAM: ::c_int = 2; + +pub const F_GETLK: ::c_int = 5; +pub const F_GETOWN: ::c_int = 9; +pub const F_SETOWN: ::c_int = 8; + +pub const PTRACE_GETFPXREGS: ::c_uint = 18; +pub const PTRACE_SETFPXREGS: ::c_uint = 19; +pub const PTRACE_SYSEMU: ::c_uint = 31; +pub const PTRACE_SYSEMU_SINGLESTEP: ::c_uint = 32; + +pub const MCL_CURRENT: ::c_int = 0x0001; +pub const MCL_FUTURE: ::c_int = 0x0002; + +pub const POLLWRNORM: ::c_short = 0x100; +pub const POLLWRBAND: ::c_short = 0x200; + +pub const EFD_NONBLOCK: ::c_int = 0x800; +pub const SFD_NONBLOCK: ::c_int = 0x0800; + +pub const SIGCHLD: ::c_int = 17; +pub const SIGBUS: ::c_int = 7; +pub const SIGUSR1: ::c_int = 10; +pub const SIGUSR2: ::c_int = 12; +pub const SIGCONT: ::c_int = 18; +pub const SIGSTOP: ::c_int = 19; +pub const SIGTSTP: ::c_int = 20; +pub const SIGURG: ::c_int = 23; +pub const SIGIO: ::c_int = 29; +pub const SIGSYS: ::c_int = 31; +pub const SIGSTKFLT: ::c_int = 16; +#[deprecated(since = "0.2.55", note = "Use SIGSYS instead")] +pub const SIGUNUSED: ::c_int = 31; +pub const SIGPOLL: ::c_int = 29; +pub const SIGPWR: ::c_int = 30; +pub const SIG_SETMASK: ::c_int = 2; +pub const SIG_BLOCK: ::c_int = 0x000000; +pub const SIG_UNBLOCK: ::c_int = 0x01; +pub const SIGTTIN: ::c_int = 21; +pub const SIGTTOU: ::c_int = 22; +pub const SIGXCPU: ::c_int = 24; +pub const SIGXFSZ: ::c_int = 25; +pub const SIGVTALRM: ::c_int = 26; +pub const SIGPROF: ::c_int = 27; +pub const SIGWINCH: ::c_int = 28; +pub const SIGSTKSZ: ::size_t = 8192; +pub const MINSIGSTKSZ: ::size_t = 2048; +pub const CBAUD: ::tcflag_t = 0o0010017; +pub const TAB1: ::tcflag_t = 0x00000800; +pub const TAB2: ::tcflag_t = 0x00001000; +pub const TAB3: ::tcflag_t = 0x00001800; +pub const CR1: ::tcflag_t = 0x00000200; +pub const CR2: ::tcflag_t = 0x00000400; +pub const CR3: ::tcflag_t = 0x00000600; +pub const FF1: ::tcflag_t = 0x00008000; +pub const BS1: ::tcflag_t = 0x00002000; +pub const VT1: ::tcflag_t = 0x00004000; +pub const VWERASE: usize = 14; +pub const VREPRINT: usize = 12; +pub const VSUSP: usize = 10; +pub const VSTART: usize = 8; +pub const VSTOP: usize = 9; +pub const VDISCARD: usize = 13; +pub const VTIME: usize = 5; +pub const IXON: ::tcflag_t = 0x00000400; +pub const IXOFF: ::tcflag_t = 0x00001000; +pub const ONLCR: ::tcflag_t = 0x4; +pub const CSIZE: ::tcflag_t = 0x00000030; +pub const CS6: ::tcflag_t = 0x00000010; +pub const CS7: ::tcflag_t = 0x00000020; +pub const CS8: ::tcflag_t = 0x00000030; +pub const CSTOPB: ::tcflag_t = 0x00000040; +pub const CREAD: ::tcflag_t = 0x00000080; +pub const PARENB: ::tcflag_t = 0x00000100; +pub const PARODD: ::tcflag_t = 0x00000200; +pub const HUPCL: ::tcflag_t = 0x00000400; +pub const CLOCAL: ::tcflag_t = 0x00000800; +pub const ECHOKE: ::tcflag_t = 0x00000800; +pub const ECHOE: ::tcflag_t = 0x00000010; +pub const ECHOK: ::tcflag_t = 0x00000020; +pub const ECHONL: ::tcflag_t = 0x00000040; +pub const ECHOPRT: ::tcflag_t = 0x00000400; +pub const ECHOCTL: ::tcflag_t = 0x00000200; +pub const ISIG: ::tcflag_t = 0x00000001; +pub const ICANON: ::tcflag_t = 0x00000002; +pub const PENDIN: ::tcflag_t = 0x00004000; +pub const NOFLSH: ::tcflag_t = 0x00000080; +pub const CIBAUD: ::tcflag_t = 0o02003600000; +pub const CBAUDEX: ::tcflag_t = 0o010000; +pub const VSWTC: usize = 7; +pub const OLCUC: ::tcflag_t = 0o000002; +pub const NLDLY: ::tcflag_t = 0o000400; +pub const CRDLY: ::tcflag_t = 0o003000; +pub const TABDLY: ::tcflag_t = 0o014000; +pub const BSDLY: ::tcflag_t = 0o020000; +pub const FFDLY: ::tcflag_t = 0o100000; +pub const VTDLY: ::tcflag_t = 0o040000; +pub const XTABS: ::tcflag_t = 0o014000; + +pub const B0: ::speed_t = 0o000000; +pub const B50: ::speed_t = 0o000001; +pub const B75: ::speed_t = 0o000002; +pub const B110: ::speed_t = 0o000003; +pub const B134: ::speed_t = 0o000004; +pub const B150: ::speed_t = 0o000005; +pub const B200: ::speed_t = 0o000006; +pub const B300: ::speed_t = 0o000007; +pub const B600: ::speed_t = 0o000010; +pub const B1200: ::speed_t = 0o000011; +pub const B1800: ::speed_t = 0o000012; +pub const B2400: ::speed_t = 0o000013; +pub const B4800: ::speed_t = 0o000014; +pub const B9600: ::speed_t = 0o000015; +pub const B19200: ::speed_t = 0o000016; +pub const B38400: ::speed_t = 0o000017; +pub const EXTA: ::speed_t = B19200; +pub const EXTB: ::speed_t = B38400; +pub const B57600: ::speed_t = 0o010001; +pub const B115200: ::speed_t = 0o010002; +pub const B230400: ::speed_t = 0o010003; +pub const B460800: ::speed_t = 0o010004; +pub const B500000: ::speed_t = 0o010005; +pub const B576000: ::speed_t = 0o010006; +pub const B921600: ::speed_t = 0o010007; +pub const B1000000: ::speed_t = 0o010010; +pub const B1152000: ::speed_t = 0o010011; +pub const B1500000: ::speed_t = 0o010012; +pub const B2000000: ::speed_t = 0o010013; +pub const B2500000: ::speed_t = 0o010014; +pub const B3000000: ::speed_t = 0o010015; +pub const B3500000: ::speed_t = 0o010016; +pub const B4000000: ::speed_t = 0o010017; + +pub const VEOL: usize = 11; +pub const VEOL2: usize = 16; +pub const VMIN: usize = 6; +pub const IEXTEN: ::tcflag_t = 0x00008000; +pub const TOSTOP: ::tcflag_t = 0x00000100; +pub const FLUSHO: ::tcflag_t = 0x00001000; +pub const EXTPROC: ::tcflag_t = 0x00010000; + +pub const TCSANOW: ::c_int = 0; +pub const TCSADRAIN: ::c_int = 1; +pub const TCSAFLUSH: ::c_int = 2; + +pub const SYS_restart_syscall: ::c_long = 0; +pub const SYS_exit: ::c_long = 1; +pub const SYS_fork: ::c_long = 2; +pub const SYS_read: ::c_long = 3; +pub const SYS_write: ::c_long = 4; +pub const SYS_open: ::c_long = 5; +pub const SYS_close: ::c_long = 6; +pub const SYS_waitpid: ::c_long = 7; +pub const SYS_creat: ::c_long = 8; +pub const SYS_link: ::c_long = 9; +pub const SYS_unlink: ::c_long = 10; +pub const SYS_execve: ::c_long = 11; +pub const SYS_chdir: ::c_long = 12; +pub const SYS_time32: ::c_long = 13; +pub const SYS_mknod: ::c_long = 14; +pub const SYS_chmod: ::c_long = 15; +pub const SYS_chown16: ::c_long = 16; +pub const SYS_stat: ::c_long = 18; +pub const SYS_lseek: ::c_long = 19; +pub const SYS_getpid: ::c_long = 20; +pub const SYS_mount: ::c_long = 21; +pub const SYS_oldumount: ::c_long = 22; +pub const SYS_setuid16: ::c_long = 23; +pub const SYS_getuid16: ::c_long = 24; +pub const SYS_stime32: ::c_long = 25; +pub const SYS_ptrace: ::c_long = 26; +pub const SYS_alarm: ::c_long = 27; +pub const SYS_fstat: ::c_long = 28; +pub const SYS_pause: ::c_long = 29; +pub const SYS_utime32: ::c_long = 30; +pub const SYS_access: ::c_long = 33; +pub const SYS_nice: ::c_long = 34; +pub const SYS_sync: ::c_long = 36; +pub const SYS_kill: ::c_long = 37; +pub const SYS_rename: ::c_long = 38; +pub const SYS_mkdir: ::c_long = 39; +pub const SYS_rmdir: ::c_long = 40; +pub const SYS_dup: ::c_long = 41; +pub const SYS_pipe: ::c_long = 42; +pub const SYS_times: ::c_long = 43; +pub const SYS_brk: ::c_long = 45; +pub const SYS_setgid16: ::c_long = 46; +pub const SYS_getgid16: ::c_long = 47; +pub const SYS_signal: ::c_long = 48; +pub const SYS_geteuid16: ::c_long = 49; +pub const SYS_getegid16: ::c_long = 50; +pub const SYS_acct: ::c_long = 51; +pub const SYS_umount: ::c_long = 52; +pub const SYS_ioctl: ::c_long = 54; +pub const SYS_fcntl: ::c_long = 55; +pub const SYS_setpgid: ::c_long = 57; +pub const SYS_umask: ::c_long = 60; +pub const SYS_chroot: ::c_long = 61; +pub const SYS_ustat: ::c_long = 62; +pub const SYS_dup2: ::c_long = 63; +pub const SYS_getppid: ::c_long = 64; +pub const SYS_getpgrp: ::c_long = 65; +pub const SYS_setsid: ::c_long = 66; +pub const SYS_sigaction: ::c_long = 67; +pub const SYS_sgetmask: ::c_long = 68; +pub const SYS_ssetmask: ::c_long = 69; +pub const SYS_setreuid16: ::c_long = 70; +pub const SYS_setregid16: ::c_long = 71; +pub const SYS_sigsuspend: ::c_long = 72; +pub const SYS_sigpending: ::c_long = 73; +pub const SYS_sethostname: ::c_long = 74; +pub const SYS_setrlimit: ::c_long = 75; +pub const SYS_old_getrlimit: ::c_long = 76; +pub const SYS_getrusage: ::c_long = 77; +pub const SYS_gettimeofday: ::c_long = 78; +pub const SYS_settimeofday: ::c_long = 79; +pub const SYS_getgroups16: ::c_long = 80; +pub const SYS_setgroups16: ::c_long = 81; +pub const SYS_old_select: ::c_long = 82; +pub const SYS_symlink: ::c_long = 83; +pub const SYS_lstat: ::c_long = 84; +pub const SYS_readlink: ::c_long = 85; +pub const SYS_uselib: ::c_long = 86; +pub const SYS_swapon: ::c_long = 87; +pub const SYS_reboot: ::c_long = 88; +pub const SYS_old_readdir: ::c_long = 89; +pub const SYS_old_mmap: ::c_long = 90; +pub const SYS_munmap: ::c_long = 91; +pub const SYS_truncate: ::c_long = 92; +pub const SYS_ftruncate: ::c_long = 93; +pub const SYS_fchmod: ::c_long = 94; +pub const SYS_fchown16: ::c_long = 95; +pub const SYS_getpriority: ::c_long = 96; +pub const SYS_setpriority: ::c_long = 97; +pub const SYS_statfs: ::c_long = 99; +pub const SYS_fstatfs: ::c_long = 100; +pub const SYS_socketcall: ::c_long = 102; +pub const SYS_syslog: ::c_long = 103; +pub const SYS_setitimer: ::c_long = 104; +pub const SYS_getitimer: ::c_long = 105; +pub const SYS_newstat: ::c_long = 106; +pub const SYS_newlstat: ::c_long = 107; +pub const SYS_newfstat: ::c_long = 108; +pub const SYS_vhangup: ::c_long = 111; +pub const SYS_wait4: ::c_long = 114; +pub const SYS_swapoff: ::c_long = 115; +pub const SYS_sysinfo: ::c_long = 116; +pub const SYS_ipc: ::c_long = 117; +pub const SYS_fsync: ::c_long = 118; +pub const SYS_sigreturn: ::c_long = 119; +pub const SYS_clone: ::c_long = 120; +pub const SYS_setdomainname: ::c_long = 121; +pub const SYS_newuname: ::c_long = 122; +pub const SYS_cacheflush: ::c_long = 123; +pub const SYS_adjtimex_time32: ::c_long = 124; +pub const SYS_mprotect: ::c_long = 125; +pub const SYS_sigprocmask: ::c_long = 126; +pub const SYS_create_module: ::c_long = 127; +pub const SYS_init_module: ::c_long = 128; +pub const SYS_delete_module: ::c_long = 129; +pub const SYS_get_kernel_syms: ::c_long = 130; +pub const SYS_quotactl: ::c_long = 131; +pub const SYS_getpgid: ::c_long = 132; +pub const SYS_fchdir: ::c_long = 133; +pub const SYS_bdflush: ::c_long = 134; +pub const SYS_sysfs: ::c_long = 135; +pub const SYS_personality: ::c_long = 136; +pub const SYS_setfsuid16: ::c_long = 138; +pub const SYS_setfsgid16: ::c_long = 139; +pub const SYS_llseek: ::c_long = 140; +pub const SYS_getdents: ::c_long = 141; +pub const SYS_select: ::c_long = 142; +pub const SYS_flock: ::c_long = 143; +pub const SYS_msync: ::c_long = 144; +pub const SYS_readv: ::c_long = 145; +pub const SYS_writev: ::c_long = 146; +pub const SYS_getsid: ::c_long = 147; +pub const SYS_fdatasync: ::c_long = 148; +pub const SYS__sysctl: ::c_long = 149; +pub const SYS_mlock: ::c_long = 150; +pub const SYS_munlock: ::c_long = 151; +pub const SYS_mlockall: ::c_long = 152; +pub const SYS_munlockall: ::c_long = 153; +pub const SYS_sched_setparam: ::c_long = 154; +pub const SYS_sched_getparam: ::c_long = 155; +pub const SYS_sched_setscheduler: ::c_long = 156; +pub const SYS_sched_getscheduler: ::c_long = 157; +pub const SYS_sched_yield: ::c_long = 158; +pub const SYS_sched_get_priority_max: ::c_long = 159; +pub const SYS_sched_get_priority_min: ::c_long = 160; +pub const SYS_sched_rr_get_interval_time32: ::c_long = 161; +pub const SYS_nanosleep_time32: ::c_long = 162; +pub const SYS_mremap: ::c_long = 163; +pub const SYS_setresuid16: ::c_long = 164; +pub const SYS_getresuid16: ::c_long = 165; +pub const SYS_getpagesize: ::c_long = 166; +pub const SYS_query_module: ::c_long = 167; +pub const SYS_poll: ::c_long = 168; +pub const SYS_nfsservctl: ::c_long = 169; +pub const SYS_setresgid16: ::c_long = 170; +pub const SYS_getresgid16: ::c_long = 171; +pub const SYS_prctl: ::c_long = 172; +pub const SYS_rt_sigreturn: ::c_long = 173; +pub const SYS_rt_sigaction: ::c_long = 174; +pub const SYS_rt_sigprocmask: ::c_long = 175; +pub const SYS_rt_sigpending: ::c_long = 176; +pub const SYS_rt_sigtimedwait_time32: ::c_long = 177; +pub const SYS_rt_sigqueueinfo: ::c_long = 178; +pub const SYS_rt_sigsuspend: ::c_long = 179; +pub const SYS_pread64: ::c_long = 180; +pub const SYS_pwrite64: ::c_long = 181; +pub const SYS_lchown16: ::c_long = 182; +pub const SYS_getcwd: ::c_long = 183; +pub const SYS_capget: ::c_long = 184; +pub const SYS_capset: ::c_long = 185; +pub const SYS_sigaltstack: ::c_long = 186; +pub const SYS_sendfile: ::c_long = 187; +pub const SYS_getpmsg: ::c_long = 188; +pub const SYS_putpmsg: ::c_long = 189; +pub const SYS_vfork: ::c_long = 190; +pub const SYS_getrlimit: ::c_long = 191; +pub const SYS_mmap2: ::c_long = 192; +pub const SYS_truncate64: ::c_long = 193; +pub const SYS_ftruncate64: ::c_long = 194; +pub const SYS_stat64: ::c_long = 195; +pub const SYS_lstat64: ::c_long = 196; +pub const SYS_fstat64: ::c_long = 197; +pub const SYS_chown: ::c_long = 198; +pub const SYS_getuid: ::c_long = 199; +pub const SYS_getgid: ::c_long = 200; +pub const SYS_geteuid: ::c_long = 201; +pub const SYS_getegid: ::c_long = 202; +pub const SYS_setreuid: ::c_long = 203; +pub const SYS_setregid: ::c_long = 204; +pub const SYS_getgroups: ::c_long = 205; +pub const SYS_setgroups: ::c_long = 206; +pub const SYS_fchown: ::c_long = 207; +pub const SYS_setresuid: ::c_long = 208; +pub const SYS_getresuid: ::c_long = 209; +pub const SYS_setresgid: ::c_long = 210; +pub const SYS_getresgid: ::c_long = 211; +pub const SYS_lchown: ::c_long = 212; +pub const SYS_setuid: ::c_long = 213; +pub const SYS_setgid: ::c_long = 214; +pub const SYS_setfsuid: ::c_long = 215; +pub const SYS_setfsgid: ::c_long = 216; +pub const SYS_pivot_root: ::c_long = 217; +pub const SYS_getdents64: ::c_long = 220; +pub const SYS_gettid: ::c_long = 221; +pub const SYS_tkill: ::c_long = 222; +pub const SYS_setxattr: ::c_long = 223; +pub const SYS_lsetxattr: ::c_long = 224; +pub const SYS_fsetxattr: ::c_long = 225; +pub const SYS_getxattr: ::c_long = 226; +pub const SYS_lgetxattr: ::c_long = 227; +pub const SYS_fgetxattr: ::c_long = 228; +pub const SYS_listxattr: ::c_long = 229; +pub const SYS_llistxattr: ::c_long = 230; +pub const SYS_flistxattr: ::c_long = 231; +pub const SYS_removexattr: ::c_long = 232; +pub const SYS_lremovexattr: ::c_long = 233; +pub const SYS_fremovexattr: ::c_long = 234; +pub const SYS_futex_time32: ::c_long = 235; +pub const SYS_sendfile64: ::c_long = 236; +pub const SYS_mincore: ::c_long = 237; +pub const SYS_madvise: ::c_long = 238; +pub const SYS_fcntl64: ::c_long = 239; +pub const SYS_readahead: ::c_long = 240; +pub const SYS_io_setup: ::c_long = 241; +pub const SYS_io_destroy: ::c_long = 242; +pub const SYS_io_getevents_time32: ::c_long = 243; +pub const SYS_io_submit: ::c_long = 244; +pub const SYS_io_cancel: ::c_long = 245; +pub const SYS_fadvise64: ::c_long = 246; +pub const SYS_exit_group: ::c_long = 247; +pub const SYS_lookup_dcookie: ::c_long = 248; +pub const SYS_epoll_create: ::c_long = 249; +pub const SYS_epoll_ctl: ::c_long = 250; +pub const SYS_epoll_wait: ::c_long = 251; +pub const SYS_remap_file_pages: ::c_long = 252; +pub const SYS_set_tid_address: ::c_long = 253; +pub const SYS_timer_create: ::c_long = 254; +pub const SYS_timer_settime32: ::c_long = 255; +pub const SYS_timer_gettime32: ::c_long = 256; +pub const SYS_timer_getoverrun: ::c_long = 257; +pub const SYS_timer_delete: ::c_long = 258; +pub const SYS_clock_settime32: ::c_long = 259; +pub const SYS_clock_gettime32: ::c_long = 260; +pub const SYS_clock_getres_time32: ::c_long = 261; +pub const SYS_clock_nanosleep_time32: ::c_long = 262; +pub const SYS_statfs64: ::c_long = 263; +pub const SYS_fstatfs64: ::c_long = 264; +pub const SYS_tgkill: ::c_long = 265; +pub const SYS_utimes_time32: ::c_long = 266; +pub const SYS_fadvise64_64: ::c_long = 267; +pub const SYS_mbind: ::c_long = 268; +pub const SYS_get_mempolicy: ::c_long = 269; +pub const SYS_set_mempolicy: ::c_long = 270; +pub const SYS_mq_open: ::c_long = 271; +pub const SYS_mq_unlink: ::c_long = 272; +pub const SYS_mq_timedsend_time32: ::c_long = 273; +pub const SYS_mq_timedreceive_time32: ::c_long = 274; +pub const SYS_mq_notify: ::c_long = 275; +pub const SYS_mq_getsetattr: ::c_long = 276; +pub const SYS_waitid: ::c_long = 277; +pub const SYS_add_key: ::c_long = 279; +pub const SYS_request_key: ::c_long = 280; +pub const SYS_keyctl: ::c_long = 281; +pub const SYS_ioprio_set: ::c_long = 282; +pub const SYS_ioprio_get: ::c_long = 283; +pub const SYS_inotify_init: ::c_long = 284; +pub const SYS_inotify_add_watch: ::c_long = 285; +pub const SYS_inotify_rm_watch: ::c_long = 286; +pub const SYS_migrate_pages: ::c_long = 287; +pub const SYS_openat: ::c_long = 288; +pub const SYS_mkdirat: ::c_long = 289; +pub const SYS_mknodat: ::c_long = 290; +pub const SYS_fchownat: ::c_long = 291; +pub const SYS_futimesat_time32: ::c_long = 292; +pub const SYS_fstatat64: ::c_long = 293; +pub const SYS_unlinkat: ::c_long = 294; +pub const SYS_renameat: ::c_long = 295; +pub const SYS_linkat: ::c_long = 296; +pub const SYS_symlinkat: ::c_long = 297; +pub const SYS_readlinkat: ::c_long = 298; +pub const SYS_fchmodat: ::c_long = 299; +pub const SYS_faccessat: ::c_long = 300; +pub const SYS_pselect6_time32: ::c_long = 301; +pub const SYS_ppoll_time32: ::c_long = 302; +pub const SYS_unshare: ::c_long = 303; +pub const SYS_set_robust_list: ::c_long = 304; +pub const SYS_get_robust_list: ::c_long = 305; +pub const SYS_splice: ::c_long = 306; +pub const SYS_sync_file_range: ::c_long = 307; +pub const SYS_tee: ::c_long = 308; +pub const SYS_vmsplice: ::c_long = 309; +pub const SYS_move_pages: ::c_long = 310; +pub const SYS_sched_setaffinity: ::c_long = 311; +pub const SYS_sched_getaffinity: ::c_long = 312; +pub const SYS_kexec_load: ::c_long = 313; +pub const SYS_getcpu: ::c_long = 314; +pub const SYS_epoll_pwait: ::c_long = 315; +pub const SYS_utimensat_time32: ::c_long = 316; +pub const SYS_signalfd: ::c_long = 317; +pub const SYS_timerfd_create: ::c_long = 318; +pub const SYS_eventfd: ::c_long = 319; +pub const SYS_fallocate: ::c_long = 320; +pub const SYS_timerfd_settime32: ::c_long = 321; +pub const SYS_timerfd_gettime32: ::c_long = 322; +pub const SYS_signalfd4: ::c_long = 323; +pub const SYS_eventfd2: ::c_long = 324; +pub const SYS_epoll_create1: ::c_long = 325; +pub const SYS_dup3: ::c_long = 326; +pub const SYS_pipe2: ::c_long = 327; +pub const SYS_inotify_init1: ::c_long = 328; +pub const SYS_preadv: ::c_long = 329; +pub const SYS_pwritev: ::c_long = 330; +pub const SYS_rt_tgsigqueueinfo: ::c_long = 331; +pub const SYS_perf_event_open: ::c_long = 332; +pub const SYS_get_thread_area: ::c_long = 333; +pub const SYS_set_thread_area: ::c_long = 334; +pub const SYS_atomic_cmpxchg_32: ::c_long = 335; +pub const SYS_atomic_barrier: ::c_long = 336; +pub const SYS_fanotify_init: ::c_long = 337; +pub const SYS_fanotify_mark: ::c_long = 338; +pub const SYS_prlimit64: ::c_long = 339; +pub const SYS_name_to_handle_at: ::c_long = 340; +pub const SYS_open_by_handle_at: ::c_long = 341; +pub const SYS_clock_adjtime32: ::c_long = 342; +pub const SYS_syncfs: ::c_long = 343; +pub const SYS_setns: ::c_long = 344; +pub const SYS_process_vm_readv: ::c_long = 345; +pub const SYS_process_vm_writev: ::c_long = 346; +pub const SYS_kcmp: ::c_long = 347; +pub const SYS_finit_module: ::c_long = 348; +pub const SYS_sched_setattr: ::c_long = 349; +pub const SYS_sched_getattr: ::c_long = 350; +pub const SYS_renameat2: ::c_long = 351; +pub const SYS_getrandom: ::c_long = 352; +pub const SYS_memfd_create: ::c_long = 353; +pub const SYS_bpf: ::c_long = 354; +pub const SYS_execveat: ::c_long = 355; +pub const SYS_socket: ::c_long = 356; +pub const SYS_socketpair: ::c_long = 357; +pub const SYS_bind: ::c_long = 358; +pub const SYS_connect: ::c_long = 359; +pub const SYS_listen: ::c_long = 360; +pub const SYS_accept4: ::c_long = 361; +pub const SYS_getsockopt: ::c_long = 362; +pub const SYS_setsockopt: ::c_long = 363; +pub const SYS_getsockname: ::c_long = 364; +pub const SYS_getpeername: ::c_long = 365; +pub const SYS_sendto: ::c_long = 366; +pub const SYS_sendmsg: ::c_long = 367; +pub const SYS_recvfrom: ::c_long = 368; +pub const SYS_recvmsg: ::c_long = 369; +pub const SYS_shutdown: ::c_long = 370; +pub const SYS_recvmmsg_time32: ::c_long = 371; +pub const SYS_sendmmsg: ::c_long = 372; +pub const SYS_userfaultfd: ::c_long = 373; +pub const SYS_membarrier: ::c_long = 374; +pub const SYS_mlock2: ::c_long = 375; +pub const SYS_copy_file_range: ::c_long = 376; +pub const SYS_preadv2: ::c_long = 377; +pub const SYS_pwritev2: ::c_long = 378; +pub const SYS_statx: ::c_long = 379; +pub const SYS_seccomp: ::c_long = 380; +pub const SYS_pkey_mprotect: ::c_long = 381; +pub const SYS_pkey_alloc: ::c_long = 382; +pub const SYS_pkey_free: ::c_long = 383; +pub const SYS_rseq: ::c_long = 384; +pub const SYS_semget: ::c_long = 393; +pub const SYS_semctl: ::c_long = 394; +pub const SYS_shmget: ::c_long = 395; +pub const SYS_shmctl: ::c_long = 396; +pub const SYS_shmat: ::c_long = 397; +pub const SYS_shmdt: ::c_long = 398; +pub const SYS_msgget: ::c_long = 399; +pub const SYS_msgsnd: ::c_long = 400; +pub const SYS_msgrcv: ::c_long = 401; +pub const SYS_msgctl: ::c_long = 402; +pub const SYS_clock_gettime: ::c_long = 403; +pub const SYS_clock_settime: ::c_long = 404; +pub const SYS_clock_adjtime: ::c_long = 405; +pub const SYS_clock_getres: ::c_long = 406; +pub const SYS_clock_nanosleep: ::c_long = 407; +pub const SYS_timer_gettime: ::c_long = 408; +pub const SYS_timer_settime: ::c_long = 409; +pub const SYS_timerfd_gettime: ::c_long = 410; +pub const SYS_timerfd_settime: ::c_long = 411; +pub const SYS_utimensat: ::c_long = 412; +pub const SYS_pselect6: ::c_long = 413; +pub const SYS_ppoll: ::c_long = 414; +pub const SYS_io_pgetevents: ::c_long = 416; +pub const SYS_recvmmsg: ::c_long = 417; +pub const SYS_mq_timedsend: ::c_long = 418; +pub const SYS_mq_timedreceive: ::c_long = 419; +pub const SYS_semtimedop: ::c_long = 420; +pub const SYS_rt_sigtimedwait: ::c_long = 421; +pub const SYS_futex: ::c_long = 422; +pub const SYS_sched_rr_get_interval: ::c_long = 423; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; +pub const SYS_quotactl_fd: ::c_long = 443; +pub const SYS_landlock_create_ruleset: ::c_long = 444; +pub const SYS_landlock_add_rule: ::c_long = 445; +pub const SYS_landlock_restrict_self: ::c_long = 446; +pub const SYS_process_mrelease: ::c_long = 448; +pub const SYS_futex_waitv: ::c_long = 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 450; diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/mips/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/mips/mod.rs index 4a3b600aa..6a03f0ba8 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/mips/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/mips/mod.rs @@ -153,17 +153,6 @@ s! { pub l_pid: ::pid_t, pad: [::c_long; 4], } - - pub struct termios2 { - pub c_iflag: ::tcflag_t, - pub c_oflag: ::tcflag_t, - pub c_cflag: ::tcflag_t, - pub c_lflag: ::tcflag_t, - pub c_line: ::cc_t, - pub c_cc: [::cc_t; 23], - pub c_ispeed: ::speed_t, - pub c_ospeed: ::speed_t, - } } pub const O_LARGEFILE: ::c_int = 0x2000; @@ -527,18 +516,39 @@ pub const SYS_pkey_mprotect: ::c_long = 4000 + 363; pub const SYS_pkey_alloc: ::c_long = 4000 + 364; pub const SYS_pkey_free: ::c_long = 4000 + 365; pub const SYS_statx: ::c_long = 4000 + 366; +pub const SYS_rseq: ::c_long = 4000 + 367; +pub const SYS_pidfd_send_signal: ::c_long = 4000 + 424; +pub const SYS_io_uring_setup: ::c_long = 4000 + 425; +pub const SYS_io_uring_enter: ::c_long = 4000 + 426; +pub const SYS_io_uring_register: ::c_long = 4000 + 427; +pub const SYS_open_tree: ::c_long = 4000 + 428; +pub const SYS_move_mount: ::c_long = 4000 + 429; +pub const SYS_fsopen: ::c_long = 4000 + 430; +pub const SYS_fsconfig: ::c_long = 4000 + 431; +pub const SYS_fsmount: ::c_long = 4000 + 432; +pub const SYS_fspick: ::c_long = 4000 + 433; +pub const SYS_pidfd_open: ::c_long = 4000 + 434; +pub const SYS_clone3: ::c_long = 4000 + 435; +pub const SYS_close_range: ::c_long = 4000 + 436; +pub const SYS_openat2: ::c_long = 4000 + 437; +pub const SYS_pidfd_getfd: ::c_long = 4000 + 438; +pub const SYS_faccessat2: ::c_long = 4000 + 439; +pub const SYS_process_madvise: ::c_long = 4000 + 440; +pub const SYS_epoll_pwait2: ::c_long = 4000 + 441; +pub const SYS_mount_setattr: ::c_long = 4000 + 442; +pub const SYS_quotactl_fd: ::c_long = 4000 + 443; +pub const SYS_landlock_create_ruleset: ::c_long = 4000 + 444; +pub const SYS_landlock_add_rule: ::c_long = 4000 + 445; +pub const SYS_landlock_restrict_self: ::c_long = 4000 + 446; +pub const SYS_memfd_secret: ::c_long = 4000 + 447; +pub const SYS_process_mrelease: ::c_long = 4000 + 448; +pub const SYS_futex_waitv: ::c_long = 4000 + 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 4000 + 450; pub const O_DIRECT: ::c_int = 0x8000; pub const O_DIRECTORY: ::c_int = 0x10000; pub const O_NOFOLLOW: ::c_int = 0x20000; -pub const RLIM_INFINITY: ::rlim_t = 0x7fffffff; -pub const RLIMIT_NOFILE: ::__rlimit_resource_t = 5; -pub const RLIMIT_AS: ::__rlimit_resource_t = 6; -pub const RLIMIT_RSS: ::__rlimit_resource_t = 7; -pub const RLIMIT_NPROC: ::__rlimit_resource_t = 8; -pub const RLIMIT_MEMLOCK: ::__rlimit_resource_t = 9; - pub const O_APPEND: ::c_int = 8; pub const O_CREAT: ::c_int = 256; pub const O_EXCL: ::c_int = 1024; @@ -649,59 +659,6 @@ pub const MAP_STACK: ::c_int = 0x40000; pub const SOCK_STREAM: ::c_int = 2; pub const SOCK_DGRAM: ::c_int = 1; -pub const SOL_SOCKET: ::c_int = 0xffff; - -pub const SO_REUSEADDR: ::c_int = 0x0004; -pub const SO_KEEPALIVE: ::c_int = 0x0008; -pub const SO_DONTROUTE: ::c_int = 0x0010; -pub const SO_BROADCAST: ::c_int = 0x0020; -pub const SO_LINGER: ::c_int = 0x0080; -pub const SO_OOBINLINE: ::c_int = 0x0100; -pub const SO_REUSEPORT: ::c_int = 0x0200; -pub const SO_TYPE: ::c_int = 0x1008; -pub const SO_STYLE: ::c_int = SO_TYPE; -pub const SO_ERROR: ::c_int = 0x1007; -pub const SO_SNDBUF: ::c_int = 0x1001; -pub const SO_RCVBUF: ::c_int = 0x1002; -pub const SO_SNDLOWAT: ::c_int = 0x1003; -pub const SO_RCVLOWAT: ::c_int = 0x1004; -pub const SO_SNDTIMEO: ::c_int = 0x1005; -pub const SO_RCVTIMEO: ::c_int = 0x1006; -pub const SO_ACCEPTCONN: ::c_int = 0x1009; -pub const SO_PROTOCOL: ::c_int = 0x1028; -pub const SO_DOMAIN: ::c_int = 0x1029; -pub const SO_NO_CHECK: ::c_int = 11; -pub const SO_PASSCRED: ::c_int = 17; -pub const SO_PEERCRED: ::c_int = 18; -pub const SO_SECURITY_AUTHENTICATION: ::c_int = 22; -pub const SO_SECURITY_ENCRYPTION_TRANSPORT: ::c_int = 23; -pub const SO_SECURITY_ENCRYPTION_NETWORK: ::c_int = 24; -pub const SO_ATTACH_FILTER: ::c_int = 26; -pub const SO_DETACH_FILTER: ::c_int = 27; -pub const SO_GET_FILTER: ::c_int = SO_ATTACH_FILTER; -pub const SO_PEERNAME: ::c_int = 28; -pub const SO_PEERSEC: ::c_int = 30; -pub const SO_SNDBUFFORCE: ::c_int = 31; -pub const SO_RCVBUFFORCE: ::c_int = 33; -pub const SO_PASSSEC: ::c_int = 34; -pub const SO_TIMESTAMPNS: ::c_int = 35; -pub const SCM_TIMESTAMPNS: ::c_int = SO_TIMESTAMPNS; - -pub const SO_WIFI_STATUS: ::c_int = 41; -pub const SCM_WIFI_STATUS: ::c_int = SO_WIFI_STATUS; -pub const SO_NOFCS: ::c_int = 43; -pub const SO_LOCK_FILTER: ::c_int = 44; -pub const SO_SELECT_ERR_QUEUE: ::c_int = 45; -pub const SO_MAX_PACING_RATE: ::c_int = 47; -pub const SO_BPF_EXTENSIONS: ::c_int = 48; -pub const SO_INCOMING_CPU: ::c_int = 49; -pub const SO_ATTACH_BPF: ::c_int = 50; -pub const SO_DETACH_BPF: ::c_int = SO_DETACH_FILTER; - -pub const FIOCLEX: ::c_ulong = 0x6601; -pub const FIONCLEX: ::c_ulong = 0x6602; -pub const FIONBIO: ::c_ulong = 0x667e; - pub const SA_SIGINFO: ::c_int = 0x00000008; pub const SA_NOCLDWAIT: ::c_int = 0x00010000; @@ -756,38 +713,6 @@ pub const F_SETOWN: ::c_int = 24; pub const SFD_NONBLOCK: ::c_int = 0x80; -pub const TCGETS: ::c_ulong = 0x540d; -pub const TCSETS: ::c_ulong = 0x540e; -pub const TCSETSW: ::c_ulong = 0x540f; -pub const TCSETSF: ::c_ulong = 0x5410; -pub const TCGETA: ::c_ulong = 0x5401; -pub const TCSETA: ::c_ulong = 0x5402; -pub const TCSETAW: ::c_ulong = 0x5403; -pub const TCSETAF: ::c_ulong = 0x5404; -pub const TCSBRK: ::c_ulong = 0x5405; -pub const TCXONC: ::c_ulong = 0x5406; -pub const TCFLSH: ::c_ulong = 0x5407; -pub const TIOCGSOFTCAR: ::c_ulong = 0x5481; -pub const TIOCSSOFTCAR: ::c_ulong = 0x5482; -pub const TIOCINQ: ::c_ulong = 0x467f; -pub const TIOCLINUX: ::c_ulong = 0x5483; -pub const TIOCGSERIAL: ::c_ulong = 0x5484; -pub const TIOCEXCL: ::c_ulong = 0x740d; -pub const TIOCNXCL: ::c_ulong = 0x740e; -pub const TIOCSCTTY: ::c_ulong = 0x5480; -pub const TIOCGPGRP: ::c_ulong = 0x40047477; -pub const TIOCSPGRP: ::c_ulong = 0x80047476; -pub const TIOCOUTQ: ::c_ulong = 0x7472; -pub const TIOCSTI: ::c_ulong = 0x5472; -pub const TIOCGWINSZ: ::c_ulong = 0x40087468; -pub const TIOCSWINSZ: ::c_ulong = 0x80087467; -pub const TIOCMGET: ::c_ulong = 0x741d; -pub const TIOCMBIS: ::c_ulong = 0x741b; -pub const TIOCMBIC: ::c_ulong = 0x741c; -pub const TIOCMSET: ::c_ulong = 0x741a; -pub const FIONREAD: ::c_ulong = 0x467f; -pub const TIOCCONS: ::c_ulong = 0x80047478; - pub const RTLD_DEEPBIND: ::c_int = 0x10; pub const RTLD_GLOBAL: ::c_int = 0x4; pub const RTLD_NOLOAD: ::c_int = 0x8; @@ -867,7 +792,6 @@ pub const B19200: ::speed_t = 0o000016; pub const B38400: ::speed_t = 0o000017; pub const EXTA: ::speed_t = B19200; pub const EXTB: ::speed_t = B38400; -pub const BOTHER: ::speed_t = 0o010000; pub const B57600: ::speed_t = 0o010001; pub const B115200: ::speed_t = 0o010002; pub const B230400: ::speed_t = 0o010003; @@ -884,13 +808,6 @@ pub const B3000000: ::speed_t = 0o010015; pub const B3500000: ::speed_t = 0o010016; pub const B4000000: ::speed_t = 0o010017; -pub const TIOCM_ST: ::c_int = 0x010; -pub const TIOCM_SR: ::c_int = 0x020; -pub const TIOCM_CTS: ::c_int = 0x040; -pub const TIOCM_CAR: ::c_int = 0x100; -pub const TIOCM_RNG: ::c_int = 0x200; -pub const TIOCM_DSR: ::c_int = 0x400; - pub const EHWPOISON: ::c_int = 168; cfg_if! { diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/mod.rs index ab8d943b4..ad0d64c51 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/mod.rs @@ -5,22 +5,41 @@ use pthread_mutex_t; pub type c_long = i32; pub type c_ulong = u32; pub type clock_t = i32; -pub type time_t = i32; -pub type suseconds_t = i32; -pub type ino_t = u32; -pub type off_t = i32; -pub type blkcnt_t = i32; - -pub type fsblkcnt_t = ::c_ulong; -pub type fsfilcnt_t = ::c_ulong; -pub type rlim_t = c_ulong; + pub type shmatt_t = ::c_ulong; pub type msgqnum_t = ::c_ulong; pub type msglen_t = ::c_ulong; -pub type blksize_t = i32; pub type nlink_t = u32; pub type __u64 = ::c_ulonglong; +pub type __s64 = ::c_longlong; pub type __fsword_t = i32; +pub type fsblkcnt64_t = u64; +pub type fsfilcnt64_t = u64; +pub type __syscall_ulong_t = ::c_ulong; + +cfg_if! { + if #[cfg(target_arch = "riscv32")] { + pub type time_t = i64; + pub type suseconds_t = i64; + pub type ino_t = u64; + pub type off_t = i64; + pub type blkcnt_t = i64; + pub type fsblkcnt_t = u64; + pub type fsfilcnt_t = u64; + pub type rlim_t = u64; + pub type blksize_t = i64; + } else { + pub type time_t = i32; + pub type suseconds_t = i32; + pub type ino_t = u32; + pub type off_t = i32; + pub type blkcnt_t = i32; + pub type fsblkcnt_t = ::c_ulong; + pub type fsfilcnt_t = ::c_ulong; + pub type rlim_t = c_ulong; + pub type blksize_t = i32; + } +} s! { pub struct stat { @@ -115,75 +134,132 @@ s! { pub mem_unit: ::c_uint, pub _f: [::c_char; 8], } -} - -pub const O_NOATIME: ::c_int = 0o1000000; -pub const O_PATH: ::c_int = 0o10000000; -pub const O_TMPFILE: ::c_int = 0o20000000 | O_DIRECTORY; - -pub const SO_PRIORITY: ::c_int = 12; -pub const SO_BSDCOMPAT: ::c_int = 14; -pub const SO_BINDTODEVICE: ::c_int = 25; -pub const SO_TIMESTAMP: ::c_int = 29; -pub const SO_MARK: ::c_int = 36; -pub const SO_RXQ_OVFL: ::c_int = 40; -pub const SO_PEEK_OFF: ::c_int = 42; -pub const SO_BUSY_POLL: ::c_int = 46; -pub const SA_ONSTACK: ::c_int = 0x08000000; + pub struct semid_ds { + pub sem_perm: ipc_perm, + #[cfg(target_arch = "powerpc")] + __reserved: ::__syscall_ulong_t, + pub sem_otime: ::time_t, + #[cfg(not(any(target_arch = "mips", target_arch = "powerpc")))] + __reserved: ::__syscall_ulong_t, + #[cfg(target_arch = "powerpc")] + __reserved2: ::__syscall_ulong_t, + pub sem_ctime: ::time_t, + #[cfg(not(any(target_arch = "mips", target_arch = "powerpc")))] + __reserved2: ::__syscall_ulong_t, + pub sem_nsems: ::__syscall_ulong_t, + __glibc_reserved3: ::__syscall_ulong_t, + __glibc_reserved4: ::__syscall_ulong_t, + } +} -pub const PTRACE_DETACH: ::c_uint = 17; pub const POSIX_FADV_DONTNEED: ::c_int = 4; pub const POSIX_FADV_NOREUSE: ::c_int = 5; -pub const F_SETLK: ::c_int = 6; -pub const F_SETLKW: ::c_int = 7; - -pub const F_RDLCK: ::c_int = 0; -pub const F_WRLCK: ::c_int = 1; -pub const F_UNLCK: ::c_int = 2; - pub const F_OFD_GETLK: ::c_int = 36; pub const F_OFD_SETLK: ::c_int = 37; pub const F_OFD_SETLKW: ::c_int = 38; -pub const SFD_CLOEXEC: ::c_int = 0x080000; +pub const __SIZEOF_PTHREAD_CONDATTR_T: usize = 4; +pub const __SIZEOF_PTHREAD_MUTEX_T: usize = 24; +pub const __SIZEOF_PTHREAD_RWLOCK_T: usize = 32; +pub const __SIZEOF_PTHREAD_MUTEXATTR_T: usize = 4; +pub const __SIZEOF_PTHREAD_RWLOCKATTR_T: usize = 8; -pub const NCCS: usize = 32; +cfg_if! { + if #[cfg(target_arch = "sparc")] { + pub const O_NOATIME: ::c_int = 0x200000; + pub const O_PATH: ::c_int = 0x1000000; + pub const O_TMPFILE: ::c_int = 0x2000000 | O_DIRECTORY; -pub const O_TRUNC: ::c_int = 512; + pub const SA_ONSTACK: ::c_int = 1; -pub const O_CLOEXEC: ::c_int = 0x80000; + pub const PTRACE_DETACH: ::c_uint = 11; -pub const EBFONT: ::c_int = 59; -pub const ENOSTR: ::c_int = 60; -pub const ENODATA: ::c_int = 61; -pub const ETIME: ::c_int = 62; -pub const ENOSR: ::c_int = 63; -pub const ENONET: ::c_int = 64; -pub const ENOPKG: ::c_int = 65; -pub const EREMOTE: ::c_int = 66; -pub const ENOLINK: ::c_int = 67; -pub const EADV: ::c_int = 68; -pub const ESRMNT: ::c_int = 69; -pub const ECOMM: ::c_int = 70; -pub const EPROTO: ::c_int = 71; -pub const EDOTDOT: ::c_int = 73; + pub const F_SETLK: ::c_int = 8; + pub const F_SETLKW: ::c_int = 9; -pub const SA_NODEFER: ::c_int = 0x40000000; -pub const SA_RESETHAND: ::c_int = 0x80000000; -pub const SA_RESTART: ::c_int = 0x10000000; -pub const SA_NOCLDSTOP: ::c_int = 0x00000001; + pub const F_RDLCK: ::c_int = 1; + pub const F_WRLCK: ::c_int = 2; + pub const F_UNLCK: ::c_int = 3; -pub const EPOLL_CLOEXEC: ::c_int = 0x80000; + pub const SFD_CLOEXEC: ::c_int = 0x400000; -pub const EFD_CLOEXEC: ::c_int = 0x80000; + pub const NCCS: usize = 17; -pub const __SIZEOF_PTHREAD_CONDATTR_T: usize = 4; -pub const __SIZEOF_PTHREAD_MUTEX_T: usize = 24; -pub const __SIZEOF_PTHREAD_RWLOCK_T: usize = 32; -pub const __SIZEOF_PTHREAD_MUTEXATTR_T: usize = 4; -pub const __SIZEOF_PTHREAD_RWLOCKATTR_T: usize = 8; + pub const O_TRUNC: ::c_int = 0x400; + pub const O_CLOEXEC: ::c_int = 0x400000; + + pub const EBFONT: ::c_int = 109; + pub const ENOSTR: ::c_int = 72; + pub const ENODATA: ::c_int = 111; + pub const ETIME: ::c_int = 73; + pub const ENOSR: ::c_int = 74; + pub const ENONET: ::c_int = 80; + pub const ENOPKG: ::c_int = 113; + pub const EREMOTE: ::c_int = 71; + pub const ENOLINK: ::c_int = 82; + pub const EADV: ::c_int = 83; + pub const ESRMNT: ::c_int = 84; + pub const ECOMM: ::c_int = 85; + pub const EPROTO: ::c_int = 86; + pub const EDOTDOT: ::c_int = 88; + + pub const SA_NODEFER: ::c_int = 0x20; + pub const SA_RESETHAND: ::c_int = 0x4; + pub const SA_RESTART: ::c_int = 0x2; + pub const SA_NOCLDSTOP: ::c_int = 0x00000008; + + pub const EPOLL_CLOEXEC: ::c_int = 0x400000; + + pub const EFD_CLOEXEC: ::c_int = 0x400000; + } else { + pub const O_NOATIME: ::c_int = 0o1000000; + pub const O_PATH: ::c_int = 0o10000000; + pub const O_TMPFILE: ::c_int = 0o20000000 | O_DIRECTORY; + + pub const SA_ONSTACK: ::c_int = 0x08000000; + + pub const PTRACE_DETACH: ::c_uint = 17; + + pub const F_SETLK: ::c_int = 6; + pub const F_SETLKW: ::c_int = 7; + + pub const F_RDLCK: ::c_int = 0; + pub const F_WRLCK: ::c_int = 1; + pub const F_UNLCK: ::c_int = 2; + + pub const SFD_CLOEXEC: ::c_int = 0x080000; + + pub const NCCS: usize = 32; + + pub const O_TRUNC: ::c_int = 512; + pub const O_CLOEXEC: ::c_int = 0x80000; + pub const EBFONT: ::c_int = 59; + pub const ENOSTR: ::c_int = 60; + pub const ENODATA: ::c_int = 61; + pub const ETIME: ::c_int = 62; + pub const ENOSR: ::c_int = 63; + pub const ENONET: ::c_int = 64; + pub const ENOPKG: ::c_int = 65; + pub const EREMOTE: ::c_int = 66; + pub const ENOLINK: ::c_int = 67; + pub const EADV: ::c_int = 68; + pub const ESRMNT: ::c_int = 69; + pub const ECOMM: ::c_int = 70; + pub const EPROTO: ::c_int = 71; + pub const EDOTDOT: ::c_int = 73; + + pub const SA_NODEFER: ::c_int = 0x40000000; + pub const SA_RESETHAND: ::c_int = 0x80000000; + pub const SA_RESTART: ::c_int = 0x10000000; + pub const SA_NOCLDSTOP: ::c_int = 0x00000001; + + pub const EPOLL_CLOEXEC: ::c_int = 0x80000; + + pub const EFD_CLOEXEC: ::c_int = 0x80000; + } +} align_const! { #[cfg(target_endian = "little")] @@ -241,7 +317,6 @@ pub const PTRACE_SETFPREGS: ::c_uint = 15; pub const PTRACE_GETREGS: ::c_uint = 12; pub const PTRACE_SETREGS: ::c_uint = 13; -#[link(name = "util")] extern "C" { pub fn sysctl( name: *mut ::c_int, @@ -263,12 +338,18 @@ cfg_if! { } else if #[cfg(target_arch = "mips")] { mod mips; pub use self::mips::*; + } else if #[cfg(target_arch = "m68k")] { + mod m68k; + pub use self::m68k::*; } else if #[cfg(target_arch = "powerpc")] { mod powerpc; pub use self::powerpc::*; } else if #[cfg(target_arch = "sparc")] { mod sparc; pub use self::sparc::*; + } else if #[cfg(target_arch = "riscv32")] { + mod riscv32; + pub use self::riscv32::*; } else { // Unknown target_arch } diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/powerpc.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/powerpc.rs index f3f9493f7..e70b216bf 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/powerpc.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/powerpc.rs @@ -163,7 +163,6 @@ s! { } } -pub const RLIM_INFINITY: ::rlim_t = !0; pub const VEOF: usize = 4; pub const RTLD_DEEPBIND: ::c_int = 0x8; pub const RTLD_GLOBAL: ::c_int = 0x100; @@ -183,24 +182,10 @@ pub const O_DSYNC: ::c_int = 4096; pub const O_FSYNC: ::c_int = 0x101000; pub const O_ASYNC: ::c_int = 0x2000; pub const O_NDELAY: ::c_int = 0x800; -pub const RLIMIT_NOFILE: ::__rlimit_resource_t = 7; -pub const RLIMIT_NPROC: ::__rlimit_resource_t = 6; -pub const RLIMIT_RSS: ::__rlimit_resource_t = 5; -pub const RLIMIT_AS: ::__rlimit_resource_t = 9; -pub const RLIMIT_MEMLOCK: ::__rlimit_resource_t = 8; pub const TCSANOW: ::c_int = 0; pub const TCSADRAIN: ::c_int = 1; pub const TCSAFLUSH: ::c_int = 2; -pub const TIOCLINUX: ::c_ulong = 0x541C; -pub const TIOCGSERIAL: ::c_ulong = 0x541E; -pub const TIOCM_ST: ::c_int = 0x008; -pub const TIOCM_SR: ::c_int = 0x010; -pub const TIOCM_CTS: ::c_int = 0x020; -pub const TIOCM_CAR: ::c_int = 0x040; -pub const TIOCM_RNG: ::c_int = 0x080; -pub const TIOCM_DSR: ::c_int = 0x100; - pub const MADV_SOFT_OFFLINE: ::c_int = 101; pub const MAP_LOCKED: ::c_int = 0x00080; pub const MAP_NORESERVE: ::c_int = 0x00040; @@ -213,9 +198,7 @@ pub const MAP_NONBLOCK: ::c_int = 0x010000; pub const MAP_STACK: ::c_int = 0x020000; pub const MAP_HUGETLB: ::c_int = 0x040000; pub const MAP_GROWSDOWN: ::c_int = 0x0100; -pub const MAP_SYNC : ::c_int = 0x080000; - -pub const SOL_SOCKET: ::c_int = 1; +pub const MAP_SYNC: ::c_int = 0x080000; pub const EDEADLOCK: ::c_int = 58; pub const EUCLEAN: ::c_int = 117; @@ -302,40 +285,12 @@ pub const ENOTRECOVERABLE: ::c_int = 131; pub const EHWPOISON: ::c_int = 133; pub const ERFKILL: ::c_int = 132; -pub const SO_REUSEADDR: ::c_int = 2; -pub const SO_TYPE: ::c_int = 3; -pub const SO_ERROR: ::c_int = 4; -pub const SO_DONTROUTE: ::c_int = 5; -pub const SO_BROADCAST: ::c_int = 6; -pub const SO_SNDBUF: ::c_int = 7; -pub const SO_RCVBUF: ::c_int = 8; -pub const SO_KEEPALIVE: ::c_int = 9; -pub const SO_OOBINLINE: ::c_int = 10; -pub const SO_LINGER: ::c_int = 13; -pub const SO_REUSEPORT: ::c_int = 15; -pub const SO_ACCEPTCONN: ::c_int = 30; -pub const SO_PROTOCOL: ::c_int = 38; -pub const SO_DOMAIN: ::c_int = 39; -pub const SO_SNDBUFFORCE: ::c_int = 32; -pub const SO_RCVBUFFORCE: ::c_int = 33; -pub const SO_NO_CHECK: ::c_int = 11; -pub const SO_RCVLOWAT: ::c_int = 16; -pub const SO_SNDLOWAT: ::c_int = 17; -pub const SO_RCVTIMEO: ::c_int = 18; -pub const SO_SNDTIMEO: ::c_int = 19; -pub const SO_PASSCRED: ::c_int = 20; -pub const SO_PEERCRED: ::c_int = 21; - pub const SA_SIGINFO: ::c_int = 0x00000004; pub const SA_NOCLDWAIT: ::c_int = 0x00000002; pub const SOCK_STREAM: ::c_int = 1; pub const SOCK_DGRAM: ::c_int = 2; -pub const FIOCLEX: ::c_ulong = 0x20006601; -pub const FIONCLEX: ::c_ulong = 0x20006602; -pub const FIONBIO: ::c_ulong = 0x8004667e; - pub const MCL_CURRENT: ::c_int = 0x2000; pub const MCL_FUTURE: ::c_int = 0x4000; @@ -349,18 +304,6 @@ pub const F_SETOWN: ::c_int = 8; pub const EFD_NONBLOCK: ::c_int = 0x800; pub const SFD_NONBLOCK: ::c_int = 0x0800; -pub const TIOCGSOFTCAR: ::c_ulong = 0x5419; -pub const TIOCSSOFTCAR: ::c_ulong = 0x541A; -pub const TIOCEXCL: ::c_ulong = 0x540C; -pub const TIOCNXCL: ::c_ulong = 0x540D; -pub const TIOCSCTTY: ::c_ulong = 0x540E; -pub const TIOCSTI: ::c_ulong = 0x5412; -pub const TIOCMGET: ::c_ulong = 0x5415; -pub const TIOCMBIS: ::c_ulong = 0x5416; -pub const TIOCMBIC: ::c_ulong = 0x5417; -pub const TIOCMSET: ::c_ulong = 0x5418; -pub const TIOCCONS: ::c_ulong = 0x541D; - pub const SIGCHLD: ::c_int = 17; pub const SIGBUS: ::c_int = 7; pub const SIGUSR1: ::c_int = 10; @@ -472,7 +415,6 @@ pub const B2500000: ::speed_t = 0o0033; pub const B3000000: ::speed_t = 0o0034; pub const B3500000: ::speed_t = 0o0035; pub const B4000000: ::speed_t = 0o0036; -pub const BOTHER: ::speed_t = 0o0037; pub const VEOL: usize = 6; pub const VEOL2: usize = 8; @@ -481,26 +423,6 @@ pub const IEXTEN: ::tcflag_t = 0x400; pub const TOSTOP: ::tcflag_t = 0x400000; pub const FLUSHO: ::tcflag_t = 0x800000; pub const EXTPROC: ::tcflag_t = 0x10000000; -pub const TCGETS: ::c_ulong = 0x403c7413; -pub const TCSETS: ::c_ulong = 0x803c7414; -pub const TCSETSW: ::c_ulong = 0x803c7415; -pub const TCSETSF: ::c_ulong = 0x803c7416; -pub const TCGETA: ::c_ulong = 0x40147417; -pub const TCSETA: ::c_ulong = 0x80147418; -pub const TCSETAW: ::c_ulong = 0x80147419; -pub const TCSETAF: ::c_ulong = 0x8014741c; -pub const TCSBRK: ::c_ulong = 0x2000741d; -pub const TCXONC: ::c_ulong = 0x2000741e; -pub const TCFLSH: ::c_ulong = 0x2000741f; -pub const TIOCINQ: ::c_ulong = 0x4004667f; -pub const TIOCGPGRP: ::c_ulong = 0x40047477; -pub const TIOCSPGRP: ::c_ulong = 0x80047476; -pub const TIOCOUTQ: ::c_ulong = 0x40047473; -pub const TIOCGWINSZ: ::c_ulong = 0x40087468; -pub const TIOCSWINSZ: ::c_ulong = 0x80087467; -pub const TIOCGRS485: ::c_int = 0x542e; -pub const TIOCSRS485: ::c_int = 0x542f; -pub const FIONREAD: ::c_ulong = 0x4004667f; pub const SYS_restart_syscall: ::c_long = 0; pub const SYS_exit: ::c_long = 1; @@ -872,3 +794,31 @@ pub const SYS_preadv2: ::c_long = 380; pub const SYS_pwritev2: ::c_long = 381; pub const SYS_kexec_file_load: ::c_long = 382; pub const SYS_statx: ::c_long = 383; +pub const SYS_rseq: ::c_long = 387; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; +pub const SYS_quotactl_fd: ::c_long = 443; +pub const SYS_landlock_create_ruleset: ::c_long = 444; +pub const SYS_landlock_add_rule: ::c_long = 445; +pub const SYS_landlock_restrict_self: ::c_long = 446; +pub const SYS_memfd_secret: ::c_long = 447; +pub const SYS_process_mrelease: ::c_long = 448; +pub const SYS_futex_waitv: ::c_long = 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 450; diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/riscv32/align.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/riscv32/align.rs new file mode 100644 index 000000000..48d152a57 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/riscv32/align.rs @@ -0,0 +1,44 @@ +s_no_extra_traits! { + #[allow(missing_debug_implementations)] + pub struct ucontext_t { + pub __uc_flags: ::c_ulong, + pub uc_link: *mut ucontext_t, + pub uc_stack: ::stack_t, + pub uc_sigmask: ::sigset_t, + pub uc_mcontext: mcontext_t, + } + + #[allow(missing_debug_implementations)] + #[repr(align(16))] + pub struct mcontext_t { + pub __gregs: [::c_ulong; 32], + pub __fpregs: __riscv_mc_fp_state, + } + + #[allow(missing_debug_implementations)] + pub union __riscv_mc_fp_state { + pub __f: __riscv_mc_f_ext_state, + pub __d: __riscv_mc_d_ext_state, + pub __q: __riscv_mc_q_ext_state, + } + + #[allow(missing_debug_implementations)] + pub struct __riscv_mc_f_ext_state { + pub __f: [::c_uint; 32], + pub __fcsr: ::c_uint, + } + + #[allow(missing_debug_implementations)] + pub struct __riscv_mc_d_ext_state { + pub __f: [::c_ulonglong; 32], + pub __fcsr: ::c_uint, + } + + #[allow(missing_debug_implementations)] + #[repr(align(16))] + pub struct __riscv_mc_q_ext_state { + pub __f: [::c_ulonglong; 64], + pub __fcsr: ::c_uint, + pub __glibc_reserved: [::c_uint; 3], + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/riscv32/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/riscv32/mod.rs new file mode 100644 index 000000000..827f85e86 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/riscv32/mod.rs @@ -0,0 +1,775 @@ +//! RISC-V-specific definitions for 32-bit linux-like values + +pub type c_char = u8; +pub type wchar_t = ::c_int; + +s! { + pub struct pthread_attr_t { + __size: [::c_ulong; 7], + } + + pub struct msqid_ds { + pub msg_perm: ::ipc_perm, + pub msg_stime: ::time_t, + pub msg_rtime: ::time_t, + pub msg_ctime: ::time_t, + __msg_cbytes: ::c_ulong, + pub msg_qnum: ::msgqnum_t, + pub msg_qbytes: ::msglen_t, + pub msg_lspid: ::pid_t, + pub msg_lrpid: ::pid_t, + __glibc_reserved4: ::c_ulong, + __glibc_reserved5: ::c_ulong, + } + + pub struct stat { + pub st_dev: ::dev_t, + pub st_ino: ::ino_t, + pub st_mode: ::mode_t, + pub st_nlink: ::nlink_t, + pub st_uid: ::uid_t, + pub st_gid: ::gid_t, + pub st_rdev: ::dev_t, + pub __pad1: ::dev_t, + pub st_size: ::off_t, + pub st_blksize: ::blksize_t, + pub __pad2: ::c_int, + pub st_blocks: ::blkcnt_t, + pub st_atime: ::time_t, + pub st_atime_nsec: ::c_long, + pub st_mtime: ::time_t, + pub st_mtime_nsec: ::c_long, + pub st_ctime: ::time_t, + pub st_ctime_nsec: ::c_long, + __unused: [::c_int; 2usize], + } + + pub struct stat64 { + pub st_dev: ::dev_t, + pub st_ino: ::ino64_t, + pub st_mode: ::mode_t, + pub st_nlink: ::nlink_t, + pub st_uid: ::uid_t, + pub st_gid: ::gid_t, + pub st_rdev: ::dev_t, + pub __pad1: ::dev_t, + pub st_size: ::off64_t, + pub st_blksize: ::blksize_t, + pub __pad2: ::c_int, + pub st_blocks: ::blkcnt64_t, + pub st_atime: ::time_t, + pub st_atime_nsec: ::c_long, + pub st_mtime: ::time_t, + pub st_mtime_nsec: ::c_long, + pub st_ctime: ::time_t, + pub st_ctime_nsec: ::c_long, + __unused: [::c_int; 2], + } + + pub struct statfs { + pub f_type: ::c_long, + pub f_bsize: ::c_long, + pub f_blocks: ::fsblkcnt_t, + pub f_bfree: ::fsblkcnt_t, + pub f_bavail: ::fsblkcnt_t, + pub f_files: ::fsfilcnt_t, + pub f_ffree: ::fsfilcnt_t, + pub f_fsid: ::fsid_t, + pub f_namelen: ::c_long, + pub f_frsize: ::c_long, + pub f_flags: ::c_long, + pub f_spare: [::c_long; 4], + } + + pub struct statfs64 { + pub f_type: ::c_long, + pub f_bsize: ::c_long, + pub f_blocks: ::fsblkcnt64_t, + pub f_bfree: ::fsblkcnt64_t, + pub f_bavail: ::fsblkcnt64_t, + pub f_files: ::fsfilcnt64_t, + pub f_ffree: ::fsfilcnt64_t, + pub f_fsid: ::fsid_t, + pub f_namelen: ::c_long, + pub f_frsize: ::c_long, + pub f_flags: ::c_long, + pub f_spare: [::c_long; 4], + } + + pub struct statvfs { + pub f_bsize: ::c_ulong, + pub f_frsize: ::c_ulong, + pub f_blocks: ::fsblkcnt_t, + pub f_bfree: ::fsblkcnt_t, + pub f_bavail: ::fsblkcnt_t, + pub f_files: ::fsfilcnt_t, + pub f_ffree: ::fsfilcnt_t, + pub f_favail: ::fsfilcnt_t, + pub f_fsid: ::c_ulong, + pub f_flag: ::c_ulong, + pub f_namemax: ::c_ulong, + pub __f_spare: [::c_int; 6], + } + + pub struct statvfs64 { + pub f_bsize: ::c_ulong, + pub f_frsize: ::c_ulong, + pub f_blocks: ::fsblkcnt64_t, + pub f_bfree: ::fsblkcnt64_t, + pub f_bavail: ::fsblkcnt64_t, + pub f_files: ::fsfilcnt64_t, + pub f_ffree: ::fsfilcnt64_t, + pub f_favail: ::fsfilcnt64_t, + pub f_fsid: ::c_ulong, + pub f_flag: ::c_ulong, + pub f_namemax: ::c_ulong, + pub __f_spare: [::c_int; 6], + } + + pub struct siginfo_t { + pub si_signo: ::c_int, + pub si_errno: ::c_int, + pub si_code: ::c_int, + #[doc(hidden)] + #[deprecated( + since="0.2.54", + note="Please leave a comment on \ + https://github.com/rust-lang/libc/pull/1316 if you're using \ + this field" + )] + pub _pad: [::c_int; 29], + _align: [u64; 0], + } + + pub struct stack_t { + pub ss_sp: *mut ::c_void, + pub ss_flags: ::c_int, + pub ss_size: ::size_t, + } + + pub struct sigaction { + pub sa_sigaction: ::sighandler_t, + pub sa_mask: ::sigset_t, + pub sa_flags: ::c_int, + pub sa_restorer: ::Option, + } + + pub struct ipc_perm { + pub __key: ::key_t, + pub uid: ::uid_t, + pub gid: ::gid_t, + pub cuid: ::uid_t, + pub cgid: ::gid_t, + pub mode: ::c_ushort, + __pad1: ::c_ushort, + pub __seq: ::c_ushort, + __pad2: ::c_ushort, + __unused1: ::c_ulong, + __unused2: ::c_ulong, + } + + pub struct shmid_ds { + pub shm_perm: ::ipc_perm, + pub shm_segsz: ::size_t, + pub shm_atime: ::time_t, + pub shm_dtime: ::time_t, + pub shm_ctime: ::time_t, + pub shm_cpid: ::pid_t, + pub shm_lpid: ::pid_t, + pub shm_nattch: ::shmatt_t, + __unused5: ::c_ulong, + __unused6: ::c_ulong, + } + + pub struct flock { + pub l_type: ::c_short, + pub l_whence: ::c_short, + pub l_start: ::off_t, + pub l_len: ::off_t, + pub l_pid: ::pid_t, + } + + pub struct flock64 { + pub l_type: ::c_short, + pub l_whence: ::c_short, + pub l_start: ::off64_t, + pub l_len: ::off64_t, + pub l_pid: ::pid_t, + } +} + +pub const O_LARGEFILE: ::c_int = 0; +pub const VEOF: usize = 4; +pub const RTLD_DEEPBIND: ::c_int = 0x8; +pub const RTLD_GLOBAL: ::c_int = 0x100; +pub const RTLD_NOLOAD: ::c_int = 0x4; +pub const O_APPEND: ::c_int = 1024; +pub const O_CREAT: ::c_int = 64; +pub const O_EXCL: ::c_int = 128; +pub const O_NOCTTY: ::c_int = 256; +pub const O_NONBLOCK: ::c_int = 2048; +pub const O_SYNC: ::c_int = 1052672; +pub const O_RSYNC: ::c_int = 1052672; +pub const O_DSYNC: ::c_int = 4096; +pub const O_FSYNC: ::c_int = 1052672; +pub const MADV_SOFT_OFFLINE: ::c_int = 101; +pub const MAP_GROWSDOWN: ::c_int = 256; +pub const EDEADLK: ::c_int = 35; +pub const ENAMETOOLONG: ::c_int = 36; +pub const ENOLCK: ::c_int = 37; +pub const ENOSYS: ::c_int = 38; +pub const ENOTEMPTY: ::c_int = 39; +pub const ELOOP: ::c_int = 40; +pub const ENOMSG: ::c_int = 42; +pub const EIDRM: ::c_int = 43; +pub const ECHRNG: ::c_int = 44; +pub const EL2NSYNC: ::c_int = 45; +pub const EL3HLT: ::c_int = 46; +pub const EL3RST: ::c_int = 47; +pub const ELNRNG: ::c_int = 48; +pub const EUNATCH: ::c_int = 49; +pub const ENOCSI: ::c_int = 50; +pub const EL2HLT: ::c_int = 51; +pub const EBADE: ::c_int = 52; +pub const EBADR: ::c_int = 53; +pub const EXFULL: ::c_int = 54; +pub const ENOANO: ::c_int = 55; +pub const EBADRQC: ::c_int = 56; +pub const EBADSLT: ::c_int = 57; +pub const EMULTIHOP: ::c_int = 72; +pub const EOVERFLOW: ::c_int = 75; +pub const ENOTUNIQ: ::c_int = 76; +pub const EBADFD: ::c_int = 77; +pub const EBADMSG: ::c_int = 74; +pub const EREMCHG: ::c_int = 78; +pub const ELIBACC: ::c_int = 79; +pub const ELIBBAD: ::c_int = 80; +pub const ELIBSCN: ::c_int = 81; +pub const ELIBMAX: ::c_int = 82; +pub const ELIBEXEC: ::c_int = 83; +pub const EILSEQ: ::c_int = 84; +pub const ERESTART: ::c_int = 85; +pub const ESTRPIPE: ::c_int = 86; +pub const EUSERS: ::c_int = 87; +pub const ENOTSOCK: ::c_int = 88; +pub const EDESTADDRREQ: ::c_int = 89; +pub const EMSGSIZE: ::c_int = 90; +pub const EPROTOTYPE: ::c_int = 91; +pub const ENOPROTOOPT: ::c_int = 92; +pub const EPROTONOSUPPORT: ::c_int = 93; +pub const ESOCKTNOSUPPORT: ::c_int = 94; +pub const EOPNOTSUPP: ::c_int = 95; +pub const EPFNOSUPPORT: ::c_int = 96; +pub const EAFNOSUPPORT: ::c_int = 97; +pub const EADDRINUSE: ::c_int = 98; +pub const EADDRNOTAVAIL: ::c_int = 99; +pub const ENETDOWN: ::c_int = 100; +pub const ENETUNREACH: ::c_int = 101; +pub const ENETRESET: ::c_int = 102; +pub const ECONNABORTED: ::c_int = 103; +pub const ECONNRESET: ::c_int = 104; +pub const ENOBUFS: ::c_int = 105; +pub const EISCONN: ::c_int = 106; +pub const ENOTCONN: ::c_int = 107; +pub const ESHUTDOWN: ::c_int = 108; +pub const ETOOMANYREFS: ::c_int = 109; +pub const ETIMEDOUT: ::c_int = 110; +pub const ECONNREFUSED: ::c_int = 111; +pub const EHOSTDOWN: ::c_int = 112; +pub const EHOSTUNREACH: ::c_int = 113; +pub const EALREADY: ::c_int = 114; +pub const EINPROGRESS: ::c_int = 115; +pub const ESTALE: ::c_int = 116; +pub const EDQUOT: ::c_int = 122; +pub const ENOMEDIUM: ::c_int = 123; +pub const EMEDIUMTYPE: ::c_int = 124; +pub const ECANCELED: ::c_int = 125; +pub const ENOKEY: ::c_int = 126; +pub const EKEYEXPIRED: ::c_int = 127; +pub const EKEYREVOKED: ::c_int = 128; +pub const EKEYREJECTED: ::c_int = 129; +pub const EOWNERDEAD: ::c_int = 130; +pub const ENOTRECOVERABLE: ::c_int = 131; +pub const EHWPOISON: ::c_int = 133; +pub const ERFKILL: ::c_int = 132; + +pub const SOCK_STREAM: ::c_int = 1; +pub const SOCK_DGRAM: ::c_int = 2; +pub const SA_SIGINFO: ::c_int = 4; +pub const SA_NOCLDWAIT: ::c_int = 2; +pub const SIGTTIN: ::c_int = 21; +pub const SIGTTOU: ::c_int = 22; +pub const SIGXCPU: ::c_int = 24; +pub const SIGXFSZ: ::c_int = 25; +pub const SIGVTALRM: ::c_int = 26; +pub const SIGPROF: ::c_int = 27; +pub const SIGWINCH: ::c_int = 28; +pub const SIGCHLD: ::c_int = 17; +pub const SIGBUS: ::c_int = 7; +pub const SIGUSR1: ::c_int = 10; +pub const SIGUSR2: ::c_int = 12; +pub const SIGCONT: ::c_int = 18; +pub const SIGSTOP: ::c_int = 19; +pub const SIGTSTP: ::c_int = 20; +pub const SIGURG: ::c_int = 23; +pub const SIGIO: ::c_int = 29; +pub const SIGSYS: ::c_int = 31; +pub const SIGSTKFLT: ::c_int = 16; +pub const SIGPOLL: ::c_int = 29; +pub const SIGPWR: ::c_int = 30; +pub const SIG_SETMASK: ::c_int = 2; +pub const SIG_BLOCK: ::c_int = 0; +pub const SIG_UNBLOCK: ::c_int = 1; +pub const POLLWRNORM: ::c_short = 256; +pub const POLLWRBAND: ::c_short = 512; +pub const O_ASYNC: ::c_int = 8192; +pub const O_NDELAY: ::c_int = 2048; +pub const EFD_NONBLOCK: ::c_int = 2048; +pub const F_GETLK: ::c_int = 5; +pub const F_GETOWN: ::c_int = 9; +pub const F_SETOWN: ::c_int = 8; +pub const SFD_NONBLOCK: ::c_int = 2048; +pub const TCSANOW: ::c_int = 0; +pub const TCSADRAIN: ::c_int = 1; +pub const TCSAFLUSH: ::c_int = 2; + +pub const __SIZEOF_PTHREAD_CONDATTR_T: usize = 4; +pub const __SIZEOF_PTHREAD_MUTEXATTR_T: usize = 4; +pub const O_DIRECT: ::c_int = 16384; +pub const O_DIRECTORY: ::c_int = 65536; +pub const O_NOFOLLOW: ::c_int = 131072; +pub const MAP_HUGETLB: ::c_int = 262144; +pub const MAP_LOCKED: ::c_int = 8192; +pub const MAP_NORESERVE: ::c_int = 16384; +pub const MAP_ANON: ::c_int = 32; +pub const MAP_ANONYMOUS: ::c_int = 32; +pub const MAP_DENYWRITE: ::c_int = 2048; +pub const MAP_EXECUTABLE: ::c_int = 4096; +pub const MAP_POPULATE: ::c_int = 32768; +pub const MAP_NONBLOCK: ::c_int = 65536; +pub const MAP_STACK: ::c_int = 131072; +pub const MAP_SYNC: ::c_int = 0x080000; +pub const EDEADLOCK: ::c_int = 35; +pub const EUCLEAN: ::c_int = 117; +pub const ENOTNAM: ::c_int = 118; +pub const ENAVAIL: ::c_int = 119; +pub const EISNAM: ::c_int = 120; +pub const EREMOTEIO: ::c_int = 121; +pub const MCL_CURRENT: ::c_int = 1; +pub const MCL_FUTURE: ::c_int = 2; +pub const SIGSTKSZ: ::size_t = 8192; +pub const MINSIGSTKSZ: ::size_t = 2048; +pub const CBAUD: ::tcflag_t = 4111; +pub const TAB1: ::tcflag_t = 2048; +pub const TAB2: ::tcflag_t = 4096; +pub const TAB3: ::tcflag_t = 6144; +pub const CR1: ::tcflag_t = 512; +pub const CR2: ::tcflag_t = 1024; +pub const CR3: ::tcflag_t = 1536; +pub const FF1: ::tcflag_t = 32768; +pub const BS1: ::tcflag_t = 8192; +pub const VT1: ::tcflag_t = 16384; +pub const VWERASE: usize = 14; +pub const VREPRINT: usize = 12; +pub const VSUSP: usize = 10; +pub const VSTART: usize = 8; +pub const VSTOP: usize = 9; +pub const VDISCARD: usize = 13; +pub const VTIME: usize = 5; +pub const IXON: ::tcflag_t = 1024; +pub const IXOFF: ::tcflag_t = 4096; +pub const ONLCR: ::tcflag_t = 4; +pub const CSIZE: ::tcflag_t = 48; +pub const CS6: ::tcflag_t = 16; +pub const CS7: ::tcflag_t = 32; +pub const CS8: ::tcflag_t = 48; +pub const CSTOPB: ::tcflag_t = 64; +pub const CREAD: ::tcflag_t = 128; +pub const PARENB: ::tcflag_t = 256; +pub const PARODD: ::tcflag_t = 512; +pub const HUPCL: ::tcflag_t = 1024; +pub const CLOCAL: ::tcflag_t = 2048; +pub const ECHOKE: ::tcflag_t = 2048; +pub const ECHOE: ::tcflag_t = 16; +pub const ECHOK: ::tcflag_t = 32; +pub const ECHONL: ::tcflag_t = 64; +pub const ECHOPRT: ::tcflag_t = 1024; +pub const ECHOCTL: ::tcflag_t = 512; +pub const ISIG: ::tcflag_t = 1; +pub const ICANON: ::tcflag_t = 2; +pub const PENDIN: ::tcflag_t = 16384; +pub const NOFLSH: ::tcflag_t = 128; +pub const CIBAUD: ::tcflag_t = 269418496; +pub const CBAUDEX: ::tcflag_t = 4096; +pub const VSWTC: usize = 7; +pub const OLCUC: ::tcflag_t = 2; +pub const NLDLY: ::tcflag_t = 256; +pub const CRDLY: ::tcflag_t = 1536; +pub const TABDLY: ::tcflag_t = 6144; +pub const BSDLY: ::tcflag_t = 8192; +pub const FFDLY: ::tcflag_t = 32768; +pub const VTDLY: ::tcflag_t = 16384; +pub const XTABS: ::tcflag_t = 6144; +pub const B0: ::speed_t = 0; +pub const B50: ::speed_t = 1; +pub const B75: ::speed_t = 2; +pub const B110: ::speed_t = 3; +pub const B134: ::speed_t = 4; +pub const B150: ::speed_t = 5; +pub const B200: ::speed_t = 6; +pub const B300: ::speed_t = 7; +pub const B600: ::speed_t = 8; +pub const B1200: ::speed_t = 9; +pub const B1800: ::speed_t = 10; +pub const B2400: ::speed_t = 11; +pub const B4800: ::speed_t = 12; +pub const B9600: ::speed_t = 13; +pub const B19200: ::speed_t = 14; +pub const B38400: ::speed_t = 15; +pub const EXTA: ::speed_t = 14; +pub const EXTB: ::speed_t = 15; +pub const B57600: ::speed_t = 4097; +pub const B115200: ::speed_t = 4098; +pub const B230400: ::speed_t = 4099; +pub const B460800: ::speed_t = 4100; +pub const B500000: ::speed_t = 4101; +pub const B576000: ::speed_t = 4102; +pub const B921600: ::speed_t = 4103; +pub const B1000000: ::speed_t = 4104; +pub const B1152000: ::speed_t = 4105; +pub const B1500000: ::speed_t = 4106; +pub const B2000000: ::speed_t = 4107; +pub const B2500000: ::speed_t = 4108; +pub const B3000000: ::speed_t = 4109; +pub const B3500000: ::speed_t = 4110; +pub const B4000000: ::speed_t = 4111; +pub const VEOL: usize = 11; +pub const VEOL2: usize = 16; +pub const VMIN: usize = 6; +pub const IEXTEN: ::tcflag_t = 32768; +pub const TOSTOP: ::tcflag_t = 256; +pub const FLUSHO: ::tcflag_t = 4096; +pub const EXTPROC: ::tcflag_t = 65536; +pub const __SIZEOF_PTHREAD_MUTEX_T: usize = 40; +pub const __SIZEOF_PTHREAD_RWLOCK_T: usize = 56; +pub const NGREG: usize = 32; +pub const REG_PC: usize = 0; +pub const REG_RA: usize = 1; +pub const REG_SP: usize = 2; +pub const REG_TP: usize = 4; +pub const REG_S0: usize = 8; +pub const REG_S1: usize = 9; +pub const REG_A0: usize = 10; +pub const REG_S2: usize = 18; +pub const REG_NARGS: usize = 8; + +pub const SYS_read: ::c_long = 63; +pub const SYS_write: ::c_long = 64; +pub const SYS_close: ::c_long = 57; +pub const SYS_fstat: ::c_long = 80; +pub const SYS_lseek: ::c_long = 62; +pub const SYS_mmap: ::c_long = 222; +pub const SYS_mprotect: ::c_long = 226; +pub const SYS_munmap: ::c_long = 215; +pub const SYS_brk: ::c_long = 214; +pub const SYS_rt_sigaction: ::c_long = 134; +pub const SYS_rt_sigprocmask: ::c_long = 135; +pub const SYS_rt_sigreturn: ::c_long = 139; +pub const SYS_ioctl: ::c_long = 29; +pub const SYS_pread64: ::c_long = 67; +pub const SYS_pwrite64: ::c_long = 68; +pub const SYS_readv: ::c_long = 65; +pub const SYS_writev: ::c_long = 66; +pub const SYS_sched_yield: ::c_long = 124; +pub const SYS_mremap: ::c_long = 216; +pub const SYS_msync: ::c_long = 227; +pub const SYS_mincore: ::c_long = 232; +pub const SYS_madvise: ::c_long = 233; +pub const SYS_shmget: ::c_long = 194; +pub const SYS_shmat: ::c_long = 196; +pub const SYS_shmctl: ::c_long = 195; +pub const SYS_dup: ::c_long = 23; +pub const SYS_nanosleep: ::c_long = 101; +pub const SYS_getitimer: ::c_long = 102; +pub const SYS_setitimer: ::c_long = 103; +pub const SYS_getpid: ::c_long = 172; +pub const SYS_sendfile: ::c_long = 71; +pub const SYS_socket: ::c_long = 198; +pub const SYS_connect: ::c_long = 203; +pub const SYS_accept: ::c_long = 202; +pub const SYS_sendto: ::c_long = 206; +pub const SYS_recvfrom: ::c_long = 207; +pub const SYS_sendmsg: ::c_long = 211; +pub const SYS_recvmsg: ::c_long = 212; +pub const SYS_shutdown: ::c_long = 210; +pub const SYS_bind: ::c_long = 200; +pub const SYS_listen: ::c_long = 201; +pub const SYS_getsockname: ::c_long = 204; +pub const SYS_getpeername: ::c_long = 205; +pub const SYS_socketpair: ::c_long = 199; +pub const SYS_setsockopt: ::c_long = 208; +pub const SYS_getsockopt: ::c_long = 209; +pub const SYS_clone: ::c_long = 220; +pub const SYS_execve: ::c_long = 221; +pub const SYS_exit: ::c_long = 93; +pub const SYS_wait4: ::c_long = 260; +pub const SYS_kill: ::c_long = 129; +pub const SYS_uname: ::c_long = 160; +pub const SYS_semget: ::c_long = 190; +pub const SYS_semop: ::c_long = 193; +pub const SYS_semctl: ::c_long = 191; +pub const SYS_shmdt: ::c_long = 197; +pub const SYS_msgget: ::c_long = 186; +pub const SYS_msgsnd: ::c_long = 189; +pub const SYS_msgrcv: ::c_long = 188; +pub const SYS_msgctl: ::c_long = 187; +pub const SYS_fcntl: ::c_long = 25; +pub const SYS_flock: ::c_long = 32; +pub const SYS_fsync: ::c_long = 82; +pub const SYS_fdatasync: ::c_long = 83; +pub const SYS_truncate: ::c_long = 45; +pub const SYS_ftruncate: ::c_long = 46; +pub const SYS_getcwd: ::c_long = 17; +pub const SYS_chdir: ::c_long = 49; +pub const SYS_fchdir: ::c_long = 50; +pub const SYS_fchmod: ::c_long = 52; +pub const SYS_fchown: ::c_long = 55; +pub const SYS_umask: ::c_long = 166; +pub const SYS_gettimeofday: ::c_long = 169; +pub const SYS_getrlimit: ::c_long = 163; +pub const SYS_getrusage: ::c_long = 165; +pub const SYS_sysinfo: ::c_long = 179; +pub const SYS_times: ::c_long = 153; +pub const SYS_ptrace: ::c_long = 117; +pub const SYS_getuid: ::c_long = 174; +pub const SYS_syslog: ::c_long = 116; +pub const SYS_getgid: ::c_long = 176; +pub const SYS_setuid: ::c_long = 146; +pub const SYS_setgid: ::c_long = 144; +pub const SYS_geteuid: ::c_long = 175; +pub const SYS_getegid: ::c_long = 177; +pub const SYS_setpgid: ::c_long = 154; +pub const SYS_getppid: ::c_long = 173; +pub const SYS_setsid: ::c_long = 157; +pub const SYS_setreuid: ::c_long = 145; +pub const SYS_setregid: ::c_long = 143; +pub const SYS_getgroups: ::c_long = 158; +pub const SYS_setgroups: ::c_long = 159; +pub const SYS_setresuid: ::c_long = 147; +pub const SYS_getresuid: ::c_long = 148; +pub const SYS_setresgid: ::c_long = 149; +pub const SYS_getresgid: ::c_long = 150; +pub const SYS_getpgid: ::c_long = 155; +pub const SYS_setfsuid: ::c_long = 151; +pub const SYS_setfsgid: ::c_long = 152; +pub const SYS_getsid: ::c_long = 156; +pub const SYS_capget: ::c_long = 90; +pub const SYS_capset: ::c_long = 91; +pub const SYS_rt_sigpending: ::c_long = 136; +pub const SYS_rt_sigtimedwait: ::c_long = 137; +pub const SYS_rt_sigqueueinfo: ::c_long = 138; +pub const SYS_rt_sigsuspend: ::c_long = 133; +pub const SYS_sigaltstack: ::c_long = 132; +pub const SYS_personality: ::c_long = 92; +pub const SYS_statfs: ::c_long = 43; +pub const SYS_fstatfs: ::c_long = 44; +pub const SYS_getpriority: ::c_long = 141; +pub const SYS_setpriority: ::c_long = 140; +pub const SYS_sched_setparam: ::c_long = 118; +pub const SYS_sched_getparam: ::c_long = 121; +pub const SYS_sched_setscheduler: ::c_long = 119; +pub const SYS_sched_getscheduler: ::c_long = 120; +pub const SYS_sched_get_priority_max: ::c_long = 125; +pub const SYS_sched_get_priority_min: ::c_long = 126; +pub const SYS_sched_rr_get_interval: ::c_long = 127; +pub const SYS_mlock: ::c_long = 228; +pub const SYS_munlock: ::c_long = 229; +pub const SYS_mlockall: ::c_long = 230; +pub const SYS_munlockall: ::c_long = 231; +pub const SYS_vhangup: ::c_long = 58; +pub const SYS_pivot_root: ::c_long = 41; +pub const SYS_prctl: ::c_long = 167; +pub const SYS_adjtimex: ::c_long = 171; +pub const SYS_setrlimit: ::c_long = 164; +pub const SYS_chroot: ::c_long = 51; +pub const SYS_sync: ::c_long = 81; +pub const SYS_acct: ::c_long = 89; +pub const SYS_settimeofday: ::c_long = 170; +pub const SYS_mount: ::c_long = 40; +pub const SYS_umount2: ::c_long = 39; +pub const SYS_swapon: ::c_long = 224; +pub const SYS_swapoff: ::c_long = 225; +pub const SYS_reboot: ::c_long = 142; +pub const SYS_sethostname: ::c_long = 161; +pub const SYS_setdomainname: ::c_long = 162; +pub const SYS_init_module: ::c_long = 105; +pub const SYS_delete_module: ::c_long = 106; +pub const SYS_quotactl: ::c_long = 60; +pub const SYS_nfsservctl: ::c_long = 42; +pub const SYS_gettid: ::c_long = 178; +pub const SYS_readahead: ::c_long = 213; +pub const SYS_setxattr: ::c_long = 5; +pub const SYS_lsetxattr: ::c_long = 6; +pub const SYS_fsetxattr: ::c_long = 7; +pub const SYS_getxattr: ::c_long = 8; +pub const SYS_lgetxattr: ::c_long = 9; +pub const SYS_fgetxattr: ::c_long = 10; +pub const SYS_listxattr: ::c_long = 11; +pub const SYS_llistxattr: ::c_long = 12; +pub const SYS_flistxattr: ::c_long = 13; +pub const SYS_removexattr: ::c_long = 14; +pub const SYS_lremovexattr: ::c_long = 15; +pub const SYS_fremovexattr: ::c_long = 16; +pub const SYS_tkill: ::c_long = 130; +pub const SYS_futex: ::c_long = 98; +pub const SYS_sched_setaffinity: ::c_long = 122; +pub const SYS_sched_getaffinity: ::c_long = 123; +pub const SYS_io_setup: ::c_long = 0; +pub const SYS_io_destroy: ::c_long = 1; +pub const SYS_io_getevents: ::c_long = 4; +pub const SYS_io_submit: ::c_long = 2; +pub const SYS_io_cancel: ::c_long = 3; +pub const SYS_lookup_dcookie: ::c_long = 18; +pub const SYS_remap_file_pages: ::c_long = 234; +pub const SYS_getdents64: ::c_long = 61; +pub const SYS_set_tid_address: ::c_long = 96; +pub const SYS_restart_syscall: ::c_long = 128; +pub const SYS_semtimedop: ::c_long = 192; +pub const SYS_fadvise64: ::c_long = 223; +pub const SYS_timer_create: ::c_long = 107; +pub const SYS_timer_settime: ::c_long = 110; +pub const SYS_timer_gettime: ::c_long = 108; +pub const SYS_timer_getoverrun: ::c_long = 109; +pub const SYS_timer_delete: ::c_long = 111; +pub const SYS_clock_settime: ::c_long = 112; +pub const SYS_clock_gettime: ::c_long = 113; +pub const SYS_clock_getres: ::c_long = 114; +pub const SYS_clock_nanosleep: ::c_long = 115; +pub const SYS_exit_group: ::c_long = 94; +pub const SYS_epoll_ctl: ::c_long = 21; +pub const SYS_tgkill: ::c_long = 131; +pub const SYS_mbind: ::c_long = 235; +pub const SYS_set_mempolicy: ::c_long = 237; +pub const SYS_get_mempolicy: ::c_long = 236; +pub const SYS_mq_open: ::c_long = 180; +pub const SYS_mq_unlink: ::c_long = 181; +pub const SYS_mq_timedsend: ::c_long = 182; +pub const SYS_mq_timedreceive: ::c_long = 183; +pub const SYS_mq_notify: ::c_long = 184; +pub const SYS_mq_getsetattr: ::c_long = 185; +pub const SYS_kexec_load: ::c_long = 104; +pub const SYS_waitid: ::c_long = 95; +pub const SYS_add_key: ::c_long = 217; +pub const SYS_request_key: ::c_long = 218; +pub const SYS_keyctl: ::c_long = 219; +pub const SYS_ioprio_set: ::c_long = 30; +pub const SYS_ioprio_get: ::c_long = 31; +pub const SYS_inotify_add_watch: ::c_long = 27; +pub const SYS_inotify_rm_watch: ::c_long = 28; +pub const SYS_migrate_pages: ::c_long = 238; +pub const SYS_openat: ::c_long = 56; +pub const SYS_mkdirat: ::c_long = 34; +pub const SYS_mknodat: ::c_long = 33; +pub const SYS_fchownat: ::c_long = 54; +pub const SYS_newfstatat: ::c_long = 79; +pub const SYS_unlinkat: ::c_long = 35; +pub const SYS_linkat: ::c_long = 37; +pub const SYS_symlinkat: ::c_long = 36; +pub const SYS_readlinkat: ::c_long = 78; +pub const SYS_fchmodat: ::c_long = 53; +pub const SYS_faccessat: ::c_long = 48; +pub const SYS_pselect6: ::c_long = 72; +pub const SYS_ppoll: ::c_long = 73; +pub const SYS_unshare: ::c_long = 97; +pub const SYS_set_robust_list: ::c_long = 99; +pub const SYS_get_robust_list: ::c_long = 100; +pub const SYS_splice: ::c_long = 76; +pub const SYS_tee: ::c_long = 77; +pub const SYS_sync_file_range: ::c_long = 84; +pub const SYS_vmsplice: ::c_long = 75; +pub const SYS_move_pages: ::c_long = 239; +pub const SYS_utimensat: ::c_long = 88; +pub const SYS_epoll_pwait: ::c_long = 22; +pub const SYS_timerfd_create: ::c_long = 85; +pub const SYS_fallocate: ::c_long = 47; +pub const SYS_timerfd_settime: ::c_long = 86; +pub const SYS_timerfd_gettime: ::c_long = 87; +pub const SYS_accept4: ::c_long = 242; +pub const SYS_signalfd4: ::c_long = 74; +pub const SYS_eventfd2: ::c_long = 19; +pub const SYS_epoll_create1: ::c_long = 20; +pub const SYS_dup3: ::c_long = 24; +pub const SYS_pipe2: ::c_long = 59; +pub const SYS_inotify_init1: ::c_long = 26; +pub const SYS_preadv: ::c_long = 69; +pub const SYS_pwritev: ::c_long = 70; +pub const SYS_rt_tgsigqueueinfo: ::c_long = 240; +pub const SYS_perf_event_open: ::c_long = 241; +pub const SYS_recvmmsg: ::c_long = 243; +pub const SYS_fanotify_init: ::c_long = 262; +pub const SYS_fanotify_mark: ::c_long = 263; +pub const SYS_prlimit64: ::c_long = 261; +pub const SYS_name_to_handle_at: ::c_long = 264; +pub const SYS_open_by_handle_at: ::c_long = 265; +pub const SYS_clock_adjtime: ::c_long = 266; +pub const SYS_syncfs: ::c_long = 267; +pub const SYS_sendmmsg: ::c_long = 269; +pub const SYS_setns: ::c_long = 268; +pub const SYS_getcpu: ::c_long = 168; +pub const SYS_process_vm_readv: ::c_long = 270; +pub const SYS_process_vm_writev: ::c_long = 271; +pub const SYS_kcmp: ::c_long = 272; +pub const SYS_finit_module: ::c_long = 273; +pub const SYS_sched_setattr: ::c_long = 274; +pub const SYS_sched_getattr: ::c_long = 275; +pub const SYS_renameat2: ::c_long = 276; +pub const SYS_seccomp: ::c_long = 277; +pub const SYS_getrandom: ::c_long = 278; +pub const SYS_memfd_create: ::c_long = 279; +pub const SYS_bpf: ::c_long = 280; +pub const SYS_execveat: ::c_long = 281; +pub const SYS_userfaultfd: ::c_long = 282; +pub const SYS_membarrier: ::c_long = 283; +pub const SYS_mlock2: ::c_long = 284; +pub const SYS_copy_file_range: ::c_long = 285; +pub const SYS_preadv2: ::c_long = 286; +pub const SYS_pwritev2: ::c_long = 287; +pub const SYS_pkey_mprotect: ::c_long = 288; +pub const SYS_pkey_alloc: ::c_long = 289; +pub const SYS_pkey_free: ::c_long = 290; +pub const SYS_statx: ::c_long = 291; +pub const SYS_rseq: ::c_long = 293; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; +pub const SYS_quotactl_fd: ::c_long = 443; +pub const SYS_landlock_create_ruleset: ::c_long = 444; +pub const SYS_landlock_add_rule: ::c_long = 445; +pub const SYS_landlock_restrict_self: ::c_long = 446; +pub const SYS_memfd_secret: ::c_long = 447; +pub const SYS_process_mrelease: ::c_long = 448; +pub const SYS_futex_waitv: ::c_long = 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 450; + +cfg_if! { + if #[cfg(libc_align)] { + mod align; + pub use self::align::*; + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/sparc/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/sparc/mod.rs index aaa800870..57ad9fe8e 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/sparc/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/sparc/mod.rs @@ -1,6 +1,6 @@ //! SPARC-specific definitions for 32-bit linux-like values -pub type c_char = u8; +pub type c_char = i8; pub type wchar_t = i32; s! { @@ -191,37 +191,13 @@ s! { __glibc_reserved1: ::c_ulong, __glibc_reserved2: ::c_ulong, } - - pub struct termios2 { - pub c_iflag: ::tcflag_t, - pub c_oflag: ::tcflag_t, - pub c_cflag: ::tcflag_t, - pub c_lflag: ::tcflag_t, - pub c_line: ::cc_t, - pub c_cc: [::cc_t; 19], - pub c_ispeed: ::speed_t, - pub c_ospeed: ::speed_t, - } } -pub const POSIX_FADV_DONTNEED: ::c_int = 4; -pub const POSIX_FADV_NOREUSE: ::c_int = 5; - -pub const RLIM_INFINITY: ::rlim_t = !0; pub const VEOF: usize = 4; pub const RTLD_DEEPBIND: ::c_int = 0x8; pub const RTLD_GLOBAL: ::c_int = 0x100; pub const RTLD_NOLOAD: ::c_int = 0x4; -pub const TIOCGSOFTCAR: ::c_ulong = 0x40047464; -pub const TIOCSSOFTCAR: ::c_ulong = 0x80047465; - -pub const RLIMIT_RSS: ::__rlimit_resource_t = 5; -pub const RLIMIT_AS: ::__rlimit_resource_t = 9; -pub const RLIMIT_MEMLOCK: ::__rlimit_resource_t = 8; -pub const RLIMIT_NOFILE: ::__rlimit_resource_t = 6; -pub const RLIMIT_NPROC: ::__rlimit_resource_t = 7; - pub const O_APPEND: ::c_int = 0x8; pub const O_CREAT: ::c_int = 0x200; pub const O_EXCL: ::c_int = 0x800; @@ -231,9 +207,6 @@ pub const O_SYNC: ::c_int = 0x802000; pub const O_RSYNC: ::c_int = 0x802000; pub const O_DSYNC: ::c_int = 0x2000; pub const O_FSYNC: ::c_int = 0x802000; -pub const O_NOATIME: ::c_int = 0x200000; -pub const O_PATH: ::c_int = 0x1000000; -pub const O_TMPFILE: ::c_int = 0x2000000 | O_DIRECTORY; pub const MADV_SOFT_OFFLINE: ::c_int = 101; pub const MAP_GROWSDOWN: ::c_int = 0x0200; @@ -245,7 +218,7 @@ pub const MAP_POPULATE: ::c_int = 0x08000; pub const MAP_NONBLOCK: ::c_int = 0x010000; pub const MAP_STACK: ::c_int = 0x020000; pub const MAP_HUGETLB: ::c_int = 0x040000; -pub const MAP_SYNC : ::c_int = 0x080000; +pub const MAP_SYNC: ::c_int = 0x080000; pub const EDEADLK: ::c_int = 78; pub const ENAMETOOLONG: ::c_int = 63; @@ -326,35 +299,9 @@ pub const ENOTRECOVERABLE: ::c_int = 133; pub const EHWPOISON: ::c_int = 135; pub const ERFKILL: ::c_int = 134; -pub const SOL_SOCKET: ::c_int = 0xffff; - -pub const SO_PASSCRED: ::c_int = 2; -pub const SO_REUSEADDR: ::c_int = 4; -pub const SO_BINDTODEVICE: ::c_int = 0x000d; -pub const SO_TIMESTAMP: ::c_int = 0x001d; -pub const SO_MARK: ::c_int = 0x0022; -pub const SO_RXQ_OVFL: ::c_int = 0x0024; -pub const SO_PEEK_OFF: ::c_int = 0x0026; -pub const SO_BUSY_POLL: ::c_int = 0x0030; -pub const SO_TYPE: ::c_int = 0x1008; -pub const SO_ERROR: ::c_int = 0x1007; -pub const SO_DONTROUTE: ::c_int = 16; -pub const SO_BROADCAST: ::c_int = 32; -pub const SO_SNDBUF: ::c_int = 0x1001; -pub const SO_RCVBUF: ::c_int = 0x1002; -pub const SO_SNDBUFFORCE: ::c_int = 0x100a; -pub const SO_RCVBUFFORCE: ::c_int = 0x100b; -pub const SO_DOMAIN: ::c_int = 0x1029; -pub const SO_KEEPALIVE: ::c_int = 8; -pub const SO_OOBINLINE: ::c_int = 0x100; -pub const SO_LINGER: ::c_int = 128; -pub const SO_REUSEPORT: ::c_int = 0x200; -pub const SO_ACCEPTCONN: ::c_int = 0x8000; - pub const SOCK_STREAM: ::c_int = 1; pub const SOCK_DGRAM: ::c_int = 2; -pub const SA_ONSTACK: ::c_int = 1; pub const SA_SIGINFO: ::c_int = 0x200; pub const SA_NOCLDWAIT: ::c_int = 0x100; @@ -387,22 +334,11 @@ pub const POLLWRBAND: ::c_short = 0x100; pub const O_ASYNC: ::c_int = 0x40; pub const O_NDELAY: ::c_int = 0x4004; -pub const PTRACE_DETACH: ::c_uint = 11; - pub const EFD_NONBLOCK: ::c_int = 0x4000; pub const F_GETLK: ::c_int = 7; pub const F_GETOWN: ::c_int = 5; pub const F_SETOWN: ::c_int = 6; -pub const F_SETLK: ::c_int = 8; -pub const F_SETLKW: ::c_int = 9; -pub const F_OFD_GETLK: ::c_int = 36; -pub const F_OFD_SETLK: ::c_int = 37; -pub const F_OFD_SETLKW: ::c_int = 38; - -pub const F_RDLCK: ::c_int = 1; -pub const F_WRLCK: ::c_int = 2; -pub const F_UNLCK: ::c_int = 3; pub const SFD_NONBLOCK: ::c_int = 0x4000; @@ -410,56 +346,6 @@ pub const TCSANOW: ::c_int = 0; pub const TCSADRAIN: ::c_int = 1; pub const TCSAFLUSH: ::c_int = 2; -pub const TIOCLINUX: ::c_ulong = 0x541C; -pub const TIOCGSERIAL: ::c_ulong = 0x541E; -pub const TIOCEXCL: ::c_ulong = 0x2000740d; -pub const TIOCNXCL: ::c_ulong = 0x2000740e; -pub const TIOCSCTTY: ::c_ulong = 0x20007484; -pub const TIOCSTI: ::c_ulong = 0x80017472; -pub const TIOCMGET: ::c_ulong = 0x4004746a; -pub const TIOCMBIS: ::c_ulong = 0x8004746c; -pub const TIOCMBIC: ::c_ulong = 0x8004746b; -pub const TIOCMSET: ::c_ulong = 0x8004746d; -pub const TIOCCONS: ::c_ulong = 0x20007424; - -pub const TIOCM_ST: ::c_int = 0x008; -pub const TIOCM_SR: ::c_int = 0x010; -pub const TIOCM_CTS: ::c_int = 0x020; -pub const TIOCM_CAR: ::c_int = 0x040; -pub const TIOCM_RNG: ::c_int = 0x080; -pub const TIOCM_DSR: ::c_int = 0x100; - -pub const SFD_CLOEXEC: ::c_int = 0x400000; - -pub const NCCS: usize = 17; -pub const O_TRUNC: ::c_int = 0x400; - -pub const O_CLOEXEC: ::c_int = 0x400000; - -pub const EBFONT: ::c_int = 109; -pub const ENOSTR: ::c_int = 72; -pub const ENODATA: ::c_int = 111; -pub const ETIME: ::c_int = 73; -pub const ENOSR: ::c_int = 74; -pub const ENONET: ::c_int = 80; -pub const ENOPKG: ::c_int = 113; -pub const EREMOTE: ::c_int = 71; -pub const ENOLINK: ::c_int = 82; -pub const EADV: ::c_int = 83; -pub const ESRMNT: ::c_int = 84; -pub const ECOMM: ::c_int = 85; -pub const EPROTO: ::c_int = 86; -pub const EDOTDOT: ::c_int = 88; - -pub const SA_NODEFER: ::c_int = 0x20; -pub const SA_RESETHAND: ::c_int = 0x4; -pub const SA_RESTART: ::c_int = 0x2; -pub const SA_NOCLDSTOP: ::c_int = 0x00000008; - -pub const EPOLL_CLOEXEC: ::c_int = 0x400000; - -pub const EFD_CLOEXEC: ::c_int = 0x400000; - pub const O_DIRECTORY: ::c_int = 0o200000; pub const O_NOFOLLOW: ::c_int = 0o400000; pub const O_LARGEFILE: ::c_int = 0x40000; @@ -475,16 +361,6 @@ pub const ENAVAIL: ::c_int = 119; pub const EISNAM: ::c_int = 120; pub const EREMOTEIO: ::c_int = 121; -pub const SO_PEERCRED: ::c_int = 0x40; -pub const SO_RCVLOWAT: ::c_int = 0x800; -pub const SO_SNDLOWAT: ::c_int = 0x1000; -pub const SO_RCVTIMEO: ::c_int = 0x2000; -pub const SO_SNDTIMEO: ::c_int = 0x4000; - -pub const FIOCLEX: ::c_ulong = 0x20006601; -pub const FIONCLEX: ::c_ulong = 0x20006602; -pub const FIONBIO: ::c_ulong = 0x8004667e; - pub const MCL_CURRENT: ::c_int = 0x2000; pub const MCL_FUTURE: ::c_int = 0x4000; @@ -560,7 +436,6 @@ pub const B19200: ::speed_t = 0o000016; pub const B38400: ::speed_t = 0o000017; pub const EXTA: ::speed_t = B19200; pub const EXTB: ::speed_t = B38400; -pub const BOTHER: ::speed_t = 0x1000; pub const B57600: ::speed_t = 0x1001; pub const B115200: ::speed_t = 0x1002; pub const B230400: ::speed_t = 0x1003; @@ -584,24 +459,6 @@ pub const IEXTEN: ::tcflag_t = 0x8000; pub const TOSTOP: ::tcflag_t = 0x100; pub const FLUSHO: ::tcflag_t = 0x1000; pub const EXTPROC: ::tcflag_t = 0x10000; -pub const TCGETS: ::c_ulong = 0x40245408; -pub const TCSETS: ::c_ulong = 0x80245409; -pub const TCSETSW: ::c_ulong = 0x8024540a; -pub const TCSETSF: ::c_ulong = 0x8024540b; -pub const TCGETA: ::c_ulong = 0x40125401; -pub const TCSETA: ::c_ulong = 0x80125402; -pub const TCSETAW: ::c_ulong = 0x80125403; -pub const TCSETAF: ::c_ulong = 0x80125404; -pub const TCSBRK: ::c_ulong = 0x20005405; -pub const TCXONC: ::c_ulong = 0x20005406; -pub const TCFLSH: ::c_ulong = 0x20005407; -pub const TIOCINQ: ::c_ulong = 0x4004667f; -pub const TIOCGPGRP: ::c_ulong = 0x40047483; -pub const TIOCSPGRP: ::c_ulong = 0x80047482; -pub const TIOCOUTQ: ::c_ulong = 0x40047473; -pub const TIOCGWINSZ: ::c_ulong = 0x40087468; -pub const TIOCSWINSZ: ::c_ulong = 0x80087467; -pub const FIONREAD: ::c_ulong = 0x4004667f; pub const SYS_restart_syscall: ::c_long = 0; pub const SYS_exit: ::c_long = 1; @@ -961,18 +818,35 @@ pub const SYS_copy_file_range: ::c_long = 357; pub const SYS_preadv2: ::c_long = 358; pub const SYS_pwritev2: ::c_long = 359; pub const SYS_statx: ::c_long = 360; - -#[link(name = "util")] -extern "C" { - pub fn sysctl( - name: *mut ::c_int, - namelen: ::c_int, - oldp: *mut ::c_void, - oldlenp: *mut ::size_t, - newp: *mut ::c_void, - newlen: ::size_t, - ) -> ::c_int; -} +pub const SYS_rseq: ::c_long = 365; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +// Reserved in the kernel, but not actually implemented yet +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; +pub const SYS_quotactl_fd: ::c_long = 443; +pub const SYS_landlock_create_ruleset: ::c_long = 444; +pub const SYS_landlock_add_rule: ::c_long = 445; +pub const SYS_landlock_restrict_self: ::c_long = 446; +pub const SYS_memfd_secret: ::c_long = 447; +pub const SYS_process_mrelease: ::c_long = 448; +pub const SYS_futex_waitv: ::c_long = 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 450; cfg_if! { if #[cfg(libc_align)] { diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/x86/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/x86/mod.rs index 1d29ac2f5..93622387e 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/x86/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b32/x86/mod.rs @@ -215,17 +215,6 @@ s! { __glibc_reserved5: ::c_ulong, } - pub struct termios2 { - pub c_iflag: ::tcflag_t, - pub c_oflag: ::tcflag_t, - pub c_cflag: ::tcflag_t, - pub c_lflag: ::tcflag_t, - pub c_line: ::cc_t, - pub c_cc: [::cc_t; 19], - pub c_ispeed: ::speed_t, - pub c_ospeed: ::speed_t, - } - pub struct siginfo_t { pub si_signo: ::c_int, pub si_errno: ::c_int, @@ -246,6 +235,12 @@ s! { pub ss_flags: ::c_int, pub ss_size: ::size_t } + + pub struct seccomp_notif_sizes { + pub seccomp_notif: ::__u16, + pub seccomp_notif_resp: ::__u16, + pub seccomp_data: ::__u16, + } } s_no_extra_traits! { @@ -375,7 +370,6 @@ cfg_if! { } } -pub const RLIM_INFINITY: ::rlim_t = !0; pub const VEOF: usize = 4; pub const RTLD_DEEPBIND: ::c_int = 0x8; pub const RTLD_GLOBAL: ::c_int = 0x100; @@ -395,13 +389,6 @@ pub const O_DSYNC: ::c_int = 4096; pub const O_FSYNC: ::c_int = 0x101000; pub const O_ASYNC: ::c_int = 0x2000; pub const O_NDELAY: ::c_int = 0x800; -pub const RLIMIT_NOFILE: ::__rlimit_resource_t = 7; -pub const RLIMIT_NPROC: ::__rlimit_resource_t = 6; -pub const RLIMIT_RSS: ::__rlimit_resource_t = 5; -pub const RLIMIT_AS: ::__rlimit_resource_t = 9; -pub const RLIMIT_MEMLOCK: ::__rlimit_resource_t = 8; - -pub const SOL_SOCKET: ::c_int = 1; pub const MADV_SOFT_OFFLINE: ::c_int = 101; pub const MAP_LOCKED: ::c_int = 0x02000; @@ -416,7 +403,7 @@ pub const MAP_NONBLOCK: ::c_int = 0x010000; pub const MAP_STACK: ::c_int = 0x020000; pub const MAP_HUGETLB: ::c_int = 0x040000; pub const MAP_GROWSDOWN: ::c_int = 0x0100; -pub const MAP_SYNC : ::c_int = 0x080000; +pub const MAP_SYNC: ::c_int = 0x080000; pub const EDEADLOCK: ::c_int = 35; pub const EUCLEAN: ::c_int = 117; @@ -503,30 +490,6 @@ pub const ENOTRECOVERABLE: ::c_int = 131; pub const EHWPOISON: ::c_int = 133; pub const ERFKILL: ::c_int = 132; -pub const SO_REUSEADDR: ::c_int = 2; -pub const SO_TYPE: ::c_int = 3; -pub const SO_ERROR: ::c_int = 4; -pub const SO_DONTROUTE: ::c_int = 5; -pub const SO_BROADCAST: ::c_int = 6; -pub const SO_SNDBUF: ::c_int = 7; -pub const SO_RCVBUF: ::c_int = 8; -pub const SO_KEEPALIVE: ::c_int = 9; -pub const SO_OOBINLINE: ::c_int = 10; -pub const SO_LINGER: ::c_int = 13; -pub const SO_REUSEPORT: ::c_int = 15; -pub const SO_ACCEPTCONN: ::c_int = 30; -pub const SO_PROTOCOL: ::c_int = 38; -pub const SO_DOMAIN: ::c_int = 39; -pub const SO_SNDBUFFORCE: ::c_int = 32; -pub const SO_RCVBUFFORCE: ::c_int = 33; -pub const SO_NO_CHECK: ::c_int = 11; -pub const SO_PASSCRED: ::c_int = 16; -pub const SO_PEERCRED: ::c_int = 17; -pub const SO_RCVLOWAT: ::c_int = 18; -pub const SO_SNDLOWAT: ::c_int = 19; -pub const SO_RCVTIMEO: ::c_int = 20; -pub const SO_SNDTIMEO: ::c_int = 21; - pub const SA_SIGINFO: ::c_int = 0x00000004; pub const SA_NOCLDWAIT: ::c_int = 0x00000002; @@ -537,12 +500,10 @@ pub const F_GETLK: ::c_int = 5; pub const F_GETOWN: ::c_int = 9; pub const F_SETOWN: ::c_int = 8; -pub const FIOCLEX: ::c_ulong = 0x5451; -pub const FIONCLEX: ::c_ulong = 0x5450; -pub const FIONBIO: ::c_ulong = 0x5421; - pub const PTRACE_GETFPXREGS: ::c_uint = 18; pub const PTRACE_SETFPXREGS: ::c_uint = 19; +pub const PTRACE_SYSEMU: ::c_uint = 31; +pub const PTRACE_SYSEMU_SINGLESTEP: ::c_uint = 32; pub const MCL_CURRENT: ::c_int = 0x0001; pub const MCL_FUTURE: ::c_int = 0x0002; @@ -632,18 +593,6 @@ pub const FFDLY: ::tcflag_t = 0o100000; pub const VTDLY: ::tcflag_t = 0o040000; pub const XTABS: ::tcflag_t = 0o014000; -pub const TIOCGSOFTCAR: ::c_ulong = 0x5419; -pub const TIOCSSOFTCAR: ::c_ulong = 0x541A; -pub const TIOCEXCL: ::c_ulong = 0x540C; -pub const TIOCNXCL: ::c_ulong = 0x540D; -pub const TIOCSCTTY: ::c_ulong = 0x540E; -pub const TIOCSTI: ::c_ulong = 0x5412; -pub const TIOCMGET: ::c_ulong = 0x5415; -pub const TIOCMBIS: ::c_ulong = 0x5416; -pub const TIOCMBIC: ::c_ulong = 0x5417; -pub const TIOCMSET: ::c_ulong = 0x5418; -pub const TIOCCONS: ::c_ulong = 0x541D; - pub const B0: ::speed_t = 0o000000; pub const B50: ::speed_t = 0o000001; pub const B75: ::speed_t = 0o000002; @@ -662,7 +611,6 @@ pub const B19200: ::speed_t = 0o000016; pub const B38400: ::speed_t = 0o000017; pub const EXTA: ::speed_t = B19200; pub const EXTB: ::speed_t = B38400; -pub const BOTHER: ::speed_t = 0o010000; pub const B57600: ::speed_t = 0o010001; pub const B115200: ::speed_t = 0o010002; pub const B230400: ::speed_t = 0o010003; @@ -686,40 +634,11 @@ pub const IEXTEN: ::tcflag_t = 0x00008000; pub const TOSTOP: ::tcflag_t = 0x00000100; pub const FLUSHO: ::tcflag_t = 0x00001000; pub const EXTPROC: ::tcflag_t = 0x00010000; -pub const TCGETS: ::c_ulong = 0x5401; -pub const TCSETS: ::c_ulong = 0x5402; -pub const TCSETSW: ::c_ulong = 0x5403; -pub const TCSETSF: ::c_ulong = 0x5404; -pub const TCGETA: ::c_ulong = 0x5405; -pub const TCSETA: ::c_ulong = 0x5406; -pub const TCSETAW: ::c_ulong = 0x5407; -pub const TCSETAF: ::c_ulong = 0x5408; -pub const TCSBRK: ::c_ulong = 0x5409; -pub const TCXONC: ::c_ulong = 0x540A; -pub const TCFLSH: ::c_ulong = 0x540B; -pub const TIOCINQ: ::c_ulong = 0x541B; -pub const TIOCGPGRP: ::c_ulong = 0x540F; -pub const TIOCSPGRP: ::c_ulong = 0x5410; -pub const TIOCOUTQ: ::c_ulong = 0x5411; -pub const TIOCGWINSZ: ::c_ulong = 0x5413; -pub const TIOCSWINSZ: ::c_ulong = 0x5414; -pub const TIOCGRS485: ::c_int = 0x542E; -pub const TIOCSRS485: ::c_int = 0x542F; -pub const FIONREAD: ::c_ulong = 0x541B; pub const TCSANOW: ::c_int = 0; pub const TCSADRAIN: ::c_int = 1; pub const TCSAFLUSH: ::c_int = 2; -pub const TIOCLINUX: ::c_ulong = 0x541C; -pub const TIOCGSERIAL: ::c_ulong = 0x541E; -pub const TIOCM_ST: ::c_int = 0x008; -pub const TIOCM_SR: ::c_int = 0x010; -pub const TIOCM_CTS: ::c_int = 0x020; -pub const TIOCM_CAR: ::c_int = 0x040; -pub const TIOCM_RNG: ::c_int = 0x080; -pub const TIOCM_DSR: ::c_int = 0x100; - // Syscall table pub const SYS_restart_syscall: ::c_long = 0; pub const SYS_exit: ::c_long = 1; @@ -1101,6 +1020,34 @@ pub const SYS_pkey_mprotect: ::c_long = 380; pub const SYS_pkey_alloc: ::c_long = 381; pub const SYS_pkey_free: ::c_long = 382; pub const SYS_statx: ::c_long = 383; +pub const SYS_rseq: ::c_long = 386; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; +pub const SYS_quotactl_fd: ::c_long = 443; +pub const SYS_landlock_create_ruleset: ::c_long = 444; +pub const SYS_landlock_add_rule: ::c_long = 445; +pub const SYS_landlock_restrict_self: ::c_long = 446; +pub const SYS_memfd_secret: ::c_long = 447; +pub const SYS_process_mrelease: ::c_long = 448; +pub const SYS_futex_waitv: ::c_long = 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 450; // offsets in user_regs_structs, from sys/reg.h pub const EBX: ::c_int = 0; @@ -1142,19 +1089,16 @@ pub const REG_EFL: ::c_int = 16; pub const REG_UESP: ::c_int = 17; pub const REG_SS: ::c_int = 18; +pub const SECCOMP_SET_MODE_STRICT: ::c_uint = 0; +pub const SECCOMP_SET_MODE_FILTER: ::c_uint = 1; +pub const SECCOMP_GET_ACTION_AVAIL: ::c_uint = 2; +pub const SECCOMP_GET_NOTIF_SIZES: ::c_uint = 3; + extern "C" { pub fn getcontext(ucp: *mut ucontext_t) -> ::c_int; pub fn setcontext(ucp: *const ucontext_t) -> ::c_int; - pub fn makecontext( - ucp: *mut ucontext_t, - func: extern "C" fn(), - argc: ::c_int, - ... - ); - pub fn swapcontext( - uocp: *mut ucontext_t, - ucp: *const ucontext_t, - ) -> ::c_int; + pub fn makecontext(ucp: *mut ucontext_t, func: extern "C" fn(), argc: ::c_int, ...); + pub fn swapcontext(uocp: *mut ucontext_t, ucp: *const ucontext_t) -> ::c_int; } cfg_if! { diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/aarch64/align.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/aarch64/align.rs index 8e949963a..06173be66 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/aarch64/align.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/aarch64/align.rs @@ -5,3 +5,54 @@ s_no_extra_traits! { priv_: [f32; 8] } } + +s! { + pub struct ucontext_t { + pub uc_flags: ::c_ulong, + pub uc_link: *mut ucontext_t, + pub uc_stack: ::stack_t, + pub uc_sigmask: ::sigset_t, + pub uc_mcontext: mcontext_t, + } + + #[repr(align(16))] + pub struct mcontext_t { + pub fault_address: ::c_ulonglong, + pub regs: [::c_ulonglong; 31], + pub sp: ::c_ulonglong, + pub pc: ::c_ulonglong, + pub pstate: ::c_ulonglong, + // nested arrays to get the right size/length while being able to + // auto-derive traits like Debug + __reserved: [[u64; 32]; 16], + } + + #[repr(align(16))] + pub struct user_fpsimd_struct { + pub vregs: [[u64; 2]; 32], + pub fpsr: ::c_uint, + pub fpcr: ::c_uint, + } + + #[repr(align(8))] + pub struct clone_args { + pub flags: ::c_ulonglong, + pub pidfd: ::c_ulonglong, + pub child_tid: ::c_ulonglong, + pub parent_tid: ::c_ulonglong, + pub exit_signal: ::c_ulonglong, + pub stack: ::c_ulonglong, + pub stack_size: ::c_ulonglong, + pub tls: ::c_ulonglong, + pub set_tid: ::c_ulonglong, + pub set_tid_size: ::c_ulonglong, + pub cgroup: ::c_ulonglong, + } +} + +extern "C" { + pub fn getcontext(ucp: *mut ucontext_t) -> ::c_int; + pub fn setcontext(ucp: *const ucontext_t) -> ::c_int; + pub fn makecontext(ucp: *mut ucontext_t, func: extern "C" fn(), argc: ::c_int, ...); + pub fn swapcontext(uocp: *mut ucontext_t, ucp: *const ucontext_t) -> ::c_int; +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/aarch64/ilp32.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/aarch64/ilp32.rs new file mode 100644 index 000000000..24b7f4e6b --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/aarch64/ilp32.rs @@ -0,0 +1,62 @@ +use pthread_mutex_t; + +pub type c_long = i32; +pub type c_ulong = u32; + +pub const __SIZEOF_PTHREAD_CONDATTR_T: usize = 4; +pub const __SIZEOF_PTHREAD_MUTEX_T: usize = 32; +pub const __SIZEOF_PTHREAD_MUTEXATTR_T: usize = 4; +pub const __SIZEOF_PTHREAD_RWLOCK_T: usize = 48; + +align_const! { + #[cfg(target_endian = "little")] + pub const PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP: ::pthread_mutex_t = + pthread_mutex_t { + size: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ], + }; + #[cfg(target_endian = "little")] + pub const PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP: ::pthread_mutex_t = + pthread_mutex_t { + size: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ], + }; + #[cfg(target_endian = "little")] + pub const PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP: ::pthread_mutex_t = + pthread_mutex_t { + size: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ], + }; + #[cfg(target_endian = "big")] + pub const PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP: ::pthread_mutex_t = + pthread_mutex_t { + size: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ], + }; + #[cfg(target_endian = "big")] + pub const PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP: ::pthread_mutex_t = + pthread_mutex_t { + size: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ], + }; + #[cfg(target_endian = "big")] + pub const PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP: ::pthread_mutex_t = + pthread_mutex_t { + size: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ], + }; +} + +pub const SYS_sync_file_range2: ::c_long = 84; diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/aarch64/int128.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/aarch64/int128.rs new file mode 100644 index 000000000..4535e73ee --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/aarch64/int128.rs @@ -0,0 +1,7 @@ +s! { + pub struct user_fpsimd_struct { + pub vregs: [::__uint128_t; 32], + pub fpsr: u32, + pub fpcr: u32, + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/aarch64/lp64.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/aarch64/lp64.rs new file mode 100644 index 000000000..14d39e543 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/aarch64/lp64.rs @@ -0,0 +1,71 @@ +use pthread_mutex_t; + +pub type c_long = i64; +pub type c_ulong = u64; + +pub const __SIZEOF_PTHREAD_CONDATTR_T: usize = 8; +pub const __SIZEOF_PTHREAD_MUTEX_T: usize = 48; +pub const __SIZEOF_PTHREAD_MUTEXATTR_T: usize = 8; +pub const __SIZEOF_PTHREAD_RWLOCK_T: usize = 56; + +align_const! { + #[cfg(target_endian = "little")] + pub const PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP: ::pthread_mutex_t = + pthread_mutex_t { + size: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + ], + }; + #[cfg(target_endian = "little")] + pub const PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP: ::pthread_mutex_t = + pthread_mutex_t { + size: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + ], + }; + #[cfg(target_endian = "little")] + pub const PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP: ::pthread_mutex_t = + pthread_mutex_t { + size: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + ], + }; + #[cfg(target_endian = "big")] + pub const PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP: ::pthread_mutex_t = + pthread_mutex_t { + size: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + ], + }; + #[cfg(target_endian = "big")] + pub const PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP: ::pthread_mutex_t = + pthread_mutex_t { + size: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + ], + }; + #[cfg(target_endian = "big")] + pub const PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP: ::pthread_mutex_t = + pthread_mutex_t { + size: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + ], + }; +} + +pub const SYS_renameat: ::c_long = 38; +pub const SYS_sync_file_range: ::c_long = 84; +pub const SYS_getrlimit: ::c_long = 163; +pub const SYS_setrlimit: ::c_long = 164; diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/aarch64/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/aarch64/mod.rs index d64a1b02b..a20a1cf68 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/aarch64/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/aarch64/mod.rs @@ -1,15 +1,12 @@ //! AArch64-specific definitions for 64-bit linux-like values -use pthread_mutex_t; - -pub type c_long = i64; -pub type c_ulong = u64; pub type c_char = u8; pub type wchar_t = u32; pub type nlink_t = u32; pub type blksize_t = i32; pub type suseconds_t = i64; pub type __u64 = ::c_ulonglong; +pub type __s64 = ::c_longlong; s! { pub struct sigaction { @@ -143,7 +140,14 @@ s! { } pub struct pthread_attr_t { - __size: [u64; 8] + __size: [usize; 8] + } + + pub struct user_regs_struct { + pub regs: [::c_ulonglong; 31], + pub sp: ::c_ulonglong, + pub pc: ::c_ulonglong, + pub pstate: ::c_ulonglong, } pub struct ipc_perm { @@ -172,17 +176,6 @@ s! { __unused5: ::c_ulong } - pub struct termios2 { - pub c_iflag: ::tcflag_t, - pub c_oflag: ::tcflag_t, - pub c_cflag: ::tcflag_t, - pub c_lflag: ::tcflag_t, - pub c_line: ::cc_t, - pub c_cc: [::cc_t; 19], - pub c_ispeed: ::speed_t, - pub c_ospeed: ::speed_t, - } - pub struct siginfo_t { pub si_signo: ::c_int, pub si_errno: ::c_int, @@ -203,26 +196,20 @@ s! { pub ss_flags: ::c_int, pub ss_size: ::size_t } + + pub struct seccomp_notif_sizes { + pub seccomp_notif: ::__u16, + pub seccomp_notif_resp: ::__u16, + pub seccomp_data: ::__u16, + } } pub const VEOF: usize = 4; -pub const __SIZEOF_PTHREAD_RWLOCK_T: usize = 56; pub const RTLD_DEEPBIND: ::c_int = 0x8; pub const RTLD_GLOBAL: ::c_int = 0x100; pub const RTLD_NOLOAD: ::c_int = 0x4; -pub const TIOCGSOFTCAR: ::c_ulong = 0x5419; -pub const TIOCSSOFTCAR: ::c_ulong = 0x541A; -pub const TIOCGRS485: ::c_int = 0x542E; -pub const TIOCSRS485: ::c_int = 0x542F; - -pub const RLIMIT_RSS: ::__rlimit_resource_t = 5; -pub const RLIMIT_AS: ::__rlimit_resource_t = 9; -pub const RLIMIT_MEMLOCK: ::__rlimit_resource_t = 8; -pub const RLIMIT_NOFILE: ::__rlimit_resource_t = 7; -pub const RLIMIT_NPROC: ::__rlimit_resource_t = 6; - pub const O_APPEND: ::c_int = 1024; pub const O_CREAT: ::c_int = 64; pub const O_EXCL: ::c_int = 128; @@ -326,61 +313,6 @@ pub const ERFKILL: ::c_int = 132; pub const POSIX_FADV_DONTNEED: ::c_int = 4; pub const POSIX_FADV_NOREUSE: ::c_int = 5; -pub const SOL_SOCKET: ::c_int = 1; - -pub const SO_REUSEADDR: ::c_int = 2; -pub const SO_TYPE: ::c_int = 3; -pub const SO_ERROR: ::c_int = 4; -pub const SO_DONTROUTE: ::c_int = 5; -pub const SO_BROADCAST: ::c_int = 6; -pub const SO_SNDBUF: ::c_int = 7; -pub const SO_RCVBUF: ::c_int = 8; -pub const SO_SNDBUFFORCE: ::c_int = 32; -pub const SO_RCVBUFFORCE: ::c_int = 33; -pub const SO_KEEPALIVE: ::c_int = 9; -pub const SO_OOBINLINE: ::c_int = 10; -pub const SO_NO_CHECK: ::c_int = 11; -pub const SO_PRIORITY: ::c_int = 12; -pub const SO_LINGER: ::c_int = 13; -pub const SO_BSDCOMPAT: ::c_int = 14; -pub const SO_REUSEPORT: ::c_int = 15; -pub const SO_PASSCRED: ::c_int = 16; -pub const SO_PEERCRED: ::c_int = 17; -pub const SO_RCVLOWAT: ::c_int = 18; -pub const SO_SNDLOWAT: ::c_int = 19; -pub const SO_RCVTIMEO: ::c_int = 20; -pub const SO_SNDTIMEO: ::c_int = 21; -pub const SO_SECURITY_AUTHENTICATION: ::c_int = 22; -pub const SO_SECURITY_ENCRYPTION_TRANSPORT: ::c_int = 23; -pub const SO_SECURITY_ENCRYPTION_NETWORK: ::c_int = 24; -pub const SO_BINDTODEVICE: ::c_int = 25; -pub const SO_ATTACH_FILTER: ::c_int = 26; -pub const SO_DETACH_FILTER: ::c_int = 27; -pub const SO_GET_FILTER: ::c_int = SO_ATTACH_FILTER; -pub const SO_PEERNAME: ::c_int = 28; -pub const SO_TIMESTAMP: ::c_int = 29; -pub const SO_ACCEPTCONN: ::c_int = 30; -pub const SO_PEERSEC: ::c_int = 31; -pub const SO_PASSSEC: ::c_int = 34; -pub const SO_TIMESTAMPNS: ::c_int = 35; -pub const SCM_TIMESTAMPNS: ::c_int = SO_TIMESTAMPNS; -pub const SO_MARK: ::c_int = 36; -pub const SO_PROTOCOL: ::c_int = 38; -pub const SO_DOMAIN: ::c_int = 39; -pub const SO_RXQ_OVFL: ::c_int = 40; -pub const SO_WIFI_STATUS: ::c_int = 41; -pub const SCM_WIFI_STATUS: ::c_int = SO_WIFI_STATUS; -pub const SO_PEEK_OFF: ::c_int = 42; -pub const SO_NOFCS: ::c_int = 43; -pub const SO_LOCK_FILTER: ::c_int = 44; -pub const SO_SELECT_ERR_QUEUE: ::c_int = 45; -pub const SO_BUSY_POLL: ::c_int = 46; -pub const SO_MAX_PACING_RATE: ::c_int = 47; -pub const SO_BPF_EXTENSIONS: ::c_int = 48; -pub const SO_INCOMING_CPU: ::c_int = 49; -pub const SO_ATTACH_BPF: ::c_int = 50; -pub const SO_DETACH_BPF: ::c_int = SO_DETACH_FILTER; - pub const SOCK_STREAM: ::c_int = 1; pub const SOCK_DGRAM: ::c_int = 2; @@ -439,23 +371,6 @@ pub const F_UNLCK: ::c_int = 2; pub const SFD_NONBLOCK: ::c_int = 0x0800; -pub const TIOCEXCL: ::c_ulong = 0x540C; -pub const TIOCNXCL: ::c_ulong = 0x540D; -pub const TIOCSCTTY: ::c_ulong = 0x540E; -pub const TIOCSTI: ::c_ulong = 0x5412; -pub const TIOCMGET: ::c_ulong = 0x5415; -pub const TIOCMBIS: ::c_ulong = 0x5416; -pub const TIOCMBIC: ::c_ulong = 0x5417; -pub const TIOCMSET: ::c_ulong = 0x5418; -pub const TIOCCONS: ::c_ulong = 0x541D; - -pub const TIOCM_ST: ::c_int = 0x008; -pub const TIOCM_SR: ::c_int = 0x010; -pub const TIOCM_CTS: ::c_int = 0x020; -pub const TIOCM_CAR: ::c_int = 0x040; -pub const TIOCM_RNG: ::c_int = 0x080; -pub const TIOCM_DSR: ::c_int = 0x100; - pub const SFD_CLOEXEC: ::c_int = 0x080000; pub const NCCS: usize = 32; @@ -488,37 +403,6 @@ pub const EPOLL_CLOEXEC: ::c_int = 0x80000; pub const EFD_CLOEXEC: ::c_int = 0x80000; -pub const __SIZEOF_PTHREAD_CONDATTR_T: usize = 8; -pub const __SIZEOF_PTHREAD_MUTEX_T: usize = 48; -pub const __SIZEOF_PTHREAD_MUTEXATTR_T: usize = 8; - -align_const! { - pub const PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP: ::pthread_mutex_t = - pthread_mutex_t { - size: [ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, - ], - }; - pub const PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP: ::pthread_mutex_t = - pthread_mutex_t { - size: [ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, - ], - }; - pub const PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP: ::pthread_mutex_t = - pthread_mutex_t { - size: [ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, - ], - }; -} - pub const O_DIRECT: ::c_int = 0x10000; pub const O_DIRECTORY: ::c_int = 0x4000; pub const O_NOFOLLOW: ::c_int = 0x8000; @@ -533,14 +417,10 @@ pub const MAP_POPULATE: ::c_int = 0x08000; pub const MAP_NONBLOCK: ::c_int = 0x010000; pub const MAP_STACK: ::c_int = 0x020000; pub const MAP_HUGETLB: ::c_int = 0x040000; -pub const MAP_SYNC : ::c_int = 0x080000; +pub const MAP_SYNC: ::c_int = 0x080000; pub const EDEADLOCK: ::c_int = 35; -pub const FIOCLEX: ::c_ulong = 0x5451; -pub const FIONCLEX: ::c_ulong = 0x5450; -pub const FIONBIO: ::c_ulong = 0x5421; - pub const MCL_CURRENT: ::c_int = 0x0001; pub const MCL_FUTURE: ::c_int = 0x0002; @@ -616,7 +496,6 @@ pub const B19200: ::speed_t = 0o000016; pub const B38400: ::speed_t = 0o000017; pub const EXTA: ::speed_t = B19200; pub const EXTB: ::speed_t = B38400; -pub const BOTHER: ::speed_t = 0o010000; pub const B57600: ::speed_t = 0o010001; pub const B115200: ::speed_t = 0o010002; pub const B230400: ::speed_t = 0o010003; @@ -633,6 +512,11 @@ pub const B3000000: ::speed_t = 0o010015; pub const B3500000: ::speed_t = 0o010016; pub const B4000000: ::speed_t = 0o010017; +pub const SECCOMP_SET_MODE_STRICT: ::c_uint = 0; +pub const SECCOMP_SET_MODE_FILTER: ::c_uint = 1; +pub const SECCOMP_GET_ACTION_AVAIL: ::c_uint = 2; +pub const SECCOMP_GET_NOTIF_SIZES: ::c_uint = 3; + pub const VEOL: usize = 11; pub const VEOL2: usize = 16; pub const VMIN: usize = 6; @@ -640,32 +524,11 @@ pub const IEXTEN: ::tcflag_t = 0x00008000; pub const TOSTOP: ::tcflag_t = 0x00000100; pub const FLUSHO: ::tcflag_t = 0x00001000; pub const EXTPROC: ::tcflag_t = 0x00010000; -pub const TCGETS: ::c_ulong = 0x5401; -pub const TCSETS: ::c_ulong = 0x5402; -pub const TCSETSW: ::c_ulong = 0x5403; -pub const TCSETSF: ::c_ulong = 0x5404; -pub const TCGETA: ::c_ulong = 0x5405; -pub const TCSETA: ::c_ulong = 0x5406; -pub const TCSETAW: ::c_ulong = 0x5407; -pub const TCSETAF: ::c_ulong = 0x5408; -pub const TCSBRK: ::c_ulong = 0x5409; -pub const TCXONC: ::c_ulong = 0x540A; -pub const TCFLSH: ::c_ulong = 0x540B; -pub const TIOCINQ: ::c_ulong = 0x541B; -pub const TIOCGPGRP: ::c_ulong = 0x540F; -pub const TIOCSPGRP: ::c_ulong = 0x5410; -pub const TIOCOUTQ: ::c_ulong = 0x5411; -pub const TIOCGWINSZ: ::c_ulong = 0x5413; -pub const TIOCSWINSZ: ::c_ulong = 0x5414; -pub const FIONREAD: ::c_ulong = 0x541B; pub const TCSANOW: ::c_int = 0; pub const TCSADRAIN: ::c_int = 1; pub const TCSAFLUSH: ::c_int = 2; -pub const TIOCLINUX: ::c_ulong = 0x541C; -pub const TIOCGSERIAL: ::c_ulong = 0x541E; - // sys/auxv.h pub const HWCAP_FP: ::c_ulong = 1 << 0; pub const HWCAP_ASIMD: ::c_ulong = 1 << 1; @@ -710,6 +573,22 @@ pub const HWCAP_PACG: ::c_ulong = 1 << 31; //pub const HWCAP2_SVESM4: ::c_ulong = 1 << 6; //pub const HWCAP2_FLAGM2: ::c_ulong = 1 << 7; //pub const HWCAP2_FRINT: ::c_ulong = 1 << 8; +//pub const HWCAP2_MTE: ::c_ulong = 1 << 18; + +// linux/prctl.h +pub const PR_PAC_RESET_KEYS: ::c_int = 54; +pub const PR_SET_TAGGED_ADDR_CTRL: ::c_int = 55; +pub const PR_GET_TAGGED_ADDR_CTRL: ::c_int = 56; +pub const PR_PAC_SET_ENABLED_KEYS: ::c_int = 60; +pub const PR_PAC_GET_ENABLED_KEYS: ::c_int = 61; + +pub const PR_TAGGED_ADDR_ENABLE: ::c_ulong = 1; + +pub const PR_PAC_APIAKEY: ::c_ulong = 1 << 0; +pub const PR_PAC_APIBKEY: ::c_ulong = 1 << 1; +pub const PR_PAC_APDAKEY: ::c_ulong = 1 << 2; +pub const PR_PAC_APDBKEY: ::c_ulong = 1 << 3; +pub const PR_PAC_APGAKEY: ::c_ulong = 1 << 4; // Syscall table pub const SYS_io_setup: ::c_long = 0; @@ -750,11 +629,15 @@ pub const SYS_mkdirat: ::c_long = 34; pub const SYS_unlinkat: ::c_long = 35; pub const SYS_symlinkat: ::c_long = 36; pub const SYS_linkat: ::c_long = 37; -pub const SYS_renameat: ::c_long = 38; +// 38 is renameat only on LP64 pub const SYS_umount2: ::c_long = 39; pub const SYS_mount: ::c_long = 40; pub const SYS_pivot_root: ::c_long = 41; pub const SYS_nfsservctl: ::c_long = 42; +pub const SYS_statfs: ::c_long = 43; +pub const SYS_fstatfs: ::c_long = 44; +pub const SYS_truncate: ::c_long = 45; +pub const SYS_ftruncate: ::c_long = 46; pub const SYS_fallocate: ::c_long = 47; pub const SYS_faccessat: ::c_long = 48; pub const SYS_chdir: ::c_long = 49; @@ -791,7 +674,7 @@ pub const SYS_fstat: ::c_long = 80; pub const SYS_sync: ::c_long = 81; pub const SYS_fsync: ::c_long = 82; pub const SYS_fdatasync: ::c_long = 83; -pub const SYS_sync_file_range: ::c_long = 84; +// 84 sync_file_range on LP64 and sync_file_range2 on ILP32 pub const SYS_timerfd_create: ::c_long = 85; pub const SYS_timerfd_settime: ::c_long = 86; pub const SYS_timerfd_gettime: ::c_long = 87; @@ -870,8 +753,8 @@ pub const SYS_setgroups: ::c_long = 159; pub const SYS_uname: ::c_long = 160; pub const SYS_sethostname: ::c_long = 161; pub const SYS_setdomainname: ::c_long = 162; -pub const SYS_getrlimit: ::c_long = 163; -pub const SYS_setrlimit: ::c_long = 164; +// 163 is getrlimit only on LP64 +// 164 is setrlimit only on LP64 pub const SYS_getrusage: ::c_long = 165; pub const SYS_umask: ::c_long = 166; pub const SYS_prctl: ::c_long = 167; @@ -982,8 +865,36 @@ pub const SYS_pkey_mprotect: ::c_long = 288; pub const SYS_pkey_alloc: ::c_long = 289; pub const SYS_pkey_free: ::c_long = 290; pub const SYS_statx: ::c_long = 291; +pub const SYS_rseq: ::c_long = 293; +pub const SYS_kexec_file_load: ::c_long = 294; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; +pub const SYS_quotactl_fd: ::c_long = 443; +pub const SYS_landlock_create_ruleset: ::c_long = 444; +pub const SYS_landlock_add_rule: ::c_long = 445; +pub const SYS_landlock_restrict_self: ::c_long = 446; +pub const SYS_memfd_secret: ::c_long = 447; +pub const SYS_process_mrelease: ::c_long = 448; +pub const SYS_futex_waitv: ::c_long = 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 450; -#[link(name = "util")] extern "C" { pub fn sysctl( name: *mut ::c_int, @@ -995,9 +906,26 @@ extern "C" { ) -> ::c_int; } +cfg_if! { + if #[cfg(target_pointer_width = "32")] { + mod ilp32; + pub use self::ilp32::*; + } else { + mod lp64; + pub use self::lp64::*; + } +} + cfg_if! { if #[cfg(libc_align)] { mod align; pub use self::align::*; } } + +cfg_if! { + if #[cfg(libc_int128)] { + mod int128; + pub use self::int128::*; + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/loongarch64/align.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/loongarch64/align.rs new file mode 100644 index 000000000..4cae9c1c3 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/loongarch64/align.rs @@ -0,0 +1,40 @@ +s_no_extra_traits! { + #[allow(missing_debug_implementations)] + #[repr(align(16))] + pub struct max_align_t { + priv_: [f64; 4] + } +} + +s! { + pub struct ucontext_t { + pub uc_flags: ::c_ulong, + pub uc_link: *mut ucontext_t, + pub uc_stack: ::stack_t, + pub uc_sigmask: ::sigset_t, + pub uc_mcontext: mcontext_t, + } + + #[repr(align(16))] + pub struct mcontext_t { + pub sc_pc: ::c_ulonglong, + pub sc_regs: [::c_ulonglong; 32], + pub sc_flags: ::c_ulong, + pub sc_extcontext: [u64; 0], + } + + #[repr(align(8))] + pub struct clone_args { + pub flags: ::c_ulonglong, + pub pidfd: ::c_ulonglong, + pub child_tid: ::c_ulonglong, + pub parent_tid: ::c_ulonglong, + pub exit_signal: ::c_ulonglong, + pub stack: ::c_ulonglong, + pub stack_size: ::c_ulonglong, + pub tls: ::c_ulonglong, + pub set_tid: ::c_ulonglong, + pub set_tid_size: ::c_ulonglong, + pub cgroup: ::c_ulonglong, + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/loongarch64/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/loongarch64/mod.rs new file mode 100644 index 000000000..d64c353bb --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/loongarch64/mod.rs @@ -0,0 +1,862 @@ +use pthread_mutex_t; + +pub type c_char = i8; +pub type c_long = i64; +pub type c_ulong = u64; +pub type wchar_t = i32; + +pub type blksize_t = i32; +pub type nlink_t = u32; +pub type suseconds_t = i64; +pub type __u64 = ::c_ulonglong; +pub type __s64 = ::c_longlong; + +s! { + pub struct stat { + pub st_dev: ::dev_t, + pub st_ino: ::ino_t, + pub st_mode: ::mode_t, + pub st_nlink: ::nlink_t, + pub st_uid: ::uid_t, + pub st_gid: ::gid_t, + pub st_rdev: ::dev_t, + __pad1: ::dev_t, + pub st_size: ::off_t, + pub st_blksize: ::blksize_t, + __pad2: ::c_int, + pub st_blocks: ::blkcnt_t, + pub st_atime: ::time_t, + pub st_atime_nsec: ::c_long, + pub st_mtime: ::time_t, + pub st_mtime_nsec: ::c_long, + pub st_ctime: ::time_t, + pub st_ctime_nsec: ::c_long, + __unused: [::c_int; 2], + } + + pub struct stat64 { + pub st_dev: ::dev_t, + pub st_ino: ::ino64_t, + pub st_mode: ::mode_t, + pub st_nlink: ::nlink_t, + pub st_uid: ::uid_t, + pub st_gid: ::gid_t, + pub st_rdev: ::dev_t, + pub __pad1: ::dev_t, + pub st_size: ::off64_t, + pub st_blksize: ::blksize_t, + pub __pad2: ::c_int, + pub st_blocks: ::blkcnt_t, + pub st_atime: ::time_t, + pub st_atime_nsec: ::c_long, + pub st_mtime: ::time_t, + pub st_mtime_nsec: ::c_long, + pub st_ctime: ::time_t, + pub st_ctime_nsec: ::c_long, + __unused: [::c_int; 2], + } + + pub struct statfs { + pub f_type: ::__fsword_t, + pub f_bsize: ::__fsword_t, + pub f_blocks: ::fsblkcnt_t, + pub f_bfree: ::fsblkcnt_t, + pub f_bavail: ::fsblkcnt_t, + pub f_files: ::fsfilcnt_t, + pub f_ffree: ::fsfilcnt_t, + pub f_fsid: ::fsid_t, + pub f_namelen: ::__fsword_t, + pub f_frsize: ::__fsword_t, + pub f_flags: ::__fsword_t, + pub f_spare: [::__fsword_t; 4], + } + + pub struct statfs64 { + pub f_type: ::__fsword_t, + pub f_bsize: ::__fsword_t, + pub f_blocks: u64, + pub f_bfree: u64, + pub f_bavail: u64, + pub f_files: u64, + pub f_ffree: u64, + pub f_fsid: ::fsid_t, + pub f_namelen: ::__fsword_t, + pub f_frsize: ::__fsword_t, + pub f_flags: ::__fsword_t, + pub f_spare: [::__fsword_t; 4], + } + + pub struct flock { + pub l_type: ::c_short, + pub l_whence: ::c_short, + pub l_start: ::off_t, + pub l_len: ::off_t, + pub l_pid: ::pid_t, + } + + pub struct flock64 { + pub l_type: ::c_short, + pub l_whence: ::c_short, + pub l_start: ::off64_t, + pub l_len: ::off64_t, + pub l_pid: ::pid_t, + } + + pub struct statvfs { + pub f_bsize: ::c_ulong, + pub f_frsize: ::c_ulong, + pub f_blocks: ::fsblkcnt_t, + pub f_bfree: ::fsblkcnt_t, + pub f_bavail: ::fsblkcnt_t, + pub f_files: ::fsfilcnt_t, + pub f_ffree: ::fsfilcnt_t, + pub f_favail: ::fsfilcnt_t, + pub f_fsid: ::c_ulong, + pub f_flag: ::c_ulong, + pub f_namemax: ::c_ulong, + __f_spare: [::c_int; 6], + } + + pub struct statvfs64 { + pub f_bsize: ::c_ulong, + pub f_frsize: ::c_ulong, + pub f_blocks: u64, + pub f_bfree: u64, + pub f_bavail: u64, + pub f_files: u64, + pub f_ffree: u64, + pub f_favail: u64, + pub f_fsid: ::c_ulong, + pub f_flag: ::c_ulong, + pub f_namemax: ::c_ulong, + __f_spare: [::c_int; 6], + } + + pub struct pthread_attr_t { + __size: [::c_ulong; 7] + } + + pub struct sigaction { + pub sa_sigaction: ::sighandler_t, + pub sa_mask: ::sigset_t, + pub sa_flags: ::c_int, + pub sa_restorer: ::Option, + } + + pub struct stack_t { + pub ss_sp: *mut ::c_void, + pub ss_flags: ::c_int, + pub ss_size: ::size_t, + } + + pub struct siginfo_t { + pub si_signo: ::c_int, + pub si_errno: ::c_int, + pub si_code: ::c_int, + #[doc(hidden)] + #[deprecated( + since="0.2.54", + note="Please leave a comment on \ + https://github.com/rust-lang/libc/pull/1316 if you're using \ + this field" + )] + pub _pad: [::c_int; 29], + _align: [u64; 0], + } + + pub struct ipc_perm { + pub __key: ::key_t, + pub uid: ::uid_t, + pub gid: ::gid_t, + pub cuid: ::uid_t, + pub cgid: ::gid_t, + pub mode: ::c_uint, + pub __seq: ::c_ushort, + __pad2: ::c_ushort, + __unused1: ::c_ulong, + __unused2: ::c_ulong + } + + pub struct shmid_ds { + pub shm_perm: ::ipc_perm, + pub shm_segsz: ::size_t, + pub shm_atime: ::time_t, + pub shm_dtime: ::time_t, + pub shm_ctime: ::time_t, + pub shm_cpid: ::pid_t, + pub shm_lpid: ::pid_t, + pub shm_nattch: ::shmatt_t, + __unused4: ::c_ulong, + __unused5: ::c_ulong + } +} + +pub const __SIZEOF_PTHREAD_CONDATTR_T: usize = 4; +pub const __SIZEOF_PTHREAD_MUTEXATTR_T: usize = 4; +pub const __SIZEOF_PTHREAD_MUTEX_T: usize = 40; +pub const __SIZEOF_PTHREAD_RWLOCK_T: usize = 56; + +align_const! { + #[cfg(target_endian = "little")] + pub const PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP: ::pthread_mutex_t = + pthread_mutex_t { + size: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ], + }; + #[cfg(target_endian = "little")] + pub const PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP: ::pthread_mutex_t = + pthread_mutex_t { + size: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ], + }; + #[cfg(target_endian = "little")] + pub const PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP: ::pthread_mutex_t = + pthread_mutex_t { + size: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ], + }; + #[cfg(target_endian = "big")] + pub const PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP: ::pthread_mutex_t = + pthread_mutex_t { + size: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ], + }; + #[cfg(target_endian = "big")] + pub const PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP: ::pthread_mutex_t = + pthread_mutex_t { + size: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ], + }; + #[cfg(target_endian = "big")] + pub const PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP: ::pthread_mutex_t = + pthread_mutex_t { + size: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ], + }; +} + +pub const SYS_io_setup: ::c_long = 0; +pub const SYS_io_destroy: ::c_long = 1; +pub const SYS_io_submit: ::c_long = 2; +pub const SYS_io_cancel: ::c_long = 3; +pub const SYS_io_getevents: ::c_long = 4; +pub const SYS_setxattr: ::c_long = 5; +pub const SYS_lsetxattr: ::c_long = 6; +pub const SYS_fsetxattr: ::c_long = 7; +pub const SYS_getxattr: ::c_long = 8; +pub const SYS_lgetxattr: ::c_long = 9; +pub const SYS_fgetxattr: ::c_long = 10; +pub const SYS_listxattr: ::c_long = 11; +pub const SYS_llistxattr: ::c_long = 12; +pub const SYS_flistxattr: ::c_long = 13; +pub const SYS_removexattr: ::c_long = 14; +pub const SYS_lremovexattr: ::c_long = 15; +pub const SYS_fremovexattr: ::c_long = 16; +pub const SYS_getcwd: ::c_long = 17; +pub const SYS_lookup_dcookie: ::c_long = 18; +pub const SYS_eventfd2: ::c_long = 19; +pub const SYS_epoll_create1: ::c_long = 20; +pub const SYS_epoll_ctl: ::c_long = 21; +pub const SYS_epoll_pwait: ::c_long = 22; +pub const SYS_dup: ::c_long = 23; +pub const SYS_dup3: ::c_long = 24; +pub const SYS_fcntl: ::c_long = 25; +pub const SYS_inotify_init1: ::c_long = 26; +pub const SYS_inotify_add_watch: ::c_long = 27; +pub const SYS_inotify_rm_watch: ::c_long = 28; +pub const SYS_ioctl: ::c_long = 29; +pub const SYS_ioprio_set: ::c_long = 30; +pub const SYS_ioprio_get: ::c_long = 31; +pub const SYS_flock: ::c_long = 32; +pub const SYS_mknodat: ::c_long = 33; +pub const SYS_mkdirat: ::c_long = 34; +pub const SYS_unlinkat: ::c_long = 35; +pub const SYS_symlinkat: ::c_long = 36; +pub const SYS_linkat: ::c_long = 37; +pub const SYS_umount2: ::c_long = 39; +pub const SYS_mount: ::c_long = 40; +pub const SYS_pivot_root: ::c_long = 41; +pub const SYS_nfsservctl: ::c_long = 42; +pub const SYS_statfs: ::c_long = 43; +pub const SYS_fstatfs: ::c_long = 44; +pub const SYS_truncate: ::c_long = 45; +pub const SYS_ftruncate: ::c_long = 46; +pub const SYS_fallocate: ::c_long = 47; +pub const SYS_faccessat: ::c_long = 48; +pub const SYS_chdir: ::c_long = 49; +pub const SYS_fchdir: ::c_long = 50; +pub const SYS_chroot: ::c_long = 51; +pub const SYS_fchmod: ::c_long = 52; +pub const SYS_fchmodat: ::c_long = 53; +pub const SYS_fchownat: ::c_long = 54; +pub const SYS_fchown: ::c_long = 55; +pub const SYS_openat: ::c_long = 56; +pub const SYS_close: ::c_long = 57; +pub const SYS_vhangup: ::c_long = 58; +pub const SYS_pipe2: ::c_long = 59; +pub const SYS_quotactl: ::c_long = 60; +pub const SYS_getdents64: ::c_long = 61; +pub const SYS_lseek: ::c_long = 62; +pub const SYS_read: ::c_long = 63; +pub const SYS_write: ::c_long = 64; +pub const SYS_readv: ::c_long = 65; +pub const SYS_writev: ::c_long = 66; +pub const SYS_pread64: ::c_long = 67; +pub const SYS_pwrite64: ::c_long = 68; +pub const SYS_preadv: ::c_long = 69; +pub const SYS_pwritev: ::c_long = 70; +pub const SYS_sendfile: ::c_long = 71; +pub const SYS_pselect6: ::c_long = 72; +pub const SYS_ppoll: ::c_long = 73; +pub const SYS_signalfd4: ::c_long = 74; +pub const SYS_vmsplice: ::c_long = 75; +pub const SYS_splice: ::c_long = 76; +pub const SYS_tee: ::c_long = 77; +pub const SYS_readlinkat: ::c_long = 78; +pub const SYS_sync: ::c_long = 81; +pub const SYS_fsync: ::c_long = 82; +pub const SYS_fdatasync: ::c_long = 83; +pub const SYS_sync_file_range: ::c_long = 84; +pub const SYS_timerfd_create: ::c_long = 85; +pub const SYS_timerfd_settime: ::c_long = 86; +pub const SYS_timerfd_gettime: ::c_long = 87; +pub const SYS_utimensat: ::c_long = 88; +pub const SYS_acct: ::c_long = 89; +pub const SYS_capget: ::c_long = 90; +pub const SYS_capset: ::c_long = 91; +pub const SYS_personality: ::c_long = 92; +pub const SYS_exit: ::c_long = 93; +pub const SYS_exit_group: ::c_long = 94; +pub const SYS_waitid: ::c_long = 95; +pub const SYS_set_tid_address: ::c_long = 96; +pub const SYS_unshare: ::c_long = 97; +pub const SYS_futex: ::c_long = 98; +pub const SYS_set_robust_list: ::c_long = 99; +pub const SYS_get_robust_list: ::c_long = 100; +pub const SYS_nanosleep: ::c_long = 101; +pub const SYS_getitimer: ::c_long = 102; +pub const SYS_setitimer: ::c_long = 103; +pub const SYS_kexec_load: ::c_long = 104; +pub const SYS_init_module: ::c_long = 105; +pub const SYS_delete_module: ::c_long = 106; +pub const SYS_timer_create: ::c_long = 107; +pub const SYS_timer_gettime: ::c_long = 108; +pub const SYS_timer_getoverrun: ::c_long = 109; +pub const SYS_timer_settime: ::c_long = 110; +pub const SYS_timer_delete: ::c_long = 111; +pub const SYS_clock_settime: ::c_long = 112; +pub const SYS_clock_gettime: ::c_long = 113; +pub const SYS_clock_getres: ::c_long = 114; +pub const SYS_clock_nanosleep: ::c_long = 115; +pub const SYS_syslog: ::c_long = 116; +pub const SYS_ptrace: ::c_long = 117; +pub const SYS_sched_setparam: ::c_long = 118; +pub const SYS_sched_setscheduler: ::c_long = 119; +pub const SYS_sched_getscheduler: ::c_long = 120; +pub const SYS_sched_getparam: ::c_long = 121; +pub const SYS_sched_setaffinity: ::c_long = 122; +pub const SYS_sched_getaffinity: ::c_long = 123; +pub const SYS_sched_yield: ::c_long = 124; +pub const SYS_sched_get_priority_max: ::c_long = 125; +pub const SYS_sched_get_priority_min: ::c_long = 126; +pub const SYS_sched_rr_get_interval: ::c_long = 127; +pub const SYS_restart_syscall: ::c_long = 128; +pub const SYS_kill: ::c_long = 129; +pub const SYS_tkill: ::c_long = 130; +pub const SYS_tgkill: ::c_long = 131; +pub const SYS_sigaltstack: ::c_long = 132; +pub const SYS_rt_sigsuspend: ::c_long = 133; +pub const SYS_rt_sigaction: ::c_long = 134; +pub const SYS_rt_sigprocmask: ::c_long = 135; +pub const SYS_rt_sigpending: ::c_long = 136; +pub const SYS_rt_sigtimedwait: ::c_long = 137; +pub const SYS_rt_sigqueueinfo: ::c_long = 138; +pub const SYS_rt_sigreturn: ::c_long = 139; +pub const SYS_setpriority: ::c_long = 140; +pub const SYS_getpriority: ::c_long = 141; +pub const SYS_reboot: ::c_long = 142; +pub const SYS_setregid: ::c_long = 143; +pub const SYS_setgid: ::c_long = 144; +pub const SYS_setreuid: ::c_long = 145; +pub const SYS_setuid: ::c_long = 146; +pub const SYS_setresuid: ::c_long = 147; +pub const SYS_getresuid: ::c_long = 148; +pub const SYS_setresgid: ::c_long = 149; +pub const SYS_getresgid: ::c_long = 150; +pub const SYS_setfsuid: ::c_long = 151; +pub const SYS_setfsgid: ::c_long = 152; +pub const SYS_times: ::c_long = 153; +pub const SYS_setpgid: ::c_long = 154; +pub const SYS_getpgid: ::c_long = 155; +pub const SYS_getsid: ::c_long = 156; +pub const SYS_setsid: ::c_long = 157; +pub const SYS_getgroups: ::c_long = 158; +pub const SYS_setgroups: ::c_long = 159; +pub const SYS_uname: ::c_long = 160; +pub const SYS_sethostname: ::c_long = 161; +pub const SYS_setdomainname: ::c_long = 162; +pub const SYS_getrusage: ::c_long = 165; +pub const SYS_umask: ::c_long = 166; +pub const SYS_prctl: ::c_long = 167; +pub const SYS_getcpu: ::c_long = 168; +pub const SYS_gettimeofday: ::c_long = 169; +pub const SYS_settimeofday: ::c_long = 170; +pub const SYS_adjtimex: ::c_long = 171; +pub const SYS_getpid: ::c_long = 172; +pub const SYS_getppid: ::c_long = 173; +pub const SYS_getuid: ::c_long = 174; +pub const SYS_geteuid: ::c_long = 175; +pub const SYS_getgid: ::c_long = 176; +pub const SYS_getegid: ::c_long = 177; +pub const SYS_gettid: ::c_long = 178; +pub const SYS_sysinfo: ::c_long = 179; +pub const SYS_mq_open: ::c_long = 180; +pub const SYS_mq_unlink: ::c_long = 181; +pub const SYS_mq_timedsend: ::c_long = 182; +pub const SYS_mq_timedreceive: ::c_long = 183; +pub const SYS_mq_notify: ::c_long = 184; +pub const SYS_mq_getsetattr: ::c_long = 185; +pub const SYS_msgget: ::c_long = 186; +pub const SYS_msgctl: ::c_long = 187; +pub const SYS_msgrcv: ::c_long = 188; +pub const SYS_msgsnd: ::c_long = 189; +pub const SYS_semget: ::c_long = 190; +pub const SYS_semctl: ::c_long = 191; +pub const SYS_semtimedop: ::c_long = 192; +pub const SYS_semop: ::c_long = 193; +pub const SYS_shmget: ::c_long = 194; +pub const SYS_shmctl: ::c_long = 195; +pub const SYS_shmat: ::c_long = 196; +pub const SYS_shmdt: ::c_long = 197; +pub const SYS_socket: ::c_long = 198; +pub const SYS_socketpair: ::c_long = 199; +pub const SYS_bind: ::c_long = 200; +pub const SYS_listen: ::c_long = 201; +pub const SYS_accept: ::c_long = 202; +pub const SYS_connect: ::c_long = 203; +pub const SYS_getsockname: ::c_long = 204; +pub const SYS_getpeername: ::c_long = 205; +pub const SYS_sendto: ::c_long = 206; +pub const SYS_recvfrom: ::c_long = 207; +pub const SYS_setsockopt: ::c_long = 208; +pub const SYS_getsockopt: ::c_long = 209; +pub const SYS_shutdown: ::c_long = 210; +pub const SYS_sendmsg: ::c_long = 211; +pub const SYS_recvmsg: ::c_long = 212; +pub const SYS_readahead: ::c_long = 213; +pub const SYS_brk: ::c_long = 214; +pub const SYS_munmap: ::c_long = 215; +pub const SYS_mremap: ::c_long = 216; +pub const SYS_add_key: ::c_long = 217; +pub const SYS_request_key: ::c_long = 218; +pub const SYS_keyctl: ::c_long = 219; +pub const SYS_clone: ::c_long = 220; +pub const SYS_execve: ::c_long = 221; +pub const SYS_mmap: ::c_long = 222; +pub const SYS_fadvise64: ::c_long = 223; +pub const SYS_swapon: ::c_long = 224; +pub const SYS_swapoff: ::c_long = 225; +pub const SYS_mprotect: ::c_long = 226; +pub const SYS_msync: ::c_long = 227; +pub const SYS_mlock: ::c_long = 228; +pub const SYS_munlock: ::c_long = 229; +pub const SYS_mlockall: ::c_long = 230; +pub const SYS_munlockall: ::c_long = 231; +pub const SYS_mincore: ::c_long = 232; +pub const SYS_madvise: ::c_long = 233; +pub const SYS_remap_file_pages: ::c_long = 234; +pub const SYS_mbind: ::c_long = 235; +pub const SYS_get_mempolicy: ::c_long = 236; +pub const SYS_set_mempolicy: ::c_long = 237; +pub const SYS_migrate_pages: ::c_long = 238; +pub const SYS_move_pages: ::c_long = 239; +pub const SYS_rt_tgsigqueueinfo: ::c_long = 240; +pub const SYS_perf_event_open: ::c_long = 241; +pub const SYS_accept4: ::c_long = 242; +pub const SYS_recvmmsg: ::c_long = 243; +//pub const SYS_arch_specific_syscall: ::c_long = 244; +pub const SYS_wait4: ::c_long = 260; +pub const SYS_prlimit64: ::c_long = 261; +pub const SYS_fanotify_init: ::c_long = 262; +pub const SYS_fanotify_mark: ::c_long = 263; +pub const SYS_name_to_handle_at: ::c_long = 264; +pub const SYS_open_by_handle_at: ::c_long = 265; +pub const SYS_clock_adjtime: ::c_long = 266; +pub const SYS_syncfs: ::c_long = 267; +pub const SYS_setns: ::c_long = 268; +pub const SYS_sendmmsg: ::c_long = 269; +pub const SYS_process_vm_readv: ::c_long = 270; +pub const SYS_process_vm_writev: ::c_long = 271; +pub const SYS_kcmp: ::c_long = 272; +pub const SYS_finit_module: ::c_long = 273; +pub const SYS_sched_setattr: ::c_long = 274; +pub const SYS_sched_getattr: ::c_long = 275; +pub const SYS_renameat2: ::c_long = 276; +pub const SYS_seccomp: ::c_long = 277; +pub const SYS_getrandom: ::c_long = 278; +pub const SYS_memfd_create: ::c_long = 279; +pub const SYS_bpf: ::c_long = 280; +pub const SYS_execveat: ::c_long = 281; +pub const SYS_userfaultfd: ::c_long = 282; +pub const SYS_membarrier: ::c_long = 283; +pub const SYS_mlock2: ::c_long = 284; +pub const SYS_copy_file_range: ::c_long = 285; +pub const SYS_preadv2: ::c_long = 286; +pub const SYS_pwritev2: ::c_long = 287; +pub const SYS_pkey_mprotect: ::c_long = 288; +pub const SYS_pkey_alloc: ::c_long = 289; +pub const SYS_pkey_free: ::c_long = 290; +pub const SYS_statx: ::c_long = 291; +pub const SYS_io_pgetevents: ::c_long = 292; +pub const SYS_rseq: ::c_long = 293; +pub const SYS_kexec_file_load: ::c_long = 294; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; +pub const SYS_quotactl_fd: ::c_long = 443; +pub const SYS_landlock_create_ruleset: ::c_long = 444; +pub const SYS_landlock_add_rule: ::c_long = 445; +pub const SYS_landlock_restrict_self: ::c_long = 446; +pub const SYS_process_mrelease: ::c_long = 448; +pub const SYS_futex_waitv: ::c_long = 449; +//pub const SYS_set_mempolicy_home_node: ::c_long = 450; + +pub const POSIX_FADV_DONTNEED: ::c_int = 4; +pub const POSIX_FADV_NOREUSE: ::c_int = 5; +pub const O_DIRECT: ::c_int = 0o00040000; +pub const O_DIRECTORY: ::c_int = 0o00200000; +pub const O_NOFOLLOW: ::c_int = 0o00400000; +pub const O_TRUNC: ::c_int = 0o00001000; +pub const O_NOATIME: ::c_int = 0o1000000; +pub const O_CLOEXEC: ::c_int = 0o02000000; +pub const O_PATH: ::c_int = 0o10000000; +pub const O_TMPFILE: ::c_int = 0o20000000 | O_DIRECTORY; +pub const O_APPEND: ::c_int = 0o00002000; +pub const O_CREAT: ::c_int = 0o00000100; +pub const O_EXCL: ::c_int = 0o00000200; +pub const O_NOCTTY: ::c_int = 0o00000400; +pub const O_NONBLOCK: ::c_int = 0o00004000; +pub const FASYNC: ::c_int = 0o00020000; +pub const O_SYNC: ::c_int = 0o04010000; +pub const O_RSYNC: ::c_int = 0o04010000; +pub const O_FSYNC: ::c_int = O_SYNC; +pub const O_ASYNC: ::c_int = 0o00020000; +pub const O_DSYNC: ::c_int = 0o00010000; +pub const O_NDELAY: ::c_int = O_NONBLOCK; +pub const F_RDLCK: ::c_int = 0; +pub const F_WRLCK: ::c_int = 1; +pub const F_UNLCK: ::c_int = 2; +pub const F_GETLK: ::c_int = 5; +pub const F_SETLK: ::c_int = 6; +pub const F_SETLKW: ::c_int = 7; +pub const F_SETOWN: ::c_int = 8; +pub const F_GETOWN: ::c_int = 9; +pub const F_OFD_GETLK: ::c_int = 36; +pub const F_OFD_SETLK: ::c_int = 37; +pub const F_OFD_SETLKW: ::c_int = 38; + +pub const EDEADLK: ::c_int = 35; +pub const EDEADLOCK: ::c_int = 35; +pub const ENAMETOOLONG: ::c_int = 36; +pub const ENOLCK: ::c_int = 37; +pub const ENOSYS: ::c_int = 38; +pub const ENOTEMPTY: ::c_int = 39; +pub const ELOOP: ::c_int = 40; +pub const ENOMSG: ::c_int = 42; +pub const EIDRM: ::c_int = 43; +pub const ECHRNG: ::c_int = 44; +pub const EL2NSYNC: ::c_int = 45; +pub const EL3HLT: ::c_int = 46; +pub const EL3RST: ::c_int = 47; +pub const ELNRNG: ::c_int = 48; +pub const EUNATCH: ::c_int = 49; +pub const ENOCSI: ::c_int = 50; +pub const EL2HLT: ::c_int = 51; +pub const EBADE: ::c_int = 52; +pub const EBADR: ::c_int = 53; +pub const EXFULL: ::c_int = 54; +pub const ENOANO: ::c_int = 55; +pub const EBADRQC: ::c_int = 56; +pub const EBADSLT: ::c_int = 57; +pub const EBFONT: ::c_int = 59; +pub const ENOSTR: ::c_int = 60; +pub const ENODATA: ::c_int = 61; +pub const ETIME: ::c_int = 62; +pub const ENOSR: ::c_int = 63; +pub const ENONET: ::c_int = 64; +pub const ENOPKG: ::c_int = 65; +pub const EREMOTE: ::c_int = 66; +pub const ENOLINK: ::c_int = 67; +pub const EADV: ::c_int = 68; +pub const ESRMNT: ::c_int = 69; +pub const ECOMM: ::c_int = 70; +pub const EPROTO: ::c_int = 71; +pub const EDOTDOT: ::c_int = 73; +pub const EMULTIHOP: ::c_int = 72; +pub const EOVERFLOW: ::c_int = 75; +pub const ENOTUNIQ: ::c_int = 76; +pub const EBADFD: ::c_int = 77; +pub const EBADMSG: ::c_int = 74; +pub const EREMCHG: ::c_int = 78; +pub const ELIBACC: ::c_int = 79; +pub const ELIBBAD: ::c_int = 80; +pub const ELIBSCN: ::c_int = 81; +pub const ELIBMAX: ::c_int = 82; +pub const ELIBEXEC: ::c_int = 83; +pub const EILSEQ: ::c_int = 84; +pub const ERESTART: ::c_int = 85; +pub const ESTRPIPE: ::c_int = 86; +pub const EUSERS: ::c_int = 87; +pub const ENOTSOCK: ::c_int = 88; +pub const EDESTADDRREQ: ::c_int = 89; +pub const EMSGSIZE: ::c_int = 90; +pub const EPROTOTYPE: ::c_int = 91; +pub const ENOPROTOOPT: ::c_int = 92; +pub const EPROTONOSUPPORT: ::c_int = 93; +pub const ESOCKTNOSUPPORT: ::c_int = 94; +pub const EOPNOTSUPP: ::c_int = 95; +pub const EPFNOSUPPORT: ::c_int = 96; +pub const EAFNOSUPPORT: ::c_int = 97; +pub const EADDRINUSE: ::c_int = 98; +pub const EADDRNOTAVAIL: ::c_int = 99; +pub const ENETDOWN: ::c_int = 100; +pub const ENETUNREACH: ::c_int = 101; +pub const ENETRESET: ::c_int = 102; +pub const ECONNABORTED: ::c_int = 103; +pub const ECONNRESET: ::c_int = 104; +pub const ENOBUFS: ::c_int = 105; +pub const EISCONN: ::c_int = 106; +pub const ENOTCONN: ::c_int = 107; +pub const ESHUTDOWN: ::c_int = 108; +pub const ETOOMANYREFS: ::c_int = 109; +pub const ETIMEDOUT: ::c_int = 110; +pub const ECONNREFUSED: ::c_int = 111; +pub const EHOSTDOWN: ::c_int = 112; +pub const EHOSTUNREACH: ::c_int = 113; +pub const EALREADY: ::c_int = 114; +pub const EINPROGRESS: ::c_int = 115; +pub const ESTALE: ::c_int = 116; +pub const EUCLEAN: ::c_int = 117; +pub const ENOTNAM: ::c_int = 118; +pub const ENAVAIL: ::c_int = 119; +pub const EISNAM: ::c_int = 120; +pub const EREMOTEIO: ::c_int = 121; +pub const EDQUOT: ::c_int = 122; +pub const ENOMEDIUM: ::c_int = 123; +pub const EMEDIUMTYPE: ::c_int = 124; +pub const ECANCELED: ::c_int = 125; +pub const ENOKEY: ::c_int = 126; +pub const EKEYEXPIRED: ::c_int = 127; +pub const EKEYREVOKED: ::c_int = 128; +pub const EKEYREJECTED: ::c_int = 129; +pub const EOWNERDEAD: ::c_int = 130; +pub const ENOTRECOVERABLE: ::c_int = 131; +pub const ERFKILL: ::c_int = 132; +pub const EHWPOISON: ::c_int = 133; + +pub const MAP_NORESERVE: ::c_int = 0x4000; +pub const MAP_ANONYMOUS: ::c_int = 0x0020; +pub const MAP_ANON: ::c_int = 0x0020; +pub const MAP_GROWSDOWN: ::c_int = 0x0100; +pub const MAP_DENYWRITE: ::c_int = 0x0800; +pub const MAP_EXECUTABLE: ::c_int = 0x1000; +pub const MAP_LOCKED: ::c_int = 0x2000; +pub const MAP_POPULATE: ::c_int = 0x8000; +pub const MAP_NONBLOCK: ::c_int = 0x10000; +pub const MAP_STACK: ::c_int = 0x20000; +pub const MAP_HUGETLB: ::c_int = 0x40000; +pub const MCL_CURRENT: ::c_int = 0x0001; +pub const MCL_FUTURE: ::c_int = 0x0002; +pub const MCL_ONFAULT: ::c_int = 0x0004; + +pub const SOCK_STREAM: ::c_int = 1; +pub const SOCK_DGRAM: ::c_int = 2; + +pub const SFD_NONBLOCK: ::c_int = 0x800; +pub const SFD_CLOEXEC: ::c_int = 0x080000; +pub const SA_NODEFER: ::c_int = 0x40000000; +pub const SA_RESETHAND: ::c_int = 0x80000000; +pub const SA_RESTART: ::c_int = 0x10000000; +pub const SA_NOCLDSTOP: ::c_int = 0x00000001; +pub const SA_ONSTACK: ::c_int = 0x08000000; +pub const SA_SIGINFO: ::c_int = 0x00000004; +pub const SA_NOCLDWAIT: ::c_int = 0x00000002; +pub const SIG_BLOCK: ::c_int = 0; +pub const SIG_UNBLOCK: ::c_int = 1; +pub const SIG_SETMASK: ::c_int = 2; +pub const SIGBUS: ::c_int = 7; +pub const SIGUSR1: ::c_int = 10; +pub const SIGUSR2: ::c_int = 12; +pub const SIGSTKFLT: ::c_int = 16; +pub const SIGCHLD: ::c_int = 17; +pub const SIGCONT: ::c_int = 18; +pub const SIGSTOP: ::c_int = 19; +pub const SIGTSTP: ::c_int = 20; +pub const SIGTTIN: ::c_int = 21; +pub const SIGTTOU: ::c_int = 22; +pub const SIGURG: ::c_int = 23; +pub const SIGXCPU: ::c_int = 24; +pub const SIGXFSZ: ::c_int = 25; +pub const SIGVTALRM: ::c_int = 26; +pub const SIGPROF: ::c_int = 27; +pub const SIGWINCH: ::c_int = 28; +pub const SIGIO: ::c_int = 29; +pub const SIGPOLL: ::c_int = 29; +pub const SIGPWR: ::c_int = 30; +pub const SIGSYS: ::c_int = 31; +pub const SIGUNUSED: ::c_int = 31; + +pub const POLLWRNORM: ::c_short = 0x100; +pub const POLLWRBAND: ::c_short = 0x200; + +pub const PTRACE_GETFPREGS: ::c_uint = 14; +pub const PTRACE_SETFPREGS: ::c_uint = 15; +pub const PTRACE_DETACH: ::c_uint = 17; +pub const PTRACE_GETFPXREGS: ::c_uint = 18; +pub const PTRACE_SETFPXREGS: ::c_uint = 19; +pub const PTRACE_GETREGS: ::c_uint = 12; +pub const PTRACE_SETREGS: ::c_uint = 13; + +pub const RTLD_DEEPBIND: ::c_int = 0x8; +pub const RTLD_GLOBAL: ::c_int = 0x100; +pub const RTLD_NOLOAD: ::c_int = 0x4; + +pub const VEOF: usize = 4; +pub const VTIME: usize = 5; +pub const VMIN: usize = 6; +pub const VSWTC: usize = 7; +pub const VSTART: usize = 8; +pub const VSTOP: usize = 9; +pub const VSUSP: usize = 10; +pub const VEOL: usize = 11; +pub const VREPRINT: usize = 12; +pub const VDISCARD: usize = 13; +pub const VWERASE: usize = 14; +pub const VEOL2: usize = 16; +pub const IEXTEN: ::tcflag_t = 0x00008000; +pub const TOSTOP: ::tcflag_t = 0x00000100; +pub const FLUSHO: ::tcflag_t = 0x00001000; +pub const EXTPROC: ::tcflag_t = 0x00010000; +pub const TCSANOW: ::c_int = 0; +pub const TCSADRAIN: ::c_int = 1; +pub const TCSAFLUSH: ::c_int = 2; +pub const SIGSTKSZ: ::size_t = 16384; +pub const MINSIGSTKSZ: ::size_t = 4096; +pub const CBAUD: ::tcflag_t = 0o0010017; +pub const CSIZE: ::tcflag_t = 0x00000030; +pub const CS6: ::tcflag_t = 0x00000010; +pub const CS7: ::tcflag_t = 0x00000020; +pub const CS8: ::tcflag_t = 0x00000030; +pub const CSTOPB: ::tcflag_t = 0x00000040; +pub const CREAD: ::tcflag_t = 0x00000080; +pub const PARENB: ::tcflag_t = 0x00000100; +pub const PARODD: ::tcflag_t = 0x00000200; +pub const HUPCL: ::tcflag_t = 0x00000400; +pub const CLOCAL: ::tcflag_t = 0x00000800; +pub const CIBAUD: ::tcflag_t = 0o02003600000; +pub const CBAUDEX: ::tcflag_t = 0o010000; +pub const B0: ::speed_t = 0o000000; +pub const B50: ::speed_t = 0o000001; +pub const B75: ::speed_t = 0o000002; +pub const B110: ::speed_t = 0o000003; +pub const B134: ::speed_t = 0o000004; +pub const B150: ::speed_t = 0o000005; +pub const B200: ::speed_t = 0o000006; +pub const B300: ::speed_t = 0o000007; +pub const B600: ::speed_t = 0o000010; +pub const B1200: ::speed_t = 0o000011; +pub const B1800: ::speed_t = 0o000012; +pub const B2400: ::speed_t = 0o000013; +pub const B4800: ::speed_t = 0o000014; +pub const B9600: ::speed_t = 0o000015; +pub const B19200: ::speed_t = 0o000016; +pub const B38400: ::speed_t = 0o000017; +pub const EXTA: ::speed_t = B19200; +pub const EXTB: ::speed_t = B38400; +pub const B57600: ::speed_t = 0o010001; +pub const B115200: ::speed_t = 0o010002; +pub const B230400: ::speed_t = 0o010003; +pub const B460800: ::speed_t = 0o010004; +pub const B500000: ::speed_t = 0o010005; +pub const B576000: ::speed_t = 0o010006; +pub const B921600: ::speed_t = 0o010007; +pub const B1000000: ::speed_t = 0o010010; +pub const B1152000: ::speed_t = 0o010011; +pub const B1500000: ::speed_t = 0o010012; +pub const B2000000: ::speed_t = 0o010013; +pub const B2500000: ::speed_t = 0o010014; +pub const B3000000: ::speed_t = 0o010015; +pub const B3500000: ::speed_t = 0o010016; +pub const B4000000: ::speed_t = 0o010017; +pub const TAB1: ::tcflag_t = 0x00000800; +pub const TAB2: ::tcflag_t = 0x00001000; +pub const TAB3: ::tcflag_t = 0x00001800; +pub const CR1: ::tcflag_t = 0x00000200; +pub const CR2: ::tcflag_t = 0x00000400; +pub const CR3: ::tcflag_t = 0x00000600; +pub const FF1: ::tcflag_t = 0x00008000; +pub const BS1: ::tcflag_t = 0x00002000; +pub const OLCUC: ::tcflag_t = 0o000002; +pub const NLDLY: ::tcflag_t = 0o000400; +pub const CRDLY: ::tcflag_t = 0o003000; +pub const TABDLY: ::tcflag_t = 0o014000; +pub const BSDLY: ::tcflag_t = 0o020000; +pub const FFDLY: ::tcflag_t = 0o100000; +pub const VTDLY: ::tcflag_t = 0o040000; +pub const XTABS: ::tcflag_t = 0o014000; +pub const VT1: ::tcflag_t = 0x00004000; +pub const ONLCR: ::tcflag_t = 0x4; +pub const IXON: ::tcflag_t = 0x00000400; +pub const IXOFF: ::tcflag_t = 0x00001000; +pub const ECHOKE: ::tcflag_t = 0x00000800; +pub const ECHOE: ::tcflag_t = 0x00000010; +pub const ECHOK: ::tcflag_t = 0x00000020; +pub const ECHONL: ::tcflag_t = 0x00000040; +pub const ECHOPRT: ::tcflag_t = 0x00000400; +pub const ECHOCTL: ::tcflag_t = 0x00000200; +pub const ISIG: ::tcflag_t = 0x00000001; +pub const ICANON: ::tcflag_t = 0x00000002; +pub const PENDIN: ::tcflag_t = 0x00004000; +pub const NOFLSH: ::tcflag_t = 0x00000080; + +pub const NCCS: usize = 32; + +pub const EPOLL_CLOEXEC: ::c_int = 0x80000; + +pub const EFD_CLOEXEC: ::c_int = 0x80000; +pub const EFD_NONBLOCK: ::c_int = 0x800; + +cfg_if! { + if #[cfg(libc_align)] { + mod align; + pub use self::align::*; + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/mips64/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/mips64/mod.rs index d27ebc90a..0bf5084d5 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/mips64/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/mips64/mod.rs @@ -8,6 +8,7 @@ pub type nlink_t = u64; pub type suseconds_t = i64; pub type wchar_t = i32; pub type __u64 = ::c_ulong; +pub type __s64 = ::c_long; s! { pub struct stat { @@ -183,17 +184,6 @@ s! { __unused4: ::c_ulong, __unused5: ::c_ulong } - - pub struct termios2 { - pub c_iflag: ::tcflag_t, - pub c_oflag: ::tcflag_t, - pub c_cflag: ::tcflag_t, - pub c_lflag: ::tcflag_t, - pub c_line: ::cc_t, - pub c_cc: [::cc_t; 23], - pub c_ispeed: ::speed_t, - pub c_ospeed: ::speed_t, - } } pub const __SIZEOF_PTHREAD_CONDATTR_T: usize = 4; @@ -577,6 +567,34 @@ pub const SYS_pkey_mprotect: ::c_long = 5000 + 323; pub const SYS_pkey_alloc: ::c_long = 5000 + 324; pub const SYS_pkey_free: ::c_long = 5000 + 325; pub const SYS_statx: ::c_long = 5000 + 326; +pub const SYS_rseq: ::c_long = 5000 + 327; +pub const SYS_pidfd_send_signal: ::c_long = 5000 + 424; +pub const SYS_io_uring_setup: ::c_long = 5000 + 425; +pub const SYS_io_uring_enter: ::c_long = 5000 + 426; +pub const SYS_io_uring_register: ::c_long = 5000 + 427; +pub const SYS_open_tree: ::c_long = 5000 + 428; +pub const SYS_move_mount: ::c_long = 5000 + 429; +pub const SYS_fsopen: ::c_long = 5000 + 430; +pub const SYS_fsconfig: ::c_long = 5000 + 431; +pub const SYS_fsmount: ::c_long = 5000 + 432; +pub const SYS_fspick: ::c_long = 5000 + 433; +pub const SYS_pidfd_open: ::c_long = 5000 + 434; +pub const SYS_clone3: ::c_long = 5000 + 435; +pub const SYS_close_range: ::c_long = 5000 + 436; +pub const SYS_openat2: ::c_long = 5000 + 437; +pub const SYS_pidfd_getfd: ::c_long = 5000 + 438; +pub const SYS_faccessat2: ::c_long = 5000 + 439; +pub const SYS_process_madvise: ::c_long = 5000 + 440; +pub const SYS_epoll_pwait2: ::c_long = 5000 + 441; +pub const SYS_mount_setattr: ::c_long = 5000 + 442; +pub const SYS_quotactl_fd: ::c_long = 5000 + 443; +pub const SYS_landlock_create_ruleset: ::c_long = 5000 + 444; +pub const SYS_landlock_add_rule: ::c_long = 5000 + 445; +pub const SYS_landlock_restrict_self: ::c_long = 5000 + 446; +pub const SYS_memfd_secret: ::c_long = 5000 + 447; +pub const SYS_process_mrelease: ::c_long = 5000 + 448; +pub const SYS_futex_waitv: ::c_long = 5000 + 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 5000 + 450; pub const SFD_CLOEXEC: ::c_int = 0x080000; @@ -620,12 +638,6 @@ pub const O_DIRECT: ::c_int = 0x8000; pub const O_DIRECTORY: ::c_int = 0x10000; pub const O_NOFOLLOW: ::c_int = 0x20000; -pub const RLIMIT_NOFILE: ::__rlimit_resource_t = 5; -pub const RLIMIT_AS: ::__rlimit_resource_t = 6; -pub const RLIMIT_RSS: ::__rlimit_resource_t = 7; -pub const RLIMIT_NPROC: ::__rlimit_resource_t = 8; -pub const RLIMIT_MEMLOCK: ::__rlimit_resource_t = 9; - pub const O_APPEND: ::c_int = 8; pub const O_CREAT: ::c_int = 256; pub const O_EXCL: ::c_int = 1024; @@ -737,66 +749,6 @@ pub const MAP_HUGETLB: ::c_int = 0x080000; pub const SOCK_STREAM: ::c_int = 2; pub const SOCK_DGRAM: ::c_int = 1; -pub const SOL_SOCKET: ::c_int = 0xffff; - -pub const SO_REUSEADDR: ::c_int = 0x0004; -pub const SO_KEEPALIVE: ::c_int = 0x0008; -pub const SO_DONTROUTE: ::c_int = 0x0010; -pub const SO_BROADCAST: ::c_int = 0x0020; -pub const SO_LINGER: ::c_int = 0x0080; -pub const SO_OOBINLINE: ::c_int = 0x0100; -pub const SO_REUSEPORT: ::c_int = 0x0200; -pub const SO_TYPE: ::c_int = 0x1008; -pub const SO_STYLE: ::c_int = SO_TYPE; -pub const SO_ERROR: ::c_int = 0x1007; -pub const SO_SNDBUF: ::c_int = 0x1001; -pub const SO_RCVBUF: ::c_int = 0x1002; -pub const SO_SNDLOWAT: ::c_int = 0x1003; -pub const SO_RCVLOWAT: ::c_int = 0x1004; -pub const SO_SNDTIMEO: ::c_int = 0x1005; -pub const SO_RCVTIMEO: ::c_int = 0x1006; -pub const SO_ACCEPTCONN: ::c_int = 0x1009; -pub const SO_PROTOCOL: ::c_int = 0x1028; -pub const SO_DOMAIN: ::c_int = 0x1029; -pub const SO_NO_CHECK: ::c_int = 11; -pub const SO_PRIORITY: ::c_int = 12; -pub const SO_BSDCOMPAT: ::c_int = 14; -pub const SO_PASSCRED: ::c_int = 17; -pub const SO_PEERCRED: ::c_int = 18; -pub const SO_SECURITY_AUTHENTICATION: ::c_int = 22; -pub const SO_SECURITY_ENCRYPTION_TRANSPORT: ::c_int = 23; -pub const SO_SECURITY_ENCRYPTION_NETWORK: ::c_int = 24; -pub const SO_BINDTODEVICE: ::c_int = 25; -pub const SO_ATTACH_FILTER: ::c_int = 26; -pub const SO_DETACH_FILTER: ::c_int = 27; -pub const SO_GET_FILTER: ::c_int = SO_ATTACH_FILTER; -pub const SO_PEERNAME: ::c_int = 28; -pub const SO_TIMESTAMP: ::c_int = 29; -pub const SO_PEERSEC: ::c_int = 30; -pub const SO_SNDBUFFORCE: ::c_int = 31; -pub const SO_RCVBUFFORCE: ::c_int = 33; -pub const SO_PASSSEC: ::c_int = 34; -pub const SO_TIMESTAMPNS: ::c_int = 35; -pub const SCM_TIMESTAMPNS: ::c_int = SO_TIMESTAMPNS; -pub const SO_MARK: ::c_int = 36; -pub const SO_RXQ_OVFL: ::c_int = 40; -pub const SO_WIFI_STATUS: ::c_int = 41; -pub const SCM_WIFI_STATUS: ::c_int = SO_WIFI_STATUS; -pub const SO_PEEK_OFF: ::c_int = 42; -pub const SO_NOFCS: ::c_int = 43; -pub const SO_LOCK_FILTER: ::c_int = 44; -pub const SO_SELECT_ERR_QUEUE: ::c_int = 45; -pub const SO_BUSY_POLL: ::c_int = 46; -pub const SO_MAX_PACING_RATE: ::c_int = 47; -pub const SO_BPF_EXTENSIONS: ::c_int = 48; -pub const SO_INCOMING_CPU: ::c_int = 49; -pub const SO_ATTACH_BPF: ::c_int = 50; -pub const SO_DETACH_BPF: ::c_int = SO_DETACH_FILTER; - -pub const FIOCLEX: ::c_ulong = 0x6601; -pub const FIONCLEX: ::c_ulong = 0x6602; -pub const FIONBIO: ::c_ulong = 0x667e; - pub const SA_ONSTACK: ::c_int = 0x08000000; pub const SA_SIGINFO: ::c_int = 0x00000008; pub const SA_NOCLDWAIT: ::c_int = 0x00010000; @@ -863,38 +815,6 @@ pub const F_OFD_SETLKW: ::c_int = 38; pub const SFD_NONBLOCK: ::c_int = 0x80; -pub const TCGETS: ::c_ulong = 0x540d; -pub const TCSETS: ::c_ulong = 0x540e; -pub const TCSETSW: ::c_ulong = 0x540f; -pub const TCSETSF: ::c_ulong = 0x5410; -pub const TCGETA: ::c_ulong = 0x5401; -pub const TCSETA: ::c_ulong = 0x5402; -pub const TCSETAW: ::c_ulong = 0x5403; -pub const TCSETAF: ::c_ulong = 0x5404; -pub const TCSBRK: ::c_ulong = 0x5405; -pub const TCXONC: ::c_ulong = 0x5406; -pub const TCFLSH: ::c_ulong = 0x5407; -pub const TIOCGSOFTCAR: ::c_ulong = 0x5481; -pub const TIOCSSOFTCAR: ::c_ulong = 0x5482; -pub const TIOCINQ: ::c_ulong = 0x467f; -pub const TIOCLINUX: ::c_ulong = 0x5483; -pub const TIOCGSERIAL: ::c_ulong = 0x5484; -pub const TIOCEXCL: ::c_ulong = 0x740d; -pub const TIOCNXCL: ::c_ulong = 0x740e; -pub const TIOCSCTTY: ::c_ulong = 0x5480; -pub const TIOCGPGRP: ::c_ulong = 0x40047477; -pub const TIOCSPGRP: ::c_ulong = 0x80047476; -pub const TIOCOUTQ: ::c_ulong = 0x7472; -pub const TIOCSTI: ::c_ulong = 0x5472; -pub const TIOCGWINSZ: ::c_ulong = 0x40087468; -pub const TIOCSWINSZ: ::c_ulong = 0x80087467; -pub const TIOCMGET: ::c_ulong = 0x741d; -pub const TIOCMBIS: ::c_ulong = 0x741b; -pub const TIOCMBIC: ::c_ulong = 0x741c; -pub const TIOCMSET: ::c_ulong = 0x741a; -pub const FIONREAD: ::c_ulong = 0x467f; -pub const TIOCCONS: ::c_ulong = 0x80047478; - pub const RTLD_DEEPBIND: ::c_int = 0x10; pub const RTLD_GLOBAL: ::c_int = 0x4; pub const RTLD_NOLOAD: ::c_int = 0x8; @@ -974,7 +894,6 @@ pub const B19200: ::speed_t = 0o000016; pub const B38400: ::speed_t = 0o000017; pub const EXTA: ::speed_t = B19200; pub const EXTB: ::speed_t = B38400; -pub const BOTHER: ::speed_t = 0o010000; pub const B57600: ::speed_t = 0o010001; pub const B115200: ::speed_t = 0o010002; pub const B230400: ::speed_t = 0o010003; @@ -991,16 +910,8 @@ pub const B3000000: ::speed_t = 0o010015; pub const B3500000: ::speed_t = 0o010016; pub const B4000000: ::speed_t = 0o010017; -pub const TIOCM_ST: ::c_int = 0x010; -pub const TIOCM_SR: ::c_int = 0x020; -pub const TIOCM_CTS: ::c_int = 0x040; -pub const TIOCM_CAR: ::c_int = 0x100; -pub const TIOCM_RNG: ::c_int = 0x200; -pub const TIOCM_DSR: ::c_int = 0x400; - pub const EHWPOISON: ::c_int = 168; -#[link(name = "util")] extern "C" { pub fn sysctl( name: *mut ::c_int, diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/mod.rs index 40ce8441a..443958cff 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/mod.rs @@ -1,7 +1,5 @@ //! 64-bit specific definitions for linux-like values -pub type clock_t = i64; -pub type time_t = i64; pub type ino_t = u64; pub type off_t = i64; pub type blkcnt_t = i64; @@ -11,7 +9,22 @@ pub type msglen_t = u64; pub type fsblkcnt_t = u64; pub type fsfilcnt_t = u64; pub type rlim_t = u64; -pub type __fsword_t = i64; +#[cfg(all(target_arch = "x86_64", target_pointer_width = "32"))] +pub type __syscall_ulong_t = ::c_ulonglong; +#[cfg(not(all(target_arch = "x86_64", target_pointer_width = "32")))] +pub type __syscall_ulong_t = ::c_ulong; + +cfg_if! { + if #[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))] { + pub type clock_t = i32; + pub type time_t = i32; + pub type __fsword_t = i32; + } else { + pub type __fsword_t = i64; + pub type clock_t = i64; + pub type time_t = i64; + } +} s! { pub struct sigset_t { @@ -52,9 +65,32 @@ s! { __glibc_reserved5: u64, } + pub struct semid_ds { + pub sem_perm: ipc_perm, + pub sem_otime: ::time_t, + #[cfg(not(any( + target_arch = "aarch64", + target_arch = "loongarch64", + target_arch = "mips64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "sparc64")))] + __reserved: ::__syscall_ulong_t, + pub sem_ctime: ::time_t, + #[cfg(not(any( + target_arch = "aarch64", + target_arch = "loongarch64", + target_arch = "mips64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "sparc64")))] + __reserved2: ::__syscall_ulong_t, + pub sem_nsems: ::__syscall_ulong_t, + __glibc_reserved3: ::__syscall_ulong_t, + __glibc_reserved4: ::__syscall_ulong_t, + } } -pub const RLIM_INFINITY: ::rlim_t = !0; pub const __SIZEOF_PTHREAD_RWLOCKATTR_T: usize = 8; pub const O_LARGEFILE: ::c_int = 0; @@ -81,6 +117,9 @@ cfg_if! { } else if #[cfg(any(target_arch = "riscv64"))] { mod riscv64; pub use self::riscv64::*; + } else if #[cfg(any(target_arch = "loongarch64"))] { + mod loongarch64; + pub use self::loongarch64::*; } else { // Unknown target_arch } diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/powerpc64/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/powerpc64/mod.rs index d77c6fa86..ce8ce97bb 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/powerpc64/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/powerpc64/mod.rs @@ -10,6 +10,7 @@ pub type nlink_t = u64; pub type blksize_t = i64; pub type suseconds_t = i64; pub type __u64 = ::c_ulong; +pub type __s64 = ::c_long; s! { pub struct sigaction { @@ -201,15 +202,6 @@ pub const RTLD_NOLOAD: ::c_int = 0x4; pub const VEOF: usize = 4; pub const __SIZEOF_PTHREAD_RWLOCK_T: usize = 56; -pub const TIOCGSOFTCAR: ::c_ulong = 0x5419; -pub const TIOCSSOFTCAR: ::c_ulong = 0x541A; - -pub const RLIMIT_RSS: ::__rlimit_resource_t = 5; -pub const RLIMIT_AS: ::__rlimit_resource_t = 9; -pub const RLIMIT_MEMLOCK: ::__rlimit_resource_t = 8; -pub const RLIMIT_NOFILE: ::__rlimit_resource_t = 7; -pub const RLIMIT_NPROC: ::__rlimit_resource_t = 6; - pub const O_APPEND: ::c_int = 1024; pub const O_CREAT: ::c_int = 64; pub const O_EXCL: ::c_int = 128; @@ -313,61 +305,6 @@ pub const ENOTRECOVERABLE: ::c_int = 131; pub const EHWPOISON: ::c_int = 133; pub const ERFKILL: ::c_int = 132; -pub const SOL_SOCKET: ::c_int = 1; - -pub const SO_REUSEADDR: ::c_int = 2; -pub const SO_TYPE: ::c_int = 3; -pub const SO_ERROR: ::c_int = 4; -pub const SO_DONTROUTE: ::c_int = 5; -pub const SO_BROADCAST: ::c_int = 6; -pub const SO_SNDBUF: ::c_int = 7; -pub const SO_RCVBUF: ::c_int = 8; -pub const SO_SNDBUFFORCE: ::c_int = 32; -pub const SO_RCVBUFFORCE: ::c_int = 33; -pub const SO_KEEPALIVE: ::c_int = 9; -pub const SO_OOBINLINE: ::c_int = 10; -pub const SO_NO_CHECK: ::c_int = 11; -pub const SO_PRIORITY: ::c_int = 12; -pub const SO_LINGER: ::c_int = 13; -pub const SO_BSDCOMPAT: ::c_int = 14; -pub const SO_REUSEPORT: ::c_int = 15; -pub const SO_PASSCRED: ::c_int = 20; -pub const SO_PEERCRED: ::c_int = 21; -pub const SO_RCVLOWAT: ::c_int = 16; -pub const SO_SNDLOWAT: ::c_int = 17; -pub const SO_RCVTIMEO: ::c_int = 18; -pub const SO_SNDTIMEO: ::c_int = 19; -pub const SO_SECURITY_AUTHENTICATION: ::c_int = 22; -pub const SO_SECURITY_ENCRYPTION_TRANSPORT: ::c_int = 23; -pub const SO_SECURITY_ENCRYPTION_NETWORK: ::c_int = 24; -pub const SO_BINDTODEVICE: ::c_int = 25; -pub const SO_ATTACH_FILTER: ::c_int = 26; -pub const SO_DETACH_FILTER: ::c_int = 27; -pub const SO_GET_FILTER: ::c_int = SO_ATTACH_FILTER; -pub const SO_PEERNAME: ::c_int = 28; -pub const SO_TIMESTAMP: ::c_int = 29; -pub const SO_ACCEPTCONN: ::c_int = 30; -pub const SO_PEERSEC: ::c_int = 31; -pub const SO_PASSSEC: ::c_int = 34; -pub const SO_TIMESTAMPNS: ::c_int = 35; -pub const SCM_TIMESTAMPNS: ::c_int = SO_TIMESTAMPNS; -pub const SO_MARK: ::c_int = 36; -pub const SO_PROTOCOL: ::c_int = 38; -pub const SO_DOMAIN: ::c_int = 39; -pub const SO_RXQ_OVFL: ::c_int = 40; -pub const SO_WIFI_STATUS: ::c_int = 41; -pub const SCM_WIFI_STATUS: ::c_int = SO_WIFI_STATUS; -pub const SO_PEEK_OFF: ::c_int = 42; -pub const SO_NOFCS: ::c_int = 43; -pub const SO_LOCK_FILTER: ::c_int = 44; -pub const SO_SELECT_ERR_QUEUE: ::c_int = 45; -pub const SO_BUSY_POLL: ::c_int = 46; -pub const SO_MAX_PACING_RATE: ::c_int = 47; -pub const SO_BPF_EXTENSIONS: ::c_int = 48; -pub const SO_INCOMING_CPU: ::c_int = 49; -pub const SO_ATTACH_BPF: ::c_int = 50; -pub const SO_DETACH_BPF: ::c_int = SO_DETACH_FILTER; - pub const SOCK_STREAM: ::c_int = 1; pub const SOCK_DGRAM: ::c_int = 2; @@ -430,27 +367,6 @@ pub const TCSANOW: ::c_int = 0; pub const TCSADRAIN: ::c_int = 1; pub const TCSAFLUSH: ::c_int = 2; -pub const TIOCLINUX: ::c_ulong = 0x541C; -pub const TIOCGSERIAL: ::c_ulong = 0x541E; -pub const TIOCEXCL: ::c_ulong = 0x540C; -pub const TIOCNXCL: ::c_ulong = 0x540D; -pub const TIOCSCTTY: ::c_ulong = 0x540E; -pub const TIOCSTI: ::c_ulong = 0x5412; -pub const TIOCMGET: ::c_ulong = 0x5415; -pub const TIOCMBIS: ::c_ulong = 0x5416; -pub const TIOCMBIC: ::c_ulong = 0x5417; -pub const TIOCMSET: ::c_ulong = 0x5418; -pub const TIOCCONS: ::c_ulong = 0x541D; -pub const TIOCGRS485: ::c_int = 0x542E; -pub const TIOCSRS485: ::c_int = 0x542F; - -pub const TIOCM_ST: ::c_int = 0x008; -pub const TIOCM_SR: ::c_int = 0x010; -pub const TIOCM_CTS: ::c_int = 0x020; -pub const TIOCM_CAR: ::c_int = 0x040; -pub const TIOCM_RNG: ::c_int = 0x080; -pub const TIOCM_DSR: ::c_int = 0x100; - pub const SFD_CLOEXEC: ::c_int = 0x080000; pub const NCCS: usize = 32; @@ -544,7 +460,7 @@ pub const O_DIRECT: ::c_int = 0x20000; pub const MAP_LOCKED: ::c_int = 0x00080; pub const MAP_NORESERVE: ::c_int = 0x00040; -pub const MAP_SYNC : ::c_int = 0x080000; +pub const MAP_SYNC: ::c_int = 0x080000; pub const EDEADLOCK: ::c_int = 58; pub const EUCLEAN: ::c_int = 117; @@ -553,10 +469,6 @@ pub const ENAVAIL: ::c_int = 119; pub const EISNAM: ::c_int = 120; pub const EREMOTEIO: ::c_int = 121; -pub const FIOCLEX: ::c_ulong = 0x20006601; -pub const FIONCLEX: ::c_ulong = 0x20006602; -pub const FIONBIO: ::c_ulong = 0x8004667e; - pub const MCL_CURRENT: ::c_int = 0x2000; pub const MCL_FUTURE: ::c_int = 0x4000; @@ -646,7 +558,6 @@ pub const B2500000: ::speed_t = 0o0033; pub const B3000000: ::speed_t = 0o0034; pub const B3500000: ::speed_t = 0o0035; pub const B4000000: ::speed_t = 0o0036; -pub const BOTHER: ::speed_t = 0o0037; pub const VEOL: usize = 6; pub const VEOL2: usize = 8; @@ -655,24 +566,6 @@ pub const IEXTEN: ::tcflag_t = 0x400; pub const TOSTOP: ::tcflag_t = 0x400000; pub const FLUSHO: ::tcflag_t = 0x800000; pub const EXTPROC: ::tcflag_t = 0x10000000; -pub const TCGETS: ::c_ulong = 0x403c7413; -pub const TCSETS: ::c_ulong = 0x803c7414; -pub const TCSETSW: ::c_ulong = 0x803c7415; -pub const TCSETSF: ::c_ulong = 0x803c7416; -pub const TCGETA: ::c_ulong = 0x40147417; -pub const TCSETA: ::c_ulong = 0x80147418; -pub const TCSETAW: ::c_ulong = 0x80147419; -pub const TCSETAF: ::c_ulong = 0x8014741c; -pub const TCSBRK: ::c_ulong = 0x2000741d; -pub const TCXONC: ::c_ulong = 0x2000741e; -pub const TCFLSH: ::c_ulong = 0x2000741f; -pub const TIOCINQ: ::c_ulong = 0x4004667f; -pub const TIOCGPGRP: ::c_ulong = 0x40047477; -pub const TIOCSPGRP: ::c_ulong = 0x80047476; -pub const TIOCOUTQ: ::c_ulong = 0x40047473; -pub const TIOCGWINSZ: ::c_ulong = 0x40087468; -pub const TIOCSWINSZ: ::c_ulong = 0x80087467; -pub const FIONREAD: ::c_ulong = 0x4004667f; // Syscall table pub const SYS_restart_syscall: ::c_long = 0; @@ -1035,8 +928,35 @@ pub const SYS_preadv2: ::c_long = 380; pub const SYS_pwritev2: ::c_long = 381; pub const SYS_kexec_file_load: ::c_long = 382; pub const SYS_statx: ::c_long = 383; +pub const SYS_rseq: ::c_long = 387; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; +pub const SYS_quotactl_fd: ::c_long = 443; +pub const SYS_landlock_create_ruleset: ::c_long = 444; +pub const SYS_landlock_add_rule: ::c_long = 445; +pub const SYS_landlock_restrict_self: ::c_long = 446; +pub const SYS_memfd_secret: ::c_long = 447; +pub const SYS_process_mrelease: ::c_long = 448; +pub const SYS_futex_waitv: ::c_long = 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 450; -#[link(name = "util")] extern "C" { pub fn sysctl( name: *mut ::c_int, diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/riscv64/align.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/riscv64/align.rs new file mode 100644 index 000000000..48d152a57 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/riscv64/align.rs @@ -0,0 +1,44 @@ +s_no_extra_traits! { + #[allow(missing_debug_implementations)] + pub struct ucontext_t { + pub __uc_flags: ::c_ulong, + pub uc_link: *mut ucontext_t, + pub uc_stack: ::stack_t, + pub uc_sigmask: ::sigset_t, + pub uc_mcontext: mcontext_t, + } + + #[allow(missing_debug_implementations)] + #[repr(align(16))] + pub struct mcontext_t { + pub __gregs: [::c_ulong; 32], + pub __fpregs: __riscv_mc_fp_state, + } + + #[allow(missing_debug_implementations)] + pub union __riscv_mc_fp_state { + pub __f: __riscv_mc_f_ext_state, + pub __d: __riscv_mc_d_ext_state, + pub __q: __riscv_mc_q_ext_state, + } + + #[allow(missing_debug_implementations)] + pub struct __riscv_mc_f_ext_state { + pub __f: [::c_uint; 32], + pub __fcsr: ::c_uint, + } + + #[allow(missing_debug_implementations)] + pub struct __riscv_mc_d_ext_state { + pub __f: [::c_ulonglong; 32], + pub __fcsr: ::c_uint, + } + + #[allow(missing_debug_implementations)] + #[repr(align(16))] + pub struct __riscv_mc_q_ext_state { + pub __f: [::c_ulonglong; 64], + pub __fcsr: ::c_uint, + pub __glibc_reserved: [::c_uint; 3], + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/riscv64/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/riscv64/mod.rs index 047505fc1..9d022f96e 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/riscv64/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/riscv64/mod.rs @@ -4,89 +4,72 @@ pub type c_char = u8; pub type c_long = i64; pub type c_ulong = u64; pub type wchar_t = ::c_int; -pub type time_t = ::c_long; -pub type dev_t = ::c_ulong; -pub type uid_t = ::c_uint; -pub type gid_t = ::c_uint; -pub type ino_t = ::c_ulong; -pub type ino64_t = ::c_ulong; -pub type mode_t = ::c_uint; pub type nlink_t = ::c_uint; -pub type off_t = ::c_long; -pub type off64_t = ::c_long; -pub type pid_t = ::c_int; pub type blksize_t = ::c_int; -pub type blkcnt_t = ::c_long; -pub type fsblkcnt_t = ::c_ulong; pub type fsblkcnt64_t = ::c_ulong; -pub type fsfilcnt_t = ::c_ulong; pub type fsfilcnt64_t = ::c_ulong; pub type suseconds_t = i64; pub type __u64 = ::c_ulonglong; +pub type __s64 = ::c_longlong; s! { pub struct pthread_attr_t { __size: [::c_ulong; 7], } - pub struct timespec { - pub tv_sec: time_t, - pub tv_nsec: ::c_long, - } - pub struct stat { - pub st_dev: dev_t, - pub st_ino: ino_t, - pub st_mode: mode_t, - pub st_nlink: nlink_t, - pub st_uid: uid_t, - pub st_gid: gid_t, - pub st_rdev: dev_t, - pub __pad1: dev_t, - pub st_size: off_t, - pub st_blksize: blksize_t, + pub st_dev: ::dev_t, + pub st_ino: ::ino_t, + pub st_mode: ::mode_t, + pub st_nlink: ::nlink_t, + pub st_uid: ::uid_t, + pub st_gid: ::gid_t, + pub st_rdev: ::dev_t, + pub __pad1: ::dev_t, + pub st_size: ::off_t, + pub st_blksize: ::blksize_t, pub __pad2: ::c_int, - pub st_blocks: blkcnt_t, - pub st_atime: time_t, + pub st_blocks: ::blkcnt_t, + pub st_atime: ::time_t, pub st_atime_nsec: ::c_long, - pub st_mtime: time_t, + pub st_mtime: ::time_t, pub st_mtime_nsec: ::c_long, - pub st_ctime: time_t, + pub st_ctime: ::time_t, pub st_ctime_nsec: ::c_long, - pub __unused: [::c_int; 2usize], + __unused: [::c_int; 2usize], } pub struct stat64 { - pub st_dev: dev_t, - pub st_ino: ino64_t, - pub st_mode: mode_t, - pub st_nlink: nlink_t, - pub st_uid: uid_t, - pub st_gid: gid_t, - pub st_rdev: dev_t, - pub __pad1: dev_t, - pub st_size: off64_t, - pub st_blksize: blksize_t, + pub st_dev: ::dev_t, + pub st_ino: ::ino64_t, + pub st_mode: ::mode_t, + pub st_nlink: ::nlink_t, + pub st_uid: ::uid_t, + pub st_gid: ::gid_t, + pub st_rdev: ::dev_t, + pub __pad1: ::dev_t, + pub st_size: ::off64_t, + pub st_blksize: ::blksize_t, pub __pad2: ::c_int, - pub st_blocks: blkcnt_t, - pub st_atime: time_t, + pub st_blocks: ::blkcnt_t, + pub st_atime: ::time_t, pub st_atime_nsec: ::c_long, - pub st_mtime: time_t, + pub st_mtime: ::time_t, pub st_mtime_nsec: ::c_long, - pub st_ctime: time_t, + pub st_ctime: ::time_t, pub st_ctime_nsec: ::c_long, - pub __unused: [::c_int; 2], + __unused: [::c_int; 2], } pub struct statfs { pub f_type: ::c_long, pub f_bsize: ::c_long, - pub f_blocks: fsblkcnt_t, - pub f_bfree: fsblkcnt_t, - pub f_bavail: fsblkcnt_t, - pub f_files: fsfilcnt_t, - pub f_ffree: fsfilcnt_t, + pub f_blocks: ::fsblkcnt_t, + pub f_bfree: ::fsblkcnt_t, + pub f_bavail: ::fsblkcnt_t, + pub f_files: ::fsfilcnt_t, + pub f_ffree: ::fsfilcnt_t, pub f_fsid: ::fsid_t, pub f_namelen: ::c_long, pub f_frsize: ::c_long, @@ -97,11 +80,11 @@ s! { pub struct statfs64 { pub f_type: ::c_long, pub f_bsize: ::c_long, - pub f_blocks: fsblkcnt64_t, - pub f_bfree: fsblkcnt64_t, - pub f_bavail: fsblkcnt64_t, - pub f_files: fsfilcnt64_t, - pub f_ffree: fsfilcnt64_t, + pub f_blocks: ::fsblkcnt64_t, + pub f_bfree: ::fsblkcnt64_t, + pub f_bavail: ::fsblkcnt64_t, + pub f_files: ::fsfilcnt64_t, + pub f_ffree: ::fsfilcnt64_t, pub f_fsid: ::fsid_t, pub f_namelen: ::c_long, pub f_frsize: ::c_long, @@ -112,12 +95,12 @@ s! { pub struct statvfs { pub f_bsize: ::c_ulong, pub f_frsize: ::c_ulong, - pub f_blocks: fsblkcnt_t, - pub f_bfree: fsblkcnt_t, - pub f_bavail: fsblkcnt_t, - pub f_files: fsfilcnt_t, - pub f_ffree: fsfilcnt_t, - pub f_favail: fsfilcnt_t, + pub f_blocks: ::fsblkcnt_t, + pub f_bfree: ::fsblkcnt_t, + pub f_bavail: ::fsblkcnt_t, + pub f_files: ::fsfilcnt_t, + pub f_ffree: ::fsfilcnt_t, + pub f_favail: ::fsfilcnt_t, pub f_fsid: ::c_ulong, pub f_flag: ::c_ulong, pub f_namemax: ::c_ulong, @@ -127,12 +110,12 @@ s! { pub struct statvfs64 { pub f_bsize: ::c_ulong, pub f_frsize: ::c_ulong, - pub f_blocks: fsblkcnt64_t, - pub f_bfree: fsblkcnt64_t, - pub f_bavail: fsblkcnt64_t, - pub f_files: fsfilcnt64_t, - pub f_ffree: fsfilcnt64_t, - pub f_favail: fsfilcnt64_t, + pub f_blocks: ::fsblkcnt64_t, + pub f_bfree: ::fsblkcnt64_t, + pub f_bavail: ::fsblkcnt64_t, + pub f_files: ::fsfilcnt64_t, + pub f_ffree: ::fsfilcnt64_t, + pub f_favail: ::fsfilcnt64_t, pub f_fsid: ::c_ulong, pub f_flag: ::c_ulong, pub f_namemax: ::c_ulong, @@ -167,35 +150,31 @@ s! { pub sa_restorer: ::Option, } - pub struct sigset_t { - pub __val: [::c_ulong; 16], - } - pub struct ipc_perm { pub __key: ::key_t, - pub uid: uid_t, - pub gid: gid_t, - pub cuid: uid_t, - pub cgid: gid_t, + pub uid: ::uid_t, + pub gid: ::gid_t, + pub cuid: ::uid_t, + pub cgid: ::gid_t, pub mode: ::c_ushort, - pub __pad1: ::c_ushort, + __pad1: ::c_ushort, pub __seq: ::c_ushort, - pub __pad2: ::c_ushort, - pub __unused1: ::c_ulong, - pub __unused2: ::c_ulong, + __pad2: ::c_ushort, + __unused1: ::c_ulong, + __unused2: ::c_ulong, } pub struct shmid_ds { - pub shm_perm: ipc_perm, + pub shm_perm: ::ipc_perm, pub shm_segsz: ::size_t, - pub shm_atime: time_t, - pub shm_dtime: time_t, - pub shm_ctime: time_t, - pub shm_cpid: pid_t, - pub shm_lpid: pid_t, + pub shm_atime: ::time_t, + pub shm_dtime: ::time_t, + pub shm_ctime: ::time_t, + pub shm_cpid: ::pid_t, + pub shm_lpid: ::pid_t, pub shm_nattch: ::shmatt_t, - pub __unused5: ::c_ulong, - pub __unused6: ::c_ulong, + __unused5: ::c_ulong, + __unused6: ::c_ulong, } pub struct flock { @@ -221,10 +200,6 @@ pub const VEOF: usize = 4; pub const RTLD_DEEPBIND: ::c_int = 0x8; pub const RTLD_GLOBAL: ::c_int = 0x100; pub const RTLD_NOLOAD: ::c_int = 0x4; -pub const TIOCGSOFTCAR: ::c_ulong = 21529; -pub const TIOCSSOFTCAR: ::c_ulong = 21530; -pub const TIOCGRS485: ::c_int = 21550; -pub const TIOCSRS485: ::c_int = 21551; pub const O_APPEND: ::c_int = 1024; pub const O_CREAT: ::c_int = 64; pub const O_EXCL: ::c_int = 128; @@ -237,6 +212,7 @@ pub const O_FSYNC: ::c_int = 1052672; pub const O_NOATIME: ::c_int = 262144; pub const O_PATH: ::c_int = 2097152; pub const O_TMPFILE: ::c_int = 4259840; +pub const MADV_SOFT_OFFLINE: ::c_int = 101; pub const MAP_GROWSDOWN: ::c_int = 256; pub const EDEADLK: ::c_int = 35; pub const ENAMETOOLONG: ::c_int = 36; @@ -316,59 +292,7 @@ pub const EOWNERDEAD: ::c_int = 130; pub const ENOTRECOVERABLE: ::c_int = 131; pub const EHWPOISON: ::c_int = 133; pub const ERFKILL: ::c_int = 132; -pub const SOL_SOCKET: ::c_int = 1; -pub const SO_REUSEADDR: ::c_int = 2; -pub const SO_TYPE: ::c_int = 3; -pub const SO_ERROR: ::c_int = 4; -pub const SO_DONTROUTE: ::c_int = 5; -pub const SO_BROADCAST: ::c_int = 6; -pub const SO_SNDBUF: ::c_int = 7; -pub const SO_RCVBUF: ::c_int = 8; -pub const SO_SNDBUFFORCE: ::c_int = 32; -pub const SO_RCVBUFFORCE: ::c_int = 33; -pub const SO_KEEPALIVE: ::c_int = 9; -pub const SO_OOBINLINE: ::c_int = 10; -pub const SO_NO_CHECK: ::c_int = 11; -pub const SO_PRIORITY: ::c_int = 12; -pub const SO_LINGER: ::c_int = 13; -pub const SO_BSDCOMPAT: ::c_int = 14; -pub const SO_REUSEPORT: ::c_int = 15; -pub const SO_PASSCRED: ::c_int = 16; -pub const SO_PEERCRED: ::c_int = 17; -pub const SO_RCVLOWAT: ::c_int = 18; -pub const SO_SNDLOWAT: ::c_int = 19; -pub const SO_RCVTIMEO: ::c_int = 20; -pub const SO_SNDTIMEO: ::c_int = 21; -pub const SO_SECURITY_AUTHENTICATION: ::c_int = 22; -pub const SO_SECURITY_ENCRYPTION_TRANSPORT: ::c_int = 23; -pub const SO_SECURITY_ENCRYPTION_NETWORK: ::c_int = 24; -pub const SO_BINDTODEVICE: ::c_int = 25; -pub const SO_ATTACH_FILTER: ::c_int = 26; -pub const SO_DETACH_FILTER: ::c_int = 27; -pub const SO_GET_FILTER: ::c_int = 26; -pub const SO_PEERNAME: ::c_int = 28; -pub const SO_TIMESTAMP: ::c_int = 29; -pub const SO_ACCEPTCONN: ::c_int = 30; -pub const SO_PEERSEC: ::c_int = 31; -pub const SO_PASSSEC: ::c_int = 34; -pub const SO_TIMESTAMPNS: ::c_int = 35; -pub const SCM_TIMESTAMPNS: ::c_int = 35; -pub const SO_MARK: ::c_int = 36; -pub const SO_PROTOCOL: ::c_int = 38; -pub const SO_DOMAIN: ::c_int = 39; -pub const SO_RXQ_OVFL: ::c_int = 40; -pub const SO_WIFI_STATUS: ::c_int = 41; -pub const SCM_WIFI_STATUS: ::c_int = 41; -pub const SO_PEEK_OFF: ::c_int = 42; -pub const SO_NOFCS: ::c_int = 43; -pub const SO_LOCK_FILTER: ::c_int = 44; -pub const SO_SELECT_ERR_QUEUE: ::c_int = 45; -pub const SO_BUSY_POLL: ::c_int = 46; -pub const SO_MAX_PACING_RATE: ::c_int = 47; -pub const SO_BPF_EXTENSIONS: ::c_int = 48; -pub const SO_INCOMING_CPU: ::c_int = 49; -pub const SO_ATTACH_BPF: ::c_int = 50; -pub const SO_DETACH_BPF: ::c_int = 27; + pub const SOCK_STREAM: ::c_int = 1; pub const SOCK_DGRAM: ::c_int = 2; pub const SA_ONSTACK: ::c_int = 134217728; @@ -418,23 +342,6 @@ pub const SFD_NONBLOCK: ::c_int = 2048; pub const TCSANOW: ::c_int = 0; pub const TCSADRAIN: ::c_int = 1; pub const TCSAFLUSH: ::c_int = 2; -pub const TIOCLINUX: ::c_ulong = 21532; -pub const TIOCGSERIAL: ::c_ulong = 21534; -pub const TIOCEXCL: ::c_ulong = 21516; -pub const TIOCNXCL: ::c_ulong = 21517; -pub const TIOCSCTTY: ::c_ulong = 21518; -pub const TIOCSTI: ::c_ulong = 21522; -pub const TIOCMGET: ::c_ulong = 21525; -pub const TIOCMBIS: ::c_ulong = 21526; -pub const TIOCMBIC: ::c_ulong = 21527; -pub const TIOCMSET: ::c_ulong = 21528; -pub const TIOCCONS: ::c_ulong = 21533; -pub const TIOCM_ST: ::c_int = 8; -pub const TIOCM_SR: ::c_int = 16; -pub const TIOCM_CTS: ::c_int = 32; -pub const TIOCM_CAR: ::c_int = 64; -pub const TIOCM_RNG: ::c_int = 128; -pub const TIOCM_DSR: ::c_int = 256; pub const SFD_CLOEXEC: ::c_int = 524288; pub const NCCS: usize = 32; pub const O_TRUNC: ::c_int = 512; @@ -474,16 +381,19 @@ pub const MAP_EXECUTABLE: ::c_int = 4096; pub const MAP_POPULATE: ::c_int = 32768; pub const MAP_NONBLOCK: ::c_int = 65536; pub const MAP_STACK: ::c_int = 131072; -pub const MAP_SYNC : ::c_int = 0x080000; +pub const MAP_SYNC: ::c_int = 0x080000; pub const EDEADLOCK: ::c_int = 35; pub const EUCLEAN: ::c_int = 117; pub const ENOTNAM: ::c_int = 118; pub const ENAVAIL: ::c_int = 119; pub const EISNAM: ::c_int = 120; pub const EREMOTEIO: ::c_int = 121; -pub const FIOCLEX: ::c_ulong = 21585; -pub const FIONCLEX: ::c_ulong = 21584; -pub const FIONBIO: ::c_ulong = 21537; +pub const PTRACE_GETFPREGS: ::c_uint = 14; +pub const PTRACE_SETFPREGS: ::c_uint = 15; +pub const PTRACE_GETFPXREGS: ::c_uint = 18; +pub const PTRACE_SETFPXREGS: ::c_uint = 19; +pub const PTRACE_GETREGS: ::c_uint = 12; +pub const PTRACE_SETREGS: ::c_uint = 13; pub const MCL_CURRENT: ::c_int = 1; pub const MCL_FUTURE: ::c_int = 2; pub const SIGSTKSZ: ::size_t = 8192; @@ -579,26 +489,19 @@ pub const IEXTEN: ::tcflag_t = 32768; pub const TOSTOP: ::tcflag_t = 256; pub const FLUSHO: ::tcflag_t = 4096; pub const EXTPROC: ::tcflag_t = 65536; -pub const TCGETS: ::c_ulong = 21505; -pub const TCSETS: ::c_ulong = 21506; -pub const TCSETSW: ::c_ulong = 21507; -pub const TCSETSF: ::c_ulong = 21508; -pub const TCGETA: ::c_ulong = 21509; -pub const TCSETA: ::c_ulong = 21510; -pub const TCSETAW: ::c_ulong = 21511; -pub const TCSETAF: ::c_ulong = 21512; -pub const TCSBRK: ::c_ulong = 21513; -pub const TCXONC: ::c_ulong = 21514; -pub const TCFLSH: ::c_ulong = 21515; -pub const TIOCINQ: ::c_ulong = 21531; -pub const TIOCGPGRP: ::c_ulong = 21519; -pub const TIOCSPGRP: ::c_ulong = 21520; -pub const TIOCOUTQ: ::c_ulong = 21521; -pub const TIOCGWINSZ: ::c_ulong = 21523; -pub const TIOCSWINSZ: ::c_ulong = 21524; -pub const FIONREAD: ::c_ulong = 21531; pub const __SIZEOF_PTHREAD_MUTEX_T: usize = 40; pub const __SIZEOF_PTHREAD_RWLOCK_T: usize = 56; +pub const NGREG: usize = 32; +pub const REG_PC: usize = 0; +pub const REG_RA: usize = 1; +pub const REG_SP: usize = 2; +pub const REG_TP: usize = 4; +pub const REG_S0: usize = 8; +pub const REG_S1: usize = 9; +pub const REG_A0: usize = 10; +pub const REG_S2: usize = 18; +pub const REG_NARGS: usize = 8; + pub const SYS_read: ::c_long = 63; pub const SYS_write: ::c_long = 64; pub const SYS_close: ::c_long = 57; @@ -874,3 +777,38 @@ pub const SYS_pkey_mprotect: ::c_long = 288; pub const SYS_pkey_alloc: ::c_long = 289; pub const SYS_pkey_free: ::c_long = 290; pub const SYS_statx: ::c_long = 291; +pub const SYS_rseq: ::c_long = 293; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; +pub const SYS_quotactl_fd: ::c_long = 443; +pub const SYS_landlock_create_ruleset: ::c_long = 444; +pub const SYS_landlock_add_rule: ::c_long = 445; +pub const SYS_landlock_restrict_self: ::c_long = 446; +pub const SYS_memfd_secret: ::c_long = 447; +pub const SYS_process_mrelease: ::c_long = 448; +pub const SYS_futex_waitv: ::c_long = 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 450; + +cfg_if! { + if #[cfg(libc_align)] { + mod align; + pub use self::align::*; + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/s390x.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/s390x.rs index e2f3b9bcf..c4bae089c 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/s390x.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/s390x.rs @@ -11,6 +11,7 @@ pub type suseconds_t = i64; pub type wchar_t = i32; pub type greg_t = u64; pub type __u64 = u64; +pub type __s64 = i64; s! { pub struct sigaction { @@ -153,17 +154,6 @@ s! { __f_spare: [::c_int; 6], } - pub struct termios2 { - pub c_iflag: ::tcflag_t, - pub c_oflag: ::tcflag_t, - pub c_cflag: ::tcflag_t, - pub c_lflag: ::tcflag_t, - pub c_line: ::cc_t, - pub c_cc: [::cc_t; 19], - pub c_ispeed: ::speed_t, - pub c_ospeed: ::speed_t, - } - pub struct __psw_t { pub mask: u64, pub addr: u64, @@ -340,9 +330,6 @@ pub const EDEADLK: ::c_int = 35; pub const ENOSYS: ::c_int = 38; pub const ENOTCONN: ::c_int = 107; pub const ETIMEDOUT: ::c_int = 110; -pub const FIOCLEX: ::c_ulong = 0x5451; -pub const FIONCLEX: ::c_ulong = 0x5450; -pub const FIONBIO: ::c_ulong = 0x5421; pub const O_APPEND: ::c_int = 1024; pub const O_CREAT: ::c_int = 64; pub const O_EXCL: ::c_int = 128; @@ -354,30 +341,10 @@ pub const SIGBUS: ::c_int = 7; pub const SIGSTKSZ: ::size_t = 0x2000; pub const MINSIGSTKSZ: ::size_t = 2048; pub const SIG_SETMASK: ::c_int = 2; -pub const SOL_SOCKET: ::c_int = 1; -pub const SO_BROADCAST: ::c_int = 6; -pub const SO_ERROR: ::c_int = 4; -pub const SO_RCVTIMEO: ::c_int = 20; -pub const SO_REUSEADDR: ::c_int = 2; -pub const SO_SNDTIMEO: ::c_int = 21; -pub const SO_BINDTODEVICE: ::c_int = 25; -pub const SO_TIMESTAMP: ::c_int = 29; -pub const SO_MARK: ::c_int = 36; -pub const SO_PROTOCOL: ::c_int = 38; -pub const SO_DOMAIN: ::c_int = 39; -pub const SO_RXQ_OVFL: ::c_int = 40; -pub const SO_PEEK_OFF: ::c_int = 42; -pub const SO_BUSY_POLL: ::c_int = 46; pub const SOCK_STREAM: ::c_int = 1; pub const SOCK_DGRAM: ::c_int = 2; -pub const RLIMIT_RSS: ::__rlimit_resource_t = 5; -pub const RLIMIT_AS: ::__rlimit_resource_t = 9; -pub const RLIMIT_MEMLOCK: ::__rlimit_resource_t = 8; -pub const RLIMIT_NOFILE: ::__rlimit_resource_t = 7; -pub const RLIMIT_NPROC: ::__rlimit_resource_t = 6; - pub const O_NOCTTY: ::c_int = 256; pub const O_SYNC: ::c_int = 1052672; pub const O_RSYNC: ::c_int = 1052672; @@ -399,6 +366,7 @@ pub const MAP_POPULATE: ::c_int = 0x08000; pub const MAP_NONBLOCK: ::c_int = 0x010000; pub const MAP_STACK: ::c_int = 0x020000; pub const MAP_HUGETLB: ::c_int = 0x040000; +pub const MAP_SYNC: ::c_int = 0x080000; pub const EDEADLOCK: ::c_int = 35; pub const ENAMETOOLONG: ::c_int = 36; @@ -471,24 +439,6 @@ pub const ENOTRECOVERABLE: ::c_int = 131; pub const EHWPOISON: ::c_int = 133; pub const ERFKILL: ::c_int = 132; -pub const SO_TYPE: ::c_int = 3; -pub const SO_DONTROUTE: ::c_int = 5; -pub const SO_SNDBUF: ::c_int = 7; -pub const SO_RCVBUF: ::c_int = 8; -pub const SO_KEEPALIVE: ::c_int = 9; -pub const SO_OOBINLINE: ::c_int = 10; -pub const SO_PRIORITY: ::c_int = 12; -pub const SO_LINGER: ::c_int = 13; -pub const SO_BSDCOMPAT: ::c_int = 14; -pub const SO_REUSEPORT: ::c_int = 15; -pub const SO_PASSCRED: ::c_int = 16; -pub const SO_PEERCRED: ::c_int = 17; -pub const SO_RCVLOWAT: ::c_int = 18; -pub const SO_SNDLOWAT: ::c_int = 19; -pub const SO_ACCEPTCONN: ::c_int = 30; -pub const SO_SNDBUFFORCE: ::c_int = 32; -pub const SO_RCVBUFFORCE: ::c_int = 33; - pub const SIGTTIN: ::c_int = 21; pub const SIGTTOU: ::c_int = 22; pub const SIGXCPU: ::c_int = 24; @@ -546,49 +496,10 @@ pub const F_OFD_SETLKW: ::c_int = 38; pub const SFD_NONBLOCK: ::c_int = 0x0800; -pub const TCGETS: ::c_ulong = 0x5401; -pub const TCSETS: ::c_ulong = 0x5402; -pub const TCSETSW: ::c_ulong = 0x5403; -pub const TCSETSF: ::c_ulong = 0x5404; -pub const TCGETA: ::c_ulong = 0x5405; -pub const TCSETA: ::c_ulong = 0x5406; -pub const TCSETAW: ::c_ulong = 0x5407; -pub const TCSETAF: ::c_ulong = 0x5408; -pub const TCSBRK: ::c_ulong = 0x5409; -pub const TCXONC: ::c_ulong = 0x540A; -pub const TCFLSH: ::c_ulong = 0x540B; -pub const TIOCGSOFTCAR: ::c_ulong = 0x5419; -pub const TIOCSSOFTCAR: ::c_ulong = 0x541A; -pub const TIOCINQ: ::c_ulong = 0x541B; -pub const TIOCEXCL: ::c_ulong = 0x540C; -pub const TIOCNXCL: ::c_ulong = 0x540D; -pub const TIOCSCTTY: ::c_ulong = 0x540E; -pub const TIOCGPGRP: ::c_ulong = 0x540F; -pub const TIOCSPGRP: ::c_ulong = 0x5410; -pub const TIOCOUTQ: ::c_ulong = 0x5411; -pub const TIOCSTI: ::c_ulong = 0x5412; -pub const TIOCGWINSZ: ::c_ulong = 0x5413; -pub const TIOCSWINSZ: ::c_ulong = 0x5414; -pub const TIOCMGET: ::c_ulong = 0x5415; -pub const TIOCMBIS: ::c_ulong = 0x5416; -pub const TIOCMBIC: ::c_ulong = 0x5417; -pub const TIOCMSET: ::c_ulong = 0x5418; -pub const FIONREAD: ::c_ulong = 0x541B; -pub const TIOCCONS: ::c_ulong = 0x541D; - pub const TCSANOW: ::c_int = 0; pub const TCSADRAIN: ::c_int = 1; pub const TCSAFLUSH: ::c_int = 2; -pub const TIOCLINUX: ::c_ulong = 0x541C; -pub const TIOCGSERIAL: ::c_ulong = 0x541E; -pub const TIOCM_ST: ::c_int = 0x008; -pub const TIOCM_SR: ::c_int = 0x010; -pub const TIOCM_CTS: ::c_int = 0x020; -pub const TIOCM_CAR: ::c_int = 0x040; -pub const TIOCM_RNG: ::c_int = 0x080; -pub const TIOCM_DSR: ::c_int = 0x100; - pub const VTIME: usize = 5; pub const VSWTC: usize = 7; pub const VSTART: usize = 8; @@ -646,7 +557,6 @@ pub const PARODD: ::tcflag_t = 0o001000; pub const HUPCL: ::tcflag_t = 0o002000; pub const CLOCAL: ::tcflag_t = 0o004000; pub const CBAUDEX: ::tcflag_t = 0o010000; -pub const BOTHER: ::speed_t = 0o010000; pub const B57600: ::speed_t = 0o010001; pub const B115200: ::speed_t = 0o010002; pub const B230400: ::speed_t = 0o010003; @@ -1005,8 +915,35 @@ pub const SYS_setfsuid: ::c_long = 215; pub const SYS_setfsgid: ::c_long = 216; pub const SYS_newfstatat: ::c_long = 293; pub const SYS_statx: ::c_long = 379; +pub const SYS_rseq: ::c_long = 383; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; +pub const SYS_quotactl_fd: ::c_long = 443; +pub const SYS_landlock_create_ruleset: ::c_long = 444; +pub const SYS_landlock_add_rule: ::c_long = 445; +pub const SYS_landlock_restrict_self: ::c_long = 446; +pub const SYS_memfd_secret: ::c_long = 447; +pub const SYS_process_mrelease: ::c_long = 448; +pub const SYS_futex_waitv: ::c_long = 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 450; -#[link(name = "util")] extern "C" { pub fn sysctl( @@ -1019,14 +956,6 @@ extern "C" { ) -> ::c_int; pub fn getcontext(ucp: *mut ::ucontext_t) -> ::c_int; pub fn setcontext(ucp: *const ::ucontext_t) -> ::c_int; - pub fn makecontext( - ucp: *mut ::ucontext_t, - func: extern "C" fn(), - argc: ::c_int, - ... - ); - pub fn swapcontext( - uocp: *mut ::ucontext_t, - ucp: *const ::ucontext_t, - ) -> ::c_int; + pub fn makecontext(ucp: *mut ::ucontext_t, func: extern "C" fn(), argc: ::c_int, ...); + pub fn swapcontext(uocp: *mut ::ucontext_t, ucp: *const ::ucontext_t) -> ::c_int; } diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/sparc64/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/sparc64/mod.rs index 3e08ebe5a..9fdacfac8 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/sparc64/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/sparc64/mod.rs @@ -10,6 +10,7 @@ pub type nlink_t = u32; pub type blksize_t = i64; pub type suseconds_t = i32; pub type __u64 = ::c_ulonglong; +pub type __s64 = ::c_longlong; s! { pub struct sigaction { @@ -193,17 +194,6 @@ s! { __reserved1: ::c_ulong, __reserved2: ::c_ulong } - - pub struct termios2 { - pub c_iflag: ::tcflag_t, - pub c_oflag: ::tcflag_t, - pub c_cflag: ::tcflag_t, - pub c_lflag: ::tcflag_t, - pub c_line: ::cc_t, - pub c_cc: [::cc_t; 19], - pub c_ispeed: ::speed_t, - pub c_ospeed: ::speed_t, - } } pub const POSIX_FADV_DONTNEED: ::c_int = 4; @@ -215,15 +205,6 @@ pub const RTLD_GLOBAL: ::c_int = 0x100; pub const RTLD_NOLOAD: ::c_int = 0x4; pub const __SIZEOF_PTHREAD_RWLOCK_T: usize = 56; -pub const TIOCGSOFTCAR: ::c_ulong = 0x40047464; -pub const TIOCSSOFTCAR: ::c_ulong = 0x80047465; - -pub const RLIMIT_RSS: ::__rlimit_resource_t = 5; -pub const RLIMIT_AS: ::__rlimit_resource_t = 9; -pub const RLIMIT_MEMLOCK: ::__rlimit_resource_t = 8; -pub const RLIMIT_NOFILE: ::__rlimit_resource_t = 6; -pub const RLIMIT_NPROC: ::__rlimit_resource_t = 7; - pub const O_APPEND: ::c_int = 0x8; pub const O_CREAT: ::c_int = 0x200; pub const O_EXCL: ::c_int = 0x800; @@ -247,7 +228,7 @@ pub const MAP_POPULATE: ::c_int = 0x08000; pub const MAP_NONBLOCK: ::c_int = 0x010000; pub const MAP_STACK: ::c_int = 0x020000; pub const MAP_HUGETLB: ::c_int = 0x040000; -pub const MAP_SYNC : ::c_int = 0x080000; +pub const MAP_SYNC: ::c_int = 0x080000; pub const EDEADLK: ::c_int = 78; pub const ENAMETOOLONG: ::c_int = 63; @@ -328,31 +309,6 @@ pub const ENOTRECOVERABLE: ::c_int = 133; pub const EHWPOISON: ::c_int = 135; pub const ERFKILL: ::c_int = 134; -pub const SOL_SOCKET: ::c_int = 0xffff; - -pub const SO_PASSCRED: ::c_int = 2; -pub const SO_REUSEADDR: ::c_int = 4; -pub const SO_BINDTODEVICE: ::c_int = 0x000d; -pub const SO_TIMESTAMP: ::c_int = 0x001d; -pub const SO_MARK: ::c_int = 0x0022; -pub const SO_RXQ_OVFL: ::c_int = 0x0024; -pub const SO_PEEK_OFF: ::c_int = 0x0026; -pub const SO_BUSY_POLL: ::c_int = 0x0030; -pub const SO_TYPE: ::c_int = 0x1008; -pub const SO_ERROR: ::c_int = 0x1007; -pub const SO_DONTROUTE: ::c_int = 16; -pub const SO_BROADCAST: ::c_int = 32; -pub const SO_SNDBUF: ::c_int = 0x1001; -pub const SO_RCVBUF: ::c_int = 0x1002; -pub const SO_SNDBUFFORCE: ::c_int = 0x100a; -pub const SO_RCVBUFFORCE: ::c_int = 0x100b; -pub const SO_DOMAIN: ::c_int = 0x1029; -pub const SO_KEEPALIVE: ::c_int = 8; -pub const SO_OOBINLINE: ::c_int = 0x100; -pub const SO_LINGER: ::c_int = 128; -pub const SO_REUSEPORT: ::c_int = 0x200; -pub const SO_ACCEPTCONN: ::c_int = 0x8000; - pub const SOCK_STREAM: ::c_int = 1; pub const SOCK_DGRAM: ::c_int = 2; @@ -389,7 +345,7 @@ pub const POLLWRBAND: ::c_short = 0x100; pub const O_ASYNC: ::c_int = 0x40; pub const O_NDELAY: ::c_int = 0x4004; -pub const PTRACE_DETACH: ::c_uint = 11; +pub const PTRACE_DETACH: ::c_uint = 17; pub const EFD_NONBLOCK: ::c_int = 0x4000; @@ -412,25 +368,6 @@ pub const TCSANOW: ::c_int = 0; pub const TCSADRAIN: ::c_int = 1; pub const TCSAFLUSH: ::c_int = 2; -pub const TIOCLINUX: ::c_ulong = 0x541C; -pub const TIOCGSERIAL: ::c_ulong = 0x541E; -pub const TIOCEXCL: ::c_ulong = 0x2000740d; -pub const TIOCNXCL: ::c_ulong = 0x2000740e; -pub const TIOCSCTTY: ::c_ulong = 0x20007484; -pub const TIOCSTI: ::c_ulong = 0x80017472; -pub const TIOCMGET: ::c_ulong = 0x4004746a; -pub const TIOCMBIS: ::c_ulong = 0x8004746c; -pub const TIOCMBIC: ::c_ulong = 0x8004746b; -pub const TIOCMSET: ::c_ulong = 0x8004746d; -pub const TIOCCONS: ::c_ulong = 0x20007424; - -pub const TIOCM_ST: ::c_int = 0x008; -pub const TIOCM_SR: ::c_int = 0x010; -pub const TIOCM_CTS: ::c_int = 0x020; -pub const TIOCM_CAR: ::c_int = 0x040; -pub const TIOCM_RNG: ::c_int = 0x080; -pub const TIOCM_DSR: ::c_int = 0x100; - pub const SFD_CLOEXEC: ::c_int = 0x400000; pub const NCCS: usize = 17; @@ -503,16 +440,6 @@ pub const ENAVAIL: ::c_int = 119; pub const EISNAM: ::c_int = 120; pub const EREMOTEIO: ::c_int = 121; -pub const SO_PEERCRED: ::c_int = 0x40; -pub const SO_RCVLOWAT: ::c_int = 0x800; -pub const SO_SNDLOWAT: ::c_int = 0x1000; -pub const SO_RCVTIMEO: ::c_int = 0x2000; -pub const SO_SNDTIMEO: ::c_int = 0x4000; - -pub const FIOCLEX: ::c_ulong = 0x20006601; -pub const FIONCLEX: ::c_ulong = 0x20006602; -pub const FIONBIO: ::c_ulong = 0x8004667e; - pub const MCL_CURRENT: ::c_int = 0x2000; pub const MCL_FUTURE: ::c_int = 0x4000; @@ -588,7 +515,6 @@ pub const B19200: ::speed_t = 0o000016; pub const B38400: ::speed_t = 0o000017; pub const EXTA: ::speed_t = B19200; pub const EXTB: ::speed_t = B38400; -pub const BOTHER: ::speed_t = 0x1000; pub const B57600: ::speed_t = 0x1001; pub const B115200: ::speed_t = 0x1002; pub const B230400: ::speed_t = 0x1003; @@ -612,24 +538,6 @@ pub const IEXTEN: ::tcflag_t = 0x8000; pub const TOSTOP: ::tcflag_t = 0x100; pub const FLUSHO: ::tcflag_t = 0x1000; pub const EXTPROC: ::tcflag_t = 0x10000; -pub const TCGETS: ::c_ulong = 0x40245408; -pub const TCSETS: ::c_ulong = 0x80245409; -pub const TCSETSW: ::c_ulong = 0x8024540a; -pub const TCSETSF: ::c_ulong = 0x8024540b; -pub const TCGETA: ::c_ulong = 0x40125401; -pub const TCSETA: ::c_ulong = 0x80125402; -pub const TCSETAW: ::c_ulong = 0x80125403; -pub const TCSETAF: ::c_ulong = 0x80125404; -pub const TCSBRK: ::c_ulong = 0x20005405; -pub const TCXONC: ::c_ulong = 0x20005406; -pub const TCFLSH: ::c_ulong = 0x20005407; -pub const TIOCINQ: ::c_ulong = 0x4004667f; -pub const TIOCGPGRP: ::c_ulong = 0x40047483; -pub const TIOCSPGRP: ::c_ulong = 0x80047482; -pub const TIOCOUTQ: ::c_ulong = 0x40047473; -pub const TIOCGWINSZ: ::c_ulong = 0x40087468; -pub const TIOCSWINSZ: ::c_ulong = 0x80087467; -pub const FIONREAD: ::c_ulong = 0x4004667f; pub const SYS_restart_syscall: ::c_long = 0; pub const SYS_exit: ::c_long = 1; @@ -971,8 +879,36 @@ pub const SYS_copy_file_range: ::c_long = 357; pub const SYS_preadv2: ::c_long = 358; pub const SYS_pwritev2: ::c_long = 359; pub const SYS_statx: ::c_long = 360; +pub const SYS_rseq: ::c_long = 365; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +// Reserved in the kernel, but not actually implemented yet +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; +pub const SYS_quotactl_fd: ::c_long = 443; +pub const SYS_landlock_create_ruleset: ::c_long = 444; +pub const SYS_landlock_add_rule: ::c_long = 445; +pub const SYS_landlock_restrict_self: ::c_long = 446; +pub const SYS_memfd_secret: ::c_long = 447; +pub const SYS_process_mrelease: ::c_long = 448; +pub const SYS_futex_waitv: ::c_long = 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 450; -#[link(name = "util")] extern "C" { pub fn sysctl( name: *mut ::c_int, diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/x86_64/align.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/x86_64/align.rs index 7ca870fd0..ba3075edd 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/x86_64/align.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/x86_64/align.rs @@ -5,3 +5,20 @@ s_no_extra_traits! { priv_: [f64; 4] } } + +s! { + #[repr(align(8))] + pub struct clone_args { + pub flags: ::c_ulonglong, + pub pidfd: ::c_ulonglong, + pub child_tid: ::c_ulonglong, + pub parent_tid: ::c_ulonglong, + pub exit_signal: ::c_ulonglong, + pub stack: ::c_ulonglong, + pub stack_size: ::c_ulonglong, + pub tls: ::c_ulonglong, + pub set_tid: ::c_ulonglong, + pub set_tid_size: ::c_ulonglong, + pub cgroup: ::c_ulonglong, + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/x86_64/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/x86_64/mod.rs index 4f5313807..d515d2231 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/x86_64/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/x86_64/mod.rs @@ -7,6 +7,7 @@ pub type blksize_t = i64; pub type greg_t = i64; pub type suseconds_t = i64; pub type __u64 = ::c_ulonglong; +pub type __s64 = ::c_longlong; s! { pub struct sigaction { @@ -259,21 +260,18 @@ s! { __unused5: u64 } - pub struct termios2 { - pub c_iflag: ::tcflag_t, - pub c_oflag: ::tcflag_t, - pub c_cflag: ::tcflag_t, - pub c_lflag: ::tcflag_t, - pub c_line: ::cc_t, - pub c_cc: [::cc_t; 19], - pub c_ispeed: ::speed_t, - pub c_ospeed: ::speed_t, + pub struct seccomp_notif_sizes { + pub seccomp_notif: ::__u16, + pub seccomp_notif_resp: ::__u16, + pub seccomp_data: ::__u16, } - pub struct ip_mreqn { - pub imr_multiaddr: ::in_addr, - pub imr_address: ::in_addr, - pub imr_ifindex: ::c_int, + pub struct ptrace_rseq_configuration { + pub rseq_abi_pointer: ::__u64, + pub rseq_abi_size: ::__u32, + pub signature: ::__u32, + pub flags: ::__u32, + pub pad: ::__u32, } } @@ -409,16 +407,6 @@ pub const VEOF: usize = 4; pub const RTLD_DEEPBIND: ::c_int = 0x8; pub const RTLD_GLOBAL: ::c_int = 0x100; pub const RTLD_NOLOAD: ::c_int = 0x4; -pub const TIOCGSOFTCAR: ::c_ulong = 0x5419; -pub const TIOCSSOFTCAR: ::c_ulong = 0x541A; -pub const TIOCGRS485: ::c_int = 0x542E; -pub const TIOCSRS485: ::c_int = 0x542F; - -pub const RLIMIT_RSS: ::__rlimit_resource_t = 5; -pub const RLIMIT_AS: ::__rlimit_resource_t = 9; -pub const RLIMIT_MEMLOCK: ::__rlimit_resource_t = 8; -pub const RLIMIT_NOFILE: ::__rlimit_resource_t = 7; -pub const RLIMIT_NPROC: ::__rlimit_resource_t = 6; pub const O_APPEND: ::c_int = 1024; pub const O_CREAT: ::c_int = 64; @@ -515,61 +503,6 @@ pub const ENOTRECOVERABLE: ::c_int = 131; pub const EHWPOISON: ::c_int = 133; pub const ERFKILL: ::c_int = 132; -pub const SOL_SOCKET: ::c_int = 1; - -pub const SO_REUSEADDR: ::c_int = 2; -pub const SO_TYPE: ::c_int = 3; -pub const SO_ERROR: ::c_int = 4; -pub const SO_DONTROUTE: ::c_int = 5; -pub const SO_BROADCAST: ::c_int = 6; -pub const SO_SNDBUF: ::c_int = 7; -pub const SO_RCVBUF: ::c_int = 8; -pub const SO_SNDBUFFORCE: ::c_int = 32; -pub const SO_RCVBUFFORCE: ::c_int = 33; -pub const SO_KEEPALIVE: ::c_int = 9; -pub const SO_OOBINLINE: ::c_int = 10; -pub const SO_NO_CHECK: ::c_int = 11; -pub const SO_PRIORITY: ::c_int = 12; -pub const SO_LINGER: ::c_int = 13; -pub const SO_BSDCOMPAT: ::c_int = 14; -pub const SO_REUSEPORT: ::c_int = 15; -pub const SO_PASSCRED: ::c_int = 16; -pub const SO_PEERCRED: ::c_int = 17; -pub const SO_RCVLOWAT: ::c_int = 18; -pub const SO_SNDLOWAT: ::c_int = 19; -pub const SO_RCVTIMEO: ::c_int = 20; -pub const SO_SNDTIMEO: ::c_int = 21; -pub const SO_SECURITY_AUTHENTICATION: ::c_int = 22; -pub const SO_SECURITY_ENCRYPTION_TRANSPORT: ::c_int = 23; -pub const SO_SECURITY_ENCRYPTION_NETWORK: ::c_int = 24; -pub const SO_BINDTODEVICE: ::c_int = 25; -pub const SO_ATTACH_FILTER: ::c_int = 26; -pub const SO_DETACH_FILTER: ::c_int = 27; -pub const SO_GET_FILTER: ::c_int = SO_ATTACH_FILTER; -pub const SO_PEERNAME: ::c_int = 28; -pub const SO_TIMESTAMP: ::c_int = 29; -pub const SO_ACCEPTCONN: ::c_int = 30; -pub const SO_PEERSEC: ::c_int = 31; -pub const SO_PASSSEC: ::c_int = 34; -pub const SO_TIMESTAMPNS: ::c_int = 35; -pub const SCM_TIMESTAMPNS: ::c_int = SO_TIMESTAMPNS; -pub const SO_MARK: ::c_int = 36; -pub const SO_PROTOCOL: ::c_int = 38; -pub const SO_DOMAIN: ::c_int = 39; -pub const SO_RXQ_OVFL: ::c_int = 40; -pub const SO_WIFI_STATUS: ::c_int = 41; -pub const SCM_WIFI_STATUS: ::c_int = SO_WIFI_STATUS; -pub const SO_PEEK_OFF: ::c_int = 42; -pub const SO_NOFCS: ::c_int = 43; -pub const SO_LOCK_FILTER: ::c_int = 44; -pub const SO_SELECT_ERR_QUEUE: ::c_int = 45; -pub const SO_BUSY_POLL: ::c_int = 46; -pub const SO_MAX_PACING_RATE: ::c_int = 47; -pub const SO_BPF_EXTENSIONS: ::c_int = 48; -pub const SO_INCOMING_CPU: ::c_int = 49; -pub const SO_ATTACH_BPF: ::c_int = 50; -pub const SO_DETACH_BPF: ::c_int = SO_DETACH_FILTER; - pub const SOCK_STREAM: ::c_int = 1; pub const SOCK_DGRAM: ::c_int = 2; @@ -610,6 +543,7 @@ pub const O_ASYNC: ::c_int = 0x2000; pub const O_NDELAY: ::c_int = 0x800; pub const PTRACE_DETACH: ::c_uint = 17; +pub const PTRACE_GET_RSEQ_CONFIGURATION: ::c_uint = 0x420f; pub const EFD_NONBLOCK: ::c_int = 0x800; @@ -632,25 +566,6 @@ pub const TCSANOW: ::c_int = 0; pub const TCSADRAIN: ::c_int = 1; pub const TCSAFLUSH: ::c_int = 2; -pub const TIOCLINUX: ::c_ulong = 0x541C; -pub const TIOCGSERIAL: ::c_ulong = 0x541E; -pub const TIOCEXCL: ::c_ulong = 0x540C; -pub const TIOCNXCL: ::c_ulong = 0x540D; -pub const TIOCSCTTY: ::c_ulong = 0x540E; -pub const TIOCSTI: ::c_ulong = 0x5412; -pub const TIOCMGET: ::c_ulong = 0x5415; -pub const TIOCMBIS: ::c_ulong = 0x5416; -pub const TIOCMBIC: ::c_ulong = 0x5417; -pub const TIOCMSET: ::c_ulong = 0x5418; -pub const TIOCCONS: ::c_ulong = 0x541D; - -pub const TIOCM_ST: ::c_int = 0x008; -pub const TIOCM_SR: ::c_int = 0x010; -pub const TIOCM_CTS: ::c_int = 0x020; -pub const TIOCM_CAR: ::c_int = 0x040; -pub const TIOCM_RNG: ::c_int = 0x080; -pub const TIOCM_DSR: ::c_int = 0x100; - pub const SFD_CLOEXEC: ::c_int = 0x080000; pub const NCCS: usize = 32; @@ -701,7 +616,7 @@ pub const MAP_EXECUTABLE: ::c_int = 0x01000; pub const MAP_POPULATE: ::c_int = 0x08000; pub const MAP_NONBLOCK: ::c_int = 0x010000; pub const MAP_STACK: ::c_int = 0x020000; -pub const MAP_SYNC : ::c_int = 0x080000; +pub const MAP_SYNC: ::c_int = 0x080000; pub const EDEADLOCK: ::c_int = 35; pub const EUCLEAN: ::c_int = 117; @@ -710,10 +625,6 @@ pub const ENAVAIL: ::c_int = 119; pub const EISNAM: ::c_int = 120; pub const EREMOTEIO: ::c_int = 121; -pub const FIOCLEX: ::c_ulong = 0x5451; -pub const FIONCLEX: ::c_ulong = 0x5450; -pub const FIONBIO: ::c_ulong = 0x5421; - pub const PTRACE_GETFPREGS: ::c_uint = 14; pub const PTRACE_SETFPREGS: ::c_uint = 15; pub const PTRACE_GETFPXREGS: ::c_uint = 18; @@ -721,6 +632,21 @@ pub const PTRACE_SETFPXREGS: ::c_uint = 19; pub const PTRACE_GETREGS: ::c_uint = 12; pub const PTRACE_SETREGS: ::c_uint = 13; pub const PTRACE_PEEKSIGINFO_SHARED: ::c_uint = 1; +pub const PTRACE_SYSEMU: ::c_uint = 31; +pub const PTRACE_SYSEMU_SINGLESTEP: ::c_uint = 32; + +pub const PR_GET_SPECULATION_CTRL: ::c_int = 52; +pub const PR_SET_SPECULATION_CTRL: ::c_int = 53; +pub const PR_SPEC_NOT_AFFECTED: ::c_uint = 0; +pub const PR_SPEC_PRCTL: ::c_uint = 1 << 0; +pub const PR_SPEC_ENABLE: ::c_uint = 1 << 1; +pub const PR_SPEC_DISABLE: ::c_uint = 1 << 2; +pub const PR_SPEC_FORCE_DISABLE: ::c_uint = 1 << 3; +pub const PR_SPEC_DISABLE_NOEXEC: ::c_uint = 1 << 4; +pub const PR_SPEC_STORE_BYPASS: ::c_int = 0; +pub const PR_SPEC_INDIRECT_BRANCH: ::c_int = 1; +// FIXME: perharps for later +//pub const PR_SPEC_L1D_FLUSH: ::c_int = 2; pub const MCL_CURRENT: ::c_int = 0x0001; pub const MCL_FUTURE: ::c_int = 0x0002; @@ -797,7 +723,6 @@ pub const B19200: ::speed_t = 0o000016; pub const B38400: ::speed_t = 0o000017; pub const EXTA: ::speed_t = B19200; pub const EXTB: ::speed_t = B38400; -pub const BOTHER: ::speed_t = 0o010000; pub const B57600: ::speed_t = 0o010001; pub const B115200: ::speed_t = 0o010002; pub const B230400: ::speed_t = 0o010003; @@ -821,24 +746,6 @@ pub const IEXTEN: ::tcflag_t = 0x00008000; pub const TOSTOP: ::tcflag_t = 0x00000100; pub const FLUSHO: ::tcflag_t = 0x00001000; pub const EXTPROC: ::tcflag_t = 0x00010000; -pub const TCGETS: ::c_ulong = 0x5401; -pub const TCSETS: ::c_ulong = 0x5402; -pub const TCSETSW: ::c_ulong = 0x5403; -pub const TCSETSF: ::c_ulong = 0x5404; -pub const TCGETA: ::c_ulong = 0x5405; -pub const TCSETA: ::c_ulong = 0x5406; -pub const TCSETAW: ::c_ulong = 0x5407; -pub const TCSETAF: ::c_ulong = 0x5408; -pub const TCSBRK: ::c_ulong = 0x5409; -pub const TCXONC: ::c_ulong = 0x540A; -pub const TCFLSH: ::c_ulong = 0x540B; -pub const TIOCINQ: ::c_ulong = 0x541B; -pub const TIOCGPGRP: ::c_ulong = 0x540F; -pub const TIOCSPGRP: ::c_ulong = 0x5410; -pub const TIOCOUTQ: ::c_ulong = 0x5411; -pub const TIOCGWINSZ: ::c_ulong = 0x5413; -pub const TIOCSWINSZ: ::c_ulong = 0x5414; -pub const FIONREAD: ::c_ulong = 0x541B; // offsets in user_regs_structs, from sys/reg.h pub const R15: ::c_int = 0; @@ -894,25 +801,18 @@ pub const REG_TRAPNO: ::c_int = 20; pub const REG_OLDMASK: ::c_int = 21; pub const REG_CR2: ::c_int = 22; +pub const SECCOMP_SET_MODE_STRICT: ::c_uint = 0; +pub const SECCOMP_SET_MODE_FILTER: ::c_uint = 1; +pub const SECCOMP_GET_ACTION_AVAIL: ::c_uint = 2; +pub const SECCOMP_GET_NOTIF_SIZES: ::c_uint = 3; + extern "C" { pub fn getcontext(ucp: *mut ucontext_t) -> ::c_int; pub fn setcontext(ucp: *const ucontext_t) -> ::c_int; - pub fn makecontext( - ucp: *mut ucontext_t, - func: extern "C" fn(), - argc: ::c_int, - ... - ); - pub fn swapcontext( - uocp: *mut ucontext_t, - ucp: *const ucontext_t, - ) -> ::c_int; + pub fn makecontext(ucp: *mut ucontext_t, func: extern "C" fn(), argc: ::c_int, ...); + pub fn swapcontext(uocp: *mut ucontext_t, ucp: *const ucontext_t) -> ::c_int; pub fn iopl(level: ::c_int) -> ::c_int; - pub fn ioperm( - from: ::c_ulong, - num: ::c_ulong, - turn_on: ::c_int, - ) -> ::c_int; + pub fn ioperm(from: ::c_ulong, num: ::c_ulong, turn_on: ::c_int) -> ::c_int; } cfg_if! { diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/x86_64/not_x32.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/x86_64/not_x32.rs index 64a6de9c6..35d2714ee 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/x86_64/not_x32.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/x86_64/not_x32.rs @@ -409,8 +409,35 @@ pub const SYS_pkey_mprotect: ::c_long = 329; pub const SYS_pkey_alloc: ::c_long = 330; pub const SYS_pkey_free: ::c_long = 331; pub const SYS_statx: ::c_long = 332; +pub const SYS_rseq: ::c_long = 334; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; +pub const SYS_quotactl_fd: ::c_long = 443; +pub const SYS_landlock_create_ruleset: ::c_long = 444; +pub const SYS_landlock_add_rule: ::c_long = 445; +pub const SYS_landlock_restrict_self: ::c_long = 446; +pub const SYS_memfd_secret: ::c_long = 447; +pub const SYS_process_mrelease: ::c_long = 448; +pub const SYS_futex_waitv: ::c_long = 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 450; -#[link(name = "util")] extern "C" { pub fn sysctl( name: *mut ::c_int, diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/x86_64/x32.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/x86_64/x32.rs index 37468818a..807b948ef 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/x86_64/x32.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/b64/x86_64/x32.rs @@ -337,6 +337,34 @@ pub const SYS_pkey_mprotect: ::c_long = __X32_SYSCALL_BIT + 329; pub const SYS_pkey_alloc: ::c_long = __X32_SYSCALL_BIT + 330; pub const SYS_pkey_free: ::c_long = __X32_SYSCALL_BIT + 331; pub const SYS_statx: ::c_long = __X32_SYSCALL_BIT + 332; +pub const SYS_rseq: ::c_long = __X32_SYSCALL_BIT + 334; +pub const SYS_pidfd_send_signal: ::c_long = __X32_SYSCALL_BIT + 424; +pub const SYS_io_uring_setup: ::c_long = __X32_SYSCALL_BIT + 425; +pub const SYS_io_uring_enter: ::c_long = __X32_SYSCALL_BIT + 426; +pub const SYS_io_uring_register: ::c_long = __X32_SYSCALL_BIT + 427; +pub const SYS_open_tree: ::c_long = __X32_SYSCALL_BIT + 428; +pub const SYS_move_mount: ::c_long = __X32_SYSCALL_BIT + 429; +pub const SYS_fsopen: ::c_long = __X32_SYSCALL_BIT + 430; +pub const SYS_fsconfig: ::c_long = __X32_SYSCALL_BIT + 431; +pub const SYS_fsmount: ::c_long = __X32_SYSCALL_BIT + 432; +pub const SYS_fspick: ::c_long = __X32_SYSCALL_BIT + 433; +pub const SYS_pidfd_open: ::c_long = __X32_SYSCALL_BIT + 434; +pub const SYS_clone3: ::c_long = __X32_SYSCALL_BIT + 435; +pub const SYS_close_range: ::c_long = __X32_SYSCALL_BIT + 436; +pub const SYS_openat2: ::c_long = __X32_SYSCALL_BIT + 437; +pub const SYS_pidfd_getfd: ::c_long = __X32_SYSCALL_BIT + 438; +pub const SYS_faccessat2: ::c_long = __X32_SYSCALL_BIT + 439; +pub const SYS_process_madvise: ::c_long = __X32_SYSCALL_BIT + 440; +pub const SYS_epoll_pwait2: ::c_long = __X32_SYSCALL_BIT + 441; +pub const SYS_mount_setattr: ::c_long = __X32_SYSCALL_BIT + 442; +pub const SYS_quotactl_fd: ::c_long = __X32_SYSCALL_BIT + 443; +pub const SYS_landlock_create_ruleset: ::c_long = __X32_SYSCALL_BIT + 444; +pub const SYS_landlock_add_rule: ::c_long = __X32_SYSCALL_BIT + 445; +pub const SYS_landlock_restrict_self: ::c_long = __X32_SYSCALL_BIT + 446; +pub const SYS_memfd_secret: ::c_long = __X32_SYSCALL_BIT + 447; +pub const SYS_process_mrelease: ::c_long = __X32_SYSCALL_BIT + 448; +pub const SYS_futex_waitv: ::c_long = __X32_SYSCALL_BIT + 449; +pub const SYS_set_mempolicy_home_node: ::c_long = __X32_SYSCALL_BIT + 450; pub const SYS_rt_sigaction: ::c_long = __X32_SYSCALL_BIT + 512; pub const SYS_rt_sigreturn: ::c_long = __X32_SYSCALL_BIT + 513; pub const SYS_ioctl: ::c_long = __X32_SYSCALL_BIT + 514; diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/mod.rs index 02645fdd1..98a58a2ac 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/gnu/mod.rs @@ -2,6 +2,17 @@ pub type pthread_t = c_ulong; pub type __priority_which_t = ::c_uint; pub type __rlimit_resource_t = ::c_uint; pub type Lmid_t = ::c_long; +pub type regoff_t = ::c_int; + +cfg_if! { + if #[cfg(doc)] { + // Used in `linux::arch` to define ioctl constants. + pub(crate) type Ioctl = ::c_ulong; + } else { + #[doc(hidden)] + pub type Ioctl = ::c_ulong; + } +} s! { pub struct statx { @@ -12,7 +23,7 @@ s! { pub stx_uid: u32, pub stx_gid: u32, pub stx_mode: u16, - pub __statx_pad1: [u16; 1], + __statx_pad1: [u16; 1], pub stx_ino: u64, pub stx_size: u64, pub stx_blocks: u64, @@ -25,7 +36,9 @@ s! { pub stx_rdev_minor: u32, pub stx_dev_major: u32, pub stx_dev_minor: u32, - pub __statx_pad2: [u64; 14], + pub stx_mnt_id: u64, + __statx_pad2: u64, + __statx_pad3: [u64; 12], } pub struct statx_timestamp { @@ -125,17 +138,17 @@ s! { pub keepcost: ::c_int, } - pub struct nlmsghdr { - pub nlmsg_len: u32, - pub nlmsg_type: u16, - pub nlmsg_flags: u16, - pub nlmsg_seq: u32, - pub nlmsg_pid: u32, - } - - pub struct nlmsgerr { - pub error: ::c_int, - pub msg: nlmsghdr, + pub struct mallinfo2 { + pub arena: ::size_t, + pub ordblks: ::size_t, + pub smblks: ::size_t, + pub hblks: ::size_t, + pub hblkhd: ::size_t, + pub usmblks: ::size_t, + pub fsmblks: ::size_t, + pub uordblks: ::size_t, + pub fordblks: ::size_t, + pub keepcost: ::size_t, } pub struct nl_pktinfo { @@ -158,11 +171,6 @@ s! { pub nm_gid: u32, } - pub struct nlattr { - pub nla_len: u16, - pub nla_type: u16, - } - pub struct rtentry { pub rt_pad1: ::c_ulong, pub rt_dst: ::sockaddr, @@ -273,6 +281,75 @@ s! { pub __glibc_reserved3: ::c_long, pub __glibc_reserved4: ::c_long, } + + pub struct regex_t { + __buffer: *mut ::c_void, + __allocated: ::size_t, + __used: ::size_t, + __syntax: ::c_ulong, + __fastmap: *mut ::c_char, + __translate: *mut ::c_char, + __re_nsub: ::size_t, + __bitfield: u8, + } + + pub struct Elf64_Chdr { + pub ch_type: ::Elf64_Word, + pub ch_reserved: ::Elf64_Word, + pub ch_size: ::Elf64_Xword, + pub ch_addralign: ::Elf64_Xword, + } + + pub struct Elf32_Chdr { + pub ch_type: ::Elf32_Word, + pub ch_size: ::Elf32_Word, + pub ch_addralign: ::Elf32_Word, + } + + pub struct seminfo { + pub semmap: ::c_int, + pub semmni: ::c_int, + pub semmns: ::c_int, + pub semmnu: ::c_int, + pub semmsl: ::c_int, + pub semopm: ::c_int, + pub semume: ::c_int, + pub semusz: ::c_int, + pub semvmx: ::c_int, + pub semaem: ::c_int, + } + + pub struct ptrace_peeksiginfo_args { + pub off: ::__u64, + pub flags: ::__u32, + pub nr: ::__s32, + } + + pub struct __c_anonymous_ptrace_syscall_info_entry { + pub nr: ::__u64, + pub args: [::__u64; 6], + } + + pub struct __c_anonymous_ptrace_syscall_info_exit { + pub sval: ::__s64, + pub is_error: ::__u8, + } + + pub struct __c_anonymous_ptrace_syscall_info_seccomp { + pub nr: ::__u64, + pub args: [::__u64; 6], + pub ret_data: ::__u32, + } + + pub struct ptrace_syscall_info { + pub op: ::__u8, + pub pad: [::__u8; 3], + pub arch: ::__u32, + pub instruction_pointer: ::__u64, + pub stack_pointer: ::__u64, + #[cfg(libc_union)] + pub u: __c_anonymous_ptrace_syscall_info_data, + } } impl siginfo_t { @@ -301,6 +378,80 @@ impl siginfo_t { } } +cfg_if! { + if #[cfg(libc_union)] { + // Internal, for casts to access union fields + #[repr(C)] + struct sifields_sigchld { + si_pid: ::pid_t, + si_uid: ::uid_t, + si_status: ::c_int, + si_utime: ::c_long, + si_stime: ::c_long, + } + impl ::Copy for sifields_sigchld {} + impl ::Clone for sifields_sigchld { + fn clone(&self) -> sifields_sigchld { + *self + } + } + + // Internal, for casts to access union fields + #[repr(C)] + union sifields { + _align_pointer: *mut ::c_void, + sigchld: sifields_sigchld, + } + + // Internal, for casts to access union fields. Note that some variants + // of sifields start with a pointer, which makes the alignment of + // sifields vary on 32-bit and 64-bit architectures. + #[repr(C)] + struct siginfo_f { + _siginfo_base: [::c_int; 3], + sifields: sifields, + } + + impl siginfo_t { + unsafe fn sifields(&self) -> &sifields { + &(*(self as *const siginfo_t as *const siginfo_f)).sifields + } + + pub unsafe fn si_pid(&self) -> ::pid_t { + self.sifields().sigchld.si_pid + } + + pub unsafe fn si_uid(&self) -> ::uid_t { + self.sifields().sigchld.si_uid + } + + pub unsafe fn si_status(&self) -> ::c_int { + self.sifields().sigchld.si_status + } + + pub unsafe fn si_utime(&self) -> ::c_long { + self.sifields().sigchld.si_utime + } + + pub unsafe fn si_stime(&self) -> ::c_long { + self.sifields().sigchld.si_stime + } + } + + pub union __c_anonymous_ptrace_syscall_info_data { + pub entry: __c_anonymous_ptrace_syscall_info_entry, + pub exit: __c_anonymous_ptrace_syscall_info_exit, + pub seccomp: __c_anonymous_ptrace_syscall_info_seccomp, + } + impl ::Copy for __c_anonymous_ptrace_syscall_info_data {} + impl ::Clone for __c_anonymous_ptrace_syscall_info_data { + fn clone(&self) -> __c_anonymous_ptrace_syscall_info_data { + *self + } + } + } +} + s_no_extra_traits! { pub struct utmpx { pub ut_type: ::c_short, @@ -314,22 +465,26 @@ s_no_extra_traits! { #[cfg(any(target_arch = "aarch64", target_arch = "s390x", + target_arch = "loongarch64", all(target_pointer_width = "32", not(target_arch = "x86_64"))))] pub ut_session: ::c_long, #[cfg(any(target_arch = "aarch64", target_arch = "s390x", + target_arch = "loongarch64", all(target_pointer_width = "32", not(target_arch = "x86_64"))))] pub ut_tv: ::timeval, #[cfg(not(any(target_arch = "aarch64", target_arch = "s390x", + target_arch = "loongarch64", all(target_pointer_width = "32", not(target_arch = "x86_64")))))] pub ut_session: i32, #[cfg(not(any(target_arch = "aarch64", target_arch = "s390x", + target_arch = "loongarch64", all(target_pointer_width = "32", not(target_arch = "x86_64")))))] pub ut_tv: __timeval, @@ -396,21 +551,91 @@ cfg_if! { self.__glibc_reserved.hash(state); } } + + #[cfg(libc_union)] + impl PartialEq for __c_anonymous_ptrace_syscall_info_data { + fn eq(&self, other: &__c_anonymous_ptrace_syscall_info_data) -> bool { + unsafe { + self.entry == other.entry || + self.exit == other.exit || + self.seccomp == other.seccomp + } + } + } + + #[cfg(libc_union)] + impl Eq for __c_anonymous_ptrace_syscall_info_data {} + + #[cfg(libc_union)] + impl ::fmt::Debug for __c_anonymous_ptrace_syscall_info_data { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + unsafe { + f.debug_struct("__c_anonymous_ptrace_syscall_info_data") + .field("entry", &self.entry) + .field("exit", &self.exit) + .field("seccomp", &self.seccomp) + .finish() + } + } + } + + #[cfg(libc_union)] + impl ::hash::Hash for __c_anonymous_ptrace_syscall_info_data { + fn hash(&self, state: &mut H) { + unsafe { + self.entry.hash(state); + self.exit.hash(state); + self.seccomp.hash(state); + } + } + } } } -pub const RLIMIT_CPU: ::__rlimit_resource_t = 0; -pub const RLIMIT_FSIZE: ::__rlimit_resource_t = 1; -pub const RLIMIT_DATA: ::__rlimit_resource_t = 2; -pub const RLIMIT_STACK: ::__rlimit_resource_t = 3; -pub const RLIMIT_CORE: ::__rlimit_resource_t = 4; -pub const RLIMIT_LOCKS: ::__rlimit_resource_t = 10; -pub const RLIMIT_SIGPENDING: ::__rlimit_resource_t = 11; -pub const RLIMIT_MSGQUEUE: ::__rlimit_resource_t = 12; -pub const RLIMIT_NICE: ::__rlimit_resource_t = 13; -pub const RLIMIT_RTPRIO: ::__rlimit_resource_t = 14; -pub const RLIMIT_RTTIME: ::__rlimit_resource_t = 15; -pub const RLIMIT_NLIMITS: ::__rlimit_resource_t = 16; +// include/uapi/asm-generic/hugetlb_encode.h +pub const HUGETLB_FLAG_ENCODE_SHIFT: ::c_int = 26; +pub const HUGETLB_FLAG_ENCODE_MASK: ::c_int = 0x3f; + +pub const HUGETLB_FLAG_ENCODE_64KB: ::c_int = 16 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_512KB: ::c_int = 19 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_1MB: ::c_int = 20 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_2MB: ::c_int = 21 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_8MB: ::c_int = 23 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_16MB: ::c_int = 24 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_32MB: ::c_int = 25 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_256MB: ::c_int = 28 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_512MB: ::c_int = 29 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_1GB: ::c_int = 30 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_2GB: ::c_int = 31 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_16GB: ::c_int = 34 << HUGETLB_FLAG_ENCODE_SHIFT; + +// include/uapi/linux/mman.h +/* + * Huge page size encoding when MAP_HUGETLB is specified, and a huge page + * size other than the default is desired. See hugetlb_encode.h. + * All known huge page size encodings are provided here. It is the + * responsibility of the application to know which sizes are supported on + * the running system. See mmap(2) man page for details. + */ +pub const MAP_HUGE_SHIFT: ::c_int = HUGETLB_FLAG_ENCODE_SHIFT; +pub const MAP_HUGE_MASK: ::c_int = HUGETLB_FLAG_ENCODE_MASK; + +pub const MAP_HUGE_64KB: ::c_int = HUGETLB_FLAG_ENCODE_64KB; +pub const MAP_HUGE_512KB: ::c_int = HUGETLB_FLAG_ENCODE_512KB; +pub const MAP_HUGE_1MB: ::c_int = HUGETLB_FLAG_ENCODE_1MB; +pub const MAP_HUGE_2MB: ::c_int = HUGETLB_FLAG_ENCODE_2MB; +pub const MAP_HUGE_8MB: ::c_int = HUGETLB_FLAG_ENCODE_8MB; +pub const MAP_HUGE_16MB: ::c_int = HUGETLB_FLAG_ENCODE_16MB; +pub const MAP_HUGE_32MB: ::c_int = HUGETLB_FLAG_ENCODE_32MB; +pub const MAP_HUGE_256MB: ::c_int = HUGETLB_FLAG_ENCODE_256MB; +pub const MAP_HUGE_512MB: ::c_int = HUGETLB_FLAG_ENCODE_512MB; +pub const MAP_HUGE_1GB: ::c_int = HUGETLB_FLAG_ENCODE_1GB; +pub const MAP_HUGE_2GB: ::c_int = HUGETLB_FLAG_ENCODE_2GB; +pub const MAP_HUGE_16GB: ::c_int = HUGETLB_FLAG_ENCODE_16GB; + +pub const PRIO_PROCESS: ::__priority_which_t = 0; +pub const PRIO_PGRP: ::__priority_which_t = 1; +pub const PRIO_USER: ::__priority_which_t = 2; pub const MS_RMT_MASK: ::c_ulong = 0x02800051; @@ -444,6 +669,7 @@ pub const RTLD_DI_TLS_MODID: ::c_int = 9; pub const RTLD_DI_TLS_DATA: ::c_int = 10; pub const SOCK_NONBLOCK: ::c_int = O_NONBLOCK; +pub const PIDFD_NONBLOCK: ::c_uint = O_NONBLOCK as ::c_uint; pub const SOL_RXRPC: ::c_int = 272; pub const SOL_PPPOL2TP: ::c_int = 273; @@ -487,17 +713,21 @@ pub const SOCK_SEQPACKET: ::c_int = 5; pub const SOCK_DCCP: ::c_int = 6; pub const SOCK_PACKET: ::c_int = 10; -pub const TCP_COOKIE_TRANSACTIONS: ::c_int = 15; -pub const TCP_THIN_LINEAR_TIMEOUTS: ::c_int = 16; -pub const TCP_THIN_DUPACK: ::c_int = 17; -pub const TCP_USER_TIMEOUT: ::c_int = 18; -pub const TCP_REPAIR: ::c_int = 19; -pub const TCP_REPAIR_QUEUE: ::c_int = 20; -pub const TCP_QUEUE_SEQ: ::c_int = 21; -pub const TCP_REPAIR_OPTIONS: ::c_int = 22; -pub const TCP_FASTOPEN: ::c_int = 23; -pub const TCP_TIMESTAMP: ::c_int = 24; -pub const TCP_FASTOPEN_CONNECT: ::c_int = 30; +pub const FAN_MARK_INODE: ::c_uint = 0x0000_0000; +pub const FAN_MARK_MOUNT: ::c_uint = 0x0000_0010; +// NOTE: FAN_MARK_FILESYSTEM requires Linux Kernel >= 4.20.0 +pub const FAN_MARK_FILESYSTEM: ::c_uint = 0x0000_0100; + +pub const AF_IB: ::c_int = 27; +pub const AF_MPLS: ::c_int = 28; +pub const AF_NFC: ::c_int = 39; +pub const AF_VSOCK: ::c_int = 40; +pub const AF_XDP: ::c_int = 44; +pub const PF_IB: ::c_int = AF_IB; +pub const PF_MPLS: ::c_int = AF_MPLS; +pub const PF_NFC: ::c_int = AF_NFC; +pub const PF_VSOCK: ::c_int = AF_VSOCK; +pub const PF_XDP: ::c_int = AF_XDP; /* DCCP socket options */ pub const DCCP_SOCKOPT_PACKET_SIZE: ::c_int = 1; @@ -525,6 +755,7 @@ pub const SIGEV_THREAD_ID: ::c_int = 4; pub const BUFSIZ: ::c_uint = 8192; pub const TMP_MAX: ::c_uint = 238328; pub const FOPEN_MAX: ::c_uint = 16; +pub const FILENAME_MAX: ::c_uint = 4096; pub const POSIX_MADV_DONTNEED: ::c_int = 4; pub const _SC_EQUIV_CLASS_MAX: ::c_int = 41; pub const _SC_CHARCLASS_NAME_MAX: ::c_int = 45; @@ -605,34 +836,18 @@ pub const O_ACCMODE: ::c_int = 3; pub const ST_RELATIME: ::c_ulong = 4096; pub const NI_MAXHOST: ::socklen_t = 1025; -pub const ADFS_SUPER_MAGIC: ::c_long = 0x0000adf5; -pub const AFFS_SUPER_MAGIC: ::c_long = 0x0000adff; -pub const CODA_SUPER_MAGIC: ::c_long = 0x73757245; -pub const CRAMFS_MAGIC: ::c_long = 0x28cd3d45; -pub const EFS_SUPER_MAGIC: ::c_long = 0x00414a53; -pub const EXT2_SUPER_MAGIC: ::c_long = 0x0000ef53; -pub const EXT3_SUPER_MAGIC: ::c_long = 0x0000ef53; -pub const EXT4_SUPER_MAGIC: ::c_long = 0x0000ef53; -pub const HPFS_SUPER_MAGIC: ::c_long = 0xf995e849; -pub const HUGETLBFS_MAGIC: ::c_long = 0x958458f6; -pub const ISOFS_SUPER_MAGIC: ::c_long = 0x00009660; -pub const JFFS2_SUPER_MAGIC: ::c_long = 0x000072b6; -pub const MINIX_SUPER_MAGIC: ::c_long = 0x0000137f; -pub const MINIX_SUPER_MAGIC2: ::c_long = 0x0000138f; -pub const MINIX2_SUPER_MAGIC: ::c_long = 0x00002468; -pub const MINIX2_SUPER_MAGIC2: ::c_long = 0x00002478; -pub const MSDOS_SUPER_MAGIC: ::c_long = 0x00004d44; -pub const NCP_SUPER_MAGIC: ::c_long = 0x0000564c; -pub const NFS_SUPER_MAGIC: ::c_long = 0x00006969; -pub const OPENPROM_SUPER_MAGIC: ::c_long = 0x00009fa1; -pub const PROC_SUPER_MAGIC: ::c_long = 0x00009fa0; -pub const QNX4_SUPER_MAGIC: ::c_long = 0x0000002f; -pub const REISERFS_SUPER_MAGIC: ::c_long = 0x52654973; -pub const SMB_SUPER_MAGIC: ::c_long = 0x0000517b; -pub const TMPFS_MAGIC: ::c_long = 0x01021994; -pub const USBDEVICE_SUPER_MAGIC: ::c_long = 0x00009fa2; -pub const CGROUP_SUPER_MAGIC: ::c_long = 0x27e0eb; -pub const CGROUP2_SUPER_MAGIC: ::c_long = 0x63677270; +// Most `*_SUPER_MAGIC` constants are defined at the `linux_like` level; the +// following are only available on newer Linux versions than the versions +// currently used in CI in some configurations, so we define them here. +cfg_if! { + if #[cfg(not(target_arch = "s390x"))] { + pub const BINDERFS_SUPER_MAGIC: ::c_long = 0x6c6f6f70; + pub const XFS_SUPER_MAGIC: ::c_long = 0x58465342; + } else if #[cfg(target_arch = "s390x")] { + pub const BINDERFS_SUPER_MAGIC: ::c_uint = 0x6c6f6f70; + pub const XFS_SUPER_MAGIC: ::c_uint = 0x58465342; + } +} pub const CPU_SETSIZE: ::c_int = 0x400; @@ -658,11 +873,16 @@ pub const PTRACE_SEIZE: ::c_uint = 0x4206; pub const PTRACE_INTERRUPT: ::c_uint = 0x4207; pub const PTRACE_LISTEN: ::c_uint = 0x4208; pub const PTRACE_PEEKSIGINFO: ::c_uint = 0x4209; +pub const PTRACE_GET_SYSCALL_INFO: ::c_uint = 0x420e; -pub const EPOLLWAKEUP: ::c_int = 0x20000000; +// linux/fs.h -pub const SEEK_DATA: ::c_int = 3; -pub const SEEK_HOLE: ::c_int = 4; +// Flags for preadv2/pwritev2 +pub const RWF_HIPRI: ::c_int = 0x00000001; +pub const RWF_DSYNC: ::c_int = 0x00000002; +pub const RWF_SYNC: ::c_int = 0x00000004; +pub const RWF_NOWAIT: ::c_int = 0x00000008; +pub const RWF_APPEND: ::c_int = 0x00000010; // linux/rtnetlink.h pub const TCA_PAD: ::c_ushort = 9; @@ -696,6 +916,10 @@ pub const NDA_MASTER: ::c_ushort = 9; pub const NDA_LINK_NETNSID: ::c_ushort = 10; pub const NDA_SRC_VNI: ::c_ushort = 11; +// linux/personality.h +pub const UNAME26: ::c_int = 0x0020000; +pub const FDPIC_FUNCPTRS: ::c_int = 0x0080000; + // linux/if_addr.h pub const IFA_FLAGS: ::c_ushort = 8; @@ -711,17 +935,26 @@ pub const GENL_UNS_ADMIN_PERM: ::c_int = 0x10; pub const GENL_ID_VFS_DQUOT: ::c_int = ::NLMSG_MIN_TYPE + 1; pub const GENL_ID_PMCRAID: ::c_int = ::NLMSG_MIN_TYPE + 2; -pub const TIOCM_LE: ::c_int = 0x001; -pub const TIOCM_DTR: ::c_int = 0x002; -pub const TIOCM_RTS: ::c_int = 0x004; -pub const TIOCM_CD: ::c_int = TIOCM_CAR; -pub const TIOCM_RI: ::c_int = TIOCM_RNG; - -pub const NF_NETDEV_INGRESS: ::c_int = 0; -pub const NF_NETDEV_NUMHOOKS: ::c_int = 1; - -pub const NFPROTO_INET: ::c_int = 1; -pub const NFPROTO_NETDEV: ::c_int = 5; +// elf.h +pub const NT_PRSTATUS: ::c_int = 1; +pub const NT_PRFPREG: ::c_int = 2; +pub const NT_FPREGSET: ::c_int = 2; +pub const NT_PRPSINFO: ::c_int = 3; +pub const NT_PRXREG: ::c_int = 4; +pub const NT_TASKSTRUCT: ::c_int = 4; +pub const NT_PLATFORM: ::c_int = 5; +pub const NT_AUXV: ::c_int = 6; +pub const NT_GWINDOWS: ::c_int = 7; +pub const NT_ASRS: ::c_int = 8; +pub const NT_PSTATUS: ::c_int = 10; +pub const NT_PSINFO: ::c_int = 13; +pub const NT_PRCRED: ::c_int = 14; +pub const NT_UTSNAME: ::c_int = 15; +pub const NT_LWPSTATUS: ::c_int = 16; +pub const NT_LWPSINFO: ::c_int = 17; +pub const NT_PRFPXREG: ::c_int = 20; + +pub const ELFOSABI_ARM_AEABI: u8 = 64; // linux/keyctl.h pub const KEYCTL_DH_COMPUTE: u32 = 23; @@ -754,193 +987,6 @@ cfg_if! { } } -// linux/netfilter/nf_tables.h -pub const NFT_TABLE_MAXNAMELEN: ::c_int = 256; -pub const NFT_CHAIN_MAXNAMELEN: ::c_int = 256; -pub const NFT_SET_MAXNAMELEN: ::c_int = 256; -pub const NFT_OBJ_MAXNAMELEN: ::c_int = 256; -pub const NFT_USERDATA_MAXLEN: ::c_int = 256; - -pub const NFT_REG_VERDICT: ::c_int = 0; -pub const NFT_REG_1: ::c_int = 1; -pub const NFT_REG_2: ::c_int = 2; -pub const NFT_REG_3: ::c_int = 3; -pub const NFT_REG_4: ::c_int = 4; -pub const __NFT_REG_MAX: ::c_int = 5; -pub const NFT_REG32_00: ::c_int = 8; -pub const NFT_REG32_01: ::c_int = 9; -pub const NFT_REG32_02: ::c_int = 10; -pub const NFT_REG32_03: ::c_int = 11; -pub const NFT_REG32_04: ::c_int = 12; -pub const NFT_REG32_05: ::c_int = 13; -pub const NFT_REG32_06: ::c_int = 14; -pub const NFT_REG32_07: ::c_int = 15; -pub const NFT_REG32_08: ::c_int = 16; -pub const NFT_REG32_09: ::c_int = 17; -pub const NFT_REG32_10: ::c_int = 18; -pub const NFT_REG32_11: ::c_int = 19; -pub const NFT_REG32_12: ::c_int = 20; -pub const NFT_REG32_13: ::c_int = 21; -pub const NFT_REG32_14: ::c_int = 22; -pub const NFT_REG32_15: ::c_int = 23; - -pub const NFT_REG_SIZE: ::c_int = 16; -pub const NFT_REG32_SIZE: ::c_int = 4; - -pub const NFT_CONTINUE: ::c_int = -1; -pub const NFT_BREAK: ::c_int = -2; -pub const NFT_JUMP: ::c_int = -3; -pub const NFT_GOTO: ::c_int = -4; -pub const NFT_RETURN: ::c_int = -5; - -pub const NFT_MSG_NEWTABLE: ::c_int = 0; -pub const NFT_MSG_GETTABLE: ::c_int = 1; -pub const NFT_MSG_DELTABLE: ::c_int = 2; -pub const NFT_MSG_NEWCHAIN: ::c_int = 3; -pub const NFT_MSG_GETCHAIN: ::c_int = 4; -pub const NFT_MSG_DELCHAIN: ::c_int = 5; -pub const NFT_MSG_NEWRULE: ::c_int = 6; -pub const NFT_MSG_GETRULE: ::c_int = 7; -pub const NFT_MSG_DELRULE: ::c_int = 8; -pub const NFT_MSG_NEWSET: ::c_int = 9; -pub const NFT_MSG_GETSET: ::c_int = 10; -pub const NFT_MSG_DELSET: ::c_int = 11; -pub const NFT_MSG_NEWSETELEM: ::c_int = 12; -pub const NFT_MSG_GETSETELEM: ::c_int = 13; -pub const NFT_MSG_DELSETELEM: ::c_int = 14; -pub const NFT_MSG_NEWGEN: ::c_int = 15; -pub const NFT_MSG_GETGEN: ::c_int = 16; -pub const NFT_MSG_TRACE: ::c_int = 17; -cfg_if! { - if #[cfg(not(target_arch = "sparc64"))] { - pub const NFT_MSG_NEWOBJ: ::c_int = 18; - pub const NFT_MSG_GETOBJ: ::c_int = 19; - pub const NFT_MSG_DELOBJ: ::c_int = 20; - pub const NFT_MSG_GETOBJ_RESET: ::c_int = 21; - } -} -pub const NFT_MSG_MAX: ::c_int = 25; - -pub const NFT_SET_ANONYMOUS: ::c_int = 0x1; -pub const NFT_SET_CONSTANT: ::c_int = 0x2; -pub const NFT_SET_INTERVAL: ::c_int = 0x4; -pub const NFT_SET_MAP: ::c_int = 0x8; -pub const NFT_SET_TIMEOUT: ::c_int = 0x10; -pub const NFT_SET_EVAL: ::c_int = 0x20; - -pub const NFT_SET_POL_PERFORMANCE: ::c_int = 0; -pub const NFT_SET_POL_MEMORY: ::c_int = 1; - -pub const NFT_SET_ELEM_INTERVAL_END: ::c_int = 0x1; - -pub const NFT_DATA_VALUE: ::c_uint = 0; -pub const NFT_DATA_VERDICT: ::c_uint = 0xffffff00; - -pub const NFT_DATA_RESERVED_MASK: ::c_uint = 0xffffff00; - -pub const NFT_DATA_VALUE_MAXLEN: ::c_int = 64; - -pub const NFT_BYTEORDER_NTOH: ::c_int = 0; -pub const NFT_BYTEORDER_HTON: ::c_int = 1; - -pub const NFT_CMP_EQ: ::c_int = 0; -pub const NFT_CMP_NEQ: ::c_int = 1; -pub const NFT_CMP_LT: ::c_int = 2; -pub const NFT_CMP_LTE: ::c_int = 3; -pub const NFT_CMP_GT: ::c_int = 4; -pub const NFT_CMP_GTE: ::c_int = 5; - -pub const NFT_RANGE_EQ: ::c_int = 0; -pub const NFT_RANGE_NEQ: ::c_int = 1; - -pub const NFT_LOOKUP_F_INV: ::c_int = 1 << 0; - -pub const NFT_DYNSET_OP_ADD: ::c_int = 0; -pub const NFT_DYNSET_OP_UPDATE: ::c_int = 1; - -pub const NFT_DYNSET_F_INV: ::c_int = 1 << 0; - -pub const NFT_PAYLOAD_LL_HEADER: ::c_int = 0; -pub const NFT_PAYLOAD_NETWORK_HEADER: ::c_int = 1; -pub const NFT_PAYLOAD_TRANSPORT_HEADER: ::c_int = 2; - -pub const NFT_PAYLOAD_CSUM_NONE: ::c_int = 0; -pub const NFT_PAYLOAD_CSUM_INET: ::c_int = 1; - -pub const NFT_META_LEN: ::c_int = 0; -pub const NFT_META_PROTOCOL: ::c_int = 1; -pub const NFT_META_PRIORITY: ::c_int = 2; -pub const NFT_META_MARK: ::c_int = 3; -pub const NFT_META_IIF: ::c_int = 4; -pub const NFT_META_OIF: ::c_int = 5; -pub const NFT_META_IIFNAME: ::c_int = 6; -pub const NFT_META_OIFNAME: ::c_int = 7; -pub const NFT_META_IIFTYPE: ::c_int = 8; -pub const NFT_META_OIFTYPE: ::c_int = 9; -pub const NFT_META_SKUID: ::c_int = 10; -pub const NFT_META_SKGID: ::c_int = 11; -pub const NFT_META_NFTRACE: ::c_int = 12; -pub const NFT_META_RTCLASSID: ::c_int = 13; -pub const NFT_META_SECMARK: ::c_int = 14; -pub const NFT_META_NFPROTO: ::c_int = 15; -pub const NFT_META_L4PROTO: ::c_int = 16; -pub const NFT_META_BRI_IIFNAME: ::c_int = 17; -pub const NFT_META_BRI_OIFNAME: ::c_int = 18; -pub const NFT_META_PKTTYPE: ::c_int = 19; -pub const NFT_META_CPU: ::c_int = 20; -pub const NFT_META_IIFGROUP: ::c_int = 21; -pub const NFT_META_OIFGROUP: ::c_int = 22; -pub const NFT_META_CGROUP: ::c_int = 23; -pub const NFT_META_PRANDOM: ::c_int = 24; - -pub const NFT_CT_STATE: ::c_int = 0; -pub const NFT_CT_DIRECTION: ::c_int = 1; -pub const NFT_CT_STATUS: ::c_int = 2; -pub const NFT_CT_MARK: ::c_int = 3; -pub const NFT_CT_SECMARK: ::c_int = 4; -pub const NFT_CT_EXPIRATION: ::c_int = 5; -pub const NFT_CT_HELPER: ::c_int = 6; -pub const NFT_CT_L3PROTOCOL: ::c_int = 7; -pub const NFT_CT_SRC: ::c_int = 8; -pub const NFT_CT_DST: ::c_int = 9; -pub const NFT_CT_PROTOCOL: ::c_int = 10; -pub const NFT_CT_PROTO_SRC: ::c_int = 11; -pub const NFT_CT_PROTO_DST: ::c_int = 12; -pub const NFT_CT_LABELS: ::c_int = 13; -pub const NFT_CT_PKTS: ::c_int = 14; -pub const NFT_CT_BYTES: ::c_int = 15; - -pub const NFT_LIMIT_PKTS: ::c_int = 0; -pub const NFT_LIMIT_PKT_BYTES: ::c_int = 1; - -pub const NFT_LIMIT_F_INV: ::c_int = 1 << 0; - -pub const NFT_QUEUE_FLAG_BYPASS: ::c_int = 0x01; -pub const NFT_QUEUE_FLAG_CPU_FANOUT: ::c_int = 0x02; -pub const NFT_QUEUE_FLAG_MASK: ::c_int = 0x03; - -pub const NFT_QUOTA_F_INV: ::c_int = 1 << 0; - -pub const NFT_REJECT_ICMP_UNREACH: ::c_int = 0; -pub const NFT_REJECT_TCP_RST: ::c_int = 1; -pub const NFT_REJECT_ICMPX_UNREACH: ::c_int = 2; - -pub const NFT_REJECT_ICMPX_NO_ROUTE: ::c_int = 0; -pub const NFT_REJECT_ICMPX_PORT_UNREACH: ::c_int = 1; -pub const NFT_REJECT_ICMPX_HOST_UNREACH: ::c_int = 2; -pub const NFT_REJECT_ICMPX_ADMIN_PROHIBITED: ::c_int = 3; - -pub const NFT_NAT_SNAT: ::c_int = 0; -pub const NFT_NAT_DNAT: ::c_int = 1; - -pub const NFT_TRACETYPE_UNSPEC: ::c_int = 0; -pub const NFT_TRACETYPE_POLICY: ::c_int = 1; -pub const NFT_TRACETYPE_RETURN: ::c_int = 2; -pub const NFT_TRACETYPE_RULE: ::c_int = 3; - -pub const NFT_NG_INCREMENTAL: ::c_int = 0; -pub const NFT_NG_RANDOM: ::c_int = 1; - pub const M_MXFAST: ::c_int = 1; pub const M_NLBLKS: ::c_int = 2; pub const M_GRAIN: ::c_int = 3; @@ -971,6 +1017,7 @@ pub const STATX_SIZE: ::c_uint = 0x0200; pub const STATX_BLOCKS: ::c_uint = 0x0400; pub const STATX_BASIC_STATS: ::c_uint = 0x07ff; pub const STATX_BTIME: ::c_uint = 0x0800; +pub const STATX_MNT_ID: ::c_uint = 0x1000; pub const STATX_ALL: ::c_uint = 0x0fff; pub const STATX__RESERVED: ::c_int = 0x80000000; pub const STATX_ATTR_COMPRESSED: ::c_int = 0x0004; @@ -979,33 +1026,11 @@ pub const STATX_ATTR_APPEND: ::c_int = 0x0020; pub const STATX_ATTR_NODUMP: ::c_int = 0x0040; pub const STATX_ATTR_ENCRYPTED: ::c_int = 0x0800; pub const STATX_ATTR_AUTOMOUNT: ::c_int = 0x1000; +pub const STATX_ATTR_MOUNT_ROOT: ::c_int = 0x2000; +pub const STATX_ATTR_VERITY: ::c_int = 0x00100000; +pub const STATX_ATTR_DAX: ::c_int = 0x00200000; -// sys/auxv.h -pub const AT_NULL: ::c_ulong = 0; -pub const AT_IGNORE: ::c_ulong = 1; -pub const AT_EXECFD: ::c_ulong = 2; -pub const AT_PHDR: ::c_ulong = 3; -pub const AT_PHENT: ::c_ulong = 4; -pub const AT_PHNUM: ::c_ulong = 5; -pub const AT_PAGESZ: ::c_ulong = 6; -pub const AT_BASE: ::c_ulong = 7; -pub const AT_FLAGS: ::c_ulong = 8; -pub const AT_ENTRY: ::c_ulong = 9; -pub const AT_NOTELF: ::c_ulong = 10; -pub const AT_UID: ::c_ulong = 11; -pub const AT_EUID: ::c_ulong = 12; -pub const AT_GID: ::c_ulong = 13; -pub const AT_EGID: ::c_ulong = 14; -pub const AT_PLATFORM: ::c_ulong = 15; -pub const AT_HWCAP: ::c_ulong = 16; -pub const AT_CLKTCK: ::c_ulong = 17; -// AT_* values 18 through 22 are reserved -pub const AT_SECURE: ::c_ulong = 23; -pub const AT_BASE_PLATFORM: ::c_ulong = 24; -pub const AT_RANDOM: ::c_ulong = 25; -pub const AT_HWCAP2: ::c_ulong = 26; - -pub const AT_EXECFN: ::c_ulong = 31; +pub const SOMAXCONN: ::c_int = 4096; //sys/timex.h pub const ADJ_OFFSET: ::c_uint = 0x0001; @@ -1072,7 +1097,8 @@ cfg_if! { target_arch = "x86", target_arch = "x86_64", target_arch = "s390x", - target_arch = "riscv64" + target_arch = "riscv64", + target_arch = "riscv32" ))] { pub const PTHREAD_STACK_MIN: ::size_t = 16384; } else if #[cfg(any( @@ -1086,6 +1112,12 @@ cfg_if! { } pub const PTHREAD_MUTEX_ADAPTIVE_NP: ::c_int = 3; +pub const REG_STARTEND: ::c_int = 4; + +pub const REG_EEND: ::c_int = 14; +pub const REG_ESIZE: ::c_int = 15; +pub const REG_ERPAREN: ::c_int = 16; + extern "C" { pub fn fgetspent_r( fp: *mut ::FILE, @@ -1112,11 +1144,7 @@ extern "C" { num: ::size_t, size: ::size_t, compar: ::Option< - unsafe extern "C" fn( - *const ::c_void, - *const ::c_void, - *mut ::c_void, - ) -> ::c_int, + unsafe extern "C" fn(*const ::c_void, *const ::c_void, *mut ::c_void) -> ::c_int, >, arg: *mut ::c_void, ); @@ -1134,22 +1162,10 @@ extern "C" { timeout: *mut ::timespec, ) -> ::c_int; - pub fn getrlimit64( - resource: ::__rlimit_resource_t, - rlim: *mut ::rlimit64, - ) -> ::c_int; - pub fn setrlimit64( - resource: ::__rlimit_resource_t, - rlim: *const ::rlimit64, - ) -> ::c_int; - pub fn getrlimit( - resource: ::__rlimit_resource_t, - rlim: *mut ::rlimit, - ) -> ::c_int; - pub fn setrlimit( - resource: ::__rlimit_resource_t, - rlim: *const ::rlimit, - ) -> ::c_int; + pub fn getrlimit64(resource: ::__rlimit_resource_t, rlim: *mut ::rlimit64) -> ::c_int; + pub fn setrlimit64(resource: ::__rlimit_resource_t, rlim: *const ::rlimit64) -> ::c_int; + pub fn getrlimit(resource: ::__rlimit_resource_t, rlim: *mut ::rlimit) -> ::c_int; + pub fn setrlimit(resource: ::__rlimit_resource_t, rlim: *const ::rlimit) -> ::c_int; pub fn prlimit( pid: ::pid_t, resource: ::__rlimit_resource_t, @@ -1180,36 +1196,82 @@ extern "C" { mask: ::c_uint, statxbuf: *mut statx, ) -> ::c_int; - pub fn getrandom( - buf: *mut ::c_void, - buflen: ::size_t, - flags: ::c_uint, - ) -> ::ssize_t; - - pub fn memmem( - haystack: *const ::c_void, - haystacklen: ::size_t, - needle: *const ::c_void, - needlelen: ::size_t, - ) -> *mut ::c_void; + pub fn getentropy(buf: *mut ::c_void, buflen: ::size_t) -> ::c_int; + pub fn getrandom(buf: *mut ::c_void, buflen: ::size_t, flags: ::c_uint) -> ::ssize_t; pub fn getauxval(type_: ::c_ulong) -> ::c_ulong; pub fn adjtimex(buf: *mut timex) -> ::c_int; pub fn ntp_adjtime(buf: *mut timex) -> ::c_int; #[link_name = "ntp_gettimex"] pub fn ntp_gettime(buf: *mut ntptimeval) -> ::c_int; + pub fn clock_adjtime(clk_id: ::clockid_t, buf: *mut ::timex) -> ::c_int; + + pub fn copy_file_range( + fd_in: ::c_int, + off_in: *mut ::off64_t, + fd_out: ::c_int, + off_out: *mut ::off64_t, + len: ::size_t, + flags: ::c_uint, + ) -> ::ssize_t; + pub fn fanotify_mark( + fd: ::c_int, + flags: ::c_uint, + mask: u64, + dirfd: ::c_int, + path: *const ::c_char, + ) -> ::c_int; + pub fn preadv2( + fd: ::c_int, + iov: *const ::iovec, + iovcnt: ::c_int, + offset: ::off_t, + flags: ::c_int, + ) -> ::ssize_t; + pub fn pwritev2( + fd: ::c_int, + iov: *const ::iovec, + iovcnt: ::c_int, + offset: ::off_t, + flags: ::c_int, + ) -> ::ssize_t; + pub fn preadv64v2( + fd: ::c_int, + iov: *const ::iovec, + iovcnt: ::c_int, + offset: ::off64_t, + flags: ::c_int, + ) -> ::ssize_t; + pub fn pwritev64v2( + fd: ::c_int, + iov: *const ::iovec, + iovcnt: ::c_int, + offset: ::off64_t, + flags: ::c_int, + ) -> ::ssize_t; + pub fn renameat2( + olddirfd: ::c_int, + oldpath: *const ::c_char, + newdirfd: ::c_int, + newpath: *const ::c_char, + flags: ::c_uint, + ) -> ::c_int; + + // Added in `glibc` 2.25 + pub fn explicit_bzero(s: *mut ::c_void, len: ::size_t); + // Added in `glibc` 2.29 + pub fn reallocarray(ptr: *mut ::c_void, nmemb: ::size_t, size: ::size_t) -> *mut ::c_void; + + pub fn ctermid(s: *mut ::c_char) -> *mut ::c_char; } -#[link(name = "util")] extern "C" { pub fn ioctl(fd: ::c_int, request: ::c_ulong, ...) -> ::c_int; pub fn backtrace(buf: *mut *mut ::c_void, sz: ::c_int) -> ::c_int; pub fn glob64( pattern: *const ::c_char, flags: ::c_int, - errfunc: ::Option< - extern "C" fn(epath: *const ::c_char, errno: ::c_int) -> ::c_int, - >, + errfunc: ::Option ::c_int>, pglob: *mut glob64_t, ) -> ::c_int; pub fn globfree64(pglob: *mut glob64_t); @@ -1225,21 +1287,7 @@ extern "C" { cpuset: *const ::cpu_set_t, ) -> ::c_int; pub fn getpriority(which: ::__priority_which_t, who: ::id_t) -> ::c_int; - pub fn setpriority( - which: ::__priority_which_t, - who: ::id_t, - prio: ::c_int, - ) -> ::c_int; - pub fn pthread_getaffinity_np( - thread: ::pthread_t, - cpusetsize: ::size_t, - cpuset: *mut ::cpu_set_t, - ) -> ::c_int; - pub fn pthread_setaffinity_np( - thread: ::pthread_t, - cpusetsize: ::size_t, - cpuset: *const ::cpu_set_t, - ) -> ::c_int; + pub fn setpriority(which: ::__priority_which_t, who: ::id_t, prio: ::c_int) -> ::c_int; pub fn pthread_rwlockattr_getkind_np( attr: *const ::pthread_rwlockattr_t, val: *mut ::c_int, @@ -1248,56 +1296,91 @@ extern "C" { attr: *mut ::pthread_rwlockattr_t, val: ::c_int, ) -> ::c_int; - pub fn sched_getcpu() -> ::c_int; + pub fn pthread_sigqueue(thread: ::pthread_t, sig: ::c_int, value: ::sigval) -> ::c_int; pub fn mallinfo() -> ::mallinfo; + pub fn mallinfo2() -> ::mallinfo2; + pub fn malloc_info(options: ::c_int, stream: *mut ::FILE) -> ::c_int; pub fn malloc_usable_size(ptr: *mut ::c_void) -> ::size_t; - #[cfg_attr(target_os = "netbsd", link_name = "__getpwent_r50")] - #[cfg_attr(target_os = "solaris", link_name = "__posix_getpwent_r")] pub fn getpwent_r( pwd: *mut ::passwd, buf: *mut ::c_char, buflen: ::size_t, result: *mut *mut ::passwd, ) -> ::c_int; - #[cfg_attr(target_os = "netbsd", link_name = "__getgrent_r50")] - #[cfg_attr(target_os = "solaris", link_name = "__posix_getgrent_r")] pub fn getgrent_r( grp: *mut ::group, buf: *mut ::c_char, buflen: ::size_t, result: *mut *mut ::group, ) -> ::c_int; - pub fn pthread_getname_np( - thread: ::pthread_t, - name: *mut ::c_char, - len: ::size_t, + pub fn fgetpwent_r( + stream: *mut ::FILE, + pwd: *mut ::passwd, + buf: *mut ::c_char, + buflen: ::size_t, + result: *mut *mut ::passwd, ) -> ::c_int; - pub fn pthread_setname_np( - thread: ::pthread_t, - name: *const ::c_char, + pub fn fgetgrent_r( + stream: *mut ::FILE, + grp: *mut ::group, + buf: *mut ::c_char, + buflen: ::size_t, + result: *mut *mut ::group, ) -> ::c_int; + + pub fn sethostid(hostid: ::c_long) -> ::c_int; + + pub fn memfd_create(name: *const ::c_char, flags: ::c_uint) -> ::c_int; + pub fn mlock2(addr: *const ::c_void, len: ::size_t, flags: ::c_uint) -> ::c_int; + + pub fn euidaccess(pathname: *const ::c_char, mode: ::c_int) -> ::c_int; + pub fn eaccess(pathname: *const ::c_char, mode: ::c_int) -> ::c_int; + + pub fn asctime_r(tm: *const ::tm, buf: *mut ::c_char) -> *mut ::c_char; + pub fn ctime_r(timep: *const time_t, buf: *mut ::c_char) -> *mut ::c_char; + + pub fn strftime( + s: *mut ::c_char, + max: ::size_t, + format: *const ::c_char, + tm: *const ::tm, + ) -> ::size_t; + pub fn strptime(s: *const ::c_char, format: *const ::c_char, tm: *mut ::tm) -> *mut ::c_char; + + pub fn dirname(path: *mut ::c_char) -> *mut ::c_char; + /// POSIX version of `basename(3)`, defined in `libgen.h`. + #[link_name = "__xpg_basename"] + pub fn posix_basename(path: *mut ::c_char) -> *mut ::c_char; + /// GNU version of `basename(3)`, defined in `string.h`. + #[link_name = "basename"] + pub fn gnu_basename(path: *const ::c_char) -> *mut ::c_char; } -#[link(name = "dl")] extern "C" { - pub fn dlmopen( - lmid: Lmid_t, - filename: *const ::c_char, - flag: ::c_int, - ) -> *mut ::c_void; - pub fn dlinfo( - handle: *mut ::c_void, - request: ::c_int, - info: *mut ::c_void, + pub fn dlmopen(lmid: Lmid_t, filename: *const ::c_char, flag: ::c_int) -> *mut ::c_void; + pub fn dlinfo(handle: *mut ::c_void, request: ::c_int, info: *mut ::c_void) -> ::c_int; + pub fn dladdr1( + addr: *const ::c_void, + info: *mut ::Dl_info, + extra_info: *mut *mut ::c_void, + flags: ::c_int, ) -> ::c_int; + pub fn malloc_trim(__pad: ::size_t) -> ::c_int; +} + +extern "C" { + pub fn gnu_get_libc_release() -> *const ::c_char; + pub fn gnu_get_libc_version() -> *const ::c_char; } cfg_if! { if #[cfg(any(target_arch = "x86", target_arch = "arm", + target_arch = "m68k", target_arch = "mips", target_arch = "powerpc", - target_arch = "sparc"))] { + target_arch = "sparc", + target_arch = "riscv32"))] { mod b32; pub use self::b32::*; } else if #[cfg(any(target_arch = "x86_64", @@ -1306,7 +1389,8 @@ cfg_if! { target_arch = "mips64", target_arch = "s390x", target_arch = "sparc64", - target_arch = "riscv64"))] { + target_arch = "riscv64", + target_arch = "loongarch64"))] { mod b64; pub use self::b64::*; } else { diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/mod.rs index e46e6b241..f0a0820c3 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/mod.rs @@ -14,6 +14,7 @@ pub type nl_item = ::c_int; pub type idtype_t = ::c_uint; pub type loff_t = ::c_longlong; pub type pthread_key_t = ::c_uint; +pub type pthread_spinlock_t = ::c_int; pub type __u8 = ::c_uchar; pub type __u16 = ::c_ushort; @@ -36,6 +37,17 @@ pub type Elf64_Sxword = i64; pub type Elf32_Section = u16; pub type Elf64_Section = u16; +// linux/can.h +pub type canid_t = u32; + +// linux/can/j1939.h +pub type can_err_mask_t = u32; +pub type pgn_t = u32; +pub type priority_t = u8; +pub type name_t = u64; + +pub type iconv_t = *mut ::c_void; + #[cfg_attr(feature = "extra_traits", derive(Debug))] pub enum fpos64_t {} // FIXME: fill this out with a struct impl ::Copy for fpos64_t {} @@ -278,6 +290,24 @@ s! { pub u: [u32; 7], } + pub struct uinput_ff_upload { + pub request_id: ::__u32, + pub retval: ::__s32, + pub effect: ff_effect, + pub old: ff_effect, + } + + pub struct uinput_ff_erase { + pub request_id: ::__u32, + pub retval: ::__s32, + pub effect_id: ::__u32, + } + + pub struct uinput_abs_setup { + pub code: ::__u16, + pub absinfo: input_absinfo, + } + pub struct dl_phdr_info { #[cfg(target_pointer_width = "64")] pub dlpi_addr: Elf64_Addr, @@ -296,9 +326,18 @@ s! { #[cfg(target_pointer_width = "32")] pub dlpi_phnum: Elf32_Half, + // As of uClibc 1.0.36, the following fields are + // gated behind a "#if 0" block which always evaluates + // to false. So I'm just removing these, and if uClibc changes + // the #if block in the future to include the following fields, these + // will probably need including here. tsidea, skrap + #[cfg(not(target_env = "uclibc"))] pub dlpi_adds: ::c_ulonglong, + #[cfg(not(target_env = "uclibc"))] pub dlpi_subs: ::c_ulonglong, + #[cfg(not(target_env = "uclibc"))] pub dlpi_tls_modid: ::size_t, + #[cfg(not(target_env = "uclibc"))] pub dlpi_tls_data: *mut ::c_void, } @@ -402,19 +441,6 @@ s! { pub sh_entsize: Elf64_Xword, } - pub struct Elf32_Chdr { - pub ch_type: Elf32_Word, - pub ch_size: Elf32_Word, - pub ch_addralign: Elf32_Word, - } - - pub struct Elf64_Chdr { - pub ch_type: Elf64_Word, - pub ch_reserved: Elf64_Word, - pub ch_size: Elf64_Xword, - pub ch_addralign: Elf64_Xword, - } - pub struct ucred { pub pid: ::pid_t, pub uid: ::uid_t, @@ -477,6 +503,11 @@ s! { pub len: u32 } + pub struct fanotify_response { + pub fd: ::c_int, + pub response: __u32, + } + pub struct sockaddr_vm { pub svm_family: ::sa_family_t, pub svm_reserved1: ::c_ushort, @@ -484,6 +515,116 @@ s! { pub svm_cid: ::c_uint, pub svm_zero: [u8; 4] } + + pub struct regmatch_t { + pub rm_so: regoff_t, + pub rm_eo: regoff_t, + } + + pub struct sock_extended_err { + pub ee_errno: u32, + pub ee_origin: u8, + pub ee_type: u8, + pub ee_code: u8, + pub ee_pad: u8, + pub ee_info: u32, + pub ee_data: u32, + } + + // linux/can.h + pub struct __c_anonymous_sockaddr_can_tp { + pub rx_id: canid_t, + pub tx_id: canid_t, + } + + pub struct __c_anonymous_sockaddr_can_j1939 { + pub name: u64, + pub pgn: u32, + pub addr: u8, + } + + pub struct can_filter { + pub can_id: canid_t, + pub can_mask: canid_t, + } + + // linux/can/j1939.h + pub struct j1939_filter { + pub name: name_t, + pub name_mask: name_t, + pub pgn: pgn_t, + pub pgn_mask: pgn_t, + pub addr: u8, + pub addr_mask: u8, + } + + // linux/filter.h + pub struct sock_filter { + pub code: ::__u16, + pub jt: ::__u8, + pub jf: ::__u8, + pub k: ::__u32, + } + + pub struct sock_fprog { + pub len: ::c_ushort, + pub filter: *mut sock_filter, + } + + // linux/seccomp.h + pub struct seccomp_data { + pub nr: ::c_int, + pub arch: ::__u32, + pub instruction_pointer: ::__u64, + pub args: [::__u64; 6], + } + + pub struct nlmsghdr { + pub nlmsg_len: u32, + pub nlmsg_type: u16, + pub nlmsg_flags: u16, + pub nlmsg_seq: u32, + pub nlmsg_pid: u32, + } + + pub struct nlmsgerr { + pub error: ::c_int, + pub msg: nlmsghdr, + } + + pub struct nlattr { + pub nla_len: u16, + pub nla_type: u16, + } + + pub struct file_clone_range { + pub src_fd: ::__s64, + pub src_offset: ::__u64, + pub src_length: ::__u64, + pub dest_offset: ::__u64, + } + + pub struct __c_anonymous_ifru_map { + pub mem_start: ::c_ulong, + pub mem_end: ::c_ulong, + pub base_addr: ::c_ushort, + pub irq: ::c_uchar, + pub dma: ::c_uchar, + pub port: ::c_uchar, + } + + pub struct in6_ifreq { + pub ifr6_addr: ::in6_addr, + pub ifr6_prefixlen: u32, + pub ifr6_ifindex: ::c_int, + } + + pub struct option { + pub name: *const ::c_char, + pub has_arg: ::c_int, + pub flag: *mut ::c_int, + pub val: ::c_int, + } } s_no_extra_traits! { @@ -518,6 +659,29 @@ s_no_extra_traits! { pub salg_name: [::c_uchar; 64], } + pub struct uinput_setup { + pub id: input_id, + pub name: [::c_char; UINPUT_MAX_NAME_SIZE], + pub ff_effects_max: ::__u32, + } + + pub struct uinput_user_dev { + pub name: [::c_char; UINPUT_MAX_NAME_SIZE], + pub id: input_id, + pub ff_effects_max: ::__u32, + pub absmax: [::__s32; ABS_CNT], + pub absmin: [::__s32; ABS_CNT], + pub absfuzz: [::__s32; ABS_CNT], + pub absflat: [::__s32; ABS_CNT], + } + + /// WARNING: The `PartialEq`, `Eq` and `Hash` implementations of this + /// type are unsound and will be removed in the future. + #[deprecated( + note = "this struct has unsafe trait implementations that will be \ + removed in the future", + since = "0.2.80" + )] pub struct af_alg_iv { pub ivlen: u32, pub iv: [::c_uchar; 0], @@ -548,6 +712,61 @@ s_no_extra_traits! { #[cfg(not(all(target_arch = "x86_64", target_pointer_width = "32")))] pad: [::c_long; 4], } + + #[cfg(libc_union)] + pub union __c_anonymous_ifr_ifru { + pub ifru_addr: ::sockaddr, + pub ifru_dstaddr: ::sockaddr, + pub ifru_broadaddr: ::sockaddr, + pub ifru_netmask: ::sockaddr, + pub ifru_hwaddr: ::sockaddr, + pub ifru_flags: ::c_short, + pub ifru_ifindex: ::c_int, + pub ifru_metric: ::c_int, + pub ifru_mtu: ::c_int, + pub ifru_map: __c_anonymous_ifru_map, + pub ifru_slave: [::c_char; ::IFNAMSIZ], + pub ifru_newname: [::c_char; ::IFNAMSIZ], + pub ifru_data: *mut ::c_char, + } + + pub struct ifreq { + /// interface name, e.g. "en0" + pub ifr_name: [::c_char; ::IFNAMSIZ], + #[cfg(libc_union)] + pub ifr_ifru: __c_anonymous_ifr_ifru, + #[cfg(not(libc_union))] + pub ifr_ifru: ::sockaddr, + } +} + +s_no_extra_traits! { + // linux/net_tstamp.h + #[allow(missing_debug_implementations)] + pub struct sock_txtime { + pub clockid: ::clockid_t, + pub flags: ::__u32, + } +} + +cfg_if! { + if #[cfg(libc_union)] { + s_no_extra_traits! { + // linux/can.h + #[allow(missing_debug_implementations)] + pub union __c_anonymous_sockaddr_can_can_addr { + pub tp: __c_anonymous_sockaddr_can_tp, + pub j1939: __c_anonymous_sockaddr_can_j1939, + } + + #[allow(missing_debug_implementations)] + pub struct sockaddr_can { + pub can_family: ::sa_family_t, + pub can_ifindex: ::c_int, + pub can_addr: __c_anonymous_sockaddr_can_can_addr, + } + } + } } cfg_if! { @@ -761,6 +980,73 @@ cfg_if! { } } + impl PartialEq for uinput_setup { + fn eq(&self, other: &uinput_setup) -> bool { + self.id == other.id + && self.name[..] == other.name[..] + && self.ff_effects_max == other.ff_effects_max + } + } + impl Eq for uinput_setup {} + + impl ::fmt::Debug for uinput_setup { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("uinput_setup") + .field("id", &self.id) + .field("name", &&self.name[..]) + .field("ff_effects_max", &self.ff_effects_max) + .finish() + } + } + + impl ::hash::Hash for uinput_setup { + fn hash(&self, state: &mut H) { + self.id.hash(state); + self.name.hash(state); + self.ff_effects_max.hash(state); + } + } + + impl PartialEq for uinput_user_dev { + fn eq(&self, other: &uinput_user_dev) -> bool { + self.name[..] == other.name[..] + && self.id == other.id + && self.ff_effects_max == other.ff_effects_max + && self.absmax[..] == other.absmax[..] + && self.absmin[..] == other.absmin[..] + && self.absfuzz[..] == other.absfuzz[..] + && self.absflat[..] == other.absflat[..] + } + } + impl Eq for uinput_user_dev {} + + impl ::fmt::Debug for uinput_user_dev { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("uinput_setup") + .field("name", &&self.name[..]) + .field("id", &self.id) + .field("ff_effects_max", &self.ff_effects_max) + .field("absmax", &&self.absmax[..]) + .field("absmin", &&self.absmin[..]) + .field("absfuzz", &&self.absfuzz[..]) + .field("absflat", &&self.absflat[..]) + .finish() + } + } + + impl ::hash::Hash for uinput_user_dev { + fn hash(&self, state: &mut H) { + self.name.hash(state); + self.id.hash(state); + self.ff_effects_max.hash(state); + self.absmax.hash(state); + self.absmin.hash(state); + self.absfuzz.hash(state); + self.absflat.hash(state); + } + } + + #[allow(deprecated)] impl af_alg_iv { fn as_slice(&self) -> &[u8] { unsafe { @@ -772,22 +1058,26 @@ cfg_if! { } } + #[allow(deprecated)] impl PartialEq for af_alg_iv { fn eq(&self, other: &af_alg_iv) -> bool { *self.as_slice() == *other.as_slice() } } + #[allow(deprecated)] impl Eq for af_alg_iv {} + #[allow(deprecated)] impl ::fmt::Debug for af_alg_iv { fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { f.debug_struct("af_alg_iv") - .field("iv", &self.as_slice()) + .field("ivlen", &self.ivlen) .finish() } } + #[allow(deprecated)] impl ::hash::Hash for af_alg_iv { fn hash(&self, state: &mut H) { self.as_slice().hash(state); @@ -821,81 +1111,107 @@ cfg_if! { self.mq_curmsgs.hash(state); } } + #[cfg(libc_union)] + impl ::fmt::Debug for __c_anonymous_ifr_ifru { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("ifr_ifru") + .field("ifru_addr", unsafe { &self.ifru_addr }) + .field("ifru_dstaddr", unsafe { &self.ifru_dstaddr }) + .field("ifru_broadaddr", unsafe { &self.ifru_broadaddr }) + .field("ifru_netmask", unsafe { &self.ifru_netmask }) + .field("ifru_hwaddr", unsafe { &self.ifru_hwaddr }) + .field("ifru_flags", unsafe { &self.ifru_flags }) + .field("ifru_ifindex", unsafe { &self.ifru_ifindex }) + .field("ifru_metric", unsafe { &self.ifru_metric }) + .field("ifru_mtu", unsafe { &self.ifru_mtu }) + .field("ifru_map", unsafe { &self.ifru_map }) + .field("ifru_slave", unsafe { &self.ifru_slave }) + .field("ifru_newname", unsafe { &self.ifru_newname }) + .field("ifru_data", unsafe { &self.ifru_data }) + .finish() + } + } + impl ::fmt::Debug for ifreq { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("ifreq") + .field("ifr_name", &self.ifr_name) + .field("ifr_ifru", &self.ifr_ifru) + .finish() + } + } } } -pub const ABDAY_1: ::nl_item = 0x20000; -pub const ABDAY_2: ::nl_item = 0x20001; -pub const ABDAY_3: ::nl_item = 0x20002; -pub const ABDAY_4: ::nl_item = 0x20003; -pub const ABDAY_5: ::nl_item = 0x20004; -pub const ABDAY_6: ::nl_item = 0x20005; -pub const ABDAY_7: ::nl_item = 0x20006; - -pub const DAY_1: ::nl_item = 0x20007; -pub const DAY_2: ::nl_item = 0x20008; -pub const DAY_3: ::nl_item = 0x20009; -pub const DAY_4: ::nl_item = 0x2000A; -pub const DAY_5: ::nl_item = 0x2000B; -pub const DAY_6: ::nl_item = 0x2000C; -pub const DAY_7: ::nl_item = 0x2000D; - -pub const ABMON_1: ::nl_item = 0x2000E; -pub const ABMON_2: ::nl_item = 0x2000F; -pub const ABMON_3: ::nl_item = 0x20010; -pub const ABMON_4: ::nl_item = 0x20011; -pub const ABMON_5: ::nl_item = 0x20012; -pub const ABMON_6: ::nl_item = 0x20013; -pub const ABMON_7: ::nl_item = 0x20014; -pub const ABMON_8: ::nl_item = 0x20015; -pub const ABMON_9: ::nl_item = 0x20016; -pub const ABMON_10: ::nl_item = 0x20017; -pub const ABMON_11: ::nl_item = 0x20018; -pub const ABMON_12: ::nl_item = 0x20019; - -pub const MON_1: ::nl_item = 0x2001A; -pub const MON_2: ::nl_item = 0x2001B; -pub const MON_3: ::nl_item = 0x2001C; -pub const MON_4: ::nl_item = 0x2001D; -pub const MON_5: ::nl_item = 0x2001E; -pub const MON_6: ::nl_item = 0x2001F; -pub const MON_7: ::nl_item = 0x20020; -pub const MON_8: ::nl_item = 0x20021; -pub const MON_9: ::nl_item = 0x20022; -pub const MON_10: ::nl_item = 0x20023; -pub const MON_11: ::nl_item = 0x20024; -pub const MON_12: ::nl_item = 0x20025; - -pub const AM_STR: ::nl_item = 0x20026; -pub const PM_STR: ::nl_item = 0x20027; - -pub const D_T_FMT: ::nl_item = 0x20028; -pub const D_FMT: ::nl_item = 0x20029; -pub const T_FMT: ::nl_item = 0x2002A; -pub const T_FMT_AMPM: ::nl_item = 0x2002B; - -pub const ERA: ::nl_item = 0x2002C; -pub const ERA_D_FMT: ::nl_item = 0x2002E; -pub const ALT_DIGITS: ::nl_item = 0x2002F; -pub const ERA_D_T_FMT: ::nl_item = 0x20030; -pub const ERA_T_FMT: ::nl_item = 0x20031; - -pub const CODESET: ::nl_item = 14; - -pub const CRNCYSTR: ::nl_item = 0x4000F; - -pub const RUSAGE_THREAD: ::c_int = 1; -pub const RUSAGE_CHILDREN: ::c_int = -1; - -pub const RADIXCHAR: ::nl_item = 0x10000; -pub const THOUSEP: ::nl_item = 0x10001; - -pub const YESEXPR: ::nl_item = 0x50000; -pub const NOEXPR: ::nl_item = 0x50001; -pub const YESSTR: ::nl_item = 0x50002; -pub const NOSTR: ::nl_item = 0x50003; +cfg_if! { + if #[cfg(any(target_env = "gnu", target_env = "musl"))] { + pub const ABDAY_1: ::nl_item = 0x20000; + pub const ABDAY_2: ::nl_item = 0x20001; + pub const ABDAY_3: ::nl_item = 0x20002; + pub const ABDAY_4: ::nl_item = 0x20003; + pub const ABDAY_5: ::nl_item = 0x20004; + pub const ABDAY_6: ::nl_item = 0x20005; + pub const ABDAY_7: ::nl_item = 0x20006; + + pub const DAY_1: ::nl_item = 0x20007; + pub const DAY_2: ::nl_item = 0x20008; + pub const DAY_3: ::nl_item = 0x20009; + pub const DAY_4: ::nl_item = 0x2000A; + pub const DAY_5: ::nl_item = 0x2000B; + pub const DAY_6: ::nl_item = 0x2000C; + pub const DAY_7: ::nl_item = 0x2000D; + + pub const ABMON_1: ::nl_item = 0x2000E; + pub const ABMON_2: ::nl_item = 0x2000F; + pub const ABMON_3: ::nl_item = 0x20010; + pub const ABMON_4: ::nl_item = 0x20011; + pub const ABMON_5: ::nl_item = 0x20012; + pub const ABMON_6: ::nl_item = 0x20013; + pub const ABMON_7: ::nl_item = 0x20014; + pub const ABMON_8: ::nl_item = 0x20015; + pub const ABMON_9: ::nl_item = 0x20016; + pub const ABMON_10: ::nl_item = 0x20017; + pub const ABMON_11: ::nl_item = 0x20018; + pub const ABMON_12: ::nl_item = 0x20019; + + pub const MON_1: ::nl_item = 0x2001A; + pub const MON_2: ::nl_item = 0x2001B; + pub const MON_3: ::nl_item = 0x2001C; + pub const MON_4: ::nl_item = 0x2001D; + pub const MON_5: ::nl_item = 0x2001E; + pub const MON_6: ::nl_item = 0x2001F; + pub const MON_7: ::nl_item = 0x20020; + pub const MON_8: ::nl_item = 0x20021; + pub const MON_9: ::nl_item = 0x20022; + pub const MON_10: ::nl_item = 0x20023; + pub const MON_11: ::nl_item = 0x20024; + pub const MON_12: ::nl_item = 0x20025; + + pub const AM_STR: ::nl_item = 0x20026; + pub const PM_STR: ::nl_item = 0x20027; + + pub const D_T_FMT: ::nl_item = 0x20028; + pub const D_FMT: ::nl_item = 0x20029; + pub const T_FMT: ::nl_item = 0x2002A; + pub const T_FMT_AMPM: ::nl_item = 0x2002B; + + pub const ERA: ::nl_item = 0x2002C; + pub const ERA_D_FMT: ::nl_item = 0x2002E; + pub const ALT_DIGITS: ::nl_item = 0x2002F; + pub const ERA_D_T_FMT: ::nl_item = 0x20030; + pub const ERA_T_FMT: ::nl_item = 0x20031; + + pub const CODESET: ::nl_item = 14; + pub const CRNCYSTR: ::nl_item = 0x4000F; + pub const RADIXCHAR: ::nl_item = 0x10000; + pub const THOUSEP: ::nl_item = 0x10001; + pub const YESEXPR: ::nl_item = 0x50000; + pub const NOEXPR: ::nl_item = 0x50001; + pub const YESSTR: ::nl_item = 0x50002; + pub const NOSTR: ::nl_item = 0x50003; + } +} -pub const FILENAME_MAX: ::c_uint = 4096; +pub const RUSAGE_CHILDREN: ::c_int = -1; pub const L_tmpnam: ::c_uint = 20; pub const _PC_LINK_MAX: ::c_int = 0; pub const _PC_MAX_CANON: ::c_int = 1; @@ -1066,6 +1382,207 @@ pub const _SC_THREAD_ROBUST_PRIO_PROTECT: ::c_int = 248; pub const RLIM_SAVED_MAX: ::rlim_t = RLIM_INFINITY; pub const RLIM_SAVED_CUR: ::rlim_t = RLIM_INFINITY; +// elf.h - Fields in the e_ident array. +pub const EI_NIDENT: usize = 16; + +pub const EI_MAG0: usize = 0; +pub const ELFMAG0: u8 = 0x7f; +pub const EI_MAG1: usize = 1; +pub const ELFMAG1: u8 = b'E'; +pub const EI_MAG2: usize = 2; +pub const ELFMAG2: u8 = b'L'; +pub const EI_MAG3: usize = 3; +pub const ELFMAG3: u8 = b'F'; +pub const SELFMAG: usize = 4; + +pub const EI_CLASS: usize = 4; +pub const ELFCLASSNONE: u8 = 0; +pub const ELFCLASS32: u8 = 1; +pub const ELFCLASS64: u8 = 2; +pub const ELFCLASSNUM: usize = 3; + +pub const EI_DATA: usize = 5; +pub const ELFDATANONE: u8 = 0; +pub const ELFDATA2LSB: u8 = 1; +pub const ELFDATA2MSB: u8 = 2; +pub const ELFDATANUM: usize = 3; + +pub const EI_VERSION: usize = 6; + +pub const EI_OSABI: usize = 7; +pub const ELFOSABI_NONE: u8 = 0; +pub const ELFOSABI_SYSV: u8 = 0; +pub const ELFOSABI_HPUX: u8 = 1; +pub const ELFOSABI_NETBSD: u8 = 2; +pub const ELFOSABI_GNU: u8 = 3; +pub const ELFOSABI_LINUX: u8 = ELFOSABI_GNU; +pub const ELFOSABI_SOLARIS: u8 = 6; +pub const ELFOSABI_AIX: u8 = 7; +pub const ELFOSABI_IRIX: u8 = 8; +pub const ELFOSABI_FREEBSD: u8 = 9; +pub const ELFOSABI_TRU64: u8 = 10; +pub const ELFOSABI_MODESTO: u8 = 11; +pub const ELFOSABI_OPENBSD: u8 = 12; +pub const ELFOSABI_ARM: u8 = 97; +pub const ELFOSABI_STANDALONE: u8 = 255; + +pub const EI_ABIVERSION: usize = 8; + +pub const EI_PAD: usize = 9; + +// elf.h - Legal values for e_type (object file type). +pub const ET_NONE: u16 = 0; +pub const ET_REL: u16 = 1; +pub const ET_EXEC: u16 = 2; +pub const ET_DYN: u16 = 3; +pub const ET_CORE: u16 = 4; +pub const ET_NUM: u16 = 5; +pub const ET_LOOS: u16 = 0xfe00; +pub const ET_HIOS: u16 = 0xfeff; +pub const ET_LOPROC: u16 = 0xff00; +pub const ET_HIPROC: u16 = 0xffff; + +// elf.h - Legal values for e_machine (architecture). +pub const EM_NONE: u16 = 0; +pub const EM_M32: u16 = 1; +pub const EM_SPARC: u16 = 2; +pub const EM_386: u16 = 3; +pub const EM_68K: u16 = 4; +pub const EM_88K: u16 = 5; +pub const EM_860: u16 = 7; +pub const EM_MIPS: u16 = 8; +pub const EM_S370: u16 = 9; +pub const EM_MIPS_RS3_LE: u16 = 10; +pub const EM_PARISC: u16 = 15; +pub const EM_VPP500: u16 = 17; +pub const EM_SPARC32PLUS: u16 = 18; +pub const EM_960: u16 = 19; +pub const EM_PPC: u16 = 20; +pub const EM_PPC64: u16 = 21; +pub const EM_S390: u16 = 22; +pub const EM_V800: u16 = 36; +pub const EM_FR20: u16 = 37; +pub const EM_RH32: u16 = 38; +pub const EM_RCE: u16 = 39; +pub const EM_ARM: u16 = 40; +pub const EM_FAKE_ALPHA: u16 = 41; +pub const EM_SH: u16 = 42; +pub const EM_SPARCV9: u16 = 43; +pub const EM_TRICORE: u16 = 44; +pub const EM_ARC: u16 = 45; +pub const EM_H8_300: u16 = 46; +pub const EM_H8_300H: u16 = 47; +pub const EM_H8S: u16 = 48; +pub const EM_H8_500: u16 = 49; +pub const EM_IA_64: u16 = 50; +pub const EM_MIPS_X: u16 = 51; +pub const EM_COLDFIRE: u16 = 52; +pub const EM_68HC12: u16 = 53; +pub const EM_MMA: u16 = 54; +pub const EM_PCP: u16 = 55; +pub const EM_NCPU: u16 = 56; +pub const EM_NDR1: u16 = 57; +pub const EM_STARCORE: u16 = 58; +pub const EM_ME16: u16 = 59; +pub const EM_ST100: u16 = 60; +pub const EM_TINYJ: u16 = 61; +pub const EM_X86_64: u16 = 62; +pub const EM_PDSP: u16 = 63; +pub const EM_FX66: u16 = 66; +pub const EM_ST9PLUS: u16 = 67; +pub const EM_ST7: u16 = 68; +pub const EM_68HC16: u16 = 69; +pub const EM_68HC11: u16 = 70; +pub const EM_68HC08: u16 = 71; +pub const EM_68HC05: u16 = 72; +pub const EM_SVX: u16 = 73; +pub const EM_ST19: u16 = 74; +pub const EM_VAX: u16 = 75; +pub const EM_CRIS: u16 = 76; +pub const EM_JAVELIN: u16 = 77; +pub const EM_FIREPATH: u16 = 78; +pub const EM_ZSP: u16 = 79; +pub const EM_MMIX: u16 = 80; +pub const EM_HUANY: u16 = 81; +pub const EM_PRISM: u16 = 82; +pub const EM_AVR: u16 = 83; +pub const EM_FR30: u16 = 84; +pub const EM_D10V: u16 = 85; +pub const EM_D30V: u16 = 86; +pub const EM_V850: u16 = 87; +pub const EM_M32R: u16 = 88; +pub const EM_MN10300: u16 = 89; +pub const EM_MN10200: u16 = 90; +pub const EM_PJ: u16 = 91; +pub const EM_OPENRISC: u16 = 92; +pub const EM_ARC_A5: u16 = 93; +pub const EM_XTENSA: u16 = 94; +pub const EM_AARCH64: u16 = 183; +pub const EM_TILEPRO: u16 = 188; +pub const EM_TILEGX: u16 = 191; +pub const EM_ALPHA: u16 = 0x9026; + +// elf.h - Legal values for e_version (version). +pub const EV_NONE: u32 = 0; +pub const EV_CURRENT: u32 = 1; +pub const EV_NUM: u32 = 2; + +// elf.h - Legal values for p_type (segment type). +pub const PT_NULL: u32 = 0; +pub const PT_LOAD: u32 = 1; +pub const PT_DYNAMIC: u32 = 2; +pub const PT_INTERP: u32 = 3; +pub const PT_NOTE: u32 = 4; +pub const PT_SHLIB: u32 = 5; +pub const PT_PHDR: u32 = 6; +pub const PT_TLS: u32 = 7; +pub const PT_NUM: u32 = 8; +pub const PT_LOOS: u32 = 0x60000000; +pub const PT_GNU_EH_FRAME: u32 = 0x6474e550; +pub const PT_GNU_STACK: u32 = 0x6474e551; +pub const PT_GNU_RELRO: u32 = 0x6474e552; +pub const PT_LOSUNW: u32 = 0x6ffffffa; +pub const PT_SUNWBSS: u32 = 0x6ffffffa; +pub const PT_SUNWSTACK: u32 = 0x6ffffffb; +pub const PT_HISUNW: u32 = 0x6fffffff; +pub const PT_HIOS: u32 = 0x6fffffff; +pub const PT_LOPROC: u32 = 0x70000000; +pub const PT_HIPROC: u32 = 0x7fffffff; + +// Legal values for p_flags (segment flags). +pub const PF_X: u32 = 1 << 0; +pub const PF_W: u32 = 1 << 1; +pub const PF_R: u32 = 1 << 2; +pub const PF_MASKOS: u32 = 0x0ff00000; +pub const PF_MASKPROC: u32 = 0xf0000000; + +// elf.h - Legal values for a_type (entry type). +pub const AT_NULL: ::c_ulong = 0; +pub const AT_IGNORE: ::c_ulong = 1; +pub const AT_EXECFD: ::c_ulong = 2; +pub const AT_PHDR: ::c_ulong = 3; +pub const AT_PHENT: ::c_ulong = 4; +pub const AT_PHNUM: ::c_ulong = 5; +pub const AT_PAGESZ: ::c_ulong = 6; +pub const AT_BASE: ::c_ulong = 7; +pub const AT_FLAGS: ::c_ulong = 8; +pub const AT_ENTRY: ::c_ulong = 9; +pub const AT_NOTELF: ::c_ulong = 10; +pub const AT_UID: ::c_ulong = 11; +pub const AT_EUID: ::c_ulong = 12; +pub const AT_GID: ::c_ulong = 13; +pub const AT_EGID: ::c_ulong = 14; +pub const AT_PLATFORM: ::c_ulong = 15; +pub const AT_HWCAP: ::c_ulong = 16; +pub const AT_CLKTCK: ::c_ulong = 17; + +pub const AT_SECURE: ::c_ulong = 23; +pub const AT_BASE_PLATFORM: ::c_ulong = 24; +pub const AT_RANDOM: ::c_ulong = 25; +pub const AT_HWCAP2: ::c_ulong = 26; + +pub const AT_EXECFN: ::c_ulong = 31; + pub const GLOB_ERR: ::c_int = 1 << 0; pub const GLOB_MARK: ::c_int = 1 << 1; pub const GLOB_NOSORT: ::c_int = 1 << 2; @@ -1082,6 +1599,8 @@ pub const POSIX_MADV_NORMAL: ::c_int = 0; pub const POSIX_MADV_RANDOM: ::c_int = 1; pub const POSIX_MADV_SEQUENTIAL: ::c_int = 2; pub const POSIX_MADV_WILLNEED: ::c_int = 3; +pub const POSIX_SPAWN_USEVFORK: ::c_int = 64; +pub const POSIX_SPAWN_SETSID: ::c_int = 128; pub const S_IEXEC: mode_t = 64; pub const S_IWRITE: mode_t = 128; @@ -1159,6 +1678,23 @@ pub const IFLA_PHYS_SWITCH_ID: ::c_ushort = 36; pub const IFLA_LINK_NETNSID: ::c_ushort = 37; pub const IFLA_PHYS_PORT_NAME: ::c_ushort = 38; pub const IFLA_PROTO_DOWN: ::c_ushort = 39; +pub const IFLA_GSO_MAX_SEGS: ::c_ushort = 40; +pub const IFLA_GSO_MAX_SIZE: ::c_ushort = 41; +pub const IFLA_PAD: ::c_ushort = 42; +pub const IFLA_XDP: ::c_ushort = 43; +pub const IFLA_EVENT: ::c_ushort = 44; +pub const IFLA_NEW_NETNSID: ::c_ushort = 45; +pub const IFLA_IF_NETNSID: ::c_ushort = 46; +pub const IFLA_TARGET_NETNSID: ::c_ushort = IFLA_IF_NETNSID; +pub const IFLA_CARRIER_UP_COUNT: ::c_ushort = 47; +pub const IFLA_CARRIER_DOWN_COUNT: ::c_ushort = 48; +pub const IFLA_NEW_IFINDEX: ::c_ushort = 49; +pub const IFLA_MIN_MTU: ::c_ushort = 50; +pub const IFLA_MAX_MTU: ::c_ushort = 51; +pub const IFLA_PROP_LIST: ::c_ushort = 52; +pub const IFLA_ALT_IFNAME: ::c_ushort = 53; +pub const IFLA_PERM_ADDRESS: ::c_ushort = 54; +pub const IFLA_PROTO_DOWN_REASON: ::c_ushort = 55; pub const IFLA_INFO_UNSPEC: ::c_ushort = 0; pub const IFLA_INFO_KIND: ::c_ushort = 1; @@ -1188,6 +1724,10 @@ pub const IFF_DETACH_QUEUE: ::c_int = 0x0400; pub const IFF_PERSIST: ::c_int = 0x0800; pub const IFF_NOFILTER: ::c_int = 0x1000; +// Since Linux 3.1 +pub const SEEK_DATA: ::c_int = 3; +pub const SEEK_HOLE: ::c_int = 4; + pub const ST_RDONLY: ::c_ulong = 1; pub const ST_NOSUID: ::c_ulong = 2; pub const ST_NODEV: ::c_ulong = 4; @@ -1207,7 +1747,15 @@ pub const RTLD_NOW: ::c_int = 0x2; pub const AT_EACCESS: ::c_int = 0x200; -pub const TCP_MD5SIG: ::c_int = 14; +// linux/mempolicy.h +pub const MPOL_DEFAULT: ::c_int = 0; +pub const MPOL_PREFERRED: ::c_int = 1; +pub const MPOL_BIND: ::c_int = 2; +pub const MPOL_INTERLEAVE: ::c_int = 3; +pub const MPOL_LOCAL: ::c_int = 4; +pub const MPOL_F_NUMA_BALANCING: ::c_int = 1 << 13; +pub const MPOL_F_RELATIVE_NODES: ::c_int = 1 << 14; +pub const MPOL_F_STATIC_NODES: ::c_int = 1 << 15; align_const! { pub const PTHREAD_MUTEX_INITIALIZER: pthread_mutex_t = pthread_mutex_t { @@ -1224,13 +1772,18 @@ pub const PTHREAD_MUTEX_NORMAL: ::c_int = 0; pub const PTHREAD_MUTEX_RECURSIVE: ::c_int = 1; pub const PTHREAD_MUTEX_ERRORCHECK: ::c_int = 2; pub const PTHREAD_MUTEX_DEFAULT: ::c_int = PTHREAD_MUTEX_NORMAL; +pub const PTHREAD_MUTEX_STALLED: ::c_int = 0; +pub const PTHREAD_MUTEX_ROBUST: ::c_int = 1; +pub const PTHREAD_PRIO_NONE: ::c_int = 0; +pub const PTHREAD_PRIO_INHERIT: ::c_int = 1; +pub const PTHREAD_PRIO_PROTECT: ::c_int = 2; pub const PTHREAD_PROCESS_PRIVATE: ::c_int = 0; pub const PTHREAD_PROCESS_SHARED: ::c_int = 1; pub const __SIZEOF_PTHREAD_COND_T: usize = 48; -pub const RENAME_NOREPLACE: ::c_int = 1; -pub const RENAME_EXCHANGE: ::c_int = 2; -pub const RENAME_WHITEOUT: ::c_int = 4; +pub const RENAME_NOREPLACE: ::c_uint = 1; +pub const RENAME_EXCHANGE: ::c_uint = 2; +pub const RENAME_WHITEOUT: ::c_uint = 4; pub const SCHED_OTHER: ::c_int = 0; pub const SCHED_FIFO: ::c_int = 1; @@ -1240,86 +1793,21 @@ pub const SCHED_IDLE: ::c_int = 5; pub const SCHED_RESET_ON_FORK: ::c_int = 0x40000000; +pub const CLONE_PIDFD: ::c_int = 0x1000; + // netinet/in.h // NOTE: These are in addition to the constants defined in src/unix/mod.rs -// IPPROTO_IP defined in src/unix/mod.rs -/// Hop-by-hop option header -pub const IPPROTO_HOPOPTS: ::c_int = 0; -// IPPROTO_ICMP defined in src/unix/mod.rs -/// group mgmt protocol -pub const IPPROTO_IGMP: ::c_int = 2; -/// for compatibility -pub const IPPROTO_IPIP: ::c_int = 4; -// IPPROTO_TCP defined in src/unix/mod.rs -/// exterior gateway protocol -pub const IPPROTO_EGP: ::c_int = 8; -/// pup -pub const IPPROTO_PUP: ::c_int = 12; -// IPPROTO_UDP defined in src/unix/mod.rs -/// xns idp -pub const IPPROTO_IDP: ::c_int = 22; -/// tp-4 w/ class negotiation -pub const IPPROTO_TP: ::c_int = 29; -/// DCCP -pub const IPPROTO_DCCP: ::c_int = 33; -// IPPROTO_IPV6 defined in src/unix/mod.rs -/// IP6 routing header -pub const IPPROTO_ROUTING: ::c_int = 43; -/// IP6 fragmentation header -pub const IPPROTO_FRAGMENT: ::c_int = 44; -/// resource reservation -pub const IPPROTO_RSVP: ::c_int = 46; -/// General Routing Encap. -pub const IPPROTO_GRE: ::c_int = 47; -/// IP6 Encap Sec. Payload -pub const IPPROTO_ESP: ::c_int = 50; -/// IP6 Auth Header -pub const IPPROTO_AH: ::c_int = 51; -// IPPROTO_ICMPV6 defined in src/unix/mod.rs -/// IP6 no next header -pub const IPPROTO_NONE: ::c_int = 59; -/// IP6 destination option -pub const IPPROTO_DSTOPTS: ::c_int = 60; -pub const IPPROTO_MTP: ::c_int = 92; -pub const IPPROTO_BEETPH: ::c_int = 94; -/// encapsulation header -pub const IPPROTO_ENCAP: ::c_int = 98; -/// Protocol indep. multicast -pub const IPPROTO_PIM: ::c_int = 103; -/// IP Payload Comp. Protocol -pub const IPPROTO_COMP: ::c_int = 108; -/// SCTP -pub const IPPROTO_SCTP: ::c_int = 132; -pub const IPPROTO_MH: ::c_int = 135; -pub const IPPROTO_UDPLITE: ::c_int = 136; -pub const IPPROTO_MPLS: ::c_int = 137; -/// raw IP packet -pub const IPPROTO_RAW: ::c_int = 255; +/// Multipath TCP +pub const IPPROTO_MPTCP: ::c_int = 262; +#[deprecated( + since = "0.2.80", + note = "This value was increased in the newer kernel \ + and we'll change this following upstream in the future release. \ + See #1896 for more info." +)] pub const IPPROTO_MAX: ::c_int = 256; -pub const IP_MSFILTER: ::c_int = 41; -pub const MCAST_JOIN_GROUP: ::c_int = 42; -pub const MCAST_BLOCK_SOURCE: ::c_int = 43; -pub const MCAST_UNBLOCK_SOURCE: ::c_int = 44; -pub const MCAST_LEAVE_GROUP: ::c_int = 45; -pub const MCAST_JOIN_SOURCE_GROUP: ::c_int = 46; -pub const MCAST_LEAVE_SOURCE_GROUP: ::c_int = 47; -pub const MCAST_MSFILTER: ::c_int = 48; -pub const IP_MULTICAST_ALL: ::c_int = 49; -pub const IP_UNICAST_IF: ::c_int = 50; - -pub const AF_IB: ::c_int = 27; -pub const AF_MPLS: ::c_int = 28; -pub const AF_NFC: ::c_int = 39; -pub const AF_VSOCK: ::c_int = 40; -pub const AF_XDP: ::c_int = 44; -pub const PF_IB: ::c_int = AF_IB; -pub const PF_MPLS: ::c_int = AF_MPLS; -pub const PF_NFC: ::c_int = AF_NFC; -pub const PF_VSOCK: ::c_int = AF_VSOCK; -pub const PF_XDP: ::c_int = AF_XDP; - // System V IPC pub const IPC_PRIVATE: ::key_t = 0; @@ -1336,7 +1824,7 @@ pub const MSG_INFO: ::c_int = 12; pub const MSG_NOERROR: ::c_int = 0o10000; pub const MSG_EXCEPT: ::c_int = 0o20000; -pub const MSG_COPY: ::c_int = 0o40000; +pub const MSG_ZEROCOPY: ::c_int = 0x4000000; pub const SHM_R: ::c_int = 0o400; pub const SHM_W: ::c_int = 0o200; @@ -1344,18 +1832,14 @@ pub const SHM_W: ::c_int = 0o200; pub const SHM_RDONLY: ::c_int = 0o10000; pub const SHM_RND: ::c_int = 0o20000; pub const SHM_REMAP: ::c_int = 0o40000; -pub const SHM_EXEC: ::c_int = 0o100000; pub const SHM_LOCK: ::c_int = 11; pub const SHM_UNLOCK: ::c_int = 12; pub const SHM_HUGETLB: ::c_int = 0o4000; +#[cfg(not(all(target_env = "uclibc", target_arch = "mips")))] pub const SHM_NORESERVE: ::c_int = 0o10000; -pub const EPOLLRDHUP: ::c_int = 0x2000; -pub const EPOLLEXCLUSIVE: ::c_int = 0x10000000; -pub const EPOLLONESHOT: ::c_int = 0x40000000; - pub const QFMT_VFS_OLD: ::c_int = 1; pub const QFMT_VFS_V0: ::c_int = 2; pub const QFMT_VFS_V1: ::c_int = 4; @@ -1405,17 +1889,31 @@ pub const SYNC_FILE_RANGE_WAIT_BEFORE: ::c_uint = 1; pub const SYNC_FILE_RANGE_WRITE: ::c_uint = 2; pub const SYNC_FILE_RANGE_WAIT_AFTER: ::c_uint = 4; -pub const AIO_CANCELED: ::c_int = 0; -pub const AIO_NOTCANCELED: ::c_int = 1; -pub const AIO_ALLDONE: ::c_int = 2; -pub const LIO_READ: ::c_int = 0; -pub const LIO_WRITE: ::c_int = 1; -pub const LIO_NOP: ::c_int = 2; -pub const LIO_WAIT: ::c_int = 0; -pub const LIO_NOWAIT: ::c_int = 1; +cfg_if! { + if #[cfg(not(target_env = "uclibc"))] { + pub const AIO_CANCELED: ::c_int = 0; + pub const AIO_NOTCANCELED: ::c_int = 1; + pub const AIO_ALLDONE: ::c_int = 2; + pub const LIO_READ: ::c_int = 0; + pub const LIO_WRITE: ::c_int = 1; + pub const LIO_NOP: ::c_int = 2; + pub const LIO_WAIT: ::c_int = 0; + pub const LIO_NOWAIT: ::c_int = 1; + pub const RUSAGE_THREAD: ::c_int = 1; + pub const MSG_COPY: ::c_int = 0o40000; + pub const SHM_EXEC: ::c_int = 0o100000; + pub const IPV6_MULTICAST_ALL: ::c_int = 29; + pub const IPV6_ROUTER_ALERT_ISOLATE: ::c_int = 30; + pub const PACKET_MR_UNICAST: ::c_int = 3; + pub const PTRACE_EVENT_STOP: ::c_int = 128; + pub const UDP_SEGMENT: ::c_int = 103; + pub const UDP_GRO: ::c_int = 104; + } +} pub const MREMAP_MAYMOVE: ::c_int = 1; pub const MREMAP_FIXED: ::c_int = 2; +pub const MREMAP_DONTUNMAP: ::c_int = 4; pub const PR_SET_PDEATHSIG: ::c_int = 1; pub const PR_GET_PDEATHSIG: ::c_int = 2; @@ -1537,13 +2035,34 @@ pub const PR_CAP_AMBIENT_RAISE: ::c_int = 2; pub const PR_CAP_AMBIENT_LOWER: ::c_int = 3; pub const PR_CAP_AMBIENT_CLEAR_ALL: ::c_int = 4; +pub const PR_SET_VMA: ::c_int = 0x53564d41; +pub const PR_SET_VMA_ANON_NAME: ::c_int = 0; + pub const GRND_NONBLOCK: ::c_uint = 0x0001; pub const GRND_RANDOM: ::c_uint = 0x0002; +pub const GRND_INSECURE: ::c_uint = 0x0004; pub const SECCOMP_MODE_DISABLED: ::c_uint = 0; pub const SECCOMP_MODE_STRICT: ::c_uint = 1; pub const SECCOMP_MODE_FILTER: ::c_uint = 2; +pub const SECCOMP_FILTER_FLAG_TSYNC: ::c_ulong = 1; +pub const SECCOMP_FILTER_FLAG_LOG: ::c_ulong = 2; +pub const SECCOMP_FILTER_FLAG_SPEC_ALLOW: ::c_ulong = 4; + +pub const SECCOMP_RET_KILL_PROCESS: ::c_uint = 0x80000000; +pub const SECCOMP_RET_KILL_THREAD: ::c_uint = 0x00000000; +pub const SECCOMP_RET_KILL: ::c_uint = SECCOMP_RET_KILL_THREAD; +pub const SECCOMP_RET_TRAP: ::c_uint = 0x00030000; +pub const SECCOMP_RET_ERRNO: ::c_uint = 0x00050000; +pub const SECCOMP_RET_TRACE: ::c_uint = 0x7ff00000; +pub const SECCOMP_RET_LOG: ::c_uint = 0x7ffc0000; +pub const SECCOMP_RET_ALLOW: ::c_uint = 0x7fff0000; + +pub const SECCOMP_RET_ACTION_FULL: ::c_uint = 0xffff0000; +pub const SECCOMP_RET_ACTION: ::c_uint = 0x7fff0000; +pub const SECCOMP_RET_DATA: ::c_uint = 0x0000ffff; + pub const ITIMER_REAL: ::c_int = 0; pub const ITIMER_VIRTUAL: ::c_int = 1; pub const ITIMER_PROF: ::c_int = 2; @@ -1551,9 +2070,7 @@ pub const ITIMER_PROF: ::c_int = 2; pub const TFD_CLOEXEC: ::c_int = O_CLOEXEC; pub const TFD_NONBLOCK: ::c_int = O_NONBLOCK; pub const TFD_TIMER_ABSTIME: ::c_int = 1; - -pub const XATTR_CREATE: ::c_int = 0x1; -pub const XATTR_REPLACE: ::c_int = 0x2; +pub const TFD_TIMER_CANCEL_ON_SET: ::c_int = 2; pub const _POSIX_VDISABLE: ::cc_t = 0; @@ -1571,38 +2088,127 @@ pub const FALLOC_FL_UNSHARE_RANGE: ::c_int = 0x40; pub const ENOATTR: ::c_int = ::ENODATA; pub const SO_ORIGINAL_DST: ::c_int = 80; -pub const IP_ORIGDSTADDR: ::c_int = 20; -pub const IP_RECVORIGDSTADDR: ::c_int = IP_ORIGDSTADDR; -pub const IPV6_ORIGDSTADDR: ::c_int = 74; -pub const IPV6_RECVORIGDSTADDR: ::c_int = IPV6_ORIGDSTADDR; + +pub const IP_RECVFRAGSIZE: ::c_int = 25; + +pub const IPV6_FLOWINFO: ::c_int = 11; pub const IPV6_FLOWLABEL_MGR: ::c_int = 32; pub const IPV6_FLOWINFO_SEND: ::c_int = 33; +pub const IPV6_RECVFRAGSIZE: ::c_int = 77; +pub const IPV6_FREEBIND: ::c_int = 78; pub const IPV6_FLOWINFO_FLOWLABEL: ::c_int = 0x000fffff; pub const IPV6_FLOWINFO_PRIORITY: ::c_int = 0x0ff00000; +pub const IPV6_RTHDR_LOOSE: ::c_int = 0; +pub const IPV6_RTHDR_STRICT: ::c_int = 1; + +// SO_MEMINFO offsets +pub const SK_MEMINFO_RMEM_ALLOC: ::c_int = 0; +pub const SK_MEMINFO_RCVBUF: ::c_int = 1; +pub const SK_MEMINFO_WMEM_ALLOC: ::c_int = 2; +pub const SK_MEMINFO_SNDBUF: ::c_int = 3; +pub const SK_MEMINFO_FWD_ALLOC: ::c_int = 4; +pub const SK_MEMINFO_WMEM_QUEUED: ::c_int = 5; +pub const SK_MEMINFO_OPTMEM: ::c_int = 6; +pub const SK_MEMINFO_BACKLOG: ::c_int = 7; +pub const SK_MEMINFO_DROPS: ::c_int = 8; + pub const IUTF8: ::tcflag_t = 0x00004000; +#[cfg(not(all(target_env = "uclibc", target_arch = "mips")))] pub const CMSPAR: ::tcflag_t = 0o10000000000; pub const MFD_CLOEXEC: ::c_uint = 0x0001; pub const MFD_ALLOW_SEALING: ::c_uint = 0x0002; pub const MFD_HUGETLB: ::c_uint = 0x0004; - -// these are used in the p_type field of Elf32_Phdr and Elf64_Phdr, which has -// the type Elf32Word and Elf64Word respectively. Luckily, both of those are u32 -// so we can use that type here to avoid having to cast. -pub const PT_NULL: u32 = 0; -pub const PT_LOAD: u32 = 1; -pub const PT_DYNAMIC: u32 = 2; -pub const PT_INTERP: u32 = 3; -pub const PT_NOTE: u32 = 4; -pub const PT_SHLIB: u32 = 5; -pub const PT_PHDR: u32 = 6; -pub const PT_TLS: u32 = 7; -pub const PT_NUM: u32 = 8; -pub const PT_LOOS: u32 = 0x60000000; -pub const PT_GNU_EH_FRAME: u32 = 0x6474e550; -pub const PT_GNU_STACK: u32 = 0x6474e551; -pub const PT_GNU_RELRO: u32 = 0x6474e552; +pub const MFD_HUGE_64KB: ::c_uint = 0x40000000; +pub const MFD_HUGE_512KB: ::c_uint = 0x4c000000; +pub const MFD_HUGE_1MB: ::c_uint = 0x50000000; +pub const MFD_HUGE_2MB: ::c_uint = 0x54000000; +pub const MFD_HUGE_8MB: ::c_uint = 0x5c000000; +pub const MFD_HUGE_16MB: ::c_uint = 0x60000000; +pub const MFD_HUGE_32MB: ::c_uint = 0x64000000; +pub const MFD_HUGE_256MB: ::c_uint = 0x70000000; +pub const MFD_HUGE_512MB: ::c_uint = 0x74000000; +pub const MFD_HUGE_1GB: ::c_uint = 0x78000000; +pub const MFD_HUGE_2GB: ::c_uint = 0x7c000000; +pub const MFD_HUGE_16GB: ::c_uint = 0x88000000; +pub const MFD_HUGE_MASK: ::c_uint = 63; +pub const MFD_HUGE_SHIFT: ::c_uint = 26; + +// linux/close_range.h +pub const CLOSE_RANGE_UNSHARE: ::c_uint = 1 << 1; +pub const CLOSE_RANGE_CLOEXEC: ::c_uint = 1 << 2; + +// linux/filter.h +pub const SKF_AD_OFF: ::c_int = -0x1000; +pub const SKF_AD_PROTOCOL: ::c_int = 0; +pub const SKF_AD_PKTTYPE: ::c_int = 4; +pub const SKF_AD_IFINDEX: ::c_int = 8; +pub const SKF_AD_NLATTR: ::c_int = 12; +pub const SKF_AD_NLATTR_NEST: ::c_int = 16; +pub const SKF_AD_MARK: ::c_int = 20; +pub const SKF_AD_QUEUE: ::c_int = 24; +pub const SKF_AD_HATYPE: ::c_int = 28; +pub const SKF_AD_RXHASH: ::c_int = 32; +pub const SKF_AD_CPU: ::c_int = 36; +pub const SKF_AD_ALU_XOR_X: ::c_int = 40; +pub const SKF_AD_VLAN_TAG: ::c_int = 44; +pub const SKF_AD_VLAN_TAG_PRESENT: ::c_int = 48; +pub const SKF_AD_PAY_OFFSET: ::c_int = 52; +pub const SKF_AD_RANDOM: ::c_int = 56; +pub const SKF_AD_VLAN_TPID: ::c_int = 60; +pub const SKF_AD_MAX: ::c_int = 64; +pub const SKF_NET_OFF: ::c_int = -0x100000; +pub const SKF_LL_OFF: ::c_int = -0x200000; +pub const BPF_NET_OFF: ::c_int = SKF_NET_OFF; +pub const BPF_LL_OFF: ::c_int = SKF_LL_OFF; +pub const BPF_MEMWORDS: ::c_int = 16; +pub const BPF_MAXINSNS: ::c_int = 4096; + +// linux/bpf_common.h +pub const BPF_LD: ::__u32 = 0x00; +pub const BPF_LDX: ::__u32 = 0x01; +pub const BPF_ST: ::__u32 = 0x02; +pub const BPF_STX: ::__u32 = 0x03; +pub const BPF_ALU: ::__u32 = 0x04; +pub const BPF_JMP: ::__u32 = 0x05; +pub const BPF_RET: ::__u32 = 0x06; +pub const BPF_MISC: ::__u32 = 0x07; +pub const BPF_W: ::__u32 = 0x00; +pub const BPF_H: ::__u32 = 0x08; +pub const BPF_B: ::__u32 = 0x10; +pub const BPF_IMM: ::__u32 = 0x00; +pub const BPF_ABS: ::__u32 = 0x20; +pub const BPF_IND: ::__u32 = 0x40; +pub const BPF_MEM: ::__u32 = 0x60; +pub const BPF_LEN: ::__u32 = 0x80; +pub const BPF_MSH: ::__u32 = 0xa0; +pub const BPF_ADD: ::__u32 = 0x00; +pub const BPF_SUB: ::__u32 = 0x10; +pub const BPF_MUL: ::__u32 = 0x20; +pub const BPF_DIV: ::__u32 = 0x30; +pub const BPF_OR: ::__u32 = 0x40; +pub const BPF_AND: ::__u32 = 0x50; +pub const BPF_LSH: ::__u32 = 0x60; +pub const BPF_RSH: ::__u32 = 0x70; +pub const BPF_NEG: ::__u32 = 0x80; +pub const BPF_MOD: ::__u32 = 0x90; +pub const BPF_XOR: ::__u32 = 0xa0; +pub const BPF_JA: ::__u32 = 0x00; +pub const BPF_JEQ: ::__u32 = 0x10; +pub const BPF_JGT: ::__u32 = 0x20; +pub const BPF_JGE: ::__u32 = 0x30; +pub const BPF_JSET: ::__u32 = 0x40; +pub const BPF_K: ::__u32 = 0x00; +pub const BPF_X: ::__u32 = 0x08; + +// linux/openat2.h +pub const RESOLVE_NO_XDEV: ::__u64 = 0x01; +pub const RESOLVE_NO_MAGICLINKS: ::__u64 = 0x02; +pub const RESOLVE_NO_SYMLINKS: ::__u64 = 0x04; +pub const RESOLVE_BENEATH: ::__u64 = 0x08; +pub const RESOLVE_IN_ROOT: ::__u64 = 0x10; +pub const RESOLVE_CACHED: ::__u64 = 0x20; // linux/if_ether.h pub const ETH_ALEN: ::c_int = 6; @@ -1719,6 +2325,7 @@ pub const NFNLGRP_CONNTRACK_EXP_UPDATE: ::c_int = 5; pub const NFNLGRP_CONNTRACK_EXP_DESTROY: ::c_int = 6; pub const NFNLGRP_NFTABLES: ::c_int = 7; pub const NFNLGRP_ACCT_QUOTA: ::c_int = 8; +pub const NFNLGRP_NFTRACE: ::c_int = 9; pub const NFNETLINK_V0: ::c_int = 0; @@ -1734,15 +2341,23 @@ pub const NFNL_SUBSYS_CTNETLINK_TIMEOUT: ::c_int = 8; pub const NFNL_SUBSYS_CTHELPER: ::c_int = 9; pub const NFNL_SUBSYS_NFTABLES: ::c_int = 10; pub const NFNL_SUBSYS_NFT_COMPAT: ::c_int = 11; -pub const NFNL_SUBSYS_COUNT: ::c_int = 12; +pub const NFNL_SUBSYS_HOOK: ::c_int = 12; +pub const NFNL_SUBSYS_COUNT: ::c_int = 13; pub const NFNL_MSG_BATCH_BEGIN: ::c_int = NLMSG_MIN_TYPE; pub const NFNL_MSG_BATCH_END: ::c_int = NLMSG_MIN_TYPE + 1; +pub const NFNL_BATCH_UNSPEC: ::c_int = 0; +pub const NFNL_BATCH_GENID: ::c_int = 1; + // linux/netfilter/nfnetlink_log.h pub const NFULNL_MSG_PACKET: ::c_int = 0; pub const NFULNL_MSG_CONFIG: ::c_int = 1; +pub const NFULA_VLAN_UNSPEC: ::c_int = 0; +pub const NFULA_VLAN_PROTO: ::c_int = 1; +pub const NFULA_VLAN_TCI: ::c_int = 2; + pub const NFULA_UNSPEC: ::c_int = 0; pub const NFULA_PACKET_HDR: ::c_int = 1; pub const NFULA_MARK: ::c_int = 2; @@ -1763,6 +2378,8 @@ pub const NFULA_HWHEADER: ::c_int = 16; pub const NFULA_HWLEN: ::c_int = 17; pub const NFULA_CT: ::c_int = 18; pub const NFULA_CT_INFO: ::c_int = 19; +pub const NFULA_VLAN: ::c_int = 20; +pub const NFULA_L2HDR: ::c_int = 21; pub const NFULNL_CFG_CMD_NONE: ::c_int = 0; pub const NFULNL_CFG_CMD_BIND: ::c_int = 1; @@ -1786,7 +2403,7 @@ pub const NFULNL_CFG_F_SEQ: ::c_int = 0x0001; pub const NFULNL_CFG_F_SEQ_GLOBAL: ::c_int = 0x0002; pub const NFULNL_CFG_F_CONNTRACK: ::c_int = 0x0004; -// linux/netfilter/nfnetlink_log.h +// linux/netfilter/nfnetlink_queue.h pub const NFQNL_MSG_PACKET: ::c_int = 0; pub const NFQNL_MSG_VERDICT: ::c_int = 1; pub const NFQNL_MSG_CONFIG: ::c_int = 2; @@ -1811,18 +2428,13 @@ pub const NFQA_EXP: ::c_int = 15; pub const NFQA_UID: ::c_int = 16; pub const NFQA_GID: ::c_int = 17; pub const NFQA_SECCTX: ::c_int = 18; -/* - FIXME: These are not yet available in musl sanitized kernel headers and - make the tests fail. Enable them once musl has them. - - See https://github.com/rust-lang/libc/pull/1628 for more details. pub const NFQA_VLAN: ::c_int = 19; pub const NFQA_L2HDR: ::c_int = 20; +pub const NFQA_PRIORITY: ::c_int = 21; pub const NFQA_VLAN_UNSPEC: ::c_int = 0; pub const NFQA_VLAN_PROTO: ::c_int = 1; pub const NFQA_VLAN_TCI: ::c_int = 2; -*/ pub const NFQNL_CFG_CMD_NONE: ::c_int = 0; pub const NFQNL_CFG_CMD_BIND: ::c_int = 1; @@ -1852,6 +2464,8 @@ pub const NFQA_SKB_CSUMNOTREADY: ::c_int = 0x0001; pub const NFQA_SKB_GSO: ::c_int = 0x0002; pub const NFQA_SKB_CSUM_NOTVERIFIED: ::c_int = 0x0004; +// linux/genetlink.h + pub const GENL_NAMSIZ: ::c_int = 16; pub const GENL_MIN_ID: ::c_int = NLMSG_MIN_TYPE; @@ -1899,7 +2513,6 @@ pub const PACKET_DROP_MEMBERSHIP: ::c_int = 2; pub const PACKET_MR_MULTICAST: ::c_int = 0; pub const PACKET_MR_PROMISC: ::c_int = 1; pub const PACKET_MR_ALLMULTI: ::c_int = 2; -pub const PACKET_MR_UNICAST: ::c_int = 3; // linux/netfilter.h pub const NF_DROP: ::c_int = 0; @@ -1933,6 +2546,11 @@ pub const NFPROTO_BRIDGE: ::c_int = 7; pub const NFPROTO_IPV6: ::c_int = 10; pub const NFPROTO_DECNET: ::c_int = 12; pub const NFPROTO_NUMPROTO: ::c_int = 13; +pub const NFPROTO_INET: ::c_int = 1; +pub const NFPROTO_NETDEV: ::c_int = 5; + +pub const NF_NETDEV_INGRESS: ::c_int = 0; +pub const NF_NETDEV_NUMHOOKS: ::c_int = 1; // linux/netfilter_ipv4.h pub const NF_IP_PRE_ROUTING: ::c_int = 0; @@ -2011,6 +2629,24 @@ pub const SIOCGIFSLAVE: ::c_ulong = 0x00008929; pub const SIOCSIFSLAVE: ::c_ulong = 0x00008930; pub const SIOCADDMULTI: ::c_ulong = 0x00008931; pub const SIOCDELMULTI: ::c_ulong = 0x00008932; +pub const SIOCGIFINDEX: ::c_ulong = 0x00008933; +pub const SIOGIFINDEX: ::c_ulong = SIOCGIFINDEX; +pub const SIOCSIFPFLAGS: ::c_ulong = 0x00008934; +pub const SIOCGIFPFLAGS: ::c_ulong = 0x00008935; +pub const SIOCDIFADDR: ::c_ulong = 0x00008936; +pub const SIOCSIFHWBROADCAST: ::c_ulong = 0x00008937; +pub const SIOCGIFCOUNT: ::c_ulong = 0x00008938; +pub const SIOCGIFBR: ::c_ulong = 0x00008940; +pub const SIOCSIFBR: ::c_ulong = 0x00008941; +pub const SIOCGIFTXQLEN: ::c_ulong = 0x00008942; +pub const SIOCSIFTXQLEN: ::c_ulong = 0x00008943; +pub const SIOCETHTOOL: ::c_ulong = 0x00008946; +pub const SIOCGMIIPHY: ::c_ulong = 0x00008947; +pub const SIOCGMIIREG: ::c_ulong = 0x00008948; +pub const SIOCSMIIREG: ::c_ulong = 0x00008949; +pub const SIOCWANDEV: ::c_ulong = 0x0000894A; +pub const SIOCOUTQNSD: ::c_ulong = 0x0000894B; +pub const SIOCGSKNS: ::c_ulong = 0x0000894C; pub const SIOCDARP: ::c_ulong = 0x00008953; pub const SIOCGARP: ::c_ulong = 0x00008954; pub const SIOCSARP: ::c_ulong = 0x00008955; @@ -2152,6 +2788,8 @@ pub const NETLINK_TX_RING: ::c_int = 7; pub const NETLINK_LISTEN_ALL_NSID: ::c_int = 8; pub const NETLINK_LIST_MEMBERSHIPS: ::c_int = 9; pub const NETLINK_CAP_ACK: ::c_int = 10; +pub const NETLINK_EXT_ACK: ::c_int = 11; +pub const NETLINK_GET_STRICT_CHK: ::c_int = 12; pub const NLA_F_NESTED: ::c_int = 1 << 15; pub const NLA_F_NET_BYTEORDER: ::c_int = 1 << 14; @@ -2288,11 +2926,61 @@ pub const ARPD_LOOKUP: ::c_ushort = 0x02; pub const ARPD_FLUSH: ::c_ushort = 0x03; pub const ATF_MAGIC: ::c_int = 0x80; -#[cfg(not(target_arch = "sparc64"))] -pub const SO_TIMESTAMPING: ::c_int = 37; -#[cfg(target_arch = "sparc64")] -pub const SO_TIMESTAMPING: ::c_int = 35; -pub const SCM_TIMESTAMPING: ::c_int = SO_TIMESTAMPING; +// userspace compat definitions for RTNLGRP_* +pub const RTMGRP_LINK: ::c_int = 0x00001; +pub const RTMGRP_NOTIFY: ::c_int = 0x00002; +pub const RTMGRP_NEIGH: ::c_int = 0x00004; +pub const RTMGRP_TC: ::c_int = 0x00008; +pub const RTMGRP_IPV4_IFADDR: ::c_int = 0x00010; +pub const RTMGRP_IPV4_MROUTE: ::c_int = 0x00020; +pub const RTMGRP_IPV4_ROUTE: ::c_int = 0x00040; +pub const RTMGRP_IPV4_RULE: ::c_int = 0x00080; +pub const RTMGRP_IPV6_IFADDR: ::c_int = 0x00100; +pub const RTMGRP_IPV6_MROUTE: ::c_int = 0x00200; +pub const RTMGRP_IPV6_ROUTE: ::c_int = 0x00400; +pub const RTMGRP_IPV6_IFINFO: ::c_int = 0x00800; +pub const RTMGRP_DECnet_IFADDR: ::c_int = 0x01000; +pub const RTMGRP_DECnet_ROUTE: ::c_int = 0x04000; +pub const RTMGRP_IPV6_PREFIX: ::c_int = 0x20000; + +// enum rtnetlink_groups +pub const RTNLGRP_NONE: ::c_uint = 0x00; +pub const RTNLGRP_LINK: ::c_uint = 0x01; +pub const RTNLGRP_NOTIFY: ::c_uint = 0x02; +pub const RTNLGRP_NEIGH: ::c_uint = 0x03; +pub const RTNLGRP_TC: ::c_uint = 0x04; +pub const RTNLGRP_IPV4_IFADDR: ::c_uint = 0x05; +pub const RTNLGRP_IPV4_MROUTE: ::c_uint = 0x06; +pub const RTNLGRP_IPV4_ROUTE: ::c_uint = 0x07; +pub const RTNLGRP_IPV4_RULE: ::c_uint = 0x08; +pub const RTNLGRP_IPV6_IFADDR: ::c_uint = 0x09; +pub const RTNLGRP_IPV6_MROUTE: ::c_uint = 0x0a; +pub const RTNLGRP_IPV6_ROUTE: ::c_uint = 0x0b; +pub const RTNLGRP_IPV6_IFINFO: ::c_uint = 0x0c; +pub const RTNLGRP_DECnet_IFADDR: ::c_uint = 0x0d; +pub const RTNLGRP_NOP2: ::c_uint = 0x0e; +pub const RTNLGRP_DECnet_ROUTE: ::c_uint = 0x0f; +pub const RTNLGRP_DECnet_RULE: ::c_uint = 0x10; +pub const RTNLGRP_NOP4: ::c_uint = 0x11; +pub const RTNLGRP_IPV6_PREFIX: ::c_uint = 0x12; +pub const RTNLGRP_IPV6_RULE: ::c_uint = 0x13; +pub const RTNLGRP_ND_USEROPT: ::c_uint = 0x14; +pub const RTNLGRP_PHONET_IFADDR: ::c_uint = 0x15; +pub const RTNLGRP_PHONET_ROUTE: ::c_uint = 0x16; +pub const RTNLGRP_DCB: ::c_uint = 0x17; +pub const RTNLGRP_IPV4_NETCONF: ::c_uint = 0x18; +pub const RTNLGRP_IPV6_NETCONF: ::c_uint = 0x19; +pub const RTNLGRP_MDB: ::c_uint = 0x1a; +pub const RTNLGRP_MPLS_ROUTE: ::c_uint = 0x1b; +pub const RTNLGRP_NSID: ::c_uint = 0x1c; +pub const RTNLGRP_MPLS_NETCONF: ::c_uint = 0x1d; +pub const RTNLGRP_IPV4_MROUTE_R: ::c_uint = 0x1e; +pub const RTNLGRP_IPV6_MROUTE_R: ::c_uint = 0x1f; +pub const RTNLGRP_NEXTHOP: ::c_uint = 0x20; +pub const RTNLGRP_BRVLAN: ::c_uint = 0x21; +pub const RTNLGRP_MCTP_IFADDR: ::c_uint = 0x22; +pub const RTNLGRP_TUNNEL: ::c_uint = 0x23; +pub const RTNLGRP_STATS: ::c_uint = 0x24; // linux/module.h pub const MODULE_INIT_IGNORE_MODVERSIONS: ::c_uint = 0x0001; @@ -2306,6 +2994,16 @@ pub const SOF_TIMESTAMPING_RX_SOFTWARE: ::c_uint = 1 << 3; pub const SOF_TIMESTAMPING_SOFTWARE: ::c_uint = 1 << 4; pub const SOF_TIMESTAMPING_SYS_HARDWARE: ::c_uint = 1 << 5; pub const SOF_TIMESTAMPING_RAW_HARDWARE: ::c_uint = 1 << 6; +pub const SOF_TIMESTAMPING_OPT_ID: ::c_uint = 1 << 7; +pub const SOF_TIMESTAMPING_TX_SCHED: ::c_uint = 1 << 8; +pub const SOF_TIMESTAMPING_TX_ACK: ::c_uint = 1 << 9; +pub const SOF_TIMESTAMPING_OPT_CMSG: ::c_uint = 1 << 10; +pub const SOF_TIMESTAMPING_OPT_TSONLY: ::c_uint = 1 << 11; +pub const SOF_TIMESTAMPING_OPT_STATS: ::c_uint = 1 << 12; +pub const SOF_TIMESTAMPING_OPT_PKTINFO: ::c_uint = 1 << 13; +pub const SOF_TIMESTAMPING_OPT_TX_SWHW: ::c_uint = 1 << 14; +pub const SOF_TXTIME_DEADLINE_MODE: u32 = 1 << 0; +pub const SOF_TXTIME_REPORT_ERRORS: u32 = 1 << 1; // linux/if_alg.h pub const ALG_SET_KEY: ::c_int = 1; @@ -2317,16 +3015,29 @@ pub const ALG_SET_AEAD_AUTHSIZE: ::c_int = 5; pub const ALG_OP_DECRYPT: ::c_int = 0; pub const ALG_OP_ENCRYPT: ::c_int = 1; +// include/uapi/linux/udp.h +pub const UDP_CORK: ::c_int = 1; +pub const UDP_ENCAP: ::c_int = 100; +pub const UDP_NO_CHECK6_TX: ::c_int = 101; +pub const UDP_NO_CHECK6_RX: ::c_int = 102; + // include/uapi/linux/mman.h pub const MAP_SHARED_VALIDATE: ::c_int = 0x3; // include/uapi/asm-generic/mman-common.h pub const MAP_FIXED_NOREPLACE: ::c_int = 0x100000; +pub const MLOCK_ONFAULT: ::c_uint = 0x01; // uapi/linux/vm_sockets.h pub const VMADDR_CID_ANY: ::c_uint = 0xFFFFFFFF; pub const VMADDR_CID_HYPERVISOR: ::c_uint = 0; +#[deprecated( + since = "0.2.74", + note = "VMADDR_CID_RESERVED is removed since Linux v5.6 and \ + replaced with VMADDR_CID_LOCAL" +)] pub const VMADDR_CID_RESERVED: ::c_uint = 1; +pub const VMADDR_CID_LOCAL: ::c_uint = 1; pub const VMADDR_CID_HOST: ::c_uint = 2; pub const VMADDR_PORT_ANY: ::c_uint = 0xFFFFFFFF; @@ -2417,6 +3128,268 @@ pub const IN_ALL_EVENTS: u32 = IN_ACCESS pub const IN_CLOEXEC: ::c_int = O_CLOEXEC; pub const IN_NONBLOCK: ::c_int = O_NONBLOCK; +// uapi/linux/netfilter/nf_tables.h +pub const NFT_TABLE_MAXNAMELEN: ::c_int = 256; +pub const NFT_CHAIN_MAXNAMELEN: ::c_int = 256; +pub const NFT_SET_MAXNAMELEN: ::c_int = 256; +pub const NFT_OBJ_MAXNAMELEN: ::c_int = 256; +pub const NFT_USERDATA_MAXLEN: ::c_int = 256; + +pub const NFT_REG_VERDICT: ::c_int = 0; +pub const NFT_REG_1: ::c_int = 1; +pub const NFT_REG_2: ::c_int = 2; +pub const NFT_REG_3: ::c_int = 3; +pub const NFT_REG_4: ::c_int = 4; +pub const __NFT_REG_MAX: ::c_int = 5; +pub const NFT_REG32_00: ::c_int = 8; +pub const NFT_REG32_01: ::c_int = 9; +pub const NFT_REG32_02: ::c_int = 10; +pub const NFT_REG32_03: ::c_int = 11; +pub const NFT_REG32_04: ::c_int = 12; +pub const NFT_REG32_05: ::c_int = 13; +pub const NFT_REG32_06: ::c_int = 14; +pub const NFT_REG32_07: ::c_int = 15; +pub const NFT_REG32_08: ::c_int = 16; +pub const NFT_REG32_09: ::c_int = 17; +pub const NFT_REG32_10: ::c_int = 18; +pub const NFT_REG32_11: ::c_int = 19; +pub const NFT_REG32_12: ::c_int = 20; +pub const NFT_REG32_13: ::c_int = 21; +pub const NFT_REG32_14: ::c_int = 22; +pub const NFT_REG32_15: ::c_int = 23; + +pub const NFT_REG_SIZE: ::c_int = 16; +pub const NFT_REG32_SIZE: ::c_int = 4; + +pub const NFT_CONTINUE: ::c_int = -1; +pub const NFT_BREAK: ::c_int = -2; +pub const NFT_JUMP: ::c_int = -3; +pub const NFT_GOTO: ::c_int = -4; +pub const NFT_RETURN: ::c_int = -5; + +pub const NFT_MSG_NEWTABLE: ::c_int = 0; +pub const NFT_MSG_GETTABLE: ::c_int = 1; +pub const NFT_MSG_DELTABLE: ::c_int = 2; +pub const NFT_MSG_NEWCHAIN: ::c_int = 3; +pub const NFT_MSG_GETCHAIN: ::c_int = 4; +pub const NFT_MSG_DELCHAIN: ::c_int = 5; +pub const NFT_MSG_NEWRULE: ::c_int = 6; +pub const NFT_MSG_GETRULE: ::c_int = 7; +pub const NFT_MSG_DELRULE: ::c_int = 8; +pub const NFT_MSG_NEWSET: ::c_int = 9; +pub const NFT_MSG_GETSET: ::c_int = 10; +pub const NFT_MSG_DELSET: ::c_int = 11; +pub const NFT_MSG_NEWSETELEM: ::c_int = 12; +pub const NFT_MSG_GETSETELEM: ::c_int = 13; +pub const NFT_MSG_DELSETELEM: ::c_int = 14; +pub const NFT_MSG_NEWGEN: ::c_int = 15; +pub const NFT_MSG_GETGEN: ::c_int = 16; +pub const NFT_MSG_TRACE: ::c_int = 17; +cfg_if! { + if #[cfg(not(target_arch = "sparc64"))] { + pub const NFT_MSG_NEWOBJ: ::c_int = 18; + pub const NFT_MSG_GETOBJ: ::c_int = 19; + pub const NFT_MSG_DELOBJ: ::c_int = 20; + pub const NFT_MSG_GETOBJ_RESET: ::c_int = 21; + } +} +pub const NFT_MSG_MAX: ::c_int = 25; + +pub const NFT_SET_ANONYMOUS: ::c_int = 0x1; +pub const NFT_SET_CONSTANT: ::c_int = 0x2; +pub const NFT_SET_INTERVAL: ::c_int = 0x4; +pub const NFT_SET_MAP: ::c_int = 0x8; +pub const NFT_SET_TIMEOUT: ::c_int = 0x10; +pub const NFT_SET_EVAL: ::c_int = 0x20; + +pub const NFT_SET_POL_PERFORMANCE: ::c_int = 0; +pub const NFT_SET_POL_MEMORY: ::c_int = 1; + +pub const NFT_SET_ELEM_INTERVAL_END: ::c_int = 0x1; + +pub const NFT_DATA_VALUE: ::c_uint = 0; +pub const NFT_DATA_VERDICT: ::c_uint = 0xffffff00; + +pub const NFT_DATA_RESERVED_MASK: ::c_uint = 0xffffff00; + +pub const NFT_DATA_VALUE_MAXLEN: ::c_int = 64; + +pub const NFT_BYTEORDER_NTOH: ::c_int = 0; +pub const NFT_BYTEORDER_HTON: ::c_int = 1; + +pub const NFT_CMP_EQ: ::c_int = 0; +pub const NFT_CMP_NEQ: ::c_int = 1; +pub const NFT_CMP_LT: ::c_int = 2; +pub const NFT_CMP_LTE: ::c_int = 3; +pub const NFT_CMP_GT: ::c_int = 4; +pub const NFT_CMP_GTE: ::c_int = 5; + +pub const NFT_RANGE_EQ: ::c_int = 0; +pub const NFT_RANGE_NEQ: ::c_int = 1; + +pub const NFT_LOOKUP_F_INV: ::c_int = 1 << 0; + +pub const NFT_DYNSET_OP_ADD: ::c_int = 0; +pub const NFT_DYNSET_OP_UPDATE: ::c_int = 1; + +pub const NFT_DYNSET_F_INV: ::c_int = 1 << 0; + +pub const NFT_PAYLOAD_LL_HEADER: ::c_int = 0; +pub const NFT_PAYLOAD_NETWORK_HEADER: ::c_int = 1; +pub const NFT_PAYLOAD_TRANSPORT_HEADER: ::c_int = 2; + +pub const NFT_PAYLOAD_CSUM_NONE: ::c_int = 0; +pub const NFT_PAYLOAD_CSUM_INET: ::c_int = 1; + +pub const NFT_META_LEN: ::c_int = 0; +pub const NFT_META_PROTOCOL: ::c_int = 1; +pub const NFT_META_PRIORITY: ::c_int = 2; +pub const NFT_META_MARK: ::c_int = 3; +pub const NFT_META_IIF: ::c_int = 4; +pub const NFT_META_OIF: ::c_int = 5; +pub const NFT_META_IIFNAME: ::c_int = 6; +pub const NFT_META_OIFNAME: ::c_int = 7; +pub const NFT_META_IIFTYPE: ::c_int = 8; +pub const NFT_META_OIFTYPE: ::c_int = 9; +pub const NFT_META_SKUID: ::c_int = 10; +pub const NFT_META_SKGID: ::c_int = 11; +pub const NFT_META_NFTRACE: ::c_int = 12; +pub const NFT_META_RTCLASSID: ::c_int = 13; +pub const NFT_META_SECMARK: ::c_int = 14; +pub const NFT_META_NFPROTO: ::c_int = 15; +pub const NFT_META_L4PROTO: ::c_int = 16; +pub const NFT_META_BRI_IIFNAME: ::c_int = 17; +pub const NFT_META_BRI_OIFNAME: ::c_int = 18; +pub const NFT_META_PKTTYPE: ::c_int = 19; +pub const NFT_META_CPU: ::c_int = 20; +pub const NFT_META_IIFGROUP: ::c_int = 21; +pub const NFT_META_OIFGROUP: ::c_int = 22; +pub const NFT_META_CGROUP: ::c_int = 23; +pub const NFT_META_PRANDOM: ::c_int = 24; + +pub const NFT_CT_STATE: ::c_int = 0; +pub const NFT_CT_DIRECTION: ::c_int = 1; +pub const NFT_CT_STATUS: ::c_int = 2; +pub const NFT_CT_MARK: ::c_int = 3; +pub const NFT_CT_SECMARK: ::c_int = 4; +pub const NFT_CT_EXPIRATION: ::c_int = 5; +pub const NFT_CT_HELPER: ::c_int = 6; +pub const NFT_CT_L3PROTOCOL: ::c_int = 7; +pub const NFT_CT_SRC: ::c_int = 8; +pub const NFT_CT_DST: ::c_int = 9; +pub const NFT_CT_PROTOCOL: ::c_int = 10; +pub const NFT_CT_PROTO_SRC: ::c_int = 11; +pub const NFT_CT_PROTO_DST: ::c_int = 12; +pub const NFT_CT_LABELS: ::c_int = 13; +pub const NFT_CT_PKTS: ::c_int = 14; +pub const NFT_CT_BYTES: ::c_int = 15; + +pub const NFT_LIMIT_PKTS: ::c_int = 0; +pub const NFT_LIMIT_PKT_BYTES: ::c_int = 1; + +pub const NFT_LIMIT_F_INV: ::c_int = 1 << 0; + +pub const NFT_QUEUE_FLAG_BYPASS: ::c_int = 0x01; +pub const NFT_QUEUE_FLAG_CPU_FANOUT: ::c_int = 0x02; +pub const NFT_QUEUE_FLAG_MASK: ::c_int = 0x03; + +pub const NFT_QUOTA_F_INV: ::c_int = 1 << 0; + +pub const NFT_REJECT_ICMP_UNREACH: ::c_int = 0; +pub const NFT_REJECT_TCP_RST: ::c_int = 1; +pub const NFT_REJECT_ICMPX_UNREACH: ::c_int = 2; + +pub const NFT_REJECT_ICMPX_NO_ROUTE: ::c_int = 0; +pub const NFT_REJECT_ICMPX_PORT_UNREACH: ::c_int = 1; +pub const NFT_REJECT_ICMPX_HOST_UNREACH: ::c_int = 2; +pub const NFT_REJECT_ICMPX_ADMIN_PROHIBITED: ::c_int = 3; + +pub const NFT_NAT_SNAT: ::c_int = 0; +pub const NFT_NAT_DNAT: ::c_int = 1; + +pub const NFT_TRACETYPE_UNSPEC: ::c_int = 0; +pub const NFT_TRACETYPE_POLICY: ::c_int = 1; +pub const NFT_TRACETYPE_RETURN: ::c_int = 2; +pub const NFT_TRACETYPE_RULE: ::c_int = 3; + +pub const NFT_NG_INCREMENTAL: ::c_int = 0; +pub const NFT_NG_RANDOM: ::c_int = 1; + +// linux/input.h +pub const FF_MAX: ::__u16 = 0x7f; +pub const FF_CNT: usize = FF_MAX as usize + 1; + +// linux/input-event-codes.h +pub const INPUT_PROP_MAX: ::__u16 = 0x1f; +pub const INPUT_PROP_CNT: usize = INPUT_PROP_MAX as usize + 1; +pub const EV_MAX: ::__u16 = 0x1f; +pub const EV_CNT: usize = EV_MAX as usize + 1; +pub const SYN_MAX: ::__u16 = 0xf; +pub const SYN_CNT: usize = SYN_MAX as usize + 1; +pub const KEY_MAX: ::__u16 = 0x2ff; +pub const KEY_CNT: usize = KEY_MAX as usize + 1; +pub const REL_MAX: ::__u16 = 0x0f; +pub const REL_CNT: usize = REL_MAX as usize + 1; +pub const ABS_MAX: ::__u16 = 0x3f; +pub const ABS_CNT: usize = ABS_MAX as usize + 1; +pub const SW_MAX: ::__u16 = 0x10; +pub const SW_CNT: usize = SW_MAX as usize + 1; +pub const MSC_MAX: ::__u16 = 0x07; +pub const MSC_CNT: usize = MSC_MAX as usize + 1; +pub const LED_MAX: ::__u16 = 0x0f; +pub const LED_CNT: usize = LED_MAX as usize + 1; +pub const REP_MAX: ::__u16 = 0x01; +pub const REP_CNT: usize = REP_MAX as usize + 1; +pub const SND_MAX: ::__u16 = 0x07; +pub const SND_CNT: usize = SND_MAX as usize + 1; + +// linux/uinput.h +pub const UINPUT_VERSION: ::c_uint = 5; +pub const UINPUT_MAX_NAME_SIZE: usize = 80; + +// uapi/linux/fanotify.h +pub const FAN_ACCESS: u64 = 0x0000_0001; +pub const FAN_MODIFY: u64 = 0x0000_0002; +pub const FAN_CLOSE_WRITE: u64 = 0x0000_0008; +pub const FAN_CLOSE_NOWRITE: u64 = 0x0000_0010; +pub const FAN_OPEN: u64 = 0x0000_0020; + +pub const FAN_Q_OVERFLOW: u64 = 0x0000_4000; + +pub const FAN_OPEN_PERM: u64 = 0x0001_0000; +pub const FAN_ACCESS_PERM: u64 = 0x0002_0000; + +pub const FAN_ONDIR: u64 = 0x4000_0000; + +pub const FAN_EVENT_ON_CHILD: u64 = 0x0800_0000; + +pub const FAN_CLOSE: u64 = FAN_CLOSE_WRITE | FAN_CLOSE_NOWRITE; + +pub const FAN_CLOEXEC: ::c_uint = 0x0000_0001; +pub const FAN_NONBLOCK: ::c_uint = 0x0000_0002; + +pub const FAN_CLASS_NOTIF: ::c_uint = 0x0000_0000; +pub const FAN_CLASS_CONTENT: ::c_uint = 0x0000_0004; +pub const FAN_CLASS_PRE_CONTENT: ::c_uint = 0x0000_0008; + +pub const FAN_UNLIMITED_QUEUE: ::c_uint = 0x0000_0010; +pub const FAN_UNLIMITED_MARKS: ::c_uint = 0x0000_0020; + +pub const FAN_MARK_ADD: ::c_uint = 0x0000_0001; +pub const FAN_MARK_REMOVE: ::c_uint = 0x0000_0002; +pub const FAN_MARK_DONT_FOLLOW: ::c_uint = 0x0000_0004; +pub const FAN_MARK_ONLYDIR: ::c_uint = 0x0000_0008; +pub const FAN_MARK_IGNORED_MASK: ::c_uint = 0x0000_0020; +pub const FAN_MARK_IGNORED_SURV_MODIFY: ::c_uint = 0x0000_0040; +pub const FAN_MARK_FLUSH: ::c_uint = 0x0000_0080; + +pub const FANOTIFY_METADATA_VERSION: u8 = 3; + +pub const FAN_ALLOW: u32 = 0x01; +pub const FAN_DENY: u32 = 0x02; + +pub const FAN_NOFD: ::c_int = -1; + pub const FUTEX_WAIT: ::c_int = 0; pub const FUTEX_WAKE: ::c_int = 1; pub const FUTEX_FD: ::c_int = 2; @@ -2430,11 +3403,11 @@ pub const FUTEX_WAIT_BITSET: ::c_int = 9; pub const FUTEX_WAKE_BITSET: ::c_int = 10; pub const FUTEX_WAIT_REQUEUE_PI: ::c_int = 11; pub const FUTEX_CMP_REQUEUE_PI: ::c_int = 12; +pub const FUTEX_LOCK_PI2: ::c_int = 13; pub const FUTEX_PRIVATE_FLAG: ::c_int = 128; pub const FUTEX_CLOCK_REALTIME: ::c_int = 256; -pub const FUTEX_CMD_MASK: ::c_int = - !(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME); +pub const FUTEX_CMD_MASK: ::c_int = !(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME); // linux/reboot.h pub const LINUX_REBOOT_MAGIC1: ::c_int = 0xfee1dead; @@ -2452,6 +3425,164 @@ pub const LINUX_REBOOT_CMD_RESTART2: ::c_int = 0xA1B2C3D4; pub const LINUX_REBOOT_CMD_SW_SUSPEND: ::c_int = 0xD000FCE2; pub const LINUX_REBOOT_CMD_KEXEC: ::c_int = 0x45584543; +pub const REG_EXTENDED: ::c_int = 1; +pub const REG_ICASE: ::c_int = 2; +pub const REG_NEWLINE: ::c_int = 4; +pub const REG_NOSUB: ::c_int = 8; + +pub const REG_NOTBOL: ::c_int = 1; +pub const REG_NOTEOL: ::c_int = 2; + +pub const REG_ENOSYS: ::c_int = -1; +pub const REG_NOMATCH: ::c_int = 1; +pub const REG_BADPAT: ::c_int = 2; +pub const REG_ECOLLATE: ::c_int = 3; +pub const REG_ECTYPE: ::c_int = 4; +pub const REG_EESCAPE: ::c_int = 5; +pub const REG_ESUBREG: ::c_int = 6; +pub const REG_EBRACK: ::c_int = 7; +pub const REG_EPAREN: ::c_int = 8; +pub const REG_EBRACE: ::c_int = 9; +pub const REG_BADBR: ::c_int = 10; +pub const REG_ERANGE: ::c_int = 11; +pub const REG_ESPACE: ::c_int = 12; +pub const REG_BADRPT: ::c_int = 13; + +// linux/errqueue.h +pub const SO_EE_ORIGIN_NONE: u8 = 0; +pub const SO_EE_ORIGIN_LOCAL: u8 = 1; +pub const SO_EE_ORIGIN_ICMP: u8 = 2; +pub const SO_EE_ORIGIN_ICMP6: u8 = 3; +pub const SO_EE_ORIGIN_TXSTATUS: u8 = 4; +pub const SO_EE_ORIGIN_TIMESTAMPING: u8 = SO_EE_ORIGIN_TXSTATUS; + +// errno.h +pub const EPERM: ::c_int = 1; +pub const ENOENT: ::c_int = 2; +pub const ESRCH: ::c_int = 3; +pub const EINTR: ::c_int = 4; +pub const EIO: ::c_int = 5; +pub const ENXIO: ::c_int = 6; +pub const E2BIG: ::c_int = 7; +pub const ENOEXEC: ::c_int = 8; +pub const EBADF: ::c_int = 9; +pub const ECHILD: ::c_int = 10; +pub const EAGAIN: ::c_int = 11; +pub const ENOMEM: ::c_int = 12; +pub const EACCES: ::c_int = 13; +pub const EFAULT: ::c_int = 14; +pub const ENOTBLK: ::c_int = 15; +pub const EBUSY: ::c_int = 16; +pub const EEXIST: ::c_int = 17; +pub const EXDEV: ::c_int = 18; +pub const ENODEV: ::c_int = 19; +pub const ENOTDIR: ::c_int = 20; +pub const EISDIR: ::c_int = 21; +pub const EINVAL: ::c_int = 22; +pub const ENFILE: ::c_int = 23; +pub const EMFILE: ::c_int = 24; +pub const ENOTTY: ::c_int = 25; +pub const ETXTBSY: ::c_int = 26; +pub const EFBIG: ::c_int = 27; +pub const ENOSPC: ::c_int = 28; +pub const ESPIPE: ::c_int = 29; +pub const EROFS: ::c_int = 30; +pub const EMLINK: ::c_int = 31; +pub const EPIPE: ::c_int = 32; +pub const EDOM: ::c_int = 33; +pub const ERANGE: ::c_int = 34; +pub const EWOULDBLOCK: ::c_int = EAGAIN; + +// linux/can.h +pub const CAN_EFF_FLAG: canid_t = 0x80000000; +pub const CAN_RTR_FLAG: canid_t = 0x40000000; +pub const CAN_ERR_FLAG: canid_t = 0x20000000; +pub const CAN_SFF_MASK: canid_t = 0x000007FF; +pub const CAN_EFF_MASK: canid_t = 0x1FFFFFFF; +pub const CAN_ERR_MASK: canid_t = 0x1FFFFFFF; + +pub const CAN_SFF_ID_BITS: ::c_int = 11; +pub const CAN_EFF_ID_BITS: ::c_int = 29; + +pub const CAN_MAX_DLC: ::c_int = 8; +pub const CAN_MAX_DLEN: usize = 8; +pub const CANFD_MAX_DLC: ::c_int = 15; +pub const CANFD_MAX_DLEN: usize = 64; + +pub const CANFD_BRS: ::c_int = 0x01; +pub const CANFD_ESI: ::c_int = 0x02; + +cfg_if! { + if #[cfg(libc_align)] { + pub const CAN_MTU: usize = ::mem::size_of::(); + pub const CANFD_MTU: usize = ::mem::size_of::(); + } +} + +pub const CAN_RAW: ::c_int = 1; +pub const CAN_BCM: ::c_int = 2; +pub const CAN_TP16: ::c_int = 3; +pub const CAN_TP20: ::c_int = 4; +pub const CAN_MCNET: ::c_int = 5; +pub const CAN_ISOTP: ::c_int = 6; +pub const CAN_J1939: ::c_int = 7; +pub const CAN_NPROTO: ::c_int = 8; + +pub const SOL_CAN_BASE: ::c_int = 100; + +pub const CAN_INV_FILTER: canid_t = 0x20000000; +pub const CAN_RAW_FILTER_MAX: ::c_int = 512; + +// linux/can/raw.h +pub const SOL_CAN_RAW: ::c_int = SOL_CAN_BASE + CAN_RAW; +pub const CAN_RAW_FILTER: ::c_int = 1; +pub const CAN_RAW_ERR_FILTER: ::c_int = 2; +pub const CAN_RAW_LOOPBACK: ::c_int = 3; +pub const CAN_RAW_RECV_OWN_MSGS: ::c_int = 4; +pub const CAN_RAW_FD_FRAMES: ::c_int = 5; +pub const CAN_RAW_JOIN_FILTERS: ::c_int = 6; + +// linux/can/j1939.h +pub const SOL_CAN_J1939: ::c_int = SOL_CAN_BASE + CAN_J1939; + +pub const J1939_MAX_UNICAST_ADDR: ::c_uchar = 0xfd; +pub const J1939_IDLE_ADDR: ::c_uchar = 0xfe; +pub const J1939_NO_ADDR: ::c_uchar = 0xff; +pub const J1939_NO_NAME: ::c_ulong = 0; +pub const J1939_PGN_REQUEST: ::c_uint = 0x0ea00; +pub const J1939_PGN_ADDRESS_CLAIMED: ::c_uint = 0x0ee00; +pub const J1939_PGN_ADDRESS_COMMANDED: ::c_uint = 0x0fed8; +pub const J1939_PGN_PDU1_MAX: ::c_uint = 0x3ff00; +pub const J1939_PGN_MAX: ::c_uint = 0x3ffff; +pub const J1939_NO_PGN: ::c_uint = 0x40000; + +pub const SO_J1939_FILTER: ::c_int = 1; +pub const SO_J1939_PROMISC: ::c_int = 2; +pub const SO_J1939_SEND_PRIO: ::c_int = 3; +pub const SO_J1939_ERRQUEUE: ::c_int = 4; + +pub const SCM_J1939_DEST_ADDR: ::c_int = 1; +pub const SCM_J1939_DEST_NAME: ::c_int = 2; +pub const SCM_J1939_PRIO: ::c_int = 3; +pub const SCM_J1939_ERRQUEUE: ::c_int = 4; + +pub const J1939_NLA_PAD: ::c_int = 0; +pub const J1939_NLA_BYTES_ACKED: ::c_int = 1; +pub const J1939_NLA_TOTAL_SIZE: ::c_int = 2; +pub const J1939_NLA_PGN: ::c_int = 3; +pub const J1939_NLA_SRC_NAME: ::c_int = 4; +pub const J1939_NLA_DEST_NAME: ::c_int = 5; +pub const J1939_NLA_SRC_ADDR: ::c_int = 6; +pub const J1939_NLA_DEST_ADDR: ::c_int = 7; + +pub const J1939_EE_INFO_NONE: ::c_int = 0; +pub const J1939_EE_INFO_TX_ABORT: ::c_int = 1; +pub const J1939_EE_INFO_RX_RTS: ::c_int = 2; +pub const J1939_EE_INFO_RX_DPO: ::c_int = 3; +pub const J1939_EE_INFO_RX_ABORT: ::c_int = 4; + +pub const J1939_FILTER_MAX: ::c_int = 512; + f! { pub fn NLA_ALIGN(len: ::c_int) -> ::c_int { return ((len) + NLA_ALIGNTO - 1) & !(NLA_ALIGNTO - 1) @@ -2476,6 +3607,12 @@ f! { } } + pub fn CPU_ALLOC_SIZE(count: ::c_int) -> ::size_t { + let _dummy: cpu_set_t = ::mem::zeroed(); + let size_in_bits = 8 * ::mem::size_of_val(&_dummy.bits[0]); + ((count as ::size_t + size_in_bits - 1) / 8) as ::size_t + } + pub fn CPU_ZERO(cpuset: &mut cpu_set_t) -> () { for slot in cpuset.bits.iter_mut() { *slot = 0; @@ -2504,6 +3641,19 @@ f! { 0 != (cpuset.bits[idx] & (1 << offset)) } + pub fn CPU_COUNT_S(size: usize, cpuset: &cpu_set_t) -> ::c_int { + let mut s: u32 = 0; + let size_of_mask = ::mem::size_of_val(&cpuset.bits[0]); + for i in cpuset.bits[..(size / size_of_mask)].iter() { + s += i.count_ones(); + }; + s as ::c_int + } + + pub fn CPU_COUNT(cpuset: &cpu_set_t) -> ::c_int { + CPU_COUNT_S(::mem::size_of::(), cpuset) + } + pub fn CPU_EQUAL(set1: &cpu_set_t, set2: &cpu_set_t) -> bool { set1.bits == set2.bits } @@ -2522,17 +3672,6 @@ f! { minor as ::c_uint } - pub fn makedev(major: ::c_uint, minor: ::c_uint) -> ::dev_t { - let major = major as ::dev_t; - let minor = minor as ::dev_t; - let mut dev = 0; - dev |= (major & 0x00000fff) << 8; - dev |= (major & 0xfffff000) << 32; - dev |= (minor & 0x000000ff) << 0; - dev |= (minor & 0xffffff00) << 12; - dev - } - pub fn IPTOS_TOS(tos: u8) -> u8 { tos & IPTOS_TOS_MASK } @@ -2552,15 +3691,113 @@ f! { pub fn RT_LOCALADDR(flags: u32) -> bool { (flags & RTF_ADDRCLASSMASK) == (RTF_LOCAL | RTF_INTERFACE) } + + pub fn SO_EE_OFFENDER(ee: *const ::sock_extended_err) -> *mut ::sockaddr { + ee.offset(1) as *mut ::sockaddr + } + + pub fn BPF_RVAL(code: ::__u32) -> ::__u32 { + code & 0x18 + } + + pub fn BPF_MISCOP(code: ::__u32) -> ::__u32 { + code & 0xf8 + } + + pub fn BPF_STMT(code: ::__u16, k: ::__u32) -> sock_filter { + sock_filter{code: code, jt: 0, jf: 0, k: k} + } + + pub fn BPF_JUMP(code: ::__u16, k: ::__u32, jt: ::__u8, jf: ::__u8) -> sock_filter { + sock_filter{code: code, jt: jt, jf: jf, k: k} + } +} + +safe_f! { + pub {const} fn makedev(major: ::c_uint, minor: ::c_uint) -> ::dev_t { + let major = major as ::dev_t; + let minor = minor as ::dev_t; + let mut dev = 0; + dev |= (major & 0x00000fff) << 8; + dev |= (major & 0xfffff000) << 32; + dev |= (minor & 0x000000ff) << 0; + dev |= (minor & 0xffffff00) << 12; + dev + } +} + +cfg_if! { + if #[cfg(not(target_env = "uclibc"))] { + extern "C" { + pub fn aio_read(aiocbp: *mut aiocb) -> ::c_int; + pub fn aio_write(aiocbp: *mut aiocb) -> ::c_int; + pub fn aio_fsync(op: ::c_int, aiocbp: *mut aiocb) -> ::c_int; + pub fn aio_error(aiocbp: *const aiocb) -> ::c_int; + pub fn aio_return(aiocbp: *mut aiocb) -> ::ssize_t; + pub fn aio_suspend( + aiocb_list: *const *const aiocb, + nitems: ::c_int, + timeout: *const ::timespec, + ) -> ::c_int; + pub fn aio_cancel(fd: ::c_int, aiocbp: *mut aiocb) -> ::c_int; + pub fn lio_listio( + mode: ::c_int, + aiocb_list: *const *mut aiocb, + nitems: ::c_int, + sevp: *mut ::sigevent, + ) -> ::c_int; + pub fn pwritev( + fd: ::c_int, + iov: *const ::iovec, + iovcnt: ::c_int, + offset: ::off_t, + ) -> ::ssize_t; + pub fn preadv( + fd: ::c_int, + iov: *const ::iovec, + iovcnt: ::c_int, + offset: ::off_t, + ) -> ::ssize_t; + pub fn getnameinfo( + sa: *const ::sockaddr, + salen: ::socklen_t, + host: *mut ::c_char, + hostlen: ::socklen_t, + serv: *mut ::c_char, + sevlen: ::socklen_t, + flags: ::c_int, + ) -> ::c_int; + pub fn getloadavg( + loadavg: *mut ::c_double, + nelem: ::c_int + ) -> ::c_int; + pub fn process_vm_readv( + pid: ::pid_t, + local_iov: *const ::iovec, + liovcnt: ::c_ulong, + remote_iov: *const ::iovec, + riovcnt: ::c_ulong, + flags: ::c_ulong, + ) -> isize; + pub fn process_vm_writev( + pid: ::pid_t, + local_iov: *const ::iovec, + liovcnt: ::c_ulong, + remote_iov: *const ::iovec, + riovcnt: ::c_ulong, + flags: ::c_ulong, + ) -> isize; + pub fn futimes( + fd: ::c_int, + times: *const ::timeval + ) -> ::c_int; + } + } } extern "C" { #[cfg_attr(not(target_env = "musl"), link_name = "__xpg_strerror_r")] - pub fn strerror_r( - errnum: ::c_int, - buf: *mut c_char, - buflen: ::size_t, - ) -> ::c_int; + pub fn strerror_r(errnum: ::c_int, buf: *mut c_char, buflen: ::size_t) -> ::c_int; pub fn abs(i: ::c_int) -> ::c_int; pub fn atof(s: *const ::c_char) -> ::c_double; @@ -2568,23 +3805,15 @@ extern "C" { pub fn rand() -> ::c_int; pub fn srand(seed: ::c_uint); - pub fn aio_read(aiocbp: *mut aiocb) -> ::c_int; - pub fn aio_write(aiocbp: *mut aiocb) -> ::c_int; - pub fn aio_fsync(op: ::c_int, aiocbp: *mut aiocb) -> ::c_int; - pub fn aio_error(aiocbp: *const aiocb) -> ::c_int; - pub fn aio_return(aiocbp: *mut aiocb) -> ::ssize_t; - pub fn aio_suspend( - aiocb_list: *const *const aiocb, - nitems: ::c_int, - timeout: *const ::timespec, - ) -> ::c_int; - pub fn aio_cancel(fd: ::c_int, aiocbp: *mut aiocb) -> ::c_int; - pub fn lio_listio( - mode: ::c_int, - aiocb_list: *const *mut aiocb, - nitems: ::c_int, - sevp: *mut ::sigevent, - ) -> ::c_int; + pub fn drand48() -> ::c_double; + pub fn erand48(xseed: *mut ::c_ushort) -> ::c_double; + pub fn lrand48() -> ::c_long; + pub fn nrand48(xseed: *mut ::c_ushort) -> ::c_long; + pub fn mrand48() -> ::c_long; + pub fn jrand48(xseed: *mut ::c_ushort) -> ::c_long; + pub fn srand48(seed: ::c_long); + pub fn seed48(xseed: *mut ::c_ushort) -> *mut ::c_ushort; + pub fn lcong48(p: *mut ::c_ushort); pub fn lutimes(file: *const ::c_char, times: *const ::timeval) -> ::c_int; @@ -2610,40 +3839,18 @@ extern "C" { spbufp: *mut *mut spwd, ) -> ::c_int; - pub fn shm_open( - name: *const c_char, - oflag: ::c_int, - mode: mode_t, - ) -> ::c_int; + pub fn shm_open(name: *const c_char, oflag: ::c_int, mode: mode_t) -> ::c_int; // System V IPC pub fn shmget(key: ::key_t, size: ::size_t, shmflg: ::c_int) -> ::c_int; - pub fn shmat( - shmid: ::c_int, - shmaddr: *const ::c_void, - shmflg: ::c_int, - ) -> *mut ::c_void; + pub fn shmat(shmid: ::c_int, shmaddr: *const ::c_void, shmflg: ::c_int) -> *mut ::c_void; pub fn shmdt(shmaddr: *const ::c_void) -> ::c_int; - pub fn shmctl( - shmid: ::c_int, - cmd: ::c_int, - buf: *mut ::shmid_ds, - ) -> ::c_int; + pub fn shmctl(shmid: ::c_int, cmd: ::c_int, buf: *mut ::shmid_ds) -> ::c_int; pub fn ftok(pathname: *const ::c_char, proj_id: ::c_int) -> ::key_t; pub fn semget(key: ::key_t, nsems: ::c_int, semflag: ::c_int) -> ::c_int; - pub fn semop( - semid: ::c_int, - sops: *mut ::sembuf, - nsops: ::size_t, - ) -> ::c_int; - pub fn semctl( - semid: ::c_int, - semnum: ::c_int, - cmd: ::c_int, - ... - ) -> ::c_int; - pub fn msgctl(msqid: ::c_int, cmd: ::c_int, buf: *mut msqid_ds) - -> ::c_int; + pub fn semop(semid: ::c_int, sops: *mut ::sembuf, nsops: ::size_t) -> ::c_int; + pub fn semctl(semid: ::c_int, semnum: ::c_int, cmd: ::c_int, ...) -> ::c_int; + pub fn msgctl(msqid: ::c_int, cmd: ::c_int, buf: *mut msqid_ds) -> ::c_int; pub fn msgget(key: ::key_t, msgflg: ::c_int) -> ::c_int; pub fn msgrcv( msqid: ::c_int, @@ -2659,17 +3866,10 @@ extern "C" { msgflg: ::c_int, ) -> ::c_int; - pub fn mprotect( - addr: *mut ::c_void, - len: ::size_t, - prot: ::c_int, - ) -> ::c_int; + pub fn mprotect(addr: *mut ::c_void, len: ::size_t, prot: ::c_int) -> ::c_int; pub fn __errno_location() -> *mut ::c_int; - pub fn fopen64( - filename: *const c_char, - mode: *const c_char, - ) -> *mut ::FILE; + pub fn fopen64(filename: *const c_char, mode: *const c_char) -> *mut ::FILE; pub fn freopen64( filename: *const c_char, mode: *const c_char, @@ -2678,39 +3878,13 @@ extern "C" { pub fn tmpfile64() -> *mut ::FILE; pub fn fgetpos64(stream: *mut ::FILE, ptr: *mut fpos64_t) -> ::c_int; pub fn fsetpos64(stream: *mut ::FILE, ptr: *const fpos64_t) -> ::c_int; - pub fn fseeko64( - stream: *mut ::FILE, - offset: ::off64_t, - whence: ::c_int, - ) -> ::c_int; + pub fn fseeko64(stream: *mut ::FILE, offset: ::off64_t, whence: ::c_int) -> ::c_int; pub fn ftello64(stream: *mut ::FILE) -> ::off64_t; - pub fn fallocate( - fd: ::c_int, - mode: ::c_int, - offset: ::off_t, - len: ::off_t, - ) -> ::c_int; - pub fn fallocate64( - fd: ::c_int, - mode: ::c_int, - offset: ::off64_t, - len: ::off64_t, - ) -> ::c_int; - pub fn posix_fallocate( - fd: ::c_int, - offset: ::off_t, - len: ::off_t, - ) -> ::c_int; - pub fn posix_fallocate64( - fd: ::c_int, - offset: ::off64_t, - len: ::off64_t, - ) -> ::c_int; - pub fn readahead( - fd: ::c_int, - offset: ::off64_t, - count: ::size_t, - ) -> ::ssize_t; + pub fn fallocate(fd: ::c_int, mode: ::c_int, offset: ::off_t, len: ::off_t) -> ::c_int; + pub fn fallocate64(fd: ::c_int, mode: ::c_int, offset: ::off64_t, len: ::off64_t) -> ::c_int; + pub fn posix_fallocate(fd: ::c_int, offset: ::off_t, len: ::off_t) -> ::c_int; + pub fn posix_fallocate64(fd: ::c_int, offset: ::off64_t, len: ::off64_t) -> ::c_int; + pub fn readahead(fd: ::c_int, offset: ::off64_t, count: ::size_t) -> ::ssize_t; pub fn getxattr( path: *const c_char, name: *const c_char, @@ -2750,52 +3924,21 @@ extern "C" { size: ::size_t, flags: ::c_int, ) -> ::c_int; - pub fn listxattr( - path: *const c_char, - list: *mut c_char, - size: ::size_t, - ) -> ::ssize_t; - pub fn llistxattr( - path: *const c_char, - list: *mut c_char, - size: ::size_t, - ) -> ::ssize_t; - pub fn flistxattr( - filedes: ::c_int, - list: *mut c_char, - size: ::size_t, - ) -> ::ssize_t; + pub fn listxattr(path: *const c_char, list: *mut c_char, size: ::size_t) -> ::ssize_t; + pub fn llistxattr(path: *const c_char, list: *mut c_char, size: ::size_t) -> ::ssize_t; + pub fn flistxattr(filedes: ::c_int, list: *mut c_char, size: ::size_t) -> ::ssize_t; pub fn removexattr(path: *const c_char, name: *const c_char) -> ::c_int; pub fn lremovexattr(path: *const c_char, name: *const c_char) -> ::c_int; pub fn fremovexattr(filedes: ::c_int, name: *const c_char) -> ::c_int; - pub fn signalfd( - fd: ::c_int, - mask: *const ::sigset_t, - flags: ::c_int, - ) -> ::c_int; - pub fn timerfd_create(clockid: ::c_int, flags: ::c_int) -> ::c_int; - pub fn timerfd_gettime( - fd: ::c_int, - curr_value: *mut itimerspec, - ) -> ::c_int; + pub fn signalfd(fd: ::c_int, mask: *const ::sigset_t, flags: ::c_int) -> ::c_int; + pub fn timerfd_create(clockid: ::clockid_t, flags: ::c_int) -> ::c_int; + pub fn timerfd_gettime(fd: ::c_int, curr_value: *mut itimerspec) -> ::c_int; pub fn timerfd_settime( fd: ::c_int, flags: ::c_int, new_value: *const itimerspec, old_value: *mut itimerspec, ) -> ::c_int; - pub fn pwritev( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - offset: ::off_t, - ) -> ::ssize_t; - pub fn preadv( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - offset: ::off_t, - ) -> ::ssize_t; pub fn quotactl( cmd: ::c_int, special: *const ::c_char, @@ -2832,11 +3975,7 @@ extern "C" { abs_timeout: *const ::timespec, ) -> ::c_int; pub fn mq_getattr(mqd: ::mqd_t, attr: *mut ::mq_attr) -> ::c_int; - pub fn mq_setattr( - mqd: ::mqd_t, - newattr: *const ::mq_attr, - oldattr: *mut ::mq_attr, - ) -> ::c_int; + pub fn mq_setattr(mqd: ::mqd_t, newattr: *const ::mq_attr, oldattr: *mut ::mq_attr) -> ::c_int; pub fn epoll_pwait( epfd: ::c_int, events: *mut ::epoll_event, @@ -2846,59 +3985,37 @@ extern "C" { ) -> ::c_int; pub fn dup3(oldfd: ::c_int, newfd: ::c_int, flags: ::c_int) -> ::c_int; pub fn mkostemp(template: *mut ::c_char, flags: ::c_int) -> ::c_int; - pub fn mkostemps( - template: *mut ::c_char, - suffixlen: ::c_int, - flags: ::c_int, - ) -> ::c_int; + pub fn mkostemps(template: *mut ::c_char, suffixlen: ::c_int, flags: ::c_int) -> ::c_int; pub fn sigtimedwait( set: *const sigset_t, info: *mut siginfo_t, timeout: *const ::timespec, ) -> ::c_int; pub fn sigwaitinfo(set: *const sigset_t, info: *mut siginfo_t) -> ::c_int; - pub fn nl_langinfo_l(item: ::nl_item, locale: ::locale_t) - -> *mut ::c_char; - pub fn getnameinfo( - sa: *const ::sockaddr, - salen: ::socklen_t, - host: *mut ::c_char, - hostlen: ::socklen_t, - serv: *mut ::c_char, - sevlen: ::socklen_t, - flags: ::c_int, + pub fn nl_langinfo_l(item: ::nl_item, locale: ::locale_t) -> *mut ::c_char; + pub fn accept4( + fd: ::c_int, + addr: *mut ::sockaddr, + len: *mut ::socklen_t, + flg: ::c_int, ) -> ::c_int; - pub fn pthread_setschedprio( - native: ::pthread_t, - priority: ::c_int, + pub fn pthread_getaffinity_np( + thread: ::pthread_t, + cpusetsize: ::size_t, + cpuset: *mut ::cpu_set_t, ) -> ::c_int; - pub fn getloadavg(loadavg: *mut ::c_double, nelem: ::c_int) -> ::c_int; - pub fn process_vm_readv( - pid: ::pid_t, - local_iov: *const ::iovec, - liovcnt: ::c_ulong, - remote_iov: *const ::iovec, - riovcnt: ::c_ulong, - flags: ::c_ulong, - ) -> isize; - pub fn process_vm_writev( - pid: ::pid_t, - local_iov: *const ::iovec, - liovcnt: ::c_ulong, - remote_iov: *const ::iovec, - riovcnt: ::c_ulong, - flags: ::c_ulong, - ) -> isize; + pub fn pthread_setaffinity_np( + thread: ::pthread_t, + cpusetsize: ::size_t, + cpuset: *const ::cpu_set_t, + ) -> ::c_int; + pub fn pthread_setschedprio(native: ::pthread_t, priority: ::c_int) -> ::c_int; pub fn reboot(how_to: ::c_int) -> ::c_int; pub fn setfsgid(gid: ::gid_t) -> ::c_int; pub fn setfsuid(uid: ::uid_t) -> ::c_int; // Not available now on Android - pub fn mkfifoat( - dirfd: ::c_int, - pathname: *const ::c_char, - mode: ::mode_t, - ) -> ::c_int; + pub fn mkfifoat(dirfd: ::c_int, pathname: *const ::c_char, mode: ::mode_t) -> ::c_int; pub fn if_nameindex() -> *mut if_nameindex; pub fn if_freenameindex(ptr: *mut if_nameindex); pub fn sync_file_range( @@ -2918,35 +4035,21 @@ extern "C" { pub fn glob( pattern: *const c_char, flags: ::c_int, - errfunc: ::Option< - extern "C" fn(epath: *const c_char, errno: ::c_int) -> ::c_int, - >, + errfunc: ::Option ::c_int>, pglob: *mut ::glob_t, ) -> ::c_int; pub fn globfree(pglob: *mut ::glob_t); - pub fn posix_madvise( - addr: *mut ::c_void, - len: ::size_t, - advice: ::c_int, - ) -> ::c_int; + pub fn posix_madvise(addr: *mut ::c_void, len: ::size_t, advice: ::c_int) -> ::c_int; pub fn shm_unlink(name: *const ::c_char) -> ::c_int; pub fn seekdir(dirp: *mut ::DIR, loc: ::c_long); pub fn telldir(dirp: *mut ::DIR) -> ::c_long; - pub fn madvise( - addr: *mut ::c_void, - len: ::size_t, - advice: ::c_int, - ) -> ::c_int; + pub fn madvise(addr: *mut ::c_void, len: ::size_t, advice: ::c_int) -> ::c_int; - pub fn msync( - addr: *mut ::c_void, - len: ::size_t, - flags: ::c_int, - ) -> ::c_int; + pub fn msync(addr: *mut ::c_void, len: ::size_t, flags: ::c_int) -> ::c_int; pub fn remap_file_pages( addr: *mut ::c_void, size: ::size_t, @@ -2963,19 +4066,17 @@ extern "C" { addrlen: *mut ::socklen_t, ) -> ::ssize_t; pub fn mkstemps(template: *mut ::c_char, suffixlen: ::c_int) -> ::c_int; - pub fn futimes(fd: ::c_int, times: *const ::timeval) -> ::c_int; + pub fn nl_langinfo(item: ::nl_item) -> *mut ::c_char; pub fn getdomainname(name: *mut ::c_char, len: ::size_t) -> ::c_int; pub fn setdomainname(name: *const ::c_char, len: ::size_t) -> ::c_int; pub fn vhangup() -> ::c_int; pub fn sync(); + pub fn syncfs(fd: ::c_int) -> ::c_int; pub fn syscall(num: ::c_long, ...) -> ::c_long; - pub fn sched_getaffinity( - pid: ::pid_t, - cpusetsize: ::size_t, - cpuset: *mut cpu_set_t, - ) -> ::c_int; + pub fn sched_getaffinity(pid: ::pid_t, cpusetsize: ::size_t, cpuset: *mut cpu_set_t) + -> ::c_int; pub fn sched_setaffinity( pid: ::pid_t, cpusetsize: ::size_t, @@ -2989,12 +4090,8 @@ extern "C" { maxevents: ::c_int, timeout: ::c_int, ) -> ::c_int; - pub fn epoll_ctl( - epfd: ::c_int, - op: ::c_int, - fd: ::c_int, - event: *mut ::epoll_event, - ) -> ::c_int; + pub fn epoll_ctl(epfd: ::c_int, op: ::c_int, fd: ::c_int, event: *mut ::epoll_event) + -> ::c_int; pub fn pthread_getschedparam( native: ::pthread_t, policy: *mut ::c_int, @@ -3003,16 +4100,8 @@ extern "C" { pub fn unshare(flags: ::c_int) -> ::c_int; pub fn umount(target: *const ::c_char) -> ::c_int; pub fn sched_get_priority_max(policy: ::c_int) -> ::c_int; - pub fn tee( - fd_in: ::c_int, - fd_out: ::c_int, - len: ::size_t, - flags: ::c_uint, - ) -> ::ssize_t; - pub fn settimeofday( - tv: *const ::timeval, - tz: *const ::timezone, - ) -> ::c_int; + pub fn tee(fd_in: ::c_int, fd_out: ::c_int, len: ::size_t, flags: ::c_uint) -> ::ssize_t; + pub fn settimeofday(tv: *const ::timeval, tz: *const ::timezone) -> ::c_int; pub fn splice( fd_in: ::c_int, off_in: *mut ::loff_t, @@ -3022,19 +4111,12 @@ extern "C" { flags: ::c_uint, ) -> ::ssize_t; pub fn eventfd(init: ::c_uint, flags: ::c_int) -> ::c_int; - pub fn sched_rr_get_interval(pid: ::pid_t, tp: *mut ::timespec) - -> ::c_int; - pub fn sem_timedwait( - sem: *mut sem_t, - abstime: *const ::timespec, - ) -> ::c_int; + pub fn sched_rr_get_interval(pid: ::pid_t, tp: *mut ::timespec) -> ::c_int; + pub fn sem_timedwait(sem: *mut sem_t, abstime: *const ::timespec) -> ::c_int; pub fn sem_getvalue(sem: *mut sem_t, sval: *mut ::c_int) -> ::c_int; - pub fn sched_setparam( - pid: ::pid_t, - param: *const ::sched_param, - ) -> ::c_int; + pub fn sched_setparam(pid: ::pid_t, param: *const ::sched_param) -> ::c_int; pub fn setns(fd: ::c_int, nstype: ::c_int) -> ::c_int; - pub fn swapoff(puath: *const ::c_char) -> ::c_int; + pub fn swapoff(path: *const ::c_char) -> ::c_int; pub fn vmsplice( fd: ::c_int, iov: *const ::iovec, @@ -3057,10 +4139,24 @@ extern "C" { timeout: *const ::timespec, sigmask: *const sigset_t, ) -> ::c_int; + pub fn pthread_mutexattr_getprotocol( + attr: *const pthread_mutexattr_t, + protocol: *mut ::c_int, + ) -> ::c_int; + pub fn pthread_mutexattr_setprotocol( + attr: *mut pthread_mutexattr_t, + protocol: ::c_int, + ) -> ::c_int; + pub fn pthread_mutex_consistent(mutex: *mut pthread_mutex_t) -> ::c_int; pub fn pthread_mutex_timedlock( lock: *mut pthread_mutex_t, abstime: *const ::timespec, ) -> ::c_int; + pub fn pthread_spin_init(lock: *mut ::pthread_spinlock_t, pshared: ::c_int) -> ::c_int; + pub fn pthread_spin_destroy(lock: *mut ::pthread_spinlock_t) -> ::c_int; + pub fn pthread_spin_lock(lock: *mut ::pthread_spinlock_t) -> ::c_int; + pub fn pthread_spin_trylock(lock: *mut ::pthread_spinlock_t) -> ::c_int; + pub fn pthread_spin_unlock(lock: *mut ::pthread_spinlock_t) -> ::c_int; pub fn clone( cb: extern "C" fn(*mut ::c_void) -> ::c_int, child_stack: *mut ::c_void, @@ -3104,8 +4200,13 @@ extern "C" { offset: *mut off_t, count: ::size_t, ) -> ::ssize_t; + pub fn sendfile64( + out_fd: ::c_int, + in_fd: ::c_int, + offset: *mut off64_t, + count: ::size_t, + ) -> ::ssize_t; pub fn sigsuspend(mask: *const ::sigset_t) -> ::c_int; - #[cfg_attr(target_os = "solaris", link_name = "__posix_getgrgid_r")] pub fn getgrgid_r( gid: ::gid_t, grp: *mut ::group, @@ -3113,15 +4214,9 @@ extern "C" { buflen: ::size_t, result: *mut *mut ::group, ) -> ::c_int; - #[cfg_attr( - all(target_os = "macos", target_arch = "x86"), - link_name = "sigaltstack$UNIX2003" - )] - #[cfg_attr(target_os = "netbsd", link_name = "__sigaltstack14")] pub fn sigaltstack(ss: *const stack_t, oss: *mut stack_t) -> ::c_int; pub fn sem_close(sem: *mut sem_t) -> ::c_int; pub fn getdtablesize() -> ::c_int; - #[cfg_attr(target_os = "solaris", link_name = "__posix_getgrnam_r")] pub fn getgrnam_r( name: *const ::c_char, grp: *mut ::group, @@ -3130,23 +4225,13 @@ extern "C" { result: *mut *mut ::group, ) -> ::c_int; pub fn initgroups(user: *const ::c_char, group: ::gid_t) -> ::c_int; - #[cfg_attr( - all(target_os = "macos", target_arch = "x86"), - link_name = "pthread_sigmask$UNIX2003" - )] - pub fn pthread_sigmask( - how: ::c_int, - set: *const sigset_t, - oldset: *mut sigset_t, - ) -> ::c_int; + pub fn pthread_sigmask(how: ::c_int, set: *const sigset_t, oldset: *mut sigset_t) -> ::c_int; pub fn sem_open(name: *const ::c_char, oflag: ::c_int, ...) -> *mut sem_t; pub fn getgrnam(name: *const ::c_char) -> *mut ::group; pub fn pthread_cancel(thread: ::pthread_t) -> ::c_int; pub fn pthread_kill(thread: ::pthread_t, sig: ::c_int) -> ::c_int; pub fn sem_unlink(name: *const ::c_char) -> ::c_int; pub fn daemon(nochdir: ::c_int, noclose: ::c_int) -> ::c_int; - #[cfg_attr(target_os = "netbsd", link_name = "__getpwnam_r50")] - #[cfg_attr(target_os = "solaris", link_name = "__posix_getpwnam_r")] pub fn getpwnam_r( name: *const ::c_char, pwd: *mut passwd, @@ -3154,8 +4239,6 @@ extern "C" { buflen: ::size_t, result: *mut *mut passwd, ) -> ::c_int; - #[cfg_attr(target_os = "netbsd", link_name = "__getpwuid_r50")] - #[cfg_attr(target_os = "solaris", link_name = "__posix_getpwuid_r")] pub fn getpwuid_r( uid: ::uid_t, pwd: *mut passwd, @@ -3163,11 +4246,6 @@ extern "C" { buflen: ::size_t, result: *mut *mut passwd, ) -> ::c_int; - #[cfg_attr( - all(target_os = "macos", target_arch = "x86"), - link_name = "sigwait$UNIX2003" - )] - #[cfg_attr(target_os = "solaris", link_name = "__posix_sigwait")] pub fn sigwait(set: *const sigset_t, sig: *mut ::c_int) -> ::c_int; pub fn pthread_atfork( prepare: ::Option, @@ -3185,10 +4263,14 @@ extern "C" { attr: *const pthread_mutexattr_t, pshared: *mut ::c_int, ) -> ::c_int; - #[cfg_attr( - all(target_os = "macos", target_arch = "x86"), - link_name = "popen$UNIX2003" - )] + pub fn pthread_mutexattr_getrobust( + attr: *const pthread_mutexattr_t, + robustness: *mut ::c_int, + ) -> ::c_int; + pub fn pthread_mutexattr_setrobust( + attr: *mut pthread_mutexattr_t, + robustness: ::c_int, + ) -> ::c_int; pub fn popen(command: *const c_char, mode: *const c_char) -> *mut ::FILE; pub fn faccessat( dirfd: ::c_int, @@ -3213,17 +4295,11 @@ extern "C" { data: *mut ::c_void, ) -> ::c_int; - pub fn setmntent( - filename: *const ::c_char, - ty: *const ::c_char, - ) -> *mut ::FILE; + pub fn setmntent(filename: *const ::c_char, ty: *const ::c_char) -> *mut ::FILE; pub fn getmntent(stream: *mut ::FILE) -> *mut ::mntent; pub fn addmntent(stream: *mut ::FILE, mnt: *const ::mntent) -> ::c_int; pub fn endmntent(streamp: *mut ::FILE) -> ::c_int; - pub fn hasmntopt( - mnt: *const ::mntent, - opt: *const ::c_char, - ) -> *mut ::c_char; + pub fn hasmntopt(mnt: *const ::mntent, opt: *const ::c_char) -> *mut ::c_char; pub fn posix_spawn( pid: *mut ::pid_t, @@ -3263,26 +4339,17 @@ extern "C" { attr: *const posix_spawnattr_t, flags: *mut ::c_short, ) -> ::c_int; - pub fn posix_spawnattr_setflags( - attr: *mut posix_spawnattr_t, - flags: ::c_short, - ) -> ::c_int; + pub fn posix_spawnattr_setflags(attr: *mut posix_spawnattr_t, flags: ::c_short) -> ::c_int; pub fn posix_spawnattr_getpgroup( attr: *const posix_spawnattr_t, flags: *mut ::pid_t, ) -> ::c_int; - pub fn posix_spawnattr_setpgroup( - attr: *mut posix_spawnattr_t, - flags: ::pid_t, - ) -> ::c_int; + pub fn posix_spawnattr_setpgroup(attr: *mut posix_spawnattr_t, flags: ::pid_t) -> ::c_int; pub fn posix_spawnattr_getschedpolicy( attr: *const posix_spawnattr_t, flags: *mut ::c_int, ) -> ::c_int; - pub fn posix_spawnattr_setschedpolicy( - attr: *mut posix_spawnattr_t, - flags: ::c_int, - ) -> ::c_int; + pub fn posix_spawnattr_setschedpolicy(attr: *mut posix_spawnattr_t, flags: ::c_int) -> ::c_int; pub fn posix_spawnattr_getschedparam( attr: *const posix_spawnattr_t, param: *mut ::sched_param, @@ -3292,12 +4359,8 @@ extern "C" { param: *const ::sched_param, ) -> ::c_int; - pub fn posix_spawn_file_actions_init( - actions: *mut posix_spawn_file_actions_t, - ) -> ::c_int; - pub fn posix_spawn_file_actions_destroy( - actions: *mut posix_spawn_file_actions_t, - ) -> ::c_int; + pub fn posix_spawn_file_actions_init(actions: *mut posix_spawn_file_actions_t) -> ::c_int; + pub fn posix_spawn_file_actions_destroy(actions: *mut posix_spawn_file_actions_t) -> ::c_int; pub fn posix_spawn_file_actions_addopen( actions: *mut posix_spawn_file_actions_t, fd: ::c_int, @@ -3323,15 +4386,82 @@ extern "C" { pub fn inotify_rm_watch(fd: ::c_int, wd: ::c_int) -> ::c_int; pub fn inotify_init() -> ::c_int; pub fn inotify_init1(flags: ::c_int) -> ::c_int; - pub fn inotify_add_watch( - fd: ::c_int, - path: *const ::c_char, - mask: u32, + pub fn inotify_add_watch(fd: ::c_int, path: *const ::c_char, mask: u32) -> ::c_int; + pub fn fanotify_init(flags: ::c_uint, event_f_flags: ::c_uint) -> ::c_int; + + pub fn regcomp(preg: *mut ::regex_t, pattern: *const ::c_char, cflags: ::c_int) -> ::c_int; + + pub fn regexec( + preg: *const ::regex_t, + input: *const ::c_char, + nmatch: ::size_t, + pmatch: *mut regmatch_t, + eflags: ::c_int, + ) -> ::c_int; + + pub fn regerror( + errcode: ::c_int, + preg: *const ::regex_t, + errbuf: *mut ::c_char, + errbuf_size: ::size_t, + ) -> ::size_t; + + pub fn regfree(preg: *mut ::regex_t); + + pub fn iconv_open(tocode: *const ::c_char, fromcode: *const ::c_char) -> iconv_t; + pub fn iconv( + cd: iconv_t, + inbuf: *mut *mut ::c_char, + inbytesleft: *mut ::size_t, + outbuf: *mut *mut ::c_char, + outbytesleft: *mut ::size_t, + ) -> ::size_t; + pub fn iconv_close(cd: iconv_t) -> ::c_int; + + pub fn gettid() -> ::pid_t; + + pub fn timer_create( + clockid: ::clockid_t, + sevp: *mut ::sigevent, + timerid: *mut ::timer_t, + ) -> ::c_int; + pub fn timer_delete(timerid: ::timer_t) -> ::c_int; + pub fn timer_getoverrun(timerid: ::timer_t) -> ::c_int; + pub fn timer_gettime(timerid: ::timer_t, curr_value: *mut ::itimerspec) -> ::c_int; + pub fn timer_settime( + timerid: ::timer_t, + flags: ::c_int, + new_value: *const ::itimerspec, + old_value: *mut ::itimerspec, + ) -> ::c_int; + + pub fn gethostid() -> ::c_long; + + pub fn pthread_getcpuclockid(thread: ::pthread_t, clk_id: *mut ::clockid_t) -> ::c_int; + pub fn memmem( + haystack: *const ::c_void, + haystacklen: ::size_t, + needle: *const ::c_void, + needlelen: ::size_t, + ) -> *mut ::c_void; + pub fn sched_getcpu() -> ::c_int; + + pub fn pthread_getname_np(thread: ::pthread_t, name: *mut ::c_char, len: ::size_t) -> ::c_int; + pub fn pthread_setname_np(thread: ::pthread_t, name: *const ::c_char) -> ::c_int; + pub fn getopt_long( + argc: ::c_int, + argv: *const *mut c_char, + optstring: *const c_char, + longopts: *const option, + longindex: *mut ::c_int, ) -> ::c_int; } cfg_if! { - if #[cfg(target_env = "musl")] { + if #[cfg(target_env = "uclibc")] { + mod uclibc; + pub use self::uclibc::*; + } else if #[cfg(target_env = "musl")] { mod musl; pub use self::musl::*; } else if #[cfg(target_env = "gnu")] { @@ -3340,6 +4470,9 @@ cfg_if! { } } +mod arch; +pub use self::arch::*; + cfg_if! { if #[cfg(libc_align)] { #[macro_use] @@ -3350,3 +4483,10 @@ cfg_if! { } } expand_align!(); + +cfg_if! { + if #[cfg(libc_non_exhaustive)] { + mod non_exhaustive; + pub use self::non_exhaustive::*; + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/arm/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/arm/mod.rs index 51237a2cb..c47fa2c4c 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/arm/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/arm/mod.rs @@ -151,22 +151,75 @@ s! { __f_spare: [::c_int; 6], } - pub struct nlmsghdr { - pub nlmsg_len: u32, - pub nlmsg_type: u16, - pub nlmsg_flags: u16, - pub nlmsg_seq: u32, - pub nlmsg_pid: u32, + pub struct mcontext_t { + pub trap_no: ::c_ulong, + pub error_code: ::c_ulong, + pub oldmask: ::c_ulong, + pub arm_r0: ::c_ulong, + pub arm_r1: ::c_ulong, + pub arm_r2: ::c_ulong, + pub arm_r3: ::c_ulong, + pub arm_r4: ::c_ulong, + pub arm_r5: ::c_ulong, + pub arm_r6: ::c_ulong, + pub arm_r7: ::c_ulong, + pub arm_r8: ::c_ulong, + pub arm_r9: ::c_ulong, + pub arm_r10: ::c_ulong, + pub arm_fp: ::c_ulong, + pub arm_ip: ::c_ulong, + pub arm_sp: ::c_ulong, + pub arm_lr: ::c_ulong, + pub arm_pc: ::c_ulong, + pub arm_cpsr: ::c_ulong, + pub fault_address: ::c_ulong, } +} - pub struct nlmsgerr { - pub error: ::c_int, - pub msg: nlmsghdr, +s_no_extra_traits! { + #[allow(missing_debug_implementations)] + pub struct ucontext_t { + pub uc_flags: ::c_ulong, + pub uc_link: *mut ucontext_t, + pub uc_stack: ::stack_t, + pub uc_mcontext: mcontext_t, + pub uc_sigmask: ::sigset_t, + pub uc_regspace: [::c_ulonglong; 64], } +} - pub struct nlattr { - pub nla_len: u16, - pub nla_type: u16, +cfg_if! { + if #[cfg(feature = "extra_traits")] { + impl PartialEq for ucontext_t { + fn eq(&self, other: &ucontext_t) -> bool { + self.uc_flags == other.uc_flags + && self.uc_link == other.uc_link + && self.uc_stack == other.uc_stack + && self.uc_mcontext == other.uc_mcontext + && self.uc_sigmask == other.uc_sigmask + } + } + impl Eq for ucontext_t {} + impl ::fmt::Debug for ucontext_t { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("ucontext_t") + .field("uc_flags", &self.uc_link) + .field("uc_link", &self.uc_link) + .field("uc_stack", &self.uc_stack) + .field("uc_mcontext", &self.uc_mcontext) + .field("uc_sigmask", &self.uc_sigmask) + .finish() + } + } + impl ::hash::Hash for ucontext_t { + fn hash(&self, state: &mut H) { + self.uc_flags.hash(state); + self.uc_link.hash(state); + self.uc_stack.hash(state); + self.uc_mcontext.hash(state); + self.uc_sigmask.hash(state); + } + } } } @@ -179,17 +232,6 @@ pub const O_NOFOLLOW: ::c_int = 0x8000; pub const O_ASYNC: ::c_int = 0x2000; pub const O_LARGEFILE: ::c_int = 0o400000; -pub const FIOCLEX: ::c_int = 0x5451; -pub const FIONCLEX: ::c_int = 0x5450; -pub const FIONBIO: ::c_int = 0x5421; - -pub const RLIMIT_RSS: ::c_int = 5; -pub const RLIMIT_NOFILE: ::c_int = 7; -pub const RLIMIT_AS: ::c_int = 9; -pub const RLIMIT_NPROC: ::c_int = 6; -pub const RLIMIT_MEMLOCK: ::c_int = 8; -pub const RLIMIT_NLIMITS: ::c_int = 15; - pub const MADV_SOFT_OFFLINE: ::c_int = 101; pub const MCL_CURRENT: ::c_int = 0x0001; pub const MCL_FUTURE: ::c_int = 0x0002; @@ -280,14 +322,12 @@ pub const MAP_NORESERVE: ::c_int = 0x04000; pub const MAP_POPULATE: ::c_int = 0x08000; pub const MAP_NONBLOCK: ::c_int = 0x010000; pub const MAP_STACK: ::c_int = 0x020000; -pub const MAP_SYNC : ::c_int = 0x080000; +pub const MAP_SYNC: ::c_int = 0x080000; pub const SOCK_STREAM: ::c_int = 1; pub const SOCK_DGRAM: ::c_int = 2; pub const SOCK_SEQPACKET: ::c_int = 5; -pub const SOL_SOCKET: ::c_int = 1; - pub const EDEADLK: ::c_int = 35; pub const ENAMETOOLONG: ::c_int = 36; pub const ENOLCK: ::c_int = 37; @@ -374,32 +414,6 @@ pub const ENOTRECOVERABLE: ::c_int = 131; pub const ERFKILL: ::c_int = 132; pub const EHWPOISON: ::c_int = 133; -pub const SO_REUSEADDR: ::c_int = 2; -pub const SO_TYPE: ::c_int = 3; -pub const SO_ERROR: ::c_int = 4; -pub const SO_DONTROUTE: ::c_int = 5; -pub const SO_BROADCAST: ::c_int = 6; -pub const SO_SNDBUF: ::c_int = 7; -pub const SO_RCVBUF: ::c_int = 8; -pub const SO_KEEPALIVE: ::c_int = 9; -pub const SO_OOBINLINE: ::c_int = 10; -pub const SO_NO_CHECK: ::c_int = 11; -pub const SO_PRIORITY: ::c_int = 12; -pub const SO_LINGER: ::c_int = 13; -pub const SO_BSDCOMPAT: ::c_int = 14; -pub const SO_REUSEPORT: ::c_int = 15; -pub const SO_PASSCRED: ::c_int = 16; -pub const SO_PEERCRED: ::c_int = 17; -pub const SO_RCVLOWAT: ::c_int = 18; -pub const SO_SNDLOWAT: ::c_int = 19; -pub const SO_RCVTIMEO: ::c_int = 20; -pub const SO_SNDTIMEO: ::c_int = 21; -pub const SO_ACCEPTCONN: ::c_int = 30; -pub const SO_SNDBUFFORCE: ::c_int = 32; -pub const SO_RCVBUFFORCE: ::c_int = 33; -pub const SO_PROTOCOL: ::c_int = 38; -pub const SO_DOMAIN: ::c_int = 39; - pub const SA_ONSTACK: ::c_int = 0x08000000; pub const SA_SIGINFO: ::c_int = 0x00000004; pub const SA_NOCLDWAIT: ::c_int = 0x00000002; @@ -449,55 +463,9 @@ pub const IEXTEN: ::tcflag_t = 0x00008000; pub const TOSTOP: ::tcflag_t = 0x00000100; pub const FLUSHO: ::tcflag_t = 0x00001000; -pub const TCGETS: ::c_int = 0x5401; -pub const TCSETS: ::c_int = 0x5402; -pub const TCSETSW: ::c_int = 0x5403; -pub const TCSETSF: ::c_int = 0x5404; -pub const TCGETA: ::c_int = 0x5405; -pub const TCSETA: ::c_int = 0x5406; -pub const TCSETAW: ::c_int = 0x5407; -pub const TCSETAF: ::c_int = 0x5408; -pub const TCSBRK: ::c_int = 0x5409; -pub const TCXONC: ::c_int = 0x540A; -pub const TCFLSH: ::c_int = 0x540B; -pub const TIOCGSOFTCAR: ::c_int = 0x5419; -pub const TIOCSSOFTCAR: ::c_int = 0x541A; -pub const TIOCLINUX: ::c_int = 0x541C; -pub const TIOCGSERIAL: ::c_int = 0x541E; -pub const TIOCEXCL: ::c_int = 0x540C; -pub const TIOCNXCL: ::c_int = 0x540D; -pub const TIOCSCTTY: ::c_int = 0x540E; -pub const TIOCGPGRP: ::c_int = 0x540F; -pub const TIOCSPGRP: ::c_int = 0x5410; -pub const TIOCOUTQ: ::c_int = 0x5411; -pub const TIOCSTI: ::c_int = 0x5412; -pub const TIOCGWINSZ: ::c_int = 0x5413; -pub const TIOCSWINSZ: ::c_int = 0x5414; -pub const TIOCMGET: ::c_int = 0x5415; -pub const TIOCMBIS: ::c_int = 0x5416; -pub const TIOCMBIC: ::c_int = 0x5417; -pub const TIOCMSET: ::c_int = 0x5418; -pub const FIONREAD: ::c_int = 0x541B; -pub const TIOCCONS: ::c_int = 0x541D; - -pub const TIOCGRS485: ::c_int = 0x542E; -pub const TIOCSRS485: ::c_int = 0x542F; - pub const POLLWRNORM: ::c_short = 0x100; pub const POLLWRBAND: ::c_short = 0x200; -pub const TIOCM_LE: ::c_int = 0x001; -pub const TIOCM_DTR: ::c_int = 0x002; -pub const TIOCM_RTS: ::c_int = 0x004; -pub const TIOCM_ST: ::c_int = 0x008; -pub const TIOCM_SR: ::c_int = 0x010; -pub const TIOCM_CTS: ::c_int = 0x020; -pub const TIOCM_CAR: ::c_int = 0x040; -pub const TIOCM_RNG: ::c_int = 0x080; -pub const TIOCM_DSR: ::c_int = 0x100; -pub const TIOCM_CD: ::c_int = TIOCM_CAR; -pub const TIOCM_RI: ::c_int = TIOCM_RNG; - // Syscall table pub const SYS_restart_syscall: ::c_long = 0; pub const SYS_exit: ::c_long = 1; @@ -850,13 +818,36 @@ pub const SYS_pkey_mprotect: ::c_long = 394; pub const SYS_pkey_alloc: ::c_long = 395; pub const SYS_pkey_free: ::c_long = 396; pub const SYS_statx: ::c_long = 397; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; +pub const SYS_quotactl_fd: ::c_long = 443; +pub const SYS_landlock_create_ruleset: ::c_long = 444; +pub const SYS_landlock_add_rule: ::c_long = 445; +pub const SYS_landlock_restrict_self: ::c_long = 446; +pub const SYS_memfd_secret: ::c_long = 447; +pub const SYS_process_mrelease: ::c_long = 448; +pub const SYS_futex_waitv: ::c_long = 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 450; extern "C" { - pub fn getrandom( - buf: *mut ::c_void, - buflen: ::size_t, - flags: ::c_uint, - ) -> ::ssize_t; + pub fn getrandom(buf: *mut ::c_void, buflen: ::size_t, flags: ::c_uint) -> ::ssize_t; } cfg_if! { diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/hexagon.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/hexagon.rs index 234069557..f83d208d5 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/hexagon.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/hexagon.rs @@ -129,17 +129,6 @@ s! { pub f_namemax: ::c_ulong, __f_spare: [::c_int; 6], } - - pub struct termios2 { - pub c_iflag: ::tcflag_t, - pub c_oflag: ::tcflag_t, - pub c_cflag: ::tcflag_t, - pub c_lflag: ::tcflag_t, - pub c_line: ::cc_t, - pub c_cc: [::cc_t; 19], - pub c_ispeed: ::speed_t, - pub c_ospeed: ::speed_t, - } } pub const AF_FILE: ::c_int = 1; @@ -222,6 +211,7 @@ pub const ESOCKTNOSUPPORT: ::c_int = 94; pub const ESTALE: ::c_int = 116; pub const ESTRPIPE: ::c_int = 86; pub const ETOOMANYREFS: ::c_int = 109; +pub const ETIMEDOUT: ::c_int = 110; pub const EUCLEAN: ::c_int = 117; pub const EUNATCH: ::c_int = 49; pub const EUSERS: ::c_int = 87; @@ -233,12 +223,6 @@ pub const F_GETOWN: ::c_int = 9; pub const F_GETOWNER_UIDS: ::c_int = 17; pub const F_GETOWN_EX: ::c_int = 16; pub const F_GETSIG: ::c_int = 11; -pub const FIOASYNC: ::c_int = 21586; -pub const FIOCLEX: ::c_int = 21585; -pub const FIONBIO: ::c_int = 21537; -pub const FIONCLEX: ::c_int = 21584; -pub const FIONREAD: ::c_int = 21531; -pub const FIOQSIZE: ::c_int = 21600; pub const F_LINUX_SPECIFIC_BASE: ::c_int = 1024; pub const FLUSHO: ::c_int = 4096; pub const F_OFD_GETLK: ::c_int = 36; @@ -258,8 +242,6 @@ pub const MAP_ANON: ::c_int = 32; pub const MAP_DENYWRITE: ::c_int = 2048; pub const MAP_EXECUTABLE: ::c_int = 4096; pub const MAP_GROWSDOWN: ::c_int = 256; -pub const MAP_HUGE_MASK: ::c_int = 63; -pub const MAP_HUGE_SHIFT: ::c_int = 26; pub const MAP_HUGETLB: ::c_int = 262144; pub const MAP_LOCKED: ::c_int = 8192; pub const MAP_NONBLOCK: ::c_int = 65536; @@ -283,13 +265,7 @@ pub const PF_FILE: ::c_int = 1; pub const PF_KCM: ::c_int = 41; pub const PF_MAX: ::c_int = 43; pub const PF_QIPCRTR: ::c_int = 42; -pub const RLIMIT_AS: ::c_int = 9; -pub const RLIMIT_MEMLOCK: ::c_int = 8; -pub const RLIMIT_NOFILE: ::c_int = 7; -pub const RLIMIT_NPROC: ::c_int = 6; -pub const RLIMIT_RSS: ::c_int = 5; #[deprecated(since = "0.2.64", note = "Not stable across OS versions")] -pub const RLIM_NLIMITS: ::c_int = 16; pub const SA_ONSTACK: ::c_int = 0x08000000; pub const SA_SIGINFO: ::c_int = 0x00000004; pub const SA_NOCLDWAIT: ::c_int = 0x00000002; @@ -301,6 +277,8 @@ pub const SIGPOLL: ::c_int = 29; pub const SIGPROF: ::c_int = 27; pub const SIGPWR: ::c_int = 30; pub const SIGSTKFLT: ::c_int = 16; +pub const SIGSTKSZ: ::size_t = 8192; +pub const MINSIGSTKSZ: ::size_t = 2048; pub const SIGSTOP: ::c_int = 19; pub const SIGSYS: ::c_int = 31; pub const SIGTSTP: ::c_int = 20; @@ -316,65 +294,19 @@ pub const SIGXFSZ: ::c_int = 25; pub const SIG_SETMASK: ::c_int = 2; // FIXME check these pub const SIG_BLOCK: ::c_int = 0x000000; pub const SIG_UNBLOCK: ::c_int = 0x01; -pub const SO_ACCEPTCONN: ::c_int = 30; -pub const SO_ATTACH_BPF: ::c_int = 50; -pub const SO_ATTACH_FILTER: ::c_int = 26; -pub const SO_ATTACH_REUSEPORT_CBPF: ::c_int = 51; -pub const SO_ATTACH_REUSEPORT_EBPF: ::c_int = 52; -pub const SO_BPF_EXTENSIONS: ::c_int = 48; -pub const SO_BROADCAST: ::c_int = 6; -pub const SO_BSDCOMPAT: ::c_int = 14; pub const SOCK_DGRAM: ::c_int = 2; pub const SOCK_NONBLOCK: ::c_int = 2048; pub const SOCK_SEQPACKET: ::c_int = 5; pub const SOCK_STREAM: ::c_int = 1; -pub const SO_CNX_ADVICE: ::c_int = 53; -pub const SO_DETACH_BPF: ::c_int = 27; -pub const SO_DETACH_FILTER: ::c_int = 27; -pub const SO_DOMAIN: ::c_int = 39; -pub const SO_DONTROUTE: ::c_int = 5; -pub const SO_ERROR: ::c_int = 4; -pub const SO_GET_FILTER: ::c_int = 26; -pub const SO_INCOMING_CPU: ::c_int = 49; -pub const SO_KEEPALIVE: ::c_int = 9; pub const SOL_CAIF: ::c_int = 278; -pub const SO_LINGER: ::c_int = 13; pub const SOL_IUCV: ::c_int = 277; pub const SOL_KCM: ::c_int = 281; pub const SOL_NFC: ::c_int = 280; -pub const SO_LOCK_FILTER: ::c_int = 44; pub const SOL_PNPIPE: ::c_int = 275; pub const SOL_PPPOL2TP: ::c_int = 273; pub const SOL_RDS: ::c_int = 276; pub const SOL_RXRPC: ::c_int = 272; -pub const SOL_SOCKET: ::c_int = 1; -pub const SO_MAX_PACING_RATE: ::c_int = 47; -pub const SO_NO_CHECK: ::c_int = 11; -pub const SO_NOFCS: ::c_int = 43; -pub const SO_OOBINLINE: ::c_int = 10; -pub const SO_PASSCRED: ::c_int = 16; -pub const SO_PASSSEC: ::c_int = 34; -pub const SO_PEERCRED: ::c_int = 17; -pub const SO_PEERNAME: ::c_int = 28; -pub const SO_PEERSEC: ::c_int = 31; -pub const SO_PRIORITY: ::c_int = 12; -pub const SO_PROTOCOL: ::c_int = 38; -pub const SO_RCVBUF: ::c_int = 8; -pub const SO_RCVBUFFORCE: ::c_int = 33; -pub const SO_RCVLOWAT: ::c_int = 18; -pub const SO_RCVTIMEO: ::c_int = 20; -pub const SO_REUSEADDR: ::c_int = 2; -pub const SO_REUSEPORT: ::c_int = 15; -pub const SO_SECURITY_AUTHENTICATION: ::c_int = 22; -pub const SO_SECURITY_ENCRYPTION_NETWORK: ::c_int = 24; -pub const SO_SECURITY_ENCRYPTION_TRANSPORT: ::c_int = 23; -pub const SO_SELECT_ERR_QUEUE: ::c_int = 45; -pub const SO_SNDBUF: ::c_int = 7; -pub const SO_SNDBUFFORCE: ::c_int = 32; -pub const SO_SNDLOWAT: ::c_int = 19; -pub const SO_SNDTIMEO: ::c_int = 21; -pub const SO_TYPE: ::c_int = 3; -pub const SO_WIFI_STATUS: ::c_int = 41; + pub const SYS3264_fadvise64: ::c_int = 223; pub const SYS3264_fcntl: ::c_int = 25; pub const SYS3264_fstatat: ::c_int = 79; @@ -711,85 +643,29 @@ pub const SYS_waitid: ::c_int = 95; pub const SYS_write: ::c_int = 64; pub const SYS_writev: ::c_int = 66; pub const SYS_statx: ::c_int = 291; -pub const TCFLSH: ::c_int = 21515; -pub const TCGETA: ::c_int = 21509; -pub const TCGETS: ::c_int = 21505; -pub const TCGETX: ::c_int = 21554; -pub const TCSBRK: ::c_int = 21513; -pub const TCSBRKP: ::c_int = 21541; -pub const TCSETA: ::c_int = 21510; -pub const TCSETAF: ::c_int = 21512; -pub const TCSETAW: ::c_int = 21511; -pub const TCSETS: ::c_int = 21506; -pub const TCSETSF: ::c_int = 21508; -pub const TCSETSW: ::c_int = 21507; -pub const TCSETX: ::c_int = 21555; -pub const TCSETXF: ::c_int = 21556; -pub const TCSETXW: ::c_int = 21557; -pub const TCXONC: ::c_int = 21514; -pub const TIOCCBRK: ::c_int = 21544; -pub const TIOCCONS: ::c_int = 21533; -pub const TIOCEXCL: ::c_int = 21516; -pub const TIOCGETD: ::c_int = 21540; -pub const TIOCGICOUNT: ::c_int = 21597; -pub const TIOCGLCKTRMIOS: ::c_int = 21590; -pub const TIOCGPGRP: ::c_int = 21519; -pub const TIOCGRS485: ::c_int = 21550; -pub const TIOCGSERIAL: ::c_int = 21534; -pub const TIOCGSID: ::c_int = 21545; -pub const TIOCGSOFTCAR: ::c_int = 21529; -pub const TIOCGWINSZ: ::c_int = 21523; -pub const TIOCLINUX: ::c_int = 21532; -pub const TIOCMBIC: ::c_int = 21527; -pub const TIOCMBIS: ::c_int = 21526; -pub const TIOCM_CAR: ::c_int = 64; -pub const TIOCM_CD: ::c_int = 64; -pub const TIOCM_CTS: ::c_int = 32; -pub const TIOCM_DSR: ::c_int = 256; -pub const TIOCM_DTR: ::c_int = 2; -pub const TIOCMGET: ::c_int = 21525; -pub const TIOCMIWAIT: ::c_int = 21596; -pub const TIOCM_LE: ::c_int = 1; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; pub const TIOCM_LOOP: ::c_int = 32768; pub const TIOCM_OUT1: ::c_int = 8192; pub const TIOCM_OUT2: ::c_int = 16384; -pub const TIOCM_RI: ::c_int = 128; -pub const TIOCM_RNG: ::c_int = 128; -pub const TIOCM_RTS: ::c_int = 4; -pub const TIOCMSET: ::c_int = 21528; -pub const TIOCM_SR: ::c_int = 16; -pub const TIOCM_ST: ::c_int = 8; -pub const TIOCNOTTY: ::c_int = 21538; -pub const TIOCNXCL: ::c_int = 21517; -pub const TIOCOUTQ: ::c_int = 21521; -pub const TIOCPKT: ::c_int = 21536; -pub const TIOCPKT_DATA: ::c_int = 0; -pub const TIOCPKT_DOSTOP: ::c_int = 32; -pub const TIOCPKT_FLUSHREAD: ::c_int = 1; -pub const TIOCPKT_FLUSHWRITE: ::c_int = 2; -pub const TIOCPKT_IOCTL: ::c_int = 64; -pub const TIOCPKT_NOSTOP: ::c_int = 16; -pub const TIOCPKT_START: ::c_int = 8; -pub const TIOCPKT_STOP: ::c_int = 4; -pub const TIOCSBRK: ::c_int = 21543; -pub const TIOCSCTTY: ::c_int = 21518; -pub const TIOCSERCONFIG: ::c_int = 21587; -pub const TIOCSERGETLSR: ::c_int = 21593; -pub const TIOCSERGETMULTI: ::c_int = 21594; -pub const TIOCSERGSTRUCT: ::c_int = 21592; -pub const TIOCSERGWILD: ::c_int = 21588; -pub const TIOCSERSETMULTI: ::c_int = 21595; -pub const TIOCSERSWILD: ::c_int = 21589; pub const TIOCSER_TEMT: ::c_int = 1; -pub const TIOCSETD: ::c_int = 21539; -pub const TIOCSLCKTRMIOS: ::c_int = 21591; -pub const TIOCSPGRP: ::c_int = 21520; -pub const TIOCSRS485: ::c_int = 21551; -pub const TIOCSSERIAL: ::c_int = 21535; -pub const TIOCSSOFTCAR: ::c_int = 21530; -pub const TIOCSTI: ::c_int = 21522; -pub const TIOCSWINSZ: ::c_int = 21524; -pub const TIOCVHANGUP: ::c_int = 21559; pub const TOSTOP: ::c_int = 256; pub const VEOF: ::c_int = 4; pub const VEOL2: ::c_int = 16; diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/mips/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/mips/mod.rs index 7dfac784d..40b507bcd 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/mips/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/mips/mod.rs @@ -161,24 +161,6 @@ s! { pub f_namemax: ::c_ulong, __f_spare: [::c_int; 6], } - - pub struct nlmsghdr { - pub nlmsg_len: u32, - pub nlmsg_type: u16, - pub nlmsg_flags: u16, - pub nlmsg_seq: u32, - pub nlmsg_pid: u32, - } - - pub struct nlmsgerr { - pub error: ::c_int, - pub msg: nlmsghdr, - } - - pub struct nlattr { - pub nla_len: u16, - pub nla_type: u16, - } } pub const SIGSTKSZ: ::size_t = 8192; @@ -190,17 +172,6 @@ pub const O_NOFOLLOW: ::c_int = 0o400000; pub const O_ASYNC: ::c_int = 0o10000; pub const O_LARGEFILE: ::c_int = 0x2000; -pub const FIOCLEX: ::c_int = 0x6601; -pub const FIONCLEX: ::c_int = 0x6602; -pub const FIONBIO: ::c_int = 0x667E; - -pub const RLIMIT_RSS: ::c_int = 7; -pub const RLIMIT_NOFILE: ::c_int = 5; -pub const RLIMIT_AS: ::c_int = 6; -pub const RLIMIT_NPROC: ::c_int = 8; -pub const RLIMIT_MEMLOCK: ::c_int = 9; -pub const RLIMIT_NLIMITS: ::c_int = 15; - pub const MCL_CURRENT: ::c_int = 0x0001; pub const MCL_FUTURE: ::c_int = 0x0002; pub const CBAUD: ::tcflag_t = 0o0010017; @@ -381,34 +352,6 @@ pub const SOCK_STREAM: ::c_int = 2; pub const SOCK_DGRAM: ::c_int = 1; pub const SOCK_SEQPACKET: ::c_int = 5; -pub const SOL_SOCKET: ::c_int = 65535; - -pub const SO_REUSEADDR: ::c_int = 0x0004; -pub const SO_KEEPALIVE: ::c_int = 0x0008; -pub const SO_DONTROUTE: ::c_int = 0x0010; -pub const SO_BROADCAST: ::c_int = 0x0020; -pub const SO_LINGER: ::c_int = 0x0080; -pub const SO_OOBINLINE: ::c_int = 0x0100; -pub const SO_REUSEPORT: ::c_int = 0x0200; -pub const SO_SNDBUF: ::c_int = 0x1001; -pub const SO_RCVBUF: ::c_int = 0x1002; -pub const SO_SNDLOWAT: ::c_int = 0x1003; -pub const SO_RCVLOWAT: ::c_int = 0x1004; -pub const SO_RCVTIMEO: ::c_int = 0x1006; -pub const SO_SNDTIMEO: ::c_int = 0x1005; -pub const SO_ERROR: ::c_int = 0x1007; -pub const SO_TYPE: ::c_int = 0x1008; -pub const SO_ACCEPTCONN: ::c_int = 0x1009; -pub const SO_PROTOCOL: ::c_int = 0x1028; -pub const SO_DOMAIN: ::c_int = 0x1029; -pub const SO_NO_CHECK: ::c_int = 11; -pub const SO_PRIORITY: ::c_int = 12; -pub const SO_BSDCOMPAT: ::c_int = 14; -pub const SO_PASSCRED: ::c_int = 17; -pub const SO_PEERCRED: ::c_int = 18; -pub const SO_SNDBUFFORCE: ::c_int = 31; -pub const SO_RCVBUFFORCE: ::c_int = 33; - pub const SA_ONSTACK: ::c_int = 0x08000000; pub const SA_SIGINFO: ::c_int = 8; pub const SA_NOCLDWAIT: ::c_int = 0x10000; @@ -458,55 +401,9 @@ pub const IEXTEN: ::tcflag_t = 0o000400; pub const TOSTOP: ::tcflag_t = 0o100000; pub const FLUSHO: ::tcflag_t = 0o020000; -pub const TCGETS: ::c_int = 0x540D; -pub const TCSETS: ::c_int = 0x540E; -pub const TCSETSW: ::c_int = 0x540F; -pub const TCSETSF: ::c_int = 0x5410; -pub const TCGETA: ::c_int = 0x5401; -pub const TCSETA: ::c_int = 0x5402; -pub const TCSETAW: ::c_int = 0x5403; -pub const TCSETAF: ::c_int = 0x5404; -pub const TCSBRK: ::c_int = 0x5405; -pub const TCXONC: ::c_int = 0x5406; -pub const TCFLSH: ::c_int = 0x5407; -pub const TIOCGSOFTCAR: ::c_int = 0x5481; -pub const TIOCSSOFTCAR: ::c_int = 0x5482; -pub const TIOCLINUX: ::c_int = 0x5483; -pub const TIOCGSERIAL: ::c_int = 0x5484; -pub const TIOCEXCL: ::c_int = 0x740D; -pub const TIOCNXCL: ::c_int = 0x740E; -pub const TIOCSCTTY: ::c_int = 0x5480; -pub const TIOCGPGRP: ::c_int = 0x40047477; -pub const TIOCSPGRP: ::c_int = 0x80047476; -pub const TIOCOUTQ: ::c_int = 0x7472; -pub const TIOCSTI: ::c_int = 0x5472; -pub const TIOCGWINSZ: ::c_int = 0x40087468; -pub const TIOCSWINSZ: ::c_int = 0x80087467; -pub const TIOCMGET: ::c_int = 0x741D; -pub const TIOCMBIS: ::c_int = 0x741B; -pub const TIOCMBIC: ::c_int = 0x741C; -pub const TIOCMSET: ::c_int = 0x741A; -pub const FIONREAD: ::c_int = 0x467F; -pub const TIOCCONS: ::c_int = 0x80047478; - -pub const TIOCGRS485: ::c_int = 0x4020542E; -pub const TIOCSRS485: ::c_int = 0xC020542F; - pub const POLLWRNORM: ::c_short = 0x4; pub const POLLWRBAND: ::c_short = 0x100; -pub const TIOCM_LE: ::c_int = 0x001; -pub const TIOCM_DTR: ::c_int = 0x002; -pub const TIOCM_RTS: ::c_int = 0x004; -pub const TIOCM_ST: ::c_int = 0x010; -pub const TIOCM_SR: ::c_int = 0x020; -pub const TIOCM_CTS: ::c_int = 0x040; -pub const TIOCM_CAR: ::c_int = 0x100; -pub const TIOCM_CD: ::c_int = TIOCM_CAR; -pub const TIOCM_RNG: ::c_int = 0x200; -pub const TIOCM_RI: ::c_int = TIOCM_RNG; -pub const TIOCM_DSR: ::c_int = 0x400; - pub const SYS_syscall: ::c_long = 4000 + 0; pub const SYS_exit: ::c_long = 4000 + 1; pub const SYS_fork: ::c_long = 4000 + 2; @@ -861,6 +758,33 @@ pub const SYS_pkey_mprotect: ::c_long = 4000 + 363; pub const SYS_pkey_alloc: ::c_long = 4000 + 364; pub const SYS_pkey_free: ::c_long = 4000 + 365; pub const SYS_statx: ::c_long = 4000 + 366; +pub const SYS_pidfd_send_signal: ::c_long = 4000 + 424; +pub const SYS_io_uring_setup: ::c_long = 4000 + 425; +pub const SYS_io_uring_enter: ::c_long = 4000 + 426; +pub const SYS_io_uring_register: ::c_long = 4000 + 427; +pub const SYS_open_tree: ::c_long = 4000 + 428; +pub const SYS_move_mount: ::c_long = 4000 + 429; +pub const SYS_fsopen: ::c_long = 4000 + 430; +pub const SYS_fsconfig: ::c_long = 4000 + 431; +pub const SYS_fsmount: ::c_long = 4000 + 432; +pub const SYS_fspick: ::c_long = 4000 + 433; +pub const SYS_pidfd_open: ::c_long = 4000 + 434; +pub const SYS_clone3: ::c_long = 4000 + 435; +pub const SYS_close_range: ::c_long = 4000 + 436; +pub const SYS_openat2: ::c_long = 4000 + 437; +pub const SYS_pidfd_getfd: ::c_long = 4000 + 438; +pub const SYS_faccessat2: ::c_long = 4000 + 439; +pub const SYS_process_madvise: ::c_long = 4000 + 440; +pub const SYS_epoll_pwait2: ::c_long = 4000 + 441; +pub const SYS_mount_setattr: ::c_long = 4000 + 442; +pub const SYS_quotactl_fd: ::c_long = 4000 + 443; +pub const SYS_landlock_create_ruleset: ::c_long = 4000 + 444; +pub const SYS_landlock_add_rule: ::c_long = 4000 + 445; +pub const SYS_landlock_restrict_self: ::c_long = 4000 + 446; +pub const SYS_memfd_secret: ::c_long = 4000 + 447; +pub const SYS_process_mrelease: ::c_long = 4000 + 448; +pub const SYS_futex_waitv: ::c_long = 4000 + 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 4000 + 450; cfg_if! { if #[cfg(libc_align)] { diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/mod.rs index 7cf6da913..63824fbf5 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/mod.rs @@ -3,6 +3,8 @@ pub type c_ulong = u32; pub type nlink_t = u32; pub type blksize_t = ::c_long; pub type __u64 = ::c_ulonglong; +pub type __s64 = ::c_longlong; +pub type regoff_t = ::c_int; s! { pub struct pthread_attr_t { @@ -37,12 +39,6 @@ s! { pub const __SIZEOF_PTHREAD_RWLOCK_T: usize = 32; pub const __SIZEOF_PTHREAD_MUTEX_T: usize = 24; -pub const TIOCINQ: ::c_int = ::FIONREAD; - -extern "C" { - pub fn ioctl(fd: ::c_int, request: ::c_int, ...) -> ::c_int; -} - cfg_if! { if #[cfg(any(target_arch = "x86"))] { mod x86; @@ -59,6 +55,9 @@ cfg_if! { } else if #[cfg(any(target_arch = "hexagon"))] { mod hexagon; pub use self::hexagon::*; + } else if #[cfg(any(target_arch = "riscv32"))] { + mod riscv32; + pub use self::riscv32::*; } else { // Unknown target_arch } diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/powerpc.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/powerpc.rs index b94bb7b15..5b1bf17ed 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/powerpc.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/powerpc.rs @@ -165,17 +165,6 @@ pub const O_NOFOLLOW: ::c_int = 0x8000; pub const O_ASYNC: ::c_int = 0x2000; pub const O_LARGEFILE: ::c_int = 0x10000; -pub const FIOCLEX: ::c_int = 0x20006601; -pub const FIONCLEX: ::c_int = 0x20006602; -pub const FIONBIO: ::c_int = 0x8004667E; - -pub const RLIMIT_RSS: ::c_int = 5; -pub const RLIMIT_NOFILE: ::c_int = 7; -pub const RLIMIT_AS: ::c_int = 9; -pub const RLIMIT_NPROC: ::c_int = 6; -pub const RLIMIT_MEMLOCK: ::c_int = 8; -pub const RLIMIT_NLIMITS: ::c_int = 15; - pub const MCL_CURRENT: ::c_int = 0x2000; pub const MCL_FUTURE: ::c_int = 0x4000; pub const CBAUD: ::tcflag_t = 0o0000377; @@ -270,8 +259,6 @@ pub const SOCK_STREAM: ::c_int = 1; pub const SOCK_DGRAM: ::c_int = 2; pub const SOCK_SEQPACKET: ::c_int = 5; -pub const SOL_SOCKET: ::c_int = 1; - pub const EDEADLK: ::c_int = 35; pub const ENAMETOOLONG: ::c_int = 36; pub const ENOLCK: ::c_int = 37; @@ -358,32 +345,6 @@ pub const ENOTRECOVERABLE: ::c_int = 131; pub const ERFKILL: ::c_int = 132; pub const EHWPOISON: ::c_int = 133; -pub const SO_REUSEADDR: ::c_int = 2; -pub const SO_TYPE: ::c_int = 3; -pub const SO_ERROR: ::c_int = 4; -pub const SO_DONTROUTE: ::c_int = 5; -pub const SO_BROADCAST: ::c_int = 6; -pub const SO_SNDBUF: ::c_int = 7; -pub const SO_RCVBUF: ::c_int = 8; -pub const SO_KEEPALIVE: ::c_int = 9; -pub const SO_OOBINLINE: ::c_int = 10; -pub const SO_NO_CHECK: ::c_int = 11; -pub const SO_PRIORITY: ::c_int = 12; -pub const SO_LINGER: ::c_int = 13; -pub const SO_BSDCOMPAT: ::c_int = 14; -pub const SO_REUSEPORT: ::c_int = 15; -pub const SO_RCVLOWAT: ::c_int = 16; -pub const SO_SNDLOWAT: ::c_int = 17; -pub const SO_RCVTIMEO: ::c_int = 18; -pub const SO_SNDTIMEO: ::c_int = 19; -pub const SO_PASSCRED: ::c_int = 20; -pub const SO_PEERCRED: ::c_int = 21; -pub const SO_ACCEPTCONN: ::c_int = 30; -pub const SO_SNDBUFFORCE: ::c_int = 32; -pub const SO_RCVBUFFORCE: ::c_int = 33; -pub const SO_PROTOCOL: ::c_int = 38; -pub const SO_DOMAIN: ::c_int = 39; - pub const SA_ONSTACK: ::c_int = 0x08000000; pub const SA_SIGINFO: ::c_int = 0x00000004; pub const SA_NOCLDWAIT: ::c_int = 0x00000002; @@ -433,55 +394,9 @@ pub const IEXTEN: ::tcflag_t = 0x00000400; pub const TOSTOP: ::tcflag_t = 0x00400000; pub const FLUSHO: ::tcflag_t = 0x00800000; -pub const TCGETS: ::c_int = 0x402C7413; -pub const TCSETS: ::c_int = 0x802C7414; -pub const TCSETSW: ::c_int = 0x802C7415; -pub const TCSETSF: ::c_int = 0x802C7416; -pub const TCGETA: ::c_int = 0x40147417; -pub const TCSETA: ::c_int = 0x80147418; -pub const TCSETAW: ::c_int = 0x80147419; -pub const TCSETAF: ::c_int = 0x8014741C; -pub const TCSBRK: ::c_int = 0x2000741D; -pub const TCXONC: ::c_int = 0x2000741E; -pub const TCFLSH: ::c_int = 0x2000741F; -pub const TIOCGSOFTCAR: ::c_int = 0x5419; -pub const TIOCSSOFTCAR: ::c_int = 0x541A; -pub const TIOCLINUX: ::c_int = 0x541C; -pub const TIOCGSERIAL: ::c_int = 0x541E; -pub const TIOCEXCL: ::c_int = 0x540C; -pub const TIOCNXCL: ::c_int = 0x540D; -pub const TIOCSCTTY: ::c_int = 0x540E; -pub const TIOCGPGRP: ::c_int = 0x40047477; -pub const TIOCSPGRP: ::c_int = 0x80047476; -pub const TIOCOUTQ: ::c_int = 0x40047473; -pub const TIOCSTI: ::c_int = 0x5412; -pub const TIOCGWINSZ: ::c_int = 0x40087468; -pub const TIOCSWINSZ: ::c_int = 0x80087467; -pub const TIOCMGET: ::c_int = 0x5415; -pub const TIOCMBIS: ::c_int = 0x5416; -pub const TIOCMBIC: ::c_int = 0x5417; -pub const TIOCMSET: ::c_int = 0x5418; -pub const FIONREAD: ::c_int = 0x4004667F; -pub const TIOCCONS: ::c_int = 0x541D; - -pub const TIOCGRS485: ::c_int = 0x542e; -pub const TIOCSRS485: ::c_int = 0x542f; - pub const POLLWRNORM: ::c_short = 0x100; pub const POLLWRBAND: ::c_short = 0x200; -pub const TIOCM_LE: ::c_int = 0x001; -pub const TIOCM_DTR: ::c_int = 0x002; -pub const TIOCM_RTS: ::c_int = 0x004; -pub const TIOCM_ST: ::c_int = 0x008; -pub const TIOCM_SR: ::c_int = 0x010; -pub const TIOCM_CTS: ::c_int = 0x020; -pub const TIOCM_CAR: ::c_int = 0x040; -pub const TIOCM_RNG: ::c_int = 0x080; -pub const TIOCM_DSR: ::c_int = 0x100; -pub const TIOCM_CD: ::c_int = TIOCM_CAR; -pub const TIOCM_RI: ::c_int = TIOCM_RNG; - // Syscall table pub const SYS_restart_syscall: ::c_long = 0; pub const SYS_exit: ::c_long = 1; @@ -856,11 +771,34 @@ pub const SYS_statx: ::c_long = 383; pub const SYS_pkey_alloc: ::c_long = 384; pub const SYS_pkey_free: ::c_long = 385; pub const SYS_pkey_mprotect: ::c_long = 386; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; +pub const SYS_quotactl_fd: ::c_long = 443; +pub const SYS_landlock_create_ruleset: ::c_long = 444; +pub const SYS_landlock_add_rule: ::c_long = 445; +pub const SYS_landlock_restrict_self: ::c_long = 446; +pub const SYS_memfd_secret: ::c_long = 447; +pub const SYS_process_mrelease: ::c_long = 448; +pub const SYS_futex_waitv: ::c_long = 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 450; extern "C" { - pub fn getrandom( - buf: *mut ::c_void, - buflen: ::size_t, - flags: ::c_uint, - ) -> ::ssize_t; + pub fn getrandom(buf: *mut ::c_void, buflen: ::size_t, flags: ::c_uint) -> ::ssize_t; } diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/riscv32/align.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/riscv32/align.rs new file mode 100644 index 000000000..048268c96 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/riscv32/align.rs @@ -0,0 +1,7 @@ +s_no_extra_traits! { + #[allow(missing_debug_implementations)] + #[repr(align(8))] + pub struct max_align_t { + priv_: (i64, f64) + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/riscv32/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/riscv32/mod.rs new file mode 100644 index 000000000..573624620 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/riscv32/mod.rs @@ -0,0 +1,808 @@ +//! RISC-V-specific definitions for 32-bit linux-like values + +pub type c_char = u8; +pub type wchar_t = ::c_int; + +s! { + pub struct pthread_attr_t { + __size: [::c_ulong; 7], + } + + pub struct stat { + pub st_dev: ::dev_t, + pub st_ino: ::ino_t, + pub st_mode: ::mode_t, + pub st_nlink: ::nlink_t, + pub st_uid: ::uid_t, + pub st_gid: ::gid_t, + pub st_rdev: ::dev_t, + pub __pad1: ::dev_t, + pub st_size: ::off_t, + pub st_blksize: ::blksize_t, + pub __pad2: ::c_int, + pub st_blocks: ::blkcnt_t, + pub st_atime: ::time_t, + pub st_atime_nsec: ::c_long, + pub st_mtime: ::time_t, + pub st_mtime_nsec: ::c_long, + pub st_ctime: ::time_t, + pub st_ctime_nsec: ::c_long, + __unused: [::c_int; 2usize], + } + + pub struct stat64 { + pub st_dev: ::dev_t, + pub st_ino: ::ino64_t, + pub st_mode: ::mode_t, + pub st_nlink: ::nlink_t, + pub st_uid: ::uid_t, + pub st_gid: ::gid_t, + pub st_rdev: ::dev_t, + pub __pad1: ::dev_t, + pub st_size: ::off64_t, + pub st_blksize: ::blksize_t, + pub __pad2: ::c_int, + pub st_blocks: ::blkcnt64_t, + pub st_atime: ::time_t, + pub st_atime_nsec: ::c_long, + pub st_mtime: ::time_t, + pub st_mtime_nsec: ::c_long, + pub st_ctime: ::time_t, + pub st_ctime_nsec: ::c_long, + __unused: [::c_int; 2], + } + + pub struct statfs { + pub f_type: ::c_long, + pub f_bsize: ::c_long, + pub f_blocks: ::fsblkcnt_t, + pub f_bfree: ::fsblkcnt_t, + pub f_bavail: ::fsblkcnt_t, + pub f_files: ::fsfilcnt_t, + pub f_ffree: ::fsfilcnt_t, + pub f_fsid: ::fsid_t, + pub f_namelen: ::c_long, + pub f_frsize: ::c_long, + pub f_flags: ::c_long, + pub f_spare: [::c_long; 4], + } + + pub struct statvfs { + pub f_bsize: ::c_ulong, + pub f_frsize: ::c_ulong, + pub f_blocks: ::fsblkcnt_t, + pub f_bfree: ::fsblkcnt_t, + pub f_bavail: ::fsblkcnt_t, + pub f_files: ::fsfilcnt_t, + pub f_ffree: ::fsfilcnt_t, + pub f_favail: ::fsfilcnt_t, + pub f_fsid: ::c_ulong, + pub f_flag: ::c_ulong, + pub f_namemax: ::c_ulong, + pub __f_spare: [::c_int; 6], + } + + pub struct statfs64 { + pub f_type: ::c_ulong, + pub f_bsize: ::c_ulong, + pub f_blocks: ::fsblkcnt_t, + pub f_bfree: ::fsblkcnt_t, + pub f_bavail: ::fsblkcnt_t, + pub f_files: ::fsfilcnt_t, + pub f_ffree: ::fsfilcnt_t, + pub f_fsid: ::fsid_t, + pub f_namelen: ::c_ulong, + pub f_frsize: ::c_ulong, + pub f_flags: ::c_ulong, + pub f_spare: [::c_ulong; 4], + } + + pub struct statvfs64 { + pub f_bsize: ::c_ulong, + pub f_frsize: ::c_ulong, + pub f_blocks: u64, + pub f_bfree: u64, + pub f_bavail: u64, + pub f_files: u64, + pub f_ffree: u64, + pub f_favail: u64, + pub f_fsid: ::c_ulong, + __f_unused: ::c_int, + pub f_flag: ::c_ulong, + pub f_namemax: ::c_ulong, + __f_spare: [::c_int; 6], + } + + pub struct siginfo_t { + pub si_signo: ::c_int, + pub si_errno: ::c_int, + pub si_code: ::c_int, + #[doc(hidden)] + #[deprecated( + since="0.2.54", + note="Please leave a comment on \ + https://github.com/rust-lang/libc/pull/1316 if you're using \ + this field" + )] + pub _pad: [::c_int; 29], + _align: [u64; 0], + } + + pub struct stack_t { + pub ss_sp: *mut ::c_void, + pub ss_flags: ::c_int, + pub ss_size: ::size_t, + } + + pub struct sigaction { + pub sa_sigaction: ::sighandler_t, + pub sa_mask: ::sigset_t, + pub sa_flags: ::c_int, + pub sa_restorer: ::Option, + } + + pub struct ipc_perm { + pub __key: ::key_t, + pub uid: ::uid_t, + pub gid: ::gid_t, + pub cuid: ::uid_t, + pub cgid: ::gid_t, + pub mode: ::c_ushort, + __pad1: ::c_ushort, + pub __seq: ::c_ushort, + __pad2: ::c_ushort, + __unused1: ::c_ulong, + __unused2: ::c_ulong, + } + + pub struct shmid_ds { + pub shm_perm: ::ipc_perm, + pub shm_segsz: ::size_t, + pub shm_atime: ::time_t, + pub shm_dtime: ::time_t, + pub shm_ctime: ::time_t, + pub shm_cpid: ::pid_t, + pub shm_lpid: ::pid_t, + pub shm_nattch: ::shmatt_t, + __unused5: ::c_ulong, + __unused6: ::c_ulong, + } + + pub struct msqid_ds { + pub msg_perm: ::ipc_perm, + pub msg_stime: ::time_t, + __unused1: ::c_int, + pub msg_rtime: ::time_t, + __unused2: ::c_int, + pub msg_ctime: ::time_t, + __unused3: ::c_int, + __msg_cbytes: ::c_ulong, + pub msg_qnum: ::msgqnum_t, + pub msg_qbytes: ::msglen_t, + pub msg_lspid: ::pid_t, + pub msg_lrpid: ::pid_t, + __pad1: ::c_ulong, + __pad2: ::c_ulong, + } + + pub struct flock { + pub l_type: ::c_short, + pub l_whence: ::c_short, + pub l_start: ::off_t, + pub l_len: ::off_t, + pub l_pid: ::pid_t, + } + + pub struct flock64 { + pub l_type: ::c_short, + pub l_whence: ::c_short, + pub l_start: ::off64_t, + pub l_len: ::off64_t, + pub l_pid: ::pid_t, + } +} + +//pub const RLIM_INFINITY: ::rlim_t = !0; +pub const VEOF: usize = 4; +pub const RTLD_DEEPBIND: ::c_int = 0x8; +pub const RTLD_GLOBAL: ::c_int = 0x100; +pub const RTLD_NOLOAD: ::c_int = 0x4; +pub const TIOCGSOFTCAR: ::c_ulong = 21529; +pub const TIOCSSOFTCAR: ::c_ulong = 21530; +pub const TIOCGRS485: ::c_int = 21550; +pub const TIOCSRS485: ::c_int = 21551; +//pub const RLIMIT_RSS: ::__rlimit_resource_t = 5; +//pub const RLIMIT_AS: ::__rlimit_resource_t = 9; +//pub const RLIMIT_MEMLOCK: ::__rlimit_resource_t = 8; +//pub const RLIMIT_NOFILE: ::__rlimit_resource_t = 7; +//pub const RLIMIT_NPROC: ::__rlimit_resource_t = 6; +pub const O_APPEND: ::c_int = 1024; +pub const O_CREAT: ::c_int = 64; +pub const O_EXCL: ::c_int = 128; +pub const O_NOCTTY: ::c_int = 256; +pub const O_NONBLOCK: ::c_int = 2048; +pub const O_SYNC: ::c_int = 1052672; +pub const O_RSYNC: ::c_int = 1052672; +pub const O_DSYNC: ::c_int = 4096; +pub const O_FSYNC: ::c_int = 1052672; +pub const MAP_GROWSDOWN: ::c_int = 256; +pub const EDEADLK: ::c_int = 35; +pub const ENAMETOOLONG: ::c_int = 36; +pub const ENOLCK: ::c_int = 37; +pub const ENOSYS: ::c_int = 38; +pub const ENOTEMPTY: ::c_int = 39; +pub const ELOOP: ::c_int = 40; +pub const ENOMSG: ::c_int = 42; +pub const EIDRM: ::c_int = 43; +pub const ECHRNG: ::c_int = 44; +pub const EL2NSYNC: ::c_int = 45; +pub const EL3HLT: ::c_int = 46; +pub const EL3RST: ::c_int = 47; +pub const ELNRNG: ::c_int = 48; +pub const EUNATCH: ::c_int = 49; +pub const ENOCSI: ::c_int = 50; +pub const EL2HLT: ::c_int = 51; +pub const EBADE: ::c_int = 52; +pub const EBADR: ::c_int = 53; +pub const EXFULL: ::c_int = 54; +pub const ENOANO: ::c_int = 55; +pub const EBADRQC: ::c_int = 56; +pub const EBADSLT: ::c_int = 57; +pub const EMULTIHOP: ::c_int = 72; +pub const EOVERFLOW: ::c_int = 75; +pub const ENOTUNIQ: ::c_int = 76; +pub const EBADFD: ::c_int = 77; +pub const EBADMSG: ::c_int = 74; +pub const EREMCHG: ::c_int = 78; +pub const ELIBACC: ::c_int = 79; +pub const ELIBBAD: ::c_int = 80; +pub const ELIBSCN: ::c_int = 81; +pub const ELIBMAX: ::c_int = 82; +pub const ELIBEXEC: ::c_int = 83; +pub const EILSEQ: ::c_int = 84; +pub const ERESTART: ::c_int = 85; +pub const ESTRPIPE: ::c_int = 86; +pub const EUSERS: ::c_int = 87; +pub const ENOTSOCK: ::c_int = 88; +pub const EDESTADDRREQ: ::c_int = 89; +pub const EMSGSIZE: ::c_int = 90; +pub const EPROTOTYPE: ::c_int = 91; +pub const ENOPROTOOPT: ::c_int = 92; +pub const EPROTONOSUPPORT: ::c_int = 93; +pub const ESOCKTNOSUPPORT: ::c_int = 94; +pub const EOPNOTSUPP: ::c_int = 95; +pub const ENOTSUP: ::c_int = EOPNOTSUPP; +pub const EPFNOSUPPORT: ::c_int = 96; +pub const EAFNOSUPPORT: ::c_int = 97; +pub const EADDRINUSE: ::c_int = 98; +pub const EADDRNOTAVAIL: ::c_int = 99; +pub const ENETDOWN: ::c_int = 100; +pub const ENETUNREACH: ::c_int = 101; +pub const ENETRESET: ::c_int = 102; +pub const ECONNABORTED: ::c_int = 103; +pub const ECONNRESET: ::c_int = 104; +pub const ENOBUFS: ::c_int = 105; +pub const EISCONN: ::c_int = 106; +pub const ENOTCONN: ::c_int = 107; +pub const ESHUTDOWN: ::c_int = 108; +pub const ETOOMANYREFS: ::c_int = 109; +pub const ETIMEDOUT: ::c_int = 110; +pub const ECONNREFUSED: ::c_int = 111; +pub const EHOSTDOWN: ::c_int = 112; +pub const EHOSTUNREACH: ::c_int = 113; +pub const EALREADY: ::c_int = 114; +pub const EINPROGRESS: ::c_int = 115; +pub const ESTALE: ::c_int = 116; +pub const EDQUOT: ::c_int = 122; +pub const ENOMEDIUM: ::c_int = 123; +pub const EMEDIUMTYPE: ::c_int = 124; +pub const ECANCELED: ::c_int = 125; +pub const ENOKEY: ::c_int = 126; +pub const EKEYEXPIRED: ::c_int = 127; +pub const EKEYREVOKED: ::c_int = 128; +pub const EKEYREJECTED: ::c_int = 129; +pub const EOWNERDEAD: ::c_int = 130; +pub const ENOTRECOVERABLE: ::c_int = 131; +pub const EHWPOISON: ::c_int = 133; +pub const ERFKILL: ::c_int = 132; + +pub const SOCK_STREAM: ::c_int = 1; +pub const SOCK_DGRAM: ::c_int = 2; +pub const SA_ONSTACK: ::c_int = 8; +pub const SA_SIGINFO: ::c_int = 4; +pub const SA_NOCLDWAIT: ::c_int = 2; +pub const SIGTTIN: ::c_int = 21; +pub const SIGTTOU: ::c_int = 22; +pub const SIGXCPU: ::c_int = 24; +pub const SIGXFSZ: ::c_int = 25; +pub const SIGVTALRM: ::c_int = 26; +pub const SIGPROF: ::c_int = 27; +pub const SIGWINCH: ::c_int = 28; +pub const SIGCHLD: ::c_int = 17; +pub const SIGBUS: ::c_int = 7; +pub const SIGUSR1: ::c_int = 10; +pub const SIGUSR2: ::c_int = 12; +pub const SIGCONT: ::c_int = 18; +pub const SIGSTOP: ::c_int = 19; +pub const SIGTSTP: ::c_int = 20; +pub const SIGURG: ::c_int = 23; +pub const SIGIO: ::c_int = 29; +pub const SIGSYS: ::c_int = 31; +pub const SIGSTKFLT: ::c_int = 16; +pub const SIGPOLL: ::c_int = 29; +pub const SIGPWR: ::c_int = 30; +pub const SIG_SETMASK: ::c_int = 2; +pub const SIG_BLOCK: ::c_int = 0; +pub const SIG_UNBLOCK: ::c_int = 1; +pub const POLLWRNORM: ::c_short = 256; +pub const POLLWRBAND: ::c_short = 512; +pub const O_ASYNC: ::c_int = 8192; +pub const O_NDELAY: ::c_int = 2048; +pub const EFD_NONBLOCK: ::c_int = 2048; +pub const F_GETLK: ::c_int = 5; +pub const F_GETOWN: ::c_int = 9; +pub const F_SETOWN: ::c_int = 8; +pub const SFD_NONBLOCK: ::c_int = 2048; +pub const TCSANOW: ::c_int = 0; +pub const TCSADRAIN: ::c_int = 1; +pub const TCSAFLUSH: ::c_int = 2; +pub const TIOCLINUX: ::c_ulong = 21532; +pub const TIOCGSERIAL: ::c_ulong = 21534; +pub const TIOCEXCL: ::c_ulong = 21516; +pub const TIOCNXCL: ::c_ulong = 21517; +pub const TIOCSCTTY: ::c_ulong = 21518; +pub const TIOCSTI: ::c_ulong = 21522; +pub const TIOCMGET: ::c_ulong = 21525; +pub const TIOCMBIS: ::c_ulong = 21526; +pub const TIOCMBIC: ::c_ulong = 21527; +pub const TIOCMSET: ::c_ulong = 21528; +pub const TIOCCONS: ::c_ulong = 21533; +pub const TIOCM_ST: ::c_int = 8; +pub const TIOCM_SR: ::c_int = 16; +pub const TIOCM_CTS: ::c_int = 32; +pub const TIOCM_CAR: ::c_int = 64; +pub const TIOCM_RNG: ::c_int = 128; +pub const TIOCM_DSR: ::c_int = 256; + +pub const __SIZEOF_PTHREAD_CONDATTR_T: usize = 4; +pub const __SIZEOF_PTHREAD_MUTEXATTR_T: usize = 4; +pub const O_DIRECT: ::c_int = 16384; +pub const O_DIRECTORY: ::c_int = 65536; +pub const O_NOFOLLOW: ::c_int = 131072; +pub const MAP_HUGETLB: ::c_int = 262144; +pub const MAP_LOCKED: ::c_int = 8192; +pub const MAP_NORESERVE: ::c_int = 16384; +pub const MAP_ANON: ::c_int = 32; +pub const MAP_ANONYMOUS: ::c_int = 32; +pub const MAP_DENYWRITE: ::c_int = 2048; +pub const MAP_EXECUTABLE: ::c_int = 4096; +pub const MAP_POPULATE: ::c_int = 32768; +pub const MAP_NONBLOCK: ::c_int = 65536; +pub const MAP_STACK: ::c_int = 131072; +pub const MAP_SYNC: ::c_int = 0x080000; +pub const EDEADLOCK: ::c_int = 35; +pub const EUCLEAN: ::c_int = 117; +pub const ENOTNAM: ::c_int = 118; +pub const ENAVAIL: ::c_int = 119; +pub const EISNAM: ::c_int = 120; +pub const EREMOTEIO: ::c_int = 121; +pub const FIOCLEX: ::c_int = 21585; +pub const FIONCLEX: ::c_int = 21584; +pub const FIONBIO: ::c_int = 21537; +pub const MCL_CURRENT: ::c_int = 1; +pub const MCL_FUTURE: ::c_int = 2; +pub const SIGSTKSZ: ::size_t = 8192; +pub const MINSIGSTKSZ: ::size_t = 2048; +pub const CBAUD: ::tcflag_t = 4111; +pub const TAB1: ::tcflag_t = 2048; +pub const TAB2: ::tcflag_t = 4096; +pub const TAB3: ::tcflag_t = 6144; +pub const CR1: ::tcflag_t = 512; +pub const CR2: ::tcflag_t = 1024; +pub const CR3: ::tcflag_t = 1536; +pub const FF1: ::tcflag_t = 32768; +pub const BS1: ::tcflag_t = 8192; +pub const VT1: ::tcflag_t = 16384; +pub const VWERASE: usize = 14; +pub const VREPRINT: usize = 12; +pub const VSUSP: usize = 10; +pub const VSTART: usize = 8; +pub const VSTOP: usize = 9; +pub const VDISCARD: usize = 13; +pub const VTIME: usize = 5; +pub const IXON: ::tcflag_t = 1024; +pub const IXOFF: ::tcflag_t = 4096; +pub const ONLCR: ::tcflag_t = 4; +pub const CSIZE: ::tcflag_t = 48; +pub const CS6: ::tcflag_t = 16; +pub const CS7: ::tcflag_t = 32; +pub const CS8: ::tcflag_t = 48; +pub const CSTOPB: ::tcflag_t = 64; +pub const CREAD: ::tcflag_t = 128; +pub const PARENB: ::tcflag_t = 256; +pub const PARODD: ::tcflag_t = 512; +pub const HUPCL: ::tcflag_t = 1024; +pub const CLOCAL: ::tcflag_t = 2048; +pub const ECHOKE: ::tcflag_t = 2048; +pub const ECHOE: ::tcflag_t = 16; +pub const ECHOK: ::tcflag_t = 32; +pub const ECHONL: ::tcflag_t = 64; +pub const ECHOPRT: ::tcflag_t = 1024; +pub const ECHOCTL: ::tcflag_t = 512; +pub const ISIG: ::tcflag_t = 1; +pub const ICANON: ::tcflag_t = 2; +pub const PENDIN: ::tcflag_t = 16384; +pub const NOFLSH: ::tcflag_t = 128; +pub const CIBAUD: ::tcflag_t = 269418496; +pub const CBAUDEX: ::tcflag_t = 4096; +pub const VSWTC: usize = 7; +pub const OLCUC: ::tcflag_t = 2; +pub const NLDLY: ::tcflag_t = 256; +pub const CRDLY: ::tcflag_t = 1536; +pub const TABDLY: ::tcflag_t = 6144; +pub const BSDLY: ::tcflag_t = 8192; +pub const FFDLY: ::tcflag_t = 32768; +pub const VTDLY: ::tcflag_t = 16384; +pub const XTABS: ::tcflag_t = 6144; +pub const B0: ::speed_t = 0; +pub const B50: ::speed_t = 1; +pub const B75: ::speed_t = 2; +pub const B110: ::speed_t = 3; +pub const B134: ::speed_t = 4; +pub const B150: ::speed_t = 5; +pub const B200: ::speed_t = 6; +pub const B300: ::speed_t = 7; +pub const B600: ::speed_t = 8; +pub const B1200: ::speed_t = 9; +pub const B1800: ::speed_t = 10; +pub const B2400: ::speed_t = 11; +pub const B4800: ::speed_t = 12; +pub const B9600: ::speed_t = 13; +pub const B19200: ::speed_t = 14; +pub const B38400: ::speed_t = 15; +pub const EXTA: ::speed_t = 14; +pub const EXTB: ::speed_t = 15; +pub const B57600: ::speed_t = 4097; +pub const B115200: ::speed_t = 4098; +pub const B230400: ::speed_t = 4099; +pub const B460800: ::speed_t = 4100; +pub const B500000: ::speed_t = 4101; +pub const B576000: ::speed_t = 4102; +pub const B921600: ::speed_t = 4103; +pub const B1000000: ::speed_t = 4104; +pub const B1152000: ::speed_t = 4105; +pub const B1500000: ::speed_t = 4106; +pub const B2000000: ::speed_t = 4107; +pub const B2500000: ::speed_t = 4108; +pub const B3000000: ::speed_t = 4109; +pub const B3500000: ::speed_t = 4110; +pub const B4000000: ::speed_t = 4111; +pub const VEOL: usize = 11; +pub const VEOL2: usize = 16; +pub const VMIN: usize = 6; +pub const IEXTEN: ::tcflag_t = 32768; +pub const TOSTOP: ::tcflag_t = 256; +pub const FLUSHO: ::tcflag_t = 4096; +pub const EXTPROC: ::tcflag_t = 65536; +pub const TCGETS: ::c_int = 21505; +pub const TCSETS: ::c_int = 21506; +pub const TCSETSW: ::c_int = 21507; +pub const TCSETSF: ::c_int = 21508; +pub const TCGETA: ::c_int = 21509; +pub const TCSETA: ::c_int = 21510; +pub const TCSETAW: ::c_int = 21511; +pub const TCSETAF: ::c_int = 21512; +pub const TCSBRK: ::c_int = 21513; +pub const TCXONC: ::c_int = 21514; +pub const TCFLSH: ::c_int = 21515; +pub const TIOCINQ: ::c_int = 21531; +pub const TIOCGPGRP: ::c_int = 21519; +pub const TIOCSPGRP: ::c_int = 21520; +pub const TIOCOUTQ: ::c_int = 21521; +pub const TIOCGWINSZ: ::c_int = 21523; +pub const TIOCSWINSZ: ::c_int = 21524; +pub const FIONREAD: ::c_int = 21531; +pub const __SIZEOF_PTHREAD_MUTEX_T: usize = 40; +pub const __SIZEOF_PTHREAD_RWLOCK_T: usize = 56; + +pub const SYS_read: ::c_long = 63; +pub const SYS_write: ::c_long = 64; +pub const SYS_close: ::c_long = 57; +pub const SYS_fstat: ::c_long = 80; +pub const SYS_lseek: ::c_long = 62; +pub const SYS_mmap: ::c_long = 222; +pub const SYS_mprotect: ::c_long = 226; +pub const SYS_munmap: ::c_long = 215; +pub const SYS_brk: ::c_long = 214; +pub const SYS_rt_sigaction: ::c_long = 134; +pub const SYS_rt_sigprocmask: ::c_long = 135; +pub const SYS_rt_sigreturn: ::c_long = 139; +pub const SYS_ioctl: ::c_long = 29; +pub const SYS_pread64: ::c_long = 67; +pub const SYS_pwrite64: ::c_long = 68; +pub const SYS_readv: ::c_long = 65; +pub const SYS_writev: ::c_long = 66; +pub const SYS_sched_yield: ::c_long = 124; +pub const SYS_mremap: ::c_long = 216; +pub const SYS_msync: ::c_long = 227; +pub const SYS_mincore: ::c_long = 232; +pub const SYS_madvise: ::c_long = 233; +pub const SYS_shmget: ::c_long = 194; +pub const SYS_shmat: ::c_long = 196; +pub const SYS_shmctl: ::c_long = 195; +pub const SYS_dup: ::c_long = 23; +pub const SYS_nanosleep: ::c_long = 101; +pub const SYS_getitimer: ::c_long = 102; +pub const SYS_setitimer: ::c_long = 103; +pub const SYS_getpid: ::c_long = 172; +pub const SYS_sendfile: ::c_long = 71; +pub const SYS_socket: ::c_long = 198; +pub const SYS_connect: ::c_long = 203; +pub const SYS_accept: ::c_long = 202; +pub const SYS_sendto: ::c_long = 206; +pub const SYS_recvfrom: ::c_long = 207; +pub const SYS_sendmsg: ::c_long = 211; +pub const SYS_recvmsg: ::c_long = 212; +pub const SYS_shutdown: ::c_long = 210; +pub const SYS_bind: ::c_long = 200; +pub const SYS_listen: ::c_long = 201; +pub const SYS_getsockname: ::c_long = 204; +pub const SYS_getpeername: ::c_long = 205; +pub const SYS_socketpair: ::c_long = 199; +pub const SYS_setsockopt: ::c_long = 208; +pub const SYS_getsockopt: ::c_long = 209; +pub const SYS_clone: ::c_long = 220; +pub const SYS_execve: ::c_long = 221; +pub const SYS_exit: ::c_long = 93; +pub const SYS_wait4: ::c_long = 260; +pub const SYS_kill: ::c_long = 129; +pub const SYS_uname: ::c_long = 160; +pub const SYS_semget: ::c_long = 190; +pub const SYS_semop: ::c_long = 193; +pub const SYS_semctl: ::c_long = 191; +pub const SYS_shmdt: ::c_long = 197; +pub const SYS_msgget: ::c_long = 186; +pub const SYS_msgsnd: ::c_long = 189; +pub const SYS_msgrcv: ::c_long = 188; +pub const SYS_msgctl: ::c_long = 187; +pub const SYS_fcntl: ::c_long = 25; +pub const SYS_flock: ::c_long = 32; +pub const SYS_fsync: ::c_long = 82; +pub const SYS_fdatasync: ::c_long = 83; +pub const SYS_truncate: ::c_long = 45; +pub const SYS_ftruncate: ::c_long = 46; +pub const SYS_getcwd: ::c_long = 17; +pub const SYS_chdir: ::c_long = 49; +pub const SYS_fchdir: ::c_long = 50; +pub const SYS_fchmod: ::c_long = 52; +pub const SYS_fchown: ::c_long = 55; +pub const SYS_umask: ::c_long = 166; +pub const SYS_gettimeofday: ::c_long = 169; +pub const SYS_getrlimit: ::c_long = 163; +pub const SYS_getrusage: ::c_long = 165; +pub const SYS_sysinfo: ::c_long = 179; +pub const SYS_times: ::c_long = 153; +pub const SYS_ptrace: ::c_long = 117; +pub const SYS_getuid: ::c_long = 174; +pub const SYS_syslog: ::c_long = 116; +pub const SYS_getgid: ::c_long = 176; +pub const SYS_setuid: ::c_long = 146; +pub const SYS_setgid: ::c_long = 144; +pub const SYS_geteuid: ::c_long = 175; +pub const SYS_getegid: ::c_long = 177; +pub const SYS_setpgid: ::c_long = 154; +pub const SYS_getppid: ::c_long = 173; +pub const SYS_setsid: ::c_long = 157; +pub const SYS_setreuid: ::c_long = 145; +pub const SYS_setregid: ::c_long = 143; +pub const SYS_getgroups: ::c_long = 158; +pub const SYS_setgroups: ::c_long = 159; +pub const SYS_setresuid: ::c_long = 147; +pub const SYS_getresuid: ::c_long = 148; +pub const SYS_setresgid: ::c_long = 149; +pub const SYS_getresgid: ::c_long = 150; +pub const SYS_getpgid: ::c_long = 155; +pub const SYS_setfsuid: ::c_long = 151; +pub const SYS_setfsgid: ::c_long = 152; +pub const SYS_getsid: ::c_long = 156; +pub const SYS_capget: ::c_long = 90; +pub const SYS_capset: ::c_long = 91; +pub const SYS_rt_sigpending: ::c_long = 136; +pub const SYS_rt_sigtimedwait: ::c_long = 137; +pub const SYS_rt_sigqueueinfo: ::c_long = 138; +pub const SYS_rt_sigsuspend: ::c_long = 133; +pub const SYS_sigaltstack: ::c_long = 132; +pub const SYS_personality: ::c_long = 92; +pub const SYS_statfs: ::c_long = 43; +pub const SYS_fstatfs: ::c_long = 44; +pub const SYS_getpriority: ::c_long = 141; +pub const SYS_setpriority: ::c_long = 140; +pub const SYS_sched_setparam: ::c_long = 118; +pub const SYS_sched_getparam: ::c_long = 121; +pub const SYS_sched_setscheduler: ::c_long = 119; +pub const SYS_sched_getscheduler: ::c_long = 120; +pub const SYS_sched_get_priority_max: ::c_long = 125; +pub const SYS_sched_get_priority_min: ::c_long = 126; +pub const SYS_sched_rr_get_interval: ::c_long = 127; +pub const SYS_mlock: ::c_long = 228; +pub const SYS_munlock: ::c_long = 229; +pub const SYS_mlockall: ::c_long = 230; +pub const SYS_munlockall: ::c_long = 231; +pub const SYS_vhangup: ::c_long = 58; +pub const SYS_pivot_root: ::c_long = 41; +pub const SYS_prctl: ::c_long = 167; +pub const SYS_adjtimex: ::c_long = 171; +pub const SYS_setrlimit: ::c_long = 164; +pub const SYS_chroot: ::c_long = 51; +pub const SYS_sync: ::c_long = 81; +pub const SYS_acct: ::c_long = 89; +pub const SYS_settimeofday: ::c_long = 170; +pub const SYS_mount: ::c_long = 40; +pub const SYS_umount2: ::c_long = 39; +pub const SYS_swapon: ::c_long = 224; +pub const SYS_swapoff: ::c_long = 225; +pub const SYS_reboot: ::c_long = 142; +pub const SYS_sethostname: ::c_long = 161; +pub const SYS_setdomainname: ::c_long = 162; +pub const SYS_init_module: ::c_long = 105; +pub const SYS_delete_module: ::c_long = 106; +pub const SYS_quotactl: ::c_long = 60; +pub const SYS_nfsservctl: ::c_long = 42; +pub const SYS_gettid: ::c_long = 178; +pub const SYS_readahead: ::c_long = 213; +pub const SYS_setxattr: ::c_long = 5; +pub const SYS_lsetxattr: ::c_long = 6; +pub const SYS_fsetxattr: ::c_long = 7; +pub const SYS_getxattr: ::c_long = 8; +pub const SYS_lgetxattr: ::c_long = 9; +pub const SYS_fgetxattr: ::c_long = 10; +pub const SYS_listxattr: ::c_long = 11; +pub const SYS_llistxattr: ::c_long = 12; +pub const SYS_flistxattr: ::c_long = 13; +pub const SYS_removexattr: ::c_long = 14; +pub const SYS_lremovexattr: ::c_long = 15; +pub const SYS_fremovexattr: ::c_long = 16; +pub const SYS_tkill: ::c_long = 130; +pub const SYS_futex: ::c_long = 98; +pub const SYS_sched_setaffinity: ::c_long = 122; +pub const SYS_sched_getaffinity: ::c_long = 123; +pub const SYS_io_setup: ::c_long = 0; +pub const SYS_io_destroy: ::c_long = 1; +pub const SYS_io_getevents: ::c_long = 4; +pub const SYS_io_submit: ::c_long = 2; +pub const SYS_io_cancel: ::c_long = 3; +pub const SYS_lookup_dcookie: ::c_long = 18; +pub const SYS_remap_file_pages: ::c_long = 234; +pub const SYS_getdents64: ::c_long = 61; +pub const SYS_set_tid_address: ::c_long = 96; +pub const SYS_restart_syscall: ::c_long = 128; +pub const SYS_semtimedop: ::c_long = 192; +pub const SYS_fadvise64: ::c_long = 223; +pub const SYS_timer_create: ::c_long = 107; +pub const SYS_timer_settime: ::c_long = 110; +pub const SYS_timer_gettime: ::c_long = 108; +pub const SYS_timer_getoverrun: ::c_long = 109; +pub const SYS_timer_delete: ::c_long = 111; +pub const SYS_clock_settime: ::c_long = 112; +pub const SYS_clock_gettime: ::c_long = 113; +pub const SYS_clock_getres: ::c_long = 114; +pub const SYS_clock_nanosleep: ::c_long = 115; +pub const SYS_exit_group: ::c_long = 94; +pub const SYS_epoll_ctl: ::c_long = 21; +pub const SYS_tgkill: ::c_long = 131; +pub const SYS_mbind: ::c_long = 235; +pub const SYS_set_mempolicy: ::c_long = 237; +pub const SYS_get_mempolicy: ::c_long = 236; +pub const SYS_mq_open: ::c_long = 180; +pub const SYS_mq_unlink: ::c_long = 181; +pub const SYS_mq_timedsend: ::c_long = 182; +pub const SYS_mq_timedreceive: ::c_long = 183; +pub const SYS_mq_notify: ::c_long = 184; +pub const SYS_mq_getsetattr: ::c_long = 185; +pub const SYS_kexec_load: ::c_long = 104; +pub const SYS_waitid: ::c_long = 95; +pub const SYS_add_key: ::c_long = 217; +pub const SYS_request_key: ::c_long = 218; +pub const SYS_keyctl: ::c_long = 219; +pub const SYS_ioprio_set: ::c_long = 30; +pub const SYS_ioprio_get: ::c_long = 31; +pub const SYS_inotify_add_watch: ::c_long = 27; +pub const SYS_inotify_rm_watch: ::c_long = 28; +pub const SYS_migrate_pages: ::c_long = 238; +pub const SYS_openat: ::c_long = 56; +pub const SYS_mkdirat: ::c_long = 34; +pub const SYS_mknodat: ::c_long = 33; +pub const SYS_fchownat: ::c_long = 54; +pub const SYS_newfstatat: ::c_long = 79; +pub const SYS_unlinkat: ::c_long = 35; +pub const SYS_linkat: ::c_long = 37; +pub const SYS_symlinkat: ::c_long = 36; +pub const SYS_readlinkat: ::c_long = 78; +pub const SYS_fchmodat: ::c_long = 53; +pub const SYS_faccessat: ::c_long = 48; +pub const SYS_pselect6: ::c_long = 72; +pub const SYS_ppoll: ::c_long = 73; +pub const SYS_unshare: ::c_long = 97; +pub const SYS_set_robust_list: ::c_long = 99; +pub const SYS_get_robust_list: ::c_long = 100; +pub const SYS_splice: ::c_long = 76; +pub const SYS_tee: ::c_long = 77; +pub const SYS_sync_file_range: ::c_long = 84; +pub const SYS_vmsplice: ::c_long = 75; +pub const SYS_move_pages: ::c_long = 239; +pub const SYS_utimensat: ::c_long = 88; +pub const SYS_epoll_pwait: ::c_long = 22; +pub const SYS_timerfd_create: ::c_long = 85; +pub const SYS_fallocate: ::c_long = 47; +pub const SYS_timerfd_settime: ::c_long = 86; +pub const SYS_timerfd_gettime: ::c_long = 87; +pub const SYS_accept4: ::c_long = 242; +pub const SYS_signalfd4: ::c_long = 74; +pub const SYS_eventfd2: ::c_long = 19; +pub const SYS_epoll_create1: ::c_long = 20; +pub const SYS_dup3: ::c_long = 24; +pub const SYS_pipe2: ::c_long = 59; +pub const SYS_inotify_init1: ::c_long = 26; +pub const SYS_preadv: ::c_long = 69; +pub const SYS_pwritev: ::c_long = 70; +pub const SYS_rt_tgsigqueueinfo: ::c_long = 240; +pub const SYS_perf_event_open: ::c_long = 241; +pub const SYS_recvmmsg: ::c_long = 243; +pub const SYS_fanotify_init: ::c_long = 262; +pub const SYS_fanotify_mark: ::c_long = 263; +pub const SYS_prlimit64: ::c_long = 261; +pub const SYS_name_to_handle_at: ::c_long = 264; +pub const SYS_open_by_handle_at: ::c_long = 265; +pub const SYS_clock_adjtime: ::c_long = 266; +pub const SYS_syncfs: ::c_long = 267; +pub const SYS_sendmmsg: ::c_long = 269; +pub const SYS_setns: ::c_long = 268; +pub const SYS_getcpu: ::c_long = 168; +pub const SYS_process_vm_readv: ::c_long = 270; +pub const SYS_process_vm_writev: ::c_long = 271; +pub const SYS_kcmp: ::c_long = 272; +pub const SYS_finit_module: ::c_long = 273; +pub const SYS_sched_setattr: ::c_long = 274; +pub const SYS_sched_getattr: ::c_long = 275; +pub const SYS_renameat2: ::c_long = 276; +pub const SYS_seccomp: ::c_long = 277; +pub const SYS_getrandom: ::c_long = 278; +pub const SYS_memfd_create: ::c_long = 279; +pub const SYS_bpf: ::c_long = 280; +pub const SYS_execveat: ::c_long = 281; +pub const SYS_userfaultfd: ::c_long = 282; +pub const SYS_membarrier: ::c_long = 283; +pub const SYS_mlock2: ::c_long = 284; +pub const SYS_copy_file_range: ::c_long = 285; +pub const SYS_preadv2: ::c_long = 286; +pub const SYS_pwritev2: ::c_long = 287; +pub const SYS_pkey_mprotect: ::c_long = 288; +pub const SYS_pkey_alloc: ::c_long = 289; +pub const SYS_pkey_free: ::c_long = 290; +pub const SYS_statx: ::c_long = 291; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; + +cfg_if! { + if #[cfg(libc_align)] { + mod align; + pub use self::align::*; + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/x86/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/x86/mod.rs index 9d00b5253..c319b91b6 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/x86/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b32/x86/mod.rs @@ -154,24 +154,6 @@ s! { pub f_namemax: ::c_ulong, __f_spare: [::c_int; 6], } - - pub struct nlmsghdr { - pub nlmsg_len: u32, - pub nlmsg_type: u16, - pub nlmsg_flags: u16, - pub nlmsg_seq: u32, - pub nlmsg_pid: u32, - } - - pub struct nlmsgerr { - pub error: ::c_int, - pub msg: nlmsghdr, - } - - pub struct nlattr { - pub nla_len: u16, - pub nla_type: u16, - } } s_no_extra_traits! { @@ -239,17 +221,6 @@ pub const O_NOFOLLOW: ::c_int = 0x20000; pub const O_ASYNC: ::c_int = 0x2000; pub const O_LARGEFILE: ::c_int = 0o0100000; -pub const FIOCLEX: ::c_int = 0x5451; -pub const FIONCLEX: ::c_int = 0x5450; -pub const FIONBIO: ::c_int = 0x5421; - -pub const RLIMIT_RSS: ::c_int = 5; -pub const RLIMIT_NOFILE: ::c_int = 7; -pub const RLIMIT_AS: ::c_int = 9; -pub const RLIMIT_NPROC: ::c_int = 6; -pub const RLIMIT_MEMLOCK: ::c_int = 8; -pub const RLIMIT_NLIMITS: ::c_int = 15; - pub const MADV_SOFT_OFFLINE: ::c_int = 101; pub const MCL_CURRENT: ::c_int = 0x0001; pub const MCL_FUTURE: ::c_int = 0x0002; @@ -340,14 +311,12 @@ pub const MAP_NORESERVE: ::c_int = 0x04000; pub const MAP_POPULATE: ::c_int = 0x08000; pub const MAP_NONBLOCK: ::c_int = 0x010000; pub const MAP_STACK: ::c_int = 0x020000; -pub const MAP_SYNC : ::c_int = 0x080000; +pub const MAP_SYNC: ::c_int = 0x080000; pub const SOCK_STREAM: ::c_int = 1; pub const SOCK_DGRAM: ::c_int = 2; pub const SOCK_SEQPACKET: ::c_int = 5; -pub const SOL_SOCKET: ::c_int = 1; - pub const EDEADLK: ::c_int = 35; pub const ENAMETOOLONG: ::c_int = 36; pub const ENOLCK: ::c_int = 37; @@ -434,32 +403,6 @@ pub const ENOTRECOVERABLE: ::c_int = 131; pub const ERFKILL: ::c_int = 132; pub const EHWPOISON: ::c_int = 133; -pub const SO_REUSEADDR: ::c_int = 2; -pub const SO_TYPE: ::c_int = 3; -pub const SO_ERROR: ::c_int = 4; -pub const SO_DONTROUTE: ::c_int = 5; -pub const SO_BROADCAST: ::c_int = 6; -pub const SO_SNDBUF: ::c_int = 7; -pub const SO_RCVBUF: ::c_int = 8; -pub const SO_KEEPALIVE: ::c_int = 9; -pub const SO_OOBINLINE: ::c_int = 10; -pub const SO_NO_CHECK: ::c_int = 11; -pub const SO_PRIORITY: ::c_int = 12; -pub const SO_LINGER: ::c_int = 13; -pub const SO_BSDCOMPAT: ::c_int = 14; -pub const SO_REUSEPORT: ::c_int = 15; -pub const SO_PASSCRED: ::c_int = 16; -pub const SO_PEERCRED: ::c_int = 17; -pub const SO_RCVLOWAT: ::c_int = 18; -pub const SO_SNDLOWAT: ::c_int = 19; -pub const SO_RCVTIMEO: ::c_int = 20; -pub const SO_SNDTIMEO: ::c_int = 21; -pub const SO_ACCEPTCONN: ::c_int = 30; -pub const SO_SNDBUFFORCE: ::c_int = 32; -pub const SO_RCVBUFFORCE: ::c_int = 33; -pub const SO_PROTOCOL: ::c_int = 38; -pub const SO_DOMAIN: ::c_int = 39; - pub const SA_ONSTACK: ::c_int = 0x08000000; pub const SA_SIGINFO: ::c_int = 0x00000004; pub const SA_NOCLDWAIT: ::c_int = 0x00000002; @@ -510,55 +453,9 @@ pub const IEXTEN: ::tcflag_t = 0x00008000; pub const TOSTOP: ::tcflag_t = 0x00000100; pub const FLUSHO: ::tcflag_t = 0x00001000; -pub const TCGETS: ::c_int = 0x5401; -pub const TCSETS: ::c_int = 0x5402; -pub const TCSETSW: ::c_int = 0x5403; -pub const TCSETSF: ::c_int = 0x5404; -pub const TCGETA: ::c_int = 0x5405; -pub const TCSETA: ::c_int = 0x5406; -pub const TCSETAW: ::c_int = 0x5407; -pub const TCSETAF: ::c_int = 0x5408; -pub const TCSBRK: ::c_int = 0x5409; -pub const TCXONC: ::c_int = 0x540A; -pub const TCFLSH: ::c_int = 0x540B; -pub const TIOCGSOFTCAR: ::c_int = 0x5419; -pub const TIOCSSOFTCAR: ::c_int = 0x541A; -pub const TIOCLINUX: ::c_int = 0x541C; -pub const TIOCGSERIAL: ::c_int = 0x541E; -pub const TIOCEXCL: ::c_int = 0x540C; -pub const TIOCNXCL: ::c_int = 0x540D; -pub const TIOCSCTTY: ::c_int = 0x540E; -pub const TIOCGPGRP: ::c_int = 0x540F; -pub const TIOCSPGRP: ::c_int = 0x5410; -pub const TIOCOUTQ: ::c_int = 0x5411; -pub const TIOCSTI: ::c_int = 0x5412; -pub const TIOCGWINSZ: ::c_int = 0x5413; -pub const TIOCSWINSZ: ::c_int = 0x5414; -pub const TIOCMGET: ::c_int = 0x5415; -pub const TIOCMBIS: ::c_int = 0x5416; -pub const TIOCMBIC: ::c_int = 0x5417; -pub const TIOCMSET: ::c_int = 0x5418; -pub const FIONREAD: ::c_int = 0x541B; -pub const TIOCCONS: ::c_int = 0x541D; - -pub const TIOCGRS485: ::c_int = 0x542E; -pub const TIOCSRS485: ::c_int = 0x542F; - pub const POLLWRNORM: ::c_short = 0x100; pub const POLLWRBAND: ::c_short = 0x200; -pub const TIOCM_LE: ::c_int = 0x001; -pub const TIOCM_DTR: ::c_int = 0x002; -pub const TIOCM_RTS: ::c_int = 0x004; -pub const TIOCM_ST: ::c_int = 0x008; -pub const TIOCM_SR: ::c_int = 0x010; -pub const TIOCM_CTS: ::c_int = 0x020; -pub const TIOCM_CAR: ::c_int = 0x040; -pub const TIOCM_RNG: ::c_int = 0x080; -pub const TIOCM_DSR: ::c_int = 0x100; -pub const TIOCM_CD: ::c_int = TIOCM_CAR; -pub const TIOCM_RI: ::c_int = TIOCM_RNG; - // Syscall table pub const SYS_restart_syscall: ::c_long = 0; pub const SYS_exit: ::c_long = 1; @@ -940,6 +837,33 @@ pub const SYS_pkey_mprotect: ::c_long = 380; pub const SYS_pkey_alloc: ::c_long = 381; pub const SYS_pkey_free: ::c_long = 382; pub const SYS_statx: ::c_long = 383; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; +pub const SYS_quotactl_fd: ::c_long = 443; +pub const SYS_landlock_create_ruleset: ::c_long = 444; +pub const SYS_landlock_add_rule: ::c_long = 445; +pub const SYS_landlock_restrict_self: ::c_long = 446; +pub const SYS_memfd_secret: ::c_long = 447; +pub const SYS_process_mrelease: ::c_long = 448; +pub const SYS_futex_waitv: ::c_long = 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 450; // offsets in user_regs_structs, from sys/reg.h pub const EBX: ::c_int = 0; @@ -961,11 +885,7 @@ pub const UESP: ::c_int = 15; pub const SS: ::c_int = 16; extern "C" { - pub fn getrandom( - buf: *mut ::c_void, - buflen: ::size_t, - flags: ::c_uint, - ) -> ::ssize_t; + pub fn getrandom(buf: *mut ::c_void, buflen: ::size_t, flags: ::c_uint) -> ::ssize_t; } cfg_if! { diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/aarch64/align.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/aarch64/align.rs index 8e949963a..a4bf9bff4 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/aarch64/align.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/aarch64/align.rs @@ -5,3 +5,38 @@ s_no_extra_traits! { priv_: [f32; 8] } } + +s! { + pub struct ucontext_t { + pub uc_flags: ::c_ulong, + pub uc_link: *mut ucontext_t, + pub uc_stack: ::stack_t, + pub uc_sigmask: ::sigset_t, + pub uc_mcontext: mcontext_t, + } + + #[repr(align(16))] + pub struct mcontext_t { + pub fault_address: ::c_ulong, + pub regs: [::c_ulong; 31], + pub sp: ::c_ulong, + pub pc: ::c_ulong, + pub pstate: ::c_ulong, + __reserved: [[u64; 32]; 16], + } + + #[repr(align(8))] + pub struct clone_args { + pub flags: ::c_ulonglong, + pub pidfd: ::c_ulonglong, + pub child_tid: ::c_ulonglong, + pub parent_tid: ::c_ulonglong, + pub exit_signal: ::c_ulonglong, + pub stack: ::c_ulonglong, + pub stack_size: ::c_ulonglong, + pub tls: ::c_ulonglong, + pub set_tid: ::c_ulonglong, + pub set_tid_size: ::c_ulonglong, + pub cgroup: ::c_ulonglong, + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/aarch64/int128.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/aarch64/int128.rs new file mode 100644 index 000000000..4535e73ee --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/aarch64/int128.rs @@ -0,0 +1,7 @@ +s! { + pub struct user_fpsimd_struct { + pub vregs: [::__uint128_t; 32], + pub fpsr: u32, + pub fpcr: u32, + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/aarch64/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/aarch64/mod.rs index d7e06cd21..14b4bc6d6 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/aarch64/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/aarch64/mod.rs @@ -1,5 +1,6 @@ pub type c_char = u8; pub type __u64 = ::c_ulonglong; +pub type __s64 = ::c_longlong; pub type wchar_t = u32; pub type nlink_t = u32; pub type blksize_t = ::c_int; @@ -49,6 +50,13 @@ s! { __unused: [::c_uint; 2], } + pub struct user_regs_struct { + pub regs: [::c_ulonglong; 31], + pub sp: ::c_ulonglong, + pub pc: ::c_ulonglong, + pub pstate: ::c_ulonglong, + } + pub struct ipc_perm { pub __ipc_perm_key: ::key_t, pub uid: ::uid_t, @@ -160,6 +168,40 @@ pub const ENOTRECOVERABLE: ::c_int = 131; pub const ERFKILL: ::c_int = 132; pub const EHWPOISON: ::c_int = 133; +// bits/hwcap.h +pub const HWCAP_FP: ::c_ulong = 1 << 0; +pub const HWCAP_ASIMD: ::c_ulong = 1 << 1; +pub const HWCAP_EVTSTRM: ::c_ulong = 1 << 2; +pub const HWCAP_AES: ::c_ulong = 1 << 3; +pub const HWCAP_PMULL: ::c_ulong = 1 << 4; +pub const HWCAP_SHA1: ::c_ulong = 1 << 5; +pub const HWCAP_SHA2: ::c_ulong = 1 << 6; +pub const HWCAP_CRC32: ::c_ulong = 1 << 7; +pub const HWCAP_ATOMICS: ::c_ulong = 1 << 8; +pub const HWCAP_FPHP: ::c_ulong = 1 << 9; +pub const HWCAP_ASIMDHP: ::c_ulong = 1 << 10; +pub const HWCAP_CPUID: ::c_ulong = 1 << 11; +pub const HWCAP_ASIMDRDM: ::c_ulong = 1 << 12; +pub const HWCAP_JSCVT: ::c_ulong = 1 << 13; +pub const HWCAP_FCMA: ::c_ulong = 1 << 14; +pub const HWCAP_LRCPC: ::c_ulong = 1 << 15; +pub const HWCAP_DCPOP: ::c_ulong = 1 << 16; +pub const HWCAP_SHA3: ::c_ulong = 1 << 17; +pub const HWCAP_SM3: ::c_ulong = 1 << 18; +pub const HWCAP_SM4: ::c_ulong = 1 << 19; +pub const HWCAP_ASIMDDP: ::c_ulong = 1 << 20; +pub const HWCAP_SHA512: ::c_ulong = 1 << 21; +pub const HWCAP_SVE: ::c_ulong = 1 << 22; +pub const HWCAP_ASIMDFHM: ::c_ulong = 1 << 23; +pub const HWCAP_DIT: ::c_ulong = 1 << 24; +pub const HWCAP_USCAT: ::c_ulong = 1 << 25; +pub const HWCAP_ILRCPC: ::c_ulong = 1 << 26; +pub const HWCAP_FLAGM: ::c_ulong = 1 << 27; +pub const HWCAP_SSBS: ::c_ulong = 1 << 28; +pub const HWCAP_SB: ::c_ulong = 1 << 29; +pub const HWCAP_PACA: ::c_ulong = 1 << 30; +pub const HWCAP_PACG: ::c_ulong = 1 << 31; + pub const MAP_ANON: ::c_int = 0x0020; pub const MAP_GROWSDOWN: ::c_int = 0x0100; pub const MAP_DENYWRITE: ::c_int = 0x0800; @@ -170,30 +212,10 @@ pub const MAP_POPULATE: ::c_int = 0x08000; pub const MAP_NONBLOCK: ::c_int = 0x010000; pub const MAP_STACK: ::c_int = 0x020000; pub const MAP_HUGETLB: ::c_int = 0x040000; -pub const MAP_SYNC : ::c_int = 0x080000; +pub const MAP_SYNC: ::c_int = 0x080000; pub const SOCK_STREAM: ::c_int = 1; pub const SOCK_DGRAM: ::c_int = 2; -pub const SOL_SOCKET: ::c_int = 1; -pub const SO_REUSEADDR: ::c_int = 2; -pub const SO_TYPE: ::c_int = 3; -pub const SO_ERROR: ::c_int = 4; -pub const SO_DONTROUTE: ::c_int = 5; -pub const SO_BROADCAST: ::c_int = 6; -pub const SO_SNDBUF: ::c_int = 7; -pub const SO_RCVBUF: ::c_int = 8; -pub const SO_KEEPALIVE: ::c_int = 9; -pub const SO_OOBINLINE: ::c_int = 10; -pub const SO_NO_CHECK: ::c_int = 11; -pub const SO_PRIORITY: ::c_int = 12; -pub const SO_LINGER: ::c_int = 13; -pub const SO_BSDCOMPAT: ::c_int = 14; -pub const SO_REUSEPORT: ::c_int = 15; -pub const SO_ACCEPTCONN: ::c_int = 30; -pub const SO_SNDBUFFORCE: ::c_int = 32; -pub const SO_RCVBUFFORCE: ::c_int = 33; -pub const SO_PROTOCOL: ::c_int = 38; -pub const SO_DOMAIN: ::c_int = 39; pub const SA_ONSTACK: ::c_int = 0x08000000; pub const SA_SIGINFO: ::c_int = 0x00000004; @@ -284,6 +306,10 @@ pub const SYS_umount2: ::c_long = 39; pub const SYS_mount: ::c_long = 40; pub const SYS_pivot_root: ::c_long = 41; pub const SYS_nfsservctl: ::c_long = 42; +pub const SYS_statfs: ::c_long = 43; +pub const SYS_fstatfs: ::c_long = 44; +pub const SYS_truncate: ::c_long = 45; +pub const SYS_ftruncate: ::c_long = 46; pub const SYS_fallocate: ::c_long = 47; pub const SYS_faccessat: ::c_long = 48; pub const SYS_chdir: ::c_long = 49; @@ -511,9 +537,34 @@ pub const SYS_pkey_mprotect: ::c_long = 288; pub const SYS_pkey_alloc: ::c_long = 289; pub const SYS_pkey_free: ::c_long = 290; pub const SYS_statx: ::c_long = 291; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; +pub const SYS_quotactl_fd: ::c_long = 443; +pub const SYS_landlock_create_ruleset: ::c_long = 444; +pub const SYS_landlock_add_rule: ::c_long = 445; +pub const SYS_landlock_restrict_self: ::c_long = 446; +pub const SYS_memfd_secret: ::c_long = 447; +pub const SYS_process_mrelease: ::c_long = 448; +pub const SYS_futex_waitv: ::c_long = 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 450; -pub const RLIMIT_NLIMITS: ::c_int = 15; -pub const TIOCINQ: ::c_int = ::FIONREAD; pub const MCL_CURRENT: ::c_int = 0x0001; pub const MCL_FUTURE: ::c_int = 0x0002; pub const CBAUD: ::tcflag_t = 0o0010017; @@ -583,17 +634,9 @@ pub const B3000000: ::speed_t = 0o010015; pub const B3500000: ::speed_t = 0o010016; pub const B4000000: ::speed_t = 0o010017; -pub const FIOCLEX: ::c_int = 0x5451; -pub const FIONCLEX: ::c_int = 0x5450; -pub const FIONBIO: ::c_int = 0x5421; pub const EDEADLK: ::c_int = 35; pub const EDEADLOCK: ::c_int = EDEADLK; -pub const SO_PASSCRED: ::c_int = 16; -pub const SO_PEERCRED: ::c_int = 17; -pub const SO_RCVLOWAT: ::c_int = 18; -pub const SO_SNDLOWAT: ::c_int = 19; -pub const SO_RCVTIMEO: ::c_int = 20; -pub const SO_SNDTIMEO: ::c_int = 21; + pub const EXTPROC: ::tcflag_t = 0x00010000; pub const VEOL: usize = 11; pub const VEOL2: usize = 16; @@ -601,55 +644,6 @@ pub const VMIN: usize = 6; pub const IEXTEN: ::tcflag_t = 0x00008000; pub const TOSTOP: ::tcflag_t = 0x00000100; pub const FLUSHO: ::tcflag_t = 0x00001000; -pub const TCGETS: ::c_int = 0x5401; -pub const TCSETS: ::c_int = 0x5402; -pub const TCSETSW: ::c_int = 0x5403; -pub const TCSETSF: ::c_int = 0x5404; -pub const TCGETA: ::c_int = 0x5405; -pub const TCSETA: ::c_int = 0x5406; -pub const TCSETAW: ::c_int = 0x5407; -pub const TCSETAF: ::c_int = 0x5408; -pub const TCSBRK: ::c_int = 0x5409; -pub const TCXONC: ::c_int = 0x540A; -pub const TCFLSH: ::c_int = 0x540B; -pub const TIOCGSOFTCAR: ::c_int = 0x5419; -pub const TIOCSSOFTCAR: ::c_int = 0x541A; -pub const TIOCLINUX: ::c_int = 0x541C; -pub const TIOCGSERIAL: ::c_int = 0x541E; -pub const TIOCEXCL: ::c_int = 0x540C; -pub const TIOCNXCL: ::c_int = 0x540D; -pub const TIOCSCTTY: ::c_int = 0x540E; -pub const TIOCGPGRP: ::c_int = 0x540F; -pub const TIOCSPGRP: ::c_int = 0x5410; -pub const TIOCOUTQ: ::c_int = 0x5411; -pub const TIOCSTI: ::c_int = 0x5412; -pub const TIOCGWINSZ: ::c_int = 0x5413; -pub const TIOCSWINSZ: ::c_int = 0x5414; -pub const TIOCMGET: ::c_int = 0x5415; -pub const TIOCMBIS: ::c_int = 0x5416; -pub const TIOCMBIC: ::c_int = 0x5417; -pub const TIOCMSET: ::c_int = 0x5418; -pub const FIONREAD: ::c_int = 0x541B; -pub const TIOCCONS: ::c_int = 0x541D; - -pub const TIOCGRS485: ::c_int = 0x542E; -pub const TIOCSRS485: ::c_int = 0x542F; - -pub const TIOCM_LE: ::c_int = 0x001; -pub const TIOCM_DTR: ::c_int = 0x002; -pub const TIOCM_RTS: ::c_int = 0x004; -pub const TIOCM_ST: ::c_int = 0x008; -pub const TIOCM_SR: ::c_int = 0x010; -pub const TIOCM_CTS: ::c_int = 0x020; -pub const TIOCM_CAR: ::c_int = 0x040; -pub const TIOCM_RNG: ::c_int = 0x080; -pub const TIOCM_DSR: ::c_int = 0x100; -pub const TIOCM_CD: ::c_int = TIOCM_CAR; -pub const TIOCM_RI: ::c_int = TIOCM_RNG; - -extern "C" { - pub fn ioctl(fd: ::c_int, request: ::c_int, ...) -> ::c_int; -} cfg_if! { if #[cfg(libc_align)] { @@ -657,3 +651,10 @@ cfg_if! { pub use self::align::*; } } + +cfg_if! { + if #[cfg(libc_int128)] { + mod int128; + pub use self::int128::*; + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/mips64.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/mips64.rs index 7c6f96390..22ac91690 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/mips64.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/mips64.rs @@ -1,6 +1,7 @@ pub type c_char = i8; pub type wchar_t = i32; pub type __u64 = ::c_ulong; +pub type __s64 = ::c_long; pub type nlink_t = u64; pub type blksize_t = i64; @@ -83,24 +84,6 @@ s! { pub f_spare: [::c_ulong; 5], } - pub struct nlmsghdr { - pub nlmsg_len: u32, - pub nlmsg_type: u16, - pub nlmsg_flags: u16, - pub nlmsg_seq: u32, - pub nlmsg_pid: u32, - } - - pub struct nlmsgerr { - pub error: ::c_int, - pub msg: nlmsghdr, - } - - pub struct nlattr { - pub nla_len: u16, - pub nla_type: u16, - } - pub struct ipc_perm { pub __ipc_perm_key: ::key_t, pub uid: ::uid_t, @@ -443,6 +426,33 @@ pub const SYS_pkey_mprotect: ::c_long = 5000 + 323; pub const SYS_pkey_alloc: ::c_long = 5000 + 324; pub const SYS_pkey_free: ::c_long = 5000 + 325; pub const SYS_statx: ::c_long = 5000 + 326; +pub const SYS_pidfd_send_signal: ::c_long = 5000 + 424; +pub const SYS_io_uring_setup: ::c_long = 5000 + 425; +pub const SYS_io_uring_enter: ::c_long = 5000 + 426; +pub const SYS_io_uring_register: ::c_long = 5000 + 427; +pub const SYS_open_tree: ::c_long = 5000 + 428; +pub const SYS_move_mount: ::c_long = 5000 + 429; +pub const SYS_fsopen: ::c_long = 5000 + 430; +pub const SYS_fsconfig: ::c_long = 5000 + 431; +pub const SYS_fsmount: ::c_long = 5000 + 432; +pub const SYS_fspick: ::c_long = 5000 + 433; +pub const SYS_pidfd_open: ::c_long = 5000 + 434; +pub const SYS_clone3: ::c_long = 5000 + 435; +pub const SYS_close_range: ::c_long = 5000 + 436; +pub const SYS_openat2: ::c_long = 5000 + 437; +pub const SYS_pidfd_getfd: ::c_long = 5000 + 438; +pub const SYS_faccessat2: ::c_long = 5000 + 439; +pub const SYS_process_madvise: ::c_long = 5000 + 440; +pub const SYS_epoll_pwait2: ::c_long = 5000 + 441; +pub const SYS_mount_setattr: ::c_long = 5000 + 442; +pub const SYS_quotactl_fd: ::c_long = 5000 + 443; +pub const SYS_landlock_create_ruleset: ::c_long = 5000 + 444; +pub const SYS_landlock_add_rule: ::c_long = 5000 + 445; +pub const SYS_landlock_restrict_self: ::c_long = 5000 + 446; +pub const SYS_memfd_secret: ::c_long = 5000 + 447; +pub const SYS_process_mrelease: ::c_long = 5000 + 448; +pub const SYS_futex_waitv: ::c_long = 5000 + 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 5000 + 450; pub const O_DIRECT: ::c_int = 0x8000; pub const O_DIRECTORY: ::c_int = 0x10000; @@ -457,6 +467,7 @@ pub const O_SYNC: ::c_int = 0x4010; pub const O_RSYNC: ::c_int = 0x4010; pub const O_DSYNC: ::c_int = 0x10; pub const O_ASYNC: ::c_int = 0x1000; +pub const O_LARGEFILE: ::c_int = 0x2000; pub const EDEADLK: ::c_int = 45; pub const ENAMETOOLONG: ::c_int = 78; @@ -504,6 +515,7 @@ pub const ENOPROTOOPT: ::c_int = 99; pub const EPROTONOSUPPORT: ::c_int = 120; pub const ESOCKTNOSUPPORT: ::c_int = 121; pub const EOPNOTSUPP: ::c_int = 122; +pub const ENOTSUP: ::c_int = EOPNOTSUPP; pub const EPFNOSUPPORT: ::c_int = 123; pub const EAFNOSUPPORT: ::c_int = 124; pub const EADDRINUSE: ::c_int = 125; @@ -556,59 +568,6 @@ pub const MAP_HUGETLB: ::c_int = 0x080000; pub const SOCK_STREAM: ::c_int = 2; pub const SOCK_DGRAM: ::c_int = 1; -pub const SOL_SOCKET: ::c_int = 0xffff; - -pub const SO_REUSEADDR: ::c_int = 0x0004; -pub const SO_KEEPALIVE: ::c_int = 0x0008; -pub const SO_DONTROUTE: ::c_int = 0x0010; -pub const SO_BROADCAST: ::c_int = 0x0020; -pub const SO_LINGER: ::c_int = 0x0080; -pub const SO_OOBINLINE: ::c_int = 0x0100; -pub const SO_REUSEPORT: ::c_int = 0x0200; -pub const SO_TYPE: ::c_int = 0x1008; -pub const SO_ERROR: ::c_int = 0x1007; -pub const SO_SNDBUF: ::c_int = 0x1001; -pub const SO_RCVBUF: ::c_int = 0x1002; -pub const SO_SNDLOWAT: ::c_int = 0x1003; -pub const SO_RCVLOWAT: ::c_int = 0x1004; -pub const SO_SNDTIMEO: ::c_int = 0x1005; -pub const SO_RCVTIMEO: ::c_int = 0x1006; -pub const SO_ACCEPTCONN: ::c_int = 0x1009; -pub const SO_PROTOCOL: ::c_int = 0x1028; -pub const SO_DOMAIN: ::c_int = 0x1029; -pub const SO_NO_CHECK: ::c_int = 11; -pub const SO_PRIORITY: ::c_int = 12; -pub const SO_BSDCOMPAT: ::c_int = 14; -pub const SO_PASSCRED: ::c_int = 17; -pub const SO_PEERCRED: ::c_int = 18; -pub const SO_SECURITY_AUTHENTICATION: ::c_int = 22; -pub const SO_SECURITY_ENCRYPTION_TRANSPORT: ::c_int = 23; -pub const SO_SECURITY_ENCRYPTION_NETWORK: ::c_int = 24; -pub const SO_ATTACH_FILTER: ::c_int = 26; -pub const SO_DETACH_FILTER: ::c_int = 27; -pub const SO_GET_FILTER: ::c_int = SO_ATTACH_FILTER; -pub const SO_PEERNAME: ::c_int = 28; -pub const SO_PEERSEC: ::c_int = 30; -pub const SO_SNDBUFFORCE: ::c_int = 31; -pub const SO_RCVBUFFORCE: ::c_int = 33; -pub const SO_PASSSEC: ::c_int = 34; -pub const SO_TIMESTAMPNS: ::c_int = 35; -pub const SCM_TIMESTAMPNS: ::c_int = SO_TIMESTAMPNS; -pub const SO_WIFI_STATUS: ::c_int = 41; -pub const SCM_WIFI_STATUS: ::c_int = SO_WIFI_STATUS; -pub const SO_NOFCS: ::c_int = 43; -pub const SO_LOCK_FILTER: ::c_int = 44; -pub const SO_SELECT_ERR_QUEUE: ::c_int = 45; -pub const SO_MAX_PACING_RATE: ::c_int = 47; -pub const SO_BPF_EXTENSIONS: ::c_int = 48; -pub const SO_INCOMING_CPU: ::c_int = 49; -pub const SO_ATTACH_BPF: ::c_int = 50; -pub const SO_DETACH_BPF: ::c_int = SO_DETACH_FILTER; - -pub const FIOCLEX: ::c_int = 0x6601; -pub const FIONCLEX: ::c_int = 0x6602; -pub const FIONBIO: ::c_int = 0x667e; - pub const SA_ONSTACK: ::c_int = 0x08000000; pub const SA_SIGINFO: ::c_int = 0x00000008; pub const SA_NOCLDWAIT: ::c_int = 0x00010000; @@ -657,38 +616,6 @@ pub const F_OFD_GETLK: ::c_int = 36; pub const F_OFD_SETLK: ::c_int = 37; pub const F_OFD_SETLKW: ::c_int = 38; -pub const TCGETS: ::c_ulong = 0x540d; -pub const TCSETS: ::c_ulong = 0x540e; -pub const TCSETSW: ::c_ulong = 0x540f; -pub const TCSETSF: ::c_ulong = 0x5410; -pub const TCGETA: ::c_ulong = 0x5401; -pub const TCSETA: ::c_ulong = 0x5402; -pub const TCSETAW: ::c_ulong = 0x5403; -pub const TCSETAF: ::c_ulong = 0x5404; -pub const TCSBRK: ::c_ulong = 0x5405; -pub const TCXONC: ::c_ulong = 0x5406; -pub const TCFLSH: ::c_ulong = 0x5407; -pub const TIOCGSOFTCAR: ::c_ulong = 0x5481; -pub const TIOCSSOFTCAR: ::c_ulong = 0x5482; -pub const TIOCINQ: ::c_ulong = 0x467f; -pub const TIOCLINUX: ::c_ulong = 0x5483; -pub const TIOCGSERIAL: ::c_ulong = 0x5484; -pub const TIOCEXCL: ::c_ulong = 0x740d; -pub const TIOCNXCL: ::c_ulong = 0x740e; -pub const TIOCSCTTY: ::c_ulong = 0x5480; -pub const TIOCGPGRP: ::c_ulong = 0x40047477; -pub const TIOCSPGRP: ::c_ulong = 0x80047476; -pub const TIOCOUTQ: ::c_ulong = 0x7472; -pub const TIOCSTI: ::c_ulong = 0x5472; -pub const TIOCGWINSZ: ::c_ulong = 0x40087468; -pub const TIOCSWINSZ: ::c_ulong = 0x80087467; -pub const TIOCMGET: ::c_ulong = 0x741d; -pub const TIOCMBIS: ::c_ulong = 0x741b; -pub const TIOCMBIC: ::c_ulong = 0x741c; -pub const TIOCMSET: ::c_ulong = 0x741a; -pub const FIONREAD: ::c_ulong = 0x467f; -pub const TIOCCONS: ::c_ulong = 0x80047478; - pub const MCL_CURRENT: ::c_int = 0x0001; pub const MCL_FUTURE: ::c_int = 0x0002; @@ -744,7 +671,6 @@ pub const FFDLY: ::tcflag_t = 0o100000; pub const VTDLY: ::tcflag_t = 0o040000; pub const XTABS: ::tcflag_t = 0o014000; -pub const BOTHER: ::speed_t = 0o010000; pub const B57600: ::speed_t = 0o010001; pub const B115200: ::speed_t = 0o010002; pub const B230400: ::speed_t = 0o010003; @@ -761,15 +687,4 @@ pub const B3000000: ::speed_t = 0o010015; pub const B3500000: ::speed_t = 0o010016; pub const B4000000: ::speed_t = 0o010017; -pub const TIOCM_ST: ::c_int = 0x010; -pub const TIOCM_SR: ::c_int = 0x020; -pub const TIOCM_CTS: ::c_int = 0x040; -pub const TIOCM_CAR: ::c_int = 0x100; -pub const TIOCM_RNG: ::c_int = 0x200; -pub const TIOCM_DSR: ::c_int = 0x400; - pub const EHWPOISON: ::c_int = 168; - -extern "C" { - pub fn ioctl(fd: ::c_int, request: ::c_int, ...) -> ::c_int; -} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/mod.rs index e6a8fc81f..34c63bc69 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/mod.rs @@ -1,5 +1,6 @@ pub type c_long = i64; pub type c_ulong = u64; +pub type regoff_t = ::c_long; s! { pub struct statfs64 { @@ -132,22 +133,12 @@ s! { pub const __SIZEOF_PTHREAD_RWLOCK_T: usize = 56; pub const __SIZEOF_PTHREAD_MUTEX_T: usize = 40; -pub const RLIMIT_RSS: ::c_int = 5; -pub const RLIMIT_NOFILE: ::c_int = 7; -pub const RLIMIT_AS: ::c_int = 9; -pub const RLIMIT_NPROC: ::c_int = 6; -pub const RLIMIT_MEMLOCK: ::c_int = 8; - pub const SOCK_NONBLOCK: ::c_int = 2048; pub const SOCK_SEQPACKET: ::c_int = 5; extern "C" { - pub fn getrandom( - buf: *mut ::c_void, - buflen: ::size_t, - flags: ::c_uint, - ) -> ::ssize_t; + pub fn getrandom(buf: *mut ::c_void, buflen: ::size_t, flags: ::c_uint) -> ::ssize_t; } cfg_if! { @@ -160,9 +151,15 @@ cfg_if! { } else if #[cfg(any(target_arch = "powerpc64"))] { mod powerpc64; pub use self::powerpc64::*; + } else if #[cfg(any(target_arch = "s390x"))] { + mod s390x; + pub use self::s390x::*; } else if #[cfg(any(target_arch = "x86_64"))] { mod x86_64; pub use self::x86_64::*; + } else if #[cfg(any(target_arch = "riscv64"))] { + mod riscv64; + pub use self::riscv64::*; } else { // Unknown target_arch } diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/powerpc64.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/powerpc64.rs index 18fcd5c33..0bb4cf837 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/powerpc64.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/powerpc64.rs @@ -1,6 +1,7 @@ pub type c_char = u8; pub type wchar_t = i32; pub type __u64 = ::c_ulong; +pub type __s64 = ::c_long; pub type nlink_t = u64; pub type blksize_t = ::c_long; @@ -170,30 +171,10 @@ pub const MAP_POPULATE: ::c_int = 0x08000; pub const MAP_NONBLOCK: ::c_int = 0x010000; pub const MAP_STACK: ::c_int = 0x020000; pub const MAP_HUGETLB: ::c_int = 0x040000; -pub const MAP_SYNC : ::c_int = 0x080000; +pub const MAP_SYNC: ::c_int = 0x080000; pub const SOCK_STREAM: ::c_int = 1; pub const SOCK_DGRAM: ::c_int = 2; -pub const SOL_SOCKET: ::c_int = 1; -pub const SO_REUSEADDR: ::c_int = 2; -pub const SO_TYPE: ::c_int = 3; -pub const SO_ERROR: ::c_int = 4; -pub const SO_DONTROUTE: ::c_int = 5; -pub const SO_BROADCAST: ::c_int = 6; -pub const SO_SNDBUF: ::c_int = 7; -pub const SO_RCVBUF: ::c_int = 8; -pub const SO_KEEPALIVE: ::c_int = 9; -pub const SO_OOBINLINE: ::c_int = 10; -pub const SO_NO_CHECK: ::c_int = 11; -pub const SO_PRIORITY: ::c_int = 12; -pub const SO_LINGER: ::c_int = 13; -pub const SO_BSDCOMPAT: ::c_int = 14; -pub const SO_REUSEPORT: ::c_int = 15; -pub const SO_ACCEPTCONN: ::c_int = 30; -pub const SO_SNDBUFFORCE: ::c_int = 32; -pub const SO_RCVBUFFORCE: ::c_int = 33; -pub const SO_PROTOCOL: ::c_int = 38; -pub const SO_DOMAIN: ::c_int = 39; pub const SA_ONSTACK: ::c_int = 0x08000000; pub const SA_SIGINFO: ::c_int = 0x00000004; @@ -601,18 +582,37 @@ pub const SYS_preadv2: ::c_long = 380; pub const SYS_pwritev2: ::c_long = 381; pub const SYS_kexec_file_load: ::c_long = 382; pub const SYS_statx: ::c_long = 383; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; +pub const SYS_quotactl_fd: ::c_long = 443; +pub const SYS_landlock_create_ruleset: ::c_long = 444; +pub const SYS_landlock_add_rule: ::c_long = 445; +pub const SYS_landlock_restrict_self: ::c_long = 446; +pub const SYS_memfd_secret: ::c_long = 447; +pub const SYS_process_mrelease: ::c_long = 448; +pub const SYS_futex_waitv: ::c_long = 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 450; -pub const FIOCLEX: ::c_int = 0x20006601; -pub const FIONCLEX: ::c_int = 0x20006602; -pub const FIONBIO: ::c_int = 0x8004667e; pub const EDEADLK: ::c_int = 58; pub const EDEADLOCK: ::c_int = EDEADLK; -pub const SO_PASSCRED: ::c_int = 20; -pub const SO_PEERCRED: ::c_int = 21; -pub const SO_RCVLOWAT: ::c_int = 16; -pub const SO_SNDLOWAT: ::c_int = 17; -pub const SO_RCVTIMEO: ::c_int = 18; -pub const SO_SNDTIMEO: ::c_int = 19; + pub const EXTPROC: ::tcflag_t = 0x10000000; pub const VEOL: usize = 6; pub const VEOL2: usize = 8; @@ -620,52 +620,7 @@ pub const VMIN: usize = 5; pub const IEXTEN: ::tcflag_t = 0x00000400; pub const TOSTOP: ::tcflag_t = 0x00400000; pub const FLUSHO: ::tcflag_t = 0x00800000; -pub const TCGETS: ::c_int = 0x403c7413; -pub const TCSETS: ::c_int = 0x803c7414; -pub const TCSETSW: ::c_int = 0x803c7415; -pub const TCSETSF: ::c_int = 0x803c7416; -pub const TCGETA: ::c_int = 0x40147417; -pub const TCSETA: ::c_int = 0x80147418; -pub const TCSETAW: ::c_int = 0x80147419; -pub const TCSETAF: ::c_int = 0x8014741c; -pub const TCSBRK: ::c_int = 0x2000741d; -pub const TCXONC: ::c_int = 0x2000741e; -pub const TCFLSH: ::c_int = 0x2000741f; -pub const TIOCGSOFTCAR: ::c_int = 0x5419; -pub const TIOCSSOFTCAR: ::c_int = 0x541A; -pub const TIOCLINUX: ::c_int = 0x541C; -pub const TIOCGSERIAL: ::c_int = 0x541E; -pub const TIOCEXCL: ::c_int = 0x540C; -pub const TIOCNXCL: ::c_int = 0x540D; -pub const TIOCSCTTY: ::c_int = 0x540E; -pub const TIOCGPGRP: ::c_int = 0x40047477; -pub const TIOCSPGRP: ::c_int = 0x80047476; -pub const TIOCOUTQ: ::c_int = 0x40047473; -pub const TIOCGWINSZ: ::c_int = 0x40087468; -pub const TIOCSWINSZ: ::c_int = 0x80087467; -pub const TIOCMGET: ::c_int = 0x5415; -pub const TIOCMBIS: ::c_int = 0x5416; -pub const TIOCMBIC: ::c_int = 0x5417; -pub const TIOCMSET: ::c_int = 0x5418; -pub const FIONREAD: ::c_int = 0x4004667f; -pub const TIOCCONS: ::c_int = 0x541D; -pub const TIOCM_LE: ::c_int = 0x001; -pub const TIOCM_DTR: ::c_int = 0x002; -pub const TIOCM_RTS: ::c_int = 0x004; -pub const TIOCM_ST: ::c_int = 0x008; -pub const TIOCM_SR: ::c_int = 0x010; -pub const TIOCM_CTS: ::c_int = 0x020; -pub const TIOCM_CAR: ::c_int = 0x040; -pub const TIOCM_RNG: ::c_int = 0x080; -pub const TIOCM_DSR: ::c_int = 0x100; -pub const TIOCM_CD: ::c_int = TIOCM_CAR; -pub const TIOCM_RI: ::c_int = TIOCM_RNG; - -pub const TIOCGRS485: ::c_int = 0x542E; -pub const TIOCSRS485: ::c_int = 0x542F; -pub const RLIMIT_NLIMITS: ::c_int = 15; -pub const TIOCINQ: ::c_int = ::FIONREAD; pub const MCL_CURRENT: ::c_int = 0x2000; pub const MCL_FUTURE: ::c_int = 0x4000; pub const CBAUD: ::tcflag_t = 0xff; @@ -737,7 +692,3 @@ pub const B2500000: ::speed_t = 0o00033; pub const B3000000: ::speed_t = 0o00034; pub const B3500000: ::speed_t = 0o00035; pub const B4000000: ::speed_t = 0o00036; - -extern "C" { - pub fn ioctl(fd: ::c_int, request: ::c_int, ...) -> ::c_int; -} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/riscv64/align.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/riscv64/align.rs new file mode 100644 index 000000000..48d152a57 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/riscv64/align.rs @@ -0,0 +1,44 @@ +s_no_extra_traits! { + #[allow(missing_debug_implementations)] + pub struct ucontext_t { + pub __uc_flags: ::c_ulong, + pub uc_link: *mut ucontext_t, + pub uc_stack: ::stack_t, + pub uc_sigmask: ::sigset_t, + pub uc_mcontext: mcontext_t, + } + + #[allow(missing_debug_implementations)] + #[repr(align(16))] + pub struct mcontext_t { + pub __gregs: [::c_ulong; 32], + pub __fpregs: __riscv_mc_fp_state, + } + + #[allow(missing_debug_implementations)] + pub union __riscv_mc_fp_state { + pub __f: __riscv_mc_f_ext_state, + pub __d: __riscv_mc_d_ext_state, + pub __q: __riscv_mc_q_ext_state, + } + + #[allow(missing_debug_implementations)] + pub struct __riscv_mc_f_ext_state { + pub __f: [::c_uint; 32], + pub __fcsr: ::c_uint, + } + + #[allow(missing_debug_implementations)] + pub struct __riscv_mc_d_ext_state { + pub __f: [::c_ulonglong; 32], + pub __fcsr: ::c_uint, + } + + #[allow(missing_debug_implementations)] + #[repr(align(16))] + pub struct __riscv_mc_q_ext_state { + pub __f: [::c_ulonglong; 64], + pub __fcsr: ::c_uint, + pub __glibc_reserved: [::c_uint; 3], + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/riscv64/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/riscv64/mod.rs new file mode 100644 index 000000000..f354293e0 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/riscv64/mod.rs @@ -0,0 +1,742 @@ +//! RISC-V-specific definitions for 64-bit linux-like values + +pub type c_char = u8; +pub type wchar_t = ::c_int; + +pub type nlink_t = ::c_uint; +pub type blksize_t = ::c_int; +pub type fsblkcnt64_t = ::c_ulong; +pub type fsfilcnt64_t = ::c_ulong; +pub type __u64 = ::c_ulonglong; +pub type __s64 = ::c_longlong; + +s! { + pub struct pthread_attr_t { + __size: [::c_ulong; 7], + } + + pub struct stat { + pub st_dev: ::dev_t, + pub st_ino: ::ino_t, + pub st_mode: ::mode_t, + pub st_nlink: ::nlink_t, + pub st_uid: ::uid_t, + pub st_gid: ::gid_t, + pub st_rdev: ::dev_t, + pub __pad1: ::dev_t, + pub st_size: ::off_t, + pub st_blksize: ::blksize_t, + pub __pad2: ::c_int, + pub st_blocks: ::blkcnt_t, + pub st_atime: ::time_t, + pub st_atime_nsec: ::c_long, + pub st_mtime: ::time_t, + pub st_mtime_nsec: ::c_long, + pub st_ctime: ::time_t, + pub st_ctime_nsec: ::c_long, + __unused: [::c_int; 2usize], + } + + pub struct stat64 { + pub st_dev: ::dev_t, + pub st_ino: ::ino64_t, + pub st_mode: ::mode_t, + pub st_nlink: ::nlink_t, + pub st_uid: ::uid_t, + pub st_gid: ::gid_t, + pub st_rdev: ::dev_t, + pub __pad1: ::dev_t, + pub st_size: ::off64_t, + pub st_blksize: ::blksize_t, + pub __pad2: ::c_int, + pub st_blocks: ::blkcnt_t, + pub st_atime: ::time_t, + pub st_atime_nsec: ::c_long, + pub st_mtime: ::time_t, + pub st_mtime_nsec: ::c_long, + pub st_ctime: ::time_t, + pub st_ctime_nsec: ::c_long, + __unused: [::c_int; 2], + } + + pub struct statfs { + pub f_type: ::c_long, + pub f_bsize: ::c_long, + pub f_blocks: ::fsblkcnt_t, + pub f_bfree: ::fsblkcnt_t, + pub f_bavail: ::fsblkcnt_t, + pub f_files: ::fsfilcnt_t, + pub f_ffree: ::fsfilcnt_t, + pub f_fsid: ::fsid_t, + pub f_namelen: ::c_long, + pub f_frsize: ::c_long, + pub f_flags: ::c_long, + pub f_spare: [::c_long; 4], + } + + pub struct statfs64 { + pub f_type: ::c_long, + pub f_bsize: ::c_long, + pub f_blocks: ::fsblkcnt64_t, + pub f_bfree: ::fsblkcnt64_t, + pub f_bavail: ::fsblkcnt64_t, + pub f_files: ::fsfilcnt64_t, + pub f_ffree: ::fsfilcnt64_t, + pub f_fsid: ::fsid_t, + pub f_namelen: ::c_long, + pub f_frsize: ::c_long, + pub f_flags: ::c_long, + pub f_spare: [::c_long; 4], + } + + pub struct statvfs { + pub f_bsize: ::c_ulong, + pub f_frsize: ::c_ulong, + pub f_blocks: ::fsblkcnt_t, + pub f_bfree: ::fsblkcnt_t, + pub f_bavail: ::fsblkcnt_t, + pub f_files: ::fsfilcnt_t, + pub f_ffree: ::fsfilcnt_t, + pub f_favail: ::fsfilcnt_t, + pub f_fsid: ::c_ulong, + pub f_flag: ::c_ulong, + pub f_namemax: ::c_ulong, + pub __f_spare: [::c_int; 6], + } + + pub struct statvfs64 { + pub f_bsize: ::c_ulong, + pub f_frsize: ::c_ulong, + pub f_blocks: ::fsblkcnt64_t, + pub f_bfree: ::fsblkcnt64_t, + pub f_bavail: ::fsblkcnt64_t, + pub f_files: ::fsfilcnt64_t, + pub f_ffree: ::fsfilcnt64_t, + pub f_favail: ::fsfilcnt64_t, + pub f_fsid: ::c_ulong, + pub f_flag: ::c_ulong, + pub f_namemax: ::c_ulong, + pub __f_spare: [::c_int; 6], + } + + pub struct siginfo_t { + pub si_signo: ::c_int, + pub si_errno: ::c_int, + pub si_code: ::c_int, + #[doc(hidden)] + #[deprecated( + since="0.2.54", + note="Please leave a comment on \ + https://github.com/rust-lang/libc/pull/1316 if you're using \ + this field" + )] + pub _pad: [::c_int; 29], + _align: [u64; 0], + } + + pub struct stack_t { + pub ss_sp: *mut ::c_void, + pub ss_flags: ::c_int, + pub ss_size: ::size_t, + } + + pub struct sigaction { + pub sa_sigaction: ::sighandler_t, + pub sa_mask: ::sigset_t, + pub sa_flags: ::c_int, + pub sa_restorer: ::Option, + } + + pub struct ipc_perm { + pub __key: ::key_t, + pub uid: ::uid_t, + pub gid: ::gid_t, + pub cuid: ::uid_t, + pub cgid: ::gid_t, + pub mode: ::c_ushort, + __pad1: ::c_ushort, + pub __seq: ::c_ushort, + __pad2: ::c_ushort, + __unused1: ::c_ulong, + __unused2: ::c_ulong, + } + + pub struct shmid_ds { + pub shm_perm: ::ipc_perm, + pub shm_segsz: ::size_t, + pub shm_atime: ::time_t, + pub shm_dtime: ::time_t, + pub shm_ctime: ::time_t, + pub shm_cpid: ::pid_t, + pub shm_lpid: ::pid_t, + pub shm_nattch: ::shmatt_t, + __unused5: ::c_ulong, + __unused6: ::c_ulong, + } + + pub struct flock { + pub l_type: ::c_short, + pub l_whence: ::c_short, + pub l_start: ::off_t, + pub l_len: ::off_t, + pub l_pid: ::pid_t, + } + + pub struct flock64 { + pub l_type: ::c_short, + pub l_whence: ::c_short, + pub l_start: ::off64_t, + pub l_len: ::off64_t, + pub l_pid: ::pid_t, + } +} + +pub const SYS_read: ::c_long = 63; +pub const SYS_write: ::c_long = 64; +pub const SYS_close: ::c_long = 57; +pub const SYS_fstat: ::c_long = 80; +pub const SYS_lseek: ::c_long = 62; +pub const SYS_mmap: ::c_long = 222; +pub const SYS_mprotect: ::c_long = 226; +pub const SYS_munmap: ::c_long = 215; +pub const SYS_brk: ::c_long = 214; +pub const SYS_rt_sigaction: ::c_long = 134; +pub const SYS_rt_sigprocmask: ::c_long = 135; +pub const SYS_rt_sigreturn: ::c_long = 139; +pub const SYS_ioctl: ::c_long = 29; +pub const SYS_pread64: ::c_long = 67; +pub const SYS_pwrite64: ::c_long = 68; +pub const SYS_readv: ::c_long = 65; +pub const SYS_writev: ::c_long = 66; +pub const SYS_sched_yield: ::c_long = 124; +pub const SYS_mremap: ::c_long = 216; +pub const SYS_msync: ::c_long = 227; +pub const SYS_mincore: ::c_long = 232; +pub const SYS_madvise: ::c_long = 233; +pub const SYS_shmget: ::c_long = 194; +pub const SYS_shmat: ::c_long = 196; +pub const SYS_shmctl: ::c_long = 195; +pub const SYS_dup: ::c_long = 23; +pub const SYS_nanosleep: ::c_long = 101; +pub const SYS_getitimer: ::c_long = 102; +pub const SYS_setitimer: ::c_long = 103; +pub const SYS_getpid: ::c_long = 172; +pub const SYS_sendfile: ::c_long = 71; +pub const SYS_socket: ::c_long = 198; +pub const SYS_connect: ::c_long = 203; +pub const SYS_accept: ::c_long = 202; +pub const SYS_sendto: ::c_long = 206; +pub const SYS_recvfrom: ::c_long = 207; +pub const SYS_sendmsg: ::c_long = 211; +pub const SYS_recvmsg: ::c_long = 212; +pub const SYS_shutdown: ::c_long = 210; +pub const SYS_bind: ::c_long = 200; +pub const SYS_listen: ::c_long = 201; +pub const SYS_getsockname: ::c_long = 204; +pub const SYS_getpeername: ::c_long = 205; +pub const SYS_socketpair: ::c_long = 199; +pub const SYS_setsockopt: ::c_long = 208; +pub const SYS_getsockopt: ::c_long = 209; +pub const SYS_clone: ::c_long = 220; +pub const SYS_execve: ::c_long = 221; +pub const SYS_exit: ::c_long = 93; +pub const SYS_wait4: ::c_long = 260; +pub const SYS_kill: ::c_long = 129; +pub const SYS_uname: ::c_long = 160; +pub const SYS_semget: ::c_long = 190; +pub const SYS_semop: ::c_long = 193; +pub const SYS_semctl: ::c_long = 191; +pub const SYS_shmdt: ::c_long = 197; +pub const SYS_msgget: ::c_long = 186; +pub const SYS_msgsnd: ::c_long = 189; +pub const SYS_msgrcv: ::c_long = 188; +pub const SYS_msgctl: ::c_long = 187; +pub const SYS_fcntl: ::c_long = 25; +pub const SYS_flock: ::c_long = 32; +pub const SYS_fsync: ::c_long = 82; +pub const SYS_fdatasync: ::c_long = 83; +pub const SYS_truncate: ::c_long = 45; +pub const SYS_ftruncate: ::c_long = 46; +pub const SYS_getcwd: ::c_long = 17; +pub const SYS_chdir: ::c_long = 49; +pub const SYS_fchdir: ::c_long = 50; +pub const SYS_fchmod: ::c_long = 52; +pub const SYS_fchown: ::c_long = 55; +pub const SYS_umask: ::c_long = 166; +pub const SYS_gettimeofday: ::c_long = 169; +pub const SYS_getrlimit: ::c_long = 163; +pub const SYS_getrusage: ::c_long = 165; +pub const SYS_sysinfo: ::c_long = 179; +pub const SYS_times: ::c_long = 153; +pub const SYS_ptrace: ::c_long = 117; +pub const SYS_getuid: ::c_long = 174; +pub const SYS_syslog: ::c_long = 116; +pub const SYS_getgid: ::c_long = 176; +pub const SYS_setuid: ::c_long = 146; +pub const SYS_setgid: ::c_long = 144; +pub const SYS_geteuid: ::c_long = 175; +pub const SYS_getegid: ::c_long = 177; +pub const SYS_setpgid: ::c_long = 154; +pub const SYS_getppid: ::c_long = 173; +pub const SYS_setsid: ::c_long = 157; +pub const SYS_setreuid: ::c_long = 145; +pub const SYS_setregid: ::c_long = 143; +pub const SYS_getgroups: ::c_long = 158; +pub const SYS_setgroups: ::c_long = 159; +pub const SYS_setresuid: ::c_long = 147; +pub const SYS_getresuid: ::c_long = 148; +pub const SYS_setresgid: ::c_long = 149; +pub const SYS_getresgid: ::c_long = 150; +pub const SYS_getpgid: ::c_long = 155; +pub const SYS_setfsuid: ::c_long = 151; +pub const SYS_setfsgid: ::c_long = 152; +pub const SYS_getsid: ::c_long = 156; +pub const SYS_capget: ::c_long = 90; +pub const SYS_capset: ::c_long = 91; +pub const SYS_rt_sigpending: ::c_long = 136; +pub const SYS_rt_sigtimedwait: ::c_long = 137; +pub const SYS_rt_sigqueueinfo: ::c_long = 138; +pub const SYS_rt_sigsuspend: ::c_long = 133; +pub const SYS_sigaltstack: ::c_long = 132; +pub const SYS_personality: ::c_long = 92; +pub const SYS_statfs: ::c_long = 43; +pub const SYS_fstatfs: ::c_long = 44; +pub const SYS_getpriority: ::c_long = 141; +pub const SYS_setpriority: ::c_long = 140; +pub const SYS_sched_setparam: ::c_long = 118; +pub const SYS_sched_getparam: ::c_long = 121; +pub const SYS_sched_setscheduler: ::c_long = 119; +pub const SYS_sched_getscheduler: ::c_long = 120; +pub const SYS_sched_get_priority_max: ::c_long = 125; +pub const SYS_sched_get_priority_min: ::c_long = 126; +pub const SYS_sched_rr_get_interval: ::c_long = 127; +pub const SYS_mlock: ::c_long = 228; +pub const SYS_munlock: ::c_long = 229; +pub const SYS_mlockall: ::c_long = 230; +pub const SYS_munlockall: ::c_long = 231; +pub const SYS_vhangup: ::c_long = 58; +pub const SYS_pivot_root: ::c_long = 41; +pub const SYS_prctl: ::c_long = 167; +pub const SYS_adjtimex: ::c_long = 171; +pub const SYS_setrlimit: ::c_long = 164; +pub const SYS_chroot: ::c_long = 51; +pub const SYS_sync: ::c_long = 81; +pub const SYS_acct: ::c_long = 89; +pub const SYS_settimeofday: ::c_long = 170; +pub const SYS_mount: ::c_long = 40; +pub const SYS_umount2: ::c_long = 39; +pub const SYS_swapon: ::c_long = 224; +pub const SYS_swapoff: ::c_long = 225; +pub const SYS_reboot: ::c_long = 142; +pub const SYS_sethostname: ::c_long = 161; +pub const SYS_setdomainname: ::c_long = 162; +pub const SYS_init_module: ::c_long = 105; +pub const SYS_delete_module: ::c_long = 106; +pub const SYS_quotactl: ::c_long = 60; +pub const SYS_nfsservctl: ::c_long = 42; +pub const SYS_gettid: ::c_long = 178; +pub const SYS_readahead: ::c_long = 213; +pub const SYS_setxattr: ::c_long = 5; +pub const SYS_lsetxattr: ::c_long = 6; +pub const SYS_fsetxattr: ::c_long = 7; +pub const SYS_getxattr: ::c_long = 8; +pub const SYS_lgetxattr: ::c_long = 9; +pub const SYS_fgetxattr: ::c_long = 10; +pub const SYS_listxattr: ::c_long = 11; +pub const SYS_llistxattr: ::c_long = 12; +pub const SYS_flistxattr: ::c_long = 13; +pub const SYS_removexattr: ::c_long = 14; +pub const SYS_lremovexattr: ::c_long = 15; +pub const SYS_fremovexattr: ::c_long = 16; +pub const SYS_tkill: ::c_long = 130; +pub const SYS_futex: ::c_long = 98; +pub const SYS_sched_setaffinity: ::c_long = 122; +pub const SYS_sched_getaffinity: ::c_long = 123; +pub const SYS_io_setup: ::c_long = 0; +pub const SYS_io_destroy: ::c_long = 1; +pub const SYS_io_getevents: ::c_long = 4; +pub const SYS_io_submit: ::c_long = 2; +pub const SYS_io_cancel: ::c_long = 3; +pub const SYS_lookup_dcookie: ::c_long = 18; +pub const SYS_remap_file_pages: ::c_long = 234; +pub const SYS_getdents64: ::c_long = 61; +pub const SYS_set_tid_address: ::c_long = 96; +pub const SYS_restart_syscall: ::c_long = 128; +pub const SYS_semtimedop: ::c_long = 192; +pub const SYS_fadvise64: ::c_long = 223; +pub const SYS_timer_create: ::c_long = 107; +pub const SYS_timer_settime: ::c_long = 110; +pub const SYS_timer_gettime: ::c_long = 108; +pub const SYS_timer_getoverrun: ::c_long = 109; +pub const SYS_timer_delete: ::c_long = 111; +pub const SYS_clock_settime: ::c_long = 112; +pub const SYS_clock_gettime: ::c_long = 113; +pub const SYS_clock_getres: ::c_long = 114; +pub const SYS_clock_nanosleep: ::c_long = 115; +pub const SYS_exit_group: ::c_long = 94; +pub const SYS_epoll_ctl: ::c_long = 21; +pub const SYS_tgkill: ::c_long = 131; +pub const SYS_mbind: ::c_long = 235; +pub const SYS_set_mempolicy: ::c_long = 237; +pub const SYS_get_mempolicy: ::c_long = 236; +pub const SYS_mq_open: ::c_long = 180; +pub const SYS_mq_unlink: ::c_long = 181; +pub const SYS_mq_timedsend: ::c_long = 182; +pub const SYS_mq_timedreceive: ::c_long = 183; +pub const SYS_mq_notify: ::c_long = 184; +pub const SYS_mq_getsetattr: ::c_long = 185; +pub const SYS_kexec_load: ::c_long = 104; +pub const SYS_waitid: ::c_long = 95; +pub const SYS_add_key: ::c_long = 217; +pub const SYS_request_key: ::c_long = 218; +pub const SYS_keyctl: ::c_long = 219; +pub const SYS_ioprio_set: ::c_long = 30; +pub const SYS_ioprio_get: ::c_long = 31; +pub const SYS_inotify_add_watch: ::c_long = 27; +pub const SYS_inotify_rm_watch: ::c_long = 28; +pub const SYS_migrate_pages: ::c_long = 238; +pub const SYS_openat: ::c_long = 56; +pub const SYS_mkdirat: ::c_long = 34; +pub const SYS_mknodat: ::c_long = 33; +pub const SYS_fchownat: ::c_long = 54; +pub const SYS_newfstatat: ::c_long = 79; +pub const SYS_unlinkat: ::c_long = 35; +pub const SYS_linkat: ::c_long = 37; +pub const SYS_symlinkat: ::c_long = 36; +pub const SYS_readlinkat: ::c_long = 78; +pub const SYS_fchmodat: ::c_long = 53; +pub const SYS_faccessat: ::c_long = 48; +pub const SYS_pselect6: ::c_long = 72; +pub const SYS_ppoll: ::c_long = 73; +pub const SYS_unshare: ::c_long = 97; +pub const SYS_set_robust_list: ::c_long = 99; +pub const SYS_get_robust_list: ::c_long = 100; +pub const SYS_splice: ::c_long = 76; +pub const SYS_tee: ::c_long = 77; +pub const SYS_sync_file_range: ::c_long = 84; +pub const SYS_vmsplice: ::c_long = 75; +pub const SYS_move_pages: ::c_long = 239; +pub const SYS_utimensat: ::c_long = 88; +pub const SYS_epoll_pwait: ::c_long = 22; +pub const SYS_timerfd_create: ::c_long = 85; +pub const SYS_fallocate: ::c_long = 47; +pub const SYS_timerfd_settime: ::c_long = 86; +pub const SYS_timerfd_gettime: ::c_long = 87; +pub const SYS_accept4: ::c_long = 242; +pub const SYS_signalfd4: ::c_long = 74; +pub const SYS_eventfd2: ::c_long = 19; +pub const SYS_epoll_create1: ::c_long = 20; +pub const SYS_dup3: ::c_long = 24; +pub const SYS_pipe2: ::c_long = 59; +pub const SYS_inotify_init1: ::c_long = 26; +pub const SYS_preadv: ::c_long = 69; +pub const SYS_pwritev: ::c_long = 70; +pub const SYS_rt_tgsigqueueinfo: ::c_long = 240; +pub const SYS_perf_event_open: ::c_long = 241; +pub const SYS_recvmmsg: ::c_long = 243; +pub const SYS_fanotify_init: ::c_long = 262; +pub const SYS_fanotify_mark: ::c_long = 263; +pub const SYS_prlimit64: ::c_long = 261; +pub const SYS_name_to_handle_at: ::c_long = 264; +pub const SYS_open_by_handle_at: ::c_long = 265; +pub const SYS_clock_adjtime: ::c_long = 266; +pub const SYS_syncfs: ::c_long = 267; +pub const SYS_sendmmsg: ::c_long = 269; +pub const SYS_setns: ::c_long = 268; +pub const SYS_getcpu: ::c_long = 168; +pub const SYS_process_vm_readv: ::c_long = 270; +pub const SYS_process_vm_writev: ::c_long = 271; +pub const SYS_kcmp: ::c_long = 272; +pub const SYS_finit_module: ::c_long = 273; +pub const SYS_sched_setattr: ::c_long = 274; +pub const SYS_sched_getattr: ::c_long = 275; +pub const SYS_renameat2: ::c_long = 276; +pub const SYS_seccomp: ::c_long = 277; +pub const SYS_getrandom: ::c_long = 278; +pub const SYS_memfd_create: ::c_long = 279; +pub const SYS_bpf: ::c_long = 280; +pub const SYS_execveat: ::c_long = 281; +pub const SYS_userfaultfd: ::c_long = 282; +pub const SYS_membarrier: ::c_long = 283; +pub const SYS_mlock2: ::c_long = 284; +pub const SYS_copy_file_range: ::c_long = 285; +pub const SYS_preadv2: ::c_long = 286; +pub const SYS_pwritev2: ::c_long = 287; +pub const SYS_pkey_mprotect: ::c_long = 288; +pub const SYS_pkey_alloc: ::c_long = 289; +pub const SYS_pkey_free: ::c_long = 290; +pub const SYS_statx: ::c_long = 291; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; + +pub const O_APPEND: ::c_int = 1024; +pub const O_DIRECT: ::c_int = 0x4000; +pub const O_DIRECTORY: ::c_int = 0x10000; +pub const O_LARGEFILE: ::c_int = 0; +pub const O_NOFOLLOW: ::c_int = 0x20000; +pub const O_CREAT: ::c_int = 64; +pub const O_EXCL: ::c_int = 128; +pub const O_NOCTTY: ::c_int = 256; +pub const O_NONBLOCK: ::c_int = 2048; +pub const O_SYNC: ::c_int = 1052672; +pub const O_RSYNC: ::c_int = 1052672; +pub const O_DSYNC: ::c_int = 4096; +pub const O_ASYNC: ::c_int = 0x2000; + +pub const SIGSTKSZ: ::size_t = 8192; +pub const MINSIGSTKSZ: ::size_t = 2048; + +pub const ENAMETOOLONG: ::c_int = 36; +pub const ENOLCK: ::c_int = 37; +pub const ENOSYS: ::c_int = 38; +pub const ENOTEMPTY: ::c_int = 39; +pub const ELOOP: ::c_int = 40; +pub const ENOMSG: ::c_int = 42; +pub const EIDRM: ::c_int = 43; +pub const ECHRNG: ::c_int = 44; +pub const EL2NSYNC: ::c_int = 45; +pub const EL3HLT: ::c_int = 46; +pub const EL3RST: ::c_int = 47; +pub const ELNRNG: ::c_int = 48; +pub const EUNATCH: ::c_int = 49; +pub const ENOCSI: ::c_int = 50; +pub const EL2HLT: ::c_int = 51; +pub const EBADE: ::c_int = 52; +pub const EBADR: ::c_int = 53; +pub const EXFULL: ::c_int = 54; +pub const ENOANO: ::c_int = 55; +pub const EBADRQC: ::c_int = 56; +pub const EBADSLT: ::c_int = 57; +pub const EMULTIHOP: ::c_int = 72; +pub const EOVERFLOW: ::c_int = 75; +pub const ENOTUNIQ: ::c_int = 76; +pub const EBADFD: ::c_int = 77; +pub const EBADMSG: ::c_int = 74; +pub const EREMCHG: ::c_int = 78; +pub const ELIBACC: ::c_int = 79; +pub const ELIBBAD: ::c_int = 80; +pub const ELIBSCN: ::c_int = 81; +pub const ELIBMAX: ::c_int = 82; +pub const ELIBEXEC: ::c_int = 83; +pub const EILSEQ: ::c_int = 84; +pub const ERESTART: ::c_int = 85; +pub const ESTRPIPE: ::c_int = 86; +pub const EUSERS: ::c_int = 87; +pub const ENOTSOCK: ::c_int = 88; +pub const EDESTADDRREQ: ::c_int = 89; +pub const EMSGSIZE: ::c_int = 90; +pub const EPROTOTYPE: ::c_int = 91; +pub const ENOPROTOOPT: ::c_int = 92; +pub const EPROTONOSUPPORT: ::c_int = 93; +pub const ESOCKTNOSUPPORT: ::c_int = 94; +pub const EOPNOTSUPP: ::c_int = 95; +pub const ENOTSUP: ::c_int = EOPNOTSUPP; +pub const EPFNOSUPPORT: ::c_int = 96; +pub const EAFNOSUPPORT: ::c_int = 97; +pub const EADDRINUSE: ::c_int = 98; +pub const EADDRNOTAVAIL: ::c_int = 99; +pub const ENETDOWN: ::c_int = 100; +pub const ENETUNREACH: ::c_int = 101; +pub const ENETRESET: ::c_int = 102; +pub const ECONNABORTED: ::c_int = 103; +pub const ECONNRESET: ::c_int = 104; +pub const ENOBUFS: ::c_int = 105; +pub const EISCONN: ::c_int = 106; +pub const ENOTCONN: ::c_int = 107; +pub const ESHUTDOWN: ::c_int = 108; +pub const ETOOMANYREFS: ::c_int = 109; +pub const ETIMEDOUT: ::c_int = 110; +pub const ECONNREFUSED: ::c_int = 111; +pub const EHOSTDOWN: ::c_int = 112; +pub const EHOSTUNREACH: ::c_int = 113; +pub const EALREADY: ::c_int = 114; +pub const EINPROGRESS: ::c_int = 115; +pub const ESTALE: ::c_int = 116; +pub const EUCLEAN: ::c_int = 117; +pub const ENOTNAM: ::c_int = 118; +pub const ENAVAIL: ::c_int = 119; +pub const EISNAM: ::c_int = 120; +pub const EREMOTEIO: ::c_int = 121; +pub const EDQUOT: ::c_int = 122; +pub const ENOMEDIUM: ::c_int = 123; +pub const EMEDIUMTYPE: ::c_int = 124; +pub const ECANCELED: ::c_int = 125; +pub const ENOKEY: ::c_int = 126; +pub const EKEYEXPIRED: ::c_int = 127; +pub const EKEYREVOKED: ::c_int = 128; +pub const EKEYREJECTED: ::c_int = 129; +pub const EOWNERDEAD: ::c_int = 130; +pub const ENOTRECOVERABLE: ::c_int = 131; +pub const EHWPOISON: ::c_int = 133; +pub const ERFKILL: ::c_int = 132; + +pub const SA_ONSTACK: ::c_int = 0x08000000; +pub const SA_SIGINFO: ::c_int = 0x00000004; +pub const SA_NOCLDWAIT: ::c_int = 0x00000002; + +pub const SIGCHLD: ::c_int = 17; +pub const SIGBUS: ::c_int = 7; +pub const SIGTTIN: ::c_int = 21; +pub const SIGTTOU: ::c_int = 22; +pub const SIGXCPU: ::c_int = 24; +pub const SIGXFSZ: ::c_int = 25; +pub const SIGVTALRM: ::c_int = 26; +pub const SIGPROF: ::c_int = 27; +pub const SIGWINCH: ::c_int = 28; +pub const SIGUSR1: ::c_int = 10; +pub const SIGUSR2: ::c_int = 12; +pub const SIGCONT: ::c_int = 18; +pub const SIGSTOP: ::c_int = 19; +pub const SIGTSTP: ::c_int = 20; +pub const SIGURG: ::c_int = 23; +pub const SIGIO: ::c_int = 29; +pub const SIGSYS: ::c_int = 31; +pub const SIGSTKFLT: ::c_int = 16; +pub const SIGPOLL: ::c_int = 29; +pub const SIGPWR: ::c_int = 30; +pub const SIG_SETMASK: ::c_int = 2; +pub const SIG_BLOCK: ::c_int = 0x000000; +pub const SIG_UNBLOCK: ::c_int = 0x01; + +pub const F_GETLK: ::c_int = 5; +pub const F_GETOWN: ::c_int = 9; +pub const F_SETLK: ::c_int = 6; +pub const F_SETLKW: ::c_int = 7; +pub const F_SETOWN: ::c_int = 8; +pub const F_OFD_GETLK: ::c_int = 36; +pub const F_OFD_SETLK: ::c_int = 37; +pub const F_OFD_SETLKW: ::c_int = 38; + +pub const VEOF: usize = 4; + +pub const POLLWRNORM: ::c_short = 0x100; +pub const POLLWRBAND: ::c_short = 0x200; + +pub const SOCK_STREAM: ::c_int = 1; +pub const SOCK_DGRAM: ::c_int = 2; + +pub const MAP_ANON: ::c_int = 0x0020; +pub const MAP_GROWSDOWN: ::c_int = 0x0100; +pub const MAP_DENYWRITE: ::c_int = 0x0800; +pub const MAP_EXECUTABLE: ::c_int = 0x01000; +pub const MAP_LOCKED: ::c_int = 0x02000; +pub const MAP_NORESERVE: ::c_int = 0x04000; +pub const MAP_POPULATE: ::c_int = 0x08000; +pub const MAP_NONBLOCK: ::c_int = 0x010000; +pub const MAP_STACK: ::c_int = 0x020000; +pub const MAP_HUGETLB: ::c_int = 0x040000; +pub const MAP_SYNC: ::c_int = 0x080000; + +pub const MCL_CURRENT: ::c_int = 0x0001; +pub const MCL_FUTURE: ::c_int = 0x0002; +pub const CBAUD: ::tcflag_t = 0o0010017; +pub const TAB1: ::c_int = 0x00000800; +pub const TAB2: ::c_int = 0x00001000; +pub const TAB3: ::c_int = 0x00001800; +pub const CR1: ::c_int = 0x00000200; +pub const CR2: ::c_int = 0x00000400; +pub const CR3: ::c_int = 0x00000600; +pub const FF1: ::c_int = 0x00008000; +pub const BS1: ::c_int = 0x00002000; +pub const VT1: ::c_int = 0x00004000; +pub const VWERASE: usize = 14; +pub const VREPRINT: usize = 12; +pub const VSUSP: usize = 10; +pub const VSTART: usize = 8; +pub const VSTOP: usize = 9; +pub const VDISCARD: usize = 13; +pub const VTIME: usize = 5; +pub const IXON: ::tcflag_t = 0x00000400; +pub const IXOFF: ::tcflag_t = 0x00001000; +pub const ONLCR: ::tcflag_t = 0x4; +pub const CSIZE: ::tcflag_t = 0x00000030; +pub const CS6: ::tcflag_t = 0x00000010; +pub const CS7: ::tcflag_t = 0x00000020; +pub const CS8: ::tcflag_t = 0x00000030; +pub const CSTOPB: ::tcflag_t = 0x00000040; +pub const CREAD: ::tcflag_t = 0x00000080; +pub const PARENB: ::tcflag_t = 0x00000100; +pub const PARODD: ::tcflag_t = 0x00000200; +pub const HUPCL: ::tcflag_t = 0x00000400; +pub const CLOCAL: ::tcflag_t = 0x00000800; +pub const ECHOKE: ::tcflag_t = 0x00000800; +pub const ECHOE: ::tcflag_t = 0x00000010; +pub const ECHOK: ::tcflag_t = 0x00000020; +pub const ECHONL: ::tcflag_t = 0x00000040; +pub const ECHOPRT: ::tcflag_t = 0x00000400; +pub const ECHOCTL: ::tcflag_t = 0x00000200; +pub const ISIG: ::tcflag_t = 0x00000001; +pub const ICANON: ::tcflag_t = 0x00000002; +pub const PENDIN: ::tcflag_t = 0x00004000; +pub const NOFLSH: ::tcflag_t = 0x00000080; +pub const CIBAUD: ::tcflag_t = 0o02003600000; +pub const CBAUDEX: ::tcflag_t = 0o010000; +pub const VSWTC: usize = 7; +pub const OLCUC: ::tcflag_t = 0o000002; +pub const NLDLY: ::tcflag_t = 0o000400; +pub const CRDLY: ::tcflag_t = 0o003000; +pub const TABDLY: ::tcflag_t = 0o014000; +pub const BSDLY: ::tcflag_t = 0o020000; +pub const FFDLY: ::tcflag_t = 0o100000; +pub const VTDLY: ::tcflag_t = 0o040000; +pub const XTABS: ::tcflag_t = 0o014000; +pub const B57600: ::speed_t = 0o010001; +pub const B115200: ::speed_t = 0o010002; +pub const B230400: ::speed_t = 0o010003; +pub const B460800: ::speed_t = 0o010004; +pub const B500000: ::speed_t = 0o010005; +pub const B576000: ::speed_t = 0o010006; +pub const B921600: ::speed_t = 0o010007; +pub const B1000000: ::speed_t = 0o010010; +pub const B1152000: ::speed_t = 0o010011; +pub const B1500000: ::speed_t = 0o010012; +pub const B2000000: ::speed_t = 0o010013; +pub const B2500000: ::speed_t = 0o010014; +pub const B3000000: ::speed_t = 0o010015; +pub const B3500000: ::speed_t = 0o010016; +pub const B4000000: ::speed_t = 0o010017; + +pub const EDEADLK: ::c_int = 35; +pub const EDEADLOCK: ::c_int = EDEADLK; +pub const EXTPROC: ::tcflag_t = 0x00010000; +pub const VEOL: usize = 11; +pub const VEOL2: usize = 16; +pub const VMIN: usize = 6; +pub const IEXTEN: ::tcflag_t = 0x00008000; +pub const TOSTOP: ::tcflag_t = 0x00000100; +pub const FLUSHO: ::tcflag_t = 0x00001000; + +pub const NGREG: usize = 32; +pub const REG_PC: usize = 0; +pub const REG_RA: usize = 1; +pub const REG_SP: usize = 2; +pub const REG_TP: usize = 4; +pub const REG_S0: usize = 8; +pub const REG_S1: usize = 9; +pub const REG_A0: usize = 10; +pub const REG_S2: usize = 18; +pub const REG_NARGS: usize = 8; + +cfg_if! { + if #[cfg(libc_align)] { + mod align; + pub use self::align::*; + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/s390x.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/s390x.rs new file mode 100644 index 000000000..60bfc8d3e --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/s390x.rs @@ -0,0 +1,721 @@ +pub type blksize_t = i64; +pub type c_char = u8; +pub type nlink_t = u64; +pub type wchar_t = i32; +pub type greg_t = u64; +pub type __u64 = u64; +pub type __s64 = i64; + +s! { + pub struct ipc_perm { + pub __ipc_perm_key: ::key_t, + pub uid: ::uid_t, + pub gid: ::gid_t, + pub cuid: ::uid_t, + pub cgid: ::gid_t, + pub mode: ::mode_t, + pub __seq: ::c_int, + __pad1: ::c_long, + __pad2: ::c_long, + } + + pub struct stat { + pub st_dev: ::dev_t, + pub st_ino: ::ino_t, + pub st_nlink: ::nlink_t, + pub st_mode: ::mode_t, + pub st_uid: ::uid_t, + pub st_gid: ::gid_t, + pub st_rdev: ::dev_t, + pub st_size: ::off_t, + pub st_atime: ::time_t, + pub st_atime_nsec: ::c_long, + pub st_mtime: ::time_t, + pub st_mtime_nsec: ::c_long, + pub st_ctime: ::time_t, + pub st_ctime_nsec: ::c_long, + pub st_blksize: ::blksize_t, + pub st_blocks: ::blkcnt_t, + __unused: [::c_long; 3], + } + + pub struct stat64 { + pub st_dev: ::dev_t, + pub st_ino: ::ino64_t, + pub st_nlink: ::nlink_t, + pub st_mode: ::mode_t, + pub st_uid: ::uid_t, + pub st_gid: ::gid_t, + pub st_rdev: ::dev_t, + pub st_size: ::off_t, + pub st_atime: ::time_t, + pub st_atime_nsec: ::c_long, + pub st_mtime: ::time_t, + pub st_mtime_nsec: ::c_long, + pub st_ctime: ::time_t, + pub st_ctime_nsec: ::c_long, + pub st_blksize: ::blksize_t, + pub st_blocks: ::blkcnt64_t, + __unused: [::c_long; 3], + } + + pub struct statfs { + pub f_type: ::c_ulong, + pub f_bsize: ::c_ulong, + pub f_blocks: ::fsblkcnt_t, + pub f_bfree: ::fsblkcnt_t, + pub f_bavail: ::fsblkcnt_t, + pub f_files: ::fsfilcnt_t, + pub f_ffree: ::fsfilcnt_t, + pub f_fsid: ::fsid_t, + pub f_namelen: ::c_ulong, + pub f_frsize: ::c_ulong, + pub f_flags: ::c_ulong, + pub f_spare: [::c_ulong; 4], + } + + pub struct statfs64 { + pub f_type: ::c_ulong, + pub f_bsize: ::c_ulong, + pub f_blocks: ::fsblkcnt_t, + pub f_bfree: ::fsblkcnt_t, + pub f_bavail: ::fsblkcnt_t, + pub f_files: ::fsfilcnt_t, + pub f_ffree: ::fsfilcnt_t, + pub f_fsid: ::fsid_t, + pub f_namelen: ::c_ulong, + pub f_frsize: ::c_ulong, + pub f_flags: ::c_ulong, + pub f_spare: [::c_ulong; 4], + } +} + +s_no_extra_traits! { + // FIXME: This is actually a union. + pub struct fpreg_t { + pub d: ::c_double, + // f: ::c_float, + } +} + +cfg_if! { + if #[cfg(feature = "extra_traits")] { + impl PartialEq for fpreg_t { + fn eq(&self, other: &fpreg_t) -> bool { + self.d == other.d + } + } + + impl Eq for fpreg_t {} + + impl ::fmt::Debug for fpreg_t { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("fpreg_t") + .field("d", &self.d) + .finish() + } + } + + impl ::hash::Hash for fpreg_t { + fn hash(&self, state: &mut H) { + let d: u64 = unsafe { ::mem::transmute(self.d) }; + d.hash(state); + } + } + } +} + +pub const VEOF: usize = 4; +pub const RTLD_DEEPBIND: ::c_int = 0x8; + +pub const EUCLEAN: ::c_int = 117; +pub const ENOTNAM: ::c_int = 118; +pub const ENAVAIL: ::c_int = 119; +pub const EISNAM: ::c_int = 120; +pub const EREMOTEIO: ::c_int = 121; +pub const EADDRINUSE: ::c_int = 98; +pub const EADDRNOTAVAIL: ::c_int = 99; +pub const ECONNABORTED: ::c_int = 103; +pub const ECONNREFUSED: ::c_int = 111; +pub const ECONNRESET: ::c_int = 104; +pub const EDEADLK: ::c_int = 35; +pub const ENOSYS: ::c_int = 38; +pub const ENOTCONN: ::c_int = 107; +pub const ETIMEDOUT: ::c_int = 110; +pub const O_APPEND: ::c_int = 1024; +pub const O_CREAT: ::c_int = 64; +pub const O_EXCL: ::c_int = 128; +pub const O_NONBLOCK: ::c_int = 2048; +pub const SA_NOCLDWAIT: ::c_int = 2; +pub const SA_ONSTACK: ::c_int = 0x08000000; +pub const SA_SIGINFO: ::c_int = 4; +pub const SIGBUS: ::c_int = 7; +pub const SIGSTKSZ: ::size_t = 0x2000; +pub const MINSIGSTKSZ: ::size_t = 2048; +pub const SIG_SETMASK: ::c_int = 2; + +pub const SOCK_STREAM: ::c_int = 1; +pub const SOCK_DGRAM: ::c_int = 2; + +pub const O_NOCTTY: ::c_int = 256; +pub const O_SYNC: ::c_int = 1052672; +pub const O_RSYNC: ::c_int = 1052672; +pub const O_DSYNC: ::c_int = 4096; +pub const O_FSYNC: ::c_int = 0x101000; +pub const O_DIRECT: ::c_int = 0x4000; +pub const O_DIRECTORY: ::c_int = 0x10000; +pub const O_NOFOLLOW: ::c_int = 0x20000; + +pub const MADV_SOFT_OFFLINE: ::c_int = 101; +pub const MAP_GROWSDOWN: ::c_int = 0x0100; +pub const MAP_LOCKED: ::c_int = 0x02000; +pub const MAP_NORESERVE: ::c_int = 0x04000; +pub const MAP_ANON: ::c_int = 0x0020; +pub const MAP_DENYWRITE: ::c_int = 0x0800; +pub const MAP_EXECUTABLE: ::c_int = 0x01000; +pub const MAP_POPULATE: ::c_int = 0x08000; +pub const MAP_NONBLOCK: ::c_int = 0x010000; +pub const MAP_STACK: ::c_int = 0x020000; +pub const MAP_HUGETLB: ::c_int = 0x040000; + +pub const EDEADLOCK: ::c_int = 35; +pub const ENAMETOOLONG: ::c_int = 36; +pub const ENOLCK: ::c_int = 37; +pub const ENOTEMPTY: ::c_int = 39; +pub const ELOOP: ::c_int = 40; +pub const ENOMSG: ::c_int = 42; +pub const EIDRM: ::c_int = 43; +pub const ECHRNG: ::c_int = 44; +pub const EL2NSYNC: ::c_int = 45; +pub const EL3HLT: ::c_int = 46; +pub const EL3RST: ::c_int = 47; +pub const ELNRNG: ::c_int = 48; +pub const EUNATCH: ::c_int = 49; +pub const ENOCSI: ::c_int = 50; +pub const EL2HLT: ::c_int = 51; +pub const EBADE: ::c_int = 52; +pub const EBADR: ::c_int = 53; +pub const EXFULL: ::c_int = 54; +pub const ENOANO: ::c_int = 55; +pub const EBADRQC: ::c_int = 56; +pub const EBADSLT: ::c_int = 57; +pub const EMULTIHOP: ::c_int = 72; +pub const EOVERFLOW: ::c_int = 75; +pub const ENOTUNIQ: ::c_int = 76; +pub const EBADFD: ::c_int = 77; +pub const EBADMSG: ::c_int = 74; +pub const EREMCHG: ::c_int = 78; +pub const ELIBACC: ::c_int = 79; +pub const ELIBBAD: ::c_int = 80; +pub const ELIBSCN: ::c_int = 81; +pub const ELIBMAX: ::c_int = 82; +pub const ELIBEXEC: ::c_int = 83; +pub const EILSEQ: ::c_int = 84; +pub const ERESTART: ::c_int = 85; +pub const ESTRPIPE: ::c_int = 86; +pub const EUSERS: ::c_int = 87; +pub const ENOTSOCK: ::c_int = 88; +pub const EDESTADDRREQ: ::c_int = 89; +pub const EMSGSIZE: ::c_int = 90; +pub const EPROTOTYPE: ::c_int = 91; +pub const ENOPROTOOPT: ::c_int = 92; +pub const EPROTONOSUPPORT: ::c_int = 93; +pub const ESOCKTNOSUPPORT: ::c_int = 94; +pub const EOPNOTSUPP: ::c_int = 95; +pub const ENOTSUP: ::c_int = EOPNOTSUPP; +pub const EPFNOSUPPORT: ::c_int = 96; +pub const EAFNOSUPPORT: ::c_int = 97; +pub const ENETDOWN: ::c_int = 100; +pub const ENETUNREACH: ::c_int = 101; +pub const ENETRESET: ::c_int = 102; +pub const ENOBUFS: ::c_int = 105; +pub const EISCONN: ::c_int = 106; +pub const ESHUTDOWN: ::c_int = 108; +pub const ETOOMANYREFS: ::c_int = 109; +pub const EHOSTDOWN: ::c_int = 112; +pub const EHOSTUNREACH: ::c_int = 113; +pub const EALREADY: ::c_int = 114; +pub const EINPROGRESS: ::c_int = 115; +pub const ESTALE: ::c_int = 116; +pub const EDQUOT: ::c_int = 122; +pub const ENOMEDIUM: ::c_int = 123; +pub const EMEDIUMTYPE: ::c_int = 124; +pub const ECANCELED: ::c_int = 125; +pub const ENOKEY: ::c_int = 126; +pub const EKEYEXPIRED: ::c_int = 127; +pub const EKEYREVOKED: ::c_int = 128; +pub const EKEYREJECTED: ::c_int = 129; +pub const EOWNERDEAD: ::c_int = 130; +pub const ENOTRECOVERABLE: ::c_int = 131; +pub const EHWPOISON: ::c_int = 133; +pub const ERFKILL: ::c_int = 132; + +pub const SIGTTIN: ::c_int = 21; +pub const SIGTTOU: ::c_int = 22; +pub const SIGXCPU: ::c_int = 24; +pub const SIGXFSZ: ::c_int = 25; +pub const SIGVTALRM: ::c_int = 26; +pub const SIGPROF: ::c_int = 27; +pub const SIGWINCH: ::c_int = 28; +pub const SIGCHLD: ::c_int = 17; +pub const SIGUSR1: ::c_int = 10; +pub const SIGUSR2: ::c_int = 12; +pub const SIGCONT: ::c_int = 18; +pub const SIGSTOP: ::c_int = 19; +pub const SIGTSTP: ::c_int = 20; +pub const SIGURG: ::c_int = 23; +pub const SIGIO: ::c_int = 29; +pub const SIGSYS: ::c_int = 31; +pub const SIGSTKFLT: ::c_int = 16; +pub const SIGPOLL: ::c_int = 29; +pub const SIGPWR: ::c_int = 30; +pub const SIG_BLOCK: ::c_int = 0x000000; +pub const SIG_UNBLOCK: ::c_int = 0x01; + +pub const O_ASYNC: ::c_int = 0x2000; + +pub const VEOL: usize = 11; +pub const VEOL2: usize = 16; +pub const VMIN: usize = 6; +pub const IEXTEN: ::tcflag_t = 0x00008000; +pub const TOSTOP: ::tcflag_t = 0x00000100; +pub const FLUSHO: ::tcflag_t = 0x00001000; + +pub const EXTPROC: ::tcflag_t = 0x00010000; + +pub const MCL_CURRENT: ::c_int = 0x0001; +pub const MCL_FUTURE: ::c_int = 0x0002; + +pub const F_GETLK: ::c_int = 5; +pub const F_GETOWN: ::c_int = 9; +pub const F_SETOWN: ::c_int = 8; +pub const F_SETLK: ::c_int = 6; +pub const F_SETLKW: ::c_int = 7; +pub const F_OFD_GETLK: ::c_int = 36; +pub const F_OFD_SETLK: ::c_int = 37; +pub const F_OFD_SETLKW: ::c_int = 38; + +pub const VTIME: usize = 5; +pub const VSWTC: usize = 7; +pub const VSTART: usize = 8; +pub const VSTOP: usize = 9; +pub const VSUSP: usize = 10; +pub const VREPRINT: usize = 12; +pub const VDISCARD: usize = 13; +pub const VWERASE: usize = 14; +pub const OLCUC: ::tcflag_t = 0o000002; +pub const ONLCR: ::tcflag_t = 0o000004; +pub const NLDLY: ::tcflag_t = 0o000400; +pub const CRDLY: ::tcflag_t = 0o003000; +pub const CR1: ::tcflag_t = 0x00000200; +pub const CR2: ::tcflag_t = 0x00000400; +pub const CR3: ::tcflag_t = 0x00000600; +pub const TABDLY: ::tcflag_t = 0o014000; +pub const TAB1: ::tcflag_t = 0x00000800; +pub const TAB2: ::tcflag_t = 0x00001000; +pub const TAB3: ::tcflag_t = 0x00001800; +pub const BSDLY: ::tcflag_t = 0o020000; +pub const BS1: ::tcflag_t = 0x00002000; +pub const FFDLY: ::tcflag_t = 0o100000; +pub const FF1: ::tcflag_t = 0x00008000; +pub const VTDLY: ::tcflag_t = 0o040000; +pub const VT1: ::tcflag_t = 0x00004000; +pub const XTABS: ::tcflag_t = 0o014000; + +pub const CBAUD: ::speed_t = 0o010017; +pub const CSIZE: ::tcflag_t = 0o000060; +pub const CS6: ::tcflag_t = 0o000020; +pub const CS7: ::tcflag_t = 0o000040; +pub const CS8: ::tcflag_t = 0o000060; +pub const CSTOPB: ::tcflag_t = 0o000100; +pub const CREAD: ::tcflag_t = 0o000200; +pub const PARENB: ::tcflag_t = 0o000400; +pub const PARODD: ::tcflag_t = 0o001000; +pub const HUPCL: ::tcflag_t = 0o002000; +pub const CLOCAL: ::tcflag_t = 0o004000; +pub const CBAUDEX: ::tcflag_t = 0o010000; +pub const B57600: ::speed_t = 0o010001; +pub const B115200: ::speed_t = 0o010002; +pub const B230400: ::speed_t = 0o010003; +pub const B460800: ::speed_t = 0o010004; +pub const B500000: ::speed_t = 0o010005; +pub const B576000: ::speed_t = 0o010006; +pub const B921600: ::speed_t = 0o010007; +pub const B1000000: ::speed_t = 0o010010; +pub const B1152000: ::speed_t = 0o010011; +pub const B1500000: ::speed_t = 0o010012; +pub const B2000000: ::speed_t = 0o010013; +pub const B2500000: ::speed_t = 0o010014; +pub const B3000000: ::speed_t = 0o010015; +pub const B3500000: ::speed_t = 0o010016; +pub const B4000000: ::speed_t = 0o010017; +pub const CIBAUD: ::tcflag_t = 0o02003600000; + +pub const ISIG: ::tcflag_t = 0o000001; +pub const ICANON: ::tcflag_t = 0o000002; +pub const XCASE: ::tcflag_t = 0o000004; +pub const ECHOE: ::tcflag_t = 0o000020; +pub const ECHOK: ::tcflag_t = 0o000040; +pub const ECHONL: ::tcflag_t = 0o000100; +pub const NOFLSH: ::tcflag_t = 0o000200; +pub const ECHOCTL: ::tcflag_t = 0o001000; +pub const ECHOPRT: ::tcflag_t = 0o002000; +pub const ECHOKE: ::tcflag_t = 0o004000; +pub const PENDIN: ::tcflag_t = 0o040000; + +pub const POLLWRNORM: ::c_short = 0x100; +pub const POLLWRBAND: ::c_short = 0x200; + +pub const IXON: ::tcflag_t = 0o002000; +pub const IXOFF: ::tcflag_t = 0o010000; + +pub const SYS_exit: ::c_long = 1; +pub const SYS_fork: ::c_long = 2; +pub const SYS_read: ::c_long = 3; +pub const SYS_write: ::c_long = 4; +pub const SYS_open: ::c_long = 5; +pub const SYS_close: ::c_long = 6; +pub const SYS_restart_syscall: ::c_long = 7; +pub const SYS_creat: ::c_long = 8; +pub const SYS_link: ::c_long = 9; +pub const SYS_unlink: ::c_long = 10; +pub const SYS_execve: ::c_long = 11; +pub const SYS_chdir: ::c_long = 12; +pub const SYS_mknod: ::c_long = 14; +pub const SYS_chmod: ::c_long = 15; +pub const SYS_lseek: ::c_long = 19; +pub const SYS_getpid: ::c_long = 20; +pub const SYS_mount: ::c_long = 21; +pub const SYS_umount: ::c_long = 22; +pub const SYS_ptrace: ::c_long = 26; +pub const SYS_alarm: ::c_long = 27; +pub const SYS_pause: ::c_long = 29; +pub const SYS_utime: ::c_long = 30; +pub const SYS_access: ::c_long = 33; +pub const SYS_nice: ::c_long = 34; +pub const SYS_sync: ::c_long = 36; +pub const SYS_kill: ::c_long = 37; +pub const SYS_rename: ::c_long = 38; +pub const SYS_mkdir: ::c_long = 39; +pub const SYS_rmdir: ::c_long = 40; +pub const SYS_dup: ::c_long = 41; +pub const SYS_pipe: ::c_long = 42; +pub const SYS_times: ::c_long = 43; +pub const SYS_brk: ::c_long = 45; +pub const SYS_signal: ::c_long = 48; +pub const SYS_acct: ::c_long = 51; +pub const SYS_umount2: ::c_long = 52; +pub const SYS_ioctl: ::c_long = 54; +pub const SYS_fcntl: ::c_long = 55; +pub const SYS_setpgid: ::c_long = 57; +pub const SYS_umask: ::c_long = 60; +pub const SYS_chroot: ::c_long = 61; +pub const SYS_ustat: ::c_long = 62; +pub const SYS_dup2: ::c_long = 63; +pub const SYS_getppid: ::c_long = 64; +pub const SYS_getpgrp: ::c_long = 65; +pub const SYS_setsid: ::c_long = 66; +pub const SYS_sigaction: ::c_long = 67; +pub const SYS_sigsuspend: ::c_long = 72; +pub const SYS_sigpending: ::c_long = 73; +pub const SYS_sethostname: ::c_long = 74; +pub const SYS_setrlimit: ::c_long = 75; +pub const SYS_getrusage: ::c_long = 77; +pub const SYS_gettimeofday: ::c_long = 78; +pub const SYS_settimeofday: ::c_long = 79; +pub const SYS_symlink: ::c_long = 83; +pub const SYS_readlink: ::c_long = 85; +pub const SYS_uselib: ::c_long = 86; +pub const SYS_swapon: ::c_long = 87; +pub const SYS_reboot: ::c_long = 88; +pub const SYS_readdir: ::c_long = 89; +pub const SYS_mmap: ::c_long = 90; +pub const SYS_munmap: ::c_long = 91; +pub const SYS_truncate: ::c_long = 92; +pub const SYS_ftruncate: ::c_long = 93; +pub const SYS_fchmod: ::c_long = 94; +pub const SYS_getpriority: ::c_long = 96; +pub const SYS_setpriority: ::c_long = 97; +pub const SYS_statfs: ::c_long = 99; +pub const SYS_fstatfs: ::c_long = 100; +pub const SYS_socketcall: ::c_long = 102; +pub const SYS_syslog: ::c_long = 103; +pub const SYS_setitimer: ::c_long = 104; +pub const SYS_getitimer: ::c_long = 105; +pub const SYS_stat: ::c_long = 106; +pub const SYS_lstat: ::c_long = 107; +pub const SYS_fstat: ::c_long = 108; +pub const SYS_lookup_dcookie: ::c_long = 110; +pub const SYS_vhangup: ::c_long = 111; +pub const SYS_idle: ::c_long = 112; +pub const SYS_wait4: ::c_long = 114; +pub const SYS_swapoff: ::c_long = 115; +pub const SYS_sysinfo: ::c_long = 116; +pub const SYS_ipc: ::c_long = 117; +pub const SYS_fsync: ::c_long = 118; +pub const SYS_sigreturn: ::c_long = 119; +pub const SYS_clone: ::c_long = 120; +pub const SYS_setdomainname: ::c_long = 121; +pub const SYS_uname: ::c_long = 122; +pub const SYS_adjtimex: ::c_long = 124; +pub const SYS_mprotect: ::c_long = 125; +pub const SYS_sigprocmask: ::c_long = 126; +pub const SYS_create_module: ::c_long = 127; +pub const SYS_init_module: ::c_long = 128; +pub const SYS_delete_module: ::c_long = 129; +pub const SYS_get_kernel_syms: ::c_long = 130; +pub const SYS_quotactl: ::c_long = 131; +pub const SYS_getpgid: ::c_long = 132; +pub const SYS_fchdir: ::c_long = 133; +pub const SYS_bdflush: ::c_long = 134; +pub const SYS_sysfs: ::c_long = 135; +pub const SYS_personality: ::c_long = 136; +pub const SYS_afs_syscall: ::c_long = 137; /* Syscall for Andrew File System */ +pub const SYS_getdents: ::c_long = 141; +pub const SYS_flock: ::c_long = 143; +pub const SYS_msync: ::c_long = 144; +pub const SYS_readv: ::c_long = 145; +pub const SYS_writev: ::c_long = 146; +pub const SYS_getsid: ::c_long = 147; +pub const SYS_fdatasync: ::c_long = 148; +pub const SYS__sysctl: ::c_long = 149; +pub const SYS_mlock: ::c_long = 150; +pub const SYS_munlock: ::c_long = 151; +pub const SYS_mlockall: ::c_long = 152; +pub const SYS_munlockall: ::c_long = 153; +pub const SYS_sched_setparam: ::c_long = 154; +pub const SYS_sched_getparam: ::c_long = 155; +pub const SYS_sched_setscheduler: ::c_long = 156; +pub const SYS_sched_getscheduler: ::c_long = 157; +pub const SYS_sched_yield: ::c_long = 158; +pub const SYS_sched_get_priority_max: ::c_long = 159; +pub const SYS_sched_get_priority_min: ::c_long = 160; +pub const SYS_sched_rr_get_interval: ::c_long = 161; +pub const SYS_nanosleep: ::c_long = 162; +pub const SYS_mremap: ::c_long = 163; +pub const SYS_query_module: ::c_long = 167; +pub const SYS_poll: ::c_long = 168; +pub const SYS_nfsservctl: ::c_long = 169; +pub const SYS_prctl: ::c_long = 172; +pub const SYS_rt_sigreturn: ::c_long = 173; +pub const SYS_rt_sigaction: ::c_long = 174; +pub const SYS_rt_sigprocmask: ::c_long = 175; +pub const SYS_rt_sigpending: ::c_long = 176; +pub const SYS_rt_sigtimedwait: ::c_long = 177; +pub const SYS_rt_sigqueueinfo: ::c_long = 178; +pub const SYS_rt_sigsuspend: ::c_long = 179; +pub const SYS_pread64: ::c_long = 180; +pub const SYS_pwrite64: ::c_long = 181; +pub const SYS_getcwd: ::c_long = 183; +pub const SYS_capget: ::c_long = 184; +pub const SYS_capset: ::c_long = 185; +pub const SYS_sigaltstack: ::c_long = 186; +pub const SYS_sendfile: ::c_long = 187; +pub const SYS_getpmsg: ::c_long = 188; +pub const SYS_putpmsg: ::c_long = 189; +pub const SYS_vfork: ::c_long = 190; +pub const SYS_pivot_root: ::c_long = 217; +pub const SYS_mincore: ::c_long = 218; +pub const SYS_madvise: ::c_long = 219; +pub const SYS_getdents64: ::c_long = 220; +pub const SYS_readahead: ::c_long = 222; +pub const SYS_setxattr: ::c_long = 224; +pub const SYS_lsetxattr: ::c_long = 225; +pub const SYS_fsetxattr: ::c_long = 226; +pub const SYS_getxattr: ::c_long = 227; +pub const SYS_lgetxattr: ::c_long = 228; +pub const SYS_fgetxattr: ::c_long = 229; +pub const SYS_listxattr: ::c_long = 230; +pub const SYS_llistxattr: ::c_long = 231; +pub const SYS_flistxattr: ::c_long = 232; +pub const SYS_removexattr: ::c_long = 233; +pub const SYS_lremovexattr: ::c_long = 234; +pub const SYS_fremovexattr: ::c_long = 235; +pub const SYS_gettid: ::c_long = 236; +pub const SYS_tkill: ::c_long = 237; +pub const SYS_futex: ::c_long = 238; +pub const SYS_sched_setaffinity: ::c_long = 239; +pub const SYS_sched_getaffinity: ::c_long = 240; +pub const SYS_tgkill: ::c_long = 241; +pub const SYS_io_setup: ::c_long = 243; +pub const SYS_io_destroy: ::c_long = 244; +pub const SYS_io_getevents: ::c_long = 245; +pub const SYS_io_submit: ::c_long = 246; +pub const SYS_io_cancel: ::c_long = 247; +pub const SYS_exit_group: ::c_long = 248; +pub const SYS_epoll_create: ::c_long = 249; +pub const SYS_epoll_ctl: ::c_long = 250; +pub const SYS_epoll_wait: ::c_long = 251; +pub const SYS_set_tid_address: ::c_long = 252; +pub const SYS_fadvise64: ::c_long = 253; +pub const SYS_timer_create: ::c_long = 254; +pub const SYS_timer_settime: ::c_long = 255; +pub const SYS_timer_gettime: ::c_long = 256; +pub const SYS_timer_getoverrun: ::c_long = 257; +pub const SYS_timer_delete: ::c_long = 258; +pub const SYS_clock_settime: ::c_long = 259; +pub const SYS_clock_gettime: ::c_long = 260; +pub const SYS_clock_getres: ::c_long = 261; +pub const SYS_clock_nanosleep: ::c_long = 262; +pub const SYS_statfs64: ::c_long = 265; +pub const SYS_fstatfs64: ::c_long = 266; +pub const SYS_remap_file_pages: ::c_long = 267; +pub const SYS_mbind: ::c_long = 268; +pub const SYS_get_mempolicy: ::c_long = 269; +pub const SYS_set_mempolicy: ::c_long = 270; +pub const SYS_mq_open: ::c_long = 271; +pub const SYS_mq_unlink: ::c_long = 272; +pub const SYS_mq_timedsend: ::c_long = 273; +pub const SYS_mq_timedreceive: ::c_long = 274; +pub const SYS_mq_notify: ::c_long = 275; +pub const SYS_mq_getsetattr: ::c_long = 276; +pub const SYS_kexec_load: ::c_long = 277; +pub const SYS_add_key: ::c_long = 278; +pub const SYS_request_key: ::c_long = 279; +pub const SYS_keyctl: ::c_long = 280; +pub const SYS_waitid: ::c_long = 281; +pub const SYS_ioprio_set: ::c_long = 282; +pub const SYS_ioprio_get: ::c_long = 283; +pub const SYS_inotify_init: ::c_long = 284; +pub const SYS_inotify_add_watch: ::c_long = 285; +pub const SYS_inotify_rm_watch: ::c_long = 286; +pub const SYS_migrate_pages: ::c_long = 287; +pub const SYS_openat: ::c_long = 288; +pub const SYS_mkdirat: ::c_long = 289; +pub const SYS_mknodat: ::c_long = 290; +pub const SYS_fchownat: ::c_long = 291; +pub const SYS_futimesat: ::c_long = 292; +pub const SYS_unlinkat: ::c_long = 294; +pub const SYS_renameat: ::c_long = 295; +pub const SYS_linkat: ::c_long = 296; +pub const SYS_symlinkat: ::c_long = 297; +pub const SYS_readlinkat: ::c_long = 298; +pub const SYS_fchmodat: ::c_long = 299; +pub const SYS_faccessat: ::c_long = 300; +pub const SYS_pselect6: ::c_long = 301; +pub const SYS_ppoll: ::c_long = 302; +pub const SYS_unshare: ::c_long = 303; +pub const SYS_set_robust_list: ::c_long = 304; +pub const SYS_get_robust_list: ::c_long = 305; +pub const SYS_splice: ::c_long = 306; +pub const SYS_sync_file_range: ::c_long = 307; +pub const SYS_tee: ::c_long = 308; +pub const SYS_vmsplice: ::c_long = 309; +pub const SYS_move_pages: ::c_long = 310; +pub const SYS_getcpu: ::c_long = 311; +pub const SYS_epoll_pwait: ::c_long = 312; +pub const SYS_utimes: ::c_long = 313; +pub const SYS_fallocate: ::c_long = 314; +pub const SYS_utimensat: ::c_long = 315; +pub const SYS_signalfd: ::c_long = 316; +pub const SYS_timerfd: ::c_long = 317; +pub const SYS_eventfd: ::c_long = 318; +pub const SYS_timerfd_create: ::c_long = 319; +pub const SYS_timerfd_settime: ::c_long = 320; +pub const SYS_timerfd_gettime: ::c_long = 321; +pub const SYS_signalfd4: ::c_long = 322; +pub const SYS_eventfd2: ::c_long = 323; +pub const SYS_inotify_init1: ::c_long = 324; +pub const SYS_pipe2: ::c_long = 325; +pub const SYS_dup3: ::c_long = 326; +pub const SYS_epoll_create1: ::c_long = 327; +pub const SYS_preadv: ::c_long = 328; +pub const SYS_pwritev: ::c_long = 329; +pub const SYS_rt_tgsigqueueinfo: ::c_long = 330; +pub const SYS_perf_event_open: ::c_long = 331; +pub const SYS_fanotify_init: ::c_long = 332; +pub const SYS_fanotify_mark: ::c_long = 333; +pub const SYS_prlimit64: ::c_long = 334; +pub const SYS_name_to_handle_at: ::c_long = 335; +pub const SYS_open_by_handle_at: ::c_long = 336; +pub const SYS_clock_adjtime: ::c_long = 337; +pub const SYS_syncfs: ::c_long = 338; +pub const SYS_setns: ::c_long = 339; +pub const SYS_process_vm_readv: ::c_long = 340; +pub const SYS_process_vm_writev: ::c_long = 341; +pub const SYS_s390_runtime_instr: ::c_long = 342; +pub const SYS_kcmp: ::c_long = 343; +pub const SYS_finit_module: ::c_long = 344; +pub const SYS_sched_setattr: ::c_long = 345; +pub const SYS_sched_getattr: ::c_long = 346; +pub const SYS_renameat2: ::c_long = 347; +pub const SYS_seccomp: ::c_long = 348; +pub const SYS_getrandom: ::c_long = 349; +pub const SYS_memfd_create: ::c_long = 350; +pub const SYS_bpf: ::c_long = 351; +pub const SYS_s390_pci_mmio_write: ::c_long = 352; +pub const SYS_s390_pci_mmio_read: ::c_long = 353; +pub const SYS_execveat: ::c_long = 354; +pub const SYS_userfaultfd: ::c_long = 355; +pub const SYS_membarrier: ::c_long = 356; +pub const SYS_recvmmsg: ::c_long = 357; +pub const SYS_sendmmsg: ::c_long = 358; +pub const SYS_socket: ::c_long = 359; +pub const SYS_socketpair: ::c_long = 360; +pub const SYS_bind: ::c_long = 361; +pub const SYS_connect: ::c_long = 362; +pub const SYS_listen: ::c_long = 363; +pub const SYS_accept4: ::c_long = 364; +pub const SYS_getsockopt: ::c_long = 365; +pub const SYS_setsockopt: ::c_long = 366; +pub const SYS_getsockname: ::c_long = 367; +pub const SYS_getpeername: ::c_long = 368; +pub const SYS_sendto: ::c_long = 369; +pub const SYS_sendmsg: ::c_long = 370; +pub const SYS_recvfrom: ::c_long = 371; +pub const SYS_recvmsg: ::c_long = 372; +pub const SYS_shutdown: ::c_long = 373; +pub const SYS_mlock2: ::c_long = 374; +pub const SYS_copy_file_range: ::c_long = 375; +pub const SYS_preadv2: ::c_long = 376; +pub const SYS_pwritev2: ::c_long = 377; +pub const SYS_lchown: ::c_long = 198; +pub const SYS_setuid: ::c_long = 213; +pub const SYS_getuid: ::c_long = 199; +pub const SYS_setgid: ::c_long = 214; +pub const SYS_getgid: ::c_long = 200; +pub const SYS_geteuid: ::c_long = 201; +pub const SYS_setreuid: ::c_long = 203; +pub const SYS_setregid: ::c_long = 204; +pub const SYS_getrlimit: ::c_long = 191; +pub const SYS_getgroups: ::c_long = 205; +pub const SYS_fchown: ::c_long = 207; +pub const SYS_setresuid: ::c_long = 208; +pub const SYS_setresgid: ::c_long = 210; +pub const SYS_getresgid: ::c_long = 211; +pub const SYS_select: ::c_long = 142; +pub const SYS_getegid: ::c_long = 202; +pub const SYS_setgroups: ::c_long = 206; +pub const SYS_getresuid: ::c_long = 209; +pub const SYS_chown: ::c_long = 212; +pub const SYS_setfsuid: ::c_long = 215; +pub const SYS_setfsgid: ::c_long = 216; +pub const SYS_newfstatat: ::c_long = 293; +pub const SYS_statx: ::c_long = 379; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; +pub const SYS_quotactl_fd: ::c_long = 443; +pub const SYS_landlock_create_ruleset: ::c_long = 444; +pub const SYS_landlock_add_rule: ::c_long = 445; +pub const SYS_landlock_restrict_self: ::c_long = 446; +pub const SYS_memfd_secret: ::c_long = 447; +pub const SYS_process_mrelease: ::c_long = 448; +pub const SYS_futex_waitv: ::c_long = 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 450; diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/x86_64/align.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/x86_64/align.rs index 7ca870fd0..94391a01a 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/x86_64/align.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/x86_64/align.rs @@ -4,4 +4,22 @@ s_no_extra_traits! { pub struct max_align_t { priv_: [f64; 4] } + +} + +s! { + #[repr(align(8))] + pub struct clone_args { + pub flags: ::c_ulonglong, + pub pidfd: ::c_ulonglong, + pub child_tid: ::c_ulonglong, + pub parent_tid: ::c_ulonglong, + pub exit_signal: ::c_ulonglong, + pub stack: ::c_ulonglong, + pub stack_size: ::c_ulonglong, + pub tls: ::c_ulonglong, + pub set_tid: ::c_ulonglong, + pub set_tid_size: ::c_ulonglong, + pub cgroup: ::c_ulonglong, + } } diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/x86_64/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/x86_64/mod.rs index 59afe8e01..8198dc2f3 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/x86_64/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/b64/x86_64/mod.rs @@ -3,6 +3,7 @@ pub type wchar_t = i32; pub type nlink_t = u64; pub type blksize_t = ::c_long; pub type __u64 = ::c_ulonglong; +pub type __s64 = ::c_longlong; pub type greg_t = i64; s! { @@ -48,24 +49,6 @@ s! { __reserved: [::c_long; 3], } - pub struct nlmsghdr { - pub nlmsg_len: u32, - pub nlmsg_type: u16, - pub nlmsg_flags: u16, - pub nlmsg_seq: u32, - pub nlmsg_pid: u32, - } - - pub struct nlmsgerr { - pub error: ::c_int, - pub msg: nlmsghdr, - } - - pub struct nlattr { - pub nla_len: u16, - pub nla_type: u16, - } - pub struct user_regs_struct { pub r15: ::c_ulong, pub r14: ::c_ulong, @@ -598,6 +581,33 @@ pub const SYS_pkey_mprotect: ::c_long = 329; pub const SYS_pkey_alloc: ::c_long = 330; pub const SYS_pkey_free: ::c_long = 331; pub const SYS_statx: ::c_long = 332; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_io_uring_setup: ::c_long = 425; +pub const SYS_io_uring_enter: ::c_long = 426; +pub const SYS_io_uring_register: ::c_long = 427; +pub const SYS_open_tree: ::c_long = 428; +pub const SYS_move_mount: ::c_long = 429; +pub const SYS_fsopen: ::c_long = 430; +pub const SYS_fsconfig: ::c_long = 431; +pub const SYS_fsmount: ::c_long = 432; +pub const SYS_fspick: ::c_long = 433; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_close_range: ::c_long = 436; +pub const SYS_openat2: ::c_long = 437; +pub const SYS_pidfd_getfd: ::c_long = 438; +pub const SYS_faccessat2: ::c_long = 439; +pub const SYS_process_madvise: ::c_long = 440; +pub const SYS_epoll_pwait2: ::c_long = 441; +pub const SYS_mount_setattr: ::c_long = 442; +pub const SYS_quotactl_fd: ::c_long = 443; +pub const SYS_landlock_create_ruleset: ::c_long = 444; +pub const SYS_landlock_add_rule: ::c_long = 445; +pub const SYS_landlock_restrict_self: ::c_long = 446; +pub const SYS_memfd_secret: ::c_long = 447; +pub const SYS_process_mrelease: ::c_long = 448; +pub const SYS_futex_waitv: ::c_long = 449; +pub const SYS_set_mempolicy_home_node: ::c_long = 450; // offsets in user_regs_structs, from sys/reg.h pub const R15: ::c_int = 0; @@ -672,9 +682,6 @@ pub const O_RSYNC: ::c_int = 1052672; pub const O_DSYNC: ::c_int = 4096; pub const O_ASYNC: ::c_int = 0x2000; -pub const TIOCGRS485: ::c_int = 0x542E; -pub const TIOCSRS485: ::c_int = 0x542F; - pub const SIGSTKSZ: ::size_t = 8192; pub const MINSIGSTKSZ: ::size_t = 2048; @@ -806,26 +813,6 @@ pub const POLLWRBAND: ::c_short = 0x200; pub const SOCK_STREAM: ::c_int = 1; pub const SOCK_DGRAM: ::c_int = 2; -pub const SOL_SOCKET: ::c_int = 1; -pub const SO_REUSEADDR: ::c_int = 2; -pub const SO_TYPE: ::c_int = 3; -pub const SO_ERROR: ::c_int = 4; -pub const SO_DONTROUTE: ::c_int = 5; -pub const SO_BROADCAST: ::c_int = 6; -pub const SO_SNDBUF: ::c_int = 7; -pub const SO_RCVBUF: ::c_int = 8; -pub const SO_KEEPALIVE: ::c_int = 9; -pub const SO_OOBINLINE: ::c_int = 10; -pub const SO_NO_CHECK: ::c_int = 11; -pub const SO_PRIORITY: ::c_int = 12; -pub const SO_LINGER: ::c_int = 13; -pub const SO_BSDCOMPAT: ::c_int = 14; -pub const SO_REUSEPORT: ::c_int = 15; -pub const SO_ACCEPTCONN: ::c_int = 30; -pub const SO_SNDBUFFORCE: ::c_int = 32; -pub const SO_RCVBUFFORCE: ::c_int = 33; -pub const SO_PROTOCOL: ::c_int = 38; -pub const SO_DOMAIN: ::c_int = 39; pub const MAP_ANON: ::c_int = 0x0020; pub const MAP_GROWSDOWN: ::c_int = 0x0100; @@ -837,10 +824,8 @@ pub const MAP_POPULATE: ::c_int = 0x08000; pub const MAP_NONBLOCK: ::c_int = 0x010000; pub const MAP_STACK: ::c_int = 0x020000; pub const MAP_HUGETLB: ::c_int = 0x040000; -pub const MAP_SYNC : ::c_int = 0x080000; +pub const MAP_SYNC: ::c_int = 0x080000; -pub const RLIMIT_NLIMITS: ::c_int = 15; -pub const TIOCINQ: ::c_int = ::FIONREAD; pub const MCL_CURRENT: ::c_int = 0x0001; pub const MCL_FUTURE: ::c_int = 0x0002; pub const CBAUD: ::tcflag_t = 0o0010017; @@ -910,17 +895,9 @@ pub const B3000000: ::speed_t = 0o010015; pub const B3500000: ::speed_t = 0o010016; pub const B4000000: ::speed_t = 0o010017; -pub const FIOCLEX: ::c_int = 0x5451; -pub const FIONCLEX: ::c_int = 0x5450; -pub const FIONBIO: ::c_int = 0x5421; pub const EDEADLK: ::c_int = 35; pub const EDEADLOCK: ::c_int = EDEADLK; -pub const SO_PASSCRED: ::c_int = 16; -pub const SO_PEERCRED: ::c_int = 17; -pub const SO_RCVLOWAT: ::c_int = 18; -pub const SO_SNDLOWAT: ::c_int = 19; -pub const SO_RCVTIMEO: ::c_int = 20; -pub const SO_SNDTIMEO: ::c_int = 21; + pub const EXTPROC: ::tcflag_t = 0x00010000; pub const VEOL: usize = 11; pub const VEOL2: usize = 16; @@ -928,52 +905,6 @@ pub const VMIN: usize = 6; pub const IEXTEN: ::tcflag_t = 0x00008000; pub const TOSTOP: ::tcflag_t = 0x00000100; pub const FLUSHO: ::tcflag_t = 0x00001000; -pub const TCGETS: ::c_int = 0x5401; -pub const TCSETS: ::c_int = 0x5402; -pub const TCSETSW: ::c_int = 0x5403; -pub const TCSETSF: ::c_int = 0x5404; -pub const TCGETA: ::c_int = 0x5405; -pub const TCSETA: ::c_int = 0x5406; -pub const TCSETAW: ::c_int = 0x5407; -pub const TCSETAF: ::c_int = 0x5408; -pub const TCSBRK: ::c_int = 0x5409; -pub const TCXONC: ::c_int = 0x540A; -pub const TCFLSH: ::c_int = 0x540B; -pub const TIOCGSOFTCAR: ::c_int = 0x5419; -pub const TIOCSSOFTCAR: ::c_int = 0x541A; -pub const TIOCLINUX: ::c_int = 0x541C; -pub const TIOCGSERIAL: ::c_int = 0x541E; -pub const TIOCEXCL: ::c_int = 0x540C; -pub const TIOCNXCL: ::c_int = 0x540D; -pub const TIOCSCTTY: ::c_int = 0x540E; -pub const TIOCGPGRP: ::c_int = 0x540F; -pub const TIOCSPGRP: ::c_int = 0x5410; -pub const TIOCOUTQ: ::c_int = 0x5411; -pub const TIOCSTI: ::c_int = 0x5412; -pub const TIOCGWINSZ: ::c_int = 0x5413; -pub const TIOCSWINSZ: ::c_int = 0x5414; -pub const TIOCMGET: ::c_int = 0x5415; -pub const TIOCMBIS: ::c_int = 0x5416; -pub const TIOCMBIC: ::c_int = 0x5417; -pub const TIOCMSET: ::c_int = 0x5418; -pub const FIONREAD: ::c_int = 0x541B; -pub const TIOCCONS: ::c_int = 0x541D; - -pub const TIOCM_LE: ::c_int = 0x001; -pub const TIOCM_DTR: ::c_int = 0x002; -pub const TIOCM_RTS: ::c_int = 0x004; -pub const TIOCM_ST: ::c_int = 0x008; -pub const TIOCM_SR: ::c_int = 0x010; -pub const TIOCM_CTS: ::c_int = 0x020; -pub const TIOCM_CAR: ::c_int = 0x040; -pub const TIOCM_RNG: ::c_int = 0x080; -pub const TIOCM_DSR: ::c_int = 0x100; -pub const TIOCM_CD: ::c_int = TIOCM_CAR; -pub const TIOCM_RI: ::c_int = TIOCM_RNG; - -extern "C" { - pub fn ioctl(fd: ::c_int, request: ::c_int, ...) -> ::c_int; -} cfg_if! { if #[cfg(libc_align)] { diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/mod.rs index 73c562c30..2a894a602 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/musl/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/musl/mod.rs @@ -1,5 +1,14 @@ pub type pthread_t = *mut ::c_void; pub type clock_t = c_long; +#[cfg_attr( + not(feature = "rustc-dep-of-std"), + deprecated( + since = "0.2.80", + note = "This type is changed to 64-bit in musl 1.2.0, \ + we'll follow that change in the future release. \ + See #1848 for more info." + ) +)] pub type time_t = c_long; pub type suseconds_t = c_long; pub type ino_t = u64; @@ -15,6 +24,16 @@ pub type rlim_t = ::c_ulonglong; pub type flock64 = flock; +cfg_if! { + if #[cfg(doc)] { + // Used in `linux::arch` to define ioctl constants. + pub(crate) type Ioctl = ::c_int; + } else { + #[doc(hidden)] + pub type Ioctl = ::c_int; + } +} + impl siginfo_t { pub unsafe fn si_addr(&self) -> *mut ::c_void { #[repr(C)] @@ -24,7 +43,6 @@ impl siginfo_t { _si_code: ::c_int, si_addr: *mut ::c_void, } - (*(self as *const siginfo_t as *const siginfo_sigfault)).si_addr } @@ -38,11 +56,72 @@ impl siginfo_t { _si_overrun: ::c_int, si_value: ::sigval, } - (*(self as *const siginfo_t as *const siginfo_si_value)).si_value } } +cfg_if! { + if #[cfg(libc_union)] { + // Internal, for casts to access union fields + #[repr(C)] + struct sifields_sigchld { + si_pid: ::pid_t, + si_uid: ::uid_t, + si_status: ::c_int, + si_utime: ::c_long, + si_stime: ::c_long, + } + impl ::Copy for sifields_sigchld {} + impl ::Clone for sifields_sigchld { + fn clone(&self) -> sifields_sigchld { + *self + } + } + + // Internal, for casts to access union fields + #[repr(C)] + union sifields { + _align_pointer: *mut ::c_void, + sigchld: sifields_sigchld, + } + + // Internal, for casts to access union fields. Note that some variants + // of sifields start with a pointer, which makes the alignment of + // sifields vary on 32-bit and 64-bit architectures. + #[repr(C)] + struct siginfo_f { + _siginfo_base: [::c_int; 3], + sifields: sifields, + } + + impl siginfo_t { + unsafe fn sifields(&self) -> &sifields { + &(*(self as *const siginfo_t as *const siginfo_f)).sifields + } + + pub unsafe fn si_pid(&self) -> ::pid_t { + self.sifields().sigchld.si_pid + } + + pub unsafe fn si_uid(&self) -> ::uid_t { + self.sifields().sigchld.si_uid + } + + pub unsafe fn si_status(&self) -> ::c_int { + self.sifields().sigchld.si_status + } + + pub unsafe fn si_utime(&self) -> ::c_long { + self.sifields().sigchld.si_utime + } + + pub unsafe fn si_stime(&self) -> ::c_long { + self.sifields().sigchld.si_stime + } + } + } +} + s! { pub struct aiocb { pub aio_fildes: ::c_int, @@ -109,6 +188,83 @@ s! { pub l_len: ::off_t, pub l_pid: ::pid_t, } + + pub struct regex_t { + __re_nsub: ::size_t, + __opaque: *mut ::c_void, + __padding: [*mut ::c_void; 4usize], + __nsub2: ::size_t, + __padding2: ::c_char, + } + + pub struct rtentry { + pub rt_pad1: ::c_ulong, + pub rt_dst: ::sockaddr, + pub rt_gateway: ::sockaddr, + pub rt_genmask: ::sockaddr, + pub rt_flags: ::c_ushort, + pub rt_pad2: ::c_short, + pub rt_pad3: ::c_ulong, + pub rt_tos: ::c_uchar, + pub rt_class: ::c_uchar, + #[cfg(target_pointer_width = "64")] + pub rt_pad4: [::c_short; 3usize], + #[cfg(not(target_pointer_width = "64"))] + pub rt_pad4: [::c_short; 1usize], + pub rt_metric: ::c_short, + pub rt_dev: *mut ::c_char, + pub rt_mtu: ::c_ulong, + pub rt_window: ::c_ulong, + pub rt_irtt: ::c_ushort, + } + + pub struct __exit_status { + pub e_termination: ::c_short, + pub e_exit: ::c_short, + } + + pub struct Elf64_Chdr { + pub ch_type: ::Elf64_Word, + pub ch_reserved: ::Elf64_Word, + pub ch_size: ::Elf64_Xword, + pub ch_addralign: ::Elf64_Xword, + } + + pub struct Elf32_Chdr { + pub ch_type: ::Elf32_Word, + pub ch_size: ::Elf32_Word, + pub ch_addralign: ::Elf32_Word, + } + + pub struct timex { + pub modes: ::c_uint, + pub offset: ::c_long, + pub freq: ::c_long, + pub maxerror: ::c_long, + pub esterror: ::c_long, + pub status: ::c_int, + pub constant: ::c_long, + pub precision: ::c_long, + pub tolerance: ::c_long, + pub time: ::timeval, + pub tick: ::c_long, + pub ppsfreq: ::c_long, + pub jitter: ::c_long, + pub shift: ::c_int, + pub stabil: ::c_long, + pub jitcnt: ::c_long, + pub calcnt: ::c_long, + pub errcnt: ::c_long, + pub stbcnt: ::c_long, + pub tai: ::c_int, + pub __padding: [::c_int; 11], + } + + pub struct ntptimeval { + pub time: ::timeval, + pub maxerror: ::c_long, + pub esterror: ::c_long, + } } s_no_extra_traits! { @@ -128,6 +284,36 @@ s_no_extra_traits! { pub mem_unit: ::c_uint, pub __reserved: [::c_char; 256], } + + // FIXME: musl added paddings and adjusted + // layout in 1.2.0 but our CI is still 1.1.24. + // So, I'm leaving some fields as comments for now. + // ref. https://github.com/bminor/musl/commit/ + // 1e7f0fcd7ff2096904fd93a2ee6d12a2392be392 + pub struct utmpx { + pub ut_type: ::c_short, + //__ut_pad1: ::c_short, + pub ut_pid: ::pid_t, + pub ut_line: [::c_char; 32], + pub ut_id: [::c_char; 4], + pub ut_user: [::c_char; 32], + pub ut_host: [::c_char; 256], + pub ut_exit: __exit_status, + + //#[cfg(target_endian = "little")] + pub ut_session: ::c_long, + //#[cfg(target_endian = "little")] + //__ut_pad2: ::c_long, + + //#[cfg(not(target_endian = "little"))] + //__ut_pad2: ::c_int, + //#[cfg(not(target_endian = "little"))] + //pub ut_session: ::c_int, + + pub ut_tv: ::timeval, + pub ut_addr_v6: [::c_uint; 4], + __unused: [::c_char; 20], + } } cfg_if! { @@ -196,9 +382,95 @@ cfg_if! { self.__reserved.hash(state); } } + + impl PartialEq for utmpx { + fn eq(&self, other: &utmpx) -> bool { + self.ut_type == other.ut_type + //&& self.__ut_pad1 == other.__ut_pad1 + && self.ut_pid == other.ut_pid + && self.ut_line == other.ut_line + && self.ut_id == other.ut_id + && self.ut_user == other.ut_user + && self + .ut_host + .iter() + .zip(other.ut_host.iter()) + .all(|(a,b)| a == b) + && self.ut_exit == other.ut_exit + && self.ut_session == other.ut_session + //&& self.__ut_pad2 == other.__ut_pad2 + && self.ut_tv == other.ut_tv + && self.ut_addr_v6 == other.ut_addr_v6 + && self.__unused == other.__unused + } + } + + impl Eq for utmpx {} + + impl ::fmt::Debug for utmpx { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("utmpx") + .field("ut_type", &self.ut_type) + //.field("__ut_pad1", &self.__ut_pad1) + .field("ut_pid", &self.ut_pid) + .field("ut_line", &self.ut_line) + .field("ut_id", &self.ut_id) + .field("ut_user", &self.ut_user) + //FIXME: .field("ut_host", &self.ut_host) + .field("ut_exit", &self.ut_exit) + .field("ut_session", &self.ut_session) + //.field("__ut_pad2", &self.__ut_pad2) + .field("ut_tv", &self.ut_tv) + .field("ut_addr_v6", &self.ut_addr_v6) + .field("__unused", &self.__unused) + .finish() + } + } + + impl ::hash::Hash for utmpx { + fn hash(&self, state: &mut H) { + self.ut_type.hash(state); + //self.__ut_pad1.hash(state); + self.ut_pid.hash(state); + self.ut_line.hash(state); + self.ut_id.hash(state); + self.ut_user.hash(state); + self.ut_host.hash(state); + self.ut_exit.hash(state); + self.ut_session.hash(state); + //self.__ut_pad2.hash(state); + self.ut_tv.hash(state); + self.ut_addr_v6.hash(state); + self.__unused.hash(state); + } + } } } +// include/sys/mman.h +/* + * Huge page size encoding when MAP_HUGETLB is specified, and a huge page + * size other than the default is desired. See hugetlb_encode.h. + * All known huge page size encodings are provided here. It is the + * responsibility of the application to know which sizes are supported on + * the running system. See mmap(2) man page for details. + */ +pub const MAP_HUGE_SHIFT: ::c_int = 26; +pub const MAP_HUGE_MASK: ::c_int = 0x3f; + +pub const MAP_HUGE_64KB: ::c_int = 16 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_512KB: ::c_int = 19 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_1MB: ::c_int = 20 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_2MB: ::c_int = 21 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_8MB: ::c_int = 23 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_16MB: ::c_int = 24 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_32MB: ::c_int = 25 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_256MB: ::c_int = 28 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_512MB: ::c_int = 29 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_1GB: ::c_int = 30 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_2GB: ::c_int = 31 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_16GB: ::c_int = 34 << MAP_HUGE_SHIFT; + pub const MS_RMT_MASK: ::c_ulong = 0x02800051; pub const SFD_CLOEXEC: ::c_int = 0x080000; @@ -241,6 +513,7 @@ pub const EFD_CLOEXEC: ::c_int = 0x80000; pub const BUFSIZ: ::c_uint = 1024; pub const TMP_MAX: ::c_uint = 10000; pub const FOPEN_MAX: ::c_uint = 1000; +pub const FILENAME_MAX: ::c_uint = 4096; pub const O_PATH: ::c_int = 0o10000000; pub const O_EXEC: ::c_int = 0o10000000; pub const O_SEARCH: ::c_int = 0o10000000; @@ -248,30 +521,15 @@ pub const O_ACCMODE: ::c_int = 0o10000003; pub const O_NDELAY: ::c_int = O_NONBLOCK; pub const NI_MAXHOST: ::socklen_t = 255; pub const PTHREAD_STACK_MIN: ::size_t = 2048; -pub const POSIX_FADV_DONTNEED: ::c_int = 4; -pub const POSIX_FADV_NOREUSE: ::c_int = 5; pub const POSIX_MADV_DONTNEED: ::c_int = 4; -pub const RLIM_INFINITY: ::rlim_t = !0; -pub const RLIMIT_RTTIME: ::c_int = 15; - pub const MAP_ANONYMOUS: ::c_int = MAP_ANON; pub const SOCK_DCCP: ::c_int = 6; pub const SOCK_PACKET: ::c_int = 10; -pub const TCP_COOKIE_TRANSACTIONS: ::c_int = 15; -pub const TCP_THIN_LINEAR_TIMEOUTS: ::c_int = 16; -pub const TCP_THIN_DUPACK: ::c_int = 17; -pub const TCP_USER_TIMEOUT: ::c_int = 18; -pub const TCP_REPAIR: ::c_int = 19; -pub const TCP_REPAIR_QUEUE: ::c_int = 20; -pub const TCP_QUEUE_SEQ: ::c_int = 21; -pub const TCP_REPAIR_OPTIONS: ::c_int = 22; -pub const TCP_FASTOPEN: ::c_int = 23; -pub const TCP_TIMESTAMP: ::c_int = 24; -pub const TCP_FASTOPEN_CONNECT: ::c_int = 30; +pub const SOMAXCONN: ::c_int = 128; #[deprecated(since = "0.2.55", note = "Use SIGSYS instead")] pub const SIGUNUSED: ::c_int = ::SIGSYS; @@ -312,15 +570,28 @@ pub const PTRACE_INTERRUPT: ::c_int = 0x4207; pub const PTRACE_LISTEN: ::c_int = 0x4208; pub const PTRACE_PEEKSIGINFO: ::c_int = 0x4209; -pub const EPOLLWAKEUP: ::c_int = 0x20000000; - -pub const SEEK_DATA: ::c_int = 3; -pub const SEEK_HOLE: ::c_int = 4; +pub const FAN_MARK_INODE: ::c_uint = 0x0000_0000; +pub const FAN_MARK_MOUNT: ::c_uint = 0x0000_0010; +// NOTE: FAN_MARK_FILESYSTEM requires Linux Kernel >= 4.20.0 +pub const FAN_MARK_FILESYSTEM: ::c_uint = 0x0000_0100; + +pub const AF_IB: ::c_int = 27; +pub const AF_MPLS: ::c_int = 28; +pub const AF_NFC: ::c_int = 39; +pub const AF_VSOCK: ::c_int = 40; +pub const AF_XDP: ::c_int = 44; +pub const PF_IB: ::c_int = AF_IB; +pub const PF_MPLS: ::c_int = AF_MPLS; +pub const PF_NFC: ::c_int = AF_NFC; +pub const PF_VSOCK: ::c_int = AF_VSOCK; +pub const PF_XDP: ::c_int = AF_XDP; pub const EFD_NONBLOCK: ::c_int = ::O_NONBLOCK; pub const SFD_NONBLOCK: ::c_int = ::O_NONBLOCK; +pub const PIDFD_NONBLOCK: ::c_uint = O_NONBLOCK as ::c_uint; + pub const TCSANOW: ::c_int = 0; pub const TCSADRAIN: ::c_int = 1; pub const TCSAFLUSH: ::c_int = 2; @@ -349,23 +620,79 @@ pub const B38400: ::speed_t = 0o000017; pub const EXTA: ::speed_t = B19200; pub const EXTB: ::speed_t = B38400; -pub const SO_BINDTODEVICE: ::c_int = 25; -pub const SO_TIMESTAMP: ::c_int = 29; -pub const SO_MARK: ::c_int = 36; -pub const SO_RXQ_OVFL: ::c_int = 40; -pub const SO_PEEK_OFF: ::c_int = 42; -pub const SO_BUSY_POLL: ::c_int = 46; - -pub const RLIMIT_CPU: ::c_int = 0; -pub const RLIMIT_FSIZE: ::c_int = 1; -pub const RLIMIT_DATA: ::c_int = 2; -pub const RLIMIT_STACK: ::c_int = 3; -pub const RLIMIT_CORE: ::c_int = 4; -pub const RLIMIT_LOCKS: ::c_int = 10; -pub const RLIMIT_SIGPENDING: ::c_int = 11; -pub const RLIMIT_MSGQUEUE: ::c_int = 12; -pub const RLIMIT_NICE: ::c_int = 13; -pub const RLIMIT_RTPRIO: ::c_int = 14; +pub const REG_OK: ::c_int = 0; + +pub const PRIO_PROCESS: ::c_int = 0; +pub const PRIO_PGRP: ::c_int = 1; +pub const PRIO_USER: ::c_int = 2; + +pub const ADJ_OFFSET: ::c_uint = 0x0001; +pub const ADJ_FREQUENCY: ::c_uint = 0x0002; +pub const ADJ_MAXERROR: ::c_uint = 0x0004; +pub const ADJ_ESTERROR: ::c_uint = 0x0008; +pub const ADJ_STATUS: ::c_uint = 0x0010; +pub const ADJ_TIMECONST: ::c_uint = 0x0020; +pub const ADJ_TAI: ::c_uint = 0x0080; +pub const ADJ_SETOFFSET: ::c_uint = 0x0100; +pub const ADJ_MICRO: ::c_uint = 0x1000; +pub const ADJ_NANO: ::c_uint = 0x2000; +pub const ADJ_TICK: ::c_uint = 0x4000; +pub const ADJ_OFFSET_SINGLESHOT: ::c_uint = 0x8001; +pub const ADJ_OFFSET_SS_READ: ::c_uint = 0xa001; +pub const MOD_OFFSET: ::c_uint = ADJ_OFFSET; +pub const MOD_FREQUENCY: ::c_uint = ADJ_FREQUENCY; +pub const MOD_MAXERROR: ::c_uint = ADJ_MAXERROR; +pub const MOD_ESTERROR: ::c_uint = ADJ_ESTERROR; +pub const MOD_STATUS: ::c_uint = ADJ_STATUS; +pub const MOD_TIMECONST: ::c_uint = ADJ_TIMECONST; +pub const MOD_CLKB: ::c_uint = ADJ_TICK; +pub const MOD_CLKA: ::c_uint = ADJ_OFFSET_SINGLESHOT; +pub const MOD_TAI: ::c_uint = ADJ_TAI; +pub const MOD_MICRO: ::c_uint = ADJ_MICRO; +pub const MOD_NANO: ::c_uint = ADJ_NANO; +pub const STA_PLL: ::c_int = 0x0001; +pub const STA_PPSFREQ: ::c_int = 0x0002; +pub const STA_PPSTIME: ::c_int = 0x0004; +pub const STA_FLL: ::c_int = 0x0008; +pub const STA_INS: ::c_int = 0x0010; +pub const STA_DEL: ::c_int = 0x0020; +pub const STA_UNSYNC: ::c_int = 0x0040; +pub const STA_FREQHOLD: ::c_int = 0x0080; +pub const STA_PPSSIGNAL: ::c_int = 0x0100; +pub const STA_PPSJITTER: ::c_int = 0x0200; +pub const STA_PPSWANDER: ::c_int = 0x0400; +pub const STA_PPSERROR: ::c_int = 0x0800; +pub const STA_CLOCKERR: ::c_int = 0x1000; +pub const STA_NANO: ::c_int = 0x2000; +pub const STA_MODE: ::c_int = 0x4000; +pub const STA_CLK: ::c_int = 0x8000; +pub const STA_RONLY: ::c_int = STA_PPSSIGNAL + | STA_PPSJITTER + | STA_PPSWANDER + | STA_PPSERROR + | STA_CLOCKERR + | STA_NANO + | STA_MODE + | STA_CLK; + +pub const TIME_OK: ::c_int = 0; +pub const TIME_INS: ::c_int = 1; +pub const TIME_DEL: ::c_int = 2; +pub const TIME_OOP: ::c_int = 3; +pub const TIME_WAIT: ::c_int = 4; +pub const TIME_ERROR: ::c_int = 5; +pub const TIME_BAD: ::c_int = TIME_ERROR; +pub const MAXTC: ::c_long = 6; + +cfg_if! { + if #[cfg(target_arch = "s390x")] { + pub const POSIX_FADV_DONTNEED: ::c_int = 6; + pub const POSIX_FADV_NOREUSE: ::c_int = 7; + } else { + pub const POSIX_FADV_DONTNEED: ::c_int = 4; + pub const POSIX_FADV_NOREUSE: ::c_int = 5; + } +} extern "C" { pub fn sendmmsg( @@ -399,40 +726,68 @@ extern "C" { old_limit: *mut ::rlimit64, ) -> ::c_int; + pub fn ioctl(fd: ::c_int, request: ::c_int, ...) -> ::c_int; pub fn gettimeofday(tp: *mut ::timeval, tz: *mut ::c_void) -> ::c_int; pub fn ptrace(request: ::c_int, ...) -> ::c_long; pub fn getpriority(which: ::c_int, who: ::id_t) -> ::c_int; pub fn setpriority(which: ::c_int, who: ::id_t, prio: ::c_int) -> ::c_int; - pub fn pthread_getaffinity_np( - thread: ::pthread_t, - cpusetsize: ::size_t, - cpuset: *mut ::cpu_set_t, - ) -> ::c_int; - pub fn pthread_setaffinity_np( - thread: ::pthread_t, - cpusetsize: ::size_t, - cpuset: *const ::cpu_set_t, + // Musl targets need the `mask` argument of `fanotify_mark` be specified + // `::c_ulonglong` instead of `u64` or there will be a type mismatch between + // `long long unsigned int` and the expected `uint64_t`. + pub fn fanotify_mark( + fd: ::c_int, + flags: ::c_uint, + mask: ::c_ulonglong, + dirfd: ::c_int, + path: *const ::c_char, ) -> ::c_int; - pub fn sched_getcpu() -> ::c_int; - pub fn memmem( - haystack: *const ::c_void, - haystacklen: ::size_t, - needle: *const ::c_void, - needlelen: ::size_t, - ) -> *mut ::c_void; + pub fn getauxval(type_: ::c_ulong) -> ::c_ulong; + + // Added in `musl` 1.1.20 + pub fn explicit_bzero(s: *mut ::c_void, len: ::size_t); + // Added in `musl` 1.2.2 + pub fn reallocarray(ptr: *mut ::c_void, nmemb: ::size_t, size: ::size_t) -> *mut ::c_void; + + pub fn adjtimex(buf: *mut ::timex) -> ::c_int; + pub fn clock_adjtime(clk_id: ::clockid_t, buf: *mut ::timex) -> ::c_int; + + pub fn ctermid(s: *mut ::c_char) -> *mut ::c_char; + + pub fn memfd_create(name: *const ::c_char, flags: ::c_uint) -> ::c_int; + pub fn mlock2(addr: *const ::c_void, len: ::size_t, flags: ::c_uint) -> ::c_int; + pub fn malloc_usable_size(ptr: *mut ::c_void) -> ::size_t; + + pub fn euidaccess(pathname: *const ::c_char, mode: ::c_int) -> ::c_int; + pub fn eaccess(pathname: *const ::c_char, mode: ::c_int) -> ::c_int; + + pub fn asctime_r(tm: *const ::tm, buf: *mut ::c_char) -> *mut ::c_char; + + pub fn strftime( + s: *mut ::c_char, + max: ::size_t, + format: *const ::c_char, + tm: *const ::tm, + ) -> ::size_t; + pub fn strptime(s: *const ::c_char, format: *const ::c_char, tm: *mut ::tm) -> *mut ::c_char; + + pub fn dirname(path: *mut ::c_char) -> *mut ::c_char; + pub fn basename(path: *mut ::c_char) -> *mut ::c_char; } cfg_if! { if #[cfg(any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "mips64", - target_arch = "powerpc64"))] { + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "riscv64"))] { mod b64; pub use self::b64::*; } else if #[cfg(any(target_arch = "x86", target_arch = "mips", target_arch = "powerpc", target_arch = "hexagon", + target_arch = "riscv32", target_arch = "arm"))] { mod b32; pub use self::b32::*; diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/no_align.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/no_align.rs index a59edcb94..2b5abb3be 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/linux/no_align.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/no_align.rs @@ -8,6 +8,8 @@ macro_rules! expand_align { target_arch = "s390x", target_arch = "sparc64", target_arch = "riscv64", + target_arch = "riscv32", + target_arch = "loongarch64", all(target_arch = "aarch64", target_env = "musl")))] __align: [::c_int; 0], @@ -17,6 +19,8 @@ macro_rules! expand_align { target_arch = "s390x", target_arch = "sparc64", target_arch = "riscv64", + target_arch = "riscv32", + target_arch = "loongarch64", all(target_arch = "aarch64", target_env = "musl"))))] __align: [::c_long; 0], @@ -35,6 +39,17 @@ macro_rules! expand_align { __align: [::c_int; 0], size: [u8; ::__SIZEOF_PTHREAD_CONDATTR_T], } + + pub struct fanotify_event_metadata { + __align: [::c_long; 0], + pub event_len: __u32, + pub vers: __u8, + pub reserved: __u8, + pub metadata_len: __u16, + pub mask: __u64, + pub fd: ::c_int, + pub pid: ::c_int, + } } s_no_extra_traits! { @@ -49,6 +64,7 @@ macro_rules! expand_align { pub struct pthread_mutex_t { #[cfg(any(target_arch = "mips", target_arch = "arm", + target_arch = "m68k", target_arch = "powerpc", target_arch = "sparc", all(target_arch = "x86_64", @@ -56,6 +72,7 @@ macro_rules! expand_align { __align: [::c_long; 0], #[cfg(not(any(target_arch = "mips", target_arch = "arm", + target_arch = "m68k", target_arch = "powerpc", target_arch = "sparc", all(target_arch = "x86_64", @@ -67,6 +84,7 @@ macro_rules! expand_align { pub struct pthread_rwlock_t { #[cfg(any(target_arch = "mips", target_arch = "arm", + target_arch = "m68k", target_arch = "powerpc", target_arch = "sparc", all(target_arch = "x86_64", @@ -74,6 +92,7 @@ macro_rules! expand_align { __align: [::c_long; 0], #[cfg(not(any(target_arch = "mips", target_arch = "arm", + target_arch = "m68k", target_arch = "powerpc", target_arch = "sparc", all(target_arch = "x86_64", diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/non_exhaustive.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/non_exhaustive.rs new file mode 100644 index 000000000..e2e2cb847 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/non_exhaustive.rs @@ -0,0 +1,9 @@ +s! { + // linux/openat2.h + #[non_exhaustive] + pub struct open_how { + pub flags: ::__u64, + pub mode: ::__u64, + pub resolve: ::__u64, + } +} diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/align.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/align.rs new file mode 100644 index 000000000..e6610bb7b --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/align.rs @@ -0,0 +1,28 @@ +macro_rules! expand_align { + () => { + s! { + #[cfg_attr(any(target_pointer_width = "32", + target_arch = "x86_64", + target_arch = "powerpc64", + target_arch = "mips64", + target_arch = "s390x", + target_arch = "sparc64"), + repr(align(4)))] + #[cfg_attr(not(any(target_pointer_width = "32", + target_arch = "x86_64", + target_arch = "powerpc64", + target_arch = "mips64", + target_arch = "s390x", + target_arch = "sparc64")), + repr(align(8)))] + pub struct pthread_mutexattr_t { + size: [u8; ::__SIZEOF_PTHREAD_MUTEXATTR_T], + } + + #[repr(align(4))] + pub struct pthread_condattr_t { + size: [u8; ::__SIZEOF_PTHREAD_CONDATTR_T], + } + } + }; +} diff --git a/crux-mir/lib/libc/src/unix/uclibc/arm/align.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/arm/align.rs similarity index 100% rename from crux-mir/lib/libc/src/unix/uclibc/arm/align.rs rename to crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/arm/align.rs diff --git a/crux-mir/lib/libc/src/unix/uclibc/arm/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/arm/mod.rs similarity index 74% rename from crux-mir/lib/libc/src/unix/uclibc/arm/mod.rs rename to crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/arm/mod.rs index 613a11fba..25125bcc9 100644 --- a/crux-mir/lib/libc/src/unix/uclibc/arm/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/arm/mod.rs @@ -10,13 +10,17 @@ pub type fsfilcnt_t = ::c_ulong; pub type ino_t = ::c_ulong; pub type off_t = ::c_long; pub type pthread_t = ::c_ulong; -pub type rlim_t = ::c_ulong; pub type suseconds_t = ::c_long; pub type nlink_t = ::c_uint; pub type blksize_t = ::c_long; pub type blkcnt_t = ::c_long; +pub type fsblkcnt64_t = u64; +pub type fsfilcnt64_t = u64; +pub type __u64 = ::c_ulonglong; +pub type __s64 = ::c_longlong; + s! { pub struct cmsghdr { pub cmsg_len: ::size_t, @@ -40,25 +44,25 @@ s! { pub struct stat { pub st_dev: ::c_ulonglong, - pub __pad1: ::c_ushort, + __pad1: ::c_ushort, pub st_ino: ::ino_t, pub st_mode: ::mode_t, pub st_nlink: ::nlink_t, pub st_uid: ::uid_t, pub st_gid: ::gid_t, pub st_rdev: ::c_ulonglong, - pub __pad2: ::c_ushort, + __pad2: ::c_ushort, pub st_size: ::off_t, pub st_blksize: ::blksize_t, pub st_blocks: ::blkcnt_t, pub st_atime: ::time_t, - pub st_atime_nsec: ::c_ulong, + pub st_atime_nsec: ::c_long, pub st_mtime: ::time_t, - pub st_mtime_nsec: ::c_ulong, + pub st_mtime_nsec: ::c_long, pub st_ctime: ::time_t, - pub st_ctime_nsec: ::c_ulong, - pub __unused4: ::c_ulong, - pub __unused5: ::c_ulong, + pub st_ctime_nsec: ::c_long, + __unused4: ::c_ulong, + __unused5: ::c_ulong, } pub struct stat64 @@ -76,11 +80,11 @@ s! { pub st_blksize: ::blksize_t, pub st_blocks: ::blkcnt64_t, pub st_atime: ::time_t, - pub st_atime_nsec: ::c_ulong, + pub st_atime_nsec: ::c_long, pub st_mtime: ::time_t, - pub st_mtime_nsec: ::c_ulong, + pub st_mtime_nsec: ::c_long, pub st_ctime: ::time_t, - pub st_ctime_nsec: ::c_ulong, + pub st_ctime_nsec: ::c_long, pub st_ino: ::ino64_t, } @@ -92,6 +96,23 @@ s! { pub l_pid: ::pid_t, } + pub struct sysinfo { + pub uptime: ::c_long, + pub loads: [::c_ulong; 3], + pub totalram: ::c_ulong, + pub freeram: ::c_ulong, + pub sharedram: ::c_ulong, + pub bufferram: ::c_ulong, + pub totalswap: ::c_ulong, + pub freeswap: ::c_ulong, + pub procs: ::c_ushort, + pub pad: ::c_ushort, + pub totalhigh: ::c_ulong, + pub freehigh: ::c_ulong, + pub mem_unit: ::c_uint, + pub _f: [::c_char; 8], + } + pub struct statfs { pub f_type: ::c_int, pub f_bsize: ::c_int, @@ -104,7 +125,39 @@ s! { pub f_fsid: ::fsid_t, pub f_namelen: ::c_int, pub f_frsize: ::c_int, - pub f_spare: [::c_int; 5], + pub f_flags: ::c_int, + pub f_spare: [::c_int; 4], + } + + pub struct statfs64 { + pub f_type: ::c_int, + pub f_bsize: ::c_int, + pub f_blocks: ::fsblkcnt64_t, + pub f_bfree: ::fsblkcnt64_t, + pub f_bavail: ::fsblkcnt64_t, + pub f_files: ::fsfilcnt64_t, + pub f_ffree: ::fsfilcnt64_t, + pub f_fsid: ::fsid_t, + pub f_namelen: ::c_int, + pub f_frsize: ::c_int, + pub f_flags: ::c_int, + pub f_spare: [::c_int; 4], + } + + pub struct statvfs64 { + pub f_bsize: ::c_ulong, + pub f_frsize: ::c_ulong, + pub f_blocks: u64, + pub f_bfree: u64, + pub f_bavail: u64, + pub f_files: u64, + pub f_ffree: u64, + pub f_favail: u64, + pub f_fsid: ::c_ulong, + __f_unused: ::c_int, + pub f_flag: ::c_ulong, + pub f_namemax: ::c_ulong, + __f_spare: [::c_int; 6], } pub struct sigset_t { @@ -113,10 +166,8 @@ s! { pub struct sigaction { pub sa_sigaction: ::sighandler_t, - // uClibc defines sa_flags as `unsigned long int`, - // but nix crate expects `int` - pub sa_flags: ::c_int, - pub sa_restorer: *mut ::c_void, + pub sa_flags: ::c_ulong, + pub sa_restorer: ::Option, pub sa_mask: sigset_t, } @@ -151,56 +202,48 @@ s! { pub cuid: ::uid_t, pub cgid: ::gid_t, pub mode: ::c_ushort, - pub __pad1: ::c_ushort, + __pad1: ::c_ushort, pub __seq: ::c_ushort, - pub __pad2: ::c_ushort, - pub __unused1: ::c_ulong, - pub __unused2: ::c_ulong, + __pad2: ::c_ushort, + __unused1: ::c_ulong, + __unused2: ::c_ulong, } pub struct msqid_ds { pub msg_perm: ::ipc_perm, pub msg_stime: ::time_t, - pub __unused1: ::c_ulong, + __unused1: ::c_ulong, pub msg_rtime: ::time_t, - pub __unused2: ::c_ulong, + __unused2: ::c_ulong, pub msg_ctime: ::time_t, - pub __unused3: ::c_ulong, - pub __msg_cbytes: ::c_ulong, + __unused3: ::c_ulong, + __msg_cbytes: ::c_ulong, pub msg_qnum: ::msgqnum_t, pub msg_qbytes: ::msglen_t, pub msg_lspid: ::pid_t, pub msg_lrpid: ::pid_t, - pub __unused4: ::c_ulong, - pub __unused5: ::c_ulong, + __unused4: ::c_ulong, + __unused5: ::c_ulong, } pub struct shmid_ds { pub shm_perm: ::ipc_perm, pub shm_segsz: ::size_t, pub shm_atime: ::time_t, - pub __unused1: ::c_ulong, + __unused1: ::c_ulong, pub shm_dtime: ::time_t, - pub __unused2: ::c_ulong, + __unused2: ::c_ulong, pub shm_ctime: ::time_t, - pub __unused3: ::c_ulong, + __unused3: ::c_ulong, pub shm_cpid: ::pid_t, pub shm_lpid: ::pid_t, pub shm_nattch: ::shmatt_t, - pub __unused4: ::c_ulong, - pub __unused5: ::c_ulong, + __unused4: ::c_ulong, + __unused5: ::c_ulong, } - - pub struct ucred { - pub pid: ::pid_t, - pub uid: ::uid_t, - pub gid: ::gid_t, - } - } pub const O_CLOEXEC: ::c_int = 0o2000000; -pub const RLIM_INFINITY: rlim_t = !0; pub const __SIZEOF_PTHREAD_ATTR_T: usize = 36; pub const __SIZEOF_PTHREAD_MUTEX_T: usize = 24; pub const __SIZEOF_PTHREAD_MUTEXATTR_T: usize = 4; @@ -214,33 +257,9 @@ pub const NCCS: usize = 32; // I wasn't able to find those constants // in uclibc build environment for armv7 -pub const AIO_ALLDONE: ::c_int = 2; // from linux/mod.rs -pub const AIO_CANCELED: ::c_int = 0; // from linux/mod.rs -pub const AIO_NOTCANCELED: ::c_int = 1; // from linux/mod.rs -pub const CLONE_NEWCGROUP: ::c_int = 0x02000000; // from linux/mod.rs -pub const EPOLLEXCLUSIVE: ::c_int = 0x10000000; // from linux/mod.rs -pub const EPOLLWAKEUP: ::c_int = 0x20000000; // from linux/other/mod.rs -pub const EXTPROC: ::tcflag_t = 0o200000; // from asm-generic/termbits.h -pub const F_GETPIPE_SZ: ::c_int = 1032; // from linux_like/mod.rs -pub const F_SETPIPE_SZ: ::c_int = 1031; // from linux_like/mod.rs -pub const LIO_NOP: ::c_int = 2; // from linux/mod.rs -pub const LIO_NOWAIT: ::c_int = 1; // from linux/mod.rs -pub const LIO_READ: ::c_int = 0; // from linux/mod.rs -pub const LIO_WAIT: ::c_int = 0; // from linux/mod.rs -pub const LIO_WRITE: ::c_int = 1; // from linux/mod.rs pub const MAP_HUGETLB: ::c_int = 0x040000; // from linux/other/mod.rs -pub const O_TMPFILE: ::c_int = 0o20000000 | O_DIRECTORY; -pub const RB_KEXEC: ::c_int = 0x45584543u32 as i32; // from linux/mod.rs -pub const RB_SW_SUSPEND: ::c_int = 0xd000fce2u32 as i32; // from linux/mod.rs -pub const SO_BUSY_POLL: ::c_int = 46; // from src/unix/linux_like/mod.rs -pub const SO_PEEK_OFF: ::c_int = 42; // from src/unix/linux_like/mod.rs -pub const SO_REUSEPORT: ::c_int = 15; // from src/unix/linux_like/mod.rs -pub const SOL_NETLINK: ::c_int = 270; // from src/unix/linux_like/mod.rs -pub const _POSIX_VDISABLE: ::cc_t = 0; // from linux/mod.rs -pub const AT_EMPTY_PATH: ::c_int = 0x1000; // from linux_like/mod.rs // autogenerated constants with hand tuned types -pub const AT_NO_AUTOMOUNT: ::c_int = 0x800; pub const B0: ::speed_t = 0; pub const B1000000: ::speed_t = 0x1008; pub const B110: ::speed_t = 0x3; @@ -278,7 +297,6 @@ pub const CBAUD: ::tcflag_t = 0x100f; pub const CBAUDEX: ::tcflag_t = 0x1000; pub const CIBAUD: ::tcflag_t = 0x100f0000; pub const CLOCAL: ::tcflag_t = 0x800; -pub const CMSPAR: ::tcflag_t = 0x40000000; pub const CPU_SETSIZE: ::c_int = 0x400; pub const CR1: ::c_int = 0x200; pub const CR2: ::c_int = 0x400; @@ -398,9 +416,6 @@ pub const EUSERS: ::c_int = 0x57; pub const EXFULL: ::c_int = 0x36; pub const FF1: ::c_int = 0x8000; pub const FFDLY: ::c_int = 0x8000; -pub const FIONBIO: ::c_ulong = 0x5421; -pub const FIOCLEX: ::c_ulong = 0x5451; -pub const FIONCLEX: ::c_ulong = 0x5450; pub const FLUSHO: ::tcflag_t = 0x1000; pub const F_GETLK: ::c_int = 0x5; pub const F_SETLK: ::c_int = 0x6; @@ -409,7 +424,6 @@ pub const HUPCL: ::tcflag_t = 0x400; pub const ICANON: ::tcflag_t = 0x2; pub const IEXTEN: ::tcflag_t = 0x8000; pub const ISIG: ::tcflag_t = 0x1; -pub const IUTF8: ::tcflag_t = 0x4000; pub const IXOFF: ::tcflag_t = 0x1000; pub const IXON: ::tcflag_t = 0x400; pub const MAP_ANON: ::c_int = 0x20; @@ -422,67 +436,46 @@ pub const MAP_NONBLOCK: ::c_int = 0x10000; pub const MAP_NORESERVE: ::c_int = 0x4000; pub const MAP_POPULATE: ::c_int = 0x8000; pub const MAP_STACK: ::c_int = 0x20000; -pub const MS_ACTIVE: u32 = 0x40000000; -pub const MS_DIRSYNC: u32 = 0x80; -pub const MS_I_VERSION: u32 = 0x800000; -pub const MS_KERNMOUNT: u32 = 0x400000; -pub const MS_MOVE: u32 = 0x2000; -pub const MS_POSIXACL: u32 = 0x10000; -pub const MS_PRIVATE: u32 = 0x40000; -pub const MS_REC: u32 = 0x4000; -pub const MS_RELATIME: u32 = 0x200000; -pub const MS_SHARED: u32 = 0x100000; -pub const MS_SILENT: u32 = 0x8000; -pub const MS_SLAVE: u32 = 0x80000; -pub const MS_STRICTATIME: u32 = 0x1000000; -pub const MS_UNBINDABLE: u32 = 0x20000; pub const NLDLY: ::tcflag_t = 0x100; pub const NOFLSH: ::tcflag_t = 0x80; -pub const OCRNL: ::c_int = 0x8; -pub const OFDEL: ::c_int = 0x80; -pub const OFILL: ::c_int = 0x40; pub const OLCUC: ::tcflag_t = 0x2; pub const ONLCR: ::tcflag_t = 0x4; -pub const ONLRET: ::tcflag_t = 0x20; -pub const ONOCR: ::tcflag_t = 0x10; pub const O_ACCMODE: ::c_int = 0x3; pub const O_APPEND: ::c_int = 0x400; +pub const O_ASYNC: ::c_int = 0o20000; pub const O_CREAT: ::c_int = 0x40; pub const O_DIRECT: ::c_int = 0x10000; pub const O_DIRECTORY: ::c_int = 0x4000; -pub const O_DSYNC: ::c_int = 0x1000; +pub const O_DSYNC: ::c_int = O_SYNC; pub const O_EXCL: ::c_int = 0x80; -pub const O_NDELAY: ::c_int = 0x800; +pub const O_FSYNC: ::c_int = O_SYNC; +pub const O_LARGEFILE: ::c_int = 0o400000; +pub const O_NDELAY: ::c_int = O_NONBLOCK; +pub const O_NOATIME: ::c_int = 0o1000000; pub const O_NOCTTY: ::c_int = 0x100; pub const O_NOFOLLOW: ::c_int = 0x8000; pub const O_NONBLOCK: ::c_int = 0x800; +pub const O_PATH: ::c_int = 0o10000000; +pub const O_RSYNC: ::c_int = O_SYNC; pub const O_SYNC: ::c_int = 0o10000; pub const O_TRUNC: ::c_int = 0x200; pub const PARENB: ::tcflag_t = 0x100; pub const PARODD: ::tcflag_t = 0x200; pub const PENDIN: ::tcflag_t = 0x4000; -pub const POLLRDBAND: ::c_short = 0x80; -pub const POLLRDNORM: ::c_short = 0x40; pub const POLLWRBAND: ::c_short = 0x200; pub const POLLWRNORM: ::c_short = 0x100; pub const PTHREAD_STACK_MIN: ::size_t = 16384; -pub const QIF_ALL: u32 = 0x3f; -pub const QIF_BLIMITS: u32 = 0x1; -pub const QIF_BTIME: u32 = 0x10; -pub const QIF_ILIMITS: u32 = 0x4; -pub const QIF_INODES: u32 = 0x8; -pub const QIF_ITIME: u32 = 0x20; -pub const QIF_LIMITS: u32 = 0x5; -pub const QIF_SPACE: u32 = 0x2; -pub const QIF_TIMES: u32 = 0x30; -pub const QIF_USAGE: u32 = 0xa; -pub const SA_NOCLDSTOP: ::c_int = 0x1; -pub const SA_NOCLDWAIT: ::c_int = 0x2; -pub const SA_NODEFER: ::c_int = 0x40000000; -pub const SA_ONSTACK: ::c_int = 0x8000000; -pub const SA_RESETHAND: ::c_int = 0x80000000; -pub const SA_RESTART: ::c_int = 0x10000000; -pub const SA_SIGINFO: ::c_int = 0x4; +pub const RTLD_GLOBAL: ::c_int = 0x00100; + +// These are typed unsigned to match sigaction +pub const SA_NOCLDSTOP: ::c_ulong = 0x1; +pub const SA_NOCLDWAIT: ::c_ulong = 0x2; +pub const SA_SIGINFO: ::c_ulong = 0x4; +pub const SA_NODEFER: ::c_ulong = 0x40000000; +pub const SA_ONSTACK: ::c_ulong = 0x8000000; +pub const SA_RESETHAND: ::c_ulong = 0x80000000; +pub const SA_RESTART: ::c_ulong = 0x10000000; + pub const SFD_CLOEXEC: ::c_int = 0x80000; pub const SFD_NONBLOCK: ::c_int = 0x800; pub const SIGBUS: ::c_int = 0x7; @@ -512,33 +505,7 @@ pub const SOCK_DGRAM: ::c_int = 0x2; pub const SOCK_NONBLOCK: ::c_int = 0o0004000; pub const SOCK_SEQPACKET: ::c_int = 0x5; pub const SOCK_STREAM: ::c_int = 0x1; -pub const SOL_SOCKET: ::c_int = 0x1; -pub const SO_ACCEPTCONN: ::c_int = 0x1e; -pub const SO_BINDTODEVICE: ::c_int = 0x19; -pub const SO_BROADCAST: ::c_int = 0x6; -pub const SO_BSDCOMPAT: ::c_int = 0xe; -pub const SO_DOMAIN: ::c_int = 0x27; -pub const SO_DONTROUTE: ::c_int = 0x5; -pub const SO_ERROR: ::c_int = 0x4; -pub const SO_KEEPALIVE: ::c_int = 0x9; -pub const SO_LINGER: ::c_int = 0xd; -pub const SO_MARK: ::c_int = 0x24; -pub const SO_OOBINLINE: ::c_int = 0xa; -pub const SO_PASSCRED: ::c_int = 0x10; -pub const SO_PEERCRED: ::c_int = 0x11; -pub const SO_PRIORITY: ::c_int = 0xc; -pub const SO_PROTOCOL: ::c_int = 0x26; -pub const SO_RCVBUF: ::c_int = 0x8; -pub const SO_RCVLOWAT: ::c_int = 0x12; -pub const SO_RCVTIMEO: ::c_int = 0x14; -pub const SO_REUSEADDR: ::c_int = 0x2; -pub const SO_RXQ_OVFL: ::c_int = 0x28; -pub const SO_SNDBUF: ::c_int = 0x7; -pub const SO_SNDBUFFORCE: ::c_int = 0x20; -pub const SO_SNDLOWAT: ::c_int = 0x13; -pub const SO_SNDTIMEO: ::c_int = 0x15; -pub const SO_TIMESTAMP: ::c_int = 0x1d; -pub const SO_TYPE: ::c_int = 0x3; + pub const TAB1: ::c_int = 0x800; pub const TAB2: ::c_int = 0x1000; pub const TAB3: ::c_int = 0x1800; @@ -562,58 +529,8 @@ pub const VTDLY: ::c_int = 0x4000; pub const VTIME: usize = 0x5; pub const VWERASE: usize = 0xe; pub const XTABS: ::tcflag_t = 0x1800; -pub const _PC_2_SYMLINKS: ::c_int = 0x14; -pub const _PC_ALLOC_SIZE_MIN: ::c_int = 0x12; -pub const _PC_ASYNC_IO: ::c_int = 0xa; -pub const _PC_FILESIZEBITS: ::c_int = 0xd; -pub const _PC_PRIO_IO: ::c_int = 0xb; -pub const _PC_REC_INCR_XFER_SIZE: ::c_int = 0xe; -pub const _PC_REC_MAX_XFER_SIZE: ::c_int = 0xf; -pub const _PC_REC_MIN_XFER_SIZE: ::c_int = 0x10; -pub const _PC_REC_XFER_ALIGN: ::c_int = 0x11; -pub const _PC_SYMLINK_MAX: ::c_int = 0x13; -pub const _PC_SYNC_IO: ::c_int = 0x9; -pub const _SC_2_PBS: ::c_int = 0xa8; -pub const _SC_2_PBS_ACCOUNTING: ::c_int = 0xa9; -pub const _SC_2_PBS_CHECKPOINT: ::c_int = 0xaf; -pub const _SC_2_PBS_LOCATE: ::c_int = 0xaa; -pub const _SC_2_PBS_MESSAGE: ::c_int = 0xab; -pub const _SC_2_PBS_TRACK: ::c_int = 0xac; -pub const _SC_ADVISORY_INFO: ::c_int = 0x84; -pub const _SC_BARRIERS: ::c_int = 0x85; -pub const _SC_CLOCK_SELECTION: ::c_int = 0x89; -pub const _SC_CPUTIME: ::c_int = 0x8a; -pub const _SC_IPV6: ::c_int = 0xeb; -pub const _SC_MONOTONIC_CLOCK: ::c_int = 0x95; -pub const _SC_RAW_SOCKETS: ::c_int = 0xec; -pub const _SC_READER_WRITER_LOCKS: ::c_int = 0x99; -pub const _SC_REGEXP: ::c_int = 0x9b; -pub const _SC_SHELL: ::c_int = 0x9d; -pub const _SC_SPAWN: ::c_int = 0x9f; -pub const _SC_SPIN_LOCKS: ::c_int = 0x9a; -pub const _SC_SPORADIC_SERVER: ::c_int = 0xa0; -pub const _SC_SS_REPL_MAX: ::c_int = 0xf1; -pub const _SC_SYMLOOP_MAX: ::c_int = 0xad; -pub const _SC_THREAD_CPUTIME: ::c_int = 0x8b; -pub const _SC_THREAD_PROCESS_SHARED: ::c_int = 0x52; -pub const _SC_THREAD_ROBUST_PRIO_INHERIT: ::c_int = 0xf7; -pub const _SC_THREAD_ROBUST_PRIO_PROTECT: ::c_int = 0xf8; -pub const _SC_THREAD_SPORADIC_SERVER: ::c_int = 0xa1; -pub const _SC_TIMEOUTS: ::c_int = 0xa4; -pub const _SC_TRACE: ::c_int = 0xb5; -pub const _SC_TRACE_EVENT_FILTER: ::c_int = 0xb6; -pub const _SC_TRACE_EVENT_NAME_MAX: ::c_int = 0xf2; -pub const _SC_TRACE_INHERIT: ::c_int = 0xb7; -pub const _SC_TRACE_LOG: ::c_int = 0xb8; -pub const _SC_TRACE_NAME_MAX: ::c_int = 0xf3; -pub const _SC_TRACE_SYS_MAX: ::c_int = 0xf4; -pub const _SC_TRACE_USER_EVENT_MAX: ::c_int = 0xf5; -pub const _SC_TYPED_MEMORY_OBJECTS: ::c_int = 0xa5; -pub const _SC_V6_ILP32_OFF32: ::c_int = 0xb0; -pub const _SC_V6_ILP32_OFFBIG: ::c_int = 0xb1; -pub const _SC_V6_LP64_OFF64: ::c_int = 0xb2; -pub const _SC_V6_LPBIG_OFFBIG: ::c_int = 0xb3; -pub const _SC_XOPEN_STREAMS: ::c_int = 0xf6; + +pub const MADV_SOFT_OFFLINE: ::c_int = 101; // Syscall table is copied from src/unix/notbsd/linux/musl/b32/arm.rs pub const SYS_restart_syscall: ::c_long = 0; @@ -966,77 +883,11 @@ pub const SYS_pwritev2: ::c_long = 393; pub const SYS_pkey_mprotect: ::c_long = 394; pub const SYS_pkey_alloc: ::c_long = 395; pub const SYS_pkey_free: ::c_long = 396; - -fn CMSG_ALIGN(len: usize) -> usize { - len + ::mem::size_of::() - 1 & !(::mem::size_of::() - 1) -} - -f! { - pub fn CMSG_FIRSTHDR(mhdr: *const msghdr) -> *mut cmsghdr { - if (*mhdr).msg_controllen as usize >= ::mem::size_of::() { - (*mhdr).msg_control as *mut cmsghdr - } else { - 0 as *mut cmsghdr - } - } - - pub fn CMSG_DATA(cmsg: *const cmsghdr) -> *mut ::c_uchar { - cmsg.offset(1) as *mut ::c_uchar - } - - pub fn CMSG_SPACE(length: ::c_uint) -> ::c_uint { - (CMSG_ALIGN(length as usize) + CMSG_ALIGN(::mem::size_of::())) - as ::c_uint - } - - pub fn CMSG_LEN(length: ::c_uint) -> ::c_uint { - CMSG_ALIGN(::mem::size_of::()) as ::c_uint + length - } - - pub fn CMSG_NXTHDR(mhdr: *const msghdr, - cmsg: *const cmsghdr) -> *mut cmsghdr { - if ((*cmsg).cmsg_len as usize) < ::mem::size_of::() { - return 0 as *mut cmsghdr; - }; - let next = (cmsg as usize + - CMSG_ALIGN((*cmsg).cmsg_len as usize)) - as *mut cmsghdr; - let max = (*mhdr).msg_control as usize - + (*mhdr).msg_controllen as usize; - if (next.offset(1)) as usize > max || - next as usize + CMSG_ALIGN((*next).cmsg_len as usize) > max - { - 0 as *mut cmsghdr - } else { - next as *mut cmsghdr - } - } - -} - -extern "C" { - pub fn ioctl(fd: ::c_int, request: ::c_ulong, ...) -> ::c_int; - pub fn openpty( - amaster: *mut ::c_int, - aslave: *mut ::c_int, - name: *mut ::c_char, - termp: *mut termios, - winp: *mut ::winsize, - ) -> ::c_int; - pub fn setns(fd: ::c_int, nstype: ::c_int) -> ::c_int; - pub fn pwritev( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - offset: ::off_t, - ) -> ::ssize_t; - pub fn preadv( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - offset: ::off_t, - ) -> ::ssize_t; -} +pub const SYS_statx: ::c_int = 397; +pub const SYS_pidfd_send_signal: ::c_long = 424; +pub const SYS_pidfd_open: ::c_long = 434; +pub const SYS_clone3: ::c_long = 435; +pub const SYS_pidfd_getfd: ::c_long = 438; cfg_if! { if #[cfg(libc_align)] { diff --git a/crux-mir/lib/libc/src/unix/uclibc/arm/no_align.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/arm/no_align.rs similarity index 100% rename from crux-mir/lib/libc/src/unix/uclibc/arm/no_align.rs rename to crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/arm/no_align.rs diff --git a/crux-mir/lib/libc/src/unix/uclibc/mips/mips32/align.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/mips/mips32/align.rs similarity index 100% rename from crux-mir/lib/libc/src/unix/uclibc/mips/mips32/align.rs rename to crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/mips/mips32/align.rs diff --git a/crux-mir/lib/libc/src/unix/uclibc/mips/mips32/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/mips/mips32/mod.rs similarity index 94% rename from crux-mir/lib/libc/src/unix/uclibc/mips/mips32/mod.rs rename to crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/mips/mips32/mod.rs index 31bca589e..7f9d3c031 100644 --- a/crux-mir/lib/libc/src/unix/uclibc/mips/mips32/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/mips/mips32/mod.rs @@ -12,7 +12,10 @@ pub type blksize_t = i32; pub type nlink_t = u32; pub type fsblkcnt_t = ::c_ulong; pub type fsfilcnt_t = ::c_ulong; -pub type rlim_t = c_ulong; +pub type __u64 = ::c_ulonglong; +pub type __s64 = ::c_longlong; +pub type fsblkcnt64_t = u64; +pub type fsfilcnt64_t = u64; s! { pub struct stat { @@ -61,6 +64,22 @@ s! { st_pad5: [::c_long; 14], } + pub struct statvfs64 { + pub f_bsize: ::c_ulong, + pub f_frsize: ::c_ulong, + pub f_blocks: ::fsblkcnt64_t, + pub f_bfree: ::fsblkcnt64_t, + pub f_bavail: ::fsblkcnt64_t, + pub f_files: ::fsfilcnt64_t, + pub f_ffree: ::fsfilcnt64_t, + pub f_favail: ::fsfilcnt64_t, + pub f_fsid: ::c_ulong, + pub __f_unused: ::c_int, + pub f_flag: ::c_ulong, + pub f_namemax: ::c_ulong, + pub __f_spare: [::c_int; 6], + } + pub struct pthread_attr_t { __size: [u32; 9] } @@ -169,6 +188,21 @@ s! { f_spare: [::c_long; 6], } + pub struct statfs64 { + pub f_type: ::c_long, + pub f_bsize: ::c_long, + pub f_frsize: ::c_long, + pub f_blocks: ::fsblkcnt64_t, + pub f_bfree: ::fsblkcnt64_t, + pub f_files: ::fsblkcnt64_t, + pub f_ffree: ::fsblkcnt64_t, + pub f_bavail: ::fsblkcnt64_t, + pub f_fsid: ::fsid_t, + pub f_namelen: ::c_long, + pub f_flags: ::c_long, + pub f_spare: [::c_long; 5], + } + pub struct msghdr { pub msg_name: *mut ::c_void, pub msg_namelen: ::socklen_t, @@ -222,12 +256,14 @@ s! { } } -pub const __SIZEOF_PTHREAD_CONDATTR_T: usize = 4; +pub const __SIZEOF_PTHREAD_ATTR_T: usize = 36; pub const __SIZEOF_PTHREAD_MUTEX_T: usize = 24; -pub const __SIZEOF_PTHREAD_RWLOCK_T: usize = 32; pub const __SIZEOF_PTHREAD_MUTEXATTR_T: usize = 4; - -pub const RLIM_INFINITY: ::rlim_t = 0x7fffffff; +pub const __SIZEOF_PTHREAD_CONDATTR_T: usize = 4; +pub const __SIZEOF_PTHREAD_RWLOCK_T: usize = 32; +pub const __SIZEOF_PTHREAD_RWLOCKATTR_T: usize = 8; +pub const __SIZEOF_PTHREAD_BARRIER_T: usize = 20; +pub const __SIZEOF_PTHREAD_BARRIERATTR_T: usize = 4; pub const SYS_syscall: ::c_long = 4000 + 0; pub const SYS_exit: ::c_long = 4000 + 1; @@ -587,6 +623,7 @@ pub const SYS_pwritev2: ::c_long = 4000 + 362; pub const SYS_pkey_mprotect: ::c_long = 4000 + 363; pub const SYS_pkey_alloc: ::c_long = 4000 + 364; pub const SYS_pkey_free: ::c_long = 4000 + 365; +pub const SYS_clone3: ::c_long = 4000 + 435; #[link(name = "util")] extern "C" { @@ -598,18 +635,13 @@ extern "C" { newp: *mut ::c_void, newlen: ::size_t, ) -> ::c_int; - pub fn ioctl(fd: ::c_int, request: ::c_ulong, ...) -> ::c_int; - pub fn backtrace(buf: *mut *mut ::c_void, sz: ::c_int) -> ::c_int; pub fn glob64( pattern: *const ::c_char, flags: ::c_int, - errfunc: ::Option< - extern "C" fn(epath: *const ::c_char, errno: ::c_int) -> ::c_int, - >, + errfunc: ::Option ::c_int>, pglob: *mut glob64_t, ) -> ::c_int; pub fn globfree64(pglob: *mut glob64_t); - pub fn ptrace(request: ::c_uint, ...) -> ::c_long; pub fn pthread_attr_getaffinity_np( attr: *const ::pthread_attr_t, cpusetsize: ::size_t, diff --git a/crux-mir/lib/libc/src/unix/uclibc/mips/mips32/no_align.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/mips/mips32/no_align.rs similarity index 100% rename from crux-mir/lib/libc/src/unix/uclibc/mips/mips32/no_align.rs rename to crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/mips/mips32/no_align.rs diff --git a/crux-mir/lib/libc/src/unix/uclibc/mips/mips64/align.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/mips/mips64/align.rs similarity index 100% rename from crux-mir/lib/libc/src/unix/uclibc/mips/mips64/align.rs rename to crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/mips/mips64/align.rs diff --git a/crux-mir/lib/libc/src/unix/uclibc/mips/mips64/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/mips/mips64/mod.rs similarity index 96% rename from crux-mir/lib/libc/src/unix/uclibc/mips/mips64/mod.rs rename to crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/mips/mips64/mod.rs index 735eb851e..be6d2813d 100644 --- a/crux-mir/lib/libc/src/unix/uclibc/mips/mips64/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/mips/mips64/mod.rs @@ -8,7 +8,6 @@ pub type fsfilcnt_t = ::c_ulong; pub type ino_t = u64; pub type nlink_t = u64; pub type off_t = i64; -pub type rlim_t = ::c_ulong; pub type suseconds_t = i64; pub type time_t = i64; pub type wchar_t = i32; @@ -193,15 +192,8 @@ pub const __SIZEOF_PTHREAD_MUTEXATTR_T: usize = 4; pub const __SIZEOF_PTHREAD_MUTEX_T: usize = 40; pub const __SIZEOF_PTHREAD_RWLOCK_T: usize = 56; -pub const RLIM_INFINITY: ::rlim_t = 0xffff_ffff_ffff_ffff; - pub const SYS_gettid: ::c_long = 5178; // Valid for n64 -#[link(name = "util")] -extern "C" { - pub fn ioctl(fd: ::c_int, request: ::c_ulong, ...) -> ::c_int; -} - cfg_if! { if #[cfg(libc_align)] { mod align; diff --git a/crux-mir/lib/libc/src/unix/uclibc/mips/mips64/no_align.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/mips/mips64/no_align.rs similarity index 100% rename from crux-mir/lib/libc/src/unix/uclibc/mips/mips64/no_align.rs rename to crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/mips/mips64/no_align.rs diff --git a/crux-mir/lib/libc/src/unix/uclibc/mips/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/mips/mod.rs similarity index 55% rename from crux-mir/lib/libc/src/unix/uclibc/mips/mod.rs rename to crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/mips/mod.rs index 6a9b41c9f..61684094f 100644 --- a/crux-mir/lib/libc/src/unix/uclibc/mips/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/mips/mod.rs @@ -32,26 +32,12 @@ pub const EPOLL_CLOEXEC: ::c_int = 0x80000; pub const EFD_CLOEXEC: ::c_int = 0x80000; -pub const BUFSIZ: ::c_uint = 4096; pub const TMP_MAX: ::c_uint = 238328; -pub const FOPEN_MAX: ::c_uint = 16; -pub const POSIX_FADV_DONTNEED: ::c_int = 4; -pub const POSIX_FADV_NOREUSE: ::c_int = 5; -pub const POSIX_MADV_DONTNEED: ::c_int = 4; pub const _SC_2_C_VERSION: ::c_int = 96; pub const O_ACCMODE: ::c_int = 3; pub const O_DIRECT: ::c_int = 0x8000; pub const O_DIRECTORY: ::c_int = 0x10000; pub const O_NOFOLLOW: ::c_int = 0x20000; -pub const ST_RELATIME: ::c_ulong = 4096; -pub const NI_MAXHOST: ::socklen_t = 1025; - -pub const RLIMIT_NOFILE: ::c_int = 5; -pub const RLIMIT_AS: ::c_int = 6; -pub const RLIMIT_RSS: ::c_int = 7; -pub const RLIMIT_NPROC: ::c_int = 8; -pub const RLIMIT_MEMLOCK: ::c_int = 9; -pub const RLIMIT_NLIMITS: ::c_int = 15; pub const O_APPEND: ::c_int = 8; pub const O_CREAT: ::c_int = 256; @@ -89,7 +75,6 @@ pub const EXFULL: ::c_int = 52; pub const ENOANO: ::c_int = 53; pub const EBADRQC: ::c_int = 54; pub const EBADSLT: ::c_int = 55; -pub const EDEADLOCK: ::c_int = 56; pub const EMULTIHOP: ::c_int = 74; pub const EOVERFLOW: ::c_int = 79; pub const ENOTUNIQ: ::c_int = 80; @@ -113,7 +98,6 @@ pub const ENOPROTOOPT: ::c_int = 99; pub const EPROTONOSUPPORT: ::c_int = 120; pub const ESOCKTNOSUPPORT: ::c_int = 121; pub const EOPNOTSUPP: ::c_int = 122; -pub const ENOTSUP: ::c_int = EOPNOTSUPP; pub const EPFNOSUPPORT: ::c_int = 123; pub const EAFNOSUPPORT: ::c_int = 124; pub const EADDRINUSE: ::c_int = 125; @@ -167,66 +151,6 @@ pub const SOCK_STREAM: ::c_int = 2; pub const SOCK_DGRAM: ::c_int = 1; pub const SOCK_SEQPACKET: ::c_int = 5; -pub const SOL_SOCKET: ::c_int = 0xffff; - -pub const SO_REUSEADDR: ::c_int = 0x0004; -pub const SO_KEEPALIVE: ::c_int = 0x0008; -pub const SO_DONTROUTE: ::c_int = 0x0010; -pub const SO_BROADCAST: ::c_int = 0x0020; -pub const SO_LINGER: ::c_int = 0x0080; -pub const SO_OOBINLINE: ::c_int = 0x0100; -pub const SO_REUSEPORT: ::c_int = 0x0200; -pub const SO_TYPE: ::c_int = 0x1008; -pub const SO_STYLE: ::c_int = SO_TYPE; -pub const SO_ERROR: ::c_int = 0x1007; -pub const SO_SNDBUF: ::c_int = 0x1001; -pub const SO_RCVBUF: ::c_int = 0x1002; -pub const SO_SNDLOWAT: ::c_int = 0x1003; -pub const SO_RCVLOWAT: ::c_int = 0x1004; -pub const SO_SNDTIMEO: ::c_int = 0x1005; -pub const SO_RCVTIMEO: ::c_int = 0x1006; -pub const SO_ACCEPTCONN: ::c_int = 0x1009; -pub const SO_PROTOCOL: ::c_int = 0x1028; -pub const SO_DOMAIN: ::c_int = 0x1029; -pub const SO_NO_CHECK: ::c_int = 11; -pub const SO_PRIORITY: ::c_int = 12; -pub const SO_BSDCOMPAT: ::c_int = 14; -pub const SO_PASSCRED: ::c_int = 17; -pub const SO_PEERCRED: ::c_int = 18; -pub const SO_SECURITY_AUTHENTICATION: ::c_int = 22; -pub const SO_SECURITY_ENCRYPTION_TRANSPORT: ::c_int = 23; -pub const SO_SECURITY_ENCRYPTION_NETWORK: ::c_int = 24; -pub const SO_BINDTODEVICE: ::c_int = 25; -pub const SO_ATTACH_FILTER: ::c_int = 26; -pub const SO_DETACH_FILTER: ::c_int = 27; -pub const SO_GET_FILTER: ::c_int = SO_ATTACH_FILTER; -pub const SO_PEERNAME: ::c_int = 28; -pub const SO_TIMESTAMP: ::c_int = 29; -pub const SCM_TIMESTAMP: ::c_int = SO_TIMESTAMP; -pub const SO_PEERSEC: ::c_int = 30; -pub const SO_SNDBUFFORCE: ::c_int = 31; -pub const SO_RCVBUFFORCE: ::c_int = 33; -pub const SO_PASSSEC: ::c_int = 34; -pub const SO_TIMESTAMPNS: ::c_int = 35; -pub const SCM_TIMESTAMPNS: ::c_int = SO_TIMESTAMPNS; -pub const SO_MARK: ::c_int = 36; -pub const SO_TIMESTAMPING: ::c_int = 37; -pub const SCM_TIMESTAMPING: ::c_int = SO_TIMESTAMPING; -pub const SO_RXQ_OVFL: ::c_int = 40; -pub const SO_WIFI_STATUS: ::c_int = 41; -pub const SCM_WIFI_STATUS: ::c_int = SO_WIFI_STATUS; -pub const SO_PEEK_OFF: ::c_int = 42; -pub const SO_NOFCS: ::c_int = 43; -pub const SO_LOCK_FILTER: ::c_int = 44; -pub const SO_SELECT_ERR_QUEUE: ::c_int = 45; -pub const SO_BUSY_POLL: ::c_int = 46; -pub const SO_MAX_PACING_RATE: ::c_int = 47; -pub const SO_BPF_EXTENSIONS: ::c_int = 48; - -pub const FIOCLEX: ::c_ulong = 0x6601; -pub const FIONCLEX: ::c_ulong = 0x6602; -pub const FIONBIO: ::c_ulong = 0x667e; - pub const SA_ONSTACK: ::c_uint = 0x08000000; pub const SA_SIGINFO: ::c_uint = 0x00000008; pub const SA_NOCLDWAIT: ::c_int = 0x00010000; @@ -248,46 +172,16 @@ pub const SIGTSTP: ::c_int = 24; pub const SIGURG: ::c_int = 21; pub const SIGIO: ::c_int = 22; pub const SIGSYS: ::c_int = 12; -pub const SIGPOLL: ::c_int = 22; pub const SIGPWR: ::c_int = 19; pub const SIG_SETMASK: ::c_int = 3; pub const SIG_BLOCK: ::c_int = 0x1; pub const SIG_UNBLOCK: ::c_int = 0x2; -pub const POLLRDNORM: ::c_short = 0x040; pub const POLLWRNORM: ::c_short = 0x004; -pub const POLLRDBAND: ::c_short = 0x080; pub const POLLWRBAND: ::c_short = 0x100; pub const PTHREAD_STACK_MIN: ::size_t = 16384; -pub const ADFS_SUPER_MAGIC: ::c_long = 0x0000adf5; -pub const AFFS_SUPER_MAGIC: ::c_long = 0x0000adff; -pub const CODA_SUPER_MAGIC: ::c_long = 0x73757245; -pub const CRAMFS_MAGIC: ::c_long = 0x28cd3d45; -pub const EFS_SUPER_MAGIC: ::c_long = 0x00414a53; -pub const EXT2_SUPER_MAGIC: ::c_long = 0x0000ef53; -pub const EXT3_SUPER_MAGIC: ::c_long = 0x0000ef53; -pub const EXT4_SUPER_MAGIC: ::c_long = 0x0000ef53; -pub const HPFS_SUPER_MAGIC: ::c_long = 0xf995e849; -pub const HUGETLBFS_MAGIC: ::c_long = 0x958458f6; -pub const ISOFS_SUPER_MAGIC: ::c_long = 0x00009660; -pub const JFFS2_SUPER_MAGIC: ::c_long = 0x000072b6; -pub const MINIX_SUPER_MAGIC: ::c_long = 0x0000137f; -pub const MINIX_SUPER_MAGIC2: ::c_long = 0x0000138f; -pub const MINIX2_SUPER_MAGIC: ::c_long = 0x00002468; -pub const MINIX2_SUPER_MAGIC2: ::c_long = 0x00002478; -pub const MSDOS_SUPER_MAGIC: ::c_long = 0x00004d44; -pub const NCP_SUPER_MAGIC: ::c_long = 0x0000564c; -pub const NFS_SUPER_MAGIC: ::c_long = 0x00006969; -pub const OPENPROM_SUPER_MAGIC: ::c_long = 0x00009fa1; -pub const PROC_SUPER_MAGIC: ::c_long = 0x00009fa0; -pub const QNX4_SUPER_MAGIC: ::c_long = 0x0000002f; -pub const REISERFS_SUPER_MAGIC: ::c_long = 0x52654973; -pub const SMB_SUPER_MAGIC: ::c_long = 0x0000517b; -pub const TMPFS_MAGIC: ::c_long = 0x01021994; -pub const USBDEVICE_SUPER_MAGIC: ::c_long = 0x00009fa2; - pub const VEOF: usize = 16; pub const VEOL: usize = 17; pub const VEOL2: usize = 6; @@ -295,100 +189,21 @@ pub const VMIN: usize = 4; pub const IEXTEN: ::tcflag_t = 0x00000100; pub const TOSTOP: ::tcflag_t = 0x00008000; pub const FLUSHO: ::tcflag_t = 0x00002000; -pub const IUTF8: ::tcflag_t = 0x00004000; pub const TCSANOW: ::c_int = 0x540e; pub const TCSADRAIN: ::c_int = 0x540f; pub const TCSAFLUSH: ::c_int = 0x5410; pub const CPU_SETSIZE: ::c_int = 0x400; -pub const PTRACE_TRACEME: ::c_uint = 0; -pub const PTRACE_PEEKTEXT: ::c_uint = 1; -pub const PTRACE_PEEKDATA: ::c_uint = 2; -pub const PTRACE_PEEKUSER: ::c_uint = 3; -pub const PTRACE_POKETEXT: ::c_uint = 4; -pub const PTRACE_POKEDATA: ::c_uint = 5; -pub const PTRACE_POKEUSER: ::c_uint = 6; -pub const PTRACE_CONT: ::c_uint = 7; -pub const PTRACE_KILL: ::c_uint = 8; -pub const PTRACE_SINGLESTEP: ::c_uint = 9; -pub const PTRACE_ATTACH: ::c_uint = 16; -pub const PTRACE_DETACH: ::c_uint = 17; -pub const PTRACE_SYSCALL: ::c_uint = 24; -pub const PTRACE_SETOPTIONS: ::c_uint = 0x4200; -pub const PTRACE_GETEVENTMSG: ::c_uint = 0x4201; -pub const PTRACE_GETSIGINFO: ::c_uint = 0x4202; -pub const PTRACE_SETSIGINFO: ::c_uint = 0x4203; -pub const PTRACE_GETFPREGS: ::c_uint = 14; -pub const PTRACE_SETFPREGS: ::c_uint = 15; -pub const PTRACE_GETFPXREGS: ::c_uint = 18; -pub const PTRACE_SETFPXREGS: ::c_uint = 19; -pub const PTRACE_GETREGS: ::c_uint = 12; -pub const PTRACE_SETREGS: ::c_uint = 13; - pub const EFD_NONBLOCK: ::c_int = 0x80; pub const F_GETLK: ::c_int = 14; -pub const F_GETOWN: ::c_int = 23; -pub const F_SETOWN: ::c_int = 24; pub const F_SETLK: ::c_int = 6; pub const F_SETLKW: ::c_int = 7; pub const SFD_NONBLOCK: ::c_int = 0x80; -pub const TCGETS: ::c_ulong = 0x540d; -pub const TCSETS: ::c_ulong = 0x540e; -pub const TCSETSW: ::c_ulong = 0x540f; -pub const TCSETSF: ::c_ulong = 0x5410; -pub const TCGETA: ::c_ulong = 0x5401; -pub const TCSETA: ::c_ulong = 0x5402; -pub const TCSETAW: ::c_ulong = 0x5403; -pub const TCSETAF: ::c_ulong = 0x5404; -pub const TCSBRK: ::c_ulong = 0x5405; -pub const TCXONC: ::c_ulong = 0x5406; -pub const TCFLSH: ::c_ulong = 0x5407; -pub const TIOCGSOFTCAR: ::c_ulong = 0x5481; -pub const TIOCSSOFTCAR: ::c_ulong = 0x5482; -pub const TIOCINQ: ::c_ulong = 0x467f; -pub const TIOCLINUX: ::c_ulong = 0x5483; -pub const TIOCGSERIAL: ::c_ulong = 0x5484; -pub const TIOCEXCL: ::c_ulong = 0x740d; -pub const TIOCNXCL: ::c_ulong = 0x740e; -pub const TIOCSCTTY: ::c_ulong = 0x5480; -pub const TIOCGPGRP: ::c_ulong = 0x40047477; -pub const TIOCSPGRP: ::c_ulong = 0x80047476; -pub const TIOCOUTQ: ::c_ulong = 0x7472; -pub const TIOCSTI: ::c_ulong = 0x5472; -pub const TIOCGWINSZ: ::c_ulong = 0x40087468; -pub const TIOCSWINSZ: ::c_ulong = 0x80087467; -pub const TIOCMGET: ::c_ulong = 0x741d; -pub const TIOCMBIS: ::c_ulong = 0x741b; -pub const TIOCMBIC: ::c_ulong = 0x741c; -pub const TIOCMSET: ::c_ulong = 0x741a; -pub const FIONREAD: ::c_ulong = 0x467f; -pub const TIOCCONS: ::c_ulong = 0x80047478; - -pub const RTLD_DEEPBIND: ::c_int = 0x10; pub const RTLD_GLOBAL: ::c_int = 0x4; -pub const RTLD_NOLOAD: ::c_int = 0x8; - -pub const LINUX_REBOOT_MAGIC1: ::c_int = 0xfee1dead; -pub const LINUX_REBOOT_MAGIC2: ::c_int = 672274793; -pub const LINUX_REBOOT_MAGIC2A: ::c_int = 85072278; -pub const LINUX_REBOOT_MAGIC2B: ::c_int = 369367448; -pub const LINUX_REBOOT_MAGIC2C: ::c_int = 537993216; - -pub const LINUX_REBOOT_CMD_RESTART: ::c_int = 0x01234567; -pub const LINUX_REBOOT_CMD_HALT: ::c_int = 0xCDEF0123; -pub const LINUX_REBOOT_CMD_CAD_ON: ::c_int = 0x89ABCDEF; -pub const LINUX_REBOOT_CMD_CAD_OFF: ::c_int = 0x00000000; -pub const LINUX_REBOOT_CMD_POWER_OFF: ::c_int = 0x4321FEDC; -pub const LINUX_REBOOT_CMD_RESTART2: ::c_int = 0xA1B2C3D4; -pub const LINUX_REBOOT_CMD_SW_SUSPEND: ::c_int = 0xD000FCE2; -pub const LINUX_REBOOT_CMD_KEXEC: ::c_int = 0x45584543; - -pub const MCL_CURRENT: ::c_int = 0x0001; -pub const MCL_FUTURE: ::c_int = 0x0002; pub const SIGSTKSZ: ::size_t = 8192; pub const CBAUD: ::tcflag_t = 0o0010017; @@ -448,8 +263,6 @@ pub const B4800: ::speed_t = 0o000014; pub const B9600: ::speed_t = 0o000015; pub const B19200: ::speed_t = 0o000016; pub const B38400: ::speed_t = 0o000017; -pub const EXTA: ::speed_t = B19200; -pub const EXTB: ::speed_t = B38400; pub const B57600: ::speed_t = 0o010001; pub const B115200: ::speed_t = 0o010002; pub const B230400: ::speed_t = 0o010003; diff --git a/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/mod.rs new file mode 100644 index 000000000..4a01e0cd8 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/mod.rs @@ -0,0 +1,392 @@ +pub type shmatt_t = ::c_ulong; +pub type msgqnum_t = ::c_ulong; +pub type msglen_t = ::c_ulong; +pub type regoff_t = ::c_int; +pub type rlim_t = ::c_ulong; +pub type __rlimit_resource_t = ::c_ulong; +pub type __priority_which_t = ::c_uint; + +cfg_if! { + if #[cfg(doc)] { + // Used in `linux::arch` to define ioctl constants. + pub(crate) type Ioctl = ::c_ulong; + } else { + #[doc(hidden)] + pub type Ioctl = ::c_ulong; + } +} + +s! { + pub struct statvfs { // Different than GNU! + pub f_bsize: ::c_ulong, + pub f_frsize: ::c_ulong, + pub f_blocks: ::fsblkcnt_t, + pub f_bfree: ::fsblkcnt_t, + pub f_bavail: ::fsblkcnt_t, + pub f_files: ::fsfilcnt_t, + pub f_ffree: ::fsfilcnt_t, + pub f_favail: ::fsfilcnt_t, + #[cfg(target_endian = "little")] + pub f_fsid: ::c_ulong, + #[cfg(target_pointer_width = "32")] + __f_unused: ::c_int, + #[cfg(target_endian = "big")] + pub f_fsid: ::c_ulong, + pub f_flag: ::c_ulong, + pub f_namemax: ::c_ulong, + __f_spare: [::c_int; 6], + } + + pub struct regex_t { + __buffer: *mut ::c_void, + __allocated: ::size_t, + __used: ::size_t, + __syntax: ::c_ulong, + __fastmap: *mut ::c_char, + __translate: *mut ::c_char, + __re_nsub: ::size_t, + __bitfield: u8, + } + + pub struct rtentry { + pub rt_pad1: ::c_ulong, + pub rt_dst: ::sockaddr, + pub rt_gateway: ::sockaddr, + pub rt_genmask: ::sockaddr, + pub rt_flags: ::c_ushort, + pub rt_pad2: ::c_short, + pub rt_pad3: ::c_ulong, + pub rt_tos: ::c_uchar, + pub rt_class: ::c_uchar, + #[cfg(target_pointer_width = "64")] + pub rt_pad4: [::c_short; 3usize], + #[cfg(not(target_pointer_width = "64"))] + pub rt_pad4: ::c_short, + pub rt_metric: ::c_short, + pub rt_dev: *mut ::c_char, + pub rt_mtu: ::c_ulong, + pub rt_window: ::c_ulong, + pub rt_irtt: ::c_ushort, + } + + pub struct __exit_status { + pub e_termination: ::c_short, + pub e_exit: ::c_short, + } + + pub struct ptrace_peeksiginfo_args { + pub off: ::__u64, + pub flags: ::__u32, + pub nr: ::__s32, + } +} + +impl siginfo_t { + pub unsafe fn si_addr(&self) -> *mut ::c_void { + #[repr(C)] + struct siginfo_sigfault { + _si_signo: ::c_int, + _si_errno: ::c_int, + _si_code: ::c_int, + si_addr: *mut ::c_void, + } + (*(self as *const siginfo_t as *const siginfo_sigfault)).si_addr + } + + pub unsafe fn si_value(&self) -> ::sigval { + #[repr(C)] + struct siginfo_si_value { + _si_signo: ::c_int, + _si_errno: ::c_int, + _si_code: ::c_int, + _si_timerid: ::c_int, + _si_overrun: ::c_int, + si_value: ::sigval, + } + (*(self as *const siginfo_t as *const siginfo_si_value)).si_value + } +} + +pub const MCL_CURRENT: ::c_int = 0x0001; +pub const MCL_FUTURE: ::c_int = 0x0002; + +pub const SIGEV_THREAD_ID: ::c_int = 4; + +pub const AF_VSOCK: ::c_int = 40; + +// Most `*_SUPER_MAGIC` constants are defined at the `linux_like` level; the +// following are only available on newer Linux versions than the versions +// currently used in CI in some configurations, so we define them here. +pub const BINDERFS_SUPER_MAGIC: ::c_long = 0x6c6f6f70; +pub const XFS_SUPER_MAGIC: ::c_long = 0x58465342; + +pub const PTRACE_TRACEME: ::c_int = 0; +pub const PTRACE_PEEKTEXT: ::c_int = 1; +pub const PTRACE_PEEKDATA: ::c_int = 2; +pub const PTRACE_PEEKUSER: ::c_int = 3; +pub const PTRACE_POKETEXT: ::c_int = 4; +pub const PTRACE_POKEDATA: ::c_int = 5; +pub const PTRACE_POKEUSER: ::c_int = 6; +pub const PTRACE_CONT: ::c_int = 7; +pub const PTRACE_KILL: ::c_int = 8; +pub const PTRACE_SINGLESTEP: ::c_int = 9; +pub const PTRACE_GETREGS: ::c_int = 12; +pub const PTRACE_SETREGS: ::c_int = 13; +pub const PTRACE_GETFPREGS: ::c_int = 14; +pub const PTRACE_SETFPREGS: ::c_int = 15; +pub const PTRACE_ATTACH: ::c_int = 16; +pub const PTRACE_DETACH: ::c_int = 17; +pub const PTRACE_GETFPXREGS: ::c_int = 18; +pub const PTRACE_SETFPXREGS: ::c_int = 19; +pub const PTRACE_SYSCALL: ::c_int = 24; +pub const PTRACE_SETOPTIONS: ::c_int = 0x4200; +pub const PTRACE_GETEVENTMSG: ::c_int = 0x4201; +pub const PTRACE_GETSIGINFO: ::c_int = 0x4202; +pub const PTRACE_SETSIGINFO: ::c_int = 0x4203; +pub const PTRACE_GETREGSET: ::c_int = 0x4204; +pub const PTRACE_SETREGSET: ::c_int = 0x4205; +pub const PTRACE_SEIZE: ::c_int = 0x4206; +pub const PTRACE_INTERRUPT: ::c_int = 0x4207; +pub const PTRACE_LISTEN: ::c_int = 0x4208; + +pub const POSIX_FADV_DONTNEED: ::c_int = 4; +pub const POSIX_FADV_NOREUSE: ::c_int = 5; + +// These are different than GNU! +pub const LC_CTYPE: ::c_int = 0; +pub const LC_NUMERIC: ::c_int = 1; +pub const LC_TIME: ::c_int = 3; +pub const LC_COLLATE: ::c_int = 4; +pub const LC_MONETARY: ::c_int = 2; +pub const LC_MESSAGES: ::c_int = 5; +pub const LC_ALL: ::c_int = 6; +// end different section + +// MS_ flags for mount(2) +pub const MS_RMT_MASK: ::c_ulong = ::MS_RDONLY | ::MS_SYNCHRONOUS | ::MS_MANDLOCK | ::MS_I_VERSION; + +pub const ENOTSUP: ::c_int = EOPNOTSUPP; + +pub const IPV6_JOIN_GROUP: ::c_int = 20; +pub const IPV6_LEAVE_GROUP: ::c_int = 21; + +// These are different from GNU +pub const ABDAY_1: ::nl_item = 0x300; +pub const ABDAY_2: ::nl_item = 0x301; +pub const ABDAY_3: ::nl_item = 0x302; +pub const ABDAY_4: ::nl_item = 0x303; +pub const ABDAY_5: ::nl_item = 0x304; +pub const ABDAY_6: ::nl_item = 0x305; +pub const ABDAY_7: ::nl_item = 0x306; +pub const DAY_1: ::nl_item = 0x307; +pub const DAY_2: ::nl_item = 0x308; +pub const DAY_3: ::nl_item = 0x309; +pub const DAY_4: ::nl_item = 0x30A; +pub const DAY_5: ::nl_item = 0x30B; +pub const DAY_6: ::nl_item = 0x30C; +pub const DAY_7: ::nl_item = 0x30D; +pub const ABMON_1: ::nl_item = 0x30E; +pub const ABMON_2: ::nl_item = 0x30F; +pub const ABMON_3: ::nl_item = 0x310; +pub const ABMON_4: ::nl_item = 0x311; +pub const ABMON_5: ::nl_item = 0x312; +pub const ABMON_6: ::nl_item = 0x313; +pub const ABMON_7: ::nl_item = 0x314; +pub const ABMON_8: ::nl_item = 0x315; +pub const ABMON_9: ::nl_item = 0x316; +pub const ABMON_10: ::nl_item = 0x317; +pub const ABMON_11: ::nl_item = 0x318; +pub const ABMON_12: ::nl_item = 0x319; +pub const MON_1: ::nl_item = 0x31A; +pub const MON_2: ::nl_item = 0x31B; +pub const MON_3: ::nl_item = 0x31C; +pub const MON_4: ::nl_item = 0x31D; +pub const MON_5: ::nl_item = 0x31E; +pub const MON_6: ::nl_item = 0x31F; +pub const MON_7: ::nl_item = 0x320; +pub const MON_8: ::nl_item = 0x321; +pub const MON_9: ::nl_item = 0x322; +pub const MON_10: ::nl_item = 0x323; +pub const MON_11: ::nl_item = 0x324; +pub const MON_12: ::nl_item = 0x325; +pub const AM_STR: ::nl_item = 0x326; +pub const PM_STR: ::nl_item = 0x327; +pub const D_T_FMT: ::nl_item = 0x328; +pub const D_FMT: ::nl_item = 0x329; +pub const T_FMT: ::nl_item = 0x32A; +pub const T_FMT_AMPM: ::nl_item = 0x32B; +pub const ERA: ::nl_item = 0x32C; +pub const ERA_D_FMT: ::nl_item = 0x32E; +pub const ALT_DIGITS: ::nl_item = 0x32F; +pub const ERA_D_T_FMT: ::nl_item = 0x330; +pub const ERA_T_FMT: ::nl_item = 0x331; +pub const CODESET: ::nl_item = 10; +pub const CRNCYSTR: ::nl_item = 0x215; +pub const RADIXCHAR: ::nl_item = 0x100; +pub const THOUSEP: ::nl_item = 0x101; +pub const NOEXPR: ::nl_item = 0x501; +pub const YESSTR: ::nl_item = 0x502; +pub const NOSTR: ::nl_item = 0x503; + +// Different than Gnu. +pub const FILENAME_MAX: ::c_uint = 4095; + +pub const PRIO_PROCESS: ::c_int = 0; +pub const PRIO_PGRP: ::c_int = 1; +pub const PRIO_USER: ::c_int = 2; + +pub const SOMAXCONN: ::c_int = 128; + +pub const ST_RELATIME: ::c_ulong = 4096; + +pub const AF_NFC: ::c_int = PF_NFC; +pub const BUFSIZ: ::c_int = 4096; +pub const EDEADLOCK: ::c_int = EDEADLK; +pub const EXTA: ::c_uint = B19200; +pub const EXTB: ::c_uint = B38400; +pub const EXTPROC: ::tcflag_t = 0200000; +pub const FAN_MARK_FILESYSTEM: ::c_int = 0x00000100; +pub const FAN_MARK_INODE: ::c_int = 0x00000000; +pub const FAN_MARK_MOUNT: ::c_int = 0x10; +pub const FOPEN_MAX: ::c_int = 16; +pub const F_GETOWN: ::c_int = 9; +pub const F_OFD_GETLK: ::c_int = 36; +pub const F_OFD_SETLK: ::c_int = 37; +pub const F_OFD_SETLKW: ::c_int = 38; +pub const F_RDLCK: ::c_int = 0; +pub const F_SETOWN: ::c_int = 8; +pub const F_UNLCK: ::c_int = 2; +pub const F_WRLCK: ::c_int = 1; +pub const IPV6_MULTICAST_ALL: ::c_int = 29; +pub const IPV6_ROUTER_ALERT_ISOLATE: ::c_int = 30; +pub const MAP_HUGE_SHIFT: ::c_int = 26; +pub const MAP_HUGE_MASK: ::c_int = 0x3f; +pub const MAP_HUGE_64KB: ::c_int = 16 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_512KB: ::c_int = 19 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_1MB: ::c_int = 20 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_2MB: ::c_int = 21 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_8MB: ::c_int = 23 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_16MB: ::c_int = 24 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_32MB: ::c_int = 25 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_256MB: ::c_int = 28 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_512MB: ::c_int = 29 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_1GB: ::c_int = 30 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_2GB: ::c_int = 31 << MAP_HUGE_SHIFT; +pub const MAP_HUGE_16GB: ::c_int = 34 << MAP_HUGE_SHIFT; +pub const MINSIGSTKSZ: ::c_int = 2048; +pub const MSG_COPY: ::c_int = 040000; +pub const NI_MAXHOST: ::socklen_t = 1025; +pub const O_TMPFILE: ::c_int = 0o20000000 | O_DIRECTORY; +pub const PACKET_MR_UNICAST: ::c_int = 3; +pub const PF_NFC: ::c_int = 39; +pub const PF_VSOCK: ::c_int = 40; +pub const POSIX_MADV_DONTNEED: ::c_int = 4; +pub const PTRACE_EVENT_STOP: ::c_int = 128; +pub const PTRACE_PEEKSIGINFO: ::c_int = 0x4209; +pub const RTLD_NOLOAD: ::c_int = 0x00004; +pub const RUSAGE_THREAD: ::c_int = 1; +pub const SHM_EXEC: ::c_int = 0100000; +pub const SIGPOLL: ::c_int = SIGIO; +pub const SOCK_DCCP: ::c_int = 6; +pub const SOCK_PACKET: ::c_int = 10; +pub const TCP_COOKIE_TRANSACTIONS: ::c_int = 15; +pub const UDP_GRO: ::c_int = 104; +pub const UDP_SEGMENT: ::c_int = 103; +pub const YESEXPR: ::c_int = ((5) << 8) | (0); + +extern "C" { + pub fn ioctl(fd: ::c_int, request: ::c_ulong, ...) -> ::c_int; + pub fn gettimeofday(tp: *mut ::timeval, tz: *mut ::timezone) -> ::c_int; + + pub fn pthread_rwlockattr_getkind_np( + attr: *const ::pthread_rwlockattr_t, + val: *mut ::c_int, + ) -> ::c_int; + pub fn pthread_rwlockattr_setkind_np( + attr: *mut ::pthread_rwlockattr_t, + val: ::c_int, + ) -> ::c_int; + + pub fn ptrace(request: ::c_uint, ...) -> ::c_long; + + pub fn sendmmsg( + sockfd: ::c_int, + msgvec: *mut ::mmsghdr, + vlen: ::c_uint, + flags: ::c_int, + ) -> ::c_int; + pub fn recvmmsg( + sockfd: ::c_int, + msgvec: *mut ::mmsghdr, + vlen: ::c_uint, + flags: ::c_int, + timeout: *mut ::timespec, + ) -> ::c_int; + + pub fn openpty( + amaster: *mut ::c_int, + aslave: *mut ::c_int, + name: *mut ::c_char, + termp: *mut termios, + winp: *mut ::winsize, + ) -> ::c_int; + pub fn forkpty( + amaster: *mut ::c_int, + name: *mut ::c_char, + termp: *mut termios, + winp: *mut ::winsize, + ) -> ::pid_t; + + pub fn getnameinfo( + sa: *const ::sockaddr, + salen: ::socklen_t, + host: *mut ::c_char, + hostlen: ::socklen_t, + serv: *mut ::c_char, + sevlen: ::socklen_t, + flags: ::c_int, + ) -> ::c_int; + + pub fn pwritev( + fd: ::c_int, + iov: *const ::iovec, + iovcnt: ::c_int, + offset: ::off64_t, + ) -> ::ssize_t; + pub fn preadv( + fd: ::c_int, + iov: *const ::iovec, + iovcnt: ::c_int, + offset: ::off64_t, + ) -> ::ssize_t; + + pub fn sethostid(hostid: ::c_long) -> ::c_int; + pub fn fanotify_mark( + fd: ::c_int, + flags: ::c_uint, + mask: u64, + dirfd: ::c_int, + path: *const ::c_char, + ) -> ::c_int; + pub fn getrlimit64(resource: ::__rlimit_resource_t, rlim: *mut ::rlimit64) -> ::c_int; + pub fn setrlimit64(resource: ::__rlimit_resource_t, rlim: *const ::rlimit64) -> ::c_int; + pub fn getrlimit(resource: ::__rlimit_resource_t, rlim: *mut ::rlimit) -> ::c_int; + pub fn setrlimit(resource: ::__rlimit_resource_t, rlim: *const ::rlimit) -> ::c_int; + pub fn getpriority(which: ::__priority_which_t, who: ::id_t) -> ::c_int; + pub fn setpriority(which: ::__priority_which_t, who: ::id_t, prio: ::c_int) -> ::c_int; +} + +cfg_if! { + if #[cfg(any(target_arch = "mips", target_arch = "mips64"))] { + mod mips; + pub use self::mips::*; + } else if #[cfg(target_arch = "x86_64")] { + mod x86_64; + pub use self::x86_64::*; + } else if #[cfg(target_arch = "arm")] { + mod arm; + pub use self::arm::*; + } else { + pub use unsupported_target; + } +} diff --git a/crux-mir/lib/libc/src/unix/uclibc/no_align.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/no_align.rs similarity index 100% rename from crux-mir/lib/libc/src/unix/uclibc/no_align.rs rename to crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/no_align.rs diff --git a/crux-mir/lib/libc/src/unix/uclibc/x86_64/l4re.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/x86_64/l4re.rs similarity index 91% rename from crux-mir/lib/libc/src/unix/uclibc/x86_64/l4re.rs rename to crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/x86_64/l4re.rs index 16ec0ef96..c7cbafa16 100644 --- a/crux-mir/lib/libc/src/unix/uclibc/x86_64/l4re.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/x86_64/l4re.rs @@ -46,3 +46,8 @@ pub struct pthread_attr_t { // L4Re requires a min stack size of 64k; that isn't defined in uClibc, but // somewhere in the core libraries. uClibc wants 16k, but that's not enough. pub const PTHREAD_STACK_MIN: usize = 65536; + +// Misc other constants required for building. +pub const SIGIO: ::c_int = 29; +pub const B19200: ::speed_t = 0o000016; +pub const B38400: ::speed_t = 0o000017; diff --git a/crux-mir/lib/libc/src/unix/uclibc/x86_64/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/x86_64/mod.rs similarity index 80% rename from crux-mir/lib/libc/src/unix/uclibc/x86_64/mod.rs rename to crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/x86_64/mod.rs index 26eca9e7e..43ac79296 100644 --- a/crux-mir/lib/libc/src/unix/uclibc/x86_64/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/x86_64/mod.rs @@ -11,14 +11,17 @@ pub type fsword_t = ::c_long; pub type ino_t = ::c_ulong; pub type nlink_t = ::c_uint; pub type off_t = ::c_long; -pub type rlim_t = c_ulong; -pub type rlim64_t = u64; // [uClibc docs] Note stat64 has the same shape as stat for x86-64. pub type stat64 = stat; pub type suseconds_t = ::c_long; pub type time_t = ::c_int; pub type wchar_t = ::c_int; +pub type fsblkcnt64_t = u64; +pub type fsfilcnt64_t = u64; +pub type __u64 = ::c_ulong; +pub type __s64 = ::c_long; + s! { pub struct ipc_perm { pub __key: ::key_t, @@ -143,7 +146,7 @@ s! { pub struct sigaction { pub sa_handler: ::sighandler_t, pub sa_flags: ::c_ulong, - pub sa_restorer: *mut ::c_void, + pub sa_restorer: ::Option, pub sa_mask: ::sigset_t, } @@ -167,6 +170,37 @@ s! { f_spare: [fsword_t; 5], } + pub struct statfs64 { + pub f_type: ::c_int, + pub f_bsize: ::c_int, + pub f_blocks: ::fsblkcnt64_t, + pub f_bfree: ::fsblkcnt64_t, + pub f_bavail: ::fsblkcnt64_t, + pub f_files: ::fsfilcnt64_t, + pub f_ffree: ::fsfilcnt64_t, + pub f_fsid: ::fsid_t, + pub f_namelen: ::c_int, + pub f_frsize: ::c_int, + pub f_flags: ::c_int, + pub f_spare: [::c_int; 4], + } + + pub struct statvfs64 { + pub f_bsize: ::c_ulong, + pub f_frsize: ::c_ulong, + pub f_blocks: u64, + pub f_bfree: u64, + pub f_bavail: u64, + pub f_files: u64, + pub f_ffree: u64, + pub f_favail: u64, + pub f_fsid: ::c_ulong, + __f_unused: ::c_int, + pub f_flag: ::c_ulong, + pub f_namemax: ::c_ulong, + __f_spare: [::c_int; 6], + } + pub struct msghdr { // FIXME pub msg_name: *mut ::c_void, pub msg_namelen: ::socklen_t, @@ -219,11 +253,6 @@ s! { __unused5: *mut ::c_void, } - pub struct rlimit64 { // FIXME - pub rlim_cur: rlim64_t, - pub rlim_max: rlim64_t, - } - pub struct cpu_set_t { // FIXME #[cfg(target_pointer_width = "32")] bits: [u32; 32], @@ -234,6 +263,21 @@ s! { pub struct fsid_t { // FIXME __val: [::c_int; 2], } + + // FIXME this is actually a union + pub struct sem_t { + #[cfg(target_pointer_width = "32")] + __size: [::c_char; 16], + #[cfg(target_pointer_width = "64")] + __size: [::c_char; 32], + __align: [::c_long; 0], + } + + pub struct cmsghdr { + pub cmsg_len: ::size_t, + pub cmsg_level: ::c_int, + pub cmsg_type: ::c_int, + } } s_no_extra_traits! { @@ -245,19 +289,16 @@ s_no_extra_traits! { pub d_type: u8, pub d_name: [::c_char; 256], } - #[allow(missing_debug_implementations)] - pub struct dirent64 { - pub d_ino: ::ino64_t, - pub d_off: ::off64_t, - pub d_reclen: u16, - pub d_type: u8, - pub d_name: [::c_char; 256], - } } // constants +pub const ENAMETOOLONG: ::c_int = 36; // File name too long +pub const ENOTEMPTY: ::c_int = 39; // Directory not empty +pub const ELOOP: ::c_int = 40; // Too many symbolic links encountered pub const EADDRINUSE: ::c_int = 98; // Address already in use pub const EADDRNOTAVAIL: ::c_int = 99; // Cannot assign requested address +pub const ENETDOWN: ::c_int = 100; // Network is down +pub const ENETUNREACH: ::c_int = 101; // Network is unreachable pub const ECONNABORTED: ::c_int = 103; // Software caused connection abort pub const ECONNREFUSED: ::c_int = 111; // Connection refused pub const ECONNRESET: ::c_int = 104; // Connection reset by peer @@ -265,30 +306,30 @@ pub const EDEADLK: ::c_int = 35; // Resource deadlock would occur pub const ENOSYS: ::c_int = 38; // Function not implemented pub const ENOTCONN: ::c_int = 107; // Transport endpoint is not connected pub const ETIMEDOUT: ::c_int = 110; // connection timed out +pub const ESTALE: ::c_int = 116; // Stale file handle +pub const EHOSTUNREACH: ::c_int = 113; // No route to host +pub const EDQUOT: ::c_int = 122; // Quota exceeded +pub const EOPNOTSUPP: ::c_int = 0x5f; +pub const ENODATA: ::c_int = 0x3d; pub const O_APPEND: ::c_int = 02000; pub const O_ACCMODE: ::c_int = 0003; pub const O_CLOEXEC: ::c_int = 0x80000; pub const O_CREAT: ::c_int = 0100; pub const O_DIRECTORY: ::c_int = 0200000; pub const O_EXCL: ::c_int = 0200; +pub const O_NOFOLLOW: ::c_int = 0x20000; pub const O_NONBLOCK: ::c_int = 04000; pub const O_TRUNC: ::c_int = 01000; pub const NCCS: usize = 32; pub const SIG_SETMASK: ::c_int = 2; // Set the set of blocked signals pub const __SIZEOF_PTHREAD_MUTEX_T: usize = 40; pub const __SIZEOF_PTHREAD_MUTEXATTR_T: usize = 4; -pub const SO_BROADCAST: ::c_int = 6; pub const SOCK_DGRAM: ::c_int = 2; // connectionless, unreliable datagrams pub const SOCK_STREAM: ::c_int = 1; // …/common/bits/socket_type.h -pub const SO_ERROR: ::c_int = 4; -pub const SOL_SOCKET: ::c_int = 1; -pub const SO_RCVTIMEO: ::c_int = 20; -pub const SO_REUSEADDR: ::c_int = 2; -pub const SO_SNDTIMEO: ::c_int = 21; -pub const RLIM_INFINITY: u64 = 0xffffffffffffffff; pub const __SIZEOF_PTHREAD_COND_T: usize = 48; pub const __SIZEOF_PTHREAD_CONDATTR_T: usize = 4; pub const __SIZEOF_PTHREAD_RWLOCK_T: usize = 56; +pub const __SIZEOF_PTHREAD_RWLOCKATTR_T: usize = 8; cfg_if! { if #[cfg(target_os = "l4re")] { @@ -299,14 +340,3 @@ cfg_if! { pub use other::*; } } - -cfg_if! { - if #[cfg(libc_align)] { - #[macro_use] - mod align; - } else { - #[macro_use] - mod no_align; - } -} -expand_align!(); diff --git a/crux-mir/lib/libc/src/unix/uclibc/x86_64/other.rs b/crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/x86_64/other.rs similarity index 100% rename from crux-mir/lib/libc/src/unix/uclibc/x86_64/other.rs rename to crux-mir/lib/libc/src/unix/linux_like/linux/uclibc/x86_64/other.rs diff --git a/crux-mir/lib/libc/src/unix/linux_like/mod.rs b/crux-mir/lib/libc/src/unix/linux_like/mod.rs index a867ed4b0..8e738d87b 100644 --- a/crux-mir/lib/libc/src/unix/linux_like/mod.rs +++ b/crux-mir/lib/libc/src/unix/linux_like/mod.rs @@ -2,6 +2,7 @@ pub type sa_family_t = u16; pub type speed_t = ::c_uint; pub type tcflag_t = ::c_uint; pub type clockid_t = ::c_int; +pub type timer_t = *mut ::c_void; pub type key_t = ::c_int; pub type id_t = ::c_uint; @@ -24,6 +25,12 @@ s! { pub imr_interface: in_addr, } + pub struct ip_mreqn { + pub imr_multiaddr: in_addr, + pub imr_address: in_addr, + pub imr_ifindex: ::c_int, + } + pub struct ip_mreq_source { pub imr_multiaddr: in_addr, pub imr_interface: in_addr, @@ -50,6 +57,8 @@ s! { pub sin6_scope_id: u32, } + // The order of the `ai_addr` field in this struct is crucial + // for converting between the Rust and C types. pub struct addrinfo { pub ai_flags: ::c_int, pub ai_family: ::c_int, @@ -199,16 +208,6 @@ s! { pub msg_hdr: ::msghdr, pub msg_len: ::c_uint, } - - pub struct sock_extended_err { - pub ee_errno: u32, - pub ee_origin: u8, - pub ee_type: u8, - pub ee_code: u8, - pub ee_pad: u8, - pub ee_info: u32, - pub ee_data: u32 - } } s_no_extra_traits! { @@ -232,11 +231,11 @@ s_no_extra_traits! { pub struct sockaddr_storage { pub ss_family: sa_family_t, - __ss_align: ::size_t, #[cfg(target_pointer_width = "32")] - __ss_pad2: [u8; 128 - 2 * 4], + __ss_pad2: [u8; 128 - 2 - 4], #[cfg(target_pointer_width = "64")] - __ss_pad2: [u8; 128 - 2 * 8], + __ss_pad2: [u8; 128 - 2 - 8], + __ss_align: ::size_t, } pub struct utsname { @@ -554,13 +553,21 @@ pub const PROT_READ: ::c_int = 1; pub const PROT_WRITE: ::c_int = 2; pub const PROT_EXEC: ::c_int = 4; -pub const LC_CTYPE: ::c_int = 0; -pub const LC_NUMERIC: ::c_int = 1; -pub const LC_TIME: ::c_int = 2; -pub const LC_COLLATE: ::c_int = 3; -pub const LC_MONETARY: ::c_int = 4; -pub const LC_MESSAGES: ::c_int = 5; -pub const LC_ALL: ::c_int = 6; +pub const XATTR_CREATE: ::c_int = 0x1; +pub const XATTR_REPLACE: ::c_int = 0x2; + +cfg_if! { + if #[cfg(not(target_env = "uclibc"))] { + pub const LC_CTYPE: ::c_int = 0; + pub const LC_NUMERIC: ::c_int = 1; + pub const LC_TIME: ::c_int = 2; + pub const LC_COLLATE: ::c_int = 3; + pub const LC_MONETARY: ::c_int = 4; + pub const LC_MESSAGES: ::c_int = 5; + pub const LC_ALL: ::c_int = 6; + } +} + pub const LC_CTYPE_MASK: ::c_int = 1 << LC_CTYPE; pub const LC_NUMERIC_MASK: ::c_int = 1 << LC_NUMERIC; pub const LC_TIME_MASK: ::c_int = 1 << LC_TIME; @@ -605,46 +612,11 @@ pub const MS_RELATIME: ::c_ulong = 0x200000; pub const MS_KERNMOUNT: ::c_ulong = 0x400000; pub const MS_I_VERSION: ::c_ulong = 0x800000; pub const MS_STRICTATIME: ::c_ulong = 0x1000000; +pub const MS_LAZYTIME: ::c_ulong = 0x2000000; pub const MS_ACTIVE: ::c_ulong = 0x40000000; pub const MS_MGC_VAL: ::c_ulong = 0xc0ed0000; pub const MS_MGC_MSK: ::c_ulong = 0xffff0000; -pub const EPERM: ::c_int = 1; -pub const ENOENT: ::c_int = 2; -pub const ESRCH: ::c_int = 3; -pub const EINTR: ::c_int = 4; -pub const EIO: ::c_int = 5; -pub const ENXIO: ::c_int = 6; -pub const E2BIG: ::c_int = 7; -pub const ENOEXEC: ::c_int = 8; -pub const EBADF: ::c_int = 9; -pub const ECHILD: ::c_int = 10; -pub const EAGAIN: ::c_int = 11; -pub const ENOMEM: ::c_int = 12; -pub const EACCES: ::c_int = 13; -pub const EFAULT: ::c_int = 14; -pub const ENOTBLK: ::c_int = 15; -pub const EBUSY: ::c_int = 16; -pub const EEXIST: ::c_int = 17; -pub const EXDEV: ::c_int = 18; -pub const ENODEV: ::c_int = 19; -pub const ENOTDIR: ::c_int = 20; -pub const EISDIR: ::c_int = 21; -pub const EINVAL: ::c_int = 22; -pub const ENFILE: ::c_int = 23; -pub const EMFILE: ::c_int = 24; -pub const ENOTTY: ::c_int = 25; -pub const ETXTBSY: ::c_int = 26; -pub const EFBIG: ::c_int = 27; -pub const ENOSPC: ::c_int = 28; -pub const ESPIPE: ::c_int = 29; -pub const EROFS: ::c_int = 30; -pub const EMLINK: ::c_int = 31; -pub const EPIPE: ::c_int = 32; -pub const EDOM: ::c_int = 33; -pub const ERANGE: ::c_int = 34; -pub const EWOULDBLOCK: ::c_int = EAGAIN; - pub const SCM_RIGHTS: ::c_int = 0x01; pub const SCM_CREDENTIALS: ::c_int = 0x02; @@ -787,8 +759,6 @@ pub const PF_IEEE802154: ::c_int = AF_IEEE802154; pub const PF_CAIF: ::c_int = AF_CAIF; pub const PF_ALG: ::c_int = AF_ALG; -pub const SOMAXCONN: ::c_int = 128; - pub const MSG_OOB: ::c_int = 1; pub const MSG_PEEK: ::c_int = 2; pub const MSG_DONTROUTE: ::c_int = 4; @@ -812,21 +782,118 @@ pub const SCM_TIMESTAMP: ::c_int = SO_TIMESTAMP; pub const SOCK_RAW: ::c_int = 3; pub const SOCK_RDM: ::c_int = 4; -pub const IP_MULTICAST_IF: ::c_int = 32; -pub const IP_MULTICAST_TTL: ::c_int = 33; -pub const IP_MULTICAST_LOOP: ::c_int = 34; pub const IP_TOS: ::c_int = 1; pub const IP_TTL: ::c_int = 2; pub const IP_HDRINCL: ::c_int = 3; +pub const IP_OPTIONS: ::c_int = 4; +pub const IP_ROUTER_ALERT: ::c_int = 5; +pub const IP_RECVOPTS: ::c_int = 6; +pub const IP_RETOPTS: ::c_int = 7; pub const IP_PKTINFO: ::c_int = 8; +pub const IP_PKTOPTIONS: ::c_int = 9; pub const IP_MTU_DISCOVER: ::c_int = 10; -pub const IP_RECVTOS: ::c_int = 13; pub const IP_RECVERR: ::c_int = 11; +pub const IP_RECVTTL: ::c_int = 12; +pub const IP_RECVTOS: ::c_int = 13; +pub const IP_MTU: ::c_int = 14; +pub const IP_FREEBIND: ::c_int = 15; +pub const IP_IPSEC_POLICY: ::c_int = 16; +pub const IP_XFRM_POLICY: ::c_int = 17; +pub const IP_PASSSEC: ::c_int = 18; +pub const IP_TRANSPARENT: ::c_int = 19; +pub const IP_ORIGDSTADDR: ::c_int = 20; +pub const IP_RECVORIGDSTADDR: ::c_int = IP_ORIGDSTADDR; +pub const IP_MINTTL: ::c_int = 21; +pub const IP_NODEFRAG: ::c_int = 22; +pub const IP_CHECKSUM: ::c_int = 23; +pub const IP_BIND_ADDRESS_NO_PORT: ::c_int = 24; +pub const IP_MULTICAST_IF: ::c_int = 32; +pub const IP_MULTICAST_TTL: ::c_int = 33; +pub const IP_MULTICAST_LOOP: ::c_int = 34; pub const IP_ADD_MEMBERSHIP: ::c_int = 35; pub const IP_DROP_MEMBERSHIP: ::c_int = 36; +pub const IP_UNBLOCK_SOURCE: ::c_int = 37; +pub const IP_BLOCK_SOURCE: ::c_int = 38; pub const IP_ADD_SOURCE_MEMBERSHIP: ::c_int = 39; pub const IP_DROP_SOURCE_MEMBERSHIP: ::c_int = 40; -pub const IP_TRANSPARENT: ::c_int = 19; +pub const IP_MSFILTER: ::c_int = 41; +pub const IP_MULTICAST_ALL: ::c_int = 49; +pub const IP_UNICAST_IF: ::c_int = 50; + +pub const IP_DEFAULT_MULTICAST_TTL: ::c_int = 1; +pub const IP_DEFAULT_MULTICAST_LOOP: ::c_int = 1; + +pub const IP_PMTUDISC_DONT: ::c_int = 0; +pub const IP_PMTUDISC_WANT: ::c_int = 1; +pub const IP_PMTUDISC_DO: ::c_int = 2; +pub const IP_PMTUDISC_PROBE: ::c_int = 3; +pub const IP_PMTUDISC_INTERFACE: ::c_int = 4; +pub const IP_PMTUDISC_OMIT: ::c_int = 5; + +// IPPROTO_IP defined in src/unix/mod.rs +/// Hop-by-hop option header +pub const IPPROTO_HOPOPTS: ::c_int = 0; +// IPPROTO_ICMP defined in src/unix/mod.rs +/// group mgmt protocol +pub const IPPROTO_IGMP: ::c_int = 2; +/// for compatibility +pub const IPPROTO_IPIP: ::c_int = 4; +// IPPROTO_TCP defined in src/unix/mod.rs +/// exterior gateway protocol +pub const IPPROTO_EGP: ::c_int = 8; +/// pup +pub const IPPROTO_PUP: ::c_int = 12; +// IPPROTO_UDP defined in src/unix/mod.rs +/// xns idp +pub const IPPROTO_IDP: ::c_int = 22; +/// tp-4 w/ class negotiation +pub const IPPROTO_TP: ::c_int = 29; +/// DCCP +pub const IPPROTO_DCCP: ::c_int = 33; +// IPPROTO_IPV6 defined in src/unix/mod.rs +/// IP6 routing header +pub const IPPROTO_ROUTING: ::c_int = 43; +/// IP6 fragmentation header +pub const IPPROTO_FRAGMENT: ::c_int = 44; +/// resource reservation +pub const IPPROTO_RSVP: ::c_int = 46; +/// General Routing Encap. +pub const IPPROTO_GRE: ::c_int = 47; +/// IP6 Encap Sec. Payload +pub const IPPROTO_ESP: ::c_int = 50; +/// IP6 Auth Header +pub const IPPROTO_AH: ::c_int = 51; +// IPPROTO_ICMPV6 defined in src/unix/mod.rs +/// IP6 no next header +pub const IPPROTO_NONE: ::c_int = 59; +/// IP6 destination option +pub const IPPROTO_DSTOPTS: ::c_int = 60; +pub const IPPROTO_MTP: ::c_int = 92; +/// encapsulation header +pub const IPPROTO_ENCAP: ::c_int = 98; +/// Protocol indep. multicast +pub const IPPROTO_PIM: ::c_int = 103; +/// IP Payload Comp. Protocol +pub const IPPROTO_COMP: ::c_int = 108; +/// SCTP +pub const IPPROTO_SCTP: ::c_int = 132; +pub const IPPROTO_MH: ::c_int = 135; +pub const IPPROTO_UDPLITE: ::c_int = 136; +/// raw IP packet +pub const IPPROTO_RAW: ::c_int = 255; +pub const IPPROTO_BEETPH: ::c_int = 94; +pub const IPPROTO_MPLS: ::c_int = 137; + +pub const MCAST_EXCLUDE: ::c_int = 0; +pub const MCAST_INCLUDE: ::c_int = 1; +pub const MCAST_JOIN_GROUP: ::c_int = 42; +pub const MCAST_BLOCK_SOURCE: ::c_int = 43; +pub const MCAST_UNBLOCK_SOURCE: ::c_int = 44; +pub const MCAST_LEAVE_GROUP: ::c_int = 45; +pub const MCAST_JOIN_SOURCE_GROUP: ::c_int = 46; +pub const MCAST_LEAVE_SOURCE_GROUP: ::c_int = 47; +pub const MCAST_MSFILTER: ::c_int = 48; + pub const IPV6_ADDRFORM: ::c_int = 1; pub const IPV6_2292PKTINFO: ::c_int = 2; pub const IPV6_2292HOPOPTS: ::c_int = 3; @@ -836,7 +903,7 @@ pub const IPV6_2292PKTOPTIONS: ::c_int = 6; pub const IPV6_CHECKSUM: ::c_int = 7; pub const IPV6_2292HOPLIMIT: ::c_int = 8; pub const IPV6_NEXTHOP: ::c_int = 9; -pub const IPV6_FLOWINFO: ::c_int = 11; +pub const IPV6_AUTHHDR: ::c_int = 10; pub const IPV6_UNICAST_HOPS: ::c_int = 16; pub const IPV6_MULTICAST_IF: ::c_int = 17; pub const IPV6_MULTICAST_HOPS: ::c_int = 18; @@ -850,15 +917,46 @@ pub const IPV6_RECVERR: ::c_int = 25; pub const IPV6_V6ONLY: ::c_int = 26; pub const IPV6_JOIN_ANYCAST: ::c_int = 27; pub const IPV6_LEAVE_ANYCAST: ::c_int = 28; +pub const IPV6_IPSEC_POLICY: ::c_int = 34; +pub const IPV6_XFRM_POLICY: ::c_int = 35; +pub const IPV6_HDRINCL: ::c_int = 36; pub const IPV6_RECVPKTINFO: ::c_int = 49; pub const IPV6_PKTINFO: ::c_int = 50; +pub const IPV6_RECVHOPLIMIT: ::c_int = 51; +pub const IPV6_HOPLIMIT: ::c_int = 52; +pub const IPV6_RECVHOPOPTS: ::c_int = 53; +pub const IPV6_HOPOPTS: ::c_int = 54; +pub const IPV6_RTHDRDSTOPTS: ::c_int = 55; +pub const IPV6_RECVRTHDR: ::c_int = 56; +pub const IPV6_RTHDR: ::c_int = 57; +pub const IPV6_RECVDSTOPTS: ::c_int = 58; +pub const IPV6_DSTOPTS: ::c_int = 59; +pub const IPV6_RECVPATHMTU: ::c_int = 60; +pub const IPV6_PATHMTU: ::c_int = 61; +pub const IPV6_DONTFRAG: ::c_int = 62; pub const IPV6_RECVTCLASS: ::c_int = 66; pub const IPV6_TCLASS: ::c_int = 67; - -pub const IP_PMTUDISC_DONT: ::c_int = 0; -pub const IP_PMTUDISC_WANT: ::c_int = 1; -pub const IP_PMTUDISC_DO: ::c_int = 2; -pub const IP_PMTUDISC_PROBE: ::c_int = 3; +pub const IPV6_AUTOFLOWLABEL: ::c_int = 70; +pub const IPV6_ADDR_PREFERENCES: ::c_int = 72; +pub const IPV6_MINHOPCOUNT: ::c_int = 73; +pub const IPV6_ORIGDSTADDR: ::c_int = 74; +pub const IPV6_RECVORIGDSTADDR: ::c_int = IPV6_ORIGDSTADDR; +pub const IPV6_TRANSPARENT: ::c_int = 75; +pub const IPV6_UNICAST_IF: ::c_int = 76; +pub const IPV6_PREFER_SRC_TMP: ::c_int = 0x0001; +pub const IPV6_PREFER_SRC_PUBLIC: ::c_int = 0x0002; +pub const IPV6_PREFER_SRC_PUBTMP_DEFAULT: ::c_int = 0x0100; +pub const IPV6_PREFER_SRC_COA: ::c_int = 0x0004; +pub const IPV6_PREFER_SRC_HOME: ::c_int = 0x0400; +pub const IPV6_PREFER_SRC_CGA: ::c_int = 0x0008; +pub const IPV6_PREFER_SRC_NONCGA: ::c_int = 0x0800; + +pub const IPV6_PMTUDISC_DONT: ::c_int = 0; +pub const IPV6_PMTUDISC_WANT: ::c_int = 1; +pub const IPV6_PMTUDISC_DO: ::c_int = 2; +pub const IPV6_PMTUDISC_PROBE: ::c_int = 3; +pub const IPV6_PMTUDISC_INTERFACE: ::c_int = 4; +pub const IPV6_PMTUDISC_OMIT: ::c_int = 5; pub const TCP_NODELAY: ::c_int = 1; pub const TCP_MAXSEG: ::c_int = 2; @@ -873,6 +971,43 @@ pub const TCP_WINDOW_CLAMP: ::c_int = 10; pub const TCP_INFO: ::c_int = 11; pub const TCP_QUICKACK: ::c_int = 12; pub const TCP_CONGESTION: ::c_int = 13; +pub const TCP_MD5SIG: ::c_int = 14; +cfg_if! { + if #[cfg(all(target_os = "linux", any(target_env = "gnu", target_env = "musl")))] { + // WARN: deprecated + pub const TCP_COOKIE_TRANSACTIONS: ::c_int = 15; + } +} +pub const TCP_THIN_LINEAR_TIMEOUTS: ::c_int = 16; +pub const TCP_THIN_DUPACK: ::c_int = 17; +pub const TCP_USER_TIMEOUT: ::c_int = 18; +pub const TCP_REPAIR: ::c_int = 19; +pub const TCP_REPAIR_QUEUE: ::c_int = 20; +pub const TCP_QUEUE_SEQ: ::c_int = 21; +pub const TCP_REPAIR_OPTIONS: ::c_int = 22; +pub const TCP_FASTOPEN: ::c_int = 23; +pub const TCP_TIMESTAMP: ::c_int = 24; +pub const TCP_NOTSENT_LOWAT: ::c_int = 25; +pub const TCP_CC_INFO: ::c_int = 26; +pub const TCP_SAVE_SYN: ::c_int = 27; +pub const TCP_SAVED_SYN: ::c_int = 28; +cfg_if! { + if #[cfg(not(target_os = "emscripten"))] { + // NOTE: emscripten doesn't support these options yet. + + pub const TCP_REPAIR_WINDOW: ::c_int = 29; + pub const TCP_FASTOPEN_CONNECT: ::c_int = 30; + pub const TCP_ULP: ::c_int = 31; + pub const TCP_MD5SIG_EXT: ::c_int = 32; + pub const TCP_FASTOPEN_KEY: ::c_int = 33; + pub const TCP_FASTOPEN_NO_COOKIE: ::c_int = 34; + pub const TCP_ZEROCOPY_RECEIVE: ::c_int = 35; + pub const TCP_INQ: ::c_int = 36; + pub const TCP_CM_INQ: ::c_int = TCP_INQ; + // NOTE: Some CI images doesn't have this option yet. + // pub const TCP_TX_DELAY: ::c_int = 37; + } +} pub const SO_DEBUG: ::c_int = 1; @@ -890,26 +1025,34 @@ pub const SS_DISABLE: ::c_int = 2; pub const PATH_MAX: ::c_int = 4096; +pub const UIO_MAXIOV: ::c_int = 1024; + pub const FD_SETSIZE: usize = 1024; pub const EPOLLIN: ::c_int = 0x1; pub const EPOLLPRI: ::c_int = 0x2; pub const EPOLLOUT: ::c_int = 0x4; +pub const EPOLLERR: ::c_int = 0x8; +pub const EPOLLHUP: ::c_int = 0x10; pub const EPOLLRDNORM: ::c_int = 0x40; pub const EPOLLRDBAND: ::c_int = 0x80; pub const EPOLLWRNORM: ::c_int = 0x100; pub const EPOLLWRBAND: ::c_int = 0x200; pub const EPOLLMSG: ::c_int = 0x400; -pub const EPOLLERR: ::c_int = 0x8; -pub const EPOLLHUP: ::c_int = 0x10; +pub const EPOLLRDHUP: ::c_int = 0x2000; +pub const EPOLLEXCLUSIVE: ::c_int = 0x10000000; +pub const EPOLLWAKEUP: ::c_int = 0x20000000; +pub const EPOLLONESHOT: ::c_int = 0x40000000; pub const EPOLLET: ::c_int = 0x80000000; pub const EPOLL_CTL_ADD: ::c_int = 1; pub const EPOLL_CTL_MOD: ::c_int = 3; pub const EPOLL_CTL_DEL: ::c_int = 2; +pub const MNT_FORCE: ::c_int = 0x1; pub const MNT_DETACH: ::c_int = 0x2; pub const MNT_EXPIRE: ::c_int = 0x4; +pub const UMOUNT_NOFOLLOW: ::c_int = 0x8; pub const Q_GETFMT: ::c_int = 0x800004; pub const Q_GETINFO: ::c_int = 0x800005; @@ -925,8 +1068,6 @@ pub const QIF_USAGE: u32 = 10; pub const QIF_TIMES: u32 = 48; pub const QIF_ALL: u32 = 63; -pub const MNT_FORCE: ::c_int = 0x1; - pub const Q_SYNC: ::c_int = 0x800001; pub const Q_QUOTAON: ::c_int = 0x800002; pub const Q_QUOTAOFF: ::c_int = 0x800003; @@ -989,13 +1130,13 @@ pub const CLONE_CHILD_CLEARTID: ::c_int = 0x200000; pub const CLONE_DETACHED: ::c_int = 0x400000; pub const CLONE_UNTRACED: ::c_int = 0x800000; pub const CLONE_CHILD_SETTID: ::c_int = 0x01000000; +pub const CLONE_NEWCGROUP: ::c_int = 0x02000000; pub const CLONE_NEWUTS: ::c_int = 0x04000000; pub const CLONE_NEWIPC: ::c_int = 0x08000000; pub const CLONE_NEWUSER: ::c_int = 0x10000000; pub const CLONE_NEWPID: ::c_int = 0x20000000; pub const CLONE_NEWNET: ::c_int = 0x40000000; pub const CLONE_IO: ::c_int = 0x80000000; -pub const CLONE_NEWCGROUP: ::c_int = 0x02000000; pub const WNOHANG: ::c_int = 0x00000001; pub const WUNTRACED: ::c_int = 0x00000002; @@ -1004,6 +1145,17 @@ pub const WEXITED: ::c_int = 0x00000004; pub const WCONTINUED: ::c_int = 0x00000008; pub const WNOWAIT: ::c_int = 0x01000000; +// Options for personality(2). +pub const ADDR_NO_RANDOMIZE: ::c_int = 0x0040000; +pub const MMAP_PAGE_ZERO: ::c_int = 0x0100000; +pub const ADDR_COMPAT_LAYOUT: ::c_int = 0x0200000; +pub const READ_IMPLIES_EXEC: ::c_int = 0x0400000; +pub const ADDR_LIMIT_32BIT: ::c_int = 0x0800000; +pub const SHORT_INODE: ::c_int = 0x1000000; +pub const WHOLE_SECONDS: ::c_int = 0x2000000; +pub const STICKY_TIMEOUTS: ::c_int = 0x4000000; +pub const ADDR_LIMIT_3GB: ::c_int = 0x8000000; + // Options set using PTRACE_SETOPTIONS. pub const PTRACE_O_TRACESYSGOOD: ::c_int = 0x00000001; pub const PTRACE_O_TRACEFORK: ::c_int = 0x00000002; @@ -1013,8 +1165,8 @@ pub const PTRACE_O_TRACEEXEC: ::c_int = 0x00000010; pub const PTRACE_O_TRACEVFORKDONE: ::c_int = 0x00000020; pub const PTRACE_O_TRACEEXIT: ::c_int = 0x00000040; pub const PTRACE_O_TRACESECCOMP: ::c_int = 0x00000080; -pub const PTRACE_O_EXITKILL: ::c_int = 0x00100000; pub const PTRACE_O_SUSPEND_SECCOMP: ::c_int = 0x00200000; +pub const PTRACE_O_EXITKILL: ::c_int = 0x00100000; pub const PTRACE_O_MASK: ::c_int = 0x003000ff; // Wait extended result codes for the above trace options. @@ -1025,8 +1177,6 @@ pub const PTRACE_EVENT_EXEC: ::c_int = 4; pub const PTRACE_EVENT_VFORK_DONE: ::c_int = 5; pub const PTRACE_EVENT_EXIT: ::c_int = 6; pub const PTRACE_EVENT_SECCOMP: ::c_int = 7; -// PTRACE_EVENT_STOP was added to glibc in 2.26 -// pub const PTRACE_EVENT_STOP: ::c_int = 128; pub const __WNOTHREAD: ::c_int = 0x20000000; pub const __WALL: ::c_int = 0x40000000; @@ -1061,6 +1211,22 @@ pub const PIPE_BUF: usize = 4096; pub const SI_LOAD_SHIFT: ::c_uint = 16; +// si_code values for SIGBUS signal +pub const BUS_ADRALN: ::c_int = 1; +pub const BUS_ADRERR: ::c_int = 2; +pub const BUS_OBJERR: ::c_int = 3; +// Linux-specific si_code values for SIGBUS signal +pub const BUS_MCEERR_AR: ::c_int = 4; +pub const BUS_MCEERR_AO: ::c_int = 5; + +// si_code values for SIGCHLD signal +pub const CLD_EXITED: ::c_int = 1; +pub const CLD_KILLED: ::c_int = 2; +pub const CLD_DUMPED: ::c_int = 3; +pub const CLD_TRAPPED: ::c_int = 4; +pub const CLD_STOPPED: ::c_int = 5; +pub const CLD_CONTINUED: ::c_int = 6; + pub const SIGEV_SIGNAL: ::c_int = 0; pub const SIGEV_NONE: ::c_int = 1; pub const SIGEV_THREAD: ::c_int = 2; @@ -1068,6 +1234,11 @@ pub const SIGEV_THREAD: ::c_int = 2; pub const P_ALL: idtype_t = 0; pub const P_PID: idtype_t = 1; pub const P_PGID: idtype_t = 2; +cfg_if! { + if #[cfg(not(target_os = "emscripten"))] { + pub const P_PIDFD: idtype_t = 3; + } +} pub const UTIME_OMIT: c_long = 1073741822; pub const UTIME_NOW: c_long = 1073741823; @@ -1080,6 +1251,10 @@ pub const POLLHUP: ::c_short = 0x10; pub const POLLNVAL: ::c_short = 0x20; pub const POLLRDNORM: ::c_short = 0x040; pub const POLLRDBAND: ::c_short = 0x080; +#[cfg(not(any(target_arch = "sparc", target_arch = "sparc64")))] +pub const POLLRDHUP: ::c_short = 0x2000; +#[cfg(any(target_arch = "sparc", target_arch = "sparc64"))] +pub const POLLRDHUP: ::c_short = 0x800; pub const IPTOS_LOWDELAY: u8 = 0x10; pub const IPTOS_THROUGHPUT: u8 = 0x08; @@ -1166,6 +1341,7 @@ pub const ARPHRD_ADAPT: u16 = 264; pub const ARPHRD_ROSE: u16 = 270; pub const ARPHRD_X25: u16 = 271; pub const ARPHRD_HWX25: u16 = 272; +pub const ARPHRD_CAN: u16 = 280; pub const ARPHRD_PPP: u16 = 512; pub const ARPHRD_CISCO: u16 = 513; pub const ARPHRD_HDLC: u16 = ARPHRD_CISCO; @@ -1202,12 +1378,119 @@ pub const ARPHRD_IEEE802154: u16 = 804; pub const ARPHRD_VOID: u16 = 0xFFFF; pub const ARPHRD_NONE: u16 = 0xFFFE; -pub const SO_EE_ORIGIN_NONE: u8 = 0; -pub const SO_EE_ORIGIN_LOCAL: u8 = 1; -pub const SO_EE_ORIGIN_ICMP: u8 = 2; -pub const SO_EE_ORIGIN_ICMP6: u8 = 3; -pub const SO_EE_ORIGIN_TXSTATUS: u8 = 4; -pub const SO_EE_ORIGIN_TIMESTAMPING: u8 = SO_EE_ORIGIN_TXSTATUS; +cfg_if! { + if #[cfg(target_os = "emscripten")] { + // Emscripten does not define any `*_SUPER_MAGIC` constants. + } else if #[cfg(not(target_arch = "s390x"))] { + pub const ADFS_SUPER_MAGIC: ::c_long = 0x0000adf5; + pub const AFFS_SUPER_MAGIC: ::c_long = 0x0000adff; + pub const AFS_SUPER_MAGIC: ::c_long = 0x5346414f; + pub const AUTOFS_SUPER_MAGIC: ::c_long = 0x0187; + pub const BPF_FS_MAGIC: ::c_long = 0xcafe4a11; + pub const BTRFS_SUPER_MAGIC: ::c_long = 0x9123683e; + pub const CGROUP2_SUPER_MAGIC: ::c_long = 0x63677270; + pub const CGROUP_SUPER_MAGIC: ::c_long = 0x27e0eb; + pub const CODA_SUPER_MAGIC: ::c_long = 0x73757245; + pub const CRAMFS_MAGIC: ::c_long = 0x28cd3d45; + pub const DEBUGFS_MAGIC: ::c_long = 0x64626720; + pub const DEVPTS_SUPER_MAGIC: ::c_long = 0x1cd1; + pub const ECRYPTFS_SUPER_MAGIC: ::c_long = 0xf15f; + pub const EFS_SUPER_MAGIC: ::c_long = 0x00414a53; + pub const EXT2_SUPER_MAGIC: ::c_long = 0x0000ef53; + pub const EXT3_SUPER_MAGIC: ::c_long = 0x0000ef53; + pub const EXT4_SUPER_MAGIC: ::c_long = 0x0000ef53; + pub const F2FS_SUPER_MAGIC: ::c_long = 0xf2f52010; + pub const FUSE_SUPER_MAGIC: ::c_long = 0x65735546; + pub const FUTEXFS_SUPER_MAGIC: ::c_long = 0xbad1dea; + pub const HOSTFS_SUPER_MAGIC: ::c_long = 0x00c0ffee; + pub const HPFS_SUPER_MAGIC: ::c_long = 0xf995e849; + pub const HUGETLBFS_MAGIC: ::c_long = 0x958458f6; + pub const ISOFS_SUPER_MAGIC: ::c_long = 0x00009660; + pub const JFFS2_SUPER_MAGIC: ::c_long = 0x000072b6; + pub const MINIX2_SUPER_MAGIC2: ::c_long = 0x00002478; + pub const MINIX2_SUPER_MAGIC: ::c_long = 0x00002468; + pub const MINIX3_SUPER_MAGIC: ::c_long = 0x4d5a; + pub const MINIX_SUPER_MAGIC2: ::c_long = 0x0000138f; + pub const MINIX_SUPER_MAGIC: ::c_long = 0x0000137f; + pub const MSDOS_SUPER_MAGIC: ::c_long = 0x00004d44; + pub const NCP_SUPER_MAGIC: ::c_long = 0x0000564c; + pub const NFS_SUPER_MAGIC: ::c_long = 0x00006969; + pub const NILFS_SUPER_MAGIC: ::c_long = 0x3434; + pub const OCFS2_SUPER_MAGIC: ::c_long = 0x7461636f; + pub const OPENPROM_SUPER_MAGIC: ::c_long = 0x00009fa1; + pub const OVERLAYFS_SUPER_MAGIC: ::c_long = 0x794c7630; + pub const PROC_SUPER_MAGIC: ::c_long = 0x00009fa0; + pub const QNX4_SUPER_MAGIC: ::c_long = 0x0000002f; + pub const QNX6_SUPER_MAGIC: ::c_long = 0x68191122; + pub const RDTGROUP_SUPER_MAGIC: ::c_long = 0x7655821; + pub const REISERFS_SUPER_MAGIC: ::c_long = 0x52654973; + pub const SECURITYFS_MAGIC: ::c_long = 0x73636673; + pub const SELINUX_MAGIC: ::c_long = 0xf97cff8c; + pub const SMACK_MAGIC: ::c_long = 0x43415d53; + pub const SMB_SUPER_MAGIC: ::c_long = 0x0000517b; + pub const SYSFS_MAGIC: ::c_long = 0x62656572; + pub const TMPFS_MAGIC: ::c_long = 0x01021994; + pub const TRACEFS_MAGIC: ::c_long = 0x74726163; + pub const UDF_SUPER_MAGIC: ::c_long = 0x15013346; + pub const USBDEVICE_SUPER_MAGIC: ::c_long = 0x00009fa2; + pub const XENFS_SUPER_MAGIC: ::c_long = 0xabba1974; + pub const NSFS_MAGIC: ::c_long = 0x6e736673; + } else if #[cfg(target_arch = "s390x")] { + pub const ADFS_SUPER_MAGIC: ::c_uint = 0x0000adf5; + pub const AFFS_SUPER_MAGIC: ::c_uint = 0x0000adff; + pub const AFS_SUPER_MAGIC: ::c_uint = 0x5346414f; + pub const AUTOFS_SUPER_MAGIC: ::c_uint = 0x0187; + pub const BPF_FS_MAGIC: ::c_uint = 0xcafe4a11; + pub const BTRFS_SUPER_MAGIC: ::c_uint = 0x9123683e; + pub const CGROUP2_SUPER_MAGIC: ::c_uint = 0x63677270; + pub const CGROUP_SUPER_MAGIC: ::c_uint = 0x27e0eb; + pub const CODA_SUPER_MAGIC: ::c_uint = 0x73757245; + pub const CRAMFS_MAGIC: ::c_uint = 0x28cd3d45; + pub const DEBUGFS_MAGIC: ::c_uint = 0x64626720; + pub const DEVPTS_SUPER_MAGIC: ::c_uint = 0x1cd1; + pub const ECRYPTFS_SUPER_MAGIC: ::c_uint = 0xf15f; + pub const EFS_SUPER_MAGIC: ::c_uint = 0x00414a53; + pub const EXT2_SUPER_MAGIC: ::c_uint = 0x0000ef53; + pub const EXT3_SUPER_MAGIC: ::c_uint = 0x0000ef53; + pub const EXT4_SUPER_MAGIC: ::c_uint = 0x0000ef53; + pub const F2FS_SUPER_MAGIC: ::c_uint = 0xf2f52010; + pub const FUSE_SUPER_MAGIC: ::c_uint = 0x65735546; + pub const FUTEXFS_SUPER_MAGIC: ::c_uint = 0xbad1dea; + pub const HOSTFS_SUPER_MAGIC: ::c_uint = 0x00c0ffee; + pub const HPFS_SUPER_MAGIC: ::c_uint = 0xf995e849; + pub const HUGETLBFS_MAGIC: ::c_uint = 0x958458f6; + pub const ISOFS_SUPER_MAGIC: ::c_uint = 0x00009660; + pub const JFFS2_SUPER_MAGIC: ::c_uint = 0x000072b6; + pub const MINIX2_SUPER_MAGIC2: ::c_uint = 0x00002478; + pub const MINIX2_SUPER_MAGIC: ::c_uint = 0x00002468; + pub const MINIX3_SUPER_MAGIC: ::c_uint = 0x4d5a; + pub const MINIX_SUPER_MAGIC2: ::c_uint = 0x0000138f; + pub const MINIX_SUPER_MAGIC: ::c_uint = 0x0000137f; + pub const MSDOS_SUPER_MAGIC: ::c_uint = 0x00004d44; + pub const NCP_SUPER_MAGIC: ::c_uint = 0x0000564c; + pub const NFS_SUPER_MAGIC: ::c_uint = 0x00006969; + pub const NILFS_SUPER_MAGIC: ::c_uint = 0x3434; + pub const OCFS2_SUPER_MAGIC: ::c_uint = 0x7461636f; + pub const OPENPROM_SUPER_MAGIC: ::c_uint = 0x00009fa1; + pub const OVERLAYFS_SUPER_MAGIC: ::c_uint = 0x794c7630; + pub const PROC_SUPER_MAGIC: ::c_uint = 0x00009fa0; + pub const QNX4_SUPER_MAGIC: ::c_uint = 0x0000002f; + pub const QNX6_SUPER_MAGIC: ::c_uint = 0x68191122; + pub const RDTGROUP_SUPER_MAGIC: ::c_uint = 0x7655821; + pub const REISERFS_SUPER_MAGIC: ::c_uint = 0x52654973; + pub const SECURITYFS_MAGIC: ::c_uint = 0x73636673; + pub const SELINUX_MAGIC: ::c_uint = 0xf97cff8c; + pub const SMACK_MAGIC: ::c_uint = 0x43415d53; + pub const SMB_SUPER_MAGIC: ::c_uint = 0x0000517b; + pub const SYSFS_MAGIC: ::c_uint = 0x62656572; + pub const TMPFS_MAGIC: ::c_uint = 0x01021994; + pub const TRACEFS_MAGIC: ::c_uint = 0x74726163; + pub const UDF_SUPER_MAGIC: ::c_uint = 0x15013346; + pub const USBDEVICE_SUPER_MAGIC: ::c_uint = 0x00009fa2; + pub const XENFS_SUPER_MAGIC: ::c_uint = 0xabba1974; + pub const NSFS_MAGIC: ::c_uint = 0x6e736673; + } +} const_fn! { {const} fn CMSG_ALIGN(len: usize) -> usize { @@ -1244,7 +1527,7 @@ f! { return } - pub fn FD_ISSET(fd: ::c_int, set: *mut fd_set) -> bool { + pub fn FD_ISSET(fd: ::c_int, set: *const fd_set) -> bool { let fd = fd as usize; let size = ::mem::size_of_val(&(*set).fds_bits[0]) * 8; return ((*set).fds_bits[fd / size] & (1 << (fd % size))) != 0 @@ -1262,89 +1545,97 @@ f! { *slot = 0; } } +} + +safe_f! { + pub fn SIGRTMAX() -> ::c_int { + unsafe { __libc_current_sigrtmax() } + } + + pub fn SIGRTMIN() -> ::c_int { + unsafe { __libc_current_sigrtmin() } + } - pub fn WIFSTOPPED(status: ::c_int) -> bool { + pub {const} fn WIFSTOPPED(status: ::c_int) -> bool { (status & 0xff) == 0x7f } - pub fn WSTOPSIG(status: ::c_int) -> ::c_int { + pub {const} fn WSTOPSIG(status: ::c_int) -> ::c_int { (status >> 8) & 0xff } - pub fn WIFCONTINUED(status: ::c_int) -> bool { + pub {const} fn WIFCONTINUED(status: ::c_int) -> bool { status == 0xffff } - pub fn WIFSIGNALED(status: ::c_int) -> bool { + pub {const} fn WIFSIGNALED(status: ::c_int) -> bool { ((status & 0x7f) + 1) as i8 >= 2 } - pub fn WTERMSIG(status: ::c_int) -> ::c_int { + pub {const} fn WTERMSIG(status: ::c_int) -> ::c_int { status & 0x7f } - pub fn WIFEXITED(status: ::c_int) -> bool { + pub {const} fn WIFEXITED(status: ::c_int) -> bool { (status & 0x7f) == 0 } - pub fn WEXITSTATUS(status: ::c_int) -> ::c_int { + pub {const} fn WEXITSTATUS(status: ::c_int) -> ::c_int { (status >> 8) & 0xff } - pub fn WCOREDUMP(status: ::c_int) -> bool { + pub {const} fn WCOREDUMP(status: ::c_int) -> bool { (status & 0x80) != 0 } - pub fn QCMD(cmd: ::c_int, type_: ::c_int) -> ::c_int { + pub {const} fn W_EXITCODE(ret: ::c_int, sig: ::c_int) -> ::c_int { + (ret << 8) | sig + } + + pub {const} fn W_STOPCODE(sig: ::c_int) -> ::c_int { + (sig << 8) | 0x7f + } + + pub {const} fn QCMD(cmd: ::c_int, type_: ::c_int) -> ::c_int { (cmd << 8) | (type_ & 0x00ff) } - pub fn IPOPT_COPIED(o: u8) -> u8 { + pub {const} fn IPOPT_COPIED(o: u8) -> u8 { o & IPOPT_COPY } - pub fn IPOPT_CLASS(o: u8) -> u8 { + pub {const} fn IPOPT_CLASS(o: u8) -> u8 { o & IPOPT_CLASS_MASK } - pub fn IPOPT_NUMBER(o: u8) -> u8 { + pub {const} fn IPOPT_NUMBER(o: u8) -> u8 { o & IPOPT_NUMBER_MASK } - pub fn IPTOS_ECN(x: u8) -> u8 { + pub {const} fn IPTOS_ECN(x: u8) -> u8 { x & ::IPTOS_ECN_MASK } - - pub fn SO_EE_OFFENDER(ee: *const ::sock_extended_err) -> *mut ::sockaddr { - ee.offset(1) as *mut ::sockaddr - } } extern "C" { + #[doc(hidden)] + pub fn __libc_current_sigrtmax() -> ::c_int; + #[doc(hidden)] + pub fn __libc_current_sigrtmin() -> ::c_int; + pub fn sem_destroy(sem: *mut sem_t) -> ::c_int; - pub fn sem_init( - sem: *mut sem_t, - pshared: ::c_int, - value: ::c_uint, - ) -> ::c_int; + pub fn sem_init(sem: *mut sem_t, pshared: ::c_int, value: ::c_uint) -> ::c_int; pub fn fdatasync(fd: ::c_int) -> ::c_int; - pub fn mincore( - addr: *mut ::c_void, - len: ::size_t, - vec: *mut ::c_uchar, - ) -> ::c_int; + pub fn mincore(addr: *mut ::c_void, len: ::size_t, vec: *mut ::c_uchar) -> ::c_int; + pub fn clock_getres(clk_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; pub fn clock_gettime(clk_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; - pub fn clock_settime( - clk_id: ::clockid_t, - tp: *const ::timespec, - ) -> ::c_int; + pub fn clock_settime(clk_id: ::clockid_t, tp: *const ::timespec) -> ::c_int; + pub fn clock_getcpuclockid(pid: ::pid_t, clk_id: *mut ::clockid_t) -> ::c_int; + pub fn dirfd(dirp: *mut ::DIR) -> ::c_int; - pub fn pthread_getattr_np( - native: ::pthread_t, - attr: *mut ::pthread_attr_t, - ) -> ::c_int; + pub fn pthread_getattr_np(native: ::pthread_t, attr: *mut ::pthread_attr_t) -> ::c_int; pub fn pthread_attr_getstack( attr: *const ::pthread_attr_t, stackaddr: *mut *mut ::c_void, @@ -1359,16 +1650,13 @@ extern "C" { pub fn fstatfs64(fd: ::c_int, buf: *mut statfs64) -> ::c_int; pub fn statvfs64(path: *const ::c_char, buf: *mut statvfs64) -> ::c_int; pub fn fstatvfs64(fd: ::c_int, buf: *mut statvfs64) -> ::c_int; - pub fn memrchr( - cx: *const ::c_void, - c: ::c_int, - n: ::size_t, - ) -> *mut ::c_void; + pub fn memrchr(cx: *const ::c_void, c: ::c_int, n: ::size_t) -> *mut ::c_void; - pub fn posix_fadvise( + pub fn posix_fadvise(fd: ::c_int, offset: ::off_t, len: ::off_t, advise: ::c_int) -> ::c_int; + pub fn posix_fadvise64( fd: ::c_int, - offset: ::off_t, - len: ::off_t, + offset: ::off64_t, + len: ::off64_t, advise: ::c_int, ) -> ::c_int; pub fn futimens(fd: ::c_int, times: *const ::timespec) -> ::c_int; @@ -1380,11 +1668,7 @@ extern "C" { ) -> ::c_int; pub fn duplocale(base: ::locale_t) -> ::locale_t; pub fn freelocale(loc: ::locale_t); - pub fn newlocale( - mask: ::c_int, - locale: *const ::c_char, - base: ::locale_t, - ) -> ::locale_t; + pub fn newlocale(mask: ::c_int, locale: *const ::c_char, base: ::locale_t) -> ::locale_t; pub fn uselocale(loc: ::locale_t) -> ::locale_t; pub fn creat64(path: *const c_char, mode: mode_t) -> ::c_int; pub fn fstat64(fildes: ::c_int, buf: *mut stat64) -> ::c_int; @@ -1406,36 +1690,14 @@ extern "C" { offset: off64_t, ) -> *mut ::c_void; pub fn open64(path: *const c_char, oflag: ::c_int, ...) -> ::c_int; - pub fn openat64( - fd: ::c_int, - path: *const c_char, - oflag: ::c_int, - ... - ) -> ::c_int; - pub fn pread64( - fd: ::c_int, - buf: *mut ::c_void, - count: ::size_t, - offset: off64_t, - ) -> ::ssize_t; - pub fn preadv64( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - offset: ::off64_t, - ) -> ::ssize_t; + pub fn openat64(fd: ::c_int, path: *const c_char, oflag: ::c_int, ...) -> ::c_int; + pub fn pread64(fd: ::c_int, buf: *mut ::c_void, count: ::size_t, offset: off64_t) -> ::ssize_t; pub fn pwrite64( fd: ::c_int, buf: *const ::c_void, count: ::size_t, offset: off64_t, ) -> ::ssize_t; - pub fn pwritev64( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - offset: ::off64_t, - ) -> ::ssize_t; pub fn readdir64(dirp: *mut ::DIR) -> *mut ::dirent64; pub fn readdir64_r( dirp: *mut ::DIR, @@ -1459,16 +1721,7 @@ extern "C" { attr: *mut pthread_condattr_t, clock_id: ::clockid_t, ) -> ::c_int; - pub fn pthread_condattr_setpshared( - attr: *mut pthread_condattr_t, - pshared: ::c_int, - ) -> ::c_int; - pub fn accept4( - fd: ::c_int, - addr: *mut ::sockaddr, - len: *mut ::socklen_t, - flg: ::c_int, - ) -> ::c_int; + pub fn pthread_condattr_setpshared(attr: *mut pthread_condattr_t, pshared: ::c_int) -> ::c_int; pub fn pthread_mutexattr_setpshared( attr: *mut pthread_mutexattr_t, pshared: ::c_int, @@ -1477,34 +1730,13 @@ extern "C" { attr: *const pthread_rwlockattr_t, val: *mut ::c_int, ) -> ::c_int; - pub fn pthread_rwlockattr_setpshared( - attr: *mut pthread_rwlockattr_t, - val: ::c_int, - ) -> ::c_int; - pub fn ptsname_r( - fd: ::c_int, - buf: *mut ::c_char, - buflen: ::size_t, - ) -> ::c_int; + pub fn pthread_rwlockattr_setpshared(attr: *mut pthread_rwlockattr_t, val: ::c_int) -> ::c_int; + pub fn ptsname_r(fd: ::c_int, buf: *mut ::c_char, buflen: ::size_t) -> ::c_int; pub fn clearenv() -> ::c_int; - pub fn waitid( - idtype: idtype_t, - id: id_t, - infop: *mut ::siginfo_t, - options: ::c_int, - ) -> ::c_int; - pub fn setreuid(ruid: ::uid_t, euid: ::uid_t) -> ::c_int; - pub fn setregid(rgid: ::gid_t, egid: ::gid_t) -> ::c_int; - pub fn getresuid( - ruid: *mut ::uid_t, - euid: *mut ::uid_t, - suid: *mut ::uid_t, - ) -> ::c_int; - pub fn getresgid( - rgid: *mut ::gid_t, - egid: *mut ::gid_t, - sgid: *mut ::gid_t, - ) -> ::c_int; + pub fn waitid(idtype: idtype_t, id: id_t, infop: *mut ::siginfo_t, options: ::c_int) + -> ::c_int; + pub fn getresuid(ruid: *mut ::uid_t, euid: *mut ::uid_t, suid: *mut ::uid_t) -> ::c_int; + pub fn getresgid(rgid: *mut ::gid_t, egid: *mut ::gid_t, sgid: *mut ::gid_t) -> ::c_int; pub fn acct(filename: *const ::c_char) -> ::c_int; pub fn brk(addr: *mut ::c_void) -> ::c_int; pub fn sbrk(increment: ::intptr_t) -> *mut ::c_void; @@ -1521,19 +1753,6 @@ extern "C" { options: ::c_int, rusage: *mut ::rusage, ) -> ::pid_t; - pub fn openpty( - amaster: *mut ::c_int, - aslave: *mut ::c_int, - name: *mut ::c_char, - termp: *const termios, - winp: *const ::winsize, - ) -> ::c_int; - pub fn forkpty( - amaster: *mut ::c_int, - name: *mut ::c_char, - termp: *const termios, - winp: *const ::winsize, - ) -> ::pid_t; pub fn login_tty(fd: ::c_int) -> ::c_int; pub fn execvpe( file: *const ::c_char, @@ -1547,36 +1766,50 @@ extern "C" { ) -> ::c_int; pub fn getifaddrs(ifap: *mut *mut ::ifaddrs) -> ::c_int; pub fn freeifaddrs(ifa: *mut ::ifaddrs); - pub fn bind( - socket: ::c_int, - address: *const ::sockaddr, - address_len: ::socklen_t, - ) -> ::c_int; + pub fn bind(socket: ::c_int, address: *const ::sockaddr, address_len: ::socklen_t) -> ::c_int; - pub fn writev( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - ) -> ::ssize_t; - pub fn readv( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - ) -> ::ssize_t; + pub fn writev(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int) -> ::ssize_t; + pub fn readv(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int) -> ::ssize_t; - pub fn sendmsg( - fd: ::c_int, - msg: *const ::msghdr, - flags: ::c_int, - ) -> ::ssize_t; - pub fn recvmsg( - fd: ::c_int, - msg: *mut ::msghdr, - flags: ::c_int, - ) -> ::ssize_t; + pub fn sendmsg(fd: ::c_int, msg: *const ::msghdr, flags: ::c_int) -> ::ssize_t; + pub fn recvmsg(fd: ::c_int, msg: *mut ::msghdr, flags: ::c_int) -> ::ssize_t; pub fn uname(buf: *mut ::utsname) -> ::c_int; } +cfg_if! { + if #[cfg(not(target_env = "uclibc"))] { + extern "C" { + pub fn preadv64( + fd: ::c_int, + iov: *const ::iovec, + iovcnt: ::c_int, + offset: ::off64_t, + ) -> ::ssize_t; + pub fn pwritev64( + fd: ::c_int, + iov: *const ::iovec, + iovcnt: ::c_int, + offset: ::off64_t, + ) -> ::ssize_t; + // uclibc has separate non-const version of this function + pub fn forkpty( + amaster: *mut ::c_int, + name: *mut ::c_char, + termp: *const termios, + winp: *const ::winsize, + ) -> ::pid_t; + // uclibc has separate non-const version of this function + pub fn openpty( + amaster: *mut ::c_int, + aslave: *mut ::c_int, + name: *mut ::c_char, + termp: *const termios, + winp: *const ::winsize, + ) -> ::c_int; + } + } +} + cfg_if! { if #[cfg(target_os = "emscripten")] { mod emscripten; @@ -1584,6 +1817,9 @@ cfg_if! { } else if #[cfg(target_os = "linux")] { mod linux; pub use self::linux::*; + } else if #[cfg(target_os = "l4re")] { + mod linux; + pub use self::linux::*; } else if #[cfg(target_os = "android")] { mod android; pub use self::android::*; diff --git a/crux-mir/lib/libc/src/unix/mod.rs b/crux-mir/lib/libc/src/unix/mod.rs index 6d3ef9443..fb9ebf792 100644 --- a/crux-mir/lib/libc/src/unix/mod.rs +++ b/crux-mir/lib/libc/src/unix/mod.rs @@ -23,13 +23,21 @@ pub type uintptr_t = usize; pub type ssize_t = isize; pub type pid_t = i32; -pub type uid_t = u32; -pub type gid_t = u32; pub type in_addr_t = u32; pub type in_port_t = u16; pub type sighandler_t = ::size_t; pub type cc_t = ::c_uchar; +cfg_if! { + if #[cfg(any(target_os = "espidf", target_os = "horizon"))] { + pub type uid_t = ::c_ushort; + pub type gid_t = ::c_ushort; + } else { + pub type uid_t = u32; + pub type gid_t = u32; + } +} + #[cfg_attr(feature = "extra_traits", derive(Debug))] pub enum DIR {} impl ::Copy for DIR {} @@ -226,8 +234,13 @@ pub const S_ISUID: ::mode_t = 0x800; pub const S_ISGID: ::mode_t = 0x400; pub const S_ISVTX: ::mode_t = 0x200; -pub const IF_NAMESIZE: ::size_t = 16; -pub const IFNAMSIZ: ::size_t = IF_NAMESIZE; +cfg_if! { + if #[cfg(not(any(target_os = "haiku", target_os = "illumos", + target_os = "solaris")))] { + pub const IF_NAMESIZE: ::size_t = 16; + pub const IFNAMSIZ: ::size_t = IF_NAMESIZE; + } +} pub const LOG_EMERG: ::c_int = 0; pub const LOG_ALERT: ::c_int = 1; @@ -256,19 +269,18 @@ pub const LOG_LOCAL5: ::c_int = 21 << 3; pub const LOG_LOCAL6: ::c_int = 22 << 3; pub const LOG_LOCAL7: ::c_int = 23 << 3; -pub const LOG_PID: ::c_int = 0x01; -pub const LOG_CONS: ::c_int = 0x02; -pub const LOG_ODELAY: ::c_int = 0x04; -pub const LOG_NDELAY: ::c_int = 0x08; -pub const LOG_NOWAIT: ::c_int = 0x10; - +cfg_if! { + if #[cfg(not(target_os = "haiku"))] { + pub const LOG_PID: ::c_int = 0x01; + pub const LOG_CONS: ::c_int = 0x02; + pub const LOG_ODELAY: ::c_int = 0x04; + pub const LOG_NDELAY: ::c_int = 0x08; + pub const LOG_NOWAIT: ::c_int = 0x10; + } +} pub const LOG_PRIMASK: ::c_int = 7; pub const LOG_FACMASK: ::c_int = 0x3f8; -pub const PRIO_PROCESS: ::c_int = 0; -pub const PRIO_PGRP: ::c_int = 1; -pub const PRIO_USER: ::c_int = 2; - pub const PRIO_MIN: ::c_int = -20; pub const PRIO_MAX: ::c_int = 20; @@ -293,14 +305,42 @@ pub const ATF_PUBL: ::c_int = 0x08; pub const ATF_USETRAILERS: ::c_int = 0x10; cfg_if! { - if #[cfg(target_os = "l4re")] { - // required libraries for L4Re are linked externally, ATM + if #[cfg(any(target_os = "l4re", target_os = "espidf"))] { + // required libraries for L4Re and the ESP-IDF framework are linked externally, ATM } else if #[cfg(feature = "std")] { // cargo build, don't pull in anything extra as the libstd dep // already pulls in all libs. + } else if #[cfg(all(target_os = "linux", + any(target_env = "gnu", target_env = "uclibc"), + feature = "rustc-dep-of-std"))] { + #[link(name = "util", kind = "static", modifiers = "-bundle", + cfg(target_feature = "crt-static"))] + #[link(name = "rt", kind = "static", modifiers = "-bundle", + cfg(target_feature = "crt-static"))] + #[link(name = "pthread", kind = "static", modifiers = "-bundle", + cfg(target_feature = "crt-static"))] + #[link(name = "m", kind = "static", modifiers = "-bundle", + cfg(target_feature = "crt-static"))] + #[link(name = "dl", kind = "static", modifiers = "-bundle", + cfg(target_feature = "crt-static"))] + #[link(name = "c", kind = "static", modifiers = "-bundle", + cfg(target_feature = "crt-static"))] + #[link(name = "gcc_eh", kind = "static", modifiers = "-bundle", + cfg(target_feature = "crt-static"))] + #[link(name = "gcc", kind = "static", modifiers = "-bundle", + cfg(target_feature = "crt-static"))] + #[link(name = "c", kind = "static", modifiers = "-bundle", + cfg(target_feature = "crt-static"))] + #[link(name = "util", cfg(not(target_feature = "crt-static")))] + #[link(name = "rt", cfg(not(target_feature = "crt-static")))] + #[link(name = "pthread", cfg(not(target_feature = "crt-static")))] + #[link(name = "m", cfg(not(target_feature = "crt-static")))] + #[link(name = "dl", cfg(not(target_feature = "crt-static")))] + #[link(name = "c", cfg(not(target_feature = "crt-static")))] + extern {} } else if #[cfg(target_env = "musl")] { #[cfg_attr(feature = "rustc-dep-of-std", - link(name = "c", kind = "static", + link(name = "c", kind = "static", modifiers = "-bundle", cfg(target_feature = "crt-static")))] #[cfg_attr(feature = "rustc-dep-of-std", link(name = "c", cfg(not(target_feature = "crt-static"))))] @@ -308,16 +348,18 @@ cfg_if! { } else if #[cfg(target_os = "emscripten")] { #[link(name = "c")] extern {} - } else if #[cfg(all(target_os = "netbsd", - feature = "rustc-dep-of-std", - target_vendor = "rumprun"))] { - // Since we don't use -nodefaultlibs on Rumprun, libc is always pulled - // in automatically by the linker. We avoid passing it explicitly, as it - // causes some versions of binutils to crash with an assertion failure. - #[link(name = "m")] + } else if #[cfg(all(target_os = "android", feature = "rustc-dep-of-std"))] { + #[link(name = "c", kind = "static", modifiers = "-bundle", + cfg(target_feature = "crt-static"))] + #[link(name = "m", kind = "static", modifiers = "-bundle", + cfg(target_feature = "crt-static"))] + #[link(name = "m", cfg(not(target_feature = "crt-static")))] + #[link(name = "c", cfg(not(target_feature = "crt-static")))] extern {} } else if #[cfg(any(target_os = "macos", target_os = "ios", + target_os = "tvos", + target_os = "watchos", target_os = "android", target_os = "openbsd"))] { #[link(name = "c")] @@ -342,7 +384,7 @@ cfg_if! { extern {} } else if #[cfg(target_os = "redox")] { #[cfg_attr(feature = "rustc-dep-of-std", - link(name = "c", kind = "static-nobundle", + link(name = "c", kind = "static", modifiers = "-bundle", cfg(target_feature = "crt-static")))] #[cfg_attr(feature = "rustc-dep-of-std", link(name = "c", cfg(not(target_feature = "crt-static"))))] @@ -392,18 +434,14 @@ extern "C" { base: *mut c_void, num: size_t, size: size_t, - compar: ::Option< - unsafe extern "C" fn(*const c_void, *const c_void) -> c_int, - >, + compar: ::Option c_int>, ); pub fn bsearch( key: *const c_void, base: *const c_void, num: size_t, size: size_t, - compar: ::Option< - unsafe extern "C" fn(*const c_void, *const c_void) -> c_int, - >, + compar: ::Option c_int>, ) -> *mut c_void; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), @@ -414,28 +452,21 @@ extern "C" { all(target_os = "macos", target_arch = "x86"), link_name = "freopen$UNIX2003" )] - pub fn freopen( - filename: *const c_char, - mode: *const c_char, - file: *mut FILE, - ) -> *mut FILE; + pub fn freopen(filename: *const c_char, mode: *const c_char, file: *mut FILE) -> *mut FILE; + pub fn fmemopen(buf: *mut c_void, size: size_t, mode: *const c_char) -> *mut FILE; + pub fn open_memstream(ptr: *mut *mut c_char, sizeloc: *mut size_t) -> *mut FILE; + pub fn fflush(file: *mut FILE) -> c_int; pub fn fclose(file: *mut FILE) -> c_int; pub fn remove(filename: *const c_char) -> c_int; pub fn rename(oldname: *const c_char, newname: *const c_char) -> c_int; pub fn tmpfile() -> *mut FILE; - pub fn setvbuf( - stream: *mut FILE, - buffer: *mut c_char, - mode: c_int, - size: size_t, - ) -> c_int; + pub fn setvbuf(stream: *mut FILE, buffer: *mut c_char, mode: c_int, size: size_t) -> c_int; pub fn setbuf(stream: *mut FILE, buf: *mut c_char); pub fn getchar() -> c_int; pub fn putchar(c: c_int) -> c_int; pub fn fgetc(stream: *mut FILE) -> c_int; - pub fn fgets(buf: *mut c_char, n: c_int, stream: *mut FILE) - -> *mut c_char; + pub fn fgets(buf: *mut c_char, n: c_int, stream: *mut FILE) -> *mut c_char; pub fn fputc(c: c_int, stream: *mut FILE) -> c_int; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), @@ -444,22 +475,12 @@ extern "C" { pub fn fputs(s: *const c_char, stream: *mut FILE) -> c_int; pub fn puts(s: *const c_char) -> c_int; pub fn ungetc(c: c_int, stream: *mut FILE) -> c_int; - pub fn fread( - ptr: *mut c_void, - size: size_t, - nobj: size_t, - stream: *mut FILE, - ) -> size_t; + pub fn fread(ptr: *mut c_void, size: size_t, nobj: size_t, stream: *mut FILE) -> size_t; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "fwrite$UNIX2003" )] - pub fn fwrite( - ptr: *const c_void, - size: size_t, - nobj: size_t, - stream: *mut FILE, - ) -> size_t; + pub fn fwrite(ptr: *const c_void, size: size_t, nobj: size_t, stream: *mut FILE) -> size_t; pub fn fseek(stream: *mut FILE, offset: c_long, whence: c_int) -> c_int; pub fn ftell(stream: *mut FILE) -> c_long; pub fn rewind(stream: *mut FILE); @@ -469,6 +490,7 @@ extern "C" { pub fn fsetpos(stream: *mut FILE, ptr: *const fpos_t) -> c_int; pub fn feof(stream: *mut FILE) -> c_int; pub fn ferror(stream: *mut FILE) -> c_int; + pub fn clearerr(stream: *mut FILE); pub fn perror(s: *const c_char); pub fn atoi(s: *const c_char) -> c_int; #[cfg_attr( @@ -476,16 +498,9 @@ extern "C" { link_name = "strtod$UNIX2003" )] pub fn strtod(s: *const c_char, endp: *mut *mut c_char) -> c_double; - pub fn strtol( - s: *const c_char, - endp: *mut *mut c_char, - base: c_int, - ) -> c_long; - pub fn strtoul( - s: *const c_char, - endp: *mut *mut c_char, - base: c_int, - ) -> c_ulong; + pub fn strtof(s: *const c_char, endp: *mut *mut c_char) -> c_float; + pub fn strtol(s: *const c_char, endp: *mut *mut c_char, base: c_int) -> c_long; + pub fn strtoul(s: *const c_char, endp: *mut *mut c_char, base: c_int) -> c_ulong; pub fn calloc(nobj: size_t, size: size_t) -> *mut c_void; pub fn malloc(size: size_t) -> *mut c_void; pub fn realloc(p: *mut c_void, size: size_t) -> *mut c_void; @@ -502,17 +517,10 @@ extern "C" { pub fn getenv(s: *const c_char) -> *mut c_char; pub fn strcpy(dst: *mut c_char, src: *const c_char) -> *mut c_char; - pub fn strncpy( - dst: *mut c_char, - src: *const c_char, - n: size_t, - ) -> *mut c_char; + pub fn strncpy(dst: *mut c_char, src: *const c_char, n: size_t) -> *mut c_char; + pub fn stpcpy(dst: *mut c_char, src: *const c_char) -> *mut c_char; pub fn strcat(s: *mut c_char, ct: *const c_char) -> *mut c_char; - pub fn strncat( - s: *mut c_char, - ct: *const c_char, - n: size_t, - ) -> *mut c_char; + pub fn strncat(s: *mut c_char, ct: *const c_char, n: size_t) -> *mut c_char; pub fn strcmp(cs: *const c_char, ct: *const c_char) -> c_int; pub fn strncmp(cs: *const c_char, ct: *const c_char, n: size_t) -> c_int; pub fn strcoll(cs: *const c_char, ct: *const c_char) -> c_int; @@ -521,14 +529,11 @@ extern "C" { pub fn strspn(cs: *const c_char, ct: *const c_char) -> size_t; pub fn strcspn(cs: *const c_char, ct: *const c_char) -> size_t; pub fn strdup(cs: *const c_char) -> *mut c_char; + pub fn strndup(cs: *const c_char, n: size_t) -> *mut c_char; pub fn strpbrk(cs: *const c_char, ct: *const c_char) -> *mut c_char; pub fn strstr(cs: *const c_char, ct: *const c_char) -> *mut c_char; pub fn strcasecmp(s1: *const c_char, s2: *const c_char) -> c_int; - pub fn strncasecmp( - s1: *const c_char, - s2: *const c_char, - n: size_t, - ) -> c_int; + pub fn strncasecmp(s1: *const c_char, s2: *const c_char, n: size_t) -> c_int; pub fn strlen(cs: *const c_char) -> size_t; pub fn strnlen(cs: *const c_char, maxlen: size_t) -> size_t; #[cfg_attr( @@ -537,27 +542,17 @@ extern "C" { )] pub fn strerror(n: c_int) -> *mut c_char; pub fn strtok(s: *mut c_char, t: *const c_char) -> *mut c_char; + pub fn strtok_r(s: *mut c_char, t: *const c_char, p: *mut *mut c_char) -> *mut c_char; pub fn strxfrm(s: *mut c_char, ct: *const c_char, n: size_t) -> size_t; + pub fn strsignal(sig: c_int) -> *mut c_char; pub fn wcslen(buf: *const wchar_t) -> size_t; - pub fn wcstombs( - dest: *mut c_char, - src: *const wchar_t, - n: size_t, - ) -> ::size_t; + pub fn wcstombs(dest: *mut c_char, src: *const wchar_t, n: size_t) -> ::size_t; pub fn memchr(cx: *const c_void, c: c_int, n: size_t) -> *mut c_void; pub fn wmemchr(cx: *const wchar_t, c: wchar_t, n: size_t) -> *mut wchar_t; pub fn memcmp(cx: *const c_void, ct: *const c_void, n: size_t) -> c_int; - pub fn memcpy( - dest: *mut c_void, - src: *const c_void, - n: size_t, - ) -> *mut c_void; - pub fn memmove( - dest: *mut c_void, - src: *const c_void, - n: size_t, - ) -> *mut c_void; + pub fn memcpy(dest: *mut c_void, src: *const c_void, n: size_t) -> *mut c_void; + pub fn memmove(dest: *mut c_void, src: *const c_void, n: size_t) -> *mut c_void; pub fn memset(dest: *mut c_void, c: c_int, n: size_t) -> *mut c_void; } @@ -567,79 +562,97 @@ extern "C" { #[cfg_attr(target_os = "netbsd", link_name = "__getpwuid50")] pub fn getpwuid(uid: ::uid_t) -> *mut passwd; - pub fn fprintf( - stream: *mut ::FILE, - format: *const ::c_char, - ... - ) -> ::c_int; + pub fn fprintf(stream: *mut ::FILE, format: *const ::c_char, ...) -> ::c_int; pub fn printf(format: *const ::c_char, ...) -> ::c_int; - pub fn snprintf( - s: *mut ::c_char, - n: ::size_t, - format: *const ::c_char, - ... - ) -> ::c_int; + pub fn snprintf(s: *mut ::c_char, n: ::size_t, format: *const ::c_char, ...) -> ::c_int; pub fn sprintf(s: *mut ::c_char, format: *const ::c_char, ...) -> ::c_int; - #[cfg_attr(target_os = "linux", link_name = "__isoc99_fscanf")] - pub fn fscanf( - stream: *mut ::FILE, - format: *const ::c_char, - ... - ) -> ::c_int; - #[cfg_attr(target_os = "linux", link_name = "__isoc99_scanf")] + #[cfg_attr( + all(target_os = "linux", not(target_env = "uclibc")), + link_name = "__isoc99_fscanf" + )] + pub fn fscanf(stream: *mut ::FILE, format: *const ::c_char, ...) -> ::c_int; + #[cfg_attr( + all(target_os = "linux", not(target_env = "uclibc")), + link_name = "__isoc99_scanf" + )] pub fn scanf(format: *const ::c_char, ...) -> ::c_int; - #[cfg_attr(target_os = "linux", link_name = "__isoc99_sscanf")] - pub fn sscanf(s: *const ::c_char, format: *const ::c_char, ...) - -> ::c_int; + #[cfg_attr( + all(target_os = "linux", not(target_env = "uclibc")), + link_name = "__isoc99_sscanf" + )] + pub fn sscanf(s: *const ::c_char, format: *const ::c_char, ...) -> ::c_int; pub fn getchar_unlocked() -> ::c_int; pub fn putchar_unlocked(c: ::c_int) -> ::c_int; + #[cfg(not(all( + libc_cfg_target_vendor, + target_arch = "powerpc", + target_vendor = "nintendo" + )))] #[cfg_attr(target_os = "netbsd", link_name = "__socket30")] #[cfg_attr(target_os = "illumos", link_name = "__xnet_socket")] + #[cfg_attr(target_os = "espidf", link_name = "lwip_socket")] pub fn socket(domain: ::c_int, ty: ::c_int, protocol: ::c_int) -> ::c_int; + #[cfg(not(all( + libc_cfg_target_vendor, + target_arch = "powerpc", + target_vendor = "nintendo" + )))] #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "connect$UNIX2003" )] #[cfg_attr(target_os = "illumos", link_name = "__xnet_connect")] - pub fn connect( - socket: ::c_int, - address: *const sockaddr, - len: socklen_t, - ) -> ::c_int; + #[cfg_attr(target_os = "espidf", link_name = "lwip_connect")] + pub fn connect(socket: ::c_int, address: *const sockaddr, len: socklen_t) -> ::c_int; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "listen$UNIX2003" )] - #[cfg_attr(target_os = "illumos", link_name = "__xnet_listen")] + #[cfg_attr(target_os = "espidf", link_name = "lwip_listen")] pub fn listen(socket: ::c_int, backlog: ::c_int) -> ::c_int; + #[cfg(not(all( + libc_cfg_target_vendor, + target_arch = "powerpc", + target_vendor = "nintendo" + )))] #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "accept$UNIX2003" )] - pub fn accept( - socket: ::c_int, - address: *mut sockaddr, - address_len: *mut socklen_t, - ) -> ::c_int; + #[cfg_attr(target_os = "espidf", link_name = "lwip_accept")] + pub fn accept(socket: ::c_int, address: *mut sockaddr, address_len: *mut socklen_t) -> ::c_int; + #[cfg(not(all( + libc_cfg_target_vendor, + target_arch = "powerpc", + target_vendor = "nintendo" + )))] #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "getpeername$UNIX2003" )] + #[cfg_attr(target_os = "espidf", link_name = "lwip_getpeername")] pub fn getpeername( socket: ::c_int, address: *mut sockaddr, address_len: *mut socklen_t, ) -> ::c_int; + #[cfg(not(all( + libc_cfg_target_vendor, + target_arch = "powerpc", + target_vendor = "nintendo" + )))] #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "getsockname$UNIX2003" )] + #[cfg_attr(target_os = "espidf", link_name = "lwip_getsockname")] pub fn getsockname( socket: ::c_int, address: *mut sockaddr, address_len: *mut socklen_t, ) -> ::c_int; + #[cfg_attr(target_os = "espidf", link_name = "lwip_setsockopt")] pub fn setsockopt( socket: ::c_int, level: ::c_int, @@ -658,11 +671,17 @@ extern "C" { protocol: ::c_int, socket_vector: *mut ::c_int, ) -> ::c_int; + #[cfg(not(all( + libc_cfg_target_vendor, + target_arch = "powerpc", + target_vendor = "nintendo" + )))] #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "sendto$UNIX2003" )] #[cfg_attr(target_os = "illumos", link_name = "__xnet_sendto")] + #[cfg_attr(target_os = "espidf", link_name = "lwip_sendto")] pub fn sendto( socket: ::c_int, buf: *const ::c_void, @@ -671,6 +690,7 @@ extern "C" { addr: *const sockaddr, addrlen: socklen_t, ) -> ::ssize_t; + #[cfg_attr(target_os = "espidf", link_name = "lwip_shutdown")] pub fn shutdown(socket: ::c_int, how: ::c_int) -> ::c_int; #[cfg_attr( @@ -684,7 +704,10 @@ extern "C" { )] pub fn fchmod(fd: ::c_int, mode: mode_t) -> ::c_int; - #[cfg_attr(target_os = "macos", link_name = "fstat$INODE64")] + #[cfg_attr( + all(target_os = "macos", not(target_arch = "aarch64")), + link_name = "fstat$INODE64" + )] #[cfg_attr(target_os = "netbsd", link_name = "__fstat50")] #[cfg_attr( all(target_os = "freebsd", any(freebsd11, freebsd10)), @@ -694,7 +717,10 @@ extern "C" { pub fn mkdir(path: *const c_char, mode: mode_t) -> ::c_int; - #[cfg_attr(target_os = "macos", link_name = "stat$INODE64")] + #[cfg_attr( + all(target_os = "macos", not(target_arch = "aarch64")), + link_name = "stat$INODE64" + )] #[cfg_attr(target_os = "netbsd", link_name = "__stat50")] #[cfg_attr( all(target_os = "freebsd", any(freebsd11, freebsd10)), @@ -737,7 +763,10 @@ extern "C" { #[cfg_attr(target_os = "netbsd", link_name = "__opendir30")] pub fn opendir(dirname: *const c_char) -> *mut ::DIR; - #[cfg_attr(target_os = "macos", link_name = "readdir$INODE64")] + #[cfg_attr( + all(target_os = "macos", not(target_arch = "aarch64")), + link_name = "readdir$INODE64" + )] #[cfg_attr(target_os = "netbsd", link_name = "__readdir30")] #[cfg_attr( all(target_os = "freebsd", any(freebsd11, freebsd10)), @@ -773,7 +802,10 @@ extern "C" { group: ::gid_t, flags: ::c_int, ) -> ::c_int; - #[cfg_attr(target_os = "macos", link_name = "fstatat$INODE64")] + #[cfg_attr( + all(target_os = "macos", not(target_arch = "aarch64")), + link_name = "fstatat$INODE64" + )] #[cfg_attr( all(target_os = "freebsd", any(freebsd11, freebsd10)), link_name = "fstatat@FBSD_1.1" @@ -802,11 +834,7 @@ extern "C" { newdirfd: ::c_int, linkpath: *const ::c_char, ) -> ::c_int; - pub fn unlinkat( - dirfd: ::c_int, - pathname: *const ::c_char, - flags: ::c_int, - ) -> ::c_int; + pub fn unlinkat(dirfd: ::c_int, pathname: *const ::c_char, flags: ::c_int) -> ::c_int; pub fn access(path: *const c_char, amode: ::c_int) -> ::c_int; pub fn alarm(seconds: ::c_uint) -> ::c_uint; @@ -830,16 +858,8 @@ extern "C" { pub fn dup(fd: ::c_int) -> ::c_int; pub fn dup2(src: ::c_int, dst: ::c_int) -> ::c_int; pub fn execl(path: *const c_char, arg0: *const c_char, ...) -> ::c_int; - pub fn execle( - path: *const ::c_char, - arg0: *const ::c_char, - ... - ) -> ::c_int; - pub fn execlp( - file: *const ::c_char, - arg0: *const ::c_char, - ... - ) -> ::c_int; + pub fn execle(path: *const ::c_char, arg0: *const ::c_char, ...) -> ::c_int; + pub fn execlp(file: *const ::c_char, arg0: *const ::c_char, ...) -> ::c_int; pub fn execv(prog: *const c_char, argv: *const *const c_char) -> ::c_int; pub fn execve( prog: *const c_char, @@ -854,16 +874,13 @@ extern "C" { pub fn geteuid() -> uid_t; pub fn getgid() -> gid_t; pub fn getgroups(ngroups_max: ::c_int, groups: *mut gid_t) -> ::c_int; + #[cfg_attr(target_os = "illumos", link_name = "getloginx")] pub fn getlogin() -> *mut c_char; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "getopt$UNIX2003" )] - pub fn getopt( - argc: ::c_int, - argv: *const *mut c_char, - optstr: *const c_char, - ) -> ::c_int; + pub fn getopt(argc: ::c_int, argv: *const *mut c_char, optstr: *const c_char) -> ::c_int; pub fn getpgid(pid: pid_t) -> pid_t; pub fn getpgrp() -> pid_t; pub fn getpid() -> pid_t; @@ -874,17 +891,12 @@ extern "C" { pub fn lseek(fd: ::c_int, offset: off_t, whence: ::c_int) -> off_t; pub fn pathconf(path: *const c_char, name: ::c_int) -> c_long; pub fn pipe(fds: *mut ::c_int) -> ::c_int; - pub fn posix_memalign( - memptr: *mut *mut ::c_void, - align: ::size_t, - size: ::size_t, - ) -> ::c_int; + pub fn posix_memalign(memptr: *mut *mut ::c_void, align: ::size_t, size: ::size_t) -> ::c_int; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "read$UNIX2003" )] - pub fn read(fd: ::c_int, buf: *mut ::c_void, count: ::size_t) - -> ::ssize_t; + pub fn read(fd: ::c_int, buf: *mut ::c_void, count: ::size_t) -> ::ssize_t; pub fn rmdir(path: *const c_char) -> ::c_int; pub fn seteuid(uid: uid_t) -> ::c_int; pub fn setegid(gid: gid_t) -> ::c_int; @@ -892,6 +904,8 @@ extern "C" { pub fn setpgid(pid: pid_t, pgid: pid_t) -> ::c_int; pub fn setsid() -> pid_t; pub fn setuid(uid: uid_t) -> ::c_int; + pub fn setreuid(ruid: uid_t, euid: uid_t) -> ::c_int; + pub fn setregid(rgid: gid_t, egid: gid_t) -> ::c_int; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "sleep$UNIX2003" @@ -910,11 +924,8 @@ extern "C" { all(target_os = "macos", target_arch = "x86"), link_name = "ttyname_r$UNIX2003" )] - pub fn ttyname_r( - fd: ::c_int, - buf: *mut c_char, - buflen: ::size_t, - ) -> ::c_int; + #[cfg_attr(target_os = "illumos", link_name = "__posix_ttyname_r")] + pub fn ttyname_r(fd: ::c_int, buf: *mut c_char, buflen: ::size_t) -> ::c_int; pub fn unlink(c: *const c_char) -> ::c_int; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), @@ -925,40 +936,22 @@ extern "C" { all(target_os = "macos", target_arch = "x86"), link_name = "waitpid$UNIX2003" )] - pub fn waitpid( - pid: pid_t, - status: *mut ::c_int, - options: ::c_int, - ) -> pid_t; + pub fn waitpid(pid: pid_t, status: *mut ::c_int, options: ::c_int) -> pid_t; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "write$UNIX2003" )] - pub fn write( - fd: ::c_int, - buf: *const ::c_void, - count: ::size_t, - ) -> ::ssize_t; + pub fn write(fd: ::c_int, buf: *const ::c_void, count: ::size_t) -> ::ssize_t; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "pread$UNIX2003" )] - pub fn pread( - fd: ::c_int, - buf: *mut ::c_void, - count: ::size_t, - offset: off_t, - ) -> ::ssize_t; + pub fn pread(fd: ::c_int, buf: *mut ::c_void, count: ::size_t, offset: off_t) -> ::ssize_t; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "pwrite$UNIX2003" )] - pub fn pwrite( - fd: ::c_int, - buf: *const ::c_void, - count: ::size_t, - offset: off_t, - ) -> ::ssize_t; + pub fn pwrite(fd: ::c_int, buf: *const ::c_void, count: ::size_t, offset: off_t) -> ::ssize_t; pub fn umask(mask: mode_t) -> mode_t; #[cfg_attr(target_os = "netbsd", link_name = "__utime50")] @@ -999,12 +992,12 @@ extern "C" { pub fn munmap(addr: *mut ::c_void, len: ::size_t) -> ::c_int; pub fn if_nametoindex(ifname: *const c_char) -> ::c_uint; - pub fn if_indextoname( - ifindex: ::c_uint, - ifname: *mut ::c_char, - ) -> *mut ::c_char; + pub fn if_indextoname(ifindex: ::c_uint, ifname: *mut ::c_char) -> *mut ::c_char; - #[cfg_attr(target_os = "macos", link_name = "lstat$INODE64")] + #[cfg_attr( + all(target_os = "macos", not(target_arch = "aarch64")), + link_name = "lstat$INODE64" + )] #[cfg_attr(target_os = "netbsd", link_name = "__lstat50")] #[cfg_attr( all(target_os = "freebsd", any(freebsd11, freebsd10)), @@ -1022,11 +1015,7 @@ extern "C" { all(target_os = "macos", target_arch = "x86"), link_name = "setenv$UNIX2003" )] - pub fn setenv( - name: *const c_char, - val: *const c_char, - overwrite: ::c_int, - ) -> ::c_int; + pub fn setenv(name: *const c_char, val: *const c_char, overwrite: ::c_int) -> ::c_int; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "unsetenv$UNIX2003" @@ -1044,13 +1033,15 @@ extern "C" { pub fn getrusage(resource: ::c_int, usage: *mut rusage) -> ::c_int; #[cfg_attr( - any(target_os = "macos", target_os = "ios"), + any( + target_os = "macos", + target_os = "ios", + target_os = "tvos", + target_os = "watchos" + ), link_name = "realpath$DARWIN_EXTSN" )] - pub fn realpath( - pathname: *const ::c_char, - resolved: *mut ::c_char, - ) -> *mut ::c_char; + pub fn realpath(pathname: *const ::c_char, resolved: *mut ::c_char) -> *mut ::c_char; pub fn flock(fd: ::c_int, operation: ::c_int) -> ::c_int; @@ -1062,21 +1053,12 @@ extern "C" { all(target_os = "macos", target_arch = "x86"), link_name = "pthread_join$UNIX2003" )] - pub fn pthread_join( - native: ::pthread_t, - value: *mut *mut ::c_void, - ) -> ::c_int; + pub fn pthread_join(native: ::pthread_t, value: *mut *mut ::c_void) -> ::c_int; pub fn pthread_exit(value: *mut ::c_void) -> !; pub fn pthread_attr_init(attr: *mut ::pthread_attr_t) -> ::c_int; pub fn pthread_attr_destroy(attr: *mut ::pthread_attr_t) -> ::c_int; - pub fn pthread_attr_setstacksize( - attr: *mut ::pthread_attr_t, - stack_size: ::size_t, - ) -> ::c_int; - pub fn pthread_attr_setdetachstate( - attr: *mut ::pthread_attr_t, - state: ::c_int, - ) -> ::c_int; + pub fn pthread_attr_setstacksize(attr: *mut ::pthread_attr_t, stack_size: ::size_t) -> ::c_int; + pub fn pthread_attr_setdetachstate(attr: *mut ::pthread_attr_t, state: ::c_int) -> ::c_int; pub fn pthread_detach(thread: ::pthread_t) -> ::c_int; #[cfg_attr(target_os = "netbsd", link_name = "__libc_thr_yield")] pub fn sched_yield() -> ::c_int; @@ -1086,10 +1068,7 @@ extern "C" { ) -> ::c_int; pub fn pthread_key_delete(key: pthread_key_t) -> ::c_int; pub fn pthread_getspecific(key: pthread_key_t) -> *mut ::c_void; - pub fn pthread_setspecific( - key: pthread_key_t, - value: *const ::c_void, - ) -> ::c_int; + pub fn pthread_setspecific(key: pthread_key_t, value: *const ::c_void) -> ::c_int; pub fn pthread_mutex_init( lock: *mut pthread_mutex_t, attr: *const pthread_mutexattr_t, @@ -1104,30 +1083,20 @@ extern "C" { all(target_os = "macos", target_arch = "x86"), link_name = "pthread_mutexattr_destroy$UNIX2003" )] - pub fn pthread_mutexattr_destroy( - attr: *mut pthread_mutexattr_t, - ) -> ::c_int; - pub fn pthread_mutexattr_settype( - attr: *mut pthread_mutexattr_t, - _type: ::c_int, - ) -> ::c_int; + pub fn pthread_mutexattr_destroy(attr: *mut pthread_mutexattr_t) -> ::c_int; + pub fn pthread_mutexattr_settype(attr: *mut pthread_mutexattr_t, _type: ::c_int) -> ::c_int; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "pthread_cond_init$UNIX2003" )] - pub fn pthread_cond_init( - cond: *mut pthread_cond_t, - attr: *const pthread_condattr_t, - ) -> ::c_int; + pub fn pthread_cond_init(cond: *mut pthread_cond_t, attr: *const pthread_condattr_t) + -> ::c_int; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "pthread_cond_wait$UNIX2003" )] - pub fn pthread_cond_wait( - cond: *mut pthread_cond_t, - lock: *mut pthread_mutex_t, - ) -> ::c_int; + pub fn pthread_cond_wait(cond: *mut pthread_cond_t, lock: *mut pthread_mutex_t) -> ::c_int; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "pthread_cond_timedwait$UNIX2003" @@ -1180,13 +1149,11 @@ extern "C" { link_name = "pthread_rwlock_unlock$UNIX2003" )] pub fn pthread_rwlock_unlock(lock: *mut pthread_rwlock_t) -> ::c_int; - pub fn pthread_rwlockattr_init(attr: *mut pthread_rwlockattr_t) - -> ::c_int; - pub fn pthread_rwlockattr_destroy( - attr: *mut pthread_rwlockattr_t, - ) -> ::c_int; + pub fn pthread_rwlockattr_init(attr: *mut pthread_rwlockattr_t) -> ::c_int; + pub fn pthread_rwlockattr_destroy(attr: *mut pthread_rwlockattr_t) -> ::c_int; #[cfg_attr(target_os = "illumos", link_name = "__xnet_getsockopt")] + #[cfg_attr(target_os = "espidf", link_name = "lwip_getsockopt")] pub fn getsockopt( sockfd: ::c_int, level: ::c_int, @@ -1196,33 +1163,37 @@ extern "C" { ) -> ::c_int; pub fn raise(signum: ::c_int) -> ::c_int; #[cfg_attr(target_os = "netbsd", link_name = "__sigaction14")] - pub fn sigaction( - signum: ::c_int, - act: *const sigaction, - oldact: *mut sigaction, - ) -> ::c_int; + pub fn sigaction(signum: ::c_int, act: *const sigaction, oldact: *mut sigaction) -> ::c_int; #[cfg_attr(target_os = "netbsd", link_name = "__utimes50")] - pub fn utimes( - filename: *const ::c_char, - times: *const ::timeval, - ) -> ::c_int; + pub fn utimes(filename: *const ::c_char, times: *const ::timeval) -> ::c_int; pub fn dlopen(filename: *const ::c_char, flag: ::c_int) -> *mut ::c_void; pub fn dlerror() -> *mut ::c_char; - pub fn dlsym( - handle: *mut ::c_void, - symbol: *const ::c_char, - ) -> *mut ::c_void; + pub fn dlsym(handle: *mut ::c_void, symbol: *const ::c_char) -> *mut ::c_void; pub fn dlclose(handle: *mut ::c_void) -> ::c_int; pub fn dladdr(addr: *const ::c_void, info: *mut Dl_info) -> ::c_int; + #[cfg(not(all( + libc_cfg_target_vendor, + target_arch = "powerpc", + target_vendor = "nintendo" + )))] + #[cfg_attr(target_os = "illumos", link_name = "__xnet_getaddrinfo")] + #[cfg_attr(target_os = "espidf", link_name = "lwip_getaddrinfo")] pub fn getaddrinfo( node: *const c_char, service: *const c_char, hints: *const addrinfo, res: *mut *mut addrinfo, ) -> ::c_int; + #[cfg(not(all( + libc_cfg_target_vendor, + target_arch = "powerpc", + target_vendor = "nintendo" + )))] + #[cfg_attr(target_os = "espidf", link_name = "lwip_freeaddrinfo")] pub fn freeaddrinfo(res: *mut addrinfo); + pub fn hstrerror(errcode: ::c_int) -> *const ::c_char; pub fn gai_strerror(errcode: ::c_int) -> *const ::c_char; #[cfg_attr( any( @@ -1234,45 +1205,65 @@ extern "C" { link_name = "__res_init" )] #[cfg_attr( - any(target_os = "macos", target_os = "ios"), + any( + target_os = "macos", + target_os = "ios", + target_os = "tvos", + target_os = "watchos" + ), link_name = "res_9_init" )] pub fn res_init() -> ::c_int; #[cfg_attr(target_os = "netbsd", link_name = "__gmtime_r50")] + #[cfg_attr(target_env = "musl", allow(deprecated))] + // FIXME: for `time_t` pub fn gmtime_r(time_p: *const time_t, result: *mut tm) -> *mut tm; #[cfg_attr(target_os = "netbsd", link_name = "__localtime_r50")] + #[cfg_attr(target_env = "musl", allow(deprecated))] + // FIXME: for `time_t` pub fn localtime_r(time_p: *const time_t, result: *mut tm) -> *mut tm; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "mktime$UNIX2003" )] #[cfg_attr(target_os = "netbsd", link_name = "__mktime50")] + #[cfg_attr(target_env = "musl", allow(deprecated))] + // FIXME: for `time_t` pub fn mktime(tm: *mut tm) -> time_t; #[cfg_attr(target_os = "netbsd", link_name = "__time50")] + #[cfg_attr(target_env = "musl", allow(deprecated))] + // FIXME: for `time_t` pub fn time(time: *mut time_t) -> time_t; #[cfg_attr(target_os = "netbsd", link_name = "__gmtime50")] + #[cfg_attr(target_env = "musl", allow(deprecated))] + // FIXME: for `time_t` pub fn gmtime(time_p: *const time_t) -> *mut tm; #[cfg_attr(target_os = "netbsd", link_name = "__locatime50")] + #[cfg_attr(target_env = "musl", allow(deprecated))] + // FIXME: for `time_t` pub fn localtime(time_p: *const time_t) -> *mut tm; #[cfg_attr(target_os = "netbsd", link_name = "__difftime50")] + #[cfg_attr(target_env = "musl", allow(deprecated))] + // FIXME: for `time_t` pub fn difftime(time1: time_t, time0: time_t) -> ::c_double; + #[cfg_attr(target_os = "netbsd", link_name = "__timegm50")] + #[cfg_attr(target_env = "musl", allow(deprecated))] + // FIXME: for `time_t` + pub fn timegm(tm: *mut ::tm) -> time_t; #[cfg_attr(target_os = "netbsd", link_name = "__mknod50")] #[cfg_attr( all(target_os = "freebsd", any(freebsd11, freebsd10)), link_name = "mknod@FBSD_1.0" )] - pub fn mknod( - pathname: *const ::c_char, - mode: ::mode_t, - dev: ::dev_t, - ) -> ::c_int; + pub fn mknod(pathname: *const ::c_char, mode: ::mode_t, dev: ::dev_t) -> ::c_int; pub fn gethostname(name: *mut ::c_char, len: ::size_t) -> ::c_int; - pub fn getservbyname( - name: *const ::c_char, - proto: *const ::c_char, - ) -> *mut servent; + pub fn endservent(); + pub fn getservbyname(name: *const ::c_char, proto: *const ::c_char) -> *mut servent; + pub fn getservbyport(port: ::c_int, proto: *const ::c_char) -> *mut servent; + pub fn getservent() -> *mut servent; + pub fn setservent(stayopen: ::c_int); pub fn getprotobyname(name: *const ::c_char) -> *mut protoent; pub fn getprotobynumber(proto: ::c_int) -> *mut protoent; pub fn chroot(name: *const ::c_char) -> ::c_int; @@ -1285,22 +1276,14 @@ extern "C" { all(target_os = "macos", target_arch = "x86"), link_name = "send$UNIX2003" )] - pub fn send( - socket: ::c_int, - buf: *const ::c_void, - len: ::size_t, - flags: ::c_int, - ) -> ::ssize_t; + #[cfg_attr(target_os = "espidf", link_name = "lwip_send")] + pub fn send(socket: ::c_int, buf: *const ::c_void, len: ::size_t, flags: ::c_int) -> ::ssize_t; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "recv$UNIX2003" )] - pub fn recv( - socket: ::c_int, - buf: *mut ::c_void, - len: ::size_t, - flags: ::c_int, - ) -> ::ssize_t; + #[cfg_attr(target_os = "espidf", link_name = "lwip_recv")] + pub fn recv(socket: ::c_int, buf: *mut ::c_void, len: ::size_t, flags: ::c_int) -> ::ssize_t; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), link_name = "putenv$UNIX2003" @@ -1323,16 +1306,13 @@ extern "C" { #[cfg_attr(target_os = "netbsd", link_name = "__select50")] pub fn select( nfds: ::c_int, - readfs: *mut fd_set, + readfds: *mut fd_set, writefds: *mut fd_set, errorfds: *mut fd_set, timeout: *mut timeval, ) -> ::c_int; #[cfg_attr(target_os = "netbsd", link_name = "__setlocale50")] - pub fn setlocale( - category: ::c_int, - locale: *const ::c_char, - ) -> *mut ::c_char; + pub fn setlocale(category: ::c_int, locale: *const ::c_char) -> *mut ::c_char; pub fn localeconv() -> *mut lconv; #[cfg_attr( @@ -1345,11 +1325,7 @@ extern "C" { pub fn statvfs(path: *const c_char, buf: *mut statvfs) -> ::c_int; pub fn fstatvfs(fd: ::c_int, buf: *mut statvfs) -> ::c_int; - pub fn readlink( - path: *const c_char, - buf: *mut c_char, - bufsz: ::size_t, - ) -> ::ssize_t; + pub fn readlink(path: *const c_char, buf: *mut c_char, bufsz: ::size_t) -> ::ssize_t; #[cfg_attr(target_os = "netbsd", link_name = "__sigemptyset14")] pub fn sigemptyset(set: *mut sigset_t) -> ::c_int; @@ -1363,17 +1339,10 @@ extern "C" { pub fn sigismember(set: *const sigset_t, signum: ::c_int) -> ::c_int; #[cfg_attr(target_os = "netbsd", link_name = "__sigprocmask14")] - pub fn sigprocmask( - how: ::c_int, - set: *const sigset_t, - oldset: *mut sigset_t, - ) -> ::c_int; + pub fn sigprocmask(how: ::c_int, set: *const sigset_t, oldset: *mut sigset_t) -> ::c_int; #[cfg_attr(target_os = "netbsd", link_name = "__sigpending14")] pub fn sigpending(set: *mut sigset_t) -> ::c_int; - #[cfg_attr(target_os = "netbsd", link_name = "__timegm50")] - pub fn timegm(tm: *mut ::tm) -> time_t; - pub fn sysconf(name: ::c_int) -> ::c_long; pub fn mkfifo(path: *const c_char, mode: mode_t) -> ::c_int; @@ -1389,17 +1358,13 @@ extern "C" { #[cfg_attr(target_os = "netbsd", link_name = "__pselect50")] pub fn pselect( nfds: ::c_int, - readfs: *mut fd_set, + readfds: *mut fd_set, writefds: *mut fd_set, errorfds: *mut fd_set, timeout: *const timespec, sigmask: *const sigset_t, ) -> ::c_int; - pub fn fseeko( - stream: *mut ::FILE, - offset: ::off_t, - whence: ::c_int, - ) -> ::c_int; + pub fn fseeko(stream: *mut ::FILE, offset: ::off_t, whence: ::c_int) -> ::c_int; pub fn ftello(stream: *mut ::FILE) -> ::off_t; #[cfg_attr( all(target_os = "macos", target_arch = "x86"), @@ -1411,11 +1376,7 @@ extern "C" { pub fn cfsetispeed(termios: *mut ::termios, speed: ::speed_t) -> ::c_int; pub fn cfsetospeed(termios: *mut ::termios, speed: ::speed_t) -> ::c_int; pub fn tcgetattr(fd: ::c_int, termios: *mut ::termios) -> ::c_int; - pub fn tcsetattr( - fd: ::c_int, - optional_actions: ::c_int, - termios: *const ::termios, - ) -> ::c_int; + pub fn tcsetattr(fd: ::c_int, optional_actions: ::c_int, termios: *const ::termios) -> ::c_int; pub fn tcflow(fd: ::c_int, action: ::c_int) -> ::c_int; pub fn tcflush(fd: ::c_int, action: ::c_int) -> ::c_int; pub fn tcgetsid(fd: ::c_int) -> ::pid_t; @@ -1442,11 +1403,31 @@ extern "C" { pub fn unlockpt(fd: ::c_int) -> ::c_int; pub fn strcasestr(cs: *const c_char, ct: *const c_char) -> *mut c_char; - pub fn getline( - lineptr: *mut *mut c_char, - n: *mut size_t, - stream: *mut FILE, - ) -> ssize_t; + pub fn getline(lineptr: *mut *mut c_char, n: *mut size_t, stream: *mut FILE) -> ssize_t; + + pub fn lockf(fd: ::c_int, cmd: ::c_int, len: ::off_t) -> ::c_int; + +} +cfg_if! { + if #[cfg(not(any(target_os = "emscripten", + target_os = "android", + target_os = "haiku")))] { + extern "C" { + pub fn adjtime(delta: *const timeval, olddelta: *mut timeval) -> ::c_int; + pub fn stpncpy(dst: *mut c_char, src: *const c_char, n: size_t) -> *mut c_char; + } + } +} + +cfg_if! { + if #[cfg(not(target_env = "uclibc"))] { + extern "C" { + pub fn open_wmemstream( + ptr: *mut *mut wchar_t, + sizeloc: *mut size_t, + ) -> *mut FILE; + } + } } cfg_if! { @@ -1473,15 +1454,17 @@ cfg_if! { link_name = "fdopendir$INODE64$UNIX2003")] pub fn fdopendir(fd: ::c_int) -> *mut ::DIR; - #[cfg_attr(target_os = "macos", link_name = "readdir_r$INODE64")] + #[cfg_attr(all(target_os = "macos", not(target_arch = "aarch64")), + link_name = "readdir_r$INODE64")] #[cfg_attr(target_os = "netbsd", link_name = "__readdir_r30")] #[cfg_attr( all(target_os = "freebsd", any(freebsd11, freebsd10)), link_name = "readdir_r@FBSD_1.0" )] - /// The 64-bit libc on Solaris and illumos only has readdir_r. If a + #[allow(non_autolinks)] // FIXME: `<>` breaks line length limit. + /// The 64-bit libc on Solaris and illumos only has readdir_r. If a /// 32-bit Solaris or illumos target is ever created, it should use - /// __posix_readdir_r. See libc(3LIB) on Solaris or illumos: + /// __posix_readdir_r. See libc(3LIB) on Solaris or illumos: /// https://illumos.org/man/3lib/libc /// https://docs.oracle.com/cd/E36784_01/html/E36873/libc-3lib.html /// https://www.unix.com/man-page/opensolaris/3LIB/libc/ @@ -1502,19 +1485,19 @@ cfg_if! { } cfg_if! { - if #[cfg(target_env = "uclibc")] { - mod uclibc; - pub use self::uclibc::*; - } else if #[cfg(target_env = "newlib")] { + if #[cfg(target_env = "newlib")] { mod newlib; pub use self::newlib::*; } else if #[cfg(any(target_os = "linux", + target_os = "l4re", target_os = "android", target_os = "emscripten"))] { mod linux_like; pub use self::linux_like::*; } else if #[cfg(any(target_os = "macos", target_os = "ios", + target_os = "tvos", + target_os = "watchos", target_os = "freebsd", target_os = "dragonfly", target_os = "openbsd", diff --git a/crux-mir/lib/libc/src/unix/newlib/aarch64/mod.rs b/crux-mir/lib/libc/src/unix/newlib/aarch64/mod.rs index 7e1b2bb70..d686b3692 100644 --- a/crux-mir/lib/libc/src/unix/newlib/aarch64/mod.rs +++ b/crux-mir/lib/libc/src/unix/newlib/aarch64/mod.rs @@ -1,3 +1,4 @@ +pub type clock_t = ::c_long; pub type c_char = u8; pub type wchar_t = u32; @@ -29,5 +30,25 @@ s! { } } +pub const AF_INET6: ::c_int = 23; + +pub const FIONBIO: ::c_ulong = 1; + +pub const POLLIN: ::c_short = 0x1; +pub const POLLPRI: ::c_short = 0x2; pub const POLLOUT: ::c_short = 0x4; +pub const POLLERR: ::c_short = 0x8; pub const POLLHUP: ::c_short = 0x10; +pub const POLLNVAL: ::c_short = 0x20; + +pub const SOL_SOCKET: ::c_int = 65535; + +pub const MSG_OOB: ::c_int = 1; +pub const MSG_PEEK: ::c_int = 2; +pub const MSG_DONTWAIT: ::c_int = 4; +pub const MSG_DONTROUTE: ::c_int = 0; +pub const MSG_WAITALL: ::c_int = 0; +pub const MSG_MORE: ::c_int = 0; +pub const MSG_NOSIGNAL: ::c_int = 0; + +pub use crate::unix::newlib::generic::{sigset_t, stat}; diff --git a/crux-mir/lib/libc/src/unix/newlib/arm/mod.rs b/crux-mir/lib/libc/src/unix/newlib/arm/mod.rs index 39cb425fe..f644349cb 100644 --- a/crux-mir/lib/libc/src/unix/newlib/arm/mod.rs +++ b/crux-mir/lib/libc/src/unix/newlib/arm/mod.rs @@ -1,3 +1,4 @@ +pub type clock_t = ::c_long; pub type c_char = u8; pub type wchar_t = u32; @@ -31,5 +32,25 @@ s! { } } -pub const POLLOUT: ::c_short = 0x10; +pub const AF_INET6: ::c_int = 23; + +pub const FIONBIO: ::c_ulong = 1; + +pub const POLLIN: ::c_short = 0x1; +pub const POLLPRI: ::c_short = 0x2; pub const POLLHUP: ::c_short = 0x4; +pub const POLLERR: ::c_short = 0x8; +pub const POLLOUT: ::c_short = 0x10; +pub const POLLNVAL: ::c_short = 0x20; + +pub const SOL_SOCKET: ::c_int = 65535; + +pub const MSG_OOB: ::c_int = 1; +pub const MSG_PEEK: ::c_int = 2; +pub const MSG_DONTWAIT: ::c_int = 4; +pub const MSG_DONTROUTE: ::c_int = 0; +pub const MSG_WAITALL: ::c_int = 0; +pub const MSG_MORE: ::c_int = 0; +pub const MSG_NOSIGNAL: ::c_int = 0; + +pub use crate::unix::newlib::generic::{sigset_t, stat}; diff --git a/crux-mir/lib/libc/src/unix/newlib/espidf/mod.rs b/crux-mir/lib/libc/src/unix/newlib/espidf/mod.rs new file mode 100644 index 000000000..804cd6645 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/newlib/espidf/mod.rs @@ -0,0 +1,110 @@ +pub type clock_t = ::c_ulong; +pub type c_char = i8; +pub type wchar_t = u32; + +pub type c_long = i32; +pub type c_ulong = u32; + +s! { + pub struct cmsghdr { + pub cmsg_len: ::socklen_t, + pub cmsg_level: ::c_int, + pub cmsg_type: ::c_int, + } + + pub struct msghdr { + pub msg_name: *mut ::c_void, + pub msg_namelen: ::socklen_t, + pub msg_iov: *mut ::iovec, + pub msg_iovlen: ::c_int, + pub msg_control: *mut ::c_void, + pub msg_controllen: ::socklen_t, + pub msg_flags: ::c_int, + } + + pub struct sockaddr_un { + pub sun_family: ::sa_family_t, + pub sun_path: [::c_char; 108], + } + + pub struct sockaddr { + pub sa_len: u8, + pub sa_family: ::sa_family_t, + pub sa_data: [::c_char; 14], + } + + pub struct sockaddr_in6 { + pub sin6_len: u8, + pub sin6_family: ::sa_family_t, + pub sin6_port: ::in_port_t, + pub sin6_flowinfo: u32, + pub sin6_addr: ::in6_addr, + pub sin6_scope_id: u32, + } + + pub struct sockaddr_in { + pub sin_len: u8, + pub sin_family: ::sa_family_t, + pub sin_port: ::in_port_t, + pub sin_addr: ::in_addr, + pub sin_zero: [::c_char; 8], + } + + pub struct sockaddr_storage { + pub s2_len: u8, + pub ss_family: ::sa_family_t, + pub s2_data1: [::c_char; 2], + pub s2_data2: [u32; 3], + pub s2_data3: [u32; 3], + } +} + +pub const AF_UNIX: ::c_int = 1; +pub const AF_INET6: ::c_int = 10; + +pub const FIONBIO: ::c_ulong = 2147772030; + +pub const POLLIN: ::c_short = 1 << 0; +pub const POLLRDNORM: ::c_short = 1 << 1; +pub const POLLRDBAND: ::c_short = 1 << 2; +pub const POLLPRI: ::c_short = POLLRDBAND; +pub const POLLOUT: ::c_short = 1 << 3; +pub const POLLWRNORM: ::c_short = POLLOUT; +pub const POLLWRBAND: ::c_short = 1 << 4; +pub const POLLERR: ::c_short = 1 << 5; +pub const POLLHUP: ::c_short = 1 << 6; + +pub const SOL_SOCKET: ::c_int = 0xfff; + +pub const MSG_OOB: ::c_int = 0x04; +pub const MSG_PEEK: ::c_int = 0x01; +pub const MSG_DONTWAIT: ::c_int = 0x08; +pub const MSG_DONTROUTE: ::c_int = 0x4; +pub const MSG_WAITALL: ::c_int = 0x02; +pub const MSG_MORE: ::c_int = 0x10; +pub const MSG_NOSIGNAL: ::c_int = 0x20; +pub const MSG_TRUNC: ::c_int = 0x04; +pub const MSG_CTRUNC: ::c_int = 0x08; +pub const MSG_EOR: ::c_int = 0x08; + +pub const PTHREAD_STACK_MIN: ::size_t = 768; + +extern "C" { + pub fn pthread_create( + native: *mut ::pthread_t, + attr: *const ::pthread_attr_t, + f: extern "C" fn(_: *mut ::c_void) -> *mut ::c_void, + value: *mut ::c_void, + ) -> ::c_int; + + pub fn getrandom(buf: *mut ::c_void, buflen: ::size_t, flags: ::c_uint) -> ::ssize_t; + + #[link_name = "lwip_sendmsg"] + pub fn sendmsg(s: ::c_int, msg: *const ::msghdr, flags: ::c_int) -> ::ssize_t; + #[link_name = "lwip_recvmsg"] + pub fn recvmsg(s: ::c_int, msg: *mut ::msghdr, flags: ::c_int) -> ::ssize_t; + + pub fn eventfd(initval: ::c_uint, flags: ::c_int) -> ::c_int; +} + +pub use crate::unix::newlib::generic::{sigset_t, stat}; diff --git a/crux-mir/lib/libc/src/unix/newlib/generic.rs b/crux-mir/lib/libc/src/unix/newlib/generic.rs new file mode 100644 index 000000000..db7797f17 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/newlib/generic.rs @@ -0,0 +1,27 @@ +//! Common types used by most newlib platforms + +s! { + pub struct sigset_t { + __val: [::c_ulong; 16], + } + + pub struct stat { + pub st_dev: ::dev_t, + pub st_ino: ::ino_t, + pub st_mode: ::mode_t, + pub st_nlink: ::nlink_t, + pub st_uid: ::uid_t, + pub st_gid: ::gid_t, + pub st_rdev: ::dev_t, + pub st_size: ::off_t, + pub st_atime: ::time_t, + pub st_spare1: ::c_long, + pub st_mtime: ::time_t, + pub st_spare2: ::c_long, + pub st_ctime: ::time_t, + pub st_spare3: ::c_long, + pub st_blksize: ::blksize_t, + pub st_blocks: ::blkcnt_t, + pub st_spare4: [::c_long; 2usize], + } +} diff --git a/crux-mir/lib/libc/src/unix/newlib/horizon/mod.rs b/crux-mir/lib/libc/src/unix/newlib/horizon/mod.rs new file mode 100644 index 000000000..bcb93ad9d --- /dev/null +++ b/crux-mir/lib/libc/src/unix/newlib/horizon/mod.rs @@ -0,0 +1,268 @@ +//! ARMv6K Nintendo 3DS C Newlib definitions + +pub type c_char = u8; +pub type c_long = i32; +pub type c_ulong = u32; + +pub type wchar_t = ::c_uint; + +pub type u_register_t = ::c_uint; +pub type u_char = ::c_uchar; +pub type u_short = ::c_ushort; +pub type u_int = ::c_uint; +pub type u_long = c_ulong; +pub type ushort = ::c_ushort; +pub type uint = ::c_uint; +pub type ulong = c_ulong; +pub type clock_t = c_ulong; +pub type daddr_t = c_long; +pub type caddr_t = *mut c_char; +pub type sbintime_t = ::c_longlong; +pub type sigset_t = ::c_ulong; + +s! { + pub struct sockaddr { + pub sa_family: ::sa_family_t, + pub sa_data: [::c_char; 26usize], + } + + pub struct sockaddr_storage { + pub ss_family: ::sa_family_t, + pub __ss_padding: [::c_char; 26usize], + } + + pub struct sockaddr_in { + pub sin_family: ::sa_family_t, + pub sin_port: ::in_port_t, + pub sin_addr: ::in_addr, + } + + pub struct sockaddr_in6 { + pub sin6_family: ::sa_family_t, + pub sin6_port: ::in_port_t, + pub sin6_flowinfo: u32, + pub sin6_addr: ::in6_addr, + pub sin6_scope_id: u32, + } + + pub struct sockaddr_un { + pub sun_len: ::c_uchar, + pub sun_family: ::sa_family_t, + pub sun_path: [::c_char; 104usize], + } + + pub struct sched_param { + pub sched_priority: ::c_int, + } + + pub struct stat { + pub st_dev: ::dev_t, + pub st_ino: ::ino_t, + pub st_mode: ::mode_t, + pub st_nlink: ::nlink_t, + pub st_uid: ::uid_t, + pub st_gid: ::gid_t, + pub st_rdev: ::dev_t, + pub st_size: ::off_t, + pub st_atim: ::timespec, + pub st_mtim: ::timespec, + pub st_ctim: ::timespec, + pub st_blksize: ::blksize_t, + pub st_blocks: ::blkcnt_t, + pub st_spare4: [::c_long; 2usize], + } +} + +pub const SIGEV_NONE: ::c_int = 1; +pub const SIGEV_SIGNAL: ::c_int = 2; +pub const SIGEV_THREAD: ::c_int = 3; +pub const SA_NOCLDSTOP: ::c_int = 1; +pub const MINSIGSTKSZ: ::c_int = 2048; +pub const SIGSTKSZ: ::c_int = 8192; +pub const SS_ONSTACK: ::c_int = 1; +pub const SS_DISABLE: ::c_int = 2; +pub const SIG_SETMASK: ::c_int = 0; +pub const SIG_BLOCK: ::c_int = 1; +pub const SIG_UNBLOCK: ::c_int = 2; +pub const SIGHUP: ::c_int = 1; +pub const SIGINT: ::c_int = 2; +pub const SIGQUIT: ::c_int = 3; +pub const SIGILL: ::c_int = 4; +pub const SIGTRAP: ::c_int = 5; +pub const SIGABRT: ::c_int = 6; +pub const SIGEMT: ::c_int = 7; +pub const SIGFPE: ::c_int = 8; +pub const SIGKILL: ::c_int = 9; +pub const SIGBUS: ::c_int = 10; +pub const SIGSEGV: ::c_int = 11; +pub const SIGSYS: ::c_int = 12; +pub const SIGPIPE: ::c_int = 13; +pub const SIGALRM: ::c_int = 14; +pub const SIGTERM: ::c_int = 15; +pub const SIGURG: ::c_int = 16; +pub const SIGSTOP: ::c_int = 17; +pub const SIGTSTP: ::c_int = 18; +pub const SIGCONT: ::c_int = 19; +pub const SIGCHLD: ::c_int = 20; +pub const SIGCLD: ::c_int = 20; +pub const SIGTTIN: ::c_int = 21; +pub const SIGTTOU: ::c_int = 22; +pub const SIGIO: ::c_int = 23; +pub const SIGPOLL: ::c_int = 23; +pub const SIGXCPU: ::c_int = 24; +pub const SIGXFSZ: ::c_int = 25; +pub const SIGVTALRM: ::c_int = 26; +pub const SIGPROF: ::c_int = 27; +pub const SIGWINCH: ::c_int = 28; +pub const SIGLOST: ::c_int = 29; +pub const SIGUSR1: ::c_int = 30; +pub const SIGUSR2: ::c_int = 31; +pub const NSIG: ::c_int = 32; +pub const CLOCK_ENABLED: ::c_uint = 1; +pub const CLOCK_DISABLED: ::c_uint = 0; +pub const CLOCK_ALLOWED: ::c_uint = 1; +pub const CLOCK_DISALLOWED: ::c_uint = 0; +pub const TIMER_ABSTIME: ::c_uint = 4; +pub const SOL_SOCKET: ::c_int = 65535; +pub const MSG_OOB: ::c_int = 1; +pub const MSG_PEEK: ::c_int = 2; +pub const MSG_DONTWAIT: ::c_int = 4; +pub const MSG_DONTROUTE: ::c_int = 0; +pub const MSG_WAITALL: ::c_int = 0; +pub const MSG_MORE: ::c_int = 0; +pub const MSG_NOSIGNAL: ::c_int = 0; +pub const SOL_CONFIG: ::c_uint = 65534; + +pub const _SC_PAGESIZE: ::c_int = 8; +pub const _SC_GETPW_R_SIZE_MAX: ::c_int = 51; + +pub const PTHREAD_STACK_MIN: ::size_t = 4096; +pub const WNOHANG: ::c_int = 1; + +pub const POLLIN: ::c_short = 0x0001; +pub const POLLPRI: ::c_short = 0x0002; +pub const POLLOUT: ::c_short = 0x0004; +pub const POLLRDNORM: ::c_short = 0x0040; +pub const POLLWRNORM: ::c_short = POLLOUT; +pub const POLLRDBAND: ::c_short = 0x0080; +pub const POLLWRBAND: ::c_short = 0x0100; +pub const POLLERR: ::c_short = 0x0008; +pub const POLLHUP: ::c_short = 0x0010; +pub const POLLNVAL: ::c_short = 0x0020; + +pub const EAI_AGAIN: ::c_int = 2; +pub const EAI_BADFLAGS: ::c_int = 3; +pub const EAI_FAIL: ::c_int = 4; +pub const EAI_SERVICE: ::c_int = 9; +pub const EAI_SYSTEM: ::c_int = 11; +pub const EAI_BADHINTS: ::c_int = 12; +pub const EAI_PROTOCOL: ::c_int = 13; +pub const EAI_OVERFLOW: ::c_int = 14; +pub const EAI_MAX: ::c_int = 15; + +pub const AF_UNIX: ::c_int = 1; +pub const AF_INET6: ::c_int = 23; + +pub const FIONBIO: ::c_ulong = 1; + +pub const RTLD_DEFAULT: *mut ::c_void = 0 as *mut ::c_void; + +// For pthread get/setschedparam +pub const SCHED_FIFO: ::c_int = 1; +pub const SCHED_RR: ::c_int = 2; + +// For getrandom() +pub const GRND_NONBLOCK: ::c_uint = 0x1; +pub const GRND_RANDOM: ::c_uint = 0x2; + +// Horizon OS works doesn't or can't hold any of this information +safe_f! { + pub {const} fn WIFSTOPPED(_status: ::c_int) -> bool { + false + } + + pub {const} fn WSTOPSIG(_status: ::c_int) -> ::c_int { + 0 + } + + pub {const} fn WIFCONTINUED(_status: ::c_int) -> bool { + true + } + + pub {const} fn WIFSIGNALED(_status: ::c_int) -> bool { + false + } + + pub {const} fn WTERMSIG(_status: ::c_int) -> ::c_int { + 0 + } + + pub {const} fn WIFEXITED(_status: ::c_int) -> bool { + true + } + + pub {const} fn WEXITSTATUS(_status: ::c_int) -> ::c_int { + 0 + } + + pub {const} fn WCOREDUMP(_status: ::c_int) -> bool { + false + } +} + +extern "C" { + pub fn pthread_create( + native: *mut ::pthread_t, + attr: *const ::pthread_attr_t, + f: extern "C" fn(_: *mut ::c_void) -> *mut ::c_void, + value: *mut ::c_void, + ) -> ::c_int; + + pub fn pthread_attr_getschedparam( + attr: *const ::pthread_attr_t, + param: *mut sched_param, + ) -> ::c_int; + + pub fn pthread_attr_setschedparam( + attr: *mut ::pthread_attr_t, + param: *const sched_param, + ) -> ::c_int; + + pub fn pthread_attr_getprocessorid_np( + attr: *const ::pthread_attr_t, + processor_id: *mut ::c_int, + ) -> ::c_int; + + pub fn pthread_attr_setprocessorid_np( + attr: *mut ::pthread_attr_t, + processor_id: ::c_int, + ) -> ::c_int; + + pub fn pthread_getschedparam( + native: ::pthread_t, + policy: *mut ::c_int, + param: *mut ::sched_param, + ) -> ::c_int; + + pub fn pthread_setschedparam( + native: ::pthread_t, + policy: ::c_int, + param: *const ::sched_param, + ) -> ::c_int; + + pub fn pthread_condattr_getclock( + attr: *const ::pthread_condattr_t, + clock_id: *mut ::clockid_t, + ) -> ::c_int; + + pub fn pthread_condattr_setclock( + attr: *mut ::pthread_condattr_t, + clock_id: ::clockid_t, + ) -> ::c_int; + + pub fn pthread_getprocessorid_np() -> ::c_int; + + pub fn getrandom(buf: *mut ::c_void, buflen: ::size_t, flags: ::c_uint) -> ::ssize_t; + + pub fn gethostid() -> ::c_long; +} diff --git a/crux-mir/lib/libc/src/unix/newlib/mod.rs b/crux-mir/lib/libc/src/unix/newlib/mod.rs index 0900e903f..1a694691a 100644 --- a/crux-mir/lib/libc/src/unix/newlib/mod.rs +++ b/crux-mir/lib/libc/src/unix/newlib/mod.rs @@ -1,38 +1,73 @@ pub type blkcnt_t = i32; pub type blksize_t = i32; -pub type clock_t = i32; pub type clockid_t = ::c_ulong; -pub type dev_t = u32; + +cfg_if! { + if #[cfg(target_os = "espidf")] { + pub type dev_t = ::c_short; + pub type ino_t = ::c_ushort; + pub type off_t = ::c_long; + } else { + pub type dev_t = u32; + pub type ino_t = u32; + pub type off_t = i64; + } +} + pub type fsblkcnt_t = u64; pub type fsfilcnt_t = u32; pub type id_t = u32; -pub type ino_t = u32; pub type key_t = ::c_int; pub type loff_t = ::c_longlong; pub type mode_t = ::c_uint; pub type nfds_t = u32; pub type nlink_t = ::c_ushort; -pub type off_t = i64; pub type pthread_t = ::c_ulong; pub type pthread_key_t = ::c_uint; pub type rlim_t = u32; -pub type sa_family_t = u8; + +cfg_if! { + if #[cfg(target_os = "horizon")] { + pub type sa_family_t = u16; + } else { + pub type sa_family_t = u8; + } +} + pub type socklen_t = u32; pub type speed_t = u32; pub type suseconds_t = i32; pub type tcflag_t = ::c_uint; -pub type time_t = i32; pub type useconds_t = u32; +cfg_if! { + if #[cfg(any(target_os = "horizon", all(target_os = "espidf", espidf_time64)))] { + pub type time_t = ::c_longlong; + } else { + pub type time_t = i32; + } +} + s! { + // The order of the `ai_addr` field in this struct is crucial + // for converting between the Rust and C types. pub struct addrinfo { pub ai_flags: ::c_int, pub ai_family: ::c_int, pub ai_socktype: ::c_int, pub ai_protocol: ::c_int, pub ai_addrlen: socklen_t, + + #[cfg(target_os = "espidf")] + pub ai_addr: *mut sockaddr, + pub ai_canonname: *mut ::c_char, + + #[cfg(not(any( + target_os = "espidf", + all(libc_cfg_target_vendor, target_arch = "powerpc", target_vendor = "nintendo"))))] pub ai_addr: *mut sockaddr, + pub ai_next: *mut addrinfo, } @@ -104,26 +139,6 @@ s! { pub tm_isdst: ::c_int, } - pub struct stat { - pub st_dev: ::dev_t, - pub st_ino: ::ino_t, - pub st_mode: ::mode_t, - pub st_nlink: ::nlink_t, - pub st_uid: ::uid_t, - pub st_gid: ::gid_t, - pub st_rdev: dev_t, - pub st_size: off_t, - pub st_atime: time_t, - pub st_spare1: ::c_long, - pub st_mtime: time_t, - pub st_spare2: ::c_long, - pub st_ctime: time_t, - pub st_spare3: ::c_long, - pub st_blksize: blksize_t, - pub st_blocks: blkcnt_t, - pub st_spare4: [::c_long; 2usize], - } - pub struct statvfs { pub f_bsize: ::c_ulong, pub f_frsize: ::c_ulong, @@ -138,10 +153,6 @@ s! { pub f_namemax: ::c_ulong, } - pub struct sigset_t { - __val: [::c_ulong; 16], - } - pub struct sigaction { pub sa_handler: extern fn(arg1: ::c_int), pub sa_mask: sigset_t, @@ -220,23 +231,37 @@ s! { // unverified constants align_const! { pub const PTHREAD_MUTEX_INITIALIZER: pthread_mutex_t = pthread_mutex_t { - size: [0; __SIZEOF_PTHREAD_MUTEX_T], + size: [__PTHREAD_INITIALIZER_BYTE; __SIZEOF_PTHREAD_MUTEX_T], }; pub const PTHREAD_COND_INITIALIZER: pthread_cond_t = pthread_cond_t { - size: [0; __SIZEOF_PTHREAD_COND_T], + size: [__PTHREAD_INITIALIZER_BYTE; __SIZEOF_PTHREAD_COND_T], }; pub const PTHREAD_RWLOCK_INITIALIZER: pthread_rwlock_t = pthread_rwlock_t { - size: [0; __SIZEOF_PTHREAD_RWLOCK_T], + size: [__PTHREAD_INITIALIZER_BYTE; __SIZEOF_PTHREAD_RWLOCK_T], }; } pub const NCCS: usize = 32; -pub const __SIZEOF_PTHREAD_ATTR_T: usize = 56; -pub const __SIZEOF_PTHREAD_MUTEX_T: usize = 40; -pub const __SIZEOF_PTHREAD_MUTEXATTR_T: usize = 4; -pub const __SIZEOF_PTHREAD_COND_T: usize = 48; -pub const __SIZEOF_PTHREAD_CONDATTR_T: usize = 4; -pub const __SIZEOF_PTHREAD_RWLOCK_T: usize = 56; -pub const __SIZEOF_PTHREAD_RWLOCKATTR_T: usize = 8; +cfg_if! { + if #[cfg(target_os = "espidf")] { + const __PTHREAD_INITIALIZER_BYTE: u8 = 0xff; + pub const __SIZEOF_PTHREAD_ATTR_T: usize = 32; + pub const __SIZEOF_PTHREAD_MUTEX_T: usize = 4; + pub const __SIZEOF_PTHREAD_MUTEXATTR_T: usize = 12; + pub const __SIZEOF_PTHREAD_COND_T: usize = 4; + pub const __SIZEOF_PTHREAD_CONDATTR_T: usize = 8; + pub const __SIZEOF_PTHREAD_RWLOCK_T: usize = 4; + pub const __SIZEOF_PTHREAD_RWLOCKATTR_T: usize = 12; + } else { + const __PTHREAD_INITIALIZER_BYTE: u8 = 0; + pub const __SIZEOF_PTHREAD_ATTR_T: usize = 56; + pub const __SIZEOF_PTHREAD_MUTEX_T: usize = 40; + pub const __SIZEOF_PTHREAD_MUTEXATTR_T: usize = 4; + pub const __SIZEOF_PTHREAD_COND_T: usize = 48; + pub const __SIZEOF_PTHREAD_CONDATTR_T: usize = 4; + pub const __SIZEOF_PTHREAD_RWLOCK_T: usize = 56; + pub const __SIZEOF_PTHREAD_RWLOCKATTR_T: usize = 8; + } +} pub const __SIZEOF_PTHREAD_BARRIER_T: usize = 32; pub const __SIZEOF_PTHREAD_BARRIERATTR_T: usize = 4; pub const __PTHREAD_MUTEX_HAVE_PREV: usize = 1; @@ -244,7 +269,14 @@ pub const __PTHREAD_RWLOCK_INT_FLAGS_SHARED: usize = 1; pub const PTHREAD_MUTEX_NORMAL: ::c_int = 0; pub const PTHREAD_MUTEX_RECURSIVE: ::c_int = 1; pub const PTHREAD_MUTEX_ERRORCHECK: ::c_int = 2; -pub const FD_SETSIZE: usize = 1024; + +cfg_if! { + if #[cfg(target_os = "horizon")] { + pub const FD_SETSIZE: usize = 64; + } else { + pub const FD_SETSIZE: usize = 1024; + } +} // intentionally not public, only used for fd_set const ULONG_SIZE: usize = 32; @@ -364,11 +396,6 @@ pub const O_NONBLOCK: ::c_int = 16384; pub const O_ACCMODE: ::c_int = 3; pub const O_CLOEXEC: ::c_int = 0x80000; -pub const POLLIN: ::c_short = 0x1; -pub const POLLPRI: ::c_short = 0x2; -pub const POLLERR: ::c_short = 0x8; -pub const POLLNVAL: ::c_short = 0x20; - pub const RTLD_LAZY: ::c_int = 0x1; pub const STDIN_FILENO: ::c_int = 0; @@ -379,7 +406,6 @@ pub const SEEK_SET: ::c_int = 0; pub const SEEK_CUR: ::c_int = 1; pub const SEEK_END: ::c_int = 2; -pub const FIONBIO: ::c_ulong = 1; pub const FIOCLEX: ::c_ulong = 0x20006601; pub const FIONCLEX: ::c_ulong = 0x20006602; @@ -406,7 +432,6 @@ pub const S_IROTH: ::mode_t = 4; pub const S_IWOTH: ::mode_t = 2; pub const S_IXOTH: ::mode_t = 1; -pub const SOL_SOCKET: ::c_int = 65535; pub const SOL_TCP: ::c_int = 6; pub const PF_UNSPEC: ::c_int = 0; @@ -415,7 +440,6 @@ pub const PF_INET6: ::c_int = 23; pub const AF_UNSPEC: ::c_int = 0; pub const AF_INET: ::c_int = 2; -pub const AF_INET6: ::c_int = 23; pub const CLOCK_REALTIME: ::clockid_t = 1; pub const CLOCK_MONOTONIC: ::clockid_t = 4; @@ -424,14 +448,6 @@ pub const CLOCK_BOOTTIME: ::clockid_t = 4; pub const SOCK_STREAM: ::c_int = 1; pub const SOCK_DGRAM: ::c_int = 2; -pub const MSG_OOB: ::c_int = 1; -pub const MSG_PEEK: ::c_int = 2; -pub const MSG_DONTWAIT: ::c_int = 4; -pub const MSG_DONTROUTE: ::c_int = 0; -pub const MSG_WAITALL: ::c_int = 0; -pub const MSG_MORE: ::c_int = 0; -pub const MSG_NOSIGNAL: ::c_int = 0; - pub const SHUT_RD: ::c_int = 0; pub const SHUT_WR: ::c_int = 1; pub const SHUT_RDWR: ::c_int = 2; @@ -469,7 +485,13 @@ pub const SO_SNDLOWAT: ::c_int = 0x1003; pub const SO_RCVLOWAT: ::c_int = 0x1004; pub const SO_SNDTIMEO: ::c_int = 0x1005; pub const SO_RCVTIMEO: ::c_int = 0x1006; -pub const SO_ERROR: ::c_int = 0x1007; +cfg_if! { + if #[cfg(target_os = "horizon")] { + pub const SO_ERROR: ::c_int = 0x1009; + } else { + pub const SO_ERROR: ::c_int = 0x1007; + } +} pub const SO_TYPE: ::c_int = 0x1008; pub const SOCK_CLOEXEC: ::c_int = O_CLOEXEC; @@ -504,7 +526,13 @@ pub const TCP_KEEPIDLE: ::c_int = 256; pub const TCP_KEEPINTVL: ::c_int = 512; pub const TCP_KEEPCNT: ::c_int = 1024; -pub const IP_TOS: ::c_int = 3; +cfg_if! { + if #[cfg(target_os = "horizon")] { + pub const IP_TOS: ::c_int = 7; + } else { + pub const IP_TOS: ::c_int = 3; + } +} pub const IP_TTL: ::c_int = 8; pub const IP_MULTICAST_IF: ::c_int = 9; pub const IP_MULTICAST_TTL: ::c_int = 10; @@ -547,6 +575,13 @@ pub const EAI_MEMORY: ::c_int = -304; pub const EAI_NONAME: ::c_int = -305; pub const EAI_SOCKTYPE: ::c_int = -307; +pub const EXIT_SUCCESS: ::c_int = 0; +pub const EXIT_FAILURE: ::c_int = 1; + +pub const PRIO_PROCESS: ::c_int = 0; +pub const PRIO_PGRP: ::c_int = 1; +pub const PRIO_USER: ::c_int = 2; + f! { pub fn FD_CLR(fd: ::c_int, set: *mut fd_set) -> () { let bits = ::mem::size_of_val(&(*set).fds_bits[0]) * 8; @@ -555,7 +590,7 @@ f! { return } - pub fn FD_ISSET(fd: ::c_int, set: *mut fd_set) -> bool { + pub fn FD_ISSET(fd: ::c_int, set: *const fd_set) -> bool { let bits = ::mem::size_of_val(&(*set).fds_bits[0]) * 8; let fd = fd as usize; return ((*set).fds_bits[fd / bits] & (1 << (fd % bits))) != 0 @@ -580,18 +615,10 @@ extern "C" { pub fn setrlimit(resource: ::c_int, rlim: *const ::rlimit) -> ::c_int; #[cfg_attr(target_os = "linux", link_name = "__xpg_strerror_r")] - pub fn strerror_r( - errnum: ::c_int, - buf: *mut c_char, - buflen: ::size_t, - ) -> ::c_int; + pub fn strerror_r(errnum: ::c_int, buf: *mut c_char, buflen: ::size_t) -> ::c_int; pub fn sem_destroy(sem: *mut sem_t) -> ::c_int; - pub fn sem_init( - sem: *mut sem_t, - pshared: ::c_int, - value: ::c_uint, - ) -> ::c_int; + pub fn sem_init(sem: *mut sem_t, pshared: ::c_int, value: ::c_uint) -> ::c_int; pub fn abs(i: ::c_int) -> ::c_int; pub fn atof(s: *const ::c_char) -> ::c_double; @@ -599,22 +626,25 @@ extern "C" { pub fn rand() -> ::c_int; pub fn srand(seed: ::c_uint); - pub fn bind(fd: ::c_int, addr: *const sockaddr, len: socklen_t) - -> ::c_int; - pub fn clock_settime( - clock_id: ::clockid_t, - tp: *const ::timespec, - ) -> ::c_int; - pub fn clock_gettime( - clock_id: ::clockid_t, - tp: *mut ::timespec, - ) -> ::c_int; - pub fn clock_getres( - clock_id: ::clockid_t, - res: *mut ::timespec, - ) -> ::c_int; + #[cfg(not(all( + libc_cfg_target_vendor, + target_arch = "powerpc", + target_vendor = "nintendo" + )))] + #[cfg_attr(target_os = "espidf", link_name = "lwip_bind")] + pub fn bind(fd: ::c_int, addr: *const sockaddr, len: socklen_t) -> ::c_int; + pub fn clock_settime(clock_id: ::clockid_t, tp: *const ::timespec) -> ::c_int; + pub fn clock_gettime(clock_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; + pub fn clock_getres(clock_id: ::clockid_t, res: *mut ::timespec) -> ::c_int; + #[cfg_attr(target_os = "espidf", link_name = "lwip_close")] pub fn closesocket(sockfd: ::c_int) -> ::c_int; pub fn ioctl(fd: ::c_int, request: ::c_ulong, ...) -> ::c_int; + #[cfg(not(all( + libc_cfg_target_vendor, + target_arch = "powerpc", + target_vendor = "nintendo" + )))] + #[cfg_attr(target_os = "espidf", link_name = "lwip_recvfrom")] pub fn recvfrom( fd: ::c_int, buf: *mut ::c_void, @@ -623,6 +653,11 @@ extern "C" { addr: *mut sockaddr, addr_len: *mut socklen_t, ) -> isize; + #[cfg(not(all( + libc_cfg_target_vendor, + target_arch = "powerpc", + target_vendor = "nintendo" + )))] pub fn getnameinfo( sa: *const sockaddr, salen: socklen_t, @@ -639,7 +674,6 @@ extern "C" { envp: *const *const ::c_char, ) -> ::c_int; pub fn gettimeofday(tp: *mut ::timeval, tz: *mut ::c_void) -> ::c_int; - #[cfg_attr(target_os = "solaris", link_name = "__posix_getgrgid_r")] pub fn getgrgid_r( gid: ::gid_t, grp: *mut ::group, @@ -647,15 +681,9 @@ extern "C" { buflen: ::size_t, result: *mut *mut ::group, ) -> ::c_int; - #[cfg_attr( - all(target_os = "macos", target_arch = "x86"), - link_name = "sigaltstack$UNIX2003" - )] - #[cfg_attr(target_os = "netbsd", link_name = "__sigaltstack14")] pub fn sigaltstack(ss: *const stack_t, oss: *mut stack_t) -> ::c_int; pub fn sem_close(sem: *mut sem_t) -> ::c_int; pub fn getdtablesize() -> ::c_int; - #[cfg_attr(target_os = "solaris", link_name = "__posix_getgrnam_r")] pub fn getgrnam_r( name: *const ::c_char, grp: *mut ::group, @@ -663,22 +691,12 @@ extern "C" { buflen: ::size_t, result: *mut *mut ::group, ) -> ::c_int; - #[cfg_attr( - all(target_os = "macos", target_arch = "x86"), - link_name = "pthread_sigmask$UNIX2003" - )] - pub fn pthread_sigmask( - how: ::c_int, - set: *const sigset_t, - oldset: *mut sigset_t, - ) -> ::c_int; + pub fn pthread_sigmask(how: ::c_int, set: *const sigset_t, oldset: *mut sigset_t) -> ::c_int; pub fn sem_open(name: *const ::c_char, oflag: ::c_int, ...) -> *mut sem_t; pub fn getgrnam(name: *const ::c_char) -> *mut ::group; pub fn pthread_kill(thread: ::pthread_t, sig: ::c_int) -> ::c_int; pub fn sem_unlink(name: *const ::c_char) -> ::c_int; pub fn daemon(nochdir: ::c_int, noclose: ::c_int) -> ::c_int; - #[cfg_attr(target_os = "netbsd", link_name = "__getpwnam_r50")] - #[cfg_attr(target_os = "solaris", link_name = "__posix_getpwnam_r")] pub fn getpwnam_r( name: *const ::c_char, pwd: *mut passwd, @@ -686,8 +704,6 @@ extern "C" { buflen: ::size_t, result: *mut *mut passwd, ) -> ::c_int; - #[cfg_attr(target_os = "netbsd", link_name = "__getpwuid_r50")] - #[cfg_attr(target_os = "solaris", link_name = "__posix_getpwuid_r")] pub fn getpwuid_r( uid: ::uid_t, pwd: *mut passwd, @@ -695,11 +711,6 @@ extern "C" { buflen: ::size_t, result: *mut *mut passwd, ) -> ::c_int; - #[cfg_attr( - all(target_os = "macos", target_arch = "x86"), - link_name = "sigwait$UNIX2003" - )] - #[cfg_attr(target_os = "solaris", link_name = "__posix_sigwait")] pub fn sigwait(set: *const sigset_t, sig: *mut ::c_int) -> ::c_int; pub fn pthread_atfork( prepare: ::Option, @@ -707,21 +718,28 @@ extern "C" { child: ::Option, ) -> ::c_int; pub fn getgrgid(gid: ::gid_t) -> *mut ::group; - #[cfg_attr( - all(target_os = "macos", target_arch = "x86"), - link_name = "popen$UNIX2003" - )] pub fn popen(command: *const c_char, mode: *const c_char) -> *mut ::FILE; pub fn uname(buf: *mut ::utsname) -> ::c_int; } +mod generic; + cfg_if! { - if #[cfg(target_arch = "arm")] { + if #[cfg(target_os = "espidf")] { + mod espidf; + pub use self::espidf::*; + } else if #[cfg(target_os = "horizon")] { + mod horizon; + pub use self::horizon::*; + } else if #[cfg(target_arch = "arm")] { mod arm; pub use self::arm::*; } else if #[cfg(target_arch = "aarch64")] { mod aarch64; pub use self::aarch64::*; + } else if #[cfg(target_arch = "powerpc")] { + mod powerpc; + pub use self::powerpc::*; } else { // Only tested on ARM so far. Other platforms might have different // definitions for types and constants. diff --git a/crux-mir/lib/libc/src/unix/newlib/no_align.rs b/crux-mir/lib/libc/src/unix/newlib/no_align.rs index 316c464ed..ce3aca4ed 100644 --- a/crux-mir/lib/libc/src/unix/newlib/no_align.rs +++ b/crux-mir/lib/libc/src/unix/newlib/no_align.rs @@ -47,5 +47,5 @@ macro_rules! expand_align { size: [u8; ::__SIZEOF_PTHREAD_CONDATTR_T], } } - } + }; } diff --git a/crux-mir/lib/libc/src/unix/newlib/powerpc/mod.rs b/crux-mir/lib/libc/src/unix/newlib/powerpc/mod.rs new file mode 100644 index 000000000..6bed1ce27 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/newlib/powerpc/mod.rs @@ -0,0 +1,16 @@ +pub type clock_t = ::c_ulong; +pub type c_char = u8; +pub type wchar_t = ::c_int; + +pub type c_long = i32; +pub type c_ulong = u32; + +pub use crate::unix::newlib::generic::{sigset_t, stat}; + +// the newlib shipped with devkitPPC does not support the following components: +// - sockaddr +// - AF_INET6 +// - FIONBIO +// - POLL* +// - SOL_SOCKET +// - MSG_* diff --git a/crux-mir/lib/libc/src/unix/redox/mod.rs b/crux-mir/lib/libc/src/unix/redox/mod.rs index 7f66f1b41..afba67727 100644 --- a/crux-mir/lib/libc/src/unix/redox/mod.rs +++ b/crux-mir/lib/libc/src/unix/redox/mod.rs @@ -1,8 +1,20 @@ pub type c_char = i8; -pub type c_long = i64; -pub type c_ulong = u64; pub type wchar_t = i32; +cfg_if! { + if #[cfg(target_pointer_width = "32")] { + pub type c_long = i32; + pub type c_ulong = u32; + } +} + +cfg_if! { + if #[cfg(target_pointer_width = "64")] { + pub type c_long = i64; + pub type c_ulong = u64; + } +} + pub type blkcnt_t = ::c_ulong; pub type blksize_t = ::c_long; pub type clock_t = ::c_long; @@ -14,7 +26,7 @@ pub type ino_t = ::c_ulong; pub type mode_t = ::c_int; pub type nfds_t = ::c_ulong; pub type nlink_t = ::c_ulong; -pub type off_t = ::c_long; +pub type off_t = ::c_longlong; pub type pthread_t = *mut ::c_void; pub type pthread_attr_t = *mut ::c_void; pub type pthread_cond_t = *mut ::c_void; @@ -34,7 +46,7 @@ pub type socklen_t = u32; pub type speed_t = u32; pub type suseconds_t = ::c_int; pub type tcflag_t = u32; -pub type time_t = ::c_long; +pub type time_t = ::c_longlong; #[cfg_attr(feature = "extra_traits", derive(Debug))] pub enum timezone {} @@ -151,12 +163,20 @@ s! { } pub struct sigaction { - pub sa_handler: ::sighandler_t, + pub sa_sigaction: ::sighandler_t, pub sa_flags: ::c_ulong, pub sa_restorer: ::Option, pub sa_mask: ::sigset_t, } + pub struct siginfo_t { + pub si_signo: ::c_int, + pub si_errno: ::c_int, + pub si_code: ::c_int, + _pad: [::c_int; 29], + _align: [usize; 0], + } + pub struct sockaddr { pub sa_family: ::sa_family_t, pub sa_data: [::c_char; 14], @@ -367,6 +387,7 @@ pub const EPROTONOSUPPORT: ::c_int = 93; /* Protocol not supported */ pub const ESOCKTNOSUPPORT: ::c_int = 94; /* Socket type not supported */ /* Operation not supported on transport endpoint */ pub const EOPNOTSUPP: ::c_int = 95; +pub const ENOTSUP: ::c_int = EOPNOTSUPP; pub const EPFNOSUPPORT: ::c_int = 96; /* Protocol family not supported */ /* Address family not supported by protocol */ pub const EAFNOSUPPORT: ::c_int = 97; @@ -439,7 +460,32 @@ pub const O_SYMLINK: ::c_int = 0x4000_0000; pub const O_NOFOLLOW: ::c_int = -0x8000_0000; // netdb.h +pub const AI_PASSIVE: ::c_int = 0x0001; +pub const AI_CANONNAME: ::c_int = 0x0002; +pub const AI_NUMERICHOST: ::c_int = 0x0004; +pub const AI_V4MAPPED: ::c_int = 0x0008; +pub const AI_ALL: ::c_int = 0x0010; +pub const AI_ADDRCONFIG: ::c_int = 0x0020; +pub const AI_NUMERICSERV: ::c_int = 0x0400; +pub const EAI_BADFLAGS: ::c_int = -1; +pub const EAI_NONAME: ::c_int = -2; +pub const EAI_AGAIN: ::c_int = -3; +pub const EAI_FAIL: ::c_int = -4; +pub const EAI_NODATA: ::c_int = -5; +pub const EAI_FAMILY: ::c_int = -6; +pub const EAI_SOCKTYPE: ::c_int = -7; +pub const EAI_SERVICE: ::c_int = -8; +pub const EAI_ADDRFAMILY: ::c_int = -9; +pub const EAI_MEMORY: ::c_int = -10; pub const EAI_SYSTEM: ::c_int = -11; +pub const EAI_OVERFLOW: ::c_int = -12; +pub const NI_MAXHOST: ::c_int = 1025; +pub const NI_MAXSERV: ::c_int = 32; +pub const NI_NUMERICHOST: ::c_int = 0x0001; +pub const NI_NUMERICSERV: ::c_int = 0x0002; +pub const NI_NOFQDN: ::c_int = 0x0004; +pub const NI_NAMEREQD: ::c_int = 0x0008; +pub const NI_DGRAM: ::c_int = 0x0010; // netinet/in.h // FIXME: relibc { @@ -526,6 +572,12 @@ pub const SA_RESTART: ::c_ulong = 0x10000000; pub const SA_NODEFER: ::c_ulong = 0x40000000; pub const SA_RESETHAND: ::c_ulong = 0x80000000; +// sys/file.h +pub const LOCK_SH: ::c_int = 1; +pub const LOCK_EX: ::c_int = 2; +pub const LOCK_NB: ::c_int = 4; +pub const LOCK_UN: ::c_int = 8; + // sys/epoll.h pub const EPOLL_CLOEXEC: ::c_int = 0x0100_0000; pub const EPOLL_CTL_ADD: ::c_int = 1; @@ -576,6 +628,7 @@ pub const EXIT_FAILURE: ::c_int = 1; // sys/ioctl.h // FIXME: relibc { +pub const FIONREAD: ::c_ulong = 0x541B; pub const FIONBIO: ::c_ulong = 0x5421; pub const FIOCLEX: ::c_ulong = 0x5451; // } @@ -587,51 +640,105 @@ pub const TIOCSPGRP: ::c_ulong = 0x5410; pub const TIOCGWINSZ: ::c_ulong = 0x5413; pub const TIOCSWINSZ: ::c_ulong = 0x5414; +// sys/mman.h +pub const PROT_NONE: ::c_int = 0x0000; +pub const PROT_READ: ::c_int = 0x0004; +pub const PROT_WRITE: ::c_int = 0x0002; +pub const PROT_EXEC: ::c_int = 0x0001; + +pub const MADV_NORMAL: ::c_int = 0; +pub const MADV_RANDOM: ::c_int = 1; +pub const MADV_SEQUENTIAL: ::c_int = 2; +pub const MADV_WILLNEED: ::c_int = 3; +pub const MADV_DONTNEED: ::c_int = 4; + +pub const MAP_SHARED: ::c_int = 0x0001; +pub const MAP_PRIVATE: ::c_int = 0x0002; +pub const MAP_ANON: ::c_int = 0x0020; +pub const MAP_ANONYMOUS: ::c_int = MAP_ANON; +pub const MAP_FIXED: ::c_int = 0x0010; +pub const MAP_FAILED: *mut ::c_void = !0 as _; + +pub const MS_ASYNC: ::c_int = 0x0001; +pub const MS_INVALIDATE: ::c_int = 0x0002; +pub const MS_SYNC: ::c_int = 0x0004; + // sys/select.h pub const FD_SETSIZE: usize = 1024; // sys/socket.h -pub const AF_UNIX: ::c_int = 1; pub const AF_INET: ::c_int = 2; pub const AF_INET6: ::c_int = 10; +pub const AF_UNIX: ::c_int = 1; +pub const AF_UNSPEC: ::c_int = 0; +pub const PF_INET: ::c_int = 2; +pub const PF_INET6: ::c_int = 10; +pub const PF_UNIX: ::c_int = 1; +pub const PF_UNSPEC: ::c_int = 0; +pub const MSG_CTRUNC: ::c_int = 8; +pub const MSG_DONTROUTE: ::c_int = 4; +pub const MSG_EOR: ::c_int = 128; +pub const MSG_OOB: ::c_int = 1; pub const MSG_PEEK: ::c_int = 2; +pub const MSG_TRUNC: ::c_int = 32; +pub const MSG_DONTWAIT: ::c_int = 64; +pub const MSG_WAITALL: ::c_int = 256; pub const SHUT_RD: ::c_int = 0; pub const SHUT_WR: ::c_int = 1; pub const SHUT_RDWR: ::c_int = 2; +pub const SO_DEBUG: ::c_int = 1; pub const SO_REUSEADDR: ::c_int = 2; +pub const SO_TYPE: ::c_int = 3; pub const SO_ERROR: ::c_int = 4; +pub const SO_DONTROUTE: ::c_int = 5; pub const SO_BROADCAST: ::c_int = 6; pub const SO_SNDBUF: ::c_int = 7; pub const SO_RCVBUF: ::c_int = 8; pub const SO_KEEPALIVE: ::c_int = 9; +pub const SO_OOBINLINE: ::c_int = 10; +pub const SO_NO_CHECK: ::c_int = 11; +pub const SO_PRIORITY: ::c_int = 12; pub const SO_LINGER: ::c_int = 13; +pub const SO_BSDCOMPAT: ::c_int = 14; pub const SO_REUSEPORT: ::c_int = 15; +pub const SO_PASSCRED: ::c_int = 16; +pub const SO_PEERCRED: ::c_int = 17; +pub const SO_RCVLOWAT: ::c_int = 18; +pub const SO_SNDLOWAT: ::c_int = 19; pub const SO_RCVTIMEO: ::c_int = 20; pub const SO_SNDTIMEO: ::c_int = 21; +pub const SO_ACCEPTCONN: ::c_int = 30; +pub const SO_PEERSEC: ::c_int = 31; +pub const SO_SNDBUFFORCE: ::c_int = 32; +pub const SO_RCVBUFFORCE: ::c_int = 33; +pub const SO_PROTOCOL: ::c_int = 38; +pub const SO_DOMAIN: ::c_int = 39; pub const SOCK_STREAM: ::c_int = 1; pub const SOCK_DGRAM: ::c_int = 2; +pub const SOCK_NONBLOCK: ::c_int = 0o4_000; +pub const SOCK_CLOEXEC: ::c_int = 0o2_000_000; +pub const SOCK_SEQPACKET: ::c_int = 5; pub const SOL_SOCKET: ::c_int = 1; // sys/termios.h -pub const NCCS: usize = 32; - -pub const VINTR: usize = 0; -pub const VQUIT: usize = 1; -pub const VERASE: usize = 2; -pub const VKILL: usize = 3; -pub const VEOF: usize = 4; -pub const VTIME: usize = 5; -pub const VMIN: usize = 6; +pub const VEOF: usize = 0; +pub const VEOL: usize = 1; +pub const VEOL2: usize = 2; +pub const VERASE: usize = 3; +pub const VWERASE: usize = 4; +pub const VKILL: usize = 5; +pub const VREPRINT: usize = 6; pub const VSWTC: usize = 7; -pub const VSTART: usize = 8; -pub const VSTOP: usize = 9; +pub const VINTR: usize = 8; +pub const VQUIT: usize = 9; pub const VSUSP: usize = 10; -pub const VEOL: usize = 11; -pub const VREPRINT: usize = 12; -pub const VDISCARD: usize = 13; -pub const VWERASE: usize = 14; -pub const VLNEXT: usize = 15; -pub const VEOL2: usize = 16; +pub const VSTART: usize = 12; +pub const VSTOP: usize = 13; +pub const VLNEXT: usize = 14; +pub const VDISCARD: usize = 15; +pub const VMIN: usize = 16; +pub const VTIME: usize = 17; +pub const NCCS: usize = 32; pub const IGNBRK: ::tcflag_t = 0o000_001; pub const BRKINT: ::tcflag_t = 0o000_002; @@ -642,25 +749,17 @@ pub const ISTRIP: ::tcflag_t = 0o000_040; pub const INLCR: ::tcflag_t = 0o000_100; pub const IGNCR: ::tcflag_t = 0o000_200; pub const ICRNL: ::tcflag_t = 0o000_400; -pub const IUCLC: ::tcflag_t = 0o001_000; -pub const IXON: ::tcflag_t = 0o002_000; -pub const IXANY: ::tcflag_t = 0o004_000; -pub const IXOFF: ::tcflag_t = 0o010_000; -pub const IMAXBEL: ::tcflag_t = 0o020_000; -pub const IUTF8: ::tcflag_t = 0o040_000; +pub const IXON: ::tcflag_t = 0o001_000; +pub const IXOFF: ::tcflag_t = 0o002_000; pub const OPOST: ::tcflag_t = 0o000_001; -pub const OLCUC: ::tcflag_t = 0o000_002; -pub const ONLCR: ::tcflag_t = 0o000_004; +pub const ONLCR: ::tcflag_t = 0o000_002; +pub const OLCUC: ::tcflag_t = 0o000_004; pub const OCRNL: ::tcflag_t = 0o000_010; pub const ONOCR: ::tcflag_t = 0o000_020; -pub const ONLRET: ::tcflag_t = 0o00_0040; -pub const OFILL: ::tcflag_t = 0o000_100; -pub const OFDEL: ::tcflag_t = 0o000_200; - -pub const VTDLY: usize = 0o040_000; -pub const VT0: usize = 0o000_000; -pub const VT1: usize = 0o040_000; +pub const ONLRET: ::tcflag_t = 0o000_040; +pub const OFILL: ::tcflag_t = 0o0000_100; +pub const OFDEL: ::tcflag_t = 0o0000_200; pub const B0: speed_t = 0o000_000; pub const B50: speed_t = 0o000_001; @@ -679,43 +778,45 @@ pub const B9600: speed_t = 0o000_015; pub const B19200: speed_t = 0o000_016; pub const B38400: speed_t = 0o000_017; -pub const B57600: speed_t = 0o010_001; -pub const B115200: speed_t = 0o010_002; -pub const B230400: speed_t = 0o010_003; -pub const B460800: speed_t = 0o010_004; -pub const B500000: speed_t = 0o010_005; -pub const B576000: speed_t = 0o010_006; -pub const B921600: speed_t = 0o010_007; -pub const B1000000: speed_t = 0o010_010; -pub const B1152000: speed_t = 0o010_011; -pub const B1500000: speed_t = 0o010_012; -pub const B2000000: speed_t = 0o010_013; -pub const B2500000: speed_t = 0o010_014; -pub const B3000000: speed_t = 0o010_015; -pub const B3500000: speed_t = 0o010_016; -pub const B4000000: speed_t = 0o010_017; - -pub const CSIZE: ::tcflag_t = 0o000_060; +pub const B57600: speed_t = 0o0_020; +pub const B115200: speed_t = 0o0_021; +pub const B230400: speed_t = 0o0_022; +pub const B460800: speed_t = 0o0_023; +pub const B500000: speed_t = 0o0_024; +pub const B576000: speed_t = 0o0_025; +pub const B921600: speed_t = 0o0_026; +pub const B1000000: speed_t = 0o0_027; +pub const B1152000: speed_t = 0o0_030; +pub const B1500000: speed_t = 0o0_031; +pub const B2000000: speed_t = 0o0_032; +pub const B2500000: speed_t = 0o0_033; +pub const B3000000: speed_t = 0o0_034; +pub const B3500000: speed_t = 0o0_035; +pub const B4000000: speed_t = 0o0_036; + +pub const CSIZE: ::tcflag_t = 0o001_400; pub const CS5: ::tcflag_t = 0o000_000; -pub const CS6: ::tcflag_t = 0o000_020; -pub const CS7: ::tcflag_t = 0o000_040; -pub const CS8: ::tcflag_t = 0o000_060; -pub const CSTOPB: ::tcflag_t = 0o000_100; -pub const CREAD: ::tcflag_t = 0o000_200; -pub const PARENB: ::tcflag_t = 0o000_400; -pub const PARODD: ::tcflag_t = 0o001_000; -pub const HUPCL: ::tcflag_t = 0o002_000; -pub const CLOCAL: ::tcflag_t = 0o004_000; - -pub const ISIG: ::tcflag_t = 0o000_001; -pub const ICANON: ::tcflag_t = 0o000_002; -pub const ECHO: ::tcflag_t = 0o000_010; -pub const ECHOE: ::tcflag_t = 0o000_020; -pub const ECHOK: ::tcflag_t = 0o000_040; -pub const ECHONL: ::tcflag_t = 0o000_100; -pub const NOFLSH: ::tcflag_t = 0o000_200; -pub const TOSTOP: ::tcflag_t = 0o000_400; -pub const IEXTEN: ::tcflag_t = 0o100_000; +pub const CS6: ::tcflag_t = 0o000_400; +pub const CS7: ::tcflag_t = 0o001_000; +pub const CS8: ::tcflag_t = 0o001_400; + +pub const CSTOPB: ::tcflag_t = 0o002_000; +pub const CREAD: ::tcflag_t = 0o004_000; +pub const PARENB: ::tcflag_t = 0o010_000; +pub const PARODD: ::tcflag_t = 0o020_000; +pub const HUPCL: ::tcflag_t = 0o040_000; + +pub const CLOCAL: ::tcflag_t = 0o0100000; + +pub const ISIG: ::tcflag_t = 0x0000_0080; +pub const ICANON: ::tcflag_t = 0x0000_0100; +pub const ECHO: ::tcflag_t = 0x0000_0008; +pub const ECHOE: ::tcflag_t = 0x0000_0002; +pub const ECHOK: ::tcflag_t = 0x0000_0004; +pub const ECHONL: ::tcflag_t = 0x0000_0010; +pub const NOFLSH: ::tcflag_t = 0x8000_0000; +pub const TOSTOP: ::tcflag_t = 0x0040_0000; +pub const IEXTEN: ::tcflag_t = 0x0000_0400; pub const TCOOFF: ::c_int = 0; pub const TCOON: ::c_int = 1; @@ -747,6 +848,8 @@ pub const __WCLONE: ::c_int = 0x8000_0000; // time.h pub const CLOCK_REALTIME: ::c_int = 1; pub const CLOCK_MONOTONIC: ::c_int = 4; +pub const CLOCK_PROCESS_CPUTIME_ID: ::clockid_t = 2; +pub const CLOCKS_PER_SEC: ::clock_t = 1_000_000; // unistd.h // POSIX.1 { @@ -806,40 +909,12 @@ pub const _PC_ALLOC_SIZE_MIN: ::c_int = 18; pub const _PC_SYMLINK_MAX: ::c_int = 19; pub const _PC_2_SYMLINKS: ::c_int = 20; +pub const PRIO_PROCESS: ::c_int = 0; +pub const PRIO_PGRP: ::c_int = 1; +pub const PRIO_USER: ::c_int = 2; + // wait.h f! { - pub fn WIFSTOPPED(status: ::c_int) -> bool { - (status & 0xff) == 0x7f - } - - pub fn WSTOPSIG(status: ::c_int) -> ::c_int { - (status >> 8) & 0xff - } - - pub fn WIFCONTINUED(status: ::c_int) -> bool { - status == 0xffff - } - - pub fn WIFSIGNALED(status: ::c_int) -> bool { - ((status & 0x7f) + 1) as i8 >= 2 - } - - pub fn WTERMSIG(status: ::c_int) -> ::c_int { - status & 0x7f - } - - pub fn WIFEXITED(status: ::c_int) -> bool { - (status & 0x7f) == 0 - } - - pub fn WEXITSTATUS(status: ::c_int) -> ::c_int { - (status >> 8) & 0xff - } - - pub fn WCOREDUMP(status: ::c_int) -> bool { - (status & 0x80) != 0 - } - pub fn FD_CLR(fd: ::c_int, set: *mut fd_set) -> () { let fd = fd as usize; let size = ::mem::size_of_val(&(*set).fds_bits[0]) * 8; @@ -847,7 +922,7 @@ f! { return } - pub fn FD_ISSET(fd: ::c_int, set: *mut fd_set) -> bool { + pub fn FD_ISSET(fd: ::c_int, set: *const fd_set) -> bool { let fd = fd as usize; let size = ::mem::size_of_val(&(*set).fds_bits[0]) * 8; return ((*set).fds_bits[fd / size] & (1 << (fd % size))) != 0 @@ -867,14 +942,44 @@ f! { } } +safe_f! { + pub {const} fn WIFSTOPPED(status: ::c_int) -> bool { + (status & 0xff) == 0x7f + } + + pub {const} fn WSTOPSIG(status: ::c_int) -> ::c_int { + (status >> 8) & 0xff + } + + pub {const} fn WIFCONTINUED(status: ::c_int) -> bool { + status == 0xffff + } + + pub {const} fn WIFSIGNALED(status: ::c_int) -> bool { + ((status & 0x7f) + 1) as i8 >= 2 + } + + pub {const} fn WTERMSIG(status: ::c_int) -> ::c_int { + status & 0x7f + } + + pub {const} fn WIFEXITED(status: ::c_int) -> bool { + (status & 0x7f) == 0 + } + + pub {const} fn WEXITSTATUS(status: ::c_int) -> ::c_int { + (status >> 8) & 0xff + } + + pub {const} fn WCOREDUMP(status: ::c_int) -> bool { + (status & 0x80) != 0 + } +} + extern "C" { // errno.h pub fn __errno_location() -> *mut ::c_int; - pub fn strerror_r( - errnum: ::c_int, - buf: *mut c_char, - buflen: ::size_t, - ) -> ::c_int; + pub fn strerror_r(errnum: ::c_int, buf: *mut c_char, buflen: ::size_t) -> ::c_int; // unistd.h pub fn pipe2(fds: *mut ::c_int, flags: ::c_int) -> ::c_int; @@ -882,6 +987,17 @@ extern "C" { // malloc.h pub fn memalign(align: ::size_t, size: ::size_t) -> *mut ::c_void; + // netdb.h + pub fn getnameinfo( + addr: *const ::sockaddr, + addrlen: ::socklen_t, + host: *mut ::c_char, + hostlen: ::socklen_t, + serv: *mut ::c_char, + servlen: ::socklen_t, + flags: ::c_int, + ) -> ::c_int; + // pthread.h pub fn pthread_atfork( prepare: ::Option, @@ -914,6 +1030,8 @@ extern "C" { set: *const ::sigset_t, oldset: *mut ::sigset_t, ) -> ::c_int; + pub fn pthread_cancel(thread: ::pthread_t) -> ::c_int; + pub fn pthread_kill(thread: ::pthread_t, sig: ::c_int) -> ::c_int; // sys/epoll.h pub fn epoll_create(size: ::c_int) -> ::c_int; @@ -924,26 +1042,25 @@ extern "C" { maxevents: ::c_int, timeout: ::c_int, ) -> ::c_int; - pub fn epoll_ctl( - epfd: ::c_int, - op: ::c_int, - fd: ::c_int, - event: *mut ::epoll_event, - ) -> ::c_int; + pub fn epoll_ctl(epfd: ::c_int, op: ::c_int, fd: ::c_int, event: *mut ::epoll_event) + -> ::c_int; // sys/ioctl.h pub fn ioctl(fd: ::c_int, request: ::c_ulong, ...) -> ::c_int; + // sys/mman.h + pub fn madvise(addr: *mut ::c_void, len: ::size_t, advice: ::c_int) -> ::c_int; + pub fn msync(addr: *mut ::c_void, len: ::size_t, flags: ::c_int) -> ::c_int; + pub fn mprotect(addr: *mut ::c_void, len: ::size_t, prot: ::c_int) -> ::c_int; + pub fn shm_open(name: *const c_char, oflag: ::c_int, mode: mode_t) -> ::c_int; + pub fn shm_unlink(name: *const ::c_char) -> ::c_int; + // sys/resource.h pub fn getrlimit(resource: ::c_int, rlim: *mut ::rlimit) -> ::c_int; pub fn setrlimit(resource: ::c_int, rlim: *const ::rlimit) -> ::c_int; // sys/socket.h - pub fn bind( - socket: ::c_int, - address: *const ::sockaddr, - address_len: ::socklen_t, - ) -> ::c_int; + pub fn bind(socket: ::c_int, address: *const ::sockaddr, address_len: ::socklen_t) -> ::c_int; pub fn recvfrom( socket: ::c_int, buf: *mut ::c_void, @@ -957,16 +1074,8 @@ extern "C" { pub fn futimens(fd: ::c_int, times: *const ::timespec) -> ::c_int; // sys/uio.h - pub fn readv( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - ) -> ::ssize_t; - pub fn writev( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - ) -> ::ssize_t; + pub fn readv(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int) -> ::ssize_t; + pub fn writev(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int) -> ::ssize_t; // sys/utsname.h pub fn uname(utsname: *mut utsname) -> ::c_int; diff --git a/crux-mir/lib/libc/src/unix/solarish/compat.rs b/crux-mir/lib/libc/src/unix/solarish/compat.rs index a33645211..cbf955a31 100644 --- a/crux-mir/lib/libc/src/unix/solarish/compat.rs +++ b/crux-mir/lib/libc/src/unix/solarish/compat.rs @@ -1,32 +1,220 @@ // Common functions that are unfortunately missing on illumos and // Solaris, but often needed by other crates. +use core::cmp::min; use unix::solarish::*; +const PTEM: &[u8] = b"ptem\0"; +const LDTERM: &[u8] = b"ldterm\0"; + pub unsafe fn cfmakeraw(termios: *mut ::termios) { - let mut t = *termios as ::termios; - t.c_iflag &= !(IMAXBEL - | IGNBRK - | BRKINT - | PARMRK - | ISTRIP - | INLCR - | IGNCR - | ICRNL - | IXON); - t.c_oflag &= !OPOST; - t.c_lflag &= !(ECHO | ECHONL | ICANON | ISIG | IEXTEN); - t.c_cflag &= !(CSIZE | PARENB); - t.c_cflag |= CS8; + (*termios).c_iflag &= + !(IMAXBEL | IGNBRK | BRKINT | PARMRK | ISTRIP | INLCR | IGNCR | ICRNL | IXON); + (*termios).c_oflag &= !OPOST; + (*termios).c_lflag &= !(ECHO | ECHONL | ICANON | ISIG | IEXTEN); + (*termios).c_cflag &= !(CSIZE | PARENB); + (*termios).c_cflag |= CS8; + + // By default, most software expects a pending read to block until at + // least one byte becomes available. As per termio(7I), this requires + // setting the MIN and TIME parameters appropriately. + // + // As a somewhat unfortunate artefact of history, the MIN and TIME slots + // in the control character array overlap with the EOF and EOL slots used + // for canonical mode processing. Because the EOF character needs to be + // the ASCII EOT value (aka Control-D), it has the byte value 4. When + // switching to raw mode, this is interpreted as a MIN value of 4; i.e., + // reads will block until at least four bytes have been input. + // + // Other platforms with a distinct MIN slot like Linux and FreeBSD appear + // to default to a MIN value of 1, so we'll force that value here: + (*termios).c_cc[VMIN] = 1; + (*termios).c_cc[VTIME] = 0; } -pub unsafe fn cfsetspeed( - termios: *mut ::termios, - speed: ::speed_t, -) -> ::c_int { +pub unsafe fn cfsetspeed(termios: *mut ::termios, speed: ::speed_t) -> ::c_int { // Neither of these functions on illumos or Solaris actually ever // return an error ::cfsetispeed(termios, speed); ::cfsetospeed(termios, speed); 0 } + +unsafe fn bail(fdm: ::c_int, fds: ::c_int) -> ::c_int { + let e = *___errno(); + if fds >= 0 { + ::close(fds); + } + if fdm >= 0 { + ::close(fdm); + } + *___errno() = e; + return -1; +} + +pub unsafe fn openpty( + amain: *mut ::c_int, + asubord: *mut ::c_int, + name: *mut ::c_char, + termp: *const termios, + winp: *const ::winsize, +) -> ::c_int { + // Open the main pseudo-terminal device, making sure not to set it as the + // controlling terminal for this process: + let fdm = ::posix_openpt(O_RDWR | O_NOCTTY); + if fdm < 0 { + return -1; + } + + // Set permissions and ownership on the subordinate device and unlock it: + if ::grantpt(fdm) < 0 || ::unlockpt(fdm) < 0 { + return bail(fdm, -1); + } + + // Get the path name of the subordinate device: + let subordpath = ::ptsname(fdm); + if subordpath.is_null() { + return bail(fdm, -1); + } + + // Open the subordinate device without setting it as the controlling + // terminal for this process: + let fds = ::open(subordpath, O_RDWR | O_NOCTTY); + if fds < 0 { + return bail(fdm, -1); + } + + // Check if the STREAMS modules are already pushed: + let setup = ::ioctl(fds, I_FIND, LDTERM.as_ptr()); + if setup < 0 { + return bail(fdm, fds); + } else if setup == 0 { + // The line discipline is not present, so push the appropriate STREAMS + // modules for the subordinate device: + if ::ioctl(fds, I_PUSH, PTEM.as_ptr()) < 0 || ::ioctl(fds, I_PUSH, LDTERM.as_ptr()) < 0 { + return bail(fdm, fds); + } + } + + // If provided, set the terminal parameters: + if !termp.is_null() && ::tcsetattr(fds, TCSAFLUSH, termp) != 0 { + return bail(fdm, fds); + } + + // If provided, set the window size: + if !winp.is_null() && ::ioctl(fds, TIOCSWINSZ, winp) < 0 { + return bail(fdm, fds); + } + + // If the caller wants the name of the subordinate device, copy it out. + // + // Note that this is a terrible interface: there appears to be no standard + // upper bound on the copy length for this pointer. Nobody should pass + // anything but NULL here, preferring instead to use ptsname(3C) directly. + if !name.is_null() { + ::strcpy(name, subordpath); + } + + *amain = fdm; + *asubord = fds; + 0 +} + +pub unsafe fn forkpty( + amain: *mut ::c_int, + name: *mut ::c_char, + termp: *const termios, + winp: *const ::winsize, +) -> ::pid_t { + let mut fds = -1; + + if openpty(amain, &mut fds, name, termp, winp) != 0 { + return -1; + } + + let pid = ::fork(); + if pid < 0 { + return bail(*amain, fds); + } else if pid > 0 { + // In the parent process, we close the subordinate device and return the + // process ID of the new child: + ::close(fds); + return pid; + } + + // The rest of this function executes in the child process. + + // Close the main side of the pseudo-terminal pair: + ::close(*amain); + + // Use TIOCSCTTY to set the subordinate device as our controlling + // terminal. This will fail (with ENOTTY) if we are not the leader in + // our own session, so we call setsid() first. Finally, arrange for + // the pseudo-terminal to occupy the standard I/O descriptors. + if ::setsid() < 0 + || ::ioctl(fds, TIOCSCTTY, 0) < 0 + || ::dup2(fds, 0) < 0 + || ::dup2(fds, 1) < 0 + || ::dup2(fds, 2) < 0 + { + // At this stage there are no particularly good ways to handle failure. + // Exit as abruptly as possible, using _exit() to avoid messing with any + // state still shared with the parent process. + ::_exit(EXIT_FAILURE); + } + // Close the inherited descriptor, taking care to avoid closing the standard + // descriptors by mistake: + if fds > 2 { + ::close(fds); + } + + 0 +} + +pub unsafe fn getpwent_r( + pwd: *mut passwd, + buf: *mut ::c_char, + buflen: ::size_t, + result: *mut *mut passwd, +) -> ::c_int { + let old_errno = *::___errno(); + *::___errno() = 0; + *result = native_getpwent_r( + pwd, + buf, + min(buflen, ::c_int::max_value() as ::size_t) as ::c_int, + ); + + let ret = if (*result).is_null() { + *::___errno() + } else { + 0 + }; + *::___errno() = old_errno; + + ret +} + +pub unsafe fn getgrent_r( + grp: *mut ::group, + buf: *mut ::c_char, + buflen: ::size_t, + result: *mut *mut ::group, +) -> ::c_int { + let old_errno = *::___errno(); + *::___errno() = 0; + *result = native_getgrent_r( + grp, + buf, + min(buflen, ::c_int::max_value() as ::size_t) as ::c_int, + ); + + let ret = if (*result).is_null() { + *::___errno() + } else { + 0 + }; + *::___errno() = old_errno; + + ret +} diff --git a/crux-mir/lib/libc/src/unix/solarish/illumos.rs b/crux-mir/lib/libc/src/unix/solarish/illumos.rs new file mode 100644 index 000000000..404f013da --- /dev/null +++ b/crux-mir/lib/libc/src/unix/solarish/illumos.rs @@ -0,0 +1,88 @@ +s! { + pub struct shmid_ds { + pub shm_perm: ::ipc_perm, + pub shm_segsz: ::size_t, + pub shm_amp: *mut ::c_void, + pub shm_lkcnt: ::c_ushort, + pub shm_lpid: ::pid_t, + pub shm_cpid: ::pid_t, + pub shm_nattch: ::shmatt_t, + pub shm_cnattch: ::c_ulong, + pub shm_atime: ::time_t, + pub shm_dtime: ::time_t, + pub shm_ctime: ::time_t, + pub shm_pad4: [i64; 4], + } + + pub struct fil_info { + pub fi_flags: ::c_int, + pub fi_pos: ::c_int, + pub fi_name: [::c_char; ::FILNAME_MAX as usize], + } +} + +pub const AF_LOCAL: ::c_int = 1; // AF_UNIX +pub const AF_FILE: ::c_int = 1; // AF_UNIX + +pub const EFD_SEMAPHORE: ::c_int = 0x1; +pub const EFD_NONBLOCK: ::c_int = 0x800; +pub const EFD_CLOEXEC: ::c_int = 0x80000; + +pub const TCP_KEEPIDLE: ::c_int = 34; +pub const TCP_KEEPCNT: ::c_int = 35; +pub const TCP_KEEPINTVL: ::c_int = 36; +pub const TCP_CONGESTION: ::c_int = 37; + +// These constants are correct for 64-bit programs or 32-bit programs that are +// not using large-file mode. If Rust ever supports anything other than 64-bit +// compilation on illumos, this may require adjustment: +pub const F_OFD_GETLK: ::c_int = 47; +pub const F_OFD_SETLK: ::c_int = 48; +pub const F_OFD_SETLKW: ::c_int = 49; +pub const F_FLOCK: ::c_int = 53; +pub const F_FLOCKW: ::c_int = 54; + +pub const F_DUPFD_CLOEXEC: ::c_int = 37; +pub const F_DUP2FD_CLOEXEC: ::c_int = 36; + +pub const FIL_ATTACH: ::c_int = 0x1; +pub const FIL_DETACH: ::c_int = 0x2; +pub const FIL_LIST: ::c_int = 0x3; +pub const FILNAME_MAX: ::c_int = 32; +pub const FILF_PROG: ::c_int = 0x1; +pub const FILF_AUTO: ::c_int = 0x2; +pub const FILF_BYPASS: ::c_int = 0x4; +pub const SOL_FILTER: ::c_int = 0xfffc; + +pub const MADV_PURGE: ::c_int = 9; + +pub const B1000000: ::speed_t = 24; +pub const B1152000: ::speed_t = 25; +pub const B1500000: ::speed_t = 26; +pub const B2000000: ::speed_t = 27; +pub const B2500000: ::speed_t = 28; +pub const B3000000: ::speed_t = 29; +pub const B3500000: ::speed_t = 30; +pub const B4000000: ::speed_t = 31; + +// sys/systeminfo.h +pub const SI_ADDRESS_WIDTH: ::c_int = 520; + +extern "C" { + pub fn eventfd(init: ::c_uint, flags: ::c_int) -> ::c_int; + + pub fn mincore(addr: ::caddr_t, len: ::size_t, vec: *mut ::c_char) -> ::c_int; + + pub fn pset_bind_lwp( + pset: ::psetid_t, + id: ::id_t, + pid: ::pid_t, + opset: *mut ::psetid_t, + ) -> ::c_int; + pub fn pset_getloadavg(pset: ::psetid_t, load: *mut ::c_double, num: ::c_int) -> ::c_int; + + pub fn preadv(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int, offset: ::off_t) -> ::ssize_t; + pub fn pwritev(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int, offset: ::off_t) + -> ::ssize_t; + pub fn getpagesizes2(pagesize: *mut ::size_t, nelem: ::c_int) -> ::c_int; +} diff --git a/crux-mir/lib/libc/src/unix/solarish/mod.rs b/crux-mir/lib/libc/src/unix/solarish/mod.rs index 35031df20..abe304e8e 100644 --- a/crux-mir/lib/libc/src/unix/solarish/mod.rs +++ b/crux-mir/lib/libc/src/unix/solarish/mod.rs @@ -1,6 +1,7 @@ pub type c_char = i8; pub type c_long = i64; pub type c_ulong = u64; +pub type caddr_t = *mut ::c_char; pub type clockid_t = ::c_int; pub type blkcnt_t = ::c_long; @@ -19,8 +20,15 @@ pub type rlim_t = ::c_ulong; pub type speed_t = ::c_uint; pub type tcflag_t = ::c_uint; pub type time_t = ::c_long; +pub type timer_t = ::c_int; pub type wchar_t = ::c_int; pub type nfds_t = ::c_ulong; +pub type projid_t = ::c_int; +pub type zoneid_t = ::c_int; +pub type psetid_t = ::c_int; +pub type processorid_t = ::c_int; +pub type chipid_t = ::c_int; +pub type ctid_t = ::id_t; pub type suseconds_t = ::c_long; pub type off_t = ::c_long; @@ -29,6 +37,7 @@ pub type socklen_t = ::c_uint; pub type sa_family_t = u16; pub type pthread_t = ::c_uint; pub type pthread_key_t = ::c_uint; +pub type thread_t = ::c_uint; pub type blksize_t = ::c_int; pub type nl_item = ::c_int; pub type mqd_t = *mut ::c_void; @@ -36,8 +45,15 @@ pub type id_t = ::c_int; pub type idtype_t = ::c_uint; pub type shmatt_t = ::c_ulong; -pub type door_attr_t = ::c_uint; -pub type door_id_t = ::c_ulonglong; +pub type lgrp_rsrc_t = ::c_int; +pub type lgrp_affinity_t = ::c_int; +pub type lgrp_id_t = ::id_t; +pub type lgrp_mem_size_t = ::c_longlong; +pub type lgrp_cookie_t = ::uintptr_t; +pub type lgrp_content_t = ::c_uint; +pub type lgrp_lat_between_t = ::c_uint; +pub type lgrp_mem_size_flag_t = ::c_uint; +pub type lgrp_view_t = ::c_uint; #[cfg_attr(feature = "extra_traits", derive(Debug))] pub enum timezone {} @@ -48,6 +64,15 @@ impl ::Clone for timezone { } } +#[cfg_attr(feature = "extra_traits", derive(Debug))] +pub enum ucred_t {} +impl ::Copy for ucred_t {} +impl ::Clone for ucred_t { + fn clone(&self) -> ucred_t { + *self + } +} + s! { pub struct in_addr { pub s_addr: ::in_addr_t, @@ -58,6 +83,12 @@ s! { pub imr_interface: in_addr, } + pub struct ip_mreq_source { + pub imr_multiaddr: in_addr, + pub imr_sourceaddr: in_addr, + pub imr_interface: in_addr, + } + pub struct ipc_perm { pub uid: ::uid_t, pub gid: ::gid_t, @@ -111,6 +142,11 @@ s! { pub ifa_data: *mut ::c_void } + pub struct itimerspec { + pub it_interval: ::timespec, + pub it_value: ::timespec, + } + pub struct tm { pub tm_sec: ::c_int, pub tm_min: ::c_int, @@ -217,33 +253,6 @@ s! { pub ai_next: *mut addrinfo, } - pub struct shmid_ds { - pub shm_perm: ipc_perm, - pub shm_segsz: ::size_t, - #[cfg(target_os = "illumos")] - pub shm_amp: *mut ::c_void, - #[cfg(target_os = "solaris")] - pub shm_flags: ::uintptr_t, - pub shm_lkcnt: ::c_ushort, - pub shm_lpid: ::pid_t, - pub shm_cpid: ::pid_t, - pub shm_nattch: ::shmatt_t, - pub shm_cnattch: ::c_ulong, - pub shm_atime: ::time_t, - pub shm_dtime: ::time_t, - pub shm_ctime: ::time_t, - #[cfg(target_os = "illumos")] - pub shm_pad4: [i64; 4], - #[cfg(target_os = "solaris")] - pub shm_amp: *mut ::c_void, - #[cfg(target_os = "solaris")] - pub shm_gransize: u64, - #[cfg(target_os = "solaris")] - pub shm_allocated: u64, - #[cfg(target_os = "solaris")] - pub shm_pad4: [i64; 1], - } - pub struct sigset_t { bits: [u32; 4], } @@ -276,6 +285,13 @@ s! { pub f_fstr: [::c_char; 32] } + pub struct sendfilevec_t { + pub sfv_fd: ::c_int, + pub sfv_flag: ::c_uint, + pub sfv_off: ::off_t, + pub sfv_len: ::size_t, + } + pub struct sched_param { pub sched_priority: ::c_int, sched_pad: [::c_int; 8] @@ -371,7 +387,7 @@ s! { pub mq_maxmsg: ::c_long, pub mq_msgsize: ::c_long, pub mq_curmsgs: ::c_long, - _pad: [::c_int; 4] + _pad: [::c_int; 12] } pub struct port_event { @@ -382,9 +398,9 @@ s! { pub portev_user: *mut ::c_void, } - pub struct door_desc_t__d_data__d_desc { - pub d_descriptor: ::c_int, - pub d_id: ::door_id_t + pub struct port_notify { + pub portnfy_port: ::c_int, + pub portnfy_user: *mut ::c_void, } pub struct exit_status { @@ -428,10 +444,46 @@ s! { pub esterror: i32, } + pub struct mmapobj_result_t { + pub mr_addr: ::caddr_t, + pub mr_msize: ::size_t, + pub mr_fsize: ::size_t, + pub mr_offset: ::size_t, + pub mr_prot: ::c_uint, + pub mr_flags: ::c_uint, + } + + pub struct lgrp_affinity_args { + pub idtype: ::idtype_t, + pub id: ::id_t, + pub lgrp: ::lgrp_id_t, + pub aff: ::lgrp_affinity_t, + } + + pub struct processor_info_t { + pub pi_state: ::c_int, + pub pi_processor_type: [::c_char; PI_TYPELEN as usize], + pub pi_fputypes: [::c_char; PI_FPUTYPE as usize], + pub pi_clock: ::c_int, + } + + pub struct option { + pub name: *const ::c_char, + pub has_arg: ::c_int, + pub flag: *mut ::c_int, + pub val: ::c_int, + } } s_no_extra_traits! { - #[cfg_attr(any(target_arch = "x86", target_arch = "x86_64"), repr(packed))] + #[cfg_attr(all( + any(target_arch = "x86", target_arch = "x86_64"), + libc_packedN + ), repr(packed(4)))] + #[cfg_attr(all( + any(target_arch = "x86", target_arch = "x86_64"), + not(libc_packedN) + ), repr(packed))] pub struct epoll_event { pub events: u32, pub u64: u64, @@ -478,13 +530,15 @@ s_no_extra_traits! { __ss_pad2: [u8; 240], } + #[cfg_attr(all(target_pointer_width = "64", libc_align), repr(align(8)))] pub struct siginfo_t { pub si_signo: ::c_int, pub si_code: ::c_int, pub si_errno: ::c_int, + #[cfg(target_pointer_width = "64")] pub si_pad: ::c_int, - pub si_addr: *mut ::c_void, - __pad: [u8; 232], + + __data_pad: [::c_int; SIGINFO_DATA_SIZE], } pub struct sockaddr_dl { @@ -506,26 +560,18 @@ s_no_extra_traits! { __sigev_pad2: ::c_int, } - #[cfg_attr(feature = "extra_traits", allow(missing_debug_implementations))] - pub union door_desc_t__d_data { - pub d_desc: door_desc_t__d_data__d_desc, - d_resv: [::c_int; 5], /* Check out /usr/include/sys/door.h */ + #[cfg(libc_union)] + #[cfg_attr(libc_align, repr(align(16)))] + pub union pad128_t { + // pub _q in this structure would be a "long double", of 16 bytes + pub _l: [i32; 4], } - #[cfg_attr(feature = "extra_traits", allow(missing_debug_implementations))] - pub struct door_desc_t { - pub d_attributes: door_attr_t, - pub d_data: door_desc_t__d_data, - } - - #[cfg_attr(feature = "extra_traits", allow(missing_debug_implementations))] - pub struct door_arg_t { - pub data_ptr: *const ::c_char, - pub data_size: ::size_t, - pub desc_ptr: *const door_desc_t, - pub dec_num: ::c_uint, - pub rbuf: *const ::c_char, - pub rsize: ::size_t, + #[cfg(libc_union)] + #[cfg_attr(libc_align, repr(align(16)))] + pub union upad128_t { + // pub _q in this structure would be a "long double", of 16 bytes + pub _l: [u32; 4], } } @@ -743,17 +789,52 @@ cfg_if! { } } + impl siginfo_t { + /// The siginfo_t will have differing contents based on the delivered signal. Based on + /// `si_signo`, this determines how many of the `c_int` pad fields contain valid data + /// exposed by the C unions. + /// + /// It is not yet exhausitive for the OS-defined types, and defaults to assuming the + /// entire data pad area is "valid" for otherwise unrecognized signal numbers. + fn data_field_count(&self) -> usize { + match self.si_signo { + ::SIGSEGV | ::SIGBUS | ::SIGILL | ::SIGTRAP | ::SIGFPE => { + ::mem::size_of::() / ::mem::size_of::<::c_int>() + } + ::SIGCLD => ::mem::size_of::() / ::mem::size_of::<::c_int>(), + ::SIGHUP + | ::SIGINT + | ::SIGQUIT + | ::SIGABRT + | ::SIGSYS + | ::SIGPIPE + | ::SIGALRM + | ::SIGTERM + | ::SIGUSR1 + | ::SIGUSR2 + | ::SIGPWR + | ::SIGWINCH + | ::SIGURG => ::mem::size_of::() / ::mem::size_of::<::c_int>(), + _ => SIGINFO_DATA_SIZE, + } + } + } impl PartialEq for siginfo_t { fn eq(&self, other: &siginfo_t) -> bool { - self.si_signo == other.si_signo + if self.si_signo == other.si_signo && self.si_code == other.si_code - && self.si_errno == other.si_errno - && self.si_addr == other.si_addr - && self - .__pad - .iter() - .zip(other.__pad.iter()) - .all(|(a, b)| a == b) + && self.si_errno == other.si_errno { + // FIXME: The `si_pad` field in the 64-bit version of the struct is ignored + // (for now) when doing comparisons. + + let field_count = self.data_field_count(); + self.__data_pad[..field_count] + .iter() + .zip(other.__data_pad[..field_count].iter()) + .all(|(a, b)| a == b) + } else { + false + } } } impl Eq for siginfo_t {} @@ -763,7 +844,6 @@ cfg_if! { .field("si_signo", &self.si_signo) .field("si_code", &self.si_code) .field("si_errno", &self.si_errno) - .field("si_addr", &self.si_addr) // FIXME: .field("__pad", &self.__pad) .finish() } @@ -773,8 +853,12 @@ cfg_if! { self.si_signo.hash(state); self.si_code.hash(state); self.si_errno.hash(state); - self.si_addr.hash(state); - self.__pad.hash(state); + + // FIXME: The `si_pad` field in the 64-bit version of the struct is ignored + // (for now) when doing hashing. + + let field_count = self.data_field_count(); + self.__data_pad[..field_count].hash(state) } } @@ -852,6 +936,178 @@ cfg_if! { } } + #[cfg(libc_union)] + impl PartialEq for pad128_t { + fn eq(&self, other: &pad128_t) -> bool { + unsafe { + // FIXME: self._q == other._q || + self._l == other._l + } + } + } + #[cfg(libc_union)] + impl Eq for pad128_t {} + #[cfg(libc_union)] + impl ::fmt::Debug for pad128_t { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + unsafe { + f.debug_struct("pad128_t") + // FIXME: .field("_q", &{self._q}) + .field("_l", &{self._l}) + .finish() + } + } + } + #[cfg(libc_union)] + impl ::hash::Hash for pad128_t { + fn hash(&self, state: &mut H) { + unsafe { + // FIXME: state.write_i64(self._q as i64); + self._l.hash(state); + } + } + } + #[cfg(libc_union)] + impl PartialEq for upad128_t { + fn eq(&self, other: &upad128_t) -> bool { + unsafe { + // FIXME: self._q == other._q || + self._l == other._l + } + } + } + #[cfg(libc_union)] + impl Eq for upad128_t {} + #[cfg(libc_union)] + impl ::fmt::Debug for upad128_t { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + unsafe { + f.debug_struct("upad128_t") + // FIXME: .field("_q", &{self._q}) + .field("_l", &{self._l}) + .finish() + } + } + } + #[cfg(libc_union)] + impl ::hash::Hash for upad128_t { + fn hash(&self, state: &mut H) { + unsafe { + // FIXME: state.write_i64(self._q as i64); + self._l.hash(state); + } + } + } + } +} + +cfg_if! { + if #[cfg(target_pointer_width = "64")] { + const SIGINFO_DATA_SIZE: usize = 60; + } else { + const SIGINFO_DATA_SIZE: usize = 29; + } +} + +#[repr(C)] +struct siginfo_fault { + addr: *mut ::c_void, + trapno: ::c_int, + pc: *mut ::caddr_t, +} +impl ::Copy for siginfo_fault {} +impl ::Clone for siginfo_fault { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +struct siginfo_cldval { + utime: ::clock_t, + status: ::c_int, + stime: ::clock_t, +} +impl ::Copy for siginfo_cldval {} +impl ::Clone for siginfo_cldval { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +struct siginfo_killval { + uid: ::uid_t, + value: ::sigval, + // Pad out to match the SIGCLD value size + _pad: *mut ::c_void, +} +impl ::Copy for siginfo_killval {} +impl ::Clone for siginfo_killval { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +struct siginfo_sigcld { + pid: ::pid_t, + val: siginfo_cldval, + ctid: ::ctid_t, + zoneid: ::zoneid_t, +} +impl ::Copy for siginfo_sigcld {} +impl ::Clone for siginfo_sigcld { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +struct siginfo_kill { + pid: ::pid_t, + val: siginfo_killval, + ctid: ::ctid_t, + zoneid: ::zoneid_t, +} +impl ::Copy for siginfo_kill {} +impl ::Clone for siginfo_kill { + fn clone(&self) -> Self { + *self + } +} + +impl siginfo_t { + unsafe fn sidata(&self) -> T { + *((&self.__data_pad) as *const ::c_int as *const T) + } + pub unsafe fn si_addr(&self) -> *mut ::c_void { + let sifault: siginfo_fault = self.sidata(); + sifault.addr + } + pub unsafe fn si_uid(&self) -> ::uid_t { + let kill: siginfo_kill = self.sidata(); + kill.val.uid + } + pub unsafe fn si_value(&self) -> ::sigval { + let kill: siginfo_kill = self.sidata(); + kill.val.value + } + pub unsafe fn si_pid(&self) -> ::pid_t { + let sigcld: siginfo_sigcld = self.sidata(); + sigcld.pid + } + pub unsafe fn si_status(&self) -> ::c_int { + let sigcld: siginfo_sigcld = self.sidata(); + sigcld.val.status + } + pub unsafe fn si_utime(&self) -> ::c_long { + let sigcld: siginfo_sigcld = self.sidata(); + sigcld.val.utime + } + pub unsafe fn si_stime(&self) -> ::c_long { + let sigcld: siginfo_sigcld = self.sidata(); + sigcld.val.stime } } @@ -963,6 +1219,7 @@ pub const FIOSETOWN: ::c_int = 0x8004667c; pub const FIOGETOWN: ::c_int = 0x4004667b; pub const SIGCHLD: ::c_int = 18; +pub const SIGCLD: ::c_int = ::SIGCHLD; pub const SIGBUS: ::c_int = 10; pub const SIGINFO: ::c_int = 41; pub const SIG_BLOCK: ::c_int = 1; @@ -973,10 +1230,22 @@ pub const SIGEV_NONE: ::c_int = 1; pub const SIGEV_SIGNAL: ::c_int = 2; pub const SIGEV_THREAD: ::c_int = 3; +pub const CLD_EXITED: ::c_int = 1; +pub const CLD_KILLED: ::c_int = 2; +pub const CLD_DUMPED: ::c_int = 3; +pub const CLD_TRAPPED: ::c_int = 4; +pub const CLD_STOPPED: ::c_int = 5; +pub const CLD_CONTINUED: ::c_int = 6; + +pub const IP_RECVDSTADDR: ::c_int = 0x7; +pub const IP_SEC_OPT: ::c_int = 0x22; + pub const IPV6_UNICAST_HOPS: ::c_int = 0x5; pub const IPV6_MULTICAST_IF: ::c_int = 0x6; pub const IPV6_MULTICAST_HOPS: ::c_int = 0x7; pub const IPV6_MULTICAST_LOOP: ::c_int = 0x8; +pub const IPV6_RECVPKTINFO: ::c_int = 0x12; +pub const IPV6_SEC_OPT: ::c_int = 0x22; pub const IPV6_V6ONLY: ::c_int = 0x27; cfg_if! { @@ -991,6 +1260,7 @@ pub const ST_RDONLY: ::c_ulong = 1; pub const ST_NOSUID: ::c_ulong = 2; pub const NI_MAXHOST: ::socklen_t = 1025; +pub const NI_MAXSERV: ::socklen_t = 32; pub const EXIT_FAILURE: ::c_int = 1; pub const EXIT_SUCCESS: ::c_int = 0; @@ -999,6 +1269,8 @@ pub const EOF: ::c_int = -1; pub const SEEK_SET: ::c_int = 0; pub const SEEK_CUR: ::c_int = 1; pub const SEEK_END: ::c_int = 2; +pub const SEEK_DATA: ::c_int = 3; +pub const SEEK_HOLE: ::c_int = 4; pub const _IOFBF: ::c_int = 0; pub const _IONBF: ::c_int = 4; pub const _IOLBF: ::c_int = 64; @@ -1008,6 +1280,9 @@ pub const FILENAME_MAX: ::c_uint = 1024; pub const L_tmpnam: ::c_uint = 25; pub const TMP_MAX: ::c_uint = 17576; +pub const GRND_NONBLOCK: ::c_int = 0x0001; +pub const GRND_RANDOM: ::c_int = 0x0002; + pub const O_RDONLY: ::c_int = 0; pub const O_WRONLY: ::c_int = 1; pub const O_RDWR: ::c_int = 2; @@ -1018,11 +1293,13 @@ pub const O_CREAT: ::c_int = 256; pub const O_EXCL: ::c_int = 1024; pub const O_NOCTTY: ::c_int = 2048; pub const O_TRUNC: ::c_int = 512; -pub const O_NOFOLLOW: ::c_int = 0x200000; +pub const O_NOFOLLOW: ::c_int = 0x20000; +pub const O_DIRECTORY: ::c_int = 0x1000000; pub const O_SEARCH: ::c_int = 0x200000; pub const O_EXEC: ::c_int = 0x400000; pub const O_CLOEXEC: ::c_int = 0x800000; pub const O_ACCMODE: ::c_int = 0x600003; +pub const O_XATTR: ::c_int = 0x4000; pub const S_IFIFO: mode_t = 4096; pub const S_IFCHR: mode_t = 8192; pub const S_IFBLK: mode_t = 24576; @@ -1057,10 +1334,23 @@ pub const F_LOCK: ::c_int = 1; pub const F_TEST: ::c_int = 3; pub const F_TLOCK: ::c_int = 2; pub const F_ULOCK: ::c_int = 0; -pub const F_DUPFD_CLOEXEC: ::c_int = 37; pub const F_SETLK: ::c_int = 6; pub const F_SETLKW: ::c_int = 7; pub const F_GETLK: ::c_int = 14; +pub const F_ALLOCSP: ::c_int = 10; +pub const F_FREESP: ::c_int = 11; +pub const F_BLOCKS: ::c_int = 18; +pub const F_BLKSIZE: ::c_int = 19; +pub const F_SHARE: ::c_int = 40; +pub const F_UNSHARE: ::c_int = 41; +pub const F_ISSTREAM: ::c_int = 13; +pub const F_PRIV: ::c_int = 15; +pub const F_NPRIV: ::c_int = 16; +pub const F_QUOTACTL: ::c_int = 17; +pub const F_GETOWN: ::c_int = 23; +pub const F_SETOWN: ::c_int = 24; +pub const F_REVOKE: ::c_int = 25; +pub const F_HASREMOTELOCKS: ::c_int = 26; pub const SIGHUP: ::c_int = 1; pub const SIGINT: ::c_int = 2; pub const SIGQUIT: ::c_int = 3; @@ -1124,6 +1414,20 @@ pub const P_CTID: idtype_t = 13; pub const P_CPUID: idtype_t = 14; pub const P_PSETID: idtype_t = 15; +pub const PBIND_NONE: ::processorid_t = -1; +pub const PBIND_QUERY: ::processorid_t = -2; +pub const PBIND_HARD: ::processorid_t = -3; +pub const PBIND_SOFT: ::processorid_t = -4; + +pub const PS_NONE: ::c_int = -1; +pub const PS_QUERY: ::c_int = -2; +pub const PS_MYID: ::c_int = -3; +pub const PS_SOFT: ::c_int = -4; +pub const PS_HARD: ::c_int = -5; +pub const PS_QUERY_TYPE: ::c_int = -6; +pub const PS_SYSTEM: ::c_int = 1; +pub const PS_PRIVATE: ::c_int = 2; + pub const UTIME_OMIT: c_long = -2; pub const UTIME_NOW: c_long = -1; @@ -1143,6 +1447,7 @@ pub const MAP_RENAME: ::c_int = 0x20; pub const MAP_ALIGN: ::c_int = 0x200; pub const MAP_TEXT: ::c_int = 0x400; pub const MAP_INITDATA: ::c_int = 0x800; +pub const MAP_32BIT: ::c_int = 0x80; pub const MAP_FAILED: *mut ::c_void = !0 as *mut ::c_void; pub const MCL_CURRENT: ::c_int = 0x0001; @@ -1151,7 +1456,11 @@ pub const MCL_FUTURE: ::c_int = 0x0002; pub const MS_SYNC: ::c_int = 0x0004; pub const MS_ASYNC: ::c_int = 0x0001; pub const MS_INVALIDATE: ::c_int = 0x0002; -pub const MS_INVALCURPROC: ::c_int = 0x0008; + +pub const MMOBJ_PADDING: ::c_uint = 0x10000; +pub const MMOBJ_INTERPRET: ::c_uint = 0x20000; +pub const MR_PADDING: ::c_uint = 0x1; +pub const MR_HDR_ELF: ::c_uint = 0x2; pub const EPERM: ::c_int = 1; pub const ENOENT: ::c_int = 2; @@ -1289,11 +1598,21 @@ pub const EAI_SOCKTYPE: ::c_int = 10; pub const EAI_SYSTEM: ::c_int = 11; pub const EAI_OVERFLOW: ::c_int = 12; +pub const NI_NOFQDN: ::c_uint = 0x0001; +pub const NI_NUMERICHOST: ::c_uint = 0x0002; +pub const NI_NAMEREQD: ::c_uint = 0x0004; +pub const NI_NUMERICSERV: ::c_uint = 0x0008; +pub const NI_DGRAM: ::c_uint = 0x0010; +pub const NI_WITHSCOPEID: ::c_uint = 0x0020; +pub const NI_NUMERICSCOPE: ::c_uint = 0x0040; + pub const F_DUPFD: ::c_int = 0; +pub const F_DUP2FD: ::c_int = 9; pub const F_GETFD: ::c_int = 1; pub const F_SETFD: ::c_int = 2; pub const F_GETFL: ::c_int = 3; pub const F_SETFL: ::c_int = 4; +pub const F_GETXFL: ::c_int = 45; pub const SIGTRAP: ::c_int = 5; @@ -1361,7 +1680,7 @@ pub const RLIMIT_AS: ::c_int = RLIMIT_VMEM; #[deprecated(since = "0.2.64", note = "Not stable across OS versions")] pub const RLIM_NLIMITS: rlim_t = 7; -pub const RLIM_INFINITY: rlim_t = 0x7fffffff; +pub const RLIM_INFINITY: rlim_t = 0xfffffffffffffffd; pub const RUSAGE_SELF: ::c_int = 0; pub const RUSAGE_CHILDREN: ::c_int = -1; @@ -1372,11 +1691,12 @@ pub const MADV_SEQUENTIAL: ::c_int = 2; pub const MADV_WILLNEED: ::c_int = 3; pub const MADV_DONTNEED: ::c_int = 4; pub const MADV_FREE: ::c_int = 5; +pub const MADV_ACCESS_DEFAULT: ::c_int = 6; +pub const MADV_ACCESS_LWP: ::c_int = 7; +pub const MADV_ACCESS_MANY: ::c_int = 8; pub const AF_UNSPEC: ::c_int = 0; pub const AF_UNIX: ::c_int = 1; -pub const AF_LOCAL: ::c_int = 0; -pub const AF_FILE: ::c_int = 0; pub const AF_INET: ::c_int = 2; pub const AF_IMPLINK: ::c_int = 3; pub const AF_PUP: ::c_int = 4; @@ -1408,7 +1728,42 @@ pub const AF_POLICY: ::c_int = 29; pub const AF_INET_OFFLOAD: ::c_int = 30; pub const AF_TRILL: ::c_int = 31; pub const AF_PACKET: ::c_int = 32; -pub const AF_LX_NETLINK: ::c_int = 33; + +pub const PF_UNSPEC: ::c_int = AF_UNSPEC; +pub const PF_UNIX: ::c_int = AF_UNIX; +pub const PF_LOCAL: ::c_int = PF_UNIX; +pub const PF_FILE: ::c_int = PF_UNIX; +pub const PF_INET: ::c_int = AF_INET; +pub const PF_IMPLINK: ::c_int = AF_IMPLINK; +pub const PF_PUP: ::c_int = AF_PUP; +pub const PF_CHAOS: ::c_int = AF_CHAOS; +pub const PF_NS: ::c_int = AF_NS; +pub const PF_NBS: ::c_int = AF_NBS; +pub const PF_ECMA: ::c_int = AF_ECMA; +pub const PF_DATAKIT: ::c_int = AF_DATAKIT; +pub const PF_CCITT: ::c_int = AF_CCITT; +pub const PF_SNA: ::c_int = AF_SNA; +pub const PF_DECnet: ::c_int = AF_DECnet; +pub const PF_DLI: ::c_int = AF_DLI; +pub const PF_LAT: ::c_int = AF_LAT; +pub const PF_HYLINK: ::c_int = AF_HYLINK; +pub const PF_APPLETALK: ::c_int = AF_APPLETALK; +pub const PF_NIT: ::c_int = AF_NIT; +pub const PF_802: ::c_int = AF_802; +pub const PF_OSI: ::c_int = AF_OSI; +pub const PF_X25: ::c_int = AF_X25; +pub const PF_OSINET: ::c_int = AF_OSINET; +pub const PF_GOSIP: ::c_int = AF_GOSIP; +pub const PF_IPX: ::c_int = AF_IPX; +pub const PF_ROUTE: ::c_int = AF_ROUTE; +pub const PF_LINK: ::c_int = AF_LINK; +pub const PF_INET6: ::c_int = AF_INET6; +pub const PF_KEY: ::c_int = AF_KEY; +pub const PF_NCA: ::c_int = AF_NCA; +pub const PF_POLICY: ::c_int = AF_POLICY; +pub const PF_INET_OFFLOAD: ::c_int = AF_INET_OFFLOAD; +pub const PF_TRILL: ::c_int = AF_TRILL; +pub const PF_PACKET: ::c_int = AF_PACKET; pub const SOCK_DGRAM: ::c_int = 1; pub const SOCK_STREAM: ::c_int = 2; @@ -1424,9 +1779,34 @@ pub const IP_ADD_MEMBERSHIP: ::c_int = 19; pub const IP_DROP_MEMBERSHIP: ::c_int = 20; pub const IPV6_JOIN_GROUP: ::c_int = 9; pub const IPV6_LEAVE_GROUP: ::c_int = 10; +pub const IP_ADD_SOURCE_MEMBERSHIP: ::c_int = 23; +pub const IP_DROP_SOURCE_MEMBERSHIP: ::c_int = 24; +pub const IP_BLOCK_SOURCE: ::c_int = 21; +pub const IP_UNBLOCK_SOURCE: ::c_int = 22; + +// These TCP socket options are common between illumos and Solaris, while higher +// numbers have generally diverged: +pub const TCP_NODELAY: ::c_int = 0x1; +pub const TCP_MAXSEG: ::c_int = 0x2; +pub const TCP_KEEPALIVE: ::c_int = 0x8; +pub const TCP_NOTIFY_THRESHOLD: ::c_int = 0x10; +pub const TCP_ABORT_THRESHOLD: ::c_int = 0x11; +pub const TCP_CONN_NOTIFY_THRESHOLD: ::c_int = 0x12; +pub const TCP_CONN_ABORT_THRESHOLD: ::c_int = 0x13; +pub const TCP_RECVDSTADDR: ::c_int = 0x14; +pub const TCP_INIT_CWND: ::c_int = 0x15; +pub const TCP_KEEPALIVE_THRESHOLD: ::c_int = 0x16; +pub const TCP_KEEPALIVE_ABORT_THRESHOLD: ::c_int = 0x17; +pub const TCP_CORK: ::c_int = 0x18; +pub const TCP_RTO_INITIAL: ::c_int = 0x19; +pub const TCP_RTO_MIN: ::c_int = 0x1a; +pub const TCP_RTO_MAX: ::c_int = 0x1b; +pub const TCP_LINGER2: ::c_int = 0x1c; + +pub const UDP_NAT_T_ENDPOINT: ::c_int = 0x0103; + +pub const SOMAXCONN: ::c_int = 128; -pub const TCP_NODELAY: ::c_int = 1; -pub const TCP_KEEPIDLE: ::c_int = 34; pub const SOL_SOCKET: ::c_int = 0xffff; pub const SO_DEBUG: ::c_int = 0x01; pub const SO_ACCEPTCONN: ::c_int = 0x0002; @@ -1465,6 +1845,9 @@ pub const MSG_DUPCTRL: ::c_int = 0x800; pub const MSG_XPG4_2: ::c_int = 0x8000; pub const MSG_MAXIOVLEN: ::c_int = 16; +pub const IF_NAMESIZE: ::size_t = 32; +pub const IFNAMSIZ: ::size_t = 16; + // https://docs.oracle.com/cd/E23824_01/html/821-1475/if-7p.html pub const IFF_UP: ::c_int = 0x0000000001; // Address is up pub const IFF_BROADCAST: ::c_int = 0x0000000002; // Broadcast address valid @@ -1520,6 +1903,14 @@ pub const IPC_RMID: ::c_int = 10; pub const IPC_SET: ::c_int = 11; pub const IPC_SEAT: ::c_int = 12; +// sys/shm.h +pub const SHM_R: ::c_int = 0o400; +pub const SHM_W: ::c_int = 0o200; +pub const SHM_RDONLY: ::c_int = 0o10000; +pub const SHM_RND: ::c_int = 0o20000; +pub const SHM_SHARE_MMU: ::c_int = 0o40000; +pub const SHM_PAGEABLE: ::c_int = 0o100000; + pub const SHUT_RD: ::c_int = 0; pub const SHUT_WR: ::c_int = 1; pub const SHUT_RDWR: ::c_int = 2; @@ -1780,8 +2171,6 @@ pub const PORT_SOURCE_FD: ::c_int = 4; pub const PORT_SOURCE_ALERT: ::c_int = 5; pub const PORT_SOURCE_MQ: ::c_int = 6; pub const PORT_SOURCE_FILE: ::c_int = 7; -pub const PORT_SOURCE_POSTWAIT: ::c_int = 8; -pub const PORT_SOURCE_SIGNAL: ::c_int = 9; pub const NONROOT_USR: ::c_short = 2; pub const _UTX_USERSIZE: usize = 32; @@ -1832,8 +2221,6 @@ pub const TIOCGWINSZ: ::c_int = _TIOC | 104; pub const TIOCSWINSZ: ::c_int = _TIOC | 103; pub const TIOCGSOFTCAR: ::c_int = _TIOC | 105; pub const TIOCSSOFTCAR: ::c_int = _TIOC | 106; -pub const TIOCSETLD: ::c_int = _TIOC | 123; -pub const TIOCGETLD: ::c_int = _TIOC | 124; pub const TIOCGPPS: ::c_int = _TIOC | 125; pub const TIOCSPPS: ::c_int = _TIOC | 126; pub const TIOCGPPSEV: ::c_int = _TIOC | 127; @@ -1876,6 +2263,18 @@ pub const TIOCMGET: ::c_int = tIOC | 29; pub const TIOCREMOTE: ::c_int = tIOC | 30; pub const TIOCSIGNAL: ::c_int = tIOC | 31; +pub const TIOCM_LE: ::c_int = 0o0001; +pub const TIOCM_DTR: ::c_int = 0o0002; +pub const TIOCM_RTS: ::c_int = 0o0004; +pub const TIOCM_ST: ::c_int = 0o0010; +pub const TIOCM_SR: ::c_int = 0o0020; +pub const TIOCM_CTS: ::c_int = 0o0040; +pub const TIOCM_CAR: ::c_int = 0o0100; +pub const TIOCM_CD: ::c_int = TIOCM_CAR; +pub const TIOCM_RNG: ::c_int = 0o0200; +pub const TIOCM_RI: ::c_int = TIOCM_RNG; +pub const TIOCM_DSR: ::c_int = 0o0400; + pub const EPOLLIN: ::c_int = 0x1; pub const EPOLLPRI: ::c_int = 0x2; pub const EPOLLOUT: ::c_int = 0x4; @@ -1888,8 +2287,9 @@ pub const EPOLLERR: ::c_int = 0x8; pub const EPOLLHUP: ::c_int = 0x10; pub const EPOLLET: ::c_int = 0x80000000; pub const EPOLLRDHUP: ::c_int = 0x2000; -pub const EPOLLEXCLUSIVE: ::c_int = 0x10000000; pub const EPOLLONESHOT: ::c_int = 0x40000000; +pub const EPOLLWAKEUP: ::c_int = 0x20000000; +pub const EPOLLEXCLUSIVE: ::c_int = 0x10000000; pub const EPOLL_CLOEXEC: ::c_int = 0x80000; pub const EPOLL_CTL_ADD: ::c_int = 1; pub const EPOLL_CTL_MOD: ::c_int = 3; @@ -1920,9 +2320,13 @@ pub const B230400: speed_t = 20; pub const B307200: speed_t = 21; pub const B460800: speed_t = 22; pub const B921600: speed_t = 23; -pub const CSTART: ::tcflag_t = 021; -pub const CSTOP: ::tcflag_t = 023; -pub const CSWTCH: ::tcflag_t = 032; +pub const CSTART: ::tcflag_t = 0o21; +pub const CSTOP: ::tcflag_t = 0o23; +pub const CSWTCH: ::tcflag_t = 0o32; +pub const CBAUD: ::tcflag_t = 0o17; +pub const CIBAUD: ::tcflag_t = 0o3600000; +pub const CBAUDEXT: ::tcflag_t = 0o10000000; +pub const CIBAUDEXT: ::tcflag_t = 0o20000000; pub const CSIZE: ::tcflag_t = 0o000060; pub const CS5: ::tcflag_t = 0; pub const CS6: ::tcflag_t = 0o000020; @@ -1946,20 +2350,26 @@ pub const ISTRIP: ::tcflag_t = 0o000040; pub const INLCR: ::tcflag_t = 0o000100; pub const IGNCR: ::tcflag_t = 0o000200; pub const ICRNL: ::tcflag_t = 0o000400; +pub const IUCLC: ::tcflag_t = 0o001000; pub const IXON: ::tcflag_t = 0o002000; pub const IXOFF: ::tcflag_t = 0o010000; pub const IXANY: ::tcflag_t = 0o004000; pub const IMAXBEL: ::tcflag_t = 0o020000; +pub const DOSMODE: ::tcflag_t = 0o100000; pub const OPOST: ::tcflag_t = 0o000001; +pub const OLCUC: ::tcflag_t = 0o000002; pub const ONLCR: ::tcflag_t = 0o000004; pub const OCRNL: ::tcflag_t = 0o000010; pub const ONOCR: ::tcflag_t = 0o000020; pub const ONLRET: ::tcflag_t = 0o000040; +pub const OFILL: ::tcflag_t = 0o0000100; +pub const OFDEL: ::tcflag_t = 0o0000200; pub const CREAD: ::tcflag_t = 0o000200; pub const PARENB: ::tcflag_t = 0o000400; pub const PARODD: ::tcflag_t = 0o001000; pub const HUPCL: ::tcflag_t = 0o002000; pub const CLOCAL: ::tcflag_t = 0o004000; +pub const CRTSXOFF: ::tcflag_t = 0o10000000000; pub const CRTSCTS: ::tcflag_t = 0o20000000000; pub const ISIG: ::tcflag_t = 0o000001; pub const ICANON: ::tcflag_t = 0o000002; @@ -1989,6 +2399,44 @@ pub const VLNEXT: usize = 15; pub const VSTATUS: usize = 16; pub const VERASE2: usize = 17; +// +const STR: ::c_int = (b'S' as ::c_int) << 8; +pub const I_NREAD: ::c_int = STR | 0o1; +pub const I_PUSH: ::c_int = STR | 0o2; +pub const I_POP: ::c_int = STR | 0o3; +pub const I_LOOK: ::c_int = STR | 0o4; +pub const I_FLUSH: ::c_int = STR | 0o5; +pub const I_SRDOPT: ::c_int = STR | 0o6; +pub const I_GRDOPT: ::c_int = STR | 0o7; +pub const I_STR: ::c_int = STR | 0o10; +pub const I_SETSIG: ::c_int = STR | 0o11; +pub const I_GETSIG: ::c_int = STR | 0o12; +pub const I_FIND: ::c_int = STR | 0o13; +pub const I_LINK: ::c_int = STR | 0o14; +pub const I_UNLINK: ::c_int = STR | 0o15; +pub const I_PEEK: ::c_int = STR | 0o17; +pub const I_FDINSERT: ::c_int = STR | 0o20; +pub const I_SENDFD: ::c_int = STR | 0o21; +pub const I_RECVFD: ::c_int = STR | 0o16; +pub const I_SWROPT: ::c_int = STR | 0o23; +pub const I_GWROPT: ::c_int = STR | 0o24; +pub const I_LIST: ::c_int = STR | 0o25; +pub const I_PLINK: ::c_int = STR | 0o26; +pub const I_PUNLINK: ::c_int = STR | 0o27; +pub const I_ANCHOR: ::c_int = STR | 0o30; +pub const I_FLUSHBAND: ::c_int = STR | 0o34; +pub const I_CKBAND: ::c_int = STR | 0o35; +pub const I_GETBAND: ::c_int = STR | 0o36; +pub const I_ATMARK: ::c_int = STR | 0o37; +pub const I_SETCLTIME: ::c_int = STR | 0o40; +pub const I_GETCLTIME: ::c_int = STR | 0o41; +pub const I_CANPUT: ::c_int = STR | 0o42; +pub const I_SERROPT: ::c_int = STR | 0o43; +pub const I_GERROPT: ::c_int = STR | 0o44; +pub const I_ESETSIG: ::c_int = STR | 0o45; +pub const I_EGETSIG: ::c_int = STR | 0o46; +pub const __I_PUSH_NOCTTY: ::c_int = STR | 0o47; + // 3SOCKET flags pub const SOCK_CLOEXEC: ::c_int = 0x080000; pub const SOCK_NONBLOCK: ::c_int = 0x100000; @@ -2034,11 +2482,8 @@ pub const STA_PPSJITTER: i32 = 0x0200; pub const STA_PPSWANDER: i32 = 0x0400; pub const STA_PPSERROR: i32 = 0x0800; pub const STA_CLOCKERR: i32 = 0x1000; -pub const STA_RONLY: i32 = STA_PPSSIGNAL - | STA_PPSJITTER - | STA_PPSWANDER - | STA_PPSERROR - | STA_CLOCKERR; +pub const STA_RONLY: i32 = + STA_PPSSIGNAL | STA_PPSJITTER | STA_PPSWANDER | STA_PPSERROR | STA_CLOCKERR; pub const TIME_OK: i32 = 0; pub const TIME_INS: i32 = 1; pub const TIME_DEL: i32 = 2; @@ -2046,7 +2491,149 @@ pub const TIME_OOP: i32 = 3; pub const TIME_WAIT: i32 = 4; pub const TIME_ERROR: i32 = 5; +pub const PRIO_PROCESS: ::c_int = 0; +pub const PRIO_PGRP: ::c_int = 1; +pub const PRIO_USER: ::c_int = 2; + +pub const SCHED_OTHER: ::c_int = 0; +pub const SCHED_FIFO: ::c_int = 1; +pub const SCHED_RR: ::c_int = 2; +pub const SCHED_SYS: ::c_int = 3; +pub const SCHED_IA: ::c_int = 4; +pub const SCHED_FSS: ::c_int = 5; +pub const SCHED_FX: ::c_int = 6; + +// sys/priv.h +pub const PRIV_DEBUG: ::c_uint = 0x0001; +pub const PRIV_AWARE: ::c_uint = 0x0002; +pub const PRIV_AWARE_INHERIT: ::c_uint = 0x0004; +pub const __PROC_PROTECT: ::c_uint = 0x0008; +pub const NET_MAC_AWARE: ::c_uint = 0x0010; +pub const NET_MAC_AWARE_INHERIT: ::c_uint = 0x0020; +pub const PRIV_AWARE_RESET: ::c_uint = 0x0040; +pub const PRIV_XPOLICY: ::c_uint = 0x0080; +pub const PRIV_PFEXEC: ::c_uint = 0x0100; +pub const PRIV_USER: ::c_uint = PRIV_DEBUG + | NET_MAC_AWARE + | NET_MAC_AWARE_INHERIT + | PRIV_XPOLICY + | PRIV_AWARE_RESET + | PRIV_PFEXEC; + +// sys/systeminfo.h +pub const SI_SYSNAME: ::c_int = 1; +pub const SI_HOSTNAME: ::c_int = 2; +pub const SI_RELEASE: ::c_int = 3; +pub const SI_VERSION: ::c_int = 4; +pub const SI_MACHINE: ::c_int = 5; +pub const SI_ARCHITECTURE: ::c_int = 6; +pub const SI_HW_SERIAL: ::c_int = 7; +pub const SI_HW_PROVIDER: ::c_int = 8; +pub const SI_SET_HOSTNAME: ::c_int = 258; +pub const SI_SET_SRPC_DOMAIN: ::c_int = 265; +pub const SI_PLATFORM: ::c_int = 513; +pub const SI_ISALIST: ::c_int = 514; +pub const SI_DHCP_CACHE: ::c_int = 515; +pub const SI_ARCHITECTURE_32: ::c_int = 516; +pub const SI_ARCHITECTURE_64: ::c_int = 517; +pub const SI_ARCHITECTURE_K: ::c_int = 518; +pub const SI_ARCHITECTURE_NATIVE: ::c_int = 519; + +// sys/lgrp_user.h +pub const LGRP_COOKIE_NONE: ::lgrp_cookie_t = 0; +pub const LGRP_AFF_NONE: ::lgrp_affinity_t = 0x0; +pub const LGRP_AFF_WEAK: ::lgrp_affinity_t = 0x10; +pub const LGRP_AFF_STRONG: ::lgrp_affinity_t = 0x100; +pub const LGRP_RSRC_COUNT: ::lgrp_rsrc_t = 2; +pub const LGRP_RSRC_CPU: ::lgrp_rsrc_t = 0; +pub const LGRP_RSRC_MEM: ::lgrp_rsrc_t = 1; +pub const LGRP_CONTENT_ALL: ::lgrp_content_t = 0; +pub const LGRP_CONTENT_HIERARCHY: ::lgrp_content_t = LGRP_CONTENT_ALL; +pub const LGRP_CONTENT_DIRECT: ::lgrp_content_t = 1; +pub const LGRP_LAT_CPU_TO_MEM: ::lgrp_lat_between_t = 0; +pub const LGRP_MEM_SZ_FREE: ::lgrp_mem_size_flag_t = 0; +pub const LGRP_MEM_SZ_INSTALLED: ::lgrp_mem_size_flag_t = 1; +pub const LGRP_VIEW_CALLER: ::lgrp_view_t = 0; +pub const LGRP_VIEW_OS: ::lgrp_view_t = 1; + +// sys/processor.h + +pub const P_OFFLINE: ::c_int = 0x001; +pub const P_ONLINE: ::c_int = 0x002; +pub const P_STATUS: ::c_int = 0x003; +pub const P_FAULTED: ::c_int = 0x004; +pub const P_POWEROFF: ::c_int = 0x005; +pub const P_NOINTR: ::c_int = 0x006; +pub const P_SPARE: ::c_int = 0x007; +pub const P_DISABLED: ::c_int = 0x008; +pub const P_FORCED: ::c_int = 0x10000000; +pub const PI_TYPELEN: ::c_int = 16; +pub const PI_FPUTYPE: ::c_int = 32; + +// sys/auxv.h +pub const AT_SUN_HWCAP: ::c_uint = 2009; +pub const AT_SUN_HWCAP2: ::c_uint = 2023; +pub const AT_SUN_FPTYPE: ::c_uint = 2027; + +// As per sys/socket.h, header alignment must be 8 bytes on SPARC +// and 4 bytes everywhere else: +#[cfg(target_arch = "sparc64")] +const _CMSG_HDR_ALIGNMENT: usize = 8; +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +const _CMSG_HDR_ALIGNMENT: usize = 4; + +const _CMSG_DATA_ALIGNMENT: usize = ::mem::size_of::<::c_int>(); + +const_fn! { + {const} fn _CMSG_HDR_ALIGN(p: usize) -> usize { + (p + _CMSG_HDR_ALIGNMENT - 1) & !(_CMSG_HDR_ALIGNMENT - 1) + } + + {const} fn _CMSG_DATA_ALIGN(p: usize) -> usize { + (p + _CMSG_DATA_ALIGNMENT - 1) & !(_CMSG_DATA_ALIGNMENT - 1) + } +} + f! { + pub fn CMSG_DATA(cmsg: *const ::cmsghdr) -> *mut ::c_uchar { + _CMSG_DATA_ALIGN(cmsg.offset(1) as usize) as *mut ::c_uchar + } + + pub fn CMSG_LEN(length: ::c_uint) -> ::c_uint { + _CMSG_DATA_ALIGN(::mem::size_of::<::cmsghdr>()) as ::c_uint + length + } + + pub fn CMSG_FIRSTHDR(mhdr: *const ::msghdr) -> *mut ::cmsghdr { + if ((*mhdr).msg_controllen as usize) < ::mem::size_of::<::cmsghdr>() { + 0 as *mut ::cmsghdr + } else { + (*mhdr).msg_control as *mut ::cmsghdr + } + } + + pub fn CMSG_NXTHDR(mhdr: *const ::msghdr, cmsg: *const ::cmsghdr) + -> *mut ::cmsghdr + { + if cmsg.is_null() { + return ::CMSG_FIRSTHDR(mhdr); + }; + let next = _CMSG_HDR_ALIGN(cmsg as usize + (*cmsg).cmsg_len as usize + + ::mem::size_of::<::cmsghdr>()); + let max = (*mhdr).msg_control as usize + + (*mhdr).msg_controllen as usize; + if next > max { + 0 as *mut ::cmsghdr + } else { + _CMSG_HDR_ALIGN(cmsg as usize + (*cmsg).cmsg_len as usize) + as *mut ::cmsghdr + } + } + + pub {const} fn CMSG_SPACE(length: ::c_uint) -> ::c_uint { + _CMSG_HDR_ALIGN(::mem::size_of::<::cmsghdr>() as usize + + length as usize) as ::c_uint + } + pub fn FD_CLR(fd: ::c_int, set: *mut fd_set) -> () { let bits = ::mem::size_of_val(&(*set).fds_bits[0]) * 8; let fd = fd as usize; @@ -2054,7 +2641,7 @@ f! { return } - pub fn FD_ISSET(fd: ::c_int, set: *mut fd_set) -> bool { + pub fn FD_ISSET(fd: ::c_int, set: *const fd_set) -> bool { let bits = ::mem::size_of_val(&(*set).fds_bits[0]) * 8; let fd = fd as usize; return ((*set).fds_bits[fd / bits] & (1 << (fd % bits))) != 0 @@ -2072,56 +2659,54 @@ f! { *slot = 0; } } +} - pub fn WIFEXITED(status: ::c_int) -> bool { +safe_f! { + pub {const} fn WIFEXITED(status: ::c_int) -> bool { (status & 0xFF) == 0 } - pub fn WEXITSTATUS(status: ::c_int) -> ::c_int { + pub {const} fn WEXITSTATUS(status: ::c_int) -> ::c_int { (status >> 8) & 0xFF } - pub fn WTERMSIG(status: ::c_int) -> ::c_int { + pub {const} fn WTERMSIG(status: ::c_int) -> ::c_int { status & 0x7F } - pub fn WIFCONTINUED(status: ::c_int) -> bool { + pub {const} fn WIFCONTINUED(status: ::c_int) -> bool { (status & 0xffff) == 0xffff } - pub fn WSTOPSIG(status: ::c_int) -> ::c_int { + pub {const} fn WSTOPSIG(status: ::c_int) -> ::c_int { (status & 0xff00) >> 8 } - pub fn WIFSIGNALED(status: ::c_int) -> bool { + pub {const} fn WIFSIGNALED(status: ::c_int) -> bool { ((status & 0xff) > 0) && (status & 0xff00 == 0) } - pub fn WIFSTOPPED(status: ::c_int) -> bool { + pub {const} fn WIFSTOPPED(status: ::c_int) -> bool { ((status & 0xff) == 0x7f) && ((status & 0xff00) != 0) } - pub fn WCOREDUMP(status: ::c_int) -> bool { + pub {const} fn WCOREDUMP(status: ::c_int) -> bool { (status & 0x80) != 0 } + + pub {const} fn MR_GET_TYPE(flags: ::c_uint) -> ::c_uint { + flags & 0x0000ffff + } } extern "C" { pub fn getrlimit(resource: ::c_int, rlim: *mut ::rlimit) -> ::c_int; pub fn setrlimit(resource: ::c_int, rlim: *const ::rlimit) -> ::c_int; - pub fn strerror_r( - errnum: ::c_int, - buf: *mut c_char, - buflen: ::size_t, - ) -> ::c_int; + pub fn strerror_r(errnum: ::c_int, buf: *mut c_char, buflen: ::size_t) -> ::c_int; pub fn sem_destroy(sem: *mut sem_t) -> ::c_int; - pub fn sem_init( - sem: *mut sem_t, - pshared: ::c_int, - value: ::c_uint, - ) -> ::c_int; + pub fn sem_init(sem: *mut sem_t, pshared: ::c_int, value: ::c_uint) -> ::c_int; pub fn abs(i: ::c_int) -> ::c_int; pub fn acct(filename: *const ::c_char) -> ::c_int; @@ -2130,6 +2715,8 @@ extern "C" { pub fn labs(i: ::c_long) -> ::c_long; pub fn rand() -> ::c_int; pub fn srand(seed: ::c_uint); + pub fn getentropy(buf: *mut ::c_void, buflen: ::size_t) -> ::c_int; + pub fn getrandom(bbuf: *mut ::c_void, buflen: ::size_t, flags: ::c_uint) -> ::ssize_t; pub fn gettimeofday(tp: *mut ::timeval, tz: *mut ::c_void) -> ::c_int; pub fn settimeofday(tp: *const ::timeval, tz: *const ::c_void) -> ::c_int; @@ -2137,19 +2724,16 @@ extern "C" { pub fn freeifaddrs(ifa: *mut ::ifaddrs); pub fn stack_getbounds(sp: *mut ::stack_t) -> ::c_int; - pub fn mincore( - addr: *const ::c_void, - len: ::size_t, - vec: *mut c_char, + pub fn getgrouplist( + name: *const ::c_char, + basegid: ::gid_t, + groups: *mut ::gid_t, + ngroups: *mut ::c_int, ) -> ::c_int; pub fn initgroups(name: *const ::c_char, basegid: ::gid_t) -> ::c_int; pub fn setgroups(ngroups: ::c_int, ptr: *const ::gid_t) -> ::c_int; pub fn ioctl(fildes: ::c_int, request: ::c_int, ...) -> ::c_int; - pub fn mprotect( - addr: *const ::c_void, - len: ::size_t, - prot: ::c_int, - ) -> ::c_int; + pub fn mprotect(addr: *const ::c_void, len: ::size_t, prot: ::c_int) -> ::c_int; pub fn ___errno() -> *mut ::c_int; pub fn clock_getres(clk_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; pub fn clock_gettime(clk_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; @@ -2159,10 +2743,7 @@ extern "C" { rqtp: *const ::timespec, rmtp: *mut ::timespec, ) -> ::c_int; - pub fn clock_settime( - clk_id: ::clockid_t, - tp: *const ::timespec, - ) -> ::c_int; + pub fn clock_settime(clk_id: ::clockid_t, tp: *const ::timespec) -> ::c_int; pub fn getnameinfo( sa: *const ::sockaddr, salen: ::socklen_t, @@ -2176,22 +2757,16 @@ extern "C" { pub fn endpwent(); pub fn getpwent() -> *mut passwd; pub fn fdatasync(fd: ::c_int) -> ::c_int; - pub fn nl_langinfo_l(item: ::nl_item, locale: ::locale_t) - -> *mut ::c_char; + pub fn nl_langinfo_l(item: ::nl_item, locale: ::locale_t) -> *mut ::c_char; pub fn duplocale(base: ::locale_t) -> ::locale_t; pub fn freelocale(loc: ::locale_t); - pub fn newlocale( - mask: ::c_int, - locale: *const ::c_char, - base: ::locale_t, - ) -> ::locale_t; + pub fn newlocale(mask: ::c_int, locale: *const ::c_char, base: ::locale_t) -> ::locale_t; pub fn uselocale(loc: ::locale_t) -> ::locale_t; pub fn getprogname() -> *const ::c_char; pub fn setprogname(name: *const ::c_char); pub fn getloadavg(loadavg: *mut ::c_double, nelem: ::c_int) -> ::c_int; pub fn getpriority(which: ::c_int, who: ::c_int) -> ::c_int; - pub fn setpriority(which: ::c_int, who: ::c_int, prio: ::c_int) - -> ::c_int; + pub fn setpriority(which: ::c_int, who: ::c_int, prio: ::c_int) -> ::c_int; pub fn mknodat( dirfd: ::c_int, @@ -2199,11 +2774,7 @@ extern "C" { mode: ::mode_t, dev: dev_t, ) -> ::c_int; - pub fn mkfifoat( - dirfd: ::c_int, - pathname: *const ::c_char, - mode: ::mode_t, - ) -> ::c_int; + pub fn mkfifoat(dirfd: ::c_int, pathname: *const ::c_char, mode: ::mode_t) -> ::c_int; pub fn sethostname(name: *const ::c_char, len: ::c_int) -> ::c_int; pub fn if_nameindex() -> *mut if_nameindex; pub fn if_freenameindex(ptr: *mut if_nameindex); @@ -2213,6 +2784,11 @@ extern "C" { f: extern "C" fn(*mut ::c_void) -> *mut ::c_void, value: *mut ::c_void, ) -> ::c_int; + pub fn pthread_attr_getstack( + attr: *const ::pthread_attr_t, + stackaddr: *mut *mut ::c_void, + stacksize: *mut ::size_t, + ) -> ::c_int; pub fn pthread_condattr_getclock( attr: *const pthread_condattr_t, clock_id: *mut clockid_t, @@ -2221,76 +2797,47 @@ extern "C" { attr: *mut pthread_condattr_t, clock_id: ::clockid_t, ) -> ::c_int; - pub fn sem_timedwait( - sem: *mut sem_t, - abstime: *const ::timespec, - ) -> ::c_int; + pub fn sem_timedwait(sem: *mut sem_t, abstime: *const ::timespec) -> ::c_int; pub fn sem_getvalue(sem: *mut sem_t, sval: *mut ::c_int) -> ::c_int; pub fn pthread_mutex_timedlock( lock: *mut pthread_mutex_t, abstime: *const ::timespec, ) -> ::c_int; - pub fn waitid( - idtype: idtype_t, - id: id_t, - infop: *mut ::siginfo_t, - options: ::c_int, - ) -> ::c_int; + pub fn pthread_getname_np(tid: ::pthread_t, name: *mut ::c_char, len: ::size_t) -> ::c_int; + pub fn pthread_setname_np(tid: ::pthread_t, name: *const ::c_char) -> ::c_int; + pub fn waitid(idtype: idtype_t, id: id_t, infop: *mut ::siginfo_t, options: ::c_int) + -> ::c_int; + #[cfg_attr(target_os = "illumos", link_name = "_glob_ext")] pub fn glob( pattern: *const ::c_char, flags: ::c_int, - errfunc: ::Option< - extern "C" fn(epath: *const ::c_char, errno: ::c_int) -> ::c_int, - >, + errfunc: ::Option ::c_int>, pglob: *mut ::glob_t, ) -> ::c_int; + #[cfg_attr(target_os = "illumos", link_name = "_globfree_ext")] pub fn globfree(pglob: *mut ::glob_t); - pub fn posix_madvise( - addr: *mut ::c_void, - len: ::size_t, - advice: ::c_int, - ) -> ::c_int; + pub fn posix_madvise(addr: *mut ::c_void, len: ::size_t, advice: ::c_int) -> ::c_int; - pub fn shmat( - shmid: ::c_int, - shmaddr: *const ::c_void, - shmflg: ::c_int, - ) -> *mut ::c_void; + pub fn shmat(shmid: ::c_int, shmaddr: *const ::c_void, shmflg: ::c_int) -> *mut ::c_void; - pub fn shmctl( - shmid: ::c_int, - cmd: ::c_int, - buf: *mut ::shmid_ds, - ) -> ::c_int; + pub fn shmctl(shmid: ::c_int, cmd: ::c_int, buf: *mut ::shmid_ds) -> ::c_int; pub fn shmdt(shmaddr: *const ::c_void) -> ::c_int; pub fn shmget(key: key_t, size: ::size_t, shmflg: ::c_int) -> ::c_int; - pub fn shm_open( - name: *const ::c_char, - oflag: ::c_int, - mode: ::mode_t, - ) -> ::c_int; + pub fn shm_open(name: *const ::c_char, oflag: ::c_int, mode: ::mode_t) -> ::c_int; pub fn shm_unlink(name: *const ::c_char) -> ::c_int; pub fn seekdir(dirp: *mut ::DIR, loc: ::c_long); pub fn telldir(dirp: *mut ::DIR) -> ::c_long; - pub fn madvise( - addr: *mut ::c_void, - len: ::size_t, - advice: ::c_int, - ) -> ::c_int; + pub fn madvise(addr: *mut ::c_void, len: ::size_t, advice: ::c_int) -> ::c_int; - pub fn msync( - addr: *mut ::c_void, - len: ::size_t, - flags: ::c_int, - ) -> ::c_int; + pub fn msync(addr: *mut ::c_void, len: ::size_t, flags: ::c_int) -> ::c_int; pub fn memalign(align: ::size_t, size: ::size_t) -> *mut ::c_void; @@ -2303,11 +2850,7 @@ extern "C" { addrlen: *mut ::socklen_t, ) -> ::ssize_t; pub fn mkstemps(template: *mut ::c_char, suffixlen: ::c_int) -> ::c_int; - pub fn futimesat( - fd: ::c_int, - path: *const ::c_char, - times: *const ::timeval, - ) -> ::c_int; + pub fn futimesat(fd: ::c_int, path: *const ::c_char, times: *const ::timeval) -> ::c_int; pub fn futimens(dirfd: ::c_int, times: *const ::timespec) -> ::c_int; pub fn utimensat( dirfd: ::c_int, @@ -2318,35 +2861,21 @@ extern "C" { pub fn nl_langinfo(item: ::nl_item) -> *mut ::c_char; #[cfg_attr(target_os = "illumos", link_name = "__xnet_bind")] - pub fn bind( - socket: ::c_int, - address: *const ::sockaddr, - address_len: ::socklen_t, - ) -> ::c_int; + pub fn bind(socket: ::c_int, address: *const ::sockaddr, address_len: ::socklen_t) -> ::c_int; - pub fn writev( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - ) -> ::ssize_t; - pub fn readv( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - ) -> ::ssize_t; + pub fn writev(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int) -> ::ssize_t; + pub fn readv(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int) -> ::ssize_t; #[cfg_attr(target_os = "illumos", link_name = "__xnet_sendmsg")] - pub fn sendmsg( - fd: ::c_int, - msg: *const ::msghdr, - flags: ::c_int, - ) -> ::ssize_t; + pub fn sendmsg(fd: ::c_int, msg: *const ::msghdr, flags: ::c_int) -> ::ssize_t; #[cfg_attr(target_os = "illumos", link_name = "__xnet_recvmsg")] - pub fn recvmsg( + pub fn recvmsg(fd: ::c_int, msg: *mut ::msghdr, flags: ::c_int) -> ::ssize_t; + pub fn accept4( fd: ::c_int, - msg: *mut ::msghdr, + address: *mut sockaddr, + address_len: *mut socklen_t, flags: ::c_int, - ) -> ::ssize_t; + ) -> ::c_int; pub fn mq_open(name: *const ::c_char, oflag: ::c_int, ...) -> ::mqd_t; pub fn mq_close(mqd: ::mqd_t) -> ::c_int; @@ -2378,11 +2907,7 @@ extern "C" { abs_timeout: *const ::timespec, ) -> ::c_int; pub fn mq_getattr(mqd: ::mqd_t, attr: *mut ::mq_attr) -> ::c_int; - pub fn mq_setattr( - mqd: ::mqd_t, - newattr: *const ::mq_attr, - oldattr: *mut ::mq_attr, - ) -> ::c_int; + pub fn mq_setattr(mqd: ::mqd_t, newattr: *const ::mq_attr, oldattr: *mut ::mq_attr) -> ::c_int; pub fn port_create() -> ::c_int; pub fn port_associate( port: ::c_int, @@ -2391,16 +2916,8 @@ extern "C" { events: ::c_int, user: *mut ::c_void, ) -> ::c_int; - pub fn port_dissociate( - port: ::c_int, - source: ::c_int, - object: ::uintptr_t, - ) -> ::c_int; - pub fn port_get( - port: ::c_int, - pe: *mut port_event, - timeout: *mut ::timespec, - ) -> ::c_int; + pub fn port_dissociate(port: ::c_int, source: ::c_int, object: ::uintptr_t) -> ::c_int; + pub fn port_get(port: ::c_int, pe: *mut port_event, timeout: *mut ::timespec) -> ::c_int; pub fn port_getn( port: ::c_int, pe_list: *mut port_event, @@ -2408,11 +2925,7 @@ extern "C" { nget: *mut ::c_uint, timeout: *mut ::timespec, ) -> ::c_int; - pub fn port_send( - port: ::c_int, - events: ::c_int, - user: *mut ::c_void, - ) -> ::c_int; + pub fn port_send(port: ::c_int, events: ::c_int, user: *mut ::c_void) -> ::c_int; pub fn port_sendn( port_list: *mut ::c_int, error_list: *mut ::c_int, @@ -2420,11 +2933,6 @@ extern "C" { events: ::c_int, user: *mut ::c_void, ) -> ::c_int; - pub fn fexecve( - fd: ::c_int, - argv: *const *const ::c_char, - envp: *const *const ::c_char, - ) -> ::c_int; #[cfg_attr( any(target_os = "solaris", target_os = "illumos"), link_name = "__posix_getgrgid_r" @@ -2442,7 +2950,7 @@ extern "C" { // The epoll functions are actually only present on illumos. However, // there are things using epoll on illumos (built using the - // x86_64-sun-solaris target) which would break until the illumos target is + // x86_64-pc-solaris target) which would break until the illumos target is // present in rustc. pub fn epoll_pwait( epfd: ::c_int, @@ -2460,12 +2968,8 @@ extern "C" { maxevents: ::c_int, timeout: ::c_int, ) -> ::c_int; - pub fn epoll_ctl( - epfd: ::c_int, - op: ::c_int, - fd: ::c_int, - event: *mut ::epoll_event, - ) -> ::c_int; + pub fn epoll_ctl(epfd: ::c_int, op: ::c_int, fd: ::c_int, event: *mut ::epoll_event) + -> ::c_int; #[cfg_attr( any(target_os = "solaris", target_os = "illumos"), @@ -2478,14 +2982,21 @@ extern "C" { buflen: ::size_t, result: *mut *mut ::group, ) -> ::c_int; - pub fn pthread_sigmask( - how: ::c_int, - set: *const sigset_t, - oldset: *mut sigset_t, - ) -> ::c_int; + pub fn thr_self() -> ::thread_t; + pub fn pthread_sigmask(how: ::c_int, set: *const sigset_t, oldset: *mut sigset_t) -> ::c_int; pub fn sem_open(name: *const ::c_char, oflag: ::c_int, ...) -> *mut sem_t; pub fn getgrnam(name: *const ::c_char) -> *mut ::group; pub fn pthread_kill(thread: ::pthread_t, sig: ::c_int) -> ::c_int; + pub fn sched_get_priority_min(policy: ::c_int) -> ::c_int; + pub fn sched_get_priority_max(policy: ::c_int) -> ::c_int; + pub fn sched_getparam(pid: ::pid_t, param: *mut sched_param) -> ::c_int; + pub fn sched_setparam(pid: ::pid_t, param: *const sched_param) -> ::c_int; + pub fn sched_getscheduler(pid: ::pid_t) -> ::c_int; + pub fn sched_setscheduler( + pid: ::pid_t, + policy: ::c_int, + param: *const ::sched_param, + ) -> ::c_int; pub fn sem_unlink(name: *const ::c_char) -> ::c_int; pub fn daemon(nochdir: ::c_int, noclose: ::c_int) -> ::c_int; #[cfg_attr( @@ -2512,24 +3023,14 @@ extern "C" { ) -> ::c_int; #[cfg_attr( any(target_os = "solaris", target_os = "illumos"), - link_name = "__posix_getpwent_r" + link_name = "getpwent_r" )] - pub fn getpwent_r( - pwd: *mut passwd, - buf: *mut ::c_char, - buflen: ::size_t, - result: *mut *mut passwd, - ) -> ::c_int; + fn native_getpwent_r(pwd: *mut passwd, buf: *mut ::c_char, buflen: ::c_int) -> *mut passwd; #[cfg_attr( any(target_os = "solaris", target_os = "illumos"), - link_name = "__posix_getgrent_r" + link_name = "getgrent_r" )] - pub fn getgrent_r( - grp: *mut ::group, - buf: *mut ::c_char, - buflen: ::size_t, - result: *mut *mut ::group, - ) -> ::c_int; + fn native_getgrent_r(grp: *mut ::group, buf: *mut ::c_char, buflen: ::c_int) -> *mut ::group; #[cfg_attr( any(target_os = "solaris", target_os = "illumos"), link_name = "__posix_sigwait" @@ -2549,25 +3050,6 @@ extern "C" { pub fn dup3(src: ::c_int, dst: ::c_int, flags: ::c_int) -> ::c_int; pub fn uname(buf: *mut ::utsname) -> ::c_int; pub fn pipe2(fds: *mut ::c_int, flags: ::c_int) -> ::c_int; - pub fn door_call(d: ::c_int, params: *const door_arg_t) -> ::c_int; - pub fn door_return( - data_ptr: *const ::c_char, - data_size: ::size_t, - desc_ptr: *const door_desc_t, - num_desc: ::c_uint, - ); - pub fn door_create( - server_procedure: extern "C" fn( - cookie: *const ::c_void, - argp: *const ::c_char, - arg_size: ::size_t, - dp: *const door_desc_t, - n_desc: ::c_uint, - ), - cookie: *const ::c_void, - attributes: door_attr_t, - ) -> ::c_int; - pub fn fattach(fildes: ::c_int, path: *const ::c_char) -> ::c_int; pub fn makeutx(ux: *const utmpx) -> *mut utmpx; pub fn modutx(ux: *const utmpx) -> *mut utmpx; @@ -2594,7 +3076,208 @@ extern "C" { pub fn ntp_adjtime(buf: *mut timex) -> ::c_int; pub fn ntp_gettime(buf: *mut ntptimeval) -> ::c_int; + + pub fn timer_create(clock_id: clockid_t, evp: *mut sigevent, timerid: *mut timer_t) -> ::c_int; + pub fn timer_delete(timerid: timer_t) -> ::c_int; + pub fn timer_getoverrun(timerid: timer_t) -> ::c_int; + pub fn timer_gettime(timerid: timer_t, value: *mut itimerspec) -> ::c_int; + pub fn timer_settime( + timerid: timer_t, + flags: ::c_int, + value: *const itimerspec, + ovalue: *mut itimerspec, + ) -> ::c_int; + + pub fn ucred_get(pid: ::pid_t) -> *mut ucred_t; + pub fn getpeerucred(fd: ::c_int, ucred: *mut *mut ucred_t) -> ::c_int; + + pub fn ucred_free(ucred: *mut ucred_t); + + pub fn ucred_geteuid(ucred: *const ucred_t) -> ::uid_t; + pub fn ucred_getruid(ucred: *const ucred_t) -> ::uid_t; + pub fn ucred_getsuid(ucred: *const ucred_t) -> ::uid_t; + pub fn ucred_getegid(ucred: *const ucred_t) -> ::gid_t; + pub fn ucred_getrgid(ucred: *const ucred_t) -> ::gid_t; + pub fn ucred_getsgid(ucred: *const ucred_t) -> ::gid_t; + pub fn ucred_getgroups(ucred: *const ucred_t, groups: *mut *const ::gid_t) -> ::c_int; + pub fn ucred_getpid(ucred: *const ucred_t) -> ::pid_t; + pub fn ucred_getprojid(ucred: *const ucred_t) -> projid_t; + pub fn ucred_getzoneid(ucred: *const ucred_t) -> zoneid_t; + pub fn ucred_getpflags(ucred: *const ucred_t, flags: ::c_uint) -> ::c_uint; + + pub fn ucred_size() -> ::size_t; + + pub fn pset_create(newpset: *mut ::psetid_t) -> ::c_int; + pub fn pset_destroy(pset: ::psetid_t) -> ::c_int; + pub fn pset_assign(pset: ::psetid_t, cpu: ::processorid_t, opset: *mut psetid_t) -> ::c_int; + pub fn pset_info( + pset: ::psetid_t, + tpe: *mut ::c_int, + numcpus: *mut ::c_uint, + cpulist: *mut processorid_t, + ) -> ::c_int; + pub fn pset_bind( + pset: ::psetid_t, + idtype: ::idtype_t, + id: ::id_t, + opset: *mut psetid_t, + ) -> ::c_int; + pub fn pset_list(pset: *mut psetid_t, numpsets: *mut ::c_uint) -> ::c_int; + pub fn pset_setattr(pset: psetid_t, attr: ::c_uint) -> ::c_int; + pub fn pset_getattr(pset: psetid_t, attr: *mut ::c_uint) -> ::c_int; + pub fn processor_bind( + idtype: ::idtype_t, + id: ::id_t, + new_binding: ::processorid_t, + old_binding: *mut processorid_t, + ) -> ::c_int; + pub fn p_online(processorid: ::processorid_t, flag: ::c_int) -> ::c_int; + pub fn processor_info(processorid: ::processorid_t, infop: *mut processor_info_t) -> ::c_int; + + pub fn getexecname() -> *const ::c_char; + + pub fn gethostid() -> ::c_long; + + pub fn getpflags(flags: ::c_uint) -> ::c_uint; + pub fn setpflags(flags: ::c_uint, value: ::c_uint) -> ::c_int; + + pub fn sysinfo(command: ::c_int, buf: *mut ::c_char, count: ::c_long) -> ::c_int; + + pub fn faccessat(fd: ::c_int, path: *const ::c_char, amode: ::c_int, flag: ::c_int) -> ::c_int; + + // #include + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + pub fn dl_iterate_phdr( + callback: ::Option< + unsafe extern "C" fn( + info: *mut dl_phdr_info, + size: usize, + data: *mut ::c_void, + ) -> ::c_int, + >, + data: *mut ::c_void, + ) -> ::c_int; + pub fn getpagesize() -> ::c_int; + pub fn getpagesizes(pagesize: *mut ::size_t, nelem: ::c_int) -> ::c_int; + pub fn mmapobj( + fd: ::c_int, + flags: ::c_uint, + storage: *mut mmapobj_result_t, + elements: *mut ::c_uint, + arg: *mut ::c_void, + ) -> ::c_int; + pub fn meminfo( + inaddr: *const u64, + addr_count: ::c_int, + info_req: *const ::c_uint, + info_count: ::c_int, + outdata: *mut u64, + validity: *mut ::c_uint, + ) -> ::c_int; + + pub fn strcasecmp_l(s1: *const ::c_char, s2: *const ::c_char, loc: ::locale_t) -> ::c_int; + pub fn strncasecmp_l( + s1: *const ::c_char, + s2: *const ::c_char, + n: ::size_t, + loc: ::locale_t, + ) -> ::c_int; + pub fn strsep(string: *mut *mut ::c_char, delim: *const ::c_char) -> *mut ::c_char; + + pub fn getisax(array: *mut u32, n: ::c_uint) -> ::c_uint; + + pub fn backtrace(buffer: *mut *mut ::c_void, size: ::c_int) -> ::c_int; + pub fn backtrace_symbols(buffer: *const *mut ::c_void, size: ::c_int) -> *mut *mut ::c_char; + pub fn backtrace_symbols_fd(buffer: *const *mut ::c_void, size: ::c_int, fd: ::c_int); + + pub fn getopt_long( + argc: ::c_int, + argv: *const *mut c_char, + optstring: *const c_char, + longopts: *const option, + longindex: *mut ::c_int, + ) -> ::c_int; +} + +#[link(name = "sendfile")] +extern "C" { + pub fn sendfile(out_fd: ::c_int, in_fd: ::c_int, off: *mut ::off_t, len: ::size_t) + -> ::ssize_t; + pub fn sendfilev( + fildes: ::c_int, + vec: *const sendfilevec_t, + sfvcnt: ::c_int, + xferred: *mut ::size_t, + ) -> ::ssize_t; +} + +#[link(name = "lgrp")] +extern "C" { + pub fn lgrp_init(view: lgrp_view_t) -> lgrp_cookie_t; + pub fn lgrp_fini(cookie: lgrp_cookie_t) -> ::c_int; + pub fn lgrp_affinity_get( + idtype: ::idtype_t, + id: ::id_t, + lgrp: ::lgrp_id_t, + ) -> ::lgrp_affinity_t; + pub fn lgrp_affinity_set( + idtype: ::idtype_t, + id: ::id_t, + lgrp: ::lgrp_id_t, + aff: lgrp_affinity_t, + ) -> ::lgrp_affinity_t; + pub fn lgrp_cpus( + cookie: ::lgrp_cookie_t, + lgrp: ::lgrp_id_t, + cpuids: *mut ::processorid_t, + count: ::c_uint, + content: ::lgrp_content_t, + ) -> ::c_int; + pub fn lgrp_mem_size( + cookie: ::lgrp_cookie_t, + lgrp: ::lgrp_id_t, + tpe: ::lgrp_mem_size_flag_t, + content: ::lgrp_content_t, + ) -> ::lgrp_mem_size_t; + pub fn lgrp_nlgrps(cookie: ::lgrp_cookie_t) -> ::c_int; + pub fn lgrp_view(cookie: ::lgrp_cookie_t) -> ::lgrp_view_t; + pub fn lgrp_home(idtype: ::idtype_t, id: ::id_t) -> ::lgrp_id_t; + pub fn lgrp_version(version: ::c_int) -> ::c_int; + pub fn lgrp_resources( + cookie: ::lgrp_cookie_t, + lgrp: ::lgrp_id_t, + lgrps: *mut ::lgrp_id_t, + count: ::c_uint, + tpe: ::lgrp_rsrc_t, + ) -> ::c_int; + pub fn lgrp_root(cookie: ::lgrp_cookie_t) -> ::lgrp_id_t; } mod compat; pub use self::compat::*; + +cfg_if! { + if #[cfg(target_os = "illumos")] { + mod illumos; + pub use self::illumos::*; + } else if #[cfg(target_os = "solaris")] { + mod solaris; + pub use self::solaris::*; + } else { + // Unknown target_os + } +} + +cfg_if! { + if #[cfg(target_arch = "x86_64")] { + mod x86_64; + mod x86_common; + pub use self::x86_64::*; + pub use self::x86_common::*; + } else if #[cfg(target_arch = "x86")] { + mod x86; + mod x86_common; + pub use self::x86::*; + pub use self::x86_common::*; + } +} diff --git a/crux-mir/lib/libc/src/unix/solarish/solaris.rs b/crux-mir/lib/libc/src/unix/solarish/solaris.rs new file mode 100644 index 000000000..80bad281e --- /dev/null +++ b/crux-mir/lib/libc/src/unix/solarish/solaris.rs @@ -0,0 +1,101 @@ +pub type door_attr_t = ::c_uint; +pub type door_id_t = ::c_ulonglong; + +s! { + pub struct shmid_ds { + pub shm_perm: ::ipc_perm, + pub shm_segsz: ::size_t, + pub shm_flags: ::uintptr_t, + pub shm_lkcnt: ::c_ushort, + pub shm_lpid: ::pid_t, + pub shm_cpid: ::pid_t, + pub shm_nattch: ::shmatt_t, + pub shm_cnattch: ::c_ulong, + pub shm_atime: ::time_t, + pub shm_dtime: ::time_t, + pub shm_ctime: ::time_t, + pub shm_amp: *mut ::c_void, + pub shm_gransize: u64, + pub shm_allocated: u64, + pub shm_pad4: [i64; 1], + } + + pub struct door_desc_t__d_data__d_desc { + pub d_descriptor: ::c_int, + pub d_id: ::door_id_t + } +} + +s_no_extra_traits! { + #[cfg_attr(feature = "extra_traits", allow(missing_debug_implementations))] + pub union door_desc_t__d_data { + pub d_desc: door_desc_t__d_data__d_desc, + d_resv: [::c_int; 5], /* Check out /usr/include/sys/door.h */ + } + + #[cfg_attr(feature = "extra_traits", allow(missing_debug_implementations))] + pub struct door_desc_t { + pub d_attributes: door_attr_t, + pub d_data: door_desc_t__d_data, + } + + #[cfg_attr(feature = "extra_traits", allow(missing_debug_implementations))] + pub struct door_arg_t { + pub data_ptr: *const ::c_char, + pub data_size: ::size_t, + pub desc_ptr: *const door_desc_t, + pub dec_num: ::c_uint, + pub rbuf: *const ::c_char, + pub rsize: ::size_t, + } +} + +pub const PORT_SOURCE_POSTWAIT: ::c_int = 8; +pub const PORT_SOURCE_SIGNAL: ::c_int = 9; + +pub const AF_LOCAL: ::c_int = 0; +pub const AF_FILE: ::c_int = 0; + +pub const TCP_KEEPIDLE: ::c_int = 0x1d; +pub const TCP_KEEPINTVL: ::c_int = 0x1e; +pub const TCP_KEEPCNT: ::c_int = 0x1f; + +pub const F_DUPFD_CLOEXEC: ::c_int = 47; +pub const F_DUPFD_CLOFORK: ::c_int = 49; +pub const F_DUP2FD_CLOEXEC: ::c_int = 48; +pub const F_DUP2FD_CLOFORK: ::c_int = 50; + +extern "C" { + pub fn fexecve( + fd: ::c_int, + argv: *const *const ::c_char, + envp: *const *const ::c_char, + ) -> ::c_int; + + pub fn mincore(addr: *const ::c_void, len: ::size_t, vec: *mut ::c_char) -> ::c_int; + + pub fn door_call(d: ::c_int, params: *const door_arg_t) -> ::c_int; + pub fn door_return( + data_ptr: *const ::c_char, + data_size: ::size_t, + desc_ptr: *const door_desc_t, + num_desc: ::c_uint, + ); + pub fn door_create( + server_procedure: extern "C" fn( + cookie: *const ::c_void, + argp: *const ::c_char, + arg_size: ::size_t, + dp: *const door_desc_t, + n_desc: ::c_uint, + ), + cookie: *const ::c_void, + attributes: door_attr_t, + ) -> ::c_int; + + pub fn fattach(fildes: ::c_int, path: *const ::c_char) -> ::c_int; + + pub fn pthread_getattr_np(thread: ::pthread_t, attr: *mut ::pthread_attr_t) -> ::c_int; + + pub fn euidaccess(path: *const ::c_char, amode: ::c_int) -> ::c_int; +} diff --git a/crux-mir/lib/libc/src/unix/solarish/x86.rs b/crux-mir/lib/libc/src/unix/solarish/x86.rs new file mode 100644 index 000000000..23f52ad3c --- /dev/null +++ b/crux-mir/lib/libc/src/unix/solarish/x86.rs @@ -0,0 +1,29 @@ +pub type Elf32_Addr = ::c_ulong; +pub type Elf32_Half = ::c_ushort; +pub type Elf32_Off = ::c_ulong; +pub type Elf32_Sword = ::c_long; +pub type Elf32_Word = ::c_ulong; +pub type Elf32_Lword = ::c_ulonglong; +pub type Elf32_Phdr = __c_anonymous_Elf32_Phdr; + +s! { + pub struct __c_anonymous_Elf32_Phdr { + pub p_type: ::Elf32_Word, + pub p_offset: ::Elf32_Off, + pub p_vaddr: ::Elf32_Addr, + pub p_paddr: ::Elf32_Addr, + pub p_filesz: ::Elf32_Word, + pub p_memsz: ::Elf32_Word, + pub p_flags: ::Elf32_Word, + pub p_align: ::Elf32_Word, + } + + pub struct dl_phdr_info { + pub dlpi_addr: ::Elf32_Addr, + pub dlpi_name: *const ::c_char, + pub dlpi_phdr: *const ::Elf32_Phdr, + pub dlpi_phnum: ::Elf32_Half, + pub dlpi_adds: ::c_ulonglong, + pub dlpi_subs: ::c_ulonglong, + } +} diff --git a/crux-mir/lib/libc/src/unix/solarish/x86_64.rs b/crux-mir/lib/libc/src/unix/solarish/x86_64.rs new file mode 100644 index 000000000..bca552f37 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/solarish/x86_64.rs @@ -0,0 +1,190 @@ +pub type greg_t = ::c_long; + +pub type Elf64_Addr = ::c_ulong; +pub type Elf64_Half = ::c_ushort; +pub type Elf64_Off = ::c_ulong; +pub type Elf64_Sword = ::c_int; +pub type Elf64_Sxword = ::c_long; +pub type Elf64_Word = ::c_uint; +pub type Elf64_Xword = ::c_ulong; +pub type Elf64_Lword = ::c_ulong; +pub type Elf64_Phdr = __c_anonymous_Elf64_Phdr; + +s! { + pub struct __c_anonymous_fpchip_state { + pub cw: u16, + pub sw: u16, + pub fctw: u8, + pub __fx_rsvd: u8, + pub fop: u16, + pub rip: u64, + pub rdp: u64, + pub mxcsr: u32, + pub mxcsr_mask: u32, + pub st: [::upad128_t; 8], + pub xmm: [::upad128_t; 16], + pub __fx_ign: [::upad128_t; 6], + pub status: u32, + pub xstatus: u32, + } + + pub struct __c_anonymous_Elf64_Phdr { + pub p_type: ::Elf64_Word, + pub p_flags: ::Elf64_Word, + pub p_offset: ::Elf64_Off, + pub p_vaddr: ::Elf64_Addr, + pub p_paddr: ::Elf64_Addr, + pub p_filesz: ::Elf64_Xword, + pub p_memsz: ::Elf64_Xword, + pub p_align: ::Elf64_Xword, + } + + pub struct dl_phdr_info { + pub dlpi_addr: ::Elf64_Addr, + pub dlpi_name: *const ::c_char, + pub dlpi_phdr: *const ::Elf64_Phdr, + pub dlpi_phnum: ::Elf64_Half, + pub dlpi_adds: ::c_ulonglong, + pub dlpi_subs: ::c_ulonglong, + } +} + +s_no_extra_traits! { + #[cfg(libc_union)] + pub union __c_anonymous_fp_reg_set { + pub fpchip_state: __c_anonymous_fpchip_state, + pub f_fpregs: [[u32; 13]; 10], + } + + pub struct fpregset_t { + pub fp_reg_set: __c_anonymous_fp_reg_set, + } + + pub struct mcontext_t { + pub gregs: [::greg_t; 28], + pub fpregs: fpregset_t, + } + + pub struct ucontext_t { + pub uc_flags: ::c_ulong, + pub uc_link: *mut ucontext_t, + pub uc_sigmask: ::sigset_t, + pub uc_stack: ::stack_t, + pub uc_mcontext: mcontext_t, + pub uc_filler: [::c_long; 5], + } +} + +cfg_if! { + if #[cfg(feature = "extra_traits")] { + #[cfg(libc_union)] + impl PartialEq for __c_anonymous_fp_reg_set { + fn eq(&self, other: &__c_anonymous_fp_reg_set) -> bool { + unsafe { + self.fpchip_state == other.fpchip_state || + self. + f_fpregs. + iter(). + zip(other.f_fpregs.iter()). + all(|(a, b)| a == b) + } + } + } + #[cfg(libc_union)] + impl Eq for __c_anonymous_fp_reg_set {} + #[cfg(libc_union)] + impl ::fmt::Debug for __c_anonymous_fp_reg_set { + fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { + unsafe { + f.debug_struct("__c_anonymous_fp_reg_set") + .field("fpchip_state", &{self.fpchip_state}) + .field("f_fpregs", &{self.f_fpregs}) + .finish() + } + } + } + impl PartialEq for fpregset_t { + fn eq(&self, other: &fpregset_t) -> bool { + self.fp_reg_set == other.fp_reg_set + } + } + impl Eq for fpregset_t {} + impl ::fmt::Debug for fpregset_t { + fn fmt(&self, f:&mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("fpregset_t") + .field("fp_reg_set", &self.fp_reg_set) + .finish() + } + } + impl PartialEq for mcontext_t { + fn eq(&self, other: &mcontext_t) -> bool { + self.gregs == other.gregs && + self.fpregs == other.fpregs + } + } + impl Eq for mcontext_t {} + impl ::fmt::Debug for mcontext_t { + fn fmt(&self, f:&mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("mcontext_t") + .field("gregs", &self.gregs) + .field("fpregs", &self.fpregs) + .finish() + } + } + impl PartialEq for ucontext_t { + fn eq(&self, other: &ucontext_t) -> bool { + self.uc_flags == other.uc_flags + && self.uc_link == other.uc_link + && self.uc_sigmask == other.uc_sigmask + && self.uc_stack == other.uc_stack + && self.uc_mcontext == other.uc_mcontext + && self.uc_filler == other.uc_filler + } + } + impl Eq for ucontext_t {} + impl ::fmt::Debug for ucontext_t { + fn fmt(&self, f:&mut ::fmt::Formatter) -> ::fmt::Result { + f.debug_struct("ucontext_t") + .field("uc_flags", &self.uc_flags) + .field("uc_link", &self.uc_link) + .field("uc_sigmask", &self.uc_sigmask) + .field("uc_stack", &self.uc_stack) + .field("uc_mcontext", &self.uc_mcontext) + .field("uc_filler", &self.uc_filler) + .finish() + } + } + + } +} + +// sys/regset.h + +pub const REG_GSBASE: ::c_int = 27; +pub const REG_FSBASE: ::c_int = 26; +pub const REG_DS: ::c_int = 25; +pub const REG_ES: ::c_int = 24; +pub const REG_GS: ::c_int = 23; +pub const REG_FS: ::c_int = 22; +pub const REG_SS: ::c_int = 21; +pub const REG_RSP: ::c_int = 20; +pub const REG_RFL: ::c_int = 19; +pub const REG_CS: ::c_int = 18; +pub const REG_RIP: ::c_int = 17; +pub const REG_ERR: ::c_int = 16; +pub const REG_TRAPNO: ::c_int = 15; +pub const REG_RAX: ::c_int = 14; +pub const REG_RCX: ::c_int = 13; +pub const REG_RDX: ::c_int = 12; +pub const REG_RBX: ::c_int = 11; +pub const REG_RBP: ::c_int = 10; +pub const REG_RSI: ::c_int = 9; +pub const REG_RDI: ::c_int = 8; +pub const REG_R8: ::c_int = 7; +pub const REG_R9: ::c_int = 6; +pub const REG_R10: ::c_int = 5; +pub const REG_R11: ::c_int = 4; +pub const REG_R12: ::c_int = 3; +pub const REG_R13: ::c_int = 2; +pub const REG_R14: ::c_int = 1; +pub const REG_R15: ::c_int = 0; diff --git a/crux-mir/lib/libc/src/unix/solarish/x86_common.rs b/crux-mir/lib/libc/src/unix/solarish/x86_common.rs new file mode 100644 index 000000000..515f23490 --- /dev/null +++ b/crux-mir/lib/libc/src/unix/solarish/x86_common.rs @@ -0,0 +1,65 @@ +// AT_SUN_HWCAP +pub const AV_386_FPU: u32 = 0x00001; +pub const AV_386_TSC: u32 = 0x00002; +pub const AV_386_CX8: u32 = 0x00004; +pub const AV_386_SEP: u32 = 0x00008; +pub const AV_386_AMD_SYSC: u32 = 0x00010; +pub const AV_386_CMOV: u32 = 0x00020; +pub const AV_386_MMX: u32 = 0x00040; +pub const AV_386_AMD_MMX: u32 = 0x00080; +pub const AV_386_AMD_3DNow: u32 = 0x00100; +pub const AV_386_AMD_3DNowx: u32 = 0x00200; +pub const AV_386_FXSR: u32 = 0x00400; +pub const AV_386_SSE: u32 = 0x00800; +pub const AV_386_SSE2: u32 = 0x01000; +pub const AV_386_CX16: u32 = 0x10000; +pub const AV_386_AHF: u32 = 0x20000; +pub const AV_386_TSCP: u32 = 0x40000; +pub const AV_386_AMD_SSE4A: u32 = 0x80000; +pub const AV_386_POPCNT: u32 = 0x100000; +pub const AV_386_AMD_LZCNT: u32 = 0x200000; +pub const AV_386_SSSE3: u32 = 0x400000; +pub const AV_386_SSE4_1: u32 = 0x800000; +pub const AV_386_SSE4_2: u32 = 0x1000000; +pub const AV_386_MOVBE: u32 = 0x2000000; +pub const AV_386_AES: u32 = 0x4000000; +pub const AV_386_PCLMULQDQ: u32 = 0x8000000; +pub const AV_386_XSAVE: u32 = 0x10000000; +pub const AV_386_AVX: u32 = 0x20000000; +pub const AV_386_VMX: u32 = 0x40000000; +pub const AV_386_AMD_SVM: u32 = 0x80000000; +// AT_SUN_HWCAP2 +pub const AV_386_2_F16C: u32 = 0x00000001; +pub const AV_386_2_RDRAND: u32 = 0x00000002; +pub const AV_386_2_BMI1: u32 = 0x00000004; +pub const AV_386_2_BMI2: u32 = 0x00000008; +pub const AV_386_2_FMA: u32 = 0x00000010; +pub const AV_386_2_AVX2: u32 = 0x00000020; +pub const AV_386_2_ADX: u32 = 0x00000040; +pub const AV_386_2_RDSEED: u32 = 0x00000080; +pub const AV_386_2_AVX512F: u32 = 0x00000100; +pub const AV_386_2_AVX512DQ: u32 = 0x00000200; +pub const AV_386_2_AVX512IFMA: u32 = 0x00000400; +pub const AV_386_2_AVX512PF: u32 = 0x00000800; +pub const AV_386_2_AVX512ER: u32 = 0x00001000; +pub const AV_386_2_AVX512CD: u32 = 0x00002000; +pub const AV_386_2_AVX512BW: u32 = 0x00004000; +pub const AV_386_2_AVX512VL: u32 = 0x00008000; +pub const AV_386_2_AVX512VBMI: u32 = 0x00010000; +pub const AV_386_2_AVX512VPOPCDQ: u32 = 0x00020000; +pub const AV_386_2_AVX512_4NNIW: u32 = 0x00040000; +pub const AV_386_2_AVX512_4FMAPS: u32 = 0x00080000; +pub const AV_386_2_SHA: u32 = 0x00100000; +pub const AV_386_2_FSGSBASE: u32 = 0x00200000; +pub const AV_386_2_CLFLUSHOPT: u32 = 0x00400000; +pub const AV_386_2_CLWB: u32 = 0x00800000; +pub const AV_386_2_MONITORX: u32 = 0x01000000; +pub const AV_386_2_CLZERO: u32 = 0x02000000; +pub const AV_386_2_AVX512_VNNI: u32 = 0x04000000; +pub const AV_386_2_VPCLMULQDQ: u32 = 0x08000000; +pub const AV_386_2_VAES: u32 = 0x10000000; +// AT_SUN_FPTYPE +pub const AT_386_FPINFO_NONE: u32 = 0; +pub const AT_386_FPINFO_FXSAVE: u32 = 1; +pub const AT_386_FPINFO_XSAVE: u32 = 2; +pub const AT_386_FPINFO_XSAVE_AMD: u32 = 3; diff --git a/crux-mir/lib/libc/src/unix/uclibc/align.rs b/crux-mir/lib/libc/src/unix/uclibc/align.rs deleted file mode 100644 index 76b524d0c..000000000 --- a/crux-mir/lib/libc/src/unix/uclibc/align.rs +++ /dev/null @@ -1,66 +0,0 @@ -macro_rules! expand_align { - () => { - s! { - #[cfg_attr(any(target_pointer_width = "32", - target_arch = "x86_64", - target_arch = "powerpc64", - target_arch = "mips64", - target_arch = "s390x", - target_arch = "sparc64"), - repr(align(4)))] - #[cfg_attr(not(any(target_pointer_width = "32", - target_arch = "x86_64", - target_arch = "powerpc64", - target_arch = "mips64", - target_arch = "s390x", - target_arch = "sparc64")), - repr(align(8)))] - pub struct pthread_mutexattr_t { - size: [u8; ::__SIZEOF_PTHREAD_MUTEXATTR_T], - } - - #[repr(align(4))] - pub struct pthread_condattr_t { - size: [u8; ::__SIZEOF_PTHREAD_CONDATTR_T], - } - } - - s_no_extra_traits! { - #[repr(align(8))] - #[allow(missing_debug_implementations)] - pub struct pthread_cond_t { - size: [u8; ::__SIZEOF_PTHREAD_COND_T], - } - - #[cfg_attr(all(target_pointer_width = "32", - any(target_arch = "mips", - target_arch = "arm", - target_arch = "powerpc")), - repr(align(4)))] - #[cfg_attr(any(target_pointer_width = "64", - not(any(target_arch = "mips", - target_arch = "arm", - target_arch = "powerpc"))), - repr(align(8)))] - #[allow(missing_debug_implementations)] - pub struct pthread_mutex_t { - size: [u8; ::__SIZEOF_PTHREAD_MUTEX_T], - } - - #[cfg_attr(all(target_pointer_width = "32", - any(target_arch = "mips", - target_arch = "arm", - target_arch = "powerpc")), - repr(align(4)))] - #[cfg_attr(any(target_pointer_width = "64", - not(any(target_arch = "mips", - target_arch = "arm", - target_arch = "powerpc"))), - repr(align(8)))] - #[allow(missing_debug_implementations)] - pub struct pthread_rwlock_t { - size: [u8; ::__SIZEOF_PTHREAD_RWLOCK_T], - } - } - }; -} diff --git a/crux-mir/lib/libc/src/unix/uclibc/mod.rs b/crux-mir/lib/libc/src/unix/uclibc/mod.rs deleted file mode 100644 index ae8c30e95..000000000 --- a/crux-mir/lib/libc/src/unix/uclibc/mod.rs +++ /dev/null @@ -1,2295 +0,0 @@ -pub type sa_family_t = u16; -pub type pthread_key_t = ::c_uint; -pub type speed_t = ::c_uint; -pub type tcflag_t = ::c_uint; -pub type loff_t = ::c_longlong; -pub type clockid_t = ::c_int; -pub type key_t = ::c_int; -pub type id_t = ::c_uint; -pub type useconds_t = u32; -pub type dev_t = u64; -pub type socklen_t = u32; -pub type mode_t = u32; -pub type ino64_t = u64; -pub type off64_t = i64; -pub type blkcnt64_t = i64; -pub type rlim64_t = u64; -pub type shmatt_t = ::c_ulong; -pub type mqd_t = ::c_int; -pub type msgqnum_t = ::c_ulong; -pub type msglen_t = ::c_ulong; -pub type nfds_t = ::c_ulong; -pub type nl_item = ::c_int; -pub type idtype_t = ::c_uint; - -#[cfg_attr(feature = "extra_traits", derive(Debug))] -pub enum fpos64_t {} // FIXME: fill this out with a struct -impl ::Copy for fpos64_t {} -impl ::Clone for fpos64_t { - fn clone(&self) -> fpos64_t { - *self - } -} - -#[cfg_attr(feature = "extra_traits", derive(Debug))] -pub enum timezone {} -impl ::Copy for timezone {} -impl ::Clone for timezone { - fn clone(&self) -> timezone { - *self - } -} - -s! { - pub struct in_addr { - pub s_addr: ::in_addr_t, - } - - pub struct ip_mreq { - pub imr_multiaddr: in_addr, - pub imr_interface: in_addr, - } - - pub struct sockaddr { - pub sa_family: sa_family_t, - pub sa_data: [::c_char; 14], - } - - pub struct sockaddr_in { - pub sin_family: sa_family_t, - pub sin_port: ::in_port_t, - pub sin_addr: ::in_addr, - pub sin_zero: [u8; 8], - } - - pub struct sockaddr_in6 { - pub sin6_family: sa_family_t, - pub sin6_port: ::in_port_t, - pub sin6_flowinfo: u32, - pub sin6_addr: ::in6_addr, - pub sin6_scope_id: u32, - } - - pub struct addrinfo { - pub ai_flags: ::c_int, - pub ai_family: ::c_int, - pub ai_socktype: ::c_int, - pub ai_protocol: ::c_int, - pub ai_addrlen: socklen_t, - - pub ai_addr: *mut ::sockaddr, - - pub ai_canonname: *mut c_char, - - pub ai_next: *mut addrinfo, - } - - pub struct sockaddr_ll { - pub sll_family: ::c_ushort, - pub sll_protocol: ::c_ushort, - pub sll_ifindex: ::c_int, - pub sll_hatype: ::c_ushort, - pub sll_pkttype: ::c_uchar, - pub sll_halen: ::c_uchar, - pub sll_addr: [::c_uchar; 8] - } - - pub struct fd_set { - fds_bits: [::c_ulong; FD_SETSIZE / ULONG_SIZE], - } - - pub struct tm { - pub tm_sec: ::c_int, - pub tm_min: ::c_int, - pub tm_hour: ::c_int, - pub tm_mday: ::c_int, - pub tm_mon: ::c_int, - pub tm_year: ::c_int, - pub tm_wday: ::c_int, - pub tm_yday: ::c_int, - pub tm_isdst: ::c_int, - pub tm_gmtoff: ::c_long, - pub tm_zone: *const ::c_char, - } - - pub struct sched_param { - pub sched_priority: ::c_int, - } - - pub struct Dl_info { - pub dli_fname: *const ::c_char, - pub dli_fbase: *mut ::c_void, - pub dli_sname: *const ::c_char, - pub dli_saddr: *mut ::c_void, - } - - pub struct lconv { - pub decimal_point: *mut ::c_char, - pub thousands_sep: *mut ::c_char, - pub grouping: *mut ::c_char, - pub int_curr_symbol: *mut ::c_char, - pub currency_symbol: *mut ::c_char, - pub mon_decimal_point: *mut ::c_char, - pub mon_thousands_sep: *mut ::c_char, - pub mon_grouping: *mut ::c_char, - pub positive_sign: *mut ::c_char, - pub negative_sign: *mut ::c_char, - pub int_frac_digits: ::c_char, - pub frac_digits: ::c_char, - pub p_cs_precedes: ::c_char, - pub p_sep_by_space: ::c_char, - pub n_cs_precedes: ::c_char, - pub n_sep_by_space: ::c_char, - pub p_sign_posn: ::c_char, - pub n_sign_posn: ::c_char, - pub int_p_cs_precedes: ::c_char, - pub int_p_sep_by_space: ::c_char, - pub int_n_cs_precedes: ::c_char, - pub int_n_sep_by_space: ::c_char, - pub int_p_sign_posn: ::c_char, - pub int_n_sign_posn: ::c_char, - } - - pub struct rlimit64 { - pub rlim_cur: rlim64_t, - pub rlim_max: rlim64_t, - } - - pub struct glob_t { - pub gl_pathc: ::size_t, - pub gl_pathv: *mut *mut c_char, - pub gl_offs: ::size_t, - pub gl_flags: ::c_int, - - __unused1: *mut ::c_void, - __unused2: *mut ::c_void, - __unused3: *mut ::c_void, - __unused4: *mut ::c_void, - __unused5: *mut ::c_void, - } - - pub struct ifaddrs { - pub ifa_next: *mut ifaddrs, - pub ifa_name: *mut c_char, - pub ifa_flags: ::c_uint, - pub ifa_addr: *mut ::sockaddr, - pub ifa_netmask: *mut ::sockaddr, - pub ifa_ifu: *mut ::sockaddr, // FIXME This should be a union - pub ifa_data: *mut ::c_void - } - - pub struct pthread_rwlockattr_t { - __lockkind: ::c_int, - __pshared: ::c_int, - } - - pub struct passwd { - pub pw_name: *mut ::c_char, - pub pw_passwd: *mut ::c_char, - pub pw_uid: ::uid_t, - pub pw_gid: ::gid_t, - pub pw_gecos: *mut ::c_char, - pub pw_dir: *mut ::c_char, - pub pw_shell: *mut ::c_char, - } - - pub struct spwd { - pub sp_namp: *mut ::c_char, - pub sp_pwdp: *mut ::c_char, - pub sp_lstchg: ::c_long, - pub sp_min: ::c_long, - pub sp_max: ::c_long, - pub sp_warn: ::c_long, - pub sp_inact: ::c_long, - pub sp_expire: ::c_long, - pub sp_flag: ::c_ulong, - } - - pub struct statvfs { - pub f_bsize: ::c_ulong, - pub f_frsize: ::c_ulong, - pub f_blocks: ::fsblkcnt_t, - pub f_bfree: ::fsblkcnt_t, - pub f_bavail: ::fsblkcnt_t, - pub f_files: ::fsfilcnt_t, - pub f_ffree: ::fsfilcnt_t, - pub f_favail: ::fsfilcnt_t, - #[cfg(target_endian = "little")] - pub f_fsid: ::c_ulong, - #[cfg(target_pointer_width = "32")] - __f_unused: ::c_int, - #[cfg(target_endian = "big")] - pub f_fsid: ::c_ulong, - pub f_flag: ::c_ulong, - pub f_namemax: ::c_ulong, - __f_spare: [::c_int; 6], - } - - pub struct dqblk { - pub dqb_bhardlimit: u32, - pub dqb_bsoftlimit: u32, - pub dqb_curblocks: u32, - pub dqb_ihardlimit: u32, - pub dqb_isoftlimit: u32, - pub dqb_curinodes: u32, - pub dqb_btime: ::time_t, - pub dqb_itime: ::time_t, - } - - pub struct signalfd_siginfo { - pub ssi_signo: u32, - pub ssi_errno: i32, - pub ssi_code: i32, - pub ssi_pid: u32, - pub ssi_uid: u32, - pub ssi_fd: i32, - pub ssi_tid: u32, - pub ssi_band: u32, - pub ssi_overrun: u32, - pub ssi_trapno: u32, - pub ssi_status: i32, - pub ssi_int: i32, - pub ssi_ptr: u64, - pub ssi_utime: u64, - pub ssi_stime: u64, - pub ssi_addr: u64, - pub ssi_addr_lsb: u16, - _pad2: u16, - pub ssi_syscall: i32, - pub ssi_call_addr: u64, - pub ssi_arch: u32, - _pad: [u8; 28], - } - - pub struct fsid_t { - __val: [::c_int; 2], - } - - pub struct cpu_set_t { - #[cfg(target_pointer_width = "32")] - bits: [u32; 32], - #[cfg(target_pointer_width = "64")] - bits: [u64; 16], - } - - pub struct if_nameindex { - pub if_index: ::c_uint, - pub if_name: *mut ::c_char, - } - - // System V IPC - pub struct msginfo { - pub msgpool: ::c_int, - pub msgmap: ::c_int, - pub msgmax: ::c_int, - pub msgmnb: ::c_int, - pub msgmni: ::c_int, - pub msgssz: ::c_int, - pub msgtql: ::c_int, - pub msgseg: ::c_ushort, - } - - pub struct ucred { - pub pid: ::pid_t, - pub uid: ::uid_t, - pub gid: ::gid_t, - } -} - -s_no_extra_traits! { - #[cfg_attr( - any(target_arch = "x86", target_arch = "x86_64"), - repr(packed) - )] - #[allow(missing_debug_implementations)] - pub struct epoll_event { - pub events: u32, - pub u64: u64, - } - - #[allow(missing_debug_implementations)] - pub struct sockaddr_un { - pub sun_family: sa_family_t, - pub sun_path: [::c_char; 108] - } - - #[allow(missing_debug_implementations)] - pub struct sockaddr_storage { - pub ss_family: sa_family_t, - __ss_align: ::size_t, - #[cfg(target_pointer_width = "32")] - __ss_pad2: [u8; 128 - 2 * 4], - #[cfg(target_pointer_width = "64")] - __ss_pad2: [u8; 128 - 2 * 8], - } - - #[allow(missing_debug_implementations)] - pub struct utsname { - pub sysname: [::c_char; 65], - pub nodename: [::c_char; 65], - pub release: [::c_char; 65], - pub version: [::c_char; 65], - pub machine: [::c_char; 65], - pub domainname: [::c_char; 65] - } - - #[allow(missing_debug_implementations)] - pub struct dirent { - pub d_ino: ::ino_t, - pub d_off: ::off_t, - pub d_reclen: ::c_ushort, - pub d_type: ::c_uchar, - pub d_name: [::c_char; 256], - } - - #[allow(missing_debug_implementations)] - pub struct dirent64 { - pub d_ino: ::ino64_t, - pub d_off: ::off64_t, - pub d_reclen: ::c_ushort, - pub d_type: ::c_uchar, - pub d_name: [::c_char; 256], - } - - pub struct mq_attr { - pub mq_flags: ::c_long, - pub mq_maxmsg: ::c_long, - pub mq_msgsize: ::c_long, - pub mq_curmsgs: ::c_long, - pad: [::c_long; 4] - } - - pub struct sockaddr_nl { - pub nl_family: ::sa_family_t, - nl_pad: ::c_ushort, - pub nl_pid: u32, - pub nl_groups: u32 - } - - pub struct sigevent { - pub sigev_value: ::sigval, - pub sigev_signo: ::c_int, - pub sigev_notify: ::c_int, - // Actually a union. We only expose sigev_notify_thread_id because it's - // the most useful member - pub sigev_notify_thread_id: ::c_int, - #[cfg(target_pointer_width = "64")] - __unused1: [::c_int; 11], - #[cfg(target_pointer_width = "32")] - __unused1: [::c_int; 12] - } -} - -cfg_if! { - if #[cfg(feature = "extra_traits")] { - impl PartialEq for mq_attr { - fn eq(&self, other: &mq_attr) -> bool { - self.mq_flags == other.mq_flags && - self.mq_maxmsg == other.mq_maxmsg && - self.mq_msgsize == other.mq_msgsize && - self.mq_curmsgs == other.mq_curmsgs - } - } - impl Eq for mq_attr {} - impl ::fmt::Debug for mq_attr { - fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { - f.debug_struct("mq_attr") - .field("mq_flags", &self.mq_flags) - .field("mq_maxmsg", &self.mq_maxmsg) - .field("mq_msgsize", &self.mq_msgsize) - .field("mq_curmsgs", &self.mq_curmsgs) - .finish() - } - } - impl ::hash::Hash for mq_attr { - fn hash(&self, state: &mut H) { - self.mq_flags.hash(state); - self.mq_maxmsg.hash(state); - self.mq_msgsize.hash(state); - self.mq_curmsgs.hash(state); - } - } - - impl PartialEq for sockaddr_nl { - fn eq(&self, other: &sockaddr_nl) -> bool { - self.nl_family == other.nl_family && - self.nl_pid == other.nl_pid && - self.nl_groups == other.nl_groups - } - } - impl Eq for sockaddr_nl {} - impl ::fmt::Debug for sockaddr_nl { - fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { - f.debug_struct("sockaddr_nl") - .field("nl_family", &self.nl_family) - .field("nl_pid", &self.nl_pid) - .field("nl_groups", &self.nl_groups) - .finish() - } - } - impl ::hash::Hash for sockaddr_nl { - fn hash(&self, state: &mut H) { - self.nl_family.hash(state); - self.nl_pid.hash(state); - self.nl_groups.hash(state); - } - } - - impl PartialEq for sigevent { - fn eq(&self, other: &sigevent) -> bool { - self.sigev_value == other.sigev_value - && self.sigev_signo == other.sigev_signo - && self.sigev_notify == other.sigev_notify - && self.sigev_notify_thread_id - == other.sigev_notify_thread_id - } - } - impl Eq for sigevent {} - impl ::fmt::Debug for sigevent { - fn fmt(&self, f: &mut ::fmt::Formatter) -> ::fmt::Result { - f.debug_struct("sigevent") - .field("sigev_value", &self.sigev_value) - .field("sigev_signo", &self.sigev_signo) - .field("sigev_notify", &self.sigev_notify) - .field("sigev_notify_thread_id", - &self.sigev_notify_thread_id) - .finish() - } - } - impl ::hash::Hash for sigevent { - fn hash(&self, state: &mut H) { - self.sigev_value.hash(state); - self.sigev_signo.hash(state); - self.sigev_notify.hash(state); - self.sigev_notify_thread_id.hash(state); - } - } - } -} - -// intentionally not public, only used for fd_set -cfg_if! { - if #[cfg(target_pointer_width = "32")] { - const ULONG_SIZE: usize = 32; - } else if #[cfg(target_pointer_width = "64")] { - const ULONG_SIZE: usize = 64; - } else { - // Unknown target_pointer_width - } -} - -pub const EXIT_FAILURE: ::c_int = 1; -pub const EXIT_SUCCESS: ::c_int = 0; -pub const RAND_MAX: ::c_int = 2147483647; -pub const EOF: ::c_int = -1; -pub const SEEK_SET: ::c_int = 0; -pub const SEEK_CUR: ::c_int = 1; -pub const SEEK_END: ::c_int = 2; -pub const _IOFBF: ::c_int = 0; -pub const _IONBF: ::c_int = 2; -pub const _IOLBF: ::c_int = 1; - -pub const F_DUPFD: ::c_int = 0; -pub const F_GETFD: ::c_int = 1; -pub const F_SETFD: ::c_int = 2; -pub const F_GETFL: ::c_int = 3; -pub const F_SETFL: ::c_int = 4; - -// Linux-specific fcntls -pub const F_SETLEASE: ::c_int = 1024; -pub const F_GETLEASE: ::c_int = 1025; -pub const F_NOTIFY: ::c_int = 1026; -pub const F_DUPFD_CLOEXEC: ::c_int = 1030; - -// FIXME(#235): Include file sealing fcntls once we have a way to verify them. - -pub const SIGTRAP: ::c_int = 5; - -pub const PTHREAD_CREATE_JOINABLE: ::c_int = 0; -pub const PTHREAD_CREATE_DETACHED: ::c_int = 1; - -pub const CLOCK_REALTIME: ::clockid_t = 0; -pub const CLOCK_MONOTONIC: ::clockid_t = 1; -pub const CLOCK_PROCESS_CPUTIME_ID: ::clockid_t = 2; -pub const CLOCK_THREAD_CPUTIME_ID: ::clockid_t = 3; -// FIXME: Add these constants once uclibc gets them. -// pub const CLOCK_SGI_CYCLE: ::clockid_t = 10; -// pub const CLOCK_TAI: ::clockid_t = 11; -pub const TIMER_ABSTIME: ::c_int = 1; - -pub const RLIMIT_CPU: ::c_int = 0; -pub const RLIMIT_FSIZE: ::c_int = 1; -pub const RLIMIT_DATA: ::c_int = 2; -pub const RLIMIT_STACK: ::c_int = 3; -pub const RLIMIT_CORE: ::c_int = 4; -pub const RLIMIT_LOCKS: ::c_int = 10; -pub const RLIMIT_SIGPENDING: ::c_int = 11; -pub const RLIMIT_MSGQUEUE: ::c_int = 12; -pub const RLIMIT_NICE: ::c_int = 13; -pub const RLIMIT_RTPRIO: ::c_int = 14; - -pub const RUSAGE_SELF: ::c_int = 0; - -pub const O_RDONLY: ::c_int = 0; -pub const O_WRONLY: ::c_int = 1; -pub const O_RDWR: ::c_int = 2; - -pub const SOCK_CLOEXEC: ::c_int = O_CLOEXEC; - -pub const S_IFIFO: ::mode_t = 4096; -pub const S_IFCHR: ::mode_t = 8192; -pub const S_IFBLK: ::mode_t = 24576; -pub const S_IFDIR: ::mode_t = 16384; -pub const S_IFREG: ::mode_t = 32768; -pub const S_IFLNK: ::mode_t = 40960; -pub const S_IFSOCK: ::mode_t = 49152; -pub const S_IFMT: ::mode_t = 61440; -pub const S_IRWXU: ::mode_t = 448; -pub const S_IXUSR: ::mode_t = 64; -pub const S_IWUSR: ::mode_t = 128; -pub const S_IRUSR: ::mode_t = 256; -pub const S_IRWXG: ::mode_t = 56; -pub const S_IXGRP: ::mode_t = 8; -pub const S_IWGRP: ::mode_t = 16; -pub const S_IRGRP: ::mode_t = 32; -pub const S_IRWXO: ::mode_t = 7; -pub const S_IXOTH: ::mode_t = 1; -pub const S_IWOTH: ::mode_t = 2; -pub const S_IROTH: ::mode_t = 4; -pub const F_OK: ::c_int = 0; -pub const R_OK: ::c_int = 4; -pub const W_OK: ::c_int = 2; -pub const X_OK: ::c_int = 1; -pub const STDIN_FILENO: ::c_int = 0; -pub const STDOUT_FILENO: ::c_int = 1; -pub const STDERR_FILENO: ::c_int = 2; -pub const SIGHUP: ::c_int = 1; -pub const SIGINT: ::c_int = 2; -pub const SIGQUIT: ::c_int = 3; -pub const SIGILL: ::c_int = 4; -pub const SIGABRT: ::c_int = 6; -pub const SIGFPE: ::c_int = 8; -pub const SIGKILL: ::c_int = 9; -pub const SIGSEGV: ::c_int = 11; -pub const SIGPIPE: ::c_int = 13; -pub const SIGALRM: ::c_int = 14; -pub const SIGTERM: ::c_int = 15; - -pub const PROT_NONE: ::c_int = 0; -pub const PROT_READ: ::c_int = 1; -pub const PROT_WRITE: ::c_int = 2; -pub const PROT_EXEC: ::c_int = 4; - -pub const LC_CTYPE: ::c_int = 0; -pub const LC_NUMERIC: ::c_int = 1; -pub const LC_MONETARY: ::c_int = 2; -pub const LC_TIME: ::c_int = 3; -pub const LC_COLLATE: ::c_int = 4; -pub const LC_MESSAGES: ::c_int = 5; -pub const LC_ALL: ::c_int = 6; -pub const LC_CTYPE_MASK: ::c_int = 1 << LC_CTYPE; -pub const LC_NUMERIC_MASK: ::c_int = 1 << LC_NUMERIC; -pub const LC_TIME_MASK: ::c_int = 1 << LC_TIME; -pub const LC_COLLATE_MASK: ::c_int = 1 << LC_COLLATE; -pub const LC_MONETARY_MASK: ::c_int = 1 << LC_MONETARY; -pub const LC_MESSAGES_MASK: ::c_int = 1 << LC_MESSAGES; -// LC_ALL_MASK defined per platform - -pub const MAP_FILE: ::c_int = 0x0000; -pub const MAP_SHARED: ::c_int = 0x0001; -pub const MAP_PRIVATE: ::c_int = 0x0002; -pub const MAP_FIXED: ::c_int = 0x0010; - -pub const MAP_FAILED: *mut ::c_void = !0 as *mut ::c_void; - -// MS_ flags for msync(2) -pub const MS_ASYNC: ::c_int = 0x0001; -pub const MS_INVALIDATE: ::c_int = 0x0002; -pub const MS_SYNC: ::c_int = 0x0004; - -// MS_ flags for mount(2) -pub const MS_RDONLY: ::c_ulong = 0x01; -pub const MS_NOSUID: ::c_ulong = 0x02; -pub const MS_NODEV: ::c_ulong = 0x04; -pub const MS_NOEXEC: ::c_ulong = 0x08; -pub const MS_SYNCHRONOUS: ::c_ulong = 0x10; -pub const MS_REMOUNT: ::c_ulong = 0x20; -pub const MS_MANDLOCK: ::c_ulong = 0x40; -pub const MS_NOATIME: ::c_ulong = 0x0400; -pub const MS_NODIRATIME: ::c_ulong = 0x0800; -pub const MS_BIND: ::c_ulong = 0x1000; -pub const MS_NOUSER: ::c_ulong = 0x80000000; -pub const MS_MGC_VAL: ::c_ulong = 0xc0ed0000; -pub const MS_MGC_MSK: ::c_ulong = 0xffff0000; -pub const MS_RMT_MASK: ::c_ulong = 0x800051; - -pub const EPERM: ::c_int = 1; -pub const ENOENT: ::c_int = 2; -pub const ESRCH: ::c_int = 3; -pub const EINTR: ::c_int = 4; -pub const EIO: ::c_int = 5; -pub const ENXIO: ::c_int = 6; -pub const E2BIG: ::c_int = 7; -pub const ENOEXEC: ::c_int = 8; -pub const EBADF: ::c_int = 9; -pub const ECHILD: ::c_int = 10; -pub const EAGAIN: ::c_int = 11; -pub const ENOMEM: ::c_int = 12; -pub const EACCES: ::c_int = 13; -pub const EFAULT: ::c_int = 14; -pub const ENOTBLK: ::c_int = 15; -pub const EBUSY: ::c_int = 16; -pub const EEXIST: ::c_int = 17; -pub const EXDEV: ::c_int = 18; -pub const ENODEV: ::c_int = 19; -pub const ENOTDIR: ::c_int = 20; -pub const EISDIR: ::c_int = 21; -pub const EINVAL: ::c_int = 22; -pub const ENFILE: ::c_int = 23; -pub const EMFILE: ::c_int = 24; -pub const ENOTTY: ::c_int = 25; -pub const ETXTBSY: ::c_int = 26; -pub const EFBIG: ::c_int = 27; -pub const ENOSPC: ::c_int = 28; -pub const ESPIPE: ::c_int = 29; -pub const EROFS: ::c_int = 30; -pub const EMLINK: ::c_int = 31; -pub const EPIPE: ::c_int = 32; -pub const EDOM: ::c_int = 33; -pub const ERANGE: ::c_int = 34; -pub const EWOULDBLOCK: ::c_int = EAGAIN; - -pub const SCM_RIGHTS: ::c_int = 0x01; -pub const SCM_CREDENTIALS: ::c_int = 0x02; - -// netinet/in.h -// NOTE: These are in addition to the constants defined in src/unix/mod.rs - -// IPPROTO_IP defined in src/unix/mod.rs -/// Hop-by-hop option header -pub const IPPROTO_HOPOPTS: ::c_int = 0; -// IPPROTO_ICMP defined in src/unix/mod.rs -/// group mgmt protocol -pub const IPPROTO_IGMP: ::c_int = 2; -/// for compatibility -pub const IPPROTO_IPIP: ::c_int = 4; -// IPPROTO_TCP defined in src/unix/mod.rs -/// exterior gateway protocol -pub const IPPROTO_EGP: ::c_int = 8; -/// pup -pub const IPPROTO_PUP: ::c_int = 12; -// IPPROTO_UDP defined in src/unix/mod.rs -/// xns idp -pub const IPPROTO_IDP: ::c_int = 22; -/// tp-4 w/ class negotiation -pub const IPPROTO_TP: ::c_int = 29; -/// DCCP -pub const IPPROTO_DCCP: ::c_int = 33; -// IPPROTO_IPV6 defined in src/unix/mod.rs -/// IP6 routing header -pub const IPPROTO_ROUTING: ::c_int = 43; -/// IP6 fragmentation header -pub const IPPROTO_FRAGMENT: ::c_int = 44; -/// resource reservation -pub const IPPROTO_RSVP: ::c_int = 46; -/// General Routing Encap. -pub const IPPROTO_GRE: ::c_int = 47; -/// IP6 Encap Sec. Payload -pub const IPPROTO_ESP: ::c_int = 50; -/// IP6 Auth Header -pub const IPPROTO_AH: ::c_int = 51; -// IPPROTO_ICMPV6 defined in src/unix/mod.rs -/// IP6 no next header -pub const IPPROTO_NONE: ::c_int = 59; -/// IP6 destination option -pub const IPPROTO_DSTOPTS: ::c_int = 60; -pub const IPPROTO_MTP: ::c_int = 92; -pub const IPPROTO_BEETPH: ::c_int = 94; -/// encapsulation header -pub const IPPROTO_ENCAP: ::c_int = 98; -/// Protocol indep. multicast -pub const IPPROTO_PIM: ::c_int = 103; -/// IP Payload Comp. Protocol -pub const IPPROTO_COMP: ::c_int = 108; -/// SCTP -pub const IPPROTO_SCTP: ::c_int = 132; -pub const IPPROTO_MH: ::c_int = 135; -pub const IPPROTO_UDPLITE: ::c_int = 136; -pub const IPPROTO_MPLS: ::c_int = 137; -/// raw IP packet -pub const IPPROTO_RAW: ::c_int = 255; -pub const IPPROTO_MAX: ::c_int = 256; - -pub const PROT_GROWSDOWN: ::c_int = 0x1000000; -pub const PROT_GROWSUP: ::c_int = 0x2000000; - -pub const MAP_TYPE: ::c_int = 0x000f; - -pub const MADV_NORMAL: ::c_int = 0; -pub const MADV_RANDOM: ::c_int = 1; -pub const MADV_SEQUENTIAL: ::c_int = 2; -pub const MADV_WILLNEED: ::c_int = 3; -pub const MADV_DONTNEED: ::c_int = 4; -pub const MADV_REMOVE: ::c_int = 9; -pub const MADV_DONTFORK: ::c_int = 10; -pub const MADV_DOFORK: ::c_int = 11; -pub const MADV_MERGEABLE: ::c_int = 12; -pub const MADV_UNMERGEABLE: ::c_int = 13; -pub const MADV_HWPOISON: ::c_int = 100; - -// https://github.com/kraj/uClibc/blob/master/include/net/if.h#L44 -pub const IFF_UP: ::c_int = 0x1; // Interface is up. -pub const IFF_BROADCAST: ::c_int = 0x2; // Broadcast address valid. -pub const IFF_DEBUG: ::c_int = 0x4; // Turn on debugging. -pub const IFF_LOOPBACK: ::c_int = 0x8; // Is a loopback net. -pub const IFF_POINTOPOINT: ::c_int = 0x10; // Interface is point-to-point link. -pub const IFF_NOTRAILERS: ::c_int = 0x20; // Avoid use of trailers. -pub const IFF_RUNNING: ::c_int = 0x40; // Resources allocated. -pub const IFF_NOARP: ::c_int = 0x80; // No address resolution protocol. -pub const IFF_PROMISC: ::c_int = 0x100; // Receive all packets. - -// Not supported -pub const IFF_ALLMULTI: ::c_int = 0x200; // Receive all multicast packets. -pub const IFF_MASTER: ::c_int = 0x400; // Master of a load balancer. -pub const IFF_SLAVE: ::c_int = 0x800; // Slave of a load balancer. -pub const IFF_MULTICAST: ::c_int = 0x1000; // Supports multicast. -pub const IFF_PORTSEL: ::c_int = 0x2000; // Can set media type. -pub const IFF_AUTOMEDIA: ::c_int = 0x4000; // Auto media select active. - -// Dialup device with changing addresses. -pub const IFF_DYNAMIC: ::c_int = 0x8000; - -pub const SOL_IP: ::c_int = 0; -pub const SOL_TCP: ::c_int = 6; -pub const SOL_IPV6: ::c_int = 41; -pub const SOL_ICMPV6: ::c_int = 58; -pub const SOL_RAW: ::c_int = 255; -pub const SOL_DECNET: ::c_int = 261; -pub const SOL_X25: ::c_int = 262; -pub const SOL_PACKET: ::c_int = 263; -pub const SOL_ATM: ::c_int = 264; -pub const SOL_AAL: ::c_int = 265; -pub const SOL_IRDA: ::c_int = 266; - -pub const AF_UNSPEC: ::c_int = 0; -pub const AF_UNIX: ::c_int = 1; -pub const AF_LOCAL: ::c_int = 1; -pub const AF_INET: ::c_int = 2; -pub const AF_AX25: ::c_int = 3; -pub const AF_IPX: ::c_int = 4; -pub const AF_APPLETALK: ::c_int = 5; -pub const AF_NETROM: ::c_int = 6; -pub const AF_BRIDGE: ::c_int = 7; -pub const AF_ATMPVC: ::c_int = 8; -pub const AF_X25: ::c_int = 9; -pub const AF_INET6: ::c_int = 10; -pub const AF_ROSE: ::c_int = 11; -pub const AF_DECnet: ::c_int = 12; -pub const AF_NETBEUI: ::c_int = 13; -pub const AF_SECURITY: ::c_int = 14; -pub const AF_KEY: ::c_int = 15; -pub const AF_NETLINK: ::c_int = 16; -pub const AF_ROUTE: ::c_int = AF_NETLINK; -pub const AF_PACKET: ::c_int = 17; -pub const AF_ASH: ::c_int = 18; -pub const AF_ECONET: ::c_int = 19; -pub const AF_ATMSVC: ::c_int = 20; -pub const AF_SNA: ::c_int = 22; -pub const AF_IRDA: ::c_int = 23; -pub const AF_PPPOX: ::c_int = 24; -pub const AF_WANPIPE: ::c_int = 25; -pub const AF_LLC: ::c_int = 26; -pub const AF_CAN: ::c_int = 29; -pub const AF_TIPC: ::c_int = 30; -pub const AF_BLUETOOTH: ::c_int = 31; -pub const AF_IUCV: ::c_int = 32; -pub const AF_RXRPC: ::c_int = 33; -pub const AF_ISDN: ::c_int = 34; -pub const AF_PHONET: ::c_int = 35; -pub const AF_IEEE802154: ::c_int = 36; -pub const AF_CAIF: ::c_int = 37; -pub const AF_ALG: ::c_int = 38; - -pub const PF_UNSPEC: ::c_int = AF_UNSPEC; -pub const PF_UNIX: ::c_int = AF_UNIX; -pub const PF_LOCAL: ::c_int = AF_LOCAL; -pub const PF_INET: ::c_int = AF_INET; -pub const PF_AX25: ::c_int = AF_AX25; -pub const PF_IPX: ::c_int = AF_IPX; -pub const PF_APPLETALK: ::c_int = AF_APPLETALK; -pub const PF_NETROM: ::c_int = AF_NETROM; -pub const PF_BRIDGE: ::c_int = AF_BRIDGE; -pub const PF_ATMPVC: ::c_int = AF_ATMPVC; -pub const PF_X25: ::c_int = AF_X25; -pub const PF_INET6: ::c_int = AF_INET6; -pub const PF_ROSE: ::c_int = AF_ROSE; -pub const PF_DECnet: ::c_int = AF_DECnet; -pub const PF_NETBEUI: ::c_int = AF_NETBEUI; -pub const PF_SECURITY: ::c_int = AF_SECURITY; -pub const PF_KEY: ::c_int = AF_KEY; -pub const PF_NETLINK: ::c_int = AF_NETLINK; -pub const PF_ROUTE: ::c_int = AF_ROUTE; -pub const PF_PACKET: ::c_int = AF_PACKET; -pub const PF_ASH: ::c_int = AF_ASH; -pub const PF_ECONET: ::c_int = AF_ECONET; -pub const PF_ATMSVC: ::c_int = AF_ATMSVC; -pub const PF_SNA: ::c_int = AF_SNA; -pub const PF_IRDA: ::c_int = AF_IRDA; -pub const PF_PPPOX: ::c_int = AF_PPPOX; -pub const PF_WANPIPE: ::c_int = AF_WANPIPE; -pub const PF_LLC: ::c_int = AF_LLC; -pub const PF_CAN: ::c_int = AF_CAN; -pub const PF_TIPC: ::c_int = AF_TIPC; -pub const PF_BLUETOOTH: ::c_int = AF_BLUETOOTH; -pub const PF_IUCV: ::c_int = AF_IUCV; -pub const PF_RXRPC: ::c_int = AF_RXRPC; -pub const PF_ISDN: ::c_int = AF_ISDN; -pub const PF_PHONET: ::c_int = AF_PHONET; -pub const PF_IEEE802154: ::c_int = AF_IEEE802154; -pub const PF_CAIF: ::c_int = AF_CAIF; -pub const PF_ALG: ::c_int = AF_ALG; - -pub const SOMAXCONN: ::c_int = 128; - -pub const MSG_OOB: ::c_int = 1; -pub const MSG_PEEK: ::c_int = 2; -pub const MSG_DONTROUTE: ::c_int = 4; -pub const MSG_CTRUNC: ::c_int = 8; -pub const MSG_TRUNC: ::c_int = 0x20; -pub const MSG_DONTWAIT: ::c_int = 0x40; -pub const MSG_EOR: ::c_int = 0x80; -pub const MSG_WAITALL: ::c_int = 0x100; -pub const MSG_FIN: ::c_int = 0x200; -pub const MSG_SYN: ::c_int = 0x400; -pub const MSG_CONFIRM: ::c_int = 0x800; -pub const MSG_RST: ::c_int = 0x1000; -pub const MSG_ERRQUEUE: ::c_int = 0x2000; -pub const MSG_NOSIGNAL: ::c_int = 0x4000; -pub const MSG_MORE: ::c_int = 0x8000; -pub const MSG_WAITFORONE: ::c_int = 0x10000; -pub const MSG_CMSG_CLOEXEC: ::c_int = 0x40000000; - -pub const SOCK_RAW: ::c_int = 3; -pub const IP_MULTICAST_TTL: ::c_int = 33; -pub const IP_MULTICAST_LOOP: ::c_int = 34; -pub const IP_TTL: ::c_int = 2; -pub const IP_HDRINCL: ::c_int = 3; -pub const IP_ADD_MEMBERSHIP: ::c_int = 35; -pub const IP_DROP_MEMBERSHIP: ::c_int = 36; -pub const IPV6_ADD_MEMBERSHIP: ::c_int = 20; -pub const IPV6_DROP_MEMBERSHIP: ::c_int = 21; - -pub const IPV6_JOIN_GROUP: ::c_int = 20; -pub const IPV6_LEAVE_GROUP: ::c_int = 21; - -pub const TCP_NODELAY: ::c_int = 1; -pub const TCP_MAXSEG: ::c_int = 2; -pub const TCP_CORK: ::c_int = 3; -pub const TCP_KEEPIDLE: ::c_int = 4; -pub const TCP_KEEPINTVL: ::c_int = 5; -pub const TCP_KEEPCNT: ::c_int = 6; -pub const TCP_SYNCNT: ::c_int = 7; -pub const TCP_LINGER2: ::c_int = 8; -pub const TCP_DEFER_ACCEPT: ::c_int = 9; -pub const TCP_WINDOW_CLAMP: ::c_int = 10; -pub const TCP_INFO: ::c_int = 11; -pub const TCP_QUICKACK: ::c_int = 12; -pub const TCP_CONGESTION: ::c_int = 13; - -// Source: -// https://github.com/kraj/uClibc/blob/ca1c74d67dd115d059a875150e10b8560a9c35a8 -// /libc/sysdeps/linux/common/bits/in.h -// Same for all architectures -pub const IPV6_MULTICAST_HOPS: ::c_int = 18; -pub const IP_MULTICAST_IF: ::c_int = 32; -pub const IPV6_MULTICAST_IF: ::c_int = 17; -pub const IPV6_UNICAST_HOPS: ::c_int = 16; - -// Source: -// https://github.com/kraj/uClibc/tree/ca1c74d67dd115d059a875150e10b8560a9c35a8 -// Same for all architectures -pub const FUTEX_WAIT: ::c_int = 0; -pub const FUTEX_PRIVATE_FLAG: ::c_int = 128; -pub const FUTEX_WAKE: ::c_int = 1; - -pub const IPV6_MULTICAST_LOOP: ::c_int = 19; -pub const IPV6_V6ONLY: ::c_int = 26; - -pub const SO_DEBUG: ::c_int = 1; - -pub const SHUT_RD: ::c_int = 0; -pub const SHUT_WR: ::c_int = 1; -pub const SHUT_RDWR: ::c_int = 2; - -pub const LOCK_SH: ::c_int = 1; -pub const LOCK_EX: ::c_int = 2; -pub const LOCK_NB: ::c_int = 4; -pub const LOCK_UN: ::c_int = 8; - -pub const SS_ONSTACK: ::c_int = 1; -pub const SS_DISABLE: ::c_int = 2; - -pub const PATH_MAX: ::c_int = 4096; - -pub const FD_SETSIZE: usize = 1024; - -pub const EPOLLIN: ::c_int = 0x1; -pub const EPOLLPRI: ::c_int = 0x2; -pub const EPOLLOUT: ::c_int = 0x4; -pub const EPOLLRDNORM: ::c_int = 0x40; -pub const EPOLLRDBAND: ::c_int = 0x80; -pub const EPOLLWRNORM: ::c_int = 0x100; -pub const EPOLLWRBAND: ::c_int = 0x200; -pub const EPOLLMSG: ::c_int = 0x400; -pub const EPOLLERR: ::c_int = 0x8; -pub const EPOLLHUP: ::c_int = 0x10; -pub const EPOLLET: ::c_int = 0x80000000; - -pub const EPOLL_CTL_ADD: ::c_int = 1; -pub const EPOLL_CTL_MOD: ::c_int = 3; -pub const EPOLL_CTL_DEL: ::c_int = 2; - -pub const MNT_DETACH: ::c_int = 0x2; -pub const MNT_EXPIRE: ::c_int = 0x4; - -pub const MNT_FORCE: ::c_int = 0x1; - -pub const Q_SYNC: ::c_int = 0x600; -pub const Q_QUOTAON: ::c_int = 0x100; -pub const Q_QUOTAOFF: ::c_int = 0x200; -pub const Q_GETQUOTA: ::c_int = 0x300; -pub const Q_SETQUOTA: ::c_int = 0x400; - -pub const TCIOFF: ::c_int = 2; -pub const TCION: ::c_int = 3; -pub const TCOOFF: ::c_int = 0; -pub const TCOON: ::c_int = 1; -pub const TCIFLUSH: ::c_int = 0; -pub const TCOFLUSH: ::c_int = 1; -pub const TCIOFLUSH: ::c_int = 2; -pub const NL0: ::tcflag_t = 0x00000000; -pub const NL1: ::tcflag_t = 0x00000100; -pub const TAB0: ::tcflag_t = 0x00000000; -pub const CR0: ::tcflag_t = 0x00000000; -pub const FF0: ::tcflag_t = 0x00000000; -pub const BS0: ::tcflag_t = 0x00000000; -pub const VT0: ::tcflag_t = 0x00000000; -pub const VERASE: usize = 2; -pub const VKILL: usize = 3; -pub const VINTR: usize = 0; -pub const VQUIT: usize = 1; -pub const VLNEXT: usize = 15; -pub const IGNBRK: ::tcflag_t = 0x00000001; -pub const BRKINT: ::tcflag_t = 0x00000002; -pub const IGNPAR: ::tcflag_t = 0x00000004; -pub const PARMRK: ::tcflag_t = 0x00000008; -pub const INPCK: ::tcflag_t = 0x00000010; -pub const ISTRIP: ::tcflag_t = 0x00000020; -pub const INLCR: ::tcflag_t = 0x00000040; -pub const IGNCR: ::tcflag_t = 0x00000080; -pub const ICRNL: ::tcflag_t = 0x00000100; -pub const IXANY: ::tcflag_t = 0x00000800; -pub const IMAXBEL: ::tcflag_t = 0x00002000; -pub const OPOST: ::tcflag_t = 0x1; -pub const CS5: ::tcflag_t = 0x00000000; -pub const CRTSCTS: ::tcflag_t = 0x80000000; -pub const ECHO: ::tcflag_t = 0x00000008; - -pub const CLONE_VM: ::c_int = 0x100; -pub const CLONE_FS: ::c_int = 0x200; -pub const CLONE_FILES: ::c_int = 0x400; -pub const CLONE_SIGHAND: ::c_int = 0x800; -pub const CLONE_PTRACE: ::c_int = 0x2000; -pub const CLONE_VFORK: ::c_int = 0x4000; -pub const CLONE_PARENT: ::c_int = 0x8000; -pub const CLONE_THREAD: ::c_int = 0x10000; -pub const CLONE_NEWNS: ::c_int = 0x20000; -pub const CLONE_SYSVSEM: ::c_int = 0x40000; -pub const CLONE_SETTLS: ::c_int = 0x80000; -pub const CLONE_PARENT_SETTID: ::c_int = 0x100000; -pub const CLONE_CHILD_CLEARTID: ::c_int = 0x200000; -pub const CLONE_DETACHED: ::c_int = 0x400000; -pub const CLONE_UNTRACED: ::c_int = 0x800000; -pub const CLONE_CHILD_SETTID: ::c_int = 0x01000000; -pub const CLONE_NEWUTS: ::c_int = 0x04000000; -pub const CLONE_NEWIPC: ::c_int = 0x08000000; -pub const CLONE_NEWUSER: ::c_int = 0x10000000; -pub const CLONE_NEWPID: ::c_int = 0x20000000; -pub const CLONE_NEWNET: ::c_int = 0x40000000; -pub const CLONE_IO: ::c_int = 0x80000000; - -pub const WNOHANG: ::c_int = 0x00000001; -pub const WUNTRACED: ::c_int = 0x00000002; -pub const WSTOPPED: ::c_int = WUNTRACED; -pub const WEXITED: ::c_int = 0x00000004; -pub const WCONTINUED: ::c_int = 0x00000008; -pub const WNOWAIT: ::c_int = 0x01000000; - -pub const __WNOTHREAD: ::c_int = 0x20000000; -pub const __WALL: ::c_int = 0x40000000; -pub const __WCLONE: ::c_int = 0x80000000; - -pub const SPLICE_F_MOVE: ::c_uint = 0x01; -pub const SPLICE_F_NONBLOCK: ::c_uint = 0x02; -pub const SPLICE_F_MORE: ::c_uint = 0x04; -pub const SPLICE_F_GIFT: ::c_uint = 0x08; - -pub const RTLD_LOCAL: ::c_int = 0; -pub const RTLD_LAZY: ::c_int = 1; - -pub const POSIX_FADV_NORMAL: ::c_int = 0; -pub const POSIX_FADV_RANDOM: ::c_int = 1; -pub const POSIX_FADV_SEQUENTIAL: ::c_int = 2; -pub const POSIX_FADV_WILLNEED: ::c_int = 3; - -pub const AT_FDCWD: ::c_int = -100; -pub const AT_SYMLINK_NOFOLLOW: ::c_int = 0x100; -pub const AT_REMOVEDIR: ::c_int = 0x200; -pub const AT_EACCESS: ::c_int = 0x200; -pub const AT_SYMLINK_FOLLOW: ::c_int = 0x400; - -pub const LOG_CRON: ::c_int = 9 << 3; -pub const LOG_AUTHPRIV: ::c_int = 10 << 3; -pub const LOG_FTP: ::c_int = 11 << 3; -pub const LOG_PERROR: ::c_int = 0x20; - -pub const POLLIN: ::c_short = 0x1; -pub const POLLPRI: ::c_short = 0x2; -pub const POLLOUT: ::c_short = 0x4; -pub const POLLERR: ::c_short = 0x8; -pub const POLLHUP: ::c_short = 0x10; -pub const POLLNVAL: ::c_short = 0x20; - -pub const PIPE_BUF: usize = 4096; - -pub const SI_LOAD_SHIFT: ::c_uint = 16; - -pub const SIGEV_SIGNAL: ::c_int = 0; -pub const SIGEV_NONE: ::c_int = 1; -pub const SIGEV_THREAD: ::c_int = 2; - -pub const P_ALL: idtype_t = 0; -pub const P_PID: idtype_t = 1; -pub const P_PGID: idtype_t = 2; - -pub const UTIME_OMIT: c_long = 1073741822; -pub const UTIME_NOW: c_long = 1073741823; - -pub const L_tmpnam: ::c_uint = 20; -pub const _PC_LINK_MAX: ::c_int = 0; -pub const _PC_MAX_CANON: ::c_int = 1; -pub const _PC_MAX_INPUT: ::c_int = 2; -pub const _PC_NAME_MAX: ::c_int = 3; -pub const _PC_PATH_MAX: ::c_int = 4; -pub const _PC_PIPE_BUF: ::c_int = 5; -pub const _PC_CHOWN_RESTRICTED: ::c_int = 6; -pub const _PC_NO_TRUNC: ::c_int = 7; -pub const _PC_VDISABLE: ::c_int = 8; - -pub const _SC_ARG_MAX: ::c_int = 0; -pub const _SC_CHILD_MAX: ::c_int = 1; -pub const _SC_CLK_TCK: ::c_int = 2; -pub const _SC_NGROUPS_MAX: ::c_int = 3; -pub const _SC_OPEN_MAX: ::c_int = 4; -pub const _SC_STREAM_MAX: ::c_int = 5; -pub const _SC_TZNAME_MAX: ::c_int = 6; -pub const _SC_JOB_CONTROL: ::c_int = 7; -pub const _SC_SAVED_IDS: ::c_int = 8; -pub const _SC_REALTIME_SIGNALS: ::c_int = 9; -pub const _SC_PRIORITY_SCHEDULING: ::c_int = 10; -pub const _SC_TIMERS: ::c_int = 11; -pub const _SC_ASYNCHRONOUS_IO: ::c_int = 12; -pub const _SC_PRIORITIZED_IO: ::c_int = 13; -pub const _SC_SYNCHRONIZED_IO: ::c_int = 14; -pub const _SC_FSYNC: ::c_int = 15; -pub const _SC_MAPPED_FILES: ::c_int = 16; -pub const _SC_MEMLOCK: ::c_int = 17; -pub const _SC_MEMLOCK_RANGE: ::c_int = 18; -pub const _SC_MEMORY_PROTECTION: ::c_int = 19; -pub const _SC_MESSAGE_PASSING: ::c_int = 20; -pub const _SC_SEMAPHORES: ::c_int = 21; -pub const _SC_SHARED_MEMORY_OBJECTS: ::c_int = 22; -pub const _SC_AIO_LISTIO_MAX: ::c_int = 23; -pub const _SC_AIO_MAX: ::c_int = 24; -pub const _SC_AIO_PRIO_DELTA_MAX: ::c_int = 25; -pub const _SC_DELAYTIMER_MAX: ::c_int = 26; -pub const _SC_MQ_OPEN_MAX: ::c_int = 27; -pub const _SC_MQ_PRIO_MAX: ::c_int = 28; -pub const _SC_VERSION: ::c_int = 29; -pub const _SC_PAGESIZE: ::c_int = 30; -pub const _SC_PAGE_SIZE: ::c_int = _SC_PAGESIZE; -pub const _SC_RTSIG_MAX: ::c_int = 31; -pub const _SC_SEM_NSEMS_MAX: ::c_int = 32; -pub const _SC_SEM_VALUE_MAX: ::c_int = 33; -pub const _SC_SIGQUEUE_MAX: ::c_int = 34; -pub const _SC_TIMER_MAX: ::c_int = 35; -pub const _SC_BC_BASE_MAX: ::c_int = 36; -pub const _SC_BC_DIM_MAX: ::c_int = 37; -pub const _SC_BC_SCALE_MAX: ::c_int = 38; -pub const _SC_BC_STRING_MAX: ::c_int = 39; -pub const _SC_COLL_WEIGHTS_MAX: ::c_int = 40; -pub const _SC_EXPR_NEST_MAX: ::c_int = 42; -pub const _SC_LINE_MAX: ::c_int = 43; -pub const _SC_RE_DUP_MAX: ::c_int = 44; -pub const _SC_2_VERSION: ::c_int = 46; -pub const _SC_2_C_BIND: ::c_int = 47; -pub const _SC_2_C_DEV: ::c_int = 48; -pub const _SC_2_FORT_DEV: ::c_int = 49; -pub const _SC_2_FORT_RUN: ::c_int = 50; -pub const _SC_2_SW_DEV: ::c_int = 51; -pub const _SC_2_LOCALEDEF: ::c_int = 52; -pub const _SC_IOV_MAX: ::c_int = 60; -pub const _SC_THREADS: ::c_int = 67; -pub const _SC_THREAD_SAFE_FUNCTIONS: ::c_int = 68; -pub const _SC_GETGR_R_SIZE_MAX: ::c_int = 69; -pub const _SC_GETPW_R_SIZE_MAX: ::c_int = 70; -pub const _SC_LOGIN_NAME_MAX: ::c_int = 71; -pub const _SC_TTY_NAME_MAX: ::c_int = 72; -pub const _SC_THREAD_DESTRUCTOR_ITERATIONS: ::c_int = 73; -pub const _SC_THREAD_KEYS_MAX: ::c_int = 74; -pub const _SC_THREAD_STACK_MIN: ::c_int = 75; -pub const _SC_THREAD_THREADS_MAX: ::c_int = 76; -pub const _SC_THREAD_ATTR_STACKADDR: ::c_int = 77; -pub const _SC_THREAD_ATTR_STACKSIZE: ::c_int = 78; -pub const _SC_THREAD_PRIORITY_SCHEDULING: ::c_int = 79; -pub const _SC_THREAD_PRIO_INHERIT: ::c_int = 80; -pub const _SC_THREAD_PRIO_PROTECT: ::c_int = 81; -pub const _SC_NPROCESSORS_ONLN: ::c_int = 84; -pub const _SC_ATEXIT_MAX: ::c_int = 87; -pub const _SC_XOPEN_VERSION: ::c_int = 89; -pub const _SC_XOPEN_XCU_VERSION: ::c_int = 90; -pub const _SC_XOPEN_UNIX: ::c_int = 91; -pub const _SC_XOPEN_CRYPT: ::c_int = 92; -pub const _SC_XOPEN_ENH_I18N: ::c_int = 93; -pub const _SC_XOPEN_SHM: ::c_int = 94; -pub const _SC_2_CHAR_TERM: ::c_int = 95; -pub const _SC_2_UPE: ::c_int = 97; -pub const _SC_XBS5_ILP32_OFF32: ::c_int = 125; -pub const _SC_XBS5_ILP32_OFFBIG: ::c_int = 126; -pub const _SC_XBS5_LPBIG_OFFBIG: ::c_int = 128; -pub const _SC_XOPEN_LEGACY: ::c_int = 129; -pub const _SC_XOPEN_REALTIME: ::c_int = 130; -pub const _SC_XOPEN_REALTIME_THREADS: ::c_int = 131; -pub const _SC_HOST_NAME_MAX: ::c_int = 180; - -pub const RLIM_SAVED_MAX: ::rlim_t = RLIM_INFINITY; -pub const RLIM_SAVED_CUR: ::rlim_t = RLIM_INFINITY; - -pub const GLOB_ERR: ::c_int = 1 << 0; -pub const GLOB_MARK: ::c_int = 1 << 1; -pub const GLOB_NOSORT: ::c_int = 1 << 2; -pub const GLOB_DOOFFS: ::c_int = 1 << 3; -pub const GLOB_NOCHECK: ::c_int = 1 << 4; -pub const GLOB_APPEND: ::c_int = 1 << 5; -pub const GLOB_NOESCAPE: ::c_int = 1 << 6; - -pub const GLOB_NOSPACE: ::c_int = 1; -pub const GLOB_ABORTED: ::c_int = 2; -pub const GLOB_NOMATCH: ::c_int = 3; - -pub const POSIX_MADV_NORMAL: ::c_int = 0; -pub const POSIX_MADV_RANDOM: ::c_int = 1; -pub const POSIX_MADV_SEQUENTIAL: ::c_int = 2; -pub const POSIX_MADV_WILLNEED: ::c_int = 3; - -pub const S_IEXEC: mode_t = 64; -pub const S_IWRITE: mode_t = 128; -pub const S_IREAD: mode_t = 256; - -pub const F_LOCK: ::c_int = 1; -pub const F_TEST: ::c_int = 3; -pub const F_TLOCK: ::c_int = 2; -pub const F_ULOCK: ::c_int = 0; - -pub const ST_RDONLY: ::c_ulong = 1; -pub const ST_NOSUID: ::c_ulong = 2; -pub const ST_NODEV: ::c_ulong = 4; -pub const ST_NOEXEC: ::c_ulong = 8; -pub const ST_SYNCHRONOUS: ::c_ulong = 16; -pub const ST_MANDLOCK: ::c_ulong = 64; -pub const ST_WRITE: ::c_ulong = 128; -pub const ST_APPEND: ::c_ulong = 256; -pub const ST_IMMUTABLE: ::c_ulong = 512; -pub const ST_NOATIME: ::c_ulong = 1024; -pub const ST_NODIRATIME: ::c_ulong = 2048; - -pub const RTLD_NEXT: *mut ::c_void = -1i64 as *mut ::c_void; -pub const RTLD_DEFAULT: *mut ::c_void = 0i64 as *mut ::c_void; -pub const RTLD_NODELETE: ::c_int = 0x1000; -pub const RTLD_NOW: ::c_int = 0x2; - -pub const TCP_MD5SIG: ::c_int = 14; - -align_const! { - pub const PTHREAD_MUTEX_INITIALIZER: pthread_mutex_t = pthread_mutex_t { - size: [0; __SIZEOF_PTHREAD_MUTEX_T], - }; - pub const PTHREAD_COND_INITIALIZER: pthread_cond_t = pthread_cond_t { - size: [0; __SIZEOF_PTHREAD_COND_T], - }; - pub const PTHREAD_RWLOCK_INITIALIZER: pthread_rwlock_t = pthread_rwlock_t { - size: [0; __SIZEOF_PTHREAD_RWLOCK_T], - }; -} -pub const PTHREAD_MUTEX_NORMAL: ::c_int = 0; -pub const PTHREAD_MUTEX_RECURSIVE: ::c_int = 1; -pub const PTHREAD_MUTEX_ERRORCHECK: ::c_int = 2; -pub const PTHREAD_MUTEX_DEFAULT: ::c_int = PTHREAD_MUTEX_NORMAL; -pub const __SIZEOF_PTHREAD_COND_T: usize = 48; - -pub const SCHED_OTHER: ::c_int = 0; -pub const SCHED_FIFO: ::c_int = 1; -pub const SCHED_RR: ::c_int = 2; -pub const SCHED_BATCH: ::c_int = 3; -pub const SCHED_IDLE: ::c_int = 5; - -// System V IPC -pub const IPC_PRIVATE: ::key_t = 0; - -pub const IPC_CREAT: ::c_int = 0o1000; -pub const IPC_EXCL: ::c_int = 0o2000; -pub const IPC_NOWAIT: ::c_int = 0o4000; - -pub const IPC_RMID: ::c_int = 0; -pub const IPC_SET: ::c_int = 1; -pub const IPC_STAT: ::c_int = 2; -pub const IPC_INFO: ::c_int = 3; -pub const MSG_STAT: ::c_int = 11; -pub const MSG_INFO: ::c_int = 12; - -pub const MSG_NOERROR: ::c_int = 0o10000; -pub const MSG_EXCEPT: ::c_int = 0o20000; - -pub const SHM_R: ::c_int = 0o400; -pub const SHM_W: ::c_int = 0o200; - -pub const SHM_RDONLY: ::c_int = 0o10000; -pub const SHM_RND: ::c_int = 0o20000; -pub const SHM_REMAP: ::c_int = 0o40000; - -pub const SHM_LOCK: ::c_int = 11; -pub const SHM_UNLOCK: ::c_int = 12; - -pub const SHM_HUGETLB: ::c_int = 0o4000; -pub const SHM_NORESERVE: ::c_int = 0o10000; - -pub const EPOLLRDHUP: ::c_int = 0x2000; -pub const EPOLLONESHOT: ::c_int = 0x40000000; - -pub const QFMT_VFS_OLD: ::c_int = 1; -pub const QFMT_VFS_V0: ::c_int = 2; - -pub const EFD_SEMAPHORE: ::c_int = 0x1; - -pub const LOG_NFACILITIES: ::c_int = 24; - -pub const SEM_FAILED: *mut ::sem_t = 0 as *mut sem_t; - -pub const RB_AUTOBOOT: ::c_int = 0x01234567u32 as i32; -pub const RB_HALT_SYSTEM: ::c_int = 0xcdef0123u32 as i32; -pub const RB_ENABLE_CAD: ::c_int = 0x89abcdefu32 as i32; -pub const RB_DISABLE_CAD: ::c_int = 0x00000000u32 as i32; -pub const RB_POWER_OFF: ::c_int = 0x4321fedcu32 as i32; - -pub const AI_PASSIVE: ::c_int = 0x0001; -pub const AI_CANONNAME: ::c_int = 0x0002; -pub const AI_NUMERICHOST: ::c_int = 0x0004; -pub const AI_V4MAPPED: ::c_int = 0x0008; -pub const AI_ALL: ::c_int = 0x0010; -pub const AI_ADDRCONFIG: ::c_int = 0x0020; - -pub const AI_NUMERICSERV: ::c_int = 0x0400; - -pub const EAI_BADFLAGS: ::c_int = -1; -pub const EAI_NONAME: ::c_int = -2; -pub const EAI_AGAIN: ::c_int = -3; -pub const EAI_FAIL: ::c_int = -4; -pub const EAI_FAMILY: ::c_int = -6; -pub const EAI_SOCKTYPE: ::c_int = -7; -pub const EAI_SERVICE: ::c_int = -8; -pub const EAI_MEMORY: ::c_int = -10; -pub const EAI_OVERFLOW: ::c_int = -12; - -pub const NI_NUMERICHOST: ::c_int = 1; -pub const NI_NUMERICSERV: ::c_int = 2; -pub const NI_NOFQDN: ::c_int = 4; -pub const NI_NAMEREQD: ::c_int = 8; -pub const NI_DGRAM: ::c_int = 16; - -pub const SYNC_FILE_RANGE_WAIT_BEFORE: ::c_uint = 1; -pub const SYNC_FILE_RANGE_WRITE: ::c_uint = 2; -pub const SYNC_FILE_RANGE_WAIT_AFTER: ::c_uint = 4; - -pub const EAI_SYSTEM: ::c_int = -11; - -pub const MREMAP_MAYMOVE: ::c_int = 1; -pub const MREMAP_FIXED: ::c_int = 2; - -pub const PR_SET_PDEATHSIG: ::c_int = 1; -pub const PR_GET_PDEATHSIG: ::c_int = 2; - -pub const PR_GET_DUMPABLE: ::c_int = 3; -pub const PR_SET_DUMPABLE: ::c_int = 4; - -pub const PR_GET_UNALIGN: ::c_int = 5; -pub const PR_SET_UNALIGN: ::c_int = 6; -pub const PR_UNALIGN_NOPRINT: ::c_int = 1; -pub const PR_UNALIGN_SIGBUS: ::c_int = 2; - -pub const PR_GET_KEEPCAPS: ::c_int = 7; -pub const PR_SET_KEEPCAPS: ::c_int = 8; - -pub const PR_GET_FPEMU: ::c_int = 9; -pub const PR_SET_FPEMU: ::c_int = 10; -pub const PR_FPEMU_NOPRINT: ::c_int = 1; -pub const PR_FPEMU_SIGFPE: ::c_int = 2; - -pub const PR_GET_FPEXC: ::c_int = 11; -pub const PR_SET_FPEXC: ::c_int = 12; -pub const PR_FP_EXC_SW_ENABLE: ::c_int = 0x80; -pub const PR_FP_EXC_DIV: ::c_int = 0x010000; -pub const PR_FP_EXC_OVF: ::c_int = 0x020000; -pub const PR_FP_EXC_UND: ::c_int = 0x040000; -pub const PR_FP_EXC_RES: ::c_int = 0x080000; -pub const PR_FP_EXC_INV: ::c_int = 0x100000; -pub const PR_FP_EXC_DISABLED: ::c_int = 0; -pub const PR_FP_EXC_NONRECOV: ::c_int = 1; -pub const PR_FP_EXC_ASYNC: ::c_int = 2; -pub const PR_FP_EXC_PRECISE: ::c_int = 3; - -pub const PR_GET_TIMING: ::c_int = 13; -pub const PR_SET_TIMING: ::c_int = 14; -pub const PR_TIMING_STATISTICAL: ::c_int = 0; -pub const PR_TIMING_TIMESTAMP: ::c_int = 1; - -pub const PR_SET_NAME: ::c_int = 15; -pub const PR_GET_NAME: ::c_int = 16; - -pub const PR_GET_ENDIAN: ::c_int = 19; -pub const PR_SET_ENDIAN: ::c_int = 20; -pub const PR_ENDIAN_BIG: ::c_int = 0; -pub const PR_ENDIAN_LITTLE: ::c_int = 1; -pub const PR_ENDIAN_PPC_LITTLE: ::c_int = 2; - -pub const PR_GET_SECCOMP: ::c_int = 21; -pub const PR_SET_SECCOMP: ::c_int = 22; - -pub const PR_CAPBSET_READ: ::c_int = 23; -pub const PR_CAPBSET_DROP: ::c_int = 24; - -pub const PR_GET_TSC: ::c_int = 25; -pub const PR_SET_TSC: ::c_int = 26; -pub const PR_TSC_ENABLE: ::c_int = 1; -pub const PR_TSC_SIGSEGV: ::c_int = 2; - -pub const PR_GET_SECUREBITS: ::c_int = 27; -pub const PR_SET_SECUREBITS: ::c_int = 28; - -pub const PR_SET_TIMERSLACK: ::c_int = 29; -pub const PR_GET_TIMERSLACK: ::c_int = 30; - -pub const PR_TASK_PERF_EVENTS_DISABLE: ::c_int = 31; -pub const PR_TASK_PERF_EVENTS_ENABLE: ::c_int = 32; - -pub const PR_MCE_KILL: ::c_int = 33; -pub const PR_MCE_KILL_CLEAR: ::c_int = 0; -pub const PR_MCE_KILL_SET: ::c_int = 1; - -pub const PR_MCE_KILL_LATE: ::c_int = 0; -pub const PR_MCE_KILL_EARLY: ::c_int = 1; -pub const PR_MCE_KILL_DEFAULT: ::c_int = 2; - -pub const PR_MCE_KILL_GET: ::c_int = 34; - -pub const PR_SET_MM: ::c_int = 35; -pub const PR_SET_MM_START_CODE: ::c_int = 1; -pub const PR_SET_MM_END_CODE: ::c_int = 2; -pub const PR_SET_MM_START_DATA: ::c_int = 3; -pub const PR_SET_MM_END_DATA: ::c_int = 4; -pub const PR_SET_MM_START_STACK: ::c_int = 5; -pub const PR_SET_MM_START_BRK: ::c_int = 6; -pub const PR_SET_MM_BRK: ::c_int = 7; -pub const PR_SET_MM_ARG_START: ::c_int = 8; -pub const PR_SET_MM_ARG_END: ::c_int = 9; -pub const PR_SET_MM_ENV_START: ::c_int = 10; -pub const PR_SET_MM_ENV_END: ::c_int = 11; -pub const PR_SET_MM_AUXV: ::c_int = 12; -pub const PR_SET_MM_EXE_FILE: ::c_int = 13; -pub const PR_SET_MM_MAP: ::c_int = 14; -pub const PR_SET_MM_MAP_SIZE: ::c_int = 15; - -pub const PR_SET_PTRACER: ::c_int = 0x59616d61; - -pub const PR_SET_CHILD_SUBREAPER: ::c_int = 36; -pub const PR_GET_CHILD_SUBREAPER: ::c_int = 37; - -pub const PR_SET_NO_NEW_PRIVS: ::c_int = 38; -pub const PR_GET_NO_NEW_PRIVS: ::c_int = 39; - -pub const PR_GET_TID_ADDRESS: ::c_int = 40; - -pub const PR_SET_THP_DISABLE: ::c_int = 41; -pub const PR_GET_THP_DISABLE: ::c_int = 42; - -pub const GRND_NONBLOCK: ::c_uint = 0x0001; -pub const GRND_RANDOM: ::c_uint = 0x0002; - -pub const ABDAY_1: ::nl_item = 0x300; -pub const ABDAY_2: ::nl_item = 0x301; -pub const ABDAY_3: ::nl_item = 0x302; -pub const ABDAY_4: ::nl_item = 0x303; -pub const ABDAY_5: ::nl_item = 0x304; -pub const ABDAY_6: ::nl_item = 0x305; -pub const ABDAY_7: ::nl_item = 0x306; - -pub const DAY_1: ::nl_item = 0x307; -pub const DAY_2: ::nl_item = 0x308; -pub const DAY_3: ::nl_item = 0x309; -pub const DAY_4: ::nl_item = 0x30A; -pub const DAY_5: ::nl_item = 0x30B; -pub const DAY_6: ::nl_item = 0x30C; -pub const DAY_7: ::nl_item = 0x30D; - -pub const ABMON_1: ::nl_item = 0x30E; -pub const ABMON_2: ::nl_item = 0x30F; -pub const ABMON_3: ::nl_item = 0x310; -pub const ABMON_4: ::nl_item = 0x311; -pub const ABMON_5: ::nl_item = 0x312; -pub const ABMON_6: ::nl_item = 0x313; -pub const ABMON_7: ::nl_item = 0x314; -pub const ABMON_8: ::nl_item = 0x315; -pub const ABMON_9: ::nl_item = 0x316; -pub const ABMON_10: ::nl_item = 0x317; -pub const ABMON_11: ::nl_item = 0x318; -pub const ABMON_12: ::nl_item = 0x319; - -pub const MON_1: ::nl_item = 0x31A; -pub const MON_2: ::nl_item = 0x31B; -pub const MON_3: ::nl_item = 0x31C; -pub const MON_4: ::nl_item = 0x31D; -pub const MON_5: ::nl_item = 0x31E; -pub const MON_6: ::nl_item = 0x31F; -pub const MON_7: ::nl_item = 0x320; -pub const MON_8: ::nl_item = 0x321; -pub const MON_9: ::nl_item = 0x322; -pub const MON_10: ::nl_item = 0x323; -pub const MON_11: ::nl_item = 0x324; -pub const MON_12: ::nl_item = 0x325; - -pub const AM_STR: ::nl_item = 0x326; -pub const PM_STR: ::nl_item = 0x327; - -pub const D_T_FMT: ::nl_item = 0x328; -pub const D_FMT: ::nl_item = 0x329; -pub const T_FMT: ::nl_item = 0x32A; -pub const T_FMT_AMPM: ::nl_item = 0x32B; - -pub const ERA: ::nl_item = 0x32C; -pub const ERA_D_FMT: ::nl_item = 0x32E; -pub const ALT_DIGITS: ::nl_item = 0x32F; -pub const ERA_D_T_FMT: ::nl_item = 0x330; -pub const ERA_T_FMT: ::nl_item = 0x331; - -pub const CODESET: ::nl_item = 10; - -pub const CRNCYSTR: ::nl_item = 0x215; - -pub const RADIXCHAR: ::nl_item = 0x100; -pub const THOUSEP: ::nl_item = 0x101; - -pub const NOEXPR: ::nl_item = 0x501; -pub const YESSTR: ::nl_item = 0x502; -pub const NOSTR: ::nl_item = 0x503; - -pub const FILENAME_MAX: ::c_uint = 4095; - -f! { - pub fn FD_CLR(fd: ::c_int, set: *mut fd_set) -> () { - let fd = fd as usize; - let size = ::mem::size_of_val(&(*set).fds_bits[0]) * 8; - (*set).fds_bits[fd / size] &= !(1 << (fd % size)); - return - } - - pub fn FD_ISSET(fd: ::c_int, set: *mut fd_set) -> bool { - let fd = fd as usize; - let size = ::mem::size_of_val(&(*set).fds_bits[0]) * 8; - return ((*set).fds_bits[fd / size] & (1 << (fd % size))) != 0 - } - - pub fn FD_SET(fd: ::c_int, set: *mut fd_set) -> () { - let fd = fd as usize; - let size = ::mem::size_of_val(&(*set).fds_bits[0]) * 8; - (*set).fds_bits[fd / size] |= 1 << (fd % size); - return - } - - pub fn FD_ZERO(set: *mut fd_set) -> () { - for slot in (*set).fds_bits.iter_mut() { - *slot = 0; - } - } - - pub fn WIFSTOPPED(status: ::c_int) -> bool { - (status & 0xff) == 0x7f - } - - pub fn WSTOPSIG(status: ::c_int) -> ::c_int { - (status >> 8) & 0xff - } - - pub fn WIFCONTINUED(status: ::c_int) -> bool { - status == 0xffff - } - - pub fn WIFSIGNALED(status: ::c_int) -> bool { - ((status & 0x7f) + 1) as i8 >= 2 - } - - pub fn WTERMSIG(status: ::c_int) -> ::c_int { - status & 0x7f - } - - pub fn WIFEXITED(status: ::c_int) -> bool { - (status & 0x7f) == 0 - } - - pub fn WEXITSTATUS(status: ::c_int) -> ::c_int { - (status >> 8) & 0xff - } - - pub fn WCOREDUMP(status: ::c_int) -> bool { - (status & 0x80) != 0 - } - - pub fn CPU_ZERO(cpuset: &mut cpu_set_t) -> () { - for slot in cpuset.bits.iter_mut() { - *slot = 0; - } - } - - pub fn CPU_SET(cpu: usize, cpuset: &mut cpu_set_t) -> () { - let size_in_bits - = 8 * ::mem::size_of_val(&cpuset.bits[0]); // 32, 64 etc - let (idx, offset) = (cpu / size_in_bits, cpu % size_in_bits); - cpuset.bits[idx] |= 1 << offset; - () - } - - pub fn CPU_CLR(cpu: usize, cpuset: &mut cpu_set_t) -> () { - let size_in_bits - = 8 * ::mem::size_of_val(&cpuset.bits[0]); // 32, 64 etc - let (idx, offset) = (cpu / size_in_bits, cpu % size_in_bits); - cpuset.bits[idx] &= !(1 << offset); - () - } - - pub fn CPU_ISSET(cpu: usize, cpuset: &cpu_set_t) -> bool { - let size_in_bits = 8 * ::mem::size_of_val(&cpuset.bits[0]); - let (idx, offset) = (cpu / size_in_bits, cpu % size_in_bits); - 0 != (cpuset.bits[idx] & (1 << offset)) - } - - pub fn CPU_EQUAL(set1: &cpu_set_t, set2: &cpu_set_t) -> bool { - set1.bits == set2.bits - } - - pub fn QCMD(cmd: ::c_int, type_: ::c_int) -> ::c_int { - (cmd << 8) | (type_ & 0x00ff) - } -} - -extern "C" { - #[cfg_attr(target_os = "linux", link_name = "__xpg_strerror_r")] - pub fn strerror_r( - errnum: ::c_int, - buf: *mut c_char, - buflen: ::size_t, - ) -> ::c_int; - - pub fn sem_destroy(sem: *mut sem_t) -> ::c_int; - pub fn sem_init( - sem: *mut sem_t, - pshared: ::c_int, - value: ::c_uint, - ) -> ::c_int; - - pub fn abs(i: ::c_int) -> ::c_int; - pub fn atof(s: *const ::c_char) -> ::c_double; - pub fn labs(i: ::c_long) -> ::c_long; - pub fn rand() -> ::c_int; - pub fn srand(seed: ::c_uint); - - pub fn fdatasync(fd: ::c_int) -> ::c_int; - pub fn gettimeofday(tp: *mut ::timeval, tz: *mut ::timezone) -> ::c_int; - pub fn mincore( - addr: *mut ::c_void, - len: ::size_t, - vec: *mut ::c_uchar, - ) -> ::c_int; - pub fn clock_getres(clk_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; - pub fn clock_gettime(clk_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; - pub fn clock_nanosleep( - clk_id: ::clockid_t, - flags: ::c_int, - rqtp: *const ::timespec, - rmtp: *mut ::timespec, - ) -> ::c_int; - pub fn clock_settime( - clk_id: ::clockid_t, - tp: *const ::timespec, - ) -> ::c_int; - pub fn prctl(option: ::c_int, ...) -> ::c_int; - pub fn pthread_getattr_np( - native: ::pthread_t, - attr: *mut ::pthread_attr_t, - ) -> ::c_int; - pub fn pthread_attr_getguardsize( - attr: *const ::pthread_attr_t, - guardsize: *mut ::size_t, - ) -> ::c_int; - pub fn pthread_attr_getstack( - attr: *const ::pthread_attr_t, - stackaddr: *mut *mut ::c_void, - stacksize: *mut ::size_t, - ) -> ::c_int; - pub fn memalign(align: ::size_t, size: ::size_t) -> *mut ::c_void; - pub fn setgroups(ngroups: ::size_t, ptr: *const ::gid_t) -> ::c_int; - pub fn initgroups(user: *const ::c_char, group: ::gid_t) -> ::c_int; - pub fn sched_setscheduler( - pid: ::pid_t, - policy: ::c_int, - param: *const ::sched_param, - ) -> ::c_int; - pub fn sched_getscheduler(pid: ::pid_t) -> ::c_int; - pub fn sched_get_priority_max(policy: ::c_int) -> ::c_int; - pub fn sched_get_priority_min(policy: ::c_int) -> ::c_int; - pub fn epoll_create(size: ::c_int) -> ::c_int; - pub fn epoll_create1(flags: ::c_int) -> ::c_int; - pub fn epoll_ctl( - epfd: ::c_int, - op: ::c_int, - fd: ::c_int, - event: *mut ::epoll_event, - ) -> ::c_int; - pub fn epoll_wait( - epfd: ::c_int, - events: *mut ::epoll_event, - maxevents: ::c_int, - timeout: ::c_int, - ) -> ::c_int; - pub fn pipe2(fds: *mut ::c_int, flags: ::c_int) -> ::c_int; - pub fn mount( - src: *const ::c_char, - target: *const ::c_char, - fstype: *const ::c_char, - flags: ::c_ulong, - data: *const ::c_void, - ) -> ::c_int; - pub fn umount(target: *const ::c_char) -> ::c_int; - pub fn umount2(target: *const ::c_char, flags: ::c_int) -> ::c_int; - pub fn clone( - cb: extern "C" fn(*mut ::c_void) -> ::c_int, - child_stack: *mut ::c_void, - flags: ::c_int, - arg: *mut ::c_void, - ... - ) -> ::c_int; - pub fn statfs(path: *const ::c_char, buf: *mut statfs) -> ::c_int; - pub fn fstatfs(fd: ::c_int, buf: *mut statfs) -> ::c_int; - pub fn memrchr( - cx: *const ::c_void, - c: ::c_int, - n: ::size_t, - ) -> *mut ::c_void; - pub fn syscall(num: ::c_long, ...) -> ::c_long; - pub fn sendfile( - out_fd: ::c_int, - in_fd: ::c_int, - offset: *mut off_t, - count: ::size_t, - ) -> ::ssize_t; - pub fn splice( - fd_in: ::c_int, - off_in: *mut ::loff_t, - fd_out: ::c_int, - off_out: *mut ::loff_t, - len: ::size_t, - flags: ::c_uint, - ) -> ::ssize_t; - pub fn tee( - fd_in: ::c_int, - fd_out: ::c_int, - len: ::size_t, - flags: ::c_uint, - ) -> ::ssize_t; - pub fn vmsplice( - fd: ::c_int, - iov: *const ::iovec, - nr_segs: ::size_t, - flags: ::c_uint, - ) -> ::ssize_t; - - pub fn posix_fadvise( - fd: ::c_int, - offset: ::off_t, - len: ::off_t, - advise: ::c_int, - ) -> ::c_int; - pub fn getrlimit(resource: ::c_int, rlim: *mut ::rlimit) -> ::c_int; - pub fn setrlimit(resource: ::c_int, rlim: *const ::rlimit) -> ::c_int; - pub fn futimens(fd: ::c_int, times: *const ::timespec) -> ::c_int; - pub fn utimensat( - dirfd: ::c_int, - path: *const ::c_char, - times: *const ::timespec, - flag: ::c_int, - ) -> ::c_int; - pub fn duplocale(base: ::locale_t) -> ::locale_t; - pub fn freelocale(loc: ::locale_t); - pub fn newlocale( - mask: ::c_int, - locale: *const ::c_char, - base: ::locale_t, - ) -> ::locale_t; - pub fn uselocale(loc: ::locale_t) -> ::locale_t; - pub fn creat64(path: *const c_char, mode: mode_t) -> ::c_int; - pub fn fstat64(fildes: ::c_int, buf: *mut stat64) -> ::c_int; - pub fn fstatat64( - fildes: ::c_int, - path: *const ::c_char, - buf: *mut stat64, - flag: ::c_int, - ) -> ::c_int; - pub fn ftruncate64(fd: ::c_int, length: off64_t) -> ::c_int; - pub fn getrlimit64(resource: ::c_int, rlim: *mut rlimit64) -> ::c_int; - pub fn lseek64(fd: ::c_int, offset: off64_t, whence: ::c_int) -> off64_t; - pub fn lstat64(path: *const c_char, buf: *mut stat64) -> ::c_int; - pub fn mmap64( - addr: *mut ::c_void, - len: ::size_t, - prot: ::c_int, - flags: ::c_int, - fd: ::c_int, - offset: off64_t, - ) -> *mut ::c_void; - pub fn open64(path: *const c_char, oflag: ::c_int, ...) -> ::c_int; - pub fn openat64( - fd: ::c_int, - path: *const c_char, - oflag: ::c_int, - ... - ) -> ::c_int; - pub fn pread64( - fd: ::c_int, - buf: *mut ::c_void, - count: ::size_t, - offset: off64_t, - ) -> ::ssize_t; - pub fn pwrite64( - fd: ::c_int, - buf: *const ::c_void, - count: ::size_t, - offset: off64_t, - ) -> ::ssize_t; - pub fn readdir64(dirp: *mut ::DIR) -> *mut ::dirent64; - pub fn readdir64_r( - dirp: *mut ::DIR, - entry: *mut ::dirent64, - result: *mut *mut ::dirent64, - ) -> ::c_int; - pub fn setrlimit64(resource: ::c_int, rlim: *const rlimit64) -> ::c_int; - pub fn stat64(path: *const c_char, buf: *mut stat64) -> ::c_int; - pub fn truncate64(path: *const c_char, length: off64_t) -> ::c_int; - pub fn eventfd(init: ::c_uint, flags: ::c_int) -> ::c_int; - - pub fn mknodat( - dirfd: ::c_int, - pathname: *const ::c_char, - mode: ::mode_t, - dev: dev_t, - ) -> ::c_int; - pub fn ppoll( - fds: *mut ::pollfd, - nfds: nfds_t, - timeout: *const ::timespec, - sigmask: *const sigset_t, - ) -> ::c_int; - pub fn pthread_condattr_getclock( - attr: *const pthread_condattr_t, - clock_id: *mut clockid_t, - ) -> ::c_int; - pub fn pthread_condattr_setclock( - attr: *mut pthread_condattr_t, - clock_id: ::clockid_t, - ) -> ::c_int; - pub fn pthread_condattr_setpshared( - attr: *mut pthread_condattr_t, - pshared: ::c_int, - ) -> ::c_int; - pub fn pthread_condattr_getpshared( - attr: *const pthread_condattr_t, - pshared: *mut ::c_int, - ) -> ::c_int; - pub fn sched_getaffinity( - pid: ::pid_t, - cpusetsize: ::size_t, - cpuset: *mut cpu_set_t, - ) -> ::c_int; - pub fn sched_setaffinity( - pid: ::pid_t, - cpusetsize: ::size_t, - cpuset: *const cpu_set_t, - ) -> ::c_int; - pub fn unshare(flags: ::c_int) -> ::c_int; - pub fn sem_timedwait( - sem: *mut sem_t, - abstime: *const ::timespec, - ) -> ::c_int; - pub fn sem_getvalue(sem: *mut sem_t, sval: *mut ::c_int) -> ::c_int; - pub fn accept4( - fd: ::c_int, - addr: *mut ::sockaddr, - len: *mut ::socklen_t, - flg: ::c_int, - ) -> ::c_int; - pub fn pthread_mutex_timedlock( - lock: *mut pthread_mutex_t, - abstime: *const ::timespec, - ) -> ::c_int; - pub fn pthread_mutexattr_setpshared( - attr: *mut pthread_mutexattr_t, - pshared: ::c_int, - ) -> ::c_int; - pub fn pthread_mutexattr_getpshared( - attr: *const pthread_mutexattr_t, - pshared: *mut ::c_int, - ) -> ::c_int; - pub fn pthread_rwlockattr_getkind_np( - attr: *const pthread_rwlockattr_t, - val: *mut ::c_int, - ) -> ::c_int; - pub fn pthread_rwlockattr_setkind_np( - attr: *mut pthread_rwlockattr_t, - val: ::c_int, - ) -> ::c_int; - pub fn pthread_rwlockattr_getpshared( - attr: *const pthread_rwlockattr_t, - val: *mut ::c_int, - ) -> ::c_int; - pub fn pthread_rwlockattr_setpshared( - attr: *mut pthread_rwlockattr_t, - val: ::c_int, - ) -> ::c_int; - pub fn ptsname_r( - fd: ::c_int, - buf: *mut ::c_char, - buflen: ::size_t, - ) -> ::c_int; - pub fn clearenv() -> ::c_int; - pub fn waitid( - idtype: idtype_t, - id: id_t, - infop: *mut ::siginfo_t, - options: ::c_int, - ) -> ::c_int; - - pub fn lutimes(file: *const ::c_char, times: *const ::timeval) -> ::c_int; - - pub fn setpwent(); - pub fn endpwent(); - pub fn getpwent() -> *mut passwd; - pub fn setspent(); - pub fn endspent(); - pub fn getspent() -> *mut spwd; - pub fn getspnam(__name: *const ::c_char) -> *mut spwd; - - pub fn shm_open( - name: *const c_char, - oflag: ::c_int, - mode: mode_t, - ) -> ::c_int; - - // System V IPC - pub fn shmget(key: ::key_t, size: ::size_t, shmflg: ::c_int) -> ::c_int; - pub fn shmat( - shmid: ::c_int, - shmaddr: *const ::c_void, - shmflg: ::c_int, - ) -> *mut ::c_void; - pub fn shmdt(shmaddr: *const ::c_void) -> ::c_int; - pub fn shmctl( - shmid: ::c_int, - cmd: ::c_int, - buf: *mut ::shmid_ds, - ) -> ::c_int; - pub fn ftok(pathname: *const ::c_char, proj_id: ::c_int) -> ::key_t; - pub fn msgctl(msqid: ::c_int, cmd: ::c_int, buf: *mut msqid_ds) - -> ::c_int; - pub fn msgget(key: ::key_t, msgflg: ::c_int) -> ::c_int; - pub fn msgrcv( - msqid: ::c_int, - msgp: *mut ::c_void, - msgsz: ::size_t, - msgtyp: ::c_long, - msgflg: ::c_int, - ) -> ::ssize_t; - pub fn msgsnd( - msqid: ::c_int, - msgp: *const ::c_void, - msgsz: ::size_t, - msgflg: ::c_int, - ) -> ::c_int; - - pub fn mprotect( - addr: *mut ::c_void, - len: ::size_t, - prot: ::c_int, - ) -> ::c_int; - pub fn __errno_location() -> *mut ::c_int; - - pub fn fopen64( - filename: *const c_char, - mode: *const c_char, - ) -> *mut ::FILE; - pub fn freopen64( - filename: *const c_char, - mode: *const c_char, - file: *mut ::FILE, - ) -> *mut ::FILE; - pub fn tmpfile64() -> *mut ::FILE; - pub fn fgetpos64(stream: *mut ::FILE, ptr: *mut fpos64_t) -> ::c_int; - pub fn fsetpos64(stream: *mut ::FILE, ptr: *const fpos64_t) -> ::c_int; - pub fn fseeko64( - stream: *mut ::FILE, - offset: ::off64_t, - whence: ::c_int, - ) -> ::c_int; - pub fn ftello64(stream: *mut ::FILE) -> ::off64_t; - pub fn readahead( - fd: ::c_int, - offset: ::off64_t, - count: ::size_t, - ) -> ::ssize_t; - pub fn getxattr( - path: *const c_char, - name: *const c_char, - value: *mut ::c_void, - size: ::size_t, - ) -> ::ssize_t; - pub fn lgetxattr( - path: *const c_char, - name: *const c_char, - value: *mut ::c_void, - size: ::size_t, - ) -> ::ssize_t; - pub fn fgetxattr( - filedes: ::c_int, - name: *const c_char, - value: *mut ::c_void, - size: ::size_t, - ) -> ::ssize_t; - pub fn setxattr( - path: *const c_char, - name: *const c_char, - value: *const ::c_void, - size: ::size_t, - flags: ::c_int, - ) -> ::c_int; - pub fn lsetxattr( - path: *const c_char, - name: *const c_char, - value: *const ::c_void, - size: ::size_t, - flags: ::c_int, - ) -> ::c_int; - pub fn fsetxattr( - filedes: ::c_int, - name: *const c_char, - value: *const ::c_void, - size: ::size_t, - flags: ::c_int, - ) -> ::c_int; - pub fn listxattr( - path: *const c_char, - list: *mut c_char, - size: ::size_t, - ) -> ::ssize_t; - pub fn llistxattr( - path: *const c_char, - list: *mut c_char, - size: ::size_t, - ) -> ::ssize_t; - pub fn flistxattr( - filedes: ::c_int, - list: *mut c_char, - size: ::size_t, - ) -> ::ssize_t; - pub fn removexattr(path: *const c_char, name: *const c_char) -> ::c_int; - pub fn lremovexattr(path: *const c_char, name: *const c_char) -> ::c_int; - pub fn fremovexattr(filedes: ::c_int, name: *const c_char) -> ::c_int; - pub fn signalfd( - fd: ::c_int, - mask: *const ::sigset_t, - flags: ::c_int, - ) -> ::c_int; - pub fn quotactl( - cmd: ::c_int, - special: *const ::c_char, - id: ::c_int, - data: *mut ::c_char, - ) -> ::c_int; - pub fn mq_open(name: *const ::c_char, oflag: ::c_int, ...) -> ::mqd_t; - pub fn mq_close(mqd: ::mqd_t) -> ::c_int; - pub fn mq_unlink(name: *const ::c_char) -> ::c_int; - pub fn mq_receive( - mqd: ::mqd_t, - msg_ptr: *mut ::c_char, - msg_len: ::size_t, - msg_prio: *mut ::c_uint, - ) -> ::ssize_t; - pub fn mq_send( - mqd: ::mqd_t, - msg_ptr: *const ::c_char, - msg_len: ::size_t, - msg_prio: ::c_uint, - ) -> ::c_int; - pub fn mq_getattr(mqd: ::mqd_t, attr: *mut ::mq_attr) -> ::c_int; - pub fn mq_setattr( - mqd: ::mqd_t, - newattr: *const ::mq_attr, - oldattr: *mut ::mq_attr, - ) -> ::c_int; - pub fn epoll_pwait( - epfd: ::c_int, - events: *mut ::epoll_event, - maxevents: ::c_int, - timeout: ::c_int, - sigmask: *const ::sigset_t, - ) -> ::c_int; - pub fn sethostname(name: *const ::c_char, len: ::size_t) -> ::c_int; - pub fn sigtimedwait( - set: *const sigset_t, - info: *mut siginfo_t, - timeout: *const ::timespec, - ) -> ::c_int; - pub fn sigwaitinfo(set: *const sigset_t, info: *mut siginfo_t) -> ::c_int; - pub fn nl_langinfo_l(item: ::nl_item, locale: ::locale_t) - -> *mut ::c_char; - pub fn prlimit( - pid: ::pid_t, - resource: ::c_int, - new_limit: *const ::rlimit, - old_limit: *mut ::rlimit, - ) -> ::c_int; - pub fn prlimit64( - pid: ::pid_t, - resource: ::c_int, - new_limit: *const ::rlimit64, - old_limit: *mut ::rlimit64, - ) -> ::c_int; - pub fn reboot(how_to: ::c_int) -> ::c_int; - pub fn setfsgid(gid: ::gid_t) -> ::c_int; - pub fn setfsuid(uid: ::uid_t) -> ::c_int; - pub fn setresgid(rgid: ::gid_t, egid: ::gid_t, sgid: ::gid_t) -> ::c_int; - pub fn setresuid(ruid: ::uid_t, euid: ::uid_t, suid: ::uid_t) -> ::c_int; - - // Not available now on Android - pub fn mkfifoat( - dirfd: ::c_int, - pathname: *const ::c_char, - mode: ::mode_t, - ) -> ::c_int; - pub fn if_nameindex() -> *mut if_nameindex; - pub fn if_freenameindex(ptr: *mut if_nameindex); - pub fn sync_file_range( - fd: ::c_int, - offset: ::off64_t, - nbytes: ::off64_t, - flags: ::c_uint, - ) -> ::c_int; - pub fn getifaddrs(ifap: *mut *mut ::ifaddrs) -> ::c_int; - pub fn freeifaddrs(ifa: *mut ::ifaddrs); - - pub fn mremap( - addr: *mut ::c_void, - len: ::size_t, - new_len: ::size_t, - flags: ::c_int, - ... - ) -> *mut ::c_void; - - pub fn glob( - pattern: *const c_char, - flags: ::c_int, - errfunc: ::Option< - extern "C" fn(epath: *const c_char, errno: ::c_int) -> ::c_int, - >, - pglob: *mut ::glob_t, - ) -> ::c_int; - pub fn globfree(pglob: *mut ::glob_t); - - pub fn shm_unlink(name: *const ::c_char) -> ::c_int; - - pub fn seekdir(dirp: *mut ::DIR, loc: ::c_long); - - pub fn dirfd(dirp: *mut ::DIR) -> ::c_int; - - pub fn telldir(dirp: *mut ::DIR) -> ::c_long; - pub fn madvise( - addr: *mut ::c_void, - len: ::size_t, - advice: ::c_int, - ) -> ::c_int; - - pub fn msync( - addr: *mut ::c_void, - len: ::size_t, - flags: ::c_int, - ) -> ::c_int; - - pub fn recvfrom( - socket: ::c_int, - buf: *mut ::c_void, - len: ::size_t, - flags: ::c_int, - addr: *mut ::sockaddr, - addrlen: *mut ::socklen_t, - ) -> ::ssize_t; - pub fn nl_langinfo(item: ::nl_item) -> *mut ::c_char; - - pub fn bind( - socket: ::c_int, - address: *const ::sockaddr, - address_len: ::socklen_t, - ) -> ::c_int; - - pub fn writev( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - ) -> ::ssize_t; - pub fn readv( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - ) -> ::ssize_t; - - pub fn sendmsg( - fd: ::c_int, - msg: *const ::msghdr, - flags: ::c_int, - ) -> ::ssize_t; - pub fn recvmsg( - fd: ::c_int, - msg: *mut ::msghdr, - flags: ::c_int, - ) -> ::ssize_t; - #[cfg_attr(target_os = "solaris", link_name = "__posix_getgrgid_r")] - pub fn getgrgid_r( - gid: ::gid_t, - grp: *mut ::group, - buf: *mut ::c_char, - buflen: ::size_t, - result: *mut *mut ::group, - ) -> ::c_int; - #[cfg_attr( - all(target_os = "macos", target_arch = "x86"), - link_name = "sigaltstack$UNIX2003" - )] - #[cfg_attr(target_os = "netbsd", link_name = "__sigaltstack14")] - pub fn sigaltstack(ss: *const stack_t, oss: *mut stack_t) -> ::c_int; - pub fn sem_close(sem: *mut sem_t) -> ::c_int; - pub fn getdtablesize() -> ::c_int; - #[cfg_attr(target_os = "solaris", link_name = "__posix_getgrnam_r")] - pub fn getgrnam_r( - name: *const ::c_char, - grp: *mut ::group, - buf: *mut ::c_char, - buflen: ::size_t, - result: *mut *mut ::group, - ) -> ::c_int; - #[cfg_attr( - all(target_os = "macos", target_arch = "x86"), - link_name = "pthread_sigmask$UNIX2003" - )] - pub fn pthread_sigmask( - how: ::c_int, - set: *const sigset_t, - oldset: *mut sigset_t, - ) -> ::c_int; - pub fn sem_open(name: *const ::c_char, oflag: ::c_int, ...) -> *mut sem_t; - pub fn getgrnam(name: *const ::c_char) -> *mut ::group; - pub fn pthread_kill(thread: ::pthread_t, sig: ::c_int) -> ::c_int; - pub fn sem_unlink(name: *const ::c_char) -> ::c_int; - pub fn daemon(nochdir: ::c_int, noclose: ::c_int) -> ::c_int; - #[cfg_attr(target_os = "netbsd", link_name = "__getpwnam_r50")] - #[cfg_attr(target_os = "solaris", link_name = "__posix_getpwnam_r")] - pub fn getpwnam_r( - name: *const ::c_char, - pwd: *mut passwd, - buf: *mut ::c_char, - buflen: ::size_t, - result: *mut *mut passwd, - ) -> ::c_int; - #[cfg_attr(target_os = "netbsd", link_name = "__getpwuid_r50")] - #[cfg_attr(target_os = "solaris", link_name = "__posix_getpwuid_r")] - pub fn getpwuid_r( - uid: ::uid_t, - pwd: *mut passwd, - buf: *mut ::c_char, - buflen: ::size_t, - result: *mut *mut passwd, - ) -> ::c_int; - #[cfg_attr( - all(target_os = "macos", target_arch = "x86"), - link_name = "sigwait$UNIX2003" - )] - #[cfg_attr(target_os = "solaris", link_name = "__posix_sigwait")] - pub fn sigwait(set: *const sigset_t, sig: *mut ::c_int) -> ::c_int; - pub fn pthread_atfork( - prepare: ::Option, - parent: ::Option, - child: ::Option, - ) -> ::c_int; - pub fn pthread_create( - native: *mut ::pthread_t, - attr: *const ::pthread_attr_t, - f: extern "C" fn(*mut ::c_void) -> *mut ::c_void, - value: *mut ::c_void, - ) -> ::c_int; - pub fn getgrgid(gid: ::gid_t) -> *mut ::group; - #[cfg_attr( - all(target_os = "macos", target_arch = "x86"), - link_name = "popen$UNIX2003" - )] - pub fn popen(command: *const c_char, mode: *const c_char) -> *mut ::FILE; - pub fn uname(buf: *mut ::utsname) -> ::c_int; -} - -cfg_if! { - if #[cfg(any(target_arch = "mips", target_arch = "mips64"))] { - mod mips; - pub use self::mips::*; - } else if #[cfg(target_arch = "x86_64")] { - mod x86_64; - pub use self::x86_64::*; - } else if #[cfg(target_arch = "arm")] { - mod arm; - pub use self::arm::*; - } else { - pub use unsupported_target; - } -} - -cfg_if! { - if #[cfg(libc_align)] { - #[macro_use] - mod align; - } else { - #[macro_use] - mod no_align; - } -} - -expand_align!(); diff --git a/crux-mir/lib/libc/src/unix/uclibc/x86_64/align.rs b/crux-mir/lib/libc/src/unix/uclibc/x86_64/align.rs deleted file mode 100644 index e2d829b50..000000000 --- a/crux-mir/lib/libc/src/unix/uclibc/x86_64/align.rs +++ /dev/null @@ -1,77 +0,0 @@ -macro_rules! expand_align { - () => { - s! { - #[cfg_attr(target_pointer_width = "32", - repr(align(4)))] - #[cfg_attr(target_pointer_width = "64", - repr(align(8)))] - pub struct sem_t { // FIXME - #[cfg(target_pointer_width = "32")] - __size: [::c_char; 16], - #[cfg(target_pointer_width = "64")] - __size: [::c_char; 32], - } - - #[cfg_attr(any(target_pointer_width = "32", - target_arch = "x86_64", - target_arch = "powerpc64", - target_arch = "mips64", - target_arch = "s390x", - target_arch = "sparc64"), - repr(align(4)))] - #[cfg_attr(not(any(target_pointer_width = "32", - target_arch = "x86_64", - target_arch = "powerpc64", - target_arch = "mips64", - target_arch = "s390x", - target_arch = "sparc64")), - repr(align(8)))] - pub struct pthread_mutexattr_t { // FIXME - size: [u8; ::__SIZEOF_PTHREAD_MUTEXATTR_T], - } - - #[repr(align(4))] - pub struct pthread_condattr_t { // FIXME - size: [u8; ::__SIZEOF_PTHREAD_CONDATTR_T], - } - } - - s_no_extra_traits! { - #[cfg_attr(all(target_pointer_width = "32", - any(target_arch = "mips", - target_arch = "arm", - target_arch = "powerpc")), - repr(align(4)))] - #[cfg_attr(all(any(target_pointer_width = "64", - not(any(target_arch = "mips", - target_arch = "arm", - target_arch = "powerpc")))), - repr(align(8)))] - #[allow(missing_debug_implementations)] - pub struct pthread_mutex_t { // FIXME - size: [u8; ::__SIZEOF_PTHREAD_MUTEX_T], - } - - #[repr(align(8))] - #[allow(missing_debug_implementations)] - pub struct pthread_cond_t { // FIXME - size: [u8; ::__SIZEOF_PTHREAD_COND_T], - } - - #[cfg_attr(all(target_pointer_width = "32", - any(target_arch = "mips", - target_arch = "arm", - target_arch = "powerpc")), - repr(align(4)))] - #[cfg_attr(any(target_pointer_width = "64", - not(any(target_arch = "mips", - target_arch = "arm", - target_arch = "powerpc"))), - repr(align(8)))] - #[allow(missing_debug_implementations)] - pub struct pthread_rwlock_t { // FIXME - size: [u8; ::__SIZEOF_PTHREAD_RWLOCK_T], - } - } - }; -} diff --git a/crux-mir/lib/libc/src/unix/uclibc/x86_64/no_align.rs b/crux-mir/lib/libc/src/unix/uclibc/x86_64/no_align.rs deleted file mode 100644 index ffa4e523f..000000000 --- a/crux-mir/lib/libc/src/unix/uclibc/x86_64/no_align.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! expand_align { - () => { - s! { - pub struct sem_t { // FIXME - #[cfg(target_pointer_width = "32")] - __size: [::c_char; 16], - #[cfg(target_pointer_width = "64")] - __size: [::c_char; 32], - __align: [::c_long; 0], - } - - pub struct pthread_mutex_t { // FIXME - #[cfg(any(target_arch = "mips", - target_arch = "arm", - target_arch = "powerpc"))] - __align: [::c_long; 0], - #[cfg(not(any(target_arch = "mips", - target_arch = "arm", - target_arch = "powerpc")))] - __align: [::c_longlong; 0], - size: [u8; ::__SIZEOF_PTHREAD_MUTEX_T], - } - - pub struct pthread_mutexattr_t { // FIXME - #[cfg(any(target_arch = "x86_64", target_arch = "powerpc64", - target_arch = "mips64", target_arch = "s390x", - target_arch = "sparc64"))] - __align: [::c_int; 0], - #[cfg(not(any(target_arch = "x86_64", target_arch = "powerpc64", - target_arch = "mips64", target_arch = "s390x", - target_arch = "sparc64")))] - __align: [::c_long; 0], - size: [u8; ::__SIZEOF_PTHREAD_MUTEXATTR_T], - } - - pub struct pthread_cond_t { // FIXME - __align: [::c_longlong; 0], - size: [u8; ::__SIZEOF_PTHREAD_COND_T], - } - - pub struct pthread_condattr_t { // FIXME - __align: [::c_int; 0], - size: [u8; ::__SIZEOF_PTHREAD_CONDATTR_T], - } - - pub struct pthread_rwlock_t { // FIXME - #[cfg(any(target_arch = "mips", - target_arch = "arm", - target_arch = "powerpc"))] - __align: [::c_long; 0], - #[cfg(not(any(target_arch = "mips", - target_arch = "arm", - target_arch = "powerpc")))] - __align: [::c_longlong; 0], - size: [u8; ::__SIZEOF_PTHREAD_RWLOCK_T], - } - } - } -} diff --git a/crux-mir/lib/libc/src/vxworks/mod.rs b/crux-mir/lib/libc/src/vxworks/mod.rs old mode 100755 new mode 100644 index 2ca38d5e2..2772d68d2 --- a/crux-mir/lib/libc/src/vxworks/mod.rs +++ b/crux-mir/lib/libc/src/vxworks/mod.rs @@ -112,6 +112,28 @@ impl ::Clone for _Vx_semaphore { } } +impl siginfo_t { + pub unsafe fn si_addr(&self) -> *mut ::c_void { + self.si_addr + } + + pub unsafe fn si_value(&self) -> ::sigval { + self.si_value + } + + pub unsafe fn si_pid(&self) -> ::pid_t { + self.si_pid + } + + pub unsafe fn si_uid(&self) -> ::uid_t { + self.si_uid + } + + pub unsafe fn si_status(&self) -> ::c_int { + self.si_status + } +} + s! { // b_pthread_condattr_t.h pub struct pthread_condattr_t { @@ -609,6 +631,7 @@ pub const EFAULT: ::c_int = 14; pub const ENOTEMPTY: ::c_int = 15; pub const EBUSY: ::c_int = 16; pub const EEXIST: ::c_int = 17; +pub const EXDEV: ::c_int = 18; pub const ENODEV: ::c_int = 19; pub const ENOTDIR: ::c_int = 20; pub const EISDIR: ::c_int = 21; @@ -616,7 +639,9 @@ pub const EINVAL: ::c_int = 22; pub const ENAMETOOLONG: ::c_int = 26; pub const EFBIG: ::c_int = 27; pub const ENOSPC: ::c_int = 28; +pub const ESPIPE: ::c_int = 29; pub const EROFS: ::c_int = 30; +pub const EMLINK: ::c_int = 31; pub const EPIPE: ::c_int = 32; pub const EDEADLK: ::c_int = 33; pub const ERANGE: ::c_int = 38; @@ -642,6 +667,10 @@ pub const ESHUTDOWN: ::c_int = 58; pub const ETOOMANYREFS: ::c_int = 59; pub const ETIMEDOUT: ::c_int = 60; pub const ECONNREFUSED: ::c_int = 61; +pub const ENETDOWN: ::c_int = 62; +pub const ETXTBSY: ::c_int = 63; +pub const ELOOP: ::c_int = 64; +pub const EHOSTUNREACH: ::c_int = 65; pub const EINPROGRESS: ::c_int = 68; pub const EALREADY: ::c_int = 69; pub const EWOULDBLOCK: ::c_int = 70; @@ -680,24 +709,16 @@ pub const S_nfsLib_NFSERR_NAMETOOLONG: ::c_int = ENAMETOOLONG; pub const S_nfsLib_NFSERR_NOTEMPTY: ::c_int = ENOTEMPTY; pub const S_nfsLib_NFSERR_DQUOT: ::c_int = EDQUOT; pub const S_nfsLib_NFSERR_STALE: ::c_int = ESTALE; -pub const S_nfsLib_NFSERR_WFLUSH: ::c_int = - M_nfsStat | nfsstat::NFSERR_WFLUSH as ::c_int; -pub const S_nfsLib_NFSERR_REMOTE: ::c_int = - M_nfsStat | nfsstat::NFSERR_REMOTE as ::c_int; -pub const S_nfsLib_NFSERR_BADHANDLE: ::c_int = - M_nfsStat | nfsstat::NFSERR_BADHANDLE as ::c_int; -pub const S_nfsLib_NFSERR_NOT_SYNC: ::c_int = - M_nfsStat | nfsstat::NFSERR_NOT_SYNC as ::c_int; -pub const S_nfsLib_NFSERR_BAD_COOKIE: ::c_int = - M_nfsStat | nfsstat::NFSERR_BAD_COOKIE as ::c_int; +pub const S_nfsLib_NFSERR_WFLUSH: ::c_int = M_nfsStat | nfsstat::NFSERR_WFLUSH as ::c_int; +pub const S_nfsLib_NFSERR_REMOTE: ::c_int = M_nfsStat | nfsstat::NFSERR_REMOTE as ::c_int; +pub const S_nfsLib_NFSERR_BADHANDLE: ::c_int = M_nfsStat | nfsstat::NFSERR_BADHANDLE as ::c_int; +pub const S_nfsLib_NFSERR_NOT_SYNC: ::c_int = M_nfsStat | nfsstat::NFSERR_NOT_SYNC as ::c_int; +pub const S_nfsLib_NFSERR_BAD_COOKIE: ::c_int = M_nfsStat | nfsstat::NFSERR_BAD_COOKIE as ::c_int; pub const S_nfsLib_NFSERR_NOTSUPP: ::c_int = EOPNOTSUPP; -pub const S_nfsLib_NFSERR_TOOSMALL: ::c_int = - M_nfsStat | nfsstat::NFSERR_TOOSMALL as ::c_int; +pub const S_nfsLib_NFSERR_TOOSMALL: ::c_int = M_nfsStat | nfsstat::NFSERR_TOOSMALL as ::c_int; pub const S_nfsLib_NFSERR_SERVERFAULT: ::c_int = EIO; -pub const S_nfsLib_NFSERR_BADTYPE: ::c_int = - M_nfsStat | nfsstat::NFSERR_BADTYPE as ::c_int; -pub const S_nfsLib_NFSERR_JUKEBOX: ::c_int = - M_nfsStat | nfsstat::NFSERR_JUKEBOX as ::c_int; +pub const S_nfsLib_NFSERR_BADTYPE: ::c_int = M_nfsStat | nfsstat::NFSERR_BADTYPE as ::c_int; +pub const S_nfsLib_NFSERR_JUKEBOX: ::c_int = M_nfsStat | nfsstat::NFSERR_JUKEBOX as ::c_int; // in.h pub const IPPROTO_IP: ::c_int = 0; @@ -781,8 +802,7 @@ pub const SOCK_PACKET: ::c_int = 10; pub const _SS_MAXSIZE: usize = 128; pub const _SS_ALIGNSIZE: usize = size_of::(); -pub const _SS_PAD1SIZE: usize = - _SS_ALIGNSIZE - size_of::<::c_uchar>() - size_of::<::sa_family_t>(); +pub const _SS_PAD1SIZE: usize = _SS_ALIGNSIZE - size_of::<::c_uchar>() - size_of::<::sa_family_t>(); pub const _SS_PAD2SIZE: usize = _SS_MAXSIZE - size_of::<::c_uchar>() - size_of::<::sa_family_t>() @@ -929,14 +949,13 @@ pub const _PARM_PATH_MAX: ::c_int = 1024; pub const WNOHANG: ::c_int = 0x01; pub const WUNTRACED: ::c_int = 0x02; -const PTHREAD_MUTEXATTR_INITIALIZER: pthread_mutexattr_t = - pthread_mutexattr_t { - mutexAttrStatus: PTHREAD_INITIALIZED_OBJ, - mutexAttrProtocol: PTHREAD_PRIO_NONE, - mutexAttrPrioceiling: 0, - mutexAttrType: PTHREAD_MUTEX_DEFAULT, - mutexAttrPshared: 1, - }; +const PTHREAD_MUTEXATTR_INITIALIZER: pthread_mutexattr_t = pthread_mutexattr_t { + mutexAttrStatus: PTHREAD_INITIALIZED_OBJ, + mutexAttrProtocol: PTHREAD_PRIO_NONE, + mutexAttrPrioceiling: 0, + mutexAttrType: PTHREAD_MUTEX_DEFAULT, + mutexAttrPshared: 1, +}; pub const PTHREAD_MUTEX_INITIALIZER: pthread_mutex_t = pthread_mutex_t { mutexSemId: null_mut(), mutexValid: PTHREAD_VALID_OBJ, @@ -962,13 +981,12 @@ pub const PTHREAD_COND_INITIALIZER: pthread_cond_t = pthread_cond_t { condSemName: [0; _PTHREAD_SHARED_SEM_NAME_MAX], }; -const PTHREAD_RWLOCKATTR_INITIALIZER: pthread_rwlockattr_t = - pthread_rwlockattr_t { - rwlockAttrStatus: PTHREAD_INITIALIZED_OBJ, - rwlockAttrPshared: 1, - rwlockAttrMaxReaders: 0, - rwlockAttrConformOpt: 1, - }; +const PTHREAD_RWLOCKATTR_INITIALIZER: pthread_rwlockattr_t = pthread_rwlockattr_t { + rwlockAttrStatus: PTHREAD_INITIALIZED_OBJ, + rwlockAttrPshared: 1, + rwlockAttrMaxReaders: 0, + rwlockAttrConformOpt: 1, +}; pub const PTHREAD_RWLOCK_INITIALIZER: pthread_rwlock_t = pthread_rwlock_t { rwlockSemId: null_mut(), rwlockReadersRefCount: 0, @@ -1018,7 +1036,7 @@ impl ::Clone for fpos_t { } f! { - pub fn CMSG_ALIGN(len: usize) -> usize { + pub {const} fn CMSG_ALIGN(len: usize) -> usize { len + ::mem::size_of::() - 1 & !(::mem::size_of::() - 1) } @@ -1049,7 +1067,7 @@ f! { .offset(CMSG_ALIGN(::mem::size_of::<::cmsghdr>()) as isize) } - pub fn CMSG_SPACE(length: ::c_uint) -> ::c_uint { + pub {const} fn CMSG_SPACE(length: ::c_uint) -> ::c_uint { (CMSG_ALIGN(length as usize) + CMSG_ALIGN(::mem::size_of::())) as ::c_uint } @@ -1075,44 +1093,24 @@ extern "C" { pub fn tolower(c: c_int) -> c_int; pub fn toupper(c: c_int) -> c_int; pub fn fopen(filename: *const c_char, mode: *const c_char) -> *mut FILE; - pub fn freopen( - filename: *const c_char, - mode: *const c_char, - file: *mut FILE, - ) -> *mut FILE; + pub fn freopen(filename: *const c_char, mode: *const c_char, file: *mut FILE) -> *mut FILE; pub fn fflush(file: *mut FILE) -> c_int; pub fn fclose(file: *mut FILE) -> c_int; pub fn remove(filename: *const c_char) -> c_int; pub fn rename(oldname: *const c_char, newname: *const c_char) -> c_int; pub fn tmpfile() -> *mut FILE; - pub fn setvbuf( - stream: *mut FILE, - buffer: *mut c_char, - mode: c_int, - size: size_t, - ) -> c_int; + pub fn setvbuf(stream: *mut FILE, buffer: *mut c_char, mode: c_int, size: size_t) -> c_int; pub fn setbuf(stream: *mut FILE, buf: *mut c_char); pub fn getchar() -> c_int; pub fn putchar(c: c_int) -> c_int; pub fn fgetc(stream: *mut FILE) -> c_int; - pub fn fgets(buf: *mut c_char, n: c_int, stream: *mut FILE) - -> *mut c_char; + pub fn fgets(buf: *mut c_char, n: c_int, stream: *mut FILE) -> *mut c_char; pub fn fputc(c: c_int, stream: *mut FILE) -> c_int; pub fn fputs(s: *const c_char, stream: *mut FILE) -> c_int; pub fn puts(s: *const c_char) -> c_int; pub fn ungetc(c: c_int, stream: *mut FILE) -> c_int; - pub fn fread( - ptr: *mut c_void, - size: size_t, - nobj: size_t, - stream: *mut FILE, - ) -> size_t; - pub fn fwrite( - ptr: *const c_void, - size: size_t, - nobj: size_t, - stream: *mut FILE, - ) -> size_t; + pub fn fread(ptr: *mut c_void, size: size_t, nobj: size_t, stream: *mut FILE) -> size_t; + pub fn fwrite(ptr: *const c_void, size: size_t, nobj: size_t, stream: *mut FILE) -> size_t; pub fn fseek(stream: *mut FILE, offset: c_long, whence: c_int) -> c_int; pub fn ftell(stream: *mut FILE) -> c_long; pub fn rewind(stream: *mut FILE); @@ -1123,16 +1121,9 @@ extern "C" { pub fn perror(s: *const c_char); pub fn atoi(s: *const c_char) -> c_int; pub fn strtod(s: *const c_char, endp: *mut *mut c_char) -> c_double; - pub fn strtol( - s: *const c_char, - endp: *mut *mut c_char, - base: c_int, - ) -> c_long; - pub fn strtoul( - s: *const c_char, - endp: *mut *mut c_char, - base: c_int, - ) -> c_ulong; + pub fn strtof(s: *const c_char, endp: *mut *mut c_char) -> c_float; + pub fn strtol(s: *const c_char, endp: *mut *mut c_char, base: c_int) -> c_long; + pub fn strtoul(s: *const c_char, endp: *mut *mut c_char, base: c_int) -> c_ulong; pub fn calloc(nobj: size_t, size: size_t) -> *mut c_void; pub fn malloc(size: size_t) -> *mut c_void; pub fn realloc(p: *mut c_void, size: size_t) -> *mut c_void; @@ -1144,17 +1135,9 @@ extern "C" { pub fn getenv(s: *const c_char) -> *mut c_char; pub fn strcpy(dst: *mut c_char, src: *const c_char) -> *mut c_char; - pub fn strncpy( - dst: *mut c_char, - src: *const c_char, - n: size_t, - ) -> *mut c_char; + pub fn strncpy(dst: *mut c_char, src: *const c_char, n: size_t) -> *mut c_char; pub fn strcat(s: *mut c_char, ct: *const c_char) -> *mut c_char; - pub fn strncat( - s: *mut c_char, - ct: *const c_char, - n: size_t, - ) -> *mut c_char; + pub fn strncat(s: *mut c_char, ct: *const c_char, n: size_t) -> *mut c_char; pub fn strcmp(cs: *const c_char, ct: *const c_char) -> c_int; pub fn strncmp(cs: *const c_char, ct: *const c_char, n: size_t) -> c_int; pub fn strcoll(cs: *const c_char, ct: *const c_char) -> c_int; @@ -1166,60 +1149,30 @@ extern "C" { pub fn strpbrk(cs: *const c_char, ct: *const c_char) -> *mut c_char; pub fn strstr(cs: *const c_char, ct: *const c_char) -> *mut c_char; pub fn strcasecmp(s1: *const c_char, s2: *const c_char) -> c_int; - pub fn strncasecmp( - s1: *const c_char, - s2: *const c_char, - n: size_t, - ) -> c_int; + pub fn strncasecmp(s1: *const c_char, s2: *const c_char, n: size_t) -> c_int; pub fn strlen(cs: *const c_char) -> size_t; pub fn strerror(n: c_int) -> *mut c_char; pub fn strtok(s: *mut c_char, t: *const c_char) -> *mut c_char; pub fn strxfrm(s: *mut c_char, ct: *const c_char, n: size_t) -> size_t; pub fn wcslen(buf: *const wchar_t) -> size_t; - pub fn wcstombs( - dest: *mut c_char, - src: *const wchar_t, - n: size_t, - ) -> ::size_t; + pub fn wcstombs(dest: *mut c_char, src: *const wchar_t, n: size_t) -> ::size_t; pub fn memchr(cx: *const c_void, c: c_int, n: size_t) -> *mut c_void; pub fn wmemchr(cx: *const wchar_t, c: wchar_t, n: size_t) -> *mut wchar_t; pub fn memcmp(cx: *const c_void, ct: *const c_void, n: size_t) -> c_int; - pub fn memcpy( - dest: *mut c_void, - src: *const c_void, - n: size_t, - ) -> *mut c_void; - pub fn memmove( - dest: *mut c_void, - src: *const c_void, - n: size_t, - ) -> *mut c_void; + pub fn memcpy(dest: *mut c_void, src: *const c_void, n: size_t) -> *mut c_void; + pub fn memmove(dest: *mut c_void, src: *const c_void, n: size_t) -> *mut c_void; pub fn memset(dest: *mut c_void, c: c_int, n: size_t) -> *mut c_void; } extern "C" { - pub fn fprintf( - stream: *mut ::FILE, - format: *const ::c_char, - ... - ) -> ::c_int; + pub fn fprintf(stream: *mut ::FILE, format: *const ::c_char, ...) -> ::c_int; pub fn printf(format: *const ::c_char, ...) -> ::c_int; - pub fn snprintf( - s: *mut ::c_char, - n: ::size_t, - format: *const ::c_char, - ... - ) -> ::c_int; + pub fn snprintf(s: *mut ::c_char, n: ::size_t, format: *const ::c_char, ...) -> ::c_int; pub fn sprintf(s: *mut ::c_char, format: *const ::c_char, ...) -> ::c_int; - pub fn fscanf( - stream: *mut ::FILE, - format: *const ::c_char, - ... - ) -> ::c_int; + pub fn fscanf(stream: *mut ::FILE, format: *const ::c_char, ...) -> ::c_int; pub fn scanf(format: *const ::c_char, ...) -> ::c_int; - pub fn sscanf(s: *const ::c_char, format: *const ::c_char, ...) - -> ::c_int; + pub fn sscanf(s: *const ::c_char, format: *const ::c_char, ...) -> ::c_int; pub fn getchar_unlocked() -> ::c_int; pub fn putchar_unlocked(c: ::c_int) -> ::c_int; pub fn stat(path: *const c_char, buf: *mut stat) -> ::c_int; @@ -1237,11 +1190,7 @@ extern "C" { pub fn geteuid() -> uid_t; pub fn getgroups(ngroups_max: ::c_int, groups: *mut gid_t) -> ::c_int; pub fn getlogin() -> *mut c_char; - pub fn getopt( - argc: ::c_int, - argv: *const *mut c_char, - optstr: *const c_char, - ) -> ::c_int; + pub fn getopt(argc: ::c_int, argv: *const *mut c_char, optstr: *const c_char) -> ::c_int; pub fn pathconf(path: *const c_char, name: ::c_int) -> c_long; pub fn pause() -> ::c_int; pub fn seteuid(uid: uid_t) -> ::c_int; @@ -1267,27 +1216,15 @@ extern "C" { pub fn gettimeofday(tp: *mut ::timeval, tz: *mut ::c_void) -> ::c_int; pub fn pthread_exit(value: *mut ::c_void) -> !; - pub fn pthread_attr_setdetachstate( - attr: *mut ::pthread_attr_t, - state: ::c_int, - ) -> ::c_int; + pub fn pthread_attr_setdetachstate(attr: *mut ::pthread_attr_t, state: ::c_int) -> ::c_int; - pub fn strerror_r( - errnum: ::c_int, - buf: *mut c_char, - buflen: ::size_t, - ) -> ::c_int; + pub fn strerror_r(errnum: ::c_int, buf: *mut c_char, buflen: ::size_t) -> ::c_int; - pub fn sigaction( - signum: ::c_int, - act: *const sigaction, - oldact: *mut sigaction, - ) -> ::c_int; + pub fn sigaddset(set: *mut sigset_t, signum: ::c_int) -> ::c_int; - pub fn utimes( - filename: *const ::c_char, - times: *const ::timeval, - ) -> ::c_int; + pub fn sigaction(signum: ::c_int, act: *const sigaction, oldact: *mut sigaction) -> ::c_int; + + pub fn utimes(filename: *const ::c_char, times: *const ::timeval) -> ::c_int; #[link_name = "_rtld_dlopen"] pub fn dlopen(filename: *const ::c_char, flag: ::c_int) -> *mut ::c_void; @@ -1296,10 +1233,7 @@ extern "C" { pub fn dlerror() -> *mut ::c_char; #[link_name = "_rtld_dlsym"] - pub fn dlsym( - handle: *mut ::c_void, - symbol: *const ::c_char, - ) -> *mut ::c_void; + pub fn dlsym(handle: *mut ::c_void, symbol: *const ::c_char) -> *mut ::c_void; #[link_name = "_rtld_dlclose"] pub fn dlclose(handle: *mut ::c_void) -> ::c_int; @@ -1319,25 +1253,14 @@ extern "C" { pub fn gethostname(name: *mut ::c_char, len: ::size_t) -> ::c_int; pub fn usleep(secs: ::useconds_t) -> ::c_int; pub fn putenv(string: *mut c_char) -> ::c_int; - pub fn setlocale( - category: ::c_int, - locale: *const ::c_char, - ) -> *mut ::c_char; - - pub fn sigprocmask( - how: ::c_int, - set: *const sigset_t, - oldset: *mut sigset_t, - ) -> ::c_int; + pub fn setlocale(category: ::c_int, locale: *const ::c_char) -> *mut ::c_char; + + pub fn sigprocmask(how: ::c_int, set: *const sigset_t, oldset: *mut sigset_t) -> ::c_int; pub fn sigpending(set: *mut sigset_t) -> ::c_int; pub fn mkfifo(path: *const c_char, mode: mode_t) -> ::c_int; - pub fn fseeko( - stream: *mut ::FILE, - offset: ::off_t, - whence: ::c_int, - ) -> ::c_int; + pub fn fseeko(stream: *mut ::FILE, offset: ::off_t, whence: ::c_int) -> ::c_int; pub fn ftello(stream: *mut ::FILE) -> ::off_t; pub fn mkstemp(template: *mut ::c_char) -> ::c_int; @@ -1347,18 +1270,13 @@ extern "C" { pub fn closelog(); pub fn setlogmask(maskpri: ::c_int) -> ::c_int; pub fn syslog(priority: ::c_int, message: *const ::c_char, ...); - pub fn getline( - lineptr: *mut *mut c_char, - n: *mut size_t, - stream: *mut FILE, - ) -> ssize_t; + pub fn getline(lineptr: *mut *mut c_char, n: *mut size_t, stream: *mut FILE) -> ssize_t; } extern "C" { // stdlib.h - pub fn memalign(block_size: ::size_t, size_arg: ::size_t) - -> *mut ::c_void; + pub fn memalign(block_size: ::size_t, size_arg: ::size_t) -> *mut ::c_void; // ioLib.h pub fn getcwd(buf: *mut ::c_char, size: ::size_t) -> *mut ::c_char; @@ -1370,15 +1288,10 @@ extern "C" { pub fn pthread_mutexattr_init(attr: *mut pthread_mutexattr_t) -> ::c_int; // pthread.h - pub fn pthread_mutexattr_destroy( - attr: *mut pthread_mutexattr_t, - ) -> ::c_int; + pub fn pthread_mutexattr_destroy(attr: *mut pthread_mutexattr_t) -> ::c_int; // pthread.h - pub fn pthread_mutexattr_settype( - pAttr: *mut ::pthread_mutexattr_t, - pType: ::c_int, - ) -> ::c_int; + pub fn pthread_mutexattr_settype(pAttr: *mut ::pthread_mutexattr_t, pType: ::c_int) -> ::c_int; // pthread.h pub fn pthread_mutex_init( @@ -1396,31 +1309,20 @@ extern "C" { pub fn pthread_mutex_trylock(mutex: *mut pthread_mutex_t) -> ::c_int; // pthread.h - pub fn pthread_mutex_timedlock( - attr: *mut pthread_mutex_t, - spec: *const timespec, - ) -> ::c_int; + pub fn pthread_mutex_timedlock(attr: *mut pthread_mutex_t, spec: *const timespec) -> ::c_int; // pthread.h pub fn pthread_mutex_unlock(mutex: *mut pthread_mutex_t) -> ::c_int; // pthread.h - pub fn pthread_attr_setname( - pAttr: *mut ::pthread_attr_t, - name: *mut ::c_char, - ) -> ::c_int; + pub fn pthread_attr_setname(pAttr: *mut ::pthread_attr_t, name: *mut ::c_char) -> ::c_int; // pthread.h - pub fn pthread_attr_setstacksize( - attr: *mut ::pthread_attr_t, - stacksize: ::size_t, - ) -> ::c_int; + pub fn pthread_attr_setstacksize(attr: *mut ::pthread_attr_t, stacksize: ::size_t) -> ::c_int; // pthread.h - pub fn pthread_attr_getstacksize( - attr: *const ::pthread_attr_t, - size: *mut ::size_t, - ) -> ::c_int; + pub fn pthread_attr_getstacksize(attr: *const ::pthread_attr_t, size: *mut ::size_t) + -> ::c_int; // pthread.h pub fn pthread_attr_init(attr: *mut ::pthread_attr_t) -> ::c_int; @@ -1455,11 +1357,8 @@ extern "C" { pub fn ftruncate(fd: ::c_int, length: off_t) -> ::c_int; // dirent.h - pub fn readdir_r( - pDir: *mut ::DIR, - entry: *mut ::dirent, - result: *mut *mut ::dirent, - ) -> ::c_int; + pub fn readdir_r(pDir: *mut ::DIR, entry: *mut ::dirent, result: *mut *mut ::dirent) + -> ::c_int; // dirent.h pub fn readdir(pDir: *mut ::DIR) -> *mut ::dirent; @@ -1475,9 +1374,7 @@ extern "C" { pub fn pthread_condattr_init(attr: *mut ::pthread_condattr_t) -> ::c_int; // pthread.h - pub fn pthread_condattr_destroy( - attr: *mut ::pthread_condattr_t, - ) -> ::c_int; + pub fn pthread_condattr_destroy(attr: *mut ::pthread_condattr_t) -> ::c_int; // pthread.h pub fn pthread_condattr_getclock( @@ -1507,20 +1404,14 @@ extern "C" { pub fn pthread_cond_broadcast(cond: *mut ::pthread_cond_t) -> ::c_int; // pthread.h - pub fn pthread_cond_wait( - cond: *mut ::pthread_cond_t, - mutex: *mut ::pthread_mutex_t, - ) -> ::c_int; + pub fn pthread_cond_wait(cond: *mut ::pthread_cond_t, mutex: *mut ::pthread_mutex_t) + -> ::c_int; // pthread.h - pub fn pthread_rwlockattr_init( - attr: *mut ::pthread_rwlockattr_t, - ) -> ::c_int; + pub fn pthread_rwlockattr_init(attr: *mut ::pthread_rwlockattr_t) -> ::c_int; // pthread.h - pub fn pthread_rwlockattr_destroy( - attr: *mut ::pthread_rwlockattr_t, - ) -> ::c_int; + pub fn pthread_rwlockattr_destroy(attr: *mut ::pthread_rwlockattr_t) -> ::c_int; // pthread.h pub fn pthread_rwlockattr_setmaxreaders( @@ -1574,10 +1465,7 @@ extern "C" { pub fn pthread_key_delete(key: ::pthread_key_t) -> ::c_int; // pthread.h - pub fn pthread_setspecific( - key: ::pthread_key_t, - value: *const ::c_void, - ) -> ::c_int; + pub fn pthread_setspecific(key: ::pthread_key_t, value: *const ::c_void) -> ::c_int; // pthread.h pub fn pthread_getspecific(key: ::pthread_key_t) -> *mut ::c_void; @@ -1590,37 +1478,22 @@ extern "C" { ) -> ::c_int; // pthread.h - pub fn pthread_attr_getname( - attr: *mut ::pthread_attr_t, - name: *mut *mut ::c_char, - ) -> ::c_int; + pub fn pthread_attr_getname(attr: *mut ::pthread_attr_t, name: *mut *mut ::c_char) -> ::c_int; // pthread.h - pub fn pthread_join( - thread: ::pthread_t, - status: *mut *mut ::c_void, - ) -> ::c_int; + pub fn pthread_join(thread: ::pthread_t, status: *mut *mut ::c_void) -> ::c_int; // pthread.h pub fn pthread_self() -> ::pthread_t; // clockLib.h - pub fn clock_gettime( - clock_id: ::clockid_t, - tp: *mut ::timespec, - ) -> ::c_int; + pub fn clock_gettime(clock_id: ::clockid_t, tp: *mut ::timespec) -> ::c_int; // clockLib.h - pub fn clock_settime( - clock_id: ::clockid_t, - tp: *const ::timespec, - ) -> ::c_int; + pub fn clock_settime(clock_id: ::clockid_t, tp: *const ::timespec) -> ::c_int; // clockLib.h - pub fn clock_getres( - clock_id: ::clockid_t, - res: *mut ::timespec, - ) -> ::c_int; + pub fn clock_getres(clock_id: ::clockid_t, res: *mut ::timespec) -> ::c_int; // clockLib.h pub fn clock_nanosleep( @@ -1631,35 +1504,19 @@ extern "C" { ) -> ::c_int; // timerLib.h - pub fn nanosleep( - rqtp: *const ::timespec, - rmtp: *mut ::timespec, - ) -> ::c_int; + pub fn nanosleep(rqtp: *const ::timespec, rmtp: *mut ::timespec) -> ::c_int; // socket.h - pub fn accept( - s: ::c_int, - addr: *mut ::sockaddr, - addrlen: *mut ::socklen_t, - ) -> ::c_int; + pub fn accept(s: ::c_int, addr: *mut ::sockaddr, addrlen: *mut ::socklen_t) -> ::c_int; // socket.h - pub fn bind(fd: ::c_int, addr: *const sockaddr, len: socklen_t) - -> ::c_int; + pub fn bind(fd: ::c_int, addr: *const sockaddr, len: socklen_t) -> ::c_int; // socket.h - pub fn connect( - s: ::c_int, - name: *const ::sockaddr, - namelen: ::socklen_t, - ) -> ::c_int; + pub fn connect(s: ::c_int, name: *const ::sockaddr, namelen: ::socklen_t) -> ::c_int; // socket.h - pub fn getpeername( - s: ::c_int, - name: *mut ::sockaddr, - namelen: *mut ::socklen_t, - ) -> ::c_int; + pub fn getpeername(s: ::c_int, name: *mut ::sockaddr, namelen: *mut ::socklen_t) -> ::c_int; // socket.h pub fn getsockname( @@ -1681,12 +1538,7 @@ extern "C" { pub fn listen(socket: ::c_int, backlog: ::c_int) -> ::c_int; // socket.h - pub fn recv( - s: ::c_int, - buf: *mut ::c_void, - bufLen: ::size_t, - flags: ::c_int, - ) -> ::ssize_t; + pub fn recv(s: ::c_int, buf: *mut ::c_void, bufLen: ::size_t, flags: ::c_int) -> ::ssize_t; // socket.h pub fn recvfrom( @@ -1698,25 +1550,12 @@ extern "C" { pFromLen: *mut ::socklen_t, ) -> ::ssize_t; - pub fn recvmsg( - socket: ::c_int, - mp: *mut ::msghdr, - flags: ::c_int, - ) -> ::ssize_t; + pub fn recvmsg(socket: ::c_int, mp: *mut ::msghdr, flags: ::c_int) -> ::ssize_t; // socket.h - pub fn send( - socket: ::c_int, - buf: *const ::c_void, - len: ::size_t, - flags: ::c_int, - ) -> ::ssize_t; + pub fn send(socket: ::c_int, buf: *const ::c_void, len: ::size_t, flags: ::c_int) -> ::ssize_t; - pub fn sendmsg( - socket: ::c_int, - mp: *const ::msghdr, - flags: ::c_int, - ) -> ::ssize_t; + pub fn sendmsg(socket: ::c_int, mp: *const ::msghdr, flags: ::c_int) -> ::ssize_t; // socket.h pub fn sendto( @@ -1741,11 +1580,7 @@ extern "C" { pub fn shutdown(s: ::c_int, how: ::c_int) -> ::c_int; // socket.h - pub fn socket( - domain: ::c_int, - _type: ::c_int, - protocol: ::c_int, - ) -> ::c_int; + pub fn socket(domain: ::c_int, _type: ::c_int, protocol: ::c_int) -> ::c_int; // icotl.h pub fn ioctl(fd: ::c_int, request: ::c_int, ...) -> ::c_int; @@ -1763,16 +1598,11 @@ extern "C" { // ioLib.h or // unistd.h - pub fn read(fd: ::c_int, buf: *mut ::c_void, count: ::size_t) - -> ::ssize_t; + pub fn read(fd: ::c_int, buf: *mut ::c_void, count: ::size_t) -> ::ssize_t; // ioLib.h or // unistd.h - pub fn write( - fd: ::c_int, - buf: *const ::c_void, - count: ::size_t, - ) -> ::ssize_t; + pub fn write(fd: ::c_int, buf: *const ::c_void, count: ::size_t) -> ::ssize_t; // ioLib.h or // unistd.h @@ -1819,8 +1649,7 @@ extern "C" { pub fn getppid() -> pid_t; // wait.h - pub fn waitpid(pid: pid_t, status: *mut ::c_int, optons: ::c_int) - -> pid_t; + pub fn waitpid(pid: pid_t, status: *mut ::c_int, optons: ::c_int) -> pid_t; // unistd.h pub fn sysconf(attr: ::c_int) -> ::c_long; @@ -1840,20 +1669,13 @@ extern "C" { ) -> ::c_int; // stdlib.h - pub fn realpath( - fileName: *const ::c_char, - resolvedName: *mut ::c_char, - ) -> *mut ::c_char; + pub fn realpath(fileName: *const ::c_char, resolvedName: *mut ::c_char) -> *mut ::c_char; // unistd.h pub fn link(src: *const ::c_char, dst: *const ::c_char) -> ::c_int; // unistd.h - pub fn readlink( - path: *const ::c_char, - buf: *mut ::c_char, - bufsize: ::size_t, - ) -> ::ssize_t; + pub fn readlink(path: *const ::c_char, buf: *mut ::c_char, bufsize: ::size_t) -> ::ssize_t; // unistd.h pub fn symlink(path1: *const ::c_char, path2: *const ::c_char) -> ::c_int; @@ -1918,11 +1740,7 @@ extern "C" { pub fn kill(__pid: pid_t, __signo: ::c_int) -> ::c_int; // signal.h for user - pub fn sigqueue( - __pid: pid_t, - __signo: ::c_int, - __value: ::sigval, - ) -> ::c_int; + pub fn sigqueue(__pid: pid_t, __signo: ::c_int, __value: ::sigval) -> ::c_int; // signal.h for user pub fn _sigqueue( @@ -1955,27 +1773,13 @@ extern "C" { ) -> RTP_ID; // ioLib.h - pub fn _realpath( - fileName: *const ::c_char, - resolvedName: *mut ::c_char, - ) -> *mut ::c_char; + pub fn _realpath(fileName: *const ::c_char, resolvedName: *mut ::c_char) -> *mut ::c_char; // pathLib.h - pub fn _pathIsAbsolute( - filepath: *const ::c_char, - pNameTail: *mut *const ::c_char, - ) -> BOOL; + pub fn _pathIsAbsolute(filepath: *const ::c_char, pNameTail: *mut *const ::c_char) -> BOOL; - pub fn writev( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - ) -> ::ssize_t; - pub fn readv( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - ) -> ::ssize_t; + pub fn writev(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int) -> ::ssize_t; + pub fn readv(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int) -> ::ssize_t; // randomNumGen.h pub fn randBytes(buf: *mut c_uchar, length: c_int) -> c_int; @@ -2014,41 +1818,34 @@ extern "C" { abs_timeout: *const ::timespec, ) -> ::c_int; pub fn mq_getattr(mqd: ::mqd_t, attr: *mut ::mq_attr) -> ::c_int; - pub fn mq_setattr( - mqd: ::mqd_t, - newattr: *const ::mq_attr, - oldattr: *mut ::mq_attr, - ) -> ::c_int; + pub fn mq_setattr(mqd: ::mqd_t, newattr: *const ::mq_attr, oldattr: *mut ::mq_attr) -> ::c_int; } //Dummy functions, these don't really exist in VxWorks. // wait.h macros -pub fn WIFEXITED(status: ::c_int) -> bool { - (status & 0xFF00) == 0 -} -pub fn WIFSIGNALED(status: ::c_int) -> bool { - (status & 0xFF00) != 0 -} -pub fn WIFSTOPPED(status: ::c_int) -> bool { - (status & 0xFF0000) != 0 -} -pub fn WEXITSTATUS(status: ::c_int) -> ::c_int { - status & 0xFF -} -pub fn WTERMSIG(status: ::c_int) -> ::c_int { - (status >> 8) & 0xFF -} -pub fn WSTOPSIG(status: ::c_int) -> ::c_int { - (status >> 16) & 0xFF +safe_f! { + pub {const} fn WIFEXITED(status: ::c_int) -> bool { + (status & 0xFF00) == 0 + } + pub {const} fn WIFSIGNALED(status: ::c_int) -> bool { + (status & 0xFF00) != 0 + } + pub {const} fn WIFSTOPPED(status: ::c_int) -> bool { + (status & 0xFF0000) != 0 + } + pub {const} fn WEXITSTATUS(status: ::c_int) -> ::c_int { + status & 0xFF + } + pub {const} fn WTERMSIG(status: ::c_int) -> ::c_int { + (status >> 8) & 0xFF + } + pub {const} fn WSTOPSIG(status: ::c_int) -> ::c_int { + (status >> 16) & 0xFF + } } -pub fn pread( - _fd: ::c_int, - _buf: *mut ::c_void, - _count: ::size_t, - _offset: off64_t, -) -> ::ssize_t { +pub fn pread(_fd: ::c_int, _buf: *mut ::c_void, _count: ::size_t, _offset: off64_t) -> ::ssize_t { -1 } @@ -2060,16 +1857,10 @@ pub fn pwrite( ) -> ::ssize_t { -1 } -pub fn posix_memalign( - memptr: *mut *mut ::c_void, - align: ::size_t, - size: ::size_t, -) -> ::c_int { +pub fn posix_memalign(memptr: *mut *mut ::c_void, align: ::size_t, size: ::size_t) -> ::c_int { // check to see if align is a power of 2 and if align is a multiple // of sizeof(void *) - if (align & align - 1 != 0) - || (align as usize % size_of::<::size_t>() != 0) - { + if (align & align - 1 != 0) || (align as usize % size_of::<::size_t>() != 0) { return ::EINVAL; } diff --git a/crux-mir/lib/libc/src/wasi.rs b/crux-mir/lib/libc/src/wasi.rs index 081141e44..c5dd67047 100644 --- a/crux-mir/lib/libc/src/wasi.rs +++ b/crux-mir/lib/libc/src/wasi.rs @@ -1,3 +1,5 @@ +use super::{Send, Sync}; + pub use ffi::c_void; pub type c_char = i8; @@ -35,9 +37,18 @@ pub type nlink_t = u64; pub type blksize_t = c_long; pub type blkcnt_t = i64; pub type nfds_t = c_ulong; - +pub type wchar_t = i32; +pub type nl_item = c_int; pub type __wasi_rights_t = u64; +s_no_extra_traits! { + #[repr(align(16))] + #[allow(missing_debug_implementations)] + pub struct max_align_t { + priv_: [f64; 4] + } +} + #[allow(missing_copy_implementations)] #[cfg_attr(feature = "extra_traits", derive(Debug))] pub enum FILE {} @@ -50,6 +61,16 @@ pub enum __locale_struct {} pub type locale_t = *mut __locale_struct; +s_paren! { + // in wasi-libc clockid_t is const struct __clockid* (where __clockid is an opaque struct), + // but that's an implementation detail that we don't want to have to deal with + #[repr(transparent)] + pub struct clockid_t(*const u8); +} + +unsafe impl Send for clockid_t {} +unsafe impl Sync for clockid_t {} + s! { #[repr(align(8))] pub struct fpos_t { @@ -180,6 +201,11 @@ pub const SEEK_END: c_int = 2; pub const _IOFBF: c_int = 0; pub const _IONBF: c_int = 2; pub const _IOLBF: c_int = 1; +pub const F_GETFD: c_int = 1; +pub const F_SETFD: c_int = 2; +pub const F_GETFL: c_int = 3; +pub const F_SETFL: c_int = 4; +pub const FD_CLOEXEC: c_int = 1; pub const FD_SETSIZE: size_t = 1024; pub const O_APPEND: c_int = 0x0001; pub const O_DSYNC: c_int = 0x0002; @@ -195,20 +221,62 @@ pub const O_EXEC: c_int = 0x02000000; pub const O_RDONLY: c_int = 0x04000000; pub const O_SEARCH: c_int = 0x08000000; pub const O_WRONLY: c_int = 0x10000000; +pub const O_CLOEXEC: c_int = 0x0; pub const O_RDWR: c_int = O_WRONLY | O_RDONLY; pub const O_ACCMODE: c_int = O_EXEC | O_RDWR | O_SEARCH; +pub const O_NOCTTY: c_int = 0x0; pub const POSIX_FADV_DONTNEED: c_int = 4; pub const POSIX_FADV_NOREUSE: c_int = 5; pub const POSIX_FADV_NORMAL: c_int = 0; pub const POSIX_FADV_RANDOM: c_int = 2; pub const POSIX_FADV_SEQUENTIAL: c_int = 1; pub const POSIX_FADV_WILLNEED: c_int = 3; +pub const AT_FDCWD: ::c_int = -2; pub const AT_EACCESS: c_int = 0x0; pub const AT_SYMLINK_NOFOLLOW: c_int = 0x1; pub const AT_SYMLINK_FOLLOW: c_int = 0x2; pub const AT_REMOVEDIR: c_int = 0x4; pub const UTIME_OMIT: c_long = 0xfffffffe; pub const UTIME_NOW: c_long = 0xffffffff; +pub const S_IFIFO: mode_t = 49152; +pub const S_IFCHR: mode_t = 8192; +pub const S_IFBLK: mode_t = 24576; +pub const S_IFDIR: mode_t = 16384; +pub const S_IFREG: mode_t = 32768; +pub const S_IFLNK: mode_t = 40960; +pub const S_IFSOCK: mode_t = 49152; +pub const S_IFMT: mode_t = 57344; +pub const S_IXOTH: mode_t = 0x1; +pub const S_IWOTH: mode_t = 0x2; +pub const S_IROTH: mode_t = 0x4; +pub const S_IXGRP: mode_t = 0x8; +pub const S_IWGRP: mode_t = 0x10; +pub const S_IRGRP: mode_t = 0x20; +pub const S_IXUSR: mode_t = 0x40; +pub const S_IWUSR: mode_t = 0x80; +pub const S_IRUSR: mode_t = 0x100; +pub const S_ISVTX: mode_t = 0x200; +pub const S_ISGID: mode_t = 0x400; +pub const S_ISUID: mode_t = 0x800; +pub const DT_UNKNOWN: u8 = 0; +pub const DT_BLK: u8 = 1; +pub const DT_CHR: u8 = 2; +pub const DT_DIR: u8 = 3; +pub const DT_REG: u8 = 4; +pub const DT_LNK: u8 = 7; +pub const FIONREAD: c_int = 1; +pub const FIONBIO: c_int = 2; +pub const F_OK: ::c_int = 0; +pub const R_OK: ::c_int = 4; +pub const W_OK: ::c_int = 2; +pub const X_OK: ::c_int = 1; +pub const POLLIN: ::c_short = 0x1; +pub const POLLOUT: ::c_short = 0x2; +pub const POLLERR: ::c_short = 0x1000; +pub const POLLHUP: ::c_short = 0x2000; +pub const POLLNVAL: ::c_short = 0x4000; +pub const POLLRDNORM: ::c_short = 0x1; +pub const POLLWRNORM: ::c_short = 0x2; pub const E2BIG: c_int = 1; pub const EACCES: c_int = 2; @@ -289,9 +357,91 @@ pub const ENOTCAPABLE: c_int = 76; pub const EOPNOTSUPP: c_int = ENOTSUP; pub const EWOULDBLOCK: c_int = EAGAIN; +pub const _SC_PAGESIZE: c_int = 30; +pub const _SC_PAGE_SIZE: ::c_int = _SC_PAGESIZE; +pub const _SC_IOV_MAX: c_int = 60; +pub const _SC_SYMLOOP_MAX: c_int = 173; + +pub static CLOCK_MONOTONIC: clockid_t = unsafe { clockid_t(ptr_addr_of!(_CLOCK_MONOTONIC)) }; +pub static CLOCK_PROCESS_CPUTIME_ID: clockid_t = + unsafe { clockid_t(ptr_addr_of!(_CLOCK_PROCESS_CPUTIME_ID)) }; +pub static CLOCK_REALTIME: clockid_t = unsafe { clockid_t(ptr_addr_of!(_CLOCK_REALTIME)) }; +pub static CLOCK_THREAD_CPUTIME_ID: clockid_t = + unsafe { clockid_t(ptr_addr_of!(_CLOCK_THREAD_CPUTIME_ID)) }; + +pub const ABDAY_1: ::nl_item = 0x20000; +pub const ABDAY_2: ::nl_item = 0x20001; +pub const ABDAY_3: ::nl_item = 0x20002; +pub const ABDAY_4: ::nl_item = 0x20003; +pub const ABDAY_5: ::nl_item = 0x20004; +pub const ABDAY_6: ::nl_item = 0x20005; +pub const ABDAY_7: ::nl_item = 0x20006; + +pub const DAY_1: ::nl_item = 0x20007; +pub const DAY_2: ::nl_item = 0x20008; +pub const DAY_3: ::nl_item = 0x20009; +pub const DAY_4: ::nl_item = 0x2000A; +pub const DAY_5: ::nl_item = 0x2000B; +pub const DAY_6: ::nl_item = 0x2000C; +pub const DAY_7: ::nl_item = 0x2000D; + +pub const ABMON_1: ::nl_item = 0x2000E; +pub const ABMON_2: ::nl_item = 0x2000F; +pub const ABMON_3: ::nl_item = 0x20010; +pub const ABMON_4: ::nl_item = 0x20011; +pub const ABMON_5: ::nl_item = 0x20012; +pub const ABMON_6: ::nl_item = 0x20013; +pub const ABMON_7: ::nl_item = 0x20014; +pub const ABMON_8: ::nl_item = 0x20015; +pub const ABMON_9: ::nl_item = 0x20016; +pub const ABMON_10: ::nl_item = 0x20017; +pub const ABMON_11: ::nl_item = 0x20018; +pub const ABMON_12: ::nl_item = 0x20019; + +pub const MON_1: ::nl_item = 0x2001A; +pub const MON_2: ::nl_item = 0x2001B; +pub const MON_3: ::nl_item = 0x2001C; +pub const MON_4: ::nl_item = 0x2001D; +pub const MON_5: ::nl_item = 0x2001E; +pub const MON_6: ::nl_item = 0x2001F; +pub const MON_7: ::nl_item = 0x20020; +pub const MON_8: ::nl_item = 0x20021; +pub const MON_9: ::nl_item = 0x20022; +pub const MON_10: ::nl_item = 0x20023; +pub const MON_11: ::nl_item = 0x20024; +pub const MON_12: ::nl_item = 0x20025; + +pub const AM_STR: ::nl_item = 0x20026; +pub const PM_STR: ::nl_item = 0x20027; + +pub const D_T_FMT: ::nl_item = 0x20028; +pub const D_FMT: ::nl_item = 0x20029; +pub const T_FMT: ::nl_item = 0x2002A; +pub const T_FMT_AMPM: ::nl_item = 0x2002B; + +pub const ERA: ::nl_item = 0x2002C; +pub const ERA_D_FMT: ::nl_item = 0x2002E; +pub const ALT_DIGITS: ::nl_item = 0x2002F; +pub const ERA_D_T_FMT: ::nl_item = 0x20030; +pub const ERA_T_FMT: ::nl_item = 0x20031; + +pub const CODESET: ::nl_item = 14; +pub const CRNCYSTR: ::nl_item = 0x4000F; +pub const RADIXCHAR: ::nl_item = 0x10000; +pub const THOUSEP: ::nl_item = 0x10001; +pub const YESEXPR: ::nl_item = 0x50000; +pub const NOEXPR: ::nl_item = 0x50001; +pub const YESSTR: ::nl_item = 0x50002; +pub const NOSTR: ::nl_item = 0x50003; + #[cfg_attr( feature = "rustc-dep-of-std", - link(name = "c", kind = "static", cfg(target_feature = "crt-static")) + link( + name = "c", + kind = "static", + modifiers = "-bundle", + cfg(target_feature = "crt-static") + ) )] #[cfg_attr( feature = "rustc-dep-of-std", @@ -318,11 +468,7 @@ extern "C" { pub fn write(fd: c_int, ptr: *const c_void, size: size_t) -> ssize_t; pub static mut environ: *mut *mut c_char; pub fn fopen(a: *const c_char, b: *const c_char) -> *mut FILE; - pub fn freopen( - a: *const c_char, - b: *const c_char, - f: *mut FILE, - ) -> *mut FILE; + pub fn freopen(a: *const c_char, b: *const c_char, f: *mut FILE) -> *mut FILE; pub fn fclose(f: *mut FILE) -> c_int; pub fn remove(a: *const c_char) -> c_int; pub fn rename(a: *const c_char, b: *const c_char) -> c_int; @@ -335,18 +481,8 @@ extern "C" { pub fn rewind(f: *mut FILE); pub fn fgetpos(f: *mut FILE, pos: *mut fpos_t) -> c_int; pub fn fsetpos(f: *mut FILE, pos: *const fpos_t) -> c_int; - pub fn fread( - buf: *mut c_void, - a: size_t, - b: size_t, - f: *mut FILE, - ) -> size_t; - pub fn fwrite( - buf: *const c_void, - a: size_t, - b: size_t, - f: *mut FILE, - ) -> size_t; + pub fn fread(buf: *mut c_void, a: size_t, b: size_t, f: *mut FILE) -> size_t; + pub fn fwrite(buf: *const c_void, a: size_t, b: size_t, f: *mut FILE) -> size_t; pub fn fgetc(f: *mut FILE) -> c_int; pub fn getc(f: *mut FILE) -> c_int; pub fn getchar() -> c_int; @@ -370,27 +506,22 @@ extern "C" { pub fn time(a: *mut time_t) -> time_t; pub fn difftime(a: time_t, b: time_t) -> c_double; pub fn mktime(a: *mut tm) -> time_t; - pub fn strftime( - a: *mut c_char, - b: size_t, - c: *const c_char, - d: *const tm, - ) -> size_t; + pub fn strftime(a: *mut c_char, b: size_t, c: *const c_char, d: *const tm) -> size_t; pub fn gmtime(a: *const time_t) -> *mut tm; pub fn gmtime_r(a: *const time_t, b: *mut tm) -> *mut tm; + pub fn localtime(a: *const time_t) -> *mut tm; pub fn localtime_r(a: *const time_t, b: *mut tm) -> *mut tm; pub fn asctime_r(a: *const tm, b: *mut c_char) -> *mut c_char; pub fn ctime_r(a: *const time_t, b: *mut c_char) -> *mut c_char; + static _CLOCK_MONOTONIC: u8; + static _CLOCK_PROCESS_CPUTIME_ID: u8; + static _CLOCK_REALTIME: u8; + static _CLOCK_THREAD_CPUTIME_ID: u8; pub fn nanosleep(a: *const timespec, b: *mut timespec) -> c_int; - // pub fn clock_getres(a: clockid_t, b: *mut timespec) -> c_int; - // pub fn clock_gettime(a: clockid_t, b: *mut timespec) -> c_int; - // pub fn clock_nanosleep( - // a: clockid_t, - // a2: c_int, - // b: *const timespec, - // c: *mut timespec, - // ) -> c_int; + pub fn clock_getres(a: clockid_t, b: *mut timespec) -> c_int; + pub fn clock_gettime(a: clockid_t, b: *mut timespec) -> c_int; + pub fn clock_nanosleep(a: clockid_t, a2: c_int, b: *const timespec, c: *mut timespec) -> c_int; pub fn isalnum(c: c_int) -> c_int; pub fn isalpha(c: c_int) -> c_int; @@ -403,43 +534,23 @@ extern "C" { pub fn isspace(c: c_int) -> c_int; pub fn isupper(c: c_int) -> c_int; pub fn isxdigit(c: c_int) -> c_int; + pub fn isblank(c: c_int) -> c_int; pub fn tolower(c: c_int) -> c_int; pub fn toupper(c: c_int) -> c_int; - pub fn setvbuf( - stream: *mut FILE, - buffer: *mut c_char, - mode: c_int, - size: size_t, - ) -> c_int; + pub fn setvbuf(stream: *mut FILE, buffer: *mut c_char, mode: c_int, size: size_t) -> c_int; pub fn setbuf(stream: *mut FILE, buf: *mut c_char); - pub fn fgets(buf: *mut c_char, n: c_int, stream: *mut FILE) - -> *mut c_char; + pub fn fgets(buf: *mut c_char, n: c_int, stream: *mut FILE) -> *mut c_char; pub fn atoi(s: *const c_char) -> c_int; pub fn atof(s: *const c_char) -> c_double; pub fn strtod(s: *const c_char, endp: *mut *mut c_char) -> c_double; - pub fn strtol( - s: *const c_char, - endp: *mut *mut c_char, - base: c_int, - ) -> c_long; - pub fn strtoul( - s: *const c_char, - endp: *mut *mut c_char, - base: c_int, - ) -> c_ulong; + pub fn strtof(s: *const c_char, endp: *mut *mut c_char) -> c_float; + pub fn strtol(s: *const c_char, endp: *mut *mut c_char, base: c_int) -> c_long; + pub fn strtoul(s: *const c_char, endp: *mut *mut c_char, base: c_int) -> c_ulong; pub fn strcpy(dst: *mut c_char, src: *const c_char) -> *mut c_char; - pub fn strncpy( - dst: *mut c_char, - src: *const c_char, - n: size_t, - ) -> *mut c_char; + pub fn strncpy(dst: *mut c_char, src: *const c_char, n: size_t) -> *mut c_char; pub fn strcat(s: *mut c_char, ct: *const c_char) -> *mut c_char; - pub fn strncat( - s: *mut c_char, - ct: *const c_char, - n: size_t, - ) -> *mut c_char; + pub fn strncat(s: *mut c_char, ct: *const c_char, n: size_t) -> *mut c_char; pub fn strcmp(cs: *const c_char, ct: *const c_char) -> c_int; pub fn strncmp(cs: *const c_char, ct: *const c_char, n: size_t) -> c_int; pub fn strcoll(cs: *const c_char, ct: *const c_char) -> c_int; @@ -448,14 +559,11 @@ extern "C" { pub fn strspn(cs: *const c_char, ct: *const c_char) -> size_t; pub fn strcspn(cs: *const c_char, ct: *const c_char) -> size_t; pub fn strdup(cs: *const c_char) -> *mut c_char; + pub fn strndup(cs: *const c_char, n: size_t) -> *mut c_char; pub fn strpbrk(cs: *const c_char, ct: *const c_char) -> *mut c_char; pub fn strstr(cs: *const c_char, ct: *const c_char) -> *mut c_char; pub fn strcasecmp(s1: *const c_char, s2: *const c_char) -> c_int; - pub fn strncasecmp( - s1: *const c_char, - s2: *const c_char, - n: size_t, - ) -> c_int; + pub fn strncasecmp(s1: *const c_char, s2: *const c_char, n: size_t) -> c_int; pub fn strlen(cs: *const c_char) -> size_t; pub fn strnlen(cs: *const c_char, maxlen: size_t) -> size_t; pub fn strerror(n: c_int) -> *mut c_char; @@ -464,39 +572,17 @@ extern "C" { pub fn memchr(cx: *const c_void, c: c_int, n: size_t) -> *mut c_void; pub fn memcmp(cx: *const c_void, ct: *const c_void, n: size_t) -> c_int; - pub fn memcpy( - dest: *mut c_void, - src: *const c_void, - n: size_t, - ) -> *mut c_void; - pub fn memmove( - dest: *mut c_void, - src: *const c_void, - n: size_t, - ) -> *mut c_void; + pub fn memcpy(dest: *mut c_void, src: *const c_void, n: size_t) -> *mut c_void; + pub fn memmove(dest: *mut c_void, src: *const c_void, n: size_t) -> *mut c_void; pub fn memset(dest: *mut c_void, c: c_int, n: size_t) -> *mut c_void; - pub fn fprintf( - stream: *mut ::FILE, - format: *const ::c_char, - ... - ) -> ::c_int; + pub fn fprintf(stream: *mut ::FILE, format: *const ::c_char, ...) -> ::c_int; pub fn printf(format: *const ::c_char, ...) -> ::c_int; - pub fn snprintf( - s: *mut ::c_char, - n: ::size_t, - format: *const ::c_char, - ... - ) -> ::c_int; + pub fn snprintf(s: *mut ::c_char, n: ::size_t, format: *const ::c_char, ...) -> ::c_int; pub fn sprintf(s: *mut ::c_char, format: *const ::c_char, ...) -> ::c_int; - pub fn fscanf( - stream: *mut ::FILE, - format: *const ::c_char, - ... - ) -> ::c_int; + pub fn fscanf(stream: *mut ::FILE, format: *const ::c_char, ...) -> ::c_int; pub fn scanf(format: *const ::c_char, ...) -> ::c_int; - pub fn sscanf(s: *const ::c_char, format: *const ::c_char, ...) - -> ::c_int; + pub fn sscanf(s: *const ::c_char, format: *const ::c_char, ...) -> ::c_int; pub fn getchar_unlocked() -> ::c_int; pub fn putchar_unlocked(c: ::c_int) -> ::c_int; @@ -515,13 +601,10 @@ extern "C" { pub fn closedir(dirp: *mut ::DIR) -> ::c_int; pub fn rewinddir(dirp: *mut ::DIR); pub fn dirfd(dirp: *mut ::DIR) -> ::c_int; + pub fn seekdir(dirp: *mut ::DIR, loc: ::c_long); + pub fn telldir(dirp: *mut ::DIR) -> ::c_long; - pub fn openat( - dirfd: ::c_int, - pathname: *const ::c_char, - flags: ::c_int, - ... - ) -> ::c_int; + pub fn openat(dirfd: ::c_int, pathname: *const ::c_char, flags: ::c_int, ...) -> ::c_int; pub fn fstatat( dirfd: ::c_int, pathname: *const ::c_char, @@ -535,11 +618,7 @@ extern "C" { newpath: *const ::c_char, flags: ::c_int, ) -> ::c_int; - pub fn mkdirat( - dirfd: ::c_int, - pathname: *const ::c_char, - mode: ::mode_t, - ) -> ::c_int; + pub fn mkdirat(dirfd: ::c_int, pathname: *const ::c_char, mode: ::mode_t) -> ::c_int; pub fn readlinkat( dirfd: ::c_int, pathname: *const ::c_char, @@ -557,40 +636,21 @@ extern "C" { newdirfd: ::c_int, linkpath: *const ::c_char, ) -> ::c_int; - pub fn unlinkat( - dirfd: ::c_int, - pathname: *const ::c_char, - flags: ::c_int, - ) -> ::c_int; + pub fn unlinkat(dirfd: ::c_int, pathname: *const ::c_char, flags: ::c_int) -> ::c_int; pub fn access(path: *const c_char, amode: ::c_int) -> ::c_int; pub fn close(fd: ::c_int) -> ::c_int; pub fn fpathconf(filedes: ::c_int, name: ::c_int) -> c_long; - pub fn getopt( - argc: ::c_int, - argv: *const *mut c_char, - optstr: *const c_char, - ) -> ::c_int; + pub fn getopt(argc: ::c_int, argv: *const *mut c_char, optstr: *const c_char) -> ::c_int; pub fn isatty(fd: ::c_int) -> ::c_int; pub fn link(src: *const c_char, dst: *const c_char) -> ::c_int; pub fn lseek(fd: ::c_int, offset: off_t, whence: ::c_int) -> off_t; pub fn pathconf(path: *const c_char, name: ::c_int) -> c_long; - pub fn pause() -> ::c_int; pub fn rmdir(path: *const c_char) -> ::c_int; pub fn sleep(secs: ::c_uint) -> ::c_uint; pub fn unlink(c: *const c_char) -> ::c_int; - pub fn pread( - fd: ::c_int, - buf: *mut ::c_void, - count: ::size_t, - offset: off_t, - ) -> ::ssize_t; - pub fn pwrite( - fd: ::c_int, - buf: *const ::c_void, - count: ::size_t, - offset: off_t, - ) -> ::ssize_t; + pub fn pread(fd: ::c_int, buf: *mut ::c_void, count: ::size_t, offset: off_t) -> ::ssize_t; + pub fn pwrite(fd: ::c_int, buf: *const ::c_void, count: ::size_t, offset: off_t) -> ::ssize_t; pub fn lstat(path: *const c_char, buf: *mut stat) -> ::c_int; @@ -607,60 +667,29 @@ extern "C" { pub fn gettimeofday(tp: *mut ::timeval, tz: *mut ::c_void) -> ::c_int; pub fn times(buf: *mut ::tms) -> ::clock_t; - pub fn strerror_r( - errnum: ::c_int, - buf: *mut c_char, - buflen: ::size_t, - ) -> ::c_int; + pub fn strerror_r(errnum: ::c_int, buf: *mut c_char, buflen: ::size_t) -> ::c_int; pub fn usleep(secs: ::c_uint) -> ::c_int; - pub fn send( - socket: ::c_int, - buf: *const ::c_void, - len: ::size_t, - flags: ::c_int, - ) -> ::ssize_t; - pub fn recv( - socket: ::c_int, - buf: *mut ::c_void, - len: ::size_t, - flags: ::c_int, - ) -> ::ssize_t; + pub fn send(socket: ::c_int, buf: *const ::c_void, len: ::size_t, flags: ::c_int) -> ::ssize_t; + pub fn recv(socket: ::c_int, buf: *mut ::c_void, len: ::size_t, flags: ::c_int) -> ::ssize_t; pub fn poll(fds: *mut pollfd, nfds: nfds_t, timeout: ::c_int) -> ::c_int; - pub fn setlocale( - category: ::c_int, - locale: *const ::c_char, - ) -> *mut ::c_char; + pub fn setlocale(category: ::c_int, locale: *const ::c_char) -> *mut ::c_char; pub fn localeconv() -> *mut lconv; - pub fn readlink( - path: *const c_char, - buf: *mut c_char, - bufsz: ::size_t, - ) -> ::ssize_t; + pub fn readlink(path: *const c_char, buf: *mut c_char, bufsz: ::size_t) -> ::ssize_t; pub fn timegm(tm: *mut ::tm) -> time_t; pub fn sysconf(name: ::c_int) -> ::c_long; - pub fn fseeko( - stream: *mut ::FILE, - offset: ::off_t, - whence: ::c_int, - ) -> ::c_int; + pub fn ioctl(fd: ::c_int, request: ::c_int, ...) -> ::c_int; + + pub fn fseeko(stream: *mut ::FILE, offset: ::off_t, whence: ::c_int) -> ::c_int; pub fn ftello(stream: *mut ::FILE) -> ::off_t; - pub fn posix_fallocate( - fd: ::c_int, - offset: ::off_t, - len: ::off_t, - ) -> ::c_int; + pub fn posix_fallocate(fd: ::c_int, offset: ::off_t, len: ::off_t) -> ::c_int; pub fn strcasestr(cs: *const c_char, ct: *const c_char) -> *mut c_char; - pub fn getline( - lineptr: *mut *mut c_char, - n: *mut size_t, - stream: *mut FILE, - ) -> ssize_t; + pub fn getline(lineptr: *mut *mut c_char, n: *mut size_t, stream: *mut FILE) -> ssize_t; pub fn faccessat( dirfd: ::c_int, @@ -668,34 +697,12 @@ extern "C" { mode: ::c_int, flags: ::c_int, ) -> ::c_int; - pub fn writev( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - ) -> ::ssize_t; - pub fn readv( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - ) -> ::ssize_t; - pub fn pwritev( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - offset: ::off_t, - ) -> ::ssize_t; - pub fn preadv( - fd: ::c_int, - iov: *const ::iovec, - iovcnt: ::c_int, - offset: ::off_t, - ) -> ::ssize_t; - pub fn posix_fadvise( - fd: ::c_int, - offset: ::off_t, - len: ::off_t, - advise: ::c_int, - ) -> ::c_int; + pub fn writev(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int) -> ::ssize_t; + pub fn readv(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int) -> ::ssize_t; + pub fn pwritev(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int, offset: ::off_t) + -> ::ssize_t; + pub fn preadv(fd: ::c_int, iov: *const ::iovec, iovcnt: ::c_int, offset: ::off_t) -> ::ssize_t; + pub fn posix_fadvise(fd: ::c_int, offset: ::off_t, len: ::off_t, advise: ::c_int) -> ::c_int; pub fn futimens(fd: ::c_int, times: *const ::timespec) -> ::c_int; pub fn utimensat( dirfd: ::c_int, @@ -704,35 +711,109 @@ extern "C" { flag: ::c_int, ) -> ::c_int; pub fn getentropy(buf: *mut ::c_void, buflen: ::size_t) -> ::c_int; - pub fn memrchr( - cx: *const ::c_void, - c: ::c_int, - n: ::size_t, - ) -> *mut ::c_void; + pub fn memrchr(cx: *const ::c_void, c: ::c_int, n: ::size_t) -> *mut ::c_void; pub fn abs(i: c_int) -> c_int; pub fn labs(i: c_long) -> c_long; pub fn duplocale(base: ::locale_t) -> ::locale_t; pub fn freelocale(loc: ::locale_t); - pub fn newlocale( - mask: ::c_int, - locale: *const ::c_char, - base: ::locale_t, - ) -> ::locale_t; + pub fn newlocale(mask: ::c_int, locale: *const ::c_char, base: ::locale_t) -> ::locale_t; pub fn uselocale(loc: ::locale_t) -> ::locale_t; pub fn sched_yield() -> ::c_int; + pub fn getcwd(buf: *mut c_char, size: ::size_t) -> *mut c_char; + pub fn chdir(dir: *const c_char) -> ::c_int; - pub fn __wasilibc_register_preopened_fd( - fd: c_int, - path: *const c_char, - ) -> c_int; + pub fn nl_langinfo(item: ::nl_item) -> *mut ::c_char; + pub fn nl_langinfo_l(item: ::nl_item, loc: ::locale_t) -> *mut ::c_char; + + pub fn __wasilibc_register_preopened_fd(fd: c_int, path: *const c_char) -> c_int; pub fn __wasilibc_fd_renumber(fd: c_int, newfd: c_int) -> c_int; pub fn __wasilibc_unlinkat(fd: c_int, path: *const c_char) -> c_int; pub fn __wasilibc_rmdirat(fd: c_int, path: *const c_char) -> c_int; pub fn __wasilibc_find_relpath( path: *const c_char, - relative_path: *mut *const c_char, + abs_prefix: *mut *const c_char, + relative_path: *mut *mut c_char, + relative_path_len: usize, ) -> c_int; pub fn __wasilibc_tell(fd: c_int) -> ::off_t; + pub fn __wasilibc_nocwd___wasilibc_unlinkat(dirfd: c_int, path: *const c_char) -> c_int; + pub fn __wasilibc_nocwd___wasilibc_rmdirat(dirfd: c_int, path: *const c_char) -> c_int; + pub fn __wasilibc_nocwd_linkat( + olddirfd: c_int, + oldpath: *const c_char, + newdirfd: c_int, + newpath: *const c_char, + flags: c_int, + ) -> c_int; + pub fn __wasilibc_nocwd_symlinkat( + target: *const c_char, + dirfd: c_int, + path: *const c_char, + ) -> c_int; + pub fn __wasilibc_nocwd_readlinkat( + dirfd: c_int, + path: *const c_char, + buf: *mut c_char, + bufsize: usize, + ) -> isize; + pub fn __wasilibc_nocwd_faccessat( + dirfd: c_int, + path: *const c_char, + mode: c_int, + flags: c_int, + ) -> c_int; + pub fn __wasilibc_nocwd_renameat( + olddirfd: c_int, + oldpath: *const c_char, + newdirfd: c_int, + newpath: *const c_char, + ) -> c_int; + pub fn __wasilibc_nocwd_openat_nomode(dirfd: c_int, path: *const c_char, flags: c_int) + -> c_int; + pub fn __wasilibc_nocwd_fstatat( + dirfd: c_int, + path: *const c_char, + buf: *mut stat, + flags: c_int, + ) -> c_int; + pub fn __wasilibc_nocwd_mkdirat_nomode(dirfd: c_int, path: *const c_char) -> c_int; + pub fn __wasilibc_nocwd_utimensat( + dirfd: c_int, + path: *const c_char, + times: *const ::timespec, + flags: c_int, + ) -> c_int; + pub fn __wasilibc_nocwd_opendirat(dirfd: c_int, path: *const c_char) -> *mut ::DIR; + pub fn __wasilibc_access(pathname: *const c_char, mode: c_int, flags: c_int) -> c_int; + pub fn __wasilibc_stat(pathname: *const c_char, buf: *mut stat, flags: c_int) -> c_int; + pub fn __wasilibc_utimens( + pathname: *const c_char, + times: *const ::timespec, + flags: c_int, + ) -> c_int; + pub fn __wasilibc_link(oldpath: *const c_char, newpath: *const c_char, flags: c_int) -> c_int; + pub fn __wasilibc_link_oldat( + olddirfd: c_int, + oldpath: *const c_char, + newpath: *const c_char, + flags: c_int, + ) -> c_int; + pub fn __wasilibc_link_newat( + oldpath: *const c_char, + newdirfd: c_int, + newpath: *const c_char, + flags: c_int, + ) -> c_int; + pub fn __wasilibc_rename_oldat( + olddirfd: c_int, + oldpath: *const c_char, + newpath: *const c_char, + ) -> c_int; + pub fn __wasilibc_rename_newat( + oldpath: *const c_char, + newdirfd: c_int, + newpath: *const c_char, + ) -> c_int; pub fn arc4random() -> u32; pub fn arc4random_buf(a: *mut c_void, b: size_t); diff --git a/crux-mir/lib/libc/src/windows/gnu/mod.rs b/crux-mir/lib/libc/src/windows/gnu/mod.rs index e74628b98..3e7d38b8e 100644 --- a/crux-mir/lib/libc/src/windows/gnu/mod.rs +++ b/crux-mir/lib/libc/src/windows/gnu/mod.rs @@ -8,11 +8,11 @@ pub const STDERR_FILENO: ::c_int = 2; extern "C" { pub fn strcasecmp(s1: *const ::c_char, s2: *const ::c_char) -> ::c_int; - pub fn strncasecmp( - s1: *const ::c_char, - s2: *const ::c_char, - n: ::size_t, - ) -> ::c_int; + pub fn strncasecmp(s1: *const ::c_char, s2: *const ::c_char, n: ::size_t) -> ::c_int; + + // NOTE: For MSVC target, `wmemchr` is only a inline function in `` + // header file. We cannot find a way to link to that symbol from Rust. + pub fn wmemchr(cx: *const ::wchar_t, c: ::wchar_t, n: ::size_t) -> *mut ::wchar_t; } cfg_if! { diff --git a/crux-mir/lib/libc/src/windows/mod.rs b/crux-mir/lib/libc/src/windows/mod.rs index fcbe0bf78..916019b1f 100644 --- a/crux-mir/lib/libc/src/windows/mod.rs +++ b/crux-mir/lib/libc/src/windows/mod.rs @@ -27,6 +27,8 @@ pub type wchar_t = u16; pub type clock_t = i32; +pub type errno_t = ::c_int; + cfg_if! { if #[cfg(all(target_arch = "x86", target_env = "gnu"))] { pub type time_t = i32; @@ -117,16 +119,27 @@ pub const BUFSIZ: ::c_uint = 512; pub const FOPEN_MAX: ::c_uint = 20; pub const FILENAME_MAX: ::c_uint = 260; -pub const O_RDONLY: ::c_int = 0; -pub const O_WRONLY: ::c_int = 1; -pub const O_RDWR: ::c_int = 2; -pub const O_APPEND: ::c_int = 8; -pub const O_CREAT: ::c_int = 256; -pub const O_EXCL: ::c_int = 1024; -pub const O_TEXT: ::c_int = 16384; -pub const O_BINARY: ::c_int = 32768; -pub const O_NOINHERIT: ::c_int = 128; -pub const O_TRUNC: ::c_int = 512; +// fcntl.h +pub const O_RDONLY: ::c_int = 0x0000; +pub const O_WRONLY: ::c_int = 0x0001; +pub const O_RDWR: ::c_int = 0x0002; +pub const O_APPEND: ::c_int = 0x0008; +pub const O_CREAT: ::c_int = 0x0100; +pub const O_TRUNC: ::c_int = 0x0200; +pub const O_EXCL: ::c_int = 0x0400; +pub const O_TEXT: ::c_int = 0x4000; +pub const O_BINARY: ::c_int = 0x8000; +pub const _O_WTEXT: ::c_int = 0x10000; +pub const _O_U16TEXT: ::c_int = 0x20000; +pub const _O_U8TEXT: ::c_int = 0x40000; +pub const O_RAW: ::c_int = O_BINARY; +pub const O_NOINHERIT: ::c_int = 0x0080; +pub const O_TEMPORARY: ::c_int = 0x0040; +pub const _O_SHORT_LIVED: ::c_int = 0x1000; +pub const _O_OBTAIN_DIR: ::c_int = 0x2000; +pub const O_SEQUENTIAL: ::c_int = 0x0020; +pub const O_RANDOM: ::c_int = 0x0010; + pub const S_IFCHR: ::c_int = 8192; pub const S_IFDIR: ::c_int = 16384; pub const S_IFREG: ::c_int = 32768; @@ -233,7 +246,13 @@ pub const SIGSEGV: ::c_int = 11; pub const SIGTERM: ::c_int = 15; pub const SIGABRT: ::c_int = 22; pub const NSIG: ::c_int = 23; + pub const SIG_ERR: ::c_int = -1; +pub const SIG_DFL: ::sighandler_t = 0; +pub const SIG_IGN: ::sighandler_t = 1; +pub const SIG_GET: ::sighandler_t = 2; +pub const SIG_SGE: ::sighandler_t = 3; +pub const SIG_ACK: ::sighandler_t = 4; // inline comment below appeases style checker #[cfg(all(target_env = "msvc", feature = "rustc-dep-of-std"))] // " if " @@ -258,6 +277,16 @@ impl ::Clone for fpos_t { } } +// Special handling for all print and scan type functions because of https://github.com/rust-lang/libc/issues/2860 +#[cfg_attr( + all(windows, target_env = "msvc"), + link(name = "legacy_stdio_definitions") +)] +extern "C" { + pub fn printf(format: *const c_char, ...) -> ::c_int; + pub fn fprintf(stream: *mut FILE, format: *const c_char, ...) -> ::c_int; +} + extern "C" { pub fn isalnum(c: c_int) -> c_int; pub fn isalpha(c: c_int) -> c_int; @@ -274,44 +303,24 @@ extern "C" { pub fn tolower(c: c_int) -> c_int; pub fn toupper(c: c_int) -> c_int; pub fn fopen(filename: *const c_char, mode: *const c_char) -> *mut FILE; - pub fn freopen( - filename: *const c_char, - mode: *const c_char, - file: *mut FILE, - ) -> *mut FILE; + pub fn freopen(filename: *const c_char, mode: *const c_char, file: *mut FILE) -> *mut FILE; pub fn fflush(file: *mut FILE) -> c_int; pub fn fclose(file: *mut FILE) -> c_int; pub fn remove(filename: *const c_char) -> c_int; pub fn rename(oldname: *const c_char, newname: *const c_char) -> c_int; pub fn tmpfile() -> *mut FILE; - pub fn setvbuf( - stream: *mut FILE, - buffer: *mut c_char, - mode: c_int, - size: size_t, - ) -> c_int; + pub fn setvbuf(stream: *mut FILE, buffer: *mut c_char, mode: c_int, size: size_t) -> c_int; pub fn setbuf(stream: *mut FILE, buf: *mut c_char); pub fn getchar() -> c_int; pub fn putchar(c: c_int) -> c_int; pub fn fgetc(stream: *mut FILE) -> c_int; - pub fn fgets(buf: *mut c_char, n: c_int, stream: *mut FILE) - -> *mut c_char; + pub fn fgets(buf: *mut c_char, n: c_int, stream: *mut FILE) -> *mut c_char; pub fn fputc(c: c_int, stream: *mut FILE) -> c_int; pub fn fputs(s: *const c_char, stream: *mut FILE) -> c_int; pub fn puts(s: *const c_char) -> c_int; pub fn ungetc(c: c_int, stream: *mut FILE) -> c_int; - pub fn fread( - ptr: *mut c_void, - size: size_t, - nobj: size_t, - stream: *mut FILE, - ) -> size_t; - pub fn fwrite( - ptr: *const c_void, - size: size_t, - nobj: size_t, - stream: *mut FILE, - ) -> size_t; + pub fn fread(ptr: *mut c_void, size: size_t, nobj: size_t, stream: *mut FILE) -> size_t; + pub fn fwrite(ptr: *const c_void, size: size_t, nobj: size_t, stream: *mut FILE) -> size_t; pub fn fseek(stream: *mut FILE, offset: c_long, whence: c_int) -> c_int; pub fn ftell(stream: *mut FILE) -> c_long; pub fn rewind(stream: *mut FILE); @@ -322,16 +331,9 @@ extern "C" { pub fn perror(s: *const c_char); pub fn atoi(s: *const c_char) -> c_int; pub fn strtod(s: *const c_char, endp: *mut *mut c_char) -> c_double; - pub fn strtol( - s: *const c_char, - endp: *mut *mut c_char, - base: c_int, - ) -> c_long; - pub fn strtoul( - s: *const c_char, - endp: *mut *mut c_char, - base: c_int, - ) -> c_ulong; + pub fn strtof(s: *const c_char, endp: *mut *mut c_char) -> c_float; + pub fn strtol(s: *const c_char, endp: *mut *mut c_char, base: c_int) -> c_long; + pub fn strtoul(s: *const c_char, endp: *mut *mut c_char, base: c_int) -> c_ulong; pub fn calloc(nobj: size_t, size: size_t) -> *mut c_void; pub fn malloc(size: size_t) -> *mut c_void; pub fn realloc(p: *mut c_void, size: size_t) -> *mut c_void; @@ -344,17 +346,9 @@ extern "C" { pub fn getenv(s: *const c_char) -> *mut c_char; pub fn strcpy(dst: *mut c_char, src: *const c_char) -> *mut c_char; - pub fn strncpy( - dst: *mut c_char, - src: *const c_char, - n: size_t, - ) -> *mut c_char; + pub fn strncpy(dst: *mut c_char, src: *const c_char, n: size_t) -> *mut c_char; pub fn strcat(s: *mut c_char, ct: *const c_char) -> *mut c_char; - pub fn strncat( - s: *mut c_char, - ct: *const c_char, - n: size_t, - ) -> *mut c_char; + pub fn strncat(s: *mut c_char, ct: *const c_char, n: size_t) -> *mut c_char; pub fn strcmp(cs: *const c_char, ct: *const c_char) -> c_int; pub fn strncmp(cs: *const c_char, ct: *const c_char, n: size_t) -> c_int; pub fn strcoll(cs: *const c_char, ct: *const c_char) -> c_int; @@ -371,25 +365,12 @@ extern "C" { pub fn strtok(s: *mut c_char, t: *const c_char) -> *mut c_char; pub fn strxfrm(s: *mut c_char, ct: *const c_char, n: size_t) -> size_t; pub fn wcslen(buf: *const wchar_t) -> size_t; - pub fn wcstombs( - dest: *mut c_char, - src: *const wchar_t, - n: size_t, - ) -> ::size_t; + pub fn wcstombs(dest: *mut c_char, src: *const wchar_t, n: size_t) -> ::size_t; pub fn memchr(cx: *const c_void, c: c_int, n: size_t) -> *mut c_void; - pub fn wmemchr(cx: *const wchar_t, c: wchar_t, n: size_t) -> *mut wchar_t; pub fn memcmp(cx: *const c_void, ct: *const c_void, n: size_t) -> c_int; - pub fn memcpy( - dest: *mut c_void, - src: *const c_void, - n: size_t, - ) -> *mut c_void; - pub fn memmove( - dest: *mut c_void, - src: *const c_void, - n: size_t, - ) -> *mut c_void; + pub fn memcpy(dest: *mut c_void, src: *const c_void, n: size_t) -> *mut c_void; + pub fn memmove(dest: *mut c_void, src: *const c_void, n: size_t) -> *mut c_void; pub fn memset(dest: *mut c_void, c: c_int, n: size_t) -> *mut c_void; pub fn abs(i: c_int) -> c_int; @@ -403,6 +384,8 @@ extern "C" { #[link_name = "_gmtime64_s"] pub fn gmtime_s(destTime: *mut tm, srcTime: *const time_t) -> ::c_int; + #[link_name = "_localtime64_s"] + pub fn localtime_s(tmDest: *mut tm, sourceTime: *const time_t) -> ::errno_t; #[link_name = "_time64"] pub fn time(destTime: *mut time_t) -> time_t; #[link_name = "_chmod"] @@ -445,11 +428,24 @@ extern "C" { pub fn dup(fd: ::c_int) -> ::c_int; #[link_name = "_dup2"] pub fn dup2(src: ::c_int, dst: ::c_int) -> ::c_int; + #[link_name = "_execl"] + pub fn execl(path: *const c_char, arg0: *const c_char, ...) -> intptr_t; + #[link_name = "_wexecl"] + pub fn wexecl(path: *const wchar_t, arg0: *const wchar_t, ...) -> intptr_t; + #[link_name = "_execle"] + pub fn execle(path: *const c_char, arg0: *const c_char, ...) -> intptr_t; + #[link_name = "_wexecle"] + pub fn wexecle(path: *const wchar_t, arg0: *const wchar_t, ...) -> intptr_t; + #[link_name = "_execlp"] + pub fn execlp(path: *const c_char, arg0: *const c_char, ...) -> intptr_t; + #[link_name = "_wexeclp"] + pub fn wexeclp(path: *const wchar_t, arg0: *const wchar_t, ...) -> intptr_t; + #[link_name = "_execlpe"] + pub fn execlpe(path: *const c_char, arg0: *const c_char, ...) -> intptr_t; + #[link_name = "_wexeclpe"] + pub fn wexeclpe(path: *const wchar_t, arg0: *const wchar_t, ...) -> intptr_t; #[link_name = "_execv"] - pub fn execv( - prog: *const c_char, - argv: *const *const c_char, - ) -> ::intptr_t; + pub fn execv(prog: *const c_char, argv: *const *const c_char) -> ::intptr_t; #[link_name = "_execve"] pub fn execve( prog: *const c_char, @@ -464,6 +460,22 @@ extern "C" { argv: *const *const c_char, envp: *const *const c_char, ) -> ::c_int; + #[link_name = "_wexecv"] + pub fn wexecv(prog: *const wchar_t, argv: *const *const wchar_t) -> ::intptr_t; + #[link_name = "_wexecve"] + pub fn wexecve( + prog: *const wchar_t, + argv: *const *const wchar_t, + envp: *const *const wchar_t, + ) -> ::intptr_t; + #[link_name = "_wexecvp"] + pub fn wexecvp(c: *const wchar_t, argv: *const *const wchar_t) -> ::intptr_t; + #[link_name = "_wexecvpe"] + pub fn wexecvpe( + c: *const wchar_t, + argv: *const *const wchar_t, + envp: *const *const wchar_t, + ) -> ::intptr_t; #[link_name = "_getcwd"] pub fn getcwd(buf: *mut c_char, size: ::c_int) -> *mut c_char; #[link_name = "_getpid"] @@ -473,17 +485,9 @@ extern "C" { #[link_name = "_lseek"] pub fn lseek(fd: ::c_int, offset: c_long, origin: ::c_int) -> c_long; #[link_name = "_lseeki64"] - pub fn lseek64( - fd: ::c_int, - offset: c_longlong, - origin: ::c_int, - ) -> c_longlong; + pub fn lseek64(fd: ::c_int, offset: c_longlong, origin: ::c_int) -> c_longlong; #[link_name = "_pipe"] - pub fn pipe( - fds: *mut ::c_int, - psize: ::c_uint, - textmode: ::c_int, - ) -> ::c_int; + pub fn pipe(fds: *mut ::c_int, psize: ::c_uint, textmode: ::c_int) -> ::c_int; #[link_name = "_read"] pub fn read(fd: ::c_int, buf: *mut ::c_void, count: ::c_uint) -> ::c_int; #[link_name = "_rmdir"] @@ -491,11 +495,7 @@ extern "C" { #[link_name = "_unlink"] pub fn unlink(c: *const c_char) -> ::c_int; #[link_name = "_write"] - pub fn write( - fd: ::c_int, - buf: *const ::c_void, - count: ::c_uint, - ) -> ::c_int; + pub fn write(fd: ::c_int, buf: *const ::c_void, count: ::c_uint) -> ::c_int; #[link_name = "_commit"] pub fn commit(fd: ::c_int) -> ::c_int; #[link_name = "_get_osfhandle"] @@ -504,39 +504,28 @@ extern "C" { pub fn open_osfhandle(osfhandle: ::intptr_t, flags: ::c_int) -> ::c_int; pub fn setlocale(category: ::c_int, locale: *const c_char) -> *mut c_char; #[link_name = "_wsetlocale"] - pub fn wsetlocale( - category: ::c_int, - locale: *const wchar_t, - ) -> *mut wchar_t; + pub fn wsetlocale(category: ::c_int, locale: *const wchar_t) -> *mut wchar_t; + #[link_name = "_aligned_malloc"] + pub fn aligned_malloc(size: size_t, alignment: size_t) -> *mut c_void; + #[link_name = "_aligned_free"] + pub fn aligned_free(ptr: *mut ::c_void); + #[link_name = "_putenv"] + pub fn putenv(envstring: *const ::c_char) -> ::c_int; + #[link_name = "_wputenv"] + pub fn wputenv(envstring: *const ::wchar_t) -> ::c_int; + #[link_name = "_putenv_s"] + pub fn putenv_s(envstring: *const ::c_char, value_string: *const ::c_char) -> ::errno_t; + #[link_name = "_wputenv_s"] + pub fn wputenv_s(envstring: *const ::wchar_t, value_string: *const ::wchar_t) -> ::errno_t; } extern "system" { pub fn listen(s: SOCKET, backlog: ::c_int) -> ::c_int; - pub fn accept( - s: SOCKET, - addr: *mut ::sockaddr, - addrlen: *mut ::c_int, - ) -> SOCKET; - pub fn bind( - s: SOCKET, - name: *const ::sockaddr, - namelen: ::c_int, - ) -> ::c_int; - pub fn connect( - s: SOCKET, - name: *const ::sockaddr, - namelen: ::c_int, - ) -> ::c_int; - pub fn getpeername( - s: SOCKET, - name: *mut ::sockaddr, - nameln: *mut ::c_int, - ) -> ::c_int; - pub fn getsockname( - s: SOCKET, - name: *mut ::sockaddr, - nameln: *mut ::c_int, - ) -> ::c_int; + pub fn accept(s: SOCKET, addr: *mut ::sockaddr, addrlen: *mut ::c_int) -> SOCKET; + pub fn bind(s: SOCKET, name: *const ::sockaddr, namelen: ::c_int) -> ::c_int; + pub fn connect(s: SOCKET, name: *const ::sockaddr, namelen: ::c_int) -> ::c_int; + pub fn getpeername(s: SOCKET, name: *mut ::sockaddr, nameln: *mut ::c_int) -> ::c_int; + pub fn getsockname(s: SOCKET, name: *mut ::sockaddr, nameln: *mut ::c_int) -> ::c_int; pub fn getsockopt( s: SOCKET, level: ::c_int, @@ -567,11 +556,7 @@ extern "system" { optval: *const ::c_char, optlen: ::c_int, ) -> ::c_int; - pub fn socket( - af: ::c_int, - socket_type: ::c_int, - protocol: ::c_int, - ) -> SOCKET; + pub fn socket(af: ::c_int, socket_type: ::c_int, protocol: ::c_int) -> SOCKET; } cfg_if! { diff --git a/crux-mir/lib/libc/src/windows/msvc.rs b/crux-mir/lib/libc/src/windows/msvc/mod.rs similarity index 58% rename from crux-mir/lib/libc/src/windows/msvc.rs rename to crux-mir/lib/libc/src/windows/msvc/mod.rs index 8f20deb5d..f5a1d95f3 100644 --- a/crux-mir/lib/libc/src/windows/msvc.rs +++ b/crux-mir/lib/libc/src/windows/msvc/mod.rs @@ -9,9 +9,12 @@ extern "C" { #[link_name = "_stricmp"] pub fn stricmp(s1: *const ::c_char, s2: *const ::c_char) -> ::c_int; #[link_name = "_strnicmp"] - pub fn strnicmp( - s1: *const ::c_char, - s2: *const ::c_char, - n: ::size_t, - ) -> ::c_int; + pub fn strnicmp(s1: *const ::c_char, s2: *const ::c_char, n: ::size_t) -> ::c_int; + #[link_name = "_memccpy"] + pub fn memccpy( + dest: *mut ::c_void, + src: *const ::c_void, + c: ::c_int, + count: ::size_t, + ) -> *mut ::c_void; } diff --git a/crux-mir/lib/memchr/.cargo-ok b/crux-mir/lib/memchr/.cargo-ok new file mode 100644 index 000000000..b5754e203 --- /dev/null +++ b/crux-mir/lib/memchr/.cargo-ok @@ -0,0 +1 @@ +ok \ No newline at end of file diff --git a/crux-mir/lib/memchr/.cargo_vcs_info.json b/crux-mir/lib/memchr/.cargo_vcs_info.json new file mode 100644 index 000000000..03d5f8609 --- /dev/null +++ b/crux-mir/lib/memchr/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "c55597192ead1896c5c7cad2632f01d57ab41da0" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/crux-mir/lib/memchr/.gitignore b/crux-mir/lib/memchr/.gitignore new file mode 100644 index 000000000..ab067f298 --- /dev/null +++ b/crux-mir/lib/memchr/.gitignore @@ -0,0 +1,10 @@ +.*.swp +doc +tags +examples/ss10pusa.csv +build +target +/Cargo.lock +scratch* +bench_large/huge +tmp/ diff --git a/crux-mir/lib/memchr/.ignore b/crux-mir/lib/memchr/.ignore new file mode 100644 index 000000000..47ec4742e --- /dev/null +++ b/crux-mir/lib/memchr/.ignore @@ -0,0 +1 @@ +!.github diff --git a/crux-mir/lib/memchr/COPYING b/crux-mir/lib/memchr/COPYING new file mode 100644 index 000000000..bb9c20a09 --- /dev/null +++ b/crux-mir/lib/memchr/COPYING @@ -0,0 +1,3 @@ +This project is dual-licensed under the Unlicense and MIT licenses. + +You may use this code under the terms of either license. diff --git a/crux-mir/lib/memchr/Cargo.toml b/crux-mir/lib/memchr/Cargo.toml new file mode 100644 index 000000000..630195281 --- /dev/null +++ b/crux-mir/lib/memchr/Cargo.toml @@ -0,0 +1,78 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2018" +name = "memchr" +version = "2.5.0" +authors = [ + "Andrew Gallant ", + "bluss", +] +exclude = [ + "/bench", + "/.github", + "/fuzz", +] +description = "Safe interface to memchr." +homepage = "https://github.com/BurntSushi/memchr" +documentation = "https://docs.rs/memchr/" +readme = "README.md" +keywords = [ + "memchr", + "char", + "scan", + "strchr", + "string", +] +license = "Unlicense/MIT" +repository = "https://github.com/BurntSushi/memchr" + +[profile.bench] +debug = true + +[profile.release] +debug = true + +[profile.test] +opt-level = 3 +debug = true + +[lib] +name = "memchr" +bench = false + +[dependencies.compiler_builtins] +version = "0.1.2" +optional = true + +[dependencies.core] +version = "1.0.0" +optional = true +package = "rustc-std-workspace-core" + +[dependencies.libc] +version = "0.2.18" +optional = true +default-features = false + +[dev-dependencies.quickcheck] +version = "1.0.3" +default-features = false + +[features] +default = ["std"] +rustc-dep-of-std = [ + "core", + "compiler_builtins", +] +std = [] +use_std = ["std"] diff --git a/crux-mir/lib/memchr/Cargo.toml.orig b/crux-mir/lib/memchr/Cargo.toml.orig new file mode 100644 index 000000000..03f126c89 --- /dev/null +++ b/crux-mir/lib/memchr/Cargo.toml.orig @@ -0,0 +1,57 @@ +[package] +name = "memchr" +version = "2.5.0" #:version +authors = ["Andrew Gallant ", "bluss"] +description = "Safe interface to memchr." +documentation = "https://docs.rs/memchr/" +homepage = "https://github.com/BurntSushi/memchr" +repository = "https://github.com/BurntSushi/memchr" +readme = "README.md" +keywords = ["memchr", "char", "scan", "strchr", "string"] +license = "Unlicense/MIT" +exclude = ["/bench", "/.github", "/fuzz"] +edition = "2018" + +[workspace] +members = ["bench"] + +[lib] +name = "memchr" +bench = false + +[features] +default = ["std"] + +# The 'std' feature permits the memchr crate to use the standard library. This +# permits this crate to use runtime CPU feature detection to automatically +# accelerate searching via vector instructions. Without the standard library, +# this automatic detection is not possible. +std = [] +# The 'use_std' feature is DEPRECATED. It will be removed in memchr 3. Until +# then, it is alias for the 'std' feature. +use_std = ["std"] + +# Internal feature, only used when building as part of libstd, not part of the +# stable interface of this crate. +rustc-dep-of-std = ['core', 'compiler_builtins'] + +[dependencies] +libc = { version = "0.2.18", default-features = false, optional = true } + +# Internal feature, only used when building as part of libstd, not part of the +# stable interface of this crate. +core = { version = '1.0.0', optional = true, package = 'rustc-std-workspace-core' } +compiler_builtins = { version = '0.1.2', optional = true } + +[dev-dependencies] +quickcheck = { version = "1.0.3", default-features = false } + +[profile.release] +debug = true + +[profile.bench] +debug = true + +[profile.test] +opt-level = 3 +debug = true diff --git a/crux-mir/lib/byteorder/LICENSE-MIT b/crux-mir/lib/memchr/LICENSE-MIT similarity index 100% rename from crux-mir/lib/byteorder/LICENSE-MIT rename to crux-mir/lib/memchr/LICENSE-MIT diff --git a/crux-mir/lib/memchr/README.md b/crux-mir/lib/memchr/README.md new file mode 100644 index 000000000..77a7a0f5b --- /dev/null +++ b/crux-mir/lib/memchr/README.md @@ -0,0 +1,107 @@ +memchr +====== +This library provides heavily optimized routines for string search primitives. + +[![Build status](https://github.com/BurntSushi/memchr/workflows/ci/badge.svg)](https://github.com/BurntSushi/memchr/actions) +[![Crates.io](https://img.shields.io/crates/v/memchr.svg)](https://crates.io/crates/memchr) + +Dual-licensed under MIT or the [UNLICENSE](https://unlicense.org/). + + +### Documentation + +[https://docs.rs/memchr](https://docs.rs/memchr) + + +### Overview + +* The top-level module provides routines for searching for 1, 2 or 3 bytes + in the forward or reverse direction. When searching for more than one byte, + positions are considered a match if the byte at that position matches any + of the bytes. +* The `memmem` sub-module provides forward and reverse substring search + routines. + +In all such cases, routines operate on `&[u8]` without regard to encoding. This +is exactly what you want when searching either UTF-8 or arbitrary bytes. + +### Compiling without the standard library + +memchr links to the standard library by default, but you can disable the +`std` feature if you want to use it in a `#![no_std]` crate: + +```toml +[dependencies] +memchr = { version = "2", default-features = false } +``` + +On x86 platforms, when the `std` feature is disabled, the SSE2 accelerated +implementations will be used. When `std` is enabled, AVX accelerated +implementations will be used if the CPU is determined to support it at runtime. + +### Using libc + +`memchr` is a routine that is part of libc, although this crate does not use +libc by default. Instead, it uses its own routines, which are either vectorized +or generic fallback routines. In general, these should be competitive with +what's in libc, although this has not been tested for all architectures. If +using `memchr` from libc is desirable and a vectorized routine is not otherwise +available in this crate, then enabling the `libc` feature will use libc's +version of `memchr`. + +The rest of the functions in this crate, e.g., `memchr2` or `memrchr3` and the +substring search routines, will always use the implementations in this crate. +One exception to this is `memrchr`, which is an extension in `libc` found on +Linux. On Linux, `memrchr` is used in precisely the same scenario as `memchr`, +as described above. + + +### Minimum Rust version policy + +This crate's minimum supported `rustc` version is `1.41.1`. + +The current policy is that the minimum Rust version required to use this crate +can be increased in minor version updates. For example, if `crate 1.0` requires +Rust 1.20.0, then `crate 1.0.z` for all values of `z` will also require Rust +1.20.0 or newer. However, `crate 1.y` for `y > 0` may require a newer minimum +version of Rust. + +In general, this crate will be conservative with respect to the minimum +supported version of Rust. + + +### Testing strategy + +Given the complexity of the code in this crate, along with the pervasive use +of `unsafe`, this crate has an extensive testing strategy. It combines multiple +approaches: + +* Hand-written tests. +* Exhaustive-style testing meant to exercise all possible branching and offset + calculations. +* Property based testing through [`quickcheck`](https://github.com/BurntSushi/quickcheck). +* Fuzz testing through [`cargo fuzz`](https://github.com/rust-fuzz/cargo-fuzz). +* A huge suite of benchmarks that are also run as tests. Benchmarks always + confirm that the expected result occurs. + +Improvements to the testing infrastructure are very welcome. + + +### Algorithms used + +At time of writing, this crate's implementation of substring search actually +has a few different algorithms to choose from depending on the situation. + +* For very small haystacks, + [Rabin-Karp](https://en.wikipedia.org/wiki/Rabin%E2%80%93Karp_algorithm) + is used to reduce latency. Rabin-Karp has very small overhead and can often + complete before other searchers have even been constructed. +* For small needles, a variant of the + ["Generic SIMD"](http://0x80.pl/articles/simd-strfind.html#algorithm-1-generic-simd) + algorithm is used. Instead of using the first and last bytes, a heuristic is + used to select bytes based on a background distribution of byte frequencies. +* In all other cases, + [Two-Way](https://en.wikipedia.org/wiki/Two-way_string-matching_algorithm) + is used. If possible, a prefilter based on the "Generic SIMD" algorithm + linked above is used to find candidates quickly. A dynamic heuristic is used + to detect if the prefilter is ineffective, and if so, disables it. diff --git a/crux-mir/lib/memchr/UNLICENSE b/crux-mir/lib/memchr/UNLICENSE new file mode 100644 index 000000000..68a49daad --- /dev/null +++ b/crux-mir/lib/memchr/UNLICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/crux-mir/lib/memchr/build.rs b/crux-mir/lib/memchr/build.rs new file mode 100644 index 000000000..584a60856 --- /dev/null +++ b/crux-mir/lib/memchr/build.rs @@ -0,0 +1,88 @@ +use std::env; + +fn main() { + enable_simd_optimizations(); + enable_libc(); +} + +// This adds various simd cfgs if this compiler and target support it. +// +// This can be disabled with RUSTFLAGS="--cfg memchr_disable_auto_simd", but +// this is generally only intended for testing. +// +// On targets which don't feature SSE2, this is disabled, as LLVM wouln't know +// how to work with SSE2 operands. Enabling SSE4.2 and AVX on SSE2-only targets +// is not a problem. In that case, the fastest option will be chosen at +// runtime. +fn enable_simd_optimizations() { + if is_env_set("CARGO_CFG_MEMCHR_DISABLE_AUTO_SIMD") { + return; + } + let arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); + match &arch[..] { + "x86_64" => { + if !target_has_feature("sse2") { + return; + } + println!("cargo:rustc-cfg=memchr_runtime_simd"); + println!("cargo:rustc-cfg=memchr_runtime_sse2"); + println!("cargo:rustc-cfg=memchr_runtime_sse42"); + println!("cargo:rustc-cfg=memchr_runtime_avx"); + } + "wasm32" | "wasm64" => { + if !target_has_feature("simd128") { + return; + } + println!("cargo:rustc-cfg=memchr_runtime_simd"); + println!("cargo:rustc-cfg=memchr_runtime_wasm128"); + } + _ => {} + } +} + +// This adds a `memchr_libc` cfg if and only if libc can be used, if no other +// better option is available. +// +// This could be performed in the source code, but it's simpler to do it once +// here and consolidate it into one cfg knob. +// +// Basically, we use libc only if its enabled and if we aren't targeting a +// known bad platform. For example, wasm32 doesn't have a libc and the +// performance of memchr on Windows is seemingly worse than the fallback +// implementation. +fn enable_libc() { + const NO_ARCH: &'static [&'static str] = &["wasm32", "windows"]; + const NO_ENV: &'static [&'static str] = &["sgx"]; + + if !is_feature_set("LIBC") { + return; + } + + let arch = match env::var("CARGO_CFG_TARGET_ARCH") { + Err(_) => return, + Ok(arch) => arch, + }; + let env = match env::var("CARGO_CFG_TARGET_ENV") { + Err(_) => return, + Ok(env) => env, + }; + if NO_ARCH.contains(&&*arch) || NO_ENV.contains(&&*env) { + return; + } + + println!("cargo:rustc-cfg=memchr_libc"); +} + +fn is_feature_set(name: &str) -> bool { + is_env_set(&format!("CARGO_FEATURE_{}", name)) +} + +fn is_env_set(name: &str) -> bool { + env::var_os(name).is_some() +} + +fn target_has_feature(feature: &str) -> bool { + env::var("CARGO_CFG_TARGET_FEATURE") + .map(|features| features.contains(feature)) + .unwrap_or(false) +} diff --git a/crux-mir/lib/memchr/rustfmt.toml b/crux-mir/lib/memchr/rustfmt.toml new file mode 100644 index 000000000..aa37a218b --- /dev/null +++ b/crux-mir/lib/memchr/rustfmt.toml @@ -0,0 +1,2 @@ +max_width = 79 +use_small_heuristics = "max" diff --git a/crux-mir/lib/memchr/scripts/make-byte-frequency-table b/crux-mir/lib/memchr/scripts/make-byte-frequency-table new file mode 100755 index 000000000..37eeca7b7 --- /dev/null +++ b/crux-mir/lib/memchr/scripts/make-byte-frequency-table @@ -0,0 +1,74 @@ +#!/usr/bin/env python + +# This does simple normalized frequency analysis on UTF-8 encoded text. The +# result of the analysis is translated to a ranked list, where every byte is +# assigned a rank. This list is written to src/freqs.rs. +# +# Currently, the frequencies are generated from the following corpuses: +# +# * The CIA world fact book +# * The source code of rustc +# * Septuaginta + +from __future__ import absolute_import, division, print_function + +import argparse +from collections import Counter +import sys + +preamble = ''' +// NOTE: The following code was generated by "scripts/frequencies.py", do not +// edit directly +'''.lstrip() + + +def eprint(*args, **kwargs): + kwargs['file'] = sys.stderr + print(*args, **kwargs) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('corpus', metavar='FILE', nargs='+') + args = p.parse_args() + + # Get frequency counts of each byte. + freqs = Counter() + for i in range(0, 256): + freqs[i] = 0 + + eprint('reading entire corpus into memory') + corpus = [] + for fpath in args.corpus: + corpus.append(open(fpath, 'rb').read()) + + eprint('computing byte frequencies') + for c in corpus: + for byte in c: + freqs[byte] += 1.0 / float(len(c)) + + eprint('writing Rust code') + # Get the rank of each byte. A lower rank => lower relative frequency. + rank = [0] * 256 + for i, (byte, _) in enumerate(freqs.most_common()): + # print(byte) + rank[byte] = 255 - i + + # Forcefully set the highest rank possible for bytes that start multi-byte + # UTF-8 sequences. The idea here is that a continuation byte will be more + # discerning in a homogenous haystack. + for byte in range(0xC0, 0xFF + 1): + rank[byte] = 255 + + # Now write Rust. + olines = ['pub const BYTE_FREQUENCIES: [u8; 256] = ['] + for byte in range(256): + olines.append(' %3d, // %r' % (rank[byte], chr(byte))) + olines.append('];') + + print(preamble) + print('\n'.join(olines)) + + +if __name__ == '__main__': + main() diff --git a/crux-mir/lib/memchr/src/cow.rs b/crux-mir/lib/memchr/src/cow.rs new file mode 100644 index 000000000..0b7d0dad0 --- /dev/null +++ b/crux-mir/lib/memchr/src/cow.rs @@ -0,0 +1,97 @@ +use core::ops; + +/// A specialized copy-on-write byte string. +/// +/// The purpose of this type is to permit usage of a "borrowed or owned +/// byte string" in a way that keeps std/no-std compatibility. That is, in +/// no-std mode, this type devolves into a simple &[u8] with no owned variant +/// available. We can't just use a plain Cow because Cow is not in core. +#[derive(Clone, Debug)] +pub struct CowBytes<'a>(Imp<'a>); + +// N.B. We don't use std::borrow::Cow here since we can get away with a +// Box<[u8]> for our use case, which is 1/3 smaller than the Vec that +// a Cow<[u8]> would use. +#[cfg(feature = "std")] +#[derive(Clone, Debug)] +enum Imp<'a> { + Borrowed(&'a [u8]), + Owned(Box<[u8]>), +} + +#[cfg(not(feature = "std"))] +#[derive(Clone, Debug)] +struct Imp<'a>(&'a [u8]); + +impl<'a> ops::Deref for CowBytes<'a> { + type Target = [u8]; + + #[inline(always)] + fn deref(&self) -> &[u8] { + self.as_slice() + } +} + +impl<'a> CowBytes<'a> { + /// Create a new borrowed CowBytes. + #[inline(always)] + pub fn new>(bytes: &'a B) -> CowBytes<'a> { + CowBytes(Imp::new(bytes.as_ref())) + } + + /// Create a new owned CowBytes. + #[cfg(feature = "std")] + #[inline(always)] + pub fn new_owned(bytes: Box<[u8]>) -> CowBytes<'static> { + CowBytes(Imp::Owned(bytes)) + } + + /// Return a borrowed byte string, regardless of whether this is an owned + /// or borrowed byte string internally. + #[inline(always)] + pub fn as_slice(&self) -> &[u8] { + self.0.as_slice() + } + + /// Return an owned version of this copy-on-write byte string. + /// + /// If this is already an owned byte string internally, then this is a + /// no-op. Otherwise, the internal byte string is copied. + #[cfg(feature = "std")] + #[inline(always)] + pub fn into_owned(self) -> CowBytes<'static> { + match self.0 { + Imp::Borrowed(b) => CowBytes::new_owned(Box::from(b)), + Imp::Owned(b) => CowBytes::new_owned(b), + } + } +} + +impl<'a> Imp<'a> { + #[cfg(feature = "std")] + #[inline(always)] + pub fn new(bytes: &'a [u8]) -> Imp<'a> { + Imp::Borrowed(bytes) + } + + #[cfg(not(feature = "std"))] + #[inline(always)] + pub fn new(bytes: &'a [u8]) -> Imp<'a> { + Imp(bytes) + } + + #[cfg(feature = "std")] + #[inline(always)] + pub fn as_slice(&self) -> &[u8] { + match self { + Imp::Owned(ref x) => x, + Imp::Borrowed(x) => x, + } + } + + #[cfg(not(feature = "std"))] + #[inline(always)] + pub fn as_slice(&self) -> &[u8] { + self.0 + } +} diff --git a/crux-mir/lib/memchr/src/lib.rs b/crux-mir/lib/memchr/src/lib.rs new file mode 100644 index 000000000..e0b4ce3fd --- /dev/null +++ b/crux-mir/lib/memchr/src/lib.rs @@ -0,0 +1,181 @@ +/*! +This library provides heavily optimized routines for string search primitives. + +# Overview + +This section gives a brief high level overview of what this crate offers. + +* The top-level module provides routines for searching for 1, 2 or 3 bytes + in the forward or reverse direction. When searching for more than one byte, + positions are considered a match if the byte at that position matches any + of the bytes. +* The [`memmem`] sub-module provides forward and reverse substring search + routines. + +In all such cases, routines operate on `&[u8]` without regard to encoding. This +is exactly what you want when searching either UTF-8 or arbitrary bytes. + +# Example: using `memchr` + +This example shows how to use `memchr` to find the first occurrence of `z` in +a haystack: + +``` +use memchr::memchr; + +let haystack = b"foo bar baz quuz"; +assert_eq!(Some(10), memchr(b'z', haystack)); +``` + +# Example: matching one of three possible bytes + +This examples shows how to use `memrchr3` to find occurrences of `a`, `b` or +`c`, starting at the end of the haystack. + +``` +use memchr::memchr3_iter; + +let haystack = b"xyzaxyzbxyzc"; + +let mut it = memchr3_iter(b'a', b'b', b'c', haystack).rev(); +assert_eq!(Some(11), it.next()); +assert_eq!(Some(7), it.next()); +assert_eq!(Some(3), it.next()); +assert_eq!(None, it.next()); +``` + +# Example: iterating over substring matches + +This example shows how to use the [`memmem`] sub-module to find occurrences of +a substring in a haystack. + +``` +use memchr::memmem; + +let haystack = b"foo bar foo baz foo"; + +let mut it = memmem::find_iter(haystack, "foo"); +assert_eq!(Some(0), it.next()); +assert_eq!(Some(8), it.next()); +assert_eq!(Some(16), it.next()); +assert_eq!(None, it.next()); +``` + +# Example: repeating a search for the same needle + +It may be possible for the overhead of constructing a substring searcher to be +measurable in some workloads. In cases where the same needle is used to search +many haystacks, it is possible to do construction once and thus to avoid it for +subsequent searches. This can be done with a [`memmem::Finder`]: + +``` +use memchr::memmem; + +let finder = memmem::Finder::new("foo"); + +assert_eq!(Some(4), finder.find(b"baz foo quux")); +assert_eq!(None, finder.find(b"quux baz bar")); +``` + +# Why use this crate? + +At first glance, the APIs provided by this crate might seem weird. Why provide +a dedicated routine like `memchr` for something that could be implemented +clearly and trivially in one line: + +``` +fn memchr(needle: u8, haystack: &[u8]) -> Option { + haystack.iter().position(|&b| b == needle) +} +``` + +Or similarly, why does this crate provide substring search routines when Rust's +core library already provides them? + +``` +fn search(haystack: &str, needle: &str) -> Option { + haystack.find(needle) +} +``` + +The primary reason for both of them to exist is performance. When it comes to +performance, at a high level at least, there are two primary ways to look at +it: + +* **Throughput**: For this, think about it as, "given some very large haystack + and a byte that never occurs in that haystack, how long does it take to + search through it and determine that it, in fact, does not occur?" +* **Latency**: For this, think about it as, "given a tiny haystack---just a + few bytes---how long does it take to determine if a byte is in it?" + +The `memchr` routine in this crate has _slightly_ worse latency than the +solution presented above, however, its throughput can easily be over an +order of magnitude faster. This is a good general purpose trade off to make. +You rarely lose, but often gain big. + +**NOTE:** The name `memchr` comes from the corresponding routine in libc. A key +advantage of using this library is that its performance is not tied to its +quality of implementation in the libc you happen to be using, which can vary +greatly from platform to platform. + +But what about substring search? This one is a bit more complicated. The +primary reason for its existence is still indeed performance, but it's also +useful because Rust's core library doesn't actually expose any substring +search routine on arbitrary bytes. The only substring search routine that +exists works exclusively on valid UTF-8. + +So if you have valid UTF-8, is there a reason to use this over the standard +library substring search routine? Yes. This routine is faster on almost every +metric, including latency. The natural question then, is why isn't this +implementation in the standard library, even if only for searching on UTF-8? +The reason is that the implementation details for using SIMD in the standard +library haven't quite been worked out yet. + +**NOTE:** Currently, only `x86_64` targets have highly accelerated +implementations of substring search. For `memchr`, all targets have +somewhat-accelerated implementations, while only `x86_64` targets have highly +accelerated implementations. This limitation is expected to be lifted once the +standard library exposes a platform independent SIMD API. + +# Crate features + +* **std** - When enabled (the default), this will permit this crate to use + features specific to the standard library. Currently, the only thing used + from the standard library is runtime SIMD CPU feature detection. This means + that this feature must be enabled to get AVX accelerated routines. When + `std` is not enabled, this crate will still attempt to use SSE2 accelerated + routines on `x86_64`. +* **libc** - When enabled (**not** the default), this library will use your + platform's libc implementation of `memchr` (and `memrchr` on Linux). This + can be useful on non-`x86_64` targets where the fallback implementation in + this crate is not as good as the one found in your libc. All other routines + (e.g., `memchr[23]` and substring search) unconditionally use the + implementation in this crate. +*/ + +#![deny(missing_docs)] +#![cfg_attr(not(feature = "std"), no_std)] +// It's not worth trying to gate all code on just miri, so turn off relevant +// dead code warnings. +#![cfg_attr(miri, allow(dead_code, unused_macros))] + +// Supporting 8-bit (or others) would be fine. If you need it, please submit a +// bug report at https://github.com/BurntSushi/memchr +#[cfg(not(any( + target_pointer_width = "16", + target_pointer_width = "32", + target_pointer_width = "64" +)))] +compile_error!("memchr currently not supported on non-{16,32,64}"); + +pub use crate::memchr::{ + memchr, memchr2, memchr2_iter, memchr3, memchr3_iter, memchr_iter, + memrchr, memrchr2, memrchr2_iter, memrchr3, memrchr3_iter, memrchr_iter, + Memchr, Memchr2, Memchr3, +}; + +mod cow; +mod memchr; +pub mod memmem; +#[cfg(test)] +mod tests; diff --git a/crux-mir/lib/memchr/src/memchr/c.rs b/crux-mir/lib/memchr/src/memchr/c.rs new file mode 100644 index 000000000..608aabc98 --- /dev/null +++ b/crux-mir/lib/memchr/src/memchr/c.rs @@ -0,0 +1,44 @@ +// This module defines safe wrappers around memchr (POSIX) and memrchr (GNU +// extension). + +#![allow(dead_code)] + +use libc::{c_int, c_void, size_t}; + +pub fn memchr(needle: u8, haystack: &[u8]) -> Option { + // SAFETY: This is safe to call since all pointers are valid. + let p = unsafe { + libc::memchr( + haystack.as_ptr() as *const c_void, + needle as c_int, + haystack.len() as size_t, + ) + }; + if p.is_null() { + None + } else { + Some(p as usize - (haystack.as_ptr() as usize)) + } +} + +// memrchr is a GNU extension. We know it's available on Linux at least. +#[cfg(target_os = "linux")] +pub fn memrchr(needle: u8, haystack: &[u8]) -> Option { + // GNU's memrchr() will - unlike memchr() - error if haystack is empty. + if haystack.is_empty() { + return None; + } + // SAFETY: This is safe to call since all pointers are valid. + let p = unsafe { + libc::memrchr( + haystack.as_ptr() as *const c_void, + needle as c_int, + haystack.len() as size_t, + ) + }; + if p.is_null() { + None + } else { + Some(p as usize - (haystack.as_ptr() as usize)) + } +} diff --git a/crux-mir/lib/memchr/src/memchr/fallback.rs b/crux-mir/lib/memchr/src/memchr/fallback.rs new file mode 100644 index 000000000..b01f224fa --- /dev/null +++ b/crux-mir/lib/memchr/src/memchr/fallback.rs @@ -0,0 +1,329 @@ +// This module defines pure Rust platform independent implementations of all +// the memchr routines. We do our best to make them fast. Some of them may even +// get auto-vectorized. + +use core::{cmp, usize}; + +#[cfg(target_pointer_width = "16")] +const USIZE_BYTES: usize = 2; + +#[cfg(target_pointer_width = "32")] +const USIZE_BYTES: usize = 4; + +#[cfg(target_pointer_width = "64")] +const USIZE_BYTES: usize = 8; + +// The number of bytes to loop at in one iteration of memchr/memrchr. +const LOOP_SIZE: usize = 2 * USIZE_BYTES; + +/// Return `true` if `x` contains any zero byte. +/// +/// From *Matters Computational*, J. Arndt +/// +/// "The idea is to subtract one from each of the bytes and then look for +/// bytes where the borrow propagated all the way to the most significant +/// bit." +#[inline(always)] +fn contains_zero_byte(x: usize) -> bool { + const LO_U64: u64 = 0x0101010101010101; + const HI_U64: u64 = 0x8080808080808080; + + const LO_USIZE: usize = LO_U64 as usize; + const HI_USIZE: usize = HI_U64 as usize; + + x.wrapping_sub(LO_USIZE) & !x & HI_USIZE != 0 +} + +/// Repeat the given byte into a word size number. That is, every 8 bits +/// is equivalent to the given byte. For example, if `b` is `\x4E` or +/// `01001110` in binary, then the returned value on a 32-bit system would be: +/// `01001110_01001110_01001110_01001110`. +#[inline(always)] +fn repeat_byte(b: u8) -> usize { + (b as usize) * (usize::MAX / 255) +} + +pub fn memchr(n1: u8, haystack: &[u8]) -> Option { + let vn1 = repeat_byte(n1); + let confirm = |byte| byte == n1; + let loop_size = cmp::min(LOOP_SIZE, haystack.len()); + let align = USIZE_BYTES - 1; + let start_ptr = haystack.as_ptr(); + let mut ptr = start_ptr; + + unsafe { + let end_ptr = start_ptr.add(haystack.len()); + if haystack.len() < USIZE_BYTES { + return forward_search(start_ptr, end_ptr, ptr, confirm); + } + + let chunk = (ptr as *const usize).read_unaligned(); + if contains_zero_byte(chunk ^ vn1) { + return forward_search(start_ptr, end_ptr, ptr, confirm); + } + + ptr = ptr.add(USIZE_BYTES - (start_ptr as usize & align)); + debug_assert!(ptr > start_ptr); + debug_assert!(end_ptr.sub(USIZE_BYTES) >= start_ptr); + while loop_size == LOOP_SIZE && ptr <= end_ptr.sub(loop_size) { + debug_assert_eq!(0, (ptr as usize) % USIZE_BYTES); + + let a = *(ptr as *const usize); + let b = *(ptr.add(USIZE_BYTES) as *const usize); + let eqa = contains_zero_byte(a ^ vn1); + let eqb = contains_zero_byte(b ^ vn1); + if eqa || eqb { + break; + } + ptr = ptr.add(LOOP_SIZE); + } + forward_search(start_ptr, end_ptr, ptr, confirm) + } +} + +/// Like `memchr`, but searches for two bytes instead of one. +pub fn memchr2(n1: u8, n2: u8, haystack: &[u8]) -> Option { + let vn1 = repeat_byte(n1); + let vn2 = repeat_byte(n2); + let confirm = |byte| byte == n1 || byte == n2; + let align = USIZE_BYTES - 1; + let start_ptr = haystack.as_ptr(); + let mut ptr = start_ptr; + + unsafe { + let end_ptr = start_ptr.add(haystack.len()); + if haystack.len() < USIZE_BYTES { + return forward_search(start_ptr, end_ptr, ptr, confirm); + } + + let chunk = (ptr as *const usize).read_unaligned(); + let eq1 = contains_zero_byte(chunk ^ vn1); + let eq2 = contains_zero_byte(chunk ^ vn2); + if eq1 || eq2 { + return forward_search(start_ptr, end_ptr, ptr, confirm); + } + + ptr = ptr.add(USIZE_BYTES - (start_ptr as usize & align)); + debug_assert!(ptr > start_ptr); + debug_assert!(end_ptr.sub(USIZE_BYTES) >= start_ptr); + while ptr <= end_ptr.sub(USIZE_BYTES) { + debug_assert_eq!(0, (ptr as usize) % USIZE_BYTES); + + let chunk = *(ptr as *const usize); + let eq1 = contains_zero_byte(chunk ^ vn1); + let eq2 = contains_zero_byte(chunk ^ vn2); + if eq1 || eq2 { + break; + } + ptr = ptr.add(USIZE_BYTES); + } + forward_search(start_ptr, end_ptr, ptr, confirm) + } +} + +/// Like `memchr`, but searches for three bytes instead of one. +pub fn memchr3(n1: u8, n2: u8, n3: u8, haystack: &[u8]) -> Option { + let vn1 = repeat_byte(n1); + let vn2 = repeat_byte(n2); + let vn3 = repeat_byte(n3); + let confirm = |byte| byte == n1 || byte == n2 || byte == n3; + let align = USIZE_BYTES - 1; + let start_ptr = haystack.as_ptr(); + let mut ptr = start_ptr; + + unsafe { + let end_ptr = start_ptr.add(haystack.len()); + if haystack.len() < USIZE_BYTES { + return forward_search(start_ptr, end_ptr, ptr, confirm); + } + + let chunk = (ptr as *const usize).read_unaligned(); + let eq1 = contains_zero_byte(chunk ^ vn1); + let eq2 = contains_zero_byte(chunk ^ vn2); + let eq3 = contains_zero_byte(chunk ^ vn3); + if eq1 || eq2 || eq3 { + return forward_search(start_ptr, end_ptr, ptr, confirm); + } + + ptr = ptr.add(USIZE_BYTES - (start_ptr as usize & align)); + debug_assert!(ptr > start_ptr); + debug_assert!(end_ptr.sub(USIZE_BYTES) >= start_ptr); + while ptr <= end_ptr.sub(USIZE_BYTES) { + debug_assert_eq!(0, (ptr as usize) % USIZE_BYTES); + + let chunk = *(ptr as *const usize); + let eq1 = contains_zero_byte(chunk ^ vn1); + let eq2 = contains_zero_byte(chunk ^ vn2); + let eq3 = contains_zero_byte(chunk ^ vn3); + if eq1 || eq2 || eq3 { + break; + } + ptr = ptr.add(USIZE_BYTES); + } + forward_search(start_ptr, end_ptr, ptr, confirm) + } +} + +/// Return the last index matching the byte `x` in `text`. +pub fn memrchr(n1: u8, haystack: &[u8]) -> Option { + let vn1 = repeat_byte(n1); + let confirm = |byte| byte == n1; + let loop_size = cmp::min(LOOP_SIZE, haystack.len()); + let align = USIZE_BYTES - 1; + let start_ptr = haystack.as_ptr(); + + unsafe { + let end_ptr = start_ptr.add(haystack.len()); + let mut ptr = end_ptr; + if haystack.len() < USIZE_BYTES { + return reverse_search(start_ptr, end_ptr, ptr, confirm); + } + + let chunk = (ptr.sub(USIZE_BYTES) as *const usize).read_unaligned(); + if contains_zero_byte(chunk ^ vn1) { + return reverse_search(start_ptr, end_ptr, ptr, confirm); + } + + ptr = (end_ptr as usize & !align) as *const u8; + debug_assert!(start_ptr <= ptr && ptr <= end_ptr); + while loop_size == LOOP_SIZE && ptr >= start_ptr.add(loop_size) { + debug_assert_eq!(0, (ptr as usize) % USIZE_BYTES); + + let a = *(ptr.sub(2 * USIZE_BYTES) as *const usize); + let b = *(ptr.sub(1 * USIZE_BYTES) as *const usize); + let eqa = contains_zero_byte(a ^ vn1); + let eqb = contains_zero_byte(b ^ vn1); + if eqa || eqb { + break; + } + ptr = ptr.sub(loop_size); + } + reverse_search(start_ptr, end_ptr, ptr, confirm) + } +} + +/// Like `memrchr`, but searches for two bytes instead of one. +pub fn memrchr2(n1: u8, n2: u8, haystack: &[u8]) -> Option { + let vn1 = repeat_byte(n1); + let vn2 = repeat_byte(n2); + let confirm = |byte| byte == n1 || byte == n2; + let align = USIZE_BYTES - 1; + let start_ptr = haystack.as_ptr(); + + unsafe { + let end_ptr = start_ptr.add(haystack.len()); + let mut ptr = end_ptr; + if haystack.len() < USIZE_BYTES { + return reverse_search(start_ptr, end_ptr, ptr, confirm); + } + + let chunk = (ptr.sub(USIZE_BYTES) as *const usize).read_unaligned(); + let eq1 = contains_zero_byte(chunk ^ vn1); + let eq2 = contains_zero_byte(chunk ^ vn2); + if eq1 || eq2 { + return reverse_search(start_ptr, end_ptr, ptr, confirm); + } + + ptr = (end_ptr as usize & !align) as *const u8; + debug_assert!(start_ptr <= ptr && ptr <= end_ptr); + while ptr >= start_ptr.add(USIZE_BYTES) { + debug_assert_eq!(0, (ptr as usize) % USIZE_BYTES); + + let chunk = *(ptr.sub(USIZE_BYTES) as *const usize); + let eq1 = contains_zero_byte(chunk ^ vn1); + let eq2 = contains_zero_byte(chunk ^ vn2); + if eq1 || eq2 { + break; + } + ptr = ptr.sub(USIZE_BYTES); + } + reverse_search(start_ptr, end_ptr, ptr, confirm) + } +} + +/// Like `memrchr`, but searches for three bytes instead of one. +pub fn memrchr3(n1: u8, n2: u8, n3: u8, haystack: &[u8]) -> Option { + let vn1 = repeat_byte(n1); + let vn2 = repeat_byte(n2); + let vn3 = repeat_byte(n3); + let confirm = |byte| byte == n1 || byte == n2 || byte == n3; + let align = USIZE_BYTES - 1; + let start_ptr = haystack.as_ptr(); + + unsafe { + let end_ptr = start_ptr.add(haystack.len()); + let mut ptr = end_ptr; + if haystack.len() < USIZE_BYTES { + return reverse_search(start_ptr, end_ptr, ptr, confirm); + } + + let chunk = (ptr.sub(USIZE_BYTES) as *const usize).read_unaligned(); + let eq1 = contains_zero_byte(chunk ^ vn1); + let eq2 = contains_zero_byte(chunk ^ vn2); + let eq3 = contains_zero_byte(chunk ^ vn3); + if eq1 || eq2 || eq3 { + return reverse_search(start_ptr, end_ptr, ptr, confirm); + } + + ptr = (end_ptr as usize & !align) as *const u8; + debug_assert!(start_ptr <= ptr && ptr <= end_ptr); + while ptr >= start_ptr.add(USIZE_BYTES) { + debug_assert_eq!(0, (ptr as usize) % USIZE_BYTES); + + let chunk = *(ptr.sub(USIZE_BYTES) as *const usize); + let eq1 = contains_zero_byte(chunk ^ vn1); + let eq2 = contains_zero_byte(chunk ^ vn2); + let eq3 = contains_zero_byte(chunk ^ vn3); + if eq1 || eq2 || eq3 { + break; + } + ptr = ptr.sub(USIZE_BYTES); + } + reverse_search(start_ptr, end_ptr, ptr, confirm) + } +} + +#[inline(always)] +unsafe fn forward_search bool>( + start_ptr: *const u8, + end_ptr: *const u8, + mut ptr: *const u8, + confirm: F, +) -> Option { + debug_assert!(start_ptr <= ptr); + debug_assert!(ptr <= end_ptr); + + while ptr < end_ptr { + if confirm(*ptr) { + return Some(sub(ptr, start_ptr)); + } + ptr = ptr.offset(1); + } + None +} + +#[inline(always)] +unsafe fn reverse_search bool>( + start_ptr: *const u8, + end_ptr: *const u8, + mut ptr: *const u8, + confirm: F, +) -> Option { + debug_assert!(start_ptr <= ptr); + debug_assert!(ptr <= end_ptr); + + while ptr > start_ptr { + ptr = ptr.offset(-1); + if confirm(*ptr) { + return Some(sub(ptr, start_ptr)); + } + } + None +} + +/// Subtract `b` from `a` and return the difference. `a` should be greater than +/// or equal to `b`. +fn sub(a: *const u8, b: *const u8) -> usize { + debug_assert!(a >= b); + (a as usize) - (b as usize) +} diff --git a/crux-mir/lib/memchr/src/memchr/iter.rs b/crux-mir/lib/memchr/src/memchr/iter.rs new file mode 100644 index 000000000..16e203f63 --- /dev/null +++ b/crux-mir/lib/memchr/src/memchr/iter.rs @@ -0,0 +1,173 @@ +use crate::{memchr, memchr2, memchr3, memrchr, memrchr2, memrchr3}; + +macro_rules! iter_next { + // Common code for the memchr iterators: + // update haystack and position and produce the index + // + // self: &mut Self where Self is the iterator + // search_result: Option which is the result of the corresponding + // memchr function. + // + // Returns Option (the next iterator element) + ($self_:expr, $search_result:expr) => { + $search_result.map(move |index| { + // split and take the remaining back half + $self_.haystack = $self_.haystack.split_at(index + 1).1; + let found_position = $self_.position + index; + $self_.position = found_position + 1; + found_position + }) + }; +} + +macro_rules! iter_next_back { + ($self_:expr, $search_result:expr) => { + $search_result.map(move |index| { + // split and take the remaining front half + $self_.haystack = $self_.haystack.split_at(index).0; + $self_.position + index + }) + }; +} + +/// An iterator for `memchr`. +pub struct Memchr<'a> { + needle: u8, + // The haystack to iterate over + haystack: &'a [u8], + // The index + position: usize, +} + +impl<'a> Memchr<'a> { + /// Creates a new iterator that yields all positions of needle in haystack. + #[inline] + pub fn new(needle: u8, haystack: &[u8]) -> Memchr<'_> { + Memchr { needle: needle, haystack: haystack, position: 0 } + } +} + +impl<'a> Iterator for Memchr<'a> { + type Item = usize; + + #[inline] + fn next(&mut self) -> Option { + iter_next!(self, memchr(self.needle, self.haystack)) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + (0, Some(self.haystack.len())) + } +} + +impl<'a> DoubleEndedIterator for Memchr<'a> { + #[inline] + fn next_back(&mut self) -> Option { + iter_next_back!(self, memrchr(self.needle, self.haystack)) + } +} + +/// An iterator for `memchr2`. +pub struct Memchr2<'a> { + needle1: u8, + needle2: u8, + // The haystack to iterate over + haystack: &'a [u8], + // The index + position: usize, +} + +impl<'a> Memchr2<'a> { + /// Creates a new iterator that yields all positions of needle in haystack. + #[inline] + pub fn new(needle1: u8, needle2: u8, haystack: &[u8]) -> Memchr2<'_> { + Memchr2 { + needle1: needle1, + needle2: needle2, + haystack: haystack, + position: 0, + } + } +} + +impl<'a> Iterator for Memchr2<'a> { + type Item = usize; + + #[inline] + fn next(&mut self) -> Option { + iter_next!(self, memchr2(self.needle1, self.needle2, self.haystack)) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + (0, Some(self.haystack.len())) + } +} + +impl<'a> DoubleEndedIterator for Memchr2<'a> { + #[inline] + fn next_back(&mut self) -> Option { + iter_next_back!( + self, + memrchr2(self.needle1, self.needle2, self.haystack) + ) + } +} + +/// An iterator for `memchr3`. +pub struct Memchr3<'a> { + needle1: u8, + needle2: u8, + needle3: u8, + // The haystack to iterate over + haystack: &'a [u8], + // The index + position: usize, +} + +impl<'a> Memchr3<'a> { + /// Create a new `Memchr3` that's initialized to zero with a haystack + #[inline] + pub fn new( + needle1: u8, + needle2: u8, + needle3: u8, + haystack: &[u8], + ) -> Memchr3<'_> { + Memchr3 { + needle1: needle1, + needle2: needle2, + needle3: needle3, + haystack: haystack, + position: 0, + } + } +} + +impl<'a> Iterator for Memchr3<'a> { + type Item = usize; + + #[inline] + fn next(&mut self) -> Option { + iter_next!( + self, + memchr3(self.needle1, self.needle2, self.needle3, self.haystack) + ) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + (0, Some(self.haystack.len())) + } +} + +impl<'a> DoubleEndedIterator for Memchr3<'a> { + #[inline] + fn next_back(&mut self) -> Option { + iter_next_back!( + self, + memrchr3(self.needle1, self.needle2, self.needle3, self.haystack) + ) + } +} diff --git a/crux-mir/lib/memchr/src/memchr/mod.rs b/crux-mir/lib/memchr/src/memchr/mod.rs new file mode 100644 index 000000000..09ce6ef3c --- /dev/null +++ b/crux-mir/lib/memchr/src/memchr/mod.rs @@ -0,0 +1,410 @@ +use core::iter::Rev; + +pub use self::iter::{Memchr, Memchr2, Memchr3}; + +// N.B. If you're looking for the cfg knobs for libc, see build.rs. +#[cfg(memchr_libc)] +mod c; +#[allow(dead_code)] +pub mod fallback; +mod iter; +pub mod naive; +#[cfg(all(not(miri), target_arch = "x86_64", memchr_runtime_simd))] +mod x86; + +/// An iterator over all occurrences of the needle in a haystack. +#[inline] +pub fn memchr_iter(needle: u8, haystack: &[u8]) -> Memchr<'_> { + Memchr::new(needle, haystack) +} + +/// An iterator over all occurrences of the needles in a haystack. +#[inline] +pub fn memchr2_iter(needle1: u8, needle2: u8, haystack: &[u8]) -> Memchr2<'_> { + Memchr2::new(needle1, needle2, haystack) +} + +/// An iterator over all occurrences of the needles in a haystack. +#[inline] +pub fn memchr3_iter( + needle1: u8, + needle2: u8, + needle3: u8, + haystack: &[u8], +) -> Memchr3<'_> { + Memchr3::new(needle1, needle2, needle3, haystack) +} + +/// An iterator over all occurrences of the needle in a haystack, in reverse. +#[inline] +pub fn memrchr_iter(needle: u8, haystack: &[u8]) -> Rev> { + Memchr::new(needle, haystack).rev() +} + +/// An iterator over all occurrences of the needles in a haystack, in reverse. +#[inline] +pub fn memrchr2_iter( + needle1: u8, + needle2: u8, + haystack: &[u8], +) -> Rev> { + Memchr2::new(needle1, needle2, haystack).rev() +} + +/// An iterator over all occurrences of the needles in a haystack, in reverse. +#[inline] +pub fn memrchr3_iter( + needle1: u8, + needle2: u8, + needle3: u8, + haystack: &[u8], +) -> Rev> { + Memchr3::new(needle1, needle2, needle3, haystack).rev() +} + +/// Search for the first occurrence of a byte in a slice. +/// +/// This returns the index corresponding to the first occurrence of `needle` in +/// `haystack`, or `None` if one is not found. If an index is returned, it is +/// guaranteed to be less than `usize::MAX`. +/// +/// While this is operationally the same as something like +/// `haystack.iter().position(|&b| b == needle)`, `memchr` will use a highly +/// optimized routine that can be up to an order of magnitude faster in some +/// cases. +/// +/// # Example +/// +/// This shows how to find the first position of a byte in a byte string. +/// +/// ``` +/// use memchr::memchr; +/// +/// let haystack = b"the quick brown fox"; +/// assert_eq!(memchr(b'k', haystack), Some(8)); +/// ``` +#[inline] +pub fn memchr(needle: u8, haystack: &[u8]) -> Option { + #[cfg(miri)] + #[inline(always)] + fn imp(n1: u8, haystack: &[u8]) -> Option { + naive::memchr(n1, haystack) + } + + #[cfg(all(target_arch = "x86_64", memchr_runtime_simd, not(miri)))] + #[inline(always)] + fn imp(n1: u8, haystack: &[u8]) -> Option { + x86::memchr(n1, haystack) + } + + #[cfg(all( + memchr_libc, + not(all(target_arch = "x86_64", memchr_runtime_simd)), + not(miri), + ))] + #[inline(always)] + fn imp(n1: u8, haystack: &[u8]) -> Option { + c::memchr(n1, haystack) + } + + #[cfg(all( + not(memchr_libc), + not(all(target_arch = "x86_64", memchr_runtime_simd)), + not(miri), + ))] + #[inline(always)] + fn imp(n1: u8, haystack: &[u8]) -> Option { + fallback::memchr(n1, haystack) + } + + if haystack.is_empty() { + None + } else { + imp(needle, haystack) + } +} + +/// Like `memchr`, but searches for either of two bytes instead of just one. +/// +/// This returns the index corresponding to the first occurrence of `needle1` +/// or the first occurrence of `needle2` in `haystack` (whichever occurs +/// earlier), or `None` if neither one is found. If an index is returned, it is +/// guaranteed to be less than `usize::MAX`. +/// +/// While this is operationally the same as something like +/// `haystack.iter().position(|&b| b == needle1 || b == needle2)`, `memchr2` +/// will use a highly optimized routine that can be up to an order of magnitude +/// faster in some cases. +/// +/// # Example +/// +/// This shows how to find the first position of either of two bytes in a byte +/// string. +/// +/// ``` +/// use memchr::memchr2; +/// +/// let haystack = b"the quick brown fox"; +/// assert_eq!(memchr2(b'k', b'q', haystack), Some(4)); +/// ``` +#[inline] +pub fn memchr2(needle1: u8, needle2: u8, haystack: &[u8]) -> Option { + #[cfg(miri)] + #[inline(always)] + fn imp(n1: u8, n2: u8, haystack: &[u8]) -> Option { + naive::memchr2(n1, n2, haystack) + } + + #[cfg(all(target_arch = "x86_64", memchr_runtime_simd, not(miri)))] + #[inline(always)] + fn imp(n1: u8, n2: u8, haystack: &[u8]) -> Option { + x86::memchr2(n1, n2, haystack) + } + + #[cfg(all( + not(all(target_arch = "x86_64", memchr_runtime_simd)), + not(miri), + ))] + #[inline(always)] + fn imp(n1: u8, n2: u8, haystack: &[u8]) -> Option { + fallback::memchr2(n1, n2, haystack) + } + + if haystack.is_empty() { + None + } else { + imp(needle1, needle2, haystack) + } +} + +/// Like `memchr`, but searches for any of three bytes instead of just one. +/// +/// This returns the index corresponding to the first occurrence of `needle1`, +/// the first occurrence of `needle2`, or the first occurrence of `needle3` in +/// `haystack` (whichever occurs earliest), or `None` if none are found. If an +/// index is returned, it is guaranteed to be less than `usize::MAX`. +/// +/// While this is operationally the same as something like +/// `haystack.iter().position(|&b| b == needle1 || b == needle2 || +/// b == needle3)`, `memchr3` will use a highly optimized routine that can be +/// up to an order of magnitude faster in some cases. +/// +/// # Example +/// +/// This shows how to find the first position of any of three bytes in a byte +/// string. +/// +/// ``` +/// use memchr::memchr3; +/// +/// let haystack = b"the quick brown fox"; +/// assert_eq!(memchr3(b'k', b'q', b'e', haystack), Some(2)); +/// ``` +#[inline] +pub fn memchr3( + needle1: u8, + needle2: u8, + needle3: u8, + haystack: &[u8], +) -> Option { + #[cfg(miri)] + #[inline(always)] + fn imp(n1: u8, n2: u8, n3: u8, haystack: &[u8]) -> Option { + naive::memchr3(n1, n2, n3, haystack) + } + + #[cfg(all(target_arch = "x86_64", memchr_runtime_simd, not(miri)))] + #[inline(always)] + fn imp(n1: u8, n2: u8, n3: u8, haystack: &[u8]) -> Option { + x86::memchr3(n1, n2, n3, haystack) + } + + #[cfg(all( + not(all(target_arch = "x86_64", memchr_runtime_simd)), + not(miri), + ))] + #[inline(always)] + fn imp(n1: u8, n2: u8, n3: u8, haystack: &[u8]) -> Option { + fallback::memchr3(n1, n2, n3, haystack) + } + + if haystack.is_empty() { + None + } else { + imp(needle1, needle2, needle3, haystack) + } +} + +/// Search for the last occurrence of a byte in a slice. +/// +/// This returns the index corresponding to the last occurrence of `needle` in +/// `haystack`, or `None` if one is not found. If an index is returned, it is +/// guaranteed to be less than `usize::MAX`. +/// +/// While this is operationally the same as something like +/// `haystack.iter().rposition(|&b| b == needle)`, `memrchr` will use a highly +/// optimized routine that can be up to an order of magnitude faster in some +/// cases. +/// +/// # Example +/// +/// This shows how to find the last position of a byte in a byte string. +/// +/// ``` +/// use memchr::memrchr; +/// +/// let haystack = b"the quick brown fox"; +/// assert_eq!(memrchr(b'o', haystack), Some(17)); +/// ``` +#[inline] +pub fn memrchr(needle: u8, haystack: &[u8]) -> Option { + #[cfg(miri)] + #[inline(always)] + fn imp(n1: u8, haystack: &[u8]) -> Option { + naive::memrchr(n1, haystack) + } + + #[cfg(all(target_arch = "x86_64", memchr_runtime_simd, not(miri)))] + #[inline(always)] + fn imp(n1: u8, haystack: &[u8]) -> Option { + x86::memrchr(n1, haystack) + } + + #[cfg(all( + memchr_libc, + target_os = "linux", + not(all(target_arch = "x86_64", memchr_runtime_simd)), + not(miri) + ))] + #[inline(always)] + fn imp(n1: u8, haystack: &[u8]) -> Option { + c::memrchr(n1, haystack) + } + + #[cfg(all( + not(all(memchr_libc, target_os = "linux")), + not(all(target_arch = "x86_64", memchr_runtime_simd)), + not(miri), + ))] + #[inline(always)] + fn imp(n1: u8, haystack: &[u8]) -> Option { + fallback::memrchr(n1, haystack) + } + + if haystack.is_empty() { + None + } else { + imp(needle, haystack) + } +} + +/// Like `memrchr`, but searches for either of two bytes instead of just one. +/// +/// This returns the index corresponding to the last occurrence of `needle1` or +/// the last occurrence of `needle2` in `haystack` (whichever occurs later), or +/// `None` if neither one is found. If an index is returned, it is guaranteed +/// to be less than `usize::MAX`. +/// +/// While this is operationally the same as something like +/// `haystack.iter().rposition(|&b| b == needle1 || b == needle2)`, `memrchr2` +/// will use a highly optimized routine that can be up to an order of magnitude +/// faster in some cases. +/// +/// # Example +/// +/// This shows how to find the last position of either of two bytes in a byte +/// string. +/// +/// ``` +/// use memchr::memrchr2; +/// +/// let haystack = b"the quick brown fox"; +/// assert_eq!(memrchr2(b'k', b'q', haystack), Some(8)); +/// ``` +#[inline] +pub fn memrchr2(needle1: u8, needle2: u8, haystack: &[u8]) -> Option { + #[cfg(miri)] + #[inline(always)] + fn imp(n1: u8, n2: u8, haystack: &[u8]) -> Option { + naive::memrchr2(n1, n2, haystack) + } + + #[cfg(all(target_arch = "x86_64", memchr_runtime_simd, not(miri)))] + #[inline(always)] + fn imp(n1: u8, n2: u8, haystack: &[u8]) -> Option { + x86::memrchr2(n1, n2, haystack) + } + + #[cfg(all( + not(all(target_arch = "x86_64", memchr_runtime_simd)), + not(miri), + ))] + #[inline(always)] + fn imp(n1: u8, n2: u8, haystack: &[u8]) -> Option { + fallback::memrchr2(n1, n2, haystack) + } + + if haystack.is_empty() { + None + } else { + imp(needle1, needle2, haystack) + } +} + +/// Like `memrchr`, but searches for any of three bytes instead of just one. +/// +/// This returns the index corresponding to the last occurrence of `needle1`, +/// the last occurrence of `needle2`, or the last occurrence of `needle3` in +/// `haystack` (whichever occurs later), or `None` if none are found. If an +/// index is returned, it is guaranteed to be less than `usize::MAX`. +/// +/// While this is operationally the same as something like +/// `haystack.iter().rposition(|&b| b == needle1 || b == needle2 || +/// b == needle3)`, `memrchr3` will use a highly optimized routine that can be +/// up to an order of magnitude faster in some cases. +/// +/// # Example +/// +/// This shows how to find the last position of any of three bytes in a byte +/// string. +/// +/// ``` +/// use memchr::memrchr3; +/// +/// let haystack = b"the quick brown fox"; +/// assert_eq!(memrchr3(b'k', b'q', b'e', haystack), Some(8)); +/// ``` +#[inline] +pub fn memrchr3( + needle1: u8, + needle2: u8, + needle3: u8, + haystack: &[u8], +) -> Option { + #[cfg(miri)] + #[inline(always)] + fn imp(n1: u8, n2: u8, n3: u8, haystack: &[u8]) -> Option { + naive::memrchr3(n1, n2, n3, haystack) + } + + #[cfg(all(target_arch = "x86_64", memchr_runtime_simd, not(miri)))] + #[inline(always)] + fn imp(n1: u8, n2: u8, n3: u8, haystack: &[u8]) -> Option { + x86::memrchr3(n1, n2, n3, haystack) + } + + #[cfg(all( + not(all(target_arch = "x86_64", memchr_runtime_simd)), + not(miri), + ))] + #[inline(always)] + fn imp(n1: u8, n2: u8, n3: u8, haystack: &[u8]) -> Option { + fallback::memrchr3(n1, n2, n3, haystack) + } + + if haystack.is_empty() { + None + } else { + imp(needle1, needle2, needle3, haystack) + } +} diff --git a/crux-mir/lib/memchr/src/memchr/naive.rs b/crux-mir/lib/memchr/src/memchr/naive.rs new file mode 100644 index 000000000..3f3053d48 --- /dev/null +++ b/crux-mir/lib/memchr/src/memchr/naive.rs @@ -0,0 +1,25 @@ +#![allow(dead_code)] + +pub fn memchr(n1: u8, haystack: &[u8]) -> Option { + haystack.iter().position(|&b| b == n1) +} + +pub fn memchr2(n1: u8, n2: u8, haystack: &[u8]) -> Option { + haystack.iter().position(|&b| b == n1 || b == n2) +} + +pub fn memchr3(n1: u8, n2: u8, n3: u8, haystack: &[u8]) -> Option { + haystack.iter().position(|&b| b == n1 || b == n2 || b == n3) +} + +pub fn memrchr(n1: u8, haystack: &[u8]) -> Option { + haystack.iter().rposition(|&b| b == n1) +} + +pub fn memrchr2(n1: u8, n2: u8, haystack: &[u8]) -> Option { + haystack.iter().rposition(|&b| b == n1 || b == n2) +} + +pub fn memrchr3(n1: u8, n2: u8, n3: u8, haystack: &[u8]) -> Option { + haystack.iter().rposition(|&b| b == n1 || b == n2 || b == n3) +} diff --git a/crux-mir/lib/memchr/src/memchr/x86/avx.rs b/crux-mir/lib/memchr/src/memchr/x86/avx.rs new file mode 100644 index 000000000..535123097 --- /dev/null +++ b/crux-mir/lib/memchr/src/memchr/x86/avx.rs @@ -0,0 +1,755 @@ +use core::{arch::x86_64::*, cmp, mem::size_of}; + +use super::sse2; + +const VECTOR_SIZE: usize = size_of::<__m256i>(); +const VECTOR_ALIGN: usize = VECTOR_SIZE - 1; + +// The number of bytes to loop at in one iteration of memchr/memrchr. +const LOOP_SIZE: usize = 4 * VECTOR_SIZE; + +// The number of bytes to loop at in one iteration of memchr2/memrchr2 and +// memchr3/memrchr3. There was no observable difference between 128 and 64 +// bytes in benchmarks. memchr3 in particular only gets a very slight speed up +// from the loop unrolling. +const LOOP_SIZE2: usize = 2 * VECTOR_SIZE; + +#[target_feature(enable = "avx2")] +pub unsafe fn memchr(n1: u8, haystack: &[u8]) -> Option { + // For a high level explanation for how this algorithm works, see the + // sse2 implementation. The avx implementation here is the same, but with + // 256-bit vectors instead of 128-bit vectors. + + // This routine is called whenever a match is detected. It is specifically + // marked as unlineable because it improves the codegen of the unrolled + // loop below. Inlining this seems to cause codegen with some extra adds + // and a load that aren't necessary. This seems to result in about a 10% + // improvement for the memchr1/crate/huge/never benchmark. + // + // Interestingly, I couldn't observe a similar improvement for memrchr. + #[cold] + #[inline(never)] + #[target_feature(enable = "avx2")] + unsafe fn matched( + start_ptr: *const u8, + ptr: *const u8, + eqa: __m256i, + eqb: __m256i, + eqc: __m256i, + eqd: __m256i, + ) -> usize { + let mut at = sub(ptr, start_ptr); + let mask = _mm256_movemask_epi8(eqa); + if mask != 0 { + return at + forward_pos(mask); + } + + at += VECTOR_SIZE; + let mask = _mm256_movemask_epi8(eqb); + if mask != 0 { + return at + forward_pos(mask); + } + + at += VECTOR_SIZE; + let mask = _mm256_movemask_epi8(eqc); + if mask != 0 { + return at + forward_pos(mask); + } + + at += VECTOR_SIZE; + let mask = _mm256_movemask_epi8(eqd); + debug_assert!(mask != 0); + at + forward_pos(mask) + } + + let start_ptr = haystack.as_ptr(); + let end_ptr = start_ptr.add(haystack.len()); + let mut ptr = start_ptr; + + if haystack.len() < VECTOR_SIZE { + // For small haystacks, defer to the SSE2 implementation. Codegen + // suggests this completely avoids touching the AVX vectors. + return sse2::memchr(n1, haystack); + } + + let vn1 = _mm256_set1_epi8(n1 as i8); + let loop_size = cmp::min(LOOP_SIZE, haystack.len()); + if let Some(i) = forward_search1(start_ptr, end_ptr, ptr, vn1) { + return Some(i); + } + + ptr = ptr.add(VECTOR_SIZE - (start_ptr as usize & VECTOR_ALIGN)); + debug_assert!(ptr > start_ptr && end_ptr.sub(VECTOR_SIZE) >= start_ptr); + while loop_size == LOOP_SIZE && ptr <= end_ptr.sub(loop_size) { + debug_assert_eq!(0, (ptr as usize) % VECTOR_SIZE); + + let a = _mm256_load_si256(ptr as *const __m256i); + let b = _mm256_load_si256(ptr.add(VECTOR_SIZE) as *const __m256i); + let c = _mm256_load_si256(ptr.add(2 * VECTOR_SIZE) as *const __m256i); + let d = _mm256_load_si256(ptr.add(3 * VECTOR_SIZE) as *const __m256i); + let eqa = _mm256_cmpeq_epi8(vn1, a); + let eqb = _mm256_cmpeq_epi8(vn1, b); + let eqc = _mm256_cmpeq_epi8(vn1, c); + let eqd = _mm256_cmpeq_epi8(vn1, d); + let or1 = _mm256_or_si256(eqa, eqb); + let or2 = _mm256_or_si256(eqc, eqd); + let or3 = _mm256_or_si256(or1, or2); + + if _mm256_movemask_epi8(or3) != 0 { + return Some(matched(start_ptr, ptr, eqa, eqb, eqc, eqd)); + } + ptr = ptr.add(loop_size); + } + while ptr <= end_ptr.sub(VECTOR_SIZE) { + debug_assert!(sub(end_ptr, ptr) >= VECTOR_SIZE); + + if let Some(i) = forward_search1(start_ptr, end_ptr, ptr, vn1) { + return Some(i); + } + ptr = ptr.add(VECTOR_SIZE); + } + if ptr < end_ptr { + debug_assert!(sub(end_ptr, ptr) < VECTOR_SIZE); + ptr = ptr.sub(VECTOR_SIZE - sub(end_ptr, ptr)); + debug_assert_eq!(sub(end_ptr, ptr), VECTOR_SIZE); + + return forward_search1(start_ptr, end_ptr, ptr, vn1); + } + None +} + +#[target_feature(enable = "avx2")] +pub unsafe fn memchr2(n1: u8, n2: u8, haystack: &[u8]) -> Option { + #[cold] + #[inline(never)] + #[target_feature(enable = "avx2")] + unsafe fn matched( + start_ptr: *const u8, + ptr: *const u8, + eqa1: __m256i, + eqa2: __m256i, + eqb1: __m256i, + eqb2: __m256i, + ) -> usize { + let mut at = sub(ptr, start_ptr); + let mask1 = _mm256_movemask_epi8(eqa1); + let mask2 = _mm256_movemask_epi8(eqa2); + if mask1 != 0 || mask2 != 0 { + return at + forward_pos2(mask1, mask2); + } + + at += VECTOR_SIZE; + let mask1 = _mm256_movemask_epi8(eqb1); + let mask2 = _mm256_movemask_epi8(eqb2); + at + forward_pos2(mask1, mask2) + } + + let vn1 = _mm256_set1_epi8(n1 as i8); + let vn2 = _mm256_set1_epi8(n2 as i8); + let len = haystack.len(); + let loop_size = cmp::min(LOOP_SIZE2, len); + let start_ptr = haystack.as_ptr(); + let end_ptr = start_ptr.add(haystack.len()); + let mut ptr = start_ptr; + + if haystack.len() < VECTOR_SIZE { + while ptr < end_ptr { + if *ptr == n1 || *ptr == n2 { + return Some(sub(ptr, start_ptr)); + } + ptr = ptr.offset(1); + } + return None; + } + + if let Some(i) = forward_search2(start_ptr, end_ptr, ptr, vn1, vn2) { + return Some(i); + } + + ptr = ptr.add(VECTOR_SIZE - (start_ptr as usize & VECTOR_ALIGN)); + debug_assert!(ptr > start_ptr && end_ptr.sub(VECTOR_SIZE) >= start_ptr); + while loop_size == LOOP_SIZE2 && ptr <= end_ptr.sub(loop_size) { + debug_assert_eq!(0, (ptr as usize) % VECTOR_SIZE); + + let a = _mm256_load_si256(ptr as *const __m256i); + let b = _mm256_load_si256(ptr.add(VECTOR_SIZE) as *const __m256i); + let eqa1 = _mm256_cmpeq_epi8(vn1, a); + let eqb1 = _mm256_cmpeq_epi8(vn1, b); + let eqa2 = _mm256_cmpeq_epi8(vn2, a); + let eqb2 = _mm256_cmpeq_epi8(vn2, b); + let or1 = _mm256_or_si256(eqa1, eqb1); + let or2 = _mm256_or_si256(eqa2, eqb2); + let or3 = _mm256_or_si256(or1, or2); + if _mm256_movemask_epi8(or3) != 0 { + return Some(matched(start_ptr, ptr, eqa1, eqa2, eqb1, eqb2)); + } + ptr = ptr.add(loop_size); + } + while ptr <= end_ptr.sub(VECTOR_SIZE) { + if let Some(i) = forward_search2(start_ptr, end_ptr, ptr, vn1, vn2) { + return Some(i); + } + ptr = ptr.add(VECTOR_SIZE); + } + if ptr < end_ptr { + debug_assert!(sub(end_ptr, ptr) < VECTOR_SIZE); + ptr = ptr.sub(VECTOR_SIZE - sub(end_ptr, ptr)); + debug_assert_eq!(sub(end_ptr, ptr), VECTOR_SIZE); + + return forward_search2(start_ptr, end_ptr, ptr, vn1, vn2); + } + None +} + +#[target_feature(enable = "avx2")] +pub unsafe fn memchr3( + n1: u8, + n2: u8, + n3: u8, + haystack: &[u8], +) -> Option { + #[cold] + #[inline(never)] + #[target_feature(enable = "avx2")] + unsafe fn matched( + start_ptr: *const u8, + ptr: *const u8, + eqa1: __m256i, + eqa2: __m256i, + eqa3: __m256i, + eqb1: __m256i, + eqb2: __m256i, + eqb3: __m256i, + ) -> usize { + let mut at = sub(ptr, start_ptr); + let mask1 = _mm256_movemask_epi8(eqa1); + let mask2 = _mm256_movemask_epi8(eqa2); + let mask3 = _mm256_movemask_epi8(eqa3); + if mask1 != 0 || mask2 != 0 || mask3 != 0 { + return at + forward_pos3(mask1, mask2, mask3); + } + + at += VECTOR_SIZE; + let mask1 = _mm256_movemask_epi8(eqb1); + let mask2 = _mm256_movemask_epi8(eqb2); + let mask3 = _mm256_movemask_epi8(eqb3); + at + forward_pos3(mask1, mask2, mask3) + } + + let vn1 = _mm256_set1_epi8(n1 as i8); + let vn2 = _mm256_set1_epi8(n2 as i8); + let vn3 = _mm256_set1_epi8(n3 as i8); + let len = haystack.len(); + let loop_size = cmp::min(LOOP_SIZE2, len); + let start_ptr = haystack.as_ptr(); + let end_ptr = start_ptr.add(haystack.len()); + let mut ptr = start_ptr; + + if haystack.len() < VECTOR_SIZE { + while ptr < end_ptr { + if *ptr == n1 || *ptr == n2 || *ptr == n3 { + return Some(sub(ptr, start_ptr)); + } + ptr = ptr.offset(1); + } + return None; + } + + if let Some(i) = forward_search3(start_ptr, end_ptr, ptr, vn1, vn2, vn3) { + return Some(i); + } + + ptr = ptr.add(VECTOR_SIZE - (start_ptr as usize & VECTOR_ALIGN)); + debug_assert!(ptr > start_ptr && end_ptr.sub(VECTOR_SIZE) >= start_ptr); + while loop_size == LOOP_SIZE2 && ptr <= end_ptr.sub(loop_size) { + debug_assert_eq!(0, (ptr as usize) % VECTOR_SIZE); + + let a = _mm256_load_si256(ptr as *const __m256i); + let b = _mm256_load_si256(ptr.add(VECTOR_SIZE) as *const __m256i); + let eqa1 = _mm256_cmpeq_epi8(vn1, a); + let eqb1 = _mm256_cmpeq_epi8(vn1, b); + let eqa2 = _mm256_cmpeq_epi8(vn2, a); + let eqb2 = _mm256_cmpeq_epi8(vn2, b); + let eqa3 = _mm256_cmpeq_epi8(vn3, a); + let eqb3 = _mm256_cmpeq_epi8(vn3, b); + let or1 = _mm256_or_si256(eqa1, eqb1); + let or2 = _mm256_or_si256(eqa2, eqb2); + let or3 = _mm256_or_si256(eqa3, eqb3); + let or4 = _mm256_or_si256(or1, or2); + let or5 = _mm256_or_si256(or3, or4); + if _mm256_movemask_epi8(or5) != 0 { + return Some(matched( + start_ptr, ptr, eqa1, eqa2, eqa3, eqb1, eqb2, eqb3, + )); + } + ptr = ptr.add(loop_size); + } + while ptr <= end_ptr.sub(VECTOR_SIZE) { + if let Some(i) = + forward_search3(start_ptr, end_ptr, ptr, vn1, vn2, vn3) + { + return Some(i); + } + ptr = ptr.add(VECTOR_SIZE); + } + if ptr < end_ptr { + debug_assert!(sub(end_ptr, ptr) < VECTOR_SIZE); + ptr = ptr.sub(VECTOR_SIZE - sub(end_ptr, ptr)); + debug_assert_eq!(sub(end_ptr, ptr), VECTOR_SIZE); + + return forward_search3(start_ptr, end_ptr, ptr, vn1, vn2, vn3); + } + None +} + +#[target_feature(enable = "avx2")] +pub unsafe fn memrchr(n1: u8, haystack: &[u8]) -> Option { + let vn1 = _mm256_set1_epi8(n1 as i8); + let len = haystack.len(); + let loop_size = cmp::min(LOOP_SIZE, len); + let start_ptr = haystack.as_ptr(); + let end_ptr = start_ptr.add(haystack.len()); + let mut ptr = end_ptr; + + if haystack.len() < VECTOR_SIZE { + while ptr > start_ptr { + ptr = ptr.offset(-1); + if *ptr == n1 { + return Some(sub(ptr, start_ptr)); + } + } + return None; + } + + ptr = ptr.sub(VECTOR_SIZE); + if let Some(i) = reverse_search1(start_ptr, end_ptr, ptr, vn1) { + return Some(i); + } + + ptr = (end_ptr as usize & !VECTOR_ALIGN) as *const u8; + debug_assert!(start_ptr <= ptr && ptr <= end_ptr); + while loop_size == LOOP_SIZE && ptr >= start_ptr.add(loop_size) { + debug_assert_eq!(0, (ptr as usize) % VECTOR_SIZE); + + ptr = ptr.sub(loop_size); + let a = _mm256_load_si256(ptr as *const __m256i); + let b = _mm256_load_si256(ptr.add(VECTOR_SIZE) as *const __m256i); + let c = _mm256_load_si256(ptr.add(2 * VECTOR_SIZE) as *const __m256i); + let d = _mm256_load_si256(ptr.add(3 * VECTOR_SIZE) as *const __m256i); + let eqa = _mm256_cmpeq_epi8(vn1, a); + let eqb = _mm256_cmpeq_epi8(vn1, b); + let eqc = _mm256_cmpeq_epi8(vn1, c); + let eqd = _mm256_cmpeq_epi8(vn1, d); + let or1 = _mm256_or_si256(eqa, eqb); + let or2 = _mm256_or_si256(eqc, eqd); + let or3 = _mm256_or_si256(or1, or2); + if _mm256_movemask_epi8(or3) != 0 { + let mut at = sub(ptr.add(3 * VECTOR_SIZE), start_ptr); + let mask = _mm256_movemask_epi8(eqd); + if mask != 0 { + return Some(at + reverse_pos(mask)); + } + + at -= VECTOR_SIZE; + let mask = _mm256_movemask_epi8(eqc); + if mask != 0 { + return Some(at + reverse_pos(mask)); + } + + at -= VECTOR_SIZE; + let mask = _mm256_movemask_epi8(eqb); + if mask != 0 { + return Some(at + reverse_pos(mask)); + } + + at -= VECTOR_SIZE; + let mask = _mm256_movemask_epi8(eqa); + debug_assert!(mask != 0); + return Some(at + reverse_pos(mask)); + } + } + while ptr >= start_ptr.add(VECTOR_SIZE) { + ptr = ptr.sub(VECTOR_SIZE); + if let Some(i) = reverse_search1(start_ptr, end_ptr, ptr, vn1) { + return Some(i); + } + } + if ptr > start_ptr { + debug_assert!(sub(ptr, start_ptr) < VECTOR_SIZE); + return reverse_search1(start_ptr, end_ptr, start_ptr, vn1); + } + None +} + +#[target_feature(enable = "avx2")] +pub unsafe fn memrchr2(n1: u8, n2: u8, haystack: &[u8]) -> Option { + let vn1 = _mm256_set1_epi8(n1 as i8); + let vn2 = _mm256_set1_epi8(n2 as i8); + let len = haystack.len(); + let loop_size = cmp::min(LOOP_SIZE2, len); + let start_ptr = haystack.as_ptr(); + let end_ptr = start_ptr.add(haystack.len()); + let mut ptr = end_ptr; + + if haystack.len() < VECTOR_SIZE { + while ptr > start_ptr { + ptr = ptr.offset(-1); + if *ptr == n1 || *ptr == n2 { + return Some(sub(ptr, start_ptr)); + } + } + return None; + } + + ptr = ptr.sub(VECTOR_SIZE); + if let Some(i) = reverse_search2(start_ptr, end_ptr, ptr, vn1, vn2) { + return Some(i); + } + + ptr = (end_ptr as usize & !VECTOR_ALIGN) as *const u8; + debug_assert!(start_ptr <= ptr && ptr <= end_ptr); + while loop_size == LOOP_SIZE2 && ptr >= start_ptr.add(loop_size) { + debug_assert_eq!(0, (ptr as usize) % VECTOR_SIZE); + + ptr = ptr.sub(loop_size); + let a = _mm256_load_si256(ptr as *const __m256i); + let b = _mm256_load_si256(ptr.add(VECTOR_SIZE) as *const __m256i); + let eqa1 = _mm256_cmpeq_epi8(vn1, a); + let eqb1 = _mm256_cmpeq_epi8(vn1, b); + let eqa2 = _mm256_cmpeq_epi8(vn2, a); + let eqb2 = _mm256_cmpeq_epi8(vn2, b); + let or1 = _mm256_or_si256(eqa1, eqb1); + let or2 = _mm256_or_si256(eqa2, eqb2); + let or3 = _mm256_or_si256(or1, or2); + if _mm256_movemask_epi8(or3) != 0 { + let mut at = sub(ptr.add(VECTOR_SIZE), start_ptr); + let mask1 = _mm256_movemask_epi8(eqb1); + let mask2 = _mm256_movemask_epi8(eqb2); + if mask1 != 0 || mask2 != 0 { + return Some(at + reverse_pos2(mask1, mask2)); + } + + at -= VECTOR_SIZE; + let mask1 = _mm256_movemask_epi8(eqa1); + let mask2 = _mm256_movemask_epi8(eqa2); + return Some(at + reverse_pos2(mask1, mask2)); + } + } + while ptr >= start_ptr.add(VECTOR_SIZE) { + ptr = ptr.sub(VECTOR_SIZE); + if let Some(i) = reverse_search2(start_ptr, end_ptr, ptr, vn1, vn2) { + return Some(i); + } + } + if ptr > start_ptr { + debug_assert!(sub(ptr, start_ptr) < VECTOR_SIZE); + return reverse_search2(start_ptr, end_ptr, start_ptr, vn1, vn2); + } + None +} + +#[target_feature(enable = "avx2")] +pub unsafe fn memrchr3( + n1: u8, + n2: u8, + n3: u8, + haystack: &[u8], +) -> Option { + let vn1 = _mm256_set1_epi8(n1 as i8); + let vn2 = _mm256_set1_epi8(n2 as i8); + let vn3 = _mm256_set1_epi8(n3 as i8); + let len = haystack.len(); + let loop_size = cmp::min(LOOP_SIZE2, len); + let start_ptr = haystack.as_ptr(); + let end_ptr = start_ptr.add(haystack.len()); + let mut ptr = end_ptr; + + if haystack.len() < VECTOR_SIZE { + while ptr > start_ptr { + ptr = ptr.offset(-1); + if *ptr == n1 || *ptr == n2 || *ptr == n3 { + return Some(sub(ptr, start_ptr)); + } + } + return None; + } + + ptr = ptr.sub(VECTOR_SIZE); + if let Some(i) = reverse_search3(start_ptr, end_ptr, ptr, vn1, vn2, vn3) { + return Some(i); + } + + ptr = (end_ptr as usize & !VECTOR_ALIGN) as *const u8; + debug_assert!(start_ptr <= ptr && ptr <= end_ptr); + while loop_size == LOOP_SIZE2 && ptr >= start_ptr.add(loop_size) { + debug_assert_eq!(0, (ptr as usize) % VECTOR_SIZE); + + ptr = ptr.sub(loop_size); + let a = _mm256_load_si256(ptr as *const __m256i); + let b = _mm256_load_si256(ptr.add(VECTOR_SIZE) as *const __m256i); + let eqa1 = _mm256_cmpeq_epi8(vn1, a); + let eqb1 = _mm256_cmpeq_epi8(vn1, b); + let eqa2 = _mm256_cmpeq_epi8(vn2, a); + let eqb2 = _mm256_cmpeq_epi8(vn2, b); + let eqa3 = _mm256_cmpeq_epi8(vn3, a); + let eqb3 = _mm256_cmpeq_epi8(vn3, b); + let or1 = _mm256_or_si256(eqa1, eqb1); + let or2 = _mm256_or_si256(eqa2, eqb2); + let or3 = _mm256_or_si256(eqa3, eqb3); + let or4 = _mm256_or_si256(or1, or2); + let or5 = _mm256_or_si256(or3, or4); + if _mm256_movemask_epi8(or5) != 0 { + let mut at = sub(ptr.add(VECTOR_SIZE), start_ptr); + let mask1 = _mm256_movemask_epi8(eqb1); + let mask2 = _mm256_movemask_epi8(eqb2); + let mask3 = _mm256_movemask_epi8(eqb3); + if mask1 != 0 || mask2 != 0 || mask3 != 0 { + return Some(at + reverse_pos3(mask1, mask2, mask3)); + } + + at -= VECTOR_SIZE; + let mask1 = _mm256_movemask_epi8(eqa1); + let mask2 = _mm256_movemask_epi8(eqa2); + let mask3 = _mm256_movemask_epi8(eqa3); + return Some(at + reverse_pos3(mask1, mask2, mask3)); + } + } + while ptr >= start_ptr.add(VECTOR_SIZE) { + ptr = ptr.sub(VECTOR_SIZE); + if let Some(i) = + reverse_search3(start_ptr, end_ptr, ptr, vn1, vn2, vn3) + { + return Some(i); + } + } + if ptr > start_ptr { + debug_assert!(sub(ptr, start_ptr) < VECTOR_SIZE); + return reverse_search3(start_ptr, end_ptr, start_ptr, vn1, vn2, vn3); + } + None +} + +#[target_feature(enable = "avx2")] +unsafe fn forward_search1( + start_ptr: *const u8, + end_ptr: *const u8, + ptr: *const u8, + vn1: __m256i, +) -> Option { + debug_assert!(sub(end_ptr, start_ptr) >= VECTOR_SIZE); + debug_assert!(start_ptr <= ptr); + debug_assert!(ptr <= end_ptr.sub(VECTOR_SIZE)); + + let chunk = _mm256_loadu_si256(ptr as *const __m256i); + let mask = _mm256_movemask_epi8(_mm256_cmpeq_epi8(chunk, vn1)); + if mask != 0 { + Some(sub(ptr, start_ptr) + forward_pos(mask)) + } else { + None + } +} + +#[target_feature(enable = "avx2")] +unsafe fn forward_search2( + start_ptr: *const u8, + end_ptr: *const u8, + ptr: *const u8, + vn1: __m256i, + vn2: __m256i, +) -> Option { + debug_assert!(sub(end_ptr, start_ptr) >= VECTOR_SIZE); + debug_assert!(start_ptr <= ptr); + debug_assert!(ptr <= end_ptr.sub(VECTOR_SIZE)); + + let chunk = _mm256_loadu_si256(ptr as *const __m256i); + let eq1 = _mm256_cmpeq_epi8(chunk, vn1); + let eq2 = _mm256_cmpeq_epi8(chunk, vn2); + if _mm256_movemask_epi8(_mm256_or_si256(eq1, eq2)) != 0 { + let mask1 = _mm256_movemask_epi8(eq1); + let mask2 = _mm256_movemask_epi8(eq2); + Some(sub(ptr, start_ptr) + forward_pos2(mask1, mask2)) + } else { + None + } +} + +#[target_feature(enable = "avx2")] +unsafe fn forward_search3( + start_ptr: *const u8, + end_ptr: *const u8, + ptr: *const u8, + vn1: __m256i, + vn2: __m256i, + vn3: __m256i, +) -> Option { + debug_assert!(sub(end_ptr, start_ptr) >= VECTOR_SIZE); + debug_assert!(start_ptr <= ptr); + debug_assert!(ptr <= end_ptr.sub(VECTOR_SIZE)); + + let chunk = _mm256_loadu_si256(ptr as *const __m256i); + let eq1 = _mm256_cmpeq_epi8(chunk, vn1); + let eq2 = _mm256_cmpeq_epi8(chunk, vn2); + let eq3 = _mm256_cmpeq_epi8(chunk, vn3); + let or = _mm256_or_si256(eq1, eq2); + if _mm256_movemask_epi8(_mm256_or_si256(or, eq3)) != 0 { + let mask1 = _mm256_movemask_epi8(eq1); + let mask2 = _mm256_movemask_epi8(eq2); + let mask3 = _mm256_movemask_epi8(eq3); + Some(sub(ptr, start_ptr) + forward_pos3(mask1, mask2, mask3)) + } else { + None + } +} + +#[target_feature(enable = "avx2")] +unsafe fn reverse_search1( + start_ptr: *const u8, + end_ptr: *const u8, + ptr: *const u8, + vn1: __m256i, +) -> Option { + debug_assert!(sub(end_ptr, start_ptr) >= VECTOR_SIZE); + debug_assert!(start_ptr <= ptr); + debug_assert!(ptr <= end_ptr.sub(VECTOR_SIZE)); + + let chunk = _mm256_loadu_si256(ptr as *const __m256i); + let mask = _mm256_movemask_epi8(_mm256_cmpeq_epi8(vn1, chunk)); + if mask != 0 { + Some(sub(ptr, start_ptr) + reverse_pos(mask)) + } else { + None + } +} + +#[target_feature(enable = "avx2")] +unsafe fn reverse_search2( + start_ptr: *const u8, + end_ptr: *const u8, + ptr: *const u8, + vn1: __m256i, + vn2: __m256i, +) -> Option { + debug_assert!(sub(end_ptr, start_ptr) >= VECTOR_SIZE); + debug_assert!(start_ptr <= ptr); + debug_assert!(ptr <= end_ptr.sub(VECTOR_SIZE)); + + let chunk = _mm256_loadu_si256(ptr as *const __m256i); + let eq1 = _mm256_cmpeq_epi8(chunk, vn1); + let eq2 = _mm256_cmpeq_epi8(chunk, vn2); + if _mm256_movemask_epi8(_mm256_or_si256(eq1, eq2)) != 0 { + let mask1 = _mm256_movemask_epi8(eq1); + let mask2 = _mm256_movemask_epi8(eq2); + Some(sub(ptr, start_ptr) + reverse_pos2(mask1, mask2)) + } else { + None + } +} + +#[target_feature(enable = "avx2")] +unsafe fn reverse_search3( + start_ptr: *const u8, + end_ptr: *const u8, + ptr: *const u8, + vn1: __m256i, + vn2: __m256i, + vn3: __m256i, +) -> Option { + debug_assert!(sub(end_ptr, start_ptr) >= VECTOR_SIZE); + debug_assert!(start_ptr <= ptr); + debug_assert!(ptr <= end_ptr.sub(VECTOR_SIZE)); + + let chunk = _mm256_loadu_si256(ptr as *const __m256i); + let eq1 = _mm256_cmpeq_epi8(chunk, vn1); + let eq2 = _mm256_cmpeq_epi8(chunk, vn2); + let eq3 = _mm256_cmpeq_epi8(chunk, vn3); + let or = _mm256_or_si256(eq1, eq2); + if _mm256_movemask_epi8(_mm256_or_si256(or, eq3)) != 0 { + let mask1 = _mm256_movemask_epi8(eq1); + let mask2 = _mm256_movemask_epi8(eq2); + let mask3 = _mm256_movemask_epi8(eq3); + Some(sub(ptr, start_ptr) + reverse_pos3(mask1, mask2, mask3)) + } else { + None + } +} + +/// Compute the position of the first matching byte from the given mask. The +/// position returned is always in the range [0, 31]. +/// +/// The mask given is expected to be the result of _mm256_movemask_epi8. +fn forward_pos(mask: i32) -> usize { + // We are dealing with little endian here, where the most significant byte + // is at a higher address. That means the least significant bit that is set + // corresponds to the position of our first matching byte. That position + // corresponds to the number of zeros after the least significant bit. + mask.trailing_zeros() as usize +} + +/// Compute the position of the first matching byte from the given masks. The +/// position returned is always in the range [0, 31]. Each mask corresponds to +/// the equality comparison of a single byte. +/// +/// The masks given are expected to be the result of _mm256_movemask_epi8, +/// where at least one of the masks is non-zero (i.e., indicates a match). +fn forward_pos2(mask1: i32, mask2: i32) -> usize { + debug_assert!(mask1 != 0 || mask2 != 0); + + forward_pos(mask1 | mask2) +} + +/// Compute the position of the first matching byte from the given masks. The +/// position returned is always in the range [0, 31]. Each mask corresponds to +/// the equality comparison of a single byte. +/// +/// The masks given are expected to be the result of _mm256_movemask_epi8, +/// where at least one of the masks is non-zero (i.e., indicates a match). +fn forward_pos3(mask1: i32, mask2: i32, mask3: i32) -> usize { + debug_assert!(mask1 != 0 || mask2 != 0 || mask3 != 0); + + forward_pos(mask1 | mask2 | mask3) +} + +/// Compute the position of the last matching byte from the given mask. The +/// position returned is always in the range [0, 31]. +/// +/// The mask given is expected to be the result of _mm256_movemask_epi8. +fn reverse_pos(mask: i32) -> usize { + // We are dealing with little endian here, where the most significant byte + // is at a higher address. That means the most significant bit that is set + // corresponds to the position of our last matching byte. The position from + // the end of the mask is therefore the number of leading zeros in a 32 + // bit integer, and the position from the start of the mask is therefore + // 32 - (leading zeros) - 1. + VECTOR_SIZE - (mask as u32).leading_zeros() as usize - 1 +} + +/// Compute the position of the last matching byte from the given masks. The +/// position returned is always in the range [0, 31]. Each mask corresponds to +/// the equality comparison of a single byte. +/// +/// The masks given are expected to be the result of _mm256_movemask_epi8, +/// where at least one of the masks is non-zero (i.e., indicates a match). +fn reverse_pos2(mask1: i32, mask2: i32) -> usize { + debug_assert!(mask1 != 0 || mask2 != 0); + + reverse_pos(mask1 | mask2) +} + +/// Compute the position of the last matching byte from the given masks. The +/// position returned is always in the range [0, 31]. Each mask corresponds to +/// the equality comparison of a single byte. +/// +/// The masks given are expected to be the result of _mm256_movemask_epi8, +/// where at least one of the masks is non-zero (i.e., indicates a match). +fn reverse_pos3(mask1: i32, mask2: i32, mask3: i32) -> usize { + debug_assert!(mask1 != 0 || mask2 != 0 || mask3 != 0); + + reverse_pos(mask1 | mask2 | mask3) +} + +/// Subtract `b` from `a` and return the difference. `a` should be greater than +/// or equal to `b`. +fn sub(a: *const u8, b: *const u8) -> usize { + debug_assert!(a >= b); + (a as usize) - (b as usize) +} diff --git a/crux-mir/lib/memchr/src/memchr/x86/mod.rs b/crux-mir/lib/memchr/src/memchr/x86/mod.rs new file mode 100644 index 000000000..aec35dbff --- /dev/null +++ b/crux-mir/lib/memchr/src/memchr/x86/mod.rs @@ -0,0 +1,148 @@ +use super::fallback; + +// We only use AVX when we can detect at runtime whether it's available, which +// requires std. +#[cfg(feature = "std")] +mod avx; +mod sse2; + +/// This macro employs a gcc-like "ifunc" trick where by upon first calling +/// `memchr` (for example), CPU feature detection will be performed at runtime +/// to determine the best implementation to use. After CPU feature detection +/// is done, we replace `memchr`'s function pointer with the selection. Upon +/// subsequent invocations, the CPU-specific routine is invoked directly, which +/// skips the CPU feature detection and subsequent branch that's required. +/// +/// While this typically doesn't matter for rare occurrences or when used on +/// larger haystacks, `memchr` can be called in tight loops where the overhead +/// of this branch can actually add up *and is measurable*. This trick was +/// necessary to bring this implementation up to glibc's speeds for the 'tiny' +/// benchmarks, for example. +/// +/// At some point, I expect the Rust ecosystem will get a nice macro for doing +/// exactly this, at which point, we can replace our hand-jammed version of it. +/// +/// N.B. The ifunc strategy does prevent function inlining of course, but +/// on modern CPUs, you'll probably end up with the AVX2 implementation, +/// which probably can't be inlined anyway---unless you've compiled your +/// entire program with AVX2 enabled. However, even then, the various memchr +/// implementations aren't exactly small, so inlining might not help anyway! +/// +/// # Safety +/// +/// Callers must ensure that fnty is function pointer type. +#[cfg(feature = "std")] +macro_rules! unsafe_ifunc { + ($fnty:ty, $name:ident, $haystack:ident, $($needle:ident),+) => {{ + use std::{mem, sync::atomic::{AtomicPtr, Ordering}}; + + type FnRaw = *mut (); + + static FN: AtomicPtr<()> = AtomicPtr::new(detect as FnRaw); + + fn detect($($needle: u8),+, haystack: &[u8]) -> Option { + let fun = + if cfg!(memchr_runtime_avx) && is_x86_feature_detected!("avx2") { + avx::$name as FnRaw + } else if cfg!(memchr_runtime_sse2) { + sse2::$name as FnRaw + } else { + fallback::$name as FnRaw + }; + FN.store(fun as FnRaw, Ordering::Relaxed); + // SAFETY: By virtue of the caller contract, $fnty is a function + // pointer, which is always safe to transmute with a *mut (). + // Also, if 'fun is the AVX routine, then it is guaranteed to be + // supported since we checked the avx2 feature. + unsafe { + mem::transmute::(fun)($($needle),+, haystack) + } + } + + // SAFETY: By virtue of the caller contract, $fnty is a function + // pointer, which is always safe to transmute with a *mut (). Also, if + // 'fun is the AVX routine, then it is guaranteed to be supported since + // we checked the avx2 feature. + unsafe { + let fun = FN.load(Ordering::Relaxed); + mem::transmute::(fun)($($needle),+, $haystack) + } + }} +} + +/// When std isn't available to provide runtime CPU feature detection, or if +/// runtime CPU feature detection has been explicitly disabled, then just +/// call our optimized SSE2 routine directly. SSE2 is avalbale on all x86_64 +/// targets, so no CPU feature detection is necessary. +/// +/// # Safety +/// +/// There are no safety requirements for this definition of the macro. It is +/// safe for all inputs since it is restricted to either the fallback routine +/// or the SSE routine, which is always safe to call on x86_64. +#[cfg(not(feature = "std"))] +macro_rules! unsafe_ifunc { + ($fnty:ty, $name:ident, $haystack:ident, $($needle:ident),+) => {{ + if cfg!(memchr_runtime_sse2) { + unsafe { sse2::$name($($needle),+, $haystack) } + } else { + fallback::$name($($needle),+, $haystack) + } + }} +} + +#[inline(always)] +pub fn memchr(n1: u8, haystack: &[u8]) -> Option { + unsafe_ifunc!(fn(u8, &[u8]) -> Option, memchr, haystack, n1) +} + +#[inline(always)] +pub fn memchr2(n1: u8, n2: u8, haystack: &[u8]) -> Option { + unsafe_ifunc!( + fn(u8, u8, &[u8]) -> Option, + memchr2, + haystack, + n1, + n2 + ) +} + +#[inline(always)] +pub fn memchr3(n1: u8, n2: u8, n3: u8, haystack: &[u8]) -> Option { + unsafe_ifunc!( + fn(u8, u8, u8, &[u8]) -> Option, + memchr3, + haystack, + n1, + n2, + n3 + ) +} + +#[inline(always)] +pub fn memrchr(n1: u8, haystack: &[u8]) -> Option { + unsafe_ifunc!(fn(u8, &[u8]) -> Option, memrchr, haystack, n1) +} + +#[inline(always)] +pub fn memrchr2(n1: u8, n2: u8, haystack: &[u8]) -> Option { + unsafe_ifunc!( + fn(u8, u8, &[u8]) -> Option, + memrchr2, + haystack, + n1, + n2 + ) +} + +#[inline(always)] +pub fn memrchr3(n1: u8, n2: u8, n3: u8, haystack: &[u8]) -> Option { + unsafe_ifunc!( + fn(u8, u8, u8, &[u8]) -> Option, + memrchr3, + haystack, + n1, + n2, + n3 + ) +} diff --git a/crux-mir/lib/memchr/src/memchr/x86/sse2.rs b/crux-mir/lib/memchr/src/memchr/x86/sse2.rs new file mode 100644 index 000000000..b7b3a9328 --- /dev/null +++ b/crux-mir/lib/memchr/src/memchr/x86/sse2.rs @@ -0,0 +1,791 @@ +use core::{arch::x86_64::*, cmp, mem::size_of}; + +const VECTOR_SIZE: usize = size_of::<__m128i>(); +const VECTOR_ALIGN: usize = VECTOR_SIZE - 1; + +// The number of bytes to loop at in one iteration of memchr/memrchr. +const LOOP_SIZE: usize = 4 * VECTOR_SIZE; + +// The number of bytes to loop at in one iteration of memchr2/memrchr2 and +// memchr3/memrchr3. There was no observable difference between 64 and 32 bytes +// in benchmarks. memchr3 in particular only gets a very slight speed up from +// the loop unrolling. +const LOOP_SIZE2: usize = 2 * VECTOR_SIZE; + +#[target_feature(enable = "sse2")] +pub unsafe fn memchr(n1: u8, haystack: &[u8]) -> Option { + // What follows is a fast SSE2-only algorithm to detect the position of + // `n1` in `haystack` if it exists. From what I know, this is the "classic" + // algorithm. I believe it can be found in places like glibc and Go's + // standard library. It appears to be well known and is elaborated on in + // more detail here: https://gms.tf/stdfind-and-memchr-optimizations.html + // + // While this routine is very long, the basic idea is actually very simple + // and can be expressed straight-forwardly in pseudo code: + // + // needle = (n1 << 15) | (n1 << 14) | ... | (n1 << 1) | n1 + // // Note: shift amount in bytes + // + // while i <= haystack.len() - 16: + // // A 16 byte vector. Each byte in chunk corresponds to a byte in + // // the haystack. + // chunk = haystack[i:i+16] + // // Compare bytes in needle with bytes in chunk. The result is a 16 + // // byte chunk where each byte is 0xFF if the corresponding bytes + // // in needle and chunk were equal, or 0x00 otherwise. + // eqs = cmpeq(needle, chunk) + // // Return a 32 bit integer where the most significant 16 bits + // // are always 0 and the lower 16 bits correspond to whether the + // // most significant bit in the correspond byte in `eqs` is set. + // // In other words, `mask as u16` has bit i set if and only if + // // needle[i] == chunk[i]. + // mask = movemask(eqs) + // + // // Mask is 0 if there is no match, and non-zero otherwise. + // if mask != 0: + // // trailing_zeros tells us the position of the least significant + // // bit that is set. + // return i + trailing_zeros(mask) + // + // // haystack length may not be a multiple of 16, so search the rest. + // while i < haystack.len(): + // if haystack[i] == n1: + // return i + // + // // No match found. + // return NULL + // + // In fact, we could loosely translate the above code to Rust line-for-line + // and it would be a pretty fast algorithm. But, we pull out all the stops + // to go as fast as possible: + // + // 1. We use aligned loads. That is, we do some finagling to make sure our + // primary loop not only proceeds in increments of 16 bytes, but that + // the address of haystack's pointer that we dereference is aligned to + // 16 bytes. 16 is a magic number here because it is the size of SSE2 + // 128-bit vector. (For the AVX2 algorithm, 32 is the magic number.) + // Therefore, to get aligned loads, our pointer's address must be evenly + // divisible by 16. + // 2. Our primary loop proceeds 64 bytes at a time instead of 16. It's + // kind of like loop unrolling, but we combine the equality comparisons + // using a vector OR such that we only need to extract a single mask to + // determine whether a match exists or not. If so, then we do some + // book-keeping to determine the precise location but otherwise mush on. + // 3. We use our "chunk" comparison routine in as many places as possible, + // even if it means using unaligned loads. In particular, if haystack + // starts with an unaligned address, then we do an unaligned load to + // search the first 16 bytes. We then start our primary loop at the + // smallest subsequent aligned address, which will actually overlap with + // previously searched bytes. But we're OK with that. We do a similar + // dance at the end of our primary loop. Finally, to avoid a + // byte-at-a-time loop at the end, we do a final 16 byte unaligned load + // that may overlap with a previous load. This is OK because it converts + // a loop into a small number of very fast vector instructions. + // + // The primary downside of this algorithm is that it's effectively + // completely unsafe. Therefore, we have to be super careful to avoid + // undefined behavior: + // + // 1. We use raw pointers everywhere. Not only does dereferencing a pointer + // require the pointer to be valid, but we actually can't even store the + // address of an invalid pointer (unless it's 1 past the end of + // haystack) without sacrificing performance. + // 2. _mm_loadu_si128 is used when you don't care about alignment, and + // _mm_load_si128 is used when you do care. You cannot use the latter + // on unaligned pointers. + // 3. We make liberal use of debug_assert! to check assumptions. + // 4. We make a concerted effort to stick with pointers instead of indices. + // Indices are nicer because there's less to worry about with them (see + // above about pointer offsets), but I could not get the compiler to + // produce as good of code as what the below produces. In any case, + // pointers are what we really care about here, and alignment is + // expressed a bit more naturally with them. + // + // In general, most of the algorithms in this crate have a similar + // structure to what you see below, so this comment applies fairly well to + // all of them. + + let vn1 = _mm_set1_epi8(n1 as i8); + let len = haystack.len(); + let loop_size = cmp::min(LOOP_SIZE, len); + let start_ptr = haystack.as_ptr(); + let end_ptr = start_ptr.add(haystack.len()); + let mut ptr = start_ptr; + + if haystack.len() < VECTOR_SIZE { + while ptr < end_ptr { + if *ptr == n1 { + return Some(sub(ptr, start_ptr)); + } + ptr = ptr.offset(1); + } + return None; + } + + if let Some(i) = forward_search1(start_ptr, end_ptr, ptr, vn1) { + return Some(i); + } + + ptr = ptr.add(VECTOR_SIZE - (start_ptr as usize & VECTOR_ALIGN)); + debug_assert!(ptr > start_ptr && end_ptr.sub(VECTOR_SIZE) >= start_ptr); + while loop_size == LOOP_SIZE && ptr <= end_ptr.sub(loop_size) { + debug_assert_eq!(0, (ptr as usize) % VECTOR_SIZE); + + let a = _mm_load_si128(ptr as *const __m128i); + let b = _mm_load_si128(ptr.add(VECTOR_SIZE) as *const __m128i); + let c = _mm_load_si128(ptr.add(2 * VECTOR_SIZE) as *const __m128i); + let d = _mm_load_si128(ptr.add(3 * VECTOR_SIZE) as *const __m128i); + let eqa = _mm_cmpeq_epi8(vn1, a); + let eqb = _mm_cmpeq_epi8(vn1, b); + let eqc = _mm_cmpeq_epi8(vn1, c); + let eqd = _mm_cmpeq_epi8(vn1, d); + let or1 = _mm_or_si128(eqa, eqb); + let or2 = _mm_or_si128(eqc, eqd); + let or3 = _mm_or_si128(or1, or2); + if _mm_movemask_epi8(or3) != 0 { + let mut at = sub(ptr, start_ptr); + let mask = _mm_movemask_epi8(eqa); + if mask != 0 { + return Some(at + forward_pos(mask)); + } + + at += VECTOR_SIZE; + let mask = _mm_movemask_epi8(eqb); + if mask != 0 { + return Some(at + forward_pos(mask)); + } + + at += VECTOR_SIZE; + let mask = _mm_movemask_epi8(eqc); + if mask != 0 { + return Some(at + forward_pos(mask)); + } + + at += VECTOR_SIZE; + let mask = _mm_movemask_epi8(eqd); + debug_assert!(mask != 0); + return Some(at + forward_pos(mask)); + } + ptr = ptr.add(loop_size); + } + while ptr <= end_ptr.sub(VECTOR_SIZE) { + debug_assert!(sub(end_ptr, ptr) >= VECTOR_SIZE); + + if let Some(i) = forward_search1(start_ptr, end_ptr, ptr, vn1) { + return Some(i); + } + ptr = ptr.add(VECTOR_SIZE); + } + if ptr < end_ptr { + debug_assert!(sub(end_ptr, ptr) < VECTOR_SIZE); + ptr = ptr.sub(VECTOR_SIZE - sub(end_ptr, ptr)); + debug_assert_eq!(sub(end_ptr, ptr), VECTOR_SIZE); + + return forward_search1(start_ptr, end_ptr, ptr, vn1); + } + None +} + +#[target_feature(enable = "sse2")] +pub unsafe fn memchr2(n1: u8, n2: u8, haystack: &[u8]) -> Option { + let vn1 = _mm_set1_epi8(n1 as i8); + let vn2 = _mm_set1_epi8(n2 as i8); + let len = haystack.len(); + let loop_size = cmp::min(LOOP_SIZE2, len); + let start_ptr = haystack.as_ptr(); + let end_ptr = start_ptr.add(haystack.len()); + let mut ptr = start_ptr; + + if haystack.len() < VECTOR_SIZE { + while ptr < end_ptr { + if *ptr == n1 || *ptr == n2 { + return Some(sub(ptr, start_ptr)); + } + ptr = ptr.offset(1); + } + return None; + } + + if let Some(i) = forward_search2(start_ptr, end_ptr, ptr, vn1, vn2) { + return Some(i); + } + + ptr = ptr.add(VECTOR_SIZE - (start_ptr as usize & VECTOR_ALIGN)); + debug_assert!(ptr > start_ptr && end_ptr.sub(VECTOR_SIZE) >= start_ptr); + while loop_size == LOOP_SIZE2 && ptr <= end_ptr.sub(loop_size) { + debug_assert_eq!(0, (ptr as usize) % VECTOR_SIZE); + + let a = _mm_load_si128(ptr as *const __m128i); + let b = _mm_load_si128(ptr.add(VECTOR_SIZE) as *const __m128i); + let eqa1 = _mm_cmpeq_epi8(vn1, a); + let eqb1 = _mm_cmpeq_epi8(vn1, b); + let eqa2 = _mm_cmpeq_epi8(vn2, a); + let eqb2 = _mm_cmpeq_epi8(vn2, b); + let or1 = _mm_or_si128(eqa1, eqb1); + let or2 = _mm_or_si128(eqa2, eqb2); + let or3 = _mm_or_si128(or1, or2); + if _mm_movemask_epi8(or3) != 0 { + let mut at = sub(ptr, start_ptr); + let mask1 = _mm_movemask_epi8(eqa1); + let mask2 = _mm_movemask_epi8(eqa2); + if mask1 != 0 || mask2 != 0 { + return Some(at + forward_pos2(mask1, mask2)); + } + + at += VECTOR_SIZE; + let mask1 = _mm_movemask_epi8(eqb1); + let mask2 = _mm_movemask_epi8(eqb2); + return Some(at + forward_pos2(mask1, mask2)); + } + ptr = ptr.add(loop_size); + } + while ptr <= end_ptr.sub(VECTOR_SIZE) { + if let Some(i) = forward_search2(start_ptr, end_ptr, ptr, vn1, vn2) { + return Some(i); + } + ptr = ptr.add(VECTOR_SIZE); + } + if ptr < end_ptr { + debug_assert!(sub(end_ptr, ptr) < VECTOR_SIZE); + ptr = ptr.sub(VECTOR_SIZE - sub(end_ptr, ptr)); + debug_assert_eq!(sub(end_ptr, ptr), VECTOR_SIZE); + + return forward_search2(start_ptr, end_ptr, ptr, vn1, vn2); + } + None +} + +#[target_feature(enable = "sse2")] +pub unsafe fn memchr3( + n1: u8, + n2: u8, + n3: u8, + haystack: &[u8], +) -> Option { + let vn1 = _mm_set1_epi8(n1 as i8); + let vn2 = _mm_set1_epi8(n2 as i8); + let vn3 = _mm_set1_epi8(n3 as i8); + let len = haystack.len(); + let loop_size = cmp::min(LOOP_SIZE2, len); + let start_ptr = haystack.as_ptr(); + let end_ptr = start_ptr.add(haystack.len()); + let mut ptr = start_ptr; + + if haystack.len() < VECTOR_SIZE { + while ptr < end_ptr { + if *ptr == n1 || *ptr == n2 || *ptr == n3 { + return Some(sub(ptr, start_ptr)); + } + ptr = ptr.offset(1); + } + return None; + } + + if let Some(i) = forward_search3(start_ptr, end_ptr, ptr, vn1, vn2, vn3) { + return Some(i); + } + + ptr = ptr.add(VECTOR_SIZE - (start_ptr as usize & VECTOR_ALIGN)); + debug_assert!(ptr > start_ptr && end_ptr.sub(VECTOR_SIZE) >= start_ptr); + while loop_size == LOOP_SIZE2 && ptr <= end_ptr.sub(loop_size) { + debug_assert_eq!(0, (ptr as usize) % VECTOR_SIZE); + + let a = _mm_load_si128(ptr as *const __m128i); + let b = _mm_load_si128(ptr.add(VECTOR_SIZE) as *const __m128i); + let eqa1 = _mm_cmpeq_epi8(vn1, a); + let eqb1 = _mm_cmpeq_epi8(vn1, b); + let eqa2 = _mm_cmpeq_epi8(vn2, a); + let eqb2 = _mm_cmpeq_epi8(vn2, b); + let eqa3 = _mm_cmpeq_epi8(vn3, a); + let eqb3 = _mm_cmpeq_epi8(vn3, b); + let or1 = _mm_or_si128(eqa1, eqb1); + let or2 = _mm_or_si128(eqa2, eqb2); + let or3 = _mm_or_si128(eqa3, eqb3); + let or4 = _mm_or_si128(or1, or2); + let or5 = _mm_or_si128(or3, or4); + if _mm_movemask_epi8(or5) != 0 { + let mut at = sub(ptr, start_ptr); + let mask1 = _mm_movemask_epi8(eqa1); + let mask2 = _mm_movemask_epi8(eqa2); + let mask3 = _mm_movemask_epi8(eqa3); + if mask1 != 0 || mask2 != 0 || mask3 != 0 { + return Some(at + forward_pos3(mask1, mask2, mask3)); + } + + at += VECTOR_SIZE; + let mask1 = _mm_movemask_epi8(eqb1); + let mask2 = _mm_movemask_epi8(eqb2); + let mask3 = _mm_movemask_epi8(eqb3); + return Some(at + forward_pos3(mask1, mask2, mask3)); + } + ptr = ptr.add(loop_size); + } + while ptr <= end_ptr.sub(VECTOR_SIZE) { + if let Some(i) = + forward_search3(start_ptr, end_ptr, ptr, vn1, vn2, vn3) + { + return Some(i); + } + ptr = ptr.add(VECTOR_SIZE); + } + if ptr < end_ptr { + debug_assert!(sub(end_ptr, ptr) < VECTOR_SIZE); + ptr = ptr.sub(VECTOR_SIZE - sub(end_ptr, ptr)); + debug_assert_eq!(sub(end_ptr, ptr), VECTOR_SIZE); + + return forward_search3(start_ptr, end_ptr, ptr, vn1, vn2, vn3); + } + None +} + +#[target_feature(enable = "sse2")] +pub unsafe fn memrchr(n1: u8, haystack: &[u8]) -> Option { + let vn1 = _mm_set1_epi8(n1 as i8); + let len = haystack.len(); + let loop_size = cmp::min(LOOP_SIZE, len); + let start_ptr = haystack.as_ptr(); + let end_ptr = start_ptr.add(haystack.len()); + let mut ptr = end_ptr; + + if haystack.len() < VECTOR_SIZE { + while ptr > start_ptr { + ptr = ptr.offset(-1); + if *ptr == n1 { + return Some(sub(ptr, start_ptr)); + } + } + return None; + } + + ptr = ptr.sub(VECTOR_SIZE); + if let Some(i) = reverse_search1(start_ptr, end_ptr, ptr, vn1) { + return Some(i); + } + + ptr = (end_ptr as usize & !VECTOR_ALIGN) as *const u8; + debug_assert!(start_ptr <= ptr && ptr <= end_ptr); + while loop_size == LOOP_SIZE && ptr >= start_ptr.add(loop_size) { + debug_assert_eq!(0, (ptr as usize) % VECTOR_SIZE); + + ptr = ptr.sub(loop_size); + let a = _mm_load_si128(ptr as *const __m128i); + let b = _mm_load_si128(ptr.add(VECTOR_SIZE) as *const __m128i); + let c = _mm_load_si128(ptr.add(2 * VECTOR_SIZE) as *const __m128i); + let d = _mm_load_si128(ptr.add(3 * VECTOR_SIZE) as *const __m128i); + let eqa = _mm_cmpeq_epi8(vn1, a); + let eqb = _mm_cmpeq_epi8(vn1, b); + let eqc = _mm_cmpeq_epi8(vn1, c); + let eqd = _mm_cmpeq_epi8(vn1, d); + let or1 = _mm_or_si128(eqa, eqb); + let or2 = _mm_or_si128(eqc, eqd); + let or3 = _mm_or_si128(or1, or2); + if _mm_movemask_epi8(or3) != 0 { + let mut at = sub(ptr.add(3 * VECTOR_SIZE), start_ptr); + let mask = _mm_movemask_epi8(eqd); + if mask != 0 { + return Some(at + reverse_pos(mask)); + } + + at -= VECTOR_SIZE; + let mask = _mm_movemask_epi8(eqc); + if mask != 0 { + return Some(at + reverse_pos(mask)); + } + + at -= VECTOR_SIZE; + let mask = _mm_movemask_epi8(eqb); + if mask != 0 { + return Some(at + reverse_pos(mask)); + } + + at -= VECTOR_SIZE; + let mask = _mm_movemask_epi8(eqa); + debug_assert!(mask != 0); + return Some(at + reverse_pos(mask)); + } + } + while ptr >= start_ptr.add(VECTOR_SIZE) { + ptr = ptr.sub(VECTOR_SIZE); + if let Some(i) = reverse_search1(start_ptr, end_ptr, ptr, vn1) { + return Some(i); + } + } + if ptr > start_ptr { + debug_assert!(sub(ptr, start_ptr) < VECTOR_SIZE); + return reverse_search1(start_ptr, end_ptr, start_ptr, vn1); + } + None +} + +#[target_feature(enable = "sse2")] +pub unsafe fn memrchr2(n1: u8, n2: u8, haystack: &[u8]) -> Option { + let vn1 = _mm_set1_epi8(n1 as i8); + let vn2 = _mm_set1_epi8(n2 as i8); + let len = haystack.len(); + let loop_size = cmp::min(LOOP_SIZE2, len); + let start_ptr = haystack.as_ptr(); + let end_ptr = start_ptr.add(haystack.len()); + let mut ptr = end_ptr; + + if haystack.len() < VECTOR_SIZE { + while ptr > start_ptr { + ptr = ptr.offset(-1); + if *ptr == n1 || *ptr == n2 { + return Some(sub(ptr, start_ptr)); + } + } + return None; + } + + ptr = ptr.sub(VECTOR_SIZE); + if let Some(i) = reverse_search2(start_ptr, end_ptr, ptr, vn1, vn2) { + return Some(i); + } + + ptr = (end_ptr as usize & !VECTOR_ALIGN) as *const u8; + debug_assert!(start_ptr <= ptr && ptr <= end_ptr); + while loop_size == LOOP_SIZE2 && ptr >= start_ptr.add(loop_size) { + debug_assert_eq!(0, (ptr as usize) % VECTOR_SIZE); + + ptr = ptr.sub(loop_size); + let a = _mm_load_si128(ptr as *const __m128i); + let b = _mm_load_si128(ptr.add(VECTOR_SIZE) as *const __m128i); + let eqa1 = _mm_cmpeq_epi8(vn1, a); + let eqb1 = _mm_cmpeq_epi8(vn1, b); + let eqa2 = _mm_cmpeq_epi8(vn2, a); + let eqb2 = _mm_cmpeq_epi8(vn2, b); + let or1 = _mm_or_si128(eqa1, eqb1); + let or2 = _mm_or_si128(eqa2, eqb2); + let or3 = _mm_or_si128(or1, or2); + if _mm_movemask_epi8(or3) != 0 { + let mut at = sub(ptr.add(VECTOR_SIZE), start_ptr); + let mask1 = _mm_movemask_epi8(eqb1); + let mask2 = _mm_movemask_epi8(eqb2); + if mask1 != 0 || mask2 != 0 { + return Some(at + reverse_pos2(mask1, mask2)); + } + + at -= VECTOR_SIZE; + let mask1 = _mm_movemask_epi8(eqa1); + let mask2 = _mm_movemask_epi8(eqa2); + return Some(at + reverse_pos2(mask1, mask2)); + } + } + while ptr >= start_ptr.add(VECTOR_SIZE) { + ptr = ptr.sub(VECTOR_SIZE); + if let Some(i) = reverse_search2(start_ptr, end_ptr, ptr, vn1, vn2) { + return Some(i); + } + } + if ptr > start_ptr { + debug_assert!(sub(ptr, start_ptr) < VECTOR_SIZE); + return reverse_search2(start_ptr, end_ptr, start_ptr, vn1, vn2); + } + None +} + +#[target_feature(enable = "sse2")] +pub unsafe fn memrchr3( + n1: u8, + n2: u8, + n3: u8, + haystack: &[u8], +) -> Option { + let vn1 = _mm_set1_epi8(n1 as i8); + let vn2 = _mm_set1_epi8(n2 as i8); + let vn3 = _mm_set1_epi8(n3 as i8); + let len = haystack.len(); + let loop_size = cmp::min(LOOP_SIZE2, len); + let start_ptr = haystack.as_ptr(); + let end_ptr = start_ptr.add(haystack.len()); + let mut ptr = end_ptr; + + if haystack.len() < VECTOR_SIZE { + while ptr > start_ptr { + ptr = ptr.offset(-1); + if *ptr == n1 || *ptr == n2 || *ptr == n3 { + return Some(sub(ptr, start_ptr)); + } + } + return None; + } + + ptr = ptr.sub(VECTOR_SIZE); + if let Some(i) = reverse_search3(start_ptr, end_ptr, ptr, vn1, vn2, vn3) { + return Some(i); + } + + ptr = (end_ptr as usize & !VECTOR_ALIGN) as *const u8; + debug_assert!(start_ptr <= ptr && ptr <= end_ptr); + while loop_size == LOOP_SIZE2 && ptr >= start_ptr.add(loop_size) { + debug_assert_eq!(0, (ptr as usize) % VECTOR_SIZE); + + ptr = ptr.sub(loop_size); + let a = _mm_load_si128(ptr as *const __m128i); + let b = _mm_load_si128(ptr.add(VECTOR_SIZE) as *const __m128i); + let eqa1 = _mm_cmpeq_epi8(vn1, a); + let eqb1 = _mm_cmpeq_epi8(vn1, b); + let eqa2 = _mm_cmpeq_epi8(vn2, a); + let eqb2 = _mm_cmpeq_epi8(vn2, b); + let eqa3 = _mm_cmpeq_epi8(vn3, a); + let eqb3 = _mm_cmpeq_epi8(vn3, b); + let or1 = _mm_or_si128(eqa1, eqb1); + let or2 = _mm_or_si128(eqa2, eqb2); + let or3 = _mm_or_si128(eqa3, eqb3); + let or4 = _mm_or_si128(or1, or2); + let or5 = _mm_or_si128(or3, or4); + if _mm_movemask_epi8(or5) != 0 { + let mut at = sub(ptr.add(VECTOR_SIZE), start_ptr); + let mask1 = _mm_movemask_epi8(eqb1); + let mask2 = _mm_movemask_epi8(eqb2); + let mask3 = _mm_movemask_epi8(eqb3); + if mask1 != 0 || mask2 != 0 || mask3 != 0 { + return Some(at + reverse_pos3(mask1, mask2, mask3)); + } + + at -= VECTOR_SIZE; + let mask1 = _mm_movemask_epi8(eqa1); + let mask2 = _mm_movemask_epi8(eqa2); + let mask3 = _mm_movemask_epi8(eqa3); + return Some(at + reverse_pos3(mask1, mask2, mask3)); + } + } + while ptr >= start_ptr.add(VECTOR_SIZE) { + ptr = ptr.sub(VECTOR_SIZE); + if let Some(i) = + reverse_search3(start_ptr, end_ptr, ptr, vn1, vn2, vn3) + { + return Some(i); + } + } + if ptr > start_ptr { + debug_assert!(sub(ptr, start_ptr) < VECTOR_SIZE); + return reverse_search3(start_ptr, end_ptr, start_ptr, vn1, vn2, vn3); + } + None +} + +#[target_feature(enable = "sse2")] +pub unsafe fn forward_search1( + start_ptr: *const u8, + end_ptr: *const u8, + ptr: *const u8, + vn1: __m128i, +) -> Option { + debug_assert!(sub(end_ptr, start_ptr) >= VECTOR_SIZE); + debug_assert!(start_ptr <= ptr); + debug_assert!(ptr <= end_ptr.sub(VECTOR_SIZE)); + + let chunk = _mm_loadu_si128(ptr as *const __m128i); + let mask = _mm_movemask_epi8(_mm_cmpeq_epi8(chunk, vn1)); + if mask != 0 { + Some(sub(ptr, start_ptr) + forward_pos(mask)) + } else { + None + } +} + +#[target_feature(enable = "sse2")] +unsafe fn forward_search2( + start_ptr: *const u8, + end_ptr: *const u8, + ptr: *const u8, + vn1: __m128i, + vn2: __m128i, +) -> Option { + debug_assert!(sub(end_ptr, start_ptr) >= VECTOR_SIZE); + debug_assert!(start_ptr <= ptr); + debug_assert!(ptr <= end_ptr.sub(VECTOR_SIZE)); + + let chunk = _mm_loadu_si128(ptr as *const __m128i); + let eq1 = _mm_cmpeq_epi8(chunk, vn1); + let eq2 = _mm_cmpeq_epi8(chunk, vn2); + if _mm_movemask_epi8(_mm_or_si128(eq1, eq2)) != 0 { + let mask1 = _mm_movemask_epi8(eq1); + let mask2 = _mm_movemask_epi8(eq2); + Some(sub(ptr, start_ptr) + forward_pos2(mask1, mask2)) + } else { + None + } +} + +#[target_feature(enable = "sse2")] +pub unsafe fn forward_search3( + start_ptr: *const u8, + end_ptr: *const u8, + ptr: *const u8, + vn1: __m128i, + vn2: __m128i, + vn3: __m128i, +) -> Option { + debug_assert!(sub(end_ptr, start_ptr) >= VECTOR_SIZE); + debug_assert!(start_ptr <= ptr); + debug_assert!(ptr <= end_ptr.sub(VECTOR_SIZE)); + + let chunk = _mm_loadu_si128(ptr as *const __m128i); + let eq1 = _mm_cmpeq_epi8(chunk, vn1); + let eq2 = _mm_cmpeq_epi8(chunk, vn2); + let eq3 = _mm_cmpeq_epi8(chunk, vn3); + let or = _mm_or_si128(eq1, eq2); + if _mm_movemask_epi8(_mm_or_si128(or, eq3)) != 0 { + let mask1 = _mm_movemask_epi8(eq1); + let mask2 = _mm_movemask_epi8(eq2); + let mask3 = _mm_movemask_epi8(eq3); + Some(sub(ptr, start_ptr) + forward_pos3(mask1, mask2, mask3)) + } else { + None + } +} + +#[target_feature(enable = "sse2")] +unsafe fn reverse_search1( + start_ptr: *const u8, + end_ptr: *const u8, + ptr: *const u8, + vn1: __m128i, +) -> Option { + debug_assert!(sub(end_ptr, start_ptr) >= VECTOR_SIZE); + debug_assert!(start_ptr <= ptr); + debug_assert!(ptr <= end_ptr.sub(VECTOR_SIZE)); + + let chunk = _mm_loadu_si128(ptr as *const __m128i); + let mask = _mm_movemask_epi8(_mm_cmpeq_epi8(vn1, chunk)); + if mask != 0 { + Some(sub(ptr, start_ptr) + reverse_pos(mask)) + } else { + None + } +} + +#[target_feature(enable = "sse2")] +unsafe fn reverse_search2( + start_ptr: *const u8, + end_ptr: *const u8, + ptr: *const u8, + vn1: __m128i, + vn2: __m128i, +) -> Option { + debug_assert!(sub(end_ptr, start_ptr) >= VECTOR_SIZE); + debug_assert!(start_ptr <= ptr); + debug_assert!(ptr <= end_ptr.sub(VECTOR_SIZE)); + + let chunk = _mm_loadu_si128(ptr as *const __m128i); + let eq1 = _mm_cmpeq_epi8(chunk, vn1); + let eq2 = _mm_cmpeq_epi8(chunk, vn2); + if _mm_movemask_epi8(_mm_or_si128(eq1, eq2)) != 0 { + let mask1 = _mm_movemask_epi8(eq1); + let mask2 = _mm_movemask_epi8(eq2); + Some(sub(ptr, start_ptr) + reverse_pos2(mask1, mask2)) + } else { + None + } +} + +#[target_feature(enable = "sse2")] +unsafe fn reverse_search3( + start_ptr: *const u8, + end_ptr: *const u8, + ptr: *const u8, + vn1: __m128i, + vn2: __m128i, + vn3: __m128i, +) -> Option { + debug_assert!(sub(end_ptr, start_ptr) >= VECTOR_SIZE); + debug_assert!(start_ptr <= ptr); + debug_assert!(ptr <= end_ptr.sub(VECTOR_SIZE)); + + let chunk = _mm_loadu_si128(ptr as *const __m128i); + let eq1 = _mm_cmpeq_epi8(chunk, vn1); + let eq2 = _mm_cmpeq_epi8(chunk, vn2); + let eq3 = _mm_cmpeq_epi8(chunk, vn3); + let or = _mm_or_si128(eq1, eq2); + if _mm_movemask_epi8(_mm_or_si128(or, eq3)) != 0 { + let mask1 = _mm_movemask_epi8(eq1); + let mask2 = _mm_movemask_epi8(eq2); + let mask3 = _mm_movemask_epi8(eq3); + Some(sub(ptr, start_ptr) + reverse_pos3(mask1, mask2, mask3)) + } else { + None + } +} + +/// Compute the position of the first matching byte from the given mask. The +/// position returned is always in the range [0, 15]. +/// +/// The mask given is expected to be the result of _mm_movemask_epi8. +fn forward_pos(mask: i32) -> usize { + // We are dealing with little endian here, where the most significant byte + // is at a higher address. That means the least significant bit that is set + // corresponds to the position of our first matching byte. That position + // corresponds to the number of zeros after the least significant bit. + mask.trailing_zeros() as usize +} + +/// Compute the position of the first matching byte from the given masks. The +/// position returned is always in the range [0, 15]. Each mask corresponds to +/// the equality comparison of a single byte. +/// +/// The masks given are expected to be the result of _mm_movemask_epi8, where +/// at least one of the masks is non-zero (i.e., indicates a match). +fn forward_pos2(mask1: i32, mask2: i32) -> usize { + debug_assert!(mask1 != 0 || mask2 != 0); + + forward_pos(mask1 | mask2) +} + +/// Compute the position of the first matching byte from the given masks. The +/// position returned is always in the range [0, 15]. Each mask corresponds to +/// the equality comparison of a single byte. +/// +/// The masks given are expected to be the result of _mm_movemask_epi8, where +/// at least one of the masks is non-zero (i.e., indicates a match). +fn forward_pos3(mask1: i32, mask2: i32, mask3: i32) -> usize { + debug_assert!(mask1 != 0 || mask2 != 0 || mask3 != 0); + + forward_pos(mask1 | mask2 | mask3) +} + +/// Compute the position of the last matching byte from the given mask. The +/// position returned is always in the range [0, 15]. +/// +/// The mask given is expected to be the result of _mm_movemask_epi8. +fn reverse_pos(mask: i32) -> usize { + // We are dealing with little endian here, where the most significant byte + // is at a higher address. That means the most significant bit that is set + // corresponds to the position of our last matching byte. The position from + // the end of the mask is therefore the number of leading zeros in a 16 + // bit integer, and the position from the start of the mask is therefore + // 16 - (leading zeros) - 1. + VECTOR_SIZE - (mask as u16).leading_zeros() as usize - 1 +} + +/// Compute the position of the last matching byte from the given masks. The +/// position returned is always in the range [0, 15]. Each mask corresponds to +/// the equality comparison of a single byte. +/// +/// The masks given are expected to be the result of _mm_movemask_epi8, where +/// at least one of the masks is non-zero (i.e., indicates a match). +fn reverse_pos2(mask1: i32, mask2: i32) -> usize { + debug_assert!(mask1 != 0 || mask2 != 0); + + reverse_pos(mask1 | mask2) +} + +/// Compute the position of the last matching byte from the given masks. The +/// position returned is always in the range [0, 15]. Each mask corresponds to +/// the equality comparison of a single byte. +/// +/// The masks given are expected to be the result of _mm_movemask_epi8, where +/// at least one of the masks is non-zero (i.e., indicates a match). +fn reverse_pos3(mask1: i32, mask2: i32, mask3: i32) -> usize { + debug_assert!(mask1 != 0 || mask2 != 0 || mask3 != 0); + + reverse_pos(mask1 | mask2 | mask3) +} + +/// Subtract `b` from `a` and return the difference. `a` should be greater than +/// or equal to `b`. +fn sub(a: *const u8, b: *const u8) -> usize { + debug_assert!(a >= b); + (a as usize) - (b as usize) +} diff --git a/crux-mir/lib/memchr/src/memchr/x86/sse42.rs b/crux-mir/lib/memchr/src/memchr/x86/sse42.rs new file mode 100644 index 000000000..da38e50c2 --- /dev/null +++ b/crux-mir/lib/memchr/src/memchr/x86/sse42.rs @@ -0,0 +1,72 @@ +// This code is unused. PCMPESTRI is gratuitously slow. I imagine it might +// start winning with a hypothetical memchr4 (or greater). This technique might +// also be good for exposing searches over ranges of bytes, but that departs +// from the standard memchr API, so it's not clear whether we actually want +// that or not. +// +// N.B. PCMPISTRI appears to be about twice as fast as PCMPESTRI, which is kind +// of neat. Unfortunately, UTF-8 strings can contain NUL bytes, which means +// I don't see a way of effectively using PCMPISTRI unless there's some fast +// way to replace zero bytes with a byte that is not not a needle byte. + +use core::{arch::x86_64::*, mem::size_of}; + +use x86::sse2; + +const VECTOR_SIZE: usize = size_of::<__m128i>(); +const CONTROL_ANY: i32 = _SIDD_UBYTE_OPS + | _SIDD_CMP_EQUAL_ANY + | _SIDD_POSITIVE_POLARITY + | _SIDD_LEAST_SIGNIFICANT; + +#[target_feature(enable = "sse4.2")] +pub unsafe fn memchr3( + n1: u8, + n2: u8, + n3: u8, + haystack: &[u8], +) -> Option { + let vn1 = _mm_set1_epi8(n1 as i8); + let vn2 = _mm_set1_epi8(n2 as i8); + let vn3 = _mm_set1_epi8(n3 as i8); + let vn = _mm_setr_epi8( + n1 as i8, n2 as i8, n3 as i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ); + let len = haystack.len(); + let start_ptr = haystack.as_ptr(); + let end_ptr = haystack[haystack.len()..].as_ptr(); + let mut ptr = start_ptr; + + if haystack.len() < VECTOR_SIZE { + while ptr < end_ptr { + if *ptr == n1 || *ptr == n2 || *ptr == n3 { + return Some(sub(ptr, start_ptr)); + } + ptr = ptr.offset(1); + } + return None; + } + while ptr <= end_ptr.sub(VECTOR_SIZE) { + let chunk = _mm_loadu_si128(ptr as *const __m128i); + let res = _mm_cmpestri(vn, 3, chunk, 16, CONTROL_ANY); + if res < 16 { + return Some(sub(ptr, start_ptr) + res as usize); + } + ptr = ptr.add(VECTOR_SIZE); + } + if ptr < end_ptr { + debug_assert!(sub(end_ptr, ptr) < VECTOR_SIZE); + ptr = ptr.sub(VECTOR_SIZE - sub(end_ptr, ptr)); + debug_assert_eq!(sub(end_ptr, ptr), VECTOR_SIZE); + + return sse2::forward_search3(start_ptr, end_ptr, ptr, vn1, vn2, vn3); + } + None +} + +/// Subtract `b` from `a` and return the difference. `a` should be greater than +/// or equal to `b`. +fn sub(a: *const u8, b: *const u8) -> usize { + debug_assert!(a >= b); + (a as usize) - (b as usize) +} diff --git a/crux-mir/lib/memchr/src/memmem/byte_frequencies.rs b/crux-mir/lib/memchr/src/memmem/byte_frequencies.rs new file mode 100644 index 000000000..c313b629d --- /dev/null +++ b/crux-mir/lib/memchr/src/memmem/byte_frequencies.rs @@ -0,0 +1,258 @@ +pub const BYTE_FREQUENCIES: [u8; 256] = [ + 55, // '\x00' + 52, // '\x01' + 51, // '\x02' + 50, // '\x03' + 49, // '\x04' + 48, // '\x05' + 47, // '\x06' + 46, // '\x07' + 45, // '\x08' + 103, // '\t' + 242, // '\n' + 66, // '\x0b' + 67, // '\x0c' + 229, // '\r' + 44, // '\x0e' + 43, // '\x0f' + 42, // '\x10' + 41, // '\x11' + 40, // '\x12' + 39, // '\x13' + 38, // '\x14' + 37, // '\x15' + 36, // '\x16' + 35, // '\x17' + 34, // '\x18' + 33, // '\x19' + 56, // '\x1a' + 32, // '\x1b' + 31, // '\x1c' + 30, // '\x1d' + 29, // '\x1e' + 28, // '\x1f' + 255, // ' ' + 148, // '!' + 164, // '"' + 149, // '#' + 136, // '$' + 160, // '%' + 155, // '&' + 173, // "'" + 221, // '(' + 222, // ')' + 134, // '*' + 122, // '+' + 232, // ',' + 202, // '-' + 215, // '.' + 224, // '/' + 208, // '0' + 220, // '1' + 204, // '2' + 187, // '3' + 183, // '4' + 179, // '5' + 177, // '6' + 168, // '7' + 178, // '8' + 200, // '9' + 226, // ':' + 195, // ';' + 154, // '<' + 184, // '=' + 174, // '>' + 126, // '?' + 120, // '@' + 191, // 'A' + 157, // 'B' + 194, // 'C' + 170, // 'D' + 189, // 'E' + 162, // 'F' + 161, // 'G' + 150, // 'H' + 193, // 'I' + 142, // 'J' + 137, // 'K' + 171, // 'L' + 176, // 'M' + 185, // 'N' + 167, // 'O' + 186, // 'P' + 112, // 'Q' + 175, // 'R' + 192, // 'S' + 188, // 'T' + 156, // 'U' + 140, // 'V' + 143, // 'W' + 123, // 'X' + 133, // 'Y' + 128, // 'Z' + 147, // '[' + 138, // '\\' + 146, // ']' + 114, // '^' + 223, // '_' + 151, // '`' + 249, // 'a' + 216, // 'b' + 238, // 'c' + 236, // 'd' + 253, // 'e' + 227, // 'f' + 218, // 'g' + 230, // 'h' + 247, // 'i' + 135, // 'j' + 180, // 'k' + 241, // 'l' + 233, // 'm' + 246, // 'n' + 244, // 'o' + 231, // 'p' + 139, // 'q' + 245, // 'r' + 243, // 's' + 251, // 't' + 235, // 'u' + 201, // 'v' + 196, // 'w' + 240, // 'x' + 214, // 'y' + 152, // 'z' + 182, // '{' + 205, // '|' + 181, // '}' + 127, // '~' + 27, // '\x7f' + 212, // '\x80' + 211, // '\x81' + 210, // '\x82' + 213, // '\x83' + 228, // '\x84' + 197, // '\x85' + 169, // '\x86' + 159, // '\x87' + 131, // '\x88' + 172, // '\x89' + 105, // '\x8a' + 80, // '\x8b' + 98, // '\x8c' + 96, // '\x8d' + 97, // '\x8e' + 81, // '\x8f' + 207, // '\x90' + 145, // '\x91' + 116, // '\x92' + 115, // '\x93' + 144, // '\x94' + 130, // '\x95' + 153, // '\x96' + 121, // '\x97' + 107, // '\x98' + 132, // '\x99' + 109, // '\x9a' + 110, // '\x9b' + 124, // '\x9c' + 111, // '\x9d' + 82, // '\x9e' + 108, // '\x9f' + 118, // '\xa0' + 141, // '¡' + 113, // '¢' + 129, // '£' + 119, // '¤' + 125, // '¥' + 165, // '¦' + 117, // '§' + 92, // '¨' + 106, // '©' + 83, // 'ª' + 72, // '«' + 99, // '¬' + 93, // '\xad' + 65, // '®' + 79, // '¯' + 166, // '°' + 237, // '±' + 163, // '²' + 199, // '³' + 190, // '´' + 225, // 'µ' + 209, // '¶' + 203, // '·' + 198, // '¸' + 217, // '¹' + 219, // 'º' + 206, // '»' + 234, // '¼' + 248, // '½' + 158, // '¾' + 239, // '¿' + 255, // 'À' + 255, // 'Á' + 255, // 'Â' + 255, // 'Ã' + 255, // 'Ä' + 255, // 'Å' + 255, // 'Æ' + 255, // 'Ç' + 255, // 'È' + 255, // 'É' + 255, // 'Ê' + 255, // 'Ë' + 255, // 'Ì' + 255, // 'Í' + 255, // 'Î' + 255, // 'Ï' + 255, // 'Ð' + 255, // 'Ñ' + 255, // 'Ò' + 255, // 'Ó' + 255, // 'Ô' + 255, // 'Õ' + 255, // 'Ö' + 255, // '×' + 255, // 'Ø' + 255, // 'Ù' + 255, // 'Ú' + 255, // 'Û' + 255, // 'Ü' + 255, // 'Ý' + 255, // 'Þ' + 255, // 'ß' + 255, // 'à' + 255, // 'á' + 255, // 'â' + 255, // 'ã' + 255, // 'ä' + 255, // 'å' + 255, // 'æ' + 255, // 'ç' + 255, // 'è' + 255, // 'é' + 255, // 'ê' + 255, // 'ë' + 255, // 'ì' + 255, // 'í' + 255, // 'î' + 255, // 'ï' + 255, // 'ð' + 255, // 'ñ' + 255, // 'ò' + 255, // 'ó' + 255, // 'ô' + 255, // 'õ' + 255, // 'ö' + 255, // '÷' + 255, // 'ø' + 255, // 'ù' + 255, // 'ú' + 255, // 'û' + 255, // 'ü' + 255, // 'ý' + 255, // 'þ' + 255, // 'ÿ' +]; diff --git a/crux-mir/lib/memchr/src/memmem/genericsimd.rs b/crux-mir/lib/memchr/src/memmem/genericsimd.rs new file mode 100644 index 000000000..28bfdab88 --- /dev/null +++ b/crux-mir/lib/memchr/src/memmem/genericsimd.rs @@ -0,0 +1,266 @@ +use core::mem::size_of; + +use crate::memmem::{util::memcmp, vector::Vector, NeedleInfo}; + +/// The minimum length of a needle required for this algorithm. The minimum +/// is 2 since a length of 1 should just use memchr and a length of 0 isn't +/// a case handled by this searcher. +pub(crate) const MIN_NEEDLE_LEN: usize = 2; + +/// The maximum length of a needle required for this algorithm. +/// +/// In reality, there is no hard max here. The code below can handle any +/// length needle. (Perhaps that suggests there are missing optimizations.) +/// Instead, this is a heuristic and a bound guaranteeing our linear time +/// complexity. +/// +/// It is a heuristic because when a candidate match is found, memcmp is run. +/// For very large needles with lots of false positives, memcmp can make the +/// code run quite slow. +/// +/// It is a bound because the worst case behavior with memcmp is multiplicative +/// in the size of the needle and haystack, and we want to keep that additive. +/// This bound ensures we still meet that bound theoretically, since it's just +/// a constant. We aren't acting in bad faith here, memcmp on tiny needles +/// is so fast that even in pathological cases (see pathological vector +/// benchmarks), this is still just as fast or faster in practice. +/// +/// This specific number was chosen by tweaking a bit and running benchmarks. +/// The rare-medium-needle, for example, gets about 5% faster by using this +/// algorithm instead of a prefilter-accelerated Two-Way. There's also a +/// theoretical desire to keep this number reasonably low, to mitigate the +/// impact of pathological cases. I did try 64, and some benchmarks got a +/// little better, and others (particularly the pathological ones), got a lot +/// worse. So... 32 it is? +pub(crate) const MAX_NEEDLE_LEN: usize = 32; + +/// The implementation of the forward vector accelerated substring search. +/// +/// This is extremely similar to the prefilter vector module by the same name. +/// The key difference is that this is not a prefilter. Instead, it handles +/// confirming its own matches. The trade off is that this only works with +/// smaller needles. The speed up here is that an inlined memcmp on a tiny +/// needle is very quick, even on pathological inputs. This is much better than +/// combining a prefilter with Two-Way, where using Two-Way to confirm the +/// match has higher latency. +/// +/// So why not use this for all needles? We could, and it would probably work +/// really well on most inputs. But its worst case is multiplicative and we +/// want to guarantee worst case additive time. Some of the benchmarks try to +/// justify this (see the pathological ones). +/// +/// The prefilter variant of this has more comments. Also note that we only +/// implement this for forward searches for now. If you have a compelling use +/// case for accelerated reverse search, please file an issue. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Forward { + rare1i: u8, + rare2i: u8, +} + +impl Forward { + /// Create a new "generic simd" forward searcher. If one could not be + /// created from the given inputs, then None is returned. + pub(crate) fn new(ninfo: &NeedleInfo, needle: &[u8]) -> Option { + let (rare1i, rare2i) = ninfo.rarebytes.as_rare_ordered_u8(); + // If the needle is too short or too long, give up. Also, give up + // if the rare bytes detected are at the same position. (It likely + // suggests a degenerate case, although it should technically not be + // possible.) + if needle.len() < MIN_NEEDLE_LEN + || needle.len() > MAX_NEEDLE_LEN + || rare1i == rare2i + { + return None; + } + Some(Forward { rare1i, rare2i }) + } + + /// Returns the minimum length of haystack that is needed for this searcher + /// to work for a particular vector. Passing a haystack with a length + /// smaller than this will cause `fwd_find` to panic. + #[inline(always)] + pub(crate) fn min_haystack_len(&self) -> usize { + self.rare2i as usize + size_of::() + } +} + +/// Searches the given haystack for the given needle. The needle given should +/// be the same as the needle that this searcher was initialized with. +/// +/// # Panics +/// +/// When the given haystack has a length smaller than `min_haystack_len`. +/// +/// # Safety +/// +/// Since this is meant to be used with vector functions, callers need to +/// specialize this inside of a function with a `target_feature` attribute. +/// Therefore, callers must ensure that whatever target feature is being used +/// supports the vector functions that this function is specialized for. (For +/// the specific vector functions used, see the Vector trait implementations.) +#[inline(always)] +pub(crate) unsafe fn fwd_find( + fwd: &Forward, + haystack: &[u8], + needle: &[u8], +) -> Option { + // It would be nice if we didn't have this check here, since the meta + // searcher should handle it for us. But without this, I don't think we + // guarantee that end_ptr.sub(needle.len()) won't result in UB. We could + // put it as part of the safety contract, but it makes it more complicated + // than necessary. + if haystack.len() < needle.len() { + return None; + } + let min_haystack_len = fwd.min_haystack_len::(); + assert!(haystack.len() >= min_haystack_len, "haystack too small"); + debug_assert!(needle.len() <= haystack.len()); + debug_assert!( + needle.len() >= MIN_NEEDLE_LEN, + "needle must be at least {} bytes", + MIN_NEEDLE_LEN, + ); + debug_assert!( + needle.len() <= MAX_NEEDLE_LEN, + "needle must be at most {} bytes", + MAX_NEEDLE_LEN, + ); + + let (rare1i, rare2i) = (fwd.rare1i as usize, fwd.rare2i as usize); + let rare1chunk = V::splat(needle[rare1i]); + let rare2chunk = V::splat(needle[rare2i]); + + let start_ptr = haystack.as_ptr(); + let end_ptr = start_ptr.add(haystack.len()); + let max_ptr = end_ptr.sub(min_haystack_len); + let mut ptr = start_ptr; + + // N.B. I did experiment with unrolling the loop to deal with size(V) + // bytes at a time and 2*size(V) bytes at a time. The double unroll was + // marginally faster while the quadruple unroll was unambiguously slower. + // In the end, I decided the complexity from unrolling wasn't worth it. I + // used the memmem/krate/prebuilt/huge-en/ benchmarks to compare. + while ptr <= max_ptr { + let m = fwd_find_in_chunk( + fwd, needle, ptr, end_ptr, rare1chunk, rare2chunk, !0, + ); + if let Some(chunki) = m { + return Some(matched(start_ptr, ptr, chunki)); + } + ptr = ptr.add(size_of::()); + } + if ptr < end_ptr { + let remaining = diff(end_ptr, ptr); + debug_assert!( + remaining < min_haystack_len, + "remaining bytes should be smaller than the minimum haystack \ + length of {}, but there are {} bytes remaining", + min_haystack_len, + remaining, + ); + if remaining < needle.len() { + return None; + } + debug_assert!( + max_ptr < ptr, + "after main loop, ptr should have exceeded max_ptr", + ); + let overlap = diff(ptr, max_ptr); + debug_assert!( + overlap > 0, + "overlap ({}) must always be non-zero", + overlap, + ); + debug_assert!( + overlap < size_of::(), + "overlap ({}) cannot possibly be >= than a vector ({})", + overlap, + size_of::(), + ); + // The mask has all of its bits set except for the first N least + // significant bits, where N=overlap. This way, any matches that + // occur in find_in_chunk within the overlap are automatically + // ignored. + let mask = !((1 << overlap) - 1); + ptr = max_ptr; + let m = fwd_find_in_chunk( + fwd, needle, ptr, end_ptr, rare1chunk, rare2chunk, mask, + ); + if let Some(chunki) = m { + return Some(matched(start_ptr, ptr, chunki)); + } + } + None +} + +/// Search for an occurrence of two rare bytes from the needle in the chunk +/// pointed to by ptr, with the end of the haystack pointed to by end_ptr. When +/// an occurrence is found, memcmp is run to check if a match occurs at the +/// corresponding position. +/// +/// rare1chunk and rare2chunk correspond to vectors with the rare1 and rare2 +/// bytes repeated in each 8-bit lane, respectively. +/// +/// mask should have bits set corresponding the positions in the chunk in which +/// matches are considered. This is only used for the last vector load where +/// the beginning of the vector might have overlapped with the last load in +/// the main loop. The mask lets us avoid visiting positions that have already +/// been discarded as matches. +/// +/// # Safety +/// +/// It must be safe to do an unaligned read of size(V) bytes starting at both +/// (ptr + rare1i) and (ptr + rare2i). It must also be safe to do unaligned +/// loads on ptr up to (end_ptr - needle.len()). +#[inline(always)] +unsafe fn fwd_find_in_chunk( + fwd: &Forward, + needle: &[u8], + ptr: *const u8, + end_ptr: *const u8, + rare1chunk: V, + rare2chunk: V, + mask: u32, +) -> Option { + let chunk0 = V::load_unaligned(ptr.add(fwd.rare1i as usize)); + let chunk1 = V::load_unaligned(ptr.add(fwd.rare2i as usize)); + + let eq0 = chunk0.cmpeq(rare1chunk); + let eq1 = chunk1.cmpeq(rare2chunk); + + let mut match_offsets = eq0.and(eq1).movemask() & mask; + while match_offsets != 0 { + let offset = match_offsets.trailing_zeros() as usize; + let ptr = ptr.add(offset); + if end_ptr.sub(needle.len()) < ptr { + return None; + } + let chunk = core::slice::from_raw_parts(ptr, needle.len()); + if memcmp(needle, chunk) { + return Some(offset); + } + match_offsets &= match_offsets - 1; + } + None +} + +/// Accepts a chunk-relative offset and returns a haystack relative offset +/// after updating the prefilter state. +/// +/// See the same function with the same name in the prefilter variant of this +/// algorithm to learned why it's tagged with inline(never). Even here, where +/// the function is simpler, inlining it leads to poorer codegen. (Although +/// it does improve some benchmarks, like prebuiltiter/huge-en/common-you.) +#[cold] +#[inline(never)] +fn matched(start_ptr: *const u8, ptr: *const u8, chunki: usize) -> usize { + diff(ptr, start_ptr) + chunki +} + +/// Subtract `b` from `a` and return the difference. `a` must be greater than +/// or equal to `b`. +fn diff(a: *const u8, b: *const u8) -> usize { + debug_assert!(a >= b); + (a as usize) - (b as usize) +} diff --git a/crux-mir/lib/memchr/src/memmem/mod.rs b/crux-mir/lib/memchr/src/memmem/mod.rs new file mode 100644 index 000000000..e1cd1aec7 --- /dev/null +++ b/crux-mir/lib/memchr/src/memmem/mod.rs @@ -0,0 +1,1321 @@ +/*! +This module provides forward and reverse substring search routines. + +Unlike the standard library's substring search routines, these work on +arbitrary bytes. For all non-empty needles, these routines will report exactly +the same values as the corresponding routines in the standard library. For +the empty needle, the standard library reports matches only at valid UTF-8 +boundaries, where as these routines will report matches at every position. + +Other than being able to work on arbitrary bytes, the primary reason to prefer +these routines over the standard library routines is that these will generally +be faster. In some cases, significantly so. + +# Example: iterating over substring matches + +This example shows how to use [`find_iter`] to find occurrences of a substring +in a haystack. + +``` +use memchr::memmem; + +let haystack = b"foo bar foo baz foo"; + +let mut it = memmem::find_iter(haystack, "foo"); +assert_eq!(Some(0), it.next()); +assert_eq!(Some(8), it.next()); +assert_eq!(Some(16), it.next()); +assert_eq!(None, it.next()); +``` + +# Example: iterating over substring matches in reverse + +This example shows how to use [`rfind_iter`] to find occurrences of a substring +in a haystack starting from the end of the haystack. + +**NOTE:** This module does not implement double ended iterators, so reverse +searches aren't done by calling `rev` on a forward iterator. + +``` +use memchr::memmem; + +let haystack = b"foo bar foo baz foo"; + +let mut it = memmem::rfind_iter(haystack, "foo"); +assert_eq!(Some(16), it.next()); +assert_eq!(Some(8), it.next()); +assert_eq!(Some(0), it.next()); +assert_eq!(None, it.next()); +``` + +# Example: repeating a search for the same needle + +It may be possible for the overhead of constructing a substring searcher to be +measurable in some workloads. In cases where the same needle is used to search +many haystacks, it is possible to do construction once and thus to avoid it for +subsequent searches. This can be done with a [`Finder`] (or a [`FinderRev`] for +reverse searches). + +``` +use memchr::memmem; + +let finder = memmem::Finder::new("foo"); + +assert_eq!(Some(4), finder.find(b"baz foo quux")); +assert_eq!(None, finder.find(b"quux baz bar")); +``` +*/ + +pub use self::prefilter::Prefilter; + +use crate::{ + cow::CowBytes, + memmem::{ + prefilter::{Pre, PrefilterFn, PrefilterState}, + rabinkarp::NeedleHash, + rarebytes::RareNeedleBytes, + }, +}; + +/// Defines a suite of quickcheck properties for forward and reverse +/// substring searching. +/// +/// This is defined in this specific spot so that it can be used freely among +/// the different substring search implementations. I couldn't be bothered to +/// fight with the macro-visibility rules enough to figure out how to stuff it +/// somewhere more convenient. +#[cfg(all(test, feature = "std"))] +macro_rules! define_memmem_quickcheck_tests { + ($fwd:expr, $rev:expr) => { + use crate::memmem::proptests; + + quickcheck::quickcheck! { + fn qc_fwd_prefix_is_substring(bs: Vec) -> bool { + proptests::prefix_is_substring(false, &bs, $fwd) + } + + fn qc_fwd_suffix_is_substring(bs: Vec) -> bool { + proptests::suffix_is_substring(false, &bs, $fwd) + } + + fn qc_fwd_matches_naive( + haystack: Vec, + needle: Vec + ) -> bool { + proptests::matches_naive(false, &haystack, &needle, $fwd) + } + + fn qc_rev_prefix_is_substring(bs: Vec) -> bool { + proptests::prefix_is_substring(true, &bs, $rev) + } + + fn qc_rev_suffix_is_substring(bs: Vec) -> bool { + proptests::suffix_is_substring(true, &bs, $rev) + } + + fn qc_rev_matches_naive( + haystack: Vec, + needle: Vec + ) -> bool { + proptests::matches_naive(true, &haystack, &needle, $rev) + } + } + }; +} + +/// Defines a suite of "simple" hand-written tests for a substring +/// implementation. +/// +/// This is defined here for the same reason that +/// define_memmem_quickcheck_tests is defined here. +#[cfg(test)] +macro_rules! define_memmem_simple_tests { + ($fwd:expr, $rev:expr) => { + use crate::memmem::testsimples; + + #[test] + fn simple_forward() { + testsimples::run_search_tests_fwd($fwd); + } + + #[test] + fn simple_reverse() { + testsimples::run_search_tests_rev($rev); + } + }; +} + +mod byte_frequencies; +#[cfg(memchr_runtime_simd)] +mod genericsimd; +mod prefilter; +mod rabinkarp; +mod rarebytes; +mod twoway; +mod util; +#[cfg(memchr_runtime_simd)] +mod vector; +#[cfg(all(memchr_runtime_wasm128))] +mod wasm; +#[cfg(all(not(miri), target_arch = "x86_64", memchr_runtime_simd))] +mod x86; + +/// Returns an iterator over all non-overlapping occurrences of a substring in +/// a haystack. +/// +/// # Complexity +/// +/// This routine is guaranteed to have worst case linear time complexity +/// with respect to both the needle and the haystack. That is, this runs +/// in `O(needle.len() + haystack.len())` time. +/// +/// This routine is also guaranteed to have worst case constant space +/// complexity. +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// use memchr::memmem; +/// +/// let haystack = b"foo bar foo baz foo"; +/// let mut it = memmem::find_iter(haystack, b"foo"); +/// assert_eq!(Some(0), it.next()); +/// assert_eq!(Some(8), it.next()); +/// assert_eq!(Some(16), it.next()); +/// assert_eq!(None, it.next()); +/// ``` +#[inline] +pub fn find_iter<'h, 'n, N: 'n + ?Sized + AsRef<[u8]>>( + haystack: &'h [u8], + needle: &'n N, +) -> FindIter<'h, 'n> { + FindIter::new(haystack, Finder::new(needle)) +} + +/// Returns a reverse iterator over all non-overlapping occurrences of a +/// substring in a haystack. +/// +/// # Complexity +/// +/// This routine is guaranteed to have worst case linear time complexity +/// with respect to both the needle and the haystack. That is, this runs +/// in `O(needle.len() + haystack.len())` time. +/// +/// This routine is also guaranteed to have worst case constant space +/// complexity. +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// use memchr::memmem; +/// +/// let haystack = b"foo bar foo baz foo"; +/// let mut it = memmem::rfind_iter(haystack, b"foo"); +/// assert_eq!(Some(16), it.next()); +/// assert_eq!(Some(8), it.next()); +/// assert_eq!(Some(0), it.next()); +/// assert_eq!(None, it.next()); +/// ``` +#[inline] +pub fn rfind_iter<'h, 'n, N: 'n + ?Sized + AsRef<[u8]>>( + haystack: &'h [u8], + needle: &'n N, +) -> FindRevIter<'h, 'n> { + FindRevIter::new(haystack, FinderRev::new(needle)) +} + +/// Returns the index of the first occurrence of the given needle. +/// +/// Note that if you're are searching for the same needle in many different +/// small haystacks, it may be faster to initialize a [`Finder`] once, +/// and reuse it for each search. +/// +/// # Complexity +/// +/// This routine is guaranteed to have worst case linear time complexity +/// with respect to both the needle and the haystack. That is, this runs +/// in `O(needle.len() + haystack.len())` time. +/// +/// This routine is also guaranteed to have worst case constant space +/// complexity. +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// use memchr::memmem; +/// +/// let haystack = b"foo bar baz"; +/// assert_eq!(Some(0), memmem::find(haystack, b"foo")); +/// assert_eq!(Some(4), memmem::find(haystack, b"bar")); +/// assert_eq!(None, memmem::find(haystack, b"quux")); +/// ``` +#[inline] +pub fn find(haystack: &[u8], needle: &[u8]) -> Option { + if haystack.len() < 64 { + rabinkarp::find(haystack, needle) + } else { + Finder::new(needle).find(haystack) + } +} + +/// Returns the index of the last occurrence of the given needle. +/// +/// Note that if you're are searching for the same needle in many different +/// small haystacks, it may be faster to initialize a [`FinderRev`] once, +/// and reuse it for each search. +/// +/// # Complexity +/// +/// This routine is guaranteed to have worst case linear time complexity +/// with respect to both the needle and the haystack. That is, this runs +/// in `O(needle.len() + haystack.len())` time. +/// +/// This routine is also guaranteed to have worst case constant space +/// complexity. +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// use memchr::memmem; +/// +/// let haystack = b"foo bar baz"; +/// assert_eq!(Some(0), memmem::rfind(haystack, b"foo")); +/// assert_eq!(Some(4), memmem::rfind(haystack, b"bar")); +/// assert_eq!(Some(8), memmem::rfind(haystack, b"ba")); +/// assert_eq!(None, memmem::rfind(haystack, b"quux")); +/// ``` +#[inline] +pub fn rfind(haystack: &[u8], needle: &[u8]) -> Option { + if haystack.len() < 64 { + rabinkarp::rfind(haystack, needle) + } else { + FinderRev::new(needle).rfind(haystack) + } +} + +/// An iterator over non-overlapping substring matches. +/// +/// Matches are reported by the byte offset at which they begin. +/// +/// `'h` is the lifetime of the haystack while `'n` is the lifetime of the +/// needle. +#[derive(Debug)] +pub struct FindIter<'h, 'n> { + haystack: &'h [u8], + prestate: PrefilterState, + finder: Finder<'n>, + pos: usize, +} + +impl<'h, 'n> FindIter<'h, 'n> { + #[inline(always)] + pub(crate) fn new( + haystack: &'h [u8], + finder: Finder<'n>, + ) -> FindIter<'h, 'n> { + let prestate = finder.searcher.prefilter_state(); + FindIter { haystack, prestate, finder, pos: 0 } + } + + /// Convert this iterator into its owned variant, such that it no longer + /// borrows the finder and needle. + /// + /// If this is already an owned iterator, then this is a no-op. Otherwise, + /// this copies the needle. + /// + /// This is only available when the `std` feature is enabled. + #[cfg(feature = "std")] + #[inline] + pub fn into_owned(self) -> FindIter<'h, 'static> { + FindIter { + haystack: self.haystack, + prestate: self.prestate, + finder: self.finder.into_owned(), + pos: self.pos, + } + } +} + +impl<'h, 'n> Iterator for FindIter<'h, 'n> { + type Item = usize; + + fn next(&mut self) -> Option { + if self.pos > self.haystack.len() { + return None; + } + let result = self + .finder + .searcher + .find(&mut self.prestate, &self.haystack[self.pos..]); + match result { + None => None, + Some(i) => { + let pos = self.pos + i; + self.pos = pos + core::cmp::max(1, self.finder.needle().len()); + Some(pos) + } + } + } +} + +/// An iterator over non-overlapping substring matches in reverse. +/// +/// Matches are reported by the byte offset at which they begin. +/// +/// `'h` is the lifetime of the haystack while `'n` is the lifetime of the +/// needle. +#[derive(Debug)] +pub struct FindRevIter<'h, 'n> { + haystack: &'h [u8], + finder: FinderRev<'n>, + /// When searching with an empty needle, this gets set to `None` after + /// we've yielded the last element at `0`. + pos: Option, +} + +impl<'h, 'n> FindRevIter<'h, 'n> { + #[inline(always)] + pub(crate) fn new( + haystack: &'h [u8], + finder: FinderRev<'n>, + ) -> FindRevIter<'h, 'n> { + let pos = Some(haystack.len()); + FindRevIter { haystack, finder, pos } + } + + /// Convert this iterator into its owned variant, such that it no longer + /// borrows the finder and needle. + /// + /// If this is already an owned iterator, then this is a no-op. Otherwise, + /// this copies the needle. + /// + /// This is only available when the `std` feature is enabled. + #[cfg(feature = "std")] + #[inline] + pub fn into_owned(self) -> FindRevIter<'h, 'static> { + FindRevIter { + haystack: self.haystack, + finder: self.finder.into_owned(), + pos: self.pos, + } + } +} + +impl<'h, 'n> Iterator for FindRevIter<'h, 'n> { + type Item = usize; + + fn next(&mut self) -> Option { + let pos = match self.pos { + None => return None, + Some(pos) => pos, + }; + let result = self.finder.rfind(&self.haystack[..pos]); + match result { + None => None, + Some(i) => { + if pos == i { + self.pos = pos.checked_sub(1); + } else { + self.pos = Some(i); + } + Some(i) + } + } + } +} + +/// A single substring searcher fixed to a particular needle. +/// +/// The purpose of this type is to permit callers to construct a substring +/// searcher that can be used to search haystacks without the overhead of +/// constructing the searcher in the first place. This is a somewhat niche +/// concern when it's necessary to re-use the same needle to search multiple +/// different haystacks with as little overhead as possible. In general, using +/// [`find`] is good enough, but `Finder` is useful when you can meaningfully +/// observe searcher construction time in a profile. +/// +/// When the `std` feature is enabled, then this type has an `into_owned` +/// version which permits building a `Finder` that is not connected to +/// the lifetime of its needle. +#[derive(Clone, Debug)] +pub struct Finder<'n> { + searcher: Searcher<'n>, +} + +impl<'n> Finder<'n> { + /// Create a new finder for the given needle. + #[inline] + pub fn new>(needle: &'n B) -> Finder<'n> { + FinderBuilder::new().build_forward(needle) + } + + /// Returns the index of the first occurrence of this needle in the given + /// haystack. + /// + /// # Complexity + /// + /// This routine is guaranteed to have worst case linear time complexity + /// with respect to both the needle and the haystack. That is, this runs + /// in `O(needle.len() + haystack.len())` time. + /// + /// This routine is also guaranteed to have worst case constant space + /// complexity. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use memchr::memmem::Finder; + /// + /// let haystack = b"foo bar baz"; + /// assert_eq!(Some(0), Finder::new("foo").find(haystack)); + /// assert_eq!(Some(4), Finder::new("bar").find(haystack)); + /// assert_eq!(None, Finder::new("quux").find(haystack)); + /// ``` + pub fn find(&self, haystack: &[u8]) -> Option { + self.searcher.find(&mut self.searcher.prefilter_state(), haystack) + } + + /// Returns an iterator over all occurrences of a substring in a haystack. + /// + /// # Complexity + /// + /// This routine is guaranteed to have worst case linear time complexity + /// with respect to both the needle and the haystack. That is, this runs + /// in `O(needle.len() + haystack.len())` time. + /// + /// This routine is also guaranteed to have worst case constant space + /// complexity. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use memchr::memmem::Finder; + /// + /// let haystack = b"foo bar foo baz foo"; + /// let finder = Finder::new(b"foo"); + /// let mut it = finder.find_iter(haystack); + /// assert_eq!(Some(0), it.next()); + /// assert_eq!(Some(8), it.next()); + /// assert_eq!(Some(16), it.next()); + /// assert_eq!(None, it.next()); + /// ``` + #[inline] + pub fn find_iter<'a, 'h>( + &'a self, + haystack: &'h [u8], + ) -> FindIter<'h, 'a> { + FindIter::new(haystack, self.as_ref()) + } + + /// Convert this finder into its owned variant, such that it no longer + /// borrows the needle. + /// + /// If this is already an owned finder, then this is a no-op. Otherwise, + /// this copies the needle. + /// + /// This is only available when the `std` feature is enabled. + #[cfg(feature = "std")] + #[inline] + pub fn into_owned(self) -> Finder<'static> { + Finder { searcher: self.searcher.into_owned() } + } + + /// Convert this finder into its borrowed variant. + /// + /// This is primarily useful if your finder is owned and you'd like to + /// store its borrowed variant in some intermediate data structure. + /// + /// Note that the lifetime parameter of the returned finder is tied to the + /// lifetime of `self`, and may be shorter than the `'n` lifetime of the + /// needle itself. Namely, a finder's needle can be either borrowed or + /// owned, so the lifetime of the needle returned must necessarily be the + /// shorter of the two. + #[inline] + pub fn as_ref(&self) -> Finder<'_> { + Finder { searcher: self.searcher.as_ref() } + } + + /// Returns the needle that this finder searches for. + /// + /// Note that the lifetime of the needle returned is tied to the lifetime + /// of the finder, and may be shorter than the `'n` lifetime. Namely, a + /// finder's needle can be either borrowed or owned, so the lifetime of the + /// needle returned must necessarily be the shorter of the two. + #[inline] + pub fn needle(&self) -> &[u8] { + self.searcher.needle() + } +} + +/// A single substring reverse searcher fixed to a particular needle. +/// +/// The purpose of this type is to permit callers to construct a substring +/// searcher that can be used to search haystacks without the overhead of +/// constructing the searcher in the first place. This is a somewhat niche +/// concern when it's necessary to re-use the same needle to search multiple +/// different haystacks with as little overhead as possible. In general, +/// using [`rfind`] is good enough, but `FinderRev` is useful when you can +/// meaningfully observe searcher construction time in a profile. +/// +/// When the `std` feature is enabled, then this type has an `into_owned` +/// version which permits building a `FinderRev` that is not connected to +/// the lifetime of its needle. +#[derive(Clone, Debug)] +pub struct FinderRev<'n> { + searcher: SearcherRev<'n>, +} + +impl<'n> FinderRev<'n> { + /// Create a new reverse finder for the given needle. + #[inline] + pub fn new>(needle: &'n B) -> FinderRev<'n> { + FinderBuilder::new().build_reverse(needle) + } + + /// Returns the index of the last occurrence of this needle in the given + /// haystack. + /// + /// The haystack may be any type that can be cheaply converted into a + /// `&[u8]`. This includes, but is not limited to, `&str` and `&[u8]`. + /// + /// # Complexity + /// + /// This routine is guaranteed to have worst case linear time complexity + /// with respect to both the needle and the haystack. That is, this runs + /// in `O(needle.len() + haystack.len())` time. + /// + /// This routine is also guaranteed to have worst case constant space + /// complexity. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use memchr::memmem::FinderRev; + /// + /// let haystack = b"foo bar baz"; + /// assert_eq!(Some(0), FinderRev::new("foo").rfind(haystack)); + /// assert_eq!(Some(4), FinderRev::new("bar").rfind(haystack)); + /// assert_eq!(None, FinderRev::new("quux").rfind(haystack)); + /// ``` + pub fn rfind>(&self, haystack: B) -> Option { + self.searcher.rfind(haystack.as_ref()) + } + + /// Returns a reverse iterator over all occurrences of a substring in a + /// haystack. + /// + /// # Complexity + /// + /// This routine is guaranteed to have worst case linear time complexity + /// with respect to both the needle and the haystack. That is, this runs + /// in `O(needle.len() + haystack.len())` time. + /// + /// This routine is also guaranteed to have worst case constant space + /// complexity. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use memchr::memmem::FinderRev; + /// + /// let haystack = b"foo bar foo baz foo"; + /// let finder = FinderRev::new(b"foo"); + /// let mut it = finder.rfind_iter(haystack); + /// assert_eq!(Some(16), it.next()); + /// assert_eq!(Some(8), it.next()); + /// assert_eq!(Some(0), it.next()); + /// assert_eq!(None, it.next()); + /// ``` + #[inline] + pub fn rfind_iter<'a, 'h>( + &'a self, + haystack: &'h [u8], + ) -> FindRevIter<'h, 'a> { + FindRevIter::new(haystack, self.as_ref()) + } + + /// Convert this finder into its owned variant, such that it no longer + /// borrows the needle. + /// + /// If this is already an owned finder, then this is a no-op. Otherwise, + /// this copies the needle. + /// + /// This is only available when the `std` feature is enabled. + #[cfg(feature = "std")] + #[inline] + pub fn into_owned(self) -> FinderRev<'static> { + FinderRev { searcher: self.searcher.into_owned() } + } + + /// Convert this finder into its borrowed variant. + /// + /// This is primarily useful if your finder is owned and you'd like to + /// store its borrowed variant in some intermediate data structure. + /// + /// Note that the lifetime parameter of the returned finder is tied to the + /// lifetime of `self`, and may be shorter than the `'n` lifetime of the + /// needle itself. Namely, a finder's needle can be either borrowed or + /// owned, so the lifetime of the needle returned must necessarily be the + /// shorter of the two. + #[inline] + pub fn as_ref(&self) -> FinderRev<'_> { + FinderRev { searcher: self.searcher.as_ref() } + } + + /// Returns the needle that this finder searches for. + /// + /// Note that the lifetime of the needle returned is tied to the lifetime + /// of the finder, and may be shorter than the `'n` lifetime. Namely, a + /// finder's needle can be either borrowed or owned, so the lifetime of the + /// needle returned must necessarily be the shorter of the two. + #[inline] + pub fn needle(&self) -> &[u8] { + self.searcher.needle() + } +} + +/// A builder for constructing non-default forward or reverse memmem finders. +/// +/// A builder is primarily useful for configuring a substring searcher. +/// Currently, the only configuration exposed is the ability to disable +/// heuristic prefilters used to speed up certain searches. +#[derive(Clone, Debug, Default)] +pub struct FinderBuilder { + config: SearcherConfig, +} + +impl FinderBuilder { + /// Create a new finder builder with default settings. + pub fn new() -> FinderBuilder { + FinderBuilder::default() + } + + /// Build a forward finder using the given needle from the current + /// settings. + pub fn build_forward<'n, B: ?Sized + AsRef<[u8]>>( + &self, + needle: &'n B, + ) -> Finder<'n> { + Finder { searcher: Searcher::new(self.config, needle.as_ref()) } + } + + /// Build a reverse finder using the given needle from the current + /// settings. + pub fn build_reverse<'n, B: ?Sized + AsRef<[u8]>>( + &self, + needle: &'n B, + ) -> FinderRev<'n> { + FinderRev { searcher: SearcherRev::new(needle.as_ref()) } + } + + /// Configure the prefilter setting for the finder. + /// + /// See the documentation for [`Prefilter`] for more discussion on why + /// you might want to configure this. + pub fn prefilter(&mut self, prefilter: Prefilter) -> &mut FinderBuilder { + self.config.prefilter = prefilter; + self + } +} + +/// The internal implementation of a forward substring searcher. +/// +/// The reality is that this is a "meta" searcher. Namely, depending on a +/// variety of parameters (CPU support, target, needle size, haystack size and +/// even dynamic properties such as prefilter effectiveness), the actual +/// algorithm employed to do substring search may change. +#[derive(Clone, Debug)] +struct Searcher<'n> { + /// The actual needle we're searching for. + /// + /// A CowBytes is like a Cow<[u8]>, except in no_std environments, it is + /// specialized to a single variant (the borrowed form). + needle: CowBytes<'n>, + /// A collection of facts computed on the needle that are useful for more + /// than one substring search algorithm. + ninfo: NeedleInfo, + /// A prefilter function, if it was deemed appropriate. + /// + /// Some substring search implementations (like Two-Way) benefit greatly + /// if we can quickly find candidate starting positions for a match. + prefn: Option, + /// The actual substring implementation in use. + kind: SearcherKind, +} + +/// A collection of facts computed about a search needle. +/// +/// We group these things together because it's useful to be able to hand them +/// to prefilters or substring algorithms that want them. +#[derive(Clone, Copy, Debug)] +pub(crate) struct NeedleInfo { + /// The offsets of "rare" bytes detected in the needle. + /// + /// This is meant to be a heuristic in order to maximize the effectiveness + /// of vectorized code. Namely, vectorized code tends to focus on only + /// one or two bytes. If we pick bytes from the needle that occur + /// infrequently, then more time will be spent in the vectorized code and + /// will likely make the overall search (much) faster. + /// + /// Of course, this is only a heuristic based on a background frequency + /// distribution of bytes. But it tends to work very well in practice. + pub(crate) rarebytes: RareNeedleBytes, + /// A Rabin-Karp hash of the needle. + /// + /// This is store here instead of in a more specific Rabin-Karp search + /// since Rabin-Karp may be used even if another SearchKind corresponds + /// to some other search implementation. e.g., If measurements suggest RK + /// is faster in some cases or if a search implementation can't handle + /// particularly small haystack. (Moreover, we cannot use RK *generally*, + /// since its worst case time is multiplicative. Instead, we only use it + /// some small haystacks, where "small" is a constant.) + pub(crate) nhash: NeedleHash, +} + +/// Configuration for substring search. +#[derive(Clone, Copy, Debug, Default)] +struct SearcherConfig { + /// This permits changing the behavior of the prefilter, since it can have + /// a variable impact on performance. + prefilter: Prefilter, +} + +#[derive(Clone, Debug)] +enum SearcherKind { + /// A special case for empty needles. An empty needle always matches, even + /// in an empty haystack. + Empty, + /// This is used whenever the needle is a single byte. In this case, we + /// always use memchr. + OneByte(u8), + /// Two-Way is the generic work horse and is what provides our additive + /// linear time guarantee. In general, it's used when the needle is bigger + /// than 8 bytes or so. + TwoWay(twoway::Forward), + #[cfg(all(not(miri), target_arch = "x86_64", memchr_runtime_simd))] + GenericSIMD128(x86::sse::Forward), + #[cfg(memchr_runtime_wasm128)] + GenericSIMD128(wasm::Forward), + #[cfg(all(not(miri), target_arch = "x86_64", memchr_runtime_simd))] + GenericSIMD256(x86::avx::Forward), +} + +impl<'n> Searcher<'n> { + fn new(config: SearcherConfig, needle: &'n [u8]) -> Searcher<'n> { + use self::SearcherKind::*; + + let ninfo = NeedleInfo::new(needle); + let mk = |kind: SearcherKind| { + let prefn = prefilter::forward( + &config.prefilter, + &ninfo.rarebytes, + needle, + ); + Searcher { needle: CowBytes::new(needle), ninfo, prefn, kind } + }; + if needle.len() == 0 { + return mk(Empty); + } + if needle.len() == 1 { + return mk(OneByte(needle[0])); + } + #[cfg(all(not(miri), target_arch = "x86_64", memchr_runtime_simd))] + { + if let Some(fwd) = x86::avx::Forward::new(&ninfo, needle) { + return mk(GenericSIMD256(fwd)); + } else if let Some(fwd) = x86::sse::Forward::new(&ninfo, needle) { + return mk(GenericSIMD128(fwd)); + } + } + #[cfg(all(target_arch = "wasm32", memchr_runtime_simd))] + { + if let Some(fwd) = wasm::Forward::new(&ninfo, needle) { + return mk(GenericSIMD128(fwd)); + } + } + + mk(TwoWay(twoway::Forward::new(needle))) + } + + /// Return a fresh prefilter state that can be used with this searcher. + /// A prefilter state is used to track the effectiveness of a searcher's + /// prefilter for speeding up searches. Therefore, the prefilter state + /// should generally be reused on subsequent searches (such as in an + /// iterator). For searches on a different haystack, then a new prefilter + /// state should be used. + /// + /// This always initializes a valid (but possibly inert) prefilter state + /// even if this searcher does not have a prefilter enabled. + fn prefilter_state(&self) -> PrefilterState { + if self.prefn.is_none() { + PrefilterState::inert() + } else { + PrefilterState::new() + } + } + + fn needle(&self) -> &[u8] { + self.needle.as_slice() + } + + fn as_ref(&self) -> Searcher<'_> { + use self::SearcherKind::*; + + let kind = match self.kind { + Empty => Empty, + OneByte(b) => OneByte(b), + TwoWay(tw) => TwoWay(tw), + #[cfg(all(not(miri), memchr_runtime_simd))] + GenericSIMD128(gs) => GenericSIMD128(gs), + #[cfg(all( + not(miri), + target_arch = "x86_64", + memchr_runtime_simd + ))] + GenericSIMD256(gs) => GenericSIMD256(gs), + }; + Searcher { + needle: CowBytes::new(self.needle()), + ninfo: self.ninfo, + prefn: self.prefn, + kind, + } + } + + #[cfg(feature = "std")] + fn into_owned(self) -> Searcher<'static> { + use self::SearcherKind::*; + + let kind = match self.kind { + Empty => Empty, + OneByte(b) => OneByte(b), + TwoWay(tw) => TwoWay(tw), + #[cfg(all(not(miri), memchr_runtime_simd))] + GenericSIMD128(gs) => GenericSIMD128(gs), + #[cfg(all( + not(miri), + target_arch = "x86_64", + memchr_runtime_simd + ))] + GenericSIMD256(gs) => GenericSIMD256(gs), + }; + Searcher { + needle: self.needle.into_owned(), + ninfo: self.ninfo, + prefn: self.prefn, + kind, + } + } + + /// Implements forward substring search by selecting the implementation + /// chosen at construction and executing it on the given haystack with the + /// prefilter's current state of effectiveness. + #[inline(always)] + fn find( + &self, + state: &mut PrefilterState, + haystack: &[u8], + ) -> Option { + use self::SearcherKind::*; + + let needle = self.needle(); + if haystack.len() < needle.len() { + return None; + } + match self.kind { + Empty => Some(0), + OneByte(b) => crate::memchr(b, haystack), + TwoWay(ref tw) => { + // For very short haystacks (e.g., where the prefilter probably + // can't run), it's faster to just run RK. + if rabinkarp::is_fast(haystack, needle) { + rabinkarp::find_with(&self.ninfo.nhash, haystack, needle) + } else { + self.find_tw(tw, state, haystack, needle) + } + } + #[cfg(all(not(miri), memchr_runtime_simd))] + GenericSIMD128(ref gs) => { + // The SIMD matcher can't handle particularly short haystacks, + // so we fall back to RK in these cases. + if haystack.len() < gs.min_haystack_len() { + rabinkarp::find_with(&self.ninfo.nhash, haystack, needle) + } else { + gs.find(haystack, needle) + } + } + #[cfg(all( + not(miri), + target_arch = "x86_64", + memchr_runtime_simd + ))] + GenericSIMD256(ref gs) => { + // The SIMD matcher can't handle particularly short haystacks, + // so we fall back to RK in these cases. + if haystack.len() < gs.min_haystack_len() { + rabinkarp::find_with(&self.ninfo.nhash, haystack, needle) + } else { + gs.find(haystack, needle) + } + } + } + } + + /// Calls Two-Way on the given haystack/needle. + /// + /// This is marked as unlineable since it seems to have a better overall + /// effect on benchmarks. However, this is one of those cases where + /// inlining it results an improvement in other benchmarks too, so I + /// suspect we just don't have enough data yet to make the right call here. + /// + /// I suspect the main problem is that this function contains two different + /// inlined copies of Two-Way: one with and one without prefilters enabled. + #[inline(never)] + fn find_tw( + &self, + tw: &twoway::Forward, + state: &mut PrefilterState, + haystack: &[u8], + needle: &[u8], + ) -> Option { + if let Some(prefn) = self.prefn { + // We used to look at the length of a haystack here. That is, if + // it was too small, then don't bother with the prefilter. But two + // things changed: the prefilter falls back to memchr for small + // haystacks, and, above, Rabin-Karp is employed for tiny haystacks + // anyway. + if state.is_effective() { + let mut pre = Pre { state, prefn, ninfo: &self.ninfo }; + return tw.find(Some(&mut pre), haystack, needle); + } + } + tw.find(None, haystack, needle) + } +} + +impl NeedleInfo { + pub(crate) fn new(needle: &[u8]) -> NeedleInfo { + NeedleInfo { + rarebytes: RareNeedleBytes::forward(needle), + nhash: NeedleHash::forward(needle), + } + } +} + +/// The internal implementation of a reverse substring searcher. +/// +/// See the forward searcher docs for more details. Currently, the reverse +/// searcher is considerably simpler since it lacks prefilter support. This +/// was done because it adds a lot of code, and more surface area to test. And +/// in particular, it's not clear whether a prefilter on reverse searching is +/// worth it. (If you have a compelling use case, please file an issue!) +#[derive(Clone, Debug)] +struct SearcherRev<'n> { + /// The actual needle we're searching for. + needle: CowBytes<'n>, + /// A Rabin-Karp hash of the needle. + nhash: NeedleHash, + /// The actual substring implementation in use. + kind: SearcherRevKind, +} + +#[derive(Clone, Debug)] +enum SearcherRevKind { + /// A special case for empty needles. An empty needle always matches, even + /// in an empty haystack. + Empty, + /// This is used whenever the needle is a single byte. In this case, we + /// always use memchr. + OneByte(u8), + /// Two-Way is the generic work horse and is what provides our additive + /// linear time guarantee. In general, it's used when the needle is bigger + /// than 8 bytes or so. + TwoWay(twoway::Reverse), +} + +impl<'n> SearcherRev<'n> { + fn new(needle: &'n [u8]) -> SearcherRev<'n> { + use self::SearcherRevKind::*; + + let kind = if needle.len() == 0 { + Empty + } else if needle.len() == 1 { + OneByte(needle[0]) + } else { + TwoWay(twoway::Reverse::new(needle)) + }; + SearcherRev { + needle: CowBytes::new(needle), + nhash: NeedleHash::reverse(needle), + kind, + } + } + + fn needle(&self) -> &[u8] { + self.needle.as_slice() + } + + fn as_ref(&self) -> SearcherRev<'_> { + use self::SearcherRevKind::*; + + let kind = match self.kind { + Empty => Empty, + OneByte(b) => OneByte(b), + TwoWay(tw) => TwoWay(tw), + }; + SearcherRev { + needle: CowBytes::new(self.needle()), + nhash: self.nhash, + kind, + } + } + + #[cfg(feature = "std")] + fn into_owned(self) -> SearcherRev<'static> { + use self::SearcherRevKind::*; + + let kind = match self.kind { + Empty => Empty, + OneByte(b) => OneByte(b), + TwoWay(tw) => TwoWay(tw), + }; + SearcherRev { + needle: self.needle.into_owned(), + nhash: self.nhash, + kind, + } + } + + /// Implements reverse substring search by selecting the implementation + /// chosen at construction and executing it on the given haystack with the + /// prefilter's current state of effectiveness. + #[inline(always)] + fn rfind(&self, haystack: &[u8]) -> Option { + use self::SearcherRevKind::*; + + let needle = self.needle(); + if haystack.len() < needle.len() { + return None; + } + match self.kind { + Empty => Some(haystack.len()), + OneByte(b) => crate::memrchr(b, haystack), + TwoWay(ref tw) => { + // For very short haystacks (e.g., where the prefilter probably + // can't run), it's faster to just run RK. + if rabinkarp::is_fast(haystack, needle) { + rabinkarp::rfind_with(&self.nhash, haystack, needle) + } else { + tw.rfind(haystack, needle) + } + } + } + } +} + +/// This module defines some generic quickcheck properties useful for testing +/// any substring search algorithm. It also runs those properties for the +/// top-level public API memmem routines. (The properties are also used to +/// test various substring search implementations more granularly elsewhere as +/// well.) +#[cfg(all(test, feature = "std", not(miri)))] +mod proptests { + // N.B. This defines the quickcheck tests using the properties defined + // below. Because of macro-visibility weirdness, the actual macro is + // defined at the top of this file. + define_memmem_quickcheck_tests!(super::find, super::rfind); + + /// Check that every prefix of the given byte string is a substring. + pub(crate) fn prefix_is_substring( + reverse: bool, + bs: &[u8], + mut search: impl FnMut(&[u8], &[u8]) -> Option, + ) -> bool { + if bs.is_empty() { + return true; + } + for i in 0..(bs.len() - 1) { + let prefix = &bs[..i]; + if reverse { + assert_eq!(naive_rfind(bs, prefix), search(bs, prefix)); + } else { + assert_eq!(naive_find(bs, prefix), search(bs, prefix)); + } + } + true + } + + /// Check that every suffix of the given byte string is a substring. + pub(crate) fn suffix_is_substring( + reverse: bool, + bs: &[u8], + mut search: impl FnMut(&[u8], &[u8]) -> Option, + ) -> bool { + if bs.is_empty() { + return true; + } + for i in 0..(bs.len() - 1) { + let suffix = &bs[i..]; + if reverse { + assert_eq!(naive_rfind(bs, suffix), search(bs, suffix)); + } else { + assert_eq!(naive_find(bs, suffix), search(bs, suffix)); + } + } + true + } + + /// Check that naive substring search matches the result of the given search + /// algorithm. + pub(crate) fn matches_naive( + reverse: bool, + haystack: &[u8], + needle: &[u8], + mut search: impl FnMut(&[u8], &[u8]) -> Option, + ) -> bool { + if reverse { + naive_rfind(haystack, needle) == search(haystack, needle) + } else { + naive_find(haystack, needle) == search(haystack, needle) + } + } + + /// Naively search forwards for the given needle in the given haystack. + fn naive_find(haystack: &[u8], needle: &[u8]) -> Option { + if needle.is_empty() { + return Some(0); + } else if haystack.len() < needle.len() { + return None; + } + for i in 0..(haystack.len() - needle.len() + 1) { + if needle == &haystack[i..i + needle.len()] { + return Some(i); + } + } + None + } + + /// Naively search in reverse for the given needle in the given haystack. + fn naive_rfind(haystack: &[u8], needle: &[u8]) -> Option { + if needle.is_empty() { + return Some(haystack.len()); + } else if haystack.len() < needle.len() { + return None; + } + for i in (0..(haystack.len() - needle.len() + 1)).rev() { + if needle == &haystack[i..i + needle.len()] { + return Some(i); + } + } + None + } +} + +/// This module defines some hand-written "simple" substring tests. It +/// also provides routines for easily running them on any substring search +/// implementation. +#[cfg(test)] +mod testsimples { + define_memmem_simple_tests!(super::find, super::rfind); + + /// Each test is a (needle, haystack, expected_fwd, expected_rev) tuple. + type SearchTest = + (&'static str, &'static str, Option, Option); + + const SEARCH_TESTS: &'static [SearchTest] = &[ + ("", "", Some(0), Some(0)), + ("", "a", Some(0), Some(1)), + ("", "ab", Some(0), Some(2)), + ("", "abc", Some(0), Some(3)), + ("a", "", None, None), + ("a", "a", Some(0), Some(0)), + ("a", "aa", Some(0), Some(1)), + ("a", "ba", Some(1), Some(1)), + ("a", "bba", Some(2), Some(2)), + ("a", "bbba", Some(3), Some(3)), + ("a", "bbbab", Some(3), Some(3)), + ("a", "bbbabb", Some(3), Some(3)), + ("a", "bbbabbb", Some(3), Some(3)), + ("a", "bbbbbb", None, None), + ("ab", "", None, None), + ("ab", "a", None, None), + ("ab", "b", None, None), + ("ab", "ab", Some(0), Some(0)), + ("ab", "aab", Some(1), Some(1)), + ("ab", "aaab", Some(2), Some(2)), + ("ab", "abaab", Some(0), Some(3)), + ("ab", "baaab", Some(3), Some(3)), + ("ab", "acb", None, None), + ("ab", "abba", Some(0), Some(0)), + ("abc", "ab", None, None), + ("abc", "abc", Some(0), Some(0)), + ("abc", "abcz", Some(0), Some(0)), + ("abc", "abczz", Some(0), Some(0)), + ("abc", "zabc", Some(1), Some(1)), + ("abc", "zzabc", Some(2), Some(2)), + ("abc", "azbc", None, None), + ("abc", "abzc", None, None), + ("abczdef", "abczdefzzzzzzzzzzzzzzzzzzzz", Some(0), Some(0)), + ("abczdef", "zzzzzzzzzzzzzzzzzzzzabczdef", Some(20), Some(20)), + ("xyz", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaxyz", Some(32), Some(32)), + // Failures caught by quickcheck. + ("\u{0}\u{15}", "\u{0}\u{15}\u{15}\u{0}", Some(0), Some(0)), + ("\u{0}\u{1e}", "\u{1e}\u{0}", None, None), + ]; + + /// Run the substring search tests. `search` should be a closure that + /// accepts a haystack and a needle and returns the starting position + /// of the first occurrence of needle in the haystack, or `None` if one + /// doesn't exist. + pub(crate) fn run_search_tests_fwd( + mut search: impl FnMut(&[u8], &[u8]) -> Option, + ) { + for &(needle, haystack, expected_fwd, _) in SEARCH_TESTS { + let (n, h) = (needle.as_bytes(), haystack.as_bytes()); + assert_eq!( + expected_fwd, + search(h, n), + "needle: {:?}, haystack: {:?}, expected: {:?}", + n, + h, + expected_fwd + ); + } + } + + /// Run the substring search tests. `search` should be a closure that + /// accepts a haystack and a needle and returns the starting position of + /// the last occurrence of needle in the haystack, or `None` if one doesn't + /// exist. + pub(crate) fn run_search_tests_rev( + mut search: impl FnMut(&[u8], &[u8]) -> Option, + ) { + for &(needle, haystack, _, expected_rev) in SEARCH_TESTS { + let (n, h) = (needle.as_bytes(), haystack.as_bytes()); + assert_eq!( + expected_rev, + search(h, n), + "needle: {:?}, haystack: {:?}, expected: {:?}", + n, + h, + expected_rev + ); + } + } +} diff --git a/crux-mir/lib/memchr/src/memmem/prefilter/fallback.rs b/crux-mir/lib/memchr/src/memmem/prefilter/fallback.rs new file mode 100644 index 000000000..ae1bbccb3 --- /dev/null +++ b/crux-mir/lib/memchr/src/memmem/prefilter/fallback.rs @@ -0,0 +1,122 @@ +/* +This module implements a "fallback" prefilter that only relies on memchr to +function. While memchr works best when it's explicitly vectorized, its +fallback implementations are fast enough to make a prefilter like this +worthwhile. + +The essence of this implementation is to identify two rare bytes in a needle +based on a background frequency distribution of bytes. We then run memchr on the +rarer byte. For each match, we use the second rare byte as a guard to quickly +check if a match is possible. If the position passes the guard test, then we do +a naive memcmp to confirm the match. + +In practice, this formulation works amazingly well, primarily because of the +heuristic use of a background frequency distribution. However, it does have a +number of weaknesses where it can get quite slow when its background frequency +distribution doesn't line up with the haystack being searched. This is why we +have specialized vector routines that essentially take this idea and move the +guard check into vectorized code. (Those specialized vector routines do still +make use of the background frequency distribution of bytes though.) + +This fallback implementation was originally formulated in regex many moons ago: +https://github.com/rust-lang/regex/blob/3db8722d0b204a85380fe2a65e13d7065d7dd968/src/literal/imp.rs#L370-L501 +Prior to that, I'm not aware of anyone using this technique in any prominent +substring search implementation. Although, I'm sure folks have had this same +insight long before me. + +Another version of this also appeared in bstr: +https://github.com/BurntSushi/bstr/blob/a444256ca7407fe180ee32534688549655b7a38e/src/search/prefilter.rs#L83-L340 +*/ + +use crate::memmem::{ + prefilter::{PrefilterFnTy, PrefilterState}, + NeedleInfo, +}; + +// Check that the functions below satisfy the Prefilter function type. +const _: PrefilterFnTy = find; + +/// Look for a possible occurrence of needle. The position returned +/// corresponds to the beginning of the occurrence, if one exists. +/// +/// Callers may assume that this never returns false negatives (i.e., it +/// never misses an actual occurrence), but must check that the returned +/// position corresponds to a match. That is, it can return false +/// positives. +/// +/// This should only be used when Freqy is constructed for forward +/// searching. +pub(crate) fn find( + prestate: &mut PrefilterState, + ninfo: &NeedleInfo, + haystack: &[u8], + needle: &[u8], +) -> Option { + let mut i = 0; + let (rare1i, rare2i) = ninfo.rarebytes.as_rare_usize(); + let (rare1, rare2) = ninfo.rarebytes.as_rare_bytes(needle); + while prestate.is_effective() { + // Use a fast vectorized implementation to skip to the next + // occurrence of the rarest byte (heuristically chosen) in the + // needle. + let found = crate::memchr(rare1, &haystack[i..])?; + prestate.update(found); + i += found; + + // If we can't align our first match with the haystack, then a + // match is impossible. + if i < rare1i { + i += 1; + continue; + } + + // Align our rare2 byte with the haystack. A mismatch means that + // a match is impossible. + let aligned_rare2i = i - rare1i + rare2i; + if haystack.get(aligned_rare2i) != Some(&rare2) { + i += 1; + continue; + } + + // We've done what we can. There might be a match here. + return Some(i - rare1i); + } + // The only way we get here is if we believe our skipping heuristic + // has become ineffective. We're allowed to return false positives, + // so return the position at which we advanced to, aligned to the + // haystack. + Some(i.saturating_sub(rare1i)) +} + +#[cfg(all(test, feature = "std"))] +mod tests { + use super::*; + + fn freqy_find(haystack: &[u8], needle: &[u8]) -> Option { + let ninfo = NeedleInfo::new(needle); + let mut prestate = PrefilterState::new(); + find(&mut prestate, &ninfo, haystack, needle) + } + + #[test] + fn freqy_forward() { + assert_eq!(Some(0), freqy_find(b"BARFOO", b"BAR")); + assert_eq!(Some(3), freqy_find(b"FOOBAR", b"BAR")); + assert_eq!(Some(0), freqy_find(b"zyzz", b"zyzy")); + assert_eq!(Some(2), freqy_find(b"zzzy", b"zyzy")); + assert_eq!(None, freqy_find(b"zazb", b"zyzy")); + assert_eq!(Some(0), freqy_find(b"yzyy", b"yzyz")); + assert_eq!(Some(2), freqy_find(b"yyyz", b"yzyz")); + assert_eq!(None, freqy_find(b"yayb", b"yzyz")); + } + + #[test] + #[cfg(not(miri))] + fn prefilter_permutations() { + use crate::memmem::prefilter::tests::PrefilterTest; + + // SAFETY: super::find is safe to call for all inputs and on all + // platforms. + unsafe { PrefilterTest::run_all_tests(super::find) }; + } +} diff --git a/crux-mir/lib/memchr/src/memmem/prefilter/genericsimd.rs b/crux-mir/lib/memchr/src/memmem/prefilter/genericsimd.rs new file mode 100644 index 000000000..1a6e38734 --- /dev/null +++ b/crux-mir/lib/memchr/src/memmem/prefilter/genericsimd.rs @@ -0,0 +1,207 @@ +use core::mem::size_of; + +use crate::memmem::{ + prefilter::{PrefilterFnTy, PrefilterState}, + vector::Vector, + NeedleInfo, +}; + +/// The implementation of the forward vector accelerated candidate finder. +/// +/// This is inspired by the "generic SIMD" algorithm described here: +/// http://0x80.pl/articles/simd-strfind.html#algorithm-1-generic-simd +/// +/// The main difference is that this is just a prefilter. That is, it reports +/// candidates once they are seen and doesn't attempt to confirm them. Also, +/// the bytes this routine uses to check for candidates are selected based on +/// an a priori background frequency distribution. This means that on most +/// haystacks, this will on average spend more time in vectorized code than you +/// would if you just selected the first and last bytes of the needle. +/// +/// Note that a non-prefilter variant of this algorithm can be found in the +/// parent module, but it only works on smaller needles. +/// +/// `prestate`, `ninfo`, `haystack` and `needle` are the four prefilter +/// function parameters. `fallback` is a prefilter that is used if the haystack +/// is too small to be handled with the given vector size. +/// +/// This routine is not safe because it is intended for callers to specialize +/// this with a particular vector (e.g., __m256i) and then call it with the +/// relevant target feature (e.g., avx2) enabled. +/// +/// # Panics +/// +/// If `needle.len() <= 1`, then this panics. +/// +/// # Safety +/// +/// Since this is meant to be used with vector functions, callers need to +/// specialize this inside of a function with a `target_feature` attribute. +/// Therefore, callers must ensure that whatever target feature is being used +/// supports the vector functions that this function is specialized for. (For +/// the specific vector functions used, see the Vector trait implementations.) +#[inline(always)] +pub(crate) unsafe fn find( + prestate: &mut PrefilterState, + ninfo: &NeedleInfo, + haystack: &[u8], + needle: &[u8], + fallback: PrefilterFnTy, +) -> Option { + assert!(needle.len() >= 2, "needle must be at least 2 bytes"); + let (rare1i, rare2i) = ninfo.rarebytes.as_rare_ordered_usize(); + let min_haystack_len = rare2i + size_of::(); + if haystack.len() < min_haystack_len { + return fallback(prestate, ninfo, haystack, needle); + } + + let start_ptr = haystack.as_ptr(); + let end_ptr = start_ptr.add(haystack.len()); + let max_ptr = end_ptr.sub(min_haystack_len); + let mut ptr = start_ptr; + + let rare1chunk = V::splat(needle[rare1i]); + let rare2chunk = V::splat(needle[rare2i]); + + // N.B. I did experiment with unrolling the loop to deal with size(V) + // bytes at a time and 2*size(V) bytes at a time. The double unroll + // was marginally faster while the quadruple unroll was unambiguously + // slower. In the end, I decided the complexity from unrolling wasn't + // worth it. I used the memmem/krate/prebuilt/huge-en/ benchmarks to + // compare. + while ptr <= max_ptr { + let m = find_in_chunk2(ptr, rare1i, rare2i, rare1chunk, rare2chunk); + if let Some(chunki) = m { + return Some(matched(prestate, start_ptr, ptr, chunki)); + } + ptr = ptr.add(size_of::()); + } + if ptr < end_ptr { + // This routine immediately quits if a candidate match is found. + // That means that if we're here, no candidate matches have been + // found at or before 'ptr'. Thus, we don't need to mask anything + // out even though we might technically search part of the haystack + // that we've already searched (because we know it can't match). + ptr = max_ptr; + let m = find_in_chunk2(ptr, rare1i, rare2i, rare1chunk, rare2chunk); + if let Some(chunki) = m { + return Some(matched(prestate, start_ptr, ptr, chunki)); + } + } + prestate.update(haystack.len()); + None +} + +// Below are two different techniques for checking whether a candidate +// match exists in a given chunk or not. find_in_chunk2 checks two bytes +// where as find_in_chunk3 checks three bytes. The idea behind checking +// three bytes is that while we do a bit more work per iteration, we +// decrease the chances of a false positive match being reported and thus +// make the search faster overall. This actually works out for the +// memmem/krate/prebuilt/huge-en/never-all-common-bytes benchmark, where +// using find_in_chunk3 is about 25% faster than find_in_chunk2. However, +// it turns out that find_in_chunk2 is faster for all other benchmarks, so +// perhaps the extra check isn't worth it in practice. +// +// For now, we go with find_in_chunk2, but we leave find_in_chunk3 around +// to make it easy to switch to and benchmark when possible. + +/// Search for an occurrence of two rare bytes from the needle in the current +/// chunk pointed to by ptr. +/// +/// rare1chunk and rare2chunk correspond to vectors with the rare1 and rare2 +/// bytes repeated in each 8-bit lane, respectively. +/// +/// # Safety +/// +/// It must be safe to do an unaligned read of size(V) bytes starting at both +/// (ptr + rare1i) and (ptr + rare2i). +#[inline(always)] +unsafe fn find_in_chunk2( + ptr: *const u8, + rare1i: usize, + rare2i: usize, + rare1chunk: V, + rare2chunk: V, +) -> Option { + let chunk0 = V::load_unaligned(ptr.add(rare1i)); + let chunk1 = V::load_unaligned(ptr.add(rare2i)); + + let eq0 = chunk0.cmpeq(rare1chunk); + let eq1 = chunk1.cmpeq(rare2chunk); + + let match_offsets = eq0.and(eq1).movemask(); + if match_offsets == 0 { + return None; + } + Some(match_offsets.trailing_zeros() as usize) +} + +/// Search for an occurrence of two rare bytes and the first byte (even if one +/// of the rare bytes is equivalent to the first byte) from the needle in the +/// current chunk pointed to by ptr. +/// +/// firstchunk, rare1chunk and rare2chunk correspond to vectors with the first, +/// rare1 and rare2 bytes repeated in each 8-bit lane, respectively. +/// +/// # Safety +/// +/// It must be safe to do an unaligned read of size(V) bytes starting at ptr, +/// (ptr + rare1i) and (ptr + rare2i). +#[allow(dead_code)] +#[inline(always)] +unsafe fn find_in_chunk3( + ptr: *const u8, + rare1i: usize, + rare2i: usize, + firstchunk: V, + rare1chunk: V, + rare2chunk: V, +) -> Option { + let chunk0 = V::load_unaligned(ptr); + let chunk1 = V::load_unaligned(ptr.add(rare1i)); + let chunk2 = V::load_unaligned(ptr.add(rare2i)); + + let eq0 = chunk0.cmpeq(firstchunk); + let eq1 = chunk1.cmpeq(rare1chunk); + let eq2 = chunk2.cmpeq(rare2chunk); + + let match_offsets = eq0.and(eq1).and(eq2).movemask(); + if match_offsets == 0 { + return None; + } + Some(match_offsets.trailing_zeros() as usize) +} + +/// Accepts a chunk-relative offset and returns a haystack relative offset +/// after updating the prefilter state. +/// +/// Why do we use this unlineable function when a search completes? Well, +/// I don't know. Really. Obviously this function was not here initially. +/// When doing profiling, the codegen for the inner loop here looked bad and +/// I didn't know why. There were a couple extra 'add' instructions and an +/// extra 'lea' instruction that I couldn't explain. I hypothesized that the +/// optimizer was having trouble untangling the hot code in the loop from the +/// code that deals with a candidate match. By putting the latter into an +/// unlineable function, it kind of forces the issue and it had the intended +/// effect: codegen improved measurably. It's good for a ~10% improvement +/// across the board on the memmem/krate/prebuilt/huge-en/ benchmarks. +#[cold] +#[inline(never)] +fn matched( + prestate: &mut PrefilterState, + start_ptr: *const u8, + ptr: *const u8, + chunki: usize, +) -> usize { + let found = diff(ptr, start_ptr) + chunki; + prestate.update(found); + found +} + +/// Subtract `b` from `a` and return the difference. `a` must be greater than +/// or equal to `b`. +fn diff(a: *const u8, b: *const u8) -> usize { + debug_assert!(a >= b); + (a as usize) - (b as usize) +} diff --git a/crux-mir/lib/memchr/src/memmem/prefilter/mod.rs b/crux-mir/lib/memchr/src/memmem/prefilter/mod.rs new file mode 100644 index 000000000..015d3b27a --- /dev/null +++ b/crux-mir/lib/memchr/src/memmem/prefilter/mod.rs @@ -0,0 +1,570 @@ +use crate::memmem::{rarebytes::RareNeedleBytes, NeedleInfo}; + +mod fallback; +#[cfg(memchr_runtime_simd)] +mod genericsimd; +#[cfg(all(not(miri), target_arch = "wasm32", memchr_runtime_simd))] +mod wasm; +#[cfg(all(not(miri), target_arch = "x86_64", memchr_runtime_simd))] +mod x86; + +/// The maximum frequency rank permitted for the fallback prefilter. If the +/// rarest byte in the needle has a frequency rank above this value, then no +/// prefilter is used if the fallback prefilter would otherwise be selected. +const MAX_FALLBACK_RANK: usize = 250; + +/// A combination of prefilter effectiveness state, the prefilter function and +/// the needle info required to run a prefilter. +/// +/// For the most part, these are grouped into a single type for convenience, +/// instead of needing to pass around all three as distinct function +/// parameters. +pub(crate) struct Pre<'a> { + /// State that tracks the effectiveness of a prefilter. + pub(crate) state: &'a mut PrefilterState, + /// The actual prefilter function. + pub(crate) prefn: PrefilterFn, + /// Information about a needle, such as its RK hash and rare byte offsets. + pub(crate) ninfo: &'a NeedleInfo, +} + +impl<'a> Pre<'a> { + /// Call this prefilter on the given haystack with the given needle. + #[inline(always)] + pub(crate) fn call( + &mut self, + haystack: &[u8], + needle: &[u8], + ) -> Option { + self.prefn.call(self.state, self.ninfo, haystack, needle) + } + + /// Return true if and only if this prefilter should be used. + #[inline(always)] + pub(crate) fn should_call(&mut self) -> bool { + self.state.is_effective() + } +} + +/// A prefilter function. +/// +/// A prefilter function describes both forward and reverse searches. +/// (Although, we don't currently implement prefilters for reverse searching.) +/// In the case of a forward search, the position returned corresponds to +/// the starting offset of a match (confirmed or possible). Its minimum +/// value is `0`, and its maximum value is `haystack.len() - 1`. In the case +/// of a reverse search, the position returned corresponds to the position +/// immediately after a match (confirmed or possible). Its minimum value is `1` +/// and its maximum value is `haystack.len()`. +/// +/// In both cases, the position returned is the starting (or ending) point of a +/// _possible_ match. That is, returning a false positive is okay. A prefilter, +/// however, must never return any false negatives. That is, if a match exists +/// at a particular position `i`, then a prefilter _must_ return that position. +/// It cannot skip past it. +/// +/// # Safety +/// +/// A prefilter function is not safe to create, since not all prefilters are +/// safe to call in all contexts. (e.g., A prefilter that uses AVX instructions +/// may only be called on x86_64 CPUs with the relevant AVX feature enabled.) +/// Thus, callers must ensure that when a prefilter function is created that it +/// is safe to call for the current environment. +#[derive(Clone, Copy)] +pub(crate) struct PrefilterFn(PrefilterFnTy); + +/// The type of a prefilter function. All prefilters must satisfy this +/// signature. +/// +/// Using a function pointer like this does inhibit inlining, but it does +/// eliminate branching and the extra costs associated with copying a larger +/// enum. Note also, that using Box can't really work +/// here, since we want to work in contexts that don't have dynamic memory +/// allocation. Moreover, in the default configuration of this crate on x86_64 +/// CPUs released in the past ~decade, we will use an AVX2-optimized prefilter, +/// which generally won't be inlineable into the surrounding code anyway. +/// (Unless AVX2 is enabled at compile time, but this is typically rare, since +/// it produces a non-portable binary.) +pub(crate) type PrefilterFnTy = unsafe fn( + prestate: &mut PrefilterState, + ninfo: &NeedleInfo, + haystack: &[u8], + needle: &[u8], +) -> Option; + +// If the haystack is too small for SSE2, then just run memchr on the +// rarest byte and be done with it. (It is likely that this code path is +// rarely exercised, since a higher level routine will probably dispatch to +// Rabin-Karp for such a small haystack.) +#[cfg(memchr_runtime_simd)] +fn simple_memchr_fallback( + _prestate: &mut PrefilterState, + ninfo: &NeedleInfo, + haystack: &[u8], + needle: &[u8], +) -> Option { + let (rare, _) = ninfo.rarebytes.as_rare_ordered_usize(); + crate::memchr(needle[rare], haystack).map(|i| i.saturating_sub(rare)) +} + +impl PrefilterFn { + /// Create a new prefilter function from the function pointer given. + /// + /// # Safety + /// + /// Callers must ensure that the given prefilter function is safe to call + /// for all inputs in the current environment. For example, if the given + /// prefilter function uses AVX instructions, then the caller must ensure + /// that the appropriate AVX CPU features are enabled. + pub(crate) unsafe fn new(prefn: PrefilterFnTy) -> PrefilterFn { + PrefilterFn(prefn) + } + + /// Call the underlying prefilter function with the given arguments. + pub fn call( + self, + prestate: &mut PrefilterState, + ninfo: &NeedleInfo, + haystack: &[u8], + needle: &[u8], + ) -> Option { + // SAFETY: Callers have the burden of ensuring that a prefilter + // function is safe to call for all inputs in the current environment. + unsafe { (self.0)(prestate, ninfo, haystack, needle) } + } +} + +impl core::fmt::Debug for PrefilterFn { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + "".fmt(f) + } +} + +/// Prefilter controls whether heuristics are used to accelerate searching. +/// +/// A prefilter refers to the idea of detecting candidate matches very quickly, +/// and then confirming whether those candidates are full matches. This +/// idea can be quite effective since it's often the case that looking for +/// candidates can be a lot faster than running a complete substring search +/// over the entire input. Namely, looking for candidates can be done with +/// extremely fast vectorized code. +/// +/// The downside of a prefilter is that it assumes false positives (which are +/// candidates generated by a prefilter that aren't matches) are somewhat rare +/// relative to the frequency of full matches. That is, if a lot of false +/// positives are generated, then it's possible for search time to be worse +/// than if the prefilter wasn't enabled in the first place. +/// +/// Another downside of a prefilter is that it can result in highly variable +/// performance, where some cases are extraordinarily fast and others aren't. +/// Typically, variable performance isn't a problem, but it may be for your use +/// case. +/// +/// The use of prefilters in this implementation does use a heuristic to detect +/// when a prefilter might not be carrying its weight, and will dynamically +/// disable its use. Nevertheless, this configuration option gives callers +/// the ability to disable prefilters if you have knowledge that they won't be +/// useful. +#[derive(Clone, Copy, Debug)] +#[non_exhaustive] +pub enum Prefilter { + /// Never used a prefilter in substring search. + None, + /// Automatically detect whether a heuristic prefilter should be used. If + /// it is used, then heuristics will be used to dynamically disable the + /// prefilter if it is believed to not be carrying its weight. + Auto, +} + +impl Default for Prefilter { + fn default() -> Prefilter { + Prefilter::Auto + } +} + +impl Prefilter { + pub(crate) fn is_none(&self) -> bool { + match *self { + Prefilter::None => true, + _ => false, + } + } +} + +/// PrefilterState tracks state associated with the effectiveness of a +/// prefilter. It is used to track how many bytes, on average, are skipped by +/// the prefilter. If this average dips below a certain threshold over time, +/// then the state renders the prefilter inert and stops using it. +/// +/// A prefilter state should be created for each search. (Where creating an +/// iterator is treated as a single search.) A prefilter state should only be +/// created from a `Freqy`. e.g., An inert `Freqy` will produce an inert +/// `PrefilterState`. +#[derive(Clone, Debug)] +pub(crate) struct PrefilterState { + /// The number of skips that has been executed. This is always 1 greater + /// than the actual number of skips. The special sentinel value of 0 + /// indicates that the prefilter is inert. This is useful to avoid + /// additional checks to determine whether the prefilter is still + /// "effective." Once a prefilter becomes inert, it should no longer be + /// used (according to our heuristics). + skips: u32, + /// The total number of bytes that have been skipped. + skipped: u32, +} + +impl PrefilterState { + /// The minimum number of skip attempts to try before considering whether + /// a prefilter is effective or not. + const MIN_SKIPS: u32 = 50; + + /// The minimum amount of bytes that skipping must average. + /// + /// This value was chosen based on varying it and checking + /// the microbenchmarks. In particular, this can impact the + /// pathological/repeated-{huge,small} benchmarks quite a bit if it's set + /// too low. + const MIN_SKIP_BYTES: u32 = 8; + + /// Create a fresh prefilter state. + pub(crate) fn new() -> PrefilterState { + PrefilterState { skips: 1, skipped: 0 } + } + + /// Create a fresh prefilter state that is always inert. + pub(crate) fn inert() -> PrefilterState { + PrefilterState { skips: 0, skipped: 0 } + } + + /// Update this state with the number of bytes skipped on the last + /// invocation of the prefilter. + #[inline] + pub(crate) fn update(&mut self, skipped: usize) { + self.skips = self.skips.saturating_add(1); + // We need to do this dance since it's technically possible for + // `skipped` to overflow a `u32`. (And we use a `u32` to reduce the + // size of a prefilter state.) + if skipped > core::u32::MAX as usize { + self.skipped = core::u32::MAX; + } else { + self.skipped = self.skipped.saturating_add(skipped as u32); + } + } + + /// Return true if and only if this state indicates that a prefilter is + /// still effective. + #[inline] + pub(crate) fn is_effective(&mut self) -> bool { + if self.is_inert() { + return false; + } + if self.skips() < PrefilterState::MIN_SKIPS { + return true; + } + if self.skipped >= PrefilterState::MIN_SKIP_BYTES * self.skips() { + return true; + } + + // We're inert. + self.skips = 0; + false + } + + #[inline] + fn is_inert(&self) -> bool { + self.skips == 0 + } + + #[inline] + fn skips(&self) -> u32 { + self.skips.saturating_sub(1) + } +} + +/// Determine which prefilter function, if any, to use. +/// +/// This only applies to x86_64 when runtime SIMD detection is enabled (which +/// is the default). In general, we try to use an AVX prefilter, followed by +/// SSE and then followed by a generic one based on memchr. +#[inline(always)] +pub(crate) fn forward( + config: &Prefilter, + rare: &RareNeedleBytes, + needle: &[u8], +) -> Option { + if config.is_none() || needle.len() <= 1 { + return None; + } + + #[cfg(all(not(miri), target_arch = "x86_64", memchr_runtime_simd))] + { + #[cfg(feature = "std")] + { + if cfg!(memchr_runtime_avx) { + if is_x86_feature_detected!("avx2") { + // SAFETY: x86::avx::find only requires the avx2 feature, + // which we've just checked above. + return unsafe { Some(PrefilterFn::new(x86::avx::find)) }; + } + } + } + if cfg!(memchr_runtime_sse2) { + // SAFETY: x86::sse::find only requires the sse2 feature, which is + // guaranteed to be available on x86_64. + return unsafe { Some(PrefilterFn::new(x86::sse::find)) }; + } + } + #[cfg(all(not(miri), target_arch = "wasm32", memchr_runtime_simd))] + { + // SAFETY: `wasm::find` is actually a safe function + // + // Also note that the `if true` is here to prevent, on wasm with simd, + // rustc warning about the code below being dead code. + if true { + return unsafe { Some(PrefilterFn::new(wasm::find)) }; + } + } + // Check that our rarest byte has a reasonably low rank. The main issue + // here is that the fallback prefilter can perform pretty poorly if it's + // given common bytes. So we try to avoid the worst cases here. + let (rare1_rank, _) = rare.as_ranks(needle); + if rare1_rank <= MAX_FALLBACK_RANK { + // SAFETY: fallback::find is safe to call in all environments. + return unsafe { Some(PrefilterFn::new(fallback::find)) }; + } + None +} + +/// Return the minimum length of the haystack in which a prefilter should be +/// used. If the haystack is below this length, then it's probably not worth +/// the overhead of running the prefilter. +/// +/// We used to look at the length of a haystack here. That is, if it was too +/// small, then don't bother with the prefilter. But two things changed: +/// the prefilter falls back to memchr for small haystacks, and, at the +/// meta-searcher level, Rabin-Karp is employed for tiny haystacks anyway. +/// +/// We keep it around for now in case we want to bring it back. +#[allow(dead_code)] +pub(crate) fn minimum_len(_haystack: &[u8], needle: &[u8]) -> usize { + // If the haystack length isn't greater than needle.len() * FACTOR, then + // no prefilter will be used. The presumption here is that since there + // are so few bytes to check, it's not worth running the prefilter since + // there will need to be a validation step anyway. Thus, the prefilter is + // largely redundant work. + // + // Increase the factor noticeably hurts the + // memmem/krate/prebuilt/teeny-*/never-john-watson benchmarks. + const PREFILTER_LENGTH_FACTOR: usize = 2; + const VECTOR_MIN_LENGTH: usize = 16; + let min = core::cmp::max( + VECTOR_MIN_LENGTH, + PREFILTER_LENGTH_FACTOR * needle.len(), + ); + // For haystacks with length==min, we still want to avoid the prefilter, + // so add 1. + min + 1 +} + +#[cfg(all(test, feature = "std", not(miri)))] +pub(crate) mod tests { + use std::convert::{TryFrom, TryInto}; + + use super::*; + use crate::memmem::{ + prefilter::PrefilterFnTy, rabinkarp, rarebytes::RareNeedleBytes, + }; + + // Below is a small jig that generates prefilter tests. The main purpose + // of this jig is to generate tests of varying needle/haystack lengths + // in order to try and exercise all code paths in our prefilters. And in + // particular, this is especially important for vectorized prefilters where + // certain code paths might only be exercised at certain lengths. + + /// A test that represents the input and expected output to a prefilter + /// function. The test should be able to run with any prefilter function + /// and get the expected output. + pub(crate) struct PrefilterTest { + // These fields represent the inputs and expected output of a forwards + // prefilter function. + pub(crate) ninfo: NeedleInfo, + pub(crate) haystack: Vec, + pub(crate) needle: Vec, + pub(crate) output: Option, + } + + impl PrefilterTest { + /// Run all generated forward prefilter tests on the given prefn. + /// + /// # Safety + /// + /// Callers must ensure that the given prefilter function pointer is + /// safe to call for all inputs in the current environment. + pub(crate) unsafe fn run_all_tests(prefn: PrefilterFnTy) { + PrefilterTest::run_all_tests_filter(prefn, |_| true) + } + + /// Run all generated forward prefilter tests that pass the given + /// predicate on the given prefn. + /// + /// # Safety + /// + /// Callers must ensure that the given prefilter function pointer is + /// safe to call for all inputs in the current environment. + pub(crate) unsafe fn run_all_tests_filter( + prefn: PrefilterFnTy, + mut predicate: impl FnMut(&PrefilterTest) -> bool, + ) { + for seed in PREFILTER_TEST_SEEDS { + for test in seed.generate() { + if predicate(&test) { + test.run(prefn); + } + } + } + } + + /// Create a new prefilter test from a seed and some chose offsets to + /// rare bytes in the seed's needle. + /// + /// If a valid test could not be constructed, then None is returned. + /// (Currently, we take the approach of massaging tests to be valid + /// instead of rejecting them outright.) + fn new( + seed: PrefilterTestSeed, + rare1i: usize, + rare2i: usize, + haystack_len: usize, + needle_len: usize, + output: Option, + ) -> Option { + let mut rare1i: u8 = rare1i.try_into().unwrap(); + let mut rare2i: u8 = rare2i.try_into().unwrap(); + // The '#' byte is never used in a haystack (unless we're expecting + // a match), while the '@' byte is never used in a needle. + let mut haystack = vec![b'@'; haystack_len]; + let mut needle = vec![b'#'; needle_len]; + needle[0] = seed.first; + needle[rare1i as usize] = seed.rare1; + needle[rare2i as usize] = seed.rare2; + // If we're expecting a match, then make sure the needle occurs + // in the haystack at the expected position. + if let Some(i) = output { + haystack[i..i + needle.len()].copy_from_slice(&needle); + } + // If the operations above lead to rare offsets pointing to the + // non-first occurrence of a byte, then adjust it. This might lead + // to redundant tests, but it's simpler than trying to change the + // generation process I think. + if let Some(i) = crate::memchr(seed.rare1, &needle) { + rare1i = u8::try_from(i).unwrap(); + } + if let Some(i) = crate::memchr(seed.rare2, &needle) { + rare2i = u8::try_from(i).unwrap(); + } + let ninfo = NeedleInfo { + rarebytes: RareNeedleBytes::new(rare1i, rare2i), + nhash: rabinkarp::NeedleHash::forward(&needle), + }; + Some(PrefilterTest { ninfo, haystack, needle, output }) + } + + /// Run this specific test on the given prefilter function. If the + /// outputs do no match, then this routine panics with a failure + /// message. + /// + /// # Safety + /// + /// Callers must ensure that the given prefilter function pointer is + /// safe to call for all inputs in the current environment. + unsafe fn run(&self, prefn: PrefilterFnTy) { + let mut prestate = PrefilterState::new(); + assert_eq!( + self.output, + prefn( + &mut prestate, + &self.ninfo, + &self.haystack, + &self.needle + ), + "ninfo: {:?}, haystack(len={}): {:?}, needle(len={}): {:?}", + self.ninfo, + self.haystack.len(), + std::str::from_utf8(&self.haystack).unwrap(), + self.needle.len(), + std::str::from_utf8(&self.needle).unwrap(), + ); + } + } + + /// A set of prefilter test seeds. Each seed serves as the base for the + /// generation of many other tests. In essence, the seed captures the + /// "rare" and first bytes among our needle. The tests generated from each + /// seed essentially vary the length of the needle and haystack, while + /// using the rare/first byte configuration from the seed. + /// + /// The purpose of this is to test many different needle/haystack lengths. + /// In particular, some of the vector optimizations might only have bugs + /// in haystacks of a certain size. + const PREFILTER_TEST_SEEDS: &[PrefilterTestSeed] = &[ + PrefilterTestSeed { first: b'x', rare1: b'y', rare2: b'z' }, + PrefilterTestSeed { first: b'x', rare1: b'x', rare2: b'z' }, + PrefilterTestSeed { first: b'x', rare1: b'y', rare2: b'x' }, + PrefilterTestSeed { first: b'x', rare1: b'x', rare2: b'x' }, + PrefilterTestSeed { first: b'x', rare1: b'y', rare2: b'y' }, + ]; + + /// Data that describes a single prefilter test seed. + #[derive(Clone, Copy)] + struct PrefilterTestSeed { + first: u8, + rare1: u8, + rare2: u8, + } + + impl PrefilterTestSeed { + /// Generate a series of prefilter tests from this seed. + fn generate(self) -> impl Iterator { + let len_start = 2; + // The iterator below generates *a lot* of tests. The number of + // tests was chosen somewhat empirically to be "bearable" when + // running the test suite. + // + // We use an iterator here because the collective haystacks of all + // these test cases add up to enough memory to OOM a conservative + // sandbox or a small laptop. + (len_start..=40).flat_map(move |needle_len| { + let rare_start = len_start - 1; + (rare_start..needle_len).flat_map(move |rare1i| { + (rare1i..needle_len).flat_map(move |rare2i| { + (needle_len..=66).flat_map(move |haystack_len| { + PrefilterTest::new( + self, + rare1i, + rare2i, + haystack_len, + needle_len, + None, + ) + .into_iter() + .chain( + (0..=(haystack_len - needle_len)).flat_map( + move |output| { + PrefilterTest::new( + self, + rare1i, + rare2i, + haystack_len, + needle_len, + Some(output), + ) + }, + ), + ) + }) + }) + }) + }) + } + } +} diff --git a/crux-mir/lib/memchr/src/memmem/prefilter/wasm.rs b/crux-mir/lib/memchr/src/memmem/prefilter/wasm.rs new file mode 100644 index 000000000..5470c922a --- /dev/null +++ b/crux-mir/lib/memchr/src/memmem/prefilter/wasm.rs @@ -0,0 +1,39 @@ +use core::arch::wasm32::v128; + +use crate::memmem::{ + prefilter::{PrefilterFnTy, PrefilterState}, + NeedleInfo, +}; + +// Check that the functions below satisfy the Prefilter function type. +const _: PrefilterFnTy = find; + +/// A `v128`-accelerated candidate finder for single-substring search. +#[target_feature(enable = "simd128")] +pub(crate) fn find( + prestate: &mut PrefilterState, + ninfo: &NeedleInfo, + haystack: &[u8], + needle: &[u8], +) -> Option { + unsafe { + super::genericsimd::find::( + prestate, + ninfo, + haystack, + needle, + super::simple_memchr_fallback, + ) + } +} + +#[cfg(all(test, feature = "std"))] +mod tests { + #[test] + #[cfg(not(miri))] + fn prefilter_permutations() { + use crate::memmem::prefilter::tests::PrefilterTest; + // SAFETY: super::find is safe to call for all inputs on x86. + unsafe { PrefilterTest::run_all_tests(super::find) }; + } +} diff --git a/crux-mir/lib/memchr/src/memmem/prefilter/x86/avx.rs b/crux-mir/lib/memchr/src/memmem/prefilter/x86/avx.rs new file mode 100644 index 000000000..fb11f335b --- /dev/null +++ b/crux-mir/lib/memchr/src/memmem/prefilter/x86/avx.rs @@ -0,0 +1,46 @@ +use core::arch::x86_64::__m256i; + +use crate::memmem::{ + prefilter::{PrefilterFnTy, PrefilterState}, + NeedleInfo, +}; + +// Check that the functions below satisfy the Prefilter function type. +const _: PrefilterFnTy = find; + +/// An AVX2 accelerated candidate finder for single-substring search. +/// +/// # Safety +/// +/// Callers must ensure that the avx2 CPU feature is enabled in the current +/// environment. +#[target_feature(enable = "avx2")] +pub(crate) unsafe fn find( + prestate: &mut PrefilterState, + ninfo: &NeedleInfo, + haystack: &[u8], + needle: &[u8], +) -> Option { + super::super::genericsimd::find::<__m256i>( + prestate, + ninfo, + haystack, + needle, + super::sse::find, + ) +} + +#[cfg(test)] +mod tests { + #[test] + #[cfg(not(miri))] + fn prefilter_permutations() { + use crate::memmem::prefilter::tests::PrefilterTest; + if !is_x86_feature_detected!("avx2") { + return; + } + // SAFETY: The safety of super::find only requires that the current + // CPU support AVX2, which we checked above. + unsafe { PrefilterTest::run_all_tests(super::find) }; + } +} diff --git a/crux-mir/lib/memchr/src/memmem/prefilter/x86/mod.rs b/crux-mir/lib/memchr/src/memmem/prefilter/x86/mod.rs new file mode 100644 index 000000000..91381e516 --- /dev/null +++ b/crux-mir/lib/memchr/src/memmem/prefilter/x86/mod.rs @@ -0,0 +1,5 @@ +// We only use AVX when we can detect at runtime whether it's available, which +// requires std. +#[cfg(feature = "std")] +pub(crate) mod avx; +pub(crate) mod sse; diff --git a/crux-mir/lib/memchr/src/memmem/prefilter/x86/sse.rs b/crux-mir/lib/memchr/src/memmem/prefilter/x86/sse.rs new file mode 100644 index 000000000..b1c48e1e1 --- /dev/null +++ b/crux-mir/lib/memchr/src/memmem/prefilter/x86/sse.rs @@ -0,0 +1,42 @@ +use core::arch::x86_64::__m128i; + +use crate::memmem::{ + prefilter::{PrefilterFnTy, PrefilterState}, + NeedleInfo, +}; + +// Check that the functions below satisfy the Prefilter function type. +const _: PrefilterFnTy = find; + +/// An SSE2 accelerated candidate finder for single-substring search. +/// +/// # Safety +/// +/// Callers must ensure that the sse2 CPU feature is enabled in the current +/// environment. This feature should be enabled in all x86_64 targets. +#[target_feature(enable = "sse2")] +pub(crate) unsafe fn find( + prestate: &mut PrefilterState, + ninfo: &NeedleInfo, + haystack: &[u8], + needle: &[u8], +) -> Option { + super::super::genericsimd::find::<__m128i>( + prestate, + ninfo, + haystack, + needle, + super::super::simple_memchr_fallback, + ) +} + +#[cfg(all(test, feature = "std"))] +mod tests { + #[test] + #[cfg(not(miri))] + fn prefilter_permutations() { + use crate::memmem::prefilter::tests::PrefilterTest; + // SAFETY: super::find is safe to call for all inputs on x86. + unsafe { PrefilterTest::run_all_tests(super::find) }; + } +} diff --git a/crux-mir/lib/memchr/src/memmem/rabinkarp.rs b/crux-mir/lib/memchr/src/memmem/rabinkarp.rs new file mode 100644 index 000000000..daa4015d5 --- /dev/null +++ b/crux-mir/lib/memchr/src/memmem/rabinkarp.rs @@ -0,0 +1,233 @@ +/* +This module implements the classical Rabin-Karp substring search algorithm, +with no extra frills. While its use would seem to break our time complexity +guarantee of O(m+n) (RK's time complexity is O(mn)), we are careful to only +ever use RK on a constant subset of haystacks. The main point here is that +RK has good latency properties for small needles/haystacks. It's very quick +to compute a needle hash and zip through the haystack when compared to +initializing Two-Way, for example. And this is especially useful for cases +where the haystack is just too short for vector instructions to do much good. + +The hashing function used here is the same one recommended by ESMAJ. + +Another choice instead of Rabin-Karp would be Shift-Or. But its latency +isn't quite as good since its preprocessing time is a bit more expensive +(both in practice and in theory). However, perhaps Shift-Or has a place +somewhere else for short patterns. I think the main problem is that it +requires space proportional to the alphabet and the needle. If we, for +example, supported needles up to length 16, then the total table size would be +len(alphabet)*size_of::()==512 bytes. Which isn't exactly small, and it's +probably bad to put that on the stack. So ideally, we'd throw it on the heap, +but we'd really like to write as much code without using alloc/std as possible. +But maybe it's worth the special casing. It's a TODO to benchmark. + +Wikipedia has a decent explanation, if a bit heavy on the theory: +https://en.wikipedia.org/wiki/Rabin%E2%80%93Karp_algorithm + +But ESMAJ provides something a bit more concrete: +http://www-igm.univ-mlv.fr/~lecroq/string/node5.html + +Finally, aho-corasick uses Rabin-Karp for multiple pattern match in some cases: +https://github.com/BurntSushi/aho-corasick/blob/3852632f10587db0ff72ef29e88d58bf305a0946/src/packed/rabinkarp.rs +*/ + +/// Whether RK is believed to be very fast for the given needle/haystack. +pub(crate) fn is_fast(haystack: &[u8], _needle: &[u8]) -> bool { + haystack.len() < 16 +} + +/// Search for the first occurrence of needle in haystack using Rabin-Karp. +pub(crate) fn find(haystack: &[u8], needle: &[u8]) -> Option { + find_with(&NeedleHash::forward(needle), haystack, needle) +} + +/// Search for the first occurrence of needle in haystack using Rabin-Karp with +/// a pre-computed needle hash. +pub(crate) fn find_with( + nhash: &NeedleHash, + mut haystack: &[u8], + needle: &[u8], +) -> Option { + if haystack.len() < needle.len() { + return None; + } + let start = haystack.as_ptr() as usize; + let mut hash = Hash::from_bytes_fwd(&haystack[..needle.len()]); + // N.B. I've experimented with unrolling this loop, but couldn't realize + // any obvious gains. + loop { + if nhash.eq(hash) && is_prefix(haystack, needle) { + return Some(haystack.as_ptr() as usize - start); + } + if needle.len() >= haystack.len() { + return None; + } + hash.roll(&nhash, haystack[0], haystack[needle.len()]); + haystack = &haystack[1..]; + } +} + +/// Search for the last occurrence of needle in haystack using Rabin-Karp. +pub(crate) fn rfind(haystack: &[u8], needle: &[u8]) -> Option { + rfind_with(&NeedleHash::reverse(needle), haystack, needle) +} + +/// Search for the last occurrence of needle in haystack using Rabin-Karp with +/// a pre-computed needle hash. +pub(crate) fn rfind_with( + nhash: &NeedleHash, + mut haystack: &[u8], + needle: &[u8], +) -> Option { + if haystack.len() < needle.len() { + return None; + } + let mut hash = + Hash::from_bytes_rev(&haystack[haystack.len() - needle.len()..]); + loop { + if nhash.eq(hash) && is_suffix(haystack, needle) { + return Some(haystack.len() - needle.len()); + } + if needle.len() >= haystack.len() { + return None; + } + hash.roll( + &nhash, + haystack[haystack.len() - 1], + haystack[haystack.len() - needle.len() - 1], + ); + haystack = &haystack[..haystack.len() - 1]; + } +} + +/// A hash derived from a needle. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct NeedleHash { + /// The actual hash. + hash: Hash, + /// The factor needed to multiply a byte by in order to subtract it from + /// the hash. It is defined to be 2^(n-1) (using wrapping exponentiation), + /// where n is the length of the needle. This is how we "remove" a byte + /// from the hash once the hash window rolls past it. + hash_2pow: u32, +} + +impl NeedleHash { + /// Create a new Rabin-Karp hash for the given needle for use in forward + /// searching. + pub(crate) fn forward(needle: &[u8]) -> NeedleHash { + let mut nh = NeedleHash { hash: Hash::new(), hash_2pow: 1 }; + if needle.is_empty() { + return nh; + } + nh.hash.add(needle[0]); + for &b in needle.iter().skip(1) { + nh.hash.add(b); + nh.hash_2pow = nh.hash_2pow.wrapping_shl(1); + } + nh + } + + /// Create a new Rabin-Karp hash for the given needle for use in reverse + /// searching. + pub(crate) fn reverse(needle: &[u8]) -> NeedleHash { + let mut nh = NeedleHash { hash: Hash::new(), hash_2pow: 1 }; + if needle.is_empty() { + return nh; + } + nh.hash.add(needle[needle.len() - 1]); + for &b in needle.iter().rev().skip(1) { + nh.hash.add(b); + nh.hash_2pow = nh.hash_2pow.wrapping_shl(1); + } + nh + } + + /// Return true if the hashes are equivalent. + fn eq(&self, hash: Hash) -> bool { + self.hash == hash + } +} + +/// A Rabin-Karp hash. This might represent the hash of a needle, or the hash +/// of a rolling window in the haystack. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(crate) struct Hash(u32); + +impl Hash { + /// Create a new hash that represents the empty string. + pub(crate) fn new() -> Hash { + Hash(0) + } + + /// Create a new hash from the bytes given for use in forward searches. + pub(crate) fn from_bytes_fwd(bytes: &[u8]) -> Hash { + let mut hash = Hash::new(); + for &b in bytes { + hash.add(b); + } + hash + } + + /// Create a new hash from the bytes given for use in reverse searches. + fn from_bytes_rev(bytes: &[u8]) -> Hash { + let mut hash = Hash::new(); + for &b in bytes.iter().rev() { + hash.add(b); + } + hash + } + + /// Add 'new' and remove 'old' from this hash. The given needle hash should + /// correspond to the hash computed for the needle being searched for. + /// + /// This is meant to be used when the rolling window of the haystack is + /// advanced. + fn roll(&mut self, nhash: &NeedleHash, old: u8, new: u8) { + self.del(nhash, old); + self.add(new); + } + + /// Add a byte to this hash. + fn add(&mut self, byte: u8) { + self.0 = self.0.wrapping_shl(1).wrapping_add(byte as u32); + } + + /// Remove a byte from this hash. The given needle hash should correspond + /// to the hash computed for the needle being searched for. + fn del(&mut self, nhash: &NeedleHash, byte: u8) { + let factor = nhash.hash_2pow; + self.0 = self.0.wrapping_sub((byte as u32).wrapping_mul(factor)); + } +} + +/// Returns true if the given needle is a prefix of the given haystack. +/// +/// We forcefully don't inline the is_prefix call and hint at the compiler that +/// it is unlikely to be called. This causes the inner rabinkarp loop above +/// to be a bit tighter and leads to some performance improvement. See the +/// memmem/krate/prebuilt/sliceslice-words/words benchmark. +#[cold] +#[inline(never)] +fn is_prefix(haystack: &[u8], needle: &[u8]) -> bool { + crate::memmem::util::is_prefix(haystack, needle) +} + +/// Returns true if the given needle is a suffix of the given haystack. +/// +/// See is_prefix for why this is forcefully not inlined. +#[cold] +#[inline(never)] +fn is_suffix(haystack: &[u8], needle: &[u8]) -> bool { + crate::memmem::util::is_suffix(haystack, needle) +} + +#[cfg(test)] +mod simpletests { + define_memmem_simple_tests!(super::find, super::rfind); +} + +#[cfg(all(test, feature = "std", not(miri)))] +mod proptests { + define_memmem_quickcheck_tests!(super::find, super::rfind); +} diff --git a/crux-mir/lib/memchr/src/memmem/rarebytes.rs b/crux-mir/lib/memchr/src/memmem/rarebytes.rs new file mode 100644 index 000000000..fb33f6894 --- /dev/null +++ b/crux-mir/lib/memchr/src/memmem/rarebytes.rs @@ -0,0 +1,136 @@ +/// A heuristic frequency based detection of rare bytes for substring search. +/// +/// This detector attempts to pick out two bytes in a needle that are predicted +/// to occur least frequently. The purpose is to use these bytes to implement +/// fast candidate search using vectorized code. +/// +/// A set of offsets is only computed for needles of length 2 or greater. +/// Smaller needles should be special cased by the substring search algorithm +/// in use. (e.g., Use memchr for single byte needles.) +/// +/// Note that we use `u8` to represent the offsets of the rare bytes in a +/// needle to reduce space usage. This means that rare byte occurring after the +/// first 255 bytes in a needle will never be used. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct RareNeedleBytes { + /// The leftmost offset of the rarest byte in the needle, according to + /// pre-computed frequency analysis. The "leftmost offset" means that + /// rare1i <= i for all i where needle[i] == needle[rare1i]. + rare1i: u8, + /// The leftmost offset of the second rarest byte in the needle, according + /// to pre-computed frequency analysis. The "leftmost offset" means that + /// rare2i <= i for all i where needle[i] == needle[rare2i]. + /// + /// The second rarest byte is used as a type of guard for quickly detecting + /// a mismatch if the first byte matches. This is a hedge against + /// pathological cases where the pre-computed frequency analysis may be + /// off. (But of course, does not prevent *all* pathological cases.) + /// + /// In general, rare1i != rare2i by construction, although there is no hard + /// requirement that they be different. However, since the case of a single + /// byte needle is handled specially by memchr itself, rare2i generally + /// always should be different from rare1i since it would otherwise be + /// ineffective as a guard. + rare2i: u8, +} + +impl RareNeedleBytes { + /// Create a new pair of rare needle bytes with the given offsets. This is + /// only used in tests for generating input data. + #[cfg(all(test, feature = "std"))] + pub(crate) fn new(rare1i: u8, rare2i: u8) -> RareNeedleBytes { + RareNeedleBytes { rare1i, rare2i } + } + + /// Detect the leftmost offsets of the two rarest bytes in the given + /// needle. + pub(crate) fn forward(needle: &[u8]) -> RareNeedleBytes { + if needle.len() <= 1 || needle.len() > core::u8::MAX as usize { + // For needles bigger than u8::MAX, our offsets aren't big enough. + // (We make our offsets small to reduce stack copying.) + // If you have a use case for it, please file an issue. In that + // case, we should probably just adjust the routine below to pick + // some rare bytes from the first 255 bytes of the needle. + // + // Also note that for needles of size 0 or 1, they are special + // cased in Two-Way. + // + // TODO: Benchmar this. + return RareNeedleBytes { rare1i: 0, rare2i: 0 }; + } + + // Find the rarest two bytes. We make them distinct by construction. + let (mut rare1, mut rare1i) = (needle[0], 0); + let (mut rare2, mut rare2i) = (needle[1], 1); + if rank(rare2) < rank(rare1) { + core::mem::swap(&mut rare1, &mut rare2); + core::mem::swap(&mut rare1i, &mut rare2i); + } + for (i, &b) in needle.iter().enumerate().skip(2) { + if rank(b) < rank(rare1) { + rare2 = rare1; + rare2i = rare1i; + rare1 = b; + rare1i = i as u8; + } else if b != rare1 && rank(b) < rank(rare2) { + rare2 = b; + rare2i = i as u8; + } + } + // While not strictly required, we really don't want these to be + // equivalent. If they were, it would reduce the effectiveness of + // candidate searching using these rare bytes by increasing the rate of + // false positives. + assert_ne!(rare1i, rare2i); + RareNeedleBytes { rare1i, rare2i } + } + + /// Return the rare bytes in the given needle in the forward direction. + /// The needle given must be the same one given to the RareNeedleBytes + /// constructor. + pub(crate) fn as_rare_bytes(&self, needle: &[u8]) -> (u8, u8) { + (needle[self.rare1i as usize], needle[self.rare2i as usize]) + } + + /// Return the rare offsets such that the first offset is always <= to the + /// second offset. This is useful when the caller doesn't care whether + /// rare1 is rarer than rare2, but just wants to ensure that they are + /// ordered with respect to one another. + #[cfg(memchr_runtime_simd)] + pub(crate) fn as_rare_ordered_usize(&self) -> (usize, usize) { + let (rare1i, rare2i) = self.as_rare_ordered_u8(); + (rare1i as usize, rare2i as usize) + } + + /// Like as_rare_ordered_usize, but returns the offsets as their native + /// u8 values. + #[cfg(memchr_runtime_simd)] + pub(crate) fn as_rare_ordered_u8(&self) -> (u8, u8) { + if self.rare1i <= self.rare2i { + (self.rare1i, self.rare2i) + } else { + (self.rare2i, self.rare1i) + } + } + + /// Return the rare offsets as usize values in the order in which they were + /// constructed. rare1, for example, is constructed as the "rarer" byte, + /// and thus, callers may want to treat it differently from rare2. + pub(crate) fn as_rare_usize(&self) -> (usize, usize) { + (self.rare1i as usize, self.rare2i as usize) + } + + /// Return the byte frequency rank of each byte. The higher the rank, the + /// more frequency the byte is predicted to be. The needle given must be + /// the same one given to the RareNeedleBytes constructor. + pub(crate) fn as_ranks(&self, needle: &[u8]) -> (usize, usize) { + let (b1, b2) = self.as_rare_bytes(needle); + (rank(b1), rank(b2)) + } +} + +/// Return the heuristical frequency rank of the given byte. A lower rank +/// means the byte is believed to occur less frequently. +fn rank(b: u8) -> usize { + crate::memmem::byte_frequencies::BYTE_FREQUENCIES[b as usize] as usize +} diff --git a/crux-mir/lib/memchr/src/memmem/twoway.rs b/crux-mir/lib/memchr/src/memmem/twoway.rs new file mode 100644 index 000000000..7f82ed15d --- /dev/null +++ b/crux-mir/lib/memchr/src/memmem/twoway.rs @@ -0,0 +1,878 @@ +use core::cmp; + +use crate::memmem::{prefilter::Pre, util}; + +/// Two-Way search in the forward direction. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Forward(TwoWay); + +/// Two-Way search in the reverse direction. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Reverse(TwoWay); + +/// An implementation of the TwoWay substring search algorithm, with heuristics +/// for accelerating search based on frequency analysis. +/// +/// This searcher supports forward and reverse search, although not +/// simultaneously. It runs in O(n + m) time and O(1) space, where +/// `n ~ len(needle)` and `m ~ len(haystack)`. +/// +/// The implementation here roughly matches that which was developed by +/// Crochemore and Perrin in their 1991 paper "Two-way string-matching." The +/// changes in this implementation are 1) the use of zero-based indices, 2) a +/// heuristic skip table based on the last byte (borrowed from Rust's standard +/// library) and 3) the addition of heuristics for a fast skip loop. That is, +/// (3) this will detect bytes that are believed to be rare in the needle and +/// use fast vectorized instructions to find their occurrences quickly. The +/// Two-Way algorithm is then used to confirm whether a match at that location +/// occurred. +/// +/// The heuristic for fast skipping is automatically shut off if it's +/// detected to be ineffective at search time. Generally, this only occurs in +/// pathological cases. But this is generally necessary in order to preserve +/// a `O(n + m)` time bound. +/// +/// The code below is fairly complex and not obviously correct at all. It's +/// likely necessary to read the Two-Way paper cited above in order to fully +/// grok this code. The essence of it is: +/// +/// 1) Do something to detect a "critical" position in the needle. +/// 2) For the current position in the haystack, look if needle[critical..] +/// matches at that position. +/// 3) If so, look if needle[..critical] matches. +/// 4) If a mismatch occurs, shift the search by some amount based on the +/// critical position and a pre-computed shift. +/// +/// This type is wrapped in Forward and Reverse types that expose consistent +/// forward or reverse APIs. +#[derive(Clone, Copy, Debug)] +struct TwoWay { + /// A small bitset used as a quick prefilter (in addition to the faster + /// SIMD based prefilter). Namely, a bit 'i' is set if and only if b%64==i + /// for any b in the needle. + /// + /// When used as a prefilter, if the last byte at the current candidate + /// position is NOT in this set, then we can skip that entire candidate + /// position (the length of the needle). This is essentially the shift + /// trick found in Boyer-Moore, but only applied to bytes that don't appear + /// in the needle. + /// + /// N.B. This trick was inspired by something similar in std's + /// implementation of Two-Way. + byteset: ApproximateByteSet, + /// A critical position in needle. Specifically, this position corresponds + /// to beginning of either the minimal or maximal suffix in needle. (N.B. + /// See SuffixType below for why "minimal" isn't quite the correct word + /// here.) + /// + /// This is the position at which every search begins. Namely, search + /// starts by scanning text to the right of this position, and only if + /// there's a match does the text to the left of this position get scanned. + critical_pos: usize, + /// The amount we shift by in the Two-Way search algorithm. This + /// corresponds to the "small period" and "large period" cases. + shift: Shift, +} + +impl Forward { + /// Create a searcher that uses the Two-Way algorithm by searching forwards + /// through any haystack. + pub(crate) fn new(needle: &[u8]) -> Forward { + if needle.is_empty() { + return Forward(TwoWay::empty()); + } + + let byteset = ApproximateByteSet::new(needle); + let min_suffix = Suffix::forward(needle, SuffixKind::Minimal); + let max_suffix = Suffix::forward(needle, SuffixKind::Maximal); + let (period_lower_bound, critical_pos) = + if min_suffix.pos > max_suffix.pos { + (min_suffix.period, min_suffix.pos) + } else { + (max_suffix.period, max_suffix.pos) + }; + let shift = Shift::forward(needle, period_lower_bound, critical_pos); + Forward(TwoWay { byteset, critical_pos, shift }) + } + + /// Find the position of the first occurrence of this searcher's needle in + /// the given haystack. If one does not exist, then return None. + /// + /// This accepts prefilter state that is useful when using the same + /// searcher multiple times, such as in an iterator. + /// + /// Callers must guarantee that the needle is non-empty and its length is + /// <= the haystack's length. + #[inline(always)] + pub(crate) fn find( + &self, + pre: Option<&mut Pre<'_>>, + haystack: &[u8], + needle: &[u8], + ) -> Option { + debug_assert!(!needle.is_empty(), "needle should not be empty"); + debug_assert!(needle.len() <= haystack.len(), "haystack too short"); + + match self.0.shift { + Shift::Small { period } => { + self.find_small_imp(pre, haystack, needle, period) + } + Shift::Large { shift } => { + self.find_large_imp(pre, haystack, needle, shift) + } + } + } + + /// Like find, but handles the degenerate substring test cases. This is + /// only useful for conveniently testing this substring implementation in + /// isolation. + #[cfg(test)] + fn find_general( + &self, + pre: Option<&mut Pre<'_>>, + haystack: &[u8], + needle: &[u8], + ) -> Option { + if needle.is_empty() { + Some(0) + } else if haystack.len() < needle.len() { + None + } else { + self.find(pre, haystack, needle) + } + } + + // Each of the two search implementations below can be accelerated by a + // prefilter, but it is not always enabled. To avoid its overhead when + // its disabled, we explicitly inline each search implementation based on + // whether a prefilter will be used or not. The decision on which to use + // is made in the parent meta searcher. + + #[inline(always)] + fn find_small_imp( + &self, + mut pre: Option<&mut Pre<'_>>, + haystack: &[u8], + needle: &[u8], + period: usize, + ) -> Option { + let last_byte = needle.len() - 1; + let mut pos = 0; + let mut shift = 0; + while pos + needle.len() <= haystack.len() { + let mut i = cmp::max(self.0.critical_pos, shift); + if let Some(pre) = pre.as_mut() { + if pre.should_call() { + pos += pre.call(&haystack[pos..], needle)?; + shift = 0; + i = self.0.critical_pos; + if pos + needle.len() > haystack.len() { + return None; + } + } + } + if !self.0.byteset.contains(haystack[pos + last_byte]) { + pos += needle.len(); + shift = 0; + continue; + } + while i < needle.len() && needle[i] == haystack[pos + i] { + i += 1; + } + if i < needle.len() { + pos += i - self.0.critical_pos + 1; + shift = 0; + } else { + let mut j = self.0.critical_pos; + while j > shift && needle[j] == haystack[pos + j] { + j -= 1; + } + if j <= shift && needle[shift] == haystack[pos + shift] { + return Some(pos); + } + pos += period; + shift = needle.len() - period; + } + } + None + } + + #[inline(always)] + fn find_large_imp( + &self, + mut pre: Option<&mut Pre<'_>>, + haystack: &[u8], + needle: &[u8], + shift: usize, + ) -> Option { + let last_byte = needle.len() - 1; + let mut pos = 0; + 'outer: while pos + needle.len() <= haystack.len() { + if let Some(pre) = pre.as_mut() { + if pre.should_call() { + pos += pre.call(&haystack[pos..], needle)?; + if pos + needle.len() > haystack.len() { + return None; + } + } + } + + if !self.0.byteset.contains(haystack[pos + last_byte]) { + pos += needle.len(); + continue; + } + let mut i = self.0.critical_pos; + while i < needle.len() && needle[i] == haystack[pos + i] { + i += 1; + } + if i < needle.len() { + pos += i - self.0.critical_pos + 1; + } else { + for j in (0..self.0.critical_pos).rev() { + if needle[j] != haystack[pos + j] { + pos += shift; + continue 'outer; + } + } + return Some(pos); + } + } + None + } +} + +impl Reverse { + /// Create a searcher that uses the Two-Way algorithm by searching in + /// reverse through any haystack. + pub(crate) fn new(needle: &[u8]) -> Reverse { + if needle.is_empty() { + return Reverse(TwoWay::empty()); + } + + let byteset = ApproximateByteSet::new(needle); + let min_suffix = Suffix::reverse(needle, SuffixKind::Minimal); + let max_suffix = Suffix::reverse(needle, SuffixKind::Maximal); + let (period_lower_bound, critical_pos) = + if min_suffix.pos < max_suffix.pos { + (min_suffix.period, min_suffix.pos) + } else { + (max_suffix.period, max_suffix.pos) + }; + // let critical_pos = needle.len() - critical_pos; + let shift = Shift::reverse(needle, period_lower_bound, critical_pos); + Reverse(TwoWay { byteset, critical_pos, shift }) + } + + /// Find the position of the last occurrence of this searcher's needle + /// in the given haystack. If one does not exist, then return None. + /// + /// This will automatically initialize prefilter state. This should only + /// be used for one-off searches. + /// + /// Callers must guarantee that the needle is non-empty and its length is + /// <= the haystack's length. + #[inline(always)] + pub(crate) fn rfind( + &self, + haystack: &[u8], + needle: &[u8], + ) -> Option { + debug_assert!(!needle.is_empty(), "needle should not be empty"); + debug_assert!(needle.len() <= haystack.len(), "haystack too short"); + // For the reverse case, we don't use a prefilter. It's plausible that + // perhaps we should, but it's a lot of additional code to do it, and + // it's not clear that it's actually worth it. If you have a really + // compelling use case for this, please file an issue. + match self.0.shift { + Shift::Small { period } => { + self.rfind_small_imp(haystack, needle, period) + } + Shift::Large { shift } => { + self.rfind_large_imp(haystack, needle, shift) + } + } + } + + /// Like rfind, but handles the degenerate substring test cases. This is + /// only useful for conveniently testing this substring implementation in + /// isolation. + #[cfg(test)] + fn rfind_general(&self, haystack: &[u8], needle: &[u8]) -> Option { + if needle.is_empty() { + Some(haystack.len()) + } else if haystack.len() < needle.len() { + None + } else { + self.rfind(haystack, needle) + } + } + + #[inline(always)] + fn rfind_small_imp( + &self, + haystack: &[u8], + needle: &[u8], + period: usize, + ) -> Option { + let nlen = needle.len(); + let mut pos = haystack.len(); + let mut shift = nlen; + while pos >= nlen { + if !self.0.byteset.contains(haystack[pos - nlen]) { + pos -= nlen; + shift = nlen; + continue; + } + let mut i = cmp::min(self.0.critical_pos, shift); + while i > 0 && needle[i - 1] == haystack[pos - nlen + i - 1] { + i -= 1; + } + if i > 0 || needle[0] != haystack[pos - nlen] { + pos -= self.0.critical_pos - i + 1; + shift = nlen; + } else { + let mut j = self.0.critical_pos; + while j < shift && needle[j] == haystack[pos - nlen + j] { + j += 1; + } + if j >= shift { + return Some(pos - nlen); + } + pos -= period; + shift = period; + } + } + None + } + + #[inline(always)] + fn rfind_large_imp( + &self, + haystack: &[u8], + needle: &[u8], + shift: usize, + ) -> Option { + let nlen = needle.len(); + let mut pos = haystack.len(); + while pos >= nlen { + if !self.0.byteset.contains(haystack[pos - nlen]) { + pos -= nlen; + continue; + } + let mut i = self.0.critical_pos; + while i > 0 && needle[i - 1] == haystack[pos - nlen + i - 1] { + i -= 1; + } + if i > 0 || needle[0] != haystack[pos - nlen] { + pos -= self.0.critical_pos - i + 1; + } else { + let mut j = self.0.critical_pos; + while j < nlen && needle[j] == haystack[pos - nlen + j] { + j += 1; + } + if j == nlen { + return Some(pos - nlen); + } + pos -= shift; + } + } + None + } +} + +impl TwoWay { + fn empty() -> TwoWay { + TwoWay { + byteset: ApproximateByteSet::new(b""), + critical_pos: 0, + shift: Shift::Large { shift: 0 }, + } + } +} + +/// A representation of the amount we're allowed to shift by during Two-Way +/// search. +/// +/// When computing a critical factorization of the needle, we find the position +/// of the critical factorization by finding the needle's maximal (or minimal) +/// suffix, along with the period of that suffix. It turns out that the period +/// of that suffix is a lower bound on the period of the needle itself. +/// +/// This lower bound is equivalent to the actual period of the needle in +/// some cases. To describe that case, we denote the needle as `x` where +/// `x = uv` and `v` is the lexicographic maximal suffix of `v`. The lower +/// bound given here is always the period of `v`, which is `<= period(x)`. The +/// case where `period(v) == period(x)` occurs when `len(u) < (len(x) / 2)` and +/// where `u` is a suffix of `v[0..period(v)]`. +/// +/// This case is important because the search algorithm for when the +/// periods are equivalent is slightly different than the search algorithm +/// for when the periods are not equivalent. In particular, when they aren't +/// equivalent, we know that the period of the needle is no less than half its +/// length. In this case, we shift by an amount less than or equal to the +/// period of the needle (determined by the maximum length of the components +/// of the critical factorization of `x`, i.e., `max(len(u), len(v))`).. +/// +/// The above two cases are represented by the variants below. Each entails +/// a different instantiation of the Two-Way search algorithm. +/// +/// N.B. If we could find a way to compute the exact period in all cases, +/// then we could collapse this case analysis and simplify the algorithm. The +/// Two-Way paper suggests this is possible, but more reading is required to +/// grok why the authors didn't pursue that path. +#[derive(Clone, Copy, Debug)] +enum Shift { + Small { period: usize }, + Large { shift: usize }, +} + +impl Shift { + /// Compute the shift for a given needle in the forward direction. + /// + /// This requires a lower bound on the period and a critical position. + /// These can be computed by extracting both the minimal and maximal + /// lexicographic suffixes, and choosing the right-most starting position. + /// The lower bound on the period is then the period of the chosen suffix. + fn forward( + needle: &[u8], + period_lower_bound: usize, + critical_pos: usize, + ) -> Shift { + let large = cmp::max(critical_pos, needle.len() - critical_pos); + if critical_pos * 2 >= needle.len() { + return Shift::Large { shift: large }; + } + + let (u, v) = needle.split_at(critical_pos); + if !util::is_suffix(&v[..period_lower_bound], u) { + return Shift::Large { shift: large }; + } + Shift::Small { period: period_lower_bound } + } + + /// Compute the shift for a given needle in the reverse direction. + /// + /// This requires a lower bound on the period and a critical position. + /// These can be computed by extracting both the minimal and maximal + /// lexicographic suffixes, and choosing the left-most starting position. + /// The lower bound on the period is then the period of the chosen suffix. + fn reverse( + needle: &[u8], + period_lower_bound: usize, + critical_pos: usize, + ) -> Shift { + let large = cmp::max(critical_pos, needle.len() - critical_pos); + if (needle.len() - critical_pos) * 2 >= needle.len() { + return Shift::Large { shift: large }; + } + + let (v, u) = needle.split_at(critical_pos); + if !util::is_prefix(&v[v.len() - period_lower_bound..], u) { + return Shift::Large { shift: large }; + } + Shift::Small { period: period_lower_bound } + } +} + +/// A suffix extracted from a needle along with its period. +#[derive(Debug)] +struct Suffix { + /// The starting position of this suffix. + /// + /// If this is a forward suffix, then `&bytes[pos..]` can be used. If this + /// is a reverse suffix, then `&bytes[..pos]` can be used. That is, for + /// forward suffixes, this is an inclusive starting position, where as for + /// reverse suffixes, this is an exclusive ending position. + pos: usize, + /// The period of this suffix. + /// + /// Note that this is NOT necessarily the period of the string from which + /// this suffix comes from. (It is always less than or equal to the period + /// of the original string.) + period: usize, +} + +impl Suffix { + fn forward(needle: &[u8], kind: SuffixKind) -> Suffix { + debug_assert!(!needle.is_empty()); + + // suffix represents our maximal (or minimal) suffix, along with + // its period. + let mut suffix = Suffix { pos: 0, period: 1 }; + // The start of a suffix in `needle` that we are considering as a + // more maximal (or minimal) suffix than what's in `suffix`. + let mut candidate_start = 1; + // The current offset of our suffixes that we're comparing. + // + // When the characters at this offset are the same, then we mush on + // to the next position since no decision is possible. When the + // candidate's character is greater (or lesser) than the corresponding + // character than our current maximal (or minimal) suffix, then the + // current suffix is changed over to the candidate and we restart our + // search. Otherwise, the candidate suffix is no good and we restart + // our search on the next candidate. + // + // The three cases above correspond to the three cases in the loop + // below. + let mut offset = 0; + + while candidate_start + offset < needle.len() { + let current = needle[suffix.pos + offset]; + let candidate = needle[candidate_start + offset]; + match kind.cmp(current, candidate) { + SuffixOrdering::Accept => { + suffix = Suffix { pos: candidate_start, period: 1 }; + candidate_start += 1; + offset = 0; + } + SuffixOrdering::Skip => { + candidate_start += offset + 1; + offset = 0; + suffix.period = candidate_start - suffix.pos; + } + SuffixOrdering::Push => { + if offset + 1 == suffix.period { + candidate_start += suffix.period; + offset = 0; + } else { + offset += 1; + } + } + } + } + suffix + } + + fn reverse(needle: &[u8], kind: SuffixKind) -> Suffix { + debug_assert!(!needle.is_empty()); + + // See the comments in `forward` for how this works. + let mut suffix = Suffix { pos: needle.len(), period: 1 }; + if needle.len() == 1 { + return suffix; + } + let mut candidate_start = needle.len() - 1; + let mut offset = 0; + + while offset < candidate_start { + let current = needle[suffix.pos - offset - 1]; + let candidate = needle[candidate_start - offset - 1]; + match kind.cmp(current, candidate) { + SuffixOrdering::Accept => { + suffix = Suffix { pos: candidate_start, period: 1 }; + candidate_start -= 1; + offset = 0; + } + SuffixOrdering::Skip => { + candidate_start -= offset + 1; + offset = 0; + suffix.period = suffix.pos - candidate_start; + } + SuffixOrdering::Push => { + if offset + 1 == suffix.period { + candidate_start -= suffix.period; + offset = 0; + } else { + offset += 1; + } + } + } + } + suffix + } +} + +/// The kind of suffix to extract. +#[derive(Clone, Copy, Debug)] +enum SuffixKind { + /// Extract the smallest lexicographic suffix from a string. + /// + /// Technically, this doesn't actually pick the smallest lexicographic + /// suffix. e.g., Given the choice between `a` and `aa`, this will choose + /// the latter over the former, even though `a < aa`. The reasoning for + /// this isn't clear from the paper, but it still smells like a minimal + /// suffix. + Minimal, + /// Extract the largest lexicographic suffix from a string. + /// + /// Unlike `Minimal`, this really does pick the maximum suffix. e.g., Given + /// the choice between `z` and `zz`, this will choose the latter over the + /// former. + Maximal, +} + +/// The result of comparing corresponding bytes between two suffixes. +#[derive(Clone, Copy, Debug)] +enum SuffixOrdering { + /// This occurs when the given candidate byte indicates that the candidate + /// suffix is better than the current maximal (or minimal) suffix. That is, + /// the current candidate suffix should supplant the current maximal (or + /// minimal) suffix. + Accept, + /// This occurs when the given candidate byte excludes the candidate suffix + /// from being better than the current maximal (or minimal) suffix. That + /// is, the current candidate suffix should be dropped and the next one + /// should be considered. + Skip, + /// This occurs when no decision to accept or skip the candidate suffix + /// can be made, e.g., when corresponding bytes are equivalent. In this + /// case, the next corresponding bytes should be compared. + Push, +} + +impl SuffixKind { + /// Returns true if and only if the given candidate byte indicates that + /// it should replace the current suffix as the maximal (or minimal) + /// suffix. + fn cmp(self, current: u8, candidate: u8) -> SuffixOrdering { + use self::SuffixOrdering::*; + + match self { + SuffixKind::Minimal if candidate < current => Accept, + SuffixKind::Minimal if candidate > current => Skip, + SuffixKind::Minimal => Push, + SuffixKind::Maximal if candidate > current => Accept, + SuffixKind::Maximal if candidate < current => Skip, + SuffixKind::Maximal => Push, + } + } +} + +/// A bitset used to track whether a particular byte exists in a needle or not. +/// +/// Namely, bit 'i' is set if and only if byte%64==i for any byte in the +/// needle. If a particular byte in the haystack is NOT in this set, then one +/// can conclude that it is also not in the needle, and thus, one can advance +/// in the haystack by needle.len() bytes. +#[derive(Clone, Copy, Debug)] +struct ApproximateByteSet(u64); + +impl ApproximateByteSet { + /// Create a new set from the given needle. + fn new(needle: &[u8]) -> ApproximateByteSet { + let mut bits = 0; + for &b in needle { + bits |= 1 << (b % 64); + } + ApproximateByteSet(bits) + } + + /// Return true if and only if the given byte might be in this set. This + /// may return a false positive, but will never return a false negative. + #[inline(always)] + fn contains(&self, byte: u8) -> bool { + self.0 & (1 << (byte % 64)) != 0 + } +} + +#[cfg(all(test, feature = "std", not(miri)))] +mod tests { + use quickcheck::quickcheck; + + use super::*; + + define_memmem_quickcheck_tests!( + super::simpletests::twoway_find, + super::simpletests::twoway_rfind + ); + + /// Convenience wrapper for computing the suffix as a byte string. + fn get_suffix_forward(needle: &[u8], kind: SuffixKind) -> (&[u8], usize) { + let s = Suffix::forward(needle, kind); + (&needle[s.pos..], s.period) + } + + /// Convenience wrapper for computing the reverse suffix as a byte string. + fn get_suffix_reverse(needle: &[u8], kind: SuffixKind) -> (&[u8], usize) { + let s = Suffix::reverse(needle, kind); + (&needle[..s.pos], s.period) + } + + /// Return all of the non-empty suffixes in the given byte string. + fn suffixes(bytes: &[u8]) -> Vec<&[u8]> { + (0..bytes.len()).map(|i| &bytes[i..]).collect() + } + + /// Return the lexicographically maximal suffix of the given byte string. + fn naive_maximal_suffix_forward(needle: &[u8]) -> &[u8] { + let mut sufs = suffixes(needle); + sufs.sort(); + sufs.pop().unwrap() + } + + /// Return the lexicographically maximal suffix of the reverse of the given + /// byte string. + fn naive_maximal_suffix_reverse(needle: &[u8]) -> Vec { + let mut reversed = needle.to_vec(); + reversed.reverse(); + let mut got = naive_maximal_suffix_forward(&reversed).to_vec(); + got.reverse(); + got + } + + #[test] + fn suffix_forward() { + macro_rules! assert_suffix_min { + ($given:expr, $expected:expr, $period:expr) => { + let (got_suffix, got_period) = + get_suffix_forward($given.as_bytes(), SuffixKind::Minimal); + let got_suffix = std::str::from_utf8(got_suffix).unwrap(); + assert_eq!(($expected, $period), (got_suffix, got_period)); + }; + } + + macro_rules! assert_suffix_max { + ($given:expr, $expected:expr, $period:expr) => { + let (got_suffix, got_period) = + get_suffix_forward($given.as_bytes(), SuffixKind::Maximal); + let got_suffix = std::str::from_utf8(got_suffix).unwrap(); + assert_eq!(($expected, $period), (got_suffix, got_period)); + }; + } + + assert_suffix_min!("a", "a", 1); + assert_suffix_max!("a", "a", 1); + + assert_suffix_min!("ab", "ab", 2); + assert_suffix_max!("ab", "b", 1); + + assert_suffix_min!("ba", "a", 1); + assert_suffix_max!("ba", "ba", 2); + + assert_suffix_min!("abc", "abc", 3); + assert_suffix_max!("abc", "c", 1); + + assert_suffix_min!("acb", "acb", 3); + assert_suffix_max!("acb", "cb", 2); + + assert_suffix_min!("cba", "a", 1); + assert_suffix_max!("cba", "cba", 3); + + assert_suffix_min!("abcabc", "abcabc", 3); + assert_suffix_max!("abcabc", "cabc", 3); + + assert_suffix_min!("abcabcabc", "abcabcabc", 3); + assert_suffix_max!("abcabcabc", "cabcabc", 3); + + assert_suffix_min!("abczz", "abczz", 5); + assert_suffix_max!("abczz", "zz", 1); + + assert_suffix_min!("zzabc", "abc", 3); + assert_suffix_max!("zzabc", "zzabc", 5); + + assert_suffix_min!("aaa", "aaa", 1); + assert_suffix_max!("aaa", "aaa", 1); + + assert_suffix_min!("foobar", "ar", 2); + assert_suffix_max!("foobar", "r", 1); + } + + #[test] + fn suffix_reverse() { + macro_rules! assert_suffix_min { + ($given:expr, $expected:expr, $period:expr) => { + let (got_suffix, got_period) = + get_suffix_reverse($given.as_bytes(), SuffixKind::Minimal); + let got_suffix = std::str::from_utf8(got_suffix).unwrap(); + assert_eq!(($expected, $period), (got_suffix, got_period)); + }; + } + + macro_rules! assert_suffix_max { + ($given:expr, $expected:expr, $period:expr) => { + let (got_suffix, got_period) = + get_suffix_reverse($given.as_bytes(), SuffixKind::Maximal); + let got_suffix = std::str::from_utf8(got_suffix).unwrap(); + assert_eq!(($expected, $period), (got_suffix, got_period)); + }; + } + + assert_suffix_min!("a", "a", 1); + assert_suffix_max!("a", "a", 1); + + assert_suffix_min!("ab", "a", 1); + assert_suffix_max!("ab", "ab", 2); + + assert_suffix_min!("ba", "ba", 2); + assert_suffix_max!("ba", "b", 1); + + assert_suffix_min!("abc", "a", 1); + assert_suffix_max!("abc", "abc", 3); + + assert_suffix_min!("acb", "a", 1); + assert_suffix_max!("acb", "ac", 2); + + assert_suffix_min!("cba", "cba", 3); + assert_suffix_max!("cba", "c", 1); + + assert_suffix_min!("abcabc", "abca", 3); + assert_suffix_max!("abcabc", "abcabc", 3); + + assert_suffix_min!("abcabcabc", "abcabca", 3); + assert_suffix_max!("abcabcabc", "abcabcabc", 3); + + assert_suffix_min!("abczz", "a", 1); + assert_suffix_max!("abczz", "abczz", 5); + + assert_suffix_min!("zzabc", "zza", 3); + assert_suffix_max!("zzabc", "zz", 1); + + assert_suffix_min!("aaa", "aaa", 1); + assert_suffix_max!("aaa", "aaa", 1); + } + + quickcheck! { + fn qc_suffix_forward_maximal(bytes: Vec) -> bool { + if bytes.is_empty() { + return true; + } + + let (got, _) = get_suffix_forward(&bytes, SuffixKind::Maximal); + let expected = naive_maximal_suffix_forward(&bytes); + got == expected + } + + fn qc_suffix_reverse_maximal(bytes: Vec) -> bool { + if bytes.is_empty() { + return true; + } + + let (got, _) = get_suffix_reverse(&bytes, SuffixKind::Maximal); + let expected = naive_maximal_suffix_reverse(&bytes); + expected == got + } + } +} + +#[cfg(test)] +mod simpletests { + use super::*; + + pub(crate) fn twoway_find( + haystack: &[u8], + needle: &[u8], + ) -> Option { + Forward::new(needle).find_general(None, haystack, needle) + } + + pub(crate) fn twoway_rfind( + haystack: &[u8], + needle: &[u8], + ) -> Option { + Reverse::new(needle).rfind_general(haystack, needle) + } + + define_memmem_simple_tests!(twoway_find, twoway_rfind); + + // This is a regression test caught by quickcheck that exercised a bug in + // the reverse small period handling. The bug was that we were using 'if j + // == shift' to determine if a match occurred, but the correct guard is 'if + // j >= shift', which matches the corresponding guard in the forward impl. + #[test] + fn regression_rev_small_period() { + let rfind = super::simpletests::twoway_rfind; + let haystack = "ababaz"; + let needle = "abab"; + assert_eq!(Some(0), rfind(haystack.as_bytes(), needle.as_bytes())); + } +} diff --git a/crux-mir/lib/memchr/src/memmem/util.rs b/crux-mir/lib/memchr/src/memmem/util.rs new file mode 100644 index 000000000..de0e385e1 --- /dev/null +++ b/crux-mir/lib/memchr/src/memmem/util.rs @@ -0,0 +1,88 @@ +// These routines are meant to be optimized specifically for low latency as +// compared to the equivalent routines offered by std. (Which may invoke the +// dynamic linker and call out to libc, which introduces a bit more latency +// than we'd like.) + +/// Returns true if and only if needle is a prefix of haystack. +#[inline(always)] +pub(crate) fn is_prefix(haystack: &[u8], needle: &[u8]) -> bool { + needle.len() <= haystack.len() && memcmp(&haystack[..needle.len()], needle) +} + +/// Returns true if and only if needle is a suffix of haystack. +#[inline(always)] +pub(crate) fn is_suffix(haystack: &[u8], needle: &[u8]) -> bool { + needle.len() <= haystack.len() + && memcmp(&haystack[haystack.len() - needle.len()..], needle) +} + +/// Return true if and only if x.len() == y.len() && x[i] == y[i] for all +/// 0 <= i < x.len(). +/// +/// Why not just use actual memcmp for this? Well, memcmp requires calling out +/// to libc, and this routine is called in fairly hot code paths. Other than +/// just calling out to libc, it also seems to result in worse codegen. By +/// rolling our own memcmp in pure Rust, it seems to appear more friendly to +/// the optimizer. +/// +/// We mark this as inline always, although, some callers may not want it +/// inlined for better codegen (like Rabin-Karp). In that case, callers are +/// advised to create a non-inlineable wrapper routine that calls memcmp. +#[inline(always)] +pub(crate) fn memcmp(x: &[u8], y: &[u8]) -> bool { + if x.len() != y.len() { + return false; + } + // If we don't have enough bytes to do 4-byte at a time loads, then + // fall back to the naive slow version. + // + // TODO: We could do a copy_nonoverlapping combined with a mask instead + // of a loop. Benchmark it. + if x.len() < 4 { + for (&b1, &b2) in x.iter().zip(y) { + if b1 != b2 { + return false; + } + } + return true; + } + // When we have 4 or more bytes to compare, then proceed in chunks of 4 at + // a time using unaligned loads. + // + // Also, why do 4 byte loads instead of, say, 8 byte loads? The reason is + // that this particular version of memcmp is likely to be called with tiny + // needles. That means that if we do 8 byte loads, then a higher proportion + // of memcmp calls will use the slower variant above. With that said, this + // is a hypothesis and is only loosely supported by benchmarks. There's + // likely some improvement that could be made here. The main thing here + // though is to optimize for latency, not throughput. + + // SAFETY: Via the conditional above, we know that both `px` and `py` + // have the same length, so `px < pxend` implies that `py < pyend`. + // Thus, derefencing both `px` and `py` in the loop below is safe. + // + // Moreover, we set `pxend` and `pyend` to be 4 bytes before the actual + // end of of `px` and `py`. Thus, the final dereference outside of the + // loop is guaranteed to be valid. (The final comparison will overlap with + // the last comparison done in the loop for lengths that aren't multiples + // of four.) + // + // Finally, we needn't worry about alignment here, since we do unaligned + // loads. + unsafe { + let (mut px, mut py) = (x.as_ptr(), y.as_ptr()); + let (pxend, pyend) = (px.add(x.len() - 4), py.add(y.len() - 4)); + while px < pxend { + let vx = (px as *const u32).read_unaligned(); + let vy = (py as *const u32).read_unaligned(); + if vx != vy { + return false; + } + px = px.add(4); + py = py.add(4); + } + let vx = (pxend as *const u32).read_unaligned(); + let vy = (pyend as *const u32).read_unaligned(); + vx == vy + } +} diff --git a/crux-mir/lib/memchr/src/memmem/vector.rs b/crux-mir/lib/memchr/src/memmem/vector.rs new file mode 100644 index 000000000..b81165f8b --- /dev/null +++ b/crux-mir/lib/memchr/src/memmem/vector.rs @@ -0,0 +1,131 @@ +/// A trait for describing vector operations used by vectorized searchers. +/// +/// The trait is highly constrained to low level vector operations needed. In +/// general, it was invented mostly to be generic over x86's __m128i and +/// __m256i types. It's likely that once std::simd becomes a thing, we can +/// migrate to that since the operations required are quite simple. +/// +/// TODO: Consider moving this trait up a level and using it to implement +/// memchr as well. The trait might need to grow one or two methods, but +/// otherwise should be close to sufficient already. +/// +/// # Safety +/// +/// All methods are not safe since they are intended to be implemented using +/// vendor intrinsics, which are also not safe. Callers must ensure that the +/// appropriate target features are enabled in the calling function, and that +/// the current CPU supports them. All implementations should avoid marking the +/// routines with #[target_feature] and instead mark them as #[inline(always)] +/// to ensure they get appropriately inlined. (inline(always) cannot be used +/// with target_feature.) +pub(crate) trait Vector: Copy + core::fmt::Debug { + /// _mm_set1_epi8 or _mm256_set1_epi8 + unsafe fn splat(byte: u8) -> Self; + /// _mm_loadu_si128 or _mm256_loadu_si256 + unsafe fn load_unaligned(data: *const u8) -> Self; + /// _mm_movemask_epi8 or _mm256_movemask_epi8 + unsafe fn movemask(self) -> u32; + /// _mm_cmpeq_epi8 or _mm256_cmpeq_epi8 + unsafe fn cmpeq(self, vector2: Self) -> Self; + /// _mm_and_si128 or _mm256_and_si256 + unsafe fn and(self, vector2: Self) -> Self; +} + +#[cfg(target_arch = "x86_64")] +mod x86sse { + use super::Vector; + use core::arch::x86_64::*; + + impl Vector for __m128i { + #[inline(always)] + unsafe fn splat(byte: u8) -> __m128i { + _mm_set1_epi8(byte as i8) + } + + #[inline(always)] + unsafe fn load_unaligned(data: *const u8) -> __m128i { + _mm_loadu_si128(data as *const __m128i) + } + + #[inline(always)] + unsafe fn movemask(self) -> u32 { + _mm_movemask_epi8(self) as u32 + } + + #[inline(always)] + unsafe fn cmpeq(self, vector2: Self) -> __m128i { + _mm_cmpeq_epi8(self, vector2) + } + + #[inline(always)] + unsafe fn and(self, vector2: Self) -> __m128i { + _mm_and_si128(self, vector2) + } + } +} + +#[cfg(all(feature = "std", target_arch = "x86_64"))] +mod x86avx { + use super::Vector; + use core::arch::x86_64::*; + + impl Vector for __m256i { + #[inline(always)] + unsafe fn splat(byte: u8) -> __m256i { + _mm256_set1_epi8(byte as i8) + } + + #[inline(always)] + unsafe fn load_unaligned(data: *const u8) -> __m256i { + _mm256_loadu_si256(data as *const __m256i) + } + + #[inline(always)] + unsafe fn movemask(self) -> u32 { + _mm256_movemask_epi8(self) as u32 + } + + #[inline(always)] + unsafe fn cmpeq(self, vector2: Self) -> __m256i { + _mm256_cmpeq_epi8(self, vector2) + } + + #[inline(always)] + unsafe fn and(self, vector2: Self) -> __m256i { + _mm256_and_si256(self, vector2) + } + } +} + +#[cfg(target_arch = "wasm32")] +mod wasm_simd128 { + use super::Vector; + use core::arch::wasm32::*; + + impl Vector for v128 { + #[inline(always)] + unsafe fn splat(byte: u8) -> v128 { + u8x16_splat(byte) + } + + #[inline(always)] + unsafe fn load_unaligned(data: *const u8) -> v128 { + v128_load(data.cast()) + } + + #[inline(always)] + unsafe fn movemask(self) -> u32 { + u8x16_bitmask(self).into() + } + + #[inline(always)] + unsafe fn cmpeq(self, vector2: Self) -> v128 { + u8x16_eq(self, vector2) + } + + #[inline(always)] + unsafe fn and(self, vector2: Self) -> v128 { + v128_and(self, vector2) + } + } +} diff --git a/crux-mir/lib/memchr/src/memmem/wasm.rs b/crux-mir/lib/memchr/src/memmem/wasm.rs new file mode 100644 index 000000000..4e3ea985c --- /dev/null +++ b/crux-mir/lib/memchr/src/memmem/wasm.rs @@ -0,0 +1,75 @@ +use core::arch::wasm32::v128; + +use crate::memmem::{genericsimd, NeedleInfo}; + +/// A `v128` accelerated vectorized substring search routine that only works on +/// small needles. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Forward(genericsimd::Forward); + +impl Forward { + /// Create a new "generic simd" forward searcher. If one could not be + /// created from the given inputs, then None is returned. + pub(crate) fn new(ninfo: &NeedleInfo, needle: &[u8]) -> Option { + if !cfg!(memchr_runtime_simd) { + return None; + } + genericsimd::Forward::new(ninfo, needle).map(Forward) + } + + /// Returns the minimum length of haystack that is needed for this searcher + /// to work. Passing a haystack with a length smaller than this will cause + /// `find` to panic. + #[inline(always)] + pub(crate) fn min_haystack_len(&self) -> usize { + self.0.min_haystack_len::() + } + + #[inline(always)] + pub(crate) fn find( + &self, + haystack: &[u8], + needle: &[u8], + ) -> Option { + self.find_impl(haystack, needle) + } + + /// The implementation of find marked with the appropriate target feature. + #[target_feature(enable = "simd128")] + fn find_impl(&self, haystack: &[u8], needle: &[u8]) -> Option { + unsafe { genericsimd::fwd_find::(&self.0, haystack, needle) } + } +} + +#[cfg(all(test, feature = "std", not(miri)))] +mod tests { + use crate::memmem::{prefilter::PrefilterState, NeedleInfo}; + + fn find( + _: &mut PrefilterState, + ninfo: &NeedleInfo, + haystack: &[u8], + needle: &[u8], + ) -> Option { + super::Forward::new(ninfo, needle).unwrap().find(haystack, needle) + } + + #[test] + fn prefilter_permutations() { + use crate::memmem::prefilter::tests::PrefilterTest; + + unsafe { + PrefilterTest::run_all_tests_filter(find, |t| { + // This substring searcher only works on certain configs, so + // filter our tests such that Forward::new will be guaranteed + // to succeed. (And also remove tests with a haystack that is + // too small.) + let fwd = match super::Forward::new(&t.ninfo, &t.needle) { + None => return false, + Some(fwd) => fwd, + }; + t.haystack.len() >= fwd.min_haystack_len() + }) + } + } +} diff --git a/crux-mir/lib/memchr/src/memmem/x86/avx.rs b/crux-mir/lib/memchr/src/memmem/x86/avx.rs new file mode 100644 index 000000000..ce168dd37 --- /dev/null +++ b/crux-mir/lib/memchr/src/memmem/x86/avx.rs @@ -0,0 +1,139 @@ +#[cfg(not(feature = "std"))] +pub(crate) use self::nostd::Forward; +#[cfg(feature = "std")] +pub(crate) use self::std::Forward; + +#[cfg(feature = "std")] +mod std { + use core::arch::x86_64::{__m128i, __m256i}; + + use crate::memmem::{genericsimd, NeedleInfo}; + + /// An AVX accelerated vectorized substring search routine that only works + /// on small needles. + #[derive(Clone, Copy, Debug)] + pub(crate) struct Forward(genericsimd::Forward); + + impl Forward { + /// Create a new "generic simd" forward searcher. If one could not be + /// created from the given inputs, then None is returned. + pub(crate) fn new( + ninfo: &NeedleInfo, + needle: &[u8], + ) -> Option { + if !cfg!(memchr_runtime_avx) || !is_x86_feature_detected!("avx2") { + return None; + } + genericsimd::Forward::new(ninfo, needle).map(Forward) + } + + /// Returns the minimum length of haystack that is needed for this + /// searcher to work. Passing a haystack with a length smaller than + /// this will cause `find` to panic. + #[inline(always)] + pub(crate) fn min_haystack_len(&self) -> usize { + self.0.min_haystack_len::<__m128i>() + } + + #[inline(always)] + pub(crate) fn find( + &self, + haystack: &[u8], + needle: &[u8], + ) -> Option { + // SAFETY: The only way a Forward value can exist is if the avx2 + // target feature is enabled. This is the only safety requirement + // for calling the genericsimd searcher. + unsafe { self.find_impl(haystack, needle) } + } + + /// The implementation of find marked with the appropriate target + /// feature. + /// + /// # Safety + /// + /// Callers must ensure that the avx2 CPU feature is enabled in the + /// current environment. + #[target_feature(enable = "avx2")] + unsafe fn find_impl( + &self, + haystack: &[u8], + needle: &[u8], + ) -> Option { + if haystack.len() < self.0.min_haystack_len::<__m256i>() { + genericsimd::fwd_find::<__m128i>(&self.0, haystack, needle) + } else { + genericsimd::fwd_find::<__m256i>(&self.0, haystack, needle) + } + } + } +} + +// We still define the avx "forward" type on nostd to make caller code a bit +// simpler. This avoids needing a lot more conditional compilation. +#[cfg(not(feature = "std"))] +mod nostd { + use crate::memmem::NeedleInfo; + + #[derive(Clone, Copy, Debug)] + pub(crate) struct Forward(()); + + impl Forward { + pub(crate) fn new( + ninfo: &NeedleInfo, + needle: &[u8], + ) -> Option { + None + } + + pub(crate) fn min_haystack_len(&self) -> usize { + unreachable!() + } + + pub(crate) fn find( + &self, + haystack: &[u8], + needle: &[u8], + ) -> Option { + unreachable!() + } + } +} + +#[cfg(all(test, feature = "std", not(miri)))] +mod tests { + use crate::memmem::{prefilter::PrefilterState, NeedleInfo}; + + fn find( + _: &mut PrefilterState, + ninfo: &NeedleInfo, + haystack: &[u8], + needle: &[u8], + ) -> Option { + super::Forward::new(ninfo, needle).unwrap().find(haystack, needle) + } + + #[test] + fn prefilter_permutations() { + use crate::memmem::prefilter::tests::PrefilterTest; + + if !is_x86_feature_detected!("avx2") { + return; + } + // SAFETY: The safety of find only requires that the current CPU + // support AVX2, which we checked above. + unsafe { + PrefilterTest::run_all_tests_filter(find, |t| { + // This substring searcher only works on certain configs, so + // filter our tests such that Forward::new will be guaranteed + // to succeed. (And also remove tests with a haystack that is + // too small.) + let fwd = match super::Forward::new(&t.ninfo, &t.needle) { + None => return false, + Some(fwd) => fwd, + }; + t.haystack.len() >= fwd.min_haystack_len() + }) + } + } +} diff --git a/crux-mir/lib/memchr/src/memmem/x86/mod.rs b/crux-mir/lib/memchr/src/memmem/x86/mod.rs new file mode 100644 index 000000000..c1cc73fee --- /dev/null +++ b/crux-mir/lib/memchr/src/memmem/x86/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod avx; +pub(crate) mod sse; diff --git a/crux-mir/lib/memchr/src/memmem/x86/sse.rs b/crux-mir/lib/memchr/src/memmem/x86/sse.rs new file mode 100644 index 000000000..22e7d9933 --- /dev/null +++ b/crux-mir/lib/memchr/src/memmem/x86/sse.rs @@ -0,0 +1,89 @@ +use core::arch::x86_64::__m128i; + +use crate::memmem::{genericsimd, NeedleInfo}; + +/// An SSE accelerated vectorized substring search routine that only works on +/// small needles. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Forward(genericsimd::Forward); + +impl Forward { + /// Create a new "generic simd" forward searcher. If one could not be + /// created from the given inputs, then None is returned. + pub(crate) fn new(ninfo: &NeedleInfo, needle: &[u8]) -> Option { + if !cfg!(memchr_runtime_sse2) { + return None; + } + genericsimd::Forward::new(ninfo, needle).map(Forward) + } + + /// Returns the minimum length of haystack that is needed for this searcher + /// to work. Passing a haystack with a length smaller than this will cause + /// `find` to panic. + #[inline(always)] + pub(crate) fn min_haystack_len(&self) -> usize { + self.0.min_haystack_len::<__m128i>() + } + + #[inline(always)] + pub(crate) fn find( + &self, + haystack: &[u8], + needle: &[u8], + ) -> Option { + // SAFETY: sse2 is enabled on all x86_64 targets, so this is always + // safe to call. + unsafe { self.find_impl(haystack, needle) } + } + + /// The implementation of find marked with the appropriate target feature. + /// + /// # Safety + /// + /// This is safe to call in all cases since sse2 is guaranteed to be part + /// of x86_64. It is marked as unsafe because of the target feature + /// attribute. + #[target_feature(enable = "sse2")] + unsafe fn find_impl( + &self, + haystack: &[u8], + needle: &[u8], + ) -> Option { + genericsimd::fwd_find::<__m128i>(&self.0, haystack, needle) + } +} + +#[cfg(all(test, feature = "std", not(miri)))] +mod tests { + use crate::memmem::{prefilter::PrefilterState, NeedleInfo}; + + fn find( + _: &mut PrefilterState, + ninfo: &NeedleInfo, + haystack: &[u8], + needle: &[u8], + ) -> Option { + super::Forward::new(ninfo, needle).unwrap().find(haystack, needle) + } + + #[test] + fn prefilter_permutations() { + use crate::memmem::prefilter::tests::PrefilterTest; + + // SAFETY: sse2 is enabled on all x86_64 targets, so this is always + // safe to call. + unsafe { + PrefilterTest::run_all_tests_filter(find, |t| { + // This substring searcher only works on certain configs, so + // filter our tests such that Forward::new will be guaranteed + // to succeed. (And also remove tests with a haystack that is + // too small.) + let fwd = match super::Forward::new(&t.ninfo, &t.needle) { + None => return false, + Some(fwd) => fwd, + }; + t.haystack.len() >= fwd.min_haystack_len() + }) + } + } +} diff --git a/crux-mir/lib/memchr/src/tests/memchr/iter.rs b/crux-mir/lib/memchr/src/tests/memchr/iter.rs new file mode 100644 index 000000000..80ea5c279 --- /dev/null +++ b/crux-mir/lib/memchr/src/tests/memchr/iter.rs @@ -0,0 +1,230 @@ +use quickcheck::quickcheck; + +use crate::{tests::memchr::testdata::memchr_tests, Memchr, Memchr2, Memchr3}; + +#[test] +fn memchr1_iter() { + for test in memchr_tests() { + test.iter_one(false, Memchr::new); + } +} + +#[test] +fn memchr2_iter() { + for test in memchr_tests() { + test.iter_two(false, Memchr2::new); + } +} + +#[test] +fn memchr3_iter() { + for test in memchr_tests() { + test.iter_three(false, Memchr3::new); + } +} + +#[test] +fn memrchr1_iter() { + for test in memchr_tests() { + test.iter_one(true, |n1, corpus| Memchr::new(n1, corpus).rev()); + } +} + +#[test] +fn memrchr2_iter() { + for test in memchr_tests() { + test.iter_two(true, |n1, n2, corpus| { + Memchr2::new(n1, n2, corpus).rev() + }) + } +} + +#[test] +fn memrchr3_iter() { + for test in memchr_tests() { + test.iter_three(true, |n1, n2, n3, corpus| { + Memchr3::new(n1, n2, n3, corpus).rev() + }) + } +} + +quickcheck! { + fn qc_memchr_double_ended_iter( + needle: u8, data: Vec, take_side: Vec + ) -> bool { + // make nonempty + let mut take_side = take_side; + if take_side.is_empty() { take_side.push(true) }; + + let iter = Memchr::new(needle, &data); + let all_found = double_ended_take( + iter, take_side.iter().cycle().cloned()); + + all_found.iter().cloned().eq(positions1(needle, &data)) + } + + fn qc_memchr2_double_ended_iter( + needle1: u8, needle2: u8, data: Vec, take_side: Vec + ) -> bool { + // make nonempty + let mut take_side = take_side; + if take_side.is_empty() { take_side.push(true) }; + + let iter = Memchr2::new(needle1, needle2, &data); + let all_found = double_ended_take( + iter, take_side.iter().cycle().cloned()); + + all_found.iter().cloned().eq(positions2(needle1, needle2, &data)) + } + + fn qc_memchr3_double_ended_iter( + needle1: u8, needle2: u8, needle3: u8, + data: Vec, take_side: Vec + ) -> bool { + // make nonempty + let mut take_side = take_side; + if take_side.is_empty() { take_side.push(true) }; + + let iter = Memchr3::new(needle1, needle2, needle3, &data); + let all_found = double_ended_take( + iter, take_side.iter().cycle().cloned()); + + all_found + .iter() + .cloned() + .eq(positions3(needle1, needle2, needle3, &data)) + } + + fn qc_memchr1_iter(data: Vec) -> bool { + let needle = 0; + let answer = positions1(needle, &data); + answer.eq(Memchr::new(needle, &data)) + } + + fn qc_memchr1_rev_iter(data: Vec) -> bool { + let needle = 0; + let answer = positions1(needle, &data); + answer.rev().eq(Memchr::new(needle, &data).rev()) + } + + fn qc_memchr2_iter(data: Vec) -> bool { + let needle1 = 0; + let needle2 = 1; + let answer = positions2(needle1, needle2, &data); + answer.eq(Memchr2::new(needle1, needle2, &data)) + } + + fn qc_memchr2_rev_iter(data: Vec) -> bool { + let needle1 = 0; + let needle2 = 1; + let answer = positions2(needle1, needle2, &data); + answer.rev().eq(Memchr2::new(needle1, needle2, &data).rev()) + } + + fn qc_memchr3_iter(data: Vec) -> bool { + let needle1 = 0; + let needle2 = 1; + let needle3 = 2; + let answer = positions3(needle1, needle2, needle3, &data); + answer.eq(Memchr3::new(needle1, needle2, needle3, &data)) + } + + fn qc_memchr3_rev_iter(data: Vec) -> bool { + let needle1 = 0; + let needle2 = 1; + let needle3 = 2; + let answer = positions3(needle1, needle2, needle3, &data); + answer.rev().eq(Memchr3::new(needle1, needle2, needle3, &data).rev()) + } + + fn qc_memchr1_iter_size_hint(data: Vec) -> bool { + // test that the size hint is within reasonable bounds + let needle = 0; + let mut iter = Memchr::new(needle, &data); + let mut real_count = data + .iter() + .filter(|&&elt| elt == needle) + .count(); + + while let Some(index) = iter.next() { + real_count -= 1; + let (lower, upper) = iter.size_hint(); + assert!(lower <= real_count); + assert!(upper.unwrap() >= real_count); + assert!(upper.unwrap() <= data.len() - index); + } + true + } +} + +// take items from a DEI, taking front for each true and back for each false. +// Return a vector with the concatenation of the fronts and the reverse of the +// backs. +fn double_ended_take(mut iter: I, take_side: J) -> Vec +where + I: DoubleEndedIterator, + J: Iterator, +{ + let mut found_front = Vec::new(); + let mut found_back = Vec::new(); + + for take_front in take_side { + if take_front { + if let Some(pos) = iter.next() { + found_front.push(pos); + } else { + break; + } + } else { + if let Some(pos) = iter.next_back() { + found_back.push(pos); + } else { + break; + } + }; + } + + let mut all_found = found_front; + all_found.extend(found_back.into_iter().rev()); + all_found +} + +// return an iterator of the 0-based indices of haystack that match the needle +fn positions1<'a>( + n1: u8, + haystack: &'a [u8], +) -> Box + 'a> { + let it = haystack + .iter() + .enumerate() + .filter(move |&(_, &b)| b == n1) + .map(|t| t.0); + Box::new(it) +} + +fn positions2<'a>( + n1: u8, + n2: u8, + haystack: &'a [u8], +) -> Box + 'a> { + let it = haystack + .iter() + .enumerate() + .filter(move |&(_, &b)| b == n1 || b == n2) + .map(|t| t.0); + Box::new(it) +} + +fn positions3<'a>( + n1: u8, + n2: u8, + n3: u8, + haystack: &'a [u8], +) -> Box + 'a> { + let it = haystack + .iter() + .enumerate() + .filter(move |&(_, &b)| b == n1 || b == n2 || b == n3) + .map(|t| t.0); + Box::new(it) +} diff --git a/crux-mir/lib/memchr/src/tests/memchr/memchr.rs b/crux-mir/lib/memchr/src/tests/memchr/memchr.rs new file mode 100644 index 000000000..ac955ed68 --- /dev/null +++ b/crux-mir/lib/memchr/src/tests/memchr/memchr.rs @@ -0,0 +1,134 @@ +use quickcheck::quickcheck; + +use crate::{ + memchr, + memchr::{fallback, naive}, + memchr2, memchr3, memrchr, memrchr2, memrchr3, + tests::memchr::testdata::memchr_tests, +}; + +#[test] +fn memchr1_find() { + for test in memchr_tests() { + test.one(false, memchr); + } +} + +#[test] +fn memchr1_fallback_find() { + for test in memchr_tests() { + test.one(false, fallback::memchr); + } +} + +#[test] +fn memchr2_find() { + for test in memchr_tests() { + test.two(false, memchr2); + } +} + +#[test] +fn memchr2_fallback_find() { + for test in memchr_tests() { + test.two(false, fallback::memchr2); + } +} + +#[test] +fn memchr3_find() { + for test in memchr_tests() { + test.three(false, memchr3); + } +} + +#[test] +fn memchr3_fallback_find() { + for test in memchr_tests() { + test.three(false, fallback::memchr3); + } +} + +#[test] +fn memrchr1_find() { + for test in memchr_tests() { + test.one(true, memrchr); + } +} + +#[test] +fn memrchr1_fallback_find() { + for test in memchr_tests() { + test.one(true, fallback::memrchr); + } +} + +#[test] +fn memrchr2_find() { + for test in memchr_tests() { + test.two(true, memrchr2); + } +} + +#[test] +fn memrchr2_fallback_find() { + for test in memchr_tests() { + test.two(true, fallback::memrchr2); + } +} + +#[test] +fn memrchr3_find() { + for test in memchr_tests() { + test.three(true, memrchr3); + } +} + +#[test] +fn memrchr3_fallback_find() { + for test in memchr_tests() { + test.three(true, fallback::memrchr3); + } +} + +quickcheck! { + fn qc_memchr1_matches_naive(n1: u8, corpus: Vec) -> bool { + memchr(n1, &corpus) == naive::memchr(n1, &corpus) + } +} + +quickcheck! { + fn qc_memchr2_matches_naive(n1: u8, n2: u8, corpus: Vec) -> bool { + memchr2(n1, n2, &corpus) == naive::memchr2(n1, n2, &corpus) + } +} + +quickcheck! { + fn qc_memchr3_matches_naive( + n1: u8, n2: u8, n3: u8, + corpus: Vec + ) -> bool { + memchr3(n1, n2, n3, &corpus) == naive::memchr3(n1, n2, n3, &corpus) + } +} + +quickcheck! { + fn qc_memrchr1_matches_naive(n1: u8, corpus: Vec) -> bool { + memrchr(n1, &corpus) == naive::memrchr(n1, &corpus) + } +} + +quickcheck! { + fn qc_memrchr2_matches_naive(n1: u8, n2: u8, corpus: Vec) -> bool { + memrchr2(n1, n2, &corpus) == naive::memrchr2(n1, n2, &corpus) + } +} + +quickcheck! { + fn qc_memrchr3_matches_naive( + n1: u8, n2: u8, n3: u8, + corpus: Vec + ) -> bool { + memrchr3(n1, n2, n3, &corpus) == naive::memrchr3(n1, n2, n3, &corpus) + } +} diff --git a/crux-mir/lib/memchr/src/tests/memchr/mod.rs b/crux-mir/lib/memchr/src/tests/memchr/mod.rs new file mode 100644 index 000000000..79f94ab56 --- /dev/null +++ b/crux-mir/lib/memchr/src/tests/memchr/mod.rs @@ -0,0 +1,7 @@ +#[cfg(all(feature = "std", not(miri)))] +mod iter; +#[cfg(all(feature = "std", not(miri)))] +mod memchr; +mod simple; +#[cfg(all(feature = "std", not(miri)))] +mod testdata; diff --git a/crux-mir/lib/memchr/src/tests/memchr/simple.rs b/crux-mir/lib/memchr/src/tests/memchr/simple.rs new file mode 100644 index 000000000..bed5b4863 --- /dev/null +++ b/crux-mir/lib/memchr/src/tests/memchr/simple.rs @@ -0,0 +1,23 @@ +// Simple tests using MIRI. These are intended only to be a simple exercise of +// memchr when tests are run under miri. These are mostly necessary because the +// other tests are far more extensive and take too long to run under miri. +// +// These tests are also run when the 'std' feature is not enabled. + +use crate::{memchr, memchr2, memchr3, memrchr, memrchr2, memrchr3}; + +#[test] +fn simple() { + assert_eq!(memchr(b'a', b"abcda"), Some(0)); + assert_eq!(memchr(b'z', b"abcda"), None); + assert_eq!(memchr2(b'a', b'z', b"abcda"), Some(0)); + assert_eq!(memchr2(b'z', b'y', b"abcda"), None); + assert_eq!(memchr3(b'a', b'z', b'b', b"abcda"), Some(0)); + assert_eq!(memchr3(b'z', b'y', b'x', b"abcda"), None); + assert_eq!(memrchr(b'a', b"abcda"), Some(4)); + assert_eq!(memrchr(b'z', b"abcda"), None); + assert_eq!(memrchr2(b'a', b'z', b"abcda"), Some(4)); + assert_eq!(memrchr2(b'z', b'y', b"abcda"), None); + assert_eq!(memrchr3(b'a', b'z', b'b', b"abcda"), Some(4)); + assert_eq!(memrchr3(b'z', b'y', b'x', b"abcda"), None); +} diff --git a/crux-mir/lib/memchr/src/tests/memchr/testdata.rs b/crux-mir/lib/memchr/src/tests/memchr/testdata.rs new file mode 100644 index 000000000..6dda5246f --- /dev/null +++ b/crux-mir/lib/memchr/src/tests/memchr/testdata.rs @@ -0,0 +1,351 @@ +use std::iter::repeat; + +/// Create a sequence of tests that should be run by memchr implementations. +pub fn memchr_tests() -> Vec { + let mut tests = Vec::new(); + for statict in MEMCHR_TESTS { + assert!(!statict.corpus.contains("%"), "% is not allowed in corpora"); + assert!(!statict.corpus.contains("#"), "# is not allowed in corpora"); + assert!(!statict.needles.contains(&b'%'), "% is an invalid needle"); + assert!(!statict.needles.contains(&b'#'), "# is an invalid needle"); + + let t = MemchrTest { + corpus: statict.corpus.to_string(), + needles: statict.needles.to_vec(), + positions: statict.positions.to_vec(), + }; + tests.push(t.clone()); + tests.extend(t.expand()); + } + tests +} + +/// A set of tests for memchr-like functions. +/// +/// These tests mostly try to cover the short string cases. We cover the longer +/// string cases via the benchmarks (which are tests themselves), via +/// quickcheck tests and via automatic expansion of each test case (by +/// increasing the corpus size). Finally, we cover different alignment cases +/// in the tests by varying the starting point of the slice. +const MEMCHR_TESTS: &[MemchrTestStatic] = &[ + // one needle (applied to memchr + memchr2 + memchr3) + MemchrTestStatic { corpus: "a", needles: &[b'a'], positions: &[0] }, + MemchrTestStatic { corpus: "aa", needles: &[b'a'], positions: &[0, 1] }, + MemchrTestStatic { + corpus: "aaa", + needles: &[b'a'], + positions: &[0, 1, 2], + }, + MemchrTestStatic { corpus: "", needles: &[b'a'], positions: &[] }, + MemchrTestStatic { corpus: "z", needles: &[b'a'], positions: &[] }, + MemchrTestStatic { corpus: "zz", needles: &[b'a'], positions: &[] }, + MemchrTestStatic { corpus: "zza", needles: &[b'a'], positions: &[2] }, + MemchrTestStatic { corpus: "zaza", needles: &[b'a'], positions: &[1, 3] }, + MemchrTestStatic { corpus: "zzza", needles: &[b'a'], positions: &[3] }, + MemchrTestStatic { corpus: "\x00a", needles: &[b'a'], positions: &[1] }, + MemchrTestStatic { corpus: "\x00", needles: &[b'\x00'], positions: &[0] }, + MemchrTestStatic { + corpus: "\x00\x00", + needles: &[b'\x00'], + positions: &[0, 1], + }, + MemchrTestStatic { + corpus: "\x00a\x00", + needles: &[b'\x00'], + positions: &[0, 2], + }, + MemchrTestStatic { + corpus: "zzzzzzzzzzzzzzzza", + needles: &[b'a'], + positions: &[16], + }, + MemchrTestStatic { + corpus: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzza", + needles: &[b'a'], + positions: &[32], + }, + // two needles (applied to memchr2 + memchr3) + MemchrTestStatic { + corpus: "az", + needles: &[b'a', b'z'], + positions: &[0, 1], + }, + MemchrTestStatic { + corpus: "az", + needles: &[b'a', b'z'], + positions: &[0, 1], + }, + MemchrTestStatic { corpus: "az", needles: &[b'x', b'y'], positions: &[] }, + MemchrTestStatic { corpus: "az", needles: &[b'a', b'y'], positions: &[0] }, + MemchrTestStatic { corpus: "az", needles: &[b'x', b'z'], positions: &[1] }, + MemchrTestStatic { + corpus: "yyyyaz", + needles: &[b'a', b'z'], + positions: &[4, 5], + }, + MemchrTestStatic { + corpus: "yyyyaz", + needles: &[b'z', b'a'], + positions: &[4, 5], + }, + // three needles (applied to memchr3) + MemchrTestStatic { + corpus: "xyz", + needles: &[b'x', b'y', b'z'], + positions: &[0, 1, 2], + }, + MemchrTestStatic { + corpus: "zxy", + needles: &[b'x', b'y', b'z'], + positions: &[0, 1, 2], + }, + MemchrTestStatic { + corpus: "zxy", + needles: &[b'x', b'a', b'z'], + positions: &[0, 1], + }, + MemchrTestStatic { + corpus: "zxy", + needles: &[b't', b'a', b'z'], + positions: &[0], + }, + MemchrTestStatic { + corpus: "yxz", + needles: &[b't', b'a', b'z'], + positions: &[2], + }, +]; + +/// A description of a test on a memchr like function. +#[derive(Clone, Debug)] +pub struct MemchrTest { + /// The thing to search. We use `&str` instead of `&[u8]` because they + /// are nicer to write in tests, and we don't miss much since memchr + /// doesn't care about UTF-8. + /// + /// Corpora cannot contain either '%' or '#'. We use these bytes when + /// expanding test cases into many test cases, and we assume they are not + /// used. If they are used, `memchr_tests` will panic. + corpus: String, + /// The needles to search for. This is intended to be an "alternation" of + /// needles. The number of needles may cause this test to be skipped for + /// some memchr variants. For example, a test with 2 needles cannot be used + /// to test `memchr`, but can be used to test `memchr2` and `memchr3`. + /// However, a test with only 1 needle can be used to test all of `memchr`, + /// `memchr2` and `memchr3`. We achieve this by filling in the needles with + /// bytes that we never used in the corpus (such as '#'). + needles: Vec, + /// The positions expected to match for all of the needles. + positions: Vec, +} + +/// Like MemchrTest, but easier to define as a constant. +#[derive(Clone, Debug)] +pub struct MemchrTestStatic { + corpus: &'static str, + needles: &'static [u8], + positions: &'static [usize], +} + +impl MemchrTest { + pub fn one Option>(&self, reverse: bool, f: F) { + let needles = match self.needles(1) { + None => return, + Some(needles) => needles, + }; + // We test different alignments here. Since some implementations use + // AVX2, which can read 32 bytes at a time, we test at least that. + // Moreover, with loop unrolling, we sometimes process 64 (sse2) or 128 + // (avx) bytes at a time, so we include that in our offsets as well. + // + // You might think this would cause most needles to not be found, but + // we actually expand our tests to include corpus sizes all the way up + // to >500 bytes, so we should exercise most branches. + for align in 0..130 { + let corpus = self.corpus(align); + assert_eq!( + self.positions(align, reverse).get(0).cloned(), + f(needles[0], corpus.as_bytes()), + "search for {:?} failed in: {:?} (len: {}, alignment: {})", + needles[0] as char, + corpus, + corpus.len(), + align + ); + } + } + + pub fn two Option>( + &self, + reverse: bool, + f: F, + ) { + let needles = match self.needles(2) { + None => return, + Some(needles) => needles, + }; + for align in 0..130 { + let corpus = self.corpus(align); + assert_eq!( + self.positions(align, reverse).get(0).cloned(), + f(needles[0], needles[1], corpus.as_bytes()), + "search for {:?}|{:?} failed in: {:?} \ + (len: {}, alignment: {})", + needles[0] as char, + needles[1] as char, + corpus, + corpus.len(), + align + ); + } + } + + pub fn three Option>( + &self, + reverse: bool, + f: F, + ) { + let needles = match self.needles(3) { + None => return, + Some(needles) => needles, + }; + for align in 0..130 { + let corpus = self.corpus(align); + assert_eq!( + self.positions(align, reverse).get(0).cloned(), + f(needles[0], needles[1], needles[2], corpus.as_bytes()), + "search for {:?}|{:?}|{:?} failed in: {:?} \ + (len: {}, alignment: {})", + needles[0] as char, + needles[1] as char, + needles[2] as char, + corpus, + corpus.len(), + align + ); + } + } + + pub fn iter_one<'a, I, F>(&'a self, reverse: bool, f: F) + where + F: FnOnce(u8, &'a [u8]) -> I, + I: Iterator, + { + if let Some(ns) = self.needles(1) { + self.iter(reverse, f(ns[0], self.corpus.as_bytes())); + } + } + + pub fn iter_two<'a, I, F>(&'a self, reverse: bool, f: F) + where + F: FnOnce(u8, u8, &'a [u8]) -> I, + I: Iterator, + { + if let Some(ns) = self.needles(2) { + self.iter(reverse, f(ns[0], ns[1], self.corpus.as_bytes())); + } + } + + pub fn iter_three<'a, I, F>(&'a self, reverse: bool, f: F) + where + F: FnOnce(u8, u8, u8, &'a [u8]) -> I, + I: Iterator, + { + if let Some(ns) = self.needles(3) { + self.iter(reverse, f(ns[0], ns[1], ns[2], self.corpus.as_bytes())); + } + } + + /// Test that the positions yielded by the given iterator match the + /// positions in this test. If reverse is true, then reverse the positions + /// before comparing them. + fn iter>(&self, reverse: bool, it: I) { + assert_eq!( + self.positions(0, reverse), + it.collect::>(), + r"search for {:?} failed in: {:?}", + self.needles.iter().map(|&b| b as char).collect::>(), + self.corpus + ); + } + + /// Expand this test into many variations of the same test. + /// + /// In particular, this will generate more tests with larger corpus sizes. + /// The expected positions are updated to maintain the integrity of the + /// test. + /// + /// This is important in testing a memchr implementation, because there are + /// often different cases depending on the length of the corpus. + /// + /// Note that we extend the corpus by adding `%` bytes, which we + /// don't otherwise use as a needle. + fn expand(&self) -> Vec { + let mut more = Vec::new(); + + // Add bytes to the start of the corpus. + for i in 1..515 { + let mut t = self.clone(); + let mut new_corpus: String = repeat('%').take(i).collect(); + new_corpus.push_str(&t.corpus); + t.corpus = new_corpus; + t.positions = t.positions.into_iter().map(|p| p + i).collect(); + more.push(t); + } + // Add bytes to the end of the corpus. + for i in 1..515 { + let mut t = self.clone(); + let padding: String = repeat('%').take(i).collect(); + t.corpus.push_str(&padding); + more.push(t); + } + + more + } + + /// Return the corpus at the given alignment. + /// + /// If the alignment exceeds the length of the corpus, then this returns + /// an empty slice. + fn corpus(&self, align: usize) -> &str { + self.corpus.get(align..).unwrap_or("") + } + + /// Return exactly `count` needles from this test. If this test has less + /// than `count` needles, then add `#` until the number of needles + /// matches `count`. If this test has more than `count` needles, then + /// return `None` (because there is no way to use this test data for a + /// search using fewer needles). + fn needles(&self, count: usize) -> Option> { + if self.needles.len() > count { + return None; + } + + let mut needles = self.needles.to_vec(); + for _ in needles.len()..count { + // we assume # is never used in tests. + needles.push(b'#'); + } + Some(needles) + } + + /// Return the positions in this test, reversed if `reverse` is true. + /// + /// If alignment is given, then all positions greater than or equal to that + /// alignment are offset by the alignment. Positions less than the + /// alignment are dropped. + fn positions(&self, align: usize, reverse: bool) -> Vec { + let positions = if reverse { + let mut positions = self.positions.to_vec(); + positions.reverse(); + positions + } else { + self.positions.to_vec() + }; + positions + .into_iter() + .filter(|&p| p >= align) + .map(|p| p - align) + .collect() + } +} diff --git a/crux-mir/lib/memchr/src/tests/mod.rs b/crux-mir/lib/memchr/src/tests/mod.rs new file mode 100644 index 000000000..f4d406cd1 --- /dev/null +++ b/crux-mir/lib/memchr/src/tests/mod.rs @@ -0,0 +1,15 @@ +mod memchr; + +// For debugging, particularly in CI, print out the byte order of the current +// target. +#[cfg(all(feature = "std", target_endian = "little"))] +#[test] +fn byte_order() { + eprintln!("LITTLE ENDIAN"); +} + +#[cfg(all(feature = "std", target_endian = "big"))] +#[test] +fn byte_order() { + eprintln!("BIG ENDIAN"); +} diff --git a/crux-mir/lib/memchr/src/tests/x86_64-soft_float.json b/crux-mir/lib/memchr/src/tests/x86_64-soft_float.json new file mode 100644 index 000000000..b77649ef9 --- /dev/null +++ b/crux-mir/lib/memchr/src/tests/x86_64-soft_float.json @@ -0,0 +1,15 @@ +{ + "llvm-target": "x86_64-unknown-none", + "target-endian": "little", + "target-pointer-width": "64", + "target-c-int-width": "32", + "os": "none", + "arch": "x86_64", + "data-layout": "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128", + "linker-flavor": "ld.lld", + "linker": "rust-lld", + "features": "-mmx,-sse,-sse2,-sse3,-ssse3,-sse4.1,-sse4.2,-3dnow,-3dnowa,-avx,-avx2,+soft-float", + "executables": true, + "disable-redzone": true, + "panic-strategy": "abort" +} diff --git a/crux-mir/lib/miniz_oxide/.cargo-ok b/crux-mir/lib/miniz_oxide/.cargo-ok new file mode 100644 index 000000000..b5754e203 --- /dev/null +++ b/crux-mir/lib/miniz_oxide/.cargo-ok @@ -0,0 +1 @@ +ok \ No newline at end of file diff --git a/crux-mir/lib/miniz_oxide/.cargo_vcs_info.json b/crux-mir/lib/miniz_oxide/.cargo_vcs_info.json new file mode 100644 index 000000000..ad204b0f6 --- /dev/null +++ b/crux-mir/lib/miniz_oxide/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "d0000c5aed8abb2ea290cc8b23ac55170fd3db6b" + }, + "path_in_vcs": "miniz_oxide" +} \ No newline at end of file diff --git a/crux-mir/lib/miniz_oxide/Cargo.toml b/crux-mir/lib/miniz_oxide/Cargo.toml new file mode 100644 index 000000000..7546128ce --- /dev/null +++ b/crux-mir/lib/miniz_oxide/Cargo.toml @@ -0,0 +1,55 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2018" +name = "miniz_oxide" +version = "0.5.3" +authors = ["Frommi ", "oyvindln "] +exclude = ["benches/*", "tests/*"] +description = "DEFLATE compression and decompression library rewritten in Rust based on miniz" +homepage = "https://github.com/Frommi/miniz_oxide/tree/master/miniz_oxide" +documentation = "https://docs.rs/miniz_oxide" +readme = "Readme.md" +keywords = ["zlib", "miniz", "deflate", "encoding"] +categories = ["compression"] +license = "MIT OR Zlib OR Apache-2.0" +repository = "https://github.com/Frommi/miniz_oxide/tree/master/miniz_oxide" + +[lib] +name = "miniz_oxide" +[dependencies.adler] +version = "1.0" +default-features = false + +[dependencies.alloc] +version = "1.0.0" +optional = true +package = "rustc-std-workspace-alloc" + +[dependencies.compiler_builtins] +version = "0.1.2" +optional = true + +[dependencies.core] +version = "1.0.0" +optional = true +package = "rustc-std-workspace-core" + +[dependencies.simd-adler32] +version = "0.3" +optional = true +default-features = false + +[features] +default = [] +rustc-dep-of-std = ["core", "alloc", "compiler_builtins", "adler/rustc-dep-of-std"] +simd = ["simd-adler32"] diff --git a/crux-mir/lib/miniz_oxide/Cargo.toml.orig b/crux-mir/lib/miniz_oxide/Cargo.toml.orig new file mode 100644 index 000000000..11caa4cc4 --- /dev/null +++ b/crux-mir/lib/miniz_oxide/Cargo.toml.orig @@ -0,0 +1,36 @@ +[package] +name = "miniz_oxide" +authors = ["Frommi ", "oyvindln "] +version = "0.5.3" +license = "MIT OR Zlib OR Apache-2.0" +readme = "Readme.md" +keywords = ["zlib", "miniz", "deflate", "encoding"] +categories = ["compression"] +repository = "https://github.com/Frommi/miniz_oxide/tree/master/miniz_oxide" +homepage = "https://github.com/Frommi/miniz_oxide/tree/master/miniz_oxide" +documentation = "https://docs.rs/miniz_oxide" +description = "DEFLATE compression and decompression library rewritten in Rust based on miniz" +edition = "2018" +exclude = ["benches/*", "tests/*"] + +[lib] +name = "miniz_oxide" + +[dependencies] +adler = { version = "1.0", default-features = false } +simd-adler32 = { version = "0.3", default-features = false, optional = true } + +# Internal feature, only used when building as part of libstd, not part of the +# stable interface of this crate. +core = { version = '1.0.0', optional = true, package = 'rustc-std-workspace-core' } +alloc = { version = '1.0.0', optional = true, package = 'rustc-std-workspace-alloc' } +compiler_builtins = { version = '0.1.2', optional = true } + +[features] +default = [] + +# Internal feature, only used when building as part of libstd, not part of the +# stable interface of this crate. +rustc-dep-of-std = ['core', 'alloc', 'compiler_builtins', 'adler/rustc-dep-of-std'] + +simd = ['simd-adler32'] diff --git a/crux-mir/lib/miniz_oxide/LICENSE b/crux-mir/lib/miniz_oxide/LICENSE new file mode 100644 index 000000000..64c53792c --- /dev/null +++ b/crux-mir/lib/miniz_oxide/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Frommi + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crux-mir/lib/miniz_oxide/LICENSE-APACHE.md b/crux-mir/lib/miniz_oxide/LICENSE-APACHE.md new file mode 100644 index 000000000..f433b1a53 --- /dev/null +++ b/crux-mir/lib/miniz_oxide/LICENSE-APACHE.md @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/crux-mir/lib/miniz_oxide/LICENSE-MIT.md b/crux-mir/lib/miniz_oxide/LICENSE-MIT.md new file mode 100644 index 000000000..64c53792c --- /dev/null +++ b/crux-mir/lib/miniz_oxide/LICENSE-MIT.md @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Frommi + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crux-mir/lib/miniz_oxide/LICENSE-ZLIB.md b/crux-mir/lib/miniz_oxide/LICENSE-ZLIB.md new file mode 100644 index 000000000..7f513d1ac --- /dev/null +++ b/crux-mir/lib/miniz_oxide/LICENSE-ZLIB.md @@ -0,0 +1,11 @@ +Copyright (c) 2020 Frommi + +This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. + +3. This notice may not be removed or altered from any source distribution. diff --git a/crux-mir/lib/miniz_oxide/Readme.md b/crux-mir/lib/miniz_oxide/Readme.md new file mode 100644 index 000000000..0eac176e8 --- /dev/null +++ b/crux-mir/lib/miniz_oxide/Readme.md @@ -0,0 +1,35 @@ +# miniz_oxide + +A fully safe, pure rust replacement for the [miniz](https://github.com/richgel999/miniz) DEFLATE/zlib encoder/decoder. +The main intention of this crate is to be used as a back-end for the [flate2](https://github.com/alexcrichton/flate2-rs), but it can also be used on it's own. Using flate2 with the ```rust_backend``` feature provides an easy to use streaming API for miniz_oxide. + +The library is fully [no_std](https://docs.rust-embedded.org/book/intro/no-std.html), though it requires the use of the `alloc` and `collection` crates as it allocates memory. + +miniz_oxide 0.5.x Requires at least rust 1.40.0 0.3.x requires at least rust 0.36.0. + +miniz_oxide features no use of unsafe code. + +miniz_oxide can optionally be made to use a simd-accelerated version of adler32 via the [simd-adler32](https://crates.io/crates/simd-adler32) crate by enabling the 'simd' feature. This is not enabled by default as due to the use of simd intrinsics, the simd-adler32 has to use unsafe. The default setup uses the [adler](https://crates.io/crates/adler) crate which features no unsafe code. + +## Usage +Simple compression/decompression: +```rust + +use miniz_oxide::deflate::compress_to_vec; +use miniz_oxide::inflate::decompress_to_vec; + +fn roundtrip(data: &[u8]) { + // Compress the input + let compressed = compress_to_vec(data, 6); + // Decompress the compressed input + let decompressed = decompress_to_vec(compressed.as_slice()).expect("Failed to decompress!"); + // Check roundtrip succeeded + assert_eq!(data, decompressed); +} + +fn main() { + roundtrip("Hello, world!".as_bytes()); +} + +``` +These simple functions will do everything in one go and are thus not recommended for use cases where the input size may be large or unknown, for that use case consider using miniz_oxide via flate2 or the low-level streaming functions instead. diff --git a/crux-mir/lib/miniz_oxide/src/deflate/buffer.rs b/crux-mir/lib/miniz_oxide/src/deflate/buffer.rs new file mode 100644 index 000000000..f246c07df --- /dev/null +++ b/crux-mir/lib/miniz_oxide/src/deflate/buffer.rs @@ -0,0 +1,58 @@ +//! Buffer wrappers implementing default so we can allocate the buffers with `Box::default()` +//! to avoid stack copies. Box::new() doesn't at the moment, and using a vec means we would lose +//! static length info. + +use crate::deflate::core::{LZ_DICT_SIZE, MAX_MATCH_LEN}; + +/// Size of the buffer of lz77 encoded data. +pub const LZ_CODE_BUF_SIZE: usize = 64 * 1024; +/// Size of the output buffer. +pub const OUT_BUF_SIZE: usize = (LZ_CODE_BUF_SIZE * 13) / 10; +pub const LZ_DICT_FULL_SIZE: usize = LZ_DICT_SIZE + MAX_MATCH_LEN - 1 + 1; + +/// Size of hash values in the hash chains. +pub const LZ_HASH_BITS: i32 = 15; +/// How many bits to shift when updating the current hash value. +pub const LZ_HASH_SHIFT: i32 = (LZ_HASH_BITS + 2) / 3; +/// Size of the chained hash tables. +pub const LZ_HASH_SIZE: usize = 1 << LZ_HASH_BITS; + +#[inline] +pub fn update_hash(current_hash: u16, byte: u8) -> u16 { + ((current_hash << LZ_HASH_SHIFT) ^ u16::from(byte)) & (LZ_HASH_SIZE as u16 - 1) +} + +pub struct HashBuffers { + pub dict: [u8; LZ_DICT_FULL_SIZE], + pub next: [u16; LZ_DICT_SIZE], + pub hash: [u16; LZ_DICT_SIZE], +} + +impl HashBuffers { + #[inline] + pub fn reset(&mut self) { + *self = HashBuffers::default(); + } +} + +impl Default for HashBuffers { + fn default() -> HashBuffers { + HashBuffers { + dict: [0; LZ_DICT_FULL_SIZE], + next: [0; LZ_DICT_SIZE], + hash: [0; LZ_DICT_SIZE], + } + } +} + +pub struct LocalBuf { + pub b: [u8; OUT_BUF_SIZE], +} + +impl Default for LocalBuf { + fn default() -> LocalBuf { + LocalBuf { + b: [0; OUT_BUF_SIZE], + } + } +} diff --git a/crux-mir/lib/miniz_oxide/src/deflate/core.rs b/crux-mir/lib/miniz_oxide/src/deflate/core.rs new file mode 100644 index 000000000..91a9bf8b8 --- /dev/null +++ b/crux-mir/lib/miniz_oxide/src/deflate/core.rs @@ -0,0 +1,2463 @@ +//! Streaming compression functionality. + +use alloc::boxed::Box; +use core::convert::TryInto; +use core::{cmp, mem}; + +use super::super::*; +use super::deflate_flags::*; +use super::CompressionLevel; +use crate::deflate::buffer::{ + update_hash, HashBuffers, LocalBuf, LZ_CODE_BUF_SIZE, LZ_DICT_FULL_SIZE, LZ_HASH_BITS, + LZ_HASH_SHIFT, LZ_HASH_SIZE, OUT_BUF_SIZE, +}; +use crate::shared::{update_adler32, HUFFMAN_LENGTH_ORDER, MZ_ADLER32_INIT}; +use crate::DataFormat; + +// Currently not bubbled up outside this module, so can fill in with more +// context eventually if needed. +type Result = core::result::Result; +struct Error {} + +const MAX_PROBES_MASK: i32 = 0xFFF; + +const MAX_SUPPORTED_HUFF_CODESIZE: usize = 32; + +/// Length code for length values. +#[rustfmt::skip] +const LEN_SYM: [u16; 256] = [ + 257, 258, 259, 260, 261, 262, 263, 264, 265, 265, 266, 266, 267, 267, 268, 268, + 269, 269, 269, 269, 270, 270, 270, 270, 271, 271, 271, 271, 272, 272, 272, 272, + 273, 273, 273, 273, 273, 273, 273, 273, 274, 274, 274, 274, 274, 274, 274, 274, + 275, 275, 275, 275, 275, 275, 275, 275, 276, 276, 276, 276, 276, 276, 276, 276, + 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, + 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, + 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, + 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, + 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, + 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, + 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 285 +]; + +/// Number of extra bits for length values. +#[rustfmt::skip] +const LEN_EXTRA: [u8; 256] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0 +]; + +/// Distance codes for distances smaller than 512. +#[rustfmt::skip] +const SMALL_DIST_SYM: [u8; 512] = [ + 0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, + 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17 +]; + +/// Number of extra bits for distances smaller than 512. +#[rustfmt::skip] +const SMALL_DIST_EXTRA: [u8; 512] = [ + 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 +]; + +/// Base values to calculate distances above 512. +#[rustfmt::skip] +const LARGE_DIST_SYM: [u8; 128] = [ + 0, 0, 18, 19, 20, 20, 21, 21, 22, 22, 22, 22, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, 25, 25, 25, 25, 25, 25, 25, 25, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, + 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, + 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, + 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, + 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29 +]; + +/// Number of extra bits distances above 512. +#[rustfmt::skip] +const LARGE_DIST_EXTRA: [u8; 128] = [ + 0, 0, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, + 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13 +]; + +#[rustfmt::skip] +const BITMASKS: [u32; 17] = [ + 0x0000, 0x0001, 0x0003, 0x0007, 0x000F, 0x001F, 0x003F, 0x007F, 0x00FF, + 0x01FF, 0x03FF, 0x07FF, 0x0FFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF +]; + +/// The maximum number of checks for matches in the hash table the compressor will make for each +/// compression level. +const NUM_PROBES: [u32; 11] = [0, 1, 6, 32, 16, 32, 128, 256, 512, 768, 1500]; + +#[derive(Copy, Clone)] +struct SymFreq { + key: u16, + sym_index: u16, +} + +pub mod deflate_flags { + /// Whether to use a zlib wrapper. + pub const TDEFL_WRITE_ZLIB_HEADER: u32 = 0x0000_1000; + /// Should we compute the adler32 checksum. + pub const TDEFL_COMPUTE_ADLER32: u32 = 0x0000_2000; + /// Should we use greedy parsing (as opposed to lazy parsing where look ahead one or more + /// bytes to check for better matches.) + pub const TDEFL_GREEDY_PARSING_FLAG: u32 = 0x0000_4000; + /// Used in miniz to skip zero-initializing hash and dict. We don't do this here, so + /// this flag is ignored. + pub const TDEFL_NONDETERMINISTIC_PARSING_FLAG: u32 = 0x0000_8000; + /// Only look for matches with a distance of 0. + pub const TDEFL_RLE_MATCHES: u32 = 0x0001_0000; + /// Only use matches that are at least 6 bytes long. + pub const TDEFL_FILTER_MATCHES: u32 = 0x0002_0000; + /// Force the compressor to only output static blocks. (Blocks using the default huffman codes + /// specified in the deflate specification.) + pub const TDEFL_FORCE_ALL_STATIC_BLOCKS: u32 = 0x0004_0000; + /// Force the compressor to only output raw/uncompressed blocks. + pub const TDEFL_FORCE_ALL_RAW_BLOCKS: u32 = 0x0008_0000; +} + +/// Strategy setting for compression. +/// +/// The non-default settings offer some special-case compression variants. +#[repr(i32)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum CompressionStrategy { + /// Don't use any of the special strategies. + Default = 0, + /// Only use matches that are at least 5 bytes long. + Filtered = 1, + /// Don't look for matches, only huffman encode the literals. + HuffmanOnly = 2, + /// Only look for matches with a distance of 1, i.e do run-length encoding only. + RLE = 3, + /// Only use static/fixed blocks. (Blocks using the default huffman codes + /// specified in the deflate specification.) + Fixed = 4, +} + +/// A list of deflate flush types. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum TDEFLFlush { + /// Normal operation. + /// + /// Compress as much as there is space for, and then return waiting for more input. + None = 0, + + /// Try to flush all the current data and output an empty raw block. + Sync = 2, + + /// Same as [`Sync`][Self::Sync], but reset the dictionary so that the following data does not + /// depend on previous data. + Full = 3, + + /// Try to flush everything and end the deflate stream. + /// + /// On success this will yield a [`TDEFLStatus::Done`] return status. + Finish = 4, +} + +impl From for TDEFLFlush { + fn from(flush: MZFlush) -> Self { + match flush { + MZFlush::None => TDEFLFlush::None, + MZFlush::Sync => TDEFLFlush::Sync, + MZFlush::Full => TDEFLFlush::Full, + MZFlush::Finish => TDEFLFlush::Finish, + _ => TDEFLFlush::None, // TODO: ??? What to do ??? + } + } +} + +impl TDEFLFlush { + pub fn new(flush: i32) -> Result { + match flush { + 0 => Ok(TDEFLFlush::None), + 2 => Ok(TDEFLFlush::Sync), + 3 => Ok(TDEFLFlush::Full), + 4 => Ok(TDEFLFlush::Finish), + _ => Err(MZError::Param), + } + } +} + +/// Return status of compression. +#[repr(i32)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum TDEFLStatus { + /// Usage error. + /// + /// This indicates that either the [`CompressorOxide`] experienced a previous error, or the + /// stream has already been [`TDEFLFlush::Finish`]'d. + BadParam = -2, + + /// Error putting data into output buffer. + /// + /// This usually indicates a too-small buffer. + PutBufFailed = -1, + + /// Compression succeeded normally. + Okay = 0, + + /// Compression succeeded and the deflate stream was ended. + /// + /// This is the result of calling compression with [`TDEFLFlush::Finish`]. + Done = 1, +} + +const MAX_HUFF_SYMBOLS: usize = 288; +/// Size of hash chain for fast compression mode. +const LEVEL1_HASH_SIZE_MASK: u32 = 4095; +/// The number of huffman tables used by the compressor. +/// Literal/length, Distances and Length of the huffman codes for the other two tables. +const MAX_HUFF_TABLES: usize = 3; +/// Literal/length codes +const MAX_HUFF_SYMBOLS_0: usize = 288; +/// Distance codes. +const MAX_HUFF_SYMBOLS_1: usize = 32; +/// Huffman length values. +const MAX_HUFF_SYMBOLS_2: usize = 19; +/// Size of the chained hash table. +pub(crate) const LZ_DICT_SIZE: usize = 32_768; +/// Mask used when stepping through the hash chains. +const LZ_DICT_SIZE_MASK: usize = (LZ_DICT_SIZE as u32 - 1) as usize; +/// The minimum length of a match. +const MIN_MATCH_LEN: u8 = 3; +/// The maximum length of a match. +pub(crate) const MAX_MATCH_LEN: usize = 258; + +const DEFAULT_FLAGS: u32 = NUM_PROBES[4] | TDEFL_WRITE_ZLIB_HEADER; + +mod zlib { + const DEFAULT_CM: u8 = 8; + const DEFAULT_CINFO: u8 = 7 << 4; + const _DEFAULT_FDICT: u8 = 0; + const DEFAULT_CMF: u8 = DEFAULT_CM | DEFAULT_CINFO; + /// The 16-bit value consisting of CMF and FLG must be divisible by this to be valid. + const FCHECK_DIVISOR: u8 = 31; + + /// Generate FCHECK from CMF and FLG (without FCKECH )so that they are correct according to the + /// specification, i.e (CMF*256 + FCHK) % 31 = 0. + /// Returns flg with the FCHKECK bits added (any existing FCHECK bits are ignored). + fn add_fcheck(cmf: u8, flg: u8) -> u8 { + let rem = ((usize::from(cmf) * 256) + usize::from(flg)) % usize::from(FCHECK_DIVISOR); + + // Clear existing FCHECK if any + let flg = flg & 0b11100000; + + // Casting is safe as rem can't overflow since it is a value mod 31 + // We can simply add the value to flg as (31 - rem) will never be above 2^5 + flg + (FCHECK_DIVISOR - rem as u8) + } + + fn zlib_level_from_flags(flags: u32) -> u8 { + use super::NUM_PROBES; + + let num_probes = flags & (super::MAX_PROBES_MASK as u32); + if flags & super::TDEFL_GREEDY_PARSING_FLAG != 0 { + if num_probes <= 1 { + 0 + } else { + 1 + } + } else if num_probes >= NUM_PROBES[9] { + 3 + } else { + 2 + } + } + + /// Get the zlib header for the level using the default window size and no + /// dictionary. + fn header_from_level(level: u8) -> [u8; 2] { + let cmf = DEFAULT_CMF; + [cmf, add_fcheck(cmf, (level as u8) << 6)] + } + + /// Create a zlib header from the given compression flags. + /// Only level is considered. + pub fn header_from_flags(flags: u32) -> [u8; 2] { + let level = zlib_level_from_flags(flags); + header_from_level(level) + } + + #[cfg(test)] + mod test { + #[test] + fn zlib() { + use super::super::*; + use super::*; + + let test_level = |level, expected| { + let flags = create_comp_flags_from_zip_params( + level, + MZ_DEFAULT_WINDOW_BITS, + CompressionStrategy::Default as i32, + ); + assert_eq!(zlib_level_from_flags(flags), expected); + }; + + assert_eq!(zlib_level_from_flags(DEFAULT_FLAGS), 2); + test_level(0, 0); + test_level(1, 0); + test_level(2, 1); + test_level(3, 1); + for i in 4..=8 { + test_level(i, 2) + } + test_level(9, 3); + test_level(10, 3); + } + + #[test] + fn test_header() { + let header = super::header_from_level(3); + assert_eq!( + ((usize::from(header[0]) * 256) + usize::from(header[1])) % 31, + 0 + ); + } + } +} + +fn memset(slice: &mut [T], val: T) { + for x in slice { + *x = val + } +} + +#[cfg(test)] +#[inline] +fn write_u16_le(val: u16, slice: &mut [u8], pos: usize) { + slice[pos] = val as u8; + slice[pos + 1] = (val >> 8) as u8; +} + +// Read the two bytes starting at pos and interpret them as an u16. +#[inline] +const fn read_u16_le(slice: &[u8], pos: usize) -> u16 { + // The compiler is smart enough to optimize this into an unaligned load. + slice[pos] as u16 | ((slice[pos + 1] as u16) << 8) +} + +/// Main compression struct. +pub struct CompressorOxide { + lz: LZOxide, + params: ParamsOxide, + huff: Box, + dict: DictOxide, +} + +impl CompressorOxide { + /// Create a new `CompressorOxide` with the given flags. + /// + /// # Notes + /// This function may be changed to take different parameters in the future. + pub fn new(flags: u32) -> Self { + CompressorOxide { + lz: LZOxide::new(), + params: ParamsOxide::new(flags), + /// Put HuffmanOxide on the heap with default trick to avoid + /// excessive stack copies. + huff: Box::default(), + dict: DictOxide::new(flags), + } + } + + /// Get the adler32 checksum of the currently encoded data. + pub const fn adler32(&self) -> u32 { + self.params.adler32 + } + + /// Get the return status of the previous [`compress`](fn.compress.html) + /// call with this compressor. + pub const fn prev_return_status(&self) -> TDEFLStatus { + self.params.prev_return_status + } + + /// Get the raw compressor flags. + /// + /// # Notes + /// This function may be deprecated or changed in the future to use more rust-style flags. + pub const fn flags(&self) -> i32 { + self.params.flags as i32 + } + + /// Returns whether the compressor is wrapping the data in a zlib format or not. + pub fn data_format(&self) -> DataFormat { + if (self.params.flags & TDEFL_WRITE_ZLIB_HEADER) != 0 { + DataFormat::Zlib + } else { + DataFormat::Raw + } + } + + /// Reset the state of the compressor, keeping the same parameters. + /// + /// This avoids re-allocating data. + pub fn reset(&mut self) { + // LZ buf and huffman has no settings or dynamic memory + // that needs to be saved, so we simply replace them. + self.lz = LZOxide::new(); + self.params.reset(); + *self.huff = HuffmanOxide::default(); + self.dict.reset(); + } + + /// Set the compression level of the compressor. + /// + /// Using this to change level after compression has started is supported. + /// # Notes + /// The compression strategy will be reset to the default one when this is called. + pub fn set_compression_level(&mut self, level: CompressionLevel) { + let format = self.data_format(); + self.set_format_and_level(format, level as u8); + } + + /// Set the compression level of the compressor using an integer value. + /// + /// Using this to change level after compression has started is supported. + /// # Notes + /// The compression strategy will be reset to the default one when this is called. + pub fn set_compression_level_raw(&mut self, level: u8) { + let format = self.data_format(); + self.set_format_and_level(format, level); + } + + /// Update the compression settings of the compressor. + /// + /// Changing the `DataFormat` after compression has started will result in + /// a corrupted stream. + /// + /// # Notes + /// This function mainly intended for setting the initial settings after e.g creating with + /// `default` or after calling `CompressorOxide::reset()`, and behaviour may be changed + /// to disallow calling it after starting compression in the future. + pub fn set_format_and_level(&mut self, data_format: DataFormat, level: u8) { + let flags = create_comp_flags_from_zip_params( + level.into(), + data_format.to_window_bits(), + CompressionStrategy::Default as i32, + ); + self.params.update_flags(flags); + self.dict.update_flags(flags); + } +} + +impl Default for CompressorOxide { + /// Initialize the compressor with a level of 4, zlib wrapper and + /// the default strategy. + #[inline(always)] + fn default() -> Self { + CompressorOxide { + lz: LZOxide::new(), + params: ParamsOxide::new(DEFAULT_FLAGS), + /// Put HuffmanOxide on the heap with default trick to avoid + /// excessive stack copies. + huff: Box::default(), + dict: DictOxide::new(DEFAULT_FLAGS), + } + } +} + +/// Callback function and user used in `compress_to_output`. +pub struct CallbackFunc<'a> { + pub put_buf_func: &'a mut dyn FnMut(&[u8]) -> bool, +} + +impl<'a> CallbackFunc<'a> { + fn flush_output( + &mut self, + saved_output: SavedOutputBufferOxide, + params: &mut ParamsOxide, + ) -> i32 { + // TODO: As this could be unsafe since + // we can't verify the function pointer + // this whole function should maybe be unsafe as well. + let call_success = (self.put_buf_func)(¶ms.local_buf.b[0..saved_output.pos as usize]); + + if !call_success { + params.prev_return_status = TDEFLStatus::PutBufFailed; + return params.prev_return_status as i32; + } + + params.flush_remaining as i32 + } +} + +struct CallbackBuf<'a> { + pub out_buf: &'a mut [u8], +} + +impl<'a> CallbackBuf<'a> { + fn flush_output( + &mut self, + saved_output: SavedOutputBufferOxide, + params: &mut ParamsOxide, + ) -> i32 { + if saved_output.local { + let n = cmp::min( + saved_output.pos as usize, + self.out_buf.len() - params.out_buf_ofs, + ); + (&mut self.out_buf[params.out_buf_ofs..params.out_buf_ofs + n]) + .copy_from_slice(¶ms.local_buf.b[..n]); + + params.out_buf_ofs += n; + if saved_output.pos != n { + params.flush_ofs = n as u32; + params.flush_remaining = (saved_output.pos - n) as u32; + } + } else { + params.out_buf_ofs += saved_output.pos; + } + + params.flush_remaining as i32 + } +} + +enum CallbackOut<'a> { + Func(CallbackFunc<'a>), + Buf(CallbackBuf<'a>), +} + +impl<'a> CallbackOut<'a> { + fn new_output_buffer<'b>( + &'b mut self, + local_buf: &'b mut [u8], + out_buf_ofs: usize, + ) -> OutputBufferOxide<'b> { + let is_local; + let buf_len = OUT_BUF_SIZE - 16; + let chosen_buffer = match *self { + CallbackOut::Buf(ref mut cb) if cb.out_buf.len() - out_buf_ofs >= OUT_BUF_SIZE => { + is_local = false; + &mut cb.out_buf[out_buf_ofs..out_buf_ofs + buf_len] + } + _ => { + is_local = true; + &mut local_buf[..buf_len] + } + }; + + OutputBufferOxide { + inner: chosen_buffer, + inner_pos: 0, + local: is_local, + bit_buffer: 0, + bits_in: 0, + } + } +} + +struct CallbackOxide<'a> { + in_buf: Option<&'a [u8]>, + in_buf_size: Option<&'a mut usize>, + out_buf_size: Option<&'a mut usize>, + out: CallbackOut<'a>, +} + +impl<'a> CallbackOxide<'a> { + fn new_callback_buf(in_buf: &'a [u8], out_buf: &'a mut [u8]) -> Self { + CallbackOxide { + in_buf: Some(in_buf), + in_buf_size: None, + out_buf_size: None, + out: CallbackOut::Buf(CallbackBuf { out_buf }), + } + } + + fn new_callback_func(in_buf: &'a [u8], callback_func: CallbackFunc<'a>) -> Self { + CallbackOxide { + in_buf: Some(in_buf), + in_buf_size: None, + out_buf_size: None, + out: CallbackOut::Func(callback_func), + } + } + + fn update_size(&mut self, in_size: Option, out_size: Option) { + if let (Some(in_size), Some(size)) = (in_size, self.in_buf_size.as_mut()) { + **size = in_size; + } + + if let (Some(out_size), Some(size)) = (out_size, self.out_buf_size.as_mut()) { + **size = out_size + } + } + + fn flush_output( + &mut self, + saved_output: SavedOutputBufferOxide, + params: &mut ParamsOxide, + ) -> i32 { + if saved_output.pos == 0 { + return params.flush_remaining as i32; + } + + self.update_size(Some(params.src_pos), None); + match self.out { + CallbackOut::Func(ref mut cf) => cf.flush_output(saved_output, params), + CallbackOut::Buf(ref mut cb) => cb.flush_output(saved_output, params), + } + } +} + +struct OutputBufferOxide<'a> { + pub inner: &'a mut [u8], + pub inner_pos: usize, + pub local: bool, + + pub bit_buffer: u32, + pub bits_in: u32, +} + +impl<'a> OutputBufferOxide<'a> { + fn put_bits(&mut self, bits: u32, len: u32) { + assert!(bits <= ((1u32 << len) - 1u32)); + self.bit_buffer |= bits << self.bits_in; + self.bits_in += len; + while self.bits_in >= 8 { + self.inner[self.inner_pos] = self.bit_buffer as u8; + self.inner_pos += 1; + self.bit_buffer >>= 8; + self.bits_in -= 8; + } + } + + const fn save(&self) -> SavedOutputBufferOxide { + SavedOutputBufferOxide { + pos: self.inner_pos, + bit_buffer: self.bit_buffer, + bits_in: self.bits_in, + local: self.local, + } + } + + fn load(&mut self, saved: SavedOutputBufferOxide) { + self.inner_pos = saved.pos; + self.bit_buffer = saved.bit_buffer; + self.bits_in = saved.bits_in; + self.local = saved.local; + } + + fn pad_to_bytes(&mut self) { + if self.bits_in != 0 { + let len = 8 - self.bits_in; + self.put_bits(0, len); + } + } +} + +struct SavedOutputBufferOxide { + pub pos: usize, + pub bit_buffer: u32, + pub bits_in: u32, + pub local: bool, +} + +struct BitBuffer { + pub bit_buffer: u64, + pub bits_in: u32, +} + +impl BitBuffer { + fn put_fast(&mut self, bits: u64, len: u32) { + self.bit_buffer |= bits << self.bits_in; + self.bits_in += len; + } + + fn flush(&mut self, output: &mut OutputBufferOxide) -> Result<()> { + let pos = output.inner_pos; + { + // isolation to please borrow checker + let inner = &mut output.inner[pos..pos + 8]; + let bytes = u64::to_le_bytes(self.bit_buffer); + inner.copy_from_slice(&bytes); + } + match output.inner_pos.checked_add((self.bits_in >> 3) as usize) { + Some(n) if n <= output.inner.len() => output.inner_pos = n, + _ => return Err(Error {}), + } + self.bit_buffer >>= self.bits_in & !7; + self.bits_in &= 7; + Ok(()) + } +} + +/// A struct containing data about huffman codes and symbol frequencies. +/// +/// NOTE: Only the literal/lengths have enough symbols to actually use +/// the full array. It's unclear why it's defined like this in miniz, +/// it could be for cache/alignment reasons. +struct HuffmanOxide { + /// Number of occurrences of each symbol. + pub count: [[u16; MAX_HUFF_SYMBOLS]; MAX_HUFF_TABLES], + /// The bits of the huffman code assigned to the symbol + pub codes: [[u16; MAX_HUFF_SYMBOLS]; MAX_HUFF_TABLES], + /// The length of the huffman code assigned to the symbol. + pub code_sizes: [[u8; MAX_HUFF_SYMBOLS]; MAX_HUFF_TABLES], +} + +/// Tables used for literal/lengths in `HuffmanOxide`. +const LITLEN_TABLE: usize = 0; +/// Tables for distances. +const DIST_TABLE: usize = 1; +/// Tables for the run-length encoded huffman lengths for literals/lengths/distances. +const HUFF_CODES_TABLE: usize = 2; + +/// Status of RLE encoding of huffman code lengths. +struct Rle { + pub z_count: u32, + pub repeat_count: u32, + pub prev_code_size: u8, +} + +impl Rle { + fn prev_code_size( + &mut self, + packed_code_sizes: &mut [u8], + packed_pos: &mut usize, + h: &mut HuffmanOxide, + ) -> Result<()> { + let mut write = |buf| write(buf, packed_code_sizes, packed_pos); + let counts = &mut h.count[HUFF_CODES_TABLE]; + if self.repeat_count != 0 { + if self.repeat_count < 3 { + counts[self.prev_code_size as usize] = + counts[self.prev_code_size as usize].wrapping_add(self.repeat_count as u16); + let code = self.prev_code_size; + write(&[code, code, code][..self.repeat_count as usize])?; + } else { + counts[16] = counts[16].wrapping_add(1); + write(&[16, (self.repeat_count - 3) as u8][..])?; + } + self.repeat_count = 0; + } + + Ok(()) + } + + fn zero_code_size( + &mut self, + packed_code_sizes: &mut [u8], + packed_pos: &mut usize, + h: &mut HuffmanOxide, + ) -> Result<()> { + let mut write = |buf| write(buf, packed_code_sizes, packed_pos); + let counts = &mut h.count[HUFF_CODES_TABLE]; + if self.z_count != 0 { + if self.z_count < 3 { + counts[0] = counts[0].wrapping_add(self.z_count as u16); + write(&[0, 0, 0][..self.z_count as usize])?; + } else if self.z_count <= 10 { + counts[17] = counts[17].wrapping_add(1); + write(&[17, (self.z_count - 3) as u8][..])?; + } else { + counts[18] = counts[18].wrapping_add(1); + write(&[18, (self.z_count - 11) as u8][..])?; + } + self.z_count = 0; + } + + Ok(()) + } +} + +fn write(src: &[u8], dst: &mut [u8], dst_pos: &mut usize) -> Result<()> { + match dst.get_mut(*dst_pos..*dst_pos + src.len()) { + Some(s) => s.copy_from_slice(src), + None => return Err(Error {}), + } + *dst_pos += src.len(); + Ok(()) +} + +impl Default for HuffmanOxide { + fn default() -> Self { + HuffmanOxide { + count: [[0; MAX_HUFF_SYMBOLS]; MAX_HUFF_TABLES], + codes: [[0; MAX_HUFF_SYMBOLS]; MAX_HUFF_TABLES], + code_sizes: [[0; MAX_HUFF_SYMBOLS]; MAX_HUFF_TABLES], + } + } +} + +impl HuffmanOxide { + fn radix_sort_symbols<'a>( + symbols0: &'a mut [SymFreq], + symbols1: &'a mut [SymFreq], + ) -> &'a mut [SymFreq] { + let mut hist = [[0; 256]; 2]; + + for freq in symbols0.iter() { + hist[0][(freq.key & 0xFF) as usize] += 1; + hist[1][((freq.key >> 8) & 0xFF) as usize] += 1; + } + + let mut n_passes = 2; + if symbols0.len() == hist[1][0] { + n_passes -= 1; + } + + let mut current_symbols = symbols0; + let mut new_symbols = symbols1; + + for (pass, hist_item) in hist.iter().enumerate().take(n_passes) { + let mut offsets = [0; 256]; + let mut offset = 0; + for i in 0..256 { + offsets[i] = offset; + offset += hist_item[i]; + } + + for sym in current_symbols.iter() { + let j = ((sym.key >> (pass * 8)) & 0xFF) as usize; + new_symbols[offsets[j]] = *sym; + offsets[j] += 1; + } + + mem::swap(&mut current_symbols, &mut new_symbols); + } + + current_symbols + } + + fn calculate_minimum_redundancy(symbols: &mut [SymFreq]) { + match symbols.len() { + 0 => (), + 1 => symbols[0].key = 1, + n => { + symbols[0].key += symbols[1].key; + let mut root = 0; + let mut leaf = 2; + for next in 1..n - 1 { + if (leaf >= n) || (symbols[root].key < symbols[leaf].key) { + symbols[next].key = symbols[root].key; + symbols[root].key = next as u16; + root += 1; + } else { + symbols[next].key = symbols[leaf].key; + leaf += 1; + } + + if (leaf >= n) || (root < next && symbols[root].key < symbols[leaf].key) { + symbols[next].key = symbols[next].key.wrapping_add(symbols[root].key); + symbols[root].key = next as u16; + root += 1; + } else { + symbols[next].key = symbols[next].key.wrapping_add(symbols[leaf].key); + leaf += 1; + } + } + + symbols[n - 2].key = 0; + for next in (0..n - 2).rev() { + symbols[next].key = symbols[symbols[next].key as usize].key + 1; + } + + let mut avbl = 1; + let mut used = 0; + let mut dpth = 0; + let mut root = (n - 2) as i32; + let mut next = (n - 1) as i32; + while avbl > 0 { + while (root >= 0) && (symbols[root as usize].key == dpth) { + used += 1; + root -= 1; + } + while avbl > used { + symbols[next as usize].key = dpth; + next -= 1; + avbl -= 1; + } + avbl = 2 * used; + dpth += 1; + used = 0; + } + } + } + } + + fn enforce_max_code_size(num_codes: &mut [i32], code_list_len: usize, max_code_size: usize) { + if code_list_len <= 1 { + return; + } + + num_codes[max_code_size] += num_codes[max_code_size + 1..].iter().sum::(); + let total = num_codes[1..=max_code_size] + .iter() + .rev() + .enumerate() + .fold(0u32, |total, (i, &x)| total + ((x as u32) << i)); + + for _ in (1 << max_code_size)..total { + num_codes[max_code_size] -= 1; + for i in (1..max_code_size).rev() { + if num_codes[i] != 0 { + num_codes[i] -= 1; + num_codes[i + 1] += 2; + break; + } + } + } + } + + fn optimize_table( + &mut self, + table_num: usize, + table_len: usize, + code_size_limit: usize, + static_table: bool, + ) { + let mut num_codes = [0i32; MAX_SUPPORTED_HUFF_CODESIZE + 1]; + let mut next_code = [0u32; MAX_SUPPORTED_HUFF_CODESIZE + 1]; + + if static_table { + for &code_size in &self.code_sizes[table_num][..table_len] { + num_codes[code_size as usize] += 1; + } + } else { + let mut symbols0 = [SymFreq { + key: 0, + sym_index: 0, + }; MAX_HUFF_SYMBOLS]; + let mut symbols1 = [SymFreq { + key: 0, + sym_index: 0, + }; MAX_HUFF_SYMBOLS]; + + let mut num_used_symbols = 0; + for i in 0..table_len { + if self.count[table_num][i] != 0 { + symbols0[num_used_symbols] = SymFreq { + key: self.count[table_num][i], + sym_index: i as u16, + }; + num_used_symbols += 1; + } + } + + let symbols = Self::radix_sort_symbols( + &mut symbols0[..num_used_symbols], + &mut symbols1[..num_used_symbols], + ); + Self::calculate_minimum_redundancy(symbols); + + for symbol in symbols.iter() { + num_codes[symbol.key as usize] += 1; + } + + Self::enforce_max_code_size(&mut num_codes, num_used_symbols, code_size_limit); + + memset(&mut self.code_sizes[table_num][..], 0); + memset(&mut self.codes[table_num][..], 0); + + let mut last = num_used_symbols; + for (i, &num_item) in num_codes + .iter() + .enumerate() + .take(code_size_limit + 1) + .skip(1) + { + let first = last - num_item as usize; + for symbol in &symbols[first..last] { + self.code_sizes[table_num][symbol.sym_index as usize] = i as u8; + } + last = first; + } + } + + let mut j = 0; + next_code[1] = 0; + for i in 2..=code_size_limit { + j = (j + num_codes[i - 1]) << 1; + next_code[i] = j as u32; + } + + for (&code_size, huff_code) in self.code_sizes[table_num] + .iter() + .take(table_len) + .zip(self.codes[table_num].iter_mut().take(table_len)) + { + if code_size == 0 { + continue; + } + + let mut code = next_code[code_size as usize]; + next_code[code_size as usize] += 1; + + let mut rev_code = 0; + for _ in 0..code_size { + rev_code = (rev_code << 1) | (code & 1); + code >>= 1; + } + *huff_code = rev_code as u16; + } + } + + fn start_static_block(&mut self, output: &mut OutputBufferOxide) { + memset(&mut self.code_sizes[LITLEN_TABLE][0..144], 8); + memset(&mut self.code_sizes[LITLEN_TABLE][144..256], 9); + memset(&mut self.code_sizes[LITLEN_TABLE][256..280], 7); + memset(&mut self.code_sizes[LITLEN_TABLE][280..288], 8); + + memset(&mut self.code_sizes[DIST_TABLE][..32], 5); + + self.optimize_table(LITLEN_TABLE, 288, 15, true); + self.optimize_table(DIST_TABLE, 32, 15, true); + + output.put_bits(0b01, 2) + } + + fn start_dynamic_block(&mut self, output: &mut OutputBufferOxide) -> Result<()> { + // There will always be one, and only one end of block code. + self.count[0][256] = 1; + + self.optimize_table(0, MAX_HUFF_SYMBOLS_0, 15, false); + self.optimize_table(1, MAX_HUFF_SYMBOLS_1, 15, false); + + let num_lit_codes = 286 + - &self.code_sizes[0][257..286] + .iter() + .rev() + .take_while(|&x| *x == 0) + .count(); + + let num_dist_codes = 30 + - &self.code_sizes[1][1..30] + .iter() + .rev() + .take_while(|&x| *x == 0) + .count(); + + let mut code_sizes_to_pack = [0u8; MAX_HUFF_SYMBOLS_0 + MAX_HUFF_SYMBOLS_1]; + let mut packed_code_sizes = [0u8; MAX_HUFF_SYMBOLS_0 + MAX_HUFF_SYMBOLS_1]; + + let total_code_sizes_to_pack = num_lit_codes + num_dist_codes; + + code_sizes_to_pack[..num_lit_codes].copy_from_slice(&self.code_sizes[0][..num_lit_codes]); + + code_sizes_to_pack[num_lit_codes..total_code_sizes_to_pack] + .copy_from_slice(&self.code_sizes[1][..num_dist_codes]); + + let mut rle = Rle { + z_count: 0, + repeat_count: 0, + prev_code_size: 0xFF, + }; + + memset(&mut self.count[HUFF_CODES_TABLE][..MAX_HUFF_SYMBOLS_2], 0); + + let mut packed_pos = 0; + for &code_size in &code_sizes_to_pack[..total_code_sizes_to_pack] { + if code_size == 0 { + rle.prev_code_size(&mut packed_code_sizes, &mut packed_pos, self)?; + rle.z_count += 1; + if rle.z_count == 138 { + rle.zero_code_size(&mut packed_code_sizes, &mut packed_pos, self)?; + } + } else { + rle.zero_code_size(&mut packed_code_sizes, &mut packed_pos, self)?; + if code_size != rle.prev_code_size { + rle.prev_code_size(&mut packed_code_sizes, &mut packed_pos, self)?; + self.count[HUFF_CODES_TABLE][code_size as usize] = + self.count[HUFF_CODES_TABLE][code_size as usize].wrapping_add(1); + write(&[code_size], &mut packed_code_sizes, &mut packed_pos)?; + } else { + rle.repeat_count += 1; + if rle.repeat_count == 6 { + rle.prev_code_size(&mut packed_code_sizes, &mut packed_pos, self)?; + } + } + } + rle.prev_code_size = code_size; + } + + if rle.repeat_count != 0 { + rle.prev_code_size(&mut packed_code_sizes, &mut packed_pos, self)?; + } else { + rle.zero_code_size(&mut packed_code_sizes, &mut packed_pos, self)?; + } + + self.optimize_table(2, MAX_HUFF_SYMBOLS_2, 7, false); + + output.put_bits(2, 2); + + output.put_bits((num_lit_codes - 257) as u32, 5); + output.put_bits((num_dist_codes - 1) as u32, 5); + + let mut num_bit_lengths = 18 + - HUFFMAN_LENGTH_ORDER + .iter() + .rev() + .take_while(|&swizzle| self.code_sizes[HUFF_CODES_TABLE][*swizzle as usize] == 0) + .count(); + + num_bit_lengths = cmp::max(4, num_bit_lengths + 1); + output.put_bits(num_bit_lengths as u32 - 4, 4); + for &swizzle in &HUFFMAN_LENGTH_ORDER[..num_bit_lengths] { + output.put_bits( + u32::from(self.code_sizes[HUFF_CODES_TABLE][swizzle as usize]), + 3, + ); + } + + let mut packed_code_size_index = 0; + while packed_code_size_index < packed_pos { + let code = packed_code_sizes[packed_code_size_index] as usize; + packed_code_size_index += 1; + assert!(code < MAX_HUFF_SYMBOLS_2); + output.put_bits( + u32::from(self.codes[HUFF_CODES_TABLE][code]), + u32::from(self.code_sizes[HUFF_CODES_TABLE][code]), + ); + if code >= 16 { + output.put_bits( + u32::from(packed_code_sizes[packed_code_size_index]), + [2, 3, 7][code - 16], + ); + packed_code_size_index += 1; + } + } + + Ok(()) + } +} + +struct DictOxide { + /// The maximum number of checks in the hash chain, for the initial, + /// and the lazy match respectively. + pub max_probes: [u32; 2], + /// Buffer of input data. + /// Padded with 1 byte to simplify matching code in `compress_fast`. + pub b: Box, + + pub code_buf_dict_pos: usize, + pub lookahead_size: usize, + pub lookahead_pos: usize, + pub size: usize, +} + +const fn probes_from_flags(flags: u32) -> [u32; 2] { + [ + 1 + ((flags & 0xFFF) + 2) / 3, + 1 + (((flags & 0xFFF) >> 2) + 2) / 3, + ] +} + +impl DictOxide { + fn new(flags: u32) -> Self { + DictOxide { + max_probes: probes_from_flags(flags), + b: Box::default(), + code_buf_dict_pos: 0, + lookahead_size: 0, + lookahead_pos: 0, + size: 0, + } + } + + fn update_flags(&mut self, flags: u32) { + self.max_probes = probes_from_flags(flags); + } + + fn reset(&mut self) { + self.b.reset(); + self.code_buf_dict_pos = 0; + self.lookahead_size = 0; + self.lookahead_pos = 0; + self.size = 0; + } + + /// Do an unaligned read of the data at `pos` in the dictionary and treat it as if it was of + /// type T. + #[inline] + fn read_unaligned_u32(&self, pos: usize) -> u32 { + // Masking the value here helps avoid bounds checks. + let pos = (pos & LZ_DICT_SIZE_MASK) as usize; + let end = pos + 4; + // Somehow this assertion makes things faster. + assert!(end < LZ_DICT_FULL_SIZE); + + let bytes: [u8; 4] = self.b.dict[pos..end].try_into().unwrap(); + u32::from_le_bytes(bytes) + } + + /// Do an unaligned read of the data at `pos` in the dictionary and treat it as if it was of + /// type T. + #[inline] + fn read_unaligned_u64(&self, pos: usize) -> u64 { + let pos = pos as usize; + let bytes: [u8; 8] = self.b.dict[pos..pos + 8].try_into().unwrap(); + u64::from_le_bytes(bytes) + } + + /// Do an unaligned read of the data at `pos` in the dictionary and treat it as if it was of + /// type T. + #[inline] + fn read_as_u16(&self, pos: usize) -> u16 { + read_u16_le(&self.b.dict[..], pos) + } + + /// Try to find a match for the data at lookahead_pos in the dictionary that is + /// longer than `match_len`. + /// Returns a tuple containing (match_distance, match_length). Will be equal to the input + /// values if no better matches were found. + fn find_match( + &self, + lookahead_pos: usize, + max_dist: usize, + max_match_len: u32, + mut match_dist: u32, + mut match_len: u32, + ) -> (u32, u32) { + // Clamp the match len and max_match_len to be valid. (It should be when this is called, but + // do it for now just in case for safety reasons.) + // This should normally end up as at worst conditional moves, + // so it shouldn't slow us down much. + // TODO: Statically verify these so we don't need to do this. + let max_match_len = cmp::min(MAX_MATCH_LEN as u32, max_match_len); + match_len = cmp::max(match_len, 1); + + let pos = lookahead_pos as usize & LZ_DICT_SIZE_MASK; + let mut probe_pos = pos; + // Number of probes into the hash chains. + let mut num_probes_left = self.max_probes[(match_len >= 32) as usize]; + + // If we already have a match of the full length don't bother searching for another one. + if max_match_len <= match_len { + return (match_dist, match_len); + } + + // Read the last byte of the current match, and the next one, used to compare matches. + let mut c01: u16 = self.read_as_u16(pos as usize + match_len as usize - 1); + // Read the two bytes at the end position of the current match. + let s01: u16 = self.read_as_u16(pos as usize); + + 'outer: loop { + let mut dist; + 'found: loop { + num_probes_left -= 1; + if num_probes_left == 0 { + // We have done as many probes in the hash chain as the current compression + // settings allow, so return the best match we found, if any. + return (match_dist, match_len); + } + + for _ in 0..3 { + let next_probe_pos = self.b.next[probe_pos as usize] as usize; + + dist = (lookahead_pos - next_probe_pos) & 0xFFFF; + if next_probe_pos == 0 || dist > max_dist { + // We reached the end of the hash chain, or the next value is further away + // than the maximum allowed distance, so return the best match we found, if + // any. + return (match_dist, match_len); + } + + // Mask the position value to get the position in the hash chain of the next + // position to match against. + probe_pos = next_probe_pos & LZ_DICT_SIZE_MASK; + + if self.read_as_u16((probe_pos + match_len as usize - 1) as usize) == c01 { + break 'found; + } + } + } + + if dist == 0 { + // We've looked through the whole match range, so return the best match we + // found. + return (match_dist, match_len); + } + + // Check if the two first bytes match. + if self.read_as_u16(probe_pos as usize) != s01 { + continue; + } + + let mut p = pos + 2; + let mut q = probe_pos + 2; + // The first two bytes matched, so check the full length of the match. + for _ in 0..32 { + let p_data: u64 = self.read_unaligned_u64(p); + let q_data: u64 = self.read_unaligned_u64(q); + // Compare of 8 bytes at a time by using unaligned loads of 64-bit integers. + let xor_data = p_data ^ q_data; + if xor_data == 0 { + p += 8; + q += 8; + } else { + // If not all of the last 8 bytes matched, check how may of them did. + let trailing = xor_data.trailing_zeros(); + + let probe_len = p - pos + (trailing as usize >> 3); + if probe_len > match_len as usize { + match_dist = dist as u32; + match_len = cmp::min(max_match_len, probe_len as u32); + if match_len == max_match_len { + // We found a match that had the maximum allowed length, + // so there is now point searching further. + return (match_dist, match_len); + } + // We found a better match, so save the last two bytes for further match + // comparisons. + c01 = self.read_as_u16(pos + match_len as usize - 1) + } + continue 'outer; + } + } + + return (dist as u32, cmp::min(max_match_len, MAX_MATCH_LEN as u32)); + } + } +} + +struct ParamsOxide { + pub flags: u32, + pub greedy_parsing: bool, + pub block_index: u32, + + pub saved_match_dist: u32, + pub saved_match_len: u32, + pub saved_lit: u8, + + pub flush: TDEFLFlush, + pub flush_ofs: u32, + pub flush_remaining: u32, + pub finished: bool, + + pub adler32: u32, + + pub src_pos: usize, + + pub out_buf_ofs: usize, + pub prev_return_status: TDEFLStatus, + + pub saved_bit_buffer: u32, + pub saved_bits_in: u32, + + pub local_buf: Box, +} + +impl ParamsOxide { + fn new(flags: u32) -> Self { + ParamsOxide { + flags, + greedy_parsing: flags & TDEFL_GREEDY_PARSING_FLAG != 0, + block_index: 0, + saved_match_dist: 0, + saved_match_len: 0, + saved_lit: 0, + flush: TDEFLFlush::None, + flush_ofs: 0, + flush_remaining: 0, + finished: false, + adler32: MZ_ADLER32_INIT, + src_pos: 0, + out_buf_ofs: 0, + prev_return_status: TDEFLStatus::Okay, + saved_bit_buffer: 0, + saved_bits_in: 0, + local_buf: Box::default(), + } + } + + fn update_flags(&mut self, flags: u32) { + self.flags = flags; + self.greedy_parsing = self.flags & TDEFL_GREEDY_PARSING_FLAG != 0; + } + + /// Reset state, saving settings. + fn reset(&mut self) { + self.block_index = 0; + self.saved_match_len = 0; + self.saved_match_dist = 0; + self.saved_lit = 0; + self.flush = TDEFLFlush::None; + self.flush_ofs = 0; + self.flush_remaining = 0; + self.finished = false; + self.adler32 = MZ_ADLER32_INIT; + self.src_pos = 0; + self.out_buf_ofs = 0; + self.prev_return_status = TDEFLStatus::Okay; + self.saved_bit_buffer = 0; + self.saved_bits_in = 0; + self.local_buf.b = [0; OUT_BUF_SIZE]; + } +} + +struct LZOxide { + pub codes: [u8; LZ_CODE_BUF_SIZE], + pub code_position: usize, + pub flag_position: usize, + + // The total number of bytes in the current block. + // (Could maybe use usize, but it's not possible to exceed a block size of ) + pub total_bytes: u32, + pub num_flags_left: u32, +} + +impl LZOxide { + const fn new() -> Self { + LZOxide { + codes: [0; LZ_CODE_BUF_SIZE], + code_position: 1, + flag_position: 0, + total_bytes: 0, + num_flags_left: 8, + } + } + + fn write_code(&mut self, val: u8) { + self.codes[self.code_position] = val; + self.code_position += 1; + } + + fn init_flag(&mut self) { + if self.num_flags_left == 8 { + *self.get_flag() = 0; + self.code_position -= 1; + } else { + *self.get_flag() >>= self.num_flags_left; + } + } + + fn get_flag(&mut self) -> &mut u8 { + &mut self.codes[self.flag_position] + } + + fn plant_flag(&mut self) { + self.flag_position = self.code_position; + self.code_position += 1; + } + + fn consume_flag(&mut self) { + self.num_flags_left -= 1; + if self.num_flags_left == 0 { + self.num_flags_left = 8; + self.plant_flag(); + } + } +} + +fn compress_lz_codes( + huff: &HuffmanOxide, + output: &mut OutputBufferOxide, + lz_code_buf: &[u8], +) -> Result { + let mut flags = 1; + let mut bb = BitBuffer { + bit_buffer: u64::from(output.bit_buffer), + bits_in: output.bits_in, + }; + + let mut i: usize = 0; + while i < lz_code_buf.len() { + if flags == 1 { + flags = u32::from(lz_code_buf[i]) | 0x100; + i += 1; + } + + // The lz code was a length code + if flags & 1 == 1 { + flags >>= 1; + + let sym; + let num_extra_bits; + + let match_len = lz_code_buf[i] as usize; + + let match_dist = read_u16_le(lz_code_buf, i + 1); + + i += 3; + + debug_assert!(huff.code_sizes[0][LEN_SYM[match_len] as usize] != 0); + bb.put_fast( + u64::from(huff.codes[0][LEN_SYM[match_len] as usize]), + u32::from(huff.code_sizes[0][LEN_SYM[match_len] as usize]), + ); + bb.put_fast( + match_len as u64 & u64::from(BITMASKS[LEN_EXTRA[match_len] as usize]), + u32::from(LEN_EXTRA[match_len]), + ); + + if match_dist < 512 { + sym = SMALL_DIST_SYM[match_dist as usize] as usize; + num_extra_bits = SMALL_DIST_EXTRA[match_dist as usize] as usize; + } else { + sym = LARGE_DIST_SYM[(match_dist >> 8) as usize] as usize; + num_extra_bits = LARGE_DIST_EXTRA[(match_dist >> 8) as usize] as usize; + } + + debug_assert!(huff.code_sizes[1][sym] != 0); + bb.put_fast( + u64::from(huff.codes[1][sym]), + u32::from(huff.code_sizes[1][sym]), + ); + bb.put_fast( + u64::from(match_dist) & u64::from(BITMASKS[num_extra_bits as usize]), + num_extra_bits as u32, + ); + } else { + // The lz code was a literal + for _ in 0..3 { + flags >>= 1; + let lit = lz_code_buf[i]; + i += 1; + + debug_assert!(huff.code_sizes[0][lit as usize] != 0); + bb.put_fast( + u64::from(huff.codes[0][lit as usize]), + u32::from(huff.code_sizes[0][lit as usize]), + ); + + if flags & 1 == 1 || i >= lz_code_buf.len() { + break; + } + } + } + + bb.flush(output)?; + } + + output.bits_in = 0; + output.bit_buffer = 0; + while bb.bits_in != 0 { + let n = cmp::min(bb.bits_in, 16); + output.put_bits(bb.bit_buffer as u32 & BITMASKS[n as usize], n); + bb.bit_buffer >>= n; + bb.bits_in -= n; + } + + // Output the end of block symbol. + output.put_bits( + u32::from(huff.codes[0][256]), + u32::from(huff.code_sizes[0][256]), + ); + + Ok(true) +} + +fn compress_block( + huff: &mut HuffmanOxide, + output: &mut OutputBufferOxide, + lz: &LZOxide, + static_block: bool, +) -> Result { + if static_block { + huff.start_static_block(output); + } else { + huff.start_dynamic_block(output)?; + } + + compress_lz_codes(huff, output, &lz.codes[..lz.code_position]) +} + +fn flush_block( + d: &mut CompressorOxide, + callback: &mut CallbackOxide, + flush: TDEFLFlush, +) -> Result { + let mut saved_buffer; + { + let mut output = callback + .out + .new_output_buffer(&mut d.params.local_buf.b, d.params.out_buf_ofs); + output.bit_buffer = d.params.saved_bit_buffer; + output.bits_in = d.params.saved_bits_in; + + let use_raw_block = (d.params.flags & TDEFL_FORCE_ALL_RAW_BLOCKS != 0) + && (d.dict.lookahead_pos - d.dict.code_buf_dict_pos) <= d.dict.size; + + assert!(d.params.flush_remaining == 0); + d.params.flush_ofs = 0; + d.params.flush_remaining = 0; + + d.lz.init_flag(); + + // If we are at the start of the stream, write the zlib header if requested. + if d.params.flags & TDEFL_WRITE_ZLIB_HEADER != 0 && d.params.block_index == 0 { + let header = zlib::header_from_flags(d.params.flags as u32); + output.put_bits(header[0].into(), 8); + output.put_bits(header[1].into(), 8); + } + + // Output the block header. + output.put_bits((flush == TDEFLFlush::Finish) as u32, 1); + + saved_buffer = output.save(); + + let comp_success = if !use_raw_block { + let use_static = + (d.params.flags & TDEFL_FORCE_ALL_STATIC_BLOCKS != 0) || (d.lz.total_bytes < 48); + compress_block(&mut d.huff, &mut output, &d.lz, use_static)? + } else { + false + }; + + // If we failed to compress anything and the output would take up more space than the output + // data, output a stored block instead, which has at most 5 bytes of overhead. + // We only use some simple heuristics for now. + // A stored block will have an overhead of at least 4 bytes containing the block length + // but usually more due to the length parameters having to start at a byte boundary and thus + // requiring up to 5 bytes of padding. + // As a static block will have an overhead of at most 1 bit per byte + // (as literals are either 8 or 9 bytes), a raw block will + // never take up less space if the number of input bytes are less than 32. + let expanded = (d.lz.total_bytes > 32) + && (output.inner_pos - saved_buffer.pos + 1 >= (d.lz.total_bytes as usize)) + && (d.dict.lookahead_pos - d.dict.code_buf_dict_pos <= d.dict.size); + + if use_raw_block || expanded { + output.load(saved_buffer); + + // Block header. + output.put_bits(0, 2); + + // Block length has to start on a byte boundary, s opad. + output.pad_to_bytes(); + + // Block length and ones complement of block length. + output.put_bits(d.lz.total_bytes & 0xFFFF, 16); + output.put_bits(!d.lz.total_bytes & 0xFFFF, 16); + + // Write the actual bytes. + for i in 0..d.lz.total_bytes { + let pos = (d.dict.code_buf_dict_pos + i as usize) & LZ_DICT_SIZE_MASK; + output.put_bits(u32::from(d.dict.b.dict[pos as usize]), 8); + } + } else if !comp_success { + output.load(saved_buffer); + compress_block(&mut d.huff, &mut output, &d.lz, true)?; + } + + if flush != TDEFLFlush::None { + if flush == TDEFLFlush::Finish { + output.pad_to_bytes(); + if d.params.flags & TDEFL_WRITE_ZLIB_HEADER != 0 { + let mut adler = d.params.adler32; + for _ in 0..4 { + output.put_bits((adler >> 24) & 0xFF, 8); + adler <<= 8; + } + } + } else { + // Sync or Full flush. + // Output an empty raw block. + output.put_bits(0, 3); + output.pad_to_bytes(); + output.put_bits(0, 16); + output.put_bits(0xFFFF, 16); + } + } + + memset(&mut d.huff.count[0][..MAX_HUFF_SYMBOLS_0], 0); + memset(&mut d.huff.count[1][..MAX_HUFF_SYMBOLS_1], 0); + + d.lz.code_position = 1; + d.lz.flag_position = 0; + d.lz.num_flags_left = 8; + d.dict.code_buf_dict_pos += d.lz.total_bytes as usize; + d.lz.total_bytes = 0; + d.params.block_index += 1; + + saved_buffer = output.save(); + + d.params.saved_bit_buffer = saved_buffer.bit_buffer; + d.params.saved_bits_in = saved_buffer.bits_in; + } + + Ok(callback.flush_output(saved_buffer, &mut d.params)) +} + +fn record_literal(h: &mut HuffmanOxide, lz: &mut LZOxide, lit: u8) { + lz.total_bytes += 1; + lz.write_code(lit); + + *lz.get_flag() >>= 1; + lz.consume_flag(); + + h.count[0][lit as usize] += 1; +} + +fn record_match(h: &mut HuffmanOxide, lz: &mut LZOxide, mut match_len: u32, mut match_dist: u32) { + assert!(match_len >= MIN_MATCH_LEN.into()); + assert!(match_dist >= 1); + assert!(match_dist as usize <= LZ_DICT_SIZE); + + lz.total_bytes += match_len; + match_dist -= 1; + match_len -= u32::from(MIN_MATCH_LEN); + lz.write_code(match_len as u8); + lz.write_code(match_dist as u8); + lz.write_code((match_dist >> 8) as u8); + + *lz.get_flag() >>= 1; + *lz.get_flag() |= 0x80; + lz.consume_flag(); + + let symbol = if match_dist < 512 { + SMALL_DIST_SYM[match_dist as usize] + } else { + LARGE_DIST_SYM[((match_dist >> 8) & 127) as usize] + } as usize; + h.count[1][symbol] += 1; + h.count[0][LEN_SYM[match_len as usize] as usize] += 1; +} + +fn compress_normal(d: &mut CompressorOxide, callback: &mut CallbackOxide) -> bool { + let mut src_pos = d.params.src_pos; + let in_buf = match callback.in_buf { + None => return true, + Some(in_buf) => in_buf, + }; + + let mut lookahead_size = d.dict.lookahead_size; + let mut lookahead_pos = d.dict.lookahead_pos; + let mut saved_lit = d.params.saved_lit; + let mut saved_match_dist = d.params.saved_match_dist; + let mut saved_match_len = d.params.saved_match_len; + + while src_pos < in_buf.len() || (d.params.flush != TDEFLFlush::None && lookahead_size != 0) { + let src_buf_left = in_buf.len() - src_pos; + let num_bytes_to_process = cmp::min(src_buf_left, MAX_MATCH_LEN - lookahead_size as usize); + + if lookahead_size + d.dict.size >= usize::from(MIN_MATCH_LEN) - 1 + && num_bytes_to_process > 0 + { + let dictb = &mut d.dict.b; + + let mut dst_pos = (lookahead_pos + lookahead_size as usize) & LZ_DICT_SIZE_MASK; + let mut ins_pos = lookahead_pos + lookahead_size as usize - 2; + // Start the hash value from the first two bytes + let mut hash = update_hash( + u16::from(dictb.dict[(ins_pos & LZ_DICT_SIZE_MASK) as usize]), + dictb.dict[((ins_pos + 1) & LZ_DICT_SIZE_MASK) as usize], + ); + + lookahead_size += num_bytes_to_process; + + for &c in &in_buf[src_pos..src_pos + num_bytes_to_process] { + // Add byte to input buffer. + dictb.dict[dst_pos as usize] = c; + if (dst_pos as usize) < MAX_MATCH_LEN - 1 { + dictb.dict[LZ_DICT_SIZE + dst_pos as usize] = c; + } + + // Generate hash from the current byte, + hash = update_hash(hash, c); + dictb.next[(ins_pos & LZ_DICT_SIZE_MASK) as usize] = dictb.hash[hash as usize]; + // and insert it into the hash chain. + dictb.hash[hash as usize] = ins_pos as u16; + dst_pos = (dst_pos + 1) & LZ_DICT_SIZE_MASK; + ins_pos += 1; + } + src_pos += num_bytes_to_process; + } else { + let dictb = &mut d.dict.b; + for &c in &in_buf[src_pos..src_pos + num_bytes_to_process] { + let dst_pos = (lookahead_pos + lookahead_size) & LZ_DICT_SIZE_MASK; + dictb.dict[dst_pos as usize] = c; + if (dst_pos as usize) < MAX_MATCH_LEN - 1 { + dictb.dict[LZ_DICT_SIZE + dst_pos as usize] = c; + } + + lookahead_size += 1; + if lookahead_size + d.dict.size >= MIN_MATCH_LEN.into() { + let ins_pos = lookahead_pos + lookahead_size - 3; + let hash = ((u32::from(dictb.dict[(ins_pos & LZ_DICT_SIZE_MASK) as usize]) + << (LZ_HASH_SHIFT * 2)) + ^ ((u32::from(dictb.dict[((ins_pos + 1) & LZ_DICT_SIZE_MASK) as usize]) + << LZ_HASH_SHIFT) + ^ u32::from(c))) + & (LZ_HASH_SIZE as u32 - 1); + + dictb.next[(ins_pos & LZ_DICT_SIZE_MASK) as usize] = dictb.hash[hash as usize]; + dictb.hash[hash as usize] = ins_pos as u16; + } + } + + src_pos += num_bytes_to_process; + } + + d.dict.size = cmp::min(LZ_DICT_SIZE - lookahead_size, d.dict.size); + if d.params.flush == TDEFLFlush::None && (lookahead_size as usize) < MAX_MATCH_LEN { + break; + } + + let mut len_to_move = 1; + let mut cur_match_dist = 0; + let mut cur_match_len = if saved_match_len != 0 { + saved_match_len + } else { + u32::from(MIN_MATCH_LEN) - 1 + }; + let cur_pos = lookahead_pos & LZ_DICT_SIZE_MASK; + if d.params.flags & (TDEFL_RLE_MATCHES | TDEFL_FORCE_ALL_RAW_BLOCKS) != 0 { + // If TDEFL_RLE_MATCHES is set, we only look for repeating sequences of the current byte. + if d.dict.size != 0 && d.params.flags & TDEFL_FORCE_ALL_RAW_BLOCKS == 0 { + let c = d.dict.b.dict[((cur_pos.wrapping_sub(1)) & LZ_DICT_SIZE_MASK) as usize]; + cur_match_len = d.dict.b.dict[cur_pos as usize..(cur_pos + lookahead_size) as usize] + .iter() + .take_while(|&x| *x == c) + .count() as u32; + if cur_match_len < MIN_MATCH_LEN.into() { + cur_match_len = 0 + } else { + cur_match_dist = 1 + } + } + } else { + // Try to find a match for the bytes at the current position. + let dist_len = d.dict.find_match( + lookahead_pos, + d.dict.size, + lookahead_size as u32, + cur_match_dist, + cur_match_len, + ); + cur_match_dist = dist_len.0; + cur_match_len = dist_len.1; + } + + let far_and_small = cur_match_len == MIN_MATCH_LEN.into() && cur_match_dist >= 8 * 1024; + let filter_small = d.params.flags & TDEFL_FILTER_MATCHES != 0 && cur_match_len <= 5; + if far_and_small || filter_small || cur_pos == cur_match_dist as usize { + cur_match_dist = 0; + cur_match_len = 0; + } + + if saved_match_len != 0 { + if cur_match_len > saved_match_len { + record_literal(&mut d.huff, &mut d.lz, saved_lit); + if cur_match_len >= 128 { + record_match(&mut d.huff, &mut d.lz, cur_match_len, cur_match_dist); + saved_match_len = 0; + len_to_move = cur_match_len as usize; + } else { + saved_lit = d.dict.b.dict[cur_pos as usize]; + saved_match_dist = cur_match_dist; + saved_match_len = cur_match_len; + } + } else { + record_match(&mut d.huff, &mut d.lz, saved_match_len, saved_match_dist); + len_to_move = (saved_match_len - 1) as usize; + saved_match_len = 0; + } + } else if cur_match_dist == 0 { + record_literal( + &mut d.huff, + &mut d.lz, + d.dict.b.dict[cmp::min(cur_pos as usize, d.dict.b.dict.len() - 1)], + ); + } else if d.params.greedy_parsing + || (d.params.flags & TDEFL_RLE_MATCHES != 0) + || cur_match_len >= 128 + { + // If we are using lazy matching, check for matches at the next byte if the current + // match was shorter than 128 bytes. + record_match(&mut d.huff, &mut d.lz, cur_match_len, cur_match_dist); + len_to_move = cur_match_len as usize; + } else { + saved_lit = d.dict.b.dict[cmp::min(cur_pos as usize, d.dict.b.dict.len() - 1)]; + saved_match_dist = cur_match_dist; + saved_match_len = cur_match_len; + } + + lookahead_pos += len_to_move; + assert!(lookahead_size >= len_to_move); + lookahead_size -= len_to_move; + d.dict.size = cmp::min(d.dict.size + len_to_move, LZ_DICT_SIZE); + + let lz_buf_tight = d.lz.code_position > LZ_CODE_BUF_SIZE - 8; + let raw = d.params.flags & TDEFL_FORCE_ALL_RAW_BLOCKS != 0; + let fat = ((d.lz.code_position * 115) >> 7) >= d.lz.total_bytes as usize; + let fat_or_raw = (d.lz.total_bytes > 31 * 1024) && (fat || raw); + + if lz_buf_tight || fat_or_raw { + d.params.src_pos = src_pos; + // These values are used in flush_block, so we need to write them back here. + d.dict.lookahead_size = lookahead_size; + d.dict.lookahead_pos = lookahead_pos; + + let n = flush_block(d, callback, TDEFLFlush::None) + .unwrap_or(TDEFLStatus::PutBufFailed as i32); + if n != 0 { + d.params.saved_lit = saved_lit; + d.params.saved_match_dist = saved_match_dist; + d.params.saved_match_len = saved_match_len; + return n > 0; + } + } + } + + d.params.src_pos = src_pos; + d.dict.lookahead_size = lookahead_size; + d.dict.lookahead_pos = lookahead_pos; + d.params.saved_lit = saved_lit; + d.params.saved_match_dist = saved_match_dist; + d.params.saved_match_len = saved_match_len; + true +} + +const COMP_FAST_LOOKAHEAD_SIZE: usize = 4096; + +fn compress_fast(d: &mut CompressorOxide, callback: &mut CallbackOxide) -> bool { + let mut src_pos = d.params.src_pos; + let mut lookahead_size = d.dict.lookahead_size; + let mut lookahead_pos = d.dict.lookahead_pos; + + let mut cur_pos = lookahead_pos & LZ_DICT_SIZE_MASK; + let in_buf = match callback.in_buf { + None => return true, + Some(in_buf) => in_buf, + }; + + debug_assert!(d.lz.code_position < LZ_CODE_BUF_SIZE - 2); + + while src_pos < in_buf.len() || (d.params.flush != TDEFLFlush::None && lookahead_size > 0) { + let mut dst_pos = ((lookahead_pos + lookahead_size) & LZ_DICT_SIZE_MASK) as usize; + let mut num_bytes_to_process = cmp::min( + in_buf.len() - src_pos, + (COMP_FAST_LOOKAHEAD_SIZE - lookahead_size) as usize, + ); + lookahead_size += num_bytes_to_process; + + while num_bytes_to_process != 0 { + let n = cmp::min(LZ_DICT_SIZE - dst_pos, num_bytes_to_process); + d.dict.b.dict[dst_pos..dst_pos + n].copy_from_slice(&in_buf[src_pos..src_pos + n]); + + if dst_pos < MAX_MATCH_LEN - 1 { + let m = cmp::min(n, MAX_MATCH_LEN - 1 - dst_pos); + d.dict.b.dict[dst_pos + LZ_DICT_SIZE..dst_pos + LZ_DICT_SIZE + m] + .copy_from_slice(&in_buf[src_pos..src_pos + m]); + } + + src_pos += n; + dst_pos = (dst_pos + n) & LZ_DICT_SIZE_MASK as usize; + num_bytes_to_process -= n; + } + + d.dict.size = cmp::min(LZ_DICT_SIZE - lookahead_size, d.dict.size); + if d.params.flush == TDEFLFlush::None && lookahead_size < COMP_FAST_LOOKAHEAD_SIZE { + break; + } + + while lookahead_size >= 4 { + let mut cur_match_len = 1; + + let first_trigram = d.dict.read_unaligned_u32(cur_pos) & 0xFF_FFFF; + + let hash = (first_trigram ^ (first_trigram >> (24 - (LZ_HASH_BITS - 8)))) + & LEVEL1_HASH_SIZE_MASK; + + let mut probe_pos = usize::from(d.dict.b.hash[hash as usize]); + d.dict.b.hash[hash as usize] = lookahead_pos as u16; + + let mut cur_match_dist = (lookahead_pos - probe_pos as usize) as u16; + if cur_match_dist as usize <= d.dict.size { + probe_pos &= LZ_DICT_SIZE_MASK; + + let trigram = d.dict.read_unaligned_u32(probe_pos) & 0xFF_FFFF; + + if first_trigram == trigram { + // Trigram was tested, so we can start with "+ 3" displacement. + let mut p = cur_pos + 3; + let mut q = probe_pos + 3; + cur_match_len = (|| { + for _ in 0..32 { + let p_data: u64 = d.dict.read_unaligned_u64(p); + let q_data: u64 = d.dict.read_unaligned_u64(q); + let xor_data = p_data ^ q_data; + if xor_data == 0 { + p += 8; + q += 8; + } else { + let trailing = xor_data.trailing_zeros(); + return p as u32 - cur_pos as u32 + (trailing >> 3); + } + } + + if cur_match_dist == 0 { + 0 + } else { + MAX_MATCH_LEN as u32 + } + })(); + + if cur_match_len < MIN_MATCH_LEN.into() + || (cur_match_len == MIN_MATCH_LEN.into() && cur_match_dist >= 8 * 1024) + { + let lit = first_trigram as u8; + cur_match_len = 1; + d.lz.write_code(lit); + *d.lz.get_flag() >>= 1; + d.huff.count[0][lit as usize] += 1; + } else { + // Limit the match to the length of the lookahead so we don't create a match + // that ends after the end of the input data. + cur_match_len = cmp::min(cur_match_len, lookahead_size as u32); + debug_assert!(cur_match_len >= MIN_MATCH_LEN.into()); + debug_assert!(cur_match_dist >= 1); + debug_assert!(cur_match_dist as usize <= LZ_DICT_SIZE); + cur_match_dist -= 1; + + d.lz.write_code((cur_match_len - u32::from(MIN_MATCH_LEN)) as u8); + d.lz.write_code(cur_match_dist as u8); + d.lz.write_code((cur_match_dist >> 8) as u8); + + *d.lz.get_flag() >>= 1; + *d.lz.get_flag() |= 0x80; + if cur_match_dist < 512 { + d.huff.count[1][SMALL_DIST_SYM[cur_match_dist as usize] as usize] += 1; + } else { + d.huff.count[1] + [LARGE_DIST_SYM[(cur_match_dist >> 8) as usize] as usize] += 1; + } + + d.huff.count[0][LEN_SYM[(cur_match_len - u32::from(MIN_MATCH_LEN)) as usize] + as usize] += 1; + } + } else { + d.lz.write_code(first_trigram as u8); + *d.lz.get_flag() >>= 1; + d.huff.count[0][first_trigram as u8 as usize] += 1; + } + + d.lz.consume_flag(); + d.lz.total_bytes += cur_match_len; + lookahead_pos += cur_match_len as usize; + d.dict.size = cmp::min(d.dict.size + cur_match_len as usize, LZ_DICT_SIZE); + cur_pos = (cur_pos + cur_match_len as usize) & LZ_DICT_SIZE_MASK; + lookahead_size -= cur_match_len as usize; + + if d.lz.code_position > LZ_CODE_BUF_SIZE - 8 { + // These values are used in flush_block, so we need to write them back here. + d.dict.lookahead_size = lookahead_size; + d.dict.lookahead_pos = lookahead_pos; + + let n = match flush_block(d, callback, TDEFLFlush::None) { + Err(_) => { + d.params.src_pos = src_pos; + d.params.prev_return_status = TDEFLStatus::PutBufFailed; + return false; + } + Ok(status) => status, + }; + if n != 0 { + d.params.src_pos = src_pos; + return n > 0; + } + debug_assert!(d.lz.code_position < LZ_CODE_BUF_SIZE - 2); + + lookahead_size = d.dict.lookahead_size; + lookahead_pos = d.dict.lookahead_pos; + } + } + } + + while lookahead_size != 0 { + let lit = d.dict.b.dict[cur_pos as usize]; + d.lz.total_bytes += 1; + d.lz.write_code(lit); + *d.lz.get_flag() >>= 1; + d.lz.consume_flag(); + + d.huff.count[0][lit as usize] += 1; + lookahead_pos += 1; + d.dict.size = cmp::min(d.dict.size + 1, LZ_DICT_SIZE); + cur_pos = (cur_pos + 1) & LZ_DICT_SIZE_MASK; + lookahead_size -= 1; + + if d.lz.code_position > LZ_CODE_BUF_SIZE - 8 { + // These values are used in flush_block, so we need to write them back here. + d.dict.lookahead_size = lookahead_size; + d.dict.lookahead_pos = lookahead_pos; + + let n = match flush_block(d, callback, TDEFLFlush::None) { + Err(_) => { + d.params.prev_return_status = TDEFLStatus::PutBufFailed; + d.params.src_pos = src_pos; + return false; + } + Ok(status) => status, + }; + if n != 0 { + d.params.src_pos = src_pos; + return n > 0; + } + + lookahead_size = d.dict.lookahead_size; + lookahead_pos = d.dict.lookahead_pos; + } + } + } + + d.params.src_pos = src_pos; + d.dict.lookahead_size = lookahead_size; + d.dict.lookahead_pos = lookahead_pos; + true +} + +fn flush_output_buffer(c: &mut CallbackOxide, p: &mut ParamsOxide) -> (TDEFLStatus, usize, usize) { + let mut res = (TDEFLStatus::Okay, p.src_pos, 0); + if let CallbackOut::Buf(ref mut cb) = c.out { + let n = cmp::min(cb.out_buf.len() - p.out_buf_ofs, p.flush_remaining as usize); + if n != 0 { + (&mut cb.out_buf[p.out_buf_ofs..p.out_buf_ofs + n]) + .copy_from_slice(&p.local_buf.b[p.flush_ofs as usize..p.flush_ofs as usize + n]); + } + p.flush_ofs += n as u32; + p.flush_remaining -= n as u32; + p.out_buf_ofs += n; + res.2 = p.out_buf_ofs; + } + + if p.finished && p.flush_remaining == 0 { + res.0 = TDEFLStatus::Done + } + res +} + +/// Main compression function. Tries to compress as much as possible from `in_buf` and +/// puts compressed output into `out_buf`. +/// +/// The value of `flush` determines if the compressor should attempt to flush all output +/// and alternatively try to finish the stream. +/// +/// Use [`TDEFLFlush::Finish`] on the final call to signal that the stream is finishing. +/// +/// Note that this function does not keep track of whether a flush marker has been output, so +/// if called using [`TDEFLFlush::Sync`], the caller needs to ensure there is enough space in the +/// output buffer if they want to avoid repeated flush markers. +/// See #105 for details. +/// +/// # Returns +/// Returns a tuple containing the current status of the compressor, the current position +/// in the input buffer and the current position in the output buffer. +pub fn compress( + d: &mut CompressorOxide, + in_buf: &[u8], + out_buf: &mut [u8], + flush: TDEFLFlush, +) -> (TDEFLStatus, usize, usize) { + compress_inner( + d, + &mut CallbackOxide::new_callback_buf(in_buf, out_buf), + flush, + ) +} + +/// Main compression function. Callbacks output. +/// +/// # Returns +/// Returns a tuple containing the current status of the compressor, the current position +/// in the input buffer. +/// +/// The caller is responsible for ensuring the `CallbackFunc` struct will not cause undefined +/// behaviour. +pub fn compress_to_output( + d: &mut CompressorOxide, + in_buf: &[u8], + flush: TDEFLFlush, + mut callback_func: impl FnMut(&[u8]) -> bool, +) -> (TDEFLStatus, usize) { + let res = compress_inner( + d, + &mut CallbackOxide::new_callback_func( + in_buf, + CallbackFunc { + put_buf_func: &mut callback_func, + }, + ), + flush, + ); + + (res.0, res.1) +} + +fn compress_inner( + d: &mut CompressorOxide, + callback: &mut CallbackOxide, + flush: TDEFLFlush, +) -> (TDEFLStatus, usize, usize) { + d.params.out_buf_ofs = 0; + d.params.src_pos = 0; + + let prev_ok = d.params.prev_return_status == TDEFLStatus::Okay; + let flush_finish_once = d.params.flush != TDEFLFlush::Finish || flush == TDEFLFlush::Finish; + + d.params.flush = flush; + if !prev_ok || !flush_finish_once { + d.params.prev_return_status = TDEFLStatus::BadParam; + return (d.params.prev_return_status, 0, 0); + } + + if d.params.flush_remaining != 0 || d.params.finished { + let res = flush_output_buffer(callback, &mut d.params); + d.params.prev_return_status = res.0; + return res; + } + + let one_probe = d.params.flags & MAX_PROBES_MASK as u32 == 1; + let greedy = d.params.flags & TDEFL_GREEDY_PARSING_FLAG != 0; + let filter_or_rle_or_raw = d.params.flags + & (TDEFL_FILTER_MATCHES | TDEFL_FORCE_ALL_RAW_BLOCKS | TDEFL_RLE_MATCHES) + != 0; + + let compress_success = if one_probe && greedy && !filter_or_rle_or_raw { + compress_fast(d, callback) + } else { + compress_normal(d, callback) + }; + + if !compress_success { + return ( + d.params.prev_return_status, + d.params.src_pos, + d.params.out_buf_ofs, + ); + } + + if let Some(in_buf) = callback.in_buf { + if d.params.flags & (TDEFL_WRITE_ZLIB_HEADER | TDEFL_COMPUTE_ADLER32) != 0 { + d.params.adler32 = update_adler32(d.params.adler32, &in_buf[..d.params.src_pos]); + } + } + + let flush_none = d.params.flush == TDEFLFlush::None; + let in_left = callback.in_buf.map_or(0, |buf| buf.len()) - d.params.src_pos; + let remaining = in_left != 0 || d.params.flush_remaining != 0; + if !flush_none && d.dict.lookahead_size == 0 && !remaining { + let flush = d.params.flush; + match flush_block(d, callback, flush) { + Err(_) => { + d.params.prev_return_status = TDEFLStatus::PutBufFailed; + return ( + d.params.prev_return_status, + d.params.src_pos, + d.params.out_buf_ofs, + ); + } + Ok(x) if x < 0 => { + return ( + d.params.prev_return_status, + d.params.src_pos, + d.params.out_buf_ofs, + ) + } + _ => { + d.params.finished = d.params.flush == TDEFLFlush::Finish; + if d.params.flush == TDEFLFlush::Full { + memset(&mut d.dict.b.hash[..], 0); + memset(&mut d.dict.b.next[..], 0); + d.dict.size = 0; + } + } + } + } + + let res = flush_output_buffer(callback, &mut d.params); + d.params.prev_return_status = res.0; + + res +} + +/// Create a set of compression flags using parameters used by zlib and other compressors. +/// Mainly intended for use with transition from c libraries as it deals with raw integers. +/// +/// # Parameters +/// `level` determines compression level. Clamped to maximum of 10. Negative values result in +/// `CompressionLevel::DefaultLevel`. +/// `window_bits`: Above 0, wraps the stream in a zlib wrapper, 0 or negative for a raw deflate +/// stream. +/// `strategy`: Sets the strategy if this conforms to any of the values in `CompressionStrategy`. +/// +/// # Notes +/// This function may be removed or moved to the `miniz_oxide_c_api` in the future. +pub fn create_comp_flags_from_zip_params(level: i32, window_bits: i32, strategy: i32) -> u32 { + let num_probes = (if level >= 0 { + cmp::min(10, level) + } else { + CompressionLevel::DefaultLevel as i32 + }) as usize; + let greedy = if level <= 3 { + TDEFL_GREEDY_PARSING_FLAG + } else { + 0 + }; + let mut comp_flags = NUM_PROBES[num_probes] | greedy; + + if window_bits > 0 { + comp_flags |= TDEFL_WRITE_ZLIB_HEADER; + } + + if level == 0 { + comp_flags |= TDEFL_FORCE_ALL_RAW_BLOCKS; + } else if strategy == CompressionStrategy::Filtered as i32 { + comp_flags |= TDEFL_FILTER_MATCHES; + } else if strategy == CompressionStrategy::HuffmanOnly as i32 { + comp_flags &= !MAX_PROBES_MASK as u32; + } else if strategy == CompressionStrategy::Fixed as i32 { + comp_flags |= TDEFL_FORCE_ALL_STATIC_BLOCKS; + } else if strategy == CompressionStrategy::RLE as i32 { + comp_flags |= TDEFL_RLE_MATCHES; + } + + comp_flags +} + +#[cfg(test)] +mod test { + use super::{ + compress_to_output, create_comp_flags_from_zip_params, read_u16_le, write_u16_le, + CompressionStrategy, CompressorOxide, TDEFLFlush, TDEFLStatus, DEFAULT_FLAGS, + MZ_DEFAULT_WINDOW_BITS, + }; + use crate::inflate::decompress_to_vec; + use alloc::vec; + + #[test] + fn u16_to_slice() { + let mut slice = [0, 0]; + write_u16_le(2000, &mut slice, 0); + assert_eq!(slice, [208, 7]); + } + + #[test] + fn u16_from_slice() { + let mut slice = [208, 7]; + assert_eq!(read_u16_le(&mut slice, 0), 2000); + } + + #[test] + fn compress_output() { + assert_eq!( + DEFAULT_FLAGS, + create_comp_flags_from_zip_params( + 4, + MZ_DEFAULT_WINDOW_BITS, + CompressionStrategy::Default as i32 + ) + ); + + let slice = [ + 1, 2, 3, 4, 1, 2, 3, 1, 2, 3, 1, 2, 6, 1, 2, 3, 1, 2, 3, 2, 3, 1, 2, 3, + ]; + let mut encoded = vec![]; + let flags = create_comp_flags_from_zip_params(6, 0, 0); + let mut d = CompressorOxide::new(flags); + let (status, in_consumed) = + compress_to_output(&mut d, &slice, TDEFLFlush::Finish, |out: &[u8]| { + encoded.extend_from_slice(out); + true + }); + + assert_eq!(status, TDEFLStatus::Done); + assert_eq!(in_consumed, slice.len()); + + let decoded = decompress_to_vec(&encoded[..]).unwrap(); + assert_eq!(&decoded[..], &slice[..]); + } + + #[test] + /// Check fast compress mode + fn compress_fast() { + let slice = [ + 1, 2, 3, 4, 1, 2, 3, 1, 2, 3, 1, 2, 6, 1, 2, 3, 1, 2, 3, 2, 3, 1, 2, 3, + ]; + let mut encoded = vec![]; + let flags = create_comp_flags_from_zip_params(1, 0, 0); + let mut d = CompressorOxide::new(flags); + let (status, in_consumed) = + compress_to_output(&mut d, &slice, TDEFLFlush::Finish, |out: &[u8]| { + encoded.extend_from_slice(out); + true + }); + + assert_eq!(status, TDEFLStatus::Done); + assert_eq!(in_consumed, slice.len()); + + // Needs to be altered if algorithm improves. + assert_eq!( + &encoded[..], + [99, 100, 98, 102, 1, 98, 48, 98, 3, 147, 204, 76, 204, 140, 76, 204, 0] + ); + + let decoded = decompress_to_vec(&encoded[..]).unwrap(); + assert_eq!(&decoded[..], &slice[..]); + } +} diff --git a/crux-mir/lib/miniz_oxide/src/deflate/mod.rs b/crux-mir/lib/miniz_oxide/src/deflate/mod.rs new file mode 100644 index 000000000..471b94b9d --- /dev/null +++ b/crux-mir/lib/miniz_oxide/src/deflate/mod.rs @@ -0,0 +1,227 @@ +//! This module contains functionality for compression. + +use crate::alloc::vec; +use crate::alloc::vec::Vec; + +mod buffer; +pub mod core; +pub mod stream; +use self::core::*; + +/// How much processing the compressor should do to compress the data. +/// `NoCompression` and `Bestspeed` have special meanings, the other levels determine the number +/// of checks for matches in the hash chains and whether to use lazy or greedy parsing. +#[repr(i32)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum CompressionLevel { + /// Don't do any compression, only output uncompressed blocks. + NoCompression = 0, + /// Fast compression. Uses a special compression routine that is optimized for speed. + BestSpeed = 1, + /// Slow/high compression. Do a lot of checks to try to find good matches. + BestCompression = 9, + /// Even more checks, can be very slow. + UberCompression = 10, + /// Default compromise between speed and compression. + DefaultLevel = 6, + /// Use the default compression level. + DefaultCompression = -1, +} + +// Missing safe rust analogue (this and mem-to-mem are quite similar) +/* +fn tdefl_compress( + d: Option<&mut CompressorOxide>, + in_buf: *const c_void, + in_size: Option<&mut usize>, + out_buf: *mut c_void, + out_size: Option<&mut usize>, + flush: TDEFLFlush, +) -> TDEFLStatus { + let res = match d { + None => { + in_size.map(|size| *size = 0); + out_size.map(|size| *size = 0); + (TDEFLStatus::BadParam, 0, 0) + }, + Some(compressor) => { + let callback_res = CallbackOxide::new( + compressor.callback_func.clone(), + in_buf, + in_size, + out_buf, + out_size, + ); + + if let Ok(mut callback) = callback_res { + let res = compress(compressor, &mut callback, flush); + callback.update_size(Some(res.1), Some(res.2)); + res + } else { + (TDEFLStatus::BadParam, 0, 0) + } + } + }; + res.0 +}*/ + +// Missing safe rust analogue +/* +fn tdefl_init( + d: Option<&mut CompressorOxide>, + put_buf_func: PutBufFuncPtr, + put_buf_user: *mut c_void, + flags: c_int, +) -> TDEFLStatus { + if let Some(d) = d { + *d = CompressorOxide::new( + put_buf_func.map(|func| + CallbackFunc { put_buf_func: func, put_buf_user: put_buf_user } + ), + flags as u32, + ); + TDEFLStatus::Okay + } else { + TDEFLStatus::BadParam + } +}*/ + +// Missing safe rust analogue (though maybe best served by flate2 front-end instead) +/* +fn tdefl_compress_mem_to_output( + buf: *const c_void, + buf_len: usize, + put_buf_func: PutBufFuncPtr, + put_buf_user: *mut c_void, + flags: c_int, +) -> bool*/ + +// Missing safe Rust analogue +/* +fn tdefl_compress_mem_to_mem( + out_buf: *mut c_void, + out_buf_len: usize, + src_buf: *const c_void, + src_buf_len: usize, + flags: c_int, +) -> usize*/ + +/// Compress the input data to a vector, using the specified compression level (0-10). +pub fn compress_to_vec(input: &[u8], level: u8) -> Vec { + compress_to_vec_inner(input, level, 0, 0) +} + +/// Compress the input data to a vector, using the specified compression level (0-10), and with a +/// zlib wrapper. +pub fn compress_to_vec_zlib(input: &[u8], level: u8) -> Vec { + compress_to_vec_inner(input, level, 1, 0) +} + +/// Simple function to compress data to a vec. +fn compress_to_vec_inner(input: &[u8], level: u8, window_bits: i32, strategy: i32) -> Vec { + // The comp flags function sets the zlib flag if the window_bits parameter is > 0. + let flags = create_comp_flags_from_zip_params(level.into(), window_bits, strategy); + let mut compressor = CompressorOxide::new(flags); + let mut output = vec![0; ::core::cmp::max(input.len() / 2, 2)]; + + let mut in_pos = 0; + let mut out_pos = 0; + loop { + let (status, bytes_in, bytes_out) = compress( + &mut compressor, + &input[in_pos..], + &mut output[out_pos..], + TDEFLFlush::Finish, + ); + + out_pos += bytes_out; + in_pos += bytes_in; + + match status { + TDEFLStatus::Done => { + output.truncate(out_pos); + break; + } + TDEFLStatus::Okay => { + // We need more space, so resize the vector. + if output.len().saturating_sub(out_pos) < 30 { + output.resize(output.len() * 2, 0) + } + } + // Not supposed to happen unless there is a bug. + _ => panic!("Bug! Unexpectedly failed to compress!"), + } + } + + output +} + +#[cfg(test)] +mod test { + use super::{compress_to_vec, compress_to_vec_inner, CompressionStrategy}; + use crate::inflate::decompress_to_vec; + use alloc::vec; + + /// Test deflate example. + /// + /// Check if the encoder produces the same code as the example given by Mark Adler here: + /// https://stackoverflow.com/questions/17398931/deflate-encoding-with-static-huffman-codes/17415203 + #[test] + fn compress_small() { + let test_data = b"Deflate late"; + let check = [ + 0x73, 0x49, 0x4d, 0xcb, 0x49, 0x2c, 0x49, 0x55, 0x00, 0x11, 0x00, + ]; + + let res = compress_to_vec(test_data, 1); + assert_eq!(&check[..], res.as_slice()); + + let res = compress_to_vec(test_data, 9); + assert_eq!(&check[..], res.as_slice()); + } + + #[test] + fn compress_huff_only() { + let test_data = b"Deflate late"; + + let res = compress_to_vec_inner(test_data, 1, 0, CompressionStrategy::HuffmanOnly as i32); + let d = decompress_to_vec(res.as_slice()).expect("Failed to decompress!"); + assert_eq!(test_data, d.as_slice()); + } + + /// Test that a raw block compresses fine. + #[test] + fn compress_raw() { + let text = b"Hello, zlib!"; + let encoded = { + let len = text.len(); + let notlen = !len; + let mut encoded = vec![ + 1, + len as u8, + (len >> 8) as u8, + notlen as u8, + (notlen >> 8) as u8, + ]; + encoded.extend_from_slice(&text[..]); + encoded + }; + + let res = compress_to_vec(text, 0); + assert_eq!(encoded, res.as_slice()); + } + + #[test] + fn short() { + let test_data = [10, 10, 10, 10, 10, 55]; + let c = compress_to_vec(&test_data, 9); + + let d = decompress_to_vec(c.as_slice()).expect("Failed to decompress!"); + assert_eq!(&test_data, d.as_slice()); + // Check that a static block is used here, rather than a raw block + // , so the data is actually compressed. + // (The optimal compressed length would be 5, but neither miniz nor zlib manages that either + // as neither checks matches against the byte at index 0.) + assert!(c.len() <= 6); + } +} diff --git a/crux-mir/lib/miniz_oxide/src/deflate/stream.rs b/crux-mir/lib/miniz_oxide/src/deflate/stream.rs new file mode 100644 index 000000000..39aa82d92 --- /dev/null +++ b/crux-mir/lib/miniz_oxide/src/deflate/stream.rs @@ -0,0 +1,121 @@ +//! Extra streaming compression functionality. +//! +//! As of now this is mainly intended for use to build a higher-level wrapper. +//! +//! There is no DeflateState as the needed state is contained in the compressor struct itself. + +use crate::deflate::core::{compress, CompressorOxide, TDEFLFlush, TDEFLStatus}; +use crate::{MZError, MZFlush, MZStatus, StreamResult}; + +/// Try to compress from input to output with the given [`CompressorOxide`]. +/// +/// # Errors +/// +/// Returns [`MZError::Buf`] If the size of the `output` slice is empty or no progress was made due +/// to lack of expected input data, or if called without [`MZFlush::Finish`] after the compression +/// was already finished. +/// +/// Returns [`MZError::Param`] if the compressor parameters are set wrong. +/// +/// Returns [`MZError::Stream`] when lower-level decompressor returns a +/// [`TDEFLStatus::PutBufFailed`]; may not actually be possible. +pub fn deflate( + compressor: &mut CompressorOxide, + input: &[u8], + output: &mut [u8], + flush: MZFlush, +) -> StreamResult { + if output.is_empty() { + return StreamResult::error(MZError::Buf); + } + + if compressor.prev_return_status() == TDEFLStatus::Done { + return if flush == MZFlush::Finish { + StreamResult { + bytes_written: 0, + bytes_consumed: 0, + status: Ok(MZStatus::StreamEnd), + } + } else { + StreamResult::error(MZError::Buf) + }; + } + + let mut bytes_written = 0; + let mut bytes_consumed = 0; + + let mut next_in = input; + let mut next_out = output; + + let status = loop { + let in_bytes; + let out_bytes; + let defl_status = { + let res = compress(compressor, next_in, next_out, TDEFLFlush::from(flush)); + in_bytes = res.1; + out_bytes = res.2; + res.0 + }; + + next_in = &next_in[in_bytes..]; + next_out = &mut next_out[out_bytes..]; + bytes_consumed += in_bytes; + bytes_written += out_bytes; + + // Check if we are done, or compression failed. + match defl_status { + TDEFLStatus::BadParam => break Err(MZError::Param), + // Don't think this can happen as we're not using a custom callback. + TDEFLStatus::PutBufFailed => break Err(MZError::Stream), + TDEFLStatus::Done => break Ok(MZStatus::StreamEnd), + _ => (), + }; + + // All the output space was used, so wait for more. + if next_out.is_empty() { + break Ok(MZStatus::Ok); + } + + if next_in.is_empty() && (flush != MZFlush::Finish) { + let total_changed = bytes_written > 0 || bytes_consumed > 0; + + break if (flush != MZFlush::None) || total_changed { + // We wrote or consumed something, and/or did a flush (sync/partial etc.). + Ok(MZStatus::Ok) + } else { + // No more input data, not flushing, and nothing was consumed or written, + // so couldn't make any progress. + Err(MZError::Buf) + }; + } + }; + StreamResult { + bytes_consumed, + bytes_written, + status, + } +} + +#[cfg(test)] +mod test { + use super::deflate; + use crate::deflate::CompressorOxide; + use crate::inflate::decompress_to_vec_zlib; + use crate::{MZFlush, MZStatus}; + use alloc::boxed::Box; + use alloc::vec; + + #[test] + fn test_state() { + let data = b"Hello zlib!"; + let mut compressed = vec![0; 50]; + let mut compressor = Box::::default(); + let res = deflate(&mut compressor, data, &mut compressed, MZFlush::Finish); + let status = res.status.expect("Failed to compress!"); + let decomp = + decompress_to_vec_zlib(&compressed).expect("Failed to decompress compressed data"); + assert_eq!(status, MZStatus::StreamEnd); + assert_eq!(decomp[..], data[..]); + assert_eq!(res.bytes_consumed, data.len()); + } +} diff --git a/crux-mir/lib/miniz_oxide/src/inflate/core.rs b/crux-mir/lib/miniz_oxide/src/inflate/core.rs new file mode 100644 index 000000000..38bdacbbd --- /dev/null +++ b/crux-mir/lib/miniz_oxide/src/inflate/core.rs @@ -0,0 +1,1931 @@ +//! Streaming decompression functionality. + +use super::*; +use crate::shared::{update_adler32, HUFFMAN_LENGTH_ORDER}; + +use ::core::convert::TryInto; +use ::core::{cmp, slice}; + +use self::output_buffer::OutputBuffer; + +pub const TINFL_LZ_DICT_SIZE: usize = 32_768; + +/// A struct containing huffman code lengths and the huffman code tree used by the decompressor. +struct HuffmanTable { + /// Length of the code at each index. + pub code_size: [u8; MAX_HUFF_SYMBOLS_0], + /// Fast lookup table for shorter huffman codes. + /// + /// See `HuffmanTable::fast_lookup`. + pub look_up: [i16; FAST_LOOKUP_SIZE as usize], + /// Full huffman tree. + /// + /// Positive values are edge nodes/symbols, negative values are + /// parent nodes/references to other nodes. + pub tree: [i16; MAX_HUFF_TREE_SIZE], +} + +impl HuffmanTable { + const fn new() -> HuffmanTable { + HuffmanTable { + code_size: [0; MAX_HUFF_SYMBOLS_0], + look_up: [0; FAST_LOOKUP_SIZE as usize], + tree: [0; MAX_HUFF_TREE_SIZE], + } + } + + /// Look for a symbol in the fast lookup table. + /// The symbol is stored in the lower 9 bits, the length in the next 6. + /// If the returned value is negative, the code wasn't found in the + /// fast lookup table and the full tree has to be traversed to find the code. + #[inline] + fn fast_lookup(&self, bit_buf: BitBuffer) -> i16 { + self.look_up[(bit_buf & BitBuffer::from(FAST_LOOKUP_SIZE - 1)) as usize] + } + + /// Get the symbol and the code length from the huffman tree. + #[inline] + fn tree_lookup(&self, fast_symbol: i32, bit_buf: BitBuffer, mut code_len: u32) -> (i32, u32) { + let mut symbol = fast_symbol; + // We step through the tree until we encounter a positive value, which indicates a + // symbol. + loop { + // symbol here indicates the position of the left (0) node, if the next bit is 1 + // we add 1 to the lookup position to get the right node. + symbol = i32::from(self.tree[(!symbol + ((bit_buf >> code_len) & 1) as i32) as usize]); + code_len += 1; + if symbol >= 0 { + break; + } + } + (symbol, code_len) + } + + #[inline] + /// Look up a symbol and code length from the bits in the provided bit buffer. + /// + /// Returns Some(symbol, length) on success, + /// None if the length is 0. + /// + /// It's possible we could avoid checking for 0 if we can guarantee a sane table. + /// TODO: Check if a smaller type for code_len helps performance. + fn lookup(&self, bit_buf: BitBuffer) -> Option<(i32, u32)> { + let symbol = self.fast_lookup(bit_buf).into(); + if symbol >= 0 { + if (symbol >> 9) as u32 != 0 { + Some((symbol, (symbol >> 9) as u32)) + } else { + // Zero-length code. + None + } + } else { + // We didn't get a symbol from the fast lookup table, so check the tree instead. + Some(self.tree_lookup(symbol, bit_buf, FAST_LOOKUP_BITS.into())) + } + } +} + +/// The number of huffman tables used. +const MAX_HUFF_TABLES: usize = 3; +/// The length of the first (literal/length) huffman table. +const MAX_HUFF_SYMBOLS_0: usize = 288; +/// The length of the second (distance) huffman table. +const MAX_HUFF_SYMBOLS_1: usize = 32; +/// The length of the last (huffman code length) huffman table. +const _MAX_HUFF_SYMBOLS_2: usize = 19; +/// The maximum length of a code that can be looked up in the fast lookup table. +const FAST_LOOKUP_BITS: u8 = 10; +/// The size of the fast lookup table. +const FAST_LOOKUP_SIZE: u32 = 1 << FAST_LOOKUP_BITS; +const MAX_HUFF_TREE_SIZE: usize = MAX_HUFF_SYMBOLS_0 * 2; +const LITLEN_TABLE: usize = 0; +const DIST_TABLE: usize = 1; +const HUFFLEN_TABLE: usize = 2; + +/// Flags to [`decompress()`] to control how inflation works. +/// +/// These define bits for a bitmask argument. +pub mod inflate_flags { + /// Should we try to parse a zlib header? + /// + /// If unset, [`decompress()`] will expect an RFC1951 deflate stream. If set, it will expect an + /// RFC1950 zlib wrapper around the deflate stream. + pub const TINFL_FLAG_PARSE_ZLIB_HEADER: u32 = 1; + + /// There will be more input that hasn't been given to the decompressor yet. + /// + /// This is useful when you want to decompress what you have so far, + /// even if you know there is probably more input that hasn't gotten here yet (_e.g._, over a + /// network connection). When [`decompress()`][super::decompress] reaches the end of the input + /// without finding the end of the compressed stream, it will return + /// [`TINFLStatus::NeedsMoreInput`][super::TINFLStatus::NeedsMoreInput] if this is set, + /// indicating that you should get more data before calling again. If not set, it will return + /// [`TINFLStatus::FailedCannotMakeProgress`][super::TINFLStatus::FailedCannotMakeProgress] + /// suggesting the stream is corrupt, since you claimed it was all there. + pub const TINFL_FLAG_HAS_MORE_INPUT: u32 = 2; + + /// The output buffer should not wrap around. + pub const TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF: u32 = 4; + + /// Calculate the adler32 checksum of the output data even if we're not inflating a zlib stream. + /// + /// If [`TINFL_FLAG_IGNORE_ADLER32`] is specified, it will override this. + /// + /// NOTE: Enabling/disabling this between calls to decompress will result in an incorect + /// checksum. + pub const TINFL_FLAG_COMPUTE_ADLER32: u32 = 8; + + /// Ignore adler32 checksum even if we are inflating a zlib stream. + /// + /// Overrides [`TINFL_FLAG_COMPUTE_ADLER32`] if both are enabled. + /// + /// NOTE: This flag does not exist in miniz as it does not support this and is a + /// custom addition for miniz_oxide. + /// + /// NOTE: Should not be changed from enabled to disabled after decompression has started, + /// this will result in checksum failure (outside the unlikely event where the checksum happens + /// to match anyway). + pub const TINFL_FLAG_IGNORE_ADLER32: u32 = 64; +} + +use self::inflate_flags::*; + +const MIN_TABLE_SIZES: [u16; 3] = [257, 1, 4]; + +#[cfg(target_pointer_width = "64")] +type BitBuffer = u64; + +#[cfg(not(target_pointer_width = "64"))] +type BitBuffer = u32; + +/// Main decompression struct. +/// +pub struct DecompressorOxide { + /// Current state of the decompressor. + state: core::State, + /// Number of bits in the bit buffer. + num_bits: u32, + /// Zlib CMF + z_header0: u32, + /// Zlib FLG + z_header1: u32, + /// Adler32 checksum from the zlib header. + z_adler32: u32, + /// 1 if the current block is the last block, 0 otherwise. + finish: u32, + /// The type of the current block. + block_type: u32, + /// 1 if the adler32 value should be checked. + check_adler32: u32, + /// Last match distance. + dist: u32, + /// Variable used for match length, symbols, and a number of other things. + counter: u32, + /// Number of extra bits for the last length or distance code. + num_extra: u32, + /// Number of entries in each huffman table. + table_sizes: [u32; MAX_HUFF_TABLES], + /// Buffer of input data. + bit_buf: BitBuffer, + /// Huffman tables. + tables: [HuffmanTable; MAX_HUFF_TABLES], + /// Raw block header. + raw_header: [u8; 4], + /// Huffman length codes. + len_codes: [u8; MAX_HUFF_SYMBOLS_0 + MAX_HUFF_SYMBOLS_1 + 137], +} + +impl DecompressorOxide { + /// Create a new tinfl_decompressor with all fields set to 0. + pub fn new() -> DecompressorOxide { + DecompressorOxide::default() + } + + /// Set the current state to `Start`. + #[inline] + pub fn init(&mut self) { + // The rest of the data is reset or overwritten when used. + self.state = core::State::Start; + } + + /// Returns the adler32 checksum of the currently decompressed data. + /// Note: Will return Some(1) if decompressing zlib but ignoring adler32. + #[inline] + pub fn adler32(&self) -> Option { + if self.state != State::Start && !self.state.is_failure() && self.z_header0 != 0 { + Some(self.check_adler32) + } else { + None + } + } + + /// Returns the adler32 that was read from the zlib header if it exists. + #[inline] + pub fn adler32_header(&self) -> Option { + if self.state != State::Start && self.state != State::BadZlibHeader && self.z_header0 != 0 { + Some(self.z_adler32) + } else { + None + } + } +} + +impl Default for DecompressorOxide { + /// Create a new tinfl_decompressor with all fields set to 0. + #[inline(always)] + fn default() -> Self { + DecompressorOxide { + state: core::State::Start, + num_bits: 0, + z_header0: 0, + z_header1: 0, + z_adler32: 0, + finish: 0, + block_type: 0, + check_adler32: 0, + dist: 0, + counter: 0, + num_extra: 0, + table_sizes: [0; MAX_HUFF_TABLES], + bit_buf: 0, + // TODO:(oyvindln) Check that copies here are optimized out in release mode. + tables: [ + HuffmanTable::new(), + HuffmanTable::new(), + HuffmanTable::new(), + ], + raw_header: [0; 4], + len_codes: [0; MAX_HUFF_SYMBOLS_0 + MAX_HUFF_SYMBOLS_1 + 137], + } + } +} + +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +enum State { + Start = 0, + ReadZlibCmf, + ReadZlibFlg, + ReadBlockHeader, + BlockTypeNoCompression, + RawHeader, + RawMemcpy1, + RawMemcpy2, + ReadTableSizes, + ReadHufflenTableCodeSize, + ReadLitlenDistTablesCodeSize, + ReadExtraBitsCodeSize, + DecodeLitlen, + WriteSymbol, + ReadExtraBitsLitlen, + DecodeDistance, + ReadExtraBitsDistance, + RawReadFirstByte, + RawStoreFirstByte, + WriteLenBytesToEnd, + BlockDone, + HuffDecodeOuterLoop1, + HuffDecodeOuterLoop2, + ReadAdler32, + + DoneForever, + + // Failure states. + BlockTypeUnexpected, + BadCodeSizeSum, + BadTotalSymbols, + BadZlibHeader, + DistanceOutOfBounds, + BadRawLength, + BadCodeSizeDistPrevLookup, + InvalidLitlen, + InvalidDist, + InvalidCodeLen, +} + +impl State { + fn is_failure(self) -> bool { + match self { + BlockTypeUnexpected => true, + BadCodeSizeSum => true, + BadTotalSymbols => true, + BadZlibHeader => true, + DistanceOutOfBounds => true, + BadRawLength => true, + BadCodeSizeDistPrevLookup => true, + InvalidLitlen => true, + InvalidDist => true, + _ => false, + } + } + + #[inline] + fn begin(&mut self, new_state: State) { + *self = new_state; + } +} + +use self::State::*; + +// Not sure why miniz uses 32-bit values for these, maybe alignment/cache again? +// # Optimization +// We add a extra value at the end and make the tables 32 elements long +// so we can use a mask to avoid bounds checks. +// The invalid values are set to something high enough to avoid underflowing +// the match length. +/// Base length for each length code. +/// +/// The base is used together with the value of the extra bits to decode the actual +/// length/distance values in a match. +#[rustfmt::skip] +const LENGTH_BASE: [u16; 32] = [ + 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, + 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 512, 512, 512 +]; + +/// Number of extra bits for each length code. +#[rustfmt::skip] +const LENGTH_EXTRA: [u8; 32] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, + 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0, 0, 0, 0 +]; + +/// Base length for each distance code. +#[rustfmt::skip] +const DIST_BASE: [u16; 32] = [ + 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, + 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, + 2049, 3073, 4097, 6145, 8193, 12_289, 16_385, 24_577, 32_768, 32_768 +]; + +/// Number of extra bits for each distance code. +#[rustfmt::skip] +const DIST_EXTRA: [u8; 32] = [ + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, + 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 13, 13 +]; + +/// The mask used when indexing the base/extra arrays. +const BASE_EXTRA_MASK: usize = 32 - 1; + +/// Sets the value of all the elements of the slice to `val`. +#[inline] +fn memset(slice: &mut [T], val: T) { + for x in slice { + *x = val + } +} + +/// Read an le u16 value from the slice iterator. +/// +/// # Panics +/// Panics if there are less than two bytes left. +#[inline] +fn read_u16_le(iter: &mut slice::Iter) -> u16 { + let ret = { + let two_bytes = iter.as_ref()[..2].try_into().unwrap(); + u16::from_le_bytes(two_bytes) + }; + iter.nth(1); + ret +} + +/// Read an le u32 value from the slice iterator. +/// +/// # Panics +/// Panics if there are less than four bytes left. +#[inline(always)] +#[cfg(target_pointer_width = "64")] +fn read_u32_le(iter: &mut slice::Iter) -> u32 { + let ret = { + let four_bytes: [u8; 4] = iter.as_ref()[..4].try_into().unwrap(); + u32::from_le_bytes(four_bytes) + }; + iter.nth(3); + ret +} + +/// Ensure that there is data in the bit buffer. +/// +/// On 64-bit platform, we use a 64-bit value so this will +/// result in there being at least 32 bits in the bit buffer. +/// This function assumes that there is at least 4 bytes left in the input buffer. +#[inline(always)] +#[cfg(target_pointer_width = "64")] +fn fill_bit_buffer(l: &mut LocalVars, in_iter: &mut slice::Iter) { + // Read four bytes into the buffer at once. + if l.num_bits < 30 { + l.bit_buf |= BitBuffer::from(read_u32_le(in_iter)) << l.num_bits; + l.num_bits += 32; + } +} + +/// Same as previous, but for non-64-bit platforms. +/// Ensures at least 16 bits are present, requires at least 2 bytes in the in buffer. +#[inline(always)] +#[cfg(not(target_pointer_width = "64"))] +fn fill_bit_buffer(l: &mut LocalVars, in_iter: &mut slice::Iter) { + // If the buffer is 32-bit wide, read 2 bytes instead. + if l.num_bits < 15 { + l.bit_buf |= BitBuffer::from(read_u16_le(in_iter)) << l.num_bits; + l.num_bits += 16; + } +} + +/// Check that the zlib header is correct and that there is enough space in the buffer +/// for the window size specified in the header. +/// +/// See https://tools.ietf.org/html/rfc1950 +#[inline] +fn validate_zlib_header(cmf: u32, flg: u32, flags: u32, mask: usize) -> Action { + let mut failed = + // cmf + flg should be divisible by 31. + (((cmf * 256) + flg) % 31 != 0) || + // If this flag is set, a dictionary was used for this zlib compressed data. + // This is currently not supported by miniz or miniz-oxide + ((flg & 0b0010_0000) != 0) || + // Compression method. Only 8(DEFLATE) is defined by the standard. + ((cmf & 15) != 8); + + let window_size = 1 << ((cmf >> 4) + 8); + if (flags & TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF) == 0 { + // Bail if the buffer is wrapping and the window size is larger than the buffer. + failed |= (mask + 1) < window_size; + } + + // Zlib doesn't allow window sizes above 32 * 1024. + failed |= window_size > 32_768; + + if failed { + Action::Jump(BadZlibHeader) + } else { + Action::Jump(ReadBlockHeader) + } +} + +enum Action { + None, + Jump(State), + End(TINFLStatus), +} + +/// Try to decode the next huffman code, and puts it in the counter field of the decompressor +/// if successful. +/// +/// # Returns +/// The specified action returned from `f` on success, +/// `Action::End` if there are not enough data left to decode a symbol. +fn decode_huffman_code( + r: &mut DecompressorOxide, + l: &mut LocalVars, + table: usize, + flags: u32, + in_iter: &mut slice::Iter, + f: F, +) -> Action +where + F: FnOnce(&mut DecompressorOxide, &mut LocalVars, i32) -> Action, +{ + // As the huffman codes can be up to 15 bits long we need at least 15 bits + // ready in the bit buffer to start decoding the next huffman code. + if l.num_bits < 15 { + // First, make sure there is enough data in the bit buffer to decode a huffman code. + if in_iter.len() < 2 { + // If there is less than 2 bytes left in the input buffer, we try to look up + // the huffman code with what's available, and return if that doesn't succeed. + // Original explanation in miniz: + // /* TINFL_HUFF_BITBUF_FILL() is only used rarely, when the number of bytes + // * remaining in the input buffer falls below 2. */ + // /* It reads just enough bytes from the input stream that are needed to decode + // * the next Huffman code (and absolutely no more). It works by trying to fully + // * decode a */ + // /* Huffman code by using whatever bits are currently present in the bit buffer. + // * If this fails, it reads another byte, and tries again until it succeeds or + // * until the */ + // /* bit buffer contains >=15 bits (deflate's max. Huffman code size). */ + loop { + let mut temp = i32::from(r.tables[table].fast_lookup(l.bit_buf)); + + if temp >= 0 { + let code_len = (temp >> 9) as u32; + if (code_len != 0) && (l.num_bits >= code_len) { + break; + } + } else if l.num_bits > FAST_LOOKUP_BITS.into() { + let mut code_len = u32::from(FAST_LOOKUP_BITS); + loop { + temp = i32::from( + r.tables[table].tree + [(!temp + ((l.bit_buf >> code_len) & 1) as i32) as usize], + ); + code_len += 1; + if temp >= 0 || l.num_bits < code_len + 1 { + break; + } + } + if temp >= 0 { + break; + } + } + + // TODO: miniz jumps straight to here after getting here again after failing to read + // a byte. + // Doing that lets miniz avoid re-doing the lookup that that was done in the + // previous call. + let mut byte = 0; + if let a @ Action::End(_) = read_byte(in_iter, flags, |b| { + byte = b; + Action::None + }) { + return a; + }; + + // Do this outside closure for now to avoid borrowing r. + l.bit_buf |= BitBuffer::from(byte) << l.num_bits; + l.num_bits += 8; + + if l.num_bits >= 15 { + break; + } + } + } else { + // There is enough data in the input buffer, so read the next two bytes + // and add them to the bit buffer. + // Unwrapping here is fine since we just checked that there are at least two + // bytes left. + l.bit_buf |= BitBuffer::from(read_u16_le(in_iter)) << l.num_bits; + l.num_bits += 16; + } + } + + // We now have at least 15 bits in the input buffer. + let mut symbol = i32::from(r.tables[table].fast_lookup(l.bit_buf)); + let code_len; + // If the symbol was found in the fast lookup table. + if symbol >= 0 { + // Get the length value from the top bits. + // As we shift down the sign bit, converting to an unsigned value + // shouldn't overflow. + code_len = (symbol >> 9) as u32; + // Mask out the length value. + symbol &= 511; + } else { + let res = r.tables[table].tree_lookup(symbol, l.bit_buf, u32::from(FAST_LOOKUP_BITS)); + symbol = res.0; + code_len = res.1 as u32; + }; + + if code_len == 0 { + return Action::Jump(InvalidCodeLen); + } + + l.bit_buf >>= code_len as u32; + l.num_bits -= code_len; + f(r, l, symbol) +} + +/// Try to read one byte from `in_iter` and call `f` with the read byte as an argument, +/// returning the result. +/// If reading fails, `Action::End is returned` +#[inline] +fn read_byte(in_iter: &mut slice::Iter, flags: u32, f: F) -> Action +where + F: FnOnce(u8) -> Action, +{ + match in_iter.next() { + None => end_of_input(flags), + Some(&byte) => f(byte), + } +} + +// TODO: `l: &mut LocalVars` may be slow similar to decompress_fast (even with inline(always)) +/// Try to read `amount` number of bits from `in_iter` and call the function `f` with the bits as an +/// an argument after reading, returning the result of that function, or `Action::End` if there are +/// not enough bytes left. +#[inline] +#[allow(clippy::while_immutable_condition)] +fn read_bits( + l: &mut LocalVars, + amount: u32, + in_iter: &mut slice::Iter, + flags: u32, + f: F, +) -> Action +where + F: FnOnce(&mut LocalVars, BitBuffer) -> Action, +{ + // Clippy gives a false positive warning here due to the closure. + // Read enough bytes from the input iterator to cover the number of bits we want. + while l.num_bits < amount { + match read_byte(in_iter, flags, |byte| { + l.bit_buf |= BitBuffer::from(byte) << l.num_bits; + l.num_bits += 8; + Action::None + }) { + Action::None => (), + // If there are not enough bytes in the input iterator, return and signal that we need + // more. + action => return action, + } + } + + let bits = l.bit_buf & ((1 << amount) - 1); + l.bit_buf >>= amount; + l.num_bits -= amount; + f(l, bits) +} + +#[inline] +fn pad_to_bytes(l: &mut LocalVars, in_iter: &mut slice::Iter, flags: u32, f: F) -> Action +where + F: FnOnce(&mut LocalVars) -> Action, +{ + let num_bits = l.num_bits & 7; + read_bits(l, num_bits, in_iter, flags, |l, _| f(l)) +} + +#[inline] +fn end_of_input(flags: u32) -> Action { + Action::End(if flags & TINFL_FLAG_HAS_MORE_INPUT != 0 { + TINFLStatus::NeedsMoreInput + } else { + TINFLStatus::FailedCannotMakeProgress + }) +} + +#[inline] +fn undo_bytes(l: &mut LocalVars, max: u32) -> u32 { + let res = cmp::min(l.num_bits >> 3, max); + l.num_bits -= res << 3; + res +} + +fn start_static_table(r: &mut DecompressorOxide) { + r.table_sizes[LITLEN_TABLE] = 288; + r.table_sizes[DIST_TABLE] = 32; + memset(&mut r.tables[LITLEN_TABLE].code_size[0..144], 8); + memset(&mut r.tables[LITLEN_TABLE].code_size[144..256], 9); + memset(&mut r.tables[LITLEN_TABLE].code_size[256..280], 7); + memset(&mut r.tables[LITLEN_TABLE].code_size[280..288], 8); + memset(&mut r.tables[DIST_TABLE].code_size[0..32], 5); +} + +fn init_tree(r: &mut DecompressorOxide, l: &mut LocalVars) -> Action { + loop { + let table = &mut r.tables[r.block_type as usize]; + let table_size = r.table_sizes[r.block_type as usize] as usize; + let mut total_symbols = [0u32; 16]; + let mut next_code = [0u32; 17]; + memset(&mut table.look_up[..], 0); + memset(&mut table.tree[..], 0); + + for &code_size in &table.code_size[..table_size] { + total_symbols[code_size as usize] += 1; + } + + let mut used_symbols = 0; + let mut total = 0; + for i in 1..16 { + used_symbols += total_symbols[i]; + total += total_symbols[i]; + total <<= 1; + next_code[i + 1] = total; + } + + if total != 65_536 && used_symbols > 1 { + return Action::Jump(BadTotalSymbols); + } + + let mut tree_next = -1; + for symbol_index in 0..table_size { + let mut rev_code = 0; + let code_size = table.code_size[symbol_index]; + if code_size == 0 { + continue; + } + + let mut cur_code = next_code[code_size as usize]; + next_code[code_size as usize] += 1; + + for _ in 0..code_size { + rev_code = (rev_code << 1) | (cur_code & 1); + cur_code >>= 1; + } + + if code_size <= FAST_LOOKUP_BITS { + let k = (i16::from(code_size) << 9) | symbol_index as i16; + while rev_code < FAST_LOOKUP_SIZE { + table.look_up[rev_code as usize] = k; + rev_code += 1 << code_size; + } + continue; + } + + let mut tree_cur = table.look_up[(rev_code & (FAST_LOOKUP_SIZE - 1)) as usize]; + if tree_cur == 0 { + table.look_up[(rev_code & (FAST_LOOKUP_SIZE - 1)) as usize] = tree_next as i16; + tree_cur = tree_next; + tree_next -= 2; + } + + rev_code >>= FAST_LOOKUP_BITS - 1; + for _ in FAST_LOOKUP_BITS + 1..code_size { + rev_code >>= 1; + tree_cur -= (rev_code & 1) as i16; + if table.tree[(-tree_cur - 1) as usize] == 0 { + table.tree[(-tree_cur - 1) as usize] = tree_next as i16; + tree_cur = tree_next; + tree_next -= 2; + } else { + tree_cur = table.tree[(-tree_cur - 1) as usize]; + } + } + + rev_code >>= 1; + tree_cur -= (rev_code & 1) as i16; + table.tree[(-tree_cur - 1) as usize] = symbol_index as i16; + } + + if r.block_type == 2 { + l.counter = 0; + return Action::Jump(ReadLitlenDistTablesCodeSize); + } + + if r.block_type == 0 { + break; + } + r.block_type -= 1; + } + + l.counter = 0; + Action::Jump(DecodeLitlen) +} + +// A helper macro for generating the state machine. +// +// As Rust doesn't have fallthrough on matches, we have to return to the match statement +// and jump for each state change. (Which would ideally be optimized away, but often isn't.) +macro_rules! generate_state { + ($state: ident, $state_machine: tt, $f: expr) => { + loop { + match $f { + Action::None => continue, + Action::Jump(new_state) => { + $state = new_state; + continue $state_machine; + }, + Action::End(result) => break $state_machine result, + } + } + }; +} + +#[derive(Copy, Clone)] +struct LocalVars { + pub bit_buf: BitBuffer, + pub num_bits: u32, + pub dist: u32, + pub counter: u32, + pub num_extra: u32, +} + +#[inline] +fn transfer( + out_slice: &mut [u8], + mut source_pos: usize, + mut out_pos: usize, + match_len: usize, + out_buf_size_mask: usize, +) { + for _ in 0..match_len >> 2 { + out_slice[out_pos] = out_slice[source_pos & out_buf_size_mask]; + out_slice[out_pos + 1] = out_slice[(source_pos + 1) & out_buf_size_mask]; + out_slice[out_pos + 2] = out_slice[(source_pos + 2) & out_buf_size_mask]; + out_slice[out_pos + 3] = out_slice[(source_pos + 3) & out_buf_size_mask]; + source_pos += 4; + out_pos += 4; + } + + match match_len & 3 { + 0 => (), + 1 => out_slice[out_pos] = out_slice[source_pos & out_buf_size_mask], + 2 => { + out_slice[out_pos] = out_slice[source_pos & out_buf_size_mask]; + out_slice[out_pos + 1] = out_slice[(source_pos + 1) & out_buf_size_mask]; + } + 3 => { + out_slice[out_pos] = out_slice[source_pos & out_buf_size_mask]; + out_slice[out_pos + 1] = out_slice[(source_pos + 1) & out_buf_size_mask]; + out_slice[out_pos + 2] = out_slice[(source_pos + 2) & out_buf_size_mask]; + } + _ => unreachable!(), + } +} + +/// Presumes that there is at least match_len bytes in output left. +#[inline] +fn apply_match( + out_slice: &mut [u8], + out_pos: usize, + dist: usize, + match_len: usize, + out_buf_size_mask: usize, +) { + debug_assert!(out_pos + match_len <= out_slice.len()); + + let source_pos = out_pos.wrapping_sub(dist) & out_buf_size_mask; + + if match_len == 3 { + // Fast path for match len 3. + out_slice[out_pos] = out_slice[source_pos]; + out_slice[out_pos + 1] = out_slice[(source_pos + 1) & out_buf_size_mask]; + out_slice[out_pos + 2] = out_slice[(source_pos + 2) & out_buf_size_mask]; + return; + } + + if cfg!(not(any(target_arch = "x86", target_arch = "x86_64"))) { + // We are not on x86 so copy manually. + transfer(out_slice, source_pos, out_pos, match_len, out_buf_size_mask); + return; + } + + if source_pos >= out_pos && (source_pos - out_pos) < match_len { + transfer(out_slice, source_pos, out_pos, match_len, out_buf_size_mask); + } else if match_len <= dist && source_pos + match_len < out_slice.len() { + // Destination and source segments does not intersect and source does not wrap. + if source_pos < out_pos { + let (from_slice, to_slice) = out_slice.split_at_mut(out_pos); + to_slice[..match_len].copy_from_slice(&from_slice[source_pos..source_pos + match_len]); + } else { + let (to_slice, from_slice) = out_slice.split_at_mut(source_pos); + to_slice[out_pos..out_pos + match_len].copy_from_slice(&from_slice[..match_len]); + } + } else { + transfer(out_slice, source_pos, out_pos, match_len, out_buf_size_mask); + } +} + +/// Fast inner decompression loop which is run while there is at least +/// 259 bytes left in the output buffer, and at least 6 bytes left in the input buffer +/// (The maximum one match would need + 1). +/// +/// This was inspired by a similar optimization in zlib, which uses this info to do +/// faster unchecked copies of multiple bytes at a time. +/// Currently we don't do this here, but this function does avoid having to jump through the +/// big match loop on each state change(as rust does not have fallthrough or gotos at the moment), +/// and already improves decompression speed a fair bit. +fn decompress_fast( + r: &mut DecompressorOxide, + in_iter: &mut slice::Iter, + out_buf: &mut OutputBuffer, + flags: u32, + local_vars: &mut LocalVars, + out_buf_size_mask: usize, +) -> (TINFLStatus, State) { + // Make a local copy of the most used variables, to avoid having to update and read from values + // in a random memory location and to encourage more register use. + let mut l = *local_vars; + let mut state; + + let status: TINFLStatus = 'o: loop { + state = State::DecodeLitlen; + loop { + // This function assumes that there is at least 259 bytes left in the output buffer, + // and that there is at least 14 bytes left in the input buffer. 14 input bytes: + // 15 (prev lit) + 15 (length) + 5 (length extra) + 15 (dist) + // + 29 + 32 (left in bit buf, including last 13 dist extra) = 111 bits < 14 bytes + // We need the one extra byte as we may write one length and one full match + // before checking again. + if out_buf.bytes_left() < 259 || in_iter.len() < 14 { + state = State::DecodeLitlen; + break 'o TINFLStatus::Done; + } + + fill_bit_buffer(&mut l, in_iter); + + if let Some((symbol, code_len)) = r.tables[LITLEN_TABLE].lookup(l.bit_buf) { + l.counter = symbol as u32; + l.bit_buf >>= code_len; + l.num_bits -= code_len; + + if (l.counter & 256) != 0 { + // The symbol is not a literal. + break; + } else { + // If we have a 32-bit buffer we need to read another two bytes now + // to have enough bits to keep going. + if cfg!(not(target_pointer_width = "64")) { + fill_bit_buffer(&mut l, in_iter); + } + + if let Some((symbol, code_len)) = r.tables[LITLEN_TABLE].lookup(l.bit_buf) { + l.bit_buf >>= code_len; + l.num_bits -= code_len; + // The previous symbol was a literal, so write it directly and check + // the next one. + out_buf.write_byte(l.counter as u8); + if (symbol & 256) != 0 { + l.counter = symbol as u32; + // The symbol is a length value. + break; + } else { + // The symbol is a literal, so write it directly and continue. + out_buf.write_byte(symbol as u8); + } + } else { + state.begin(InvalidCodeLen); + break 'o TINFLStatus::Failed; + } + } + } else { + state.begin(InvalidCodeLen); + break 'o TINFLStatus::Failed; + } + } + + // Mask the top bits since they may contain length info. + l.counter &= 511; + if l.counter == 256 { + // We hit the end of block symbol. + state.begin(BlockDone); + break 'o TINFLStatus::Done; + } else if l.counter > 285 { + // Invalid code. + // We already verified earlier that the code is > 256. + state.begin(InvalidLitlen); + break 'o TINFLStatus::Failed; + } else { + // The symbol was a length code. + // # Optimization + // Mask the value to avoid bounds checks + // We could use get_unchecked later if can statically verify that + // this will never go out of bounds. + l.num_extra = u32::from(LENGTH_EXTRA[(l.counter - 257) as usize & BASE_EXTRA_MASK]); + l.counter = u32::from(LENGTH_BASE[(l.counter - 257) as usize & BASE_EXTRA_MASK]); + // Length and distance codes have a number of extra bits depending on + // the base, which together with the base gives us the exact value. + + fill_bit_buffer(&mut l, in_iter); + if l.num_extra != 0 { + let extra_bits = l.bit_buf & ((1 << l.num_extra) - 1); + l.bit_buf >>= l.num_extra; + l.num_bits -= l.num_extra; + l.counter += extra_bits as u32; + } + + // We found a length code, so a distance code should follow. + + if cfg!(not(target_pointer_width = "64")) { + fill_bit_buffer(&mut l, in_iter); + } + + if let Some((mut symbol, code_len)) = r.tables[DIST_TABLE].lookup(l.bit_buf) { + symbol &= 511; + l.bit_buf >>= code_len; + l.num_bits -= code_len; + if symbol > 29 { + state.begin(InvalidDist); + break 'o TINFLStatus::Failed; + } + + l.num_extra = u32::from(DIST_EXTRA[symbol as usize]); + l.dist = u32::from(DIST_BASE[symbol as usize]); + } else { + state.begin(InvalidCodeLen); + break 'o TINFLStatus::Failed; + } + + if l.num_extra != 0 { + fill_bit_buffer(&mut l, in_iter); + let extra_bits = l.bit_buf & ((1 << l.num_extra) - 1); + l.bit_buf >>= l.num_extra; + l.num_bits -= l.num_extra; + l.dist += extra_bits as u32; + } + + let position = out_buf.position(); + if l.dist as usize > out_buf.position() + && (flags & TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF != 0) + { + // We encountered a distance that refers a position before + // the start of the decoded data, so we can't continue. + state.begin(DistanceOutOfBounds); + break TINFLStatus::Failed; + } + + apply_match( + out_buf.get_mut(), + position, + l.dist as usize, + l.counter as usize, + out_buf_size_mask, + ); + + out_buf.set_position(position + l.counter as usize); + } + }; + + *local_vars = l; + (status, state) +} + +/// Main decompression function. Keeps decompressing data from `in_buf` until the `in_buf` is +/// empty, `out` is full, the end of the deflate stream is hit, or there is an error in the +/// deflate stream. +/// +/// # Arguments +/// +/// `r` is a [`DecompressorOxide`] struct with the state of this stream. +/// +/// `in_buf` is a reference to the compressed data that is to be decompressed. The decompressor will +/// start at the first byte of this buffer. +/// +/// `out` is a reference to the buffer that will store the decompressed data, and that +/// stores previously decompressed data if any. +/// +/// * The offset given by `out_pos` indicates where in the output buffer slice writing should start. +/// * If [`TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF`] is not set, the output buffer is used in a +/// wrapping manner, and it's size is required to be a power of 2. +/// * The decompression function normally needs access to 32KiB of the previously decompressed data +///(or to the beginning of the decompressed data if less than 32KiB has been decompressed.) +/// - If this data is not available, decompression may fail. +/// - Some deflate compressors allow specifying a window size which limits match distances to +/// less than this, or alternatively an RLE mode where matches will only refer to the previous byte +/// and thus allows a smaller output buffer. The window size can be specified in the zlib +/// header structure, however, the header data should not be relied on to be correct. +/// +/// `flags` indicates settings and status to the decompression function. +/// * The [`TINFL_FLAG_HAS_MORE_INPUT`] has to be specified if more compressed data is to be provided +/// in a subsequent call to this function. +/// * See the the [`inflate_flags`] module for details on other flags. +/// +/// # Returns +/// +/// Returns a tuple containing the status of the compressor, the number of input bytes read, and the +/// number of bytes output to `out`. +/// +/// This function shouldn't panic pending any bugs. +pub fn decompress( + r: &mut DecompressorOxide, + in_buf: &[u8], + out: &mut [u8], + out_pos: usize, + flags: u32, +) -> (TINFLStatus, usize, usize) { + let out_buf_size_mask = if flags & TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF != 0 { + usize::max_value() + } else { + // In the case of zero len, any attempt to write would produce HasMoreOutput, + // so to gracefully process the case of there really being no output, + // set the mask to all zeros. + out.len().saturating_sub(1) + }; + + // Ensure the output buffer's size is a power of 2, unless the output buffer + // is large enough to hold the entire output file (in which case it doesn't + // matter). + // Also make sure that the output buffer position is not past the end of the output buffer. + if (out_buf_size_mask.wrapping_add(1) & out_buf_size_mask) != 0 || out_pos > out.len() { + return (TINFLStatus::BadParam, 0, 0); + } + + let mut in_iter = in_buf.iter(); + + let mut state = r.state; + + let mut out_buf = OutputBuffer::from_slice_and_pos(out, out_pos); + + // Make a local copy of the important variables here so we can work with them on the stack. + let mut l = LocalVars { + bit_buf: r.bit_buf, + num_bits: r.num_bits, + dist: r.dist, + counter: r.counter, + num_extra: r.num_extra, + }; + + let mut status = 'state_machine: loop { + match state { + Start => generate_state!(state, 'state_machine, { + l.bit_buf = 0; + l.num_bits = 0; + l.dist = 0; + l.counter = 0; + l.num_extra = 0; + r.z_header0 = 0; + r.z_header1 = 0; + r.z_adler32 = 1; + r.check_adler32 = 1; + if flags & TINFL_FLAG_PARSE_ZLIB_HEADER != 0 { + Action::Jump(State::ReadZlibCmf) + } else { + Action::Jump(State::ReadBlockHeader) + } + }), + + ReadZlibCmf => generate_state!(state, 'state_machine, { + read_byte(&mut in_iter, flags, |cmf| { + r.z_header0 = u32::from(cmf); + Action::Jump(State::ReadZlibFlg) + }) + }), + + ReadZlibFlg => generate_state!(state, 'state_machine, { + read_byte(&mut in_iter, flags, |flg| { + r.z_header1 = u32::from(flg); + validate_zlib_header(r.z_header0, r.z_header1, flags, out_buf_size_mask) + }) + }), + + // Read the block header and jump to the relevant section depending on the block type. + ReadBlockHeader => generate_state!(state, 'state_machine, { + read_bits(&mut l, 3, &mut in_iter, flags, |l, bits| { + r.finish = (bits & 1) as u32; + r.block_type = (bits >> 1) as u32 & 3; + match r.block_type { + 0 => Action::Jump(BlockTypeNoCompression), + 1 => { + start_static_table(r); + init_tree(r, l) + }, + 2 => { + l.counter = 0; + Action::Jump(ReadTableSizes) + }, + 3 => Action::Jump(BlockTypeUnexpected), + _ => unreachable!() + } + }) + }), + + // Raw/Stored/uncompressed block. + BlockTypeNoCompression => generate_state!(state, 'state_machine, { + pad_to_bytes(&mut l, &mut in_iter, flags, |l| { + l.counter = 0; + Action::Jump(RawHeader) + }) + }), + + // Check that the raw block header is correct. + RawHeader => generate_state!(state, 'state_machine, { + if l.counter < 4 { + // Read block length and block length check. + if l.num_bits != 0 { + read_bits(&mut l, 8, &mut in_iter, flags, |l, bits| { + r.raw_header[l.counter as usize] = bits as u8; + l.counter += 1; + Action::None + }) + } else { + read_byte(&mut in_iter, flags, |byte| { + r.raw_header[l.counter as usize] = byte; + l.counter += 1; + Action::None + }) + } + } else { + // Check if the length value of a raw block is correct. + // The 2 first (2-byte) words in a raw header are the length and the + // ones complement of the length. + let length = u16::from(r.raw_header[0]) | (u16::from(r.raw_header[1]) << 8); + let check = u16::from(r.raw_header[2]) | (u16::from(r.raw_header[3]) << 8); + let valid = length == !check; + l.counter = length.into(); + + if !valid { + Action::Jump(BadRawLength) + } else if l.counter == 0 { + // Empty raw block. Sometimes used for synchronization. + Action::Jump(BlockDone) + } else if l.num_bits != 0 { + // There is some data in the bit buffer, so we need to write that first. + Action::Jump(RawReadFirstByte) + } else { + // The bit buffer is empty, so memcpy the rest of the uncompressed data from + // the block. + Action::Jump(RawMemcpy1) + } + } + }), + + // Read the byte from the bit buffer. + RawReadFirstByte => generate_state!(state, 'state_machine, { + read_bits(&mut l, 8, &mut in_iter, flags, |l, bits| { + l.dist = bits as u32; + Action::Jump(RawStoreFirstByte) + }) + }), + + // Write the byte we just read to the output buffer. + RawStoreFirstByte => generate_state!(state, 'state_machine, { + if out_buf.bytes_left() == 0 { + Action::End(TINFLStatus::HasMoreOutput) + } else { + out_buf.write_byte(l.dist as u8); + l.counter -= 1; + if l.counter == 0 || l.num_bits == 0 { + Action::Jump(RawMemcpy1) + } else { + // There is still some data left in the bit buffer that needs to be output. + // TODO: Changed this to jump to `RawReadfirstbyte` rather than + // `RawStoreFirstByte` as that seemed to be the correct path, but this + // needs testing. + Action::Jump(RawReadFirstByte) + } + } + }), + + RawMemcpy1 => generate_state!(state, 'state_machine, { + if l.counter == 0 { + Action::Jump(BlockDone) + } else if out_buf.bytes_left() == 0 { + Action::End(TINFLStatus::HasMoreOutput) + } else { + Action::Jump(RawMemcpy2) + } + }), + + RawMemcpy2 => generate_state!(state, 'state_machine, { + if in_iter.len() > 0 { + // Copy as many raw bytes as possible from the input to the output using memcpy. + // Raw block lengths are limited to 64 * 1024, so casting through usize and u32 + // is not an issue. + let space_left = out_buf.bytes_left(); + let bytes_to_copy = cmp::min(cmp::min( + space_left, + in_iter.len()), + l.counter as usize + ); + + out_buf.write_slice(&in_iter.as_slice()[..bytes_to_copy]); + + (&mut in_iter).nth(bytes_to_copy - 1); + l.counter -= bytes_to_copy as u32; + Action::Jump(RawMemcpy1) + } else { + end_of_input(flags) + } + }), + + // Read how many huffman codes/symbols are used for each table. + ReadTableSizes => generate_state!(state, 'state_machine, { + if l.counter < 3 { + let num_bits = [5, 5, 4][l.counter as usize]; + read_bits(&mut l, num_bits, &mut in_iter, flags, |l, bits| { + r.table_sizes[l.counter as usize] = + bits as u32 + u32::from(MIN_TABLE_SIZES[l.counter as usize]); + l.counter += 1; + Action::None + }) + } else { + memset(&mut r.tables[HUFFLEN_TABLE].code_size[..], 0); + l.counter = 0; + Action::Jump(ReadHufflenTableCodeSize) + } + }), + + // Read the 3-bit lengths of the huffman codes describing the huffman code lengths used + // to decode the lengths of the main tables. + ReadHufflenTableCodeSize => generate_state!(state, 'state_machine, { + if l.counter < r.table_sizes[HUFFLEN_TABLE] { + read_bits(&mut l, 3, &mut in_iter, flags, |l, bits| { + // These lengths are not stored in a normal ascending order, but rather one + // specified by the deflate specification intended to put the most used + // values at the front as trailing zero lengths do not have to be stored. + r.tables[HUFFLEN_TABLE] + .code_size[HUFFMAN_LENGTH_ORDER[l.counter as usize] as usize] = + bits as u8; + l.counter += 1; + Action::None + }) + } else { + r.table_sizes[HUFFLEN_TABLE] = 19; + init_tree(r, &mut l) + } + }), + + ReadLitlenDistTablesCodeSize => generate_state!(state, 'state_machine, { + if l.counter < r.table_sizes[LITLEN_TABLE] + r.table_sizes[DIST_TABLE] { + decode_huffman_code( + r, &mut l, HUFFLEN_TABLE, + flags, &mut in_iter, |r, l, symbol| { + l.dist = symbol as u32; + if l.dist < 16 { + r.len_codes[l.counter as usize] = l.dist as u8; + l.counter += 1; + Action::None + } else if l.dist == 16 && l.counter == 0 { + Action::Jump(BadCodeSizeDistPrevLookup) + } else { + l.num_extra = [2, 3, 7][l.dist as usize - 16]; + Action::Jump(ReadExtraBitsCodeSize) + } + } + ) + } else if l.counter != r.table_sizes[LITLEN_TABLE] + r.table_sizes[DIST_TABLE] { + Action::Jump(BadCodeSizeSum) + } else { + r.tables[LITLEN_TABLE].code_size[..r.table_sizes[LITLEN_TABLE] as usize] + .copy_from_slice(&r.len_codes[..r.table_sizes[LITLEN_TABLE] as usize]); + + let dist_table_start = r.table_sizes[LITLEN_TABLE] as usize; + let dist_table_end = (r.table_sizes[LITLEN_TABLE] + + r.table_sizes[DIST_TABLE]) as usize; + r.tables[DIST_TABLE].code_size[..r.table_sizes[DIST_TABLE] as usize] + .copy_from_slice(&r.len_codes[dist_table_start..dist_table_end]); + + r.block_type -= 1; + init_tree(r, &mut l) + } + }), + + ReadExtraBitsCodeSize => generate_state!(state, 'state_machine, { + let num_extra = l.num_extra; + read_bits(&mut l, num_extra, &mut in_iter, flags, |l, mut extra_bits| { + // Mask to avoid a bounds check. + extra_bits += [3, 3, 11][(l.dist as usize - 16) & 3]; + let val = if l.dist == 16 { + r.len_codes[l.counter as usize - 1] + } else { + 0 + }; + + memset( + &mut r.len_codes[ + l.counter as usize..l.counter as usize + extra_bits as usize + ], + val, + ); + l.counter += extra_bits as u32; + Action::Jump(ReadLitlenDistTablesCodeSize) + }) + }), + + DecodeLitlen => generate_state!(state, 'state_machine, { + if in_iter.len() < 4 || out_buf.bytes_left() < 2 { + // See if we can decode a literal with the data we have left. + // Jumps to next state (WriteSymbol) if successful. + decode_huffman_code( + r, + &mut l, + LITLEN_TABLE, + flags, + &mut in_iter, + |_r, l, symbol| { + l.counter = symbol as u32; + Action::Jump(WriteSymbol) + }, + ) + } else if + // If there is enough space, use the fast inner decompression + // function. + out_buf.bytes_left() >= 259 && + in_iter.len() >= 14 + { + let (status, new_state) = decompress_fast( + r, + &mut in_iter, + &mut out_buf, + flags, + &mut l, + out_buf_size_mask, + ); + + state = new_state; + if status == TINFLStatus::Done { + Action::Jump(new_state) + } else { + Action::End(status) + } + } else { + fill_bit_buffer(&mut l, &mut in_iter); + + if let Some((symbol, code_len)) = r.tables[LITLEN_TABLE].lookup(l.bit_buf) { + + l.counter = symbol as u32; + l.bit_buf >>= code_len; + l.num_bits -= code_len; + + if (l.counter & 256) != 0 { + // The symbol is not a literal. + Action::Jump(HuffDecodeOuterLoop1) + } else { + // If we have a 32-bit buffer we need to read another two bytes now + // to have enough bits to keep going. + if cfg!(not(target_pointer_width = "64")) { + fill_bit_buffer(&mut l, &mut in_iter); + } + + if let Some((symbol, code_len)) = r.tables[LITLEN_TABLE].lookup(l.bit_buf) { + + l.bit_buf >>= code_len; + l.num_bits -= code_len; + // The previous symbol was a literal, so write it directly and check + // the next one. + out_buf.write_byte(l.counter as u8); + if (symbol & 256) != 0 { + l.counter = symbol as u32; + // The symbol is a length value. + Action::Jump(HuffDecodeOuterLoop1) + } else { + // The symbol is a literal, so write it directly and continue. + out_buf.write_byte(symbol as u8); + Action::None + } + } else { + Action::Jump(InvalidCodeLen) + } + } + } else { + Action::Jump(InvalidCodeLen) + } + } + }), + + WriteSymbol => generate_state!(state, 'state_machine, { + if l.counter >= 256 { + Action::Jump(HuffDecodeOuterLoop1) + } else if out_buf.bytes_left() > 0 { + out_buf.write_byte(l.counter as u8); + Action::Jump(DecodeLitlen) + } else { + Action::End(TINFLStatus::HasMoreOutput) + } + }), + + HuffDecodeOuterLoop1 => generate_state!(state, 'state_machine, { + // Mask the top bits since they may contain length info. + l.counter &= 511; + + if l.counter == 256 { + // We hit the end of block symbol. + Action::Jump(BlockDone) + } else if l.counter > 285 { + // Invalid code. + // We already verified earlier that the code is > 256. + Action::Jump(InvalidLitlen) + } else { + // # Optimization + // Mask the value to avoid bounds checks + // We could use get_unchecked later if can statically verify that + // this will never go out of bounds. + l.num_extra = + u32::from(LENGTH_EXTRA[(l.counter - 257) as usize & BASE_EXTRA_MASK]); + l.counter = u32::from(LENGTH_BASE[(l.counter - 257) as usize & BASE_EXTRA_MASK]); + // Length and distance codes have a number of extra bits depending on + // the base, which together with the base gives us the exact value. + if l.num_extra != 0 { + Action::Jump(ReadExtraBitsLitlen) + } else { + Action::Jump(DecodeDistance) + } + } + }), + + ReadExtraBitsLitlen => generate_state!(state, 'state_machine, { + let num_extra = l.num_extra; + read_bits(&mut l, num_extra, &mut in_iter, flags, |l, extra_bits| { + l.counter += extra_bits as u32; + Action::Jump(DecodeDistance) + }) + }), + + DecodeDistance => generate_state!(state, 'state_machine, { + // Try to read a huffman code from the input buffer and look up what + // length code the decoded symbol refers to. + decode_huffman_code(r, &mut l, DIST_TABLE, flags, &mut in_iter, |_r, l, symbol| { + if symbol > 29 { + // Invalid distance code. + return Action::Jump(InvalidDist) + } + // # Optimization + // Mask the value to avoid bounds checks + // We could use get_unchecked later if can statically verify that + // this will never go out of bounds. + l.num_extra = u32::from(DIST_EXTRA[symbol as usize & BASE_EXTRA_MASK]); + l.dist = u32::from(DIST_BASE[symbol as usize & BASE_EXTRA_MASK]); + if l.num_extra != 0 { + // ReadEXTRA_BITS_DISTACNE + Action::Jump(ReadExtraBitsDistance) + } else { + Action::Jump(HuffDecodeOuterLoop2) + } + }) + }), + + ReadExtraBitsDistance => generate_state!(state, 'state_machine, { + let num_extra = l.num_extra; + read_bits(&mut l, num_extra, &mut in_iter, flags, |l, extra_bits| { + l.dist += extra_bits as u32; + Action::Jump(HuffDecodeOuterLoop2) + }) + }), + + HuffDecodeOuterLoop2 => generate_state!(state, 'state_machine, { + if l.dist as usize > out_buf.position() && + (flags & TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF != 0) + { + // We encountered a distance that refers a position before + // the start of the decoded data, so we can't continue. + Action::Jump(DistanceOutOfBounds) + } else { + let out_pos = out_buf.position(); + let source_pos = out_buf.position() + .wrapping_sub(l.dist as usize) & out_buf_size_mask; + + let out_len = out_buf.get_ref().len() as usize; + let match_end_pos = out_buf.position() + l.counter as usize; + + if match_end_pos > out_len || + // miniz doesn't do this check here. Not sure how it makes sure + // that this case doesn't happen. + (source_pos >= out_pos && (source_pos - out_pos) < l.counter as usize) + { + // Not enough space for all of the data in the output buffer, + // so copy what we have space for. + if l.counter == 0 { + Action::Jump(DecodeLitlen) + } else { + Action::Jump(WriteLenBytesToEnd) + } + } else { + apply_match( + out_buf.get_mut(), + out_pos, + l.dist as usize, + l.counter as usize, + out_buf_size_mask + ); + out_buf.set_position(out_pos + l.counter as usize); + Action::Jump(DecodeLitlen) + } + } + }), + + WriteLenBytesToEnd => generate_state!(state, 'state_machine, { + if out_buf.bytes_left() > 0 { + let out_pos = out_buf.position(); + let source_pos = out_buf.position() + .wrapping_sub(l.dist as usize) & out_buf_size_mask; + + + let len = cmp::min(out_buf.bytes_left(), l.counter as usize); + + transfer(out_buf.get_mut(), source_pos, out_pos, len, out_buf_size_mask); + + out_buf.set_position(out_pos + len); + l.counter -= len as u32; + if l.counter == 0 { + Action::Jump(DecodeLitlen) + } else { + Action::None + } + } else { + Action::End(TINFLStatus::HasMoreOutput) + } + }), + + BlockDone => generate_state!(state, 'state_machine, { + // End once we've read the last block. + if r.finish != 0 { + pad_to_bytes(&mut l, &mut in_iter, flags, |_| Action::None); + + let in_consumed = in_buf.len() - in_iter.len(); + let undo = undo_bytes(&mut l, in_consumed as u32) as usize; + in_iter = in_buf[in_consumed - undo..].iter(); + + l.bit_buf &= ((1 as BitBuffer) << l.num_bits) - 1; + debug_assert_eq!(l.num_bits, 0); + + if flags & TINFL_FLAG_PARSE_ZLIB_HEADER != 0 { + l.counter = 0; + Action::Jump(ReadAdler32) + } else { + Action::Jump(DoneForever) + } + } else { + Action::Jump(ReadBlockHeader) + } + }), + + ReadAdler32 => generate_state!(state, 'state_machine, { + if l.counter < 4 { + if l.num_bits != 0 { + read_bits(&mut l, 8, &mut in_iter, flags, |l, bits| { + r.z_adler32 <<= 8; + r.z_adler32 |= bits as u32; + l.counter += 1; + Action::None + }) + } else { + read_byte(&mut in_iter, flags, |byte| { + r.z_adler32 <<= 8; + r.z_adler32 |= u32::from(byte); + l.counter += 1; + Action::None + }) + } + } else { + Action::Jump(DoneForever) + } + }), + + // We are done. + DoneForever => break TINFLStatus::Done, + + // Anything else indicates failure. + // BadZlibHeader | BadRawLength | BlockTypeUnexpected | DistanceOutOfBounds | + // BadTotalSymbols | BadCodeSizeDistPrevLookup | BadCodeSizeSum | InvalidLitlen | + // InvalidDist | InvalidCodeLen + _ => break TINFLStatus::Failed, + }; + }; + + let in_undo = if status != TINFLStatus::NeedsMoreInput + && status != TINFLStatus::FailedCannotMakeProgress + { + undo_bytes(&mut l, (in_buf.len() - in_iter.len()) as u32) as usize + } else { + 0 + }; + + // Make sure HasMoreOutput overrides NeedsMoreInput if the output buffer is full. + // (Unless the missing input is the adler32 value in which case we don't need to write anything.) + // TODO: May want to see if we can do this in a better way. + if status == TINFLStatus::NeedsMoreInput + && out_buf.bytes_left() == 0 + && state != State::ReadAdler32 + { + status = TINFLStatus::HasMoreOutput + } + + r.state = state; + r.bit_buf = l.bit_buf; + r.num_bits = l.num_bits; + r.dist = l.dist; + r.counter = l.counter; + r.num_extra = l.num_extra; + + r.bit_buf &= ((1 as BitBuffer) << r.num_bits) - 1; + + // If this is a zlib stream, and update the adler32 checksum with the decompressed bytes if + // requested. + let need_adler = if (flags & TINFL_FLAG_IGNORE_ADLER32) == 0 { + flags & (TINFL_FLAG_PARSE_ZLIB_HEADER | TINFL_FLAG_COMPUTE_ADLER32) != 0 + } else { + // If TINFL_FLAG_IGNORE_ADLER32 is enabled, ignore the checksum. + false + }; + if need_adler && status as i32 >= 0 { + let out_buf_pos = out_buf.position(); + r.check_adler32 = update_adler32(r.check_adler32, &out_buf.get_ref()[out_pos..out_buf_pos]); + + // disabled so that random input from fuzzer would not be rejected early, + // before it has a chance to reach interesting parts of code + if !cfg!(fuzzing) { + // Once we are done, check if the checksum matches with the one provided in the zlib header. + if status == TINFLStatus::Done + && flags & TINFL_FLAG_PARSE_ZLIB_HEADER != 0 + && r.check_adler32 != r.z_adler32 + { + status = TINFLStatus::Adler32Mismatch; + } + } + } + + ( + status, + in_buf.len() - in_iter.len() - in_undo, + out_buf.position() - out_pos, + ) +} + +#[cfg(test)] +mod test { + use super::*; + + //TODO: Fix these. + + fn tinfl_decompress_oxide<'i>( + r: &mut DecompressorOxide, + input_buffer: &'i [u8], + output_buffer: &mut [u8], + flags: u32, + ) -> (TINFLStatus, &'i [u8], usize) { + let (status, in_pos, out_pos) = decompress(r, input_buffer, output_buffer, 0, flags); + (status, &input_buffer[in_pos..], out_pos) + } + + #[test] + fn decompress_zlib() { + let encoded = [ + 120, 156, 243, 72, 205, 201, 201, 215, 81, 168, 202, 201, 76, 82, 4, 0, 27, 101, 4, 19, + ]; + let flags = TINFL_FLAG_COMPUTE_ADLER32 | TINFL_FLAG_PARSE_ZLIB_HEADER; + + let mut b = DecompressorOxide::new(); + const LEN: usize = 32; + let mut b_buf = vec![0; LEN]; + + // This should fail with the out buffer being to small. + let b_status = tinfl_decompress_oxide(&mut b, &encoded[..], b_buf.as_mut_slice(), flags); + + assert_eq!(b_status.0, TINFLStatus::Failed); + + let flags = flags | TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF; + + b = DecompressorOxide::new(); + + // With TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF set this should no longer fail. + let b_status = tinfl_decompress_oxide(&mut b, &encoded[..], b_buf.as_mut_slice(), flags); + + assert_eq!(b_buf[..b_status.2], b"Hello, zlib!"[..]); + assert_eq!(b_status.0, TINFLStatus::Done); + } + + #[test] + fn raw_block() { + const LEN: usize = 64; + + let text = b"Hello, zlib!"; + let encoded = { + let len = text.len(); + let notlen = !len; + let mut encoded = vec![ + 1, + len as u8, + (len >> 8) as u8, + notlen as u8, + (notlen >> 8) as u8, + ]; + encoded.extend_from_slice(&text[..]); + encoded + }; + + //let flags = TINFL_FLAG_COMPUTE_ADLER32 | TINFL_FLAG_PARSE_ZLIB_HEADER | + let flags = TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF; + + let mut b = DecompressorOxide::new(); + + let mut b_buf = vec![0; LEN]; + + let b_status = tinfl_decompress_oxide(&mut b, &encoded[..], b_buf.as_mut_slice(), flags); + assert_eq!(b_buf[..b_status.2], text[..]); + assert_eq!(b_status.0, TINFLStatus::Done); + } + + fn masked_lookup(table: &HuffmanTable, bit_buf: BitBuffer) -> (i32, u32) { + let ret = table.lookup(bit_buf).unwrap(); + (ret.0 & 511, ret.1) + } + + #[test] + fn fixed_table_lookup() { + let mut d = DecompressorOxide::new(); + d.block_type = 1; + start_static_table(&mut d); + let mut l = LocalVars { + bit_buf: d.bit_buf, + num_bits: d.num_bits, + dist: d.dist, + counter: d.counter, + num_extra: d.num_extra, + }; + init_tree(&mut d, &mut l); + let llt = &d.tables[LITLEN_TABLE]; + let dt = &d.tables[DIST_TABLE]; + assert_eq!(masked_lookup(llt, 0b00001100), (0, 8)); + assert_eq!(masked_lookup(llt, 0b00011110), (72, 8)); + assert_eq!(masked_lookup(llt, 0b01011110), (74, 8)); + assert_eq!(masked_lookup(llt, 0b11111101), (143, 8)); + assert_eq!(masked_lookup(llt, 0b000010011), (144, 9)); + assert_eq!(masked_lookup(llt, 0b111111111), (255, 9)); + assert_eq!(masked_lookup(llt, 0b00000000), (256, 7)); + assert_eq!(masked_lookup(llt, 0b1110100), (279, 7)); + assert_eq!(masked_lookup(llt, 0b00000011), (280, 8)); + assert_eq!(masked_lookup(llt, 0b11100011), (287, 8)); + + assert_eq!(masked_lookup(dt, 0), (0, 5)); + assert_eq!(masked_lookup(dt, 20), (5, 5)); + } + + fn check_result(input: &[u8], expected_status: TINFLStatus, expected_state: State, zlib: bool) { + let mut r = DecompressorOxide::default(); + let mut output_buf = vec![0; 1024 * 32]; + let flags = if zlib { + inflate_flags::TINFL_FLAG_PARSE_ZLIB_HEADER + } else { + 0 + } | TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF + | TINFL_FLAG_HAS_MORE_INPUT; + let (d_status, _in_bytes, _out_bytes) = + decompress(&mut r, input, &mut output_buf, 0, flags); + assert_eq!(expected_status, d_status); + assert_eq!(expected_state, r.state); + } + + #[test] + fn bogus_input() { + use self::check_result as cr; + const F: TINFLStatus = TINFLStatus::Failed; + const OK: TINFLStatus = TINFLStatus::Done; + // Bad CM. + cr(&[0x77, 0x85], F, State::BadZlibHeader, true); + // Bad window size (but check is correct). + cr(&[0x88, 0x98], F, State::BadZlibHeader, true); + // Bad check bits. + cr(&[0x78, 0x98], F, State::BadZlibHeader, true); + + // Too many code lengths. (From inflate library issues) + cr( + b"M\xff\xffM*\xad\xad\xad\xad\xad\xad\xad\xcd\xcd\xcdM", + F, + State::BadTotalSymbols, + false, + ); + // Bad CLEN (also from inflate library issues) + cr( + b"\xdd\xff\xff*M\x94ffffffffff", + F, + State::BadTotalSymbols, + false, + ); + + // Port of inflate coverage tests from zlib-ng + // https://github.com/Dead2/zlib-ng/blob/develop/test/infcover.c + let c = |a, b, c| cr(a, b, c, false); + + // Invalid uncompressed/raw block length. + c(&[0, 0, 0, 0, 0], F, State::BadRawLength); + // Ok empty uncompressed block. + c(&[3, 0], OK, State::DoneForever); + // Invalid block type. + c(&[6], F, State::BlockTypeUnexpected); + // Ok uncompressed block. + c(&[1, 1, 0, 0xfe, 0xff, 0], OK, State::DoneForever); + // Too many litlens, we handle this later than zlib, so this test won't + // give the same result. + // c(&[0xfc, 0, 0], F, State::BadTotalSymbols); + // Invalid set of code lengths - TODO Check if this is the correct error for this. + c(&[4, 0, 0xfe, 0xff], F, State::BadTotalSymbols); + // Invalid repeat in list of code lengths. + // (Try to repeat a non-existant code.) + c(&[4, 0, 0x24, 0x49, 0], F, State::BadCodeSizeDistPrevLookup); + // Missing end of block code (should we have a separate error for this?) - fails on futher input + // c(&[4, 0, 0x24, 0xe9, 0xff, 0x6d], F, State::BadTotalSymbols); + // Invalid set of literals/lengths + c( + &[ + 4, 0x80, 0x49, 0x92, 0x24, 0x49, 0x92, 0x24, 0x71, 0xff, 0xff, 0x93, 0x11, 0, + ], + F, + State::BadTotalSymbols, + ); + // Invalid set of distances _ needsmoreinput + // c(&[4, 0x80, 0x49, 0x92, 0x24, 0x49, 0x92, 0x24, 0x0f, 0xb4, 0xff, 0xff, 0xc3, 0x84], F, State::BadTotalSymbols); + // Invalid distance code + c(&[2, 0x7e, 0xff, 0xff], F, State::InvalidDist); + + // Distance refers to position before the start + c( + &[0x0c, 0xc0, 0x81, 0, 0, 0, 0, 0, 0x90, 0xff, 0x6b, 0x4, 0], + F, + State::DistanceOutOfBounds, + ); + + // Trailer + // Bad gzip trailer checksum GZip header not handled by miniz_oxide + //cr(&[0x1f, 0x8b, 0x08 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0x03, 0, 0, 0, 0, 0x01], F, State::BadCRC, false) + // Bad gzip trailer length + //cr(&[0x1f, 0x8b, 0x08 ,0 ,0 ,0 ,0 ,0 ,0 ,0 ,0x03, 0, 0, 0, 0, 0, 0, 0, 0, 0x01], F, State::BadCRC, false) + } + + #[test] + fn empty_output_buffer_non_wrapping() { + let encoded = [ + 120, 156, 243, 72, 205, 201, 201, 215, 81, 168, 202, 201, 76, 82, 4, 0, 27, 101, 4, 19, + ]; + let flags = TINFL_FLAG_COMPUTE_ADLER32 + | TINFL_FLAG_PARSE_ZLIB_HEADER + | TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF; + let mut r = DecompressorOxide::new(); + let mut output_buf = vec![]; + // Check that we handle an empty buffer properly and not panicking. + // https://github.com/Frommi/miniz_oxide/issues/23 + let res = decompress(&mut r, &encoded, &mut output_buf, 0, flags); + assert_eq!(res, (TINFLStatus::HasMoreOutput, 4, 0)); + } + + #[test] + fn empty_output_buffer_wrapping() { + let encoded = [ + 0x73, 0x49, 0x4d, 0xcb, 0x49, 0x2c, 0x49, 0x55, 0x00, 0x11, 0x00, + ]; + let flags = TINFL_FLAG_COMPUTE_ADLER32; + let mut r = DecompressorOxide::new(); + let mut output_buf = vec![]; + // Check that we handle an empty buffer properly and not panicking. + // https://github.com/Frommi/miniz_oxide/issues/23 + let res = decompress(&mut r, &encoded, &mut output_buf, 0, flags); + assert_eq!(res, (TINFLStatus::HasMoreOutput, 2, 0)); + } +} diff --git a/crux-mir/lib/miniz_oxide/src/inflate/mod.rs b/crux-mir/lib/miniz_oxide/src/inflate/mod.rs new file mode 100644 index 000000000..535392327 --- /dev/null +++ b/crux-mir/lib/miniz_oxide/src/inflate/mod.rs @@ -0,0 +1,279 @@ +//! This module contains functionality for decompression. + +use crate::alloc::boxed::Box; +use crate::alloc::vec; +use crate::alloc::vec::Vec; +use ::core::cmp::min; +use ::core::usize; + +pub mod core; +mod output_buffer; +pub mod stream; +use self::core::*; + +const TINFL_STATUS_FAILED_CANNOT_MAKE_PROGRESS: i32 = -4; +const TINFL_STATUS_BAD_PARAM: i32 = -3; +const TINFL_STATUS_ADLER32_MISMATCH: i32 = -2; +const TINFL_STATUS_FAILED: i32 = -1; +const TINFL_STATUS_DONE: i32 = 0; +const TINFL_STATUS_NEEDS_MORE_INPUT: i32 = 1; +const TINFL_STATUS_HAS_MORE_OUTPUT: i32 = 2; + +/// Return status codes. +#[repr(i8)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum TINFLStatus { + /// More input data was expected, but the caller indicated that there was no more data, so the + /// input stream is likely truncated. + /// + /// This can't happen if you have provided the + /// [`TINFL_FLAG_HAS_MORE_INPUT`][core::inflate_flags::TINFL_FLAG_HAS_MORE_INPUT] flag to the + /// decompression. By setting that flag, you indicate more input exists but is not provided, + /// and so reaching the end of the input data without finding the end of the compressed stream + /// would instead return a [`NeedsMoreInput`][Self::NeedsMoreInput] status. + FailedCannotMakeProgress = TINFL_STATUS_FAILED_CANNOT_MAKE_PROGRESS as i8, + + /// The output buffer is an invalid size; consider the `flags` parameter. + BadParam = TINFL_STATUS_BAD_PARAM as i8, + + /// The decompression went fine, but the adler32 checksum did not match the one + /// provided in the header. + Adler32Mismatch = TINFL_STATUS_ADLER32_MISMATCH as i8, + + /// Failed to decompress due to invalid data. + Failed = TINFL_STATUS_FAILED as i8, + + /// Finished decompression without issues. + /// + /// This indicates the end of the compressed stream has been reached. + Done = TINFL_STATUS_DONE as i8, + + /// The decompressor needs more input data to continue decompressing. + /// + /// This occurs when there's no more consumable input, but the end of the stream hasn't been + /// reached, and you have supplied the + /// [`TINFL_FLAG_HAS_MORE_INPUT`][core::inflate_flags::TINFL_FLAG_HAS_MORE_INPUT] flag to the + /// decompressor. Had you not supplied that flag (which would mean you were asserting that you + /// believed all the data was available) you would have gotten a + /// [`FailedCannotMakeProcess`][Self::FailedCannotMakeProgress] instead. + NeedsMoreInput = TINFL_STATUS_NEEDS_MORE_INPUT as i8, + + /// There is still pending data that didn't fit in the output buffer. + HasMoreOutput = TINFL_STATUS_HAS_MORE_OUTPUT as i8, +} + +impl TINFLStatus { + pub fn from_i32(value: i32) -> Option { + use self::TINFLStatus::*; + match value { + TINFL_STATUS_FAILED_CANNOT_MAKE_PROGRESS => Some(FailedCannotMakeProgress), + TINFL_STATUS_BAD_PARAM => Some(BadParam), + TINFL_STATUS_ADLER32_MISMATCH => Some(Adler32Mismatch), + TINFL_STATUS_FAILED => Some(Failed), + TINFL_STATUS_DONE => Some(Done), + TINFL_STATUS_NEEDS_MORE_INPUT => Some(NeedsMoreInput), + TINFL_STATUS_HAS_MORE_OUTPUT => Some(HasMoreOutput), + _ => None, + } + } +} + +/// Decompress the deflate-encoded data in `input` to a vector. +/// +/// Returns a tuple of the [`Vec`] of decompressed data and the [status result][TINFLStatus]. +#[inline] +pub fn decompress_to_vec(input: &[u8]) -> Result, TINFLStatus> { + decompress_to_vec_inner(input, 0, usize::max_value()) +} + +/// Decompress the deflate-encoded data (with a zlib wrapper) in `input` to a vector. +/// +/// Returns a tuple of the [`Vec`] of decompressed data and the [status result][TINFLStatus]. +#[inline] +pub fn decompress_to_vec_zlib(input: &[u8]) -> Result, TINFLStatus> { + decompress_to_vec_inner( + input, + inflate_flags::TINFL_FLAG_PARSE_ZLIB_HEADER, + usize::max_value(), + ) +} + +/// Decompress the deflate-encoded data in `input` to a vector. +/// The vector is grown to at most `max_size` bytes; if the data does not fit in that size, +/// [`TINFLStatus::HasMoreOutput`] error is returned. +/// +/// Returns a tuple of the [`Vec`] of decompressed data and the [status result][TINFLStatus]. +#[inline] +pub fn decompress_to_vec_with_limit(input: &[u8], max_size: usize) -> Result, TINFLStatus> { + decompress_to_vec_inner(input, 0, max_size) +} + +/// Decompress the deflate-encoded data (with a zlib wrapper) in `input` to a vector. +/// The vector is grown to at most `max_size` bytes; if the data does not fit in that size, +/// [`TINFLStatus::HasMoreOutput`] error is returned. +/// +/// Returns a tuple of the [`Vec`] of decompressed data and the [status result][TINFLStatus]. +#[inline] +pub fn decompress_to_vec_zlib_with_limit( + input: &[u8], + max_size: usize, +) -> Result, TINFLStatus> { + decompress_to_vec_inner(input, inflate_flags::TINFL_FLAG_PARSE_ZLIB_HEADER, max_size) +} + +/// Backend of various to-[`Vec`] decompressions. +/// +/// Returns a tuple of the [`Vec`] of decompressed data and the [status result][TINFLStatus]. +fn decompress_to_vec_inner( + input: &[u8], + flags: u32, + max_output_size: usize, +) -> Result, TINFLStatus> { + let flags = flags | inflate_flags::TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF; + let mut ret: Vec = vec![0; min(input.len().saturating_mul(2), max_output_size)]; + + let mut decomp = Box::::default(); + + let mut in_pos = 0; + let mut out_pos = 0; + loop { + // Wrap the whole output slice so we know we have enough of the + // decompressed data for matches. + let (status, in_consumed, out_consumed) = + decompress(&mut decomp, &input[in_pos..], &mut ret, out_pos, flags); + in_pos += in_consumed; + out_pos += out_consumed; + + match status { + TINFLStatus::Done => { + ret.truncate(out_pos); + return Ok(ret); + } + + TINFLStatus::HasMoreOutput => { + // We need more space, so check if we can resize the buffer and do it. + let new_len = ret + .len() + .checked_add(out_pos) + .ok_or(TINFLStatus::HasMoreOutput)?; + if new_len > max_output_size { + return Err(TINFLStatus::HasMoreOutput); + }; + ret.resize(new_len, 0); + } + + _ => return Err(status), + } + } +} + +/// Decompress one or more source slices from an iterator into the output slice. +/// +/// * On success, returns the number of bytes that were written. +/// * On failure, returns the failure status code. +/// +/// This will fail if the output buffer is not large enough, but in that case +/// the output buffer will still contain the partial decompression. +/// +/// * `out` the output buffer. +/// * `it` the iterator of input slices. +/// * `zlib_header` if the first slice out of the iterator is expected to have a +/// Zlib header. Otherwise the slices are assumed to be the deflate data only. +/// * `ignore_adler32` if the adler32 checksum should be calculated or not. +pub fn decompress_slice_iter_to_slice<'out, 'inp>( + out: &'out mut [u8], + it: impl Iterator, + zlib_header: bool, + ignore_adler32: bool, +) -> Result { + use self::core::inflate_flags::*; + + let mut it = it.peekable(); + let r = &mut DecompressorOxide::new(); + let mut out_pos = 0; + while let Some(in_buf) = it.next() { + let has_more = it.peek().is_some(); + let flags = { + let mut f = TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF; + if zlib_header { + f |= TINFL_FLAG_PARSE_ZLIB_HEADER; + } + if ignore_adler32 { + f |= TINFL_FLAG_IGNORE_ADLER32; + } + if has_more { + f |= TINFL_FLAG_HAS_MORE_INPUT; + } + f + }; + let (status, _input_read, bytes_written) = decompress(r, in_buf, out, out_pos, flags); + out_pos += bytes_written; + match status { + TINFLStatus::NeedsMoreInput => continue, + TINFLStatus::Done => return Ok(out_pos), + e => return Err(e), + } + } + // If we ran out of source slices without getting a `Done` from the + // decompression we can call it a failure. + Err(TINFLStatus::FailedCannotMakeProgress) +} + +#[cfg(test)] +mod test { + use super::{ + decompress_slice_iter_to_slice, decompress_to_vec_zlib, decompress_to_vec_zlib_with_limit, + TINFLStatus, + }; + const ENCODED: [u8; 20] = [ + 120, 156, 243, 72, 205, 201, 201, 215, 81, 168, 202, 201, 76, 82, 4, 0, 27, 101, 4, 19, + ]; + + #[test] + fn decompress_vec() { + let res = decompress_to_vec_zlib(&ENCODED[..]).unwrap(); + assert_eq!(res.as_slice(), &b"Hello, zlib!"[..]); + } + + #[test] + fn decompress_vec_with_high_limit() { + let res = decompress_to_vec_zlib_with_limit(&ENCODED[..], 100_000).unwrap(); + assert_eq!(res.as_slice(), &b"Hello, zlib!"[..]); + } + + #[test] + fn fail_to_decompress_with_limit() { + let res = decompress_to_vec_zlib_with_limit(&ENCODED[..], 8); + match res { + Err(TINFLStatus::HasMoreOutput) => (), // expected result + _ => panic!("Decompression output size limit was not enforced"), + } + } + + #[test] + fn test_decompress_slice_iter_to_slice() { + // one slice + let mut out = [0_u8; 12_usize]; + let r = + decompress_slice_iter_to_slice(&mut out, Some(&ENCODED[..]).into_iter(), true, false); + assert_eq!(r, Ok(12)); + assert_eq!(&out[..12], &b"Hello, zlib!"[..]); + + // some chunks at a time + for chunk_size in 1..13 { + // Note: because of https://github.com/Frommi/miniz_oxide/issues/110 our + // out buffer needs to have +1 byte available when the chunk size cuts + // the adler32 data off from the last actual data. + let mut out = [0_u8; 12_usize + 1]; + let r = + decompress_slice_iter_to_slice(&mut out, ENCODED.chunks(chunk_size), true, false); + assert_eq!(r, Ok(12)); + assert_eq!(&out[..12], &b"Hello, zlib!"[..]); + } + + // output buffer too small + let mut out = [0_u8; 3_usize]; + let r = decompress_slice_iter_to_slice(&mut out, ENCODED.chunks(7), true, false); + assert!(r.is_err()); + } +} diff --git a/crux-mir/lib/miniz_oxide/src/inflate/output_buffer.rs b/crux-mir/lib/miniz_oxide/src/inflate/output_buffer.rs new file mode 100644 index 000000000..5218a807d --- /dev/null +++ b/crux-mir/lib/miniz_oxide/src/inflate/output_buffer.rs @@ -0,0 +1,60 @@ +/// A wrapper for the output slice used when decompressing. +/// +/// Using this rather than `Cursor` lets us implement the writing methods directly on +/// the buffer and lets us use a usize rather than u64 for the position which helps with +/// performance on 32-bit systems. +pub struct OutputBuffer<'a> { + slice: &'a mut [u8], + position: usize, +} + +impl<'a> OutputBuffer<'a> { + #[inline] + pub fn from_slice_and_pos(slice: &'a mut [u8], position: usize) -> OutputBuffer<'a> { + OutputBuffer { slice, position } + } + + #[inline] + pub const fn position(&self) -> usize { + self.position + } + + #[inline] + pub fn set_position(&mut self, position: usize) { + self.position = position; + } + + /// Write a byte to the current position and increment + /// + /// Assumes that there is space. + #[inline] + pub fn write_byte(&mut self, byte: u8) { + self.slice[self.position] = byte; + self.position += 1; + } + + /// Write a slice to the current position and increment + /// + /// Assumes that there is space. + #[inline] + pub fn write_slice(&mut self, data: &[u8]) { + let len = data.len(); + self.slice[self.position..self.position + len].copy_from_slice(data); + self.position += data.len(); + } + + #[inline] + pub const fn bytes_left(&self) -> usize { + self.slice.len() - self.position + } + + #[inline] + pub const fn get_ref(&self) -> &[u8] { + self.slice + } + + #[inline] + pub fn get_mut(&mut self) -> &mut [u8] { + self.slice + } +} diff --git a/crux-mir/lib/miniz_oxide/src/inflate/stream.rs b/crux-mir/lib/miniz_oxide/src/inflate/stream.rs new file mode 100644 index 000000000..715747166 --- /dev/null +++ b/crux-mir/lib/miniz_oxide/src/inflate/stream.rs @@ -0,0 +1,415 @@ +//! Extra streaming decompression functionality. +//! +//! As of now this is mainly intended for use to build a higher-level wrapper. +use crate::alloc::boxed::Box; +use core::{cmp, mem}; + +use crate::inflate::core::{decompress, inflate_flags, DecompressorOxide, TINFL_LZ_DICT_SIZE}; +use crate::inflate::TINFLStatus; +use crate::{DataFormat, MZError, MZFlush, MZResult, MZStatus, StreamResult}; + +/// Tag that determines reset policy of [InflateState](struct.InflateState.html) +pub trait ResetPolicy { + /// Performs reset + fn reset(&self, state: &mut InflateState); +} + +/// Resets state, without performing expensive ops (e.g. zeroing buffer) +/// +/// Note that not zeroing buffer can lead to security issues when dealing with untrusted input. +pub struct MinReset; + +impl ResetPolicy for MinReset { + fn reset(&self, state: &mut InflateState) { + state.decompressor().init(); + state.dict_ofs = 0; + state.dict_avail = 0; + state.first_call = true; + state.has_flushed = false; + state.last_status = TINFLStatus::NeedsMoreInput; + } +} + +/// Resets state and zero memory, continuing to use the same data format. +pub struct ZeroReset; + +impl ResetPolicy for ZeroReset { + #[inline] + fn reset(&self, state: &mut InflateState) { + MinReset.reset(state); + state.dict = [0; TINFL_LZ_DICT_SIZE]; + } +} + +/// Full reset of the state, including zeroing memory. +/// +/// Requires to provide new data format. +pub struct FullReset(pub DataFormat); + +impl ResetPolicy for FullReset { + #[inline] + fn reset(&self, state: &mut InflateState) { + ZeroReset.reset(state); + state.data_format = self.0; + } +} + +/// A struct that compbines a decompressor with extra data for streaming decompression. +/// +pub struct InflateState { + /// Inner decompressor struct + decomp: DecompressorOxide, + + /// Buffer of input bytes for matches. + /// TODO: Could probably do this a bit cleaner with some + /// Cursor-like class. + /// We may also look into whether we need to keep a buffer here, or just one in the + /// decompressor struct. + dict: [u8; TINFL_LZ_DICT_SIZE], + /// Where in the buffer are we currently at? + dict_ofs: usize, + /// How many bytes of data to be flushed is there currently in the buffer? + dict_avail: usize, + + first_call: bool, + has_flushed: bool, + + /// Whether the input data is wrapped in a zlib header and checksum. + /// TODO: This should be stored in the decompressor. + data_format: DataFormat, + last_status: TINFLStatus, +} + +impl Default for InflateState { + fn default() -> Self { + InflateState { + decomp: DecompressorOxide::default(), + dict: [0; TINFL_LZ_DICT_SIZE], + dict_ofs: 0, + dict_avail: 0, + first_call: true, + has_flushed: false, + data_format: DataFormat::Raw, + last_status: TINFLStatus::NeedsMoreInput, + } + } +} +impl InflateState { + /// Create a new state. + /// + /// Note that this struct is quite large due to internal buffers, and as such storing it on + /// the stack is not recommended. + /// + /// # Parameters + /// `data_format`: Determines whether the compressed data is assumed to wrapped with zlib + /// metadata. + pub fn new(data_format: DataFormat) -> InflateState { + InflateState { + data_format, + ..Default::default() + } + } + + /// Create a new state on the heap. + /// + /// # Parameters + /// `data_format`: Determines whether the compressed data is assumed to wrapped with zlib + /// metadata. + pub fn new_boxed(data_format: DataFormat) -> Box { + let mut b: Box = Box::default(); + b.data_format = data_format; + b + } + + /// Access the innner decompressor. + pub fn decompressor(&mut self) -> &mut DecompressorOxide { + &mut self.decomp + } + + /// Return the status of the last call to `inflate` with this `InflateState`. + pub const fn last_status(&self) -> TINFLStatus { + self.last_status + } + + /// Create a new state using miniz/zlib style window bits parameter. + /// + /// The decompressor does not support different window sizes. As such, + /// any positive (>0) value will set the zlib header flag, while a negative one + /// will not. + pub fn new_boxed_with_window_bits(window_bits: i32) -> Box { + let mut b: Box = Box::default(); + b.data_format = DataFormat::from_window_bits(window_bits); + b + } + + #[inline] + /// Reset the decompressor without re-allocating memory, using the given + /// data format. + pub fn reset(&mut self, data_format: DataFormat) { + self.reset_as(FullReset(data_format)); + } + + #[inline] + /// Resets the state according to specified policy. + pub fn reset_as(&mut self, policy: T) { + policy.reset(self) + } +} + +/// Try to decompress from `input` to `output` with the given [`InflateState`] +/// +/// # `flush` +/// +/// Generally, the various [`MZFlush`] flags have meaning only on the compression side. They can be +/// supplied here, but the only one that has any semantic meaning is [`MZFlush::Finish`], which is a +/// signal that the stream is expected to finish, and failing to do so is an error. It isn't +/// necessary to specify it when the stream ends; you'll still get returned a +/// [`MZStatus::StreamEnd`] anyway. Other values either have no effect or cause errors. It's +/// likely that you'll almost always just want to use [`MZFlush::None`]. +/// +/// # Errors +/// +/// Returns [`MZError::Buf`] if the size of the `output` slice is empty or no progress was made due +/// to lack of expected input data, or if called with [`MZFlush::Finish`] and input wasn't all +/// consumed. +/// +/// Returns [`MZError::Data`] if this or a a previous call failed with an error return from +/// [`TINFLStatus`]; probably indicates corrupted data. +/// +/// Returns [`MZError::Stream`] when called with [`MZFlush::Full`] (meaningless on +/// decompression), or when called without [`MZFlush::Finish`] after an earlier call with +/// [`MZFlush::Finish`] has been made. +pub fn inflate( + state: &mut InflateState, + input: &[u8], + output: &mut [u8], + flush: MZFlush, +) -> StreamResult { + let mut bytes_consumed = 0; + let mut bytes_written = 0; + let mut next_in = input; + let mut next_out = output; + + if flush == MZFlush::Full { + return StreamResult::error(MZError::Stream); + } + + let mut decomp_flags = if state.data_format == DataFormat::Zlib { + inflate_flags::TINFL_FLAG_COMPUTE_ADLER32 + } else { + inflate_flags::TINFL_FLAG_IGNORE_ADLER32 + }; + + if (state.data_format == DataFormat::Zlib) + | (state.data_format == DataFormat::ZLibIgnoreChecksum) + { + decomp_flags |= inflate_flags::TINFL_FLAG_PARSE_ZLIB_HEADER; + } + + let first_call = state.first_call; + state.first_call = false; + if (state.last_status as i32) < 0 { + return StreamResult::error(MZError::Data); + } + + if state.has_flushed && (flush != MZFlush::Finish) { + return StreamResult::error(MZError::Stream); + } + state.has_flushed |= flush == MZFlush::Finish; + + if (flush == MZFlush::Finish) && first_call { + decomp_flags |= inflate_flags::TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF; + + let status = decompress(&mut state.decomp, next_in, next_out, 0, decomp_flags); + let in_bytes = status.1; + let out_bytes = status.2; + let status = status.0; + + state.last_status = status; + + bytes_consumed += in_bytes; + bytes_written += out_bytes; + + let ret_status = { + if (status as i32) < 0 { + Err(MZError::Data) + } else if status != TINFLStatus::Done { + state.last_status = TINFLStatus::Failed; + Err(MZError::Buf) + } else { + Ok(MZStatus::StreamEnd) + } + }; + return StreamResult { + bytes_consumed, + bytes_written, + status: ret_status, + }; + } + + if flush != MZFlush::Finish { + decomp_flags |= inflate_flags::TINFL_FLAG_HAS_MORE_INPUT; + } + + if state.dict_avail != 0 { + bytes_written += push_dict_out(state, &mut next_out); + return StreamResult { + bytes_consumed, + bytes_written, + status: Ok( + if (state.last_status == TINFLStatus::Done) && (state.dict_avail == 0) { + MZStatus::StreamEnd + } else { + MZStatus::Ok + }, + ), + }; + } + + let status = inflate_loop( + state, + &mut next_in, + &mut next_out, + &mut bytes_consumed, + &mut bytes_written, + decomp_flags, + flush, + ); + StreamResult { + bytes_consumed, + bytes_written, + status, + } +} + +fn inflate_loop( + state: &mut InflateState, + next_in: &mut &[u8], + next_out: &mut &mut [u8], + total_in: &mut usize, + total_out: &mut usize, + decomp_flags: u32, + flush: MZFlush, +) -> MZResult { + let orig_in_len = next_in.len(); + loop { + let status = decompress( + &mut state.decomp, + *next_in, + &mut state.dict, + state.dict_ofs, + decomp_flags, + ); + + let in_bytes = status.1; + let out_bytes = status.2; + let status = status.0; + + state.last_status = status; + + *next_in = &next_in[in_bytes..]; + *total_in += in_bytes; + + state.dict_avail = out_bytes; + *total_out += push_dict_out(state, next_out); + + // The stream was corrupted, and decompression failed. + if (status as i32) < 0 { + return Err(MZError::Data); + } + + // The decompressor has flushed all it's data and is waiting for more input, but + // there was no more input provided. + if (status == TINFLStatus::NeedsMoreInput) && orig_in_len == 0 { + return Err(MZError::Buf); + } + + if flush == MZFlush::Finish { + if status == TINFLStatus::Done { + // There is not enough space in the output buffer to flush the remaining + // decompressed data in the internal buffer. + return if state.dict_avail != 0 { + Err(MZError::Buf) + } else { + Ok(MZStatus::StreamEnd) + }; + // No more space in the output buffer, but we're not done. + } else if next_out.is_empty() { + return Err(MZError::Buf); + } + } else { + // We're not expected to finish, so it's fine if we can't flush everything yet. + let empty_buf = next_in.is_empty() || next_out.is_empty(); + if (status == TINFLStatus::Done) || empty_buf || (state.dict_avail != 0) { + return if (status == TINFLStatus::Done) && (state.dict_avail == 0) { + // No more data left, we're done. + Ok(MZStatus::StreamEnd) + } else { + // Ok for now, still waiting for more input data or output space. + Ok(MZStatus::Ok) + }; + } + } + } +} + +fn push_dict_out(state: &mut InflateState, next_out: &mut &mut [u8]) -> usize { + let n = cmp::min(state.dict_avail as usize, next_out.len()); + (next_out[..n]).copy_from_slice(&state.dict[state.dict_ofs..state.dict_ofs + n]); + *next_out = &mut mem::take(next_out)[n..]; + state.dict_avail -= n; + state.dict_ofs = (state.dict_ofs + (n)) & (TINFL_LZ_DICT_SIZE - 1); + n +} + +#[cfg(test)] +mod test { + use super::{inflate, InflateState}; + use crate::{DataFormat, MZFlush, MZStatus}; + use alloc::vec; + + #[test] + fn test_state() { + let encoded = [ + 120u8, 156, 243, 72, 205, 201, 201, 215, 81, 168, 202, 201, 76, 82, 4, 0, 27, 101, 4, + 19, + ]; + let mut out = vec![0; 50]; + let mut state = InflateState::new_boxed(DataFormat::Zlib); + let res = inflate(&mut state, &encoded, &mut out, MZFlush::Finish); + let status = res.status.expect("Failed to decompress!"); + assert_eq!(status, MZStatus::StreamEnd); + assert_eq!(out[..res.bytes_written as usize], b"Hello, zlib!"[..]); + assert_eq!(res.bytes_consumed, encoded.len()); + + state.reset_as(super::ZeroReset); + out.iter_mut().map(|x| *x = 0).count(); + let res = inflate(&mut state, &encoded, &mut out, MZFlush::Finish); + let status = res.status.expect("Failed to decompress!"); + assert_eq!(status, MZStatus::StreamEnd); + assert_eq!(out[..res.bytes_written as usize], b"Hello, zlib!"[..]); + assert_eq!(res.bytes_consumed, encoded.len()); + + state.reset_as(super::MinReset); + out.iter_mut().map(|x| *x = 0).count(); + let res = inflate(&mut state, &encoded, &mut out, MZFlush::Finish); + let status = res.status.expect("Failed to decompress!"); + assert_eq!(status, MZStatus::StreamEnd); + assert_eq!(out[..res.bytes_written as usize], b"Hello, zlib!"[..]); + assert_eq!(res.bytes_consumed, encoded.len()); + assert_eq!(state.decompressor().adler32(), Some(459605011)); + + // Test state when not computing adler. + state = InflateState::new_boxed(DataFormat::ZLibIgnoreChecksum); + out.iter_mut().map(|x| *x = 0).count(); + let res = inflate(&mut state, &encoded, &mut out, MZFlush::Finish); + let status = res.status.expect("Failed to decompress!"); + assert_eq!(status, MZStatus::StreamEnd); + assert_eq!(out[..res.bytes_written as usize], b"Hello, zlib!"[..]); + assert_eq!(res.bytes_consumed, encoded.len()); + // Not computed, so should be Some(1) + assert_eq!(state.decompressor().adler32(), Some(1)); + // Should still have the checksum read from the header file. + assert_eq!(state.decompressor().adler32_header(), Some(459605011)) + } +} diff --git a/crux-mir/lib/miniz_oxide/src/lib.rs b/crux-mir/lib/miniz_oxide/src/lib.rs new file mode 100644 index 000000000..8357c5200 --- /dev/null +++ b/crux-mir/lib/miniz_oxide/src/lib.rs @@ -0,0 +1,208 @@ +//! A pure rust replacement for the [miniz](https://github.com/richgel999/miniz) +//! DEFLATE/zlib encoder/decoder. +//! The plan for this crate is to be used as a back-end for the +//! [flate2](https://github.com/alexcrichton/flate2-rs) crate and eventually remove the +//! need to depend on a C library. +//! +//! # Usage +//! ## Simple compression/decompression: +//! ``` rust +//! +//! use miniz_oxide::inflate::decompress_to_vec; +//! use miniz_oxide::deflate::compress_to_vec; +//! +//! fn roundtrip(data: &[u8]) { +//! let compressed = compress_to_vec(data, 6); +//! let decompressed = decompress_to_vec(compressed.as_slice()).expect("Failed to decompress!"); +//! # let _ = decompressed; +//! } +//! +//! # roundtrip(b"Test_data test data lalalal blabla"); +//! +//! ``` + +#![forbid(unsafe_code)] +#![no_std] + +extern crate alloc; + +pub mod deflate; +pub mod inflate; +mod shared; + +pub use crate::shared::update_adler32 as mz_adler32_oxide; +pub use crate::shared::{MZ_ADLER32_INIT, MZ_DEFAULT_WINDOW_BITS}; + +/// A list of flush types. +/// +/// See for more in-depth info. +#[repr(i32)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum MZFlush { + /// Don't force any flushing. + /// Used when more input data is expected. + None = 0, + /// Zlib partial flush. + /// Currently treated as [`Sync`]. + Partial = 1, + /// Finish compressing the currently buffered data, and output an empty raw block. + /// Has no use in decompression. + Sync = 2, + /// Same as [`Sync`], but resets the compression dictionary so that further compressed + /// data does not depend on data compressed before the flush. + /// + /// Has no use in decompression, and is an error to supply in that case. + Full = 3, + /// Attempt to flush the remaining data and end the stream. + Finish = 4, + /// Not implemented. + Block = 5, +} + +impl MZFlush { + /// Create an MZFlush value from an integer value. + /// + /// Returns `MZError::Param` on invalid values. + pub fn new(flush: i32) -> Result { + match flush { + 0 => Ok(MZFlush::None), + 1 | 2 => Ok(MZFlush::Sync), + 3 => Ok(MZFlush::Full), + 4 => Ok(MZFlush::Finish), + _ => Err(MZError::Param), + } + } +} + +/// A list of miniz successful status codes. +/// +/// These are emitted as the [`Ok`] side of a [`MZResult`] in the [`StreamResult`] returned from +/// [`deflate::stream::deflate()`] or [`inflate::stream::inflate()`]. +#[repr(i32)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum MZStatus { + /// Operation succeeded. + /// + /// Some data was decompressed or compressed; see the byte counters in the [`StreamResult`] for + /// details. + Ok = 0, + + /// Operation succeeded and end of deflate stream was found. + /// + /// X-ref [`TINFLStatus::Done`][inflate::TINFLStatus::Done] or + /// [`TDEFLStatus::Done`][deflate::core::TDEFLStatus::Done] for `inflate` or `deflate` + /// respectively. + StreamEnd = 1, + + /// Unused + NeedDict = 2, +} + +/// A list of miniz failed status codes. +/// +/// These are emitted as the [`Err`] side of a [`MZResult`] in the [`StreamResult`] returned from +/// [`deflate::stream::deflate()`] or [`inflate::stream::inflate()`]. +#[repr(i32)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum MZError { + /// Unused + ErrNo = -1, + + /// General stream error. + /// + /// See [`inflate::stream::inflate()`] docs for details of how it can occur there. + /// + /// See [`deflate::stream::deflate()`] docs for how it can in principle occur there, though it's + /// believed impossible in practice. + Stream = -2, + + /// Error in inflation; see [`inflate::stream::inflate()`] for details. + /// + /// Not returned from [`deflate::stream::deflate()`]. + Data = -3, + + /// Unused + Mem = -4, + + /// Buffer-related error. + /// + /// See the docs of [`deflate::stream::deflate()`] or [`inflate::stream::inflate()`] for details + /// of when it would trigger in the one you're using. + Buf = -5, + + /// Unused + Version = -6, + + /// Bad parameters. + /// + /// This can be returned from [`deflate::stream::deflate()`] in the case of bad parameters. See + /// [`TDEFLStatus::BadParam`][deflate::core::TDEFLStatus::BadParam]. + Param = -10_000, +} + +/// How compressed data is wrapped. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum DataFormat { + /// Wrapped using the [zlib](http://www.zlib.org/rfc-zlib.html) format. + Zlib, + /// Zlib wrapped but ignore and don't compute the adler32 checksum. + /// Currently only used for inflate, behaves the same as Zlib for compression. + ZLibIgnoreChecksum, + /// Raw DEFLATE. + Raw, +} + +impl DataFormat { + pub(crate) fn from_window_bits(window_bits: i32) -> DataFormat { + if window_bits > 0 { + DataFormat::Zlib + } else { + DataFormat::Raw + } + } + + pub(crate) fn to_window_bits(self) -> i32 { + match self { + DataFormat::Zlib | DataFormat::ZLibIgnoreChecksum => shared::MZ_DEFAULT_WINDOW_BITS, + DataFormat::Raw => -shared::MZ_DEFAULT_WINDOW_BITS, + } + } +} + +/// `Result` alias for all miniz status codes both successful and failed. +pub type MZResult = Result; + +/// A structure containg the result of a call to the inflate or deflate streaming functions. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct StreamResult { + /// The number of bytes consumed from the input slice. + pub bytes_consumed: usize, + /// The number of bytes written to the output slice. + pub bytes_written: usize, + /// The return status of the call. + pub status: MZResult, +} + +impl StreamResult { + #[inline] + pub(crate) const fn error(error: MZError) -> StreamResult { + StreamResult { + bytes_consumed: 0, + bytes_written: 0, + status: Err(error), + } + } +} + +impl core::convert::From for MZResult { + fn from(res: StreamResult) -> Self { + res.status + } +} + +impl core::convert::From<&StreamResult> for MZResult { + fn from(res: &StreamResult) -> Self { + res.status + } +} diff --git a/crux-mir/lib/miniz_oxide/src/shared.rs b/crux-mir/lib/miniz_oxide/src/shared.rs new file mode 100644 index 000000000..8b81fb112 --- /dev/null +++ b/crux-mir/lib/miniz_oxide/src/shared.rs @@ -0,0 +1,25 @@ +#[doc(hidden)] +pub const MZ_ADLER32_INIT: u32 = 1; + +#[doc(hidden)] +pub const MZ_DEFAULT_WINDOW_BITS: i32 = 15; + +pub const HUFFMAN_LENGTH_ORDER: [u8; 19] = [ + 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15, +]; + +#[doc(hidden)] +#[cfg(not(feature = "simd"))] +pub fn update_adler32(adler: u32, data: &[u8]) -> u32 { + let mut hash = adler::Adler32::from_checksum(adler); + hash.write_slice(data); + hash.checksum() +} + +#[doc(hidden)] +#[cfg(feature = "simd")] +pub fn update_adler32(adler: u32, data: &[u8]) -> u32 { + let mut hash = simd_adler32::Adler32::from_checksum(adler); + hash.write(data); + hash.finish() +} diff --git a/crux-mir/lib/object/.cargo-ok b/crux-mir/lib/object/.cargo-ok new file mode 100644 index 000000000..b5754e203 --- /dev/null +++ b/crux-mir/lib/object/.cargo-ok @@ -0,0 +1 @@ +ok \ No newline at end of file diff --git a/crux-mir/lib/object/.cargo_vcs_info.json b/crux-mir/lib/object/.cargo_vcs_info.json new file mode 100644 index 000000000..603670d7b --- /dev/null +++ b/crux-mir/lib/object/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "0a38064531fef4ddbaf93770a3551d333338980e" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/crux-mir/lib/object/.gitignore b/crux-mir/lib/object/.gitignore new file mode 100644 index 000000000..a9d37c560 --- /dev/null +++ b/crux-mir/lib/object/.gitignore @@ -0,0 +1,2 @@ +target +Cargo.lock diff --git a/crux-mir/lib/object/.gitmodules b/crux-mir/lib/object/.gitmodules new file mode 100644 index 000000000..c3282e292 --- /dev/null +++ b/crux-mir/lib/object/.gitmodules @@ -0,0 +1,3 @@ +[submodule "testfiles"] + path = testfiles + url = https://github.com/gimli-rs/object-testfiles diff --git a/crux-mir/lib/object/CHANGELOG.md b/crux-mir/lib/object/CHANGELOG.md new file mode 100644 index 000000000..d74a55e63 --- /dev/null +++ b/crux-mir/lib/object/CHANGELOG.md @@ -0,0 +1,408 @@ +# `object` Change Log + +-------------------------------------------------------------------------------- + +## 0.29.0 + +Released 2022/06/22. + +### Breaking changes + +* The `write` feature now has a minimum supported rust version of 1.56.1. + [#444](https://github.com/gimli-rs/object/pull/444) + +* Added `os_abi` and `abi_version` fields to `FileFlags::Elf`. + [#438](https://github.com/gimli-rs/object/pull/438) + [#441](https://github.com/gimli-rs/object/pull/441) + +### Changed + +* Fixed handling of empty symbol tables in `read::elf::ElfFile::symbol_table` and + `read::elf::ElfFile::dynamic_symbol_table`. + [#443](https://github.com/gimli-rs/object/pull/443) + +### Added + +* Added more `ELF_OSABI_*` constants. + [#439](https://github.com/gimli-rs/object/pull/439) + +-------------------------------------------------------------------------------- + +## 0.28.4 + +Released 2022/05/09. + +### Added + +* Added `read::pe::DataDirectories::resource_directory`. + [#425](https://github.com/gimli-rs/object/pull/425) + [#427](https://github.com/gimli-rs/object/pull/427) + +* Added PE support for more ARM relocations. + [#428](https://github.com/gimli-rs/object/pull/428) + +* Added support for `Architecture::LoongArch64`. + [#430](https://github.com/gimli-rs/object/pull/430) + [#432](https://github.com/gimli-rs/object/pull/432) + +* Added `elf::EF_MIPS_ABI` and associated constants. + [#433](https://github.com/gimli-rs/object/pull/433) + +-------------------------------------------------------------------------------- + +## 0.28.3 + +Released 2022/01/19. + +### Changed + +* For the Mach-O support in `write::Object`, accept `RelocationKind::MachO` for all + architecures, and accept `RelocationKind::Absolute` for ARM64. + [#422](https://github.com/gimli-rs/object/pull/422) + +### Added + +* Added `pe::ImageDataDirectory::file_range`, `read::pe::SectionTable::pe_file_range_at` + and `pe::ImageSectionHeader::pe_file_range_at`. + [#421](https://github.com/gimli-rs/object/pull/421) + +* Added `write::Object::add_coff_exports`. + [#423](https://github.com/gimli-rs/object/pull/423) + +-------------------------------------------------------------------------------- + +## 0.28.2 + +Released 2022/01/09. + +### Changed + +* Ignored errors for the Wasm extended name section in `read::WasmFile::parse`. + [#408](https://github.com/gimli-rs/object/pull/408) + +* Ignored errors for the COFF symbol table in `read::PeFile::parse`. + [#410](https://github.com/gimli-rs/object/pull/410) + +* Fixed handling of `SectionFlags::Coff` in `write::Object::coff_write`. + [#412](https://github.com/gimli-rs/object/pull/412) + +### Added + +* Added `read::ObjectSegment::flags`. + [#416](https://github.com/gimli-rs/object/pull/416) + [#418](https://github.com/gimli-rs/object/pull/418) + +-------------------------------------------------------------------------------- + +## 0.28.1 + +Released 2021/12/12. + +### Changed + +* Fixed `read::elf::SymbolTable::shndx_section`. + [#405](https://github.com/gimli-rs/object/pull/405) + +* Fixed build warnings. + [#405](https://github.com/gimli-rs/object/pull/405) + [#406](https://github.com/gimli-rs/object/pull/406) + +-------------------------------------------------------------------------------- + +## 0.28.0 + +Released 2021/12/12. + +### Breaking changes + +* `write_core` feature no longer enables `std` support. Use `write_std` instead. + [#400](https://github.com/gimli-rs/object/pull/400) + +* Multiple changes related to Mach-O split dyld cache support. + [#398](https://github.com/gimli-rs/object/pull/398) + +### Added + +* Added `write::pe::Writer::write_file_align`. + [#397](https://github.com/gimli-rs/object/pull/397) + +* Added support for Mach-O split dyld cache. + [#398](https://github.com/gimli-rs/object/pull/398) + +* Added support for `IMAGE_SCN_LNK_NRELOC_OVFL` when reading and writing COFF. + [#399](https://github.com/gimli-rs/object/pull/399) + +* Added `write::elf::Writer::reserve_null_symbol_index`. + [#402](https://github.com/gimli-rs/object/pull/402) + +-------------------------------------------------------------------------------- + +## 0.27.1 + +Released 2021/10/22. + +### Changed + +* Fixed build error with older Rust versions due to cargo resolver version. + +-------------------------------------------------------------------------------- + +## 0.27.0 + +Released 2021/10/17. + +### Breaking changes + +* Changed `read::elf` to use `SectionIndex` instead of `usize` in more places. + [#341](https://github.com/gimli-rs/object/pull/341) + +* Changed some `read::elf` section methods to additionally return the linked section index. + [#341](https://github.com/gimli-rs/object/pull/341) + +* Changed `read::pe::ImageNtHeaders::parse` to return `DataDirectories` instead of a slice. + [#357](https://github.com/gimli-rs/object/pull/357) + +* Deleted `value` parameter for `write:WritableBuffer::resize`. + [#369](https://github.com/gimli-rs/object/pull/369) + +* Changed `write::Object` and `write::Section` to use `Cow` for section data. + This added a lifetime parameter, which existing users can set to `'static`. + [#370](https://github.com/gimli-rs/object/pull/370) + +### Changed + +* Fixed parsing when PE import directory has zero size. + [#341](https://github.com/gimli-rs/object/pull/341) + +* Fixed parsing when PE import directory has zero for original first thunk. + [#385](https://github.com/gimli-rs/object/pull/385) + [#387](https://github.com/gimli-rs/object/pull/387) + +* Fixed parsing when PE export directory has zero number of names. + [#353](https://github.com/gimli-rs/object/pull/353) + +* Fixed parsing when PE export directory has zero number of names and addresses. + [#362](https://github.com/gimli-rs/object/pull/362) + +* Fixed parsing when PE sections are contiguous. + [#354](https://github.com/gimli-rs/object/pull/354) + +* Fixed `std` feature for `indexmap` dependency. + [#374](https://github.com/gimli-rs/object/pull/374) + +* Fixed overflow in COFF section name offset parsing. + [#390](https://github.com/gimli-rs/object/pull/390) + +### Added + +* Added `name_bytes` methods to unified `read` traits. + [#351](https://github.com/gimli-rs/object/pull/351) + +* Added `read::Object::kind`. + [#352](https://github.com/gimli-rs/object/pull/352) + +* Added `read::elf::VersionTable` and related helpers. + [#341](https://github.com/gimli-rs/object/pull/341) + +* Added `read::elf::SectionTable::dynamic` and related helpers. + [#345](https://github.com/gimli-rs/object/pull/345) + +* Added `read::coff::SectionTable::max_section_file_offset`. + [#344](https://github.com/gimli-rs/object/pull/344) + +* Added `read::pe::ExportTable` and related helpers. + [#349](https://github.com/gimli-rs/object/pull/349) + [#353](https://github.com/gimli-rs/object/pull/353) + +* Added `read::pe::ImportTable` and related helpers. + [#357](https://github.com/gimli-rs/object/pull/357) + +* Added `read::pe::DataDirectories` and related helpers. + [#357](https://github.com/gimli-rs/object/pull/357) + [#384](https://github.com/gimli-rs/object/pull/384) + +* Added `read::pe::RichHeaderInfo` and related helpers. + [#375](https://github.com/gimli-rs/object/pull/375) + [#379](https://github.com/gimli-rs/object/pull/379) + +* Added `read::pe::RelocationBlocks` and related helpers. + [#378](https://github.com/gimli-rs/object/pull/378) + +* Added `write::elf::Writer`. + [#350](https://github.com/gimli-rs/object/pull/350) + +* Added `write::pe::Writer`. + [#382](https://github.com/gimli-rs/object/pull/382) + [#388](https://github.com/gimli-rs/object/pull/388) + +* Added `write::Section::data/data_mut`. + [#367](https://github.com/gimli-rs/object/pull/367) + +* Added `write::Object::write_stream`. + [#369](https://github.com/gimli-rs/object/pull/369) + +* Added MIPSr6 ELF header flag definitions. + [#372](https://github.com/gimli-rs/object/pull/372) + +-------------------------------------------------------------------------------- + +## 0.26.2 + +Released 2021/08/28. + +### Added + +* Added support for 64-bit symbol table names to `read::archive`. + [#366](https://github.com/gimli-rs/object/pull/366) + +-------------------------------------------------------------------------------- + +## 0.26.1 + +Released 2021/08/19. + +### Changed + +* Activate `memchr`'s `rustc-dep-of-std` feature + [#356](https://github.com/gimli-rs/object/pull/356) + +-------------------------------------------------------------------------------- + +## 0.26.0 + +Released 2021/07/26. + +### Breaking changes + +* Changed `ReadRef::read_bytes_at_until` to accept a range parameter. + [#326](https://github.com/gimli-rs/object/pull/326) + +* Added `ReadRef` type parameter to `read::StringTable` and types that + contain it. String table entries are now only read as required. + [#326](https://github.com/gimli-rs/object/pull/326) + +* Changed result type of `read::elf::SectionHeader::data` and `data_as_array`. + [#332](https://github.com/gimli-rs/object/pull/332) + +* Moved `pod::WritableBuffer` to `write::WritableBuffer`. + Renamed `WritableBuffer::extend` to `write_bytes`. + Added more provided methods to `WritableBuffer`. + [#335](https://github.com/gimli-rs/object/pull/335) + +* Moved `pod::Bytes` to `read::Bytes`. + [#336](https://github.com/gimli-rs/object/pull/336) + +* Added `is_mips64el` parameter to `elf::Rela64::r_info/set_r_info`. + [#337](https://github.com/gimli-rs/object/pull/337) + +### Changed + +* Removed `alloc` dependency when no features are enabled. + [#336](https://github.com/gimli-rs/object/pull/336) + +### Added + +* Added `read::pe::PeFile` methods: `section_table`, `data_directory`, and `data`. + [#324](https://github.com/gimli-rs/object/pull/324) + +* Added more ELF definitions. + [#332](https://github.com/gimli-rs/object/pull/332) + +* Added `read::elf::SectionTable` methods for hash tables and symbol version + information. + [#332](https://github.com/gimli-rs/object/pull/332) + +* Added PE RISC-V definitions. + [#333](https://github.com/gimli-rs/object/pull/333) + +* Added `WritableBuffer` implementation for `Vec`. + [#335](https://github.com/gimli-rs/object/pull/335) + +-------------------------------------------------------------------------------- + +## 0.25.3 + +Released 2021/06/12. + +### Added + +* Added `RelocationEncoding::AArch64Call`. + [#322](https://github.com/gimli-rs/object/pull/322) + +-------------------------------------------------------------------------------- + +## 0.25.2 + +Released 2021/06/04. + +### Added + +* Added `Architecture::X86_64_X32`. + [#320](https://github.com/gimli-rs/object/pull/320) + +-------------------------------------------------------------------------------- + +## 0.25.1 + +Released 2021/06/03. + +### Changed + +* write: Fix choice of `SHT_REL` or `SHT_RELA` for most architectures. + [#318](https://github.com/gimli-rs/object/pull/318) + +* write: Fix relocation encoding for MIPS64EL. + [#318](https://github.com/gimli-rs/object/pull/318) + +-------------------------------------------------------------------------------- + +## 0.25.0 + +Released 2021/06/02. + +### Breaking changes + +* Added `non_exhaustive` to most public enums. + [#306](https://github.com/gimli-rs/object/pull/306) + +* `MachHeader::parse` and `MachHeader::load_commands` now require a header offset. + [#304](https://github.com/gimli-rs/object/pull/304) + +* Added `ReadRef::read_bytes_at_until`. + [#308](https://github.com/gimli-rs/object/pull/308) + +* `PeFile::entry`, `PeSection::address` and `PeSegment::address` now return a + virtual address instead of a RVA. + [#315](https://github.com/gimli-rs/object/pull/315) + +### Added + +* Added `pod::from_bytes_mut`, `pod::slice_from_bytes_mut`, `pod::bytes_of_mut`, + and `pod::bytes_of_slice_mut`. + [#296](https://github.com/gimli-rs/object/pull/296) + [#297](https://github.com/gimli-rs/object/pull/297) + +* Added `Object::pdb_info`. + [#298](https://github.com/gimli-rs/object/pull/298) + +* Added `read::macho::DyldCache`, other associated definitions, + and support for these in the examples. + [#308](https://github.com/gimli-rs/object/pull/308) + +* Added more architecture support. + [#303](https://github.com/gimli-rs/object/pull/303) + [#309](https://github.com/gimli-rs/object/pull/309) + +* Derive more traits for enums. + [#311](https://github.com/gimli-rs/object/pull/311) + +* Added `Object::relative_address_base`. + [#315](https://github.com/gimli-rs/object/pull/315) + +### Changed + +* Improved performance for string parsing. + [#302](https://github.com/gimli-rs/object/pull/302) + +* `objdump` example allows selecting container members. + [#308](https://github.com/gimli-rs/object/pull/308) diff --git a/crux-mir/lib/object/Cargo.toml b/crux-mir/lib/object/Cargo.toml new file mode 100644 index 000000000..a94180ee1 --- /dev/null +++ b/crux-mir/lib/object/Cargo.toml @@ -0,0 +1,145 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2018" +name = "object" +version = "0.29.0" +exclude = [ + "/.github", + "/testfiles", +] +description = "A unified interface for reading and writing object file formats." +keywords = [ + "object", + "elf", + "mach-o", + "pe", + "coff", +] +license = "Apache-2.0 OR MIT" +repository = "https://github.com/gimli-rs/object" + +[package.metadata.docs.rs] +features = ["doc"] + +[dependencies.alloc] +version = "1.0.0" +optional = true +package = "rustc-std-workspace-alloc" + +[dependencies.compiler_builtins] +version = "0.1.2" +optional = true + +[dependencies.core] +version = "1.0.0" +optional = true +package = "rustc-std-workspace-core" + +[dependencies.crc32fast] +version = "1.2" +optional = true +default-features = false + +[dependencies.flate2] +version = "1" +optional = true + +[dependencies.hashbrown] +version = "0.12.0" +features = ["ahash"] +optional = true +default-features = false + +[dependencies.indexmap] +version = "1.6" +optional = true + +[dependencies.memchr] +version = "2.4.1" +default-features = false + +[dependencies.wasmparser] +version = "0.57" +optional = true + +[features] +all = [ + "read", + "write", + "std", + "compression", + "wasm", +] +archive = [] +cargo-all = [] +coff = [] +compression = [ + "flate2", + "std", +] +default = [ + "read", + "compression", +] +doc = [ + "read_core", + "write_std", + "std", + "compression", + "archive", + "coff", + "elf", + "macho", + "pe", + "wasm", +] +elf = [] +macho = [] +pe = ["coff"] +read = [ + "read_core", + "archive", + "coff", + "elf", + "macho", + "pe", + "unaligned", +] +read_core = [] +rustc-dep-of-std = [ + "core", + "compiler_builtins", + "alloc", + "memchr/rustc-dep-of-std", +] +std = ["memchr/std"] +unaligned = [] +wasm = ["wasmparser"] +write = [ + "write_std", + "coff", + "elf", + "macho", + "pe", +] +write_core = [ + "crc32fast", + "indexmap", + "hashbrown", +] +write_std = [ + "write_core", + "std", + "indexmap/std", + "crc32fast/std", +] diff --git a/crux-mir/lib/object/Cargo.toml.orig b/crux-mir/lib/object/Cargo.toml.orig new file mode 100755 index 000000000..e4457590e --- /dev/null +++ b/crux-mir/lib/object/Cargo.toml.orig @@ -0,0 +1,98 @@ +[package] +name = "object" +# Note: disable resolver in workspace setting before releases. +version = "0.29.0" +edition = "2018" +exclude = ["/.github", "/testfiles"] +keywords = ["object", "elf", "mach-o", "pe", "coff"] +license = "Apache-2.0 OR MIT" +repository = "https://github.com/gimli-rs/object" +description = "A unified interface for reading and writing object file formats." + +[package.metadata.docs.rs] +features = ['doc'] + +[dependencies] +crc32fast = { version = "1.2", default-features = false, optional = true } +flate2 = { version = "1", optional = true } +indexmap = { version = "1.6", optional = true } +wasmparser = { version = "0.57", optional = true } +memchr = { version = "2.4.1", default-features = false } +hashbrown = { version = "0.12.0", features = ["ahash"], default-features = false, optional = true } + +# Internal feature, only used when building as part of libstd, not part of the +# stable interface of this crate. +core = { version = '1.0.0', optional = true, package = 'rustc-std-workspace-core' } +compiler_builtins = { version = '0.1.2', optional = true } +alloc = { version = '1.0.0', optional = true, package = 'rustc-std-workspace-alloc' } + +[features] +#======================================= +# Read/write features. + +# Core read support. You will need to enable some file formats too. +read_core = [] +# Read support for all file formats (including unaligned files). +read = ["read_core", "archive", "coff", "elf", "macho", "pe", "unaligned"] +# Core write support. You will need to enable some file formats too. +write_core = ["crc32fast", "indexmap", "hashbrown"] +# Core write support with libstd features. You will need to enable some file formats too. +write_std = ["write_core", "std", "indexmap/std", "crc32fast/std"] +# Write support for all file formats, including libstd features. +write = ["write_std", "coff", "elf", "macho", "pe"] + +#======================================= +# Misc features. + +# Enable things that require libstd. +# Currently, this provides an `Error` implementation. +std = ["memchr/std"] +# Enable decompression of compressed sections. +# This feature is not required if you want to do your own decompression. +compression = ["flate2", "std"] +# Treat all types as unaligned. +# Normally types use the alignment required by the specifications, but +# sometimes files do not strictly follow the specifications. +# This may be useful to enable when processing files for architectures +# that have no alignment constraints. +unaligned = [] + +#======================================= +# File format features. +archive = [] +coff = [] +elf = [] +macho = [] +pe = ["coff"] +wasm = ["wasmparser"] + +#======================================= +# By default, support all read features. +default = ["read", "compression"] + +#======================================= +# Umbrella feature for enabling all user-facing features of this crate. Does not +# enable internal features like `rustc-dep-of-std`. +all = ["read", "write", "std", "compression", "wasm"] + +# Use of --all-features is not supported. +# This is a dummy feature to detect when --all-features is used. +cargo-all = [] + +#======================================= +# Documentation should be generated with everything in "all" except for "unaligned". +doc = [ + "read_core", "write_std", + "std", "compression", + "archive", "coff", "elf", "macho", "pe", "wasm", +] + +#======================================= +# Internal feature, only used when building as part of libstd, not part of the +# stable interface of this crate. +rustc-dep-of-std = ['core', 'compiler_builtins', 'alloc', 'memchr/rustc-dep-of-std'] + +[workspace] +members = ["crates/examples"] +default-members = [".", "crates/examples"] +#resolver = "2" diff --git a/crux-mir/lib/rustc-demangle/LICENSE-APACHE b/crux-mir/lib/object/LICENSE-APACHE similarity index 100% rename from crux-mir/lib/rustc-demangle/LICENSE-APACHE rename to crux-mir/lib/object/LICENSE-APACHE diff --git a/crux-mir/lib/object/LICENSE-MIT b/crux-mir/lib/object/LICENSE-MIT new file mode 100644 index 000000000..8440c7e95 --- /dev/null +++ b/crux-mir/lib/object/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2015 The Gimli Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/crux-mir/lib/object/README.md b/crux-mir/lib/object/README.md new file mode 100644 index 000000000..19268a953 --- /dev/null +++ b/crux-mir/lib/object/README.md @@ -0,0 +1,58 @@ +# `object` + +The `object` crate provides a unified interface to working with object files +across platforms. It supports reading object files and executable files, +and writing COFF/ELF/Mach-O object files and ELF/PE executable files. + +For reading files, it provides multiple levels of support: + +* raw struct definitions suitable for zero copy access +* low level APIs for accessing the raw structs ([example](crates/examples/src/readobj/)) +* a higher level unified API for accessing common features of object files, such + as sections and symbols ([example](crates/examples/src/objdump.rs)) + +Supported file formats: ELF, Mach-O, Windows PE/COFF, Wasm, and Unix archive. + +## Example for unified read API +```rust +use object::{Object, ObjectSection}; +use std::error::Error; +use std::fs; + +/// Reads a file and displays the content of the ".boot" section. +fn main() -> Result<(), Box> { + let bin_data = fs::read("./multiboot2-binary.elf")?; + let obj_file = object::File::parse(&*bin_data)?; + if let Some(section) = obj_file.section_by_name(".boot") { + println!("{:#x?}", section.data()?); + } else { + eprintln!("section not available"); + } + Ok(()) +} +``` + +See [`crates/examples`](crates/examples) for more examples. + +## Minimum Supported Rust Version (MSRV) + +Changes to MSRV are considered breaking changes. We are conservative about changing the MSRV, +but sometimes are required to due to dependencies. The MSRV is: + + * 1.42.0 for the `read` feature and its dependencies. + * 1.56.1 for the `write` feature and its dependencies. + +## License + +Licensed under either of + + * Apache License, Version 2.0 ([`LICENSE-APACHE`](./LICENSE-APACHE) or https://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([`LICENSE-MIT`](./LICENSE-MIT) or https://opensource.org/licenses/MIT) + +at your option. + +## Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in the work by you, as defined in the Apache-2.0 license, shall be +dual licensed as above, without any additional terms or conditions. diff --git a/crux-mir/lib/object/clippy.toml b/crux-mir/lib/object/clippy.toml new file mode 100644 index 000000000..f97e544b6 --- /dev/null +++ b/crux-mir/lib/object/clippy.toml @@ -0,0 +1 @@ +msrv = "1.42.0" diff --git a/crux-mir/lib/object/src/archive.rs b/crux-mir/lib/object/src/archive.rs new file mode 100644 index 000000000..d4b419beb --- /dev/null +++ b/crux-mir/lib/object/src/archive.rs @@ -0,0 +1,39 @@ +//! Archive definitions. +//! +//! These definitions are independent of read/write support, although we do implement +//! some traits useful for those. + +use crate::pod::Pod; + +/// File identification bytes stored at the beginning of the file. +pub const MAGIC: [u8; 8] = *b"!\n"; + +/// File identification bytes stored at the beginning of a thin archive. +/// +/// A thin archive only contains a symbol table and file names. +pub const THIN_MAGIC: [u8; 8] = *b"!\n"; + +/// The terminator for each archive member header. +pub const TERMINATOR: [u8; 2] = *b"`\n"; + +/// The header at the start of an archive member. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Header { + /// The file name. + pub name: [u8; 16], + /// File modification timestamp in decimal. + pub date: [u8; 12], + /// User ID in decimal. + pub uid: [u8; 6], + /// Group ID in decimal. + pub gid: [u8; 6], + /// File mode in octal. + pub mode: [u8; 8], + /// File size in decimal. + pub size: [u8; 10], + /// Must be equal to `TERMINATOR`. + pub terminator: [u8; 2], +} + +unsafe_impl_pod!(Header); diff --git a/crux-mir/lib/object/src/common.rs b/crux-mir/lib/object/src/common.rs new file mode 100644 index 000000000..20dde991a --- /dev/null +++ b/crux-mir/lib/object/src/common.rs @@ -0,0 +1,450 @@ +/// A CPU architecture. +#[allow(missing_docs)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum Architecture { + Unknown, + Aarch64, + Arm, + Avr, + Bpf, + I386, + X86_64, + #[allow(non_camel_case_types)] + X86_64_X32, + Hexagon, + LoongArch64, + Mips, + Mips64, + Msp430, + PowerPc, + PowerPc64, + Riscv32, + Riscv64, + S390x, + Sparc64, + Wasm32, +} + +impl Architecture { + /// The size of an address value for this architecture. + /// + /// Returns `None` for unknown architectures. + pub fn address_size(self) -> Option { + match self { + Architecture::Unknown => None, + Architecture::Aarch64 => Some(AddressSize::U64), + Architecture::Arm => Some(AddressSize::U32), + Architecture::Avr => Some(AddressSize::U8), + Architecture::Bpf => Some(AddressSize::U64), + Architecture::I386 => Some(AddressSize::U32), + Architecture::X86_64 => Some(AddressSize::U64), + Architecture::X86_64_X32 => Some(AddressSize::U32), + Architecture::Hexagon => Some(AddressSize::U32), + Architecture::LoongArch64 => Some(AddressSize::U64), + Architecture::Mips => Some(AddressSize::U32), + Architecture::Mips64 => Some(AddressSize::U64), + Architecture::Msp430 => Some(AddressSize::U16), + Architecture::PowerPc => Some(AddressSize::U32), + Architecture::PowerPc64 => Some(AddressSize::U64), + Architecture::Riscv32 => Some(AddressSize::U32), + Architecture::Riscv64 => Some(AddressSize::U64), + Architecture::S390x => Some(AddressSize::U64), + Architecture::Sparc64 => Some(AddressSize::U64), + Architecture::Wasm32 => Some(AddressSize::U32), + } + } +} + +/// The size of an address value for an architecture. +/// +/// This may differ from the address size supported by the file format (such as for COFF). +#[allow(missing_docs)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +#[repr(u8)] +pub enum AddressSize { + U8 = 1, + U16 = 2, + U32 = 4, + U64 = 8, +} + +impl AddressSize { + /// The size in bytes of an address value. + #[inline] + pub fn bytes(self) -> u8 { + self as u8 + } +} + +/// A binary file format. +#[allow(missing_docs)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum BinaryFormat { + Coff, + Elf, + MachO, + Pe, + Wasm, +} + +/// The kind of a section. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum SectionKind { + /// The section kind is unknown. + Unknown, + /// An executable code section. + /// + /// Example ELF sections: `.text` + /// + /// Example Mach-O sections: `__TEXT/__text` + Text, + /// A data section. + /// + /// Example ELF sections: `.data` + /// + /// Example Mach-O sections: `__DATA/__data` + Data, + /// A read only data section. + /// + /// Example ELF sections: `.rodata` + /// + /// Example Mach-O sections: `__TEXT/__const`, `__DATA/__const`, `__TEXT/__literal4` + ReadOnlyData, + /// A loadable string section. + /// + /// Example ELF sections: `.rodata.str` + /// + /// Example Mach-O sections: `__TEXT/__cstring` + ReadOnlyString, + /// An uninitialized data section. + /// + /// Example ELF sections: `.bss` + /// + /// Example Mach-O sections: `__DATA/__bss` + UninitializedData, + /// An uninitialized common data section. + /// + /// Example Mach-O sections: `__DATA/__common` + Common, + /// A TLS data section. + /// + /// Example ELF sections: `.tdata` + /// + /// Example Mach-O sections: `__DATA/__thread_data` + Tls, + /// An uninitialized TLS data section. + /// + /// Example ELF sections: `.tbss` + /// + /// Example Mach-O sections: `__DATA/__thread_bss` + UninitializedTls, + /// A TLS variables section. + /// + /// This contains TLS variable structures, rather than the variable initializers. + /// + /// Example Mach-O sections: `__DATA/__thread_vars` + TlsVariables, + /// A non-loadable string section. + /// + /// Example ELF sections: `.comment`, `.debug_str` + OtherString, + /// Some other non-loadable section. + /// + /// Example ELF sections: `.debug_info` + Other, + /// Debug information. + /// + /// Example Mach-O sections: `__DWARF/__debug_info` + Debug, + /// Information for the linker. + /// + /// Example COFF sections: `.drectve` + Linker, + /// ELF note section. + Note, + /// Metadata such as symbols or relocations. + /// + /// Example ELF sections: `.symtab`, `.strtab`, `.group` + Metadata, + /// Some other ELF section type. + /// + /// This is the `sh_type` field in the section header. + /// The meaning may be dependent on the architecture. + Elf(u32), +} + +impl SectionKind { + /// Return true if this section contains zerofill data. + pub fn is_bss(self) -> bool { + self == SectionKind::UninitializedData + || self == SectionKind::UninitializedTls + || self == SectionKind::Common + } +} + +/// The selection kind for a COMDAT section group. +/// +/// This determines the way in which the linker resolves multiple definitions of the COMDAT +/// sections. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum ComdatKind { + /// The selection kind is unknown. + Unknown, + /// Multiple definitions are allowed. + /// + /// An arbitrary definition is selected, and the rest are removed. + /// + /// This is the only supported selection kind for ELF. + Any, + /// Multiple definitions are not allowed. + /// + /// This is used to group sections without allowing duplicates. + NoDuplicates, + /// Multiple definitions must have the same size. + /// + /// An arbitrary definition is selected, and the rest are removed. + SameSize, + /// Multiple definitions must match exactly. + /// + /// An arbitrary definition is selected, and the rest are removed. + ExactMatch, + /// Multiple definitions are allowed, and the largest is selected. + /// + /// An arbitrary definition with the largest size is selected, and the rest are removed. + Largest, + /// Multiple definitions are allowed, and the newest is selected. + Newest, +} + +/// The kind of a symbol. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum SymbolKind { + /// The symbol kind is unknown. + Unknown, + /// The symbol is a null placeholder. + Null, + /// The symbol is for executable code. + Text, + /// The symbol is for a data object. + Data, + /// The symbol is for a section. + Section, + /// The symbol is the name of a file. It precedes symbols within that file. + File, + /// The symbol is for a code label. + Label, + /// The symbol is for a thread local storage entity. + Tls, +} + +/// A symbol scope. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum SymbolScope { + /// Unknown scope. + Unknown, + /// Symbol is visible to the compilation unit. + Compilation, + /// Symbol is visible to the static linkage unit. + Linkage, + /// Symbol is visible to dynamically linked objects. + Dynamic, +} + +/// The operation used to calculate the result of the relocation. +/// +/// The relocation descriptions use the following definitions. Note that +/// these definitions probably don't match any ELF ABI. +/// +/// * A - The value of the addend. +/// * G - The address of the symbol's entry within the global offset table. +/// * L - The address of the symbol's entry within the procedure linkage table. +/// * P - The address of the place of the relocation. +/// * S - The address of the symbol. +/// * GotBase - The address of the global offset table. +/// * Image - The base address of the image. +/// * Section - The address of the section containing the symbol. +/// +/// 'XxxRelative' means 'Xxx + A - P'. 'XxxOffset' means 'S + A - Xxx'. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum RelocationKind { + /// S + A + Absolute, + /// S + A - P + Relative, + /// G + A - GotBase + Got, + /// G + A - P + GotRelative, + /// GotBase + A - P + GotBaseRelative, + /// S + A - GotBase + GotBaseOffset, + /// L + A - P + PltRelative, + /// S + A - Image + ImageOffset, + /// S + A - Section + SectionOffset, + /// The index of the section containing the symbol. + SectionIndex, + /// Some other ELF relocation. The value is dependent on the architecture. + Elf(u32), + /// Some other Mach-O relocation. The value is dependent on the architecture. + MachO { + /// The relocation type. + value: u8, + /// Whether the relocation is relative to the place. + relative: bool, + }, + /// Some other COFF relocation. The value is dependent on the architecture. + Coff(u16), +} + +/// Information about how the result of the relocation operation is encoded in the place. +/// +/// This is usually architecture specific, such as specifying an addressing mode or +/// a specific instruction. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum RelocationEncoding { + /// Generic encoding. + Generic, + + /// x86 sign extension at runtime. + /// + /// Used with `RelocationKind::Absolute`. + X86Signed, + /// x86 rip-relative addressing. + /// + /// The `RelocationKind` must be PC relative. + X86RipRelative, + /// x86 rip-relative addressing in movq instruction. + /// + /// The `RelocationKind` must be PC relative. + X86RipRelativeMovq, + /// x86 branch instruction. + /// + /// The `RelocationKind` must be PC relative. + X86Branch, + + /// s390x PC-relative offset shifted right by one bit. + /// + /// The `RelocationKind` must be PC relative. + S390xDbl, + + /// AArch64 call target. + /// + /// The `RelocationKind` must be PC relative. + AArch64Call, +} + +/// File flags that are specific to each file format. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum FileFlags { + /// No file flags. + None, + /// ELF file flags. + Elf { + /// `os_abi` field in the ELF file header. + os_abi: u8, + /// `abi_version` field in the ELF file header. + abi_version: u8, + /// `e_flags` field in the ELF file header. + e_flags: u32, + }, + /// Mach-O file flags. + MachO { + /// `flags` field in the Mach-O file header. + flags: u32, + }, + /// COFF file flags. + Coff { + /// `Characteristics` field in the COFF file header. + characteristics: u16, + }, +} + +/// Segment flags that are specific to each file format. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum SegmentFlags { + /// No segment flags. + None, + /// ELF segment flags. + Elf { + /// `p_flags` field in the segment header. + p_flags: u32, + }, + /// Mach-O segment flags. + MachO { + /// `flags` field in the segment header. + flags: u32, + /// `maxprot` field in the segment header. + maxprot: u32, + /// `initprot` field in the segment header. + initprot: u32, + }, + /// COFF segment flags. + Coff { + /// `Characteristics` field in the segment header. + characteristics: u32, + }, +} + +/// Section flags that are specific to each file format. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum SectionFlags { + /// No section flags. + None, + /// ELF section flags. + Elf { + /// `sh_flags` field in the section header. + sh_flags: u64, + }, + /// Mach-O section flags. + MachO { + /// `flags` field in the section header. + flags: u32, + }, + /// COFF section flags. + Coff { + /// `Characteristics` field in the section header. + characteristics: u32, + }, +} + +/// Symbol flags that are specific to each file format. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum SymbolFlags
{ + /// No symbol flags. + None, + /// ELF symbol flags. + Elf { + /// `st_info` field in the ELF symbol. + st_info: u8, + /// `st_other` field in the ELF symbol. + st_other: u8, + }, + /// Mach-O symbol flags. + MachO { + /// `n_desc` field in the Mach-O symbol. + n_desc: u16, + }, + /// COFF flags for a section symbol. + CoffSection { + /// `Selection` field in the auxiliary symbol for the section. + selection: u8, + /// `Number` field in the auxiliary symbol for the section. + associative_section: Option
, + }, +} diff --git a/crux-mir/lib/object/src/elf.rs b/crux-mir/lib/object/src/elf.rs new file mode 100644 index 000000000..9f6577269 --- /dev/null +++ b/crux-mir/lib/object/src/elf.rs @@ -0,0 +1,6404 @@ +//! ELF definitions. +//! +//! These definitions are independent of read/write support, although we do implement +//! some traits useful for those. +//! +//! This module is the equivalent of /usr/include/elf.h, and is based heavily on it. + +#![allow(clippy::identity_op)] + +use crate::endian::{Endian, U32Bytes, U64Bytes, I32, I64, U16, U32, U64}; +use crate::pod::Pod; + +/// The header at the start of every 32-bit ELF file. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct FileHeader32 { + /// Magic number and other information. + pub e_ident: Ident, + /// Object file type. One of the `ET_*` constants. + pub e_type: U16, + /// Architecture. One of the `EM_*` constants. + pub e_machine: U16, + /// Object file version. Must be `EV_CURRENT`. + pub e_version: U32, + /// Entry point virtual address. + pub e_entry: U32, + /// Program header table file offset. + pub e_phoff: U32, + /// Section header table file offset. + pub e_shoff: U32, + /// Processor-specific flags. + /// + /// A combination of the `EF_*` constants. + pub e_flags: U32, + /// Size in bytes of this header. + pub e_ehsize: U16, + /// Program header table entry size. + pub e_phentsize: U16, + /// Program header table entry count. + /// + /// If the count is greater than or equal to `PN_XNUM` then this field is set to + /// `PN_XNUM` and the count is stored in the `sh_info` field of section 0. + pub e_phnum: U16, + /// Section header table entry size. + pub e_shentsize: U16, + /// Section header table entry count. + /// + /// If the count is greater than or equal to `SHN_LORESERVE` then this field is set to + /// `0` and the count is stored in the `sh_size` field of section 0. + /// first section header. + pub e_shnum: U16, + /// Section header string table index. + /// + /// If the index is greater than or equal to `SHN_LORESERVE` then this field is set to + /// `SHN_XINDEX` and the index is stored in the `sh_link` field of section 0. + pub e_shstrndx: U16, +} + +/// The header at the start of every 64-bit ELF file. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct FileHeader64 { + /// Magic number and other information. + pub e_ident: Ident, + /// Object file type. One of the `ET_*` constants. + pub e_type: U16, + /// Architecture. One of the `EM_*` constants. + pub e_machine: U16, + /// Object file version. Must be `EV_CURRENT`. + pub e_version: U32, + /// Entry point virtual address. + pub e_entry: U64, + /// Program header table file offset. + pub e_phoff: U64, + /// Section header table file offset. + pub e_shoff: U64, + /// Processor-specific flags. + /// + /// A combination of the `EF_*` constants. + pub e_flags: U32, + /// Size in bytes of this header. + pub e_ehsize: U16, + /// Program header table entry size. + pub e_phentsize: U16, + /// Program header table entry count. + /// + /// If the count is greater than or equal to `PN_XNUM` then this field is set to + /// `PN_XNUM` and the count is stored in the `sh_info` field of section 0. + pub e_phnum: U16, + /// Section header table entry size. + pub e_shentsize: U16, + /// Section header table entry count. + /// + /// If the count is greater than or equal to `SHN_LORESERVE` then this field is set to + /// `0` and the count is stored in the `sh_size` field of section 0. + /// first section header. + pub e_shnum: U16, + /// Section header string table index. + /// + /// If the index is greater than or equal to `SHN_LORESERVE` then this field is set to + /// `SHN_XINDEX` and the index is stored in the `sh_link` field of section 0. + pub e_shstrndx: U16, +} + +/// Magic number and other information. +/// +/// Contained in the file header. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Ident { + /// Magic number. Must be `ELFMAG`. + pub magic: [u8; 4], + /// File class. One of the `ELFCLASS*` constants. + pub class: u8, + /// Data encoding. One of the `ELFDATA*` constants. + pub data: u8, + /// ELF version. Must be `EV_CURRENT`. + pub version: u8, + /// OS ABI identification. One of the `ELFOSABI*` constants. + pub os_abi: u8, + /// ABI version. + /// + /// The meaning of this field depends on the `os_abi` value. + pub abi_version: u8, + /// Padding bytes. + pub padding: [u8; 7], +} + +/// File identification bytes stored in `Ident::magic`. +pub const ELFMAG: [u8; 4] = [0x7f, b'E', b'L', b'F']; + +// Values for `Ident::class`. +/// Invalid class. +pub const ELFCLASSNONE: u8 = 0; +/// 32-bit object. +pub const ELFCLASS32: u8 = 1; +/// 64-bit object. +pub const ELFCLASS64: u8 = 2; + +// Values for `Ident::data`. +/// Invalid data encoding. +pub const ELFDATANONE: u8 = 0; +/// 2's complement, little endian. +pub const ELFDATA2LSB: u8 = 1; +/// 2's complement, big endian. +pub const ELFDATA2MSB: u8 = 2; + +// Values for `Ident::os_abi`. +/// UNIX System V ABI. +pub const ELFOSABI_NONE: u8 = 0; +/// UNIX System V ABI. +/// +/// Alias. +pub const ELFOSABI_SYSV: u8 = 0; +/// HP-UX. +pub const ELFOSABI_HPUX: u8 = 1; +/// NetBSD. +pub const ELFOSABI_NETBSD: u8 = 2; +/// Object uses GNU ELF extensions. +pub const ELFOSABI_GNU: u8 = 3; +/// Object uses GNU ELF extensions. +/// +/// Compatibility alias. +pub const ELFOSABI_LINUX: u8 = ELFOSABI_GNU; +/// GNU/Hurd. +pub const ELFOSABI_HURD: u8 = 4; +/// Sun Solaris. +pub const ELFOSABI_SOLARIS: u8 = 6; +/// IBM AIX. +pub const ELFOSABI_AIX: u8 = 7; +/// SGI Irix. +pub const ELFOSABI_IRIX: u8 = 8; +/// FreeBSD. +pub const ELFOSABI_FREEBSD: u8 = 9; +/// Compaq TRU64 UNIX. +pub const ELFOSABI_TRU64: u8 = 10; +/// Novell Modesto. +pub const ELFOSABI_MODESTO: u8 = 11; +/// OpenBSD. +pub const ELFOSABI_OPENBSD: u8 = 12; +/// OpenVMS. +pub const ELFOSABI_OPENVMS: u8 = 13; +/// Hewlett-Packard Non-Stop Kernel. +pub const ELFOSABI_NSK: u8 = 14; +/// AROS +pub const ELFOSABI_AROS: u8 = 15; +/// FenixOS +pub const ELFOSABI_FENIXOS: u8 = 16; +/// Nuxi CloudABI +pub const ELFOSABI_CLOUDABI: u8 = 17; +/// ARM EABI. +pub const ELFOSABI_ARM_AEABI: u8 = 64; +/// ARM. +pub const ELFOSABI_ARM: u8 = 97; +/// Standalone (embedded) application. +pub const ELFOSABI_STANDALONE: u8 = 255; + +// Values for `FileHeader*::e_type`. +/// No file type. +pub const ET_NONE: u16 = 0; +/// Relocatable file. +pub const ET_REL: u16 = 1; +/// Executable file. +pub const ET_EXEC: u16 = 2; +/// Shared object file. +pub const ET_DYN: u16 = 3; +/// Core file. +pub const ET_CORE: u16 = 4; +/// OS-specific range start. +pub const ET_LOOS: u16 = 0xfe00; +/// OS-specific range end. +pub const ET_HIOS: u16 = 0xfeff; +/// Processor-specific range start. +pub const ET_LOPROC: u16 = 0xff00; +/// Processor-specific range end. +pub const ET_HIPROC: u16 = 0xffff; + +// Values for `FileHeader*::e_machine`. +/// No machine +pub const EM_NONE: u16 = 0; +/// AT&T WE 32100 +pub const EM_M32: u16 = 1; +/// SUN SPARC +pub const EM_SPARC: u16 = 2; +/// Intel 80386 +pub const EM_386: u16 = 3; +/// Motorola m68k family +pub const EM_68K: u16 = 4; +/// Motorola m88k family +pub const EM_88K: u16 = 5; +/// Intel MCU +pub const EM_IAMCU: u16 = 6; +/// Intel 80860 +pub const EM_860: u16 = 7; +/// MIPS R3000 big-endian +pub const EM_MIPS: u16 = 8; +/// IBM System/370 +pub const EM_S370: u16 = 9; +/// MIPS R3000 little-endian +pub const EM_MIPS_RS3_LE: u16 = 10; +/// HPPA +pub const EM_PARISC: u16 = 15; +/// Fujitsu VPP500 +pub const EM_VPP500: u16 = 17; +/// Sun's "v8plus" +pub const EM_SPARC32PLUS: u16 = 18; +/// Intel 80960 +pub const EM_960: u16 = 19; +/// PowerPC +pub const EM_PPC: u16 = 20; +/// PowerPC 64-bit +pub const EM_PPC64: u16 = 21; +/// IBM S390 +pub const EM_S390: u16 = 22; +/// IBM SPU/SPC +pub const EM_SPU: u16 = 23; +/// NEC V800 series +pub const EM_V800: u16 = 36; +/// Fujitsu FR20 +pub const EM_FR20: u16 = 37; +/// TRW RH-32 +pub const EM_RH32: u16 = 38; +/// Motorola RCE +pub const EM_RCE: u16 = 39; +/// ARM +pub const EM_ARM: u16 = 40; +/// Digital Alpha +pub const EM_FAKE_ALPHA: u16 = 41; +/// Hitachi SH +pub const EM_SH: u16 = 42; +/// SPARC v9 64-bit +pub const EM_SPARCV9: u16 = 43; +/// Siemens Tricore +pub const EM_TRICORE: u16 = 44; +/// Argonaut RISC Core +pub const EM_ARC: u16 = 45; +/// Hitachi H8/300 +pub const EM_H8_300: u16 = 46; +/// Hitachi H8/300H +pub const EM_H8_300H: u16 = 47; +/// Hitachi H8S +pub const EM_H8S: u16 = 48; +/// Hitachi H8/500 +pub const EM_H8_500: u16 = 49; +/// Intel Merced +pub const EM_IA_64: u16 = 50; +/// Stanford MIPS-X +pub const EM_MIPS_X: u16 = 51; +/// Motorola Coldfire +pub const EM_COLDFIRE: u16 = 52; +/// Motorola M68HC12 +pub const EM_68HC12: u16 = 53; +/// Fujitsu MMA Multimedia Accelerator +pub const EM_MMA: u16 = 54; +/// Siemens PCP +pub const EM_PCP: u16 = 55; +/// Sony nCPU embeeded RISC +pub const EM_NCPU: u16 = 56; +/// Denso NDR1 microprocessor +pub const EM_NDR1: u16 = 57; +/// Motorola Start*Core processor +pub const EM_STARCORE: u16 = 58; +/// Toyota ME16 processor +pub const EM_ME16: u16 = 59; +/// STMicroelectronic ST100 processor +pub const EM_ST100: u16 = 60; +/// Advanced Logic Corp. Tinyj emb.fam +pub const EM_TINYJ: u16 = 61; +/// AMD x86-64 architecture +pub const EM_X86_64: u16 = 62; +/// Sony DSP Processor +pub const EM_PDSP: u16 = 63; +/// Digital PDP-10 +pub const EM_PDP10: u16 = 64; +/// Digital PDP-11 +pub const EM_PDP11: u16 = 65; +/// Siemens FX66 microcontroller +pub const EM_FX66: u16 = 66; +/// STMicroelectronics ST9+ 8/16 mc +pub const EM_ST9PLUS: u16 = 67; +/// STmicroelectronics ST7 8 bit mc +pub const EM_ST7: u16 = 68; +/// Motorola MC68HC16 microcontroller +pub const EM_68HC16: u16 = 69; +/// Motorola MC68HC11 microcontroller +pub const EM_68HC11: u16 = 70; +/// Motorola MC68HC08 microcontroller +pub const EM_68HC08: u16 = 71; +/// Motorola MC68HC05 microcontroller +pub const EM_68HC05: u16 = 72; +/// Silicon Graphics SVx +pub const EM_SVX: u16 = 73; +/// STMicroelectronics ST19 8 bit mc +pub const EM_ST19: u16 = 74; +/// Digital VAX +pub const EM_VAX: u16 = 75; +/// Axis Communications 32-bit emb.proc +pub const EM_CRIS: u16 = 76; +/// Infineon Technologies 32-bit emb.proc +pub const EM_JAVELIN: u16 = 77; +/// Element 14 64-bit DSP Processor +pub const EM_FIREPATH: u16 = 78; +/// LSI Logic 16-bit DSP Processor +pub const EM_ZSP: u16 = 79; +/// Donald Knuth's educational 64-bit proc +pub const EM_MMIX: u16 = 80; +/// Harvard University machine-independent object files +pub const EM_HUANY: u16 = 81; +/// SiTera Prism +pub const EM_PRISM: u16 = 82; +/// Atmel AVR 8-bit microcontroller +pub const EM_AVR: u16 = 83; +/// Fujitsu FR30 +pub const EM_FR30: u16 = 84; +/// Mitsubishi D10V +pub const EM_D10V: u16 = 85; +/// Mitsubishi D30V +pub const EM_D30V: u16 = 86; +/// NEC v850 +pub const EM_V850: u16 = 87; +/// Mitsubishi M32R +pub const EM_M32R: u16 = 88; +/// Matsushita MN10300 +pub const EM_MN10300: u16 = 89; +/// Matsushita MN10200 +pub const EM_MN10200: u16 = 90; +/// picoJava +pub const EM_PJ: u16 = 91; +/// OpenRISC 32-bit embedded processor +pub const EM_OPENRISC: u16 = 92; +/// ARC International ARCompact +pub const EM_ARC_COMPACT: u16 = 93; +/// Tensilica Xtensa Architecture +pub const EM_XTENSA: u16 = 94; +/// Alphamosaic VideoCore +pub const EM_VIDEOCORE: u16 = 95; +/// Thompson Multimedia General Purpose Proc +pub const EM_TMM_GPP: u16 = 96; +/// National Semi. 32000 +pub const EM_NS32K: u16 = 97; +/// Tenor Network TPC +pub const EM_TPC: u16 = 98; +/// Trebia SNP 1000 +pub const EM_SNP1K: u16 = 99; +/// STMicroelectronics ST200 +pub const EM_ST200: u16 = 100; +/// Ubicom IP2xxx +pub const EM_IP2K: u16 = 101; +/// MAX processor +pub const EM_MAX: u16 = 102; +/// National Semi. CompactRISC +pub const EM_CR: u16 = 103; +/// Fujitsu F2MC16 +pub const EM_F2MC16: u16 = 104; +/// Texas Instruments msp430 +pub const EM_MSP430: u16 = 105; +/// Analog Devices Blackfin DSP +pub const EM_BLACKFIN: u16 = 106; +/// Seiko Epson S1C33 family +pub const EM_SE_C33: u16 = 107; +/// Sharp embedded microprocessor +pub const EM_SEP: u16 = 108; +/// Arca RISC +pub const EM_ARCA: u16 = 109; +/// PKU-Unity & MPRC Peking Uni. mc series +pub const EM_UNICORE: u16 = 110; +/// eXcess configurable cpu +pub const EM_EXCESS: u16 = 111; +/// Icera Semi. Deep Execution Processor +pub const EM_DXP: u16 = 112; +/// Altera Nios II +pub const EM_ALTERA_NIOS2: u16 = 113; +/// National Semi. CompactRISC CRX +pub const EM_CRX: u16 = 114; +/// Motorola XGATE +pub const EM_XGATE: u16 = 115; +/// Infineon C16x/XC16x +pub const EM_C166: u16 = 116; +/// Renesas M16C +pub const EM_M16C: u16 = 117; +/// Microchip Technology dsPIC30F +pub const EM_DSPIC30F: u16 = 118; +/// Freescale Communication Engine RISC +pub const EM_CE: u16 = 119; +/// Renesas M32C +pub const EM_M32C: u16 = 120; +/// Altium TSK3000 +pub const EM_TSK3000: u16 = 131; +/// Freescale RS08 +pub const EM_RS08: u16 = 132; +/// Analog Devices SHARC family +pub const EM_SHARC: u16 = 133; +/// Cyan Technology eCOG2 +pub const EM_ECOG2: u16 = 134; +/// Sunplus S+core7 RISC +pub const EM_SCORE7: u16 = 135; +/// New Japan Radio (NJR) 24-bit DSP +pub const EM_DSP24: u16 = 136; +/// Broadcom VideoCore III +pub const EM_VIDEOCORE3: u16 = 137; +/// RISC for Lattice FPGA +pub const EM_LATTICEMICO32: u16 = 138; +/// Seiko Epson C17 +pub const EM_SE_C17: u16 = 139; +/// Texas Instruments TMS320C6000 DSP +pub const EM_TI_C6000: u16 = 140; +/// Texas Instruments TMS320C2000 DSP +pub const EM_TI_C2000: u16 = 141; +/// Texas Instruments TMS320C55x DSP +pub const EM_TI_C5500: u16 = 142; +/// Texas Instruments App. Specific RISC +pub const EM_TI_ARP32: u16 = 143; +/// Texas Instruments Prog. Realtime Unit +pub const EM_TI_PRU: u16 = 144; +/// STMicroelectronics 64bit VLIW DSP +pub const EM_MMDSP_PLUS: u16 = 160; +/// Cypress M8C +pub const EM_CYPRESS_M8C: u16 = 161; +/// Renesas R32C +pub const EM_R32C: u16 = 162; +/// NXP Semi. TriMedia +pub const EM_TRIMEDIA: u16 = 163; +/// QUALCOMM Hexagon +pub const EM_HEXAGON: u16 = 164; +/// Intel 8051 and variants +pub const EM_8051: u16 = 165; +/// STMicroelectronics STxP7x +pub const EM_STXP7X: u16 = 166; +/// Andes Tech. compact code emb. RISC +pub const EM_NDS32: u16 = 167; +/// Cyan Technology eCOG1X +pub const EM_ECOG1X: u16 = 168; +/// Dallas Semi. MAXQ30 mc +pub const EM_MAXQ30: u16 = 169; +/// New Japan Radio (NJR) 16-bit DSP +pub const EM_XIMO16: u16 = 170; +/// M2000 Reconfigurable RISC +pub const EM_MANIK: u16 = 171; +/// Cray NV2 vector architecture +pub const EM_CRAYNV2: u16 = 172; +/// Renesas RX +pub const EM_RX: u16 = 173; +/// Imagination Tech. META +pub const EM_METAG: u16 = 174; +/// MCST Elbrus +pub const EM_MCST_ELBRUS: u16 = 175; +/// Cyan Technology eCOG16 +pub const EM_ECOG16: u16 = 176; +/// National Semi. CompactRISC CR16 +pub const EM_CR16: u16 = 177; +/// Freescale Extended Time Processing Unit +pub const EM_ETPU: u16 = 178; +/// Infineon Tech. SLE9X +pub const EM_SLE9X: u16 = 179; +/// Intel L10M +pub const EM_L10M: u16 = 180; +/// Intel K10M +pub const EM_K10M: u16 = 181; +/// ARM AARCH64 +pub const EM_AARCH64: u16 = 183; +/// Amtel 32-bit microprocessor +pub const EM_AVR32: u16 = 185; +/// STMicroelectronics STM8 +pub const EM_STM8: u16 = 186; +/// Tileta TILE64 +pub const EM_TILE64: u16 = 187; +/// Tilera TILEPro +pub const EM_TILEPRO: u16 = 188; +/// Xilinx MicroBlaze +pub const EM_MICROBLAZE: u16 = 189; +/// NVIDIA CUDA +pub const EM_CUDA: u16 = 190; +/// Tilera TILE-Gx +pub const EM_TILEGX: u16 = 191; +/// CloudShield +pub const EM_CLOUDSHIELD: u16 = 192; +/// KIPO-KAIST Core-A 1st gen. +pub const EM_COREA_1ST: u16 = 193; +/// KIPO-KAIST Core-A 2nd gen. +pub const EM_COREA_2ND: u16 = 194; +/// Synopsys ARCompact V2 +pub const EM_ARC_COMPACT2: u16 = 195; +/// Open8 RISC +pub const EM_OPEN8: u16 = 196; +/// Renesas RL78 +pub const EM_RL78: u16 = 197; +/// Broadcom VideoCore V +pub const EM_VIDEOCORE5: u16 = 198; +/// Renesas 78KOR +pub const EM_78KOR: u16 = 199; +/// Freescale 56800EX DSC +pub const EM_56800EX: u16 = 200; +/// Beyond BA1 +pub const EM_BA1: u16 = 201; +/// Beyond BA2 +pub const EM_BA2: u16 = 202; +/// XMOS xCORE +pub const EM_XCORE: u16 = 203; +/// Microchip 8-bit PIC(r) +pub const EM_MCHP_PIC: u16 = 204; +/// KM211 KM32 +pub const EM_KM32: u16 = 210; +/// KM211 KMX32 +pub const EM_KMX32: u16 = 211; +/// KM211 KMX16 +pub const EM_EMX16: u16 = 212; +/// KM211 KMX8 +pub const EM_EMX8: u16 = 213; +/// KM211 KVARC +pub const EM_KVARC: u16 = 214; +/// Paneve CDP +pub const EM_CDP: u16 = 215; +/// Cognitive Smart Memory Processor +pub const EM_COGE: u16 = 216; +/// Bluechip CoolEngine +pub const EM_COOL: u16 = 217; +/// Nanoradio Optimized RISC +pub const EM_NORC: u16 = 218; +/// CSR Kalimba +pub const EM_CSR_KALIMBA: u16 = 219; +/// Zilog Z80 +pub const EM_Z80: u16 = 220; +/// Controls and Data Services VISIUMcore +pub const EM_VISIUM: u16 = 221; +/// FTDI Chip FT32 +pub const EM_FT32: u16 = 222; +/// Moxie processor +pub const EM_MOXIE: u16 = 223; +/// AMD GPU +pub const EM_AMDGPU: u16 = 224; +/// RISC-V +pub const EM_RISCV: u16 = 243; +/// Linux BPF -- in-kernel virtual machine +pub const EM_BPF: u16 = 247; +/// C-SKY +pub const EM_CSKY: u16 = 252; +/// Loongson LoongArch +pub const EM_LOONGARCH: u16 = 258; +/// Digital Alpha +pub const EM_ALPHA: u16 = 0x9026; + +// Values for `FileHeader*::e_version` and `Ident::version`. +/// Invalid ELF version. +pub const EV_NONE: u8 = 0; +/// Current ELF version. +pub const EV_CURRENT: u8 = 1; + +/// Section header. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct SectionHeader32 { + /// Section name. + /// + /// This is an offset into the section header string table. + pub sh_name: U32, + /// Section type. One of the `SHT_*` constants. + pub sh_type: U32, + /// Section flags. A combination of the `SHF_*` constants. + pub sh_flags: U32, + /// Section virtual address at execution. + pub sh_addr: U32, + /// Section file offset. + pub sh_offset: U32, + /// Section size in bytes. + pub sh_size: U32, + /// Link to another section. + /// + /// The section relationship depends on the `sh_type` value. + pub sh_link: U32, + /// Additional section information. + /// + /// The meaning of this field depends on the `sh_type` value. + pub sh_info: U32, + /// Section alignment. + pub sh_addralign: U32, + /// Entry size if the section holds a table. + pub sh_entsize: U32, +} + +/// Section header. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct SectionHeader64 { + /// Section name. + /// + /// This is an offset into the section header string table. + pub sh_name: U32, + /// Section type. One of the `SHT_*` constants. + pub sh_type: U32, + /// Section flags. A combination of the `SHF_*` constants. + pub sh_flags: U64, + /// Section virtual address at execution. + pub sh_addr: U64, + /// Section file offset. + pub sh_offset: U64, + /// Section size in bytes. + pub sh_size: U64, + /// Link to another section. + /// + /// The section relationship depends on the `sh_type` value. + pub sh_link: U32, + /// Additional section information. + /// + /// The meaning of this field depends on the `sh_type` value. + pub sh_info: U32, + /// Section alignment. + pub sh_addralign: U64, + /// Entry size if the section holds a table. + pub sh_entsize: U64, +} + +// Special values for section indices. +/// Undefined section. +pub const SHN_UNDEF: u16 = 0; +/// OS-specific range start. +/// Start of reserved section indices. +pub const SHN_LORESERVE: u16 = 0xff00; +/// Start of processor-specific section indices. +pub const SHN_LOPROC: u16 = 0xff00; +/// End of processor-specific section indices. +pub const SHN_HIPROC: u16 = 0xff1f; +/// Start of OS-specific section indices. +pub const SHN_LOOS: u16 = 0xff20; +/// End of OS-specific section indices. +pub const SHN_HIOS: u16 = 0xff3f; +/// Associated symbol is absolute. +pub const SHN_ABS: u16 = 0xfff1; +/// Associated symbol is common. +pub const SHN_COMMON: u16 = 0xfff2; +/// Section index is in the `SHT_SYMTAB_SHNDX` section. +pub const SHN_XINDEX: u16 = 0xffff; +/// End of reserved section indices. +pub const SHN_HIRESERVE: u16 = 0xffff; + +// Values for `SectionHeader*::sh_type`. +/// Section header table entry is unused. +pub const SHT_NULL: u32 = 0; +/// Program data. +pub const SHT_PROGBITS: u32 = 1; +/// Symbol table. +pub const SHT_SYMTAB: u32 = 2; +/// String table. +pub const SHT_STRTAB: u32 = 3; +/// Relocation entries with explicit addends. +pub const SHT_RELA: u32 = 4; +/// Symbol hash table. +pub const SHT_HASH: u32 = 5; +/// Dynamic linking information. +pub const SHT_DYNAMIC: u32 = 6; +/// Notes. +pub const SHT_NOTE: u32 = 7; +/// Program space with no data (bss). +pub const SHT_NOBITS: u32 = 8; +/// Relocation entries without explicit addends. +pub const SHT_REL: u32 = 9; +/// Reserved section type. +pub const SHT_SHLIB: u32 = 10; +/// Dynamic linker symbol table. +pub const SHT_DYNSYM: u32 = 11; +/// Array of constructors. +pub const SHT_INIT_ARRAY: u32 = 14; +/// Array of destructors. +pub const SHT_FINI_ARRAY: u32 = 15; +/// Array of pre-constructors. +pub const SHT_PREINIT_ARRAY: u32 = 16; +/// Section group. +pub const SHT_GROUP: u32 = 17; +/// Extended section indices for a symbol table. +pub const SHT_SYMTAB_SHNDX: u32 = 18; +/// Start of OS-specific section types. +pub const SHT_LOOS: u32 = 0x6000_0000; +/// Object attributes. +pub const SHT_GNU_ATTRIBUTES: u32 = 0x6fff_fff5; +/// GNU-style hash table. +pub const SHT_GNU_HASH: u32 = 0x6fff_fff6; +/// Prelink library list +pub const SHT_GNU_LIBLIST: u32 = 0x6fff_fff7; +/// Checksum for DSO content. +pub const SHT_CHECKSUM: u32 = 0x6fff_fff8; +/// Sun-specific low bound. +pub const SHT_LOSUNW: u32 = 0x6fff_fffa; +#[allow(missing_docs, non_upper_case_globals)] +pub const SHT_SUNW_move: u32 = 0x6fff_fffa; +#[allow(missing_docs)] +pub const SHT_SUNW_COMDAT: u32 = 0x6fff_fffb; +#[allow(missing_docs, non_upper_case_globals)] +pub const SHT_SUNW_syminfo: u32 = 0x6fff_fffc; +/// Version definition section. +#[allow(non_upper_case_globals)] +pub const SHT_GNU_VERDEF: u32 = 0x6fff_fffd; +/// Version needs section. +#[allow(non_upper_case_globals)] +pub const SHT_GNU_VERNEED: u32 = 0x6fff_fffe; +/// Version symbol table. +#[allow(non_upper_case_globals)] +pub const SHT_GNU_VERSYM: u32 = 0x6fff_ffff; +/// Sun-specific high bound. +pub const SHT_HISUNW: u32 = 0x6fff_ffff; +/// End of OS-specific section types. +pub const SHT_HIOS: u32 = 0x6fff_ffff; +/// Start of processor-specific section types. +pub const SHT_LOPROC: u32 = 0x7000_0000; +/// End of processor-specific section types. +pub const SHT_HIPROC: u32 = 0x7fff_ffff; +/// Start of application-specific section types. +pub const SHT_LOUSER: u32 = 0x8000_0000; +/// End of application-specific section types. +pub const SHT_HIUSER: u32 = 0x8fff_ffff; + +// Values for `SectionHeader*::sh_flags`. +/// Section is writable. +pub const SHF_WRITE: u32 = 1 << 0; +/// Section occupies memory during execution. +pub const SHF_ALLOC: u32 = 1 << 1; +/// Section is executable. +pub const SHF_EXECINSTR: u32 = 1 << 2; +/// Section may be be merged to eliminate duplication. +pub const SHF_MERGE: u32 = 1 << 4; +/// Section contains nul-terminated strings. +pub const SHF_STRINGS: u32 = 1 << 5; +/// The `sh_info` field contains a section header table index. +pub const SHF_INFO_LINK: u32 = 1 << 6; +/// Section has special ordering requirements when combining sections. +pub const SHF_LINK_ORDER: u32 = 1 << 7; +/// Section requires special OS-specific handling. +pub const SHF_OS_NONCONFORMING: u32 = 1 << 8; +/// Section is a member of a group. +pub const SHF_GROUP: u32 = 1 << 9; +/// Section holds thread-local storage. +pub const SHF_TLS: u32 = 1 << 10; +/// Section is compressed. +/// +/// Compressed sections begin with one of the `CompressionHeader*` headers. +pub const SHF_COMPRESSED: u32 = 1 << 11; +/// OS-specific section flags. +pub const SHF_MASKOS: u32 = 0x0ff0_0000; +/// Processor-specific section flags. +pub const SHF_MASKPROC: u32 = 0xf000_0000; +/// This section is excluded from the final executable or shared library. +pub const SHF_EXCLUDE: u32 = 0x8000_0000; + +/// Section compression header. +/// +/// Used when `SHF_COMPRESSED` is set. +/// +/// Note: this type currently allows for misaligned headers, but that may be +/// changed in a future version. +#[derive(Debug, Default, Clone, Copy)] +#[repr(C)] +pub struct CompressionHeader32 { + /// Compression format. One of the `ELFCOMPRESS_*` values. + pub ch_type: U32Bytes, + /// Uncompressed data size. + pub ch_size: U32Bytes, + /// Uncompressed data alignment. + pub ch_addralign: U32Bytes, +} + +/// Section compression header. +/// +/// Used when `SHF_COMPRESSED` is set. +/// +/// Note: this type currently allows for misaligned headers, but that may be +/// changed in a future version. +#[derive(Debug, Default, Clone, Copy)] +#[repr(C)] +pub struct CompressionHeader64 { + /// Compression format. One of the `ELFCOMPRESS_*` values. + pub ch_type: U32Bytes, + /// Reserved. + pub ch_reserved: U32Bytes, + /// Uncompressed data size. + pub ch_size: U64Bytes, + /// Uncompressed data alignment. + pub ch_addralign: U64Bytes, +} + +/// ZLIB/DEFLATE algorithm. +pub const ELFCOMPRESS_ZLIB: u32 = 1; +/// Start of OS-specific compression types. +pub const ELFCOMPRESS_LOOS: u32 = 0x6000_0000; +/// End of OS-specific compression types. +pub const ELFCOMPRESS_HIOS: u32 = 0x6fff_ffff; +/// Start of processor-specific compression types. +pub const ELFCOMPRESS_LOPROC: u32 = 0x7000_0000; +/// End of processor-specific compression types. +pub const ELFCOMPRESS_HIPROC: u32 = 0x7fff_ffff; + +// Values for the flag entry for section groups. +/// Mark group as COMDAT. +pub const GRP_COMDAT: u32 = 1; + +/// Symbol table entry. +#[derive(Debug, Default, Clone, Copy)] +#[repr(C)] +pub struct Sym32 { + /// Symbol name. + /// + /// This is an offset into the symbol string table. + pub st_name: U32, + /// Symbol value. + pub st_value: U32, + /// Symbol size. + pub st_size: U32, + /// Symbol type and binding. + /// + /// Use the `st_type` and `st_bind` methods to access this value. + pub st_info: u8, + /// Symbol visibility. + /// + /// Use the `st_visibility` method to access this value. + pub st_other: u8, + /// Section index or one of the `SHN_*` values. + pub st_shndx: U16, +} + +impl Sym32 { + /// Get the `st_bind` component of the `st_info` field. + #[inline] + pub fn st_bind(&self) -> u8 { + self.st_info >> 4 + } + + /// Get the `st_type` component of the `st_info` field. + #[inline] + pub fn st_type(&self) -> u8 { + self.st_info & 0xf + } + + /// Set the `st_info` field given the `st_bind` and `st_type` components. + #[inline] + pub fn set_st_info(&mut self, st_bind: u8, st_type: u8) { + self.st_info = (st_bind << 4) + (st_type & 0xf); + } + + /// Get the `st_visibility` component of the `st_info` field. + #[inline] + pub fn st_visibility(&self) -> u8 { + self.st_other & 0x3 + } +} + +/// Symbol table entry. +#[derive(Debug, Default, Clone, Copy)] +#[repr(C)] +pub struct Sym64 { + /// Symbol name. + /// + /// This is an offset into the symbol string table. + pub st_name: U32, + /// Symbol type and binding. + /// + /// Use the `st_bind` and `st_type` methods to access this value. + pub st_info: u8, + /// Symbol visibility. + /// + /// Use the `st_visibility` method to access this value. + pub st_other: u8, + /// Section index or one of the `SHN_*` values. + pub st_shndx: U16, + /// Symbol value. + pub st_value: U64, + /// Symbol size. + pub st_size: U64, +} + +impl Sym64 { + /// Get the `st_bind` component of the `st_info` field. + #[inline] + pub fn st_bind(&self) -> u8 { + self.st_info >> 4 + } + + /// Get the `st_type` component of the `st_info` field. + #[inline] + pub fn st_type(&self) -> u8 { + self.st_info & 0xf + } + + /// Set the `st_info` field given the `st_bind` and `st_type` components. + #[inline] + pub fn set_st_info(&mut self, st_bind: u8, st_type: u8) { + self.st_info = (st_bind << 4) + (st_type & 0xf); + } + + /// Get the `st_visibility` component of the `st_info` field. + #[inline] + pub fn st_visibility(&self) -> u8 { + self.st_other & 0x3 + } +} + +/// Additional information about a `Sym32`. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Syminfo32 { + /// Direct bindings, symbol bound to. + pub si_boundto: U16, + /// Per symbol flags. + pub si_flags: U16, +} + +/// Additional information about a `Sym64`. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Syminfo64 { + /// Direct bindings, symbol bound to. + pub si_boundto: U16, + /// Per symbol flags. + pub si_flags: U16, +} + +// Values for `Syminfo*::si_boundto`. +/// Symbol bound to self +pub const SYMINFO_BT_SELF: u16 = 0xffff; +/// Symbol bound to parent +pub const SYMINFO_BT_PARENT: u16 = 0xfffe; +/// Beginning of reserved entries +pub const SYMINFO_BT_LOWRESERVE: u16 = 0xff00; + +// Values for `Syminfo*::si_flags`. +/// Direct bound symbol +pub const SYMINFO_FLG_DIRECT: u16 = 0x0001; +/// Pass-thru symbol for translator +pub const SYMINFO_FLG_PASSTHRU: u16 = 0x0002; +/// Symbol is a copy-reloc +pub const SYMINFO_FLG_COPY: u16 = 0x0004; +/// Symbol bound to object to be lazy loaded +pub const SYMINFO_FLG_LAZYLOAD: u16 = 0x0008; + +// Syminfo version values. +#[allow(missing_docs)] +pub const SYMINFO_NONE: u16 = 0; +#[allow(missing_docs)] +pub const SYMINFO_CURRENT: u16 = 1; +#[allow(missing_docs)] +pub const SYMINFO_NUM: u16 = 2; + +// Values for bind component of `Sym*::st_info`. +/// Local symbol. +pub const STB_LOCAL: u8 = 0; +/// Global symbol. +pub const STB_GLOBAL: u8 = 1; +/// Weak symbol. +pub const STB_WEAK: u8 = 2; +/// Start of OS-specific symbol binding. +pub const STB_LOOS: u8 = 10; +/// Unique symbol. +pub const STB_GNU_UNIQUE: u8 = 10; +/// End of OS-specific symbol binding. +pub const STB_HIOS: u8 = 12; +/// Start of processor-specific symbol binding. +pub const STB_LOPROC: u8 = 13; +/// End of processor-specific symbol binding. +pub const STB_HIPROC: u8 = 15; + +// Values for type component of `Sym*::st_info`. +/// Symbol type is unspecified. +pub const STT_NOTYPE: u8 = 0; +/// Symbol is a data object. +pub const STT_OBJECT: u8 = 1; +/// Symbol is a code object. +pub const STT_FUNC: u8 = 2; +/// Symbol is associated with a section. +pub const STT_SECTION: u8 = 3; +/// Symbol's name is a file name. +pub const STT_FILE: u8 = 4; +/// Symbol is a common data object. +pub const STT_COMMON: u8 = 5; +/// Symbol is a thread-local storage object. +pub const STT_TLS: u8 = 6; +/// Start of OS-specific symbol types. +pub const STT_LOOS: u8 = 10; +/// Symbol is an indirect code object. +pub const STT_GNU_IFUNC: u8 = 10; +/// End of OS-specific symbol types. +pub const STT_HIOS: u8 = 12; +/// Start of processor-specific symbol types. +pub const STT_LOPROC: u8 = 13; +/// End of processor-specific symbol types. +pub const STT_HIPROC: u8 = 15; + +// Values for visibility component of `Symbol*::st_other`. +/// Default symbol visibility rules. +pub const STV_DEFAULT: u8 = 0; +/// Processor specific hidden class. +pub const STV_INTERNAL: u8 = 1; +/// Symbol is not visible to other components. +pub const STV_HIDDEN: u8 = 2; +/// Symbol is visible to other components, but is not preemptible. +pub const STV_PROTECTED: u8 = 3; + +/// Relocation table entry without explicit addend. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Rel32 { + /// Relocation address. + pub r_offset: U32, + /// Relocation type and symbol index. + pub r_info: U32, +} + +impl Rel32 { + /// Get the `r_sym` component of the `r_info` field. + #[inline] + pub fn r_sym(&self, endian: E) -> u32 { + self.r_info.get(endian) >> 8 + } + + /// Get the `r_type` component of the `r_info` field. + #[inline] + pub fn r_type(&self, endian: E) -> u32 { + self.r_info.get(endian) & 0xff + } + + /// Calculate the `r_info` field given the `r_sym` and `r_type` components. + pub fn r_info(endian: E, r_sym: u32, r_type: u8) -> U32 { + U32::new(endian, (r_sym << 8) | u32::from(r_type)) + } + + /// Set the `r_info` field given the `r_sym` and `r_type` components. + pub fn set_r_info(&mut self, endian: E, r_sym: u32, r_type: u8) { + self.r_info = Self::r_info(endian, r_sym, r_type) + } +} + +/// Relocation table entry with explicit addend. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Rela32 { + /// Relocation address. + pub r_offset: U32, + /// Relocation type and symbol index. + pub r_info: U32, + /// Explicit addend. + pub r_addend: I32, +} + +impl Rela32 { + /// Get the `r_sym` component of the `r_info` field. + #[inline] + pub fn r_sym(&self, endian: E) -> u32 { + self.r_info.get(endian) >> 8 + } + + /// Get the `r_type` component of the `r_info` field. + #[inline] + pub fn r_type(&self, endian: E) -> u32 { + self.r_info.get(endian) & 0xff + } + + /// Calculate the `r_info` field given the `r_sym` and `r_type` components. + pub fn r_info(endian: E, r_sym: u32, r_type: u8) -> U32 { + U32::new(endian, (r_sym << 8) | u32::from(r_type)) + } + + /// Set the `r_info` field given the `r_sym` and `r_type` components. + pub fn set_r_info(&mut self, endian: E, r_sym: u32, r_type: u8) { + self.r_info = Self::r_info(endian, r_sym, r_type) + } +} + +impl From> for Rela32 { + fn from(rel: Rel32) -> Self { + Rela32 { + r_offset: rel.r_offset, + r_info: rel.r_info, + r_addend: I32::default(), + } + } +} + +/// Relocation table entry without explicit addend. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Rel64 { + /// Relocation address. + pub r_offset: U64, + /// Relocation type and symbol index. + pub r_info: U64, +} + +impl Rel64 { + /// Get the `r_sym` component of the `r_info` field. + #[inline] + pub fn r_sym(&self, endian: E) -> u32 { + (self.r_info.get(endian) >> 32) as u32 + } + + /// Get the `r_type` component of the `r_info` field. + #[inline] + pub fn r_type(&self, endian: E) -> u32 { + (self.r_info.get(endian) & 0xffff_ffff) as u32 + } + + /// Calculate the `r_info` field given the `r_sym` and `r_type` components. + pub fn r_info(endian: E, r_sym: u32, r_type: u32) -> U64 { + U64::new(endian, (u64::from(r_sym) << 32) | u64::from(r_type)) + } + + /// Set the `r_info` field given the `r_sym` and `r_type` components. + pub fn set_r_info(&mut self, endian: E, r_sym: u32, r_type: u32) { + self.r_info = Self::r_info(endian, r_sym, r_type) + } +} + +impl From> for Rela64 { + fn from(rel: Rel64) -> Self { + Rela64 { + r_offset: rel.r_offset, + r_info: rel.r_info, + r_addend: I64::default(), + } + } +} + +/// Relocation table entry with explicit addend. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Rela64 { + /// Relocation address. + pub r_offset: U64, + /// Relocation type and symbol index. + pub r_info: U64, + /// Explicit addend. + pub r_addend: I64, +} + +impl Rela64 { + pub(crate) fn get_r_info(&self, endian: E, is_mips64el: bool) -> u64 { + let mut t = self.r_info.get(endian); + if is_mips64el { + t = (t << 32) + | ((t >> 8) & 0xff000000) + | ((t >> 24) & 0x00ff0000) + | ((t >> 40) & 0x0000ff00) + | ((t >> 56) & 0x000000ff); + } + t + } + + /// Get the `r_sym` component of the `r_info` field. + #[inline] + pub fn r_sym(&self, endian: E, is_mips64el: bool) -> u32 { + (self.get_r_info(endian, is_mips64el) >> 32) as u32 + } + + /// Get the `r_type` component of the `r_info` field. + #[inline] + pub fn r_type(&self, endian: E, is_mips64el: bool) -> u32 { + (self.get_r_info(endian, is_mips64el) & 0xffff_ffff) as u32 + } + + /// Calculate the `r_info` field given the `r_sym` and `r_type` components. + pub fn r_info(endian: E, is_mips64el: bool, r_sym: u32, r_type: u32) -> U64 { + let mut t = (u64::from(r_sym) << 32) | u64::from(r_type); + if is_mips64el { + t = (t >> 32) + | ((t & 0xff000000) << 8) + | ((t & 0x00ff0000) << 24) + | ((t & 0x0000ff00) << 40) + | ((t & 0x000000ff) << 56); + } + U64::new(endian, t) + } + + /// Set the `r_info` field given the `r_sym` and `r_type` components. + pub fn set_r_info(&mut self, endian: E, is_mips64el: bool, r_sym: u32, r_type: u32) { + self.r_info = Self::r_info(endian, is_mips64el, r_sym, r_type); + } +} + +/// Program segment header. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ProgramHeader32 { + /// Segment type. One of the `PT_*` constants. + pub p_type: U32, + /// Segment file offset. + pub p_offset: U32, + /// Segment virtual address. + pub p_vaddr: U32, + /// Segment physical address. + pub p_paddr: U32, + /// Segment size in the file. + pub p_filesz: U32, + /// Segment size in memory. + pub p_memsz: U32, + /// Segment flags. A combination of the `PF_*` constants. + pub p_flags: U32, + /// Segment alignment. + pub p_align: U32, +} + +/// Program segment header. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ProgramHeader64 { + /// Segment type. One of the `PT_*` constants. + pub p_type: U32, + /// Segment flags. A combination of the `PF_*` constants. + pub p_flags: U32, + /// Segment file offset. + pub p_offset: U64, + /// Segment virtual address. + pub p_vaddr: U64, + /// Segment physical address. + pub p_paddr: U64, + /// Segment size in the file. + pub p_filesz: U64, + /// Segment size in memory. + pub p_memsz: U64, + /// Segment alignment. + pub p_align: U64, +} + +/// Special value for `FileHeader*::e_phnum`. +/// +/// This indicates that the real number of program headers is too large to fit into e_phnum. +/// Instead the real value is in the field `sh_info` of section 0. +pub const PN_XNUM: u16 = 0xffff; + +// Values for `ProgramHeader*::p_type`. +/// Program header table entry is unused. +pub const PT_NULL: u32 = 0; +/// Loadable program segment. +pub const PT_LOAD: u32 = 1; +/// Dynamic linking information. +pub const PT_DYNAMIC: u32 = 2; +/// Program interpreter. +pub const PT_INTERP: u32 = 3; +/// Auxiliary information. +pub const PT_NOTE: u32 = 4; +/// Reserved. +pub const PT_SHLIB: u32 = 5; +/// Segment contains the program header table. +pub const PT_PHDR: u32 = 6; +/// Thread-local storage segment. +pub const PT_TLS: u32 = 7; +/// Start of OS-specific segment types. +pub const PT_LOOS: u32 = 0x6000_0000; +/// GCC `.eh_frame_hdr` segment. +pub const PT_GNU_EH_FRAME: u32 = 0x6474_e550; +/// Indicates stack executability. +pub const PT_GNU_STACK: u32 = 0x6474_e551; +/// Read-only after relocation. +pub const PT_GNU_RELRO: u32 = 0x6474_e552; +/// End of OS-specific segment types. +pub const PT_HIOS: u32 = 0x6fff_ffff; +/// Start of processor-specific segment types. +pub const PT_LOPROC: u32 = 0x7000_0000; +/// End of processor-specific segment types. +pub const PT_HIPROC: u32 = 0x7fff_ffff; + +// Values for `ProgramHeader*::p_flags`. +/// Segment is executable. +pub const PF_X: u32 = 1 << 0; +/// Segment is writable. +pub const PF_W: u32 = 1 << 1; +/// Segment is readable. +pub const PF_R: u32 = 1 << 2; +/// OS-specific segment flags. +pub const PF_MASKOS: u32 = 0x0ff0_0000; +/// Processor-specific segment flags. +pub const PF_MASKPROC: u32 = 0xf000_0000; + +/// Note name for core files. +pub static ELF_NOTE_CORE: &[u8] = b"CORE"; +/// Note name for linux core files. +/// +/// Notes in linux core files may also use `ELF_NOTE_CORE`. +pub static ELF_NOTE_LINUX: &[u8] = b"LINUX"; + +// Values for `NoteHeader*::n_type` in core files. +// +/// Contains copy of prstatus struct. +pub const NT_PRSTATUS: u32 = 1; +/// Contains copy of fpregset struct. +pub const NT_PRFPREG: u32 = 2; +/// Contains copy of fpregset struct. +pub const NT_FPREGSET: u32 = 2; +/// Contains copy of prpsinfo struct. +pub const NT_PRPSINFO: u32 = 3; +/// Contains copy of prxregset struct. +pub const NT_PRXREG: u32 = 4; +/// Contains copy of task structure. +pub const NT_TASKSTRUCT: u32 = 4; +/// String from sysinfo(SI_PLATFORM). +pub const NT_PLATFORM: u32 = 5; +/// Contains copy of auxv array. +pub const NT_AUXV: u32 = 6; +/// Contains copy of gwindows struct. +pub const NT_GWINDOWS: u32 = 7; +/// Contains copy of asrset struct. +pub const NT_ASRS: u32 = 8; +/// Contains copy of pstatus struct. +pub const NT_PSTATUS: u32 = 10; +/// Contains copy of psinfo struct. +pub const NT_PSINFO: u32 = 13; +/// Contains copy of prcred struct. +pub const NT_PRCRED: u32 = 14; +/// Contains copy of utsname struct. +pub const NT_UTSNAME: u32 = 15; +/// Contains copy of lwpstatus struct. +pub const NT_LWPSTATUS: u32 = 16; +/// Contains copy of lwpinfo struct. +pub const NT_LWPSINFO: u32 = 17; +/// Contains copy of fprxregset struct. +pub const NT_PRFPXREG: u32 = 20; +/// Contains copy of siginfo_t, size might increase. +pub const NT_SIGINFO: u32 = 0x5349_4749; +/// Contains information about mapped files. +pub const NT_FILE: u32 = 0x4649_4c45; +/// Contains copy of user_fxsr_struct. +pub const NT_PRXFPREG: u32 = 0x46e6_2b7f; +/// PowerPC Altivec/VMX registers. +pub const NT_PPC_VMX: u32 = 0x100; +/// PowerPC SPE/EVR registers. +pub const NT_PPC_SPE: u32 = 0x101; +/// PowerPC VSX registers. +pub const NT_PPC_VSX: u32 = 0x102; +/// Target Address Register. +pub const NT_PPC_TAR: u32 = 0x103; +/// Program Priority Register. +pub const NT_PPC_PPR: u32 = 0x104; +/// Data Stream Control Register. +pub const NT_PPC_DSCR: u32 = 0x105; +/// Event Based Branch Registers. +pub const NT_PPC_EBB: u32 = 0x106; +/// Performance Monitor Registers. +pub const NT_PPC_PMU: u32 = 0x107; +/// TM checkpointed GPR Registers. +pub const NT_PPC_TM_CGPR: u32 = 0x108; +/// TM checkpointed FPR Registers. +pub const NT_PPC_TM_CFPR: u32 = 0x109; +/// TM checkpointed VMX Registers. +pub const NT_PPC_TM_CVMX: u32 = 0x10a; +/// TM checkpointed VSX Registers. +pub const NT_PPC_TM_CVSX: u32 = 0x10b; +/// TM Special Purpose Registers. +pub const NT_PPC_TM_SPR: u32 = 0x10c; +/// TM checkpointed Target Address Register. +pub const NT_PPC_TM_CTAR: u32 = 0x10d; +/// TM checkpointed Program Priority Register. +pub const NT_PPC_TM_CPPR: u32 = 0x10e; +/// TM checkpointed Data Stream Control Register. +pub const NT_PPC_TM_CDSCR: u32 = 0x10f; +/// Memory Protection Keys registers. +pub const NT_PPC_PKEY: u32 = 0x110; +/// i386 TLS slots (struct user_desc). +pub const NT_386_TLS: u32 = 0x200; +/// x86 io permission bitmap (1=deny). +pub const NT_386_IOPERM: u32 = 0x201; +/// x86 extended state using xsave. +pub const NT_X86_XSTATE: u32 = 0x202; +/// s390 upper register halves. +pub const NT_S390_HIGH_GPRS: u32 = 0x300; +/// s390 timer register. +pub const NT_S390_TIMER: u32 = 0x301; +/// s390 TOD clock comparator register. +pub const NT_S390_TODCMP: u32 = 0x302; +/// s390 TOD programmable register. +pub const NT_S390_TODPREG: u32 = 0x303; +/// s390 control registers. +pub const NT_S390_CTRS: u32 = 0x304; +/// s390 prefix register. +pub const NT_S390_PREFIX: u32 = 0x305; +/// s390 breaking event address. +pub const NT_S390_LAST_BREAK: u32 = 0x306; +/// s390 system call restart data. +pub const NT_S390_SYSTEM_CALL: u32 = 0x307; +/// s390 transaction diagnostic block. +pub const NT_S390_TDB: u32 = 0x308; +/// s390 vector registers 0-15 upper half. +pub const NT_S390_VXRS_LOW: u32 = 0x309; +/// s390 vector registers 16-31. +pub const NT_S390_VXRS_HIGH: u32 = 0x30a; +/// s390 guarded storage registers. +pub const NT_S390_GS_CB: u32 = 0x30b; +/// s390 guarded storage broadcast control block. +pub const NT_S390_GS_BC: u32 = 0x30c; +/// s390 runtime instrumentation. +pub const NT_S390_RI_CB: u32 = 0x30d; +/// ARM VFP/NEON registers. +pub const NT_ARM_VFP: u32 = 0x400; +/// ARM TLS register. +pub const NT_ARM_TLS: u32 = 0x401; +/// ARM hardware breakpoint registers. +pub const NT_ARM_HW_BREAK: u32 = 0x402; +/// ARM hardware watchpoint registers. +pub const NT_ARM_HW_WATCH: u32 = 0x403; +/// ARM system call number. +pub const NT_ARM_SYSTEM_CALL: u32 = 0x404; +/// ARM Scalable Vector Extension registers. +pub const NT_ARM_SVE: u32 = 0x405; +/// Vmcore Device Dump Note. +pub const NT_VMCOREDD: u32 = 0x700; +/// MIPS DSP ASE registers. +pub const NT_MIPS_DSP: u32 = 0x800; +/// MIPS floating-point mode. +pub const NT_MIPS_FP_MODE: u32 = 0x801; + +/// Note type for version string. +/// +/// This note may appear in object files. +/// +/// It must be handled as a special case because it has no descriptor, and instead +/// uses the note name as the version string. +pub const NT_VERSION: u32 = 1; + +/// Dynamic section entry. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Dyn32 { + /// Dynamic entry type. + pub d_tag: U32, + /// Value (integer or address). + pub d_val: U32, +} + +/// Dynamic section entry. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Dyn64 { + /// Dynamic entry type. + pub d_tag: U64, + /// Value (integer or address). + pub d_val: U64, +} + +// Values for `Dyn*::d_tag`. + +/// Marks end of dynamic section +pub const DT_NULL: u32 = 0; +/// Name of needed library +pub const DT_NEEDED: u32 = 1; +/// Size in bytes of PLT relocs +pub const DT_PLTRELSZ: u32 = 2; +/// Processor defined value +pub const DT_PLTGOT: u32 = 3; +/// Address of symbol hash table +pub const DT_HASH: u32 = 4; +/// Address of string table +pub const DT_STRTAB: u32 = 5; +/// Address of symbol table +pub const DT_SYMTAB: u32 = 6; +/// Address of Rela relocs +pub const DT_RELA: u32 = 7; +/// Total size of Rela relocs +pub const DT_RELASZ: u32 = 8; +/// Size of one Rela reloc +pub const DT_RELAENT: u32 = 9; +/// Size of string table +pub const DT_STRSZ: u32 = 10; +/// Size of one symbol table entry +pub const DT_SYMENT: u32 = 11; +/// Address of init function +pub const DT_INIT: u32 = 12; +/// Address of termination function +pub const DT_FINI: u32 = 13; +/// Name of shared object +pub const DT_SONAME: u32 = 14; +/// Library search path (deprecated) +pub const DT_RPATH: u32 = 15; +/// Start symbol search here +pub const DT_SYMBOLIC: u32 = 16; +/// Address of Rel relocs +pub const DT_REL: u32 = 17; +/// Total size of Rel relocs +pub const DT_RELSZ: u32 = 18; +/// Size of one Rel reloc +pub const DT_RELENT: u32 = 19; +/// Type of reloc in PLT +pub const DT_PLTREL: u32 = 20; +/// For debugging; unspecified +pub const DT_DEBUG: u32 = 21; +/// Reloc might modify .text +pub const DT_TEXTREL: u32 = 22; +/// Address of PLT relocs +pub const DT_JMPREL: u32 = 23; +/// Process relocations of object +pub const DT_BIND_NOW: u32 = 24; +/// Array with addresses of init fct +pub const DT_INIT_ARRAY: u32 = 25; +/// Array with addresses of fini fct +pub const DT_FINI_ARRAY: u32 = 26; +/// Size in bytes of DT_INIT_ARRAY +pub const DT_INIT_ARRAYSZ: u32 = 27; +/// Size in bytes of DT_FINI_ARRAY +pub const DT_FINI_ARRAYSZ: u32 = 28; +/// Library search path +pub const DT_RUNPATH: u32 = 29; +/// Flags for the object being loaded +pub const DT_FLAGS: u32 = 30; +/// Start of encoded range +pub const DT_ENCODING: u32 = 32; +/// Array with addresses of preinit fct +pub const DT_PREINIT_ARRAY: u32 = 32; +/// size in bytes of DT_PREINIT_ARRAY +pub const DT_PREINIT_ARRAYSZ: u32 = 33; +/// Address of SYMTAB_SHNDX section +pub const DT_SYMTAB_SHNDX: u32 = 34; +/// Start of OS-specific +pub const DT_LOOS: u32 = 0x6000_000d; +/// End of OS-specific +pub const DT_HIOS: u32 = 0x6fff_f000; +/// Start of processor-specific +pub const DT_LOPROC: u32 = 0x7000_0000; +/// End of processor-specific +pub const DT_HIPROC: u32 = 0x7fff_ffff; + +// `DT_*` entries between `DT_VALRNGHI` & `DT_VALRNGLO` use `d_val` as a value. +#[allow(missing_docs)] +pub const DT_VALRNGLO: u32 = 0x6fff_fd00; +/// Prelinking timestamp +pub const DT_GNU_PRELINKED: u32 = 0x6fff_fdf5; +/// Size of conflict section +pub const DT_GNU_CONFLICTSZ: u32 = 0x6fff_fdf6; +/// Size of library list +pub const DT_GNU_LIBLISTSZ: u32 = 0x6fff_fdf7; +#[allow(missing_docs)] +pub const DT_CHECKSUM: u32 = 0x6fff_fdf8; +#[allow(missing_docs)] +pub const DT_PLTPADSZ: u32 = 0x6fff_fdf9; +#[allow(missing_docs)] +pub const DT_MOVEENT: u32 = 0x6fff_fdfa; +#[allow(missing_docs)] +pub const DT_MOVESZ: u32 = 0x6fff_fdfb; +/// Feature selection (DTF_*). +pub const DT_FEATURE_1: u32 = 0x6fff_fdfc; +/// Flags for DT_* entries, affecting the following DT_* entry. +pub const DT_POSFLAG_1: u32 = 0x6fff_fdfd; +/// Size of syminfo table (in bytes) +pub const DT_SYMINSZ: u32 = 0x6fff_fdfe; +/// Entry size of syminfo +pub const DT_SYMINENT: u32 = 0x6fff_fdff; +#[allow(missing_docs)] +pub const DT_VALRNGHI: u32 = 0x6fff_fdff; + +// `DT_*` entries between `DT_ADDRRNGHI` & `DT_ADDRRNGLO` use `d_val` as an address. +// +// If any adjustment is made to the ELF object after it has been +// built these entries will need to be adjusted. +#[allow(missing_docs)] +pub const DT_ADDRRNGLO: u32 = 0x6fff_fe00; +/// GNU-style hash table. +pub const DT_GNU_HASH: u32 = 0x6fff_fef5; +#[allow(missing_docs)] +pub const DT_TLSDESC_PLT: u32 = 0x6fff_fef6; +#[allow(missing_docs)] +pub const DT_TLSDESC_GOT: u32 = 0x6fff_fef7; +/// Start of conflict section +pub const DT_GNU_CONFLICT: u32 = 0x6fff_fef8; +/// Library list +pub const DT_GNU_LIBLIST: u32 = 0x6fff_fef9; +/// Configuration information. +pub const DT_CONFIG: u32 = 0x6fff_fefa; +/// Dependency auditing. +pub const DT_DEPAUDIT: u32 = 0x6fff_fefb; +/// Object auditing. +pub const DT_AUDIT: u32 = 0x6fff_fefc; +/// PLT padding. +pub const DT_PLTPAD: u32 = 0x6fff_fefd; +/// Move table. +pub const DT_MOVETAB: u32 = 0x6fff_fefe; +/// Syminfo table. +pub const DT_SYMINFO: u32 = 0x6fff_feff; +#[allow(missing_docs)] +pub const DT_ADDRRNGHI: u32 = 0x6fff_feff; + +// The versioning entry types. The next are defined as part of the +// GNU extension. +#[allow(missing_docs)] +pub const DT_VERSYM: u32 = 0x6fff_fff0; +#[allow(missing_docs)] +pub const DT_RELACOUNT: u32 = 0x6fff_fff9; +#[allow(missing_docs)] +pub const DT_RELCOUNT: u32 = 0x6fff_fffa; +/// State flags, see DF_1_* below. +pub const DT_FLAGS_1: u32 = 0x6fff_fffb; +/// Address of version definition table +pub const DT_VERDEF: u32 = 0x6fff_fffc; +/// Number of version definitions +pub const DT_VERDEFNUM: u32 = 0x6fff_fffd; +/// Address of table with needed versions +pub const DT_VERNEED: u32 = 0x6fff_fffe; +/// Number of needed versions +pub const DT_VERNEEDNUM: u32 = 0x6fff_ffff; + +// Machine-independent extensions in the "processor-specific" range. +/// Shared object to load before self +pub const DT_AUXILIARY: u32 = 0x7fff_fffd; +/// Shared object to get values from +pub const DT_FILTER: u32 = 0x7fff_ffff; + +// Values of `Dyn*::d_val` in the `DT_FLAGS` entry. +/// Object may use DF_ORIGIN +pub const DF_ORIGIN: u32 = 0x0000_0001; +/// Symbol resolutions starts here +pub const DF_SYMBOLIC: u32 = 0x0000_0002; +/// Object contains text relocations +pub const DF_TEXTREL: u32 = 0x0000_0004; +/// No lazy binding for this object +pub const DF_BIND_NOW: u32 = 0x0000_0008; +/// Module uses the static TLS model +pub const DF_STATIC_TLS: u32 = 0x0000_0010; + +// Values of `Dyn*::d_val` in the `DT_FLAGS_1` entry. +/// Set RTLD_NOW for this object. +pub const DF_1_NOW: u32 = 0x0000_0001; +/// Set RTLD_GLOBAL for this object. +pub const DF_1_GLOBAL: u32 = 0x0000_0002; +/// Set RTLD_GROUP for this object. +pub const DF_1_GROUP: u32 = 0x0000_0004; +/// Set RTLD_NODELETE for this object. +pub const DF_1_NODELETE: u32 = 0x0000_0008; +/// Trigger filtee loading at runtime. +pub const DF_1_LOADFLTR: u32 = 0x0000_0010; +/// Set RTLD_INITFIRST for this object. +pub const DF_1_INITFIRST: u32 = 0x0000_0020; +/// Set RTLD_NOOPEN for this object. +pub const DF_1_NOOPEN: u32 = 0x0000_0040; +/// $ORIGIN must be handled. +pub const DF_1_ORIGIN: u32 = 0x0000_0080; +/// Direct binding enabled. +pub const DF_1_DIRECT: u32 = 0x0000_0100; +#[allow(missing_docs)] +pub const DF_1_TRANS: u32 = 0x0000_0200; +/// Object is used to interpose. +pub const DF_1_INTERPOSE: u32 = 0x0000_0400; +/// Ignore default lib search path. +pub const DF_1_NODEFLIB: u32 = 0x0000_0800; +/// Object can't be dldump'ed. +pub const DF_1_NODUMP: u32 = 0x0000_1000; +/// Configuration alternative created. +pub const DF_1_CONFALT: u32 = 0x0000_2000; +/// Filtee terminates filters search. +pub const DF_1_ENDFILTEE: u32 = 0x0000_4000; +/// Disp reloc applied at build time. +pub const DF_1_DISPRELDNE: u32 = 0x0000_8000; +/// Disp reloc applied at run-time. +pub const DF_1_DISPRELPND: u32 = 0x0001_0000; +/// Object has no-direct binding. +pub const DF_1_NODIRECT: u32 = 0x0002_0000; +#[allow(missing_docs)] +pub const DF_1_IGNMULDEF: u32 = 0x0004_0000; +#[allow(missing_docs)] +pub const DF_1_NOKSYMS: u32 = 0x0008_0000; +#[allow(missing_docs)] +pub const DF_1_NOHDR: u32 = 0x0010_0000; +/// Object is modified after built. +pub const DF_1_EDITED: u32 = 0x0020_0000; +#[allow(missing_docs)] +pub const DF_1_NORELOC: u32 = 0x0040_0000; +/// Object has individual interposers. +pub const DF_1_SYMINTPOSE: u32 = 0x0080_0000; +/// Global auditing required. +pub const DF_1_GLOBAUDIT: u32 = 0x0100_0000; +/// Singleton symbols are used. +pub const DF_1_SINGLETON: u32 = 0x0200_0000; +#[allow(missing_docs)] +pub const DF_1_STUB: u32 = 0x0400_0000; +#[allow(missing_docs)] +pub const DF_1_PIE: u32 = 0x0800_0000; + +/// Version symbol information +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Versym(pub U16); + +/// Symbol is hidden. +pub const VERSYM_HIDDEN: u16 = 0x8000; +/// Symbol version index. +pub const VERSYM_VERSION: u16 = 0x7fff; + +/// Version definition sections +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Verdef { + /// Version revision + pub vd_version: U16, + /// Version information + pub vd_flags: U16, + /// Version Index + pub vd_ndx: U16, + /// Number of associated aux entries + pub vd_cnt: U16, + /// Version name hash value + pub vd_hash: U32, + /// Offset in bytes to verdaux array + pub vd_aux: U32, + /// Offset in bytes to next verdef entry + pub vd_next: U32, +} + +// Legal values for vd_version (version revision). +/// No version +pub const VER_DEF_NONE: u16 = 0; +/// Current version +pub const VER_DEF_CURRENT: u16 = 1; + +// Legal values for vd_flags (version information flags). +/// Version definition of file itself +pub const VER_FLG_BASE: u16 = 0x1; +// Legal values for vd_flags and vna_flags (version information flags). +/// Weak version identifier +pub const VER_FLG_WEAK: u16 = 0x2; + +// Versym symbol index values. +/// Symbol is local. +pub const VER_NDX_LOCAL: u16 = 0; +/// Symbol is global. +pub const VER_NDX_GLOBAL: u16 = 1; + +/// Auxiliary version information. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Verdaux { + /// Version or dependency names + pub vda_name: U32, + /// Offset in bytes to next verdaux + pub vda_next: U32, +} + +/// Version dependency. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Verneed { + /// Version of structure + pub vn_version: U16, + /// Number of associated aux entries + pub vn_cnt: U16, + /// Offset of filename for this dependency + pub vn_file: U32, + /// Offset in bytes to vernaux array + pub vn_aux: U32, + /// Offset in bytes to next verneed entry + pub vn_next: U32, +} + +// Legal values for vn_version (version revision). +/// No version +pub const VER_NEED_NONE: u16 = 0; +/// Current version +pub const VER_NEED_CURRENT: u16 = 1; + +/// Auxiliary needed version information. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Vernaux { + /// Hash value of dependency name + pub vna_hash: U32, + /// Dependency specific information + pub vna_flags: U16, + /// Version Index + pub vna_other: U16, + /// Dependency name string offset + pub vna_name: U32, + /// Offset in bytes to next vernaux entry + pub vna_next: U32, +} + +// TODO: Elf*_auxv_t, AT_* + +/// Note section entry header. +/// +/// A note consists of a header followed by a variable length name and descriptor. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct NoteHeader32 { + /// Length of the note's name. + /// + /// Some known names are defined by the `ELF_NOTE_*` constants. + pub n_namesz: U32, + /// Length of the note's descriptor. + /// + /// The content of the descriptor depends on the note name and type. + pub n_descsz: U32, + /// Type of the note. + /// + /// One of the `NT_*` constants. The note name determines which + /// `NT_*` constants are valid. + pub n_type: U32, +} + +/// Note section entry header. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct NoteHeader64 { + /// Length of the note's name. + /// + /// Some known names are defined by the `ELF_NOTE_*` constants. + pub n_namesz: U32, + /// Length of the note's descriptor. + /// + /// The content of the descriptor depends on the note name and type. + pub n_descsz: U32, + /// Type of the note. + /// + /// One of the `NT_*` constants. The note name determines which + /// `NT_*` constants are valid. + pub n_type: U32, +} + +/// Solaris entries in the note section have this name. +pub static ELF_NOTE_SOLARIS: &[u8] = b"SUNW Solaris"; + +// Values for `n_type` when the name is `ELF_NOTE_SOLARIS`. +/// Desired pagesize for the binary. +pub const NT_SOLARIS_PAGESIZE_HINT: u32 = 1; + +/// GNU entries in the note section have this name. +pub static ELF_NOTE_GNU: &[u8] = b"GNU"; + +// Note types for `ELF_NOTE_GNU`. + +/// ABI information. +/// +/// The descriptor consists of words: +/// - word 0: OS descriptor +/// - word 1: major version of the ABI +/// - word 2: minor version of the ABI +/// - word 3: subminor version of the ABI +pub const NT_GNU_ABI_TAG: u32 = 1; + +/// OS descriptor for `NT_GNU_ABI_TAG`. +pub const ELF_NOTE_OS_LINUX: u32 = 0; +/// OS descriptor for `NT_GNU_ABI_TAG`. +pub const ELF_NOTE_OS_GNU: u32 = 1; +/// OS descriptor for `NT_GNU_ABI_TAG`. +pub const ELF_NOTE_OS_SOLARIS2: u32 = 2; +/// OS descriptor for `NT_GNU_ABI_TAG`. +pub const ELF_NOTE_OS_FREEBSD: u32 = 3; + +/// Synthetic hwcap information. +/// +/// The descriptor begins with two words: +/// - word 0: number of entries +/// - word 1: bitmask of enabled entries +/// Then follow variable-length entries, one byte followed by a +/// '\0'-terminated hwcap name string. The byte gives the bit +/// number to test if enabled, (1U << bit) & bitmask. */ +pub const NT_GNU_HWCAP: u32 = 2; + +/// Build ID bits as generated by `ld --build-id`. +/// +/// The descriptor consists of any nonzero number of bytes. +pub const NT_GNU_BUILD_ID: u32 = 3; + +/// Version note generated by GNU gold containing a version string. +pub const NT_GNU_GOLD_VERSION: u32 = 4; + +/// Program property. +pub const NT_GNU_PROPERTY_TYPE_0: u32 = 5; + +// TODO: GNU_PROPERTY_* +// TODO: Elf*_Move + +/// Header of `SHT_HASH` section. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct HashHeader { + /// The number of hash buckets. + pub bucket_count: U32, + /// The number of chain values. + pub chain_count: U32, + // Array of hash bucket start indices. + // buckets: U32[bucket_count] + // Array of hash chain links. An index of 0 terminates the chain. + // chains: U32[chain_count] +} + +/// Calculate the SysV hash for a symbol name. +/// +/// Used for `SHT_HASH`. +pub fn hash(name: &[u8]) -> u32 { + let mut hash = 0u32; + for byte in name { + hash = hash.wrapping_mul(16).wrapping_add(u32::from(*byte)); + hash ^= (hash >> 24) & 0xf0; + } + hash & 0xfff_ffff +} + +/// Header of `SHT_GNU_HASH` section. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct GnuHashHeader { + /// The number of hash buckets. + pub bucket_count: U32, + /// The symbol table index of the first symbol in the hash. + pub symbol_base: U32, + /// The number of words in the bloom filter. + /// + /// Must be a non-zero power of 2. + pub bloom_count: U32, + /// The bit shift count for the bloom filter. + pub bloom_shift: U32, + // Array of bloom filter words. + // bloom_filters: U32[bloom_count] or U64[bloom_count] + // Array of hash bucket start indices. + // buckets: U32[bucket_count] + // Array of hash values, one for each symbol starting at symbol_base. + // values: U32[symbol_count] +} + +/// Calculate the GNU hash for a symbol name. +/// +/// Used for `SHT_GNU_HASH`. +pub fn gnu_hash(name: &[u8]) -> u32 { + let mut hash = 5381u32; + for byte in name { + hash = hash.wrapping_mul(33).wrapping_add(u32::from(*byte)); + } + hash +} + +// Motorola 68k specific definitions. + +// m68k values for `Rel*::r_type`. + +/// No reloc +pub const R_68K_NONE: u32 = 0; +/// Direct 32 bit +pub const R_68K_32: u32 = 1; +/// Direct 16 bit +pub const R_68K_16: u32 = 2; +/// Direct 8 bit +pub const R_68K_8: u32 = 3; +/// PC relative 32 bit +pub const R_68K_PC32: u32 = 4; +/// PC relative 16 bit +pub const R_68K_PC16: u32 = 5; +/// PC relative 8 bit +pub const R_68K_PC8: u32 = 6; +/// 32 bit PC relative GOT entry +pub const R_68K_GOT32: u32 = 7; +/// 16 bit PC relative GOT entry +pub const R_68K_GOT16: u32 = 8; +/// 8 bit PC relative GOT entry +pub const R_68K_GOT8: u32 = 9; +/// 32 bit GOT offset +pub const R_68K_GOT32O: u32 = 10; +/// 16 bit GOT offset +pub const R_68K_GOT16O: u32 = 11; +/// 8 bit GOT offset +pub const R_68K_GOT8O: u32 = 12; +/// 32 bit PC relative PLT address +pub const R_68K_PLT32: u32 = 13; +/// 16 bit PC relative PLT address +pub const R_68K_PLT16: u32 = 14; +/// 8 bit PC relative PLT address +pub const R_68K_PLT8: u32 = 15; +/// 32 bit PLT offset +pub const R_68K_PLT32O: u32 = 16; +/// 16 bit PLT offset +pub const R_68K_PLT16O: u32 = 17; +/// 8 bit PLT offset +pub const R_68K_PLT8O: u32 = 18; +/// Copy symbol at runtime +pub const R_68K_COPY: u32 = 19; +/// Create GOT entry +pub const R_68K_GLOB_DAT: u32 = 20; +/// Create PLT entry +pub const R_68K_JMP_SLOT: u32 = 21; +/// Adjust by program base +pub const R_68K_RELATIVE: u32 = 22; +/// 32 bit GOT offset for GD +pub const R_68K_TLS_GD32: u32 = 25; +/// 16 bit GOT offset for GD +pub const R_68K_TLS_GD16: u32 = 26; +/// 8 bit GOT offset for GD +pub const R_68K_TLS_GD8: u32 = 27; +/// 32 bit GOT offset for LDM +pub const R_68K_TLS_LDM32: u32 = 28; +/// 16 bit GOT offset for LDM +pub const R_68K_TLS_LDM16: u32 = 29; +/// 8 bit GOT offset for LDM +pub const R_68K_TLS_LDM8: u32 = 30; +/// 32 bit module-relative offset +pub const R_68K_TLS_LDO32: u32 = 31; +/// 16 bit module-relative offset +pub const R_68K_TLS_LDO16: u32 = 32; +/// 8 bit module-relative offset +pub const R_68K_TLS_LDO8: u32 = 33; +/// 32 bit GOT offset for IE +pub const R_68K_TLS_IE32: u32 = 34; +/// 16 bit GOT offset for IE +pub const R_68K_TLS_IE16: u32 = 35; +/// 8 bit GOT offset for IE +pub const R_68K_TLS_IE8: u32 = 36; +/// 32 bit offset relative to static TLS block +pub const R_68K_TLS_LE32: u32 = 37; +/// 16 bit offset relative to static TLS block +pub const R_68K_TLS_LE16: u32 = 38; +/// 8 bit offset relative to static TLS block +pub const R_68K_TLS_LE8: u32 = 39; +/// 32 bit module number +pub const R_68K_TLS_DTPMOD32: u32 = 40; +/// 32 bit module-relative offset +pub const R_68K_TLS_DTPREL32: u32 = 41; +/// 32 bit TP-relative offset +pub const R_68K_TLS_TPREL32: u32 = 42; + +// Intel 80386 specific definitions. + +// i386 values for `Rel*::r_type`. + +/// No reloc +pub const R_386_NONE: u32 = 0; +/// Direct 32 bit +pub const R_386_32: u32 = 1; +/// PC relative 32 bit +pub const R_386_PC32: u32 = 2; +/// 32 bit GOT entry +pub const R_386_GOT32: u32 = 3; +/// 32 bit PLT address +pub const R_386_PLT32: u32 = 4; +/// Copy symbol at runtime +pub const R_386_COPY: u32 = 5; +/// Create GOT entry +pub const R_386_GLOB_DAT: u32 = 6; +/// Create PLT entry +pub const R_386_JMP_SLOT: u32 = 7; +/// Adjust by program base +pub const R_386_RELATIVE: u32 = 8; +/// 32 bit offset to GOT +pub const R_386_GOTOFF: u32 = 9; +/// 32 bit PC relative offset to GOT +pub const R_386_GOTPC: u32 = 10; +/// Direct 32 bit PLT address +pub const R_386_32PLT: u32 = 11; +/// Offset in static TLS block +pub const R_386_TLS_TPOFF: u32 = 14; +/// Address of GOT entry for static TLS block offset +pub const R_386_TLS_IE: u32 = 15; +/// GOT entry for static TLS block offset +pub const R_386_TLS_GOTIE: u32 = 16; +/// Offset relative to static TLS block +pub const R_386_TLS_LE: u32 = 17; +/// Direct 32 bit for GNU version of general dynamic thread local data +pub const R_386_TLS_GD: u32 = 18; +/// Direct 32 bit for GNU version of local dynamic thread local data in LE code +pub const R_386_TLS_LDM: u32 = 19; +/// Direct 16 bit +pub const R_386_16: u32 = 20; +/// PC relative 16 bit +pub const R_386_PC16: u32 = 21; +/// Direct 8 bit +pub const R_386_8: u32 = 22; +/// PC relative 8 bit +pub const R_386_PC8: u32 = 23; +/// Direct 32 bit for general dynamic thread local data +pub const R_386_TLS_GD_32: u32 = 24; +/// Tag for pushl in GD TLS code +pub const R_386_TLS_GD_PUSH: u32 = 25; +/// Relocation for call to __tls_get_addr() +pub const R_386_TLS_GD_CALL: u32 = 26; +/// Tag for popl in GD TLS code +pub const R_386_TLS_GD_POP: u32 = 27; +/// Direct 32 bit for local dynamic thread local data in LE code +pub const R_386_TLS_LDM_32: u32 = 28; +/// Tag for pushl in LDM TLS code +pub const R_386_TLS_LDM_PUSH: u32 = 29; +/// Relocation for call to __tls_get_addr() in LDM code +pub const R_386_TLS_LDM_CALL: u32 = 30; +/// Tag for popl in LDM TLS code +pub const R_386_TLS_LDM_POP: u32 = 31; +/// Offset relative to TLS block +pub const R_386_TLS_LDO_32: u32 = 32; +/// GOT entry for negated static TLS block offset +pub const R_386_TLS_IE_32: u32 = 33; +/// Negated offset relative to static TLS block +pub const R_386_TLS_LE_32: u32 = 34; +/// ID of module containing symbol +pub const R_386_TLS_DTPMOD32: u32 = 35; +/// Offset in TLS block +pub const R_386_TLS_DTPOFF32: u32 = 36; +/// Negated offset in static TLS block +pub const R_386_TLS_TPOFF32: u32 = 37; +/// 32-bit symbol size +pub const R_386_SIZE32: u32 = 38; +/// GOT offset for TLS descriptor. +pub const R_386_TLS_GOTDESC: u32 = 39; +/// Marker of call through TLS descriptor for relaxation. +pub const R_386_TLS_DESC_CALL: u32 = 40; +/// TLS descriptor containing pointer to code and to argument, returning the TLS offset for the symbol. +pub const R_386_TLS_DESC: u32 = 41; +/// Adjust indirectly by program base +pub const R_386_IRELATIVE: u32 = 42; +/// Load from 32 bit GOT entry, relaxable. +pub const R_386_GOT32X: u32 = 43; + +// SUN SPARC specific definitions. + +// SPARC values for `st_type` component of `Sym*::st_info`. + +/// Global register reserved to app. +pub const STT_SPARC_REGISTER: u8 = 13; + +// SPARC values for `FileHeader64::e_flags`. + +#[allow(missing_docs)] +pub const EF_SPARCV9_MM: u32 = 3; +#[allow(missing_docs)] +pub const EF_SPARCV9_TSO: u32 = 0; +#[allow(missing_docs)] +pub const EF_SPARCV9_PSO: u32 = 1; +#[allow(missing_docs)] +pub const EF_SPARCV9_RMO: u32 = 2; +/// little endian data +pub const EF_SPARC_LEDATA: u32 = 0x80_0000; +#[allow(missing_docs)] +pub const EF_SPARC_EXT_MASK: u32 = 0xFF_FF00; +/// generic V8+ features +pub const EF_SPARC_32PLUS: u32 = 0x00_0100; +/// Sun UltraSPARC1 extensions +pub const EF_SPARC_SUN_US1: u32 = 0x00_0200; +/// HAL R1 extensions +pub const EF_SPARC_HAL_R1: u32 = 0x00_0400; +/// Sun UltraSPARCIII extensions +pub const EF_SPARC_SUN_US3: u32 = 0x00_0800; + +// SPARC values for `Rel*::r_type`. + +/// No reloc +pub const R_SPARC_NONE: u32 = 0; +/// Direct 8 bit +pub const R_SPARC_8: u32 = 1; +/// Direct 16 bit +pub const R_SPARC_16: u32 = 2; +/// Direct 32 bit +pub const R_SPARC_32: u32 = 3; +/// PC relative 8 bit +pub const R_SPARC_DISP8: u32 = 4; +/// PC relative 16 bit +pub const R_SPARC_DISP16: u32 = 5; +/// PC relative 32 bit +pub const R_SPARC_DISP32: u32 = 6; +/// PC relative 30 bit shifted +pub const R_SPARC_WDISP30: u32 = 7; +/// PC relative 22 bit shifted +pub const R_SPARC_WDISP22: u32 = 8; +/// High 22 bit +pub const R_SPARC_HI22: u32 = 9; +/// Direct 22 bit +pub const R_SPARC_22: u32 = 10; +/// Direct 13 bit +pub const R_SPARC_13: u32 = 11; +/// Truncated 10 bit +pub const R_SPARC_LO10: u32 = 12; +/// Truncated 10 bit GOT entry +pub const R_SPARC_GOT10: u32 = 13; +/// 13 bit GOT entry +pub const R_SPARC_GOT13: u32 = 14; +/// 22 bit GOT entry shifted +pub const R_SPARC_GOT22: u32 = 15; +/// PC relative 10 bit truncated +pub const R_SPARC_PC10: u32 = 16; +/// PC relative 22 bit shifted +pub const R_SPARC_PC22: u32 = 17; +/// 30 bit PC relative PLT address +pub const R_SPARC_WPLT30: u32 = 18; +/// Copy symbol at runtime +pub const R_SPARC_COPY: u32 = 19; +/// Create GOT entry +pub const R_SPARC_GLOB_DAT: u32 = 20; +/// Create PLT entry +pub const R_SPARC_JMP_SLOT: u32 = 21; +/// Adjust by program base +pub const R_SPARC_RELATIVE: u32 = 22; +/// Direct 32 bit unaligned +pub const R_SPARC_UA32: u32 = 23; + +// Sparc64 values for `Rel*::r_type`. + +/// Direct 32 bit ref to PLT entry +pub const R_SPARC_PLT32: u32 = 24; +/// High 22 bit PLT entry +pub const R_SPARC_HIPLT22: u32 = 25; +/// Truncated 10 bit PLT entry +pub const R_SPARC_LOPLT10: u32 = 26; +/// PC rel 32 bit ref to PLT entry +pub const R_SPARC_PCPLT32: u32 = 27; +/// PC rel high 22 bit PLT entry +pub const R_SPARC_PCPLT22: u32 = 28; +/// PC rel trunc 10 bit PLT entry +pub const R_SPARC_PCPLT10: u32 = 29; +/// Direct 10 bit +pub const R_SPARC_10: u32 = 30; +/// Direct 11 bit +pub const R_SPARC_11: u32 = 31; +/// Direct 64 bit +pub const R_SPARC_64: u32 = 32; +/// 10bit with secondary 13bit addend +pub const R_SPARC_OLO10: u32 = 33; +/// Top 22 bits of direct 64 bit +pub const R_SPARC_HH22: u32 = 34; +/// High middle 10 bits of ... +pub const R_SPARC_HM10: u32 = 35; +/// Low middle 22 bits of ... +pub const R_SPARC_LM22: u32 = 36; +/// Top 22 bits of pc rel 64 bit +pub const R_SPARC_PC_HH22: u32 = 37; +/// High middle 10 bit of ... +pub const R_SPARC_PC_HM10: u32 = 38; +/// Low miggle 22 bits of ... +pub const R_SPARC_PC_LM22: u32 = 39; +/// PC relative 16 bit shifted +pub const R_SPARC_WDISP16: u32 = 40; +/// PC relative 19 bit shifted +pub const R_SPARC_WDISP19: u32 = 41; +/// was part of v9 ABI but was removed +pub const R_SPARC_GLOB_JMP: u32 = 42; +/// Direct 7 bit +pub const R_SPARC_7: u32 = 43; +/// Direct 5 bit +pub const R_SPARC_5: u32 = 44; +/// Direct 6 bit +pub const R_SPARC_6: u32 = 45; +/// PC relative 64 bit +pub const R_SPARC_DISP64: u32 = 46; +/// Direct 64 bit ref to PLT entry +pub const R_SPARC_PLT64: u32 = 47; +/// High 22 bit complemented +pub const R_SPARC_HIX22: u32 = 48; +/// Truncated 11 bit complemented +pub const R_SPARC_LOX10: u32 = 49; +/// Direct high 12 of 44 bit +pub const R_SPARC_H44: u32 = 50; +/// Direct mid 22 of 44 bit +pub const R_SPARC_M44: u32 = 51; +/// Direct low 10 of 44 bit +pub const R_SPARC_L44: u32 = 52; +/// Global register usage +pub const R_SPARC_REGISTER: u32 = 53; +/// Direct 64 bit unaligned +pub const R_SPARC_UA64: u32 = 54; +/// Direct 16 bit unaligned +pub const R_SPARC_UA16: u32 = 55; +#[allow(missing_docs)] +pub const R_SPARC_TLS_GD_HI22: u32 = 56; +#[allow(missing_docs)] +pub const R_SPARC_TLS_GD_LO10: u32 = 57; +#[allow(missing_docs)] +pub const R_SPARC_TLS_GD_ADD: u32 = 58; +#[allow(missing_docs)] +pub const R_SPARC_TLS_GD_CALL: u32 = 59; +#[allow(missing_docs)] +pub const R_SPARC_TLS_LDM_HI22: u32 = 60; +#[allow(missing_docs)] +pub const R_SPARC_TLS_LDM_LO10: u32 = 61; +#[allow(missing_docs)] +pub const R_SPARC_TLS_LDM_ADD: u32 = 62; +#[allow(missing_docs)] +pub const R_SPARC_TLS_LDM_CALL: u32 = 63; +#[allow(missing_docs)] +pub const R_SPARC_TLS_LDO_HIX22: u32 = 64; +#[allow(missing_docs)] +pub const R_SPARC_TLS_LDO_LOX10: u32 = 65; +#[allow(missing_docs)] +pub const R_SPARC_TLS_LDO_ADD: u32 = 66; +#[allow(missing_docs)] +pub const R_SPARC_TLS_IE_HI22: u32 = 67; +#[allow(missing_docs)] +pub const R_SPARC_TLS_IE_LO10: u32 = 68; +#[allow(missing_docs)] +pub const R_SPARC_TLS_IE_LD: u32 = 69; +#[allow(missing_docs)] +pub const R_SPARC_TLS_IE_LDX: u32 = 70; +#[allow(missing_docs)] +pub const R_SPARC_TLS_IE_ADD: u32 = 71; +#[allow(missing_docs)] +pub const R_SPARC_TLS_LE_HIX22: u32 = 72; +#[allow(missing_docs)] +pub const R_SPARC_TLS_LE_LOX10: u32 = 73; +#[allow(missing_docs)] +pub const R_SPARC_TLS_DTPMOD32: u32 = 74; +#[allow(missing_docs)] +pub const R_SPARC_TLS_DTPMOD64: u32 = 75; +#[allow(missing_docs)] +pub const R_SPARC_TLS_DTPOFF32: u32 = 76; +#[allow(missing_docs)] +pub const R_SPARC_TLS_DTPOFF64: u32 = 77; +#[allow(missing_docs)] +pub const R_SPARC_TLS_TPOFF32: u32 = 78; +#[allow(missing_docs)] +pub const R_SPARC_TLS_TPOFF64: u32 = 79; +#[allow(missing_docs)] +pub const R_SPARC_GOTDATA_HIX22: u32 = 80; +#[allow(missing_docs)] +pub const R_SPARC_GOTDATA_LOX10: u32 = 81; +#[allow(missing_docs)] +pub const R_SPARC_GOTDATA_OP_HIX22: u32 = 82; +#[allow(missing_docs)] +pub const R_SPARC_GOTDATA_OP_LOX10: u32 = 83; +#[allow(missing_docs)] +pub const R_SPARC_GOTDATA_OP: u32 = 84; +#[allow(missing_docs)] +pub const R_SPARC_H34: u32 = 85; +#[allow(missing_docs)] +pub const R_SPARC_SIZE32: u32 = 86; +#[allow(missing_docs)] +pub const R_SPARC_SIZE64: u32 = 87; +#[allow(missing_docs)] +pub const R_SPARC_WDISP10: u32 = 88; +#[allow(missing_docs)] +pub const R_SPARC_JMP_IREL: u32 = 248; +#[allow(missing_docs)] +pub const R_SPARC_IRELATIVE: u32 = 249; +#[allow(missing_docs)] +pub const R_SPARC_GNU_VTINHERIT: u32 = 250; +#[allow(missing_docs)] +pub const R_SPARC_GNU_VTENTRY: u32 = 251; +#[allow(missing_docs)] +pub const R_SPARC_REV32: u32 = 252; + +// Sparc64 values for `Dyn32::d_tag`. + +#[allow(missing_docs)] +pub const DT_SPARC_REGISTER: u32 = 0x7000_0001; + +// MIPS R3000 specific definitions. + +// MIPS values for `FileHeader32::e_flags`. + +/// A .noreorder directive was used. +pub const EF_MIPS_NOREORDER: u32 = 1; +/// Contains PIC code. +pub const EF_MIPS_PIC: u32 = 2; +/// Uses PIC calling sequence. +pub const EF_MIPS_CPIC: u32 = 4; +#[allow(missing_docs)] +pub const EF_MIPS_XGOT: u32 = 8; +#[allow(missing_docs)] +pub const EF_MIPS_64BIT_WHIRL: u32 = 16; +#[allow(missing_docs)] +pub const EF_MIPS_ABI2: u32 = 32; +#[allow(missing_docs)] +pub const EF_MIPS_ABI_ON32: u32 = 64; +/// Uses FP64 (12 callee-saved). +pub const EF_MIPS_FP64: u32 = 512; +/// Uses IEEE 754-2008 NaN encoding. +pub const EF_MIPS_NAN2008: u32 = 1024; +/// MIPS architecture level. +pub const EF_MIPS_ARCH: u32 = 0xf000_0000; + +/// The first MIPS 32 bit ABI +pub const EF_MIPS_ABI_O32: u32 = 0x0000_1000; +/// O32 ABI extended for 64-bit architectures +pub const EF_MIPS_ABI_O64: u32 = 0x0000_2000; +/// EABI in 32-bit mode +pub const EF_MIPS_ABI_EABI32: u32 = 0x0000_3000; +/// EABI in 64-bit mode +pub const EF_MIPS_ABI_EABI64: u32 = 0x0000_4000; +/// Mask for selecting EF_MIPS_ABI_ variant +pub const EF_MIPS_ABI: u32 = 0x0000_f000; + +// Legal values for MIPS architecture level. + +/// -mips1 code. +pub const EF_MIPS_ARCH_1: u32 = 0x0000_0000; +/// -mips2 code. +pub const EF_MIPS_ARCH_2: u32 = 0x1000_0000; +/// -mips3 code. +pub const EF_MIPS_ARCH_3: u32 = 0x2000_0000; +/// -mips4 code. +pub const EF_MIPS_ARCH_4: u32 = 0x3000_0000; +/// -mips5 code. +pub const EF_MIPS_ARCH_5: u32 = 0x4000_0000; +/// MIPS32 code. +pub const EF_MIPS_ARCH_32: u32 = 0x5000_0000; +/// MIPS64 code. +pub const EF_MIPS_ARCH_64: u32 = 0x6000_0000; +/// MIPS32r2 code. +pub const EF_MIPS_ARCH_32R2: u32 = 0x7000_0000; +/// MIPS64r2 code. +pub const EF_MIPS_ARCH_64R2: u32 = 0x8000_0000; +/// MIPS32r6 code +pub const EF_MIPS_ARCH_32R6: u32 = 0x9000_0000; +/// MIPS64r6 code +pub const EF_MIPS_ARCH_64R6: u32 = 0xa000_0000; + +// MIPS values for `Sym32::st_shndx`. + +/// Allocated common symbols. +pub const SHN_MIPS_ACOMMON: u16 = 0xff00; +/// Allocated test symbols. +pub const SHN_MIPS_TEXT: u16 = 0xff01; +/// Allocated data symbols. +pub const SHN_MIPS_DATA: u16 = 0xff02; +/// Small common symbols. +pub const SHN_MIPS_SCOMMON: u16 = 0xff03; +/// Small undefined symbols. +pub const SHN_MIPS_SUNDEFINED: u16 = 0xff04; + +// MIPS values for `SectionHeader32::sh_type`. + +/// Shared objects used in link. +pub const SHT_MIPS_LIBLIST: u32 = 0x7000_0000; +#[allow(missing_docs)] +pub const SHT_MIPS_MSYM: u32 = 0x7000_0001; +/// Conflicting symbols. +pub const SHT_MIPS_CONFLICT: u32 = 0x7000_0002; +/// Global data area sizes. +pub const SHT_MIPS_GPTAB: u32 = 0x7000_0003; +/// Reserved for SGI/MIPS compilers +pub const SHT_MIPS_UCODE: u32 = 0x7000_0004; +/// MIPS ECOFF debugging info. +pub const SHT_MIPS_DEBUG: u32 = 0x7000_0005; +/// Register usage information. +pub const SHT_MIPS_REGINFO: u32 = 0x7000_0006; +#[allow(missing_docs)] +pub const SHT_MIPS_PACKAGE: u32 = 0x7000_0007; +#[allow(missing_docs)] +pub const SHT_MIPS_PACKSYM: u32 = 0x7000_0008; +#[allow(missing_docs)] +pub const SHT_MIPS_RELD: u32 = 0x7000_0009; +#[allow(missing_docs)] +pub const SHT_MIPS_IFACE: u32 = 0x7000_000b; +#[allow(missing_docs)] +pub const SHT_MIPS_CONTENT: u32 = 0x7000_000c; +/// Miscellaneous options. +pub const SHT_MIPS_OPTIONS: u32 = 0x7000_000d; +#[allow(missing_docs)] +pub const SHT_MIPS_SHDR: u32 = 0x7000_0010; +#[allow(missing_docs)] +pub const SHT_MIPS_FDESC: u32 = 0x7000_0011; +#[allow(missing_docs)] +pub const SHT_MIPS_EXTSYM: u32 = 0x7000_0012; +#[allow(missing_docs)] +pub const SHT_MIPS_DENSE: u32 = 0x7000_0013; +#[allow(missing_docs)] +pub const SHT_MIPS_PDESC: u32 = 0x7000_0014; +#[allow(missing_docs)] +pub const SHT_MIPS_LOCSYM: u32 = 0x7000_0015; +#[allow(missing_docs)] +pub const SHT_MIPS_AUXSYM: u32 = 0x7000_0016; +#[allow(missing_docs)] +pub const SHT_MIPS_OPTSYM: u32 = 0x7000_0017; +#[allow(missing_docs)] +pub const SHT_MIPS_LOCSTR: u32 = 0x7000_0018; +#[allow(missing_docs)] +pub const SHT_MIPS_LINE: u32 = 0x7000_0019; +#[allow(missing_docs)] +pub const SHT_MIPS_RFDESC: u32 = 0x7000_001a; +#[allow(missing_docs)] +pub const SHT_MIPS_DELTASYM: u32 = 0x7000_001b; +#[allow(missing_docs)] +pub const SHT_MIPS_DELTAINST: u32 = 0x7000_001c; +#[allow(missing_docs)] +pub const SHT_MIPS_DELTACLASS: u32 = 0x7000_001d; +/// DWARF debugging information. +pub const SHT_MIPS_DWARF: u32 = 0x7000_001e; +#[allow(missing_docs)] +pub const SHT_MIPS_DELTADECL: u32 = 0x7000_001f; +#[allow(missing_docs)] +pub const SHT_MIPS_SYMBOL_LIB: u32 = 0x7000_0020; +/// Event section. +pub const SHT_MIPS_EVENTS: u32 = 0x7000_0021; +#[allow(missing_docs)] +pub const SHT_MIPS_TRANSLATE: u32 = 0x7000_0022; +#[allow(missing_docs)] +pub const SHT_MIPS_PIXIE: u32 = 0x7000_0023; +#[allow(missing_docs)] +pub const SHT_MIPS_XLATE: u32 = 0x7000_0024; +#[allow(missing_docs)] +pub const SHT_MIPS_XLATE_DEBUG: u32 = 0x7000_0025; +#[allow(missing_docs)] +pub const SHT_MIPS_WHIRL: u32 = 0x7000_0026; +#[allow(missing_docs)] +pub const SHT_MIPS_EH_REGION: u32 = 0x7000_0027; +#[allow(missing_docs)] +pub const SHT_MIPS_XLATE_OLD: u32 = 0x7000_0028; +#[allow(missing_docs)] +pub const SHT_MIPS_PDR_EXCEPTION: u32 = 0x7000_0029; + +// MIPS values for `SectionHeader32::sh_flags`. + +/// Must be in global data area. +pub const SHF_MIPS_GPREL: u32 = 0x1000_0000; +#[allow(missing_docs)] +pub const SHF_MIPS_MERGE: u32 = 0x2000_0000; +#[allow(missing_docs)] +pub const SHF_MIPS_ADDR: u32 = 0x4000_0000; +#[allow(missing_docs)] +pub const SHF_MIPS_STRINGS: u32 = 0x8000_0000; +#[allow(missing_docs)] +pub const SHF_MIPS_NOSTRIP: u32 = 0x0800_0000; +#[allow(missing_docs)] +pub const SHF_MIPS_LOCAL: u32 = 0x0400_0000; +#[allow(missing_docs)] +pub const SHF_MIPS_NAMES: u32 = 0x0200_0000; +#[allow(missing_docs)] +pub const SHF_MIPS_NODUPE: u32 = 0x0100_0000; + +// MIPS values for `Sym32::st_other`. + +#[allow(missing_docs)] +pub const STO_MIPS_PLT: u8 = 0x8; +/// Only valid for `STB_MIPS_SPLIT_COMMON`. +pub const STO_MIPS_SC_ALIGN_UNUSED: u8 = 0xff; + +// MIPS values for `Sym32::st_info'. +#[allow(missing_docs)] +pub const STB_MIPS_SPLIT_COMMON: u8 = 13; + +// Entries found in sections of type `SHT_MIPS_GPTAB`. + +// TODO: Elf32_gptab, Elf32_RegInfo, Elf_Options + +// Values for `Elf_Options::kind`. + +/// Undefined. +pub const ODK_NULL: u32 = 0; +/// Register usage information. +pub const ODK_REGINFO: u32 = 1; +/// Exception processing options. +pub const ODK_EXCEPTIONS: u32 = 2; +/// Section padding options. +pub const ODK_PAD: u32 = 3; +/// Hardware workarounds performed +pub const ODK_HWPATCH: u32 = 4; +/// record the fill value used by the linker. +pub const ODK_FILL: u32 = 5; +/// reserve space for desktop tools to write. +pub const ODK_TAGS: u32 = 6; +/// HW workarounds. 'AND' bits when merging. +pub const ODK_HWAND: u32 = 7; +/// HW workarounds. 'OR' bits when merging. +pub const ODK_HWOR: u32 = 8; + +// Values for `Elf_Options::info` for `ODK_EXCEPTIONS` entries. + +/// FPE's which MUST be enabled. +pub const OEX_FPU_MIN: u32 = 0x1f; +/// FPE's which MAY be enabled. +pub const OEX_FPU_MAX: u32 = 0x1f00; +/// page zero must be mapped. +pub const OEX_PAGE0: u32 = 0x10000; +/// Force sequential memory mode? +pub const OEX_SMM: u32 = 0x20000; +/// Force floating point debug mode? +pub const OEX_FPDBUG: u32 = 0x40000; +#[allow(missing_docs)] +pub const OEX_PRECISEFP: u32 = OEX_FPDBUG; +/// Dismiss invalid address faults? +pub const OEX_DISMISS: u32 = 0x80000; + +#[allow(missing_docs)] +pub const OEX_FPU_INVAL: u32 = 0x10; +#[allow(missing_docs)] +pub const OEX_FPU_DIV0: u32 = 0x08; +#[allow(missing_docs)] +pub const OEX_FPU_OFLO: u32 = 0x04; +#[allow(missing_docs)] +pub const OEX_FPU_UFLO: u32 = 0x02; +#[allow(missing_docs)] +pub const OEX_FPU_INEX: u32 = 0x01; + +// Masks for `Elf_Options::info` for an `ODK_HWPATCH` entry. */ +/// R4000 end-of-page patch. +pub const OHW_R4KEOP: u32 = 0x1; +/// may need R8000 prefetch patch. +pub const OHW_R8KPFETCH: u32 = 0x2; +/// R5000 end-of-page patch. +pub const OHW_R5KEOP: u32 = 0x4; +/// R5000 cvt.\[ds\].l bug. clean=1. +pub const OHW_R5KCVTL: u32 = 0x8; + +#[allow(missing_docs)] +pub const OPAD_PREFIX: u32 = 0x1; +#[allow(missing_docs)] +pub const OPAD_POSTFIX: u32 = 0x2; +#[allow(missing_docs)] +pub const OPAD_SYMBOL: u32 = 0x4; + +// Entries found in sections of type `SHT_MIPS_OPTIONS`. + +// TODO: Elf_Options_Hw + +// Masks for `ElfOptions::info` for `ODK_HWAND` and `ODK_HWOR` entries. + +#[allow(missing_docs)] +pub const OHWA0_R4KEOP_CHECKED: u32 = 0x0000_0001; +#[allow(missing_docs)] +pub const OHWA1_R4KEOP_CLEAN: u32 = 0x0000_0002; + +// MIPS values for `Rel*::r_type`. + +/// No reloc +pub const R_MIPS_NONE: u32 = 0; +/// Direct 16 bit +pub const R_MIPS_16: u32 = 1; +/// Direct 32 bit +pub const R_MIPS_32: u32 = 2; +/// PC relative 32 bit +pub const R_MIPS_REL32: u32 = 3; +/// Direct 26 bit shifted +pub const R_MIPS_26: u32 = 4; +/// High 16 bit +pub const R_MIPS_HI16: u32 = 5; +/// Low 16 bit +pub const R_MIPS_LO16: u32 = 6; +/// GP relative 16 bit +pub const R_MIPS_GPREL16: u32 = 7; +/// 16 bit literal entry +pub const R_MIPS_LITERAL: u32 = 8; +/// 16 bit GOT entry +pub const R_MIPS_GOT16: u32 = 9; +/// PC relative 16 bit +pub const R_MIPS_PC16: u32 = 10; +/// 16 bit GOT entry for function +pub const R_MIPS_CALL16: u32 = 11; +/// GP relative 32 bit +pub const R_MIPS_GPREL32: u32 = 12; + +#[allow(missing_docs)] +pub const R_MIPS_SHIFT5: u32 = 16; +#[allow(missing_docs)] +pub const R_MIPS_SHIFT6: u32 = 17; +#[allow(missing_docs)] +pub const R_MIPS_64: u32 = 18; +#[allow(missing_docs)] +pub const R_MIPS_GOT_DISP: u32 = 19; +#[allow(missing_docs)] +pub const R_MIPS_GOT_PAGE: u32 = 20; +#[allow(missing_docs)] +pub const R_MIPS_GOT_OFST: u32 = 21; +#[allow(missing_docs)] +pub const R_MIPS_GOT_HI16: u32 = 22; +#[allow(missing_docs)] +pub const R_MIPS_GOT_LO16: u32 = 23; +#[allow(missing_docs)] +pub const R_MIPS_SUB: u32 = 24; +#[allow(missing_docs)] +pub const R_MIPS_INSERT_A: u32 = 25; +#[allow(missing_docs)] +pub const R_MIPS_INSERT_B: u32 = 26; +#[allow(missing_docs)] +pub const R_MIPS_DELETE: u32 = 27; +#[allow(missing_docs)] +pub const R_MIPS_HIGHER: u32 = 28; +#[allow(missing_docs)] +pub const R_MIPS_HIGHEST: u32 = 29; +#[allow(missing_docs)] +pub const R_MIPS_CALL_HI16: u32 = 30; +#[allow(missing_docs)] +pub const R_MIPS_CALL_LO16: u32 = 31; +#[allow(missing_docs)] +pub const R_MIPS_SCN_DISP: u32 = 32; +#[allow(missing_docs)] +pub const R_MIPS_REL16: u32 = 33; +#[allow(missing_docs)] +pub const R_MIPS_ADD_IMMEDIATE: u32 = 34; +#[allow(missing_docs)] +pub const R_MIPS_PJUMP: u32 = 35; +#[allow(missing_docs)] +pub const R_MIPS_RELGOT: u32 = 36; +#[allow(missing_docs)] +pub const R_MIPS_JALR: u32 = 37; +/// Module number 32 bit +pub const R_MIPS_TLS_DTPMOD32: u32 = 38; +/// Module-relative offset 32 bit +pub const R_MIPS_TLS_DTPREL32: u32 = 39; +/// Module number 64 bit +pub const R_MIPS_TLS_DTPMOD64: u32 = 40; +/// Module-relative offset 64 bit +pub const R_MIPS_TLS_DTPREL64: u32 = 41; +/// 16 bit GOT offset for GD +pub const R_MIPS_TLS_GD: u32 = 42; +/// 16 bit GOT offset for LDM +pub const R_MIPS_TLS_LDM: u32 = 43; +/// Module-relative offset, high 16 bits +pub const R_MIPS_TLS_DTPREL_HI16: u32 = 44; +/// Module-relative offset, low 16 bits +pub const R_MIPS_TLS_DTPREL_LO16: u32 = 45; +/// 16 bit GOT offset for IE +pub const R_MIPS_TLS_GOTTPREL: u32 = 46; +/// TP-relative offset, 32 bit +pub const R_MIPS_TLS_TPREL32: u32 = 47; +/// TP-relative offset, 64 bit +pub const R_MIPS_TLS_TPREL64: u32 = 48; +/// TP-relative offset, high 16 bits +pub const R_MIPS_TLS_TPREL_HI16: u32 = 49; +/// TP-relative offset, low 16 bits +pub const R_MIPS_TLS_TPREL_LO16: u32 = 50; +#[allow(missing_docs)] +pub const R_MIPS_GLOB_DAT: u32 = 51; +#[allow(missing_docs)] +pub const R_MIPS_COPY: u32 = 126; +#[allow(missing_docs)] +pub const R_MIPS_JUMP_SLOT: u32 = 127; + +// MIPS values for `ProgramHeader32::p_type`. + +/// Register usage information. +pub const PT_MIPS_REGINFO: u32 = 0x7000_0000; +/// Runtime procedure table. +pub const PT_MIPS_RTPROC: u32 = 0x7000_0001; +#[allow(missing_docs)] +pub const PT_MIPS_OPTIONS: u32 = 0x7000_0002; +/// FP mode requirement. +pub const PT_MIPS_ABIFLAGS: u32 = 0x7000_0003; + +// MIPS values for `ProgramHeader32::p_flags`. + +#[allow(missing_docs)] +pub const PF_MIPS_LOCAL: u32 = 0x1000_0000; + +// MIPS values for `Dyn32::d_tag`. + +/// Runtime linker interface version +pub const DT_MIPS_RLD_VERSION: u32 = 0x7000_0001; +/// Timestamp +pub const DT_MIPS_TIME_STAMP: u32 = 0x7000_0002; +/// Checksum +pub const DT_MIPS_ICHECKSUM: u32 = 0x7000_0003; +/// Version string (string tbl index) +pub const DT_MIPS_IVERSION: u32 = 0x7000_0004; +/// Flags +pub const DT_MIPS_FLAGS: u32 = 0x7000_0005; +/// Base address +pub const DT_MIPS_BASE_ADDRESS: u32 = 0x7000_0006; +#[allow(missing_docs)] +pub const DT_MIPS_MSYM: u32 = 0x7000_0007; +/// Address of CONFLICT section +pub const DT_MIPS_CONFLICT: u32 = 0x7000_0008; +/// Address of LIBLIST section +pub const DT_MIPS_LIBLIST: u32 = 0x7000_0009; +/// Number of local GOT entries +pub const DT_MIPS_LOCAL_GOTNO: u32 = 0x7000_000a; +/// Number of CONFLICT entries +pub const DT_MIPS_CONFLICTNO: u32 = 0x7000_000b; +/// Number of LIBLIST entries +pub const DT_MIPS_LIBLISTNO: u32 = 0x7000_0010; +/// Number of DYNSYM entries +pub const DT_MIPS_SYMTABNO: u32 = 0x7000_0011; +/// First external DYNSYM +pub const DT_MIPS_UNREFEXTNO: u32 = 0x7000_0012; +/// First GOT entry in DYNSYM +pub const DT_MIPS_GOTSYM: u32 = 0x7000_0013; +/// Number of GOT page table entries +pub const DT_MIPS_HIPAGENO: u32 = 0x7000_0014; +/// Address of run time loader map. +pub const DT_MIPS_RLD_MAP: u32 = 0x7000_0016; +/// Delta C++ class definition. +pub const DT_MIPS_DELTA_CLASS: u32 = 0x7000_0017; +/// Number of entries in DT_MIPS_DELTA_CLASS. +pub const DT_MIPS_DELTA_CLASS_NO: u32 = 0x7000_0018; +/// Delta C++ class instances. +pub const DT_MIPS_DELTA_INSTANCE: u32 = 0x7000_0019; +/// Number of entries in DT_MIPS_DELTA_INSTANCE. +pub const DT_MIPS_DELTA_INSTANCE_NO: u32 = 0x7000_001a; +/// Delta relocations. +pub const DT_MIPS_DELTA_RELOC: u32 = 0x7000_001b; +/// Number of entries in DT_MIPS_DELTA_RELOC. +pub const DT_MIPS_DELTA_RELOC_NO: u32 = 0x7000_001c; +/// Delta symbols that Delta relocations refer to. +pub const DT_MIPS_DELTA_SYM: u32 = 0x7000_001d; +/// Number of entries in DT_MIPS_DELTA_SYM. +pub const DT_MIPS_DELTA_SYM_NO: u32 = 0x7000_001e; +/// Delta symbols that hold the class declaration. +pub const DT_MIPS_DELTA_CLASSSYM: u32 = 0x7000_0020; +/// Number of entries in DT_MIPS_DELTA_CLASSSYM. +pub const DT_MIPS_DELTA_CLASSSYM_NO: u32 = 0x7000_0021; +/// Flags indicating for C++ flavor. +pub const DT_MIPS_CXX_FLAGS: u32 = 0x7000_0022; +#[allow(missing_docs)] +pub const DT_MIPS_PIXIE_INIT: u32 = 0x7000_0023; +#[allow(missing_docs)] +pub const DT_MIPS_SYMBOL_LIB: u32 = 0x7000_0024; +#[allow(missing_docs)] +pub const DT_MIPS_LOCALPAGE_GOTIDX: u32 = 0x7000_0025; +#[allow(missing_docs)] +pub const DT_MIPS_LOCAL_GOTIDX: u32 = 0x7000_0026; +#[allow(missing_docs)] +pub const DT_MIPS_HIDDEN_GOTIDX: u32 = 0x7000_0027; +#[allow(missing_docs)] +pub const DT_MIPS_PROTECTED_GOTIDX: u32 = 0x7000_0028; +/// Address of .options. +pub const DT_MIPS_OPTIONS: u32 = 0x7000_0029; +/// Address of .interface. +pub const DT_MIPS_INTERFACE: u32 = 0x7000_002a; +#[allow(missing_docs)] +pub const DT_MIPS_DYNSTR_ALIGN: u32 = 0x7000_002b; +/// Size of the .interface section. +pub const DT_MIPS_INTERFACE_SIZE: u32 = 0x7000_002c; +/// Address of rld_text_rsolve function stored in GOT. +pub const DT_MIPS_RLD_TEXT_RESOLVE_ADDR: u32 = 0x7000_002d; +/// Default suffix of dso to be added by rld on dlopen() calls. +pub const DT_MIPS_PERF_SUFFIX: u32 = 0x7000_002e; +/// (O32)Size of compact rel section. +pub const DT_MIPS_COMPACT_SIZE: u32 = 0x7000_002f; +/// GP value for aux GOTs. +pub const DT_MIPS_GP_VALUE: u32 = 0x7000_0030; +/// Address of aux .dynamic. +pub const DT_MIPS_AUX_DYNAMIC: u32 = 0x7000_0031; +/// The address of .got.plt in an executable using the new non-PIC ABI. +pub const DT_MIPS_PLTGOT: u32 = 0x7000_0032; +/// The base of the PLT in an executable using the new non-PIC ABI if that PLT is writable. For a non-writable PLT, this is omitted or has a zero value. +pub const DT_MIPS_RWPLT: u32 = 0x7000_0034; +/// An alternative description of the classic MIPS RLD_MAP that is usable in a PIE as it stores a relative offset from the address of the tag rather than an absolute address. +pub const DT_MIPS_RLD_MAP_REL: u32 = 0x7000_0035; + +// Values for `DT_MIPS_FLAGS` `Dyn32` entry. + +/// No flags +pub const RHF_NONE: u32 = 0; +/// Use quickstart +pub const RHF_QUICKSTART: u32 = 1 << 0; +/// Hash size not power of 2 +pub const RHF_NOTPOT: u32 = 1 << 1; +/// Ignore LD_LIBRARY_PATH +pub const RHF_NO_LIBRARY_REPLACEMENT: u32 = 1 << 2; +#[allow(missing_docs)] +pub const RHF_NO_MOVE: u32 = 1 << 3; +#[allow(missing_docs)] +pub const RHF_SGI_ONLY: u32 = 1 << 4; +#[allow(missing_docs)] +pub const RHF_GUARANTEE_INIT: u32 = 1 << 5; +#[allow(missing_docs)] +pub const RHF_DELTA_C_PLUS_PLUS: u32 = 1 << 6; +#[allow(missing_docs)] +pub const RHF_GUARANTEE_START_INIT: u32 = 1 << 7; +#[allow(missing_docs)] +pub const RHF_PIXIE: u32 = 1 << 8; +#[allow(missing_docs)] +pub const RHF_DEFAULT_DELAY_LOAD: u32 = 1 << 9; +#[allow(missing_docs)] +pub const RHF_REQUICKSTART: u32 = 1 << 10; +#[allow(missing_docs)] +pub const RHF_REQUICKSTARTED: u32 = 1 << 11; +#[allow(missing_docs)] +pub const RHF_CORD: u32 = 1 << 12; +#[allow(missing_docs)] +pub const RHF_NO_UNRES_UNDEF: u32 = 1 << 13; +#[allow(missing_docs)] +pub const RHF_RLD_ORDER_SAFE: u32 = 1 << 14; + +// Entries found in sections of type `SHT_MIPS_LIBLIST`. + +// TODO: Elf32_Lib, Elf64_Lib + +// Values for `Lib*::l_flags`. + +#[allow(missing_docs)] +pub const LL_NONE: u32 = 0; +/// Require exact match +pub const LL_EXACT_MATCH: u32 = 1 << 0; +/// Ignore interface version +pub const LL_IGNORE_INT_VER: u32 = 1 << 1; +#[allow(missing_docs)] +pub const LL_REQUIRE_MINOR: u32 = 1 << 2; +#[allow(missing_docs)] +pub const LL_EXPORTS: u32 = 1 << 3; +#[allow(missing_docs)] +pub const LL_DELAY_LOAD: u32 = 1 << 4; +#[allow(missing_docs)] +pub const LL_DELTA: u32 = 1 << 5; + +// TODO: MIPS ABI flags + +// PA-RISC specific definitions. + +// PA-RISC values for `FileHeader32::e_flags`. + +/// Trap nil pointer dereference. +pub const EF_PARISC_TRAPNIL: u32 = 0x0001_0000; +/// Program uses arch. extensions. +pub const EF_PARISC_EXT: u32 = 0x0002_0000; +/// Program expects little endian. +pub const EF_PARISC_LSB: u32 = 0x0004_0000; +/// Program expects wide mode. +pub const EF_PARISC_WIDE: u32 = 0x0008_0000; +/// No kernel assisted branch prediction. +pub const EF_PARISC_NO_KABP: u32 = 0x0010_0000; +/// Allow lazy swapping. +pub const EF_PARISC_LAZYSWAP: u32 = 0x0040_0000; +/// Architecture version. +pub const EF_PARISC_ARCH: u32 = 0x0000_ffff; + +// Values for `EF_PARISC_ARCH'. + +/// PA-RISC 1.0 big-endian. +pub const EFA_PARISC_1_0: u32 = 0x020b; +/// PA-RISC 1.1 big-endian. +pub const EFA_PARISC_1_1: u32 = 0x0210; +/// PA-RISC 2.0 big-endian. +pub const EFA_PARISC_2_0: u32 = 0x0214; + +// PA-RISC values for `Sym*::st_shndx`. + +/// Section for tenatively declared symbols in ANSI C. +pub const SHN_PARISC_ANSI_COMMON: u16 = 0xff00; +/// Common blocks in huge model. +pub const SHN_PARISC_HUGE_COMMON: u16 = 0xff01; + +// PA-RISC values for `SectionHeader32::sh_type`. + +/// Contains product specific ext. +pub const SHT_PARISC_EXT: u32 = 0x7000_0000; +/// Unwind information. +pub const SHT_PARISC_UNWIND: u32 = 0x7000_0001; +/// Debug info for optimized code. +pub const SHT_PARISC_DOC: u32 = 0x7000_0002; + +// PA-RISC values for `SectionHeader32::sh_flags`. + +/// Section with short addressing. +pub const SHF_PARISC_SHORT: u32 = 0x2000_0000; +/// Section far from gp. +pub const SHF_PARISC_HUGE: u32 = 0x4000_0000; +/// Static branch prediction code. +pub const SHF_PARISC_SBP: u32 = 0x8000_0000; + +// PA-RISC values for `st_type` component of `Sym32::st_info`. + +/// Millicode function entry point. +pub const STT_PARISC_MILLICODE: u8 = 13; + +#[allow(missing_docs)] +pub const STT_HP_OPAQUE: u8 = STT_LOOS + 0x1; +#[allow(missing_docs)] +pub const STT_HP_STUB: u8 = STT_LOOS + 0x2; + +// PA-RISC values for `Rel*::r_type`. + +/// No reloc. +pub const R_PARISC_NONE: u32 = 0; +/// Direct 32-bit reference. +pub const R_PARISC_DIR32: u32 = 1; +/// Left 21 bits of eff. address. +pub const R_PARISC_DIR21L: u32 = 2; +/// Right 17 bits of eff. address. +pub const R_PARISC_DIR17R: u32 = 3; +/// 17 bits of eff. address. +pub const R_PARISC_DIR17F: u32 = 4; +/// Right 14 bits of eff. address. +pub const R_PARISC_DIR14R: u32 = 6; +/// 32-bit rel. address. +pub const R_PARISC_PCREL32: u32 = 9; +/// Left 21 bits of rel. address. +pub const R_PARISC_PCREL21L: u32 = 10; +/// Right 17 bits of rel. address. +pub const R_PARISC_PCREL17R: u32 = 11; +/// 17 bits of rel. address. +pub const R_PARISC_PCREL17F: u32 = 12; +/// Right 14 bits of rel. address. +pub const R_PARISC_PCREL14R: u32 = 14; +/// Left 21 bits of rel. address. +pub const R_PARISC_DPREL21L: u32 = 18; +/// Right 14 bits of rel. address. +pub const R_PARISC_DPREL14R: u32 = 22; +/// GP-relative, left 21 bits. +pub const R_PARISC_GPREL21L: u32 = 26; +/// GP-relative, right 14 bits. +pub const R_PARISC_GPREL14R: u32 = 30; +/// LT-relative, left 21 bits. +pub const R_PARISC_LTOFF21L: u32 = 34; +/// LT-relative, right 14 bits. +pub const R_PARISC_LTOFF14R: u32 = 38; +/// 32 bits section rel. address. +pub const R_PARISC_SECREL32: u32 = 41; +/// No relocation, set segment base. +pub const R_PARISC_SEGBASE: u32 = 48; +/// 32 bits segment rel. address. +pub const R_PARISC_SEGREL32: u32 = 49; +/// PLT rel. address, left 21 bits. +pub const R_PARISC_PLTOFF21L: u32 = 50; +/// PLT rel. address, right 14 bits. +pub const R_PARISC_PLTOFF14R: u32 = 54; +/// 32 bits LT-rel. function pointer. +pub const R_PARISC_LTOFF_FPTR32: u32 = 57; +/// LT-rel. fct ptr, left 21 bits. +pub const R_PARISC_LTOFF_FPTR21L: u32 = 58; +/// LT-rel. fct ptr, right 14 bits. +pub const R_PARISC_LTOFF_FPTR14R: u32 = 62; +/// 64 bits function address. +pub const R_PARISC_FPTR64: u32 = 64; +/// 32 bits function address. +pub const R_PARISC_PLABEL32: u32 = 65; +/// Left 21 bits of fdesc address. +pub const R_PARISC_PLABEL21L: u32 = 66; +/// Right 14 bits of fdesc address. +pub const R_PARISC_PLABEL14R: u32 = 70; +/// 64 bits PC-rel. address. +pub const R_PARISC_PCREL64: u32 = 72; +/// 22 bits PC-rel. address. +pub const R_PARISC_PCREL22F: u32 = 74; +/// PC-rel. address, right 14 bits. +pub const R_PARISC_PCREL14WR: u32 = 75; +/// PC rel. address, right 14 bits. +pub const R_PARISC_PCREL14DR: u32 = 76; +/// 16 bits PC-rel. address. +pub const R_PARISC_PCREL16F: u32 = 77; +/// 16 bits PC-rel. address. +pub const R_PARISC_PCREL16WF: u32 = 78; +/// 16 bits PC-rel. address. +pub const R_PARISC_PCREL16DF: u32 = 79; +/// 64 bits of eff. address. +pub const R_PARISC_DIR64: u32 = 80; +/// 14 bits of eff. address. +pub const R_PARISC_DIR14WR: u32 = 83; +/// 14 bits of eff. address. +pub const R_PARISC_DIR14DR: u32 = 84; +/// 16 bits of eff. address. +pub const R_PARISC_DIR16F: u32 = 85; +/// 16 bits of eff. address. +pub const R_PARISC_DIR16WF: u32 = 86; +/// 16 bits of eff. address. +pub const R_PARISC_DIR16DF: u32 = 87; +/// 64 bits of GP-rel. address. +pub const R_PARISC_GPREL64: u32 = 88; +/// GP-rel. address, right 14 bits. +pub const R_PARISC_GPREL14WR: u32 = 91; +/// GP-rel. address, right 14 bits. +pub const R_PARISC_GPREL14DR: u32 = 92; +/// 16 bits GP-rel. address. +pub const R_PARISC_GPREL16F: u32 = 93; +/// 16 bits GP-rel. address. +pub const R_PARISC_GPREL16WF: u32 = 94; +/// 16 bits GP-rel. address. +pub const R_PARISC_GPREL16DF: u32 = 95; +/// 64 bits LT-rel. address. +pub const R_PARISC_LTOFF64: u32 = 96; +/// LT-rel. address, right 14 bits. +pub const R_PARISC_LTOFF14WR: u32 = 99; +/// LT-rel. address, right 14 bits. +pub const R_PARISC_LTOFF14DR: u32 = 100; +/// 16 bits LT-rel. address. +pub const R_PARISC_LTOFF16F: u32 = 101; +/// 16 bits LT-rel. address. +pub const R_PARISC_LTOFF16WF: u32 = 102; +/// 16 bits LT-rel. address. +pub const R_PARISC_LTOFF16DF: u32 = 103; +/// 64 bits section rel. address. +pub const R_PARISC_SECREL64: u32 = 104; +/// 64 bits segment rel. address. +pub const R_PARISC_SEGREL64: u32 = 112; +/// PLT-rel. address, right 14 bits. +pub const R_PARISC_PLTOFF14WR: u32 = 115; +/// PLT-rel. address, right 14 bits. +pub const R_PARISC_PLTOFF14DR: u32 = 116; +/// 16 bits LT-rel. address. +pub const R_PARISC_PLTOFF16F: u32 = 117; +/// 16 bits PLT-rel. address. +pub const R_PARISC_PLTOFF16WF: u32 = 118; +/// 16 bits PLT-rel. address. +pub const R_PARISC_PLTOFF16DF: u32 = 119; +/// 64 bits LT-rel. function ptr. +pub const R_PARISC_LTOFF_FPTR64: u32 = 120; +/// LT-rel. fct. ptr., right 14 bits. +pub const R_PARISC_LTOFF_FPTR14WR: u32 = 123; +/// LT-rel. fct. ptr., right 14 bits. +pub const R_PARISC_LTOFF_FPTR14DR: u32 = 124; +/// 16 bits LT-rel. function ptr. +pub const R_PARISC_LTOFF_FPTR16F: u32 = 125; +/// 16 bits LT-rel. function ptr. +pub const R_PARISC_LTOFF_FPTR16WF: u32 = 126; +/// 16 bits LT-rel. function ptr. +pub const R_PARISC_LTOFF_FPTR16DF: u32 = 127; +#[allow(missing_docs)] +pub const R_PARISC_LORESERVE: u32 = 128; +/// Copy relocation. +pub const R_PARISC_COPY: u32 = 128; +/// Dynamic reloc, imported PLT +pub const R_PARISC_IPLT: u32 = 129; +/// Dynamic reloc, exported PLT +pub const R_PARISC_EPLT: u32 = 130; +/// 32 bits TP-rel. address. +pub const R_PARISC_TPREL32: u32 = 153; +/// TP-rel. address, left 21 bits. +pub const R_PARISC_TPREL21L: u32 = 154; +/// TP-rel. address, right 14 bits. +pub const R_PARISC_TPREL14R: u32 = 158; +/// LT-TP-rel. address, left 21 bits. +pub const R_PARISC_LTOFF_TP21L: u32 = 162; +/// LT-TP-rel. address, right 14 bits. +pub const R_PARISC_LTOFF_TP14R: u32 = 166; +/// 14 bits LT-TP-rel. address. +pub const R_PARISC_LTOFF_TP14F: u32 = 167; +/// 64 bits TP-rel. address. +pub const R_PARISC_TPREL64: u32 = 216; +/// TP-rel. address, right 14 bits. +pub const R_PARISC_TPREL14WR: u32 = 219; +/// TP-rel. address, right 14 bits. +pub const R_PARISC_TPREL14DR: u32 = 220; +/// 16 bits TP-rel. address. +pub const R_PARISC_TPREL16F: u32 = 221; +/// 16 bits TP-rel. address. +pub const R_PARISC_TPREL16WF: u32 = 222; +/// 16 bits TP-rel. address. +pub const R_PARISC_TPREL16DF: u32 = 223; +/// 64 bits LT-TP-rel. address. +pub const R_PARISC_LTOFF_TP64: u32 = 224; +/// LT-TP-rel. address, right 14 bits. +pub const R_PARISC_LTOFF_TP14WR: u32 = 227; +/// LT-TP-rel. address, right 14 bits. +pub const R_PARISC_LTOFF_TP14DR: u32 = 228; +/// 16 bits LT-TP-rel. address. +pub const R_PARISC_LTOFF_TP16F: u32 = 229; +/// 16 bits LT-TP-rel. address. +pub const R_PARISC_LTOFF_TP16WF: u32 = 230; +/// 16 bits LT-TP-rel. address. +pub const R_PARISC_LTOFF_TP16DF: u32 = 231; +#[allow(missing_docs)] +pub const R_PARISC_GNU_VTENTRY: u32 = 232; +#[allow(missing_docs)] +pub const R_PARISC_GNU_VTINHERIT: u32 = 233; +/// GD 21-bit left. +pub const R_PARISC_TLS_GD21L: u32 = 234; +/// GD 14-bit right. +pub const R_PARISC_TLS_GD14R: u32 = 235; +/// GD call to __t_g_a. +pub const R_PARISC_TLS_GDCALL: u32 = 236; +/// LD module 21-bit left. +pub const R_PARISC_TLS_LDM21L: u32 = 237; +/// LD module 14-bit right. +pub const R_PARISC_TLS_LDM14R: u32 = 238; +/// LD module call to __t_g_a. +pub const R_PARISC_TLS_LDMCALL: u32 = 239; +/// LD offset 21-bit left. +pub const R_PARISC_TLS_LDO21L: u32 = 240; +/// LD offset 14-bit right. +pub const R_PARISC_TLS_LDO14R: u32 = 241; +/// DTP module 32-bit. +pub const R_PARISC_TLS_DTPMOD32: u32 = 242; +/// DTP module 64-bit. +pub const R_PARISC_TLS_DTPMOD64: u32 = 243; +/// DTP offset 32-bit. +pub const R_PARISC_TLS_DTPOFF32: u32 = 244; +/// DTP offset 32-bit. +pub const R_PARISC_TLS_DTPOFF64: u32 = 245; +#[allow(missing_docs)] +pub const R_PARISC_TLS_LE21L: u32 = R_PARISC_TPREL21L; +#[allow(missing_docs)] +pub const R_PARISC_TLS_LE14R: u32 = R_PARISC_TPREL14R; +#[allow(missing_docs)] +pub const R_PARISC_TLS_IE21L: u32 = R_PARISC_LTOFF_TP21L; +#[allow(missing_docs)] +pub const R_PARISC_TLS_IE14R: u32 = R_PARISC_LTOFF_TP14R; +#[allow(missing_docs)] +pub const R_PARISC_TLS_TPREL32: u32 = R_PARISC_TPREL32; +#[allow(missing_docs)] +pub const R_PARISC_TLS_TPREL64: u32 = R_PARISC_TPREL64; +#[allow(missing_docs)] +pub const R_PARISC_HIRESERVE: u32 = 255; + +// PA-RISC values for `ProgramHeader*::p_type`. + +#[allow(missing_docs)] +pub const PT_HP_TLS: u32 = PT_LOOS + 0x0; +#[allow(missing_docs)] +pub const PT_HP_CORE_NONE: u32 = PT_LOOS + 0x1; +#[allow(missing_docs)] +pub const PT_HP_CORE_VERSION: u32 = PT_LOOS + 0x2; +#[allow(missing_docs)] +pub const PT_HP_CORE_KERNEL: u32 = PT_LOOS + 0x3; +#[allow(missing_docs)] +pub const PT_HP_CORE_COMM: u32 = PT_LOOS + 0x4; +#[allow(missing_docs)] +pub const PT_HP_CORE_PROC: u32 = PT_LOOS + 0x5; +#[allow(missing_docs)] +pub const PT_HP_CORE_LOADABLE: u32 = PT_LOOS + 0x6; +#[allow(missing_docs)] +pub const PT_HP_CORE_STACK: u32 = PT_LOOS + 0x7; +#[allow(missing_docs)] +pub const PT_HP_CORE_SHM: u32 = PT_LOOS + 0x8; +#[allow(missing_docs)] +pub const PT_HP_CORE_MMF: u32 = PT_LOOS + 0x9; +#[allow(missing_docs)] +pub const PT_HP_PARALLEL: u32 = PT_LOOS + 0x10; +#[allow(missing_docs)] +pub const PT_HP_FASTBIND: u32 = PT_LOOS + 0x11; +#[allow(missing_docs)] +pub const PT_HP_OPT_ANNOT: u32 = PT_LOOS + 0x12; +#[allow(missing_docs)] +pub const PT_HP_HSL_ANNOT: u32 = PT_LOOS + 0x13; +#[allow(missing_docs)] +pub const PT_HP_STACK: u32 = PT_LOOS + 0x14; + +#[allow(missing_docs)] +pub const PT_PARISC_ARCHEXT: u32 = 0x7000_0000; +#[allow(missing_docs)] +pub const PT_PARISC_UNWIND: u32 = 0x7000_0001; + +// PA-RISC values for `ProgramHeader*::p_flags`. + +#[allow(missing_docs)] +pub const PF_PARISC_SBP: u32 = 0x0800_0000; + +#[allow(missing_docs)] +pub const PF_HP_PAGE_SIZE: u32 = 0x0010_0000; +#[allow(missing_docs)] +pub const PF_HP_FAR_SHARED: u32 = 0x0020_0000; +#[allow(missing_docs)] +pub const PF_HP_NEAR_SHARED: u32 = 0x0040_0000; +#[allow(missing_docs)] +pub const PF_HP_CODE: u32 = 0x0100_0000; +#[allow(missing_docs)] +pub const PF_HP_MODIFY: u32 = 0x0200_0000; +#[allow(missing_docs)] +pub const PF_HP_LAZYSWAP: u32 = 0x0400_0000; +#[allow(missing_docs)] +pub const PF_HP_SBP: u32 = 0x0800_0000; + +// Alpha specific definitions. + +// Alpha values for `FileHeader64::e_flags`. + +/// All addresses must be < 2GB. +pub const EF_ALPHA_32BIT: u32 = 1; +/// Relocations for relaxing exist. +pub const EF_ALPHA_CANRELAX: u32 = 2; + +// Alpha values for `SectionHeader64::sh_type`. + +// These two are primerily concerned with ECOFF debugging info. +#[allow(missing_docs)] +pub const SHT_ALPHA_DEBUG: u32 = 0x7000_0001; +#[allow(missing_docs)] +pub const SHT_ALPHA_REGINFO: u32 = 0x7000_0002; + +// Alpha values for `SectionHeader64::sh_flags`. + +#[allow(missing_docs)] +pub const SHF_ALPHA_GPREL: u32 = 0x1000_0000; + +// Alpha values for `Sym64::st_other`. +/// No PV required. +pub const STO_ALPHA_NOPV: u8 = 0x80; +/// PV only used for initial ldgp. +pub const STO_ALPHA_STD_GPLOAD: u8 = 0x88; + +// Alpha values for `Rel64::r_type`. + +/// No reloc +pub const R_ALPHA_NONE: u32 = 0; +/// Direct 32 bit +pub const R_ALPHA_REFLONG: u32 = 1; +/// Direct 64 bit +pub const R_ALPHA_REFQUAD: u32 = 2; +/// GP relative 32 bit +pub const R_ALPHA_GPREL32: u32 = 3; +/// GP relative 16 bit w/optimization +pub const R_ALPHA_LITERAL: u32 = 4; +/// Optimization hint for LITERAL +pub const R_ALPHA_LITUSE: u32 = 5; +/// Add displacement to GP +pub const R_ALPHA_GPDISP: u32 = 6; +/// PC+4 relative 23 bit shifted +pub const R_ALPHA_BRADDR: u32 = 7; +/// PC+4 relative 16 bit shifted +pub const R_ALPHA_HINT: u32 = 8; +/// PC relative 16 bit +pub const R_ALPHA_SREL16: u32 = 9; +/// PC relative 32 bit +pub const R_ALPHA_SREL32: u32 = 10; +/// PC relative 64 bit +pub const R_ALPHA_SREL64: u32 = 11; +/// GP relative 32 bit, high 16 bits +pub const R_ALPHA_GPRELHIGH: u32 = 17; +/// GP relative 32 bit, low 16 bits +pub const R_ALPHA_GPRELLOW: u32 = 18; +/// GP relative 16 bit +pub const R_ALPHA_GPREL16: u32 = 19; +/// Copy symbol at runtime +pub const R_ALPHA_COPY: u32 = 24; +/// Create GOT entry +pub const R_ALPHA_GLOB_DAT: u32 = 25; +/// Create PLT entry +pub const R_ALPHA_JMP_SLOT: u32 = 26; +/// Adjust by program base +pub const R_ALPHA_RELATIVE: u32 = 27; +#[allow(missing_docs)] +pub const R_ALPHA_TLS_GD_HI: u32 = 28; +#[allow(missing_docs)] +pub const R_ALPHA_TLSGD: u32 = 29; +#[allow(missing_docs)] +pub const R_ALPHA_TLS_LDM: u32 = 30; +#[allow(missing_docs)] +pub const R_ALPHA_DTPMOD64: u32 = 31; +#[allow(missing_docs)] +pub const R_ALPHA_GOTDTPREL: u32 = 32; +#[allow(missing_docs)] +pub const R_ALPHA_DTPREL64: u32 = 33; +#[allow(missing_docs)] +pub const R_ALPHA_DTPRELHI: u32 = 34; +#[allow(missing_docs)] +pub const R_ALPHA_DTPRELLO: u32 = 35; +#[allow(missing_docs)] +pub const R_ALPHA_DTPREL16: u32 = 36; +#[allow(missing_docs)] +pub const R_ALPHA_GOTTPREL: u32 = 37; +#[allow(missing_docs)] +pub const R_ALPHA_TPREL64: u32 = 38; +#[allow(missing_docs)] +pub const R_ALPHA_TPRELHI: u32 = 39; +#[allow(missing_docs)] +pub const R_ALPHA_TPRELLO: u32 = 40; +#[allow(missing_docs)] +pub const R_ALPHA_TPREL16: u32 = 41; + +// Magic values of the `R_ALPHA_LITUSE` relocation addend. +#[allow(missing_docs)] +pub const LITUSE_ALPHA_ADDR: u32 = 0; +#[allow(missing_docs)] +pub const LITUSE_ALPHA_BASE: u32 = 1; +#[allow(missing_docs)] +pub const LITUSE_ALPHA_BYTOFF: u32 = 2; +#[allow(missing_docs)] +pub const LITUSE_ALPHA_JSR: u32 = 3; +#[allow(missing_docs)] +pub const LITUSE_ALPHA_TLS_GD: u32 = 4; +#[allow(missing_docs)] +pub const LITUSE_ALPHA_TLS_LDM: u32 = 5; + +// Alpha values for `Dyn64::d_tag`. +#[allow(missing_docs)] +pub const DT_ALPHA_PLTRO: u32 = DT_LOPROC + 0; + +// PowerPC specific declarations. + +// PowerPC values for `FileHeader*::e_flags`. +/// PowerPC embedded flag +pub const EF_PPC_EMB: u32 = 0x8000_0000; + +// Cygnus local bits below . +/// PowerPC -mrelocatable flag +pub const EF_PPC_RELOCATABLE: u32 = 0x0001_0000; +/// PowerPC -mrelocatable-lib flag +pub const EF_PPC_RELOCATABLE_LIB: u32 = 0x0000_8000; + +// PowerPC values for `Rel*::r_type` defined by the ABIs. +#[allow(missing_docs)] +pub const R_PPC_NONE: u32 = 0; +/// 32bit absolute address +pub const R_PPC_ADDR32: u32 = 1; +/// 26bit address, 2 bits ignored. +pub const R_PPC_ADDR24: u32 = 2; +/// 16bit absolute address +pub const R_PPC_ADDR16: u32 = 3; +/// lower 16bit of absolute address +pub const R_PPC_ADDR16_LO: u32 = 4; +/// high 16bit of absolute address +pub const R_PPC_ADDR16_HI: u32 = 5; +/// adjusted high 16bit +pub const R_PPC_ADDR16_HA: u32 = 6; +/// 16bit address, 2 bits ignored +pub const R_PPC_ADDR14: u32 = 7; +#[allow(missing_docs)] +pub const R_PPC_ADDR14_BRTAKEN: u32 = 8; +#[allow(missing_docs)] +pub const R_PPC_ADDR14_BRNTAKEN: u32 = 9; +/// PC relative 26 bit +pub const R_PPC_REL24: u32 = 10; +/// PC relative 16 bit +pub const R_PPC_REL14: u32 = 11; +#[allow(missing_docs)] +pub const R_PPC_REL14_BRTAKEN: u32 = 12; +#[allow(missing_docs)] +pub const R_PPC_REL14_BRNTAKEN: u32 = 13; +#[allow(missing_docs)] +pub const R_PPC_GOT16: u32 = 14; +#[allow(missing_docs)] +pub const R_PPC_GOT16_LO: u32 = 15; +#[allow(missing_docs)] +pub const R_PPC_GOT16_HI: u32 = 16; +#[allow(missing_docs)] +pub const R_PPC_GOT16_HA: u32 = 17; +#[allow(missing_docs)] +pub const R_PPC_PLTREL24: u32 = 18; +#[allow(missing_docs)] +pub const R_PPC_COPY: u32 = 19; +#[allow(missing_docs)] +pub const R_PPC_GLOB_DAT: u32 = 20; +#[allow(missing_docs)] +pub const R_PPC_JMP_SLOT: u32 = 21; +#[allow(missing_docs)] +pub const R_PPC_RELATIVE: u32 = 22; +#[allow(missing_docs)] +pub const R_PPC_LOCAL24PC: u32 = 23; +#[allow(missing_docs)] +pub const R_PPC_UADDR32: u32 = 24; +#[allow(missing_docs)] +pub const R_PPC_UADDR16: u32 = 25; +#[allow(missing_docs)] +pub const R_PPC_REL32: u32 = 26; +#[allow(missing_docs)] +pub const R_PPC_PLT32: u32 = 27; +#[allow(missing_docs)] +pub const R_PPC_PLTREL32: u32 = 28; +#[allow(missing_docs)] +pub const R_PPC_PLT16_LO: u32 = 29; +#[allow(missing_docs)] +pub const R_PPC_PLT16_HI: u32 = 30; +#[allow(missing_docs)] +pub const R_PPC_PLT16_HA: u32 = 31; +#[allow(missing_docs)] +pub const R_PPC_SDAREL16: u32 = 32; +#[allow(missing_docs)] +pub const R_PPC_SECTOFF: u32 = 33; +#[allow(missing_docs)] +pub const R_PPC_SECTOFF_LO: u32 = 34; +#[allow(missing_docs)] +pub const R_PPC_SECTOFF_HI: u32 = 35; +#[allow(missing_docs)] +pub const R_PPC_SECTOFF_HA: u32 = 36; + +// PowerPC values for `Rel*::r_type` defined for the TLS access ABI. +/// none (sym+add)@tls +pub const R_PPC_TLS: u32 = 67; +/// word32 (sym+add)@dtpmod +pub const R_PPC_DTPMOD32: u32 = 68; +/// half16* (sym+add)@tprel +pub const R_PPC_TPREL16: u32 = 69; +/// half16 (sym+add)@tprel@l +pub const R_PPC_TPREL16_LO: u32 = 70; +/// half16 (sym+add)@tprel@h +pub const R_PPC_TPREL16_HI: u32 = 71; +/// half16 (sym+add)@tprel@ha +pub const R_PPC_TPREL16_HA: u32 = 72; +/// word32 (sym+add)@tprel +pub const R_PPC_TPREL32: u32 = 73; +/// half16*(sym+add)@dtprel +pub const R_PPC_DTPREL16: u32 = 74; +/// half16 (sym+add)@dtprel@l +pub const R_PPC_DTPREL16_LO: u32 = 75; +/// half16 (sym+add)@dtprel@h +pub const R_PPC_DTPREL16_HI: u32 = 76; +/// half16 (sym+add)@dtprel@ha +pub const R_PPC_DTPREL16_HA: u32 = 77; +/// word32 (sym+add)@dtprel +pub const R_PPC_DTPREL32: u32 = 78; +/// half16* (sym+add)@got@tlsgd +pub const R_PPC_GOT_TLSGD16: u32 = 79; +/// half16 (sym+add)@got@tlsgd@l +pub const R_PPC_GOT_TLSGD16_LO: u32 = 80; +/// half16 (sym+add)@got@tlsgd@h +pub const R_PPC_GOT_TLSGD16_HI: u32 = 81; +/// half16 (sym+add)@got@tlsgd@ha +pub const R_PPC_GOT_TLSGD16_HA: u32 = 82; +/// half16* (sym+add)@got@tlsld +pub const R_PPC_GOT_TLSLD16: u32 = 83; +/// half16 (sym+add)@got@tlsld@l +pub const R_PPC_GOT_TLSLD16_LO: u32 = 84; +/// half16 (sym+add)@got@tlsld@h +pub const R_PPC_GOT_TLSLD16_HI: u32 = 85; +/// half16 (sym+add)@got@tlsld@ha +pub const R_PPC_GOT_TLSLD16_HA: u32 = 86; +/// half16* (sym+add)@got@tprel +pub const R_PPC_GOT_TPREL16: u32 = 87; +/// half16 (sym+add)@got@tprel@l +pub const R_PPC_GOT_TPREL16_LO: u32 = 88; +/// half16 (sym+add)@got@tprel@h +pub const R_PPC_GOT_TPREL16_HI: u32 = 89; +/// half16 (sym+add)@got@tprel@ha +pub const R_PPC_GOT_TPREL16_HA: u32 = 90; +/// half16* (sym+add)@got@dtprel +pub const R_PPC_GOT_DTPREL16: u32 = 91; +/// half16* (sym+add)@got@dtprel@l +pub const R_PPC_GOT_DTPREL16_LO: u32 = 92; +/// half16* (sym+add)@got@dtprel@h +pub const R_PPC_GOT_DTPREL16_HI: u32 = 93; +/// half16* (sym+add)@got@dtprel@ha +pub const R_PPC_GOT_DTPREL16_HA: u32 = 94; +/// none (sym+add)@tlsgd +pub const R_PPC_TLSGD: u32 = 95; +/// none (sym+add)@tlsld +pub const R_PPC_TLSLD: u32 = 96; + +// PowerPC values for `Rel*::r_type` from the Embedded ELF ABI. +#[allow(missing_docs)] +pub const R_PPC_EMB_NADDR32: u32 = 101; +#[allow(missing_docs)] +pub const R_PPC_EMB_NADDR16: u32 = 102; +#[allow(missing_docs)] +pub const R_PPC_EMB_NADDR16_LO: u32 = 103; +#[allow(missing_docs)] +pub const R_PPC_EMB_NADDR16_HI: u32 = 104; +#[allow(missing_docs)] +pub const R_PPC_EMB_NADDR16_HA: u32 = 105; +#[allow(missing_docs)] +pub const R_PPC_EMB_SDAI16: u32 = 106; +#[allow(missing_docs)] +pub const R_PPC_EMB_SDA2I16: u32 = 107; +#[allow(missing_docs)] +pub const R_PPC_EMB_SDA2REL: u32 = 108; +/// 16 bit offset in SDA +pub const R_PPC_EMB_SDA21: u32 = 109; +#[allow(missing_docs)] +pub const R_PPC_EMB_MRKREF: u32 = 110; +#[allow(missing_docs)] +pub const R_PPC_EMB_RELSEC16: u32 = 111; +#[allow(missing_docs)] +pub const R_PPC_EMB_RELST_LO: u32 = 112; +#[allow(missing_docs)] +pub const R_PPC_EMB_RELST_HI: u32 = 113; +#[allow(missing_docs)] +pub const R_PPC_EMB_RELST_HA: u32 = 114; +#[allow(missing_docs)] +pub const R_PPC_EMB_BIT_FLD: u32 = 115; +/// 16 bit relative offset in SDA +pub const R_PPC_EMB_RELSDA: u32 = 116; + +// Diab tool values for `Rel*::r_type`. +/// like EMB_SDA21, but lower 16 bit +pub const R_PPC_DIAB_SDA21_LO: u32 = 180; +/// like EMB_SDA21, but high 16 bit +pub const R_PPC_DIAB_SDA21_HI: u32 = 181; +/// like EMB_SDA21, adjusted high 16 +pub const R_PPC_DIAB_SDA21_HA: u32 = 182; +/// like EMB_RELSDA, but lower 16 bit +pub const R_PPC_DIAB_RELSDA_LO: u32 = 183; +/// like EMB_RELSDA, but high 16 bit +pub const R_PPC_DIAB_RELSDA_HI: u32 = 184; +/// like EMB_RELSDA, adjusted high 16 +pub const R_PPC_DIAB_RELSDA_HA: u32 = 185; + +/// GNU extension to support local ifunc. +pub const R_PPC_IRELATIVE: u32 = 248; + +// GNU relocs used in PIC code sequences. +/// half16 (sym+add-.) +pub const R_PPC_REL16: u32 = 249; +/// half16 (sym+add-.)@l +pub const R_PPC_REL16_LO: u32 = 250; +/// half16 (sym+add-.)@h +pub const R_PPC_REL16_HI: u32 = 251; +/// half16 (sym+add-.)@ha +pub const R_PPC_REL16_HA: u32 = 252; + +/// This is a phony reloc to handle any old fashioned TOC16 references that may +/// still be in object files. +pub const R_PPC_TOC16: u32 = 255; + +// PowerPC specific values for `Dyn*::d_tag`. +#[allow(missing_docs)] +pub const DT_PPC_GOT: u32 = DT_LOPROC + 0; +#[allow(missing_docs)] +pub const DT_PPC_OPT: u32 = DT_LOPROC + 1; + +// PowerPC specific values for the `DT_PPC_OPT` entry. +#[allow(missing_docs)] +pub const PPC_OPT_TLS: u32 = 1; + +// PowerPC64 values for `Rel*::r_type` defined by the ABIs. +#[allow(missing_docs)] +pub const R_PPC64_NONE: u32 = R_PPC_NONE; +/// 32bit absolute address +pub const R_PPC64_ADDR32: u32 = R_PPC_ADDR32; +/// 26bit address, word aligned +pub const R_PPC64_ADDR24: u32 = R_PPC_ADDR24; +/// 16bit absolute address +pub const R_PPC64_ADDR16: u32 = R_PPC_ADDR16; +/// lower 16bits of address +pub const R_PPC64_ADDR16_LO: u32 = R_PPC_ADDR16_LO; +/// high 16bits of address. +pub const R_PPC64_ADDR16_HI: u32 = R_PPC_ADDR16_HI; +/// adjusted high 16bits. +pub const R_PPC64_ADDR16_HA: u32 = R_PPC_ADDR16_HA; +/// 16bit address, word aligned +pub const R_PPC64_ADDR14: u32 = R_PPC_ADDR14; +#[allow(missing_docs)] +pub const R_PPC64_ADDR14_BRTAKEN: u32 = R_PPC_ADDR14_BRTAKEN; +#[allow(missing_docs)] +pub const R_PPC64_ADDR14_BRNTAKEN: u32 = R_PPC_ADDR14_BRNTAKEN; +/// PC-rel. 26 bit, word aligned +pub const R_PPC64_REL24: u32 = R_PPC_REL24; +/// PC relative 16 bit +pub const R_PPC64_REL14: u32 = R_PPC_REL14; +#[allow(missing_docs)] +pub const R_PPC64_REL14_BRTAKEN: u32 = R_PPC_REL14_BRTAKEN; +#[allow(missing_docs)] +pub const R_PPC64_REL14_BRNTAKEN: u32 = R_PPC_REL14_BRNTAKEN; +#[allow(missing_docs)] +pub const R_PPC64_GOT16: u32 = R_PPC_GOT16; +#[allow(missing_docs)] +pub const R_PPC64_GOT16_LO: u32 = R_PPC_GOT16_LO; +#[allow(missing_docs)] +pub const R_PPC64_GOT16_HI: u32 = R_PPC_GOT16_HI; +#[allow(missing_docs)] +pub const R_PPC64_GOT16_HA: u32 = R_PPC_GOT16_HA; + +#[allow(missing_docs)] +pub const R_PPC64_COPY: u32 = R_PPC_COPY; +#[allow(missing_docs)] +pub const R_PPC64_GLOB_DAT: u32 = R_PPC_GLOB_DAT; +#[allow(missing_docs)] +pub const R_PPC64_JMP_SLOT: u32 = R_PPC_JMP_SLOT; +#[allow(missing_docs)] +pub const R_PPC64_RELATIVE: u32 = R_PPC_RELATIVE; + +#[allow(missing_docs)] +pub const R_PPC64_UADDR32: u32 = R_PPC_UADDR32; +#[allow(missing_docs)] +pub const R_PPC64_UADDR16: u32 = R_PPC_UADDR16; +#[allow(missing_docs)] +pub const R_PPC64_REL32: u32 = R_PPC_REL32; +#[allow(missing_docs)] +pub const R_PPC64_PLT32: u32 = R_PPC_PLT32; +#[allow(missing_docs)] +pub const R_PPC64_PLTREL32: u32 = R_PPC_PLTREL32; +#[allow(missing_docs)] +pub const R_PPC64_PLT16_LO: u32 = R_PPC_PLT16_LO; +#[allow(missing_docs)] +pub const R_PPC64_PLT16_HI: u32 = R_PPC_PLT16_HI; +#[allow(missing_docs)] +pub const R_PPC64_PLT16_HA: u32 = R_PPC_PLT16_HA; + +#[allow(missing_docs)] +pub const R_PPC64_SECTOFF: u32 = R_PPC_SECTOFF; +#[allow(missing_docs)] +pub const R_PPC64_SECTOFF_LO: u32 = R_PPC_SECTOFF_LO; +#[allow(missing_docs)] +pub const R_PPC64_SECTOFF_HI: u32 = R_PPC_SECTOFF_HI; +#[allow(missing_docs)] +pub const R_PPC64_SECTOFF_HA: u32 = R_PPC_SECTOFF_HA; +/// word30 (S + A - P) >> 2 +pub const R_PPC64_ADDR30: u32 = 37; +/// doubleword64 S + A +pub const R_PPC64_ADDR64: u32 = 38; +/// half16 #higher(S + A) +pub const R_PPC64_ADDR16_HIGHER: u32 = 39; +/// half16 #highera(S + A) +pub const R_PPC64_ADDR16_HIGHERA: u32 = 40; +/// half16 #highest(S + A) +pub const R_PPC64_ADDR16_HIGHEST: u32 = 41; +/// half16 #highesta(S + A) +pub const R_PPC64_ADDR16_HIGHESTA: u32 = 42; +/// doubleword64 S + A +pub const R_PPC64_UADDR64: u32 = 43; +/// doubleword64 S + A - P +pub const R_PPC64_REL64: u32 = 44; +/// doubleword64 L + A +pub const R_PPC64_PLT64: u32 = 45; +/// doubleword64 L + A - P +pub const R_PPC64_PLTREL64: u32 = 46; +/// half16* S + A - .TOC +pub const R_PPC64_TOC16: u32 = 47; +/// half16 #lo(S + A - .TOC.) +pub const R_PPC64_TOC16_LO: u32 = 48; +/// half16 #hi(S + A - .TOC.) +pub const R_PPC64_TOC16_HI: u32 = 49; +/// half16 #ha(S + A - .TOC.) +pub const R_PPC64_TOC16_HA: u32 = 50; +/// doubleword64 .TOC +pub const R_PPC64_TOC: u32 = 51; +/// half16* M + A +pub const R_PPC64_PLTGOT16: u32 = 52; +/// half16 #lo(M + A) +pub const R_PPC64_PLTGOT16_LO: u32 = 53; +/// half16 #hi(M + A) +pub const R_PPC64_PLTGOT16_HI: u32 = 54; +/// half16 #ha(M + A) +pub const R_PPC64_PLTGOT16_HA: u32 = 55; + +/// half16ds* (S + A) >> 2 +pub const R_PPC64_ADDR16_DS: u32 = 56; +/// half16ds #lo(S + A) >> 2 +pub const R_PPC64_ADDR16_LO_DS: u32 = 57; +/// half16ds* (G + A) >> 2 +pub const R_PPC64_GOT16_DS: u32 = 58; +/// half16ds #lo(G + A) >> 2 +pub const R_PPC64_GOT16_LO_DS: u32 = 59; +/// half16ds #lo(L + A) >> 2 +pub const R_PPC64_PLT16_LO_DS: u32 = 60; +/// half16ds* (R + A) >> 2 +pub const R_PPC64_SECTOFF_DS: u32 = 61; +/// half16ds #lo(R + A) >> 2 +pub const R_PPC64_SECTOFF_LO_DS: u32 = 62; +/// half16ds* (S + A - .TOC.) >> 2 +pub const R_PPC64_TOC16_DS: u32 = 63; +/// half16ds #lo(S + A - .TOC.) >> 2 +pub const R_PPC64_TOC16_LO_DS: u32 = 64; +/// half16ds* (M + A) >> 2 +pub const R_PPC64_PLTGOT16_DS: u32 = 65; +/// half16ds #lo(M + A) >> 2 +pub const R_PPC64_PLTGOT16_LO_DS: u32 = 66; + +// PowerPC64 values for `Rel*::r_type` defined for the TLS access ABI. +/// none (sym+add)@tls +pub const R_PPC64_TLS: u32 = 67; +/// doubleword64 (sym+add)@dtpmod +pub const R_PPC64_DTPMOD64: u32 = 68; +/// half16* (sym+add)@tprel +pub const R_PPC64_TPREL16: u32 = 69; +/// half16 (sym+add)@tprel@l +pub const R_PPC64_TPREL16_LO: u32 = 70; +/// half16 (sym+add)@tprel@h +pub const R_PPC64_TPREL16_HI: u32 = 71; +/// half16 (sym+add)@tprel@ha +pub const R_PPC64_TPREL16_HA: u32 = 72; +/// doubleword64 (sym+add)@tprel +pub const R_PPC64_TPREL64: u32 = 73; +/// half16* (sym+add)@dtprel +pub const R_PPC64_DTPREL16: u32 = 74; +/// half16 (sym+add)@dtprel@l +pub const R_PPC64_DTPREL16_LO: u32 = 75; +/// half16 (sym+add)@dtprel@h +pub const R_PPC64_DTPREL16_HI: u32 = 76; +/// half16 (sym+add)@dtprel@ha +pub const R_PPC64_DTPREL16_HA: u32 = 77; +/// doubleword64 (sym+add)@dtprel +pub const R_PPC64_DTPREL64: u32 = 78; +/// half16* (sym+add)@got@tlsgd +pub const R_PPC64_GOT_TLSGD16: u32 = 79; +/// half16 (sym+add)@got@tlsgd@l +pub const R_PPC64_GOT_TLSGD16_LO: u32 = 80; +/// half16 (sym+add)@got@tlsgd@h +pub const R_PPC64_GOT_TLSGD16_HI: u32 = 81; +/// half16 (sym+add)@got@tlsgd@ha +pub const R_PPC64_GOT_TLSGD16_HA: u32 = 82; +/// half16* (sym+add)@got@tlsld +pub const R_PPC64_GOT_TLSLD16: u32 = 83; +/// half16 (sym+add)@got@tlsld@l +pub const R_PPC64_GOT_TLSLD16_LO: u32 = 84; +/// half16 (sym+add)@got@tlsld@h +pub const R_PPC64_GOT_TLSLD16_HI: u32 = 85; +/// half16 (sym+add)@got@tlsld@ha +pub const R_PPC64_GOT_TLSLD16_HA: u32 = 86; +/// half16ds* (sym+add)@got@tprel +pub const R_PPC64_GOT_TPREL16_DS: u32 = 87; +/// half16ds (sym+add)@got@tprel@l +pub const R_PPC64_GOT_TPREL16_LO_DS: u32 = 88; +/// half16 (sym+add)@got@tprel@h +pub const R_PPC64_GOT_TPREL16_HI: u32 = 89; +/// half16 (sym+add)@got@tprel@ha +pub const R_PPC64_GOT_TPREL16_HA: u32 = 90; +/// half16ds* (sym+add)@got@dtprel +pub const R_PPC64_GOT_DTPREL16_DS: u32 = 91; +/// half16ds (sym+add)@got@dtprel@l +pub const R_PPC64_GOT_DTPREL16_LO_DS: u32 = 92; +/// half16 (sym+add)@got@dtprel@h +pub const R_PPC64_GOT_DTPREL16_HI: u32 = 93; +/// half16 (sym+add)@got@dtprel@ha +pub const R_PPC64_GOT_DTPREL16_HA: u32 = 94; +/// half16ds* (sym+add)@tprel +pub const R_PPC64_TPREL16_DS: u32 = 95; +/// half16ds (sym+add)@tprel@l +pub const R_PPC64_TPREL16_LO_DS: u32 = 96; +/// half16 (sym+add)@tprel@higher +pub const R_PPC64_TPREL16_HIGHER: u32 = 97; +/// half16 (sym+add)@tprel@highera +pub const R_PPC64_TPREL16_HIGHERA: u32 = 98; +/// half16 (sym+add)@tprel@highest +pub const R_PPC64_TPREL16_HIGHEST: u32 = 99; +/// half16 (sym+add)@tprel@highesta +pub const R_PPC64_TPREL16_HIGHESTA: u32 = 100; +/// half16ds* (sym+add)@dtprel +pub const R_PPC64_DTPREL16_DS: u32 = 101; +/// half16ds (sym+add)@dtprel@l +pub const R_PPC64_DTPREL16_LO_DS: u32 = 102; +/// half16 (sym+add)@dtprel@higher +pub const R_PPC64_DTPREL16_HIGHER: u32 = 103; +/// half16 (sym+add)@dtprel@highera +pub const R_PPC64_DTPREL16_HIGHERA: u32 = 104; +/// half16 (sym+add)@dtprel@highest +pub const R_PPC64_DTPREL16_HIGHEST: u32 = 105; +/// half16 (sym+add)@dtprel@highesta +pub const R_PPC64_DTPREL16_HIGHESTA: u32 = 106; +/// none (sym+add)@tlsgd +pub const R_PPC64_TLSGD: u32 = 107; +/// none (sym+add)@tlsld +pub const R_PPC64_TLSLD: u32 = 108; +/// none +pub const R_PPC64_TOCSAVE: u32 = 109; + +// Added when HA and HI relocs were changed to report overflows. +#[allow(missing_docs)] +pub const R_PPC64_ADDR16_HIGH: u32 = 110; +#[allow(missing_docs)] +pub const R_PPC64_ADDR16_HIGHA: u32 = 111; +#[allow(missing_docs)] +pub const R_PPC64_TPREL16_HIGH: u32 = 112; +#[allow(missing_docs)] +pub const R_PPC64_TPREL16_HIGHA: u32 = 113; +#[allow(missing_docs)] +pub const R_PPC64_DTPREL16_HIGH: u32 = 114; +#[allow(missing_docs)] +pub const R_PPC64_DTPREL16_HIGHA: u32 = 115; + +/// GNU extension to support local ifunc. +#[allow(missing_docs)] +pub const R_PPC64_JMP_IREL: u32 = 247; +/// GNU extension to support local ifunc. +#[allow(missing_docs)] +pub const R_PPC64_IRELATIVE: u32 = 248; +/// half16 (sym+add-.) +pub const R_PPC64_REL16: u32 = 249; +/// half16 (sym+add-.)@l +pub const R_PPC64_REL16_LO: u32 = 250; +/// half16 (sym+add-.)@h +pub const R_PPC64_REL16_HI: u32 = 251; +/// half16 (sym+add-.)@ha +pub const R_PPC64_REL16_HA: u32 = 252; + +// PowerPC64 values for `FileHeader64::e_flags. +/// PowerPC64 bits specifying ABI. +/// +/// 1 for original function descriptor using ABI, +/// 2 for revised ABI without function descriptors, +/// 0 for unspecified or not using any features affected by the differences. +pub const EF_PPC64_ABI: u32 = 3; + +// PowerPC64 values for `Dyn64::d_tag. +#[allow(missing_docs)] +pub const DT_PPC64_GLINK: u32 = DT_LOPROC + 0; +#[allow(missing_docs)] +pub const DT_PPC64_OPD: u32 = DT_LOPROC + 1; +#[allow(missing_docs)] +pub const DT_PPC64_OPDSZ: u32 = DT_LOPROC + 2; +#[allow(missing_docs)] +pub const DT_PPC64_OPT: u32 = DT_LOPROC + 3; + +// PowerPC64 bits for `DT_PPC64_OPT` entry. +#[allow(missing_docs)] +pub const PPC64_OPT_TLS: u32 = 1; +#[allow(missing_docs)] +pub const PPC64_OPT_MULTI_TOC: u32 = 2; +#[allow(missing_docs)] +pub const PPC64_OPT_LOCALENTRY: u32 = 4; + +// PowerPC64 values for `Sym64::st_other. +#[allow(missing_docs)] +pub const STO_PPC64_LOCAL_BIT: u8 = 5; +#[allow(missing_docs)] +pub const STO_PPC64_LOCAL_MASK: u8 = 7 << STO_PPC64_LOCAL_BIT; + +// ARM specific declarations. + +// ARM values for `FileHeader*::e_flags`. +#[allow(missing_docs)] +pub const EF_ARM_RELEXEC: u32 = 0x01; +#[allow(missing_docs)] +pub const EF_ARM_HASENTRY: u32 = 0x02; +#[allow(missing_docs)] +pub const EF_ARM_INTERWORK: u32 = 0x04; +#[allow(missing_docs)] +pub const EF_ARM_APCS_26: u32 = 0x08; +#[allow(missing_docs)] +pub const EF_ARM_APCS_FLOAT: u32 = 0x10; +#[allow(missing_docs)] +pub const EF_ARM_PIC: u32 = 0x20; +/// 8-bit structure alignment is in use +pub const EF_ARM_ALIGN8: u32 = 0x40; +#[allow(missing_docs)] +pub const EF_ARM_NEW_ABI: u32 = 0x80; +#[allow(missing_docs)] +pub const EF_ARM_OLD_ABI: u32 = 0x100; +#[allow(missing_docs)] +pub const EF_ARM_SOFT_FLOAT: u32 = 0x200; +#[allow(missing_docs)] +pub const EF_ARM_VFP_FLOAT: u32 = 0x400; +#[allow(missing_docs)] +pub const EF_ARM_MAVERICK_FLOAT: u32 = 0x800; + +/// NB conflicts with EF_ARM_SOFT_FLOAT +pub const EF_ARM_ABI_FLOAT_SOFT: u32 = 0x200; +/// NB conflicts with EF_ARM_VFP_FLOAT +pub const EF_ARM_ABI_FLOAT_HARD: u32 = 0x400; + +// Other constants defined in the ARM ELF spec. version B-01. +// NB. These conflict with values defined above. +#[allow(missing_docs)] +pub const EF_ARM_SYMSARESORTED: u32 = 0x04; +#[allow(missing_docs)] +pub const EF_ARM_DYNSYMSUSESEGIDX: u32 = 0x08; +#[allow(missing_docs)] +pub const EF_ARM_MAPSYMSFIRST: u32 = 0x10; + +// Constants defined in AAELF. +#[allow(missing_docs)] +pub const EF_ARM_BE8: u32 = 0x0080_0000; +#[allow(missing_docs)] +pub const EF_ARM_LE8: u32 = 0x0040_0000; + +#[allow(missing_docs)] +pub const EF_ARM_EABIMASK: u32 = 0xff00_0000; +#[allow(missing_docs)] +pub const EF_ARM_EABI_UNKNOWN: u32 = 0x0000_0000; +#[allow(missing_docs)] +pub const EF_ARM_EABI_VER1: u32 = 0x0100_0000; +#[allow(missing_docs)] +pub const EF_ARM_EABI_VER2: u32 = 0x0200_0000; +#[allow(missing_docs)] +pub const EF_ARM_EABI_VER3: u32 = 0x0300_0000; +#[allow(missing_docs)] +pub const EF_ARM_EABI_VER4: u32 = 0x0400_0000; +#[allow(missing_docs)] +pub const EF_ARM_EABI_VER5: u32 = 0x0500_0000; + +// ARM Thumb values for `st_type` component of `Sym*::st_info`. +/// A Thumb function. +pub const STT_ARM_TFUNC: u8 = STT_LOPROC; +/// A Thumb label. +pub const STT_ARM_16BIT: u8 = STT_HIPROC; + +// ARM values for `SectionHeader*::sh_flags`. +/// Section contains an entry point +pub const SHF_ARM_ENTRYSECT: u32 = 0x1000_0000; +/// Section may be multiply defined in the input to a link step. +pub const SHF_ARM_COMDEF: u32 = 0x8000_0000; + +// ARM values for `ProgramHeader*::p_flags`. +/// Segment contains the location addressed by the static base. +pub const PF_ARM_SB: u32 = 0x1000_0000; +/// Position-independent segment. +pub const PF_ARM_PI: u32 = 0x2000_0000; +/// Absolute segment. +pub const PF_ARM_ABS: u32 = 0x4000_0000; + +// ARM values for `ProgramHeader*::p_type`. +/// ARM unwind segment. +pub const PT_ARM_EXIDX: u32 = PT_LOPROC + 1; + +// ARM values for `SectionHeader*::sh_type`. +/// ARM unwind section. +pub const SHT_ARM_EXIDX: u32 = SHT_LOPROC + 1; +/// Preemption details. +pub const SHT_ARM_PREEMPTMAP: u32 = SHT_LOPROC + 2; +/// ARM attributes section. +pub const SHT_ARM_ATTRIBUTES: u32 = SHT_LOPROC + 3; + +// AArch64 values for `Rel*::r_type`. + +/// No relocation. +pub const R_AARCH64_NONE: u32 = 0; + +// ILP32 AArch64 relocs. +/// Direct 32 bit. +pub const R_AARCH64_P32_ABS32: u32 = 1; +/// Copy symbol at runtime. +pub const R_AARCH64_P32_COPY: u32 = 180; +/// Create GOT entry. +pub const R_AARCH64_P32_GLOB_DAT: u32 = 181; +/// Create PLT entry. +pub const R_AARCH64_P32_JUMP_SLOT: u32 = 182; +/// Adjust by program base. +pub const R_AARCH64_P32_RELATIVE: u32 = 183; +/// Module number, 32 bit. +pub const R_AARCH64_P32_TLS_DTPMOD: u32 = 184; +/// Module-relative offset, 32 bit. +pub const R_AARCH64_P32_TLS_DTPREL: u32 = 185; +/// TP-relative offset, 32 bit. +pub const R_AARCH64_P32_TLS_TPREL: u32 = 186; +/// TLS Descriptor. +pub const R_AARCH64_P32_TLSDESC: u32 = 187; +/// STT_GNU_IFUNC relocation. +pub const R_AARCH64_P32_IRELATIVE: u32 = 188; + +// LP64 AArch64 relocs. +/// Direct 64 bit. +pub const R_AARCH64_ABS64: u32 = 257; +/// Direct 32 bit. +pub const R_AARCH64_ABS32: u32 = 258; +/// Direct 16-bit. +pub const R_AARCH64_ABS16: u32 = 259; +/// PC-relative 64-bit. +pub const R_AARCH64_PREL64: u32 = 260; +/// PC-relative 32-bit. +pub const R_AARCH64_PREL32: u32 = 261; +/// PC-relative 16-bit. +pub const R_AARCH64_PREL16: u32 = 262; +/// Dir. MOVZ imm. from bits 15:0. +pub const R_AARCH64_MOVW_UABS_G0: u32 = 263; +/// Likewise for MOVK; no check. +pub const R_AARCH64_MOVW_UABS_G0_NC: u32 = 264; +/// Dir. MOVZ imm. from bits 31:16. +pub const R_AARCH64_MOVW_UABS_G1: u32 = 265; +/// Likewise for MOVK; no check. +pub const R_AARCH64_MOVW_UABS_G1_NC: u32 = 266; +/// Dir. MOVZ imm. from bits 47:32. +pub const R_AARCH64_MOVW_UABS_G2: u32 = 267; +/// Likewise for MOVK; no check. +pub const R_AARCH64_MOVW_UABS_G2_NC: u32 = 268; +/// Dir. MOV{K,Z} imm. from 63:48. +pub const R_AARCH64_MOVW_UABS_G3: u32 = 269; +/// Dir. MOV{N,Z} imm. from 15:0. +pub const R_AARCH64_MOVW_SABS_G0: u32 = 270; +/// Dir. MOV{N,Z} imm. from 31:16. +pub const R_AARCH64_MOVW_SABS_G1: u32 = 271; +/// Dir. MOV{N,Z} imm. from 47:32. +pub const R_AARCH64_MOVW_SABS_G2: u32 = 272; +/// PC-rel. LD imm. from bits 20:2. +pub const R_AARCH64_LD_PREL_LO19: u32 = 273; +/// PC-rel. ADR imm. from bits 20:0. +pub const R_AARCH64_ADR_PREL_LO21: u32 = 274; +/// Page-rel. ADRP imm. from 32:12. +pub const R_AARCH64_ADR_PREL_PG_HI21: u32 = 275; +/// Likewise; no overflow check. +pub const R_AARCH64_ADR_PREL_PG_HI21_NC: u32 = 276; +/// Dir. ADD imm. from bits 11:0. +pub const R_AARCH64_ADD_ABS_LO12_NC: u32 = 277; +/// Likewise for LD/ST; no check. +pub const R_AARCH64_LDST8_ABS_LO12_NC: u32 = 278; +/// PC-rel. TBZ/TBNZ imm. from 15:2. +pub const R_AARCH64_TSTBR14: u32 = 279; +/// PC-rel. cond. br. imm. from 20:2. +pub const R_AARCH64_CONDBR19: u32 = 280; +/// PC-rel. B imm. from bits 27:2. +pub const R_AARCH64_JUMP26: u32 = 282; +/// Likewise for CALL. +pub const R_AARCH64_CALL26: u32 = 283; +/// Dir. ADD imm. from bits 11:1. +pub const R_AARCH64_LDST16_ABS_LO12_NC: u32 = 284; +/// Likewise for bits 11:2. +pub const R_AARCH64_LDST32_ABS_LO12_NC: u32 = 285; +/// Likewise for bits 11:3. +pub const R_AARCH64_LDST64_ABS_LO12_NC: u32 = 286; +/// PC-rel. MOV{N,Z} imm. from 15:0. +pub const R_AARCH64_MOVW_PREL_G0: u32 = 287; +/// Likewise for MOVK; no check. +pub const R_AARCH64_MOVW_PREL_G0_NC: u32 = 288; +/// PC-rel. MOV{N,Z} imm. from 31:16. +pub const R_AARCH64_MOVW_PREL_G1: u32 = 289; +/// Likewise for MOVK; no check. +pub const R_AARCH64_MOVW_PREL_G1_NC: u32 = 290; +/// PC-rel. MOV{N,Z} imm. from 47:32. +pub const R_AARCH64_MOVW_PREL_G2: u32 = 291; +/// Likewise for MOVK; no check. +pub const R_AARCH64_MOVW_PREL_G2_NC: u32 = 292; +/// PC-rel. MOV{N,Z} imm. from 63:48. +pub const R_AARCH64_MOVW_PREL_G3: u32 = 293; +/// Dir. ADD imm. from bits 11:4. +pub const R_AARCH64_LDST128_ABS_LO12_NC: u32 = 299; +/// GOT-rel. off. MOV{N,Z} imm. 15:0. +pub const R_AARCH64_MOVW_GOTOFF_G0: u32 = 300; +/// Likewise for MOVK; no check. +pub const R_AARCH64_MOVW_GOTOFF_G0_NC: u32 = 301; +/// GOT-rel. o. MOV{N,Z} imm. 31:16. +pub const R_AARCH64_MOVW_GOTOFF_G1: u32 = 302; +/// Likewise for MOVK; no check. +pub const R_AARCH64_MOVW_GOTOFF_G1_NC: u32 = 303; +/// GOT-rel. o. MOV{N,Z} imm. 47:32. +pub const R_AARCH64_MOVW_GOTOFF_G2: u32 = 304; +/// Likewise for MOVK; no check. +pub const R_AARCH64_MOVW_GOTOFF_G2_NC: u32 = 305; +/// GOT-rel. o. MOV{N,Z} imm. 63:48. +pub const R_AARCH64_MOVW_GOTOFF_G3: u32 = 306; +/// GOT-relative 64-bit. +pub const R_AARCH64_GOTREL64: u32 = 307; +/// GOT-relative 32-bit. +pub const R_AARCH64_GOTREL32: u32 = 308; +/// PC-rel. GOT off. load imm. 20:2. +pub const R_AARCH64_GOT_LD_PREL19: u32 = 309; +/// GOT-rel. off. LD/ST imm. 14:3. +pub const R_AARCH64_LD64_GOTOFF_LO15: u32 = 310; +/// P-page-rel. GOT off. ADRP 32:12. +pub const R_AARCH64_ADR_GOT_PAGE: u32 = 311; +/// Dir. GOT off. LD/ST imm. 11:3. +pub const R_AARCH64_LD64_GOT_LO12_NC: u32 = 312; +/// GOT-page-rel. GOT off. LD/ST 14:3 +pub const R_AARCH64_LD64_GOTPAGE_LO15: u32 = 313; +/// PC-relative ADR imm. 20:0. +pub const R_AARCH64_TLSGD_ADR_PREL21: u32 = 512; +/// page-rel. ADRP imm. 32:12. +pub const R_AARCH64_TLSGD_ADR_PAGE21: u32 = 513; +/// direct ADD imm. from 11:0. +pub const R_AARCH64_TLSGD_ADD_LO12_NC: u32 = 514; +/// GOT-rel. MOV{N,Z} 31:16. +pub const R_AARCH64_TLSGD_MOVW_G1: u32 = 515; +/// GOT-rel. MOVK imm. 15:0. +pub const R_AARCH64_TLSGD_MOVW_G0_NC: u32 = 516; +/// Like 512; local dynamic model. +pub const R_AARCH64_TLSLD_ADR_PREL21: u32 = 517; +/// Like 513; local dynamic model. +pub const R_AARCH64_TLSLD_ADR_PAGE21: u32 = 518; +/// Like 514; local dynamic model. +pub const R_AARCH64_TLSLD_ADD_LO12_NC: u32 = 519; +/// Like 515; local dynamic model. +pub const R_AARCH64_TLSLD_MOVW_G1: u32 = 520; +/// Like 516; local dynamic model. +pub const R_AARCH64_TLSLD_MOVW_G0_NC: u32 = 521; +/// TLS PC-rel. load imm. 20:2. +pub const R_AARCH64_TLSLD_LD_PREL19: u32 = 522; +/// TLS DTP-rel. MOV{N,Z} 47:32. +pub const R_AARCH64_TLSLD_MOVW_DTPREL_G2: u32 = 523; +/// TLS DTP-rel. MOV{N,Z} 31:16. +pub const R_AARCH64_TLSLD_MOVW_DTPREL_G1: u32 = 524; +/// Likewise; MOVK; no check. +pub const R_AARCH64_TLSLD_MOVW_DTPREL_G1_NC: u32 = 525; +/// TLS DTP-rel. MOV{N,Z} 15:0. +pub const R_AARCH64_TLSLD_MOVW_DTPREL_G0: u32 = 526; +/// Likewise; MOVK; no check. +pub const R_AARCH64_TLSLD_MOVW_DTPREL_G0_NC: u32 = 527; +/// DTP-rel. ADD imm. from 23:12. +pub const R_AARCH64_TLSLD_ADD_DTPREL_HI12: u32 = 528; +/// DTP-rel. ADD imm. from 11:0. +pub const R_AARCH64_TLSLD_ADD_DTPREL_LO12: u32 = 529; +/// Likewise; no ovfl. check. +pub const R_AARCH64_TLSLD_ADD_DTPREL_LO12_NC: u32 = 530; +/// DTP-rel. LD/ST imm. 11:0. +pub const R_AARCH64_TLSLD_LDST8_DTPREL_LO12: u32 = 531; +/// Likewise; no check. +pub const R_AARCH64_TLSLD_LDST8_DTPREL_LO12_NC: u32 = 532; +/// DTP-rel. LD/ST imm. 11:1. +pub const R_AARCH64_TLSLD_LDST16_DTPREL_LO12: u32 = 533; +/// Likewise; no check. +pub const R_AARCH64_TLSLD_LDST16_DTPREL_LO12_NC: u32 = 534; +/// DTP-rel. LD/ST imm. 11:2. +pub const R_AARCH64_TLSLD_LDST32_DTPREL_LO12: u32 = 535; +/// Likewise; no check. +pub const R_AARCH64_TLSLD_LDST32_DTPREL_LO12_NC: u32 = 536; +/// DTP-rel. LD/ST imm. 11:3. +pub const R_AARCH64_TLSLD_LDST64_DTPREL_LO12: u32 = 537; +/// Likewise; no check. +pub const R_AARCH64_TLSLD_LDST64_DTPREL_LO12_NC: u32 = 538; +/// GOT-rel. MOV{N,Z} 31:16. +pub const R_AARCH64_TLSIE_MOVW_GOTTPREL_G1: u32 = 539; +/// GOT-rel. MOVK 15:0. +pub const R_AARCH64_TLSIE_MOVW_GOTTPREL_G0_NC: u32 = 540; +/// Page-rel. ADRP 32:12. +pub const R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: u32 = 541; +/// Direct LD off. 11:3. +pub const R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: u32 = 542; +/// PC-rel. load imm. 20:2. +pub const R_AARCH64_TLSIE_LD_GOTTPREL_PREL19: u32 = 543; +/// TLS TP-rel. MOV{N,Z} 47:32. +pub const R_AARCH64_TLSLE_MOVW_TPREL_G2: u32 = 544; +/// TLS TP-rel. MOV{N,Z} 31:16. +pub const R_AARCH64_TLSLE_MOVW_TPREL_G1: u32 = 545; +/// Likewise; MOVK; no check. +pub const R_AARCH64_TLSLE_MOVW_TPREL_G1_NC: u32 = 546; +/// TLS TP-rel. MOV{N,Z} 15:0. +pub const R_AARCH64_TLSLE_MOVW_TPREL_G0: u32 = 547; +/// Likewise; MOVK; no check. +pub const R_AARCH64_TLSLE_MOVW_TPREL_G0_NC: u32 = 548; +/// TP-rel. ADD imm. 23:12. +pub const R_AARCH64_TLSLE_ADD_TPREL_HI12: u32 = 549; +/// TP-rel. ADD imm. 11:0. +pub const R_AARCH64_TLSLE_ADD_TPREL_LO12: u32 = 550; +/// Likewise; no ovfl. check. +pub const R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: u32 = 551; +/// TP-rel. LD/ST off. 11:0. +pub const R_AARCH64_TLSLE_LDST8_TPREL_LO12: u32 = 552; +/// Likewise; no ovfl. check. +pub const R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC: u32 = 553; +/// TP-rel. LD/ST off. 11:1. +pub const R_AARCH64_TLSLE_LDST16_TPREL_LO12: u32 = 554; +/// Likewise; no check. +pub const R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC: u32 = 555; +/// TP-rel. LD/ST off. 11:2. +pub const R_AARCH64_TLSLE_LDST32_TPREL_LO12: u32 = 556; +/// Likewise; no check. +pub const R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC: u32 = 557; +/// TP-rel. LD/ST off. 11:3. +pub const R_AARCH64_TLSLE_LDST64_TPREL_LO12: u32 = 558; +/// Likewise; no check. +pub const R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC: u32 = 559; +/// PC-rel. load immediate 20:2. +pub const R_AARCH64_TLSDESC_LD_PREL19: u32 = 560; +/// PC-rel. ADR immediate 20:0. +pub const R_AARCH64_TLSDESC_ADR_PREL21: u32 = 561; +/// Page-rel. ADRP imm. 32:12. +pub const R_AARCH64_TLSDESC_ADR_PAGE21: u32 = 562; +/// Direct LD off. from 11:3. +pub const R_AARCH64_TLSDESC_LD64_LO12: u32 = 563; +/// Direct ADD imm. from 11:0. +pub const R_AARCH64_TLSDESC_ADD_LO12: u32 = 564; +/// GOT-rel. MOV{N,Z} imm. 31:16. +pub const R_AARCH64_TLSDESC_OFF_G1: u32 = 565; +/// GOT-rel. MOVK imm. 15:0; no ck. +pub const R_AARCH64_TLSDESC_OFF_G0_NC: u32 = 566; +/// Relax LDR. +pub const R_AARCH64_TLSDESC_LDR: u32 = 567; +/// Relax ADD. +pub const R_AARCH64_TLSDESC_ADD: u32 = 568; +/// Relax BLR. +pub const R_AARCH64_TLSDESC_CALL: u32 = 569; +/// TP-rel. LD/ST off. 11:4. +pub const R_AARCH64_TLSLE_LDST128_TPREL_LO12: u32 = 570; +/// Likewise; no check. +pub const R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: u32 = 571; +/// DTP-rel. LD/ST imm. 11:4. +pub const R_AARCH64_TLSLD_LDST128_DTPREL_LO12: u32 = 572; +/// Likewise; no check. +pub const R_AARCH64_TLSLD_LDST128_DTPREL_LO12_NC: u32 = 573; +/// Copy symbol at runtime. +pub const R_AARCH64_COPY: u32 = 1024; +/// Create GOT entry. +pub const R_AARCH64_GLOB_DAT: u32 = 1025; +/// Create PLT entry. +pub const R_AARCH64_JUMP_SLOT: u32 = 1026; +/// Adjust by program base. +pub const R_AARCH64_RELATIVE: u32 = 1027; +/// Module number, 64 bit. +pub const R_AARCH64_TLS_DTPMOD: u32 = 1028; +/// Module-relative offset, 64 bit. +pub const R_AARCH64_TLS_DTPREL: u32 = 1029; +/// TP-relative offset, 64 bit. +pub const R_AARCH64_TLS_TPREL: u32 = 1030; +/// TLS Descriptor. +pub const R_AARCH64_TLSDESC: u32 = 1031; +/// STT_GNU_IFUNC relocation. +pub const R_AARCH64_IRELATIVE: u32 = 1032; + +// AVR values for `Rel*::r_type`. + +/// Direct 32 bit +pub const R_AVR_32: u32 = 1; +/// Direct 16 bit +pub const R_AVR_16: u32 = 4; + +// MSP430 values for `Rel*::r_type`. + +/// Direct 32 bit +pub const R_MSP430_32: u32 = 1; +/// Direct 16 bit +pub const R_MSP430_16_BYTE: u32 = 5; + +// Hexagon values for `Rel*::r_type`. + +/// Direct 32 bit +pub const R_HEX_32: u32 = 6; + +// ARM values for `Rel*::r_type`. + +/// No reloc +pub const R_ARM_NONE: u32 = 0; +/// Deprecated PC relative 26 bit branch. +pub const R_ARM_PC24: u32 = 1; +/// Direct 32 bit +pub const R_ARM_ABS32: u32 = 2; +/// PC relative 32 bit +pub const R_ARM_REL32: u32 = 3; +#[allow(missing_docs)] +pub const R_ARM_PC13: u32 = 4; +/// Direct 16 bit +pub const R_ARM_ABS16: u32 = 5; +/// Direct 12 bit +pub const R_ARM_ABS12: u32 = 6; +/// Direct & 0x7C (LDR, STR). +pub const R_ARM_THM_ABS5: u32 = 7; +/// Direct 8 bit +pub const R_ARM_ABS8: u32 = 8; +#[allow(missing_docs)] +pub const R_ARM_SBREL32: u32 = 9; +/// PC relative 24 bit (Thumb32 BL). +pub const R_ARM_THM_PC22: u32 = 10; +/// PC relative & 0x3FC (Thumb16 LDR, ADD, ADR). +pub const R_ARM_THM_PC8: u32 = 11; +#[allow(missing_docs)] +pub const R_ARM_AMP_VCALL9: u32 = 12; +/// Obsolete static relocation. +pub const R_ARM_SWI24: u32 = 13; +/// Dynamic relocation. +pub const R_ARM_TLS_DESC: u32 = 13; +/// Reserved. +pub const R_ARM_THM_SWI8: u32 = 14; +/// Reserved. +pub const R_ARM_XPC25: u32 = 15; +/// Reserved. +pub const R_ARM_THM_XPC22: u32 = 16; +/// ID of module containing symbol +pub const R_ARM_TLS_DTPMOD32: u32 = 17; +/// Offset in TLS block +pub const R_ARM_TLS_DTPOFF32: u32 = 18; +/// Offset in static TLS block +pub const R_ARM_TLS_TPOFF32: u32 = 19; +/// Copy symbol at runtime +pub const R_ARM_COPY: u32 = 20; +/// Create GOT entry +pub const R_ARM_GLOB_DAT: u32 = 21; +/// Create PLT entry +pub const R_ARM_JUMP_SLOT: u32 = 22; +/// Adjust by program base +pub const R_ARM_RELATIVE: u32 = 23; +/// 32 bit offset to GOT +pub const R_ARM_GOTOFF: u32 = 24; +/// 32 bit PC relative offset to GOT +pub const R_ARM_GOTPC: u32 = 25; +/// 32 bit GOT entry +pub const R_ARM_GOT32: u32 = 26; +/// Deprecated, 32 bit PLT address. +pub const R_ARM_PLT32: u32 = 27; +/// PC relative 24 bit (BL, BLX). +pub const R_ARM_CALL: u32 = 28; +/// PC relative 24 bit (B, BL). +pub const R_ARM_JUMP24: u32 = 29; +/// PC relative 24 bit (Thumb32 B.W). +pub const R_ARM_THM_JUMP24: u32 = 30; +/// Adjust by program base. +pub const R_ARM_BASE_ABS: u32 = 31; +/// Obsolete. +pub const R_ARM_ALU_PCREL_7_0: u32 = 32; +/// Obsolete. +pub const R_ARM_ALU_PCREL_15_8: u32 = 33; +/// Obsolete. +pub const R_ARM_ALU_PCREL_23_15: u32 = 34; +/// Deprecated, prog. base relative. +pub const R_ARM_LDR_SBREL_11_0: u32 = 35; +/// Deprecated, prog. base relative. +pub const R_ARM_ALU_SBREL_19_12: u32 = 36; +/// Deprecated, prog. base relative. +pub const R_ARM_ALU_SBREL_27_20: u32 = 37; +#[allow(missing_docs)] +pub const R_ARM_TARGET1: u32 = 38; +/// Program base relative. +pub const R_ARM_SBREL31: u32 = 39; +#[allow(missing_docs)] +pub const R_ARM_V4BX: u32 = 40; +#[allow(missing_docs)] +pub const R_ARM_TARGET2: u32 = 41; +/// 32 bit PC relative. +pub const R_ARM_PREL31: u32 = 42; +/// Direct 16-bit (MOVW). +pub const R_ARM_MOVW_ABS_NC: u32 = 43; +/// Direct high 16-bit (MOVT). +pub const R_ARM_MOVT_ABS: u32 = 44; +/// PC relative 16-bit (MOVW). +pub const R_ARM_MOVW_PREL_NC: u32 = 45; +/// PC relative (MOVT). +pub const R_ARM_MOVT_PREL: u32 = 46; +/// Direct 16 bit (Thumb32 MOVW). +pub const R_ARM_THM_MOVW_ABS_NC: u32 = 47; +/// Direct high 16 bit (Thumb32 MOVT). +pub const R_ARM_THM_MOVT_ABS: u32 = 48; +/// PC relative 16 bit (Thumb32 MOVW). +pub const R_ARM_THM_MOVW_PREL_NC: u32 = 49; +/// PC relative high 16 bit (Thumb32 MOVT). +pub const R_ARM_THM_MOVT_PREL: u32 = 50; +/// PC relative 20 bit (Thumb32 B.W). +pub const R_ARM_THM_JUMP19: u32 = 51; +/// PC relative X & 0x7E (Thumb16 CBZ, CBNZ). +pub const R_ARM_THM_JUMP6: u32 = 52; +/// PC relative 12 bit (Thumb32 ADR.W). +pub const R_ARM_THM_ALU_PREL_11_0: u32 = 53; +/// PC relative 12 bit (Thumb32 LDR{D,SB,H,SH}). +pub const R_ARM_THM_PC12: u32 = 54; +/// Direct 32-bit. +pub const R_ARM_ABS32_NOI: u32 = 55; +/// PC relative 32-bit. +pub const R_ARM_REL32_NOI: u32 = 56; +/// PC relative (ADD, SUB). +pub const R_ARM_ALU_PC_G0_NC: u32 = 57; +/// PC relative (ADD, SUB). +pub const R_ARM_ALU_PC_G0: u32 = 58; +/// PC relative (ADD, SUB). +pub const R_ARM_ALU_PC_G1_NC: u32 = 59; +/// PC relative (ADD, SUB). +pub const R_ARM_ALU_PC_G1: u32 = 60; +/// PC relative (ADD, SUB). +pub const R_ARM_ALU_PC_G2: u32 = 61; +/// PC relative (LDR,STR,LDRB,STRB). +pub const R_ARM_LDR_PC_G1: u32 = 62; +/// PC relative (LDR,STR,LDRB,STRB). +pub const R_ARM_LDR_PC_G2: u32 = 63; +/// PC relative (STR{D,H}, LDR{D,SB,H,SH}). +pub const R_ARM_LDRS_PC_G0: u32 = 64; +/// PC relative (STR{D,H}, LDR{D,SB,H,SH}). +pub const R_ARM_LDRS_PC_G1: u32 = 65; +/// PC relative (STR{D,H}, LDR{D,SB,H,SH}). +pub const R_ARM_LDRS_PC_G2: u32 = 66; +/// PC relative (LDC, STC). +pub const R_ARM_LDC_PC_G0: u32 = 67; +/// PC relative (LDC, STC). +pub const R_ARM_LDC_PC_G1: u32 = 68; +/// PC relative (LDC, STC). +pub const R_ARM_LDC_PC_G2: u32 = 69; +/// Program base relative (ADD,SUB). +pub const R_ARM_ALU_SB_G0_NC: u32 = 70; +/// Program base relative (ADD,SUB). +pub const R_ARM_ALU_SB_G0: u32 = 71; +/// Program base relative (ADD,SUB). +pub const R_ARM_ALU_SB_G1_NC: u32 = 72; +/// Program base relative (ADD,SUB). +pub const R_ARM_ALU_SB_G1: u32 = 73; +/// Program base relative (ADD,SUB). +pub const R_ARM_ALU_SB_G2: u32 = 74; +/// Program base relative (LDR, STR, LDRB, STRB). +pub const R_ARM_LDR_SB_G0: u32 = 75; +/// Program base relative (LDR, STR, LDRB, STRB). +pub const R_ARM_LDR_SB_G1: u32 = 76; +/// Program base relative (LDR, STR, LDRB, STRB). +pub const R_ARM_LDR_SB_G2: u32 = 77; +/// Program base relative (LDR, STR, LDRB, STRB). +pub const R_ARM_LDRS_SB_G0: u32 = 78; +/// Program base relative (LDR, STR, LDRB, STRB). +pub const R_ARM_LDRS_SB_G1: u32 = 79; +/// Program base relative (LDR, STR, LDRB, STRB). +pub const R_ARM_LDRS_SB_G2: u32 = 80; +/// Program base relative (LDC,STC). +pub const R_ARM_LDC_SB_G0: u32 = 81; +/// Program base relative (LDC,STC). +pub const R_ARM_LDC_SB_G1: u32 = 82; +/// Program base relative (LDC,STC). +pub const R_ARM_LDC_SB_G2: u32 = 83; +/// Program base relative 16 bit (MOVW). +pub const R_ARM_MOVW_BREL_NC: u32 = 84; +/// Program base relative high 16 bit (MOVT). +pub const R_ARM_MOVT_BREL: u32 = 85; +/// Program base relative 16 bit (MOVW). +pub const R_ARM_MOVW_BREL: u32 = 86; +/// Program base relative 16 bit (Thumb32 MOVW). +pub const R_ARM_THM_MOVW_BREL_NC: u32 = 87; +/// Program base relative high 16 bit (Thumb32 MOVT). +pub const R_ARM_THM_MOVT_BREL: u32 = 88; +/// Program base relative 16 bit (Thumb32 MOVW). +pub const R_ARM_THM_MOVW_BREL: u32 = 89; +#[allow(missing_docs)] +pub const R_ARM_TLS_GOTDESC: u32 = 90; +#[allow(missing_docs)] +pub const R_ARM_TLS_CALL: u32 = 91; +/// TLS relaxation. +pub const R_ARM_TLS_DESCSEQ: u32 = 92; +#[allow(missing_docs)] +pub const R_ARM_THM_TLS_CALL: u32 = 93; +#[allow(missing_docs)] +pub const R_ARM_PLT32_ABS: u32 = 94; +/// GOT entry. +pub const R_ARM_GOT_ABS: u32 = 95; +/// PC relative GOT entry. +pub const R_ARM_GOT_PREL: u32 = 96; +/// GOT entry relative to GOT origin (LDR). +pub const R_ARM_GOT_BREL12: u32 = 97; +/// 12 bit, GOT entry relative to GOT origin (LDR, STR). +pub const R_ARM_GOTOFF12: u32 = 98; +#[allow(missing_docs)] +pub const R_ARM_GOTRELAX: u32 = 99; +#[allow(missing_docs)] +pub const R_ARM_GNU_VTENTRY: u32 = 100; +#[allow(missing_docs)] +pub const R_ARM_GNU_VTINHERIT: u32 = 101; +/// PC relative & 0xFFE (Thumb16 B). +pub const R_ARM_THM_PC11: u32 = 102; +/// PC relative & 0x1FE (Thumb16 B/B). +pub const R_ARM_THM_PC9: u32 = 103; +/// PC-rel 32 bit for global dynamic thread local data +pub const R_ARM_TLS_GD32: u32 = 104; +/// PC-rel 32 bit for local dynamic thread local data +pub const R_ARM_TLS_LDM32: u32 = 105; +/// 32 bit offset relative to TLS block +pub const R_ARM_TLS_LDO32: u32 = 106; +/// PC-rel 32 bit for GOT entry of static TLS block offset +pub const R_ARM_TLS_IE32: u32 = 107; +/// 32 bit offset relative to static TLS block +pub const R_ARM_TLS_LE32: u32 = 108; +/// 12 bit relative to TLS block (LDR, STR). +pub const R_ARM_TLS_LDO12: u32 = 109; +/// 12 bit relative to static TLS block (LDR, STR). +pub const R_ARM_TLS_LE12: u32 = 110; +/// 12 bit GOT entry relative to GOT origin (LDR). +pub const R_ARM_TLS_IE12GP: u32 = 111; +/// Obsolete. +pub const R_ARM_ME_TOO: u32 = 128; +#[allow(missing_docs)] +pub const R_ARM_THM_TLS_DESCSEQ: u32 = 129; +#[allow(missing_docs)] +pub const R_ARM_THM_TLS_DESCSEQ16: u32 = 129; +#[allow(missing_docs)] +pub const R_ARM_THM_TLS_DESCSEQ32: u32 = 130; +/// GOT entry relative to GOT origin, 12 bit (Thumb32 LDR). +pub const R_ARM_THM_GOT_BREL12: u32 = 131; +#[allow(missing_docs)] +pub const R_ARM_IRELATIVE: u32 = 160; +#[allow(missing_docs)] +pub const R_ARM_RXPC25: u32 = 249; +#[allow(missing_docs)] +pub const R_ARM_RSBREL32: u32 = 250; +#[allow(missing_docs)] +pub const R_ARM_THM_RPC22: u32 = 251; +#[allow(missing_docs)] +pub const R_ARM_RREL32: u32 = 252; +#[allow(missing_docs)] +pub const R_ARM_RABS22: u32 = 253; +#[allow(missing_docs)] +pub const R_ARM_RPC24: u32 = 254; +#[allow(missing_docs)] +pub const R_ARM_RBASE: u32 = 255; + +// C-SKY values for `Rel*::r_type`. +/// no reloc +pub const R_CKCORE_NONE: u32 = 0; +/// direct 32 bit (S + A) +pub const R_CKCORE_ADDR32: u32 = 1; +/// disp ((S + A - P) >> 2) & 0xff +pub const R_CKCORE_PCRELIMM8BY4: u32 = 2; +/// disp ((S + A - P) >> 1) & 0x7ff +pub const R_CKCORE_PCRELIMM11BY2: u32 = 3; +/// 32-bit rel (S + A - P) +pub const R_CKCORE_PCREL32: u32 = 5; +/// disp ((S + A - P) >>1) & 0x7ff +pub const R_CKCORE_PCRELJSR_IMM11BY2: u32 = 6; +/// 32 bit adjust program base(B + A) +pub const R_CKCORE_RELATIVE: u32 = 9; +/// 32 bit adjust by program base +pub const R_CKCORE_COPY: u32 = 10; +/// off between got and sym (S) +pub const R_CKCORE_GLOB_DAT: u32 = 11; +/// PLT entry (S) +pub const R_CKCORE_JUMP_SLOT: u32 = 12; +/// offset to GOT (S + A - GOT) +pub const R_CKCORE_GOTOFF: u32 = 13; +/// PC offset to GOT (GOT + A - P) +pub const R_CKCORE_GOTPC: u32 = 14; +/// 32 bit GOT entry (G) +pub const R_CKCORE_GOT32: u32 = 15; +/// 32 bit PLT entry (G) +pub const R_CKCORE_PLT32: u32 = 16; +/// GOT entry in GLOB_DAT (GOT + G) +pub const R_CKCORE_ADDRGOT: u32 = 17; +/// PLT entry in GLOB_DAT (GOT + G) +pub const R_CKCORE_ADDRPLT: u32 = 18; +/// ((S + A - P) >> 1) & 0x3ff_ffff +pub const R_CKCORE_PCREL_IMM26BY2: u32 = 19; +/// disp ((S + A - P) >> 1) & 0xffff +pub const R_CKCORE_PCREL_IMM16BY2: u32 = 20; +/// disp ((S + A - P) >> 2) & 0xffff +pub const R_CKCORE_PCREL_IMM16BY4: u32 = 21; +/// disp ((S + A - P) >> 1) & 0x3ff +pub const R_CKCORE_PCREL_IMM10BY2: u32 = 22; +/// disp ((S + A - P) >> 2) & 0x3ff +pub const R_CKCORE_PCREL_IMM10BY4: u32 = 23; +/// high & low 16 bit ADDR, ((S + A) >> 16) & 0xffff +pub const R_CKCORE_ADDR_HI16: u32 = 24; +/// (S + A) & 0xffff +pub const R_CKCORE_ADDR_LO16: u32 = 25; +/// high & low 16 bit GOTPC, ((GOT + A - P) >> 16) & 0xffff +pub const R_CKCORE_GOTPC_HI16: u32 = 26; +/// (GOT + A - P) & 0xffff +pub const R_CKCORE_GOTPC_LO16: u32 = 27; +/// high & low 16 bit GOTOFF, ((S + A - GOT) >> 16) & 0xffff +pub const R_CKCORE_GOTOFF_HI16: u32 = 28; +/// (S + A - GOT) & 0xffff +pub const R_CKCORE_GOTOFF_LO16: u32 = 29; +/// 12 bit disp GOT entry (G) +pub const R_CKCORE_GOT12: u32 = 30; +/// high & low 16 bit GOT, (G >> 16) & 0xffff +pub const R_CKCORE_GOT_HI16: u32 = 31; +/// (G & 0xffff) +pub const R_CKCORE_GOT_LO16: u32 = 32; +/// 12 bit disp PLT entry (G) +pub const R_CKCORE_PLT12: u32 = 33; +/// high & low 16 bit PLT, (G >> 16) & 0xffff +pub const R_CKCORE_PLT_HI16: u32 = 34; +/// G & 0xffff +pub const R_CKCORE_PLT_LO16: u32 = 35; +/// high & low 16 bit ADDRGOT, (GOT + G * 4) & 0xffff +pub const R_CKCORE_ADDRGOT_HI16: u32 = 36; +/// (GOT + G * 4) & 0xffff +pub const R_CKCORE_ADDRGOT_LO16: u32 = 37; +/// high & low 16 bit ADDRPLT, ((GOT + G * 4) >> 16) & 0xFFFF +pub const R_CKCORE_ADDRPLT_HI16: u32 = 38; +/// (GOT+G*4) & 0xffff +pub const R_CKCORE_ADDRPLT_LO16: u32 = 39; +/// disp ((S+A-P) >>1) & x3ff_ffff +pub const R_CKCORE_PCREL_JSR_IMM26BY2: u32 = 40; +/// (S+A-BTEXT) & 0xffff +pub const R_CKCORE_TOFFSET_LO16: u32 = 41; +/// (S+A-BTEXT) & 0xffff +pub const R_CKCORE_DOFFSET_LO16: u32 = 42; +/// disp ((S+A-P) >>1) & 0x3ffff +pub const R_CKCORE_PCREL_IMM18BY2: u32 = 43; +/// disp (S+A-BDATA) & 0x3ffff +pub const R_CKCORE_DOFFSET_IMM18: u32 = 44; +/// disp ((S+A-BDATA)>>1) & 0x3ffff +pub const R_CKCORE_DOFFSET_IMM18BY2: u32 = 45; +/// disp ((S+A-BDATA)>>2) & 0x3ffff +pub const R_CKCORE_DOFFSET_IMM18BY4: u32 = 46; +/// disp (G >> 2) +pub const R_CKCORE_GOT_IMM18BY4: u32 = 48; +/// disp (G >> 2) +pub const R_CKCORE_PLT_IMM18BY4: u32 = 49; +/// disp ((S+A-P) >>2) & 0x7f +pub const R_CKCORE_PCREL_IMM7BY4: u32 = 50; +/// 32 bit offset to TLS block +pub const R_CKCORE_TLS_LE32: u32 = 51; +#[allow(missing_docs)] +pub const R_CKCORE_TLS_IE32: u32 = 52; +#[allow(missing_docs)] +pub const R_CKCORE_TLS_GD32: u32 = 53; +#[allow(missing_docs)] +pub const R_CKCORE_TLS_LDM32: u32 = 54; +#[allow(missing_docs)] +pub const R_CKCORE_TLS_LDO32: u32 = 55; +#[allow(missing_docs)] +pub const R_CKCORE_TLS_DTPMOD32: u32 = 56; +#[allow(missing_docs)] +pub const R_CKCORE_TLS_DTPOFF32: u32 = 57; +#[allow(missing_docs)] +pub const R_CKCORE_TLS_TPOFF32: u32 = 58; + +// C-SKY values for `FileHeader*::e_flags`. +#[allow(missing_docs)] +pub const EF_CSKY_ABIMASK: u32 = 0xF000_0000; +#[allow(missing_docs)] +pub const EF_CSKY_OTHER: u32 = 0x0FFF_0000; +#[allow(missing_docs)] +pub const EF_CSKY_PROCESSOR: u32 = 0x0000_FFFF; + +#[allow(missing_docs)] +pub const EF_CSKY_ABIV1: u32 = 0x1000_0000; +#[allow(missing_docs)] +pub const EF_CSKY_ABIV2: u32 = 0x2000_0000; + +// C-SKY values for `SectionHeader*::sh_type`. +/// C-SKY attributes section. +pub const SHT_CSKY_ATTRIBUTES: u32 = SHT_LOPROC + 1; + +// IA-64 specific declarations. + +// IA-64 values for `FileHeader64::e_flags`. +/// os-specific flags +pub const EF_IA_64_MASKOS: u32 = 0x0000_000f; +/// 64-bit ABI +pub const EF_IA_64_ABI64: u32 = 0x0000_0010; +/// arch. version mask +pub const EF_IA_64_ARCH: u32 = 0xff00_0000; + +// IA-64 values for `ProgramHeader64::p_type`. +/// arch extension bits +pub const PT_IA_64_ARCHEXT: u32 = PT_LOPROC + 0; +/// ia64 unwind bits +pub const PT_IA_64_UNWIND: u32 = PT_LOPROC + 1; +#[allow(missing_docs)] +pub const PT_IA_64_HP_OPT_ANOT: u32 = PT_LOOS + 0x12; +#[allow(missing_docs)] +pub const PT_IA_64_HP_HSL_ANOT: u32 = PT_LOOS + 0x13; +#[allow(missing_docs)] +pub const PT_IA_64_HP_STACK: u32 = PT_LOOS + 0x14; + +// IA-64 values for `ProgramHeader64::p_flags`. +/// spec insns w/o recovery +pub const PF_IA_64_NORECOV: u32 = 0x8000_0000; + +// IA-64 values for `SectionHeader64::sh_type`. +/// extension bits +pub const SHT_IA_64_EXT: u32 = SHT_LOPROC + 0; +/// unwind bits +pub const SHT_IA_64_UNWIND: u32 = SHT_LOPROC + 1; + +// IA-64 values for `SectionHeader64::sh_flags`. +/// section near gp +pub const SHF_IA_64_SHORT: u32 = 0x1000_0000; +/// spec insns w/o recovery +pub const SHF_IA_64_NORECOV: u32 = 0x2000_0000; + +// IA-64 values for `Dyn64::d_tag`. +#[allow(missing_docs)] +pub const DT_IA_64_PLT_RESERVE: u32 = DT_LOPROC + 0; + +// IA-64 values for `Rel*::r_type`. +/// none +pub const R_IA64_NONE: u32 = 0x00; +/// symbol + addend, add imm14 +pub const R_IA64_IMM14: u32 = 0x21; +/// symbol + addend, add imm22 +pub const R_IA64_IMM22: u32 = 0x22; +/// symbol + addend, mov imm64 +pub const R_IA64_IMM64: u32 = 0x23; +/// symbol + addend, data4 MSB +pub const R_IA64_DIR32MSB: u32 = 0x24; +/// symbol + addend, data4 LSB +pub const R_IA64_DIR32LSB: u32 = 0x25; +/// symbol + addend, data8 MSB +pub const R_IA64_DIR64MSB: u32 = 0x26; +/// symbol + addend, data8 LSB +pub const R_IA64_DIR64LSB: u32 = 0x27; +/// @gprel(sym + add), add imm22 +pub const R_IA64_GPREL22: u32 = 0x2a; +/// @gprel(sym + add), mov imm64 +pub const R_IA64_GPREL64I: u32 = 0x2b; +/// @gprel(sym + add), data4 MSB +pub const R_IA64_GPREL32MSB: u32 = 0x2c; +/// @gprel(sym + add), data4 LSB +pub const R_IA64_GPREL32LSB: u32 = 0x2d; +/// @gprel(sym + add), data8 MSB +pub const R_IA64_GPREL64MSB: u32 = 0x2e; +/// @gprel(sym + add), data8 LSB +pub const R_IA64_GPREL64LSB: u32 = 0x2f; +/// @ltoff(sym + add), add imm22 +pub const R_IA64_LTOFF22: u32 = 0x32; +/// @ltoff(sym + add), mov imm64 +pub const R_IA64_LTOFF64I: u32 = 0x33; +/// @pltoff(sym + add), add imm22 +pub const R_IA64_PLTOFF22: u32 = 0x3a; +/// @pltoff(sym + add), mov imm64 +pub const R_IA64_PLTOFF64I: u32 = 0x3b; +/// @pltoff(sym + add), data8 MSB +pub const R_IA64_PLTOFF64MSB: u32 = 0x3e; +/// @pltoff(sym + add), data8 LSB +pub const R_IA64_PLTOFF64LSB: u32 = 0x3f; +/// @fptr(sym + add), mov imm64 +pub const R_IA64_FPTR64I: u32 = 0x43; +/// @fptr(sym + add), data4 MSB +pub const R_IA64_FPTR32MSB: u32 = 0x44; +/// @fptr(sym + add), data4 LSB +pub const R_IA64_FPTR32LSB: u32 = 0x45; +/// @fptr(sym + add), data8 MSB +pub const R_IA64_FPTR64MSB: u32 = 0x46; +/// @fptr(sym + add), data8 LSB +pub const R_IA64_FPTR64LSB: u32 = 0x47; +/// @pcrel(sym + add), brl +pub const R_IA64_PCREL60B: u32 = 0x48; +/// @pcrel(sym + add), ptb, call +pub const R_IA64_PCREL21B: u32 = 0x49; +/// @pcrel(sym + add), chk.s +pub const R_IA64_PCREL21M: u32 = 0x4a; +/// @pcrel(sym + add), fchkf +pub const R_IA64_PCREL21F: u32 = 0x4b; +/// @pcrel(sym + add), data4 MSB +pub const R_IA64_PCREL32MSB: u32 = 0x4c; +/// @pcrel(sym + add), data4 LSB +pub const R_IA64_PCREL32LSB: u32 = 0x4d; +/// @pcrel(sym + add), data8 MSB +pub const R_IA64_PCREL64MSB: u32 = 0x4e; +/// @pcrel(sym + add), data8 LSB +pub const R_IA64_PCREL64LSB: u32 = 0x4f; +/// @ltoff(@fptr(s+a)), imm22 +pub const R_IA64_LTOFF_FPTR22: u32 = 0x52; +/// @ltoff(@fptr(s+a)), imm64 +pub const R_IA64_LTOFF_FPTR64I: u32 = 0x53; +/// @ltoff(@fptr(s+a)), data4 MSB +pub const R_IA64_LTOFF_FPTR32MSB: u32 = 0x54; +/// @ltoff(@fptr(s+a)), data4 LSB +pub const R_IA64_LTOFF_FPTR32LSB: u32 = 0x55; +/// @ltoff(@fptr(s+a)), data8 MSB +pub const R_IA64_LTOFF_FPTR64MSB: u32 = 0x56; +/// @ltoff(@fptr(s+a)), data8 LSB +pub const R_IA64_LTOFF_FPTR64LSB: u32 = 0x57; +/// @segrel(sym + add), data4 MSB +pub const R_IA64_SEGREL32MSB: u32 = 0x5c; +/// @segrel(sym + add), data4 LSB +pub const R_IA64_SEGREL32LSB: u32 = 0x5d; +/// @segrel(sym + add), data8 MSB +pub const R_IA64_SEGREL64MSB: u32 = 0x5e; +/// @segrel(sym + add), data8 LSB +pub const R_IA64_SEGREL64LSB: u32 = 0x5f; +/// @secrel(sym + add), data4 MSB +pub const R_IA64_SECREL32MSB: u32 = 0x64; +/// @secrel(sym + add), data4 LSB +pub const R_IA64_SECREL32LSB: u32 = 0x65; +/// @secrel(sym + add), data8 MSB +pub const R_IA64_SECREL64MSB: u32 = 0x66; +/// @secrel(sym + add), data8 LSB +pub const R_IA64_SECREL64LSB: u32 = 0x67; +/// data 4 + REL +pub const R_IA64_REL32MSB: u32 = 0x6c; +/// data 4 + REL +pub const R_IA64_REL32LSB: u32 = 0x6d; +/// data 8 + REL +pub const R_IA64_REL64MSB: u32 = 0x6e; +/// data 8 + REL +pub const R_IA64_REL64LSB: u32 = 0x6f; +/// symbol + addend, data4 MSB +pub const R_IA64_LTV32MSB: u32 = 0x74; +/// symbol + addend, data4 LSB +pub const R_IA64_LTV32LSB: u32 = 0x75; +/// symbol + addend, data8 MSB +pub const R_IA64_LTV64MSB: u32 = 0x76; +/// symbol + addend, data8 LSB +pub const R_IA64_LTV64LSB: u32 = 0x77; +/// @pcrel(sym + add), 21bit inst +pub const R_IA64_PCREL21BI: u32 = 0x79; +/// @pcrel(sym + add), 22bit inst +pub const R_IA64_PCREL22: u32 = 0x7a; +/// @pcrel(sym + add), 64bit inst +pub const R_IA64_PCREL64I: u32 = 0x7b; +/// dynamic reloc, imported PLT, MSB +pub const R_IA64_IPLTMSB: u32 = 0x80; +/// dynamic reloc, imported PLT, LSB +pub const R_IA64_IPLTLSB: u32 = 0x81; +/// copy relocation +pub const R_IA64_COPY: u32 = 0x84; +/// Addend and symbol difference +pub const R_IA64_SUB: u32 = 0x85; +/// LTOFF22, relaxable. +pub const R_IA64_LTOFF22X: u32 = 0x86; +/// Use of LTOFF22X. +pub const R_IA64_LDXMOV: u32 = 0x87; +/// @tprel(sym + add), imm14 +pub const R_IA64_TPREL14: u32 = 0x91; +/// @tprel(sym + add), imm22 +pub const R_IA64_TPREL22: u32 = 0x92; +/// @tprel(sym + add), imm64 +pub const R_IA64_TPREL64I: u32 = 0x93; +/// @tprel(sym + add), data8 MSB +pub const R_IA64_TPREL64MSB: u32 = 0x96; +/// @tprel(sym + add), data8 LSB +pub const R_IA64_TPREL64LSB: u32 = 0x97; +/// @ltoff(@tprel(s+a)), imm2 +pub const R_IA64_LTOFF_TPREL22: u32 = 0x9a; +/// @dtpmod(sym + add), data8 MSB +pub const R_IA64_DTPMOD64MSB: u32 = 0xa6; +/// @dtpmod(sym + add), data8 LSB +pub const R_IA64_DTPMOD64LSB: u32 = 0xa7; +/// @ltoff(@dtpmod(sym + add)), imm22 +pub const R_IA64_LTOFF_DTPMOD22: u32 = 0xaa; +/// @dtprel(sym + add), imm14 +pub const R_IA64_DTPREL14: u32 = 0xb1; +/// @dtprel(sym + add), imm22 +pub const R_IA64_DTPREL22: u32 = 0xb2; +/// @dtprel(sym + add), imm64 +pub const R_IA64_DTPREL64I: u32 = 0xb3; +/// @dtprel(sym + add), data4 MSB +pub const R_IA64_DTPREL32MSB: u32 = 0xb4; +/// @dtprel(sym + add), data4 LSB +pub const R_IA64_DTPREL32LSB: u32 = 0xb5; +/// @dtprel(sym + add), data8 MSB +pub const R_IA64_DTPREL64MSB: u32 = 0xb6; +/// @dtprel(sym + add), data8 LSB +pub const R_IA64_DTPREL64LSB: u32 = 0xb7; +/// @ltoff(@dtprel(s+a)), imm22 +pub const R_IA64_LTOFF_DTPREL22: u32 = 0xba; + +// SH specific declarations. + +// SH values `FileHeader*::e_flags`. +#[allow(missing_docs)] +pub const EF_SH_MACH_MASK: u32 = 0x1f; +#[allow(missing_docs)] +pub const EF_SH_UNKNOWN: u32 = 0x0; +#[allow(missing_docs)] +pub const EF_SH1: u32 = 0x1; +#[allow(missing_docs)] +pub const EF_SH2: u32 = 0x2; +#[allow(missing_docs)] +pub const EF_SH3: u32 = 0x3; +#[allow(missing_docs)] +pub const EF_SH_DSP: u32 = 0x4; +#[allow(missing_docs)] +pub const EF_SH3_DSP: u32 = 0x5; +#[allow(missing_docs)] +pub const EF_SH4AL_DSP: u32 = 0x6; +#[allow(missing_docs)] +pub const EF_SH3E: u32 = 0x8; +#[allow(missing_docs)] +pub const EF_SH4: u32 = 0x9; +#[allow(missing_docs)] +pub const EF_SH2E: u32 = 0xb; +#[allow(missing_docs)] +pub const EF_SH4A: u32 = 0xc; +#[allow(missing_docs)] +pub const EF_SH2A: u32 = 0xd; +#[allow(missing_docs)] +pub const EF_SH4_NOFPU: u32 = 0x10; +#[allow(missing_docs)] +pub const EF_SH4A_NOFPU: u32 = 0x11; +#[allow(missing_docs)] +pub const EF_SH4_NOMMU_NOFPU: u32 = 0x12; +#[allow(missing_docs)] +pub const EF_SH2A_NOFPU: u32 = 0x13; +#[allow(missing_docs)] +pub const EF_SH3_NOMMU: u32 = 0x14; +#[allow(missing_docs)] +pub const EF_SH2A_SH4_NOFPU: u32 = 0x15; +#[allow(missing_docs)] +pub const EF_SH2A_SH3_NOFPU: u32 = 0x16; +#[allow(missing_docs)] +pub const EF_SH2A_SH4: u32 = 0x17; +#[allow(missing_docs)] +pub const EF_SH2A_SH3E: u32 = 0x18; + +// SH values `Rel*::r_type`. +#[allow(missing_docs)] +pub const R_SH_NONE: u32 = 0; +#[allow(missing_docs)] +pub const R_SH_DIR32: u32 = 1; +#[allow(missing_docs)] +pub const R_SH_REL32: u32 = 2; +#[allow(missing_docs)] +pub const R_SH_DIR8WPN: u32 = 3; +#[allow(missing_docs)] +pub const R_SH_IND12W: u32 = 4; +#[allow(missing_docs)] +pub const R_SH_DIR8WPL: u32 = 5; +#[allow(missing_docs)] +pub const R_SH_DIR8WPZ: u32 = 6; +#[allow(missing_docs)] +pub const R_SH_DIR8BP: u32 = 7; +#[allow(missing_docs)] +pub const R_SH_DIR8W: u32 = 8; +#[allow(missing_docs)] +pub const R_SH_DIR8L: u32 = 9; +#[allow(missing_docs)] +pub const R_SH_SWITCH16: u32 = 25; +#[allow(missing_docs)] +pub const R_SH_SWITCH32: u32 = 26; +#[allow(missing_docs)] +pub const R_SH_USES: u32 = 27; +#[allow(missing_docs)] +pub const R_SH_COUNT: u32 = 28; +#[allow(missing_docs)] +pub const R_SH_ALIGN: u32 = 29; +#[allow(missing_docs)] +pub const R_SH_CODE: u32 = 30; +#[allow(missing_docs)] +pub const R_SH_DATA: u32 = 31; +#[allow(missing_docs)] +pub const R_SH_LABEL: u32 = 32; +#[allow(missing_docs)] +pub const R_SH_SWITCH8: u32 = 33; +#[allow(missing_docs)] +pub const R_SH_GNU_VTINHERIT: u32 = 34; +#[allow(missing_docs)] +pub const R_SH_GNU_VTENTRY: u32 = 35; +#[allow(missing_docs)] +pub const R_SH_TLS_GD_32: u32 = 144; +#[allow(missing_docs)] +pub const R_SH_TLS_LD_32: u32 = 145; +#[allow(missing_docs)] +pub const R_SH_TLS_LDO_32: u32 = 146; +#[allow(missing_docs)] +pub const R_SH_TLS_IE_32: u32 = 147; +#[allow(missing_docs)] +pub const R_SH_TLS_LE_32: u32 = 148; +#[allow(missing_docs)] +pub const R_SH_TLS_DTPMOD32: u32 = 149; +#[allow(missing_docs)] +pub const R_SH_TLS_DTPOFF32: u32 = 150; +#[allow(missing_docs)] +pub const R_SH_TLS_TPOFF32: u32 = 151; +#[allow(missing_docs)] +pub const R_SH_GOT32: u32 = 160; +#[allow(missing_docs)] +pub const R_SH_PLT32: u32 = 161; +#[allow(missing_docs)] +pub const R_SH_COPY: u32 = 162; +#[allow(missing_docs)] +pub const R_SH_GLOB_DAT: u32 = 163; +#[allow(missing_docs)] +pub const R_SH_JMP_SLOT: u32 = 164; +#[allow(missing_docs)] +pub const R_SH_RELATIVE: u32 = 165; +#[allow(missing_docs)] +pub const R_SH_GOTOFF: u32 = 166; +#[allow(missing_docs)] +pub const R_SH_GOTPC: u32 = 167; + +// S/390 specific definitions. + +// S/390 values `FileHeader*::e_flags`. + +/// High GPRs kernel facility needed. +pub const EF_S390_HIGH_GPRS: u32 = 0x0000_0001; + +// S/390 values `Rel*::r_type`. + +/// No reloc. +pub const R_390_NONE: u32 = 0; +/// Direct 8 bit. +pub const R_390_8: u32 = 1; +/// Direct 12 bit. +pub const R_390_12: u32 = 2; +/// Direct 16 bit. +pub const R_390_16: u32 = 3; +/// Direct 32 bit. +pub const R_390_32: u32 = 4; +/// PC relative 32 bit. +pub const R_390_PC32: u32 = 5; +/// 12 bit GOT offset. +pub const R_390_GOT12: u32 = 6; +/// 32 bit GOT offset. +pub const R_390_GOT32: u32 = 7; +/// 32 bit PC relative PLT address. +pub const R_390_PLT32: u32 = 8; +/// Copy symbol at runtime. +pub const R_390_COPY: u32 = 9; +/// Create GOT entry. +pub const R_390_GLOB_DAT: u32 = 10; +/// Create PLT entry. +pub const R_390_JMP_SLOT: u32 = 11; +/// Adjust by program base. +pub const R_390_RELATIVE: u32 = 12; +/// 32 bit offset to GOT. +pub const R_390_GOTOFF32: u32 = 13; +/// 32 bit PC relative offset to GOT. +pub const R_390_GOTPC: u32 = 14; +/// 16 bit GOT offset. +pub const R_390_GOT16: u32 = 15; +/// PC relative 16 bit. +pub const R_390_PC16: u32 = 16; +/// PC relative 16 bit shifted by 1. +pub const R_390_PC16DBL: u32 = 17; +/// 16 bit PC rel. PLT shifted by 1. +pub const R_390_PLT16DBL: u32 = 18; +/// PC relative 32 bit shifted by 1. +pub const R_390_PC32DBL: u32 = 19; +/// 32 bit PC rel. PLT shifted by 1. +pub const R_390_PLT32DBL: u32 = 20; +/// 32 bit PC rel. GOT shifted by 1. +pub const R_390_GOTPCDBL: u32 = 21; +/// Direct 64 bit. +pub const R_390_64: u32 = 22; +/// PC relative 64 bit. +pub const R_390_PC64: u32 = 23; +/// 64 bit GOT offset. +pub const R_390_GOT64: u32 = 24; +/// 64 bit PC relative PLT address. +pub const R_390_PLT64: u32 = 25; +/// 32 bit PC rel. to GOT entry >> 1. +pub const R_390_GOTENT: u32 = 26; +/// 16 bit offset to GOT. +pub const R_390_GOTOFF16: u32 = 27; +/// 64 bit offset to GOT. +pub const R_390_GOTOFF64: u32 = 28; +/// 12 bit offset to jump slot. +pub const R_390_GOTPLT12: u32 = 29; +/// 16 bit offset to jump slot. +pub const R_390_GOTPLT16: u32 = 30; +/// 32 bit offset to jump slot. +pub const R_390_GOTPLT32: u32 = 31; +/// 64 bit offset to jump slot. +pub const R_390_GOTPLT64: u32 = 32; +/// 32 bit rel. offset to jump slot. +pub const R_390_GOTPLTENT: u32 = 33; +/// 16 bit offset from GOT to PLT. +pub const R_390_PLTOFF16: u32 = 34; +/// 32 bit offset from GOT to PLT. +pub const R_390_PLTOFF32: u32 = 35; +/// 16 bit offset from GOT to PLT. +pub const R_390_PLTOFF64: u32 = 36; +/// Tag for load insn in TLS code. +pub const R_390_TLS_LOAD: u32 = 37; +/// Tag for function call in general dynamic TLS code. +pub const R_390_TLS_GDCALL: u32 = 38; +/// Tag for function call in local dynamic TLS code. +pub const R_390_TLS_LDCALL: u32 = 39; +/// Direct 32 bit for general dynamic thread local data. +pub const R_390_TLS_GD32: u32 = 40; +/// Direct 64 bit for general dynamic thread local data. +pub const R_390_TLS_GD64: u32 = 41; +/// 12 bit GOT offset for static TLS block offset. +pub const R_390_TLS_GOTIE12: u32 = 42; +/// 32 bit GOT offset for static TLS block offset. +pub const R_390_TLS_GOTIE32: u32 = 43; +/// 64 bit GOT offset for static TLS block offset. +pub const R_390_TLS_GOTIE64: u32 = 44; +/// Direct 32 bit for local dynamic thread local data in LE code. +pub const R_390_TLS_LDM32: u32 = 45; +/// Direct 64 bit for local dynamic thread local data in LE code. +pub const R_390_TLS_LDM64: u32 = 46; +/// 32 bit address of GOT entry for negated static TLS block offset. +pub const R_390_TLS_IE32: u32 = 47; +/// 64 bit address of GOT entry for negated static TLS block offset. +pub const R_390_TLS_IE64: u32 = 48; +/// 32 bit rel. offset to GOT entry for negated static TLS block offset. +pub const R_390_TLS_IEENT: u32 = 49; +/// 32 bit negated offset relative to static TLS block. +pub const R_390_TLS_LE32: u32 = 50; +/// 64 bit negated offset relative to static TLS block. +pub const R_390_TLS_LE64: u32 = 51; +/// 32 bit offset relative to TLS block. +pub const R_390_TLS_LDO32: u32 = 52; +/// 64 bit offset relative to TLS block. +pub const R_390_TLS_LDO64: u32 = 53; +/// ID of module containing symbol. +pub const R_390_TLS_DTPMOD: u32 = 54; +/// Offset in TLS block. +pub const R_390_TLS_DTPOFF: u32 = 55; +/// Negated offset in static TLS block. +pub const R_390_TLS_TPOFF: u32 = 56; +/// Direct 20 bit. +pub const R_390_20: u32 = 57; +/// 20 bit GOT offset. +pub const R_390_GOT20: u32 = 58; +/// 20 bit offset to jump slot. +pub const R_390_GOTPLT20: u32 = 59; +/// 20 bit GOT offset for static TLS block offset. +pub const R_390_TLS_GOTIE20: u32 = 60; +/// STT_GNU_IFUNC relocation. +pub const R_390_IRELATIVE: u32 = 61; + +// CRIS values `Rel*::r_type`. +#[allow(missing_docs)] +pub const R_CRIS_NONE: u32 = 0; +#[allow(missing_docs)] +pub const R_CRIS_8: u32 = 1; +#[allow(missing_docs)] +pub const R_CRIS_16: u32 = 2; +#[allow(missing_docs)] +pub const R_CRIS_32: u32 = 3; +#[allow(missing_docs)] +pub const R_CRIS_8_PCREL: u32 = 4; +#[allow(missing_docs)] +pub const R_CRIS_16_PCREL: u32 = 5; +#[allow(missing_docs)] +pub const R_CRIS_32_PCREL: u32 = 6; +#[allow(missing_docs)] +pub const R_CRIS_GNU_VTINHERIT: u32 = 7; +#[allow(missing_docs)] +pub const R_CRIS_GNU_VTENTRY: u32 = 8; +#[allow(missing_docs)] +pub const R_CRIS_COPY: u32 = 9; +#[allow(missing_docs)] +pub const R_CRIS_GLOB_DAT: u32 = 10; +#[allow(missing_docs)] +pub const R_CRIS_JUMP_SLOT: u32 = 11; +#[allow(missing_docs)] +pub const R_CRIS_RELATIVE: u32 = 12; +#[allow(missing_docs)] +pub const R_CRIS_16_GOT: u32 = 13; +#[allow(missing_docs)] +pub const R_CRIS_32_GOT: u32 = 14; +#[allow(missing_docs)] +pub const R_CRIS_16_GOTPLT: u32 = 15; +#[allow(missing_docs)] +pub const R_CRIS_32_GOTPLT: u32 = 16; +#[allow(missing_docs)] +pub const R_CRIS_32_GOTREL: u32 = 17; +#[allow(missing_docs)] +pub const R_CRIS_32_PLT_GOTREL: u32 = 18; +#[allow(missing_docs)] +pub const R_CRIS_32_PLT_PCREL: u32 = 19; + +// AMD x86-64 values `Rel*::r_type`. +/// No reloc +pub const R_X86_64_NONE: u32 = 0; +/// Direct 64 bit +pub const R_X86_64_64: u32 = 1; +/// PC relative 32 bit signed +pub const R_X86_64_PC32: u32 = 2; +/// 32 bit GOT entry +pub const R_X86_64_GOT32: u32 = 3; +/// 32 bit PLT address +pub const R_X86_64_PLT32: u32 = 4; +/// Copy symbol at runtime +pub const R_X86_64_COPY: u32 = 5; +/// Create GOT entry +pub const R_X86_64_GLOB_DAT: u32 = 6; +/// Create PLT entry +pub const R_X86_64_JUMP_SLOT: u32 = 7; +/// Adjust by program base +pub const R_X86_64_RELATIVE: u32 = 8; +/// 32 bit signed PC relative offset to GOT +pub const R_X86_64_GOTPCREL: u32 = 9; +/// Direct 32 bit zero extended +pub const R_X86_64_32: u32 = 10; +/// Direct 32 bit sign extended +pub const R_X86_64_32S: u32 = 11; +/// Direct 16 bit zero extended +pub const R_X86_64_16: u32 = 12; +/// 16 bit sign extended pc relative +pub const R_X86_64_PC16: u32 = 13; +/// Direct 8 bit sign extended +pub const R_X86_64_8: u32 = 14; +/// 8 bit sign extended pc relative +pub const R_X86_64_PC8: u32 = 15; +/// ID of module containing symbol +pub const R_X86_64_DTPMOD64: u32 = 16; +/// Offset in module's TLS block +pub const R_X86_64_DTPOFF64: u32 = 17; +/// Offset in initial TLS block +pub const R_X86_64_TPOFF64: u32 = 18; +/// 32 bit signed PC relative offset to two GOT entries for GD symbol +pub const R_X86_64_TLSGD: u32 = 19; +/// 32 bit signed PC relative offset to two GOT entries for LD symbol +pub const R_X86_64_TLSLD: u32 = 20; +/// Offset in TLS block +pub const R_X86_64_DTPOFF32: u32 = 21; +/// 32 bit signed PC relative offset to GOT entry for IE symbol +pub const R_X86_64_GOTTPOFF: u32 = 22; +/// Offset in initial TLS block +pub const R_X86_64_TPOFF32: u32 = 23; +/// PC relative 64 bit +pub const R_X86_64_PC64: u32 = 24; +/// 64 bit offset to GOT +pub const R_X86_64_GOTOFF64: u32 = 25; +/// 32 bit signed pc relative offset to GOT +pub const R_X86_64_GOTPC32: u32 = 26; +/// 64-bit GOT entry offset +pub const R_X86_64_GOT64: u32 = 27; +/// 64-bit PC relative offset to GOT entry +pub const R_X86_64_GOTPCREL64: u32 = 28; +/// 64-bit PC relative offset to GOT +pub const R_X86_64_GOTPC64: u32 = 29; +/// like GOT64, says PLT entry needed +pub const R_X86_64_GOTPLT64: u32 = 30; +/// 64-bit GOT relative offset to PLT entry +pub const R_X86_64_PLTOFF64: u32 = 31; +/// Size of symbol plus 32-bit addend +pub const R_X86_64_SIZE32: u32 = 32; +/// Size of symbol plus 64-bit addend +pub const R_X86_64_SIZE64: u32 = 33; +/// GOT offset for TLS descriptor. +pub const R_X86_64_GOTPC32_TLSDESC: u32 = 34; +/// Marker for call through TLS descriptor. +pub const R_X86_64_TLSDESC_CALL: u32 = 35; +/// TLS descriptor. +pub const R_X86_64_TLSDESC: u32 = 36; +/// Adjust indirectly by program base +pub const R_X86_64_IRELATIVE: u32 = 37; +/// 64-bit adjust by program base +pub const R_X86_64_RELATIVE64: u32 = 38; +// 39 Reserved was R_X86_64_PC32_BND +// 40 Reserved was R_X86_64_PLT32_BND +/// Load from 32 bit signed pc relative offset to GOT entry without REX prefix, relaxable. +pub const R_X86_64_GOTPCRELX: u32 = 41; +/// Load from 32 bit signed pc relative offset to GOT entry with REX prefix, relaxable. +pub const R_X86_64_REX_GOTPCRELX: u32 = 42; + +// AMD x86-64 values `SectionHeader*::sh_type`. +/// Unwind information. +pub const SHT_X86_64_UNWIND: u32 = 0x7000_0001; + +// AM33 values `Rel*::r_type`. +/// No reloc. +pub const R_MN10300_NONE: u32 = 0; +/// Direct 32 bit. +pub const R_MN10300_32: u32 = 1; +/// Direct 16 bit. +pub const R_MN10300_16: u32 = 2; +/// Direct 8 bit. +pub const R_MN10300_8: u32 = 3; +/// PC-relative 32-bit. +pub const R_MN10300_PCREL32: u32 = 4; +/// PC-relative 16-bit signed. +pub const R_MN10300_PCREL16: u32 = 5; +/// PC-relative 8-bit signed. +pub const R_MN10300_PCREL8: u32 = 6; +/// Ancient C++ vtable garbage... +pub const R_MN10300_GNU_VTINHERIT: u32 = 7; +/// ... collection annotation. +pub const R_MN10300_GNU_VTENTRY: u32 = 8; +/// Direct 24 bit. +pub const R_MN10300_24: u32 = 9; +/// 32-bit PCrel offset to GOT. +pub const R_MN10300_GOTPC32: u32 = 10; +/// 16-bit PCrel offset to GOT. +pub const R_MN10300_GOTPC16: u32 = 11; +/// 32-bit offset from GOT. +pub const R_MN10300_GOTOFF32: u32 = 12; +/// 24-bit offset from GOT. +pub const R_MN10300_GOTOFF24: u32 = 13; +/// 16-bit offset from GOT. +pub const R_MN10300_GOTOFF16: u32 = 14; +/// 32-bit PCrel to PLT entry. +pub const R_MN10300_PLT32: u32 = 15; +/// 16-bit PCrel to PLT entry. +pub const R_MN10300_PLT16: u32 = 16; +/// 32-bit offset to GOT entry. +pub const R_MN10300_GOT32: u32 = 17; +/// 24-bit offset to GOT entry. +pub const R_MN10300_GOT24: u32 = 18; +/// 16-bit offset to GOT entry. +pub const R_MN10300_GOT16: u32 = 19; +/// Copy symbol at runtime. +pub const R_MN10300_COPY: u32 = 20; +/// Create GOT entry. +pub const R_MN10300_GLOB_DAT: u32 = 21; +/// Create PLT entry. +pub const R_MN10300_JMP_SLOT: u32 = 22; +/// Adjust by program base. +pub const R_MN10300_RELATIVE: u32 = 23; +/// 32-bit offset for global dynamic. +pub const R_MN10300_TLS_GD: u32 = 24; +/// 32-bit offset for local dynamic. +pub const R_MN10300_TLS_LD: u32 = 25; +/// Module-relative offset. +pub const R_MN10300_TLS_LDO: u32 = 26; +/// GOT offset for static TLS block offset. +pub const R_MN10300_TLS_GOTIE: u32 = 27; +/// GOT address for static TLS block offset. +pub const R_MN10300_TLS_IE: u32 = 28; +/// Offset relative to static TLS block. +pub const R_MN10300_TLS_LE: u32 = 29; +/// ID of module containing symbol. +pub const R_MN10300_TLS_DTPMOD: u32 = 30; +/// Offset in module TLS block. +pub const R_MN10300_TLS_DTPOFF: u32 = 31; +/// Offset in static TLS block. +pub const R_MN10300_TLS_TPOFF: u32 = 32; +/// Adjustment for next reloc as needed by linker relaxation. +pub const R_MN10300_SYM_DIFF: u32 = 33; +/// Alignment requirement for linker relaxation. +pub const R_MN10300_ALIGN: u32 = 34; + +// M32R values `Rel32::r_type`. +/// No reloc. +pub const R_M32R_NONE: u32 = 0; +/// Direct 16 bit. +pub const R_M32R_16: u32 = 1; +/// Direct 32 bit. +pub const R_M32R_32: u32 = 2; +/// Direct 24 bit. +pub const R_M32R_24: u32 = 3; +/// PC relative 10 bit shifted. +pub const R_M32R_10_PCREL: u32 = 4; +/// PC relative 18 bit shifted. +pub const R_M32R_18_PCREL: u32 = 5; +/// PC relative 26 bit shifted. +pub const R_M32R_26_PCREL: u32 = 6; +/// High 16 bit with unsigned low. +pub const R_M32R_HI16_ULO: u32 = 7; +/// High 16 bit with signed low. +pub const R_M32R_HI16_SLO: u32 = 8; +/// Low 16 bit. +pub const R_M32R_LO16: u32 = 9; +/// 16 bit offset in SDA. +pub const R_M32R_SDA16: u32 = 10; +#[allow(missing_docs)] +pub const R_M32R_GNU_VTINHERIT: u32 = 11; +#[allow(missing_docs)] +pub const R_M32R_GNU_VTENTRY: u32 = 12; +// M32R values `Rela32::r_type`. +/// Direct 16 bit. +pub const R_M32R_16_RELA: u32 = 33; +/// Direct 32 bit. +pub const R_M32R_32_RELA: u32 = 34; +/// Direct 24 bit. +pub const R_M32R_24_RELA: u32 = 35; +/// PC relative 10 bit shifted. +pub const R_M32R_10_PCREL_RELA: u32 = 36; +/// PC relative 18 bit shifted. +pub const R_M32R_18_PCREL_RELA: u32 = 37; +/// PC relative 26 bit shifted. +pub const R_M32R_26_PCREL_RELA: u32 = 38; +/// High 16 bit with unsigned low +pub const R_M32R_HI16_ULO_RELA: u32 = 39; +/// High 16 bit with signed low +pub const R_M32R_HI16_SLO_RELA: u32 = 40; +/// Low 16 bit +pub const R_M32R_LO16_RELA: u32 = 41; +/// 16 bit offset in SDA +pub const R_M32R_SDA16_RELA: u32 = 42; +#[allow(missing_docs)] +pub const R_M32R_RELA_GNU_VTINHERIT: u32 = 43; +#[allow(missing_docs)] +pub const R_M32R_RELA_GNU_VTENTRY: u32 = 44; +/// PC relative 32 bit. +pub const R_M32R_REL32: u32 = 45; + +/// 24 bit GOT entry +pub const R_M32R_GOT24: u32 = 48; +/// 26 bit PC relative to PLT shifted +pub const R_M32R_26_PLTREL: u32 = 49; +/// Copy symbol at runtime +pub const R_M32R_COPY: u32 = 50; +/// Create GOT entry +pub const R_M32R_GLOB_DAT: u32 = 51; +/// Create PLT entry +pub const R_M32R_JMP_SLOT: u32 = 52; +/// Adjust by program base +pub const R_M32R_RELATIVE: u32 = 53; +/// 24 bit offset to GOT +pub const R_M32R_GOTOFF: u32 = 54; +/// 24 bit PC relative offset to GOT +pub const R_M32R_GOTPC24: u32 = 55; +/// High 16 bit GOT entry with unsigned low +pub const R_M32R_GOT16_HI_ULO: u32 = 56; +/// High 16 bit GOT entry with signed low +pub const R_M32R_GOT16_HI_SLO: u32 = 57; +/// Low 16 bit GOT entry +pub const R_M32R_GOT16_LO: u32 = 58; +/// High 16 bit PC relative offset to GOT with unsigned low +pub const R_M32R_GOTPC_HI_ULO: u32 = 59; +/// High 16 bit PC relative offset to GOT with signed low +pub const R_M32R_GOTPC_HI_SLO: u32 = 60; +/// Low 16 bit PC relative offset to GOT +pub const R_M32R_GOTPC_LO: u32 = 61; +/// High 16 bit offset to GOT with unsigned low +pub const R_M32R_GOTOFF_HI_ULO: u32 = 62; +/// High 16 bit offset to GOT with signed low +pub const R_M32R_GOTOFF_HI_SLO: u32 = 63; +/// Low 16 bit offset to GOT +pub const R_M32R_GOTOFF_LO: u32 = 64; +/// Keep this the last entry. +pub const R_M32R_NUM: u32 = 256; + +// MicroBlaze values `Rel*::r_type`. +/// No reloc. +pub const R_MICROBLAZE_NONE: u32 = 0; +/// Direct 32 bit. +pub const R_MICROBLAZE_32: u32 = 1; +/// PC relative 32 bit. +pub const R_MICROBLAZE_32_PCREL: u32 = 2; +/// PC relative 64 bit. +pub const R_MICROBLAZE_64_PCREL: u32 = 3; +/// Low 16 bits of PCREL32. +pub const R_MICROBLAZE_32_PCREL_LO: u32 = 4; +/// Direct 64 bit. +pub const R_MICROBLAZE_64: u32 = 5; +/// Low 16 bit. +pub const R_MICROBLAZE_32_LO: u32 = 6; +/// Read-only small data area. +pub const R_MICROBLAZE_SRO32: u32 = 7; +/// Read-write small data area. +pub const R_MICROBLAZE_SRW32: u32 = 8; +/// No reloc. +pub const R_MICROBLAZE_64_NONE: u32 = 9; +/// Symbol Op Symbol relocation. +pub const R_MICROBLAZE_32_SYM_OP_SYM: u32 = 10; +/// GNU C++ vtable hierarchy. +pub const R_MICROBLAZE_GNU_VTINHERIT: u32 = 11; +/// GNU C++ vtable member usage. +pub const R_MICROBLAZE_GNU_VTENTRY: u32 = 12; +/// PC-relative GOT offset. +pub const R_MICROBLAZE_GOTPC_64: u32 = 13; +/// GOT entry offset. +pub const R_MICROBLAZE_GOT_64: u32 = 14; +/// PLT offset (PC-relative). +pub const R_MICROBLAZE_PLT_64: u32 = 15; +/// Adjust by program base. +pub const R_MICROBLAZE_REL: u32 = 16; +/// Create PLT entry. +pub const R_MICROBLAZE_JUMP_SLOT: u32 = 17; +/// Create GOT entry. +pub const R_MICROBLAZE_GLOB_DAT: u32 = 18; +/// 64 bit offset to GOT. +pub const R_MICROBLAZE_GOTOFF_64: u32 = 19; +/// 32 bit offset to GOT. +pub const R_MICROBLAZE_GOTOFF_32: u32 = 20; +/// Runtime copy. +pub const R_MICROBLAZE_COPY: u32 = 21; +/// TLS Reloc. +pub const R_MICROBLAZE_TLS: u32 = 22; +/// TLS General Dynamic. +pub const R_MICROBLAZE_TLSGD: u32 = 23; +/// TLS Local Dynamic. +pub const R_MICROBLAZE_TLSLD: u32 = 24; +/// TLS Module ID. +pub const R_MICROBLAZE_TLSDTPMOD32: u32 = 25; +/// TLS Offset Within TLS Block. +pub const R_MICROBLAZE_TLSDTPREL32: u32 = 26; +/// TLS Offset Within TLS Block. +pub const R_MICROBLAZE_TLSDTPREL64: u32 = 27; +/// TLS Offset From Thread Pointer. +pub const R_MICROBLAZE_TLSGOTTPREL32: u32 = 28; +/// TLS Offset From Thread Pointer. +pub const R_MICROBLAZE_TLSTPREL32: u32 = 29; + +// Nios II values `Dyn::d_tag`. +/// Address of _gp. +pub const DT_NIOS2_GP: u32 = 0x7000_0002; + +// Nios II values `Rel*::r_type`. +/// No reloc. +pub const R_NIOS2_NONE: u32 = 0; +/// Direct signed 16 bit. +pub const R_NIOS2_S16: u32 = 1; +/// Direct unsigned 16 bit. +pub const R_NIOS2_U16: u32 = 2; +/// PC relative 16 bit. +pub const R_NIOS2_PCREL16: u32 = 3; +/// Direct call. +pub const R_NIOS2_CALL26: u32 = 4; +/// 5 bit constant expression. +pub const R_NIOS2_IMM5: u32 = 5; +/// 5 bit expression, shift 22. +pub const R_NIOS2_CACHE_OPX: u32 = 6; +/// 6 bit constant expression. +pub const R_NIOS2_IMM6: u32 = 7; +/// 8 bit constant expression. +pub const R_NIOS2_IMM8: u32 = 8; +/// High 16 bit. +pub const R_NIOS2_HI16: u32 = 9; +/// Low 16 bit. +pub const R_NIOS2_LO16: u32 = 10; +/// High 16 bit, adjusted. +pub const R_NIOS2_HIADJ16: u32 = 11; +/// 32 bit symbol value + addend. +pub const R_NIOS2_BFD_RELOC_32: u32 = 12; +/// 16 bit symbol value + addend. +pub const R_NIOS2_BFD_RELOC_16: u32 = 13; +/// 8 bit symbol value + addend. +pub const R_NIOS2_BFD_RELOC_8: u32 = 14; +/// 16 bit GP pointer offset. +pub const R_NIOS2_GPREL: u32 = 15; +/// GNU C++ vtable hierarchy. +pub const R_NIOS2_GNU_VTINHERIT: u32 = 16; +/// GNU C++ vtable member usage. +pub const R_NIOS2_GNU_VTENTRY: u32 = 17; +/// Unconditional branch. +pub const R_NIOS2_UJMP: u32 = 18; +/// Conditional branch. +pub const R_NIOS2_CJMP: u32 = 19; +/// Indirect call through register. +pub const R_NIOS2_CALLR: u32 = 20; +/// Alignment requirement for linker relaxation. +pub const R_NIOS2_ALIGN: u32 = 21; +/// 16 bit GOT entry. +pub const R_NIOS2_GOT16: u32 = 22; +/// 16 bit GOT entry for function. +pub const R_NIOS2_CALL16: u32 = 23; +/// %lo of offset to GOT pointer. +pub const R_NIOS2_GOTOFF_LO: u32 = 24; +/// %hiadj of offset to GOT pointer. +pub const R_NIOS2_GOTOFF_HA: u32 = 25; +/// %lo of PC relative offset. +pub const R_NIOS2_PCREL_LO: u32 = 26; +/// %hiadj of PC relative offset. +pub const R_NIOS2_PCREL_HA: u32 = 27; +/// 16 bit GOT offset for TLS GD. +pub const R_NIOS2_TLS_GD16: u32 = 28; +/// 16 bit GOT offset for TLS LDM. +pub const R_NIOS2_TLS_LDM16: u32 = 29; +/// 16 bit module relative offset. +pub const R_NIOS2_TLS_LDO16: u32 = 30; +/// 16 bit GOT offset for TLS IE. +pub const R_NIOS2_TLS_IE16: u32 = 31; +/// 16 bit LE TP-relative offset. +pub const R_NIOS2_TLS_LE16: u32 = 32; +/// Module number. +pub const R_NIOS2_TLS_DTPMOD: u32 = 33; +/// Module-relative offset. +pub const R_NIOS2_TLS_DTPREL: u32 = 34; +/// TP-relative offset. +pub const R_NIOS2_TLS_TPREL: u32 = 35; +/// Copy symbol at runtime. +pub const R_NIOS2_COPY: u32 = 36; +/// Create GOT entry. +pub const R_NIOS2_GLOB_DAT: u32 = 37; +/// Create PLT entry. +pub const R_NIOS2_JUMP_SLOT: u32 = 38; +/// Adjust by program base. +pub const R_NIOS2_RELATIVE: u32 = 39; +/// 16 bit offset to GOT pointer. +pub const R_NIOS2_GOTOFF: u32 = 40; +/// Direct call in .noat section. +pub const R_NIOS2_CALL26_NOAT: u32 = 41; +/// %lo() of GOT entry. +pub const R_NIOS2_GOT_LO: u32 = 42; +/// %hiadj() of GOT entry. +pub const R_NIOS2_GOT_HA: u32 = 43; +/// %lo() of function GOT entry. +pub const R_NIOS2_CALL_LO: u32 = 44; +/// %hiadj() of function GOT entry. +pub const R_NIOS2_CALL_HA: u32 = 45; + +// TILEPro values `Rel*::r_type`. +/// No reloc +pub const R_TILEPRO_NONE: u32 = 0; +/// Direct 32 bit +pub const R_TILEPRO_32: u32 = 1; +/// Direct 16 bit +pub const R_TILEPRO_16: u32 = 2; +/// Direct 8 bit +pub const R_TILEPRO_8: u32 = 3; +/// PC relative 32 bit +pub const R_TILEPRO_32_PCREL: u32 = 4; +/// PC relative 16 bit +pub const R_TILEPRO_16_PCREL: u32 = 5; +/// PC relative 8 bit +pub const R_TILEPRO_8_PCREL: u32 = 6; +/// Low 16 bit +pub const R_TILEPRO_LO16: u32 = 7; +/// High 16 bit +pub const R_TILEPRO_HI16: u32 = 8; +/// High 16 bit, adjusted +pub const R_TILEPRO_HA16: u32 = 9; +/// Copy relocation +pub const R_TILEPRO_COPY: u32 = 10; +/// Create GOT entry +pub const R_TILEPRO_GLOB_DAT: u32 = 11; +/// Create PLT entry +pub const R_TILEPRO_JMP_SLOT: u32 = 12; +/// Adjust by program base +pub const R_TILEPRO_RELATIVE: u32 = 13; +/// X1 pipe branch offset +pub const R_TILEPRO_BROFF_X1: u32 = 14; +/// X1 pipe jump offset +pub const R_TILEPRO_JOFFLONG_X1: u32 = 15; +/// X1 pipe jump offset to PLT +pub const R_TILEPRO_JOFFLONG_X1_PLT: u32 = 16; +/// X0 pipe 8-bit +pub const R_TILEPRO_IMM8_X0: u32 = 17; +/// Y0 pipe 8-bit +pub const R_TILEPRO_IMM8_Y0: u32 = 18; +/// X1 pipe 8-bit +pub const R_TILEPRO_IMM8_X1: u32 = 19; +/// Y1 pipe 8-bit +pub const R_TILEPRO_IMM8_Y1: u32 = 20; +/// X1 pipe mtspr +pub const R_TILEPRO_MT_IMM15_X1: u32 = 21; +/// X1 pipe mfspr +pub const R_TILEPRO_MF_IMM15_X1: u32 = 22; +/// X0 pipe 16-bit +pub const R_TILEPRO_IMM16_X0: u32 = 23; +/// X1 pipe 16-bit +pub const R_TILEPRO_IMM16_X1: u32 = 24; +/// X0 pipe low 16-bit +pub const R_TILEPRO_IMM16_X0_LO: u32 = 25; +/// X1 pipe low 16-bit +pub const R_TILEPRO_IMM16_X1_LO: u32 = 26; +/// X0 pipe high 16-bit +pub const R_TILEPRO_IMM16_X0_HI: u32 = 27; +/// X1 pipe high 16-bit +pub const R_TILEPRO_IMM16_X1_HI: u32 = 28; +/// X0 pipe high 16-bit, adjusted +pub const R_TILEPRO_IMM16_X0_HA: u32 = 29; +/// X1 pipe high 16-bit, adjusted +pub const R_TILEPRO_IMM16_X1_HA: u32 = 30; +/// X0 pipe PC relative 16 bit +pub const R_TILEPRO_IMM16_X0_PCREL: u32 = 31; +/// X1 pipe PC relative 16 bit +pub const R_TILEPRO_IMM16_X1_PCREL: u32 = 32; +/// X0 pipe PC relative low 16 bit +pub const R_TILEPRO_IMM16_X0_LO_PCREL: u32 = 33; +/// X1 pipe PC relative low 16 bit +pub const R_TILEPRO_IMM16_X1_LO_PCREL: u32 = 34; +/// X0 pipe PC relative high 16 bit +pub const R_TILEPRO_IMM16_X0_HI_PCREL: u32 = 35; +/// X1 pipe PC relative high 16 bit +pub const R_TILEPRO_IMM16_X1_HI_PCREL: u32 = 36; +/// X0 pipe PC relative ha() 16 bit +pub const R_TILEPRO_IMM16_X0_HA_PCREL: u32 = 37; +/// X1 pipe PC relative ha() 16 bit +pub const R_TILEPRO_IMM16_X1_HA_PCREL: u32 = 38; +/// X0 pipe 16-bit GOT offset +pub const R_TILEPRO_IMM16_X0_GOT: u32 = 39; +/// X1 pipe 16-bit GOT offset +pub const R_TILEPRO_IMM16_X1_GOT: u32 = 40; +/// X0 pipe low 16-bit GOT offset +pub const R_TILEPRO_IMM16_X0_GOT_LO: u32 = 41; +/// X1 pipe low 16-bit GOT offset +pub const R_TILEPRO_IMM16_X1_GOT_LO: u32 = 42; +/// X0 pipe high 16-bit GOT offset +pub const R_TILEPRO_IMM16_X0_GOT_HI: u32 = 43; +/// X1 pipe high 16-bit GOT offset +pub const R_TILEPRO_IMM16_X1_GOT_HI: u32 = 44; +/// X0 pipe ha() 16-bit GOT offset +pub const R_TILEPRO_IMM16_X0_GOT_HA: u32 = 45; +/// X1 pipe ha() 16-bit GOT offset +pub const R_TILEPRO_IMM16_X1_GOT_HA: u32 = 46; +/// X0 pipe mm "start" +pub const R_TILEPRO_MMSTART_X0: u32 = 47; +/// X0 pipe mm "end" +pub const R_TILEPRO_MMEND_X0: u32 = 48; +/// X1 pipe mm "start" +pub const R_TILEPRO_MMSTART_X1: u32 = 49; +/// X1 pipe mm "end" +pub const R_TILEPRO_MMEND_X1: u32 = 50; +/// X0 pipe shift amount +pub const R_TILEPRO_SHAMT_X0: u32 = 51; +/// X1 pipe shift amount +pub const R_TILEPRO_SHAMT_X1: u32 = 52; +/// Y0 pipe shift amount +pub const R_TILEPRO_SHAMT_Y0: u32 = 53; +/// Y1 pipe shift amount +pub const R_TILEPRO_SHAMT_Y1: u32 = 54; +/// X1 pipe destination 8-bit +pub const R_TILEPRO_DEST_IMM8_X1: u32 = 55; +// Relocs 56-59 are currently not defined. +/// "jal" for TLS GD +pub const R_TILEPRO_TLS_GD_CALL: u32 = 60; +/// X0 pipe "addi" for TLS GD +pub const R_TILEPRO_IMM8_X0_TLS_GD_ADD: u32 = 61; +/// X1 pipe "addi" for TLS GD +pub const R_TILEPRO_IMM8_X1_TLS_GD_ADD: u32 = 62; +/// Y0 pipe "addi" for TLS GD +pub const R_TILEPRO_IMM8_Y0_TLS_GD_ADD: u32 = 63; +/// Y1 pipe "addi" for TLS GD +pub const R_TILEPRO_IMM8_Y1_TLS_GD_ADD: u32 = 64; +/// "lw_tls" for TLS IE +pub const R_TILEPRO_TLS_IE_LOAD: u32 = 65; +/// X0 pipe 16-bit TLS GD offset +pub const R_TILEPRO_IMM16_X0_TLS_GD: u32 = 66; +/// X1 pipe 16-bit TLS GD offset +pub const R_TILEPRO_IMM16_X1_TLS_GD: u32 = 67; +/// X0 pipe low 16-bit TLS GD offset +pub const R_TILEPRO_IMM16_X0_TLS_GD_LO: u32 = 68; +/// X1 pipe low 16-bit TLS GD offset +pub const R_TILEPRO_IMM16_X1_TLS_GD_LO: u32 = 69; +/// X0 pipe high 16-bit TLS GD offset +pub const R_TILEPRO_IMM16_X0_TLS_GD_HI: u32 = 70; +/// X1 pipe high 16-bit TLS GD offset +pub const R_TILEPRO_IMM16_X1_TLS_GD_HI: u32 = 71; +/// X0 pipe ha() 16-bit TLS GD offset +pub const R_TILEPRO_IMM16_X0_TLS_GD_HA: u32 = 72; +/// X1 pipe ha() 16-bit TLS GD offset +pub const R_TILEPRO_IMM16_X1_TLS_GD_HA: u32 = 73; +/// X0 pipe 16-bit TLS IE offset +pub const R_TILEPRO_IMM16_X0_TLS_IE: u32 = 74; +/// X1 pipe 16-bit TLS IE offset +pub const R_TILEPRO_IMM16_X1_TLS_IE: u32 = 75; +/// X0 pipe low 16-bit TLS IE offset +pub const R_TILEPRO_IMM16_X0_TLS_IE_LO: u32 = 76; +/// X1 pipe low 16-bit TLS IE offset +pub const R_TILEPRO_IMM16_X1_TLS_IE_LO: u32 = 77; +/// X0 pipe high 16-bit TLS IE offset +pub const R_TILEPRO_IMM16_X0_TLS_IE_HI: u32 = 78; +/// X1 pipe high 16-bit TLS IE offset +pub const R_TILEPRO_IMM16_X1_TLS_IE_HI: u32 = 79; +/// X0 pipe ha() 16-bit TLS IE offset +pub const R_TILEPRO_IMM16_X0_TLS_IE_HA: u32 = 80; +/// X1 pipe ha() 16-bit TLS IE offset +pub const R_TILEPRO_IMM16_X1_TLS_IE_HA: u32 = 81; +/// ID of module containing symbol +pub const R_TILEPRO_TLS_DTPMOD32: u32 = 82; +/// Offset in TLS block +pub const R_TILEPRO_TLS_DTPOFF32: u32 = 83; +/// Offset in static TLS block +pub const R_TILEPRO_TLS_TPOFF32: u32 = 84; +/// X0 pipe 16-bit TLS LE offset +pub const R_TILEPRO_IMM16_X0_TLS_LE: u32 = 85; +/// X1 pipe 16-bit TLS LE offset +pub const R_TILEPRO_IMM16_X1_TLS_LE: u32 = 86; +/// X0 pipe low 16-bit TLS LE offset +pub const R_TILEPRO_IMM16_X0_TLS_LE_LO: u32 = 87; +/// X1 pipe low 16-bit TLS LE offset +pub const R_TILEPRO_IMM16_X1_TLS_LE_LO: u32 = 88; +/// X0 pipe high 16-bit TLS LE offset +pub const R_TILEPRO_IMM16_X0_TLS_LE_HI: u32 = 89; +/// X1 pipe high 16-bit TLS LE offset +pub const R_TILEPRO_IMM16_X1_TLS_LE_HI: u32 = 90; +/// X0 pipe ha() 16-bit TLS LE offset +pub const R_TILEPRO_IMM16_X0_TLS_LE_HA: u32 = 91; +/// X1 pipe ha() 16-bit TLS LE offset +pub const R_TILEPRO_IMM16_X1_TLS_LE_HA: u32 = 92; + +/// GNU C++ vtable hierarchy +pub const R_TILEPRO_GNU_VTINHERIT: u32 = 128; +/// GNU C++ vtable member usage +pub const R_TILEPRO_GNU_VTENTRY: u32 = 129; + +// TILE-Gx values `Rel*::r_type`. +/// No reloc +pub const R_TILEGX_NONE: u32 = 0; +/// Direct 64 bit +pub const R_TILEGX_64: u32 = 1; +/// Direct 32 bit +pub const R_TILEGX_32: u32 = 2; +/// Direct 16 bit +pub const R_TILEGX_16: u32 = 3; +/// Direct 8 bit +pub const R_TILEGX_8: u32 = 4; +/// PC relative 64 bit +pub const R_TILEGX_64_PCREL: u32 = 5; +/// PC relative 32 bit +pub const R_TILEGX_32_PCREL: u32 = 6; +/// PC relative 16 bit +pub const R_TILEGX_16_PCREL: u32 = 7; +/// PC relative 8 bit +pub const R_TILEGX_8_PCREL: u32 = 8; +/// hword 0 16-bit +pub const R_TILEGX_HW0: u32 = 9; +/// hword 1 16-bit +pub const R_TILEGX_HW1: u32 = 10; +/// hword 2 16-bit +pub const R_TILEGX_HW2: u32 = 11; +/// hword 3 16-bit +pub const R_TILEGX_HW3: u32 = 12; +/// last hword 0 16-bit +pub const R_TILEGX_HW0_LAST: u32 = 13; +/// last hword 1 16-bit +pub const R_TILEGX_HW1_LAST: u32 = 14; +/// last hword 2 16-bit +pub const R_TILEGX_HW2_LAST: u32 = 15; +/// Copy relocation +pub const R_TILEGX_COPY: u32 = 16; +/// Create GOT entry +pub const R_TILEGX_GLOB_DAT: u32 = 17; +/// Create PLT entry +pub const R_TILEGX_JMP_SLOT: u32 = 18; +/// Adjust by program base +pub const R_TILEGX_RELATIVE: u32 = 19; +/// X1 pipe branch offset +pub const R_TILEGX_BROFF_X1: u32 = 20; +/// X1 pipe jump offset +pub const R_TILEGX_JUMPOFF_X1: u32 = 21; +/// X1 pipe jump offset to PLT +pub const R_TILEGX_JUMPOFF_X1_PLT: u32 = 22; +/// X0 pipe 8-bit +pub const R_TILEGX_IMM8_X0: u32 = 23; +/// Y0 pipe 8-bit +pub const R_TILEGX_IMM8_Y0: u32 = 24; +/// X1 pipe 8-bit +pub const R_TILEGX_IMM8_X1: u32 = 25; +/// Y1 pipe 8-bit +pub const R_TILEGX_IMM8_Y1: u32 = 26; +/// X1 pipe destination 8-bit +pub const R_TILEGX_DEST_IMM8_X1: u32 = 27; +/// X1 pipe mtspr +pub const R_TILEGX_MT_IMM14_X1: u32 = 28; +/// X1 pipe mfspr +pub const R_TILEGX_MF_IMM14_X1: u32 = 29; +/// X0 pipe mm "start" +pub const R_TILEGX_MMSTART_X0: u32 = 30; +/// X0 pipe mm "end" +pub const R_TILEGX_MMEND_X0: u32 = 31; +/// X0 pipe shift amount +pub const R_TILEGX_SHAMT_X0: u32 = 32; +/// X1 pipe shift amount +pub const R_TILEGX_SHAMT_X1: u32 = 33; +/// Y0 pipe shift amount +pub const R_TILEGX_SHAMT_Y0: u32 = 34; +/// Y1 pipe shift amount +pub const R_TILEGX_SHAMT_Y1: u32 = 35; +/// X0 pipe hword 0 +pub const R_TILEGX_IMM16_X0_HW0: u32 = 36; +/// X1 pipe hword 0 +pub const R_TILEGX_IMM16_X1_HW0: u32 = 37; +/// X0 pipe hword 1 +pub const R_TILEGX_IMM16_X0_HW1: u32 = 38; +/// X1 pipe hword 1 +pub const R_TILEGX_IMM16_X1_HW1: u32 = 39; +/// X0 pipe hword 2 +pub const R_TILEGX_IMM16_X0_HW2: u32 = 40; +/// X1 pipe hword 2 +pub const R_TILEGX_IMM16_X1_HW2: u32 = 41; +/// X0 pipe hword 3 +pub const R_TILEGX_IMM16_X0_HW3: u32 = 42; +/// X1 pipe hword 3 +pub const R_TILEGX_IMM16_X1_HW3: u32 = 43; +/// X0 pipe last hword 0 +pub const R_TILEGX_IMM16_X0_HW0_LAST: u32 = 44; +/// X1 pipe last hword 0 +pub const R_TILEGX_IMM16_X1_HW0_LAST: u32 = 45; +/// X0 pipe last hword 1 +pub const R_TILEGX_IMM16_X0_HW1_LAST: u32 = 46; +/// X1 pipe last hword 1 +pub const R_TILEGX_IMM16_X1_HW1_LAST: u32 = 47; +/// X0 pipe last hword 2 +pub const R_TILEGX_IMM16_X0_HW2_LAST: u32 = 48; +/// X1 pipe last hword 2 +pub const R_TILEGX_IMM16_X1_HW2_LAST: u32 = 49; +/// X0 pipe PC relative hword 0 +pub const R_TILEGX_IMM16_X0_HW0_PCREL: u32 = 50; +/// X1 pipe PC relative hword 0 +pub const R_TILEGX_IMM16_X1_HW0_PCREL: u32 = 51; +/// X0 pipe PC relative hword 1 +pub const R_TILEGX_IMM16_X0_HW1_PCREL: u32 = 52; +/// X1 pipe PC relative hword 1 +pub const R_TILEGX_IMM16_X1_HW1_PCREL: u32 = 53; +/// X0 pipe PC relative hword 2 +pub const R_TILEGX_IMM16_X0_HW2_PCREL: u32 = 54; +/// X1 pipe PC relative hword 2 +pub const R_TILEGX_IMM16_X1_HW2_PCREL: u32 = 55; +/// X0 pipe PC relative hword 3 +pub const R_TILEGX_IMM16_X0_HW3_PCREL: u32 = 56; +/// X1 pipe PC relative hword 3 +pub const R_TILEGX_IMM16_X1_HW3_PCREL: u32 = 57; +/// X0 pipe PC-rel last hword 0 +pub const R_TILEGX_IMM16_X0_HW0_LAST_PCREL: u32 = 58; +/// X1 pipe PC-rel last hword 0 +pub const R_TILEGX_IMM16_X1_HW0_LAST_PCREL: u32 = 59; +/// X0 pipe PC-rel last hword 1 +pub const R_TILEGX_IMM16_X0_HW1_LAST_PCREL: u32 = 60; +/// X1 pipe PC-rel last hword 1 +pub const R_TILEGX_IMM16_X1_HW1_LAST_PCREL: u32 = 61; +/// X0 pipe PC-rel last hword 2 +pub const R_TILEGX_IMM16_X0_HW2_LAST_PCREL: u32 = 62; +/// X1 pipe PC-rel last hword 2 +pub const R_TILEGX_IMM16_X1_HW2_LAST_PCREL: u32 = 63; +/// X0 pipe hword 0 GOT offset +pub const R_TILEGX_IMM16_X0_HW0_GOT: u32 = 64; +/// X1 pipe hword 0 GOT offset +pub const R_TILEGX_IMM16_X1_HW0_GOT: u32 = 65; +/// X0 pipe PC-rel PLT hword 0 +pub const R_TILEGX_IMM16_X0_HW0_PLT_PCREL: u32 = 66; +/// X1 pipe PC-rel PLT hword 0 +pub const R_TILEGX_IMM16_X1_HW0_PLT_PCREL: u32 = 67; +/// X0 pipe PC-rel PLT hword 1 +pub const R_TILEGX_IMM16_X0_HW1_PLT_PCREL: u32 = 68; +/// X1 pipe PC-rel PLT hword 1 +pub const R_TILEGX_IMM16_X1_HW1_PLT_PCREL: u32 = 69; +/// X0 pipe PC-rel PLT hword 2 +pub const R_TILEGX_IMM16_X0_HW2_PLT_PCREL: u32 = 70; +/// X1 pipe PC-rel PLT hword 2 +pub const R_TILEGX_IMM16_X1_HW2_PLT_PCREL: u32 = 71; +/// X0 pipe last hword 0 GOT offset +pub const R_TILEGX_IMM16_X0_HW0_LAST_GOT: u32 = 72; +/// X1 pipe last hword 0 GOT offset +pub const R_TILEGX_IMM16_X1_HW0_LAST_GOT: u32 = 73; +/// X0 pipe last hword 1 GOT offset +pub const R_TILEGX_IMM16_X0_HW1_LAST_GOT: u32 = 74; +/// X1 pipe last hword 1 GOT offset +pub const R_TILEGX_IMM16_X1_HW1_LAST_GOT: u32 = 75; +/// X0 pipe PC-rel PLT hword 3 +pub const R_TILEGX_IMM16_X0_HW3_PLT_PCREL: u32 = 76; +/// X1 pipe PC-rel PLT hword 3 +pub const R_TILEGX_IMM16_X1_HW3_PLT_PCREL: u32 = 77; +/// X0 pipe hword 0 TLS GD offset +pub const R_TILEGX_IMM16_X0_HW0_TLS_GD: u32 = 78; +/// X1 pipe hword 0 TLS GD offset +pub const R_TILEGX_IMM16_X1_HW0_TLS_GD: u32 = 79; +/// X0 pipe hword 0 TLS LE offset +pub const R_TILEGX_IMM16_X0_HW0_TLS_LE: u32 = 80; +/// X1 pipe hword 0 TLS LE offset +pub const R_TILEGX_IMM16_X1_HW0_TLS_LE: u32 = 81; +/// X0 pipe last hword 0 LE off +pub const R_TILEGX_IMM16_X0_HW0_LAST_TLS_LE: u32 = 82; +/// X1 pipe last hword 0 LE off +pub const R_TILEGX_IMM16_X1_HW0_LAST_TLS_LE: u32 = 83; +/// X0 pipe last hword 1 LE off +pub const R_TILEGX_IMM16_X0_HW1_LAST_TLS_LE: u32 = 84; +/// X1 pipe last hword 1 LE off +pub const R_TILEGX_IMM16_X1_HW1_LAST_TLS_LE: u32 = 85; +/// X0 pipe last hword 0 GD off +pub const R_TILEGX_IMM16_X0_HW0_LAST_TLS_GD: u32 = 86; +/// X1 pipe last hword 0 GD off +pub const R_TILEGX_IMM16_X1_HW0_LAST_TLS_GD: u32 = 87; +/// X0 pipe last hword 1 GD off +pub const R_TILEGX_IMM16_X0_HW1_LAST_TLS_GD: u32 = 88; +/// X1 pipe last hword 1 GD off +pub const R_TILEGX_IMM16_X1_HW1_LAST_TLS_GD: u32 = 89; +// Relocs 90-91 are currently not defined. +/// X0 pipe hword 0 TLS IE offset +pub const R_TILEGX_IMM16_X0_HW0_TLS_IE: u32 = 92; +/// X1 pipe hword 0 TLS IE offset +pub const R_TILEGX_IMM16_X1_HW0_TLS_IE: u32 = 93; +/// X0 pipe PC-rel PLT last hword 0 +pub const R_TILEGX_IMM16_X0_HW0_LAST_PLT_PCREL: u32 = 94; +/// X1 pipe PC-rel PLT last hword 0 +pub const R_TILEGX_IMM16_X1_HW0_LAST_PLT_PCREL: u32 = 95; +/// X0 pipe PC-rel PLT last hword 1 +pub const R_TILEGX_IMM16_X0_HW1_LAST_PLT_PCREL: u32 = 96; +/// X1 pipe PC-rel PLT last hword 1 +pub const R_TILEGX_IMM16_X1_HW1_LAST_PLT_PCREL: u32 = 97; +/// X0 pipe PC-rel PLT last hword 2 +pub const R_TILEGX_IMM16_X0_HW2_LAST_PLT_PCREL: u32 = 98; +/// X1 pipe PC-rel PLT last hword 2 +pub const R_TILEGX_IMM16_X1_HW2_LAST_PLT_PCREL: u32 = 99; +/// X0 pipe last hword 0 IE off +pub const R_TILEGX_IMM16_X0_HW0_LAST_TLS_IE: u32 = 100; +/// X1 pipe last hword 0 IE off +pub const R_TILEGX_IMM16_X1_HW0_LAST_TLS_IE: u32 = 101; +/// X0 pipe last hword 1 IE off +pub const R_TILEGX_IMM16_X0_HW1_LAST_TLS_IE: u32 = 102; +/// X1 pipe last hword 1 IE off +pub const R_TILEGX_IMM16_X1_HW1_LAST_TLS_IE: u32 = 103; +// Relocs 104-105 are currently not defined. +/// 64-bit ID of symbol's module +pub const R_TILEGX_TLS_DTPMOD64: u32 = 106; +/// 64-bit offset in TLS block +pub const R_TILEGX_TLS_DTPOFF64: u32 = 107; +/// 64-bit offset in static TLS block +pub const R_TILEGX_TLS_TPOFF64: u32 = 108; +/// 32-bit ID of symbol's module +pub const R_TILEGX_TLS_DTPMOD32: u32 = 109; +/// 32-bit offset in TLS block +pub const R_TILEGX_TLS_DTPOFF32: u32 = 110; +/// 32-bit offset in static TLS block +pub const R_TILEGX_TLS_TPOFF32: u32 = 111; +/// "jal" for TLS GD +pub const R_TILEGX_TLS_GD_CALL: u32 = 112; +/// X0 pipe "addi" for TLS GD +pub const R_TILEGX_IMM8_X0_TLS_GD_ADD: u32 = 113; +/// X1 pipe "addi" for TLS GD +pub const R_TILEGX_IMM8_X1_TLS_GD_ADD: u32 = 114; +/// Y0 pipe "addi" for TLS GD +pub const R_TILEGX_IMM8_Y0_TLS_GD_ADD: u32 = 115; +/// Y1 pipe "addi" for TLS GD +pub const R_TILEGX_IMM8_Y1_TLS_GD_ADD: u32 = 116; +/// "ld_tls" for TLS IE +pub const R_TILEGX_TLS_IE_LOAD: u32 = 117; +/// X0 pipe "addi" for TLS GD/IE +pub const R_TILEGX_IMM8_X0_TLS_ADD: u32 = 118; +/// X1 pipe "addi" for TLS GD/IE +pub const R_TILEGX_IMM8_X1_TLS_ADD: u32 = 119; +/// Y0 pipe "addi" for TLS GD/IE +pub const R_TILEGX_IMM8_Y0_TLS_ADD: u32 = 120; +/// Y1 pipe "addi" for TLS GD/IE +pub const R_TILEGX_IMM8_Y1_TLS_ADD: u32 = 121; + +/// GNU C++ vtable hierarchy +pub const R_TILEGX_GNU_VTINHERIT: u32 = 128; +/// GNU C++ vtable member usage +pub const R_TILEGX_GNU_VTENTRY: u32 = 129; + +// RISC-V values `FileHeader*::e_flags`. +#[allow(missing_docs)] +pub const EF_RISCV_RVC: u32 = 0x0001; +#[allow(missing_docs)] +pub const EF_RISCV_FLOAT_ABI: u32 = 0x0006; +#[allow(missing_docs)] +pub const EF_RISCV_FLOAT_ABI_SOFT: u32 = 0x0000; +#[allow(missing_docs)] +pub const EF_RISCV_FLOAT_ABI_SINGLE: u32 = 0x0002; +#[allow(missing_docs)] +pub const EF_RISCV_FLOAT_ABI_DOUBLE: u32 = 0x0004; +#[allow(missing_docs)] +pub const EF_RISCV_FLOAT_ABI_QUAD: u32 = 0x0006; + +// RISC-V values `Rel*::r_type`. +#[allow(missing_docs)] +pub const R_RISCV_NONE: u32 = 0; +#[allow(missing_docs)] +pub const R_RISCV_32: u32 = 1; +#[allow(missing_docs)] +pub const R_RISCV_64: u32 = 2; +#[allow(missing_docs)] +pub const R_RISCV_RELATIVE: u32 = 3; +#[allow(missing_docs)] +pub const R_RISCV_COPY: u32 = 4; +#[allow(missing_docs)] +pub const R_RISCV_JUMP_SLOT: u32 = 5; +#[allow(missing_docs)] +pub const R_RISCV_TLS_DTPMOD32: u32 = 6; +#[allow(missing_docs)] +pub const R_RISCV_TLS_DTPMOD64: u32 = 7; +#[allow(missing_docs)] +pub const R_RISCV_TLS_DTPREL32: u32 = 8; +#[allow(missing_docs)] +pub const R_RISCV_TLS_DTPREL64: u32 = 9; +#[allow(missing_docs)] +pub const R_RISCV_TLS_TPREL32: u32 = 10; +#[allow(missing_docs)] +pub const R_RISCV_TLS_TPREL64: u32 = 11; +#[allow(missing_docs)] +pub const R_RISCV_BRANCH: u32 = 16; +#[allow(missing_docs)] +pub const R_RISCV_JAL: u32 = 17; +#[allow(missing_docs)] +pub const R_RISCV_CALL: u32 = 18; +#[allow(missing_docs)] +pub const R_RISCV_CALL_PLT: u32 = 19; +#[allow(missing_docs)] +pub const R_RISCV_GOT_HI20: u32 = 20; +#[allow(missing_docs)] +pub const R_RISCV_TLS_GOT_HI20: u32 = 21; +#[allow(missing_docs)] +pub const R_RISCV_TLS_GD_HI20: u32 = 22; +#[allow(missing_docs)] +pub const R_RISCV_PCREL_HI20: u32 = 23; +#[allow(missing_docs)] +pub const R_RISCV_PCREL_LO12_I: u32 = 24; +#[allow(missing_docs)] +pub const R_RISCV_PCREL_LO12_S: u32 = 25; +#[allow(missing_docs)] +pub const R_RISCV_HI20: u32 = 26; +#[allow(missing_docs)] +pub const R_RISCV_LO12_I: u32 = 27; +#[allow(missing_docs)] +pub const R_RISCV_LO12_S: u32 = 28; +#[allow(missing_docs)] +pub const R_RISCV_TPREL_HI20: u32 = 29; +#[allow(missing_docs)] +pub const R_RISCV_TPREL_LO12_I: u32 = 30; +#[allow(missing_docs)] +pub const R_RISCV_TPREL_LO12_S: u32 = 31; +#[allow(missing_docs)] +pub const R_RISCV_TPREL_ADD: u32 = 32; +#[allow(missing_docs)] +pub const R_RISCV_ADD8: u32 = 33; +#[allow(missing_docs)] +pub const R_RISCV_ADD16: u32 = 34; +#[allow(missing_docs)] +pub const R_RISCV_ADD32: u32 = 35; +#[allow(missing_docs)] +pub const R_RISCV_ADD64: u32 = 36; +#[allow(missing_docs)] +pub const R_RISCV_SUB8: u32 = 37; +#[allow(missing_docs)] +pub const R_RISCV_SUB16: u32 = 38; +#[allow(missing_docs)] +pub const R_RISCV_SUB32: u32 = 39; +#[allow(missing_docs)] +pub const R_RISCV_SUB64: u32 = 40; +#[allow(missing_docs)] +pub const R_RISCV_GNU_VTINHERIT: u32 = 41; +#[allow(missing_docs)] +pub const R_RISCV_GNU_VTENTRY: u32 = 42; +#[allow(missing_docs)] +pub const R_RISCV_ALIGN: u32 = 43; +#[allow(missing_docs)] +pub const R_RISCV_RVC_BRANCH: u32 = 44; +#[allow(missing_docs)] +pub const R_RISCV_RVC_JUMP: u32 = 45; +#[allow(missing_docs)] +pub const R_RISCV_RVC_LUI: u32 = 46; +#[allow(missing_docs)] +pub const R_RISCV_GPREL_I: u32 = 47; +#[allow(missing_docs)] +pub const R_RISCV_GPREL_S: u32 = 48; +#[allow(missing_docs)] +pub const R_RISCV_TPREL_I: u32 = 49; +#[allow(missing_docs)] +pub const R_RISCV_TPREL_S: u32 = 50; +#[allow(missing_docs)] +pub const R_RISCV_RELAX: u32 = 51; +#[allow(missing_docs)] +pub const R_RISCV_SUB6: u32 = 52; +#[allow(missing_docs)] +pub const R_RISCV_SET6: u32 = 53; +#[allow(missing_docs)] +pub const R_RISCV_SET8: u32 = 54; +#[allow(missing_docs)] +pub const R_RISCV_SET16: u32 = 55; +#[allow(missing_docs)] +pub const R_RISCV_SET32: u32 = 56; +#[allow(missing_docs)] +pub const R_RISCV_32_PCREL: u32 = 57; + +// BPF values `Rel*::r_type`. +/// No reloc +pub const R_BPF_NONE: u32 = 0; +#[allow(missing_docs)] +pub const R_BPF_64_64: u32 = 1; +#[allow(missing_docs)] +pub const R_BPF_64_32: u32 = 10; + +// Imagination Meta values `Rel*::r_type`. + +#[allow(missing_docs)] +pub const R_METAG_HIADDR16: u32 = 0; +#[allow(missing_docs)] +pub const R_METAG_LOADDR16: u32 = 1; +/// 32bit absolute address +pub const R_METAG_ADDR32: u32 = 2; +/// No reloc +pub const R_METAG_NONE: u32 = 3; +#[allow(missing_docs)] +pub const R_METAG_RELBRANCH: u32 = 4; +#[allow(missing_docs)] +pub const R_METAG_GETSETOFF: u32 = 5; + +// Backward compatability +#[allow(missing_docs)] +pub const R_METAG_REG32OP1: u32 = 6; +#[allow(missing_docs)] +pub const R_METAG_REG32OP2: u32 = 7; +#[allow(missing_docs)] +pub const R_METAG_REG32OP3: u32 = 8; +#[allow(missing_docs)] +pub const R_METAG_REG16OP1: u32 = 9; +#[allow(missing_docs)] +pub const R_METAG_REG16OP2: u32 = 10; +#[allow(missing_docs)] +pub const R_METAG_REG16OP3: u32 = 11; +#[allow(missing_docs)] +pub const R_METAG_REG32OP4: u32 = 12; + +#[allow(missing_docs)] +pub const R_METAG_HIOG: u32 = 13; +#[allow(missing_docs)] +pub const R_METAG_LOOG: u32 = 14; + +#[allow(missing_docs)] +pub const R_METAG_REL8: u32 = 15; +#[allow(missing_docs)] +pub const R_METAG_REL16: u32 = 16; + +#[allow(missing_docs)] +pub const R_METAG_GNU_VTINHERIT: u32 = 30; +#[allow(missing_docs)] +pub const R_METAG_GNU_VTENTRY: u32 = 31; + +// PIC relocations +#[allow(missing_docs)] +pub const R_METAG_HI16_GOTOFF: u32 = 32; +#[allow(missing_docs)] +pub const R_METAG_LO16_GOTOFF: u32 = 33; +#[allow(missing_docs)] +pub const R_METAG_GETSET_GOTOFF: u32 = 34; +#[allow(missing_docs)] +pub const R_METAG_GETSET_GOT: u32 = 35; +#[allow(missing_docs)] +pub const R_METAG_HI16_GOTPC: u32 = 36; +#[allow(missing_docs)] +pub const R_METAG_LO16_GOTPC: u32 = 37; +#[allow(missing_docs)] +pub const R_METAG_HI16_PLT: u32 = 38; +#[allow(missing_docs)] +pub const R_METAG_LO16_PLT: u32 = 39; +#[allow(missing_docs)] +pub const R_METAG_RELBRANCH_PLT: u32 = 40; +#[allow(missing_docs)] +pub const R_METAG_GOTOFF: u32 = 41; +#[allow(missing_docs)] +pub const R_METAG_PLT: u32 = 42; +#[allow(missing_docs)] +pub const R_METAG_COPY: u32 = 43; +#[allow(missing_docs)] +pub const R_METAG_JMP_SLOT: u32 = 44; +#[allow(missing_docs)] +pub const R_METAG_RELATIVE: u32 = 45; +#[allow(missing_docs)] +pub const R_METAG_GLOB_DAT: u32 = 46; + +// TLS relocations +#[allow(missing_docs)] +pub const R_METAG_TLS_GD: u32 = 47; +#[allow(missing_docs)] +pub const R_METAG_TLS_LDM: u32 = 48; +#[allow(missing_docs)] +pub const R_METAG_TLS_LDO_HI16: u32 = 49; +#[allow(missing_docs)] +pub const R_METAG_TLS_LDO_LO16: u32 = 50; +#[allow(missing_docs)] +pub const R_METAG_TLS_LDO: u32 = 51; +#[allow(missing_docs)] +pub const R_METAG_TLS_IE: u32 = 52; +#[allow(missing_docs)] +pub const R_METAG_TLS_IENONPIC: u32 = 53; +#[allow(missing_docs)] +pub const R_METAG_TLS_IENONPIC_HI16: u32 = 54; +#[allow(missing_docs)] +pub const R_METAG_TLS_IENONPIC_LO16: u32 = 55; +#[allow(missing_docs)] +pub const R_METAG_TLS_TPOFF: u32 = 56; +#[allow(missing_docs)] +pub const R_METAG_TLS_DTPMOD: u32 = 57; +#[allow(missing_docs)] +pub const R_METAG_TLS_DTPOFF: u32 = 58; +#[allow(missing_docs)] +pub const R_METAG_TLS_LE: u32 = 59; +#[allow(missing_docs)] +pub const R_METAG_TLS_LE_HI16: u32 = 60; +#[allow(missing_docs)] +pub const R_METAG_TLS_LE_LO16: u32 = 61; + +// NDS32 values `Rel*::r_type`. +#[allow(missing_docs)] +pub const R_NDS32_NONE: u32 = 0; +#[allow(missing_docs)] +pub const R_NDS32_32_RELA: u32 = 20; +#[allow(missing_docs)] +pub const R_NDS32_COPY: u32 = 39; +#[allow(missing_docs)] +pub const R_NDS32_GLOB_DAT: u32 = 40; +#[allow(missing_docs)] +pub const R_NDS32_JMP_SLOT: u32 = 41; +#[allow(missing_docs)] +pub const R_NDS32_RELATIVE: u32 = 42; +#[allow(missing_docs)] +pub const R_NDS32_TLS_TPOFF: u32 = 102; +#[allow(missing_docs)] +pub const R_NDS32_TLS_DESC: u32 = 119; + +// LoongArch values `FileHeader*::e_flags`. +/// Uses 64-bit GPRs and the stack for parameter passing +pub const EF_LARCH_ABI_LP64S: u32 = 0x1; +/// Uses 64-bit GPRs, 32-bit FPRs and the stack for parameter passing +pub const EF_LARCH_ABI_LP64F: u32 = 0x2; +/// Uses 64-bit GPRs, 64-bit FPRs and the stack for parameter passing +pub const EF_LARCH_ABI_LP64D: u32 = 0x3; +/// Uses 32-bit GPRs and the stack for parameter passing +pub const EF_LARCH_ABI_ILP32S: u32 = 0x5; +/// Uses 32-bit GPRs, 32-bit FPRs and the stack for parameter passing +pub const EF_LARCH_ABI_ILP32F: u32 = 0x6; +/// Uses 32-bit GPRs, 64-bit FPRs and the stack for parameter passing +pub const EF_LARCH_ABI_ILP32D: u32 = 0x7; + +// LoongArch values `Rel*::r_type`. +/// No reloc +pub const R_LARCH_NONE: u32 = 0; +/// Runtime address resolving +pub const R_LARCH_32: u32 = 1; +/// Runtime address resolving +pub const R_LARCH_64: u32 = 2; +/// Runtime fixup for load-address +pub const R_LARCH_RELATIVE: u32 = 3; +/// Runtime memory copy in executable +pub const R_LARCH_COPY: u32 = 4; +/// Runtime PLT supporting +pub const R_LARCH_JUMP_SLOT: u32 = 5; +/// Runtime relocation for TLS-GD +pub const R_LARCH_TLS_DTPMOD32: u32 = 6; +/// Runtime relocation for TLS-GD +pub const R_LARCH_TLS_DTPMOD64: u32 = 7; +/// Runtime relocation for TLS-GD +pub const R_LARCH_TLS_DTPREL32: u32 = 8; +/// Runtime relocation for TLS-GD +pub const R_LARCH_TLS_DTPREL64: u32 = 9; +/// Runtime relocation for TLE-IE +pub const R_LARCH_TLS_TPREL32: u32 = 10; +/// Runtime relocation for TLE-IE +pub const R_LARCH_TLS_TPREL64: u32 = 11; +/// Runtime local indirect function resolving +pub const R_LARCH_IRELATIVE: u32 = 12; +/// Mark la.abs: load absolute address for static link. +pub const R_LARCH_MARK_LA: u32 = 20; +/// Mark external label branch: access PC relative address for static link. +pub const R_LARCH_MARK_PCREL: u32 = 21; +/// Push PC-relative offset +pub const R_LARCH_SOP_PUSH_PCREL: u32 = 22; +/// Push constant or absolute address +pub const R_LARCH_SOP_PUSH_ABSOLUTE: u32 = 23; +/// Duplicate stack top +pub const R_LARCH_SOP_PUSH_DUP: u32 = 24; +/// Push for access GOT entry +pub const R_LARCH_SOP_PUSH_GPREL: u32 = 25; +/// Push for TLS-LE +pub const R_LARCH_SOP_PUSH_TLS_TPREL: u32 = 26; +/// Push for TLS-IE +pub const R_LARCH_SOP_PUSH_TLS_GOT: u32 = 27; +/// Push for TLS-GD +pub const R_LARCH_SOP_PUSH_TLS_GD: u32 = 28; +/// Push for external function calling +pub const R_LARCH_SOP_PUSH_PLT_PCREL: u32 = 29; +/// Assert stack top +pub const R_LARCH_SOP_ASSERT: u32 = 30; +/// Stack top logical not (unary) +pub const R_LARCH_SOP_NOT: u32 = 31; +/// Stack top subtraction (binary) +pub const R_LARCH_SOP_SUB: u32 = 32; +/// Stack top left shift (binary) +pub const R_LARCH_SOP_SL: u32 = 33; +/// Stack top right shift (binary) +pub const R_LARCH_SOP_SR: u32 = 34; +/// Stack top addition (binary) +pub const R_LARCH_SOP_ADD: u32 = 35; +/// Stack top bitwise and (binary) +pub const R_LARCH_SOP_AND: u32 = 36; +/// Stack top selection (tertiary) +pub const R_LARCH_SOP_IF_ELSE: u32 = 37; +/// Pop stack top to fill 5-bit signed immediate operand +pub const R_LARCH_SOP_POP_32_S_10_5: u32 = 38; +/// Pop stack top to fill 12-bit unsigned immediate operand +pub const R_LARCH_SOP_POP_32_U_10_12: u32 = 39; +/// Pop stack top to fill 12-bit signed immediate operand +pub const R_LARCH_SOP_POP_32_S_10_12: u32 = 40; +/// Pop stack top to fill 16-bit signed immediate operand +pub const R_LARCH_SOP_POP_32_S_10_16: u32 = 41; +/// Pop stack top to fill 18-bit signed immediate operand with two trailing +/// zeros implied +pub const R_LARCH_SOP_POP_32_S_10_16_S2: u32 = 42; +/// Pop stack top to fill 20-bit signed immediate operand +pub const R_LARCH_SOP_POP_32_S_5_20: u32 = 43; +/// Pop stack top to fill 23-bit signed immediate operand with two trailing +/// zeros implied +pub const R_LARCH_SOP_POP_32_S_0_5_10_16_S2: u32 = 44; +/// Pop stack top to fill 28-bit signed immediate operand with two trailing +/// zeros implied +pub const R_LARCH_SOP_POP_32_S_0_10_10_16_S2: u32 = 45; +/// Pop stack top to fill an instruction +pub const R_LARCH_SOP_POP_32_U: u32 = 46; +/// 8-bit in-place addition +pub const R_LARCH_ADD8: u32 = 47; +/// 16-bit in-place addition +pub const R_LARCH_ADD16: u32 = 48; +/// 24-bit in-place addition +pub const R_LARCH_ADD24: u32 = 49; +/// 32-bit in-place addition +pub const R_LARCH_ADD32: u32 = 50; +/// 64-bit in-place addition +pub const R_LARCH_ADD64: u32 = 51; +/// 8-bit in-place subtraction +pub const R_LARCH_SUB8: u32 = 52; +/// 16-bit in-place subtraction +pub const R_LARCH_SUB16: u32 = 53; +/// 24-bit in-place subtraction +pub const R_LARCH_SUB24: u32 = 54; +/// 32-bit in-place subtraction +pub const R_LARCH_SUB32: u32 = 55; +/// 64-bit in-place subtraction +pub const R_LARCH_SUB64: u32 = 56; +/// GNU C++ vtable hierarchy +pub const R_LARCH_GNU_VTINHERIT: u32 = 57; +/// GNU C++ vtable member usage +pub const R_LARCH_GNU_VTENTRY: u32 = 58; + +unsafe_impl_endian_pod!( + FileHeader32, + FileHeader64, + SectionHeader32, + SectionHeader64, + CompressionHeader32, + CompressionHeader64, + Sym32, + Sym64, + Syminfo32, + Syminfo64, + Rel32, + Rel64, + Rela32, + Rela64, + ProgramHeader32, + ProgramHeader64, + Dyn32, + Dyn64, + Versym, + Verdef, + Verdaux, + Verneed, + Vernaux, + NoteHeader32, + NoteHeader64, + HashHeader, + GnuHashHeader, +); diff --git a/crux-mir/lib/object/src/endian.rs b/crux-mir/lib/object/src/endian.rs new file mode 100644 index 000000000..1287aa654 --- /dev/null +++ b/crux-mir/lib/object/src/endian.rs @@ -0,0 +1,771 @@ +//! Types for compile-time and run-time endianness. + +use crate::pod::Pod; +use core::fmt::{self, Debug}; +use core::marker::PhantomData; + +/// A trait for using an endianness specification. +/// +/// Provides methods for converting between the specified endianness and +/// the native endianness of the target machine. +/// +/// This trait does not require that the endianness is known at compile time. +pub trait Endian: Debug + Default + Clone + Copy + PartialEq + Eq + 'static { + /// Construct a specification for the endianness of some values. + /// + /// Returns `None` if the type does not support specifying the given endianness. + fn from_big_endian(big_endian: bool) -> Option; + + /// Construct a specification for the endianness of some values. + /// + /// Returns `None` if the type does not support specifying the given endianness. + fn from_little_endian(little_endian: bool) -> Option { + Self::from_big_endian(!little_endian) + } + + /// Return true for big endian byte order. + fn is_big_endian(self) -> bool; + + /// Return true for little endian byte order. + #[inline] + fn is_little_endian(self) -> bool { + !self.is_big_endian() + } + + /// Converts an unsigned 16 bit integer to native endian. + #[inline] + fn read_u16(self, n: u16) -> u16 { + if self.is_big_endian() { + u16::from_be(n) + } else { + u16::from_le(n) + } + } + + /// Converts an unsigned 32 bit integer to native endian. + #[inline] + fn read_u32(self, n: u32) -> u32 { + if self.is_big_endian() { + u32::from_be(n) + } else { + u32::from_le(n) + } + } + + /// Converts an unsigned 64 bit integer to native endian. + #[inline] + fn read_u64(self, n: u64) -> u64 { + if self.is_big_endian() { + u64::from_be(n) + } else { + u64::from_le(n) + } + } + + /// Converts a signed 16 bit integer to native endian. + #[inline] + fn read_i16(self, n: i16) -> i16 { + if self.is_big_endian() { + i16::from_be(n) + } else { + i16::from_le(n) + } + } + + /// Converts a signed 32 bit integer to native endian. + #[inline] + fn read_i32(self, n: i32) -> i32 { + if self.is_big_endian() { + i32::from_be(n) + } else { + i32::from_le(n) + } + } + + /// Converts a signed 64 bit integer to native endian. + #[inline] + fn read_i64(self, n: i64) -> i64 { + if self.is_big_endian() { + i64::from_be(n) + } else { + i64::from_le(n) + } + } + + /// Converts an unaligned unsigned 16 bit integer to native endian. + #[inline] + fn read_u16_bytes(self, n: [u8; 2]) -> u16 { + if self.is_big_endian() { + u16::from_be_bytes(n) + } else { + u16::from_le_bytes(n) + } + } + + /// Converts an unaligned unsigned 32 bit integer to native endian. + #[inline] + fn read_u32_bytes(self, n: [u8; 4]) -> u32 { + if self.is_big_endian() { + u32::from_be_bytes(n) + } else { + u32::from_le_bytes(n) + } + } + + /// Converts an unaligned unsigned 64 bit integer to native endian. + #[inline] + fn read_u64_bytes(self, n: [u8; 8]) -> u64 { + if self.is_big_endian() { + u64::from_be_bytes(n) + } else { + u64::from_le_bytes(n) + } + } + + /// Converts an unaligned signed 16 bit integer to native endian. + #[inline] + fn read_i16_bytes(self, n: [u8; 2]) -> i16 { + if self.is_big_endian() { + i16::from_be_bytes(n) + } else { + i16::from_le_bytes(n) + } + } + + /// Converts an unaligned signed 32 bit integer to native endian. + #[inline] + fn read_i32_bytes(self, n: [u8; 4]) -> i32 { + if self.is_big_endian() { + i32::from_be_bytes(n) + } else { + i32::from_le_bytes(n) + } + } + + /// Converts an unaligned signed 64 bit integer to native endian. + #[inline] + fn read_i64_bytes(self, n: [u8; 8]) -> i64 { + if self.is_big_endian() { + i64::from_be_bytes(n) + } else { + i64::from_le_bytes(n) + } + } + + /// Converts an unsigned 16 bit integer from native endian. + #[inline] + fn write_u16(self, n: u16) -> u16 { + if self.is_big_endian() { + u16::to_be(n) + } else { + u16::to_le(n) + } + } + + /// Converts an unsigned 32 bit integer from native endian. + #[inline] + fn write_u32(self, n: u32) -> u32 { + if self.is_big_endian() { + u32::to_be(n) + } else { + u32::to_le(n) + } + } + + /// Converts an unsigned 64 bit integer from native endian. + #[inline] + fn write_u64(self, n: u64) -> u64 { + if self.is_big_endian() { + u64::to_be(n) + } else { + u64::to_le(n) + } + } + + /// Converts a signed 16 bit integer from native endian. + #[inline] + fn write_i16(self, n: i16) -> i16 { + if self.is_big_endian() { + i16::to_be(n) + } else { + i16::to_le(n) + } + } + + /// Converts a signed 32 bit integer from native endian. + #[inline] + fn write_i32(self, n: i32) -> i32 { + if self.is_big_endian() { + i32::to_be(n) + } else { + i32::to_le(n) + } + } + + /// Converts a signed 64 bit integer from native endian. + #[inline] + fn write_i64(self, n: i64) -> i64 { + if self.is_big_endian() { + i64::to_be(n) + } else { + i64::to_le(n) + } + } + + /// Converts an unaligned unsigned 16 bit integer from native endian. + #[inline] + fn write_u16_bytes(self, n: u16) -> [u8; 2] { + if self.is_big_endian() { + u16::to_be_bytes(n) + } else { + u16::to_le_bytes(n) + } + } + + /// Converts an unaligned unsigned 32 bit integer from native endian. + #[inline] + fn write_u32_bytes(self, n: u32) -> [u8; 4] { + if self.is_big_endian() { + u32::to_be_bytes(n) + } else { + u32::to_le_bytes(n) + } + } + + /// Converts an unaligned unsigned 64 bit integer from native endian. + #[inline] + fn write_u64_bytes(self, n: u64) -> [u8; 8] { + if self.is_big_endian() { + u64::to_be_bytes(n) + } else { + u64::to_le_bytes(n) + } + } + + /// Converts an unaligned signed 16 bit integer from native endian. + #[inline] + fn write_i16_bytes(self, n: i16) -> [u8; 2] { + if self.is_big_endian() { + i16::to_be_bytes(n) + } else { + i16::to_le_bytes(n) + } + } + + /// Converts an unaligned signed 32 bit integer from native endian. + #[inline] + fn write_i32_bytes(self, n: i32) -> [u8; 4] { + if self.is_big_endian() { + i32::to_be_bytes(n) + } else { + i32::to_le_bytes(n) + } + } + + /// Converts an unaligned signed 64 bit integer from native endian. + #[inline] + fn write_i64_bytes(self, n: i64) -> [u8; 8] { + if self.is_big_endian() { + i64::to_be_bytes(n) + } else { + i64::to_le_bytes(n) + } + } +} + +/// An endianness that is selectable at run-time. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Endianness { + /// Little endian byte order. + Little, + /// Big endian byte order. + Big, +} + +impl Default for Endianness { + #[cfg(target_endian = "little")] + #[inline] + fn default() -> Endianness { + Endianness::Little + } + + #[cfg(target_endian = "big")] + #[inline] + fn default() -> Endianness { + Endianness::Big + } +} + +impl Endian for Endianness { + #[inline] + fn from_big_endian(big_endian: bool) -> Option { + Some(if big_endian { + Endianness::Big + } else { + Endianness::Little + }) + } + + #[inline] + fn is_big_endian(self) -> bool { + self != Endianness::Little + } +} + +/// Compile-time little endian byte order. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct LittleEndian; + +impl Default for LittleEndian { + #[inline] + fn default() -> LittleEndian { + LittleEndian + } +} + +impl Endian for LittleEndian { + #[inline] + fn from_big_endian(big_endian: bool) -> Option { + if big_endian { + None + } else { + Some(LittleEndian) + } + } + + #[inline] + fn is_big_endian(self) -> bool { + false + } +} + +/// Compile-time big endian byte order. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct BigEndian; + +impl Default for BigEndian { + #[inline] + fn default() -> BigEndian { + BigEndian + } +} + +impl Endian for BigEndian { + #[inline] + fn from_big_endian(big_endian: bool) -> Option { + if big_endian { + Some(BigEndian) + } else { + None + } + } + + #[inline] + fn is_big_endian(self) -> bool { + true + } +} + +/// The native endianness for the target platform. +#[cfg(target_endian = "little")] +pub type NativeEndian = LittleEndian; + +#[cfg(target_endian = "little")] +#[allow(non_upper_case_globals)] +#[doc(hidden)] +pub const NativeEndian: LittleEndian = LittleEndian; + +/// The native endianness for the target platform. +#[cfg(target_endian = "big")] +pub type NativeEndian = BigEndian; + +#[cfg(target_endian = "big")] +#[allow(non_upper_case_globals)] +#[doc(hidden)] +pub const NativeEndian: BigEndian = BigEndian; + +macro_rules! unsafe_impl_endian_pod { + ($($struct_name:ident),+ $(,)?) => { + $( + unsafe impl Pod for $struct_name { } + )+ + } +} + +#[cfg(not(feature = "unaligned"))] +mod aligned { + use super::{fmt, Endian, PhantomData, Pod}; + + /// A `u16` value with an externally specified endianness of type `E`. + #[derive(Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] + #[repr(transparent)] + pub struct U16(u16, PhantomData); + + impl U16 { + /// Construct a new value given a native endian value. + pub fn new(e: E, n: u16) -> Self { + Self(e.write_u16(n), PhantomData) + } + + /// Return the value as a native endian value. + pub fn get(self, e: E) -> u16 { + e.read_u16(self.0) + } + + /// Set the value given a native endian value. + pub fn set(&mut self, e: E, n: u16) { + self.0 = e.write_u16(n); + } + } + + /// A `u32` value with an externally specified endianness of type `E`. + #[derive(Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] + #[repr(transparent)] + pub struct U32(u32, PhantomData); + + impl U32 { + /// Construct a new value given a native endian value. + pub fn new(e: E, n: u32) -> Self { + Self(e.write_u32(n), PhantomData) + } + /// Return the value as a native endian value. + pub fn get(self, e: E) -> u32 { + e.read_u32(self.0) + } + /// Set the value given a native endian value. + pub fn set(&mut self, e: E, n: u32) { + self.0 = e.write_u32(n); + } + } + + /// A `u64` value with an externally specified endianness of type `E`. + #[derive(Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] + #[repr(transparent)] + pub struct U64(u64, PhantomData); + + impl U64 { + /// Construct a new value given a native endian value. + pub fn new(e: E, n: u64) -> Self { + Self(e.write_u64(n), PhantomData) + } + /// Return the value as a native endian value. + pub fn get(self, e: E) -> u64 { + e.read_u64(self.0) + } + /// Set the value given a native endian value. + pub fn set(&mut self, e: E, n: u64) { + self.0 = e.write_u64(n); + } + } + + /// An `i16` value with an externally specified endianness of type `E`. + #[derive(Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] + #[repr(transparent)] + pub struct I16(i16, PhantomData); + + impl I16 { + /// Construct a new value given a native endian value. + pub fn new(e: E, n: i16) -> Self { + Self(e.write_i16(n), PhantomData) + } + /// Return the value as a native endian value. + pub fn get(self, e: E) -> i16 { + e.read_i16(self.0) + } + /// Set the value given a native endian value. + pub fn set(&mut self, e: E, n: i16) { + self.0 = e.write_i16(n); + } + } + + /// An `i32` value with an externally specified endianness of type `E`. + #[derive(Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] + #[repr(transparent)] + pub struct I32(i32, PhantomData); + + impl I32 { + /// Construct a new value given a native endian value. + pub fn new(e: E, n: i32) -> Self { + Self(e.write_i32(n), PhantomData) + } + /// Return the value as a native endian value. + pub fn get(self, e: E) -> i32 { + e.read_i32(self.0) + } + /// Set the value given a native endian value. + pub fn set(&mut self, e: E, n: i32) { + self.0 = e.write_i32(n); + } + } + + /// An `i64` value with an externally specified endianness of type `E`. + #[derive(Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] + #[repr(transparent)] + pub struct I64(i64, PhantomData); + + impl I64 { + /// Construct a new value given a native endian value. + pub fn new(e: E, n: i64) -> Self { + Self(e.write_i64(n), PhantomData) + } + /// Return the value as a native endian value. + pub fn get(self, e: E) -> i64 { + e.read_i64(self.0) + } + /// Set the value given a native endian value. + pub fn set(&mut self, e: E, n: i64) { + self.0 = e.write_i64(n); + } + } + + impl fmt::Debug for U16 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "U16({:x})", self.0) + } + } + + impl fmt::Debug for U32 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "U32({:x})", self.0) + } + } + + impl fmt::Debug for U64 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "U64({:x})", self.0) + } + } + + impl fmt::Debug for I16 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "I16({:x})", self.0) + } + } + + impl fmt::Debug for I32 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "I32({:x})", self.0) + } + } + + impl fmt::Debug for I64 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "I64({:x})", self.0) + } + } + + unsafe_impl_endian_pod!(U16, U32, U64, I16, I32, I64); +} + +#[cfg(not(feature = "unaligned"))] +pub use aligned::*; + +/// A `u16` value with an externally specified endianness of type `E`. +#[cfg(feature = "unaligned")] +pub type U16 = U16Bytes; + +/// A `u32` value with an externally specified endianness of type `E`. +#[cfg(feature = "unaligned")] +pub type U32 = U32Bytes; + +/// A `u64` value with an externally specified endianness of type `E`. +#[cfg(feature = "unaligned")] +pub type U64 = U64Bytes; + +/// An `i16` value with an externally specified endianness of type `E`. +#[cfg(feature = "unaligned")] +pub type I16 = I16Bytes; + +/// An `i32` value with an externally specified endianness of type `E`. +#[cfg(feature = "unaligned")] +pub type I32 = I32Bytes; + +/// An `i64` value with an externally specified endianness of type `E`. +#[cfg(feature = "unaligned")] +pub type I64 = I64Bytes; + +/// An unaligned `u16` value with an externally specified endianness of type `E`. +#[derive(Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[repr(transparent)] +pub struct U16Bytes([u8; 2], PhantomData); + +impl U16Bytes { + /// Construct a new value given a native endian value. + pub fn new(e: E, n: u16) -> Self { + Self(e.write_u16_bytes(n), PhantomData) + } + + /// Return the value as a native endian value. + pub fn get(self, e: E) -> u16 { + e.read_u16_bytes(self.0) + } + + /// Set the value given a native endian value. + pub fn set(&mut self, e: E, n: u16) { + self.0 = e.write_u16_bytes(n); + } +} + +/// An unaligned `u32` value with an externally specified endianness of type `E`. +#[derive(Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[repr(transparent)] +pub struct U32Bytes([u8; 4], PhantomData); + +impl U32Bytes { + /// Construct a new value given a native endian value. + pub fn new(e: E, n: u32) -> Self { + Self(e.write_u32_bytes(n), PhantomData) + } + + /// Return the value as a native endian value. + pub fn get(self, e: E) -> u32 { + e.read_u32_bytes(self.0) + } + + /// Set the value given a native endian value. + pub fn set(&mut self, e: E, n: u32) { + self.0 = e.write_u32_bytes(n); + } +} + +/// An unaligned `u64` value with an externally specified endianness of type `E`. +#[derive(Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[repr(transparent)] +pub struct U64Bytes([u8; 8], PhantomData); + +impl U64Bytes { + /// Construct a new value given a native endian value. + pub fn new(e: E, n: u64) -> Self { + Self(e.write_u64_bytes(n), PhantomData) + } + + /// Return the value as a native endian value. + pub fn get(self, e: E) -> u64 { + e.read_u64_bytes(self.0) + } + + /// Set the value given a native endian value. + pub fn set(&mut self, e: E, n: u64) { + self.0 = e.write_u64_bytes(n); + } +} + +/// An unaligned `i16` value with an externally specified endianness of type `E`. +#[derive(Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[repr(transparent)] +pub struct I16Bytes([u8; 2], PhantomData); + +impl I16Bytes { + /// Construct a new value given a native endian value. + pub fn new(e: E, n: i16) -> Self { + Self(e.write_i16_bytes(n), PhantomData) + } + + /// Return the value as a native endian value. + pub fn get(self, e: E) -> i16 { + e.read_i16_bytes(self.0) + } + + /// Set the value given a native endian value. + pub fn set(&mut self, e: E, n: i16) { + self.0 = e.write_i16_bytes(n); + } +} + +/// An unaligned `i32` value with an externally specified endianness of type `E`. +#[derive(Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[repr(transparent)] +pub struct I32Bytes([u8; 4], PhantomData); + +impl I32Bytes { + /// Construct a new value given a native endian value. + pub fn new(e: E, n: i32) -> Self { + Self(e.write_i32_bytes(n), PhantomData) + } + + /// Return the value as a native endian value. + pub fn get(self, e: E) -> i32 { + e.read_i32_bytes(self.0) + } + + /// Set the value given a native endian value. + pub fn set(&mut self, e: E, n: i32) { + self.0 = e.write_i32_bytes(n); + } +} + +/// An unaligned `i64` value with an externally specified endianness of type `E`. +#[derive(Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[repr(transparent)] +pub struct I64Bytes([u8; 8], PhantomData); + +impl I64Bytes { + /// Construct a new value given a native endian value. + pub fn new(e: E, n: i64) -> Self { + Self(e.write_i64_bytes(n), PhantomData) + } + + /// Return the value as a native endian value. + pub fn get(self, e: E) -> i64 { + e.read_i64_bytes(self.0) + } + + /// Set the value given a native endian value. + pub fn set(&mut self, e: E, n: i64) { + self.0 = e.write_i64_bytes(n); + } +} + +impl fmt::Debug for U16Bytes { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "U16({:x}, {:x})", self.0[0], self.0[1],) + } +} + +impl fmt::Debug for U32Bytes { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "U32({:x}, {:x}, {:x}, {:x})", + self.0[0], self.0[1], self.0[2], self.0[3], + ) + } +} + +impl fmt::Debug for U64Bytes { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "U64({:x}, {:x}, {:x}, {:x}, {:x}, {:x}, {:x}, {:x})", + self.0[0], self.0[1], self.0[2], self.0[3], self.0[4], self.0[5], self.0[6], self.0[7], + ) + } +} + +impl fmt::Debug for I16Bytes { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "I16({:x}, {:x})", self.0[0], self.0[1],) + } +} + +impl fmt::Debug for I32Bytes { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "I32({:x}, {:x}, {:x}, {:x})", + self.0[0], self.0[1], self.0[2], self.0[3], + ) + } +} + +impl fmt::Debug for I64Bytes { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "I64({:x}, {:x}, {:x}, {:x}, {:x}, {:x}, {:x}, {:x})", + self.0[0], self.0[1], self.0[2], self.0[3], self.0[4], self.0[5], self.0[6], self.0[7], + ) + } +} + +unsafe_impl_endian_pod!(U16Bytes, U32Bytes, U64Bytes, I16Bytes, I32Bytes, I64Bytes); diff --git a/crux-mir/lib/object/src/lib.rs b/crux-mir/lib/object/src/lib.rs new file mode 100644 index 000000000..d50009f38 --- /dev/null +++ b/crux-mir/lib/object/src/lib.rs @@ -0,0 +1,112 @@ +//! # `object` +//! +//! The `object` crate provides a unified interface to working with object files +//! across platforms. It supports reading object files and executable files, +//! and writing object files and some executable files. +//! +//! ## Raw struct definitions +//! +//! Raw structs are defined for: [ELF](elf), [Mach-O](macho), [PE/COFF](pe), [archive]. +//! Types and traits for zerocopy support are defined in [pod] and [endian]. +//! +//! ## Unified read API +//! +//! The [read::Object] trait defines the unified interace. This trait is implemented +//! by [read::File], which allows reading any file format, as well as implementations +//! for each file format: [ELF](read::elf::ElfFile), [Mach-O](read::macho::MachOFile), +//! [COFF](read::coff::CoffFile), [PE](read::pe::PeFile), [Wasm](read::wasm::WasmFile). +//! +//! ## Low level read API +//! +//! In addition to the unified read API, the various `read` modules define helpers that +//! operate on the raw structs. These also provide traits that abstract over the differences +//! between 32-bit and 64-bit versions of the file format. +//! +//! ## Unified write API +//! +//! [write::Object] allows building a COFF/ELF/Mach-O object and then writing it out. +//! +//! ## Low level executable writers +//! +//! [write::elf::Writer] and [write::pe::Writer] allow writing executable files. +//! +//! ## Example for unified read API +//! ```no_run +//! # #[cfg(feature = "read")] +//! use object::{Object, ObjectSection}; +//! use std::error::Error; +//! use std::fs; +//! +//! /// Reads a file and displays the content of the ".boot" section. +//! fn main() -> Result<(), Box> { +//! # #[cfg(all(feature = "read", feature = "std"))] { +//! let bin_data = fs::read("./multiboot2-binary.elf")?; +//! let obj_file = object::File::parse(&*bin_data)?; +//! if let Some(section) = obj_file.section_by_name(".boot") { +//! println!("{:#x?}", section.data()?); +//! } else { +//! eprintln!("section not available"); +//! } +//! # } +//! Ok(()) +//! } +//! ``` + +#![deny(missing_docs)] +#![deny(missing_debug_implementations)] +#![no_std] +// Style. +#![allow(clippy::collapsible_if)] +#![allow(clippy::comparison_chain)] +#![allow(clippy::match_like_matches_macro)] +#![allow(clippy::single_match)] +#![allow(clippy::type_complexity)] +// Occurs due to fallible iteration. +#![allow(clippy::should_implement_trait)] +// Unit errors are converted to other types by callers. +#![allow(clippy::result_unit_err)] +// Clippy is wrong. +#![allow(clippy::transmute_ptr_to_ptr)] +// Worse readability sometimes. +#![allow(clippy::collapsible_else_if)] + +#[cfg(feature = "cargo-all")] +compile_error!("'--all-features' is not supported; use '--features all' instead"); + +#[cfg(any(feature = "read_core", feature = "write_core"))] +#[allow(unused_imports)] +#[macro_use] +extern crate alloc; + +#[cfg(feature = "std")] +#[allow(unused_imports)] +#[macro_use] +extern crate std; + +mod common; +pub use common::*; + +#[macro_use] +pub mod endian; +pub use endian::*; + +#[macro_use] +pub mod pod; +pub use pod::*; + +#[cfg(feature = "read_core")] +pub mod read; +#[cfg(feature = "read_core")] +pub use read::*; + +#[cfg(feature = "write_core")] +pub mod write; + +#[cfg(feature = "archive")] +pub mod archive; +#[cfg(feature = "elf")] +pub mod elf; +#[cfg(feature = "macho")] +pub mod macho; +#[cfg(any(feature = "coff", feature = "pe"))] +pub mod pe; diff --git a/crux-mir/lib/object/src/macho.rs b/crux-mir/lib/object/src/macho.rs new file mode 100644 index 000000000..86a5bf626 --- /dev/null +++ b/crux-mir/lib/object/src/macho.rs @@ -0,0 +1,3307 @@ +//! Mach-O definitions. +//! +//! These definitions are independent of read/write support, although we do implement +//! some traits useful for those. +//! +//! This module is based heavily on header files from MacOSX11.1.sdk. + +#![allow(missing_docs)] + +use crate::endian::{BigEndian, Endian, U64Bytes, U16, U32, U64}; +use crate::pod::Pod; + +// Definitions from "/usr/include/mach/machine.h". + +/* + * Capability bits used in the definition of cpu_type. + */ + +/// mask for architecture bits +pub const CPU_ARCH_MASK: u32 = 0xff00_0000; +/// 64 bit ABI +pub const CPU_ARCH_ABI64: u32 = 0x0100_0000; +/// ABI for 64-bit hardware with 32-bit types; LP32 +pub const CPU_ARCH_ABI64_32: u32 = 0x0200_0000; + +/* + * Machine types known by all. + */ + +pub const CPU_TYPE_ANY: u32 = !0; + +pub const CPU_TYPE_VAX: u32 = 1; +pub const CPU_TYPE_MC680X0: u32 = 6; +pub const CPU_TYPE_X86: u32 = 7; +pub const CPU_TYPE_X86_64: u32 = CPU_TYPE_X86 | CPU_ARCH_ABI64; +pub const CPU_TYPE_MIPS: u32 = 8; +pub const CPU_TYPE_MC98000: u32 = 10; +pub const CPU_TYPE_HPPA: u32 = 11; +pub const CPU_TYPE_ARM: u32 = 12; +pub const CPU_TYPE_ARM64: u32 = CPU_TYPE_ARM | CPU_ARCH_ABI64; +pub const CPU_TYPE_ARM64_32: u32 = CPU_TYPE_ARM | CPU_ARCH_ABI64_32; +pub const CPU_TYPE_MC88000: u32 = 13; +pub const CPU_TYPE_SPARC: u32 = 14; +pub const CPU_TYPE_I860: u32 = 15; +pub const CPU_TYPE_ALPHA: u32 = 16; +pub const CPU_TYPE_POWERPC: u32 = 18; +pub const CPU_TYPE_POWERPC64: u32 = CPU_TYPE_POWERPC | CPU_ARCH_ABI64; + +/* + * Capability bits used in the definition of cpu_subtype. + */ +/// mask for feature flags +pub const CPU_SUBTYPE_MASK: u32 = 0xff00_0000; +/// 64 bit libraries +pub const CPU_SUBTYPE_LIB64: u32 = 0x8000_0000; +/// pointer authentication with versioned ABI +pub const CPU_SUBTYPE_PTRAUTH_ABI: u32 = 0x8000_0000; + +/// When selecting a slice, ANY will pick the slice with the best +/// grading for the selected cpu_type_t, unlike the "ALL" subtypes, +/// which are the slices that can run on any hardware for that cpu type. +pub const CPU_SUBTYPE_ANY: u32 = !0; + +/* + * Object files that are hand-crafted to run on any + * implementation of an architecture are tagged with + * CPU_SUBTYPE_MULTIPLE. This functions essentially the same as + * the "ALL" subtype of an architecture except that it allows us + * to easily find object files that may need to be modified + * whenever a new implementation of an architecture comes out. + * + * It is the responsibility of the implementor to make sure the + * software handles unsupported implementations elegantly. + */ +pub const CPU_SUBTYPE_MULTIPLE: u32 = !0; +pub const CPU_SUBTYPE_LITTLE_ENDIAN: u32 = 0; +pub const CPU_SUBTYPE_BIG_ENDIAN: u32 = 1; + +/* + * VAX subtypes (these do *not* necessary conform to the actual cpu + * ID assigned by DEC available via the SID register). + */ + +pub const CPU_SUBTYPE_VAX_ALL: u32 = 0; +pub const CPU_SUBTYPE_VAX780: u32 = 1; +pub const CPU_SUBTYPE_VAX785: u32 = 2; +pub const CPU_SUBTYPE_VAX750: u32 = 3; +pub const CPU_SUBTYPE_VAX730: u32 = 4; +pub const CPU_SUBTYPE_UVAXI: u32 = 5; +pub const CPU_SUBTYPE_UVAXII: u32 = 6; +pub const CPU_SUBTYPE_VAX8200: u32 = 7; +pub const CPU_SUBTYPE_VAX8500: u32 = 8; +pub const CPU_SUBTYPE_VAX8600: u32 = 9; +pub const CPU_SUBTYPE_VAX8650: u32 = 10; +pub const CPU_SUBTYPE_VAX8800: u32 = 11; +pub const CPU_SUBTYPE_UVAXIII: u32 = 12; + +/* + * 680x0 subtypes + * + * The subtype definitions here are unusual for historical reasons. + * NeXT used to consider 68030 code as generic 68000 code. For + * backwards compatability: + * + * CPU_SUBTYPE_MC68030 symbol has been preserved for source code + * compatability. + * + * CPU_SUBTYPE_MC680x0_ALL has been defined to be the same + * subtype as CPU_SUBTYPE_MC68030 for binary comatability. + * + * CPU_SUBTYPE_MC68030_ONLY has been added to allow new object + * files to be tagged as containing 68030-specific instructions. + */ + +pub const CPU_SUBTYPE_MC680X0_ALL: u32 = 1; +// compat +pub const CPU_SUBTYPE_MC68030: u32 = 1; +pub const CPU_SUBTYPE_MC68040: u32 = 2; +pub const CPU_SUBTYPE_MC68030_ONLY: u32 = 3; + +/* + * I386 subtypes + */ + +#[inline] +pub const fn cpu_subtype_intel(f: u32, m: u32) -> u32 { + f + (m << 4) +} + +pub const CPU_SUBTYPE_I386_ALL: u32 = cpu_subtype_intel(3, 0); +pub const CPU_SUBTYPE_386: u32 = cpu_subtype_intel(3, 0); +pub const CPU_SUBTYPE_486: u32 = cpu_subtype_intel(4, 0); +pub const CPU_SUBTYPE_486SX: u32 = cpu_subtype_intel(4, 8); +pub const CPU_SUBTYPE_586: u32 = cpu_subtype_intel(5, 0); +pub const CPU_SUBTYPE_PENT: u32 = cpu_subtype_intel(5, 0); +pub const CPU_SUBTYPE_PENTPRO: u32 = cpu_subtype_intel(6, 1); +pub const CPU_SUBTYPE_PENTII_M3: u32 = cpu_subtype_intel(6, 3); +pub const CPU_SUBTYPE_PENTII_M5: u32 = cpu_subtype_intel(6, 5); +pub const CPU_SUBTYPE_CELERON: u32 = cpu_subtype_intel(7, 6); +pub const CPU_SUBTYPE_CELERON_MOBILE: u32 = cpu_subtype_intel(7, 7); +pub const CPU_SUBTYPE_PENTIUM_3: u32 = cpu_subtype_intel(8, 0); +pub const CPU_SUBTYPE_PENTIUM_3_M: u32 = cpu_subtype_intel(8, 1); +pub const CPU_SUBTYPE_PENTIUM_3_XEON: u32 = cpu_subtype_intel(8, 2); +pub const CPU_SUBTYPE_PENTIUM_M: u32 = cpu_subtype_intel(9, 0); +pub const CPU_SUBTYPE_PENTIUM_4: u32 = cpu_subtype_intel(10, 0); +pub const CPU_SUBTYPE_PENTIUM_4_M: u32 = cpu_subtype_intel(10, 1); +pub const CPU_SUBTYPE_ITANIUM: u32 = cpu_subtype_intel(11, 0); +pub const CPU_SUBTYPE_ITANIUM_2: u32 = cpu_subtype_intel(11, 1); +pub const CPU_SUBTYPE_XEON: u32 = cpu_subtype_intel(12, 0); +pub const CPU_SUBTYPE_XEON_MP: u32 = cpu_subtype_intel(12, 1); + +#[inline] +pub const fn cpu_subtype_intel_family(x: u32) -> u32 { + x & 15 +} +pub const CPU_SUBTYPE_INTEL_FAMILY_MAX: u32 = 15; + +#[inline] +pub const fn cpu_subtype_intel_model(x: u32) -> u32 { + x >> 4 +} +pub const CPU_SUBTYPE_INTEL_MODEL_ALL: u32 = 0; + +/* + * X86 subtypes. + */ + +pub const CPU_SUBTYPE_X86_ALL: u32 = 3; +pub const CPU_SUBTYPE_X86_64_ALL: u32 = 3; +pub const CPU_SUBTYPE_X86_ARCH1: u32 = 4; +/// Haswell feature subset +pub const CPU_SUBTYPE_X86_64_H: u32 = 8; + +/* + * Mips subtypes. + */ + +pub const CPU_SUBTYPE_MIPS_ALL: u32 = 0; +pub const CPU_SUBTYPE_MIPS_R2300: u32 = 1; +pub const CPU_SUBTYPE_MIPS_R2600: u32 = 2; +pub const CPU_SUBTYPE_MIPS_R2800: u32 = 3; +/// pmax +pub const CPU_SUBTYPE_MIPS_R2000A: u32 = 4; +pub const CPU_SUBTYPE_MIPS_R2000: u32 = 5; +/// 3max +pub const CPU_SUBTYPE_MIPS_R3000A: u32 = 6; +pub const CPU_SUBTYPE_MIPS_R3000: u32 = 7; + +/* + * MC98000 (PowerPC) subtypes + */ +pub const CPU_SUBTYPE_MC98000_ALL: u32 = 0; +pub const CPU_SUBTYPE_MC98601: u32 = 1; + +/* + * HPPA subtypes for Hewlett-Packard HP-PA family of + * risc processors. Port by NeXT to 700 series. + */ + +pub const CPU_SUBTYPE_HPPA_ALL: u32 = 0; +pub const CPU_SUBTYPE_HPPA_7100LC: u32 = 1; + +/* + * MC88000 subtypes. + */ +pub const CPU_SUBTYPE_MC88000_ALL: u32 = 0; +pub const CPU_SUBTYPE_MC88100: u32 = 1; +pub const CPU_SUBTYPE_MC88110: u32 = 2; + +/* + * SPARC subtypes + */ +pub const CPU_SUBTYPE_SPARC_ALL: u32 = 0; + +/* + * I860 subtypes + */ +pub const CPU_SUBTYPE_I860_ALL: u32 = 0; +pub const CPU_SUBTYPE_I860_860: u32 = 1; + +/* + * PowerPC subtypes + */ +pub const CPU_SUBTYPE_POWERPC_ALL: u32 = 0; +pub const CPU_SUBTYPE_POWERPC_601: u32 = 1; +pub const CPU_SUBTYPE_POWERPC_602: u32 = 2; +pub const CPU_SUBTYPE_POWERPC_603: u32 = 3; +pub const CPU_SUBTYPE_POWERPC_603E: u32 = 4; +pub const CPU_SUBTYPE_POWERPC_603EV: u32 = 5; +pub const CPU_SUBTYPE_POWERPC_604: u32 = 6; +pub const CPU_SUBTYPE_POWERPC_604E: u32 = 7; +pub const CPU_SUBTYPE_POWERPC_620: u32 = 8; +pub const CPU_SUBTYPE_POWERPC_750: u32 = 9; +pub const CPU_SUBTYPE_POWERPC_7400: u32 = 10; +pub const CPU_SUBTYPE_POWERPC_7450: u32 = 11; +pub const CPU_SUBTYPE_POWERPC_970: u32 = 100; + +/* + * ARM subtypes + */ +pub const CPU_SUBTYPE_ARM_ALL: u32 = 0; +pub const CPU_SUBTYPE_ARM_V4T: u32 = 5; +pub const CPU_SUBTYPE_ARM_V6: u32 = 6; +pub const CPU_SUBTYPE_ARM_V5TEJ: u32 = 7; +pub const CPU_SUBTYPE_ARM_XSCALE: u32 = 8; +/// ARMv7-A and ARMv7-R +pub const CPU_SUBTYPE_ARM_V7: u32 = 9; +/// Cortex A9 +pub const CPU_SUBTYPE_ARM_V7F: u32 = 10; +/// Swift +pub const CPU_SUBTYPE_ARM_V7S: u32 = 11; +pub const CPU_SUBTYPE_ARM_V7K: u32 = 12; +pub const CPU_SUBTYPE_ARM_V8: u32 = 13; +/// Not meant to be run under xnu +pub const CPU_SUBTYPE_ARM_V6M: u32 = 14; +/// Not meant to be run under xnu +pub const CPU_SUBTYPE_ARM_V7M: u32 = 15; +/// Not meant to be run under xnu +pub const CPU_SUBTYPE_ARM_V7EM: u32 = 16; +/// Not meant to be run under xnu +pub const CPU_SUBTYPE_ARM_V8M: u32 = 17; + +/* + * ARM64 subtypes + */ +pub const CPU_SUBTYPE_ARM64_ALL: u32 = 0; +pub const CPU_SUBTYPE_ARM64_V8: u32 = 1; +pub const CPU_SUBTYPE_ARM64E: u32 = 2; + +/* + * ARM64_32 subtypes + */ +pub const CPU_SUBTYPE_ARM64_32_ALL: u32 = 0; +pub const CPU_SUBTYPE_ARM64_32_V8: u32 = 1; + +// Definitions from "/usr/include/mach/vm_prot.h". + +/// read permission +pub const VM_PROT_READ: u32 = 0x01; +/// write permission +pub const VM_PROT_WRITE: u32 = 0x02; +/// execute permission +pub const VM_PROT_EXECUTE: u32 = 0x04; + +// Definitions from https://opensource.apple.com/source/dyld/dyld-210.2.3/launch-cache/dyld_cache_format.h.auto.html + +/// The dyld cache header. +/// Corresponds to struct dyld_cache_header from dyld_cache_format.h. +/// This header has grown over time. Only the fields up to and including dyld_base_address +/// are guaranteed to be present. For all other fields, check the header size before +/// accessing the field. The header size is stored in mapping_offset; the mappings start +/// right after the theader. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct DyldCacheHeader { + /// e.g. "dyld_v0 i386" + pub magic: [u8; 16], + /// file offset to first dyld_cache_mapping_info + pub mapping_offset: U32, // offset: 0x10 + /// number of dyld_cache_mapping_info entries + pub mapping_count: U32, // offset: 0x14 + /// file offset to first dyld_cache_image_info + pub images_offset: U32, // offset: 0x18 + /// number of dyld_cache_image_info entries + pub images_count: U32, // offset: 0x1c + /// base address of dyld when cache was built + pub dyld_base_address: U64, // offset: 0x20 + /// + reserved1: [u8; 32], // offset: 0x28 + /// file offset of where local symbols are stored + pub local_symbols_offset: U64, // offset: 0x48 + /// size of local symbols information + pub local_symbols_size: U64, // offset: 0x50 + /// unique value for each shared cache file + pub uuid: [u8; 16], // offset: 0x58 + /// + reserved2: [u8; 32], // offset: 0x68 + /// + reserved3: [u8; 32], // offset: 0x88 + /// + reserved4: [u8; 32], // offset: 0xa8 + /// + reserved5: [u8; 32], // offset: 0xc8 + /// + reserved6: [u8; 32], // offset: 0xe8 + /// + reserved7: [u8; 32], // offset: 0x108 + /// + reserved8: [u8; 32], // offset: 0x128 + /// + reserved9: [u8; 32], // offset: 0x148 + /// + reserved10: [u8; 32], // offset: 0x168 + /// file offset to first dyld_subcache_info + pub subcaches_offset: U32, // offset: 0x188 + /// number of dyld_subcache_info entries + pub subcaches_count: U32, // offset: 0x18c + /// the UUID of the .symbols subcache + pub symbols_subcache_uuid: [u8; 16], // offset: 0x190 + /// + reserved11: [u8; 32], // offset: 0x1a0 + /// file offset to first dyld_cache_image_info + /// Use this instead of images_offset if mapping_offset is at least 0x1c4. + pub images_across_all_subcaches_offset: U32, // offset: 0x1c0 + /// number of dyld_cache_image_info entries + /// Use this instead of images_count if mapping_offset is at least 0x1c4. + pub images_across_all_subcaches_count: U32, // offset: 0x1c4 +} + +/// Corresponds to struct dyld_cache_mapping_info from dyld_cache_format.h. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct DyldCacheMappingInfo { + /// + pub address: U64, + /// + pub size: U64, + /// + pub file_offset: U64, + /// + pub max_prot: U32, + /// + pub init_prot: U32, +} + +/// Corresponds to struct dyld_cache_image_info from dyld_cache_format.h. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct DyldCacheImageInfo { + /// + pub address: U64, + /// + pub mod_time: U64, + /// + pub inode: U64, + /// + pub path_file_offset: U32, + /// + pub pad: U32, +} + +/// Corresponds to a struct whose source code has not been published as of Nov 2021. +/// Added in the dyld cache version which shipped with macOS 12 / iOS 15. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct DyldSubCacheInfo { + /// The UUID of this subcache. + pub uuid: [u8; 16], + /// The size of this subcache plus all previous subcaches. + pub cumulative_size: U64, +} + +// Definitions from "/usr/include/mach-o/loader.h". + +/* + * This header file describes the structures of the file format for "fat" + * architecture specific file (wrapper design). At the begining of the file + * there is one `FatHeader` structure followed by a number of `FatArch*` + * structures. For each architecture in the file, specified by a pair of + * cputype and cpusubtype, the `FatHeader` describes the file offset, file + * size and alignment in the file of the architecture specific member. + * The padded bytes in the file to place each member on it's specific alignment + * are defined to be read as zeros and can be left as "holes" if the file system + * can support them as long as they read as zeros. + * + * All structures defined here are always written and read to/from disk + * in big-endian order. + */ + +pub const FAT_MAGIC: u32 = 0xcafe_babe; +/// NXSwapLong(FAT_MAGIC) +pub const FAT_CIGAM: u32 = 0xbeba_feca; + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct FatHeader { + /// FAT_MAGIC or FAT_MAGIC_64 + pub magic: U32, + /// number of structs that follow + pub nfat_arch: U32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct FatArch32 { + /// cpu specifier (int) + pub cputype: U32, + /// machine specifier (int) + pub cpusubtype: U32, + /// file offset to this object file + pub offset: U32, + /// size of this object file + pub size: U32, + /// alignment as a power of 2 + pub align: U32, +} + +/* + * The support for the 64-bit fat file format described here is a work in + * progress and not yet fully supported in all the Apple Developer Tools. + * + * When a slice is greater than 4mb or an offset to a slice is greater than 4mb + * then the 64-bit fat file format is used. + */ +pub const FAT_MAGIC_64: u32 = 0xcafe_babf; +/// NXSwapLong(FAT_MAGIC_64) +pub const FAT_CIGAM_64: u32 = 0xbfba_feca; + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct FatArch64 { + /// cpu specifier (int) + pub cputype: U32, + /// machine specifier (int) + pub cpusubtype: U32, + /// file offset to this object file + pub offset: U64, + /// size of this object file + pub size: U64, + /// alignment as a power of 2 + pub align: U32, + /// reserved + pub reserved: U32, +} + +// Definitions from "/usr/include/mach-o/loader.h". + +/// The 32-bit mach header. +/// +/// Appears at the very beginning of the object file for 32-bit architectures. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct MachHeader32 { + /// mach magic number identifier + pub magic: U32, + /// cpu specifier + pub cputype: U32, + /// machine specifier + pub cpusubtype: U32, + /// type of file + pub filetype: U32, + /// number of load commands + pub ncmds: U32, + /// the size of all the load commands + pub sizeofcmds: U32, + /// flags + pub flags: U32, +} + +// Values for `MachHeader32::magic`. +/// the mach magic number +pub const MH_MAGIC: u32 = 0xfeed_face; +/// NXSwapInt(MH_MAGIC) +pub const MH_CIGAM: u32 = 0xcefa_edfe; + +/// The 64-bit mach header. +/// +/// Appears at the very beginning of object files for 64-bit architectures. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct MachHeader64 { + /// mach magic number identifier + pub magic: U32, + /// cpu specifier + pub cputype: U32, + /// machine specifier + pub cpusubtype: U32, + /// type of file + pub filetype: U32, + /// number of load commands + pub ncmds: U32, + /// the size of all the load commands + pub sizeofcmds: U32, + /// flags + pub flags: U32, + /// reserved + pub reserved: U32, +} + +// Values for `MachHeader64::magic`. +/// the 64-bit mach magic number +pub const MH_MAGIC_64: u32 = 0xfeed_facf; +/// NXSwapInt(MH_MAGIC_64) +pub const MH_CIGAM_64: u32 = 0xcffa_edfe; + +/* + * The layout of the file depends on the filetype. For all but the MH_OBJECT + * file type the segments are padded out and aligned on a segment alignment + * boundary for efficient demand pageing. The MH_EXECUTE, MH_FVMLIB, MH_DYLIB, + * MH_DYLINKER and MH_BUNDLE file types also have the headers included as part + * of their first segment. + * + * The file type MH_OBJECT is a compact format intended as output of the + * assembler and input (and possibly output) of the link editor (the .o + * format). All sections are in one unnamed segment with no segment padding. + * This format is used as an executable format when the file is so small the + * segment padding greatly increases its size. + * + * The file type MH_PRELOAD is an executable format intended for things that + * are not executed under the kernel (proms, stand alones, kernels, etc). The + * format can be executed under the kernel but may demand paged it and not + * preload it before execution. + * + * A core file is in MH_CORE format and can be any in an arbritray legal + * Mach-O file. + */ + +// Values for `MachHeader*::filetype`. +/// relocatable object file +pub const MH_OBJECT: u32 = 0x1; +/// demand paged executable file +pub const MH_EXECUTE: u32 = 0x2; +/// fixed VM shared library file +pub const MH_FVMLIB: u32 = 0x3; +/// core file +pub const MH_CORE: u32 = 0x4; +/// preloaded executable file +pub const MH_PRELOAD: u32 = 0x5; +/// dynamically bound shared library +pub const MH_DYLIB: u32 = 0x6; +/// dynamic link editor +pub const MH_DYLINKER: u32 = 0x7; +/// dynamically bound bundle file +pub const MH_BUNDLE: u32 = 0x8; +/// shared library stub for static linking only, no section contents +pub const MH_DYLIB_STUB: u32 = 0x9; +/// companion file with only debug sections +pub const MH_DSYM: u32 = 0xa; +/// x86_64 kexts +pub const MH_KEXT_BUNDLE: u32 = 0xb; +/// set of mach-o's +pub const MH_FILESET: u32 = 0xc; + +// Values for `MachHeader*::flags`. +/// the object file has no undefined references +pub const MH_NOUNDEFS: u32 = 0x1; +/// the object file is the output of an incremental link against a base file and can't be link edited again +pub const MH_INCRLINK: u32 = 0x2; +/// the object file is input for the dynamic linker and can't be staticly link edited again +pub const MH_DYLDLINK: u32 = 0x4; +/// the object file's undefined references are bound by the dynamic linker when loaded. +pub const MH_BINDATLOAD: u32 = 0x8; +/// the file has its dynamic undefined references prebound. +pub const MH_PREBOUND: u32 = 0x10; +/// the file has its read-only and read-write segments split +pub const MH_SPLIT_SEGS: u32 = 0x20; +/// the shared library init routine is to be run lazily via catching memory faults to its writeable segments (obsolete) +pub const MH_LAZY_INIT: u32 = 0x40; +/// the image is using two-level name space bindings +pub const MH_TWOLEVEL: u32 = 0x80; +/// the executable is forcing all images to use flat name space bindings +pub const MH_FORCE_FLAT: u32 = 0x100; +/// this umbrella guarantees no multiple defintions of symbols in its sub-images so the two-level namespace hints can always be used. +pub const MH_NOMULTIDEFS: u32 = 0x200; +/// do not have dyld notify the prebinding agent about this executable +pub const MH_NOFIXPREBINDING: u32 = 0x400; +/// the binary is not prebound but can have its prebinding redone. only used when MH_PREBOUND is not set. +pub const MH_PREBINDABLE: u32 = 0x800; +/// indicates that this binary binds to all two-level namespace modules of its dependent libraries. only used when MH_PREBINDABLE and MH_TWOLEVEL are both set. +pub const MH_ALLMODSBOUND: u32 = 0x1000; +/// safe to divide up the sections into sub-sections via symbols for dead code stripping +pub const MH_SUBSECTIONS_VIA_SYMBOLS: u32 = 0x2000; +/// the binary has been canonicalized via the unprebind operation +pub const MH_CANONICAL: u32 = 0x4000; +/// the final linked image contains external weak symbols +pub const MH_WEAK_DEFINES: u32 = 0x8000; +/// the final linked image uses weak symbols +pub const MH_BINDS_TO_WEAK: u32 = 0x10000; +/// When this bit is set, all stacks in the task will be given stack execution privilege. Only used in MH_EXECUTE filetypes. +pub const MH_ALLOW_STACK_EXECUTION: u32 = 0x20000; +/// When this bit is set, the binary declares it is safe for use in processes with uid zero +pub const MH_ROOT_SAFE: u32 = 0x40000; +/// When this bit is set, the binary declares it is safe for use in processes when issetugid() is true +pub const MH_SETUID_SAFE: u32 = 0x80000; +/// When this bit is set on a dylib, the static linker does not need to examine dependent dylibs to see if any are re-exported +pub const MH_NO_REEXPORTED_DYLIBS: u32 = 0x10_0000; +/// When this bit is set, the OS will load the main executable at a random address. Only used in MH_EXECUTE filetypes. +pub const MH_PIE: u32 = 0x20_0000; +/// Only for use on dylibs. When linking against a dylib that has this bit set, the static linker will automatically not create a LC_LOAD_DYLIB load command to the dylib if no symbols are being referenced from the dylib. +pub const MH_DEAD_STRIPPABLE_DYLIB: u32 = 0x40_0000; +/// Contains a section of type S_THREAD_LOCAL_VARIABLES +pub const MH_HAS_TLV_DESCRIPTORS: u32 = 0x80_0000; +/// When this bit is set, the OS will run the main executable with a non-executable heap even on platforms (e.g. i386) that don't require it. Only used in MH_EXECUTE filetypes. +pub const MH_NO_HEAP_EXECUTION: u32 = 0x100_0000; +/// The code was linked for use in an application extension. +pub const MH_APP_EXTENSION_SAFE: u32 = 0x0200_0000; +/// The external symbols listed in the nlist symbol table do not include all the symbols listed in the dyld info. +pub const MH_NLIST_OUTOFSYNC_WITH_DYLDINFO: u32 = 0x0400_0000; +/// Allow LC_MIN_VERSION_MACOS and LC_BUILD_VERSION load commands with +/// the platforms macOS, iOSMac, iOSSimulator, tvOSSimulator and watchOSSimulator. +pub const MH_SIM_SUPPORT: u32 = 0x0800_0000; +/// Only for use on dylibs. When this bit is set, the dylib is part of the dyld +/// shared cache, rather than loose in the filesystem. +pub const MH_DYLIB_IN_CACHE: u32 = 0x8000_0000; + +/// Common fields at the start of every load command. +/// +/// The load commands directly follow the mach_header. The total size of all +/// of the commands is given by the sizeofcmds field in the mach_header. All +/// load commands must have as their first two fields `cmd` and `cmdsize`. The `cmd` +/// field is filled in with a constant for that command type. Each command type +/// has a structure specifically for it. The `cmdsize` field is the size in bytes +/// of the particular load command structure plus anything that follows it that +/// is a part of the load command (i.e. section structures, strings, etc.). To +/// advance to the next load command the `cmdsize` can be added to the offset or +/// pointer of the current load command. The `cmdsize` for 32-bit architectures +/// MUST be a multiple of 4 bytes and for 64-bit architectures MUST be a multiple +/// of 8 bytes (these are forever the maximum alignment of any load commands). +/// The padded bytes must be zero. All tables in the object file must also +/// follow these rules so the file can be memory mapped. Otherwise the pointers +/// to these tables will not work well or at all on some machines. With all +/// padding zeroed like objects will compare byte for byte. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct LoadCommand { + /// Type of load command. + /// + /// One of the `LC_*` constants. + pub cmd: U32, + /// Total size of command in bytes. + pub cmdsize: U32, +} + +/* + * After MacOS X 10.1 when a new load command is added that is required to be + * understood by the dynamic linker for the image to execute properly the + * LC_REQ_DYLD bit will be or'ed into the load command constant. If the dynamic + * linker sees such a load command it it does not understand will issue a + * "unknown load command required for execution" error and refuse to use the + * image. Other load commands without this bit that are not understood will + * simply be ignored. + */ +pub const LC_REQ_DYLD: u32 = 0x8000_0000; + +/* Constants for the cmd field of all load commands, the type */ +/// segment of this file to be mapped +pub const LC_SEGMENT: u32 = 0x1; +/// link-edit stab symbol table info +pub const LC_SYMTAB: u32 = 0x2; +/// link-edit gdb symbol table info (obsolete) +pub const LC_SYMSEG: u32 = 0x3; +/// thread +pub const LC_THREAD: u32 = 0x4; +/// unix thread (includes a stack) +pub const LC_UNIXTHREAD: u32 = 0x5; +/// load a specified fixed VM shared library +pub const LC_LOADFVMLIB: u32 = 0x6; +/// fixed VM shared library identification +pub const LC_IDFVMLIB: u32 = 0x7; +/// object identification info (obsolete) +pub const LC_IDENT: u32 = 0x8; +/// fixed VM file inclusion (internal use) +pub const LC_FVMFILE: u32 = 0x9; +/// prepage command (internal use) +pub const LC_PREPAGE: u32 = 0xa; +/// dynamic link-edit symbol table info +pub const LC_DYSYMTAB: u32 = 0xb; +/// load a dynamically linked shared library +pub const LC_LOAD_DYLIB: u32 = 0xc; +/// dynamically linked shared lib ident +pub const LC_ID_DYLIB: u32 = 0xd; +/// load a dynamic linker +pub const LC_LOAD_DYLINKER: u32 = 0xe; +/// dynamic linker identification +pub const LC_ID_DYLINKER: u32 = 0xf; +/// modules prebound for a dynamically linked shared library +pub const LC_PREBOUND_DYLIB: u32 = 0x10; +/// image routines +pub const LC_ROUTINES: u32 = 0x11; +/// sub framework +pub const LC_SUB_FRAMEWORK: u32 = 0x12; +/// sub umbrella +pub const LC_SUB_UMBRELLA: u32 = 0x13; +/// sub client +pub const LC_SUB_CLIENT: u32 = 0x14; +/// sub library +pub const LC_SUB_LIBRARY: u32 = 0x15; +/// two-level namespace lookup hints +pub const LC_TWOLEVEL_HINTS: u32 = 0x16; +/// prebind checksum +pub const LC_PREBIND_CKSUM: u32 = 0x17; +/// load a dynamically linked shared library that is allowed to be missing +/// (all symbols are weak imported). +pub const LC_LOAD_WEAK_DYLIB: u32 = 0x18 | LC_REQ_DYLD; +/// 64-bit segment of this file to be mapped +pub const LC_SEGMENT_64: u32 = 0x19; +/// 64-bit image routines +pub const LC_ROUTINES_64: u32 = 0x1a; +/// the uuid +pub const LC_UUID: u32 = 0x1b; +/// runpath additions +pub const LC_RPATH: u32 = 0x1c | LC_REQ_DYLD; +/// local of code signature +pub const LC_CODE_SIGNATURE: u32 = 0x1d; +/// local of info to split segments +pub const LC_SEGMENT_SPLIT_INFO: u32 = 0x1e; +/// load and re-export dylib +pub const LC_REEXPORT_DYLIB: u32 = 0x1f | LC_REQ_DYLD; +/// delay load of dylib until first use +pub const LC_LAZY_LOAD_DYLIB: u32 = 0x20; +/// encrypted segment information +pub const LC_ENCRYPTION_INFO: u32 = 0x21; +/// compressed dyld information +pub const LC_DYLD_INFO: u32 = 0x22; +/// compressed dyld information only +pub const LC_DYLD_INFO_ONLY: u32 = 0x22 | LC_REQ_DYLD; +/// load upward dylib +pub const LC_LOAD_UPWARD_DYLIB: u32 = 0x23 | LC_REQ_DYLD; +/// build for MacOSX min OS version +pub const LC_VERSION_MIN_MACOSX: u32 = 0x24; +/// build for iPhoneOS min OS version +pub const LC_VERSION_MIN_IPHONEOS: u32 = 0x25; +/// compressed table of function start addresses +pub const LC_FUNCTION_STARTS: u32 = 0x26; +/// string for dyld to treat like environment variable +pub const LC_DYLD_ENVIRONMENT: u32 = 0x27; +/// replacement for LC_UNIXTHREAD +pub const LC_MAIN: u32 = 0x28 | LC_REQ_DYLD; +/// table of non-instructions in __text +pub const LC_DATA_IN_CODE: u32 = 0x29; +/// source version used to build binary +pub const LC_SOURCE_VERSION: u32 = 0x2A; +/// Code signing DRs copied from linked dylibs +pub const LC_DYLIB_CODE_SIGN_DRS: u32 = 0x2B; +/// 64-bit encrypted segment information +pub const LC_ENCRYPTION_INFO_64: u32 = 0x2C; +/// linker options in MH_OBJECT files +pub const LC_LINKER_OPTION: u32 = 0x2D; +/// optimization hints in MH_OBJECT files +pub const LC_LINKER_OPTIMIZATION_HINT: u32 = 0x2E; +/// build for AppleTV min OS version +pub const LC_VERSION_MIN_TVOS: u32 = 0x2F; +/// build for Watch min OS version +pub const LC_VERSION_MIN_WATCHOS: u32 = 0x30; +/// arbitrary data included within a Mach-O file +pub const LC_NOTE: u32 = 0x31; +/// build for platform min OS version +pub const LC_BUILD_VERSION: u32 = 0x32; +/// used with `LinkeditDataCommand`, payload is trie +pub const LC_DYLD_EXPORTS_TRIE: u32 = 0x33 | LC_REQ_DYLD; +/// used with `LinkeditDataCommand` +pub const LC_DYLD_CHAINED_FIXUPS: u32 = 0x34 | LC_REQ_DYLD; +/// used with `FilesetEntryCommand` +pub const LC_FILESET_ENTRY: u32 = 0x35 | LC_REQ_DYLD; + +/// A variable length string in a load command. +/// +/// The strings are stored just after the load command structure and +/// the offset is from the start of the load command structure. The size +/// of the string is reflected in the `cmdsize` field of the load command. +/// Once again any padded bytes to bring the `cmdsize` field to a multiple +/// of 4 bytes must be zero. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct LcStr { + /// offset to the string + pub offset: U32, +} + +/// 32-bit segment load command. +/// +/// The segment load command indicates that a part of this file is to be +/// mapped into the task's address space. The size of this segment in memory, +/// vmsize, maybe equal to or larger than the amount to map from this file, +/// filesize. The file is mapped starting at fileoff to the beginning of +/// the segment in memory, vmaddr. The rest of the memory of the segment, +/// if any, is allocated zero fill on demand. The segment's maximum virtual +/// memory protection and initial virtual memory protection are specified +/// by the maxprot and initprot fields. If the segment has sections then the +/// `Section32` structures directly follow the segment command and their size is +/// reflected in `cmdsize`. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct SegmentCommand32 { + /// LC_SEGMENT + pub cmd: U32, + /// includes sizeof section structs + pub cmdsize: U32, + /// segment name + pub segname: [u8; 16], + /// memory address of this segment + pub vmaddr: U32, + /// memory size of this segment + pub vmsize: U32, + /// file offset of this segment + pub fileoff: U32, + /// amount to map from the file + pub filesize: U32, + /// maximum VM protection + pub maxprot: U32, + /// initial VM protection + pub initprot: U32, + /// number of sections in segment + pub nsects: U32, + /// flags + pub flags: U32, +} + +/// 64-bit segment load command. +/// +/// The 64-bit segment load command indicates that a part of this file is to be +/// mapped into a 64-bit task's address space. If the 64-bit segment has +/// sections then `Section64` structures directly follow the 64-bit segment +/// command and their size is reflected in `cmdsize`. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct SegmentCommand64 { + /// LC_SEGMENT_64 + pub cmd: U32, + /// includes sizeof section_64 structs + pub cmdsize: U32, + /// segment name + pub segname: [u8; 16], + /// memory address of this segment + pub vmaddr: U64, + /// memory size of this segment + pub vmsize: U64, + /// file offset of this segment + pub fileoff: U64, + /// amount to map from the file + pub filesize: U64, + /// maximum VM protection + pub maxprot: U32, + /// initial VM protection + pub initprot: U32, + /// number of sections in segment + pub nsects: U32, + /// flags + pub flags: U32, +} + +// Values for `SegmentCommand*::flags`. +/// the file contents for this segment is for the high part of the VM space, the low part is zero filled (for stacks in core files) +pub const SG_HIGHVM: u32 = 0x1; +/// this segment is the VM that is allocated by a fixed VM library, for overlap checking in the link editor +pub const SG_FVMLIB: u32 = 0x2; +/// this segment has nothing that was relocated in it and nothing relocated to it, that is it maybe safely replaced without relocation +pub const SG_NORELOC: u32 = 0x4; +/// This segment is protected. If the segment starts at file offset 0, the first page of the segment is not protected. All other pages of the segment are protected. +pub const SG_PROTECTED_VERSION_1: u32 = 0x8; +/// This segment is made read-only after fixups +pub const SG_READ_ONLY: u32 = 0x10; + +/* + * A segment is made up of zero or more sections. Non-MH_OBJECT files have + * all of their segments with the proper sections in each, and padded to the + * specified segment alignment when produced by the link editor. The first + * segment of a MH_EXECUTE and MH_FVMLIB format file contains the mach_header + * and load commands of the object file before its first section. The zero + * fill sections are always last in their segment (in all formats). This + * allows the zeroed segment padding to be mapped into memory where zero fill + * sections might be. The gigabyte zero fill sections, those with the section + * type S_GB_ZEROFILL, can only be in a segment with sections of this type. + * These segments are then placed after all other segments. + * + * The MH_OBJECT format has all of its sections in one segment for + * compactness. There is no padding to a specified segment boundary and the + * mach_header and load commands are not part of the segment. + * + * Sections with the same section name, sectname, going into the same segment, + * segname, are combined by the link editor. The resulting section is aligned + * to the maximum alignment of the combined sections and is the new section's + * alignment. The combined sections are aligned to their original alignment in + * the combined section. Any padded bytes to get the specified alignment are + * zeroed. + * + * The format of the relocation entries referenced by the reloff and nreloc + * fields of the section structure for mach object files is described in the + * header file . + */ +/// 32-bit section. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Section32 { + /// name of this section + pub sectname: [u8; 16], + /// segment this section goes in + pub segname: [u8; 16], + /// memory address of this section + pub addr: U32, + /// size in bytes of this section + pub size: U32, + /// file offset of this section + pub offset: U32, + /// section alignment (power of 2) + pub align: U32, + /// file offset of relocation entries + pub reloff: U32, + /// number of relocation entries + pub nreloc: U32, + /// flags (section type and attributes) + pub flags: U32, + /// reserved (for offset or index) + pub reserved1: U32, + /// reserved (for count or sizeof) + pub reserved2: U32, +} + +/// 64-bit section. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Section64 { + /// name of this section + pub sectname: [u8; 16], + /// segment this section goes in + pub segname: [u8; 16], + /// memory address of this section + pub addr: U64, + /// size in bytes of this section + pub size: U64, + /// file offset of this section + pub offset: U32, + /// section alignment (power of 2) + pub align: U32, + /// file offset of relocation entries + pub reloff: U32, + /// number of relocation entries + pub nreloc: U32, + /// flags (section type and attributes) + pub flags: U32, + /// reserved (for offset or index) + pub reserved1: U32, + /// reserved (for count or sizeof) + pub reserved2: U32, + /// reserved + pub reserved3: U32, +} + +/* + * The flags field of a section structure is separated into two parts a section + * type and section attributes. The section types are mutually exclusive (it + * can only have one type) but the section attributes are not (it may have more + * than one attribute). + */ +/// 256 section types +pub const SECTION_TYPE: u32 = 0x0000_00ff; +/// 24 section attributes +pub const SECTION_ATTRIBUTES: u32 = 0xffff_ff00; + +/* Constants for the type of a section */ +/// regular section +pub const S_REGULAR: u32 = 0x0; +/// zero fill on demand section +pub const S_ZEROFILL: u32 = 0x1; +/// section with only literal C strings +pub const S_CSTRING_LITERALS: u32 = 0x2; +/// section with only 4 byte literals +pub const S_4BYTE_LITERALS: u32 = 0x3; +/// section with only 8 byte literals +pub const S_8BYTE_LITERALS: u32 = 0x4; +/// section with only pointers to literals +pub const S_LITERAL_POINTERS: u32 = 0x5; +/* + * For the two types of symbol pointers sections and the symbol stubs section + * they have indirect symbol table entries. For each of the entries in the + * section the indirect symbol table entries, in corresponding order in the + * indirect symbol table, start at the index stored in the reserved1 field + * of the section structure. Since the indirect symbol table entries + * correspond to the entries in the section the number of indirect symbol table + * entries is inferred from the size of the section divided by the size of the + * entries in the section. For symbol pointers sections the size of the entries + * in the section is 4 bytes and for symbol stubs sections the byte size of the + * stubs is stored in the reserved2 field of the section structure. + */ +/// section with only non-lazy symbol pointers +pub const S_NON_LAZY_SYMBOL_POINTERS: u32 = 0x6; +/// section with only lazy symbol pointers +pub const S_LAZY_SYMBOL_POINTERS: u32 = 0x7; +/// section with only symbol stubs, byte size of stub in the reserved2 field +pub const S_SYMBOL_STUBS: u32 = 0x8; +/// section with only function pointers for initialization +pub const S_MOD_INIT_FUNC_POINTERS: u32 = 0x9; +/// section with only function pointers for termination +pub const S_MOD_TERM_FUNC_POINTERS: u32 = 0xa; +/// section contains symbols that are to be coalesced +pub const S_COALESCED: u32 = 0xb; +/// zero fill on demand section (that can be larger than 4 gigabytes) +pub const S_GB_ZEROFILL: u32 = 0xc; +/// section with only pairs of function pointers for interposing +pub const S_INTERPOSING: u32 = 0xd; +/// section with only 16 byte literals +pub const S_16BYTE_LITERALS: u32 = 0xe; +/// section contains DTrace Object Format +pub const S_DTRACE_DOF: u32 = 0xf; +/// section with only lazy symbol pointers to lazy loaded dylibs +pub const S_LAZY_DYLIB_SYMBOL_POINTERS: u32 = 0x10; +/* + * Section types to support thread local variables + */ +/// template of initial values for TLVs +pub const S_THREAD_LOCAL_REGULAR: u32 = 0x11; +/// template of initial values for TLVs +pub const S_THREAD_LOCAL_ZEROFILL: u32 = 0x12; +/// TLV descriptors +pub const S_THREAD_LOCAL_VARIABLES: u32 = 0x13; +/// pointers to TLV descriptors +pub const S_THREAD_LOCAL_VARIABLE_POINTERS: u32 = 0x14; +/// functions to call to initialize TLV values +pub const S_THREAD_LOCAL_INIT_FUNCTION_POINTERS: u32 = 0x15; +/// 32-bit offsets to initializers +pub const S_INIT_FUNC_OFFSETS: u32 = 0x16; + +/* + * Constants for the section attributes part of the flags field of a section + * structure. + */ +/// User setable attributes +pub const SECTION_ATTRIBUTES_USR: u32 = 0xff00_0000; +/// section contains only true machine instructions +pub const S_ATTR_PURE_INSTRUCTIONS: u32 = 0x8000_0000; +/// section contains coalesced symbols that are not to be in a ranlib table of contents +pub const S_ATTR_NO_TOC: u32 = 0x4000_0000; +/// ok to strip static symbols in this section in files with the MH_DYLDLINK flag +pub const S_ATTR_STRIP_STATIC_SYMS: u32 = 0x2000_0000; +/// no dead stripping +pub const S_ATTR_NO_DEAD_STRIP: u32 = 0x1000_0000; +/// blocks are live if they reference live blocks +pub const S_ATTR_LIVE_SUPPORT: u32 = 0x0800_0000; +/// Used with i386 code stubs written on by dyld +pub const S_ATTR_SELF_MODIFYING_CODE: u32 = 0x0400_0000; +/* + * If a segment contains any sections marked with S_ATTR_DEBUG then all + * sections in that segment must have this attribute. No section other than + * a section marked with this attribute may reference the contents of this + * section. A section with this attribute may contain no symbols and must have + * a section type S_REGULAR. The static linker will not copy section contents + * from sections with this attribute into its output file. These sections + * generally contain DWARF debugging info. + */ +/// a debug section +pub const S_ATTR_DEBUG: u32 = 0x0200_0000; +/// system setable attributes +pub const SECTION_ATTRIBUTES_SYS: u32 = 0x00ff_ff00; +/// section contains some machine instructions +pub const S_ATTR_SOME_INSTRUCTIONS: u32 = 0x0000_0400; +/// section has external relocation entries +pub const S_ATTR_EXT_RELOC: u32 = 0x0000_0200; +/// section has local relocation entries +pub const S_ATTR_LOC_RELOC: u32 = 0x0000_0100; + +/* + * The names of segments and sections in them are mostly meaningless to the + * link-editor. But there are few things to support traditional UNIX + * executables that require the link-editor and assembler to use some names + * agreed upon by convention. + * + * The initial protection of the "__TEXT" segment has write protection turned + * off (not writeable). + * + * The link-editor will allocate common symbols at the end of the "__common" + * section in the "__DATA" segment. It will create the section and segment + * if needed. + */ + +/* The currently known segment names and the section names in those segments */ + +/// the pagezero segment which has no protections and catches NULL references for MH_EXECUTE files +pub const SEG_PAGEZERO: &str = "__PAGEZERO"; + +/// the tradition UNIX text segment +pub const SEG_TEXT: &str = "__TEXT"; +/// the real text part of the text section no headers, and no padding +pub const SECT_TEXT: &str = "__text"; +/// the fvmlib initialization section +pub const SECT_FVMLIB_INIT0: &str = "__fvmlib_init0"; +/// the section following the fvmlib initialization section +pub const SECT_FVMLIB_INIT1: &str = "__fvmlib_init1"; + +/// the tradition UNIX data segment +pub const SEG_DATA: &str = "__DATA"; +/// the real initialized data section no padding, no bss overlap +pub const SECT_DATA: &str = "__data"; +/// the real uninitialized data section no padding +pub const SECT_BSS: &str = "__bss"; +/// the section common symbols are allocated in by the link editor +pub const SECT_COMMON: &str = "__common"; + +/// objective-C runtime segment +pub const SEG_OBJC: &str = "__OBJC"; +/// symbol table +pub const SECT_OBJC_SYMBOLS: &str = "__symbol_table"; +/// module information +pub const SECT_OBJC_MODULES: &str = "__module_info"; +/// string table +pub const SECT_OBJC_STRINGS: &str = "__selector_strs"; +/// string table +pub const SECT_OBJC_REFS: &str = "__selector_refs"; + +/// the icon segment +pub const SEG_ICON: &str = "__ICON"; +/// the icon headers +pub const SECT_ICON_HEADER: &str = "__header"; +/// the icons in tiff format +pub const SECT_ICON_TIFF: &str = "__tiff"; + +/// the segment containing all structs created and maintained by the link editor. Created with -seglinkedit option to ld(1) for MH_EXECUTE and FVMLIB file types only +pub const SEG_LINKEDIT: &str = "__LINKEDIT"; + +/// the segment overlapping with linkedit containing linking information +pub const SEG_LINKINFO: &str = "__LINKINFO"; + +/// the unix stack segment +pub const SEG_UNIXSTACK: &str = "__UNIXSTACK"; + +/// the segment for the self (dyld) modifing code stubs that has read, write and execute permissions +pub const SEG_IMPORT: &str = "__IMPORT"; + +/* + * Fixed virtual memory shared libraries are identified by two things. The + * target pathname (the name of the library as found for execution), and the + * minor version number. The address of where the headers are loaded is in + * header_addr. (THIS IS OBSOLETE and no longer supported). + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Fvmlib { + /// library's target pathname + pub name: LcStr, + /// library's minor version number + pub minor_version: U32, + /// library's header address + pub header_addr: U32, +} + +/* + * A fixed virtual shared library (filetype == MH_FVMLIB in the mach header) + * contains a `FvmlibCommand` (cmd == LC_IDFVMLIB) to identify the library. + * An object that uses a fixed virtual shared library also contains a + * `FvmlibCommand` (cmd == LC_LOADFVMLIB) for each library it uses. + * (THIS IS OBSOLETE and no longer supported). + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct FvmlibCommand { + /// LC_IDFVMLIB or LC_LOADFVMLIB + pub cmd: U32, + /// includes pathname string + pub cmdsize: U32, + /// the library identification + pub fvmlib: Fvmlib, +} + +/* + * Dynamicly linked shared libraries are identified by two things. The + * pathname (the name of the library as found for execution), and the + * compatibility version number. The pathname must match and the compatibility + * number in the user of the library must be greater than or equal to the + * library being used. The time stamp is used to record the time a library was + * built and copied into user so it can be use to determined if the library used + * at runtime is exactly the same as used to built the program. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Dylib { + /// library's path name + pub name: LcStr, + /// library's build time stamp + pub timestamp: U32, + /// library's current version number + pub current_version: U32, + /// library's compatibility vers number + pub compatibility_version: U32, +} + +/* + * A dynamically linked shared library (filetype == MH_DYLIB in the mach header) + * contains a `DylibCommand` (cmd == LC_ID_DYLIB) to identify the library. + * An object that uses a dynamically linked shared library also contains a + * `DylibCommand` (cmd == LC_LOAD_DYLIB, LC_LOAD_WEAK_DYLIB, or + * LC_REEXPORT_DYLIB) for each library it uses. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct DylibCommand { + /// LC_ID_DYLIB, LC_LOAD_{,WEAK_}DYLIB, LC_REEXPORT_DYLIB + pub cmd: U32, + /// includes pathname string + pub cmdsize: U32, + /// the library identification + pub dylib: Dylib, +} + +/* + * A dynamically linked shared library may be a subframework of an umbrella + * framework. If so it will be linked with "-umbrella umbrella_name" where + * Where "umbrella_name" is the name of the umbrella framework. A subframework + * can only be linked against by its umbrella framework or other subframeworks + * that are part of the same umbrella framework. Otherwise the static link + * editor produces an error and states to link against the umbrella framework. + * The name of the umbrella framework for subframeworks is recorded in the + * following structure. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct SubFrameworkCommand { + /// LC_SUB_FRAMEWORK + pub cmd: U32, + /// includes umbrella string + pub cmdsize: U32, + /// the umbrella framework name + pub umbrella: LcStr, +} + +/* + * For dynamically linked shared libraries that are subframework of an umbrella + * framework they can allow clients other than the umbrella framework or other + * subframeworks in the same umbrella framework. To do this the subframework + * is built with "-allowable_client client_name" and an LC_SUB_CLIENT load + * command is created for each -allowable_client flag. The client_name is + * usually a framework name. It can also be a name used for bundles clients + * where the bundle is built with "-client_name client_name". + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct SubClientCommand { + /// LC_SUB_CLIENT + pub cmd: U32, + /// includes client string + pub cmdsize: U32, + /// the client name + pub client: LcStr, +} + +/* + * A dynamically linked shared library may be a sub_umbrella of an umbrella + * framework. If so it will be linked with "-sub_umbrella umbrella_name" where + * Where "umbrella_name" is the name of the sub_umbrella framework. When + * staticly linking when -twolevel_namespace is in effect a twolevel namespace + * umbrella framework will only cause its subframeworks and those frameworks + * listed as sub_umbrella frameworks to be implicited linked in. Any other + * dependent dynamic libraries will not be linked it when -twolevel_namespace + * is in effect. The primary library recorded by the static linker when + * resolving a symbol in these libraries will be the umbrella framework. + * Zero or more sub_umbrella frameworks may be use by an umbrella framework. + * The name of a sub_umbrella framework is recorded in the following structure. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct SubUmbrellaCommand { + /// LC_SUB_UMBRELLA + pub cmd: U32, + /// includes sub_umbrella string + pub cmdsize: U32, + /// the sub_umbrella framework name + pub sub_umbrella: LcStr, +} + +/* + * A dynamically linked shared library may be a sub_library of another shared + * library. If so it will be linked with "-sub_library library_name" where + * Where "library_name" is the name of the sub_library shared library. When + * staticly linking when -twolevel_namespace is in effect a twolevel namespace + * shared library will only cause its subframeworks and those frameworks + * listed as sub_umbrella frameworks and libraries listed as sub_libraries to + * be implicited linked in. Any other dependent dynamic libraries will not be + * linked it when -twolevel_namespace is in effect. The primary library + * recorded by the static linker when resolving a symbol in these libraries + * will be the umbrella framework (or dynamic library). Zero or more sub_library + * shared libraries may be use by an umbrella framework or (or dynamic library). + * The name of a sub_library framework is recorded in the following structure. + * For example /usr/lib/libobjc_profile.A.dylib would be recorded as "libobjc". + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct SubLibraryCommand { + /// LC_SUB_LIBRARY + pub cmd: U32, + /// includes sub_library string + pub cmdsize: U32, + /// the sub_library name + pub sub_library: LcStr, +} + +/* + * A program (filetype == MH_EXECUTE) that is + * prebound to its dynamic libraries has one of these for each library that + * the static linker used in prebinding. It contains a bit vector for the + * modules in the library. The bits indicate which modules are bound (1) and + * which are not (0) from the library. The bit for module 0 is the low bit + * of the first byte. So the bit for the Nth module is: + * (linked_modules[N/8] >> N%8) & 1 + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct PreboundDylibCommand { + /// LC_PREBOUND_DYLIB + pub cmd: U32, + /// includes strings + pub cmdsize: U32, + /// library's path name + pub name: LcStr, + /// number of modules in library + pub nmodules: U32, + /// bit vector of linked modules + pub linked_modules: LcStr, +} + +/* + * A program that uses a dynamic linker contains a `DylinkerCommand` to identify + * the name of the dynamic linker (LC_LOAD_DYLINKER). And a dynamic linker + * contains a `DylinkerCommand` to identify the dynamic linker (LC_ID_DYLINKER). + * A file can have at most one of these. + * This struct is also used for the LC_DYLD_ENVIRONMENT load command and + * contains string for dyld to treat like environment variable. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct DylinkerCommand { + /// LC_ID_DYLINKER, LC_LOAD_DYLINKER or LC_DYLD_ENVIRONMENT + pub cmd: U32, + /// includes pathname string + pub cmdsize: U32, + /// dynamic linker's path name + pub name: LcStr, +} + +/* + * Thread commands contain machine-specific data structures suitable for + * use in the thread state primitives. The machine specific data structures + * follow the struct `ThreadCommand` as follows. + * Each flavor of machine specific data structure is preceded by an uint32_t + * constant for the flavor of that data structure, an uint32_t that is the + * count of uint32_t's of the size of the state data structure and then + * the state data structure follows. This triple may be repeated for many + * flavors. The constants for the flavors, counts and state data structure + * definitions are expected to be in the header file . + * These machine specific data structures sizes must be multiples of + * 4 bytes. The `cmdsize` reflects the total size of the `ThreadCommand` + * and all of the sizes of the constants for the flavors, counts and state + * data structures. + * + * For executable objects that are unix processes there will be one + * `ThreadCommand` (cmd == LC_UNIXTHREAD) created for it by the link-editor. + * This is the same as a LC_THREAD, except that a stack is automatically + * created (based on the shell's limit for the stack size). Command arguments + * and environment variables are copied onto that stack. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ThreadCommand { + /// LC_THREAD or LC_UNIXTHREAD + pub cmd: U32, + /// total size of this command + pub cmdsize: U32, + /* uint32_t flavor flavor of thread state */ + /* uint32_t count count of uint32_t's in thread state */ + /* struct XXX_thread_state state thread state for this flavor */ + /* ... */ +} + +/* + * The routines command contains the address of the dynamic shared library + * initialization routine and an index into the module table for the module + * that defines the routine. Before any modules are used from the library the + * dynamic linker fully binds the module that defines the initialization routine + * and then calls it. This gets called before any module initialization + * routines (used for C++ static constructors) in the library. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct RoutinesCommand32 { + /* for 32-bit architectures */ + /// LC_ROUTINES + pub cmd: U32, + /// total size of this command + pub cmdsize: U32, + /// address of initialization routine + pub init_address: U32, + /// index into the module table that the init routine is defined in + pub init_module: U32, + pub reserved1: U32, + pub reserved2: U32, + pub reserved3: U32, + pub reserved4: U32, + pub reserved5: U32, + pub reserved6: U32, +} + +/* + * The 64-bit routines command. Same use as above. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct RoutinesCommand64 { + /* for 64-bit architectures */ + /// LC_ROUTINES_64 + pub cmd: U32, + /// total size of this command + pub cmdsize: U32, + /// address of initialization routine + pub init_address: U64, + /// index into the module table that the init routine is defined in + pub init_module: U64, + pub reserved1: U64, + pub reserved2: U64, + pub reserved3: U64, + pub reserved4: U64, + pub reserved5: U64, + pub reserved6: U64, +} + +/* + * The `SymtabCommand` contains the offsets and sizes of the link-edit 4.3BSD + * "stab" style symbol table information as described in the header files + * and . + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct SymtabCommand { + /// LC_SYMTAB + pub cmd: U32, + /// sizeof(struct SymtabCommand) + pub cmdsize: U32, + /// symbol table offset + pub symoff: U32, + /// number of symbol table entries + pub nsyms: U32, + /// string table offset + pub stroff: U32, + /// string table size in bytes + pub strsize: U32, +} + +/* + * This is the second set of the symbolic information which is used to support + * the data structures for the dynamically link editor. + * + * The original set of symbolic information in the `SymtabCommand` which contains + * the symbol and string tables must also be present when this load command is + * present. When this load command is present the symbol table is organized + * into three groups of symbols: + * local symbols (static and debugging symbols) - grouped by module + * defined external symbols - grouped by module (sorted by name if not lib) + * undefined external symbols (sorted by name if MH_BINDATLOAD is not set, + * and in order the were seen by the static + * linker if MH_BINDATLOAD is set) + * In this load command there are offsets and counts to each of the three groups + * of symbols. + * + * This load command contains a the offsets and sizes of the following new + * symbolic information tables: + * table of contents + * module table + * reference symbol table + * indirect symbol table + * The first three tables above (the table of contents, module table and + * reference symbol table) are only present if the file is a dynamically linked + * shared library. For executable and object modules, which are files + * containing only one module, the information that would be in these three + * tables is determined as follows: + * table of contents - the defined external symbols are sorted by name + * module table - the file contains only one module so everything in the + * file is part of the module. + * reference symbol table - is the defined and undefined external symbols + * + * For dynamically linked shared library files this load command also contains + * offsets and sizes to the pool of relocation entries for all sections + * separated into two groups: + * external relocation entries + * local relocation entries + * For executable and object modules the relocation entries continue to hang + * off the section structures. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct DysymtabCommand { + /// LC_DYSYMTAB + pub cmd: U32, + /// sizeof(struct DysymtabCommand) + pub cmdsize: U32, + + /* + * The symbols indicated by symoff and nsyms of the LC_SYMTAB load command + * are grouped into the following three groups: + * local symbols (further grouped by the module they are from) + * defined external symbols (further grouped by the module they are from) + * undefined symbols + * + * The local symbols are used only for debugging. The dynamic binding + * process may have to use them to indicate to the debugger the local + * symbols for a module that is being bound. + * + * The last two groups are used by the dynamic binding process to do the + * binding (indirectly through the module table and the reference symbol + * table when this is a dynamically linked shared library file). + */ + /// index to local symbols + pub ilocalsym: U32, + /// number of local symbols + pub nlocalsym: U32, + + /// index to externally defined symbols + pub iextdefsym: U32, + /// number of externally defined symbols + pub nextdefsym: U32, + + /// index to undefined symbols + pub iundefsym: U32, + /// number of undefined symbols + pub nundefsym: U32, + + /* + * For the for the dynamic binding process to find which module a symbol + * is defined in the table of contents is used (analogous to the ranlib + * structure in an archive) which maps defined external symbols to modules + * they are defined in. This exists only in a dynamically linked shared + * library file. For executable and object modules the defined external + * symbols are sorted by name and is use as the table of contents. + */ + /// file offset to table of contents + pub tocoff: U32, + /// number of entries in table of contents + pub ntoc: U32, + + /* + * To support dynamic binding of "modules" (whole object files) the symbol + * table must reflect the modules that the file was created from. This is + * done by having a module table that has indexes and counts into the merged + * tables for each module. The module structure that these two entries + * refer to is described below. This exists only in a dynamically linked + * shared library file. For executable and object modules the file only + * contains one module so everything in the file belongs to the module. + */ + /// file offset to module table + pub modtaboff: U32, + /// number of module table entries + pub nmodtab: U32, + + /* + * To support dynamic module binding the module structure for each module + * indicates the external references (defined and undefined) each module + * makes. For each module there is an offset and a count into the + * reference symbol table for the symbols that the module references. + * This exists only in a dynamically linked shared library file. For + * executable and object modules the defined external symbols and the + * undefined external symbols indicates the external references. + */ + /// offset to referenced symbol table + pub extrefsymoff: U32, + /// number of referenced symbol table entries + pub nextrefsyms: U32, + + /* + * The sections that contain "symbol pointers" and "routine stubs" have + * indexes and (implied counts based on the size of the section and fixed + * size of the entry) into the "indirect symbol" table for each pointer + * and stub. For every section of these two types the index into the + * indirect symbol table is stored in the section header in the field + * reserved1. An indirect symbol table entry is simply a 32bit index into + * the symbol table to the symbol that the pointer or stub is referring to. + * The indirect symbol table is ordered to match the entries in the section. + */ + /// file offset to the indirect symbol table + pub indirectsymoff: U32, + /// number of indirect symbol table entries + pub nindirectsyms: U32, + + /* + * To support relocating an individual module in a library file quickly the + * external relocation entries for each module in the library need to be + * accessed efficiently. Since the relocation entries can't be accessed + * through the section headers for a library file they are separated into + * groups of local and external entries further grouped by module. In this + * case the presents of this load command who's extreloff, nextrel, + * locreloff and nlocrel fields are non-zero indicates that the relocation + * entries of non-merged sections are not referenced through the section + * structures (and the reloff and nreloc fields in the section headers are + * set to zero). + * + * Since the relocation entries are not accessed through the section headers + * this requires the r_address field to be something other than a section + * offset to identify the item to be relocated. In this case r_address is + * set to the offset from the vmaddr of the first LC_SEGMENT command. + * For MH_SPLIT_SEGS images r_address is set to the the offset from the + * vmaddr of the first read-write LC_SEGMENT command. + * + * The relocation entries are grouped by module and the module table + * entries have indexes and counts into them for the group of external + * relocation entries for that the module. + * + * For sections that are merged across modules there must not be any + * remaining external relocation entries for them (for merged sections + * remaining relocation entries must be local). + */ + /// offset to external relocation entries + pub extreloff: U32, + /// number of external relocation entries + pub nextrel: U32, + + /* + * All the local relocation entries are grouped together (they are not + * grouped by their module since they are only used if the object is moved + * from it staticly link edited address). + */ + /// offset to local relocation entries + pub locreloff: U32, + /// number of local relocation entries + pub nlocrel: U32, +} + +/* + * An indirect symbol table entry is simply a 32bit index into the symbol table + * to the symbol that the pointer or stub is refering to. Unless it is for a + * non-lazy symbol pointer section for a defined symbol which strip(1) as + * removed. In which case it has the value INDIRECT_SYMBOL_LOCAL. If the + * symbol was also absolute INDIRECT_SYMBOL_ABS is or'ed with that. + */ +pub const INDIRECT_SYMBOL_LOCAL: u32 = 0x8000_0000; +pub const INDIRECT_SYMBOL_ABS: u32 = 0x4000_0000; + +/* a table of contents entry */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct DylibTableOfContents { + /// the defined external symbol (index into the symbol table) + pub symbol_index: U32, + /// index into the module table this symbol is defined in + pub module_index: U32, +} + +/* a module table entry */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct DylibModule32 { + /// the module name (index into string table) + pub module_name: U32, + + /// index into externally defined symbols + pub iextdefsym: U32, + /// number of externally defined symbols + pub nextdefsym: U32, + /// index into reference symbol table + pub irefsym: U32, + /// number of reference symbol table entries + pub nrefsym: U32, + /// index into symbols for local symbols + pub ilocalsym: U32, + /// number of local symbols + pub nlocalsym: U32, + + /// index into external relocation entries + pub iextrel: U32, + /// number of external relocation entries + pub nextrel: U32, + + /// low 16 bits are the index into the init section, high 16 bits are the index into the term section + pub iinit_iterm: U32, + /// low 16 bits are the number of init section entries, high 16 bits are the number of term section entries + pub ninit_nterm: U32, + + /// for this module address of the start of the (__OBJC,__module_info) section + pub objc_module_info_addr: U32, + /// for this module size of the (__OBJC,__module_info) section + pub objc_module_info_size: U32, +} + +/* a 64-bit module table entry */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct DylibModule64 { + /// the module name (index into string table) + pub module_name: U32, + + /// index into externally defined symbols + pub iextdefsym: U32, + /// number of externally defined symbols + pub nextdefsym: U32, + /// index into reference symbol table + pub irefsym: U32, + /// number of reference symbol table entries + pub nrefsym: U32, + /// index into symbols for local symbols + pub ilocalsym: U32, + /// number of local symbols + pub nlocalsym: U32, + + /// index into external relocation entries + pub iextrel: U32, + /// number of external relocation entries + pub nextrel: U32, + + /// low 16 bits are the index into the init section, high 16 bits are the index into the term section + pub iinit_iterm: U32, + /// low 16 bits are the number of init section entries, high 16 bits are the number of term section entries + pub ninit_nterm: U32, + + /// for this module size of the (__OBJC,__module_info) section + pub objc_module_info_size: U32, + /// for this module address of the start of the (__OBJC,__module_info) section + pub objc_module_info_addr: U64, +} + +/* + * The entries in the reference symbol table are used when loading the module + * (both by the static and dynamic link editors) and if the module is unloaded + * or replaced. Therefore all external symbols (defined and undefined) are + * listed in the module's reference table. The flags describe the type of + * reference that is being made. The constants for the flags are defined in + * as they are also used for symbol table entries. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct DylibReference { + /* TODO: + uint32_t isym:24, /* index into the symbol table */ + flags:8; /* flags to indicate the type of reference */ + */ + pub bitfield: U32, +} + +/* + * The TwolevelHintsCommand contains the offset and number of hints in the + * two-level namespace lookup hints table. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct TwolevelHintsCommand { + /// LC_TWOLEVEL_HINTS + pub cmd: U32, + /// sizeof(struct TwolevelHintsCommand) + pub cmdsize: U32, + /// offset to the hint table + pub offset: U32, + /// number of hints in the hint table + pub nhints: U32, +} + +/* + * The entries in the two-level namespace lookup hints table are TwolevelHint + * structs. These provide hints to the dynamic link editor where to start + * looking for an undefined symbol in a two-level namespace image. The + * isub_image field is an index into the sub-images (sub-frameworks and + * sub-umbrellas list) that made up the two-level image that the undefined + * symbol was found in when it was built by the static link editor. If + * isub-image is 0 the the symbol is expected to be defined in library and not + * in the sub-images. If isub-image is non-zero it is an index into the array + * of sub-images for the umbrella with the first index in the sub-images being + * 1. The array of sub-images is the ordered list of sub-images of the umbrella + * that would be searched for a symbol that has the umbrella recorded as its + * primary library. The table of contents index is an index into the + * library's table of contents. This is used as the starting point of the + * binary search or a directed linear search. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct TwolevelHint { + /* TODO: + uint32_t + isub_image:8, /* index into the sub images */ + itoc:24; /* index into the table of contents */ + */ + pub bitfield: U32, +} + +/* + * The PrebindCksumCommand contains the value of the original check sum for + * prebound files or zero. When a prebound file is first created or modified + * for other than updating its prebinding information the value of the check sum + * is set to zero. When the file has it prebinding re-done and if the value of + * the check sum is zero the original check sum is calculated and stored in + * cksum field of this load command in the output file. If when the prebinding + * is re-done and the cksum field is non-zero it is left unchanged from the + * input file. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct PrebindCksumCommand { + /// LC_PREBIND_CKSUM + pub cmd: U32, + /// sizeof(struct PrebindCksumCommand) + pub cmdsize: U32, + /// the check sum or zero + pub cksum: U32, +} + +/* + * The uuid load command contains a single 128-bit unique random number that + * identifies an object produced by the static link editor. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct UuidCommand { + /// LC_UUID + pub cmd: U32, + /// sizeof(struct UuidCommand) + pub cmdsize: U32, + /// the 128-bit uuid + pub uuid: [u8; 16], +} + +/* + * The RpathCommand contains a path which at runtime should be added to + * the current run path used to find @rpath prefixed dylibs. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct RpathCommand { + /// LC_RPATH + pub cmd: U32, + /// includes string + pub cmdsize: U32, + /// path to add to run path + pub path: LcStr, +} + +/* + * The LinkeditDataCommand contains the offsets and sizes of a blob + * of data in the __LINKEDIT segment. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct LinkeditDataCommand { + /// `LC_CODE_SIGNATURE`, `LC_SEGMENT_SPLIT_INFO`, `LC_FUNCTION_STARTS`, + /// `LC_DATA_IN_CODE`, `LC_DYLIB_CODE_SIGN_DRS`, `LC_LINKER_OPTIMIZATION_HINT`, + /// `LC_DYLD_EXPORTS_TRIE`, or `LC_DYLD_CHAINED_FIXUPS`. + pub cmd: U32, + /// sizeof(struct LinkeditDataCommand) + pub cmdsize: U32, + /// file offset of data in __LINKEDIT segment + pub dataoff: U32, + /// file size of data in __LINKEDIT segment + pub datasize: U32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct FilesetEntryCommand { + // LC_FILESET_ENTRY + pub cmd: U32, + /// includes id string + pub cmdsize: U32, + /// memory address of the dylib + pub vmaddr: U64, + /// file offset of the dylib + pub fileoff: U64, + /// contained entry id + pub entry_id: LcStr, + /// entry_id is 32-bits long, so this is the reserved padding + pub reserved: U32, +} + +/* + * The EncryptionInfoCommand32 contains the file offset and size of an + * of an encrypted segment. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct EncryptionInfoCommand32 { + /// LC_ENCRYPTION_INFO + pub cmd: U32, + /// sizeof(struct EncryptionInfoCommand32) + pub cmdsize: U32, + /// file offset of encrypted range + pub cryptoff: U32, + /// file size of encrypted range + pub cryptsize: U32, + /// which enryption system, 0 means not-encrypted yet + pub cryptid: U32, +} + +/* + * The EncryptionInfoCommand64 contains the file offset and size of an + * of an encrypted segment (for use in x86_64 targets). + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct EncryptionInfoCommand64 { + /// LC_ENCRYPTION_INFO_64 + pub cmd: U32, + /// sizeof(struct EncryptionInfoCommand64) + pub cmdsize: U32, + /// file offset of encrypted range + pub cryptoff: U32, + /// file size of encrypted range + pub cryptsize: U32, + /// which enryption system, 0 means not-encrypted yet + pub cryptid: U32, + /// padding to make this struct's size a multiple of 8 bytes + pub pad: U32, +} + +/* + * The VersionMinCommand contains the min OS version on which this + * binary was built to run. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct VersionMinCommand { + /// LC_VERSION_MIN_MACOSX or LC_VERSION_MIN_IPHONEOS or LC_VERSION_MIN_WATCHOS or LC_VERSION_MIN_TVOS + pub cmd: U32, + /// sizeof(struct VersionMinCommand) + pub cmdsize: U32, + /// X.Y.Z is encoded in nibbles xxxx.yy.zz + pub version: U32, + /// X.Y.Z is encoded in nibbles xxxx.yy.zz + pub sdk: U32, +} + +/* + * The BuildVersionCommand contains the min OS version on which this + * binary was built to run for its platform. The list of known platforms and + * tool values following it. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct BuildVersionCommand { + /// LC_BUILD_VERSION + pub cmd: U32, + /// sizeof(struct BuildVersionCommand) plus ntools * sizeof(struct BuildToolVersion) + pub cmdsize: U32, + /// platform + pub platform: U32, + /// X.Y.Z is encoded in nibbles xxxx.yy.zz + pub minos: U32, + /// X.Y.Z is encoded in nibbles xxxx.yy.zz + pub sdk: U32, + /// number of tool entries following this + pub ntools: U32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct BuildToolVersion { + /// enum for the tool + pub tool: U32, + /// version number of the tool + pub version: U32, +} + +/* Known values for the platform field above. */ +pub const PLATFORM_MACOS: u32 = 1; +pub const PLATFORM_IOS: u32 = 2; +pub const PLATFORM_TVOS: u32 = 3; +pub const PLATFORM_WATCHOS: u32 = 4; +pub const PLATFORM_BRIDGEOS: u32 = 5; +pub const PLATFORM_MACCATALYST: u32 = 6; +pub const PLATFORM_IOSSIMULATOR: u32 = 7; +pub const PLATFORM_TVOSSIMULATOR: u32 = 8; +pub const PLATFORM_WATCHOSSIMULATOR: u32 = 9; +pub const PLATFORM_DRIVERKIT: u32 = 10; + +/* Known values for the tool field above. */ +pub const TOOL_CLANG: u32 = 1; +pub const TOOL_SWIFT: u32 = 2; +pub const TOOL_LD: u32 = 3; + +/* + * The DyldInfoCommand contains the file offsets and sizes of + * the new compressed form of the information dyld needs to + * load the image. This information is used by dyld on Mac OS X + * 10.6 and later. All information pointed to by this command + * is encoded using byte streams, so no endian swapping is needed + * to interpret it. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct DyldInfoCommand { + /// LC_DYLD_INFO or LC_DYLD_INFO_ONLY + pub cmd: U32, + /// sizeof(struct DyldInfoCommand) + pub cmdsize: U32, + + /* + * Dyld rebases an image whenever dyld loads it at an address different + * from its preferred address. The rebase information is a stream + * of byte sized opcodes whose symbolic names start with REBASE_OPCODE_. + * Conceptually the rebase information is a table of tuples: + * + * The opcodes are a compressed way to encode the table by only + * encoding when a column changes. In addition simple patterns + * like "every n'th offset for m times" can be encoded in a few + * bytes. + */ + /// file offset to rebase info + pub rebase_off: U32, + /// size of rebase info + pub rebase_size: U32, + + /* + * Dyld binds an image during the loading process, if the image + * requires any pointers to be initialized to symbols in other images. + * The bind information is a stream of byte sized + * opcodes whose symbolic names start with BIND_OPCODE_. + * Conceptually the bind information is a table of tuples: + * + * The opcodes are a compressed way to encode the table by only + * encoding when a column changes. In addition simple patterns + * like for runs of pointers initialzed to the same value can be + * encoded in a few bytes. + */ + /// file offset to binding info + pub bind_off: U32, + /// size of binding info + pub bind_size: U32, + + /* + * Some C++ programs require dyld to unique symbols so that all + * images in the process use the same copy of some code/data. + * This step is done after binding. The content of the weak_bind + * info is an opcode stream like the bind_info. But it is sorted + * alphabetically by symbol name. This enable dyld to walk + * all images with weak binding information in order and look + * for collisions. If there are no collisions, dyld does + * no updating. That means that some fixups are also encoded + * in the bind_info. For instance, all calls to "operator new" + * are first bound to libstdc++.dylib using the information + * in bind_info. Then if some image overrides operator new + * that is detected when the weak_bind information is processed + * and the call to operator new is then rebound. + */ + /// file offset to weak binding info + pub weak_bind_off: U32, + /// size of weak binding info + pub weak_bind_size: U32, + + /* + * Some uses of external symbols do not need to be bound immediately. + * Instead they can be lazily bound on first use. The lazy_bind + * are contains a stream of BIND opcodes to bind all lazy symbols. + * Normal use is that dyld ignores the lazy_bind section when + * loading an image. Instead the static linker arranged for the + * lazy pointer to initially point to a helper function which + * pushes the offset into the lazy_bind area for the symbol + * needing to be bound, then jumps to dyld which simply adds + * the offset to lazy_bind_off to get the information on what + * to bind. + */ + /// file offset to lazy binding info + pub lazy_bind_off: U32, + /// size of lazy binding infs + pub lazy_bind_size: U32, + + /* + * The symbols exported by a dylib are encoded in a trie. This + * is a compact representation that factors out common prefixes. + * It also reduces LINKEDIT pages in RAM because it encodes all + * information (name, address, flags) in one small, contiguous range. + * The export area is a stream of nodes. The first node sequentially + * is the start node for the trie. + * + * Nodes for a symbol start with a uleb128 that is the length of + * the exported symbol information for the string so far. + * If there is no exported symbol, the node starts with a zero byte. + * If there is exported info, it follows the length. + * + * First is a uleb128 containing flags. Normally, it is followed by + * a uleb128 encoded offset which is location of the content named + * by the symbol from the mach_header for the image. If the flags + * is EXPORT_SYMBOL_FLAGS_REEXPORT, then following the flags is + * a uleb128 encoded library ordinal, then a zero terminated + * UTF8 string. If the string is zero length, then the symbol + * is re-export from the specified dylib with the same name. + * If the flags is EXPORT_SYMBOL_FLAGS_STUB_AND_RESOLVER, then following + * the flags is two uleb128s: the stub offset and the resolver offset. + * The stub is used by non-lazy pointers. The resolver is used + * by lazy pointers and must be called to get the actual address to use. + * + * After the optional exported symbol information is a byte of + * how many edges (0-255) that this node has leaving it, + * followed by each edge. + * Each edge is a zero terminated UTF8 of the addition chars + * in the symbol, followed by a uleb128 offset for the node that + * edge points to. + * + */ + /// file offset to lazy binding info + pub export_off: U32, + /// size of lazy binding infs + pub export_size: U32, +} + +/* + * The following are used to encode rebasing information + */ +pub const REBASE_TYPE_POINTER: u8 = 1; +pub const REBASE_TYPE_TEXT_ABSOLUTE32: u8 = 2; +pub const REBASE_TYPE_TEXT_PCREL32: u8 = 3; + +pub const REBASE_OPCODE_MASK: u8 = 0xF0; +pub const REBASE_IMMEDIATE_MASK: u8 = 0x0F; +pub const REBASE_OPCODE_DONE: u8 = 0x00; +pub const REBASE_OPCODE_SET_TYPE_IMM: u8 = 0x10; +pub const REBASE_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB: u8 = 0x20; +pub const REBASE_OPCODE_ADD_ADDR_ULEB: u8 = 0x30; +pub const REBASE_OPCODE_ADD_ADDR_IMM_SCALED: u8 = 0x40; +pub const REBASE_OPCODE_DO_REBASE_IMM_TIMES: u8 = 0x50; +pub const REBASE_OPCODE_DO_REBASE_ULEB_TIMES: u8 = 0x60; +pub const REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB: u8 = 0x70; +pub const REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB: u8 = 0x80; + +/* + * The following are used to encode binding information + */ +pub const BIND_TYPE_POINTER: u8 = 1; +pub const BIND_TYPE_TEXT_ABSOLUTE32: u8 = 2; +pub const BIND_TYPE_TEXT_PCREL32: u8 = 3; + +pub const BIND_SPECIAL_DYLIB_SELF: i8 = 0; +pub const BIND_SPECIAL_DYLIB_MAIN_EXECUTABLE: i8 = -1; +pub const BIND_SPECIAL_DYLIB_FLAT_LOOKUP: i8 = -2; +pub const BIND_SPECIAL_DYLIB_WEAK_LOOKUP: i8 = -3; + +pub const BIND_SYMBOL_FLAGS_WEAK_IMPORT: u8 = 0x1; +pub const BIND_SYMBOL_FLAGS_NON_WEAK_DEFINITION: u8 = 0x8; + +pub const BIND_OPCODE_MASK: u8 = 0xF0; +pub const BIND_IMMEDIATE_MASK: u8 = 0x0F; +pub const BIND_OPCODE_DONE: u8 = 0x00; +pub const BIND_OPCODE_SET_DYLIB_ORDINAL_IMM: u8 = 0x10; +pub const BIND_OPCODE_SET_DYLIB_ORDINAL_ULEB: u8 = 0x20; +pub const BIND_OPCODE_SET_DYLIB_SPECIAL_IMM: u8 = 0x30; +pub const BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM: u8 = 0x40; +pub const BIND_OPCODE_SET_TYPE_IMM: u8 = 0x50; +pub const BIND_OPCODE_SET_ADDEND_SLEB: u8 = 0x60; +pub const BIND_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB: u8 = 0x70; +pub const BIND_OPCODE_ADD_ADDR_ULEB: u8 = 0x80; +pub const BIND_OPCODE_DO_BIND: u8 = 0x90; +pub const BIND_OPCODE_DO_BIND_ADD_ADDR_ULEB: u8 = 0xA0; +pub const BIND_OPCODE_DO_BIND_ADD_ADDR_IMM_SCALED: u8 = 0xB0; +pub const BIND_OPCODE_DO_BIND_ULEB_TIMES_SKIPPING_ULEB: u8 = 0xC0; +pub const BIND_OPCODE_THREADED: u8 = 0xD0; +pub const BIND_SUBOPCODE_THREADED_SET_BIND_ORDINAL_TABLE_SIZE_ULEB: u8 = 0x00; +pub const BIND_SUBOPCODE_THREADED_APPLY: u8 = 0x01; + +/* + * The following are used on the flags byte of a terminal node + * in the export information. + */ +pub const EXPORT_SYMBOL_FLAGS_KIND_MASK: u32 = 0x03; +pub const EXPORT_SYMBOL_FLAGS_KIND_REGULAR: u32 = 0x00; +pub const EXPORT_SYMBOL_FLAGS_KIND_THREAD_LOCAL: u32 = 0x01; +pub const EXPORT_SYMBOL_FLAGS_KIND_ABSOLUTE: u32 = 0x02; +pub const EXPORT_SYMBOL_FLAGS_WEAK_DEFINITION: u32 = 0x04; +pub const EXPORT_SYMBOL_FLAGS_REEXPORT: u32 = 0x08; +pub const EXPORT_SYMBOL_FLAGS_STUB_AND_RESOLVER: u32 = 0x10; + +/* + * The LinkerOptionCommand contains linker options embedded in object files. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct LinkerOptionCommand { + /// LC_LINKER_OPTION only used in MH_OBJECT filetypes + pub cmd: U32, + pub cmdsize: U32, + /// number of strings + pub count: U32, + /* concatenation of zero terminated UTF8 strings. + Zero filled at end to align */ +} + +/* + * The SymsegCommand contains the offset and size of the GNU style + * symbol table information as described in the header file . + * The symbol roots of the symbol segments must also be aligned properly + * in the file. So the requirement of keeping the offsets aligned to a + * multiple of a 4 bytes translates to the length field of the symbol + * roots also being a multiple of a long. Also the padding must again be + * zeroed. (THIS IS OBSOLETE and no longer supported). + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct SymsegCommand { + /// LC_SYMSEG + pub cmd: U32, + /// sizeof(struct SymsegCommand) + pub cmdsize: U32, + /// symbol segment offset + pub offset: U32, + /// symbol segment size in bytes + pub size: U32, +} + +/* + * The IdentCommand contains a free format string table following the + * IdentCommand structure. The strings are null terminated and the size of + * the command is padded out with zero bytes to a multiple of 4 bytes/ + * (THIS IS OBSOLETE and no longer supported). + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct IdentCommand { + /// LC_IDENT + pub cmd: U32, + /// strings that follow this command + pub cmdsize: U32, +} + +/* + * The FvmfileCommand contains a reference to a file to be loaded at the + * specified virtual address. (Presently, this command is reserved for + * internal use. The kernel ignores this command when loading a program into + * memory). + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct FvmfileCommand { + /// LC_FVMFILE + pub cmd: U32, + /// includes pathname string + pub cmdsize: U32, + /// files pathname + pub name: LcStr, + /// files virtual address + pub header_addr: U32, +} + +/* + * The EntryPointCommand is a replacement for thread_command. + * It is used for main executables to specify the location (file offset) + * of main(). If -stack_size was used at link time, the stacksize + * field will contain the stack size need for the main thread. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct EntryPointCommand { + /// LC_MAIN only used in MH_EXECUTE filetypes + pub cmd: U32, + /// 24 + pub cmdsize: U32, + /// file (__TEXT) offset of main() + pub entryoff: U64, + /// if not zero, initial stack size + pub stacksize: U64, +} + +/* + * The SourceVersionCommand is an optional load command containing + * the version of the sources used to build the binary. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct SourceVersionCommand { + /// LC_SOURCE_VERSION + pub cmd: U32, + /// 16 + pub cmdsize: U32, + /// A.B.C.D.E packed as a24.b10.c10.d10.e10 + pub version: U64, +} + +/* + * The LC_DATA_IN_CODE load commands uses a LinkeditDataCommand + * to point to an array of DataInCodeEntry entries. Each entry + * describes a range of data in a code section. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct DataInCodeEntry { + /// from mach_header to start of data range + pub offset: U32, + /// number of bytes in data range + pub length: U16, + /// a DICE_KIND_* value + pub kind: U16, +} +pub const DICE_KIND_DATA: u32 = 0x0001; +pub const DICE_KIND_JUMP_TABLE8: u32 = 0x0002; +pub const DICE_KIND_JUMP_TABLE16: u32 = 0x0003; +pub const DICE_KIND_JUMP_TABLE32: u32 = 0x0004; +pub const DICE_KIND_ABS_JUMP_TABLE32: u32 = 0x0005; + +/* + * Sections of type S_THREAD_LOCAL_VARIABLES contain an array + * of TlvDescriptor structures. + */ +/* TODO: +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct TlvDescriptor +{ + void* (*thunk)(struct TlvDescriptor*); + unsigned long key; + unsigned long offset; +} +*/ + +/* + * LC_NOTE commands describe a region of arbitrary data included in a Mach-O + * file. Its initial use is to record extra data in MH_CORE files. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct NoteCommand { + /// LC_NOTE + pub cmd: U32, + /// sizeof(struct NoteCommand) + pub cmdsize: U32, + /// owner name for this LC_NOTE + pub data_owner: [u8; 16], + /// file offset of this data + pub offset: U64, + /// length of data region + pub size: U64, +} + +// Definitions from "/usr/include/mach-o/nlist.h". + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Nlist32 { + /// index into the string table + pub n_strx: U32, + /// type flag, see below + pub n_type: u8, + /// section number or NO_SECT + pub n_sect: u8, + /// see + pub n_desc: U16, + /// value of this symbol (or stab offset) + pub n_value: U32, +} + +/* + * This is the symbol table entry structure for 64-bit architectures. + */ +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Nlist64 { + /// index into the string table + pub n_strx: U32, + /// type flag, see below + pub n_type: u8, + /// section number or NO_SECT + pub n_sect: u8, + /// see + pub n_desc: U16, + /// value of this symbol (or stab offset) + // Note: 4 byte alignment has been observed in practice. + pub n_value: U64Bytes, +} + +/* + * Symbols with a index into the string table of zero (n_un.n_strx == 0) are + * defined to have a null, "", name. Therefore all string indexes to non null + * names must not have a zero string index. This is bit historical information + * that has never been well documented. + */ + +/* + * The n_type field really contains four fields: + * unsigned char N_STAB:3, + * N_PEXT:1, + * N_TYPE:3, + * N_EXT:1; + * which are used via the following masks. + */ +/// if any of these bits set, a symbolic debugging entry +pub const N_STAB: u8 = 0xe0; +/// private external symbol bit +pub const N_PEXT: u8 = 0x10; +/// mask for the type bits +pub const N_TYPE: u8 = 0x0e; +/// external symbol bit, set for external symbols +pub const N_EXT: u8 = 0x01; + +/* + * Only symbolic debugging entries have some of the N_STAB bits set and if any + * of these bits are set then it is a symbolic debugging entry (a stab). In + * which case then the values of the n_type field (the entire field) are given + * in + */ + +/* + * Values for N_TYPE bits of the n_type field. + */ +/// undefined, n_sect == NO_SECT +pub const N_UNDF: u8 = 0x0; +/// absolute, n_sect == NO_SECT +pub const N_ABS: u8 = 0x2; +/// defined in section number n_sect +pub const N_SECT: u8 = 0xe; +/// prebound undefined (defined in a dylib) +pub const N_PBUD: u8 = 0xc; +/// indirect +pub const N_INDR: u8 = 0xa; + +/* + * If the type is N_INDR then the symbol is defined to be the same as another + * symbol. In this case the n_value field is an index into the string table + * of the other symbol's name. When the other symbol is defined then they both + * take on the defined type and value. + */ + +/* + * If the type is N_SECT then the n_sect field contains an ordinal of the + * section the symbol is defined in. The sections are numbered from 1 and + * refer to sections in order they appear in the load commands for the file + * they are in. This means the same ordinal may very well refer to different + * sections in different files. + * + * The n_value field for all symbol table entries (including N_STAB's) gets + * updated by the link editor based on the value of it's n_sect field and where + * the section n_sect references gets relocated. If the value of the n_sect + * field is NO_SECT then it's n_value field is not changed by the link editor. + */ +/// symbol is not in any section +pub const NO_SECT: u8 = 0; +/// 1 thru 255 inclusive +pub const MAX_SECT: u8 = 255; + +/* + * Common symbols are represented by undefined (N_UNDF) external (N_EXT) types + * who's values (n_value) are non-zero. In which case the value of the n_value + * field is the size (in bytes) of the common symbol. The n_sect field is set + * to NO_SECT. The alignment of a common symbol may be set as a power of 2 + * between 2^1 and 2^15 as part of the n_desc field using the macros below. If + * the alignment is not set (a value of zero) then natural alignment based on + * the size is used. + */ +/* TODO: +#define GET_COMM_ALIGN(n_desc) (((n_desc) >> 8) & 0x0f) +#define SET_COMM_ALIGN(n_desc,align) \ + (n_desc) = (((n_desc) & 0xf0ff) | (((align) & 0x0f) << 8)) + */ + +/* + * To support the lazy binding of undefined symbols in the dynamic link-editor, + * the undefined symbols in the symbol table (the nlist structures) are marked + * with the indication if the undefined reference is a lazy reference or + * non-lazy reference. If both a non-lazy reference and a lazy reference is + * made to the same symbol the non-lazy reference takes precedence. A reference + * is lazy only when all references to that symbol are made through a symbol + * pointer in a lazy symbol pointer section. + * + * The implementation of marking nlist structures in the symbol table for + * undefined symbols will be to use some of the bits of the n_desc field as a + * reference type. The mask REFERENCE_TYPE will be applied to the n_desc field + * of an nlist structure for an undefined symbol to determine the type of + * undefined reference (lazy or non-lazy). + * + * The constants for the REFERENCE FLAGS are propagated to the reference table + * in a shared library file. In that case the constant for a defined symbol, + * REFERENCE_FLAG_DEFINED, is also used. + */ +/* Reference type bits of the n_desc field of undefined symbols */ +pub const REFERENCE_TYPE: u16 = 0x7; +/* types of references */ +pub const REFERENCE_FLAG_UNDEFINED_NON_LAZY: u16 = 0; +pub const REFERENCE_FLAG_UNDEFINED_LAZY: u16 = 1; +pub const REFERENCE_FLAG_DEFINED: u16 = 2; +pub const REFERENCE_FLAG_PRIVATE_DEFINED: u16 = 3; +pub const REFERENCE_FLAG_PRIVATE_UNDEFINED_NON_LAZY: u16 = 4; +pub const REFERENCE_FLAG_PRIVATE_UNDEFINED_LAZY: u16 = 5; + +/* + * To simplify stripping of objects that use are used with the dynamic link + * editor, the static link editor marks the symbols defined an object that are + * referenced by a dynamicly bound object (dynamic shared libraries, bundles). + * With this marking strip knows not to strip these symbols. + */ +pub const REFERENCED_DYNAMICALLY: u16 = 0x0010; + +/* + * For images created by the static link editor with the -twolevel_namespace + * option in effect the flags field of the mach header is marked with + * MH_TWOLEVEL. And the binding of the undefined references of the image are + * determined by the static link editor. Which library an undefined symbol is + * bound to is recorded by the static linker in the high 8 bits of the n_desc + * field using the SET_LIBRARY_ORDINAL macro below. The ordinal recorded + * references the libraries listed in the Mach-O's LC_LOAD_DYLIB, + * LC_LOAD_WEAK_DYLIB, LC_REEXPORT_DYLIB, LC_LOAD_UPWARD_DYLIB, and + * LC_LAZY_LOAD_DYLIB, etc. load commands in the order they appear in the + * headers. The library ordinals start from 1. + * For a dynamic library that is built as a two-level namespace image the + * undefined references from module defined in another use the same nlist struct + * an in that case SELF_LIBRARY_ORDINAL is used as the library ordinal. For + * defined symbols in all images they also must have the library ordinal set to + * SELF_LIBRARY_ORDINAL. The EXECUTABLE_ORDINAL refers to the executable + * image for references from plugins that refer to the executable that loads + * them. + * + * The DYNAMIC_LOOKUP_ORDINAL is for undefined symbols in a two-level namespace + * image that are looked up by the dynamic linker with flat namespace semantics. + * This ordinal was added as a feature in Mac OS X 10.3 by reducing the + * value of MAX_LIBRARY_ORDINAL by one. So it is legal for existing binaries + * or binaries built with older tools to have 0xfe (254) dynamic libraries. In + * this case the ordinal value 0xfe (254) must be treated as a library ordinal + * for compatibility. + */ +/* TODO: +#define GET_LIBRARY_ORDINAL(n_desc) (((n_desc) >> 8) & 0xff) +#define SET_LIBRARY_ORDINAL(n_desc,ordinal) \ + (n_desc) = (((n_desc) & 0x00ff) | (((ordinal) & 0xff) << 8)) + */ +pub const SELF_LIBRARY_ORDINAL: u8 = 0x0; +pub const MAX_LIBRARY_ORDINAL: u8 = 0xfd; +pub const DYNAMIC_LOOKUP_ORDINAL: u8 = 0xfe; +pub const EXECUTABLE_ORDINAL: u8 = 0xff; + +/* + * The bit 0x0020 of the n_desc field is used for two non-overlapping purposes + * and has two different symbolic names, N_NO_DEAD_STRIP and N_DESC_DISCARDED. + */ + +/* + * The N_NO_DEAD_STRIP bit of the n_desc field only ever appears in a + * relocatable .o file (MH_OBJECT filetype). And is used to indicate to the + * static link editor it is never to dead strip the symbol. + */ +/// symbol is not to be dead stripped +pub const N_NO_DEAD_STRIP: u16 = 0x0020; + +/* + * The N_DESC_DISCARDED bit of the n_desc field never appears in linked image. + * But is used in very rare cases by the dynamic link editor to mark an in + * memory symbol as discared and longer used for linking. + */ +/// symbol is discarded +pub const N_DESC_DISCARDED: u16 = 0x0020; + +/* + * The N_WEAK_REF bit of the n_desc field indicates to the dynamic linker that + * the undefined symbol is allowed to be missing and is to have the address of + * zero when missing. + */ +/// symbol is weak referenced +pub const N_WEAK_REF: u16 = 0x0040; + +/* + * The N_WEAK_DEF bit of the n_desc field indicates to the static and dynamic + * linkers that the symbol definition is weak, allowing a non-weak symbol to + * also be used which causes the weak definition to be discared. Currently this + * is only supported for symbols in coalesed sections. + */ +/// coalesed symbol is a weak definition +pub const N_WEAK_DEF: u16 = 0x0080; + +/* + * The N_REF_TO_WEAK bit of the n_desc field indicates to the dynamic linker + * that the undefined symbol should be resolved using flat namespace searching. + */ +/// reference to a weak symbol +pub const N_REF_TO_WEAK: u16 = 0x0080; + +/* + * The N_ARM_THUMB_DEF bit of the n_desc field indicates that the symbol is + * a defintion of a Thumb function. + */ +/// symbol is a Thumb function (ARM) +pub const N_ARM_THUMB_DEF: u16 = 0x0008; + +/* + * The N_SYMBOL_RESOLVER bit of the n_desc field indicates that the + * that the function is actually a resolver function and should + * be called to get the address of the real function to use. + * This bit is only available in .o files (MH_OBJECT filetype) + */ +pub const N_SYMBOL_RESOLVER: u16 = 0x0100; + +/* + * The N_ALT_ENTRY bit of the n_desc field indicates that the + * symbol is pinned to the previous content. + */ +pub const N_ALT_ENTRY: u16 = 0x0200; + +// Definitions from "/usr/include/mach-o/stab.h". + +/* + * This file gives definitions supplementing for permanent symbol + * table entries of Mach-O files. Modified from the BSD definitions. The + * modifications from the original definitions were changing what the values of + * what was the n_other field (an unused field) which is now the n_sect field. + * These modifications are required to support symbols in an arbitrary number of + * sections not just the three sections (text, data and bss) in a BSD file. + * The values of the defined constants have NOT been changed. + * + * These must have one of the N_STAB bits on. The n_value fields are subject + * to relocation according to the value of their n_sect field. So for types + * that refer to things in sections the n_sect field must be filled in with the + * proper section ordinal. For types that are not to have their n_value field + * relocatated the n_sect field must be NO_SECT. + */ + +/* + * Symbolic debugger symbols. The comments give the conventional use for + * + * .stabs "n_name", n_type, n_sect, n_desc, n_value + * + * where n_type is the defined constant and not listed in the comment. Other + * fields not listed are zero. n_sect is the section ordinal the entry is + * refering to. + */ +/// global symbol: name,,NO_SECT,type,0 +pub const N_GSYM: u8 = 0x20; +/// procedure name (f77 kludge): name,,NO_SECT,0,0 +pub const N_FNAME: u8 = 0x22; +/// procedure: name,,n_sect,linenumber,address +pub const N_FUN: u8 = 0x24; +/// static symbol: name,,n_sect,type,address +pub const N_STSYM: u8 = 0x26; +/// .lcomm symbol: name,,n_sect,type,address +pub const N_LCSYM: u8 = 0x28; +/// begin nsect sym: 0,,n_sect,0,address +pub const N_BNSYM: u8 = 0x2e; +/// AST file path: name,,NO_SECT,0,0 +pub const N_AST: u8 = 0x32; +/// emitted with gcc2_compiled and in gcc source +pub const N_OPT: u8 = 0x3c; +/// register sym: name,,NO_SECT,type,register +pub const N_RSYM: u8 = 0x40; +/// src line: 0,,n_sect,linenumber,address +pub const N_SLINE: u8 = 0x44; +/// end nsect sym: 0,,n_sect,0,address +pub const N_ENSYM: u8 = 0x4e; +/// structure elt: name,,NO_SECT,type,struct_offset +pub const N_SSYM: u8 = 0x60; +/// source file name: name,,n_sect,0,address +pub const N_SO: u8 = 0x64; +/// object file name: name,,0,0,st_mtime +pub const N_OSO: u8 = 0x66; +/// local sym: name,,NO_SECT,type,offset +pub const N_LSYM: u8 = 0x80; +/// include file beginning: name,,NO_SECT,0,sum +pub const N_BINCL: u8 = 0x82; +/// #included file name: name,,n_sect,0,address +pub const N_SOL: u8 = 0x84; +/// compiler parameters: name,,NO_SECT,0,0 +pub const N_PARAMS: u8 = 0x86; +/// compiler version: name,,NO_SECT,0,0 +pub const N_VERSION: u8 = 0x88; +/// compiler -O level: name,,NO_SECT,0,0 +pub const N_OLEVEL: u8 = 0x8A; +/// parameter: name,,NO_SECT,type,offset +pub const N_PSYM: u8 = 0xa0; +/// include file end: name,,NO_SECT,0,0 +pub const N_EINCL: u8 = 0xa2; +/// alternate entry: name,,n_sect,linenumber,address +pub const N_ENTRY: u8 = 0xa4; +/// left bracket: 0,,NO_SECT,nesting level,address +pub const N_LBRAC: u8 = 0xc0; +/// deleted include file: name,,NO_SECT,0,sum +pub const N_EXCL: u8 = 0xc2; +/// right bracket: 0,,NO_SECT,nesting level,address +pub const N_RBRAC: u8 = 0xe0; +/// begin common: name,,NO_SECT,0,0 +pub const N_BCOMM: u8 = 0xe2; +/// end common: name,,n_sect,0,0 +pub const N_ECOMM: u8 = 0xe4; +/// end common (local name): 0,,n_sect,0,address +pub const N_ECOML: u8 = 0xe8; +/// second stab entry with length information +pub const N_LENG: u8 = 0xfe; + +/* + * for the berkeley pascal compiler, pc(1): + */ +/// global pascal symbol: name,,NO_SECT,subtype,line +pub const N_PC: u8 = 0x30; + +// Definitions from "/usr/include/mach-o/reloc.h". + +/// A relocation entry. +/// +/// Mach-O relocations have plain and scattered variants, with the +/// meaning of the fields depending on the variant. +/// +/// This type provides functions for determining whether the relocation +/// is scattered, and for accessing the fields of each variant. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Relocation { + pub r_word0: U32, + pub r_word1: U32, +} + +impl Relocation { + /// Determine whether this is a scattered relocation. + #[inline] + pub fn r_scattered(self, endian: E, cputype: u32) -> bool { + if cputype == CPU_TYPE_X86_64 { + false + } else { + self.r_word0.get(endian) & R_SCATTERED != 0 + } + } + + /// Return the fields of a plain relocation. + pub fn info(self, endian: E) -> RelocationInfo { + let r_address = self.r_word0.get(endian); + let r_word1 = self.r_word1.get(endian); + if endian.is_little_endian() { + RelocationInfo { + r_address, + r_symbolnum: r_word1 & 0x00ff_ffff, + r_pcrel: ((r_word1 >> 24) & 0x1) != 0, + r_length: ((r_word1 >> 25) & 0x3) as u8, + r_extern: ((r_word1 >> 27) & 0x1) != 0, + r_type: (r_word1 >> 28) as u8, + } + } else { + RelocationInfo { + r_address, + r_symbolnum: r_word1 >> 8, + r_pcrel: ((r_word1 >> 7) & 0x1) != 0, + r_length: ((r_word1 >> 5) & 0x3) as u8, + r_extern: ((r_word1 >> 4) & 0x1) != 0, + r_type: (r_word1 & 0xf) as u8, + } + } + } + + /// Return the fields of a scattered relocation. + pub fn scattered_info(self, endian: E) -> ScatteredRelocationInfo { + let r_word0 = self.r_word0.get(endian); + let r_value = self.r_word1.get(endian); + ScatteredRelocationInfo { + r_address: r_word0 & 0x00ff_ffff, + r_type: ((r_word0 >> 24) & 0xf) as u8, + r_length: ((r_word0 >> 28) & 0x3) as u8, + r_pcrel: ((r_word0 >> 30) & 0x1) != 0, + r_value, + } + } +} + +/* + * Format of a relocation entry of a Mach-O file. Modified from the 4.3BSD + * format. The modifications from the original format were changing the value + * of the r_symbolnum field for "local" (r_extern == 0) relocation entries. + * This modification is required to support symbols in an arbitrary number of + * sections not just the three sections (text, data and bss) in a 4.3BSD file. + * Also the last 4 bits have had the r_type tag added to them. + */ + +#[derive(Debug, Clone, Copy)] +pub struct RelocationInfo { + /// offset in the section to what is being relocated + pub r_address: u32, + /// symbol index if r_extern == 1 or section ordinal if r_extern == 0 + pub r_symbolnum: u32, + /// was relocated pc relative already + pub r_pcrel: bool, + /// 0=byte, 1=word, 2=long, 3=quad + pub r_length: u8, + /// does not include value of sym referenced + pub r_extern: bool, + /// if not 0, machine specific relocation type + pub r_type: u8, +} + +impl RelocationInfo { + /// Combine the fields into a `Relocation`. + pub fn relocation(self, endian: E) -> Relocation { + let r_word0 = U32::new(endian, self.r_address); + let r_word1 = U32::new( + endian, + if endian.is_little_endian() { + self.r_symbolnum & 0x00ff_ffff + | u32::from(self.r_pcrel) << 24 + | u32::from(self.r_length & 0x3) << 25 + | u32::from(self.r_extern) << 27 + | u32::from(self.r_type) << 28 + } else { + self.r_symbolnum >> 8 + | u32::from(self.r_pcrel) << 7 + | u32::from(self.r_length & 0x3) << 5 + | u32::from(self.r_extern) << 4 + | u32::from(self.r_type) & 0xf + }, + ); + Relocation { r_word0, r_word1 } + } +} + +/// absolute relocation type for Mach-O files +pub const R_ABS: u8 = 0; + +/* + * The r_address is not really the address as it's name indicates but an offset. + * In 4.3BSD a.out objects this offset is from the start of the "segment" for + * which relocation entry is for (text or data). For Mach-O object files it is + * also an offset but from the start of the "section" for which the relocation + * entry is for. See comments in about the r_address feild + * in images for used with the dynamic linker. + * + * In 4.3BSD a.out objects if r_extern is zero then r_symbolnum is an ordinal + * for the segment the symbol being relocated is in. These ordinals are the + * symbol types N_TEXT, N_DATA, N_BSS or N_ABS. In Mach-O object files these + * ordinals refer to the sections in the object file in the order their section + * structures appear in the headers of the object file they are in. The first + * section has the ordinal 1, the second 2, and so on. This means that the + * same ordinal in two different object files could refer to two different + * sections. And further could have still different ordinals when combined + * by the link-editor. The value R_ABS is used for relocation entries for + * absolute symbols which need no further relocation. + */ + +/* + * For RISC machines some of the references are split across two instructions + * and the instruction does not contain the complete value of the reference. + * In these cases a second, or paired relocation entry, follows each of these + * relocation entries, using a PAIR r_type, which contains the other part of the + * reference not contained in the instruction. This other part is stored in the + * pair's r_address field. The exact number of bits of the other part of the + * reference store in the r_address field is dependent on the particular + * relocation type for the particular architecture. + */ + +/* + * To make scattered loading by the link editor work correctly "local" + * relocation entries can't be used when the item to be relocated is the value + * of a symbol plus an offset (where the resulting expresion is outside the + * block the link editor is moving, a blocks are divided at symbol addresses). + * In this case. where the item is a symbol value plus offset, the link editor + * needs to know more than just the section the symbol was defined. What is + * needed is the actual value of the symbol without the offset so it can do the + * relocation correctly based on where the value of the symbol got relocated to + * not the value of the expression (with the offset added to the symbol value). + * So for the NeXT 2.0 release no "local" relocation entries are ever used when + * there is a non-zero offset added to a symbol. The "external" and "local" + * relocation entries remain unchanged. + * + * The implemention is quite messy given the compatibility with the existing + * relocation entry format. The ASSUMPTION is that a section will never be + * bigger than 2**24 - 1 (0x00ffffff or 16,777,215) bytes. This assumption + * allows the r_address (which is really an offset) to fit in 24 bits and high + * bit of the r_address field in the relocation_info structure to indicate + * it is really a scattered_relocation_info structure. Since these are only + * used in places where "local" relocation entries are used and not where + * "external" relocation entries are used the r_extern field has been removed. + * + * For scattered loading to work on a RISC machine where some of the references + * are split across two instructions the link editor needs to be assured that + * each reference has a unique 32 bit reference (that more than one reference is + * NOT sharing the same high 16 bits for example) so it move each referenced + * item independent of each other. Some compilers guarantees this but the + * compilers don't so scattered loading can be done on those that do guarantee + * this. + */ + +/// Bit set in `Relocation::r_word0` for scattered relocations. +pub const R_SCATTERED: u32 = 0x8000_0000; + +#[derive(Debug, Clone, Copy)] +pub struct ScatteredRelocationInfo { + /// offset in the section to what is being relocated + pub r_address: u32, + /// if not 0, machine specific relocation type + pub r_type: u8, + /// 0=byte, 1=word, 2=long, 3=quad + pub r_length: u8, + /// was relocated pc relative already + pub r_pcrel: bool, + /// the value the item to be relocated is refering to (without any offset added) + pub r_value: u32, +} + +impl ScatteredRelocationInfo { + /// Combine the fields into a `Relocation`. + pub fn relocation(self, endian: E) -> Relocation { + let r_word0 = U32::new( + endian, + self.r_address & 0x00ff_ffff + | u32::from(self.r_type & 0xf) << 24 + | u32::from(self.r_length & 0x3) << 28 + | u32::from(self.r_pcrel) << 30 + | R_SCATTERED, + ); + let r_word1 = U32::new(endian, self.r_value); + Relocation { r_word0, r_word1 } + } +} + +/* + * Relocation types used in a generic implementation. Relocation entries for + * normal things use the generic relocation as discribed above and their r_type + * is GENERIC_RELOC_VANILLA (a value of zero). + * + * Another type of generic relocation, GENERIC_RELOC_SECTDIFF, is to support + * the difference of two symbols defined in different sections. That is the + * expression "symbol1 - symbol2 + constant" is a relocatable expression when + * both symbols are defined in some section. For this type of relocation the + * both relocations entries are scattered relocation entries. The value of + * symbol1 is stored in the first relocation entry's r_value field and the + * value of symbol2 is stored in the pair's r_value field. + * + * A special case for a prebound lazy pointer is needed to beable to set the + * value of the lazy pointer back to its non-prebound state. This is done + * using the GENERIC_RELOC_PB_LA_PTR r_type. This is a scattered relocation + * entry where the r_value feild is the value of the lazy pointer not prebound. + */ +/// generic relocation as discribed above +pub const GENERIC_RELOC_VANILLA: u8 = 0; +/// Only follows a GENERIC_RELOC_SECTDIFF +pub const GENERIC_RELOC_PAIR: u8 = 1; +pub const GENERIC_RELOC_SECTDIFF: u8 = 2; +/// prebound lazy pointer +pub const GENERIC_RELOC_PB_LA_PTR: u8 = 3; +pub const GENERIC_RELOC_LOCAL_SECTDIFF: u8 = 4; +/// thread local variables +pub const GENERIC_RELOC_TLV: u8 = 5; + +// Definitions from "/usr/include/mach-o/arm/reloc.h". + +/* + * Relocation types used in the arm implementation. Relocation entries for + * things other than instructions use the same generic relocation as discribed + * in and their r_type is ARM_RELOC_VANILLA, one of the + * *_SECTDIFF or the *_PB_LA_PTR types. The rest of the relocation types are + * for instructions. Since they are for instructions the r_address field + * indicates the 32 bit instruction that the relocation is to be preformed on. + */ +/// generic relocation as discribed above +pub const ARM_RELOC_VANILLA: u8 = 0; +/// the second relocation entry of a pair +pub const ARM_RELOC_PAIR: u8 = 1; +/// a PAIR follows with subtract symbol value +pub const ARM_RELOC_SECTDIFF: u8 = 2; +/// like ARM_RELOC_SECTDIFF, but the symbol referenced was local. +pub const ARM_RELOC_LOCAL_SECTDIFF: u8 = 3; +/// prebound lazy pointer +pub const ARM_RELOC_PB_LA_PTR: u8 = 4; +/// 24 bit branch displacement (to a word address) +pub const ARM_RELOC_BR24: u8 = 5; +/// 22 bit branch displacement (to a half-word address) +pub const ARM_THUMB_RELOC_BR22: u8 = 6; +/// obsolete - a thumb 32-bit branch instruction possibly needing page-spanning branch workaround +pub const ARM_THUMB_32BIT_BRANCH: u8 = 7; + +/* + * For these two r_type relocations they always have a pair following them + * and the r_length bits are used differently. The encoding of the + * r_length is as follows: + * low bit of r_length: + * 0 - :lower16: for movw instructions + * 1 - :upper16: for movt instructions + * high bit of r_length: + * 0 - arm instructions + * 1 - thumb instructions + * the other half of the relocated expression is in the following pair + * relocation entry in the the low 16 bits of r_address field. + */ +pub const ARM_RELOC_HALF: u8 = 8; +pub const ARM_RELOC_HALF_SECTDIFF: u8 = 9; + +// Definitions from "/usr/include/mach-o/arm64/reloc.h". + +/* + * Relocation types used in the arm64 implementation. + */ +/// for pointers +pub const ARM64_RELOC_UNSIGNED: u8 = 0; +/// must be followed by a ARM64_RELOC_UNSIGNED +pub const ARM64_RELOC_SUBTRACTOR: u8 = 1; +/// a B/BL instruction with 26-bit displacement +pub const ARM64_RELOC_BRANCH26: u8 = 2; +/// pc-rel distance to page of target +pub const ARM64_RELOC_PAGE21: u8 = 3; +/// offset within page, scaled by r_length +pub const ARM64_RELOC_PAGEOFF12: u8 = 4; +/// pc-rel distance to page of GOT slot +pub const ARM64_RELOC_GOT_LOAD_PAGE21: u8 = 5; +/// offset within page of GOT slot, scaled by r_length +pub const ARM64_RELOC_GOT_LOAD_PAGEOFF12: u8 = 6; +/// for pointers to GOT slots +pub const ARM64_RELOC_POINTER_TO_GOT: u8 = 7; +/// pc-rel distance to page of TLVP slot +pub const ARM64_RELOC_TLVP_LOAD_PAGE21: u8 = 8; +/// offset within page of TLVP slot, scaled by r_length +pub const ARM64_RELOC_TLVP_LOAD_PAGEOFF12: u8 = 9; +/// must be followed by PAGE21 or PAGEOFF12 +pub const ARM64_RELOC_ADDEND: u8 = 10; + +// An arm64e authenticated pointer. +// +// Represents a pointer to a symbol (like ARM64_RELOC_UNSIGNED). +// Additionally, the resulting pointer is signed. The signature is +// specified in the target location: the addend is restricted to the lower +// 32 bits (instead of the full 64 bits for ARM64_RELOC_UNSIGNED): +// +// |63|62|61-51|50-49| 48 |47 - 32|31 - 0| +// | 1| 0| 0 | key | addr | discriminator | addend | +// +// The key is one of: +// IA: 00 IB: 01 +// DA: 10 DB: 11 +// +// The discriminator field is used as extra signature diversification. +// +// The addr field indicates whether the target address should be blended +// into the discriminator. +// +pub const ARM64_RELOC_AUTHENTICATED_POINTER: u8 = 11; + +// Definitions from "/usr/include/mach-o/ppc/reloc.h". + +/* + * Relocation types used in the ppc implementation. Relocation entries for + * things other than instructions use the same generic relocation as discribed + * above and their r_type is RELOC_VANILLA. The rest of the relocation types + * are for instructions. Since they are for instructions the r_address field + * indicates the 32 bit instruction that the relocation is to be preformed on. + * The fields r_pcrel and r_length are ignored for non-RELOC_VANILLA r_types + * except for PPC_RELOC_BR14. + * + * For PPC_RELOC_BR14 if the r_length is the unused value 3, then the branch was + * statically predicted setting or clearing the Y-bit based on the sign of the + * displacement or the opcode. If this is the case the static linker must flip + * the value of the Y-bit if the sign of the displacement changes for non-branch + * always conditions. + */ +/// generic relocation as discribed above +pub const PPC_RELOC_VANILLA: u8 = 0; +/// the second relocation entry of a pair +pub const PPC_RELOC_PAIR: u8 = 1; +/// 14 bit branch displacement (to a word address) +pub const PPC_RELOC_BR14: u8 = 2; +/// 24 bit branch displacement (to a word address) +pub const PPC_RELOC_BR24: u8 = 3; +/// a PAIR follows with the low half +pub const PPC_RELOC_HI16: u8 = 4; +/// a PAIR follows with the high half +pub const PPC_RELOC_LO16: u8 = 5; +/// Same as the RELOC_HI16 except the low 16 bits and the high 16 bits are added together +/// with the low 16 bits sign extened first. This means if bit 15 of the low 16 bits is +/// set the high 16 bits stored in the instruction will be adjusted. +pub const PPC_RELOC_HA16: u8 = 6; +/// Same as the LO16 except that the low 2 bits are not stored in the instruction and are +/// always zero. This is used in double word load/store instructions. +pub const PPC_RELOC_LO14: u8 = 7; +/// a PAIR follows with subtract symbol value +pub const PPC_RELOC_SECTDIFF: u8 = 8; +/// prebound lazy pointer +pub const PPC_RELOC_PB_LA_PTR: u8 = 9; +/// section difference forms of above. a PAIR +pub const PPC_RELOC_HI16_SECTDIFF: u8 = 10; +/// follows these with subtract symbol value +pub const PPC_RELOC_LO16_SECTDIFF: u8 = 11; +pub const PPC_RELOC_HA16_SECTDIFF: u8 = 12; +pub const PPC_RELOC_JBSR: u8 = 13; +pub const PPC_RELOC_LO14_SECTDIFF: u8 = 14; +/// like PPC_RELOC_SECTDIFF, but the symbol referenced was local. +pub const PPC_RELOC_LOCAL_SECTDIFF: u8 = 15; + +// Definitions from "/usr/include/mach-o/x86_64/reloc.h". + +/* + * Relocations for x86_64 are a bit different than for other architectures in + * Mach-O: Scattered relocations are not used. Almost all relocations produced + * by the compiler are external relocations. An external relocation has the + * r_extern bit set to 1 and the r_symbolnum field contains the symbol table + * index of the target label. + * + * When the assembler is generating relocations, if the target label is a local + * label (begins with 'L'), then the previous non-local label in the same + * section is used as the target of the external relocation. An addend is used + * with the distance from that non-local label to the target label. Only when + * there is no previous non-local label in the section is an internal + * relocation used. + * + * The addend (i.e. the 4 in _foo+4) is encoded in the instruction (Mach-O does + * not have RELA relocations). For PC-relative relocations, the addend is + * stored directly in the instruction. This is different from other Mach-O + * architectures, which encode the addend minus the current section offset. + * + * The relocation types are: + * + * X86_64_RELOC_UNSIGNED // for absolute addresses + * X86_64_RELOC_SIGNED // for signed 32-bit displacement + * X86_64_RELOC_BRANCH // a CALL/JMP instruction with 32-bit displacement + * X86_64_RELOC_GOT_LOAD // a MOVQ load of a GOT entry + * X86_64_RELOC_GOT // other GOT references + * X86_64_RELOC_SUBTRACTOR // must be followed by a X86_64_RELOC_UNSIGNED + * + * The following are sample assembly instructions, followed by the relocation + * and section content they generate in an object file: + * + * call _foo + * r_type=X86_64_RELOC_BRANCH, r_length=2, r_extern=1, r_pcrel=1, r_symbolnum=_foo + * E8 00 00 00 00 + * + * call _foo+4 + * r_type=X86_64_RELOC_BRANCH, r_length=2, r_extern=1, r_pcrel=1, r_symbolnum=_foo + * E8 04 00 00 00 + * + * movq _foo@GOTPCREL(%rip), %rax + * r_type=X86_64_RELOC_GOT_LOAD, r_length=2, r_extern=1, r_pcrel=1, r_symbolnum=_foo + * 48 8B 05 00 00 00 00 + * + * pushq _foo@GOTPCREL(%rip) + * r_type=X86_64_RELOC_GOT, r_length=2, r_extern=1, r_pcrel=1, r_symbolnum=_foo + * FF 35 00 00 00 00 + * + * movl _foo(%rip), %eax + * r_type=X86_64_RELOC_SIGNED, r_length=2, r_extern=1, r_pcrel=1, r_symbolnum=_foo + * 8B 05 00 00 00 00 + * + * movl _foo+4(%rip), %eax + * r_type=X86_64_RELOC_SIGNED, r_length=2, r_extern=1, r_pcrel=1, r_symbolnum=_foo + * 8B 05 04 00 00 00 + * + * movb $0x12, _foo(%rip) + * r_type=X86_64_RELOC_SIGNED, r_length=2, r_extern=1, r_pcrel=1, r_symbolnum=_foo + * C6 05 FF FF FF FF 12 + * + * movl $0x12345678, _foo(%rip) + * r_type=X86_64_RELOC_SIGNED, r_length=2, r_extern=1, r_pcrel=1, r_symbolnum=_foo + * C7 05 FC FF FF FF 78 56 34 12 + * + * .quad _foo + * r_type=X86_64_RELOC_UNSIGNED, r_length=3, r_extern=1, r_pcrel=0, r_symbolnum=_foo + * 00 00 00 00 00 00 00 00 + * + * .quad _foo+4 + * r_type=X86_64_RELOC_UNSIGNED, r_length=3, r_extern=1, r_pcrel=0, r_symbolnum=_foo + * 04 00 00 00 00 00 00 00 + * + * .quad _foo - _bar + * r_type=X86_64_RELOC_SUBTRACTOR, r_length=3, r_extern=1, r_pcrel=0, r_symbolnum=_bar + * r_type=X86_64_RELOC_UNSIGNED, r_length=3, r_extern=1, r_pcrel=0, r_symbolnum=_foo + * 00 00 00 00 00 00 00 00 + * + * .quad _foo - _bar + 4 + * r_type=X86_64_RELOC_SUBTRACTOR, r_length=3, r_extern=1, r_pcrel=0, r_symbolnum=_bar + * r_type=X86_64_RELOC_UNSIGNED, r_length=3, r_extern=1, r_pcrel=0, r_symbolnum=_foo + * 04 00 00 00 00 00 00 00 + * + * .long _foo - _bar + * r_type=X86_64_RELOC_SUBTRACTOR, r_length=2, r_extern=1, r_pcrel=0, r_symbolnum=_bar + * r_type=X86_64_RELOC_UNSIGNED, r_length=2, r_extern=1, r_pcrel=0, r_symbolnum=_foo + * 00 00 00 00 + * + * lea L1(%rip), %rax + * r_type=X86_64_RELOC_SIGNED, r_length=2, r_extern=1, r_pcrel=1, r_symbolnum=_prev + * 48 8d 05 12 00 00 00 + * // assumes _prev is the first non-local label 0x12 bytes before L1 + * + * lea L0(%rip), %rax + * r_type=X86_64_RELOC_SIGNED, r_length=2, r_extern=0, r_pcrel=1, r_symbolnum=3 + * 48 8d 05 56 00 00 00 + * // assumes L0 is in third section and there is no previous non-local label. + * // The rip-relative-offset of 0x00000056 is L0-address_of_next_instruction. + * // address_of_next_instruction is the address of the relocation + 4. + * + * add $6,L0(%rip) + * r_type=X86_64_RELOC_SIGNED_1, r_length=2, r_extern=0, r_pcrel=1, r_symbolnum=3 + * 83 05 18 00 00 00 06 + * // assumes L0 is in third section and there is no previous non-local label. + * // The rip-relative-offset of 0x00000018 is L0-address_of_next_instruction. + * // address_of_next_instruction is the address of the relocation + 4 + 1. + * // The +1 comes from SIGNED_1. This is used because the relocation is not + * // at the end of the instruction. + * + * .quad L1 + * r_type=X86_64_RELOC_UNSIGNED, r_length=3, r_extern=1, r_pcrel=0, r_symbolnum=_prev + * 12 00 00 00 00 00 00 00 + * // assumes _prev is the first non-local label 0x12 bytes before L1 + * + * .quad L0 + * r_type=X86_64_RELOC_UNSIGNED, r_length=3, r_extern=0, r_pcrel=0, r_symbolnum=3 + * 56 00 00 00 00 00 00 00 + * // assumes L0 is in third section, has an address of 0x00000056 in .o + * // file, and there is no previous non-local label + * + * .quad _foo - . + * r_type=X86_64_RELOC_SUBTRACTOR, r_length=3, r_extern=1, r_pcrel=0, r_symbolnum=_prev + * r_type=X86_64_RELOC_UNSIGNED, r_length=3, r_extern=1, r_pcrel=0, r_symbolnum=_foo + * EE FF FF FF FF FF FF FF + * // assumes _prev is the first non-local label 0x12 bytes before this + * // .quad + * + * .quad _foo - L1 + * r_type=X86_64_RELOC_SUBTRACTOR, r_length=3, r_extern=1, r_pcrel=0, r_symbolnum=_prev + * r_type=X86_64_RELOC_UNSIGNED, r_length=3, r_extern=1, r_pcrel=0, r_symbolnum=_foo + * EE FF FF FF FF FF FF FF + * // assumes _prev is the first non-local label 0x12 bytes before L1 + * + * .quad L1 - _prev + * // No relocations. This is an assembly time constant. + * 12 00 00 00 00 00 00 00 + * // assumes _prev is the first non-local label 0x12 bytes before L1 + * + * + * + * In final linked images, there are only two valid relocation kinds: + * + * r_type=X86_64_RELOC_UNSIGNED, r_length=3, r_pcrel=0, r_extern=1, r_symbolnum=sym_index + * This tells dyld to add the address of a symbol to a pointer sized (8-byte) + * piece of data (i.e on disk the 8-byte piece of data contains the addend). The + * r_symbolnum contains the index into the symbol table of the target symbol. + * + * r_type=X86_64_RELOC_UNSIGNED, r_length=3, r_pcrel=0, r_extern=0, r_symbolnum=0 + * This tells dyld to adjust the pointer sized (8-byte) piece of data by the amount + * the containing image was loaded from its base address (e.g. slide). + * + */ +/// for absolute addresses +pub const X86_64_RELOC_UNSIGNED: u8 = 0; +/// for signed 32-bit displacement +pub const X86_64_RELOC_SIGNED: u8 = 1; +/// a CALL/JMP instruction with 32-bit displacement +pub const X86_64_RELOC_BRANCH: u8 = 2; +/// a MOVQ load of a GOT entry +pub const X86_64_RELOC_GOT_LOAD: u8 = 3; +/// other GOT references +pub const X86_64_RELOC_GOT: u8 = 4; +/// must be followed by a X86_64_RELOC_UNSIGNED +pub const X86_64_RELOC_SUBTRACTOR: u8 = 5; +/// for signed 32-bit displacement with a -1 addend +pub const X86_64_RELOC_SIGNED_1: u8 = 6; +/// for signed 32-bit displacement with a -2 addend +pub const X86_64_RELOC_SIGNED_2: u8 = 7; +/// for signed 32-bit displacement with a -4 addend +pub const X86_64_RELOC_SIGNED_4: u8 = 8; +/// for thread local variables +pub const X86_64_RELOC_TLV: u8 = 9; + +unsafe_impl_pod!(FatHeader, FatArch32, FatArch64,); +unsafe_impl_endian_pod!( + DyldCacheHeader, + DyldCacheMappingInfo, + DyldCacheImageInfo, + DyldSubCacheInfo, + MachHeader32, + MachHeader64, + LoadCommand, + LcStr, + SegmentCommand32, + SegmentCommand64, + Section32, + Section64, + Fvmlib, + FvmlibCommand, + Dylib, + DylibCommand, + SubFrameworkCommand, + SubClientCommand, + SubUmbrellaCommand, + SubLibraryCommand, + PreboundDylibCommand, + DylinkerCommand, + ThreadCommand, + RoutinesCommand32, + RoutinesCommand64, + SymtabCommand, + DysymtabCommand, + DylibTableOfContents, + DylibModule32, + DylibModule64, + DylibReference, + TwolevelHintsCommand, + TwolevelHint, + PrebindCksumCommand, + UuidCommand, + RpathCommand, + LinkeditDataCommand, + FilesetEntryCommand, + EncryptionInfoCommand32, + EncryptionInfoCommand64, + VersionMinCommand, + BuildVersionCommand, + BuildToolVersion, + DyldInfoCommand, + LinkerOptionCommand, + SymsegCommand, + IdentCommand, + FvmfileCommand, + EntryPointCommand, + SourceVersionCommand, + DataInCodeEntry, + //TlvDescriptor, + NoteCommand, + Nlist32, + Nlist64, + Relocation, +); diff --git a/crux-mir/lib/object/src/pe.rs b/crux-mir/lib/object/src/pe.rs new file mode 100644 index 000000000..c89b86caa --- /dev/null +++ b/crux-mir/lib/object/src/pe.rs @@ -0,0 +1,3035 @@ +//! PE/COFF definitions. +//! +//! These definitions are independent of read/write support, although we do implement +//! some traits useful for those. +//! +//! This module is based heavily on "winnt.h" (10.0.17763.0). + +#![allow(missing_docs)] + +use crate::endian::{LittleEndian as LE, U16Bytes, U32Bytes, I32, U16, U32, U64}; +use crate::pod::Pod; + +/// MZ +pub const IMAGE_DOS_SIGNATURE: u16 = 0x5A4D; +/// NE +pub const IMAGE_OS2_SIGNATURE: u16 = 0x454E; +/// LE +pub const IMAGE_OS2_SIGNATURE_LE: u16 = 0x454C; +/// LE +pub const IMAGE_VXD_SIGNATURE: u16 = 0x454C; +/// PE00 +pub const IMAGE_NT_SIGNATURE: u32 = 0x0000_4550; + +/// DOS .EXE header +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageDosHeader { + /// Magic number + pub e_magic: U16, + /// Bytes on last page of file + pub e_cblp: U16, + /// Pages in file + pub e_cp: U16, + /// Relocations + pub e_crlc: U16, + /// Size of header in paragraphs + pub e_cparhdr: U16, + /// Minimum extra paragraphs needed + pub e_minalloc: U16, + /// Maximum extra paragraphs needed + pub e_maxalloc: U16, + /// Initial (relative) SS value + pub e_ss: U16, + /// Initial SP value + pub e_sp: U16, + /// Checksum + pub e_csum: U16, + /// Initial IP value + pub e_ip: U16, + /// Initial (relative) CS value + pub e_cs: U16, + /// File address of relocation table + pub e_lfarlc: U16, + /// Overlay number + pub e_ovno: U16, + /// Reserved words + pub e_res: [U16; 4], + /// OEM identifier (for e_oeminfo) + pub e_oemid: U16, + /// OEM information; e_oemid specific + pub e_oeminfo: U16, + /// Reserved words + pub e_res2: [U16; 10], + /// File address of new exe header + pub e_lfanew: U32, +} + +/// OS/2 .EXE header +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageOs2Header { + /// Magic number + pub ne_magic: U16, + /// Version number + pub ne_ver: i8, + /// Revision number + pub ne_rev: i8, + /// Offset of Entry Table + pub ne_enttab: U16, + /// Number of bytes in Entry Table + pub ne_cbenttab: U16, + /// Checksum of whole file + pub ne_crc: I32, + /// Flag word + pub ne_flags: U16, + /// Automatic data segment number + pub ne_autodata: U16, + /// Initial heap allocation + pub ne_heap: U16, + /// Initial stack allocation + pub ne_stack: U16, + /// Initial CS:IP setting + pub ne_csip: I32, + /// Initial SS:SP setting + pub ne_sssp: I32, + /// Count of file segments + pub ne_cseg: U16, + /// Entries in Module Reference Table + pub ne_cmod: U16, + /// Size of non-resident name table + pub ne_cbnrestab: U16, + /// Offset of Segment Table + pub ne_segtab: U16, + /// Offset of Resource Table + pub ne_rsrctab: U16, + /// Offset of resident name table + pub ne_restab: U16, + /// Offset of Module Reference Table + pub ne_modtab: U16, + /// Offset of Imported Names Table + pub ne_imptab: U16, + /// Offset of Non-resident Names Table + pub ne_nrestab: I32, + /// Count of movable entries + pub ne_cmovent: U16, + /// Segment alignment shift count + pub ne_align: U16, + /// Count of resource segments + pub ne_cres: U16, + /// Target Operating system + pub ne_exetyp: u8, + /// Other .EXE flags + pub ne_flagsothers: u8, + /// offset to return thunks + pub ne_pretthunks: U16, + /// offset to segment ref. bytes + pub ne_psegrefbytes: U16, + /// Minimum code swap area size + pub ne_swaparea: U16, + /// Expected Windows version number + pub ne_expver: U16, +} + +/// Windows VXD header +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageVxdHeader { + /// Magic number + pub e32_magic: U16, + /// The byte ordering for the VXD + pub e32_border: u8, + /// The word ordering for the VXD + pub e32_worder: u8, + /// The EXE format level for now = 0 + pub e32_level: U32, + /// The CPU type + pub e32_cpu: U16, + /// The OS type + pub e32_os: U16, + /// Module version + pub e32_ver: U32, + /// Module flags + pub e32_mflags: U32, + /// Module # pages + pub e32_mpages: U32, + /// Object # for instruction pointer + pub e32_startobj: U32, + /// Extended instruction pointer + pub e32_eip: U32, + /// Object # for stack pointer + pub e32_stackobj: U32, + /// Extended stack pointer + pub e32_esp: U32, + /// VXD page size + pub e32_pagesize: U32, + /// Last page size in VXD + pub e32_lastpagesize: U32, + /// Fixup section size + pub e32_fixupsize: U32, + /// Fixup section checksum + pub e32_fixupsum: U32, + /// Loader section size + pub e32_ldrsize: U32, + /// Loader section checksum + pub e32_ldrsum: U32, + /// Object table offset + pub e32_objtab: U32, + /// Number of objects in module + pub e32_objcnt: U32, + /// Object page map offset + pub e32_objmap: U32, + /// Object iterated data map offset + pub e32_itermap: U32, + /// Offset of Resource Table + pub e32_rsrctab: U32, + /// Number of resource entries + pub e32_rsrccnt: U32, + /// Offset of resident name table + pub e32_restab: U32, + /// Offset of Entry Table + pub e32_enttab: U32, + /// Offset of Module Directive Table + pub e32_dirtab: U32, + /// Number of module directives + pub e32_dircnt: U32, + /// Offset of Fixup Page Table + pub e32_fpagetab: U32, + /// Offset of Fixup Record Table + pub e32_frectab: U32, + /// Offset of Import Module Name Table + pub e32_impmod: U32, + /// Number of entries in Import Module Name Table + pub e32_impmodcnt: U32, + /// Offset of Import Procedure Name Table + pub e32_impproc: U32, + /// Offset of Per-Page Checksum Table + pub e32_pagesum: U32, + /// Offset of Enumerated Data Pages + pub e32_datapage: U32, + /// Number of preload pages + pub e32_preload: U32, + /// Offset of Non-resident Names Table + pub e32_nrestab: U32, + /// Size of Non-resident Name Table + pub e32_cbnrestab: U32, + /// Non-resident Name Table Checksum + pub e32_nressum: U32, + /// Object # for automatic data object + pub e32_autodata: U32, + /// Offset of the debugging information + pub e32_debuginfo: U32, + /// The length of the debugging info. in bytes + pub e32_debuglen: U32, + /// Number of instance pages in preload section of VXD file + pub e32_instpreload: U32, + /// Number of instance pages in demand load section of VXD file + pub e32_instdemand: U32, + /// Size of heap - for 16-bit apps + pub e32_heapsize: U32, + /// Reserved words + pub e32_res3: [u8; 12], + pub e32_winresoff: U32, + pub e32_winreslen: U32, + /// Device ID for VxD + pub e32_devid: U16, + /// DDK version for VxD + pub e32_ddkver: U16, +} + +/// A PE rich header entry. +/// +/// Rich headers have no official documentation, but have been heavily +/// reversed-engineered and documented in the wild, e.g.: +/// * `http://www.ntcore.com/files/richsign.htm` +/// * `https://www.researchgate.net/figure/Structure-of-the-Rich-Header_fig1_318145388` +/// +/// This data is "masked", i.e. XORed with a checksum derived from the file data. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct MaskedRichHeaderEntry { + pub masked_comp_id: U32, + pub masked_count: U32, +} + +// +// File header format. +// + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageFileHeader { + pub machine: U16, + pub number_of_sections: U16, + pub time_date_stamp: U32, + pub pointer_to_symbol_table: U32, + pub number_of_symbols: U32, + pub size_of_optional_header: U16, + pub characteristics: U16, +} + +pub const IMAGE_SIZEOF_FILE_HEADER: usize = 20; + +/// Relocation info stripped from file. +pub const IMAGE_FILE_RELOCS_STRIPPED: u16 = 0x0001; +/// File is executable (i.e. no unresolved external references). +pub const IMAGE_FILE_EXECUTABLE_IMAGE: u16 = 0x0002; +/// Line nunbers stripped from file. +pub const IMAGE_FILE_LINE_NUMS_STRIPPED: u16 = 0x0004; +/// Local symbols stripped from file. +pub const IMAGE_FILE_LOCAL_SYMS_STRIPPED: u16 = 0x0008; +/// Aggressively trim working set +pub const IMAGE_FILE_AGGRESIVE_WS_TRIM: u16 = 0x0010; +/// App can handle >2gb addresses +pub const IMAGE_FILE_LARGE_ADDRESS_AWARE: u16 = 0x0020; +/// Bytes of machine word are reversed. +pub const IMAGE_FILE_BYTES_REVERSED_LO: u16 = 0x0080; +/// 32 bit word machine. +pub const IMAGE_FILE_32BIT_MACHINE: u16 = 0x0100; +/// Debugging info stripped from file in .DBG file +pub const IMAGE_FILE_DEBUG_STRIPPED: u16 = 0x0200; +/// If Image is on removable media, copy and run from the swap file. +pub const IMAGE_FILE_REMOVABLE_RUN_FROM_SWAP: u16 = 0x0400; +/// If Image is on Net, copy and run from the swap file. +pub const IMAGE_FILE_NET_RUN_FROM_SWAP: u16 = 0x0800; +/// System File. +pub const IMAGE_FILE_SYSTEM: u16 = 0x1000; +/// File is a DLL. +pub const IMAGE_FILE_DLL: u16 = 0x2000; +/// File should only be run on a UP machine +pub const IMAGE_FILE_UP_SYSTEM_ONLY: u16 = 0x4000; +/// Bytes of machine word are reversed. +pub const IMAGE_FILE_BYTES_REVERSED_HI: u16 = 0x8000; + +pub const IMAGE_FILE_MACHINE_UNKNOWN: u16 = 0; +/// Useful for indicating we want to interact with the host and not a WoW guest. +pub const IMAGE_FILE_MACHINE_TARGET_HOST: u16 = 0x0001; +/// Intel 386. +pub const IMAGE_FILE_MACHINE_I386: u16 = 0x014c; +/// MIPS little-endian, 0x160 big-endian +pub const IMAGE_FILE_MACHINE_R3000: u16 = 0x0162; +/// MIPS little-endian +pub const IMAGE_FILE_MACHINE_R4000: u16 = 0x0166; +/// MIPS little-endian +pub const IMAGE_FILE_MACHINE_R10000: u16 = 0x0168; +/// MIPS little-endian WCE v2 +pub const IMAGE_FILE_MACHINE_WCEMIPSV2: u16 = 0x0169; +/// Alpha_AXP +pub const IMAGE_FILE_MACHINE_ALPHA: u16 = 0x0184; +/// SH3 little-endian +pub const IMAGE_FILE_MACHINE_SH3: u16 = 0x01a2; +pub const IMAGE_FILE_MACHINE_SH3DSP: u16 = 0x01a3; +/// SH3E little-endian +pub const IMAGE_FILE_MACHINE_SH3E: u16 = 0x01a4; +/// SH4 little-endian +pub const IMAGE_FILE_MACHINE_SH4: u16 = 0x01a6; +/// SH5 +pub const IMAGE_FILE_MACHINE_SH5: u16 = 0x01a8; +/// ARM Little-Endian +pub const IMAGE_FILE_MACHINE_ARM: u16 = 0x01c0; +/// ARM Thumb/Thumb-2 Little-Endian +pub const IMAGE_FILE_MACHINE_THUMB: u16 = 0x01c2; +/// ARM Thumb-2 Little-Endian +pub const IMAGE_FILE_MACHINE_ARMNT: u16 = 0x01c4; +pub const IMAGE_FILE_MACHINE_AM33: u16 = 0x01d3; +/// IBM PowerPC Little-Endian +pub const IMAGE_FILE_MACHINE_POWERPC: u16 = 0x01F0; +pub const IMAGE_FILE_MACHINE_POWERPCFP: u16 = 0x01f1; +/// Intel 64 +pub const IMAGE_FILE_MACHINE_IA64: u16 = 0x0200; +/// MIPS +pub const IMAGE_FILE_MACHINE_MIPS16: u16 = 0x0266; +/// ALPHA64 +pub const IMAGE_FILE_MACHINE_ALPHA64: u16 = 0x0284; +/// MIPS +pub const IMAGE_FILE_MACHINE_MIPSFPU: u16 = 0x0366; +/// MIPS +pub const IMAGE_FILE_MACHINE_MIPSFPU16: u16 = 0x0466; +pub const IMAGE_FILE_MACHINE_AXP64: u16 = IMAGE_FILE_MACHINE_ALPHA64; +/// Infineon +pub const IMAGE_FILE_MACHINE_TRICORE: u16 = 0x0520; +pub const IMAGE_FILE_MACHINE_CEF: u16 = 0x0CEF; +/// EFI Byte Code +pub const IMAGE_FILE_MACHINE_EBC: u16 = 0x0EBC; +/// AMD64 (K8) +pub const IMAGE_FILE_MACHINE_AMD64: u16 = 0x8664; +/// M32R little-endian +pub const IMAGE_FILE_MACHINE_M32R: u16 = 0x9041; +/// ARM64 Little-Endian +pub const IMAGE_FILE_MACHINE_ARM64: u16 = 0xAA64; +pub const IMAGE_FILE_MACHINE_CEE: u16 = 0xC0EE; +/// RISCV32 +pub const IMAGE_FILE_MACHINE_RISCV32: u16 = 0x5032; +/// RISCV64 +pub const IMAGE_FILE_MACHINE_RISCV64: u16 = 0x5064; +/// RISCV128 +pub const IMAGE_FILE_MACHINE_RISCV128: u16 = 0x5128; + +// +// Directory format. +// + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageDataDirectory { + pub virtual_address: U32, + pub size: U32, +} + +pub const IMAGE_NUMBEROF_DIRECTORY_ENTRIES: usize = 16; + +// +// Optional header format. +// + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageOptionalHeader32 { + // Standard fields. + pub magic: U16, + pub major_linker_version: u8, + pub minor_linker_version: u8, + pub size_of_code: U32, + pub size_of_initialized_data: U32, + pub size_of_uninitialized_data: U32, + pub address_of_entry_point: U32, + pub base_of_code: U32, + pub base_of_data: U32, + + // NT additional fields. + pub image_base: U32, + pub section_alignment: U32, + pub file_alignment: U32, + pub major_operating_system_version: U16, + pub minor_operating_system_version: U16, + pub major_image_version: U16, + pub minor_image_version: U16, + pub major_subsystem_version: U16, + pub minor_subsystem_version: U16, + pub win32_version_value: U32, + pub size_of_image: U32, + pub size_of_headers: U32, + pub check_sum: U32, + pub subsystem: U16, + pub dll_characteristics: U16, + pub size_of_stack_reserve: U32, + pub size_of_stack_commit: U32, + pub size_of_heap_reserve: U32, + pub size_of_heap_commit: U32, + pub loader_flags: U32, + pub number_of_rva_and_sizes: U32, + //pub data_directory: [ImageDataDirectory; IMAGE_NUMBEROF_DIRECTORY_ENTRIES], +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageRomOptionalHeader { + pub magic: U16, + pub major_linker_version: u8, + pub minor_linker_version: u8, + pub size_of_code: U32, + pub size_of_initialized_data: U32, + pub size_of_uninitialized_data: U32, + pub address_of_entry_point: U32, + pub base_of_code: U32, + pub base_of_data: U32, + pub base_of_bss: U32, + pub gpr_mask: U32, + pub cpr_mask: [U32; 4], + pub gp_value: U32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageOptionalHeader64 { + pub magic: U16, + pub major_linker_version: u8, + pub minor_linker_version: u8, + pub size_of_code: U32, + pub size_of_initialized_data: U32, + pub size_of_uninitialized_data: U32, + pub address_of_entry_point: U32, + pub base_of_code: U32, + pub image_base: U64, + pub section_alignment: U32, + pub file_alignment: U32, + pub major_operating_system_version: U16, + pub minor_operating_system_version: U16, + pub major_image_version: U16, + pub minor_image_version: U16, + pub major_subsystem_version: U16, + pub minor_subsystem_version: U16, + pub win32_version_value: U32, + pub size_of_image: U32, + pub size_of_headers: U32, + pub check_sum: U32, + pub subsystem: U16, + pub dll_characteristics: U16, + pub size_of_stack_reserve: U64, + pub size_of_stack_commit: U64, + pub size_of_heap_reserve: U64, + pub size_of_heap_commit: U64, + pub loader_flags: U32, + pub number_of_rva_and_sizes: U32, + //pub data_directory: [ImageDataDirectory; IMAGE_NUMBEROF_DIRECTORY_ENTRIES], +} + +pub const IMAGE_NT_OPTIONAL_HDR32_MAGIC: u16 = 0x10b; +pub const IMAGE_NT_OPTIONAL_HDR64_MAGIC: u16 = 0x20b; +pub const IMAGE_ROM_OPTIONAL_HDR_MAGIC: u16 = 0x107; + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageNtHeaders64 { + pub signature: U32, + pub file_header: ImageFileHeader, + pub optional_header: ImageOptionalHeader64, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageNtHeaders32 { + pub signature: U32, + pub file_header: ImageFileHeader, + pub optional_header: ImageOptionalHeader32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageRomHeaders { + pub file_header: ImageFileHeader, + pub optional_header: ImageRomOptionalHeader, +} + +// Values for `ImageOptionalHeader*::subsystem`. + +/// Unknown subsystem. +pub const IMAGE_SUBSYSTEM_UNKNOWN: u16 = 0; +/// Image doesn't require a subsystem. +pub const IMAGE_SUBSYSTEM_NATIVE: u16 = 1; +/// Image runs in the Windows GUI subsystem. +pub const IMAGE_SUBSYSTEM_WINDOWS_GUI: u16 = 2; +/// Image runs in the Windows character subsystem. +pub const IMAGE_SUBSYSTEM_WINDOWS_CUI: u16 = 3; +/// image runs in the OS/2 character subsystem. +pub const IMAGE_SUBSYSTEM_OS2_CUI: u16 = 5; +/// image runs in the Posix character subsystem. +pub const IMAGE_SUBSYSTEM_POSIX_CUI: u16 = 7; +/// image is a native Win9x driver. +pub const IMAGE_SUBSYSTEM_NATIVE_WINDOWS: u16 = 8; +/// Image runs in the Windows CE subsystem. +pub const IMAGE_SUBSYSTEM_WINDOWS_CE_GUI: u16 = 9; +pub const IMAGE_SUBSYSTEM_EFI_APPLICATION: u16 = 10; +pub const IMAGE_SUBSYSTEM_EFI_BOOT_SERVICE_DRIVER: u16 = 11; +pub const IMAGE_SUBSYSTEM_EFI_RUNTIME_DRIVER: u16 = 12; +pub const IMAGE_SUBSYSTEM_EFI_ROM: u16 = 13; +pub const IMAGE_SUBSYSTEM_XBOX: u16 = 14; +pub const IMAGE_SUBSYSTEM_WINDOWS_BOOT_APPLICATION: u16 = 16; +pub const IMAGE_SUBSYSTEM_XBOX_CODE_CATALOG: u16 = 17; + +// Values for `ImageOptionalHeader*::dll_characteristics`. + +// IMAGE_LIBRARY_PROCESS_INIT 0x0001 // Reserved. +// IMAGE_LIBRARY_PROCESS_TERM 0x0002 // Reserved. +// IMAGE_LIBRARY_THREAD_INIT 0x0004 // Reserved. +// IMAGE_LIBRARY_THREAD_TERM 0x0008 // Reserved. +/// Image can handle a high entropy 64-bit virtual address space. +pub const IMAGE_DLLCHARACTERISTICS_HIGH_ENTROPY_VA: u16 = 0x0020; +/// DLL can move. +pub const IMAGE_DLLCHARACTERISTICS_DYNAMIC_BASE: u16 = 0x0040; +/// Code Integrity Image +pub const IMAGE_DLLCHARACTERISTICS_FORCE_INTEGRITY: u16 = 0x0080; +/// Image is NX compatible +pub const IMAGE_DLLCHARACTERISTICS_NX_COMPAT: u16 = 0x0100; +/// Image understands isolation and doesn't want it +pub const IMAGE_DLLCHARACTERISTICS_NO_ISOLATION: u16 = 0x0200; +/// Image does not use SEH. No SE handler may reside in this image +pub const IMAGE_DLLCHARACTERISTICS_NO_SEH: u16 = 0x0400; +/// Do not bind this image. +pub const IMAGE_DLLCHARACTERISTICS_NO_BIND: u16 = 0x0800; +/// Image should execute in an AppContainer +pub const IMAGE_DLLCHARACTERISTICS_APPCONTAINER: u16 = 0x1000; +/// Driver uses WDM model +pub const IMAGE_DLLCHARACTERISTICS_WDM_DRIVER: u16 = 0x2000; +/// Image supports Control Flow Guard. +pub const IMAGE_DLLCHARACTERISTICS_GUARD_CF: u16 = 0x4000; +pub const IMAGE_DLLCHARACTERISTICS_TERMINAL_SERVER_AWARE: u16 = 0x8000; + +// Indices for `ImageOptionalHeader*::data_directory`. + +/// Export Directory +pub const IMAGE_DIRECTORY_ENTRY_EXPORT: usize = 0; +/// Import Directory +pub const IMAGE_DIRECTORY_ENTRY_IMPORT: usize = 1; +/// Resource Directory +pub const IMAGE_DIRECTORY_ENTRY_RESOURCE: usize = 2; +/// Exception Directory +pub const IMAGE_DIRECTORY_ENTRY_EXCEPTION: usize = 3; +/// Security Directory +pub const IMAGE_DIRECTORY_ENTRY_SECURITY: usize = 4; +/// Base Relocation Table +pub const IMAGE_DIRECTORY_ENTRY_BASERELOC: usize = 5; +/// Debug Directory +pub const IMAGE_DIRECTORY_ENTRY_DEBUG: usize = 6; +// IMAGE_DIRECTORY_ENTRY_COPYRIGHT 7 // (X86 usage) +/// Architecture Specific Data +pub const IMAGE_DIRECTORY_ENTRY_ARCHITECTURE: usize = 7; +/// RVA of GP +pub const IMAGE_DIRECTORY_ENTRY_GLOBALPTR: usize = 8; +/// TLS Directory +pub const IMAGE_DIRECTORY_ENTRY_TLS: usize = 9; +/// Load Configuration Directory +pub const IMAGE_DIRECTORY_ENTRY_LOAD_CONFIG: usize = 10; +/// Bound Import Directory in headers +pub const IMAGE_DIRECTORY_ENTRY_BOUND_IMPORT: usize = 11; +/// Import Address Table +pub const IMAGE_DIRECTORY_ENTRY_IAT: usize = 12; +/// Delay Load Import Descriptors +pub const IMAGE_DIRECTORY_ENTRY_DELAY_IMPORT: usize = 13; +/// COM Runtime descriptor +pub const IMAGE_DIRECTORY_ENTRY_COM_DESCRIPTOR: usize = 14; + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Guid { + pub data1: U32, + pub data2: U16, + pub data3: U16, + pub data4: [u8; 8], +} + +pub type ClsId = Guid; + +/// Non-COFF Object file header +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct AnonObjectHeader { + /// Must be IMAGE_FILE_MACHINE_UNKNOWN + pub sig1: U16, + /// Must be 0xffff + pub sig2: U16, + /// >= 1 (implies the ClsId field is present) + pub version: U16, + pub machine: U16, + pub time_date_stamp: U32, + /// Used to invoke CoCreateInstance + pub class_id: ClsId, + /// Size of data that follows the header + pub size_of_data: U32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct AnonObjectHeaderV2 { + /// Must be IMAGE_FILE_MACHINE_UNKNOWN + pub sig1: U16, + /// Must be 0xffff + pub sig2: U16, + /// >= 2 (implies the Flags field is present - otherwise V1) + pub version: U16, + pub machine: U16, + pub time_date_stamp: U32, + /// Used to invoke CoCreateInstance + pub class_id: ClsId, + /// Size of data that follows the header + pub size_of_data: U32, + /// 0x1 -> contains metadata + pub flags: U32, + /// Size of CLR metadata + pub meta_data_size: U32, + /// Offset of CLR metadata + pub meta_data_offset: U32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct AnonObjectHeaderBigobj { + /* same as ANON_OBJECT_HEADER_V2 */ + /// Must be IMAGE_FILE_MACHINE_UNKNOWN + pub sig1: U16, + /// Must be 0xffff + pub sig2: U16, + /// >= 2 (implies the Flags field is present) + pub version: U16, + /// Actual machine - IMAGE_FILE_MACHINE_xxx + pub machine: U16, + pub time_date_stamp: U32, + /// {D1BAA1C7-BAEE-4ba9-AF20-FAF66AA4DCB8} + pub class_id: ClsId, + /// Size of data that follows the header + pub size_of_data: U32, + /// 0x1 -> contains metadata + pub flags: U32, + /// Size of CLR metadata + pub meta_data_size: U32, + /// Offset of CLR metadata + pub meta_data_offset: U32, + + /* bigobj specifics */ + /// extended from WORD + pub number_of_sections: U32, + pub pointer_to_symbol_table: U32, + pub number_of_symbols: U32, +} + +pub const IMAGE_SIZEOF_SHORT_NAME: usize = 8; + +// +// Section header format. +// + +#[derive(Debug, Default, Clone, Copy)] +#[repr(C)] +pub struct ImageSectionHeader { + pub name: [u8; IMAGE_SIZEOF_SHORT_NAME], + pub virtual_size: U32, + pub virtual_address: U32, + pub size_of_raw_data: U32, + pub pointer_to_raw_data: U32, + pub pointer_to_relocations: U32, + pub pointer_to_linenumbers: U32, + pub number_of_relocations: U16, + pub number_of_linenumbers: U16, + pub characteristics: U32, +} + +pub const IMAGE_SIZEOF_SECTION_HEADER: usize = 40; + +// Values for `ImageSectionHeader::characteristics`. + +// IMAGE_SCN_TYPE_REG 0x00000000 // Reserved. +// IMAGE_SCN_TYPE_DSECT 0x00000001 // Reserved. +// IMAGE_SCN_TYPE_NOLOAD 0x00000002 // Reserved. +// IMAGE_SCN_TYPE_GROUP 0x00000004 // Reserved. +/// Reserved. +pub const IMAGE_SCN_TYPE_NO_PAD: u32 = 0x0000_0008; +// IMAGE_SCN_TYPE_COPY 0x00000010 // Reserved. + +/// Section contains code. +pub const IMAGE_SCN_CNT_CODE: u32 = 0x0000_0020; +/// Section contains initialized data. +pub const IMAGE_SCN_CNT_INITIALIZED_DATA: u32 = 0x0000_0040; +/// Section contains uninitialized data. +pub const IMAGE_SCN_CNT_UNINITIALIZED_DATA: u32 = 0x0000_0080; + +/// Reserved. +pub const IMAGE_SCN_LNK_OTHER: u32 = 0x0000_0100; +/// Section contains comments or some other type of information. +pub const IMAGE_SCN_LNK_INFO: u32 = 0x0000_0200; +// IMAGE_SCN_TYPE_OVER 0x00000400 // Reserved. +/// Section contents will not become part of image. +pub const IMAGE_SCN_LNK_REMOVE: u32 = 0x0000_0800; +/// Section contents comdat. +pub const IMAGE_SCN_LNK_COMDAT: u32 = 0x0000_1000; +// 0x00002000 // Reserved. +// IMAGE_SCN_MEM_PROTECTED - Obsolete 0x00004000 +/// Reset speculative exceptions handling bits in the TLB entries for this section. +pub const IMAGE_SCN_NO_DEFER_SPEC_EXC: u32 = 0x0000_4000; +/// Section content can be accessed relative to GP +pub const IMAGE_SCN_GPREL: u32 = 0x0000_8000; +pub const IMAGE_SCN_MEM_FARDATA: u32 = 0x0000_8000; +// IMAGE_SCN_MEM_SYSHEAP - Obsolete 0x00010000 +pub const IMAGE_SCN_MEM_PURGEABLE: u32 = 0x0002_0000; +pub const IMAGE_SCN_MEM_16BIT: u32 = 0x0002_0000; +pub const IMAGE_SCN_MEM_LOCKED: u32 = 0x0004_0000; +pub const IMAGE_SCN_MEM_PRELOAD: u32 = 0x0008_0000; + +pub const IMAGE_SCN_ALIGN_1BYTES: u32 = 0x0010_0000; +pub const IMAGE_SCN_ALIGN_2BYTES: u32 = 0x0020_0000; +pub const IMAGE_SCN_ALIGN_4BYTES: u32 = 0x0030_0000; +pub const IMAGE_SCN_ALIGN_8BYTES: u32 = 0x0040_0000; +/// Default alignment if no others are specified. +pub const IMAGE_SCN_ALIGN_16BYTES: u32 = 0x0050_0000; +pub const IMAGE_SCN_ALIGN_32BYTES: u32 = 0x0060_0000; +pub const IMAGE_SCN_ALIGN_64BYTES: u32 = 0x0070_0000; +pub const IMAGE_SCN_ALIGN_128BYTES: u32 = 0x0080_0000; +pub const IMAGE_SCN_ALIGN_256BYTES: u32 = 0x0090_0000; +pub const IMAGE_SCN_ALIGN_512BYTES: u32 = 0x00A0_0000; +pub const IMAGE_SCN_ALIGN_1024BYTES: u32 = 0x00B0_0000; +pub const IMAGE_SCN_ALIGN_2048BYTES: u32 = 0x00C0_0000; +pub const IMAGE_SCN_ALIGN_4096BYTES: u32 = 0x00D0_0000; +pub const IMAGE_SCN_ALIGN_8192BYTES: u32 = 0x00E0_0000; +// Unused 0x00F0_0000 +pub const IMAGE_SCN_ALIGN_MASK: u32 = 0x00F0_0000; + +/// Section contains extended relocations. +pub const IMAGE_SCN_LNK_NRELOC_OVFL: u32 = 0x0100_0000; +/// Section can be discarded. +pub const IMAGE_SCN_MEM_DISCARDABLE: u32 = 0x0200_0000; +/// Section is not cachable. +pub const IMAGE_SCN_MEM_NOT_CACHED: u32 = 0x0400_0000; +/// Section is not pageable. +pub const IMAGE_SCN_MEM_NOT_PAGED: u32 = 0x0800_0000; +/// Section is shareable. +pub const IMAGE_SCN_MEM_SHARED: u32 = 0x1000_0000; +/// Section is executable. +pub const IMAGE_SCN_MEM_EXECUTE: u32 = 0x2000_0000; +/// Section is readable. +pub const IMAGE_SCN_MEM_READ: u32 = 0x4000_0000; +/// Section is writeable. +pub const IMAGE_SCN_MEM_WRITE: u32 = 0x8000_0000; + +// +// TLS Characteristic Flags +// +/// Tls index is scaled +pub const IMAGE_SCN_SCALE_INDEX: u32 = 0x0000_0001; + +// +// Symbol format. +// + +// This struct has alignment 1. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageSymbol { + /// If first 4 bytes are 0, then second 4 bytes are offset into string table. + pub name: [u8; 8], + pub value: U32Bytes, + pub section_number: U16Bytes, + pub typ: U16Bytes, + pub storage_class: u8, + pub number_of_aux_symbols: u8, +} + +pub const IMAGE_SIZEOF_SYMBOL: usize = 18; + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageSymbolBytes(pub [u8; IMAGE_SIZEOF_SYMBOL]); + +// This struct has alignment 1. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageSymbolEx { + /// If first 4 bytes are 0, then second 4 bytes are offset into string table. + pub name: [u8; 8], + pub value: U32Bytes, + pub section_number: U32Bytes, + pub typ: U16Bytes, + pub storage_class: u8, + pub number_of_aux_symbols: u8, +} + +pub const IMAGE_SIZEOF_SYMBOL_EX: usize = 20; + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageSymbolExBytes(pub [u8; IMAGE_SIZEOF_SYMBOL_EX]); + +// Values for `ImageSymbol::section_number`. +// +// Symbols have a section number of the section in which they are +// defined. Otherwise, section numbers have the following meanings: + +/// Symbol is undefined or is common. +pub const IMAGE_SYM_UNDEFINED: u16 = 0; +/// Symbol is an absolute value. +pub const IMAGE_SYM_ABSOLUTE: u16 = 0xffff; +/// Symbol is a special debug item. +pub const IMAGE_SYM_DEBUG: u16 = 0xfffe; +/// Values 0xFF00-0xFFFF are special +pub const IMAGE_SYM_SECTION_MAX: u16 = 0xFEFF; +pub const IMAGE_SYM_SECTION_MAX_EX: u32 = 0x7fff_ffff; + +// Values for `ImageSymbol::typ` (basic component). + +/// no type. +pub const IMAGE_SYM_TYPE_NULL: u16 = 0x0000; +pub const IMAGE_SYM_TYPE_VOID: u16 = 0x0001; +/// type character. +pub const IMAGE_SYM_TYPE_CHAR: u16 = 0x0002; +/// type short integer. +pub const IMAGE_SYM_TYPE_SHORT: u16 = 0x0003; +pub const IMAGE_SYM_TYPE_INT: u16 = 0x0004; +pub const IMAGE_SYM_TYPE_LONG: u16 = 0x0005; +pub const IMAGE_SYM_TYPE_FLOAT: u16 = 0x0006; +pub const IMAGE_SYM_TYPE_DOUBLE: u16 = 0x0007; +pub const IMAGE_SYM_TYPE_STRUCT: u16 = 0x0008; +pub const IMAGE_SYM_TYPE_UNION: u16 = 0x0009; +/// enumeration. +pub const IMAGE_SYM_TYPE_ENUM: u16 = 0x000A; +/// member of enumeration. +pub const IMAGE_SYM_TYPE_MOE: u16 = 0x000B; +pub const IMAGE_SYM_TYPE_BYTE: u16 = 0x000C; +pub const IMAGE_SYM_TYPE_WORD: u16 = 0x000D; +pub const IMAGE_SYM_TYPE_UINT: u16 = 0x000E; +pub const IMAGE_SYM_TYPE_DWORD: u16 = 0x000F; +pub const IMAGE_SYM_TYPE_PCODE: u16 = 0x8000; + +// Values for `ImageSymbol::typ` (derived component). + +/// no derived type. +pub const IMAGE_SYM_DTYPE_NULL: u16 = 0; +/// pointer. +pub const IMAGE_SYM_DTYPE_POINTER: u16 = 1; +/// function. +pub const IMAGE_SYM_DTYPE_FUNCTION: u16 = 2; +/// array. +pub const IMAGE_SYM_DTYPE_ARRAY: u16 = 3; + +// Values for `ImageSymbol::storage_class`. +pub const IMAGE_SYM_CLASS_END_OF_FUNCTION: u8 = 0xff; +pub const IMAGE_SYM_CLASS_NULL: u8 = 0x00; +pub const IMAGE_SYM_CLASS_AUTOMATIC: u8 = 0x01; +pub const IMAGE_SYM_CLASS_EXTERNAL: u8 = 0x02; +pub const IMAGE_SYM_CLASS_STATIC: u8 = 0x03; +pub const IMAGE_SYM_CLASS_REGISTER: u8 = 0x04; +pub const IMAGE_SYM_CLASS_EXTERNAL_DEF: u8 = 0x05; +pub const IMAGE_SYM_CLASS_LABEL: u8 = 0x06; +pub const IMAGE_SYM_CLASS_UNDEFINED_LABEL: u8 = 0x07; +pub const IMAGE_SYM_CLASS_MEMBER_OF_STRUCT: u8 = 0x08; +pub const IMAGE_SYM_CLASS_ARGUMENT: u8 = 0x09; +pub const IMAGE_SYM_CLASS_STRUCT_TAG: u8 = 0x0A; +pub const IMAGE_SYM_CLASS_MEMBER_OF_UNION: u8 = 0x0B; +pub const IMAGE_SYM_CLASS_UNION_TAG: u8 = 0x0C; +pub const IMAGE_SYM_CLASS_TYPE_DEFINITION: u8 = 0x0D; +pub const IMAGE_SYM_CLASS_UNDEFINED_STATIC: u8 = 0x0E; +pub const IMAGE_SYM_CLASS_ENUM_TAG: u8 = 0x0F; +pub const IMAGE_SYM_CLASS_MEMBER_OF_ENUM: u8 = 0x10; +pub const IMAGE_SYM_CLASS_REGISTER_PARAM: u8 = 0x11; +pub const IMAGE_SYM_CLASS_BIT_FIELD: u8 = 0x12; + +pub const IMAGE_SYM_CLASS_FAR_EXTERNAL: u8 = 0x44; + +pub const IMAGE_SYM_CLASS_BLOCK: u8 = 0x64; +pub const IMAGE_SYM_CLASS_FUNCTION: u8 = 0x65; +pub const IMAGE_SYM_CLASS_END_OF_STRUCT: u8 = 0x66; +pub const IMAGE_SYM_CLASS_FILE: u8 = 0x67; +// new +pub const IMAGE_SYM_CLASS_SECTION: u8 = 0x68; +pub const IMAGE_SYM_CLASS_WEAK_EXTERNAL: u8 = 0x69; + +pub const IMAGE_SYM_CLASS_CLR_TOKEN: u8 = 0x6B; + +// type packing constants + +pub const N_BTMASK: u16 = 0x000F; +pub const N_TMASK: u16 = 0x0030; +pub const N_TMASK1: u16 = 0x00C0; +pub const N_TMASK2: u16 = 0x00F0; +pub const N_BTSHFT: usize = 4; +pub const N_TSHIFT: usize = 2; + +pub const IMAGE_SYM_DTYPE_SHIFT: usize = N_BTSHFT; + +impl ImageSymbol { + #[inline] + pub fn base_type(&self) -> u16 { + self.typ.get(LE) & N_BTMASK + } + + #[inline] + pub fn derived_type(&self) -> u16 { + (self.typ.get(LE) & N_TMASK) >> N_BTSHFT + } +} + +impl ImageSymbolEx { + #[inline] + pub fn base_type(&self) -> u16 { + self.typ.get(LE) & N_BTMASK + } + + #[inline] + pub fn derived_type(&self) -> u16 { + (self.typ.get(LE) & N_TMASK) >> N_BTSHFT + } +} + +// +// Auxiliary entry format. +// + +// Used for both ImageSymbol and ImageSymbolEx (with padding). +// This struct has alignment 1. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageAuxSymbolTokenDef { + /// IMAGE_AUX_SYMBOL_TYPE + pub aux_type: u8, + /// Must be 0 + pub reserved1: u8, + pub symbol_table_index: U32Bytes, + /// Must be 0 + pub reserved2: [u8; 12], +} + +pub const IMAGE_AUX_SYMBOL_TYPE_TOKEN_DEF: u16 = 1; + +/// Auxiliary symbol format 1: function definitions. +// This struct has alignment 1. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageAuxSymbolFunction { + pub tag_index: U32Bytes, + pub total_size: U32Bytes, + pub pointer_to_linenumber: U32Bytes, + pub pointer_to_next_function: U32Bytes, + pub unused: [u8; 2], +} + +/// Auxiliary symbol format 2: .bf and .ef symbols. +// This struct has alignment 1. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageAuxSymbolFunctionBeginEnd { + pub unused1: [u8; 4], + /// declaration line number + pub linenumber: U16Bytes, + pub unused2: [u8; 6], + pub pointer_to_next_function: U32Bytes, + pub unused3: [u8; 2], +} + +/// Auxiliary symbol format 3: weak externals. +/// +/// Used for both `ImageSymbol` and `ImageSymbolEx` (both with padding). +// This struct has alignment 1. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageAuxSymbolWeak { + /// the weak extern default symbol index + pub weak_default_sym_index: U32Bytes, + pub weak_search_type: U32Bytes, +} + +/// Auxiliary symbol format 5: sections. +/// +/// Used for both `ImageSymbol` and `ImageSymbolEx` (with padding). +// This struct has alignment 1. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageAuxSymbolSection { + /// section length + pub length: U32Bytes, + /// number of relocation entries + pub number_of_relocations: U16Bytes, + /// number of line numbers + pub number_of_linenumbers: U16Bytes, + /// checksum for communal + pub check_sum: U32Bytes, + /// section number to associate with + pub number: U16Bytes, + /// communal selection type + pub selection: u8, + pub reserved: u8, + /// high bits of the section number + pub high_number: U16Bytes, +} + +// Used for both ImageSymbol and ImageSymbolEx (both with padding). +// This struct has alignment 1. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageAuxSymbolCrc { + pub crc: U32Bytes, +} + +// +// Communal selection types. +// + +pub const IMAGE_COMDAT_SELECT_NODUPLICATES: u8 = 1; +pub const IMAGE_COMDAT_SELECT_ANY: u8 = 2; +pub const IMAGE_COMDAT_SELECT_SAME_SIZE: u8 = 3; +pub const IMAGE_COMDAT_SELECT_EXACT_MATCH: u8 = 4; +pub const IMAGE_COMDAT_SELECT_ASSOCIATIVE: u8 = 5; +pub const IMAGE_COMDAT_SELECT_LARGEST: u8 = 6; +pub const IMAGE_COMDAT_SELECT_NEWEST: u8 = 7; + +pub const IMAGE_WEAK_EXTERN_SEARCH_NOLIBRARY: u16 = 1; +pub const IMAGE_WEAK_EXTERN_SEARCH_LIBRARY: u16 = 2; +pub const IMAGE_WEAK_EXTERN_SEARCH_ALIAS: u16 = 3; +pub const IMAGE_WEAK_EXTERN_ANTI_DEPENDENCY: u16 = 4; + +// +// Relocation format. +// + +// This struct has alignment 1. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageRelocation { + /// Also `RelocCount` when IMAGE_SCN_LNK_NRELOC_OVFL is set + pub virtual_address: U32Bytes, + pub symbol_table_index: U32Bytes, + pub typ: U16Bytes, +} + +// +// I386 relocation types. +// +/// Reference is absolute, no relocation is necessary +pub const IMAGE_REL_I386_ABSOLUTE: u16 = 0x0000; +/// Direct 16-bit reference to the symbols virtual address +pub const IMAGE_REL_I386_DIR16: u16 = 0x0001; +/// PC-relative 16-bit reference to the symbols virtual address +pub const IMAGE_REL_I386_REL16: u16 = 0x0002; +/// Direct 32-bit reference to the symbols virtual address +pub const IMAGE_REL_I386_DIR32: u16 = 0x0006; +/// Direct 32-bit reference to the symbols virtual address, base not included +pub const IMAGE_REL_I386_DIR32NB: u16 = 0x0007; +/// Direct 16-bit reference to the segment-selector bits of a 32-bit virtual address +pub const IMAGE_REL_I386_SEG12: u16 = 0x0009; +pub const IMAGE_REL_I386_SECTION: u16 = 0x000A; +pub const IMAGE_REL_I386_SECREL: u16 = 0x000B; +/// clr token +pub const IMAGE_REL_I386_TOKEN: u16 = 0x000C; +/// 7 bit offset from base of section containing target +pub const IMAGE_REL_I386_SECREL7: u16 = 0x000D; +/// PC-relative 32-bit reference to the symbols virtual address +pub const IMAGE_REL_I386_REL32: u16 = 0x0014; + +// +// MIPS relocation types. +// +/// Reference is absolute, no relocation is necessary +pub const IMAGE_REL_MIPS_ABSOLUTE: u16 = 0x0000; +pub const IMAGE_REL_MIPS_REFHALF: u16 = 0x0001; +pub const IMAGE_REL_MIPS_REFWORD: u16 = 0x0002; +pub const IMAGE_REL_MIPS_JMPADDR: u16 = 0x0003; +pub const IMAGE_REL_MIPS_REFHI: u16 = 0x0004; +pub const IMAGE_REL_MIPS_REFLO: u16 = 0x0005; +pub const IMAGE_REL_MIPS_GPREL: u16 = 0x0006; +pub const IMAGE_REL_MIPS_LITERAL: u16 = 0x0007; +pub const IMAGE_REL_MIPS_SECTION: u16 = 0x000A; +pub const IMAGE_REL_MIPS_SECREL: u16 = 0x000B; +/// Low 16-bit section relative referemce (used for >32k TLS) +pub const IMAGE_REL_MIPS_SECRELLO: u16 = 0x000C; +/// High 16-bit section relative reference (used for >32k TLS) +pub const IMAGE_REL_MIPS_SECRELHI: u16 = 0x000D; +/// clr token +pub const IMAGE_REL_MIPS_TOKEN: u16 = 0x000E; +pub const IMAGE_REL_MIPS_JMPADDR16: u16 = 0x0010; +pub const IMAGE_REL_MIPS_REFWORDNB: u16 = 0x0022; +pub const IMAGE_REL_MIPS_PAIR: u16 = 0x0025; + +// +// Alpha Relocation types. +// +pub const IMAGE_REL_ALPHA_ABSOLUTE: u16 = 0x0000; +pub const IMAGE_REL_ALPHA_REFLONG: u16 = 0x0001; +pub const IMAGE_REL_ALPHA_REFQUAD: u16 = 0x0002; +pub const IMAGE_REL_ALPHA_GPREL32: u16 = 0x0003; +pub const IMAGE_REL_ALPHA_LITERAL: u16 = 0x0004; +pub const IMAGE_REL_ALPHA_LITUSE: u16 = 0x0005; +pub const IMAGE_REL_ALPHA_GPDISP: u16 = 0x0006; +pub const IMAGE_REL_ALPHA_BRADDR: u16 = 0x0007; +pub const IMAGE_REL_ALPHA_HINT: u16 = 0x0008; +pub const IMAGE_REL_ALPHA_INLINE_REFLONG: u16 = 0x0009; +pub const IMAGE_REL_ALPHA_REFHI: u16 = 0x000A; +pub const IMAGE_REL_ALPHA_REFLO: u16 = 0x000B; +pub const IMAGE_REL_ALPHA_PAIR: u16 = 0x000C; +pub const IMAGE_REL_ALPHA_MATCH: u16 = 0x000D; +pub const IMAGE_REL_ALPHA_SECTION: u16 = 0x000E; +pub const IMAGE_REL_ALPHA_SECREL: u16 = 0x000F; +pub const IMAGE_REL_ALPHA_REFLONGNB: u16 = 0x0010; +/// Low 16-bit section relative reference +pub const IMAGE_REL_ALPHA_SECRELLO: u16 = 0x0011; +/// High 16-bit section relative reference +pub const IMAGE_REL_ALPHA_SECRELHI: u16 = 0x0012; +/// High 16 bits of 48 bit reference +pub const IMAGE_REL_ALPHA_REFQ3: u16 = 0x0013; +/// Middle 16 bits of 48 bit reference +pub const IMAGE_REL_ALPHA_REFQ2: u16 = 0x0014; +/// Low 16 bits of 48 bit reference +pub const IMAGE_REL_ALPHA_REFQ1: u16 = 0x0015; +/// Low 16-bit GP relative reference +pub const IMAGE_REL_ALPHA_GPRELLO: u16 = 0x0016; +/// High 16-bit GP relative reference +pub const IMAGE_REL_ALPHA_GPRELHI: u16 = 0x0017; + +// +// IBM PowerPC relocation types. +// +/// NOP +pub const IMAGE_REL_PPC_ABSOLUTE: u16 = 0x0000; +/// 64-bit address +pub const IMAGE_REL_PPC_ADDR64: u16 = 0x0001; +/// 32-bit address +pub const IMAGE_REL_PPC_ADDR32: u16 = 0x0002; +/// 26-bit address, shifted left 2 (branch absolute) +pub const IMAGE_REL_PPC_ADDR24: u16 = 0x0003; +/// 16-bit address +pub const IMAGE_REL_PPC_ADDR16: u16 = 0x0004; +/// 16-bit address, shifted left 2 (load doubleword) +pub const IMAGE_REL_PPC_ADDR14: u16 = 0x0005; +/// 26-bit PC-relative offset, shifted left 2 (branch relative) +pub const IMAGE_REL_PPC_REL24: u16 = 0x0006; +/// 16-bit PC-relative offset, shifted left 2 (br cond relative) +pub const IMAGE_REL_PPC_REL14: u16 = 0x0007; +/// 16-bit offset from TOC base +pub const IMAGE_REL_PPC_TOCREL16: u16 = 0x0008; +/// 16-bit offset from TOC base, shifted left 2 (load doubleword) +pub const IMAGE_REL_PPC_TOCREL14: u16 = 0x0009; + +/// 32-bit addr w/o image base +pub const IMAGE_REL_PPC_ADDR32NB: u16 = 0x000A; +/// va of containing section (as in an image sectionhdr) +pub const IMAGE_REL_PPC_SECREL: u16 = 0x000B; +/// sectionheader number +pub const IMAGE_REL_PPC_SECTION: u16 = 0x000C; +/// substitute TOC restore instruction iff symbol is glue code +pub const IMAGE_REL_PPC_IFGLUE: u16 = 0x000D; +/// symbol is glue code; virtual address is TOC restore instruction +pub const IMAGE_REL_PPC_IMGLUE: u16 = 0x000E; +/// va of containing section (limited to 16 bits) +pub const IMAGE_REL_PPC_SECREL16: u16 = 0x000F; +pub const IMAGE_REL_PPC_REFHI: u16 = 0x0010; +pub const IMAGE_REL_PPC_REFLO: u16 = 0x0011; +pub const IMAGE_REL_PPC_PAIR: u16 = 0x0012; +/// Low 16-bit section relative reference (used for >32k TLS) +pub const IMAGE_REL_PPC_SECRELLO: u16 = 0x0013; +/// High 16-bit section relative reference (used for >32k TLS) +pub const IMAGE_REL_PPC_SECRELHI: u16 = 0x0014; +pub const IMAGE_REL_PPC_GPREL: u16 = 0x0015; +/// clr token +pub const IMAGE_REL_PPC_TOKEN: u16 = 0x0016; + +/// mask to isolate above values in IMAGE_RELOCATION.Type +pub const IMAGE_REL_PPC_TYPEMASK: u16 = 0x00FF; + +// Flag bits in `ImageRelocation::typ`. + +/// subtract reloc value rather than adding it +pub const IMAGE_REL_PPC_NEG: u16 = 0x0100; +/// fix branch prediction bit to predict branch taken +pub const IMAGE_REL_PPC_BRTAKEN: u16 = 0x0200; +/// fix branch prediction bit to predict branch not taken +pub const IMAGE_REL_PPC_BRNTAKEN: u16 = 0x0400; +/// toc slot defined in file (or, data in toc) +pub const IMAGE_REL_PPC_TOCDEFN: u16 = 0x0800; + +// +// Hitachi SH3 relocation types. +// +/// No relocation +pub const IMAGE_REL_SH3_ABSOLUTE: u16 = 0x0000; +/// 16 bit direct +pub const IMAGE_REL_SH3_DIRECT16: u16 = 0x0001; +/// 32 bit direct +pub const IMAGE_REL_SH3_DIRECT32: u16 = 0x0002; +/// 8 bit direct, -128..255 +pub const IMAGE_REL_SH3_DIRECT8: u16 = 0x0003; +/// 8 bit direct .W (0 ext.) +pub const IMAGE_REL_SH3_DIRECT8_WORD: u16 = 0x0004; +/// 8 bit direct .L (0 ext.) +pub const IMAGE_REL_SH3_DIRECT8_LONG: u16 = 0x0005; +/// 4 bit direct (0 ext.) +pub const IMAGE_REL_SH3_DIRECT4: u16 = 0x0006; +/// 4 bit direct .W (0 ext.) +pub const IMAGE_REL_SH3_DIRECT4_WORD: u16 = 0x0007; +/// 4 bit direct .L (0 ext.) +pub const IMAGE_REL_SH3_DIRECT4_LONG: u16 = 0x0008; +/// 8 bit PC relative .W +pub const IMAGE_REL_SH3_PCREL8_WORD: u16 = 0x0009; +/// 8 bit PC relative .L +pub const IMAGE_REL_SH3_PCREL8_LONG: u16 = 0x000A; +/// 12 LSB PC relative .W +pub const IMAGE_REL_SH3_PCREL12_WORD: u16 = 0x000B; +/// Start of EXE section +pub const IMAGE_REL_SH3_STARTOF_SECTION: u16 = 0x000C; +/// Size of EXE section +pub const IMAGE_REL_SH3_SIZEOF_SECTION: u16 = 0x000D; +/// Section table index +pub const IMAGE_REL_SH3_SECTION: u16 = 0x000E; +/// Offset within section +pub const IMAGE_REL_SH3_SECREL: u16 = 0x000F; +/// 32 bit direct not based +pub const IMAGE_REL_SH3_DIRECT32_NB: u16 = 0x0010; +/// GP-relative addressing +pub const IMAGE_REL_SH3_GPREL4_LONG: u16 = 0x0011; +/// clr token +pub const IMAGE_REL_SH3_TOKEN: u16 = 0x0012; +/// Offset from current instruction in longwords +/// if not NOMODE, insert the inverse of the low bit at bit 32 to select PTA/PTB +pub const IMAGE_REL_SHM_PCRELPT: u16 = 0x0013; +/// Low bits of 32-bit address +pub const IMAGE_REL_SHM_REFLO: u16 = 0x0014; +/// High bits of 32-bit address +pub const IMAGE_REL_SHM_REFHALF: u16 = 0x0015; +/// Low bits of relative reference +pub const IMAGE_REL_SHM_RELLO: u16 = 0x0016; +/// High bits of relative reference +pub const IMAGE_REL_SHM_RELHALF: u16 = 0x0017; +/// offset operand for relocation +pub const IMAGE_REL_SHM_PAIR: u16 = 0x0018; + +/// relocation ignores section mode +pub const IMAGE_REL_SH_NOMODE: u16 = 0x8000; + +/// No relocation required +pub const IMAGE_REL_ARM_ABSOLUTE: u16 = 0x0000; +/// 32 bit address +pub const IMAGE_REL_ARM_ADDR32: u16 = 0x0001; +/// 32 bit address w/o image base +pub const IMAGE_REL_ARM_ADDR32NB: u16 = 0x0002; +/// 24 bit offset << 2 & sign ext. +pub const IMAGE_REL_ARM_BRANCH24: u16 = 0x0003; +/// Thumb: 2 11 bit offsets +pub const IMAGE_REL_ARM_BRANCH11: u16 = 0x0004; +/// clr token +pub const IMAGE_REL_ARM_TOKEN: u16 = 0x0005; +/// GP-relative addressing (ARM) +pub const IMAGE_REL_ARM_GPREL12: u16 = 0x0006; +/// GP-relative addressing (Thumb) +pub const IMAGE_REL_ARM_GPREL7: u16 = 0x0007; +pub const IMAGE_REL_ARM_BLX24: u16 = 0x0008; +pub const IMAGE_REL_ARM_BLX11: u16 = 0x0009; +/// 32-bit relative address from byte following reloc +pub const IMAGE_REL_ARM_REL32: u16 = 0x000A; +/// Section table index +pub const IMAGE_REL_ARM_SECTION: u16 = 0x000E; +/// Offset within section +pub const IMAGE_REL_ARM_SECREL: u16 = 0x000F; +/// ARM: MOVW/MOVT +pub const IMAGE_REL_ARM_MOV32A: u16 = 0x0010; +/// ARM: MOVW/MOVT (deprecated) +pub const IMAGE_REL_ARM_MOV32: u16 = 0x0010; +/// Thumb: MOVW/MOVT +pub const IMAGE_REL_ARM_MOV32T: u16 = 0x0011; +/// Thumb: MOVW/MOVT (deprecated) +pub const IMAGE_REL_THUMB_MOV32: u16 = 0x0011; +/// Thumb: 32-bit conditional B +pub const IMAGE_REL_ARM_BRANCH20T: u16 = 0x0012; +/// Thumb: 32-bit conditional B (deprecated) +pub const IMAGE_REL_THUMB_BRANCH20: u16 = 0x0012; +/// Thumb: 32-bit B or BL +pub const IMAGE_REL_ARM_BRANCH24T: u16 = 0x0014; +/// Thumb: 32-bit B or BL (deprecated) +pub const IMAGE_REL_THUMB_BRANCH24: u16 = 0x0014; +/// Thumb: BLX immediate +pub const IMAGE_REL_ARM_BLX23T: u16 = 0x0015; +/// Thumb: BLX immediate (deprecated) +pub const IMAGE_REL_THUMB_BLX23: u16 = 0x0015; + +pub const IMAGE_REL_AM_ABSOLUTE: u16 = 0x0000; +pub const IMAGE_REL_AM_ADDR32: u16 = 0x0001; +pub const IMAGE_REL_AM_ADDR32NB: u16 = 0x0002; +pub const IMAGE_REL_AM_CALL32: u16 = 0x0003; +pub const IMAGE_REL_AM_FUNCINFO: u16 = 0x0004; +pub const IMAGE_REL_AM_REL32_1: u16 = 0x0005; +pub const IMAGE_REL_AM_REL32_2: u16 = 0x0006; +pub const IMAGE_REL_AM_SECREL: u16 = 0x0007; +pub const IMAGE_REL_AM_SECTION: u16 = 0x0008; +pub const IMAGE_REL_AM_TOKEN: u16 = 0x0009; + +// +// ARM64 relocations types. +// + +/// No relocation required +pub const IMAGE_REL_ARM64_ABSOLUTE: u16 = 0x0000; +/// 32 bit address. Review! do we need it? +pub const IMAGE_REL_ARM64_ADDR32: u16 = 0x0001; +/// 32 bit address w/o image base (RVA: for Data/PData/XData) +pub const IMAGE_REL_ARM64_ADDR32NB: u16 = 0x0002; +/// 26 bit offset << 2 & sign ext. for B & BL +pub const IMAGE_REL_ARM64_BRANCH26: u16 = 0x0003; +/// ADRP +pub const IMAGE_REL_ARM64_PAGEBASE_REL21: u16 = 0x0004; +/// ADR +pub const IMAGE_REL_ARM64_REL21: u16 = 0x0005; +/// ADD/ADDS (immediate) with zero shift, for page offset +pub const IMAGE_REL_ARM64_PAGEOFFSET_12A: u16 = 0x0006; +/// LDR (indexed, unsigned immediate), for page offset +pub const IMAGE_REL_ARM64_PAGEOFFSET_12L: u16 = 0x0007; +/// Offset within section +pub const IMAGE_REL_ARM64_SECREL: u16 = 0x0008; +/// ADD/ADDS (immediate) with zero shift, for bit 0:11 of section offset +pub const IMAGE_REL_ARM64_SECREL_LOW12A: u16 = 0x0009; +/// ADD/ADDS (immediate) with zero shift, for bit 12:23 of section offset +pub const IMAGE_REL_ARM64_SECREL_HIGH12A: u16 = 0x000A; +/// LDR (indexed, unsigned immediate), for bit 0:11 of section offset +pub const IMAGE_REL_ARM64_SECREL_LOW12L: u16 = 0x000B; +pub const IMAGE_REL_ARM64_TOKEN: u16 = 0x000C; +/// Section table index +pub const IMAGE_REL_ARM64_SECTION: u16 = 0x000D; +/// 64 bit address +pub const IMAGE_REL_ARM64_ADDR64: u16 = 0x000E; +/// 19 bit offset << 2 & sign ext. for conditional B +pub const IMAGE_REL_ARM64_BRANCH19: u16 = 0x000F; +/// TBZ/TBNZ +pub const IMAGE_REL_ARM64_BRANCH14: u16 = 0x0010; +/// 32-bit relative address from byte following reloc +pub const IMAGE_REL_ARM64_REL32: u16 = 0x0011; + +// +// x64 relocations +// +/// Reference is absolute, no relocation is necessary +pub const IMAGE_REL_AMD64_ABSOLUTE: u16 = 0x0000; +/// 64-bit address (VA). +pub const IMAGE_REL_AMD64_ADDR64: u16 = 0x0001; +/// 32-bit address (VA). +pub const IMAGE_REL_AMD64_ADDR32: u16 = 0x0002; +/// 32-bit address w/o image base (RVA). +pub const IMAGE_REL_AMD64_ADDR32NB: u16 = 0x0003; +/// 32-bit relative address from byte following reloc +pub const IMAGE_REL_AMD64_REL32: u16 = 0x0004; +/// 32-bit relative address from byte distance 1 from reloc +pub const IMAGE_REL_AMD64_REL32_1: u16 = 0x0005; +/// 32-bit relative address from byte distance 2 from reloc +pub const IMAGE_REL_AMD64_REL32_2: u16 = 0x0006; +/// 32-bit relative address from byte distance 3 from reloc +pub const IMAGE_REL_AMD64_REL32_3: u16 = 0x0007; +/// 32-bit relative address from byte distance 4 from reloc +pub const IMAGE_REL_AMD64_REL32_4: u16 = 0x0008; +/// 32-bit relative address from byte distance 5 from reloc +pub const IMAGE_REL_AMD64_REL32_5: u16 = 0x0009; +/// Section index +pub const IMAGE_REL_AMD64_SECTION: u16 = 0x000A; +/// 32 bit offset from base of section containing target +pub const IMAGE_REL_AMD64_SECREL: u16 = 0x000B; +/// 7 bit unsigned offset from base of section containing target +pub const IMAGE_REL_AMD64_SECREL7: u16 = 0x000C; +/// 32 bit metadata token +pub const IMAGE_REL_AMD64_TOKEN: u16 = 0x000D; +/// 32 bit signed span-dependent value emitted into object +pub const IMAGE_REL_AMD64_SREL32: u16 = 0x000E; +pub const IMAGE_REL_AMD64_PAIR: u16 = 0x000F; +/// 32 bit signed span-dependent value applied at link time +pub const IMAGE_REL_AMD64_SSPAN32: u16 = 0x0010; +pub const IMAGE_REL_AMD64_EHANDLER: u16 = 0x0011; +/// Indirect branch to an import +pub const IMAGE_REL_AMD64_IMPORT_BR: u16 = 0x0012; +/// Indirect call to an import +pub const IMAGE_REL_AMD64_IMPORT_CALL: u16 = 0x0013; +/// Indirect branch to a CFG check +pub const IMAGE_REL_AMD64_CFG_BR: u16 = 0x0014; +/// Indirect branch to a CFG check, with REX.W prefix +pub const IMAGE_REL_AMD64_CFG_BR_REX: u16 = 0x0015; +/// Indirect call to a CFG check +pub const IMAGE_REL_AMD64_CFG_CALL: u16 = 0x0016; +/// Indirect branch to a target in RAX (no CFG) +pub const IMAGE_REL_AMD64_INDIR_BR: u16 = 0x0017; +/// Indirect branch to a target in RAX, with REX.W prefix (no CFG) +pub const IMAGE_REL_AMD64_INDIR_BR_REX: u16 = 0x0018; +/// Indirect call to a target in RAX (no CFG) +pub const IMAGE_REL_AMD64_INDIR_CALL: u16 = 0x0019; +/// Indirect branch for a switch table using Reg 0 (RAX) +pub const IMAGE_REL_AMD64_INDIR_BR_SWITCHTABLE_FIRST: u16 = 0x0020; +/// Indirect branch for a switch table using Reg 15 (R15) +pub const IMAGE_REL_AMD64_INDIR_BR_SWITCHTABLE_LAST: u16 = 0x002F; + +// +// IA64 relocation types. +// +pub const IMAGE_REL_IA64_ABSOLUTE: u16 = 0x0000; +pub const IMAGE_REL_IA64_IMM14: u16 = 0x0001; +pub const IMAGE_REL_IA64_IMM22: u16 = 0x0002; +pub const IMAGE_REL_IA64_IMM64: u16 = 0x0003; +pub const IMAGE_REL_IA64_DIR32: u16 = 0x0004; +pub const IMAGE_REL_IA64_DIR64: u16 = 0x0005; +pub const IMAGE_REL_IA64_PCREL21B: u16 = 0x0006; +pub const IMAGE_REL_IA64_PCREL21M: u16 = 0x0007; +pub const IMAGE_REL_IA64_PCREL21F: u16 = 0x0008; +pub const IMAGE_REL_IA64_GPREL22: u16 = 0x0009; +pub const IMAGE_REL_IA64_LTOFF22: u16 = 0x000A; +pub const IMAGE_REL_IA64_SECTION: u16 = 0x000B; +pub const IMAGE_REL_IA64_SECREL22: u16 = 0x000C; +pub const IMAGE_REL_IA64_SECREL64I: u16 = 0x000D; +pub const IMAGE_REL_IA64_SECREL32: u16 = 0x000E; +// +pub const IMAGE_REL_IA64_DIR32NB: u16 = 0x0010; +pub const IMAGE_REL_IA64_SREL14: u16 = 0x0011; +pub const IMAGE_REL_IA64_SREL22: u16 = 0x0012; +pub const IMAGE_REL_IA64_SREL32: u16 = 0x0013; +pub const IMAGE_REL_IA64_UREL32: u16 = 0x0014; +/// This is always a BRL and never converted +pub const IMAGE_REL_IA64_PCREL60X: u16 = 0x0015; +/// If possible, convert to MBB bundle with NOP.B in slot 1 +pub const IMAGE_REL_IA64_PCREL60B: u16 = 0x0016; +/// If possible, convert to MFB bundle with NOP.F in slot 1 +pub const IMAGE_REL_IA64_PCREL60F: u16 = 0x0017; +/// If possible, convert to MIB bundle with NOP.I in slot 1 +pub const IMAGE_REL_IA64_PCREL60I: u16 = 0x0018; +/// If possible, convert to MMB bundle with NOP.M in slot 1 +pub const IMAGE_REL_IA64_PCREL60M: u16 = 0x0019; +pub const IMAGE_REL_IA64_IMMGPREL64: u16 = 0x001A; +/// clr token +pub const IMAGE_REL_IA64_TOKEN: u16 = 0x001B; +pub const IMAGE_REL_IA64_GPREL32: u16 = 0x001C; +pub const IMAGE_REL_IA64_ADDEND: u16 = 0x001F; + +// +// CEF relocation types. +// +/// Reference is absolute, no relocation is necessary +pub const IMAGE_REL_CEF_ABSOLUTE: u16 = 0x0000; +/// 32-bit address (VA). +pub const IMAGE_REL_CEF_ADDR32: u16 = 0x0001; +/// 64-bit address (VA). +pub const IMAGE_REL_CEF_ADDR64: u16 = 0x0002; +/// 32-bit address w/o image base (RVA). +pub const IMAGE_REL_CEF_ADDR32NB: u16 = 0x0003; +/// Section index +pub const IMAGE_REL_CEF_SECTION: u16 = 0x0004; +/// 32 bit offset from base of section containing target +pub const IMAGE_REL_CEF_SECREL: u16 = 0x0005; +/// 32 bit metadata token +pub const IMAGE_REL_CEF_TOKEN: u16 = 0x0006; + +// +// clr relocation types. +// +/// Reference is absolute, no relocation is necessary +pub const IMAGE_REL_CEE_ABSOLUTE: u16 = 0x0000; +/// 32-bit address (VA). +pub const IMAGE_REL_CEE_ADDR32: u16 = 0x0001; +/// 64-bit address (VA). +pub const IMAGE_REL_CEE_ADDR64: u16 = 0x0002; +/// 32-bit address w/o image base (RVA). +pub const IMAGE_REL_CEE_ADDR32NB: u16 = 0x0003; +/// Section index +pub const IMAGE_REL_CEE_SECTION: u16 = 0x0004; +/// 32 bit offset from base of section containing target +pub const IMAGE_REL_CEE_SECREL: u16 = 0x0005; +/// 32 bit metadata token +pub const IMAGE_REL_CEE_TOKEN: u16 = 0x0006; + +/// No relocation required +pub const IMAGE_REL_M32R_ABSOLUTE: u16 = 0x0000; +/// 32 bit address +pub const IMAGE_REL_M32R_ADDR32: u16 = 0x0001; +/// 32 bit address w/o image base +pub const IMAGE_REL_M32R_ADDR32NB: u16 = 0x0002; +/// 24 bit address +pub const IMAGE_REL_M32R_ADDR24: u16 = 0x0003; +/// GP relative addressing +pub const IMAGE_REL_M32R_GPREL16: u16 = 0x0004; +/// 24 bit offset << 2 & sign ext. +pub const IMAGE_REL_M32R_PCREL24: u16 = 0x0005; +/// 16 bit offset << 2 & sign ext. +pub const IMAGE_REL_M32R_PCREL16: u16 = 0x0006; +/// 8 bit offset << 2 & sign ext. +pub const IMAGE_REL_M32R_PCREL8: u16 = 0x0007; +/// 16 MSBs +pub const IMAGE_REL_M32R_REFHALF: u16 = 0x0008; +/// 16 MSBs; adj for LSB sign ext. +pub const IMAGE_REL_M32R_REFHI: u16 = 0x0009; +/// 16 LSBs +pub const IMAGE_REL_M32R_REFLO: u16 = 0x000A; +/// Link HI and LO +pub const IMAGE_REL_M32R_PAIR: u16 = 0x000B; +/// Section table index +pub const IMAGE_REL_M32R_SECTION: u16 = 0x000C; +/// 32 bit section relative reference +pub const IMAGE_REL_M32R_SECREL32: u16 = 0x000D; +/// clr token +pub const IMAGE_REL_M32R_TOKEN: u16 = 0x000E; + +/// No relocation required +pub const IMAGE_REL_EBC_ABSOLUTE: u16 = 0x0000; +/// 32 bit address w/o image base +pub const IMAGE_REL_EBC_ADDR32NB: u16 = 0x0001; +/// 32-bit relative address from byte following reloc +pub const IMAGE_REL_EBC_REL32: u16 = 0x0002; +/// Section table index +pub const IMAGE_REL_EBC_SECTION: u16 = 0x0003; +/// Offset within section +pub const IMAGE_REL_EBC_SECREL: u16 = 0x0004; + +/* +// TODO? +#define EXT_IMM64(Value, Address, Size, InstPos, ValPos) /* Intel-IA64-Filler */ \ + Value |= (((ULONGLONG)((*(Address) >> InstPos) & (((ULONGLONG)1 << Size) - 1))) << ValPos) // Intel-IA64-Filler + +#define INS_IMM64(Value, Address, Size, InstPos, ValPos) /* Intel-IA64-Filler */\ + *(PDWORD)Address = (*(PDWORD)Address & ~(((1 << Size) - 1) << InstPos)) | /* Intel-IA64-Filler */\ + ((DWORD)((((ULONGLONG)Value >> ValPos) & (((ULONGLONG)1 << Size) - 1))) << InstPos) // Intel-IA64-Filler +*/ + +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM7B_INST_WORD_X: u16 = 3; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM7B_SIZE_X: u16 = 7; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM7B_INST_WORD_POS_X: u16 = 4; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM7B_VAL_POS_X: u16 = 0; + +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM9D_INST_WORD_X: u16 = 3; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM9D_SIZE_X: u16 = 9; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM9D_INST_WORD_POS_X: u16 = 18; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM9D_VAL_POS_X: u16 = 7; + +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM5C_INST_WORD_X: u16 = 3; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM5C_SIZE_X: u16 = 5; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM5C_INST_WORD_POS_X: u16 = 13; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM5C_VAL_POS_X: u16 = 16; + +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IC_INST_WORD_X: u16 = 3; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IC_SIZE_X: u16 = 1; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IC_INST_WORD_POS_X: u16 = 12; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IC_VAL_POS_X: u16 = 21; + +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM41A_INST_WORD_X: u16 = 1; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM41A_SIZE_X: u16 = 10; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM41A_INST_WORD_POS_X: u16 = 14; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM41A_VAL_POS_X: u16 = 22; + +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM41B_INST_WORD_X: u16 = 1; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM41B_SIZE_X: u16 = 8; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM41B_INST_WORD_POS_X: u16 = 24; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM41B_VAL_POS_X: u16 = 32; + +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM41C_INST_WORD_X: u16 = 2; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM41C_SIZE_X: u16 = 23; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM41C_INST_WORD_POS_X: u16 = 0; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_IMM41C_VAL_POS_X: u16 = 40; + +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_SIGN_INST_WORD_X: u16 = 3; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_SIGN_SIZE_X: u16 = 1; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_SIGN_INST_WORD_POS_X: u16 = 27; +/// Intel-IA64-Filler +pub const EMARCH_ENC_I17_SIGN_VAL_POS_X: u16 = 63; + +/// Intel-IA64-Filler +pub const X3_OPCODE_INST_WORD_X: u16 = 3; +/// Intel-IA64-Filler +pub const X3_OPCODE_SIZE_X: u16 = 4; +/// Intel-IA64-Filler +pub const X3_OPCODE_INST_WORD_POS_X: u16 = 28; +/// Intel-IA64-Filler +pub const X3_OPCODE_SIGN_VAL_POS_X: u16 = 0; + +/// Intel-IA64-Filler +pub const X3_I_INST_WORD_X: u16 = 3; +/// Intel-IA64-Filler +pub const X3_I_SIZE_X: u16 = 1; +/// Intel-IA64-Filler +pub const X3_I_INST_WORD_POS_X: u16 = 27; +/// Intel-IA64-Filler +pub const X3_I_SIGN_VAL_POS_X: u16 = 59; + +/// Intel-IA64-Filler +pub const X3_D_WH_INST_WORD_X: u16 = 3; +/// Intel-IA64-Filler +pub const X3_D_WH_SIZE_X: u16 = 3; +/// Intel-IA64-Filler +pub const X3_D_WH_INST_WORD_POS_X: u16 = 24; +/// Intel-IA64-Filler +pub const X3_D_WH_SIGN_VAL_POS_X: u16 = 0; + +/// Intel-IA64-Filler +pub const X3_IMM20_INST_WORD_X: u16 = 3; +/// Intel-IA64-Filler +pub const X3_IMM20_SIZE_X: u16 = 20; +/// Intel-IA64-Filler +pub const X3_IMM20_INST_WORD_POS_X: u16 = 4; +/// Intel-IA64-Filler +pub const X3_IMM20_SIGN_VAL_POS_X: u16 = 0; + +/// Intel-IA64-Filler +pub const X3_IMM39_1_INST_WORD_X: u16 = 2; +/// Intel-IA64-Filler +pub const X3_IMM39_1_SIZE_X: u16 = 23; +/// Intel-IA64-Filler +pub const X3_IMM39_1_INST_WORD_POS_X: u16 = 0; +/// Intel-IA64-Filler +pub const X3_IMM39_1_SIGN_VAL_POS_X: u16 = 36; + +/// Intel-IA64-Filler +pub const X3_IMM39_2_INST_WORD_X: u16 = 1; +/// Intel-IA64-Filler +pub const X3_IMM39_2_SIZE_X: u16 = 16; +/// Intel-IA64-Filler +pub const X3_IMM39_2_INST_WORD_POS_X: u16 = 16; +/// Intel-IA64-Filler +pub const X3_IMM39_2_SIGN_VAL_POS_X: u16 = 20; + +/// Intel-IA64-Filler +pub const X3_P_INST_WORD_X: u16 = 3; +/// Intel-IA64-Filler +pub const X3_P_SIZE_X: u16 = 4; +/// Intel-IA64-Filler +pub const X3_P_INST_WORD_POS_X: u16 = 0; +/// Intel-IA64-Filler +pub const X3_P_SIGN_VAL_POS_X: u16 = 0; + +/// Intel-IA64-Filler +pub const X3_TMPLT_INST_WORD_X: u16 = 0; +/// Intel-IA64-Filler +pub const X3_TMPLT_SIZE_X: u16 = 4; +/// Intel-IA64-Filler +pub const X3_TMPLT_INST_WORD_POS_X: u16 = 0; +/// Intel-IA64-Filler +pub const X3_TMPLT_SIGN_VAL_POS_X: u16 = 0; + +/// Intel-IA64-Filler +pub const X3_BTYPE_QP_INST_WORD_X: u16 = 2; +/// Intel-IA64-Filler +pub const X3_BTYPE_QP_SIZE_X: u16 = 9; +/// Intel-IA64-Filler +pub const X3_BTYPE_QP_INST_WORD_POS_X: u16 = 23; +/// Intel-IA64-Filler +pub const X3_BTYPE_QP_INST_VAL_POS_X: u16 = 0; + +/// Intel-IA64-Filler +pub const X3_EMPTY_INST_WORD_X: u16 = 1; +/// Intel-IA64-Filler +pub const X3_EMPTY_SIZE_X: u16 = 2; +/// Intel-IA64-Filler +pub const X3_EMPTY_INST_WORD_POS_X: u16 = 14; +/// Intel-IA64-Filler +pub const X3_EMPTY_INST_VAL_POS_X: u16 = 0; + +// +// Line number format. +// + +// This struct has alignment 1. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageLinenumber { + /// Symbol table index of function name if Linenumber is 0. + /// Otherwise virtual address of line number. + pub symbol_table_index_or_virtual_address: U32Bytes, + /// Line number. + pub linenumber: U16Bytes, +} + +// +// Based relocation format. +// + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageBaseRelocation { + pub virtual_address: U32, + pub size_of_block: U32, + // pub type_offset[1]: U16, +} + +// +// Based relocation types. +// + +pub const IMAGE_REL_BASED_ABSOLUTE: u16 = 0; +pub const IMAGE_REL_BASED_HIGH: u16 = 1; +pub const IMAGE_REL_BASED_LOW: u16 = 2; +pub const IMAGE_REL_BASED_HIGHLOW: u16 = 3; +pub const IMAGE_REL_BASED_HIGHADJ: u16 = 4; +pub const IMAGE_REL_BASED_MACHINE_SPECIFIC_5: u16 = 5; +pub const IMAGE_REL_BASED_RESERVED: u16 = 6; +pub const IMAGE_REL_BASED_MACHINE_SPECIFIC_7: u16 = 7; +pub const IMAGE_REL_BASED_MACHINE_SPECIFIC_8: u16 = 8; +pub const IMAGE_REL_BASED_MACHINE_SPECIFIC_9: u16 = 9; +pub const IMAGE_REL_BASED_DIR64: u16 = 10; + +// +// Platform-specific based relocation types. +// + +pub const IMAGE_REL_BASED_IA64_IMM64: u16 = 9; + +pub const IMAGE_REL_BASED_MIPS_JMPADDR: u16 = 5; +pub const IMAGE_REL_BASED_MIPS_JMPADDR16: u16 = 9; + +pub const IMAGE_REL_BASED_ARM_MOV32: u16 = 5; +pub const IMAGE_REL_BASED_THUMB_MOV32: u16 = 7; + +pub const IMAGE_REL_BASED_RISCV_HIGH20: u16 = 5; +pub const IMAGE_REL_BASED_RISCV_LOW12I: u16 = 7; +pub const IMAGE_REL_BASED_RISCV_LOW12S: u16 = 8; + +// +// Archive format. +// + +pub const IMAGE_ARCHIVE_START_SIZE: usize = 8; +pub const IMAGE_ARCHIVE_START: &[u8; 8] = b"!\n"; +pub const IMAGE_ARCHIVE_END: &[u8] = b"`\n"; +pub const IMAGE_ARCHIVE_PAD: &[u8] = b"\n"; +pub const IMAGE_ARCHIVE_LINKER_MEMBER: &[u8; 16] = b"/ "; +pub const IMAGE_ARCHIVE_LONGNAMES_MEMBER: &[u8; 16] = b"// "; +pub const IMAGE_ARCHIVE_HYBRIDMAP_MEMBER: &[u8; 16] = b"// "; + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageArchiveMemberHeader { + /// File member name - `/' terminated. + pub name: [u8; 16], + /// File member date - decimal. + pub date: [u8; 12], + /// File member user id - decimal. + pub user_id: [u8; 6], + /// File member group id - decimal. + pub group_id: [u8; 6], + /// File member mode - octal. + pub mode: [u8; 8], + /// File member size - decimal. + pub size: [u8; 10], + /// String to end header. + pub end_header: [u8; 2], +} + +pub const IMAGE_SIZEOF_ARCHIVE_MEMBER_HDR: u16 = 60; + +// +// DLL support. +// + +// +// Export Format +// + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageExportDirectory { + pub characteristics: U32, + pub time_date_stamp: U32, + pub major_version: U16, + pub minor_version: U16, + pub name: U32, + pub base: U32, + pub number_of_functions: U32, + pub number_of_names: U32, + /// RVA from base of image + pub address_of_functions: U32, + /// RVA from base of image + pub address_of_names: U32, + /// RVA from base of image + pub address_of_name_ordinals: U32, +} + +// +// Import Format +// + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageImportByName { + pub hint: U16, + //pub name: [i8; 1], +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageThunkData64(pub U64); +/* + union { +/// PBYTE + pub forwarder_string: U64, +/// PDWORD + pub function: U64, + pub ordinal: U64, +/// PIMAGE_IMPORT_BY_NAME + pub address_of_data: U64, + } u1; +*/ + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageThunkData32(pub U32); +/* + union { +/// PBYTE + pub forwarder_string: U32, +/// PDWORD + pub function: U32, + pub ordinal: U32, +/// PIMAGE_IMPORT_BY_NAME + pub address_of_data: U32, + } u1; +} +*/ + +pub const IMAGE_ORDINAL_FLAG64: u64 = 0x8000000000000000; +pub const IMAGE_ORDINAL_FLAG32: u32 = 0x80000000; + +/* +#define IMAGE_ORDINAL64(Ordinal) (Ordinal & 0xffff) +#define IMAGE_ORDINAL32(Ordinal) (Ordinal & 0xffff) +#define IMAGE_SNAP_BY_ORDINAL64(Ordinal) ((Ordinal & IMAGE_ORDINAL_FLAG64) != 0) +#define IMAGE_SNAP_BY_ORDINAL32(Ordinal) ((Ordinal & IMAGE_ORDINAL_FLAG32) != 0) + +*/ + +// +// Thread Local Storage +// + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageTlsDirectory64 { + pub start_address_of_raw_data: U64, + pub end_address_of_raw_data: U64, + /// PDWORD + pub address_of_index: U64, + /// PIMAGE_TLS_CALLBACK *; + pub address_of_call_backs: U64, + pub size_of_zero_fill: U32, + pub characteristics: U32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageTlsDirectory32 { + pub start_address_of_raw_data: U32, + pub end_address_of_raw_data: U32, + /// PDWORD + pub address_of_index: U32, + /// PIMAGE_TLS_CALLBACK * + pub address_of_call_backs: U32, + pub size_of_zero_fill: U32, + pub characteristics: U32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageImportDescriptor { + /// RVA to original unbound IAT (`ImageThunkData32`/`ImageThunkData64`) + /// 0 for terminating null import descriptor + pub original_first_thunk: U32Bytes, + /// 0 if not bound, + /// -1 if bound, and real date\time stamp + /// in IMAGE_DIRECTORY_ENTRY_BOUND_IMPORT (new BIND) + /// O.W. date/time stamp of DLL bound to (Old BIND) + pub time_date_stamp: U32Bytes, + /// -1 if no forwarders + pub forwarder_chain: U32Bytes, + pub name: U32Bytes, + /// RVA to IAT (if bound this IAT has actual addresses) + pub first_thunk: U32Bytes, +} + +impl ImageImportDescriptor { + /// Tell whether this import descriptor is the null descriptor + /// (used to mark the end of the iterator array in a PE) + pub fn is_null(&self) -> bool { + self.original_first_thunk.get(LE) == 0 + && self.time_date_stamp.get(LE) == 0 + && self.forwarder_chain.get(LE) == 0 + && self.name.get(LE) == 0 + && self.first_thunk.get(LE) == 0 + } +} + +// +// New format import descriptors pointed to by DataDirectory[ IMAGE_DIRECTORY_ENTRY_BOUND_IMPORT ] +// + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageBoundImportDescriptor { + pub time_date_stamp: U32, + pub offset_module_name: U16, + pub number_of_module_forwarder_refs: U16, + // Array of zero or more IMAGE_BOUND_FORWARDER_REF follows +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageBoundForwarderRef { + pub time_date_stamp: U32, + pub offset_module_name: U16, + pub reserved: U16, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageDelayloadDescriptor { + pub attributes: U32, + + /// RVA to the name of the target library (NULL-terminate ASCII string) + pub dll_name_rva: U32, + /// RVA to the HMODULE caching location (PHMODULE) + pub module_handle_rva: U32, + /// RVA to the start of the IAT (PIMAGE_THUNK_DATA) + pub import_address_table_rva: U32, + /// RVA to the start of the name table (PIMAGE_THUNK_DATA::AddressOfData) + pub import_name_table_rva: U32, + /// RVA to an optional bound IAT + pub bound_import_address_table_rva: U32, + /// RVA to an optional unload info table + pub unload_information_table_rva: U32, + /// 0 if not bound, otherwise, date/time of the target DLL + pub time_date_stamp: U32, +} + +/// Delay load version 2 flag for `ImageDelayloadDescriptor::attributes`. +pub const IMAGE_DELAYLOAD_RVA_BASED: u32 = 0x8000_0000; + +// +// Resource Format. +// + +// +// Resource directory consists of two counts, following by a variable length +// array of directory entries. The first count is the number of entries at +// beginning of the array that have actual names associated with each entry. +// The entries are in ascending order, case insensitive strings. The second +// count is the number of entries that immediately follow the named entries. +// This second count identifies the number of entries that have 16-bit integer +// Ids as their name. These entries are also sorted in ascending order. +// +// This structure allows fast lookup by either name or number, but for any +// given resource entry only one form of lookup is supported, not both. +// This is consistant with the syntax of the .RC file and the .RES file. +// + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageResourceDirectory { + pub characteristics: U32, + pub time_date_stamp: U32, + pub major_version: U16, + pub minor_version: U16, + pub number_of_named_entries: U16, + pub number_of_id_entries: U16, +} + +pub const IMAGE_RESOURCE_NAME_IS_STRING: u32 = 0x8000_0000; +pub const IMAGE_RESOURCE_DATA_IS_DIRECTORY: u32 = 0x8000_0000; +// +// Each directory contains the 32-bit Name of the entry and an offset, +// relative to the beginning of the resource directory of the data associated +// with this directory entry. If the name of the entry is an actual text +// string instead of an integer Id, then the high order bit of the name field +// is set to one and the low order 31-bits are an offset, relative to the +// beginning of the resource directory of the string, which is of type +// IMAGE_RESOURCE_DIRECTORY_STRING. Otherwise the high bit is clear and the +// low-order 16-bits are the integer Id that identify this resource directory +// entry. If the directory entry is yet another resource directory (i.e. a +// subdirectory), then the high order bit of the offset field will be +// set to indicate this. Otherwise the high bit is clear and the offset +// field points to a resource data entry. +// + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageResourceDirectoryEntry { + pub name_or_id: U32, + pub offset_to_data_or_directory: U32, +} + +// +// For resource directory entries that have actual string names, the Name +// field of the directory entry points to an object of the following type. +// All of these string objects are stored together after the last resource +// directory entry and before the first resource data object. This minimizes +// the impact of these variable length objects on the alignment of the fixed +// size directory entry objects. +// + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageResourceDirectoryString { + pub length: U16, + //pub name_string: [i8; 1], +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageResourceDirStringU { + pub length: U16, + //pub name_string: [U16; 1], +} + +// +// Each resource data entry describes a leaf node in the resource directory +// tree. It contains an offset, relative to the beginning of the resource +// directory of the data for the resource, a size field that gives the number +// of bytes of data at that offset, a CodePage that should be used when +// decoding code point values within the resource data. Typically for new +// applications the code page would be the unicode code page. +// + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageResourceDataEntry { + /// RVA of the data. + pub offset_to_data: U32, + pub size: U32, + pub code_page: U32, + pub reserved: U32, +} + +// Resource type: https://docs.microsoft.com/en-us/windows/win32/menurc/resource-types + +/// ID for: Hardware-dependent cursor resource. +pub const RT_CURSOR: u16 = 1; +/// ID for: Bitmap resource. +pub const RT_BITMAP: u16 = 2; +/// ID for: Hardware-dependent icon resource. +pub const RT_ICON: u16 = 3; +/// ID for: Menu resource. +pub const RT_MENU: u16 = 4; +/// ID for: Dialog box. +pub const RT_DIALOG: u16 = 5; +/// ID for: String-table entry. +pub const RT_STRING: u16 = 6; +/// ID for: Font directory resource. +pub const RT_FONTDIR: u16 = 7; +/// ID for: Font resource. +pub const RT_FONT: u16 = 8; +/// ID for: Accelerator table. +pub const RT_ACCELERATOR: u16 = 9; +/// ID for: Application-defined resource (raw data). +pub const RT_RCDATA: u16 = 10; +/// ID for: Message-table entry. +pub const RT_MESSAGETABLE: u16 = 11; +/// ID for: Hardware-independent cursor resource. +pub const RT_GROUP_CURSOR: u16 = 12; +/// ID for: Hardware-independent icon resource. +pub const RT_GROUP_ICON: u16 = 14; +/// ID for: Version resource. +pub const RT_VERSION: u16 = 16; +/// ID for: Allows a resource editing tool to associate a string with an .rc file. +pub const RT_DLGINCLUDE: u16 = 17; +/// ID for: Plug and Play resource. +pub const RT_PLUGPLAY: u16 = 19; +/// ID for: VXD. +pub const RT_VXD: u16 = 20; +/// ID for: Animated cursor. +pub const RT_ANICURSOR: u16 = 21; +/// ID for: Animated icon. +pub const RT_ANIICON: u16 = 22; +/// ID for: HTML resource. +pub const RT_HTML: u16 = 23; +/// ID for: Side-by-Side Assembly Manifest. +pub const RT_MANIFEST: u16 = 24; + +// +// Code Integrity in loadconfig (CI) +// + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageLoadConfigCodeIntegrity { + /// Flags to indicate if CI information is available, etc. + pub flags: U16, + /// 0xFFFF means not available + pub catalog: U16, + pub catalog_offset: U32, + /// Additional bitmask to be defined later + pub reserved: U32, +} + +// +// Dynamic value relocation table in loadconfig +// + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageDynamicRelocationTable { + pub version: U32, + pub size: U32, + // DynamicRelocations: [ImageDynamicRelocation; 0], +} + +// +// Dynamic value relocation entries following IMAGE_DYNAMIC_RELOCATION_TABLE +// + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageDynamicRelocation32 { + pub symbol: U32, + pub base_reloc_size: U32, + // BaseRelocations: [ImageBaseRelocation; 0], +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageDynamicRelocation64 { + pub symbol: U64, + pub base_reloc_size: U32, + // BaseRelocations: [ImageBaseRelocation; 0], +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageDynamicRelocation32V2 { + pub header_size: U32, + pub fixup_info_size: U32, + pub symbol: U32, + pub symbol_group: U32, + pub flags: U32, + // ... variable length header fields + // pub fixup_info: [u8; fixup_info_size] +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageDynamicRelocation64V2 { + pub header_size: U32, + pub fixup_info_size: U32, + pub symbol: U64, + pub symbol_group: U32, + pub flags: U32, + // ... variable length header fields + // pub fixup_info[u8; fixup_info_size] +} + +// +// Defined symbolic dynamic relocation entries. +// + +pub const IMAGE_DYNAMIC_RELOCATION_GUARD_RF_PROLOGUE: u32 = 0x0000_0001; +pub const IMAGE_DYNAMIC_RELOCATION_GUARD_RF_EPILOGUE: u32 = 0x0000_0002; +pub const IMAGE_DYNAMIC_RELOCATION_GUARD_IMPORT_CONTROL_TRANSFER: u32 = 0x0000_0003; +pub const IMAGE_DYNAMIC_RELOCATION_GUARD_INDIR_CONTROL_TRANSFER: u32 = 0x0000_0004; +pub const IMAGE_DYNAMIC_RELOCATION_GUARD_SWITCHTABLE_BRANCH: u32 = 0x0000_0005; + +// This struct has alignment 1. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImagePrologueDynamicRelocationHeader { + pub prologue_byte_count: u8, + // pub prologue_bytes: [u8; prologue_byte_count], +} + +// This struct has alignment 1. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageEpilogueDynamicRelocationHeader { + pub epilogue_count: U32Bytes, + pub epilogue_byte_count: u8, + pub branch_descriptor_element_size: u8, + pub branch_descriptor_count: U16Bytes, + // pub branch_descriptors[...], + // pub branch_descriptor_bit_map[...], +} + +/* +// TODO? bitfields +// TODO: unaligned? +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageImportControlTransferDynamicRelocation { + DWORD PageRelativeOffset : 12; + DWORD IndirectCall : 1; + DWORD IATIndex : 19; +} + +// TODO: unaligned? +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageIndirControlTransferDynamicRelocation { + WORD PageRelativeOffset : 12; + WORD IndirectCall : 1; + WORD RexWPrefix : 1; + WORD CfgCheck : 1; + WORD Reserved : 1; +} + +// TODO: unaligned? +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageSwitchtableBranchDynamicRelocation { + WORD PageRelativeOffset : 12; + WORD RegisterNumber : 4; +} +*/ + +// +// Load Configuration Directory Entry +// + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageLoadConfigDirectory32 { + pub size: U32, + pub time_date_stamp: U32, + pub major_version: U16, + pub minor_version: U16, + pub global_flags_clear: U32, + pub global_flags_set: U32, + pub critical_section_default_timeout: U32, + pub de_commit_free_block_threshold: U32, + pub de_commit_total_free_threshold: U32, + /// VA + pub lock_prefix_table: U32, + pub maximum_allocation_size: U32, + pub virtual_memory_threshold: U32, + pub process_heap_flags: U32, + pub process_affinity_mask: U32, + pub csd_version: U16, + pub dependent_load_flags: U16, + /// VA + pub edit_list: U32, + /// VA + pub security_cookie: U32, + /// VA + pub sehandler_table: U32, + pub sehandler_count: U32, + /// VA + pub guard_cf_check_function_pointer: U32, + /// VA + pub guard_cf_dispatch_function_pointer: U32, + /// VA + pub guard_cf_function_table: U32, + pub guard_cf_function_count: U32, + pub guard_flags: U32, + pub code_integrity: ImageLoadConfigCodeIntegrity, + /// VA + pub guard_address_taken_iat_entry_table: U32, + pub guard_address_taken_iat_entry_count: U32, + /// VA + pub guard_long_jump_target_table: U32, + pub guard_long_jump_target_count: U32, + /// VA + pub dynamic_value_reloc_table: U32, + pub chpe_metadata_pointer: U32, + /// VA + pub guard_rf_failure_routine: U32, + /// VA + pub guard_rf_failure_routine_function_pointer: U32, + pub dynamic_value_reloc_table_offset: U32, + pub dynamic_value_reloc_table_section: U16, + pub reserved2: U16, + /// VA + pub guard_rf_verify_stack_pointer_function_pointer: U32, + pub hot_patch_table_offset: U32, + pub reserved3: U32, + /// VA + pub enclave_configuration_pointer: U32, + /// VA + pub volatile_metadata_pointer: U32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageLoadConfigDirectory64 { + pub size: U32, + pub time_date_stamp: U32, + pub major_version: U16, + pub minor_version: U16, + pub global_flags_clear: U32, + pub global_flags_set: U32, + pub critical_section_default_timeout: U32, + pub de_commit_free_block_threshold: U64, + pub de_commit_total_free_threshold: U64, + /// VA + pub lock_prefix_table: U64, + pub maximum_allocation_size: U64, + pub virtual_memory_threshold: U64, + pub process_affinity_mask: U64, + pub process_heap_flags: U32, + pub csd_version: U16, + pub dependent_load_flags: U16, + /// VA + pub edit_list: U64, + /// VA + pub security_cookie: U64, + /// VA + pub sehandler_table: U64, + pub sehandler_count: U64, + /// VA + pub guard_cf_check_function_pointer: U64, + /// VA + pub guard_cf_dispatch_function_pointer: U64, + /// VA + pub guard_cf_function_table: U64, + pub guard_cf_function_count: U64, + pub guard_flags: U32, + pub code_integrity: ImageLoadConfigCodeIntegrity, + /// VA + pub guard_address_taken_iat_entry_table: U64, + pub guard_address_taken_iat_entry_count: U64, + /// VA + pub guard_long_jump_target_table: U64, + pub guard_long_jump_target_count: U64, + /// VA + pub dynamic_value_reloc_table: U64, + /// VA + pub chpe_metadata_pointer: U64, + /// VA + pub guard_rf_failure_routine: U64, + /// VA + pub guard_rf_failure_routine_function_pointer: U64, + pub dynamic_value_reloc_table_offset: U32, + pub dynamic_value_reloc_table_section: U16, + pub reserved2: U16, + /// VA + pub guard_rf_verify_stack_pointer_function_pointer: U64, + pub hot_patch_table_offset: U32, + pub reserved3: U32, + /// VA + pub enclave_configuration_pointer: U64, + /// VA + pub volatile_metadata_pointer: U64, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageHotPatchInfo { + pub version: U32, + pub size: U32, + pub sequence_number: U32, + pub base_image_list: U32, + pub base_image_count: U32, + /// Version 2 and later + pub buffer_offset: U32, + /// Version 3 and later + pub extra_patch_size: U32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageHotPatchBase { + pub sequence_number: U32, + pub flags: U32, + pub original_time_date_stamp: U32, + pub original_check_sum: U32, + pub code_integrity_info: U32, + pub code_integrity_size: U32, + pub patch_table: U32, + /// Version 2 and later + pub buffer_offset: U32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageHotPatchHashes { + pub sha256: [u8; 32], + pub sha1: [u8; 20], +} + +pub const IMAGE_HOT_PATCH_BASE_OBLIGATORY: u32 = 0x0000_0001; +pub const IMAGE_HOT_PATCH_BASE_CAN_ROLL_BACK: u32 = 0x0000_0002; + +pub const IMAGE_HOT_PATCH_CHUNK_INVERSE: u32 = 0x8000_0000; +pub const IMAGE_HOT_PATCH_CHUNK_OBLIGATORY: u32 = 0x4000_0000; +pub const IMAGE_HOT_PATCH_CHUNK_RESERVED: u32 = 0x3FF0_3000; +pub const IMAGE_HOT_PATCH_CHUNK_TYPE: u32 = 0x000F_C000; +pub const IMAGE_HOT_PATCH_CHUNK_SOURCE_RVA: u32 = 0x0000_8000; +pub const IMAGE_HOT_PATCH_CHUNK_TARGET_RVA: u32 = 0x0000_4000; +pub const IMAGE_HOT_PATCH_CHUNK_SIZE: u32 = 0x0000_0FFF; + +pub const IMAGE_HOT_PATCH_NONE: u32 = 0x0000_0000; +pub const IMAGE_HOT_PATCH_FUNCTION: u32 = 0x0001_C000; +pub const IMAGE_HOT_PATCH_ABSOLUTE: u32 = 0x0002_C000; +pub const IMAGE_HOT_PATCH_REL32: u32 = 0x0003_C000; +pub const IMAGE_HOT_PATCH_CALL_TARGET: u32 = 0x0004_4000; +pub const IMAGE_HOT_PATCH_INDIRECT: u32 = 0x0005_C000; +pub const IMAGE_HOT_PATCH_NO_CALL_TARGET: u32 = 0x0006_4000; +pub const IMAGE_HOT_PATCH_DYNAMIC_VALUE: u32 = 0x0007_8000; + +/// Module performs control flow integrity checks using system-supplied support +pub const IMAGE_GUARD_CF_INSTRUMENTED: u32 = 0x0000_0100; +/// Module performs control flow and write integrity checks +pub const IMAGE_GUARD_CFW_INSTRUMENTED: u32 = 0x0000_0200; +/// Module contains valid control flow target metadata +pub const IMAGE_GUARD_CF_FUNCTION_TABLE_PRESENT: u32 = 0x0000_0400; +/// Module does not make use of the /GS security cookie +pub const IMAGE_GUARD_SECURITY_COOKIE_UNUSED: u32 = 0x0000_0800; +/// Module supports read only delay load IAT +pub const IMAGE_GUARD_PROTECT_DELAYLOAD_IAT: u32 = 0x0000_1000; +/// Delayload import table in its own .didat section (with nothing else in it) that can be freely reprotected +pub const IMAGE_GUARD_DELAYLOAD_IAT_IN_ITS_OWN_SECTION: u32 = 0x0000_2000; +/// Module contains suppressed export information. +/// +/// This also infers that the address taken taken IAT table is also present in the load config. +pub const IMAGE_GUARD_CF_EXPORT_SUPPRESSION_INFO_PRESENT: u32 = 0x0000_4000; +/// Module enables suppression of exports +pub const IMAGE_GUARD_CF_ENABLE_EXPORT_SUPPRESSION: u32 = 0x0000_8000; +/// Module contains longjmp target information +pub const IMAGE_GUARD_CF_LONGJUMP_TABLE_PRESENT: u32 = 0x0001_0000; +/// Module contains return flow instrumentation and metadata +pub const IMAGE_GUARD_RF_INSTRUMENTED: u32 = 0x0002_0000; +/// Module requests that the OS enable return flow protection +pub const IMAGE_GUARD_RF_ENABLE: u32 = 0x0004_0000; +/// Module requests that the OS enable return flow protection in strict mode +pub const IMAGE_GUARD_RF_STRICT: u32 = 0x0008_0000; +/// Module was built with retpoline support +pub const IMAGE_GUARD_RETPOLINE_PRESENT: u32 = 0x0010_0000; + +/// Stride of Guard CF function table encoded in these bits (additional count of bytes per element) +pub const IMAGE_GUARD_CF_FUNCTION_TABLE_SIZE_MASK: u32 = 0xF000_0000; +/// Shift to right-justify Guard CF function table stride +pub const IMAGE_GUARD_CF_FUNCTION_TABLE_SIZE_SHIFT: u32 = 28; + +// +// GFIDS table entry flags. +// + +/// The containing GFID entry is suppressed +pub const IMAGE_GUARD_FLAG_FID_SUPPRESSED: u16 = 0x01; +/// The containing GFID entry is export suppressed +pub const IMAGE_GUARD_FLAG_EXPORT_SUPPRESSED: u16 = 0x02; + +// +// WIN CE Exception table format +// + +// +// Function table entry format. Function table is pointed to by the +// IMAGE_DIRECTORY_ENTRY_EXCEPTION directory entry. +// + +/* +// TODO? bitfields +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageCeRuntimeFunctionEntry { + pub func_start: U32, + DWORD PrologLen : 8; + DWORD FuncLen : 22; + DWORD ThirtyTwoBit : 1; + DWORD ExceptionFlag : 1; +} +*/ + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageArmRuntimeFunctionEntry { + pub begin_address: U32, + pub unwind_data: U32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageArm64RuntimeFunctionEntry { + pub begin_address: U32, + pub unwind_data: U32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageAlpha64RuntimeFunctionEntry { + pub begin_address: U64, + pub end_address: U64, + pub exception_handler: U64, + pub handler_data: U64, + pub prolog_end_address: U64, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageAlphaRuntimeFunctionEntry { + pub begin_address: U32, + pub end_address: U32, + pub exception_handler: U32, + pub handler_data: U32, + pub prolog_end_address: U32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageRuntimeFunctionEntry { + pub begin_address: U32, + pub end_address: U32, + pub unwind_info_address_or_data: U32, +} + +// +// Sofware enclave information +// + +pub const IMAGE_ENCLAVE_LONG_ID_LENGTH: usize = 32; +pub const IMAGE_ENCLAVE_SHORT_ID_LENGTH: usize = 16; + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageEnclaveConfig32 { + pub size: U32, + pub minimum_required_config_size: U32, + pub policy_flags: U32, + pub number_of_imports: U32, + pub import_list: U32, + pub import_entry_size: U32, + pub family_id: [u8; IMAGE_ENCLAVE_SHORT_ID_LENGTH], + pub image_id: [u8; IMAGE_ENCLAVE_SHORT_ID_LENGTH], + pub image_version: U32, + pub security_version: U32, + pub enclave_size: U32, + pub number_of_threads: U32, + pub enclave_flags: U32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageEnclaveConfig64 { + pub size: U32, + pub minimum_required_config_size: U32, + pub policy_flags: U32, + pub number_of_imports: U32, + pub import_list: U32, + pub import_entry_size: U32, + pub family_id: [u8; IMAGE_ENCLAVE_SHORT_ID_LENGTH], + pub image_id: [u8; IMAGE_ENCLAVE_SHORT_ID_LENGTH], + pub image_version: U32, + pub security_version: U32, + pub enclave_size: U64, + pub number_of_threads: U32, + pub enclave_flags: U32, +} + +//pub const IMAGE_ENCLAVE_MINIMUM_CONFIG_SIZE: usize = FIELD_OFFSET(IMAGE_ENCLAVE_CONFIG, EnclaveFlags); + +pub const IMAGE_ENCLAVE_POLICY_DEBUGGABLE: u32 = 0x0000_0001; + +pub const IMAGE_ENCLAVE_FLAG_PRIMARY_IMAGE: u32 = 0x0000_0001; + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageEnclaveImport { + pub match_type: U32, + pub minimum_security_version: U32, + pub unique_or_author_id: [u8; IMAGE_ENCLAVE_LONG_ID_LENGTH], + pub family_id: [u8; IMAGE_ENCLAVE_SHORT_ID_LENGTH], + pub image_id: [u8; IMAGE_ENCLAVE_SHORT_ID_LENGTH], + pub import_name: U32, + pub reserved: U32, +} + +pub const IMAGE_ENCLAVE_IMPORT_MATCH_NONE: u32 = 0x0000_0000; +pub const IMAGE_ENCLAVE_IMPORT_MATCH_UNIQUE_ID: u32 = 0x0000_0001; +pub const IMAGE_ENCLAVE_IMPORT_MATCH_AUTHOR_ID: u32 = 0x0000_0002; +pub const IMAGE_ENCLAVE_IMPORT_MATCH_FAMILY_ID: u32 = 0x0000_0003; +pub const IMAGE_ENCLAVE_IMPORT_MATCH_IMAGE_ID: u32 = 0x0000_0004; + +// +// Debug Format +// + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageDebugDirectory { + pub characteristics: U32, + pub time_date_stamp: U32, + pub major_version: U16, + pub minor_version: U16, + pub typ: U32, + pub size_of_data: U32, + pub address_of_raw_data: U32, + pub pointer_to_raw_data: U32, +} + +pub const IMAGE_DEBUG_TYPE_UNKNOWN: u32 = 0; +pub const IMAGE_DEBUG_TYPE_COFF: u32 = 1; +pub const IMAGE_DEBUG_TYPE_CODEVIEW: u32 = 2; +pub const IMAGE_DEBUG_TYPE_FPO: u32 = 3; +pub const IMAGE_DEBUG_TYPE_MISC: u32 = 4; +pub const IMAGE_DEBUG_TYPE_EXCEPTION: u32 = 5; +pub const IMAGE_DEBUG_TYPE_FIXUP: u32 = 6; +pub const IMAGE_DEBUG_TYPE_OMAP_TO_SRC: u32 = 7; +pub const IMAGE_DEBUG_TYPE_OMAP_FROM_SRC: u32 = 8; +pub const IMAGE_DEBUG_TYPE_BORLAND: u32 = 9; +pub const IMAGE_DEBUG_TYPE_RESERVED10: u32 = 10; +pub const IMAGE_DEBUG_TYPE_CLSID: u32 = 11; +pub const IMAGE_DEBUG_TYPE_VC_FEATURE: u32 = 12; +pub const IMAGE_DEBUG_TYPE_POGO: u32 = 13; +pub const IMAGE_DEBUG_TYPE_ILTCG: u32 = 14; +pub const IMAGE_DEBUG_TYPE_MPX: u32 = 15; +pub const IMAGE_DEBUG_TYPE_REPRO: u32 = 16; + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageCoffSymbolsHeader { + pub number_of_symbols: U32, + pub lva_to_first_symbol: U32, + pub number_of_linenumbers: U32, + pub lva_to_first_linenumber: U32, + pub rva_to_first_byte_of_code: U32, + pub rva_to_last_byte_of_code: U32, + pub rva_to_first_byte_of_data: U32, + pub rva_to_last_byte_of_data: U32, +} + +pub const FRAME_FPO: u16 = 0; +pub const FRAME_TRAP: u16 = 1; +pub const FRAME_TSS: u16 = 2; +pub const FRAME_NONFPO: u16 = 3; + +/* +// TODO? bitfields +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct FpoData { +/// offset 1st byte of function code + pub ul_off_start: U32, +/// # bytes in function + pub cb_proc_size: U32, +/// # bytes in locals/4 + pub cdw_locals: U32, +/// # bytes in params/4 + pub cdw_params: U16, +/// # bytes in prolog + WORD cbProlog : 8; +/// # regs saved + WORD cbRegs : 3; +/// TRUE if SEH in func + WORD fHasSEH : 1; +/// TRUE if EBP has been allocated + WORD fUseBP : 1; +/// reserved for future use + WORD reserved : 1; +/// frame type + WORD cbFrame : 2; +} +pub const SIZEOF_RFPO_DATA: usize = 16; +*/ + +pub const IMAGE_DEBUG_MISC_EXENAME: u16 = 1; + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageDebugMisc { + /// type of misc data, see defines + pub data_type: U32, + /// total length of record, rounded to four byte multiple. + pub length: U32, + /// TRUE if data is unicode string + pub unicode: u8, + pub reserved: [u8; 3], + // Actual data + //pub data: [u8; 1], +} + +// +// Function table extracted from MIPS/ALPHA/IA64 images. Does not contain +// information needed only for runtime support. Just those fields for +// each entry needed by a debugger. +// + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageFunctionEntry { + pub starting_address: U32, + pub ending_address: U32, + pub end_of_prologue: U32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageFunctionEntry64 { + pub starting_address: U64, + pub ending_address: U64, + pub end_of_prologue_or_unwind_info_address: U64, +} + +// +// Debugging information can be stripped from an image file and placed +// in a separate .DBG file, whose file name part is the same as the +// image file name part (e.g. symbols for CMD.EXE could be stripped +// and placed in CMD.DBG). This is indicated by the IMAGE_FILE_DEBUG_STRIPPED +// flag in the Characteristics field of the file header. The beginning of +// the .DBG file contains the following structure which captures certain +// information from the image file. This allows a debug to proceed even if +// the original image file is not accessable. This header is followed by +// zero of more IMAGE_SECTION_HEADER structures, followed by zero or more +// IMAGE_DEBUG_DIRECTORY structures. The latter structures and those in +// the image file contain file offsets relative to the beginning of the +// .DBG file. +// +// If symbols have been stripped from an image, the IMAGE_DEBUG_MISC structure +// is left in the image file, but not mapped. This allows a debugger to +// compute the name of the .DBG file, from the name of the image in the +// IMAGE_DEBUG_MISC structure. +// + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageSeparateDebugHeader { + pub signature: U16, + pub flags: U16, + pub machine: U16, + pub characteristics: U16, + pub time_date_stamp: U32, + pub check_sum: U32, + pub image_base: U32, + pub size_of_image: U32, + pub number_of_sections: U32, + pub exported_names_size: U32, + pub debug_directory_size: U32, + pub section_alignment: U32, + pub reserved: [U32; 2], +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct NonPagedDebugInfo { + pub signature: U16, + pub flags: U16, + pub size: U32, + pub machine: U16, + pub characteristics: U16, + pub time_date_stamp: U32, + pub check_sum: U32, + pub size_of_image: U32, + pub image_base: U64, + //debug_directory_size + //ImageDebugDirectory +} + +pub const IMAGE_SEPARATE_DEBUG_SIGNATURE: u16 = 0x4944; +pub const NON_PAGED_DEBUG_SIGNATURE: u16 = 0x494E; + +pub const IMAGE_SEPARATE_DEBUG_FLAGS_MASK: u16 = 0x8000; +/// when DBG was updated, the old checksum didn't match. +pub const IMAGE_SEPARATE_DEBUG_MISMATCH: u16 = 0x8000; + +// +// The .arch section is made up of headers, each describing an amask position/value +// pointing to an array of IMAGE_ARCHITECTURE_ENTRY's. Each "array" (both the header +// and entry arrays) are terminiated by a quadword of 0xffffffffL. +// +// NOTE: There may be quadwords of 0 sprinkled around and must be skipped. +// + +/* +// TODO? bitfields +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageArchitectureHeader { + /// 1 -> code section depends on mask bit + /// 0 -> new instruction depends on mask bit + unsigned int AmaskValue: 1; + /// MBZ + int :7; + /// Amask bit in question for this fixup + unsigned int AmaskShift: 8; + /// MBZ + int :16; + /// RVA into .arch section to array of ARCHITECTURE_ENTRY's + pub first_entry_rva: U32, +} +*/ + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageArchitectureEntry { + /// RVA of instruction to fixup + pub fixup_inst_rva: U32, + /// fixup instruction (see alphaops.h) + pub new_inst: U32, +} + +// The following structure defines the new import object. Note the values of the first two fields, +// which must be set as stated in order to differentiate old and new import members. +// Following this structure, the linker emits two null-terminated strings used to recreate the +// import at the time of use. The first string is the import's name, the second is the dll's name. + +pub const IMPORT_OBJECT_HDR_SIG2: u16 = 0xffff; + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImportObjectHeader { + /// Must be IMAGE_FILE_MACHINE_UNKNOWN + pub sig1: U16, + /// Must be IMPORT_OBJECT_HDR_SIG2. + pub sig2: U16, + pub version: U16, + pub machine: U16, + /// Time/date stamp + pub time_date_stamp: U32, + /// particularly useful for incremental links + pub size_of_data: U32, + + /// if grf & IMPORT_OBJECT_ORDINAL + pub ordinal_or_hint: U16, + + // WORD Type : 2; + // WORD NameType : 3; + // WORD Reserved : 11; + pub name_type: U16, +} + +pub const IMPORT_OBJECT_CODE: u16 = 0; +pub const IMPORT_OBJECT_DATA: u16 = 1; +pub const IMPORT_OBJECT_CONST: u16 = 2; + +/// Import by ordinal +pub const IMPORT_OBJECT_ORDINAL: u16 = 0; +/// Import name == public symbol name. +pub const IMPORT_OBJECT_NAME: u16 = 1; +/// Import name == public symbol name skipping leading ?, @, or optionally _. +pub const IMPORT_OBJECT_NAME_NO_PREFIX: u16 = 2; +/// Import name == public symbol name skipping leading ?, @, or optionally _ and truncating at first @. +pub const IMPORT_OBJECT_NAME_UNDECORATE: u16 = 3; +/// Import name == a name is explicitly provided after the DLL name. +pub const IMPORT_OBJECT_NAME_EXPORTAS: u16 = 4; + +// COM+ Header entry point flags. +pub const COMIMAGE_FLAGS_ILONLY: u32 = 0x0000_0001; +pub const COMIMAGE_FLAGS_32BITREQUIRED: u32 = 0x0000_0002; +pub const COMIMAGE_FLAGS_IL_LIBRARY: u32 = 0x0000_0004; +pub const COMIMAGE_FLAGS_STRONGNAMESIGNED: u32 = 0x0000_0008; +pub const COMIMAGE_FLAGS_NATIVE_ENTRYPOINT: u32 = 0x0000_0010; +pub const COMIMAGE_FLAGS_TRACKDEBUGDATA: u32 = 0x0001_0000; +pub const COMIMAGE_FLAGS_32BITPREFERRED: u32 = 0x0002_0000; + +// Version flags for image. +pub const COR_VERSION_MAJOR_V2: u16 = 2; +pub const COR_VERSION_MAJOR: u16 = COR_VERSION_MAJOR_V2; +pub const COR_VERSION_MINOR: u16 = 5; +pub const COR_DELETED_NAME_LENGTH: usize = 8; +pub const COR_VTABLEGAP_NAME_LENGTH: usize = 8; + +// Maximum size of a NativeType descriptor. +pub const NATIVE_TYPE_MAX_CB: u16 = 1; +pub const COR_ILMETHOD_SECT_SMALL_MAX_DATASIZE: u16 = 0xFF; + +// Consts for the MIH FLAGS +pub const IMAGE_COR_MIH_METHODRVA: u16 = 0x01; +pub const IMAGE_COR_MIH_EHRVA: u16 = 0x02; +pub const IMAGE_COR_MIH_BASICBLOCK: u16 = 0x08; + +// V-table constants +/// V-table slots are 32-bits in size. +pub const COR_VTABLE_32BIT: u16 = 0x01; +/// V-table slots are 64-bits in size. +pub const COR_VTABLE_64BIT: u16 = 0x02; +/// If set, transition from unmanaged. +pub const COR_VTABLE_FROM_UNMANAGED: u16 = 0x04; +/// If set, transition from unmanaged with keeping the current appdomain. +pub const COR_VTABLE_FROM_UNMANAGED_RETAIN_APPDOMAIN: u16 = 0x08; +/// Call most derived method described by +pub const COR_VTABLE_CALL_MOST_DERIVED: u16 = 0x10; + +// EATJ constants +/// Size of a jump thunk reserved range. +pub const IMAGE_COR_EATJ_THUNK_SIZE: usize = 32; + +// Max name lengths +pub const MAX_CLASS_NAME: usize = 1024; +pub const MAX_PACKAGE_NAME: usize = 1024; + +// CLR 2.0 header structure. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct ImageCor20Header { + // Header versioning + pub cb: U32, + pub major_runtime_version: U16, + pub minor_runtime_version: U16, + + // Symbol table and startup information + pub meta_data: ImageDataDirectory, + pub flags: U32, + + // If COMIMAGE_FLAGS_NATIVE_ENTRYPOINT is not set, EntryPointToken represents a managed entrypoint. + // If COMIMAGE_FLAGS_NATIVE_ENTRYPOINT is set, EntryPointRVA represents an RVA to a native entrypoint. + pub entry_point_token_or_rva: U32, + + // Binding information + pub resources: ImageDataDirectory, + pub strong_name_signature: ImageDataDirectory, + + // Regular fixup and binding information + pub code_manager_table: ImageDataDirectory, + pub vtable_fixups: ImageDataDirectory, + pub export_address_table_jumps: ImageDataDirectory, + + // Precompiled image info (internal use only - set to zero) + pub managed_native_header: ImageDataDirectory, +} + +unsafe_impl_pod!( + ImageDosHeader, + ImageOs2Header, + ImageVxdHeader, + ImageFileHeader, + ImageDataDirectory, + ImageOptionalHeader32, + ImageRomOptionalHeader, + ImageOptionalHeader64, + ImageNtHeaders64, + ImageNtHeaders32, + ImageRomHeaders, + Guid, + AnonObjectHeader, + AnonObjectHeaderV2, + AnonObjectHeaderBigobj, + ImageSectionHeader, + ImageSymbol, + ImageSymbolBytes, + ImageSymbolEx, + ImageSymbolExBytes, + ImageAuxSymbolTokenDef, + ImageAuxSymbolFunction, + ImageAuxSymbolFunctionBeginEnd, + ImageAuxSymbolWeak, + ImageAuxSymbolSection, + ImageAuxSymbolCrc, + ImageRelocation, + ImageLinenumber, + ImageBaseRelocation, + ImageArchiveMemberHeader, + ImageExportDirectory, + ImageImportByName, + ImageThunkData64, + ImageThunkData32, + ImageTlsDirectory64, + ImageTlsDirectory32, + ImageImportDescriptor, + ImageBoundImportDescriptor, + ImageBoundForwarderRef, + ImageDelayloadDescriptor, + ImageResourceDirectory, + ImageResourceDirectoryEntry, + ImageResourceDirectoryString, + ImageResourceDirStringU, + ImageResourceDataEntry, + ImageLoadConfigCodeIntegrity, + ImageDynamicRelocationTable, + ImageDynamicRelocation32, + ImageDynamicRelocation64, + ImageDynamicRelocation32V2, + ImageDynamicRelocation64V2, + ImagePrologueDynamicRelocationHeader, + ImageEpilogueDynamicRelocationHeader, + //ImageImportControlTransferDynamicRelocation, + //ImageIndirControlTransferDynamicRelocation, + //ImageSwitchtableBranchDynamicRelocation, + ImageLoadConfigDirectory32, + ImageLoadConfigDirectory64, + ImageHotPatchInfo, + ImageHotPatchBase, + ImageHotPatchHashes, + //ImageCeRuntimeFunctionEntry, + ImageArmRuntimeFunctionEntry, + ImageArm64RuntimeFunctionEntry, + ImageAlpha64RuntimeFunctionEntry, + ImageAlphaRuntimeFunctionEntry, + ImageRuntimeFunctionEntry, + ImageEnclaveConfig32, + ImageEnclaveConfig64, + ImageEnclaveImport, + ImageDebugDirectory, + ImageCoffSymbolsHeader, + //FpoData, + ImageDebugMisc, + ImageFunctionEntry, + ImageFunctionEntry64, + ImageSeparateDebugHeader, + NonPagedDebugInfo, + //ImageArchitectureHeader, + ImageArchitectureEntry, + ImportObjectHeader, + ImageCor20Header, + MaskedRichHeaderEntry, +); diff --git a/crux-mir/lib/object/src/pod.rs b/crux-mir/lib/object/src/pod.rs new file mode 100644 index 000000000..805cf8249 --- /dev/null +++ b/crux-mir/lib/object/src/pod.rs @@ -0,0 +1,238 @@ +//! Tools for converting file format structures to and from bytes. +//! +//! This module should be replaced once rust provides safe transmutes. + +// This module provides functions for both read and write features. +#![cfg_attr( + not(all(feature = "read_core", feature = "write_core")), + allow(dead_code) +)] + +use core::{mem, result, slice}; + +type Result = result::Result; + +/// A trait for types that can safely be converted from and to byte slices. +/// +/// A type that is `Pod` must: +/// - be `#[repr(C)]` or `#[repr(transparent)]` +/// - have no invalid byte values +/// - have no padding +pub unsafe trait Pod: Copy + 'static {} + +/// Cast a byte slice to a `Pod` type. +/// +/// Returns the type and the tail of the slice. +#[inline] +pub fn from_bytes(data: &[u8]) -> Result<(&T, &[u8])> { + let size = mem::size_of::(); + let tail = data.get(size..).ok_or(())?; + let ptr = data.as_ptr(); + if (ptr as usize) % mem::align_of::() != 0 { + return Err(()); + } + // Safety: + // The alignment and size are checked by this function. + // The Pod trait ensures the type is valid to cast from bytes. + let val = unsafe { &*ptr.cast() }; + Ok((val, tail)) +} + +/// Cast a mutable byte slice to a `Pod` type. +/// +/// Returns the type and the tail of the slice. +#[inline] +pub fn from_bytes_mut(data: &mut [u8]) -> Result<(&mut T, &mut [u8])> { + let size = mem::size_of::(); + if size > data.len() { + return Err(()); + } + let (data, tail) = data.split_at_mut(size); + let ptr = data.as_mut_ptr(); + if (ptr as usize) % mem::align_of::() != 0 { + return Err(()); + } + // Safety: + // The alignment and size are checked by this function. + // The Pod trait ensures the type is valid to cast from bytes. + let val = unsafe { &mut *ptr.cast() }; + Ok((val, tail)) +} + +/// Cast a byte slice to a slice of a `Pod` type. +/// +/// Returns the type slice and the tail of the byte slice. +#[inline] +pub fn slice_from_bytes(data: &[u8], count: usize) -> Result<(&[T], &[u8])> { + let size = count.checked_mul(mem::size_of::()).ok_or(())?; + let tail = data.get(size..).ok_or(())?; + let ptr = data.as_ptr(); + if (ptr as usize) % mem::align_of::() != 0 { + return Err(()); + } + // Safety: + // The alignment and size are checked by this function. + // The Pod trait ensures the type is valid to cast from bytes. + let slice = unsafe { slice::from_raw_parts(ptr.cast(), count) }; + Ok((slice, tail)) +} + +/// Cast a mutable byte slice to a slice of a `Pod` type. +/// +/// Returns the type slice and the tail of the byte slice. +#[inline] +pub fn slice_from_bytes_mut( + data: &mut [u8], + count: usize, +) -> Result<(&mut [T], &mut [u8])> { + let size = count.checked_mul(mem::size_of::()).ok_or(())?; + if size > data.len() { + return Err(()); + } + let (data, tail) = data.split_at_mut(size); + let ptr = data.as_mut_ptr(); + if (ptr as usize) % mem::align_of::() != 0 { + return Err(()); + } + // Safety: + // The alignment and size are checked by this function. + // The Pod trait ensures the type is valid to cast from bytes. + let slice = unsafe { slice::from_raw_parts_mut(ptr.cast(), count) }; + Ok((slice, tail)) +} + +/// Cast a `Pod` type to a byte slice. +#[inline] +pub fn bytes_of(val: &T) -> &[u8] { + let size = mem::size_of::(); + // Safety: + // Any alignment is allowed. + // The size is determined in this function. + // The Pod trait ensures the type is valid to cast to bytes. + unsafe { slice::from_raw_parts(slice::from_ref(val).as_ptr().cast(), size) } +} + +/// Cast a `Pod` type to a mutable byte slice. +#[inline] +pub fn bytes_of_mut(val: &mut T) -> &mut [u8] { + let size = mem::size_of::(); + // Safety: + // Any alignment is allowed. + // The size is determined in this function. + // The Pod trait ensures the type is valid to cast to bytes. + unsafe { slice::from_raw_parts_mut(slice::from_mut(val).as_mut_ptr().cast(), size) } +} + +/// Cast a slice of a `Pod` type to a byte slice. +#[inline] +pub fn bytes_of_slice(val: &[T]) -> &[u8] { + let size = val.len().wrapping_mul(mem::size_of::()); + // Safety: + // Any alignment is allowed. + // The size is determined in this function. + // The Pod trait ensures the type is valid to cast to bytes. + unsafe { slice::from_raw_parts(val.as_ptr().cast(), size) } +} + +/// Cast a slice of a `Pod` type to a mutable byte slice. +#[inline] +pub fn bytes_of_slice_mut(val: &mut [T]) -> &mut [u8] { + let size = val.len().wrapping_mul(mem::size_of::()); + // Safety: + // Any alignment is allowed. + // The size is determined in this function. + // The Pod trait ensures the type is valid to cast to bytes. + unsafe { slice::from_raw_parts_mut(val.as_mut_ptr().cast(), size) } +} + +macro_rules! unsafe_impl_pod { + ($($struct_name:ident),+ $(,)?) => { + $( + unsafe impl Pod for $struct_name { } + )+ + } +} + +unsafe_impl_pod!(u8, u16, u32, u64); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn single() { + let x = u32::to_be(0x0123_4567); + let mut x_mut = x; + let bytes = bytes_of(&x); + let bytes_mut = bytes_of_mut(&mut x_mut); + assert_eq!(bytes, [0x01, 0x23, 0x45, 0x67]); + assert_eq!(bytes, bytes_mut); + + let x16 = [u16::to_be(0x0123), u16::to_be(0x4567)]; + + let (y, tail) = from_bytes::(bytes).unwrap(); + let (y_mut, tail_mut) = from_bytes_mut::(bytes_mut).unwrap(); + assert_eq!(*y, x); + assert_eq!(y, y_mut); + assert_eq!(tail, &[]); + assert_eq!(tail, tail_mut); + + let (y, tail) = from_bytes::(bytes).unwrap(); + let (y_mut, tail_mut) = from_bytes_mut::(bytes_mut).unwrap(); + assert_eq!(*y, x16[0]); + assert_eq!(y, y_mut); + assert_eq!(tail, &bytes[2..]); + assert_eq!(tail, tail_mut); + + let (y, tail) = from_bytes::(&bytes[2..]).unwrap(); + let (y_mut, tail_mut) = from_bytes_mut::(&mut bytes_mut[2..]).unwrap(); + assert_eq!(*y, x16[1]); + assert_eq!(y, y_mut); + assert_eq!(tail, &[]); + assert_eq!(tail, tail_mut); + + assert_eq!(from_bytes::(&bytes[1..]), Err(())); + assert_eq!(from_bytes::(&bytes[3..]), Err(())); + assert_eq!(from_bytes::(&bytes[4..]), Err(())); + assert_eq!(from_bytes_mut::(&mut bytes_mut[1..]), Err(())); + assert_eq!(from_bytes_mut::(&mut bytes_mut[3..]), Err(())); + assert_eq!(from_bytes_mut::(&mut bytes_mut[4..]), Err(())); + } + + #[test] + fn slice() { + let x = [ + u16::to_be(0x0123), + u16::to_be(0x4567), + u16::to_be(0x89ab), + u16::to_be(0xcdef), + ]; + let mut x_mut = x; + + let bytes = bytes_of_slice(&x); + let bytes_mut = bytes_of_slice_mut(&mut x_mut); + assert_eq!(bytes, [0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]); + assert_eq!(bytes, bytes_mut); + + let (y, tail) = slice_from_bytes::(bytes, 4).unwrap(); + let (y_mut, tail_mut) = slice_from_bytes_mut::(bytes_mut, 4).unwrap(); + assert_eq!(y, x); + assert_eq!(y, y_mut); + assert_eq!(tail, &[]); + assert_eq!(tail, tail_mut); + + let (y, tail) = slice_from_bytes::(&bytes[2..], 2).unwrap(); + let (y_mut, tail_mut) = slice_from_bytes::(&mut bytes_mut[2..], 2).unwrap(); + assert_eq!(y, &x[1..3]); + assert_eq!(y, y_mut); + assert_eq!(tail, &bytes[6..]); + assert_eq!(tail, tail_mut); + + assert_eq!(slice_from_bytes::(bytes, 5), Err(())); + assert_eq!(slice_from_bytes::(&bytes[2..], 4), Err(())); + assert_eq!(slice_from_bytes::(&bytes[1..], 2), Err(())); + assert_eq!(slice_from_bytes_mut::(bytes_mut, 5), Err(())); + assert_eq!(slice_from_bytes_mut::(&mut bytes_mut[2..], 4), Err(())); + assert_eq!(slice_from_bytes_mut::(&mut bytes_mut[1..], 2), Err(())); + } +} diff --git a/crux-mir/lib/object/src/read/any.rs b/crux-mir/lib/object/src/read/any.rs new file mode 100644 index 000000000..02e76dcdd --- /dev/null +++ b/crux-mir/lib/object/src/read/any.rs @@ -0,0 +1,1251 @@ +use alloc::fmt; +use alloc::vec::Vec; +use core::marker::PhantomData; + +#[cfg(feature = "coff")] +use crate::read::coff; +#[cfg(feature = "elf")] +use crate::read::elf; +#[cfg(feature = "macho")] +use crate::read::macho; +#[cfg(feature = "pe")] +use crate::read::pe; +#[cfg(feature = "wasm")] +use crate::read::wasm; +use crate::read::{ + self, Architecture, BinaryFormat, CodeView, ComdatKind, CompressedData, CompressedFileRange, + Error, Export, FileFlags, FileKind, Import, Object, ObjectComdat, ObjectKind, ObjectMap, + ObjectSection, ObjectSegment, ObjectSymbol, ObjectSymbolTable, ReadRef, Relocation, Result, + SectionFlags, SectionIndex, SectionKind, SegmentFlags, SymbolFlags, SymbolIndex, SymbolKind, + SymbolMap, SymbolMapName, SymbolScope, SymbolSection, +}; +#[allow(unused_imports)] +use crate::{AddressSize, Endian, Endianness}; + +/// Evaluate an expression on the contents of a file format enum. +/// +/// This is a hack to avoid virtual calls. +macro_rules! with_inner { + ($inner:expr, $enum:ident, | $var:ident | $body:expr) => { + match $inner { + #[cfg(feature = "coff")] + $enum::Coff(ref $var) => $body, + #[cfg(feature = "elf")] + $enum::Elf32(ref $var) => $body, + #[cfg(feature = "elf")] + $enum::Elf64(ref $var) => $body, + #[cfg(feature = "macho")] + $enum::MachO32(ref $var) => $body, + #[cfg(feature = "macho")] + $enum::MachO64(ref $var) => $body, + #[cfg(feature = "pe")] + $enum::Pe32(ref $var) => $body, + #[cfg(feature = "pe")] + $enum::Pe64(ref $var) => $body, + #[cfg(feature = "wasm")] + $enum::Wasm(ref $var) => $body, + } + }; +} + +macro_rules! with_inner_mut { + ($inner:expr, $enum:ident, | $var:ident | $body:expr) => { + match $inner { + #[cfg(feature = "coff")] + $enum::Coff(ref mut $var) => $body, + #[cfg(feature = "elf")] + $enum::Elf32(ref mut $var) => $body, + #[cfg(feature = "elf")] + $enum::Elf64(ref mut $var) => $body, + #[cfg(feature = "macho")] + $enum::MachO32(ref mut $var) => $body, + #[cfg(feature = "macho")] + $enum::MachO64(ref mut $var) => $body, + #[cfg(feature = "pe")] + $enum::Pe32(ref mut $var) => $body, + #[cfg(feature = "pe")] + $enum::Pe64(ref mut $var) => $body, + #[cfg(feature = "wasm")] + $enum::Wasm(ref mut $var) => $body, + } + }; +} + +/// Like `with_inner!`, but wraps the result in another enum. +macro_rules! map_inner { + ($inner:expr, $from:ident, $to:ident, | $var:ident | $body:expr) => { + match $inner { + #[cfg(feature = "coff")] + $from::Coff(ref $var) => $to::Coff($body), + #[cfg(feature = "elf")] + $from::Elf32(ref $var) => $to::Elf32($body), + #[cfg(feature = "elf")] + $from::Elf64(ref $var) => $to::Elf64($body), + #[cfg(feature = "macho")] + $from::MachO32(ref $var) => $to::MachO32($body), + #[cfg(feature = "macho")] + $from::MachO64(ref $var) => $to::MachO64($body), + #[cfg(feature = "pe")] + $from::Pe32(ref $var) => $to::Pe32($body), + #[cfg(feature = "pe")] + $from::Pe64(ref $var) => $to::Pe64($body), + #[cfg(feature = "wasm")] + $from::Wasm(ref $var) => $to::Wasm($body), + } + }; +} + +/// Like `map_inner!`, but the result is a Result or Option. +macro_rules! map_inner_option { + ($inner:expr, $from:ident, $to:ident, | $var:ident | $body:expr) => { + match $inner { + #[cfg(feature = "coff")] + $from::Coff(ref $var) => $body.map($to::Coff), + #[cfg(feature = "elf")] + $from::Elf32(ref $var) => $body.map($to::Elf32), + #[cfg(feature = "elf")] + $from::Elf64(ref $var) => $body.map($to::Elf64), + #[cfg(feature = "macho")] + $from::MachO32(ref $var) => $body.map($to::MachO32), + #[cfg(feature = "macho")] + $from::MachO64(ref $var) => $body.map($to::MachO64), + #[cfg(feature = "pe")] + $from::Pe32(ref $var) => $body.map($to::Pe32), + #[cfg(feature = "pe")] + $from::Pe64(ref $var) => $body.map($to::Pe64), + #[cfg(feature = "wasm")] + $from::Wasm(ref $var) => $body.map($to::Wasm), + } + }; +} + +macro_rules! map_inner_option_mut { + ($inner:expr, $from:ident, $to:ident, | $var:ident | $body:expr) => { + match $inner { + #[cfg(feature = "coff")] + $from::Coff(ref mut $var) => $body.map($to::Coff), + #[cfg(feature = "elf")] + $from::Elf32(ref mut $var) => $body.map($to::Elf32), + #[cfg(feature = "elf")] + $from::Elf64(ref mut $var) => $body.map($to::Elf64), + #[cfg(feature = "macho")] + $from::MachO32(ref mut $var) => $body.map($to::MachO32), + #[cfg(feature = "macho")] + $from::MachO64(ref mut $var) => $body.map($to::MachO64), + #[cfg(feature = "pe")] + $from::Pe32(ref mut $var) => $body.map($to::Pe32), + #[cfg(feature = "pe")] + $from::Pe64(ref mut $var) => $body.map($to::Pe64), + #[cfg(feature = "wasm")] + $from::Wasm(ref mut $var) => $body.map($to::Wasm), + } + }; +} + +/// Call `next` for a file format iterator. +macro_rules! next_inner { + ($inner:expr, $from:ident, $to:ident) => { + match $inner { + #[cfg(feature = "coff")] + $from::Coff(ref mut iter) => iter.next().map($to::Coff), + #[cfg(feature = "elf")] + $from::Elf32(ref mut iter) => iter.next().map($to::Elf32), + #[cfg(feature = "elf")] + $from::Elf64(ref mut iter) => iter.next().map($to::Elf64), + #[cfg(feature = "macho")] + $from::MachO32(ref mut iter) => iter.next().map($to::MachO32), + #[cfg(feature = "macho")] + $from::MachO64(ref mut iter) => iter.next().map($to::MachO64), + #[cfg(feature = "pe")] + $from::Pe32(ref mut iter) => iter.next().map($to::Pe32), + #[cfg(feature = "pe")] + $from::Pe64(ref mut iter) => iter.next().map($to::Pe64), + #[cfg(feature = "wasm")] + $from::Wasm(ref mut iter) => iter.next().map($to::Wasm), + } + }; +} + +/// An object file. +/// +/// Most functionality is provided by the `Object` trait implementation. +#[derive(Debug)] +pub struct File<'data, R: ReadRef<'data> = &'data [u8]> { + inner: FileInternal<'data, R>, +} + +#[derive(Debug)] +enum FileInternal<'data, R: ReadRef<'data>> { + #[cfg(feature = "coff")] + Coff(coff::CoffFile<'data, R>), + #[cfg(feature = "elf")] + Elf32(elf::ElfFile32<'data, Endianness, R>), + #[cfg(feature = "elf")] + Elf64(elf::ElfFile64<'data, Endianness, R>), + #[cfg(feature = "macho")] + MachO32(macho::MachOFile32<'data, Endianness, R>), + #[cfg(feature = "macho")] + MachO64(macho::MachOFile64<'data, Endianness, R>), + #[cfg(feature = "pe")] + Pe32(pe::PeFile32<'data, R>), + #[cfg(feature = "pe")] + Pe64(pe::PeFile64<'data, R>), + #[cfg(feature = "wasm")] + Wasm(wasm::WasmFile<'data, R>), +} + +impl<'data, R: ReadRef<'data>> File<'data, R> { + /// Parse the raw file data. + pub fn parse(data: R) -> Result { + let inner = match FileKind::parse(data)? { + #[cfg(feature = "elf")] + FileKind::Elf32 => FileInternal::Elf32(elf::ElfFile32::parse(data)?), + #[cfg(feature = "elf")] + FileKind::Elf64 => FileInternal::Elf64(elf::ElfFile64::parse(data)?), + #[cfg(feature = "macho")] + FileKind::MachO32 => FileInternal::MachO32(macho::MachOFile32::parse(data)?), + #[cfg(feature = "macho")] + FileKind::MachO64 => FileInternal::MachO64(macho::MachOFile64::parse(data)?), + #[cfg(feature = "wasm")] + FileKind::Wasm => FileInternal::Wasm(wasm::WasmFile::parse(data)?), + #[cfg(feature = "pe")] + FileKind::Pe32 => FileInternal::Pe32(pe::PeFile32::parse(data)?), + #[cfg(feature = "pe")] + FileKind::Pe64 => FileInternal::Pe64(pe::PeFile64::parse(data)?), + #[cfg(feature = "coff")] + FileKind::Coff => FileInternal::Coff(coff::CoffFile::parse(data)?), + #[allow(unreachable_patterns)] + _ => return Err(Error("Unsupported file format")), + }; + Ok(File { inner }) + } + + /// Parse a Mach-O image from the dyld shared cache. + #[cfg(feature = "macho")] + pub fn parse_dyld_cache_image<'cache, E: Endian>( + image: &macho::DyldCacheImage<'data, 'cache, E, R>, + ) -> Result { + let inner = match image.cache.architecture().address_size() { + Some(AddressSize::U64) => { + FileInternal::MachO64(macho::MachOFile64::parse_dyld_cache_image(image)?) + } + Some(AddressSize::U32) => { + FileInternal::MachO32(macho::MachOFile32::parse_dyld_cache_image(image)?) + } + _ => return Err(Error("Unsupported file format")), + }; + Ok(File { inner }) + } + + /// Return the file format. + pub fn format(&self) -> BinaryFormat { + match self.inner { + #[cfg(feature = "coff")] + FileInternal::Coff(_) => BinaryFormat::Coff, + #[cfg(feature = "elf")] + FileInternal::Elf32(_) | FileInternal::Elf64(_) => BinaryFormat::Elf, + #[cfg(feature = "macho")] + FileInternal::MachO32(_) | FileInternal::MachO64(_) => BinaryFormat::MachO, + #[cfg(feature = "pe")] + FileInternal::Pe32(_) | FileInternal::Pe64(_) => BinaryFormat::Pe, + #[cfg(feature = "wasm")] + FileInternal::Wasm(_) => BinaryFormat::Wasm, + } + } +} + +impl<'data, R: ReadRef<'data>> read::private::Sealed for File<'data, R> {} + +impl<'data, 'file, R> Object<'data, 'file> for File<'data, R> +where + 'data: 'file, + R: 'file + ReadRef<'data>, +{ + type Segment = Segment<'data, 'file, R>; + type SegmentIterator = SegmentIterator<'data, 'file, R>; + type Section = Section<'data, 'file, R>; + type SectionIterator = SectionIterator<'data, 'file, R>; + type Comdat = Comdat<'data, 'file, R>; + type ComdatIterator = ComdatIterator<'data, 'file, R>; + type Symbol = Symbol<'data, 'file, R>; + type SymbolIterator = SymbolIterator<'data, 'file, R>; + type SymbolTable = SymbolTable<'data, 'file, R>; + type DynamicRelocationIterator = DynamicRelocationIterator<'data, 'file, R>; + + fn architecture(&self) -> Architecture { + with_inner!(self.inner, FileInternal, |x| x.architecture()) + } + + fn is_little_endian(&self) -> bool { + with_inner!(self.inner, FileInternal, |x| x.is_little_endian()) + } + + fn is_64(&self) -> bool { + with_inner!(self.inner, FileInternal, |x| x.is_64()) + } + + fn kind(&self) -> ObjectKind { + with_inner!(self.inner, FileInternal, |x| x.kind()) + } + + fn segments(&'file self) -> SegmentIterator<'data, 'file, R> { + SegmentIterator { + inner: map_inner!(self.inner, FileInternal, SegmentIteratorInternal, |x| x + .segments()), + } + } + + fn section_by_name_bytes(&'file self, section_name: &[u8]) -> Option> { + map_inner_option!(self.inner, FileInternal, SectionInternal, |x| x + .section_by_name_bytes(section_name)) + .map(|inner| Section { inner }) + } + + fn section_by_index(&'file self, index: SectionIndex) -> Result> { + map_inner_option!(self.inner, FileInternal, SectionInternal, |x| x + .section_by_index(index)) + .map(|inner| Section { inner }) + } + + fn sections(&'file self) -> SectionIterator<'data, 'file, R> { + SectionIterator { + inner: map_inner!(self.inner, FileInternal, SectionIteratorInternal, |x| x + .sections()), + } + } + + fn comdats(&'file self) -> ComdatIterator<'data, 'file, R> { + ComdatIterator { + inner: map_inner!(self.inner, FileInternal, ComdatIteratorInternal, |x| x + .comdats()), + } + } + + fn symbol_by_index(&'file self, index: SymbolIndex) -> Result> { + map_inner_option!(self.inner, FileInternal, SymbolInternal, |x| x + .symbol_by_index(index) + .map(|x| (x, PhantomData))) + .map(|inner| Symbol { inner }) + } + + fn symbols(&'file self) -> SymbolIterator<'data, 'file, R> { + SymbolIterator { + inner: map_inner!(self.inner, FileInternal, SymbolIteratorInternal, |x| ( + x.symbols(), + PhantomData + )), + } + } + + fn symbol_table(&'file self) -> Option> { + map_inner_option!(self.inner, FileInternal, SymbolTableInternal, |x| x + .symbol_table() + .map(|x| (x, PhantomData))) + .map(|inner| SymbolTable { inner }) + } + + fn dynamic_symbols(&'file self) -> SymbolIterator<'data, 'file, R> { + SymbolIterator { + inner: map_inner!(self.inner, FileInternal, SymbolIteratorInternal, |x| ( + x.dynamic_symbols(), + PhantomData + )), + } + } + + fn dynamic_symbol_table(&'file self) -> Option> { + map_inner_option!(self.inner, FileInternal, SymbolTableInternal, |x| x + .dynamic_symbol_table() + .map(|x| (x, PhantomData))) + .map(|inner| SymbolTable { inner }) + } + + #[cfg(feature = "elf")] + fn dynamic_relocations(&'file self) -> Option> { + let inner = match self.inner { + FileInternal::Elf32(ref elf) => { + DynamicRelocationIteratorInternal::Elf32(elf.dynamic_relocations()?) + } + FileInternal::Elf64(ref elf) => { + DynamicRelocationIteratorInternal::Elf64(elf.dynamic_relocations()?) + } + #[allow(unreachable_patterns)] + _ => return None, + }; + Some(DynamicRelocationIterator { inner }) + } + + #[cfg(not(feature = "elf"))] + fn dynamic_relocations(&'file self) -> Option> { + None + } + + fn symbol_map(&self) -> SymbolMap> { + with_inner!(self.inner, FileInternal, |x| x.symbol_map()) + } + + fn object_map(&self) -> ObjectMap<'data> { + with_inner!(self.inner, FileInternal, |x| x.object_map()) + } + + fn imports(&self) -> Result>> { + with_inner!(self.inner, FileInternal, |x| x.imports()) + } + + fn exports(&self) -> Result>> { + with_inner!(self.inner, FileInternal, |x| x.exports()) + } + + fn has_debug_symbols(&self) -> bool { + with_inner!(self.inner, FileInternal, |x| x.has_debug_symbols()) + } + + #[inline] + fn mach_uuid(&self) -> Result> { + with_inner!(self.inner, FileInternal, |x| x.mach_uuid()) + } + + #[inline] + fn build_id(&self) -> Result> { + with_inner!(self.inner, FileInternal, |x| x.build_id()) + } + + #[inline] + fn gnu_debuglink(&self) -> Result> { + with_inner!(self.inner, FileInternal, |x| x.gnu_debuglink()) + } + + #[inline] + fn gnu_debugaltlink(&self) -> Result> { + with_inner!(self.inner, FileInternal, |x| x.gnu_debugaltlink()) + } + + #[inline] + fn pdb_info(&self) -> Result> { + with_inner!(self.inner, FileInternal, |x| x.pdb_info()) + } + + fn relative_address_base(&self) -> u64 { + with_inner!(self.inner, FileInternal, |x| x.relative_address_base()) + } + + fn entry(&self) -> u64 { + with_inner!(self.inner, FileInternal, |x| x.entry()) + } + + fn flags(&self) -> FileFlags { + with_inner!(self.inner, FileInternal, |x| x.flags()) + } +} + +/// An iterator over the segments of a `File`. +#[derive(Debug)] +pub struct SegmentIterator<'data, 'file, R: ReadRef<'data> = &'data [u8]> +where + 'data: 'file, +{ + inner: SegmentIteratorInternal<'data, 'file, R>, +} + +#[derive(Debug)] +enum SegmentIteratorInternal<'data, 'file, R: ReadRef<'data>> +where + 'data: 'file, +{ + #[cfg(feature = "coff")] + Coff(coff::CoffSegmentIterator<'data, 'file, R>), + #[cfg(feature = "elf")] + Elf32(elf::ElfSegmentIterator32<'data, 'file, Endianness, R>), + #[cfg(feature = "elf")] + Elf64(elf::ElfSegmentIterator64<'data, 'file, Endianness, R>), + #[cfg(feature = "macho")] + MachO32(macho::MachOSegmentIterator32<'data, 'file, Endianness, R>), + #[cfg(feature = "macho")] + MachO64(macho::MachOSegmentIterator64<'data, 'file, Endianness, R>), + #[cfg(feature = "pe")] + Pe32(pe::PeSegmentIterator32<'data, 'file, R>), + #[cfg(feature = "pe")] + Pe64(pe::PeSegmentIterator64<'data, 'file, R>), + #[cfg(feature = "wasm")] + Wasm(wasm::WasmSegmentIterator<'data, 'file, R>), +} + +impl<'data, 'file, R: ReadRef<'data>> Iterator for SegmentIterator<'data, 'file, R> { + type Item = Segment<'data, 'file, R>; + + fn next(&mut self) -> Option { + next_inner!(self.inner, SegmentIteratorInternal, SegmentInternal) + .map(|inner| Segment { inner }) + } +} + +/// A segment of a `File`. +pub struct Segment<'data, 'file, R: ReadRef<'data> = &'data [u8]> +where + 'data: 'file, +{ + inner: SegmentInternal<'data, 'file, R>, +} + +#[derive(Debug)] +enum SegmentInternal<'data, 'file, R: ReadRef<'data>> +where + 'data: 'file, +{ + #[cfg(feature = "coff")] + Coff(coff::CoffSegment<'data, 'file, R>), + #[cfg(feature = "elf")] + Elf32(elf::ElfSegment32<'data, 'file, Endianness, R>), + #[cfg(feature = "elf")] + Elf64(elf::ElfSegment64<'data, 'file, Endianness, R>), + #[cfg(feature = "macho")] + MachO32(macho::MachOSegment32<'data, 'file, Endianness, R>), + #[cfg(feature = "macho")] + MachO64(macho::MachOSegment64<'data, 'file, Endianness, R>), + #[cfg(feature = "pe")] + Pe32(pe::PeSegment32<'data, 'file, R>), + #[cfg(feature = "pe")] + Pe64(pe::PeSegment64<'data, 'file, R>), + #[cfg(feature = "wasm")] + Wasm(wasm::WasmSegment<'data, 'file, R>), +} + +impl<'data, 'file, R: ReadRef<'data>> fmt::Debug for Segment<'data, 'file, R> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // It's painful to do much better than this + let mut s = f.debug_struct("Segment"); + match self.name() { + Ok(Some(ref name)) => { + s.field("name", name); + } + Ok(None) => {} + Err(_) => { + s.field("name", &""); + } + } + s.field("address", &self.address()) + .field("size", &self.size()) + .finish() + } +} + +impl<'data, 'file, R: ReadRef<'data>> read::private::Sealed for Segment<'data, 'file, R> {} + +impl<'data, 'file, R: ReadRef<'data>> ObjectSegment<'data> for Segment<'data, 'file, R> { + fn address(&self) -> u64 { + with_inner!(self.inner, SegmentInternal, |x| x.address()) + } + + fn size(&self) -> u64 { + with_inner!(self.inner, SegmentInternal, |x| x.size()) + } + + fn align(&self) -> u64 { + with_inner!(self.inner, SegmentInternal, |x| x.align()) + } + + fn file_range(&self) -> (u64, u64) { + with_inner!(self.inner, SegmentInternal, |x| x.file_range()) + } + + fn data(&self) -> Result<&'data [u8]> { + with_inner!(self.inner, SegmentInternal, |x| x.data()) + } + + fn data_range(&self, address: u64, size: u64) -> Result> { + with_inner!(self.inner, SegmentInternal, |x| x.data_range(address, size)) + } + + fn name_bytes(&self) -> Result> { + with_inner!(self.inner, SegmentInternal, |x| x.name_bytes()) + } + + fn name(&self) -> Result> { + with_inner!(self.inner, SegmentInternal, |x| x.name()) + } + + fn flags(&self) -> SegmentFlags { + with_inner!(self.inner, SegmentInternal, |x| x.flags()) + } +} + +/// An iterator of the sections of a `File`. +#[derive(Debug)] +pub struct SectionIterator<'data, 'file, R: ReadRef<'data> = &'data [u8]> +where + 'data: 'file, +{ + inner: SectionIteratorInternal<'data, 'file, R>, +} + +// we wrap our enums in a struct so that they are kept private. +#[derive(Debug)] +enum SectionIteratorInternal<'data, 'file, R: ReadRef<'data>> +where + 'data: 'file, +{ + #[cfg(feature = "coff")] + Coff(coff::CoffSectionIterator<'data, 'file, R>), + #[cfg(feature = "elf")] + Elf32(elf::ElfSectionIterator32<'data, 'file, Endianness, R>), + #[cfg(feature = "elf")] + Elf64(elf::ElfSectionIterator64<'data, 'file, Endianness, R>), + #[cfg(feature = "macho")] + MachO32(macho::MachOSectionIterator32<'data, 'file, Endianness, R>), + #[cfg(feature = "macho")] + MachO64(macho::MachOSectionIterator64<'data, 'file, Endianness, R>), + #[cfg(feature = "pe")] + Pe32(pe::PeSectionIterator32<'data, 'file, R>), + #[cfg(feature = "pe")] + Pe64(pe::PeSectionIterator64<'data, 'file, R>), + #[cfg(feature = "wasm")] + Wasm(wasm::WasmSectionIterator<'data, 'file, R>), +} + +impl<'data, 'file, R: ReadRef<'data>> Iterator for SectionIterator<'data, 'file, R> { + type Item = Section<'data, 'file, R>; + + fn next(&mut self) -> Option { + next_inner!(self.inner, SectionIteratorInternal, SectionInternal) + .map(|inner| Section { inner }) + } +} + +/// A Section of a File +pub struct Section<'data, 'file, R: ReadRef<'data> = &'data [u8]> +where + 'data: 'file, +{ + inner: SectionInternal<'data, 'file, R>, +} + +enum SectionInternal<'data, 'file, R: ReadRef<'data>> +where + 'data: 'file, +{ + #[cfg(feature = "coff")] + Coff(coff::CoffSection<'data, 'file, R>), + #[cfg(feature = "elf")] + Elf32(elf::ElfSection32<'data, 'file, Endianness, R>), + #[cfg(feature = "elf")] + Elf64(elf::ElfSection64<'data, 'file, Endianness, R>), + #[cfg(feature = "macho")] + MachO32(macho::MachOSection32<'data, 'file, Endianness, R>), + #[cfg(feature = "macho")] + MachO64(macho::MachOSection64<'data, 'file, Endianness, R>), + #[cfg(feature = "pe")] + Pe32(pe::PeSection32<'data, 'file, R>), + #[cfg(feature = "pe")] + Pe64(pe::PeSection64<'data, 'file, R>), + #[cfg(feature = "wasm")] + Wasm(wasm::WasmSection<'data, 'file, R>), +} + +impl<'data, 'file, R: ReadRef<'data>> fmt::Debug for Section<'data, 'file, R> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // It's painful to do much better than this + let mut s = f.debug_struct("Section"); + match self.segment_name() { + Ok(Some(ref name)) => { + s.field("segment", name); + } + Ok(None) => {} + Err(_) => { + s.field("segment", &""); + } + } + s.field("name", &self.name().unwrap_or("")) + .field("address", &self.address()) + .field("size", &self.size()) + .field("align", &self.align()) + .field("kind", &self.kind()) + .field("flags", &self.flags()) + .finish() + } +} + +impl<'data, 'file, R: ReadRef<'data>> read::private::Sealed for Section<'data, 'file, R> {} + +impl<'data, 'file, R: ReadRef<'data>> ObjectSection<'data> for Section<'data, 'file, R> { + type RelocationIterator = SectionRelocationIterator<'data, 'file, R>; + + fn index(&self) -> SectionIndex { + with_inner!(self.inner, SectionInternal, |x| x.index()) + } + + fn address(&self) -> u64 { + with_inner!(self.inner, SectionInternal, |x| x.address()) + } + + fn size(&self) -> u64 { + with_inner!(self.inner, SectionInternal, |x| x.size()) + } + + fn align(&self) -> u64 { + with_inner!(self.inner, SectionInternal, |x| x.align()) + } + + fn file_range(&self) -> Option<(u64, u64)> { + with_inner!(self.inner, SectionInternal, |x| x.file_range()) + } + + fn data(&self) -> Result<&'data [u8]> { + with_inner!(self.inner, SectionInternal, |x| x.data()) + } + + fn data_range(&self, address: u64, size: u64) -> Result> { + with_inner!(self.inner, SectionInternal, |x| x.data_range(address, size)) + } + + fn compressed_file_range(&self) -> Result { + with_inner!(self.inner, SectionInternal, |x| x.compressed_file_range()) + } + + fn compressed_data(&self) -> Result> { + with_inner!(self.inner, SectionInternal, |x| x.compressed_data()) + } + + fn name_bytes(&self) -> Result<&[u8]> { + with_inner!(self.inner, SectionInternal, |x| x.name_bytes()) + } + + fn name(&self) -> Result<&str> { + with_inner!(self.inner, SectionInternal, |x| x.name()) + } + + fn segment_name_bytes(&self) -> Result> { + with_inner!(self.inner, SectionInternal, |x| x.segment_name_bytes()) + } + + fn segment_name(&self) -> Result> { + with_inner!(self.inner, SectionInternal, |x| x.segment_name()) + } + + fn kind(&self) -> SectionKind { + with_inner!(self.inner, SectionInternal, |x| x.kind()) + } + + fn relocations(&self) -> SectionRelocationIterator<'data, 'file, R> { + SectionRelocationIterator { + inner: map_inner!( + self.inner, + SectionInternal, + SectionRelocationIteratorInternal, + |x| x.relocations() + ), + } + } + + fn flags(&self) -> SectionFlags { + with_inner!(self.inner, SectionInternal, |x| x.flags()) + } +} + +/// An iterator of the COMDAT section groups of a `File`. +#[derive(Debug)] +pub struct ComdatIterator<'data, 'file, R: ReadRef<'data> = &'data [u8]> +where + 'data: 'file, +{ + inner: ComdatIteratorInternal<'data, 'file, R>, +} + +#[derive(Debug)] +enum ComdatIteratorInternal<'data, 'file, R: ReadRef<'data>> +where + 'data: 'file, +{ + #[cfg(feature = "coff")] + Coff(coff::CoffComdatIterator<'data, 'file, R>), + #[cfg(feature = "elf")] + Elf32(elf::ElfComdatIterator32<'data, 'file, Endianness, R>), + #[cfg(feature = "elf")] + Elf64(elf::ElfComdatIterator64<'data, 'file, Endianness, R>), + #[cfg(feature = "macho")] + MachO32(macho::MachOComdatIterator32<'data, 'file, Endianness, R>), + #[cfg(feature = "macho")] + MachO64(macho::MachOComdatIterator64<'data, 'file, Endianness, R>), + #[cfg(feature = "pe")] + Pe32(pe::PeComdatIterator32<'data, 'file, R>), + #[cfg(feature = "pe")] + Pe64(pe::PeComdatIterator64<'data, 'file, R>), + #[cfg(feature = "wasm")] + Wasm(wasm::WasmComdatIterator<'data, 'file, R>), +} + +impl<'data, 'file, R: ReadRef<'data>> Iterator for ComdatIterator<'data, 'file, R> { + type Item = Comdat<'data, 'file, R>; + + fn next(&mut self) -> Option { + next_inner!(self.inner, ComdatIteratorInternal, ComdatInternal) + .map(|inner| Comdat { inner }) + } +} + +/// A COMDAT section group of a `File`. +pub struct Comdat<'data, 'file, R: ReadRef<'data> = &'data [u8]> +where + 'data: 'file, +{ + inner: ComdatInternal<'data, 'file, R>, +} + +enum ComdatInternal<'data, 'file, R: ReadRef<'data>> +where + 'data: 'file, +{ + #[cfg(feature = "coff")] + Coff(coff::CoffComdat<'data, 'file, R>), + #[cfg(feature = "elf")] + Elf32(elf::ElfComdat32<'data, 'file, Endianness, R>), + #[cfg(feature = "elf")] + Elf64(elf::ElfComdat64<'data, 'file, Endianness, R>), + #[cfg(feature = "macho")] + MachO32(macho::MachOComdat32<'data, 'file, Endianness, R>), + #[cfg(feature = "macho")] + MachO64(macho::MachOComdat64<'data, 'file, Endianness, R>), + #[cfg(feature = "pe")] + Pe32(pe::PeComdat32<'data, 'file, R>), + #[cfg(feature = "pe")] + Pe64(pe::PeComdat64<'data, 'file, R>), + #[cfg(feature = "wasm")] + Wasm(wasm::WasmComdat<'data, 'file, R>), +} + +impl<'data, 'file, R: ReadRef<'data>> fmt::Debug for Comdat<'data, 'file, R> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut s = f.debug_struct("Comdat"); + s.field("symbol", &self.symbol()) + .field("name", &self.name().unwrap_or("")) + .field("kind", &self.kind()) + .finish() + } +} + +impl<'data, 'file, R: ReadRef<'data>> read::private::Sealed for Comdat<'data, 'file, R> {} + +impl<'data, 'file, R: ReadRef<'data>> ObjectComdat<'data> for Comdat<'data, 'file, R> { + type SectionIterator = ComdatSectionIterator<'data, 'file, R>; + + fn kind(&self) -> ComdatKind { + with_inner!(self.inner, ComdatInternal, |x| x.kind()) + } + + fn symbol(&self) -> SymbolIndex { + with_inner!(self.inner, ComdatInternal, |x| x.symbol()) + } + + fn name_bytes(&self) -> Result<&[u8]> { + with_inner!(self.inner, ComdatInternal, |x| x.name_bytes()) + } + + fn name(&self) -> Result<&str> { + with_inner!(self.inner, ComdatInternal, |x| x.name()) + } + + fn sections(&self) -> ComdatSectionIterator<'data, 'file, R> { + ComdatSectionIterator { + inner: map_inner!( + self.inner, + ComdatInternal, + ComdatSectionIteratorInternal, + |x| x.sections() + ), + } + } +} + +/// An iterator over COMDAT section entries. +#[derive(Debug)] +pub struct ComdatSectionIterator<'data, 'file, R: ReadRef<'data> = &'data [u8]> +where + 'data: 'file, +{ + inner: ComdatSectionIteratorInternal<'data, 'file, R>, +} + +#[derive(Debug)] +enum ComdatSectionIteratorInternal<'data, 'file, R: ReadRef<'data>> +where + 'data: 'file, +{ + #[cfg(feature = "coff")] + Coff(coff::CoffComdatSectionIterator<'data, 'file, R>), + #[cfg(feature = "elf")] + Elf32(elf::ElfComdatSectionIterator32<'data, 'file, Endianness, R>), + #[cfg(feature = "elf")] + Elf64(elf::ElfComdatSectionIterator64<'data, 'file, Endianness, R>), + #[cfg(feature = "macho")] + MachO32(macho::MachOComdatSectionIterator32<'data, 'file, Endianness, R>), + #[cfg(feature = "macho")] + MachO64(macho::MachOComdatSectionIterator64<'data, 'file, Endianness, R>), + #[cfg(feature = "pe")] + Pe32(pe::PeComdatSectionIterator32<'data, 'file, R>), + #[cfg(feature = "pe")] + Pe64(pe::PeComdatSectionIterator64<'data, 'file, R>), + #[cfg(feature = "wasm")] + Wasm(wasm::WasmComdatSectionIterator<'data, 'file, R>), +} + +impl<'data, 'file, R: ReadRef<'data>> Iterator for ComdatSectionIterator<'data, 'file, R> { + type Item = SectionIndex; + + fn next(&mut self) -> Option { + with_inner_mut!(self.inner, ComdatSectionIteratorInternal, |x| x.next()) + } +} + +/// A symbol table. +#[derive(Debug)] +pub struct SymbolTable<'data, 'file, R = &'data [u8]> +where + 'data: 'file, + R: ReadRef<'data>, +{ + inner: SymbolTableInternal<'data, 'file, R>, +} + +#[derive(Debug)] +enum SymbolTableInternal<'data, 'file, R> +where + 'data: 'file, + R: ReadRef<'data>, +{ + #[cfg(feature = "coff")] + Coff((coff::CoffSymbolTable<'data, 'file, R>, PhantomData)), + #[cfg(feature = "elf")] + Elf32( + ( + elf::ElfSymbolTable32<'data, 'file, Endianness, R>, + PhantomData, + ), + ), + #[cfg(feature = "elf")] + Elf64( + ( + elf::ElfSymbolTable64<'data, 'file, Endianness, R>, + PhantomData, + ), + ), + #[cfg(feature = "macho")] + MachO32( + ( + macho::MachOSymbolTable32<'data, 'file, Endianness, R>, + PhantomData<()>, + ), + ), + #[cfg(feature = "macho")] + MachO64( + ( + macho::MachOSymbolTable64<'data, 'file, Endianness, R>, + PhantomData<()>, + ), + ), + #[cfg(feature = "pe")] + Pe32((coff::CoffSymbolTable<'data, 'file, R>, PhantomData)), + #[cfg(feature = "pe")] + Pe64((coff::CoffSymbolTable<'data, 'file, R>, PhantomData)), + #[cfg(feature = "wasm")] + Wasm((wasm::WasmSymbolTable<'data, 'file>, PhantomData)), +} + +impl<'data, 'file, R: ReadRef<'data>> read::private::Sealed for SymbolTable<'data, 'file, R> {} + +impl<'data, 'file, R: ReadRef<'data>> ObjectSymbolTable<'data> for SymbolTable<'data, 'file, R> { + type Symbol = Symbol<'data, 'file, R>; + type SymbolIterator = SymbolIterator<'data, 'file, R>; + + fn symbols(&self) -> Self::SymbolIterator { + SymbolIterator { + inner: map_inner!( + self.inner, + SymbolTableInternal, + SymbolIteratorInternal, + |x| (x.0.symbols(), PhantomData) + ), + } + } + + fn symbol_by_index(&self, index: SymbolIndex) -> Result { + map_inner_option!(self.inner, SymbolTableInternal, SymbolInternal, |x| x + .0 + .symbol_by_index(index) + .map(|x| (x, PhantomData))) + .map(|inner| Symbol { inner }) + } +} + +/// An iterator over symbol table entries. +#[derive(Debug)] +pub struct SymbolIterator<'data, 'file, R = &'data [u8]> +where + 'data: 'file, + R: ReadRef<'data>, +{ + inner: SymbolIteratorInternal<'data, 'file, R>, +} + +#[derive(Debug)] +enum SymbolIteratorInternal<'data, 'file, R> +where + 'data: 'file, + R: ReadRef<'data>, +{ + #[cfg(feature = "coff")] + Coff((coff::CoffSymbolIterator<'data, 'file, R>, PhantomData)), + #[cfg(feature = "elf")] + Elf32( + ( + elf::ElfSymbolIterator32<'data, 'file, Endianness, R>, + PhantomData, + ), + ), + #[cfg(feature = "elf")] + Elf64( + ( + elf::ElfSymbolIterator64<'data, 'file, Endianness, R>, + PhantomData, + ), + ), + #[cfg(feature = "macho")] + MachO32( + ( + macho::MachOSymbolIterator32<'data, 'file, Endianness, R>, + PhantomData<()>, + ), + ), + #[cfg(feature = "macho")] + MachO64( + ( + macho::MachOSymbolIterator64<'data, 'file, Endianness, R>, + PhantomData<()>, + ), + ), + #[cfg(feature = "pe")] + Pe32((coff::CoffSymbolIterator<'data, 'file, R>, PhantomData)), + #[cfg(feature = "pe")] + Pe64((coff::CoffSymbolIterator<'data, 'file, R>, PhantomData)), + #[cfg(feature = "wasm")] + Wasm((wasm::WasmSymbolIterator<'data, 'file>, PhantomData)), +} + +impl<'data, 'file, R: ReadRef<'data>> Iterator for SymbolIterator<'data, 'file, R> { + type Item = Symbol<'data, 'file, R>; + + fn next(&mut self) -> Option { + map_inner_option_mut!(self.inner, SymbolIteratorInternal, SymbolInternal, |iter| { + iter.0.next().map(|x| (x, PhantomData)) + }) + .map(|inner| Symbol { inner }) + } +} + +/// A symbol table entry. +pub struct Symbol<'data, 'file, R = &'data [u8]> +where + 'data: 'file, + R: ReadRef<'data>, +{ + inner: SymbolInternal<'data, 'file, R>, +} + +enum SymbolInternal<'data, 'file, R> +where + 'data: 'file, + R: ReadRef<'data>, +{ + #[cfg(feature = "coff")] + Coff((coff::CoffSymbol<'data, 'file, R>, PhantomData)), + #[cfg(feature = "elf")] + Elf32( + ( + elf::ElfSymbol32<'data, 'file, Endianness, R>, + PhantomData, + ), + ), + #[cfg(feature = "elf")] + Elf64( + ( + elf::ElfSymbol64<'data, 'file, Endianness, R>, + PhantomData, + ), + ), + #[cfg(feature = "macho")] + MachO32( + ( + macho::MachOSymbol32<'data, 'file, Endianness, R>, + PhantomData<()>, + ), + ), + #[cfg(feature = "macho")] + MachO64( + ( + macho::MachOSymbol64<'data, 'file, Endianness, R>, + PhantomData<()>, + ), + ), + #[cfg(feature = "pe")] + Pe32((coff::CoffSymbol<'data, 'file, R>, PhantomData)), + #[cfg(feature = "pe")] + Pe64((coff::CoffSymbol<'data, 'file, R>, PhantomData)), + #[cfg(feature = "wasm")] + Wasm((wasm::WasmSymbol<'data, 'file>, PhantomData)), +} + +impl<'data, 'file, R: ReadRef<'data>> fmt::Debug for Symbol<'data, 'file, R> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Symbol") + .field("name", &self.name().unwrap_or("")) + .field("address", &self.address()) + .field("size", &self.size()) + .field("kind", &self.kind()) + .field("section", &self.section()) + .field("scope", &self.scope()) + .field("weak", &self.is_weak()) + .field("flags", &self.flags()) + .finish() + } +} + +impl<'data, 'file, R: ReadRef<'data>> read::private::Sealed for Symbol<'data, 'file, R> {} + +impl<'data, 'file, R: ReadRef<'data>> ObjectSymbol<'data> for Symbol<'data, 'file, R> { + fn index(&self) -> SymbolIndex { + with_inner!(self.inner, SymbolInternal, |x| x.0.index()) + } + + fn name_bytes(&self) -> Result<&'data [u8]> { + with_inner!(self.inner, SymbolInternal, |x| x.0.name_bytes()) + } + + fn name(&self) -> Result<&'data str> { + with_inner!(self.inner, SymbolInternal, |x| x.0.name()) + } + + fn address(&self) -> u64 { + with_inner!(self.inner, SymbolInternal, |x| x.0.address()) + } + + fn size(&self) -> u64 { + with_inner!(self.inner, SymbolInternal, |x| x.0.size()) + } + + fn kind(&self) -> SymbolKind { + with_inner!(self.inner, SymbolInternal, |x| x.0.kind()) + } + + fn section(&self) -> SymbolSection { + with_inner!(self.inner, SymbolInternal, |x| x.0.section()) + } + + fn is_undefined(&self) -> bool { + with_inner!(self.inner, SymbolInternal, |x| x.0.is_undefined()) + } + + fn is_definition(&self) -> bool { + with_inner!(self.inner, SymbolInternal, |x| x.0.is_definition()) + } + + fn is_common(&self) -> bool { + with_inner!(self.inner, SymbolInternal, |x| x.0.is_common()) + } + + fn is_weak(&self) -> bool { + with_inner!(self.inner, SymbolInternal, |x| x.0.is_weak()) + } + + fn scope(&self) -> SymbolScope { + with_inner!(self.inner, SymbolInternal, |x| x.0.scope()) + } + + fn is_global(&self) -> bool { + with_inner!(self.inner, SymbolInternal, |x| x.0.is_global()) + } + + fn is_local(&self) -> bool { + with_inner!(self.inner, SymbolInternal, |x| x.0.is_local()) + } + + fn flags(&self) -> SymbolFlags { + with_inner!(self.inner, SymbolInternal, |x| x.0.flags()) + } +} + +/// An iterator over dynamic relocation entries. +#[derive(Debug)] +pub struct DynamicRelocationIterator<'data, 'file, R = &'data [u8]> +where + 'data: 'file, + R: ReadRef<'data>, +{ + inner: DynamicRelocationIteratorInternal<'data, 'file, R>, +} + +#[derive(Debug)] +enum DynamicRelocationIteratorInternal<'data, 'file, R> +where + 'data: 'file, + R: ReadRef<'data>, +{ + #[cfg(feature = "elf")] + Elf32(elf::ElfDynamicRelocationIterator32<'data, 'file, Endianness, R>), + #[cfg(feature = "elf")] + Elf64(elf::ElfDynamicRelocationIterator64<'data, 'file, Endianness, R>), + // We need to always use the lifetime parameters. + #[allow(unused)] + None(PhantomData<(&'data (), &'file (), R)>), +} + +impl<'data, 'file, R: ReadRef<'data>> Iterator for DynamicRelocationIterator<'data, 'file, R> { + type Item = (u64, Relocation); + + fn next(&mut self) -> Option { + match self.inner { + #[cfg(feature = "elf")] + DynamicRelocationIteratorInternal::Elf32(ref mut elf) => elf.next(), + #[cfg(feature = "elf")] + DynamicRelocationIteratorInternal::Elf64(ref mut elf) => elf.next(), + DynamicRelocationIteratorInternal::None(_) => None, + } + } +} + +/// An iterator over section relocation entries. +#[derive(Debug)] +pub struct SectionRelocationIterator<'data, 'file, R: ReadRef<'data> = &'data [u8]> +where + 'data: 'file, +{ + inner: SectionRelocationIteratorInternal<'data, 'file, R>, +} + +#[derive(Debug)] +enum SectionRelocationIteratorInternal<'data, 'file, R: ReadRef<'data>> +where + 'data: 'file, +{ + #[cfg(feature = "coff")] + Coff(coff::CoffRelocationIterator<'data, 'file, R>), + #[cfg(feature = "elf")] + Elf32(elf::ElfSectionRelocationIterator32<'data, 'file, Endianness, R>), + #[cfg(feature = "elf")] + Elf64(elf::ElfSectionRelocationIterator64<'data, 'file, Endianness, R>), + #[cfg(feature = "macho")] + MachO32(macho::MachORelocationIterator32<'data, 'file, Endianness, R>), + #[cfg(feature = "macho")] + MachO64(macho::MachORelocationIterator64<'data, 'file, Endianness, R>), + #[cfg(feature = "pe")] + Pe32(pe::PeRelocationIterator<'data, 'file, R>), + #[cfg(feature = "pe")] + Pe64(pe::PeRelocationIterator<'data, 'file, R>), + #[cfg(feature = "wasm")] + Wasm(wasm::WasmRelocationIterator<'data, 'file, R>), +} + +impl<'data, 'file, R: ReadRef<'data>> Iterator for SectionRelocationIterator<'data, 'file, R> { + type Item = (u64, Relocation); + + fn next(&mut self) -> Option { + with_inner_mut!(self.inner, SectionRelocationIteratorInternal, |x| x.next()) + } +} diff --git a/crux-mir/lib/object/src/read/archive.rs b/crux-mir/lib/object/src/read/archive.rs new file mode 100644 index 000000000..0208878e4 --- /dev/null +++ b/crux-mir/lib/object/src/read/archive.rs @@ -0,0 +1,502 @@ +//! Support for archive files. + +use core::convert::TryInto; + +use crate::archive; +use crate::read::{self, Error, ReadError, ReadRef}; + +/// The kind of archive format. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum ArchiveKind { + /// There are no special files that indicate the archive format. + Unknown, + /// The GNU (or System V) archive format. + Gnu, + /// The GNU (or System V) archive format with 64-bit symbol table. + Gnu64, + /// The BSD archive format. + Bsd, + /// The BSD archive format with 64-bit symbol table. + /// + /// This is used for Darwin. + Bsd64, + /// The Windows COFF archive format. + Coff, +} + +/// A partially parsed archive file. +#[derive(Debug)] +pub struct ArchiveFile<'data, R: ReadRef<'data> = &'data [u8]> { + data: R, + len: u64, + offset: u64, + kind: ArchiveKind, + symbols: (u64, u64), + names: &'data [u8], +} + +impl<'data, R: ReadRef<'data>> ArchiveFile<'data, R> { + /// Parse the archive header and special members. + pub fn parse(data: R) -> read::Result { + let len = data.len().read_error("Unknown archive length")?; + let mut tail = 0; + let magic = data + .read_bytes(&mut tail, archive::MAGIC.len() as u64) + .read_error("Invalid archive size")?; + if magic != &archive::MAGIC[..] { + return Err(Error("Unsupported archive identifier")); + } + + let mut file = ArchiveFile { + data, + offset: tail, + len, + kind: ArchiveKind::Unknown, + symbols: (0, 0), + names: &[], + }; + + // The first few members may be special, so parse them. + // GNU has: + // - "/" or "/SYM64/": symbol table (optional) + // - "//": names table (optional) + // COFF has: + // - "/": first linker member + // - "/": second linker member + // - "//": names table + // BSD has: + // - "__.SYMDEF" or "__.SYMDEF SORTED": symbol table (optional) + // BSD 64-bit has: + // - "__.SYMDEF_64" or "__.SYMDEF_64 SORTED": symbol table (optional) + // BSD may use the extended name for the symbol table. This is handled + // by `ArchiveMember::parse`. + if tail < len { + let member = ArchiveMember::parse(data, &mut tail, &[])?; + if member.name == b"/" { + // GNU symbol table (unless we later determine this is COFF). + file.kind = ArchiveKind::Gnu; + file.symbols = member.file_range(); + file.offset = tail; + + if tail < len { + let member = ArchiveMember::parse(data, &mut tail, &[])?; + if member.name == b"/" { + // COFF linker member. + file.kind = ArchiveKind::Coff; + file.symbols = member.file_range(); + file.offset = tail; + + if tail < len { + let member = ArchiveMember::parse(data, &mut tail, &[])?; + if member.name == b"//" { + // COFF names table. + file.names = member.data(data)?; + file.offset = tail; + } + } + } else if member.name == b"//" { + // GNU names table. + file.names = member.data(data)?; + file.offset = tail; + } + } + } else if member.name == b"/SYM64/" { + // GNU 64-bit symbol table. + file.kind = ArchiveKind::Gnu64; + file.symbols = member.file_range(); + file.offset = tail; + + if tail < len { + let member = ArchiveMember::parse(data, &mut tail, &[])?; + if member.name == b"//" { + // GNU names table. + file.names = member.data(data)?; + file.offset = tail; + } + } + } else if member.name == b"//" { + // GNU names table. + file.kind = ArchiveKind::Gnu; + file.names = member.data(data)?; + file.offset = tail; + } else if member.name == b"__.SYMDEF" || member.name == b"__.SYMDEF SORTED" { + // BSD symbol table. + file.kind = ArchiveKind::Bsd; + file.symbols = member.file_range(); + file.offset = tail; + } else if member.name == b"__.SYMDEF_64" || member.name == b"__.SYMDEF_64 SORTED" { + // BSD 64-bit symbol table. + file.kind = ArchiveKind::Bsd64; + file.symbols = member.file_range(); + file.offset = tail; + } else { + // TODO: This could still be a BSD file. We leave this as unknown for now. + } + } + Ok(file) + } + + /// Return the archive format. + #[inline] + pub fn kind(&self) -> ArchiveKind { + self.kind + } + + /// Iterate over the members of the archive. + /// + /// This does not return special members. + #[inline] + pub fn members(&self) -> ArchiveMemberIterator<'data, R> { + ArchiveMemberIterator { + data: self.data, + offset: self.offset, + len: self.len, + names: self.names, + } + } +} + +/// An iterator over the members of an archive. +#[derive(Debug)] +pub struct ArchiveMemberIterator<'data, R: ReadRef<'data> = &'data [u8]> { + data: R, + offset: u64, + len: u64, + names: &'data [u8], +} + +impl<'data, R: ReadRef<'data>> Iterator for ArchiveMemberIterator<'data, R> { + type Item = read::Result>; + + fn next(&mut self) -> Option { + if self.offset >= self.len { + return None; + } + let member = ArchiveMember::parse(self.data, &mut self.offset, self.names); + if member.is_err() { + self.offset = self.len; + } + Some(member) + } +} + +/// A partially parsed archive member. +#[derive(Debug)] +pub struct ArchiveMember<'data> { + header: &'data archive::Header, + name: &'data [u8], + offset: u64, + size: u64, +} + +impl<'data> ArchiveMember<'data> { + /// Parse the archive member header, name, and file data. + /// + /// This reads the extended name (if any) and adjusts the file size. + fn parse>( + data: R, + offset: &mut u64, + names: &'data [u8], + ) -> read::Result { + let header = data + .read::(offset) + .read_error("Invalid archive member header")?; + if header.terminator != archive::TERMINATOR { + return Err(Error("Invalid archive terminator")); + } + + let mut file_offset = *offset; + let mut file_size = + parse_u64_digits(&header.size, 10).read_error("Invalid archive member size")?; + *offset = offset + .checked_add(file_size) + .read_error("Archive member size is too large")?; + // Entries are padded to an even number of bytes. + if (file_size & 1) != 0 { + *offset = offset.saturating_add(1); + } + + let name = if header.name[0] == b'/' && (header.name[1] as char).is_digit(10) { + // Read file name from the names table. + parse_sysv_extended_name(&header.name[1..], names) + .read_error("Invalid archive extended name offset")? + } else if &header.name[..3] == b"#1/" && (header.name[3] as char).is_digit(10) { + // Read file name from the start of the file data. + parse_bsd_extended_name(&header.name[3..], data, &mut file_offset, &mut file_size) + .read_error("Invalid archive extended name length")? + } else if header.name[0] == b'/' { + let name_len = memchr::memchr(b' ', &header.name).unwrap_or(header.name.len()); + &header.name[..name_len] + } else { + let name_len = memchr::memchr(b'/', &header.name) + .or_else(|| memchr::memchr(b' ', &header.name)) + .unwrap_or(header.name.len()); + &header.name[..name_len] + }; + + Ok(ArchiveMember { + header, + name, + offset: file_offset, + size: file_size, + }) + } + + /// Return the raw header. + #[inline] + pub fn header(&self) -> &'data archive::Header { + self.header + } + + /// Return the parsed file name. + /// + /// This may be an extended file name. + #[inline] + pub fn name(&self) -> &'data [u8] { + self.name + } + + /// Parse the file modification timestamp from the header. + #[inline] + pub fn date(&self) -> Option { + parse_u64_digits(&self.header.date, 10) + } + + /// Parse the user ID from the header. + #[inline] + pub fn uid(&self) -> Option { + parse_u64_digits(&self.header.uid, 10) + } + + /// Parse the group ID from the header. + #[inline] + pub fn gid(&self) -> Option { + parse_u64_digits(&self.header.gid, 10) + } + + /// Parse the file mode from the header. + #[inline] + pub fn mode(&self) -> Option { + parse_u64_digits(&self.header.mode, 8) + } + + /// Return the offset and size of the file data. + pub fn file_range(&self) -> (u64, u64) { + (self.offset, self.size) + } + + /// Return the file data. + #[inline] + pub fn data>(&self, data: R) -> read::Result<&'data [u8]> { + data.read_bytes_at(self.offset, self.size) + .read_error("Archive member size is too large") + } +} + +// Ignores bytes starting from the first space. +fn parse_u64_digits(digits: &[u8], radix: u32) -> Option { + if let [b' ', ..] = digits { + return None; + } + let mut result: u64 = 0; + for &c in digits { + if c == b' ' { + return Some(result); + } else { + let x = (c as char).to_digit(radix)?; + result = result + .checked_mul(u64::from(radix))? + .checked_add(u64::from(x))?; + } + } + Some(result) +} + +fn parse_sysv_extended_name<'data>(digits: &[u8], names: &'data [u8]) -> Result<&'data [u8], ()> { + let offset = parse_u64_digits(digits, 10).ok_or(())?; + let offset = offset.try_into().map_err(|_| ())?; + let name_data = names.get(offset..).ok_or(())?; + let name = match memchr::memchr2(b'/', b'\0', name_data) { + Some(len) => &name_data[..len], + None => name_data, + }; + Ok(name) +} + +/// Modifies `data` to start after the extended name. +fn parse_bsd_extended_name<'data, R: ReadRef<'data>>( + digits: &[u8], + data: R, + offset: &mut u64, + size: &mut u64, +) -> Result<&'data [u8], ()> { + let len = parse_u64_digits(digits, 10).ok_or(())?; + *size = size.checked_sub(len).ok_or(())?; + let name_data = data.read_bytes(offset, len)?; + let name = match memchr::memchr(b'\0', name_data) { + Some(len) => &name_data[..len], + None => name_data, + }; + Ok(name) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn kind() { + let data = b"!\n"; + let archive = ArchiveFile::parse(&data[..]).unwrap(); + assert_eq!(archive.kind(), ArchiveKind::Unknown); + + let data = b"\ + !\n\ + / 4 `\n\ + 0000"; + let archive = ArchiveFile::parse(&data[..]).unwrap(); + assert_eq!(archive.kind(), ArchiveKind::Gnu); + + let data = b"\ + !\n\ + // 4 `\n\ + 0000"; + let archive = ArchiveFile::parse(&data[..]).unwrap(); + assert_eq!(archive.kind(), ArchiveKind::Gnu); + + let data = b"\ + !\n\ + / 4 `\n\ + 0000\ + // 4 `\n\ + 0000"; + let archive = ArchiveFile::parse(&data[..]).unwrap(); + assert_eq!(archive.kind(), ArchiveKind::Gnu); + + let data = b"\ + !\n\ + /SYM64/ 4 `\n\ + 0000"; + let archive = ArchiveFile::parse(&data[..]).unwrap(); + assert_eq!(archive.kind(), ArchiveKind::Gnu64); + + let data = b"\ + !\n\ + /SYM64/ 4 `\n\ + 0000\ + // 4 `\n\ + 0000"; + let archive = ArchiveFile::parse(&data[..]).unwrap(); + assert_eq!(archive.kind(), ArchiveKind::Gnu64); + + let data = b"\ + !\n\ + __.SYMDEF 4 `\n\ + 0000"; + let archive = ArchiveFile::parse(&data[..]).unwrap(); + assert_eq!(archive.kind(), ArchiveKind::Bsd); + + let data = b"\ + !\n\ + #1/9 13 `\n\ + __.SYMDEF0000"; + let archive = ArchiveFile::parse(&data[..]).unwrap(); + assert_eq!(archive.kind(), ArchiveKind::Bsd); + + let data = b"\ + !\n\ + #1/16 20 `\n\ + __.SYMDEF SORTED0000"; + let archive = ArchiveFile::parse(&data[..]).unwrap(); + assert_eq!(archive.kind(), ArchiveKind::Bsd); + + let data = b"\ + !\n\ + __.SYMDEF_64 4 `\n\ + 0000"; + let archive = ArchiveFile::parse(&data[..]).unwrap(); + assert_eq!(archive.kind(), ArchiveKind::Bsd64); + + let data = b"\ + !\n\ + #1/12 16 `\n\ + __.SYMDEF_640000"; + let archive = ArchiveFile::parse(&data[..]).unwrap(); + assert_eq!(archive.kind(), ArchiveKind::Bsd64); + + let data = b"\ + !\n\ + #1/19 23 `\n\ + __.SYMDEF_64 SORTED0000"; + let archive = ArchiveFile::parse(&data[..]).unwrap(); + assert_eq!(archive.kind(), ArchiveKind::Bsd64); + + let data = b"\ + !\n\ + / 4 `\n\ + 0000\ + / 4 `\n\ + 0000\ + // 4 `\n\ + 0000"; + let archive = ArchiveFile::parse(&data[..]).unwrap(); + assert_eq!(archive.kind(), ArchiveKind::Coff); + } + + #[test] + fn gnu_names() { + let data = b"\ + !\n\ + // 18 `\n\ + 0123456789abcdef/\n\ + s p a c e/ 0 0 0 644 4 `\n\ + 0000\ + 0123456789abcde/0 0 0 644 3 `\n\ + odd\n\ + /0 0 0 0 644 4 `\n\ + even"; + let data = &data[..]; + let archive = ArchiveFile::parse(data).unwrap(); + assert_eq!(archive.kind(), ArchiveKind::Gnu); + let mut members = archive.members(); + + let member = members.next().unwrap().unwrap(); + assert_eq!(member.name(), b"s p a c e"); + assert_eq!(member.data(data).unwrap(), &b"0000"[..]); + + let member = members.next().unwrap().unwrap(); + assert_eq!(member.name(), b"0123456789abcde"); + assert_eq!(member.data(data).unwrap(), &b"odd"[..]); + + let member = members.next().unwrap().unwrap(); + assert_eq!(member.name(), b"0123456789abcdef"); + assert_eq!(member.data(data).unwrap(), &b"even"[..]); + + assert!(members.next().is_none()); + } + + #[test] + fn bsd_names() { + let data = b"\ + !\n\ + 0123456789abcde 0 0 0 644 3 `\n\ + odd\n\ + #1/16 0 0 0 644 20 `\n\ + 0123456789abcdefeven"; + let data = &data[..]; + let archive = ArchiveFile::parse(data).unwrap(); + assert_eq!(archive.kind(), ArchiveKind::Unknown); + let mut members = archive.members(); + + let member = members.next().unwrap().unwrap(); + assert_eq!(member.name(), b"0123456789abcde"); + assert_eq!(member.data(data).unwrap(), &b"odd"[..]); + + let member = members.next().unwrap().unwrap(); + assert_eq!(member.name(), b"0123456789abcdef"); + assert_eq!(member.data(data).unwrap(), &b"even"[..]); + + assert!(members.next().is_none()); + } +} diff --git a/crux-mir/lib/object/src/read/coff/comdat.rs b/crux-mir/lib/object/src/read/coff/comdat.rs new file mode 100644 index 000000000..3be69ecc2 --- /dev/null +++ b/crux-mir/lib/object/src/read/coff/comdat.rs @@ -0,0 +1,167 @@ +use core::str; + +use crate::endian::LittleEndian as LE; +use crate::pe; +use crate::read::{ + self, ComdatKind, ObjectComdat, ReadError, ReadRef, Result, SectionIndex, SymbolIndex, +}; + +use super::CoffFile; + +/// An iterator over the COMDAT section groups of a `CoffFile`. +#[derive(Debug)] +pub struct CoffComdatIterator<'data, 'file, R: ReadRef<'data> = &'data [u8]> { + pub(super) file: &'file CoffFile<'data, R>, + pub(super) index: usize, +} + +impl<'data, 'file, R: ReadRef<'data>> Iterator for CoffComdatIterator<'data, 'file, R> { + type Item = CoffComdat<'data, 'file, R>; + + fn next(&mut self) -> Option { + loop { + let index = self.index; + let symbol = self.file.common.symbols.symbol(index).ok()?; + self.index += 1 + symbol.number_of_aux_symbols as usize; + if let Some(comdat) = CoffComdat::parse(self.file, symbol, index) { + return Some(comdat); + } + } + } +} + +/// A COMDAT section group of a `CoffFile`. +#[derive(Debug)] +pub struct CoffComdat<'data, 'file, R: ReadRef<'data> = &'data [u8]> { + file: &'file CoffFile<'data, R>, + symbol_index: SymbolIndex, + symbol: &'data pe::ImageSymbol, + selection: u8, +} + +impl<'data, 'file, R: ReadRef<'data>> CoffComdat<'data, 'file, R> { + fn parse( + file: &'file CoffFile<'data, R>, + section_symbol: &'data pe::ImageSymbol, + index: usize, + ) -> Option> { + // Must be a section symbol. + if !section_symbol.has_aux_section() { + return None; + } + + // Auxiliary record must have a non-associative selection. + let aux = file.common.symbols.aux_section(index).ok()?; + let selection = aux.selection; + if selection == 0 || selection == pe::IMAGE_COMDAT_SELECT_ASSOCIATIVE { + return None; + } + + // Find the COMDAT symbol. + let mut symbol_index = index; + let mut symbol = section_symbol; + let section_number = section_symbol.section_number.get(LE); + loop { + symbol_index += 1 + symbol.number_of_aux_symbols as usize; + symbol = file.common.symbols.symbol(symbol_index).ok()?; + if section_number == symbol.section_number.get(LE) { + break; + } + } + + Some(CoffComdat { + file, + symbol_index: SymbolIndex(symbol_index), + symbol, + selection, + }) + } +} + +impl<'data, 'file, R: ReadRef<'data>> read::private::Sealed for CoffComdat<'data, 'file, R> {} + +impl<'data, 'file, R: ReadRef<'data>> ObjectComdat<'data> for CoffComdat<'data, 'file, R> { + type SectionIterator = CoffComdatSectionIterator<'data, 'file, R>; + + #[inline] + fn kind(&self) -> ComdatKind { + match self.selection { + pe::IMAGE_COMDAT_SELECT_NODUPLICATES => ComdatKind::NoDuplicates, + pe::IMAGE_COMDAT_SELECT_ANY => ComdatKind::Any, + pe::IMAGE_COMDAT_SELECT_SAME_SIZE => ComdatKind::SameSize, + pe::IMAGE_COMDAT_SELECT_EXACT_MATCH => ComdatKind::ExactMatch, + pe::IMAGE_COMDAT_SELECT_LARGEST => ComdatKind::Largest, + pe::IMAGE_COMDAT_SELECT_NEWEST => ComdatKind::Newest, + _ => ComdatKind::Unknown, + } + } + + #[inline] + fn symbol(&self) -> SymbolIndex { + self.symbol_index + } + + #[inline] + fn name_bytes(&self) -> Result<&[u8]> { + // Find the name of first symbol referring to the section. + self.symbol.name(self.file.common.symbols.strings()) + } + + #[inline] + fn name(&self) -> Result<&str> { + let bytes = self.name_bytes()?; + str::from_utf8(bytes) + .ok() + .read_error("Non UTF-8 COFF COMDAT name") + } + + #[inline] + fn sections(&self) -> Self::SectionIterator { + CoffComdatSectionIterator { + file: self.file, + section_number: self.symbol.section_number.get(LE), + index: 0, + } + } +} + +/// An iterator over the sections in a COMDAT section group of a `CoffFile`. +#[derive(Debug)] +pub struct CoffComdatSectionIterator<'data, 'file, R: ReadRef<'data> = &'data [u8]> { + file: &'file CoffFile<'data, R>, + section_number: u16, + index: usize, +} + +impl<'data, 'file, R: ReadRef<'data>> Iterator for CoffComdatSectionIterator<'data, 'file, R> { + type Item = SectionIndex; + + fn next(&mut self) -> Option { + // Find associated COMDAT symbols. + // TODO: it seems gcc doesn't use associated symbols for this + loop { + let index = self.index; + let symbol = self.file.common.symbols.symbol(index).ok()?; + self.index += 1 + symbol.number_of_aux_symbols as usize; + + // Must be a section symbol. + if !symbol.has_aux_section() { + continue; + } + + let section_number = symbol.section_number.get(LE); + + let aux = self.file.common.symbols.aux_section(index).ok()?; + if aux.selection == pe::IMAGE_COMDAT_SELECT_ASSOCIATIVE { + // TODO: use high_number for bigobj + if aux.number.get(LE) == self.section_number { + return Some(SectionIndex(section_number as usize)); + } + } else if aux.selection != 0 { + if section_number == self.section_number { + return Some(SectionIndex(section_number as usize)); + } + } + } + } +} diff --git a/crux-mir/lib/object/src/read/coff/file.rs b/crux-mir/lib/object/src/read/coff/file.rs new file mode 100644 index 000000000..c6cc9f846 --- /dev/null +++ b/crux-mir/lib/object/src/read/coff/file.rs @@ -0,0 +1,247 @@ +use alloc::vec::Vec; + +use crate::read::{ + self, Architecture, Export, FileFlags, Import, NoDynamicRelocationIterator, Object, ObjectKind, + ObjectSection, ReadError, ReadRef, Result, SectionIndex, SymbolIndex, +}; +use crate::{pe, LittleEndian as LE}; + +use super::{ + CoffComdat, CoffComdatIterator, CoffSection, CoffSectionIterator, CoffSegment, + CoffSegmentIterator, CoffSymbol, CoffSymbolIterator, CoffSymbolTable, SectionTable, + SymbolTable, +}; + +/// The common parts of `PeFile` and `CoffFile`. +#[derive(Debug)] +pub(crate) struct CoffCommon<'data, R: ReadRef<'data>> { + pub(crate) sections: SectionTable<'data>, + // TODO: ImageSymbolExBytes + pub(crate) symbols: SymbolTable<'data, R>, + pub(crate) image_base: u64, +} + +/// A COFF object file. +#[derive(Debug)] +pub struct CoffFile<'data, R: ReadRef<'data> = &'data [u8]> { + pub(super) header: &'data pe::ImageFileHeader, + pub(super) common: CoffCommon<'data, R>, + pub(super) data: R, +} + +impl<'data, R: ReadRef<'data>> CoffFile<'data, R> { + /// Parse the raw COFF file data. + pub fn parse(data: R) -> Result { + let mut offset = 0; + let header = pe::ImageFileHeader::parse(data, &mut offset)?; + let sections = header.sections(data, offset)?; + let symbols = header.symbols(data)?; + + Ok(CoffFile { + header, + common: CoffCommon { + sections, + symbols, + image_base: 0, + }, + data, + }) + } +} + +impl<'data, R: ReadRef<'data>> read::private::Sealed for CoffFile<'data, R> {} + +impl<'data, 'file, R> Object<'data, 'file> for CoffFile<'data, R> +where + 'data: 'file, + R: 'file + ReadRef<'data>, +{ + type Segment = CoffSegment<'data, 'file, R>; + type SegmentIterator = CoffSegmentIterator<'data, 'file, R>; + type Section = CoffSection<'data, 'file, R>; + type SectionIterator = CoffSectionIterator<'data, 'file, R>; + type Comdat = CoffComdat<'data, 'file, R>; + type ComdatIterator = CoffComdatIterator<'data, 'file, R>; + type Symbol = CoffSymbol<'data, 'file, R>; + type SymbolIterator = CoffSymbolIterator<'data, 'file, R>; + type SymbolTable = CoffSymbolTable<'data, 'file, R>; + type DynamicRelocationIterator = NoDynamicRelocationIterator; + + fn architecture(&self) -> Architecture { + match self.header.machine.get(LE) { + pe::IMAGE_FILE_MACHINE_ARMNT => Architecture::Arm, + pe::IMAGE_FILE_MACHINE_ARM64 => Architecture::Aarch64, + pe::IMAGE_FILE_MACHINE_I386 => Architecture::I386, + pe::IMAGE_FILE_MACHINE_AMD64 => Architecture::X86_64, + _ => Architecture::Unknown, + } + } + + #[inline] + fn is_little_endian(&self) -> bool { + true + } + + #[inline] + fn is_64(&self) -> bool { + // Windows COFF is always 32-bit, even for 64-bit architectures. This could be confusing. + false + } + + fn kind(&self) -> ObjectKind { + ObjectKind::Relocatable + } + + fn segments(&'file self) -> CoffSegmentIterator<'data, 'file, R> { + CoffSegmentIterator { + file: self, + iter: self.common.sections.iter(), + } + } + + fn section_by_name_bytes( + &'file self, + section_name: &[u8], + ) -> Option> { + self.sections() + .find(|section| section.name_bytes() == Ok(section_name)) + } + + fn section_by_index(&'file self, index: SectionIndex) -> Result> { + let section = self.common.sections.section(index.0)?; + Ok(CoffSection { + file: self, + index, + section, + }) + } + + fn sections(&'file self) -> CoffSectionIterator<'data, 'file, R> { + CoffSectionIterator { + file: self, + iter: self.common.sections.iter().enumerate(), + } + } + + fn comdats(&'file self) -> CoffComdatIterator<'data, 'file, R> { + CoffComdatIterator { + file: self, + index: 0, + } + } + + fn symbol_by_index(&'file self, index: SymbolIndex) -> Result> { + let symbol = self.common.symbols.symbol(index.0)?; + Ok(CoffSymbol { + file: &self.common, + index, + symbol, + }) + } + + fn symbols(&'file self) -> CoffSymbolIterator<'data, 'file, R> { + CoffSymbolIterator { + file: &self.common, + index: 0, + } + } + + #[inline] + fn symbol_table(&'file self) -> Option> { + Some(CoffSymbolTable { file: &self.common }) + } + + fn dynamic_symbols(&'file self) -> CoffSymbolIterator<'data, 'file, R> { + CoffSymbolIterator { + file: &self.common, + // Hack: don't return any. + index: self.common.symbols.len(), + } + } + + #[inline] + fn dynamic_symbol_table(&'file self) -> Option> { + None + } + + #[inline] + fn dynamic_relocations(&'file self) -> Option { + None + } + + #[inline] + fn imports(&self) -> Result>> { + // TODO: this could return undefined symbols, but not needed yet. + Ok(Vec::new()) + } + + #[inline] + fn exports(&self) -> Result>> { + // TODO: this could return global symbols, but not needed yet. + Ok(Vec::new()) + } + + fn has_debug_symbols(&self) -> bool { + self.section_by_name(".debug_info").is_some() + } + + fn relative_address_base(&self) -> u64 { + 0 + } + + #[inline] + fn entry(&self) -> u64 { + 0 + } + + fn flags(&self) -> FileFlags { + FileFlags::Coff { + characteristics: self.header.characteristics.get(LE), + } + } +} + +impl pe::ImageFileHeader { + /// Read the file header. + /// + /// `data` must be the entire file data. + /// `offset` must be the file header offset. It is updated to point after the optional header, + /// which is where the section headers are located. + pub fn parse<'data, R: ReadRef<'data>>(data: R, offset: &mut u64) -> read::Result<&'data Self> { + let header = data + .read::(offset) + .read_error("Invalid COFF file header size or alignment")?; + + // Skip over the optional header. + *offset = offset + .checked_add(header.size_of_optional_header.get(LE).into()) + .read_error("Invalid COFF optional header size")?; + + // TODO: maybe validate that the machine is known? + Ok(header) + } + + /// Read the section table. + /// + /// `data` must be the entire file data. + /// `offset` must be after the optional file header. + #[inline] + pub fn sections<'data, R: ReadRef<'data>>( + &self, + data: R, + offset: u64, + ) -> read::Result> { + SectionTable::parse(self, data, offset) + } + + /// Read the symbol table and string table. + /// + /// `data` must be the entire file data. + #[inline] + pub fn symbols<'data, R: ReadRef<'data>>( + &self, + data: R, + ) -> read::Result> { + SymbolTable::parse(self, data) + } +} diff --git a/crux-mir/lib/object/src/read/coff/mod.rs b/crux-mir/lib/object/src/read/coff/mod.rs new file mode 100644 index 000000000..d5b3caf32 --- /dev/null +++ b/crux-mir/lib/object/src/read/coff/mod.rs @@ -0,0 +1,18 @@ +//! Support for reading Windows COFF files. +//! +//! Provides `CoffFile` and related types which implement the `Object` trait. + +mod file; +pub use file::*; + +mod section; +pub use section::*; + +mod symbol; +pub use symbol::*; + +mod relocation; +pub use relocation::*; + +mod comdat; +pub use comdat::*; diff --git a/crux-mir/lib/object/src/read/coff/relocation.rs b/crux-mir/lib/object/src/read/coff/relocation.rs new file mode 100644 index 000000000..9a1fcb618 --- /dev/null +++ b/crux-mir/lib/object/src/read/coff/relocation.rs @@ -0,0 +1,91 @@ +use alloc::fmt; +use core::slice; + +use crate::endian::LittleEndian as LE; +use crate::pe; +use crate::read::{ + ReadRef, Relocation, RelocationEncoding, RelocationKind, RelocationTarget, SymbolIndex, +}; + +use super::CoffFile; + +/// An iterator over the relocations in a `CoffSection`. +pub struct CoffRelocationIterator<'data, 'file, R: ReadRef<'data> = &'data [u8]> { + pub(super) file: &'file CoffFile<'data, R>, + pub(super) iter: slice::Iter<'data, pe::ImageRelocation>, +} + +impl<'data, 'file, R: ReadRef<'data>> Iterator for CoffRelocationIterator<'data, 'file, R> { + type Item = (u64, Relocation); + + fn next(&mut self) -> Option { + self.iter.next().map(|relocation| { + let (kind, size, addend) = match self.file.header.machine.get(LE) { + pe::IMAGE_FILE_MACHINE_ARMNT => match relocation.typ.get(LE) { + pe::IMAGE_REL_ARM_ADDR32 => (RelocationKind::Absolute, 32, 0), + pe::IMAGE_REL_ARM_ADDR32NB => (RelocationKind::ImageOffset, 32, 0), + pe::IMAGE_REL_ARM_REL32 => (RelocationKind::Relative, 32, -4), + pe::IMAGE_REL_ARM_SECTION => (RelocationKind::SectionIndex, 16, 0), + pe::IMAGE_REL_ARM_SECREL => (RelocationKind::SectionOffset, 32, 0), + typ => (RelocationKind::Coff(typ), 0, 0), + }, + pe::IMAGE_FILE_MACHINE_ARM64 => match relocation.typ.get(LE) { + pe::IMAGE_REL_ARM64_ADDR32 => (RelocationKind::Absolute, 32, 0), + pe::IMAGE_REL_ARM64_ADDR32NB => (RelocationKind::ImageOffset, 32, 0), + pe::IMAGE_REL_ARM64_SECREL => (RelocationKind::SectionOffset, 32, 0), + pe::IMAGE_REL_ARM64_SECTION => (RelocationKind::SectionIndex, 16, 0), + pe::IMAGE_REL_ARM64_ADDR64 => (RelocationKind::Absolute, 64, 0), + pe::IMAGE_REL_ARM64_REL32 => (RelocationKind::Relative, 32, -4), + typ => (RelocationKind::Coff(typ), 0, 0), + }, + pe::IMAGE_FILE_MACHINE_I386 => match relocation.typ.get(LE) { + pe::IMAGE_REL_I386_DIR16 => (RelocationKind::Absolute, 16, 0), + pe::IMAGE_REL_I386_REL16 => (RelocationKind::Relative, 16, 0), + pe::IMAGE_REL_I386_DIR32 => (RelocationKind::Absolute, 32, 0), + pe::IMAGE_REL_I386_DIR32NB => (RelocationKind::ImageOffset, 32, 0), + pe::IMAGE_REL_I386_SECTION => (RelocationKind::SectionIndex, 16, 0), + pe::IMAGE_REL_I386_SECREL => (RelocationKind::SectionOffset, 32, 0), + pe::IMAGE_REL_I386_SECREL7 => (RelocationKind::SectionOffset, 7, 0), + pe::IMAGE_REL_I386_REL32 => (RelocationKind::Relative, 32, -4), + typ => (RelocationKind::Coff(typ), 0, 0), + }, + pe::IMAGE_FILE_MACHINE_AMD64 => match relocation.typ.get(LE) { + pe::IMAGE_REL_AMD64_ADDR64 => (RelocationKind::Absolute, 64, 0), + pe::IMAGE_REL_AMD64_ADDR32 => (RelocationKind::Absolute, 32, 0), + pe::IMAGE_REL_AMD64_ADDR32NB => (RelocationKind::ImageOffset, 32, 0), + pe::IMAGE_REL_AMD64_REL32 => (RelocationKind::Relative, 32, -4), + pe::IMAGE_REL_AMD64_REL32_1 => (RelocationKind::Relative, 32, -5), + pe::IMAGE_REL_AMD64_REL32_2 => (RelocationKind::Relative, 32, -6), + pe::IMAGE_REL_AMD64_REL32_3 => (RelocationKind::Relative, 32, -7), + pe::IMAGE_REL_AMD64_REL32_4 => (RelocationKind::Relative, 32, -8), + pe::IMAGE_REL_AMD64_REL32_5 => (RelocationKind::Relative, 32, -9), + pe::IMAGE_REL_AMD64_SECTION => (RelocationKind::SectionIndex, 16, 0), + pe::IMAGE_REL_AMD64_SECREL => (RelocationKind::SectionOffset, 32, 0), + pe::IMAGE_REL_AMD64_SECREL7 => (RelocationKind::SectionOffset, 7, 0), + typ => (RelocationKind::Coff(typ), 0, 0), + }, + _ => (RelocationKind::Coff(relocation.typ.get(LE)), 0, 0), + }; + let target = RelocationTarget::Symbol(SymbolIndex( + relocation.symbol_table_index.get(LE) as usize, + )); + ( + u64::from(relocation.virtual_address.get(LE)), + Relocation { + kind, + encoding: RelocationEncoding::Generic, + size, + target, + addend, + implicit_addend: true, + }, + ) + }) + } +} + +impl<'data, 'file, R: ReadRef<'data>> fmt::Debug for CoffRelocationIterator<'data, 'file, R> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CoffRelocationIterator").finish() + } +} diff --git a/crux-mir/lib/object/src/read/coff/section.rs b/crux-mir/lib/object/src/read/coff/section.rs new file mode 100644 index 000000000..731e37ca8 --- /dev/null +++ b/crux-mir/lib/object/src/read/coff/section.rs @@ -0,0 +1,524 @@ +use core::convert::TryFrom; +use core::{iter, result, slice, str}; + +use crate::endian::LittleEndian as LE; +use crate::pe; +use crate::read::util::StringTable; +use crate::read::{ + self, CompressedData, CompressedFileRange, Error, ObjectSection, ObjectSegment, ReadError, + ReadRef, Result, SectionFlags, SectionIndex, SectionKind, SegmentFlags, +}; + +use super::{CoffFile, CoffRelocationIterator}; + +/// The table of section headers in a COFF or PE file. +#[derive(Debug, Default, Clone, Copy)] +pub struct SectionTable<'data> { + sections: &'data [pe::ImageSectionHeader], +} + +impl<'data> SectionTable<'data> { + /// Parse the section table. + /// + /// `data` must be the entire file data. + /// `offset` must be after the optional file header. + pub fn parse>( + header: &pe::ImageFileHeader, + data: R, + offset: u64, + ) -> Result { + let sections = data + .read_slice_at(offset, header.number_of_sections.get(LE).into()) + .read_error("Invalid COFF/PE section headers")?; + Ok(SectionTable { sections }) + } + + /// Iterate over the section headers. + /// + /// Warning: sections indices start at 1. + #[inline] + pub fn iter(&self) -> slice::Iter<'data, pe::ImageSectionHeader> { + self.sections.iter() + } + + /// Return true if the section table is empty. + #[inline] + pub fn is_empty(&self) -> bool { + self.sections.is_empty() + } + + /// The number of section headers. + #[inline] + pub fn len(&self) -> usize { + self.sections.len() + } + + /// Return the section header at the given index. + /// + /// The index is 1-based. + pub fn section(&self, index: usize) -> read::Result<&'data pe::ImageSectionHeader> { + self.sections + .get(index.wrapping_sub(1)) + .read_error("Invalid COFF/PE section index") + } + + /// Return the section header with the given name. + /// + /// The returned index is 1-based. + /// + /// Ignores sections with invalid names. + pub fn section_by_name>( + &self, + strings: StringTable<'data, R>, + name: &[u8], + ) -> Option<(usize, &'data pe::ImageSectionHeader)> { + self.sections + .iter() + .enumerate() + .find(|(_, section)| section.name(strings) == Ok(name)) + .map(|(index, section)| (index + 1, section)) + } + + /// Compute the maximum file offset used by sections. + /// + /// This will usually match the end of file, unless the PE file has a + /// [data overlay](https://security.stackexchange.com/questions/77336/how-is-the-file-overlay-read-by-an-exe-virus) + pub fn max_section_file_offset(&self) -> u64 { + let mut max = 0; + for section in self.iter() { + match (section.pointer_to_raw_data.get(LE) as u64) + .checked_add(section.size_of_raw_data.get(LE) as u64) + { + None => { + // This cannot happen, we're suming two u32 into a u64 + continue; + } + Some(end_of_section) => { + if end_of_section > max { + max = end_of_section; + } + } + } + } + max + } +} + +/// An iterator over the loadable sections of a `CoffFile`. +#[derive(Debug)] +pub struct CoffSegmentIterator<'data, 'file, R: ReadRef<'data> = &'data [u8]> { + pub(super) file: &'file CoffFile<'data, R>, + pub(super) iter: slice::Iter<'data, pe::ImageSectionHeader>, +} + +impl<'data, 'file, R: ReadRef<'data>> Iterator for CoffSegmentIterator<'data, 'file, R> { + type Item = CoffSegment<'data, 'file, R>; + + fn next(&mut self) -> Option { + self.iter.next().map(|section| CoffSegment { + file: self.file, + section, + }) + } +} + +/// A loadable section of a `CoffFile`. +#[derive(Debug)] +pub struct CoffSegment<'data, 'file, R: ReadRef<'data> = &'data [u8]> { + pub(super) file: &'file CoffFile<'data, R>, + pub(super) section: &'data pe::ImageSectionHeader, +} + +impl<'data, 'file, R: ReadRef<'data>> CoffSegment<'data, 'file, R> { + fn bytes(&self) -> Result<&'data [u8]> { + self.section + .coff_data(self.file.data) + .read_error("Invalid COFF section offset or size") + } +} + +impl<'data, 'file, R: ReadRef<'data>> read::private::Sealed for CoffSegment<'data, 'file, R> {} + +impl<'data, 'file, R: ReadRef<'data>> ObjectSegment<'data> for CoffSegment<'data, 'file, R> { + #[inline] + fn address(&self) -> u64 { + u64::from(self.section.virtual_address.get(LE)) + } + + #[inline] + fn size(&self) -> u64 { + u64::from(self.section.virtual_size.get(LE)) + } + + #[inline] + fn align(&self) -> u64 { + self.section.coff_alignment() + } + + #[inline] + fn file_range(&self) -> (u64, u64) { + let (offset, size) = self.section.coff_file_range().unwrap_or((0, 0)); + (u64::from(offset), u64::from(size)) + } + + fn data(&self) -> Result<&'data [u8]> { + self.bytes() + } + + fn data_range(&self, address: u64, size: u64) -> Result> { + Ok(read::util::data_range( + self.bytes()?, + self.address(), + address, + size, + )) + } + + #[inline] + fn name_bytes(&self) -> Result> { + self.section + .name(self.file.common.symbols.strings()) + .map(Some) + } + + #[inline] + fn name(&self) -> Result> { + let name = self.section.name(self.file.common.symbols.strings())?; + str::from_utf8(name) + .ok() + .read_error("Non UTF-8 COFF section name") + .map(Some) + } + + #[inline] + fn flags(&self) -> SegmentFlags { + let characteristics = self.section.characteristics.get(LE); + SegmentFlags::Coff { characteristics } + } +} + +/// An iterator over the sections of a `CoffFile`. +#[derive(Debug)] +pub struct CoffSectionIterator<'data, 'file, R: ReadRef<'data> = &'data [u8]> { + pub(super) file: &'file CoffFile<'data, R>, + pub(super) iter: iter::Enumerate>, +} + +impl<'data, 'file, R: ReadRef<'data>> Iterator for CoffSectionIterator<'data, 'file, R> { + type Item = CoffSection<'data, 'file, R>; + + fn next(&mut self) -> Option { + self.iter.next().map(|(index, section)| CoffSection { + file: self.file, + index: SectionIndex(index + 1), + section, + }) + } +} + +/// A section of a `CoffFile`. +#[derive(Debug)] +pub struct CoffSection<'data, 'file, R: ReadRef<'data> = &'data [u8]> { + pub(super) file: &'file CoffFile<'data, R>, + pub(super) index: SectionIndex, + pub(super) section: &'data pe::ImageSectionHeader, +} + +impl<'data, 'file, R: ReadRef<'data>> CoffSection<'data, 'file, R> { + fn bytes(&self) -> Result<&'data [u8]> { + self.section + .coff_data(self.file.data) + .read_error("Invalid COFF section offset or size") + } +} + +impl<'data, 'file, R: ReadRef<'data>> read::private::Sealed for CoffSection<'data, 'file, R> {} + +impl<'data, 'file, R: ReadRef<'data>> ObjectSection<'data> for CoffSection<'data, 'file, R> { + type RelocationIterator = CoffRelocationIterator<'data, 'file, R>; + + #[inline] + fn index(&self) -> SectionIndex { + self.index + } + + #[inline] + fn address(&self) -> u64 { + u64::from(self.section.virtual_address.get(LE)) + } + + #[inline] + fn size(&self) -> u64 { + // TODO: This may need to be the length from the auxiliary symbol for this section. + u64::from(self.section.size_of_raw_data.get(LE)) + } + + #[inline] + fn align(&self) -> u64 { + self.section.coff_alignment() + } + + #[inline] + fn file_range(&self) -> Option<(u64, u64)> { + let (offset, size) = self.section.coff_file_range()?; + Some((u64::from(offset), u64::from(size))) + } + + fn data(&self) -> Result<&'data [u8]> { + self.bytes() + } + + fn data_range(&self, address: u64, size: u64) -> Result> { + Ok(read::util::data_range( + self.bytes()?, + self.address(), + address, + size, + )) + } + + #[inline] + fn compressed_file_range(&self) -> Result { + Ok(CompressedFileRange::none(self.file_range())) + } + + #[inline] + fn compressed_data(&self) -> Result> { + self.data().map(CompressedData::none) + } + + #[inline] + fn name_bytes(&self) -> Result<&[u8]> { + self.section.name(self.file.common.symbols.strings()) + } + + #[inline] + fn name(&self) -> Result<&str> { + let name = self.name_bytes()?; + str::from_utf8(name) + .ok() + .read_error("Non UTF-8 COFF section name") + } + + #[inline] + fn segment_name_bytes(&self) -> Result> { + Ok(None) + } + + #[inline] + fn segment_name(&self) -> Result> { + Ok(None) + } + + #[inline] + fn kind(&self) -> SectionKind { + self.section.kind() + } + + fn relocations(&self) -> CoffRelocationIterator<'data, 'file, R> { + let relocations = self.section.coff_relocations(self.file.data).unwrap_or(&[]); + CoffRelocationIterator { + file: self.file, + iter: relocations.iter(), + } + } + + fn flags(&self) -> SectionFlags { + SectionFlags::Coff { + characteristics: self.section.characteristics.get(LE), + } + } +} + +impl pe::ImageSectionHeader { + pub(crate) fn kind(&self) -> SectionKind { + let characteristics = self.characteristics.get(LE); + if characteristics & (pe::IMAGE_SCN_CNT_CODE | pe::IMAGE_SCN_MEM_EXECUTE) != 0 { + SectionKind::Text + } else if characteristics & pe::IMAGE_SCN_CNT_INITIALIZED_DATA != 0 { + if characteristics & pe::IMAGE_SCN_MEM_DISCARDABLE != 0 { + SectionKind::Other + } else if characteristics & pe::IMAGE_SCN_MEM_WRITE != 0 { + SectionKind::Data + } else { + SectionKind::ReadOnlyData + } + } else if characteristics & pe::IMAGE_SCN_CNT_UNINITIALIZED_DATA != 0 { + SectionKind::UninitializedData + } else if characteristics & pe::IMAGE_SCN_LNK_INFO != 0 { + SectionKind::Linker + } else { + SectionKind::Unknown + } + } +} + +impl pe::ImageSectionHeader { + /// Return the string table offset of the section name. + /// + /// Returns `Ok(None)` if the name doesn't use the string table + /// and can be obtained with `raw_name` instead. + pub fn name_offset(&self) -> Result> { + let bytes = &self.name; + if bytes[0] != b'/' { + return Ok(None); + } + + if bytes[1] == b'/' { + let mut offset = 0; + for byte in bytes[2..].iter() { + let digit = match byte { + b'A'..=b'Z' => byte - b'A', + b'a'..=b'z' => byte - b'a' + 26, + b'0'..=b'9' => byte - b'0' + 52, + b'+' => 62, + b'/' => 63, + _ => return Err(Error("Invalid COFF section name base-64 offset")), + }; + offset = offset * 64 + digit as u64; + } + u32::try_from(offset) + .ok() + .read_error("Invalid COFF section name base-64 offset") + .map(Some) + } else { + let mut offset = 0; + for byte in bytes[1..].iter() { + let digit = match byte { + b'0'..=b'9' => byte - b'0', + 0 => break, + _ => return Err(Error("Invalid COFF section name base-10 offset")), + }; + offset = offset * 10 + digit as u32; + } + Ok(Some(offset)) + } + } + + /// Return the section name. + /// + /// This handles decoding names that are offsets into the symbol string table. + pub fn name<'data, R: ReadRef<'data>>( + &'data self, + strings: StringTable<'data, R>, + ) -> Result<&'data [u8]> { + if let Some(offset) = self.name_offset()? { + strings + .get(offset) + .read_error("Invalid COFF section name offset") + } else { + Ok(self.raw_name()) + } + } + + /// Return the raw section name. + pub fn raw_name(&self) -> &[u8] { + let bytes = &self.name; + match memchr::memchr(b'\0', bytes) { + Some(end) => &bytes[..end], + None => &bytes[..], + } + } + + /// Return the offset and size of the section in a COFF file. + /// + /// Returns `None` for sections that have no data in the file. + pub fn coff_file_range(&self) -> Option<(u32, u32)> { + if self.characteristics.get(LE) & pe::IMAGE_SCN_CNT_UNINITIALIZED_DATA != 0 { + None + } else { + let offset = self.pointer_to_raw_data.get(LE); + // Note: virtual size is not used for COFF. + let size = self.size_of_raw_data.get(LE); + Some((offset, size)) + } + } + + /// Return the section data in a COFF file. + /// + /// Returns `Ok(&[])` if the section has no data. + /// Returns `Err` for invalid values. + pub fn coff_data<'data, R: ReadRef<'data>>(&self, data: R) -> result::Result<&'data [u8], ()> { + if let Some((offset, size)) = self.coff_file_range() { + data.read_bytes_at(offset.into(), size.into()) + } else { + Ok(&[]) + } + } + + /// Return the section alignment in bytes. + /// + /// This is only valid for sections in a COFF file. + pub fn coff_alignment(&self) -> u64 { + match self.characteristics.get(LE) & pe::IMAGE_SCN_ALIGN_MASK { + pe::IMAGE_SCN_ALIGN_1BYTES => 1, + pe::IMAGE_SCN_ALIGN_2BYTES => 2, + pe::IMAGE_SCN_ALIGN_4BYTES => 4, + pe::IMAGE_SCN_ALIGN_8BYTES => 8, + pe::IMAGE_SCN_ALIGN_16BYTES => 16, + pe::IMAGE_SCN_ALIGN_32BYTES => 32, + pe::IMAGE_SCN_ALIGN_64BYTES => 64, + pe::IMAGE_SCN_ALIGN_128BYTES => 128, + pe::IMAGE_SCN_ALIGN_256BYTES => 256, + pe::IMAGE_SCN_ALIGN_512BYTES => 512, + pe::IMAGE_SCN_ALIGN_1024BYTES => 1024, + pe::IMAGE_SCN_ALIGN_2048BYTES => 2048, + pe::IMAGE_SCN_ALIGN_4096BYTES => 4096, + pe::IMAGE_SCN_ALIGN_8192BYTES => 8192, + _ => 16, + } + } + + /// Read the relocations in a COFF file. + /// + /// `data` must be the entire file data. + pub fn coff_relocations<'data, R: ReadRef<'data>>( + &self, + data: R, + ) -> read::Result<&'data [pe::ImageRelocation]> { + let mut pointer = self.pointer_to_relocations.get(LE).into(); + let mut number: usize = self.number_of_relocations.get(LE).into(); + if number == core::u16::MAX.into() + && self.characteristics.get(LE) & pe::IMAGE_SCN_LNK_NRELOC_OVFL != 0 + { + // Extended relocations. Read first relocation (which contains extended count) & adjust + // relocations pointer. + let extended_relocation_info = data + .read_at::(pointer) + .read_error("Invalid COFF relocation offset or number")?; + number = extended_relocation_info.virtual_address.get(LE) as usize; + if number == 0 { + return Err(Error("Invalid COFF relocation number")); + } + pointer += core::mem::size_of::() as u64; + // Extended relocation info does not contribute to the count of sections. + number -= 1; + } + data.read_slice_at(pointer, number) + .read_error("Invalid COFF relocation offset or number") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn name_offset() { + let mut section = pe::ImageSectionHeader::default(); + section.name = *b"xxxxxxxx"; + assert_eq!(section.name_offset(), Ok(None)); + section.name = *b"/0\0\0\0\0\0\0"; + assert_eq!(section.name_offset(), Ok(Some(0))); + section.name = *b"/9999999"; + assert_eq!(section.name_offset(), Ok(Some(999_9999))); + section.name = *b"//AAAAAA"; + assert_eq!(section.name_offset(), Ok(Some(0))); + section.name = *b"//D/////"; + assert_eq!(section.name_offset(), Ok(Some(0xffff_ffff))); + section.name = *b"//EAAAAA"; + assert!(section.name_offset().is_err()); + section.name = *b"////////"; + assert!(section.name_offset().is_err()); + } +} diff --git a/crux-mir/lib/object/src/read/coff/symbol.rs b/crux-mir/lib/object/src/read/coff/symbol.rs new file mode 100644 index 000000000..c954c8a29 --- /dev/null +++ b/crux-mir/lib/object/src/read/coff/symbol.rs @@ -0,0 +1,518 @@ +use alloc::fmt; +use alloc::vec::Vec; +use core::convert::TryInto; +use core::str; + +use super::{CoffCommon, SectionTable}; +use crate::endian::{LittleEndian as LE, U32Bytes}; +use crate::pe; +use crate::pod::{bytes_of_slice, Pod}; +use crate::read::util::StringTable; +use crate::read::{ + self, Bytes, ObjectSymbol, ObjectSymbolTable, ReadError, ReadRef, Result, SectionIndex, + SymbolFlags, SymbolIndex, SymbolKind, SymbolMap, SymbolMapEntry, SymbolScope, SymbolSection, +}; + +/// A table of symbol entries in a COFF or PE file. +/// +/// Also includes the string table used for the symbol names. +#[derive(Debug)] +pub struct SymbolTable<'data, R = &'data [u8]> +where + R: ReadRef<'data>, +{ + symbols: &'data [pe::ImageSymbolBytes], + strings: StringTable<'data, R>, +} + +impl<'data, R: ReadRef<'data>> Default for SymbolTable<'data, R> { + fn default() -> Self { + Self { + symbols: &[], + strings: StringTable::default(), + } + } +} + +impl<'data, R: ReadRef<'data>> SymbolTable<'data, R> { + /// Read the symbol table. + pub fn parse(header: &pe::ImageFileHeader, data: R) -> Result { + // The symbol table may not be present. + let mut offset = header.pointer_to_symbol_table.get(LE).into(); + let (symbols, strings) = if offset != 0 { + let symbols = data + .read_slice(&mut offset, header.number_of_symbols.get(LE) as usize) + .read_error("Invalid COFF symbol table offset or size")?; + + // Note: don't update data when reading length; the length includes itself. + let length = data + .read_at::>(offset) + .read_error("Missing COFF string table")? + .get(LE); + let str_end = offset + .checked_add(length as u64) + .read_error("Invalid COFF string table length")?; + let strings = StringTable::new(data, offset, str_end); + + (symbols, strings) + } else { + (&[][..], StringTable::default()) + }; + + Ok(SymbolTable { symbols, strings }) + } + + /// Return the string table used for the symbol names. + #[inline] + pub fn strings(&self) -> StringTable<'data, R> { + self.strings + } + + /// Return true if the symbol table is empty. + #[inline] + pub fn is_empty(&self) -> bool { + self.symbols.is_empty() + } + + /// The number of symbol table entries. + /// + /// This includes auxiliary symbol table entries. + #[inline] + pub fn len(&self) -> usize { + self.symbols.len() + } + + /// Iterate over the symbols. + #[inline] + pub fn iter<'table>(&'table self) -> SymbolIterator<'data, 'table, R> { + SymbolIterator { + symbols: self, + index: 0, + } + } + + /// Return the symbol table entry at the given index. + #[inline] + pub fn symbol(&self, index: usize) -> Result<&'data pe::ImageSymbol> { + self.get::(index, 0) + } + + /// Return the auxiliary function symbol for the symbol table entry at the given index. + /// + /// Note that the index is of the symbol, not the first auxiliary record. + #[inline] + pub fn aux_function(&self, index: usize) -> Result<&'data pe::ImageAuxSymbolFunction> { + self.get::(index, 1) + } + + /// Return the auxiliary section symbol for the symbol table entry at the given index. + /// + /// Note that the index is of the symbol, not the first auxiliary record. + #[inline] + pub fn aux_section(&self, index: usize) -> Result<&'data pe::ImageAuxSymbolSection> { + self.get::(index, 1) + } + + /// Return the auxiliary file name for the symbol table entry at the given index. + /// + /// Note that the index is of the symbol, not the first auxiliary record. + pub fn aux_file_name(&self, index: usize, aux_count: u8) -> Result<&'data [u8]> { + let entries = index + .checked_add(1) + .and_then(|x| Some(x..x.checked_add(aux_count.into())?)) + .and_then(|x| self.symbols.get(x)) + .read_error("Invalid COFF symbol index")?; + let bytes = bytes_of_slice(entries); + // The name is padded with nulls. + Ok(match memchr::memchr(b'\0', bytes) { + Some(end) => &bytes[..end], + None => bytes, + }) + } + + /// Return the symbol table entry or auxiliary record at the given index and offset. + pub fn get(&self, index: usize, offset: usize) -> Result<&'data T> { + let bytes = index + .checked_add(offset) + .and_then(|x| self.symbols.get(x)) + .read_error("Invalid COFF symbol index")?; + Bytes(&bytes.0[..]) + .read() + .read_error("Invalid COFF symbol data") + } + + /// Construct a map from addresses to a user-defined map entry. + pub fn map Option>( + &self, + f: F, + ) -> SymbolMap { + let mut symbols = Vec::with_capacity(self.symbols.len()); + for (_, symbol) in self.iter() { + if !symbol.is_definition() { + continue; + } + if let Some(entry) = f(symbol) { + symbols.push(entry); + } + } + SymbolMap::new(symbols) + } +} + +/// An iterator for symbol entries in a COFF or PE file. +/// +/// Yields the index and symbol structure for each symbol. +#[derive(Debug)] +pub struct SymbolIterator<'data, 'table, R = &'data [u8]> +where + R: ReadRef<'data>, +{ + symbols: &'table SymbolTable<'data, R>, + index: usize, +} + +impl<'data, 'table, R: ReadRef<'data>> Iterator for SymbolIterator<'data, 'table, R> { + type Item = (usize, &'data pe::ImageSymbol); + + fn next(&mut self) -> Option { + let index = self.index; + let symbol = self.symbols.symbol(index).ok()?; + self.index += 1 + symbol.number_of_aux_symbols as usize; + Some((index, symbol)) + } +} + +impl pe::ImageSymbol { + /// Parse a COFF symbol name. + /// + /// `strings` must be the string table used for symbol names. + pub fn name<'data, R: ReadRef<'data>>( + &'data self, + strings: StringTable<'data, R>, + ) -> Result<&'data [u8]> { + if self.name[0] == 0 { + // If the name starts with 0 then the last 4 bytes are a string table offset. + let offset = u32::from_le_bytes(self.name[4..8].try_into().unwrap()); + strings + .get(offset) + .read_error("Invalid COFF symbol name offset") + } else { + // The name is inline and padded with nulls. + Ok(match memchr::memchr(b'\0', &self.name) { + Some(end) => &self.name[..end], + None => &self.name[..], + }) + } + } + + /// Return the symbol address. + /// + /// This takes into account the image base and the section address. + pub fn address(&self, image_base: u64, sections: &SectionTable) -> Result { + let section_number = self.section_number.get(LE) as usize; + let section = sections.section(section_number)?; + let virtual_address = u64::from(section.virtual_address.get(LE)); + let value = u64::from(self.value.get(LE)); + Ok(image_base + virtual_address + value) + } + + /// Return true if the symbol is a definition of a function or data object. + pub fn is_definition(&self) -> bool { + let section_number = self.section_number.get(LE); + if section_number == pe::IMAGE_SYM_UNDEFINED { + return false; + } + match self.storage_class { + pe::IMAGE_SYM_CLASS_STATIC => { + // Exclude section symbols. + !(self.value.get(LE) == 0 && self.number_of_aux_symbols > 0) + } + pe::IMAGE_SYM_CLASS_EXTERNAL | pe::IMAGE_SYM_CLASS_WEAK_EXTERNAL => true, + _ => false, + } + } + + /// Return true if the symbol has an auxiliary file name. + pub fn has_aux_file_name(&self) -> bool { + self.number_of_aux_symbols > 0 && self.storage_class == pe::IMAGE_SYM_CLASS_FILE + } + + /// Return true if the symbol has an auxiliary function symbol. + pub fn has_aux_function(&self) -> bool { + self.number_of_aux_symbols > 0 && self.derived_type() == pe::IMAGE_SYM_DTYPE_FUNCTION + } + + /// Return true if the symbol has an auxiliary section symbol. + pub fn has_aux_section(&self) -> bool { + self.number_of_aux_symbols > 0 + && self.storage_class == pe::IMAGE_SYM_CLASS_STATIC + && self.value.get(LE) == 0 + } +} + +/// A symbol table of a `CoffFile`. +#[derive(Debug, Clone, Copy)] +pub struct CoffSymbolTable<'data, 'file, R = &'data [u8]> +where + R: ReadRef<'data>, +{ + pub(crate) file: &'file CoffCommon<'data, R>, +} + +impl<'data, 'file, R: ReadRef<'data>> read::private::Sealed for CoffSymbolTable<'data, 'file, R> {} + +impl<'data, 'file, R: ReadRef<'data>> ObjectSymbolTable<'data> + for CoffSymbolTable<'data, 'file, R> +{ + type Symbol = CoffSymbol<'data, 'file, R>; + type SymbolIterator = CoffSymbolIterator<'data, 'file, R>; + + fn symbols(&self) -> Self::SymbolIterator { + CoffSymbolIterator { + file: self.file, + index: 0, + } + } + + fn symbol_by_index(&self, index: SymbolIndex) -> Result { + let symbol = self.file.symbols.symbol(index.0)?; + Ok(CoffSymbol { + file: self.file, + index, + symbol, + }) + } +} + +/// An iterator over the symbols of a `CoffFile`. +pub struct CoffSymbolIterator<'data, 'file, R = &'data [u8]> +where + R: ReadRef<'data>, +{ + pub(crate) file: &'file CoffCommon<'data, R>, + pub(crate) index: usize, +} + +impl<'data, 'file, R: ReadRef<'data>> fmt::Debug for CoffSymbolIterator<'data, 'file, R> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CoffSymbolIterator").finish() + } +} + +impl<'data, 'file, R: ReadRef<'data>> Iterator for CoffSymbolIterator<'data, 'file, R> { + type Item = CoffSymbol<'data, 'file, R>; + + fn next(&mut self) -> Option { + let index = self.index; + let symbol = self.file.symbols.symbol(index).ok()?; + self.index += 1 + symbol.number_of_aux_symbols as usize; + Some(CoffSymbol { + file: self.file, + index: SymbolIndex(index), + symbol, + }) + } +} + +/// A symbol of a `CoffFile`. +#[derive(Debug, Clone, Copy)] +pub struct CoffSymbol<'data, 'file, R = &'data [u8]> +where + R: ReadRef<'data>, +{ + pub(crate) file: &'file CoffCommon<'data, R>, + pub(crate) index: SymbolIndex, + pub(crate) symbol: &'data pe::ImageSymbol, +} + +impl<'data, 'file, R: ReadRef<'data>> read::private::Sealed for CoffSymbol<'data, 'file, R> {} + +impl<'data, 'file, R: ReadRef<'data>> ObjectSymbol<'data> for CoffSymbol<'data, 'file, R> { + #[inline] + fn index(&self) -> SymbolIndex { + self.index + } + + fn name_bytes(&self) -> read::Result<&'data [u8]> { + if self.symbol.has_aux_file_name() { + self.file + .symbols + .aux_file_name(self.index.0, self.symbol.number_of_aux_symbols) + } else { + self.symbol.name(self.file.symbols.strings()) + } + } + + fn name(&self) -> read::Result<&'data str> { + let name = self.name_bytes()?; + str::from_utf8(name) + .ok() + .read_error("Non UTF-8 COFF symbol name") + } + + fn address(&self) -> u64 { + // Only return an address for storage classes that we know use an address. + match self.symbol.storage_class { + pe::IMAGE_SYM_CLASS_STATIC + | pe::IMAGE_SYM_CLASS_WEAK_EXTERNAL + | pe::IMAGE_SYM_CLASS_LABEL => {} + pe::IMAGE_SYM_CLASS_EXTERNAL => { + if self.symbol.section_number.get(LE) == pe::IMAGE_SYM_UNDEFINED { + // Undefined or common data, neither of which have an address. + return 0; + } + } + _ => return 0, + } + self.symbol + .address(self.file.image_base, &self.file.sections) + .unwrap_or(0) + } + + fn size(&self) -> u64 { + match self.symbol.storage_class { + pe::IMAGE_SYM_CLASS_STATIC => { + // Section symbols may duplicate the size from the section table. + if self.symbol.has_aux_section() { + if let Ok(aux) = self.file.symbols.aux_section(self.index.0) { + u64::from(aux.length.get(LE)) + } else { + 0 + } + } else { + 0 + } + } + pe::IMAGE_SYM_CLASS_EXTERNAL => { + if self.symbol.section_number.get(LE) == pe::IMAGE_SYM_UNDEFINED { + // For undefined symbols, symbol.value is 0 and the size is 0. + // For common data, symbol.value is the size. + u64::from(self.symbol.value.get(LE)) + } else if self.symbol.has_aux_function() { + // Function symbols may have a size. + if let Ok(aux) = self.file.symbols.aux_function(self.index.0) { + u64::from(aux.total_size.get(LE)) + } else { + 0 + } + } else { + 0 + } + } + // Most symbols don't have sizes. + _ => 0, + } + } + + fn kind(&self) -> SymbolKind { + let derived_kind = if self.symbol.derived_type() == pe::IMAGE_SYM_DTYPE_FUNCTION { + SymbolKind::Text + } else { + SymbolKind::Data + }; + match self.symbol.storage_class { + pe::IMAGE_SYM_CLASS_STATIC => { + if self.symbol.value.get(LE) == 0 && self.symbol.number_of_aux_symbols > 0 { + SymbolKind::Section + } else { + derived_kind + } + } + pe::IMAGE_SYM_CLASS_EXTERNAL | pe::IMAGE_SYM_CLASS_WEAK_EXTERNAL => derived_kind, + pe::IMAGE_SYM_CLASS_SECTION => SymbolKind::Section, + pe::IMAGE_SYM_CLASS_FILE => SymbolKind::File, + pe::IMAGE_SYM_CLASS_LABEL => SymbolKind::Label, + _ => SymbolKind::Unknown, + } + } + + fn section(&self) -> SymbolSection { + match self.symbol.section_number.get(LE) { + pe::IMAGE_SYM_UNDEFINED => { + if self.symbol.storage_class == pe::IMAGE_SYM_CLASS_EXTERNAL + && self.symbol.value.get(LE) == 0 + { + SymbolSection::Undefined + } else { + SymbolSection::Common + } + } + pe::IMAGE_SYM_ABSOLUTE => SymbolSection::Absolute, + pe::IMAGE_SYM_DEBUG => { + if self.symbol.storage_class == pe::IMAGE_SYM_CLASS_FILE { + SymbolSection::None + } else { + SymbolSection::Unknown + } + } + index if index > 0 => SymbolSection::Section(SectionIndex(index.into())), + _ => SymbolSection::Unknown, + } + } + + #[inline] + fn is_undefined(&self) -> bool { + self.symbol.storage_class == pe::IMAGE_SYM_CLASS_EXTERNAL + && self.symbol.section_number.get(LE) == pe::IMAGE_SYM_UNDEFINED + && self.symbol.value.get(LE) == 0 + } + + #[inline] + fn is_definition(&self) -> bool { + self.symbol.is_definition() + } + + #[inline] + fn is_common(&self) -> bool { + self.symbol.storage_class == pe::IMAGE_SYM_CLASS_EXTERNAL + && self.symbol.section_number.get(LE) == pe::IMAGE_SYM_UNDEFINED + && self.symbol.value.get(LE) != 0 + } + + #[inline] + fn is_weak(&self) -> bool { + self.symbol.storage_class == pe::IMAGE_SYM_CLASS_WEAK_EXTERNAL + } + + #[inline] + fn scope(&self) -> SymbolScope { + match self.symbol.storage_class { + pe::IMAGE_SYM_CLASS_EXTERNAL | pe::IMAGE_SYM_CLASS_WEAK_EXTERNAL => { + // TODO: determine if symbol is exported + SymbolScope::Linkage + } + _ => SymbolScope::Compilation, + } + } + + #[inline] + fn is_global(&self) -> bool { + match self.symbol.storage_class { + pe::IMAGE_SYM_CLASS_EXTERNAL | pe::IMAGE_SYM_CLASS_WEAK_EXTERNAL => true, + _ => false, + } + } + + #[inline] + fn is_local(&self) -> bool { + !self.is_global() + } + + fn flags(&self) -> SymbolFlags { + if self.symbol.has_aux_section() { + if let Ok(aux) = self.file.symbols.aux_section(self.index.0) { + // TODO: use high_number for bigobj + let number = aux.number.get(LE) as usize; + return SymbolFlags::CoffSection { + selection: aux.selection, + associative_section: if number == 0 { + None + } else { + Some(SectionIndex(number)) + }, + }; + } + } + SymbolFlags::None + } +} diff --git a/crux-mir/lib/object/src/read/elf/comdat.rs b/crux-mir/lib/object/src/read/elf/comdat.rs new file mode 100644 index 000000000..c9f0076f9 --- /dev/null +++ b/crux-mir/lib/object/src/read/elf/comdat.rs @@ -0,0 +1,162 @@ +use core::fmt::Debug; +use core::{iter, slice, str}; + +use crate::elf; +use crate::endian::{Endianness, U32Bytes}; +use crate::read::{self, ComdatKind, ObjectComdat, ReadError, ReadRef, SectionIndex, SymbolIndex}; + +use super::{ElfFile, FileHeader, SectionHeader, Sym}; + +/// An iterator over the COMDAT section groups of an `ElfFile32`. +pub type ElfComdatIterator32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfComdatIterator<'data, 'file, elf::FileHeader32, R>; +/// An iterator over the COMDAT section groups of an `ElfFile64`. +pub type ElfComdatIterator64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfComdatIterator<'data, 'file, elf::FileHeader64, R>; + +/// An iterator over the COMDAT section groups of an `ElfFile`. +#[derive(Debug)] +pub struct ElfComdatIterator<'data, 'file, Elf, R = &'data [u8]> +where + 'data: 'file, + Elf: FileHeader, + R: ReadRef<'data>, +{ + pub(super) file: &'file ElfFile<'data, Elf, R>, + pub(super) iter: iter::Enumerate>, +} + +impl<'data, 'file, Elf, R> Iterator for ElfComdatIterator<'data, 'file, Elf, R> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ + type Item = ElfComdat<'data, 'file, Elf, R>; + + fn next(&mut self) -> Option { + while let Some((_index, section)) = self.iter.next() { + if let Some(comdat) = ElfComdat::parse(self.file, section) { + return Some(comdat); + } + } + None + } +} + +/// A COMDAT section group of an `ElfFile32`. +pub type ElfComdat32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfComdat<'data, 'file, elf::FileHeader32, R>; +/// A COMDAT section group of an `ElfFile64`. +pub type ElfComdat64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfComdat<'data, 'file, elf::FileHeader64, R>; + +/// A COMDAT section group of an `ElfFile`. +#[derive(Debug)] +pub struct ElfComdat<'data, 'file, Elf, R = &'data [u8]> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ + file: &'file ElfFile<'data, Elf, R>, + section: &'data Elf::SectionHeader, + sections: &'data [U32Bytes], +} + +impl<'data, 'file, Elf, R> ElfComdat<'data, 'file, Elf, R> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ + fn parse( + file: &'file ElfFile<'data, Elf, R>, + section: &'data Elf::SectionHeader, + ) -> Option> { + let (flag, sections) = section.group(file.endian, file.data).ok()??; + if flag != elf::GRP_COMDAT { + return None; + } + Some(ElfComdat { + file, + section, + sections, + }) + } +} + +impl<'data, 'file, Elf, R> read::private::Sealed for ElfComdat<'data, 'file, Elf, R> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ +} + +impl<'data, 'file, Elf, R> ObjectComdat<'data> for ElfComdat<'data, 'file, Elf, R> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ + type SectionIterator = ElfComdatSectionIterator<'data, 'file, Elf, R>; + + #[inline] + fn kind(&self) -> ComdatKind { + ComdatKind::Any + } + + #[inline] + fn symbol(&self) -> SymbolIndex { + SymbolIndex(self.section.sh_info(self.file.endian) as usize) + } + + fn name_bytes(&self) -> read::Result<&[u8]> { + // FIXME: check sh_link + let index = self.section.sh_info(self.file.endian) as usize; + let symbol = self.file.symbols.symbol(index)?; + symbol.name(self.file.endian, self.file.symbols.strings()) + } + + fn name(&self) -> read::Result<&str> { + let name = self.name_bytes()?; + str::from_utf8(name) + .ok() + .read_error("Non UTF-8 ELF COMDAT name") + } + + fn sections(&self) -> Self::SectionIterator { + ElfComdatSectionIterator { + file: self.file, + sections: self.sections.iter(), + } + } +} + +/// An iterator over the sections in a COMDAT section group of an `ElfFile32`. +pub type ElfComdatSectionIterator32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfComdatSectionIterator<'data, 'file, elf::FileHeader32, R>; +/// An iterator over the sections in a COMDAT section group of an `ElfFile64`. +pub type ElfComdatSectionIterator64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfComdatSectionIterator<'data, 'file, elf::FileHeader64, R>; + +/// An iterator over the sections in a COMDAT section group of an `ElfFile`. +#[derive(Debug)] +pub struct ElfComdatSectionIterator<'data, 'file, Elf, R = &'data [u8]> +where + 'data: 'file, + Elf: FileHeader, + R: ReadRef<'data>, +{ + file: &'file ElfFile<'data, Elf, R>, + sections: slice::Iter<'data, U32Bytes>, +} + +impl<'data, 'file, Elf, R> Iterator for ElfComdatSectionIterator<'data, 'file, Elf, R> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ + type Item = SectionIndex; + + fn next(&mut self) -> Option { + let index = self.sections.next()?; + Some(SectionIndex(index.get(self.file.endian) as usize)) + } +} diff --git a/crux-mir/lib/object/src/read/elf/compression.rs b/crux-mir/lib/object/src/read/elf/compression.rs new file mode 100644 index 000000000..7242dd39c --- /dev/null +++ b/crux-mir/lib/object/src/read/elf/compression.rs @@ -0,0 +1,56 @@ +use core::fmt::Debug; + +use crate::elf; +use crate::endian; +use crate::pod::Pod; + +/// A trait for generic access to `CompressionHeader32` and `CompressionHeader64`. +#[allow(missing_docs)] +pub trait CompressionHeader: Debug + Pod { + type Word: Into; + type Endian: endian::Endian; + + fn ch_type(&self, endian: Self::Endian) -> u32; + fn ch_size(&self, endian: Self::Endian) -> Self::Word; + fn ch_addralign(&self, endian: Self::Endian) -> Self::Word; +} + +impl CompressionHeader for elf::CompressionHeader32 { + type Word = u32; + type Endian = Endian; + + #[inline] + fn ch_type(&self, endian: Self::Endian) -> u32 { + self.ch_type.get(endian) + } + + #[inline] + fn ch_size(&self, endian: Self::Endian) -> Self::Word { + self.ch_size.get(endian) + } + + #[inline] + fn ch_addralign(&self, endian: Self::Endian) -> Self::Word { + self.ch_addralign.get(endian) + } +} + +impl CompressionHeader for elf::CompressionHeader64 { + type Word = u64; + type Endian = Endian; + + #[inline] + fn ch_type(&self, endian: Self::Endian) -> u32 { + self.ch_type.get(endian) + } + + #[inline] + fn ch_size(&self, endian: Self::Endian) -> Self::Word { + self.ch_size.get(endian) + } + + #[inline] + fn ch_addralign(&self, endian: Self::Endian) -> Self::Word { + self.ch_addralign.get(endian) + } +} diff --git a/crux-mir/lib/object/src/read/elf/dynamic.rs b/crux-mir/lib/object/src/read/elf/dynamic.rs new file mode 100644 index 000000000..5fe15b560 --- /dev/null +++ b/crux-mir/lib/object/src/read/elf/dynamic.rs @@ -0,0 +1,117 @@ +use core::convert::TryInto; +use core::fmt::Debug; + +use crate::elf; +use crate::endian; +use crate::pod::Pod; +use crate::read::{ReadError, Result, StringTable}; + +/// A trait for generic access to `Dyn32` and `Dyn64`. +#[allow(missing_docs)] +pub trait Dyn: Debug + Pod { + type Word: Into; + type Endian: endian::Endian; + + fn d_tag(&self, endian: Self::Endian) -> Self::Word; + fn d_val(&self, endian: Self::Endian) -> Self::Word; + + /// Try to convert the tag to a `u32`. + fn tag32(&self, endian: Self::Endian) -> Option { + self.d_tag(endian).into().try_into().ok() + } + + /// Try to convert the value to a `u32`. + fn val32(&self, endian: Self::Endian) -> Option { + self.d_val(endian).into().try_into().ok() + } + + /// Return true if the value is an offset in the dynamic string table. + fn is_string(&self, endian: Self::Endian) -> bool { + if let Some(tag) = self.tag32(endian) { + match tag { + elf::DT_NEEDED + | elf::DT_SONAME + | elf::DT_RPATH + | elf::DT_RUNPATH + | elf::DT_AUXILIARY + | elf::DT_FILTER => true, + _ => false, + } + } else { + false + } + } + + /// Use the value to get a string in a string table. + /// + /// Does not check for an appropriate tag. + fn string<'data>( + &self, + endian: Self::Endian, + strings: StringTable<'data>, + ) -> Result<&'data [u8]> { + self.val32(endian) + .and_then(|val| strings.get(val).ok()) + .read_error("Invalid ELF dyn string") + } + + /// Return true if the value is an address. + fn is_address(&self, endian: Self::Endian) -> bool { + if let Some(tag) = self.tag32(endian) { + match tag { + elf::DT_PLTGOT + | elf::DT_HASH + | elf::DT_STRTAB + | elf::DT_SYMTAB + | elf::DT_RELA + | elf::DT_INIT + | elf::DT_FINI + | elf::DT_SYMBOLIC + | elf::DT_REL + | elf::DT_DEBUG + | elf::DT_JMPREL + | elf::DT_FINI_ARRAY + | elf::DT_INIT_ARRAY + | elf::DT_PREINIT_ARRAY + | elf::DT_SYMTAB_SHNDX + | elf::DT_VERDEF + | elf::DT_VERNEED + | elf::DT_VERSYM + | elf::DT_ADDRRNGLO..=elf::DT_ADDRRNGHI => true, + _ => false, + } + } else { + false + } + } +} + +impl Dyn for elf::Dyn32 { + type Word = u32; + type Endian = Endian; + + #[inline] + fn d_tag(&self, endian: Self::Endian) -> Self::Word { + self.d_tag.get(endian) + } + + #[inline] + fn d_val(&self, endian: Self::Endian) -> Self::Word { + self.d_val.get(endian) + } +} + +impl Dyn for elf::Dyn64 { + type Word = u64; + type Endian = Endian; + + #[inline] + fn d_tag(&self, endian: Self::Endian) -> Self::Word { + self.d_tag.get(endian) + } + + #[inline] + fn d_val(&self, endian: Self::Endian) -> Self::Word { + self.d_val.get(endian) + } +} diff --git a/crux-mir/lib/object/src/read/elf/file.rs b/crux-mir/lib/object/src/read/elf/file.rs new file mode 100644 index 000000000..e1f76a38c --- /dev/null +++ b/crux-mir/lib/object/src/read/elf/file.rs @@ -0,0 +1,882 @@ +use alloc::vec::Vec; +use core::convert::TryInto; +use core::fmt::Debug; +use core::mem; + +use crate::read::{ + self, util, Architecture, ByteString, Bytes, Error, Export, FileFlags, Import, Object, + ObjectKind, ReadError, ReadRef, SectionIndex, StringTable, SymbolIndex, +}; +use crate::{elf, endian, Endian, Endianness, Pod, U32}; + +use super::{ + CompressionHeader, Dyn, ElfComdat, ElfComdatIterator, ElfDynamicRelocationIterator, ElfSection, + ElfSectionIterator, ElfSegment, ElfSegmentIterator, ElfSymbol, ElfSymbolIterator, + ElfSymbolTable, NoteHeader, ProgramHeader, Rel, Rela, RelocationSections, SectionHeader, + SectionTable, Sym, SymbolTable, +}; + +/// A 32-bit ELF object file. +pub type ElfFile32<'data, Endian = Endianness, R = &'data [u8]> = + ElfFile<'data, elf::FileHeader32, R>; +/// A 64-bit ELF object file. +pub type ElfFile64<'data, Endian = Endianness, R = &'data [u8]> = + ElfFile<'data, elf::FileHeader64, R>; + +/// A partially parsed ELF file. +/// +/// Most of the functionality of this type is provided by the `Object` trait implementation. +#[derive(Debug)] +pub struct ElfFile<'data, Elf, R = &'data [u8]> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ + pub(super) endian: Elf::Endian, + pub(super) data: R, + pub(super) header: &'data Elf, + pub(super) segments: &'data [Elf::ProgramHeader], + pub(super) sections: SectionTable<'data, Elf, R>, + pub(super) relocations: RelocationSections, + pub(super) symbols: SymbolTable<'data, Elf, R>, + pub(super) dynamic_symbols: SymbolTable<'data, Elf, R>, +} + +impl<'data, Elf, R> ElfFile<'data, Elf, R> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ + /// Parse the raw ELF file data. + pub fn parse(data: R) -> read::Result { + let header = Elf::parse(data)?; + let endian = header.endian()?; + let segments = header.program_headers(endian, data)?; + let sections = header.sections(endian, data)?; + let symbols = sections.symbols(endian, data, elf::SHT_SYMTAB)?; + // TODO: get dynamic symbols from DT_SYMTAB if there are no sections + let dynamic_symbols = sections.symbols(endian, data, elf::SHT_DYNSYM)?; + // The API we provide requires a mapping from section to relocations, so build it now. + let relocations = sections.relocation_sections(endian, symbols.section())?; + + Ok(ElfFile { + endian, + data, + header, + segments, + sections, + relocations, + symbols, + dynamic_symbols, + }) + } + + /// Returns the endianness. + pub fn endian(&self) -> Elf::Endian { + self.endian + } + + /// Returns the raw data. + pub fn data(&self) -> R { + self.data + } + + /// Returns the raw ELF file header. + pub fn raw_header(&self) -> &'data Elf { + self.header + } + + /// Returns the raw ELF segments. + pub fn raw_segments(&self) -> &'data [Elf::ProgramHeader] { + self.segments + } + + fn raw_section_by_name<'file>( + &'file self, + section_name: &[u8], + ) -> Option> { + self.sections + .section_by_name(self.endian, section_name) + .map(|(index, section)| ElfSection { + file: self, + index: SectionIndex(index), + section, + }) + } + + #[cfg(feature = "compression")] + fn zdebug_section_by_name<'file>( + &'file self, + section_name: &[u8], + ) -> Option> { + if !section_name.starts_with(b".debug_") { + return None; + } + let mut name = Vec::with_capacity(section_name.len() + 1); + name.extend_from_slice(b".zdebug_"); + name.extend_from_slice(§ion_name[7..]); + self.raw_section_by_name(&name) + } + + #[cfg(not(feature = "compression"))] + fn zdebug_section_by_name<'file>( + &'file self, + _section_name: &[u8], + ) -> Option> { + None + } +} + +impl<'data, Elf, R> read::private::Sealed for ElfFile<'data, Elf, R> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ +} + +impl<'data, 'file, Elf, R> Object<'data, 'file> for ElfFile<'data, Elf, R> +where + 'data: 'file, + Elf: FileHeader, + R: 'file + ReadRef<'data>, +{ + type Segment = ElfSegment<'data, 'file, Elf, R>; + type SegmentIterator = ElfSegmentIterator<'data, 'file, Elf, R>; + type Section = ElfSection<'data, 'file, Elf, R>; + type SectionIterator = ElfSectionIterator<'data, 'file, Elf, R>; + type Comdat = ElfComdat<'data, 'file, Elf, R>; + type ComdatIterator = ElfComdatIterator<'data, 'file, Elf, R>; + type Symbol = ElfSymbol<'data, 'file, Elf, R>; + type SymbolIterator = ElfSymbolIterator<'data, 'file, Elf, R>; + type SymbolTable = ElfSymbolTable<'data, 'file, Elf, R>; + type DynamicRelocationIterator = ElfDynamicRelocationIterator<'data, 'file, Elf, R>; + + fn architecture(&self) -> Architecture { + match ( + self.header.e_machine(self.endian), + self.header.is_class_64(), + ) { + (elf::EM_AARCH64, _) => Architecture::Aarch64, + (elf::EM_ARM, _) => Architecture::Arm, + (elf::EM_AVR, _) => Architecture::Avr, + (elf::EM_BPF, _) => Architecture::Bpf, + (elf::EM_386, _) => Architecture::I386, + (elf::EM_X86_64, false) => Architecture::X86_64_X32, + (elf::EM_X86_64, true) => Architecture::X86_64, + (elf::EM_HEXAGON, _) => Architecture::Hexagon, + (elf::EM_LOONGARCH, true) => Architecture::LoongArch64, + (elf::EM_MIPS, false) => Architecture::Mips, + (elf::EM_MIPS, true) => Architecture::Mips64, + (elf::EM_MSP430, _) => Architecture::Msp430, + (elf::EM_PPC, _) => Architecture::PowerPc, + (elf::EM_PPC64, _) => Architecture::PowerPc64, + (elf::EM_RISCV, false) => Architecture::Riscv32, + (elf::EM_RISCV, true) => Architecture::Riscv64, + // This is either s390 or s390x, depending on the ELF class. + // We only support the 64-bit variant s390x here. + (elf::EM_S390, true) => Architecture::S390x, + (elf::EM_SPARCV9, true) => Architecture::Sparc64, + _ => Architecture::Unknown, + } + } + + #[inline] + fn is_little_endian(&self) -> bool { + self.header.is_little_endian() + } + + #[inline] + fn is_64(&self) -> bool { + self.header.is_class_64() + } + + fn kind(&self) -> ObjectKind { + match self.header.e_type(self.endian) { + elf::ET_REL => ObjectKind::Relocatable, + elf::ET_EXEC => ObjectKind::Executable, + // TODO: check for `DF_1_PIE`? + elf::ET_DYN => ObjectKind::Dynamic, + elf::ET_CORE => ObjectKind::Core, + _ => ObjectKind::Unknown, + } + } + + fn segments(&'file self) -> ElfSegmentIterator<'data, 'file, Elf, R> { + ElfSegmentIterator { + file: self, + iter: self.segments.iter(), + } + } + + fn section_by_name_bytes( + &'file self, + section_name: &[u8], + ) -> Option> { + self.raw_section_by_name(section_name) + .or_else(|| self.zdebug_section_by_name(section_name)) + } + + fn section_by_index( + &'file self, + index: SectionIndex, + ) -> read::Result> { + let section = self.sections.section(index)?; + Ok(ElfSection { + file: self, + index, + section, + }) + } + + fn sections(&'file self) -> ElfSectionIterator<'data, 'file, Elf, R> { + ElfSectionIterator { + file: self, + iter: self.sections.iter().enumerate(), + } + } + + fn comdats(&'file self) -> ElfComdatIterator<'data, 'file, Elf, R> { + ElfComdatIterator { + file: self, + iter: self.sections.iter().enumerate(), + } + } + + fn symbol_by_index( + &'file self, + index: SymbolIndex, + ) -> read::Result> { + let symbol = self.symbols.symbol(index.0)?; + Ok(ElfSymbol { + endian: self.endian, + symbols: &self.symbols, + index, + symbol, + }) + } + + fn symbols(&'file self) -> ElfSymbolIterator<'data, 'file, Elf, R> { + ElfSymbolIterator { + endian: self.endian, + symbols: &self.symbols, + index: 0, + } + } + + fn symbol_table(&'file self) -> Option> { + if self.symbols.is_empty() { + return None; + } + Some(ElfSymbolTable { + endian: self.endian, + symbols: &self.symbols, + }) + } + + fn dynamic_symbols(&'file self) -> ElfSymbolIterator<'data, 'file, Elf, R> { + ElfSymbolIterator { + endian: self.endian, + symbols: &self.dynamic_symbols, + index: 0, + } + } + + fn dynamic_symbol_table(&'file self) -> Option> { + if self.dynamic_symbols.is_empty() { + return None; + } + Some(ElfSymbolTable { + endian: self.endian, + symbols: &self.dynamic_symbols, + }) + } + + fn dynamic_relocations( + &'file self, + ) -> Option> { + Some(ElfDynamicRelocationIterator { + section_index: SectionIndex(1), + file: self, + relocations: None, + }) + } + + /// Get the imported symbols. + fn imports(&self) -> read::Result>> { + let mut imports = Vec::new(); + for symbol in self.dynamic_symbols.iter() { + if symbol.is_undefined(self.endian) { + let name = symbol.name(self.endian, self.dynamic_symbols.strings())?; + if !name.is_empty() { + // TODO: use symbol versioning to determine library + imports.push(Import { + name: ByteString(name), + library: ByteString(&[]), + }); + } + } + } + Ok(imports) + } + + /// Get the exported symbols. + fn exports(&self) -> read::Result>> { + let mut exports = Vec::new(); + for symbol in self.dynamic_symbols.iter() { + if symbol.is_definition(self.endian) { + let name = symbol.name(self.endian, self.dynamic_symbols.strings())?; + let address = symbol.st_value(self.endian).into(); + exports.push(Export { + name: ByteString(name), + address, + }); + } + } + Ok(exports) + } + + fn has_debug_symbols(&self) -> bool { + for section in self.sections.iter() { + if let Ok(name) = self.sections.section_name(self.endian, section) { + if name == b".debug_info" || name == b".zdebug_info" { + return true; + } + } + } + false + } + + fn build_id(&self) -> read::Result> { + let endian = self.endian; + // Use section headers if present, otherwise use program headers. + if !self.sections.is_empty() { + for section in self.sections.iter() { + if let Some(mut notes) = section.notes(endian, self.data)? { + while let Some(note) = notes.next()? { + if note.name() == elf::ELF_NOTE_GNU + && note.n_type(endian) == elf::NT_GNU_BUILD_ID + { + return Ok(Some(note.desc())); + } + } + } + } + } else { + for segment in self.segments { + if let Some(mut notes) = segment.notes(endian, self.data)? { + while let Some(note) = notes.next()? { + if note.name() == elf::ELF_NOTE_GNU + && note.n_type(endian) == elf::NT_GNU_BUILD_ID + { + return Ok(Some(note.desc())); + } + } + } + } + } + Ok(None) + } + + fn gnu_debuglink(&self) -> read::Result> { + let section = match self.raw_section_by_name(b".gnu_debuglink") { + Some(section) => section, + None => return Ok(None), + }; + let data = section + .section + .data(self.endian, self.data) + .read_error("Invalid ELF .gnu_debuglink section offset or size") + .map(Bytes)?; + let filename = data + .read_string_at(0) + .read_error("Missing ELF .gnu_debuglink filename")?; + let crc_offset = util::align(filename.len() + 1, 4); + let crc = data + .read_at::>(crc_offset) + .read_error("Missing ELF .gnu_debuglink crc")? + .get(self.endian); + Ok(Some((filename, crc))) + } + + fn gnu_debugaltlink(&self) -> read::Result> { + let section = match self.raw_section_by_name(b".gnu_debugaltlink") { + Some(section) => section, + None => return Ok(None), + }; + let mut data = section + .section + .data(self.endian, self.data) + .read_error("Invalid ELF .gnu_debugaltlink section offset or size") + .map(Bytes)?; + let filename = data + .read_string() + .read_error("Missing ELF .gnu_debugaltlink filename")?; + let build_id = data.0; + Ok(Some((filename, build_id))) + } + + fn relative_address_base(&self) -> u64 { + 0 + } + + fn entry(&self) -> u64 { + self.header.e_entry(self.endian).into() + } + + fn flags(&self) -> FileFlags { + FileFlags::Elf { + os_abi: self.header.e_ident().os_abi, + abi_version: self.header.e_ident().abi_version, + e_flags: self.header.e_flags(self.endian), + } + } +} + +/// A trait for generic access to `FileHeader32` and `FileHeader64`. +#[allow(missing_docs)] +pub trait FileHeader: Debug + Pod { + // Ideally this would be a `u64: From`, but can't express that. + type Word: Into; + type Sword: Into; + type Endian: endian::Endian; + type ProgramHeader: ProgramHeader; + type SectionHeader: SectionHeader; + type CompressionHeader: CompressionHeader; + type NoteHeader: NoteHeader; + type Dyn: Dyn; + type Sym: Sym; + type Rel: Rel; + type Rela: Rela + From; + + /// Return true if this type is a 64-bit header. + /// + /// This is a property of the type, not a value in the header data. + fn is_type_64(&self) -> bool; + + fn e_ident(&self) -> &elf::Ident; + fn e_type(&self, endian: Self::Endian) -> u16; + fn e_machine(&self, endian: Self::Endian) -> u16; + fn e_version(&self, endian: Self::Endian) -> u32; + fn e_entry(&self, endian: Self::Endian) -> Self::Word; + fn e_phoff(&self, endian: Self::Endian) -> Self::Word; + fn e_shoff(&self, endian: Self::Endian) -> Self::Word; + fn e_flags(&self, endian: Self::Endian) -> u32; + fn e_ehsize(&self, endian: Self::Endian) -> u16; + fn e_phentsize(&self, endian: Self::Endian) -> u16; + fn e_phnum(&self, endian: Self::Endian) -> u16; + fn e_shentsize(&self, endian: Self::Endian) -> u16; + fn e_shnum(&self, endian: Self::Endian) -> u16; + fn e_shstrndx(&self, endian: Self::Endian) -> u16; + + // Provided methods. + + /// Read the file header. + /// + /// Also checks that the ident field in the file header is a supported format. + fn parse<'data, R: ReadRef<'data>>(data: R) -> read::Result<&'data Self> { + let header = data + .read_at::(0) + .read_error("Invalid ELF header size or alignment")?; + if !header.is_supported() { + return Err(Error("Unsupported ELF header")); + } + // TODO: Check self.e_ehsize? + Ok(header) + } + + /// Check that the ident field in the file header is a supported format. + /// + /// This checks the magic number, version, class, and endianness. + fn is_supported(&self) -> bool { + let ident = self.e_ident(); + // TODO: Check self.e_version too? Requires endian though. + ident.magic == elf::ELFMAG + && (self.is_type_64() || self.is_class_32()) + && (!self.is_type_64() || self.is_class_64()) + && (self.is_little_endian() || self.is_big_endian()) + && ident.version == elf::EV_CURRENT + } + + fn is_class_32(&self) -> bool { + self.e_ident().class == elf::ELFCLASS32 + } + + fn is_class_64(&self) -> bool { + self.e_ident().class == elf::ELFCLASS64 + } + + fn is_little_endian(&self) -> bool { + self.e_ident().data == elf::ELFDATA2LSB + } + + fn is_big_endian(&self) -> bool { + self.e_ident().data == elf::ELFDATA2MSB + } + + fn endian(&self) -> read::Result { + Self::Endian::from_big_endian(self.is_big_endian()).read_error("Unsupported ELF endian") + } + + /// Return the first section header, if present. + /// + /// Section 0 is a special case because getting the section headers normally + /// requires `shnum`, but `shnum` may be in the first section header. + fn section_0<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result> { + let shoff: u64 = self.e_shoff(endian).into(); + if shoff == 0 { + // No section headers is ok. + return Ok(None); + } + let shentsize = usize::from(self.e_shentsize(endian)); + if shentsize != mem::size_of::() { + // Section header size must match. + return Err(Error("Invalid ELF section header entry size")); + } + data.read_at(shoff) + .map(Some) + .read_error("Invalid ELF section header offset or size") + } + + /// Return the `e_phnum` field of the header. Handles extended values. + /// + /// Returns `Err` for invalid values. + fn phnum<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result { + let e_phnum = self.e_phnum(endian); + if e_phnum < elf::PN_XNUM { + Ok(e_phnum as usize) + } else if let Some(section_0) = self.section_0(endian, data)? { + Ok(section_0.sh_info(endian) as usize) + } else { + // Section 0 must exist if e_phnum overflows. + Err(Error("Missing ELF section headers for e_phnum overflow")) + } + } + + /// Return the `e_shnum` field of the header. Handles extended values. + /// + /// Returns `Err` for invalid values. + fn shnum<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result { + let e_shnum = self.e_shnum(endian); + if e_shnum > 0 { + Ok(e_shnum as usize) + } else if let Some(section_0) = self.section_0(endian, data)? { + section_0 + .sh_size(endian) + .into() + .try_into() + .ok() + .read_error("Invalid ELF extended e_shnum") + } else { + // No section headers is ok. + Ok(0) + } + } + + /// Return the `e_shstrndx` field of the header. Handles extended values. + /// + /// Returns `Err` for invalid values (including if the index is 0). + fn shstrndx<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result { + let e_shstrndx = self.e_shstrndx(endian); + let index = if e_shstrndx != elf::SHN_XINDEX { + e_shstrndx.into() + } else if let Some(section_0) = self.section_0(endian, data)? { + section_0.sh_link(endian) + } else { + // Section 0 must exist if we're trying to read e_shstrndx. + return Err(Error("Missing ELF section headers for e_shstrndx overflow")); + }; + if index == 0 { + return Err(Error("Missing ELF e_shstrndx")); + } + Ok(index) + } + + /// Return the slice of program headers. + /// + /// Returns `Ok(&[])` if there are no program headers. + /// Returns `Err` for invalid values. + fn program_headers<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result<&'data [Self::ProgramHeader]> { + let phoff: u64 = self.e_phoff(endian).into(); + if phoff == 0 { + // No program headers is ok. + return Ok(&[]); + } + let phnum = self.phnum(endian, data)?; + if phnum == 0 { + // No program headers is ok. + return Ok(&[]); + } + let phentsize = self.e_phentsize(endian) as usize; + if phentsize != mem::size_of::() { + // Program header size must match. + return Err(Error("Invalid ELF program header entry size")); + } + data.read_slice_at(phoff, phnum) + .read_error("Invalid ELF program header size or alignment") + } + + /// Return the slice of section headers. + /// + /// Returns `Ok(&[])` if there are no section headers. + /// Returns `Err` for invalid values. + fn section_headers<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result<&'data [Self::SectionHeader]> { + let shoff: u64 = self.e_shoff(endian).into(); + if shoff == 0 { + // No section headers is ok. + return Ok(&[]); + } + let shnum = self.shnum(endian, data)?; + if shnum == 0 { + // No section headers is ok. + return Ok(&[]); + } + let shentsize = usize::from(self.e_shentsize(endian)); + if shentsize != mem::size_of::() { + // Section header size must match. + return Err(Error("Invalid ELF section header entry size")); + } + data.read_slice_at(shoff, shnum) + .read_error("Invalid ELF section header offset/size/alignment") + } + + /// Return the string table for the section headers. + fn section_strings<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + sections: &[Self::SectionHeader], + ) -> read::Result> { + if sections.is_empty() { + return Ok(StringTable::default()); + } + let index = self.shstrndx(endian, data)? as usize; + let shstrtab = sections.get(index).read_error("Invalid ELF e_shstrndx")?; + let strings = if let Some((shstrtab_offset, shstrtab_size)) = shstrtab.file_range(endian) { + let shstrtab_end = shstrtab_offset + .checked_add(shstrtab_size) + .read_error("Invalid ELF shstrtab size")?; + StringTable::new(data, shstrtab_offset, shstrtab_end) + } else { + StringTable::default() + }; + Ok(strings) + } + + /// Return the section table. + fn sections<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result> { + let sections = self.section_headers(endian, data)?; + let strings = self.section_strings(endian, data, sections)?; + Ok(SectionTable::new(sections, strings)) + } + + /// Returns whether this is a mips64el elf file. + fn is_mips64el(&self, endian: Self::Endian) -> bool { + self.is_class_64() && self.is_little_endian() && self.e_machine(endian) == elf::EM_MIPS + } +} + +impl FileHeader for elf::FileHeader32 { + type Word = u32; + type Sword = i32; + type Endian = Endian; + type ProgramHeader = elf::ProgramHeader32; + type SectionHeader = elf::SectionHeader32; + type CompressionHeader = elf::CompressionHeader32; + type NoteHeader = elf::NoteHeader32; + type Dyn = elf::Dyn32; + type Sym = elf::Sym32; + type Rel = elf::Rel32; + type Rela = elf::Rela32; + + #[inline] + fn is_type_64(&self) -> bool { + false + } + + #[inline] + fn e_ident(&self) -> &elf::Ident { + &self.e_ident + } + + #[inline] + fn e_type(&self, endian: Self::Endian) -> u16 { + self.e_type.get(endian) + } + + #[inline] + fn e_machine(&self, endian: Self::Endian) -> u16 { + self.e_machine.get(endian) + } + + #[inline] + fn e_version(&self, endian: Self::Endian) -> u32 { + self.e_version.get(endian) + } + + #[inline] + fn e_entry(&self, endian: Self::Endian) -> Self::Word { + self.e_entry.get(endian) + } + + #[inline] + fn e_phoff(&self, endian: Self::Endian) -> Self::Word { + self.e_phoff.get(endian) + } + + #[inline] + fn e_shoff(&self, endian: Self::Endian) -> Self::Word { + self.e_shoff.get(endian) + } + + #[inline] + fn e_flags(&self, endian: Self::Endian) -> u32 { + self.e_flags.get(endian) + } + + #[inline] + fn e_ehsize(&self, endian: Self::Endian) -> u16 { + self.e_ehsize.get(endian) + } + + #[inline] + fn e_phentsize(&self, endian: Self::Endian) -> u16 { + self.e_phentsize.get(endian) + } + + #[inline] + fn e_phnum(&self, endian: Self::Endian) -> u16 { + self.e_phnum.get(endian) + } + + #[inline] + fn e_shentsize(&self, endian: Self::Endian) -> u16 { + self.e_shentsize.get(endian) + } + + #[inline] + fn e_shnum(&self, endian: Self::Endian) -> u16 { + self.e_shnum.get(endian) + } + + #[inline] + fn e_shstrndx(&self, endian: Self::Endian) -> u16 { + self.e_shstrndx.get(endian) + } +} + +impl FileHeader for elf::FileHeader64 { + type Word = u64; + type Sword = i64; + type Endian = Endian; + type ProgramHeader = elf::ProgramHeader64; + type SectionHeader = elf::SectionHeader64; + type CompressionHeader = elf::CompressionHeader64; + type NoteHeader = elf::NoteHeader32; + type Dyn = elf::Dyn64; + type Sym = elf::Sym64; + type Rel = elf::Rel64; + type Rela = elf::Rela64; + + #[inline] + fn is_type_64(&self) -> bool { + true + } + + #[inline] + fn e_ident(&self) -> &elf::Ident { + &self.e_ident + } + + #[inline] + fn e_type(&self, endian: Self::Endian) -> u16 { + self.e_type.get(endian) + } + + #[inline] + fn e_machine(&self, endian: Self::Endian) -> u16 { + self.e_machine.get(endian) + } + + #[inline] + fn e_version(&self, endian: Self::Endian) -> u32 { + self.e_version.get(endian) + } + + #[inline] + fn e_entry(&self, endian: Self::Endian) -> Self::Word { + self.e_entry.get(endian) + } + + #[inline] + fn e_phoff(&self, endian: Self::Endian) -> Self::Word { + self.e_phoff.get(endian) + } + + #[inline] + fn e_shoff(&self, endian: Self::Endian) -> Self::Word { + self.e_shoff.get(endian) + } + + #[inline] + fn e_flags(&self, endian: Self::Endian) -> u32 { + self.e_flags.get(endian) + } + + #[inline] + fn e_ehsize(&self, endian: Self::Endian) -> u16 { + self.e_ehsize.get(endian) + } + + #[inline] + fn e_phentsize(&self, endian: Self::Endian) -> u16 { + self.e_phentsize.get(endian) + } + + #[inline] + fn e_phnum(&self, endian: Self::Endian) -> u16 { + self.e_phnum.get(endian) + } + + #[inline] + fn e_shentsize(&self, endian: Self::Endian) -> u16 { + self.e_shentsize.get(endian) + } + + #[inline] + fn e_shnum(&self, endian: Self::Endian) -> u16 { + self.e_shnum.get(endian) + } + + #[inline] + fn e_shstrndx(&self, endian: Self::Endian) -> u16 { + self.e_shstrndx.get(endian) + } +} diff --git a/crux-mir/lib/object/src/read/elf/hash.rs b/crux-mir/lib/object/src/read/elf/hash.rs new file mode 100644 index 000000000..aa1039ac1 --- /dev/null +++ b/crux-mir/lib/object/src/read/elf/hash.rs @@ -0,0 +1,220 @@ +use core::mem; + +use crate::elf; +use crate::read::{ReadError, ReadRef, Result}; +use crate::{U32, U64}; + +use super::{FileHeader, Sym, SymbolTable, Version, VersionTable}; + +/// A SysV symbol hash table in an ELF file. +#[derive(Debug)] +pub struct HashTable<'data, Elf: FileHeader> { + buckets: &'data [U32], + chains: &'data [U32], +} + +impl<'data, Elf: FileHeader> HashTable<'data, Elf> { + /// Parse a SysV hash table. + /// + /// `data` should be from a `SHT_HASH` section, or from a + /// segment pointed to via the `DT_HASH` entry. + /// + /// The header is read at offset 0 in the given `data`. + pub fn parse(endian: Elf::Endian, data: &'data [u8]) -> Result { + let mut offset = 0; + let header = data + .read::>(&mut offset) + .read_error("Invalid hash header")?; + let buckets = data + .read_slice(&mut offset, header.bucket_count.get(endian) as usize) + .read_error("Invalid hash buckets")?; + let chains = data + .read_slice(&mut offset, header.chain_count.get(endian) as usize) + .read_error("Invalid hash chains")?; + Ok(HashTable { buckets, chains }) + } + + /// Return the symbol table length. + pub fn symbol_table_length(&self) -> u32 { + self.chains.len() as u32 + } + + /// Use the hash table to find the symbol table entry with the given name, hash and version. + pub fn find>( + &self, + endian: Elf::Endian, + name: &[u8], + hash: u32, + version: Option<&Version>, + symbols: &SymbolTable<'data, Elf, R>, + versions: &VersionTable<'data, Elf>, + ) -> Option<(usize, &'data Elf::Sym)> { + // Get the chain start from the bucket for this hash. + let mut index = self.buckets[(hash as usize) % self.buckets.len()].get(endian) as usize; + // Avoid infinite loop. + let mut i = 0; + let strings = symbols.strings(); + while index != 0 && i < self.chains.len() { + if let Ok(symbol) = symbols.symbol(index) { + if symbol.name(endian, strings) == Ok(name) + && versions.matches(endian, index, version) + { + return Some((index, symbol)); + } + } + index = self.chains.get(index)?.get(endian) as usize; + i += 1; + } + None + } +} + +/// A GNU symbol hash table in an ELF file. +#[derive(Debug)] +pub struct GnuHashTable<'data, Elf: FileHeader> { + symbol_base: u32, + bloom_shift: u32, + bloom_filters: &'data [u8], + buckets: &'data [U32], + values: &'data [U32], +} + +impl<'data, Elf: FileHeader> GnuHashTable<'data, Elf> { + /// Parse a GNU hash table. + /// + /// `data` should be from a `SHT_GNU_HASH` section, or from a + /// segment pointed to via the `DT_GNU_HASH` entry. + /// + /// The header is read at offset 0 in the given `data`. + /// + /// The header does not contain a length field, and so all of `data` + /// will be used as the hash table values. It does not matter if this + /// is longer than needed, and this will often the case when accessing + /// the hash table via the `DT_GNU_HASH` entry. + pub fn parse(endian: Elf::Endian, data: &'data [u8]) -> Result { + let mut offset = 0; + let header = data + .read::>(&mut offset) + .read_error("Invalid GNU hash header")?; + let bloom_len = + u64::from(header.bloom_count.get(endian)) * mem::size_of::() as u64; + let bloom_filters = data + .read_bytes(&mut offset, bloom_len) + .read_error("Invalid GNU hash bloom filters")?; + let buckets = data + .read_slice(&mut offset, header.bucket_count.get(endian) as usize) + .read_error("Invalid GNU hash buckets")?; + let chain_count = (data.len() - offset as usize) / 4; + let values = data + .read_slice(&mut offset, chain_count) + .read_error("Invalid GNU hash values")?; + Ok(GnuHashTable { + symbol_base: header.symbol_base.get(endian), + bloom_shift: header.bloom_shift.get(endian), + bloom_filters, + buckets, + values, + }) + } + + /// Return the symbol table index of the first symbol in the hash table. + pub fn symbol_base(&self) -> u32 { + self.symbol_base + } + + /// Determine the symbol table length by finding the last entry in the hash table. + /// + /// Returns `None` if the hash table is empty or invalid. + pub fn symbol_table_length(&self, endian: Elf::Endian) -> Option { + // Ensure we find a non-empty bucket. + if self.symbol_base == 0 { + return None; + } + + // Find the highest chain index in a bucket. + let mut max_symbol = 0; + for bucket in self.buckets { + let bucket = bucket.get(endian); + if max_symbol < bucket { + max_symbol = bucket; + } + } + + // Find the end of the chain. + for value in self + .values + .get(max_symbol.checked_sub(self.symbol_base)? as usize..)? + { + max_symbol += 1; + if value.get(endian) & 1 != 0 { + return Some(max_symbol); + } + } + + None + } + + /// Use the hash table to find the symbol table entry with the given name, hash, and version. + pub fn find>( + &self, + endian: Elf::Endian, + name: &[u8], + hash: u32, + version: Option<&Version>, + symbols: &SymbolTable<'data, Elf, R>, + versions: &VersionTable<'data, Elf>, + ) -> Option<(usize, &'data Elf::Sym)> { + let word_bits = mem::size_of::() as u32 * 8; + + // Test against bloom filter. + let bloom_count = self.bloom_filters.len() / mem::size_of::(); + let offset = + ((hash / word_bits) & (bloom_count as u32 - 1)) * mem::size_of::() as u32; + let filter = if word_bits == 64 { + self.bloom_filters + .read_at::>(offset.into()) + .ok()? + .get(endian) + } else { + self.bloom_filters + .read_at::>(offset.into()) + .ok()? + .get(endian) + .into() + }; + if filter & (1 << (hash % word_bits)) == 0 { + return None; + } + if filter & (1 << ((hash >> self.bloom_shift) % word_bits)) == 0 { + return None; + } + + // Get the chain start from the bucket for this hash. + let mut index = self.buckets[(hash as usize) % self.buckets.len()].get(endian) as usize; + if index == 0 { + return None; + } + + // Test symbols in the chain. + let strings = symbols.strings(); + let symbols = symbols.symbols().get(index..)?; + let values = self + .values + .get(index.checked_sub(self.symbol_base as usize)?..)?; + for (symbol, value) in symbols.iter().zip(values.iter()) { + let value = value.get(endian); + if value | 1 == hash | 1 { + if symbol.name(endian, strings) == Ok(name) + && versions.matches(endian, index, version) + { + return Some((index, symbol)); + } + } + if value & 1 != 0 { + break; + } + index += 1; + } + None + } +} diff --git a/crux-mir/lib/object/src/read/elf/mod.rs b/crux-mir/lib/object/src/read/elf/mod.rs new file mode 100644 index 000000000..5b7d7f9f7 --- /dev/null +++ b/crux-mir/lib/object/src/read/elf/mod.rs @@ -0,0 +1,39 @@ +//! Support for reading ELF files. +//! +//! Defines traits to abstract over the difference between ELF32/ELF64, +//! and implements read functionality in terms of these traits. +//! +//! Also provides `ElfFile` and related types which implement the `Object` trait. + +mod file; +pub use file::*; + +mod segment; +pub use segment::*; + +mod section; +pub use section::*; + +mod symbol; +pub use symbol::*; + +mod relocation; +pub use relocation::*; + +mod comdat; +pub use comdat::*; + +mod dynamic; +pub use dynamic::*; + +mod compression; +pub use compression::*; + +mod note; +pub use note::*; + +mod hash; +pub use hash::*; + +mod version; +pub use version::*; diff --git a/crux-mir/lib/object/src/read/elf/note.rs b/crux-mir/lib/object/src/read/elf/note.rs new file mode 100644 index 000000000..34024dbb8 --- /dev/null +++ b/crux-mir/lib/object/src/read/elf/note.rs @@ -0,0 +1,185 @@ +use core::fmt::Debug; +use core::mem; + +use crate::elf; +use crate::endian; +use crate::pod::Pod; +use crate::read::util; +use crate::read::{self, Bytes, Error, ReadError}; + +use super::FileHeader; + +/// An iterator over the notes in an ELF section or segment. +#[derive(Debug)] +pub struct NoteIterator<'data, Elf> +where + Elf: FileHeader, +{ + endian: Elf::Endian, + align: usize, + data: Bytes<'data>, +} + +impl<'data, Elf> NoteIterator<'data, Elf> +where + Elf: FileHeader, +{ + /// Returns `Err` if `align` is invalid. + pub(super) fn new( + endian: Elf::Endian, + align: Elf::Word, + data: &'data [u8], + ) -> read::Result { + let align = match align.into() { + 0u64..=4 => 4, + 8 => 8, + _ => return Err(Error("Invalid ELF note alignment")), + }; + // TODO: check data alignment? + Ok(NoteIterator { + endian, + align, + data: Bytes(data), + }) + } + + /// Returns the next note. + pub fn next(&mut self) -> read::Result>> { + let mut data = self.data; + if data.is_empty() { + return Ok(None); + } + + let header = data + .read_at::(0) + .read_error("ELF note is too short")?; + + // The name has no alignment requirement. + let offset = mem::size_of::(); + let namesz = header.n_namesz(self.endian) as usize; + let name = data + .read_bytes_at(offset, namesz) + .read_error("Invalid ELF note namesz")? + .0; + + // The descriptor must be aligned. + let offset = util::align(offset + namesz, self.align); + let descsz = header.n_descsz(self.endian) as usize; + let desc = data + .read_bytes_at(offset, descsz) + .read_error("Invalid ELF note descsz")? + .0; + + // The next note (if any) must be aligned. + let offset = util::align(offset + descsz, self.align); + if data.skip(offset).is_err() { + data = Bytes(&[]); + } + self.data = data; + + Ok(Some(Note { header, name, desc })) + } +} + +/// A parsed `NoteHeader`. +#[derive(Debug)] +pub struct Note<'data, Elf> +where + Elf: FileHeader, +{ + header: &'data Elf::NoteHeader, + name: &'data [u8], + desc: &'data [u8], +} + +impl<'data, Elf: FileHeader> Note<'data, Elf> { + /// Return the `n_type` field of the `NoteHeader`. + /// + /// The meaning of this field is determined by `name`. + pub fn n_type(&self, endian: Elf::Endian) -> u32 { + self.header.n_type(endian) + } + + /// Return the `n_namesz` field of the `NoteHeader`. + pub fn n_namesz(&self, endian: Elf::Endian) -> u32 { + self.header.n_namesz(endian) + } + + /// Return the `n_descsz` field of the `NoteHeader`. + pub fn n_descsz(&self, endian: Elf::Endian) -> u32 { + self.header.n_descsz(endian) + } + + /// Return the bytes for the name field following the `NoteHeader`, + /// excluding any null terminator. + /// + /// This field is usually a string including a null terminator + /// (but it is not required to be). + /// + /// The length of this field (including any null terminator) is given by + /// `n_namesz`. + pub fn name(&self) -> &'data [u8] { + if let Some((last, name)) = self.name.split_last() { + if *last == 0 { + return name; + } + } + self.name + } + + /// Return the bytes for the desc field following the `NoteHeader`. + /// + /// The length of this field is given by `n_descsz`. The meaning + /// of this field is determined by `name` and `n_type`. + pub fn desc(&self) -> &'data [u8] { + self.desc + } +} + +/// A trait for generic access to `NoteHeader32` and `NoteHeader64`. +#[allow(missing_docs)] +pub trait NoteHeader: Debug + Pod { + type Endian: endian::Endian; + + fn n_namesz(&self, endian: Self::Endian) -> u32; + fn n_descsz(&self, endian: Self::Endian) -> u32; + fn n_type(&self, endian: Self::Endian) -> u32; +} + +impl NoteHeader for elf::NoteHeader32 { + type Endian = Endian; + + #[inline] + fn n_namesz(&self, endian: Self::Endian) -> u32 { + self.n_namesz.get(endian) + } + + #[inline] + fn n_descsz(&self, endian: Self::Endian) -> u32 { + self.n_descsz.get(endian) + } + + #[inline] + fn n_type(&self, endian: Self::Endian) -> u32 { + self.n_type.get(endian) + } +} + +impl NoteHeader for elf::NoteHeader64 { + type Endian = Endian; + + #[inline] + fn n_namesz(&self, endian: Self::Endian) -> u32 { + self.n_namesz.get(endian) + } + + #[inline] + fn n_descsz(&self, endian: Self::Endian) -> u32 { + self.n_descsz.get(endian) + } + + #[inline] + fn n_type(&self, endian: Self::Endian) -> u32 { + self.n_type.get(endian) + } +} diff --git a/crux-mir/lib/object/src/read/elf/relocation.rs b/crux-mir/lib/object/src/read/elf/relocation.rs new file mode 100644 index 000000000..557b80efc --- /dev/null +++ b/crux-mir/lib/object/src/read/elf/relocation.rs @@ -0,0 +1,539 @@ +use alloc::fmt; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::slice; + +use crate::elf; +use crate::endian::{self, Endianness}; +use crate::pod::Pod; +use crate::read::{ + self, Error, ReadRef, Relocation, RelocationEncoding, RelocationKind, RelocationTarget, + SectionIndex, SymbolIndex, +}; + +use super::{ElfFile, FileHeader, SectionHeader, SectionTable}; + +/// A mapping from section index to associated relocation sections. +#[derive(Debug)] +pub struct RelocationSections { + relocations: Vec, +} + +impl RelocationSections { + /// Create a new mapping using the section table. + /// + /// Skips relocation sections that do not use the given symbol table section. + pub fn parse<'data, Elf: FileHeader, R: ReadRef<'data>>( + endian: Elf::Endian, + sections: &SectionTable<'data, Elf, R>, + symbol_section: SectionIndex, + ) -> read::Result { + let mut relocations = vec![0; sections.len()]; + for (index, section) in sections.iter().enumerate().rev() { + let sh_type = section.sh_type(endian); + if sh_type == elf::SHT_REL || sh_type == elf::SHT_RELA { + // The symbol indices used in relocations must be for the symbol table + // we are expecting to use. + let sh_link = SectionIndex(section.sh_link(endian) as usize); + if sh_link != symbol_section { + continue; + } + + let sh_info = section.sh_info(endian) as usize; + if sh_info == 0 { + // Skip dynamic relocations. + continue; + } + if sh_info >= relocations.len() { + return Err(Error("Invalid ELF sh_info for relocation section")); + } + + // Handle multiple relocation sections by chaining them. + let next = relocations[sh_info]; + relocations[sh_info] = index; + relocations[index] = next; + } + } + Ok(Self { relocations }) + } + + /// Given a section index, return the section index of the associated relocation section. + /// + /// This may also be called with a relocation section index, and it will return the + /// next associated relocation section. + pub fn get(&self, index: usize) -> Option { + self.relocations.get(index).cloned().filter(|x| *x != 0) + } +} + +pub(super) enum ElfRelaIterator<'data, Elf: FileHeader> { + Rel(slice::Iter<'data, Elf::Rel>), + Rela(slice::Iter<'data, Elf::Rela>), +} + +impl<'data, Elf: FileHeader> ElfRelaIterator<'data, Elf> { + fn is_rel(&self) -> bool { + match self { + ElfRelaIterator::Rel(_) => true, + ElfRelaIterator::Rela(_) => false, + } + } +} + +impl<'data, Elf: FileHeader> Iterator for ElfRelaIterator<'data, Elf> { + type Item = Elf::Rela; + + fn next(&mut self) -> Option { + match self { + ElfRelaIterator::Rel(ref mut i) => i.next().cloned().map(Self::Item::from), + ElfRelaIterator::Rela(ref mut i) => i.next().cloned(), + } + } +} + +/// An iterator over the dynamic relocations for an `ElfFile32`. +pub type ElfDynamicRelocationIterator32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfDynamicRelocationIterator<'data, 'file, elf::FileHeader32, R>; +/// An iterator over the dynamic relocations for an `ElfFile64`. +pub type ElfDynamicRelocationIterator64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfDynamicRelocationIterator<'data, 'file, elf::FileHeader64, R>; + +/// An iterator over the dynamic relocations for an `ElfFile`. +pub struct ElfDynamicRelocationIterator<'data, 'file, Elf, R = &'data [u8]> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ + /// The current relocation section index. + pub(super) section_index: SectionIndex, + pub(super) file: &'file ElfFile<'data, Elf, R>, + pub(super) relocations: Option>, +} + +impl<'data, 'file, Elf, R> Iterator for ElfDynamicRelocationIterator<'data, 'file, Elf, R> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ + type Item = (u64, Relocation); + + fn next(&mut self) -> Option { + let endian = self.file.endian; + loop { + if let Some(ref mut relocations) = self.relocations { + if let Some(reloc) = relocations.next() { + let relocation = + parse_relocation(self.file.header, endian, reloc, relocations.is_rel()); + return Some((reloc.r_offset(endian).into(), relocation)); + } + self.relocations = None; + } + + let section = self.file.sections.section(self.section_index).ok()?; + self.section_index.0 += 1; + + let sh_link = SectionIndex(section.sh_link(endian) as usize); + if sh_link != self.file.dynamic_symbols.section() { + continue; + } + + match section.sh_type(endian) { + elf::SHT_REL => { + if let Ok(relocations) = section.data_as_array(endian, self.file.data) { + self.relocations = Some(ElfRelaIterator::Rel(relocations.iter())); + } + } + elf::SHT_RELA => { + if let Ok(relocations) = section.data_as_array(endian, self.file.data) { + self.relocations = Some(ElfRelaIterator::Rela(relocations.iter())); + } + } + _ => {} + } + } + } +} + +impl<'data, 'file, Elf, R> fmt::Debug for ElfDynamicRelocationIterator<'data, 'file, Elf, R> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ElfDynamicRelocationIterator").finish() + } +} + +/// An iterator over the relocations for an `ElfSection32`. +pub type ElfSectionRelocationIterator32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfSectionRelocationIterator<'data, 'file, elf::FileHeader32, R>; +/// An iterator over the relocations for an `ElfSection64`. +pub type ElfSectionRelocationIterator64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfSectionRelocationIterator<'data, 'file, elf::FileHeader64, R>; + +/// An iterator over the relocations for an `ElfSection`. +pub struct ElfSectionRelocationIterator<'data, 'file, Elf, R = &'data [u8]> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ + /// The current pointer in the chain of relocation sections. + pub(super) section_index: SectionIndex, + pub(super) file: &'file ElfFile<'data, Elf, R>, + pub(super) relocations: Option>, +} + +impl<'data, 'file, Elf, R> Iterator for ElfSectionRelocationIterator<'data, 'file, Elf, R> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ + type Item = (u64, Relocation); + + fn next(&mut self) -> Option { + let endian = self.file.endian; + loop { + if let Some(ref mut relocations) = self.relocations { + if let Some(reloc) = relocations.next() { + let relocation = + parse_relocation(self.file.header, endian, reloc, relocations.is_rel()); + return Some((reloc.r_offset(endian).into(), relocation)); + } + self.relocations = None; + } + self.section_index = SectionIndex(self.file.relocations.get(self.section_index.0)?); + // The construction of RelocationSections ensures section_index is valid. + let section = self.file.sections.section(self.section_index).unwrap(); + match section.sh_type(endian) { + elf::SHT_REL => { + if let Ok(relocations) = section.data_as_array(endian, self.file.data) { + self.relocations = Some(ElfRelaIterator::Rel(relocations.iter())); + } + } + elf::SHT_RELA => { + if let Ok(relocations) = section.data_as_array(endian, self.file.data) { + self.relocations = Some(ElfRelaIterator::Rela(relocations.iter())); + } + } + _ => {} + } + } + } +} + +impl<'data, 'file, Elf, R> fmt::Debug for ElfSectionRelocationIterator<'data, 'file, Elf, R> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ElfSectionRelocationIterator").finish() + } +} + +fn parse_relocation( + header: &Elf, + endian: Elf::Endian, + reloc: Elf::Rela, + implicit_addend: bool, +) -> Relocation { + let mut encoding = RelocationEncoding::Generic; + let is_mips64el = header.is_mips64el(endian); + let (kind, size) = match header.e_machine(endian) { + elf::EM_AARCH64 => match reloc.r_type(endian, false) { + elf::R_AARCH64_ABS64 => (RelocationKind::Absolute, 64), + elf::R_AARCH64_ABS32 => (RelocationKind::Absolute, 32), + elf::R_AARCH64_ABS16 => (RelocationKind::Absolute, 16), + elf::R_AARCH64_PREL64 => (RelocationKind::Relative, 64), + elf::R_AARCH64_PREL32 => (RelocationKind::Relative, 32), + elf::R_AARCH64_PREL16 => (RelocationKind::Relative, 16), + elf::R_AARCH64_CALL26 => { + encoding = RelocationEncoding::AArch64Call; + (RelocationKind::PltRelative, 26) + } + r_type => (RelocationKind::Elf(r_type), 0), + }, + elf::EM_ARM => match reloc.r_type(endian, false) { + elf::R_ARM_ABS32 => (RelocationKind::Absolute, 32), + r_type => (RelocationKind::Elf(r_type), 0), + }, + elf::EM_AVR => match reloc.r_type(endian, false) { + elf::R_AVR_32 => (RelocationKind::Absolute, 32), + elf::R_AVR_16 => (RelocationKind::Absolute, 16), + r_type => (RelocationKind::Elf(r_type), 0), + }, + elf::EM_BPF => match reloc.r_type(endian, false) { + elf::R_BPF_64_64 => (RelocationKind::Absolute, 64), + elf::R_BPF_64_32 => (RelocationKind::Absolute, 32), + r_type => (RelocationKind::Elf(r_type), 0), + }, + elf::EM_386 => match reloc.r_type(endian, false) { + elf::R_386_32 => (RelocationKind::Absolute, 32), + elf::R_386_PC32 => (RelocationKind::Relative, 32), + elf::R_386_GOT32 => (RelocationKind::Got, 32), + elf::R_386_PLT32 => (RelocationKind::PltRelative, 32), + elf::R_386_GOTOFF => (RelocationKind::GotBaseOffset, 32), + elf::R_386_GOTPC => (RelocationKind::GotBaseRelative, 32), + elf::R_386_16 => (RelocationKind::Absolute, 16), + elf::R_386_PC16 => (RelocationKind::Relative, 16), + elf::R_386_8 => (RelocationKind::Absolute, 8), + elf::R_386_PC8 => (RelocationKind::Relative, 8), + r_type => (RelocationKind::Elf(r_type), 0), + }, + elf::EM_X86_64 => match reloc.r_type(endian, false) { + elf::R_X86_64_64 => (RelocationKind::Absolute, 64), + elf::R_X86_64_PC32 => (RelocationKind::Relative, 32), + elf::R_X86_64_GOT32 => (RelocationKind::Got, 32), + elf::R_X86_64_PLT32 => (RelocationKind::PltRelative, 32), + elf::R_X86_64_GOTPCREL => (RelocationKind::GotRelative, 32), + elf::R_X86_64_32 => (RelocationKind::Absolute, 32), + elf::R_X86_64_32S => { + encoding = RelocationEncoding::X86Signed; + (RelocationKind::Absolute, 32) + } + elf::R_X86_64_16 => (RelocationKind::Absolute, 16), + elf::R_X86_64_PC16 => (RelocationKind::Relative, 16), + elf::R_X86_64_8 => (RelocationKind::Absolute, 8), + elf::R_X86_64_PC8 => (RelocationKind::Relative, 8), + r_type => (RelocationKind::Elf(r_type), 0), + }, + elf::EM_HEXAGON => match reloc.r_type(endian, false) { + elf::R_HEX_32 => (RelocationKind::Absolute, 32), + r_type => (RelocationKind::Elf(r_type), 0), + }, + elf::EM_LOONGARCH => match reloc.r_type(endian, false) { + elf::R_LARCH_32 => (RelocationKind::Absolute, 32), + elf::R_LARCH_64 => (RelocationKind::Absolute, 64), + r_type => (RelocationKind::Elf(r_type), 0), + }, + elf::EM_MIPS => match reloc.r_type(endian, is_mips64el) { + elf::R_MIPS_16 => (RelocationKind::Absolute, 16), + elf::R_MIPS_32 => (RelocationKind::Absolute, 32), + elf::R_MIPS_64 => (RelocationKind::Absolute, 64), + r_type => (RelocationKind::Elf(r_type), 0), + }, + elf::EM_MSP430 => match reloc.r_type(endian, false) { + elf::R_MSP430_32 => (RelocationKind::Absolute, 32), + elf::R_MSP430_16_BYTE => (RelocationKind::Absolute, 16), + r_type => (RelocationKind::Elf(r_type), 0), + }, + elf::EM_PPC => match reloc.r_type(endian, false) { + elf::R_PPC_ADDR32 => (RelocationKind::Absolute, 32), + r_type => (RelocationKind::Elf(r_type), 0), + }, + elf::EM_PPC64 => match reloc.r_type(endian, false) { + elf::R_PPC64_ADDR32 => (RelocationKind::Absolute, 32), + elf::R_PPC64_ADDR64 => (RelocationKind::Absolute, 64), + r_type => (RelocationKind::Elf(r_type), 0), + }, + elf::EM_RISCV => match reloc.r_type(endian, false) { + elf::R_RISCV_32 => (RelocationKind::Absolute, 32), + elf::R_RISCV_64 => (RelocationKind::Absolute, 64), + r_type => (RelocationKind::Elf(r_type), 0), + }, + elf::EM_S390 => match reloc.r_type(endian, false) { + elf::R_390_8 => (RelocationKind::Absolute, 8), + elf::R_390_16 => (RelocationKind::Absolute, 16), + elf::R_390_32 => (RelocationKind::Absolute, 32), + elf::R_390_64 => (RelocationKind::Absolute, 64), + elf::R_390_PC16 => (RelocationKind::Relative, 16), + elf::R_390_PC32 => (RelocationKind::Relative, 32), + elf::R_390_PC64 => (RelocationKind::Relative, 64), + elf::R_390_PC16DBL => { + encoding = RelocationEncoding::S390xDbl; + (RelocationKind::Relative, 16) + } + elf::R_390_PC32DBL => { + encoding = RelocationEncoding::S390xDbl; + (RelocationKind::Relative, 32) + } + elf::R_390_PLT16DBL => { + encoding = RelocationEncoding::S390xDbl; + (RelocationKind::PltRelative, 16) + } + elf::R_390_PLT32DBL => { + encoding = RelocationEncoding::S390xDbl; + (RelocationKind::PltRelative, 32) + } + elf::R_390_GOT16 => (RelocationKind::Got, 16), + elf::R_390_GOT32 => (RelocationKind::Got, 32), + elf::R_390_GOT64 => (RelocationKind::Got, 64), + elf::R_390_GOTENT => { + encoding = RelocationEncoding::S390xDbl; + (RelocationKind::GotRelative, 32) + } + elf::R_390_GOTOFF16 => (RelocationKind::GotBaseOffset, 16), + elf::R_390_GOTOFF32 => (RelocationKind::GotBaseOffset, 32), + elf::R_390_GOTOFF64 => (RelocationKind::GotBaseOffset, 64), + elf::R_390_GOTPC => (RelocationKind::GotBaseRelative, 64), + elf::R_390_GOTPCDBL => { + encoding = RelocationEncoding::S390xDbl; + (RelocationKind::GotBaseRelative, 32) + } + r_type => (RelocationKind::Elf(r_type), 0), + }, + elf::EM_SPARC | elf::EM_SPARC32PLUS | elf::EM_SPARCV9 => { + match reloc.r_type(endian, false) { + elf::R_SPARC_32 | elf::R_SPARC_UA32 => (RelocationKind::Absolute, 32), + elf::R_SPARC_64 | elf::R_SPARC_UA64 => (RelocationKind::Absolute, 64), + r_type => (RelocationKind::Elf(r_type), 0), + } + } + _ => (RelocationKind::Elf(reloc.r_type(endian, false)), 0), + }; + let sym = reloc.r_sym(endian, is_mips64el) as usize; + let target = if sym == 0 { + RelocationTarget::Absolute + } else { + RelocationTarget::Symbol(SymbolIndex(sym)) + }; + Relocation { + kind, + encoding, + size, + target, + addend: reloc.r_addend(endian).into(), + implicit_addend, + } +} + +/// A trait for generic access to `Rel32` and `Rel64`. +#[allow(missing_docs)] +pub trait Rel: Debug + Pod + Clone { + type Word: Into; + type Sword: Into; + type Endian: endian::Endian; + + fn r_offset(&self, endian: Self::Endian) -> Self::Word; + fn r_info(&self, endian: Self::Endian) -> Self::Word; + fn r_sym(&self, endian: Self::Endian) -> u32; + fn r_type(&self, endian: Self::Endian) -> u32; +} + +impl Rel for elf::Rel32 { + type Word = u32; + type Sword = i32; + type Endian = Endian; + + #[inline] + fn r_offset(&self, endian: Self::Endian) -> Self::Word { + self.r_offset.get(endian) + } + + #[inline] + fn r_info(&self, endian: Self::Endian) -> Self::Word { + self.r_info.get(endian) + } + + #[inline] + fn r_sym(&self, endian: Self::Endian) -> u32 { + self.r_sym(endian) + } + + #[inline] + fn r_type(&self, endian: Self::Endian) -> u32 { + self.r_type(endian) + } +} + +impl Rel for elf::Rel64 { + type Word = u64; + type Sword = i64; + type Endian = Endian; + + #[inline] + fn r_offset(&self, endian: Self::Endian) -> Self::Word { + self.r_offset.get(endian) + } + + #[inline] + fn r_info(&self, endian: Self::Endian) -> Self::Word { + self.r_info.get(endian) + } + + #[inline] + fn r_sym(&self, endian: Self::Endian) -> u32 { + self.r_sym(endian) + } + + #[inline] + fn r_type(&self, endian: Self::Endian) -> u32 { + self.r_type(endian) + } +} + +/// A trait for generic access to `Rela32` and `Rela64`. +#[allow(missing_docs)] +pub trait Rela: Debug + Pod + Clone { + type Word: Into; + type Sword: Into; + type Endian: endian::Endian; + + fn r_offset(&self, endian: Self::Endian) -> Self::Word; + fn r_info(&self, endian: Self::Endian, is_mips64el: bool) -> Self::Word; + fn r_addend(&self, endian: Self::Endian) -> Self::Sword; + fn r_sym(&self, endian: Self::Endian, is_mips64el: bool) -> u32; + fn r_type(&self, endian: Self::Endian, is_mips64el: bool) -> u32; +} + +impl Rela for elf::Rela32 { + type Word = u32; + type Sword = i32; + type Endian = Endian; + + #[inline] + fn r_offset(&self, endian: Self::Endian) -> Self::Word { + self.r_offset.get(endian) + } + + #[inline] + fn r_info(&self, endian: Self::Endian, _is_mips64el: bool) -> Self::Word { + self.r_info.get(endian) + } + + #[inline] + fn r_addend(&self, endian: Self::Endian) -> Self::Sword { + self.r_addend.get(endian) + } + + #[inline] + fn r_sym(&self, endian: Self::Endian, _is_mips64el: bool) -> u32 { + self.r_sym(endian) + } + + #[inline] + fn r_type(&self, endian: Self::Endian, _is_mips64el: bool) -> u32 { + self.r_type(endian) + } +} + +impl Rela for elf::Rela64 { + type Word = u64; + type Sword = i64; + type Endian = Endian; + + #[inline] + fn r_offset(&self, endian: Self::Endian) -> Self::Word { + self.r_offset.get(endian) + } + + #[inline] + fn r_info(&self, endian: Self::Endian, is_mips64el: bool) -> Self::Word { + self.get_r_info(endian, is_mips64el) + } + + #[inline] + fn r_addend(&self, endian: Self::Endian) -> Self::Sword { + self.r_addend.get(endian) + } + + #[inline] + fn r_sym(&self, endian: Self::Endian, is_mips64el: bool) -> u32 { + self.r_sym(endian, is_mips64el) + } + + #[inline] + fn r_type(&self, endian: Self::Endian, is_mips64el: bool) -> u32 { + self.r_type(endian, is_mips64el) + } +} diff --git a/crux-mir/lib/object/src/read/elf/section.rs b/crux-mir/lib/object/src/read/elf/section.rs new file mode 100644 index 000000000..3f8a08216 --- /dev/null +++ b/crux-mir/lib/object/src/read/elf/section.rs @@ -0,0 +1,1090 @@ +use core::fmt::Debug; +use core::{iter, mem, slice, str}; + +use crate::elf; +use crate::endian::{self, Endianness, U32Bytes}; +use crate::pod::Pod; +use crate::read::{ + self, Bytes, CompressedData, CompressedFileRange, CompressionFormat, Error, ObjectSection, + ReadError, ReadRef, SectionFlags, SectionIndex, SectionKind, StringTable, +}; + +use super::{ + CompressionHeader, ElfFile, ElfSectionRelocationIterator, FileHeader, GnuHashTable, HashTable, + NoteIterator, RelocationSections, SymbolTable, VerdefIterator, VerneedIterator, VersionTable, +}; + +/// The table of section headers in an ELF file. +/// +/// Also includes the string table used for the section names. +#[derive(Debug, Default, Clone, Copy)] +pub struct SectionTable<'data, Elf: FileHeader, R = &'data [u8]> +where + R: ReadRef<'data>, +{ + sections: &'data [Elf::SectionHeader], + strings: StringTable<'data, R>, +} + +impl<'data, Elf: FileHeader, R: ReadRef<'data>> SectionTable<'data, Elf, R> { + /// Create a new section table. + #[inline] + pub fn new(sections: &'data [Elf::SectionHeader], strings: StringTable<'data, R>) -> Self { + SectionTable { sections, strings } + } + + /// Iterate over the section headers. + #[inline] + pub fn iter(&self) -> slice::Iter<'data, Elf::SectionHeader> { + self.sections.iter() + } + + /// Return true if the section table is empty. + #[inline] + pub fn is_empty(&self) -> bool { + self.sections.is_empty() + } + + /// The number of section headers. + #[inline] + pub fn len(&self) -> usize { + self.sections.len() + } + + /// Return the section header at the given index. + pub fn section(&self, index: SectionIndex) -> read::Result<&'data Elf::SectionHeader> { + self.sections + .get(index.0) + .read_error("Invalid ELF section index") + } + + /// Return the section header with the given name. + /// + /// Ignores sections with invalid names. + pub fn section_by_name( + &self, + endian: Elf::Endian, + name: &[u8], + ) -> Option<(usize, &'data Elf::SectionHeader)> { + self.sections + .iter() + .enumerate() + .find(|(_, section)| self.section_name(endian, section) == Ok(name)) + } + + /// Return the section name for the given section header. + pub fn section_name( + &self, + endian: Elf::Endian, + section: &'data Elf::SectionHeader, + ) -> read::Result<&'data [u8]> { + section.name(endian, self.strings) + } + + /// Return the string table at the given section index. + /// + /// Returns an error if the section is not a string table. + #[inline] + pub fn strings( + &self, + endian: Elf::Endian, + data: R, + index: SectionIndex, + ) -> read::Result> { + self.section(index)? + .strings(endian, data)? + .read_error("Invalid ELF string section type") + } + + /// Return the symbol table of the given section type. + /// + /// Returns an empty symbol table if the symbol table does not exist. + #[inline] + pub fn symbols( + &self, + endian: Elf::Endian, + data: R, + sh_type: u32, + ) -> read::Result> { + debug_assert!(sh_type == elf::SHT_DYNSYM || sh_type == elf::SHT_SYMTAB); + + let (index, section) = match self + .iter() + .enumerate() + .find(|s| s.1.sh_type(endian) == sh_type) + { + Some(s) => s, + None => return Ok(SymbolTable::default()), + }; + + SymbolTable::parse(endian, data, self, SectionIndex(index), section) + } + + /// Return the symbol table at the given section index. + /// + /// Returns an error if the section is not a symbol table. + #[inline] + pub fn symbol_table_by_index( + &self, + endian: Elf::Endian, + data: R, + index: SectionIndex, + ) -> read::Result> { + let section = self.section(index)?; + match section.sh_type(endian) { + elf::SHT_DYNSYM | elf::SHT_SYMTAB => {} + _ => return Err(Error("Invalid ELF symbol table section type")), + } + SymbolTable::parse(endian, data, self, index, section) + } + + /// Create a mapping from section index to associated relocation sections. + #[inline] + pub fn relocation_sections( + &self, + endian: Elf::Endian, + symbol_section: SectionIndex, + ) -> read::Result { + RelocationSections::parse(endian, self, symbol_section) + } + + /// Return the contents of a dynamic section. + /// + /// Also returns the linked string table index. + /// + /// Returns `Ok(None)` if there is no `SHT_DYNAMIC` section. + /// Returns `Err` for invalid values. + pub fn dynamic( + &self, + endian: Elf::Endian, + data: R, + ) -> read::Result> { + for section in self.sections { + if let Some(dynamic) = section.dynamic(endian, data)? { + return Ok(Some(dynamic)); + } + } + Ok(None) + } + + /// Return the header of a SysV hash section. + /// + /// Returns `Ok(None)` if there is no SysV GNU hash section. + /// Returns `Err` for invalid values. + pub fn hash_header( + &self, + endian: Elf::Endian, + data: R, + ) -> read::Result>> { + for section in self.sections { + if let Some(hash) = section.hash_header(endian, data)? { + return Ok(Some(hash)); + } + } + Ok(None) + } + + /// Return the contents of a SysV hash section. + /// + /// Also returns the linked symbol table index. + /// + /// Returns `Ok(None)` if there is no SysV hash section. + /// Returns `Err` for invalid values. + pub fn hash( + &self, + endian: Elf::Endian, + data: R, + ) -> read::Result, SectionIndex)>> { + for section in self.sections { + if let Some(hash) = section.hash(endian, data)? { + return Ok(Some(hash)); + } + } + Ok(None) + } + + /// Return the header of a GNU hash section. + /// + /// Returns `Ok(None)` if there is no GNU hash section. + /// Returns `Err` for invalid values. + pub fn gnu_hash_header( + &self, + endian: Elf::Endian, + data: R, + ) -> read::Result>> { + for section in self.sections { + if let Some(hash) = section.gnu_hash_header(endian, data)? { + return Ok(Some(hash)); + } + } + Ok(None) + } + + /// Return the contents of a GNU hash section. + /// + /// Also returns the linked symbol table index. + /// + /// Returns `Ok(None)` if there is no GNU hash section. + /// Returns `Err` for invalid values. + pub fn gnu_hash( + &self, + endian: Elf::Endian, + data: R, + ) -> read::Result, SectionIndex)>> { + for section in self.sections { + if let Some(hash) = section.gnu_hash(endian, data)? { + return Ok(Some(hash)); + } + } + Ok(None) + } + + /// Return the contents of a `SHT_GNU_VERSYM` section. + /// + /// Also returns the linked symbol table index. + /// + /// Returns `Ok(None)` if there is no `SHT_GNU_VERSYM` section. + /// Returns `Err` for invalid values. + pub fn gnu_versym( + &self, + endian: Elf::Endian, + data: R, + ) -> read::Result], SectionIndex)>> { + for section in self.sections { + if let Some(syms) = section.gnu_versym(endian, data)? { + return Ok(Some(syms)); + } + } + Ok(None) + } + + /// Return the contents of a `SHT_GNU_VERDEF` section. + /// + /// Also returns the linked string table index. + /// + /// Returns `Ok(None)` if there is no `SHT_GNU_VERDEF` section. + /// Returns `Err` for invalid values. + pub fn gnu_verdef( + &self, + endian: Elf::Endian, + data: R, + ) -> read::Result, SectionIndex)>> { + for section in self.sections { + if let Some(defs) = section.gnu_verdef(endian, data)? { + return Ok(Some(defs)); + } + } + Ok(None) + } + + /// Return the contents of a `SHT_GNU_VERNEED` section. + /// + /// Also returns the linked string table index. + /// + /// Returns `Ok(None)` if there is no `SHT_GNU_VERNEED` section. + /// Returns `Err` for invalid values. + pub fn gnu_verneed( + &self, + endian: Elf::Endian, + data: R, + ) -> read::Result, SectionIndex)>> { + for section in self.sections { + if let Some(needs) = section.gnu_verneed(endian, data)? { + return Ok(Some(needs)); + } + } + Ok(None) + } + + /// Returns the symbol version table. + /// + /// Returns `Ok(None)` if there is no `SHT_GNU_VERSYM` section. + /// Returns `Err` for invalid values. + pub fn versions( + &self, + endian: Elf::Endian, + data: R, + ) -> read::Result>> { + let (versyms, link) = match self.gnu_versym(endian, data)? { + Some(val) => val, + None => return Ok(None), + }; + let strings = self.symbol_table_by_index(endian, data, link)?.strings(); + // TODO: check links? + let verdefs = self.gnu_verdef(endian, data)?.map(|x| x.0); + let verneeds = self.gnu_verneed(endian, data)?.map(|x| x.0); + VersionTable::parse(endian, versyms, verdefs, verneeds, strings).map(Some) + } +} + +/// An iterator over the sections of an `ElfFile32`. +pub type ElfSectionIterator32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfSectionIterator<'data, 'file, elf::FileHeader32, R>; +/// An iterator over the sections of an `ElfFile64`. +pub type ElfSectionIterator64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfSectionIterator<'data, 'file, elf::FileHeader64, R>; + +/// An iterator over the sections of an `ElfFile`. +#[derive(Debug)] +pub struct ElfSectionIterator<'data, 'file, Elf, R = &'data [u8]> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ + pub(super) file: &'file ElfFile<'data, Elf, R>, + pub(super) iter: iter::Enumerate>, +} + +impl<'data, 'file, Elf, R> Iterator for ElfSectionIterator<'data, 'file, Elf, R> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ + type Item = ElfSection<'data, 'file, Elf, R>; + + fn next(&mut self) -> Option { + self.iter.next().map(|(index, section)| ElfSection { + index: SectionIndex(index), + file: self.file, + section, + }) + } +} + +/// A section of an `ElfFile32`. +pub type ElfSection32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfSection<'data, 'file, elf::FileHeader32, R>; +/// A section of an `ElfFile64`. +pub type ElfSection64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfSection<'data, 'file, elf::FileHeader64, R>; + +/// A section of an `ElfFile`. +#[derive(Debug)] +pub struct ElfSection<'data, 'file, Elf, R = &'data [u8]> +where + 'data: 'file, + Elf: FileHeader, + R: ReadRef<'data>, +{ + pub(super) file: &'file ElfFile<'data, Elf, R>, + pub(super) index: SectionIndex, + pub(super) section: &'data Elf::SectionHeader, +} + +impl<'data, 'file, Elf: FileHeader, R: ReadRef<'data>> ElfSection<'data, 'file, Elf, R> { + fn bytes(&self) -> read::Result<&'data [u8]> { + self.section + .data(self.file.endian, self.file.data) + .read_error("Invalid ELF section size or offset") + } + + fn maybe_compressed(&self) -> read::Result> { + let endian = self.file.endian; + if (self.section.sh_flags(endian).into() & u64::from(elf::SHF_COMPRESSED)) == 0 { + return Ok(None); + } + let (section_offset, section_size) = self + .section + .file_range(endian) + .read_error("Invalid ELF compressed section type")?; + let mut offset = section_offset; + let header = self + .file + .data + .read::(&mut offset) + .read_error("Invalid ELF compressed section offset")?; + if header.ch_type(endian) != elf::ELFCOMPRESS_ZLIB { + return Err(Error("Unsupported ELF compression type")); + } + let uncompressed_size = header.ch_size(endian).into(); + let compressed_size = section_size + .checked_sub(offset - section_offset) + .read_error("Invalid ELF compressed section size")?; + Ok(Some(CompressedFileRange { + format: CompressionFormat::Zlib, + offset, + compressed_size, + uncompressed_size, + })) + } + + /// Try GNU-style "ZLIB" header decompression. + fn maybe_compressed_gnu(&self) -> read::Result> { + let name = match self.name() { + Ok(name) => name, + // I think it's ok to ignore this error? + Err(_) => return Ok(None), + }; + if !name.starts_with(".zdebug_") { + return Ok(None); + } + let (section_offset, section_size) = self + .section + .file_range(self.file.endian) + .read_error("Invalid ELF GNU compressed section type")?; + let mut offset = section_offset; + let data = self.file.data; + // Assume ZLIB-style uncompressed data is no more than 4GB to avoid accidentally + // huge allocations. This also reduces the chance of accidentally matching on a + // .debug_str that happens to start with "ZLIB". + if data + .read_bytes(&mut offset, 8) + .read_error("ELF GNU compressed section is too short")? + != b"ZLIB\0\0\0\0" + { + return Err(Error("Invalid ELF GNU compressed section header")); + } + let uncompressed_size = data + .read::>(&mut offset) + .read_error("ELF GNU compressed section is too short")? + .get(endian::BigEndian) + .into(); + let compressed_size = section_size + .checked_sub(offset - section_offset) + .read_error("ELF GNU compressed section is too short")?; + Ok(Some(CompressedFileRange { + format: CompressionFormat::Zlib, + offset, + compressed_size, + uncompressed_size, + })) + } +} + +impl<'data, 'file, Elf, R> read::private::Sealed for ElfSection<'data, 'file, Elf, R> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ +} + +impl<'data, 'file, Elf, R> ObjectSection<'data> for ElfSection<'data, 'file, Elf, R> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ + type RelocationIterator = ElfSectionRelocationIterator<'data, 'file, Elf, R>; + + #[inline] + fn index(&self) -> SectionIndex { + self.index + } + + #[inline] + fn address(&self) -> u64 { + self.section.sh_addr(self.file.endian).into() + } + + #[inline] + fn size(&self) -> u64 { + self.section.sh_size(self.file.endian).into() + } + + #[inline] + fn align(&self) -> u64 { + self.section.sh_addralign(self.file.endian).into() + } + + #[inline] + fn file_range(&self) -> Option<(u64, u64)> { + self.section.file_range(self.file.endian) + } + + #[inline] + fn data(&self) -> read::Result<&'data [u8]> { + self.bytes() + } + + fn data_range(&self, address: u64, size: u64) -> read::Result> { + Ok(read::util::data_range( + self.bytes()?, + self.address(), + address, + size, + )) + } + + fn compressed_file_range(&self) -> read::Result { + Ok(if let Some(data) = self.maybe_compressed()? { + data + } else if let Some(data) = self.maybe_compressed_gnu()? { + data + } else { + CompressedFileRange::none(self.file_range()) + }) + } + + fn compressed_data(&self) -> read::Result> { + self.compressed_file_range()?.data(self.file.data) + } + + fn name_bytes(&self) -> read::Result<&[u8]> { + self.file + .sections + .section_name(self.file.endian, self.section) + } + + fn name(&self) -> read::Result<&str> { + let name = self.name_bytes()?; + str::from_utf8(name) + .ok() + .read_error("Non UTF-8 ELF section name") + } + + #[inline] + fn segment_name_bytes(&self) -> read::Result> { + Ok(None) + } + + #[inline] + fn segment_name(&self) -> read::Result> { + Ok(None) + } + + fn kind(&self) -> SectionKind { + let flags = self.section.sh_flags(self.file.endian).into(); + let sh_type = self.section.sh_type(self.file.endian); + match sh_type { + elf::SHT_PROGBITS => { + if flags & u64::from(elf::SHF_ALLOC) != 0 { + if flags & u64::from(elf::SHF_EXECINSTR) != 0 { + SectionKind::Text + } else if flags & u64::from(elf::SHF_TLS) != 0 { + SectionKind::Tls + } else if flags & u64::from(elf::SHF_WRITE) != 0 { + SectionKind::Data + } else if flags & u64::from(elf::SHF_STRINGS) != 0 { + SectionKind::ReadOnlyString + } else { + SectionKind::ReadOnlyData + } + } else if flags & u64::from(elf::SHF_STRINGS) != 0 { + SectionKind::OtherString + } else { + SectionKind::Other + } + } + elf::SHT_NOBITS => { + if flags & u64::from(elf::SHF_TLS) != 0 { + SectionKind::UninitializedTls + } else { + SectionKind::UninitializedData + } + } + elf::SHT_NOTE => SectionKind::Note, + elf::SHT_NULL + | elf::SHT_SYMTAB + | elf::SHT_STRTAB + | elf::SHT_RELA + | elf::SHT_HASH + | elf::SHT_DYNAMIC + | elf::SHT_REL + | elf::SHT_DYNSYM + | elf::SHT_GROUP => SectionKind::Metadata, + _ => SectionKind::Elf(sh_type), + } + } + + fn relocations(&self) -> ElfSectionRelocationIterator<'data, 'file, Elf, R> { + ElfSectionRelocationIterator { + section_index: self.index, + file: self.file, + relocations: None, + } + } + + fn flags(&self) -> SectionFlags { + SectionFlags::Elf { + sh_flags: self.section.sh_flags(self.file.endian).into(), + } + } +} + +/// A trait for generic access to `SectionHeader32` and `SectionHeader64`. +#[allow(missing_docs)] +pub trait SectionHeader: Debug + Pod { + type Elf: FileHeader; + type Word: Into; + type Endian: endian::Endian; + + fn sh_name(&self, endian: Self::Endian) -> u32; + fn sh_type(&self, endian: Self::Endian) -> u32; + fn sh_flags(&self, endian: Self::Endian) -> Self::Word; + fn sh_addr(&self, endian: Self::Endian) -> Self::Word; + fn sh_offset(&self, endian: Self::Endian) -> Self::Word; + fn sh_size(&self, endian: Self::Endian) -> Self::Word; + fn sh_link(&self, endian: Self::Endian) -> u32; + fn sh_info(&self, endian: Self::Endian) -> u32; + fn sh_addralign(&self, endian: Self::Endian) -> Self::Word; + fn sh_entsize(&self, endian: Self::Endian) -> Self::Word; + + /// Parse the section name from the string table. + fn name<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + strings: StringTable<'data, R>, + ) -> read::Result<&'data [u8]> { + strings + .get(self.sh_name(endian)) + .read_error("Invalid ELF section name offset") + } + + /// Return the offset and size of the section in the file. + /// + /// Returns `None` for sections that have no data in the file. + fn file_range(&self, endian: Self::Endian) -> Option<(u64, u64)> { + if self.sh_type(endian) == elf::SHT_NOBITS { + None + } else { + Some((self.sh_offset(endian).into(), self.sh_size(endian).into())) + } + } + + /// Return the section data. + /// + /// Returns `Ok(&[])` if the section has no data. + /// Returns `Err` for invalid values. + fn data<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result<&'data [u8]> { + if let Some((offset, size)) = self.file_range(endian) { + data.read_bytes_at(offset, size) + .read_error("Invalid ELF section size or offset") + } else { + Ok(&[]) + } + } + + /// Return the section data as a slice of the given type. + /// + /// Allows padding at the end of the data. + /// Returns `Ok(&[])` if the section has no data. + /// Returns `Err` for invalid values, including bad alignment. + fn data_as_array<'data, T: Pod, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result<&'data [T]> { + let mut data = self.data(endian, data).map(Bytes)?; + data.read_slice(data.len() / mem::size_of::()) + .read_error("Invalid ELF section size or offset") + } + + /// Return the strings in the section. + /// + /// Returns `Ok(None)` if the section does not contain strings. + /// Returns `Err` for invalid values. + fn strings<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result>> { + if self.sh_type(endian) != elf::SHT_STRTAB { + return Ok(None); + } + let str_offset = self.sh_offset(endian).into(); + let str_size = self.sh_size(endian).into(); + let str_end = str_offset + .checked_add(str_size) + .read_error("Invalid ELF string section offset or size")?; + Ok(Some(StringTable::new(data, str_offset, str_end))) + } + + /// Return the symbols in the section. + /// + /// Also finds the linked string table in `sections`. + /// + /// `section_index` must be the 0-based index of this section, and is used + /// to find the corresponding extended section index table in `sections`. + /// + /// Returns `Ok(None)` if the section does not contain symbols. + /// Returns `Err` for invalid values. + fn symbols<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + sections: &SectionTable<'data, Self::Elf, R>, + section_index: SectionIndex, + ) -> read::Result>> { + let sh_type = self.sh_type(endian); + if sh_type != elf::SHT_SYMTAB && sh_type != elf::SHT_DYNSYM { + return Ok(None); + } + SymbolTable::parse(endian, data, sections, section_index, self).map(Some) + } + + /// Return the `Elf::Rel` entries in the section. + /// + /// Also returns the linked symbol table index. + /// + /// Returns `Ok(None)` if the section does not contain relocations. + /// Returns `Err` for invalid values. + fn rel<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result::Rel], SectionIndex)>> { + if self.sh_type(endian) != elf::SHT_REL { + return Ok(None); + } + let rel = self + .data_as_array(endian, data) + .read_error("Invalid ELF relocation section offset or size")?; + let link = SectionIndex(self.sh_link(endian) as usize); + Ok(Some((rel, link))) + } + + /// Return the `Elf::Rela` entries in the section. + /// + /// Also returns the linked symbol table index. + /// + /// Returns `Ok(None)` if the section does not contain relocations. + /// Returns `Err` for invalid values. + fn rela<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result::Rela], SectionIndex)>> { + if self.sh_type(endian) != elf::SHT_RELA { + return Ok(None); + } + let rela = self + .data_as_array(endian, data) + .read_error("Invalid ELF relocation section offset or size")?; + let link = SectionIndex(self.sh_link(endian) as usize); + Ok(Some((rela, link))) + } + + /// Return entries in a dynamic section. + /// + /// Also returns the linked string table index. + /// + /// Returns `Ok(None)` if the section type is not `SHT_DYNAMIC`. + /// Returns `Err` for invalid values. + fn dynamic<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result::Dyn], SectionIndex)>> { + if self.sh_type(endian) != elf::SHT_DYNAMIC { + return Ok(None); + } + let dynamic = self + .data_as_array(endian, data) + .read_error("Invalid ELF dynamic section offset or size")?; + let link = SectionIndex(self.sh_link(endian) as usize); + Ok(Some((dynamic, link))) + } + + /// Return a note iterator for the section data. + /// + /// Returns `Ok(None)` if the section does not contain notes. + /// Returns `Err` for invalid values. + fn notes<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result>> { + if self.sh_type(endian) != elf::SHT_NOTE { + return Ok(None); + } + let data = self + .data(endian, data) + .read_error("Invalid ELF note section offset or size")?; + let notes = NoteIterator::new(endian, self.sh_addralign(endian), data)?; + Ok(Some(notes)) + } + + /// Return the contents of a group section. + /// + /// The first value is a `GRP_*` value, and the remaining values + /// are section indices. + /// + /// Returns `Ok(None)` if the section does not define a group. + /// Returns `Err` for invalid values. + fn group<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result])>> { + if self.sh_type(endian) != elf::SHT_GROUP { + return Ok(None); + } + let mut data = self + .data(endian, data) + .read_error("Invalid ELF group section offset or size") + .map(Bytes)?; + let flag = data + .read::>() + .read_error("Invalid ELF group section offset or size")? + .get(endian); + let count = data.len() / mem::size_of::>(); + let sections = data + .read_slice(count) + .read_error("Invalid ELF group section offset or size")?; + Ok(Some((flag, sections))) + } + + /// Return the header of a SysV hash section. + /// + /// Returns `Ok(None)` if the section does not contain a SysV hash. + /// Returns `Err` for invalid values. + fn hash_header<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result>> { + if self.sh_type(endian) != elf::SHT_HASH { + return Ok(None); + } + let data = self + .data(endian, data) + .read_error("Invalid ELF hash section offset or size")?; + let header = data + .read_at::>(0) + .read_error("Invalid hash header")?; + Ok(Some(header)) + } + + /// Return the contents of a SysV hash section. + /// + /// Also returns the linked symbol table index. + /// + /// Returns `Ok(None)` if the section does not contain a SysV hash. + /// Returns `Err` for invalid values. + fn hash<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result, SectionIndex)>> { + if self.sh_type(endian) != elf::SHT_HASH { + return Ok(None); + } + let data = self + .data(endian, data) + .read_error("Invalid ELF hash section offset or size")?; + let hash = HashTable::parse(endian, data)?; + let link = SectionIndex(self.sh_link(endian) as usize); + Ok(Some((hash, link))) + } + + /// Return the header of a GNU hash section. + /// + /// Returns `Ok(None)` if the section does not contain a GNU hash. + /// Returns `Err` for invalid values. + fn gnu_hash_header<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result>> { + if self.sh_type(endian) != elf::SHT_GNU_HASH { + return Ok(None); + } + let data = self + .data(endian, data) + .read_error("Invalid ELF GNU hash section offset or size")?; + let header = data + .read_at::>(0) + .read_error("Invalid GNU hash header")?; + Ok(Some(header)) + } + + /// Return the contents of a GNU hash section. + /// + /// Also returns the linked symbol table index. + /// + /// Returns `Ok(None)` if the section does not contain a GNU hash. + /// Returns `Err` for invalid values. + fn gnu_hash<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result, SectionIndex)>> { + if self.sh_type(endian) != elf::SHT_GNU_HASH { + return Ok(None); + } + let data = self + .data(endian, data) + .read_error("Invalid ELF GNU hash section offset or size")?; + let hash = GnuHashTable::parse(endian, data)?; + let link = SectionIndex(self.sh_link(endian) as usize); + Ok(Some((hash, link))) + } + + /// Return the contents of a `SHT_GNU_VERSYM` section. + /// + /// Also returns the linked symbol table index. + /// + /// Returns `Ok(None)` if the section type is not `SHT_GNU_VERSYM`. + /// Returns `Err` for invalid values. + fn gnu_versym<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result], SectionIndex)>> { + if self.sh_type(endian) != elf::SHT_GNU_VERSYM { + return Ok(None); + } + let versym = self + .data_as_array(endian, data) + .read_error("Invalid ELF GNU versym section offset or size")?; + let link = SectionIndex(self.sh_link(endian) as usize); + Ok(Some((versym, link))) + } + + /// Return an iterator for the entries of a `SHT_GNU_VERDEF` section. + /// + /// Also returns the linked string table index. + /// + /// Returns `Ok(None)` if the section type is not `SHT_GNU_VERDEF`. + /// Returns `Err` for invalid values. + fn gnu_verdef<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result, SectionIndex)>> { + if self.sh_type(endian) != elf::SHT_GNU_VERDEF { + return Ok(None); + } + let verdef = self + .data(endian, data) + .read_error("Invalid ELF GNU verdef section offset or size")?; + let link = SectionIndex(self.sh_link(endian) as usize); + Ok(Some((VerdefIterator::new(endian, verdef), link))) + } + + /// Return an iterator for the entries of a `SHT_GNU_VERNEED` section. + /// + /// Also returns the linked string table index. + /// + /// Returns `Ok(None)` if the section type is not `SHT_GNU_VERNEED`. + /// Returns `Err` for invalid values. + fn gnu_verneed<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result, SectionIndex)>> { + if self.sh_type(endian) != elf::SHT_GNU_VERNEED { + return Ok(None); + } + let verneed = self + .data(endian, data) + .read_error("Invalid ELF GNU verneed section offset or size")?; + let link = SectionIndex(self.sh_link(endian) as usize); + Ok(Some((VerneedIterator::new(endian, verneed), link))) + } +} + +impl SectionHeader for elf::SectionHeader32 { + type Elf = elf::FileHeader32; + type Word = u32; + type Endian = Endian; + + #[inline] + fn sh_name(&self, endian: Self::Endian) -> u32 { + self.sh_name.get(endian) + } + + #[inline] + fn sh_type(&self, endian: Self::Endian) -> u32 { + self.sh_type.get(endian) + } + + #[inline] + fn sh_flags(&self, endian: Self::Endian) -> Self::Word { + self.sh_flags.get(endian) + } + + #[inline] + fn sh_addr(&self, endian: Self::Endian) -> Self::Word { + self.sh_addr.get(endian) + } + + #[inline] + fn sh_offset(&self, endian: Self::Endian) -> Self::Word { + self.sh_offset.get(endian) + } + + #[inline] + fn sh_size(&self, endian: Self::Endian) -> Self::Word { + self.sh_size.get(endian) + } + + #[inline] + fn sh_link(&self, endian: Self::Endian) -> u32 { + self.sh_link.get(endian) + } + + #[inline] + fn sh_info(&self, endian: Self::Endian) -> u32 { + self.sh_info.get(endian) + } + + #[inline] + fn sh_addralign(&self, endian: Self::Endian) -> Self::Word { + self.sh_addralign.get(endian) + } + + #[inline] + fn sh_entsize(&self, endian: Self::Endian) -> Self::Word { + self.sh_entsize.get(endian) + } +} + +impl SectionHeader for elf::SectionHeader64 { + type Word = u64; + type Endian = Endian; + type Elf = elf::FileHeader64; + + #[inline] + fn sh_name(&self, endian: Self::Endian) -> u32 { + self.sh_name.get(endian) + } + + #[inline] + fn sh_type(&self, endian: Self::Endian) -> u32 { + self.sh_type.get(endian) + } + + #[inline] + fn sh_flags(&self, endian: Self::Endian) -> Self::Word { + self.sh_flags.get(endian) + } + + #[inline] + fn sh_addr(&self, endian: Self::Endian) -> Self::Word { + self.sh_addr.get(endian) + } + + #[inline] + fn sh_offset(&self, endian: Self::Endian) -> Self::Word { + self.sh_offset.get(endian) + } + + #[inline] + fn sh_size(&self, endian: Self::Endian) -> Self::Word { + self.sh_size.get(endian) + } + + #[inline] + fn sh_link(&self, endian: Self::Endian) -> u32 { + self.sh_link.get(endian) + } + + #[inline] + fn sh_info(&self, endian: Self::Endian) -> u32 { + self.sh_info.get(endian) + } + + #[inline] + fn sh_addralign(&self, endian: Self::Endian) -> Self::Word { + self.sh_addralign.get(endian) + } + + #[inline] + fn sh_entsize(&self, endian: Self::Endian) -> Self::Word { + self.sh_entsize.get(endian) + } +} diff --git a/crux-mir/lib/object/src/read/elf/segment.rs b/crux-mir/lib/object/src/read/elf/segment.rs new file mode 100644 index 000000000..874ea92b8 --- /dev/null +++ b/crux-mir/lib/object/src/read/elf/segment.rs @@ -0,0 +1,333 @@ +use core::fmt::Debug; +use core::{mem, slice, str}; + +use crate::elf; +use crate::endian::{self, Endianness}; +use crate::pod::Pod; +use crate::read::{self, Bytes, ObjectSegment, ReadError, ReadRef, SegmentFlags}; + +use super::{ElfFile, FileHeader, NoteIterator}; + +/// An iterator over the segments of an `ElfFile32`. +pub type ElfSegmentIterator32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfSegmentIterator<'data, 'file, elf::FileHeader32, R>; +/// An iterator over the segments of an `ElfFile64`. +pub type ElfSegmentIterator64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfSegmentIterator<'data, 'file, elf::FileHeader64, R>; + +/// An iterator over the segments of an `ElfFile`. +#[derive(Debug)] +pub struct ElfSegmentIterator<'data, 'file, Elf, R = &'data [u8]> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ + pub(super) file: &'file ElfFile<'data, Elf, R>, + pub(super) iter: slice::Iter<'data, Elf::ProgramHeader>, +} + +impl<'data, 'file, Elf, R> Iterator for ElfSegmentIterator<'data, 'file, Elf, R> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ + type Item = ElfSegment<'data, 'file, Elf, R>; + + fn next(&mut self) -> Option { + while let Some(segment) = self.iter.next() { + if segment.p_type(self.file.endian) == elf::PT_LOAD { + return Some(ElfSegment { + file: self.file, + segment, + }); + } + } + None + } +} + +/// A segment of an `ElfFile32`. +pub type ElfSegment32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfSegment<'data, 'file, elf::FileHeader32, R>; +/// A segment of an `ElfFile64`. +pub type ElfSegment64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfSegment<'data, 'file, elf::FileHeader64, R>; + +/// A segment of an `ElfFile`. +#[derive(Debug)] +pub struct ElfSegment<'data, 'file, Elf, R = &'data [u8]> +where + 'data: 'file, + Elf: FileHeader, + R: ReadRef<'data>, +{ + pub(super) file: &'file ElfFile<'data, Elf, R>, + pub(super) segment: &'data Elf::ProgramHeader, +} + +impl<'data, 'file, Elf: FileHeader, R: ReadRef<'data>> ElfSegment<'data, 'file, Elf, R> { + fn bytes(&self) -> read::Result<&'data [u8]> { + self.segment + .data(self.file.endian, self.file.data) + .read_error("Invalid ELF segment size or offset") + } +} + +impl<'data, 'file, Elf, R> read::private::Sealed for ElfSegment<'data, 'file, Elf, R> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ +} + +impl<'data, 'file, Elf, R> ObjectSegment<'data> for ElfSegment<'data, 'file, Elf, R> +where + Elf: FileHeader, + R: ReadRef<'data>, +{ + #[inline] + fn address(&self) -> u64 { + self.segment.p_vaddr(self.file.endian).into() + } + + #[inline] + fn size(&self) -> u64 { + self.segment.p_memsz(self.file.endian).into() + } + + #[inline] + fn align(&self) -> u64 { + self.segment.p_align(self.file.endian).into() + } + + #[inline] + fn file_range(&self) -> (u64, u64) { + self.segment.file_range(self.file.endian) + } + + #[inline] + fn data(&self) -> read::Result<&'data [u8]> { + self.bytes() + } + + fn data_range(&self, address: u64, size: u64) -> read::Result> { + Ok(read::util::data_range( + self.bytes()?, + self.address(), + address, + size, + )) + } + + #[inline] + fn name_bytes(&self) -> read::Result> { + Ok(None) + } + + #[inline] + fn name(&self) -> read::Result> { + Ok(None) + } + + #[inline] + fn flags(&self) -> SegmentFlags { + let p_flags = self.segment.p_flags(self.file.endian); + SegmentFlags::Elf { p_flags } + } +} + +/// A trait for generic access to `ProgramHeader32` and `ProgramHeader64`. +#[allow(missing_docs)] +pub trait ProgramHeader: Debug + Pod { + type Elf: FileHeader; + type Word: Into; + type Endian: endian::Endian; + + fn p_type(&self, endian: Self::Endian) -> u32; + fn p_flags(&self, endian: Self::Endian) -> u32; + fn p_offset(&self, endian: Self::Endian) -> Self::Word; + fn p_vaddr(&self, endian: Self::Endian) -> Self::Word; + fn p_paddr(&self, endian: Self::Endian) -> Self::Word; + fn p_filesz(&self, endian: Self::Endian) -> Self::Word; + fn p_memsz(&self, endian: Self::Endian) -> Self::Word; + fn p_align(&self, endian: Self::Endian) -> Self::Word; + + /// Return the offset and size of the segment in the file. + fn file_range(&self, endian: Self::Endian) -> (u64, u64) { + (self.p_offset(endian).into(), self.p_filesz(endian).into()) + } + + /// Return the segment data. + /// + /// Returns `Err` for invalid values. + fn data<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> Result<&'data [u8], ()> { + let (offset, size) = self.file_range(endian); + data.read_bytes_at(offset, size) + } + + /// Return the segment data as a slice of the given type. + /// + /// Allows padding at the end of the data. + /// Returns `Ok(&[])` if the segment has no data. + /// Returns `Err` for invalid values, including bad alignment. + fn data_as_array<'data, T: Pod, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> Result<&'data [T], ()> { + let mut data = self.data(endian, data).map(Bytes)?; + data.read_slice(data.len() / mem::size_of::()) + } + + /// Return the segment data in the given virtual address range + /// + /// Returns `Ok(None)` if the segment does not contain the address. + /// Returns `Err` for invalid values. + fn data_range<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + address: u64, + size: u64, + ) -> Result, ()> { + Ok(read::util::data_range( + self.data(endian, data)?, + self.p_vaddr(endian).into(), + address, + size, + )) + } + + /// Return entries in a dynamic segment. + /// + /// Returns `Ok(None)` if the segment is not `PT_DYNAMIC`. + /// Returns `Err` for invalid values. + fn dynamic<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result::Dyn]>> { + if self.p_type(endian) != elf::PT_DYNAMIC { + return Ok(None); + } + let dynamic = self + .data_as_array(endian, data) + .read_error("Invalid ELF dynamic segment offset or size")?; + Ok(Some(dynamic)) + } + + /// Return a note iterator for the segment data. + /// + /// Returns `Ok(None)` if the segment does not contain notes. + /// Returns `Err` for invalid values. + fn notes<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> read::Result>> { + if self.p_type(endian) != elf::PT_NOTE { + return Ok(None); + } + let data = self + .data(endian, data) + .read_error("Invalid ELF note segment offset or size")?; + let notes = NoteIterator::new(endian, self.p_align(endian), data)?; + Ok(Some(notes)) + } +} + +impl ProgramHeader for elf::ProgramHeader32 { + type Word = u32; + type Endian = Endian; + type Elf = elf::FileHeader32; + + #[inline] + fn p_type(&self, endian: Self::Endian) -> u32 { + self.p_type.get(endian) + } + + #[inline] + fn p_flags(&self, endian: Self::Endian) -> u32 { + self.p_flags.get(endian) + } + + #[inline] + fn p_offset(&self, endian: Self::Endian) -> Self::Word { + self.p_offset.get(endian) + } + + #[inline] + fn p_vaddr(&self, endian: Self::Endian) -> Self::Word { + self.p_vaddr.get(endian) + } + + #[inline] + fn p_paddr(&self, endian: Self::Endian) -> Self::Word { + self.p_paddr.get(endian) + } + + #[inline] + fn p_filesz(&self, endian: Self::Endian) -> Self::Word { + self.p_filesz.get(endian) + } + + #[inline] + fn p_memsz(&self, endian: Self::Endian) -> Self::Word { + self.p_memsz.get(endian) + } + + #[inline] + fn p_align(&self, endian: Self::Endian) -> Self::Word { + self.p_align.get(endian) + } +} + +impl ProgramHeader for elf::ProgramHeader64 { + type Word = u64; + type Endian = Endian; + type Elf = elf::FileHeader64; + + #[inline] + fn p_type(&self, endian: Self::Endian) -> u32 { + self.p_type.get(endian) + } + + #[inline] + fn p_flags(&self, endian: Self::Endian) -> u32 { + self.p_flags.get(endian) + } + + #[inline] + fn p_offset(&self, endian: Self::Endian) -> Self::Word { + self.p_offset.get(endian) + } + + #[inline] + fn p_vaddr(&self, endian: Self::Endian) -> Self::Word { + self.p_vaddr.get(endian) + } + + #[inline] + fn p_paddr(&self, endian: Self::Endian) -> Self::Word { + self.p_paddr.get(endian) + } + + #[inline] + fn p_filesz(&self, endian: Self::Endian) -> Self::Word { + self.p_filesz.get(endian) + } + + #[inline] + fn p_memsz(&self, endian: Self::Endian) -> Self::Word { + self.p_memsz.get(endian) + } + + #[inline] + fn p_align(&self, endian: Self::Endian) -> Self::Word { + self.p_align.get(endian) + } +} diff --git a/crux-mir/lib/object/src/read/elf/symbol.rs b/crux-mir/lib/object/src/read/elf/symbol.rs new file mode 100644 index 000000000..f52eff20e --- /dev/null +++ b/crux-mir/lib/object/src/read/elf/symbol.rs @@ -0,0 +1,579 @@ +use alloc::fmt; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::slice; +use core::str; + +use crate::elf; +use crate::endian::{self, Endianness}; +use crate::pod::Pod; +use crate::read::util::StringTable; +use crate::read::{ + self, ObjectSymbol, ObjectSymbolTable, ReadError, ReadRef, SectionIndex, SymbolFlags, + SymbolIndex, SymbolKind, SymbolMap, SymbolMapEntry, SymbolScope, SymbolSection, +}; + +use super::{FileHeader, SectionHeader, SectionTable}; + +/// A table of symbol entries in an ELF file. +/// +/// Also includes the string table used for the symbol names. +#[derive(Debug, Clone, Copy)] +pub struct SymbolTable<'data, Elf: FileHeader, R = &'data [u8]> +where + R: ReadRef<'data>, +{ + section: SectionIndex, + string_section: SectionIndex, + shndx_section: SectionIndex, + symbols: &'data [Elf::Sym], + strings: StringTable<'data, R>, + shndx: &'data [u32], +} + +impl<'data, Elf: FileHeader, R: ReadRef<'data>> Default for SymbolTable<'data, Elf, R> { + fn default() -> Self { + SymbolTable { + section: SectionIndex(0), + string_section: SectionIndex(0), + shndx_section: SectionIndex(0), + symbols: &[], + strings: Default::default(), + shndx: &[], + } + } +} + +impl<'data, Elf: FileHeader, R: ReadRef<'data>> SymbolTable<'data, Elf, R> { + /// Parse the given symbol table section. + pub fn parse( + endian: Elf::Endian, + data: R, + sections: &SectionTable<'data, Elf, R>, + section_index: SectionIndex, + section: &Elf::SectionHeader, + ) -> read::Result> { + debug_assert!( + section.sh_type(endian) == elf::SHT_DYNSYM + || section.sh_type(endian) == elf::SHT_SYMTAB + ); + + let symbols = section + .data_as_array(endian, data) + .read_error("Invalid ELF symbol table data")?; + + let link = SectionIndex(section.sh_link(endian) as usize); + let strings = sections.strings(endian, data, link)?; + + let mut shndx_section = SectionIndex(0); + let mut shndx = &[][..]; + for (i, s) in sections.iter().enumerate() { + if s.sh_type(endian) == elf::SHT_SYMTAB_SHNDX + && s.sh_link(endian) as usize == section_index.0 + { + shndx_section = SectionIndex(i); + shndx = s + .data_as_array(endian, data) + .read_error("Invalid ELF symtab_shndx data")?; + } + } + + Ok(SymbolTable { + section: section_index, + string_section: link, + symbols, + strings, + shndx, + shndx_section, + }) + } + + /// Return the section index of this symbol table. + #[inline] + pub fn section(&self) -> SectionIndex { + self.section + } + + /// Return the section index of the shndx table. + #[inline] + pub fn shndx_section(&self) -> SectionIndex { + self.shndx_section + } + + /// Return the section index of the linked string table. + #[inline] + pub fn string_section(&self) -> SectionIndex { + self.string_section + } + + /// Return the string table used for the symbol names. + #[inline] + pub fn strings(&self) -> StringTable<'data, R> { + self.strings + } + + /// Return the symbol table. + #[inline] + pub fn symbols(&self) -> &'data [Elf::Sym] { + self.symbols + } + + /// Iterate over the symbols. + #[inline] + pub fn iter(&self) -> slice::Iter<'data, Elf::Sym> { + self.symbols.iter() + } + + /// Return true if the symbol table is empty. + #[inline] + pub fn is_empty(&self) -> bool { + self.symbols.is_empty() + } + + /// The number of symbols. + #[inline] + pub fn len(&self) -> usize { + self.symbols.len() + } + + /// Return the symbol at the given index. + pub fn symbol(&self, index: usize) -> read::Result<&'data Elf::Sym> { + self.symbols + .get(index) + .read_error("Invalid ELF symbol index") + } + + /// Return the extended section index for the given symbol if present. + #[inline] + pub fn shndx(&self, index: usize) -> Option { + self.shndx.get(index).copied() + } + + /// Return the section index for the given symbol. + /// + /// This uses the extended section index if present. + pub fn symbol_section( + &self, + endian: Elf::Endian, + symbol: &'data Elf::Sym, + index: usize, + ) -> read::Result> { + match symbol.st_shndx(endian) { + elf::SHN_UNDEF => Ok(None), + elf::SHN_XINDEX => self + .shndx(index) + .read_error("Missing ELF symbol extended index") + .map(|index| Some(SectionIndex(index as usize))), + shndx if shndx < elf::SHN_LORESERVE => Ok(Some(SectionIndex(shndx.into()))), + _ => Ok(None), + } + } + + /// Return the symbol name for the given symbol. + pub fn symbol_name( + &self, + endian: Elf::Endian, + symbol: &'data Elf::Sym, + ) -> read::Result<&'data [u8]> { + symbol.name(endian, self.strings) + } + + /// Construct a map from addresses to a user-defined map entry. + pub fn map Option>( + &self, + endian: Elf::Endian, + f: F, + ) -> SymbolMap { + let mut symbols = Vec::with_capacity(self.symbols.len()); + for symbol in self.symbols { + if !symbol.is_definition(endian) { + continue; + } + if let Some(entry) = f(symbol) { + symbols.push(entry); + } + } + SymbolMap::new(symbols) + } +} + +/// A symbol table of an `ElfFile32`. +pub type ElfSymbolTable32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfSymbolTable<'data, 'file, elf::FileHeader32, R>; +/// A symbol table of an `ElfFile32`. +pub type ElfSymbolTable64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfSymbolTable<'data, 'file, elf::FileHeader64, R>; + +/// A symbol table of an `ElfFile`. +#[derive(Debug, Clone, Copy)] +pub struct ElfSymbolTable<'data, 'file, Elf, R = &'data [u8]> +where + 'data: 'file, + Elf: FileHeader, + R: ReadRef<'data>, +{ + pub(super) endian: Elf::Endian, + pub(super) symbols: &'file SymbolTable<'data, Elf, R>, +} + +impl<'data, 'file, Elf: FileHeader, R: ReadRef<'data>> read::private::Sealed + for ElfSymbolTable<'data, 'file, Elf, R> +{ +} + +impl<'data, 'file, Elf: FileHeader, R: ReadRef<'data>> ObjectSymbolTable<'data> + for ElfSymbolTable<'data, 'file, Elf, R> +{ + type Symbol = ElfSymbol<'data, 'file, Elf, R>; + type SymbolIterator = ElfSymbolIterator<'data, 'file, Elf, R>; + + fn symbols(&self) -> Self::SymbolIterator { + ElfSymbolIterator { + endian: self.endian, + symbols: self.symbols, + index: 0, + } + } + + fn symbol_by_index(&self, index: SymbolIndex) -> read::Result { + let symbol = self.symbols.symbol(index.0)?; + Ok(ElfSymbol { + endian: self.endian, + symbols: self.symbols, + index, + symbol, + }) + } +} + +/// An iterator over the symbols of an `ElfFile32`. +pub type ElfSymbolIterator32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfSymbolIterator<'data, 'file, elf::FileHeader32, R>; +/// An iterator over the symbols of an `ElfFile64`. +pub type ElfSymbolIterator64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfSymbolIterator<'data, 'file, elf::FileHeader64, R>; + +/// An iterator over the symbols of an `ElfFile`. +pub struct ElfSymbolIterator<'data, 'file, Elf, R = &'data [u8]> +where + 'data: 'file, + Elf: FileHeader, + R: ReadRef<'data>, +{ + pub(super) endian: Elf::Endian, + pub(super) symbols: &'file SymbolTable<'data, Elf, R>, + pub(super) index: usize, +} + +impl<'data, 'file, Elf: FileHeader, R: ReadRef<'data>> fmt::Debug + for ElfSymbolIterator<'data, 'file, Elf, R> +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ElfSymbolIterator").finish() + } +} + +impl<'data, 'file, Elf: FileHeader, R: ReadRef<'data>> Iterator + for ElfSymbolIterator<'data, 'file, Elf, R> +{ + type Item = ElfSymbol<'data, 'file, Elf, R>; + + fn next(&mut self) -> Option { + let index = self.index; + let symbol = self.symbols.symbols.get(index)?; + self.index += 1; + Some(ElfSymbol { + endian: self.endian, + symbols: self.symbols, + index: SymbolIndex(index), + symbol, + }) + } +} + +/// A symbol of an `ElfFile32`. +pub type ElfSymbol32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfSymbol<'data, 'file, elf::FileHeader32, R>; +/// A symbol of an `ElfFile64`. +pub type ElfSymbol64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + ElfSymbol<'data, 'file, elf::FileHeader64, R>; + +/// A symbol of an `ElfFile`. +#[derive(Debug, Clone, Copy)] +pub struct ElfSymbol<'data, 'file, Elf, R = &'data [u8]> +where + 'data: 'file, + Elf: FileHeader, + R: ReadRef<'data>, +{ + pub(super) endian: Elf::Endian, + pub(super) symbols: &'file SymbolTable<'data, Elf, R>, + pub(super) index: SymbolIndex, + pub(super) symbol: &'data Elf::Sym, +} + +impl<'data, 'file, Elf: FileHeader, R: ReadRef<'data>> read::private::Sealed + for ElfSymbol<'data, 'file, Elf, R> +{ +} + +impl<'data, 'file, Elf: FileHeader, R: ReadRef<'data>> ObjectSymbol<'data> + for ElfSymbol<'data, 'file, Elf, R> +{ + #[inline] + fn index(&self) -> SymbolIndex { + self.index + } + + fn name_bytes(&self) -> read::Result<&'data [u8]> { + self.symbol.name(self.endian, self.symbols.strings()) + } + + fn name(&self) -> read::Result<&'data str> { + let name = self.name_bytes()?; + str::from_utf8(name) + .ok() + .read_error("Non UTF-8 ELF symbol name") + } + + #[inline] + fn address(&self) -> u64 { + self.symbol.st_value(self.endian).into() + } + + #[inline] + fn size(&self) -> u64 { + self.symbol.st_size(self.endian).into() + } + + fn kind(&self) -> SymbolKind { + match self.symbol.st_type() { + elf::STT_NOTYPE if self.index.0 == 0 => SymbolKind::Null, + elf::STT_OBJECT | elf::STT_COMMON => SymbolKind::Data, + elf::STT_FUNC => SymbolKind::Text, + elf::STT_SECTION => SymbolKind::Section, + elf::STT_FILE => SymbolKind::File, + elf::STT_TLS => SymbolKind::Tls, + _ => SymbolKind::Unknown, + } + } + + fn section(&self) -> SymbolSection { + match self.symbol.st_shndx(self.endian) { + elf::SHN_UNDEF => SymbolSection::Undefined, + elf::SHN_ABS => { + if self.symbol.st_type() == elf::STT_FILE { + SymbolSection::None + } else { + SymbolSection::Absolute + } + } + elf::SHN_COMMON => SymbolSection::Common, + elf::SHN_XINDEX => match self.symbols.shndx(self.index.0) { + Some(index) => SymbolSection::Section(SectionIndex(index as usize)), + None => SymbolSection::Unknown, + }, + index if index < elf::SHN_LORESERVE => { + SymbolSection::Section(SectionIndex(index as usize)) + } + _ => SymbolSection::Unknown, + } + } + + #[inline] + fn is_undefined(&self) -> bool { + self.symbol.st_shndx(self.endian) == elf::SHN_UNDEF + } + + #[inline] + fn is_definition(&self) -> bool { + self.symbol.is_definition(self.endian) + } + + #[inline] + fn is_common(&self) -> bool { + self.symbol.st_shndx(self.endian) == elf::SHN_COMMON + } + + #[inline] + fn is_weak(&self) -> bool { + self.symbol.st_bind() == elf::STB_WEAK + } + + fn scope(&self) -> SymbolScope { + if self.symbol.st_shndx(self.endian) == elf::SHN_UNDEF { + SymbolScope::Unknown + } else { + match self.symbol.st_bind() { + elf::STB_LOCAL => SymbolScope::Compilation, + elf::STB_GLOBAL | elf::STB_WEAK => { + if self.symbol.st_visibility() == elf::STV_HIDDEN { + SymbolScope::Linkage + } else { + SymbolScope::Dynamic + } + } + _ => SymbolScope::Unknown, + } + } + } + + #[inline] + fn is_global(&self) -> bool { + self.symbol.st_bind() != elf::STB_LOCAL + } + + #[inline] + fn is_local(&self) -> bool { + self.symbol.st_bind() == elf::STB_LOCAL + } + + #[inline] + fn flags(&self) -> SymbolFlags { + SymbolFlags::Elf { + st_info: self.symbol.st_info(), + st_other: self.symbol.st_other(), + } + } +} + +/// A trait for generic access to `Sym32` and `Sym64`. +#[allow(missing_docs)] +pub trait Sym: Debug + Pod { + type Word: Into; + type Endian: endian::Endian; + + fn st_name(&self, endian: Self::Endian) -> u32; + fn st_info(&self) -> u8; + fn st_bind(&self) -> u8; + fn st_type(&self) -> u8; + fn st_other(&self) -> u8; + fn st_visibility(&self) -> u8; + fn st_shndx(&self, endian: Self::Endian) -> u16; + fn st_value(&self, endian: Self::Endian) -> Self::Word; + fn st_size(&self, endian: Self::Endian) -> Self::Word; + + /// Parse the symbol name from the string table. + fn name<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + strings: StringTable<'data, R>, + ) -> read::Result<&'data [u8]> { + strings + .get(self.st_name(endian)) + .read_error("Invalid ELF symbol name offset") + } + + /// Return true if the symbol is undefined. + #[inline] + fn is_undefined(&self, endian: Self::Endian) -> bool { + self.st_shndx(endian) == elf::SHN_UNDEF + } + + /// Return true if the symbol is a definition of a function or data object. + fn is_definition(&self, endian: Self::Endian) -> bool { + let st_type = self.st_type(); + (st_type == elf::STT_NOTYPE || st_type == elf::STT_FUNC || st_type == elf::STT_OBJECT) + && self.st_shndx(endian) != elf::SHN_UNDEF + } +} + +impl Sym for elf::Sym32 { + type Word = u32; + type Endian = Endian; + + #[inline] + fn st_name(&self, endian: Self::Endian) -> u32 { + self.st_name.get(endian) + } + + #[inline] + fn st_info(&self) -> u8 { + self.st_info + } + + #[inline] + fn st_bind(&self) -> u8 { + self.st_bind() + } + + #[inline] + fn st_type(&self) -> u8 { + self.st_type() + } + + #[inline] + fn st_other(&self) -> u8 { + self.st_other + } + + #[inline] + fn st_visibility(&self) -> u8 { + self.st_visibility() + } + + #[inline] + fn st_shndx(&self, endian: Self::Endian) -> u16 { + self.st_shndx.get(endian) + } + + #[inline] + fn st_value(&self, endian: Self::Endian) -> Self::Word { + self.st_value.get(endian) + } + + #[inline] + fn st_size(&self, endian: Self::Endian) -> Self::Word { + self.st_size.get(endian) + } +} + +impl Sym for elf::Sym64 { + type Word = u64; + type Endian = Endian; + + #[inline] + fn st_name(&self, endian: Self::Endian) -> u32 { + self.st_name.get(endian) + } + + #[inline] + fn st_info(&self) -> u8 { + self.st_info + } + + #[inline] + fn st_bind(&self) -> u8 { + self.st_bind() + } + + #[inline] + fn st_type(&self) -> u8 { + self.st_type() + } + + #[inline] + fn st_other(&self) -> u8 { + self.st_other + } + + #[inline] + fn st_visibility(&self) -> u8 { + self.st_visibility() + } + + #[inline] + fn st_shndx(&self, endian: Self::Endian) -> u16 { + self.st_shndx.get(endian) + } + + #[inline] + fn st_value(&self, endian: Self::Endian) -> Self::Word { + self.st_value.get(endian) + } + + #[inline] + fn st_size(&self, endian: Self::Endian) -> Self::Word { + self.st_size.get(endian) + } +} diff --git a/crux-mir/lib/object/src/read/elf/version.rs b/crux-mir/lib/object/src/read/elf/version.rs new file mode 100644 index 000000000..6d80ba1e3 --- /dev/null +++ b/crux-mir/lib/object/src/read/elf/version.rs @@ -0,0 +1,421 @@ +use alloc::vec::Vec; + +use crate::read::{Bytes, ReadError, ReadRef, Result, StringTable}; +use crate::{elf, endian}; + +use super::FileHeader; + +/// A version index. +#[derive(Debug, Default, Clone, Copy)] +pub struct VersionIndex(pub u16); + +impl VersionIndex { + /// Return the version index. + pub fn index(&self) -> u16 { + self.0 & elf::VERSYM_VERSION + } + + /// Return true if it is the local index. + pub fn is_local(&self) -> bool { + self.index() == elf::VER_NDX_LOCAL + } + + /// Return true if it is the global index. + pub fn is_global(&self) -> bool { + self.index() == elf::VER_NDX_GLOBAL + } + + /// Return the hidden flag. + pub fn is_hidden(&self) -> bool { + self.0 & elf::VERSYM_HIDDEN != 0 + } +} + +/// A version definition or requirement. +/// +/// This is derived from entries in the `SHT_GNU_verdef` and `SHT_GNU_verneed` sections. +#[derive(Debug, Default, Clone, Copy)] +pub struct Version<'data> { + name: &'data [u8], + hash: u32, + // Used to keep track of valid indices in `VersionTable`. + valid: bool, +} + +impl<'data> Version<'data> { + /// Return the version name. + pub fn name(&self) -> &'data [u8] { + self.name + } + + /// Return hash of the version name. + pub fn hash(&self) -> u32 { + self.hash + } +} + +/// A table of version definitions and requirements. +/// +/// It allows looking up the version information for a given symbol index. +/// +/// This is derived from entries in the `SHT_GNU_versym`, `SHT_GNU_verdef` and `SHT_GNU_verneed` sections. +#[derive(Debug, Clone)] +pub struct VersionTable<'data, Elf: FileHeader> { + symbols: &'data [elf::Versym], + versions: Vec>, +} + +impl<'data, Elf: FileHeader> Default for VersionTable<'data, Elf> { + fn default() -> Self { + VersionTable { + symbols: &[], + versions: Vec::new(), + } + } +} + +impl<'data, Elf: FileHeader> VersionTable<'data, Elf> { + /// Parse the version sections. + pub fn parse>( + endian: Elf::Endian, + versyms: &'data [elf::Versym], + verdefs: Option>, + verneeds: Option>, + strings: StringTable<'data, R>, + ) -> Result { + let mut max_index = 0; + if let Some(mut verdefs) = verdefs.clone() { + while let Some((verdef, _)) = verdefs.next()? { + if verdef.vd_flags.get(endian) & elf::VER_FLG_BASE != 0 { + continue; + } + let index = verdef.vd_ndx.get(endian) & elf::VERSYM_VERSION; + if max_index < index { + max_index = index; + } + } + } + if let Some(mut verneeds) = verneeds.clone() { + while let Some((_, mut vernauxs)) = verneeds.next()? { + while let Some(vernaux) = vernauxs.next()? { + let index = vernaux.vna_other.get(endian) & elf::VERSYM_VERSION; + if max_index < index { + max_index = index; + } + } + } + } + + // Indices should be sequential, but this could be up to + // 32k * size_of::() if max_index is bad. + let mut versions = vec![Version::default(); max_index as usize + 1]; + + if let Some(mut verdefs) = verdefs { + while let Some((verdef, mut verdauxs)) = verdefs.next()? { + if verdef.vd_flags.get(endian) & elf::VER_FLG_BASE != 0 { + continue; + } + let index = verdef.vd_ndx.get(endian) & elf::VERSYM_VERSION; + if index <= elf::VER_NDX_GLOBAL { + // TODO: return error? + continue; + } + if let Some(verdaux) = verdauxs.next()? { + versions[usize::from(index)] = Version { + name: verdaux.name(endian, strings)?, + hash: verdef.vd_hash.get(endian), + valid: true, + }; + } + } + } + if let Some(mut verneeds) = verneeds { + while let Some((_, mut vernauxs)) = verneeds.next()? { + while let Some(vernaux) = vernauxs.next()? { + let index = vernaux.vna_other.get(endian) & elf::VERSYM_VERSION; + if index <= elf::VER_NDX_GLOBAL { + // TODO: return error? + continue; + } + versions[usize::from(index)] = Version { + name: vernaux.name(endian, strings)?, + hash: vernaux.vna_hash.get(endian), + valid: true, + }; + } + } + } + + Ok(VersionTable { + symbols: versyms, + versions, + }) + } + + /// Return true if the version table is empty. + pub fn is_empty(&self) -> bool { + self.symbols.is_empty() + } + + /// Return version index for a given symbol index. + pub fn version_index(&self, endian: Elf::Endian, index: usize) -> VersionIndex { + let version_index = match self.symbols.get(index) { + Some(x) => x.0.get(endian), + // Ideally this would be VER_NDX_LOCAL for undefined symbols, + // but currently there are no checks that need this distinction. + None => elf::VER_NDX_GLOBAL, + }; + VersionIndex(version_index) + } + + /// Return version information for a given symbol version index. + /// + /// Returns `Ok(None)` for local and global versions. + /// Returns `Err(_)` if index is invalid. + pub fn version(&self, index: VersionIndex) -> Result>> { + if index.index() <= elf::VER_NDX_GLOBAL { + return Ok(None); + } + self.versions + .get(usize::from(index.index())) + .filter(|version| version.valid) + .read_error("Invalid ELF symbol version index") + .map(Some) + } + + /// Return true if the given symbol index satisifies the requirements of `need`. + /// + /// Returns false for any error. + /// + /// Note: this function hasn't been fully tested and is likely to be incomplete. + pub fn matches(&self, endian: Elf::Endian, index: usize, need: Option<&Version>) -> bool { + let version_index = self.version_index(endian, index); + let def = match self.version(version_index) { + Ok(def) => def, + Err(_) => return false, + }; + match (def, need) { + (Some(def), Some(need)) => need.hash == def.hash && need.name == def.name, + (None, Some(_need)) => { + // Version must be present if needed. + false + } + (Some(_def), None) => { + // For a dlsym call, use the newest version. + // TODO: if not a dlsym call, then use the oldest version. + !version_index.is_hidden() + } + (None, None) => true, + } + } +} + +/// An iterator over the entries in an ELF `SHT_GNU_verdef` section. +#[derive(Debug, Clone)] +pub struct VerdefIterator<'data, Elf: FileHeader> { + endian: Elf::Endian, + data: Bytes<'data>, +} + +impl<'data, Elf: FileHeader> VerdefIterator<'data, Elf> { + pub(super) fn new(endian: Elf::Endian, data: &'data [u8]) -> Self { + VerdefIterator { + endian, + data: Bytes(data), + } + } + + /// Return the next `Verdef` entry. + pub fn next( + &mut self, + ) -> Result, VerdauxIterator<'data, Elf>)>> { + if self.data.is_empty() { + return Ok(None); + } + + let verdef = self + .data + .read_at::>(0) + .read_error("ELF verdef is too short")?; + + let mut verdaux_data = self.data; + verdaux_data + .skip(verdef.vd_aux.get(self.endian) as usize) + .read_error("Invalid ELF vd_aux")?; + let verdaux = + VerdauxIterator::new(self.endian, verdaux_data.0, verdef.vd_cnt.get(self.endian)); + + let next = verdef.vd_next.get(self.endian); + if next != 0 { + self.data + .skip(next as usize) + .read_error("Invalid ELF vd_next")?; + } else { + self.data = Bytes(&[]); + } + Ok(Some((verdef, verdaux))) + } +} + +/// An iterator over the auxiliary records for an entry in an ELF `SHT_GNU_verdef` section. +#[derive(Debug, Clone)] +pub struct VerdauxIterator<'data, Elf: FileHeader> { + endian: Elf::Endian, + data: Bytes<'data>, + count: u16, +} + +impl<'data, Elf: FileHeader> VerdauxIterator<'data, Elf> { + pub(super) fn new(endian: Elf::Endian, data: &'data [u8], count: u16) -> Self { + VerdauxIterator { + endian, + data: Bytes(data), + count, + } + } + + /// Return the next `Verdaux` entry. + pub fn next(&mut self) -> Result>> { + if self.count == 0 { + return Ok(None); + } + + let verdaux = self + .data + .read_at::>(0) + .read_error("ELF verdaux is too short")?; + + self.data + .skip(verdaux.vda_next.get(self.endian) as usize) + .read_error("Invalid ELF vda_next")?; + self.count -= 1; + Ok(Some(verdaux)) + } +} + +/// An iterator over the entries in an ELF `SHT_GNU_verneed` section. +#[derive(Debug, Clone)] +pub struct VerneedIterator<'data, Elf: FileHeader> { + endian: Elf::Endian, + data: Bytes<'data>, +} + +impl<'data, Elf: FileHeader> VerneedIterator<'data, Elf> { + pub(super) fn new(endian: Elf::Endian, data: &'data [u8]) -> Self { + VerneedIterator { + endian, + data: Bytes(data), + } + } + + /// Return the next `Verneed` entry. + pub fn next( + &mut self, + ) -> Result< + Option<( + &'data elf::Verneed, + VernauxIterator<'data, Elf>, + )>, + > { + if self.data.is_empty() { + return Ok(None); + } + + let verneed = self + .data + .read_at::>(0) + .read_error("ELF verneed is too short")?; + + let mut vernaux_data = self.data; + vernaux_data + .skip(verneed.vn_aux.get(self.endian) as usize) + .read_error("Invalid ELF vn_aux")?; + let vernaux = + VernauxIterator::new(self.endian, vernaux_data.0, verneed.vn_cnt.get(self.endian)); + + let next = verneed.vn_next.get(self.endian); + if next != 0 { + self.data + .skip(next as usize) + .read_error("Invalid ELF vn_next")?; + } else { + self.data = Bytes(&[]); + } + Ok(Some((verneed, vernaux))) + } +} + +/// An iterator over the auxiliary records for an entry in an ELF `SHT_GNU_verneed` section. +#[derive(Debug, Clone)] +pub struct VernauxIterator<'data, Elf: FileHeader> { + endian: Elf::Endian, + data: Bytes<'data>, + count: u16, +} + +impl<'data, Elf: FileHeader> VernauxIterator<'data, Elf> { + pub(super) fn new(endian: Elf::Endian, data: &'data [u8], count: u16) -> Self { + VernauxIterator { + endian, + data: Bytes(data), + count, + } + } + + /// Return the next `Vernaux` entry. + pub fn next(&mut self) -> Result>> { + if self.count == 0 { + return Ok(None); + } + + let vernaux = self + .data + .read_at::>(0) + .read_error("ELF vernaux is too short")?; + + self.data + .skip(vernaux.vna_next.get(self.endian) as usize) + .read_error("Invalid ELF vna_next")?; + self.count -= 1; + Ok(Some(vernaux)) + } +} + +impl elf::Verdaux { + /// Parse the version name from the string table. + pub fn name<'data, R: ReadRef<'data>>( + &self, + endian: Endian, + strings: StringTable<'data, R>, + ) -> Result<&'data [u8]> { + strings + .get(self.vda_name.get(endian)) + .read_error("Invalid ELF vda_name") + } +} + +impl elf::Verneed { + /// Parse the file from the string table. + pub fn file<'data, R: ReadRef<'data>>( + &self, + endian: Endian, + strings: StringTable<'data, R>, + ) -> Result<&'data [u8]> { + strings + .get(self.vn_file.get(endian)) + .read_error("Invalid ELF vn_file") + } +} + +impl elf::Vernaux { + /// Parse the version name from the string table. + pub fn name<'data, R: ReadRef<'data>>( + &self, + endian: Endian, + strings: StringTable<'data, R>, + ) -> Result<&'data [u8]> { + strings + .get(self.vna_name.get(endian)) + .read_error("Invalid ELF vna_name") + } +} diff --git a/crux-mir/lib/object/src/read/macho/dyld_cache.rs b/crux-mir/lib/object/src/read/macho/dyld_cache.rs new file mode 100644 index 000000000..0839ded7d --- /dev/null +++ b/crux-mir/lib/object/src/read/macho/dyld_cache.rs @@ -0,0 +1,343 @@ +use alloc::vec::Vec; +use core::slice; + +use crate::read::{Error, File, ReadError, ReadRef, Result}; +use crate::{macho, Architecture, Endian, Endianness}; + +/// A parsed representation of the dyld shared cache. +#[derive(Debug)] +pub struct DyldCache<'data, E = Endianness, R = &'data [u8]> +where + E: Endian, + R: ReadRef<'data>, +{ + endian: E, + data: R, + subcaches: Vec>, + mappings: &'data [macho::DyldCacheMappingInfo], + images: &'data [macho::DyldCacheImageInfo], + arch: Architecture, +} + +/// Information about a subcache. +#[derive(Debug)] +pub struct DyldSubCache<'data, E = Endianness, R = &'data [u8]> +where + E: Endian, + R: ReadRef<'data>, +{ + data: R, + mappings: &'data [macho::DyldCacheMappingInfo], +} + +// This is the offset of the images_across_all_subcaches_count field. +const MIN_HEADER_SIZE_SUBCACHES: u32 = 0x1c4; + +impl<'data, E, R> DyldCache<'data, E, R> +where + E: Endian, + R: ReadRef<'data>, +{ + /// Parse the raw dyld shared cache data. + /// For shared caches from macOS 12 / iOS 15 and above, the subcache files need to be + /// supplied as well, in the correct order, with the .symbols subcache last (if present). + /// For example, data would be the data for dyld_shared_cache_x86_64, + /// and subcache_data would be the data for [dyld_shared_cache_x86_64.1, dyld_shared_cache_x86_64.2, ...] + pub fn parse(data: R, subcache_data: &[R]) -> Result { + let header = macho::DyldCacheHeader::parse(data)?; + let (arch, endian) = header.parse_magic()?; + let mappings = header.mappings(endian, data)?; + + let symbols_subcache_uuid = header.symbols_subcache_uuid(endian); + let subcaches_info = header.subcaches(endian, data)?.unwrap_or(&[]); + + if subcache_data.len() != subcaches_info.len() + symbols_subcache_uuid.is_some() as usize { + return Err(Error("Incorrect number of SubCaches")); + } + + // Split out the .symbols subcache data from the other subcaches. + let (symbols_subcache_data_and_uuid, subcache_data) = + if let Some(symbols_uuid) = symbols_subcache_uuid { + let (sym_data, rest_data) = subcache_data.split_last().unwrap(); + (Some((*sym_data, symbols_uuid)), rest_data) + } else { + (None, subcache_data) + }; + + // Read the regular SubCaches (.1, .2, ...), if present. + let mut subcaches = Vec::new(); + for (&data, info) in subcache_data.iter().zip(subcaches_info.iter()) { + let sc_header = macho::DyldCacheHeader::::parse(data)?; + if sc_header.uuid != info.uuid { + return Err(Error("Unexpected SubCache UUID")); + } + let mappings = sc_header.mappings(endian, data)?; + subcaches.push(DyldSubCache { data, mappings }); + } + + // Read the .symbols SubCache, if present. + // Other than the UUID verification, the symbols SubCache is currently unused. + let _symbols_subcache = match symbols_subcache_data_and_uuid { + Some((data, uuid)) => { + let sc_header = macho::DyldCacheHeader::::parse(data)?; + if sc_header.uuid != uuid { + return Err(Error("Unexpected .symbols SubCache UUID")); + } + let mappings = sc_header.mappings(endian, data)?; + Some(DyldSubCache { data, mappings }) + } + None => None, + }; + + let images = header.images(endian, data)?; + Ok(DyldCache { + endian, + data, + subcaches, + mappings, + images, + arch, + }) + } + + /// Get the architecture type of the file. + pub fn architecture(&self) -> Architecture { + self.arch + } + + /// Get the endianness of the file. + #[inline] + pub fn endianness(&self) -> Endianness { + if self.is_little_endian() { + Endianness::Little + } else { + Endianness::Big + } + } + + /// Return true if the file is little endian, false if it is big endian. + pub fn is_little_endian(&self) -> bool { + self.endian.is_little_endian() + } + + /// Iterate over the images in this cache. + pub fn images<'cache>(&'cache self) -> DyldCacheImageIterator<'data, 'cache, E, R> { + DyldCacheImageIterator { + cache: self, + iter: self.images.iter(), + } + } + + /// Find the address in a mapping and return the cache or subcache data it was found in, + /// together with the translated file offset. + pub fn data_and_offset_for_address(&self, address: u64) -> Option<(R, u64)> { + if let Some(file_offset) = address_to_file_offset(address, self.endian, self.mappings) { + return Some((self.data, file_offset)); + } + for subcache in &self.subcaches { + if let Some(file_offset) = + address_to_file_offset(address, self.endian, subcache.mappings) + { + return Some((subcache.data, file_offset)); + } + } + None + } +} + +/// An iterator over all the images (dylibs) in the dyld shared cache. +#[derive(Debug)] +pub struct DyldCacheImageIterator<'data, 'cache, E = Endianness, R = &'data [u8]> +where + E: Endian, + R: ReadRef<'data>, +{ + cache: &'cache DyldCache<'data, E, R>, + iter: slice::Iter<'data, macho::DyldCacheImageInfo>, +} + +impl<'data, 'cache, E, R> Iterator for DyldCacheImageIterator<'data, 'cache, E, R> +where + E: Endian, + R: ReadRef<'data>, +{ + type Item = DyldCacheImage<'data, 'cache, E, R>; + + fn next(&mut self) -> Option> { + let image_info = self.iter.next()?; + Some(DyldCacheImage { + cache: self.cache, + image_info, + }) + } +} + +/// One image (dylib) from inside the dyld shared cache. +#[derive(Debug)] +pub struct DyldCacheImage<'data, 'cache, E = Endianness, R = &'data [u8]> +where + E: Endian, + R: ReadRef<'data>, +{ + pub(crate) cache: &'cache DyldCache<'data, E, R>, + image_info: &'data macho::DyldCacheImageInfo, +} + +impl<'data, 'cache, E, R> DyldCacheImage<'data, 'cache, E, R> +where + E: Endian, + R: ReadRef<'data>, +{ + /// The file system path of this image. + pub fn path(&self) -> Result<&'data str> { + let path = self.image_info.path(self.cache.endian, self.cache.data)?; + // The path should always be ascii, so from_utf8 should alway succeed. + let path = core::str::from_utf8(path).map_err(|_| Error("Path string not valid utf-8"))?; + Ok(path) + } + + /// The subcache data which contains the Mach-O header for this image, + /// together with the file offset at which this image starts. + pub fn image_data_and_offset(&self) -> Result<(R, u64)> { + let address = self.image_info.address.get(self.cache.endian); + self.cache + .data_and_offset_for_address(address) + .ok_or(Error("Address not found in any mapping")) + } + + /// Parse this image into an Object. + pub fn parse_object(&self) -> Result> { + File::parse_dyld_cache_image(self) + } +} + +impl macho::DyldCacheHeader { + /// Read the dyld cache header. + pub fn parse<'data, R: ReadRef<'data>>(data: R) -> Result<&'data Self> { + data.read_at::>(0) + .read_error("Invalid dyld cache header size or alignment") + } + + /// Returns (arch, endian) based on the magic string. + pub fn parse_magic(&self) -> Result<(Architecture, E)> { + let (arch, is_big_endian) = match &self.magic { + b"dyld_v1 i386\0" => (Architecture::I386, false), + b"dyld_v1 x86_64\0" => (Architecture::X86_64, false), + b"dyld_v1 x86_64h\0" => (Architecture::X86_64, false), + b"dyld_v1 ppc\0" => (Architecture::PowerPc, true), + b"dyld_v1 armv6\0" => (Architecture::Arm, false), + b"dyld_v1 armv7\0" => (Architecture::Arm, false), + b"dyld_v1 armv7f\0" => (Architecture::Arm, false), + b"dyld_v1 armv7s\0" => (Architecture::Arm, false), + b"dyld_v1 armv7k\0" => (Architecture::Arm, false), + b"dyld_v1 arm64\0" => (Architecture::Aarch64, false), + b"dyld_v1 arm64e\0" => (Architecture::Aarch64, false), + _ => return Err(Error("Unrecognized dyld cache magic")), + }; + let endian = + E::from_big_endian(is_big_endian).read_error("Unsupported dyld cache endian")?; + Ok((arch, endian)) + } + + /// Return the mapping information table. + pub fn mappings<'data, R: ReadRef<'data>>( + &self, + endian: E, + data: R, + ) -> Result<&'data [macho::DyldCacheMappingInfo]> { + data.read_slice_at::>( + self.mapping_offset.get(endian).into(), + self.mapping_count.get(endian) as usize, + ) + .read_error("Invalid dyld cache mapping size or alignment") + } + + /// Return the information about subcaches, if present. + pub fn subcaches<'data, R: ReadRef<'data>>( + &self, + endian: E, + data: R, + ) -> Result]>> { + if self.mapping_offset.get(endian) >= MIN_HEADER_SIZE_SUBCACHES { + let subcaches = data + .read_slice_at::>( + self.subcaches_offset.get(endian).into(), + self.subcaches_count.get(endian) as usize, + ) + .read_error("Invalid dyld subcaches size or alignment")?; + Ok(Some(subcaches)) + } else { + Ok(None) + } + } + + /// Return the UUID for the .symbols subcache, if present. + pub fn symbols_subcache_uuid(&self, endian: E) -> Option<[u8; 16]> { + if self.mapping_offset.get(endian) >= MIN_HEADER_SIZE_SUBCACHES { + let uuid = self.symbols_subcache_uuid; + if uuid != [0; 16] { + return Some(uuid); + } + } + None + } + + /// Return the image information table. + pub fn images<'data, R: ReadRef<'data>>( + &self, + endian: E, + data: R, + ) -> Result<&'data [macho::DyldCacheImageInfo]> { + if self.mapping_offset.get(endian) >= MIN_HEADER_SIZE_SUBCACHES { + data.read_slice_at::>( + self.images_across_all_subcaches_offset.get(endian).into(), + self.images_across_all_subcaches_count.get(endian) as usize, + ) + .read_error("Invalid dyld cache image size or alignment") + } else { + data.read_slice_at::>( + self.images_offset.get(endian).into(), + self.images_count.get(endian) as usize, + ) + .read_error("Invalid dyld cache image size or alignment") + } + } +} + +impl macho::DyldCacheImageInfo { + /// The file system path of this image. + pub fn path<'data, R: ReadRef<'data>>(&self, endian: E, data: R) -> Result<&'data [u8]> { + let r_start = self.path_file_offset.get(endian).into(); + let r_end = data.len().read_error("Couldn't get data len()")?; + data.read_bytes_at_until(r_start..r_end, 0) + .read_error("Couldn't read dyld cache image path") + } + + /// Find the file offset of the image by looking up its address in the mappings. + pub fn file_offset( + &self, + endian: E, + mappings: &[macho::DyldCacheMappingInfo], + ) -> Result { + let address = self.address.get(endian); + address_to_file_offset(address, endian, mappings) + .read_error("Invalid dyld cache image address") + } +} + +/// Find the file offset of the image by looking up its address in the mappings. +pub fn address_to_file_offset( + address: u64, + endian: E, + mappings: &[macho::DyldCacheMappingInfo], +) -> Option { + for mapping in mappings { + let mapping_address = mapping.address.get(endian); + if address >= mapping_address + && address < mapping_address.wrapping_add(mapping.size.get(endian)) + { + return Some(address - mapping_address + mapping.file_offset.get(endian)); + } + } + None +} diff --git a/crux-mir/lib/object/src/read/macho/fat.rs b/crux-mir/lib/object/src/read/macho/fat.rs new file mode 100644 index 000000000..6fc649f31 --- /dev/null +++ b/crux-mir/lib/object/src/read/macho/fat.rs @@ -0,0 +1,120 @@ +use crate::read::{Architecture, Error, ReadError, ReadRef, Result}; +use crate::{macho, BigEndian, Pod}; + +pub use macho::{FatArch32, FatArch64, FatHeader}; + +impl FatHeader { + /// Attempt to parse a fat header. + /// + /// Does not validate the magic value. + pub fn parse<'data, R: ReadRef<'data>>(file: R) -> Result<&'data FatHeader> { + file.read_at::(0) + .read_error("Invalid fat header size or alignment") + } + + /// Attempt to parse a fat header and 32-bit fat arches. + pub fn parse_arch32<'data, R: ReadRef<'data>>(file: R) -> Result<&'data [FatArch32]> { + let mut offset = 0; + let header = file + .read::(&mut offset) + .read_error("Invalid fat header size or alignment")?; + if header.magic.get(BigEndian) != macho::FAT_MAGIC { + return Err(Error("Invalid 32-bit fat magic")); + } + file.read_slice::(&mut offset, header.nfat_arch.get(BigEndian) as usize) + .read_error("Invalid nfat_arch") + } + + /// Attempt to parse a fat header and 64-bit fat arches. + pub fn parse_arch64<'data, R: ReadRef<'data>>(file: R) -> Result<&'data [FatArch64]> { + let mut offset = 0; + let header = file + .read::(&mut offset) + .read_error("Invalid fat header size or alignment")?; + if header.magic.get(BigEndian) != macho::FAT_MAGIC_64 { + return Err(Error("Invalid 64-bit fat magic")); + } + file.read_slice::(&mut offset, header.nfat_arch.get(BigEndian) as usize) + .read_error("Invalid nfat_arch") + } +} + +/// A trait for generic access to `FatArch32` and `FatArch64`. +#[allow(missing_docs)] +pub trait FatArch: Pod { + type Word: Into; + + fn cputype(&self) -> u32; + fn cpusubtype(&self) -> u32; + fn offset(&self) -> Self::Word; + fn size(&self) -> Self::Word; + fn align(&self) -> u32; + + fn architecture(&self) -> Architecture { + match self.cputype() { + macho::CPU_TYPE_ARM => Architecture::Arm, + macho::CPU_TYPE_ARM64 => Architecture::Aarch64, + macho::CPU_TYPE_X86 => Architecture::I386, + macho::CPU_TYPE_X86_64 => Architecture::X86_64, + macho::CPU_TYPE_MIPS => Architecture::Mips, + _ => Architecture::Unknown, + } + } + + fn file_range(&self) -> (u64, u64) { + (self.offset().into(), self.size().into()) + } + + fn data<'data, R: ReadRef<'data>>(&self, file: R) -> Result<&'data [u8]> { + file.read_bytes_at(self.offset().into(), self.size().into()) + .read_error("Invalid fat arch offset or size") + } +} + +impl FatArch for FatArch32 { + type Word = u32; + + fn cputype(&self) -> u32 { + self.cputype.get(BigEndian) + } + + fn cpusubtype(&self) -> u32 { + self.cpusubtype.get(BigEndian) + } + + fn offset(&self) -> Self::Word { + self.offset.get(BigEndian) + } + + fn size(&self) -> Self::Word { + self.size.get(BigEndian) + } + + fn align(&self) -> u32 { + self.align.get(BigEndian) + } +} + +impl FatArch for FatArch64 { + type Word = u64; + + fn cputype(&self) -> u32 { + self.cputype.get(BigEndian) + } + + fn cpusubtype(&self) -> u32 { + self.cpusubtype.get(BigEndian) + } + + fn offset(&self) -> Self::Word { + self.offset.get(BigEndian) + } + + fn size(&self) -> Self::Word { + self.size.get(BigEndian) + } + + fn align(&self) -> u32 { + self.align.get(BigEndian) + } +} diff --git a/crux-mir/lib/object/src/read/macho/file.rs b/crux-mir/lib/object/src/read/macho/file.rs new file mode 100644 index 000000000..e028de3b9 --- /dev/null +++ b/crux-mir/lib/object/src/read/macho/file.rs @@ -0,0 +1,729 @@ +use alloc::vec::Vec; +use core::fmt::Debug; +use core::{mem, str}; + +use crate::read::{ + self, Architecture, ComdatKind, Error, Export, FileFlags, Import, NoDynamicRelocationIterator, + Object, ObjectComdat, ObjectKind, ObjectMap, ObjectSection, ReadError, ReadRef, Result, + SectionIndex, SymbolIndex, +}; +use crate::{endian, macho, BigEndian, ByteString, Endian, Endianness, Pod}; + +use super::{ + DyldCacheImage, LoadCommandIterator, MachOSection, MachOSectionInternal, MachOSectionIterator, + MachOSegment, MachOSegmentInternal, MachOSegmentIterator, MachOSymbol, MachOSymbolIterator, + MachOSymbolTable, Nlist, Section, Segment, SymbolTable, +}; + +/// A 32-bit Mach-O object file. +pub type MachOFile32<'data, Endian = Endianness, R = &'data [u8]> = + MachOFile<'data, macho::MachHeader32, R>; +/// A 64-bit Mach-O object file. +pub type MachOFile64<'data, Endian = Endianness, R = &'data [u8]> = + MachOFile<'data, macho::MachHeader64, R>; + +/// A partially parsed Mach-O file. +/// +/// Most of the functionality of this type is provided by the `Object` trait implementation. +#[derive(Debug)] +pub struct MachOFile<'data, Mach, R = &'data [u8]> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + pub(super) endian: Mach::Endian, + pub(super) data: R, + pub(super) header_offset: u64, + pub(super) header: &'data Mach, + pub(super) segments: Vec>, + pub(super) sections: Vec>, + pub(super) symbols: SymbolTable<'data, Mach, R>, +} + +impl<'data, Mach, R> MachOFile<'data, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + /// Parse the raw Mach-O file data. + pub fn parse(data: R) -> Result { + let header = Mach::parse(data, 0)?; + let endian = header.endian()?; + + // Build a list of segments and sections to make some operations more efficient. + let mut segments = Vec::new(); + let mut sections = Vec::new(); + let mut symbols = SymbolTable::default(); + if let Ok(mut commands) = header.load_commands(endian, data, 0) { + while let Ok(Some(command)) = commands.next() { + if let Some((segment, section_data)) = Mach::Segment::from_command(command)? { + let segment_index = segments.len(); + segments.push(MachOSegmentInternal { segment, data }); + for section in segment.sections(endian, section_data)? { + let index = SectionIndex(sections.len() + 1); + sections.push(MachOSectionInternal::parse(index, segment_index, section)); + } + } else if let Some(symtab) = command.symtab()? { + symbols = symtab.symbols(endian, data)?; + } + } + } + + Ok(MachOFile { + endian, + data, + header_offset: 0, + header, + segments, + sections, + symbols, + }) + } + + /// Parse the Mach-O file for the given image from the dyld shared cache. + /// This will read different sections from different subcaches, if necessary. + pub fn parse_dyld_cache_image<'cache, E: Endian>( + image: &DyldCacheImage<'data, 'cache, E, R>, + ) -> Result { + let (data, header_offset) = image.image_data_and_offset()?; + let header = Mach::parse(data, header_offset)?; + let endian = header.endian()?; + + // Build a list of sections to make some operations more efficient. + // Also build a list of segments, because we need to remember which ReadRef + // to read each section's data from. Only the DyldCache knows this information, + // and we won't have access to it once we've exited this function. + let mut segments = Vec::new(); + let mut sections = Vec::new(); + let mut linkedit_data: Option = None; + let mut symtab = None; + if let Ok(mut commands) = header.load_commands(endian, data, header_offset) { + while let Ok(Some(command)) = commands.next() { + if let Some((segment, section_data)) = Mach::Segment::from_command(command)? { + // Each segment can be stored in a different subcache. Get the segment's + // address and look it up in the cache mappings, to find the correct cache data. + let addr = segment.vmaddr(endian).into(); + let (data, _offset) = image + .cache + .data_and_offset_for_address(addr) + .read_error("Could not find segment data in dyld shared cache")?; + if segment.name() == macho::SEG_LINKEDIT.as_bytes() { + linkedit_data = Some(data); + } + let segment_index = segments.len(); + segments.push(MachOSegmentInternal { segment, data }); + + for section in segment.sections(endian, section_data)? { + let index = SectionIndex(sections.len() + 1); + sections.push(MachOSectionInternal::parse(index, segment_index, section)); + } + } else if let Some(st) = command.symtab()? { + symtab = Some(st); + } + } + } + + // The symbols are found in the __LINKEDIT segment, so make sure to read them from the + // correct subcache. + let symbols = match (symtab, linkedit_data) { + (Some(symtab), Some(linkedit_data)) => symtab.symbols(endian, linkedit_data)?, + _ => SymbolTable::default(), + }; + + Ok(MachOFile { + endian, + data, + header_offset, + header, + segments, + sections, + symbols, + }) + } + + /// Return the section at the given index. + #[inline] + pub(super) fn section_internal( + &self, + index: SectionIndex, + ) -> Result<&MachOSectionInternal<'data, Mach>> { + index + .0 + .checked_sub(1) + .and_then(|index| self.sections.get(index)) + .read_error("Invalid Mach-O section index") + } + + pub(super) fn segment_internal( + &self, + index: usize, + ) -> Result<&MachOSegmentInternal<'data, Mach, R>> { + self.segments + .get(index) + .read_error("Invalid Mach-O segment index") + } +} + +impl<'data, Mach, R> read::private::Sealed for MachOFile<'data, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ +} + +impl<'data, 'file, Mach, R> Object<'data, 'file> for MachOFile<'data, Mach, R> +where + 'data: 'file, + Mach: MachHeader, + R: 'file + ReadRef<'data>, +{ + type Segment = MachOSegment<'data, 'file, Mach, R>; + type SegmentIterator = MachOSegmentIterator<'data, 'file, Mach, R>; + type Section = MachOSection<'data, 'file, Mach, R>; + type SectionIterator = MachOSectionIterator<'data, 'file, Mach, R>; + type Comdat = MachOComdat<'data, 'file, Mach, R>; + type ComdatIterator = MachOComdatIterator<'data, 'file, Mach, R>; + type Symbol = MachOSymbol<'data, 'file, Mach, R>; + type SymbolIterator = MachOSymbolIterator<'data, 'file, Mach, R>; + type SymbolTable = MachOSymbolTable<'data, 'file, Mach, R>; + type DynamicRelocationIterator = NoDynamicRelocationIterator; + + fn architecture(&self) -> Architecture { + match self.header.cputype(self.endian) { + macho::CPU_TYPE_ARM => Architecture::Arm, + macho::CPU_TYPE_ARM64 => Architecture::Aarch64, + macho::CPU_TYPE_X86 => Architecture::I386, + macho::CPU_TYPE_X86_64 => Architecture::X86_64, + macho::CPU_TYPE_MIPS => Architecture::Mips, + _ => Architecture::Unknown, + } + } + + #[inline] + fn is_little_endian(&self) -> bool { + self.header.is_little_endian() + } + + #[inline] + fn is_64(&self) -> bool { + self.header.is_type_64() + } + + fn kind(&self) -> ObjectKind { + match self.header.filetype(self.endian) { + macho::MH_OBJECT => ObjectKind::Relocatable, + macho::MH_EXECUTE => ObjectKind::Executable, + macho::MH_CORE => ObjectKind::Core, + macho::MH_DYLIB => ObjectKind::Dynamic, + _ => ObjectKind::Unknown, + } + } + + fn segments(&'file self) -> MachOSegmentIterator<'data, 'file, Mach, R> { + MachOSegmentIterator { + file: self, + iter: self.segments.iter(), + } + } + + fn section_by_name_bytes( + &'file self, + section_name: &[u8], + ) -> Option> { + // Translate the "." prefix to the "__" prefix used by OSX/Mach-O, eg + // ".debug_info" to "__debug_info", and limit to 16 bytes total. + let system_name = if section_name.starts_with(b".") { + if section_name.len() > 15 { + Some(§ion_name[1..15]) + } else { + Some(§ion_name[1..]) + } + } else { + None + }; + let cmp_section_name = |section: &MachOSection<'data, 'file, Mach, R>| { + section + .name_bytes() + .map(|name| { + section_name == name + || system_name + .filter(|system_name| { + name.starts_with(b"__") && name[2..] == **system_name + }) + .is_some() + }) + .unwrap_or(false) + }; + + self.sections().find(cmp_section_name) + } + + fn section_by_index( + &'file self, + index: SectionIndex, + ) -> Result> { + let internal = *self.section_internal(index)?; + Ok(MachOSection { + file: self, + internal, + }) + } + + fn sections(&'file self) -> MachOSectionIterator<'data, 'file, Mach, R> { + MachOSectionIterator { + file: self, + iter: self.sections.iter(), + } + } + + fn comdats(&'file self) -> MachOComdatIterator<'data, 'file, Mach, R> { + MachOComdatIterator { file: self } + } + + fn symbol_by_index( + &'file self, + index: SymbolIndex, + ) -> Result> { + let nlist = self.symbols.symbol(index.0)?; + MachOSymbol::new(self, index, nlist).read_error("Unsupported Mach-O symbol index") + } + + fn symbols(&'file self) -> MachOSymbolIterator<'data, 'file, Mach, R> { + MachOSymbolIterator { + file: self, + index: 0, + } + } + + #[inline] + fn symbol_table(&'file self) -> Option> { + Some(MachOSymbolTable { file: self }) + } + + fn dynamic_symbols(&'file self) -> MachOSymbolIterator<'data, 'file, Mach, R> { + MachOSymbolIterator { + file: self, + index: self.symbols.len(), + } + } + + #[inline] + fn dynamic_symbol_table(&'file self) -> Option> { + None + } + + fn object_map(&'file self) -> ObjectMap<'data> { + self.symbols.object_map(self.endian) + } + + fn imports(&self) -> Result>> { + let mut dysymtab = None; + let mut libraries = Vec::new(); + let twolevel = self.header.flags(self.endian) & macho::MH_TWOLEVEL != 0; + if twolevel { + libraries.push(&[][..]); + } + let mut commands = self + .header + .load_commands(self.endian, self.data, self.header_offset)?; + while let Some(command) = commands.next()? { + if let Some(command) = command.dysymtab()? { + dysymtab = Some(command); + } + if twolevel { + if let Some(dylib) = command.dylib()? { + libraries.push(command.string(self.endian, dylib.dylib.name)?); + } + } + } + + let mut imports = Vec::new(); + if let Some(dysymtab) = dysymtab { + let index = dysymtab.iundefsym.get(self.endian) as usize; + let number = dysymtab.nundefsym.get(self.endian) as usize; + for i in index..(index.wrapping_add(number)) { + let symbol = self.symbols.symbol(i)?; + let name = symbol.name(self.endian, self.symbols.strings())?; + let library = if twolevel { + libraries + .get(symbol.library_ordinal(self.endian) as usize) + .copied() + .read_error("Invalid Mach-O symbol library ordinal")? + } else { + &[] + }; + imports.push(Import { + name: ByteString(name), + library: ByteString(library), + }); + } + } + Ok(imports) + } + + fn exports(&self) -> Result>> { + let mut dysymtab = None; + let mut commands = self + .header + .load_commands(self.endian, self.data, self.header_offset)?; + while let Some(command) = commands.next()? { + if let Some(command) = command.dysymtab()? { + dysymtab = Some(command); + break; + } + } + + let mut exports = Vec::new(); + if let Some(dysymtab) = dysymtab { + let index = dysymtab.iextdefsym.get(self.endian) as usize; + let number = dysymtab.nextdefsym.get(self.endian) as usize; + for i in index..(index.wrapping_add(number)) { + let symbol = self.symbols.symbol(i)?; + let name = symbol.name(self.endian, self.symbols.strings())?; + let address = symbol.n_value(self.endian).into(); + exports.push(Export { + name: ByteString(name), + address, + }); + } + } + Ok(exports) + } + + #[inline] + fn dynamic_relocations(&'file self) -> Option { + None + } + + fn has_debug_symbols(&self) -> bool { + self.section_by_name(".debug_info").is_some() + } + + fn mach_uuid(&self) -> Result> { + self.header.uuid(self.endian, self.data, self.header_offset) + } + + fn relative_address_base(&self) -> u64 { + 0 + } + + fn entry(&self) -> u64 { + if let Ok(mut commands) = + self.header + .load_commands(self.endian, self.data, self.header_offset) + { + while let Ok(Some(command)) = commands.next() { + if let Ok(Some(command)) = command.entry_point() { + return command.entryoff.get(self.endian); + } + } + } + 0 + } + + fn flags(&self) -> FileFlags { + FileFlags::MachO { + flags: self.header.flags(self.endian), + } + } +} + +/// An iterator over the COMDAT section groups of a `MachOFile64`. +pub type MachOComdatIterator32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachOComdatIterator<'data, 'file, macho::MachHeader32, R>; +/// An iterator over the COMDAT section groups of a `MachOFile64`. +pub type MachOComdatIterator64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachOComdatIterator<'data, 'file, macho::MachHeader64, R>; + +/// An iterator over the COMDAT section groups of a `MachOFile`. +#[derive(Debug)] +pub struct MachOComdatIterator<'data, 'file, Mach, R = &'data [u8]> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + #[allow(unused)] + file: &'file MachOFile<'data, Mach, R>, +} + +impl<'data, 'file, Mach, R> Iterator for MachOComdatIterator<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + type Item = MachOComdat<'data, 'file, Mach, R>; + + #[inline] + fn next(&mut self) -> Option { + None + } +} + +/// A COMDAT section group of a `MachOFile32`. +pub type MachOComdat32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachOComdat<'data, 'file, macho::MachHeader32, R>; + +/// A COMDAT section group of a `MachOFile64`. +pub type MachOComdat64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachOComdat<'data, 'file, macho::MachHeader64, R>; + +/// A COMDAT section group of a `MachOFile`. +#[derive(Debug)] +pub struct MachOComdat<'data, 'file, Mach, R = &'data [u8]> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + #[allow(unused)] + file: &'file MachOFile<'data, Mach, R>, +} + +impl<'data, 'file, Mach, R> read::private::Sealed for MachOComdat<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ +} + +impl<'data, 'file, Mach, R> ObjectComdat<'data> for MachOComdat<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + type SectionIterator = MachOComdatSectionIterator<'data, 'file, Mach, R>; + + #[inline] + fn kind(&self) -> ComdatKind { + unreachable!(); + } + + #[inline] + fn symbol(&self) -> SymbolIndex { + unreachable!(); + } + + #[inline] + fn name_bytes(&self) -> Result<&[u8]> { + unreachable!(); + } + + #[inline] + fn name(&self) -> Result<&str> { + unreachable!(); + } + + #[inline] + fn sections(&self) -> Self::SectionIterator { + unreachable!(); + } +} + +/// An iterator over the sections in a COMDAT section group of a `MachOFile32`. +pub type MachOComdatSectionIterator32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachOComdatSectionIterator<'data, 'file, macho::MachHeader32, R>; +/// An iterator over the sections in a COMDAT section group of a `MachOFile64`. +pub type MachOComdatSectionIterator64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachOComdatSectionIterator<'data, 'file, macho::MachHeader64, R>; + +/// An iterator over the sections in a COMDAT section group of a `MachOFile`. +#[derive(Debug)] +pub struct MachOComdatSectionIterator<'data, 'file, Mach, R = &'data [u8]> +where + 'data: 'file, + Mach: MachHeader, + R: ReadRef<'data>, +{ + #[allow(unused)] + file: &'file MachOFile<'data, Mach, R>, +} + +impl<'data, 'file, Mach, R> Iterator for MachOComdatSectionIterator<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + type Item = SectionIndex; + + fn next(&mut self) -> Option { + None + } +} + +/// A trait for generic access to `MachHeader32` and `MachHeader64`. +#[allow(missing_docs)] +pub trait MachHeader: Debug + Pod { + type Word: Into; + type Endian: endian::Endian; + type Segment: Segment; + type Section: Section; + type Nlist: Nlist; + + /// Return true if this type is a 64-bit header. + /// + /// This is a property of the type, not a value in the header data. + fn is_type_64(&self) -> bool; + + /// Return true if the `magic` field signifies big-endian. + fn is_big_endian(&self) -> bool; + + /// Return true if the `magic` field signifies little-endian. + fn is_little_endian(&self) -> bool; + + fn magic(&self) -> u32; + fn cputype(&self, endian: Self::Endian) -> u32; + fn cpusubtype(&self, endian: Self::Endian) -> u32; + fn filetype(&self, endian: Self::Endian) -> u32; + fn ncmds(&self, endian: Self::Endian) -> u32; + fn sizeofcmds(&self, endian: Self::Endian) -> u32; + fn flags(&self, endian: Self::Endian) -> u32; + + // Provided methods. + + /// Read the file header. + /// + /// Also checks that the magic field in the file header is a supported format. + fn parse<'data, R: ReadRef<'data>>(data: R, offset: u64) -> read::Result<&'data Self> { + let header = data + .read_at::(offset) + .read_error("Invalid Mach-O header size or alignment")?; + if !header.is_supported() { + return Err(Error("Unsupported Mach-O header")); + } + Ok(header) + } + + fn is_supported(&self) -> bool { + self.is_little_endian() || self.is_big_endian() + } + + fn endian(&self) -> Result { + Self::Endian::from_big_endian(self.is_big_endian()).read_error("Unsupported Mach-O endian") + } + + fn load_commands<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + header_offset: u64, + ) -> Result> { + let data = data + .read_bytes_at( + header_offset + mem::size_of::() as u64, + self.sizeofcmds(endian).into(), + ) + .read_error("Invalid Mach-O load command table size")?; + Ok(LoadCommandIterator::new(endian, data, self.ncmds(endian))) + } + + /// Return the UUID from the `LC_UUID` load command, if one is present. + fn uuid<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + header_offset: u64, + ) -> Result> { + let mut commands = self.load_commands(endian, data, header_offset)?; + while let Some(command) = commands.next()? { + if let Ok(Some(uuid)) = command.uuid() { + return Ok(Some(uuid.uuid)); + } + } + Ok(None) + } +} + +impl MachHeader for macho::MachHeader32 { + type Word = u32; + type Endian = Endian; + type Segment = macho::SegmentCommand32; + type Section = macho::Section32; + type Nlist = macho::Nlist32; + + fn is_type_64(&self) -> bool { + false + } + + fn is_big_endian(&self) -> bool { + self.magic() == macho::MH_MAGIC + } + + fn is_little_endian(&self) -> bool { + self.magic() == macho::MH_CIGAM + } + + fn magic(&self) -> u32 { + self.magic.get(BigEndian) + } + + fn cputype(&self, endian: Self::Endian) -> u32 { + self.cputype.get(endian) + } + + fn cpusubtype(&self, endian: Self::Endian) -> u32 { + self.cpusubtype.get(endian) + } + + fn filetype(&self, endian: Self::Endian) -> u32 { + self.filetype.get(endian) + } + + fn ncmds(&self, endian: Self::Endian) -> u32 { + self.ncmds.get(endian) + } + + fn sizeofcmds(&self, endian: Self::Endian) -> u32 { + self.sizeofcmds.get(endian) + } + + fn flags(&self, endian: Self::Endian) -> u32 { + self.flags.get(endian) + } +} + +impl MachHeader for macho::MachHeader64 { + type Word = u64; + type Endian = Endian; + type Segment = macho::SegmentCommand64; + type Section = macho::Section64; + type Nlist = macho::Nlist64; + + fn is_type_64(&self) -> bool { + true + } + + fn is_big_endian(&self) -> bool { + self.magic() == macho::MH_MAGIC_64 + } + + fn is_little_endian(&self) -> bool { + self.magic() == macho::MH_CIGAM_64 + } + + fn magic(&self) -> u32 { + self.magic.get(BigEndian) + } + + fn cputype(&self, endian: Self::Endian) -> u32 { + self.cputype.get(endian) + } + + fn cpusubtype(&self, endian: Self::Endian) -> u32 { + self.cpusubtype.get(endian) + } + + fn filetype(&self, endian: Self::Endian) -> u32 { + self.filetype.get(endian) + } + + fn ncmds(&self, endian: Self::Endian) -> u32 { + self.ncmds.get(endian) + } + + fn sizeofcmds(&self, endian: Self::Endian) -> u32 { + self.sizeofcmds.get(endian) + } + + fn flags(&self, endian: Self::Endian) -> u32 { + self.flags.get(endian) + } +} diff --git a/crux-mir/lib/object/src/read/macho/load_command.rs b/crux-mir/lib/object/src/read/macho/load_command.rs new file mode 100644 index 000000000..29fab6e0e --- /dev/null +++ b/crux-mir/lib/object/src/read/macho/load_command.rs @@ -0,0 +1,348 @@ +use core::marker::PhantomData; + +use crate::endian::Endian; +use crate::macho; +use crate::pod::Pod; +use crate::read::macho::{MachHeader, SymbolTable}; +use crate::read::{Bytes, ReadError, ReadRef, Result, StringTable}; + +/// An iterator over the load commands of a `MachHeader`. +#[derive(Debug, Default, Clone, Copy)] +pub struct LoadCommandIterator<'data, E: Endian> { + endian: E, + data: Bytes<'data>, + ncmds: u32, +} + +impl<'data, E: Endian> LoadCommandIterator<'data, E> { + pub(super) fn new(endian: E, data: &'data [u8], ncmds: u32) -> Self { + LoadCommandIterator { + endian, + data: Bytes(data), + ncmds, + } + } + + /// Return the next load command. + pub fn next(&mut self) -> Result>> { + if self.ncmds == 0 { + return Ok(None); + } + let header = self + .data + .read_at::>(0) + .read_error("Invalid Mach-O load command header")?; + let cmd = header.cmd.get(self.endian); + let cmdsize = header.cmdsize.get(self.endian) as usize; + let data = self + .data + .read_bytes(cmdsize) + .read_error("Invalid Mach-O load command size")?; + self.ncmds -= 1; + Ok(Some(LoadCommandData { + cmd, + data, + marker: Default::default(), + })) + } +} + +/// The data for a `LoadCommand`. +#[derive(Debug, Clone, Copy)] +pub struct LoadCommandData<'data, E: Endian> { + cmd: u32, + // Includes the header. + data: Bytes<'data>, + marker: PhantomData, +} + +impl<'data, E: Endian> LoadCommandData<'data, E> { + /// Return the `cmd` field of the `LoadCommand`. + /// + /// This is one of the `LC_` constants. + pub fn cmd(&self) -> u32 { + self.cmd + } + + /// Return the `cmdsize` field of the `LoadCommand`. + pub fn cmdsize(&self) -> u32 { + self.data.len() as u32 + } + + /// Parse the data as the given type. + #[inline] + pub fn data(&self) -> Result<&'data T> { + self.data + .read_at(0) + .read_error("Invalid Mach-O command size") + } + + /// Parse a load command string value. + /// + /// Strings used by load commands are specified by offsets that are + /// relative to the load command header. + pub fn string(&self, endian: E, s: macho::LcStr) -> Result<&'data [u8]> { + self.data + .read_string_at(s.offset.get(endian) as usize) + .read_error("Invalid load command string offset") + } + + /// Parse the command data according to the `cmd` field. + pub fn variant(&self) -> Result> { + Ok(match self.cmd { + macho::LC_SEGMENT => { + let mut data = self.data; + let segment = data.read().read_error("Invalid Mach-O command size")?; + LoadCommandVariant::Segment32(segment, data.0) + } + macho::LC_SYMTAB => LoadCommandVariant::Symtab(self.data()?), + macho::LC_THREAD | macho::LC_UNIXTHREAD => { + let mut data = self.data; + let thread = data.read().read_error("Invalid Mach-O command size")?; + LoadCommandVariant::Thread(thread, data.0) + } + macho::LC_DYSYMTAB => LoadCommandVariant::Dysymtab(self.data()?), + macho::LC_LOAD_DYLIB + | macho::LC_LOAD_WEAK_DYLIB + | macho::LC_REEXPORT_DYLIB + | macho::LC_LAZY_LOAD_DYLIB + | macho::LC_LOAD_UPWARD_DYLIB => LoadCommandVariant::Dylib(self.data()?), + macho::LC_ID_DYLIB => LoadCommandVariant::IdDylib(self.data()?), + macho::LC_LOAD_DYLINKER => LoadCommandVariant::LoadDylinker(self.data()?), + macho::LC_ID_DYLINKER => LoadCommandVariant::IdDylinker(self.data()?), + macho::LC_PREBOUND_DYLIB => LoadCommandVariant::PreboundDylib(self.data()?), + macho::LC_ROUTINES => LoadCommandVariant::Routines32(self.data()?), + macho::LC_SUB_FRAMEWORK => LoadCommandVariant::SubFramework(self.data()?), + macho::LC_SUB_UMBRELLA => LoadCommandVariant::SubUmbrella(self.data()?), + macho::LC_SUB_CLIENT => LoadCommandVariant::SubClient(self.data()?), + macho::LC_SUB_LIBRARY => LoadCommandVariant::SubLibrary(self.data()?), + macho::LC_TWOLEVEL_HINTS => LoadCommandVariant::TwolevelHints(self.data()?), + macho::LC_PREBIND_CKSUM => LoadCommandVariant::PrebindCksum(self.data()?), + macho::LC_SEGMENT_64 => { + let mut data = self.data; + let segment = data.read().read_error("Invalid Mach-O command size")?; + LoadCommandVariant::Segment64(segment, data.0) + } + macho::LC_ROUTINES_64 => LoadCommandVariant::Routines64(self.data()?), + macho::LC_UUID => LoadCommandVariant::Uuid(self.data()?), + macho::LC_RPATH => LoadCommandVariant::Rpath(self.data()?), + macho::LC_CODE_SIGNATURE + | macho::LC_SEGMENT_SPLIT_INFO + | macho::LC_FUNCTION_STARTS + | macho::LC_DATA_IN_CODE + | macho::LC_DYLIB_CODE_SIGN_DRS + | macho::LC_LINKER_OPTIMIZATION_HINT + | macho::LC_DYLD_EXPORTS_TRIE + | macho::LC_DYLD_CHAINED_FIXUPS => LoadCommandVariant::LinkeditData(self.data()?), + macho::LC_ENCRYPTION_INFO => LoadCommandVariant::EncryptionInfo32(self.data()?), + macho::LC_DYLD_INFO | macho::LC_DYLD_INFO_ONLY => { + LoadCommandVariant::DyldInfo(self.data()?) + } + macho::LC_VERSION_MIN_MACOSX + | macho::LC_VERSION_MIN_IPHONEOS + | macho::LC_VERSION_MIN_TVOS + | macho::LC_VERSION_MIN_WATCHOS => LoadCommandVariant::VersionMin(self.data()?), + macho::LC_DYLD_ENVIRONMENT => LoadCommandVariant::DyldEnvironment(self.data()?), + macho::LC_MAIN => LoadCommandVariant::EntryPoint(self.data()?), + macho::LC_SOURCE_VERSION => LoadCommandVariant::SourceVersion(self.data()?), + macho::LC_ENCRYPTION_INFO_64 => LoadCommandVariant::EncryptionInfo64(self.data()?), + macho::LC_LINKER_OPTION => LoadCommandVariant::LinkerOption(self.data()?), + macho::LC_NOTE => LoadCommandVariant::Note(self.data()?), + macho::LC_BUILD_VERSION => LoadCommandVariant::BuildVersion(self.data()?), + macho::LC_FILESET_ENTRY => LoadCommandVariant::FilesetEntry(self.data()?), + _ => LoadCommandVariant::Other, + }) + } + + /// Try to parse this command as a `SegmentCommand32`. + /// + /// Returns the segment command and the data containing the sections. + pub fn segment_32(self) -> Result, &'data [u8])>> { + if self.cmd == macho::LC_SEGMENT { + let mut data = self.data; + let segment = data.read().read_error("Invalid Mach-O command size")?; + Ok(Some((segment, data.0))) + } else { + Ok(None) + } + } + + /// Try to parse this command as a `SymtabCommand`. + /// + /// Returns the segment command and the data containing the sections. + pub fn symtab(self) -> Result>> { + if self.cmd == macho::LC_SYMTAB { + Some(self.data()).transpose() + } else { + Ok(None) + } + } + + /// Try to parse this command as a `DysymtabCommand`. + pub fn dysymtab(self) -> Result>> { + if self.cmd == macho::LC_DYSYMTAB { + Some(self.data()).transpose() + } else { + Ok(None) + } + } + + /// Try to parse this command as a `DylibCommand`. + pub fn dylib(self) -> Result>> { + if self.cmd == macho::LC_LOAD_DYLIB + || self.cmd == macho::LC_LOAD_WEAK_DYLIB + || self.cmd == macho::LC_REEXPORT_DYLIB + || self.cmd == macho::LC_LAZY_LOAD_DYLIB + || self.cmd == macho::LC_LOAD_UPWARD_DYLIB + { + Some(self.data()).transpose() + } else { + Ok(None) + } + } + + /// Try to parse this command as a `UuidCommand`. + pub fn uuid(self) -> Result>> { + if self.cmd == macho::LC_UUID { + Some(self.data()).transpose() + } else { + Ok(None) + } + } + + /// Try to parse this command as a `SegmentCommand64`. + pub fn segment_64(self) -> Result, &'data [u8])>> { + if self.cmd == macho::LC_SEGMENT_64 { + let mut data = self.data; + let command = data.read().read_error("Invalid Mach-O command size")?; + Ok(Some((command, data.0))) + } else { + Ok(None) + } + } + + /// Try to parse this command as a `DyldInfoCommand`. + pub fn dyld_info(self) -> Result>> { + if self.cmd == macho::LC_DYLD_INFO || self.cmd == macho::LC_DYLD_INFO_ONLY { + Some(self.data()).transpose() + } else { + Ok(None) + } + } + + /// Try to parse this command as an `EntryPointCommand`. + pub fn entry_point(self) -> Result>> { + if self.cmd == macho::LC_MAIN { + Some(self.data()).transpose() + } else { + Ok(None) + } + } +} + +/// A `LoadCommand` that has been interpreted according to its `cmd` field. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub enum LoadCommandVariant<'data, E: Endian> { + /// `LC_SEGMENT` + Segment32(&'data macho::SegmentCommand32, &'data [u8]), + /// `LC_SYMTAB` + Symtab(&'data macho::SymtabCommand), + // obsolete: `LC_SYMSEG` + //Symseg(&'data macho::SymsegCommand), + /// `LC_THREAD` or `LC_UNIXTHREAD` + Thread(&'data macho::ThreadCommand, &'data [u8]), + // obsolete: `LC_IDFVMLIB` or `LC_LOADFVMLIB` + //Fvmlib(&'data macho::FvmlibCommand), + // obsolete: `LC_IDENT` + //Ident(&'data macho::IdentCommand), + // internal: `LC_FVMFILE` + //Fvmfile(&'data macho::FvmfileCommand), + // internal: `LC_PREPAGE` + /// `LC_DYSYMTAB` + Dysymtab(&'data macho::DysymtabCommand), + /// `LC_LOAD_DYLIB`, `LC_LOAD_WEAK_DYLIB`, `LC_REEXPORT_DYLIB`, + /// `LC_LAZY_LOAD_DYLIB`, or `LC_LOAD_UPWARD_DYLIB` + Dylib(&'data macho::DylibCommand), + /// `LC_ID_DYLIB` + IdDylib(&'data macho::DylibCommand), + /// `LC_LOAD_DYLINKER` + LoadDylinker(&'data macho::DylinkerCommand), + /// `LC_ID_DYLINKER` + IdDylinker(&'data macho::DylinkerCommand), + /// `LC_PREBOUND_DYLIB` + PreboundDylib(&'data macho::PreboundDylibCommand), + /// `LC_ROUTINES` + Routines32(&'data macho::RoutinesCommand32), + /// `LC_SUB_FRAMEWORK` + SubFramework(&'data macho::SubFrameworkCommand), + /// `LC_SUB_UMBRELLA` + SubUmbrella(&'data macho::SubUmbrellaCommand), + /// `LC_SUB_CLIENT` + SubClient(&'data macho::SubClientCommand), + /// `LC_SUB_LIBRARY` + SubLibrary(&'data macho::SubLibraryCommand), + /// `LC_TWOLEVEL_HINTS` + TwolevelHints(&'data macho::TwolevelHintsCommand), + /// `LC_PREBIND_CKSUM` + PrebindCksum(&'data macho::PrebindCksumCommand), + /// `LC_SEGMENT_64` + Segment64(&'data macho::SegmentCommand64, &'data [u8]), + /// `LC_ROUTINES_64` + Routines64(&'data macho::RoutinesCommand64), + /// `LC_UUID` + Uuid(&'data macho::UuidCommand), + /// `LC_RPATH` + Rpath(&'data macho::RpathCommand), + /// `LC_CODE_SIGNATURE`, `LC_SEGMENT_SPLIT_INFO`, `LC_FUNCTION_STARTS`, + /// `LC_DATA_IN_CODE`, `LC_DYLIB_CODE_SIGN_DRS`, `LC_LINKER_OPTIMIZATION_HINT`, + /// `LC_DYLD_EXPORTS_TRIE`, or `LC_DYLD_CHAINED_FIXUPS`. + LinkeditData(&'data macho::LinkeditDataCommand), + /// `LC_ENCRYPTION_INFO` + EncryptionInfo32(&'data macho::EncryptionInfoCommand32), + /// `LC_DYLD_INFO` or `LC_DYLD_INFO_ONLY` + DyldInfo(&'data macho::DyldInfoCommand), + /// `LC_VERSION_MIN_MACOSX`, `LC_VERSION_MIN_IPHONEOS`, `LC_VERSION_MIN_WATCHOS`, + /// or `LC_VERSION_MIN_TVOS` + VersionMin(&'data macho::VersionMinCommand), + /// `LC_DYLD_ENVIRONMENT` + DyldEnvironment(&'data macho::DylinkerCommand), + /// `LC_MAIN` + EntryPoint(&'data macho::EntryPointCommand), + /// `LC_SOURCE_VERSION` + SourceVersion(&'data macho::SourceVersionCommand), + /// `LC_ENCRYPTION_INFO_64` + EncryptionInfo64(&'data macho::EncryptionInfoCommand64), + /// `LC_LINKER_OPTION` + LinkerOption(&'data macho::LinkerOptionCommand), + /// `LC_NOTE` + Note(&'data macho::NoteCommand), + /// `LC_BUILD_VERSION` + BuildVersion(&'data macho::BuildVersionCommand), + /// `LC_FILESET_ENTRY` + FilesetEntry(&'data macho::FilesetEntryCommand), + /// An unrecognized or obsolete load command. + Other, +} + +impl macho::SymtabCommand { + /// Return the symbol table that this command references. + pub fn symbols<'data, Mach: MachHeader, R: ReadRef<'data>>( + &self, + endian: E, + data: R, + ) -> Result> { + let symbols = data + .read_slice_at( + self.symoff.get(endian).into(), + self.nsyms.get(endian) as usize, + ) + .read_error("Invalid Mach-O symbol table offset or size")?; + let str_start: u64 = self.stroff.get(endian).into(); + let str_end = str_start + .checked_add(self.strsize.get(endian).into()) + .read_error("Invalid Mach-O string table length")?; + let strings = StringTable::new(data, str_start, str_end); + Ok(SymbolTable::new(symbols, strings)) + } +} diff --git a/crux-mir/lib/object/src/read/macho/mod.rs b/crux-mir/lib/object/src/read/macho/mod.rs new file mode 100644 index 000000000..f07ed581b --- /dev/null +++ b/crux-mir/lib/object/src/read/macho/mod.rs @@ -0,0 +1,30 @@ +//! Support for reading Mach-O files. +//! +//! Defines traits to abstract over the difference between 32-bit and 64-bit +//! Mach-O files, and implements read functionality in terms of these traits. +//! +//! Also provides `MachOFile` and related types which implement the `Object` trait. + +mod dyld_cache; +pub use dyld_cache::*; + +mod fat; +pub use fat::*; + +mod file; +pub use file::*; + +mod load_command; +pub use load_command::*; + +mod segment; +pub use segment::*; + +mod section; +pub use section::*; + +mod symbol; +pub use symbol::*; + +mod relocation; +pub use relocation::*; diff --git a/crux-mir/lib/object/src/read/macho/relocation.rs b/crux-mir/lib/object/src/read/macho/relocation.rs new file mode 100644 index 000000000..5dd7df896 --- /dev/null +++ b/crux-mir/lib/object/src/read/macho/relocation.rs @@ -0,0 +1,126 @@ +use core::{fmt, slice}; + +use crate::endian::Endianness; +use crate::macho; +use crate::read::{ + ReadRef, Relocation, RelocationEncoding, RelocationKind, RelocationTarget, SectionIndex, + SymbolIndex, +}; + +use super::{MachHeader, MachOFile}; + +/// An iterator over the relocations in a `MachOSection32`. +pub type MachORelocationIterator32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachORelocationIterator<'data, 'file, macho::MachHeader32, R>; +/// An iterator over the relocations in a `MachOSection64`. +pub type MachORelocationIterator64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachORelocationIterator<'data, 'file, macho::MachHeader64, R>; + +/// An iterator over the relocations in a `MachOSection`. +pub struct MachORelocationIterator<'data, 'file, Mach, R = &'data [u8]> +where + 'data: 'file, + Mach: MachHeader, + R: ReadRef<'data>, +{ + pub(super) file: &'file MachOFile<'data, Mach, R>, + pub(super) relocations: slice::Iter<'data, macho::Relocation>, +} + +impl<'data, 'file, Mach, R> Iterator for MachORelocationIterator<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + type Item = (u64, Relocation); + + fn next(&mut self) -> Option { + loop { + let reloc = self.relocations.next()?; + let endian = self.file.endian; + let cputype = self.file.header.cputype(endian); + if reloc.r_scattered(endian, cputype) { + // FIXME: handle scattered relocations + // We need to add `RelocationTarget::Address` for this. + continue; + } + let reloc = reloc.info(self.file.endian); + let mut encoding = RelocationEncoding::Generic; + let kind = match cputype { + macho::CPU_TYPE_ARM => match (reloc.r_type, reloc.r_pcrel) { + (macho::ARM_RELOC_VANILLA, false) => RelocationKind::Absolute, + _ => RelocationKind::MachO { + value: reloc.r_type, + relative: reloc.r_pcrel, + }, + }, + macho::CPU_TYPE_ARM64 => match (reloc.r_type, reloc.r_pcrel) { + (macho::ARM64_RELOC_UNSIGNED, false) => RelocationKind::Absolute, + _ => RelocationKind::MachO { + value: reloc.r_type, + relative: reloc.r_pcrel, + }, + }, + macho::CPU_TYPE_X86 => match (reloc.r_type, reloc.r_pcrel) { + (macho::GENERIC_RELOC_VANILLA, false) => RelocationKind::Absolute, + _ => RelocationKind::MachO { + value: reloc.r_type, + relative: reloc.r_pcrel, + }, + }, + macho::CPU_TYPE_X86_64 => match (reloc.r_type, reloc.r_pcrel) { + (macho::X86_64_RELOC_UNSIGNED, false) => RelocationKind::Absolute, + (macho::X86_64_RELOC_SIGNED, true) => { + encoding = RelocationEncoding::X86RipRelative; + RelocationKind::Relative + } + (macho::X86_64_RELOC_BRANCH, true) => { + encoding = RelocationEncoding::X86Branch; + RelocationKind::Relative + } + (macho::X86_64_RELOC_GOT, true) => RelocationKind::GotRelative, + (macho::X86_64_RELOC_GOT_LOAD, true) => { + encoding = RelocationEncoding::X86RipRelativeMovq; + RelocationKind::GotRelative + } + _ => RelocationKind::MachO { + value: reloc.r_type, + relative: reloc.r_pcrel, + }, + }, + _ => RelocationKind::MachO { + value: reloc.r_type, + relative: reloc.r_pcrel, + }, + }; + let size = 8 << reloc.r_length; + let target = if reloc.r_extern { + RelocationTarget::Symbol(SymbolIndex(reloc.r_symbolnum as usize)) + } else { + RelocationTarget::Section(SectionIndex(reloc.r_symbolnum as usize)) + }; + let addend = if reloc.r_pcrel { -4 } else { 0 }; + return Some(( + reloc.r_address as u64, + Relocation { + kind, + encoding, + size, + target, + addend, + implicit_addend: true, + }, + )); + } + } +} + +impl<'data, 'file, Mach, R> fmt::Debug for MachORelocationIterator<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("MachORelocationIterator").finish() + } +} diff --git a/crux-mir/lib/object/src/read/macho/section.rs b/crux-mir/lib/object/src/read/macho/section.rs new file mode 100644 index 000000000..9e71aa8fd --- /dev/null +++ b/crux-mir/lib/object/src/read/macho/section.rs @@ -0,0 +1,384 @@ +use core::fmt::Debug; +use core::{fmt, result, slice, str}; + +use crate::endian::{self, Endianness}; +use crate::macho; +use crate::pod::Pod; +use crate::read::{ + self, CompressedData, CompressedFileRange, ObjectSection, ReadError, ReadRef, Result, + SectionFlags, SectionIndex, SectionKind, +}; + +use super::{MachHeader, MachOFile, MachORelocationIterator}; + +/// An iterator over the sections of a `MachOFile32`. +pub type MachOSectionIterator32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachOSectionIterator<'data, 'file, macho::MachHeader32, R>; +/// An iterator over the sections of a `MachOFile64`. +pub type MachOSectionIterator64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachOSectionIterator<'data, 'file, macho::MachHeader64, R>; + +/// An iterator over the sections of a `MachOFile`. +pub struct MachOSectionIterator<'data, 'file, Mach, R = &'data [u8]> +where + 'data: 'file, + Mach: MachHeader, + R: ReadRef<'data>, +{ + pub(super) file: &'file MachOFile<'data, Mach, R>, + pub(super) iter: slice::Iter<'file, MachOSectionInternal<'data, Mach>>, +} + +impl<'data, 'file, Mach, R> fmt::Debug for MachOSectionIterator<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // It's painful to do much better than this + f.debug_struct("MachOSectionIterator").finish() + } +} + +impl<'data, 'file, Mach, R> Iterator for MachOSectionIterator<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + type Item = MachOSection<'data, 'file, Mach, R>; + + fn next(&mut self) -> Option { + self.iter.next().map(|&internal| MachOSection { + file: self.file, + internal, + }) + } +} + +/// A section of a `MachOFile32`. +pub type MachOSection32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachOSection<'data, 'file, macho::MachHeader32, R>; +/// A section of a `MachOFile64`. +pub type MachOSection64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachOSection<'data, 'file, macho::MachHeader64, R>; + +/// A section of a `MachOFile`. +#[derive(Debug)] +pub struct MachOSection<'data, 'file, Mach, R = &'data [u8]> +where + 'data: 'file, + Mach: MachHeader, + R: ReadRef<'data>, +{ + pub(super) file: &'file MachOFile<'data, Mach, R>, + pub(super) internal: MachOSectionInternal<'data, Mach>, +} + +impl<'data, 'file, Mach, R> MachOSection<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + fn bytes(&self) -> Result<&'data [u8]> { + let segment_index = self.internal.segment_index; + let segment = self.file.segment_internal(segment_index)?; + self.internal + .section + .data(self.file.endian, segment.data) + .read_error("Invalid Mach-O section size or offset") + } +} + +impl<'data, 'file, Mach, R> read::private::Sealed for MachOSection<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ +} + +impl<'data, 'file, Mach, R> ObjectSection<'data> for MachOSection<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + type RelocationIterator = MachORelocationIterator<'data, 'file, Mach, R>; + + #[inline] + fn index(&self) -> SectionIndex { + self.internal.index + } + + #[inline] + fn address(&self) -> u64 { + self.internal.section.addr(self.file.endian).into() + } + + #[inline] + fn size(&self) -> u64 { + self.internal.section.size(self.file.endian).into() + } + + #[inline] + fn align(&self) -> u64 { + 1 << self.internal.section.align(self.file.endian) + } + + #[inline] + fn file_range(&self) -> Option<(u64, u64)> { + self.internal.section.file_range(self.file.endian) + } + + #[inline] + fn data(&self) -> Result<&'data [u8]> { + self.bytes() + } + + fn data_range(&self, address: u64, size: u64) -> Result> { + Ok(read::util::data_range( + self.bytes()?, + self.address(), + address, + size, + )) + } + + #[inline] + fn compressed_file_range(&self) -> Result { + Ok(CompressedFileRange::none(self.file_range())) + } + + #[inline] + fn compressed_data(&self) -> Result> { + self.data().map(CompressedData::none) + } + + #[inline] + fn name_bytes(&self) -> Result<&[u8]> { + Ok(self.internal.section.name()) + } + + #[inline] + fn name(&self) -> Result<&str> { + str::from_utf8(self.internal.section.name()) + .ok() + .read_error("Non UTF-8 Mach-O section name") + } + + #[inline] + fn segment_name_bytes(&self) -> Result> { + Ok(Some(self.internal.section.segment_name())) + } + + #[inline] + fn segment_name(&self) -> Result> { + Ok(Some( + str::from_utf8(self.internal.section.segment_name()) + .ok() + .read_error("Non UTF-8 Mach-O segment name")?, + )) + } + + fn kind(&self) -> SectionKind { + self.internal.kind + } + + fn relocations(&self) -> MachORelocationIterator<'data, 'file, Mach, R> { + MachORelocationIterator { + file: self.file, + relocations: self + .internal + .section + .relocations(self.file.endian, self.file.data) + .unwrap_or(&[]) + .iter(), + } + } + + fn flags(&self) -> SectionFlags { + SectionFlags::MachO { + flags: self.internal.section.flags(self.file.endian), + } + } +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct MachOSectionInternal<'data, Mach: MachHeader> { + pub index: SectionIndex, + pub segment_index: usize, + pub kind: SectionKind, + pub section: &'data Mach::Section, +} + +impl<'data, Mach: MachHeader> MachOSectionInternal<'data, Mach> { + pub(super) fn parse( + index: SectionIndex, + segment_index: usize, + section: &'data Mach::Section, + ) -> Self { + // TODO: we don't validate flags, should we? + let kind = match (section.segment_name(), section.name()) { + (b"__TEXT", b"__text") => SectionKind::Text, + (b"__TEXT", b"__const") => SectionKind::ReadOnlyData, + (b"__TEXT", b"__cstring") => SectionKind::ReadOnlyString, + (b"__TEXT", b"__literal4") => SectionKind::ReadOnlyData, + (b"__TEXT", b"__literal8") => SectionKind::ReadOnlyData, + (b"__TEXT", b"__literal16") => SectionKind::ReadOnlyData, + (b"__TEXT", b"__eh_frame") => SectionKind::ReadOnlyData, + (b"__TEXT", b"__gcc_except_tab") => SectionKind::ReadOnlyData, + (b"__DATA", b"__data") => SectionKind::Data, + (b"__DATA", b"__const") => SectionKind::ReadOnlyData, + (b"__DATA", b"__bss") => SectionKind::UninitializedData, + (b"__DATA", b"__common") => SectionKind::Common, + (b"__DATA", b"__thread_data") => SectionKind::Tls, + (b"__DATA", b"__thread_bss") => SectionKind::UninitializedTls, + (b"__DATA", b"__thread_vars") => SectionKind::TlsVariables, + (b"__DWARF", _) => SectionKind::Debug, + _ => SectionKind::Unknown, + }; + MachOSectionInternal { + index, + segment_index, + kind, + section, + } + } +} + +/// A trait for generic access to `Section32` and `Section64`. +#[allow(missing_docs)] +pub trait Section: Debug + Pod { + type Word: Into; + type Endian: endian::Endian; + + fn sectname(&self) -> &[u8; 16]; + fn segname(&self) -> &[u8; 16]; + fn addr(&self, endian: Self::Endian) -> Self::Word; + fn size(&self, endian: Self::Endian) -> Self::Word; + fn offset(&self, endian: Self::Endian) -> u32; + fn align(&self, endian: Self::Endian) -> u32; + fn reloff(&self, endian: Self::Endian) -> u32; + fn nreloc(&self, endian: Self::Endian) -> u32; + fn flags(&self, endian: Self::Endian) -> u32; + + /// Return the `sectname` bytes up until the null terminator. + fn name(&self) -> &[u8] { + let sectname = &self.sectname()[..]; + match memchr::memchr(b'\0', sectname) { + Some(end) => §name[..end], + None => sectname, + } + } + + /// Return the `segname` bytes up until the null terminator. + fn segment_name(&self) -> &[u8] { + let segname = &self.segname()[..]; + match memchr::memchr(b'\0', segname) { + Some(end) => &segname[..end], + None => segname, + } + } + + /// Return the offset and size of the section in the file. + /// + /// Returns `None` for sections that have no data in the file. + fn file_range(&self, endian: Self::Endian) -> Option<(u64, u64)> { + match self.flags(endian) & macho::SECTION_TYPE { + macho::S_ZEROFILL | macho::S_GB_ZEROFILL | macho::S_THREAD_LOCAL_ZEROFILL => None, + _ => Some((self.offset(endian).into(), self.size(endian).into())), + } + } + + /// Return the section data. + /// + /// Returns `Ok(&[])` if the section has no data. + /// Returns `Err` for invalid values. + fn data<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> result::Result<&'data [u8], ()> { + if let Some((offset, size)) = self.file_range(endian) { + data.read_bytes_at(offset, size) + } else { + Ok(&[]) + } + } + + /// Return the relocation array. + /// + /// Returns `Err` for invalid values. + fn relocations<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> Result<&'data [macho::Relocation]> { + data.read_slice_at(self.reloff(endian).into(), self.nreloc(endian) as usize) + .read_error("Invalid Mach-O relocations offset or number") + } +} + +impl Section for macho::Section32 { + type Word = u32; + type Endian = Endian; + + fn sectname(&self) -> &[u8; 16] { + &self.sectname + } + fn segname(&self) -> &[u8; 16] { + &self.segname + } + fn addr(&self, endian: Self::Endian) -> Self::Word { + self.addr.get(endian) + } + fn size(&self, endian: Self::Endian) -> Self::Word { + self.size.get(endian) + } + fn offset(&self, endian: Self::Endian) -> u32 { + self.offset.get(endian) + } + fn align(&self, endian: Self::Endian) -> u32 { + self.align.get(endian) + } + fn reloff(&self, endian: Self::Endian) -> u32 { + self.reloff.get(endian) + } + fn nreloc(&self, endian: Self::Endian) -> u32 { + self.nreloc.get(endian) + } + fn flags(&self, endian: Self::Endian) -> u32 { + self.flags.get(endian) + } +} + +impl Section for macho::Section64 { + type Word = u64; + type Endian = Endian; + + fn sectname(&self) -> &[u8; 16] { + &self.sectname + } + fn segname(&self) -> &[u8; 16] { + &self.segname + } + fn addr(&self, endian: Self::Endian) -> Self::Word { + self.addr.get(endian) + } + fn size(&self, endian: Self::Endian) -> Self::Word { + self.size.get(endian) + } + fn offset(&self, endian: Self::Endian) -> u32 { + self.offset.get(endian) + } + fn align(&self, endian: Self::Endian) -> u32 { + self.align.get(endian) + } + fn reloff(&self, endian: Self::Endian) -> u32 { + self.reloff.get(endian) + } + fn nreloc(&self, endian: Self::Endian) -> u32 { + self.nreloc.get(endian) + } + fn flags(&self, endian: Self::Endian) -> u32 { + self.flags.get(endian) + } +} diff --git a/crux-mir/lib/object/src/read/macho/segment.rs b/crux-mir/lib/object/src/read/macho/segment.rs new file mode 100644 index 000000000..c7eaa6fff --- /dev/null +++ b/crux-mir/lib/object/src/read/macho/segment.rs @@ -0,0 +1,303 @@ +use core::fmt::Debug; +use core::{result, slice, str}; + +use crate::endian::{self, Endianness}; +use crate::macho; +use crate::pod::Pod; +use crate::read::{self, ObjectSegment, ReadError, ReadRef, Result, SegmentFlags}; + +use super::{LoadCommandData, MachHeader, MachOFile, Section}; + +/// An iterator over the segments of a `MachOFile32`. +pub type MachOSegmentIterator32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachOSegmentIterator<'data, 'file, macho::MachHeader32, R>; +/// An iterator over the segments of a `MachOFile64`. +pub type MachOSegmentIterator64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachOSegmentIterator<'data, 'file, macho::MachHeader64, R>; + +/// An iterator over the segments of a `MachOFile`. +#[derive(Debug)] +pub struct MachOSegmentIterator<'data, 'file, Mach, R = &'data [u8]> +where + 'data: 'file, + Mach: MachHeader, + R: ReadRef<'data>, +{ + pub(super) file: &'file MachOFile<'data, Mach, R>, + pub(super) iter: slice::Iter<'file, MachOSegmentInternal<'data, Mach, R>>, +} + +impl<'data, 'file, Mach, R> Iterator for MachOSegmentIterator<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + type Item = MachOSegment<'data, 'file, Mach, R>; + + fn next(&mut self) -> Option { + self.iter.next().map(|internal| MachOSegment { + file: self.file, + internal, + }) + } +} + +/// A segment of a `MachOFile32`. +pub type MachOSegment32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachOSegment<'data, 'file, macho::MachHeader32, R>; +/// A segment of a `MachOFile64`. +pub type MachOSegment64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachOSegment<'data, 'file, macho::MachHeader64, R>; + +/// A segment of a `MachOFile`. +#[derive(Debug)] +pub struct MachOSegment<'data, 'file, Mach, R = &'data [u8]> +where + 'data: 'file, + Mach: MachHeader, + R: ReadRef<'data>, +{ + file: &'file MachOFile<'data, Mach, R>, + internal: &'file MachOSegmentInternal<'data, Mach, R>, +} + +impl<'data, 'file, Mach, R> MachOSegment<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + fn bytes(&self) -> Result<&'data [u8]> { + self.internal + .segment + .data(self.file.endian, self.file.data) + .read_error("Invalid Mach-O segment size or offset") + } +} + +impl<'data, 'file, Mach, R> read::private::Sealed for MachOSegment<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ +} + +impl<'data, 'file, Mach, R> ObjectSegment<'data> for MachOSegment<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + #[inline] + fn address(&self) -> u64 { + self.internal.segment.vmaddr(self.file.endian).into() + } + + #[inline] + fn size(&self) -> u64 { + self.internal.segment.vmsize(self.file.endian).into() + } + + #[inline] + fn align(&self) -> u64 { + // Page size. + 0x1000 + } + + #[inline] + fn file_range(&self) -> (u64, u64) { + self.internal.segment.file_range(self.file.endian) + } + + fn data(&self) -> Result<&'data [u8]> { + self.bytes() + } + + fn data_range(&self, address: u64, size: u64) -> Result> { + Ok(read::util::data_range( + self.bytes()?, + self.address(), + address, + size, + )) + } + + #[inline] + fn name_bytes(&self) -> Result> { + Ok(Some(self.internal.segment.name())) + } + + #[inline] + fn name(&self) -> Result> { + Ok(Some( + str::from_utf8(self.internal.segment.name()) + .ok() + .read_error("Non UTF-8 Mach-O segment name")?, + )) + } + + #[inline] + fn flags(&self) -> SegmentFlags { + let flags = self.internal.segment.flags(self.file.endian); + let maxprot = self.internal.segment.maxprot(self.file.endian); + let initprot = self.internal.segment.initprot(self.file.endian); + SegmentFlags::MachO { + flags, + maxprot, + initprot, + } + } +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct MachOSegmentInternal<'data, Mach: MachHeader, R: ReadRef<'data>> { + pub data: R, + pub segment: &'data Mach::Segment, +} + +/// A trait for generic access to `SegmentCommand32` and `SegmentCommand64`. +#[allow(missing_docs)] +pub trait Segment: Debug + Pod { + type Word: Into; + type Endian: endian::Endian; + type Section: Section; + + fn from_command(command: LoadCommandData) -> Result>; + + fn cmd(&self, endian: Self::Endian) -> u32; + fn cmdsize(&self, endian: Self::Endian) -> u32; + fn segname(&self) -> &[u8; 16]; + fn vmaddr(&self, endian: Self::Endian) -> Self::Word; + fn vmsize(&self, endian: Self::Endian) -> Self::Word; + fn fileoff(&self, endian: Self::Endian) -> Self::Word; + fn filesize(&self, endian: Self::Endian) -> Self::Word; + fn maxprot(&self, endian: Self::Endian) -> u32; + fn initprot(&self, endian: Self::Endian) -> u32; + fn nsects(&self, endian: Self::Endian) -> u32; + fn flags(&self, endian: Self::Endian) -> u32; + + /// Return the `segname` bytes up until the null terminator. + fn name(&self) -> &[u8] { + let segname = &self.segname()[..]; + match memchr::memchr(b'\0', segname) { + Some(end) => &segname[..end], + None => segname, + } + } + + /// Return the offset and size of the segment in the file. + fn file_range(&self, endian: Self::Endian) -> (u64, u64) { + (self.fileoff(endian).into(), self.filesize(endian).into()) + } + + /// Get the segment data from the file data. + /// + /// Returns `Err` for invalid values. + fn data<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + data: R, + ) -> result::Result<&'data [u8], ()> { + let (offset, size) = self.file_range(endian); + data.read_bytes_at(offset, size) + } + + /// Get the array of sections from the data following the segment command. + /// + /// Returns `Err` for invalid values. + fn sections<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + section_data: R, + ) -> Result<&'data [Self::Section]> { + section_data + .read_slice_at(0, self.nsects(endian) as usize) + .read_error("Invalid Mach-O number of sections") + } +} + +impl Segment for macho::SegmentCommand32 { + type Word = u32; + type Endian = Endian; + type Section = macho::Section32; + + fn from_command(command: LoadCommandData) -> Result> { + command.segment_32() + } + + fn cmd(&self, endian: Self::Endian) -> u32 { + self.cmd.get(endian) + } + fn cmdsize(&self, endian: Self::Endian) -> u32 { + self.cmdsize.get(endian) + } + fn segname(&self) -> &[u8; 16] { + &self.segname + } + fn vmaddr(&self, endian: Self::Endian) -> Self::Word { + self.vmaddr.get(endian) + } + fn vmsize(&self, endian: Self::Endian) -> Self::Word { + self.vmsize.get(endian) + } + fn fileoff(&self, endian: Self::Endian) -> Self::Word { + self.fileoff.get(endian) + } + fn filesize(&self, endian: Self::Endian) -> Self::Word { + self.filesize.get(endian) + } + fn maxprot(&self, endian: Self::Endian) -> u32 { + self.maxprot.get(endian) + } + fn initprot(&self, endian: Self::Endian) -> u32 { + self.initprot.get(endian) + } + fn nsects(&self, endian: Self::Endian) -> u32 { + self.nsects.get(endian) + } + fn flags(&self, endian: Self::Endian) -> u32 { + self.flags.get(endian) + } +} + +impl Segment for macho::SegmentCommand64 { + type Word = u64; + type Endian = Endian; + type Section = macho::Section64; + + fn from_command(command: LoadCommandData) -> Result> { + command.segment_64() + } + + fn cmd(&self, endian: Self::Endian) -> u32 { + self.cmd.get(endian) + } + fn cmdsize(&self, endian: Self::Endian) -> u32 { + self.cmdsize.get(endian) + } + fn segname(&self) -> &[u8; 16] { + &self.segname + } + fn vmaddr(&self, endian: Self::Endian) -> Self::Word { + self.vmaddr.get(endian) + } + fn vmsize(&self, endian: Self::Endian) -> Self::Word { + self.vmsize.get(endian) + } + fn fileoff(&self, endian: Self::Endian) -> Self::Word { + self.fileoff.get(endian) + } + fn filesize(&self, endian: Self::Endian) -> Self::Word { + self.filesize.get(endian) + } + fn maxprot(&self, endian: Self::Endian) -> u32 { + self.maxprot.get(endian) + } + fn initprot(&self, endian: Self::Endian) -> u32 { + self.initprot.get(endian) + } + fn nsects(&self, endian: Self::Endian) -> u32 { + self.nsects.get(endian) + } + fn flags(&self, endian: Self::Endian) -> u32 { + self.flags.get(endian) + } +} diff --git a/crux-mir/lib/object/src/read/macho/symbol.rs b/crux-mir/lib/object/src/read/macho/symbol.rs new file mode 100644 index 000000000..e102c5d0b --- /dev/null +++ b/crux-mir/lib/object/src/read/macho/symbol.rs @@ -0,0 +1,488 @@ +use alloc::vec::Vec; +use core::fmt::Debug; +use core::{fmt, slice, str}; + +use crate::endian::{self, Endianness}; +use crate::macho; +use crate::pod::Pod; +use crate::read::util::StringTable; +use crate::read::{ + self, ObjectMap, ObjectMapEntry, ObjectSymbol, ObjectSymbolTable, ReadError, ReadRef, Result, + SectionIndex, SectionKind, SymbolFlags, SymbolIndex, SymbolKind, SymbolMap, SymbolMapEntry, + SymbolScope, SymbolSection, +}; + +use super::{MachHeader, MachOFile}; + +/// A table of symbol entries in a Mach-O file. +/// +/// Also includes the string table used for the symbol names. +#[derive(Debug, Clone, Copy)] +pub struct SymbolTable<'data, Mach: MachHeader, R = &'data [u8]> +where + R: ReadRef<'data>, +{ + symbols: &'data [Mach::Nlist], + strings: StringTable<'data, R>, +} + +impl<'data, Mach: MachHeader, R: ReadRef<'data>> Default for SymbolTable<'data, Mach, R> { + fn default() -> Self { + SymbolTable { + symbols: &[], + strings: Default::default(), + } + } +} + +impl<'data, Mach: MachHeader, R: ReadRef<'data>> SymbolTable<'data, Mach, R> { + #[inline] + pub(super) fn new(symbols: &'data [Mach::Nlist], strings: StringTable<'data, R>) -> Self { + SymbolTable { symbols, strings } + } + + /// Return the string table used for the symbol names. + #[inline] + pub fn strings(&self) -> StringTable<'data, R> { + self.strings + } + + /// Iterate over the symbols. + #[inline] + pub fn iter(&self) -> slice::Iter<'data, Mach::Nlist> { + self.symbols.iter() + } + + /// Return true if the symbol table is empty. + #[inline] + pub fn is_empty(&self) -> bool { + self.symbols.is_empty() + } + + /// The number of symbols. + #[inline] + pub fn len(&self) -> usize { + self.symbols.len() + } + + /// Return the symbol at the given index. + pub fn symbol(&self, index: usize) -> Result<&'data Mach::Nlist> { + self.symbols + .get(index) + .read_error("Invalid Mach-O symbol index") + } + + /// Construct a map from addresses to a user-defined map entry. + pub fn map Option>( + &self, + f: F, + ) -> SymbolMap { + let mut symbols = Vec::new(); + for nlist in self.symbols { + if !nlist.is_definition() { + continue; + } + if let Some(entry) = f(nlist) { + symbols.push(entry); + } + } + SymbolMap::new(symbols) + } + + /// Construct a map from addresses to symbol names and object file names. + pub fn object_map(&self, endian: Mach::Endian) -> ObjectMap<'data> { + let mut symbols = Vec::new(); + let mut objects = Vec::new(); + let mut object = None; + let mut current_function = None; + // Each module starts with one or two N_SO symbols (path, or directory + filename) + // and one N_OSO symbol. The module is terminated by an empty N_SO symbol. + for nlist in self.symbols { + let n_type = nlist.n_type(); + if n_type & macho::N_STAB == 0 { + continue; + } + // TODO: includes variables too (N_GSYM, N_STSYM). These may need to get their + // address from regular symbols though. + match n_type { + macho::N_SO => { + object = None; + } + macho::N_OSO => { + object = None; + if let Ok(name) = nlist.name(endian, self.strings) { + if !name.is_empty() { + object = Some(objects.len()); + objects.push(name); + } + } + } + macho::N_FUN => { + if let Ok(name) = nlist.name(endian, self.strings) { + if !name.is_empty() { + current_function = Some((name, nlist.n_value(endian).into())) + } else if let Some((name, address)) = current_function.take() { + if let Some(object) = object { + symbols.push(ObjectMapEntry { + address, + size: nlist.n_value(endian).into(), + name, + object, + }); + } + } + } + } + _ => {} + } + } + ObjectMap { + symbols: SymbolMap::new(symbols), + objects, + } + } +} + +/// An iterator over the symbols of a `MachOFile32`. +pub type MachOSymbolTable32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachOSymbolTable<'data, 'file, macho::MachHeader32, R>; +/// An iterator over the symbols of a `MachOFile64`. +pub type MachOSymbolTable64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachOSymbolTable<'data, 'file, macho::MachHeader64, R>; + +/// A symbol table of a `MachOFile`. +#[derive(Debug, Clone, Copy)] +pub struct MachOSymbolTable<'data, 'file, Mach, R = &'data [u8]> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + pub(super) file: &'file MachOFile<'data, Mach, R>, +} + +impl<'data, 'file, Mach, R> read::private::Sealed for MachOSymbolTable<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ +} + +impl<'data, 'file, Mach, R> ObjectSymbolTable<'data> for MachOSymbolTable<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + type Symbol = MachOSymbol<'data, 'file, Mach, R>; + type SymbolIterator = MachOSymbolIterator<'data, 'file, Mach, R>; + + fn symbols(&self) -> Self::SymbolIterator { + MachOSymbolIterator { + file: self.file, + index: 0, + } + } + + fn symbol_by_index(&self, index: SymbolIndex) -> Result { + let nlist = self.file.symbols.symbol(index.0)?; + MachOSymbol::new(self.file, index, nlist).read_error("Unsupported Mach-O symbol index") + } +} + +/// An iterator over the symbols of a `MachOFile32`. +pub type MachOSymbolIterator32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachOSymbolIterator<'data, 'file, macho::MachHeader32, R>; +/// An iterator over the symbols of a `MachOFile64`. +pub type MachOSymbolIterator64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachOSymbolIterator<'data, 'file, macho::MachHeader64, R>; + +/// An iterator over the symbols of a `MachOFile`. +pub struct MachOSymbolIterator<'data, 'file, Mach, R = &'data [u8]> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + pub(super) file: &'file MachOFile<'data, Mach, R>, + pub(super) index: usize, +} + +impl<'data, 'file, Mach, R> fmt::Debug for MachOSymbolIterator<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("MachOSymbolIterator").finish() + } +} + +impl<'data, 'file, Mach, R> Iterator for MachOSymbolIterator<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + type Item = MachOSymbol<'data, 'file, Mach, R>; + + fn next(&mut self) -> Option { + loop { + let index = self.index; + let nlist = self.file.symbols.symbols.get(index)?; + self.index += 1; + if let Some(symbol) = MachOSymbol::new(self.file, SymbolIndex(index), nlist) { + return Some(symbol); + } + } + } +} + +/// A symbol of a `MachOFile32`. +pub type MachOSymbol32<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachOSymbol<'data, 'file, macho::MachHeader32, R>; +/// A symbol of a `MachOFile64`. +pub type MachOSymbol64<'data, 'file, Endian = Endianness, R = &'data [u8]> = + MachOSymbol<'data, 'file, macho::MachHeader64, R>; + +/// A symbol of a `MachOFile`. +#[derive(Debug, Clone, Copy)] +pub struct MachOSymbol<'data, 'file, Mach, R = &'data [u8]> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + file: &'file MachOFile<'data, Mach, R>, + index: SymbolIndex, + nlist: &'data Mach::Nlist, +} + +impl<'data, 'file, Mach, R> MachOSymbol<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + pub(super) fn new( + file: &'file MachOFile<'data, Mach, R>, + index: SymbolIndex, + nlist: &'data Mach::Nlist, + ) -> Option { + if nlist.n_type() & macho::N_STAB != 0 { + return None; + } + Some(MachOSymbol { file, index, nlist }) + } +} + +impl<'data, 'file, Mach, R> read::private::Sealed for MachOSymbol<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ +} + +impl<'data, 'file, Mach, R> ObjectSymbol<'data> for MachOSymbol<'data, 'file, Mach, R> +where + Mach: MachHeader, + R: ReadRef<'data>, +{ + #[inline] + fn index(&self) -> SymbolIndex { + self.index + } + + fn name_bytes(&self) -> Result<&'data [u8]> { + self.nlist.name(self.file.endian, self.file.symbols.strings) + } + + fn name(&self) -> Result<&'data str> { + let name = self.name_bytes()?; + str::from_utf8(name) + .ok() + .read_error("Non UTF-8 Mach-O symbol name") + } + + #[inline] + fn address(&self) -> u64 { + self.nlist.n_value(self.file.endian).into() + } + + #[inline] + fn size(&self) -> u64 { + 0 + } + + fn kind(&self) -> SymbolKind { + self.section() + .index() + .and_then(|index| self.file.section_internal(index).ok()) + .map(|section| match section.kind { + SectionKind::Text => SymbolKind::Text, + SectionKind::Data + | SectionKind::ReadOnlyData + | SectionKind::ReadOnlyString + | SectionKind::UninitializedData + | SectionKind::Common => SymbolKind::Data, + SectionKind::Tls | SectionKind::UninitializedTls | SectionKind::TlsVariables => { + SymbolKind::Tls + } + _ => SymbolKind::Unknown, + }) + .unwrap_or(SymbolKind::Unknown) + } + + fn section(&self) -> SymbolSection { + match self.nlist.n_type() & macho::N_TYPE { + macho::N_UNDF => SymbolSection::Undefined, + macho::N_ABS => SymbolSection::Absolute, + macho::N_SECT => { + let n_sect = self.nlist.n_sect(); + if n_sect != 0 { + SymbolSection::Section(SectionIndex(n_sect as usize)) + } else { + SymbolSection::Unknown + } + } + _ => SymbolSection::Unknown, + } + } + + #[inline] + fn is_undefined(&self) -> bool { + self.nlist.n_type() & macho::N_TYPE == macho::N_UNDF + } + + #[inline] + fn is_definition(&self) -> bool { + self.nlist.is_definition() + } + + #[inline] + fn is_common(&self) -> bool { + // Mach-O common symbols are based on section, not symbol + false + } + + #[inline] + fn is_weak(&self) -> bool { + self.nlist.n_desc(self.file.endian) & (macho::N_WEAK_REF | macho::N_WEAK_DEF) != 0 + } + + fn scope(&self) -> SymbolScope { + let n_type = self.nlist.n_type(); + if n_type & macho::N_TYPE == macho::N_UNDF { + SymbolScope::Unknown + } else if n_type & macho::N_EXT == 0 { + SymbolScope::Compilation + } else if n_type & macho::N_PEXT != 0 { + SymbolScope::Linkage + } else { + SymbolScope::Dynamic + } + } + + #[inline] + fn is_global(&self) -> bool { + self.scope() != SymbolScope::Compilation + } + + #[inline] + fn is_local(&self) -> bool { + self.scope() == SymbolScope::Compilation + } + + #[inline] + fn flags(&self) -> SymbolFlags { + let n_desc = self.nlist.n_desc(self.file.endian); + SymbolFlags::MachO { n_desc } + } +} + +/// A trait for generic access to `Nlist32` and `Nlist64`. +#[allow(missing_docs)] +pub trait Nlist: Debug + Pod { + type Word: Into; + type Endian: endian::Endian; + + fn n_strx(&self, endian: Self::Endian) -> u32; + fn n_type(&self) -> u8; + fn n_sect(&self) -> u8; + fn n_desc(&self, endian: Self::Endian) -> u16; + fn n_value(&self, endian: Self::Endian) -> Self::Word; + + fn name<'data, R: ReadRef<'data>>( + &self, + endian: Self::Endian, + strings: StringTable<'data, R>, + ) -> Result<&'data [u8]> { + strings + .get(self.n_strx(endian)) + .read_error("Invalid Mach-O symbol name offset") + } + + /// Return true if this is a STAB symbol. + /// + /// This determines the meaning of the `n_type` field. + fn is_stab(&self) -> bool { + self.n_type() & macho::N_STAB != 0 + } + + /// Return true if this is an undefined symbol. + fn is_undefined(&self) -> bool { + let n_type = self.n_type(); + n_type & macho::N_STAB == 0 && n_type & macho::N_TYPE == macho::N_UNDF + } + + /// Return true if the symbol is a definition of a function or data object. + fn is_definition(&self) -> bool { + let n_type = self.n_type(); + n_type & macho::N_STAB == 0 && n_type & macho::N_TYPE != macho::N_UNDF + } + + /// Return the library ordinal. + /// + /// This is either a 1-based index into the dylib load commands, + /// or a special ordinal. + #[inline] + fn library_ordinal(&self, endian: Self::Endian) -> u8 { + (self.n_desc(endian) >> 8) as u8 + } +} + +impl Nlist for macho::Nlist32 { + type Word = u32; + type Endian = Endian; + + fn n_strx(&self, endian: Self::Endian) -> u32 { + self.n_strx.get(endian) + } + fn n_type(&self) -> u8 { + self.n_type + } + fn n_sect(&self) -> u8 { + self.n_sect + } + fn n_desc(&self, endian: Self::Endian) -> u16 { + self.n_desc.get(endian) + } + fn n_value(&self, endian: Self::Endian) -> Self::Word { + self.n_value.get(endian) + } +} + +impl Nlist for macho::Nlist64 { + type Word = u64; + type Endian = Endian; + + fn n_strx(&self, endian: Self::Endian) -> u32 { + self.n_strx.get(endian) + } + fn n_type(&self) -> u8 { + self.n_type + } + fn n_sect(&self) -> u8 { + self.n_sect + } + fn n_desc(&self, endian: Self::Endian) -> u16 { + self.n_desc.get(endian) + } + fn n_value(&self, endian: Self::Endian) -> Self::Word { + self.n_value.get(endian) + } +} diff --git a/crux-mir/lib/object/src/read/mod.rs b/crux-mir/lib/object/src/read/mod.rs new file mode 100644 index 000000000..41d344111 --- /dev/null +++ b/crux-mir/lib/object/src/read/mod.rs @@ -0,0 +1,710 @@ +//! Interface for reading object files. + +use alloc::borrow::Cow; +use alloc::vec::Vec; +use core::{fmt, result}; + +use crate::common::*; + +mod read_ref; +pub use read_ref::*; + +#[cfg(feature = "std")] +mod read_cache; +#[cfg(feature = "std")] +pub use read_cache::*; + +mod util; +pub use util::*; + +#[cfg(any( + feature = "coff", + feature = "elf", + feature = "macho", + feature = "pe", + feature = "wasm" +))] +mod any; +#[cfg(any( + feature = "coff", + feature = "elf", + feature = "macho", + feature = "pe", + feature = "wasm" +))] +pub use any::*; + +#[cfg(feature = "archive")] +pub mod archive; + +#[cfg(feature = "coff")] +pub mod coff; + +#[cfg(feature = "elf")] +pub mod elf; + +#[cfg(feature = "macho")] +pub mod macho; + +#[cfg(feature = "pe")] +pub mod pe; + +mod traits; +pub use traits::*; + +#[cfg(feature = "wasm")] +pub mod wasm; + +mod private { + pub trait Sealed {} +} + +/// The error type used within the read module. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Error(&'static str); + +impl fmt::Display for Error { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str(self.0) + } +} + +#[cfg(feature = "std")] +impl std::error::Error for Error {} + +/// The result type used within the read module. +pub type Result = result::Result; + +trait ReadError { + fn read_error(self, error: &'static str) -> Result; +} + +impl ReadError for result::Result { + fn read_error(self, error: &'static str) -> Result { + self.map_err(|()| Error(error)) + } +} + +impl ReadError for result::Result { + fn read_error(self, error: &'static str) -> Result { + self.map_err(|_| Error(error)) + } +} + +impl ReadError for Option { + fn read_error(self, error: &'static str) -> Result { + self.ok_or(Error(error)) + } +} + +/// The native executable file for the target platform. +#[cfg(all( + unix, + not(target_os = "macos"), + target_pointer_width = "32", + feature = "elf" +))] +pub type NativeFile<'data, R = &'data [u8]> = elf::ElfFile32<'data, crate::Endianness, R>; + +/// The native executable file for the target platform. +#[cfg(all( + unix, + not(target_os = "macos"), + target_pointer_width = "64", + feature = "elf" +))] +pub type NativeFile<'data, R = &'data [u8]> = elf::ElfFile64<'data, crate::Endianness, R>; + +/// The native executable file for the target platform. +#[cfg(all(target_os = "macos", target_pointer_width = "32", feature = "macho"))] +pub type NativeFile<'data, R = &'data [u8]> = macho::MachOFile32<'data, crate::Endianness, R>; + +/// The native executable file for the target platform. +#[cfg(all(target_os = "macos", target_pointer_width = "64", feature = "macho"))] +pub type NativeFile<'data, R = &'data [u8]> = macho::MachOFile64<'data, crate::Endianness, R>; + +/// The native executable file for the target platform. +#[cfg(all(target_os = "windows", target_pointer_width = "32", feature = "pe"))] +pub type NativeFile<'data, R = &'data [u8]> = pe::PeFile32<'data, R>; + +/// The native executable file for the target platform. +#[cfg(all(target_os = "windows", target_pointer_width = "64", feature = "pe"))] +pub type NativeFile<'data, R = &'data [u8]> = pe::PeFile64<'data, R>; + +/// The native executable file for the target platform. +#[cfg(all(feature = "wasm", target_arch = "wasm32", feature = "wasm"))] +pub type NativeFile<'data, R = &'data [u8]> = wasm::WasmFile<'data, R>; + +/// A file format kind. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum FileKind { + /// A Unix archive. + #[cfg(feature = "archive")] + Archive, + /// A COFF object file. + #[cfg(feature = "coff")] + Coff, + /// A dyld cache file containing Mach-O images. + #[cfg(feature = "macho")] + DyldCache, + /// A 32-bit ELF file. + #[cfg(feature = "elf")] + Elf32, + /// A 64-bit ELF file. + #[cfg(feature = "elf")] + Elf64, + /// A 32-bit Mach-O file. + #[cfg(feature = "macho")] + MachO32, + /// A 64-bit Mach-O file. + #[cfg(feature = "macho")] + MachO64, + /// A 32-bit Mach-O fat binary. + #[cfg(feature = "macho")] + MachOFat32, + /// A 64-bit Mach-O fat binary. + #[cfg(feature = "macho")] + MachOFat64, + /// A 32-bit PE file. + #[cfg(feature = "pe")] + Pe32, + /// A 64-bit PE file. + #[cfg(feature = "pe")] + Pe64, + /// A Wasm file. + #[cfg(feature = "wasm")] + Wasm, +} + +impl FileKind { + /// Determine a file kind by parsing the start of the file. + pub fn parse<'data, R: ReadRef<'data>>(data: R) -> Result { + Self::parse_at(data, 0) + } + + /// Determine a file kind by parsing at the given offset. + pub fn parse_at<'data, R: ReadRef<'data>>(data: R, offset: u64) -> Result { + let magic = data + .read_bytes_at(offset, 16) + .read_error("Could not read file magic")?; + if magic.len() < 16 { + return Err(Error("File too short")); + } + + let kind = match [magic[0], magic[1], magic[2], magic[3], magic[4], magic[5], magic[6], magic[7]] { + #[cfg(feature = "archive")] + [b'!', b'<', b'a', b'r', b'c', b'h', b'>', b'\n'] => FileKind::Archive, + #[cfg(feature = "macho")] + [b'd', b'y', b'l', b'd', b'_', b'v', b'1', b' '] => FileKind::DyldCache, + #[cfg(feature = "elf")] + [0x7f, b'E', b'L', b'F', 1, ..] => FileKind::Elf32, + #[cfg(feature = "elf")] + [0x7f, b'E', b'L', b'F', 2, ..] => FileKind::Elf64, + #[cfg(feature = "macho")] + [0xfe, 0xed, 0xfa, 0xce, ..] + | [0xce, 0xfa, 0xed, 0xfe, ..] => FileKind::MachO32, + #[cfg(feature = "macho")] + | [0xfe, 0xed, 0xfa, 0xcf, ..] + | [0xcf, 0xfa, 0xed, 0xfe, ..] => FileKind::MachO64, + #[cfg(feature = "macho")] + [0xca, 0xfe, 0xba, 0xbe, ..] => FileKind::MachOFat32, + #[cfg(feature = "macho")] + [0xca, 0xfe, 0xba, 0xbf, ..] => FileKind::MachOFat64, + #[cfg(feature = "wasm")] + [0x00, b'a', b's', b'm', ..] => FileKind::Wasm, + #[cfg(feature = "pe")] + [b'M', b'Z', ..] => { + match pe::optional_header_magic(data) { + Ok(crate::pe::IMAGE_NT_OPTIONAL_HDR32_MAGIC) => { + FileKind::Pe32 + } + Ok(crate::pe::IMAGE_NT_OPTIONAL_HDR64_MAGIC) => { + FileKind::Pe64 + } + _ => return Err(Error("Unknown MS-DOS file")), + } + } + // TODO: more COFF machines + #[cfg(feature = "coff")] + // COFF arm + [0xc4, 0x01, ..] + // COFF arm64 + | [0x64, 0xaa, ..] + // COFF x86 + | [0x4c, 0x01, ..] + // COFF x86-64 + | [0x64, 0x86, ..] => FileKind::Coff, + _ => return Err(Error("Unknown file magic")), + }; + Ok(kind) + } +} + +/// An object kind. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum ObjectKind { + /// The object kind is unknown. + Unknown, + /// Relocatable object. + Relocatable, + /// Executable. + Executable, + /// Dynamic shared object. + Dynamic, + /// Core. + Core, +} + +/// The index used to identify a section of a file. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct SectionIndex(pub usize); + +/// The index used to identify a symbol of a file. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct SymbolIndex(pub usize); + +/// The section where a symbol is defined. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum SymbolSection { + /// The section is unknown. + Unknown, + /// The section is not applicable for this symbol (such as file symbols). + None, + /// The symbol is undefined. + Undefined, + /// The symbol has an absolute value. + Absolute, + /// The symbol is a zero-initialized symbol that will be combined with duplicate definitions. + Common, + /// The symbol is defined in the given section. + Section(SectionIndex), +} + +impl SymbolSection { + /// Returns the section index for the section where the symbol is defined. + /// + /// May return `None` if the symbol is not defined in a section. + #[inline] + pub fn index(self) -> Option { + if let SymbolSection::Section(index) = self { + Some(index) + } else { + None + } + } +} + +/// An entry in a `SymbolMap`. +pub trait SymbolMapEntry { + /// The symbol address. + fn address(&self) -> u64; +} + +/// A map from addresses to symbols. +#[derive(Debug, Default, Clone)] +pub struct SymbolMap { + symbols: Vec, +} + +impl SymbolMap { + /// Construct a new symbol map. + /// + /// This function will sort the symbols by address. + pub fn new(mut symbols: Vec) -> Self { + symbols.sort_unstable_by_key(|s| s.address()); + SymbolMap { symbols } + } + + /// Get the symbol before the given address. + pub fn get(&self, address: u64) -> Option<&T> { + let index = match self + .symbols + .binary_search_by_key(&address, |symbol| symbol.address()) + { + Ok(index) => index, + Err(index) => index.checked_sub(1)?, + }; + self.symbols.get(index) + } + + /// Get all symbols in the map. + #[inline] + pub fn symbols(&self) -> &[T] { + &self.symbols + } +} + +/// A `SymbolMap` entry for symbol names. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct SymbolMapName<'data> { + address: u64, + name: &'data str, +} + +impl<'data> SymbolMapName<'data> { + /// Construct a `SymbolMapName`. + pub fn new(address: u64, name: &'data str) -> Self { + SymbolMapName { address, name } + } + + /// The symbol address. + #[inline] + pub fn address(&self) -> u64 { + self.address + } + + /// The symbol name. + #[inline] + pub fn name(&self) -> &'data str { + self.name + } +} + +impl<'data> SymbolMapEntry for SymbolMapName<'data> { + #[inline] + fn address(&self) -> u64 { + self.address + } +} + +/// A map from addresses to symbol names and object files. +/// +/// This is derived from STAB entries in Mach-O files. +#[derive(Debug, Default, Clone)] +pub struct ObjectMap<'data> { + symbols: SymbolMap>, + objects: Vec<&'data [u8]>, +} + +impl<'data> ObjectMap<'data> { + /// Get the entry containing the given address. + pub fn get(&self, address: u64) -> Option<&ObjectMapEntry<'data>> { + self.symbols + .get(address) + .filter(|entry| entry.size == 0 || address.wrapping_sub(entry.address) < entry.size) + } + + /// Get all symbols in the map. + #[inline] + pub fn symbols(&self) -> &[ObjectMapEntry<'data>] { + self.symbols.symbols() + } + + /// Get all objects in the map. + #[inline] + pub fn objects(&self) -> &[&'data [u8]] { + &self.objects + } +} + +/// A `ObjectMap` entry. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ObjectMapEntry<'data> { + address: u64, + size: u64, + name: &'data [u8], + object: usize, +} + +impl<'data> ObjectMapEntry<'data> { + /// Get the symbol address. + #[inline] + pub fn address(&self) -> u64 { + self.address + } + + /// Get the symbol size. + /// + /// This may be 0 if the size is unknown. + #[inline] + pub fn size(&self) -> u64 { + self.size + } + + /// Get the symbol name. + #[inline] + pub fn name(&self) -> &'data [u8] { + self.name + } + + /// Get the index of the object file name. + #[inline] + pub fn object_index(&self) -> usize { + self.object + } + + /// Get the object file name. + #[inline] + pub fn object(&self, map: &ObjectMap<'data>) -> &'data [u8] { + map.objects[self.object] + } +} + +impl<'data> SymbolMapEntry for ObjectMapEntry<'data> { + #[inline] + fn address(&self) -> u64 { + self.address + } +} + +/// An imported symbol. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Import<'data> { + library: ByteString<'data>, + // TODO: or ordinal + name: ByteString<'data>, +} + +impl<'data> Import<'data> { + /// The symbol name. + #[inline] + pub fn name(&self) -> &'data [u8] { + self.name.0 + } + + /// The name of the library to import the symbol from. + #[inline] + pub fn library(&self) -> &'data [u8] { + self.library.0 + } +} + +/// An exported symbol. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Export<'data> { + // TODO: and ordinal? + name: ByteString<'data>, + address: u64, +} + +impl<'data> Export<'data> { + /// The symbol name. + #[inline] + pub fn name(&self) -> &'data [u8] { + self.name.0 + } + + /// The virtual address of the symbol. + #[inline] + pub fn address(&self) -> u64 { + self.address + } +} + +/// PDB Information +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CodeView<'data> { + guid: [u8; 16], + path: ByteString<'data>, + age: u32, +} + +impl<'data> CodeView<'data> { + /// The path to the PDB as stored in CodeView + #[inline] + pub fn path(&self) -> &'data [u8] { + self.path.0 + } + + /// The age of the PDB + #[inline] + pub fn age(&self) -> u32 { + self.age + } + + /// The GUID of the PDB. + #[inline] + pub fn guid(&self) -> [u8; 16] { + self.guid + } +} + +/// The target referenced by a relocation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum RelocationTarget { + /// The target is a symbol. + Symbol(SymbolIndex), + /// The target is a section. + Section(SectionIndex), + /// The offset is an absolute address. + Absolute, +} + +/// A relocation entry. +#[derive(Debug)] +pub struct Relocation { + kind: RelocationKind, + encoding: RelocationEncoding, + size: u8, + target: RelocationTarget, + addend: i64, + implicit_addend: bool, +} + +impl Relocation { + /// The operation used to calculate the result of the relocation. + #[inline] + pub fn kind(&self) -> RelocationKind { + self.kind + } + + /// Information about how the result of the relocation operation is encoded in the place. + #[inline] + pub fn encoding(&self) -> RelocationEncoding { + self.encoding + } + + /// The size in bits of the place of the relocation. + /// + /// If 0, then the size is determined by the relocation kind. + #[inline] + pub fn size(&self) -> u8 { + self.size + } + + /// The target of the relocation. + #[inline] + pub fn target(&self) -> RelocationTarget { + self.target + } + + /// The addend to use in the relocation calculation. + #[inline] + pub fn addend(&self) -> i64 { + self.addend + } + + /// Set the addend to use in the relocation calculation. + #[inline] + pub fn set_addend(&mut self, addend: i64) { + self.addend = addend + } + + /// Returns true if there is an implicit addend stored in the data at the offset + /// to be relocated. + #[inline] + pub fn has_implicit_addend(&self) -> bool { + self.implicit_addend + } +} + +/// A data compression format. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum CompressionFormat { + /// The data is uncompressed. + None, + /// The data is compressed, but the compression format is unknown. + Unknown, + /// ZLIB/DEFLATE. + /// + /// Used for ELF compression and GNU compressed debug information. + Zlib, +} + +/// A range in a file that may be compressed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct CompressedFileRange { + /// The data compression format. + pub format: CompressionFormat, + /// The file offset of the compressed data. + pub offset: u64, + /// The compressed data size. + pub compressed_size: u64, + /// The uncompressed data size. + pub uncompressed_size: u64, +} + +impl CompressedFileRange { + /// Data that is uncompressed. + #[inline] + pub fn none(range: Option<(u64, u64)>) -> Self { + if let Some((offset, size)) = range { + CompressedFileRange { + format: CompressionFormat::None, + offset, + compressed_size: size, + uncompressed_size: size, + } + } else { + CompressedFileRange { + format: CompressionFormat::None, + offset: 0, + compressed_size: 0, + uncompressed_size: 0, + } + } + } + + /// Convert to `CompressedData` by reading from the file. + pub fn data<'data, R: ReadRef<'data>>(self, file: R) -> Result> { + let data = file + .read_bytes_at(self.offset, self.compressed_size) + .read_error("Invalid compressed data size or offset")?; + Ok(CompressedData { + format: self.format, + data, + uncompressed_size: self.uncompressed_size, + }) + } +} + +/// Data that may be compressed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct CompressedData<'data> { + /// The data compression format. + pub format: CompressionFormat, + /// The compressed data. + pub data: &'data [u8], + /// The uncompressed data size. + pub uncompressed_size: u64, +} + +impl<'data> CompressedData<'data> { + /// Data that is uncompressed. + #[inline] + pub fn none(data: &'data [u8]) -> Self { + CompressedData { + format: CompressionFormat::None, + data, + uncompressed_size: data.len() as u64, + } + } + + /// Return the uncompressed data. + /// + /// Returns an error for invalid data or unsupported compression. + /// This includes if the data is compressed but the `compression` feature + /// for this crate is disabled. + pub fn decompress(self) -> Result> { + match self.format { + CompressionFormat::None => Ok(Cow::Borrowed(self.data)), + #[cfg(feature = "compression")] + CompressionFormat::Zlib => { + use core::convert::TryInto; + let size = self + .uncompressed_size + .try_into() + .ok() + .read_error("Uncompressed data size is too large.")?; + let mut decompressed = Vec::with_capacity(size); + let mut decompress = flate2::Decompress::new(true); + decompress + .decompress_vec( + self.data, + &mut decompressed, + flate2::FlushDecompress::Finish, + ) + .ok() + .read_error("Invalid zlib compressed data")?; + Ok(Cow::Owned(decompressed)) + } + _ => Err(Error("Unsupported compressed data.")), + } + } +} diff --git a/crux-mir/lib/object/src/read/pe/data_directory.rs b/crux-mir/lib/object/src/read/pe/data_directory.rs new file mode 100644 index 000000000..8c1955355 --- /dev/null +++ b/crux-mir/lib/object/src/read/pe/data_directory.rs @@ -0,0 +1,185 @@ +use core::slice; + +use crate::read::{Error, ReadError, ReadRef, Result}; +use crate::{pe, LittleEndian as LE}; + +use super::{ExportTable, ImportTable, RelocationBlockIterator, ResourceDirectory, SectionTable}; + +/// The table of data directories in a PE file. +#[derive(Debug, Clone, Copy)] +pub struct DataDirectories<'data> { + entries: &'data [pe::ImageDataDirectory], +} + +impl<'data> DataDirectories<'data> { + /// Parse the data directory table. + /// + /// `data` must be the remaining optional data following the + /// [optional header](pe::ImageOptionalHeader64). `number` must be from the + /// [`number_of_rva_and_sizes`](pe::ImageOptionalHeader64::number_of_rva_and_sizes) + /// field of the optional header. + pub fn parse(data: &'data [u8], number: u32) -> Result { + let entries = data + .read_slice_at(0, number as usize) + .read_error("Invalid PE number of RVA and sizes")?; + Ok(DataDirectories { entries }) + } + + /// The number of data directories. + #[allow(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + self.entries.len() + } + + /// Iterator over the data directories. + pub fn iter(&self) -> slice::Iter<'data, pe::ImageDataDirectory> { + self.entries.iter() + } + + /// Iterator which gives the directories as well as their index (one of the IMAGE_DIRECTORY_ENTRY_* constants). + pub fn enumerate(&self) -> core::iter::Enumerate> { + self.entries.iter().enumerate() + } + + /// Returns the data directory at the given index. + /// + /// Index should be one of the `IMAGE_DIRECTORY_ENTRY_*` constants. + /// + /// Returns `None` if the index is larger than the table size, + /// or if the entry at the index has a zero virtual address. + pub fn get(&self, index: usize) -> Option<&'data pe::ImageDataDirectory> { + self.entries + .get(index) + .filter(|d| d.virtual_address.get(LE) != 0) + } + + /// Returns the unparsed export directory. + /// + /// `data` must be the entire file data. + pub fn export_directory>( + &self, + data: R, + sections: &SectionTable<'data>, + ) -> Result> { + let data_dir = match self.get(pe::IMAGE_DIRECTORY_ENTRY_EXPORT) { + Some(data_dir) => data_dir, + None => return Ok(None), + }; + let export_data = data_dir.data(data, sections)?; + ExportTable::parse_directory(export_data).map(Some) + } + + /// Returns the partially parsed export directory. + /// + /// `data` must be the entire file data. + pub fn export_table>( + &self, + data: R, + sections: &SectionTable<'data>, + ) -> Result>> { + let data_dir = match self.get(pe::IMAGE_DIRECTORY_ENTRY_EXPORT) { + Some(data_dir) => data_dir, + None => return Ok(None), + }; + let export_va = data_dir.virtual_address.get(LE); + let export_data = data_dir.data(data, sections)?; + ExportTable::parse(export_data, export_va).map(Some) + } + + /// Returns the partially parsed import directory. + /// + /// `data` must be the entire file data. + pub fn import_table>( + &self, + data: R, + sections: &SectionTable<'data>, + ) -> Result>> { + let data_dir = match self.get(pe::IMAGE_DIRECTORY_ENTRY_IMPORT) { + Some(data_dir) => data_dir, + None => return Ok(None), + }; + let import_va = data_dir.virtual_address.get(LE); + let (section_data, section_va) = sections + .pe_data_containing(data, import_va) + .read_error("Invalid import data dir virtual address")?; + Ok(Some(ImportTable::new(section_data, section_va, import_va))) + } + + /// Returns the blocks in the base relocation directory. + /// + /// `data` must be the entire file data. + pub fn relocation_blocks>( + &self, + data: R, + sections: &SectionTable<'data>, + ) -> Result>> { + let data_dir = match self.get(pe::IMAGE_DIRECTORY_ENTRY_BASERELOC) { + Some(data_dir) => data_dir, + None => return Ok(None), + }; + let reloc_data = data_dir.data(data, sections)?; + Ok(Some(RelocationBlockIterator::new(reloc_data))) + } + + /// Returns the resource directory. + /// + /// `data` must be the entire file data. + pub fn resource_directory>( + &self, + data: R, + sections: &SectionTable<'data>, + ) -> Result>> { + let data_dir = match self.get(pe::IMAGE_DIRECTORY_ENTRY_RESOURCE) { + Some(data_dir) => data_dir, + None => return Ok(None), + }; + let rsrc_data = data_dir.data(data, sections)?; + Ok(Some(ResourceDirectory::new(rsrc_data))) + } +} + +impl pe::ImageDataDirectory { + /// Return the virtual address range of this directory entry. + pub fn address_range(&self) -> (u32, u32) { + (self.virtual_address.get(LE), self.size.get(LE)) + } + + /// Return the file offset and size of this directory entry. + /// + /// This function has some limitations: + /// - It requires that the data is contained in a single section. + /// - It uses the size field of the directory entry, which is + /// not desirable for all data directories. + /// - It uses the `virtual_address` of the directory entry as an address, + /// which is not valid for `IMAGE_DIRECTORY_ENTRY_SECURITY`. + pub fn file_range<'data>(&self, sections: &SectionTable<'data>) -> Result<(u32, u32)> { + let (offset, section_size) = sections + .pe_file_range_at(self.virtual_address.get(LE)) + .read_error("Invalid data dir virtual address")?; + let size = self.size.get(LE); + if size > section_size { + return Err(Error("Invalid data dir size")); + } + Ok((offset, size)) + } + + /// Get the data referenced by this directory entry. + /// + /// This function has some limitations: + /// - It requires that the data is contained in a single section. + /// - It uses the size field of the directory entry, which is + /// not desirable for all data directories. + /// - It uses the `virtual_address` of the directory entry as an address, + /// which is not valid for `IMAGE_DIRECTORY_ENTRY_SECURITY`. + pub fn data<'data, R: ReadRef<'data>>( + &self, + data: R, + sections: &SectionTable<'data>, + ) -> Result<&'data [u8]> { + sections + .pe_data_at(data, self.virtual_address.get(LE)) + .read_error("Invalid data dir virtual address")? + .get(..self.size.get(LE) as usize) + .read_error("Invalid data dir size") + } +} diff --git a/crux-mir/lib/object/src/read/pe/export.rs b/crux-mir/lib/object/src/read/pe/export.rs new file mode 100644 index 000000000..88dc78d50 --- /dev/null +++ b/crux-mir/lib/object/src/read/pe/export.rs @@ -0,0 +1,331 @@ +use alloc::vec::Vec; +use core::fmt::Debug; + +use crate::read::{ByteString, Bytes, Error, ReadError, ReadRef, Result}; +use crate::{pe, LittleEndian as LE, U16Bytes, U32Bytes}; + +/// Where an export is pointing to. +#[derive(Clone, Copy)] +pub enum ExportTarget<'data> { + /// The address of the export, relative to the image base. + Address(u32), + /// Forwarded to an export ordinal in another DLL. + /// + /// This gives the name of the DLL, and the ordinal. + ForwardByOrdinal(&'data [u8], u32), + /// Forwarded to an export name in another DLL. + /// + /// This gives the name of the DLL, and the export name. + ForwardByName(&'data [u8], &'data [u8]), +} + +impl<'data> ExportTarget<'data> { + /// Returns true if the target is an address. + pub fn is_address(&self) -> bool { + match self { + ExportTarget::Address(_) => true, + _ => false, + } + } + + /// Returns true if the export is forwarded to another DLL. + pub fn is_forward(&self) -> bool { + !self.is_address() + } +} + +/// An export from a PE file. +/// +/// There are multiple kinds of PE exports (with or without a name, and local or forwarded). +#[derive(Clone, Copy)] +pub struct Export<'data> { + /// The ordinal of the export. + /// + /// These are sequential, starting at a base specified in the DLL. + pub ordinal: u32, + /// The name of the export, if known. + pub name: Option<&'data [u8]>, + /// The target of this export. + pub target: ExportTarget<'data>, +} + +impl<'a> Debug for Export<'a> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::result::Result<(), core::fmt::Error> { + f.debug_struct("Export") + .field("ordinal", &self.ordinal) + .field("name", &self.name.map(ByteString)) + .field("target", &self.target) + .finish() + } +} + +impl<'a> Debug for ExportTarget<'a> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::result::Result<(), core::fmt::Error> { + match self { + ExportTarget::Address(address) => write!(f, "Address({:#x})", address), + ExportTarget::ForwardByOrdinal(library, ordinal) => write!( + f, + "ForwardByOrdinal({:?}.#{})", + ByteString(library), + ordinal + ), + ExportTarget::ForwardByName(library, name) => write!( + f, + "ForwardByName({:?}.{:?})", + ByteString(library), + ByteString(name) + ), + } + } +} + +/// A partially parsed PE export table. +#[derive(Debug, Clone)] +pub struct ExportTable<'data> { + data: Bytes<'data>, + virtual_address: u32, + directory: &'data pe::ImageExportDirectory, + addresses: &'data [U32Bytes], + names: &'data [U32Bytes], + name_ordinals: &'data [U16Bytes], +} + +impl<'data> ExportTable<'data> { + /// Parse the export table given its section data and address. + pub fn parse(data: &'data [u8], virtual_address: u32) -> Result { + let directory = Self::parse_directory(data)?; + let data = Bytes(data); + + let mut addresses = &[][..]; + let address_of_functions = directory.address_of_functions.get(LE); + if address_of_functions != 0 { + addresses = data + .read_slice_at::>( + address_of_functions.wrapping_sub(virtual_address) as usize, + directory.number_of_functions.get(LE) as usize, + ) + .read_error("Invalid PE export address table")?; + } + + let mut names = &[][..]; + let mut name_ordinals = &[][..]; + let address_of_names = directory.address_of_names.get(LE); + let address_of_name_ordinals = directory.address_of_name_ordinals.get(LE); + if address_of_names != 0 { + if address_of_name_ordinals == 0 { + return Err(Error("Missing PE export ordinal table")); + } + + let number = directory.number_of_names.get(LE) as usize; + names = data + .read_slice_at::>( + address_of_names.wrapping_sub(virtual_address) as usize, + number, + ) + .read_error("Invalid PE export name pointer table")?; + name_ordinals = data + .read_slice_at::>( + address_of_name_ordinals.wrapping_sub(virtual_address) as usize, + number, + ) + .read_error("Invalid PE export ordinal table")?; + } + + Ok(ExportTable { + data, + virtual_address, + directory, + addresses, + names, + name_ordinals, + }) + } + + /// Parse the export directory given its section data. + pub fn parse_directory(data: &'data [u8]) -> Result<&'data pe::ImageExportDirectory> { + data.read_at::(0) + .read_error("Invalid PE export dir size") + } + + /// Returns the header of the export table. + pub fn directory(&self) -> &'data pe::ImageExportDirectory { + self.directory + } + + /// Returns the base value of ordinals. + /// + /// Adding this to an address index will give an ordinal. + pub fn ordinal_base(&self) -> u32 { + self.directory.base.get(LE) + } + + /// Returns the unparsed address table. + /// + /// An address table entry may be a local address, or the address of a forwarded export entry. + /// See [`Self::is_forward`] and [`Self::target_from_address`]. + pub fn addresses(&self) -> &'data [U32Bytes] { + self.addresses + } + + /// Returns the unparsed name pointer table. + /// + /// A name pointer table entry can be used with [`Self::name_from_pointer`]. + pub fn name_pointers(&self) -> &'data [U32Bytes] { + self.names + } + + /// Returns the unparsed ordinal table. + /// + /// An ordinal table entry is a 0-based index into the address table. + /// See [`Self::address_by_index`] and [`Self::target_by_index`]. + pub fn name_ordinals(&self) -> &'data [U16Bytes] { + self.name_ordinals + } + + /// Returns an iterator for the entries in the name pointer table and ordinal table. + /// + /// A name pointer table entry can be used with [`Self::name_from_pointer`]. + /// + /// An ordinal table entry is a 0-based index into the address table. + /// See [`Self::address_by_index`] and [`Self::target_by_index`]. + pub fn name_iter(&self) -> impl Iterator + 'data { + self.names + .iter() + .map(|x| x.get(LE)) + .zip(self.name_ordinals.iter().map(|x| x.get(LE))) + } + + /// Returns the export address table entry at the given address index. + /// + /// This may be a local address, or the address of a forwarded export entry. + /// See [`Self::is_forward`] and [`Self::target_from_address`]. + /// + /// `index` is a 0-based index into the export address table. + pub fn address_by_index(&self, index: u32) -> Result { + Ok(self + .addresses + .get(index as usize) + .read_error("Invalid PE export address index")? + .get(LE)) + } + + /// Returns the export address table entry at the given ordinal. + /// + /// This may be a local address, or the address of a forwarded export entry. + /// See [`Self::is_forward`] and [`Self::target_from_address`]. + pub fn address_by_ordinal(&self, ordinal: u32) -> Result { + self.address_by_index(ordinal.wrapping_sub(self.ordinal_base())) + } + + /// Returns the target of the export at the given address index. + /// + /// `index` is a 0-based index into the export address table. + pub fn target_by_index(&self, index: u32) -> Result> { + self.target_from_address(self.address_by_index(index)?) + } + + /// Returns the target of the export at the given ordinal. + pub fn target_by_ordinal(&self, ordinal: u32) -> Result> { + self.target_from_address(self.address_by_ordinal(ordinal)?) + } + + /// Convert an export address table entry into a target. + pub fn target_from_address(&self, address: u32) -> Result> { + Ok(if let Some(forward) = self.forward_string(address)? { + let i = forward + .iter() + .position(|x| *x == b'.') + .read_error("Missing PE forwarded export separator")?; + let library = &forward[..i]; + match &forward[i + 1..] { + [b'#', digits @ ..] => { + let ordinal = + parse_ordinal(digits).read_error("Invalid PE forwarded export ordinal")?; + ExportTarget::ForwardByOrdinal(library, ordinal) + } + [] => { + return Err(Error("Missing PE forwarded export name")); + } + name => ExportTarget::ForwardByName(library, name), + } + } else { + ExportTarget::Address(address) + }) + } + + fn forward_offset(&self, address: u32) -> Option { + let offset = address.wrapping_sub(self.virtual_address) as usize; + if offset < self.data.len() { + Some(offset) + } else { + None + } + } + + /// Return true if the export address table entry is a forward. + pub fn is_forward(&self, address: u32) -> bool { + self.forward_offset(address).is_some() + } + + /// Return the forward string if the export address table entry is a forward. + pub fn forward_string(&self, address: u32) -> Result> { + if let Some(offset) = self.forward_offset(address) { + self.data + .read_string_at(offset) + .read_error("Invalid PE forwarded export address") + .map(Some) + } else { + Ok(None) + } + } + + /// Convert an export name pointer table entry into a name. + pub fn name_from_pointer(&self, name_pointer: u32) -> Result<&'data [u8]> { + let offset = name_pointer.wrapping_sub(self.virtual_address); + self.data + .read_string_at(offset as usize) + .read_error("Invalid PE export name pointer") + } + + /// Returns the parsed exports in this table. + pub fn exports(&self) -> Result>> { + // First, let's list all exports. + let mut exports = Vec::new(); + let ordinal_base = self.ordinal_base(); + for (i, address) in self.addresses.iter().enumerate() { + // Convert from an array index to an ordinal. + let ordinal = ordinal_base.wrapping_add(i as u32); + let target = self.target_from_address(address.get(LE))?; + exports.push(Export { + ordinal, + target, + // Might be populated later. + name: None, + }); + } + + // Now, check whether some (or all) of them have an associated name. + // `ordinal_index` is a 0-based index into `addresses`. + for (name_pointer, ordinal_index) in self.name_iter() { + let name = self.name_from_pointer(name_pointer)?; + exports + .get_mut(ordinal_index as usize) + .read_error("Invalid PE export ordinal")? + .name = Some(name); + } + + Ok(exports) + } +} + +fn parse_ordinal(digits: &[u8]) -> Option { + if digits.is_empty() { + return None; + } + let mut result: u32 = 0; + for &c in digits { + let x = (c as char).to_digit(10)?; + result = result.checked_mul(10)?.checked_add(x)?; + } + Some(result) +} diff --git a/crux-mir/lib/object/src/read/pe/file.rs b/crux-mir/lib/object/src/read/pe/file.rs new file mode 100644 index 000000000..15b42074a --- /dev/null +++ b/crux-mir/lib/object/src/read/pe/file.rs @@ -0,0 +1,1018 @@ +use alloc::vec::Vec; +use core::fmt::Debug; +use core::{mem, str}; + +use core::convert::TryInto; + +use crate::read::coff::{CoffCommon, CoffSymbol, CoffSymbolIterator, CoffSymbolTable, SymbolTable}; +use crate::read::{ + self, Architecture, ComdatKind, Error, Export, FileFlags, Import, NoDynamicRelocationIterator, + Object, ObjectComdat, ObjectKind, ReadError, ReadRef, Result, SectionIndex, SymbolIndex, +}; +use crate::{pe, ByteString, Bytes, CodeView, LittleEndian as LE, Pod, U32}; + +use super::{ + DataDirectories, ExportTable, ImageThunkData, ImportTable, PeSection, PeSectionIterator, + PeSegment, PeSegmentIterator, RichHeaderInfo, SectionTable, +}; + +/// A PE32 (32-bit) image file. +pub type PeFile32<'data, R = &'data [u8]> = PeFile<'data, pe::ImageNtHeaders32, R>; +/// A PE32+ (64-bit) image file. +pub type PeFile64<'data, R = &'data [u8]> = PeFile<'data, pe::ImageNtHeaders64, R>; + +/// A PE object file. +#[derive(Debug)] +pub struct PeFile<'data, Pe, R = &'data [u8]> +where + Pe: ImageNtHeaders, + R: ReadRef<'data>, +{ + pub(super) dos_header: &'data pe::ImageDosHeader, + pub(super) nt_headers: &'data Pe, + pub(super) data_directories: DataDirectories<'data>, + pub(super) common: CoffCommon<'data, R>, + pub(super) data: R, +} + +impl<'data, Pe, R> PeFile<'data, Pe, R> +where + Pe: ImageNtHeaders, + R: ReadRef<'data>, +{ + /// Parse the raw PE file data. + pub fn parse(data: R) -> Result { + let dos_header = pe::ImageDosHeader::parse(data)?; + let mut offset = dos_header.nt_headers_offset().into(); + let (nt_headers, data_directories) = Pe::parse(data, &mut offset)?; + let sections = nt_headers.sections(data, offset)?; + let coff_symbols = nt_headers.symbols(data); + let image_base = nt_headers.optional_header().image_base(); + + Ok(PeFile { + dos_header, + nt_headers, + data_directories, + common: CoffCommon { + sections, + // The PE file format deprecates the COFF symbol table (https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#coff-file-header-object-and-image) + // We do not want to prevent parsing the rest of the PE file for a corrupt COFF header, but rather return an empty symbol table + symbols: coff_symbols.unwrap_or_default(), + image_base, + }, + data, + }) + } + + /// Returns this binary data. + pub fn data(&self) -> R { + self.data + } + + /// Return the DOS header of this file. + pub fn dos_header(&self) -> &'data pe::ImageDosHeader { + self.dos_header + } + + /// Return the NT Headers of this file. + pub fn nt_headers(&self) -> &'data Pe { + self.nt_headers + } + + /// Returns information about the rich header of this file (if any). + pub fn rich_header_info(&self) -> Option { + RichHeaderInfo::parse(self.data, self.dos_header.nt_headers_offset().into()) + } + + /// Returns the section table of this binary. + pub fn section_table(&self) -> SectionTable<'data> { + self.common.sections + } + + /// Returns the data directories of this file. + pub fn data_directories(&self) -> DataDirectories<'data> { + self.data_directories + } + + /// Returns the data directory at the given index. + pub fn data_directory(&self, id: usize) -> Option<&'data pe::ImageDataDirectory> { + self.data_directories.get(id) + } + + /// Returns the export table of this file. + /// + /// The export table is located using the data directory. + pub fn export_table(&self) -> Result>> { + self.data_directories + .export_table(self.data, &self.common.sections) + } + + /// Returns the import table of this file. + /// + /// The import table is located using the data directory. + pub fn import_table(&self) -> Result>> { + self.data_directories + .import_table(self.data, &self.common.sections) + } + + pub(super) fn section_alignment(&self) -> u64 { + u64::from(self.nt_headers.optional_header().section_alignment()) + } +} + +impl<'data, Pe, R> read::private::Sealed for PeFile<'data, Pe, R> +where + Pe: ImageNtHeaders, + R: ReadRef<'data>, +{ +} + +impl<'data, 'file, Pe, R> Object<'data, 'file> for PeFile<'data, Pe, R> +where + 'data: 'file, + Pe: ImageNtHeaders, + R: 'file + ReadRef<'data>, +{ + type Segment = PeSegment<'data, 'file, Pe, R>; + type SegmentIterator = PeSegmentIterator<'data, 'file, Pe, R>; + type Section = PeSection<'data, 'file, Pe, R>; + type SectionIterator = PeSectionIterator<'data, 'file, Pe, R>; + type Comdat = PeComdat<'data, 'file, Pe, R>; + type ComdatIterator = PeComdatIterator<'data, 'file, Pe, R>; + type Symbol = CoffSymbol<'data, 'file, R>; + type SymbolIterator = CoffSymbolIterator<'data, 'file, R>; + type SymbolTable = CoffSymbolTable<'data, 'file, R>; + type DynamicRelocationIterator = NoDynamicRelocationIterator; + + fn architecture(&self) -> Architecture { + match self.nt_headers.file_header().machine.get(LE) { + pe::IMAGE_FILE_MACHINE_ARMNT => Architecture::Arm, + pe::IMAGE_FILE_MACHINE_ARM64 => Architecture::Aarch64, + pe::IMAGE_FILE_MACHINE_I386 => Architecture::I386, + pe::IMAGE_FILE_MACHINE_AMD64 => Architecture::X86_64, + _ => Architecture::Unknown, + } + } + + #[inline] + fn is_little_endian(&self) -> bool { + // Only little endian is supported. + true + } + + #[inline] + fn is_64(&self) -> bool { + self.nt_headers.is_type_64() + } + + fn kind(&self) -> ObjectKind { + let characteristics = self.nt_headers.file_header().characteristics.get(LE); + if characteristics & pe::IMAGE_FILE_DLL != 0 { + ObjectKind::Dynamic + } else if characteristics & pe::IMAGE_FILE_SYSTEM != 0 { + ObjectKind::Unknown + } else { + ObjectKind::Executable + } + } + + fn segments(&'file self) -> PeSegmentIterator<'data, 'file, Pe, R> { + PeSegmentIterator { + file: self, + iter: self.common.sections.iter(), + } + } + + fn section_by_name_bytes( + &'file self, + section_name: &[u8], + ) -> Option> { + self.common + .sections + .section_by_name(self.common.symbols.strings(), section_name) + .map(|(index, section)| PeSection { + file: self, + index: SectionIndex(index), + section, + }) + } + + fn section_by_index( + &'file self, + index: SectionIndex, + ) -> Result> { + let section = self.common.sections.section(index.0)?; + Ok(PeSection { + file: self, + index, + section, + }) + } + + fn sections(&'file self) -> PeSectionIterator<'data, 'file, Pe, R> { + PeSectionIterator { + file: self, + iter: self.common.sections.iter().enumerate(), + } + } + + fn comdats(&'file self) -> PeComdatIterator<'data, 'file, Pe, R> { + PeComdatIterator { file: self } + } + + fn symbol_by_index(&'file self, index: SymbolIndex) -> Result> { + let symbol = self.common.symbols.symbol(index.0)?; + Ok(CoffSymbol { + file: &self.common, + index, + symbol, + }) + } + + fn symbols(&'file self) -> CoffSymbolIterator<'data, 'file, R> { + CoffSymbolIterator { + file: &self.common, + index: 0, + } + } + + fn symbol_table(&'file self) -> Option> { + Some(CoffSymbolTable { file: &self.common }) + } + + fn dynamic_symbols(&'file self) -> CoffSymbolIterator<'data, 'file, R> { + CoffSymbolIterator { + file: &self.common, + // Hack: don't return any. + index: self.common.symbols.len(), + } + } + + fn dynamic_symbol_table(&'file self) -> Option> { + None + } + + fn dynamic_relocations(&'file self) -> Option { + None + } + + fn imports(&self) -> Result>> { + let mut imports = Vec::new(); + if let Some(import_table) = self.import_table()? { + let mut import_descs = import_table.descriptors()?; + while let Some(import_desc) = import_descs.next()? { + let library = import_table.name(import_desc.name.get(LE))?; + let mut first_thunk = import_desc.original_first_thunk.get(LE); + if first_thunk == 0 { + first_thunk = import_desc.first_thunk.get(LE); + } + let mut thunks = import_table.thunks(first_thunk)?; + while let Some(thunk) = thunks.next::()? { + if !thunk.is_ordinal() { + let (_hint, name) = import_table.hint_name(thunk.address())?; + imports.push(Import { + library: ByteString(library), + name: ByteString(name), + }); + } + } + } + } + Ok(imports) + } + + fn exports(&self) -> Result>> { + let mut exports = Vec::new(); + if let Some(export_table) = self.export_table()? { + for (name_pointer, address_index) in export_table.name_iter() { + let name = export_table.name_from_pointer(name_pointer)?; + let address = export_table.address_by_index(address_index.into())?; + if !export_table.is_forward(address) { + exports.push(Export { + name: ByteString(name), + address: self.common.image_base.wrapping_add(address.into()), + }) + } + } + } + Ok(exports) + } + + fn pdb_info(&self) -> Result> { + let data_dir = match self.data_directory(pe::IMAGE_DIRECTORY_ENTRY_DEBUG) { + Some(data_dir) => data_dir, + None => return Ok(None), + }; + let debug_data = data_dir.data(self.data, &self.common.sections).map(Bytes)?; + let debug_dir = debug_data + .read_at::(0) + .read_error("Invalid PE debug dir size")?; + + if debug_dir.typ.get(LE) != pe::IMAGE_DEBUG_TYPE_CODEVIEW { + return Ok(None); + } + + let info = self + .data + .read_slice_at::( + debug_dir.pointer_to_raw_data.get(LE) as u64, + debug_dir.size_of_data.get(LE) as usize, + ) + .read_error("Invalid CodeView Info address")?; + + let mut info = Bytes(info); + + let sig = info + .read_bytes(4) + .read_error("Invalid CodeView signature")?; + if sig.0 != b"RSDS" { + return Ok(None); + } + + let guid: [u8; 16] = info + .read_bytes(16) + .read_error("Invalid CodeView GUID")? + .0 + .try_into() + .unwrap(); + + let age = info.read::>().read_error("Invalid CodeView Age")?; + + let path = info + .read_string() + .read_error("Invalid CodeView file path")?; + + Ok(Some(CodeView { + path: ByteString(path), + guid, + age: age.get(LE), + })) + } + + fn has_debug_symbols(&self) -> bool { + self.section_by_name(".debug_info").is_some() + } + + fn relative_address_base(&self) -> u64 { + self.common.image_base + } + + fn entry(&self) -> u64 { + u64::from(self.nt_headers.optional_header().address_of_entry_point()) + .wrapping_add(self.common.image_base) + } + + fn flags(&self) -> FileFlags { + FileFlags::Coff { + characteristics: self.nt_headers.file_header().characteristics.get(LE), + } + } +} + +/// An iterator over the COMDAT section groups of a `PeFile32`. +pub type PeComdatIterator32<'data, 'file, R = &'data [u8]> = + PeComdatIterator<'data, 'file, pe::ImageNtHeaders32, R>; +/// An iterator over the COMDAT section groups of a `PeFile64`. +pub type PeComdatIterator64<'data, 'file, R = &'data [u8]> = + PeComdatIterator<'data, 'file, pe::ImageNtHeaders64, R>; + +/// An iterator over the COMDAT section groups of a `PeFile`. +#[derive(Debug)] +pub struct PeComdatIterator<'data, 'file, Pe, R = &'data [u8]> +where + Pe: ImageNtHeaders, + R: ReadRef<'data>, +{ + #[allow(unused)] + file: &'file PeFile<'data, Pe, R>, +} + +impl<'data, 'file, Pe, R> Iterator for PeComdatIterator<'data, 'file, Pe, R> +where + Pe: ImageNtHeaders, + R: ReadRef<'data>, +{ + type Item = PeComdat<'data, 'file, Pe, R>; + + #[inline] + fn next(&mut self) -> Option { + None + } +} + +/// A COMDAT section group of a `PeFile32`. +pub type PeComdat32<'data, 'file, R = &'data [u8]> = + PeComdat<'data, 'file, pe::ImageNtHeaders32, R>; +/// A COMDAT section group of a `PeFile64`. +pub type PeComdat64<'data, 'file, R = &'data [u8]> = + PeComdat<'data, 'file, pe::ImageNtHeaders64, R>; + +/// A COMDAT section group of a `PeFile`. +#[derive(Debug)] +pub struct PeComdat<'data, 'file, Pe, R = &'data [u8]> +where + Pe: ImageNtHeaders, + R: ReadRef<'data>, +{ + #[allow(unused)] + file: &'file PeFile<'data, Pe, R>, +} + +impl<'data, 'file, Pe, R> read::private::Sealed for PeComdat<'data, 'file, Pe, R> +where + Pe: ImageNtHeaders, + R: ReadRef<'data>, +{ +} + +impl<'data, 'file, Pe, R> ObjectComdat<'data> for PeComdat<'data, 'file, Pe, R> +where + Pe: ImageNtHeaders, + R: ReadRef<'data>, +{ + type SectionIterator = PeComdatSectionIterator<'data, 'file, Pe, R>; + + #[inline] + fn kind(&self) -> ComdatKind { + unreachable!(); + } + + #[inline] + fn symbol(&self) -> SymbolIndex { + unreachable!(); + } + + #[inline] + fn name_bytes(&self) -> Result<&[u8]> { + unreachable!(); + } + + #[inline] + fn name(&self) -> Result<&str> { + unreachable!(); + } + + #[inline] + fn sections(&self) -> Self::SectionIterator { + unreachable!(); + } +} + +/// An iterator over the sections in a COMDAT section group of a `PeFile32`. +pub type PeComdatSectionIterator32<'data, 'file, R = &'data [u8]> = + PeComdatSectionIterator<'data, 'file, pe::ImageNtHeaders32, R>; +/// An iterator over the sections in a COMDAT section group of a `PeFile64`. +pub type PeComdatSectionIterator64<'data, 'file, R = &'data [u8]> = + PeComdatSectionIterator<'data, 'file, pe::ImageNtHeaders64, R>; + +/// An iterator over the sections in a COMDAT section group of a `PeFile`. +#[derive(Debug)] +pub struct PeComdatSectionIterator<'data, 'file, Pe, R = &'data [u8]> +where + Pe: ImageNtHeaders, + R: ReadRef<'data>, +{ + #[allow(unused)] + file: &'file PeFile<'data, Pe, R>, +} + +impl<'data, 'file, Pe, R> Iterator for PeComdatSectionIterator<'data, 'file, Pe, R> +where + Pe: ImageNtHeaders, + R: ReadRef<'data>, +{ + type Item = SectionIndex; + + fn next(&mut self) -> Option { + None + } +} + +impl pe::ImageDosHeader { + /// Read the DOS header. + /// + /// Also checks that the `e_magic` field in the header is valid. + pub fn parse<'data, R: ReadRef<'data>>(data: R) -> read::Result<&'data Self> { + // DOS header comes first. + let dos_header = data + .read_at::(0) + .read_error("Invalid DOS header size or alignment")?; + if dos_header.e_magic.get(LE) != pe::IMAGE_DOS_SIGNATURE { + return Err(Error("Invalid DOS magic")); + } + Ok(dos_header) + } + + /// Return the file offset of the nt_headers. + #[inline] + pub fn nt_headers_offset(&self) -> u32 { + self.e_lfanew.get(LE) + } +} + +/// Find the optional header and read the `optional_header.magic`. +/// +/// It can be useful to know this magic value before trying to +/// fully parse the NT headers. +pub fn optional_header_magic<'data, R: ReadRef<'data>>(data: R) -> Result { + let dos_header = pe::ImageDosHeader::parse(data)?; + // NT headers are at an offset specified in the DOS header. + let offset = dos_header.nt_headers_offset().into(); + // It doesn't matter which NT header type is used for the purpose + // of reading the optional header magic. + let nt_headers = data + .read_at::(offset) + .read_error("Invalid NT headers offset, size, or alignment")?; + if nt_headers.signature() != pe::IMAGE_NT_SIGNATURE { + return Err(Error("Invalid PE magic")); + } + Ok(nt_headers.optional_header().magic()) +} + +/// A trait for generic access to `ImageNtHeaders32` and `ImageNtHeaders64`. +#[allow(missing_docs)] +pub trait ImageNtHeaders: Debug + Pod { + type ImageOptionalHeader: ImageOptionalHeader; + type ImageThunkData: ImageThunkData; + + /// Return true if this type is a 64-bit header. + /// + /// This is a property of the type, not a value in the header data. + fn is_type_64(&self) -> bool; + + /// Return true if the magic field in the optional header is valid. + fn is_valid_optional_magic(&self) -> bool; + + /// Return the signature + fn signature(&self) -> u32; + + /// Return the file header. + fn file_header(&self) -> &pe::ImageFileHeader; + + /// Return the optional header. + fn optional_header(&self) -> &Self::ImageOptionalHeader; + + // Provided methods. + + /// Read the NT headers, including the data directories. + /// + /// `data` must be for the entire file. + /// + /// `offset` must be headers offset, which can be obtained from `ImageDosHeader::nt_headers_offset`. + /// It is updated to point after the optional header, which is where the section headers are located. + /// + /// Also checks that the `signature` and `magic` fields in the headers are valid. + fn parse<'data, R: ReadRef<'data>>( + data: R, + offset: &mut u64, + ) -> read::Result<(&'data Self, DataDirectories<'data>)> { + // Note that this does not include the data directories in the optional header. + let nt_headers = data + .read::(offset) + .read_error("Invalid PE headers offset or size")?; + if nt_headers.signature() != pe::IMAGE_NT_SIGNATURE { + return Err(Error("Invalid PE magic")); + } + if !nt_headers.is_valid_optional_magic() { + return Err(Error("Invalid PE optional header magic")); + } + + // Read the rest of the optional header, and then read the data directories from that. + let optional_data_size = + u64::from(nt_headers.file_header().size_of_optional_header.get(LE)) + .checked_sub(mem::size_of::() as u64) + .read_error("PE optional header size is too small")?; + let optional_data = data + .read_bytes(offset, optional_data_size) + .read_error("Invalid PE optional header size")?; + let data_directories = DataDirectories::parse( + optional_data, + nt_headers.optional_header().number_of_rva_and_sizes(), + )?; + + Ok((nt_headers, data_directories)) + } + + /// Read the section table. + /// + /// `data` must be for the entire file. + /// `offset` must be after the optional file header. + #[inline] + fn sections<'data, R: ReadRef<'data>>( + &self, + data: R, + offset: u64, + ) -> read::Result> { + SectionTable::parse(self.file_header(), data, offset) + } + + /// Read the COFF symbol table and string table. + /// + /// `data` must be the entire file data. + #[inline] + fn symbols<'data, R: ReadRef<'data>>(&self, data: R) -> read::Result> { + SymbolTable::parse(self.file_header(), data) + } +} + +/// A trait for generic access to `ImageOptionalHeader32` and `ImageOptionalHeader64`. +#[allow(missing_docs)] +pub trait ImageOptionalHeader: Debug + Pod { + // Standard fields. + fn magic(&self) -> u16; + fn major_linker_version(&self) -> u8; + fn minor_linker_version(&self) -> u8; + fn size_of_code(&self) -> u32; + fn size_of_initialized_data(&self) -> u32; + fn size_of_uninitialized_data(&self) -> u32; + fn address_of_entry_point(&self) -> u32; + fn base_of_code(&self) -> u32; + fn base_of_data(&self) -> Option; + + // NT additional fields. + fn image_base(&self) -> u64; + fn section_alignment(&self) -> u32; + fn file_alignment(&self) -> u32; + fn major_operating_system_version(&self) -> u16; + fn minor_operating_system_version(&self) -> u16; + fn major_image_version(&self) -> u16; + fn minor_image_version(&self) -> u16; + fn major_subsystem_version(&self) -> u16; + fn minor_subsystem_version(&self) -> u16; + fn win32_version_value(&self) -> u32; + fn size_of_image(&self) -> u32; + fn size_of_headers(&self) -> u32; + fn check_sum(&self) -> u32; + fn subsystem(&self) -> u16; + fn dll_characteristics(&self) -> u16; + fn size_of_stack_reserve(&self) -> u64; + fn size_of_stack_commit(&self) -> u64; + fn size_of_heap_reserve(&self) -> u64; + fn size_of_heap_commit(&self) -> u64; + fn loader_flags(&self) -> u32; + fn number_of_rva_and_sizes(&self) -> u32; +} + +impl ImageNtHeaders for pe::ImageNtHeaders32 { + type ImageOptionalHeader = pe::ImageOptionalHeader32; + type ImageThunkData = pe::ImageThunkData32; + + #[inline] + fn is_type_64(&self) -> bool { + false + } + + #[inline] + fn is_valid_optional_magic(&self) -> bool { + self.optional_header.magic.get(LE) == pe::IMAGE_NT_OPTIONAL_HDR32_MAGIC + } + + #[inline] + fn signature(&self) -> u32 { + self.signature.get(LE) + } + + #[inline] + fn file_header(&self) -> &pe::ImageFileHeader { + &self.file_header + } + + #[inline] + fn optional_header(&self) -> &Self::ImageOptionalHeader { + &self.optional_header + } +} + +impl ImageOptionalHeader for pe::ImageOptionalHeader32 { + #[inline] + fn magic(&self) -> u16 { + self.magic.get(LE) + } + + #[inline] + fn major_linker_version(&self) -> u8 { + self.major_linker_version + } + + #[inline] + fn minor_linker_version(&self) -> u8 { + self.minor_linker_version + } + + #[inline] + fn size_of_code(&self) -> u32 { + self.size_of_code.get(LE) + } + + #[inline] + fn size_of_initialized_data(&self) -> u32 { + self.size_of_initialized_data.get(LE) + } + + #[inline] + fn size_of_uninitialized_data(&self) -> u32 { + self.size_of_uninitialized_data.get(LE) + } + + #[inline] + fn address_of_entry_point(&self) -> u32 { + self.address_of_entry_point.get(LE) + } + + #[inline] + fn base_of_code(&self) -> u32 { + self.base_of_code.get(LE) + } + + #[inline] + fn base_of_data(&self) -> Option { + Some(self.base_of_data.get(LE)) + } + + #[inline] + fn image_base(&self) -> u64 { + self.image_base.get(LE).into() + } + + #[inline] + fn section_alignment(&self) -> u32 { + self.section_alignment.get(LE) + } + + #[inline] + fn file_alignment(&self) -> u32 { + self.file_alignment.get(LE) + } + + #[inline] + fn major_operating_system_version(&self) -> u16 { + self.major_operating_system_version.get(LE) + } + + #[inline] + fn minor_operating_system_version(&self) -> u16 { + self.minor_operating_system_version.get(LE) + } + + #[inline] + fn major_image_version(&self) -> u16 { + self.major_image_version.get(LE) + } + + #[inline] + fn minor_image_version(&self) -> u16 { + self.minor_image_version.get(LE) + } + + #[inline] + fn major_subsystem_version(&self) -> u16 { + self.major_subsystem_version.get(LE) + } + + #[inline] + fn minor_subsystem_version(&self) -> u16 { + self.minor_subsystem_version.get(LE) + } + + #[inline] + fn win32_version_value(&self) -> u32 { + self.win32_version_value.get(LE) + } + + #[inline] + fn size_of_image(&self) -> u32 { + self.size_of_image.get(LE) + } + + #[inline] + fn size_of_headers(&self) -> u32 { + self.size_of_headers.get(LE) + } + + #[inline] + fn check_sum(&self) -> u32 { + self.check_sum.get(LE) + } + + #[inline] + fn subsystem(&self) -> u16 { + self.subsystem.get(LE) + } + + #[inline] + fn dll_characteristics(&self) -> u16 { + self.dll_characteristics.get(LE) + } + + #[inline] + fn size_of_stack_reserve(&self) -> u64 { + self.size_of_stack_reserve.get(LE).into() + } + + #[inline] + fn size_of_stack_commit(&self) -> u64 { + self.size_of_stack_commit.get(LE).into() + } + + #[inline] + fn size_of_heap_reserve(&self) -> u64 { + self.size_of_heap_reserve.get(LE).into() + } + + #[inline] + fn size_of_heap_commit(&self) -> u64 { + self.size_of_heap_commit.get(LE).into() + } + + #[inline] + fn loader_flags(&self) -> u32 { + self.loader_flags.get(LE) + } + + #[inline] + fn number_of_rva_and_sizes(&self) -> u32 { + self.number_of_rva_and_sizes.get(LE) + } +} + +impl ImageNtHeaders for pe::ImageNtHeaders64 { + type ImageOptionalHeader = pe::ImageOptionalHeader64; + type ImageThunkData = pe::ImageThunkData64; + + #[inline] + fn is_type_64(&self) -> bool { + true + } + + #[inline] + fn is_valid_optional_magic(&self) -> bool { + self.optional_header.magic.get(LE) == pe::IMAGE_NT_OPTIONAL_HDR64_MAGIC + } + + #[inline] + fn signature(&self) -> u32 { + self.signature.get(LE) + } + + #[inline] + fn file_header(&self) -> &pe::ImageFileHeader { + &self.file_header + } + + #[inline] + fn optional_header(&self) -> &Self::ImageOptionalHeader { + &self.optional_header + } +} + +impl ImageOptionalHeader for pe::ImageOptionalHeader64 { + #[inline] + fn magic(&self) -> u16 { + self.magic.get(LE) + } + + #[inline] + fn major_linker_version(&self) -> u8 { + self.major_linker_version + } + + #[inline] + fn minor_linker_version(&self) -> u8 { + self.minor_linker_version + } + + #[inline] + fn size_of_code(&self) -> u32 { + self.size_of_code.get(LE) + } + + #[inline] + fn size_of_initialized_data(&self) -> u32 { + self.size_of_initialized_data.get(LE) + } + + #[inline] + fn size_of_uninitialized_data(&self) -> u32 { + self.size_of_uninitialized_data.get(LE) + } + + #[inline] + fn address_of_entry_point(&self) -> u32 { + self.address_of_entry_point.get(LE) + } + + #[inline] + fn base_of_code(&self) -> u32 { + self.base_of_code.get(LE) + } + + #[inline] + fn base_of_data(&self) -> Option { + None + } + + #[inline] + fn image_base(&self) -> u64 { + self.image_base.get(LE) + } + + #[inline] + fn section_alignment(&self) -> u32 { + self.section_alignment.get(LE) + } + + #[inline] + fn file_alignment(&self) -> u32 { + self.file_alignment.get(LE) + } + + #[inline] + fn major_operating_system_version(&self) -> u16 { + self.major_operating_system_version.get(LE) + } + + #[inline] + fn minor_operating_system_version(&self) -> u16 { + self.minor_operating_system_version.get(LE) + } + + #[inline] + fn major_image_version(&self) -> u16 { + self.major_image_version.get(LE) + } + + #[inline] + fn minor_image_version(&self) -> u16 { + self.minor_image_version.get(LE) + } + + #[inline] + fn major_subsystem_version(&self) -> u16 { + self.major_subsystem_version.get(LE) + } + + #[inline] + fn minor_subsystem_version(&self) -> u16 { + self.minor_subsystem_version.get(LE) + } + + #[inline] + fn win32_version_value(&self) -> u32 { + self.win32_version_value.get(LE) + } + + #[inline] + fn size_of_image(&self) -> u32 { + self.size_of_image.get(LE) + } + + #[inline] + fn size_of_headers(&self) -> u32 { + self.size_of_headers.get(LE) + } + + #[inline] + fn check_sum(&self) -> u32 { + self.check_sum.get(LE) + } + + #[inline] + fn subsystem(&self) -> u16 { + self.subsystem.get(LE) + } + + #[inline] + fn dll_characteristics(&self) -> u16 { + self.dll_characteristics.get(LE) + } + + #[inline] + fn size_of_stack_reserve(&self) -> u64 { + self.size_of_stack_reserve.get(LE) + } + + #[inline] + fn size_of_stack_commit(&self) -> u64 { + self.size_of_stack_commit.get(LE) + } + + #[inline] + fn size_of_heap_reserve(&self) -> u64 { + self.size_of_heap_reserve.get(LE) + } + + #[inline] + fn size_of_heap_commit(&self) -> u64 { + self.size_of_heap_commit.get(LE) + } + + #[inline] + fn loader_flags(&self) -> u32 { + self.loader_flags.get(LE) + } + + #[inline] + fn number_of_rva_and_sizes(&self) -> u32 { + self.number_of_rva_and_sizes.get(LE) + } +} diff --git a/crux-mir/lib/object/src/read/pe/import.rs b/crux-mir/lib/object/src/read/pe/import.rs new file mode 100644 index 000000000..809a96286 --- /dev/null +++ b/crux-mir/lib/object/src/read/pe/import.rs @@ -0,0 +1,218 @@ +use core::fmt::Debug; +use core::mem; + +use crate::read::{Bytes, ReadError, Result}; +use crate::{pe, LittleEndian as LE, Pod, U16Bytes}; + +use super::ImageNtHeaders; + +/// Information for parsing a PE import table. +#[derive(Debug, Clone)] +pub struct ImportTable<'data> { + section_data: Bytes<'data>, + section_address: u32, + import_address: u32, +} + +impl<'data> ImportTable<'data> { + /// Create a new import table parser. + /// + /// The import descriptors start at `import_address`. + /// The size declared in the `IMAGE_DIRECTORY_ENTRY_IMPORT` data directory is + /// ignored by the Windows loader, and so descriptors will be parsed until a null entry. + /// + /// `section_data` should be from the section containing `import_address`, and + /// `section_address` should be the address of that section. Pointers within the + /// descriptors and thunks may point to anywhere within the section data. + pub fn new(section_data: &'data [u8], section_address: u32, import_address: u32) -> Self { + ImportTable { + section_data: Bytes(section_data), + section_address, + import_address, + } + } + + /// Return an iterator for the import descriptors. + pub fn descriptors(&self) -> Result> { + let offset = self.import_address.wrapping_sub(self.section_address); + let mut data = self.section_data; + data.skip(offset as usize) + .read_error("Invalid PE import descriptor address")?; + Ok(ImportDescriptorIterator { data }) + } + + /// Return a library name given its address. + /// + /// This address may be from [`pe::ImageImportDescriptor::name`]. + pub fn name(&self, address: u32) -> Result<&'data [u8]> { + self.section_data + .read_string_at(address.wrapping_sub(self.section_address) as usize) + .read_error("Invalid PE import descriptor name") + } + + /// Return a list of thunks given its address. + /// + /// This address may be from [`pe::ImageImportDescriptor::original_first_thunk`] + /// or [`pe::ImageImportDescriptor::first_thunk`]. + pub fn thunks(&self, address: u32) -> Result> { + let offset = address.wrapping_sub(self.section_address); + let mut data = self.section_data; + data.skip(offset as usize) + .read_error("Invalid PE import thunk table address")?; + Ok(ImportThunkList { data }) + } + + /// Parse a thunk. + pub fn import(&self, thunk: Pe::ImageThunkData) -> Result> { + if thunk.is_ordinal() { + Ok(Import::Ordinal(thunk.ordinal())) + } else { + let (hint, name) = self.hint_name(thunk.address())?; + Ok(Import::Name(hint, name)) + } + } + + /// Return the hint and name at the given address. + /// + /// This address may be from [`pe::ImageThunkData32`] or [`pe::ImageThunkData64`]. + /// + /// The hint is an index into the export name pointer table in the target library. + pub fn hint_name(&self, address: u32) -> Result<(u16, &'data [u8])> { + let offset = address.wrapping_sub(self.section_address); + let mut data = self.section_data; + data.skip(offset as usize) + .read_error("Invalid PE import thunk address")?; + let hint = data + .read::>() + .read_error("Missing PE import thunk hint")? + .get(LE); + let name = data + .read_string() + .read_error("Missing PE import thunk name")?; + Ok((hint, name)) + } +} + +/// A fallible iterator for the descriptors in the import data directory. +#[derive(Debug, Clone)] +pub struct ImportDescriptorIterator<'data> { + data: Bytes<'data>, +} + +impl<'data> ImportDescriptorIterator<'data> { + /// Return the next descriptor. + /// + /// Returns `Ok(None)` when a null descriptor is found. + pub fn next(&mut self) -> Result> { + let import_desc = self + .data + .read::() + .read_error("Missing PE null import descriptor")?; + if import_desc.is_null() { + Ok(None) + } else { + Ok(Some(import_desc)) + } + } +} + +/// A list of import thunks. +/// +/// These may be in the import lookup table, or the import address table. +#[derive(Debug, Clone)] +pub struct ImportThunkList<'data> { + data: Bytes<'data>, +} + +impl<'data> ImportThunkList<'data> { + /// Get the thunk at the given index. + pub fn get(&self, index: usize) -> Result { + let thunk = self + .data + .read_at(index * mem::size_of::()) + .read_error("Invalid PE import thunk index")?; + Ok(*thunk) + } + + /// Return the first thunk in the list, and update `self` to point after it. + /// + /// Returns `Ok(None)` when a null thunk is found. + pub fn next(&mut self) -> Result> { + let thunk = self + .data + .read::() + .read_error("Missing PE null import thunk")?; + if thunk.address() == 0 { + Ok(None) + } else { + Ok(Some(*thunk)) + } + } +} + +/// A parsed import thunk. +#[derive(Debug, Clone, Copy)] +pub enum Import<'data> { + /// Import by ordinal. + Ordinal(u16), + /// Import by name. + /// + /// Includes a hint for the index into the export name pointer table in the target library. + Name(u16, &'data [u8]), +} + +/// A trait for generic access to [`pe::ImageThunkData32`] and [`pe::ImageThunkData64`]. +#[allow(missing_docs)] +pub trait ImageThunkData: Debug + Pod { + /// Return the raw thunk value. + fn raw(self) -> u64; + + /// Returns true if the ordinal flag is set. + fn is_ordinal(self) -> bool; + + /// Return the ordinal portion of the thunk. + /// + /// Does not check the ordinal flag. + fn ordinal(self) -> u16; + + /// Return the RVA portion of the thunk. + /// + /// Does not check the ordinal flag. + fn address(self) -> u32; +} + +impl ImageThunkData for pe::ImageThunkData64 { + fn raw(self) -> u64 { + self.0.get(LE) + } + + fn is_ordinal(self) -> bool { + self.0.get(LE) & pe::IMAGE_ORDINAL_FLAG64 != 0 + } + + fn ordinal(self) -> u16 { + self.0.get(LE) as u16 + } + + fn address(self) -> u32 { + self.0.get(LE) as u32 & 0x7fff_ffff + } +} + +impl ImageThunkData for pe::ImageThunkData32 { + fn raw(self) -> u64 { + self.0.get(LE).into() + } + + fn is_ordinal(self) -> bool { + self.0.get(LE) & pe::IMAGE_ORDINAL_FLAG32 != 0 + } + + fn ordinal(self) -> u16 { + self.0.get(LE) as u16 + } + + fn address(self) -> u32 { + self.0.get(LE) & 0x7fff_ffff + } +} diff --git a/crux-mir/lib/object/src/read/pe/mod.rs b/crux-mir/lib/object/src/read/pe/mod.rs new file mode 100644 index 000000000..2b7cc5d7a --- /dev/null +++ b/crux-mir/lib/object/src/read/pe/mod.rs @@ -0,0 +1,34 @@ +//! Support for reading PE files. +//! +//! Defines traits to abstract over the difference between PE32/PE32+, +//! and implements read functionality in terms of these traits. +//! +//! This module reuses some of the COFF functionality. +//! +//! Also provides `PeFile` and related types which implement the `Object` trait. + +mod file; +pub use file::*; + +mod section; +pub use section::*; + +mod data_directory; +pub use data_directory::*; + +mod export; +pub use export::*; + +mod import; +pub use import::*; + +mod relocation; +pub use relocation::*; + +mod resource; +pub use resource::*; + +mod rich; +pub use rich::*; + +pub use super::coff::{SectionTable, SymbolTable}; diff --git a/crux-mir/lib/object/src/read/pe/relocation.rs b/crux-mir/lib/object/src/read/pe/relocation.rs new file mode 100644 index 000000000..06215bd1a --- /dev/null +++ b/crux-mir/lib/object/src/read/pe/relocation.rs @@ -0,0 +1,90 @@ +use core::slice; + +use crate::endian::{LittleEndian as LE, U16}; +use crate::pe; +use crate::read::{Bytes, Error, ReadError, Result}; + +/// An iterator over the relocation blocks in the `.reloc` section of a PE file. +#[derive(Debug, Default, Clone, Copy)] +pub struct RelocationBlockIterator<'data> { + data: Bytes<'data>, +} + +impl<'data> RelocationBlockIterator<'data> { + /// Construct a new iterator from the data of the `.reloc` section. + pub fn new(data: &'data [u8]) -> Self { + RelocationBlockIterator { data: Bytes(data) } + } + + /// Read the next relocation page. + pub fn next(&mut self) -> Result>> { + if self.data.is_empty() { + return Ok(None); + } + let header = self + .data + .read::() + .read_error("Invalid PE reloc section size")?; + let virtual_address = header.virtual_address.get(LE); + let size = header.size_of_block.get(LE); + if size <= 8 || size & 3 != 0 { + return Err(Error("Invalid PE reloc block size")); + } + let count = (size - 8) / 2; + let relocs = self + .data + .read_slice::>(count as usize) + .read_error("Invalid PE reloc block size")? + .iter(); + Ok(Some(RelocationIterator { + virtual_address, + size, + relocs, + })) + } +} + +/// An iterator of the relocations in a block in the `.reloc` section of a PE file. +#[derive(Debug, Clone)] +pub struct RelocationIterator<'data> { + virtual_address: u32, + size: u32, + relocs: slice::Iter<'data, U16>, +} + +impl<'data> RelocationIterator<'data> { + /// Return the virtual address of the page that this block of relocations applies to. + pub fn virtual_address(&self) -> u32 { + self.virtual_address + } + + /// Return the size in bytes of this block of relocations. + pub fn size(&self) -> u32 { + self.size + } +} + +impl<'data> Iterator for RelocationIterator<'data> { + type Item = Relocation; + + fn next(&mut self) -> Option { + loop { + let reloc = self.relocs.next()?.get(LE); + if reloc != 0 { + return Some(Relocation { + virtual_address: self.virtual_address.wrapping_add((reloc & 0xfff) as u32), + typ: reloc >> 12, + }); + } + } + } +} + +/// A relocation in the `.reloc` section of a PE file. +#[derive(Debug, Default, Clone, Copy)] +pub struct Relocation { + /// The virtual address of the relocation. + pub virtual_address: u32, + /// One of the `pe::IMAGE_REL_BASED_*` constants. + pub typ: u16, +} diff --git a/crux-mir/lib/object/src/read/pe/resource.rs b/crux-mir/lib/object/src/read/pe/resource.rs new file mode 100644 index 000000000..bfbb609f5 --- /dev/null +++ b/crux-mir/lib/object/src/read/pe/resource.rs @@ -0,0 +1,195 @@ +use alloc::string::String; + +use crate::read::{ReadError, ReadRef, Result}; +use crate::{pe, LittleEndian as LE, U16}; + +/// The `.rsrc` section of a PE file. +#[derive(Debug, Clone, Copy)] +pub struct ResourceDirectory<'data> { + data: &'data [u8], +} + +impl<'data> ResourceDirectory<'data> { + /// Construct from the data of the `.rsrc` section. + pub fn new(data: &'data [u8]) -> Self { + ResourceDirectory { data } + } + + /// Parses the root resource directory. + pub fn root(&self) -> Result> { + ResourceDirectoryTable::parse(&self.data, 0) + } +} + +/// A table of resource entries. +#[derive(Debug, Clone)] +pub struct ResourceDirectoryTable<'data> { + /// The table header. + pub header: &'data pe::ImageResourceDirectory, + /// The table entries. + pub entries: &'data [pe::ImageResourceDirectoryEntry], +} + +impl<'data> ResourceDirectoryTable<'data> { + fn parse(data: &'data [u8], offset: u32) -> Result { + let mut offset = u64::from(offset); + let header = data + .read::(&mut offset) + .read_error("Invalid resource table header")?; + let entries_count = header.number_of_id_entries.get(LE) as usize + + header.number_of_named_entries.get(LE) as usize; + let entries = data + .read_slice::(&mut offset, entries_count) + .read_error("Invalid resource table entries")?; + Ok(Self { header, entries }) + } +} + +impl pe::ImageResourceDirectoryEntry { + /// Returns true if the entry has a name, rather than an ID. + pub fn has_name(&self) -> bool { + self.name_or_id.get(LE) & pe::IMAGE_RESOURCE_NAME_IS_STRING != 0 + } + + /// Returns the section offset of the name. + /// + /// Valid if `has_name()` returns true. + fn name(&self) -> ResourceName { + let offset = self.name_or_id.get(LE) & !pe::IMAGE_RESOURCE_NAME_IS_STRING; + ResourceName { offset } + } + + /// Returns the ID. + /// + /// Valid if `has_string_name()` returns false. + fn id(&self) -> u16 { + (self.name_or_id.get(LE) & 0x0000_FFFF) as u16 + } + + /// Returns the entry name + pub fn name_or_id(&self) -> ResourceNameOrId { + if self.has_name() { + ResourceNameOrId::Name(self.name()) + } else { + ResourceNameOrId::Id(self.id()) + } + } + + /// Returns true if the entry is a subtable. + pub fn is_table(&self) -> bool { + self.offset_to_data_or_directory.get(LE) & pe::IMAGE_RESOURCE_DATA_IS_DIRECTORY != 0 + } + + /// Returns the section offset of the associated table or data. + pub fn data_offset(&self) -> u32 { + self.offset_to_data_or_directory.get(LE) & !pe::IMAGE_RESOURCE_DATA_IS_DIRECTORY + } + + /// Returns the data associated to this directory entry. + pub fn data<'data>( + &self, + section: ResourceDirectory<'data>, + ) -> Result> { + if self.is_table() { + ResourceDirectoryTable::parse(section.data, self.data_offset()) + .map(|t| ResourceDirectoryEntryData::Table(t)) + } else { + section + .data + .read_at::(self.data_offset().into()) + .read_error("Invalid resource entry") + .map(|d| ResourceDirectoryEntryData::Data(d)) + } + } +} + +/// Data associated with a resource directory entry. +#[derive(Debug, Clone)] +pub enum ResourceDirectoryEntryData<'data> { + /// A subtable entry. + Table(ResourceDirectoryTable<'data>), + /// A resource data entry. + Data(&'data pe::ImageResourceDataEntry), +} + +impl<'data> ResourceDirectoryEntryData<'data> { + /// Converts to an option of table. + /// + /// Helper for iterator filtering. + pub fn table(self) -> Option> { + match self { + Self::Table(dir) => Some(dir), + _ => None, + } + } + + /// Converts to an option of data entry. + /// + /// Helper for iterator filtering. + pub fn data(self) -> Option<&'data pe::ImageResourceDataEntry> { + match self { + Self::Data(rsc) => Some(rsc), + _ => None, + } + } +} + +/// A resource name. +#[derive(Debug, Clone, Copy)] +pub struct ResourceName { + offset: u32, +} + +impl ResourceName { + /// Converts to a `String`. + pub fn to_string_lossy(&self, directory: ResourceDirectory) -> Result { + let d = self.data(directory)?; + Ok(String::from_utf16_lossy(d)) + } + + /// Returns the string unicode buffer. + pub fn data<'data>(&self, directory: ResourceDirectory<'data>) -> Result<&'data [u16]> { + let mut offset = u64::from(self.offset); + let len = directory + .data + .read::>(&mut offset) + .read_error("Invalid resource name offset")?; + directory + .data + .read_slice::(&mut offset, len.get(LE).into()) + .read_error("Invalid resource name length") + } +} + +/// A resource name or ID. +/// +/// Can be either a string or a numeric ID. +#[derive(Debug)] +pub enum ResourceNameOrId { + /// A resource name. + Name(ResourceName), + /// A resource ID. + Id(u16), +} + +impl ResourceNameOrId { + /// Converts to an option of name. + /// + /// Helper for iterator filtering. + pub fn name(self) -> Option { + match self { + Self::Name(name) => Some(name), + _ => None, + } + } + + /// Converts to an option of ID. + /// + /// Helper for iterator filtering. + pub fn id(self) -> Option { + match self { + Self::Id(id) => Some(id), + _ => None, + } + } +} diff --git a/crux-mir/lib/object/src/read/pe/rich.rs b/crux-mir/lib/object/src/read/pe/rich.rs new file mode 100644 index 000000000..687dfc995 --- /dev/null +++ b/crux-mir/lib/object/src/read/pe/rich.rs @@ -0,0 +1,91 @@ +//! PE rich header handling + +use core::mem; + +use crate::pod::bytes_of_slice; +use crate::read::Bytes; +use crate::{pe, LittleEndian as LE, ReadRef, U32}; + +/// Parsed information about a Rich Header. +#[derive(Debug, Clone, Copy)] +pub struct RichHeaderInfo<'data> { + /// The offset at which the rich header starts. + pub offset: usize, + /// The length (in bytes) of the rich header. + /// + /// This includes the payload, but also the 16-byte start sequence and the + /// 8-byte final "Rich" and XOR key. + pub length: usize, + /// The XOR key used to mask the rich header. + /// + /// Unless the file has been tampered with, it should be equal to a checksum + /// of the file header. + pub xor_key: u32, + masked_entries: &'data [pe::MaskedRichHeaderEntry], +} + +/// A PE rich header entry after it has been unmasked. +/// +/// See [`pe::MaskedRichHeaderEntry`]. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct RichHeaderEntry { + /// ID of the component. + pub comp_id: u32, + /// Number of times this component has been used when building this PE. + pub count: u32, +} + +impl<'data> RichHeaderInfo<'data> { + /// Try to locate a rich header and its entries in the current PE file. + pub fn parse>(data: R, nt_header_offset: u64) -> Option { + // Locate the rich header, if any. + // It ends with the "Rich" string and an XOR key, before the NT header. + let data = data.read_bytes_at(0, nt_header_offset).map(Bytes).ok()?; + let end_marker_offset = memmem(data.0, b"Rich", 4)?; + let xor_key = *data.read_at::>(end_marker_offset + 4).ok()?; + + // It starts at the masked "DanS" string and 3 masked zeroes. + let masked_start_marker = U32::new(LE, 0x536e_6144 ^ xor_key.get(LE)); + let start_header = [masked_start_marker, xor_key, xor_key, xor_key]; + let start_sequence = bytes_of_slice(&start_header); + let start_marker_offset = memmem(&data.0[..end_marker_offset], start_sequence, 4)?; + + // Extract the items between the markers. + let items_offset = start_marker_offset + start_sequence.len(); + let items_len = end_marker_offset - items_offset; + let item_count = items_len / mem::size_of::(); + let items = data.read_slice_at(items_offset, item_count).ok()?; + Some(RichHeaderInfo { + offset: start_marker_offset, + // Includes "Rich" marker and the XOR key. + length: end_marker_offset - start_marker_offset + 8, + xor_key: xor_key.get(LE), + masked_entries: items, + }) + } + + /// Returns an iterator over the unmasked entries. + pub fn unmasked_entries(&self) -> impl Iterator + 'data { + let xor_key = self.xor_key; + self.masked_entries + .iter() + .map(move |entry| RichHeaderEntry { + comp_id: entry.masked_comp_id.get(LE) ^ xor_key, + count: entry.masked_count.get(LE) ^ xor_key, + }) + } +} + +/// Find the offset of the first occurence of needle in the data. +/// +/// The offset must have the given alignment. +fn memmem(data: &[u8], needle: &[u8], align: usize) -> Option { + let mut offset = 0; + loop { + if data.get(offset..)?.get(..needle.len())? == needle { + return Some(offset); + } + offset += align; + } +} diff --git a/crux-mir/lib/object/src/read/pe/section.rs b/crux-mir/lib/object/src/read/pe/section.rs new file mode 100644 index 000000000..439d42dac --- /dev/null +++ b/crux-mir/lib/object/src/read/pe/section.rs @@ -0,0 +1,436 @@ +use core::marker::PhantomData; +use core::{cmp, iter, slice, str}; + +use crate::endian::LittleEndian as LE; +use crate::pe; +use crate::pe::ImageSectionHeader; +use crate::read::{ + self, CompressedData, CompressedFileRange, ObjectSection, ObjectSegment, ReadError, ReadRef, + Relocation, Result, SectionFlags, SectionIndex, SectionKind, SegmentFlags, +}; + +use super::{ImageNtHeaders, PeFile, SectionTable}; + +/// An iterator over the loadable sections of a `PeFile32`. +pub type PeSegmentIterator32<'data, 'file, R = &'data [u8]> = + PeSegmentIterator<'data, 'file, pe::ImageNtHeaders32, R>; +/// An iterator over the loadable sections of a `PeFile64`. +pub type PeSegmentIterator64<'data, 'file, R = &'data [u8]> = + PeSegmentIterator<'data, 'file, pe::ImageNtHeaders64, R>; + +/// An iterator over the loadable sections of a `PeFile`. +#[derive(Debug)] +pub struct PeSegmentIterator<'data, 'file, Pe, R = &'data [u8]> +where + Pe: ImageNtHeaders, + R: ReadRef<'data>, +{ + pub(super) file: &'file PeFile<'data, Pe, R>, + pub(super) iter: slice::Iter<'data, pe::ImageSectionHeader>, +} + +impl<'data, 'file, Pe, R> Iterator for PeSegmentIterator<'data, 'file, Pe, R> +where + Pe: ImageNtHeaders, + R: ReadRef<'data>, +{ + type Item = PeSegment<'data, 'file, Pe, R>; + + fn next(&mut self) -> Option { + self.iter.next().map(|section| PeSegment { + file: self.file, + section, + }) + } +} + +/// A loadable section of a `PeFile32`. +pub type PeSegment32<'data, 'file, R = &'data [u8]> = + PeSegment<'data, 'file, pe::ImageNtHeaders32, R>; +/// A loadable section of a `PeFile64`. +pub type PeSegment64<'data, 'file, R = &'data [u8]> = + PeSegment<'data, 'file, pe::ImageNtHeaders64, R>; + +/// A loadable section of a `PeFile`. +#[derive(Debug)] +pub struct PeSegment<'data, 'file, Pe, R = &'data [u8]> +where + Pe: ImageNtHeaders, + R: ReadRef<'data>, +{ + file: &'file PeFile<'data, Pe, R>, + section: &'data pe::ImageSectionHeader, +} + +impl<'data, 'file, Pe, R> read::private::Sealed for PeSegment<'data, 'file, Pe, R> +where + Pe: ImageNtHeaders, + R: ReadRef<'data>, +{ +} + +impl<'data, 'file, Pe, R> ObjectSegment<'data> for PeSegment<'data, 'file, Pe, R> +where + Pe: ImageNtHeaders, + R: ReadRef<'data>, +{ + #[inline] + fn address(&self) -> u64 { + u64::from(self.section.virtual_address.get(LE)).wrapping_add(self.file.common.image_base) + } + + #[inline] + fn size(&self) -> u64 { + u64::from(self.section.virtual_size.get(LE)) + } + + #[inline] + fn align(&self) -> u64 { + self.file.section_alignment() + } + + #[inline] + fn file_range(&self) -> (u64, u64) { + let (offset, size) = self.section.pe_file_range(); + (u64::from(offset), u64::from(size)) + } + + fn data(&self) -> Result<&'data [u8]> { + self.section.pe_data(self.file.data) + } + + fn data_range(&self, address: u64, size: u64) -> Result> { + Ok(read::util::data_range( + self.data()?, + self.address(), + address, + size, + )) + } + + #[inline] + fn name_bytes(&self) -> Result> { + self.section + .name(self.file.common.symbols.strings()) + .map(Some) + } + + #[inline] + fn name(&self) -> Result> { + let name = self.section.name(self.file.common.symbols.strings())?; + Ok(Some( + str::from_utf8(name) + .ok() + .read_error("Non UTF-8 PE section name")?, + )) + } + + #[inline] + fn flags(&self) -> SegmentFlags { + let characteristics = self.section.characteristics.get(LE); + SegmentFlags::Coff { characteristics } + } +} + +/// An iterator over the sections of a `PeFile32`. +pub type PeSectionIterator32<'data, 'file, R = &'data [u8]> = + PeSectionIterator<'data, 'file, pe::ImageNtHeaders32, R>; +/// An iterator over the sections of a `PeFile64`. +pub type PeSectionIterator64<'data, 'file, R = &'data [u8]> = + PeSectionIterator<'data, 'file, pe::ImageNtHeaders64, R>; + +/// An iterator over the sections of a `PeFile`. +#[derive(Debug)] +pub struct PeSectionIterator<'data, 'file, Pe, R = &'data [u8]> +where + 'data: 'file, + Pe: ImageNtHeaders, + R: ReadRef<'data>, +{ + pub(super) file: &'file PeFile<'data, Pe, R>, + pub(super) iter: iter::Enumerate>, +} + +impl<'data, 'file, Pe, R> Iterator for PeSectionIterator<'data, 'file, Pe, R> +where + Pe: ImageNtHeaders, + R: ReadRef<'data>, +{ + type Item = PeSection<'data, 'file, Pe, R>; + + fn next(&mut self) -> Option { + self.iter.next().map(|(index, section)| PeSection { + file: self.file, + index: SectionIndex(index + 1), + section, + }) + } +} + +/// A section of a `PeFile32`. +pub type PeSection32<'data, 'file, R = &'data [u8]> = + PeSection<'data, 'file, pe::ImageNtHeaders32, R>; +/// A section of a `PeFile64`. +pub type PeSection64<'data, 'file, R = &'data [u8]> = + PeSection<'data, 'file, pe::ImageNtHeaders64, R>; + +/// A section of a `PeFile`. +#[derive(Debug)] +pub struct PeSection<'data, 'file, Pe, R = &'data [u8]> +where + 'data: 'file, + Pe: ImageNtHeaders, + R: ReadRef<'data>, +{ + pub(super) file: &'file PeFile<'data, Pe, R>, + pub(super) index: SectionIndex, + pub(super) section: &'data pe::ImageSectionHeader, +} + +impl<'data, 'file, Pe, R> read::private::Sealed for PeSection<'data, 'file, Pe, R> +where + Pe: ImageNtHeaders, + R: ReadRef<'data>, +{ +} + +impl<'data, 'file, Pe, R> ObjectSection<'data> for PeSection<'data, 'file, Pe, R> +where + Pe: ImageNtHeaders, + R: ReadRef<'data>, +{ + type RelocationIterator = PeRelocationIterator<'data, 'file, R>; + + #[inline] + fn index(&self) -> SectionIndex { + self.index + } + + #[inline] + fn address(&self) -> u64 { + u64::from(self.section.virtual_address.get(LE)).wrapping_add(self.file.common.image_base) + } + + #[inline] + fn size(&self) -> u64 { + u64::from(self.section.virtual_size.get(LE)) + } + + #[inline] + fn align(&self) -> u64 { + self.file.section_alignment() + } + + #[inline] + fn file_range(&self) -> Option<(u64, u64)> { + let (offset, size) = self.section.pe_file_range(); + if size == 0 { + None + } else { + Some((u64::from(offset), u64::from(size))) + } + } + + fn data(&self) -> Result<&'data [u8]> { + self.section.pe_data(self.file.data) + } + + fn data_range(&self, address: u64, size: u64) -> Result> { + Ok(read::util::data_range( + self.data()?, + self.address(), + address, + size, + )) + } + + #[inline] + fn compressed_file_range(&self) -> Result { + Ok(CompressedFileRange::none(self.file_range())) + } + + #[inline] + fn compressed_data(&self) -> Result> { + self.data().map(CompressedData::none) + } + + #[inline] + fn name_bytes(&self) -> Result<&[u8]> { + self.section.name(self.file.common.symbols.strings()) + } + + #[inline] + fn name(&self) -> Result<&str> { + let name = self.name_bytes()?; + str::from_utf8(name) + .ok() + .read_error("Non UTF-8 PE section name") + } + + #[inline] + fn segment_name_bytes(&self) -> Result> { + Ok(None) + } + + #[inline] + fn segment_name(&self) -> Result> { + Ok(None) + } + + #[inline] + fn kind(&self) -> SectionKind { + self.section.kind() + } + + fn relocations(&self) -> PeRelocationIterator<'data, 'file, R> { + PeRelocationIterator(PhantomData) + } + + fn flags(&self) -> SectionFlags { + SectionFlags::Coff { + characteristics: self.section.characteristics.get(LE), + } + } +} + +impl<'data> SectionTable<'data> { + /// Return the file offset of the given virtual address, and the size up + /// to the end of the section containing it. + /// + /// Returns `None` if no section contains the address. + pub fn pe_file_range_at(&self, va: u32) -> Option<(u32, u32)> { + self.iter().find_map(|section| section.pe_file_range_at(va)) + } + + /// Return the data starting at the given virtual address, up to the end of the + /// section containing it. + /// + /// Ignores sections with invalid data. + /// + /// Returns `None` if no section contains the address. + pub fn pe_data_at>(&self, data: R, va: u32) -> Option<&'data [u8]> { + self.iter().find_map(|section| section.pe_data_at(data, va)) + } + + /// Return the data of the section that contains the given virtual address in a PE file. + /// + /// Also returns the virtual address of that section. + /// + /// Ignores sections with invalid data. + pub fn pe_data_containing>( + &self, + data: R, + va: u32, + ) -> Option<(&'data [u8], u32)> { + self.iter() + .find_map(|section| section.pe_data_containing(data, va)) + } + + /// Return the section that contains a given virtual address. + pub fn section_containing(&self, va: u32) -> Option<&'data ImageSectionHeader> { + self.iter().find(|section| section.contains_rva(va)) + } +} + +impl pe::ImageSectionHeader { + /// Return the offset and size of the section in a PE file. + /// + /// The size of the range will be the minimum of the file size and virtual size. + pub fn pe_file_range(&self) -> (u32, u32) { + // Pointer and size will be zero for uninitialized data; we don't need to validate this. + let offset = self.pointer_to_raw_data.get(LE); + let size = cmp::min(self.virtual_size.get(LE), self.size_of_raw_data.get(LE)); + (offset, size) + } + + /// Return the file offset of the given virtual address, and the remaining size up + /// to the end of the section. + /// + /// Returns `None` if the section does not contain the address. + pub fn pe_file_range_at(&self, va: u32) -> Option<(u32, u32)> { + let section_va = self.virtual_address.get(LE); + let offset = va.checked_sub(section_va)?; + let (section_offset, section_size) = self.pe_file_range(); + // Address must be within section (and not at its end). + if offset < section_size { + Some((section_offset.checked_add(offset)?, section_size - offset)) + } else { + None + } + } + + /// Return the virtual address and size of the section. + pub fn pe_address_range(&self) -> (u32, u32) { + (self.virtual_address.get(LE), self.virtual_size.get(LE)) + } + + /// Return the section data in a PE file. + /// + /// The length of the data will be the minimum of the file size and virtual size. + pub fn pe_data<'data, R: ReadRef<'data>>(&self, data: R) -> Result<&'data [u8]> { + let (offset, size) = self.pe_file_range(); + data.read_bytes_at(offset.into(), size.into()) + .read_error("Invalid PE section offset or size") + } + + /// Return the data starting at the given virtual address, up to the end of the + /// section. + /// + /// Ignores sections with invalid data. + /// + /// Returns `None` if the section does not contain the address. + pub fn pe_data_at<'data, R: ReadRef<'data>>(&self, data: R, va: u32) -> Option<&'data [u8]> { + let (offset, size) = self.pe_file_range_at(va)?; + data.read_bytes_at(offset.into(), size.into()).ok() + } + + /// Tests whether a given RVA is part of this section + pub fn contains_rva(&self, va: u32) -> bool { + let section_va = self.virtual_address.get(LE); + match va.checked_sub(section_va) { + None => false, + Some(offset) => { + // Address must be within section (and not at its end). + offset < self.virtual_size.get(LE) + } + } + } + + /// Return the section data if it contains the given virtual address. + /// + /// Also returns the virtual address of that section. + /// + /// Ignores sections with invalid data. + pub fn pe_data_containing<'data, R: ReadRef<'data>>( + &self, + data: R, + va: u32, + ) -> Option<(&'data [u8], u32)> { + let section_va = self.virtual_address.get(LE); + let offset = va.checked_sub(section_va)?; + let (section_offset, section_size) = self.pe_file_range(); + // Address must be within section (and not at its end). + if offset < section_size { + let section_data = data + .read_bytes_at(section_offset.into(), section_size.into()) + .ok()?; + Some((section_data, section_va)) + } else { + None + } + } +} + +/// An iterator over the relocations in an `PeSection`. +#[derive(Debug)] +pub struct PeRelocationIterator<'data, 'file, R = &'data [u8]>( + PhantomData<(&'data (), &'file (), R)>, +); + +impl<'data, 'file, R> Iterator for PeRelocationIterator<'data, 'file, R> { + type Item = (u64, Relocation); + + fn next(&mut self) -> Option { + None + } +} diff --git a/crux-mir/lib/object/src/read/read_cache.rs b/crux-mir/lib/object/src/read/read_cache.rs new file mode 100644 index 000000000..19a98a44d --- /dev/null +++ b/crux-mir/lib/object/src/read/read_cache.rs @@ -0,0 +1,185 @@ +use core::ops::Range; +use std::boxed::Box; +use std::cell::RefCell; +use std::collections::hash_map::Entry; +use std::collections::HashMap; +use std::convert::TryInto; +use std::io::{Read, Seek, SeekFrom}; +use std::mem; +use std::vec::Vec; + +use crate::read::ReadRef; + +/// An implementation of `ReadRef` for data in a stream that implements +/// `Read + Seek`. +/// +/// Contains a cache of read-only blocks of data, allowing references to +/// them to be returned. Entries in the cache are never removed. +/// Entries are keyed on the offset and size of the read. +/// Currently overlapping reads are considered separate reads. +#[derive(Debug)] +pub struct ReadCache { + cache: RefCell>, +} + +#[derive(Debug)] +struct ReadCacheInternal { + read: R, + bufs: HashMap<(u64, u64), Box<[u8]>>, + strings: HashMap<(u64, u8), Box<[u8]>>, +} + +impl ReadCache { + /// Create an empty `ReadCache` for the given stream. + pub fn new(read: R) -> Self { + ReadCache { + cache: RefCell::new(ReadCacheInternal { + read, + bufs: HashMap::new(), + strings: HashMap::new(), + }), + } + } + + /// Return an implementation of `ReadRef` that restricts reads + /// to the given range of the stream. + pub fn range(&self, offset: u64, size: u64) -> ReadCacheRange<'_, R> { + ReadCacheRange { + r: self, + offset, + size, + } + } + + /// Free buffers used by the cache. + pub fn clear(&mut self) { + self.cache.borrow_mut().bufs.clear(); + } + + /// Unwrap this `ReadCache`, returning the underlying reader. + pub fn into_inner(self) -> R { + self.cache.into_inner().read + } +} + +impl<'a, R: Read + Seek> ReadRef<'a> for &'a ReadCache { + fn len(self) -> Result { + let cache = &mut *self.cache.borrow_mut(); + cache.read.seek(SeekFrom::End(0)).map_err(|_| ()) + } + + fn read_bytes_at(self, offset: u64, size: u64) -> Result<&'a [u8], ()> { + if size == 0 { + return Ok(&[]); + } + let cache = &mut *self.cache.borrow_mut(); + let buf = match cache.bufs.entry((offset, size)) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(entry) => { + let size = size.try_into().map_err(|_| ())?; + cache + .read + .seek(SeekFrom::Start(offset as u64)) + .map_err(|_| ())?; + let mut bytes = vec![0; size].into_boxed_slice(); + cache.read.read_exact(&mut bytes).map_err(|_| ())?; + entry.insert(bytes) + } + }; + // Extend the lifetime to that of self. + // This is OK because we never mutate or remove entries. + Ok(unsafe { mem::transmute::<&[u8], &[u8]>(buf) }) + } + + fn read_bytes_at_until(self, range: Range, delimiter: u8) -> Result<&'a [u8], ()> { + let cache = &mut *self.cache.borrow_mut(); + let buf = match cache.strings.entry((range.start, delimiter)) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(entry) => { + cache + .read + .seek(SeekFrom::Start(range.start)) + .map_err(|_| ())?; + + let max_check: usize = (range.end - range.start).try_into().map_err(|_| ())?; + // Strings should be relatively small. + // TODO: make this configurable? + let max_check = max_check.min(4096); + + let mut bytes = Vec::new(); + let mut checked = 0; + loop { + bytes.resize((checked + 256).min(max_check), 0); + let read = cache.read.read(&mut bytes[checked..]).map_err(|_| ())?; + if read == 0 { + return Err(()); + } + if let Some(len) = memchr::memchr(delimiter, &bytes[checked..][..read]) { + bytes.truncate(checked + len); + break entry.insert(bytes.into_boxed_slice()); + } + checked += read; + if checked >= max_check { + return Err(()); + } + } + } + }; + // Extend the lifetime to that of self. + // This is OK because we never mutate or remove entries. + Ok(unsafe { mem::transmute::<&[u8], &[u8]>(buf) }) + } +} + +/// An implementation of `ReadRef` for a range of data in a stream that +/// implements `Read + Seek`. +/// +/// Shares an underlying `ReadCache` with a lifetime of `'a`. +#[derive(Debug)] +pub struct ReadCacheRange<'a, R: Read + Seek> { + r: &'a ReadCache, + offset: u64, + size: u64, +} + +impl<'a, R: Read + Seek> Clone for ReadCacheRange<'a, R> { + fn clone(&self) -> Self { + Self { + r: self.r, + offset: self.offset, + size: self.size, + } + } +} + +impl<'a, R: Read + Seek> Copy for ReadCacheRange<'a, R> {} + +impl<'a, R: Read + Seek> ReadRef<'a> for ReadCacheRange<'a, R> { + fn len(self) -> Result { + Ok(self.size) + } + + fn read_bytes_at(self, offset: u64, size: u64) -> Result<&'a [u8], ()> { + if size == 0 { + return Ok(&[]); + } + let end = offset.checked_add(size).ok_or(())?; + if end > self.size { + return Err(()); + } + let r_offset = self.offset.checked_add(offset).ok_or(())?; + self.r.read_bytes_at(r_offset, size) + } + + fn read_bytes_at_until(self, range: Range, delimiter: u8) -> Result<&'a [u8], ()> { + let r_start = self.offset.checked_add(range.start).ok_or(())?; + let r_end = self.offset.checked_add(range.end).ok_or(())?; + let bytes = self.r.read_bytes_at_until(r_start..r_end, delimiter)?; + let size = bytes.len().try_into().map_err(|_| ())?; + let end = range.start.checked_add(size).ok_or(())?; + if end > self.size { + return Err(()); + } + Ok(bytes) + } +} diff --git a/crux-mir/lib/object/src/read/read_ref.rs b/crux-mir/lib/object/src/read/read_ref.rs new file mode 100644 index 000000000..2f547a4e2 --- /dev/null +++ b/crux-mir/lib/object/src/read/read_ref.rs @@ -0,0 +1,137 @@ +#![allow(clippy::len_without_is_empty)] + +use core::convert::TryInto; +use core::ops::Range; +use core::{mem, result}; + +use crate::pod::{from_bytes, slice_from_bytes, Pod}; + +type Result = result::Result; + +/// A trait for reading references to `Pod` types from a block of data. +/// +/// This allows parsers to handle both of these cases: +/// - the block of data exists in memory, and it is desirable +/// to use references to this block instead of copying it, +/// - the block of data exists in storage, and it is desirable +/// to read on demand to minimize I/O and memory usage. +/// +/// The methods accept `self` by value because `Self` is expected to behave +/// similar to a reference: it may be a reference with a lifetime of `'a`, +/// or it may be a wrapper of a reference. +/// +/// The `Clone` and `Copy` bounds are for convenience, and since `Self` is +/// expected to be similar to a reference, these are easily satisfied. +/// +/// Object file parsers typically use offsets to locate the structures +/// in the block, and will most commonly use the `*_at` methods to +/// read a structure at a known offset. +/// +/// Occasionally file parsers will need to treat the block as a stream, +/// and so convenience methods are provided that update an offset with +/// the size that was read. +// +// An alternative would be for methods to accept `&mut self` and use a +// `seek` method instead of the `offset` parameters, but this is less +// convenient for implementers. +pub trait ReadRef<'a>: Clone + Copy { + /// The total size of the block of data. + fn len(self) -> Result; + + /// Get a reference to a `u8` slice at the given offset. + /// + /// Returns an error if offset or size are out of bounds. + fn read_bytes_at(self, offset: u64, size: u64) -> Result<&'a [u8]>; + + /// Get a reference to a delimited `u8` slice which starts at range.start. + /// + /// Does not include the delimiter. + /// + /// Returns an error if the range is out of bounds or the delimiter is + /// not found in the range. + fn read_bytes_at_until(self, range: Range, delimiter: u8) -> Result<&'a [u8]>; + + /// Get a reference to a `u8` slice at the given offset, and update the offset. + /// + /// Returns an error if offset or size are out of bounds. + fn read_bytes(self, offset: &mut u64, size: u64) -> Result<&'a [u8]> { + let bytes = self.read_bytes_at(*offset, size)?; + *offset = offset.wrapping_add(size); + Ok(bytes) + } + + /// Get a reference to a `Pod` type at the given offset, and update the offset. + /// + /// Returns an error if offset or size are out of bounds. + /// + /// The default implementation uses `read_bytes`, and returns an error if + /// `read_bytes` does not return bytes with the correct alignment for `T`. + /// Implementors may want to provide their own implementation that ensures + /// the alignment can be satisified. Alternatively, only use this method with + /// types that do not need alignment (see the `unaligned` feature of this crate). + fn read(self, offset: &mut u64) -> Result<&'a T> { + let size = mem::size_of::().try_into().map_err(|_| ())?; + let bytes = self.read_bytes(offset, size)?; + let (t, _) = from_bytes(bytes)?; + Ok(t) + } + + /// Get a reference to a `Pod` type at the given offset. + /// + /// Returns an error if offset or size are out of bounds. + /// + /// Also see the `read` method for information regarding alignment of `T`. + fn read_at(self, mut offset: u64) -> Result<&'a T> { + self.read(&mut offset) + } + + /// Get a reference to a slice of a `Pod` type at the given offset, and update the offset. + /// + /// Returns an error if offset or size are out of bounds. + /// + /// Also see the `read` method for information regarding alignment of `T`. + fn read_slice(self, offset: &mut u64, count: usize) -> Result<&'a [T]> { + let size = count + .checked_mul(mem::size_of::()) + .ok_or(())? + .try_into() + .map_err(|_| ())?; + let bytes = self.read_bytes(offset, size)?; + let (t, _) = slice_from_bytes(bytes, count)?; + Ok(t) + } + + /// Get a reference to a slice of a `Pod` type at the given offset. + /// + /// Returns an error if offset or size are out of bounds. + /// + /// Also see the `read` method for information regarding alignment of `T`. + fn read_slice_at(self, mut offset: u64, count: usize) -> Result<&'a [T]> { + self.read_slice(&mut offset, count) + } +} + +impl<'a> ReadRef<'a> for &'a [u8] { + fn len(self) -> Result { + self.len().try_into().map_err(|_| ()) + } + + fn read_bytes_at(self, offset: u64, size: u64) -> Result<&'a [u8]> { + let offset: usize = offset.try_into().map_err(|_| ())?; + let size: usize = size.try_into().map_err(|_| ())?; + self.get(offset..).ok_or(())?.get(..size).ok_or(()) + } + + fn read_bytes_at_until(self, range: Range, delimiter: u8) -> Result<&'a [u8]> { + let start: usize = range.start.try_into().map_err(|_| ())?; + let end: usize = range.end.try_into().map_err(|_| ())?; + let bytes = self.get(start..end).ok_or(())?; + match memchr::memchr(delimiter, bytes) { + Some(len) => { + // This will never fail. + bytes.get(..len).ok_or(()) + } + None => Err(()), + } + } +} diff --git a/crux-mir/lib/object/src/read/traits.rs b/crux-mir/lib/object/src/read/traits.rs new file mode 100644 index 000000000..f1a473e0a --- /dev/null +++ b/crux-mir/lib/object/src/read/traits.rs @@ -0,0 +1,469 @@ +use alloc::borrow::Cow; +use alloc::vec::Vec; + +use crate::read::{ + self, Architecture, CodeView, ComdatKind, CompressedData, CompressedFileRange, Export, + FileFlags, Import, ObjectKind, ObjectMap, Relocation, Result, SectionFlags, SectionIndex, + SectionKind, SegmentFlags, SymbolFlags, SymbolIndex, SymbolKind, SymbolMap, SymbolMapName, + SymbolScope, SymbolSection, +}; +use crate::Endianness; + +/// An object file. +pub trait Object<'data: 'file, 'file>: read::private::Sealed { + /// A segment in the object file. + type Segment: ObjectSegment<'data>; + + /// An iterator over the segments in the object file. + type SegmentIterator: Iterator; + + /// A section in the object file. + type Section: ObjectSection<'data>; + + /// An iterator over the sections in the object file. + type SectionIterator: Iterator; + + /// A COMDAT section group in the object file. + type Comdat: ObjectComdat<'data>; + + /// An iterator over the COMDAT section groups in the object file. + type ComdatIterator: Iterator; + + /// A symbol in the object file. + type Symbol: ObjectSymbol<'data>; + + /// An iterator over symbols in the object file. + type SymbolIterator: Iterator; + + /// A symbol table in the object file. + type SymbolTable: ObjectSymbolTable< + 'data, + Symbol = Self::Symbol, + SymbolIterator = Self::SymbolIterator, + >; + + /// An iterator over dynamic relocations in the file. + /// + /// The first field in the item tuple is the address + /// that the relocation applies to. + type DynamicRelocationIterator: Iterator; + + /// Get the architecture type of the file. + fn architecture(&self) -> Architecture; + + /// Get the endianness of the file. + #[inline] + fn endianness(&self) -> Endianness { + if self.is_little_endian() { + Endianness::Little + } else { + Endianness::Big + } + } + + /// Return true if the file is little endian, false if it is big endian. + fn is_little_endian(&self) -> bool; + + /// Return true if the file can contain 64-bit addresses. + fn is_64(&self) -> bool; + + /// Return the kind of this object. + fn kind(&self) -> ObjectKind; + + /// Get an iterator over the segments in the file. + fn segments(&'file self) -> Self::SegmentIterator; + + /// Get the section named `section_name`, if such a section exists. + /// + /// If `section_name` starts with a '.' then it is treated as a system section name, + /// and is compared using the conventions specific to the object file format. This + /// includes: + /// - if ".debug_str_offsets" is requested for a Mach-O object file, then the actual + /// section name that is searched for is "__debug_str_offs". + /// - if ".debug_info" is requested for an ELF object file, then + /// ".zdebug_info" may be returned (and similarly for other debug sections). + /// + /// For some object files, multiple segments may contain sections with the same + /// name. In this case, the first matching section will be used. + /// + /// This method skips over sections with invalid names. + fn section_by_name(&'file self, section_name: &str) -> Option { + self.section_by_name_bytes(section_name.as_bytes()) + } + + /// Like [`Self::section_by_name`], but allows names that are not UTF-8. + fn section_by_name_bytes(&'file self, section_name: &[u8]) -> Option; + + /// Get the section at the given index. + /// + /// The meaning of the index depends on the object file. + /// + /// For some object files, this requires iterating through all sections. + /// + /// Returns an error if the index is invalid. + fn section_by_index(&'file self, index: SectionIndex) -> Result; + + /// Get an iterator over the sections in the file. + fn sections(&'file self) -> Self::SectionIterator; + + /// Get an iterator over the COMDAT section groups in the file. + fn comdats(&'file self) -> Self::ComdatIterator; + + /// Get the symbol table, if any. + fn symbol_table(&'file self) -> Option; + + /// Get the debugging symbol at the given index. + /// + /// The meaning of the index depends on the object file. + /// + /// Returns an error if the index is invalid. + fn symbol_by_index(&'file self, index: SymbolIndex) -> Result; + + /// Get an iterator over the debugging symbols in the file. + /// + /// This may skip over symbols that are malformed or unsupported. + /// + /// For Mach-O files, this does not include STAB entries. + fn symbols(&'file self) -> Self::SymbolIterator; + + /// Get the dynamic linking symbol table, if any. + /// + /// Only ELF has a separate dynamic linking symbol table. + fn dynamic_symbol_table(&'file self) -> Option; + + /// Get an iterator over the dynamic linking symbols in the file. + /// + /// This may skip over symbols that are malformed or unsupported. + /// + /// Only ELF has separate dynamic linking symbols. + /// Other file formats will return an empty iterator. + fn dynamic_symbols(&'file self) -> Self::SymbolIterator; + + /// Get the dynamic relocations for this file. + /// + /// Symbol indices in these relocations refer to the dynamic symbol table. + /// + /// Only ELF has dynamic relocations. + fn dynamic_relocations(&'file self) -> Option; + + /// Construct a map from addresses to symbol names. + /// + /// The map will only contain defined text and data symbols. + /// The dynamic symbol table will only be used if there are no debugging symbols. + fn symbol_map(&'file self) -> SymbolMap> { + let mut symbols = Vec::new(); + if let Some(table) = self.symbol_table().or_else(|| self.dynamic_symbol_table()) { + for symbol in table.symbols() { + if !symbol.is_definition() { + continue; + } + if let Ok(name) = symbol.name() { + symbols.push(SymbolMapName::new(symbol.address(), name)); + } + } + } + SymbolMap::new(symbols) + } + + /// Construct a map from addresses to symbol names and object file names. + /// + /// This is derived from Mach-O STAB entries. + fn object_map(&'file self) -> ObjectMap<'data> { + ObjectMap::default() + } + + /// Get the imported symbols. + fn imports(&self) -> Result>>; + + /// Get the exported symbols that expose both a name and an address. + /// + /// Some file formats may provide other kinds of symbols, that can be retrieved using + /// the lower-level API. + fn exports(&self) -> Result>>; + + /// Return true if the file contains debug information sections, false if not. + fn has_debug_symbols(&self) -> bool; + + /// The UUID from a Mach-O `LC_UUID` load command. + #[inline] + fn mach_uuid(&self) -> Result> { + Ok(None) + } + + /// The build ID from an ELF `NT_GNU_BUILD_ID` note. + #[inline] + fn build_id(&self) -> Result> { + Ok(None) + } + + /// The filename and CRC from a `.gnu_debuglink` section. + #[inline] + fn gnu_debuglink(&self) -> Result> { + Ok(None) + } + + /// The filename and build ID from a `.gnu_debugaltlink` section. + #[inline] + fn gnu_debugaltlink(&self) -> Result> { + Ok(None) + } + + /// The filename and GUID from the PE CodeView section + #[inline] + fn pdb_info(&self) -> Result> { + Ok(None) + } + + /// Get the base address used for relative virtual addresses. + /// + /// Currently this is only non-zero for PE. + fn relative_address_base(&'file self) -> u64; + + /// Get the virtual address of the entry point of the binary + fn entry(&'file self) -> u64; + + /// File flags that are specific to each file format. + fn flags(&self) -> FileFlags; +} + +/// A loadable segment defined in an object file. +/// +/// For ELF, this is a program header with type `PT_LOAD`. +/// For Mach-O, this is a load command with type `LC_SEGMENT` or `LC_SEGMENT_64`. +pub trait ObjectSegment<'data>: read::private::Sealed { + /// Returns the virtual address of the segment. + fn address(&self) -> u64; + + /// Returns the size of the segment in memory. + fn size(&self) -> u64; + + /// Returns the alignment of the segment in memory. + fn align(&self) -> u64; + + /// Returns the offset and size of the segment in the file. + fn file_range(&self) -> (u64, u64); + + /// Returns a reference to the file contents of the segment. + /// + /// The length of this data may be different from the size of the + /// segment in memory. + fn data(&self) -> Result<&'data [u8]>; + + /// Return the segment data in the given range. + /// + /// Returns `Ok(None)` if the segment does not contain the given range. + fn data_range(&self, address: u64, size: u64) -> Result>; + + /// Returns the name of the segment. + fn name_bytes(&self) -> Result>; + + /// Returns the name of the segment. + /// + /// Returns an error if the name is not UTF-8. + fn name(&self) -> Result>; + + /// Return the flags of segment. + fn flags(&self) -> SegmentFlags; +} + +/// A section defined in an object file. +pub trait ObjectSection<'data>: read::private::Sealed { + /// An iterator over the relocations for a section. + /// + /// The first field in the item tuple is the section offset + /// that the relocation applies to. + type RelocationIterator: Iterator; + + /// Returns the section index. + fn index(&self) -> SectionIndex; + + /// Returns the address of the section. + fn address(&self) -> u64; + + /// Returns the size of the section in memory. + fn size(&self) -> u64; + + /// Returns the alignment of the section in memory. + fn align(&self) -> u64; + + /// Returns offset and size of on-disk segment (if any). + fn file_range(&self) -> Option<(u64, u64)>; + + /// Returns the raw contents of the section. + /// + /// The length of this data may be different from the size of the + /// section in memory. + /// + /// This does not do any decompression. + fn data(&self) -> Result<&'data [u8]>; + + /// Return the raw contents of the section data in the given range. + /// + /// This does not do any decompression. + /// + /// Returns `Ok(None)` if the section does not contain the given range. + fn data_range(&self, address: u64, size: u64) -> Result>; + + /// Returns the potentially compressed file range of the section, + /// along with information about the compression. + fn compressed_file_range(&self) -> Result; + + /// Returns the potentially compressed contents of the section, + /// along with information about the compression. + fn compressed_data(&self) -> Result>; + + /// Returns the uncompressed contents of the section. + /// + /// The length of this data may be different from the size of the + /// section in memory. + /// + /// If no compression is detected, then returns the data unchanged. + /// Returns `Err` if decompression fails. + fn uncompressed_data(&self) -> Result> { + self.compressed_data()?.decompress() + } + + /// Returns the name of the section. + fn name_bytes(&self) -> Result<&[u8]>; + + /// Returns the name of the section. + /// + /// Returns an error if the name is not UTF-8. + fn name(&self) -> Result<&str>; + + /// Returns the name of the segment for this section. + fn segment_name_bytes(&self) -> Result>; + + /// Returns the name of the segment for this section. + /// + /// Returns an error if the name is not UTF-8. + fn segment_name(&self) -> Result>; + + /// Return the kind of this section. + fn kind(&self) -> SectionKind; + + /// Get the relocations for this section. + fn relocations(&self) -> Self::RelocationIterator; + + /// Section flags that are specific to each file format. + fn flags(&self) -> SectionFlags; +} + +/// A COMDAT section group defined in an object file. +pub trait ObjectComdat<'data>: read::private::Sealed { + /// An iterator over the sections in the object file. + type SectionIterator: Iterator; + + /// Returns the COMDAT selection kind. + fn kind(&self) -> ComdatKind; + + /// Returns the index of the symbol used for the name of COMDAT section group. + fn symbol(&self) -> SymbolIndex; + + /// Returns the name of the COMDAT section group. + fn name_bytes(&self) -> Result<&[u8]>; + + /// Returns the name of the COMDAT section group. + /// + /// Returns an error if the name is not UTF-8. + fn name(&self) -> Result<&str>; + + /// Get the sections in this section group. + fn sections(&self) -> Self::SectionIterator; +} + +/// A symbol table. +pub trait ObjectSymbolTable<'data>: read::private::Sealed { + /// A symbol table entry. + type Symbol: ObjectSymbol<'data>; + + /// An iterator over the symbols in a symbol table. + type SymbolIterator: Iterator; + + /// Get an iterator over the symbols in the table. + /// + /// This may skip over symbols that are malformed or unsupported. + fn symbols(&self) -> Self::SymbolIterator; + + /// Get the symbol at the given index. + /// + /// The meaning of the index depends on the object file. + /// + /// Returns an error if the index is invalid. + fn symbol_by_index(&self, index: SymbolIndex) -> Result; +} + +/// A symbol table entry. +pub trait ObjectSymbol<'data>: read::private::Sealed { + /// The index of the symbol. + fn index(&self) -> SymbolIndex; + + /// The name of the symbol. + fn name_bytes(&self) -> Result<&'data [u8]>; + + /// The name of the symbol. + /// + /// Returns an error if the name is not UTF-8. + fn name(&self) -> Result<&'data str>; + + /// The address of the symbol. May be zero if the address is unknown. + fn address(&self) -> u64; + + /// The size of the symbol. May be zero if the size is unknown. + fn size(&self) -> u64; + + /// Return the kind of this symbol. + fn kind(&self) -> SymbolKind; + + /// Returns the section where the symbol is defined. + fn section(&self) -> SymbolSection; + + /// Returns the section index for the section containing this symbol. + /// + /// May return `None` if the symbol is not defined in a section. + fn section_index(&self) -> Option { + self.section().index() + } + + /// Return true if the symbol is undefined. + fn is_undefined(&self) -> bool; + + /// Return true if the symbol is a definition of a function or data object + /// that has a known address. + fn is_definition(&self) -> bool; + + /// Return true if the symbol is common data. + /// + /// Note: does not check for `SymbolSection::Section` with `SectionKind::Common`. + fn is_common(&self) -> bool; + + /// Return true if the symbol is weak. + fn is_weak(&self) -> bool; + + /// Returns the symbol scope. + fn scope(&self) -> SymbolScope; + + /// Return true if the symbol visible outside of the compilation unit. + /// + /// This treats `SymbolScope::Unknown` as global. + fn is_global(&self) -> bool; + + /// Return true if the symbol is only visible within the compilation unit. + fn is_local(&self) -> bool; + + /// Symbol flags that are specific to each file format. + fn flags(&self) -> SymbolFlags; +} + +/// An iterator for files that don't have dynamic relocations. +#[derive(Debug)] +pub struct NoDynamicRelocationIterator; + +impl Iterator for NoDynamicRelocationIterator { + type Item = (u64, Relocation); + + #[inline] + fn next(&mut self) -> Option { + None + } +} diff --git a/crux-mir/lib/object/src/read/util.rs b/crux-mir/lib/object/src/read/util.rs new file mode 100644 index 000000000..842bd6ca1 --- /dev/null +++ b/crux-mir/lib/object/src/read/util.rs @@ -0,0 +1,383 @@ +use alloc::string::String; +use core::convert::TryInto; +use core::fmt; +use core::marker::PhantomData; + +use crate::pod::{from_bytes, slice_from_bytes, Pod}; +use crate::ReadRef; + +/// A newtype for byte slices. +/// +/// It has these important features: +/// - no methods that can panic, such as `Index` +/// - convenience methods for `Pod` types +/// - a useful `Debug` implementation +#[derive(Default, Clone, Copy, PartialEq, Eq)] +pub struct Bytes<'data>(pub &'data [u8]); + +impl<'data> fmt::Debug for Bytes<'data> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + debug_list_bytes(self.0, fmt) + } +} + +impl<'data> Bytes<'data> { + /// Return the length of the byte slice. + #[inline] + pub fn len(&self) -> usize { + self.0.len() + } + + /// Return true if the byte slice is empty. + #[inline] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Skip over the given number of bytes at the start of the byte slice. + /// + /// Modifies the byte slice to start after the bytes. + /// + /// Returns an error if there are too few bytes. + #[inline] + pub fn skip(&mut self, offset: usize) -> Result<(), ()> { + match self.0.get(offset..) { + Some(tail) => { + self.0 = tail; + Ok(()) + } + None => { + self.0 = &[]; + Err(()) + } + } + } + + /// Return a reference to the given number of bytes at the start of the byte slice. + /// + /// Modifies the byte slice to start after the bytes. + /// + /// Returns an error if there are too few bytes. + #[inline] + pub fn read_bytes(&mut self, count: usize) -> Result, ()> { + match (self.0.get(..count), self.0.get(count..)) { + (Some(head), Some(tail)) => { + self.0 = tail; + Ok(Bytes(head)) + } + _ => { + self.0 = &[]; + Err(()) + } + } + } + + /// Return a reference to the given number of bytes at the given offset of the byte slice. + /// + /// Returns an error if the offset is invalid or there are too few bytes. + #[inline] + pub fn read_bytes_at(mut self, offset: usize, count: usize) -> Result, ()> { + self.skip(offset)?; + self.read_bytes(count) + } + + /// Return a reference to a `Pod` struct at the start of the byte slice. + /// + /// Modifies the byte slice to start after the bytes. + /// + /// Returns an error if there are too few bytes or the slice is incorrectly aligned. + #[inline] + pub fn read(&mut self) -> Result<&'data T, ()> { + match from_bytes(self.0) { + Ok((value, tail)) => { + self.0 = tail; + Ok(value) + } + Err(()) => { + self.0 = &[]; + Err(()) + } + } + } + + /// Return a reference to a `Pod` struct at the given offset of the byte slice. + /// + /// Returns an error if there are too few bytes or the offset is incorrectly aligned. + #[inline] + pub fn read_at(mut self, offset: usize) -> Result<&'data T, ()> { + self.skip(offset)?; + self.read() + } + + /// Return a reference to a slice of `Pod` structs at the start of the byte slice. + /// + /// Modifies the byte slice to start after the bytes. + /// + /// Returns an error if there are too few bytes or the offset is incorrectly aligned. + #[inline] + pub fn read_slice(&mut self, count: usize) -> Result<&'data [T], ()> { + match slice_from_bytes(self.0, count) { + Ok((value, tail)) => { + self.0 = tail; + Ok(value) + } + Err(()) => { + self.0 = &[]; + Err(()) + } + } + } + + /// Return a reference to a slice of `Pod` structs at the given offset of the byte slice. + /// + /// Returns an error if there are too few bytes or the offset is incorrectly aligned. + #[inline] + pub fn read_slice_at(mut self, offset: usize, count: usize) -> Result<&'data [T], ()> { + self.skip(offset)?; + self.read_slice(count) + } + + /// Read a null terminated string. + /// + /// Does not assume any encoding. + /// Reads past the null byte, but doesn't return it. + #[inline] + pub fn read_string(&mut self) -> Result<&'data [u8], ()> { + match memchr::memchr(b'\0', self.0) { + Some(null) => { + // These will never fail. + let bytes = self.read_bytes(null)?; + self.skip(1)?; + Ok(bytes.0) + } + None => { + self.0 = &[]; + Err(()) + } + } + } + + /// Read a null terminated string at an offset. + /// + /// Does not assume any encoding. Does not return the null byte. + #[inline] + pub fn read_string_at(mut self, offset: usize) -> Result<&'data [u8], ()> { + self.skip(offset)?; + self.read_string() + } +} + +// Only for Debug impl of `Bytes`. +fn debug_list_bytes(bytes: &[u8], fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut list = fmt.debug_list(); + list.entries(bytes.iter().take(8).copied().map(DebugByte)); + if bytes.len() > 8 { + list.entry(&DebugLen(bytes.len())); + } + list.finish() +} + +struct DebugByte(u8); + +impl fmt::Debug for DebugByte { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(fmt, "0x{:02x}", self.0) + } +} + +struct DebugLen(usize); + +impl fmt::Debug for DebugLen { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(fmt, "...; {}", self.0) + } +} + +/// A newtype for byte strings. +/// +/// For byte slices that are strings of an unknown encoding. +/// +/// Provides a `Debug` implementation that interprets the bytes as UTF-8. +#[derive(Default, Clone, Copy, PartialEq, Eq)] +pub(crate) struct ByteString<'data>(pub &'data [u8]); + +impl<'data> fmt::Debug for ByteString<'data> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(fmt, "\"{}\"", String::from_utf8_lossy(self.0)) + } +} + +#[allow(dead_code)] +#[inline] +pub(crate) fn align(offset: usize, size: usize) -> usize { + (offset + (size - 1)) & !(size - 1) +} + +#[allow(dead_code)] +pub(crate) fn data_range( + data: &[u8], + data_address: u64, + range_address: u64, + size: u64, +) -> Option<&[u8]> { + let offset = range_address.checked_sub(data_address)?; + data.get(offset.try_into().ok()?..)? + .get(..size.try_into().ok()?) +} + +/// A table of zero-terminated strings. +/// +/// This is used for most file formats. +#[derive(Debug, Clone, Copy)] +pub struct StringTable<'data, R = &'data [u8]> +where + R: ReadRef<'data>, +{ + data: Option, + start: u64, + end: u64, + marker: PhantomData<&'data ()>, +} + +impl<'data, R: ReadRef<'data>> StringTable<'data, R> { + /// Interpret the given data as a string table. + pub fn new(data: R, start: u64, end: u64) -> Self { + StringTable { + data: Some(data), + start, + end, + marker: PhantomData, + } + } + + /// Return the string at the given offset. + pub fn get(&self, offset: u32) -> Result<&'data [u8], ()> { + match self.data { + Some(data) => { + let r_start = self.start.checked_add(offset.into()).ok_or(())?; + data.read_bytes_at_until(r_start..self.end, 0) + } + None => Err(()), + } + } +} + +impl<'data, R: ReadRef<'data>> Default for StringTable<'data, R> { + fn default() -> Self { + StringTable { + data: None, + start: 0, + end: 0, + marker: PhantomData, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::pod::bytes_of; + + #[test] + fn bytes() { + let x = u32::to_be(0x0123_4567); + let data = Bytes(bytes_of(&x)); + + let mut bytes = data; + assert_eq!(bytes.skip(0), Ok(())); + assert_eq!(bytes, data); + + let mut bytes = data; + assert_eq!(bytes.skip(4), Ok(())); + assert_eq!(bytes, Bytes(&[])); + + let mut bytes = data; + assert_eq!(bytes.skip(5), Err(())); + assert_eq!(bytes, Bytes(&[])); + + let mut bytes = data; + assert_eq!(bytes.read_bytes(0), Ok(Bytes(&[]))); + assert_eq!(bytes, data); + + let mut bytes = data; + assert_eq!(bytes.read_bytes(4), Ok(data)); + assert_eq!(bytes, Bytes(&[])); + + let mut bytes = data; + assert_eq!(bytes.read_bytes(5), Err(())); + assert_eq!(bytes, Bytes(&[])); + + assert_eq!(data.read_bytes_at(0, 0), Ok(Bytes(&[]))); + assert_eq!(data.read_bytes_at(4, 0), Ok(Bytes(&[]))); + assert_eq!(data.read_bytes_at(0, 4), Ok(data)); + assert_eq!(data.read_bytes_at(1, 4), Err(())); + + let mut bytes = data; + assert_eq!(bytes.read::(), Ok(&u16::to_be(0x0123))); + assert_eq!(bytes, Bytes(&[0x45, 0x67])); + assert_eq!(data.read_at::(2), Ok(&u16::to_be(0x4567))); + assert_eq!(data.read_at::(3), Err(())); + assert_eq!(data.read_at::(4), Err(())); + + let mut bytes = data; + assert_eq!(bytes.read::(), Ok(&x)); + assert_eq!(bytes, Bytes(&[])); + + let mut bytes = data; + assert_eq!(bytes.read::(), Err(())); + assert_eq!(bytes, Bytes(&[])); + + let mut bytes = data; + assert_eq!(bytes.read_slice::(0), Ok(&[][..])); + assert_eq!(bytes, data); + + let mut bytes = data; + assert_eq!(bytes.read_slice::(4), Ok(data.0)); + assert_eq!(bytes, Bytes(&[])); + + let mut bytes = data; + assert_eq!(bytes.read_slice::(5), Err(())); + assert_eq!(bytes, Bytes(&[])); + + assert_eq!(data.read_slice_at::(0, 0), Ok(&[][..])); + assert_eq!(data.read_slice_at::(4, 0), Ok(&[][..])); + assert_eq!(data.read_slice_at::(0, 4), Ok(data.0)); + assert_eq!(data.read_slice_at::(1, 4), Err(())); + + let data = Bytes(&[0x01, 0x02, 0x00, 0x04]); + + let mut bytes = data; + assert_eq!(bytes.read_string(), Ok(&data.0[..2])); + assert_eq!(bytes.0, &data.0[3..]); + + let mut bytes = data; + bytes.skip(3).unwrap(); + assert_eq!(bytes.read_string(), Err(())); + assert_eq!(bytes.0, &[]); + + assert_eq!(data.read_string_at(0), Ok(&data.0[..2])); + assert_eq!(data.read_string_at(1), Ok(&data.0[1..2])); + assert_eq!(data.read_string_at(2), Ok(&[][..])); + assert_eq!(data.read_string_at(3), Err(())); + } + + #[test] + fn bytes_debug() { + assert_eq!(format!("{:?}", Bytes(&[])), "[]"); + assert_eq!(format!("{:?}", Bytes(&[0x01])), "[0x01]"); + assert_eq!( + format!( + "{:?}", + Bytes(&[0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]) + ), + "[0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]" + ); + assert_eq!( + format!( + "{:?}", + Bytes(&[0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09]) + ), + "[0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, ...; 9]" + ); + } +} diff --git a/crux-mir/lib/object/src/read/wasm.rs b/crux-mir/lib/object/src/read/wasm.rs new file mode 100644 index 000000000..0113f5971 --- /dev/null +++ b/crux-mir/lib/object/src/read/wasm.rs @@ -0,0 +1,908 @@ +//! Support for reading Wasm files. +//! +//! Provides `WasmFile` and related types which implement the `Object` trait. +//! +//! Currently implements the minimum required to access DWARF debugging information. +use alloc::boxed::Box; +use alloc::vec::Vec; +use core::marker::PhantomData; +use core::{slice, str}; +use wasmparser as wp; + +use crate::read::{ + self, Architecture, ComdatKind, CompressedData, CompressedFileRange, Error, Export, FileFlags, + Import, NoDynamicRelocationIterator, Object, ObjectComdat, ObjectKind, ObjectSection, + ObjectSegment, ObjectSymbol, ObjectSymbolTable, ReadError, ReadRef, Relocation, Result, + SectionFlags, SectionIndex, SectionKind, SegmentFlags, SymbolFlags, SymbolIndex, SymbolKind, + SymbolScope, SymbolSection, +}; + +const SECTION_CUSTOM: usize = 0; +const SECTION_TYPE: usize = 1; +const SECTION_IMPORT: usize = 2; +const SECTION_FUNCTION: usize = 3; +const SECTION_TABLE: usize = 4; +const SECTION_MEMORY: usize = 5; +const SECTION_GLOBAL: usize = 6; +const SECTION_EXPORT: usize = 7; +const SECTION_START: usize = 8; +const SECTION_ELEMENT: usize = 9; +const SECTION_CODE: usize = 10; +const SECTION_DATA: usize = 11; +const SECTION_DATA_COUNT: usize = 12; +// Update this constant when adding new section id: +const MAX_SECTION_ID: usize = SECTION_DATA_COUNT; + +/// A WebAssembly object file. +#[derive(Debug)] +pub struct WasmFile<'data, R = &'data [u8]> { + // All sections, including custom sections. + sections: Vec>, + // Indices into `sections` of sections with a non-zero id. + id_sections: Box<[Option; MAX_SECTION_ID + 1]>, + // Whether the file has DWARF information. + has_debug_symbols: bool, + // Symbols collected from imports, exports, code and name sections. + symbols: Vec>, + // Address of the function body for the entry point. + entry: u64, + marker: PhantomData, +} + +#[derive(Clone)] +enum LocalFunctionKind { + Unknown, + Exported { symbol_ids: Vec }, + Local { symbol_id: u32 }, +} + +impl ReadError for wasmparser::Result { + fn read_error(self, error: &'static str) -> Result { + self.map_err(|_| Error(error)) + } +} + +impl<'data, R: ReadRef<'data>> WasmFile<'data, R> { + /// Parse the raw wasm data. + pub fn parse(data: R) -> Result { + let len = data.len().read_error("Unknown Wasm file size")?; + let data = data.read_bytes_at(0, len).read_error("Wasm read failed")?; + let module = wp::ModuleReader::new(data).read_error("Invalid Wasm header")?; + + let mut file = WasmFile { + sections: Vec::new(), + id_sections: Default::default(), + has_debug_symbols: false, + symbols: Vec::new(), + entry: 0, + marker: PhantomData, + }; + + let mut main_file_symbol = Some(WasmSymbolInternal { + name: "", + address: 0, + size: 0, + kind: SymbolKind::File, + section: SymbolSection::None, + scope: SymbolScope::Compilation, + }); + + let mut imported_funcs_count = 0; + let mut local_func_kinds = Vec::new(); + let mut entry_func_id = None; + + for section in module { + let section = section.read_error("Invalid Wasm section header")?; + + match section.code { + wp::SectionCode::Import => { + let mut last_module_name = None; + + for import in section + .get_import_section_reader() + .read_error("Couldn't read header of the import section")? + { + let import = import.read_error("Couldn't read an import item")?; + let module_name = import.module; + + if last_module_name != Some(module_name) { + file.symbols.push(WasmSymbolInternal { + name: module_name, + address: 0, + size: 0, + kind: SymbolKind::File, + section: SymbolSection::None, + scope: SymbolScope::Dynamic, + }); + last_module_name = Some(module_name); + } + + let kind = match import.ty { + wp::ImportSectionEntryType::Function(_) => { + imported_funcs_count += 1; + SymbolKind::Text + } + wp::ImportSectionEntryType::Table(_) + | wp::ImportSectionEntryType::Memory(_) + | wp::ImportSectionEntryType::Global(_) => SymbolKind::Data, + }; + + file.symbols.push(WasmSymbolInternal { + name: import.field, + address: 0, + size: 0, + kind, + section: SymbolSection::Undefined, + scope: SymbolScope::Dynamic, + }); + } + } + wp::SectionCode::Function => { + local_func_kinds = vec![ + LocalFunctionKind::Unknown; + section + .get_function_section_reader() + .read_error("Couldn't read header of the function section")? + .get_count() as usize + ]; + } + wp::SectionCode::Export => { + if let Some(main_file_symbol) = main_file_symbol.take() { + file.symbols.push(main_file_symbol); + } + + for export in section + .get_export_section_reader() + .read_error("Couldn't read header of the export section")? + { + let export = export.read_error("Couldn't read an export item")?; + + let (kind, section_idx) = match export.kind { + wp::ExternalKind::Function => { + if let Some(local_func_id) = + export.index.checked_sub(imported_funcs_count) + { + let local_func_kind = + &mut local_func_kinds[local_func_id as usize]; + if let LocalFunctionKind::Unknown = local_func_kind { + *local_func_kind = LocalFunctionKind::Exported { + symbol_ids: Vec::new(), + }; + } + let symbol_ids = match local_func_kind { + LocalFunctionKind::Exported { symbol_ids } => symbol_ids, + _ => unreachable!(), + }; + symbol_ids.push(file.symbols.len() as u32); + } + (SymbolKind::Text, SECTION_CODE) + } + wp::ExternalKind::Table + | wp::ExternalKind::Memory + | wp::ExternalKind::Global => (SymbolKind::Data, SECTION_DATA), + }; + + file.symbols.push(WasmSymbolInternal { + name: export.field, + address: 0, + size: 0, + kind, + section: SymbolSection::Section(SectionIndex(section_idx)), + scope: SymbolScope::Dynamic, + }); + } + } + wp::SectionCode::Start => { + entry_func_id = Some( + section + .get_start_section_content() + .read_error("Couldn't read contents of the start section")?, + ); + } + wp::SectionCode::Code => { + if let Some(main_file_symbol) = main_file_symbol.take() { + file.symbols.push(main_file_symbol); + } + + for (i, (body, local_func_kind)) in section + .get_code_section_reader() + .read_error("Couldn't read header of the code section")? + .into_iter() + .zip(&mut local_func_kinds) + .enumerate() + { + let body = body.read_error("Couldn't read a function body")?; + let range = body.range(); + + let address = range.start as u64 - section.range().start as u64; + let size = (range.end - range.start) as u64; + + if entry_func_id == Some(i as u32) { + file.entry = address; + } + + match local_func_kind { + LocalFunctionKind::Unknown => { + *local_func_kind = LocalFunctionKind::Local { + symbol_id: file.symbols.len() as u32, + }; + file.symbols.push(WasmSymbolInternal { + name: "", + address, + size, + kind: SymbolKind::Text, + section: SymbolSection::Section(SectionIndex(SECTION_CODE)), + scope: SymbolScope::Compilation, + }); + } + LocalFunctionKind::Exported { symbol_ids } => { + for symbol_id in core::mem::take(symbol_ids) { + let export_symbol = &mut file.symbols[symbol_id as usize]; + export_symbol.address = address; + export_symbol.size = size; + } + } + _ => unreachable!(), + } + } + } + wp::SectionCode::Custom { + kind: wp::CustomSectionKind::Name, + .. + } => { + for name in section + .get_name_section_reader() + .read_error("Couldn't read header of the name section")? + { + // TODO: Right now, ill-formed name subsections + // are silently ignored in order to maintain + // compatibility with extended name sections, which + // are not yet supported by the version of + // `wasmparser` currently used. + // A better fix would be to update `wasmparser` to + // the newest version, but this requires + // a major rewrite of this file. + if let Ok(wp::Name::Function(name)) = name { + let mut name_map = name.get_map().read_error( + "Couldn't read header of the function name subsection", + )?; + for _ in 0..name_map.get_count() { + let naming = name_map + .read() + .read_error("Couldn't read a function name")?; + if let Some(local_index) = + naming.index.checked_sub(imported_funcs_count) + { + if let LocalFunctionKind::Local { symbol_id } = + local_func_kinds[local_index as usize] + { + file.symbols[symbol_id as usize].name = naming.name; + } + } + } + } + } + } + wp::SectionCode::Custom { name, .. } if name.starts_with(".debug_") => { + file.has_debug_symbols = true; + } + _ => {} + } + + let id = section_code_to_id(section.code); + file.id_sections[id] = Some(file.sections.len()); + + file.sections.push(section); + } + + Ok(file) + } +} + +impl<'data, R> read::private::Sealed for WasmFile<'data, R> {} + +impl<'data, 'file, R> Object<'data, 'file> for WasmFile<'data, R> +where + 'data: 'file, + R: 'file, +{ + type Segment = WasmSegment<'data, 'file, R>; + type SegmentIterator = WasmSegmentIterator<'data, 'file, R>; + type Section = WasmSection<'data, 'file, R>; + type SectionIterator = WasmSectionIterator<'data, 'file, R>; + type Comdat = WasmComdat<'data, 'file, R>; + type ComdatIterator = WasmComdatIterator<'data, 'file, R>; + type Symbol = WasmSymbol<'data, 'file>; + type SymbolIterator = WasmSymbolIterator<'data, 'file>; + type SymbolTable = WasmSymbolTable<'data, 'file>; + type DynamicRelocationIterator = NoDynamicRelocationIterator; + + #[inline] + fn architecture(&self) -> Architecture { + Architecture::Wasm32 + } + + #[inline] + fn is_little_endian(&self) -> bool { + true + } + + #[inline] + fn is_64(&self) -> bool { + false + } + + fn kind(&self) -> ObjectKind { + // TODO: check for `linking` custom section + ObjectKind::Unknown + } + + fn segments(&'file self) -> Self::SegmentIterator { + WasmSegmentIterator { file: self } + } + + fn section_by_name_bytes( + &'file self, + section_name: &[u8], + ) -> Option> { + self.sections() + .find(|section| section.name_bytes() == Ok(section_name)) + } + + fn section_by_index(&'file self, index: SectionIndex) -> Result> { + // TODO: Missing sections should return an empty section. + let id_section = self + .id_sections + .get(index.0) + .and_then(|x| *x) + .read_error("Invalid Wasm section index")?; + let section = self.sections.get(id_section).unwrap(); + Ok(WasmSection { + section, + marker: PhantomData, + }) + } + + fn sections(&'file self) -> Self::SectionIterator { + WasmSectionIterator { + sections: self.sections.iter(), + marker: PhantomData, + } + } + + fn comdats(&'file self) -> Self::ComdatIterator { + WasmComdatIterator { file: self } + } + + #[inline] + fn symbol_by_index(&'file self, index: SymbolIndex) -> Result> { + let symbol = self + .symbols + .get(index.0) + .read_error("Invalid Wasm symbol index")?; + Ok(WasmSymbol { index, symbol }) + } + + fn symbols(&'file self) -> Self::SymbolIterator { + WasmSymbolIterator { + symbols: self.symbols.iter().enumerate(), + } + } + + fn symbol_table(&'file self) -> Option> { + Some(WasmSymbolTable { + symbols: &self.symbols, + }) + } + + fn dynamic_symbols(&'file self) -> Self::SymbolIterator { + WasmSymbolIterator { + symbols: [].iter().enumerate(), + } + } + + #[inline] + fn dynamic_symbol_table(&'file self) -> Option> { + None + } + + #[inline] + fn dynamic_relocations(&self) -> Option { + None + } + + fn imports(&self) -> Result>> { + // TODO: return entries in the import section + Ok(Vec::new()) + } + + fn exports(&self) -> Result>> { + // TODO: return entries in the export section + Ok(Vec::new()) + } + + fn has_debug_symbols(&self) -> bool { + self.has_debug_symbols + } + + fn relative_address_base(&self) -> u64 { + 0 + } + + #[inline] + fn entry(&'file self) -> u64 { + self.entry + } + + #[inline] + fn flags(&self) -> FileFlags { + FileFlags::None + } +} + +/// An iterator over the segments of a `WasmFile`. +#[derive(Debug)] +pub struct WasmSegmentIterator<'data, 'file, R = &'data [u8]> { + #[allow(unused)] + file: &'file WasmFile<'data, R>, +} + +impl<'data, 'file, R> Iterator for WasmSegmentIterator<'data, 'file, R> { + type Item = WasmSegment<'data, 'file, R>; + + #[inline] + fn next(&mut self) -> Option { + None + } +} + +/// A segment of a `WasmFile`. +#[derive(Debug)] +pub struct WasmSegment<'data, 'file, R = &'data [u8]> { + #[allow(unused)] + file: &'file WasmFile<'data, R>, +} + +impl<'data, 'file, R> read::private::Sealed for WasmSegment<'data, 'file, R> {} + +impl<'data, 'file, R> ObjectSegment<'data> for WasmSegment<'data, 'file, R> { + #[inline] + fn address(&self) -> u64 { + unreachable!() + } + + #[inline] + fn size(&self) -> u64 { + unreachable!() + } + + #[inline] + fn align(&self) -> u64 { + unreachable!() + } + + #[inline] + fn file_range(&self) -> (u64, u64) { + unreachable!() + } + + fn data(&self) -> Result<&'data [u8]> { + unreachable!() + } + + fn data_range(&self, _address: u64, _size: u64) -> Result> { + unreachable!() + } + + #[inline] + fn name_bytes(&self) -> Result> { + unreachable!() + } + + #[inline] + fn name(&self) -> Result> { + unreachable!() + } + + #[inline] + fn flags(&self) -> SegmentFlags { + unreachable!() + } +} + +/// An iterator over the sections of a `WasmFile`. +#[derive(Debug)] +pub struct WasmSectionIterator<'data, 'file, R = &'data [u8]> { + sections: slice::Iter<'file, wp::Section<'data>>, + marker: PhantomData, +} + +impl<'data, 'file, R> Iterator for WasmSectionIterator<'data, 'file, R> { + type Item = WasmSection<'data, 'file, R>; + + fn next(&mut self) -> Option { + let section = self.sections.next()?; + Some(WasmSection { + section, + marker: PhantomData, + }) + } +} + +/// A section of a `WasmFile`. +#[derive(Debug)] +pub struct WasmSection<'data, 'file, R = &'data [u8]> { + section: &'file wp::Section<'data>, + marker: PhantomData, +} + +impl<'data, 'file, R> read::private::Sealed for WasmSection<'data, 'file, R> {} + +impl<'data, 'file, R> ObjectSection<'data> for WasmSection<'data, 'file, R> { + type RelocationIterator = WasmRelocationIterator<'data, 'file, R>; + + #[inline] + fn index(&self) -> SectionIndex { + // Note that we treat all custom sections as index 0. + // This is ok because they are never looked up by index. + SectionIndex(section_code_to_id(self.section.code)) + } + + #[inline] + fn address(&self) -> u64 { + 0 + } + + #[inline] + fn size(&self) -> u64 { + let range = self.section.range(); + (range.end - range.start) as u64 + } + + #[inline] + fn align(&self) -> u64 { + 1 + } + + #[inline] + fn file_range(&self) -> Option<(u64, u64)> { + let range = self.section.range(); + Some((range.start as _, range.end as _)) + } + + #[inline] + fn data(&self) -> Result<&'data [u8]> { + let mut reader = self.section.get_binary_reader(); + // TODO: raise a feature request upstream to be able + // to get remaining slice from a BinaryReader directly. + Ok(reader.read_bytes(reader.bytes_remaining()).unwrap()) + } + + fn data_range(&self, _address: u64, _size: u64) -> Result> { + unimplemented!() + } + + #[inline] + fn compressed_file_range(&self) -> Result { + Ok(CompressedFileRange::none(self.file_range())) + } + + #[inline] + fn compressed_data(&self) -> Result> { + self.data().map(CompressedData::none) + } + + #[inline] + fn name_bytes(&self) -> Result<&[u8]> { + self.name().map(str::as_bytes) + } + + #[inline] + fn name(&self) -> Result<&str> { + Ok(match self.section.code { + wp::SectionCode::Custom { name, .. } => name, + wp::SectionCode::Type => "", + wp::SectionCode::Import => "", + wp::SectionCode::Function => "", + wp::SectionCode::Table => "", + wp::SectionCode::Memory => "", + wp::SectionCode::Global => "", + wp::SectionCode::Export => "", + wp::SectionCode::Start => "", + wp::SectionCode::Element => "", + wp::SectionCode::Code => "", + wp::SectionCode::Data => "", + wp::SectionCode::DataCount => "", + }) + } + + #[inline] + fn segment_name_bytes(&self) -> Result> { + Ok(None) + } + + #[inline] + fn segment_name(&self) -> Result> { + Ok(None) + } + + #[inline] + fn kind(&self) -> SectionKind { + match self.section.code { + wp::SectionCode::Custom { kind, .. } => match kind { + wp::CustomSectionKind::Reloc | wp::CustomSectionKind::Linking => { + SectionKind::Linker + } + _ => SectionKind::Other, + }, + wp::SectionCode::Type => SectionKind::Metadata, + wp::SectionCode::Import => SectionKind::Linker, + wp::SectionCode::Function => SectionKind::Metadata, + wp::SectionCode::Table => SectionKind::UninitializedData, + wp::SectionCode::Memory => SectionKind::UninitializedData, + wp::SectionCode::Global => SectionKind::Data, + wp::SectionCode::Export => SectionKind::Linker, + wp::SectionCode::Start => SectionKind::Linker, + wp::SectionCode::Element => SectionKind::Data, + wp::SectionCode::Code => SectionKind::Text, + wp::SectionCode::Data => SectionKind::Data, + wp::SectionCode::DataCount => SectionKind::UninitializedData, + } + } + + #[inline] + fn relocations(&self) -> WasmRelocationIterator<'data, 'file, R> { + WasmRelocationIterator(PhantomData) + } + + #[inline] + fn flags(&self) -> SectionFlags { + SectionFlags::None + } +} + +/// An iterator over the COMDAT section groups of a `WasmFile`. +#[derive(Debug)] +pub struct WasmComdatIterator<'data, 'file, R = &'data [u8]> { + #[allow(unused)] + file: &'file WasmFile<'data, R>, +} + +impl<'data, 'file, R> Iterator for WasmComdatIterator<'data, 'file, R> { + type Item = WasmComdat<'data, 'file, R>; + + #[inline] + fn next(&mut self) -> Option { + None + } +} + +/// A COMDAT section group of a `WasmFile`. +#[derive(Debug)] +pub struct WasmComdat<'data, 'file, R = &'data [u8]> { + #[allow(unused)] + file: &'file WasmFile<'data, R>, +} + +impl<'data, 'file, R> read::private::Sealed for WasmComdat<'data, 'file, R> {} + +impl<'data, 'file, R> ObjectComdat<'data> for WasmComdat<'data, 'file, R> { + type SectionIterator = WasmComdatSectionIterator<'data, 'file, R>; + + #[inline] + fn kind(&self) -> ComdatKind { + unreachable!(); + } + + #[inline] + fn symbol(&self) -> SymbolIndex { + unreachable!(); + } + + #[inline] + fn name_bytes(&self) -> Result<&[u8]> { + unreachable!(); + } + + #[inline] + fn name(&self) -> Result<&str> { + unreachable!(); + } + + #[inline] + fn sections(&self) -> Self::SectionIterator { + unreachable!(); + } +} + +/// An iterator over the sections in a COMDAT section group of a `WasmFile`. +#[derive(Debug)] +pub struct WasmComdatSectionIterator<'data, 'file, R = &'data [u8]> +where + 'data: 'file, +{ + #[allow(unused)] + file: &'file WasmFile<'data, R>, +} + +impl<'data, 'file, R> Iterator for WasmComdatSectionIterator<'data, 'file, R> { + type Item = SectionIndex; + + fn next(&mut self) -> Option { + None + } +} + +/// A symbol table of a `WasmFile`. +#[derive(Debug)] +pub struct WasmSymbolTable<'data, 'file> { + symbols: &'file [WasmSymbolInternal<'data>], +} + +impl<'data, 'file> read::private::Sealed for WasmSymbolTable<'data, 'file> {} + +impl<'data, 'file> ObjectSymbolTable<'data> for WasmSymbolTable<'data, 'file> { + type Symbol = WasmSymbol<'data, 'file>; + type SymbolIterator = WasmSymbolIterator<'data, 'file>; + + fn symbols(&self) -> Self::SymbolIterator { + WasmSymbolIterator { + symbols: self.symbols.iter().enumerate(), + } + } + + fn symbol_by_index(&self, index: SymbolIndex) -> Result { + let symbol = self + .symbols + .get(index.0) + .read_error("Invalid Wasm symbol index")?; + Ok(WasmSymbol { index, symbol }) + } +} + +/// An iterator over the symbols of a `WasmFile`. +#[derive(Debug)] +pub struct WasmSymbolIterator<'data, 'file> { + symbols: core::iter::Enumerate>>, +} + +impl<'data, 'file> Iterator for WasmSymbolIterator<'data, 'file> { + type Item = WasmSymbol<'data, 'file>; + + fn next(&mut self) -> Option { + let (index, symbol) = self.symbols.next()?; + Some(WasmSymbol { + index: SymbolIndex(index), + symbol, + }) + } +} + +/// A symbol of a `WasmFile`. +#[derive(Clone, Copy, Debug)] +pub struct WasmSymbol<'data, 'file> { + index: SymbolIndex, + symbol: &'file WasmSymbolInternal<'data>, +} + +#[derive(Clone, Debug)] +struct WasmSymbolInternal<'data> { + name: &'data str, + address: u64, + size: u64, + kind: SymbolKind, + section: SymbolSection, + scope: SymbolScope, +} + +impl<'data, 'file> read::private::Sealed for WasmSymbol<'data, 'file> {} + +impl<'data, 'file> ObjectSymbol<'data> for WasmSymbol<'data, 'file> { + #[inline] + fn index(&self) -> SymbolIndex { + self.index + } + + #[inline] + fn name_bytes(&self) -> read::Result<&'data [u8]> { + Ok(self.symbol.name.as_bytes()) + } + + #[inline] + fn name(&self) -> read::Result<&'data str> { + Ok(self.symbol.name) + } + + #[inline] + fn address(&self) -> u64 { + self.symbol.address + } + + #[inline] + fn size(&self) -> u64 { + self.symbol.size + } + + #[inline] + fn kind(&self) -> SymbolKind { + self.symbol.kind + } + + #[inline] + fn section(&self) -> SymbolSection { + self.symbol.section + } + + #[inline] + fn is_undefined(&self) -> bool { + self.symbol.section == SymbolSection::Undefined + } + + #[inline] + fn is_definition(&self) -> bool { + self.symbol.kind == SymbolKind::Text && self.symbol.section != SymbolSection::Undefined + } + + #[inline] + fn is_common(&self) -> bool { + self.symbol.section == SymbolSection::Common + } + + #[inline] + fn is_weak(&self) -> bool { + false + } + + #[inline] + fn scope(&self) -> SymbolScope { + self.symbol.scope + } + + #[inline] + fn is_global(&self) -> bool { + self.symbol.scope != SymbolScope::Compilation + } + + #[inline] + fn is_local(&self) -> bool { + self.symbol.scope == SymbolScope::Compilation + } + + #[inline] + fn flags(&self) -> SymbolFlags { + SymbolFlags::None + } +} + +/// An iterator over the relocations in a `WasmSection`. +#[derive(Debug)] +pub struct WasmRelocationIterator<'data, 'file, R = &'data [u8]>( + PhantomData<(&'data (), &'file (), R)>, +); + +impl<'data, 'file, R> Iterator for WasmRelocationIterator<'data, 'file, R> { + type Item = (u64, Relocation); + + #[inline] + fn next(&mut self) -> Option { + None + } +} + +fn section_code_to_id(code: wp::SectionCode) -> usize { + match code { + wp::SectionCode::Custom { .. } => SECTION_CUSTOM, + wp::SectionCode::Type => SECTION_TYPE, + wp::SectionCode::Import => SECTION_IMPORT, + wp::SectionCode::Function => SECTION_FUNCTION, + wp::SectionCode::Table => SECTION_TABLE, + wp::SectionCode::Memory => SECTION_MEMORY, + wp::SectionCode::Global => SECTION_GLOBAL, + wp::SectionCode::Export => SECTION_EXPORT, + wp::SectionCode::Start => SECTION_START, + wp::SectionCode::Element => SECTION_ELEMENT, + wp::SectionCode::Code => SECTION_CODE, + wp::SectionCode::Data => SECTION_DATA, + wp::SectionCode::DataCount => SECTION_DATA_COUNT, + } +} diff --git a/crux-mir/lib/object/src/write/coff.rs b/crux-mir/lib/object/src/write/coff.rs new file mode 100644 index 000000000..6647b9611 --- /dev/null +++ b/crux-mir/lib/object/src/write/coff.rs @@ -0,0 +1,712 @@ +use alloc::vec::Vec; +use core::mem; + +use crate::endian::{LittleEndian as LE, U16Bytes, U32Bytes, U16, U32}; +use crate::pe as coff; +use crate::write::string::*; +use crate::write::util::*; +use crate::write::*; + +#[derive(Default, Clone, Copy)] +struct SectionOffsets { + offset: usize, + str_id: Option, + reloc_offset: usize, + selection: u8, + associative_section: u16, +} + +#[derive(Default, Clone, Copy)] +struct SymbolOffsets { + index: usize, + str_id: Option, + aux_count: u8, +} + +/// Internal format to use for the `.drectve` section containing linker +/// directives for symbol exports. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CoffExportStyle { + /// MSVC format supported by link.exe and LLD. + Msvc, + /// Gnu format supported by GNU LD and LLD. + Gnu, +} + +impl<'a> Object<'a> { + pub(crate) fn coff_section_info( + &self, + section: StandardSection, + ) -> (&'static [u8], &'static [u8], SectionKind) { + match section { + StandardSection::Text => (&[], &b".text"[..], SectionKind::Text), + StandardSection::Data => (&[], &b".data"[..], SectionKind::Data), + StandardSection::ReadOnlyData + | StandardSection::ReadOnlyDataWithRel + | StandardSection::ReadOnlyString => (&[], &b".rdata"[..], SectionKind::ReadOnlyData), + StandardSection::UninitializedData => { + (&[], &b".bss"[..], SectionKind::UninitializedData) + } + // TLS sections are data sections with a special name. + StandardSection::Tls => (&[], &b".tls$"[..], SectionKind::Data), + StandardSection::UninitializedTls => { + // Unsupported section. + (&[], &[], SectionKind::UninitializedTls) + } + StandardSection::TlsVariables => { + // Unsupported section. + (&[], &[], SectionKind::TlsVariables) + } + StandardSection::Common => { + // Unsupported section. + (&[], &[], SectionKind::Common) + } + } + } + + pub(crate) fn coff_subsection_name(&self, section: &[u8], value: &[u8]) -> Vec { + let mut name = section.to_vec(); + name.push(b'$'); + name.extend_from_slice(value); + name + } + + pub(crate) fn coff_fixup_relocation(&mut self, mut relocation: &mut Relocation) -> i64 { + if relocation.kind == RelocationKind::GotRelative { + // Use a stub symbol for the relocation instead. + // This isn't really a GOT, but it's a similar purpose. + // TODO: need to handle DLL imports differently? + relocation.kind = RelocationKind::Relative; + relocation.symbol = self.coff_add_stub_symbol(relocation.symbol); + } else if relocation.kind == RelocationKind::PltRelative { + // Windows doesn't need a separate relocation type for + // references to functions in import libraries. + // For convenience, treat this the same as Relative. + relocation.kind = RelocationKind::Relative; + } + + let constant = match self.architecture { + Architecture::I386 | Architecture::Arm | Architecture::Aarch64 => match relocation.kind + { + RelocationKind::Relative => { + // IMAGE_REL_I386_REL32, IMAGE_REL_ARM_REL32, IMAGE_REL_ARM64_REL32 + relocation.addend + 4 + } + _ => relocation.addend, + }, + Architecture::X86_64 => match relocation.kind { + RelocationKind::Relative => { + // IMAGE_REL_AMD64_REL32 through to IMAGE_REL_AMD64_REL32_5 + if relocation.addend <= -4 && relocation.addend >= -9 { + 0 + } else { + relocation.addend + 4 + } + } + _ => relocation.addend, + }, + _ => unimplemented!(), + }; + relocation.addend -= constant; + constant + } + + fn coff_add_stub_symbol(&mut self, symbol_id: SymbolId) -> SymbolId { + if let Some(stub_id) = self.stub_symbols.get(&symbol_id) { + return *stub_id; + } + let stub_size = self.architecture.address_size().unwrap().bytes(); + + let mut name = b".rdata$.refptr.".to_vec(); + name.extend_from_slice(&self.symbols[symbol_id.0].name); + let section_id = self.add_section(Vec::new(), name, SectionKind::ReadOnlyData); + let section = self.section_mut(section_id); + section.set_data(vec![0; stub_size as usize], u64::from(stub_size)); + section.relocations = vec![Relocation { + offset: 0, + size: stub_size * 8, + kind: RelocationKind::Absolute, + encoding: RelocationEncoding::Generic, + symbol: symbol_id, + addend: 0, + }]; + + let mut name = b".refptr.".to_vec(); + name.extend_from_slice(&self.symbol(symbol_id).name); + let stub_id = self.add_raw_symbol(Symbol { + name, + value: 0, + size: u64::from(stub_size), + kind: SymbolKind::Data, + scope: SymbolScope::Compilation, + weak: false, + section: SymbolSection::Section(section_id), + flags: SymbolFlags::None, + }); + self.stub_symbols.insert(symbol_id, stub_id); + + stub_id + } + + /// Appends linker directives to the `.drectve` section to tell the linker + /// to export all symbols with `SymbolScope::Dynamic`. + /// + /// This must be called after all symbols have been defined. + pub fn add_coff_exports(&mut self, style: CoffExportStyle) { + assert_eq!(self.format, BinaryFormat::Coff); + + let mut directives = vec![]; + for symbol in &self.symbols { + if symbol.scope == SymbolScope::Dynamic { + match style { + CoffExportStyle::Msvc => directives.extend(b" /EXPORT:\""), + CoffExportStyle::Gnu => directives.extend(b" -export:\""), + } + directives.extend(&symbol.name); + directives.extend(b"\""); + if symbol.kind != SymbolKind::Text { + match style { + CoffExportStyle::Msvc => directives.extend(b",DATA"), + CoffExportStyle::Gnu => directives.extend(b",data"), + } + } + } + } + let drectve = self.add_section(vec![], b".drectve".to_vec(), SectionKind::Linker); + self.append_section_data(drectve, &directives, 1); + } + + pub(crate) fn coff_write(&self, buffer: &mut dyn WritableBuffer) -> Result<()> { + // Calculate offsets of everything, and build strtab. + let mut offset = 0; + let mut strtab = StringTable::default(); + + // COFF header. + offset += mem::size_of::(); + + // Section headers. + offset += self.sections.len() * mem::size_of::(); + + // Calculate size of section data and add section strings to strtab. + let mut section_offsets = vec![SectionOffsets::default(); self.sections.len()]; + for (index, section) in self.sections.iter().enumerate() { + if section.name.len() > 8 { + section_offsets[index].str_id = Some(strtab.add(§ion.name)); + } + + let len = section.data.len(); + if len != 0 { + // TODO: not sure what alignment is required here, but this seems to match LLVM + offset = align(offset, 4); + section_offsets[index].offset = offset; + offset += len; + } else { + section_offsets[index].offset = 0; + } + + // Calculate size of relocations. + let mut count = section.relocations.len(); + if count != 0 { + section_offsets[index].reloc_offset = offset; + if count > 0xffff { + count += 1; + } + offset += count * mem::size_of::(); + } + } + + // Set COMDAT flags. + for comdat in &self.comdats { + let symbol = &self.symbols[comdat.symbol.0]; + let comdat_section = match symbol.section { + SymbolSection::Section(id) => id.0, + _ => { + return Err(Error(format!( + "unsupported COMDAT symbol `{}` section {:?}", + symbol.name().unwrap_or(""), + symbol.section + ))); + } + }; + section_offsets[comdat_section].selection = match comdat.kind { + ComdatKind::NoDuplicates => coff::IMAGE_COMDAT_SELECT_NODUPLICATES, + ComdatKind::Any => coff::IMAGE_COMDAT_SELECT_ANY, + ComdatKind::SameSize => coff::IMAGE_COMDAT_SELECT_SAME_SIZE, + ComdatKind::ExactMatch => coff::IMAGE_COMDAT_SELECT_EXACT_MATCH, + ComdatKind::Largest => coff::IMAGE_COMDAT_SELECT_LARGEST, + ComdatKind::Newest => coff::IMAGE_COMDAT_SELECT_NEWEST, + ComdatKind::Unknown => { + return Err(Error(format!( + "unsupported COMDAT symbol `{}` kind {:?}", + symbol.name().unwrap_or(""), + comdat.kind + ))); + } + }; + for id in &comdat.sections { + let section = &self.sections[id.0]; + if section.symbol.is_none() { + return Err(Error(format!( + "missing symbol for COMDAT section `{}`", + section.name().unwrap_or(""), + ))); + } + if id.0 != comdat_section { + section_offsets[id.0].selection = coff::IMAGE_COMDAT_SELECT_ASSOCIATIVE; + section_offsets[id.0].associative_section = comdat_section as u16 + 1; + } + } + } + + // Calculate size of symbols and add symbol strings to strtab. + let mut symbol_offsets = vec![SymbolOffsets::default(); self.symbols.len()]; + let mut symtab_count = 0; + for (index, symbol) in self.symbols.iter().enumerate() { + symbol_offsets[index].index = symtab_count; + symtab_count += 1; + match symbol.kind { + SymbolKind::File => { + // Name goes in auxilary symbol records. + let aux_count = (symbol.name.len() + coff::IMAGE_SIZEOF_SYMBOL - 1) + / coff::IMAGE_SIZEOF_SYMBOL; + symbol_offsets[index].aux_count = aux_count as u8; + symtab_count += aux_count; + // Don't add name to strtab. + continue; + } + SymbolKind::Section => { + symbol_offsets[index].aux_count = 1; + symtab_count += 1; + } + _ => {} + } + if symbol.name.len() > 8 { + symbol_offsets[index].str_id = Some(strtab.add(&symbol.name)); + } + } + + // Calculate size of symtab. + let symtab_offset = offset; + let symtab_len = symtab_count * coff::IMAGE_SIZEOF_SYMBOL; + offset += symtab_len; + + // Calculate size of strtab. + let strtab_offset = offset; + let mut strtab_data = Vec::new(); + // First 4 bytes of strtab are the length. + strtab.write(4, &mut strtab_data); + let strtab_len = strtab_data.len() + 4; + offset += strtab_len; + + // Start writing. + buffer + .reserve(offset) + .map_err(|_| Error(String::from("Cannot allocate buffer")))?; + + // Write file header. + let header = coff::ImageFileHeader { + machine: U16::new( + LE, + match self.architecture { + Architecture::Arm => coff::IMAGE_FILE_MACHINE_ARMNT, + Architecture::Aarch64 => coff::IMAGE_FILE_MACHINE_ARM64, + Architecture::I386 => coff::IMAGE_FILE_MACHINE_I386, + Architecture::X86_64 => coff::IMAGE_FILE_MACHINE_AMD64, + _ => { + return Err(Error(format!( + "unimplemented architecture {:?}", + self.architecture + ))); + } + }, + ), + number_of_sections: U16::new(LE, self.sections.len() as u16), + time_date_stamp: U32::default(), + pointer_to_symbol_table: U32::new(LE, symtab_offset as u32), + number_of_symbols: U32::new(LE, symtab_count as u32), + size_of_optional_header: U16::default(), + characteristics: match self.flags { + FileFlags::Coff { characteristics } => U16::new(LE, characteristics), + _ => U16::default(), + }, + }; + buffer.write(&header); + + // Write section headers. + for (index, section) in self.sections.iter().enumerate() { + let mut characteristics = if let SectionFlags::Coff { + characteristics, .. + } = section.flags + { + characteristics + } else { + match section.kind { + SectionKind::Text => { + coff::IMAGE_SCN_CNT_CODE + | coff::IMAGE_SCN_MEM_EXECUTE + | coff::IMAGE_SCN_MEM_READ + } + SectionKind::Data => { + coff::IMAGE_SCN_CNT_INITIALIZED_DATA + | coff::IMAGE_SCN_MEM_READ + | coff::IMAGE_SCN_MEM_WRITE + } + SectionKind::UninitializedData => { + coff::IMAGE_SCN_CNT_UNINITIALIZED_DATA + | coff::IMAGE_SCN_MEM_READ + | coff::IMAGE_SCN_MEM_WRITE + } + SectionKind::ReadOnlyData | SectionKind::ReadOnlyString => { + coff::IMAGE_SCN_CNT_INITIALIZED_DATA | coff::IMAGE_SCN_MEM_READ + } + SectionKind::Debug | SectionKind::Other | SectionKind::OtherString => { + coff::IMAGE_SCN_CNT_INITIALIZED_DATA + | coff::IMAGE_SCN_MEM_READ + | coff::IMAGE_SCN_MEM_DISCARDABLE + } + SectionKind::Linker => coff::IMAGE_SCN_LNK_INFO | coff::IMAGE_SCN_LNK_REMOVE, + SectionKind::Common + | SectionKind::Tls + | SectionKind::UninitializedTls + | SectionKind::TlsVariables + | SectionKind::Note + | SectionKind::Unknown + | SectionKind::Metadata + | SectionKind::Elf(_) => { + return Err(Error(format!( + "unimplemented section `{}` kind {:?}", + section.name().unwrap_or(""), + section.kind + ))); + } + } + }; + if section_offsets[index].selection != 0 { + characteristics |= coff::IMAGE_SCN_LNK_COMDAT; + }; + if section.relocations.len() > 0xffff { + characteristics |= coff::IMAGE_SCN_LNK_NRELOC_OVFL; + } + characteristics |= match section.align { + 1 => coff::IMAGE_SCN_ALIGN_1BYTES, + 2 => coff::IMAGE_SCN_ALIGN_2BYTES, + 4 => coff::IMAGE_SCN_ALIGN_4BYTES, + 8 => coff::IMAGE_SCN_ALIGN_8BYTES, + 16 => coff::IMAGE_SCN_ALIGN_16BYTES, + 32 => coff::IMAGE_SCN_ALIGN_32BYTES, + 64 => coff::IMAGE_SCN_ALIGN_64BYTES, + 128 => coff::IMAGE_SCN_ALIGN_128BYTES, + 256 => coff::IMAGE_SCN_ALIGN_256BYTES, + 512 => coff::IMAGE_SCN_ALIGN_512BYTES, + 1024 => coff::IMAGE_SCN_ALIGN_1024BYTES, + 2048 => coff::IMAGE_SCN_ALIGN_2048BYTES, + 4096 => coff::IMAGE_SCN_ALIGN_4096BYTES, + 8192 => coff::IMAGE_SCN_ALIGN_8192BYTES, + _ => { + return Err(Error(format!( + "unimplemented section `{}` align {}", + section.name().unwrap_or(""), + section.align + ))); + } + }; + let mut coff_section = coff::ImageSectionHeader { + name: [0; 8], + virtual_size: U32::default(), + virtual_address: U32::default(), + size_of_raw_data: U32::new(LE, section.size as u32), + pointer_to_raw_data: U32::new(LE, section_offsets[index].offset as u32), + pointer_to_relocations: U32::new(LE, section_offsets[index].reloc_offset as u32), + pointer_to_linenumbers: U32::default(), + number_of_relocations: if section.relocations.len() > 0xffff { + U16::new(LE, 0xffff) + } else { + U16::new(LE, section.relocations.len() as u16) + }, + number_of_linenumbers: U16::default(), + characteristics: U32::new(LE, characteristics), + }; + if section.name.len() <= 8 { + coff_section.name[..section.name.len()].copy_from_slice(§ion.name); + } else { + let mut str_offset = strtab.get_offset(section_offsets[index].str_id.unwrap()); + if str_offset <= 9_999_999 { + let mut name = [0; 7]; + let mut len = 0; + if str_offset == 0 { + name[6] = b'0'; + len = 1; + } else { + while str_offset != 0 { + let rem = (str_offset % 10) as u8; + str_offset /= 10; + name[6 - len] = b'0' + rem; + len += 1; + } + } + coff_section.name = [0; 8]; + coff_section.name[0] = b'/'; + coff_section.name[1..][..len].copy_from_slice(&name[7 - len..]); + } else if str_offset as u64 <= 0xf_ffff_ffff { + coff_section.name[0] = b'/'; + coff_section.name[1] = b'/'; + for i in 0..6 { + let rem = (str_offset % 64) as u8; + str_offset /= 64; + let c = match rem { + 0..=25 => b'A' + rem, + 26..=51 => b'a' + rem - 26, + 52..=61 => b'0' + rem - 52, + 62 => b'+', + 63 => b'/', + _ => unreachable!(), + }; + coff_section.name[7 - i] = c; + } + } else { + return Err(Error(format!("invalid section name offset {}", str_offset))); + } + } + buffer.write(&coff_section); + } + + // Write section data and relocations. + for (index, section) in self.sections.iter().enumerate() { + let len = section.data.len(); + if len != 0 { + write_align(buffer, 4); + debug_assert_eq!(section_offsets[index].offset, buffer.len()); + buffer.write_bytes(§ion.data); + } + + if !section.relocations.is_empty() { + debug_assert_eq!(section_offsets[index].reloc_offset, buffer.len()); + if section.relocations.len() > 0xffff { + let coff_relocation = coff::ImageRelocation { + virtual_address: U32Bytes::new(LE, section.relocations.len() as u32 + 1), + symbol_table_index: U32Bytes::new(LE, 0), + typ: U16Bytes::new(LE, 0), + }; + buffer.write(&coff_relocation); + } + for reloc in §ion.relocations { + //assert!(reloc.implicit_addend); + let typ = match self.architecture { + Architecture::I386 => match (reloc.kind, reloc.size, reloc.addend) { + (RelocationKind::Absolute, 16, 0) => coff::IMAGE_REL_I386_DIR16, + (RelocationKind::Relative, 16, 0) => coff::IMAGE_REL_I386_REL16, + (RelocationKind::Absolute, 32, 0) => coff::IMAGE_REL_I386_DIR32, + (RelocationKind::ImageOffset, 32, 0) => coff::IMAGE_REL_I386_DIR32NB, + (RelocationKind::SectionIndex, 16, 0) => coff::IMAGE_REL_I386_SECTION, + (RelocationKind::SectionOffset, 32, 0) => coff::IMAGE_REL_I386_SECREL, + (RelocationKind::SectionOffset, 7, 0) => coff::IMAGE_REL_I386_SECREL7, + (RelocationKind::Relative, 32, -4) => coff::IMAGE_REL_I386_REL32, + (RelocationKind::Coff(x), _, _) => x, + _ => { + return Err(Error(format!("unimplemented relocation {:?}", reloc))); + } + }, + Architecture::X86_64 => match (reloc.kind, reloc.size, reloc.addend) { + (RelocationKind::Absolute, 64, 0) => coff::IMAGE_REL_AMD64_ADDR64, + (RelocationKind::Absolute, 32, 0) => coff::IMAGE_REL_AMD64_ADDR32, + (RelocationKind::ImageOffset, 32, 0) => coff::IMAGE_REL_AMD64_ADDR32NB, + (RelocationKind::Relative, 32, -4) => coff::IMAGE_REL_AMD64_REL32, + (RelocationKind::Relative, 32, -5) => coff::IMAGE_REL_AMD64_REL32_1, + (RelocationKind::Relative, 32, -6) => coff::IMAGE_REL_AMD64_REL32_2, + (RelocationKind::Relative, 32, -7) => coff::IMAGE_REL_AMD64_REL32_3, + (RelocationKind::Relative, 32, -8) => coff::IMAGE_REL_AMD64_REL32_4, + (RelocationKind::Relative, 32, -9) => coff::IMAGE_REL_AMD64_REL32_5, + (RelocationKind::SectionIndex, 16, 0) => coff::IMAGE_REL_AMD64_SECTION, + (RelocationKind::SectionOffset, 32, 0) => coff::IMAGE_REL_AMD64_SECREL, + (RelocationKind::SectionOffset, 7, 0) => coff::IMAGE_REL_AMD64_SECREL7, + (RelocationKind::Coff(x), _, _) => x, + _ => { + return Err(Error(format!("unimplemented relocation {:?}", reloc))); + } + }, + Architecture::Arm => match (reloc.kind, reloc.size, reloc.addend) { + (RelocationKind::Absolute, 32, 0) => coff::IMAGE_REL_ARM_ADDR32, + (RelocationKind::ImageOffset, 32, 0) => coff::IMAGE_REL_ARM_ADDR32NB, + (RelocationKind::Relative, 32, -4) => coff::IMAGE_REL_ARM_REL32, + (RelocationKind::SectionIndex, 16, 0) => coff::IMAGE_REL_ARM_SECTION, + (RelocationKind::SectionOffset, 32, 0) => coff::IMAGE_REL_ARM_SECREL, + (RelocationKind::Coff(x), _, _) => x, + _ => { + return Err(Error(format!("unimplemented relocation {:?}", reloc))); + } + }, + Architecture::Aarch64 => match (reloc.kind, reloc.size, reloc.addend) { + (RelocationKind::Absolute, 32, 0) => coff::IMAGE_REL_ARM64_ADDR32, + (RelocationKind::ImageOffset, 32, 0) => coff::IMAGE_REL_ARM64_ADDR32NB, + (RelocationKind::SectionIndex, 16, 0) => coff::IMAGE_REL_ARM64_SECTION, + (RelocationKind::SectionOffset, 32, 0) => coff::IMAGE_REL_ARM64_SECREL, + (RelocationKind::Absolute, 64, 0) => coff::IMAGE_REL_ARM64_ADDR64, + (RelocationKind::Relative, 32, -4) => coff::IMAGE_REL_ARM64_REL32, + (RelocationKind::Coff(x), _, _) => x, + _ => { + return Err(Error(format!("unimplemented relocation {:?}", reloc))); + } + }, + _ => { + return Err(Error(format!( + "unimplemented architecture {:?}", + self.architecture + ))); + } + }; + let coff_relocation = coff::ImageRelocation { + virtual_address: U32Bytes::new(LE, reloc.offset as u32), + symbol_table_index: U32Bytes::new( + LE, + symbol_offsets[reloc.symbol.0].index as u32, + ), + typ: U16Bytes::new(LE, typ), + }; + buffer.write(&coff_relocation); + } + } + } + + // Write symbols. + debug_assert_eq!(symtab_offset, buffer.len()); + for (index, symbol) in self.symbols.iter().enumerate() { + let mut name = &symbol.name[..]; + let section_number = match symbol.section { + SymbolSection::None => { + debug_assert_eq!(symbol.kind, SymbolKind::File); + coff::IMAGE_SYM_DEBUG + } + SymbolSection::Undefined => coff::IMAGE_SYM_UNDEFINED, + SymbolSection::Absolute => coff::IMAGE_SYM_ABSOLUTE, + SymbolSection::Common => coff::IMAGE_SYM_UNDEFINED, + SymbolSection::Section(id) => id.0 as u16 + 1, + }; + let typ = if symbol.kind == SymbolKind::Text { + coff::IMAGE_SYM_DTYPE_FUNCTION << coff::IMAGE_SYM_DTYPE_SHIFT + } else { + coff::IMAGE_SYM_TYPE_NULL + }; + let storage_class = match symbol.kind { + SymbolKind::File => { + // Name goes in auxilary symbol records. + name = b".file"; + coff::IMAGE_SYM_CLASS_FILE + } + SymbolKind::Section => coff::IMAGE_SYM_CLASS_STATIC, + SymbolKind::Label => coff::IMAGE_SYM_CLASS_LABEL, + SymbolKind::Text | SymbolKind::Data | SymbolKind::Tls => { + match symbol.section { + SymbolSection::None => { + return Err(Error(format!( + "missing section for symbol `{}`", + symbol.name().unwrap_or("") + ))); + } + SymbolSection::Undefined | SymbolSection::Common => { + coff::IMAGE_SYM_CLASS_EXTERNAL + } + SymbolSection::Absolute | SymbolSection::Section(_) => { + match symbol.scope { + // TODO: does this need aux symbol records too? + _ if symbol.weak => coff::IMAGE_SYM_CLASS_WEAK_EXTERNAL, + SymbolScope::Unknown => { + return Err(Error(format!( + "unimplemented symbol `{}` scope {:?}", + symbol.name().unwrap_or(""), + symbol.scope + ))); + } + SymbolScope::Compilation => coff::IMAGE_SYM_CLASS_STATIC, + SymbolScope::Linkage | SymbolScope::Dynamic => { + coff::IMAGE_SYM_CLASS_EXTERNAL + } + } + } + } + } + SymbolKind::Unknown | SymbolKind::Null => { + return Err(Error(format!( + "unimplemented symbol `{}` kind {:?}", + symbol.name().unwrap_or(""), + symbol.kind + ))); + } + }; + let number_of_aux_symbols = symbol_offsets[index].aux_count; + let value = if symbol.section == SymbolSection::Common { + symbol.size as u32 + } else { + symbol.value as u32 + }; + let mut coff_symbol = coff::ImageSymbol { + name: [0; 8], + value: U32Bytes::new(LE, value), + section_number: U16Bytes::new(LE, section_number as u16), + typ: U16Bytes::new(LE, typ), + storage_class, + number_of_aux_symbols, + }; + if name.len() <= 8 { + coff_symbol.name[..name.len()].copy_from_slice(name); + } else { + let str_offset = strtab.get_offset(symbol_offsets[index].str_id.unwrap()); + coff_symbol.name[4..8].copy_from_slice(&u32::to_le_bytes(str_offset as u32)); + } + buffer.write(&coff_symbol); + + // Write auxiliary symbols. + match symbol.kind { + SymbolKind::File => { + let aux_len = number_of_aux_symbols as usize * coff::IMAGE_SIZEOF_SYMBOL; + debug_assert!(aux_len >= symbol.name.len()); + let old_len = buffer.len(); + buffer.write_bytes(&symbol.name); + buffer.resize(old_len + aux_len); + } + SymbolKind::Section => { + debug_assert_eq!(number_of_aux_symbols, 1); + let section_index = symbol.section.id().unwrap().0; + let section = &self.sections[section_index]; + let aux = coff::ImageAuxSymbolSection { + length: U32Bytes::new(LE, section.size as u32), + number_of_relocations: if section.relocations.len() > 0xffff { + U16Bytes::new(LE, 0xffff) + } else { + U16Bytes::new(LE, section.relocations.len() as u16) + }, + number_of_linenumbers: U16Bytes::default(), + check_sum: U32Bytes::new(LE, checksum(section.data())), + number: U16Bytes::new( + LE, + section_offsets[section_index].associative_section, + ), + selection: section_offsets[section_index].selection, + reserved: 0, + // TODO: bigobj + high_number: U16Bytes::default(), + }; + buffer.write(&aux); + } + _ => { + debug_assert_eq!(number_of_aux_symbols, 0); + } + } + } + + // Write strtab section. + debug_assert_eq!(strtab_offset, buffer.len()); + buffer.write_bytes(&u32::to_le_bytes(strtab_len as u32)); + buffer.write_bytes(&strtab_data); + + debug_assert_eq!(offset, buffer.len()); + + Ok(()) + } +} + +// JamCRC +fn checksum(data: &[u8]) -> u32 { + let mut hasher = crc32fast::Hasher::new_with_initial(0xffff_ffff); + hasher.update(data); + !hasher.finalize() +} diff --git a/crux-mir/lib/object/src/write/elf/mod.rs b/crux-mir/lib/object/src/write/elf/mod.rs new file mode 100644 index 000000000..3a4f3716e --- /dev/null +++ b/crux-mir/lib/object/src/write/elf/mod.rs @@ -0,0 +1,9 @@ +//! Support for writing ELF files. +//! +//! Provides [`Writer`] for low level writing of ELF files. +//! This is also used to provide ELF support for [`write::Object`](crate::write::Object). + +mod object; + +mod writer; +pub use writer::*; diff --git a/crux-mir/lib/object/src/write/elf/object.rs b/crux-mir/lib/object/src/write/elf/object.rs new file mode 100644 index 000000000..8c1fa4717 --- /dev/null +++ b/crux-mir/lib/object/src/write/elf/object.rs @@ -0,0 +1,774 @@ +use alloc::vec::Vec; + +use crate::elf; +use crate::write::elf::writer::*; +use crate::write::string::StringId; +use crate::write::*; +use crate::AddressSize; + +#[derive(Clone, Copy)] +struct ComdatOffsets { + offset: usize, + str_id: StringId, +} + +#[derive(Clone, Copy)] +struct SectionOffsets { + index: SectionIndex, + offset: usize, + str_id: StringId, + reloc_offset: usize, + reloc_str_id: Option, +} + +#[derive(Default, Clone, Copy)] +struct SymbolOffsets { + index: SymbolIndex, + str_id: Option, +} + +impl<'a> Object<'a> { + pub(crate) fn elf_section_info( + &self, + section: StandardSection, + ) -> (&'static [u8], &'static [u8], SectionKind) { + match section { + StandardSection::Text => (&[], &b".text"[..], SectionKind::Text), + StandardSection::Data => (&[], &b".data"[..], SectionKind::Data), + StandardSection::ReadOnlyData | StandardSection::ReadOnlyString => { + (&[], &b".rodata"[..], SectionKind::ReadOnlyData) + } + StandardSection::ReadOnlyDataWithRel => (&[], b".data.rel.ro", SectionKind::Data), + StandardSection::UninitializedData => { + (&[], &b".bss"[..], SectionKind::UninitializedData) + } + StandardSection::Tls => (&[], &b".tdata"[..], SectionKind::Tls), + StandardSection::UninitializedTls => { + (&[], &b".tbss"[..], SectionKind::UninitializedTls) + } + StandardSection::TlsVariables => { + // Unsupported section. + (&[], &[], SectionKind::TlsVariables) + } + StandardSection::Common => { + // Unsupported section. + (&[], &[], SectionKind::Common) + } + } + } + + pub(crate) fn elf_subsection_name(&self, section: &[u8], value: &[u8]) -> Vec { + let mut name = section.to_vec(); + name.push(b'.'); + name.extend_from_slice(value); + name + } + + fn elf_has_relocation_addend(&self) -> Result { + Ok(match self.architecture { + Architecture::Aarch64 => true, + Architecture::Arm => false, + Architecture::Avr => true, + Architecture::Bpf => false, + Architecture::I386 => false, + Architecture::X86_64 => true, + Architecture::X86_64_X32 => true, + Architecture::Hexagon => true, + Architecture::LoongArch64 => true, + Architecture::Mips => false, + Architecture::Mips64 => true, + Architecture::Msp430 => true, + Architecture::PowerPc => true, + Architecture::PowerPc64 => true, + Architecture::Riscv64 => true, + Architecture::Riscv32 => true, + Architecture::S390x => true, + Architecture::Sparc64 => true, + _ => { + return Err(Error(format!( + "unimplemented architecture {:?}", + self.architecture + ))); + } + }) + } + + pub(crate) fn elf_fixup_relocation(&mut self, mut relocation: &mut Relocation) -> Result { + // Return true if we should use a section symbol to avoid preemption. + fn want_section_symbol(relocation: &Relocation, symbol: &Symbol) -> bool { + if symbol.scope != SymbolScope::Dynamic { + // Only dynamic symbols can be preemptible. + return false; + } + match symbol.kind { + SymbolKind::Text | SymbolKind::Data => {} + _ => return false, + } + match relocation.kind { + // Anything using GOT or PLT is preemptible. + // We also require that `Other` relocations must already be correct. + RelocationKind::Got + | RelocationKind::GotRelative + | RelocationKind::GotBaseRelative + | RelocationKind::PltRelative + | RelocationKind::Elf(_) => return false, + // Absolute relocations are preemptible for non-local data. + // TODO: not sure if this rule is exactly correct + // This rule was added to handle global data references in debuginfo. + // Maybe this should be a new relocation kind so that the caller can decide. + RelocationKind::Absolute => { + if symbol.kind == SymbolKind::Data { + return false; + } + } + _ => {} + } + true + } + + // Use section symbols for relocations where required to avoid preemption. + // Otherwise, the linker will fail with: + // relocation R_X86_64_PC32 against symbol `SomeSymbolName' can not be used when + // making a shared object; recompile with -fPIC + let symbol = &self.symbols[relocation.symbol.0]; + if want_section_symbol(relocation, symbol) { + if let Some(section) = symbol.section.id() { + relocation.addend += symbol.value as i64; + relocation.symbol = self.section_symbol(section); + } + } + + // Determine whether the addend is stored in the relocation or the data. + if self.elf_has_relocation_addend()? { + Ok(0) + } else { + let constant = relocation.addend; + relocation.addend = 0; + Ok(constant) + } + } + + pub(crate) fn elf_write(&self, buffer: &mut dyn WritableBuffer) -> Result<()> { + // Create reloc section header names so we can reference them. + let is_rela = self.elf_has_relocation_addend()?; + let reloc_names: Vec<_> = self + .sections + .iter() + .map(|section| { + let mut reloc_name = Vec::with_capacity( + if is_rela { ".rela".len() } else { ".rel".len() } + section.name.len(), + ); + if !section.relocations.is_empty() { + reloc_name.extend_from_slice(if is_rela { + &b".rela"[..] + } else { + &b".rel"[..] + }); + reloc_name.extend_from_slice(§ion.name); + } + reloc_name + }) + .collect(); + + // Start calculating offsets of everything. + let is_64 = match self.architecture.address_size().unwrap() { + AddressSize::U8 | AddressSize::U16 | AddressSize::U32 => false, + AddressSize::U64 => true, + }; + let mut writer = Writer::new(self.endian, is_64, buffer); + writer.reserve_file_header(); + + // Calculate size of section data. + let mut comdat_offsets = Vec::with_capacity(self.comdats.len()); + for comdat in &self.comdats { + if comdat.kind != ComdatKind::Any { + return Err(Error(format!( + "unsupported COMDAT symbol `{}` kind {:?}", + self.symbols[comdat.symbol.0].name().unwrap_or(""), + comdat.kind + ))); + } + + writer.reserve_section_index(); + let offset = writer.reserve_comdat(comdat.sections.len()); + let str_id = writer.add_section_name(b".group"); + comdat_offsets.push(ComdatOffsets { offset, str_id }); + } + let mut section_offsets = Vec::with_capacity(self.sections.len()); + for (section, reloc_name) in self.sections.iter().zip(reloc_names.iter()) { + let index = writer.reserve_section_index(); + let offset = writer.reserve(section.data.len(), section.align as usize); + let str_id = writer.add_section_name(§ion.name); + let mut reloc_str_id = None; + if !section.relocations.is_empty() { + writer.reserve_section_index(); + reloc_str_id = Some(writer.add_section_name(reloc_name)); + } + section_offsets.push(SectionOffsets { + index, + offset, + str_id, + // Relocation data is reserved later. + reloc_offset: 0, + reloc_str_id, + }); + } + + // Calculate index of symbols and add symbol strings to strtab. + let mut symbol_offsets = vec![SymbolOffsets::default(); self.symbols.len()]; + writer.reserve_null_symbol_index(); + // Local symbols must come before global. + for (index, symbol) in self.symbols.iter().enumerate() { + if symbol.is_local() { + let section_index = symbol.section.id().map(|s| section_offsets[s.0].index); + symbol_offsets[index].index = writer.reserve_symbol_index(section_index); + } + } + let symtab_num_local = writer.symbol_count(); + for (index, symbol) in self.symbols.iter().enumerate() { + if !symbol.is_local() { + let section_index = symbol.section.id().map(|s| section_offsets[s.0].index); + symbol_offsets[index].index = writer.reserve_symbol_index(section_index); + } + } + for (index, symbol) in self.symbols.iter().enumerate() { + if symbol.kind != SymbolKind::Section && !symbol.name.is_empty() { + symbol_offsets[index].str_id = Some(writer.add_string(&symbol.name)); + } + } + + // Calculate size of symbols. + writer.reserve_symtab_section_index(); + writer.reserve_symtab(); + if writer.symtab_shndx_needed() { + writer.reserve_symtab_shndx_section_index(); + } + writer.reserve_symtab_shndx(); + writer.reserve_strtab_section_index(); + writer.reserve_strtab(); + + // Calculate size of relocations. + for (index, section) in self.sections.iter().enumerate() { + let count = section.relocations.len(); + if count != 0 { + section_offsets[index].reloc_offset = writer.reserve_relocations(count, is_rela); + } + } + + // Calculate size of section headers. + writer.reserve_shstrtab_section_index(); + writer.reserve_shstrtab(); + writer.reserve_section_headers(); + + // Start writing. + let e_type = elf::ET_REL; + let e_machine = match self.architecture { + Architecture::Aarch64 => elf::EM_AARCH64, + Architecture::Arm => elf::EM_ARM, + Architecture::Avr => elf::EM_AVR, + Architecture::Bpf => elf::EM_BPF, + Architecture::I386 => elf::EM_386, + Architecture::X86_64 => elf::EM_X86_64, + Architecture::X86_64_X32 => elf::EM_X86_64, + Architecture::Hexagon => elf::EM_HEXAGON, + Architecture::LoongArch64 => elf::EM_LOONGARCH, + Architecture::Mips => elf::EM_MIPS, + Architecture::Mips64 => elf::EM_MIPS, + Architecture::Msp430 => elf::EM_MSP430, + Architecture::PowerPc => elf::EM_PPC, + Architecture::PowerPc64 => elf::EM_PPC64, + Architecture::Riscv32 => elf::EM_RISCV, + Architecture::Riscv64 => elf::EM_RISCV, + Architecture::S390x => elf::EM_S390, + Architecture::Sparc64 => elf::EM_SPARCV9, + _ => { + return Err(Error(format!( + "unimplemented architecture {:?}", + self.architecture + ))); + } + }; + let (os_abi, abi_version, e_flags) = if let FileFlags::Elf { + os_abi, + abi_version, + e_flags, + } = self.flags + { + (os_abi, abi_version, e_flags) + } else { + (elf::ELFOSABI_NONE, 0, 0) + }; + writer.write_file_header(&FileHeader { + os_abi, + abi_version, + e_type, + e_machine, + e_entry: 0, + e_flags, + })?; + + // Write section data. + for comdat in &self.comdats { + writer.write_comdat_header(); + for section in &comdat.sections { + writer.write_comdat_entry(section_offsets[section.0].index); + } + } + for (index, section) in self.sections.iter().enumerate() { + let len = section.data.len(); + if len != 0 { + writer.write_align(section.align as usize); + debug_assert_eq!(section_offsets[index].offset, writer.len()); + writer.write(§ion.data); + } + } + + // Write symbols. + writer.write_null_symbol(); + let mut write_symbol = |index: usize, symbol: &Symbol| -> Result<()> { + let st_info = if let SymbolFlags::Elf { st_info, .. } = symbol.flags { + st_info + } else { + let st_type = match symbol.kind { + SymbolKind::Null => elf::STT_NOTYPE, + SymbolKind::Text => { + if symbol.is_undefined() { + elf::STT_NOTYPE + } else { + elf::STT_FUNC + } + } + SymbolKind::Data => { + if symbol.is_undefined() { + elf::STT_NOTYPE + } else if symbol.is_common() { + elf::STT_COMMON + } else { + elf::STT_OBJECT + } + } + SymbolKind::Section => elf::STT_SECTION, + SymbolKind::File => elf::STT_FILE, + SymbolKind::Tls => elf::STT_TLS, + SymbolKind::Label => elf::STT_NOTYPE, + SymbolKind::Unknown => { + if symbol.is_undefined() { + elf::STT_NOTYPE + } else { + return Err(Error(format!( + "unimplemented symbol `{}` kind {:?}", + symbol.name().unwrap_or(""), + symbol.kind + ))); + } + } + }; + let st_bind = if symbol.weak { + elf::STB_WEAK + } else if symbol.is_undefined() { + elf::STB_GLOBAL + } else if symbol.is_local() { + elf::STB_LOCAL + } else { + elf::STB_GLOBAL + }; + (st_bind << 4) + st_type + }; + let st_other = if let SymbolFlags::Elf { st_other, .. } = symbol.flags { + st_other + } else if symbol.scope == SymbolScope::Linkage { + elf::STV_HIDDEN + } else { + elf::STV_DEFAULT + }; + let (st_shndx, section) = match symbol.section { + SymbolSection::None => { + debug_assert_eq!(symbol.kind, SymbolKind::File); + (elf::SHN_ABS, None) + } + SymbolSection::Undefined => (elf::SHN_UNDEF, None), + SymbolSection::Absolute => (elf::SHN_ABS, None), + SymbolSection::Common => (elf::SHN_COMMON, None), + SymbolSection::Section(id) => (0, Some(section_offsets[id.0].index)), + }; + writer.write_symbol(&Sym { + name: symbol_offsets[index].str_id, + section, + st_info, + st_other, + st_shndx, + st_value: symbol.value, + st_size: symbol.size, + }); + Ok(()) + }; + for (index, symbol) in self.symbols.iter().enumerate() { + if symbol.is_local() { + write_symbol(index, symbol)?; + } + } + for (index, symbol) in self.symbols.iter().enumerate() { + if !symbol.is_local() { + write_symbol(index, symbol)?; + } + } + writer.write_symtab_shndx(); + writer.write_strtab(); + + // Write relocations. + for (index, section) in self.sections.iter().enumerate() { + if !section.relocations.is_empty() { + writer.write_align_relocation(); + debug_assert_eq!(section_offsets[index].reloc_offset, writer.len()); + for reloc in §ion.relocations { + let r_type = match self.architecture { + Architecture::Aarch64 => match (reloc.kind, reloc.encoding, reloc.size) { + (RelocationKind::Absolute, RelocationEncoding::Generic, 64) => { + elf::R_AARCH64_ABS64 + } + (RelocationKind::Absolute, RelocationEncoding::Generic, 32) => { + elf::R_AARCH64_ABS32 + } + (RelocationKind::Absolute, RelocationEncoding::Generic, 16) => { + elf::R_AARCH64_ABS16 + } + (RelocationKind::Relative, RelocationEncoding::Generic, 64) => { + elf::R_AARCH64_PREL64 + } + (RelocationKind::Relative, RelocationEncoding::Generic, 32) => { + elf::R_AARCH64_PREL32 + } + (RelocationKind::Relative, RelocationEncoding::Generic, 16) => { + elf::R_AARCH64_PREL16 + } + (RelocationKind::Relative, RelocationEncoding::AArch64Call, 26) + | (RelocationKind::PltRelative, RelocationEncoding::AArch64Call, 26) => { + elf::R_AARCH64_CALL26 + } + (RelocationKind::Elf(x), _, _) => x, + _ => { + return Err(Error(format!("unimplemented relocation {:?}", reloc))); + } + }, + Architecture::Arm => match (reloc.kind, reloc.encoding, reloc.size) { + (RelocationKind::Absolute, _, 32) => elf::R_ARM_ABS32, + (RelocationKind::Elf(x), _, _) => x, + _ => { + return Err(Error(format!("unimplemented relocation {:?}", reloc))); + } + }, + Architecture::Avr => match (reloc.kind, reloc.encoding, reloc.size) { + (RelocationKind::Absolute, _, 32) => elf::R_AVR_32, + (RelocationKind::Absolute, _, 16) => elf::R_AVR_16, + (RelocationKind::Elf(x), _, _) => x, + _ => { + return Err(Error(format!("unimplemented relocation {:?}", reloc))); + } + }, + Architecture::Bpf => match (reloc.kind, reloc.encoding, reloc.size) { + (RelocationKind::Absolute, _, 64) => elf::R_BPF_64_64, + (RelocationKind::Absolute, _, 32) => elf::R_BPF_64_32, + (RelocationKind::Elf(x), _, _) => x, + _ => { + return Err(Error(format!("unimplemented relocation {:?}", reloc))); + } + }, + Architecture::I386 => match (reloc.kind, reloc.size) { + (RelocationKind::Absolute, 32) => elf::R_386_32, + (RelocationKind::Relative, 32) => elf::R_386_PC32, + (RelocationKind::Got, 32) => elf::R_386_GOT32, + (RelocationKind::PltRelative, 32) => elf::R_386_PLT32, + (RelocationKind::GotBaseOffset, 32) => elf::R_386_GOTOFF, + (RelocationKind::GotBaseRelative, 32) => elf::R_386_GOTPC, + (RelocationKind::Absolute, 16) => elf::R_386_16, + (RelocationKind::Relative, 16) => elf::R_386_PC16, + (RelocationKind::Absolute, 8) => elf::R_386_8, + (RelocationKind::Relative, 8) => elf::R_386_PC8, + (RelocationKind::Elf(x), _) => x, + _ => { + return Err(Error(format!("unimplemented relocation {:?}", reloc))); + } + }, + Architecture::X86_64 | Architecture::X86_64_X32 => { + match (reloc.kind, reloc.encoding, reloc.size) { + (RelocationKind::Absolute, RelocationEncoding::Generic, 64) => { + elf::R_X86_64_64 + } + (RelocationKind::Relative, _, 32) => elf::R_X86_64_PC32, + (RelocationKind::Got, _, 32) => elf::R_X86_64_GOT32, + (RelocationKind::PltRelative, _, 32) => elf::R_X86_64_PLT32, + (RelocationKind::GotRelative, _, 32) => elf::R_X86_64_GOTPCREL, + (RelocationKind::Absolute, RelocationEncoding::Generic, 32) => { + elf::R_X86_64_32 + } + (RelocationKind::Absolute, RelocationEncoding::X86Signed, 32) => { + elf::R_X86_64_32S + } + (RelocationKind::Absolute, _, 16) => elf::R_X86_64_16, + (RelocationKind::Relative, _, 16) => elf::R_X86_64_PC16, + (RelocationKind::Absolute, _, 8) => elf::R_X86_64_8, + (RelocationKind::Relative, _, 8) => elf::R_X86_64_PC8, + (RelocationKind::Elf(x), _, _) => x, + _ => { + return Err(Error(format!( + "unimplemented relocation {:?}", + reloc + ))); + } + } + } + Architecture::Hexagon => match (reloc.kind, reloc.encoding, reloc.size) { + (RelocationKind::Absolute, _, 32) => elf::R_HEX_32, + (RelocationKind::Elf(x), _, _) => x, + _ => { + return Err(Error(format!("unimplemented relocation {:?}", reloc))); + } + }, + Architecture::LoongArch64 => match (reloc.kind, reloc.encoding, reloc.size) + { + (RelocationKind::Absolute, _, 32) => elf::R_LARCH_32, + (RelocationKind::Absolute, _, 64) => elf::R_LARCH_64, + (RelocationKind::Elf(x), _, _) => x, + _ => { + return Err(Error(format!("unimplemented relocation {:?}", reloc))); + } + }, + Architecture::Mips | Architecture::Mips64 => { + match (reloc.kind, reloc.encoding, reloc.size) { + (RelocationKind::Absolute, _, 16) => elf::R_MIPS_16, + (RelocationKind::Absolute, _, 32) => elf::R_MIPS_32, + (RelocationKind::Absolute, _, 64) => elf::R_MIPS_64, + (RelocationKind::Elf(x), _, _) => x, + _ => { + return Err(Error(format!( + "unimplemented relocation {:?}", + reloc + ))); + } + } + } + Architecture::Msp430 => match (reloc.kind, reloc.encoding, reloc.size) { + (RelocationKind::Absolute, _, 32) => elf::R_MSP430_32, + (RelocationKind::Absolute, _, 16) => elf::R_MSP430_16_BYTE, + (RelocationKind::Elf(x), _, _) => x, + _ => { + return Err(Error(format!("unimplemented relocation {:?}", reloc))); + } + }, + Architecture::PowerPc => match (reloc.kind, reloc.encoding, reloc.size) { + (RelocationKind::Absolute, _, 32) => elf::R_PPC_ADDR32, + (RelocationKind::Elf(x), _, _) => x, + _ => { + return Err(Error(format!("unimplemented relocation {:?}", reloc))); + } + }, + Architecture::PowerPc64 => match (reloc.kind, reloc.encoding, reloc.size) { + (RelocationKind::Absolute, _, 32) => elf::R_PPC64_ADDR32, + (RelocationKind::Absolute, _, 64) => elf::R_PPC64_ADDR64, + (RelocationKind::Elf(x), _, _) => x, + _ => { + return Err(Error(format!("unimplemented relocation {:?}", reloc))); + } + }, + Architecture::Riscv32 | Architecture::Riscv64 => { + match (reloc.kind, reloc.encoding, reloc.size) { + (RelocationKind::Absolute, _, 32) => elf::R_RISCV_32, + (RelocationKind::Absolute, _, 64) => elf::R_RISCV_64, + (RelocationKind::Elf(x), _, _) => x, + _ => { + return Err(Error(format!( + "unimplemented relocation {:?}", + reloc + ))); + } + } + } + Architecture::S390x => match (reloc.kind, reloc.encoding, reloc.size) { + (RelocationKind::Absolute, RelocationEncoding::Generic, 8) => { + elf::R_390_8 + } + (RelocationKind::Absolute, RelocationEncoding::Generic, 16) => { + elf::R_390_16 + } + (RelocationKind::Absolute, RelocationEncoding::Generic, 32) => { + elf::R_390_32 + } + (RelocationKind::Absolute, RelocationEncoding::Generic, 64) => { + elf::R_390_64 + } + (RelocationKind::Relative, RelocationEncoding::Generic, 16) => { + elf::R_390_PC16 + } + (RelocationKind::Relative, RelocationEncoding::Generic, 32) => { + elf::R_390_PC32 + } + (RelocationKind::Relative, RelocationEncoding::Generic, 64) => { + elf::R_390_PC64 + } + (RelocationKind::Relative, RelocationEncoding::S390xDbl, 16) => { + elf::R_390_PC16DBL + } + (RelocationKind::Relative, RelocationEncoding::S390xDbl, 32) => { + elf::R_390_PC32DBL + } + (RelocationKind::PltRelative, RelocationEncoding::S390xDbl, 16) => { + elf::R_390_PLT16DBL + } + (RelocationKind::PltRelative, RelocationEncoding::S390xDbl, 32) => { + elf::R_390_PLT32DBL + } + (RelocationKind::Got, RelocationEncoding::Generic, 16) => { + elf::R_390_GOT16 + } + (RelocationKind::Got, RelocationEncoding::Generic, 32) => { + elf::R_390_GOT32 + } + (RelocationKind::Got, RelocationEncoding::Generic, 64) => { + elf::R_390_GOT64 + } + (RelocationKind::GotRelative, RelocationEncoding::S390xDbl, 32) => { + elf::R_390_GOTENT + } + (RelocationKind::GotBaseOffset, RelocationEncoding::Generic, 16) => { + elf::R_390_GOTOFF16 + } + (RelocationKind::GotBaseOffset, RelocationEncoding::Generic, 32) => { + elf::R_390_GOTOFF32 + } + (RelocationKind::GotBaseOffset, RelocationEncoding::Generic, 64) => { + elf::R_390_GOTOFF64 + } + (RelocationKind::GotBaseRelative, RelocationEncoding::Generic, 64) => { + elf::R_390_GOTPC + } + (RelocationKind::GotBaseRelative, RelocationEncoding::S390xDbl, 32) => { + elf::R_390_GOTPCDBL + } + (RelocationKind::Elf(x), _, _) => x, + _ => { + return Err(Error(format!("unimplemented relocation {:?}", reloc))); + } + }, + Architecture::Sparc64 => match (reloc.kind, reloc.encoding, reloc.size) { + // TODO: use R_SPARC_32/R_SPARC_64 if aligned. + (RelocationKind::Absolute, _, 32) => elf::R_SPARC_UA32, + (RelocationKind::Absolute, _, 64) => elf::R_SPARC_UA64, + (RelocationKind::Elf(x), _, _) => x, + _ => { + return Err(Error(format!("unimplemented relocation {:?}", reloc))); + } + }, + _ => { + if let RelocationKind::Elf(x) = reloc.kind { + x + } else { + return Err(Error(format!("unimplemented relocation {:?}", reloc))); + } + } + }; + let r_sym = symbol_offsets[reloc.symbol.0].index.0; + writer.write_relocation( + is_rela, + &Rel { + r_offset: reloc.offset, + r_sym, + r_type, + r_addend: reloc.addend, + }, + ); + } + } + } + + writer.write_shstrtab(); + + // Write section headers. + writer.write_null_section_header(); + + let symtab_index = writer.symtab_index(); + for (comdat, comdat_offset) in self.comdats.iter().zip(comdat_offsets.iter()) { + writer.write_comdat_section_header( + comdat_offset.str_id, + symtab_index, + symbol_offsets[comdat.symbol.0].index, + comdat_offset.offset, + comdat.sections.len(), + ); + } + for (index, section) in self.sections.iter().enumerate() { + let sh_type = match section.kind { + SectionKind::UninitializedData | SectionKind::UninitializedTls => elf::SHT_NOBITS, + SectionKind::Note => elf::SHT_NOTE, + SectionKind::Elf(sh_type) => sh_type, + _ => elf::SHT_PROGBITS, + }; + let sh_flags = if let SectionFlags::Elf { sh_flags } = section.flags { + sh_flags + } else { + match section.kind { + SectionKind::Text => elf::SHF_ALLOC | elf::SHF_EXECINSTR, + SectionKind::Data => elf::SHF_ALLOC | elf::SHF_WRITE, + SectionKind::Tls => elf::SHF_ALLOC | elf::SHF_WRITE | elf::SHF_TLS, + SectionKind::UninitializedData => elf::SHF_ALLOC | elf::SHF_WRITE, + SectionKind::UninitializedTls => elf::SHF_ALLOC | elf::SHF_WRITE | elf::SHF_TLS, + SectionKind::ReadOnlyData => elf::SHF_ALLOC, + SectionKind::ReadOnlyString => { + elf::SHF_ALLOC | elf::SHF_STRINGS | elf::SHF_MERGE + } + SectionKind::OtherString => elf::SHF_STRINGS | elf::SHF_MERGE, + SectionKind::Other + | SectionKind::Debug + | SectionKind::Metadata + | SectionKind::Linker + | SectionKind::Note + | SectionKind::Elf(_) => 0, + SectionKind::Unknown | SectionKind::Common | SectionKind::TlsVariables => { + return Err(Error(format!( + "unimplemented section `{}` kind {:?}", + section.name().unwrap_or(""), + section.kind + ))); + } + } + .into() + }; + // TODO: not sure if this is correct, maybe user should determine this + let sh_entsize = match section.kind { + SectionKind::ReadOnlyString | SectionKind::OtherString => 1, + _ => 0, + }; + writer.write_section_header(&SectionHeader { + name: Some(section_offsets[index].str_id), + sh_type, + sh_flags, + sh_addr: 0, + sh_offset: section_offsets[index].offset as u64, + sh_size: section.size, + sh_link: 0, + sh_info: 0, + sh_addralign: section.align, + sh_entsize, + }); + + if !section.relocations.is_empty() { + writer.write_relocation_section_header( + section_offsets[index].reloc_str_id.unwrap(), + section_offsets[index].index, + symtab_index, + section_offsets[index].reloc_offset, + section.relocations.len(), + is_rela, + ); + } + } + + writer.write_symtab_section_header(symtab_num_local); + writer.write_symtab_shndx_section_header(); + writer.write_strtab_section_header(); + writer.write_shstrtab_section_header(); + + debug_assert_eq!(writer.reserved_len(), writer.len()); + + Ok(()) + } +} diff --git a/crux-mir/lib/object/src/write/elf/writer.rs b/crux-mir/lib/object/src/write/elf/writer.rs new file mode 100644 index 000000000..3c9d85a12 --- /dev/null +++ b/crux-mir/lib/object/src/write/elf/writer.rs @@ -0,0 +1,1955 @@ +//! Helper for writing ELF files. +use alloc::string::String; +use alloc::vec::Vec; +use core::mem; + +use crate::elf; +use crate::endian::*; +use crate::write::string::{StringId, StringTable}; +use crate::write::util; +use crate::write::{Error, Result, WritableBuffer}; + +/// The index of an ELF section. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct SectionIndex(pub u32); + +/// The index of an ELF symbol. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct SymbolIndex(pub u32); + +/// A helper for writing ELF files. +/// +/// Writing uses a two phase approach. The first phase builds up all of the information +/// that may need to be known ahead of time: +/// - build string tables +/// - reserve section indices +/// - reserve symbol indices +/// - reserve file ranges for headers and sections +/// +/// Some of the information has ordering requirements. For example, strings must be added +/// to string tables before reserving the file range for the string table. Symbol indices +/// must be reserved after reserving the section indices they reference. There are debug +/// asserts to check some of these requirements. +/// +/// The second phase writes everything out in order. Thus the caller must ensure writing +/// is in the same order that file ranges were reserved. There are debug asserts to assist +/// with checking this. +#[allow(missing_debug_implementations)] +pub struct Writer<'a> { + endian: Endianness, + is_64: bool, + is_mips64el: bool, + elf_align: usize, + + buffer: &'a mut dyn WritableBuffer, + len: usize, + + segment_offset: usize, + segment_num: u32, + + section_offset: usize, + section_num: u32, + + shstrtab: StringTable<'a>, + shstrtab_str_id: Option, + shstrtab_index: SectionIndex, + shstrtab_offset: usize, + shstrtab_data: Vec, + + need_strtab: bool, + strtab: StringTable<'a>, + strtab_str_id: Option, + strtab_index: SectionIndex, + strtab_offset: usize, + strtab_data: Vec, + + symtab_str_id: Option, + symtab_index: SectionIndex, + symtab_offset: usize, + symtab_num: u32, + + need_symtab_shndx: bool, + symtab_shndx_str_id: Option, + symtab_shndx_offset: usize, + symtab_shndx_data: Vec, + + need_dynstr: bool, + dynstr: StringTable<'a>, + dynstr_str_id: Option, + dynstr_index: SectionIndex, + dynstr_offset: usize, + dynstr_data: Vec, + + dynsym_str_id: Option, + dynsym_index: SectionIndex, + dynsym_offset: usize, + dynsym_num: u32, + + dynamic_str_id: Option, + dynamic_offset: usize, + dynamic_num: usize, + + hash_str_id: Option, + hash_offset: usize, + hash_size: usize, + + gnu_hash_str_id: Option, + gnu_hash_offset: usize, + gnu_hash_size: usize, + + gnu_versym_str_id: Option, + gnu_versym_offset: usize, + + gnu_verdef_str_id: Option, + gnu_verdef_offset: usize, + gnu_verdef_size: usize, + gnu_verdef_count: u16, + gnu_verdef_remaining: u16, + gnu_verdaux_remaining: u16, + + gnu_verneed_str_id: Option, + gnu_verneed_offset: usize, + gnu_verneed_size: usize, + gnu_verneed_count: u16, + gnu_verneed_remaining: u16, + gnu_vernaux_remaining: u16, +} + +impl<'a> Writer<'a> { + /// Create a new `Writer` for the given endianness and ELF class. + pub fn new(endian: Endianness, is_64: bool, buffer: &'a mut dyn WritableBuffer) -> Self { + let elf_align = if is_64 { 8 } else { 4 }; + Writer { + endian, + is_64, + // Determined later. + is_mips64el: false, + elf_align, + + buffer, + len: 0, + + segment_offset: 0, + segment_num: 0, + + section_offset: 0, + section_num: 0, + + shstrtab: StringTable::default(), + shstrtab_str_id: None, + shstrtab_index: SectionIndex(0), + shstrtab_offset: 0, + shstrtab_data: Vec::new(), + + need_strtab: false, + strtab: StringTable::default(), + strtab_str_id: None, + strtab_index: SectionIndex(0), + strtab_offset: 0, + strtab_data: Vec::new(), + + symtab_str_id: None, + symtab_index: SectionIndex(0), + symtab_offset: 0, + symtab_num: 0, + + need_symtab_shndx: false, + symtab_shndx_str_id: None, + symtab_shndx_offset: 0, + symtab_shndx_data: Vec::new(), + + need_dynstr: false, + dynstr: StringTable::default(), + dynstr_str_id: None, + dynstr_index: SectionIndex(0), + dynstr_offset: 0, + dynstr_data: Vec::new(), + + dynsym_str_id: None, + dynsym_index: SectionIndex(0), + dynsym_offset: 0, + dynsym_num: 0, + + dynamic_str_id: None, + dynamic_offset: 0, + dynamic_num: 0, + + hash_str_id: None, + hash_offset: 0, + hash_size: 0, + + gnu_hash_str_id: None, + gnu_hash_offset: 0, + gnu_hash_size: 0, + + gnu_versym_str_id: None, + gnu_versym_offset: 0, + + gnu_verdef_str_id: None, + gnu_verdef_offset: 0, + gnu_verdef_size: 0, + gnu_verdef_count: 0, + gnu_verdef_remaining: 0, + gnu_verdaux_remaining: 0, + + gnu_verneed_str_id: None, + gnu_verneed_offset: 0, + gnu_verneed_size: 0, + gnu_verneed_count: 0, + gnu_verneed_remaining: 0, + gnu_vernaux_remaining: 0, + } + } + + /// Return the current file length that has been reserved. + pub fn reserved_len(&self) -> usize { + self.len + } + + /// Return the current file length that has been written. + #[allow(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + self.buffer.len() + } + + /// Reserve a file range with the given size and starting alignment. + /// + /// Returns the aligned offset of the start of the range. + pub fn reserve(&mut self, len: usize, align_start: usize) -> usize { + if len == 0 { + return self.len; + } + self.len = util::align(self.len, align_start); + let offset = self.len; + self.len += len; + offset + } + + /// Write alignment padding bytes. + pub fn write_align(&mut self, align_start: usize) { + util::write_align(self.buffer, align_start); + } + + /// Write data. + /// + /// This is typically used to write section data. + pub fn write(&mut self, data: &[u8]) { + self.buffer.write_bytes(data); + } + + /// Reserve the file range up to the given file offset. + pub fn reserve_until(&mut self, offset: usize) { + debug_assert!(self.len <= offset); + self.len = offset; + } + + /// Write padding up to the given file offset. + pub fn pad_until(&mut self, offset: usize) { + debug_assert!(self.buffer.len() <= offset); + self.buffer.resize(offset); + } + + fn file_header_size(&self) -> usize { + if self.is_64 { + mem::size_of::>() + } else { + mem::size_of::>() + } + } + + /// Reserve the range for the file header. + /// + /// This must be at the start of the file. + pub fn reserve_file_header(&mut self) { + debug_assert_eq!(self.len, 0); + self.reserve(self.file_header_size(), 1); + } + + /// Write the file header. + /// + /// This must be at the start of the file. + /// + /// Fields that can be derived from known information are automatically set by this function. + pub fn write_file_header(&mut self, header: &FileHeader) -> Result<()> { + debug_assert_eq!(self.buffer.len(), 0); + + self.is_mips64el = + self.is_64 && self.endian.is_little_endian() && header.e_machine == elf::EM_MIPS; + + // Start writing. + self.buffer + .reserve(self.len) + .map_err(|_| Error(String::from("Cannot allocate buffer")))?; + + // Write file header. + let e_ident = elf::Ident { + magic: elf::ELFMAG, + class: if self.is_64 { + elf::ELFCLASS64 + } else { + elf::ELFCLASS32 + }, + data: if self.endian.is_little_endian() { + elf::ELFDATA2LSB + } else { + elf::ELFDATA2MSB + }, + version: elf::EV_CURRENT, + os_abi: header.os_abi, + abi_version: header.abi_version, + padding: [0; 7], + }; + + let e_ehsize = self.file_header_size() as u16; + + let e_phoff = self.segment_offset as u64; + let e_phentsize = if self.segment_num == 0 { + 0 + } else { + self.program_header_size() as u16 + }; + // TODO: overflow + let e_phnum = self.segment_num as u16; + + let e_shoff = self.section_offset as u64; + let e_shentsize = if self.section_num == 0 { + 0 + } else { + self.section_header_size() as u16 + }; + let e_shnum = if self.section_num >= elf::SHN_LORESERVE.into() { + 0 + } else { + self.section_num as u16 + }; + let e_shstrndx = if self.shstrtab_index.0 >= elf::SHN_LORESERVE.into() { + elf::SHN_XINDEX + } else { + self.shstrtab_index.0 as u16 + }; + + let endian = self.endian; + if self.is_64 { + let file = elf::FileHeader64 { + e_ident, + e_type: U16::new(endian, header.e_type), + e_machine: U16::new(endian, header.e_machine), + e_version: U32::new(endian, elf::EV_CURRENT.into()), + e_entry: U64::new(endian, header.e_entry), + e_phoff: U64::new(endian, e_phoff), + e_shoff: U64::new(endian, e_shoff), + e_flags: U32::new(endian, header.e_flags), + e_ehsize: U16::new(endian, e_ehsize), + e_phentsize: U16::new(endian, e_phentsize), + e_phnum: U16::new(endian, e_phnum), + e_shentsize: U16::new(endian, e_shentsize), + e_shnum: U16::new(endian, e_shnum), + e_shstrndx: U16::new(endian, e_shstrndx), + }; + self.buffer.write(&file) + } else { + let file = elf::FileHeader32 { + e_ident, + e_type: U16::new(endian, header.e_type), + e_machine: U16::new(endian, header.e_machine), + e_version: U32::new(endian, elf::EV_CURRENT.into()), + e_entry: U32::new(endian, header.e_entry as u32), + e_phoff: U32::new(endian, e_phoff as u32), + e_shoff: U32::new(endian, e_shoff as u32), + e_flags: U32::new(endian, header.e_flags), + e_ehsize: U16::new(endian, e_ehsize), + e_phentsize: U16::new(endian, e_phentsize), + e_phnum: U16::new(endian, e_phnum), + e_shentsize: U16::new(endian, e_shentsize), + e_shnum: U16::new(endian, e_shnum), + e_shstrndx: U16::new(endian, e_shstrndx), + }; + self.buffer.write(&file); + } + + Ok(()) + } + + fn program_header_size(&self) -> usize { + if self.is_64 { + mem::size_of::>() + } else { + mem::size_of::>() + } + } + + /// Reserve the range for the program headers. + pub fn reserve_program_headers(&mut self, num: u32) { + debug_assert_eq!(self.segment_offset, 0); + if num == 0 { + return; + } + self.segment_num = num; + self.segment_offset = + self.reserve(num as usize * self.program_header_size(), self.elf_align); + } + + /// Write alignment padding bytes prior to the program headers. + pub fn write_align_program_headers(&mut self) { + if self.segment_offset == 0 { + return; + } + util::write_align(self.buffer, self.elf_align); + debug_assert_eq!(self.segment_offset, self.buffer.len()); + } + + /// Write a program header. + pub fn write_program_header(&mut self, header: &ProgramHeader) { + let endian = self.endian; + if self.is_64 { + let header = elf::ProgramHeader64 { + p_type: U32::new(endian, header.p_type), + p_flags: U32::new(endian, header.p_flags), + p_offset: U64::new(endian, header.p_offset), + p_vaddr: U64::new(endian, header.p_vaddr), + p_paddr: U64::new(endian, header.p_paddr), + p_filesz: U64::new(endian, header.p_filesz), + p_memsz: U64::new(endian, header.p_memsz), + p_align: U64::new(endian, header.p_align), + }; + self.buffer.write(&header); + } else { + let header = elf::ProgramHeader32 { + p_type: U32::new(endian, header.p_type), + p_offset: U32::new(endian, header.p_offset as u32), + p_vaddr: U32::new(endian, header.p_vaddr as u32), + p_paddr: U32::new(endian, header.p_paddr as u32), + p_filesz: U32::new(endian, header.p_filesz as u32), + p_memsz: U32::new(endian, header.p_memsz as u32), + p_flags: U32::new(endian, header.p_flags), + p_align: U32::new(endian, header.p_align as u32), + }; + self.buffer.write(&header); + } + } + + /// Reserve the section index for the null section header. + /// + /// The null section header is usually automatically reserved, + /// but this can be used to force an empty section table. + /// + /// This must be called before [`Self::reserve_section_headers`]. + pub fn reserve_null_section_index(&mut self) -> SectionIndex { + debug_assert_eq!(self.section_num, 0); + if self.section_num == 0 { + self.section_num = 1; + } + SectionIndex(0) + } + + /// Reserve a section table index. + /// + /// Automatically also reserves the null section header if required. + /// + /// This must be called before [`Self::reserve_section_headers`]. + pub fn reserve_section_index(&mut self) -> SectionIndex { + debug_assert_eq!(self.section_offset, 0); + if self.section_num == 0 { + self.section_num = 1; + } + let index = self.section_num; + self.section_num += 1; + SectionIndex(index) + } + + fn section_header_size(&self) -> usize { + if self.is_64 { + mem::size_of::>() + } else { + mem::size_of::>() + } + } + + /// Reserve the range for the section headers. + /// + /// This function does nothing if no sections were reserved. + /// This must be called after [`Self::reserve_section_index`] + /// and other functions that reserve section indices. + pub fn reserve_section_headers(&mut self) { + debug_assert_eq!(self.section_offset, 0); + if self.section_num == 0 { + return; + } + self.section_offset = self.reserve( + self.section_num as usize * self.section_header_size(), + self.elf_align, + ); + } + + /// Write the null section header. + /// + /// This must be the first section header that is written. + /// This function does nothing if no sections were reserved. + pub fn write_null_section_header(&mut self) { + if self.section_num == 0 { + return; + } + util::write_align(self.buffer, self.elf_align); + debug_assert_eq!(self.section_offset, self.buffer.len()); + self.write_section_header(&SectionHeader { + name: None, + sh_type: 0, + sh_flags: 0, + sh_addr: 0, + sh_offset: 0, + sh_size: if self.section_num >= elf::SHN_LORESERVE.into() { + self.section_num.into() + } else { + 0 + }, + sh_link: if self.shstrtab_index.0 >= elf::SHN_LORESERVE.into() { + self.shstrtab_index.0 + } else { + 0 + }, + // TODO: e_phnum overflow + sh_info: 0, + sh_addralign: 0, + sh_entsize: 0, + }); + } + + /// Write a section header. + pub fn write_section_header(&mut self, section: &SectionHeader) { + let sh_name = if let Some(name) = section.name { + self.shstrtab.get_offset(name) as u32 + } else { + 0 + }; + let endian = self.endian; + if self.is_64 { + let section = elf::SectionHeader64 { + sh_name: U32::new(endian, sh_name), + sh_type: U32::new(endian, section.sh_type), + sh_flags: U64::new(endian, section.sh_flags), + sh_addr: U64::new(endian, section.sh_addr), + sh_offset: U64::new(endian, section.sh_offset), + sh_size: U64::new(endian, section.sh_size), + sh_link: U32::new(endian, section.sh_link), + sh_info: U32::new(endian, section.sh_info), + sh_addralign: U64::new(endian, section.sh_addralign), + sh_entsize: U64::new(endian, section.sh_entsize), + }; + self.buffer.write(§ion); + } else { + let section = elf::SectionHeader32 { + sh_name: U32::new(endian, sh_name), + sh_type: U32::new(endian, section.sh_type), + sh_flags: U32::new(endian, section.sh_flags as u32), + sh_addr: U32::new(endian, section.sh_addr as u32), + sh_offset: U32::new(endian, section.sh_offset as u32), + sh_size: U32::new(endian, section.sh_size as u32), + sh_link: U32::new(endian, section.sh_link), + sh_info: U32::new(endian, section.sh_info), + sh_addralign: U32::new(endian, section.sh_addralign as u32), + sh_entsize: U32::new(endian, section.sh_entsize as u32), + }; + self.buffer.write(§ion); + } + } + + /// Add a section name to the section header string table. + /// + /// This will be stored in the `.shstrtab` section. + /// + /// This must be called before [`Self::reserve_shstrtab`]. + pub fn add_section_name(&mut self, name: &'a [u8]) -> StringId { + debug_assert_eq!(self.shstrtab_offset, 0); + self.shstrtab.add(name) + } + + /// Reserve the range for the section header string table. + /// + /// This range is used for a section named `.shstrtab`. + /// + /// This function does nothing if no sections were reserved. + /// This must be called after [`Self::add_section_name`]. + /// and other functions that reserve section names and indices. + pub fn reserve_shstrtab(&mut self) { + debug_assert_eq!(self.shstrtab_offset, 0); + if self.section_num == 0 { + return; + } + // Start with null section name. + self.shstrtab_data = vec![0]; + self.shstrtab.write(1, &mut self.shstrtab_data); + self.shstrtab_offset = self.reserve(self.shstrtab_data.len(), 1); + } + + /// Write the section header string table. + /// + /// This function does nothing if the section was not reserved. + pub fn write_shstrtab(&mut self) { + if self.shstrtab_offset == 0 { + return; + } + debug_assert_eq!(self.shstrtab_offset, self.buffer.len()); + self.buffer.write_bytes(&self.shstrtab_data); + } + + /// Reserve the section index for the section header string table. + /// + /// This must be called before [`Self::reserve_shstrtab`] + /// and [`Self::reserve_section_headers`]. + pub fn reserve_shstrtab_section_index(&mut self) -> SectionIndex { + debug_assert_eq!(self.shstrtab_index, SectionIndex(0)); + self.shstrtab_str_id = Some(self.add_section_name(&b".shstrtab"[..])); + self.shstrtab_index = self.reserve_section_index(); + self.shstrtab_index + } + + /// Write the section header for the section header string table. + /// + /// This function does nothing if the section index was not reserved. + pub fn write_shstrtab_section_header(&mut self) { + if self.shstrtab_index == SectionIndex(0) { + return; + } + self.write_section_header(&SectionHeader { + name: self.shstrtab_str_id, + sh_type: elf::SHT_STRTAB, + sh_flags: 0, + sh_addr: 0, + sh_offset: self.shstrtab_offset as u64, + sh_size: self.shstrtab_data.len() as u64, + sh_link: 0, + sh_info: 0, + sh_addralign: 1, + sh_entsize: 0, + }); + } + + /// Add a string to the string table. + /// + /// This will be stored in the `.strtab` section. + /// + /// This must be called before [`Self::reserve_strtab`]. + pub fn add_string(&mut self, name: &'a [u8]) -> StringId { + debug_assert_eq!(self.strtab_offset, 0); + self.need_strtab = true; + self.strtab.add(name) + } + + /// Return true if `.strtab` is needed. + pub fn strtab_needed(&self) -> bool { + self.need_strtab + } + + /// Reserve the range for the string table. + /// + /// This range is used for a section named `.strtab`. + /// + /// This function does nothing if no strings or symbols were defined. + /// This must be called after [`Self::add_string`]. + pub fn reserve_strtab(&mut self) { + debug_assert_eq!(self.strtab_offset, 0); + if !self.need_strtab { + return; + } + // Start with null string. + self.strtab_data = vec![0]; + self.strtab.write(1, &mut self.strtab_data); + self.strtab_offset = self.reserve(self.strtab_data.len(), 1); + } + + /// Write the string table. + /// + /// This function does nothing if the section was not reserved. + pub fn write_strtab(&mut self) { + if self.strtab_offset == 0 { + return; + } + debug_assert_eq!(self.strtab_offset, self.buffer.len()); + self.buffer.write_bytes(&self.strtab_data); + } + + /// Reserve the section index for the string table. + /// + /// This must be called before [`Self::reserve_section_headers`]. + pub fn reserve_strtab_section_index(&mut self) -> SectionIndex { + debug_assert_eq!(self.strtab_index, SectionIndex(0)); + self.strtab_str_id = Some(self.add_section_name(&b".strtab"[..])); + self.strtab_index = self.reserve_section_index(); + self.strtab_index + } + + /// Write the section header for the string table. + /// + /// This function does nothing if the section index was not reserved. + pub fn write_strtab_section_header(&mut self) { + if self.strtab_index == SectionIndex(0) { + return; + } + self.write_section_header(&SectionHeader { + name: self.strtab_str_id, + sh_type: elf::SHT_STRTAB, + sh_flags: 0, + sh_addr: 0, + sh_offset: self.strtab_offset as u64, + sh_size: self.strtab_data.len() as u64, + sh_link: 0, + sh_info: 0, + sh_addralign: 1, + sh_entsize: 0, + }); + } + + /// Reserve the null symbol table entry. + /// + /// This will be stored in the `.symtab` section. + /// + /// The null symbol table entry is usually automatically reserved, + /// but this can be used to force an empty symbol table. + /// + /// This must be called before [`Self::reserve_symtab`]. + pub fn reserve_null_symbol_index(&mut self) -> SymbolIndex { + debug_assert_eq!(self.symtab_offset, 0); + debug_assert_eq!(self.symtab_num, 0); + self.symtab_num = 1; + // The symtab must link to a strtab. + self.need_strtab = true; + SymbolIndex(0) + } + + /// Reserve a symbol table entry. + /// + /// This will be stored in the `.symtab` section. + /// + /// `section_index` is used to determine whether `.symtab_shndx` is required. + /// + /// Automatically also reserves the null symbol if required. + /// Callers may assume that the returned indices will be sequential + /// starting at 1. + /// + /// This must be called before [`Self::reserve_symtab`] and + /// [`Self::reserve_symtab_shndx`]. + pub fn reserve_symbol_index(&mut self, section_index: Option) -> SymbolIndex { + debug_assert_eq!(self.symtab_offset, 0); + debug_assert_eq!(self.symtab_shndx_offset, 0); + if self.symtab_num == 0 { + self.symtab_num = 1; + // The symtab must link to a strtab. + self.need_strtab = true; + } + let index = self.symtab_num; + self.symtab_num += 1; + if let Some(section_index) = section_index { + if section_index.0 >= elf::SHN_LORESERVE.into() { + self.need_symtab_shndx = true; + } + } + SymbolIndex(index) + } + + /// Return the number of reserved symbol table entries. + /// + /// Includes the null symbol. + pub fn symbol_count(&self) -> u32 { + self.symtab_num + } + + fn symbol_size(&self) -> usize { + if self.is_64 { + mem::size_of::>() + } else { + mem::size_of::>() + } + } + + /// Reserve the range for the symbol table. + /// + /// This range is used for a section named `.symtab`. + /// This function does nothing if no symbols were reserved. + /// This must be called after [`Self::reserve_symbol_index`]. + pub fn reserve_symtab(&mut self) { + debug_assert_eq!(self.symtab_offset, 0); + if self.symtab_num == 0 { + return; + } + self.symtab_offset = self.reserve( + self.symtab_num as usize * self.symbol_size(), + self.elf_align, + ); + } + + /// Write the null symbol. + /// + /// This must be the first symbol that is written. + /// This function does nothing if no symbols were reserved. + pub fn write_null_symbol(&mut self) { + if self.symtab_num == 0 { + return; + } + util::write_align(self.buffer, self.elf_align); + debug_assert_eq!(self.symtab_offset, self.buffer.len()); + if self.is_64 { + self.buffer.write(&elf::Sym64::::default()); + } else { + self.buffer.write(&elf::Sym32::::default()); + } + + if self.need_symtab_shndx { + self.symtab_shndx_data.write_pod(&U32::new(self.endian, 0)); + } + } + + /// Write a symbol. + pub fn write_symbol(&mut self, sym: &Sym) { + let st_name = if let Some(name) = sym.name { + self.strtab.get_offset(name) as u32 + } else { + 0 + }; + let st_shndx = if let Some(section) = sym.section { + if section.0 >= elf::SHN_LORESERVE as u32 { + elf::SHN_XINDEX + } else { + section.0 as u16 + } + } else { + sym.st_shndx + }; + + let endian = self.endian; + if self.is_64 { + let sym = elf::Sym64 { + st_name: U32::new(endian, st_name), + st_info: sym.st_info, + st_other: sym.st_other, + st_shndx: U16::new(endian, st_shndx), + st_value: U64::new(endian, sym.st_value), + st_size: U64::new(endian, sym.st_size), + }; + self.buffer.write(&sym); + } else { + let sym = elf::Sym32 { + st_name: U32::new(endian, st_name), + st_info: sym.st_info, + st_other: sym.st_other, + st_shndx: U16::new(endian, st_shndx), + st_value: U32::new(endian, sym.st_value as u32), + st_size: U32::new(endian, sym.st_size as u32), + }; + self.buffer.write(&sym); + } + + if self.need_symtab_shndx { + let section_index = sym.section.unwrap_or(SectionIndex(0)); + self.symtab_shndx_data + .write_pod(&U32::new(self.endian, section_index.0)); + } + } + + /// Reserve the section index for the symbol table. + /// + /// This must be called before [`Self::reserve_section_headers`]. + pub fn reserve_symtab_section_index(&mut self) -> SectionIndex { + debug_assert_eq!(self.symtab_index, SectionIndex(0)); + self.symtab_str_id = Some(self.add_section_name(&b".symtab"[..])); + self.symtab_index = self.reserve_section_index(); + self.symtab_index + } + + /// Return the section index of the symbol table. + pub fn symtab_index(&mut self) -> SectionIndex { + self.symtab_index + } + + /// Write the section header for the symbol table. + /// + /// This function does nothing if the section index was not reserved. + pub fn write_symtab_section_header(&mut self, num_local: u32) { + if self.symtab_index == SectionIndex(0) { + return; + } + self.write_section_header(&SectionHeader { + name: self.symtab_str_id, + sh_type: elf::SHT_SYMTAB, + sh_flags: 0, + sh_addr: 0, + sh_offset: self.symtab_offset as u64, + sh_size: self.symtab_num as u64 * self.symbol_size() as u64, + sh_link: self.strtab_index.0, + sh_info: num_local, + sh_addralign: self.elf_align as u64, + sh_entsize: self.symbol_size() as u64, + }); + } + + /// Return true if `.symtab_shndx` is needed. + pub fn symtab_shndx_needed(&self) -> bool { + self.need_symtab_shndx + } + + /// Reserve the range for the extended section indices for the symbol table. + /// + /// This range is used for a section named `.symtab_shndx`. + /// This also reserves a section index. + /// + /// This function does nothing if extended section indices are not needed. + /// This must be called after [`Self::reserve_symbol_index`]. + pub fn reserve_symtab_shndx(&mut self) { + debug_assert_eq!(self.symtab_shndx_offset, 0); + if !self.need_symtab_shndx { + return; + } + self.symtab_shndx_offset = self.reserve(self.symtab_num as usize * 4, 4); + self.symtab_shndx_data.reserve(self.symtab_num as usize * 4); + } + + /// Write the extended section indices for the symbol table. + /// + /// This function does nothing if the section was not reserved. + pub fn write_symtab_shndx(&mut self) { + if self.symtab_shndx_offset == 0 { + return; + } + debug_assert_eq!(self.symtab_shndx_offset, self.buffer.len()); + debug_assert_eq!(self.symtab_num as usize * 4, self.symtab_shndx_data.len()); + self.buffer.write_bytes(&self.symtab_shndx_data); + } + + /// Reserve the section index for the extended section indices symbol table. + /// + /// You should check [`Self::symtab_shndx_needed`] before calling this + /// unless you have other means of knowing if this section is needed. + /// + /// This must be called before [`Self::reserve_section_headers`]. + pub fn reserve_symtab_shndx_section_index(&mut self) -> SectionIndex { + debug_assert!(self.symtab_shndx_str_id.is_none()); + self.symtab_shndx_str_id = Some(self.add_section_name(&b".symtab_shndx"[..])); + self.reserve_section_index() + } + + /// Write the section header for the extended section indices for the symbol table. + /// + /// This function does nothing if the section index was not reserved. + pub fn write_symtab_shndx_section_header(&mut self) { + if self.symtab_shndx_str_id.is_none() { + return; + } + let sh_size = if self.symtab_shndx_offset == 0 { + 0 + } else { + (self.symtab_num * 4) as u64 + }; + self.write_section_header(&SectionHeader { + name: self.symtab_shndx_str_id, + sh_type: elf::SHT_SYMTAB_SHNDX, + sh_flags: 0, + sh_addr: 0, + sh_offset: self.symtab_shndx_offset as u64, + sh_size, + sh_link: self.symtab_index.0, + sh_info: 0, + sh_addralign: 4, + sh_entsize: 4, + }); + } + + /// Add a string to the dynamic string table. + /// + /// This will be stored in the `.dynstr` section. + /// + /// This must be called before [`Self::reserve_dynstr`]. + pub fn add_dynamic_string(&mut self, name: &'a [u8]) -> StringId { + debug_assert_eq!(self.dynstr_offset, 0); + self.need_dynstr = true; + self.dynstr.add(name) + } + + /// Get a string that was previously added to the dynamic string table. + /// + /// Panics if the string was not added. + pub fn get_dynamic_string(&self, name: &'a [u8]) -> StringId { + self.dynstr.get_id(name) + } + + /// Return true if `.dynstr` is needed. + pub fn dynstr_needed(&self) -> bool { + self.need_dynstr + } + + /// Reserve the range for the dynamic string table. + /// + /// This range is used for a section named `.dynstr`. + /// + /// This function does nothing if no dynamic strings or symbols were defined. + /// This must be called after [`Self::add_dynamic_string`]. + pub fn reserve_dynstr(&mut self) { + debug_assert_eq!(self.dynstr_offset, 0); + if !self.need_dynstr { + return; + } + // Start with null string. + self.dynstr_data = vec![0]; + self.dynstr.write(1, &mut self.dynstr_data); + self.dynstr_offset = self.reserve(self.dynstr_data.len(), 1); + } + + /// Write the dynamic string table. + /// + /// This function does nothing if the section was not reserved. + pub fn write_dynstr(&mut self) { + if self.dynstr_offset == 0 { + return; + } + debug_assert_eq!(self.dynstr_offset, self.buffer.len()); + self.buffer.write_bytes(&self.dynstr_data); + } + + /// Reserve the section index for the dynamic string table. + /// + /// This must be called before [`Self::reserve_section_headers`]. + pub fn reserve_dynstr_section_index(&mut self) -> SectionIndex { + debug_assert_eq!(self.dynstr_index, SectionIndex(0)); + self.dynstr_str_id = Some(self.add_section_name(&b".dynstr"[..])); + self.dynstr_index = self.reserve_section_index(); + self.dynstr_index + } + + /// Return the section index of the dynamic string table. + pub fn dynstr_index(&mut self) -> SectionIndex { + self.dynstr_index + } + + /// Write the section header for the dynamic string table. + /// + /// This function does nothing if the section index was not reserved. + pub fn write_dynstr_section_header(&mut self, sh_addr: u64) { + if self.dynstr_index == SectionIndex(0) { + return; + } + self.write_section_header(&SectionHeader { + name: self.dynstr_str_id, + sh_type: elf::SHT_STRTAB, + sh_flags: elf::SHF_ALLOC.into(), + sh_addr, + sh_offset: self.dynstr_offset as u64, + sh_size: self.dynstr_data.len() as u64, + sh_link: 0, + sh_info: 0, + sh_addralign: 1, + sh_entsize: 0, + }); + } + + /// Reserve the null dynamic symbol table entry. + /// + /// This will be stored in the `.dynsym` section. + /// + /// The null dynamic symbol table entry is usually automatically reserved, + /// but this can be used to force an empty dynamic symbol table. + /// + /// This must be called before [`Self::reserve_dynsym`]. + pub fn reserve_null_dynamic_symbol_index(&mut self) -> SymbolIndex { + debug_assert_eq!(self.dynsym_offset, 0); + debug_assert_eq!(self.dynsym_num, 0); + self.dynsym_num = 1; + // The symtab must link to a strtab. + self.need_dynstr = true; + SymbolIndex(0) + } + + /// Reserve a dynamic symbol table entry. + /// + /// This will be stored in the `.dynsym` section. + /// + /// Automatically also reserves the null symbol if required. + /// Callers may assume that the returned indices will be sequential + /// starting at 1. + /// + /// This must be called before [`Self::reserve_dynsym`]. + pub fn reserve_dynamic_symbol_index(&mut self) -> SymbolIndex { + debug_assert_eq!(self.dynsym_offset, 0); + if self.dynsym_num == 0 { + self.dynsym_num = 1; + // The symtab must link to a strtab. + self.need_dynstr = true; + } + let index = self.dynsym_num; + self.dynsym_num += 1; + SymbolIndex(index) + } + + /// Return the number of reserved dynamic symbols. + /// + /// Includes the null symbol. + pub fn dynamic_symbol_count(&mut self) -> u32 { + self.dynsym_num + } + + /// Reserve the range for the dynamic symbol table. + /// + /// This range is used for a section named `.dynsym`. + /// + /// This function does nothing if no dynamic symbols were reserved. + /// This must be called after [`Self::reserve_dynamic_symbol_index`]. + pub fn reserve_dynsym(&mut self) { + debug_assert_eq!(self.dynsym_offset, 0); + if self.dynsym_num == 0 { + return; + } + self.dynsym_offset = self.reserve( + self.dynsym_num as usize * self.symbol_size(), + self.elf_align, + ); + } + + /// Write the null dynamic symbol. + /// + /// This must be the first dynamic symbol that is written. + /// This function does nothing if no dynamic symbols were reserved. + pub fn write_null_dynamic_symbol(&mut self) { + if self.dynsym_num == 0 { + return; + } + util::write_align(self.buffer, self.elf_align); + debug_assert_eq!(self.dynsym_offset, self.buffer.len()); + if self.is_64 { + self.buffer.write(&elf::Sym64::::default()); + } else { + self.buffer.write(&elf::Sym32::::default()); + } + } + + /// Write a dynamic symbol. + pub fn write_dynamic_symbol(&mut self, sym: &Sym) { + let st_name = if let Some(name) = sym.name { + self.dynstr.get_offset(name) as u32 + } else { + 0 + }; + + let st_shndx = if let Some(section) = sym.section { + if section.0 >= elf::SHN_LORESERVE as u32 { + // TODO: we don't actually write out .dynsym_shndx yet. + // This is unlikely to be needed though. + elf::SHN_XINDEX + } else { + section.0 as u16 + } + } else { + sym.st_shndx + }; + + let endian = self.endian; + if self.is_64 { + let sym = elf::Sym64 { + st_name: U32::new(endian, st_name), + st_info: sym.st_info, + st_other: sym.st_other, + st_shndx: U16::new(endian, st_shndx), + st_value: U64::new(endian, sym.st_value), + st_size: U64::new(endian, sym.st_size), + }; + self.buffer.write(&sym); + } else { + let sym = elf::Sym32 { + st_name: U32::new(endian, st_name), + st_info: sym.st_info, + st_other: sym.st_other, + st_shndx: U16::new(endian, st_shndx), + st_value: U32::new(endian, sym.st_value as u32), + st_size: U32::new(endian, sym.st_size as u32), + }; + self.buffer.write(&sym); + } + } + + /// Reserve the section index for the dynamic symbol table. + /// + /// This must be called before [`Self::reserve_section_headers`]. + pub fn reserve_dynsym_section_index(&mut self) -> SectionIndex { + debug_assert_eq!(self.dynsym_index, SectionIndex(0)); + self.dynsym_str_id = Some(self.add_section_name(&b".dynsym"[..])); + self.dynsym_index = self.reserve_section_index(); + self.dynsym_index + } + + /// Return the section index of the dynamic symbol table. + pub fn dynsym_index(&mut self) -> SectionIndex { + self.dynsym_index + } + + /// Write the section header for the dynamic symbol table. + /// + /// This function does nothing if the section index was not reserved. + pub fn write_dynsym_section_header(&mut self, sh_addr: u64, num_local: u32) { + if self.dynsym_index == SectionIndex(0) { + return; + } + self.write_section_header(&SectionHeader { + name: self.dynsym_str_id, + sh_type: elf::SHT_DYNSYM, + sh_flags: elf::SHF_ALLOC.into(), + sh_addr, + sh_offset: self.dynsym_offset as u64, + sh_size: self.dynsym_num as u64 * self.symbol_size() as u64, + sh_link: self.dynstr_index.0, + sh_info: num_local, + sh_addralign: self.elf_align as u64, + sh_entsize: self.symbol_size() as u64, + }); + } + + fn dyn_size(&self) -> usize { + if self.is_64 { + mem::size_of::>() + } else { + mem::size_of::>() + } + } + + /// Reserve the range for the `.dynamic` section. + /// + /// This function does nothing if `dynamic_num` is zero. + pub fn reserve_dynamic(&mut self, dynamic_num: usize) { + debug_assert_eq!(self.dynamic_offset, 0); + if dynamic_num == 0 { + return; + } + self.dynamic_num = dynamic_num; + self.dynamic_offset = self.reserve(dynamic_num * self.dyn_size(), self.elf_align); + } + + /// Write alignment padding bytes prior to the `.dynamic` section. + /// + /// This function does nothing if the section was not reserved. + pub fn write_align_dynamic(&mut self) { + if self.dynamic_offset == 0 { + return; + } + util::write_align(self.buffer, self.elf_align); + debug_assert_eq!(self.dynamic_offset, self.buffer.len()); + } + + /// Write a dynamic string entry. + pub fn write_dynamic_string(&mut self, tag: u32, id: StringId) { + self.write_dynamic(tag, self.dynstr.get_offset(id) as u64); + } + + /// Write a dynamic value entry. + pub fn write_dynamic(&mut self, d_tag: u32, d_val: u64) { + debug_assert!(self.dynamic_offset <= self.buffer.len()); + let endian = self.endian; + if self.is_64 { + let d = elf::Dyn64 { + d_tag: U64::new(endian, d_tag.into()), + d_val: U64::new(endian, d_val), + }; + self.buffer.write(&d); + } else { + let d = elf::Dyn32 { + d_tag: U32::new(endian, d_tag), + d_val: U32::new(endian, d_val as u32), + }; + self.buffer.write(&d); + } + debug_assert!( + self.dynamic_offset + self.dynamic_num * self.dyn_size() >= self.buffer.len() + ); + } + + /// Reserve the section index for the dynamic table. + pub fn reserve_dynamic_section_index(&mut self) -> SectionIndex { + debug_assert!(self.dynamic_str_id.is_none()); + self.dynamic_str_id = Some(self.add_section_name(&b".dynamic"[..])); + self.reserve_section_index() + } + + /// Write the section header for the dynamic table. + /// + /// This function does nothing if the section index was not reserved. + pub fn write_dynamic_section_header(&mut self, sh_addr: u64) { + if self.dynamic_str_id.is_none() { + return; + } + self.write_section_header(&SectionHeader { + name: self.dynamic_str_id, + sh_type: elf::SHT_DYNAMIC, + sh_flags: (elf::SHF_WRITE | elf::SHF_ALLOC).into(), + sh_addr, + sh_offset: self.dynamic_offset as u64, + sh_size: (self.dynamic_num * self.dyn_size()) as u64, + sh_link: self.dynstr_index.0, + sh_info: 0, + sh_addralign: self.elf_align as u64, + sh_entsize: self.dyn_size() as u64, + }); + } + + fn rel_size(&self, is_rela: bool) -> usize { + if self.is_64 { + if is_rela { + mem::size_of::>() + } else { + mem::size_of::>() + } + } else { + if is_rela { + mem::size_of::>() + } else { + mem::size_of::>() + } + } + } + + /// Reserve a file range for a SysV hash section. + /// + /// `symbol_count` is the number of symbols in the hash, + /// not the total number of symbols. + pub fn reserve_hash(&mut self, bucket_count: u32, chain_count: u32) { + self.hash_size = mem::size_of::>() + + bucket_count as usize * 4 + + chain_count as usize * 4; + self.hash_offset = self.reserve(self.hash_size, self.elf_align); + } + + /// Write a SysV hash section. + /// + /// `chain_count` is the number of symbols in the hash. + /// The argument to `hash` will be in the range `0..chain_count`. + pub fn write_hash(&mut self, bucket_count: u32, chain_count: u32, hash: F) + where + F: Fn(u32) -> Option, + { + let mut buckets = vec![U32::new(self.endian, 0); bucket_count as usize]; + let mut chains = vec![U32::new(self.endian, 0); chain_count as usize]; + for i in 0..chain_count { + if let Some(hash) = hash(i) { + let bucket = hash % bucket_count; + chains[i as usize] = buckets[bucket as usize]; + buckets[bucket as usize] = U32::new(self.endian, i); + } + } + + util::write_align(self.buffer, self.elf_align); + debug_assert_eq!(self.hash_offset, self.buffer.len()); + self.buffer.write(&elf::HashHeader { + bucket_count: U32::new(self.endian, bucket_count), + chain_count: U32::new(self.endian, chain_count), + }); + self.buffer.write_slice(&buckets); + self.buffer.write_slice(&chains); + } + + /// Reserve the section index for the SysV hash table. + pub fn reserve_hash_section_index(&mut self) -> SectionIndex { + debug_assert!(self.hash_str_id.is_none()); + self.hash_str_id = Some(self.add_section_name(&b".hash"[..])); + self.reserve_section_index() + } + + /// Write the section header for the SysV hash table. + /// + /// This function does nothing if the section index was not reserved. + pub fn write_hash_section_header(&mut self, sh_addr: u64) { + if self.hash_str_id.is_none() { + return; + } + self.write_section_header(&SectionHeader { + name: self.hash_str_id, + sh_type: elf::SHT_HASH, + sh_flags: elf::SHF_ALLOC.into(), + sh_addr, + sh_offset: self.hash_offset as u64, + sh_size: self.hash_size as u64, + sh_link: self.dynsym_index.0, + sh_info: 0, + sh_addralign: self.elf_align as u64, + sh_entsize: 4, + }); + } + + /// Reserve a file range for a GNU hash section. + /// + /// `symbol_count` is the number of symbols in the hash, + /// not the total number of symbols. + pub fn reserve_gnu_hash(&mut self, bloom_count: u32, bucket_count: u32, symbol_count: u32) { + self.gnu_hash_size = mem::size_of::>() + + bloom_count as usize * self.elf_align + + bucket_count as usize * 4 + + symbol_count as usize * 4; + self.gnu_hash_offset = self.reserve(self.gnu_hash_size, self.elf_align); + } + + /// Write a GNU hash section. + /// + /// `symbol_count` is the number of symbols in the hash. + /// The argument to `hash` will be in the range `0..symbol_count`. + /// + /// This requires that symbols are already sorted by bucket. + pub fn write_gnu_hash( + &mut self, + symbol_base: u32, + bloom_shift: u32, + bloom_count: u32, + bucket_count: u32, + symbol_count: u32, + hash: F, + ) where + F: Fn(u32) -> u32, + { + util::write_align(self.buffer, self.elf_align); + debug_assert_eq!(self.gnu_hash_offset, self.buffer.len()); + self.buffer.write(&elf::GnuHashHeader { + bucket_count: U32::new(self.endian, bucket_count), + symbol_base: U32::new(self.endian, symbol_base), + bloom_count: U32::new(self.endian, bloom_count), + bloom_shift: U32::new(self.endian, bloom_shift), + }); + + // Calculate and write bloom filter. + if self.is_64 { + let mut bloom_filters = vec![0; bloom_count as usize]; + for i in 0..symbol_count { + let h = hash(i); + bloom_filters[((h / 64) & (bloom_count - 1)) as usize] |= + 1 << (h % 64) | 1 << ((h >> bloom_shift) % 64); + } + for bloom_filter in bloom_filters { + self.buffer.write(&U64::new(self.endian, bloom_filter)); + } + } else { + let mut bloom_filters = vec![0; bloom_count as usize]; + for i in 0..symbol_count { + let h = hash(i); + bloom_filters[((h / 32) & (bloom_count - 1)) as usize] |= + 1 << (h % 32) | 1 << ((h >> bloom_shift) % 32); + } + for bloom_filter in bloom_filters { + self.buffer.write(&U32::new(self.endian, bloom_filter)); + } + } + + // Write buckets. + // + // This requires that symbols are already sorted by bucket. + let mut bucket = 0; + for i in 0..symbol_count { + let symbol_bucket = hash(i) % bucket_count; + while bucket < symbol_bucket { + self.buffer.write(&U32::new(self.endian, 0)); + bucket += 1; + } + if bucket == symbol_bucket { + self.buffer.write(&U32::new(self.endian, symbol_base + i)); + bucket += 1; + } + } + while bucket < bucket_count { + self.buffer.write(&U32::new(self.endian, 0)); + bucket += 1; + } + + // Write hash values. + for i in 0..symbol_count { + let mut h = hash(i); + if i == symbol_count - 1 || h % bucket_count != hash(i + 1) % bucket_count { + h |= 1; + } else { + h &= !1; + } + self.buffer.write(&U32::new(self.endian, h)); + } + } + + /// Reserve the section index for the GNU hash table. + pub fn reserve_gnu_hash_section_index(&mut self) -> SectionIndex { + debug_assert!(self.gnu_hash_str_id.is_none()); + self.gnu_hash_str_id = Some(self.add_section_name(&b".gnu.hash"[..])); + self.reserve_section_index() + } + + /// Write the section header for the GNU hash table. + /// + /// This function does nothing if the section index was not reserved. + pub fn write_gnu_hash_section_header(&mut self, sh_addr: u64) { + if self.gnu_hash_str_id.is_none() { + return; + } + self.write_section_header(&SectionHeader { + name: self.gnu_hash_str_id, + sh_type: elf::SHT_GNU_HASH, + sh_flags: elf::SHF_ALLOC.into(), + sh_addr, + sh_offset: self.gnu_hash_offset as u64, + sh_size: self.gnu_hash_size as u64, + sh_link: self.dynsym_index.0, + sh_info: 0, + sh_addralign: self.elf_align as u64, + sh_entsize: 0, + }); + } + + /// Reserve the range for the `.gnu.version` section. + /// + /// This function does nothing if no dynamic symbols were reserved. + pub fn reserve_gnu_versym(&mut self) { + debug_assert_eq!(self.gnu_versym_offset, 0); + if self.dynsym_num == 0 { + return; + } + self.gnu_versym_offset = self.reserve(self.dynsym_num as usize * 2, 2); + } + + /// Write the null symbol version entry. + /// + /// This must be the first symbol version that is written. + /// This function does nothing if no dynamic symbols were reserved. + pub fn write_null_gnu_versym(&mut self) { + if self.dynsym_num == 0 { + return; + } + util::write_align(self.buffer, 2); + debug_assert_eq!(self.gnu_versym_offset, self.buffer.len()); + self.write_gnu_versym(0); + } + + /// Write a symbol version entry. + pub fn write_gnu_versym(&mut self, versym: u16) { + self.buffer.write(&U16::new(self.endian, versym)); + } + + /// Reserve the section index for the `.gnu.version` section. + pub fn reserve_gnu_versym_section_index(&mut self) -> SectionIndex { + debug_assert!(self.gnu_versym_str_id.is_none()); + self.gnu_versym_str_id = Some(self.add_section_name(&b".gnu.version"[..])); + self.reserve_section_index() + } + + /// Write the section header for the `.gnu.version` section. + /// + /// This function does nothing if the section index was not reserved. + pub fn write_gnu_versym_section_header(&mut self, sh_addr: u64) { + if self.gnu_versym_str_id.is_none() { + return; + } + self.write_section_header(&SectionHeader { + name: self.gnu_versym_str_id, + sh_type: elf::SHT_GNU_VERSYM, + sh_flags: elf::SHF_ALLOC.into(), + sh_addr, + sh_offset: self.gnu_versym_offset as u64, + sh_size: self.dynsym_num as u64 * 2, + sh_link: self.dynsym_index.0, + sh_info: 0, + sh_addralign: 2, + sh_entsize: 2, + }); + } + + /// Reserve the range for the `.gnu.version_d` section. + pub fn reserve_gnu_verdef(&mut self, verdef_count: usize, verdaux_count: usize) { + debug_assert_eq!(self.gnu_verdef_offset, 0); + if verdef_count == 0 { + return; + } + self.gnu_verdef_size = verdef_count * mem::size_of::>() + + verdaux_count * mem::size_of::>(); + self.gnu_verdef_offset = self.reserve(self.gnu_verdef_size, self.elf_align); + self.gnu_verdef_count = verdef_count as u16; + self.gnu_verdef_remaining = self.gnu_verdef_count; + } + + /// Write alignment padding bytes prior to a `.gnu.version_d` section. + pub fn write_align_gnu_verdef(&mut self) { + if self.gnu_verdef_offset == 0 { + return; + } + util::write_align(self.buffer, self.elf_align); + debug_assert_eq!(self.gnu_verdef_offset, self.buffer.len()); + } + + /// Write a version definition entry. + pub fn write_gnu_verdef(&mut self, verdef: &Verdef) { + debug_assert_ne!(self.gnu_verdef_remaining, 0); + self.gnu_verdef_remaining -= 1; + let vd_next = if self.gnu_verdef_remaining == 0 { + 0 + } else { + mem::size_of::>() as u32 + + verdef.aux_count as u32 * mem::size_of::>() as u32 + }; + + self.gnu_verdaux_remaining = verdef.aux_count; + let vd_aux = if verdef.aux_count == 0 { + 0 + } else { + mem::size_of::>() as u32 + }; + + self.buffer.write(&elf::Verdef { + vd_version: U16::new(self.endian, verdef.version), + vd_flags: U16::new(self.endian, verdef.flags), + vd_ndx: U16::new(self.endian, verdef.index), + vd_cnt: U16::new(self.endian, verdef.aux_count), + vd_hash: U32::new(self.endian, elf::hash(self.dynstr.get_string(verdef.name))), + vd_aux: U32::new(self.endian, vd_aux), + vd_next: U32::new(self.endian, vd_next), + }); + self.write_gnu_verdaux(verdef.name); + } + + /// Write a version definition auxiliary entry. + pub fn write_gnu_verdaux(&mut self, name: StringId) { + debug_assert_ne!(self.gnu_verdaux_remaining, 0); + self.gnu_verdaux_remaining -= 1; + let vda_next = if self.gnu_verdaux_remaining == 0 { + 0 + } else { + mem::size_of::>() as u32 + }; + self.buffer.write(&elf::Verdaux { + vda_name: U32::new(self.endian, self.dynstr.get_offset(name) as u32), + vda_next: U32::new(self.endian, vda_next), + }); + } + + /// Reserve the section index for the `.gnu.version_d` section. + pub fn reserve_gnu_verdef_section_index(&mut self) -> SectionIndex { + debug_assert!(self.gnu_verdef_str_id.is_none()); + self.gnu_verdef_str_id = Some(self.add_section_name(&b".gnu.version_d"[..])); + self.reserve_section_index() + } + + /// Write the section header for the `.gnu.version_d` section. + /// + /// This function does nothing if the section index was not reserved. + pub fn write_gnu_verdef_section_header(&mut self, sh_addr: u64) { + if self.gnu_verdef_str_id.is_none() { + return; + } + self.write_section_header(&SectionHeader { + name: self.gnu_verdef_str_id, + sh_type: elf::SHT_GNU_VERDEF, + sh_flags: elf::SHF_ALLOC.into(), + sh_addr, + sh_offset: self.gnu_verdef_offset as u64, + sh_size: self.gnu_verdef_size as u64, + sh_link: self.dynstr_index.0, + sh_info: self.gnu_verdef_count.into(), + sh_addralign: self.elf_align as u64, + sh_entsize: 0, + }); + } + + /// Reserve the range for the `.gnu.version_r` section. + pub fn reserve_gnu_verneed(&mut self, verneed_count: usize, vernaux_count: usize) { + debug_assert_eq!(self.gnu_verneed_offset, 0); + if verneed_count == 0 { + return; + } + self.gnu_verneed_size = verneed_count * mem::size_of::>() + + vernaux_count * mem::size_of::>(); + self.gnu_verneed_offset = self.reserve(self.gnu_verneed_size, self.elf_align); + self.gnu_verneed_count = verneed_count as u16; + self.gnu_verneed_remaining = self.gnu_verneed_count; + } + + /// Write alignment padding bytes prior to a `.gnu.version_r` section. + pub fn write_align_gnu_verneed(&mut self) { + if self.gnu_verneed_offset == 0 { + return; + } + util::write_align(self.buffer, self.elf_align); + debug_assert_eq!(self.gnu_verneed_offset, self.buffer.len()); + } + + /// Write a version need entry. + pub fn write_gnu_verneed(&mut self, verneed: &Verneed) { + debug_assert_ne!(self.gnu_verneed_remaining, 0); + self.gnu_verneed_remaining -= 1; + let vn_next = if self.gnu_verneed_remaining == 0 { + 0 + } else { + mem::size_of::>() as u32 + + verneed.aux_count as u32 * mem::size_of::>() as u32 + }; + + self.gnu_vernaux_remaining = verneed.aux_count; + let vn_aux = if verneed.aux_count == 0 { + 0 + } else { + mem::size_of::>() as u32 + }; + + self.buffer.write(&elf::Verneed { + vn_version: U16::new(self.endian, verneed.version), + vn_cnt: U16::new(self.endian, verneed.aux_count), + vn_file: U32::new(self.endian, self.dynstr.get_offset(verneed.file) as u32), + vn_aux: U32::new(self.endian, vn_aux), + vn_next: U32::new(self.endian, vn_next), + }); + } + + /// Write a version need auxiliary entry. + pub fn write_gnu_vernaux(&mut self, vernaux: &Vernaux) { + debug_assert_ne!(self.gnu_vernaux_remaining, 0); + self.gnu_vernaux_remaining -= 1; + let vna_next = if self.gnu_vernaux_remaining == 0 { + 0 + } else { + mem::size_of::>() as u32 + }; + self.buffer.write(&elf::Vernaux { + vna_hash: U32::new(self.endian, elf::hash(self.dynstr.get_string(vernaux.name))), + vna_flags: U16::new(self.endian, vernaux.flags), + vna_other: U16::new(self.endian, vernaux.index), + vna_name: U32::new(self.endian, self.dynstr.get_offset(vernaux.name) as u32), + vna_next: U32::new(self.endian, vna_next), + }); + } + + /// Reserve the section index for the `.gnu.version_r` section. + pub fn reserve_gnu_verneed_section_index(&mut self) -> SectionIndex { + debug_assert!(self.gnu_verneed_str_id.is_none()); + self.gnu_verneed_str_id = Some(self.add_section_name(&b".gnu.version_r"[..])); + self.reserve_section_index() + } + + /// Write the section header for the `.gnu.version_r` section. + /// + /// This function does nothing if the section index was not reserved. + pub fn write_gnu_verneed_section_header(&mut self, sh_addr: u64) { + if self.gnu_verneed_str_id.is_none() { + return; + } + self.write_section_header(&SectionHeader { + name: self.gnu_verneed_str_id, + sh_type: elf::SHT_GNU_VERNEED, + sh_flags: elf::SHF_ALLOC.into(), + sh_addr, + sh_offset: self.gnu_verneed_offset as u64, + sh_size: self.gnu_verneed_size as u64, + sh_link: self.dynstr_index.0, + sh_info: self.gnu_verneed_count.into(), + sh_addralign: self.elf_align as u64, + sh_entsize: 0, + }); + } + + /// Reserve a file range for the given number of relocations. + /// + /// Returns the offset of the range. + pub fn reserve_relocations(&mut self, count: usize, is_rela: bool) -> usize { + self.reserve(count * self.rel_size(is_rela), self.elf_align) + } + + /// Write alignment padding bytes prior to a relocation section. + pub fn write_align_relocation(&mut self) { + util::write_align(self.buffer, self.elf_align); + } + + /// Write a relocation. + pub fn write_relocation(&mut self, is_rela: bool, rel: &Rel) { + let endian = self.endian; + if self.is_64 { + if is_rela { + let rel = elf::Rela64 { + r_offset: U64::new(endian, rel.r_offset), + r_info: elf::Rela64::r_info(endian, self.is_mips64el, rel.r_sym, rel.r_type), + r_addend: I64::new(endian, rel.r_addend), + }; + self.buffer.write(&rel); + } else { + let rel = elf::Rel64 { + r_offset: U64::new(endian, rel.r_offset), + r_info: elf::Rel64::r_info(endian, rel.r_sym, rel.r_type), + }; + self.buffer.write(&rel); + } + } else { + if is_rela { + let rel = elf::Rela32 { + r_offset: U32::new(endian, rel.r_offset as u32), + r_info: elf::Rel32::r_info(endian, rel.r_sym, rel.r_type as u8), + r_addend: I32::new(endian, rel.r_addend as i32), + }; + self.buffer.write(&rel); + } else { + let rel = elf::Rel32 { + r_offset: U32::new(endian, rel.r_offset as u32), + r_info: elf::Rel32::r_info(endian, rel.r_sym, rel.r_type as u8), + }; + self.buffer.write(&rel); + } + } + } + + /// Write the section header for a relocation section. + /// + /// `section` is the index of the section the relocations apply to, + /// or 0 if none. + /// + /// `symtab` is the index of the symbol table the relocations refer to, + /// or 0 if none. + /// + /// `offset` is the file offset of the relocations. + pub fn write_relocation_section_header( + &mut self, + name: StringId, + section: SectionIndex, + symtab: SectionIndex, + offset: usize, + count: usize, + is_rela: bool, + ) { + self.write_section_header(&SectionHeader { + name: Some(name), + sh_type: if is_rela { elf::SHT_RELA } else { elf::SHT_REL }, + sh_flags: elf::SHF_INFO_LINK.into(), + sh_addr: 0, + sh_offset: offset as u64, + sh_size: (count * self.rel_size(is_rela)) as u64, + sh_link: symtab.0, + sh_info: section.0, + sh_addralign: self.elf_align as u64, + sh_entsize: self.rel_size(is_rela) as u64, + }); + } + + /// Reserve a file range for a COMDAT section. + /// + /// `count` is the number of sections in the COMDAT group. + /// + /// Returns the offset of the range. + pub fn reserve_comdat(&mut self, count: usize) -> usize { + self.reserve((count + 1) * 4, 4) + } + + /// Write `GRP_COMDAT` at the start of the COMDAT section. + pub fn write_comdat_header(&mut self) { + util::write_align(self.buffer, 4); + self.buffer.write(&U32::new(self.endian, elf::GRP_COMDAT)); + } + + /// Write an entry in a COMDAT section. + pub fn write_comdat_entry(&mut self, entry: SectionIndex) { + self.buffer.write(&U32::new(self.endian, entry.0)); + } + + /// Write the section header for a COMDAT section. + pub fn write_comdat_section_header( + &mut self, + name: StringId, + symtab: SectionIndex, + symbol: SymbolIndex, + offset: usize, + count: usize, + ) { + self.write_section_header(&SectionHeader { + name: Some(name), + sh_type: elf::SHT_GROUP, + sh_flags: 0, + sh_addr: 0, + sh_offset: offset as u64, + sh_size: ((count + 1) * 4) as u64, + sh_link: symtab.0, + sh_info: symbol.0, + sh_addralign: 4, + sh_entsize: 4, + }); + } +} + +/// Native endian version of [`elf::FileHeader64`]. +#[allow(missing_docs)] +#[derive(Debug, Clone)] +pub struct FileHeader { + pub os_abi: u8, + pub abi_version: u8, + pub e_type: u16, + pub e_machine: u16, + pub e_entry: u64, + pub e_flags: u32, +} + +/// Native endian version of [`elf::ProgramHeader64`]. +#[allow(missing_docs)] +#[derive(Debug, Clone)] +pub struct ProgramHeader { + pub p_type: u32, + pub p_flags: u32, + pub p_offset: u64, + pub p_vaddr: u64, + pub p_paddr: u64, + pub p_filesz: u64, + pub p_memsz: u64, + pub p_align: u64, +} + +/// Native endian version of [`elf::SectionHeader64`]. +#[allow(missing_docs)] +#[derive(Debug, Clone)] +pub struct SectionHeader { + pub name: Option, + pub sh_type: u32, + pub sh_flags: u64, + pub sh_addr: u64, + pub sh_offset: u64, + pub sh_size: u64, + pub sh_link: u32, + pub sh_info: u32, + pub sh_addralign: u64, + pub sh_entsize: u64, +} + +/// Native endian version of [`elf::Sym64`]. +#[allow(missing_docs)] +#[derive(Debug, Clone)] +pub struct Sym { + pub name: Option, + pub section: Option, + pub st_info: u8, + pub st_other: u8, + pub st_shndx: u16, + pub st_value: u64, + pub st_size: u64, +} + +/// Unified native endian version of [`elf::Rel64`] and [`elf::Rela64`]. +#[allow(missing_docs)] +#[derive(Debug, Clone)] +pub struct Rel { + pub r_offset: u64, + pub r_sym: u32, + pub r_type: u32, + pub r_addend: i64, +} + +/// Information required for writing [`elf::Verdef`]. +#[allow(missing_docs)] +#[derive(Debug, Clone)] +pub struct Verdef { + pub version: u16, + pub flags: u16, + pub index: u16, + pub aux_count: u16, + /// The name for the first [`elf::Verdaux`] entry. + pub name: StringId, +} + +/// Information required for writing [`elf::Verneed`]. +#[allow(missing_docs)] +#[derive(Debug, Clone)] +pub struct Verneed { + pub version: u16, + pub aux_count: u16, + pub file: StringId, +} + +/// Information required for writing [`elf::Vernaux`]. +#[allow(missing_docs)] +#[derive(Debug, Clone)] +pub struct Vernaux { + pub flags: u16, + pub index: u16, + pub name: StringId, +} diff --git a/crux-mir/lib/object/src/write/macho.rs b/crux-mir/lib/object/src/write/macho.rs new file mode 100644 index 000000000..f689dec51 --- /dev/null +++ b/crux-mir/lib/object/src/write/macho.rs @@ -0,0 +1,858 @@ +use core::mem; + +use crate::endian::*; +use crate::macho; +use crate::write::string::*; +use crate::write::util::*; +use crate::write::*; +use crate::AddressSize; + +#[derive(Default, Clone, Copy)] +struct SectionOffsets { + index: usize, + offset: usize, + address: u64, + reloc_offset: usize, +} + +#[derive(Default, Clone, Copy)] +struct SymbolOffsets { + emit: bool, + index: usize, + str_id: Option, +} + +impl<'a> Object<'a> { + pub(crate) fn macho_set_subsections_via_symbols(&mut self) { + let flags = match self.flags { + FileFlags::MachO { flags } => flags, + _ => 0, + }; + self.flags = FileFlags::MachO { + flags: flags | macho::MH_SUBSECTIONS_VIA_SYMBOLS, + }; + } + + pub(crate) fn macho_segment_name(&self, segment: StandardSegment) -> &'static [u8] { + match segment { + StandardSegment::Text => &b"__TEXT"[..], + StandardSegment::Data => &b"__DATA"[..], + StandardSegment::Debug => &b"__DWARF"[..], + } + } + + pub(crate) fn macho_section_info( + &self, + section: StandardSection, + ) -> (&'static [u8], &'static [u8], SectionKind) { + match section { + StandardSection::Text => (&b"__TEXT"[..], &b"__text"[..], SectionKind::Text), + StandardSection::Data => (&b"__DATA"[..], &b"__data"[..], SectionKind::Data), + StandardSection::ReadOnlyData => { + (&b"__TEXT"[..], &b"__const"[..], SectionKind::ReadOnlyData) + } + StandardSection::ReadOnlyDataWithRel => { + (&b"__DATA"[..], &b"__const"[..], SectionKind::ReadOnlyData) + } + StandardSection::ReadOnlyString => ( + &b"__TEXT"[..], + &b"__cstring"[..], + SectionKind::ReadOnlyString, + ), + StandardSection::UninitializedData => ( + &b"__DATA"[..], + &b"__bss"[..], + SectionKind::UninitializedData, + ), + StandardSection::Tls => (&b"__DATA"[..], &b"__thread_data"[..], SectionKind::Tls), + StandardSection::UninitializedTls => ( + &b"__DATA"[..], + &b"__thread_bss"[..], + SectionKind::UninitializedTls, + ), + StandardSection::TlsVariables => ( + &b"__DATA"[..], + &b"__thread_vars"[..], + SectionKind::TlsVariables, + ), + StandardSection::Common => (&b"__DATA"[..], &b"__common"[..], SectionKind::Common), + } + } + + fn macho_tlv_bootstrap(&mut self) -> SymbolId { + match self.tlv_bootstrap { + Some(id) => id, + None => { + let id = self.add_symbol(Symbol { + name: b"_tlv_bootstrap".to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Text, + scope: SymbolScope::Dynamic, + weak: false, + section: SymbolSection::Undefined, + flags: SymbolFlags::None, + }); + self.tlv_bootstrap = Some(id); + id + } + } + } + + /// Create the `__thread_vars` entry for a TLS variable. + /// + /// The symbol given by `symbol_id` will be updated to point to this entry. + /// + /// A new `SymbolId` will be returned. The caller must update this symbol + /// to point to the initializer. + /// + /// If `symbol_id` is not for a TLS variable, then it is returned unchanged. + pub(crate) fn macho_add_thread_var(&mut self, symbol_id: SymbolId) -> SymbolId { + let symbol = self.symbol_mut(symbol_id); + if symbol.kind != SymbolKind::Tls { + return symbol_id; + } + + // Create the initializer symbol. + let mut name = symbol.name.clone(); + name.extend_from_slice(b"$tlv$init"); + let init_symbol_id = self.add_raw_symbol(Symbol { + name, + value: 0, + size: 0, + kind: SymbolKind::Tls, + scope: SymbolScope::Compilation, + weak: false, + section: SymbolSection::Undefined, + flags: SymbolFlags::None, + }); + + // Add the tlv entry. + // Three pointers in size: + // - __tlv_bootstrap - used to make sure support exists + // - spare pointer - used when mapped by the runtime + // - pointer to symbol initializer + let section = self.section_id(StandardSection::TlsVariables); + let address_size = self.architecture.address_size().unwrap().bytes(); + let size = u64::from(address_size) * 3; + let data = vec![0; size as usize]; + let offset = self.append_section_data(section, &data, u64::from(address_size)); + + let tlv_bootstrap = self.macho_tlv_bootstrap(); + self.add_relocation( + section, + Relocation { + offset, + size: address_size * 8, + kind: RelocationKind::Absolute, + encoding: RelocationEncoding::Generic, + symbol: tlv_bootstrap, + addend: 0, + }, + ) + .unwrap(); + self.add_relocation( + section, + Relocation { + offset: offset + u64::from(address_size) * 2, + size: address_size * 8, + kind: RelocationKind::Absolute, + encoding: RelocationEncoding::Generic, + symbol: init_symbol_id, + addend: 0, + }, + ) + .unwrap(); + + // Update the symbol to point to the tlv. + let symbol = self.symbol_mut(symbol_id); + symbol.value = offset; + symbol.size = size; + symbol.section = SymbolSection::Section(section); + + init_symbol_id + } + + pub(crate) fn macho_fixup_relocation(&mut self, mut relocation: &mut Relocation) -> i64 { + let constant = match relocation.kind { + RelocationKind::Relative + | RelocationKind::GotRelative + | RelocationKind::PltRelative => relocation.addend + 4, + _ => relocation.addend, + }; + relocation.addend -= constant; + constant + } + + pub(crate) fn macho_write(&self, buffer: &mut dyn WritableBuffer) -> Result<()> { + let address_size = self.architecture.address_size().unwrap(); + let endian = self.endian; + let macho32 = MachO32 { endian }; + let macho64 = MachO64 { endian }; + let macho: &dyn MachO = match address_size { + AddressSize::U8 | AddressSize::U16 | AddressSize::U32 => &macho32, + AddressSize::U64 => &macho64, + }; + let pointer_align = address_size.bytes() as usize; + + // Calculate offsets of everything, and build strtab. + let mut offset = 0; + + // Calculate size of Mach-O header. + offset += macho.mach_header_size(); + + // Calculate size of commands. + let mut ncmds = 0; + let command_offset = offset; + + // Calculate size of segment command and section headers. + let segment_command_offset = offset; + let segment_command_len = + macho.segment_command_size() + self.sections.len() * macho.section_header_size(); + offset += segment_command_len; + ncmds += 1; + + // Calculate size of symtab command. + let symtab_command_offset = offset; + let symtab_command_len = mem::size_of::>(); + offset += symtab_command_len; + ncmds += 1; + + let sizeofcmds = offset - command_offset; + + // Calculate size of section data. + let mut segment_file_offset = None; + let mut section_offsets = vec![SectionOffsets::default(); self.sections.len()]; + let mut address = 0; + for (index, section) in self.sections.iter().enumerate() { + section_offsets[index].index = 1 + index; + if !section.is_bss() { + let len = section.data.len(); + if len != 0 { + offset = align(offset, section.align as usize); + section_offsets[index].offset = offset; + if segment_file_offset.is_none() { + segment_file_offset = Some(offset); + } + offset += len; + } else { + section_offsets[index].offset = offset; + } + address = align_u64(address, section.align); + section_offsets[index].address = address; + address += section.size; + } + } + for (index, section) in self.sections.iter().enumerate() { + if section.kind.is_bss() { + assert!(section.data.is_empty()); + address = align_u64(address, section.align); + section_offsets[index].address = address; + address += section.size; + } + } + let segment_file_offset = segment_file_offset.unwrap_or(offset); + let segment_file_size = offset - segment_file_offset; + debug_assert!(segment_file_size as u64 <= address); + + // Count symbols and add symbol strings to strtab. + let mut strtab = StringTable::default(); + let mut symbol_offsets = vec![SymbolOffsets::default(); self.symbols.len()]; + let mut nsyms = 0; + for (index, symbol) in self.symbols.iter().enumerate() { + // The unified API allows creating symbols that we don't emit, so filter + // them out here. + // + // Since we don't actually emit the symbol kind, we validate it here too. + match symbol.kind { + SymbolKind::Text | SymbolKind::Data | SymbolKind::Tls => {} + SymbolKind::File | SymbolKind::Section => continue, + SymbolKind::Unknown => { + if symbol.section != SymbolSection::Undefined { + return Err(Error(format!( + "defined symbol `{}` with unknown kind", + symbol.name().unwrap_or(""), + ))); + } + } + SymbolKind::Null | SymbolKind::Label => { + return Err(Error(format!( + "unimplemented symbol `{}` kind {:?}", + symbol.name().unwrap_or(""), + symbol.kind + ))); + } + } + symbol_offsets[index].emit = true; + symbol_offsets[index].index = nsyms; + nsyms += 1; + if !symbol.name.is_empty() { + symbol_offsets[index].str_id = Some(strtab.add(&symbol.name)); + } + } + + // Calculate size of symtab. + offset = align(offset, pointer_align); + let symtab_offset = offset; + let symtab_len = nsyms * macho.nlist_size(); + offset += symtab_len; + + // Calculate size of strtab. + let strtab_offset = offset; + // Start with null name. + let mut strtab_data = vec![0]; + strtab.write(1, &mut strtab_data); + offset += strtab_data.len(); + + // Calculate size of relocations. + for (index, section) in self.sections.iter().enumerate() { + let count = section.relocations.len(); + if count != 0 { + offset = align(offset, 4); + section_offsets[index].reloc_offset = offset; + let len = count * mem::size_of::>(); + offset += len; + } + } + + // Start writing. + buffer + .reserve(offset) + .map_err(|_| Error(String::from("Cannot allocate buffer")))?; + + // Write file header. + let (cputype, cpusubtype) = match self.architecture { + Architecture::Arm => (macho::CPU_TYPE_ARM, macho::CPU_SUBTYPE_ARM_ALL), + Architecture::Aarch64 => (macho::CPU_TYPE_ARM64, macho::CPU_SUBTYPE_ARM64_ALL), + Architecture::I386 => (macho::CPU_TYPE_X86, macho::CPU_SUBTYPE_I386_ALL), + Architecture::X86_64 => (macho::CPU_TYPE_X86_64, macho::CPU_SUBTYPE_X86_64_ALL), + _ => { + return Err(Error(format!( + "unimplemented architecture {:?}", + self.architecture + ))); + } + }; + + let flags = match self.flags { + FileFlags::MachO { flags } => flags, + _ => 0, + }; + macho.write_mach_header( + buffer, + MachHeader { + cputype, + cpusubtype, + filetype: macho::MH_OBJECT, + ncmds, + sizeofcmds: sizeofcmds as u32, + flags, + }, + ); + + // Write segment command. + debug_assert_eq!(segment_command_offset, buffer.len()); + macho.write_segment_command( + buffer, + SegmentCommand { + cmdsize: segment_command_len as u32, + segname: [0; 16], + vmaddr: 0, + vmsize: address, + fileoff: segment_file_offset as u64, + filesize: segment_file_size as u64, + maxprot: macho::VM_PROT_READ | macho::VM_PROT_WRITE | macho::VM_PROT_EXECUTE, + initprot: macho::VM_PROT_READ | macho::VM_PROT_WRITE | macho::VM_PROT_EXECUTE, + nsects: self.sections.len() as u32, + flags: 0, + }, + ); + + // Write section headers. + for (index, section) in self.sections.iter().enumerate() { + let mut sectname = [0; 16]; + sectname + .get_mut(..section.name.len()) + .ok_or_else(|| { + Error(format!( + "section name `{}` is too long", + section.name().unwrap_or(""), + )) + })? + .copy_from_slice(§ion.name); + let mut segname = [0; 16]; + segname + .get_mut(..section.segment.len()) + .ok_or_else(|| { + Error(format!( + "segment name `{}` is too long", + section.segment().unwrap_or(""), + )) + })? + .copy_from_slice(§ion.segment); + let flags = if let SectionFlags::MachO { flags } = section.flags { + flags + } else { + match section.kind { + SectionKind::Text => { + macho::S_ATTR_PURE_INSTRUCTIONS | macho::S_ATTR_SOME_INSTRUCTIONS + } + SectionKind::Data => 0, + SectionKind::ReadOnlyData => 0, + SectionKind::ReadOnlyString => macho::S_CSTRING_LITERALS, + SectionKind::UninitializedData | SectionKind::Common => macho::S_ZEROFILL, + SectionKind::Tls => macho::S_THREAD_LOCAL_REGULAR, + SectionKind::UninitializedTls => macho::S_THREAD_LOCAL_ZEROFILL, + SectionKind::TlsVariables => macho::S_THREAD_LOCAL_VARIABLES, + SectionKind::Debug => macho::S_ATTR_DEBUG, + SectionKind::OtherString => macho::S_CSTRING_LITERALS, + SectionKind::Other | SectionKind::Linker | SectionKind::Metadata => 0, + SectionKind::Note | SectionKind::Unknown | SectionKind::Elf(_) => { + return Err(Error(format!( + "unimplemented section `{}` kind {:?}", + section.name().unwrap_or(""), + section.kind + ))); + } + } + }; + macho.write_section( + buffer, + SectionHeader { + sectname, + segname, + addr: section_offsets[index].address, + size: section.size, + offset: section_offsets[index].offset as u32, + align: section.align.trailing_zeros(), + reloff: section_offsets[index].reloc_offset as u32, + nreloc: section.relocations.len() as u32, + flags, + }, + ); + } + + // Write symtab command. + debug_assert_eq!(symtab_command_offset, buffer.len()); + let symtab_command = macho::SymtabCommand { + cmd: U32::new(endian, macho::LC_SYMTAB), + cmdsize: U32::new(endian, symtab_command_len as u32), + symoff: U32::new(endian, symtab_offset as u32), + nsyms: U32::new(endian, nsyms as u32), + stroff: U32::new(endian, strtab_offset as u32), + strsize: U32::new(endian, strtab_data.len() as u32), + }; + buffer.write(&symtab_command); + + // Write section data. + for (index, section) in self.sections.iter().enumerate() { + let len = section.data.len(); + if len != 0 { + write_align(buffer, section.align as usize); + debug_assert_eq!(section_offsets[index].offset, buffer.len()); + buffer.write_bytes(§ion.data); + } + } + debug_assert_eq!(segment_file_offset + segment_file_size, buffer.len()); + + // Write symtab. + write_align(buffer, pointer_align); + debug_assert_eq!(symtab_offset, buffer.len()); + for (index, symbol) in self.symbols.iter().enumerate() { + if !symbol_offsets[index].emit { + continue; + } + // TODO: N_STAB + let (mut n_type, n_sect) = match symbol.section { + SymbolSection::Undefined => (macho::N_UNDF | macho::N_EXT, 0), + SymbolSection::Absolute => (macho::N_ABS, 0), + SymbolSection::Section(id) => (macho::N_SECT, id.0 + 1), + SymbolSection::None | SymbolSection::Common => { + return Err(Error(format!( + "unimplemented symbol `{}` section {:?}", + symbol.name().unwrap_or(""), + symbol.section + ))); + } + }; + match symbol.scope { + SymbolScope::Unknown | SymbolScope::Compilation => {} + SymbolScope::Linkage => { + n_type |= macho::N_EXT | macho::N_PEXT; + } + SymbolScope::Dynamic => { + n_type |= macho::N_EXT; + } + } + + let n_desc = if let SymbolFlags::MachO { n_desc } = symbol.flags { + n_desc + } else { + let mut n_desc = 0; + if symbol.weak { + if symbol.is_undefined() { + n_desc |= macho::N_WEAK_REF; + } else { + n_desc |= macho::N_WEAK_DEF; + } + } + n_desc + }; + + let n_value = match symbol.section.id() { + Some(section) => section_offsets[section.0].address + symbol.value, + None => symbol.value, + }; + + let n_strx = symbol_offsets[index] + .str_id + .map(|id| strtab.get_offset(id)) + .unwrap_or(0); + + macho.write_nlist( + buffer, + Nlist { + n_strx: n_strx as u32, + n_type, + n_sect: n_sect as u8, + n_desc, + n_value, + }, + ); + } + + // Write strtab. + debug_assert_eq!(strtab_offset, buffer.len()); + buffer.write_bytes(&strtab_data); + + // Write relocations. + for (index, section) in self.sections.iter().enumerate() { + if !section.relocations.is_empty() { + write_align(buffer, 4); + debug_assert_eq!(section_offsets[index].reloc_offset, buffer.len()); + for reloc in §ion.relocations { + let r_extern; + let r_symbolnum; + let symbol = &self.symbols[reloc.symbol.0]; + if symbol.kind == SymbolKind::Section { + r_symbolnum = section_offsets[symbol.section.id().unwrap().0].index as u32; + r_extern = false; + } else { + r_symbolnum = symbol_offsets[reloc.symbol.0].index as u32; + r_extern = true; + } + let r_length = match reloc.size { + 8 => 0, + 16 => 1, + 32 => 2, + 64 => 3, + _ => return Err(Error(format!("unimplemented reloc size {:?}", reloc))), + }; + let (r_pcrel, r_type) = match self.architecture { + Architecture::I386 => match reloc.kind { + RelocationKind::Absolute => (false, macho::GENERIC_RELOC_VANILLA), + _ => { + return Err(Error(format!("unimplemented relocation {:?}", reloc))); + } + }, + Architecture::X86_64 => match (reloc.kind, reloc.encoding, reloc.addend) { + (RelocationKind::Absolute, RelocationEncoding::Generic, 0) => { + (false, macho::X86_64_RELOC_UNSIGNED) + } + (RelocationKind::Relative, RelocationEncoding::Generic, -4) => { + (true, macho::X86_64_RELOC_SIGNED) + } + (RelocationKind::Relative, RelocationEncoding::X86RipRelative, -4) => { + (true, macho::X86_64_RELOC_SIGNED) + } + (RelocationKind::Relative, RelocationEncoding::X86Branch, -4) => { + (true, macho::X86_64_RELOC_BRANCH) + } + (RelocationKind::PltRelative, RelocationEncoding::X86Branch, -4) => { + (true, macho::X86_64_RELOC_BRANCH) + } + (RelocationKind::GotRelative, RelocationEncoding::Generic, -4) => { + (true, macho::X86_64_RELOC_GOT) + } + ( + RelocationKind::GotRelative, + RelocationEncoding::X86RipRelativeMovq, + -4, + ) => (true, macho::X86_64_RELOC_GOT_LOAD), + (RelocationKind::MachO { value, relative }, _, _) => (relative, value), + _ => { + return Err(Error(format!("unimplemented relocation {:?}", reloc))); + } + }, + Architecture::Aarch64 => match (reloc.kind, reloc.encoding, reloc.addend) { + (RelocationKind::Absolute, RelocationEncoding::Generic, 0) => { + (false, macho::ARM64_RELOC_UNSIGNED) + } + ( + RelocationKind::MachO { value, relative }, + RelocationEncoding::Generic, + 0, + ) => (relative, value), + _ => { + return Err(Error(format!("unimplemented relocation {:?}", reloc))); + } + }, + _ => { + if let RelocationKind::MachO { value, relative } = reloc.kind { + (relative, value) + } else { + return Err(Error(format!("unimplemented relocation {:?}", reloc))); + } + } + }; + let reloc_info = macho::RelocationInfo { + r_address: reloc.offset as u32, + r_symbolnum, + r_pcrel, + r_length, + r_extern, + r_type, + }; + buffer.write(&reloc_info.relocation(endian)); + } + } + } + + debug_assert_eq!(offset, buffer.len()); + + Ok(()) + } +} + +struct MachHeader { + cputype: u32, + cpusubtype: u32, + filetype: u32, + ncmds: u32, + sizeofcmds: u32, + flags: u32, +} + +struct SegmentCommand { + cmdsize: u32, + segname: [u8; 16], + vmaddr: u64, + vmsize: u64, + fileoff: u64, + filesize: u64, + maxprot: u32, + initprot: u32, + nsects: u32, + flags: u32, +} + +pub struct SectionHeader { + sectname: [u8; 16], + segname: [u8; 16], + addr: u64, + size: u64, + offset: u32, + align: u32, + reloff: u32, + nreloc: u32, + flags: u32, +} + +struct Nlist { + n_strx: u32, + n_type: u8, + n_sect: u8, + n_desc: u16, + n_value: u64, +} + +trait MachO { + fn mach_header_size(&self) -> usize; + fn segment_command_size(&self) -> usize; + fn section_header_size(&self) -> usize; + fn nlist_size(&self) -> usize; + fn write_mach_header(&self, buffer: &mut dyn WritableBuffer, section: MachHeader); + fn write_segment_command(&self, buffer: &mut dyn WritableBuffer, segment: SegmentCommand); + fn write_section(&self, buffer: &mut dyn WritableBuffer, section: SectionHeader); + fn write_nlist(&self, buffer: &mut dyn WritableBuffer, nlist: Nlist); +} + +struct MachO32 { + endian: E, +} + +impl MachO for MachO32 { + fn mach_header_size(&self) -> usize { + mem::size_of::>() + } + + fn segment_command_size(&self) -> usize { + mem::size_of::>() + } + + fn section_header_size(&self) -> usize { + mem::size_of::>() + } + + fn nlist_size(&self) -> usize { + mem::size_of::>() + } + + fn write_mach_header(&self, buffer: &mut dyn WritableBuffer, header: MachHeader) { + let endian = self.endian; + let magic = if endian.is_big_endian() { + macho::MH_MAGIC + } else { + macho::MH_CIGAM + }; + let header = macho::MachHeader32 { + magic: U32::new(BigEndian, magic), + cputype: U32::new(endian, header.cputype), + cpusubtype: U32::new(endian, header.cpusubtype), + filetype: U32::new(endian, header.filetype), + ncmds: U32::new(endian, header.ncmds), + sizeofcmds: U32::new(endian, header.sizeofcmds), + flags: U32::new(endian, header.flags), + }; + buffer.write(&header); + } + + fn write_segment_command(&self, buffer: &mut dyn WritableBuffer, segment: SegmentCommand) { + let endian = self.endian; + let segment = macho::SegmentCommand32 { + cmd: U32::new(endian, macho::LC_SEGMENT), + cmdsize: U32::new(endian, segment.cmdsize), + segname: segment.segname, + vmaddr: U32::new(endian, segment.vmaddr as u32), + vmsize: U32::new(endian, segment.vmsize as u32), + fileoff: U32::new(endian, segment.fileoff as u32), + filesize: U32::new(endian, segment.filesize as u32), + maxprot: U32::new(endian, segment.maxprot), + initprot: U32::new(endian, segment.initprot), + nsects: U32::new(endian, segment.nsects), + flags: U32::new(endian, segment.flags), + }; + buffer.write(&segment); + } + + fn write_section(&self, buffer: &mut dyn WritableBuffer, section: SectionHeader) { + let endian = self.endian; + let section = macho::Section32 { + sectname: section.sectname, + segname: section.segname, + addr: U32::new(endian, section.addr as u32), + size: U32::new(endian, section.size as u32), + offset: U32::new(endian, section.offset), + align: U32::new(endian, section.align), + reloff: U32::new(endian, section.reloff), + nreloc: U32::new(endian, section.nreloc), + flags: U32::new(endian, section.flags), + reserved1: U32::default(), + reserved2: U32::default(), + }; + buffer.write(§ion); + } + + fn write_nlist(&self, buffer: &mut dyn WritableBuffer, nlist: Nlist) { + let endian = self.endian; + let nlist = macho::Nlist32 { + n_strx: U32::new(endian, nlist.n_strx), + n_type: nlist.n_type, + n_sect: nlist.n_sect, + n_desc: U16::new(endian, nlist.n_desc), + n_value: U32::new(endian, nlist.n_value as u32), + }; + buffer.write(&nlist); + } +} + +struct MachO64 { + endian: E, +} + +impl MachO for MachO64 { + fn mach_header_size(&self) -> usize { + mem::size_of::>() + } + + fn segment_command_size(&self) -> usize { + mem::size_of::>() + } + + fn section_header_size(&self) -> usize { + mem::size_of::>() + } + + fn nlist_size(&self) -> usize { + mem::size_of::>() + } + + fn write_mach_header(&self, buffer: &mut dyn WritableBuffer, header: MachHeader) { + let endian = self.endian; + let magic = if endian.is_big_endian() { + macho::MH_MAGIC_64 + } else { + macho::MH_CIGAM_64 + }; + let header = macho::MachHeader64 { + magic: U32::new(BigEndian, magic), + cputype: U32::new(endian, header.cputype), + cpusubtype: U32::new(endian, header.cpusubtype), + filetype: U32::new(endian, header.filetype), + ncmds: U32::new(endian, header.ncmds), + sizeofcmds: U32::new(endian, header.sizeofcmds), + flags: U32::new(endian, header.flags), + reserved: U32::default(), + }; + buffer.write(&header); + } + + fn write_segment_command(&self, buffer: &mut dyn WritableBuffer, segment: SegmentCommand) { + let endian = self.endian; + let segment = macho::SegmentCommand64 { + cmd: U32::new(endian, macho::LC_SEGMENT_64), + cmdsize: U32::new(endian, segment.cmdsize), + segname: segment.segname, + vmaddr: U64::new(endian, segment.vmaddr), + vmsize: U64::new(endian, segment.vmsize), + fileoff: U64::new(endian, segment.fileoff), + filesize: U64::new(endian, segment.filesize), + maxprot: U32::new(endian, segment.maxprot), + initprot: U32::new(endian, segment.initprot), + nsects: U32::new(endian, segment.nsects), + flags: U32::new(endian, segment.flags), + }; + buffer.write(&segment); + } + + fn write_section(&self, buffer: &mut dyn WritableBuffer, section: SectionHeader) { + let endian = self.endian; + let section = macho::Section64 { + sectname: section.sectname, + segname: section.segname, + addr: U64::new(endian, section.addr), + size: U64::new(endian, section.size), + offset: U32::new(endian, section.offset), + align: U32::new(endian, section.align), + reloff: U32::new(endian, section.reloff), + nreloc: U32::new(endian, section.nreloc), + flags: U32::new(endian, section.flags), + reserved1: U32::default(), + reserved2: U32::default(), + reserved3: U32::default(), + }; + buffer.write(§ion); + } + + fn write_nlist(&self, buffer: &mut dyn WritableBuffer, nlist: Nlist) { + let endian = self.endian; + let nlist = macho::Nlist64 { + n_strx: U32::new(endian, nlist.n_strx), + n_type: nlist.n_type, + n_sect: nlist.n_sect, + n_desc: U16::new(endian, nlist.n_desc), + n_value: U64Bytes::new(endian, nlist.n_value), + }; + buffer.write(&nlist); + } +} diff --git a/crux-mir/lib/object/src/write/mod.rs b/crux-mir/lib/object/src/write/mod.rs new file mode 100644 index 000000000..aa4980b1e --- /dev/null +++ b/crux-mir/lib/object/src/write/mod.rs @@ -0,0 +1,918 @@ +//! Interface for writing object files. + +use alloc::borrow::Cow; +use alloc::string::String; +use alloc::vec::Vec; +use core::{fmt, result, str}; +#[cfg(not(feature = "std"))] +use hashbrown::HashMap; +#[cfg(feature = "std")] +use std::{boxed::Box, collections::HashMap, error, io}; + +use crate::endian::{Endianness, U32, U64}; +use crate::{ + Architecture, BinaryFormat, ComdatKind, FileFlags, RelocationEncoding, RelocationKind, + SectionFlags, SectionKind, SymbolFlags, SymbolKind, SymbolScope, +}; + +#[cfg(feature = "coff")] +mod coff; +#[cfg(feature = "coff")] +pub use coff::CoffExportStyle; + +#[cfg(feature = "elf")] +pub mod elf; + +#[cfg(feature = "macho")] +mod macho; + +#[cfg(feature = "pe")] +pub mod pe; + +mod string; +pub use string::StringId; + +mod util; +pub use util::*; + +/// The error type used within the write module. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Error(String); + +impl fmt::Display for Error { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str(&self.0) + } +} + +#[cfg(feature = "std")] +impl error::Error for Error {} + +/// The result type used within the write module. +pub type Result = result::Result; + +/// A writable object file. +#[derive(Debug)] +pub struct Object<'a> { + format: BinaryFormat, + architecture: Architecture, + endian: Endianness, + sections: Vec>, + standard_sections: HashMap, + symbols: Vec, + symbol_map: HashMap, SymbolId>, + stub_symbols: HashMap, + comdats: Vec, + /// File flags that are specific to each file format. + pub flags: FileFlags, + /// The symbol name mangling scheme. + pub mangling: Mangling, + /// Mach-O "_tlv_bootstrap" symbol. + tlv_bootstrap: Option, +} + +impl<'a> Object<'a> { + /// Create an empty object file. + pub fn new(format: BinaryFormat, architecture: Architecture, endian: Endianness) -> Object<'a> { + Object { + format, + architecture, + endian, + sections: Vec::new(), + standard_sections: HashMap::new(), + symbols: Vec::new(), + symbol_map: HashMap::new(), + stub_symbols: HashMap::new(), + comdats: Vec::new(), + flags: FileFlags::None, + mangling: Mangling::default(format, architecture), + tlv_bootstrap: None, + } + } + + /// Return the file format. + #[inline] + pub fn format(&self) -> BinaryFormat { + self.format + } + + /// Return the architecture. + #[inline] + pub fn architecture(&self) -> Architecture { + self.architecture + } + + /// Return the current mangling setting. + #[inline] + pub fn mangling(&self) -> Mangling { + self.mangling + } + + /// Specify the mangling setting. + #[inline] + pub fn set_mangling(&mut self, mangling: Mangling) { + self.mangling = mangling; + } + + /// Return the name for a standard segment. + /// + /// This will vary based on the file format. + #[allow(unused_variables)] + pub fn segment_name(&self, segment: StandardSegment) -> &'static [u8] { + match self.format { + #[cfg(feature = "coff")] + BinaryFormat::Coff => &[], + #[cfg(feature = "elf")] + BinaryFormat::Elf => &[], + #[cfg(feature = "macho")] + BinaryFormat::MachO => self.macho_segment_name(segment), + _ => unimplemented!(), + } + } + + /// Get the section with the given `SectionId`. + #[inline] + pub fn section(&self, section: SectionId) -> &Section<'a> { + &self.sections[section.0] + } + + /// Mutably get the section with the given `SectionId`. + #[inline] + pub fn section_mut(&mut self, section: SectionId) -> &mut Section<'a> { + &mut self.sections[section.0] + } + + /// Set the data for an existing section. + /// + /// Must not be called for sections that already have data, or that contain uninitialized data. + pub fn set_section_data(&mut self, section: SectionId, data: T, align: u64) + where + T: Into>, + { + self.sections[section.0].set_data(data, align) + } + + /// Append data to an existing section. Returns the section offset of the data. + pub fn append_section_data(&mut self, section: SectionId, data: &[u8], align: u64) -> u64 { + self.sections[section.0].append_data(data, align) + } + + /// Append zero-initialized data to an existing section. Returns the section offset of the data. + pub fn append_section_bss(&mut self, section: SectionId, size: u64, align: u64) -> u64 { + self.sections[section.0].append_bss(size, align) + } + + /// Return the `SectionId` of a standard section. + /// + /// If the section doesn't already exist then it is created. + pub fn section_id(&mut self, section: StandardSection) -> SectionId { + self.standard_sections + .get(§ion) + .cloned() + .unwrap_or_else(|| { + let (segment, name, kind) = self.section_info(section); + self.add_section(segment.to_vec(), name.to_vec(), kind) + }) + } + + /// Add a new section and return its `SectionId`. + /// + /// This also creates a section symbol. + pub fn add_section(&mut self, segment: Vec, name: Vec, kind: SectionKind) -> SectionId { + let id = SectionId(self.sections.len()); + self.sections.push(Section { + segment, + name, + kind, + size: 0, + align: 1, + data: Cow::Borrowed(&[]), + relocations: Vec::new(), + symbol: None, + flags: SectionFlags::None, + }); + + // Add to self.standard_sections if required. This may match multiple standard sections. + let section = &self.sections[id.0]; + for standard_section in StandardSection::all() { + if !self.standard_sections.contains_key(standard_section) { + let (segment, name, kind) = self.section_info(*standard_section); + if segment == &*section.segment && name == &*section.name && kind == section.kind { + self.standard_sections.insert(*standard_section, id); + } + } + } + + id + } + + fn section_info( + &self, + section: StandardSection, + ) -> (&'static [u8], &'static [u8], SectionKind) { + match self.format { + #[cfg(feature = "coff")] + BinaryFormat::Coff => self.coff_section_info(section), + #[cfg(feature = "elf")] + BinaryFormat::Elf => self.elf_section_info(section), + #[cfg(feature = "macho")] + BinaryFormat::MachO => self.macho_section_info(section), + _ => unimplemented!(), + } + } + + /// Add a subsection. Returns the `SectionId` and section offset of the data. + pub fn add_subsection( + &mut self, + section: StandardSection, + name: &[u8], + data: &[u8], + align: u64, + ) -> (SectionId, u64) { + let section_id = if self.has_subsections_via_symbols() { + self.set_subsections_via_symbols(); + self.section_id(section) + } else { + let (segment, name, kind) = self.subsection_info(section, name); + self.add_section(segment.to_vec(), name, kind) + }; + let offset = self.append_section_data(section_id, data, align); + (section_id, offset) + } + + fn has_subsections_via_symbols(&self) -> bool { + match self.format { + BinaryFormat::Coff | BinaryFormat::Elf => false, + BinaryFormat::MachO => true, + _ => unimplemented!(), + } + } + + fn set_subsections_via_symbols(&mut self) { + match self.format { + #[cfg(feature = "macho")] + BinaryFormat::MachO => self.macho_set_subsections_via_symbols(), + _ => unimplemented!(), + } + } + + fn subsection_info( + &self, + section: StandardSection, + value: &[u8], + ) -> (&'static [u8], Vec, SectionKind) { + let (segment, section, kind) = self.section_info(section); + let name = self.subsection_name(section, value); + (segment, name, kind) + } + + #[allow(unused_variables)] + fn subsection_name(&self, section: &[u8], value: &[u8]) -> Vec { + debug_assert!(!self.has_subsections_via_symbols()); + match self.format { + #[cfg(feature = "coff")] + BinaryFormat::Coff => self.coff_subsection_name(section, value), + #[cfg(feature = "elf")] + BinaryFormat::Elf => self.elf_subsection_name(section, value), + _ => unimplemented!(), + } + } + + /// Get the COMDAT section group with the given `ComdatId`. + #[inline] + pub fn comdat(&self, comdat: ComdatId) -> &Comdat { + &self.comdats[comdat.0] + } + + /// Mutably get the COMDAT section group with the given `ComdatId`. + #[inline] + pub fn comdat_mut(&mut self, comdat: ComdatId) -> &mut Comdat { + &mut self.comdats[comdat.0] + } + + /// Add a new COMDAT section group and return its `ComdatId`. + pub fn add_comdat(&mut self, comdat: Comdat) -> ComdatId { + let comdat_id = ComdatId(self.comdats.len()); + self.comdats.push(comdat); + comdat_id + } + + /// Get the `SymbolId` of the symbol with the given name. + pub fn symbol_id(&self, name: &[u8]) -> Option { + self.symbol_map.get(name).cloned() + } + + /// Get the symbol with the given `SymbolId`. + #[inline] + pub fn symbol(&self, symbol: SymbolId) -> &Symbol { + &self.symbols[symbol.0] + } + + /// Mutably get the symbol with the given `SymbolId`. + #[inline] + pub fn symbol_mut(&mut self, symbol: SymbolId) -> &mut Symbol { + &mut self.symbols[symbol.0] + } + + /// Add a new symbol and return its `SymbolId`. + pub fn add_symbol(&mut self, mut symbol: Symbol) -> SymbolId { + // Defined symbols must have a scope. + debug_assert!(symbol.is_undefined() || symbol.scope != SymbolScope::Unknown); + if symbol.kind == SymbolKind::Section { + // There can only be one section symbol, but update its flags, since + // the automatically generated section symbol will have none. + let symbol_id = self.section_symbol(symbol.section.id().unwrap()); + if symbol.flags != SymbolFlags::None { + self.symbol_mut(symbol_id).flags = symbol.flags; + } + return symbol_id; + } + if !symbol.name.is_empty() + && (symbol.kind == SymbolKind::Text + || symbol.kind == SymbolKind::Data + || symbol.kind == SymbolKind::Tls) + { + let unmangled_name = symbol.name.clone(); + if let Some(prefix) = self.mangling.global_prefix() { + symbol.name.insert(0, prefix); + } + let symbol_id = self.add_raw_symbol(symbol); + self.symbol_map.insert(unmangled_name, symbol_id); + symbol_id + } else { + self.add_raw_symbol(symbol) + } + } + + fn add_raw_symbol(&mut self, symbol: Symbol) -> SymbolId { + let symbol_id = SymbolId(self.symbols.len()); + self.symbols.push(symbol); + symbol_id + } + + /// Return true if the file format supports `StandardSection::UninitializedTls`. + #[inline] + pub fn has_uninitialized_tls(&self) -> bool { + self.format != BinaryFormat::Coff + } + + /// Return true if the file format supports `StandardSection::Common`. + #[inline] + pub fn has_common(&self) -> bool { + self.format == BinaryFormat::MachO + } + + /// Add a new common symbol and return its `SymbolId`. + /// + /// For Mach-O, this appends the symbol to the `__common` section. + pub fn add_common_symbol(&mut self, mut symbol: Symbol, size: u64, align: u64) -> SymbolId { + if self.has_common() { + let symbol_id = self.add_symbol(symbol); + let section = self.section_id(StandardSection::Common); + self.add_symbol_bss(symbol_id, section, size, align); + symbol_id + } else { + symbol.section = SymbolSection::Common; + symbol.size = size; + self.add_symbol(symbol) + } + } + + /// Add a new file symbol and return its `SymbolId`. + pub fn add_file_symbol(&mut self, name: Vec) -> SymbolId { + self.add_raw_symbol(Symbol { + name, + value: 0, + size: 0, + kind: SymbolKind::File, + scope: SymbolScope::Compilation, + weak: false, + section: SymbolSection::None, + flags: SymbolFlags::None, + }) + } + + /// Get the symbol for a section. + pub fn section_symbol(&mut self, section_id: SectionId) -> SymbolId { + let section = &mut self.sections[section_id.0]; + if let Some(symbol) = section.symbol { + return symbol; + } + let name = if self.format == BinaryFormat::Coff { + section.name.clone() + } else { + Vec::new() + }; + let symbol_id = SymbolId(self.symbols.len()); + self.symbols.push(Symbol { + name, + value: 0, + size: 0, + kind: SymbolKind::Section, + scope: SymbolScope::Compilation, + weak: false, + section: SymbolSection::Section(section_id), + flags: SymbolFlags::None, + }); + section.symbol = Some(symbol_id); + symbol_id + } + + /// Append data to an existing section, and update a symbol to refer to it. + /// + /// For Mach-O, this also creates a `__thread_vars` entry for TLS symbols, and the + /// symbol will indirectly point to the added data via the `__thread_vars` entry. + /// + /// Returns the section offset of the data. + pub fn add_symbol_data( + &mut self, + symbol_id: SymbolId, + section: SectionId, + data: &[u8], + align: u64, + ) -> u64 { + let offset = self.append_section_data(section, data, align); + self.set_symbol_data(symbol_id, section, offset, data.len() as u64); + offset + } + + /// Append zero-initialized data to an existing section, and update a symbol to refer to it. + /// + /// For Mach-O, this also creates a `__thread_vars` entry for TLS symbols, and the + /// symbol will indirectly point to the added data via the `__thread_vars` entry. + /// + /// Returns the section offset of the data. + pub fn add_symbol_bss( + &mut self, + symbol_id: SymbolId, + section: SectionId, + size: u64, + align: u64, + ) -> u64 { + let offset = self.append_section_bss(section, size, align); + self.set_symbol_data(symbol_id, section, offset, size); + offset + } + + /// Update a symbol to refer to the given data within a section. + /// + /// For Mach-O, this also creates a `__thread_vars` entry for TLS symbols, and the + /// symbol will indirectly point to the data via the `__thread_vars` entry. + #[allow(unused_mut)] + pub fn set_symbol_data( + &mut self, + mut symbol_id: SymbolId, + section: SectionId, + offset: u64, + size: u64, + ) { + // Defined symbols must have a scope. + debug_assert!(self.symbol(symbol_id).scope != SymbolScope::Unknown); + match self.format { + #[cfg(feature = "macho")] + BinaryFormat::MachO => symbol_id = self.macho_add_thread_var(symbol_id), + _ => {} + } + let symbol = self.symbol_mut(symbol_id); + symbol.value = offset; + symbol.size = size; + symbol.section = SymbolSection::Section(section); + } + + /// Convert a symbol to a section symbol and offset. + /// + /// Returns `None` if the symbol does not have a section. + pub fn symbol_section_and_offset(&mut self, symbol_id: SymbolId) -> Option<(SymbolId, u64)> { + let symbol = self.symbol(symbol_id); + if symbol.kind == SymbolKind::Section { + return Some((symbol_id, 0)); + } + let symbol_offset = symbol.value; + let section = symbol.section.id()?; + let section_symbol = self.section_symbol(section); + Some((section_symbol, symbol_offset)) + } + + /// Add a relocation to a section. + /// + /// Relocations must only be added after the referenced symbols have been added + /// and defined (if applicable). + pub fn add_relocation(&mut self, section: SectionId, mut relocation: Relocation) -> Result<()> { + let addend = match self.format { + #[cfg(feature = "coff")] + BinaryFormat::Coff => self.coff_fixup_relocation(&mut relocation), + #[cfg(feature = "elf")] + BinaryFormat::Elf => self.elf_fixup_relocation(&mut relocation)?, + #[cfg(feature = "macho")] + BinaryFormat::MachO => self.macho_fixup_relocation(&mut relocation), + _ => unimplemented!(), + }; + if addend != 0 { + self.write_relocation_addend(section, &relocation, addend)?; + } + self.sections[section.0].relocations.push(relocation); + Ok(()) + } + + fn write_relocation_addend( + &mut self, + section: SectionId, + relocation: &Relocation, + addend: i64, + ) -> Result<()> { + let data = self.sections[section.0].data_mut(); + let offset = relocation.offset as usize; + match relocation.size { + 32 => data.write_at(offset, &U32::new(self.endian, addend as u32)), + 64 => data.write_at(offset, &U64::new(self.endian, addend as u64)), + _ => { + return Err(Error(format!( + "unimplemented relocation addend {:?}", + relocation + ))); + } + } + .map_err(|_| { + Error(format!( + "invalid relocation offset {}+{} (max {})", + relocation.offset, + relocation.size, + data.len() + )) + }) + } + + /// Write the object to a `Vec`. + pub fn write(&self) -> Result> { + let mut buffer = Vec::new(); + self.emit(&mut buffer)?; + Ok(buffer) + } + + /// Write the object to a `Write` implementation. + /// + /// Also flushes the writer. + /// + /// It is advisable to use a buffered writer like [`BufWriter`](std::io::BufWriter) + /// instead of an unbuffered writer like [`File`](std::fs::File). + #[cfg(feature = "std")] + pub fn write_stream(&self, w: W) -> result::Result<(), Box> { + let mut stream = StreamingBuffer::new(w); + self.emit(&mut stream)?; + stream.result()?; + stream.into_inner().flush()?; + Ok(()) + } + + /// Write the object to a `WritableBuffer`. + pub fn emit(&self, buffer: &mut dyn WritableBuffer) -> Result<()> { + match self.format { + #[cfg(feature = "coff")] + BinaryFormat::Coff => self.coff_write(buffer), + #[cfg(feature = "elf")] + BinaryFormat::Elf => self.elf_write(buffer), + #[cfg(feature = "macho")] + BinaryFormat::MachO => self.macho_write(buffer), + _ => unimplemented!(), + } + } +} + +/// A standard segment kind. +#[allow(missing_docs)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[non_exhaustive] +pub enum StandardSegment { + Text, + Data, + Debug, +} + +/// A standard section kind. +#[allow(missing_docs)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[non_exhaustive] +pub enum StandardSection { + Text, + Data, + ReadOnlyData, + ReadOnlyDataWithRel, + ReadOnlyString, + UninitializedData, + Tls, + /// Zero-fill TLS initializers. Unsupported for COFF. + UninitializedTls, + /// TLS variable structures. Only supported for Mach-O. + TlsVariables, + /// Common data. Only supported for Mach-O. + Common, +} + +impl StandardSection { + /// Return the section kind of a standard section. + pub fn kind(self) -> SectionKind { + match self { + StandardSection::Text => SectionKind::Text, + StandardSection::Data => SectionKind::Data, + StandardSection::ReadOnlyData | StandardSection::ReadOnlyDataWithRel => { + SectionKind::ReadOnlyData + } + StandardSection::ReadOnlyString => SectionKind::ReadOnlyString, + StandardSection::UninitializedData => SectionKind::UninitializedData, + StandardSection::Tls => SectionKind::Tls, + StandardSection::UninitializedTls => SectionKind::UninitializedTls, + StandardSection::TlsVariables => SectionKind::TlsVariables, + StandardSection::Common => SectionKind::Common, + } + } + + // TODO: remembering to update this is error-prone, can we do better? + fn all() -> &'static [StandardSection] { + &[ + StandardSection::Text, + StandardSection::Data, + StandardSection::ReadOnlyData, + StandardSection::ReadOnlyDataWithRel, + StandardSection::ReadOnlyString, + StandardSection::UninitializedData, + StandardSection::Tls, + StandardSection::UninitializedTls, + StandardSection::TlsVariables, + StandardSection::Common, + ] + } +} + +/// An identifier used to reference a section. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct SectionId(usize); + +/// A section in an object file. +#[derive(Debug)] +pub struct Section<'a> { + segment: Vec, + name: Vec, + kind: SectionKind, + size: u64, + align: u64, + data: Cow<'a, [u8]>, + relocations: Vec, + symbol: Option, + /// Section flags that are specific to each file format. + pub flags: SectionFlags, +} + +impl<'a> Section<'a> { + /// Try to convert the name to a utf8 string. + #[inline] + pub fn name(&self) -> Option<&str> { + str::from_utf8(&self.name).ok() + } + + /// Try to convert the segment to a utf8 string. + #[inline] + pub fn segment(&self) -> Option<&str> { + str::from_utf8(&self.segment).ok() + } + + /// Return true if this section contains zerofill data. + #[inline] + pub fn is_bss(&self) -> bool { + self.kind.is_bss() + } + + /// Set the data for a section. + /// + /// Must not be called for sections that already have data, or that contain uninitialized data. + pub fn set_data(&mut self, data: T, align: u64) + where + T: Into>, + { + debug_assert!(!self.is_bss()); + debug_assert_eq!(align & (align - 1), 0); + debug_assert!(self.data.is_empty()); + self.data = data.into(); + self.size = self.data.len() as u64; + self.align = align; + } + + /// Append data to a section. + /// + /// Must not be called for sections that contain uninitialized data. + pub fn append_data(&mut self, append_data: &[u8], align: u64) -> u64 { + debug_assert!(!self.is_bss()); + debug_assert_eq!(align & (align - 1), 0); + if self.align < align { + self.align = align; + } + let align = align as usize; + let data = self.data.to_mut(); + let mut offset = data.len(); + if offset & (align - 1) != 0 { + offset += align - (offset & (align - 1)); + data.resize(offset, 0); + } + data.extend_from_slice(append_data); + self.size = data.len() as u64; + offset as u64 + } + + /// Append unitialized data to a section. + /// + /// Must not be called for sections that contain initialized data. + pub fn append_bss(&mut self, size: u64, align: u64) -> u64 { + debug_assert!(self.is_bss()); + debug_assert_eq!(align & (align - 1), 0); + if self.align < align { + self.align = align; + } + let mut offset = self.size; + if offset & (align - 1) != 0 { + offset += align - (offset & (align - 1)); + self.size = offset; + } + self.size += size; + offset as u64 + } + + /// Returns the section as-built so far. + /// + /// This requires that the section is not a bss section. + pub fn data(&self) -> &[u8] { + debug_assert!(!self.is_bss()); + &self.data + } + + /// Returns the section as-built so far. + /// + /// This requires that the section is not a bss section. + pub fn data_mut(&mut self) -> &mut [u8] { + debug_assert!(!self.is_bss()); + self.data.to_mut() + } +} + +/// The section where a symbol is defined. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum SymbolSection { + /// The section is not applicable for this symbol (such as file symbols). + None, + /// The symbol is undefined. + Undefined, + /// The symbol has an absolute value. + Absolute, + /// The symbol is a zero-initialized symbol that will be combined with duplicate definitions. + Common, + /// The symbol is defined in the given section. + Section(SectionId), +} + +impl SymbolSection { + /// Returns the section id for the section where the symbol is defined. + /// + /// May return `None` if the symbol is not defined in a section. + #[inline] + pub fn id(self) -> Option { + if let SymbolSection::Section(id) = self { + Some(id) + } else { + None + } + } +} + +/// An identifier used to reference a symbol. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct SymbolId(usize); + +/// A symbol in an object file. +#[derive(Debug)] +pub struct Symbol { + /// The name of the symbol. + pub name: Vec, + /// The value of the symbol. + /// + /// If the symbol defined in a section, then this is the section offset of the symbol. + pub value: u64, + /// The size of the symbol. + pub size: u64, + /// The kind of the symbol. + pub kind: SymbolKind, + /// The scope of the symbol. + pub scope: SymbolScope, + /// Whether the symbol has weak binding. + pub weak: bool, + /// The section containing the symbol. + pub section: SymbolSection, + /// Symbol flags that are specific to each file format. + pub flags: SymbolFlags, +} + +impl Symbol { + /// Try to convert the name to a utf8 string. + #[inline] + pub fn name(&self) -> Option<&str> { + str::from_utf8(&self.name).ok() + } + + /// Return true if the symbol is undefined. + #[inline] + pub fn is_undefined(&self) -> bool { + self.section == SymbolSection::Undefined + } + + /// Return true if the symbol is common data. + /// + /// Note: does not check for `SymbolSection::Section` with `SectionKind::Common`. + #[inline] + pub fn is_common(&self) -> bool { + self.section == SymbolSection::Common + } + + /// Return true if the symbol scope is local. + #[inline] + pub fn is_local(&self) -> bool { + self.scope == SymbolScope::Compilation + } +} + +/// A relocation in an object file. +#[derive(Debug)] +pub struct Relocation { + /// The section offset of the place of the relocation. + pub offset: u64, + /// The size in bits of the place of relocation. + pub size: u8, + /// The operation used to calculate the result of the relocation. + pub kind: RelocationKind, + /// Information about how the result of the relocation operation is encoded in the place. + pub encoding: RelocationEncoding, + /// The symbol referred to by the relocation. + /// + /// This may be a section symbol. + pub symbol: SymbolId, + /// The addend to use in the relocation calculation. + /// + /// This may be in addition to an implicit addend stored at the place of the relocation. + pub addend: i64, +} + +/// An identifier used to reference a COMDAT section group. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ComdatId(usize); + +/// A COMDAT section group. +#[derive(Debug)] +pub struct Comdat { + /// The COMDAT selection kind. + /// + /// This determines the way in which the linker resolves multiple definitions of the COMDAT + /// sections. + pub kind: ComdatKind, + /// The COMDAT symbol. + /// + /// If this symbol is referenced, then all sections in the group will be included by the + /// linker. + pub symbol: SymbolId, + /// The sections in the group. + pub sections: Vec, +} + +/// The symbol name mangling scheme. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum Mangling { + /// No symbol mangling. + None, + /// Windows COFF symbol mangling. + Coff, + /// Windows COFF i386 symbol mangling. + CoffI386, + /// ELF symbol mangling. + Elf, + /// Mach-O symbol mangling. + MachO, +} + +impl Mangling { + /// Return the default symboling mangling for the given format and architecture. + pub fn default(format: BinaryFormat, architecture: Architecture) -> Self { + match (format, architecture) { + (BinaryFormat::Coff, Architecture::I386) => Mangling::CoffI386, + (BinaryFormat::Coff, _) => Mangling::Coff, + (BinaryFormat::Elf, _) => Mangling::Elf, + (BinaryFormat::MachO, _) => Mangling::MachO, + _ => Mangling::None, + } + } + + /// Return the prefix to use for global symbols. + pub fn global_prefix(self) -> Option { + match self { + Mangling::None | Mangling::Elf | Mangling::Coff => None, + Mangling::CoffI386 | Mangling::MachO => Some(b'_'), + } + } +} diff --git a/crux-mir/lib/object/src/write/pe.rs b/crux-mir/lib/object/src/write/pe.rs new file mode 100644 index 000000000..70da3a093 --- /dev/null +++ b/crux-mir/lib/object/src/write/pe.rs @@ -0,0 +1,847 @@ +//! Helper for writing PE files. +use alloc::string::String; +use alloc::vec::Vec; +use core::mem; + +use crate::endian::{LittleEndian as LE, *}; +use crate::pe; +use crate::write::util; +use crate::write::{Error, Result, WritableBuffer}; + +/// A helper for writing PE files. +/// +/// Writing uses a two phase approach. The first phase reserves file ranges and virtual +/// address ranges for everything in the order that they will be written. +/// +/// The second phase writes everything out in order. Thus the caller must ensure writing +/// is in the same order that file ranges were reserved. +#[allow(missing_debug_implementations)] +pub struct Writer<'a> { + is_64: bool, + section_alignment: u32, + file_alignment: u32, + + buffer: &'a mut dyn WritableBuffer, + len: u32, + virtual_len: u32, + headers_len: u32, + + code_address: u32, + data_address: u32, + code_len: u32, + data_len: u32, + bss_len: u32, + + nt_headers_offset: u32, + data_directories: Vec, + section_header_num: u16, + sections: Vec
, + + symbol_offset: u32, + symbol_num: u32, + + reloc_blocks: Vec, + relocs: Vec>, + reloc_offset: u32, +} + +impl<'a> Writer<'a> { + /// Create a new `Writer`. + pub fn new( + is_64: bool, + section_alignment: u32, + file_alignment: u32, + buffer: &'a mut dyn WritableBuffer, + ) -> Self { + Writer { + is_64, + section_alignment, + file_alignment, + + buffer, + len: 0, + virtual_len: 0, + headers_len: 0, + + code_address: 0, + data_address: 0, + code_len: 0, + data_len: 0, + bss_len: 0, + + nt_headers_offset: 0, + data_directories: Vec::new(), + section_header_num: 0, + sections: Vec::new(), + + symbol_offset: 0, + symbol_num: 0, + + reloc_blocks: Vec::new(), + relocs: Vec::new(), + reloc_offset: 0, + } + } + + /// Return the current virtual address size that has been reserved. + /// + /// This is only valid after section headers have been reserved. + pub fn virtual_len(&self) -> u32 { + self.virtual_len + } + + /// Reserve a virtual address range with the given size. + /// + /// The reserved length will be increased to match the section alignment. + /// + /// Returns the aligned offset of the start of the range. + pub fn reserve_virtual(&mut self, len: u32) -> u32 { + let offset = self.virtual_len; + self.virtual_len += len; + self.virtual_len = util::align_u32(self.virtual_len, self.section_alignment); + offset + } + + /// Reserve up to the given virtual address. + /// + /// The reserved length will be increased to match the section alignment. + pub fn reserve_virtual_until(&mut self, address: u32) { + debug_assert!(self.virtual_len <= address); + self.virtual_len = util::align_u32(address, self.section_alignment); + } + + /// Return the current file length that has been reserved. + pub fn reserved_len(&self) -> u32 { + self.len + } + + /// Return the current file length that has been written. + #[allow(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + self.buffer.len() + } + + /// Reserve a file range with the given size and starting alignment. + /// + /// Returns the aligned offset of the start of the range. + pub fn reserve(&mut self, len: u32, align_start: u32) -> u32 { + if len == 0 { + return self.len; + } + self.reserve_align(align_start); + let offset = self.len; + self.len += len; + offset + } + + /// Reserve a file range with the given size and using the file alignment. + /// + /// Returns the aligned offset of the start of the range. + pub fn reserve_file(&mut self, len: u32) -> u32 { + self.reserve(len, self.file_alignment) + } + + /// Write data. + pub fn write(&mut self, data: &[u8]) { + self.buffer.write_bytes(data); + } + + /// Reserve alignment padding bytes. + pub fn reserve_align(&mut self, align_start: u32) { + self.len = util::align_u32(self.len, align_start); + } + + /// Write alignment padding bytes. + pub fn write_align(&mut self, align_start: u32) { + util::write_align(self.buffer, align_start as usize); + } + + /// Write padding up to the next multiple of file alignment. + pub fn write_file_align(&mut self) { + self.write_align(self.file_alignment); + } + + /// Reserve the file range up to the given file offset. + pub fn reserve_until(&mut self, offset: u32) { + debug_assert!(self.len <= offset); + self.len = offset; + } + + /// Write padding up to the given file offset. + pub fn pad_until(&mut self, offset: u32) { + debug_assert!(self.buffer.len() <= offset as usize); + self.buffer.resize(offset as usize); + } + + /// Reserve the range for the DOS header. + /// + /// This must be at the start of the file. + /// + /// When writing, you may use `write_custom_dos_header` or `write_empty_dos_header`. + pub fn reserve_dos_header(&mut self) { + debug_assert_eq!(self.len, 0); + self.reserve(mem::size_of::() as u32, 1); + } + + /// Write a custom DOS header. + /// + /// This must be at the start of the file. + pub fn write_custom_dos_header(&mut self, dos_header: &pe::ImageDosHeader) -> Result<()> { + debug_assert_eq!(self.buffer.len(), 0); + + // Start writing. + self.buffer + .reserve(self.len as usize) + .map_err(|_| Error(String::from("Cannot allocate buffer")))?; + + self.buffer.write(dos_header); + Ok(()) + } + + /// Write the DOS header for a file without a stub. + /// + /// This must be at the start of the file. + /// + /// Uses default values for all fields. + pub fn write_empty_dos_header(&mut self) -> Result<()> { + self.write_custom_dos_header(&pe::ImageDosHeader { + e_magic: U16::new(LE, pe::IMAGE_DOS_SIGNATURE), + e_cblp: U16::new(LE, 0), + e_cp: U16::new(LE, 0), + e_crlc: U16::new(LE, 0), + e_cparhdr: U16::new(LE, 0), + e_minalloc: U16::new(LE, 0), + e_maxalloc: U16::new(LE, 0), + e_ss: U16::new(LE, 0), + e_sp: U16::new(LE, 0), + e_csum: U16::new(LE, 0), + e_ip: U16::new(LE, 0), + e_cs: U16::new(LE, 0), + e_lfarlc: U16::new(LE, 0), + e_ovno: U16::new(LE, 0), + e_res: [U16::new(LE, 0); 4], + e_oemid: U16::new(LE, 0), + e_oeminfo: U16::new(LE, 0), + e_res2: [U16::new(LE, 0); 10], + e_lfanew: U32::new(LE, self.nt_headers_offset), + }) + } + + /// Reserve a fixed DOS header and stub. + /// + /// Use `reserve_dos_header` and `reserve` if you need a custom stub. + pub fn reserve_dos_header_and_stub(&mut self) { + self.reserve_dos_header(); + self.reserve(64, 1); + } + + /// Write a fixed DOS header and stub. + /// + /// Use `write_custom_dos_header` and `write` if you need a custom stub. + pub fn write_dos_header_and_stub(&mut self) -> Result<()> { + self.write_custom_dos_header(&pe::ImageDosHeader { + e_magic: U16::new(LE, pe::IMAGE_DOS_SIGNATURE), + e_cblp: U16::new(LE, 0x90), + e_cp: U16::new(LE, 3), + e_crlc: U16::new(LE, 0), + e_cparhdr: U16::new(LE, 4), + e_minalloc: U16::new(LE, 0), + e_maxalloc: U16::new(LE, 0xffff), + e_ss: U16::new(LE, 0), + e_sp: U16::new(LE, 0xb8), + e_csum: U16::new(LE, 0), + e_ip: U16::new(LE, 0), + e_cs: U16::new(LE, 0), + e_lfarlc: U16::new(LE, 0x40), + e_ovno: U16::new(LE, 0), + e_res: [U16::new(LE, 0); 4], + e_oemid: U16::new(LE, 0), + e_oeminfo: U16::new(LE, 0), + e_res2: [U16::new(LE, 0); 10], + e_lfanew: U32::new(LE, self.nt_headers_offset), + })?; + + #[rustfmt::skip] + self.buffer.write_bytes(&[ + 0x0e, 0x1f, 0xba, 0x0e, 0x00, 0xb4, 0x09, 0xcd, + 0x21, 0xb8, 0x01, 0x4c, 0xcd, 0x21, 0x54, 0x68, + 0x69, 0x73, 0x20, 0x70, 0x72, 0x6f, 0x67, 0x72, + 0x61, 0x6d, 0x20, 0x63, 0x61, 0x6e, 0x6e, 0x6f, + 0x74, 0x20, 0x62, 0x65, 0x20, 0x72, 0x75, 0x6e, + 0x20, 0x69, 0x6e, 0x20, 0x44, 0x4f, 0x53, 0x20, + 0x6d, 0x6f, 0x64, 0x65, 0x2e, 0x0d, 0x0d, 0x0a, + 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]); + + Ok(()) + } + + fn nt_headers_size(&self) -> u32 { + if self.is_64 { + mem::size_of::() as u32 + } else { + mem::size_of::() as u32 + } + } + + fn optional_header_size(&self) -> u32 { + let size = if self.is_64 { + mem::size_of::() as u32 + } else { + mem::size_of::() as u32 + }; + size + self.data_directories.len() as u32 * mem::size_of::() as u32 + } + + /// Return the offset of the NT headers, if reserved. + pub fn nt_headers_offset(&self) -> u32 { + self.nt_headers_offset + } + + /// Reserve the range for the NT headers. + pub fn reserve_nt_headers(&mut self, data_directory_num: usize) { + debug_assert_eq!(self.nt_headers_offset, 0); + self.nt_headers_offset = self.reserve(self.nt_headers_size(), 8); + self.data_directories = vec![DataDirectory::default(); data_directory_num]; + self.reserve( + data_directory_num as u32 * mem::size_of::() as u32, + 1, + ); + } + + /// Set the virtual address and size of a data directory. + pub fn set_data_directory(&mut self, index: usize, virtual_address: u32, size: u32) { + self.data_directories[index] = DataDirectory { + virtual_address, + size, + } + } + + /// Write the NT headers. + pub fn write_nt_headers(&mut self, nt_headers: NtHeaders) { + self.pad_until(self.nt_headers_offset); + self.buffer.write(&U32::new(LE, pe::IMAGE_NT_SIGNATURE)); + let file_header = pe::ImageFileHeader { + machine: U16::new(LE, nt_headers.machine), + number_of_sections: U16::new(LE, self.section_header_num), + time_date_stamp: U32::new(LE, nt_headers.time_date_stamp), + pointer_to_symbol_table: U32::new(LE, self.symbol_offset), + number_of_symbols: U32::new(LE, self.symbol_num), + size_of_optional_header: U16::new(LE, self.optional_header_size() as u16), + characteristics: U16::new(LE, nt_headers.characteristics), + }; + self.buffer.write(&file_header); + if self.is_64 { + let optional_header = pe::ImageOptionalHeader64 { + magic: U16::new(LE, pe::IMAGE_NT_OPTIONAL_HDR64_MAGIC), + major_linker_version: nt_headers.major_linker_version, + minor_linker_version: nt_headers.minor_linker_version, + size_of_code: U32::new(LE, self.code_len), + size_of_initialized_data: U32::new(LE, self.data_len), + size_of_uninitialized_data: U32::new(LE, self.bss_len), + address_of_entry_point: U32::new(LE, nt_headers.address_of_entry_point), + base_of_code: U32::new(LE, self.code_address), + image_base: U64::new(LE, nt_headers.image_base), + section_alignment: U32::new(LE, self.section_alignment), + file_alignment: U32::new(LE, self.file_alignment), + major_operating_system_version: U16::new( + LE, + nt_headers.major_operating_system_version, + ), + minor_operating_system_version: U16::new( + LE, + nt_headers.minor_operating_system_version, + ), + major_image_version: U16::new(LE, nt_headers.major_image_version), + minor_image_version: U16::new(LE, nt_headers.minor_image_version), + major_subsystem_version: U16::new(LE, nt_headers.major_subsystem_version), + minor_subsystem_version: U16::new(LE, nt_headers.minor_subsystem_version), + win32_version_value: U32::new(LE, 0), + size_of_image: U32::new(LE, self.virtual_len), + size_of_headers: U32::new(LE, self.headers_len), + check_sum: U32::new(LE, 0), + subsystem: U16::new(LE, nt_headers.subsystem), + dll_characteristics: U16::new(LE, nt_headers.dll_characteristics), + size_of_stack_reserve: U64::new(LE, nt_headers.size_of_stack_reserve), + size_of_stack_commit: U64::new(LE, nt_headers.size_of_stack_commit), + size_of_heap_reserve: U64::new(LE, nt_headers.size_of_heap_reserve), + size_of_heap_commit: U64::new(LE, nt_headers.size_of_heap_commit), + loader_flags: U32::new(LE, 0), + number_of_rva_and_sizes: U32::new(LE, self.data_directories.len() as u32), + }; + self.buffer.write(&optional_header); + } else { + let optional_header = pe::ImageOptionalHeader32 { + magic: U16::new(LE, pe::IMAGE_NT_OPTIONAL_HDR32_MAGIC), + major_linker_version: nt_headers.major_linker_version, + minor_linker_version: nt_headers.minor_linker_version, + size_of_code: U32::new(LE, self.code_len), + size_of_initialized_data: U32::new(LE, self.data_len), + size_of_uninitialized_data: U32::new(LE, self.bss_len), + address_of_entry_point: U32::new(LE, nt_headers.address_of_entry_point), + base_of_code: U32::new(LE, self.code_address), + base_of_data: U32::new(LE, self.data_address), + image_base: U32::new(LE, nt_headers.image_base as u32), + section_alignment: U32::new(LE, self.section_alignment), + file_alignment: U32::new(LE, self.file_alignment), + major_operating_system_version: U16::new( + LE, + nt_headers.major_operating_system_version, + ), + minor_operating_system_version: U16::new( + LE, + nt_headers.minor_operating_system_version, + ), + major_image_version: U16::new(LE, nt_headers.major_image_version), + minor_image_version: U16::new(LE, nt_headers.minor_image_version), + major_subsystem_version: U16::new(LE, nt_headers.major_subsystem_version), + minor_subsystem_version: U16::new(LE, nt_headers.minor_subsystem_version), + win32_version_value: U32::new(LE, 0), + size_of_image: U32::new(LE, self.virtual_len), + size_of_headers: U32::new(LE, self.headers_len), + check_sum: U32::new(LE, 0), + subsystem: U16::new(LE, nt_headers.subsystem), + dll_characteristics: U16::new(LE, nt_headers.dll_characteristics), + size_of_stack_reserve: U32::new(LE, nt_headers.size_of_stack_reserve as u32), + size_of_stack_commit: U32::new(LE, nt_headers.size_of_stack_commit as u32), + size_of_heap_reserve: U32::new(LE, nt_headers.size_of_heap_reserve as u32), + size_of_heap_commit: U32::new(LE, nt_headers.size_of_heap_commit as u32), + loader_flags: U32::new(LE, 0), + number_of_rva_and_sizes: U32::new(LE, self.data_directories.len() as u32), + }; + self.buffer.write(&optional_header); + } + + for dir in &self.data_directories { + self.buffer.write(&pe::ImageDataDirectory { + virtual_address: U32::new(LE, dir.virtual_address), + size: U32::new(LE, dir.size), + }) + } + } + + /// Reserve the section headers. + /// + /// The number of reserved section headers must be the same as the number of sections that + /// are later reserved. + // TODO: change this to a maximum number of sections? + pub fn reserve_section_headers(&mut self, section_header_num: u16) { + debug_assert_eq!(self.section_header_num, 0); + self.section_header_num = section_header_num; + self.reserve( + u32::from(section_header_num) * mem::size_of::() as u32, + 1, + ); + // Padding before sections must be included in headers_len. + self.reserve_align(self.file_alignment); + self.headers_len = self.len; + self.reserve_virtual(self.len); + } + + /// Write the section headers. + /// + /// This uses information that was recorded when the sections were reserved. + pub fn write_section_headers(&mut self) { + debug_assert_eq!(self.section_header_num as usize, self.sections.len()); + for section in &self.sections { + let section_header = pe::ImageSectionHeader { + name: section.name, + virtual_size: U32::new(LE, section.range.virtual_size), + virtual_address: U32::new(LE, section.range.virtual_address), + size_of_raw_data: U32::new(LE, section.range.file_size), + pointer_to_raw_data: U32::new(LE, section.range.file_offset), + pointer_to_relocations: U32::new(LE, 0), + pointer_to_linenumbers: U32::new(LE, 0), + number_of_relocations: U16::new(LE, 0), + number_of_linenumbers: U16::new(LE, 0), + characteristics: U32::new(LE, section.characteristics), + }; + self.buffer.write(§ion_header); + } + } + + /// Reserve a section. + /// + /// Returns the file range and virtual address range that are reserved + /// for the section. + pub fn reserve_section( + &mut self, + name: [u8; 8], + characteristics: u32, + virtual_size: u32, + data_size: u32, + ) -> SectionRange { + let virtual_address = self.reserve_virtual(virtual_size); + + // Padding after section must be included in section file size. + let file_size = util::align_u32(data_size, self.file_alignment); + let file_offset = if file_size != 0 { + self.reserve(file_size, self.file_alignment) + } else { + 0 + }; + + // Sizes in optional header use the virtual size with the file alignment. + let aligned_virtual_size = util::align_u32(virtual_size, self.file_alignment); + if characteristics & pe::IMAGE_SCN_CNT_CODE != 0 { + if self.code_address == 0 { + self.code_address = virtual_address; + } + self.code_len += aligned_virtual_size; + } else if characteristics & pe::IMAGE_SCN_CNT_INITIALIZED_DATA != 0 { + if self.data_address == 0 { + self.data_address = virtual_address; + } + self.data_len += aligned_virtual_size; + } else if characteristics & pe::IMAGE_SCN_CNT_UNINITIALIZED_DATA != 0 { + if self.data_address == 0 { + self.data_address = virtual_address; + } + self.bss_len += aligned_virtual_size; + } + + let range = SectionRange { + virtual_address, + virtual_size, + file_offset, + file_size, + }; + self.sections.push(Section { + name, + characteristics, + range, + }); + range + } + + /// Write the data for a section. + pub fn write_section(&mut self, offset: u32, data: &[u8]) { + if data.is_empty() { + return; + } + self.pad_until(offset); + self.write(data); + self.write_align(self.file_alignment); + } + + /// Reserve a `.text` section. + /// + /// Contains executable code. + pub fn reserve_text_section(&mut self, size: u32) -> SectionRange { + self.reserve_section( + *b".text\0\0\0", + pe::IMAGE_SCN_CNT_CODE | pe::IMAGE_SCN_MEM_EXECUTE | pe::IMAGE_SCN_MEM_READ, + size, + size, + ) + } + + /// Reserve a `.data` section. + /// + /// Contains initialized data. + /// + /// May also contain uninitialized data if `virtual_size` is greater than `data_size`. + pub fn reserve_data_section(&mut self, virtual_size: u32, data_size: u32) -> SectionRange { + self.reserve_section( + *b".data\0\0\0", + pe::IMAGE_SCN_CNT_INITIALIZED_DATA | pe::IMAGE_SCN_MEM_READ | pe::IMAGE_SCN_MEM_WRITE, + virtual_size, + data_size, + ) + } + + /// Reserve a `.rdata` section. + /// + /// Contains read-only initialized data. + pub fn reserve_rdata_section(&mut self, size: u32) -> SectionRange { + self.reserve_section( + *b".rdata\0\0", + pe::IMAGE_SCN_CNT_INITIALIZED_DATA | pe::IMAGE_SCN_MEM_READ, + size, + size, + ) + } + + /// Reserve a `.bss` section. + /// + /// Contains uninitialized data. + pub fn reserve_bss_section(&mut self, size: u32) -> SectionRange { + self.reserve_section( + *b".bss\0\0\0\0", + pe::IMAGE_SCN_CNT_UNINITIALIZED_DATA | pe::IMAGE_SCN_MEM_READ | pe::IMAGE_SCN_MEM_WRITE, + size, + 0, + ) + } + + /// Reserve an `.idata` section. + /// + /// Contains import tables. Note that it is permissible to store import tables in a different + /// section. + /// + /// This also sets the `pe::IMAGE_DIRECTORY_ENTRY_IMPORT` data directory. + pub fn reserve_idata_section(&mut self, size: u32) -> SectionRange { + let range = self.reserve_section( + *b".idata\0\0", + pe::IMAGE_SCN_CNT_INITIALIZED_DATA | pe::IMAGE_SCN_MEM_READ | pe::IMAGE_SCN_MEM_WRITE, + size, + size, + ); + let dir = &mut self.data_directories[pe::IMAGE_DIRECTORY_ENTRY_IMPORT]; + debug_assert_eq!(dir.virtual_address, 0); + *dir = DataDirectory { + virtual_address: range.virtual_address, + size, + }; + range + } + + /// Reserve an `.edata` section. + /// + /// Contains export tables. + /// + /// This also sets the `pe::IMAGE_DIRECTORY_ENTRY_EXPORT` data directory. + pub fn reserve_edata_section(&mut self, size: u32) -> SectionRange { + let range = self.reserve_section( + *b".edata\0\0", + pe::IMAGE_SCN_CNT_INITIALIZED_DATA | pe::IMAGE_SCN_MEM_READ, + size, + size, + ); + let dir = &mut self.data_directories[pe::IMAGE_DIRECTORY_ENTRY_EXPORT]; + debug_assert_eq!(dir.virtual_address, 0); + *dir = DataDirectory { + virtual_address: range.virtual_address, + size, + }; + range + } + + /// Reserve a `.pdata` section. + /// + /// Contains exception information. + /// + /// This also sets the `pe::IMAGE_DIRECTORY_ENTRY_EXCEPTION` data directory. + pub fn reserve_pdata_section(&mut self, size: u32) -> SectionRange { + let range = self.reserve_section( + *b".pdata\0\0", + pe::IMAGE_SCN_CNT_INITIALIZED_DATA | pe::IMAGE_SCN_MEM_READ, + size, + size, + ); + let dir = &mut self.data_directories[pe::IMAGE_DIRECTORY_ENTRY_EXCEPTION]; + debug_assert_eq!(dir.virtual_address, 0); + *dir = DataDirectory { + virtual_address: range.virtual_address, + size, + }; + range + } + + /// Reserve a `.xdata` section. + /// + /// Contains exception information. + pub fn reserve_xdata_section(&mut self, size: u32) -> SectionRange { + self.reserve_section( + *b".xdata\0\0", + pe::IMAGE_SCN_CNT_INITIALIZED_DATA | pe::IMAGE_SCN_MEM_READ, + size, + size, + ) + } + + /// Reserve a `.rsrc` section. + /// + /// Contains the resource directory. + /// + /// This also sets the `pe::IMAGE_DIRECTORY_ENTRY_RESOURCE` data directory. + pub fn reserve_rsrc_section(&mut self, size: u32) -> SectionRange { + let range = self.reserve_section( + *b".rsrc\0\0\0", + pe::IMAGE_SCN_CNT_INITIALIZED_DATA | pe::IMAGE_SCN_MEM_READ, + size, + size, + ); + let dir = &mut self.data_directories[pe::IMAGE_DIRECTORY_ENTRY_RESOURCE]; + debug_assert_eq!(dir.virtual_address, 0); + *dir = DataDirectory { + virtual_address: range.virtual_address, + size, + }; + range + } + + /// Add a base relocation. + /// + /// `typ` must be one of the `IMAGE_REL_BASED_*` constants. + pub fn add_reloc(&mut self, mut virtual_address: u32, typ: u16) { + let reloc = U16::new(LE, typ << 12 | (virtual_address & 0xfff) as u16); + virtual_address &= !0xfff; + if let Some(block) = self.reloc_blocks.last_mut() { + if block.virtual_address == virtual_address { + self.relocs.push(reloc); + block.count += 1; + return; + } + // Blocks must have an even number of relocations. + if block.count & 1 != 0 { + self.relocs.push(U16::new(LE, 0)); + block.count += 1; + } + debug_assert!(block.virtual_address < virtual_address); + } + self.relocs.push(reloc); + self.reloc_blocks.push(RelocBlock { + virtual_address, + count: 1, + }); + } + + /// Return true if a base relocation has been added. + pub fn has_relocs(&mut self) -> bool { + !self.relocs.is_empty() + } + + /// Reserve a `.reloc` section. + /// + /// This contains the base relocations that were added with `add_reloc`. + /// + /// This also sets the `pe::IMAGE_DIRECTORY_ENTRY_BASERELOC` data directory. + pub fn reserve_reloc_section(&mut self) -> SectionRange { + if let Some(block) = self.reloc_blocks.last_mut() { + // Blocks must have an even number of relocations. + if block.count & 1 != 0 { + self.relocs.push(U16::new(LE, 0)); + block.count += 1; + } + } + let size = self.reloc_blocks.iter().map(RelocBlock::size).sum(); + let range = self.reserve_section( + *b".reloc\0\0", + pe::IMAGE_SCN_CNT_INITIALIZED_DATA + | pe::IMAGE_SCN_MEM_READ + | pe::IMAGE_SCN_MEM_DISCARDABLE, + size, + size, + ); + let dir = &mut self.data_directories[pe::IMAGE_DIRECTORY_ENTRY_BASERELOC]; + debug_assert_eq!(dir.virtual_address, 0); + *dir = DataDirectory { + virtual_address: range.virtual_address, + size, + }; + self.reloc_offset = range.file_offset; + range + } + + /// Write a `.reloc` section. + /// + /// This contains the base relocations that were added with `add_reloc`. + pub fn write_reloc_section(&mut self) { + if self.reloc_offset == 0 { + return; + } + self.pad_until(self.reloc_offset); + + let mut total = 0; + for block in &self.reloc_blocks { + self.buffer.write(&pe::ImageBaseRelocation { + virtual_address: U32::new(LE, block.virtual_address), + size_of_block: U32::new(LE, block.size()), + }); + self.buffer + .write_slice(&self.relocs[total..][..block.count as usize]); + total += block.count as usize; + } + debug_assert_eq!(total, self.relocs.len()); + + self.write_align(self.file_alignment); + } + + /// Reserve the certificate table. + /// + /// This also sets the `pe::IMAGE_DIRECTORY_ENTRY_SECURITY` data directory. + // TODO: reserve individual certificates + pub fn reserve_certificate_table(&mut self, size: u32) { + let size = util::align_u32(size, 8); + let offset = self.reserve(size, 8); + let dir = &mut self.data_directories[pe::IMAGE_DIRECTORY_ENTRY_SECURITY]; + debug_assert_eq!(dir.virtual_address, 0); + *dir = DataDirectory { + virtual_address: offset, + size, + }; + } + + /// Write the certificate table. + // TODO: write individual certificates + pub fn write_certificate_table(&mut self, data: &[u8]) { + let dir = self.data_directories[pe::IMAGE_DIRECTORY_ENTRY_SECURITY]; + self.pad_until(dir.virtual_address); + self.write(data); + self.pad_until(dir.virtual_address + dir.size); + } +} + +/// Information required for writing [`pe::ImageNtHeaders32`] or [`pe::ImageNtHeaders64`]. +#[allow(missing_docs)] +#[derive(Debug, Clone)] +pub struct NtHeaders { + // ImageFileHeader + pub machine: u16, + pub time_date_stamp: u32, + pub characteristics: u16, + // ImageOptionalHeader + pub major_linker_version: u8, + pub minor_linker_version: u8, + pub address_of_entry_point: u32, + pub image_base: u64, + pub major_operating_system_version: u16, + pub minor_operating_system_version: u16, + pub major_image_version: u16, + pub minor_image_version: u16, + pub major_subsystem_version: u16, + pub minor_subsystem_version: u16, + pub subsystem: u16, + pub dll_characteristics: u16, + pub size_of_stack_reserve: u64, + pub size_of_stack_commit: u64, + pub size_of_heap_reserve: u64, + pub size_of_heap_commit: u64, +} + +#[derive(Default, Clone, Copy)] +struct DataDirectory { + virtual_address: u32, + size: u32, +} + +/// Information required for writing [`pe::ImageSectionHeader`]. +#[allow(missing_docs)] +#[derive(Debug, Clone)] +pub struct Section { + pub name: [u8; pe::IMAGE_SIZEOF_SHORT_NAME], + pub characteristics: u32, + pub range: SectionRange, +} + +/// The file range and virtual address range for a section. +#[allow(missing_docs)] +#[derive(Debug, Default, Clone, Copy)] +pub struct SectionRange { + pub virtual_address: u32, + pub virtual_size: u32, + pub file_offset: u32, + pub file_size: u32, +} + +struct RelocBlock { + virtual_address: u32, + count: u32, +} + +impl RelocBlock { + fn size(&self) -> u32 { + mem::size_of::() as u32 + self.count * mem::size_of::() as u32 + } +} diff --git a/crux-mir/lib/object/src/write/string.rs b/crux-mir/lib/object/src/write/string.rs new file mode 100644 index 000000000..3bdfcccaa --- /dev/null +++ b/crux-mir/lib/object/src/write/string.rs @@ -0,0 +1,159 @@ +use alloc::vec::Vec; + +#[cfg(feature = "std")] +type IndexSet = indexmap::IndexSet; +#[cfg(not(feature = "std"))] +type IndexSet = indexmap::IndexSet; + +/// An identifer for an entry in a string table. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StringId(usize); + +#[derive(Debug, Default)] +pub(crate) struct StringTable<'a> { + strings: IndexSet<&'a [u8]>, + offsets: Vec, +} + +impl<'a> StringTable<'a> { + /// Add a string to the string table. + /// + /// Panics if the string table has already been written, or + /// if the string contains a null byte. + pub fn add(&mut self, string: &'a [u8]) -> StringId { + assert!(self.offsets.is_empty()); + assert!(!string.contains(&0)); + let id = self.strings.insert_full(string).0; + StringId(id) + } + + /// Return the id of the given string. + /// + /// Panics if the string is not in the string table. + pub fn get_id(&self, string: &[u8]) -> StringId { + let id = self.strings.get_index_of(string).unwrap(); + StringId(id) + } + + /// Return the string for the given id. + /// + /// Panics if the string is not in the string table. + pub fn get_string(&self, id: StringId) -> &'a [u8] { + self.strings.get_index(id.0).unwrap() + } + + /// Return the offset of the given string. + /// + /// Panics if the string table has not been written, or + /// if the string is not in the string table. + pub fn get_offset(&self, id: StringId) -> usize { + self.offsets[id.0] + } + + /// Append the string table to the given `Vec`, and + /// calculate the list of string offsets. + /// + /// `base` is the initial string table offset. For example, + /// this should be 1 for ELF, to account for the initial + /// null byte (which must have been written by the caller). + pub fn write(&mut self, base: usize, w: &mut Vec) { + assert!(self.offsets.is_empty()); + + let mut ids: Vec<_> = (0..self.strings.len()).collect(); + sort(&mut ids, 1, &self.strings); + + self.offsets = vec![0; ids.len()]; + let mut offset = base; + let mut previous = &[][..]; + for id in ids { + let string = self.strings.get_index(id).unwrap(); + if previous.ends_with(string) { + self.offsets[id] = offset - string.len() - 1; + } else { + self.offsets[id] = offset; + w.extend_from_slice(string); + w.push(0); + offset += string.len() + 1; + previous = string; + } + } + } +} + +// Multi-key quicksort. +// +// Ordering is such that if a string is a suffix of at least one other string, +// then it is placed immediately after one of those strings. That is: +// - comparison starts at the end of the string +// - shorter strings come later +// +// Based on the implementation in LLVM. +fn sort(mut ids: &mut [usize], mut pos: usize, strings: &IndexSet<&[u8]>) { + loop { + if ids.len() <= 1 { + return; + } + + let pivot = byte(ids[0], pos, strings); + let mut lower = 0; + let mut upper = ids.len(); + let mut i = 1; + while i < upper { + let b = byte(ids[i], pos, strings); + if b > pivot { + ids.swap(lower, i); + lower += 1; + i += 1; + } else if b < pivot { + upper -= 1; + ids.swap(upper, i); + } else { + i += 1; + } + } + + sort(&mut ids[..lower], pos, strings); + sort(&mut ids[upper..], pos, strings); + + if pivot == 0 { + return; + } + ids = &mut ids[lower..upper]; + pos += 1; + } +} + +fn byte(id: usize, pos: usize, strings: &IndexSet<&[u8]>) -> u8 { + let string = strings.get_index(id).unwrap(); + let len = string.len(); + if len >= pos { + string[len - pos] + } else { + // We know the strings don't contain null bytes. + 0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn string_table() { + let mut table = StringTable::default(); + let id0 = table.add(b""); + let id1 = table.add(b"foo"); + let id2 = table.add(b"bar"); + let id3 = table.add(b"foobar"); + + let mut data = Vec::new(); + data.push(0); + table.write(1, &mut data); + assert_eq!(data, b"\0foobar\0foo\0"); + + assert_eq!(table.get_offset(id0), 11); + assert_eq!(table.get_offset(id1), 8); + assert_eq!(table.get_offset(id2), 4); + assert_eq!(table.get_offset(id3), 1); + } +} diff --git a/crux-mir/lib/object/src/write/util.rs b/crux-mir/lib/object/src/write/util.rs new file mode 100644 index 000000000..51bc3515f --- /dev/null +++ b/crux-mir/lib/object/src/write/util.rs @@ -0,0 +1,210 @@ +use alloc::vec::Vec; +#[cfg(feature = "std")] +use std::{io, mem}; + +use crate::pod::{bytes_of, bytes_of_slice, Pod}; + +/// Trait for writable buffer. +#[allow(clippy::len_without_is_empty)] +pub trait WritableBuffer { + /// Returns position/offset for data to be written at. + /// + /// Should only be used in debug assertions + fn len(&self) -> usize; + + /// Reserves specified number of bytes in the buffer. + /// + /// This will be called exactly once before writing anything to the buffer, + /// and the given size is the exact total number of bytes that will be written. + fn reserve(&mut self, size: usize) -> Result<(), ()>; + + /// Writes zero bytes at the end of the buffer until the buffer + /// has the specified length. + fn resize(&mut self, new_len: usize); + + /// Writes the specified slice of bytes at the end of the buffer. + fn write_bytes(&mut self, val: &[u8]); + + /// Writes the specified `Pod` type at the end of the buffer. + fn write_pod(&mut self, val: &T) + where + Self: Sized, + { + self.write_bytes(bytes_of(val)) + } + + /// Writes the specified `Pod` slice at the end of the buffer. + fn write_pod_slice(&mut self, val: &[T]) + where + Self: Sized, + { + self.write_bytes(bytes_of_slice(val)) + } +} + +impl<'a> dyn WritableBuffer + 'a { + /// Writes the specified `Pod` type at the end of the buffer. + pub fn write(&mut self, val: &T) { + self.write_bytes(bytes_of(val)) + } + + /// Writes the specified `Pod` slice at the end of the buffer. + pub fn write_slice(&mut self, val: &[T]) { + self.write_bytes(bytes_of_slice(val)) + } +} + +impl WritableBuffer for Vec { + #[inline] + fn len(&self) -> usize { + self.len() + } + + #[inline] + fn reserve(&mut self, size: usize) -> Result<(), ()> { + debug_assert!(self.is_empty()); + self.reserve(size); + Ok(()) + } + + #[inline] + fn resize(&mut self, new_len: usize) { + debug_assert!(new_len >= self.len()); + self.resize(new_len, 0); + } + + #[inline] + fn write_bytes(&mut self, val: &[u8]) { + debug_assert!(self.len() + val.len() <= self.capacity()); + self.extend_from_slice(val) + } +} + +/// A [`WritableBuffer`] that streams data to a [`Write`](std::io::Write) implementation. +/// +/// [`Self::result`] must be called to determine if an I/O error occurred during writing. +/// +/// It is advisable to use a buffered writer like [`BufWriter`](std::io::BufWriter) +/// instead of an unbuffered writer like [`File`](std::fs::File). +#[cfg(feature = "std")] +#[derive(Debug)] +pub struct StreamingBuffer { + writer: W, + len: usize, + result: Result<(), io::Error>, +} + +#[cfg(feature = "std")] +impl StreamingBuffer { + /// Create a new `StreamingBuffer` backed by the given writer. + pub fn new(writer: W) -> Self { + StreamingBuffer { + writer, + len: 0, + result: Ok(()), + } + } + + /// Unwraps this [`StreamingBuffer`] giving back the original writer. + pub fn into_inner(self) -> W { + self.writer + } + + /// Returns any error that occurred during writing. + pub fn result(&mut self) -> Result<(), io::Error> { + mem::replace(&mut self.result, Ok(())) + } +} + +#[cfg(feature = "std")] +impl WritableBuffer for StreamingBuffer { + #[inline] + fn len(&self) -> usize { + self.len + } + + #[inline] + fn reserve(&mut self, _size: usize) -> Result<(), ()> { + Ok(()) + } + + #[inline] + fn resize(&mut self, new_len: usize) { + debug_assert!(self.len <= new_len); + while self.len < new_len { + let write_amt = (new_len - self.len - 1) % 1024 + 1; + self.write_bytes(&[0; 1024][..write_amt]); + } + } + + #[inline] + fn write_bytes(&mut self, val: &[u8]) { + if self.result.is_ok() { + self.result = self.writer.write_all(val); + } + self.len += val.len(); + } +} + +/// A trait for mutable byte slices. +/// +/// It provides convenience methods for `Pod` types. +pub(crate) trait BytesMut { + fn write_at(self, offset: usize, val: &T) -> Result<(), ()>; +} + +impl<'a> BytesMut for &'a mut [u8] { + #[inline] + fn write_at(self, offset: usize, val: &T) -> Result<(), ()> { + let src = bytes_of(val); + let dest = self.get_mut(offset..).ok_or(())?; + let dest = dest.get_mut(..src.len()).ok_or(())?; + dest.copy_from_slice(src); + Ok(()) + } +} + +pub(crate) fn align(offset: usize, size: usize) -> usize { + (offset + (size - 1)) & !(size - 1) +} + +#[allow(dead_code)] +pub(crate) fn align_u32(offset: u32, size: u32) -> u32 { + (offset + (size - 1)) & !(size - 1) +} + +#[allow(dead_code)] +pub(crate) fn align_u64(offset: u64, size: u64) -> u64 { + (offset + (size - 1)) & !(size - 1) +} + +pub(crate) fn write_align(buffer: &mut dyn WritableBuffer, size: usize) { + let new_len = align(buffer.len(), size); + buffer.resize(new_len); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bytes_mut() { + let data = vec![0x01, 0x23, 0x45, 0x67]; + + let mut bytes = data.clone(); + bytes.extend_from_slice(bytes_of(&u16::to_be(0x89ab))); + assert_eq!(bytes, [0x01, 0x23, 0x45, 0x67, 0x89, 0xab]); + + let mut bytes = data.clone(); + assert_eq!(bytes.write_at(0, &u16::to_be(0x89ab)), Ok(())); + assert_eq!(bytes, [0x89, 0xab, 0x45, 0x67]); + + let mut bytes = data.clone(); + assert_eq!(bytes.write_at(2, &u16::to_be(0x89ab)), Ok(())); + assert_eq!(bytes, [0x01, 0x23, 0x89, 0xab]); + + assert_eq!(bytes.write_at(3, &u16::to_be(0x89ab)), Err(())); + assert_eq!(bytes.write_at(4, &u16::to_be(0x89ab)), Err(())); + assert_eq!(vec![].write_at(0, &u32::to_be(0x89ab)), Err(())); + } +} diff --git a/crux-mir/lib/object/tests/integration.rs b/crux-mir/lib/object/tests/integration.rs new file mode 100644 index 000000000..6ebcb547f --- /dev/null +++ b/crux-mir/lib/object/tests/integration.rs @@ -0,0 +1,2 @@ +mod read; +mod round_trip; diff --git a/crux-mir/lib/object/tests/parse_self.rs b/crux-mir/lib/object/tests/parse_self.rs new file mode 100644 index 000000000..1e7df6715 --- /dev/null +++ b/crux-mir/lib/object/tests/parse_self.rs @@ -0,0 +1,25 @@ +#![cfg(feature = "read")] +use object::{File, Object}; +use std::{env, fs}; + +#[test] +fn parse_self() { + let exe = env::current_exe().unwrap(); + let data = fs::read(exe).unwrap(); + let object = File::parse(&*data).unwrap(); + assert!(object.entry() != 0); + assert!(object.sections().count() != 0); +} + +#[cfg(feature = "std")] +#[test] +fn parse_self_cache() { + use object::read::{ReadCache, ReadRef}; + let exe = env::current_exe().unwrap(); + let file = fs::File::open(exe).unwrap(); + let cache = ReadCache::new(file); + let data = cache.range(0, cache.len().unwrap()); + let object = File::parse(data).unwrap(); + assert!(object.entry() != 0); + assert!(object.sections().count() != 0); +} diff --git a/crux-mir/lib/object/tests/read/coff.rs b/crux-mir/lib/object/tests/read/coff.rs new file mode 100644 index 000000000..3e61ec284 --- /dev/null +++ b/crux-mir/lib/object/tests/read/coff.rs @@ -0,0 +1,23 @@ +use object::{pe, read, Object, ObjectSection}; +use std::fs; +use std::path::PathBuf; + +#[cfg(feature = "coff")] +#[test] +fn coff_extended_relocations() { + let path_to_obj: PathBuf = ["testfiles", "coff", "relocs_overflow.o"].iter().collect(); + let contents = fs::read(&path_to_obj).expect("Could not read relocs_overflow.o"); + let file = + read::coff::CoffFile::parse(&contents[..]).expect("Could not parse relocs_overflow.o"); + let code_section = file + .section_by_name(".text") + .expect("Could not find .text section in relocs_overflow.o"); + match code_section.flags() { + object::SectionFlags::Coff { characteristics } => { + assert!(characteristics & pe::IMAGE_SCN_LNK_NRELOC_OVFL != 0) + } + _ => panic!("Invalid section flags flavour."), + }; + let relocations = code_section.relocations().collect::>(); + assert_eq!(relocations.len(), 65536); +} diff --git a/crux-mir/lib/object/tests/read/mod.rs b/crux-mir/lib/object/tests/read/mod.rs new file mode 100644 index 000000000..d60d1933b --- /dev/null +++ b/crux-mir/lib/object/tests/read/mod.rs @@ -0,0 +1,3 @@ +#![cfg(feature = "read")] + +mod coff; diff --git a/crux-mir/lib/object/tests/round_trip/bss.rs b/crux-mir/lib/object/tests/round_trip/bss.rs new file mode 100644 index 000000000..1354fcc78 --- /dev/null +++ b/crux-mir/lib/object/tests/round_trip/bss.rs @@ -0,0 +1,255 @@ +#![cfg(all(feature = "read", feature = "write"))] + +use object::read::{Object, ObjectSection, ObjectSymbol}; +use object::{read, write}; +use object::{ + Architecture, BinaryFormat, Endianness, SectionKind, SymbolFlags, SymbolKind, SymbolScope, +}; + +#[test] +fn coff_x86_64_bss() { + let mut object = + write::Object::new(BinaryFormat::Coff, Architecture::X86_64, Endianness::Little); + + let section = object.section_id(write::StandardSection::UninitializedData); + + let symbol = object.add_symbol(write::Symbol { + name: b"v1".to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Data, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Undefined, + flags: SymbolFlags::None, + }); + object.add_symbol_bss(symbol, section, 18, 4); + + let symbol = object.add_symbol(write::Symbol { + name: b"v2".to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Data, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Undefined, + flags: SymbolFlags::None, + }); + object.add_symbol_bss(symbol, section, 34, 8); + + let bytes = object.write().unwrap(); + + //std::fs::write(&"bss.o", &bytes).unwrap(); + + let object = read::File::parse(&*bytes).unwrap(); + assert_eq!(object.format(), BinaryFormat::Coff); + assert_eq!(object.architecture(), Architecture::X86_64); + + let mut sections = object.sections(); + + let bss = sections.next().unwrap(); + println!("{:?}", bss); + let bss_index = bss.index(); + assert_eq!(bss.name(), Ok(".bss")); + assert_eq!(bss.kind(), SectionKind::UninitializedData); + assert_eq!(bss.size(), 58); + assert_eq!(bss.data(), Ok(&[][..])); + + let section = sections.next(); + assert!(section.is_none(), "unexpected section {:?}", section); + + let mut symbols = object.symbols(); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("v1")); + assert_eq!(symbol.kind(), SymbolKind::Data); + assert_eq!(symbol.section_index(), Some(bss_index)); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + assert_eq!(symbol.address(), 0); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("v2")); + assert_eq!(symbol.kind(), SymbolKind::Data); + assert_eq!(symbol.section_index(), Some(bss_index)); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + assert_eq!(symbol.address(), 24); + + let symbol = symbols.next(); + assert!(symbol.is_none(), "unexpected symbol {:?}", symbol); +} + +#[test] +fn elf_x86_64_bss() { + let mut object = + write::Object::new(BinaryFormat::Elf, Architecture::X86_64, Endianness::Little); + + let section = object.section_id(write::StandardSection::UninitializedData); + + let symbol = object.add_symbol(write::Symbol { + name: b"v1".to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Data, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Undefined, + flags: SymbolFlags::None, + }); + object.add_symbol_bss(symbol, section, 18, 4); + + let symbol = object.add_symbol(write::Symbol { + name: b"v2".to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Data, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Undefined, + flags: SymbolFlags::None, + }); + object.add_symbol_bss(symbol, section, 34, 8); + + let bytes = object.write().unwrap(); + + //std::fs::write(&"bss.o", &bytes).unwrap(); + + let object = read::File::parse(&*bytes).unwrap(); + assert_eq!(object.format(), BinaryFormat::Elf); + assert_eq!(object.architecture(), Architecture::X86_64); + + let mut sections = object.sections(); + + let section = sections.next().unwrap(); + println!("{:?}", section); + assert_eq!(section.name(), Ok("")); + assert_eq!(section.kind(), SectionKind::Metadata); + assert_eq!(section.address(), 0); + assert_eq!(section.size(), 0); + + let bss = sections.next().unwrap(); + println!("{:?}", bss); + let bss_index = bss.index(); + assert_eq!(bss.name(), Ok(".bss")); + assert_eq!(bss.kind(), SectionKind::UninitializedData); + assert_eq!(bss.size(), 58); + assert_eq!(bss.data(), Ok(&[][..])); + + let mut symbols = object.symbols(); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("")); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("v1")); + assert_eq!(symbol.kind(), SymbolKind::Data); + assert_eq!(symbol.section_index(), Some(bss_index)); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + assert_eq!(symbol.address(), 0); + assert_eq!(symbol.size(), 18); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("v2")); + assert_eq!(symbol.kind(), SymbolKind::Data); + assert_eq!(symbol.section_index(), Some(bss_index)); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + assert_eq!(symbol.address(), 24); + assert_eq!(symbol.size(), 34); + + let symbol = symbols.next(); + assert!(symbol.is_none(), "unexpected symbol {:?}", symbol); +} + +#[test] +fn macho_x86_64_bss() { + let mut object = write::Object::new( + BinaryFormat::MachO, + Architecture::X86_64, + Endianness::Little, + ); + + let section = object.section_id(write::StandardSection::UninitializedData); + + let symbol = object.add_symbol(write::Symbol { + name: b"v1".to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Data, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Undefined, + flags: SymbolFlags::None, + }); + object.add_symbol_bss(symbol, section, 18, 4); + + let symbol = object.add_symbol(write::Symbol { + name: b"v2".to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Data, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Undefined, + flags: SymbolFlags::None, + }); + object.add_symbol_bss(symbol, section, 34, 8); + + let bytes = object.write().unwrap(); + + //std::fs::write(&"bss.o", &bytes).unwrap(); + + let object = read::File::parse(&*bytes).unwrap(); + assert_eq!(object.format(), BinaryFormat::MachO); + assert_eq!(object.architecture(), Architecture::X86_64); + + let mut sections = object.sections(); + + let bss = sections.next().unwrap(); + println!("{:?}", bss); + let bss_index = bss.index(); + assert_eq!(bss.name(), Ok("__bss")); + assert_eq!(bss.segment_name(), Ok(Some("__DATA"))); + assert_eq!(bss.kind(), SectionKind::UninitializedData); + assert_eq!(bss.size(), 58); + assert_eq!(bss.data(), Ok(&[][..])); + + let section = sections.next(); + assert!(section.is_none(), "unexpected section {:?}", section); + + let mut symbols = object.symbols(); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("_v1")); + assert_eq!(symbol.kind(), SymbolKind::Data); + assert_eq!(symbol.section_index(), Some(bss_index)); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + assert_eq!(symbol.address(), 0); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("_v2")); + assert_eq!(symbol.kind(), SymbolKind::Data); + assert_eq!(symbol.section_index(), Some(bss_index)); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + assert_eq!(symbol.address(), 24); + + let symbol = symbols.next(); + assert!(symbol.is_none(), "unexpected symbol {:?}", symbol); +} diff --git a/crux-mir/lib/object/tests/round_trip/coff.rs b/crux-mir/lib/object/tests/round_trip/coff.rs new file mode 100644 index 000000000..6785dc367 --- /dev/null +++ b/crux-mir/lib/object/tests/round_trip/coff.rs @@ -0,0 +1,56 @@ +use object::read::{Object, ObjectSection}; +use object::{read, write}; +use object::{ + Architecture, BinaryFormat, Endianness, RelocationEncoding, RelocationKind, SymbolFlags, + SymbolKind, SymbolScope, +}; + +#[test] +fn reloc_overflow() { + let mut object = + write::Object::new(BinaryFormat::Coff, Architecture::X86_64, Endianness::Little); + let text = object.section_id(write::StandardSection::Text); + object.append_section_data(text, &[0; 4], 4); + let symbol = object.add_symbol(write::Symbol { + name: b"f".to_vec(), + value: 0, + size: 4, + kind: SymbolKind::Text, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Section(text), + flags: SymbolFlags::None, + }); + for i in 0..0x10000 { + object + .add_relocation( + text, + write::Relocation { + offset: i, + size: 64, + kind: RelocationKind::Absolute, + encoding: RelocationEncoding::Generic, + symbol, + addend: 0, + }, + ) + .unwrap(); + } + let bytes = object.write().unwrap(); + + //std::fs::write(&"reloc_overflow.o", &bytes).unwrap(); + + let object = read::File::parse(&*bytes).unwrap(); + assert_eq!(object.format(), BinaryFormat::Coff); + assert_eq!(object.architecture(), Architecture::X86_64); + + let section = object.sections().next().unwrap(); + assert_eq!(section.name(), Ok(".text")); + + let mut i = 0; + for (offset, _relocation) in section.relocations() { + assert_eq!(offset, i); + i += 1; + } + assert_eq!(i, 0x10000); +} diff --git a/crux-mir/lib/object/tests/round_trip/comdat.rs b/crux-mir/lib/object/tests/round_trip/comdat.rs new file mode 100644 index 000000000..7b697a04c --- /dev/null +++ b/crux-mir/lib/object/tests/round_trip/comdat.rs @@ -0,0 +1,225 @@ +#![cfg(all(feature = "read", feature = "write"))] + +use object::pe; +use object::read::{Object, ObjectComdat, ObjectSection, ObjectSymbol}; +use object::{read, write}; +use object::{ + Architecture, BinaryFormat, ComdatKind, Endianness, SectionKind, SymbolFlags, SymbolKind, + SymbolScope, +}; + +#[test] +fn coff_x86_64_comdat() { + let mut object = + write::Object::new(BinaryFormat::Coff, Architecture::X86_64, Endianness::Little); + + let (section1, offset) = + object.add_subsection(write::StandardSection::Text, b"s1", &[0, 1, 2, 3], 4); + object.section_symbol(section1); + let (section2, _) = + object.add_subsection(write::StandardSection::Data, b"s1", &[0, 1, 2, 3], 4); + object.section_symbol(section2); + + let symbol = object.add_symbol(write::Symbol { + name: b"s1".to_vec(), + value: offset, + size: 4, + kind: SymbolKind::Data, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Section(section1), + flags: SymbolFlags::None, + }); + + object.add_comdat(write::Comdat { + kind: ComdatKind::NoDuplicates, + symbol, + sections: vec![section1, section2], + }); + + let bytes = object.write().unwrap(); + + //std::fs::write(&"comdat.o", &bytes).unwrap(); + + let object = read::File::parse(&*bytes).unwrap(); + assert_eq!(object.format(), BinaryFormat::Coff); + assert_eq!(object.architecture(), Architecture::X86_64); + + let mut sections = object.sections(); + + let section1 = sections.next().unwrap(); + println!("{:?}", section1); + let section1_index = section1.index(); + assert_eq!(section1.name(), Ok(".text$s1")); + assert_eq!(section1.kind(), SectionKind::Text); + assert_eq!(section1.address(), 0); + assert_eq!(section1.size(), 4); + + let section2 = sections.next().unwrap(); + println!("{:?}", section2); + let section2_index = section2.index(); + assert_eq!(section2.name(), Ok(".data$s1")); + assert_eq!(section2.kind(), SectionKind::Data); + assert_eq!(section2.address(), 0); + assert_eq!(section2.size(), 4); + + let mut symbols = object.symbols(); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok(".text$s1")); + assert_eq!(symbol.kind(), SymbolKind::Section); + assert_eq!( + symbol.section(), + read::SymbolSection::Section(section1.index()) + ); + assert_eq!( + symbol.flags(), + SymbolFlags::CoffSection { + selection: pe::IMAGE_COMDAT_SELECT_NODUPLICATES, + associative_section: None + } + ); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok(".data$s1")); + assert_eq!(symbol.kind(), SymbolKind::Section); + assert_eq!( + symbol.section(), + read::SymbolSection::Section(section2.index()) + ); + assert_eq!( + symbol.flags(), + SymbolFlags::CoffSection { + selection: pe::IMAGE_COMDAT_SELECT_ASSOCIATIVE, + associative_section: Some(section1_index) + } + ); + + let symbol = symbols.next().unwrap(); + let symbol_index = symbol.index(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("s1")); + assert_eq!(symbol.kind(), SymbolKind::Data); + assert_eq!( + symbol.section(), + read::SymbolSection::Section(section1.index()) + ); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + assert_eq!(symbol.address(), 0); + + let symbol = symbols.next(); + assert!(symbol.is_none(), "unexpected symbol {:?}", symbol); + + let mut comdats = object.comdats(); + + let comdat = comdats.next().unwrap(); + println!("{:?}", comdat); + assert_eq!(comdat.kind(), ComdatKind::NoDuplicates); + assert_eq!(comdat.symbol(), symbol_index); + + let mut comdat_sections = comdat.sections(); + assert_eq!(comdat_sections.next(), Some(section1_index)); + assert_eq!(comdat_sections.next(), Some(section2_index)); + assert_eq!(comdat_sections.next(), None); +} + +#[test] +fn elf_x86_64_comdat() { + let mut object = + write::Object::new(BinaryFormat::Elf, Architecture::X86_64, Endianness::Little); + + let (section1, offset) = + object.add_subsection(write::StandardSection::Text, b"s1", &[0, 1, 2, 3], 4); + let (section2, _) = + object.add_subsection(write::StandardSection::Data, b"s1", &[0, 1, 2, 3], 4); + + let symbol = object.add_symbol(write::Symbol { + name: b"s1".to_vec(), + value: offset, + size: 4, + kind: SymbolKind::Data, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Section(section1), + flags: SymbolFlags::None, + }); + + object.add_comdat(write::Comdat { + kind: ComdatKind::Any, + symbol, + sections: vec![section1, section2], + }); + + let bytes = object.write().unwrap(); + + //std::fs::write(&"comdat.o", &bytes).unwrap(); + + let object = read::File::parse(&*bytes).unwrap(); + assert_eq!(object.format(), BinaryFormat::Elf); + assert_eq!(object.architecture(), Architecture::X86_64); + + let mut sections = object.sections(); + + let section = sections.next().unwrap(); + println!("{:?}", section); + assert_eq!(section.name(), Ok("")); + + let section = sections.next().unwrap(); + println!("{:?}", section); + assert_eq!(section.name(), Ok(".group")); + + let section1 = sections.next().unwrap(); + println!("{:?}", section1); + let section1_index = section1.index(); + assert_eq!(section1.name(), Ok(".text.s1")); + assert_eq!(section1.kind(), SectionKind::Text); + assert_eq!(section1.address(), 0); + assert_eq!(section1.size(), 4); + + let section2 = sections.next().unwrap(); + println!("{:?}", section2); + let section2_index = section2.index(); + assert_eq!(section2.name(), Ok(".data.s1")); + assert_eq!(section2.kind(), SectionKind::Data); + assert_eq!(section2.address(), 0); + assert_eq!(section2.size(), 4); + + let mut symbols = object.symbols(); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("")); + + let symbol = symbols.next().unwrap(); + let symbol_index = symbol.index(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("s1")); + assert_eq!(symbol.kind(), SymbolKind::Data); + assert_eq!( + symbol.section(), + read::SymbolSection::Section(section1.index()) + ); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + assert_eq!(symbol.address(), 0); + + let symbol = symbols.next(); + assert!(symbol.is_none(), "unexpected symbol {:?}", symbol); + + let mut comdats = object.comdats(); + + let comdat = comdats.next().unwrap(); + println!("{:?}", comdat); + assert_eq!(comdat.kind(), ComdatKind::Any); + assert_eq!(comdat.symbol(), symbol_index); + + let mut comdat_sections = comdat.sections(); + assert_eq!(comdat_sections.next(), Some(section1_index)); + assert_eq!(comdat_sections.next(), Some(section2_index)); + assert_eq!(comdat_sections.next(), None); +} diff --git a/crux-mir/lib/object/tests/round_trip/common.rs b/crux-mir/lib/object/tests/round_trip/common.rs new file mode 100644 index 000000000..74d443830 --- /dev/null +++ b/crux-mir/lib/object/tests/round_trip/common.rs @@ -0,0 +1,245 @@ +#![cfg(all(feature = "read", feature = "write"))] + +use object::read::{Object, ObjectSection, ObjectSymbol}; +use object::{read, write}; +use object::{ + Architecture, BinaryFormat, Endianness, SectionKind, SymbolFlags, SymbolKind, SymbolScope, +}; + +#[test] +fn coff_x86_64_common() { + let mut object = + write::Object::new(BinaryFormat::Coff, Architecture::X86_64, Endianness::Little); + + let symbol = write::Symbol { + name: b"v1".to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Data, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Undefined, + flags: SymbolFlags::None, + }; + object.add_common_symbol(symbol, 4, 4); + + let symbol = write::Symbol { + name: b"v2".to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Data, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Undefined, + flags: SymbolFlags::None, + }; + object.add_common_symbol(symbol, 8, 8); + + // Also check undefined symbols, which are very similar. + let symbol = write::Symbol { + name: b"v3".to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Data, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Undefined, + flags: SymbolFlags::None, + }; + object.add_symbol(symbol); + + let bytes = object.write().unwrap(); + + //std::fs::write(&"common.o", &bytes).unwrap(); + + let object = read::File::parse(&*bytes).unwrap(); + assert_eq!(object.format(), BinaryFormat::Coff); + assert_eq!(object.architecture(), Architecture::X86_64); + + let mut symbols = object.symbols(); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("v1")); + assert_eq!(symbol.kind(), SymbolKind::Data); + assert_eq!(symbol.section(), read::SymbolSection::Common); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + assert_eq!(symbol.address(), 0); + assert_eq!(symbol.size(), 4); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("v2")); + assert_eq!(symbol.kind(), SymbolKind::Data); + assert_eq!(symbol.section(), read::SymbolSection::Common); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + assert_eq!(symbol.address(), 0); + assert_eq!(symbol.size(), 8); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("v3")); + assert_eq!(symbol.kind(), SymbolKind::Data); + assert_eq!(symbol.section(), read::SymbolSection::Undefined); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), true); + assert_eq!(symbol.address(), 0); + assert_eq!(symbol.size(), 0); + + let symbol = symbols.next(); + assert!(symbol.is_none(), "unexpected symbol {:?}", symbol); +} + +#[test] +fn elf_x86_64_common() { + let mut object = + write::Object::new(BinaryFormat::Elf, Architecture::X86_64, Endianness::Little); + + let symbol = write::Symbol { + name: b"v1".to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Data, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Undefined, + flags: SymbolFlags::None, + }; + object.add_common_symbol(symbol, 4, 4); + + let symbol = write::Symbol { + name: b"v2".to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Data, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Undefined, + flags: SymbolFlags::None, + }; + object.add_common_symbol(symbol, 8, 8); + + let bytes = object.write().unwrap(); + + //std::fs::write(&"common.o", &bytes).unwrap(); + + let object = read::File::parse(&*bytes).unwrap(); + assert_eq!(object.format(), BinaryFormat::Elf); + assert_eq!(object.architecture(), Architecture::X86_64); + + let mut symbols = object.symbols(); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("")); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("v1")); + assert_eq!(symbol.kind(), SymbolKind::Data); + assert_eq!(symbol.section(), read::SymbolSection::Common); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + assert_eq!(symbol.address(), 0); + assert_eq!(symbol.size(), 4); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("v2")); + assert_eq!(symbol.kind(), SymbolKind::Data); + assert_eq!(symbol.section(), read::SymbolSection::Common); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + assert_eq!(symbol.address(), 0); + assert_eq!(symbol.size(), 8); + + let symbol = symbols.next(); + assert!(symbol.is_none(), "unexpected symbol {:?}", symbol); +} + +#[test] +fn macho_x86_64_common() { + let mut object = write::Object::new( + BinaryFormat::MachO, + Architecture::X86_64, + Endianness::Little, + ); + + let symbol = write::Symbol { + name: b"v1".to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Data, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Undefined, + flags: SymbolFlags::None, + }; + object.add_common_symbol(symbol, 4, 4); + + let symbol = write::Symbol { + name: b"v2".to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Data, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Undefined, + flags: SymbolFlags::None, + }; + object.add_common_symbol(symbol, 8, 8); + + let bytes = object.write().unwrap(); + + //std::fs::write(&"common.o", &bytes).unwrap(); + + let object = read::File::parse(&*bytes).unwrap(); + assert_eq!(object.format(), BinaryFormat::MachO); + assert_eq!(object.architecture(), Architecture::X86_64); + + let mut sections = object.sections(); + + let common = sections.next().unwrap(); + println!("{:?}", common); + let common_index = common.index(); + assert_eq!(common.name(), Ok("__common")); + assert_eq!(common.segment_name(), Ok(Some("__DATA"))); + assert_eq!(common.kind(), SectionKind::Common); + assert_eq!(common.size(), 16); + assert_eq!(common.data(), Ok(&[][..])); + + let section = sections.next(); + assert!(section.is_none(), "unexpected section {:?}", section); + + let mut symbols = object.symbols(); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("_v1")); + assert_eq!(symbol.kind(), SymbolKind::Data); + assert_eq!(symbol.section_index(), Some(common_index)); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + assert_eq!(symbol.address(), 0); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("_v2")); + assert_eq!(symbol.kind(), SymbolKind::Data); + assert_eq!(symbol.section_index(), Some(common_index)); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + assert_eq!(symbol.address(), 8); + + let symbol = symbols.next(); + assert!(symbol.is_none(), "unexpected symbol {:?}", symbol); +} diff --git a/crux-mir/lib/object/tests/round_trip/elf.rs b/crux-mir/lib/object/tests/round_trip/elf.rs new file mode 100644 index 000000000..048db5795 --- /dev/null +++ b/crux-mir/lib/object/tests/round_trip/elf.rs @@ -0,0 +1,218 @@ +use object::read::elf::{FileHeader, SectionHeader}; +use object::read::{Object, ObjectSymbol}; +use object::{ + elf, read, write, Architecture, BinaryFormat, Endianness, LittleEndian, SectionIndex, + SectionKind, SymbolFlags, SymbolKind, SymbolScope, SymbolSection, U32, +}; +use std::io::Write; + +#[test] +fn symtab_shndx() { + let mut object = + write::Object::new(BinaryFormat::Elf, Architecture::X86_64, Endianness::Little); + + for i in 0..0x10000 { + let name = format!("func{}", i).into_bytes(); + let (section, offset) = + object.add_subsection(write::StandardSection::Text, &name, &[0xcc], 1); + object.add_symbol(write::Symbol { + name, + value: offset, + size: 1, + kind: SymbolKind::Text, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Section(section), + flags: SymbolFlags::None, + }); + } + let bytes = object.write().unwrap(); + + //std::fs::write(&"symtab_shndx.o", &bytes).unwrap(); + + let object = read::File::parse(&*bytes).unwrap(); + assert_eq!(object.format(), BinaryFormat::Elf); + assert_eq!(object.architecture(), Architecture::X86_64); + + for symbol in object.symbols().skip(1) { + assert_eq!( + symbol.section(), + SymbolSection::Section(SectionIndex(symbol.index().0)) + ); + } +} + +#[cfg(feature = "compression")] +#[test] +fn compression_zlib() { + use object::read::ObjectSection; + use object::LittleEndian as LE; + + let data = b"test data data data"; + let len = data.len() as u64; + + let mut ch = object::elf::CompressionHeader64::::default(); + ch.ch_type.set(LE, object::elf::ELFCOMPRESS_ZLIB); + ch.ch_size.set(LE, len); + ch.ch_addralign.set(LE, 1); + + let mut buf = Vec::new(); + buf.write(object::bytes_of(&ch)).unwrap(); + let mut encoder = flate2::write::ZlibEncoder::new(buf, flate2::Compression::default()); + encoder.write_all(data).unwrap(); + let compressed = encoder.finish().unwrap(); + + let mut object = + write::Object::new(BinaryFormat::Elf, Architecture::X86_64, Endianness::Little); + let section = object.add_section( + Vec::new(), + b".debug_info".to_vec(), + object::SectionKind::Other, + ); + object.section_mut(section).set_data(compressed, 1); + object.section_mut(section).flags = object::SectionFlags::Elf { + sh_flags: object::elf::SHF_COMPRESSED.into(), + }; + let bytes = object.write().unwrap(); + + //std::fs::write(&"compression.o", &bytes).unwrap(); + + let object = read::File::parse(&*bytes).unwrap(); + assert_eq!(object.format(), BinaryFormat::Elf); + assert_eq!(object.architecture(), Architecture::X86_64); + + let section = object.section_by_name(".debug_info").unwrap(); + let uncompressed = section.uncompressed_data().unwrap(); + assert_eq!(data, &*uncompressed); +} + +#[cfg(feature = "compression")] +#[test] +fn compression_gnu() { + use object::read::ObjectSection; + use std::io::Write; + + let data = b"test data data data"; + let len = data.len() as u32; + + let mut buf = Vec::new(); + buf.write_all(b"ZLIB\0\0\0\0").unwrap(); + buf.write_all(&len.to_be_bytes()).unwrap(); + let mut encoder = flate2::write::ZlibEncoder::new(buf, flate2::Compression::default()); + encoder.write_all(data).unwrap(); + let compressed = encoder.finish().unwrap(); + + let mut object = + write::Object::new(BinaryFormat::Elf, Architecture::X86_64, Endianness::Little); + let section = object.add_section( + Vec::new(), + b".zdebug_info".to_vec(), + object::SectionKind::Other, + ); + object.section_mut(section).set_data(compressed, 1); + let bytes = object.write().unwrap(); + + //std::fs::write(&"compression.o", &bytes).unwrap(); + + let object = read::File::parse(&*bytes).unwrap(); + assert_eq!(object.format(), BinaryFormat::Elf); + assert_eq!(object.architecture(), Architecture::X86_64); + + let section = object.section_by_name(".zdebug_info").unwrap(); + let uncompressed = section.uncompressed_data().unwrap(); + assert_eq!(data, &*uncompressed); +} + +#[test] +fn note() { + let endian = Endianness::Little; + let mut object = write::Object::new(BinaryFormat::Elf, Architecture::X86_64, endian); + + // Add note section with align = 4. + let mut buffer = Vec::new(); + + buffer + .write(object::bytes_of(&elf::NoteHeader32 { + n_namesz: U32::new(endian, 6), + n_descsz: U32::new(endian, 11), + n_type: U32::new(endian, 1), + })) + .unwrap(); + buffer.write(b"name1\0\0\0").unwrap(); + buffer.write(b"descriptor\0\0").unwrap(); + + buffer + .write(object::bytes_of(&elf::NoteHeader32 { + n_namesz: U32::new(endian, 6), + n_descsz: U32::new(endian, 11), + n_type: U32::new(endian, 2), + })) + .unwrap(); + buffer.write(b"name2\0\0\0").unwrap(); + buffer.write(b"descriptor\0\0").unwrap(); + + let section = object.add_section(Vec::new(), b".note4".to_vec(), SectionKind::Note); + object.section_mut(section).set_data(buffer, 4); + + // Add note section with align = 8. + let mut buffer = Vec::new(); + + buffer + .write(object::bytes_of(&elf::NoteHeader32 { + n_namesz: U32::new(endian, 6), + n_descsz: U32::new(endian, 11), + n_type: U32::new(endian, 1), + })) + .unwrap(); + buffer.write(b"name1\0\0\0\0\0\0\0").unwrap(); + buffer.write(b"descriptor\0\0\0\0\0\0").unwrap(); + + buffer + .write(object::bytes_of(&elf::NoteHeader32 { + n_namesz: U32::new(endian, 4), + n_descsz: U32::new(endian, 11), + n_type: U32::new(endian, 2), + })) + .unwrap(); + buffer.write(b"abc\0").unwrap(); + buffer.write(b"descriptor\0\0\0\0\0\0").unwrap(); + + let section = object.add_section(Vec::new(), b".note8".to_vec(), SectionKind::Note); + object.section_mut(section).set_data(buffer, 8); + + let bytes = &*object.write().unwrap(); + + //std::fs::write(&"note.o", &bytes).unwrap(); + + let header = elf::FileHeader64::parse(bytes).unwrap(); + let endian: LittleEndian = header.endian().unwrap(); + let sections = header.sections(endian, bytes).unwrap(); + + let section = sections.section(SectionIndex(1)).unwrap(); + assert_eq!(sections.section_name(endian, section).unwrap(), b".note4"); + assert_eq!(section.sh_addralign(endian), 4); + let mut notes = section.notes(endian, bytes).unwrap().unwrap(); + let note = notes.next().unwrap().unwrap(); + assert_eq!(note.name(), b"name1"); + assert_eq!(note.desc(), b"descriptor\0"); + assert_eq!(note.n_type(endian), 1); + let note = notes.next().unwrap().unwrap(); + assert_eq!(note.name(), b"name2"); + assert_eq!(note.desc(), b"descriptor\0"); + assert_eq!(note.n_type(endian), 2); + assert!(notes.next().unwrap().is_none()); + + let section = sections.section(SectionIndex(2)).unwrap(); + assert_eq!(sections.section_name(endian, section).unwrap(), b".note8"); + assert_eq!(section.sh_addralign(endian), 8); + let mut notes = section.notes(endian, bytes).unwrap().unwrap(); + let note = notes.next().unwrap().unwrap(); + assert_eq!(note.name(), b"name1"); + assert_eq!(note.desc(), b"descriptor\0"); + assert_eq!(note.n_type(endian), 1); + let note = notes.next().unwrap().unwrap(); + assert_eq!(note.name(), b"abc"); + assert_eq!(note.desc(), b"descriptor\0"); + assert_eq!(note.n_type(endian), 2); + assert!(notes.next().unwrap().is_none()); +} diff --git a/crux-mir/lib/object/tests/round_trip/macho.rs b/crux-mir/lib/object/tests/round_trip/macho.rs new file mode 100644 index 000000000..ca3ad5ca2 --- /dev/null +++ b/crux-mir/lib/object/tests/round_trip/macho.rs @@ -0,0 +1,24 @@ +use object::read::macho::MachHeader; +use object::{macho, write, Architecture, BinaryFormat, Endianness}; + +#[test] +// Test that segment size is valid when the first section needs alignment. +fn issue_286_segment_file_size() { + let mut object = write::Object::new( + BinaryFormat::MachO, + Architecture::X86_64, + Endianness::Little, + ); + + let text = object.section_id(write::StandardSection::Text); + object.append_section_data(text, &[1; 30], 0x1000); + + let bytes = &*object.write().unwrap(); + let header = macho::MachHeader64::parse(bytes, 0).unwrap(); + let endian: Endianness = header.endian().unwrap(); + let mut commands = header.load_commands(endian, bytes, 0).unwrap(); + let command = commands.next().unwrap().unwrap(); + let (segment, _) = command.segment_64().unwrap().unwrap(); + assert_eq!(segment.vmsize.get(endian), 30); + assert_eq!(segment.filesize.get(endian), 30); +} diff --git a/crux-mir/lib/object/tests/round_trip/mod.rs b/crux-mir/lib/object/tests/round_trip/mod.rs new file mode 100644 index 000000000..120092ee9 --- /dev/null +++ b/crux-mir/lib/object/tests/round_trip/mod.rs @@ -0,0 +1,448 @@ +#![cfg(all(feature = "read", feature = "write"))] + +use object::read::{Object, ObjectSection, ObjectSymbol}; +use object::{read, write}; +use object::{ + Architecture, BinaryFormat, Endianness, RelocationEncoding, RelocationKind, SectionKind, + SymbolFlags, SymbolKind, SymbolScope, SymbolSection, +}; + +mod bss; +mod coff; +mod comdat; +mod common; +mod elf; +mod macho; +mod section_flags; +mod tls; + +#[test] +fn coff_x86_64() { + let mut object = + write::Object::new(BinaryFormat::Coff, Architecture::X86_64, Endianness::Little); + + object.add_file_symbol(b"file.c".to_vec()); + + let text = object.section_id(write::StandardSection::Text); + object.append_section_data(text, &[1; 30], 4); + + let func1_offset = object.append_section_data(text, &[1; 30], 4); + assert_eq!(func1_offset, 32); + let func1_symbol = object.add_symbol(write::Symbol { + name: b"func1".to_vec(), + value: func1_offset, + size: 32, + kind: SymbolKind::Text, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Section(text), + flags: SymbolFlags::None, + }); + object + .add_relocation( + text, + write::Relocation { + offset: 8, + size: 64, + kind: RelocationKind::Absolute, + encoding: RelocationEncoding::Generic, + symbol: func1_symbol, + addend: 0, + }, + ) + .unwrap(); + + let bytes = object.write().unwrap(); + let object = read::File::parse(&*bytes).unwrap(); + assert_eq!(object.format(), BinaryFormat::Coff); + assert_eq!(object.architecture(), Architecture::X86_64); + assert_eq!(object.endianness(), Endianness::Little); + + let mut sections = object.sections(); + + let text = sections.next().unwrap(); + println!("{:?}", text); + let text_index = text.index(); + assert_eq!(text.name(), Ok(".text")); + assert_eq!(text.kind(), SectionKind::Text); + assert_eq!(text.address(), 0); + assert_eq!(text.size(), 62); + assert_eq!(&text.data().unwrap()[..30], &[1; 30]); + assert_eq!(&text.data().unwrap()[32..62], &[1; 30]); + + let mut symbols = object.symbols(); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("file.c")); + assert_eq!(symbol.address(), 0); + assert_eq!(symbol.kind(), SymbolKind::File); + assert_eq!(symbol.section(), SymbolSection::None); + assert_eq!(symbol.scope(), SymbolScope::Compilation); + assert_eq!(symbol.is_weak(), false); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + let func1_symbol = symbol.index(); + assert_eq!(symbol.name(), Ok("func1")); + assert_eq!(symbol.address(), func1_offset); + assert_eq!(symbol.kind(), SymbolKind::Text); + assert_eq!(symbol.section_index(), Some(text_index)); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + + let mut relocations = text.relocations(); + + let (offset, relocation) = relocations.next().unwrap(); + println!("{:?}", relocation); + assert_eq!(offset, 8); + assert_eq!(relocation.kind(), RelocationKind::Absolute); + assert_eq!(relocation.encoding(), RelocationEncoding::Generic); + assert_eq!(relocation.size(), 64); + assert_eq!( + relocation.target(), + read::RelocationTarget::Symbol(func1_symbol) + ); + assert_eq!(relocation.addend(), 0); + + let map = object.symbol_map(); + let symbol = map.get(func1_offset + 1).unwrap(); + assert_eq!(symbol.address(), func1_offset); + assert_eq!(symbol.name(), "func1"); + assert_eq!(map.get(func1_offset - 1), None); +} + +#[test] +fn elf_x86_64() { + let mut object = + write::Object::new(BinaryFormat::Elf, Architecture::X86_64, Endianness::Little); + + object.add_file_symbol(b"file.c".to_vec()); + + let text = object.section_id(write::StandardSection::Text); + object.append_section_data(text, &[1; 30], 4); + + let func1_offset = object.append_section_data(text, &[1; 30], 4); + assert_eq!(func1_offset, 32); + let func1_symbol = object.add_symbol(write::Symbol { + name: b"func1".to_vec(), + value: func1_offset, + size: 32, + kind: SymbolKind::Text, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Section(text), + flags: SymbolFlags::None, + }); + object + .add_relocation( + text, + write::Relocation { + offset: 8, + size: 64, + kind: RelocationKind::Absolute, + encoding: RelocationEncoding::Generic, + symbol: func1_symbol, + addend: 0, + }, + ) + .unwrap(); + + let bytes = object.write().unwrap(); + let object = read::File::parse(&*bytes).unwrap(); + assert_eq!(object.format(), BinaryFormat::Elf); + assert_eq!(object.architecture(), Architecture::X86_64); + assert_eq!(object.endianness(), Endianness::Little); + + let mut sections = object.sections(); + + let section = sections.next().unwrap(); + println!("{:?}", section); + assert_eq!(section.name(), Ok("")); + assert_eq!(section.kind(), SectionKind::Metadata); + assert_eq!(section.address(), 0); + assert_eq!(section.size(), 0); + + let text = sections.next().unwrap(); + println!("{:?}", text); + let text_index = text.index(); + assert_eq!(text.name(), Ok(".text")); + assert_eq!(text.kind(), SectionKind::Text); + assert_eq!(text.address(), 0); + assert_eq!(text.size(), 62); + assert_eq!(&text.data().unwrap()[..30], &[1; 30]); + assert_eq!(&text.data().unwrap()[32..62], &[1; 30]); + + let mut symbols = object.symbols(); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("")); + assert_eq!(symbol.address(), 0); + assert_eq!(symbol.kind(), SymbolKind::Null); + assert_eq!(symbol.section_index(), None); + assert_eq!(symbol.scope(), SymbolScope::Unknown); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), true); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("file.c")); + assert_eq!(symbol.address(), 0); + assert_eq!(symbol.kind(), SymbolKind::File); + assert_eq!(symbol.section(), SymbolSection::None); + assert_eq!(symbol.scope(), SymbolScope::Compilation); + assert_eq!(symbol.is_weak(), false); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + let func1_symbol = symbol.index(); + assert_eq!(symbol.name(), Ok("func1")); + assert_eq!(symbol.address(), func1_offset); + assert_eq!(symbol.kind(), SymbolKind::Text); + assert_eq!(symbol.section_index(), Some(text_index)); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + + let mut relocations = text.relocations(); + + let (offset, relocation) = relocations.next().unwrap(); + println!("{:?}", relocation); + assert_eq!(offset, 8); + assert_eq!(relocation.kind(), RelocationKind::Absolute); + assert_eq!(relocation.encoding(), RelocationEncoding::Generic); + assert_eq!(relocation.size(), 64); + assert_eq!( + relocation.target(), + read::RelocationTarget::Symbol(func1_symbol) + ); + assert_eq!(relocation.addend(), 0); + + let map = object.symbol_map(); + let symbol = map.get(func1_offset + 1).unwrap(); + assert_eq!(symbol.address(), func1_offset); + assert_eq!(symbol.name(), "func1"); + assert_eq!(map.get(func1_offset - 1), None); +} + +#[test] +fn elf_any() { + for (arch, endian) in [ + (Architecture::Aarch64, Endianness::Little), + (Architecture::Arm, Endianness::Little), + (Architecture::Avr, Endianness::Little), + (Architecture::Bpf, Endianness::Little), + (Architecture::I386, Endianness::Little), + (Architecture::X86_64, Endianness::Little), + (Architecture::X86_64_X32, Endianness::Little), + (Architecture::Hexagon, Endianness::Little), + (Architecture::LoongArch64, Endianness::Little), + (Architecture::Mips, Endianness::Little), + (Architecture::Mips64, Endianness::Little), + (Architecture::Msp430, Endianness::Little), + (Architecture::PowerPc, Endianness::Big), + (Architecture::PowerPc64, Endianness::Big), + (Architecture::Riscv32, Endianness::Little), + (Architecture::Riscv64, Endianness::Little), + (Architecture::S390x, Endianness::Big), + (Architecture::Sparc64, Endianness::Big), + ] + .iter() + .copied() + { + let mut object = write::Object::new(BinaryFormat::Elf, arch, endian); + + let section = object.section_id(write::StandardSection::Data); + object.append_section_data(section, &[1; 30], 4); + let symbol = object.section_symbol(section); + + object + .add_relocation( + section, + write::Relocation { + offset: 8, + size: 32, + kind: RelocationKind::Absolute, + encoding: RelocationEncoding::Generic, + symbol, + addend: 0, + }, + ) + .unwrap(); + if arch.address_size().unwrap().bytes() >= 8 { + object + .add_relocation( + section, + write::Relocation { + offset: 16, + size: 64, + kind: RelocationKind::Absolute, + encoding: RelocationEncoding::Generic, + symbol, + addend: 0, + }, + ) + .unwrap(); + } + + let bytes = object.write().unwrap(); + let object = read::File::parse(&*bytes).unwrap(); + println!("{:?}", object.architecture()); + assert_eq!(object.format(), BinaryFormat::Elf); + assert_eq!(object.architecture(), arch); + assert_eq!(object.endianness(), endian); + + let mut sections = object.sections(); + + let section = sections.next().unwrap(); + println!("{:?}", section); + assert_eq!(section.name(), Ok("")); + assert_eq!(section.kind(), SectionKind::Metadata); + assert_eq!(section.address(), 0); + assert_eq!(section.size(), 0); + + let data = sections.next().unwrap(); + println!("{:?}", data); + assert_eq!(data.name(), Ok(".data")); + assert_eq!(data.kind(), SectionKind::Data); + + let mut relocations = data.relocations(); + + let (offset, relocation) = relocations.next().unwrap(); + println!("{:?}", relocation); + assert_eq!(offset, 8); + assert_eq!(relocation.kind(), RelocationKind::Absolute); + assert_eq!(relocation.encoding(), RelocationEncoding::Generic); + assert_eq!(relocation.size(), 32); + assert_eq!(relocation.addend(), 0); + + if arch.address_size().unwrap().bytes() >= 8 { + let (offset, relocation) = relocations.next().unwrap(); + println!("{:?}", relocation); + assert_eq!(offset, 16); + assert_eq!(relocation.kind(), RelocationKind::Absolute); + assert_eq!(relocation.encoding(), RelocationEncoding::Generic); + assert_eq!(relocation.size(), 64); + assert_eq!(relocation.addend(), 0); + } + } +} + +#[test] +fn macho_x86_64() { + let mut object = write::Object::new( + BinaryFormat::MachO, + Architecture::X86_64, + Endianness::Little, + ); + + object.add_file_symbol(b"file.c".to_vec()); + + let text = object.section_id(write::StandardSection::Text); + object.append_section_data(text, &[1; 30], 4); + + let func1_offset = object.append_section_data(text, &[1; 30], 4); + assert_eq!(func1_offset, 32); + let func1_symbol = object.add_symbol(write::Symbol { + name: b"func1".to_vec(), + value: func1_offset, + size: 32, + kind: SymbolKind::Text, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Section(text), + flags: SymbolFlags::None, + }); + object + .add_relocation( + text, + write::Relocation { + offset: 8, + size: 64, + kind: RelocationKind::Absolute, + encoding: RelocationEncoding::Generic, + symbol: func1_symbol, + addend: 0, + }, + ) + .unwrap(); + object + .add_relocation( + text, + write::Relocation { + offset: 16, + size: 32, + kind: RelocationKind::Relative, + encoding: RelocationEncoding::Generic, + symbol: func1_symbol, + addend: -4, + }, + ) + .unwrap(); + + let bytes = object.write().unwrap(); + let object = read::File::parse(&*bytes).unwrap(); + assert_eq!(object.format(), BinaryFormat::MachO); + assert_eq!(object.architecture(), Architecture::X86_64); + assert_eq!(object.endianness(), Endianness::Little); + + let mut sections = object.sections(); + + let text = sections.next().unwrap(); + println!("{:?}", text); + let text_index = text.index(); + assert_eq!(text.name(), Ok("__text")); + assert_eq!(text.segment_name(), Ok(Some("__TEXT"))); + assert_eq!(text.kind(), SectionKind::Text); + assert_eq!(text.address(), 0); + assert_eq!(text.size(), 62); + assert_eq!(&text.data().unwrap()[..30], &[1; 30]); + assert_eq!(&text.data().unwrap()[32..62], &[1; 30]); + + let mut symbols = object.symbols(); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + let func1_symbol = symbol.index(); + assert_eq!(symbol.name(), Ok("_func1")); + assert_eq!(symbol.address(), func1_offset); + assert_eq!(symbol.kind(), SymbolKind::Text); + assert_eq!(symbol.section_index(), Some(text_index)); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + + let mut relocations = text.relocations(); + + let (offset, relocation) = relocations.next().unwrap(); + println!("{:?}", relocation); + assert_eq!(offset, 8); + assert_eq!(relocation.kind(), RelocationKind::Absolute); + assert_eq!(relocation.encoding(), RelocationEncoding::Generic); + assert_eq!(relocation.size(), 64); + assert_eq!( + relocation.target(), + read::RelocationTarget::Symbol(func1_symbol) + ); + assert_eq!(relocation.addend(), 0); + + let (offset, relocation) = relocations.next().unwrap(); + println!("{:?}", relocation); + assert_eq!(offset, 16); + assert_eq!(relocation.kind(), RelocationKind::Relative); + assert_eq!(relocation.encoding(), RelocationEncoding::X86RipRelative); + assert_eq!(relocation.size(), 32); + assert_eq!( + relocation.target(), + read::RelocationTarget::Symbol(func1_symbol) + ); + assert_eq!(relocation.addend(), -4); + + let map = object.symbol_map(); + let symbol = map.get(func1_offset + 1).unwrap(); + assert_eq!(symbol.address(), func1_offset); + assert_eq!(symbol.name(), "_func1"); + assert_eq!(map.get(func1_offset - 1), None); +} diff --git a/crux-mir/lib/object/tests/round_trip/section_flags.rs b/crux-mir/lib/object/tests/round_trip/section_flags.rs new file mode 100644 index 000000000..b1ca398b4 --- /dev/null +++ b/crux-mir/lib/object/tests/round_trip/section_flags.rs @@ -0,0 +1,90 @@ +#![cfg(all(feature = "read", feature = "write"))] + +use object::read::{Object, ObjectSection}; +use object::{read, write}; +use object::{Architecture, BinaryFormat, Endianness, SectionFlags, SectionKind}; + +#[test] +fn coff_x86_64_section_flags() { + let mut object = + write::Object::new(BinaryFormat::Coff, Architecture::X86_64, Endianness::Little); + + let section = object.add_section(Vec::new(), b".text".to_vec(), SectionKind::Text); + object.section_mut(section).flags = SectionFlags::Coff { + characteristics: object::pe::IMAGE_SCN_MEM_WRITE, + }; + + let bytes = object.write().unwrap(); + + let object = read::File::parse(&*bytes).unwrap(); + assert_eq!(object.format(), BinaryFormat::Coff); + assert_eq!(object.architecture(), Architecture::X86_64); + + let mut sections = object.sections(); + let section = sections.next().unwrap(); + assert_eq!(section.name(), Ok(".text")); + assert_eq!( + section.flags(), + SectionFlags::Coff { + characteristics: object::pe::IMAGE_SCN_MEM_WRITE | object::pe::IMAGE_SCN_ALIGN_1BYTES, + } + ); +} + +#[test] +fn elf_x86_64_section_flags() { + let mut object = + write::Object::new(BinaryFormat::Elf, Architecture::X86_64, Endianness::Little); + + let section = object.add_section(Vec::new(), b".text".to_vec(), SectionKind::Text); + object.section_mut(section).flags = SectionFlags::Elf { + sh_flags: object::elf::SHF_WRITE.into(), + }; + + let bytes = object.write().unwrap(); + + let object = read::File::parse(&*bytes).unwrap(); + assert_eq!(object.format(), BinaryFormat::Elf); + assert_eq!(object.architecture(), Architecture::X86_64); + + let mut sections = object.sections(); + sections.next().unwrap(); + let section = sections.next().unwrap(); + assert_eq!(section.name(), Ok(".text")); + assert_eq!( + section.flags(), + SectionFlags::Elf { + sh_flags: object::elf::SHF_WRITE.into(), + } + ); +} + +#[test] +fn macho_x86_64_section_flags() { + let mut object = write::Object::new( + BinaryFormat::MachO, + Architecture::X86_64, + Endianness::Little, + ); + + let section = object.add_section(Vec::new(), b".text".to_vec(), SectionKind::Text); + object.section_mut(section).flags = SectionFlags::MachO { + flags: object::macho::S_ATTR_SELF_MODIFYING_CODE, + }; + + let bytes = object.write().unwrap(); + + let object = read::File::parse(&*bytes).unwrap(); + assert_eq!(object.format(), BinaryFormat::MachO); + assert_eq!(object.architecture(), Architecture::X86_64); + + let mut sections = object.sections(); + let section = sections.next().unwrap(); + assert_eq!(section.name(), Ok(".text")); + assert_eq!( + section.flags(), + SectionFlags::MachO { + flags: object::macho::S_ATTR_SELF_MODIFYING_CODE, + } + ); +} diff --git a/crux-mir/lib/object/tests/round_trip/tls.rs b/crux-mir/lib/object/tests/round_trip/tls.rs new file mode 100644 index 000000000..999e2f181 --- /dev/null +++ b/crux-mir/lib/object/tests/round_trip/tls.rs @@ -0,0 +1,316 @@ +#![cfg(all(feature = "read", feature = "write"))] + +use object::read::{Object, ObjectSection, ObjectSymbol}; +use object::{read, write}; +use object::{ + Architecture, BinaryFormat, Endianness, RelocationEncoding, RelocationKind, SectionKind, + SymbolFlags, SymbolKind, SymbolScope, +}; + +#[test] +fn coff_x86_64_tls() { + let mut object = + write::Object::new(BinaryFormat::Coff, Architecture::X86_64, Endianness::Little); + + let section = object.section_id(write::StandardSection::Tls); + let symbol = object.add_symbol(write::Symbol { + name: b"tls1".to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Tls, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Undefined, + flags: SymbolFlags::None, + }); + object.add_symbol_data(symbol, section, &[1; 30], 4); + + let bytes = object.write().unwrap(); + + //std::fs::write(&"tls.o", &bytes).unwrap(); + + let object = read::File::parse(&*bytes).unwrap(); + assert_eq!(object.format(), BinaryFormat::Coff); + assert_eq!(object.architecture(), Architecture::X86_64); + + let mut sections = object.sections(); + + let section = sections.next().unwrap(); + println!("{:?}", section); + let tls_index = section.index(); + assert_eq!(section.name(), Ok(".tls$")); + assert_eq!(section.kind(), SectionKind::Data); + assert_eq!(section.size(), 30); + assert_eq!(§ion.data().unwrap()[..], &[1; 30]); + + let mut symbols = object.symbols(); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("tls1")); + assert_eq!(symbol.kind(), SymbolKind::Data); + assert_eq!(symbol.section_index(), Some(tls_index)); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); +} + +#[test] +fn elf_x86_64_tls() { + let mut object = + write::Object::new(BinaryFormat::Elf, Architecture::X86_64, Endianness::Little); + + let section = object.section_id(write::StandardSection::Tls); + let symbol = object.add_symbol(write::Symbol { + name: b"tls1".to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Tls, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Undefined, + flags: SymbolFlags::None, + }); + object.add_symbol_data(symbol, section, &[1; 30], 4); + + let section = object.section_id(write::StandardSection::UninitializedTls); + let symbol = object.add_symbol(write::Symbol { + name: b"tls2".to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Tls, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Undefined, + flags: SymbolFlags::None, + }); + object.add_symbol_bss(symbol, section, 31, 4); + + let bytes = object.write().unwrap(); + + //std::fs::write(&"tls.o", &bytes).unwrap(); + + let object = read::File::parse(&*bytes).unwrap(); + assert_eq!(object.format(), BinaryFormat::Elf); + assert_eq!(object.architecture(), Architecture::X86_64); + + let mut sections = object.sections(); + + let section = sections.next().unwrap(); + println!("{:?}", section); + assert_eq!(section.name(), Ok("")); + + let section = sections.next().unwrap(); + println!("{:?}", section); + let tdata_index = section.index(); + assert_eq!(section.name(), Ok(".tdata")); + assert_eq!(section.kind(), SectionKind::Tls); + assert_eq!(section.size(), 30); + assert_eq!(§ion.data().unwrap()[..], &[1; 30]); + + let section = sections.next().unwrap(); + println!("{:?}", section); + let tbss_index = section.index(); + assert_eq!(section.name(), Ok(".tbss")); + assert_eq!(section.kind(), SectionKind::UninitializedTls); + assert_eq!(section.size(), 31); + assert_eq!(§ion.data().unwrap()[..], &[]); + + let mut symbols = object.symbols(); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("")); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("tls1")); + assert_eq!(symbol.kind(), SymbolKind::Tls); + assert_eq!(symbol.section_index(), Some(tdata_index)); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + assert_eq!(symbol.size(), 30); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("tls2")); + assert_eq!(symbol.kind(), SymbolKind::Tls); + assert_eq!(symbol.section_index(), Some(tbss_index)); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + assert_eq!(symbol.size(), 31); +} + +#[test] +fn macho_x86_64_tls() { + let mut object = write::Object::new( + BinaryFormat::MachO, + Architecture::X86_64, + Endianness::Little, + ); + + let section = object.section_id(write::StandardSection::Tls); + let symbol = object.add_symbol(write::Symbol { + name: b"tls1".to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Tls, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Undefined, + flags: SymbolFlags::None, + }); + object.add_symbol_data(symbol, section, &[1; 30], 4); + + let section = object.section_id(write::StandardSection::UninitializedTls); + let symbol = object.add_symbol(write::Symbol { + name: b"tls2".to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Tls, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Undefined, + flags: SymbolFlags::None, + }); + object.add_symbol_bss(symbol, section, 31, 4); + + let bytes = object.write().unwrap(); + + //std::fs::write(&"tls.o", &bytes).unwrap(); + + let object = read::File::parse(&*bytes).unwrap(); + assert_eq!(object.format(), BinaryFormat::MachO); + assert_eq!(object.architecture(), Architecture::X86_64); + + let mut sections = object.sections(); + + let thread_data = sections.next().unwrap(); + println!("{:?}", thread_data); + let thread_data_index = thread_data.index(); + assert_eq!(thread_data.name(), Ok("__thread_data")); + assert_eq!(thread_data.segment_name(), Ok(Some("__DATA"))); + assert_eq!(thread_data.kind(), SectionKind::Tls); + assert_eq!(thread_data.size(), 30); + assert_eq!(&thread_data.data().unwrap()[..], &[1; 30]); + + let thread_vars = sections.next().unwrap(); + println!("{:?}", thread_vars); + let thread_vars_index = thread_vars.index(); + assert_eq!(thread_vars.name(), Ok("__thread_vars")); + assert_eq!(thread_vars.segment_name(), Ok(Some("__DATA"))); + assert_eq!(thread_vars.kind(), SectionKind::TlsVariables); + assert_eq!(thread_vars.size(), 2 * 3 * 8); + assert_eq!(&thread_vars.data().unwrap()[..], &[0; 48][..]); + + let thread_bss = sections.next().unwrap(); + println!("{:?}", thread_bss); + let thread_bss_index = thread_bss.index(); + assert_eq!(thread_bss.name(), Ok("__thread_bss")); + assert_eq!(thread_bss.segment_name(), Ok(Some("__DATA"))); + assert_eq!(thread_bss.kind(), SectionKind::UninitializedTls); + assert_eq!(thread_bss.size(), 31); + assert_eq!(thread_bss.data(), Ok(&[][..])); + + let mut symbols = object.symbols(); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("_tls1")); + assert_eq!(symbol.kind(), SymbolKind::Tls); + assert_eq!(symbol.section_index(), Some(thread_vars_index)); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + let tls1_init_symbol = symbol.index(); + assert_eq!(symbol.name(), Ok("_tls1$tlv$init")); + assert_eq!(symbol.kind(), SymbolKind::Tls); + assert_eq!(symbol.section_index(), Some(thread_data_index)); + assert_eq!(symbol.scope(), SymbolScope::Compilation); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + let tlv_bootstrap_symbol = symbol.index(); + assert_eq!(symbol.name(), Ok("__tlv_bootstrap")); + assert_eq!(symbol.kind(), SymbolKind::Unknown); + assert_eq!(symbol.section_index(), None); + assert_eq!(symbol.scope(), SymbolScope::Unknown); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), true); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + assert_eq!(symbol.name(), Ok("_tls2")); + assert_eq!(symbol.kind(), SymbolKind::Tls); + assert_eq!(symbol.section_index(), Some(thread_vars_index)); + assert_eq!(symbol.scope(), SymbolScope::Linkage); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + + let symbol = symbols.next().unwrap(); + println!("{:?}", symbol); + let tls2_init_symbol = symbol.index(); + assert_eq!(symbol.name(), Ok("_tls2$tlv$init")); + assert_eq!(symbol.kind(), SymbolKind::Tls); + assert_eq!(symbol.section_index(), Some(thread_bss_index)); + assert_eq!(symbol.scope(), SymbolScope::Compilation); + assert_eq!(symbol.is_weak(), false); + assert_eq!(symbol.is_undefined(), false); + + let mut relocations = thread_vars.relocations(); + + let (offset, relocation) = relocations.next().unwrap(); + println!("{:?}", relocation); + assert_eq!(offset, 0); + assert_eq!(relocation.kind(), RelocationKind::Absolute); + assert_eq!(relocation.encoding(), RelocationEncoding::Generic); + assert_eq!(relocation.size(), 64); + assert_eq!( + relocation.target(), + read::RelocationTarget::Symbol(tlv_bootstrap_symbol) + ); + assert_eq!(relocation.addend(), 0); + + let (offset, relocation) = relocations.next().unwrap(); + println!("{:?}", relocation); + assert_eq!(offset, 16); + assert_eq!(relocation.kind(), RelocationKind::Absolute); + assert_eq!(relocation.encoding(), RelocationEncoding::Generic); + assert_eq!(relocation.size(), 64); + assert_eq!( + relocation.target(), + read::RelocationTarget::Symbol(tls1_init_symbol) + ); + assert_eq!(relocation.addend(), 0); + + let (offset, relocation) = relocations.next().unwrap(); + println!("{:?}", relocation); + assert_eq!(offset, 24); + assert_eq!(relocation.kind(), RelocationKind::Absolute); + assert_eq!(relocation.encoding(), RelocationEncoding::Generic); + assert_eq!(relocation.size(), 64); + assert_eq!( + relocation.target(), + read::RelocationTarget::Symbol(tlv_bootstrap_symbol) + ); + assert_eq!(relocation.addend(), 0); + + let (offset, relocation) = relocations.next().unwrap(); + println!("{:?}", relocation); + assert_eq!(offset, 40); + assert_eq!(relocation.kind(), RelocationKind::Absolute); + assert_eq!(relocation.encoding(), RelocationEncoding::Generic); + assert_eq!(relocation.size(), 64); + assert_eq!( + relocation.target(), + read::RelocationTarget::Symbol(tls2_init_symbol) + ); + assert_eq!(relocation.addend(), 0); +} diff --git a/crux-mir/lib/panic_abort/Cargo.toml b/crux-mir/lib/panic_abort/Cargo.toml index 2bee0b716..e6ea2b184 100644 --- a/crux-mir/lib/panic_abort/Cargo.toml +++ b/crux-mir/lib/panic_abort/Cargo.toml @@ -1,16 +1,19 @@ [package] -authors = ["The Rust Project Developers"] name = "panic_abort" version = "0.0.0" -edition = "2018" +license = "MIT OR Apache-2.0" +repository = "https://github.com/rust-lang/rust.git" +description = "Implementation of Rust panics via process aborts" +edition = "2021" [lib] -path = "lib.rs" test = false bench = false doc = false [dependencies] -core = { path = "../libcore" } +alloc = { path = "../alloc" } +cfg-if = { version = "1.0", features = ['rustc-dep-of-std'] } +core = { path = "../core" } libc = { version = "0.2", default-features = false } compiler_builtins = "0.1.0" diff --git a/crux-mir/lib/panic_abort/lib.rs b/crux-mir/lib/panic_abort/lib.rs deleted file mode 100644 index f44a875c9..000000000 --- a/crux-mir/lib/panic_abort/lib.rs +++ /dev/null @@ -1,126 +0,0 @@ -//! Implementation of Rust panics via process aborts -//! -//! When compared to the implementation via unwinding, this crate is *much* -//! simpler! That being said, it's not quite as versatile, but here goes! - -#![no_std] -#![unstable(feature = "panic_abort", issue = "32837")] -#![doc( - html_root_url = "https://doc.rust-lang.org/nightly/", - issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/" -)] -#![panic_runtime] -#![allow(unused_features)] -#![feature(core_intrinsics)] -#![feature(libc)] -#![feature(nll)] -#![feature(panic_runtime)] -#![feature(staged_api)] -#![feature(rustc_attrs)] - -use core::any::Any; - -#[rustc_std_internal_symbol] -pub unsafe extern "C" fn __rust_panic_cleanup(_: *mut u8) -> *mut (dyn Any + Send + 'static) { - unreachable!() -} - -// "Leak" the payload and shim to the relevant abort on the platform in -// question. -// -// For Unix we just use `abort` from libc as it'll trigger debuggers, core -// dumps, etc, as one might expect. On Windows, however, the best option we have -// is the `__fastfail` intrinsics, but that's unfortunately not defined in LLVM, -// and the `RaiseFailFastException` function isn't available until Windows 7 -// which would break compat with XP. For now just use `intrinsics::abort` which -// will kill us with an illegal instruction, which will do a good enough job for -// now hopefully. -#[rustc_std_internal_symbol] -pub unsafe extern "C" fn __rust_start_panic(_payload: usize) -> u32 { - abort(); - - #[cfg(any(unix, target_os = "cloudabi"))] - unsafe fn abort() -> ! { - libc::abort(); - } - - #[cfg(any(windows, all(target_arch = "wasm32", not(target_os = "emscripten"))))] - unsafe fn abort() -> ! { - core::intrinsics::abort(); - } - - #[cfg(any(target_os = "hermit", all(target_vendor = "fortanix", target_env = "sgx")))] - unsafe fn abort() -> ! { - // call std::sys::abort_internal - extern "C" { - pub fn __rust_abort() -> !; - } - __rust_abort(); - } -} - -// This... is a bit of an oddity. The tl;dr; is that this is required to link -// correctly, the longer explanation is below. -// -// Right now the binaries of libcore/libstd that we ship are all compiled with -// `-C panic=unwind`. This is done to ensure that the binaries are maximally -// compatible with as many situations as possible. The compiler, however, -// requires a "personality function" for all functions compiled with `-C -// panic=unwind`. This personality function is hardcoded to the symbol -// `rust_eh_personality` and is defined by the `eh_personality` lang item. -// -// So... why not just define that lang item here? Good question! The way that -// panic runtimes are linked in is actually a little subtle in that they're -// "sort of" in the compiler's crate store, but only actually linked if another -// isn't actually linked. This ends up meaning that both this crate and the -// panic_unwind crate can appear in the compiler's crate store, and if both -// define the `eh_personality` lang item then that'll hit an error. -// -// To handle this the compiler only requires the `eh_personality` is defined if -// the panic runtime being linked in is the unwinding runtime, and otherwise -// it's not required to be defined (rightfully so). In this case, however, this -// library just defines this symbol so there's at least some personality -// somewhere. -// -// Essentially this symbol is just defined to get wired up to libcore/libstd -// binaries, but it should never be called as we don't link in an unwinding -// runtime at all. -pub mod personalities { - #[rustc_std_internal_symbol] - #[cfg(not(any( - all(target_arch = "wasm32", not(target_os = "emscripten"),), - all(target_os = "windows", target_env = "gnu", target_arch = "x86_64",), - )))] - pub extern "C" fn rust_eh_personality() {} - - // On x86_64-pc-windows-gnu we use our own personality function that needs - // to return `ExceptionContinueSearch` as we're passing on all our frames. - #[rustc_std_internal_symbol] - #[cfg(all(target_os = "windows", target_env = "gnu", target_arch = "x86_64"))] - pub extern "C" fn rust_eh_personality( - _record: usize, - _frame: usize, - _context: usize, - _dispatcher: usize, - ) -> u32 { - 1 // `ExceptionContinueSearch` - } - - // Similar to above, this corresponds to the `eh_unwind_resume` lang item - // that's only used on Windows currently. - // - // Note that we don't execute landing pads, so this is never called, so it's - // body is empty. - #[rustc_std_internal_symbol] - #[cfg(all(bootstrap, target_os = "windows", target_env = "gnu"))] - pub extern "C" fn rust_eh_unwind_resume() {} - - // These two are called by our startup objects on i686-pc-windows-gnu, but - // they don't need to do anything so the bodies are nops. - #[rustc_std_internal_symbol] - #[cfg(all(target_os = "windows", target_env = "gnu", target_arch = "x86"))] - pub extern "C" fn rust_eh_register_frames() {} - #[rustc_std_internal_symbol] - #[cfg(all(target_os = "windows", target_env = "gnu", target_arch = "x86"))] - pub extern "C" fn rust_eh_unregister_frames() {} -} diff --git a/crux-mir/lib/panic_abort/src/android.rs b/crux-mir/lib/panic_abort/src/android.rs new file mode 100644 index 000000000..0fd824f8a --- /dev/null +++ b/crux-mir/lib/panic_abort/src/android.rs @@ -0,0 +1,49 @@ +use alloc::string::String; +use core::mem::transmute; +use core::panic::BoxMeUp; +use core::ptr::copy_nonoverlapping; + +const ANDROID_SET_ABORT_MESSAGE: &[u8] = b"android_set_abort_message\0"; +type SetAbortMessageType = unsafe extern "C" fn(*const libc::c_char) -> (); + +// Forward the abort message to libc's android_set_abort_message. We try our best to populate the +// message but as this function may already be called as part of a failed allocation, it might not be +// possible to do so. +// +// Some methods of core are on purpose avoided (such as try_reserve) as these rely on the correct +// resolution of rust_eh_personality which is loosely defined in panic_abort. +// +// Weakly resolve the symbol for android_set_abort_message. This function is only available +// for API >= 21. +pub(crate) unsafe fn android_set_abort_message(payload: *mut &mut dyn BoxMeUp) { + let func_addr = + libc::dlsym(libc::RTLD_DEFAULT, ANDROID_SET_ABORT_MESSAGE.as_ptr() as *const libc::c_char) + as usize; + if func_addr == 0 { + return; + } + + let payload = (*payload).get(); + let msg = match payload.downcast_ref::<&'static str>() { + Some(msg) => msg.as_bytes(), + None => match payload.downcast_ref::() { + Some(msg) => msg.as_bytes(), + None => &[], + }, + }; + if msg.is_empty() { + return; + } + + // Allocate a new buffer to append the null byte. + let size = msg.len() + 1usize; + let buf = libc::malloc(size) as *mut libc::c_char; + if buf.is_null() { + return; // allocation failure + } + copy_nonoverlapping(msg.as_ptr(), buf as *mut u8, msg.len()); + buf.add(msg.len()).write(0); + + let func = transmute::(func_addr); + func(buf); +} diff --git a/crux-mir/lib/panic_abort/src/lib.rs b/crux-mir/lib/panic_abort/src/lib.rs index f44a875c9..a3cebf99c 100644 --- a/crux-mir/lib/panic_abort/src/lib.rs +++ b/crux-mir/lib/panic_abort/src/lib.rs @@ -5,64 +5,91 @@ #![no_std] #![unstable(feature = "panic_abort", issue = "32837")] -#![doc( - html_root_url = "https://doc.rust-lang.org/nightly/", - issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/" -)] +#![doc(issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/")] #![panic_runtime] #![allow(unused_features)] #![feature(core_intrinsics)] -#![feature(libc)] -#![feature(nll)] #![feature(panic_runtime)] +#![feature(std_internals)] #![feature(staged_api)] #![feature(rustc_attrs)] +#![feature(c_unwind)] + +#[cfg(target_os = "android")] +mod android; use core::any::Any; +use core::panic::BoxMeUp; #[rustc_std_internal_symbol] +#[allow(improper_ctypes_definitions)] pub unsafe extern "C" fn __rust_panic_cleanup(_: *mut u8) -> *mut (dyn Any + Send + 'static) { unreachable!() } -// "Leak" the payload and shim to the relevant abort on the platform in -// question. -// -// For Unix we just use `abort` from libc as it'll trigger debuggers, core -// dumps, etc, as one might expect. On Windows, however, the best option we have -// is the `__fastfail` intrinsics, but that's unfortunately not defined in LLVM, -// and the `RaiseFailFastException` function isn't available until Windows 7 -// which would break compat with XP. For now just use `intrinsics::abort` which -// will kill us with an illegal instruction, which will do a good enough job for -// now hopefully. +// "Leak" the payload and shim to the relevant abort on the platform in question. #[rustc_std_internal_symbol] -pub unsafe extern "C" fn __rust_start_panic(_payload: usize) -> u32 { - abort(); +pub unsafe fn __rust_start_panic(_payload: *mut &mut dyn BoxMeUp) -> u32 { + // Android has the ability to attach a message as part of the abort. + #[cfg(target_os = "android")] + android::android_set_abort_message(_payload); - #[cfg(any(unix, target_os = "cloudabi"))] - unsafe fn abort() -> ! { - libc::abort(); - } - - #[cfg(any(windows, all(target_arch = "wasm32", not(target_os = "emscripten"))))] - unsafe fn abort() -> ! { - core::intrinsics::abort(); - } + abort(); - #[cfg(any(target_os = "hermit", all(target_vendor = "fortanix", target_env = "sgx")))] - unsafe fn abort() -> ! { - // call std::sys::abort_internal - extern "C" { - pub fn __rust_abort() -> !; + cfg_if::cfg_if! { + if #[cfg(any(unix, target_os = "solid_asp3"))] { + unsafe fn abort() -> ! { + libc::abort(); + } + } else if #[cfg(any(target_os = "hermit", + all(target_vendor = "fortanix", target_env = "sgx") + ))] { + unsafe fn abort() -> ! { + // call std::sys::abort_internal + extern "C" { + pub fn __rust_abort() -> !; + } + __rust_abort(); + } + } else if #[cfg(all(windows, not(miri)))] { + // On Windows, use the processor-specific __fastfail mechanism. In Windows 8 + // and later, this will terminate the process immediately without running any + // in-process exception handlers. In earlier versions of Windows, this + // sequence of instructions will be treated as an access violation, + // terminating the process but without necessarily bypassing all exception + // handlers. + // + // https://docs.microsoft.com/en-us/cpp/intrinsics/fastfail + // + // Note: this is the same implementation as in std's `abort_internal` + unsafe fn abort() -> ! { + #[allow(unused)] + const FAST_FAIL_FATAL_APP_EXIT: usize = 7; + cfg_if::cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + core::arch::asm!("int $$0x29", in("ecx") FAST_FAIL_FATAL_APP_EXIT); + } else if #[cfg(all(target_arch = "arm", target_feature = "thumb-mode"))] { + core::arch::asm!(".inst 0xDEFB", in("r0") FAST_FAIL_FATAL_APP_EXIT); + } else if #[cfg(target_arch = "aarch64")] { + core::arch::asm!("brk 0xF003", in("x0") FAST_FAIL_FATAL_APP_EXIT); + } else { + core::intrinsics::abort(); + } + } + core::intrinsics::unreachable(); + } + } else { + unsafe fn abort() -> ! { + core::intrinsics::abort(); + } } - __rust_abort(); } } // This... is a bit of an oddity. The tl;dr; is that this is required to link // correctly, the longer explanation is below. // -// Right now the binaries of libcore/libstd that we ship are all compiled with +// Right now the binaries of core/std that we ship are all compiled with // `-C panic=unwind`. This is done to ensure that the binaries are maximally // compatible with as many situations as possible. The compiler, however, // requires a "personality function" for all functions compiled with `-C @@ -82,45 +109,22 @@ pub unsafe extern "C" fn __rust_start_panic(_payload: usize) -> u32 { // library just defines this symbol so there's at least some personality // somewhere. // -// Essentially this symbol is just defined to get wired up to libcore/libstd +// Essentially this symbol is just defined to get wired up to core/std // binaries, but it should never be called as we don't link in an unwinding // runtime at all. pub mod personalities { - #[rustc_std_internal_symbol] - #[cfg(not(any( - all(target_arch = "wasm32", not(target_os = "emscripten"),), - all(target_os = "windows", target_env = "gnu", target_arch = "x86_64",), - )))] - pub extern "C" fn rust_eh_personality() {} - - // On x86_64-pc-windows-gnu we use our own personality function that needs - // to return `ExceptionContinueSearch` as we're passing on all our frames. - #[rustc_std_internal_symbol] - #[cfg(all(target_os = "windows", target_env = "gnu", target_arch = "x86_64"))] - pub extern "C" fn rust_eh_personality( - _record: usize, - _frame: usize, - _context: usize, - _dispatcher: usize, - ) -> u32 { - 1 // `ExceptionContinueSearch` - } + // In the past this module used to contain stubs for the personality + // functions of various platforms, but these where removed when personality + // functions were moved to std. - // Similar to above, this corresponds to the `eh_unwind_resume` lang item - // that's only used on Windows currently. + // This corresponds to the `eh_catch_typeinfo` lang item + // that's only used on Emscripten currently. // - // Note that we don't execute landing pads, so this is never called, so it's - // body is empty. - #[rustc_std_internal_symbol] - #[cfg(all(bootstrap, target_os = "windows", target_env = "gnu"))] - pub extern "C" fn rust_eh_unwind_resume() {} - - // These two are called by our startup objects on i686-pc-windows-gnu, but - // they don't need to do anything so the bodies are nops. - #[rustc_std_internal_symbol] - #[cfg(all(target_os = "windows", target_env = "gnu", target_arch = "x86"))] - pub extern "C" fn rust_eh_register_frames() {} + // Since panics don't generate exceptions and foreign exceptions are + // currently UB with -C panic=abort (although this may be subject to + // change), any catch_unwind calls will never use this typeinfo. #[rustc_std_internal_symbol] - #[cfg(all(target_os = "windows", target_env = "gnu", target_arch = "x86"))] - pub extern "C" fn rust_eh_unregister_frames() {} + #[allow(non_upper_case_globals)] + #[cfg(target_os = "emscripten")] + static rust_eh_catch_typeinfo: [usize; 2] = [0; 2]; } diff --git a/crux-mir/lib/panic_unwind/Cargo.toml b/crux-mir/lib/panic_unwind/Cargo.toml index 47cd09f1b..85386976d 100644 --- a/crux-mir/lib/panic_unwind/Cargo.toml +++ b/crux-mir/lib/panic_unwind/Cargo.toml @@ -1,19 +1,20 @@ [package] -authors = ["The Rust Project Developers"] name = "panic_unwind" version = "0.0.0" -edition = "2018" +license = "MIT OR Apache-2.0" +repository = "https://github.com/rust-lang/rust.git" +description = "Implementation of Rust panics via stack unwinding" +edition = "2021" [lib] -path = "lib.rs" test = false bench = false doc = false [dependencies] -alloc = { path = "../liballoc" } -core = { path = "../libcore" } +alloc = { path = "../alloc" } +core = { path = "../core" } libc = { version = "0.2", default-features = false } -unwind = { path = "../libunwind" } +unwind = { path = "../unwind" } compiler_builtins = "0.1.0" -cfg-if = "0.1.8" +cfg-if = "1.0" diff --git a/crux-mir/lib/panic_unwind/dummy.rs b/crux-mir/lib/panic_unwind/src/dummy.rs similarity index 67% rename from crux-mir/lib/panic_unwind/dummy.rs rename to crux-mir/lib/panic_unwind/src/dummy.rs index 4667ede2b..a4bcd216c 100644 --- a/crux-mir/lib/panic_unwind/dummy.rs +++ b/crux-mir/lib/panic_unwind/src/dummy.rs @@ -1,6 +1,6 @@ -//! Unwinding for *wasm32* target. +//! Unwinding for unsupported target. //! -//! Right now we don't support this, so this is just stubs. +//! Stubs that simply abort for targets that don't support unwinding otherwise. use alloc::boxed::Box; use core::any::Any; diff --git a/crux-mir/lib/panic_unwind/emcc.rs b/crux-mir/lib/panic_unwind/src/emcc.rs similarity index 62% rename from crux-mir/lib/panic_unwind/emcc.rs rename to crux-mir/lib/panic_unwind/src/emcc.rs index a0bdb1481..c6d423085 100644 --- a/crux-mir/lib/panic_unwind/emcc.rs +++ b/crux-mir/lib/panic_unwind/src/emcc.rs @@ -8,9 +8,10 @@ use alloc::boxed::Box; use core::any::Any; +use core::intrinsics; use core::mem; use core::ptr; -use libc::{self, c_int}; +use core::sync::atomic::{AtomicBool, Ordering}; use unwind as uw; // This matches the layout of std::type_info in C++ @@ -46,7 +47,17 @@ static EXCEPTION_TYPE_INFO: TypeInfo = TypeInfo { name: b"rust_panic\0".as_ptr(), }; +// NOTE(nbdd0121): The `canary` field will be part of stable ABI after `c_unwind` stabilization. +#[repr(C)] struct Exception { + // See `gcc.rs` on why this is present. We already have a static here so just use it. + canary: *const TypeInfo, + + // This is necessary because C++ code can capture our exception with + // std::exception_ptr and rethrow it multiple times, possibly even in + // another thread. + caught: AtomicBool, + // This needs to be an Option because the object's lifetime follows C++ // semantics: when catch_unwind moves the Box out of the exception it must // still leave the exception object in a valid state because its destructor @@ -55,53 +66,60 @@ struct Exception { } pub unsafe fn cleanup(ptr: *mut u8) -> Box { - assert!(!ptr.is_null()); - let adjusted_ptr = __cxa_begin_catch(ptr as *mut libc::c_void) as *mut Exception; - let ex = (*adjusted_ptr).data.take(); + // intrinsics::try actually gives us a pointer to this structure. + #[repr(C)] + struct CatchData { + ptr: *mut u8, + is_rust_panic: bool, + } + let catch_data = &*(ptr as *mut CatchData); + + let adjusted_ptr = __cxa_begin_catch(catch_data.ptr as *mut libc::c_void) as *mut Exception; + if !catch_data.is_rust_panic { + super::__rust_foreign_exception(); + } + + let canary = ptr::addr_of!((*adjusted_ptr).canary).read(); + if !ptr::eq(canary, &EXCEPTION_TYPE_INFO) { + super::__rust_foreign_exception(); + } + + let was_caught = (*adjusted_ptr).caught.swap(true, Ordering::SeqCst); + if was_caught { + // Since cleanup() isn't allowed to panic, we just abort instead. + intrinsics::abort(); + } + let out = (*adjusted_ptr).data.take().unwrap(); __cxa_end_catch(); - ex.unwrap() + out } pub unsafe fn panic(data: Box) -> u32 { - let sz = mem::size_of_val(&data); - let exception = __cxa_allocate_exception(sz) as *mut Exception; + let exception = __cxa_allocate_exception(mem::size_of::()) as *mut Exception; if exception.is_null() { return uw::_URC_FATAL_PHASE1_ERROR as u32; } - ptr::write(exception, Exception { data: Some(data) }); + ptr::write( + exception, + Exception { + canary: &EXCEPTION_TYPE_INFO, + caught: AtomicBool::new(false), + data: Some(data), + }, + ); __cxa_throw(exception as *mut _, &EXCEPTION_TYPE_INFO, exception_cleanup); } -// On WASM and ARM, the destructor returns the pointer to the object. -cfg_if::cfg_if! { - if #[cfg(any(target_arch = "arm", target_arch = "wasm32"))] { - type DestructorRet = *mut libc::c_void; - } else { - type DestructorRet = (); - } -} -extern "C" fn exception_cleanup(ptr: *mut libc::c_void) -> DestructorRet { +extern "C" fn exception_cleanup(ptr: *mut libc::c_void) -> *mut libc::c_void { unsafe { if let Some(b) = (ptr as *mut Exception).read().data { drop(b); super::__rust_drop_panic(); } - #[cfg(any(target_arch = "arm", target_arch = "wasm32"))] ptr } } -#[lang = "eh_personality"] -unsafe extern "C" fn rust_eh_personality( - version: c_int, - actions: uw::_Unwind_Action, - exception_class: uw::_Unwind_Exception_Class, - exception_object: *mut uw::_Unwind_Exception, - context: *mut uw::_Unwind_Context, -) -> uw::_Unwind_Reason_Code { - __gxx_personality_v0(version, actions, exception_class, exception_object, context) -} - extern "C" { fn __cxa_allocate_exception(thrown_size: libc::size_t) -> *mut libc::c_void; fn __cxa_begin_catch(thrown_exception: *mut libc::c_void) -> *mut libc::c_void; @@ -109,13 +127,6 @@ extern "C" { fn __cxa_throw( thrown_exception: *mut libc::c_void, tinfo: *const TypeInfo, - dest: extern "C" fn(*mut libc::c_void) -> DestructorRet, + dest: extern "C" fn(*mut libc::c_void) -> *mut libc::c_void, ) -> !; - fn __gxx_personality_v0( - version: c_int, - actions: uw::_Unwind_Action, - exception_class: uw::_Unwind_Exception_Class, - exception_object: *mut uw::_Unwind_Exception, - context: *mut uw::_Unwind_Context, - ) -> uw::_Unwind_Reason_Code; } diff --git a/crux-mir/lib/panic_unwind/src/gcc.rs b/crux-mir/lib/panic_unwind/src/gcc.rs new file mode 100644 index 000000000..0b7a873a6 --- /dev/null +++ b/crux-mir/lib/panic_unwind/src/gcc.rs @@ -0,0 +1,113 @@ +//! Implementation of panics backed by libgcc/libunwind (in some form). +//! +//! For background on exception handling and stack unwinding please see +//! "Exception Handling in LLVM" (llvm.org/docs/ExceptionHandling.html) and +//! documents linked from it. +//! These are also good reads: +//! * +//! * +//! * +//! +//! ## A brief summary +//! +//! Exception handling happens in two phases: a search phase and a cleanup +//! phase. +//! +//! In both phases the unwinder walks stack frames from top to bottom using +//! information from the stack frame unwind sections of the current process's +//! modules ("module" here refers to an OS module, i.e., an executable or a +//! dynamic library). +//! +//! For each stack frame, it invokes the associated "personality routine", whose +//! address is also stored in the unwind info section. +//! +//! In the search phase, the job of a personality routine is to examine +//! exception object being thrown, and to decide whether it should be caught at +//! that stack frame. Once the handler frame has been identified, cleanup phase +//! begins. +//! +//! In the cleanup phase, the unwinder invokes each personality routine again. +//! This time it decides which (if any) cleanup code needs to be run for +//! the current stack frame. If so, the control is transferred to a special +//! branch in the function body, the "landing pad", which invokes destructors, +//! frees memory, etc. At the end of the landing pad, control is transferred +//! back to the unwinder and unwinding resumes. +//! +//! Once stack has been unwound down to the handler frame level, unwinding stops +//! and the last personality routine transfers control to the catch block. + +use alloc::boxed::Box; +use core::any::Any; +use core::ptr; + +use unwind as uw; + +// In case where multiple copies of std exist in a single process, +// we use address of this static variable to distinguish an exception raised by +// this copy and some other copy (which needs to be treated as foreign exception). +static CANARY: u8 = 0; + +// NOTE(nbdd0121) +// Once `c_unwind` feature is stabilized, there will be ABI stability requirement +// on this struct. The first two field must be `_Unwind_Exception` and `canary`, +// as it may be accessed by a different version of the std with a different compiler. +#[repr(C)] +struct Exception { + _uwe: uw::_Unwind_Exception, + canary: *const u8, + cause: Box, +} + +pub unsafe fn panic(data: Box) -> u32 { + let exception = Box::new(Exception { + _uwe: uw::_Unwind_Exception { + exception_class: rust_exception_class(), + exception_cleanup, + private: [0; uw::unwinder_private_data_size], + }, + canary: &CANARY, + cause: data, + }); + let exception_param = Box::into_raw(exception) as *mut uw::_Unwind_Exception; + return uw::_Unwind_RaiseException(exception_param) as u32; + + extern "C" fn exception_cleanup( + _unwind_code: uw::_Unwind_Reason_Code, + exception: *mut uw::_Unwind_Exception, + ) { + unsafe { + let _: Box = Box::from_raw(exception as *mut Exception); + super::__rust_drop_panic(); + } + } +} + +pub unsafe fn cleanup(ptr: *mut u8) -> Box { + let exception = ptr as *mut uw::_Unwind_Exception; + if (*exception).exception_class != rust_exception_class() { + uw::_Unwind_DeleteException(exception); + super::__rust_foreign_exception(); + } + + let exception = exception.cast::(); + // Just access the canary field, avoid accessing the entire `Exception` as + // it can be a foreign Rust exception. + let canary = ptr::addr_of!((*exception).canary).read(); + if !ptr::eq(canary, &CANARY) { + // A foreign Rust exception, treat it slightly differently from other + // foreign exceptions, because call into `_Unwind_DeleteException` will + // call into `__rust_drop_panic` which produces a confusing + // "Rust panic must be rethrown" message. + super::__rust_foreign_exception(); + } + + let exception = Box::from_raw(exception as *mut Exception); + exception.cause +} + +// Rust's exception class identifier. This is used by personality routines to +// determine whether the exception was thrown by their own runtime. +fn rust_exception_class() -> uw::_Unwind_Exception_Class { + // M O Z \0 R U S T -- vendor, language + 0x4d4f5a_00_52555354 +} diff --git a/crux-mir/lib/panic_unwind/hermit.rs b/crux-mir/lib/panic_unwind/src/hermit.rs similarity index 100% rename from crux-mir/lib/panic_unwind/hermit.rs rename to crux-mir/lib/panic_unwind/src/hermit.rs diff --git a/crux-mir/lib/panic_unwind/lib.rs b/crux-mir/lib/panic_unwind/src/lib.rs similarity index 63% rename from crux-mir/lib/panic_unwind/lib.rs rename to crux-mir/lib/panic_unwind/src/lib.rs index 0a2a0e9e0..ea3c9a7a6 100644 --- a/crux-mir/lib/panic_unwind/lib.rs +++ b/crux-mir/lib/panic_unwind/src/lib.rs @@ -13,23 +13,17 @@ #![no_std] #![unstable(feature = "panic_unwind", issue = "32837")] -#![doc( - html_root_url = "https://doc.rust-lang.org/nightly/", - issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/" -)] +#![doc(issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/")] #![feature(core_intrinsics)] #![feature(lang_items)] -#![feature(libc)] -#![feature(nll)] #![feature(panic_unwind)] #![feature(staged_api)] #![feature(std_internals)] -#![feature(unwind_attributes)] #![feature(abi_thiscall)] #![feature(rustc_attrs)] -#![feature(raw)] #![panic_runtime] #![feature(panic_runtime)] +#![feature(c_unwind)] // `real_imp` is unused with Miri, so silence warnings. #![cfg_attr(miri, allow(dead_code))] @@ -41,24 +35,36 @@ cfg_if::cfg_if! { if #[cfg(target_os = "emscripten")] { #[path = "emcc.rs"] mod real_imp; - } else if #[cfg(target_arch = "wasm32")] { - #[path = "dummy.rs"] - mod real_imp; } else if #[cfg(target_os = "hermit")] { #[path = "hermit.rs"] mod real_imp; - } else if #[cfg(all(target_env = "msvc", target_arch = "aarch64"))] { + } else if #[cfg(target_os = "l4re")] { + // L4Re is unix family but does not yet support unwinding. #[path = "dummy.rs"] mod real_imp; - } else if #[cfg(target_env = "msvc")] { + } else if #[cfg(all(target_env = "msvc", not(target_arch = "arm")))] { + // LLVM does not support unwinding on 32 bit ARM msvc (thumbv7a-pc-windows-msvc) #[path = "seh.rs"] mod real_imp; - } else { - // Rust runtime's startup objects depend on these symbols, so make them public. - #[cfg(all(target_os="windows", target_arch = "x86", target_env="gnu"))] - pub use real_imp::eh_frame_registry::*; + } else if #[cfg(any( + all(target_family = "windows", target_env = "gnu"), + target_os = "psp", + target_os = "solid_asp3", + all(target_family = "unix", not(target_os = "espidf")), + all(target_vendor = "fortanix", target_env = "sgx"), + ))] { #[path = "gcc.rs"] mod real_imp; + } else { + // Targets that don't support unwinding. + // - family=wasm + // - os=none ("bare metal" targets) + // - os=uefi + // - os=espidf + // - nvptx64-nvidia-cuda + // - arch=avr + #[path = "dummy.rs"] + mod real_imp; } } @@ -76,14 +82,16 @@ cfg_if::cfg_if! { } extern "C" { - /// Handler in libstd called when a panic object is dropped outside of + /// Handler in std called when a panic object is dropped outside of /// `catch_unwind`. fn __rust_drop_panic() -> !; -} -mod dwarf; + /// Handler in std called when a foreign exception is caught. + fn __rust_foreign_exception() -> !; +} #[rustc_std_internal_symbol] +#[allow(improper_ctypes_definitions)] pub unsafe extern "C" fn __rust_panic_cleanup(payload: *mut u8) -> *mut (dyn Any + Send + 'static) { Box::into_raw(imp::cleanup(payload)) } @@ -91,10 +99,8 @@ pub unsafe extern "C" fn __rust_panic_cleanup(payload: *mut u8) -> *mut (dyn Any // Entry point for raising an exception, just delegates to the platform-specific // implementation. #[rustc_std_internal_symbol] -#[unwind(allowed)] -pub unsafe extern "C" fn __rust_start_panic(payload: usize) -> u32 { - let payload = payload as *mut &mut dyn BoxMeUp; - let payload = (*payload).take_box(); +pub unsafe fn __rust_start_panic(payload: *mut &mut dyn BoxMeUp) -> u32 { + let payload = Box::from_raw((*payload).take_box()); - imp::panic(Box::from_raw(payload)) + imp::panic(payload) } diff --git a/crux-mir/lib/panic_unwind/miri.rs b/crux-mir/lib/panic_unwind/src/miri.rs similarity index 79% rename from crux-mir/lib/panic_unwind/miri.rs rename to crux-mir/lib/panic_unwind/src/miri.rs index 9d92b2b2f..d941b73b5 100644 --- a/crux-mir/lib/panic_unwind/miri.rs +++ b/crux-mir/lib/panic_unwind/src/miri.rs @@ -6,11 +6,16 @@ use core::any::Any; // Must be pointer-sized. type Payload = Box>; +extern "Rust" { + /// Miri-provided extern function to begin unwinding. + fn miri_start_panic(payload: *mut u8) -> !; +} + pub unsafe fn panic(payload: Box) -> u32 { // The payload we pass to `miri_start_panic` will be exactly the argument we get // in `cleanup` below. So we just box it up once, to get something pointer-sized. let payload_box: Payload = Box::new(payload); - core::intrinsics::miri_start_panic(Box::into_raw(payload_box) as *mut u8) + miri_start_panic(Box::into_raw(payload_box) as *mut u8) } pub unsafe fn cleanup(payload_box: *mut u8) -> Box { diff --git a/crux-mir/lib/panic_unwind/seh.rs b/crux-mir/lib/panic_unwind/src/seh.rs similarity index 86% rename from crux-mir/lib/panic_unwind/seh.rs rename to crux-mir/lib/panic_unwind/src/seh.rs index 10b765a5b..651115a82 100644 --- a/crux-mir/lib/panic_unwind/seh.rs +++ b/crux-mir/lib/panic_unwind/src/seh.rs @@ -42,16 +42,22 @@ //! of the `try` intrinsic. //! //! [win64]: https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64 -//! [llvm]: http://llvm.org/docs/ExceptionHandling.html#background-on-windows-exceptions +//! [llvm]: https://llvm.org/docs/ExceptionHandling.html#background-on-windows-exceptions #![allow(nonstandard_style)] use alloc::boxed::Box; use core::any::Any; use core::mem::{self, ManuallyDrop}; +use core::ptr; use libc::{c_int, c_uint, c_void}; +// NOTE(nbdd0121): The `canary` field will be part of stable ABI after `c_unwind` stabilization. +#[repr(C)] struct Exception { + // See `gcc.rs` on why this is present. We already have a static here so just use it. + canary: *const _TypeDescriptor, + // This needs to be an Option because we catch the exception by reference // and its destructor is executed by the C++ runtime. When we take the Box // out of the exception, we need to leave the exception in a valid state @@ -100,7 +106,7 @@ struct Exception { // In any case, these structures are all constructed in a similar manner, and // it's just somewhat verbose for us. // -// [1]: http://www.geoffchappell.com/studies/msvc/language/predefined/ +// [1]: https://www.geoffchappell.com/studies/msvc/language/predefined/ #[cfg(target_arch = "x86")] #[macro_use] @@ -117,7 +123,7 @@ mod imp { } } -#[cfg(any(target_arch = "x86_64", target_arch = "arm"))] +#[cfg(not(target_arch = "x86"))] #[macro_use] mod imp { pub type ptr_t = u32; @@ -175,7 +181,7 @@ pub struct _TypeDescriptor { // to be able to catch Rust panics by simply declaring a `struct rust_panic`. // // When modifying, make sure that the type name string exactly matches -// the one used in src/librustc_codegen_llvm/intrinsic.rs. +// the one used in `compiler/rustc_codegen_llvm/src/intrinsic.rs`. const TYPE_NAME: [u8; 11] = *b"rust_panic\0"; static mut THROW_INFO: _ThrowInfo = _ThrowInfo { @@ -213,7 +219,6 @@ extern "C" { // // This is fine since the MSVC runtime uses string comparison on the type name // to match TypeDescriptors rather than pointer equality. -#[cfg_attr(bootstrap, lang = "eh_catch_typeinfo")] static mut TYPE_DESCRIPTOR: _TypeDescriptor = _TypeDescriptor { pVFTable: unsafe { &TYPE_INFO_VTABLE } as *const _ as *const _, spare: core::ptr::null_mut(), @@ -234,15 +239,14 @@ static mut TYPE_DESCRIPTOR: _TypeDescriptor = _TypeDescriptor { // support capturing exceptions with std::exception_ptr, which we can't support // because Box isn't clonable. macro_rules! define_cleanup { - ($abi:tt) => { + ($abi:tt $abi2:tt) => { unsafe extern $abi fn exception_cleanup(e: *mut Exception) { - if let Exception { data: Some(b) } = e.read() { + if let Exception { data: Some(b), .. } = e.read() { drop(b); super::__rust_drop_panic(); } } - #[unwind(allowed)] - unsafe extern $abi fn exception_copy(_dest: *mut Exception, + unsafe extern $abi2 fn exception_copy(_dest: *mut Exception, _src: *mut Exception) -> *mut Exception { panic!("Rust panics cannot be copied"); @@ -251,14 +255,14 @@ macro_rules! define_cleanup { } cfg_if::cfg_if! { if #[cfg(target_arch = "x86")] { - define_cleanup!("thiscall"); + define_cleanup!("thiscall" "thiscall-unwind"); } else { - define_cleanup!("C"); + define_cleanup!("C" "C-unwind"); } } pub unsafe fn panic(data: Box) -> u32 { - use core::intrinsics::atomic_store; + use core::intrinsics::atomic_store_seqcst; // _CxxThrowException executes entirely on this stack frame, so there's no // need to otherwise transfer `data` to the heap. We just pass a stack @@ -267,7 +271,7 @@ pub unsafe fn panic(data: Box) -> u32 { // The ManuallyDrop is needed here since we don't want Exception to be // dropped when unwinding. Instead it will be dropped by exception_cleanup // which is invoked by the C++ runtime. - let mut exception = ManuallyDrop::new(Exception { data: Some(data) }); + let mut exception = ManuallyDrop::new(Exception { canary: &TYPE_DESCRIPTOR, data: Some(data) }); let throw_ptr = &mut exception as *mut _ as *mut _; // This... may seems surprising, and justifiably so. On 32-bit MSVC the @@ -290,43 +294,45 @@ pub unsafe fn panic(data: Box) -> u32 { // // In any case, we basically need to do something like this until we can // express more operations in statics (and we may never be able to). - atomic_store(&mut THROW_INFO.pmfnUnwind as *mut _ as *mut u32, ptr!(exception_cleanup) as u32); - atomic_store( + atomic_store_seqcst( + &mut THROW_INFO.pmfnUnwind as *mut _ as *mut u32, + ptr!(exception_cleanup) as u32, + ); + atomic_store_seqcst( &mut THROW_INFO.pCatchableTypeArray as *mut _ as *mut u32, ptr!(&CATCHABLE_TYPE_ARRAY as *const _) as u32, ); - atomic_store( + atomic_store_seqcst( &mut CATCHABLE_TYPE_ARRAY.arrayOfCatchableTypes[0] as *mut _ as *mut u32, ptr!(&CATCHABLE_TYPE as *const _) as u32, ); - atomic_store( + atomic_store_seqcst( &mut CATCHABLE_TYPE.pType as *mut _ as *mut u32, ptr!(&TYPE_DESCRIPTOR as *const _) as u32, ); - atomic_store( + atomic_store_seqcst( &mut CATCHABLE_TYPE.copyFunction as *mut _ as *mut u32, ptr!(exception_copy) as u32, ); - extern "system" { - #[unwind(allowed)] - pub fn _CxxThrowException(pExceptionObject: *mut c_void, pThrowInfo: *mut u8) -> !; + extern "system-unwind" { + fn _CxxThrowException(pExceptionObject: *mut c_void, pThrowInfo: *mut u8) -> !; } _CxxThrowException(throw_ptr, &mut THROW_INFO as *mut _ as *mut _); } pub unsafe fn cleanup(payload: *mut u8) -> Box { - let exception = &mut *(payload as *mut Exception); - exception.data.take().unwrap() -} - -// This is required by the compiler to exist (e.g., it's a lang item), but -// it's never actually called by the compiler because __C_specific_handler -// or _except_handler3 is the personality function that is always used. -// Hence this is just an aborting stub. -#[lang = "eh_personality"] -#[cfg(not(test))] -fn rust_eh_personality() { - unsafe { core::intrinsics::abort() } + // A null payload here means that we got here from the catch (...) of + // __rust_try. This happens when a non-Rust foreign exception is caught. + if payload.is_null() { + super::__rust_foreign_exception(); + } + let exception = payload as *mut Exception; + let canary = ptr::addr_of!((*exception).canary).read(); + if !ptr::eq(canary, &TYPE_DESCRIPTOR) { + // A foreign Rust exception. + super::__rust_foreign_exception(); + } + (*exception).data.take().unwrap() } diff --git a/crux-mir/lib/portable-simd/.github/ISSUE_TEMPLATE/blank_issue.md b/crux-mir/lib/portable-simd/.github/ISSUE_TEMPLATE/blank_issue.md new file mode 100644 index 000000000..9aef3ebe6 --- /dev/null +++ b/crux-mir/lib/portable-simd/.github/ISSUE_TEMPLATE/blank_issue.md @@ -0,0 +1,4 @@ +--- +name: Blank Issue +about: Create a blank issue. +--- diff --git a/crux-mir/lib/portable-simd/.github/ISSUE_TEMPLATE/bug_report.md b/crux-mir/lib/portable-simd/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..16a8251d5 --- /dev/null +++ b/crux-mir/lib/portable-simd/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,50 @@ +--- +name: Bug Report +about: Create a bug report for Rust. +labels: C-bug +--- + + +I tried this code: + +```rust + +``` + +I expected to see this happen: *explanation* + +Instead, this happened: *explanation* + +### Meta + +`rustc --version --verbose`: +``` + +``` + + +`crate version in Cargo.toml`: +```toml +[dependencies] +stdsimd = +``` + + + + +
Backtrace +

+ +``` + +``` + +

+
diff --git a/crux-mir/lib/portable-simd/.github/ISSUE_TEMPLATE/config.yml b/crux-mir/lib/portable-simd/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..1567542c0 --- /dev/null +++ b/crux-mir/lib/portable-simd/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,10 @@ +# This only controls whether a tiny, hard-to-find "open a blank issue" link appears at the end of +# the template list. +blank_issues_enabled: true +contact_links: + - name: Intrinsic Support + url: https://github.com/rust-lang/stdarch/issues + about: Please direct issues about Rust's support for vendor intrinsics to core::arch + - name: Internal Compiler Error + url: https://github.com/rust-lang/rust/issues + about: Please report ICEs to the rustc repository diff --git a/crux-mir/lib/portable-simd/.github/ISSUE_TEMPLATE/feature_request.md b/crux-mir/lib/portable-simd/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..be6c9e3d1 --- /dev/null +++ b/crux-mir/lib/portable-simd/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,14 @@ +--- +name: Feature Request +about: Request an addition to the core::simd API +labels: C-feature-request +--- + diff --git a/crux-mir/lib/portable-simd/.github/PULL_REQUEST_TEMPLATE.md b/crux-mir/lib/portable-simd/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..31422b793 --- /dev/null +++ b/crux-mir/lib/portable-simd/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,18 @@ +Hello, welcome to `std::simd`! + +It seems this pull request template checklist was created while a lot of vector math ops were being implemented, and only really applies to ops. Feel free to delete everything here if it's not applicable, or ask for help if you're not sure what it means! + +For a given vector math operation on TxN, please add tests for interactions with: + - [ ] `T::MAX` + - [ ] `T::MIN` + - [ ] -1 + - [ ] 1 + - [ ] 0 + + +For a given vector math operation on TxN where T is a float, please add tests for test interactions with: + - [ ] a really large number, larger than the mantissa + - [ ] a really small "subnormal" number + - [ ] NaN + - [ ] Infinity + - [ ] Negative Infinity diff --git a/crux-mir/lib/portable-simd/.github/workflows/ci.yml b/crux-mir/lib/portable-simd/.github/workflows/ci.yml new file mode 100644 index 000000000..d50dfa1be --- /dev/null +++ b/crux-mir/lib/portable-simd/.github/workflows/ci.yml @@ -0,0 +1,260 @@ +name: CI + +on: + pull_request: + push: + branches: + - master + +env: + CARGO_NET_RETRY: 10 + RUSTUP_MAX_RETRIES: 10 + +jobs: + rustfmt: + name: "rustfmt" + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Setup Rust + run: | + rustup update nightly --no-self-update + rustup default nightly + rustup component add rustfmt + - name: Run rustfmt + run: cargo fmt --all -- --check + + clippy: + name: "clippy on ${{ matrix.target }}" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + target: + # We shouldn't really have any OS-specific code, so think of this as a list of architectures + - x86_64-unknown-linux-gnu + - i686-unknown-linux-gnu + - i586-unknown-linux-gnu + - aarch64-unknown-linux-gnu + - armv7-unknown-linux-gnueabihf + - mips-unknown-linux-gnu + - mips64-unknown-linux-gnuabi64 + - powerpc-unknown-linux-gnu + - powerpc64-unknown-linux-gnu + - riscv64gc-unknown-linux-gnu + - s390x-unknown-linux-gnu + - sparc64-unknown-linux-gnu + - wasm32-unknown-unknown + + steps: + - uses: actions/checkout@v2 + - name: Setup Rust + run: | + rustup update nightly --no-self-update + rustup default nightly + rustup target add ${{ matrix.target }} + rustup component add clippy + - name: Run Clippy + run: cargo clippy --all-targets --target ${{ matrix.target }} + + x86-tests: + name: "${{ matrix.target_feature }} on ${{ matrix.target }}" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + target: [x86_64-pc-windows-msvc, i686-pc-windows-msvc, i586-pc-windows-msvc, x86_64-unknown-linux-gnu, x86_64-apple-darwin] + # `default` means we use the default target config for the target, + # `native` means we run with `-Ctarget-cpu=native`, and anything else is + # an arg to `-Ctarget-feature` + target_feature: [default, native, +sse3, +ssse3, +sse4.1, +sse4.2, +avx, +avx2] + + exclude: + # The macos runners seem to only reliably support up to `avx`. + - { target: x86_64-apple-darwin, target_feature: +avx2 } + # These features are statically known to be present for all 64 bit + # macs, and thus are covered by the `default` test + - { target: x86_64-apple-darwin, target_feature: +sse3 } + - { target: x86_64-apple-darwin, target_feature: +ssse3 } + # -Ctarget-cpu=native sounds like bad-news if target != host + - { target: i686-pc-windows-msvc, target_feature: native } + - { target: i586-pc-windows-msvc, target_feature: native } + + include: + # Populate the `matrix.os` field + - { target: x86_64-apple-darwin, os: macos-latest } + - { target: x86_64-unknown-linux-gnu, os: ubuntu-latest } + - { target: x86_64-pc-windows-msvc, os: windows-latest } + - { target: i686-pc-windows-msvc, os: windows-latest } + - { target: i586-pc-windows-msvc, os: windows-latest } + + # These are globally available on all the other targets. + - { target: i586-pc-windows-msvc, target_feature: +sse, os: windows-latest } + - { target: i586-pc-windows-msvc, target_feature: +sse2, os: windows-latest } + + # Annoyingly, the x86_64-unknown-linux-gnu runner *almost* always has + # avx512vl, but occasionally doesn't. Maybe one day we can enable it. + + steps: + - uses: actions/checkout@v2 + - name: Setup Rust + run: | + rustup update nightly --no-self-update + rustup default nightly + rustup target add ${{ matrix.target }} + + - name: Configure RUSTFLAGS + shell: bash + run: | + case "${{ matrix.target_feature }}" in + default) + echo "RUSTFLAGS=-Dwarnings" >> $GITHUB_ENV;; + native) + echo "RUSTFLAGS=-Dwarnings -Ctarget-cpu=native" >> $GITHUB_ENV + ;; + *) + echo "RUSTFLAGS=-Dwarnings -Ctarget-feature=${{ matrix.target_feature }}" >> $GITHUB_ENV + ;; + esac + + # Super useful for debugging why a SIGILL occurred. + - name: Dump target configuration and support + run: | + rustc -Vv + + echo "Caveat: not all target features are expected to be logged" + + echo "## Requested target configuration (RUSTFLAGS=$RUSTFLAGS)" + rustc --print=cfg --target=${{ matrix.target }} $RUSTFLAGS + + echo "## Supported target configuration for --target=${{ matrix.target }}" + rustc --print=cfg --target=${{ matrix.target }} -Ctarget-cpu=native + + echo "## Natively supported target configuration" + rustc --print=cfg -Ctarget-cpu=native + + - name: Test (debug) + run: cargo test --verbose --target=${{ matrix.target }} + + - name: Test (release) + run: cargo test --verbose --target=${{ matrix.target }} --release + + wasm-tests: + name: "wasm (firefox, ${{ matrix.name }})" + runs-on: ubuntu-latest + strategy: + matrix: + include: + - { name: default, RUSTFLAGS: "" } + - { name: simd128, RUSTFLAGS: "-C target-feature=+simd128" } + steps: + - uses: actions/checkout@v2 + - name: Setup Rust + run: | + rustup update nightly --no-self-update + rustup default nightly + - name: Install wasm-pack + run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + - name: Test (debug) + run: wasm-pack test --firefox --headless crates/core_simd + env: + RUSTFLAGS: ${{ matrix.rustflags }} + - name: Test (release) + run: wasm-pack test --firefox --headless crates/core_simd --release + env: + RUSTFLAGS: ${{ matrix.rustflags }} + + cross-tests: + name: "${{ matrix.target }} (via cross)" + runs-on: ubuntu-latest + strategy: + fail-fast: false + # TODO: Sadly, we cant configure target-feature in a meaningful way + # because `cross` doesn't tell qemu to enable any non-default cpu + # features, nor does it give us a way to do so. + # + # Ultimately, we'd like to do something like [rust-lang/stdarch][stdarch]. + # This is a lot more complex... but in practice it's likely that we can just + # snarf the docker config from around [here][1000-dockerfiles]. + # + # [stdarch]: https://github.com/rust-lang/stdarch/blob/a5db4eaf/.github/workflows/main.yml#L67 + # [1000-dockerfiles]: https://github.com/rust-lang/stdarch/tree/a5db4eaf/ci/docker + + matrix: + target: + - i586-unknown-linux-gnu + # 32-bit arm has a few idiosyncracies like having subnormal flushing + # to zero on by default. Ideally we'd set + - armv7-unknown-linux-gnueabihf + - aarch64-unknown-linux-gnu + # Note: The issue above means neither of these mips targets will use + # MSA (mips simd) but MIPS uses a nonstandard binary representation + # for NaNs which makes it worth testing on despite that. + - mips-unknown-linux-gnu + - mips64-unknown-linux-gnuabi64 + - riscv64gc-unknown-linux-gnu + # TODO this test works, but it appears to time out + # - powerpc-unknown-linux-gnu + # TODO this test is broken, but it appears to be a problem with QEMU, not us. + # - powerpc64le-unknown-linux-gnu + # TODO enable this once a new version of cross is released + # - powerpc64-unknown-linux-gnu + + steps: + - uses: actions/checkout@v2 + - name: Setup Rust + run: | + rustup update nightly --no-self-update + rustup default nightly + rustup target add ${{ matrix.target }} + rustup component add rust-src + + - name: Install Cross + # Equivalent to `cargo install cross`, but downloading a prebuilt + # binary. Ideally we wouldn't hardcode a version, but the version number + # being part of the tarball means we can't just use the download/latest + # URL :( + run: | + CROSS_URL=https://github.com/rust-embedded/cross/releases/download/v0.2.1/cross-v0.2.1-x86_64-unknown-linux-gnu.tar.gz + mkdir -p "$HOME/.bin" + curl -sfSL --retry-delay 10 --retry 5 "${CROSS_URL}" | tar zxf - -C "$HOME/.bin" + echo "$HOME/.bin" >> $GITHUB_PATH + + - name: Test (debug) + run: cross test --verbose --target=${{ matrix.target }} + + - name: Test (release) + run: cross test --verbose --target=${{ matrix.target }} --release + + features: + name: "Check cargo features (${{ matrix.simd }} × ${{ matrix.features }})" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + simd: + - "" + - "avx512" + features: + - "" + - "--features std" + - "--features generic_const_exprs" + - "--features std --features generic_const_exprs" + + steps: + - uses: actions/checkout@v2 + - name: Setup Rust + run: | + rustup update nightly --no-self-update + rustup default nightly + - name: Detect AVX512 + run: echo "CPU_FEATURE=$(lscpu | grep -o avx512[a-z]* | sed s/avx/+avx/ | tr '\n' ',' )" >> $GITHUB_ENV + - name: Check build + if: ${{ matrix.simd == '' }} + run: RUSTFLAGS="-Dwarnings" cargo check --all-targets --no-default-features ${{ matrix.features }} + - name: Check AVX + if: ${{ matrix.simd == 'avx512' && contains(env.CPU_FEATURE, 'avx512') }} + run: | + echo "Found AVX features: $CPU_FEATURE" + RUSTFLAGS="-Dwarnings -Ctarget-feature=$CPU_FEATURE" cargo check --all-targets --no-default-features ${{ matrix.features }} diff --git a/crux-mir/lib/portable-simd/.github/workflows/doc.yml b/crux-mir/lib/portable-simd/.github/workflows/doc.yml new file mode 100644 index 000000000..9d1fa66cc --- /dev/null +++ b/crux-mir/lib/portable-simd/.github/workflows/doc.yml @@ -0,0 +1,30 @@ +name: Documentation + +on: + push: + branches: + - master + +jobs: + release: + name: Deploy Documentation + runs-on: ubuntu-latest + + steps: + - name: Checkout Repository + uses: actions/checkout@v1 + + - name: Setup Rust + run: | + rustup update nightly --no-self-update + rustup default nightly + + - name: Build Documentation + run: cargo doc --no-deps + + - name: Deploy Documentation + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_branch: gh-pages + publish_dir: ./target/doc diff --git a/crux-mir/lib/portable-simd/CONTRIBUTING.md b/crux-mir/lib/portable-simd/CONTRIBUTING.md new file mode 100644 index 000000000..9612fe871 --- /dev/null +++ b/crux-mir/lib/portable-simd/CONTRIBUTING.md @@ -0,0 +1,32 @@ +# Contributing to `std::simd` + +Simple version: +1. Fork it and `git clone` it +2. Create your feature branch: `git checkout -b my-branch` +3. Write your changes. +4. Test it: `cargo test`. Remember to enable whatever SIMD features you intend to test by setting `RUSTFLAGS`. +5. Commit your changes: `git commit add ./path/to/changes && git commit -m 'Fix some bug'` +6. Push the branch: `git push --set-upstream origin my-branch` +7. Submit a pull request! + +## Taking on an Issue + +SIMD can be quite complex, and even a "simple" issue can be huge. If an issue is organized like a tracking issue, with an itemized list of items that don't necessarily have to be done in a specific order, please take the issue one item at a time. This will help by letting work proceed apace on the rest of the issue. If it's a (relatively) small issue, feel free to announce your intention to solve it on the issue tracker and take it in one go! + +## CI + +We currently use GitHub Actions which will automatically build and test your change in order to verify that `std::simd`'s portable API is, in fact, portable. If your change builds locally, but does not build in CI, this is likely due to a platform-specific concern that your code has not addressed. Please consult the build logs and address the error, or ask for help if you need it. + +## Beyond stdsimd + +A large amount of the core SIMD implementation is found in the rustc_codegen_* crates in the [main rustc repo](https://github.com/rust-lang/rust). In addition, actual platform-specific functions are implemented in [stdarch]. Not all changes to `std::simd` require interacting with either of these, but if you're wondering where something is and it doesn't seem to be in this repository, those might be where to start looking. + +## Questions? Concerns? Need Help? + +Please feel free to ask in the [#project-portable-simd][zulip-portable-simd] stream on the [rust-lang Zulip][zulip] for help with making changes to `std::simd`! +If your changes include directly modifying the compiler, it might also be useful to ask in [#t-compiler/help][zulip-compiler-help]. + +[zulip-portable-simd]: https://rust-lang.zulipchat.com/#narrow/stream/257879-project-portable-simd +[zulip-compiler-help]: https://rust-lang.zulipchat.com/#narrow/stream/182449-t-compiler.2Fhelp +[zulip]: https://rust-lang.zulipchat.com +[stdarch]: https://github.com/rust-lang/stdarch diff --git a/crux-mir/lib/portable-simd/Cargo.toml b/crux-mir/lib/portable-simd/Cargo.toml new file mode 100644 index 000000000..9802386e4 --- /dev/null +++ b/crux-mir/lib/portable-simd/Cargo.toml @@ -0,0 +1,7 @@ +[workspace] + +members = [ + "crates/core_simd", + "crates/std_float", + "crates/test_helpers", +] diff --git a/crux-mir/lib/portable-simd/LICENSE-APACHE b/crux-mir/lib/portable-simd/LICENSE-APACHE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/crux-mir/lib/portable-simd/LICENSE-APACHE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crux-mir/lib/portable-simd/LICENSE-MIT b/crux-mir/lib/portable-simd/LICENSE-MIT new file mode 100644 index 000000000..0e9d2f43a --- /dev/null +++ b/crux-mir/lib/portable-simd/LICENSE-MIT @@ -0,0 +1,19 @@ +Copyright (c) 2020 The Rust Project Developers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crux-mir/lib/portable-simd/README.md b/crux-mir/lib/portable-simd/README.md new file mode 100644 index 000000000..db0af2da6 --- /dev/null +++ b/crux-mir/lib/portable-simd/README.md @@ -0,0 +1,69 @@ +# The Rust standard library's portable SIMD API +![Build Status](https://github.com/rust-lang/portable-simd/actions/workflows/ci.yml/badge.svg?branch=master) + +Code repository for the [Portable SIMD Project Group](https://github.com/rust-lang/project-portable-simd). +Please refer to [CONTRIBUTING.md](./CONTRIBUTING.md) for our contributing guidelines. + +The docs for this crate are published from the main branch. +You can [read them here][docs]. + +If you have questions about SIMD, we have begun writing a [guide][simd-guide]. +We can also be found on [Zulip][zulip-project-portable-simd]. + +If you are interested in support for a specific architecture, you may want [stdarch] instead. + +## Hello World + +Now we're gonna dip our toes into this world with a small SIMD "Hello, World!" example. Make sure your compiler is up to date and using `nightly`. We can do that by running + +```bash +rustup update -- nightly +``` + +or by setting up `rustup default nightly` or else with `cargo +nightly {build,test,run}`. After updating, run +```bash +cargo new hellosimd +``` +to create a new crate. Edit `hellosimd/Cargo.toml` to be +```toml +[package] +name = "hellosimd" +version = "0.1.0" +edition = "2018" +[dependencies] +core_simd = { git = "https://github.com/rust-lang/portable-simd" } +``` + +and finally write this in `src/main.rs`: +```rust +use core_simd::*; +fn main() { + let a = f32x4::splat(10.0); + let b = f32x4::from_array([1.0, 2.0, 3.0, 4.0]); + println!("{:?}", a + b); +} +``` + +Explanation: We import all the bindings from the crate with the first line. Then, we construct our SIMD vectors with methods like `splat` or `from_array`. Finally, we can use operators on them like `+` and the appropriate SIMD instructions will be carried out. When we run `cargo run` you should get `[11.0, 12.0, 13.0, 14.0]`. + +## Code Organization + +Currently the crate is organized so that each element type is a file, and then the 64-bit, 128-bit, 256-bit, and 512-bit vectors using those types are contained in said file. + +All types are then exported as a single, flat module. + +Depending on the size of the primitive type, the number of lanes the vector will have varies. For example, 128-bit vectors have four `f32` lanes and two `f64` lanes. + +The supported element types are as follows: +* **Floating Point:** `f32`, `f64` +* **Signed Integers:** `i8`, `i16`, `i32`, `i64`, `i128`, `isize` +* **Unsigned Integers:** `u8`, `u16`, `u32`, `u64`, `u128`, `usize` +* **Masks:** `mask8`, `mask16`, `mask32`, `mask64`, `mask128`, `masksize` + +Floating point, signed integers, and unsigned integers are the [primitive types](https://doc.rust-lang.org/core/primitive/index.html) you're already used to. +The `mask` types are "truthy" values, but they use the number of bits in their name instead of just 1 bit like a normal `bool` uses. + +[simd-guide]: ./beginners-guide.md +[zulip-project-portable-simd]: https://rust-lang.zulipchat.com/#narrow/stream/257879-project-portable-simd +[stdarch]: https://github.com/rust-lang/stdarch +[docs]: https://rust-lang.github.io/portable-simd/core_simd diff --git a/crux-mir/lib/portable-simd/beginners-guide.md b/crux-mir/lib/portable-simd/beginners-guide.md new file mode 100644 index 000000000..17ade06ae --- /dev/null +++ b/crux-mir/lib/portable-simd/beginners-guide.md @@ -0,0 +1,91 @@ + +# Beginner's Guide To SIMD + +Hello and welcome to our SIMD basics guide! + +Because SIMD is a subject that many programmers haven't worked with before, we thought that it's best to outline some terms and other basics for you to get started with. + +## Quick Background + +**SIMD** stands for *Single Instruction, Multiple Data*. In other words, SIMD is when the CPU performs a single action on more than one logical piece of data at the same time. Instead of adding two registers that each contain one `f32` value and getting an `f32` as the result, you might add two registers that each contain `f32x4` (128 bits of data) and then you get an `f32x4` as the output. + +This might seem a tiny bit weird at first, but there's a good reason for it. Back in the day, as CPUs got faster and faster, eventually they got so fast that the CPU would just melt itself. The heat management (heat sinks, fans, etc) simply couldn't keep up with how much electricity was going through the metal. Two main strategies were developed to help get around the limits of physics. +* One of them you're probably familiar with: Multi-core processors. By giving a processor more than one core, each core can do its own work, and because they're physically distant (at least on the CPU's scale) the heat can still be managed. Unfortunately, not all tasks can just be split up across cores in an efficient way. +* The second strategy is SIMD. If you can't make the register go any faster, you can still make the register *wider*. This lets you process more data at a time, which is *almost* as good as just having a faster CPU. As with multi-core programming, SIMD doesn't fit every kind of task, so you have to know when it will improve your program. + +## Terms + +SIMD has a few special vocabulary terms you should know: + +* **Vector:** A SIMD value is called a vector. This shouldn't be confused with the `Vec` type. A SIMD vector has a fixed size, known at compile time. All of the elements within the vector are of the same type. This makes vectors *similar to* arrays. One difference is that a vector is generally aligned to its *entire* size (eg: 16 bytes, 32 bytes, etc), not just the size of an individual element. Sometimes vector data is called "packed" data. + +* **Vectorize**: An operation that uses SIMD instructions to operate over a vector is often referred to as "vectorized". + +* **Autovectorization**: Also known as _implicit vectorization_. This is when a compiler can automatically recognize a situation where scalar instructions may be replaced with SIMD instructions, and use those instead. + +* **Scalar:** "Scalar" in mathematical contexts refers to values that can be represented as a single element, mostly numbers like 6, 3.14, or -2. It can also be used to describe "scalar operations" that use strictly scalar values, like addition. This term is mostly used to differentiate between vectorized operations that use SIMD instructions and scalar operations that don't. + +* **Lane:** A single element position within a vector is called a lane. If you have `N` lanes available then they're numbered from `0` to `N-1` when referring to them, again like an array. The biggest difference between an array element and a vector lane is that in general is *relatively costly* to access an individual lane value. On most architectures, the vector has to be pushed out of the SIMD register onto the stack, then an individual lane is accessed while it's on the stack (and possibly the stack value is read back into a register). For this reason, when working with SIMD you should avoid reading or writing the value of an individual lane during hot loops. + +* **Bit Widths:** When talking about SIMD, the bit widths used are the bit size of the vectors involved, *not* the individual elements. So "128-bit SIMD" has 128-bit vectors, and that might be `f32x4`, `i32x4`, `i16x8`, or other variations. While 128-bit SIMD is the most common, there's also 64-bit, 256-bit, and even 512-bit on the newest CPUs. + +* **Vector Register:** The extra-wide registers that are used for SIMD operations are commonly called vector registers, though you may also see "SIMD registers", vendor names for specific features, or even "floating-point register" as it is common for the same registers to be used with both scalar and vectorized floating-point operations. + +* **Vertical:** When an operation is "vertical", each lane processes individually without regard to the other lanes in the same vector. For example, a "vertical add" between two vectors would add lane 0 in `a` with lane 0 in `b`, with the total in lane 0 of `out`, and then the same thing for lanes 1, 2, etc. Most SIMD operations are vertical operations, so if your problem is a vertical problem then you can probably solve it with SIMD. + +* **Reducing/Reduce:** When an operation is "reducing" (functions named `reduce_*`), the lanes within a single vector are merged using some operation such as addition, returning the merged value as a scalar. For instance, a reducing add would return the sum of all the lanes' values. + +* **Target Feature:** Rust calls a CPU architecture extension a `target_feature`. Proper SIMD requires various CPU extensions to be enabled (details below). Don't confuse this with `feature`, which is a Cargo crate concept. + +## Target Features + +When using SIMD, you should be familiar with the CPU feature set that you're targeting. + +On `arm` and `aarch64` it's fairly simple. There's just one CPU feature that controls if SIMD is available: `neon` (or "NEON", all caps, as the ARM docs often put it). Neon registers can be used as 64-bit or 128-bit. When doing 128-bit operations it just uses two 64-bit registers as a single 128-bit register. + +> By default, the `aarch64`, `arm`, and `thumb` Rust targets generally do not enable `neon` unless it's in the target string. + +On `x86` and `x86_64` it's slightly more complicated. The SIMD support is split into many levels: +* 128-bit: `sse`, `sse2`, `sse3`, `ssse3` (not a typo!), `sse4.1`, `sse4.2`, `sse4a` (AMD only) +* 256-bit (mostly): `avx`, `avx2`, `fma` +* 512-bit (mostly): a *wide* range of `avx512` variations + +The list notes the bit widths available at each feature level, though the operations of the more advanced features can generally be used with the smaller register sizes as well. For example, new operations introduced in `avx` generally have a 128-bit form as well as a 256-bit form. This means that even if you only do 128-bit work you can still benefit from the later feature levels. + +> By default, the `i686` and `x86_64` Rust targets enable `sse` and `sse2`. + +### Selecting Additional Target Features + +If you want to enable support for a target feature within your build, generally you should use a [target-feature](https://rust-lang.github.io/packed_simd/perf-guide/target-feature/rustflags.html#target-feature) setting within you `RUSTFLAGS` setting. + +If you know that you're targeting a specific CPU you can instead use the [target-cpu](https://rust-lang.github.io/packed_simd/perf-guide/target-feature/rustflags.html#target-cpu) flag and the compiler will enable the correct set of features for that CPU. + +The [Steam Hardware Survey](https://store.steampowered.com/hwsurvey/Steam-Hardware-Software-Survey-Welcome-to-Steam) is one of the few places with data on how common various CPU features are. The dataset is limited to "the kinds of computers owned by people who play computer games", so the info only covers `x86`/`x86_64`, and it also probably skews to slightly higher quality computers than average. Still, we can see that the `sse` levels have very high support, `avx` and `avx2` are quite common as well, and the `avx-512` family is still so early in adoption you can barely find it in consumer grade stuff. + +## Running a program compiled for a CPU feature level that the CPU doesn't support is automatic undefined behavior. + +This means that if you build your program with `avx` support enabled and run it on a CPU without `avx` support, it's **instantly** undefined behavior. + +Even without an `unsafe` block in sight. + +This is no bug in Rust, or soundness hole in the type system. You just plain can't make a CPU do what it doesn't know how to do. + +This is why the various Rust targets *don't* enable many CPU feature flags by default: requiring a more advanced CPU makes the final binary *less* portable. + +So please select an appropriate CPU feature level when building your programs. + +## Size, Alignment, and Unsafe Code + +Most of the portable SIMD API is designed to allow the user to gloss over the details of different architectures and avoid using unsafe code. However, there are plenty of reasons to want to use unsafe code with these SIMD types, such as using an intrinsic function from `core::arch` to further accelerate particularly specialized SIMD operations on a given platform, while still using the portable API elsewhere. For these cases, there are some rules to keep in mind. + +Fortunately, most SIMD types have a fairly predictable size. `i32x4` is bit-equivalent to `[i32; 4]` and so can be bitcast to it, e.g. using [`mem::transmute`], though the API usually offers a safe cast you can use instead. + +However, this is not the same as alignment. Computer architectures generally prefer aligned accesses, especially when moving data between memory and vector registers, and while some support specialized operations that can bend the rules to help with this, unaligned access is still typically slow, or even undefined behavior. In addition, different architectures can require different alignments when interacting with their native SIMD types. For this reason, any `#[repr(simd)]` type has a non-portable alignment. If it is necessary to directly interact with the alignment of these types, it should be via [`mem::align_of`]. + +When working with slices, data correctly aligned for SIMD can be acquired using the [`as_simd`] and [`as_simd_mut`] methods of the slice primitive. + +[`mem::transmute`]: https://doc.rust-lang.org/core/mem/fn.transmute.html +[`mem::align_of`]: https://doc.rust-lang.org/core/mem/fn.align_of.html +[`as_simd`]: https://doc.rust-lang.org/nightly/std/primitive.slice.html#method.as_simd +[`as_simd_mut`]: https://doc.rust-lang.org/nightly/std/primitive.slice.html#method.as_simd_mut + diff --git a/crux-mir/lib/portable-simd/crates/core_simd/Cargo.toml b/crux-mir/lib/portable-simd/crates/core_simd/Cargo.toml new file mode 100644 index 000000000..8a29cf156 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "core_simd" +version = "0.1.0" +edition = "2021" +homepage = "https://github.com/rust-lang/portable-simd" +repository = "https://github.com/rust-lang/portable-simd" +keywords = ["core", "simd", "intrinsics"] +categories = ["hardware-support", "no-std"] +license = "MIT OR Apache-2.0" + +[features] +default = ["as_crate"] +as_crate = [] +std = [] +generic_const_exprs = [] + +[target.'cfg(target_arch = "wasm32")'.dev-dependencies.wasm-bindgen] +version = "0.2" + +[dev-dependencies.wasm-bindgen-test] +version = "0.3" + +[dev-dependencies.proptest] +version = "0.10" +default-features = false +features = ["alloc"] + +[dev-dependencies.test_helpers] +path = "../test_helpers" + +[dev-dependencies] +std_float = { path = "../std_float/", features = ["as_crate"] } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/LICENSE-APACHE b/crux-mir/lib/portable-simd/crates/core_simd/LICENSE-APACHE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/LICENSE-APACHE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crux-mir/lib/portable-simd/crates/core_simd/LICENSE-MIT b/crux-mir/lib/portable-simd/crates/core_simd/LICENSE-MIT new file mode 100644 index 000000000..0e9d2f43a --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/LICENSE-MIT @@ -0,0 +1,19 @@ +Copyright (c) 2020 The Rust Project Developers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crux-mir/lib/portable-simd/crates/core_simd/examples/matrix_inversion.rs b/crux-mir/lib/portable-simd/crates/core_simd/examples/matrix_inversion.rs new file mode 100644 index 000000000..39f530f68 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/examples/matrix_inversion.rs @@ -0,0 +1,316 @@ +//! 4x4 matrix inverse +// Code ported from the `packed_simd` crate +// Run this code with `cargo test --example matrix_inversion` +#![feature(array_chunks, portable_simd)] +use core_simd::simd::*; +use Which::*; + +// Gotta define our own 4x4 matrix since Rust doesn't ship multidim arrays yet :^) +#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)] +pub struct Matrix4x4([[f32; 4]; 4]); + +#[allow(clippy::too_many_lines)] +pub fn scalar_inv4x4(m: Matrix4x4) -> Option { + let m = m.0; + + #[rustfmt::skip] + let mut inv = [ + // row 0: + [ + // 0,0: + m[1][1] * m[2][2] * m[3][3] - + m[1][1] * m[2][3] * m[3][2] - + m[2][1] * m[1][2] * m[3][3] + + m[2][1] * m[1][3] * m[3][2] + + m[3][1] * m[1][2] * m[2][3] - + m[3][1] * m[1][3] * m[2][2], + // 0,1: + -m[0][1] * m[2][2] * m[3][3] + + m[0][1] * m[2][3] * m[3][2] + + m[2][1] * m[0][2] * m[3][3] - + m[2][1] * m[0][3] * m[3][2] - + m[3][1] * m[0][2] * m[2][3] + + m[3][1] * m[0][3] * m[2][2], + // 0,2: + m[0][1] * m[1][2] * m[3][3] - + m[0][1] * m[1][3] * m[3][2] - + m[1][1] * m[0][2] * m[3][3] + + m[1][1] * m[0][3] * m[3][2] + + m[3][1] * m[0][2] * m[1][3] - + m[3][1] * m[0][3] * m[1][2], + // 0,3: + -m[0][1] * m[1][2] * m[2][3] + + m[0][1] * m[1][3] * m[2][2] + + m[1][1] * m[0][2] * m[2][3] - + m[1][1] * m[0][3] * m[2][2] - + m[2][1] * m[0][2] * m[1][3] + + m[2][1] * m[0][3] * m[1][2], + ], + // row 1 + [ + // 1,0: + -m[1][0] * m[2][2] * m[3][3] + + m[1][0] * m[2][3] * m[3][2] + + m[2][0] * m[1][2] * m[3][3] - + m[2][0] * m[1][3] * m[3][2] - + m[3][0] * m[1][2] * m[2][3] + + m[3][0] * m[1][3] * m[2][2], + // 1,1: + m[0][0] * m[2][2] * m[3][3] - + m[0][0] * m[2][3] * m[3][2] - + m[2][0] * m[0][2] * m[3][3] + + m[2][0] * m[0][3] * m[3][2] + + m[3][0] * m[0][2] * m[2][3] - + m[3][0] * m[0][3] * m[2][2], + // 1,2: + -m[0][0] * m[1][2] * m[3][3] + + m[0][0] * m[1][3] * m[3][2] + + m[1][0] * m[0][2] * m[3][3] - + m[1][0] * m[0][3] * m[3][2] - + m[3][0] * m[0][2] * m[1][3] + + m[3][0] * m[0][3] * m[1][2], + // 1,3: + m[0][0] * m[1][2] * m[2][3] - + m[0][0] * m[1][3] * m[2][2] - + m[1][0] * m[0][2] * m[2][3] + + m[1][0] * m[0][3] * m[2][2] + + m[2][0] * m[0][2] * m[1][3] - + m[2][0] * m[0][3] * m[1][2], + ], + // row 2 + [ + // 2,0: + m[1][0] * m[2][1] * m[3][3] - + m[1][0] * m[2][3] * m[3][1] - + m[2][0] * m[1][1] * m[3][3] + + m[2][0] * m[1][3] * m[3][1] + + m[3][0] * m[1][1] * m[2][3] - + m[3][0] * m[1][3] * m[2][1], + // 2,1: + -m[0][0] * m[2][1] * m[3][3] + + m[0][0] * m[2][3] * m[3][1] + + m[2][0] * m[0][1] * m[3][3] - + m[2][0] * m[0][3] * m[3][1] - + m[3][0] * m[0][1] * m[2][3] + + m[3][0] * m[0][3] * m[2][1], + // 2,2: + m[0][0] * m[1][1] * m[3][3] - + m[0][0] * m[1][3] * m[3][1] - + m[1][0] * m[0][1] * m[3][3] + + m[1][0] * m[0][3] * m[3][1] + + m[3][0] * m[0][1] * m[1][3] - + m[3][0] * m[0][3] * m[1][1], + // 2,3: + -m[0][0] * m[1][1] * m[2][3] + + m[0][0] * m[1][3] * m[2][1] + + m[1][0] * m[0][1] * m[2][3] - + m[1][0] * m[0][3] * m[2][1] - + m[2][0] * m[0][1] * m[1][3] + + m[2][0] * m[0][3] * m[1][1], + ], + // row 3 + [ + // 3,0: + -m[1][0] * m[2][1] * m[3][2] + + m[1][0] * m[2][2] * m[3][1] + + m[2][0] * m[1][1] * m[3][2] - + m[2][0] * m[1][2] * m[3][1] - + m[3][0] * m[1][1] * m[2][2] + + m[3][0] * m[1][2] * m[2][1], + // 3,1: + m[0][0] * m[2][1] * m[3][2] - + m[0][0] * m[2][2] * m[3][1] - + m[2][0] * m[0][1] * m[3][2] + + m[2][0] * m[0][2] * m[3][1] + + m[3][0] * m[0][1] * m[2][2] - + m[3][0] * m[0][2] * m[2][1], + // 3,2: + -m[0][0] * m[1][1] * m[3][2] + + m[0][0] * m[1][2] * m[3][1] + + m[1][0] * m[0][1] * m[3][2] - + m[1][0] * m[0][2] * m[3][1] - + m[3][0] * m[0][1] * m[1][2] + + m[3][0] * m[0][2] * m[1][1], + // 3,3: + m[0][0] * m[1][1] * m[2][2] - + m[0][0] * m[1][2] * m[2][1] - + m[1][0] * m[0][1] * m[2][2] + + m[1][0] * m[0][2] * m[2][1] + + m[2][0] * m[0][1] * m[1][2] - + m[2][0] * m[0][2] * m[1][1], + ], + ]; + + let det = m[0][0] * inv[0][0] + m[0][1] * inv[1][0] + m[0][2] * inv[2][0] + m[0][3] * inv[3][0]; + if det == 0. { + return None; + } + + let det_inv = 1. / det; + + for row in &mut inv { + for elem in row.iter_mut() { + *elem *= det_inv; + } + } + + Some(Matrix4x4(inv)) +} + +pub fn simd_inv4x4(m: Matrix4x4) -> Option { + let m = m.0; + let m_0 = f32x4::from_array(m[0]); + let m_1 = f32x4::from_array(m[1]); + let m_2 = f32x4::from_array(m[2]); + let m_3 = f32x4::from_array(m[3]); + + const SHUFFLE01: [Which; 4] = [First(0), First(1), Second(0), Second(1)]; + const SHUFFLE02: [Which; 4] = [First(0), First(2), Second(0), Second(2)]; + const SHUFFLE13: [Which; 4] = [First(1), First(3), Second(1), Second(3)]; + const SHUFFLE23: [Which; 4] = [First(2), First(3), Second(2), Second(3)]; + + let tmp = simd_swizzle!(m_0, m_1, SHUFFLE01); + let row1 = simd_swizzle!(m_2, m_3, SHUFFLE01); + + let row0 = simd_swizzle!(tmp, row1, SHUFFLE02); + let row1 = simd_swizzle!(row1, tmp, SHUFFLE13); + + let tmp = simd_swizzle!(m_0, m_1, SHUFFLE23); + let row3 = simd_swizzle!(m_2, m_3, SHUFFLE23); + let row2 = simd_swizzle!(tmp, row3, SHUFFLE02); + let row3 = simd_swizzle!(row3, tmp, SHUFFLE13); + + let tmp = (row2 * row3).reverse().rotate_lanes_right::<2>(); + let minor0 = row1 * tmp; + let minor1 = row0 * tmp; + let tmp = tmp.rotate_lanes_right::<2>(); + let minor0 = (row1 * tmp) - minor0; + let minor1 = (row0 * tmp) - minor1; + let minor1 = minor1.rotate_lanes_right::<2>(); + + let tmp = (row1 * row2).reverse().rotate_lanes_right::<2>(); + let minor0 = (row3 * tmp) + minor0; + let minor3 = row0 * tmp; + let tmp = tmp.rotate_lanes_right::<2>(); + + let minor0 = minor0 - row3 * tmp; + let minor3 = row0 * tmp - minor3; + let minor3 = minor3.rotate_lanes_right::<2>(); + + let tmp = (row3 * row1.rotate_lanes_right::<2>()) + .reverse() + .rotate_lanes_right::<2>(); + let row2 = row2.rotate_lanes_right::<2>(); + let minor0 = row2 * tmp + minor0; + let minor2 = row0 * tmp; + let tmp = tmp.rotate_lanes_right::<2>(); + let minor0 = minor0 - row2 * tmp; + let minor2 = row0 * tmp - minor2; + let minor2 = minor2.rotate_lanes_right::<2>(); + + let tmp = (row0 * row1).reverse().rotate_lanes_right::<2>(); + let minor2 = minor2 + row3 * tmp; + let minor3 = row2 * tmp - minor3; + let tmp = tmp.rotate_lanes_right::<2>(); + let minor2 = row3 * tmp - minor2; + let minor3 = minor3 - row2 * tmp; + + let tmp = (row0 * row3).reverse().rotate_lanes_right::<2>(); + let minor1 = minor1 - row2 * tmp; + let minor2 = row1 * tmp + minor2; + let tmp = tmp.rotate_lanes_right::<2>(); + let minor1 = row2 * tmp + minor1; + let minor2 = minor2 - row1 * tmp; + + let tmp = (row0 * row2).reverse().rotate_lanes_right::<2>(); + let minor1 = row3 * tmp + minor1; + let minor3 = minor3 - row1 * tmp; + let tmp = tmp.rotate_lanes_right::<2>(); + let minor1 = minor1 - row3 * tmp; + let minor3 = row1 * tmp + minor3; + + let det = row0 * minor0; + let det = det.rotate_lanes_right::<2>() + det; + let det = det.reverse().rotate_lanes_right::<2>() + det; + + if det.reduce_sum() == 0. { + return None; + } + // calculate the reciprocal + let tmp = f32x4::splat(1.0) / det; + let det = tmp + tmp - det * tmp * tmp; + + let res0 = minor0 * det; + let res1 = minor1 * det; + let res2 = minor2 * det; + let res3 = minor3 * det; + + let mut m = m; + + m[0] = res0.to_array(); + m[1] = res1.to_array(); + m[2] = res2.to_array(); + m[3] = res3.to_array(); + + Some(Matrix4x4(m)) +} + +#[cfg(test)] +#[rustfmt::skip] +mod tests { + use super::*; + + #[test] + fn test() { + let tests: &[(Matrix4x4, Option)] = &[ + // Identity: + (Matrix4x4([ + [1., 0., 0., 0.], + [0., 1., 0., 0.], + [0., 0., 1., 0.], + [0., 0., 0., 1.], + ]), + Some(Matrix4x4([ + [1., 0., 0., 0.], + [0., 1., 0., 0.], + [0., 0., 1., 0.], + [0., 0., 0., 1.], + ])) + ), + // None: + (Matrix4x4([ + [1., 2., 3., 4.], + [12., 11., 10., 9.], + [5., 6., 7., 8.], + [16., 15., 14., 13.], + ]), + None + ), + // Other: + (Matrix4x4([ + [1., 1., 1., 0.], + [0., 3., 1., 2.], + [2., 3., 1., 0.], + [1., 0., 2., 1.], + ]), + Some(Matrix4x4([ + [-3., -0.5, 1.5, 1.0], + [ 1., 0.25, -0.25, -0.5], + [ 3., 0.25, -1.25, -0.5], + [-3., 0.0, 1.0, 1.0], + ])) + ), + + + ]; + + for &(input, output) in tests { + assert_eq!(scalar_inv4x4(input), output); + assert_eq!(simd_inv4x4(input), output); + } + } +} + +fn main() { + // Empty main to make cargo happy +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/examples/nbody.rs b/crux-mir/lib/portable-simd/crates/core_simd/examples/nbody.rs new file mode 100644 index 000000000..df38a0096 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/examples/nbody.rs @@ -0,0 +1,193 @@ +#![feature(portable_simd)] +extern crate std_float; + +/// Benchmarks game nbody code +/// Taken from the `packed_simd` crate +/// Run this benchmark with `cargo test --example nbody` +mod nbody { + use core_simd::simd::*; + #[allow(unused)] // False positive? + use std_float::StdFloat; + + use std::f64::consts::PI; + const SOLAR_MASS: f64 = 4.0 * PI * PI; + const DAYS_PER_YEAR: f64 = 365.24; + + #[derive(Debug, Clone, Copy)] + struct Body { + pub x: f64x4, + pub v: f64x4, + pub mass: f64, + } + + const N_BODIES: usize = 5; + const BODIES: [Body; N_BODIES] = [ + // sun: + Body { + x: f64x4::from_array([0., 0., 0., 0.]), + v: f64x4::from_array([0., 0., 0., 0.]), + mass: SOLAR_MASS, + }, + // jupiter: + Body { + x: f64x4::from_array([ + 4.84143144246472090e+00, + -1.16032004402742839e+00, + -1.03622044471123109e-01, + 0., + ]), + v: f64x4::from_array([ + 1.66007664274403694e-03 * DAYS_PER_YEAR, + 7.69901118419740425e-03 * DAYS_PER_YEAR, + -6.90460016972063023e-05 * DAYS_PER_YEAR, + 0., + ]), + mass: 9.54791938424326609e-04 * SOLAR_MASS, + }, + // saturn: + Body { + x: f64x4::from_array([ + 8.34336671824457987e+00, + 4.12479856412430479e+00, + -4.03523417114321381e-01, + 0., + ]), + v: f64x4::from_array([ + -2.76742510726862411e-03 * DAYS_PER_YEAR, + 4.99852801234917238e-03 * DAYS_PER_YEAR, + 2.30417297573763929e-05 * DAYS_PER_YEAR, + 0., + ]), + mass: 2.85885980666130812e-04 * SOLAR_MASS, + }, + // uranus: + Body { + x: f64x4::from_array([ + 1.28943695621391310e+01, + -1.51111514016986312e+01, + -2.23307578892655734e-01, + 0., + ]), + v: f64x4::from_array([ + 2.96460137564761618e-03 * DAYS_PER_YEAR, + 2.37847173959480950e-03 * DAYS_PER_YEAR, + -2.96589568540237556e-05 * DAYS_PER_YEAR, + 0., + ]), + mass: 4.36624404335156298e-05 * SOLAR_MASS, + }, + // neptune: + Body { + x: f64x4::from_array([ + 1.53796971148509165e+01, + -2.59193146099879641e+01, + 1.79258772950371181e-01, + 0., + ]), + v: f64x4::from_array([ + 2.68067772490389322e-03 * DAYS_PER_YEAR, + 1.62824170038242295e-03 * DAYS_PER_YEAR, + -9.51592254519715870e-05 * DAYS_PER_YEAR, + 0., + ]), + mass: 5.15138902046611451e-05 * SOLAR_MASS, + }, + ]; + + fn offset_momentum(bodies: &mut [Body; N_BODIES]) { + let (sun, rest) = bodies.split_at_mut(1); + let sun = &mut sun[0]; + for body in rest { + let m_ratio = body.mass / SOLAR_MASS; + sun.v -= body.v * Simd::splat(m_ratio); + } + } + + fn energy(bodies: &[Body; N_BODIES]) -> f64 { + let mut e = 0.; + for i in 0..N_BODIES { + let bi = &bodies[i]; + e += bi.mass * (bi.v * bi.v).reduce_sum() * 0.5; + for bj in bodies.iter().take(N_BODIES).skip(i + 1) { + let dx = bi.x - bj.x; + e -= bi.mass * bj.mass / (dx * dx).reduce_sum().sqrt() + } + } + e + } + + fn advance(bodies: &mut [Body; N_BODIES], dt: f64) { + const N: usize = N_BODIES * (N_BODIES - 1) / 2; + + // compute distance between bodies: + let mut r = [f64x4::splat(0.); N]; + { + let mut i = 0; + for j in 0..N_BODIES { + for k in j + 1..N_BODIES { + r[i] = bodies[j].x - bodies[k].x; + i += 1; + } + } + } + + let mut mag = [0.0; N]; + for i in (0..N).step_by(2) { + let d2s = f64x2::from_array([ + (r[i] * r[i]).reduce_sum(), + (r[i + 1] * r[i + 1]).reduce_sum(), + ]); + let dmags = f64x2::splat(dt) / (d2s * d2s.sqrt()); + mag[i] = dmags[0]; + mag[i + 1] = dmags[1]; + } + + let mut i = 0; + for j in 0..N_BODIES { + for k in j + 1..N_BODIES { + let f = r[i] * Simd::splat(mag[i]); + bodies[j].v -= f * Simd::splat(bodies[k].mass); + bodies[k].v += f * Simd::splat(bodies[j].mass); + i += 1 + } + } + for body in bodies { + body.x += Simd::splat(dt) * body.v + } + } + + pub fn run(n: usize) -> (f64, f64) { + let mut bodies = BODIES; + offset_momentum(&mut bodies); + let energy_before = energy(&bodies); + for _ in 0..n { + advance(&mut bodies, 0.01); + } + let energy_after = energy(&bodies); + + (energy_before, energy_after) + } +} + +#[cfg(test)] +mod tests { + // Good enough for demonstration purposes, not going for strictness here. + fn approx_eq_f64(a: f64, b: f64) -> bool { + (a - b).abs() < 0.00001 + } + #[test] + fn test() { + const OUTPUT: [f64; 2] = [-0.169075164, -0.169087605]; + let (energy_before, energy_after) = super::nbody::run(1000); + assert!(approx_eq_f64(energy_before, OUTPUT[0])); + assert!(approx_eq_f64(energy_after, OUTPUT[1])); + } +} + +fn main() { + { + let (energy_before, energy_after) = nbody::run(1000); + println!("Energy before: {energy_before}"); + println!("Energy after: {energy_after}"); + } +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/examples/spectral_norm.rs b/crux-mir/lib/portable-simd/crates/core_simd/examples/spectral_norm.rs new file mode 100644 index 000000000..d576bd0cc --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/examples/spectral_norm.rs @@ -0,0 +1,77 @@ +#![feature(portable_simd)] + +use core_simd::simd::*; + +fn a(i: usize, j: usize) -> f64 { + ((i + j) * (i + j + 1) / 2 + i + 1) as f64 +} + +fn mult_av(v: &[f64], out: &mut [f64]) { + assert!(v.len() == out.len()); + assert!(v.len() % 2 == 0); + + for (i, out) in out.iter_mut().enumerate() { + let mut sum = f64x2::splat(0.0); + + let mut j = 0; + while j < v.len() { + let b = f64x2::from_slice(&v[j..]); + let a = f64x2::from_array([a(i, j), a(i, j + 1)]); + sum += b / a; + j += 2 + } + *out = sum.reduce_sum(); + } +} + +fn mult_atv(v: &[f64], out: &mut [f64]) { + assert!(v.len() == out.len()); + assert!(v.len() % 2 == 0); + + for (i, out) in out.iter_mut().enumerate() { + let mut sum = f64x2::splat(0.0); + + let mut j = 0; + while j < v.len() { + let b = f64x2::from_slice(&v[j..]); + let a = f64x2::from_array([a(j, i), a(j + 1, i)]); + sum += b / a; + j += 2 + } + *out = sum.reduce_sum(); + } +} + +fn mult_atav(v: &[f64], out: &mut [f64], tmp: &mut [f64]) { + mult_av(v, tmp); + mult_atv(tmp, out); +} + +pub fn spectral_norm(n: usize) -> f64 { + assert!(n % 2 == 0, "only even lengths are accepted"); + + let mut u = vec![1.0; n]; + let mut v = u.clone(); + let mut tmp = u.clone(); + + for _ in 0..10 { + mult_atav(&u, &mut v, &mut tmp); + mult_atav(&v, &mut u, &mut tmp); + } + (dot(&u, &v) / dot(&v, &v)).sqrt() +} + +fn dot(x: &[f64], y: &[f64]) -> f64 { + // This is auto-vectorized: + x.iter().zip(y).map(|(&x, &y)| x * y).sum() +} + +#[cfg(test)] +#[test] +fn test() { + assert_eq!(format!("{:.9}", spectral_norm(100)), "1.274219991"); +} + +fn main() { + // Empty main to make cargo happy +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/core_simd_docs.md b/crux-mir/lib/portable-simd/crates/core_simd/src/core_simd_docs.md new file mode 100644 index 000000000..15e8ed025 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/core_simd_docs.md @@ -0,0 +1,4 @@ +Portable SIMD module. + +This module offers a portable abstraction for SIMD operations +that is not bound to any particular hardware architecture. diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/elements.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/elements.rs new file mode 100644 index 000000000..701eb66b2 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/elements.rs @@ -0,0 +1,11 @@ +mod float; +mod int; +mod uint; + +mod sealed { + pub trait Sealed {} +} + +pub use float::*; +pub use int::*; +pub use uint::*; diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/elements/float.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/elements/float.rs new file mode 100644 index 000000000..d60223270 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/elements/float.rs @@ -0,0 +1,357 @@ +use super::sealed::Sealed; +use crate::simd::{ + intrinsics, LaneCount, Mask, Simd, SimdElement, SimdPartialEq, SimdPartialOrd, + SupportedLaneCount, +}; + +/// Operations on SIMD vectors of floats. +pub trait SimdFloat: Copy + Sealed { + /// Mask type used for manipulating this SIMD vector type. + type Mask; + + /// Scalar type contained by this SIMD vector type. + type Scalar; + + /// Bit representation of this SIMD vector type. + type Bits; + + /// Raw transmutation to an unsigned integer vector type with the + /// same size and number of lanes. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn to_bits(self) -> Self::Bits; + + /// Raw transmutation from an unsigned integer vector type with the + /// same size and number of lanes. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn from_bits(bits: Self::Bits) -> Self; + + /// Produces a vector where every lane has the absolute value of the + /// equivalently-indexed lane in `self`. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn abs(self) -> Self; + + /// Takes the reciprocal (inverse) of each lane, `1/x`. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn recip(self) -> Self; + + /// Converts each lane from radians to degrees. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn to_degrees(self) -> Self; + + /// Converts each lane from degrees to radians. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn to_radians(self) -> Self; + + /// Returns true for each lane if it has a positive sign, including + /// `+0.0`, `NaN`s with positive sign bit and positive infinity. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_sign_positive(self) -> Self::Mask; + + /// Returns true for each lane if it has a negative sign, including + /// `-0.0`, `NaN`s with negative sign bit and negative infinity. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_sign_negative(self) -> Self::Mask; + + /// Returns true for each lane if its value is `NaN`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_nan(self) -> Self::Mask; + + /// Returns true for each lane if its value is positive infinity or negative infinity. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_infinite(self) -> Self::Mask; + + /// Returns true for each lane if its value is neither infinite nor `NaN`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_finite(self) -> Self::Mask; + + /// Returns true for each lane if its value is subnormal. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_subnormal(self) -> Self::Mask; + + /// Returns true for each lane if its value is neither zero, infinite, + /// subnormal, nor `NaN`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_normal(self) -> Self::Mask; + + /// Replaces each lane with a number that represents its sign. + /// + /// * `1.0` if the number is positive, `+0.0`, or `INFINITY` + /// * `-1.0` if the number is negative, `-0.0`, or `NEG_INFINITY` + /// * `NAN` if the number is `NAN` + #[must_use = "method returns a new vector and does not mutate the original value"] + fn signum(self) -> Self; + + /// Returns each lane with the magnitude of `self` and the sign of `sign`. + /// + /// For any lane containing a `NAN`, a `NAN` with the sign of `sign` is returned. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn copysign(self, sign: Self) -> Self; + + /// Returns the minimum of each lane. + /// + /// If one of the values is `NAN`, then the other value is returned. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn simd_min(self, other: Self) -> Self; + + /// Returns the maximum of each lane. + /// + /// If one of the values is `NAN`, then the other value is returned. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn simd_max(self, other: Self) -> Self; + + /// Restrict each lane to a certain interval unless it is NaN. + /// + /// For each lane in `self`, returns the corresponding lane in `max` if the lane is + /// greater than `max`, and the corresponding lane in `min` if the lane is less + /// than `min`. Otherwise returns the lane in `self`. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn simd_clamp(self, min: Self, max: Self) -> Self; + + /// Returns the sum of the lanes of the vector. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{f32x2, SimdFloat}; + /// let v = f32x2::from_array([1., 2.]); + /// assert_eq!(v.reduce_sum(), 3.); + /// ``` + fn reduce_sum(self) -> Self::Scalar; + + /// Reducing multiply. Returns the product of the lanes of the vector. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{f32x2, SimdFloat}; + /// let v = f32x2::from_array([3., 4.]); + /// assert_eq!(v.reduce_product(), 12.); + /// ``` + fn reduce_product(self) -> Self::Scalar; + + /// Returns the maximum lane in the vector. + /// + /// Returns values based on equality, so a vector containing both `0.` and `-0.` may + /// return either. + /// + /// This function will not return `NaN` unless all lanes are `NaN`. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{f32x2, SimdFloat}; + /// let v = f32x2::from_array([1., 2.]); + /// assert_eq!(v.reduce_max(), 2.); + /// + /// // NaN values are skipped... + /// let v = f32x2::from_array([1., f32::NAN]); + /// assert_eq!(v.reduce_max(), 1.); + /// + /// // ...unless all values are NaN + /// let v = f32x2::from_array([f32::NAN, f32::NAN]); + /// assert!(v.reduce_max().is_nan()); + /// ``` + fn reduce_max(self) -> Self::Scalar; + + /// Returns the minimum lane in the vector. + /// + /// Returns values based on equality, so a vector containing both `0.` and `-0.` may + /// return either. + /// + /// This function will not return `NaN` unless all lanes are `NaN`. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{f32x2, SimdFloat}; + /// let v = f32x2::from_array([3., 7.]); + /// assert_eq!(v.reduce_min(), 3.); + /// + /// // NaN values are skipped... + /// let v = f32x2::from_array([1., f32::NAN]); + /// assert_eq!(v.reduce_min(), 1.); + /// + /// // ...unless all values are NaN + /// let v = f32x2::from_array([f32::NAN, f32::NAN]); + /// assert!(v.reduce_min().is_nan()); + /// ``` + fn reduce_min(self) -> Self::Scalar; +} + +macro_rules! impl_trait { + { $($ty:ty { bits: $bits_ty:ty, mask: $mask_ty:ty }),* } => { + $( + impl Sealed for Simd<$ty, LANES> + where + LaneCount: SupportedLaneCount, + { + } + + impl SimdFloat for Simd<$ty, LANES> + where + LaneCount: SupportedLaneCount, + { + type Mask = Mask<<$mask_ty as SimdElement>::Mask, LANES>; + type Scalar = $ty; + type Bits = Simd<$bits_ty, LANES>; + + #[inline] + fn to_bits(self) -> Simd<$bits_ty, LANES> { + assert_eq!(core::mem::size_of::(), core::mem::size_of::()); + // Safety: transmuting between vector types is safe + unsafe { core::mem::transmute_copy(&self) } + } + + #[inline] + fn from_bits(bits: Simd<$bits_ty, LANES>) -> Self { + assert_eq!(core::mem::size_of::(), core::mem::size_of::()); + // Safety: transmuting between vector types is safe + unsafe { core::mem::transmute_copy(&bits) } + } + + #[inline] + fn abs(self) -> Self { + // Safety: `self` is a float vector + unsafe { intrinsics::simd_fabs(self) } + } + + #[inline] + fn recip(self) -> Self { + Self::splat(1.0) / self + } + + #[inline] + fn to_degrees(self) -> Self { + // to_degrees uses a special constant for better precision, so extract that constant + self * Self::splat(Self::Scalar::to_degrees(1.)) + } + + #[inline] + fn to_radians(self) -> Self { + self * Self::splat(Self::Scalar::to_radians(1.)) + } + + #[inline] + fn is_sign_positive(self) -> Self::Mask { + !self.is_sign_negative() + } + + #[inline] + fn is_sign_negative(self) -> Self::Mask { + let sign_bits = self.to_bits() & Simd::splat((!0 >> 1) + 1); + sign_bits.simd_gt(Simd::splat(0)) + } + + #[inline] + fn is_nan(self) -> Self::Mask { + self.simd_ne(self) + } + + #[inline] + fn is_infinite(self) -> Self::Mask { + self.abs().simd_eq(Self::splat(Self::Scalar::INFINITY)) + } + + #[inline] + fn is_finite(self) -> Self::Mask { + self.abs().simd_lt(Self::splat(Self::Scalar::INFINITY)) + } + + #[inline] + fn is_subnormal(self) -> Self::Mask { + self.abs().simd_ne(Self::splat(0.0)) & (self.to_bits() & Self::splat(Self::Scalar::INFINITY).to_bits()).simd_eq(Simd::splat(0)) + } + + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_normal(self) -> Self::Mask { + !(self.abs().simd_eq(Self::splat(0.0)) | self.is_nan() | self.is_subnormal() | self.is_infinite()) + } + + #[inline] + fn signum(self) -> Self { + self.is_nan().select(Self::splat(Self::Scalar::NAN), Self::splat(1.0).copysign(self)) + } + + #[inline] + fn copysign(self, sign: Self) -> Self { + let sign_bit = sign.to_bits() & Self::splat(-0.).to_bits(); + let magnitude = self.to_bits() & !Self::splat(-0.).to_bits(); + Self::from_bits(sign_bit | magnitude) + } + + #[inline] + fn simd_min(self, other: Self) -> Self { + // Safety: `self` and `other` are float vectors + unsafe { intrinsics::simd_fmin(self, other) } + } + + #[inline] + fn simd_max(self, other: Self) -> Self { + // Safety: `self` and `other` are floating point vectors + unsafe { intrinsics::simd_fmax(self, other) } + } + + #[inline] + fn simd_clamp(self, min: Self, max: Self) -> Self { + assert!( + min.simd_le(max).all(), + "each lane in `min` must be less than or equal to the corresponding lane in `max`", + ); + let mut x = self; + x = x.simd_lt(min).select(min, x); + x = x.simd_gt(max).select(max, x); + x + } + + #[inline] + fn reduce_sum(self) -> Self::Scalar { + // LLVM sum is inaccurate on i586 + if cfg!(all(target_arch = "x86", not(target_feature = "sse2"))) { + self.as_array().iter().sum() + } else { + // Safety: `self` is a float vector + unsafe { intrinsics::simd_reduce_add_ordered(self, 0.) } + } + } + + #[inline] + fn reduce_product(self) -> Self::Scalar { + // LLVM product is inaccurate on i586 + if cfg!(all(target_arch = "x86", not(target_feature = "sse2"))) { + self.as_array().iter().product() + } else { + // Safety: `self` is a float vector + unsafe { intrinsics::simd_reduce_mul_ordered(self, 1.) } + } + } + + #[inline] + fn reduce_max(self) -> Self::Scalar { + // Safety: `self` is a float vector + unsafe { intrinsics::simd_reduce_max(self) } + } + + #[inline] + fn reduce_min(self) -> Self::Scalar { + // Safety: `self` is a float vector + unsafe { intrinsics::simd_reduce_min(self) } + } + } + )* + } +} + +impl_trait! { f32 { bits: u32, mask: i32 }, f64 { bits: u64, mask: i64 } } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/elements/int.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/elements/int.rs new file mode 100644 index 000000000..9b8c37ed4 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/elements/int.rs @@ -0,0 +1,298 @@ +use super::sealed::Sealed; +use crate::simd::{ + intrinsics, LaneCount, Mask, Simd, SimdElement, SimdPartialOrd, SupportedLaneCount, +}; + +/// Operations on SIMD vectors of signed integers. +pub trait SimdInt: Copy + Sealed { + /// Mask type used for manipulating this SIMD vector type. + type Mask; + + /// Scalar type contained by this SIMD vector type. + type Scalar; + + /// Lanewise saturating add. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, SimdInt}; + /// use core::i32::{MIN, MAX}; + /// let x = Simd::from_array([MIN, 0, 1, MAX]); + /// let max = Simd::splat(MAX); + /// let unsat = x + max; + /// let sat = x.saturating_add(max); + /// assert_eq!(unsat, Simd::from_array([-1, MAX, MIN, -2])); + /// assert_eq!(sat, Simd::from_array([-1, MAX, MAX, MAX])); + /// ``` + fn saturating_add(self, second: Self) -> Self; + + /// Lanewise saturating subtract. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, SimdInt}; + /// use core::i32::{MIN, MAX}; + /// let x = Simd::from_array([MIN, -2, -1, MAX]); + /// let max = Simd::splat(MAX); + /// let unsat = x - max; + /// let sat = x.saturating_sub(max); + /// assert_eq!(unsat, Simd::from_array([1, MAX, MIN, 0])); + /// assert_eq!(sat, Simd::from_array([MIN, MIN, MIN, 0])); + fn saturating_sub(self, second: Self) -> Self; + + /// Lanewise absolute value, implemented in Rust. + /// Every lane becomes its absolute value. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, SimdInt}; + /// use core::i32::{MIN, MAX}; + /// let xs = Simd::from_array([MIN, MIN +1, -5, 0]); + /// assert_eq!(xs.abs(), Simd::from_array([MIN, MAX, 5, 0])); + /// ``` + fn abs(self) -> Self; + + /// Lanewise saturating absolute value, implemented in Rust. + /// As abs(), except the MIN value becomes MAX instead of itself. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, SimdInt}; + /// use core::i32::{MIN, MAX}; + /// let xs = Simd::from_array([MIN, -2, 0, 3]); + /// let unsat = xs.abs(); + /// let sat = xs.saturating_abs(); + /// assert_eq!(unsat, Simd::from_array([MIN, 2, 0, 3])); + /// assert_eq!(sat, Simd::from_array([MAX, 2, 0, 3])); + /// ``` + fn saturating_abs(self) -> Self; + + /// Lanewise saturating negation, implemented in Rust. + /// As neg(), except the MIN value becomes MAX instead of itself. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, SimdInt}; + /// use core::i32::{MIN, MAX}; + /// let x = Simd::from_array([MIN, -2, 3, MAX]); + /// let unsat = -x; + /// let sat = x.saturating_neg(); + /// assert_eq!(unsat, Simd::from_array([MIN, 2, -3, MIN + 1])); + /// assert_eq!(sat, Simd::from_array([MAX, 2, -3, MIN + 1])); + /// ``` + fn saturating_neg(self) -> Self; + + /// Returns true for each positive lane and false if it is zero or negative. + fn is_positive(self) -> Self::Mask; + + /// Returns true for each negative lane and false if it is zero or positive. + fn is_negative(self) -> Self::Mask; + + /// Returns numbers representing the sign of each lane. + /// * `0` if the number is zero + /// * `1` if the number is positive + /// * `-1` if the number is negative + fn signum(self) -> Self; + + /// Returns the sum of the lanes of the vector, with wrapping addition. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{i32x4, SimdInt}; + /// let v = i32x4::from_array([1, 2, 3, 4]); + /// assert_eq!(v.reduce_sum(), 10); + /// + /// // SIMD integer addition is always wrapping + /// let v = i32x4::from_array([i32::MAX, 1, 0, 0]); + /// assert_eq!(v.reduce_sum(), i32::MIN); + /// ``` + fn reduce_sum(self) -> Self::Scalar; + + /// Returns the product of the lanes of the vector, with wrapping multiplication. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{i32x4, SimdInt}; + /// let v = i32x4::from_array([1, 2, 3, 4]); + /// assert_eq!(v.reduce_product(), 24); + /// + /// // SIMD integer multiplication is always wrapping + /// let v = i32x4::from_array([i32::MAX, 2, 1, 1]); + /// assert!(v.reduce_product() < i32::MAX); + /// ``` + fn reduce_product(self) -> Self::Scalar; + + /// Returns the maximum lane in the vector. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{i32x4, SimdInt}; + /// let v = i32x4::from_array([1, 2, 3, 4]); + /// assert_eq!(v.reduce_max(), 4); + /// ``` + fn reduce_max(self) -> Self::Scalar; + + /// Returns the minimum lane in the vector. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{i32x4, SimdInt}; + /// let v = i32x4::from_array([1, 2, 3, 4]); + /// assert_eq!(v.reduce_min(), 1); + /// ``` + fn reduce_min(self) -> Self::Scalar; + + /// Returns the cumulative bitwise "and" across the lanes of the vector. + fn reduce_and(self) -> Self::Scalar; + + /// Returns the cumulative bitwise "or" across the lanes of the vector. + fn reduce_or(self) -> Self::Scalar; + + /// Returns the cumulative bitwise "xor" across the lanes of the vector. + fn reduce_xor(self) -> Self::Scalar; +} + +macro_rules! impl_trait { + { $($ty:ty),* } => { + $( + impl Sealed for Simd<$ty, LANES> + where + LaneCount: SupportedLaneCount, + { + } + + impl SimdInt for Simd<$ty, LANES> + where + LaneCount: SupportedLaneCount, + { + type Mask = Mask<<$ty as SimdElement>::Mask, LANES>; + type Scalar = $ty; + + #[inline] + fn saturating_add(self, second: Self) -> Self { + // Safety: `self` is a vector + unsafe { intrinsics::simd_saturating_add(self, second) } + } + + #[inline] + fn saturating_sub(self, second: Self) -> Self { + // Safety: `self` is a vector + unsafe { intrinsics::simd_saturating_sub(self, second) } + } + + #[inline] + fn abs(self) -> Self { + const SHR: $ty = <$ty>::BITS as $ty - 1; + let m = self >> Simd::splat(SHR); + (self^m) - m + } + + #[inline] + fn saturating_abs(self) -> Self { + // arith shift for -1 or 0 mask based on sign bit, giving 2s complement + const SHR: $ty = <$ty>::BITS as $ty - 1; + let m = self >> Simd::splat(SHR); + (self^m).saturating_sub(m) + } + + #[inline] + fn saturating_neg(self) -> Self { + Self::splat(0).saturating_sub(self) + } + + #[inline] + fn is_positive(self) -> Self::Mask { + self.simd_gt(Self::splat(0)) + } + + #[inline] + fn is_negative(self) -> Self::Mask { + self.simd_lt(Self::splat(0)) + } + + #[inline] + fn signum(self) -> Self { + self.is_positive().select( + Self::splat(1), + self.is_negative().select(Self::splat(-1), Self::splat(0)) + ) + } + + #[inline] + fn reduce_sum(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_add_ordered(self, 0) } + } + + #[inline] + fn reduce_product(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_mul_ordered(self, 1) } + } + + #[inline] + fn reduce_max(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_max(self) } + } + + #[inline] + fn reduce_min(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_min(self) } + } + + #[inline] + fn reduce_and(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_and(self) } + } + + #[inline] + fn reduce_or(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_or(self) } + } + + #[inline] + fn reduce_xor(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_xor(self) } + } + } + )* + } +} + +impl_trait! { i8, i16, i32, i64, isize } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/elements/uint.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/elements/uint.rs new file mode 100644 index 000000000..21e7e76eb --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/elements/uint.rs @@ -0,0 +1,139 @@ +use super::sealed::Sealed; +use crate::simd::{intrinsics, LaneCount, Simd, SupportedLaneCount}; + +/// Operations on SIMD vectors of unsigned integers. +pub trait SimdUint: Copy + Sealed { + /// Scalar type contained by this SIMD vector type. + type Scalar; + + /// Lanewise saturating add. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, SimdUint}; + /// use core::u32::MAX; + /// let x = Simd::from_array([2, 1, 0, MAX]); + /// let max = Simd::splat(MAX); + /// let unsat = x + max; + /// let sat = x.saturating_add(max); + /// assert_eq!(unsat, Simd::from_array([1, 0, MAX, MAX - 1])); + /// assert_eq!(sat, max); + /// ``` + fn saturating_add(self, second: Self) -> Self; + + /// Lanewise saturating subtract. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, SimdUint}; + /// use core::u32::MAX; + /// let x = Simd::from_array([2, 1, 0, MAX]); + /// let max = Simd::splat(MAX); + /// let unsat = x - max; + /// let sat = x.saturating_sub(max); + /// assert_eq!(unsat, Simd::from_array([3, 2, 1, 0])); + /// assert_eq!(sat, Simd::splat(0)); + fn saturating_sub(self, second: Self) -> Self; + + /// Returns the sum of the lanes of the vector, with wrapping addition. + fn reduce_sum(self) -> Self::Scalar; + + /// Returns the product of the lanes of the vector, with wrapping multiplication. + fn reduce_product(self) -> Self::Scalar; + + /// Returns the maximum lane in the vector. + fn reduce_max(self) -> Self::Scalar; + + /// Returns the minimum lane in the vector. + fn reduce_min(self) -> Self::Scalar; + + /// Returns the cumulative bitwise "and" across the lanes of the vector. + fn reduce_and(self) -> Self::Scalar; + + /// Returns the cumulative bitwise "or" across the lanes of the vector. + fn reduce_or(self) -> Self::Scalar; + + /// Returns the cumulative bitwise "xor" across the lanes of the vector. + fn reduce_xor(self) -> Self::Scalar; +} + +macro_rules! impl_trait { + { $($ty:ty),* } => { + $( + impl Sealed for Simd<$ty, LANES> + where + LaneCount: SupportedLaneCount, + { + } + + impl SimdUint for Simd<$ty, LANES> + where + LaneCount: SupportedLaneCount, + { + type Scalar = $ty; + + #[inline] + fn saturating_add(self, second: Self) -> Self { + // Safety: `self` is a vector + unsafe { intrinsics::simd_saturating_add(self, second) } + } + + #[inline] + fn saturating_sub(self, second: Self) -> Self { + // Safety: `self` is a vector + unsafe { intrinsics::simd_saturating_sub(self, second) } + } + + #[inline] + fn reduce_sum(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_add_ordered(self, 0) } + } + + #[inline] + fn reduce_product(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_mul_ordered(self, 1) } + } + + #[inline] + fn reduce_max(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_max(self) } + } + + #[inline] + fn reduce_min(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_min(self) } + } + + #[inline] + fn reduce_and(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_and(self) } + } + + #[inline] + fn reduce_or(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_or(self) } + } + + #[inline] + fn reduce_xor(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_xor(self) } + } + } + )* + } +} + +impl_trait! { u8, u16, u32, u64, usize } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/eq.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/eq.rs new file mode 100644 index 000000000..c7111f720 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/eq.rs @@ -0,0 +1,73 @@ +use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdElement, SupportedLaneCount}; + +/// Parallel `PartialEq`. +pub trait SimdPartialEq { + /// The mask type returned by each comparison. + type Mask; + + /// Test if each lane is equal to the corresponding lane in `other`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn simd_eq(self, other: Self) -> Self::Mask; + + /// Test if each lane is equal to the corresponding lane in `other`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn simd_ne(self, other: Self) -> Self::Mask; +} + +macro_rules! impl_number { + { $($number:ty),* } => { + $( + impl SimdPartialEq for Simd<$number, LANES> + where + LaneCount: SupportedLaneCount, + { + type Mask = Mask<<$number as SimdElement>::Mask, LANES>; + + #[inline] + fn simd_eq(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_eq(self, other)) } + } + + #[inline] + fn simd_ne(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_ne(self, other)) } + } + } + )* + } +} + +impl_number! { f32, f64, u8, u16, u32, u64, usize, i8, i16, i32, i64, isize } + +macro_rules! impl_mask { + { $($integer:ty),* } => { + $( + impl SimdPartialEq for Mask<$integer, LANES> + where + LaneCount: SupportedLaneCount, + { + type Mask = Self; + + #[inline] + fn simd_eq(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Self::from_int_unchecked(intrinsics::simd_eq(self.to_int(), other.to_int())) } + } + + #[inline] + fn simd_ne(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Self::from_int_unchecked(intrinsics::simd_ne(self.to_int(), other.to_int())) } + } + } + )* + } +} + +impl_mask! { i8, i16, i32, i64, isize } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/fmt.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/fmt.rs new file mode 100644 index 000000000..dbd9839c4 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/fmt.rs @@ -0,0 +1,39 @@ +use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount}; +use core::fmt; + +macro_rules! impl_fmt_trait { + { $($trait:ident,)* } => { + $( + impl fmt::$trait for Simd + where + LaneCount: SupportedLaneCount, + T: SimdElement + fmt::$trait, + { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + #[repr(transparent)] + struct Wrapper<'a, T: fmt::$trait>(&'a T); + + impl fmt::Debug for Wrapper<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } + } + + f.debug_list() + .entries(self.as_array().iter().map(|x| Wrapper(x))) + .finish() + } + } + )* + } +} + +impl_fmt_trait! { + Debug, + Binary, + LowerExp, + UpperExp, + Octal, + LowerHex, + UpperHex, +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/intrinsics.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/intrinsics.rs new file mode 100644 index 000000000..704e6ed01 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/intrinsics.rs @@ -0,0 +1,153 @@ +//! This module contains the LLVM intrinsics bindings that provide the functionality for this +//! crate. +//! +//! The LLVM assembly language is documented here: +//! +//! A quick glossary of jargon that may appear in this module, mostly paraphrasing LLVM's LangRef: +//! - poison: "undefined behavior as a value". specifically, it is like uninit memory (such as padding bytes). it is "safe" to create poison, BUT +//! poison MUST NOT be observed from safe code, as operations on poison return poison, like NaN. unlike NaN, which has defined comparisons, +//! poison is neither true nor false, and LLVM may also convert it to undef (at which point it is both). so, it can't be conditioned on, either. +//! - undef: "a value that is every value". functionally like poison, insofar as Rust is concerned. poison may become this. note: +//! this means that division by poison or undef is like division by zero, which means it inflicts... +//! - "UB": poison and undef cover most of what people call "UB". "UB" means this operation immediately invalidates the program: +//! LLVM is allowed to lower it to `ud2` or other opcodes that may cause an illegal instruction exception, and this is the "good end". +//! The "bad end" is that LLVM may reverse time to the moment control flow diverged on a path towards undefined behavior, +//! and destroy the other branch, potentially deleting safe code and violating Rust's `unsafe` contract. +//! +//! Note that according to LLVM, vectors are not arrays, but they are equivalent when stored to and loaded from memory. +//! +//! Unless stated otherwise, all intrinsics for binary operations require SIMD vectors of equal types and lengths. + +// These intrinsics aren't linked directly from LLVM and are mostly undocumented, however they are +// mostly lowered to the matching LLVM instructions by the compiler in a fairly straightforward manner. +// The associated LLVM instruction or intrinsic is documented alongside each Rust intrinsic function. +extern "platform-intrinsic" { + /// add/fadd + pub(crate) fn simd_add(x: T, y: T) -> T; + + /// sub/fsub + pub(crate) fn simd_sub(lhs: T, rhs: T) -> T; + + /// mul/fmul + pub(crate) fn simd_mul(x: T, y: T) -> T; + + /// udiv/sdiv/fdiv + /// ints and uints: {s,u}div incur UB if division by zero occurs. + /// ints: sdiv is UB for int::MIN / -1. + /// floats: fdiv is never UB, but may create NaNs or infinities. + pub(crate) fn simd_div(lhs: T, rhs: T) -> T; + + /// urem/srem/frem + /// ints and uints: {s,u}rem incur UB if division by zero occurs. + /// ints: srem is UB for int::MIN / -1. + /// floats: frem is equivalent to libm::fmod in the "default" floating point environment, sans errno. + pub(crate) fn simd_rem(lhs: T, rhs: T) -> T; + + /// shl + /// for (u)ints. poison if rhs >= lhs::BITS + pub(crate) fn simd_shl(lhs: T, rhs: T) -> T; + + /// ints: ashr + /// uints: lshr + /// poison if rhs >= lhs::BITS + pub(crate) fn simd_shr(lhs: T, rhs: T) -> T; + + /// and + pub(crate) fn simd_and(x: T, y: T) -> T; + + /// or + pub(crate) fn simd_or(x: T, y: T) -> T; + + /// xor + pub(crate) fn simd_xor(x: T, y: T) -> T; + + /// getelementptr (without inbounds) + pub(crate) fn simd_arith_offset(ptrs: T, offsets: U) -> T; + + /// fptoui/fptosi/uitofp/sitofp + /// casting floats to integers is truncating, so it is safe to convert values like e.g. 1.5 + /// but the truncated value must fit in the target type or the result is poison. + /// use `simd_as` instead for a cast that performs a saturating conversion. + pub(crate) fn simd_cast(x: T) -> U; + /// follows Rust's `T as U` semantics, including saturating float casts + /// which amounts to the same as `simd_cast` for many cases + pub(crate) fn simd_as(x: T) -> U; + + /// neg/fneg + /// ints: ultimately becomes a call to cg_ssa's BuilderMethods::neg. cg_llvm equates this to `simd_sub(Simd::splat(0), x)`. + /// floats: LLVM's fneg, which changes the floating point sign bit. Some arches have instructions for it. + /// Rust panics for Neg::neg(int::MIN) due to overflow, but it is not UB in LLVM without `nsw`. + pub(crate) fn simd_neg(x: T) -> T; + + /// fabs + pub(crate) fn simd_fabs(x: T) -> T; + + // minnum/maxnum + pub(crate) fn simd_fmin(x: T, y: T) -> T; + pub(crate) fn simd_fmax(x: T, y: T) -> T; + + // these return Simd with the same BITS size as the inputs + pub(crate) fn simd_eq(x: T, y: T) -> U; + pub(crate) fn simd_ne(x: T, y: T) -> U; + pub(crate) fn simd_lt(x: T, y: T) -> U; + pub(crate) fn simd_le(x: T, y: T) -> U; + pub(crate) fn simd_gt(x: T, y: T) -> U; + pub(crate) fn simd_ge(x: T, y: T) -> U; + + // shufflevector + // idx: LLVM calls it a "shuffle mask vector constant", a vector of i32s + pub(crate) fn simd_shuffle(x: T, y: T, idx: U) -> V; + + /// llvm.masked.gather + /// like a loop of pointer reads + /// val: vector of values to select if a lane is masked + /// ptr: vector of pointers to read from + /// mask: a "wide" mask of integers, selects as if simd_select(mask, read(ptr), val) + /// note, the LLVM intrinsic accepts a mask vector of `` + /// FIXME: review this if/when we fix up our mask story in general? + pub(crate) fn simd_gather(val: T, ptr: U, mask: V) -> T; + /// llvm.masked.scatter + /// like gather, but more spicy, as it writes instead of reads + pub(crate) fn simd_scatter(val: T, ptr: U, mask: V); + + // {s,u}add.sat + pub(crate) fn simd_saturating_add(x: T, y: T) -> T; + + // {s,u}sub.sat + pub(crate) fn simd_saturating_sub(lhs: T, rhs: T) -> T; + + // reductions + // llvm.vector.reduce.{add,fadd} + pub(crate) fn simd_reduce_add_ordered(x: T, y: U) -> U; + // llvm.vector.reduce.{mul,fmul} + pub(crate) fn simd_reduce_mul_ordered(x: T, y: U) -> U; + #[allow(unused)] + pub(crate) fn simd_reduce_all(x: T) -> bool; + #[allow(unused)] + pub(crate) fn simd_reduce_any(x: T) -> bool; + pub(crate) fn simd_reduce_max(x: T) -> U; + pub(crate) fn simd_reduce_min(x: T) -> U; + pub(crate) fn simd_reduce_and(x: T) -> U; + pub(crate) fn simd_reduce_or(x: T) -> U; + pub(crate) fn simd_reduce_xor(x: T) -> U; + + // truncate integer vector to bitmask + // `fn simd_bitmask(vector) -> unsigned integer` takes a vector of integers and + // returns either an unsigned integer or array of `u8`. + // Every element in the vector becomes a single bit in the returned bitmask. + // If the vector has less than 8 lanes, a u8 is returned with zeroed trailing bits. + // The bit order of the result depends on the byte endianness. LSB-first for little + // endian and MSB-first for big endian. + // + // UB if called on a vector with values other than 0 and -1. + #[allow(unused)] + pub(crate) fn simd_bitmask(x: T) -> U; + + // select + // first argument is a vector of integers, -1 (all bits 1) is "true" + // logically equivalent to (yes & m) | (no & (m^-1), + // but you can use it on floats. + pub(crate) fn simd_select(m: M, yes: T, no: T) -> T; + #[allow(unused)] + pub(crate) fn simd_select_bitmask(m: M, yes: T, no: T) -> T; +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/iter.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/iter.rs new file mode 100644 index 000000000..3275b4db8 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/iter.rs @@ -0,0 +1,58 @@ +use crate::simd::{LaneCount, Simd, SupportedLaneCount}; +use core::{ + iter::{Product, Sum}, + ops::{Add, Mul}, +}; + +macro_rules! impl_traits { + { $type:ty } => { + impl Sum for Simd<$type, LANES> + where + LaneCount: SupportedLaneCount, + { + fn sum>(iter: I) -> Self { + iter.fold(Simd::splat(0 as $type), Add::add) + } + } + + impl Product for Simd<$type, LANES> + where + LaneCount: SupportedLaneCount, + { + fn product>(iter: I) -> Self { + iter.fold(Simd::splat(1 as $type), Mul::mul) + } + } + + impl<'a, const LANES: usize> Sum<&'a Self> for Simd<$type, LANES> + where + LaneCount: SupportedLaneCount, + { + fn sum>(iter: I) -> Self { + iter.fold(Simd::splat(0 as $type), Add::add) + } + } + + impl<'a, const LANES: usize> Product<&'a Self> for Simd<$type, LANES> + where + LaneCount: SupportedLaneCount, + { + fn product>(iter: I) -> Self { + iter.fold(Simd::splat(1 as $type), Mul::mul) + } + } + } +} + +impl_traits! { f32 } +impl_traits! { f64 } +impl_traits! { u8 } +impl_traits! { u16 } +impl_traits! { u32 } +impl_traits! { u64 } +impl_traits! { usize } +impl_traits! { i8 } +impl_traits! { i16 } +impl_traits! { i32 } +impl_traits! { i64 } +impl_traits! { isize } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/lane_count.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/lane_count.rs new file mode 100644 index 000000000..63723e2ec --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/lane_count.rs @@ -0,0 +1,46 @@ +mod sealed { + pub trait Sealed {} +} +use sealed::Sealed; + +/// Specifies the number of lanes in a SIMD vector as a type. +pub struct LaneCount; + +impl LaneCount { + /// The number of bytes in a bitmask with this many lanes. + pub const BITMASK_LEN: usize = (LANES + 7) / 8; +} + +/// Statically guarantees that a lane count is marked as supported. +/// +/// This trait is *sealed*: the list of implementors below is total. +/// Users do not have the ability to mark additional `LaneCount` values as supported. +/// Only SIMD vectors with supported lane counts are constructable. +pub trait SupportedLaneCount: Sealed { + #[doc(hidden)] + type BitMask: Copy + Default + AsRef<[u8]> + AsMut<[u8]>; +} + +impl Sealed for LaneCount {} + +impl SupportedLaneCount for LaneCount<1> { + type BitMask = [u8; 1]; +} +impl SupportedLaneCount for LaneCount<2> { + type BitMask = [u8; 1]; +} +impl SupportedLaneCount for LaneCount<4> { + type BitMask = [u8; 1]; +} +impl SupportedLaneCount for LaneCount<8> { + type BitMask = [u8; 1]; +} +impl SupportedLaneCount for LaneCount<16> { + type BitMask = [u8; 2]; +} +impl SupportedLaneCount for LaneCount<32> { + type BitMask = [u8; 4]; +} +impl SupportedLaneCount for LaneCount<64> { + type BitMask = [u8; 8]; +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/lib.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/lib.rs new file mode 100644 index 000000000..715f258f6 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/lib.rs @@ -0,0 +1,22 @@ +#![no_std] +#![feature( + convert_float_to_int, + decl_macro, + intra_doc_pointers, + platform_intrinsics, + repr_simd, + simd_ffi, + staged_api, + stdsimd +)] +#![cfg_attr(feature = "generic_const_exprs", feature(generic_const_exprs))] +#![cfg_attr(feature = "generic_const_exprs", allow(incomplete_features))] +#![warn(missing_docs)] +#![deny(unsafe_op_in_unsafe_fn, clippy::undocumented_unsafe_blocks)] +#![unstable(feature = "portable_simd", issue = "86656")] +//! Portable SIMD module. + +#[path = "mod.rs"] +mod core_simd; +pub use self::core_simd::simd; +pub use simd::*; diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/masks.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/masks.rs new file mode 100644 index 000000000..c36c336d8 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/masks.rs @@ -0,0 +1,595 @@ +//! Types and traits associated with masking lanes of vectors. +//! Types representing +#![allow(non_camel_case_types)] + +#[cfg_attr( + not(all(target_arch = "x86_64", target_feature = "avx512f")), + path = "masks/full_masks.rs" +)] +#[cfg_attr( + all(target_arch = "x86_64", target_feature = "avx512f"), + path = "masks/bitmask.rs" +)] +mod mask_impl; + +mod to_bitmask; +pub use to_bitmask::ToBitMask; + +#[cfg(feature = "generic_const_exprs")] +pub use to_bitmask::{bitmask_len, ToBitMaskArray}; + +use crate::simd::{intrinsics, LaneCount, Simd, SimdElement, SimdPartialEq, SupportedLaneCount}; +use core::cmp::Ordering; +use core::{fmt, mem}; + +mod sealed { + use super::*; + + /// Not only does this seal the `MaskElement` trait, but these functions prevent other traits + /// from bleeding into the parent bounds. + /// + /// For example, `eq` could be provided by requiring `MaskElement: PartialEq`, but that would + /// prevent us from ever removing that bound, or from implementing `MaskElement` on + /// non-`PartialEq` types in the future. + pub trait Sealed { + fn valid(values: Simd) -> bool + where + LaneCount: SupportedLaneCount, + Self: SimdElement; + + fn eq(self, other: Self) -> bool; + + const TRUE: Self; + + const FALSE: Self; + } +} +use sealed::Sealed; + +/// Marker trait for types that may be used as SIMD mask elements. +/// +/// # Safety +/// Type must be a signed integer. +pub unsafe trait MaskElement: SimdElement + Sealed {} + +macro_rules! impl_element { + { $ty:ty } => { + impl Sealed for $ty { + fn valid(value: Simd) -> bool + where + LaneCount: SupportedLaneCount, + { + (value.simd_eq(Simd::splat(0 as _)) | value.simd_eq(Simd::splat(-1 as _))).all() + } + + fn eq(self, other: Self) -> bool { self == other } + + const TRUE: Self = -1; + const FALSE: Self = 0; + } + + // Safety: this is a valid mask element type + unsafe impl MaskElement for $ty {} + } +} + +impl_element! { i8 } +impl_element! { i16 } +impl_element! { i32 } +impl_element! { i64 } +impl_element! { isize } + +/// A SIMD vector mask for `LANES` elements of width specified by `Element`. +/// +/// Masks represent boolean inclusion/exclusion on a per-lane basis. +/// +/// The layout of this type is unspecified. +#[repr(transparent)] +pub struct Mask(mask_impl::Mask) +where + T: MaskElement, + LaneCount: SupportedLaneCount; + +impl Copy for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ +} + +impl Clone for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + fn clone(&self) -> Self { + *self + } +} + +impl Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + /// Construct a mask by setting all lanes to the given value. + pub fn splat(value: bool) -> Self { + Self(mask_impl::Mask::splat(value)) + } + + /// Converts an array of bools to a SIMD mask. + pub fn from_array(array: [bool; LANES]) -> Self { + // SAFETY: Rust's bool has a layout of 1 byte (u8) with a value of + // true: 0b_0000_0001 + // false: 0b_0000_0000 + // Thus, an array of bools is also a valid array of bytes: [u8; N] + // This would be hypothetically valid as an "in-place" transmute, + // but these are "dependently-sized" types, so copy elision it is! + unsafe { + let bytes: [u8; LANES] = mem::transmute_copy(&array); + let bools: Simd = + intrinsics::simd_ne(Simd::from_array(bytes), Simd::splat(0u8)); + Mask::from_int_unchecked(intrinsics::simd_cast(bools)) + } + } + + /// Converts a SIMD mask to an array of bools. + pub fn to_array(self) -> [bool; LANES] { + // This follows mostly the same logic as from_array. + // SAFETY: Rust's bool has a layout of 1 byte (u8) with a value of + // true: 0b_0000_0001 + // false: 0b_0000_0000 + // Thus, an array of bools is also a valid array of bytes: [u8; N] + // Since our masks are equal to integers where all bits are set, + // we can simply convert them to i8s, and then bitand them by the + // bitpattern for Rust's "true" bool. + // This would be hypothetically valid as an "in-place" transmute, + // but these are "dependently-sized" types, so copy elision it is! + unsafe { + let mut bytes: Simd = intrinsics::simd_cast(self.to_int()); + bytes &= Simd::splat(1i8); + mem::transmute_copy(&bytes) + } + } + + /// Converts a vector of integers to a mask, where 0 represents `false` and -1 + /// represents `true`. + /// + /// # Safety + /// All lanes must be either 0 or -1. + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + pub unsafe fn from_int_unchecked(value: Simd) -> Self { + // Safety: the caller must confirm this invariant + unsafe { Self(mask_impl::Mask::from_int_unchecked(value)) } + } + + /// Converts a vector of integers to a mask, where 0 represents `false` and -1 + /// represents `true`. + /// + /// # Panics + /// Panics if any lane is not 0 or -1. + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + pub fn from_int(value: Simd) -> Self { + assert!(T::valid(value), "all values must be either 0 or -1",); + // Safety: the validity has been checked + unsafe { Self::from_int_unchecked(value) } + } + + /// Converts the mask to a vector of integers, where 0 represents `false` and -1 + /// represents `true`. + #[inline] + #[must_use = "method returns a new vector and does not mutate the original value"] + pub fn to_int(self) -> Simd { + self.0.to_int() + } + + /// Converts the mask to a mask of any other lane size. + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + pub fn cast(self) -> Mask { + Mask(self.0.convert()) + } + + /// Tests the value of the specified lane. + /// + /// # Safety + /// `lane` must be less than `LANES`. + #[inline] + #[must_use = "method returns a new bool and does not mutate the original value"] + pub unsafe fn test_unchecked(&self, lane: usize) -> bool { + // Safety: the caller must confirm this invariant + unsafe { self.0.test_unchecked(lane) } + } + + /// Tests the value of the specified lane. + /// + /// # Panics + /// Panics if `lane` is greater than or equal to the number of lanes in the vector. + #[inline] + #[must_use = "method returns a new bool and does not mutate the original value"] + pub fn test(&self, lane: usize) -> bool { + assert!(lane < LANES, "lane index out of range"); + // Safety: the lane index has been checked + unsafe { self.test_unchecked(lane) } + } + + /// Sets the value of the specified lane. + /// + /// # Safety + /// `lane` must be less than `LANES`. + #[inline] + pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { + // Safety: the caller must confirm this invariant + unsafe { + self.0.set_unchecked(lane, value); + } + } + + /// Sets the value of the specified lane. + /// + /// # Panics + /// Panics if `lane` is greater than or equal to the number of lanes in the vector. + #[inline] + pub fn set(&mut self, lane: usize, value: bool) { + assert!(lane < LANES, "lane index out of range"); + // Safety: the lane index has been checked + unsafe { + self.set_unchecked(lane, value); + } + } + + /// Returns true if any lane is set, or false otherwise. + #[inline] + #[must_use = "method returns a new bool and does not mutate the original value"] + pub fn any(self) -> bool { + self.0.any() + } + + /// Returns true if all lanes are set, or false otherwise. + #[inline] + #[must_use = "method returns a new bool and does not mutate the original value"] + pub fn all(self) -> bool { + self.0.all() + } +} + +// vector/array conversion +impl From<[bool; LANES]> for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + fn from(array: [bool; LANES]) -> Self { + Self::from_array(array) + } +} + +impl From> for [bool; LANES] +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + fn from(vector: Mask) -> Self { + vector.to_array() + } +} + +impl Default for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + #[inline] + #[must_use = "method returns a defaulted mask with all lanes set to false (0)"] + fn default() -> Self { + Self::splat(false) + } +} + +impl PartialEq for Mask +where + T: MaskElement + PartialEq, + LaneCount: SupportedLaneCount, +{ + #[inline] + #[must_use = "method returns a new bool and does not mutate the original value"] + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } +} + +impl PartialOrd for Mask +where + T: MaskElement + PartialOrd, + LaneCount: SupportedLaneCount, +{ + #[inline] + #[must_use = "method returns a new Ordering and does not mutate the original value"] + fn partial_cmp(&self, other: &Self) -> Option { + self.0.partial_cmp(&other.0) + } +} + +impl fmt::Debug for Mask +where + T: MaskElement + fmt::Debug, + LaneCount: SupportedLaneCount, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list() + .entries((0..LANES).map(|lane| self.test(lane))) + .finish() + } +} + +impl core::ops::BitAnd for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn bitand(self, rhs: Self) -> Self { + Self(self.0 & rhs.0) + } +} + +impl core::ops::BitAnd for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn bitand(self, rhs: bool) -> Self { + self & Self::splat(rhs) + } +} + +impl core::ops::BitAnd> for bool +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Mask; + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn bitand(self, rhs: Mask) -> Mask { + Mask::splat(self) & rhs + } +} + +impl core::ops::BitOr for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn bitor(self, rhs: Self) -> Self { + Self(self.0 | rhs.0) + } +} + +impl core::ops::BitOr for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn bitor(self, rhs: bool) -> Self { + self | Self::splat(rhs) + } +} + +impl core::ops::BitOr> for bool +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Mask; + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn bitor(self, rhs: Mask) -> Mask { + Mask::splat(self) | rhs + } +} + +impl core::ops::BitXor for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn bitxor(self, rhs: Self) -> Self::Output { + Self(self.0 ^ rhs.0) + } +} + +impl core::ops::BitXor for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn bitxor(self, rhs: bool) -> Self::Output { + self ^ Self::splat(rhs) + } +} + +impl core::ops::BitXor> for bool +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Mask; + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn bitxor(self, rhs: Mask) -> Self::Output { + Mask::splat(self) ^ rhs + } +} + +impl core::ops::Not for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Mask; + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn not(self) -> Self::Output { + Self(!self.0) + } +} + +impl core::ops::BitAndAssign for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + #[inline] + fn bitand_assign(&mut self, rhs: Self) { + self.0 = self.0 & rhs.0; + } +} + +impl core::ops::BitAndAssign for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + #[inline] + fn bitand_assign(&mut self, rhs: bool) { + *self &= Self::splat(rhs); + } +} + +impl core::ops::BitOrAssign for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + #[inline] + fn bitor_assign(&mut self, rhs: Self) { + self.0 = self.0 | rhs.0; + } +} + +impl core::ops::BitOrAssign for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + #[inline] + fn bitor_assign(&mut self, rhs: bool) { + *self |= Self::splat(rhs); + } +} + +impl core::ops::BitXorAssign for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + #[inline] + fn bitxor_assign(&mut self, rhs: Self) { + self.0 = self.0 ^ rhs.0; + } +} + +impl core::ops::BitXorAssign for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + #[inline] + fn bitxor_assign(&mut self, rhs: bool) { + *self ^= Self::splat(rhs); + } +} + +/// A mask for SIMD vectors with eight elements of 8 bits. +pub type mask8x8 = Mask; + +/// A mask for SIMD vectors with 16 elements of 8 bits. +pub type mask8x16 = Mask; + +/// A mask for SIMD vectors with 32 elements of 8 bits. +pub type mask8x32 = Mask; + +/// A mask for SIMD vectors with 64 elements of 8 bits. +pub type mask8x64 = Mask; + +/// A mask for SIMD vectors with four elements of 16 bits. +pub type mask16x4 = Mask; + +/// A mask for SIMD vectors with eight elements of 16 bits. +pub type mask16x8 = Mask; + +/// A mask for SIMD vectors with 16 elements of 16 bits. +pub type mask16x16 = Mask; + +/// A mask for SIMD vectors with 32 elements of 16 bits. +pub type mask16x32 = Mask; + +/// A mask for SIMD vectors with two elements of 32 bits. +pub type mask32x2 = Mask; + +/// A mask for SIMD vectors with four elements of 32 bits. +pub type mask32x4 = Mask; + +/// A mask for SIMD vectors with eight elements of 32 bits. +pub type mask32x8 = Mask; + +/// A mask for SIMD vectors with 16 elements of 32 bits. +pub type mask32x16 = Mask; + +/// A mask for SIMD vectors with two elements of 64 bits. +pub type mask64x2 = Mask; + +/// A mask for SIMD vectors with four elements of 64 bits. +pub type mask64x4 = Mask; + +/// A mask for SIMD vectors with eight elements of 64 bits. +pub type mask64x8 = Mask; + +/// A mask for SIMD vectors with two elements of pointer width. +pub type masksizex2 = Mask; + +/// A mask for SIMD vectors with four elements of pointer width. +pub type masksizex4 = Mask; + +/// A mask for SIMD vectors with eight elements of pointer width. +pub type masksizex8 = Mask; + +macro_rules! impl_from { + { $from:ty => $($to:ty),* } => { + $( + impl From> for Mask<$to, LANES> + where + LaneCount: SupportedLaneCount, + { + fn from(value: Mask<$from, LANES>) -> Self { + value.cast() + } + } + )* + } +} +impl_from! { i8 => i16, i32, i64, isize } +impl_from! { i16 => i32, i64, isize, i8 } +impl_from! { i32 => i64, isize, i8, i16 } +impl_from! { i64 => isize, i8, i16, i32 } +impl_from! { isize => i8, i16, i32, i64 } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/masks/bitmask.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/masks/bitmask.rs new file mode 100644 index 000000000..365ecc0a3 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/masks/bitmask.rs @@ -0,0 +1,246 @@ +#![allow(unused_imports)] +use super::MaskElement; +use crate::simd::intrinsics; +use crate::simd::{LaneCount, Simd, SupportedLaneCount, ToBitMask}; +use core::marker::PhantomData; + +/// A mask where each lane is represented by a single bit. +#[repr(transparent)] +pub struct Mask( + as SupportedLaneCount>::BitMask, + PhantomData, +) +where + T: MaskElement, + LaneCount: SupportedLaneCount; + +impl Copy for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ +} + +impl Clone for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + fn clone(&self) -> Self { + *self + } +} + +impl PartialEq for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + fn eq(&self, other: &Self) -> bool { + self.0.as_ref() == other.0.as_ref() + } +} + +impl PartialOrd for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + fn partial_cmp(&self, other: &Self) -> Option { + self.0.as_ref().partial_cmp(other.0.as_ref()) + } +} + +impl Eq for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ +} + +impl Ord for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + fn cmp(&self, other: &Self) -> core::cmp::Ordering { + self.0.as_ref().cmp(other.0.as_ref()) + } +} + +impl Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + pub fn splat(value: bool) -> Self { + let mut mask = as SupportedLaneCount>::BitMask::default(); + if value { + mask.as_mut().fill(u8::MAX) + } else { + mask.as_mut().fill(u8::MIN) + } + if LANES % 8 > 0 { + *mask.as_mut().last_mut().unwrap() &= u8::MAX >> (8 - LANES % 8); + } + Self(mask, PhantomData) + } + + #[inline] + #[must_use = "method returns a new bool and does not mutate the original value"] + pub unsafe fn test_unchecked(&self, lane: usize) -> bool { + (self.0.as_ref()[lane / 8] >> (lane % 8)) & 0x1 > 0 + } + + #[inline] + pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { + unsafe { + self.0.as_mut()[lane / 8] ^= ((value ^ self.test_unchecked(lane)) as u8) << (lane % 8) + } + } + + #[inline] + #[must_use = "method returns a new vector and does not mutate the original value"] + pub fn to_int(self) -> Simd { + unsafe { + intrinsics::simd_select_bitmask(self.0, Simd::splat(T::TRUE), Simd::splat(T::FALSE)) + } + } + + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + pub unsafe fn from_int_unchecked(value: Simd) -> Self { + unsafe { Self(intrinsics::simd_bitmask(value), PhantomData) } + } + + #[cfg(feature = "generic_const_exprs")] + #[inline] + #[must_use = "method returns a new array and does not mutate the original value"] + pub fn to_bitmask_array(self) -> [u8; N] { + assert!(core::mem::size_of::() == N); + + // Safety: converting an integer to an array of bytes of the same size is safe + unsafe { core::mem::transmute_copy(&self.0) } + } + + #[cfg(feature = "generic_const_exprs")] + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + pub fn from_bitmask_array(bitmask: [u8; N]) -> Self { + assert!(core::mem::size_of::() == N); + + // Safety: converting an array of bytes to an integer of the same size is safe + Self(unsafe { core::mem::transmute_copy(&bitmask) }, PhantomData) + } + + #[inline] + pub fn to_bitmask_integer(self) -> U + where + super::Mask: ToBitMask, + { + // Safety: these are the same types + unsafe { core::mem::transmute_copy(&self.0) } + } + + #[inline] + pub fn from_bitmask_integer(bitmask: U) -> Self + where + super::Mask: ToBitMask, + { + // Safety: these are the same types + unsafe { Self(core::mem::transmute_copy(&bitmask), PhantomData) } + } + + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + pub fn convert(self) -> Mask + where + U: MaskElement, + { + // Safety: bitmask layout does not depend on the element width + unsafe { core::mem::transmute_copy(&self) } + } + + #[inline] + #[must_use = "method returns a new bool and does not mutate the original value"] + pub fn any(self) -> bool { + self != Self::splat(false) + } + + #[inline] + #[must_use = "method returns a new bool and does not mutate the original value"] + pub fn all(self) -> bool { + self == Self::splat(true) + } +} + +impl core::ops::BitAnd for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, + as SupportedLaneCount>::BitMask: AsRef<[u8]> + AsMut<[u8]>, +{ + type Output = Self; + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn bitand(mut self, rhs: Self) -> Self { + for (l, r) in self.0.as_mut().iter_mut().zip(rhs.0.as_ref().iter()) { + *l &= r; + } + self + } +} + +impl core::ops::BitOr for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, + as SupportedLaneCount>::BitMask: AsRef<[u8]> + AsMut<[u8]>, +{ + type Output = Self; + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn bitor(mut self, rhs: Self) -> Self { + for (l, r) in self.0.as_mut().iter_mut().zip(rhs.0.as_ref().iter()) { + *l |= r; + } + self + } +} + +impl core::ops::BitXor for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn bitxor(mut self, rhs: Self) -> Self::Output { + for (l, r) in self.0.as_mut().iter_mut().zip(rhs.0.as_ref().iter()) { + *l ^= r; + } + self + } +} + +impl core::ops::Not for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn not(mut self) -> Self::Output { + for x in self.0.as_mut() { + *x = !*x; + } + if LANES % 8 > 0 { + *self.0.as_mut().last_mut().unwrap() &= u8::MAX >> (8 - LANES % 8); + } + self + } +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/masks/full_masks.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/masks/full_masks.rs new file mode 100644 index 000000000..adf0fcbea --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/masks/full_masks.rs @@ -0,0 +1,323 @@ +//! Masks that take up full SIMD vector registers. + +use super::MaskElement; +use crate::simd::intrinsics; +use crate::simd::{LaneCount, Simd, SupportedLaneCount, ToBitMask}; + +#[cfg(feature = "generic_const_exprs")] +use crate::simd::ToBitMaskArray; + +#[repr(transparent)] +pub struct Mask(Simd) +where + T: MaskElement, + LaneCount: SupportedLaneCount; + +impl Copy for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ +} + +impl Clone for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn clone(&self) -> Self { + *self + } +} + +impl PartialEq for Mask +where + T: MaskElement + PartialEq, + LaneCount: SupportedLaneCount, +{ + fn eq(&self, other: &Self) -> bool { + self.0.eq(&other.0) + } +} + +impl PartialOrd for Mask +where + T: MaskElement + PartialOrd, + LaneCount: SupportedLaneCount, +{ + fn partial_cmp(&self, other: &Self) -> Option { + self.0.partial_cmp(&other.0) + } +} + +impl Eq for Mask +where + T: MaskElement + Eq, + LaneCount: SupportedLaneCount, +{ +} + +impl Ord for Mask +where + T: MaskElement + Ord, + LaneCount: SupportedLaneCount, +{ + fn cmp(&self, other: &Self) -> core::cmp::Ordering { + self.0.cmp(&other.0) + } +} + +// Used for bitmask bit order workaround +pub(crate) trait ReverseBits { + // Reverse the least significant `n` bits of `self`. + // (Remaining bits must be 0.) + fn reverse_bits(self, n: usize) -> Self; +} + +macro_rules! impl_reverse_bits { + { $($int:ty),* } => { + $( + impl ReverseBits for $int { + #[inline(always)] + fn reverse_bits(self, n: usize) -> Self { + let rev = <$int>::reverse_bits(self); + let bitsize = core::mem::size_of::<$int>() * 8; + if n < bitsize { + // Shift things back to the right + rev >> (bitsize - n) + } else { + rev + } + } + } + )* + } +} + +impl_reverse_bits! { u8, u16, u32, u64 } + +impl Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + pub fn splat(value: bool) -> Self { + Self(Simd::splat(if value { T::TRUE } else { T::FALSE })) + } + + #[inline] + #[must_use = "method returns a new bool and does not mutate the original value"] + pub unsafe fn test_unchecked(&self, lane: usize) -> bool { + T::eq(self.0[lane], T::TRUE) + } + + #[inline] + pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { + self.0[lane] = if value { T::TRUE } else { T::FALSE } + } + + #[inline] + #[must_use = "method returns a new vector and does not mutate the original value"] + pub fn to_int(self) -> Simd { + self.0 + } + + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + pub unsafe fn from_int_unchecked(value: Simd) -> Self { + Self(value) + } + + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + pub fn convert(self) -> Mask + where + U: MaskElement, + { + // Safety: masks are simply integer vectors of 0 and -1, and we can cast the element type. + unsafe { Mask(intrinsics::simd_cast(self.0)) } + } + + #[cfg(feature = "generic_const_exprs")] + #[inline] + #[must_use = "method returns a new array and does not mutate the original value"] + pub fn to_bitmask_array(self) -> [u8; N] + where + super::Mask: ToBitMaskArray, + [(); as ToBitMaskArray>::BYTES]: Sized, + { + assert_eq!( as ToBitMaskArray>::BYTES, N); + + // Safety: N is the correct bitmask size + unsafe { + // Compute the bitmask + let bitmask: [u8; as ToBitMaskArray>::BYTES] = + intrinsics::simd_bitmask(self.0); + + // Transmute to the return type, previously asserted to be the same size + let mut bitmask: [u8; N] = core::mem::transmute_copy(&bitmask); + + // LLVM assumes bit order should match endianness + if cfg!(target_endian = "big") { + for x in bitmask.as_mut() { + *x = x.reverse_bits(); + } + }; + + bitmask + } + } + + #[cfg(feature = "generic_const_exprs")] + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + pub fn from_bitmask_array(mut bitmask: [u8; N]) -> Self + where + super::Mask: ToBitMaskArray, + [(); as ToBitMaskArray>::BYTES]: Sized, + { + assert_eq!( as ToBitMaskArray>::BYTES, N); + + // Safety: N is the correct bitmask size + unsafe { + // LLVM assumes bit order should match endianness + if cfg!(target_endian = "big") { + for x in bitmask.as_mut() { + *x = x.reverse_bits(); + } + } + + // Transmute to the bitmask type, previously asserted to be the same size + let bitmask: [u8; as ToBitMaskArray>::BYTES] = + core::mem::transmute_copy(&bitmask); + + // Compute the regular mask + Self::from_int_unchecked(intrinsics::simd_select_bitmask( + bitmask, + Self::splat(true).to_int(), + Self::splat(false).to_int(), + )) + } + } + + #[inline] + pub(crate) fn to_bitmask_integer(self) -> U + where + super::Mask: ToBitMask, + { + // Safety: U is required to be the appropriate bitmask type + let bitmask: U = unsafe { intrinsics::simd_bitmask(self.0) }; + + // LLVM assumes bit order should match endianness + if cfg!(target_endian = "big") { + bitmask.reverse_bits(LANES) + } else { + bitmask + } + } + + #[inline] + pub(crate) fn from_bitmask_integer(bitmask: U) -> Self + where + super::Mask: ToBitMask, + { + // LLVM assumes bit order should match endianness + let bitmask = if cfg!(target_endian = "big") { + bitmask.reverse_bits(LANES) + } else { + bitmask + }; + + // Safety: U is required to be the appropriate bitmask type + unsafe { + Self::from_int_unchecked(intrinsics::simd_select_bitmask( + bitmask, + Self::splat(true).to_int(), + Self::splat(false).to_int(), + )) + } + } + + #[inline] + #[must_use = "method returns a new bool and does not mutate the original value"] + pub fn any(self) -> bool { + // Safety: use `self` as an integer vector + unsafe { intrinsics::simd_reduce_any(self.to_int()) } + } + + #[inline] + #[must_use = "method returns a new vector and does not mutate the original value"] + pub fn all(self) -> bool { + // Safety: use `self` as an integer vector + unsafe { intrinsics::simd_reduce_all(self.to_int()) } + } +} + +impl core::convert::From> for Simd +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + fn from(value: Mask) -> Self { + value.0 + } +} + +impl core::ops::BitAnd for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn bitand(self, rhs: Self) -> Self { + // Safety: `self` is an integer vector + unsafe { Self(intrinsics::simd_and(self.0, rhs.0)) } + } +} + +impl core::ops::BitOr for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn bitor(self, rhs: Self) -> Self { + // Safety: `self` is an integer vector + unsafe { Self(intrinsics::simd_or(self.0, rhs.0)) } + } +} + +impl core::ops::BitXor for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn bitxor(self, rhs: Self) -> Self { + // Safety: `self` is an integer vector + unsafe { Self(intrinsics::simd_xor(self.0, rhs.0)) } + } +} + +impl core::ops::Not for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn not(self) -> Self::Output { + Self::splat(true) ^ self + } +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/masks/to_bitmask.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/masks/to_bitmask.rs new file mode 100644 index 000000000..2235f016c --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/masks/to_bitmask.rs @@ -0,0 +1,93 @@ +use super::{mask_impl, Mask, MaskElement}; +use crate::simd::{LaneCount, SupportedLaneCount}; + +mod sealed { + pub trait Sealed {} +} +pub use sealed::Sealed; + +impl Sealed for Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ +} + +/// Converts masks to and from integer bitmasks. +/// +/// Each bit of the bitmask corresponds to a mask lane, starting with the LSB. +pub trait ToBitMask: Sealed { + /// The integer bitmask type. + type BitMask; + + /// Converts a mask to a bitmask. + fn to_bitmask(self) -> Self::BitMask; + + /// Converts a bitmask to a mask. + fn from_bitmask(bitmask: Self::BitMask) -> Self; +} + +/// Converts masks to and from byte array bitmasks. +/// +/// Each bit of the bitmask corresponds to a mask lane, starting with the LSB of the first byte. +#[cfg(feature = "generic_const_exprs")] +pub trait ToBitMaskArray: Sealed { + /// The length of the bitmask array. + const BYTES: usize; + + /// Converts a mask to a bitmask. + fn to_bitmask_array(self) -> [u8; Self::BYTES]; + + /// Converts a bitmask to a mask. + fn from_bitmask_array(bitmask: [u8; Self::BYTES]) -> Self; +} + +macro_rules! impl_integer_intrinsic { + { $(impl ToBitMask for Mask<_, $lanes:literal>)* } => { + $( + impl ToBitMask for Mask { + type BitMask = $int; + + fn to_bitmask(self) -> $int { + self.0.to_bitmask_integer() + } + + fn from_bitmask(bitmask: $int) -> Self { + Self(mask_impl::Mask::from_bitmask_integer(bitmask)) + } + } + )* + } +} + +impl_integer_intrinsic! { + impl ToBitMask for Mask<_, 1> + impl ToBitMask for Mask<_, 2> + impl ToBitMask for Mask<_, 4> + impl ToBitMask for Mask<_, 8> + impl ToBitMask for Mask<_, 16> + impl ToBitMask for Mask<_, 32> + impl ToBitMask for Mask<_, 64> +} + +/// Returns the minimum number of bytes in a bitmask with `lanes` lanes. +#[cfg(feature = "generic_const_exprs")] +pub const fn bitmask_len(lanes: usize) -> usize { + (lanes + 7) / 8 +} + +#[cfg(feature = "generic_const_exprs")] +impl ToBitMaskArray for Mask +where + LaneCount: SupportedLaneCount, +{ + const BYTES: usize = bitmask_len(LANES); + + fn to_bitmask_array(self) -> [u8; Self::BYTES] { + self.0.to_bitmask_array() + } + + fn from_bitmask_array(bitmask: [u8; Self::BYTES]) -> Self { + Mask(mask_impl::Mask::from_bitmask_array(bitmask)) + } +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/mod.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/mod.rs new file mode 100644 index 000000000..b472aa3ab --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/mod.rs @@ -0,0 +1,32 @@ +#[macro_use] +mod swizzle; + +pub(crate) mod intrinsics; + +#[cfg(feature = "generic_const_exprs")] +mod to_bytes; + +mod elements; +mod eq; +mod fmt; +mod iter; +mod lane_count; +mod masks; +mod ops; +mod ord; +mod select; +mod vector; +mod vendor; + +#[doc = include_str!("core_simd_docs.md")] +pub mod simd { + pub(crate) use crate::core_simd::intrinsics; + + pub use crate::core_simd::elements::*; + pub use crate::core_simd::eq::*; + pub use crate::core_simd::lane_count::{LaneCount, SupportedLaneCount}; + pub use crate::core_simd::masks::*; + pub use crate::core_simd::ord::*; + pub use crate::core_simd::swizzle::*; + pub use crate::core_simd::vector::*; +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/ops.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/ops.rs new file mode 100644 index 000000000..fc1e0bc42 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/ops.rs @@ -0,0 +1,254 @@ +use crate::simd::{LaneCount, Simd, SimdElement, SimdPartialEq, SupportedLaneCount}; +use core::ops::{Add, Mul}; +use core::ops::{BitAnd, BitOr, BitXor}; +use core::ops::{Div, Rem, Sub}; +use core::ops::{Shl, Shr}; + +mod assign; +mod deref; +mod unary; + +impl core::ops::Index for Simd +where + T: SimdElement, + LaneCount: SupportedLaneCount, + I: core::slice::SliceIndex<[T]>, +{ + type Output = I::Output; + fn index(&self, index: I) -> &Self::Output { + &self.as_array()[index] + } +} + +impl core::ops::IndexMut for Simd +where + T: SimdElement, + LaneCount: SupportedLaneCount, + I: core::slice::SliceIndex<[T]>, +{ + fn index_mut(&mut self, index: I) -> &mut Self::Output { + &mut self.as_mut_array()[index] + } +} + +macro_rules! unsafe_base { + ($lhs:ident, $rhs:ident, {$simd_call:ident}, $($_:tt)*) => { + // Safety: $lhs and $rhs are vectors + unsafe { $crate::simd::intrinsics::$simd_call($lhs, $rhs) } + }; +} + +/// SAFETY: This macro should not be used for anything except Shl or Shr, and passed the appropriate shift intrinsic. +/// It handles performing a bitand in addition to calling the shift operator, so that the result +/// is well-defined: LLVM can return a poison value if you shl, lshr, or ashr if `rhs >= ::BITS` +/// At worst, this will maybe add another instruction and cycle, +/// at best, it may open up more optimization opportunities, +/// or simply be elided entirely, especially for SIMD ISAs which default to this. +/// +// FIXME: Consider implementing this in cg_llvm instead? +// cg_clif defaults to this, and scalar MIR shifts also default to wrapping +macro_rules! wrap_bitshift { + ($lhs:ident, $rhs:ident, {$simd_call:ident}, $int:ident) => { + #[allow(clippy::suspicious_arithmetic_impl)] + // Safety: $lhs and the bitand result are vectors + unsafe { + $crate::simd::intrinsics::$simd_call( + $lhs, + $rhs.bitand(Simd::splat(<$int>::BITS as $int - 1)), + ) + } + }; +} + +/// SAFETY: This macro must only be used to impl Div or Rem and given the matching intrinsic. +/// It guards against LLVM's UB conditions for integer div or rem using masks and selects, +/// thus guaranteeing a Rust value returns instead. +/// +/// | | LLVM | Rust +/// | :--------------: | :--- | :---------- +/// | N {/,%} 0 | UB | panic!() +/// | <$int>::MIN / -1 | UB | <$int>::MIN +/// | <$int>::MIN % -1 | UB | 0 +/// +macro_rules! int_divrem_guard { + ( $lhs:ident, + $rhs:ident, + { const PANIC_ZERO: &'static str = $zero:literal; + $simd_call:ident + }, + $int:ident ) => { + if $rhs.simd_eq(Simd::splat(0 as _)).any() { + panic!($zero); + } else { + // Prevent otherwise-UB overflow on the MIN / -1 case. + let rhs = if <$int>::MIN != 0 { + // This should, at worst, optimize to a few branchless logical ops + // Ideally, this entire conditional should evaporate + // Fire LLVM and implement those manually if it doesn't get the hint + ($lhs.simd_eq(Simd::splat(<$int>::MIN)) + // type inference can break here, so cut an SInt to size + & $rhs.simd_eq(Simd::splat(-1i64 as _))) + .select(Simd::splat(1 as _), $rhs) + } else { + // Nice base case to make it easy to const-fold away the other branch. + $rhs + }; + // Safety: $lhs and rhs are vectors + unsafe { $crate::simd::intrinsics::$simd_call($lhs, rhs) } + } + }; +} + +macro_rules! for_base_types { + ( T = ($($scalar:ident),*); + type Lhs = Simd; + type Rhs = Simd; + type Output = $out:ty; + + impl $op:ident::$call:ident { + $macro_impl:ident $inner:tt + }) => { + $( + impl $op for Simd<$scalar, N> + where + $scalar: SimdElement, + LaneCount: SupportedLaneCount, + { + type Output = $out; + + #[inline] + #[must_use = "operator returns a new vector without mutating the inputs"] + fn $call(self, rhs: Self) -> Self::Output { + $macro_impl!(self, rhs, $inner, $scalar) + } + })* + } +} + +// A "TokenTree muncher": takes a set of scalar types `T = {};` +// type parameters for the ops it implements, `Op::fn` names, +// and a macro that expands into an expr, substituting in an intrinsic. +// It passes that to for_base_types, which expands an impl for the types, +// using the expanded expr in the function, and recurses with itself. +// +// tl;dr impls a set of ops::{Traits} for a set of types +macro_rules! for_base_ops { + ( + T = $types:tt; + type Lhs = Simd; + type Rhs = Simd; + type Output = $out:ident; + impl $op:ident::$call:ident + $inner:tt + $($rest:tt)* + ) => { + for_base_types! { + T = $types; + type Lhs = Simd; + type Rhs = Simd; + type Output = $out; + impl $op::$call + $inner + } + for_base_ops! { + T = $types; + type Lhs = Simd; + type Rhs = Simd; + type Output = $out; + $($rest)* + } + }; + ($($done:tt)*) => { + // Done. + } +} + +// Integers can always accept add, mul, sub, bitand, bitor, and bitxor. +// For all of these operations, simd_* intrinsics apply wrapping logic. +for_base_ops! { + T = (i8, i16, i32, i64, isize, u8, u16, u32, u64, usize); + type Lhs = Simd; + type Rhs = Simd; + type Output = Self; + + impl Add::add { + unsafe_base { simd_add } + } + + impl Mul::mul { + unsafe_base { simd_mul } + } + + impl Sub::sub { + unsafe_base { simd_sub } + } + + impl BitAnd::bitand { + unsafe_base { simd_and } + } + + impl BitOr::bitor { + unsafe_base { simd_or } + } + + impl BitXor::bitxor { + unsafe_base { simd_xor } + } + + impl Div::div { + int_divrem_guard { + const PANIC_ZERO: &'static str = "attempt to divide by zero"; + simd_div + } + } + + impl Rem::rem { + int_divrem_guard { + const PANIC_ZERO: &'static str = "attempt to calculate the remainder with a divisor of zero"; + simd_rem + } + } + + // The only question is how to handle shifts >= ::BITS? + // Our current solution uses wrapping logic. + impl Shl::shl { + wrap_bitshift { simd_shl } + } + + impl Shr::shr { + wrap_bitshift { + // This automatically monomorphizes to lshr or ashr, depending, + // so it's fine to use it for both UInts and SInts. + simd_shr + } + } +} + +// We don't need any special precautions here: +// Floats always accept arithmetic ops, but may become NaN. +for_base_ops! { + T = (f32, f64); + type Lhs = Simd; + type Rhs = Simd; + type Output = Self; + + impl Add::add { + unsafe_base { simd_add } + } + + impl Mul::mul { + unsafe_base { simd_mul } + } + + impl Sub::sub { + unsafe_base { simd_sub } + } + + impl Div::div { + unsafe_base { simd_div } + } + + impl Rem::rem { + unsafe_base { simd_rem } + } +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/ops/assign.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/ops/assign.rs new file mode 100644 index 000000000..d2b48614f --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/ops/assign.rs @@ -0,0 +1,124 @@ +//! Assignment operators +use super::*; +use core::ops::{AddAssign, MulAssign}; // commutative binary op-assignment +use core::ops::{BitAndAssign, BitOrAssign, BitXorAssign}; // commutative bit binary op-assignment +use core::ops::{DivAssign, RemAssign, SubAssign}; // non-commutative binary op-assignment +use core::ops::{ShlAssign, ShrAssign}; // non-commutative bit binary op-assignment + +// Arithmetic + +macro_rules! assign_ops { + ($(impl $assignTrait:ident for Simd + where + Self: $trait:ident, + { + fn $assign_call:ident(rhs: U) { + $call:ident + } + })*) => { + $(impl $assignTrait for Simd + where + Self: $trait, + T: SimdElement, + LaneCount: SupportedLaneCount, + { + #[inline] + fn $assign_call(&mut self, rhs: U) { + *self = self.$call(rhs); + } + })* + } +} + +assign_ops! { + // Arithmetic + impl AddAssign for Simd + where + Self: Add, + { + fn add_assign(rhs: U) { + add + } + } + + impl MulAssign for Simd + where + Self: Mul, + { + fn mul_assign(rhs: U) { + mul + } + } + + impl SubAssign for Simd + where + Self: Sub, + { + fn sub_assign(rhs: U) { + sub + } + } + + impl DivAssign for Simd + where + Self: Div, + { + fn div_assign(rhs: U) { + div + } + } + impl RemAssign for Simd + where + Self: Rem, + { + fn rem_assign(rhs: U) { + rem + } + } + + // Bitops + impl BitAndAssign for Simd + where + Self: BitAnd, + { + fn bitand_assign(rhs: U) { + bitand + } + } + + impl BitOrAssign for Simd + where + Self: BitOr, + { + fn bitor_assign(rhs: U) { + bitor + } + } + + impl BitXorAssign for Simd + where + Self: BitXor, + { + fn bitxor_assign(rhs: U) { + bitxor + } + } + + impl ShlAssign for Simd + where + Self: Shl, + { + fn shl_assign(rhs: U) { + shl + } + } + + impl ShrAssign for Simd + where + Self: Shr, + { + fn shr_assign(rhs: U) { + shr + } + } +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/ops/deref.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/ops/deref.rs new file mode 100644 index 000000000..9883a74c9 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/ops/deref.rs @@ -0,0 +1,124 @@ +//! This module hacks in "implicit deref" for Simd's operators. +//! Ideally, Rust would take care of this itself, +//! and method calls usually handle the LHS implicitly. +//! But this is not the case with arithmetic ops. +use super::*; + +macro_rules! deref_lhs { + (impl $trait:ident for $simd:ty { + fn $call:ident + }) => { + impl $trait<$simd> for &$simd + where + T: SimdElement, + $simd: $trait<$simd, Output = $simd>, + LaneCount: SupportedLaneCount, + { + type Output = Simd; + + #[inline] + #[must_use = "operator returns a new vector without mutating the inputs"] + fn $call(self, rhs: $simd) -> Self::Output { + (*self).$call(rhs) + } + } + }; +} + +macro_rules! deref_rhs { + (impl $trait:ident for $simd:ty { + fn $call:ident + }) => { + impl $trait<&$simd> for $simd + where + T: SimdElement, + $simd: $trait<$simd, Output = $simd>, + LaneCount: SupportedLaneCount, + { + type Output = Simd; + + #[inline] + #[must_use = "operator returns a new vector without mutating the inputs"] + fn $call(self, rhs: &$simd) -> Self::Output { + self.$call(*rhs) + } + } + }; +} + +macro_rules! deref_ops { + ($(impl $trait:ident for $simd:ty { + fn $call:ident + })*) => { + $( + deref_rhs! { + impl $trait for $simd { + fn $call + } + } + deref_lhs! { + impl $trait for $simd { + fn $call + } + } + impl<'lhs, 'rhs, T, const LANES: usize> $trait<&'rhs $simd> for &'lhs $simd + where + T: SimdElement, + $simd: $trait<$simd, Output = $simd>, + LaneCount: SupportedLaneCount, + { + type Output = $simd; + + #[inline] + #[must_use = "operator returns a new vector without mutating the inputs"] + fn $call(self, rhs: &$simd) -> Self::Output { + (*self).$call(*rhs) + } + } + )* + } +} + +deref_ops! { + // Arithmetic + impl Add for Simd { + fn add + } + + impl Mul for Simd { + fn mul + } + + impl Sub for Simd { + fn sub + } + + impl Div for Simd { + fn div + } + + impl Rem for Simd { + fn rem + } + + // Bitops + impl BitAnd for Simd { + fn bitand + } + + impl BitOr for Simd { + fn bitor + } + + impl BitXor for Simd { + fn bitxor + } + + impl Shl for Simd { + fn shl + } + + impl Shr for Simd { + fn shr + } +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/ops/unary.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/ops/unary.rs new file mode 100644 index 000000000..4ad022150 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/ops/unary.rs @@ -0,0 +1,78 @@ +use crate::simd::intrinsics; +use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount}; +use core::ops::{Neg, Not}; // unary ops + +macro_rules! neg { + ($(impl Neg for Simd<$scalar:ty, LANES>)*) => { + $(impl Neg for Simd<$scalar, LANES> + where + $scalar: SimdElement, + LaneCount: SupportedLaneCount, + { + type Output = Self; + + #[inline] + #[must_use = "operator returns a new vector without mutating the input"] + fn neg(self) -> Self::Output { + // Safety: `self` is a signed vector + unsafe { intrinsics::simd_neg(self) } + } + })* + } +} + +neg! { + impl Neg for Simd + + impl Neg for Simd + + impl Neg for Simd + + impl Neg for Simd + + impl Neg for Simd + + impl Neg for Simd + + impl Neg for Simd +} + +macro_rules! not { + ($(impl Not for Simd<$scalar:ty, LANES>)*) => { + $(impl Not for Simd<$scalar, LANES> + where + $scalar: SimdElement, + LaneCount: SupportedLaneCount, + { + type Output = Self; + + #[inline] + #[must_use = "operator returns a new vector without mutating the input"] + fn not(self) -> Self::Output { + self ^ (Simd::splat(!(0 as $scalar))) + } + })* + } +} + +not! { + impl Not for Simd + + impl Not for Simd + + impl Not for Simd + + impl Not for Simd + + impl Not for Simd + + impl Not for Simd + + impl Not for Simd + + impl Not for Simd + + impl Not for Simd + + impl Not for Simd +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/ord.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/ord.rs new file mode 100644 index 000000000..9a87bc2e3 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/ord.rs @@ -0,0 +1,213 @@ +use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount}; + +/// Parallel `PartialOrd`. +pub trait SimdPartialOrd: SimdPartialEq { + /// Test if each lane is less than the corresponding lane in `other`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn simd_lt(self, other: Self) -> Self::Mask; + + /// Test if each lane is less than or equal to the corresponding lane in `other`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn simd_le(self, other: Self) -> Self::Mask; + + /// Test if each lane is greater than the corresponding lane in `other`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn simd_gt(self, other: Self) -> Self::Mask; + + /// Test if each lane is greater than or equal to the corresponding lane in `other`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn simd_ge(self, other: Self) -> Self::Mask; +} + +/// Parallel `Ord`. +pub trait SimdOrd: SimdPartialOrd { + /// Returns the lane-wise maximum with `other`. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn simd_max(self, other: Self) -> Self; + + /// Returns the lane-wise minimum with `other`. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn simd_min(self, other: Self) -> Self; + + /// Restrict each lane to a certain interval. + /// + /// For each lane, returns `max` if `self` is greater than `max`, and `min` if `self` is + /// less than `min`. Otherwise returns `self`. + /// + /// # Panics + /// + /// Panics if `min > max` on any lane. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn simd_clamp(self, min: Self, max: Self) -> Self; +} + +macro_rules! impl_integer { + { $($integer:ty),* } => { + $( + impl SimdPartialOrd for Simd<$integer, LANES> + where + LaneCount: SupportedLaneCount, + { + #[inline] + fn simd_lt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_lt(self, other)) } + } + + #[inline] + fn simd_le(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_le(self, other)) } + } + + #[inline] + fn simd_gt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_gt(self, other)) } + } + + #[inline] + fn simd_ge(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_ge(self, other)) } + } + } + + impl SimdOrd for Simd<$integer, LANES> + where + LaneCount: SupportedLaneCount, + { + #[inline] + fn simd_max(self, other: Self) -> Self { + self.simd_lt(other).select(other, self) + } + + #[inline] + fn simd_min(self, other: Self) -> Self { + self.simd_gt(other).select(other, self) + } + + #[inline] + fn simd_clamp(self, min: Self, max: Self) -> Self { + assert!( + min.simd_le(max).all(), + "each lane in `min` must be less than or equal to the corresponding lane in `max`", + ); + self.simd_max(min).simd_min(max) + } + } + )* + } +} + +impl_integer! { u8, u16, u32, u64, usize, i8, i16, i32, i64, isize } + +macro_rules! impl_float { + { $($float:ty),* } => { + $( + impl SimdPartialOrd for Simd<$float, LANES> + where + LaneCount: SupportedLaneCount, + { + #[inline] + fn simd_lt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_lt(self, other)) } + } + + #[inline] + fn simd_le(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_le(self, other)) } + } + + #[inline] + fn simd_gt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_gt(self, other)) } + } + + #[inline] + fn simd_ge(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_ge(self, other)) } + } + } + )* + } +} + +impl_float! { f32, f64 } + +macro_rules! impl_mask { + { $($integer:ty),* } => { + $( + impl SimdPartialOrd for Mask<$integer, LANES> + where + LaneCount: SupportedLaneCount, + { + #[inline] + fn simd_lt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Self::from_int_unchecked(intrinsics::simd_lt(self.to_int(), other.to_int())) } + } + + #[inline] + fn simd_le(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Self::from_int_unchecked(intrinsics::simd_le(self.to_int(), other.to_int())) } + } + + #[inline] + fn simd_gt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Self::from_int_unchecked(intrinsics::simd_gt(self.to_int(), other.to_int())) } + } + + #[inline] + fn simd_ge(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Self::from_int_unchecked(intrinsics::simd_ge(self.to_int(), other.to_int())) } + } + } + + impl SimdOrd for Mask<$integer, LANES> + where + LaneCount: SupportedLaneCount, + { + #[inline] + fn simd_max(self, other: Self) -> Self { + self.simd_gt(other).select_mask(other, self) + } + + #[inline] + fn simd_min(self, other: Self) -> Self { + self.simd_lt(other).select_mask(other, self) + } + + #[inline] + fn simd_clamp(self, min: Self, max: Self) -> Self { + assert!( + min.simd_le(max).all(), + "each lane in `min` must be less than or equal to the corresponding lane in `max`", + ); + self.simd_max(min).simd_min(max) + } + } + )* + } +} + +impl_mask! { i8, i16, i32, i64, isize } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/select.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/select.rs new file mode 100644 index 000000000..065c5987d --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/select.rs @@ -0,0 +1,59 @@ +use crate::simd::intrinsics; +use crate::simd::{LaneCount, Mask, MaskElement, Simd, SimdElement, SupportedLaneCount}; + +impl Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + /// Choose lanes from two vectors. + /// + /// For each lane in the mask, choose the corresponding lane from `true_values` if + /// that lane mask is true, and `false_values` if that lane mask is false. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # use core::simd::{Simd, Mask}; + /// let a = Simd::from_array([0, 1, 2, 3]); + /// let b = Simd::from_array([4, 5, 6, 7]); + /// let mask = Mask::from_array([true, false, false, true]); + /// let c = mask.select(a, b); + /// assert_eq!(c.to_array(), [0, 5, 6, 3]); + /// ``` + #[inline] + #[must_use = "method returns a new vector and does not mutate the original inputs"] + pub fn select( + self, + true_values: Simd, + false_values: Simd, + ) -> Simd + where + U: SimdElement, + { + // Safety: The mask has been cast to a vector of integers, + // and the operands to select between are vectors of the same type and length. + unsafe { intrinsics::simd_select(self.to_int(), true_values, false_values) } + } + + /// Choose lanes from two masks. + /// + /// For each lane in the mask, choose the corresponding lane from `true_values` if + /// that lane mask is true, and `false_values` if that lane mask is false. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # use core::simd::Mask; + /// let a = Mask::::from_array([true, true, false, false]); + /// let b = Mask::::from_array([false, false, true, true]); + /// let mask = Mask::::from_array([true, false, false, true]); + /// let c = mask.select_mask(a, b); + /// assert_eq!(c.to_array(), [true, false, true, false]); + /// ``` + #[inline] + #[must_use = "method returns a new mask and does not mutate the original inputs"] + pub fn select_mask(self, true_values: Self, false_values: Self) -> Self { + self & true_values | !self & false_values + } +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/swizzle.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/swizzle.rs new file mode 100644 index 000000000..22999d249 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/swizzle.rs @@ -0,0 +1,385 @@ +use crate::simd::intrinsics; +use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount}; + +/// Constructs a new SIMD vector by copying elements from selected lanes in other vectors. +/// +/// When swizzling one vector, lanes are selected by a `const` array of `usize`, +/// like [`Swizzle`]. +/// +/// When swizzling two vectors, lanes are selected by a `const` array of [`Which`], +/// like [`Swizzle2`]. +/// +/// # Examples +/// +/// With a single SIMD vector, the const array specifies lane indices in that vector: +/// ``` +/// # #![feature(portable_simd)] +/// # use core::simd::{u32x2, u32x4, simd_swizzle}; +/// let v = u32x4::from_array([10, 11, 12, 13]); +/// +/// // Keeping the same size +/// let r: u32x4 = simd_swizzle!(v, [3, 0, 1, 2]); +/// assert_eq!(r.to_array(), [13, 10, 11, 12]); +/// +/// // Changing the number of lanes +/// let r: u32x2 = simd_swizzle!(v, [3, 1]); +/// assert_eq!(r.to_array(), [13, 11]); +/// ``` +/// +/// With two input SIMD vectors, the const array uses `Which` to specify the source of each index: +/// ``` +/// # #![feature(portable_simd)] +/// # use core::simd::{u32x2, u32x4, simd_swizzle, Which}; +/// use Which::{First, Second}; +/// let a = u32x4::from_array([0, 1, 2, 3]); +/// let b = u32x4::from_array([4, 5, 6, 7]); +/// +/// // Keeping the same size +/// let r: u32x4 = simd_swizzle!(a, b, [First(0), First(1), Second(2), Second(3)]); +/// assert_eq!(r.to_array(), [0, 1, 6, 7]); +/// +/// // Changing the number of lanes +/// let r: u32x2 = simd_swizzle!(a, b, [First(0), Second(0)]); +/// assert_eq!(r.to_array(), [0, 4]); +/// ``` +#[allow(unused_macros)] +pub macro simd_swizzle { + ( + $vector:expr, $index:expr $(,)? + ) => { + { + use $crate::simd::Swizzle; + struct Impl; + impl Swizzle for Impl { + const INDEX: [usize; {$index.len()}] = $index; + } + Impl::swizzle($vector) + } + }, + ( + $first:expr, $second:expr, $index:expr $(,)? + ) => { + { + use $crate::simd::{Which, Swizzle2}; + struct Impl; + impl Swizzle2 for Impl { + const INDEX: [Which; {$index.len()}] = $index; + } + Impl::swizzle2($first, $second) + } + } +} + +/// Specifies a lane index into one of two SIMD vectors. +/// +/// This is an input type for [Swizzle2] and helper macros like [simd_swizzle]. +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Which { + /// Index of a lane in the first input SIMD vector. + First(usize), + /// Index of a lane in the second input SIMD vector. + Second(usize), +} + +/// Create a vector from the elements of another vector. +pub trait Swizzle { + /// Map from the lanes of the input vector to the output vector. + const INDEX: [usize; OUTPUT_LANES]; + + /// Create a new vector from the lanes of `vector`. + /// + /// Lane `i` of the output is `vector[Self::INDEX[i]]`. + #[inline] + #[must_use = "method returns a new vector and does not mutate the original inputs"] + fn swizzle(vector: Simd) -> Simd + where + T: SimdElement, + LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, + { + // Safety: `vector` is a vector, and `INDEX_IMPL` is a const array of u32. + unsafe { intrinsics::simd_shuffle(vector, vector, Self::INDEX_IMPL) } + } +} + +/// Create a vector from the elements of two other vectors. +pub trait Swizzle2 { + /// Map from the lanes of the input vectors to the output vector + const INDEX: [Which; OUTPUT_LANES]; + + /// Create a new vector from the lanes of `first` and `second`. + /// + /// Lane `i` is `first[j]` when `Self::INDEX[i]` is `First(j)`, or `second[j]` when it is + /// `Second(j)`. + #[inline] + #[must_use = "method returns a new vector and does not mutate the original inputs"] + fn swizzle2( + first: Simd, + second: Simd, + ) -> Simd + where + T: SimdElement, + LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, + { + // Safety: `first` and `second` are vectors, and `INDEX_IMPL` is a const array of u32. + unsafe { intrinsics::simd_shuffle(first, second, Self::INDEX_IMPL) } + } +} + +/// The `simd_shuffle` intrinsic expects `u32`, so do error checking and conversion here. +/// This trait hides `INDEX_IMPL` from the public API. +trait SwizzleImpl { + const INDEX_IMPL: [u32; OUTPUT_LANES]; +} + +impl SwizzleImpl + for T +where + T: Swizzle + ?Sized, +{ + const INDEX_IMPL: [u32; OUTPUT_LANES] = { + let mut output = [0; OUTPUT_LANES]; + let mut i = 0; + while i < OUTPUT_LANES { + let index = Self::INDEX[i]; + assert!(index as u32 as usize == index); + assert!(index < INPUT_LANES, "source lane exceeds input lane count",); + output[i] = index as u32; + i += 1; + } + output + }; +} + +/// The `simd_shuffle` intrinsic expects `u32`, so do error checking and conversion here. +/// This trait hides `INDEX_IMPL` from the public API. +trait Swizzle2Impl { + const INDEX_IMPL: [u32; OUTPUT_LANES]; +} + +impl Swizzle2Impl + for T +where + T: Swizzle2 + ?Sized, +{ + const INDEX_IMPL: [u32; OUTPUT_LANES] = { + let mut output = [0; OUTPUT_LANES]; + let mut i = 0; + while i < OUTPUT_LANES { + let (offset, index) = match Self::INDEX[i] { + Which::First(index) => (false, index), + Which::Second(index) => (true, index), + }; + assert!(index < INPUT_LANES, "source lane exceeds input lane count",); + + // lanes are indexed by the first vector, then second vector + let index = if offset { index + INPUT_LANES } else { index }; + assert!(index as u32 as usize == index); + output[i] = index as u32; + i += 1; + } + output + }; +} + +impl Simd +where + T: SimdElement, + LaneCount: SupportedLaneCount, +{ + /// Reverse the order of the lanes in the vector. + #[inline] + #[must_use = "method returns a new vector and does not mutate the original inputs"] + pub fn reverse(self) -> Self { + const fn reverse_index() -> [usize; LANES] { + let mut index = [0; LANES]; + let mut i = 0; + while i < LANES { + index[i] = LANES - i - 1; + i += 1; + } + index + } + + struct Reverse; + + impl Swizzle for Reverse { + const INDEX: [usize; LANES] = reverse_index::(); + } + + Reverse::swizzle(self) + } + + /// Rotates the vector such that the first `OFFSET` elements of the slice move to the end + /// while the last `LANES - OFFSET` elements move to the front. After calling `rotate_lanes_left`, + /// the element previously in lane `OFFSET` will become the first element in the slice. + #[inline] + #[must_use = "method returns a new vector and does not mutate the original inputs"] + pub fn rotate_lanes_left(self) -> Self { + const fn rotate_index() -> [usize; LANES] { + let offset = OFFSET % LANES; + let mut index = [0; LANES]; + let mut i = 0; + while i < LANES { + index[i] = (i + offset) % LANES; + i += 1; + } + index + } + + struct Rotate; + + impl Swizzle for Rotate { + const INDEX: [usize; LANES] = rotate_index::(); + } + + Rotate::::swizzle(self) + } + + /// Rotates the vector such that the first `LANES - OFFSET` elements of the vector move to + /// the end while the last `OFFSET` elements move to the front. After calling `rotate_lanes_right`, + /// the element previously at index `LANES - OFFSET` will become the first element in the slice. + #[inline] + #[must_use = "method returns a new vector and does not mutate the original inputs"] + pub fn rotate_lanes_right(self) -> Self { + const fn rotate_index() -> [usize; LANES] { + let offset = LANES - OFFSET % LANES; + let mut index = [0; LANES]; + let mut i = 0; + while i < LANES { + index[i] = (i + offset) % LANES; + i += 1; + } + index + } + + struct Rotate; + + impl Swizzle for Rotate { + const INDEX: [usize; LANES] = rotate_index::(); + } + + Rotate::::swizzle(self) + } + + /// Interleave two vectors. + /// + /// Produces two vectors with lanes taken alternately from `self` and `other`. + /// + /// The first result contains the first `LANES / 2` lanes from `self` and `other`, + /// alternating, starting with the first lane of `self`. + /// + /// The second result contains the last `LANES / 2` lanes from `self` and `other`, + /// alternating, starting with the lane `LANES / 2` from the start of `self`. + /// + /// ``` + /// #![feature(portable_simd)] + /// # use core::simd::Simd; + /// let a = Simd::from_array([0, 1, 2, 3]); + /// let b = Simd::from_array([4, 5, 6, 7]); + /// let (x, y) = a.interleave(b); + /// assert_eq!(x.to_array(), [0, 4, 1, 5]); + /// assert_eq!(y.to_array(), [2, 6, 3, 7]); + /// ``` + #[inline] + #[must_use = "method returns a new vector and does not mutate the original inputs"] + pub fn interleave(self, other: Self) -> (Self, Self) { + const fn lo() -> [Which; LANES] { + let mut idx = [Which::First(0); LANES]; + let mut i = 0; + while i < LANES { + let offset = i / 2; + idx[i] = if i % 2 == 0 { + Which::First(offset) + } else { + Which::Second(offset) + }; + i += 1; + } + idx + } + const fn hi() -> [Which; LANES] { + let mut idx = [Which::First(0); LANES]; + let mut i = 0; + while i < LANES { + let offset = (LANES + i) / 2; + idx[i] = if i % 2 == 0 { + Which::First(offset) + } else { + Which::Second(offset) + }; + i += 1; + } + idx + } + + struct Lo; + struct Hi; + + impl Swizzle2 for Lo { + const INDEX: [Which; LANES] = lo::(); + } + + impl Swizzle2 for Hi { + const INDEX: [Which; LANES] = hi::(); + } + + (Lo::swizzle2(self, other), Hi::swizzle2(self, other)) + } + + /// Deinterleave two vectors. + /// + /// The first result takes every other lane of `self` and then `other`, starting with + /// the first lane. + /// + /// The second result takes every other lane of `self` and then `other`, starting with + /// the second lane. + /// + /// ``` + /// #![feature(portable_simd)] + /// # use core::simd::Simd; + /// let a = Simd::from_array([0, 4, 1, 5]); + /// let b = Simd::from_array([2, 6, 3, 7]); + /// let (x, y) = a.deinterleave(b); + /// assert_eq!(x.to_array(), [0, 1, 2, 3]); + /// assert_eq!(y.to_array(), [4, 5, 6, 7]); + /// ``` + #[inline] + #[must_use = "method returns a new vector and does not mutate the original inputs"] + pub fn deinterleave(self, other: Self) -> (Self, Self) { + const fn even() -> [Which; LANES] { + let mut idx = [Which::First(0); LANES]; + let mut i = 0; + while i < LANES / 2 { + idx[i] = Which::First(2 * i); + idx[i + LANES / 2] = Which::Second(2 * i); + i += 1; + } + idx + } + const fn odd() -> [Which; LANES] { + let mut idx = [Which::First(0); LANES]; + let mut i = 0; + while i < LANES / 2 { + idx[i] = Which::First(2 * i + 1); + idx[i + LANES / 2] = Which::Second(2 * i + 1); + i += 1; + } + idx + } + + struct Even; + struct Odd; + + impl Swizzle2 for Even { + const INDEX: [Which; LANES] = even::(); + } + + impl Swizzle2 for Odd { + const INDEX: [Which; LANES] = odd::(); + } + + (Even::swizzle2(self, other), Odd::swizzle2(self, other)) + } +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/to_bytes.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/to_bytes.rs new file mode 100644 index 000000000..b36b1a347 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/to_bytes.rs @@ -0,0 +1,41 @@ +macro_rules! impl_to_bytes { + { $ty:ty, $size:literal } => { + impl crate::simd::Simd<$ty, LANES> + where + crate::simd::LaneCount: crate::simd::SupportedLaneCount, + crate::simd::LaneCount<{{ $size * LANES }}>: crate::simd::SupportedLaneCount, + { + /// Return the memory representation of this integer as a byte array in native byte + /// order. + pub fn to_ne_bytes(self) -> crate::simd::Simd { + // Safety: transmuting between vectors is safe + unsafe { core::mem::transmute_copy(&self) } + } + + /// Create a native endian integer value from its memory representation as a byte array + /// in native endianness. + pub fn from_ne_bytes(bytes: crate::simd::Simd) -> Self { + // Safety: transmuting between vectors is safe + unsafe { core::mem::transmute_copy(&bytes) } + } + } + } +} + +impl_to_bytes! { u8, 1 } +impl_to_bytes! { u16, 2 } +impl_to_bytes! { u32, 4 } +impl_to_bytes! { u64, 8 } +#[cfg(target_pointer_width = "32")] +impl_to_bytes! { usize, 4 } +#[cfg(target_pointer_width = "64")] +impl_to_bytes! { usize, 8 } + +impl_to_bytes! { i8, 1 } +impl_to_bytes! { i16, 2 } +impl_to_bytes! { i32, 4 } +impl_to_bytes! { i64, 8 } +#[cfg(target_pointer_width = "32")] +impl_to_bytes! { isize, 4 } +#[cfg(target_pointer_width = "64")] +impl_to_bytes! { isize, 8 } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/vector.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/vector.rs new file mode 100644 index 000000000..78f56402e --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/vector.rs @@ -0,0 +1,742 @@ +mod float; +mod int; +mod uint; + +pub use float::*; +pub use int::*; +pub use uint::*; + +// Vectors of pointers are not for public use at the current time. +pub(crate) mod ptr; + +use crate::simd::{ + intrinsics, LaneCount, Mask, MaskElement, SimdPartialOrd, SupportedLaneCount, Swizzle, +}; + +/// A SIMD vector of `LANES` elements of type `T`. `Simd` has the same shape as [`[T; N]`](array), but operates like `T`. +/// +/// Two vectors of the same type and length will, by convention, support the operators (+, *, etc.) that `T` does. +/// These take the lanes at each index on the left-hand side and right-hand side, perform the operation, +/// and return the result in the same lane in a vector of equal size. For a given operator, this is equivalent to zipping +/// the two arrays together and mapping the operator over each lane. +/// +/// ```rust +/// # #![feature(array_zip, portable_simd)] +/// # use core::simd::{Simd}; +/// let a0: [i32; 4] = [-2, 0, 2, 4]; +/// let a1 = [10, 9, 8, 7]; +/// let zm_add = a0.zip(a1).map(|(lhs, rhs)| lhs + rhs); +/// let zm_mul = a0.zip(a1).map(|(lhs, rhs)| lhs * rhs); +/// +/// // `Simd` implements `From<[T; N]> +/// let (v0, v1) = (Simd::from(a0), Simd::from(a1)); +/// // Which means arrays implement `Into>`. +/// assert_eq!(v0 + v1, zm_add.into()); +/// assert_eq!(v0 * v1, zm_mul.into()); +/// ``` +/// +/// `Simd` with integers has the quirk that these operations are also inherently wrapping, as if `T` was [`Wrapping`]. +/// Thus, `Simd` does not implement `wrapping_add`, because that is the default behavior. +/// This means there is no warning on overflows, even in "debug" builds. +/// For most applications where `Simd` is appropriate, it is "not a bug" to wrap, +/// and even "debug builds" are unlikely to tolerate the loss of performance. +/// You may want to consider using explicitly checked arithmetic if such is required. +/// Division by zero still causes a panic, so you may want to consider using floating point numbers if that is unacceptable. +/// +/// [`Wrapping`]: core::num::Wrapping +/// +/// # Layout +/// `Simd` has a layout similar to `[T; N]` (identical "shapes"), but with a greater alignment. +/// `[T; N]` is aligned to `T`, but `Simd` will have an alignment based on both `T` and `N`. +/// It is thus sound to [`transmute`] `Simd` to `[T; N]`, and will typically optimize to zero cost, +/// but the reverse transmutation is more likely to require a copy the compiler cannot simply elide. +/// +/// # ABI "Features" +/// Due to Rust's safety guarantees, `Simd` is currently passed to and from functions via memory, not SIMD registers, +/// except as an optimization. `#[inline]` hints are recommended on functions that accept `Simd` or return it. +/// The need for this may be corrected in the future. +/// +/// # Safe SIMD with Unsafe Rust +/// +/// Operations with `Simd` are typically safe, but there are many reasons to want to combine SIMD with `unsafe` code. +/// Care must be taken to respect differences between `Simd` and other types it may be transformed into or derived from. +/// In particular, the layout of `Simd` may be similar to `[T; N]`, and may allow some transmutations, +/// but references to `[T; N]` are not interchangeable with those to `Simd`. +/// Thus, when using `unsafe` Rust to read and write `Simd` through [raw pointers], it is a good idea to first try with +/// [`read_unaligned`] and [`write_unaligned`]. This is because: +/// - [`read`] and [`write`] require full alignment (in this case, `Simd`'s alignment) +/// - the likely source for reading or destination for writing `Simd` is [`[T]`](slice) and similar types, aligned to `T` +/// - combining these actions would violate the `unsafe` contract and explode the program into a puff of **undefined behavior** +/// - the compiler can implicitly adjust layouts to make unaligned reads or writes fully aligned if it sees the optimization +/// - most contemporary processors suffer no performance penalty for "unaligned" reads and writes that are aligned at runtime +/// +/// By imposing less obligations, unaligned functions are less likely to make the program unsound, +/// and may be just as fast as stricter alternatives. +/// When trying to guarantee alignment, [`[T]::as_simd`][as_simd] is an option for converting `[T]` to `[Simd]`, +/// and allows soundly operating on an aligned SIMD body, but it may cost more time when handling the scalar head and tail. +/// If these are not sufficient, then it is most ideal to design data structures to be already aligned +/// to the `Simd` you wish to use before using `unsafe` Rust to read or write. +/// More conventional ways to compensate for these facts, like materializing `Simd` to or from an array first, +/// are handled by safe methods like [`Simd::from_array`] and [`Simd::from_slice`]. +/// +/// [`transmute`]: core::mem::transmute +/// [raw pointers]: pointer +/// [`read_unaligned`]: pointer::read_unaligned +/// [`write_unaligned`]: pointer::write_unaligned +/// [`read`]: pointer::read +/// [`write`]: pointer::write +/// [as_simd]: slice::as_simd +#[repr(simd)] +pub struct Simd([T; LANES]) +where + T: SimdElement, + LaneCount: SupportedLaneCount; + +impl Simd +where + LaneCount: SupportedLaneCount, + T: SimdElement, +{ + /// Number of lanes in this vector. + pub const LANES: usize = LANES; + + /// Returns the number of lanes in this SIMD vector. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # use core::simd::u32x4; + /// let v = u32x4::splat(0); + /// assert_eq!(v.lanes(), 4); + /// ``` + pub const fn lanes(&self) -> usize { + LANES + } + + /// Constructs a new SIMD vector with all lanes set to the given value. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # use core::simd::u32x4; + /// let v = u32x4::splat(8); + /// assert_eq!(v.as_array(), &[8, 8, 8, 8]); + /// ``` + pub fn splat(value: T) -> Self { + // This is preferred over `[value; LANES]`, since it's explicitly a splat: + // https://github.com/rust-lang/rust/issues/97804 + struct Splat; + impl Swizzle<1, LANES> for Splat { + const INDEX: [usize; LANES] = [0; LANES]; + } + Splat::swizzle(Simd::::from([value])) + } + + /// Returns an array reference containing the entire SIMD vector. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # use core::simd::{Simd, u64x4}; + /// let v: u64x4 = Simd::from_array([0, 1, 2, 3]); + /// assert_eq!(v.as_array(), &[0, 1, 2, 3]); + /// ``` + pub const fn as_array(&self) -> &[T; LANES] { + &self.0 + } + + /// Returns a mutable array reference containing the entire SIMD vector. + pub fn as_mut_array(&mut self) -> &mut [T; LANES] { + &mut self.0 + } + + /// Converts an array to a SIMD vector. + pub const fn from_array(array: [T; LANES]) -> Self { + Self(array) + } + + /// Converts a SIMD vector to an array. + pub const fn to_array(self) -> [T; LANES] { + self.0 + } + + /// Converts a slice to a SIMD vector containing `slice[..LANES]`. + /// + /// # Panics + /// + /// Panics if the slice's length is less than the vector's `Simd::LANES`. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # use core::simd::u32x4; + /// let source = vec![1, 2, 3, 4, 5, 6]; + /// let v = u32x4::from_slice(&source); + /// assert_eq!(v.as_array(), &[1, 2, 3, 4]); + /// ``` + #[must_use] + pub const fn from_slice(slice: &[T]) -> Self { + assert!(slice.len() >= LANES, "slice length must be at least the number of lanes"); + let mut array = [slice[0]; LANES]; + let mut i = 0; + while i < LANES { + array[i] = slice[i]; + i += 1; + } + Self(array) + } + + /// Performs lanewise conversion of a SIMD vector's elements to another SIMD-valid type. + /// + /// This follows the semantics of Rust's `as` conversion for casting + /// integers to unsigned integers (interpreting as the other type, so `-1` to `MAX`), + /// and from floats to integers (truncating, or saturating at the limits) for each lane, + /// or vice versa. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # use core::simd::Simd; + /// let floats: Simd = Simd::from_array([1.9, -4.5, f32::INFINITY, f32::NAN]); + /// let ints = floats.cast::(); + /// assert_eq!(ints, Simd::from_array([1, -4, i32::MAX, 0])); + /// + /// // Formally equivalent, but `Simd::cast` can optimize better. + /// assert_eq!(ints, Simd::from_array(floats.to_array().map(|x| x as i32))); + /// + /// // The float conversion does not round-trip. + /// let floats_again = ints.cast(); + /// assert_ne!(floats, floats_again); + /// assert_eq!(floats_again, Simd::from_array([1.0, -4.0, 2147483647.0, 0.0])); + /// ``` + #[must_use] + #[inline] + pub fn cast(self) -> Simd { + // Safety: The input argument is a vector of a valid SIMD element type. + unsafe { intrinsics::simd_as(self) } + } + + /// Rounds toward zero and converts to the same-width integer type, assuming that + /// the value is finite and fits in that type. + /// + /// # Safety + /// The value must: + /// + /// * Not be NaN + /// * Not be infinite + /// * Be representable in the return type, after truncating off its fractional part + /// + /// If these requirements are infeasible or costly, consider using the safe function [cast], + /// which saturates on conversion. + /// + /// [cast]: Simd::cast + #[inline] + pub unsafe fn to_int_unchecked(self) -> Simd + where + T: core::convert::FloatToInt, + I: SimdElement, + { + // Safety: `self` is a vector, and `FloatToInt` ensures the type can be casted to + // an integer. + unsafe { intrinsics::simd_cast(self) } + } + + /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector. + /// If an index is out-of-bounds, the lane is instead selected from the `or` vector. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # use core::simd::Simd; + /// let vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; + /// let idxs = Simd::from_array([9, 3, 0, 5]); + /// let alt = Simd::from_array([-5, -4, -3, -2]); + /// + /// let result = Simd::gather_or(&vec, idxs, alt); // Note the lane that is out-of-bounds. + /// assert_eq!(result, Simd::from_array([-5, 13, 10, 15])); + /// ``` + #[must_use] + #[inline] + pub fn gather_or(slice: &[T], idxs: Simd, or: Self) -> Self { + Self::gather_select(slice, Mask::splat(true), idxs, or) + } + + /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector. + /// If an index is out-of-bounds, the lane is set to the default value for the type. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # use core::simd::Simd; + /// let vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; + /// let idxs = Simd::from_array([9, 3, 0, 5]); + /// + /// let result = Simd::gather_or_default(&vec, idxs); // Note the lane that is out-of-bounds. + /// assert_eq!(result, Simd::from_array([0, 13, 10, 15])); + /// ``` + #[must_use] + #[inline] + pub fn gather_or_default(slice: &[T], idxs: Simd) -> Self + where + T: Default, + { + Self::gather_or(slice, idxs, Self::splat(T::default())) + } + + /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector. + /// The mask `enable`s all `true` lanes and disables all `false` lanes. + /// If an index is disabled or is out-of-bounds, the lane is selected from the `or` vector. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # use core::simd::{Simd, Mask}; + /// let vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; + /// let idxs = Simd::from_array([9, 3, 0, 5]); + /// let alt = Simd::from_array([-5, -4, -3, -2]); + /// let enable = Mask::from_array([true, true, true, false]); // Note the mask of the last lane. + /// + /// let result = Simd::gather_select(&vec, enable, idxs, alt); // Note the lane that is out-of-bounds. + /// assert_eq!(result, Simd::from_array([-5, 13, 10, -2])); + /// ``` + #[must_use] + #[inline] + pub fn gather_select( + slice: &[T], + enable: Mask, + idxs: Simd, + or: Self, + ) -> Self { + let enable: Mask = enable & idxs.simd_lt(Simd::splat(slice.len())); + // Safety: We have masked-off out-of-bounds lanes. + unsafe { Self::gather_select_unchecked(slice, enable, idxs, or) } + } + + /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector. + /// The mask `enable`s all `true` lanes and disables all `false` lanes. + /// If an index is disabled, the lane is selected from the `or` vector. + /// + /// # Safety + /// + /// Calling this function with an `enable`d out-of-bounds index is *[undefined behavior]* + /// even if the resulting value is not used. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, SimdPartialOrd, Mask}; + /// let vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; + /// let idxs = Simd::from_array([9, 3, 0, 5]); + /// let alt = Simd::from_array([-5, -4, -3, -2]); + /// let enable = Mask::from_array([true, true, true, false]); // Note the final mask lane. + /// // If this mask was used to gather, it would be unsound. Let's fix that. + /// let enable = enable & idxs.simd_lt(Simd::splat(vec.len())); + /// + /// // We have masked the OOB lane, so it's safe to gather now. + /// let result = unsafe { Simd::gather_select_unchecked(&vec, enable, idxs, alt) }; + /// assert_eq!(result, Simd::from_array([-5, 13, 10, -2])); + /// ``` + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html + #[must_use] + #[inline] + pub unsafe fn gather_select_unchecked( + slice: &[T], + enable: Mask, + idxs: Simd, + or: Self, + ) -> Self { + let base_ptr = crate::simd::ptr::SimdConstPtr::splat(slice.as_ptr()); + // Ferris forgive me, I have done pointer arithmetic here. + let ptrs = base_ptr.wrapping_add(idxs); + // Safety: The ptrs have been bounds-masked to prevent memory-unsafe reads insha'allah + unsafe { intrinsics::simd_gather(or, ptrs, enable.to_int()) } + } + + /// Writes the values in a SIMD vector to potentially discontiguous indices in `slice`. + /// If two lanes in the scattered vector would write to the same index + /// only the last lane is guaranteed to actually be written. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # use core::simd::Simd; + /// let mut vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; + /// let idxs = Simd::from_array([9, 3, 0, 0]); + /// let vals = Simd::from_array([-27, 82, -41, 124]); + /// + /// vals.scatter(&mut vec, idxs); // index 0 receives two writes. + /// assert_eq!(vec, vec![124, 11, 12, 82, 14, 15, 16, 17, 18]); + /// ``` + #[inline] + pub fn scatter(self, slice: &mut [T], idxs: Simd) { + self.scatter_select(slice, Mask::splat(true), idxs) + } + + /// Writes the values in a SIMD vector to multiple potentially discontiguous indices in `slice`. + /// The mask `enable`s all `true` lanes and disables all `false` lanes. + /// If an enabled index is out-of-bounds, the lane is not written. + /// If two enabled lanes in the scattered vector would write to the same index, + /// only the last lane is guaranteed to actually be written. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, Mask}; + /// let mut vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; + /// let idxs = Simd::from_array([9, 3, 0, 0]); + /// let vals = Simd::from_array([-27, 82, -41, 124]); + /// let enable = Mask::from_array([true, true, true, false]); // Note the mask of the last lane. + /// + /// vals.scatter_select(&mut vec, enable, idxs); // index 0's second write is masked, thus omitted. + /// assert_eq!(vec, vec![-41, 11, 12, 82, 14, 15, 16, 17, 18]); + /// ``` + #[inline] + pub fn scatter_select( + self, + slice: &mut [T], + enable: Mask, + idxs: Simd, + ) { + let enable: Mask = enable & idxs.simd_lt(Simd::splat(slice.len())); + // Safety: We have masked-off out-of-bounds lanes. + unsafe { self.scatter_select_unchecked(slice, enable, idxs) } + } + + /// Writes the values in a SIMD vector to multiple potentially discontiguous indices in `slice`. + /// The mask `enable`s all `true` lanes and disables all `false` lanes. + /// If two enabled lanes in the scattered vector would write to the same index, + /// only the last lane is guaranteed to actually be written. + /// + /// # Safety + /// + /// Calling this function with an enabled out-of-bounds index is *[undefined behavior]*, + /// and may lead to memory corruption. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, SimdPartialOrd, Mask}; + /// let mut vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; + /// let idxs = Simd::from_array([9, 3, 0, 0]); + /// let vals = Simd::from_array([-27, 82, -41, 124]); + /// let enable = Mask::from_array([true, true, true, false]); // Note the mask of the last lane. + /// // If this mask was used to scatter, it would be unsound. Let's fix that. + /// let enable = enable & idxs.simd_lt(Simd::splat(vec.len())); + /// + /// // We have masked the OOB lane, so it's safe to scatter now. + /// unsafe { vals.scatter_select_unchecked(&mut vec, enable, idxs); } + /// // index 0's second write is masked, thus was omitted. + /// assert_eq!(vec, vec![-41, 11, 12, 82, 14, 15, 16, 17, 18]); + /// ``` + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html + #[inline] + pub unsafe fn scatter_select_unchecked( + self, + slice: &mut [T], + enable: Mask, + idxs: Simd, + ) { + // Safety: This block works with *mut T derived from &mut 'a [T], + // which means it is delicate in Rust's borrowing model, circa 2021: + // &mut 'a [T] asserts uniqueness, so deriving &'a [T] invalidates live *mut Ts! + // Even though this block is largely safe methods, it must be exactly this way + // to prevent invalidating the raw ptrs while they're live. + // Thus, entering this block requires all values to use being already ready: + // 0. idxs we want to write to, which are used to construct the mask. + // 1. enable, which depends on an initial &'a [T] and the idxs. + // 2. actual values to scatter (self). + // 3. &mut [T] which will become our base ptr. + unsafe { + // Now Entering ☢️ *mut T Zone + let base_ptr = crate::simd::ptr::SimdMutPtr::splat(slice.as_mut_ptr()); + // Ferris forgive me, I have done pointer arithmetic here. + let ptrs = base_ptr.wrapping_add(idxs); + // The ptrs have been bounds-masked to prevent memory-unsafe writes insha'allah + intrinsics::simd_scatter(self, ptrs, enable.to_int()) + // Cleared ☢️ *mut T Zone + } + } +} + +impl Copy for Simd +where + T: SimdElement, + LaneCount: SupportedLaneCount, +{ +} + +impl Clone for Simd +where + T: SimdElement, + LaneCount: SupportedLaneCount, +{ + fn clone(&self) -> Self { + *self + } +} + +impl Default for Simd +where + LaneCount: SupportedLaneCount, + T: SimdElement + Default, +{ + #[inline] + fn default() -> Self { + Self::splat(T::default()) + } +} + +impl PartialEq for Simd +where + LaneCount: SupportedLaneCount, + T: SimdElement + PartialEq, +{ + #[inline] + fn eq(&self, other: &Self) -> bool { + // Safety: All SIMD vectors are SimdPartialEq, and the comparison produces a valid mask. + let mask = unsafe { + let tfvec: Simd<::Mask, LANES> = intrinsics::simd_eq(*self, *other); + Mask::from_int_unchecked(tfvec) + }; + + // Two vectors are equal if all lanes tested true for vertical equality. + mask.all() + } + + #[allow(clippy::partialeq_ne_impl)] + #[inline] + fn ne(&self, other: &Self) -> bool { + // Safety: All SIMD vectors are SimdPartialEq, and the comparison produces a valid mask. + let mask = unsafe { + let tfvec: Simd<::Mask, LANES> = intrinsics::simd_ne(*self, *other); + Mask::from_int_unchecked(tfvec) + }; + + // Two vectors are non-equal if any lane tested true for vertical non-equality. + mask.any() + } +} + +impl PartialOrd for Simd +where + LaneCount: SupportedLaneCount, + T: SimdElement + PartialOrd, +{ + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + // TODO use SIMD equality + self.to_array().partial_cmp(other.as_ref()) + } +} + +impl Eq for Simd +where + LaneCount: SupportedLaneCount, + T: SimdElement + Eq, +{ +} + +impl Ord for Simd +where + LaneCount: SupportedLaneCount, + T: SimdElement + Ord, +{ + #[inline] + fn cmp(&self, other: &Self) -> core::cmp::Ordering { + // TODO use SIMD equality + self.to_array().cmp(other.as_ref()) + } +} + +impl core::hash::Hash for Simd +where + LaneCount: SupportedLaneCount, + T: SimdElement + core::hash::Hash, +{ + #[inline] + fn hash(&self, state: &mut H) + where + H: core::hash::Hasher, + { + self.as_array().hash(state) + } +} + +// array references +impl AsRef<[T; LANES]> for Simd +where + LaneCount: SupportedLaneCount, + T: SimdElement, +{ + #[inline] + fn as_ref(&self) -> &[T; LANES] { + &self.0 + } +} + +impl AsMut<[T; LANES]> for Simd +where + LaneCount: SupportedLaneCount, + T: SimdElement, +{ + #[inline] + fn as_mut(&mut self) -> &mut [T; LANES] { + &mut self.0 + } +} + +// slice references +impl AsRef<[T]> for Simd +where + LaneCount: SupportedLaneCount, + T: SimdElement, +{ + #[inline] + fn as_ref(&self) -> &[T] { + &self.0 + } +} + +impl AsMut<[T]> for Simd +where + LaneCount: SupportedLaneCount, + T: SimdElement, +{ + #[inline] + fn as_mut(&mut self) -> &mut [T] { + &mut self.0 + } +} + +// vector/array conversion +impl From<[T; LANES]> for Simd +where + LaneCount: SupportedLaneCount, + T: SimdElement, +{ + fn from(array: [T; LANES]) -> Self { + Self(array) + } +} + +impl From> for [T; LANES] +where + LaneCount: SupportedLaneCount, + T: SimdElement, +{ + fn from(vector: Simd) -> Self { + vector.to_array() + } +} + +mod sealed { + pub trait Sealed {} +} +use sealed::Sealed; + +/// Marker trait for types that may be used as SIMD vector elements. +/// +/// # Safety +/// This trait, when implemented, asserts the compiler can monomorphize +/// `#[repr(simd)]` structs with the marked type as an element. +/// Strictly, it is valid to impl if the vector will not be miscompiled. +/// Practically, it is user-unfriendly to impl it if the vector won't compile, +/// even when no soundness guarantees are broken by allowing the user to try. +pub unsafe trait SimdElement: Sealed + Copy { + /// The mask element type corresponding to this element type. + type Mask: MaskElement; +} + +impl Sealed for u8 {} + +// Safety: u8 is a valid SIMD element type, and is supported by this API +unsafe impl SimdElement for u8 { + type Mask = i8; +} + +impl Sealed for u16 {} + +// Safety: u16 is a valid SIMD element type, and is supported by this API +unsafe impl SimdElement for u16 { + type Mask = i16; +} + +impl Sealed for u32 {} + +// Safety: u32 is a valid SIMD element type, and is supported by this API +unsafe impl SimdElement for u32 { + type Mask = i32; +} + +impl Sealed for u64 {} + +// Safety: u64 is a valid SIMD element type, and is supported by this API +unsafe impl SimdElement for u64 { + type Mask = i64; +} + +impl Sealed for usize {} + +// Safety: usize is a valid SIMD element type, and is supported by this API +unsafe impl SimdElement for usize { + type Mask = isize; +} + +impl Sealed for i8 {} + +// Safety: i8 is a valid SIMD element type, and is supported by this API +unsafe impl SimdElement for i8 { + type Mask = i8; +} + +impl Sealed for i16 {} + +// Safety: i16 is a valid SIMD element type, and is supported by this API +unsafe impl SimdElement for i16 { + type Mask = i16; +} + +impl Sealed for i32 {} + +// Safety: i32 is a valid SIMD element type, and is supported by this API +unsafe impl SimdElement for i32 { + type Mask = i32; +} + +impl Sealed for i64 {} + +// Safety: i64 is a valid SIMD element type, and is supported by this API +unsafe impl SimdElement for i64 { + type Mask = i64; +} + +impl Sealed for isize {} + +// Safety: isize is a valid SIMD element type, and is supported by this API +unsafe impl SimdElement for isize { + type Mask = isize; +} + +impl Sealed for f32 {} + +// Safety: f32 is a valid SIMD element type, and is supported by this API +unsafe impl SimdElement for f32 { + type Mask = i32; +} + +impl Sealed for f64 {} + +// Safety: f64 is a valid SIMD element type, and is supported by this API +unsafe impl SimdElement for f64 { + type Mask = i64; +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/vector/float.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/vector/float.rs new file mode 100644 index 000000000..f836c99b1 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/vector/float.rs @@ -0,0 +1,24 @@ +#![allow(non_camel_case_types)] + +use crate::simd::Simd; + +/// A 64-bit SIMD vector with two elements of type `f32`. +pub type f32x2 = Simd; + +/// A 128-bit SIMD vector with four elements of type `f32`. +pub type f32x4 = Simd; + +/// A 256-bit SIMD vector with eight elements of type `f32`. +pub type f32x8 = Simd; + +/// A 512-bit SIMD vector with 16 elements of type `f32`. +pub type f32x16 = Simd; + +/// A 128-bit SIMD vector with two elements of type `f64`. +pub type f64x2 = Simd; + +/// A 256-bit SIMD vector with four elements of type `f64`. +pub type f64x4 = Simd; + +/// A 512-bit SIMD vector with eight elements of type `f64`. +pub type f64x8 = Simd; diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/vector/int.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/vector/int.rs new file mode 100644 index 000000000..20e56c7dc --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/vector/int.rs @@ -0,0 +1,63 @@ +#![allow(non_camel_case_types)] + +use crate::simd::Simd; + +/// A SIMD vector with two elements of type `isize`. +pub type isizex2 = Simd; + +/// A SIMD vector with four elements of type `isize`. +pub type isizex4 = Simd; + +/// A SIMD vector with eight elements of type `isize`. +pub type isizex8 = Simd; + +/// A 32-bit SIMD vector with two elements of type `i16`. +pub type i16x2 = Simd; + +/// A 64-bit SIMD vector with four elements of type `i16`. +pub type i16x4 = Simd; + +/// A 128-bit SIMD vector with eight elements of type `i16`. +pub type i16x8 = Simd; + +/// A 256-bit SIMD vector with 16 elements of type `i16`. +pub type i16x16 = Simd; + +/// A 512-bit SIMD vector with 32 elements of type `i16`. +pub type i16x32 = Simd; + +/// A 64-bit SIMD vector with two elements of type `i32`. +pub type i32x2 = Simd; + +/// A 128-bit SIMD vector with four elements of type `i32`. +pub type i32x4 = Simd; + +/// A 256-bit SIMD vector with eight elements of type `i32`. +pub type i32x8 = Simd; + +/// A 512-bit SIMD vector with 16 elements of type `i32`. +pub type i32x16 = Simd; + +/// A 128-bit SIMD vector with two elements of type `i64`. +pub type i64x2 = Simd; + +/// A 256-bit SIMD vector with four elements of type `i64`. +pub type i64x4 = Simd; + +/// A 512-bit SIMD vector with eight elements of type `i64`. +pub type i64x8 = Simd; + +/// A 32-bit SIMD vector with four elements of type `i8`. +pub type i8x4 = Simd; + +/// A 64-bit SIMD vector with eight elements of type `i8`. +pub type i8x8 = Simd; + +/// A 128-bit SIMD vector with 16 elements of type `i8`. +pub type i8x16 = Simd; + +/// A 256-bit SIMD vector with 32 elements of type `i8`. +pub type i8x32 = Simd; + +/// A 512-bit SIMD vector with 64 elements of type `i8`. +pub type i8x64 = Simd; diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/vector/ptr.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/vector/ptr.rs new file mode 100644 index 000000000..fa756344d --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/vector/ptr.rs @@ -0,0 +1,51 @@ +//! Private implementation details of public gather/scatter APIs. +use crate::simd::intrinsics; +use crate::simd::{LaneCount, Simd, SupportedLaneCount}; + +/// A vector of *const T. +#[derive(Debug, Copy, Clone)] +#[repr(simd)] +pub(crate) struct SimdConstPtr([*const T; LANES]); + +impl SimdConstPtr +where + LaneCount: SupportedLaneCount, + T: Sized, +{ + #[inline] + #[must_use] + pub fn splat(ptr: *const T) -> Self { + Self([ptr; LANES]) + } + + #[inline] + #[must_use] + pub fn wrapping_add(self, addend: Simd) -> Self { + // Safety: this intrinsic doesn't have a precondition + unsafe { intrinsics::simd_arith_offset(self, addend) } + } +} + +/// A vector of *mut T. Be very careful around potential aliasing. +#[derive(Debug, Copy, Clone)] +#[repr(simd)] +pub(crate) struct SimdMutPtr([*mut T; LANES]); + +impl SimdMutPtr +where + LaneCount: SupportedLaneCount, + T: Sized, +{ + #[inline] + #[must_use] + pub fn splat(ptr: *mut T) -> Self { + Self([ptr; LANES]) + } + + #[inline] + #[must_use] + pub fn wrapping_add(self, addend: Simd) -> Self { + // Safety: this intrinsic doesn't have a precondition + unsafe { intrinsics::simd_arith_offset(self, addend) } + } +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/vector/uint.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/vector/uint.rs new file mode 100644 index 000000000..b4a69c443 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/vector/uint.rs @@ -0,0 +1,63 @@ +#![allow(non_camel_case_types)] + +use crate::simd::Simd; + +/// A SIMD vector with two elements of type `usize`. +pub type usizex2 = Simd; + +/// A SIMD vector with four elements of type `usize`. +pub type usizex4 = Simd; + +/// A SIMD vector with eight elements of type `usize`. +pub type usizex8 = Simd; + +/// A 32-bit SIMD vector with two elements of type `u16`. +pub type u16x2 = Simd; + +/// A 64-bit SIMD vector with four elements of type `u16`. +pub type u16x4 = Simd; + +/// A 128-bit SIMD vector with eight elements of type `u16`. +pub type u16x8 = Simd; + +/// A 256-bit SIMD vector with 16 elements of type `u16`. +pub type u16x16 = Simd; + +/// A 512-bit SIMD vector with 32 elements of type `u16`. +pub type u16x32 = Simd; + +/// A 64-bit SIMD vector with two elements of type `u32`. +pub type u32x2 = Simd; + +/// A 128-bit SIMD vector with four elements of type `u32`. +pub type u32x4 = Simd; + +/// A 256-bit SIMD vector with eight elements of type `u32`. +pub type u32x8 = Simd; + +/// A 512-bit SIMD vector with 16 elements of type `u32`. +pub type u32x16 = Simd; + +/// A 128-bit SIMD vector with two elements of type `u64`. +pub type u64x2 = Simd; + +/// A 256-bit SIMD vector with four elements of type `u64`. +pub type u64x4 = Simd; + +/// A 512-bit SIMD vector with eight elements of type `u64`. +pub type u64x8 = Simd; + +/// A 32-bit SIMD vector with four elements of type `u8`. +pub type u8x4 = Simd; + +/// A 64-bit SIMD vector with eight elements of type `u8`. +pub type u8x8 = Simd; + +/// A 128-bit SIMD vector with 16 elements of type `u8`. +pub type u8x16 = Simd; + +/// A 256-bit SIMD vector with 32 elements of type `u8`. +pub type u8x32 = Simd; + +/// A 512-bit SIMD vector with 64 elements of type `u8`. +pub type u8x64 = Simd; diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/vendor.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/vendor.rs new file mode 100644 index 000000000..9fb70218c --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/vendor.rs @@ -0,0 +1,31 @@ +/// Provides implementations of `From<$a> for $b` and `From<$b> for $a` that transmutes the value. +#[allow(unused)] +macro_rules! from_transmute { + { unsafe $a:ty => $b:ty } => { + from_transmute!{ @impl $a => $b } + from_transmute!{ @impl $b => $a } + }; + { @impl $from:ty => $to:ty } => { + impl core::convert::From<$from> for $to { + #[inline] + fn from(value: $from) -> $to { + // Safety: transmuting between vectors is safe, but the caller of this macro + // checks the invariants + unsafe { core::mem::transmute(value) } + } + } + }; +} + +/// Conversions to x86's SIMD types. +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod x86; + +#[cfg(any(target_arch = "wasm32"))] +mod wasm32; + +#[cfg(any(target_arch = "aarch64", target_arch = "arm",))] +mod arm; + +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +mod powerpc; diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/vendor/arm.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/vendor/arm.rs new file mode 100644 index 000000000..ff3b69ccf --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/vendor/arm.rs @@ -0,0 +1,76 @@ +#![allow(unused)] +use crate::simd::*; + +#[cfg(target_arch = "arm")] +use core::arch::arm::*; + +#[cfg(target_arch = "aarch64")] +use core::arch::aarch64::*; + +#[cfg(any( + target_arch = "aarch64", + all(target_arch = "arm", target_feature = "v7"), +))] +mod neon { + use super::*; + + from_transmute! { unsafe f32x2 => float32x2_t } + from_transmute! { unsafe f32x4 => float32x4_t } + + from_transmute! { unsafe u8x8 => uint8x8_t } + from_transmute! { unsafe u8x16 => uint8x16_t } + from_transmute! { unsafe i8x8 => int8x8_t } + from_transmute! { unsafe i8x16 => int8x16_t } + from_transmute! { unsafe u8x8 => poly8x8_t } + from_transmute! { unsafe u8x16 => poly8x16_t } + + from_transmute! { unsafe u16x4 => uint16x4_t } + from_transmute! { unsafe u16x8 => uint16x8_t } + from_transmute! { unsafe i16x4 => int16x4_t } + from_transmute! { unsafe i16x8 => int16x8_t } + from_transmute! { unsafe u16x4 => poly16x4_t } + from_transmute! { unsafe u16x8 => poly16x8_t } + + from_transmute! { unsafe u32x2 => uint32x2_t } + from_transmute! { unsafe u32x4 => uint32x4_t } + from_transmute! { unsafe i32x2 => int32x2_t } + from_transmute! { unsafe i32x4 => int32x4_t } + + from_transmute! { unsafe Simd => uint64x1_t } + from_transmute! { unsafe u64x2 => uint64x2_t } + from_transmute! { unsafe Simd => int64x1_t } + from_transmute! { unsafe i64x2 => int64x2_t } + from_transmute! { unsafe Simd => poly64x1_t } + from_transmute! { unsafe u64x2 => poly64x2_t } +} + +#[cfg(any( + all(target_feature = "v5te", not(target_feature = "mclass")), + all(target_feature = "mclass", target_feature = "dsp"), +))] +mod dsp { + use super::*; + + from_transmute! { unsafe Simd => uint16x2_t } + from_transmute! { unsafe Simd => int16x2_t } +} + +#[cfg(any( + all(target_feature = "v6", not(target_feature = "mclass")), + all(target_feature = "mclass", target_feature = "dsp"), +))] +mod simd32 { + use super::*; + + from_transmute! { unsafe Simd => uint8x4_t } + from_transmute! { unsafe Simd => int8x4_t } +} + +#[cfg(target_arch = "aarch64")] +mod aarch64 { + use super::neon::*; + use super::*; + + from_transmute! { unsafe Simd => float64x1_t } + from_transmute! { unsafe f64x2 => float64x2_t } +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/vendor/powerpc.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/vendor/powerpc.rs new file mode 100644 index 000000000..92f97d471 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/vendor/powerpc.rs @@ -0,0 +1,11 @@ +use crate::simd::*; + +#[cfg(target_arch = "powerpc")] +use core::arch::powerpc::*; + +#[cfg(target_arch = "powerpc64")] +use core::arch::powerpc64::*; + +from_transmute! { unsafe f64x2 => vector_double } +from_transmute! { unsafe i64x2 => vector_signed_long } +from_transmute! { unsafe u64x2 => vector_unsigned_long } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/vendor/wasm32.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/vendor/wasm32.rs new file mode 100644 index 000000000..ef3baf885 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/vendor/wasm32.rs @@ -0,0 +1,30 @@ +use crate::simd::*; +use core::arch::wasm32::v128; + +from_transmute! { unsafe u8x16 => v128 } +from_transmute! { unsafe i8x16 => v128 } + +from_transmute! { unsafe u16x8 => v128 } +from_transmute! { unsafe i16x8 => v128 } + +from_transmute! { unsafe u32x4 => v128 } +from_transmute! { unsafe i32x4 => v128 } +from_transmute! { unsafe f32x4 => v128 } + +from_transmute! { unsafe u64x2 => v128 } +from_transmute! { unsafe i64x2 => v128 } +from_transmute! { unsafe f64x2 => v128 } + +#[cfg(target_pointer_width = "32")] +mod p32 { + use super::*; + from_transmute! { unsafe usizex4 => v128 } + from_transmute! { unsafe isizex4 => v128 } +} + +#[cfg(target_pointer_width = "64")] +mod p64 { + use super::*; + from_transmute! { unsafe usizex2 => v128 } + from_transmute! { unsafe isizex2 => v128 } +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/src/vendor/x86.rs b/crux-mir/lib/portable-simd/crates/core_simd/src/vendor/x86.rs new file mode 100644 index 000000000..0dd47015e --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/src/vendor/x86.rs @@ -0,0 +1,63 @@ +use crate::simd::*; + +#[cfg(any(target_arch = "x86"))] +use core::arch::x86::*; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +from_transmute! { unsafe u8x16 => __m128i } +from_transmute! { unsafe u8x32 => __m256i } +from_transmute! { unsafe u8x64 => __m512i } +from_transmute! { unsafe i8x16 => __m128i } +from_transmute! { unsafe i8x32 => __m256i } +from_transmute! { unsafe i8x64 => __m512i } + +from_transmute! { unsafe u16x8 => __m128i } +from_transmute! { unsafe u16x16 => __m256i } +from_transmute! { unsafe u16x32 => __m512i } +from_transmute! { unsafe i16x8 => __m128i } +from_transmute! { unsafe i16x16 => __m256i } +from_transmute! { unsafe i16x32 => __m512i } + +from_transmute! { unsafe u32x4 => __m128i } +from_transmute! { unsafe u32x8 => __m256i } +from_transmute! { unsafe u32x16 => __m512i } +from_transmute! { unsafe i32x4 => __m128i } +from_transmute! { unsafe i32x8 => __m256i } +from_transmute! { unsafe i32x16 => __m512i } +from_transmute! { unsafe f32x4 => __m128 } +from_transmute! { unsafe f32x8 => __m256 } +from_transmute! { unsafe f32x16 => __m512 } + +from_transmute! { unsafe u64x2 => __m128i } +from_transmute! { unsafe u64x4 => __m256i } +from_transmute! { unsafe u64x8 => __m512i } +from_transmute! { unsafe i64x2 => __m128i } +from_transmute! { unsafe i64x4 => __m256i } +from_transmute! { unsafe i64x8 => __m512i } +from_transmute! { unsafe f64x2 => __m128d } +from_transmute! { unsafe f64x4 => __m256d } +from_transmute! { unsafe f64x8 => __m512d } + +#[cfg(target_pointer_width = "32")] +mod p32 { + use super::*; + from_transmute! { unsafe usizex4 => __m128i } + from_transmute! { unsafe usizex8 => __m256i } + from_transmute! { unsafe Simd => __m512i } + from_transmute! { unsafe isizex4 => __m128i } + from_transmute! { unsafe isizex8 => __m256i } + from_transmute! { unsafe Simd => __m512i } +} + +#[cfg(target_pointer_width = "64")] +mod p64 { + use super::*; + from_transmute! { unsafe usizex2 => __m128i } + from_transmute! { unsafe usizex4 => __m256i } + from_transmute! { unsafe usizex8 => __m512i } + from_transmute! { unsafe isizex2 => __m128i } + from_transmute! { unsafe isizex4 => __m256i } + from_transmute! { unsafe isizex8 => __m512i } +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/autoderef.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/autoderef.rs new file mode 100644 index 000000000..9359da16e --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/autoderef.rs @@ -0,0 +1,22 @@ +// Test that we handle all our "auto-deref" cases correctly. +#![feature(portable_simd)] +use core_simd::f32x4; + +#[cfg(target_arch = "wasm32")] +use wasm_bindgen_test::*; + +#[cfg(target_arch = "wasm32")] +wasm_bindgen_test_configure!(run_in_browser); + +#[test] +#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] +fn deref() { + let x = f32x4::splat(1.0); + let y = f32x4::splat(2.0); + let a = &x; + let b = &y; + assert_eq!(f32x4::splat(3.0), x + y); + assert_eq!(f32x4::splat(3.0), x + b); + assert_eq!(f32x4::splat(3.0), a + y); + assert_eq!(f32x4::splat(3.0), a + b); +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/cast.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/cast.rs new file mode 100644 index 000000000..ab5650f07 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/cast.rs @@ -0,0 +1,37 @@ +#![feature(portable_simd)] +macro_rules! cast_types { + ($start:ident, $($target:ident),*) => { + mod $start { + use core_simd::simd::Simd; + type Vector = Simd<$start, N>; + $( + mod $target { + use super::*; + test_helpers::test_lanes! { + fn cast_as() { + test_helpers::test_unary_elementwise( + &Vector::::cast::<$target>, + &|x| x as $target, + &|_| true, + ) + } + } + } + )* + } + }; +} + +// The hypothesis is that widening conversions aren't terribly interesting. +cast_types!(f32, f64, i8, u8, usize, isize); +cast_types!(f64, f32, i8, u8, usize, isize); +cast_types!(i8, u8, f32); +cast_types!(u8, i8, f32); +cast_types!(i16, u16, i8, u8, f32); +cast_types!(u16, i16, i8, u8, f32); +cast_types!(i32, u32, i8, u8, f32, f64); +cast_types!(u32, i32, i8, u8, f32, f64); +cast_types!(i64, u64, i8, u8, isize, usize, f32, f64); +cast_types!(u64, i64, i8, u8, isize, usize, f32, f64); +cast_types!(isize, usize, i8, u8, f32, f64); +cast_types!(usize, isize, i8, u8, f32, f64); diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/f32_ops.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/f32_ops.rs new file mode 100644 index 000000000..414a832b1 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/f32_ops.rs @@ -0,0 +1,5 @@ +#![feature(portable_simd)] + +#[macro_use] +mod ops_macros; +impl_float_tests! { f32, i32 } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/f64_ops.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/f64_ops.rs new file mode 100644 index 000000000..e0a1fa33f --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/f64_ops.rs @@ -0,0 +1,5 @@ +#![feature(portable_simd)] + +#[macro_use] +mod ops_macros; +impl_float_tests! { f64, i64 } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/i16_ops.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/i16_ops.rs new file mode 100644 index 000000000..f6c5d74fb --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/i16_ops.rs @@ -0,0 +1,5 @@ +#![feature(portable_simd)] + +#[macro_use] +mod ops_macros; +impl_signed_tests! { i16 } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/i32_ops.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/i32_ops.rs new file mode 100644 index 000000000..69a831c52 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/i32_ops.rs @@ -0,0 +1,5 @@ +#![feature(portable_simd)] + +#[macro_use] +mod ops_macros; +impl_signed_tests! { i32 } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/i64_ops.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/i64_ops.rs new file mode 100644 index 000000000..37ac08117 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/i64_ops.rs @@ -0,0 +1,5 @@ +#![feature(portable_simd)] + +#[macro_use] +mod ops_macros; +impl_signed_tests! { i64 } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/i8_ops.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/i8_ops.rs new file mode 100644 index 000000000..11e4a5cd6 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/i8_ops.rs @@ -0,0 +1,5 @@ +#![feature(portable_simd)] + +#[macro_use] +mod ops_macros; +impl_signed_tests! { i8 } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/isize_ops.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/isize_ops.rs new file mode 100644 index 000000000..5cc9de2b7 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/isize_ops.rs @@ -0,0 +1,5 @@ +#![feature(portable_simd)] + +#[macro_use] +mod ops_macros; +impl_signed_tests! { isize } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops.rs new file mode 100644 index 000000000..f113b50cb --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops.rs @@ -0,0 +1,3 @@ +#![feature(portable_simd)] + +mod mask_ops_impl; diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mask16.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mask16.rs new file mode 100644 index 000000000..0fe82fa68 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mask16.rs @@ -0,0 +1,4 @@ +mask_tests! { mask16x4, 4 } +mask_tests! { mask16x8, 8 } +mask_tests! { mask16x16, 16 } +mask_tests! { mask16x32, 32 } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mask32.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mask32.rs new file mode 100644 index 000000000..66d987a43 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mask32.rs @@ -0,0 +1,4 @@ +mask_tests! { mask32x2, 2 } +mask_tests! { mask32x4, 4 } +mask_tests! { mask32x8, 8 } +mask_tests! { mask32x16, 16 } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mask64.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mask64.rs new file mode 100644 index 000000000..a1f1f67b2 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mask64.rs @@ -0,0 +1,3 @@ +mask_tests! { mask64x2, 2 } +mask_tests! { mask64x4, 4 } +mask_tests! { mask64x8, 8 } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mask8.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mask8.rs new file mode 100644 index 000000000..9c06fbc04 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mask8.rs @@ -0,0 +1,3 @@ +mask_tests! { mask8x8, 8 } +mask_tests! { mask8x16, 16 } +mask_tests! { mask8x32, 32 } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mask_macros.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mask_macros.rs new file mode 100644 index 000000000..795f9e27c --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mask_macros.rs @@ -0,0 +1,225 @@ +macro_rules! mask_tests { + { $vector:ident, $lanes:literal } => { + #[cfg(test)] + mod $vector { + use core_simd::$vector as Vector; + const LANES: usize = $lanes; + + #[cfg(target_arch = "wasm32")] + use wasm_bindgen_test::*; + + #[cfg(target_arch = "wasm32")] + wasm_bindgen_test_configure!(run_in_browser); + + fn from_slice(slice: &[bool]) -> Vector { + let mut value = Vector::default(); + for (i, b) in slice.iter().take(LANES).enumerate() { + value.set(i, *b); + } + value + } + + fn apply_unary_lanewise(x: Vector, f: impl Fn(bool) -> bool) -> Vector { + let mut value = Vector::default(); + for i in 0..LANES { + value.set(i, f(x.test(i))); + } + value + } + + fn apply_binary_lanewise(x: Vector, y: Vector, f: impl Fn(bool, bool) -> bool) -> Vector { + let mut value = Vector::default(); + for i in 0..LANES { + value.set(i, f(x.test(i), y.test(i))); + } + value + } + + fn apply_binary_scalar_lhs_lanewise(x: bool, mut y: Vector, f: impl Fn(bool, bool) -> bool) -> Vector { + for i in 0..LANES { + y.set(i, f(x, y.test(i))); + } + y + } + + fn apply_binary_scalar_rhs_lanewise(mut x: Vector, y: bool, f: impl Fn(bool, bool) -> bool) -> Vector { + for i in 0..LANES { + x.set(i, f(x.test(i), y)); + } + x + } + + const A: [bool; 64] = [ + false, true, false, true, false, false, true, true, + false, true, false, true, false, false, true, true, + false, true, false, true, false, false, true, true, + false, true, false, true, false, false, true, true, + false, true, false, true, false, false, true, true, + false, true, false, true, false, false, true, true, + false, true, false, true, false, false, true, true, + false, true, false, true, false, false, true, true, + ]; + const B: [bool; 64] = [ + false, false, true, true, false, true, false, true, + false, false, true, true, false, true, false, true, + false, false, true, true, false, true, false, true, + false, false, true, true, false, true, false, true, + false, false, true, true, false, true, false, true, + false, false, true, true, false, true, false, true, + false, false, true, true, false, true, false, true, + false, false, true, true, false, true, false, true, + ]; + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + fn bitand() { + let a = from_slice(&A); + let b = from_slice(&B); + let expected = apply_binary_lanewise(a, b, core::ops::BitAnd::bitand); + assert_eq!(a & b, expected); + } + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + fn bitand_assign() { + let mut a = from_slice(&A); + let b = from_slice(&B); + let expected = apply_binary_lanewise(a, b, core::ops::BitAnd::bitand); + a &= b; + assert_eq!(a, expected); + } + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + fn bitand_scalar_rhs() { + let a = from_slice(&A); + let expected = a; + assert_eq!(a & true, expected); + assert_eq!(a & false, Vector::splat(false)); + } + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + fn bitand_scalar_lhs() { + let a = from_slice(&A); + let expected = a; + assert_eq!(true & a, expected); + assert_eq!(false & a, Vector::splat(false)); + } + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + fn bitand_assign_scalar() { + let mut a = from_slice(&A); + let expected = a; + a &= true; + assert_eq!(a, expected); + a &= false; + assert_eq!(a, Vector::splat(false)); + } + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + fn bitor() { + let a = from_slice(&A); + let b = from_slice(&B); + let expected = apply_binary_lanewise(a, b, core::ops::BitOr::bitor); + assert_eq!(a | b, expected); + } + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + fn bitor_assign() { + let mut a = from_slice(&A); + let b = from_slice(&B); + let expected = apply_binary_lanewise(a, b, core::ops::BitOr::bitor); + a |= b; + assert_eq!(a, expected); + } + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + fn bitor_scalar_rhs() { + let a = from_slice(&A); + assert_eq!(a | false, a); + assert_eq!(a | true, Vector::splat(true)); + } + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + fn bitor_scalar_lhs() { + let a = from_slice(&A); + assert_eq!(false | a, a); + assert_eq!(true | a, Vector::splat(true)); + } + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + fn bitor_assign_scalar() { + let mut a = from_slice(&A); + let expected = a; + a |= false; + assert_eq!(a, expected); + a |= true; + assert_eq!(a, Vector::splat(true)); + } + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + fn bitxor() { + let a = from_slice(&A); + let b = from_slice(&B); + let expected = apply_binary_lanewise(a, b, core::ops::BitXor::bitxor); + assert_eq!(a ^ b, expected); + } + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + fn bitxor_assign() { + let mut a = from_slice(&A); + let b = from_slice(&B); + let expected = apply_binary_lanewise(a, b, core::ops::BitXor::bitxor); + a ^= b; + assert_eq!(a, expected); + } + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + fn bitxor_scalar_rhs() { + let a = from_slice(&A); + let expected = apply_binary_scalar_rhs_lanewise(a, true, core::ops::BitXor::bitxor); + assert_eq!(a ^ false, a); + assert_eq!(a ^ true, expected); + } + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + fn bitxor_scalar_lhs() { + let a = from_slice(&A); + let expected = apply_binary_scalar_lhs_lanewise(true, a, core::ops::BitXor::bitxor); + assert_eq!(false ^ a, a); + assert_eq!(true ^ a, expected); + } + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + fn bitxor_assign_scalar() { + let mut a = from_slice(&A); + let expected_unset = a; + let expected_set = apply_binary_scalar_rhs_lanewise(a, true, core::ops::BitXor::bitxor); + a ^= false; + assert_eq!(a, expected_unset); + a ^= true; + assert_eq!(a, expected_set); + } + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + fn not() { + let v = from_slice(&A); + let expected = apply_unary_lanewise(v, core::ops::Not::not); + assert_eq!(!v, expected); + } + } + } +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/masksize.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/masksize.rs new file mode 100644 index 000000000..e0a44d870 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/masksize.rs @@ -0,0 +1,3 @@ +mask_tests! { masksizex2, 2 } +mask_tests! { masksizex4, 4 } +mask_tests! { masksizex8, 8 } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mod.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mod.rs new file mode 100644 index 000000000..b9ec8462a --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/mask_ops_impl/mod.rs @@ -0,0 +1,9 @@ +#[macro_use] +mod mask_macros; + +#[rustfmt::skip] +mod mask8; +mod mask16; +mod mask32; +mod mask64; +mod masksize; diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/masks.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/masks.rs new file mode 100644 index 000000000..673d0db93 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/masks.rs @@ -0,0 +1,158 @@ +#![feature(portable_simd)] + +#[cfg(target_arch = "wasm32")] +use wasm_bindgen_test::*; + +#[cfg(target_arch = "wasm32")] +wasm_bindgen_test_configure!(run_in_browser); + +macro_rules! test_mask_api { + { $type:ident } => { + #[allow(non_snake_case)] + mod $type { + #[cfg(target_arch = "wasm32")] + use wasm_bindgen_test::*; + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + fn set_and_test() { + let values = [true, false, false, true, false, false, true, false]; + let mut mask = core_simd::Mask::<$type, 8>::splat(false); + for (lane, value) in values.iter().copied().enumerate() { + mask.set(lane, value); + } + for (lane, value) in values.iter().copied().enumerate() { + assert_eq!(mask.test(lane), value); + } + } + + #[test] + #[should_panic] + fn set_invalid_lane() { + let mut mask = core_simd::Mask::<$type, 8>::splat(false); + mask.set(8, true); + let _ = mask; + } + + #[test] + #[should_panic] + fn test_invalid_lane() { + let mask = core_simd::Mask::<$type, 8>::splat(false); + let _ = mask.test(8); + } + + #[test] + fn any() { + assert!(!core_simd::Mask::<$type, 8>::splat(false).any()); + assert!(core_simd::Mask::<$type, 8>::splat(true).any()); + let mut v = core_simd::Mask::<$type, 8>::splat(false); + v.set(2, true); + assert!(v.any()); + } + + #[test] + fn all() { + assert!(!core_simd::Mask::<$type, 8>::splat(false).all()); + assert!(core_simd::Mask::<$type, 8>::splat(true).all()); + let mut v = core_simd::Mask::<$type, 8>::splat(false); + v.set(2, true); + assert!(!v.all()); + } + + #[test] + fn roundtrip_int_conversion() { + let values = [true, false, false, true, false, false, true, false]; + let mask = core_simd::Mask::<$type, 8>::from_array(values); + let int = mask.to_int(); + assert_eq!(int.to_array(), [-1, 0, 0, -1, 0, 0, -1, 0]); + assert_eq!(core_simd::Mask::<$type, 8>::from_int(int), mask); + } + + #[test] + fn roundtrip_bitmask_conversion() { + use core_simd::ToBitMask; + let values = [ + true, false, false, true, false, false, true, false, + true, true, false, false, false, false, false, true, + ]; + let mask = core_simd::Mask::<$type, 16>::from_array(values); + let bitmask = mask.to_bitmask(); + assert_eq!(bitmask, 0b1000001101001001); + assert_eq!(core_simd::Mask::<$type, 16>::from_bitmask(bitmask), mask); + } + + #[test] + fn roundtrip_bitmask_conversion_short() { + use core_simd::ToBitMask; + + let values = [ + false, false, false, true, + ]; + let mask = core_simd::Mask::<$type, 4>::from_array(values); + let bitmask = mask.to_bitmask(); + assert_eq!(bitmask, 0b1000); + assert_eq!(core_simd::Mask::<$type, 4>::from_bitmask(bitmask), mask); + + let values = [true, false]; + let mask = core_simd::Mask::<$type, 2>::from_array(values); + let bitmask = mask.to_bitmask(); + assert_eq!(bitmask, 0b01); + assert_eq!(core_simd::Mask::<$type, 2>::from_bitmask(bitmask), mask); + } + + #[test] + fn cast() { + fn cast_impl() + where + core_simd::Mask<$type, 8>: Into>, + { + let values = [true, false, false, true, false, false, true, false]; + let mask = core_simd::Mask::<$type, 8>::from_array(values); + + let cast_mask = mask.cast::(); + assert_eq!(values, cast_mask.to_array()); + + let into_mask: core_simd::Mask = mask.into(); + assert_eq!(values, into_mask.to_array()); + } + + cast_impl::(); + cast_impl::(); + cast_impl::(); + cast_impl::(); + cast_impl::(); + } + + #[cfg(feature = "generic_const_exprs")] + #[test] + fn roundtrip_bitmask_array_conversion() { + use core_simd::ToBitMaskArray; + let values = [ + true, false, false, true, false, false, true, false, + true, true, false, false, false, false, false, true, + ]; + let mask = core_simd::Mask::<$type, 16>::from_array(values); + let bitmask = mask.to_bitmask_array(); + assert_eq!(bitmask, [0b01001001, 0b10000011]); + assert_eq!(core_simd::Mask::<$type, 16>::from_bitmask_array(bitmask), mask); + } + } + } +} + +mod mask_api { + test_mask_api! { i8 } + test_mask_api! { i16 } + test_mask_api! { i32 } + test_mask_api! { i64 } + test_mask_api! { isize } +} + +#[test] +fn convert() { + let values = [true, false, false, true, false, false, true, false]; + assert_eq!( + core_simd::Mask::::from_array(values), + core_simd::Mask::::from_array(values).into() + ); +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/ops_macros.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/ops_macros.rs new file mode 100644 index 000000000..f759394d0 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/ops_macros.rs @@ -0,0 +1,607 @@ +/// Implements a test on a unary operation using proptest. +/// +/// Compares the vector operation to the equivalent scalar operation. +#[macro_export] +macro_rules! impl_unary_op_test { + { $scalar:ty, $trait:ident :: $fn:ident, $scalar_fn:expr } => { + test_helpers::test_lanes! { + fn $fn() { + test_helpers::test_unary_elementwise( + & as core::ops::$trait>::$fn, + &$scalar_fn, + &|_| true, + ); + } + } + }; + { $scalar:ty, $trait:ident :: $fn:ident } => { + impl_unary_op_test! { $scalar, $trait::$fn, <$scalar as core::ops::$trait>::$fn } + }; +} + +/// Implements a test on a binary operation using proptest. +/// +/// Compares the vector operation to the equivalent scalar operation. +#[macro_export] +macro_rules! impl_binary_op_test { + { $scalar:ty, $trait:ident :: $fn:ident, $trait_assign:ident :: $fn_assign:ident, $scalar_fn:expr } => { + mod $fn { + use super::*; + use core_simd::Simd; + + test_helpers::test_lanes! { + fn normal() { + test_helpers::test_binary_elementwise( + & as core::ops::$trait>::$fn, + &$scalar_fn, + &|_, _| true, + ); + } + + fn assign() { + test_helpers::test_binary_elementwise( + &|mut a, b| { as core::ops::$trait_assign>::$fn_assign(&mut a, b); a }, + &$scalar_fn, + &|_, _| true, + ); + } + } + } + }; + { $scalar:ty, $trait:ident :: $fn:ident, $trait_assign:ident :: $fn_assign:ident } => { + impl_binary_op_test! { $scalar, $trait::$fn, $trait_assign::$fn_assign, <$scalar as core::ops::$trait>::$fn } + }; +} + +/// Implements a test on a binary operation using proptest. +/// +/// Like `impl_binary_op_test`, but allows providing a function for rejecting particular inputs +/// (like the `proptest_assume` macro). +/// +/// Compares the vector operation to the equivalent scalar operation. +#[macro_export] +macro_rules! impl_binary_checked_op_test { + { $scalar:ty, $trait:ident :: $fn:ident, $trait_assign:ident :: $fn_assign:ident, $scalar_fn:expr, $check_fn:expr } => { + mod $fn { + use super::*; + use core_simd::Simd; + + test_helpers::test_lanes! { + fn normal() { + test_helpers::test_binary_elementwise( + & as core::ops::$trait>::$fn, + &$scalar_fn, + &|x, y| x.iter().zip(y.iter()).all(|(x, y)| $check_fn(*x, *y)), + ); + } + + fn assign() { + test_helpers::test_binary_elementwise( + &|mut a, b| { as core::ops::$trait_assign>::$fn_assign(&mut a, b); a }, + &$scalar_fn, + &|x, y| x.iter().zip(y.iter()).all(|(x, y)| $check_fn(*x, *y)), + ) + } + } + } + }; + { $scalar:ty, $trait:ident :: $fn:ident, $trait_assign:ident :: $fn_assign:ident, $check_fn:expr } => { + impl_binary_checked_op_test! { $scalar, $trait::$fn, $trait_assign::$fn_assign, <$scalar as core::ops::$trait>::$fn, $check_fn } + }; +} + +#[macro_export] +macro_rules! impl_common_integer_tests { + { $vector:ident, $scalar:ident } => { + test_helpers::test_lanes! { + fn reduce_sum() { + test_helpers::test_1(&|x| { + test_helpers::prop_assert_biteq! ( + $vector::::from_array(x).reduce_sum(), + x.iter().copied().fold(0 as $scalar, $scalar::wrapping_add), + ); + Ok(()) + }); + } + + fn reduce_product() { + test_helpers::test_1(&|x| { + test_helpers::prop_assert_biteq! ( + $vector::::from_array(x).reduce_product(), + x.iter().copied().fold(1 as $scalar, $scalar::wrapping_mul), + ); + Ok(()) + }); + } + + fn reduce_and() { + test_helpers::test_1(&|x| { + test_helpers::prop_assert_biteq! ( + $vector::::from_array(x).reduce_and(), + x.iter().copied().fold(-1i8 as $scalar, <$scalar as core::ops::BitAnd>::bitand), + ); + Ok(()) + }); + } + + fn reduce_or() { + test_helpers::test_1(&|x| { + test_helpers::prop_assert_biteq! ( + $vector::::from_array(x).reduce_or(), + x.iter().copied().fold(0 as $scalar, <$scalar as core::ops::BitOr>::bitor), + ); + Ok(()) + }); + } + + fn reduce_xor() { + test_helpers::test_1(&|x| { + test_helpers::prop_assert_biteq! ( + $vector::::from_array(x).reduce_xor(), + x.iter().copied().fold(0 as $scalar, <$scalar as core::ops::BitXor>::bitxor), + ); + Ok(()) + }); + } + + fn reduce_max() { + test_helpers::test_1(&|x| { + test_helpers::prop_assert_biteq! ( + $vector::::from_array(x).reduce_max(), + x.iter().copied().max().unwrap(), + ); + Ok(()) + }); + } + + fn reduce_min() { + test_helpers::test_1(&|x| { + test_helpers::prop_assert_biteq! ( + $vector::::from_array(x).reduce_min(), + x.iter().copied().min().unwrap(), + ); + Ok(()) + }); + } + } + } +} + +/// Implement tests for signed integers. +#[macro_export] +macro_rules! impl_signed_tests { + { $scalar:tt } => { + mod $scalar { + use core_simd::simd::SimdInt; + type Vector = core_simd::Simd; + type Scalar = $scalar; + + impl_common_integer_tests! { Vector, Scalar } + + test_helpers::test_lanes! { + fn neg() { + test_helpers::test_unary_elementwise( + & as core::ops::Neg>::neg, + &::neg, + &|x| !x.contains(&Scalar::MIN), + ); + } + + fn is_positive() { + test_helpers::test_unary_mask_elementwise( + &Vector::::is_positive, + &Scalar::is_positive, + &|_| true, + ); + } + + fn is_negative() { + test_helpers::test_unary_mask_elementwise( + &Vector::::is_negative, + &Scalar::is_negative, + &|_| true, + ); + } + + fn signum() { + test_helpers::test_unary_elementwise( + &Vector::::signum, + &Scalar::signum, + &|_| true, + ) + } + + fn div_min_may_overflow() { + let a = Vector::::splat(Scalar::MIN); + let b = Vector::::splat(-1); + assert_eq!(a / b, a); + } + + fn rem_min_may_overflow() { + let a = Vector::::splat(Scalar::MIN); + let b = Vector::::splat(-1); + assert_eq!(a % b, Vector::::splat(0)); + } + + fn simd_min() { + use core_simd::simd::SimdOrd; + let a = Vector::::splat(Scalar::MIN); + let b = Vector::::splat(0); + assert_eq!(a.simd_min(b), a); + let a = Vector::::splat(Scalar::MAX); + let b = Vector::::splat(0); + assert_eq!(a.simd_min(b), b); + } + + fn simd_max() { + use core_simd::simd::SimdOrd; + let a = Vector::::splat(Scalar::MIN); + let b = Vector::::splat(0); + assert_eq!(a.simd_max(b), b); + let a = Vector::::splat(Scalar::MAX); + let b = Vector::::splat(0); + assert_eq!(a.simd_max(b), a); + } + + fn simd_clamp() { + use core_simd::simd::SimdOrd; + let min = Vector::::splat(Scalar::MIN); + let max = Vector::::splat(Scalar::MAX); + let zero = Vector::::splat(0); + let one = Vector::::splat(1); + let negone = Vector::::splat(-1); + assert_eq!(zero.simd_clamp(min, max), zero); + assert_eq!(zero.simd_clamp(min, one), zero); + assert_eq!(zero.simd_clamp(one, max), one); + assert_eq!(zero.simd_clamp(min, negone), negone); + } + } + + test_helpers::test_lanes_panic! { + fn div_by_all_zeros_panics() { + let a = Vector::::splat(42); + let b = Vector::::splat(0); + let _ = a / b; + } + + fn div_by_one_zero_panics() { + let a = Vector::::splat(42); + let mut b = Vector::::splat(21); + b[0] = 0 as _; + let _ = a / b; + } + + fn rem_zero_panic() { + let a = Vector::::splat(42); + let b = Vector::::splat(0); + let _ = a % b; + } + } + + test_helpers::test_lanes! { + fn div_neg_one_no_panic() { + let a = Vector::::splat(42); + let b = Vector::::splat(-1); + let _ = a / b; + } + + fn rem_neg_one_no_panic() { + let a = Vector::::splat(42); + let b = Vector::::splat(-1); + let _ = a % b; + } + } + + impl_binary_op_test!(Scalar, Add::add, AddAssign::add_assign, Scalar::wrapping_add); + impl_binary_op_test!(Scalar, Sub::sub, SubAssign::sub_assign, Scalar::wrapping_sub); + impl_binary_op_test!(Scalar, Mul::mul, MulAssign::mul_assign, Scalar::wrapping_mul); + + // Exclude Div and Rem panicking cases + impl_binary_checked_op_test!(Scalar, Div::div, DivAssign::div_assign, Scalar::wrapping_div, |x, y| y != 0 && !(x == Scalar::MIN && y == -1)); + impl_binary_checked_op_test!(Scalar, Rem::rem, RemAssign::rem_assign, Scalar::wrapping_rem, |x, y| y != 0 && !(x == Scalar::MIN && y == -1)); + + impl_unary_op_test!(Scalar, Not::not); + impl_binary_op_test!(Scalar, BitAnd::bitand, BitAndAssign::bitand_assign); + impl_binary_op_test!(Scalar, BitOr::bitor, BitOrAssign::bitor_assign); + impl_binary_op_test!(Scalar, BitXor::bitxor, BitXorAssign::bitxor_assign); + } + } +} + +/// Implement tests for unsigned integers. +#[macro_export] +macro_rules! impl_unsigned_tests { + { $scalar:tt } => { + mod $scalar { + use core_simd::simd::SimdUint; + type Vector = core_simd::Simd; + type Scalar = $scalar; + + impl_common_integer_tests! { Vector, Scalar } + + test_helpers::test_lanes_panic! { + fn rem_zero_panic() { + let a = Vector::::splat(42); + let b = Vector::::splat(0); + let _ = a % b; + } + } + + impl_binary_op_test!(Scalar, Add::add, AddAssign::add_assign, Scalar::wrapping_add); + impl_binary_op_test!(Scalar, Sub::sub, SubAssign::sub_assign, Scalar::wrapping_sub); + impl_binary_op_test!(Scalar, Mul::mul, MulAssign::mul_assign, Scalar::wrapping_mul); + + // Exclude Div and Rem panicking cases + impl_binary_checked_op_test!(Scalar, Div::div, DivAssign::div_assign, Scalar::wrapping_div, |_, y| y != 0); + impl_binary_checked_op_test!(Scalar, Rem::rem, RemAssign::rem_assign, Scalar::wrapping_rem, |_, y| y != 0); + + impl_unary_op_test!(Scalar, Not::not); + impl_binary_op_test!(Scalar, BitAnd::bitand, BitAndAssign::bitand_assign); + impl_binary_op_test!(Scalar, BitOr::bitor, BitOrAssign::bitor_assign); + impl_binary_op_test!(Scalar, BitXor::bitxor, BitXorAssign::bitxor_assign); + } + } +} + +/// Implement tests for floating point numbers. +#[macro_export] +macro_rules! impl_float_tests { + { $scalar:tt, $int_scalar:tt } => { + mod $scalar { + use core_simd::SimdFloat; + type Vector = core_simd::Simd; + type Scalar = $scalar; + + impl_unary_op_test!(Scalar, Neg::neg); + impl_binary_op_test!(Scalar, Add::add, AddAssign::add_assign); + impl_binary_op_test!(Scalar, Sub::sub, SubAssign::sub_assign); + impl_binary_op_test!(Scalar, Mul::mul, MulAssign::mul_assign); + impl_binary_op_test!(Scalar, Div::div, DivAssign::div_assign); + impl_binary_op_test!(Scalar, Rem::rem, RemAssign::rem_assign); + + test_helpers::test_lanes! { + fn is_sign_positive() { + test_helpers::test_unary_mask_elementwise( + &Vector::::is_sign_positive, + &Scalar::is_sign_positive, + &|_| true, + ); + } + + fn is_sign_negative() { + test_helpers::test_unary_mask_elementwise( + &Vector::::is_sign_negative, + &Scalar::is_sign_negative, + &|_| true, + ); + } + + fn is_finite() { + test_helpers::test_unary_mask_elementwise( + &Vector::::is_finite, + &Scalar::is_finite, + &|_| true, + ); + } + + fn is_infinite() { + test_helpers::test_unary_mask_elementwise( + &Vector::::is_infinite, + &Scalar::is_infinite, + &|_| true, + ); + } + + fn is_nan() { + test_helpers::test_unary_mask_elementwise( + &Vector::::is_nan, + &Scalar::is_nan, + &|_| true, + ); + } + + fn is_normal() { + test_helpers::test_unary_mask_elementwise( + &Vector::::is_normal, + &Scalar::is_normal, + &|_| true, + ); + } + + fn is_subnormal() { + test_helpers::test_unary_mask_elementwise( + &Vector::::is_subnormal, + &Scalar::is_subnormal, + &|_| true, + ); + } + + fn abs() { + test_helpers::test_unary_elementwise( + &Vector::::abs, + &Scalar::abs, + &|_| true, + ) + } + + fn recip() { + test_helpers::test_unary_elementwise( + &Vector::::recip, + &Scalar::recip, + &|_| true, + ) + } + + fn to_degrees() { + test_helpers::test_unary_elementwise( + &Vector::::to_degrees, + &Scalar::to_degrees, + &|_| true, + ) + } + + fn to_radians() { + test_helpers::test_unary_elementwise( + &Vector::::to_radians, + &Scalar::to_radians, + &|_| true, + ) + } + + fn signum() { + test_helpers::test_unary_elementwise( + &Vector::::signum, + &Scalar::signum, + &|_| true, + ) + } + + fn copysign() { + test_helpers::test_binary_elementwise( + &Vector::::copysign, + &Scalar::copysign, + &|_, _| true, + ) + } + + fn simd_min() { + // Regular conditions (both values aren't zero) + test_helpers::test_binary_elementwise( + &Vector::::simd_min, + &Scalar::min, + // Reject the case where both values are zero with different signs + &|a, b| { + for (a, b) in a.iter().zip(b.iter()) { + if *a == 0. && *b == 0. && a.signum() != b.signum() { + return false; + } + } + true + } + ); + + // Special case where both values are zero + let p_zero = Vector::::splat(0.); + let n_zero = Vector::::splat(-0.); + assert!(p_zero.simd_min(n_zero).to_array().iter().all(|x| *x == 0.)); + assert!(n_zero.simd_min(p_zero).to_array().iter().all(|x| *x == 0.)); + } + + fn simd_max() { + // Regular conditions (both values aren't zero) + test_helpers::test_binary_elementwise( + &Vector::::simd_max, + &Scalar::max, + // Reject the case where both values are zero with different signs + &|a, b| { + for (a, b) in a.iter().zip(b.iter()) { + if *a == 0. && *b == 0. && a.signum() != b.signum() { + return false; + } + } + true + } + ); + + // Special case where both values are zero + let p_zero = Vector::::splat(0.); + let n_zero = Vector::::splat(-0.); + assert!(p_zero.simd_max(n_zero).to_array().iter().all(|x| *x == 0.)); + assert!(n_zero.simd_max(p_zero).to_array().iter().all(|x| *x == 0.)); + } + + fn simd_clamp() { + test_helpers::test_3(&|value: [Scalar; LANES], mut min: [Scalar; LANES], mut max: [Scalar; LANES]| { + for (min, max) in min.iter_mut().zip(max.iter_mut()) { + if max < min { + core::mem::swap(min, max); + } + if min.is_nan() { + *min = Scalar::NEG_INFINITY; + } + if max.is_nan() { + *max = Scalar::INFINITY; + } + } + + let mut result_scalar = [Scalar::default(); LANES]; + for i in 0..LANES { + result_scalar[i] = value[i].clamp(min[i], max[i]); + } + let result_vector = Vector::from_array(value).simd_clamp(min.into(), max.into()).to_array(); + test_helpers::prop_assert_biteq!(result_scalar, result_vector); + Ok(()) + }) + } + + fn reduce_sum() { + test_helpers::test_1(&|x| { + test_helpers::prop_assert_biteq! ( + Vector::::from_array(x).reduce_sum(), + x.iter().sum(), + ); + Ok(()) + }); + } + + fn reduce_product() { + test_helpers::test_1(&|x| { + test_helpers::prop_assert_biteq! ( + Vector::::from_array(x).reduce_product(), + x.iter().product(), + ); + Ok(()) + }); + } + + fn reduce_max() { + test_helpers::test_1(&|x| { + let vmax = Vector::::from_array(x).reduce_max(); + let smax = x.iter().copied().fold(Scalar::NAN, Scalar::max); + // 0 and -0 are treated the same + if !(x.contains(&0.) && x.contains(&-0.) && vmax.abs() == 0. && smax.abs() == 0.) { + test_helpers::prop_assert_biteq!(vmax, smax); + } + Ok(()) + }); + } + + fn reduce_min() { + test_helpers::test_1(&|x| { + let vmax = Vector::::from_array(x).reduce_min(); + let smax = x.iter().copied().fold(Scalar::NAN, Scalar::min); + // 0 and -0 are treated the same + if !(x.contains(&0.) && x.contains(&-0.) && vmax.abs() == 0. && smax.abs() == 0.) { + test_helpers::prop_assert_biteq!(vmax, smax); + } + Ok(()) + }); + } + } + + #[cfg(feature = "std")] + mod std { + use std_float::StdFloat; + + use super::*; + test_helpers::test_lanes! { + fn sqrt() { + test_helpers::test_unary_elementwise( + &Vector::::sqrt, + &Scalar::sqrt, + &|_| true, + ) + } + + fn mul_add() { + test_helpers::test_ternary_elementwise( + &Vector::::mul_add, + &Scalar::mul_add, + &|_, _, _| true, + ) + } + } + } + } + } +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/round.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/round.rs new file mode 100644 index 000000000..484fd5bf4 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/round.rs @@ -0,0 +1,85 @@ +#![feature(portable_simd)] + +macro_rules! float_rounding_test { + { $scalar:tt, $int_scalar:tt } => { + mod $scalar { + use std_float::StdFloat; + + type Vector = core_simd::Simd<$scalar, LANES>; + type Scalar = $scalar; + type IntScalar = $int_scalar; + + test_helpers::test_lanes! { + fn ceil() { + test_helpers::test_unary_elementwise( + &Vector::::ceil, + &Scalar::ceil, + &|_| true, + ) + } + + fn floor() { + test_helpers::test_unary_elementwise( + &Vector::::floor, + &Scalar::floor, + &|_| true, + ) + } + + fn round() { + test_helpers::test_unary_elementwise( + &Vector::::round, + &Scalar::round, + &|_| true, + ) + } + + fn trunc() { + test_helpers::test_unary_elementwise( + &Vector::::trunc, + &Scalar::trunc, + &|_| true, + ) + } + + fn fract() { + test_helpers::test_unary_elementwise( + &Vector::::fract, + &Scalar::fract, + &|_| true, + ) + } + } + + test_helpers::test_lanes! { + fn to_int_unchecked() { + // The maximum integer that can be represented by the equivalently sized float has + // all of the mantissa digits set to 1, pushed up to the MSB. + const ALL_MANTISSA_BITS: IntScalar = ((1 << ::MANTISSA_DIGITS) - 1); + const MAX_REPRESENTABLE_VALUE: Scalar = + (ALL_MANTISSA_BITS << (core::mem::size_of::() * 8 - ::MANTISSA_DIGITS as usize - 1)) as Scalar; + + let mut runner = test_helpers::make_runner(); + runner.run( + &test_helpers::array::UniformArrayStrategy::new(-MAX_REPRESENTABLE_VALUE..MAX_REPRESENTABLE_VALUE), + |x| { + let result_1 = unsafe { Vector::from_array(x).to_int_unchecked::().to_array() }; + let result_2 = { + let mut result: [IntScalar; LANES] = [0; LANES]; + for (i, o) in x.iter().zip(result.iter_mut()) { + *o = unsafe { i.to_int_unchecked::() }; + } + result + }; + test_helpers::prop_assert_biteq!(result_1, result_2); + Ok(()) + }, + ).unwrap(); + } + } + } + } +} + +float_rounding_test! { f32, i32 } +float_rounding_test! { f64, i64 } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/swizzle.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/swizzle.rs new file mode 100644 index 000000000..51c63611a --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/swizzle.rs @@ -0,0 +1,62 @@ +#![feature(portable_simd)] +use core_simd::{Simd, Swizzle}; + +#[cfg(target_arch = "wasm32")] +use wasm_bindgen_test::*; + +#[cfg(target_arch = "wasm32")] +wasm_bindgen_test_configure!(run_in_browser); + +#[test] +#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] +fn swizzle() { + struct Index; + impl Swizzle<4, 4> for Index { + const INDEX: [usize; 4] = [2, 1, 3, 0]; + } + impl Swizzle<4, 2> for Index { + const INDEX: [usize; 2] = [1, 1]; + } + + let vector = Simd::from_array([2, 4, 1, 9]); + assert_eq!(Index::swizzle(vector).to_array(), [1, 4, 9, 2]); + assert_eq!(Index::swizzle(vector).to_array(), [4, 4]); +} + +#[test] +#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] +fn reverse() { + let a = Simd::from_array([1, 2, 3, 4]); + assert_eq!(a.reverse().to_array(), [4, 3, 2, 1]); +} + +#[test] +#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] +fn rotate() { + let a = Simd::from_array([1, 2, 3, 4]); + assert_eq!(a.rotate_lanes_left::<0>().to_array(), [1, 2, 3, 4]); + assert_eq!(a.rotate_lanes_left::<1>().to_array(), [2, 3, 4, 1]); + assert_eq!(a.rotate_lanes_left::<2>().to_array(), [3, 4, 1, 2]); + assert_eq!(a.rotate_lanes_left::<3>().to_array(), [4, 1, 2, 3]); + assert_eq!(a.rotate_lanes_left::<4>().to_array(), [1, 2, 3, 4]); + assert_eq!(a.rotate_lanes_left::<5>().to_array(), [2, 3, 4, 1]); + assert_eq!(a.rotate_lanes_right::<0>().to_array(), [1, 2, 3, 4]); + assert_eq!(a.rotate_lanes_right::<1>().to_array(), [4, 1, 2, 3]); + assert_eq!(a.rotate_lanes_right::<2>().to_array(), [3, 4, 1, 2]); + assert_eq!(a.rotate_lanes_right::<3>().to_array(), [2, 3, 4, 1]); + assert_eq!(a.rotate_lanes_right::<4>().to_array(), [1, 2, 3, 4]); + assert_eq!(a.rotate_lanes_right::<5>().to_array(), [4, 1, 2, 3]); +} + +#[test] +#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] +fn interleave() { + let a = Simd::from_array([0, 1, 2, 3, 4, 5, 6, 7]); + let b = Simd::from_array([8, 9, 10, 11, 12, 13, 14, 15]); + let (lo, hi) = a.interleave(b); + assert_eq!(lo.to_array(), [0, 8, 1, 9, 2, 10, 3, 11]); + assert_eq!(hi.to_array(), [4, 12, 5, 13, 6, 14, 7, 15]); + let (even, odd) = lo.deinterleave(hi); + assert_eq!(even, a); + assert_eq!(odd, b); +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/to_bytes.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/to_bytes.rs new file mode 100644 index 000000000..debb4335e --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/to_bytes.rs @@ -0,0 +1,14 @@ +#![feature(portable_simd, generic_const_exprs, adt_const_params)] +#![allow(incomplete_features)] +#![cfg(feature = "generic_const_exprs")] + +use core_simd::Simd; + +#[test] +fn byte_convert() { + let int = Simd::::from_array([0xdeadbeef, 0x8badf00d]); + let bytes = int.to_ne_bytes(); + assert_eq!(int[0].to_ne_bytes(), bytes[..4]); + assert_eq!(int[1].to_ne_bytes(), bytes[4..]); + assert_eq!(Simd::::from_ne_bytes(bytes), int); +} diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/u16_ops.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/u16_ops.rs new file mode 100644 index 000000000..9ae3bd6a4 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/u16_ops.rs @@ -0,0 +1,5 @@ +#![feature(portable_simd)] + +#[macro_use] +mod ops_macros; +impl_unsigned_tests! { u16 } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/u32_ops.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/u32_ops.rs new file mode 100644 index 000000000..de34b73d6 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/u32_ops.rs @@ -0,0 +1,5 @@ +#![feature(portable_simd)] + +#[macro_use] +mod ops_macros; +impl_unsigned_tests! { u32 } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/u64_ops.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/u64_ops.rs new file mode 100644 index 000000000..8ee5a318c --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/u64_ops.rs @@ -0,0 +1,5 @@ +#![feature(portable_simd)] + +#[macro_use] +mod ops_macros; +impl_unsigned_tests! { u64 } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/u8_ops.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/u8_ops.rs new file mode 100644 index 000000000..6d7211121 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/u8_ops.rs @@ -0,0 +1,5 @@ +#![feature(portable_simd)] + +#[macro_use] +mod ops_macros; +impl_unsigned_tests! { u8 } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/tests/usize_ops.rs b/crux-mir/lib/portable-simd/crates/core_simd/tests/usize_ops.rs new file mode 100644 index 000000000..9c7b1687a --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/tests/usize_ops.rs @@ -0,0 +1,5 @@ +#![feature(portable_simd)] + +#[macro_use] +mod ops_macros; +impl_unsigned_tests! { usize } diff --git a/crux-mir/lib/portable-simd/crates/core_simd/webdriver.json b/crux-mir/lib/portable-simd/crates/core_simd/webdriver.json new file mode 100644 index 000000000..f1d5734f1 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/core_simd/webdriver.json @@ -0,0 +1,7 @@ +{ + "goog:chromeOptions": { + "args": [ + "--enable-features=WebAssemblySimd" + ] + } +} diff --git a/crux-mir/lib/portable-simd/crates/std_float/Cargo.toml b/crux-mir/lib/portable-simd/crates/std_float/Cargo.toml new file mode 100644 index 000000000..84c69774c --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/std_float/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "std_float" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +core_simd = { path = "../core_simd", default-features = false } + +[features] +default = ["as_crate"] +as_crate = [] diff --git a/crux-mir/lib/portable-simd/crates/std_float/src/lib.rs b/crux-mir/lib/portable-simd/crates/std_float/src/lib.rs new file mode 100644 index 000000000..4ac60b10c --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/std_float/src/lib.rs @@ -0,0 +1,161 @@ +#![cfg_attr(feature = "as_crate", no_std)] // We are std! +#![cfg_attr(feature = "as_crate", feature(platform_intrinsics), feature(portable_simd))] +#[cfg(not(feature = "as_crate"))] +use core::simd; +#[cfg(feature = "as_crate")] +use core_simd::simd; + +use simd::{LaneCount, Simd, SupportedLaneCount}; + +#[cfg(feature = "as_crate")] +mod experimental { + pub trait Sealed {} +} + +#[cfg(feature = "as_crate")] +use experimental as sealed; + +use crate::sealed::Sealed; + +// "platform intrinsics" are essentially "codegen intrinsics" +// each of these may be scalarized and lowered to a libm call +extern "platform-intrinsic" { + // ceil + fn simd_ceil(x: T) -> T; + + // floor + fn simd_floor(x: T) -> T; + + // round + fn simd_round(x: T) -> T; + + // trunc + fn simd_trunc(x: T) -> T; + + // fsqrt + fn simd_fsqrt(x: T) -> T; + + // fma + fn simd_fma(x: T, y: T, z: T) -> T; +} + +/// This trait provides a possibly-temporary implementation of float functions +/// that may, in the absence of hardware support, canonicalize to calling an +/// operating system's `math.h` dynamically-loaded library (also known as a +/// shared object). As these conditionally require runtime support, they +/// should only appear in binaries built assuming OS support: `std`. +/// +/// However, there is no reason SIMD types, in general, need OS support, +/// as for many architectures an embedded binary may simply configure that +/// support itself. This means these types must be visible in `core` +/// but have these functions available in `std`. +/// +/// [`f32`] and [`f64`] achieve a similar trick by using "lang items", but +/// due to compiler limitations, it is harder to implement this approach for +/// abstract data types like [`Simd`]. From that need, this trait is born. +/// +/// It is possible this trait will be replaced in some manner in the future, +/// when either the compiler or its supporting runtime functions are improved. +/// For now this trait is available to permit experimentation with SIMD float +/// operations that may lack hardware support, such as `mul_add`. +pub trait StdFloat: Sealed + Sized { + /// Fused multiply-add. Computes `(self * a) + b` with only one rounding error, + /// yielding a more accurate result than an unfused multiply-add. + /// + /// Using `mul_add` *may* be more performant than an unfused multiply-add if the target + /// architecture has a dedicated `fma` CPU instruction. However, this is not always + /// true, and will be heavily dependent on designing algorithms with specific target + /// hardware in mind. + #[inline] + #[must_use = "method returns a new vector and does not mutate the original value"] + fn mul_add(self, a: Self, b: Self) -> Self { + unsafe { simd_fma(self, a, b) } + } + + /// Produces a vector where every lane has the square root value + /// of the equivalently-indexed lane in `self` + #[inline] + #[must_use = "method returns a new vector and does not mutate the original value"] + fn sqrt(self) -> Self { + unsafe { simd_fsqrt(self) } + } + + /// Returns the smallest integer greater than or equal to each lane. + #[must_use = "method returns a new vector and does not mutate the original value"] + #[inline] + fn ceil(self) -> Self { + unsafe { simd_ceil(self) } + } + + /// Returns the largest integer value less than or equal to each lane. + #[must_use = "method returns a new vector and does not mutate the original value"] + #[inline] + fn floor(self) -> Self { + unsafe { simd_floor(self) } + } + + /// Rounds to the nearest integer value. Ties round toward zero. + #[must_use = "method returns a new vector and does not mutate the original value"] + #[inline] + fn round(self) -> Self { + unsafe { simd_round(self) } + } + + /// Returns the floating point's integer value, with its fractional part removed. + #[must_use = "method returns a new vector and does not mutate the original value"] + #[inline] + fn trunc(self) -> Self { + unsafe { simd_trunc(self) } + } + + /// Returns the floating point's fractional value, with its integer part removed. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn fract(self) -> Self; +} + +impl Sealed for Simd where LaneCount: SupportedLaneCount {} +impl Sealed for Simd where LaneCount: SupportedLaneCount {} + +// We can safely just use all the defaults. +impl StdFloat for Simd +where + LaneCount: SupportedLaneCount, +{ + /// Returns the floating point's fractional value, with its integer part removed. + #[must_use = "method returns a new vector and does not mutate the original value"] + #[inline] + fn fract(self) -> Self { + self - self.trunc() + } +} + +impl StdFloat for Simd +where + LaneCount: SupportedLaneCount, +{ + /// Returns the floating point's fractional value, with its integer part removed. + #[must_use = "method returns a new vector and does not mutate the original value"] + #[inline] + fn fract(self) -> Self { + self - self.trunc() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use simd::*; + + #[test] + fn everything_works() { + let x = f32x4::from_array([0.1, 0.5, 0.6, -1.5]); + let x2 = x + x; + let _xc = x.ceil(); + let _xf = x.floor(); + let _xr = x.round(); + let _xt = x.trunc(); + let _xfma = x.mul_add(x, x); + let _xsqrt = x.sqrt(); + let _ = x2.abs() * x2; + } +} diff --git a/crux-mir/lib/portable-simd/crates/test_helpers/Cargo.toml b/crux-mir/lib/portable-simd/crates/test_helpers/Cargo.toml new file mode 100644 index 000000000..a04b0961d --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/test_helpers/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "test_helpers" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies.proptest] +version = "0.10" +default-features = false +features = ["alloc"] diff --git a/crux-mir/lib/portable-simd/crates/test_helpers/src/array.rs b/crux-mir/lib/portable-simd/crates/test_helpers/src/array.rs new file mode 100644 index 000000000..5ffc92269 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/test_helpers/src/array.rs @@ -0,0 +1,97 @@ +//! Generic-length array strategy. + +// Adapted from proptest's array code +// Copyright 2017 Jason Lingle + +use core::{marker::PhantomData, mem::MaybeUninit}; +use proptest::{ + strategy::{NewTree, Strategy, ValueTree}, + test_runner::TestRunner, +}; + +#[must_use = "strategies do nothing unless used"] +#[derive(Clone, Copy, Debug)] +pub struct UniformArrayStrategy { + strategy: S, + _marker: PhantomData, +} + +impl UniformArrayStrategy { + pub const fn new(strategy: S) -> Self { + Self { + strategy, + _marker: PhantomData, + } + } +} + +pub struct ArrayValueTree { + tree: T, + shrinker: usize, + last_shrinker: Option, +} + +impl Strategy for UniformArrayStrategy +where + T: core::fmt::Debug, + S: Strategy, +{ + type Tree = ArrayValueTree<[S::Tree; LANES]>; + type Value = [T; LANES]; + + fn new_tree(&self, runner: &mut TestRunner) -> NewTree { + let tree: [S::Tree; LANES] = unsafe { + let mut tree: [MaybeUninit; LANES] = MaybeUninit::uninit().assume_init(); + for t in tree.iter_mut() { + *t = MaybeUninit::new(self.strategy.new_tree(runner)?) + } + core::mem::transmute_copy(&tree) + }; + Ok(ArrayValueTree { + tree, + shrinker: 0, + last_shrinker: None, + }) + } +} + +impl ValueTree for ArrayValueTree<[T; LANES]> { + type Value = [T::Value; LANES]; + + fn current(&self) -> Self::Value { + unsafe { + let mut value: [MaybeUninit; LANES] = MaybeUninit::uninit().assume_init(); + for (tree_elem, value_elem) in self.tree.iter().zip(value.iter_mut()) { + *value_elem = MaybeUninit::new(tree_elem.current()); + } + core::mem::transmute_copy(&value) + } + } + + fn simplify(&mut self) -> bool { + while self.shrinker < LANES { + if self.tree[self.shrinker].simplify() { + self.last_shrinker = Some(self.shrinker); + return true; + } else { + self.shrinker += 1; + } + } + + false + } + + fn complicate(&mut self) -> bool { + if let Some(shrinker) = self.last_shrinker { + self.shrinker = shrinker; + if self.tree[shrinker].complicate() { + true + } else { + self.last_shrinker = None; + false + } + } else { + false + } + } +} diff --git a/crux-mir/lib/portable-simd/crates/test_helpers/src/biteq.rs b/crux-mir/lib/portable-simd/crates/test_helpers/src/biteq.rs new file mode 100644 index 000000000..00350e224 --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/test_helpers/src/biteq.rs @@ -0,0 +1,106 @@ +//! Compare numeric types by exact bit value. + +pub trait BitEq { + fn biteq(&self, other: &Self) -> bool; + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result; +} + +impl BitEq for bool { + fn biteq(&self, other: &Self) -> bool { + self == other + } + + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "{:?}", self) + } +} + +macro_rules! impl_integer_biteq { + { $($type:ty),* } => { + $( + impl BitEq for $type { + fn biteq(&self, other: &Self) -> bool { + self == other + } + + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "{:?} ({:x})", self, self) + } + } + )* + }; +} + +impl_integer_biteq! { u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize } + +macro_rules! impl_float_biteq { + { $($type:ty),* } => { + $( + impl BitEq for $type { + fn biteq(&self, other: &Self) -> bool { + if self.is_nan() && other.is_nan() { + true // exact nan bits don't matter + } else { + self.to_bits() == other.to_bits() + } + } + + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "{:?} ({:x})", self, self.to_bits()) + } + } + )* + }; +} + +impl_float_biteq! { f32, f64 } + +impl BitEq for [T; N] { + fn biteq(&self, other: &Self) -> bool { + self.iter() + .zip(other.iter()) + .fold(true, |value, (left, right)| value && left.biteq(right)) + } + + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + #[repr(transparent)] + struct Wrapper<'a, T: BitEq>(&'a T); + + impl core::fmt::Debug for Wrapper<'_, T> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + self.0.fmt(f) + } + } + + f.debug_list() + .entries(self.iter().map(|x| Wrapper(x))) + .finish() + } +} + +#[doc(hidden)] +pub struct BitEqWrapper<'a, T>(pub &'a T); + +impl PartialEq for BitEqWrapper<'_, T> { + fn eq(&self, other: &Self) -> bool { + self.0.biteq(other.0) + } +} + +impl core::fmt::Debug for BitEqWrapper<'_, T> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + self.0.fmt(f) + } +} + +#[macro_export] +macro_rules! prop_assert_biteq { + { $a:expr, $b:expr $(,)? } => { + { + use $crate::biteq::BitEqWrapper; + let a = $a; + let b = $b; + proptest::prop_assert_eq!(BitEqWrapper(&a), BitEqWrapper(&b)); + } + } +} diff --git a/crux-mir/lib/portable-simd/crates/test_helpers/src/lib.rs b/crux-mir/lib/portable-simd/crates/test_helpers/src/lib.rs new file mode 100644 index 000000000..141bee18a --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/test_helpers/src/lib.rs @@ -0,0 +1,463 @@ +pub mod array; + +#[cfg(target_arch = "wasm32")] +pub mod wasm; + +#[macro_use] +pub mod biteq; + +/// Specifies the default strategy for testing a type. +/// +/// This strategy should be what "makes sense" to test. +pub trait DefaultStrategy { + type Strategy: proptest::strategy::Strategy; + fn default_strategy() -> Self::Strategy; +} + +macro_rules! impl_num { + { $type:tt } => { + impl DefaultStrategy for $type { + type Strategy = proptest::num::$type::Any; + fn default_strategy() -> Self::Strategy { + proptest::num::$type::ANY + } + } + } +} + +impl_num! { i8 } +impl_num! { i16 } +impl_num! { i32 } +impl_num! { i64 } +impl_num! { isize } +impl_num! { u8 } +impl_num! { u16 } +impl_num! { u32 } +impl_num! { u64 } +impl_num! { usize } +impl_num! { f32 } +impl_num! { f64 } + +#[cfg(not(target_arch = "wasm32"))] +impl DefaultStrategy for u128 { + type Strategy = proptest::num::u128::Any; + fn default_strategy() -> Self::Strategy { + proptest::num::u128::ANY + } +} + +#[cfg(not(target_arch = "wasm32"))] +impl DefaultStrategy for i128 { + type Strategy = proptest::num::i128::Any; + fn default_strategy() -> Self::Strategy { + proptest::num::i128::ANY + } +} + +#[cfg(target_arch = "wasm32")] +impl DefaultStrategy for u128 { + type Strategy = crate::wasm::u128::Any; + fn default_strategy() -> Self::Strategy { + crate::wasm::u128::ANY + } +} + +#[cfg(target_arch = "wasm32")] +impl DefaultStrategy for i128 { + type Strategy = crate::wasm::i128::Any; + fn default_strategy() -> Self::Strategy { + crate::wasm::i128::ANY + } +} + +impl DefaultStrategy for [T; LANES] { + type Strategy = crate::array::UniformArrayStrategy; + fn default_strategy() -> Self::Strategy { + Self::Strategy::new(T::default_strategy()) + } +} + +#[cfg(not(miri))] +pub fn make_runner() -> proptest::test_runner::TestRunner { + Default::default() +} +#[cfg(miri)] +pub fn make_runner() -> proptest::test_runner::TestRunner { + // Only run a few tests on Miri + proptest::test_runner::TestRunner::new(proptest::test_runner::Config::with_cases(4)) +} + +/// Test a function that takes a single value. +pub fn test_1( + f: &dyn Fn(A) -> proptest::test_runner::TestCaseResult, +) { + let mut runner = make_runner(); + runner.run(&A::default_strategy(), f).unwrap(); +} + +/// Test a function that takes two values. +pub fn test_2( + f: &dyn Fn(A, B) -> proptest::test_runner::TestCaseResult, +) { + let mut runner = make_runner(); + runner + .run(&(A::default_strategy(), B::default_strategy()), |(a, b)| { + f(a, b) + }) + .unwrap(); +} + +/// Test a function that takes two values. +pub fn test_3< + A: core::fmt::Debug + DefaultStrategy, + B: core::fmt::Debug + DefaultStrategy, + C: core::fmt::Debug + DefaultStrategy, +>( + f: &dyn Fn(A, B, C) -> proptest::test_runner::TestCaseResult, +) { + let mut runner = make_runner(); + runner + .run( + &( + A::default_strategy(), + B::default_strategy(), + C::default_strategy(), + ), + |(a, b, c)| f(a, b, c), + ) + .unwrap(); +} + +/// Test a unary vector function against a unary scalar function, applied elementwise. +#[inline(never)] +pub fn test_unary_elementwise( + fv: &dyn Fn(Vector) -> VectorResult, + fs: &dyn Fn(Scalar) -> ScalarResult, + check: &dyn Fn([Scalar; LANES]) -> bool, +) where + Scalar: Copy + Default + core::fmt::Debug + DefaultStrategy, + ScalarResult: Copy + Default + biteq::BitEq + core::fmt::Debug + DefaultStrategy, + Vector: Into<[Scalar; LANES]> + From<[Scalar; LANES]> + Copy, + VectorResult: Into<[ScalarResult; LANES]> + From<[ScalarResult; LANES]> + Copy, +{ + test_1(&|x: [Scalar; LANES]| { + proptest::prop_assume!(check(x)); + let result_1: [ScalarResult; LANES] = fv(x.into()).into(); + let result_2: [ScalarResult; LANES] = { + let mut result = [ScalarResult::default(); LANES]; + for (i, o) in x.iter().zip(result.iter_mut()) { + *o = fs(*i); + } + result + }; + crate::prop_assert_biteq!(result_1, result_2); + Ok(()) + }); +} + +/// Test a unary vector function against a unary scalar function, applied elementwise. +#[inline(never)] +pub fn test_unary_mask_elementwise( + fv: &dyn Fn(Vector) -> Mask, + fs: &dyn Fn(Scalar) -> bool, + check: &dyn Fn([Scalar; LANES]) -> bool, +) where + Scalar: Copy + Default + core::fmt::Debug + DefaultStrategy, + Vector: Into<[Scalar; LANES]> + From<[Scalar; LANES]> + Copy, + Mask: Into<[bool; LANES]> + From<[bool; LANES]> + Copy, +{ + test_1(&|x: [Scalar; LANES]| { + proptest::prop_assume!(check(x)); + let result_1: [bool; LANES] = fv(x.into()).into(); + let result_2: [bool; LANES] = { + let mut result = [false; LANES]; + for (i, o) in x.iter().zip(result.iter_mut()) { + *o = fs(*i); + } + result + }; + crate::prop_assert_biteq!(result_1, result_2); + Ok(()) + }); +} + +/// Test a binary vector function against a binary scalar function, applied elementwise. +#[inline(never)] +pub fn test_binary_elementwise< + Scalar1, + Scalar2, + ScalarResult, + Vector1, + Vector2, + VectorResult, + const LANES: usize, +>( + fv: &dyn Fn(Vector1, Vector2) -> VectorResult, + fs: &dyn Fn(Scalar1, Scalar2) -> ScalarResult, + check: &dyn Fn([Scalar1; LANES], [Scalar2; LANES]) -> bool, +) where + Scalar1: Copy + Default + core::fmt::Debug + DefaultStrategy, + Scalar2: Copy + Default + core::fmt::Debug + DefaultStrategy, + ScalarResult: Copy + Default + biteq::BitEq + core::fmt::Debug + DefaultStrategy, + Vector1: Into<[Scalar1; LANES]> + From<[Scalar1; LANES]> + Copy, + Vector2: Into<[Scalar2; LANES]> + From<[Scalar2; LANES]> + Copy, + VectorResult: Into<[ScalarResult; LANES]> + From<[ScalarResult; LANES]> + Copy, +{ + test_2(&|x: [Scalar1; LANES], y: [Scalar2; LANES]| { + proptest::prop_assume!(check(x, y)); + let result_1: [ScalarResult; LANES] = fv(x.into(), y.into()).into(); + let result_2: [ScalarResult; LANES] = { + let mut result = [ScalarResult::default(); LANES]; + for ((i1, i2), o) in x.iter().zip(y.iter()).zip(result.iter_mut()) { + *o = fs(*i1, *i2); + } + result + }; + crate::prop_assert_biteq!(result_1, result_2); + Ok(()) + }); +} + +/// Test a binary vector-scalar function against a binary scalar function, applied elementwise. +#[inline(never)] +pub fn test_binary_scalar_rhs_elementwise< + Scalar1, + Scalar2, + ScalarResult, + Vector, + VectorResult, + const LANES: usize, +>( + fv: &dyn Fn(Vector, Scalar2) -> VectorResult, + fs: &dyn Fn(Scalar1, Scalar2) -> ScalarResult, + check: &dyn Fn([Scalar1; LANES], Scalar2) -> bool, +) where + Scalar1: Copy + Default + core::fmt::Debug + DefaultStrategy, + Scalar2: Copy + Default + core::fmt::Debug + DefaultStrategy, + ScalarResult: Copy + Default + biteq::BitEq + core::fmt::Debug + DefaultStrategy, + Vector: Into<[Scalar1; LANES]> + From<[Scalar1; LANES]> + Copy, + VectorResult: Into<[ScalarResult; LANES]> + From<[ScalarResult; LANES]> + Copy, +{ + test_2(&|x: [Scalar1; LANES], y: Scalar2| { + proptest::prop_assume!(check(x, y)); + let result_1: [ScalarResult; LANES] = fv(x.into(), y).into(); + let result_2: [ScalarResult; LANES] = { + let mut result = [ScalarResult::default(); LANES]; + for (i, o) in x.iter().zip(result.iter_mut()) { + *o = fs(*i, y); + } + result + }; + crate::prop_assert_biteq!(result_1, result_2); + Ok(()) + }); +} + +/// Test a binary vector-scalar function against a binary scalar function, applied elementwise. +#[inline(never)] +pub fn test_binary_scalar_lhs_elementwise< + Scalar1, + Scalar2, + ScalarResult, + Vector, + VectorResult, + const LANES: usize, +>( + fv: &dyn Fn(Scalar1, Vector) -> VectorResult, + fs: &dyn Fn(Scalar1, Scalar2) -> ScalarResult, + check: &dyn Fn(Scalar1, [Scalar2; LANES]) -> bool, +) where + Scalar1: Copy + Default + core::fmt::Debug + DefaultStrategy, + Scalar2: Copy + Default + core::fmt::Debug + DefaultStrategy, + ScalarResult: Copy + Default + biteq::BitEq + core::fmt::Debug + DefaultStrategy, + Vector: Into<[Scalar2; LANES]> + From<[Scalar2; LANES]> + Copy, + VectorResult: Into<[ScalarResult; LANES]> + From<[ScalarResult; LANES]> + Copy, +{ + test_2(&|x: Scalar1, y: [Scalar2; LANES]| { + proptest::prop_assume!(check(x, y)); + let result_1: [ScalarResult; LANES] = fv(x, y.into()).into(); + let result_2: [ScalarResult; LANES] = { + let mut result = [ScalarResult::default(); LANES]; + for (i, o) in y.iter().zip(result.iter_mut()) { + *o = fs(x, *i); + } + result + }; + crate::prop_assert_biteq!(result_1, result_2); + Ok(()) + }); +} + +/// Test a ternary vector function against a ternary scalar function, applied elementwise. +#[inline(never)] +pub fn test_ternary_elementwise< + Scalar1, + Scalar2, + Scalar3, + ScalarResult, + Vector1, + Vector2, + Vector3, + VectorResult, + const LANES: usize, +>( + fv: &dyn Fn(Vector1, Vector2, Vector3) -> VectorResult, + fs: &dyn Fn(Scalar1, Scalar2, Scalar3) -> ScalarResult, + check: &dyn Fn([Scalar1; LANES], [Scalar2; LANES], [Scalar3; LANES]) -> bool, +) where + Scalar1: Copy + Default + core::fmt::Debug + DefaultStrategy, + Scalar2: Copy + Default + core::fmt::Debug + DefaultStrategy, + Scalar3: Copy + Default + core::fmt::Debug + DefaultStrategy, + ScalarResult: Copy + Default + biteq::BitEq + core::fmt::Debug + DefaultStrategy, + Vector1: Into<[Scalar1; LANES]> + From<[Scalar1; LANES]> + Copy, + Vector2: Into<[Scalar2; LANES]> + From<[Scalar2; LANES]> + Copy, + Vector3: Into<[Scalar3; LANES]> + From<[Scalar3; LANES]> + Copy, + VectorResult: Into<[ScalarResult; LANES]> + From<[ScalarResult; LANES]> + Copy, +{ + test_3( + &|x: [Scalar1; LANES], y: [Scalar2; LANES], z: [Scalar3; LANES]| { + proptest::prop_assume!(check(x, y, z)); + let result_1: [ScalarResult; LANES] = fv(x.into(), y.into(), z.into()).into(); + let result_2: [ScalarResult; LANES] = { + let mut result = [ScalarResult::default(); LANES]; + for ((i1, (i2, i3)), o) in + x.iter().zip(y.iter().zip(z.iter())).zip(result.iter_mut()) + { + *o = fs(*i1, *i2, *i3); + } + result + }; + crate::prop_assert_biteq!(result_1, result_2); + Ok(()) + }, + ); +} + +/// Expand a const-generic test into separate tests for each possible lane count. +#[macro_export] +macro_rules! test_lanes { + { + $(fn $test:ident() $body:tt)* + } => { + $( + mod $test { + use super::*; + + fn implementation() + where + core_simd::LaneCount<$lanes>: core_simd::SupportedLaneCount, + $body + + #[cfg(target_arch = "wasm32")] + wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)] + fn lanes_1() { + implementation::<1>(); + } + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)] + fn lanes_2() { + implementation::<2>(); + } + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)] + fn lanes_4() { + implementation::<4>(); + } + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)] + #[cfg(not(miri))] // Miri intrinsic implementations are uniform and larger tests are sloooow + fn lanes_8() { + implementation::<8>(); + } + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)] + #[cfg(not(miri))] // Miri intrinsic implementations are uniform and larger tests are sloooow + fn lanes_16() { + implementation::<16>(); + } + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)] + #[cfg(not(miri))] // Miri intrinsic implementations are uniform and larger tests are sloooow + fn lanes_32() { + implementation::<32>(); + } + + #[test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)] + #[cfg(not(miri))] // Miri intrinsic implementations are uniform and larger tests are sloooow + fn lanes_64() { + implementation::<64>(); + } + } + )* + } +} + +/// Expand a const-generic `#[should_panic]` test into separate tests for each possible lane count. +#[macro_export] +macro_rules! test_lanes_panic { + { + $(fn $test:ident() $body:tt)* + } => { + $( + mod $test { + use super::*; + + fn implementation() + where + core_simd::LaneCount<$lanes>: core_simd::SupportedLaneCount, + $body + + #[test] + #[should_panic] + fn lanes_1() { + implementation::<1>(); + } + + #[test] + #[should_panic] + fn lanes_2() { + implementation::<2>(); + } + + #[test] + #[should_panic] + fn lanes_4() { + implementation::<4>(); + } + + #[test] + #[should_panic] + fn lanes_8() { + implementation::<8>(); + } + + #[test] + #[should_panic] + fn lanes_16() { + implementation::<16>(); + } + + #[test] + #[should_panic] + fn lanes_32() { + implementation::<32>(); + } + + #[test] + #[should_panic] + fn lanes_64() { + implementation::<64>(); + } + } + )* + } +} diff --git a/crux-mir/lib/portable-simd/crates/test_helpers/src/wasm.rs b/crux-mir/lib/portable-simd/crates/test_helpers/src/wasm.rs new file mode 100644 index 000000000..3f11d67cb --- /dev/null +++ b/crux-mir/lib/portable-simd/crates/test_helpers/src/wasm.rs @@ -0,0 +1,51 @@ +//! Strategies for `u128` and `i128`, since proptest doesn't provide them for the wasm target. + +macro_rules! impl_num { + { $name:ident } => { + pub(crate) mod $name { + type InnerStrategy = crate::array::UniformArrayStrategy; + use proptest::strategy::{Strategy, ValueTree, NewTree}; + + + #[must_use = "strategies do nothing unless used"] + #[derive(Clone, Copy, Debug)] + pub struct Any { + strategy: InnerStrategy, + } + + pub struct BinarySearch { + inner: ::Tree, + } + + impl ValueTree for BinarySearch { + type Value = $name; + + fn current(&self) -> $name { + unsafe { core::mem::transmute(self.inner.current()) } + } + + fn simplify(&mut self) -> bool { + self.inner.simplify() + } + + fn complicate(&mut self) -> bool { + self.inner.complicate() + } + } + + impl Strategy for Any { + type Tree = BinarySearch; + type Value = $name; + + fn new_tree(&self, runner: &mut proptest::test_runner::TestRunner) -> NewTree { + Ok(BinarySearch { inner: self.strategy.new_tree(runner)? }) + } + } + + pub const ANY: Any = Any { strategy: InnerStrategy::new(proptest::num::u64::ANY) }; + } + } +} + +impl_num! { u128 } +impl_num! { i128 } diff --git a/crux-mir/lib/proc_macro/Cargo.toml b/crux-mir/lib/proc_macro/Cargo.toml new file mode 100644 index 000000000..e54a50aa1 --- /dev/null +++ b/crux-mir/lib/proc_macro/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "proc_macro" +version = "0.0.0" +edition = "2021" + +[dependencies] +std = { path = "../std" } +# Workaround: when documenting this crate rustdoc will try to load crate named +# `core` when resolving doc links. Without this line a different `core` will be +# loaded from sysroot causing duplicate lang items and other similar errors. +core = { path = "../core" } diff --git a/crux-mir/lib/proc_macro/src/bridge/arena.rs b/crux-mir/lib/proc_macro/src/bridge/arena.rs new file mode 100644 index 000000000..fa72d2816 --- /dev/null +++ b/crux-mir/lib/proc_macro/src/bridge/arena.rs @@ -0,0 +1,113 @@ +//! A minimal arena allocator inspired by `rustc_arena::DroplessArena`. +//! +//! This is unfortunately a minimal re-implementation rather than a dependency +//! as it is difficult to depend on crates from within `proc_macro`, due to it +//! being built at the same time as `std`. + +use std::cell::{Cell, RefCell}; +use std::cmp; +use std::mem::MaybeUninit; +use std::ops::Range; +use std::ptr; +use std::slice; +use std::str; + +// The arenas start with PAGE-sized chunks, and then each new chunk is twice as +// big as its predecessor, up until we reach HUGE_PAGE-sized chunks, whereupon +// we stop growing. This scales well, from arenas that are barely used up to +// arenas that are used for 100s of MiBs. Note also that the chosen sizes match +// the usual sizes of pages and huge pages on Linux. +const PAGE: usize = 4096; +const HUGE_PAGE: usize = 2 * 1024 * 1024; + +/// A minimal arena allocator inspired by `rustc_arena::DroplessArena`. +/// +/// This is unfortunately a complete re-implementation rather than a dependency +/// as it is difficult to depend on crates from within `proc_macro`, due to it +/// being built at the same time as `std`. +/// +/// This arena doesn't have support for allocating anything other than byte +/// slices, as that is all that is necessary. +pub(crate) struct Arena { + start: Cell<*mut MaybeUninit>, + end: Cell<*mut MaybeUninit>, + chunks: RefCell]>>>, +} + +impl Arena { + pub(crate) fn new() -> Self { + Arena { + start: Cell::new(ptr::null_mut()), + end: Cell::new(ptr::null_mut()), + chunks: RefCell::new(Vec::new()), + } + } + + /// Add a new chunk with at least `additional` free bytes. + #[inline(never)] + #[cold] + fn grow(&self, additional: usize) { + let mut chunks = self.chunks.borrow_mut(); + let mut new_cap; + if let Some(last_chunk) = chunks.last_mut() { + // If the previous chunk's len is less than HUGE_PAGE + // bytes, then this chunk will be least double the previous + // chunk's size. + new_cap = last_chunk.len().min(HUGE_PAGE / 2); + new_cap *= 2; + } else { + new_cap = PAGE; + } + // Also ensure that this chunk can fit `additional`. + new_cap = cmp::max(additional, new_cap); + + let mut chunk = Box::new_uninit_slice(new_cap); + let Range { start, end } = chunk.as_mut_ptr_range(); + self.start.set(start); + self.end.set(end); + chunks.push(chunk); + } + + /// Allocates a byte slice with specified size from the current memory + /// chunk. Returns `None` if there is no free space left to satisfy the + /// request. + fn alloc_raw_without_grow(&self, bytes: usize) -> Option<&mut [MaybeUninit]> { + let start = self.start.get().addr(); + let old_end = self.end.get(); + let end = old_end.addr(); + + let new_end = end.checked_sub(bytes)?; + if start <= new_end { + let new_end = old_end.with_addr(new_end); + self.end.set(new_end); + // SAFETY: `bytes` bytes starting at `new_end` were just reserved. + Some(unsafe { slice::from_raw_parts_mut(new_end, bytes) }) + } else { + None + } + } + + fn alloc_raw(&self, bytes: usize) -> &mut [MaybeUninit] { + if bytes == 0 { + return &mut []; + } + + loop { + if let Some(a) = self.alloc_raw_without_grow(bytes) { + break a; + } + // No free space left. Allocate a new chunk to satisfy the request. + // On failure the grow will panic or abort. + self.grow(bytes); + } + } + + pub(crate) fn alloc_str<'a>(&'a self, string: &str) -> &'a mut str { + let alloc = self.alloc_raw(string.len()); + let bytes = MaybeUninit::write_slice(alloc, string.as_bytes()); + + // SAFETY: we convert from `&str` to `&[u8]`, clone it into the arena, + // and immediately convert the clone back to `&str`. + unsafe { str::from_utf8_unchecked_mut(bytes) } + } +} diff --git a/crux-mir/lib/proc_macro/src/bridge/buffer.rs b/crux-mir/lib/proc_macro/src/bridge/buffer.rs new file mode 100644 index 000000000..48030f8d8 --- /dev/null +++ b/crux-mir/lib/proc_macro/src/bridge/buffer.rs @@ -0,0 +1,156 @@ +//! Buffer management for same-process client<->server communication. + +use std::io::{self, Write}; +use std::mem; +use std::ops::{Deref, DerefMut}; +use std::slice; + +#[repr(C)] +pub struct Buffer { + data: *mut u8, + len: usize, + capacity: usize, + reserve: extern "C" fn(Buffer, usize) -> Buffer, + drop: extern "C" fn(Buffer), +} + +unsafe impl Sync for Buffer {} +unsafe impl Send for Buffer {} + +impl Default for Buffer { + #[inline] + fn default() -> Self { + Self::from(vec![]) + } +} + +impl Deref for Buffer { + type Target = [u8]; + #[inline] + fn deref(&self) -> &[u8] { + unsafe { slice::from_raw_parts(self.data as *const u8, self.len) } + } +} + +impl DerefMut for Buffer { + #[inline] + fn deref_mut(&mut self) -> &mut [u8] { + unsafe { slice::from_raw_parts_mut(self.data, self.len) } + } +} + +impl Buffer { + #[inline] + pub(super) fn new() -> Self { + Self::default() + } + + #[inline] + pub(super) fn clear(&mut self) { + self.len = 0; + } + + #[inline] + pub(super) fn take(&mut self) -> Self { + mem::take(self) + } + + // We have the array method separate from extending from a slice. This is + // because in the case of small arrays, codegen can be more efficient + // (avoiding a memmove call). With extend_from_slice, LLVM at least + // currently is not able to make that optimization. + #[inline] + pub(super) fn extend_from_array(&mut self, xs: &[u8; N]) { + if xs.len() > (self.capacity - self.len) { + let b = self.take(); + *self = (b.reserve)(b, xs.len()); + } + unsafe { + xs.as_ptr().copy_to_nonoverlapping(self.data.add(self.len), xs.len()); + self.len += xs.len(); + } + } + + #[inline] + pub(super) fn extend_from_slice(&mut self, xs: &[u8]) { + if xs.len() > (self.capacity - self.len) { + let b = self.take(); + *self = (b.reserve)(b, xs.len()); + } + unsafe { + xs.as_ptr().copy_to_nonoverlapping(self.data.add(self.len), xs.len()); + self.len += xs.len(); + } + } + + #[inline] + pub(super) fn push(&mut self, v: u8) { + // The code here is taken from Vec::push, and we know that reserve() + // will panic if we're exceeding isize::MAX bytes and so there's no need + // to check for overflow. + if self.len == self.capacity { + let b = self.take(); + *self = (b.reserve)(b, 1); + } + unsafe { + *self.data.add(self.len) = v; + self.len += 1; + } + } +} + +impl Write for Buffer { + #[inline] + fn write(&mut self, xs: &[u8]) -> io::Result { + self.extend_from_slice(xs); + Ok(xs.len()) + } + + #[inline] + fn write_all(&mut self, xs: &[u8]) -> io::Result<()> { + self.extend_from_slice(xs); + Ok(()) + } + + #[inline] + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl Drop for Buffer { + #[inline] + fn drop(&mut self) { + let b = self.take(); + (b.drop)(b); + } +} + +impl From> for Buffer { + fn from(mut v: Vec) -> Self { + let (data, len, capacity) = (v.as_mut_ptr(), v.len(), v.capacity()); + mem::forget(v); + + // This utility function is nested in here because it can *only* + // be safely called on `Buffer`s created by *this* `proc_macro`. + fn to_vec(b: Buffer) -> Vec { + unsafe { + let Buffer { data, len, capacity, .. } = b; + mem::forget(b); + Vec::from_raw_parts(data, len, capacity) + } + } + + extern "C" fn reserve(b: Buffer, additional: usize) -> Buffer { + let mut v = to_vec(b); + v.reserve(additional); + Buffer::from(v) + } + + extern "C" fn drop(b: Buffer) { + mem::drop(to_vec(b)); + } + + Buffer { data, len, capacity, reserve, drop } + } +} diff --git a/crux-mir/lib/proc_macro/src/bridge/client.rs b/crux-mir/lib/proc_macro/src/bridge/client.rs new file mode 100644 index 000000000..52a08cad9 --- /dev/null +++ b/crux-mir/lib/proc_macro/src/bridge/client.rs @@ -0,0 +1,508 @@ +//! Client-side types. + +use super::*; + +use std::marker::PhantomData; + +macro_rules! define_handles { + ( + 'owned: $($oty:ident,)* + 'interned: $($ity:ident,)* + ) => { + #[repr(C)] + #[allow(non_snake_case)] + pub struct HandleCounters { + $($oty: AtomicUsize,)* + $($ity: AtomicUsize,)* + } + + impl HandleCounters { + // FIXME(eddyb) use a reference to the `static COUNTERS`, instead of + // a wrapper `fn` pointer, once `const fn` can reference `static`s. + extern "C" fn get() -> &'static Self { + static COUNTERS: HandleCounters = HandleCounters { + $($oty: AtomicUsize::new(1),)* + $($ity: AtomicUsize::new(1),)* + }; + &COUNTERS + } + } + + // FIXME(eddyb) generate the definition of `HandleStore` in `server.rs`. + #[allow(non_snake_case)] + pub(super) struct HandleStore { + $($oty: handle::OwnedStore,)* + $($ity: handle::InternedStore,)* + } + + impl HandleStore { + pub(super) fn new(handle_counters: &'static HandleCounters) -> Self { + HandleStore { + $($oty: handle::OwnedStore::new(&handle_counters.$oty),)* + $($ity: handle::InternedStore::new(&handle_counters.$ity),)* + } + } + } + + $( + pub(crate) struct $oty { + handle: handle::Handle, + // Prevent Send and Sync impls. `!Send`/`!Sync` is the usual + // way of doing this, but that requires unstable features. + // rust-analyzer uses this code and avoids unstable features. + _marker: PhantomData<*mut ()>, + } + + // Forward `Drop::drop` to the inherent `drop` method. + impl Drop for $oty { + fn drop(&mut self) { + $oty { + handle: self.handle, + _marker: PhantomData, + }.drop(); + } + } + + impl Encode for $oty { + fn encode(self, w: &mut Writer, s: &mut S) { + let handle = self.handle; + mem::forget(self); + handle.encode(w, s); + } + } + + impl DecodeMut<'_, '_, HandleStore>> + for Marked + { + fn decode(r: &mut Reader<'_>, s: &mut HandleStore>) -> Self { + s.$oty.take(handle::Handle::decode(r, &mut ())) + } + } + + impl Encode for &$oty { + fn encode(self, w: &mut Writer, s: &mut S) { + self.handle.encode(w, s); + } + } + + impl<'s, S: server::Types> Decode<'_, 's, HandleStore>> + for &'s Marked + { + fn decode(r: &mut Reader<'_>, s: &'s HandleStore>) -> Self { + &s.$oty[handle::Handle::decode(r, &mut ())] + } + } + + impl Encode for &mut $oty { + fn encode(self, w: &mut Writer, s: &mut S) { + self.handle.encode(w, s); + } + } + + impl<'s, S: server::Types> DecodeMut<'_, 's, HandleStore>> + for &'s mut Marked + { + fn decode( + r: &mut Reader<'_>, + s: &'s mut HandleStore> + ) -> Self { + &mut s.$oty[handle::Handle::decode(r, &mut ())] + } + } + + impl Encode>> + for Marked + { + fn encode(self, w: &mut Writer, s: &mut HandleStore>) { + s.$oty.alloc(self).encode(w, s); + } + } + + impl DecodeMut<'_, '_, S> for $oty { + fn decode(r: &mut Reader<'_>, s: &mut S) -> Self { + $oty { + handle: handle::Handle::decode(r, s), + _marker: PhantomData, + } + } + } + )* + + $( + #[derive(Copy, Clone, PartialEq, Eq, Hash)] + pub(crate) struct $ity { + handle: handle::Handle, + // Prevent Send and Sync impls. `!Send`/`!Sync` is the usual + // way of doing this, but that requires unstable features. + // rust-analyzer uses this code and avoids unstable features. + _marker: PhantomData<*mut ()>, + } + + impl Encode for $ity { + fn encode(self, w: &mut Writer, s: &mut S) { + self.handle.encode(w, s); + } + } + + impl DecodeMut<'_, '_, HandleStore>> + for Marked + { + fn decode(r: &mut Reader<'_>, s: &mut HandleStore>) -> Self { + s.$ity.copy(handle::Handle::decode(r, &mut ())) + } + } + + impl Encode>> + for Marked + { + fn encode(self, w: &mut Writer, s: &mut HandleStore>) { + s.$ity.alloc(self).encode(w, s); + } + } + + impl DecodeMut<'_, '_, S> for $ity { + fn decode(r: &mut Reader<'_>, s: &mut S) -> Self { + $ity { + handle: handle::Handle::decode(r, s), + _marker: PhantomData, + } + } + } + )* + } +} +define_handles! { + 'owned: + FreeFunctions, + TokenStream, + SourceFile, + + 'interned: + Span, +} + +// FIXME(eddyb) generate these impls by pattern-matching on the +// names of methods - also could use the presence of `fn drop` +// to distinguish between 'owned and 'interned, above. +// Alternatively, special "modes" could be listed of types in with_api +// instead of pattern matching on methods, here and in server decl. + +impl Clone for TokenStream { + fn clone(&self) -> Self { + self.clone() + } +} + +impl Clone for SourceFile { + fn clone(&self) -> Self { + self.clone() + } +} + +impl Span { + pub(crate) fn def_site() -> Span { + Bridge::with(|bridge| bridge.globals.def_site) + } + + pub(crate) fn call_site() -> Span { + Bridge::with(|bridge| bridge.globals.call_site) + } + + pub(crate) fn mixed_site() -> Span { + Bridge::with(|bridge| bridge.globals.mixed_site) + } +} + +impl fmt::Debug for Span { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.debug()) + } +} + +pub(crate) use super::symbol::Symbol; + +macro_rules! define_client_side { + ($($name:ident { + $(fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* + }),* $(,)?) => { + $(impl $name { + $(pub(crate) fn $method($($arg: $arg_ty),*) $(-> $ret_ty)? { + Bridge::with(|bridge| { + let mut buf = bridge.cached_buffer.take(); + + buf.clear(); + api_tags::Method::$name(api_tags::$name::$method).encode(&mut buf, &mut ()); + reverse_encode!(buf; $($arg),*); + + buf = bridge.dispatch.call(buf); + + let r = Result::<_, PanicMessage>::decode(&mut &buf[..], &mut ()); + + bridge.cached_buffer = buf; + + r.unwrap_or_else(|e| panic::resume_unwind(e.into())) + }) + })* + })* + } +} +with_api!(self, self, define_client_side); + +struct Bridge<'a> { + /// Reusable buffer (only `clear`-ed, never shrunk), primarily + /// used for making requests. + cached_buffer: Buffer, + + /// Server-side function that the client uses to make requests. + dispatch: closure::Closure<'a, Buffer, Buffer>, + + /// Provided globals for this macro expansion. + globals: ExpnGlobals, +} + +impl<'a> !Send for Bridge<'a> {} +impl<'a> !Sync for Bridge<'a> {} + +enum BridgeState<'a> { + /// No server is currently connected to this client. + NotConnected, + + /// A server is connected and available for requests. + Connected(Bridge<'a>), + + /// Access to the bridge is being exclusively acquired + /// (e.g., during `BridgeState::with`). + InUse, +} + +enum BridgeStateL {} + +impl<'a> scoped_cell::ApplyL<'a> for BridgeStateL { + type Out = BridgeState<'a>; +} + +thread_local! { + static BRIDGE_STATE: scoped_cell::ScopedCell = + scoped_cell::ScopedCell::new(BridgeState::NotConnected); +} + +impl BridgeState<'_> { + /// Take exclusive control of the thread-local + /// `BridgeState`, and pass it to `f`, mutably. + /// The state will be restored after `f` exits, even + /// by panic, including modifications made to it by `f`. + /// + /// N.B., while `f` is running, the thread-local state + /// is `BridgeState::InUse`. + fn with(f: impl FnOnce(&mut BridgeState<'_>) -> R) -> R { + BRIDGE_STATE.with(|state| { + state.replace(BridgeState::InUse, |mut state| { + // FIXME(#52812) pass `f` directly to `replace` when `RefMutL` is gone + f(&mut *state) + }) + }) + } +} + +impl Bridge<'_> { + fn with(f: impl FnOnce(&mut Bridge<'_>) -> R) -> R { + BridgeState::with(|state| match state { + BridgeState::NotConnected => { + panic!("procedural macro API is used outside of a procedural macro"); + } + BridgeState::InUse => { + panic!("procedural macro API is used while it's already in use"); + } + BridgeState::Connected(bridge) => f(bridge), + }) + } +} + +pub(crate) fn is_available() -> bool { + BridgeState::with(|state| match state { + BridgeState::Connected(_) | BridgeState::InUse => true, + BridgeState::NotConnected => false, + }) +} + +/// A client-side RPC entry-point, which may be using a different `proc_macro` +/// from the one used by the server, but can be invoked compatibly. +/// +/// Note that the (phantom) `I` ("input") and `O` ("output") type parameters +/// decorate the `Client` with the RPC "interface" of the entry-point, but +/// do not themselves participate in ABI, at all, only facilitate type-checking. +/// +/// E.g. `Client` is the common proc macro interface, +/// used for `#[proc_macro] fn foo(input: TokenStream) -> TokenStream`, +/// indicating that the RPC input and output will be serialized token streams, +/// and forcing the use of APIs that take/return `S::TokenStream`, server-side. +#[repr(C)] +pub struct Client { + // FIXME(eddyb) use a reference to the `static COUNTERS`, instead of + // a wrapper `fn` pointer, once `const fn` can reference `static`s. + pub(super) get_handle_counters: extern "C" fn() -> &'static HandleCounters, + + pub(super) run: extern "C" fn(BridgeConfig<'_>) -> Buffer, + + pub(super) _marker: PhantomData O>, +} + +impl Copy for Client {} +impl Clone for Client { + fn clone(&self) -> Self { + *self + } +} + +fn maybe_install_panic_hook(force_show_panics: bool) { + // Hide the default panic output within `proc_macro` expansions. + // NB. the server can't do this because it may use a different std. + static HIDE_PANICS_DURING_EXPANSION: Once = Once::new(); + HIDE_PANICS_DURING_EXPANSION.call_once(|| { + let prev = panic::take_hook(); + panic::set_hook(Box::new(move |info| { + let show = BridgeState::with(|state| match state { + BridgeState::NotConnected => true, + BridgeState::Connected(_) | BridgeState::InUse => force_show_panics, + }); + if show { + prev(info) + } + })); + }); +} + +/// Client-side helper for handling client panics, entering the bridge, +/// deserializing input and serializing output. +// FIXME(eddyb) maybe replace `Bridge::enter` with this? +fn run_client DecodeMut<'a, 's, ()>, R: Encode<()>>( + config: BridgeConfig<'_>, + f: impl FnOnce(A) -> R, +) -> Buffer { + let BridgeConfig { input: mut buf, dispatch, force_show_panics, .. } = config; + + panic::catch_unwind(panic::AssertUnwindSafe(|| { + maybe_install_panic_hook(force_show_panics); + + // Make sure the symbol store is empty before decoding inputs. + Symbol::invalidate_all(); + + let reader = &mut &buf[..]; + let (globals, input) = <(ExpnGlobals, A)>::decode(reader, &mut ()); + + // Put the buffer we used for input back in the `Bridge` for requests. + let new_state = + BridgeState::Connected(Bridge { cached_buffer: buf.take(), dispatch, globals }); + + BRIDGE_STATE.with(|state| { + state.set(new_state, || { + let output = f(input); + + // Take the `cached_buffer` back out, for the output value. + buf = Bridge::with(|bridge| bridge.cached_buffer.take()); + + // HACK(eddyb) Separate encoding a success value (`Ok(output)`) + // from encoding a panic (`Err(e: PanicMessage)`) to avoid + // having handles outside the `bridge.enter(|| ...)` scope, and + // to catch panics that could happen while encoding the success. + // + // Note that panics should be impossible beyond this point, but + // this is defensively trying to avoid any accidental panicking + // reaching the `extern "C"` (which should `abort` but might not + // at the moment, so this is also potentially preventing UB). + buf.clear(); + Ok::<_, ()>(output).encode(&mut buf, &mut ()); + }) + }) + })) + .map_err(PanicMessage::from) + .unwrap_or_else(|e| { + buf.clear(); + Err::<(), _>(e).encode(&mut buf, &mut ()); + }); + + // Now that a response has been serialized, invalidate all symbols + // registered with the interner. + Symbol::invalidate_all(); + buf +} + +impl Client { + pub const fn expand1(f: impl Fn(crate::TokenStream) -> crate::TokenStream + Copy) -> Self { + Client { + get_handle_counters: HandleCounters::get, + run: super::selfless_reify::reify_to_extern_c_fn_hrt_bridge(move |bridge| { + run_client(bridge, |input| f(crate::TokenStream(Some(input))).0) + }), + _marker: PhantomData, + } + } +} + +impl Client<(crate::TokenStream, crate::TokenStream), crate::TokenStream> { + pub const fn expand2( + f: impl Fn(crate::TokenStream, crate::TokenStream) -> crate::TokenStream + Copy, + ) -> Self { + Client { + get_handle_counters: HandleCounters::get, + run: super::selfless_reify::reify_to_extern_c_fn_hrt_bridge(move |bridge| { + run_client(bridge, |(input, input2)| { + f(crate::TokenStream(Some(input)), crate::TokenStream(Some(input2))).0 + }) + }), + _marker: PhantomData, + } + } +} + +#[repr(C)] +#[derive(Copy, Clone)] +pub enum ProcMacro { + CustomDerive { + trait_name: &'static str, + attributes: &'static [&'static str], + client: Client, + }, + + Attr { + name: &'static str, + client: Client<(crate::TokenStream, crate::TokenStream), crate::TokenStream>, + }, + + Bang { + name: &'static str, + client: Client, + }, +} + +impl ProcMacro { + pub fn name(&self) -> &'static str { + match self { + ProcMacro::CustomDerive { trait_name, .. } => trait_name, + ProcMacro::Attr { name, .. } => name, + ProcMacro::Bang { name, .. } => name, + } + } + + pub const fn custom_derive( + trait_name: &'static str, + attributes: &'static [&'static str], + expand: impl Fn(crate::TokenStream) -> crate::TokenStream + Copy, + ) -> Self { + ProcMacro::CustomDerive { trait_name, attributes, client: Client::expand1(expand) } + } + + pub const fn attr( + name: &'static str, + expand: impl Fn(crate::TokenStream, crate::TokenStream) -> crate::TokenStream + Copy, + ) -> Self { + ProcMacro::Attr { name, client: Client::expand2(expand) } + } + + pub const fn bang( + name: &'static str, + expand: impl Fn(crate::TokenStream) -> crate::TokenStream + Copy, + ) -> Self { + ProcMacro::Bang { name, client: Client::expand1(expand) } + } +} diff --git a/crux-mir/lib/proc_macro/src/bridge/closure.rs b/crux-mir/lib/proc_macro/src/bridge/closure.rs new file mode 100644 index 000000000..d371ae3ce --- /dev/null +++ b/crux-mir/lib/proc_macro/src/bridge/closure.rs @@ -0,0 +1,32 @@ +//! Closure type (equivalent to `&mut dyn FnMut(A) -> R`) that's `repr(C)`. + +use std::marker::PhantomData; + +#[repr(C)] +pub struct Closure<'a, A, R> { + call: unsafe extern "C" fn(*mut Env, A) -> R, + env: *mut Env, + // Prevent Send and Sync impls. `!Send`/`!Sync` is the usual way of doing + // this, but that requires unstable features. rust-analyzer uses this code + // and avoids unstable features. + // + // The `'a` lifetime parameter represents the lifetime of `Env`. + _marker: PhantomData<*mut &'a mut ()>, +} + +struct Env; + +impl<'a, A, R, F: FnMut(A) -> R> From<&'a mut F> for Closure<'a, A, R> { + fn from(f: &'a mut F) -> Self { + unsafe extern "C" fn call R>(env: *mut Env, arg: A) -> R { + (*(env as *mut _ as *mut F))(arg) + } + Closure { call: call::, env: f as *mut _ as *mut Env, _marker: PhantomData } + } +} + +impl<'a, A, R> Closure<'a, A, R> { + pub fn call(&mut self, arg: A) -> R { + unsafe { (self.call)(self.env, arg) } + } +} diff --git a/crux-mir/lib/proc_macro/src/bridge/fxhash.rs b/crux-mir/lib/proc_macro/src/bridge/fxhash.rs new file mode 100644 index 000000000..17bd0a1b3 --- /dev/null +++ b/crux-mir/lib/proc_macro/src/bridge/fxhash.rs @@ -0,0 +1,117 @@ +//! This is a copy of the `rustc_hash` crate, adapted to work as a module. +//! +//! If in the future it becomes more reasonable to add dependencies to +//! `proc_macro`, this module should be removed and replaced with a dependency +//! on the `rustc_hash` crate. + +use std::collections::HashMap; +use std::convert::TryInto; +use std::default::Default; +use std::hash::BuildHasherDefault; +use std::hash::Hasher; +use std::mem::size_of; +use std::ops::BitXor; + +/// Type alias for a hashmap using the `fx` hash algorithm. +pub type FxHashMap = HashMap>; + +/// A speedy hash algorithm for use within rustc. The hashmap in alloc by +/// default uses SipHash which isn't quite as speedy as we want. In the compiler +/// we're not really worried about DOS attempts, so we use a fast +/// non-cryptographic hash. +/// +/// This is the same as the algorithm used by Firefox -- which is a homespun +/// one not based on any widely-known algorithm -- though modified to produce +/// 64-bit hash values instead of 32-bit hash values. It consistently +/// out-performs an FNV-based hash within rustc itself -- the collision rate is +/// similar or slightly worse than FNV, but the speed of the hash function +/// itself is much higher because it works on up to 8 bytes at a time. +pub struct FxHasher { + hash: usize, +} + +#[cfg(target_pointer_width = "32")] +const K: usize = 0x9e3779b9; +#[cfg(target_pointer_width = "64")] +const K: usize = 0x517cc1b727220a95; + +impl Default for FxHasher { + #[inline] + fn default() -> FxHasher { + FxHasher { hash: 0 } + } +} + +impl FxHasher { + #[inline] + fn add_to_hash(&mut self, i: usize) { + self.hash = self.hash.rotate_left(5).bitxor(i).wrapping_mul(K); + } +} + +impl Hasher for FxHasher { + #[inline] + fn write(&mut self, mut bytes: &[u8]) { + #[cfg(target_pointer_width = "32")] + let read_usize = |bytes: &[u8]| u32::from_ne_bytes(bytes[..4].try_into().unwrap()); + #[cfg(target_pointer_width = "64")] + let read_usize = |bytes: &[u8]| u64::from_ne_bytes(bytes[..8].try_into().unwrap()); + + let mut hash = FxHasher { hash: self.hash }; + assert!(size_of::() <= 8); + while bytes.len() >= size_of::() { + hash.add_to_hash(read_usize(bytes) as usize); + bytes = &bytes[size_of::()..]; + } + if (size_of::() > 4) && (bytes.len() >= 4) { + hash.add_to_hash(u32::from_ne_bytes(bytes[..4].try_into().unwrap()) as usize); + bytes = &bytes[4..]; + } + if (size_of::() > 2) && bytes.len() >= 2 { + hash.add_to_hash(u16::from_ne_bytes(bytes[..2].try_into().unwrap()) as usize); + bytes = &bytes[2..]; + } + if (size_of::() > 1) && bytes.len() >= 1 { + hash.add_to_hash(bytes[0] as usize); + } + self.hash = hash.hash; + } + + #[inline] + fn write_u8(&mut self, i: u8) { + self.add_to_hash(i as usize); + } + + #[inline] + fn write_u16(&mut self, i: u16) { + self.add_to_hash(i as usize); + } + + #[inline] + fn write_u32(&mut self, i: u32) { + self.add_to_hash(i as usize); + } + + #[cfg(target_pointer_width = "32")] + #[inline] + fn write_u64(&mut self, i: u64) { + self.add_to_hash(i as usize); + self.add_to_hash((i >> 32) as usize); + } + + #[cfg(target_pointer_width = "64")] + #[inline] + fn write_u64(&mut self, i: u64) { + self.add_to_hash(i as usize); + } + + #[inline] + fn write_usize(&mut self, i: usize) { + self.add_to_hash(i); + } + + #[inline] + fn finish(&self) -> u64 { + self.hash as u64 + } +} diff --git a/crux-mir/lib/proc_macro/src/bridge/handle.rs b/crux-mir/lib/proc_macro/src/bridge/handle.rs new file mode 100644 index 000000000..00954107b --- /dev/null +++ b/crux-mir/lib/proc_macro/src/bridge/handle.rs @@ -0,0 +1,75 @@ +//! Server-side handles and storage for per-handle data. + +use std::collections::BTreeMap; +use std::hash::Hash; +use std::num::NonZeroU32; +use std::ops::{Index, IndexMut}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use super::fxhash::FxHashMap; + +pub(super) type Handle = NonZeroU32; + +/// A store that associates values of type `T` with numeric handles. A value can +/// be looked up using its handle. +pub(super) struct OwnedStore { + counter: &'static AtomicUsize, + data: BTreeMap, +} + +impl OwnedStore { + pub(super) fn new(counter: &'static AtomicUsize) -> Self { + // Ensure the handle counter isn't 0, which would panic later, + // when `NonZeroU32::new` (aka `Handle::new`) is called in `alloc`. + assert_ne!(counter.load(Ordering::SeqCst), 0); + + OwnedStore { counter, data: BTreeMap::new() } + } +} + +impl OwnedStore { + pub(super) fn alloc(&mut self, x: T) -> Handle { + let counter = self.counter.fetch_add(1, Ordering::SeqCst); + let handle = Handle::new(counter as u32).expect("`proc_macro` handle counter overflowed"); + assert!(self.data.insert(handle, x).is_none()); + handle + } + + pub(super) fn take(&mut self, h: Handle) -> T { + self.data.remove(&h).expect("use-after-free in `proc_macro` handle") + } +} + +impl Index for OwnedStore { + type Output = T; + fn index(&self, h: Handle) -> &T { + self.data.get(&h).expect("use-after-free in `proc_macro` handle") + } +} + +impl IndexMut for OwnedStore { + fn index_mut(&mut self, h: Handle) -> &mut T { + self.data.get_mut(&h).expect("use-after-free in `proc_macro` handle") + } +} + +/// Like `OwnedStore`, but avoids storing any value more than once. +pub(super) struct InternedStore { + owned: OwnedStore, + interner: FxHashMap, +} + +impl InternedStore { + pub(super) fn new(counter: &'static AtomicUsize) -> Self { + InternedStore { owned: OwnedStore::new(counter), interner: FxHashMap::default() } + } + + pub(super) fn alloc(&mut self, x: T) -> Handle { + let owned = &mut self.owned; + *self.interner.entry(x).or_insert_with(|| owned.alloc(x)) + } + + pub(super) fn copy(&mut self, h: Handle) -> T { + self.owned[h] + } +} diff --git a/crux-mir/lib/proc_macro/src/bridge/mod.rs b/crux-mir/lib/proc_macro/src/bridge/mod.rs new file mode 100644 index 000000000..4c1e196b5 --- /dev/null +++ b/crux-mir/lib/proc_macro/src/bridge/mod.rs @@ -0,0 +1,521 @@ +//! Internal interface for communicating between a `proc_macro` client +//! (a proc macro crate) and a `proc_macro` server (a compiler front-end). +//! +//! Serialization (with C ABI buffers) and unique integer handles are employed +//! to allow safely interfacing between two copies of `proc_macro` built +//! (from the same source) by different compilers with potentially mismatching +//! Rust ABIs (e.g., stage0/bin/rustc vs stage1/bin/rustc during bootstrap). + +#![deny(unsafe_code)] + +use crate::{Delimiter, Level, LineColumn, Spacing}; +use std::fmt; +use std::hash::Hash; +use std::marker; +use std::mem; +use std::ops::Bound; +use std::panic; +use std::sync::atomic::AtomicUsize; +use std::sync::Once; +use std::thread; + +/// Higher-order macro describing the server RPC API, allowing automatic +/// generation of type-safe Rust APIs, both client-side and server-side. +/// +/// `with_api!(MySelf, my_self, my_macro)` expands to: +/// ```rust,ignore (pseudo-code) +/// my_macro! { +/// // ... +/// Literal { +/// // ... +/// fn character(ch: char) -> MySelf::Literal; +/// // ... +/// fn span(my_self: &MySelf::Literal) -> MySelf::Span; +/// fn set_span(my_self: &mut MySelf::Literal, span: MySelf::Span); +/// }, +/// // ... +/// } +/// ``` +/// +/// The first two arguments serve to customize the arguments names +/// and argument/return types, to enable several different usecases: +/// +/// If `my_self` is just `self`, then each `fn` signature can be used +/// as-is for a method. If it's anything else (`self_` in practice), +/// then the signatures don't have a special `self` argument, and +/// can, therefore, have a different one introduced. +/// +/// If `MySelf` is just `Self`, then the types are only valid inside +/// a trait or a trait impl, where the trait has associated types +/// for each of the API types. If non-associated types are desired, +/// a module name (`self` in practice) can be used instead of `Self`. +macro_rules! with_api { + ($S:ident, $self:ident, $m:ident) => { + $m! { + FreeFunctions { + fn drop($self: $S::FreeFunctions); + fn track_env_var(var: &str, value: Option<&str>); + fn track_path(path: &str); + fn literal_from_str(s: &str) -> Result, ()>; + fn emit_diagnostic(diagnostic: Diagnostic<$S::Span>); + }, + TokenStream { + fn drop($self: $S::TokenStream); + fn clone($self: &$S::TokenStream) -> $S::TokenStream; + fn is_empty($self: &$S::TokenStream) -> bool; + fn expand_expr($self: &$S::TokenStream) -> Result<$S::TokenStream, ()>; + fn from_str(src: &str) -> $S::TokenStream; + fn to_string($self: &$S::TokenStream) -> String; + fn from_token_tree( + tree: TokenTree<$S::TokenStream, $S::Span, $S::Symbol>, + ) -> $S::TokenStream; + fn concat_trees( + base: Option<$S::TokenStream>, + trees: Vec>, + ) -> $S::TokenStream; + fn concat_streams( + base: Option<$S::TokenStream>, + streams: Vec<$S::TokenStream>, + ) -> $S::TokenStream; + fn into_trees( + $self: $S::TokenStream + ) -> Vec>; + }, + SourceFile { + fn drop($self: $S::SourceFile); + fn clone($self: &$S::SourceFile) -> $S::SourceFile; + fn eq($self: &$S::SourceFile, other: &$S::SourceFile) -> bool; + fn path($self: &$S::SourceFile) -> String; + fn is_real($self: &$S::SourceFile) -> bool; + }, + Span { + fn debug($self: $S::Span) -> String; + fn source_file($self: $S::Span) -> $S::SourceFile; + fn parent($self: $S::Span) -> Option<$S::Span>; + fn source($self: $S::Span) -> $S::Span; + fn start($self: $S::Span) -> LineColumn; + fn end($self: $S::Span) -> LineColumn; + fn before($self: $S::Span) -> $S::Span; + fn after($self: $S::Span) -> $S::Span; + fn join($self: $S::Span, other: $S::Span) -> Option<$S::Span>; + fn subspan($self: $S::Span, start: Bound, end: Bound) -> Option<$S::Span>; + fn resolved_at($self: $S::Span, at: $S::Span) -> $S::Span; + fn source_text($self: $S::Span) -> Option; + fn save_span($self: $S::Span) -> usize; + fn recover_proc_macro_span(id: usize) -> $S::Span; + }, + Symbol { + fn normalize_and_validate_ident(string: &str) -> Result<$S::Symbol, ()>; + }, + } + }; +} + +// FIXME(eddyb) this calls `encode` for each argument, but in reverse, +// to match the ordering in `reverse_decode`. +macro_rules! reverse_encode { + ($writer:ident;) => {}; + ($writer:ident; $first:ident $(, $rest:ident)*) => { + reverse_encode!($writer; $($rest),*); + $first.encode(&mut $writer, &mut ()); + } +} + +// FIXME(eddyb) this calls `decode` for each argument, but in reverse, +// to avoid borrow conflicts from borrows started by `&mut` arguments. +macro_rules! reverse_decode { + ($reader:ident, $s:ident;) => {}; + ($reader:ident, $s:ident; $first:ident: $first_ty:ty $(, $rest:ident: $rest_ty:ty)*) => { + reverse_decode!($reader, $s; $($rest: $rest_ty),*); + let $first = <$first_ty>::decode(&mut $reader, $s); + } +} + +#[allow(unsafe_code)] +mod arena; +#[allow(unsafe_code)] +mod buffer; +#[forbid(unsafe_code)] +pub mod client; +#[allow(unsafe_code)] +mod closure; +#[forbid(unsafe_code)] +mod fxhash; +#[forbid(unsafe_code)] +mod handle; +#[macro_use] +#[forbid(unsafe_code)] +mod rpc; +#[allow(unsafe_code)] +mod scoped_cell; +#[allow(unsafe_code)] +mod selfless_reify; +#[forbid(unsafe_code)] +pub mod server; +#[allow(unsafe_code)] +mod symbol; + +use buffer::Buffer; +pub use rpc::PanicMessage; +use rpc::{Decode, DecodeMut, Encode, Reader, Writer}; + +/// Configuration for establishing an active connection between a server and a +/// client. The server creates the bridge config (`run_server` in `server.rs`), +/// then passes it to the client through the function pointer in the `run` field +/// of `client::Client`. The client constructs a local `Bridge` from the config +/// in TLS during its execution (`Bridge::{enter, with}` in `client.rs`). +#[repr(C)] +pub struct BridgeConfig<'a> { + /// Buffer used to pass initial input to the client. + input: Buffer, + + /// Server-side function that the client uses to make requests. + dispatch: closure::Closure<'a, Buffer, Buffer>, + + /// If 'true', always invoke the default panic hook + force_show_panics: bool, + + // Prevent Send and Sync impls. `!Send`/`!Sync` is the usual way of doing + // this, but that requires unstable features. rust-analyzer uses this code + // and avoids unstable features. + _marker: marker::PhantomData<*mut ()>, +} + +#[forbid(unsafe_code)] +#[allow(non_camel_case_types)] +mod api_tags { + use super::rpc::{DecodeMut, Encode, Reader, Writer}; + + macro_rules! declare_tags { + ($($name:ident { + $(fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)*;)* + }),* $(,)?) => { + $( + pub(super) enum $name { + $($method),* + } + rpc_encode_decode!(enum $name { $($method),* }); + )* + + pub(super) enum Method { + $($name($name)),* + } + rpc_encode_decode!(enum Method { $($name(m)),* }); + } + } + with_api!(self, self, declare_tags); +} + +/// Helper to wrap associated types to allow trait impl dispatch. +/// That is, normally a pair of impls for `T::Foo` and `T::Bar` +/// can overlap, but if the impls are, instead, on types like +/// `Marked` and `Marked`, they can't. +trait Mark { + type Unmarked; + fn mark(unmarked: Self::Unmarked) -> Self; +} + +/// Unwrap types wrapped by `Mark::mark` (see `Mark` for details). +trait Unmark { + type Unmarked; + fn unmark(self) -> Self::Unmarked; +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +struct Marked { + value: T, + _marker: marker::PhantomData, +} + +impl Mark for Marked { + type Unmarked = T; + fn mark(unmarked: Self::Unmarked) -> Self { + Marked { value: unmarked, _marker: marker::PhantomData } + } +} +impl Unmark for Marked { + type Unmarked = T; + fn unmark(self) -> Self::Unmarked { + self.value + } +} +impl<'a, T, M> Unmark for &'a Marked { + type Unmarked = &'a T; + fn unmark(self) -> Self::Unmarked { + &self.value + } +} +impl<'a, T, M> Unmark for &'a mut Marked { + type Unmarked = &'a mut T; + fn unmark(self) -> Self::Unmarked { + &mut self.value + } +} + +impl Mark for Vec { + type Unmarked = Vec; + fn mark(unmarked: Self::Unmarked) -> Self { + // Should be a no-op due to std's in-place collect optimizations. + unmarked.into_iter().map(T::mark).collect() + } +} +impl Unmark for Vec { + type Unmarked = Vec; + fn unmark(self) -> Self::Unmarked { + // Should be a no-op due to std's in-place collect optimizations. + self.into_iter().map(T::unmark).collect() + } +} + +macro_rules! mark_noop { + ($($ty:ty),* $(,)?) => { + $( + impl Mark for $ty { + type Unmarked = Self; + fn mark(unmarked: Self::Unmarked) -> Self { + unmarked + } + } + impl Unmark for $ty { + type Unmarked = Self; + fn unmark(self) -> Self::Unmarked { + self + } + } + )* + } +} +mark_noop! { + (), + bool, + char, + &'_ [u8], + &'_ str, + String, + u8, + usize, + Delimiter, + LitKind, + Level, + LineColumn, + Spacing, +} + +rpc_encode_decode!( + enum Delimiter { + Parenthesis, + Brace, + Bracket, + None, + } +); +rpc_encode_decode!( + enum Level { + Error, + Warning, + Note, + Help, + } +); +rpc_encode_decode!(struct LineColumn { line, column }); +rpc_encode_decode!( + enum Spacing { + Alone, + Joint, + } +); + +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +pub enum LitKind { + Byte, + Char, + Integer, + Float, + Str, + StrRaw(u8), + ByteStr, + ByteStrRaw(u8), + Err, +} + +rpc_encode_decode!( + enum LitKind { + Byte, + Char, + Integer, + Float, + Str, + StrRaw(n), + ByteStr, + ByteStrRaw(n), + Err, + } +); + +macro_rules! mark_compound { + (struct $name:ident <$($T:ident),+> { $($field:ident),* $(,)? }) => { + impl<$($T: Mark),+> Mark for $name <$($T),+> { + type Unmarked = $name <$($T::Unmarked),+>; + fn mark(unmarked: Self::Unmarked) -> Self { + $name { + $($field: Mark::mark(unmarked.$field)),* + } + } + } + + impl<$($T: Unmark),+> Unmark for $name <$($T),+> { + type Unmarked = $name <$($T::Unmarked),+>; + fn unmark(self) -> Self::Unmarked { + $name { + $($field: Unmark::unmark(self.$field)),* + } + } + } + }; + (enum $name:ident <$($T:ident),+> { $($variant:ident $(($field:ident))?),* $(,)? }) => { + impl<$($T: Mark),+> Mark for $name <$($T),+> { + type Unmarked = $name <$($T::Unmarked),+>; + fn mark(unmarked: Self::Unmarked) -> Self { + match unmarked { + $($name::$variant $(($field))? => { + $name::$variant $((Mark::mark($field)))? + })* + } + } + } + + impl<$($T: Unmark),+> Unmark for $name <$($T),+> { + type Unmarked = $name <$($T::Unmarked),+>; + fn unmark(self) -> Self::Unmarked { + match self { + $($name::$variant $(($field))? => { + $name::$variant $((Unmark::unmark($field)))? + })* + } + } + } + } +} + +macro_rules! compound_traits { + ($($t:tt)*) => { + rpc_encode_decode!($($t)*); + mark_compound!($($t)*); + }; +} + +compound_traits!( + enum Bound { + Included(x), + Excluded(x), + Unbounded, + } +); + +compound_traits!( + enum Option { + Some(t), + None, + } +); + +compound_traits!( + enum Result { + Ok(t), + Err(e), + } +); + +#[derive(Copy, Clone)] +pub struct DelimSpan { + pub open: Span, + pub close: Span, + pub entire: Span, +} + +impl DelimSpan { + pub fn from_single(span: Span) -> Self { + DelimSpan { open: span, close: span, entire: span } + } +} + +compound_traits!(struct DelimSpan { open, close, entire }); + +#[derive(Clone)] +pub struct Group { + pub delimiter: Delimiter, + pub stream: Option, + pub span: DelimSpan, +} + +compound_traits!(struct Group { delimiter, stream, span }); + +#[derive(Clone)] +pub struct Punct { + pub ch: u8, + pub joint: bool, + pub span: Span, +} + +compound_traits!(struct Punct { ch, joint, span }); + +#[derive(Copy, Clone, Eq, PartialEq)] +pub struct Ident { + pub sym: Symbol, + pub is_raw: bool, + pub span: Span, +} + +compound_traits!(struct Ident { sym, is_raw, span }); + +#[derive(Clone, Eq, PartialEq)] +pub struct Literal { + pub kind: LitKind, + pub symbol: Symbol, + pub suffix: Option, + pub span: Span, +} + +compound_traits!(struct Literal { kind, symbol, suffix, span }); + +#[derive(Clone)] +pub enum TokenTree { + Group(Group), + Punct(Punct), + Ident(Ident), + Literal(Literal), +} + +compound_traits!( + enum TokenTree { + Group(tt), + Punct(tt), + Ident(tt), + Literal(tt), + } +); + +#[derive(Clone, Debug)] +pub struct Diagnostic { + pub level: Level, + pub message: String, + pub spans: Vec, + pub children: Vec>, +} + +compound_traits!( + struct Diagnostic { level, message, spans, children } +); + +/// Globals provided alongside the initial inputs for a macro expansion. +/// Provides values such as spans which are used frequently to avoid RPC. +#[derive(Clone)] +pub struct ExpnGlobals { + pub def_site: Span, + pub call_site: Span, + pub mixed_site: Span, +} + +compound_traits!( + struct ExpnGlobals { def_site, call_site, mixed_site } +); diff --git a/crux-mir/lib/proc_macro/src/bridge/rpc.rs b/crux-mir/lib/proc_macro/src/bridge/rpc.rs new file mode 100644 index 000000000..e9d7a46c0 --- /dev/null +++ b/crux-mir/lib/proc_macro/src/bridge/rpc.rs @@ -0,0 +1,304 @@ +//! Serialization for client-server communication. + +use std::any::Any; +use std::char; +use std::io::Write; +use std::num::NonZeroU32; +use std::str; + +pub(super) type Writer = super::buffer::Buffer; + +pub(super) trait Encode: Sized { + fn encode(self, w: &mut Writer, s: &mut S); +} + +pub(super) type Reader<'a> = &'a [u8]; + +pub(super) trait Decode<'a, 's, S>: Sized { + fn decode(r: &mut Reader<'a>, s: &'s S) -> Self; +} + +pub(super) trait DecodeMut<'a, 's, S>: Sized { + fn decode(r: &mut Reader<'a>, s: &'s mut S) -> Self; +} + +macro_rules! rpc_encode_decode { + (le $ty:ty) => { + impl Encode for $ty { + fn encode(self, w: &mut Writer, _: &mut S) { + w.extend_from_array(&self.to_le_bytes()); + } + } + + impl DecodeMut<'_, '_, S> for $ty { + fn decode(r: &mut Reader<'_>, _: &mut S) -> Self { + const N: usize = ::std::mem::size_of::<$ty>(); + + let mut bytes = [0; N]; + bytes.copy_from_slice(&r[..N]); + *r = &r[N..]; + + Self::from_le_bytes(bytes) + } + } + }; + (struct $name:ident $(<$($T:ident),+>)? { $($field:ident),* $(,)? }) => { + impl),+)?> Encode for $name $(<$($T),+>)? { + fn encode(self, w: &mut Writer, s: &mut S) { + $(self.$field.encode(w, s);)* + } + } + + impl<'a, S, $($($T: for<'s> DecodeMut<'a, 's, S>),+)?> DecodeMut<'a, '_, S> + for $name $(<$($T),+>)? + { + fn decode(r: &mut Reader<'a>, s: &mut S) -> Self { + $name { + $($field: DecodeMut::decode(r, s)),* + } + } + } + }; + (enum $name:ident $(<$($T:ident),+>)? { $($variant:ident $(($field:ident))*),* $(,)? }) => { + impl),+)?> Encode for $name $(<$($T),+>)? { + fn encode(self, w: &mut Writer, s: &mut S) { + // HACK(eddyb): `Tag` enum duplicated between the + // two impls as there's no other place to stash it. + #[allow(non_upper_case_globals)] + mod tag { + #[repr(u8)] enum Tag { $($variant),* } + + $(pub const $variant: u8 = Tag::$variant as u8;)* + } + + match self { + $($name::$variant $(($field))* => { + tag::$variant.encode(w, s); + $($field.encode(w, s);)* + })* + } + } + } + + impl<'a, S, $($($T: for<'s> DecodeMut<'a, 's, S>),+)?> DecodeMut<'a, '_, S> + for $name $(<$($T),+>)? + { + fn decode(r: &mut Reader<'a>, s: &mut S) -> Self { + // HACK(eddyb): `Tag` enum duplicated between the + // two impls as there's no other place to stash it. + #[allow(non_upper_case_globals)] + mod tag { + #[repr(u8)] enum Tag { $($variant),* } + + $(pub const $variant: u8 = Tag::$variant as u8;)* + } + + match u8::decode(r, s) { + $(tag::$variant => { + $(let $field = DecodeMut::decode(r, s);)* + $name::$variant $(($field))* + })* + _ => unreachable!(), + } + } + } + } +} + +impl Encode for () { + fn encode(self, _: &mut Writer, _: &mut S) {} +} + +impl DecodeMut<'_, '_, S> for () { + fn decode(_: &mut Reader<'_>, _: &mut S) -> Self {} +} + +impl Encode for u8 { + fn encode(self, w: &mut Writer, _: &mut S) { + w.push(self); + } +} + +impl DecodeMut<'_, '_, S> for u8 { + fn decode(r: &mut Reader<'_>, _: &mut S) -> Self { + let x = r[0]; + *r = &r[1..]; + x + } +} + +rpc_encode_decode!(le u32); +rpc_encode_decode!(le usize); + +impl Encode for bool { + fn encode(self, w: &mut Writer, s: &mut S) { + (self as u8).encode(w, s); + } +} + +impl DecodeMut<'_, '_, S> for bool { + fn decode(r: &mut Reader<'_>, s: &mut S) -> Self { + match u8::decode(r, s) { + 0 => false, + 1 => true, + _ => unreachable!(), + } + } +} + +impl Encode for char { + fn encode(self, w: &mut Writer, s: &mut S) { + (self as u32).encode(w, s); + } +} + +impl DecodeMut<'_, '_, S> for char { + fn decode(r: &mut Reader<'_>, s: &mut S) -> Self { + char::from_u32(u32::decode(r, s)).unwrap() + } +} + +impl Encode for NonZeroU32 { + fn encode(self, w: &mut Writer, s: &mut S) { + self.get().encode(w, s); + } +} + +impl DecodeMut<'_, '_, S> for NonZeroU32 { + fn decode(r: &mut Reader<'_>, s: &mut S) -> Self { + Self::new(u32::decode(r, s)).unwrap() + } +} + +impl, B: Encode> Encode for (A, B) { + fn encode(self, w: &mut Writer, s: &mut S) { + self.0.encode(w, s); + self.1.encode(w, s); + } +} + +impl<'a, S, A: for<'s> DecodeMut<'a, 's, S>, B: for<'s> DecodeMut<'a, 's, S>> DecodeMut<'a, '_, S> + for (A, B) +{ + fn decode(r: &mut Reader<'a>, s: &mut S) -> Self { + (DecodeMut::decode(r, s), DecodeMut::decode(r, s)) + } +} + +impl Encode for &[u8] { + fn encode(self, w: &mut Writer, s: &mut S) { + self.len().encode(w, s); + w.write_all(self).unwrap(); + } +} + +impl<'a, S> DecodeMut<'a, '_, S> for &'a [u8] { + fn decode(r: &mut Reader<'a>, s: &mut S) -> Self { + let len = usize::decode(r, s); + let xs = &r[..len]; + *r = &r[len..]; + xs + } +} + +impl Encode for &str { + fn encode(self, w: &mut Writer, s: &mut S) { + self.as_bytes().encode(w, s); + } +} + +impl<'a, S> DecodeMut<'a, '_, S> for &'a str { + fn decode(r: &mut Reader<'a>, s: &mut S) -> Self { + str::from_utf8(<&[u8]>::decode(r, s)).unwrap() + } +} + +impl Encode for String { + fn encode(self, w: &mut Writer, s: &mut S) { + self[..].encode(w, s); + } +} + +impl DecodeMut<'_, '_, S> for String { + fn decode(r: &mut Reader<'_>, s: &mut S) -> Self { + <&str>::decode(r, s).to_string() + } +} + +impl> Encode for Vec { + fn encode(self, w: &mut Writer, s: &mut S) { + self.len().encode(w, s); + for x in self { + x.encode(w, s); + } + } +} + +impl<'a, S, T: for<'s> DecodeMut<'a, 's, S>> DecodeMut<'a, '_, S> for Vec { + fn decode(r: &mut Reader<'a>, s: &mut S) -> Self { + let len = usize::decode(r, s); + let mut vec = Vec::with_capacity(len); + for _ in 0..len { + vec.push(T::decode(r, s)); + } + vec + } +} + +/// Simplified version of panic payloads, ignoring +/// types other than `&'static str` and `String`. +pub enum PanicMessage { + StaticStr(&'static str), + String(String), + Unknown, +} + +impl From> for PanicMessage { + fn from(payload: Box) -> Self { + if let Some(s) = payload.downcast_ref::<&'static str>() { + return PanicMessage::StaticStr(s); + } + if let Ok(s) = payload.downcast::() { + return PanicMessage::String(*s); + } + PanicMessage::Unknown + } +} + +impl Into> for PanicMessage { + fn into(self) -> Box { + match self { + PanicMessage::StaticStr(s) => Box::new(s), + PanicMessage::String(s) => Box::new(s), + PanicMessage::Unknown => { + struct UnknownPanicMessage; + Box::new(UnknownPanicMessage) + } + } + } +} + +impl PanicMessage { + pub fn as_str(&self) -> Option<&str> { + match self { + PanicMessage::StaticStr(s) => Some(s), + PanicMessage::String(s) => Some(s), + PanicMessage::Unknown => None, + } + } +} + +impl Encode for PanicMessage { + fn encode(self, w: &mut Writer, s: &mut S) { + self.as_str().encode(w, s); + } +} + +impl DecodeMut<'_, '_, S> for PanicMessage { + fn decode(r: &mut Reader<'_>, s: &mut S) -> Self { + match Option::::decode(r, s) { + Some(s) => PanicMessage::String(s), + None => PanicMessage::Unknown, + } + } +} diff --git a/crux-mir/lib/proc_macro/src/bridge/scoped_cell.rs b/crux-mir/lib/proc_macro/src/bridge/scoped_cell.rs new file mode 100644 index 000000000..2cde1f65a --- /dev/null +++ b/crux-mir/lib/proc_macro/src/bridge/scoped_cell.rs @@ -0,0 +1,81 @@ +//! `Cell` variant for (scoped) existential lifetimes. + +use std::cell::Cell; +use std::mem; +use std::ops::{Deref, DerefMut}; + +/// Type lambda application, with a lifetime. +#[allow(unused_lifetimes)] +pub trait ApplyL<'a> { + type Out; +} + +/// Type lambda taking a lifetime, i.e., `Lifetime -> Type`. +pub trait LambdaL: for<'a> ApplyL<'a> {} + +impl ApplyL<'a>> LambdaL for T {} + +// HACK(eddyb) work around projection limitations with a newtype +// FIXME(#52812) replace with `&'a mut >::Out` +pub struct RefMutL<'a, 'b, T: LambdaL>(&'a mut >::Out); + +impl<'a, 'b, T: LambdaL> Deref for RefMutL<'a, 'b, T> { + type Target = >::Out; + fn deref(&self) -> &Self::Target { + self.0 + } +} + +impl<'a, 'b, T: LambdaL> DerefMut for RefMutL<'a, 'b, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.0 + } +} + +pub struct ScopedCell(Cell<>::Out>); + +impl ScopedCell { + pub const fn new(value: >::Out) -> Self { + ScopedCell(Cell::new(value)) + } + + /// Sets the value in `self` to `replacement` while + /// running `f`, which gets the old value, mutably. + /// The old value will be restored after `f` exits, even + /// by panic, including modifications made to it by `f`. + pub fn replace<'a, R>( + &self, + replacement: >::Out, + f: impl for<'b, 'c> FnOnce(RefMutL<'b, 'c, T>) -> R, + ) -> R { + /// Wrapper that ensures that the cell always gets filled + /// (with the original state, optionally changed by `f`), + /// even if `f` had panicked. + struct PutBackOnDrop<'a, T: LambdaL> { + cell: &'a ScopedCell, + value: Option<>::Out>, + } + + impl<'a, T: LambdaL> Drop for PutBackOnDrop<'a, T> { + fn drop(&mut self) { + self.cell.0.set(self.value.take().unwrap()); + } + } + + let mut put_back_on_drop = PutBackOnDrop { + cell: self, + value: Some(self.0.replace(unsafe { + let erased = mem::transmute_copy(&replacement); + mem::forget(replacement); + erased + })), + }; + + f(RefMutL(put_back_on_drop.value.as_mut().unwrap())) + } + + /// Sets the value in `self` to `value` while running `f`. + pub fn set(&self, value: >::Out, f: impl FnOnce() -> R) -> R { + self.replace(value, |_| f()) + } +} diff --git a/crux-mir/lib/proc_macro/src/bridge/selfless_reify.rs b/crux-mir/lib/proc_macro/src/bridge/selfless_reify.rs new file mode 100644 index 000000000..907ad256e --- /dev/null +++ b/crux-mir/lib/proc_macro/src/bridge/selfless_reify.rs @@ -0,0 +1,84 @@ +//! Abstraction for creating `fn` pointers from any callable that *effectively* +//! has the equivalent of implementing `Default`, even if the compiler neither +//! provides `Default` nor allows reifying closures (i.e. creating `fn` pointers) +//! other than those with absolutely no captures. +//! +//! More specifically, for a closure-like type to be "effectively `Default`": +//! * it must be a ZST (zero-sized type): no information contained within, so +//! that `Default`'s return value (if it were implemented) is unambiguous +//! * it must be `Copy`: no captured "unique ZST tokens" or any other similar +//! types that would make duplicating values at will unsound +//! * combined with the ZST requirement, this confers a kind of "telecopy" +//! ability: similar to `Copy`, but without keeping the value around, and +//! instead "reconstructing" it (a noop given it's a ZST) when needed +//! * it must be *provably* inhabited: no captured uninhabited types or any +//! other types that cannot be constructed by the user of this abstraction +//! * the proof is a value of the closure-like type itself, in a sense the +//! "seed" for the "telecopy" process made possible by ZST + `Copy` +//! * this requirement is the only reason an abstraction limited to a specific +//! usecase is required: ZST + `Copy` can be checked with *at worst* a panic +//! at the "attempted `::default()` call" time, but that doesn't guarantee +//! that the value can be soundly created, and attempting to use the typical +//! "proof ZST token" approach leads yet again to having a ZST + `Copy` type +//! that is not proof of anything without a value (i.e. isomorphic to a +//! newtype of the type it's trying to prove the inhabitation of) +//! +//! A more flexible (and safer) solution to the general problem could exist once +//! `const`-generic parameters can have type parameters in their types: +//! +//! ```rust,ignore (needs future const-generics) +//! extern "C" fn ffi_wrapper< +//! A, R, +//! F: Fn(A) -> R, +//! const f: F, // <-- this `const`-generic is not yet allowed +//! >(arg: A) -> R { +//! f(arg) +//! } +//! ``` + +use std::mem; + +// FIXME(eddyb) this could be `trait` impls except for the `const fn` requirement. +macro_rules! define_reify_functions { + ($( + fn $name:ident $(<$($param:ident),*>)? + for $(extern $abi:tt)? fn($($arg:ident: $arg_ty:ty),*) -> $ret_ty:ty; + )+) => { + $(pub const fn $name< + $($($param,)*)? + F: Fn($($arg_ty),*) -> $ret_ty + Copy + >(f: F) -> $(extern $abi)? fn($($arg_ty),*) -> $ret_ty { + // FIXME(eddyb) describe the `F` type (e.g. via `type_name::`) once panic + // formatting becomes possible in `const fn`. + assert!(mem::size_of::() == 0, "selfless_reify: closure must be zero-sized"); + + $(extern $abi)? fn wrapper< + $($($param,)*)? + F: Fn($($arg_ty),*) -> $ret_ty + Copy + >($($arg: $arg_ty),*) -> $ret_ty { + let f = unsafe { + // SAFETY: `F` satisfies all criteria for "out of thin air" + // reconstructability (see module-level doc comment). + mem::MaybeUninit::::uninit().assume_init() + }; + f($($arg),*) + } + let _f_proof = f; + wrapper::< + $($($param,)*)? + F + > + })+ + } +} + +define_reify_functions! { + fn _reify_to_extern_c_fn_unary for extern "C" fn(arg: A) -> R; + + // HACK(eddyb) this abstraction is used with `for<'a> fn(BridgeConfig<'a>) + // -> T` but that doesn't work with just `reify_to_extern_c_fn_unary` + // because of the `fn` pointer type being "higher-ranked" (i.e. the + // `for<'a>` binder). + // FIXME(eddyb) try to remove the lifetime from `BridgeConfig`, that'd help. + fn reify_to_extern_c_fn_hrt_bridge for extern "C" fn(bridge: super::BridgeConfig<'_>) -> R; +} diff --git a/crux-mir/lib/proc_macro/src/bridge/server.rs b/crux-mir/lib/proc_macro/src/bridge/server.rs new file mode 100644 index 000000000..2ea87d866 --- /dev/null +++ b/crux-mir/lib/proc_macro/src/bridge/server.rs @@ -0,0 +1,372 @@ +//! Server-side traits. + +use super::*; + +use std::cell::Cell; +use std::marker::PhantomData; + +// FIXME(eddyb) generate the definition of `HandleStore` in `server.rs`. +use super::client::HandleStore; + +pub trait Types { + type FreeFunctions: 'static; + type TokenStream: 'static + Clone; + type SourceFile: 'static + Clone; + type Span: 'static + Copy + Eq + Hash; + type Symbol: 'static; +} + +/// Declare an associated fn of one of the traits below, adding necessary +/// default bodies. +macro_rules! associated_fn { + (fn drop(&mut self, $arg:ident: $arg_ty:ty)) => + (fn drop(&mut self, $arg: $arg_ty) { mem::drop($arg) }); + + (fn clone(&mut self, $arg:ident: $arg_ty:ty) -> $ret_ty:ty) => + (fn clone(&mut self, $arg: $arg_ty) -> $ret_ty { $arg.clone() }); + + ($($item:tt)*) => ($($item)*;) +} + +macro_rules! declare_server_traits { + ($($name:ident { + $(fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* + }),* $(,)?) => { + $(pub trait $name: Types { + $(associated_fn!(fn $method(&mut self, $($arg: $arg_ty),*) $(-> $ret_ty)?);)* + })* + + pub trait Server: Types $(+ $name)* { + fn globals(&mut self) -> ExpnGlobals; + + /// Intern a symbol received from RPC + fn intern_symbol(ident: &str) -> Self::Symbol; + + /// Recover the string value of a symbol, and invoke a callback with it. + fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)); + } + } +} +with_api!(Self, self_, declare_server_traits); + +pub(super) struct MarkedTypes(S); + +impl Server for MarkedTypes { + fn globals(&mut self) -> ExpnGlobals { + <_>::mark(Server::globals(&mut self.0)) + } + fn intern_symbol(ident: &str) -> Self::Symbol { + <_>::mark(S::intern_symbol(ident)) + } + fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) { + S::with_symbol_string(symbol.unmark(), f) + } +} + +macro_rules! define_mark_types_impls { + ($($name:ident { + $(fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* + }),* $(,)?) => { + impl Types for MarkedTypes { + $(type $name = Marked;)* + } + + $(impl $name for MarkedTypes { + $(fn $method(&mut self, $($arg: $arg_ty),*) $(-> $ret_ty)? { + <_>::mark($name::$method(&mut self.0, $($arg.unmark()),*)) + })* + })* + } +} +with_api!(Self, self_, define_mark_types_impls); + +struct Dispatcher { + handle_store: HandleStore, + server: S, +} + +macro_rules! define_dispatcher_impl { + ($($name:ident { + $(fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* + }),* $(,)?) => { + // FIXME(eddyb) `pub` only for `ExecutionStrategy` below. + pub trait DispatcherTrait { + // HACK(eddyb) these are here to allow `Self::$name` to work below. + $(type $name;)* + + fn dispatch(&mut self, buf: Buffer) -> Buffer; + } + + impl DispatcherTrait for Dispatcher> { + $(type $name = as Types>::$name;)* + + fn dispatch(&mut self, mut buf: Buffer) -> Buffer { + let Dispatcher { handle_store, server } = self; + + let mut reader = &buf[..]; + match api_tags::Method::decode(&mut reader, &mut ()) { + $(api_tags::Method::$name(m) => match m { + $(api_tags::$name::$method => { + let mut call_method = || { + reverse_decode!(reader, handle_store; $($arg: $arg_ty),*); + $name::$method(server, $($arg),*) + }; + // HACK(eddyb) don't use `panic::catch_unwind` in a panic. + // If client and server happen to use the same `std`, + // `catch_unwind` asserts that the panic counter was 0, + // even when the closure passed to it didn't panic. + let r = if thread::panicking() { + Ok(call_method()) + } else { + panic::catch_unwind(panic::AssertUnwindSafe(call_method)) + .map_err(PanicMessage::from) + }; + + buf.clear(); + r.encode(&mut buf, handle_store); + })* + }),* + } + buf + } + } + } +} +with_api!(Self, self_, define_dispatcher_impl); + +pub trait ExecutionStrategy { + fn run_bridge_and_client( + &self, + dispatcher: &mut impl DispatcherTrait, + input: Buffer, + run_client: extern "C" fn(BridgeConfig<'_>) -> Buffer, + force_show_panics: bool, + ) -> Buffer; +} + +thread_local! { + /// While running a proc-macro with the same-thread executor, this flag will + /// be set, forcing nested proc-macro invocations (e.g. due to + /// `TokenStream::expand_expr`) to be run using a cross-thread executor. + /// + /// This is required as the thread-local state in the proc_macro client does + /// not handle being re-entered, and will invalidate all `Symbol`s when + /// entering a nested macro. + static ALREADY_RUNNING_SAME_THREAD: Cell = Cell::new(false); +} + +/// Keep `ALREADY_RUNNING_SAME_THREAD` (see also its documentation) +/// set to `true`, preventing same-thread reentrance. +struct RunningSameThreadGuard(()); + +impl RunningSameThreadGuard { + fn new() -> Self { + let already_running = ALREADY_RUNNING_SAME_THREAD.replace(true); + assert!( + !already_running, + "same-thread nesting (\"reentrance\") of proc macro executions is not supported" + ); + RunningSameThreadGuard(()) + } +} + +impl Drop for RunningSameThreadGuard { + fn drop(&mut self) { + ALREADY_RUNNING_SAME_THREAD.set(false); + } +} + +pub struct MaybeCrossThread

{ + cross_thread: bool, + marker: PhantomData

, +} + +impl

MaybeCrossThread

{ + pub const fn new(cross_thread: bool) -> Self { + MaybeCrossThread { cross_thread, marker: PhantomData } + } +} + +impl

ExecutionStrategy for MaybeCrossThread

+where + P: MessagePipe + Send + 'static, +{ + fn run_bridge_and_client( + &self, + dispatcher: &mut impl DispatcherTrait, + input: Buffer, + run_client: extern "C" fn(BridgeConfig<'_>) -> Buffer, + force_show_panics: bool, + ) -> Buffer { + if self.cross_thread || ALREADY_RUNNING_SAME_THREAD.get() { + >::new().run_bridge_and_client( + dispatcher, + input, + run_client, + force_show_panics, + ) + } else { + SameThread.run_bridge_and_client(dispatcher, input, run_client, force_show_panics) + } + } +} + +pub struct SameThread; + +impl ExecutionStrategy for SameThread { + fn run_bridge_and_client( + &self, + dispatcher: &mut impl DispatcherTrait, + input: Buffer, + run_client: extern "C" fn(BridgeConfig<'_>) -> Buffer, + force_show_panics: bool, + ) -> Buffer { + let _guard = RunningSameThreadGuard::new(); + + let mut dispatch = |buf| dispatcher.dispatch(buf); + + run_client(BridgeConfig { + input, + dispatch: (&mut dispatch).into(), + force_show_panics, + _marker: marker::PhantomData, + }) + } +} + +pub struct CrossThread

(PhantomData

); + +impl

CrossThread

{ + pub const fn new() -> Self { + CrossThread(PhantomData) + } +} + +impl

ExecutionStrategy for CrossThread

+where + P: MessagePipe + Send + 'static, +{ + fn run_bridge_and_client( + &self, + dispatcher: &mut impl DispatcherTrait, + input: Buffer, + run_client: extern "C" fn(BridgeConfig<'_>) -> Buffer, + force_show_panics: bool, + ) -> Buffer { + let (mut server, mut client) = P::new(); + + let join_handle = thread::spawn(move || { + let mut dispatch = |b: Buffer| -> Buffer { + client.send(b); + client.recv().expect("server died while client waiting for reply") + }; + + run_client(BridgeConfig { + input, + dispatch: (&mut dispatch).into(), + force_show_panics, + _marker: marker::PhantomData, + }) + }); + + while let Some(b) = server.recv() { + server.send(dispatcher.dispatch(b)); + } + + join_handle.join().unwrap() + } +} + +/// A message pipe used for communicating between server and client threads. +pub trait MessagePipe: Sized { + /// Create a new pair of endpoints for the message pipe. + fn new() -> (Self, Self); + + /// Send a message to the other endpoint of this pipe. + fn send(&mut self, value: T); + + /// Receive a message from the other endpoint of this pipe. + /// + /// Returns `None` if the other end of the pipe has been destroyed, and no + /// message was received. + fn recv(&mut self) -> Option; +} + +fn run_server< + S: Server, + I: Encode>>, + O: for<'a, 's> DecodeMut<'a, 's, HandleStore>>, +>( + strategy: &impl ExecutionStrategy, + handle_counters: &'static client::HandleCounters, + server: S, + input: I, + run_client: extern "C" fn(BridgeConfig<'_>) -> Buffer, + force_show_panics: bool, +) -> Result { + let mut dispatcher = + Dispatcher { handle_store: HandleStore::new(handle_counters), server: MarkedTypes(server) }; + + let globals = dispatcher.server.globals(); + + let mut buf = Buffer::new(); + (globals, input).encode(&mut buf, &mut dispatcher.handle_store); + + buf = strategy.run_bridge_and_client(&mut dispatcher, buf, run_client, force_show_panics); + + Result::decode(&mut &buf[..], &mut dispatcher.handle_store) +} + +impl client::Client { + pub fn run( + &self, + strategy: &impl ExecutionStrategy, + server: S, + input: S::TokenStream, + force_show_panics: bool, + ) -> Result + where + S: Server, + S::TokenStream: Default, + { + let client::Client { get_handle_counters, run, _marker } = *self; + run_server( + strategy, + get_handle_counters(), + server, + as Types>::TokenStream::mark(input), + run, + force_show_panics, + ) + .map(|s| as Types>::TokenStream>>::unmark(s).unwrap_or_default()) + } +} + +impl client::Client<(crate::TokenStream, crate::TokenStream), crate::TokenStream> { + pub fn run( + &self, + strategy: &impl ExecutionStrategy, + server: S, + input: S::TokenStream, + input2: S::TokenStream, + force_show_panics: bool, + ) -> Result + where + S: Server, + S::TokenStream: Default, + { + let client::Client { get_handle_counters, run, _marker } = *self; + run_server( + strategy, + get_handle_counters(), + server, + ( + as Types>::TokenStream::mark(input), + as Types>::TokenStream::mark(input2), + ), + run, + force_show_panics, + ) + .map(|s| as Types>::TokenStream>>::unmark(s).unwrap_or_default()) + } +} diff --git a/crux-mir/lib/proc_macro/src/bridge/symbol.rs b/crux-mir/lib/proc_macro/src/bridge/symbol.rs new file mode 100644 index 000000000..930c11145 --- /dev/null +++ b/crux-mir/lib/proc_macro/src/bridge/symbol.rs @@ -0,0 +1,205 @@ +//! Client-side interner used for symbols. +//! +//! This is roughly based on the symbol interner from `rustc_span` and the +//! DroplessArena from `rustc_arena`. It is unfortunately a complete +//! copy/re-implementation rather than a dependency as it is difficult to depend +//! on crates from within `proc_macro`, due to it being built at the same time +//! as `std`. +//! +//! If at some point in the future it becomes easier to add dependencies to +//! proc_macro, this module should probably be removed or simplified. + +use std::cell::RefCell; +use std::num::NonZeroU32; +use std::str; + +use super::*; + +/// Handle for a symbol string stored within the Interner. +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +pub struct Symbol(NonZeroU32); + +impl !Send for Symbol {} +impl !Sync for Symbol {} + +impl Symbol { + /// Intern a new `Symbol` + pub(crate) fn new(string: &str) -> Self { + INTERNER.with_borrow_mut(|i| i.intern(string)) + } + + /// Create a new `Symbol` for an identifier. + /// + /// Validates and normalizes before converting it to a symbol. + pub(crate) fn new_ident(string: &str, is_raw: bool) -> Self { + // Fast-path: check if this is a valid ASCII identifier + if Self::is_valid_ascii_ident(string.as_bytes()) { + if is_raw && !Self::can_be_raw(string) { + panic!("`{}` cannot be a raw identifier", string); + } + return Self::new(string); + } + + // Slow-path: If the string is already ASCII we're done, otherwise ask + // our server to do this for us over RPC. + // We don't need to check for identifiers which can't be raw here, + // because all of them are ASCII. + if string.is_ascii() { + Err(()) + } else { + client::Symbol::normalize_and_validate_ident(string) + } + .unwrap_or_else(|_| panic!("`{:?}` is not a valid identifier", string)) + } + + /// Run a callback with the symbol's string value. + pub(crate) fn with(self, f: impl FnOnce(&str) -> R) -> R { + INTERNER.with_borrow(|i| f(i.get(self))) + } + + /// Clear out the thread-local symbol interner, making all previously + /// created symbols invalid such that `with` will panic when called on them. + pub(crate) fn invalidate_all() { + INTERNER.with_borrow_mut(|i| i.clear()); + } + + /// Check if the ident is a valid ASCII identifier. + /// + /// This is a short-circuit which is cheap to implement within the + /// proc-macro client to avoid RPC when creating simple idents, but may + /// return `false` for a valid identifier if it contains non-ASCII + /// characters. + fn is_valid_ascii_ident(bytes: &[u8]) -> bool { + matches!(bytes.first(), Some(b'_' | b'a'..=b'z' | b'A'..=b'Z')) + && bytes[1..] + .iter() + .all(|b| matches!(b, b'_' | b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9')) + } + + // Mimics the behaviour of `Symbol::can_be_raw` from `rustc_span` + fn can_be_raw(string: &str) -> bool { + match string { + "_" | "super" | "self" | "Self" | "crate" => false, + _ => true, + } + } +} + +impl fmt::Debug for Symbol { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.with(|s| fmt::Debug::fmt(s, f)) + } +} + +impl ToString for Symbol { + fn to_string(&self) -> String { + self.with(|s| s.to_owned()) + } +} + +impl fmt::Display for Symbol { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.with(|s| fmt::Display::fmt(s, f)) + } +} + +impl Encode for Symbol { + fn encode(self, w: &mut Writer, s: &mut S) { + self.with(|sym| sym.encode(w, s)) + } +} + +impl DecodeMut<'_, '_, client::HandleStore>> + for Marked +{ + fn decode(r: &mut Reader<'_>, s: &mut client::HandleStore>) -> Self { + Mark::mark(S::intern_symbol(<&str>::decode(r, s))) + } +} + +impl Encode>> + for Marked +{ + fn encode(self, w: &mut Writer, s: &mut client::HandleStore>) { + S::with_symbol_string(&self.unmark(), |sym| sym.encode(w, s)) + } +} + +impl DecodeMut<'_, '_, S> for Symbol { + fn decode(r: &mut Reader<'_>, s: &mut S) -> Self { + Symbol::new(<&str>::decode(r, s)) + } +} + +thread_local! { + static INTERNER: RefCell = RefCell::new(Interner { + arena: arena::Arena::new(), + names: fxhash::FxHashMap::default(), + strings: Vec::new(), + // Start with a base of 1 to make sure that `NonZeroU32` works. + sym_base: NonZeroU32::new(1).unwrap(), + }); +} + +/// Basic interner for a `Symbol`, inspired by the one in `rustc_span`. +struct Interner { + arena: arena::Arena, + // SAFETY: These `'static` lifetimes are actually references to data owned + // by the Arena. This is safe, as we never return them as static references + // from `Interner`. + names: fxhash::FxHashMap<&'static str, Symbol>, + strings: Vec<&'static str>, + // The offset to apply to symbol names stored in the interner. This is used + // to ensure that symbol names are not re-used after the interner is + // cleared. + sym_base: NonZeroU32, +} + +impl Interner { + fn intern(&mut self, string: &str) -> Symbol { + if let Some(&name) = self.names.get(string) { + return name; + } + + let name = Symbol( + self.sym_base + .checked_add(self.strings.len() as u32) + .expect("`proc_macro` symbol name overflow"), + ); + + let string: &str = self.arena.alloc_str(string); + + // SAFETY: we can extend the arena allocation to `'static` because we + // only access these while the arena is still alive. + let string: &'static str = unsafe { &*(string as *const str) }; + self.strings.push(string); + self.names.insert(string, name); + name + } + + /// Read a symbol's value from the store while it is held. + fn get(&self, symbol: Symbol) -> &str { + // NOTE: Subtract out the offset which was added to make the symbol + // nonzero and prevent symbol name re-use. + let name = symbol + .0 + .get() + .checked_sub(self.sym_base.get()) + .expect("use-after-free of `proc_macro` symbol"); + self.strings[name as usize] + } + + /// Clear all symbols from the store, invalidating them such that `get` will + /// panic if they are accessed in the future. + fn clear(&mut self) { + // NOTE: Be careful not to panic here, as we may be called on the client + // when a `catch_unwind` isn't installed. + self.sym_base = self.sym_base.saturating_add(self.strings.len() as u32); + self.names.clear(); + self.strings.clear(); + + // SAFETY: This is cleared after the names and strings tables are + // cleared out, so no references into the arena should remain. + self.arena = arena::Arena::new(); + } +} diff --git a/crux-mir/lib/proc_macro/src/diagnostic.rs b/crux-mir/lib/proc_macro/src/diagnostic.rs new file mode 100644 index 000000000..5a209f7c7 --- /dev/null +++ b/crux-mir/lib/proc_macro/src/diagnostic.rs @@ -0,0 +1,175 @@ +use crate::Span; + +/// An enum representing a diagnostic level. +#[unstable(feature = "proc_macro_diagnostic", issue = "54140")] +#[derive(Copy, Clone, Debug)] +#[non_exhaustive] +pub enum Level { + /// An error. + Error, + /// A warning. + Warning, + /// A note. + Note, + /// A help message. + Help, +} + +/// Trait implemented by types that can be converted into a set of `Span`s. +#[unstable(feature = "proc_macro_diagnostic", issue = "54140")] +pub trait MultiSpan { + /// Converts `self` into a `Vec`. + fn into_spans(self) -> Vec; +} + +#[unstable(feature = "proc_macro_diagnostic", issue = "54140")] +impl MultiSpan for Span { + fn into_spans(self) -> Vec { + vec![self] + } +} + +#[unstable(feature = "proc_macro_diagnostic", issue = "54140")] +impl MultiSpan for Vec { + fn into_spans(self) -> Vec { + self + } +} + +#[unstable(feature = "proc_macro_diagnostic", issue = "54140")] +impl<'a> MultiSpan for &'a [Span] { + fn into_spans(self) -> Vec { + self.to_vec() + } +} + +/// A structure representing a diagnostic message and associated children +/// messages. +#[unstable(feature = "proc_macro_diagnostic", issue = "54140")] +#[derive(Clone, Debug)] +pub struct Diagnostic { + level: Level, + message: String, + spans: Vec, + children: Vec, +} + +macro_rules! diagnostic_child_methods { + ($spanned:ident, $regular:ident, $level:expr) => { + #[unstable(feature = "proc_macro_diagnostic", issue = "54140")] + #[doc = concat!("Adds a new child diagnostics message to `self` with the [`", + stringify!($level), "`] level, and the given `spans` and `message`.")] + pub fn $spanned(mut self, spans: S, message: T) -> Diagnostic + where + S: MultiSpan, + T: Into, + { + self.children.push(Diagnostic::spanned(spans, $level, message)); + self + } + + #[unstable(feature = "proc_macro_diagnostic", issue = "54140")] + #[doc = concat!("Adds a new child diagnostic message to `self` with the [`", + stringify!($level), "`] level, and the given `message`.")] + pub fn $regular>(mut self, message: T) -> Diagnostic { + self.children.push(Diagnostic::new($level, message)); + self + } + }; +} + +/// Iterator over the children diagnostics of a `Diagnostic`. +#[derive(Debug, Clone)] +#[unstable(feature = "proc_macro_diagnostic", issue = "54140")] +pub struct Children<'a>(std::slice::Iter<'a, Diagnostic>); + +#[unstable(feature = "proc_macro_diagnostic", issue = "54140")] +impl<'a> Iterator for Children<'a> { + type Item = &'a Diagnostic; + + fn next(&mut self) -> Option { + self.0.next() + } +} + +#[unstable(feature = "proc_macro_diagnostic", issue = "54140")] +impl Diagnostic { + /// Creates a new diagnostic with the given `level` and `message`. + #[unstable(feature = "proc_macro_diagnostic", issue = "54140")] + pub fn new>(level: Level, message: T) -> Diagnostic { + Diagnostic { level, message: message.into(), spans: vec![], children: vec![] } + } + + /// Creates a new diagnostic with the given `level` and `message` pointing to + /// the given set of `spans`. + #[unstable(feature = "proc_macro_diagnostic", issue = "54140")] + pub fn spanned(spans: S, level: Level, message: T) -> Diagnostic + where + S: MultiSpan, + T: Into, + { + Diagnostic { level, message: message.into(), spans: spans.into_spans(), children: vec![] } + } + + diagnostic_child_methods!(span_error, error, Level::Error); + diagnostic_child_methods!(span_warning, warning, Level::Warning); + diagnostic_child_methods!(span_note, note, Level::Note); + diagnostic_child_methods!(span_help, help, Level::Help); + + /// Returns the diagnostic `level` for `self`. + #[unstable(feature = "proc_macro_diagnostic", issue = "54140")] + pub fn level(&self) -> Level { + self.level + } + + /// Sets the level in `self` to `level`. + #[unstable(feature = "proc_macro_diagnostic", issue = "54140")] + pub fn set_level(&mut self, level: Level) { + self.level = level; + } + + /// Returns the message in `self`. + #[unstable(feature = "proc_macro_diagnostic", issue = "54140")] + pub fn message(&self) -> &str { + &self.message + } + + /// Sets the message in `self` to `message`. + #[unstable(feature = "proc_macro_diagnostic", issue = "54140")] + pub fn set_message>(&mut self, message: T) { + self.message = message.into(); + } + + /// Returns the `Span`s in `self`. + #[unstable(feature = "proc_macro_diagnostic", issue = "54140")] + pub fn spans(&self) -> &[Span] { + &self.spans + } + + /// Sets the `Span`s in `self` to `spans`. + #[unstable(feature = "proc_macro_diagnostic", issue = "54140")] + pub fn set_spans(&mut self, spans: S) { + self.spans = spans.into_spans(); + } + + /// Returns an iterator over the children diagnostics of `self`. + #[unstable(feature = "proc_macro_diagnostic", issue = "54140")] + pub fn children(&self) -> Children<'_> { + Children(self.children.iter()) + } + + /// Emit the diagnostic. + #[unstable(feature = "proc_macro_diagnostic", issue = "54140")] + pub fn emit(self) { + fn to_internal(diag: Diagnostic) -> crate::bridge::Diagnostic { + crate::bridge::Diagnostic { + level: diag.level, + message: diag.message, + spans: diag.spans.into_iter().map(|s| s.0).collect(), + children: diag.children.into_iter().map(to_internal).collect(), + } + } + + crate::bridge::client::FreeFunctions::emit_diagnostic(to_internal(self)); + } +} diff --git a/crux-mir/lib/proc_macro/src/lib.rs b/crux-mir/lib/proc_macro/src/lib.rs new file mode 100644 index 000000000..f0e4f5d8a --- /dev/null +++ b/crux-mir/lib/proc_macro/src/lib.rs @@ -0,0 +1,1521 @@ +//! A support library for macro authors when defining new macros. +//! +//! This library, provided by the standard distribution, provides the types +//! consumed in the interfaces of procedurally defined macro definitions such as +//! function-like macros `#[proc_macro]`, macro attributes `#[proc_macro_attribute]` and +//! custom derive attributes`#[proc_macro_derive]`. +//! +//! See [the book] for more. +//! +//! [the book]: ../book/ch19-06-macros.html#procedural-macros-for-generating-code-from-attributes + +#![stable(feature = "proc_macro_lib", since = "1.15.0")] +#![deny(missing_docs)] +#![doc( + html_playground_url = "https://play.rust-lang.org/", + issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/", + test(no_crate_inject, attr(deny(warnings))), + test(attr(allow(dead_code, deprecated, unused_variables, unused_mut))) +)] +// This library is copied into rust-analyzer to allow loading rustc compiled proc macros. +// Please avoid unstable features where possible to minimize the amount of changes necessary +// to make it compile with rust-analyzer on stable. +#![feature(rustc_allow_const_fn_unstable)] +#![feature(staged_api)] +#![feature(allow_internal_unstable)] +#![feature(decl_macro)] +#![feature(local_key_cell_methods)] +#![feature(maybe_uninit_write_slice)] +#![feature(negative_impls)] +#![feature(new_uninit)] +#![feature(restricted_std)] +#![feature(rustc_attrs)] +#![feature(min_specialization)] +#![feature(strict_provenance)] +#![recursion_limit = "256"] + +#[unstable(feature = "proc_macro_internals", issue = "27812")] +#[doc(hidden)] +pub mod bridge; + +mod diagnostic; + +#[unstable(feature = "proc_macro_diagnostic", issue = "54140")] +pub use diagnostic::{Diagnostic, Level, MultiSpan}; + +use std::cmp::Ordering; +use std::ops::RangeBounds; +use std::path::PathBuf; +use std::str::FromStr; +use std::{error, fmt, iter}; + +/// Determines whether proc_macro has been made accessible to the currently +/// running program. +/// +/// The proc_macro crate is only intended for use inside the implementation of +/// procedural macros. All the functions in this crate panic if invoked from +/// outside of a procedural macro, such as from a build script or unit test or +/// ordinary Rust binary. +/// +/// With consideration for Rust libraries that are designed to support both +/// macro and non-macro use cases, `proc_macro::is_available()` provides a +/// non-panicking way to detect whether the infrastructure required to use the +/// API of proc_macro is presently available. Returns true if invoked from +/// inside of a procedural macro, false if invoked from any other binary. +#[stable(feature = "proc_macro_is_available", since = "1.57.0")] +pub fn is_available() -> bool { + bridge::client::is_available() +} + +/// The main type provided by this crate, representing an abstract stream of +/// tokens, or, more specifically, a sequence of token trees. +/// The type provide interfaces for iterating over those token trees and, conversely, +/// collecting a number of token trees into one stream. +/// +/// This is both the input and output of `#[proc_macro]`, `#[proc_macro_attribute]` +/// and `#[proc_macro_derive]` definitions. +#[stable(feature = "proc_macro_lib", since = "1.15.0")] +#[derive(Clone)] +pub struct TokenStream(Option); + +#[stable(feature = "proc_macro_lib", since = "1.15.0")] +impl !Send for TokenStream {} +#[stable(feature = "proc_macro_lib", since = "1.15.0")] +impl !Sync for TokenStream {} + +/// Error returned from `TokenStream::from_str`. +#[stable(feature = "proc_macro_lib", since = "1.15.0")] +#[non_exhaustive] +#[derive(Debug)] +pub struct LexError; + +#[stable(feature = "proc_macro_lexerror_impls", since = "1.44.0")] +impl fmt::Display for LexError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("cannot parse string into token stream") + } +} + +#[stable(feature = "proc_macro_lexerror_impls", since = "1.44.0")] +impl error::Error for LexError {} + +#[stable(feature = "proc_macro_lib", since = "1.15.0")] +impl !Send for LexError {} +#[stable(feature = "proc_macro_lib", since = "1.15.0")] +impl !Sync for LexError {} + +/// Error returned from `TokenStream::expand_expr`. +#[unstable(feature = "proc_macro_expand", issue = "90765")] +#[non_exhaustive] +#[derive(Debug)] +pub struct ExpandError; + +#[unstable(feature = "proc_macro_expand", issue = "90765")] +impl fmt::Display for ExpandError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("macro expansion failed") + } +} + +#[unstable(feature = "proc_macro_expand", issue = "90765")] +impl error::Error for ExpandError {} + +#[unstable(feature = "proc_macro_expand", issue = "90765")] +impl !Send for ExpandError {} + +#[unstable(feature = "proc_macro_expand", issue = "90765")] +impl !Sync for ExpandError {} + +impl TokenStream { + /// Returns an empty `TokenStream` containing no token trees. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn new() -> TokenStream { + TokenStream(None) + } + + /// Checks if this `TokenStream` is empty. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn is_empty(&self) -> bool { + self.0.as_ref().map(|h| h.is_empty()).unwrap_or(true) + } + + /// Parses this `TokenStream` as an expression and attempts to expand any + /// macros within it. Returns the expanded `TokenStream`. + /// + /// Currently only expressions expanding to literals will succeed, although + /// this may be relaxed in the future. + /// + /// NOTE: In error conditions, `expand_expr` may leave macros unexpanded, + /// report an error, failing compilation, and/or return an `Err(..)`. The + /// specific behavior for any error condition, and what conditions are + /// considered errors, is unspecified and may change in the future. + #[unstable(feature = "proc_macro_expand", issue = "90765")] + pub fn expand_expr(&self) -> Result { + let stream = self.0.as_ref().ok_or(ExpandError)?; + match bridge::client::TokenStream::expand_expr(stream) { + Ok(stream) => Ok(TokenStream(Some(stream))), + Err(_) => Err(ExpandError), + } + } +} + +/// Attempts to break the string into tokens and parse those tokens into a token stream. +/// May fail for a number of reasons, for example, if the string contains unbalanced delimiters +/// or characters not existing in the language. +/// All tokens in the parsed stream get `Span::call_site()` spans. +/// +/// NOTE: some errors may cause panics instead of returning `LexError`. We reserve the right to +/// change these errors into `LexError`s later. +#[stable(feature = "proc_macro_lib", since = "1.15.0")] +impl FromStr for TokenStream { + type Err = LexError; + + fn from_str(src: &str) -> Result { + Ok(TokenStream(Some(bridge::client::TokenStream::from_str(src)))) + } +} + +// N.B., the bridge only provides `to_string`, implement `fmt::Display` +// based on it (the reverse of the usual relationship between the two). +#[stable(feature = "proc_macro_lib", since = "1.15.0")] +impl ToString for TokenStream { + fn to_string(&self) -> String { + self.0.as_ref().map(|t| t.to_string()).unwrap_or_default() + } +} + +/// Prints the token stream as a string that is supposed to be losslessly convertible back +/// into the same token stream (modulo spans), except for possibly `TokenTree::Group`s +/// with `Delimiter::None` delimiters and negative numeric literals. +#[stable(feature = "proc_macro_lib", since = "1.15.0")] +impl fmt::Display for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.to_string()) + } +} + +/// Prints token in a form convenient for debugging. +#[stable(feature = "proc_macro_lib", since = "1.15.0")] +impl fmt::Debug for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("TokenStream ")?; + f.debug_list().entries(self.clone()).finish() + } +} + +#[stable(feature = "proc_macro_token_stream_default", since = "1.45.0")] +impl Default for TokenStream { + fn default() -> Self { + TokenStream::new() + } +} + +#[unstable(feature = "proc_macro_quote", issue = "54722")] +pub use quote::{quote, quote_span}; + +fn tree_to_bridge_tree( + tree: TokenTree, +) -> bridge::TokenTree { + match tree { + TokenTree::Group(tt) => bridge::TokenTree::Group(tt.0), + TokenTree::Punct(tt) => bridge::TokenTree::Punct(tt.0), + TokenTree::Ident(tt) => bridge::TokenTree::Ident(tt.0), + TokenTree::Literal(tt) => bridge::TokenTree::Literal(tt.0), + } +} + +/// Creates a token stream containing a single token tree. +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl From for TokenStream { + fn from(tree: TokenTree) -> TokenStream { + TokenStream(Some(bridge::client::TokenStream::from_token_tree(tree_to_bridge_tree(tree)))) + } +} + +/// Non-generic helper for implementing `FromIterator` and +/// `Extend` with less monomorphization in calling crates. +struct ConcatTreesHelper { + trees: Vec< + bridge::TokenTree< + bridge::client::TokenStream, + bridge::client::Span, + bridge::client::Symbol, + >, + >, +} + +impl ConcatTreesHelper { + fn new(capacity: usize) -> Self { + ConcatTreesHelper { trees: Vec::with_capacity(capacity) } + } + + fn push(&mut self, tree: TokenTree) { + self.trees.push(tree_to_bridge_tree(tree)); + } + + fn build(self) -> TokenStream { + if self.trees.is_empty() { + TokenStream(None) + } else { + TokenStream(Some(bridge::client::TokenStream::concat_trees(None, self.trees))) + } + } + + fn append_to(self, stream: &mut TokenStream) { + if self.trees.is_empty() { + return; + } + stream.0 = Some(bridge::client::TokenStream::concat_trees(stream.0.take(), self.trees)) + } +} + +/// Non-generic helper for implementing `FromIterator` and +/// `Extend` with less monomorphization in calling crates. +struct ConcatStreamsHelper { + streams: Vec, +} + +impl ConcatStreamsHelper { + fn new(capacity: usize) -> Self { + ConcatStreamsHelper { streams: Vec::with_capacity(capacity) } + } + + fn push(&mut self, stream: TokenStream) { + if let Some(stream) = stream.0 { + self.streams.push(stream); + } + } + + fn build(mut self) -> TokenStream { + if self.streams.len() <= 1 { + TokenStream(self.streams.pop()) + } else { + TokenStream(Some(bridge::client::TokenStream::concat_streams(None, self.streams))) + } + } + + fn append_to(mut self, stream: &mut TokenStream) { + if self.streams.is_empty() { + return; + } + let base = stream.0.take(); + if base.is_none() && self.streams.len() == 1 { + stream.0 = self.streams.pop(); + } else { + stream.0 = Some(bridge::client::TokenStream::concat_streams(base, self.streams)); + } + } +} + +/// Collects a number of token trees into a single stream. +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl iter::FromIterator for TokenStream { + fn from_iter>(trees: I) -> Self { + let iter = trees.into_iter(); + let mut builder = ConcatTreesHelper::new(iter.size_hint().0); + iter.for_each(|tree| builder.push(tree)); + builder.build() + } +} + +/// A "flattening" operation on token streams, collects token trees +/// from multiple token streams into a single stream. +#[stable(feature = "proc_macro_lib", since = "1.15.0")] +impl iter::FromIterator for TokenStream { + fn from_iter>(streams: I) -> Self { + let iter = streams.into_iter(); + let mut builder = ConcatStreamsHelper::new(iter.size_hint().0); + iter.for_each(|stream| builder.push(stream)); + builder.build() + } +} + +#[stable(feature = "token_stream_extend", since = "1.30.0")] +impl Extend for TokenStream { + fn extend>(&mut self, trees: I) { + let iter = trees.into_iter(); + let mut builder = ConcatTreesHelper::new(iter.size_hint().0); + iter.for_each(|tree| builder.push(tree)); + builder.append_to(self); + } +} + +#[stable(feature = "token_stream_extend", since = "1.30.0")] +impl Extend for TokenStream { + fn extend>(&mut self, streams: I) { + let iter = streams.into_iter(); + let mut builder = ConcatStreamsHelper::new(iter.size_hint().0); + iter.for_each(|stream| builder.push(stream)); + builder.append_to(self); + } +} + +/// Public implementation details for the `TokenStream` type, such as iterators. +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +pub mod token_stream { + use crate::{bridge, Group, Ident, Literal, Punct, TokenStream, TokenTree}; + + /// An iterator over `TokenStream`'s `TokenTree`s. + /// The iteration is "shallow", e.g., the iterator doesn't recurse into delimited groups, + /// and returns whole groups as token trees. + #[derive(Clone)] + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub struct IntoIter( + std::vec::IntoIter< + bridge::TokenTree< + bridge::client::TokenStream, + bridge::client::Span, + bridge::client::Symbol, + >, + >, + ); + + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + impl Iterator for IntoIter { + type Item = TokenTree; + + fn next(&mut self) -> Option { + self.0.next().map(|tree| match tree { + bridge::TokenTree::Group(tt) => TokenTree::Group(Group(tt)), + bridge::TokenTree::Punct(tt) => TokenTree::Punct(Punct(tt)), + bridge::TokenTree::Ident(tt) => TokenTree::Ident(Ident(tt)), + bridge::TokenTree::Literal(tt) => TokenTree::Literal(Literal(tt)), + }) + } + + fn size_hint(&self) -> (usize, Option) { + self.0.size_hint() + } + + fn count(self) -> usize { + self.0.count() + } + } + + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + impl IntoIterator for TokenStream { + type Item = TokenTree; + type IntoIter = IntoIter; + + fn into_iter(self) -> IntoIter { + IntoIter(self.0.map(|v| v.into_trees()).unwrap_or_default().into_iter()) + } + } +} + +/// `quote!(..)` accepts arbitrary tokens and expands into a `TokenStream` describing the input. +/// For example, `quote!(a + b)` will produce an expression, that, when evaluated, constructs +/// the `TokenStream` `[Ident("a"), Punct('+', Alone), Ident("b")]`. +/// +/// Unquoting is done with `$`, and works by taking the single next ident as the unquoted term. +/// To quote `$` itself, use `$$`. +#[unstable(feature = "proc_macro_quote", issue = "54722")] +#[allow_internal_unstable(proc_macro_def_site, proc_macro_internals)] +#[rustc_builtin_macro] +pub macro quote($($t:tt)*) { + /* compiler built-in */ +} + +#[unstable(feature = "proc_macro_internals", issue = "27812")] +#[doc(hidden)] +mod quote; + +/// A region of source code, along with macro expansion information. +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +#[derive(Copy, Clone)] +pub struct Span(bridge::client::Span); + +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl !Send for Span {} +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl !Sync for Span {} + +macro_rules! diagnostic_method { + ($name:ident, $level:expr) => { + /// Creates a new `Diagnostic` with the given `message` at the span + /// `self`. + #[unstable(feature = "proc_macro_diagnostic", issue = "54140")] + pub fn $name>(self, message: T) -> Diagnostic { + Diagnostic::spanned(self, $level, message) + } + }; +} + +impl Span { + /// A span that resolves at the macro definition site. + #[unstable(feature = "proc_macro_def_site", issue = "54724")] + pub fn def_site() -> Span { + Span(bridge::client::Span::def_site()) + } + + /// The span of the invocation of the current procedural macro. + /// Identifiers created with this span will be resolved as if they were written + /// directly at the macro call location (call-site hygiene) and other code + /// at the macro call site will be able to refer to them as well. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn call_site() -> Span { + Span(bridge::client::Span::call_site()) + } + + /// A span that represents `macro_rules` hygiene, and sometimes resolves at the macro + /// definition site (local variables, labels, `$crate`) and sometimes at the macro + /// call site (everything else). + /// The span location is taken from the call-site. + #[stable(feature = "proc_macro_mixed_site", since = "1.45.0")] + pub fn mixed_site() -> Span { + Span(bridge::client::Span::mixed_site()) + } + + /// The original source file into which this span points. + #[unstable(feature = "proc_macro_span", issue = "54725")] + pub fn source_file(&self) -> SourceFile { + SourceFile(self.0.source_file()) + } + + /// The `Span` for the tokens in the previous macro expansion from which + /// `self` was generated from, if any. + #[unstable(feature = "proc_macro_span", issue = "54725")] + pub fn parent(&self) -> Option { + self.0.parent().map(Span) + } + + /// The span for the origin source code that `self` was generated from. If + /// this `Span` wasn't generated from other macro expansions then the return + /// value is the same as `*self`. + #[unstable(feature = "proc_macro_span", issue = "54725")] + pub fn source(&self) -> Span { + Span(self.0.source()) + } + + /// Gets the starting line/column in the source file for this span. + #[unstable(feature = "proc_macro_span", issue = "54725")] + pub fn start(&self) -> LineColumn { + self.0.start().add_1_to_column() + } + + /// Gets the ending line/column in the source file for this span. + #[unstable(feature = "proc_macro_span", issue = "54725")] + pub fn end(&self) -> LineColumn { + self.0.end().add_1_to_column() + } + + /// Creates an empty span pointing to directly before this span. + #[unstable(feature = "proc_macro_span_shrink", issue = "87552")] + pub fn before(&self) -> Span { + Span(self.0.before()) + } + + /// Creates an empty span pointing to directly after this span. + #[unstable(feature = "proc_macro_span_shrink", issue = "87552")] + pub fn after(&self) -> Span { + Span(self.0.after()) + } + + /// Creates a new span encompassing `self` and `other`. + /// + /// Returns `None` if `self` and `other` are from different files. + #[unstable(feature = "proc_macro_span", issue = "54725")] + pub fn join(&self, other: Span) -> Option { + self.0.join(other.0).map(Span) + } + + /// Creates a new span with the same line/column information as `self` but + /// that resolves symbols as though it were at `other`. + #[stable(feature = "proc_macro_span_resolved_at", since = "1.45.0")] + pub fn resolved_at(&self, other: Span) -> Span { + Span(self.0.resolved_at(other.0)) + } + + /// Creates a new span with the same name resolution behavior as `self` but + /// with the line/column information of `other`. + #[stable(feature = "proc_macro_span_located_at", since = "1.45.0")] + pub fn located_at(&self, other: Span) -> Span { + other.resolved_at(*self) + } + + /// Compares two spans to see if they're equal. + #[unstable(feature = "proc_macro_span", issue = "54725")] + pub fn eq(&self, other: &Span) -> bool { + self.0 == other.0 + } + + /// Returns the source text behind a span. This preserves the original source + /// code, including spaces and comments. It only returns a result if the span + /// corresponds to real source code. + /// + /// Note: The observable result of a macro should only rely on the tokens and + /// not on this source text. The result of this function is a best effort to + /// be used for diagnostics only. + #[stable(feature = "proc_macro_source_text", since = "1.66.0")] + pub fn source_text(&self) -> Option { + self.0.source_text() + } + + // Used by the implementation of `Span::quote` + #[doc(hidden)] + #[unstable(feature = "proc_macro_internals", issue = "27812")] + pub fn save_span(&self) -> usize { + self.0.save_span() + } + + // Used by the implementation of `Span::quote` + #[doc(hidden)] + #[unstable(feature = "proc_macro_internals", issue = "27812")] + pub fn recover_proc_macro_span(id: usize) -> Span { + Span(bridge::client::Span::recover_proc_macro_span(id)) + } + + diagnostic_method!(error, Level::Error); + diagnostic_method!(warning, Level::Warning); + diagnostic_method!(note, Level::Note); + diagnostic_method!(help, Level::Help); +} + +/// Prints a span in a form convenient for debugging. +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl fmt::Debug for Span { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +/// A line-column pair representing the start or end of a `Span`. +#[unstable(feature = "proc_macro_span", issue = "54725")] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub struct LineColumn { + /// The 1-indexed line in the source file on which the span starts or ends (inclusive). + #[unstable(feature = "proc_macro_span", issue = "54725")] + pub line: usize, + /// The 1-indexed column (number of bytes in UTF-8 encoding) in the source + /// file on which the span starts or ends (inclusive). + #[unstable(feature = "proc_macro_span", issue = "54725")] + pub column: usize, +} + +impl LineColumn { + fn add_1_to_column(self) -> Self { + LineColumn { line: self.line, column: self.column + 1 } + } +} + +#[unstable(feature = "proc_macro_span", issue = "54725")] +impl !Send for LineColumn {} +#[unstable(feature = "proc_macro_span", issue = "54725")] +impl !Sync for LineColumn {} + +#[unstable(feature = "proc_macro_span", issue = "54725")] +impl Ord for LineColumn { + fn cmp(&self, other: &Self) -> Ordering { + self.line.cmp(&other.line).then(self.column.cmp(&other.column)) + } +} + +#[unstable(feature = "proc_macro_span", issue = "54725")] +impl PartialOrd for LineColumn { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +/// The source file of a given `Span`. +#[unstable(feature = "proc_macro_span", issue = "54725")] +#[derive(Clone)] +pub struct SourceFile(bridge::client::SourceFile); + +impl SourceFile { + /// Gets the path to this source file. + /// + /// ### Note + /// If the code span associated with this `SourceFile` was generated by an external macro, this + /// macro, this might not be an actual path on the filesystem. Use [`is_real`] to check. + /// + /// Also note that even if `is_real` returns `true`, if `--remap-path-prefix` was passed on + /// the command line, the path as given might not actually be valid. + /// + /// [`is_real`]: Self::is_real + #[unstable(feature = "proc_macro_span", issue = "54725")] + pub fn path(&self) -> PathBuf { + PathBuf::from(self.0.path()) + } + + /// Returns `true` if this source file is a real source file, and not generated by an external + /// macro's expansion. + #[unstable(feature = "proc_macro_span", issue = "54725")] + pub fn is_real(&self) -> bool { + // This is a hack until intercrate spans are implemented and we can have real source files + // for spans generated in external macros. + // https://github.com/rust-lang/rust/pull/43604#issuecomment-333334368 + self.0.is_real() + } +} + +#[unstable(feature = "proc_macro_span", issue = "54725")] +impl fmt::Debug for SourceFile { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SourceFile") + .field("path", &self.path()) + .field("is_real", &self.is_real()) + .finish() + } +} + +#[unstable(feature = "proc_macro_span", issue = "54725")] +impl PartialEq for SourceFile { + fn eq(&self, other: &Self) -> bool { + self.0.eq(&other.0) + } +} + +#[unstable(feature = "proc_macro_span", issue = "54725")] +impl Eq for SourceFile {} + +/// A single token or a delimited sequence of token trees (e.g., `[1, (), ..]`). +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +#[derive(Clone)] +pub enum TokenTree { + /// A token stream surrounded by bracket delimiters. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + Group(#[stable(feature = "proc_macro_lib2", since = "1.29.0")] Group), + /// An identifier. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + Ident(#[stable(feature = "proc_macro_lib2", since = "1.29.0")] Ident), + /// A single punctuation character (`+`, `,`, `$`, etc.). + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + Punct(#[stable(feature = "proc_macro_lib2", since = "1.29.0")] Punct), + /// A literal character (`'a'`), string (`"hello"`), number (`2.3`), etc. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + Literal(#[stable(feature = "proc_macro_lib2", since = "1.29.0")] Literal), +} + +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl !Send for TokenTree {} +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl !Sync for TokenTree {} + +impl TokenTree { + /// Returns the span of this tree, delegating to the `span` method of + /// the contained token or a delimited stream. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn span(&self) -> Span { + match *self { + TokenTree::Group(ref t) => t.span(), + TokenTree::Ident(ref t) => t.span(), + TokenTree::Punct(ref t) => t.span(), + TokenTree::Literal(ref t) => t.span(), + } + } + + /// Configures the span for *only this token*. + /// + /// Note that if this token is a `Group` then this method will not configure + /// the span of each of the internal tokens, this will simply delegate to + /// the `set_span` method of each variant. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn set_span(&mut self, span: Span) { + match *self { + TokenTree::Group(ref mut t) => t.set_span(span), + TokenTree::Ident(ref mut t) => t.set_span(span), + TokenTree::Punct(ref mut t) => t.set_span(span), + TokenTree::Literal(ref mut t) => t.set_span(span), + } + } +} + +/// Prints token tree in a form convenient for debugging. +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl fmt::Debug for TokenTree { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Each of these has the name in the struct type in the derived debug, + // so don't bother with an extra layer of indirection + match *self { + TokenTree::Group(ref tt) => tt.fmt(f), + TokenTree::Ident(ref tt) => tt.fmt(f), + TokenTree::Punct(ref tt) => tt.fmt(f), + TokenTree::Literal(ref tt) => tt.fmt(f), + } + } +} + +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl From for TokenTree { + fn from(g: Group) -> TokenTree { + TokenTree::Group(g) + } +} + +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl From for TokenTree { + fn from(g: Ident) -> TokenTree { + TokenTree::Ident(g) + } +} + +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl From for TokenTree { + fn from(g: Punct) -> TokenTree { + TokenTree::Punct(g) + } +} + +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl From for TokenTree { + fn from(g: Literal) -> TokenTree { + TokenTree::Literal(g) + } +} + +// N.B., the bridge only provides `to_string`, implement `fmt::Display` +// based on it (the reverse of the usual relationship between the two). +#[stable(feature = "proc_macro_lib", since = "1.15.0")] +impl ToString for TokenTree { + fn to_string(&self) -> String { + match *self { + TokenTree::Group(ref t) => t.to_string(), + TokenTree::Ident(ref t) => t.to_string(), + TokenTree::Punct(ref t) => t.to_string(), + TokenTree::Literal(ref t) => t.to_string(), + } + } +} + +/// Prints the token tree as a string that is supposed to be losslessly convertible back +/// into the same token tree (modulo spans), except for possibly `TokenTree::Group`s +/// with `Delimiter::None` delimiters and negative numeric literals. +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl fmt::Display for TokenTree { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.to_string()) + } +} + +/// A delimited token stream. +/// +/// A `Group` internally contains a `TokenStream` which is surrounded by `Delimiter`s. +#[derive(Clone)] +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +pub struct Group(bridge::Group); + +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl !Send for Group {} +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl !Sync for Group {} + +/// Describes how a sequence of token trees is delimited. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +pub enum Delimiter { + /// `( ... )` + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + Parenthesis, + /// `{ ... }` + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + Brace, + /// `[ ... ]` + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + Bracket, + /// `Ø ... Ø` + /// An invisible delimiter, that may, for example, appear around tokens coming from a + /// "macro variable" `$var`. It is important to preserve operator priorities in cases like + /// `$var * 3` where `$var` is `1 + 2`. + /// Invisible delimiters might not survive roundtrip of a token stream through a string. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + None, +} + +impl Group { + /// Creates a new `Group` with the given delimiter and token stream. + /// + /// This constructor will set the span for this group to + /// `Span::call_site()`. To change the span you can use the `set_span` + /// method below. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn new(delimiter: Delimiter, stream: TokenStream) -> Group { + Group(bridge::Group { + delimiter, + stream: stream.0, + span: bridge::DelimSpan::from_single(Span::call_site().0), + }) + } + + /// Returns the delimiter of this `Group` + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn delimiter(&self) -> Delimiter { + self.0.delimiter + } + + /// Returns the `TokenStream` of tokens that are delimited in this `Group`. + /// + /// Note that the returned token stream does not include the delimiter + /// returned above. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn stream(&self) -> TokenStream { + TokenStream(self.0.stream.clone()) + } + + /// Returns the span for the delimiters of this token stream, spanning the + /// entire `Group`. + /// + /// ```text + /// pub fn span(&self) -> Span { + /// ^^^^^^^ + /// ``` + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn span(&self) -> Span { + Span(self.0.span.entire) + } + + /// Returns the span pointing to the opening delimiter of this group. + /// + /// ```text + /// pub fn span_open(&self) -> Span { + /// ^ + /// ``` + #[stable(feature = "proc_macro_group_span", since = "1.55.0")] + pub fn span_open(&self) -> Span { + Span(self.0.span.open) + } + + /// Returns the span pointing to the closing delimiter of this group. + /// + /// ```text + /// pub fn span_close(&self) -> Span { + /// ^ + /// ``` + #[stable(feature = "proc_macro_group_span", since = "1.55.0")] + pub fn span_close(&self) -> Span { + Span(self.0.span.close) + } + + /// Configures the span for this `Group`'s delimiters, but not its internal + /// tokens. + /// + /// This method will **not** set the span of all the internal tokens spanned + /// by this group, but rather it will only set the span of the delimiter + /// tokens at the level of the `Group`. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn set_span(&mut self, span: Span) { + self.0.span = bridge::DelimSpan::from_single(span.0); + } +} + +// N.B., the bridge only provides `to_string`, implement `fmt::Display` +// based on it (the reverse of the usual relationship between the two). +#[stable(feature = "proc_macro_lib", since = "1.15.0")] +impl ToString for Group { + fn to_string(&self) -> String { + TokenStream::from(TokenTree::from(self.clone())).to_string() + } +} + +/// Prints the group as a string that should be losslessly convertible back +/// into the same group (modulo spans), except for possibly `TokenTree::Group`s +/// with `Delimiter::None` delimiters. +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl fmt::Display for Group { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.to_string()) + } +} + +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl fmt::Debug for Group { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Group") + .field("delimiter", &self.delimiter()) + .field("stream", &self.stream()) + .field("span", &self.span()) + .finish() + } +} + +/// A `Punct` is a single punctuation character such as `+`, `-` or `#`. +/// +/// Multi-character operators like `+=` are represented as two instances of `Punct` with different +/// forms of `Spacing` returned. +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +#[derive(Clone)] +pub struct Punct(bridge::Punct); + +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl !Send for Punct {} +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl !Sync for Punct {} + +/// Describes whether a `Punct` is followed immediately by another `Punct` ([`Spacing::Joint`]) or +/// by a different token or whitespace ([`Spacing::Alone`]). +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +pub enum Spacing { + /// A `Punct` is not immediately followed by another `Punct`. + /// E.g. `+` is `Alone` in `+ =`, `+ident` and `+()`. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + Alone, + /// A `Punct` is immediately followed by another `Punct`. + /// E.g. `+` is `Joint` in `+=` and `++`. + /// + /// Additionally, single quote `'` can join with identifiers to form lifetimes: `'ident`. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + Joint, +} + +impl Punct { + /// Creates a new `Punct` from the given character and spacing. + /// The `ch` argument must be a valid punctuation character permitted by the language, + /// otherwise the function will panic. + /// + /// The returned `Punct` will have the default span of `Span::call_site()` + /// which can be further configured with the `set_span` method below. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn new(ch: char, spacing: Spacing) -> Punct { + const LEGAL_CHARS: &[char] = &[ + '=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', '&', '|', '@', '.', ',', ';', + ':', '#', '$', '?', '\'', + ]; + if !LEGAL_CHARS.contains(&ch) { + panic!("unsupported character `{:?}`", ch); + } + Punct(bridge::Punct { + ch: ch as u8, + joint: spacing == Spacing::Joint, + span: Span::call_site().0, + }) + } + + /// Returns the value of this punctuation character as `char`. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn as_char(&self) -> char { + self.0.ch as char + } + + /// Returns the spacing of this punctuation character, indicating whether it's immediately + /// followed by another `Punct` in the token stream, so they can potentially be combined into + /// a multi-character operator (`Joint`), or it's followed by some other token or whitespace + /// (`Alone`) so the operator has certainly ended. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn spacing(&self) -> Spacing { + if self.0.joint { Spacing::Joint } else { Spacing::Alone } + } + + /// Returns the span for this punctuation character. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn span(&self) -> Span { + Span(self.0.span) + } + + /// Configure the span for this punctuation character. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn set_span(&mut self, span: Span) { + self.0.span = span.0; + } +} + +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl ToString for Punct { + fn to_string(&self) -> String { + self.as_char().to_string() + } +} + +/// Prints the punctuation character as a string that should be losslessly convertible +/// back into the same character. +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl fmt::Display for Punct { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.as_char()) + } +} + +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl fmt::Debug for Punct { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Punct") + .field("ch", &self.as_char()) + .field("spacing", &self.spacing()) + .field("span", &self.span()) + .finish() + } +} + +#[stable(feature = "proc_macro_punct_eq", since = "1.50.0")] +impl PartialEq for Punct { + fn eq(&self, rhs: &char) -> bool { + self.as_char() == *rhs + } +} + +#[stable(feature = "proc_macro_punct_eq_flipped", since = "1.52.0")] +impl PartialEq for char { + fn eq(&self, rhs: &Punct) -> bool { + *self == rhs.as_char() + } +} + +/// An identifier (`ident`). +#[derive(Clone)] +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +pub struct Ident(bridge::Ident); + +impl Ident { + /// Creates a new `Ident` with the given `string` as well as the specified + /// `span`. + /// The `string` argument must be a valid identifier permitted by the + /// language (including keywords, e.g. `self` or `fn`). Otherwise, the function will panic. + /// + /// Note that `span`, currently in rustc, configures the hygiene information + /// for this identifier. + /// + /// As of this time `Span::call_site()` explicitly opts-in to "call-site" hygiene + /// meaning that identifiers created with this span will be resolved as if they were written + /// directly at the location of the macro call, and other code at the macro call site will be + /// able to refer to them as well. + /// + /// Later spans like `Span::def_site()` will allow to opt-in to "definition-site" hygiene + /// meaning that identifiers created with this span will be resolved at the location of the + /// macro definition and other code at the macro call site will not be able to refer to them. + /// + /// Due to the current importance of hygiene this constructor, unlike other + /// tokens, requires a `Span` to be specified at construction. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn new(string: &str, span: Span) -> Ident { + Ident(bridge::Ident { + sym: bridge::client::Symbol::new_ident(string, false), + is_raw: false, + span: span.0, + }) + } + + /// Same as `Ident::new`, but creates a raw identifier (`r#ident`). + /// The `string` argument be a valid identifier permitted by the language + /// (including keywords, e.g. `fn`). Keywords which are usable in path segments + /// (e.g. `self`, `super`) are not supported, and will cause a panic. + #[stable(feature = "proc_macro_raw_ident", since = "1.47.0")] + pub fn new_raw(string: &str, span: Span) -> Ident { + Ident(bridge::Ident { + sym: bridge::client::Symbol::new_ident(string, true), + is_raw: true, + span: span.0, + }) + } + + /// Returns the span of this `Ident`, encompassing the entire string returned + /// by [`to_string`](ToString::to_string). + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn span(&self) -> Span { + Span(self.0.span) + } + + /// Configures the span of this `Ident`, possibly changing its hygiene context. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn set_span(&mut self, span: Span) { + self.0.span = span.0; + } +} + +/// Converts the identifier to a string that should be losslessly convertible +/// back into the same identifier. +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl ToString for Ident { + fn to_string(&self) -> String { + self.0.sym.with(|sym| if self.0.is_raw { ["r#", sym].concat() } else { sym.to_owned() }) + } +} + +/// Prints the identifier as a string that should be losslessly convertible back +/// into the same identifier. +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl fmt::Display for Ident { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.0.is_raw { + f.write_str("r#")?; + } + fmt::Display::fmt(&self.0.sym, f) + } +} + +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl fmt::Debug for Ident { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Ident") + .field("ident", &self.to_string()) + .field("span", &self.span()) + .finish() + } +} + +/// A literal string (`"hello"`), byte string (`b"hello"`), +/// character (`'a'`), byte character (`b'a'`), an integer or floating point number +/// with or without a suffix (`1`, `1u8`, `2.3`, `2.3f32`). +/// Boolean literals like `true` and `false` do not belong here, they are `Ident`s. +#[derive(Clone)] +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +pub struct Literal(bridge::Literal); + +macro_rules! suffixed_int_literals { + ($($name:ident => $kind:ident,)*) => ($( + /// Creates a new suffixed integer literal with the specified value. + /// + /// This function will create an integer like `1u32` where the integer + /// value specified is the first part of the token and the integral is + /// also suffixed at the end. + /// Literals created from negative numbers might not survive round-trips through + /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal). + /// + /// Literals created through this method have the `Span::call_site()` + /// span by default, which can be configured with the `set_span` method + /// below. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn $name(n: $kind) -> Literal { + Literal(bridge::Literal { + kind: bridge::LitKind::Integer, + symbol: bridge::client::Symbol::new(&n.to_string()), + suffix: Some(bridge::client::Symbol::new(stringify!($kind))), + span: Span::call_site().0, + }) + } + )*) +} + +macro_rules! unsuffixed_int_literals { + ($($name:ident => $kind:ident,)*) => ($( + /// Creates a new unsuffixed integer literal with the specified value. + /// + /// This function will create an integer like `1` where the integer + /// value specified is the first part of the token. No suffix is + /// specified on this token, meaning that invocations like + /// `Literal::i8_unsuffixed(1)` are equivalent to + /// `Literal::u32_unsuffixed(1)`. + /// Literals created from negative numbers might not survive rountrips through + /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal). + /// + /// Literals created through this method have the `Span::call_site()` + /// span by default, which can be configured with the `set_span` method + /// below. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn $name(n: $kind) -> Literal { + Literal(bridge::Literal { + kind: bridge::LitKind::Integer, + symbol: bridge::client::Symbol::new(&n.to_string()), + suffix: None, + span: Span::call_site().0, + }) + } + )*) +} + +impl Literal { + fn new(kind: bridge::LitKind, value: &str, suffix: Option<&str>) -> Self { + Literal(bridge::Literal { + kind, + symbol: bridge::client::Symbol::new(value), + suffix: suffix.map(bridge::client::Symbol::new), + span: Span::call_site().0, + }) + } + + suffixed_int_literals! { + u8_suffixed => u8, + u16_suffixed => u16, + u32_suffixed => u32, + u64_suffixed => u64, + u128_suffixed => u128, + usize_suffixed => usize, + i8_suffixed => i8, + i16_suffixed => i16, + i32_suffixed => i32, + i64_suffixed => i64, + i128_suffixed => i128, + isize_suffixed => isize, + } + + unsuffixed_int_literals! { + u8_unsuffixed => u8, + u16_unsuffixed => u16, + u32_unsuffixed => u32, + u64_unsuffixed => u64, + u128_unsuffixed => u128, + usize_unsuffixed => usize, + i8_unsuffixed => i8, + i16_unsuffixed => i16, + i32_unsuffixed => i32, + i64_unsuffixed => i64, + i128_unsuffixed => i128, + isize_unsuffixed => isize, + } + + /// Creates a new unsuffixed floating-point literal. + /// + /// This constructor is similar to those like `Literal::i8_unsuffixed` where + /// the float's value is emitted directly into the token but no suffix is + /// used, so it may be inferred to be a `f64` later in the compiler. + /// Literals created from negative numbers might not survive rountrips through + /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal). + /// + /// # Panics + /// + /// This function requires that the specified float is finite, for + /// example if it is infinity or NaN this function will panic. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn f32_unsuffixed(n: f32) -> Literal { + if !n.is_finite() { + panic!("Invalid float literal {n}"); + } + let mut repr = n.to_string(); + if !repr.contains('.') { + repr.push_str(".0"); + } + Literal::new(bridge::LitKind::Float, &repr, None) + } + + /// Creates a new suffixed floating-point literal. + /// + /// This constructor will create a literal like `1.0f32` where the value + /// specified is the preceding part of the token and `f32` is the suffix of + /// the token. This token will always be inferred to be an `f32` in the + /// compiler. + /// Literals created from negative numbers might not survive rountrips through + /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal). + /// + /// # Panics + /// + /// This function requires that the specified float is finite, for + /// example if it is infinity or NaN this function will panic. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn f32_suffixed(n: f32) -> Literal { + if !n.is_finite() { + panic!("Invalid float literal {n}"); + } + Literal::new(bridge::LitKind::Float, &n.to_string(), Some("f32")) + } + + /// Creates a new unsuffixed floating-point literal. + /// + /// This constructor is similar to those like `Literal::i8_unsuffixed` where + /// the float's value is emitted directly into the token but no suffix is + /// used, so it may be inferred to be a `f64` later in the compiler. + /// Literals created from negative numbers might not survive rountrips through + /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal). + /// + /// # Panics + /// + /// This function requires that the specified float is finite, for + /// example if it is infinity or NaN this function will panic. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn f64_unsuffixed(n: f64) -> Literal { + if !n.is_finite() { + panic!("Invalid float literal {n}"); + } + let mut repr = n.to_string(); + if !repr.contains('.') { + repr.push_str(".0"); + } + Literal::new(bridge::LitKind::Float, &repr, None) + } + + /// Creates a new suffixed floating-point literal. + /// + /// This constructor will create a literal like `1.0f64` where the value + /// specified is the preceding part of the token and `f64` is the suffix of + /// the token. This token will always be inferred to be an `f64` in the + /// compiler. + /// Literals created from negative numbers might not survive rountrips through + /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal). + /// + /// # Panics + /// + /// This function requires that the specified float is finite, for + /// example if it is infinity or NaN this function will panic. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn f64_suffixed(n: f64) -> Literal { + if !n.is_finite() { + panic!("Invalid float literal {n}"); + } + Literal::new(bridge::LitKind::Float, &n.to_string(), Some("f64")) + } + + /// String literal. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn string(string: &str) -> Literal { + let quoted = format!("{:?}", string); + assert!(quoted.starts_with('"') && quoted.ends_with('"')); + let symbol = "ed[1..quoted.len() - 1]; + Literal::new(bridge::LitKind::Str, symbol, None) + } + + /// Character literal. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn character(ch: char) -> Literal { + let quoted = format!("{:?}", ch); + assert!(quoted.starts_with('\'') && quoted.ends_with('\'')); + let symbol = "ed[1..quoted.len() - 1]; + Literal::new(bridge::LitKind::Char, symbol, None) + } + + /// Byte string literal. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn byte_string(bytes: &[u8]) -> Literal { + let string = bytes.escape_ascii().to_string(); + Literal::new(bridge::LitKind::ByteStr, &string, None) + } + + /// Returns the span encompassing this literal. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn span(&self) -> Span { + Span(self.0.span) + } + + /// Configures the span associated for this literal. + #[stable(feature = "proc_macro_lib2", since = "1.29.0")] + pub fn set_span(&mut self, span: Span) { + self.0.span = span.0; + } + + /// Returns a `Span` that is a subset of `self.span()` containing only the + /// source bytes in range `range`. Returns `None` if the would-be trimmed + /// span is outside the bounds of `self`. + // FIXME(SergioBenitez): check that the byte range starts and ends at a + // UTF-8 boundary of the source. otherwise, it's likely that a panic will + // occur elsewhere when the source text is printed. + // FIXME(SergioBenitez): there is no way for the user to know what + // `self.span()` actually maps to, so this method can currently only be + // called blindly. For example, `to_string()` for the character 'c' returns + // "'\u{63}'"; there is no way for the user to know whether the source text + // was 'c' or whether it was '\u{63}'. + #[unstable(feature = "proc_macro_span", issue = "54725")] + pub fn subspan>(&self, range: R) -> Option { + self.0.span.subspan(range.start_bound().cloned(), range.end_bound().cloned()).map(Span) + } + + fn with_symbol_and_suffix(&self, f: impl FnOnce(&str, &str) -> R) -> R { + self.0.symbol.with(|symbol| match self.0.suffix { + Some(suffix) => suffix.with(|suffix| f(symbol, suffix)), + None => f(symbol, ""), + }) + } + + /// Invokes the callback with a `&[&str]` consisting of each part of the + /// literal's representation. This is done to allow the `ToString` and + /// `Display` implementations to borrow references to symbol values, and + /// both be optimized to reduce overhead. + fn with_stringify_parts(&self, f: impl FnOnce(&[&str]) -> R) -> R { + /// Returns a string containing exactly `num` '#' characters. + /// Uses a 256-character source string literal which is always safe to + /// index with a `u8` index. + fn get_hashes_str(num: u8) -> &'static str { + const HASHES: &str = "\ + ################################################################\ + ################################################################\ + ################################################################\ + ################################################################\ + "; + const _: () = assert!(HASHES.len() == 256); + &HASHES[..num as usize] + } + + self.with_symbol_and_suffix(|symbol, suffix| match self.0.kind { + bridge::LitKind::Byte => f(&["b'", symbol, "'", suffix]), + bridge::LitKind::Char => f(&["'", symbol, "'", suffix]), + bridge::LitKind::Str => f(&["\"", symbol, "\"", suffix]), + bridge::LitKind::StrRaw(n) => { + let hashes = get_hashes_str(n); + f(&["r", hashes, "\"", symbol, "\"", hashes, suffix]) + } + bridge::LitKind::ByteStr => f(&["b\"", symbol, "\"", suffix]), + bridge::LitKind::ByteStrRaw(n) => { + let hashes = get_hashes_str(n); + f(&["br", hashes, "\"", symbol, "\"", hashes, suffix]) + } + _ => f(&[symbol, suffix]), + }) + } +} + +/// Parse a single literal from its stringified representation. +/// +/// In order to parse successfully, the input string must not contain anything +/// but the literal token. Specifically, it must not contain whitespace or +/// comments in addition to the literal. +/// +/// The resulting literal token will have a `Span::call_site()` span. +/// +/// NOTE: some errors may cause panics instead of returning `LexError`. We +/// reserve the right to change these errors into `LexError`s later. +#[stable(feature = "proc_macro_literal_parse", since = "1.54.0")] +impl FromStr for Literal { + type Err = LexError; + + fn from_str(src: &str) -> Result { + match bridge::client::FreeFunctions::literal_from_str(src) { + Ok(literal) => Ok(Literal(literal)), + Err(()) => Err(LexError), + } + } +} + +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl ToString for Literal { + fn to_string(&self) -> String { + self.with_stringify_parts(|parts| parts.concat()) + } +} + +/// Prints the literal as a string that should be losslessly convertible +/// back into the same literal (except for possible rounding for floating point literals). +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl fmt::Display for Literal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.with_stringify_parts(|parts| { + for part in parts { + fmt::Display::fmt(part, f)?; + } + Ok(()) + }) + } +} + +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl fmt::Debug for Literal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Literal") + // format the kind on one line even in {:#?} mode + .field("kind", &format_args!("{:?}", &self.0.kind)) + .field("symbol", &self.0.symbol) + // format `Some("...")` on one line even in {:#?} mode + .field("suffix", &format_args!("{:?}", &self.0.suffix)) + .field("span", &self.0.span) + .finish() + } +} + +/// Tracked access to environment variables. +#[unstable(feature = "proc_macro_tracked_env", issue = "99515")] +pub mod tracked_env { + use std::env::{self, VarError}; + use std::ffi::OsStr; + + /// Retrieve an environment variable and add it to build dependency info. + /// The build system executing the compiler will know that the variable was accessed during + /// compilation, and will be able to rerun the build when the value of that variable changes. + /// Besides the dependency tracking this function should be equivalent to `env::var` from the + /// standard library, except that the argument must be UTF-8. + #[unstable(feature = "proc_macro_tracked_env", issue = "99515")] + pub fn var + AsRef>(key: K) -> Result { + let key: &str = key.as_ref(); + let value = env::var(key); + crate::bridge::client::FreeFunctions::track_env_var(key, value.as_deref().ok()); + value + } +} + +/// Tracked access to additional files. +#[unstable(feature = "track_path", issue = "99515")] +pub mod tracked_path { + + /// Track a file explicitly. + /// + /// Commonly used for tracking asset preprocessing. + #[unstable(feature = "track_path", issue = "99515")] + pub fn path>(path: P) { + let path: &str = path.as_ref(); + crate::bridge::client::FreeFunctions::track_path(path); + } +} diff --git a/crux-mir/lib/proc_macro/src/quote.rs b/crux-mir/lib/proc_macro/src/quote.rs new file mode 100644 index 000000000..04fa696d5 --- /dev/null +++ b/crux-mir/lib/proc_macro/src/quote.rs @@ -0,0 +1,141 @@ +//! # Quasiquoter +//! This file contains the implementation internals of the quasiquoter provided by `quote!`. + +//! This quasiquoter uses macros 2.0 hygiene to reliably access +//! items from `proc_macro`, to build a `proc_macro::TokenStream`. + +use crate::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree}; + +macro_rules! quote_tt { + (($($t:tt)*)) => { Group::new(Delimiter::Parenthesis, quote!($($t)*)) }; + ([$($t:tt)*]) => { Group::new(Delimiter::Bracket, quote!($($t)*)) }; + ({$($t:tt)*}) => { Group::new(Delimiter::Brace, quote!($($t)*)) }; + (,) => { Punct::new(',', Spacing::Alone) }; + (.) => { Punct::new('.', Spacing::Alone) }; + (;) => { Punct::new(';', Spacing::Alone) }; + (!) => { Punct::new('!', Spacing::Alone) }; + (<) => { Punct::new('<', Spacing::Alone) }; + (>) => { Punct::new('>', Spacing::Alone) }; + (&) => { Punct::new('&', Spacing::Alone) }; + (=) => { Punct::new('=', Spacing::Alone) }; + ($i:ident) => { Ident::new(stringify!($i), Span::def_site()) }; +} + +macro_rules! quote_ts { + ((@ $($t:tt)*)) => { $($t)* }; + (::) => { + [ + TokenTree::from(Punct::new(':', Spacing::Joint)), + TokenTree::from(Punct::new(':', Spacing::Alone)), + ].iter() + .cloned() + .map(|mut x| { + x.set_span(Span::def_site()); + x + }) + .collect::() + }; + ($t:tt) => { TokenTree::from(quote_tt!($t)) }; +} + +/// Simpler version of the real `quote!` macro, implemented solely +/// through `macro_rules`, for bootstrapping the real implementation +/// (see the `quote` function), which does not have access to the +/// real `quote!` macro due to the `proc_macro` crate not being +/// able to depend on itself. +/// +/// Note: supported tokens are a subset of the real `quote!`, but +/// unquoting is different: instead of `$x`, this uses `(@ expr)`. +macro_rules! quote { + () => { TokenStream::new() }; + ($($t:tt)*) => { + [ + $(TokenStream::from(quote_ts!($t)),)* + ].iter().cloned().collect::() + }; +} + +/// Quote a `TokenStream` into a `TokenStream`. +/// This is the actual implementation of the `quote!()` proc macro. +/// +/// It is loaded by the compiler in `register_builtin_macros`. +#[unstable(feature = "proc_macro_quote", issue = "54722")] +pub fn quote(stream: TokenStream) -> TokenStream { + if stream.is_empty() { + return quote!(crate::TokenStream::new()); + } + let proc_macro_crate = quote!(crate); + let mut after_dollar = false; + let tokens = stream + .into_iter() + .filter_map(|tree| { + if after_dollar { + after_dollar = false; + match tree { + TokenTree::Ident(_) => { + return Some(quote!(Into::::into( + Clone::clone(&(@ tree))),)); + } + TokenTree::Punct(ref tt) if tt.as_char() == '$' => {} + _ => panic!("`$` must be followed by an ident or `$` in `quote!`"), + } + } else if let TokenTree::Punct(ref tt) = tree { + if tt.as_char() == '$' { + after_dollar = true; + return None; + } + } + + Some(quote!(crate::TokenStream::from((@ match tree { + TokenTree::Punct(tt) => quote!(crate::TokenTree::Punct(crate::Punct::new( + (@ TokenTree::from(Literal::character(tt.as_char()))), + (@ match tt.spacing() { + Spacing::Alone => quote!(crate::Spacing::Alone), + Spacing::Joint => quote!(crate::Spacing::Joint), + }), + ))), + TokenTree::Group(tt) => quote!(crate::TokenTree::Group(crate::Group::new( + (@ match tt.delimiter() { + Delimiter::Parenthesis => quote!(crate::Delimiter::Parenthesis), + Delimiter::Brace => quote!(crate::Delimiter::Brace), + Delimiter::Bracket => quote!(crate::Delimiter::Bracket), + Delimiter::None => quote!(crate::Delimiter::None), + }), + (@ quote(tt.stream())), + ))), + TokenTree::Ident(tt) => quote!(crate::TokenTree::Ident(crate::Ident::new( + (@ TokenTree::from(Literal::string(&tt.to_string()))), + (@ quote_span(proc_macro_crate.clone(), tt.span())), + ))), + TokenTree::Literal(tt) => quote!(crate::TokenTree::Literal({ + let mut iter = (@ TokenTree::from(Literal::string(&tt.to_string()))) + .parse::() + .unwrap() + .into_iter(); + if let (Some(crate::TokenTree::Literal(mut lit)), None) = + (iter.next(), iter.next()) + { + lit.set_span((@ quote_span(proc_macro_crate.clone(), tt.span()))); + lit + } else { + unreachable!() + } + })) + })),)) + }) + .collect::(); + + if after_dollar { + panic!("unexpected trailing `$` in `quote!`"); + } + + quote!([(@ tokens)].iter().cloned().collect::()) +} + +/// Quote a `Span` into a `TokenStream`. +/// This is needed to implement a custom quoter. +#[unstable(feature = "proc_macro_quote", issue = "54722")] +pub fn quote_span(proc_macro_crate: TokenStream, span: Span) -> TokenStream { + let id = span.save_span(); + quote!((@ proc_macro_crate ) ::Span::recover_proc_macro_span((@ TokenTree::from(Literal::usize_unsuffixed(id))))) +} diff --git a/crux-mir/lib/rustc-demangle/Cargo.toml b/crux-mir/lib/rustc-demangle/Cargo.toml deleted file mode 100644 index aeab73dfa..000000000 --- a/crux-mir/lib/rustc-demangle/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "rustc-demangle" -version = "0.1.4" -authors = ["Alex Crichton "] -license = "MIT/Apache-2.0" -readme = "README.md" -repository = "https://github.com/alexcrichton/rustc-demangle" -homepage = "https://github.com/alexcrichton/rustc-demangle" -documentation = "http://alexcrichton.com/rustc-demangle" -description = """ -Rust compiler symbol demangling. -""" diff --git a/crux-mir/lib/rustc-demangle/README.md b/crux-mir/lib/rustc-demangle/README.md deleted file mode 100644 index d2cb46726..000000000 --- a/crux-mir/lib/rustc-demangle/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# rustc-demangle - -Symbol demangling for Rust - -[![Build Status](https://travis-ci.org/alexcrichton/rustc-demangle.svg?branch=master)](https://travis-ci.org/alexcrichton/rustc-demangle) - -[Documentation](http://alexcrichton.com/rustc-demangle) - -# License - -`rustc-demangle` is primarily distributed under the terms of both the MIT license and -the Apache License (Version 2.0), with portions covered by various BSD-like -licenses. - -See LICENSE-APACHE, and LICENSE-MIT for details. diff --git a/crux-mir/lib/rustc-demangle/src/lib.rs b/crux-mir/lib/rustc-demangle/src/lib.rs deleted file mode 100644 index b46246afb..000000000 --- a/crux-mir/lib/rustc-demangle/src/lib.rs +++ /dev/null @@ -1,350 +0,0 @@ -//! Demangle Rust compiler symbol names. -//! -//! This crate provides a `demangle` function which will return a `Demangle` -//! sentinel value that can be used to learn about the demangled version of a -//! symbol name. The demangled representation will be the same as the original -//! if it doesn't look like a mangled symbol name. -//! -//! `Demangle` can be formatted with the `Display` trait. The alternate -//! modifier (`#`) can be used to format the symbol name without the -//! trailing hash value. -//! -//! # Examples -//! -//! ``` -//! use rustc_demangle::demangle; -//! -//! assert_eq!(demangle("_ZN4testE").to_string(), "test"); -//! assert_eq!(demangle("_ZN3foo3barE").to_string(), "foo::bar"); -//! assert_eq!(demangle("foo").to_string(), "foo"); -//! // With hash -//! assert_eq!(format!("{}", demangle("_ZN3foo17h05af221e174051e9E")), "foo::h05af221e174051e9"); -//! // Without hash -//! assert_eq!(format!("{:#}", demangle("_ZN3foo17h05af221e174051e9E")), "foo"); -//! ``` - -#![no_std] -#![deny(missing_docs)] - -#[cfg(test)] -#[macro_use] -extern crate std; - -use core::fmt; - -/// Representation of a demangled symbol name. -pub struct Demangle<'a> { - original: &'a str, - inner: &'a str, - valid: bool, - /// The number of ::-separated elements in the original name. - elements: usize, -} - -/// De-mangles a Rust symbol into a more readable version -/// -/// All rust symbols by default are mangled as they contain characters that -/// cannot be represented in all object files. The mangling mechanism is similar -/// to C++'s, but Rust has a few specifics to handle items like lifetimes in -/// symbols. -/// -/// This function will take a **mangled** symbol (typically acquired from a -/// `Symbol` which is in turn resolved from a `Frame`) and then writes the -/// de-mangled version into the given `writer`. If the symbol does not look like -/// a mangled symbol, it is still written to `writer`. -/// -/// # Examples -/// -/// ``` -/// use rustc_demangle::demangle; -/// -/// assert_eq!(demangle("_ZN4testE").to_string(), "test"); -/// assert_eq!(demangle("_ZN3foo3barE").to_string(), "foo::bar"); -/// assert_eq!(demangle("foo").to_string(), "foo"); -/// ``` - -// All rust symbols are in theory lists of "::"-separated identifiers. Some -// assemblers, however, can't handle these characters in symbol names. To get -// around this, we use C++-style mangling. The mangling method is: -// -// 1. Prefix the symbol with "_ZN" -// 2. For each element of the path, emit the length plus the element -// 3. End the path with "E" -// -// For example, "_ZN4testE" => "test" and "_ZN3foo3barE" => "foo::bar". -// -// We're the ones printing our backtraces, so we can't rely on anything else to -// demangle our symbols. It's *much* nicer to look at demangled symbols, so -// this function is implemented to give us nice pretty output. -// -// Note that this demangler isn't quite as fancy as it could be. We have lots -// of other information in our symbols like hashes, version, type information, -// etc. Additionally, this doesn't handle glue symbols at all. -pub fn demangle(s: &str) -> Demangle { - // First validate the symbol. If it doesn't look like anything we're - // expecting, we just print it literally. Note that we must handle non-rust - // symbols because we could have any function in the backtrace. - let mut valid = true; - let mut inner = s; - if s.len() > 4 && s.starts_with("_ZN") && s.ends_with('E') { - inner = &s[3..s.len() - 1]; - } else if s.len() > 3 && s.starts_with("ZN") && s.ends_with('E') { - // On Windows, dbghelp strips leading underscores, so we accept "ZN...E" - // form too. - inner = &s[2..s.len() - 1]; - } else { - valid = false; - } - - let mut elements = 0; - if valid { - let mut chars = inner.chars(); - while valid { - let mut i = 0; - for c in chars.by_ref() { - if c.is_digit(10) { - i = i * 10 + c as usize - '0' as usize; - } else { - break; - } - } - if i == 0 { - valid = chars.next().is_none(); - break; - } else if chars.by_ref().take(i - 1).count() != i - 1 { - valid = false; - } else { - elements += 1; - } - } - } - - Demangle { - inner: inner, - valid: valid, - elements: elements, - original: s, - } -} - -/// Error returned from the `try_demangle` function below when demangling fails. -#[derive(Debug, Clone)] -pub struct TryDemangleError { - _priv: (), -} - -/// The same as `demangle`, except return an `Err` if the string does not appear -/// to be a Rust symbol, rather than "demangling" the given string as a no-op. -/// -/// ``` -/// extern crate rustc_demangle; -/// -/// let not_a_rust_symbol = "la la la"; -/// -/// // The `try_demangle` function will reject strings which are not Rust symbols. -/// assert!(rustc_demangle::try_demangle(not_a_rust_symbol).is_err()); -/// -/// // While `demangle` will just pass the non-symbol through as a no-op. -/// assert_eq!(rustc_demangle::demangle(not_a_rust_symbol).as_str(), not_a_rust_symbol); -/// ``` -pub fn try_demangle(s: &str) -> Result { - let sym = demangle(s); - if sym.valid { - Ok(sym) - } else { - Err(TryDemangleError { _priv: () }) - } -} - -impl<'a> Demangle<'a> { - /// Returns the underlying string that's being demangled. - pub fn as_str(&self) -> &'a str { - self.original - } -} - -// Rust hashes are hex digits with an `h` prepended. -fn is_rust_hash(s: &str) -> bool { - s.starts_with('h') && s[1..].chars().all(|c| c.is_digit(16)) -} - -impl<'a> fmt::Display for Demangle<'a> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - // Alright, let's do this. - if !self.valid { - return f.write_str(self.inner); - } - - let mut inner = self.inner; - for element in 0..self.elements { - let mut rest = inner; - while rest.chars().next().unwrap().is_digit(10) { - rest = &rest[1..]; - } - let i: usize = inner[..(inner.len() - rest.len())].parse().unwrap(); - inner = &rest[i..]; - rest = &rest[..i]; - // Skip printing the hash if alternate formatting - // was requested. - if f.alternate() && element+1 == self.elements && is_rust_hash(&rest) { - break; - } - if element != 0 { - try!(f.write_str("::")); - } - if rest.starts_with("_$") { - rest = &rest[1..]; - } - while !rest.is_empty() { - if rest.starts_with('.') { - if let Some('.') = rest[1..].chars().next() { - try!(f.write_str("::")); - rest = &rest[2..]; - } else { - try!(f.write_str(".")); - rest = &rest[1..]; - } - } else if rest.starts_with('$') { - macro_rules! demangle { - ($($pat:expr => $demangled:expr),*) => ({ - $(if rest.starts_with($pat) { - try!(f.write_str($demangled)); - rest = &rest[$pat.len()..]; - } else)* - { - try!(f.write_str(rest)); - break; - } - - }) - } - - // see src/librustc/back/link.rs for these mappings - demangle! { - "$SP$" => "@", - "$BP$" => "*", - "$RF$" => "&", - "$LT$" => "<", - "$GT$" => ">", - "$LP$" => "(", - "$RP$" => ")", - "$C$" => ",", - - // in theory we can demangle any Unicode code point, but - // for simplicity we just catch the common ones. - "$u7e$" => "~", - "$u20$" => " ", - "$u27$" => "'", - "$u5b$" => "[", - "$u5d$" => "]", - "$u7b$" => "{", - "$u7d$" => "}", - "$u3b$" => ";", - "$u2b$" => "+", - "$u22$" => "\"" - } - } else { - let idx = match rest.char_indices().find(|&(_, c)| c == '$' || c == '.') { - None => rest.len(), - Some((i, _)) => i, - }; - try!(f.write_str(&rest[..idx])); - rest = &rest[idx..]; - } - } - } - - Ok(()) - } -} - -impl<'a> fmt::Debug for Demangle<'a> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Display::fmt(self, f) - } -} - -#[cfg(test)] -mod tests { - use std::prelude::v1::*; - - macro_rules! t { - ($a:expr, $b:expr) => ({ - assert_eq!(super::demangle($a).to_string(), $b); - }) - } - - macro_rules! t_nohash { - ($a:expr, $b:expr) => ({ - assert_eq!(format!("{:#}", super::demangle($a)), $b); - }) - } - - #[test] - fn demangle() { - t!("test", "test"); - t!("_ZN4testE", "test"); - t!("_ZN4test", "_ZN4test"); - t!("_ZN4test1a2bcE", "test::a::bc"); - } - - #[test] - fn demangle_dollars() { - t!("_ZN4$RP$E", ")"); - t!("_ZN8$RF$testE", "&test"); - t!("_ZN8$BP$test4foobE", "*test::foob"); - t!("_ZN9$u20$test4foobE", " test::foob"); - t!("_ZN35Bar$LT$$u5b$u32$u3b$$u20$4$u5d$$GT$E", "Bar<[u32; 4]>"); - } - - #[test] - fn demangle_many_dollars() { - t!("_ZN13test$u20$test4foobE", "test test::foob"); - t!("_ZN12test$BP$test4foobE", "test*test::foob"); - } - - #[test] - fn demangle_windows() { - t!("ZN4testE", "test"); - t!("ZN13test$u20$test4foobE", "test test::foob"); - t!("ZN12test$RF$test4foobE", "test&test::foob"); - } - - #[test] - fn demangle_elements_beginning_with_underscore() { - t!("_ZN13_$LT$test$GT$E", ""); - t!("_ZN28_$u7b$$u7b$closure$u7d$$u7d$E", "{{closure}}"); - t!("_ZN15__STATIC_FMTSTRE", "__STATIC_FMTSTR"); - } - - #[test] - fn demangle_trait_impls() { - t!("_ZN71_$LT$Test$u20$$u2b$$u20$$u27$static$u20$as$u20$foo..Bar$LT$Test$GT$$GT$3barE", - ">::bar"); - } - - #[test] - fn demangle_without_hash() { - let s = "_ZN3foo17h05af221e174051e9E"; - t!(s, "foo::h05af221e174051e9"); - t_nohash!(s, "foo"); - } - - #[test] - fn demangle_without_hash_edgecases() { - // One element, no hash. - t_nohash!("_ZN3fooE", "foo"); - // Two elements, no hash. - t_nohash!("_ZN3foo3barE", "foo::bar"); - // Longer-than-normal hash. - t_nohash!("_ZN3foo20h05af221e174051e9abcE", "foo"); - // Shorter-than-normal hash. - t_nohash!("_ZN3foo5h05afE", "foo"); - // Valid hash, but not at the end. - t_nohash!("_ZN17h05af221e174051e93fooE", "h05af221e174051e9::foo"); - // Not a valid hash, missing the 'h'. - t_nohash!("_ZN3foo16ffaf221e174051e9E", "foo::ffaf221e174051e9"); - // Not a valid hash, has a non-hex-digit. - t_nohash!("_ZN3foo17hg5af221e174051e9E", "foo::hg5af221e174051e9"); - } -} diff --git a/crux-mir/lib/rustc_demangle/.cargo-ok b/crux-mir/lib/rustc_demangle/.cargo-ok new file mode 100644 index 000000000..b5754e203 --- /dev/null +++ b/crux-mir/lib/rustc_demangle/.cargo-ok @@ -0,0 +1 @@ +ok \ No newline at end of file diff --git a/crux-mir/lib/rustc_demangle/.cargo_vcs_info.json b/crux-mir/lib/rustc_demangle/.cargo_vcs_info.json new file mode 100644 index 000000000..8dc2ce5c1 --- /dev/null +++ b/crux-mir/lib/rustc_demangle/.cargo_vcs_info.json @@ -0,0 +1,5 @@ +{ + "git": { + "sha1": "2811a1ad6f7c8bead2ef3671e4fdc10de1553e96" + } +} diff --git a/crux-mir/lib/rustc_demangle/.github/dependabot.yml b/crux-mir/lib/rustc_demangle/.github/dependabot.yml new file mode 100644 index 000000000..7377d3759 --- /dev/null +++ b/crux-mir/lib/rustc_demangle/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: +- package-ecosystem: cargo + directory: "/" + schedule: + interval: daily + time: "08:00" + open-pull-requests-limit: 10 diff --git a/crux-mir/lib/rustc_demangle/.github/workflows/main.yml b/crux-mir/lib/rustc_demangle/.github/workflows/main.yml new file mode 100644 index 000000000..6ae8d0cf0 --- /dev/null +++ b/crux-mir/lib/rustc_demangle/.github/workflows/main.yml @@ -0,0 +1,54 @@ +name: CI +on: [push, pull_request] + +jobs: + test: + name: Test + runs-on: ubuntu-latest + strategy: + matrix: + rust: [stable, beta, nightly] + steps: + - uses: actions/checkout@v2 + - name: Install Rust + run: rustup update ${{ matrix.rust }} && rustup default ${{ matrix.rust }} + - run: cargo build --all + - run: cargo test --all + + fuzz_targets: + name: Fuzz Targets + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + # Note that building with fuzzers requires nightly since it uses unstable + # flags to rustc. + - run: rustup update nightly && rustup default nightly + - run: cargo install cargo-fuzz --vers "^0.10" + - run: cargo fuzz build --dev + + rustfmt: + name: Rustfmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install Rust + run: rustup update stable && rustup default stable && rustup component add rustfmt + - run: cargo fmt -- --check + + publish_docs: + name: Publish Documentation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install Rust + run: rustup update stable && rustup default stable + - name: Build documentation + run: cargo doc --no-deps + - name: Publish documentation + run: | + cd target/doc + git init + git add . + git -c user.name='ci' -c user.email='ci' commit -m init + git push -f -q https://git:${{ secrets.github_token }}@github.com/${{ github.repository }} HEAD:gh-pages + if: github.event_name == 'push' && github.event.ref == 'refs/heads/main' diff --git a/crux-mir/lib/rustc_demangle/.gitignore b/crux-mir/lib/rustc_demangle/.gitignore new file mode 100644 index 000000000..a9d37c560 --- /dev/null +++ b/crux-mir/lib/rustc_demangle/.gitignore @@ -0,0 +1,2 @@ +target +Cargo.lock diff --git a/crux-mir/lib/rustc_demangle/Cargo.toml b/crux-mir/lib/rustc_demangle/Cargo.toml new file mode 100644 index 000000000..2253b63b1 --- /dev/null +++ b/crux-mir/lib/rustc_demangle/Cargo.toml @@ -0,0 +1,35 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies +# +# If you believe there's an error in this file please file an +# issue against the rust-lang/cargo repository. If you're +# editing this file be aware that the upstream Cargo.toml +# will likely look very different (and much more reasonable) + +[package] +name = "rustc-demangle" +version = "0.1.21" +authors = ["Alex Crichton "] +description = "Rust compiler symbol demangling.\n" +homepage = "https://github.com/alexcrichton/rustc-demangle" +documentation = "https://docs.rs/rustc-demangle" +readme = "README.md" +license = "MIT/Apache-2.0" +repository = "https://github.com/alexcrichton/rustc-demangle" +[profile.release] +lto = true +[dependencies.compiler_builtins] +version = "0.1.2" +optional = true + +[dependencies.core] +version = "1.0.0" +optional = true +package = "rustc-std-workspace-core" + +[features] +rustc-dep-of-std = ["core", "compiler_builtins"] diff --git a/crux-mir/lib/rustc_demangle/Cargo.toml.orig b/crux-mir/lib/rustc_demangle/Cargo.toml.orig new file mode 100644 index 000000000..552e0698e --- /dev/null +++ b/crux-mir/lib/rustc_demangle/Cargo.toml.orig @@ -0,0 +1,25 @@ +[package] +name = "rustc-demangle" +version = "0.1.21" +authors = ["Alex Crichton "] +license = "MIT/Apache-2.0" +readme = "README.md" +repository = "https://github.com/alexcrichton/rustc-demangle" +homepage = "https://github.com/alexcrichton/rustc-demangle" +documentation = "https://docs.rs/rustc-demangle" +description = """ +Rust compiler symbol demangling. +""" + +[workspace] +members = ["crates/capi", "fuzz"] + +[dependencies] +core = { version = '1.0.0', optional = true, package = 'rustc-std-workspace-core' } +compiler_builtins = { version = '0.1.2', optional = true } + +[features] +rustc-dep-of-std = ['core', 'compiler_builtins'] + +[profile.release] +lto = true diff --git a/crux-mir/lib/unicode-width/LICENSE-APACHE b/crux-mir/lib/rustc_demangle/LICENSE-APACHE similarity index 100% rename from crux-mir/lib/unicode-width/LICENSE-APACHE rename to crux-mir/lib/rustc_demangle/LICENSE-APACHE diff --git a/crux-mir/lib/cfg-if/LICENSE-MIT b/crux-mir/lib/rustc_demangle/LICENSE-MIT similarity index 100% rename from crux-mir/lib/cfg-if/LICENSE-MIT rename to crux-mir/lib/rustc_demangle/LICENSE-MIT diff --git a/crux-mir/lib/rustc_demangle/README.md b/crux-mir/lib/rustc_demangle/README.md new file mode 100644 index 000000000..0833e1e24 --- /dev/null +++ b/crux-mir/lib/rustc_demangle/README.md @@ -0,0 +1,48 @@ +# rustc-demangle + +Demangling for Rust symbols, written in Rust. + +[Documentation](https://docs.rs/rustc-demangle) + +## Usage + +You can add this as a dependency via your `Cargo.toml` + +```toml +[dependencies] +rustc-demangle = "0.1" +``` + +and then be sure to check out the [crate +documentation](https://docs.rs/rustc-demangle) for usage. + +## Usage from non-Rust languages + +You can also use this crate from other languages via the C API wrapper in the +`crates/capi` directory. This can be build with: + +```sh +$ cargo build -p rustc-demangle-capi --release +``` + +You'll then find `target/release/librustc_demangle.a` and +`target/release/librustc_demangle.so` (or a different name depending on your +platform). These objects implement the interface specified in +`crates/capi/include/rustc_demangle.h`. + +# License + +This project is licensed under either of + + * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or + http://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([LICENSE-MIT](LICENSE-MIT) or + http://opensource.org/licenses/MIT) + +at your option. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in rustc-demangle you, as defined in the Apache-2.0 license, shall +be dual licensed as above, without any additional terms or conditions. diff --git a/crux-mir/lib/rustc_demangle/src/legacy.rs b/crux-mir/lib/rustc_demangle/src/legacy.rs new file mode 100644 index 000000000..d55f3a1fd --- /dev/null +++ b/crux-mir/lib/rustc_demangle/src/legacy.rs @@ -0,0 +1,392 @@ +use core::char; +use core::fmt; + +/// Representation of a demangled symbol name. +pub struct Demangle<'a> { + inner: &'a str, + /// The number of ::-separated elements in the original name. + elements: usize, +} + +/// De-mangles a Rust symbol into a more readable version +/// +/// All Rust symbols by default are mangled as they contain characters that +/// cannot be represented in all object files. The mangling mechanism is similar +/// to C++'s, but Rust has a few specifics to handle items like lifetimes in +/// symbols. +/// +/// This function will take a **mangled** symbol and return a value. When printed, +/// the de-mangled version will be written. If the symbol does not look like +/// a mangled symbol, the original value will be written instead. +/// +/// # Examples +/// +/// ``` +/// use rustc_demangle::demangle; +/// +/// assert_eq!(demangle("_ZN4testE").to_string(), "test"); +/// assert_eq!(demangle("_ZN3foo3barE").to_string(), "foo::bar"); +/// assert_eq!(demangle("foo").to_string(), "foo"); +/// ``` + +// All Rust symbols are in theory lists of "::"-separated identifiers. Some +// assemblers, however, can't handle these characters in symbol names. To get +// around this, we use C++-style mangling. The mangling method is: +// +// 1. Prefix the symbol with "_ZN" +// 2. For each element of the path, emit the length plus the element +// 3. End the path with "E" +// +// For example, "_ZN4testE" => "test" and "_ZN3foo3barE" => "foo::bar". +// +// We're the ones printing our backtraces, so we can't rely on anything else to +// demangle our symbols. It's *much* nicer to look at demangled symbols, so +// this function is implemented to give us nice pretty output. +// +// Note that this demangler isn't quite as fancy as it could be. We have lots +// of other information in our symbols like hashes, version, type information, +// etc. Additionally, this doesn't handle glue symbols at all. +pub fn demangle(s: &str) -> Result<(Demangle, &str), ()> { + // First validate the symbol. If it doesn't look like anything we're + // expecting, we just print it literally. Note that we must handle non-Rust + // symbols because we could have any function in the backtrace. + let inner = if s.starts_with("_ZN") { + &s[3..] + } else if s.starts_with("ZN") { + // On Windows, dbghelp strips leading underscores, so we accept "ZN...E" + // form too. + &s[2..] + } else if s.starts_with("__ZN") { + // On OSX, symbols are prefixed with an extra _ + &s[4..] + } else { + return Err(()); + }; + + // only work with ascii text + if inner.bytes().any(|c| c & 0x80 != 0) { + return Err(()); + } + + let mut elements = 0; + let mut chars = inner.chars(); + let mut c = chars.next().ok_or(())?; + while c != 'E' { + // Decode an identifier element's length. + if !c.is_digit(10) { + return Err(()); + } + let mut len = 0usize; + while let Some(d) = c.to_digit(10) { + len = len + .checked_mul(10) + .and_then(|len| len.checked_add(d as usize)) + .ok_or(())?; + c = chars.next().ok_or(())?; + } + + // `c` already contains the first character of this identifier, skip it and + // all the other characters of this identifier, to reach the next element. + for _ in 0..len { + c = chars.next().ok_or(())?; + } + + elements += 1; + } + + Ok((Demangle { inner, elements }, chars.as_str())) +} + +// Rust hashes are hex digits with an `h` prepended. +fn is_rust_hash(s: &str) -> bool { + s.starts_with('h') && s[1..].chars().all(|c| c.is_digit(16)) +} + +impl<'a> fmt::Display for Demangle<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Alright, let's do this. + let mut inner = self.inner; + for element in 0..self.elements { + let mut rest = inner; + while rest.chars().next().unwrap().is_digit(10) { + rest = &rest[1..]; + } + let i: usize = inner[..(inner.len() - rest.len())].parse().unwrap(); + inner = &rest[i..]; + rest = &rest[..i]; + // Skip printing the hash if alternate formatting + // was requested. + if f.alternate() && element + 1 == self.elements && is_rust_hash(&rest) { + break; + } + if element != 0 { + f.write_str("::")?; + } + if rest.starts_with("_$") { + rest = &rest[1..]; + } + loop { + if rest.starts_with('.') { + if let Some('.') = rest[1..].chars().next() { + f.write_str("::")?; + rest = &rest[2..]; + } else { + f.write_str(".")?; + rest = &rest[1..]; + } + } else if rest.starts_with('$') { + let (escape, after_escape) = if let Some(end) = rest[1..].find('$') { + (&rest[1..=end], &rest[end + 2..]) + } else { + break; + }; + + // see src/librustc_codegen_utils/symbol_names/legacy.rs for these mappings + let unescaped = match escape { + "SP" => "@", + "BP" => "*", + "RF" => "&", + "LT" => "<", + "GT" => ">", + "LP" => "(", + "RP" => ")", + "C" => ",", + + _ => { + if escape.starts_with('u') { + let digits = &escape[1..]; + let all_lower_hex = digits.chars().all(|c| match c { + '0'..='9' | 'a'..='f' => true, + _ => false, + }); + let c = u32::from_str_radix(digits, 16) + .ok() + .and_then(char::from_u32); + if let (true, Some(c)) = (all_lower_hex, c) { + // FIXME(eddyb) do we need to filter out control codepoints? + if !c.is_control() { + c.fmt(f)?; + rest = after_escape; + continue; + } + } + } + break; + } + }; + f.write_str(unescaped)?; + rest = after_escape; + } else if let Some(i) = rest.find(|c| c == '$' || c == '.') { + f.write_str(&rest[..i])?; + rest = &rest[i..]; + } else { + break; + } + } + f.write_str(rest)?; + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::prelude::v1::*; + + macro_rules! t { + ($a:expr, $b:expr) => { + assert!(ok($a, $b)) + }; + } + + macro_rules! t_err { + ($a:expr) => { + assert!(ok_err($a)) + }; + } + + macro_rules! t_nohash { + ($a:expr, $b:expr) => {{ + assert_eq!(format!("{:#}", ::demangle($a)), $b); + }}; + } + + fn ok(sym: &str, expected: &str) -> bool { + match ::try_demangle(sym) { + Ok(s) => { + if s.to_string() == expected { + true + } else { + println!("\n{}\n!=\n{}\n", s, expected); + false + } + } + Err(_) => { + println!("error demangling"); + false + } + } + } + + fn ok_err(sym: &str) -> bool { + match ::try_demangle(sym) { + Ok(_) => { + println!("succeeded in demangling"); + false + } + Err(_) => ::demangle(sym).to_string() == sym, + } + } + + #[test] + fn demangle() { + t_err!("test"); + t!("_ZN4testE", "test"); + t_err!("_ZN4test"); + t!("_ZN4test1a2bcE", "test::a::bc"); + } + + #[test] + fn demangle_dollars() { + t!("_ZN4$RP$E", ")"); + t!("_ZN8$RF$testE", "&test"); + t!("_ZN8$BP$test4foobE", "*test::foob"); + t!("_ZN9$u20$test4foobE", " test::foob"); + t!("_ZN35Bar$LT$$u5b$u32$u3b$$u20$4$u5d$$GT$E", "Bar<[u32; 4]>"); + } + + #[test] + fn demangle_many_dollars() { + t!("_ZN13test$u20$test4foobE", "test test::foob"); + t!("_ZN12test$BP$test4foobE", "test*test::foob"); + } + + #[test] + fn demangle_osx() { + t!( + "__ZN5alloc9allocator6Layout9for_value17h02a996811f781011E", + "alloc::allocator::Layout::for_value::h02a996811f781011" + ); + t!("__ZN38_$LT$core..option..Option$LT$T$GT$$GT$6unwrap18_MSG_FILE_LINE_COL17haf7cb8d5824ee659E", ">::unwrap::_MSG_FILE_LINE_COL::haf7cb8d5824ee659"); + t!("__ZN4core5slice89_$LT$impl$u20$core..iter..traits..IntoIterator$u20$for$u20$$RF$$u27$a$u20$$u5b$T$u5d$$GT$9into_iter17h450e234d27262170E", "core::slice::::into_iter::h450e234d27262170"); + } + + #[test] + fn demangle_windows() { + t!("ZN4testE", "test"); + t!("ZN13test$u20$test4foobE", "test test::foob"); + t!("ZN12test$RF$test4foobE", "test&test::foob"); + } + + #[test] + fn demangle_elements_beginning_with_underscore() { + t!("_ZN13_$LT$test$GT$E", ""); + t!("_ZN28_$u7b$$u7b$closure$u7d$$u7d$E", "{{closure}}"); + t!("_ZN15__STATIC_FMTSTRE", "__STATIC_FMTSTR"); + } + + #[test] + fn demangle_trait_impls() { + t!( + "_ZN71_$LT$Test$u20$$u2b$$u20$$u27$static$u20$as$u20$foo..Bar$LT$Test$GT$$GT$3barE", + ">::bar" + ); + } + + #[test] + fn demangle_without_hash() { + let s = "_ZN3foo17h05af221e174051e9E"; + t!(s, "foo::h05af221e174051e9"); + t_nohash!(s, "foo"); + } + + #[test] + fn demangle_without_hash_edgecases() { + // One element, no hash. + t_nohash!("_ZN3fooE", "foo"); + // Two elements, no hash. + t_nohash!("_ZN3foo3barE", "foo::bar"); + // Longer-than-normal hash. + t_nohash!("_ZN3foo20h05af221e174051e9abcE", "foo"); + // Shorter-than-normal hash. + t_nohash!("_ZN3foo5h05afE", "foo"); + // Valid hash, but not at the end. + t_nohash!("_ZN17h05af221e174051e93fooE", "h05af221e174051e9::foo"); + // Not a valid hash, missing the 'h'. + t_nohash!("_ZN3foo16ffaf221e174051e9E", "foo::ffaf221e174051e9"); + // Not a valid hash, has a non-hex-digit. + t_nohash!("_ZN3foo17hg5af221e174051e9E", "foo::hg5af221e174051e9"); + } + + #[test] + fn demangle_thinlto() { + // One element, no hash. + t!("_ZN3fooE.llvm.9D1C9369", "foo"); + t!("_ZN3fooE.llvm.9D1C9369@@16", "foo"); + t_nohash!( + "_ZN9backtrace3foo17hbb467fcdaea5d79bE.llvm.A5310EB9", + "backtrace::foo" + ); + } + + #[test] + fn demangle_llvm_ir_branch_labels() { + t!("_ZN4core5slice77_$LT$impl$u20$core..ops..index..IndexMut$LT$I$GT$$u20$for$u20$$u5b$T$u5d$$GT$9index_mut17haf9727c2edfbc47bE.exit.i.i", "core::slice:: for [T]>::index_mut::haf9727c2edfbc47b.exit.i.i"); + t_nohash!("_ZN4core5slice77_$LT$impl$u20$core..ops..index..IndexMut$LT$I$GT$$u20$for$u20$$u5b$T$u5d$$GT$9index_mut17haf9727c2edfbc47bE.exit.i.i", "core::slice:: for [T]>::index_mut.exit.i.i"); + } + + #[test] + fn demangle_ignores_suffix_that_doesnt_look_like_a_symbol() { + t_err!("_ZN3fooE.llvm moocow"); + } + + #[test] + fn dont_panic() { + ::demangle("_ZN2222222222222222222222EE").to_string(); + ::demangle("_ZN5*70527e27.ll34csaғE").to_string(); + ::demangle("_ZN5*70527a54.ll34_$b.1E").to_string(); + ::demangle( + "\ + _ZN5~saäb4e\n\ + 2734cOsbE\n\ + 5usage20h)3\0\0\0\0\0\0\07e2734cOsbE\ + ", + ) + .to_string(); + } + + #[test] + fn invalid_no_chop() { + t_err!("_ZNfooE"); + } + + #[test] + fn handle_assoc_types() { + t!("_ZN151_$LT$alloc..boxed..Box$LT$alloc..boxed..FnBox$LT$A$C$$u20$Output$u3d$R$GT$$u20$$u2b$$u20$$u27$a$GT$$u20$as$u20$core..ops..function..FnOnce$LT$A$GT$$GT$9call_once17h69e8f44b3723e1caE", " + 'a> as core::ops::function::FnOnce>::call_once::h69e8f44b3723e1ca"); + } + + #[test] + fn handle_bang() { + t!( + "_ZN88_$LT$core..result..Result$LT$$u21$$C$$u20$E$GT$$u20$as$u20$std..process..Termination$GT$6report17hfc41d0da4a40b3e8E", + " as std::process::Termination>::report::hfc41d0da4a40b3e8" + ); + } + + #[test] + fn demangle_utf8_idents() { + t_nohash!( + "_ZN11utf8_idents157_$u10e1$$u10d0$$u10ed$$u10db$$u10d4$$u10da$$u10d0$$u10d3$_$u10d2$$u10d4$$u10db$$u10e0$$u10d8$$u10d4$$u10da$$u10d8$_$u10e1$$u10d0$$u10d3$$u10d8$$u10da$$u10d8$17h21634fd5714000aaE", + "utf8_idents::საჭმელად_გემრიელი_სადილი" + ); + } + + #[test] + fn demangle_issue_60925() { + t_nohash!( + "_ZN11issue_609253foo37Foo$LT$issue_60925..llv$u6d$..Foo$GT$3foo17h059a991a004536adE", + "issue_60925::foo::Foo::foo" + ); + } +} diff --git a/crux-mir/lib/rustc_demangle/src/lib.rs b/crux-mir/lib/rustc_demangle/src/lib.rs new file mode 100644 index 000000000..1ecb13fee --- /dev/null +++ b/crux-mir/lib/rustc_demangle/src/lib.rs @@ -0,0 +1,493 @@ +//! Demangle Rust compiler symbol names. +//! +//! This crate provides a `demangle` function which will return a `Demangle` +//! sentinel value that can be used to learn about the demangled version of a +//! symbol name. The demangled representation will be the same as the original +//! if it doesn't look like a mangled symbol name. +//! +//! `Demangle` can be formatted with the `Display` trait. The alternate +//! modifier (`#`) can be used to format the symbol name without the +//! trailing hash value. +//! +//! # Examples +//! +//! ``` +//! use rustc_demangle::demangle; +//! +//! assert_eq!(demangle("_ZN4testE").to_string(), "test"); +//! assert_eq!(demangle("_ZN3foo3barE").to_string(), "foo::bar"); +//! assert_eq!(demangle("foo").to_string(), "foo"); +//! // With hash +//! assert_eq!(format!("{}", demangle("_ZN3foo17h05af221e174051e9E")), "foo::h05af221e174051e9"); +//! // Without hash +//! assert_eq!(format!("{:#}", demangle("_ZN3foo17h05af221e174051e9E")), "foo"); +//! ``` + +#![no_std] +#![deny(missing_docs)] + +#[cfg(test)] +#[macro_use] +extern crate std; + +// HACK(eddyb) helper macros for tests. +#[cfg(test)] +macro_rules! assert_contains { + ($s:expr, $needle:expr) => {{ + let (s, needle) = ($s, $needle); + assert!( + s.contains(needle), + "{:?} should've contained {:?}", + s, + needle + ); + }}; +} +#[cfg(test)] +macro_rules! assert_ends_with { + ($s:expr, $suffix:expr) => {{ + let (s, suffix) = ($s, $suffix); + assert!( + s.ends_with(suffix), + "{:?} should've ended in {:?}", + s, + suffix + ); + }}; +} + +mod legacy; +mod v0; + +use core::fmt::{self, Write as _}; + +/// Representation of a demangled symbol name. +pub struct Demangle<'a> { + style: Option>, + original: &'a str, + suffix: &'a str, +} + +enum DemangleStyle<'a> { + Legacy(legacy::Demangle<'a>), + V0(v0::Demangle<'a>), +} + +/// De-mangles a Rust symbol into a more readable version +/// +/// This function will take a **mangled** symbol and return a value. When printed, +/// the de-mangled version will be written. If the symbol does not look like +/// a mangled symbol, the original value will be written instead. +/// +/// # Examples +/// +/// ``` +/// use rustc_demangle::demangle; +/// +/// assert_eq!(demangle("_ZN4testE").to_string(), "test"); +/// assert_eq!(demangle("_ZN3foo3barE").to_string(), "foo::bar"); +/// assert_eq!(demangle("foo").to_string(), "foo"); +/// ``` +pub fn demangle(mut s: &str) -> Demangle { + // During ThinLTO LLVM may import and rename internal symbols, so strip out + // those endings first as they're one of the last manglings applied to symbol + // names. + let llvm = ".llvm."; + if let Some(i) = s.find(llvm) { + let candidate = &s[i + llvm.len()..]; + let all_hex = candidate.chars().all(|c| match c { + 'A'..='F' | '0'..='9' | '@' => true, + _ => false, + }); + + if all_hex { + s = &s[..i]; + } + } + + let mut suffix = ""; + let mut style = match legacy::demangle(s) { + Ok((d, s)) => { + suffix = s; + Some(DemangleStyle::Legacy(d)) + } + Err(()) => match v0::demangle(s) { + Ok((d, s)) => { + suffix = s; + Some(DemangleStyle::V0(d)) + } + // FIXME(eddyb) would it make sense to treat an unknown-validity + // symbol (e.g. one that errored with `RecursedTooDeep`) as + // v0-mangled, and have the error show up in the demangling? + // (that error already gets past this initial check, and therefore + // will show up in the demangling, if hidden behind a backref) + Err(v0::ParseError::Invalid) | Err(v0::ParseError::RecursedTooDeep) => None, + }, + }; + + // Output like LLVM IR adds extra period-delimited words. See if + // we are in that case and save the trailing words if so. + if !suffix.is_empty() { + if suffix.starts_with('.') && is_symbol_like(suffix) { + // Keep the suffix. + } else { + // Reset the suffix and invalidate the demangling. + suffix = ""; + style = None; + } + } + + Demangle { + style, + original: s, + suffix, + } +} + +/// Error returned from the `try_demangle` function below when demangling fails. +#[derive(Debug, Clone)] +pub struct TryDemangleError { + _priv: (), +} + +/// The same as `demangle`, except return an `Err` if the string does not appear +/// to be a Rust symbol, rather than "demangling" the given string as a no-op. +/// +/// ``` +/// extern crate rustc_demangle; +/// +/// let not_a_rust_symbol = "la la la"; +/// +/// // The `try_demangle` function will reject strings which are not Rust symbols. +/// assert!(rustc_demangle::try_demangle(not_a_rust_symbol).is_err()); +/// +/// // While `demangle` will just pass the non-symbol through as a no-op. +/// assert_eq!(rustc_demangle::demangle(not_a_rust_symbol).as_str(), not_a_rust_symbol); +/// ``` +pub fn try_demangle(s: &str) -> Result { + let sym = demangle(s); + if sym.style.is_some() { + Ok(sym) + } else { + Err(TryDemangleError { _priv: () }) + } +} + +impl<'a> Demangle<'a> { + /// Returns the underlying string that's being demangled. + pub fn as_str(&self) -> &'a str { + self.original + } +} + +fn is_symbol_like(s: &str) -> bool { + s.chars().all(|c| { + // Once `char::is_ascii_punctuation` and `char::is_ascii_alphanumeric` + // have been stable for long enough, use those instead for clarity + is_ascii_alphanumeric(c) || is_ascii_punctuation(c) + }) +} + +// Copied from the documentation of `char::is_ascii_alphanumeric` +fn is_ascii_alphanumeric(c: char) -> bool { + match c { + '\u{0041}'..='\u{005A}' | '\u{0061}'..='\u{007A}' | '\u{0030}'..='\u{0039}' => true, + _ => false, + } +} + +// Copied from the documentation of `char::is_ascii_punctuation` +fn is_ascii_punctuation(c: char) -> bool { + match c { + '\u{0021}'..='\u{002F}' + | '\u{003A}'..='\u{0040}' + | '\u{005B}'..='\u{0060}' + | '\u{007B}'..='\u{007E}' => true, + _ => false, + } +} + +impl<'a> fmt::Display for DemangleStyle<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + DemangleStyle::Legacy(ref d) => fmt::Display::fmt(d, f), + DemangleStyle::V0(ref d) => fmt::Display::fmt(d, f), + } + } +} + +// Maximum size of the symbol that we'll print. +const MAX_SIZE: usize = 1_000_000; + +#[derive(Copy, Clone, Debug)] +struct SizeLimitExhausted; + +struct SizeLimitedFmtAdapter { + remaining: Result, + inner: F, +} + +impl fmt::Write for SizeLimitedFmtAdapter { + fn write_str(&mut self, s: &str) -> fmt::Result { + self.remaining = self + .remaining + .and_then(|r| r.checked_sub(s.len()).ok_or(SizeLimitExhausted)); + + match self.remaining { + Ok(_) => self.inner.write_str(s), + Err(SizeLimitExhausted) => Err(fmt::Error), + } + } +} + +impl<'a> fmt::Display for Demangle<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self.style { + None => f.write_str(self.original)?, + Some(ref d) => { + let alternate = f.alternate(); + let mut size_limited_fmt = SizeLimitedFmtAdapter { + remaining: Ok(MAX_SIZE), + inner: &mut *f, + }; + let fmt_result = if alternate { + write!(size_limited_fmt, "{:#}", d) + } else { + write!(size_limited_fmt, "{}", d) + }; + let size_limit_result = size_limited_fmt.remaining.map(|_| ()); + + // Translate a `fmt::Error` generated by `SizeLimitedFmtAdapter` + // into an error message, instead of propagating it upwards + // (which could cause panicking from inside e.g. `std::io::print`). + match (fmt_result, size_limit_result) { + (Err(_), Err(SizeLimitExhausted)) => f.write_str("{size limit reached}")?, + + _ => { + fmt_result?; + size_limit_result + .expect("`fmt::Error` from `SizeLimitedFmtAdapter` was discarded"); + } + } + } + } + f.write_str(self.suffix) + } +} + +impl<'a> fmt::Debug for Demangle<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +#[cfg(test)] +mod tests { + use std::prelude::v1::*; + + macro_rules! t { + ($a:expr, $b:expr) => { + assert!(ok($a, $b)) + }; + } + + macro_rules! t_err { + ($a:expr) => { + assert!(ok_err($a)) + }; + } + + macro_rules! t_nohash { + ($a:expr, $b:expr) => {{ + assert_eq!(format!("{:#}", super::demangle($a)), $b); + }}; + } + + fn ok(sym: &str, expected: &str) -> bool { + match super::try_demangle(sym) { + Ok(s) => { + if s.to_string() == expected { + true + } else { + println!("\n{}\n!=\n{}\n", s, expected); + false + } + } + Err(_) => { + println!("error demangling"); + false + } + } + } + + fn ok_err(sym: &str) -> bool { + match super::try_demangle(sym) { + Ok(_) => { + println!("succeeded in demangling"); + false + } + Err(_) => super::demangle(sym).to_string() == sym, + } + } + + #[test] + fn demangle() { + t_err!("test"); + t!("_ZN4testE", "test"); + t_err!("_ZN4test"); + t!("_ZN4test1a2bcE", "test::a::bc"); + } + + #[test] + fn demangle_dollars() { + t!("_ZN4$RP$E", ")"); + t!("_ZN8$RF$testE", "&test"); + t!("_ZN8$BP$test4foobE", "*test::foob"); + t!("_ZN9$u20$test4foobE", " test::foob"); + t!("_ZN35Bar$LT$$u5b$u32$u3b$$u20$4$u5d$$GT$E", "Bar<[u32; 4]>"); + } + + #[test] + fn demangle_many_dollars() { + t!("_ZN13test$u20$test4foobE", "test test::foob"); + t!("_ZN12test$BP$test4foobE", "test*test::foob"); + } + + #[test] + fn demangle_osx() { + t!( + "__ZN5alloc9allocator6Layout9for_value17h02a996811f781011E", + "alloc::allocator::Layout::for_value::h02a996811f781011" + ); + t!("__ZN38_$LT$core..option..Option$LT$T$GT$$GT$6unwrap18_MSG_FILE_LINE_COL17haf7cb8d5824ee659E", ">::unwrap::_MSG_FILE_LINE_COL::haf7cb8d5824ee659"); + t!("__ZN4core5slice89_$LT$impl$u20$core..iter..traits..IntoIterator$u20$for$u20$$RF$$u27$a$u20$$u5b$T$u5d$$GT$9into_iter17h450e234d27262170E", "core::slice::::into_iter::h450e234d27262170"); + } + + #[test] + fn demangle_windows() { + t!("ZN4testE", "test"); + t!("ZN13test$u20$test4foobE", "test test::foob"); + t!("ZN12test$RF$test4foobE", "test&test::foob"); + } + + #[test] + fn demangle_elements_beginning_with_underscore() { + t!("_ZN13_$LT$test$GT$E", ""); + t!("_ZN28_$u7b$$u7b$closure$u7d$$u7d$E", "{{closure}}"); + t!("_ZN15__STATIC_FMTSTRE", "__STATIC_FMTSTR"); + } + + #[test] + fn demangle_trait_impls() { + t!( + "_ZN71_$LT$Test$u20$$u2b$$u20$$u27$static$u20$as$u20$foo..Bar$LT$Test$GT$$GT$3barE", + ">::bar" + ); + } + + #[test] + fn demangle_without_hash() { + let s = "_ZN3foo17h05af221e174051e9E"; + t!(s, "foo::h05af221e174051e9"); + t_nohash!(s, "foo"); + } + + #[test] + fn demangle_without_hash_edgecases() { + // One element, no hash. + t_nohash!("_ZN3fooE", "foo"); + // Two elements, no hash. + t_nohash!("_ZN3foo3barE", "foo::bar"); + // Longer-than-normal hash. + t_nohash!("_ZN3foo20h05af221e174051e9abcE", "foo"); + // Shorter-than-normal hash. + t_nohash!("_ZN3foo5h05afE", "foo"); + // Valid hash, but not at the end. + t_nohash!("_ZN17h05af221e174051e93fooE", "h05af221e174051e9::foo"); + // Not a valid hash, missing the 'h'. + t_nohash!("_ZN3foo16ffaf221e174051e9E", "foo::ffaf221e174051e9"); + // Not a valid hash, has a non-hex-digit. + t_nohash!("_ZN3foo17hg5af221e174051e9E", "foo::hg5af221e174051e9"); + } + + #[test] + fn demangle_thinlto() { + // One element, no hash. + t!("_ZN3fooE.llvm.9D1C9369", "foo"); + t!("_ZN3fooE.llvm.9D1C9369@@16", "foo"); + t_nohash!( + "_ZN9backtrace3foo17hbb467fcdaea5d79bE.llvm.A5310EB9", + "backtrace::foo" + ); + } + + #[test] + fn demangle_llvm_ir_branch_labels() { + t!("_ZN4core5slice77_$LT$impl$u20$core..ops..index..IndexMut$LT$I$GT$$u20$for$u20$$u5b$T$u5d$$GT$9index_mut17haf9727c2edfbc47bE.exit.i.i", "core::slice:: for [T]>::index_mut::haf9727c2edfbc47b.exit.i.i"); + t_nohash!("_ZN4core5slice77_$LT$impl$u20$core..ops..index..IndexMut$LT$I$GT$$u20$for$u20$$u5b$T$u5d$$GT$9index_mut17haf9727c2edfbc47bE.exit.i.i", "core::slice:: for [T]>::index_mut.exit.i.i"); + } + + #[test] + fn demangle_ignores_suffix_that_doesnt_look_like_a_symbol() { + t_err!("_ZN3fooE.llvm moocow"); + } + + #[test] + fn dont_panic() { + super::demangle("_ZN2222222222222222222222EE").to_string(); + super::demangle("_ZN5*70527e27.ll34csaғE").to_string(); + super::demangle("_ZN5*70527a54.ll34_$b.1E").to_string(); + super::demangle( + "\ + _ZN5~saäb4e\n\ + 2734cOsbE\n\ + 5usage20h)3\0\0\0\0\0\0\07e2734cOsbE\ + ", + ) + .to_string(); + } + + #[test] + fn invalid_no_chop() { + t_err!("_ZNfooE"); + } + + #[test] + fn handle_assoc_types() { + t!("_ZN151_$LT$alloc..boxed..Box$LT$alloc..boxed..FnBox$LT$A$C$$u20$Output$u3d$R$GT$$u20$$u2b$$u20$$u27$a$GT$$u20$as$u20$core..ops..function..FnOnce$LT$A$GT$$GT$9call_once17h69e8f44b3723e1caE", " + 'a> as core::ops::function::FnOnce>::call_once::h69e8f44b3723e1ca"); + } + + #[test] + fn handle_bang() { + t!( + "_ZN88_$LT$core..result..Result$LT$$u21$$C$$u20$E$GT$$u20$as$u20$std..process..Termination$GT$6report17hfc41d0da4a40b3e8E", + " as std::process::Termination>::report::hfc41d0da4a40b3e8" + ); + } + + #[test] + fn limit_recursion() { + assert_contains!( + super::demangle("_RNvB_1a").to_string(), + "{recursion limit reached}" + ); + assert_contains!( + super::demangle("_RMC0RB2_").to_string(), + "{recursion limit reached}" + ); + } + + #[test] + fn limit_output() { + assert_ends_with!( + super::demangle("RYFG_FGyyEvRYFF_EvRYFFEvERLB_B_B_ERLRjB_B_B_").to_string(), + "{size limit reached}" + ); + // NOTE(eddyb) somewhat reduced version of the above, effectively + // ` fn()>` with a larger number of lifetimes in `...`. + assert_ends_with!( + super::demangle("_RMC0FGZZZ_Eu").to_string(), + "{size limit reached}" + ); + } +} diff --git a/crux-mir/lib/rustc_demangle/src/v0-large-test-symbols/early-recursion-limit b/crux-mir/lib/rustc_demangle/src/v0-large-test-symbols/early-recursion-limit new file mode 100644 index 000000000..914d416f7 --- /dev/null +++ b/crux-mir/lib/rustc_demangle/src/v0-large-test-symbols/early-recursion-limit @@ -0,0 +1,10 @@ +# NOTE: empty lines, and lines starting with `#`, are ignored. + +# Large test symbols for `v0::test::demangling_limits`, that specifically cause +# a `RecursedTooDeep` error from `v0::demangle`'s shallow traversal (sanity check) +# of the mangled symbol, i.e. before any printing coudl be attempted. + +RICu4$TTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOSOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTxxxxxRICu4,-xOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTxxxxxRICu4,-xOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTxxxxxRICu4,-xOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTxxxxxRICu4,-xOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTxxxxxRICu4,-xOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTxxxxxRICu4,-xOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTxxxxxRICu4,-xOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTxxxxxRICu4,-xOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTYTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTxxxxxRICu4,-xOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTxxxxxRICu4,-xOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTxxxxxRICu5,-xOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTxxxxxRICu3.,-xOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOxxxRICu4,-xxxxffff..ffffffffffffffffffffffffffffffffffffffffffffffffffffffffxxxxxxxxxxxxxxxxxxxRaRBRaR>R>xxxu2IC,-xxxxxxRIC4xxxOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTxxxxxRICu4.,-xOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOTTTOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOxxxRICu4,-xxxxffff..ffffffffffffffffffffffffffffffffffffffffffffffffffffffffxxxxxxxxxxxxxxxxxxxRaRBRaR>R>xxxu2IC,-xxxxxxRIC4xxx..K..xRBRaR>RICu6$-RBKIQARICu6$-RBKIQAA........TvvKKKKKKKKKxxxxxxxxxxxxxxxBKIQARICu6$-RBKIQAA...._.xxx +RIYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRYYYYYYYYYXB_RXB_lYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRXB_RXYYYYYYYYYYYYYYYYYYYYYXYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYRXB_RXYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRXB_RXYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRYYYYYYYYYXB_RXB_lYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRXB_RXYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRYFhhhhYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYNYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRYYYYYYYYYXB_RXB_lYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRXB_RXYYYYYYYYYYYYYYYYYYYYYXYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRYYYYYYYYYXB_RXB_lYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRXB_RXYYYYYYYYYYYYYYYYYYYYYXYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYNYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRYYYYYYYYYXB_RXB_lYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRXB_RXYYYYYYYYYYYYYYYYYYYYYXYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYRXB_RXYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRXB_RXYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYSYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRYYYYYYYYYXB_RXB_lYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRXB_RXYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRYFhhhhYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYNYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRYYYYYYYYYXB_RXB_lYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRXB_RXYYYYYYYYYYYYYYYYYYYYYXYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYRXB_RXYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRXB_RXYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRYYYYYYYYYXB_RXB_lYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRYYYYYYYYYXB_RXB_lYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRXB_RXYYYYYYYYYYYYYYYYYYYYYXYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYMYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRXB_RXYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYIBRIIRIIBRCIByEEj_ByEEj_EEj +RYYYYYSSSSSSSSSSSSSSSSSSSSSSSSSSSYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRSSSSSSRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRSSSSSSSYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSYYYYYYYYYYYYRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSYYYYSSSSSSSSSSSSSSSSSYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRSSSSSSSYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYSSSSRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSSSSSSYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRSSSSSSSYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRSSSSSSSYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRSSSSSSSRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSSSSSSYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRSSSSSSSYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSYYYYYYYYYYYYRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRSSSSSSSYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSSSSSSYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRSSSSSSSYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRSSSSSSSYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRSSSSSSSRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSSSSSSYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRSSSSSSSYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSSSSSSYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRSSSSSSSYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSSSSSSYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRSSSSSSSYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSSSSSSYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRSSSSSSSYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSSRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYyYYYbYYYYYYYYYYYYYYRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRYYYYYYYYYYYYYmYYYYYYYYYYYYYYYYYYYYYRCu3YYYYYYYYYYYPYYYYYYbYYYYYYYYYRYYYYYYYYYYYYYYYYYYSSSSSSSSSSSS +RYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYyYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYR; diff --git a/crux-mir/lib/rustc_demangle/src/v0.rs b/crux-mir/lib/rustc_demangle/src/v0.rs new file mode 100644 index 000000000..3e88fa64d --- /dev/null +++ b/crux-mir/lib/rustc_demangle/src/v0.rs @@ -0,0 +1,1530 @@ +use core::convert::TryFrom; +use core::{char, fmt, iter, mem, str}; + +#[allow(unused_macros)] +macro_rules! write { + ($($ignored:tt)*) => { + compile_error!( + "use `self.print(value)` or `fmt::Trait::fmt(&value, self.out)`, \ + instead of `write!(self.out, \"{...}\", value)`" + ) + }; +} + +// Maximum recursion depth when parsing symbols before we just bail out saying +// "this symbol is invalid" +const MAX_DEPTH: u32 = 500; + +/// Representation of a demangled symbol name. +pub struct Demangle<'a> { + inner: &'a str, +} + +#[derive(PartialEq, Eq, Debug)] +pub enum ParseError { + /// Symbol doesn't match the expected `v0` grammar. + Invalid, + + /// Parsing the symbol crossed the recursion limit (see `MAX_DEPTH`). + RecursedTooDeep, +} + +/// De-mangles a Rust symbol into a more readable version +/// +/// This function will take a **mangled** symbol and return a value. When printed, +/// the de-mangled version will be written. If the symbol does not look like +/// a mangled symbol, the original value will be written instead. +pub fn demangle(s: &str) -> Result<(Demangle, &str), ParseError> { + // First validate the symbol. If it doesn't look like anything we're + // expecting, we just print it literally. Note that we must handle non-Rust + // symbols because we could have any function in the backtrace. + let inner; + if s.len() > 2 && s.starts_with("_R") { + inner = &s[2..]; + } else if s.len() > 1 && s.starts_with('R') { + // On Windows, dbghelp strips leading underscores, so we accept "R..." + // form too. + inner = &s[1..]; + } else if s.len() > 3 && s.starts_with("__R") { + // On OSX, symbols are prefixed with an extra _ + inner = &s[3..]; + } else { + return Err(ParseError::Invalid); + } + + // Paths always start with uppercase characters. + match inner.as_bytes()[0] { + b'A'..=b'Z' => {} + _ => return Err(ParseError::Invalid), + } + + // only work with ascii text + if inner.bytes().any(|c| c & 0x80 != 0) { + return Err(ParseError::Invalid); + } + + // Verify that the symbol is indeed a valid path. + let try_parse_path = |parser| { + let mut dummy_printer = Printer { + parser: Ok(parser), + out: None, + bound_lifetime_depth: 0, + }; + dummy_printer + .print_path(false) + .expect("`fmt::Error`s should be impossible without a `fmt::Formatter`"); + dummy_printer.parser + }; + let mut parser = Parser { + sym: inner, + next: 0, + depth: 0, + }; + parser = try_parse_path(parser)?; + + // Instantiating crate (paths always start with uppercase characters). + if let Some(&(b'A'..=b'Z')) = parser.sym.as_bytes().get(parser.next) { + parser = try_parse_path(parser)?; + } + + Ok((Demangle { inner }, &parser.sym[parser.next..])) +} + +impl<'s> fmt::Display for Demangle<'s> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut printer = Printer { + parser: Ok(Parser { + sym: self.inner, + next: 0, + depth: 0, + }), + out: Some(f), + bound_lifetime_depth: 0, + }; + printer.print_path(true) + } +} + +struct Ident<'s> { + /// ASCII part of the identifier. + ascii: &'s str, + /// Punycode insertion codes for Unicode codepoints, if any. + punycode: &'s str, +} + +const SMALL_PUNYCODE_LEN: usize = 128; + +impl<'s> Ident<'s> { + /// Attempt to decode punycode on the stack (allocation-free), + /// and pass the char slice to the closure, if successful. + /// This supports up to `SMALL_PUNYCODE_LEN` characters. + fn try_small_punycode_decode R, R>(&self, f: F) -> Option { + let mut out = ['\0'; SMALL_PUNYCODE_LEN]; + let mut out_len = 0; + let r = self.punycode_decode(|i, c| { + // Check there's space left for another character. + out.get(out_len).ok_or(())?; + + // Move the characters after the insert position. + let mut j = out_len; + out_len += 1; + + while j > i { + out[j] = out[j - 1]; + j -= 1; + } + + // Insert the new character. + out[i] = c; + + Ok(()) + }); + if r.is_ok() { + Some(f(&out[..out_len])) + } else { + None + } + } + + /// Decode punycode as insertion positions and characters + /// and pass them to the closure, which can return `Err(())` + /// to stop the decoding process. + fn punycode_decode Result<(), ()>>( + &self, + mut insert: F, + ) -> Result<(), ()> { + let mut punycode_bytes = self.punycode.bytes().peekable(); + if punycode_bytes.peek().is_none() { + return Err(()); + } + + let mut len = 0; + + // Populate initial output from ASCII fragment. + for c in self.ascii.chars() { + insert(len, c)?; + len += 1; + } + + // Punycode parameters and initial state. + let base = 36; + let t_min = 1; + let t_max = 26; + let skew = 38; + let mut damp = 700; + let mut bias = 72; + let mut i: usize = 0; + let mut n: usize = 0x80; + + loop { + // Read one delta value. + let mut delta: usize = 0; + let mut w = 1; + let mut k: usize = 0; + loop { + use core::cmp::{max, min}; + + k += base; + let t = min(max(k.saturating_sub(bias), t_min), t_max); + + let d = match punycode_bytes.next() { + Some(d @ b'a'..=b'z') => d - b'a', + Some(d @ b'0'..=b'9') => 26 + (d - b'0'), + _ => return Err(()), + }; + let d = d as usize; + delta = delta.checked_add(d.checked_mul(w).ok_or(())?).ok_or(())?; + if d < t { + break; + } + w = w.checked_mul(base - t).ok_or(())?; + } + + // Compute the new insert position and character. + len += 1; + i = i.checked_add(delta).ok_or(())?; + n = n.checked_add(i / len).ok_or(())?; + i %= len; + + let n_u32 = n as u32; + let c = if n_u32 as usize == n { + char::from_u32(n_u32).ok_or(())? + } else { + return Err(()); + }; + + // Insert the new character and increment the insert position. + insert(i, c)?; + i += 1; + + // If there are no more deltas, decoding is complete. + if punycode_bytes.peek().is_none() { + return Ok(()); + } + + // Perform bias adaptation. + delta /= damp; + damp = 2; + + delta += delta / len; + let mut k = 0; + while delta > ((base - t_min) * t_max) / 2 { + delta /= base - t_min; + k += base; + } + bias = k + ((base - t_min + 1) * delta) / (delta + skew); + } + } +} + +impl<'s> fmt::Display for Ident<'s> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.try_small_punycode_decode(|chars| { + for &c in chars { + c.fmt(f)?; + } + Ok(()) + }) + .unwrap_or_else(|| { + if !self.punycode.is_empty() { + f.write_str("punycode{")?; + + // Reconstruct a standard Punycode encoding, + // by using `-` as the separator. + if !self.ascii.is_empty() { + f.write_str(self.ascii)?; + f.write_str("-")?; + } + f.write_str(self.punycode)?; + + f.write_str("}") + } else { + f.write_str(self.ascii) + } + }) + } +} + +/// Sequence of lowercase hexadecimal nibbles (`0-9a-f`), used by leaf consts. +struct HexNibbles<'s> { + nibbles: &'s str, +} + +impl<'s> HexNibbles<'s> { + /// Decode an integer value (with the "most significant nibble" first), + /// returning `None` if it can't fit in an `u64`. + // FIXME(eddyb) should this "just" use `u128` instead? + fn try_parse_uint(&self) -> Option { + let nibbles = self.nibbles.trim_start_matches("0"); + + if nibbles.len() > 16 { + return None; + } + + let mut v = 0; + for nibble in nibbles.chars() { + v = (v << 4) | (nibble.to_digit(16).unwrap() as u64); + } + Some(v) + } + + /// Decode a UTF-8 byte sequence (with each byte using a pair of nibbles) + /// into individual `char`s, returning `None` for invalid UTF-8. + fn try_parse_str_chars(&self) -> Option + 's> { + if self.nibbles.len() % 2 != 0 { + return None; + } + + // FIXME(eddyb) use `array_chunks` instead, when that becomes stable. + let mut bytes = self + .nibbles + .as_bytes() + .chunks_exact(2) + .map(|slice| match slice { + [a, b] => [a, b], + _ => unreachable!(), + }) + .map(|[&hi, &lo]| { + let half = |nibble: u8| (nibble as char).to_digit(16).unwrap() as u8; + (half(hi) << 4) | half(lo) + }); + + let chars = iter::from_fn(move || { + // As long as there are any bytes left, there's at least one more + // UTF-8-encoded `char` to decode (or the possibility of error). + bytes.next().map(|first_byte| -> Result { + // FIXME(eddyb) this `enum` and `fn` should be somewhere in `core`. + enum Utf8FirstByteError { + ContinuationByte, + TooLong, + } + fn utf8_len_from_first_byte(byte: u8) -> Result { + match byte { + 0x00..=0x7f => Ok(1), + 0x80..=0xbf => Err(Utf8FirstByteError::ContinuationByte), + 0xc0..=0xdf => Ok(2), + 0xe0..=0xef => Ok(3), + 0xf0..=0xf7 => Ok(4), + 0xf8..=0xff => Err(Utf8FirstByteError::TooLong), + } + } + + // Collect the appropriate amount of bytes (up to 4), according + // to the UTF-8 length implied by the first byte. + let utf8_len = utf8_len_from_first_byte(first_byte).map_err(|_| ())?; + let utf8 = &mut [first_byte, 0, 0, 0][..utf8_len]; + for i in 1..utf8_len { + utf8[i] = bytes.next().ok_or(())?; + } + + // Fully validate the UTF-8 sequence. + let s = str::from_utf8(utf8).map_err(|_| ())?; + + // Since we included exactly one UTF-8 sequence, and validation + // succeeded, `str::chars` should return exactly one `char`. + let mut chars = s.chars(); + match (chars.next(), chars.next()) { + (Some(c), None) => Ok(c), + _ => unreachable!( + "str::from_utf8({:?}) = {:?} was expected to have 1 char, \ + but {} chars were found", + utf8, + s, + s.chars().count() + ), + } + }) + }); + + // HACK(eddyb) doing a separate validation iteration like this might be + // wasteful, but it's easier to avoid starting to print a string literal + // in the first place, than to abort it mid-string. + if chars.clone().any(|r| r.is_err()) { + None + } else { + Some(chars.map(Result::unwrap)) + } + } +} + +fn basic_type(tag: u8) -> Option<&'static str> { + Some(match tag { + b'b' => "bool", + b'c' => "char", + b'e' => "str", + b'u' => "()", + b'a' => "i8", + b's' => "i16", + b'l' => "i32", + b'x' => "i64", + b'n' => "i128", + b'i' => "isize", + b'h' => "u8", + b't' => "u16", + b'm' => "u32", + b'y' => "u64", + b'o' => "u128", + b'j' => "usize", + b'f' => "f32", + b'd' => "f64", + b'z' => "!", + b'p' => "_", + b'v' => "...", + + _ => return None, + }) +} + +struct Parser<'s> { + sym: &'s str, + next: usize, + depth: u32, +} + +impl<'s> Parser<'s> { + fn push_depth(&mut self) -> Result<(), ParseError> { + self.depth += 1; + if self.depth > MAX_DEPTH { + Err(ParseError::RecursedTooDeep) + } else { + Ok(()) + } + } + + fn pop_depth(&mut self) { + self.depth -= 1; + } + + fn peek(&self) -> Option { + self.sym.as_bytes().get(self.next).cloned() + } + + fn eat(&mut self, b: u8) -> bool { + if self.peek() == Some(b) { + self.next += 1; + true + } else { + false + } + } + + fn next(&mut self) -> Result { + let b = self.peek().ok_or(ParseError::Invalid)?; + self.next += 1; + Ok(b) + } + + fn hex_nibbles(&mut self) -> Result, ParseError> { + let start = self.next; + loop { + match self.next()? { + b'0'..=b'9' | b'a'..=b'f' => {} + b'_' => break, + _ => return Err(ParseError::Invalid), + } + } + Ok(HexNibbles { + nibbles: &self.sym[start..self.next - 1], + }) + } + + fn digit_10(&mut self) -> Result { + let d = match self.peek() { + Some(d @ b'0'..=b'9') => d - b'0', + _ => return Err(ParseError::Invalid), + }; + self.next += 1; + Ok(d) + } + + fn digit_62(&mut self) -> Result { + let d = match self.peek() { + Some(d @ b'0'..=b'9') => d - b'0', + Some(d @ b'a'..=b'z') => 10 + (d - b'a'), + Some(d @ b'A'..=b'Z') => 10 + 26 + (d - b'A'), + _ => return Err(ParseError::Invalid), + }; + self.next += 1; + Ok(d) + } + + fn integer_62(&mut self) -> Result { + if self.eat(b'_') { + return Ok(0); + } + + let mut x: u64 = 0; + while !self.eat(b'_') { + let d = self.digit_62()? as u64; + x = x.checked_mul(62).ok_or(ParseError::Invalid)?; + x = x.checked_add(d).ok_or(ParseError::Invalid)?; + } + x.checked_add(1).ok_or(ParseError::Invalid) + } + + fn opt_integer_62(&mut self, tag: u8) -> Result { + if !self.eat(tag) { + return Ok(0); + } + self.integer_62()?.checked_add(1).ok_or(ParseError::Invalid) + } + + fn disambiguator(&mut self) -> Result { + self.opt_integer_62(b's') + } + + fn namespace(&mut self) -> Result, ParseError> { + match self.next()? { + // Special namespaces, like closures and shims. + ns @ b'A'..=b'Z' => Ok(Some(ns as char)), + + // Implementation-specific/unspecified namespaces. + b'a'..=b'z' => Ok(None), + + _ => Err(ParseError::Invalid), + } + } + + fn backref(&mut self) -> Result, ParseError> { + let s_start = self.next - 1; + let i = self.integer_62()?; + if i >= s_start as u64 { + return Err(ParseError::Invalid); + } + let mut new_parser = Parser { + sym: self.sym, + next: i as usize, + depth: self.depth, + }; + new_parser.push_depth()?; + Ok(new_parser) + } + + fn ident(&mut self) -> Result, ParseError> { + let is_punycode = self.eat(b'u'); + let mut len = self.digit_10()? as usize; + if len != 0 { + while let Ok(d) = self.digit_10() { + len = len.checked_mul(10).ok_or(ParseError::Invalid)?; + len = len.checked_add(d as usize).ok_or(ParseError::Invalid)?; + } + } + + // Skip past the optional `_` separator. + self.eat(b'_'); + + let start = self.next; + self.next = self.next.checked_add(len).ok_or(ParseError::Invalid)?; + if self.next > self.sym.len() { + return Err(ParseError::Invalid); + } + + let ident = &self.sym[start..self.next]; + + if is_punycode { + let ident = match ident.bytes().rposition(|b| b == b'_') { + Some(i) => Ident { + ascii: &ident[..i], + punycode: &ident[i + 1..], + }, + None => Ident { + ascii: "", + punycode: ident, + }, + }; + if ident.punycode.is_empty() { + return Err(ParseError::Invalid); + } + Ok(ident) + } else { + Ok(Ident { + ascii: ident, + punycode: "", + }) + } + } +} + +struct Printer<'a, 'b: 'a, 's> { + /// The input parser to demangle from, or `Err` if any (parse) error was + /// encountered (in order to disallow further likely-incorrect demangling). + /// + /// See also the documentation on the `invalid!` and `parse!` macros below. + parser: Result, ParseError>, + + /// The output formatter to demangle to, or `None` while skipping printing. + out: Option<&'a mut fmt::Formatter<'b>>, + + /// Cumulative number of lifetimes bound by `for<...>` binders ('G'), + /// anywhere "around" the current entity (e.g. type) being demangled. + /// This value is not tracked while skipping printing, as it'd be unused. + /// + /// See also the documentation on the `Printer::in_binder` method. + bound_lifetime_depth: u32, +} + +impl ParseError { + /// Snippet to print when the error is initially encountered. + fn message(&self) -> &str { + match self { + ParseError::Invalid => "{invalid syntax}", + ParseError::RecursedTooDeep => "{recursion limit reached}", + } + } +} + +/// Mark the parser as errored (with `ParseError::Invalid`), print the +/// appropriate message (see `ParseError::message`) and return early. +macro_rules! invalid { + ($printer:ident) => {{ + let err = ParseError::Invalid; + $printer.print(err.message())?; + $printer.parser = Err(err); + return Ok(()); + }}; +} + +/// Call a parser method (if the parser hasn't errored yet), +/// and mark the parser as errored if it returns `Err`. +/// +/// If the parser errored, before or now, this returns early, +/// from the current function, after printing either: +/// * for a new error, the appropriate message (see `ParseError::message`) +/// * for an earlier error, only `?` - this allows callers to keep printing +/// the approximate syntax of the path/type/const, despite having errors, +/// e.g. `Vec<[(A, ?); ?]>` instead of `Vec<[(A, ?` +macro_rules! parse { + ($printer:ident, $method:ident $(($($arg:expr),*))*) => { + match $printer.parser { + Ok(ref mut parser) => match parser.$method($($($arg),*)*) { + Ok(x) => x, + Err(err) => { + $printer.print(err.message())?; + $printer.parser = Err(err); + return Ok(()); + } + } + Err(_) => return $printer.print("?"), + } + }; +} + +impl<'a, 'b, 's> Printer<'a, 'b, 's> { + /// Eat the given character from the parser, + /// returning `false` if the parser errored. + fn eat(&mut self, b: u8) -> bool { + self.parser.as_mut().map(|p| p.eat(b)) == Ok(true) + } + + /// Skip printing (i.e. `self.out` will be `None`) for the duration of the + /// given closure. This should not change parsing behavior, only disable the + /// output, but there may be optimizations (such as not traversing backrefs). + fn skipping_printing(&mut self, f: F) + where + F: FnOnce(&mut Self) -> fmt::Result, + { + let orig_out = self.out.take(); + f(self).expect("`fmt::Error`s should be impossible without a `fmt::Formatter`"); + self.out = orig_out; + } + + /// Print the target of a backref, using the given closure. + /// When printing is being skipped, the backref will only be parsed, + /// ignoring the backref's target completely. + fn print_backref(&mut self, f: F) -> fmt::Result + where + F: FnOnce(&mut Self) -> fmt::Result, + { + let backref_parser = parse!(self, backref); + + if self.out.is_none() { + return Ok(()); + } + + let orig_parser = mem::replace(&mut self.parser, Ok(backref_parser)); + let r = f(self); + self.parser = orig_parser; + r + } + + fn pop_depth(&mut self) { + if let Ok(ref mut parser) = self.parser { + parser.pop_depth(); + } + } + + /// Output the given value to `self.out` (using `fmt::Display` formatting), + /// if printing isn't being skipped. + fn print(&mut self, x: impl fmt::Display) -> fmt::Result { + if let Some(out) = &mut self.out { + fmt::Display::fmt(&x, out)?; + } + Ok(()) + } + + /// Output the given `char`s (escaped using `char::escape_debug`), with the + /// whole sequence wrapped in quotes, for either a `char` or `&str` literal, + /// if printing isn't being skipped. + fn print_quoted_escaped_chars( + &mut self, + quote: char, + chars: impl Iterator, + ) -> fmt::Result { + if let Some(out) = &mut self.out { + use core::fmt::Write; + + out.write_char(quote)?; + for c in chars { + // Special-case not escaping a single/double quote, when + // inside the opposite kind of quote. + if matches!((quote, c), ('\'', '"') | ('"', '\'')) { + out.write_char(c)?; + continue; + } + + for escaped in c.escape_debug() { + out.write_char(escaped)?; + } + } + out.write_char(quote)?; + } + Ok(()) + } + + /// Print the lifetime according to the previously decoded index. + /// An index of `0` always refers to `'_`, but starting with `1`, + /// indices refer to late-bound lifetimes introduced by a binder. + fn print_lifetime_from_index(&mut self, lt: u64) -> fmt::Result { + // Bound lifetimes aren't tracked when skipping printing. + if self.out.is_none() { + return Ok(()); + } + + self.print("'")?; + if lt == 0 { + return self.print("_"); + } + match (self.bound_lifetime_depth as u64).checked_sub(lt) { + Some(depth) => { + // Try to print lifetimes alphabetically first. + if depth < 26 { + let c = (b'a' + depth as u8) as char; + self.print(c) + } else { + // Use `'_123` after running out of letters. + self.print("_")?; + self.print(depth) + } + } + None => invalid!(self), + } + } + + /// Optionally enter a binder ('G') for late-bound lifetimes, + /// printing e.g. `for<'a, 'b> ` before calling the closure, + /// and make those lifetimes visible to it (via depth level). + fn in_binder(&mut self, f: F) -> fmt::Result + where + F: FnOnce(&mut Self) -> fmt::Result, + { + let bound_lifetimes = parse!(self, opt_integer_62(b'G')); + + // Don't track bound lifetimes when skipping printing. + if self.out.is_none() { + return f(self); + } + + if bound_lifetimes > 0 { + self.print("for<")?; + for i in 0..bound_lifetimes { + if i > 0 { + self.print(", ")?; + } + self.bound_lifetime_depth += 1; + self.print_lifetime_from_index(1)?; + } + self.print("> ")?; + } + + let r = f(self); + + // Restore `bound_lifetime_depth` to the previous value. + self.bound_lifetime_depth -= bound_lifetimes as u32; + + r + } + + /// Print list elements using the given closure and separator, + /// until the end of the list ('E') is found, or the parser errors. + /// Returns the number of elements printed. + fn print_sep_list(&mut self, f: F, sep: &str) -> Result + where + F: Fn(&mut Self) -> fmt::Result, + { + let mut i = 0; + while self.parser.is_ok() && !self.eat(b'E') { + if i > 0 { + self.print(sep)?; + } + f(self)?; + i += 1; + } + Ok(i) + } + + fn print_path(&mut self, in_value: bool) -> fmt::Result { + parse!(self, push_depth); + + let tag = parse!(self, next); + match tag { + b'C' => { + let dis = parse!(self, disambiguator); + let name = parse!(self, ident); + + self.print(name)?; + if let Some(out) = &mut self.out { + if !out.alternate() { + out.write_str("[")?; + fmt::LowerHex::fmt(&dis, out)?; + out.write_str("]")?; + } + } + } + b'N' => { + let ns = parse!(self, namespace); + + self.print_path(in_value)?; + + // HACK(eddyb) if the parser is already marked as having errored, + // `parse!` below will print a `?` without its preceding `::` + // (because printing the `::` is skipped in certain conditions, + // i.e. a lowercase namespace with an empty identifier), + // so in order to get `::?`, the `::` has to be printed here. + if self.parser.is_err() { + self.print("::")?; + } + + let dis = parse!(self, disambiguator); + let name = parse!(self, ident); + + match ns { + // Special namespaces, like closures and shims. + Some(ns) => { + self.print("::{")?; + match ns { + 'C' => self.print("closure")?, + 'S' => self.print("shim")?, + _ => self.print(ns)?, + } + if !name.ascii.is_empty() || !name.punycode.is_empty() { + self.print(":")?; + self.print(name)?; + } + self.print("#")?; + self.print(dis)?; + self.print("}")?; + } + + // Implementation-specific/unspecified namespaces. + None => { + if !name.ascii.is_empty() || !name.punycode.is_empty() { + self.print("::")?; + self.print(name)?; + } + } + } + } + b'M' | b'X' | b'Y' => { + if tag != b'Y' { + // Ignore the `impl`'s own path. + parse!(self, disambiguator); + self.skipping_printing(|this| this.print_path(false)); + } + + self.print("<")?; + self.print_type()?; + if tag != b'M' { + self.print(" as ")?; + self.print_path(false)?; + } + self.print(">")?; + } + b'I' => { + self.print_path(in_value)?; + if in_value { + self.print("::")?; + } + self.print("<")?; + self.print_sep_list(Self::print_generic_arg, ", ")?; + self.print(">")?; + } + b'B' => { + self.print_backref(|this| this.print_path(in_value))?; + } + _ => invalid!(self), + } + + self.pop_depth(); + Ok(()) + } + + fn print_generic_arg(&mut self) -> fmt::Result { + if self.eat(b'L') { + let lt = parse!(self, integer_62); + self.print_lifetime_from_index(lt) + } else if self.eat(b'K') { + self.print_const(false) + } else { + self.print_type() + } + } + + fn print_type(&mut self) -> fmt::Result { + let tag = parse!(self, next); + + if let Some(ty) = basic_type(tag) { + return self.print(ty); + } + + parse!(self, push_depth); + + match tag { + b'R' | b'Q' => { + self.print("&")?; + if self.eat(b'L') { + let lt = parse!(self, integer_62); + if lt != 0 { + self.print_lifetime_from_index(lt)?; + self.print(" ")?; + } + } + if tag != b'R' { + self.print("mut ")?; + } + self.print_type()?; + } + + b'P' | b'O' => { + self.print("*")?; + if tag != b'P' { + self.print("mut ")?; + } else { + self.print("const ")?; + } + self.print_type()?; + } + + b'A' | b'S' => { + self.print("[")?; + self.print_type()?; + if tag == b'A' { + self.print("; ")?; + self.print_const(true)?; + } + self.print("]")?; + } + b'T' => { + self.print("(")?; + let count = self.print_sep_list(Self::print_type, ", ")?; + if count == 1 { + self.print(",")?; + } + self.print(")")?; + } + b'F' => self.in_binder(|this| { + let is_unsafe = this.eat(b'U'); + let abi = if this.eat(b'K') { + if this.eat(b'C') { + Some("C") + } else { + let abi = parse!(this, ident); + if abi.ascii.is_empty() || !abi.punycode.is_empty() { + invalid!(this); + } + Some(abi.ascii) + } + } else { + None + }; + + if is_unsafe { + this.print("unsafe ")?; + } + + if let Some(abi) = abi { + this.print("extern \"")?; + + // If the ABI had any `-`, they were replaced with `_`, + // so the parts between `_` have to be re-joined with `-`. + let mut parts = abi.split('_'); + this.print(parts.next().unwrap())?; + for part in parts { + this.print("-")?; + this.print(part)?; + } + + this.print("\" ")?; + } + + this.print("fn(")?; + this.print_sep_list(Self::print_type, ", ")?; + this.print(")")?; + + if this.eat(b'u') { + // Skip printing the return type if it's 'u', i.e. `()`. + } else { + this.print(" -> ")?; + this.print_type()?; + } + + Ok(()) + })?, + b'D' => { + self.print("dyn ")?; + self.in_binder(|this| { + this.print_sep_list(Self::print_dyn_trait, " + ")?; + Ok(()) + })?; + + if !self.eat(b'L') { + invalid!(self); + } + let lt = parse!(self, integer_62); + if lt != 0 { + self.print(" + ")?; + self.print_lifetime_from_index(lt)?; + } + } + b'B' => { + self.print_backref(Self::print_type)?; + } + _ => { + // Go back to the tag, so `print_path` also sees it. + let _ = self.parser.as_mut().map(|p| p.next -= 1); + self.print_path(false)?; + } + } + + self.pop_depth(); + Ok(()) + } + + /// A trait in a trait object may have some "existential projections" + /// (i.e. associated type bindings) after it, which should be printed + /// in the `<...>` of the trait, e.g. `dyn Trait`. + /// To this end, this method will keep the `<...>` of an 'I' path + /// open, by omitting the `>`, and return `Ok(true)` in that case. + fn print_path_maybe_open_generics(&mut self) -> Result { + if self.eat(b'B') { + // NOTE(eddyb) the closure may not run if printing is being skipped, + // but in that case the returned boolean doesn't matter. + let mut open = false; + self.print_backref(|this| { + open = this.print_path_maybe_open_generics()?; + Ok(()) + })?; + Ok(open) + } else if self.eat(b'I') { + self.print_path(false)?; + self.print("<")?; + self.print_sep_list(Self::print_generic_arg, ", ")?; + Ok(true) + } else { + self.print_path(false)?; + Ok(false) + } + } + + fn print_dyn_trait(&mut self) -> fmt::Result { + let mut open = self.print_path_maybe_open_generics()?; + + while self.eat(b'p') { + if !open { + self.print("<")?; + open = true; + } else { + self.print(", ")?; + } + + let name = parse!(self, ident); + self.print(name)?; + self.print(" = ")?; + self.print_type()?; + } + + if open { + self.print(">")?; + } + + Ok(()) + } + + fn print_const(&mut self, in_value: bool) -> fmt::Result { + let tag = parse!(self, next); + + parse!(self, push_depth); + + // Only literals (and the names of `const` generic parameters, but they + // don't get mangled at all), can appear in generic argument position + // without any disambiguation, all other expressions require braces. + // To avoid duplicating the mapping between `tag` and what syntax gets + // used (especially any special-casing), every case that needs braces + // has to call `open_brace(self)?` (and the closing brace is automatic). + let mut opened_brace = false; + let mut open_brace_if_outside_expr = |this: &mut Self| { + // If this expression is nested in another, braces aren't required. + if in_value { + return Ok(()); + } + + opened_brace = true; + this.print("{") + }; + + match tag { + b'p' => self.print("_")?, + + // Primitive leaves with hex-encoded values (see `basic_type`). + b'h' | b't' | b'm' | b'y' | b'o' | b'j' => self.print_const_uint(tag)?, + b'a' | b's' | b'l' | b'x' | b'n' | b'i' => { + if self.eat(b'n') { + self.print("-")?; + } + + self.print_const_uint(tag)?; + } + b'b' => match parse!(self, hex_nibbles).try_parse_uint() { + Some(0) => self.print("false")?, + Some(1) => self.print("true")?, + _ => invalid!(self), + }, + b'c' => { + let valid_char = parse!(self, hex_nibbles) + .try_parse_uint() + .and_then(|v| u32::try_from(v).ok()) + .and_then(char::from_u32); + match valid_char { + Some(c) => self.print_quoted_escaped_chars('\'', iter::once(c))?, + None => invalid!(self), + } + } + b'e' => { + // NOTE(eddyb) a string literal `"..."` has type `&str`, so + // to get back the type `str`, `*"..."` syntax is needed + // (even if that may not be valid in Rust itself). + open_brace_if_outside_expr(self)?; + self.print("*")?; + + self.print_const_str_literal()?; + } + + b'R' | b'Q' => { + // NOTE(eddyb) this prints `"..."` instead of `&*"..."`, which + // is what `Re..._` would imply (see comment for `str` above). + if tag == b'R' && self.eat(b'e') { + self.print_const_str_literal()?; + } else { + open_brace_if_outside_expr(self)?; + self.print("&")?; + if tag != b'R' { + self.print("mut ")?; + } + self.print_const(true)?; + } + } + b'A' => { + open_brace_if_outside_expr(self)?; + self.print("[")?; + self.print_sep_list(|this| this.print_const(true), ", ")?; + self.print("]")?; + } + b'T' => { + open_brace_if_outside_expr(self)?; + self.print("(")?; + let count = self.print_sep_list(|this| this.print_const(true), ", ")?; + if count == 1 { + self.print(",")?; + } + self.print(")")?; + } + b'V' => { + open_brace_if_outside_expr(self)?; + self.print_path(true)?; + match parse!(self, next) { + b'U' => {} + b'T' => { + self.print("(")?; + self.print_sep_list(|this| this.print_const(true), ", ")?; + self.print(")")?; + } + b'S' => { + self.print(" { ")?; + self.print_sep_list( + |this| { + parse!(this, disambiguator); + let name = parse!(this, ident); + this.print(name)?; + this.print(": ")?; + this.print_const(true) + }, + ", ", + )?; + self.print(" }")?; + } + _ => invalid!(self), + } + } + b'B' => { + self.print_backref(|this| this.print_const(in_value))?; + } + _ => invalid!(self), + } + + if opened_brace { + self.print("}")?; + } + + self.pop_depth(); + Ok(()) + } + + fn print_const_uint(&mut self, ty_tag: u8) -> fmt::Result { + let hex = parse!(self, hex_nibbles); + + match hex.try_parse_uint() { + Some(v) => self.print(v)?, + + // Print anything that doesn't fit in `u64` verbatim. + None => { + self.print("0x")?; + self.print(hex.nibbles)?; + } + } + + if let Some(out) = &mut self.out { + if !out.alternate() { + let ty = basic_type(ty_tag).unwrap(); + self.print(ty)?; + } + } + + Ok(()) + } + + fn print_const_str_literal(&mut self) -> fmt::Result { + match parse!(self, hex_nibbles).try_parse_str_chars() { + Some(chars) => self.print_quoted_escaped_chars('"', chars), + None => invalid!(self), + } + } +} + +#[cfg(test)] +mod tests { + use std::prelude::v1::*; + + macro_rules! t { + ($a:expr, $b:expr) => {{ + assert_eq!(format!("{}", ::demangle($a)), $b); + }}; + } + macro_rules! t_nohash { + ($a:expr, $b:expr) => {{ + assert_eq!(format!("{:#}", ::demangle($a)), $b); + }}; + } + macro_rules! t_nohash_type { + ($a:expr, $b:expr) => { + t_nohash!(concat!("_RMC0", $a), concat!("<", $b, ">")) + }; + } + macro_rules! t_const { + ($mangled:expr, $value:expr) => { + t_nohash!( + concat!("_RIC0K", $mangled, "E"), + concat!("::<", $value, ">") + ) + }; + } + macro_rules! t_const_suffixed { + ($mangled:expr, $value:expr, $value_ty_suffix:expr) => {{ + t_const!($mangled, $value); + t!( + concat!("_RIC0K", $mangled, "E"), + concat!("[0]::<", $value, $value_ty_suffix, ">") + ); + }}; + } + + #[test] + fn demangle_crate_with_leading_digit() { + t_nohash!("_RNvC6_123foo3bar", "123foo::bar"); + } + + #[test] + fn demangle_utf8_idents() { + t_nohash!( + "_RNqCs4fqI2P2rA04_11utf8_identsu30____7hkackfecea1cbdathfdh9hlq6y", + "utf8_idents::საჭმელად_გემრიელი_სადილი" + ); + } + + #[test] + fn demangle_closure() { + t_nohash!( + "_RNCNCNgCs6DXkGYLi8lr_2cc5spawn00B5_", + "cc::spawn::{closure#0}::{closure#0}" + ); + t_nohash!( + "_RNCINkXs25_NgCsbmNqQUJIY6D_4core5sliceINyB9_4IterhENuNgNoBb_4iter8iterator8Iterator9rpositionNCNgNpB9_6memchr7memrchrs_0E0Bb_", + " as core::iter::iterator::Iterator>::rposition::::{closure#0}" + ); + } + + #[test] + fn demangle_dyn_trait() { + t_nohash!( + "_RINbNbCskIICzLVDPPb_5alloc5alloc8box_freeDINbNiB4_5boxed5FnBoxuEp6OutputuEL_ECs1iopQbuBiw2_3std", + "alloc::alloc::box_free::>" + ); + } + + #[test] + fn demangle_const_generics_preview() { + // NOTE(eddyb) this was hand-written, before rustc had working + // const generics support (but the mangling format did include them). + t_nohash_type!( + "INtC8arrayvec8ArrayVechKj7b_E", + "arrayvec::ArrayVec" + ); + t_const_suffixed!("j7b_", "123", "usize"); + } + + #[test] + fn demangle_min_const_generics() { + t_const!("p", "_"); + t_const_suffixed!("hb_", "11", "u8"); + t_const_suffixed!("off00ff00ff00ff00ff_", "0xff00ff00ff00ff00ff", "u128"); + t_const_suffixed!("s98_", "152", "i16"); + t_const_suffixed!("anb_", "-11", "i8"); + t_const!("b0_", "false"); + t_const!("b1_", "true"); + t_const!("c76_", "'v'"); + t_const!("c22_", r#"'"'"#); + t_const!("ca_", "'\\n'"); + t_const!("c2202_", "'∂'"); + } + + #[test] + fn demangle_const_str() { + t_const!("e616263_", "{*\"abc\"}"); + t_const!("e27_", r#"{*"'"}"#); + t_const!("e090a_", "{*\"\\t\\n\"}"); + t_const!("ee28882c3bc_", "{*\"∂ü\"}"); + t_const!( + "ee183a1e18390e183ade1839be18394e1839ae18390e183935fe18392e18394e1839b\ + e183a0e18398e18394e1839ae183985fe183a1e18390e18393e18398e1839ae18398_", + "{*\"საჭმელად_გემრიელი_სადილი\"}" + ); + t_const!( + "ef09f908af09fa688f09fa686f09f90ae20c2a720f09f90b6f09f9192e298\ + 95f09f94a520c2a720f09fa7a1f09f929bf09f929af09f9299f09f929c_", + "{*\"🐊🦈🦆🐮 § 🐶👒☕🔥 § 🧡💛💚💙💜\"}" + ); + } + + // NOTE(eddyb) this uses the same strings as `demangle_const_str` and should + // be kept in sync with it - while a macro could be used to generate both + // `str` and `&str` tests, from a single list of strings, this seems clearer. + #[test] + fn demangle_const_ref_str() { + t_const!("Re616263_", "\"abc\""); + t_const!("Re27_", r#""'""#); + t_const!("Re090a_", "\"\\t\\n\""); + t_const!("Ree28882c3bc_", "\"∂ü\""); + t_const!( + "Ree183a1e18390e183ade1839be18394e1839ae18390e183935fe18392e18394e1839b\ + e183a0e18398e18394e1839ae183985fe183a1e18390e18393e18398e1839ae18398_", + "\"საჭმელად_გემრიელი_სადილი\"" + ); + t_const!( + "Ref09f908af09fa688f09fa686f09f90ae20c2a720f09f90b6f09f9192e298\ + 95f09f94a520c2a720f09fa7a1f09f929bf09f929af09f9299f09f929c_", + "\"🐊🦈🦆🐮 § 🐶👒☕🔥 § 🧡💛💚💙💜\"" + ); + } + + #[test] + fn demangle_const_ref() { + t_const!("Rp", "{&_}"); + t_const!("Rh7b_", "{&123}"); + t_const!("Rb0_", "{&false}"); + t_const!("Rc58_", "{&'X'}"); + t_const!("RRRh0_", "{&&&0}"); + t_const!("RRRe_", "{&&\"\"}"); + t_const!("QAE", "{&mut []}"); + } + + #[test] + fn demangle_const_array() { + t_const!("AE", "{[]}"); + t_const!("Aj0_E", "{[0]}"); + t_const!("Ah1_h2_h3_E", "{[1, 2, 3]}"); + t_const!("ARe61_Re62_Re63_E", "{[\"a\", \"b\", \"c\"]}"); + t_const!("AAh1_h2_EAh3_h4_EE", "{[[1, 2], [3, 4]]}"); + } + + #[test] + fn demangle_const_tuple() { + t_const!("TE", "{()}"); + t_const!("Tj0_E", "{(0,)}"); + t_const!("Th1_b0_E", "{(1, false)}"); + t_const!( + "TRe616263_c78_RAh1_h2_h3_EE", + "{(\"abc\", 'x', &[1, 2, 3])}" + ); + } + + #[test] + fn demangle_const_adt() { + t_const!( + "VNvINtNtC4core6option6OptionjE4NoneU", + "{core::option::Option::::None}" + ); + t_const!( + "VNvINtNtC4core6option6OptionjE4SomeTj0_E", + "{core::option::Option::::Some(0)}" + ); + t_const!( + "VNtC3foo3BarS1sRe616263_2chc78_5sliceRAh1_h2_h3_EE", + "{foo::Bar { s: \"abc\", ch: 'x', slice: &[1, 2, 3] }}" + ); + } + + #[test] + fn demangle_exponential_explosion() { + // NOTE(eddyb) because of the prefix added by `t_nohash_type!` is + // 3 bytes long, `B2_` refers to the start of the type, not `B_`. + // 6 backrefs (`B8_E` through `B3_E`) result in 2^6 = 64 copies of `_`. + // Also, because the `p` (`_`) type is after all of the starts of the + // backrefs, it can be replaced with any other type, independently. + t_nohash_type!( + concat!("TTTTTT", "p", "B8_E", "B7_E", "B6_E", "B5_E", "B4_E", "B3_E"), + "((((((_, _), (_, _)), ((_, _), (_, _))), (((_, _), (_, _)), ((_, _), (_, _)))), \ + ((((_, _), (_, _)), ((_, _), (_, _))), (((_, _), (_, _)), ((_, _), (_, _))))), \ + (((((_, _), (_, _)), ((_, _), (_, _))), (((_, _), (_, _)), ((_, _), (_, _)))), \ + ((((_, _), (_, _)), ((_, _), (_, _))), (((_, _), (_, _)), ((_, _), (_, _))))))" + ); + } + + #[test] + fn demangle_thinlto() { + t_nohash!("_RC3foo.llvm.9D1C9369", "foo"); + t_nohash!("_RC3foo.llvm.9D1C9369@@16", "foo"); + t_nohash!("_RNvC9backtrace3foo.llvm.A5310EB9", "backtrace::foo"); + } + + #[test] + fn demangle_extra_suffix() { + // From alexcrichton/rustc-demangle#27: + t_nohash!( + "_RNvNtNtNtNtCs92dm3009vxr_4rand4rngs7adapter9reseeding4fork23FORK_HANDLER_REGISTERED.0.0", + "rand::rngs::adapter::reseeding::fork::FORK_HANDLER_REGISTERED.0.0" + ); + } + + #[test] + fn demangling_limits() { + // Stress tests found via fuzzing. + + for sym in include_str!("v0-large-test-symbols/early-recursion-limit") + .lines() + .filter(|line| !line.is_empty() && !line.starts_with('#')) + { + assert_eq!( + super::demangle(sym).map(|_| ()), + Err(super::ParseError::RecursedTooDeep) + ); + } + + assert_contains!( + ::demangle( + "RIC20tRYIMYNRYFG05_EB5_B_B6_RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR\ + RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRB_E", + ) + .to_string(), + "{recursion limit reached}" + ); + } + + #[test] + fn recursion_limit_leaks() { + // NOTE(eddyb) this test checks that both paths and types support the + // recursion limit correctly, i.e. matching `push_depth` and `pop_depth`, + // and don't leak "recursion levels" and trip the limit. + // The test inputs are generated on the fly, using a repeated pattern, + // as hardcoding the actual strings would be too verbose. + // Also, `MAX_DEPTH` can be directly used, instead of assuming its value. + for &(sym_leaf, expected_leaf) in &[("p", "_"), ("Rp", "&_"), ("C1x", "x")] { + let mut sym = format!("_RIC0p"); + let mut expected = format!("::<_"); + for _ in 0..(super::MAX_DEPTH * 2) { + sym.push_str(sym_leaf); + expected.push_str(", "); + expected.push_str(expected_leaf); + } + sym.push('E'); + expected.push('>'); + + t_nohash!(&sym, expected); + } + } + + #[test] + fn recursion_limit_backref_free_bypass() { + // NOTE(eddyb) this test checks that long symbols cannot bypass the + // recursion limit by not using backrefs, and cause a stack overflow. + + // This value was chosen to be high enough that stack overflows were + // observed even with `cargo test --release`. + let depth = 100_000; + + // In order to hide the long mangling from the initial "shallow" parse, + // it's nested in an identifier (crate name), preceding its use. + let mut sym = format!("_RIC{}", depth); + let backref_start = sym.len() - 2; + for _ in 0..depth { + sym.push('R'); + } + + // Write a backref to just after the length of the identifier. + sym.push('B'); + sym.push(char::from_digit((backref_start - 1) as u32, 36).unwrap()); + sym.push('_'); + + // Close the `I` at the start. + sym.push('E'); + + assert_contains!(::demangle(&sym).to_string(), "{recursion limit reached}"); + } +} diff --git a/crux-mir/lib/rustc_std_workspace_alloc/Cargo.toml b/crux-mir/lib/rustc_std_workspace_alloc/Cargo.toml new file mode 100644 index 000000000..049ca3e46 --- /dev/null +++ b/crux-mir/lib/rustc_std_workspace_alloc/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "rustc-std-workspace-alloc" +version = "1.99.0" +license = 'MIT OR Apache-2.0' +description = """ +Hack for the compiler's own build system +""" +edition = "2021" + +[lib] +path = "lib.rs" + +[dependencies] +alloc = { path = "../alloc" } diff --git a/crux-mir/lib/rustc_std_workspace_alloc/lib.rs b/crux-mir/lib/rustc_std_workspace_alloc/lib.rs new file mode 100644 index 000000000..87db7af44 --- /dev/null +++ b/crux-mir/lib/rustc_std_workspace_alloc/lib.rs @@ -0,0 +1,9 @@ +#![feature(no_core)] +#![no_core] + +// See rustc-std-workspace-core for why this crate is needed. + +// Rename the crate to avoid conflicting with the alloc module in alloc. +extern crate alloc as foo; + +pub use foo::*; diff --git a/crux-mir/lib/rustc_std_workspace_core/Cargo.toml b/crux-mir/lib/rustc_std_workspace_core/Cargo.toml new file mode 100644 index 000000000..ff5cfcbd6 --- /dev/null +++ b/crux-mir/lib/rustc_std_workspace_core/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "rustc-std-workspace-core" +version = "1.99.0" +license = 'MIT OR Apache-2.0' +description = """ +Hack for the compiler's own build system +""" +edition = "2021" + +[lib] +path = "lib.rs" + +[dependencies] +core = { path = "../core" } diff --git a/crux-mir/lib/rustc_std_workspace_core/README.md b/crux-mir/lib/rustc_std_workspace_core/README.md new file mode 100644 index 000000000..40e0b62af --- /dev/null +++ b/crux-mir/lib/rustc_std_workspace_core/README.md @@ -0,0 +1,29 @@ +# The `rustc-std-workspace-core` crate + +This crate is a shim and empty crate which simply depends on `libcore` and +reexports all of its contents. The crate is the crux of empowering the standard +library to depend on crates from crates.io + +Crates on crates.io that the standard library depend on need to depend on the +`rustc-std-workspace-core` crate from crates.io, which is empty. We use +`[patch]` to override it to this crate in this repository. As a result, crates +on crates.io will draw a dependency edge to `libcore`, the version defined in +this repository. That should draw all the dependency edges to ensure Cargo +builds crates successfully! + +Note that crates on crates.io need to depend on this crate with the name `core` +for everything to work correctly. To do that they can use: + +```toml +core = { version = "1.0.0", optional = true, package = 'rustc-std-workspace-core' } +``` + +Through the use of the `package` key the crate is renamed to `core`, meaning +it'll look like + +``` +--extern core=.../librustc_std_workspace_core-XXXXXXX.rlib +``` + +when Cargo invokes the compiler, satisfying the implicit `extern crate core` +directive injected by the compiler. diff --git a/crux-mir/lib/rustc_std_workspace_core/lib.rs b/crux-mir/lib/rustc_std_workspace_core/lib.rs new file mode 100644 index 000000000..143278525 --- /dev/null +++ b/crux-mir/lib/rustc_std_workspace_core/lib.rs @@ -0,0 +1,4 @@ +#![feature(no_core)] +#![no_core] + +pub use core::*; diff --git a/crux-mir/lib/std/Cargo.toml b/crux-mir/lib/std/Cargo.toml new file mode 100644 index 000000000..adf521d9b --- /dev/null +++ b/crux-mir/lib/std/Cargo.toml @@ -0,0 +1,87 @@ +[package] +name = "std" +version = "0.0.0" +license = "MIT OR Apache-2.0" +repository = "https://github.com/rust-lang/rust.git" +description = "The Rust Standard Library" +edition = "2021" + +[lib] +crate-type = ["dylib", "rlib"] + +[dependencies] +alloc = { path = "../alloc" } +cfg-if = { version = "1.0", features = ['rustc-dep-of-std'] } +panic_unwind = { path = "../panic_unwind", optional = true } +panic_abort = { path = "../panic_abort" } +core = { path = "../core" } +libc = { version = "0.2.138", default-features = false, features = ['rustc-dep-of-std'] } +compiler_builtins = { version = "0.1.85" } +profiler_builtins = { path = "../profiler_builtins", optional = true } +unwind = { path = "../unwind" } +hashbrown = { version = "0.12", default-features = false, features = ['rustc-dep-of-std'] } +std_detect = { path = "../stdarch/crates/std_detect", default-features = false, features = ['rustc-dep-of-std'] } + +# Dependencies of the `backtrace` crate +addr2line = { version = "0.17.0", optional = true, default-features = false } +rustc-demangle = { version = "0.1.21", features = ['rustc-dep-of-std'] } +miniz_oxide = { version = "0.5.0", optional = true, default-features = false } +[dependencies.object] +version = "0.29.0" +optional = true +default-features = false +features = ['read_core', 'elf', 'macho', 'pe', 'unaligned', 'archive'] + +[dev-dependencies] +rand = { version = "0.8.5", default-features = false, features = ["alloc"] } +rand_xorshift = "0.3.0" + +[target.'cfg(any(all(target_family = "wasm", not(target_os = "emscripten")), all(target_vendor = "fortanix", target_env = "sgx")))'.dependencies] +dlmalloc = { version = "0.2.3", features = ['rustc-dep-of-std'] } + +[target.x86_64-fortanix-unknown-sgx.dependencies] +fortanix-sgx-abi = { version = "0.5.0", features = ['rustc-dep-of-std'] } + +[target.'cfg(target_os = "hermit")'.dependencies] +hermit-abi = { version = "0.2.6", features = ['rustc-dep-of-std'] } + +[target.wasm32-wasi.dependencies] +wasi = { version = "0.11.0", features = ['rustc-dep-of-std'], default-features = false } + +[features] +backtrace = [ + "gimli-symbolize", + 'addr2line/rustc-dep-of-std', + 'object/rustc-dep-of-std', + 'miniz_oxide/rustc-dep-of-std', +] +gimli-symbolize = [] + +panic-unwind = ["panic_unwind"] +profiler = ["profiler_builtins"] +compiler-builtins-c = ["alloc/compiler-builtins-c"] +compiler-builtins-mem = ["alloc/compiler-builtins-mem"] +compiler-builtins-no-asm = ["alloc/compiler-builtins-no-asm"] +compiler-builtins-mangled-names = ["alloc/compiler-builtins-mangled-names"] +llvm-libunwind = ["unwind/llvm-libunwind"] +system-llvm-libunwind = ["unwind/system-llvm-libunwind"] + +# Make panics and failed asserts immediately abort without formatting any message +panic_immediate_abort = ["core/panic_immediate_abort"] + +# Enable std_detect default features for stdarch/crates/std_detect: +# https://github.com/rust-lang/stdarch/blob/master/crates/std_detect/Cargo.toml +std_detect_file_io = ["std_detect/std_detect_file_io"] +std_detect_dlsym_getauxval = ["std_detect/std_detect_dlsym_getauxval"] +std_detect_env_override = ["std_detect/std_detect_env_override"] + +[package.metadata.fortanix-sgx] +# Maximum possible number of threads when testing +threads = 125 +# Maximum heap size +heap_size = 0x8000000 + +[[bench]] +name = "stdbenches" +path = "benches/lib.rs" +test = true diff --git a/crux-mir/lib/std/build.rs b/crux-mir/lib/std/build.rs new file mode 100644 index 000000000..8b1a06ee7 --- /dev/null +++ b/crux-mir/lib/std/build.rs @@ -0,0 +1,52 @@ +use std::env; + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + let target = env::var("TARGET").expect("TARGET was not set"); + if target.contains("freebsd") { + if env::var("RUST_STD_FREEBSD_12_ABI").is_ok() { + println!("cargo:rustc-cfg=freebsd12"); + } + } else if target.contains("linux") + || target.contains("netbsd") + || target.contains("dragonfly") + || target.contains("openbsd") + || target.contains("solaris") + || target.contains("illumos") + || target.contains("apple-darwin") + || target.contains("apple-ios") + || target.contains("apple-watchos") + || target.contains("uwp") + || target.contains("windows") + || target.contains("fuchsia") + || (target.contains("sgx") && target.contains("fortanix")) + || target.contains("hermit") + || target.contains("l4re") + || target.contains("redox") + || target.contains("haiku") + || target.contains("vxworks") + || target.contains("wasm32") + || target.contains("wasm64") + || target.contains("asmjs") + || target.contains("espidf") + || target.contains("solid") + || target.contains("nintendo-3ds") + { + // These platforms don't have any special requirements. + } else { + // This is for Cargo's build-std support, to mark std as unstable for + // typically no_std platforms. + // This covers: + // - os=none ("bare metal" targets) + // - mipsel-sony-psp + // - nvptx64-nvidia-cuda + // - arch=avr + // - tvos (aarch64-apple-tvos, x86_64-apple-tvos) + // - uefi (x86_64-unknown-uefi, i686-unknown-uefi) + // - JSON targets + // - Any new targets that have not been explicitly added above. + println!("cargo:rustc-cfg=feature=\"restricted-std\""); + } + println!("cargo:rustc-env=STD_ENV_ARCH={}", env::var("CARGO_CFG_TARGET_ARCH").unwrap()); + println!("cargo:rustc-cfg=backtrace_in_libstd"); +} diff --git a/crux-mir/lib/std/primitive_docs/box_into_raw.md b/crux-mir/lib/std/primitive_docs/box_into_raw.md new file mode 100644 index 000000000..307b9c85b --- /dev/null +++ b/crux-mir/lib/std/primitive_docs/box_into_raw.md @@ -0,0 +1 @@ +Box::into_raw diff --git a/crux-mir/lib/std/primitive_docs/fs_file.md b/crux-mir/lib/std/primitive_docs/fs_file.md new file mode 100644 index 000000000..13e454083 --- /dev/null +++ b/crux-mir/lib/std/primitive_docs/fs_file.md @@ -0,0 +1 @@ +fs::File diff --git a/crux-mir/lib/std/primitive_docs/io_bufread.md b/crux-mir/lib/std/primitive_docs/io_bufread.md new file mode 100644 index 000000000..bb688e3a5 --- /dev/null +++ b/crux-mir/lib/std/primitive_docs/io_bufread.md @@ -0,0 +1 @@ +io::BufRead diff --git a/crux-mir/lib/std/primitive_docs/io_read.md b/crux-mir/lib/std/primitive_docs/io_read.md new file mode 100644 index 000000000..5118d7c48 --- /dev/null +++ b/crux-mir/lib/std/primitive_docs/io_read.md @@ -0,0 +1 @@ +io::Read diff --git a/crux-mir/lib/std/primitive_docs/io_seek.md b/crux-mir/lib/std/primitive_docs/io_seek.md new file mode 100644 index 000000000..122e6df77 --- /dev/null +++ b/crux-mir/lib/std/primitive_docs/io_seek.md @@ -0,0 +1 @@ +io::Seek diff --git a/crux-mir/lib/std/primitive_docs/io_write.md b/crux-mir/lib/std/primitive_docs/io_write.md new file mode 100644 index 000000000..15dfc907a --- /dev/null +++ b/crux-mir/lib/std/primitive_docs/io_write.md @@ -0,0 +1 @@ +io::Write diff --git a/crux-mir/lib/std/primitive_docs/net_tosocketaddrs.md b/crux-mir/lib/std/primitive_docs/net_tosocketaddrs.md new file mode 100644 index 000000000..a01f318e8 --- /dev/null +++ b/crux-mir/lib/std/primitive_docs/net_tosocketaddrs.md @@ -0,0 +1 @@ +net::ToSocketAddrs diff --git a/crux-mir/lib/std/primitive_docs/process_exit.md b/crux-mir/lib/std/primitive_docs/process_exit.md new file mode 100644 index 000000000..565a71375 --- /dev/null +++ b/crux-mir/lib/std/primitive_docs/process_exit.md @@ -0,0 +1 @@ +process::exit diff --git a/crux-mir/lib/std/primitive_docs/string_string.md b/crux-mir/lib/std/primitive_docs/string_string.md new file mode 100644 index 000000000..ce7815ff9 --- /dev/null +++ b/crux-mir/lib/std/primitive_docs/string_string.md @@ -0,0 +1 @@ +string::String diff --git a/crux-mir/lib/std/src/Cargo.toml b/crux-mir/lib/std/src/Cargo.toml deleted file mode 100644 index 41a1f94ff..000000000 --- a/crux-mir/lib/std/src/Cargo.toml +++ /dev/null @@ -1,77 +0,0 @@ -[package] -authors = ["The Rust Project Developers"] -name = "std" -version = "0.0.0" -build = "build.rs" -license = "MIT OR Apache-2.0" -repository = "https://github.com/rust-lang/rust.git" -description = "The Rust Standard Library" -edition = "2018" - -[lib] -name = "std" -path = "lib.rs" -crate-type = ["dylib", "rlib"] - -[dependencies] -alloc = { path = "../liballoc" } -cfg-if = { version = "0.1.8", features = ['rustc-dep-of-std'] } -panic_unwind = { path = "../libpanic_unwind", optional = true } -panic_abort = { path = "../libpanic_abort" } -core = { path = "../libcore" } -libc = { version = "0.2.51", default-features = false, features = ['rustc-dep-of-std'] } -compiler_builtins = { version = "0.1.16" } -profiler_builtins = { path = "../libprofiler_builtins", optional = true } -unwind = { path = "../libunwind" } -hashbrown = { version = "0.6.2", default-features = false, features = ['rustc-dep-of-std'] } - -[dependencies.backtrace_rs] -package = "backtrace" -version = "0.3.44" -default-features = false # without the libstd `backtrace` feature, stub out everything -features = [ "rustc-dep-of-std" ] # enable build support for integrating into libstd - -[dev-dependencies] -rand = "0.7" - -[target.'cfg(any(all(target_arch = "wasm32", not(target_os = "emscripten")), all(target_vendor = "fortanix", target_env = "sgx")))'.dependencies] -dlmalloc = { version = "0.1", features = ['rustc-dep-of-std'] } - -[target.x86_64-fortanix-unknown-sgx.dependencies] -fortanix-sgx-abi = { version = "0.3.2", features = ['rustc-dep-of-std'] } - -[target.'cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_os = "hermit"))'.dependencies] -hermit-abi = { version = "0.1", features = ['rustc-dep-of-std'] } - -[target.wasm32-wasi.dependencies] -wasi = { version = "0.9.0", features = ['rustc-dep-of-std'], default-features = false } - -[features] -default = ["std_detect_file_io", "std_detect_dlsym_getauxval"] -opaque-poison-atomics = [] - -backtrace = [ - "backtrace_rs/dbghelp", # backtrace/symbolize on MSVC - "backtrace_rs/libbacktrace", # symbolize on most platforms - "backtrace_rs/libunwind", # backtrace on most platforms - "backtrace_rs/dladdr", # symbolize on platforms w/o libbacktrace -] - -panic-unwind = ["panic_unwind"] -profiler = ["profiler_builtins"] -compiler-builtins-c = ["alloc/compiler-builtins-c"] -llvm-libunwind = ["unwind/llvm-libunwind"] - -# Make panics and failed asserts immediately abort without formatting any message -panic_immediate_abort = ["core/panic_immediate_abort"] - -# Enable std_detect default features for stdarch/crates/std_detect: -# https://github.com/rust-lang/stdarch/blob/master/crates/std_detect/Cargo.toml -std_detect_file_io = [] -std_detect_dlsym_getauxval = [] - -[package.metadata.fortanix-sgx] -# Maximum possible number of threads when testing -threads = 125 -# Maximum heap size -heap_size = 0x8000000 diff --git a/crux-mir/lib/std/src/alloc.rs b/crux-mir/lib/std/src/alloc.rs index 25f3ddcbe..c5a5991cc 100644 --- a/crux-mir/lib/std/src/alloc.rs +++ b/crux-mir/lib/std/src/alloc.rs @@ -1,4 +1,4 @@ -//! Memory allocation APIs +//! Memory allocation APIs. //! //! In a given program, the standard library has one “global” memory allocator //! that is used for example by `Box` and `Vec`. @@ -7,8 +7,6 @@ //! like `cdylib`s and `staticlib`s are guaranteed to use the [`System`] by //! default. //! -//! [`System`]: struct.System.html -//! //! # The `#[global_allocator]` attribute //! //! This attribute allows configuring the choice of global allocator. @@ -43,11 +41,7 @@ //! The attribute is used on a `static` item whose type implements the //! [`GlobalAlloc`] trait. This type can be provided by an external library: //! -//! [`GlobalAlloc`]: ../../core/alloc/trait.GlobalAlloc.html -//! //! ```rust,ignore (demonstrates crates.io usage) -//! extern crate jemallocator; -//! //! use jemallocator::Jemalloc; //! //! #[global_allocator] @@ -59,14 +53,14 @@ //! The `#[global_allocator]` can only be used once in a crate //! or its recursive dependencies. +#![deny(unsafe_op_in_unsafe_fn)] #![stable(feature = "alloc_module", since = "1.28.0")] +use core::intrinsics; use core::ptr::NonNull; use core::sync::atomic::{AtomicPtr, Ordering}; use core::{mem, ptr}; -use crate::sys_common::util::dumb_print; - #[stable(feature = "alloc_module", since = "1.28.0")] #[doc(inline)] pub use alloc_crate::alloc::*; @@ -74,7 +68,10 @@ pub use alloc_crate::alloc::*; /// The default memory allocator provided by the operating system. /// /// This is based on `malloc` on Unix platforms and `HeapAlloc` on Windows, -/// plus related functions. +/// plus related functions. However, it is not valid to mix use of the backing +/// system allocator with `System`, as this implementation may include extra +/// work, such as to serve alignment requests greater than the alignment +/// provided directly by the backing system allocator. /// /// This type implements the `GlobalAlloc` trait and Rust programs by default /// work as if they had this definition: @@ -87,7 +84,7 @@ pub use alloc_crate::alloc::*; /// /// fn main() { /// let a = Box::new(4); // Allocates from the system allocator. -/// println!("{}", a); +/// println!("{a}"); /// } /// ``` /// @@ -108,7 +105,7 @@ pub use alloc_crate::alloc::*; /// if !ret.is_null() { /// ALLOCATED.fetch_add(layout.size(), SeqCst); /// } -/// return ret +/// ret /// } /// /// unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { @@ -133,60 +130,158 @@ pub use alloc_crate::alloc::*; #[derive(Debug, Default, Copy, Clone)] pub struct System; -// The AllocRef impl checks the layout size to be non-zero and forwards to the GlobalAlloc impl, -// which is in `std::sys::*::alloc`. -#[unstable(feature = "allocator_api", issue = "32838")] -unsafe impl AllocRef for System { +impl System { #[inline] - fn alloc(&mut self, layout: Layout) -> Result<(NonNull, usize), AllocErr> { - if layout.size() == 0 { - Ok((layout.dangling(), 0)) - } else { - unsafe { - NonNull::new(GlobalAlloc::alloc(self, layout)) - .ok_or(AllocErr) - .map(|p| (p, layout.size())) - } + fn alloc_impl(&self, layout: Layout, zeroed: bool) -> Result, AllocError> { + match layout.size() { + 0 => Ok(NonNull::slice_from_raw_parts(layout.dangling(), 0)), + // SAFETY: `layout` is non-zero in size, + size => unsafe { + let raw_ptr = if zeroed { + GlobalAlloc::alloc_zeroed(self, layout) + } else { + GlobalAlloc::alloc(self, layout) + }; + let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; + Ok(NonNull::slice_from_raw_parts(ptr, size)) + }, } } + // SAFETY: Same as `Allocator::grow` #[inline] - fn alloc_zeroed(&mut self, layout: Layout) -> Result<(NonNull, usize), AllocErr> { - if layout.size() == 0 { - Ok((layout.dangling(), 0)) - } else { - unsafe { - NonNull::new(GlobalAlloc::alloc_zeroed(self, layout)) - .ok_or(AllocErr) - .map(|p| (p, layout.size())) - } + unsafe fn grow_impl( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + zeroed: bool, + ) -> Result, AllocError> { + debug_assert!( + new_layout.size() >= old_layout.size(), + "`new_layout.size()` must be greater than or equal to `old_layout.size()`" + ); + + match old_layout.size() { + 0 => self.alloc_impl(new_layout, zeroed), + + // SAFETY: `new_size` is non-zero as `new_size` is greater than or equal to `old_size` + // as required by safety conditions and the `old_size == 0` case was handled in the + // previous match arm. Other conditions must be upheld by the caller + old_size if old_layout.align() == new_layout.align() => unsafe { + let new_size = new_layout.size(); + + // `realloc` probably checks for `new_size >= old_layout.size()` or something similar. + intrinsics::assume(new_size >= old_layout.size()); + + let raw_ptr = GlobalAlloc::realloc(self, ptr.as_ptr(), old_layout, new_size); + let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; + if zeroed { + raw_ptr.add(old_size).write_bytes(0, new_size - old_size); + } + Ok(NonNull::slice_from_raw_parts(ptr, new_size)) + }, + + // SAFETY: because `new_layout.size()` must be greater than or equal to `old_size`, + // both the old and new memory allocation are valid for reads and writes for `old_size` + // bytes. Also, because the old allocation wasn't yet deallocated, it cannot overlap + // `new_ptr`. Thus, the call to `copy_nonoverlapping` is safe. The safety contract + // for `dealloc` must be upheld by the caller. + old_size => unsafe { + let new_ptr = self.alloc_impl(new_layout, zeroed)?; + ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_mut_ptr(), old_size); + Allocator::deallocate(self, ptr, old_layout); + Ok(new_ptr) + }, } } +} + +// The Allocator impl checks the layout size to be non-zero and forwards to the GlobalAlloc impl, +// which is in `std::sys::*::alloc`. +#[unstable(feature = "allocator_api", issue = "32838")] +unsafe impl Allocator for System { + #[inline] + fn allocate(&self, layout: Layout) -> Result, AllocError> { + self.alloc_impl(layout, false) + } + + #[inline] + fn allocate_zeroed(&self, layout: Layout) -> Result, AllocError> { + self.alloc_impl(layout, true) + } #[inline] - unsafe fn dealloc(&mut self, ptr: NonNull, layout: Layout) { + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { if layout.size() != 0 { - GlobalAlloc::dealloc(self, ptr.as_ptr(), layout) + // SAFETY: `layout` is non-zero in size, + // other conditions must be upheld by the caller + unsafe { GlobalAlloc::dealloc(self, ptr.as_ptr(), layout) } } } #[inline] - unsafe fn realloc( - &mut self, + unsafe fn grow( + &self, ptr: NonNull, - layout: Layout, - new_size: usize, - ) -> Result<(NonNull, usize), AllocErr> { - match (layout.size(), new_size) { - (0, 0) => Ok((layout.dangling(), 0)), - (0, _) => self.alloc(Layout::from_size_align_unchecked(new_size, layout.align())), - (_, 0) => { - self.dealloc(ptr, layout); - Ok((layout.dangling(), 0)) - } - (_, _) => NonNull::new(GlobalAlloc::realloc(self, ptr.as_ptr(), layout, new_size)) - .ok_or(AllocErr) - .map(|p| (p, new_size)), + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + // SAFETY: all conditions must be upheld by the caller + unsafe { self.grow_impl(ptr, old_layout, new_layout, false) } + } + + #[inline] + unsafe fn grow_zeroed( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + // SAFETY: all conditions must be upheld by the caller + unsafe { self.grow_impl(ptr, old_layout, new_layout, true) } + } + + #[inline] + unsafe fn shrink( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + debug_assert!( + new_layout.size() <= old_layout.size(), + "`new_layout.size()` must be smaller than or equal to `old_layout.size()`" + ); + + match new_layout.size() { + // SAFETY: conditions must be upheld by the caller + 0 => unsafe { + Allocator::deallocate(self, ptr, old_layout); + Ok(NonNull::slice_from_raw_parts(new_layout.dangling(), 0)) + }, + + // SAFETY: `new_size` is non-zero. Other conditions must be upheld by the caller + new_size if old_layout.align() == new_layout.align() => unsafe { + // `realloc` probably checks for `new_size <= old_layout.size()` or something similar. + intrinsics::assume(new_size <= old_layout.size()); + + let raw_ptr = GlobalAlloc::realloc(self, ptr.as_ptr(), old_layout, new_size); + let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; + Ok(NonNull::slice_from_raw_parts(ptr, new_size)) + }, + + // SAFETY: because `new_size` must be smaller than or equal to `old_layout.size()`, + // both the old and new memory allocation are valid for reads and writes for `new_size` + // bytes. Also, because the old allocation wasn't yet deallocated, it cannot overlap + // `new_ptr`. Thus, the call to `copy_nonoverlapping` is safe. The safety contract + // for `dealloc` must be upheld by the caller. + new_size => unsafe { + let new_ptr = Allocator::allocate(self, new_layout)?; + ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_mut_ptr(), new_size); + Allocator::deallocate(self, ptr, old_layout); + Ok(new_ptr) + }, } } } @@ -205,8 +300,19 @@ static HOOK: AtomicPtr<()> = AtomicPtr::new(ptr::null_mut()); /// /// The allocation error hook is a global resource. /// -/// [`set_alloc_error_hook`]: fn.set_alloc_error_hook.html -/// [`take_alloc_error_hook`]: fn.take_alloc_error_hook.html +/// # Examples +/// +/// ``` +/// #![feature(alloc_error_hook)] +/// +/// use std::alloc::{Layout, set_alloc_error_hook}; +/// +/// fn custom_alloc_error_hook(layout: Layout) { +/// panic!("memory allocation of {} bytes failed", layout.size()); +/// } +/// +/// set_alloc_error_hook(custom_alloc_error_hook); +/// ``` #[unstable(feature = "alloc_error_hook", issue = "51245")] pub fn set_alloc_error_hook(hook: fn(Layout)) { HOOK.store(hook as *mut (), Ordering::SeqCst); @@ -217,8 +323,6 @@ pub fn set_alloc_error_hook(hook: fn(Layout)) { /// *See also the function [`set_alloc_error_hook`].* /// /// If no custom hook is registered, the default hook will be returned. -/// -/// [`set_alloc_error_hook`]: fn.set_alloc_error_hook.html #[unstable(feature = "alloc_error_hook", issue = "51245")] pub fn take_alloc_error_hook() -> fn(Layout) { let hook = HOOK.swap(ptr::null_mut(), Ordering::SeqCst); @@ -226,7 +330,18 @@ pub fn take_alloc_error_hook() -> fn(Layout) { } fn default_alloc_error_hook(layout: Layout) { - dumb_print(format_args!("memory allocation of {} bytes failed", layout.size())); + extern "Rust" { + // This symbol is emitted by rustc next to __rust_alloc_error_handler. + // Its value depends on the -Zoom={panic,abort} compiler option. + static __rust_alloc_error_handler_should_panic: u8; + } + + #[allow(unused_unsafe)] + if unsafe { __rust_alloc_error_handler_should_panic != 0 } { + panic!("memory allocation of {} bytes failed", layout.size()); + } else { + rtprintpanic!("memory allocation of {} bytes failed\n", layout.size()); + } } #[cfg(not(test))] @@ -238,9 +353,7 @@ pub fn rust_oom(layout: Layout) -> ! { let hook: fn(Layout) = if hook.is_null() { default_alloc_error_hook } else { unsafe { mem::transmute(hook) } }; hook(layout); - unsafe { - crate::sys::abort_internal(); - } + crate::process::abort() } #[cfg(not(test))] @@ -250,10 +363,10 @@ pub fn rust_oom(layout: Layout) -> ! { pub mod __default_lib_allocator { use super::{GlobalAlloc, Layout, System}; // These magic symbol names are used as a fallback for implementing the - // `__rust_alloc` etc symbols (see `src/liballoc/alloc.rs) when there is + // `__rust_alloc` etc symbols (see `src/liballoc/alloc.rs`) when there is // no `#[global_allocator]` attribute. - // for symbol names src/librustc/middle/allocator.rs + // for symbol names src/librustc_ast/expand/allocator.rs // for signatures src/librustc_allocator/lib.rs // linkage directives are provided as part of the current compiler allocator @@ -261,13 +374,19 @@ pub mod __default_lib_allocator { #[rustc_std_internal_symbol] pub unsafe extern "C" fn __rdl_alloc(size: usize, align: usize) -> *mut u8 { - let layout = Layout::from_size_align_unchecked(size, align); - System.alloc(layout) + // SAFETY: see the guarantees expected by `Layout::from_size_align` and + // `GlobalAlloc::alloc`. + unsafe { + let layout = Layout::from_size_align_unchecked(size, align); + System.alloc(layout) + } } #[rustc_std_internal_symbol] pub unsafe extern "C" fn __rdl_dealloc(ptr: *mut u8, size: usize, align: usize) { - System.dealloc(ptr, Layout::from_size_align_unchecked(size, align)) + // SAFETY: see the guarantees expected by `Layout::from_size_align` and + // `GlobalAlloc::dealloc`. + unsafe { System.dealloc(ptr, Layout::from_size_align_unchecked(size, align)) } } #[rustc_std_internal_symbol] @@ -277,13 +396,21 @@ pub mod __default_lib_allocator { align: usize, new_size: usize, ) -> *mut u8 { - let old_layout = Layout::from_size_align_unchecked(old_size, align); - System.realloc(ptr, old_layout, new_size) + // SAFETY: see the guarantees expected by `Layout::from_size_align` and + // `GlobalAlloc::realloc`. + unsafe { + let old_layout = Layout::from_size_align_unchecked(old_size, align); + System.realloc(ptr, old_layout, new_size) + } } #[rustc_std_internal_symbol] pub unsafe extern "C" fn __rdl_alloc_zeroed(size: usize, align: usize) -> *mut u8 { - let layout = Layout::from_size_align_unchecked(size, align); - System.alloc_zeroed(layout) + // SAFETY: see the guarantees expected by `Layout::from_size_align` and + // `GlobalAlloc::alloc_zeroed`. + unsafe { + let layout = Layout::from_size_align_unchecked(size, align); + System.alloc_zeroed(layout) + } } } diff --git a/crux-mir/lib/std/src/ascii.rs b/crux-mir/lib/std/src/ascii.rs index 41bdfea53..c29f01577 100644 --- a/crux-mir/lib/std/src/ascii.rs +++ b/crux-mir/lib/std/src/ascii.rs @@ -10,9 +10,6 @@ //! //! The [`escape_default`] function provides an iterator over the bytes of an //! escaped version of the character given. -//! -//! [`AsciiExt`]: trait.AsciiExt.html -//! [`escape_default`]: fn.escape_default.html #![stable(feature = "rust1", since = "1.0.0")] @@ -42,7 +39,7 @@ pub use core::ascii::{escape_default, EscapeDefault}; /// /// [combining character]: https://en.wikipedia.org/wiki/Combining_character #[stable(feature = "rust1", since = "1.0.0")] -#[rustc_deprecated(since = "1.26.0", reason = "use inherent methods instead")] +#[deprecated(since = "1.26.0", note = "use inherent methods instead")] pub trait AsciiExt { /// Container type for copied ASCII characters. #[stable(feature = "rust1", since = "1.0.0")] @@ -52,7 +49,7 @@ pub trait AsciiExt { /// /// # Note /// - /// This method will be deprecated in favor of the identically-named + /// This method is deprecated in favor of the identically-named /// inherent methods on `u8`, `char`, `[u8]` and `str`. #[stable(feature = "rust1", since = "1.0.0")] fn is_ascii(&self) -> bool; @@ -69,11 +66,10 @@ pub trait AsciiExt { /// /// # Note /// - /// This method will be deprecated in favor of the identically-named + /// This method is deprecated in favor of the identically-named /// inherent methods on `u8`, `char`, `[u8]` and `str`. /// - /// [`make_ascii_uppercase`]: #tymethod.make_ascii_uppercase - /// [`str::to_uppercase`]: ../primitive.str.html#method.to_uppercase + /// [`make_ascii_uppercase`]: AsciiExt::make_ascii_uppercase #[stable(feature = "rust1", since = "1.0.0")] #[allow(deprecated)] fn to_ascii_uppercase(&self) -> Self::Owned; @@ -90,11 +86,10 @@ pub trait AsciiExt { /// /// # Note /// - /// This method will be deprecated in favor of the identically-named + /// This method is deprecated in favor of the identically-named /// inherent methods on `u8`, `char`, `[u8]` and `str`. /// - /// [`make_ascii_lowercase`]: #tymethod.make_ascii_lowercase - /// [`str::to_lowercase`]: ../primitive.str.html#method.to_lowercase + /// [`make_ascii_lowercase`]: AsciiExt::make_ascii_lowercase #[stable(feature = "rust1", since = "1.0.0")] #[allow(deprecated)] fn to_ascii_lowercase(&self) -> Self::Owned; @@ -106,7 +101,7 @@ pub trait AsciiExt { /// /// # Note /// - /// This method will be deprecated in favor of the identically-named + /// This method is deprecated in favor of the identically-named /// inherent methods on `u8`, `char`, `[u8]` and `str`. #[stable(feature = "rust1", since = "1.0.0")] fn eq_ignore_ascii_case(&self, other: &Self) -> bool; @@ -121,10 +116,10 @@ pub trait AsciiExt { /// /// # Note /// - /// This method will be deprecated in favor of the identically-named + /// This method is deprecated in favor of the identically-named /// inherent methods on `u8`, `char`, `[u8]` and `str`. /// - /// [`to_ascii_uppercase`]: #tymethod.to_ascii_uppercase + /// [`to_ascii_uppercase`]: AsciiExt::to_ascii_uppercase #[stable(feature = "ascii", since = "1.9.0")] fn make_ascii_uppercase(&mut self); @@ -138,10 +133,10 @@ pub trait AsciiExt { /// /// # Note /// - /// This method will be deprecated in favor of the identically-named + /// This method is deprecated in favor of the identically-named /// inherent methods on `u8`, `char`, `[u8]` and `str`. /// - /// [`to_ascii_lowercase`]: #tymethod.to_ascii_lowercase + /// [`to_ascii_lowercase`]: AsciiExt::to_ascii_lowercase #[stable(feature = "ascii", since = "1.9.0")] fn make_ascii_lowercase(&mut self); } @@ -149,23 +144,35 @@ pub trait AsciiExt { macro_rules! delegating_ascii_methods { () => { #[inline] - fn is_ascii(&self) -> bool { self.is_ascii() } + fn is_ascii(&self) -> bool { + self.is_ascii() + } #[inline] - fn to_ascii_uppercase(&self) -> Self::Owned { self.to_ascii_uppercase() } + fn to_ascii_uppercase(&self) -> Self::Owned { + self.to_ascii_uppercase() + } #[inline] - fn to_ascii_lowercase(&self) -> Self::Owned { self.to_ascii_lowercase() } + fn to_ascii_lowercase(&self) -> Self::Owned { + self.to_ascii_lowercase() + } #[inline] - fn eq_ignore_ascii_case(&self, o: &Self) -> bool { self.eq_ignore_ascii_case(o) } + fn eq_ignore_ascii_case(&self, o: &Self) -> bool { + self.eq_ignore_ascii_case(o) + } #[inline] - fn make_ascii_uppercase(&mut self) { self.make_ascii_uppercase(); } + fn make_ascii_uppercase(&mut self) { + self.make_ascii_uppercase(); + } #[inline] - fn make_ascii_lowercase(&mut self) { self.make_ascii_lowercase(); } - } + fn make_ascii_lowercase(&mut self) { + self.make_ascii_lowercase(); + } + }; } #[stable(feature = "rust1", since = "1.0.0")] diff --git a/crux-mir/lib/std/src/backtrace.rs b/crux-mir/lib/std/src/backtrace.rs index e10d46603..7543ffadd 100644 --- a/crux-mir/lib/std/src/backtrace.rs +++ b/crux-mir/lib/std/src/backtrace.rs @@ -9,19 +9,13 @@ //! implementing `std::error::Error`) to get a causal chain of where an error //! was generated. //! -//! > **Note**: this module is unstable and is designed in [RFC 2504], and you -//! > can learn more about its status in the [tracking issue]. -//! -//! [RFC 2504]: https://github.com/rust-lang/rfcs/blob/master/text/2504-fix-error.md -//! [tracking issue]: https://github.com/rust-lang/rust/issues/53487 -//! //! ## Accuracy //! //! Backtraces are attempted to be as accurate as possible, but no guarantees //! are provided about the exact accuracy of a backtrace. Instruction pointers, //! symbol names, filenames, line numbers, etc, may all be incorrect when -//! reported. Accuracy is attempted on a best-effort basis, however, and bugs -//! are always welcome to indicate areas of improvement! +//! reported. Accuracy is attempted on a best-effort basis, however, any bug +//! reports are always welcome to indicate areas of improvement! //! //! For most platforms a backtrace with a filename/line number requires that //! programs be compiled with debug information. Without debug information @@ -29,23 +23,23 @@ //! //! ## Platform support //! -//! Not all platforms that libstd compiles for support capturing backtraces. -//! Some platforms simply do nothing when capturing a backtrace. To check -//! whether the platform supports capturing backtraces you can consult the -//! `BacktraceStatus` enum as a result of `Backtrace::status`. +//! Not all platforms that std compiles for support capturing backtraces. Some +//! platforms simply do nothing when capturing a backtrace. To check whether the +//! platform supports capturing backtraces you can consult the `BacktraceStatus` +//! enum as a result of `Backtrace::status`. //! //! Like above with accuracy platform support is done on a best effort basis. -//! Sometimes libraries may not be available at runtime or something may go +//! Sometimes libraries might not be available at runtime or something may go //! wrong which would cause a backtrace to not be captured. Please feel free to //! report issues with platforms where a backtrace cannot be captured though! //! //! ## Environment Variables //! -//! The `Backtrace::capture` function may not actually capture a backtrace by +//! The `Backtrace::capture` function might not actually capture a backtrace by //! default. Its behavior is governed by two environment variables: //! //! * `RUST_LIB_BACKTRACE` - if this is set to `0` then `Backtrace::capture` -//! will never capture a backtrace. Any other value this is set to will enable +//! will never capture a backtrace. Any other value set will enable //! `Backtrace::capture`. //! //! * `RUST_BACKTRACE` - if `RUST_LIB_BACKTRACE` is not set, then this variable @@ -61,10 +55,13 @@ //! Note that the `Backtrace::force_capture` function can be used to ignore //! these environment variables. Also note that the state of environment //! variables is cached once the first backtrace is created, so altering -//! `RUST_LIB_BACKTRACE` or `RUST_BACKTRACE` at runtime may not actually change +//! `RUST_LIB_BACKTRACE` or `RUST_BACKTRACE` at runtime might not actually change //! how backtraces are captured. -#![unstable(feature = "backtrace", issue = "53487")] +#![stable(feature = "backtrace", since = "1.65.0")] + +#[cfg(test)] +mod tests; // NB: A note on resolution of a backtrace: // @@ -91,15 +88,15 @@ // `Backtrace`, but that's a relatively small price to pay relative to capturing // a backtrace or actually symbolizing it. +use crate::backtrace_rs::{self, BytesOrWideString}; +use crate::cell::UnsafeCell; use crate::env; use crate::ffi::c_void; use crate::fmt; -use crate::sync::atomic::{AtomicUsize, Ordering::SeqCst}; -use crate::sync::Mutex; +use crate::sync::atomic::{AtomicUsize, Ordering::Relaxed}; +use crate::sync::Once; use crate::sys_common::backtrace::{lock, output_filename}; use crate::vec::Vec; -use backtrace::BytesOrWideString; -use backtrace_rs as backtrace; /// A captured OS thread stack backtrace. /// @@ -107,30 +104,36 @@ use backtrace_rs as backtrace; /// previous point in time. In some instances the `Backtrace` type may /// internally be empty due to configuration. For more information see /// `Backtrace::capture`. +#[stable(feature = "backtrace", since = "1.65.0")] +#[must_use] pub struct Backtrace { inner: Inner, } /// The current status of a backtrace, indicating whether it was captured or /// whether it is empty for some other reason. +#[stable(feature = "backtrace", since = "1.65.0")] #[non_exhaustive] #[derive(Debug, PartialEq, Eq)] pub enum BacktraceStatus { /// Capturing a backtrace is not supported, likely because it's not /// implemented for the current platform. + #[stable(feature = "backtrace", since = "1.65.0")] Unsupported, /// Capturing a backtrace has been disabled through either the /// `RUST_LIB_BACKTRACE` or `RUST_BACKTRACE` environment variables. + #[stable(feature = "backtrace", since = "1.65.0")] Disabled, /// A backtrace has been captured and the `Backtrace` should print /// reasonable information when rendered. + #[stable(feature = "backtrace", since = "1.65.0")] Captured, } enum Inner { Unsupported, Disabled, - Captured(Mutex), + Captured(LazilyResolvedCapture), } struct Capture { @@ -144,13 +147,16 @@ fn _assert_send_sync() { _assert::(); } -struct BacktraceFrame { +/// A single frame of a backtrace. +#[unstable(feature = "backtrace_frames", issue = "79676")] +pub struct BacktraceFrame { frame: RawFrame, symbols: Vec, } +#[derive(Debug)] enum RawFrame { - Actual(backtrace::Frame), + Actual(backtrace_rs::Frame), #[cfg(test)] Fake, } @@ -159,6 +165,7 @@ struct BacktraceSymbol { name: Option>, filename: Option, lineno: Option, + colno: Option, } enum BytesOrWide { @@ -166,14 +173,14 @@ enum BytesOrWide { Wide(Vec), } +#[stable(feature = "backtrace", since = "1.65.0")] impl fmt::Debug for Backtrace { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - let mut capture = match &self.inner { + let capture = match &self.inner { Inner::Unsupported => return fmt.write_str(""), Inner::Disabled => return fmt.write_str(""), - Inner::Captured(c) => c.lock().unwrap(), + Inner::Captured(c) => c.force(), }; - capture.resolve(); let frames = &capture.frames[capture.actual_start..]; @@ -193,11 +200,24 @@ impl fmt::Debug for Backtrace { } } +#[unstable(feature = "backtrace_frames", issue = "79676")] +impl fmt::Debug for BacktraceFrame { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut dbg = fmt.debug_list(); + dbg.entries(&self.symbols); + dbg.finish() + } +} + impl fmt::Debug for BacktraceSymbol { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + // FIXME: improve formatting: https://github.com/rust-lang/rust/issues/65280 + // FIXME: Also, include column numbers into the debug format as Display already has them. + // Until there are stable per-frame accessors, the format shouldn't be changed: + // https://github.com/rust-lang/rust/issues/65280#issuecomment-638966585 write!(fmt, "{{ ")?; - if let Some(fn_name) = self.name.as_ref().map(|b| backtrace::SymbolName::new(b)) { + if let Some(fn_name) = self.name.as_ref().map(|b| backtrace_rs::SymbolName::new(b)) { write!(fmt, "fn: \"{:#}\"", fn_name)?; } else { write!(fmt, "fn: ")?; @@ -207,7 +227,7 @@ impl fmt::Debug for BacktraceSymbol { write!(fmt, ", file: \"{:?}\"", fname)?; } - if let Some(line) = self.lineno.as_ref() { + if let Some(line) = self.lineno { write!(fmt, ", line: {:?}", line)?; } @@ -223,7 +243,7 @@ impl fmt::Debug for BytesOrWide { BytesOrWide::Bytes(w) => BytesOrWideString::Bytes(w), BytesOrWide::Wide(w) => BytesOrWideString::Wide(w), }, - backtrace::PrintFmt::Short, + backtrace_rs::PrintFmt::Short, crate::env::current_dir().as_ref().ok(), ) } @@ -237,7 +257,7 @@ impl Backtrace { // backtrace captures speedy, because otherwise reading environment // variables every time can be somewhat slow. static ENABLED: AtomicUsize = AtomicUsize::new(0); - match ENABLED.load(SeqCst) { + match ENABLED.load(Relaxed) { 0 => {} 1 => return false, _ => return true, @@ -249,7 +269,7 @@ impl Backtrace { Err(_) => false, }, }; - ENABLED.store(enabled as usize + 1, SeqCst); + ENABLED.store(enabled as usize + 1, Relaxed); enabled } @@ -269,6 +289,7 @@ impl Backtrace { /// /// To forcibly capture a backtrace regardless of environment variables, use /// the `Backtrace::force_capture` function. + #[stable(feature = "backtrace", since = "1.65.0")] #[inline(never)] // want to make sure there's a frame here to remove pub fn capture() -> Backtrace { if !Backtrace::enabled() { @@ -287,11 +308,20 @@ impl Backtrace { /// Note that capturing a backtrace can be an expensive operation on some /// platforms, so this should be used with caution in performance-sensitive /// parts of code. + #[stable(feature = "backtrace", since = "1.65.0")] #[inline(never)] // want to make sure there's a frame here to remove pub fn force_capture() -> Backtrace { Backtrace::create(Backtrace::force_capture as usize) } + /// Forcibly captures a disabled backtrace, regardless of environment + /// variable configuration. + #[stable(feature = "backtrace", since = "1.65.0")] + #[rustc_const_stable(feature = "backtrace", since = "1.65.0")] + pub const fn disabled() -> Backtrace { + Backtrace { inner: Inner::Disabled } + } + // Capture a backtrace which start just before the function addressed by // `ip` fn create(ip: usize) -> Backtrace { @@ -299,12 +329,12 @@ impl Backtrace { let mut frames = Vec::new(); let mut actual_start = None; unsafe { - backtrace::trace_unsynchronized(|frame| { + backtrace_rs::trace_unsynchronized(|frame| { frames.push(BacktraceFrame { frame: RawFrame::Actual(frame.clone()), symbols: Vec::new(), }); - if frame.symbol_address() as usize == ip && actual_start.is_none() { + if frame.symbol_address().addr() == ip && actual_start.is_none() { actual_start = Some(frames.len()); } true @@ -317,7 +347,7 @@ impl Backtrace { let inner = if frames.is_empty() { Inner::Unsupported } else { - Inner::Captured(Mutex::new(Capture { + Inner::Captured(LazilyResolvedCapture::new(Capture { actual_start: actual_start.unwrap_or(0), frames, resolved: false, @@ -330,6 +360,8 @@ impl Backtrace { /// Returns the status of this backtrace, indicating whether this backtrace /// request was unsupported, disabled, or a stack trace was actually /// captured. + #[stable(feature = "backtrace", since = "1.65.0")] + #[must_use] pub fn status(&self) -> BacktraceStatus { match self.inner { Inner::Unsupported => BacktraceStatus::Unsupported, @@ -339,20 +371,29 @@ impl Backtrace { } } +impl<'a> Backtrace { + /// Returns an iterator over the backtrace frames. + #[must_use] + #[unstable(feature = "backtrace_frames", issue = "79676")] + pub fn frames(&'a self) -> &'a [BacktraceFrame] { + if let Inner::Captured(c) = &self.inner { &c.force().frames } else { &[] } + } +} + +#[stable(feature = "backtrace", since = "1.65.0")] impl fmt::Display for Backtrace { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - let mut capture = match &self.inner { + let capture = match &self.inner { Inner::Unsupported => return fmt.write_str("unsupported backtrace"), Inner::Disabled => return fmt.write_str("disabled backtrace"), - Inner::Captured(c) => c.lock().unwrap(), + Inner::Captured(c) => c.force(), }; - capture.resolve(); let full = fmt.alternate(); let (frames, style) = if full { - (&capture.frames[..], backtrace::PrintFmt::Full) + (&capture.frames[..], backtrace_rs::PrintFmt::Full) } else { - (&capture.frames[capture.actual_start..], backtrace::PrintFmt::Short) + (&capture.frames[capture.actual_start..], backtrace_rs::PrintFmt::Short) }; // When printing paths we try to strip the cwd if it exists, otherwise @@ -364,22 +405,22 @@ impl fmt::Display for Backtrace { output_filename(fmt, path, style, cwd.as_ref().ok()) }; - let mut f = backtrace::BacktraceFmt::new(fmt, style, &mut print_path); + let mut f = backtrace_rs::BacktraceFmt::new(fmt, style, &mut print_path); f.add_context()?; for frame in frames { - let mut f = f.frame(); if frame.symbols.is_empty() { - f.print_raw(frame.frame.ip(), None, None, None)?; + f.frame().print_raw(frame.frame.ip(), None, None, None)?; } else { for symbol in frame.symbols.iter() { - f.print_raw( + f.frame().print_raw_with_column( frame.frame.ip(), - symbol.name.as_ref().map(|b| backtrace::SymbolName::new(b)), + symbol.name.as_ref().map(|b| backtrace_rs::SymbolName::new(b)), symbol.filename.as_ref().map(|b| match b { BytesOrWide::Bytes(w) => BytesOrWideString::Bytes(w), BytesOrWide::Wide(w) => BytesOrWideString::Wide(w), }), symbol.lineno, + symbol.colno, )?; } } @@ -389,6 +430,33 @@ impl fmt::Display for Backtrace { } } +struct LazilyResolvedCapture { + sync: Once, + capture: UnsafeCell, +} + +impl LazilyResolvedCapture { + fn new(capture: Capture) -> Self { + LazilyResolvedCapture { sync: Once::new(), capture: UnsafeCell::new(capture) } + } + + fn force(&self) -> &Capture { + self.sync.call_once(|| { + // SAFETY: This exclusive reference can't overlap with any others + // `Once` guarantees callers will block until this closure returns + // `Once` also guarantees only a single caller will enter this closure + unsafe { &mut *self.capture.get() }.resolve(); + }); + + // SAFETY: This shared reference can't overlap with the exclusive reference above + unsafe { &*self.capture.get() } + } +} + +// SAFETY: Access to the inner value is synchronized using a thread-safe `Once` +// So long as `Capture` is `Sync`, `LazilyResolvedCapture` is too +unsafe impl Sync for LazilyResolvedCapture where Capture: Sync {} + impl Capture { fn resolve(&mut self) { // If we're already resolved, nothing to do! @@ -409,7 +477,7 @@ impl Capture { RawFrame::Fake => unimplemented!(), }; unsafe { - backtrace::resolve_frame_unsynchronized(frame, |symbol| { + backtrace_rs::resolve_frame_unsynchronized(frame, |symbol| { symbols.push(BacktraceSymbol { name: symbol.name().map(|m| m.as_bytes().to_vec()), filename: symbol.filename_raw().map(|b| match b { @@ -417,6 +485,7 @@ impl Capture { BytesOrWideString::Wide(b) => BytesOrWide::Wide(b.to_owned()), }), lineno: symbol.lineno(), + colno: symbol.colno(), }); }); } @@ -429,59 +498,7 @@ impl RawFrame { match self { RawFrame::Actual(frame) => frame.ip(), #[cfg(test)] - RawFrame::Fake => 1 as *mut c_void, + RawFrame::Fake => crate::ptr::invalid_mut(1), } } } - -#[test] -fn test_debug() { - let backtrace = Backtrace { - inner: Inner::Captured(Mutex::new(Capture { - actual_start: 1, - resolved: true, - frames: vec![ - BacktraceFrame { - frame: RawFrame::Fake, - symbols: vec![BacktraceSymbol { - name: Some(b"std::backtrace::Backtrace::create".to_vec()), - filename: Some(BytesOrWide::Bytes(b"rust/backtrace.rs".to_vec())), - lineno: Some(100), - }], - }, - BacktraceFrame { - frame: RawFrame::Fake, - symbols: vec![BacktraceSymbol { - name: Some(b"__rust_maybe_catch_panic".to_vec()), - filename: None, - lineno: None, - }], - }, - BacktraceFrame { - frame: RawFrame::Fake, - symbols: vec![ - BacktraceSymbol { - name: Some(b"std::rt::lang_start_internal".to_vec()), - filename: Some(BytesOrWide::Bytes(b"rust/rt.rs".to_vec())), - lineno: Some(300), - }, - BacktraceSymbol { - name: Some(b"std::rt::lang_start".to_vec()), - filename: Some(BytesOrWide::Bytes(b"rust/rt.rs".to_vec())), - lineno: Some(400), - }, - ], - }, - ], - })), - }; - - #[rustfmt::skip] - let expected = "Backtrace [\ - \n { fn: \"__rust_maybe_catch_panic\" },\ - \n { fn: \"std::rt::lang_start_internal\", file: \"rust/rt.rs\", line: 300 },\ - \n { fn: \"std::rt::lang_start\", file: \"rust/rt.rs\", line: 400 },\ - \n]"; - - assert_eq!(format!("{:#?}", backtrace), expected); -} diff --git a/crux-mir/lib/std/src/backtrace/tests.rs b/crux-mir/lib/std/src/backtrace/tests.rs new file mode 100644 index 000000000..4dfbf88e8 --- /dev/null +++ b/crux-mir/lib/std/src/backtrace/tests.rs @@ -0,0 +1,95 @@ +use super::*; + +fn generate_fake_frames() -> Vec { + vec![ + BacktraceFrame { + frame: RawFrame::Fake, + symbols: vec![BacktraceSymbol { + name: Some(b"std::backtrace::Backtrace::create".to_vec()), + filename: Some(BytesOrWide::Bytes(b"rust/backtrace.rs".to_vec())), + lineno: Some(100), + colno: None, + }], + }, + BacktraceFrame { + frame: RawFrame::Fake, + symbols: vec![BacktraceSymbol { + name: Some(b"__rust_maybe_catch_panic".to_vec()), + filename: None, + lineno: None, + colno: None, + }], + }, + BacktraceFrame { + frame: RawFrame::Fake, + symbols: vec![ + BacktraceSymbol { + name: Some(b"std::rt::lang_start_internal".to_vec()), + filename: Some(BytesOrWide::Bytes(b"rust/rt.rs".to_vec())), + lineno: Some(300), + colno: Some(5), + }, + BacktraceSymbol { + name: Some(b"std::rt::lang_start".to_vec()), + filename: Some(BytesOrWide::Bytes(b"rust/rt.rs".to_vec())), + lineno: Some(400), + colno: None, + }, + ], + }, + ] +} + +#[test] +fn test_debug() { + let backtrace = Backtrace { + inner: Inner::Captured(LazilyResolvedCapture::new(Capture { + actual_start: 1, + resolved: true, + frames: generate_fake_frames(), + })), + }; + + #[rustfmt::skip] + let expected = "Backtrace [\ + \n { fn: \"__rust_maybe_catch_panic\" },\ + \n { fn: \"std::rt::lang_start_internal\", file: \"rust/rt.rs\", line: 300 },\ + \n { fn: \"std::rt::lang_start\", file: \"rust/rt.rs\", line: 400 },\ + \n]"; + + assert_eq!(format!("{backtrace:#?}"), expected); + + // Format the backtrace a second time, just to make sure lazily resolved state is stable + assert_eq!(format!("{backtrace:#?}"), expected); +} + +#[test] +fn test_frames() { + let backtrace = Backtrace { + inner: Inner::Captured(LazilyResolvedCapture::new(Capture { + actual_start: 1, + resolved: true, + frames: generate_fake_frames(), + })), + }; + + let frames = backtrace.frames(); + + #[rustfmt::skip] + let expected = vec![ + "[ + { fn: \"std::backtrace::Backtrace::create\", file: \"rust/backtrace.rs\", line: 100 }, +]", + "[ + { fn: \"__rust_maybe_catch_panic\" }, +]", + "[ + { fn: \"std::rt::lang_start_internal\", file: \"rust/rt.rs\", line: 300 }, + { fn: \"std::rt::lang_start\", file: \"rust/rt.rs\", line: 400 }, +]" + ]; + + let mut iter = frames.iter().zip(expected.iter()); + + assert!(iter.all(|(f, e)| format!("{f:#?}") == *e)); +} diff --git a/crux-mir/lib/std/src/build.rs b/crux-mir/lib/std/src/build.rs deleted file mode 100644 index 8db7bc12c..000000000 --- a/crux-mir/lib/std/src/build.rs +++ /dev/null @@ -1,58 +0,0 @@ -use std::env; - -fn main() { - let target = env::var("TARGET").expect("TARGET was not set"); - if target.contains("linux") { - if target.contains("android") { - println!("cargo:rustc-link-lib=dl"); - println!("cargo:rustc-link-lib=log"); - println!("cargo:rustc-link-lib=gcc"); - } else if !target.contains("musl") { - println!("cargo:rustc-link-lib=dl"); - println!("cargo:rustc-link-lib=rt"); - println!("cargo:rustc-link-lib=pthread"); - } - } else if target.contains("freebsd") { - println!("cargo:rustc-link-lib=execinfo"); - println!("cargo:rustc-link-lib=pthread"); - } else if target.contains("netbsd") { - println!("cargo:rustc-link-lib=pthread"); - println!("cargo:rustc-link-lib=rt"); - } else if target.contains("dragonfly") || target.contains("openbsd") { - println!("cargo:rustc-link-lib=pthread"); - } else if target.contains("solaris") { - println!("cargo:rustc-link-lib=socket"); - println!("cargo:rustc-link-lib=posix4"); - println!("cargo:rustc-link-lib=pthread"); - println!("cargo:rustc-link-lib=resolv"); - } else if target.contains("apple-darwin") { - println!("cargo:rustc-link-lib=System"); - - // res_init and friends require -lresolv on macOS/iOS. - // See #41582 and http://blog.achernya.com/2013/03/os-x-has-silly-libsystem.html - println!("cargo:rustc-link-lib=resolv"); - } else if target.contains("apple-ios") { - println!("cargo:rustc-link-lib=System"); - println!("cargo:rustc-link-lib=objc"); - println!("cargo:rustc-link-lib=framework=Security"); - println!("cargo:rustc-link-lib=framework=Foundation"); - println!("cargo:rustc-link-lib=resolv"); - } else if target.contains("uwp") { - println!("cargo:rustc-link-lib=ws2_32"); - // For BCryptGenRandom - println!("cargo:rustc-link-lib=bcrypt"); - } else if target.contains("windows") { - println!("cargo:rustc-link-lib=advapi32"); - println!("cargo:rustc-link-lib=ws2_32"); - println!("cargo:rustc-link-lib=userenv"); - } else if target.contains("fuchsia") { - println!("cargo:rustc-link-lib=zircon"); - println!("cargo:rustc-link-lib=fdio"); - } else if target.contains("cloudabi") { - if cfg!(feature = "backtrace") { - println!("cargo:rustc-link-lib=unwind"); - } - println!("cargo:rustc-link-lib=c"); - println!("cargo:rustc-link-lib=compiler_rt"); - } -} diff --git a/crux-mir/lib/std/src/collections/hash/crucible_map.rs b/crux-mir/lib/std/src/collections/hash/crucible_map.rs deleted file mode 100644 index a12be49e6..000000000 --- a/crux-mir/lib/std/src/collections/hash/crucible_map.rs +++ /dev/null @@ -1,408 +0,0 @@ -use crate::borrow::Borrow; -use crate::collections::TryReserveError; -use crate::hash::Hash; -use crate::marker::PhantomData; -use crate::mem; -use crate::slice; -use crate::vec; - -/// A simple Vec-based key/value map. Presents the same API as `hashbrown::HashMap`, so it can be -/// subbed in for `base::HashMap` in the implementation of `std::collections::HashMap`. -#[derive(Clone)] -pub struct HashMap { - items: Vec<(K, V)>, - hasher: S, - _marker: PhantomData, -} - -impl HashMap { - pub fn with_hasher(hash_builder: S) -> HashMap { - Self::with_capacity_and_hasher(0, hash_builder) - } - - pub fn with_capacity_and_hasher(cap: usize, hash_builder: S) -> HashMap { - HashMap { - items: Vec::with_capacity(cap), - hasher: hash_builder, - _marker: PhantomData, - } - } - - pub fn capacity(&self) -> usize { - self.items.capacity() - } - - pub fn iter(&self) -> Iter<'_, K, V> { - Iter { it: self.items.iter() } - } - - pub fn iter_mut(&mut self) -> IterMut<'_, K, V> { - IterMut { it: self.items.iter_mut() } - } - - pub fn len(&self) -> usize { - self.items.len() - } - - pub fn is_empty(&self) -> bool { - self.items.is_empty() - } - - pub fn drain(&mut self) -> Drain<'_, K, V> { - Drain { it: self.items.drain(..) } - } - - pub fn clear(&mut self) { - self.items.clear(); - } - - pub fn hasher(&self) -> &S { - &self.hasher - } -} - -impl HashMap { - pub fn reserve(&mut self, additional: usize) { - self.items.reserve(additional) - } - - pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { - self.items.try_reserve(additional) - } - - pub fn shrink_to_fit(&mut self) { - self.items.shrink_to_fit(); - } - - pub fn shrink_to(&mut self, min_capacity: usize) { - self.items.shrink_to(min_capacity); - } - - pub fn rustc_entry(&mut self, k: K) -> RustcEntry<'_, K, V> { - match self.items.iter().position(|&(ref k2, _)| k2 == &k) { - Some(idx) => RustcEntry::Occupied(RustcOccupiedEntry { - items: &mut self.items, - idx, - k: Some(k), - }), - None => RustcEntry::Vacant(RustcVacantEntry { - items: &mut self.items, - k, - }), - } - } - - pub fn get(&self, k: &Q) -> Option<&V> - where - K: Borrow, - Q: Hash + Eq, - { - self.get_key_value(k).map(|(_, v)| v) - } - - pub fn get_key_value(&self, k: &Q) -> Option<(&K, &V)> - where - K: Borrow, - Q: Hash + Eq, - { - self.items.iter() - .find(|&(ref k2, _)| k2.borrow() == k) - .map(|&(ref k, ref v)| (k, v)) - } - - pub fn contains_key(&self, k: &Q) -> bool - where - K: Borrow, - Q: Hash + Eq, - { - self.get_key_value(k).is_some() - } - - pub fn get_mut(&mut self, k: &Q) -> Option<&mut V> - where - K: Borrow, - Q: Hash + Eq, - { - self.items.iter_mut() - .find(|&(ref k2, _)| k2.borrow() == k) - .map(|&mut (_, ref mut v)| v) - } - - pub fn insert(&mut self, k: K, v: V) -> Option { - match self.rustc_entry(k) { - RustcEntry::Occupied(mut e) => { Some(e.insert(v)) }, - RustcEntry::Vacant(e) => { - e.insert(v); - None - }, - } - } - - pub fn remove(&mut self, k: &Q) -> Option - where - K: Borrow, - Q: Hash + Eq, - { - self.remove_entry(k).map(|(_, v)| v) - } - - pub fn remove_entry(&mut self, k: &Q) -> Option<(K, V)> - where - K: Borrow, - Q: Hash + Eq, - { - let idx = self.items.iter().position(|&(ref k2, _)| k2.borrow() == k)?; - Some(self.items.swap_remove(idx)) - } - - pub fn retain(&mut self, mut f: F) - where - F: FnMut(&K, &mut V) -> bool, - { - for i in (0 .. self.items.len()).rev() { - let (ref k, ref mut v) = self.items[i]; - if !f(k, v) { - self.items.swap_remove(i); - } - } - } -} - - -pub struct Iter<'a, K: 'a, V: 'a> { - it: slice::Iter<'a, (K, V)>, -} - -impl<'a, K, V> Clone for Iter<'a, K, V> { - fn clone(&self) -> Self { - Iter { it: self.it.clone() } - } -} - -impl<'a, K, V> Iterator for Iter<'a, K, V> { - type Item = (&'a K, &'a V); - - fn next(&mut self) -> Option<(&'a K, &'a V)> { - let &(ref k, ref v) = self.it.next()?; - Some((k, v)) - } - - fn size_hint(&self) -> (usize, Option) { - self.it.size_hint() - } -} - -impl<'a, K, V> ExactSizeIterator for Iter<'a, K, V> { - fn len(&self) -> usize { - self.it.len() - } -} - - -pub struct IterMut<'a, K: 'a, V: 'a> { - it: slice::IterMut<'a, (K, V)>, -} - -impl<'a, K, V> IterMut<'a, K, V> { - pub fn rustc_iter(&self) -> Iter<'_, K, V> { - Iter { - it: self.it.as_slice().iter(), - } - } -} - -impl<'a, K, V> Iterator for IterMut<'a, K, V> { - type Item = (&'a K, &'a mut V); - - fn next(&mut self) -> Option<(&'a K, &'a mut V)> { - let &mut (ref k, ref mut v) = self.it.next()?; - Some((k, v)) - } - - fn size_hint(&self) -> (usize, Option) { - self.it.size_hint() - } -} - -impl<'a, K, V> ExactSizeIterator for IterMut<'a, K, V> { - fn len(&self) -> usize { - self.it.len() - } -} - - -impl IntoIterator for HashMap { - type Item = (K, V); - type IntoIter = IntoIter; - - #[inline] - fn into_iter(self) -> IntoIter { - IntoIter { - it: self.items.into_iter(), - } - } -} - -pub struct IntoIter { - it: vec::IntoIter<(K, V)>, -} - -impl IntoIter { - pub fn rustc_iter(&self) -> Iter<'_, K, V> { - Iter { - it: self.it.as_slice().iter(), - } - } -} - -impl Iterator for IntoIter { - type Item = (K, V); - - fn next(&mut self) -> Option<(K, V)> { - self.it.next() - } - - fn size_hint(&self) -> (usize, Option) { - self.it.size_hint() - } -} - -impl ExactSizeIterator for IntoIter { - fn len(&self) -> usize { - self.it.len() - } -} - - -pub struct Drain<'a, K, V> { - it: vec::Drain<'a, (K, V)>, -} - -impl<'a, K, V> Drain<'a, K, V> { - pub fn rustc_iter(&self) -> Iter<'_, K, V> { - Iter { - it: self.it.as_slice().iter(), - } - } -} - -impl<'a, K, V> Iterator for Drain<'a, K, V> { - type Item = (K, V); - - fn next(&mut self) -> Option<(K, V)> { - self.it.next() - } - - fn size_hint(&self) -> (usize, Option) { - self.it.size_hint() - } -} - -impl<'a, K, V> ExactSizeIterator for Drain<'a, K, V> { - fn len(&self) -> usize { - self.it.len() - } -} - - -impl Extend<(K, V)> for HashMap -where - K: Eq + Hash, -{ - fn extend>(&mut self, iter: T) { - for (k, v) in iter { - self.insert(k, v); - } - } -} - -impl<'a, K, V, S> Extend<(&'a K, &'a V)> for HashMap -where - K: Eq + Hash + Copy, - V: Copy, -{ - #[inline] - fn extend>(&mut self, iter: T) { - self.extend(iter.into_iter().map(|(&key, &value)| (key, value))); - } -} - - -pub enum RustcEntry<'a, K: 'a, V: 'a> { - Occupied(RustcOccupiedEntry<'a, K, V>), - Vacant(RustcVacantEntry<'a, K, V>), -} - -pub struct RustcOccupiedEntry<'a, K, V> { - items: &'a mut Vec<(K, V)>, - idx: usize, - k: Option, -} - -pub struct RustcVacantEntry<'a, K, V> { - items: &'a mut Vec<(K, V)>, - k: K, -} - -impl<'a, K, V> RustcOccupiedEntry<'a, K, V> { - pub fn key(&self) -> &K { - &self.items[self.idx].0 - } - - pub fn remove_entry(self) -> (K, V) { - self.items.swap_remove(self.idx) - } - - pub fn get(&self) -> &V { - &self.items[self.idx].1 - } - - pub fn get_mut(&mut self) -> &mut V { - &mut self.items[self.idx].1 - } - - pub fn into_mut(self) -> &'a mut V { - &mut self.items[self.idx].1 - } - - pub fn insert(&mut self, v: V) -> V { - mem::replace(self.get_mut(), v) - } - - pub fn remove(self) -> V { - self.remove_entry().1 - } - - pub fn replace_entry(self, v: V) -> (K, V) { - mem::replace(&mut self.items[self.idx], (self.k.unwrap(), v)) - } - - pub fn replace_key(self) -> K { - mem::replace(&mut self.items[self.idx].0, self.k.unwrap()) - } -} - -impl<'a, K, V> RustcVacantEntry<'a, K, V> { - pub fn key(&self) -> &K { - &self.k - } - - pub fn into_key(self) -> K { - self.k - } - - pub fn insert(self, v: V) -> &'a mut V { - self.items.push((self.k, v)); - &mut self.items.last_mut().unwrap().1 - } - - pub fn insert_entry(self, v: V) -> RustcOccupiedEntry<'a, K, V> { - let idx = self.items.len(); - self.items.push((self.k, v)); - RustcOccupiedEntry { - items: self.items, - idx, - k: None, - } - } -} diff --git a/crux-mir/lib/std/src/collections/hash/map.rs b/crux-mir/lib/std/src/collections/hash/map.rs index c606a5dfb..df4903588 100644 --- a/crux-mir/lib/std/src/collections/hash/map.rs +++ b/crux-mir/lib/std/src/collections/hash/map.rs @@ -1,20 +1,23 @@ -// ignore-tidy-filelength +#[cfg(test)] +mod tests; use self::Entry::*; -use super::crucible_map as base; +use hashbrown::hash_map as base; use crate::borrow::Borrow; use crate::cell::Cell; use crate::collections::TryReserveError; +use crate::collections::TryReserveErrorKind; +use crate::error::Error; use crate::fmt::{self, Debug}; #[allow(deprecated)] use crate::hash::{BuildHasher, Hash, Hasher, SipHasher13}; -use crate::iter::{FromIterator, FusedIterator}; +use crate::iter::FusedIterator; use crate::ops::Index; use crate::sys; -/// A hash map implemented with quadratic probing and SIMD lookup. +/// A [hash map] implemented with quadratic probing and SIMD lookup. /// /// By default, `HashMap` uses a hashing algorithm selected to provide /// resistance against HashDoS attacks. The algorithm is randomly seeded, and a @@ -33,8 +36,8 @@ use crate::sys; /// attacks such as HashDoS. /// /// The hashing algorithm can be replaced on a per-`HashMap` basis using the -/// [`default`], [`with_hasher`], and [`with_capacity_and_hasher`] methods. Many -/// alternative algorithms are available on crates.io, such as the [`fnv`] crate. +/// [`default`], [`with_hasher`], and [`with_capacity_and_hasher`] methods. +/// There are many alternative [hashing algorithms available on crates.io]. /// /// It is required that the keys implement the [`Eq`] and [`Hash`] traits, although /// this can frequently be achieved by using `#[derive(PartialEq, Eq, Hash)]`. @@ -51,11 +54,17 @@ use crate::sys; /// hash, as determined by the [`Hash`] trait, or its equality, as determined by /// the [`Eq`] trait, changes while it is in the map. This is normally only /// possible through [`Cell`], [`RefCell`], global state, I/O, or unsafe code. +/// The behavior resulting from such a logic error is not specified, but will +/// be encapsulated to the `HashMap` that observed the logic error and not +/// result in undefined behavior. This could include panics, incorrect results, +/// aborts, memory leaks, and non-termination. /// /// The hash table implementation is a Rust port of Google's [SwissTable]. /// The original C++ version of SwissTable can be found [here], and this /// [CppCon talk] gives an overview of how the algorithm works. /// +/// [hash map]: crate::collections#use-a-hashmap-when +/// [hashing algorithms available on crates.io]: https://crates.io/keywords/hasher /// [SwissTable]: https://abseil.io/blog/20180927-swisstables /// [here]: https://github.com/abseil/abseil-cpp/blob/master/absl/container/internal/raw_hash_set.h /// [CppCon talk]: https://www.youtube.com/watch?v=ncHmEUmJZf4 @@ -102,8 +111,8 @@ use crate::sys; /// let to_find = ["Pride and Prejudice", "Alice's Adventure in Wonderland"]; /// for &book in &to_find { /// match book_reviews.get(book) { -/// Some(review) => println!("{}: {}", book, review), -/// None => println!("{} is unreviewed.", book) +/// Some(review) => println!("{book}: {review}"), +/// None => println!("{book} is unreviewed.") /// } /// } /// @@ -112,12 +121,25 @@ use crate::sys; /// /// // Iterate over everything. /// for (book, review) in &book_reviews { -/// println!("{}: \"{}\"", book, review); +/// println!("{book}: \"{review}\""); /// } /// ``` /// -/// `HashMap` also implements an [`Entry API`](#method.entry), which allows -/// for more complex methods of getting, setting, updating and removing keys and +/// A `HashMap` with a known list of items can be initialized from an array: +/// +/// ``` +/// use std::collections::HashMap; +/// +/// let solar_distance = HashMap::from([ +/// ("Mercury", 0.4), +/// ("Venus", 0.7), +/// ("Earth", 1.0), +/// ("Mars", 1.5), +/// ]); +/// ``` +/// +/// `HashMap` implements an [`Entry` API](#method.entry), which allows +/// for complex methods of getting, setting, updating and removing keys and /// their values: /// /// ``` @@ -143,20 +165,19 @@ use crate::sys; /// // update a key, guarding against the key possibly not being set /// let stat = player_stats.entry("attack").or_insert(100); /// *stat += random_stat_buff(); +/// +/// // modify an entry before an insert with in-place mutation +/// player_stats.entry("mana").and_modify(|mana| *mana += 200).or_insert(100); /// ``` /// /// The easiest way to use `HashMap` with a custom key type is to derive [`Eq`] and [`Hash`]. /// We must also derive [`PartialEq`]. /// -/// [`Eq`]: ../../std/cmp/trait.Eq.html -/// [`Hash`]: ../../std/hash/trait.Hash.html -/// [`PartialEq`]: ../../std/cmp/trait.PartialEq.html -/// [`RefCell`]: ../../std/cell/struct.RefCell.html -/// [`Cell`]: ../../std/cell/struct.Cell.html -/// [`default`]: #method.default -/// [`with_hasher`]: #method.with_hasher -/// [`with_capacity_and_hasher`]: #method.with_capacity_and_hasher -/// [`fnv`]: https://crates.io/crates/fnv +/// [`RefCell`]: crate::cell::RefCell +/// [`Cell`]: crate::cell::Cell +/// [`default`]: Default::default +/// [`with_hasher`]: Self::with_hasher +/// [`with_capacity_and_hasher`]: Self::with_capacity_and_hasher /// /// ``` /// use std::collections::HashMap; @@ -175,30 +196,21 @@ use crate::sys; /// } /// /// // Use a HashMap to store the vikings' health points. -/// let mut vikings = HashMap::new(); -/// -/// vikings.insert(Viking::new("Einar", "Norway"), 25); -/// vikings.insert(Viking::new("Olaf", "Denmark"), 24); -/// vikings.insert(Viking::new("Harald", "Iceland"), 12); +/// let vikings = HashMap::from([ +/// (Viking::new("Einar", "Norway"), 25), +/// (Viking::new("Olaf", "Denmark"), 24), +/// (Viking::new("Harald", "Iceland"), 12), +/// ]); /// /// // Use derived implementation to print the status of the vikings. /// for (viking, health) in &vikings { -/// println!("{:?} has {} hp", viking, health); +/// println!("{viking:?} has {health} hp"); /// } /// ``` -/// -/// A `HashMap` with fixed list of elements can be initialized from an array: -/// -/// ``` -/// use std::collections::HashMap; -/// -/// let timber_resources: HashMap<&str, i32> = [("Norway", 100), ("Denmark", 50), ("Iceland", 10)] -/// .iter().cloned().collect(); -/// // use the values stored in map -/// ``` -#[derive(Clone)] +#[cfg_attr(not(test), rustc_diagnostic_item = "HashMap")] #[stable(feature = "rust1", since = "1.0.0")] +#[rustc_insignificant_dtor] pub struct HashMap { base: base::HashMap, } @@ -216,15 +228,17 @@ impl HashMap { /// let mut map: HashMap<&str, i32> = HashMap::new(); /// ``` #[inline] + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn new() -> HashMap { Default::default() } - /// Creates an empty `HashMap` with the specified capacity. + /// Creates an empty `HashMap` with at least the specified capacity. /// /// The hash map will be able to hold at least `capacity` elements without - /// reallocating. If `capacity` is 0, the hash map will not allocate. + /// reallocating. This method is allowed to allocate for more elements than + /// `capacity`. If `capacity` is 0, the hash set will not allocate. /// /// # Examples /// @@ -233,6 +247,7 @@ impl HashMap { /// let mut map: HashMap<&str, i32> = HashMap::with_capacity(10); /// ``` #[inline] + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn with_capacity(capacity: usize) -> HashMap { HashMap::with_capacity_and_hasher(capacity, Default::default()) @@ -250,6 +265,9 @@ impl HashMap { /// cause many collisions and very poor performance. Setting it /// manually using this function can expose a DoS attack vector. /// + /// The `hash_builder` passed should implement the [`BuildHasher`] trait for + /// the HashMap to be useful, see its documentation for details. + /// /// # Examples /// /// ``` @@ -262,21 +280,26 @@ impl HashMap { /// ``` #[inline] #[stable(feature = "hashmap_build_hasher", since = "1.7.0")] - pub fn with_hasher(hash_builder: S) -> HashMap { + #[rustc_const_unstable(feature = "const_collections_with_hasher", issue = "102575")] + pub const fn with_hasher(hash_builder: S) -> HashMap { HashMap { base: base::HashMap::with_hasher(hash_builder) } } - /// Creates an empty `HashMap` with the specified capacity, using `hash_builder` - /// to hash the keys. + /// Creates an empty `HashMap` with at least the specified capacity, using + /// `hasher` to hash the keys. /// /// The hash map will be able to hold at least `capacity` elements without - /// reallocating. If `capacity` is 0, the hash map will not allocate. + /// reallocating. This method is allowed to allocate for more elements than + /// `capacity`. If `capacity` is 0, the hash map will not allocate. /// - /// Warning: `hash_builder` is normally randomly generated, and + /// Warning: `hasher` is normally randomly generated, and /// is designed to allow HashMaps to be resistant to attacks that /// cause many collisions and very poor performance. Setting it /// manually using this function can expose a DoS attack vector. /// + /// The `hasher` passed should implement the [`BuildHasher`] trait for + /// the HashMap to be useful, see its documentation for details. + /// /// # Examples /// /// ``` @@ -289,8 +312,8 @@ impl HashMap { /// ``` #[inline] #[stable(feature = "hashmap_build_hasher", since = "1.7.0")] - pub fn with_capacity_and_hasher(capacity: usize, hash_builder: S) -> HashMap { - HashMap { base: base::HashMap::with_capacity_and_hasher(capacity, hash_builder) } + pub fn with_capacity_and_hasher(capacity: usize, hasher: S) -> HashMap { + HashMap { base: base::HashMap::with_capacity_and_hasher(capacity, hasher) } } /// Returns the number of elements the map can hold without reallocating. @@ -319,20 +342,59 @@ impl HashMap { /// ``` /// use std::collections::HashMap; /// - /// let mut map = HashMap::new(); - /// map.insert("a", 1); - /// map.insert("b", 2); - /// map.insert("c", 3); + /// let map = HashMap::from([ + /// ("a", 1), + /// ("b", 2), + /// ("c", 3), + /// ]); /// /// for key in map.keys() { - /// println!("{}", key); + /// println!("{key}"); /// } /// ``` + /// + /// # Performance + /// + /// In the current implementation, iterating over keys takes O(capacity) time + /// instead of O(len) because it internally visits empty buckets too. #[stable(feature = "rust1", since = "1.0.0")] pub fn keys(&self) -> Keys<'_, K, V> { Keys { inner: self.iter() } } + /// Creates a consuming iterator visiting all the keys in arbitrary order. + /// The map cannot be used after calling this. + /// The iterator element type is `K`. + /// + /// # Examples + /// + /// ``` + /// use std::collections::HashMap; + /// + /// let map = HashMap::from([ + /// ("a", 1), + /// ("b", 2), + /// ("c", 3), + /// ]); + /// + /// let mut vec: Vec<&str> = map.into_keys().collect(); + /// // The `IntoKeys` iterator produces keys in arbitrary order, so the + /// // keys must be sorted to test them against a sorted array. + /// vec.sort_unstable(); + /// assert_eq!(vec, ["a", "b", "c"]); + /// ``` + /// + /// # Performance + /// + /// In the current implementation, iterating over keys takes O(capacity) time + /// instead of O(len) because it internally visits empty buckets too. + #[inline] + #[rustc_lint_query_instability] + #[stable(feature = "map_into_keys_values", since = "1.54.0")] + pub fn into_keys(self) -> IntoKeys { + IntoKeys { inner: self.into_iter() } + } + /// An iterator visiting all values in arbitrary order. /// The iterator element type is `&'a V`. /// @@ -341,15 +403,21 @@ impl HashMap { /// ``` /// use std::collections::HashMap; /// - /// let mut map = HashMap::new(); - /// map.insert("a", 1); - /// map.insert("b", 2); - /// map.insert("c", 3); + /// let map = HashMap::from([ + /// ("a", 1), + /// ("b", 2), + /// ("c", 3), + /// ]); /// /// for val in map.values() { - /// println!("{}", val); + /// println!("{val}"); /// } /// ``` + /// + /// # Performance + /// + /// In the current implementation, iterating over values takes O(capacity) time + /// instead of O(len) because it internally visits empty buckets too. #[stable(feature = "rust1", since = "1.0.0")] pub fn values(&self) -> Values<'_, K, V> { Values { inner: self.iter() } @@ -363,25 +431,63 @@ impl HashMap { /// ``` /// use std::collections::HashMap; /// - /// let mut map = HashMap::new(); - /// - /// map.insert("a", 1); - /// map.insert("b", 2); - /// map.insert("c", 3); + /// let mut map = HashMap::from([ + /// ("a", 1), + /// ("b", 2), + /// ("c", 3), + /// ]); /// /// for val in map.values_mut() { /// *val = *val + 10; /// } /// /// for val in map.values() { - /// println!("{}", val); + /// println!("{val}"); /// } /// ``` + /// + /// # Performance + /// + /// In the current implementation, iterating over values takes O(capacity) time + /// instead of O(len) because it internally visits empty buckets too. #[stable(feature = "map_values_mut", since = "1.10.0")] pub fn values_mut(&mut self) -> ValuesMut<'_, K, V> { ValuesMut { inner: self.iter_mut() } } + /// Creates a consuming iterator visiting all the values in arbitrary order. + /// The map cannot be used after calling this. + /// The iterator element type is `V`. + /// + /// # Examples + /// + /// ``` + /// use std::collections::HashMap; + /// + /// let map = HashMap::from([ + /// ("a", 1), + /// ("b", 2), + /// ("c", 3), + /// ]); + /// + /// let mut vec: Vec = map.into_values().collect(); + /// // The `IntoValues` iterator produces values in arbitrary order, so + /// // the values must be sorted to test them against a sorted array. + /// vec.sort_unstable(); + /// assert_eq!(vec, [1, 2, 3]); + /// ``` + /// + /// # Performance + /// + /// In the current implementation, iterating over values takes O(capacity) time + /// instead of O(len) because it internally visits empty buckets too. + #[inline] + #[rustc_lint_query_instability] + #[stable(feature = "map_into_keys_values", since = "1.54.0")] + pub fn into_values(self) -> IntoValues { + IntoValues { inner: self.into_iter() } + } + /// An iterator visiting all key-value pairs in arbitrary order. /// The iterator element type is `(&'a K, &'a V)`. /// @@ -390,15 +496,22 @@ impl HashMap { /// ``` /// use std::collections::HashMap; /// - /// let mut map = HashMap::new(); - /// map.insert("a", 1); - /// map.insert("b", 2); - /// map.insert("c", 3); + /// let map = HashMap::from([ + /// ("a", 1), + /// ("b", 2), + /// ("c", 3), + /// ]); /// /// for (key, val) in map.iter() { - /// println!("key: {} val: {}", key, val); + /// println!("key: {key} val: {val}"); /// } /// ``` + /// + /// # Performance + /// + /// In the current implementation, iterating over map takes O(capacity) time + /// instead of O(len) because it internally visits empty buckets too. + #[rustc_lint_query_instability] #[stable(feature = "rust1", since = "1.0.0")] pub fn iter(&self) -> Iter<'_, K, V> { Iter { base: self.base.iter() } @@ -413,10 +526,11 @@ impl HashMap { /// ``` /// use std::collections::HashMap; /// - /// let mut map = HashMap::new(); - /// map.insert("a", 1); - /// map.insert("b", 2); - /// map.insert("c", 3); + /// let mut map = HashMap::from([ + /// ("a", 1), + /// ("b", 2), + /// ("c", 3), + /// ]); /// /// // Update all values /// for (_, val) in map.iter_mut() { @@ -424,9 +538,15 @@ impl HashMap { /// } /// /// for (key, val) in &map { - /// println!("key: {} val: {}", key, val); + /// println!("key: {key} val: {val}"); /// } /// ``` + /// + /// # Performance + /// + /// In the current implementation, iterating over map takes O(capacity) time + /// instead of O(len) because it internally visits empty buckets too. + #[rustc_lint_query_instability] #[stable(feature = "rust1", since = "1.0.0")] pub fn iter_mut(&mut self) -> IterMut<'_, K, V> { IterMut { base: self.base.iter_mut() } @@ -470,6 +590,10 @@ impl HashMap { /// Clears the map, returning all key-value pairs as an iterator. Keeps the /// allocated memory for reuse. /// + /// If the returned iterator is dropped before being fully consumed, it + /// drops the remaining key-value pairs. The returned iterator keeps a + /// mutable borrow on the map to optimize its implementation. + /// /// # Examples /// /// ``` @@ -487,11 +611,86 @@ impl HashMap { /// assert!(a.is_empty()); /// ``` #[inline] + #[rustc_lint_query_instability] #[stable(feature = "drain", since = "1.6.0")] pub fn drain(&mut self) -> Drain<'_, K, V> { Drain { base: self.base.drain() } } + /// Creates an iterator which uses a closure to determine if an element should be removed. + /// + /// If the closure returns true, the element is removed from the map and yielded. + /// If the closure returns false, or panics, the element remains in the map and will not be + /// yielded. + /// + /// Note that `drain_filter` lets you mutate every value in the filter closure, regardless of + /// whether you choose to keep or remove it. + /// + /// If the iterator is only partially consumed or not consumed at all, each of the remaining + /// elements will still be subjected to the closure and removed and dropped if it returns true. + /// + /// It is unspecified how many more elements will be subjected to the closure + /// if a panic occurs in the closure, or a panic occurs while dropping an element, + /// or if the `DrainFilter` value is leaked. + /// + /// # Examples + /// + /// Splitting a map into even and odd keys, reusing the original map: + /// + /// ``` + /// #![feature(hash_drain_filter)] + /// use std::collections::HashMap; + /// + /// let mut map: HashMap = (0..8).map(|x| (x, x)).collect(); + /// let drained: HashMap = map.drain_filter(|k, _v| k % 2 == 0).collect(); + /// + /// let mut evens = drained.keys().copied().collect::>(); + /// let mut odds = map.keys().copied().collect::>(); + /// evens.sort(); + /// odds.sort(); + /// + /// assert_eq!(evens, vec![0, 2, 4, 6]); + /// assert_eq!(odds, vec![1, 3, 5, 7]); + /// ``` + #[inline] + #[rustc_lint_query_instability] + #[unstable(feature = "hash_drain_filter", issue = "59618")] + pub fn drain_filter(&mut self, pred: F) -> DrainFilter<'_, K, V, F> + where + F: FnMut(&K, &mut V) -> bool, + { + DrainFilter { base: self.base.drain_filter(pred) } + } + + /// Retains only the elements specified by the predicate. + /// + /// In other words, remove all pairs `(k, v)` for which `f(&k, &mut v)` returns `false`. + /// The elements are visited in unsorted (and unspecified) order. + /// + /// # Examples + /// + /// ``` + /// use std::collections::HashMap; + /// + /// let mut map: HashMap = (0..8).map(|x| (x, x*10)).collect(); + /// map.retain(|&k, _| k % 2 == 0); + /// assert_eq!(map.len(), 4); + /// ``` + /// + /// # Performance + /// + /// In the current implementation, this operation takes O(capacity) time + /// instead of O(len) because it internally visits empty buckets too. + #[inline] + #[rustc_lint_query_instability] + #[stable(feature = "retain_hash_collection", since = "1.18.0")] + pub fn retain(&mut self, f: F) + where + F: FnMut(&K, &mut V) -> bool, + { + self.base.retain(f) + } + /// Clears the map, removing all key-value pairs. Keeps the allocated memory /// for reuse. /// @@ -505,16 +704,14 @@ impl HashMap { /// a.clear(); /// assert!(a.is_empty()); /// ``` - #[stable(feature = "rust1", since = "1.0.0")] #[inline] + #[stable(feature = "rust1", since = "1.0.0")] pub fn clear(&mut self) { self.base.clear(); } /// Returns a reference to the map's [`BuildHasher`]. /// - /// [`BuildHasher`]: ../../std/hash/trait.BuildHasher.html - /// /// # Examples /// /// ``` @@ -538,15 +735,15 @@ where S: BuildHasher, { /// Reserves capacity for at least `additional` more elements to be inserted - /// in the `HashMap`. The collection may reserve more space to avoid - /// frequent reallocations. + /// in the `HashMap`. The collection may reserve more space to speculatively + /// avoid frequent reallocations. After calling `reserve`, + /// capacity will be greater than or equal to `self.len() + additional`. + /// Does nothing if capacity is already sufficient. /// /// # Panics /// /// Panics if the new allocation size overflows [`usize`]. /// - /// [`usize`]: ../../std/primitive.usize.html - /// /// # Examples /// /// ``` @@ -561,8 +758,11 @@ where } /// Tries to reserve capacity for at least `additional` more elements to be inserted - /// in the given `HashMap`. The collection may reserve more space to avoid - /// frequent reallocations. + /// in the `HashMap`. The collection may reserve more space to speculatively + /// avoid frequent reallocations. After calling `try_reserve`, + /// capacity will be greater than or equal to `self.len() + additional` if + /// it returns `Ok(())`. + /// Does nothing if capacity is already sufficient. /// /// # Errors /// @@ -572,15 +772,15 @@ where /// # Examples /// /// ``` - /// #![feature(try_reserve)] /// use std::collections::HashMap; + /// /// let mut map: HashMap<&str, isize> = HashMap::new(); - /// map.try_reserve(10).expect("why is the test harness OOMing on 10 bytes?"); + /// map.try_reserve(10).expect("why is the test harness OOMing on a handful of bytes?"); /// ``` #[inline] - #[unstable(feature = "try_reserve", reason = "new API", issue = "48043")] + #[stable(feature = "try_reserve", since = "1.57.0")] pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { - self.base.try_reserve(additional).map_err(map_collection_alloc_err) + self.base.try_reserve(additional).map_err(map_try_reserve_error) } /// Shrinks the capacity of the map as much as possible. It will drop @@ -609,13 +809,11 @@ where /// down no lower than the supplied limit while maintaining the internal rules /// and possibly leaving some space in accordance with the resize policy. /// - /// Panics if the current capacity is smaller than the supplied - /// minimum capacity. + /// If the current capacity is less than the lower limit, this is a no-op. /// /// # Examples /// /// ``` - /// #![feature(shrink_to)] /// use std::collections::HashMap; /// /// let mut map: HashMap = HashMap::with_capacity(100); @@ -628,9 +826,8 @@ where /// assert!(map.capacity() >= 2); /// ``` #[inline] - #[unstable(feature = "shrink_to", reason = "new API", issue = "56431")] + #[stable(feature = "shrink_to", since = "1.56.0")] pub fn shrink_to(&mut self, min_capacity: usize) { - assert!(self.capacity() >= min_capacity, "Tried to shrink to a larger capacity"); self.base.shrink_to(min_capacity); } @@ -644,8 +841,7 @@ where /// let mut letters = HashMap::new(); /// /// for ch in "a short treatise on fungi".chars() { - /// let counter = letters.entry(ch).or_insert(0); - /// *counter += 1; + /// letters.entry(ch).and_modify(|counter| *counter += 1).or_insert(1); /// } /// /// assert_eq!(letters[&'s'], 2); @@ -665,9 +861,6 @@ where /// [`Hash`] and [`Eq`] on the borrowed form *must* match those for /// the key type. /// - /// [`Eq`]: ../../std/cmp/trait.Eq.html - /// [`Hash`]: ../../std/hash/trait.Hash.html - /// /// # Examples /// /// ``` @@ -694,9 +887,6 @@ where /// [`Hash`] and [`Eq`] on the borrowed form *must* match those for /// the key type. /// - /// [`Eq`]: ../../std/cmp/trait.Eq.html - /// [`Hash`]: ../../std/hash/trait.Hash.html - /// /// # Examples /// /// ``` @@ -707,8 +897,8 @@ where /// assert_eq!(map.get_key_value(&1), Some((&1, &"a"))); /// assert_eq!(map.get_key_value(&2), None); /// ``` - #[stable(feature = "map_get_key_value", since = "1.40.0")] #[inline] + #[stable(feature = "map_get_key_value", since = "1.40.0")] pub fn get_key_value(&self, k: &Q) -> Option<(&K, &V)> where K: Borrow, @@ -717,15 +907,125 @@ where self.base.get_key_value(k) } + /// Attempts to get mutable references to `N` values in the map at once. + /// + /// Returns an array of length `N` with the results of each query. For soundness, at most one + /// mutable reference will be returned to any value. `None` will be returned if any of the + /// keys are duplicates or missing. + /// + /// # Examples + /// + /// ``` + /// #![feature(map_many_mut)] + /// use std::collections::HashMap; + /// + /// let mut libraries = HashMap::new(); + /// libraries.insert("Bodleian Library".to_string(), 1602); + /// libraries.insert("Athenæum".to_string(), 1807); + /// libraries.insert("Herzogin-Anna-Amalia-Bibliothek".to_string(), 1691); + /// libraries.insert("Library of Congress".to_string(), 1800); + /// + /// let got = libraries.get_many_mut([ + /// "Athenæum", + /// "Library of Congress", + /// ]); + /// assert_eq!( + /// got, + /// Some([ + /// &mut 1807, + /// &mut 1800, + /// ]), + /// ); + /// + /// // Missing keys result in None + /// let got = libraries.get_many_mut([ + /// "Athenæum", + /// "New York Public Library", + /// ]); + /// assert_eq!(got, None); + /// + /// // Duplicate keys result in None + /// let got = libraries.get_many_mut([ + /// "Athenæum", + /// "Athenæum", + /// ]); + /// assert_eq!(got, None); + /// ``` + #[inline] + #[unstable(feature = "map_many_mut", issue = "97601")] + pub fn get_many_mut(&mut self, ks: [&Q; N]) -> Option<[&'_ mut V; N]> + where + K: Borrow, + Q: Hash + Eq, + { + self.base.get_many_mut(ks) + } + + /// Attempts to get mutable references to `N` values in the map at once, without validating that + /// the values are unique. + /// + /// Returns an array of length `N` with the results of each query. `None` will be returned if + /// any of the keys are missing. + /// + /// For a safe alternative see [`get_many_mut`](Self::get_many_mut). + /// + /// # Safety + /// + /// Calling this method with overlapping keys is *[undefined behavior]* even if the resulting + /// references are not used. + /// + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html + /// + /// # Examples + /// + /// ``` + /// #![feature(map_many_mut)] + /// use std::collections::HashMap; + /// + /// let mut libraries = HashMap::new(); + /// libraries.insert("Bodleian Library".to_string(), 1602); + /// libraries.insert("Athenæum".to_string(), 1807); + /// libraries.insert("Herzogin-Anna-Amalia-Bibliothek".to_string(), 1691); + /// libraries.insert("Library of Congress".to_string(), 1800); + /// + /// let got = libraries.get_many_mut([ + /// "Athenæum", + /// "Library of Congress", + /// ]); + /// assert_eq!( + /// got, + /// Some([ + /// &mut 1807, + /// &mut 1800, + /// ]), + /// ); + /// + /// // Missing keys result in None + /// let got = libraries.get_many_mut([ + /// "Athenæum", + /// "New York Public Library", + /// ]); + /// assert_eq!(got, None); + /// ``` + #[inline] + #[unstable(feature = "map_many_mut", issue = "97601")] + pub unsafe fn get_many_unchecked_mut( + &mut self, + ks: [&Q; N], + ) -> Option<[&'_ mut V; N]> + where + K: Borrow, + Q: Hash + Eq, + { + self.base.get_many_unchecked_mut(ks) + } + /// Returns `true` if the map contains a value for the specified key. /// /// The key may be any borrowed form of the map's key type, but /// [`Hash`] and [`Eq`] on the borrowed form *must* match those for /// the key type. /// - /// [`Eq`]: ../../std/cmp/trait.Eq.html - /// [`Hash`]: ../../std/hash/trait.Hash.html - /// /// # Examples /// /// ``` @@ -736,8 +1036,8 @@ where /// assert_eq!(map.contains_key(&1), true); /// assert_eq!(map.contains_key(&2), false); /// ``` - #[stable(feature = "rust1", since = "1.0.0")] #[inline] + #[stable(feature = "rust1", since = "1.0.0")] pub fn contains_key(&self, k: &Q) -> bool where K: Borrow, @@ -752,9 +1052,6 @@ where /// [`Hash`] and [`Eq`] on the borrowed form *must* match those for /// the key type. /// - /// [`Eq`]: ../../std/cmp/trait.Eq.html - /// [`Hash`]: ../../std/hash/trait.Hash.html - /// /// # Examples /// /// ``` @@ -767,8 +1064,8 @@ where /// } /// assert_eq!(map[&1], "b"); /// ``` - #[stable(feature = "rust1", since = "1.0.0")] #[inline] + #[stable(feature = "rust1", since = "1.0.0")] pub fn get_mut(&mut self, k: &Q) -> Option<&mut V> where K: Borrow, @@ -786,8 +1083,7 @@ where /// types that can be `==` without being identical. See the [module-level /// documentation] for more. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// [module-level documentation]: index.html#insert-and-complex-keys + /// [module-level documentation]: crate::collections#insert-and-complex-keys /// /// # Examples /// @@ -802,12 +1098,43 @@ where /// assert_eq!(map.insert(37, "c"), Some("b")); /// assert_eq!(map[&37], "c"); /// ``` - #[stable(feature = "rust1", since = "1.0.0")] #[inline] + #[stable(feature = "rust1", since = "1.0.0")] pub fn insert(&mut self, k: K, v: V) -> Option { self.base.insert(k, v) } + /// Tries to insert a key-value pair into the map, and returns + /// a mutable reference to the value in the entry. + /// + /// If the map already had this key present, nothing is updated, and + /// an error containing the occupied entry and the value is returned. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(map_try_insert)] + /// + /// use std::collections::HashMap; + /// + /// let mut map = HashMap::new(); + /// assert_eq!(map.try_insert(37, "a").unwrap(), &"a"); + /// + /// let err = map.try_insert(37, "b").unwrap_err(); + /// assert_eq!(err.entry.key(), &37); + /// assert_eq!(err.entry.get(), &"a"); + /// assert_eq!(err.value, "b"); + /// ``` + #[unstable(feature = "map_try_insert", issue = "82766")] + pub fn try_insert(&mut self, key: K, value: V) -> Result<&mut V, OccupiedError<'_, K, V>> { + match self.entry(key) { + Occupied(entry) => Err(OccupiedError { entry, value }), + Vacant(entry) => Ok(entry.insert(value)), + } + } + /// Removes a key from the map, returning the value at the key if the key /// was previously in the map. /// @@ -815,9 +1142,6 @@ where /// [`Hash`] and [`Eq`] on the borrowed form *must* match those for /// the key type. /// - /// [`Eq`]: ../../std/cmp/trait.Eq.html - /// [`Hash`]: ../../std/hash/trait.Hash.html - /// /// # Examples /// /// ``` @@ -828,8 +1152,8 @@ where /// assert_eq!(map.remove(&1), Some("a")); /// assert_eq!(map.remove(&1), None); /// ``` - #[stable(feature = "rust1", since = "1.0.0")] #[inline] + #[stable(feature = "rust1", since = "1.0.0")] pub fn remove(&mut self, k: &Q) -> Option where K: Borrow, @@ -845,9 +1169,6 @@ where /// [`Hash`] and [`Eq`] on the borrowed form *must* match those for /// the key type. /// - /// [`Eq`]: ../../std/cmp/trait.Eq.html - /// [`Hash`]: ../../std/hash/trait.Hash.html - /// /// # Examples /// /// ``` @@ -860,8 +1181,8 @@ where /// assert_eq!(map.remove(&1), None); /// # } /// ``` - #[stable(feature = "hash_map_remove_entry", since = "1.27.0")] #[inline] + #[stable(feature = "hash_map_remove_entry", since = "1.27.0")] pub fn remove_entry(&mut self, k: &Q) -> Option<(K, V)> where K: Borrow, @@ -869,27 +1190,86 @@ where { self.base.remove_entry(k) } +} - /// Retains only the elements specified by the predicate. +impl HashMap +where + S: BuildHasher, +{ + /// Creates a raw entry builder for the HashMap. /// - /// In other words, remove all pairs `(k, v)` such that `f(&k,&mut v)` returns `false`. + /// Raw entries provide the lowest level of control for searching and + /// manipulating a map. They must be manually initialized with a hash and + /// then manually searched. After this, insertions into a vacant entry + /// still require an owned key to be provided. /// - /// # Examples + /// Raw entries are useful for such exotic situations as: /// - /// ``` - /// use std::collections::HashMap; + /// * Hash memoization + /// * Deferring the creation of an owned key until it is known to be required + /// * Using a search key that doesn't work with the Borrow trait + /// * Using custom comparison logic without newtype wrappers /// - /// let mut map: HashMap = (0..8).map(|x|(x, x*10)).collect(); - /// map.retain(|&k, _| k % 2 == 0); - /// assert_eq!(map.len(), 4); - /// ``` - #[stable(feature = "retain_hash_collection", since = "1.18.0")] + /// Because raw entries provide much more low-level control, it's much easier + /// to put the HashMap into an inconsistent state which, while memory-safe, + /// will cause the map to produce seemingly random results. Higher-level and + /// more foolproof APIs like `entry` should be preferred when possible. + /// + /// In particular, the hash used to initialized the raw entry must still be + /// consistent with the hash of the key that is ultimately stored in the entry. + /// This is because implementations of HashMap may need to recompute hashes + /// when resizing, at which point only the keys are available. + /// + /// Raw entries give mutable access to the keys. This must not be used + /// to modify how the key would compare or hash, as the map will not re-evaluate + /// where the key should go, meaning the keys may become "lost" if their + /// location does not reflect their state. For instance, if you change a key + /// so that the map now contains keys which compare equal, search may start + /// acting erratically, with two keys randomly masking each other. Implementations + /// are free to assume this doesn't happen (within the limits of memory-safety). #[inline] - pub fn retain(&mut self, f: F) - where - F: FnMut(&K, &mut V) -> bool, - { - self.base.retain(f) + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn raw_entry_mut(&mut self) -> RawEntryBuilderMut<'_, K, V, S> { + RawEntryBuilderMut { map: self } + } + + /// Creates a raw immutable entry builder for the HashMap. + /// + /// Raw entries provide the lowest level of control for searching and + /// manipulating a map. They must be manually initialized with a hash and + /// then manually searched. + /// + /// This is useful for + /// * Hash memoization + /// * Using a search key that doesn't work with the Borrow trait + /// * Using custom comparison logic without newtype wrappers + /// + /// Unless you are in such a situation, higher-level and more foolproof APIs like + /// `get` should be preferred. + /// + /// Immutable raw entries have very limited use; you might instead want `raw_entry_mut`. + #[inline] + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn raw_entry(&self) -> RawEntryBuilder<'_, K, V, S> { + RawEntryBuilder { map: self } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Clone for HashMap +where + K: Clone, + V: Clone, + S: Clone, +{ + #[inline] + fn clone(&self) -> Self { + Self { base: self.base.clone() } + } + + #[inline] + fn clone_from(&mut self, other: &Self) { + self.base.clone_from(&other.base); } } @@ -961,13 +1341,54 @@ where } } +#[stable(feature = "std_collections_from_array", since = "1.56.0")] +// Note: as what is currently the most convenient built-in way to construct +// a HashMap, a simple usage of this function must not *require* the user +// to provide a type annotation in order to infer the third type parameter +// (the hasher parameter, conventionally "S"). +// To that end, this impl is defined using RandomState as the concrete +// type of S, rather than being generic over `S: BuildHasher + Default`. +// It is expected that users who want to specify a hasher will manually use +// `with_capacity_and_hasher`. +// If type parameter defaults worked on impls, and if type parameter +// defaults could be mixed with const generics, then perhaps +// this could be generalized. +// See also the equivalent impl on HashSet. +impl From<[(K, V); N]> for HashMap +where + K: Eq + Hash, +{ + /// # Examples + /// + /// ``` + /// use std::collections::HashMap; + /// + /// let map1 = HashMap::from([(1, 2), (3, 4)]); + /// let map2: HashMap<_, _> = [(1, 2), (3, 4)].into(); + /// assert_eq!(map1, map2); + /// ``` + fn from(arr: [(K, V); N]) -> Self { + Self::from_iter(arr) + } +} + /// An iterator over the entries of a `HashMap`. /// /// This `struct` is created by the [`iter`] method on [`HashMap`]. See its /// documentation for more. /// -/// [`iter`]: struct.HashMap.html#method.iter -/// [`HashMap`]: struct.HashMap.html +/// [`iter`]: HashMap::iter +/// +/// # Example +/// +/// ``` +/// use std::collections::HashMap; +/// +/// let map = HashMap::from([ +/// ("a", 1), +/// ]); +/// let iter = map.iter(); +/// ``` #[stable(feature = "rust1", since = "1.0.0")] pub struct Iter<'a, K: 'a, V: 'a> { base: base::Iter<'a, K, V>, @@ -994,15 +1415,25 @@ impl fmt::Debug for Iter<'_, K, V> { /// This `struct` is created by the [`iter_mut`] method on [`HashMap`]. See its /// documentation for more. /// -/// [`iter_mut`]: struct.HashMap.html#method.iter_mut -/// [`HashMap`]: struct.HashMap.html +/// [`iter_mut`]: HashMap::iter_mut +/// +/// # Example +/// +/// ``` +/// use std::collections::HashMap; +/// +/// let mut map = HashMap::from([ +/// ("a", 1), +/// ]); +/// let iter = map.iter_mut(); +/// ``` #[stable(feature = "rust1", since = "1.0.0")] pub struct IterMut<'a, K: 'a, V: 'a> { base: base::IterMut<'a, K, V>, } impl<'a, K, V> IterMut<'a, K, V> { - /// Returns a iterator of references over the remaining items. + /// Returns an iterator of references over the remaining items. #[inline] pub(super) fn iter(&self) -> Iter<'_, K, V> { Iter { base: self.base.rustc_iter() } @@ -1012,17 +1443,28 @@ impl<'a, K, V> IterMut<'a, K, V> { /// An owning iterator over the entries of a `HashMap`. /// /// This `struct` is created by the [`into_iter`] method on [`HashMap`] -/// (provided by the `IntoIterator` trait). See its documentation for more. +/// (provided by the [`IntoIterator`] trait). See its documentation for more. +/// +/// [`into_iter`]: IntoIterator::into_iter +/// [`IntoIterator`]: crate::iter::IntoIterator +/// +/// # Example +/// +/// ``` +/// use std::collections::HashMap; /// -/// [`into_iter`]: struct.HashMap.html#method.into_iter -/// [`HashMap`]: struct.HashMap.html +/// let map = HashMap::from([ +/// ("a", 1), +/// ]); +/// let iter = map.into_iter(); +/// ``` #[stable(feature = "rust1", since = "1.0.0")] pub struct IntoIter { base: base::IntoIter, } impl IntoIter { - /// Returns a iterator of references over the remaining items. + /// Returns an iterator of references over the remaining items. #[inline] pub(super) fn iter(&self) -> Iter<'_, K, V> { Iter { base: self.base.rustc_iter() } @@ -1034,8 +1476,18 @@ impl IntoIter { /// This `struct` is created by the [`keys`] method on [`HashMap`]. See its /// documentation for more. /// -/// [`keys`]: struct.HashMap.html#method.keys -/// [`HashMap`]: struct.HashMap.html +/// [`keys`]: HashMap::keys +/// +/// # Example +/// +/// ``` +/// use std::collections::HashMap; +/// +/// let map = HashMap::from([ +/// ("a", 1), +/// ]); +/// let iter_keys = map.keys(); +/// ``` #[stable(feature = "rust1", since = "1.0.0")] pub struct Keys<'a, K: 'a, V: 'a> { inner: Iter<'a, K, V>, @@ -1062,8 +1514,18 @@ impl fmt::Debug for Keys<'_, K, V> { /// This `struct` is created by the [`values`] method on [`HashMap`]. See its /// documentation for more. /// -/// [`values`]: struct.HashMap.html#method.values -/// [`HashMap`]: struct.HashMap.html +/// [`values`]: HashMap::values +/// +/// # Example +/// +/// ``` +/// use std::collections::HashMap; +/// +/// let map = HashMap::from([ +/// ("a", 1), +/// ]); +/// let iter_values = map.values(); +/// ``` #[stable(feature = "rust1", since = "1.0.0")] pub struct Values<'a, K: 'a, V: 'a> { inner: Iter<'a, K, V>, @@ -1090,191 +1552,725 @@ impl fmt::Debug for Values<'_, K, V> { /// This `struct` is created by the [`drain`] method on [`HashMap`]. See its /// documentation for more. /// -/// [`drain`]: struct.HashMap.html#method.drain -/// [`HashMap`]: struct.HashMap.html -#[stable(feature = "drain", since = "1.6.0")] -pub struct Drain<'a, K: 'a, V: 'a> { - base: base::Drain<'a, K, V>, -} - +/// [`drain`]: HashMap::drain +/// +/// # Example +/// +/// ``` +/// use std::collections::HashMap; +/// +/// let mut map = HashMap::from([ +/// ("a", 1), +/// ]); +/// let iter = map.drain(); +/// ``` +#[stable(feature = "drain", since = "1.6.0")] +pub struct Drain<'a, K: 'a, V: 'a> { + base: base::Drain<'a, K, V>, +} + impl<'a, K, V> Drain<'a, K, V> { - /// Returns a iterator of references over the remaining items. + /// Returns an iterator of references over the remaining items. #[inline] pub(super) fn iter(&self) -> Iter<'_, K, V> { Iter { base: self.base.rustc_iter() } } } +/// A draining, filtering iterator over the entries of a `HashMap`. +/// +/// This `struct` is created by the [`drain_filter`] method on [`HashMap`]. +/// +/// [`drain_filter`]: HashMap::drain_filter +/// +/// # Example +/// +/// ``` +/// #![feature(hash_drain_filter)] +/// +/// use std::collections::HashMap; +/// +/// let mut map = HashMap::from([ +/// ("a", 1), +/// ]); +/// let iter = map.drain_filter(|_k, v| *v % 2 == 0); +/// ``` +#[unstable(feature = "hash_drain_filter", issue = "59618")] +pub struct DrainFilter<'a, K, V, F> +where + F: FnMut(&K, &mut V) -> bool, +{ + base: base::DrainFilter<'a, K, V, F>, +} + /// A mutable iterator over the values of a `HashMap`. /// /// This `struct` is created by the [`values_mut`] method on [`HashMap`]. See its /// documentation for more. /// -/// [`values_mut`]: struct.HashMap.html#method.values_mut -/// [`HashMap`]: struct.HashMap.html +/// [`values_mut`]: HashMap::values_mut +/// +/// # Example +/// +/// ``` +/// use std::collections::HashMap; +/// +/// let mut map = HashMap::from([ +/// ("a", 1), +/// ]); +/// let iter_values = map.values_mut(); +/// ``` #[stable(feature = "map_values_mut", since = "1.10.0")] pub struct ValuesMut<'a, K: 'a, V: 'a> { inner: IterMut<'a, K, V>, } -/// A view into a single entry in a map, which may either be vacant or occupied. +/// An owning iterator over the keys of a `HashMap`. /// -/// This `enum` is constructed from the [`entry`] method on [`HashMap`]. +/// This `struct` is created by the [`into_keys`] method on [`HashMap`]. +/// See its documentation for more. /// -/// [`HashMap`]: struct.HashMap.html -/// [`entry`]: struct.HashMap.html#method.entry -#[stable(feature = "rust1", since = "1.0.0")] -pub enum Entry<'a, K: 'a, V: 'a> { - /// An occupied entry. - #[stable(feature = "rust1", since = "1.0.0")] - Occupied(#[stable(feature = "rust1", since = "1.0.0")] OccupiedEntry<'a, K, V>), +/// [`into_keys`]: HashMap::into_keys +/// +/// # Example +/// +/// ``` +/// use std::collections::HashMap; +/// +/// let map = HashMap::from([ +/// ("a", 1), +/// ]); +/// let iter_keys = map.into_keys(); +/// ``` +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +pub struct IntoKeys { + inner: IntoIter, +} - /// A vacant entry. - #[stable(feature = "rust1", since = "1.0.0")] - Vacant(#[stable(feature = "rust1", since = "1.0.0")] VacantEntry<'a, K, V>), +/// An owning iterator over the values of a `HashMap`. +/// +/// This `struct` is created by the [`into_values`] method on [`HashMap`]. +/// See its documentation for more. +/// +/// [`into_values`]: HashMap::into_values +/// +/// # Example +/// +/// ``` +/// use std::collections::HashMap; +/// +/// let map = HashMap::from([ +/// ("a", 1), +/// ]); +/// let iter_keys = map.into_values(); +/// ``` +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +pub struct IntoValues { + inner: IntoIter, } -#[stable(feature = "debug_hash_map", since = "1.12.0")] -impl Debug for Entry<'_, K, V> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match *self { - Vacant(ref v) => f.debug_tuple("Entry").field(v).finish(), - Occupied(ref o) => f.debug_tuple("Entry").field(o).finish(), - } - } +/// A builder for computing where in a HashMap a key-value pair would be stored. +/// +/// See the [`HashMap::raw_entry_mut`] docs for usage examples. +#[unstable(feature = "hash_raw_entry", issue = "56167")] +pub struct RawEntryBuilderMut<'a, K: 'a, V: 'a, S: 'a> { + map: &'a mut HashMap, } -/// A view into an occupied entry in a `HashMap`. -/// It is part of the [`Entry`] enum. +/// A view into a single entry in a map, which may either be vacant or occupied. /// -/// [`Entry`]: enum.Entry.html -#[stable(feature = "rust1", since = "1.0.0")] -pub struct OccupiedEntry<'a, K: 'a, V: 'a> { - base: base::RustcOccupiedEntry<'a, K, V>, +/// This is a lower-level version of [`Entry`]. +/// +/// This `enum` is constructed through the [`raw_entry_mut`] method on [`HashMap`], +/// then calling one of the methods of that [`RawEntryBuilderMut`]. +/// +/// [`raw_entry_mut`]: HashMap::raw_entry_mut +#[unstable(feature = "hash_raw_entry", issue = "56167")] +pub enum RawEntryMut<'a, K: 'a, V: 'a, S: 'a> { + /// An occupied entry. + Occupied(RawOccupiedEntryMut<'a, K, V, S>), + /// A vacant entry. + Vacant(RawVacantEntryMut<'a, K, V, S>), } -#[stable(feature = "debug_hash_map", since = "1.12.0")] -impl Debug for OccupiedEntry<'_, K, V> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("OccupiedEntry").field("key", self.key()).field("value", self.get()).finish() - } +/// A view into an occupied entry in a `HashMap`. +/// It is part of the [`RawEntryMut`] enum. +#[unstable(feature = "hash_raw_entry", issue = "56167")] +pub struct RawOccupiedEntryMut<'a, K: 'a, V: 'a, S: 'a> { + base: base::RawOccupiedEntryMut<'a, K, V, S>, } /// A view into a vacant entry in a `HashMap`. -/// It is part of the [`Entry`] enum. +/// It is part of the [`RawEntryMut`] enum. +#[unstable(feature = "hash_raw_entry", issue = "56167")] +pub struct RawVacantEntryMut<'a, K: 'a, V: 'a, S: 'a> { + base: base::RawVacantEntryMut<'a, K, V, S>, +} + +/// A builder for computing where in a HashMap a key-value pair would be stored. /// -/// [`Entry`]: enum.Entry.html -#[stable(feature = "rust1", since = "1.0.0")] -pub struct VacantEntry<'a, K: 'a, V: 'a> { - base: base::RustcVacantEntry<'a, K, V>, +/// See the [`HashMap::raw_entry`] docs for usage examples. +#[unstable(feature = "hash_raw_entry", issue = "56167")] +pub struct RawEntryBuilder<'a, K: 'a, V: 'a, S: 'a> { + map: &'a HashMap, } -#[stable(feature = "debug_hash_map", since = "1.12.0")] -impl Debug for VacantEntry<'_, K, V> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("VacantEntry").field(self.key()).finish() +impl<'a, K, V, S> RawEntryBuilderMut<'a, K, V, S> +where + S: BuildHasher, +{ + /// Creates a `RawEntryMut` from the given key. + #[inline] + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn from_key(self, k: &Q) -> RawEntryMut<'a, K, V, S> + where + K: Borrow, + Q: Hash + Eq, + { + map_raw_entry(self.map.base.raw_entry_mut().from_key(k)) } -} -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, K, V, S> IntoIterator for &'a HashMap { - type Item = (&'a K, &'a V); - type IntoIter = Iter<'a, K, V>; + /// Creates a `RawEntryMut` from the given key and its hash. + #[inline] + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn from_key_hashed_nocheck(self, hash: u64, k: &Q) -> RawEntryMut<'a, K, V, S> + where + K: Borrow, + Q: Eq, + { + map_raw_entry(self.map.base.raw_entry_mut().from_key_hashed_nocheck(hash, k)) + } + /// Creates a `RawEntryMut` from the given hash. #[inline] - fn into_iter(self) -> Iter<'a, K, V> { - self.iter() + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn from_hash(self, hash: u64, is_match: F) -> RawEntryMut<'a, K, V, S> + where + for<'b> F: FnMut(&'b K) -> bool, + { + map_raw_entry(self.map.base.raw_entry_mut().from_hash(hash, is_match)) } } -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, K, V, S> IntoIterator for &'a mut HashMap { - type Item = (&'a K, &'a mut V); - type IntoIter = IterMut<'a, K, V>; +impl<'a, K, V, S> RawEntryBuilder<'a, K, V, S> +where + S: BuildHasher, +{ + /// Access an entry by key. + #[inline] + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn from_key(self, k: &Q) -> Option<(&'a K, &'a V)> + where + K: Borrow, + Q: Hash + Eq, + { + self.map.base.raw_entry().from_key(k) + } + /// Access an entry by a key and its hash. #[inline] - fn into_iter(self) -> IterMut<'a, K, V> { - self.iter_mut() + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn from_key_hashed_nocheck(self, hash: u64, k: &Q) -> Option<(&'a K, &'a V)> + where + K: Borrow, + Q: Hash + Eq, + { + self.map.base.raw_entry().from_key_hashed_nocheck(hash, k) + } + + /// Access an entry by hash. + #[inline] + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn from_hash(self, hash: u64, is_match: F) -> Option<(&'a K, &'a V)> + where + F: FnMut(&K) -> bool, + { + self.map.base.raw_entry().from_hash(hash, is_match) } } -#[stable(feature = "rust1", since = "1.0.0")] -impl IntoIterator for HashMap { - type Item = (K, V); - type IntoIter = IntoIter; +impl<'a, K, V, S> RawEntryMut<'a, K, V, S> { + /// Ensures a value is in the entry by inserting the default if empty, and returns + /// mutable references to the key and value in the entry. + /// + /// # Examples + /// + /// ``` + /// #![feature(hash_raw_entry)] + /// use std::collections::HashMap; + /// + /// let mut map: HashMap<&str, u32> = HashMap::new(); + /// + /// map.raw_entry_mut().from_key("poneyland").or_insert("poneyland", 3); + /// assert_eq!(map["poneyland"], 3); + /// + /// *map.raw_entry_mut().from_key("poneyland").or_insert("poneyland", 10).1 *= 2; + /// assert_eq!(map["poneyland"], 6); + /// ``` + #[inline] + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn or_insert(self, default_key: K, default_val: V) -> (&'a mut K, &'a mut V) + where + K: Hash, + S: BuildHasher, + { + match self { + RawEntryMut::Occupied(entry) => entry.into_key_value(), + RawEntryMut::Vacant(entry) => entry.insert(default_key, default_val), + } + } - /// Creates a consuming iterator, that is, one that moves each key-value - /// pair out of the map in arbitrary order. The map cannot be used after - /// calling this. + /// Ensures a value is in the entry by inserting the result of the default function if empty, + /// and returns mutable references to the key and value in the entry. /// /// # Examples /// /// ``` + /// #![feature(hash_raw_entry)] /// use std::collections::HashMap; /// - /// let mut map = HashMap::new(); - /// map.insert("a", 1); - /// map.insert("b", 2); - /// map.insert("c", 3); + /// let mut map: HashMap<&str, String> = HashMap::new(); /// - /// // Not possible with .iter() - /// let vec: Vec<(&str, i32)> = map.into_iter().collect(); + /// map.raw_entry_mut().from_key("poneyland").or_insert_with(|| { + /// ("poneyland", "hoho".to_string()) + /// }); + /// + /// assert_eq!(map["poneyland"], "hoho".to_string()); /// ``` #[inline] - fn into_iter(self) -> IntoIter { - IntoIter { base: self.base.into_iter() } + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn or_insert_with(self, default: F) -> (&'a mut K, &'a mut V) + where + F: FnOnce() -> (K, V), + K: Hash, + S: BuildHasher, + { + match self { + RawEntryMut::Occupied(entry) => entry.into_key_value(), + RawEntryMut::Vacant(entry) => { + let (k, v) = default(); + entry.insert(k, v) + } + } + } + + /// Provides in-place mutable access to an occupied entry before any + /// potential inserts into the map. + /// + /// # Examples + /// + /// ``` + /// #![feature(hash_raw_entry)] + /// use std::collections::HashMap; + /// + /// let mut map: HashMap<&str, u32> = HashMap::new(); + /// + /// map.raw_entry_mut() + /// .from_key("poneyland") + /// .and_modify(|_k, v| { *v += 1 }) + /// .or_insert("poneyland", 42); + /// assert_eq!(map["poneyland"], 42); + /// + /// map.raw_entry_mut() + /// .from_key("poneyland") + /// .and_modify(|_k, v| { *v += 1 }) + /// .or_insert("poneyland", 0); + /// assert_eq!(map["poneyland"], 43); + /// ``` + #[inline] + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn and_modify(self, f: F) -> Self + where + F: FnOnce(&mut K, &mut V), + { + match self { + RawEntryMut::Occupied(mut entry) => { + { + let (k, v) = entry.get_key_value_mut(); + f(k, v); + } + RawEntryMut::Occupied(entry) + } + RawEntryMut::Vacant(entry) => RawEntryMut::Vacant(entry), + } } } -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, K, V> Iterator for Iter<'a, K, V> { - type Item = (&'a K, &'a V); +impl<'a, K, V, S> RawOccupiedEntryMut<'a, K, V, S> { + /// Gets a reference to the key in the entry. + #[inline] + #[must_use] + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn key(&self) -> &K { + self.base.key() + } + /// Gets a mutable reference to the key in the entry. #[inline] - fn next(&mut self) -> Option<(&'a K, &'a V)> { - self.base.next() + #[must_use] + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn key_mut(&mut self) -> &mut K { + self.base.key_mut() } + + /// Converts the entry into a mutable reference to the key in the entry + /// with a lifetime bound to the map itself. #[inline] - fn size_hint(&self) -> (usize, Option) { - self.base.size_hint() + #[must_use = "`self` will be dropped if the result is not used"] + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn into_key(self) -> &'a mut K { + self.base.into_key() } -} -#[stable(feature = "rust1", since = "1.0.0")] -impl ExactSizeIterator for Iter<'_, K, V> { + + /// Gets a reference to the value in the entry. #[inline] - fn len(&self) -> usize { - self.base.len() + #[must_use] + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn get(&self) -> &V { + self.base.get() } -} -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Iter<'_, K, V> {} + /// Converts the `OccupiedEntry` into a mutable reference to the value in the entry + /// with a lifetime bound to the map itself. + #[inline] + #[must_use = "`self` will be dropped if the result is not used"] + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn into_mut(self) -> &'a mut V { + self.base.into_mut() + } -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, K, V> Iterator for IterMut<'a, K, V> { - type Item = (&'a K, &'a mut V); + /// Gets a mutable reference to the value in the entry. + #[inline] + #[must_use] + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn get_mut(&mut self) -> &mut V { + self.base.get_mut() + } + /// Gets a reference to the key and value in the entry. #[inline] - fn next(&mut self) -> Option<(&'a K, &'a mut V)> { - self.base.next() + #[must_use] + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn get_key_value(&mut self) -> (&K, &V) { + self.base.get_key_value() } + + /// Gets a mutable reference to the key and value in the entry. #[inline] - fn size_hint(&self) -> (usize, Option) { - self.base.size_hint() + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn get_key_value_mut(&mut self) -> (&mut K, &mut V) { + self.base.get_key_value_mut() } -} -#[stable(feature = "rust1", since = "1.0.0")] -impl ExactSizeIterator for IterMut<'_, K, V> { + + /// Converts the `OccupiedEntry` into a mutable reference to the key and value in the entry + /// with a lifetime bound to the map itself. #[inline] - fn len(&self) -> usize { - self.base.len() + #[must_use = "`self` will be dropped if the result is not used"] + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn into_key_value(self) -> (&'a mut K, &'a mut V) { + self.base.into_key_value() } -} -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for IterMut<'_, K, V> {} -#[stable(feature = "std_debug", since = "1.16.0")] -impl fmt::Debug for IterMut<'_, K, V> + /// Sets the value of the entry, and returns the entry's old value. + #[inline] + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn insert(&mut self, value: V) -> V { + self.base.insert(value) + } + + /// Sets the value of the entry, and returns the entry's old value. + #[inline] + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn insert_key(&mut self, key: K) -> K { + self.base.insert_key(key) + } + + /// Takes the value out of the entry, and returns it. + #[inline] + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn remove(self) -> V { + self.base.remove() + } + + /// Take the ownership of the key and value from the map. + #[inline] + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn remove_entry(self) -> (K, V) { + self.base.remove_entry() + } +} + +impl<'a, K, V, S> RawVacantEntryMut<'a, K, V, S> { + /// Sets the value of the entry with the `VacantEntry`'s key, + /// and returns a mutable reference to it. + #[inline] + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn insert(self, key: K, value: V) -> (&'a mut K, &'a mut V) + where + K: Hash, + S: BuildHasher, + { + self.base.insert(key, value) + } + + /// Sets the value of the entry with the VacantEntry's key, + /// and returns a mutable reference to it. + #[inline] + #[unstable(feature = "hash_raw_entry", issue = "56167")] + pub fn insert_hashed_nocheck(self, hash: u64, key: K, value: V) -> (&'a mut K, &'a mut V) + where + K: Hash, + S: BuildHasher, + { + self.base.insert_hashed_nocheck(hash, key, value) + } +} + +#[unstable(feature = "hash_raw_entry", issue = "56167")] +impl Debug for RawEntryBuilderMut<'_, K, V, S> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RawEntryBuilder").finish_non_exhaustive() + } +} + +#[unstable(feature = "hash_raw_entry", issue = "56167")] +impl Debug for RawEntryMut<'_, K, V, S> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + RawEntryMut::Vacant(ref v) => f.debug_tuple("RawEntry").field(v).finish(), + RawEntryMut::Occupied(ref o) => f.debug_tuple("RawEntry").field(o).finish(), + } + } +} + +#[unstable(feature = "hash_raw_entry", issue = "56167")] +impl Debug for RawOccupiedEntryMut<'_, K, V, S> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RawOccupiedEntryMut") + .field("key", self.key()) + .field("value", self.get()) + .finish_non_exhaustive() + } +} + +#[unstable(feature = "hash_raw_entry", issue = "56167")] +impl Debug for RawVacantEntryMut<'_, K, V, S> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RawVacantEntryMut").finish_non_exhaustive() + } +} + +#[unstable(feature = "hash_raw_entry", issue = "56167")] +impl Debug for RawEntryBuilder<'_, K, V, S> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RawEntryBuilder").finish_non_exhaustive() + } +} + +/// A view into a single entry in a map, which may either be vacant or occupied. +/// +/// This `enum` is constructed from the [`entry`] method on [`HashMap`]. +/// +/// [`entry`]: HashMap::entry +#[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "HashMapEntry")] +pub enum Entry<'a, K: 'a, V: 'a> { + /// An occupied entry. + #[stable(feature = "rust1", since = "1.0.0")] + Occupied(#[stable(feature = "rust1", since = "1.0.0")] OccupiedEntry<'a, K, V>), + + /// A vacant entry. + #[stable(feature = "rust1", since = "1.0.0")] + Vacant(#[stable(feature = "rust1", since = "1.0.0")] VacantEntry<'a, K, V>), +} + +#[stable(feature = "debug_hash_map", since = "1.12.0")] +impl Debug for Entry<'_, K, V> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + Vacant(ref v) => f.debug_tuple("Entry").field(v).finish(), + Occupied(ref o) => f.debug_tuple("Entry").field(o).finish(), + } + } +} + +/// A view into an occupied entry in a `HashMap`. +/// It is part of the [`Entry`] enum. +#[stable(feature = "rust1", since = "1.0.0")] +pub struct OccupiedEntry<'a, K: 'a, V: 'a> { + base: base::RustcOccupiedEntry<'a, K, V>, +} + +#[stable(feature = "debug_hash_map", since = "1.12.0")] +impl Debug for OccupiedEntry<'_, K, V> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OccupiedEntry") + .field("key", self.key()) + .field("value", self.get()) + .finish_non_exhaustive() + } +} + +/// A view into a vacant entry in a `HashMap`. +/// It is part of the [`Entry`] enum. +#[stable(feature = "rust1", since = "1.0.0")] +pub struct VacantEntry<'a, K: 'a, V: 'a> { + base: base::RustcVacantEntry<'a, K, V>, +} + +#[stable(feature = "debug_hash_map", since = "1.12.0")] +impl Debug for VacantEntry<'_, K, V> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("VacantEntry").field(self.key()).finish() + } +} + +/// The error returned by [`try_insert`](HashMap::try_insert) when the key already exists. +/// +/// Contains the occupied entry, and the value that was not inserted. +#[unstable(feature = "map_try_insert", issue = "82766")] +pub struct OccupiedError<'a, K: 'a, V: 'a> { + /// The entry in the map that was already occupied. + pub entry: OccupiedEntry<'a, K, V>, + /// The value which was not inserted, because the entry was already occupied. + pub value: V, +} + +#[unstable(feature = "map_try_insert", issue = "82766")] +impl Debug for OccupiedError<'_, K, V> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OccupiedError") + .field("key", self.entry.key()) + .field("old_value", self.entry.get()) + .field("new_value", &self.value) + .finish_non_exhaustive() + } +} + +#[unstable(feature = "map_try_insert", issue = "82766")] +impl<'a, K: Debug, V: Debug> fmt::Display for OccupiedError<'a, K, V> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "failed to insert {:?}, key {:?} already exists with value {:?}", + self.value, + self.entry.key(), + self.entry.get(), + ) + } +} + +#[unstable(feature = "map_try_insert", issue = "82766")] +impl<'a, K: fmt::Debug, V: fmt::Debug> Error for OccupiedError<'a, K, V> { + #[allow(deprecated)] + fn description(&self) -> &str { + "key already exists" + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, K, V, S> IntoIterator for &'a HashMap { + type Item = (&'a K, &'a V); + type IntoIter = Iter<'a, K, V>; + + #[inline] + #[rustc_lint_query_instability] + fn into_iter(self) -> Iter<'a, K, V> { + self.iter() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, K, V, S> IntoIterator for &'a mut HashMap { + type Item = (&'a K, &'a mut V); + type IntoIter = IterMut<'a, K, V>; + + #[inline] + #[rustc_lint_query_instability] + fn into_iter(self) -> IterMut<'a, K, V> { + self.iter_mut() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl IntoIterator for HashMap { + type Item = (K, V); + type IntoIter = IntoIter; + + /// Creates a consuming iterator, that is, one that moves each key-value + /// pair out of the map in arbitrary order. The map cannot be used after + /// calling this. + /// + /// # Examples + /// + /// ``` + /// use std::collections::HashMap; + /// + /// let map = HashMap::from([ + /// ("a", 1), + /// ("b", 2), + /// ("c", 3), + /// ]); + /// + /// // Not possible with .iter() + /// let vec: Vec<(&str, i32)> = map.into_iter().collect(); + /// ``` + #[inline] + #[rustc_lint_query_instability] + fn into_iter(self) -> IntoIter { + IntoIter { base: self.base.into_iter() } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, K, V> Iterator for Iter<'a, K, V> { + type Item = (&'a K, &'a V); + + #[inline] + fn next(&mut self) -> Option<(&'a K, &'a V)> { + self.base.next() + } + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.base.size_hint() + } +} +#[stable(feature = "rust1", since = "1.0.0")] +impl ExactSizeIterator for Iter<'_, K, V> { + #[inline] + fn len(&self) -> usize { + self.base.len() + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Iter<'_, K, V> {} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, K, V> Iterator for IterMut<'a, K, V> { + type Item = (&'a K, &'a mut V); + + #[inline] + fn next(&mut self) -> Option<(&'a K, &'a mut V)> { + self.base.next() + } + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.base.size_hint() + } +} +#[stable(feature = "rust1", since = "1.0.0")] +impl ExactSizeIterator for IterMut<'_, K, V> { + #[inline] + fn len(&self) -> usize { + self.base.len() + } +} +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for IterMut<'_, K, V> {} + +#[stable(feature = "std_debug", since = "1.16.0")] +impl fmt::Debug for IterMut<'_, K, V> where K: fmt::Debug, V: fmt::Debug, @@ -1384,60 +2380,145 @@ impl ExactSizeIterator for ValuesMut<'_, K, V> { impl FusedIterator for ValuesMut<'_, K, V> {} #[stable(feature = "std_debug", since = "1.16.0")] -impl fmt::Debug for ValuesMut<'_, K, V> -where - K: fmt::Debug, - V: fmt::Debug, -{ +impl fmt::Debug for ValuesMut<'_, K, V> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_list().entries(self.inner.iter()).finish() + f.debug_list().entries(self.inner.iter().map(|(_, val)| val)).finish() } } -#[stable(feature = "drain", since = "1.6.0")] -impl<'a, K, V> Iterator for Drain<'a, K, V> { - type Item = (K, V); +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +impl Iterator for IntoKeys { + type Item = K; #[inline] - fn next(&mut self) -> Option<(K, V)> { - self.base.next() + fn next(&mut self) -> Option { + self.inner.next().map(|(k, _)| k) } #[inline] fn size_hint(&self) -> (usize, Option) { - self.base.size_hint() + self.inner.size_hint() } } -#[stable(feature = "drain", since = "1.6.0")] -impl ExactSizeIterator for Drain<'_, K, V> { +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +impl ExactSizeIterator for IntoKeys { #[inline] fn len(&self) -> usize { - self.base.len() + self.inner.len() } } -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Drain<'_, K, V> {} +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +impl FusedIterator for IntoKeys {} -#[stable(feature = "std_debug", since = "1.16.0")] -impl fmt::Debug for Drain<'_, K, V> -where - K: fmt::Debug, - V: fmt::Debug, -{ +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +impl fmt::Debug for IntoKeys { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_list().entries(self.iter()).finish() + f.debug_list().entries(self.inner.iter().map(|(k, _)| k)).finish() } } -impl<'a, K, V> Entry<'a, K, V> { - #[stable(feature = "rust1", since = "1.0.0")] - /// Ensures a value is in the entry by inserting the default if empty, and returns - /// a mutable reference to the value in the entry. - /// - /// # Examples - /// - /// ``` - /// use std::collections::HashMap; - /// +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +impl Iterator for IntoValues { + type Item = V; + + #[inline] + fn next(&mut self) -> Option { + self.inner.next().map(|(_, v)| v) + } + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +impl ExactSizeIterator for IntoValues { + #[inline] + fn len(&self) -> usize { + self.inner.len() + } +} +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +impl FusedIterator for IntoValues {} + +#[stable(feature = "map_into_keys_values", since = "1.54.0")] +impl fmt::Debug for IntoValues { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.inner.iter().map(|(_, v)| v)).finish() + } +} + +#[stable(feature = "drain", since = "1.6.0")] +impl<'a, K, V> Iterator for Drain<'a, K, V> { + type Item = (K, V); + + #[inline] + fn next(&mut self) -> Option<(K, V)> { + self.base.next() + } + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.base.size_hint() + } +} +#[stable(feature = "drain", since = "1.6.0")] +impl ExactSizeIterator for Drain<'_, K, V> { + #[inline] + fn len(&self) -> usize { + self.base.len() + } +} +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Drain<'_, K, V> {} + +#[stable(feature = "std_debug", since = "1.16.0")] +impl fmt::Debug for Drain<'_, K, V> +where + K: fmt::Debug, + V: fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.iter()).finish() + } +} + +#[unstable(feature = "hash_drain_filter", issue = "59618")] +impl Iterator for DrainFilter<'_, K, V, F> +where + F: FnMut(&K, &mut V) -> bool, +{ + type Item = (K, V); + + #[inline] + fn next(&mut self) -> Option<(K, V)> { + self.base.next() + } + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.base.size_hint() + } +} + +#[unstable(feature = "hash_drain_filter", issue = "59618")] +impl FusedIterator for DrainFilter<'_, K, V, F> where F: FnMut(&K, &mut V) -> bool {} + +#[unstable(feature = "hash_drain_filter", issue = "59618")] +impl<'a, K, V, F> fmt::Debug for DrainFilter<'a, K, V, F> +where + F: FnMut(&K, &mut V) -> bool, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DrainFilter").finish_non_exhaustive() + } +} + +impl<'a, K, V> Entry<'a, K, V> { + /// Ensures a value is in the entry by inserting the default if empty, and returns + /// a mutable reference to the value in the entry. + /// + /// # Examples + /// + /// ``` + /// use std::collections::HashMap; + /// /// let mut map: HashMap<&str, u32> = HashMap::new(); /// /// map.entry("poneyland").or_insert(3); @@ -1447,6 +2528,7 @@ impl<'a, K, V> Entry<'a, K, V> { /// assert_eq!(map["poneyland"], 6); /// ``` #[inline] + #[stable(feature = "rust1", since = "1.0.0")] pub fn or_insert(self, default: V) -> &'a mut V { match self { Occupied(entry) => entry.into_mut(), @@ -1454,7 +2536,6 @@ impl<'a, K, V> Entry<'a, K, V> { } } - #[stable(feature = "rust1", since = "1.0.0")] /// Ensures a value is in the entry by inserting the result of the default function if empty, /// and returns a mutable reference to the value in the entry. /// @@ -1471,6 +2552,7 @@ impl<'a, K, V> Entry<'a, K, V> { /// assert_eq!(map["poneyland"], "hoho".to_string()); /// ``` #[inline] + #[stable(feature = "rust1", since = "1.0.0")] pub fn or_insert_with V>(self, default: F) -> &'a mut V { match self { Occupied(entry) => entry.into_mut(), @@ -1478,6 +2560,36 @@ impl<'a, K, V> Entry<'a, K, V> { } } + /// Ensures a value is in the entry by inserting, if empty, the result of the default function. + /// This method allows for generating key-derived values for insertion by providing the default + /// function a reference to the key that was moved during the `.entry(key)` method call. + /// + /// The reference to the moved key is provided so that cloning or copying the key is + /// unnecessary, unlike with `.or_insert_with(|| ... )`. + /// + /// # Examples + /// + /// ``` + /// use std::collections::HashMap; + /// + /// let mut map: HashMap<&str, usize> = HashMap::new(); + /// + /// map.entry("poneyland").or_insert_with_key(|key| key.chars().count()); + /// + /// assert_eq!(map["poneyland"], 9); + /// ``` + #[inline] + #[stable(feature = "or_insert_with_key", since = "1.50.0")] + pub fn or_insert_with_key V>(self, default: F) -> &'a mut V { + match self { + Occupied(entry) => entry.into_mut(), + Vacant(entry) => { + let value = default(entry.key()); + entry.insert(value) + } + } + } + /// Returns a reference to this entry's key. /// /// # Examples @@ -1532,7 +2644,7 @@ impl<'a, K, V> Entry<'a, K, V> { } } - /// Sets the value of the entry, and returns an OccupiedEntry. + /// Sets the value of the entry, and returns an `OccupiedEntry`. /// /// # Examples /// @@ -1541,13 +2653,13 @@ impl<'a, K, V> Entry<'a, K, V> { /// use std::collections::HashMap; /// /// let mut map: HashMap<&str, String> = HashMap::new(); - /// let entry = map.entry("poneyland").insert("hoho".to_string()); + /// let entry = map.entry("poneyland").insert_entry("hoho".to_string()); /// /// assert_eq!(entry.key(), &"poneyland"); /// ``` #[inline] #[unstable(feature = "entry_insert", issue = "65225")] - pub fn insert(self, value: V) -> OccupiedEntry<'a, K, V> { + pub fn insert_entry(self, value: V) -> OccupiedEntry<'a, K, V> { match self { Occupied(mut entry) => { entry.insert(value); @@ -1559,7 +2671,6 @@ impl<'a, K, V> Entry<'a, K, V> { } impl<'a, K, V: Default> Entry<'a, K, V> { - #[stable(feature = "entry_or_default", since = "1.28.0")] /// Ensures a value is in the entry by inserting the default value if empty, /// and returns a mutable reference to the value in the entry. /// @@ -1576,6 +2687,7 @@ impl<'a, K, V: Default> Entry<'a, K, V> { /// # } /// ``` #[inline] + #[stable(feature = "entry_or_default", since = "1.28.0")] pub fn or_default(self) -> &'a mut V { match self { Occupied(entry) => entry.into_mut(), @@ -1652,7 +2764,7 @@ impl<'a, K, V> OccupiedEntry<'a, K, V> { /// If you need a reference to the `OccupiedEntry` which may outlive the /// destruction of the `Entry` value, see [`into_mut`]. /// - /// [`into_mut`]: #method.into_mut + /// [`into_mut`]: Self::into_mut /// /// # Examples /// @@ -1680,12 +2792,12 @@ impl<'a, K, V> OccupiedEntry<'a, K, V> { self.base.get_mut() } - /// Converts the OccupiedEntry into a mutable reference to the value in the entry + /// Converts the `OccupiedEntry` into a mutable reference to the value in the entry /// with a lifetime bound to the map itself. /// /// If you need multiple references to the `OccupiedEntry`, see [`get_mut`]. /// - /// [`get_mut`]: #method.get_mut + /// [`get_mut`]: Self::get_mut /// /// # Examples /// @@ -1792,7 +2904,7 @@ impl<'a, K, V> OccupiedEntry<'a, K, V> { /// use std::rc::Rc; /// /// let mut map: HashMap, u32> = HashMap::new(); - /// let mut known_strings: Vec> = Vec::new(); + /// let known_strings: Vec> = Vec::new(); /// /// // Initialise known strings, run program, etc. /// @@ -1800,7 +2912,7 @@ impl<'a, K, V> OccupiedEntry<'a, K, V> { /// /// fn reclaim_memory(map: &mut HashMap, u32>, known_strings: &[Rc] ) { /// for s in known_strings { - /// if let Entry::Occupied(entry) = map.entry(s.clone()) { + /// if let Entry::Occupied(entry) = map.entry(Rc::clone(s)) { /// // Replaces the entry's key with our version of it in `known_strings`. /// entry.replace_key(); /// } @@ -1852,7 +2964,7 @@ impl<'a, K: 'a, V: 'a> VacantEntry<'a, K, V> { self.base.into_key() } - /// Sets the value of the entry with the VacantEntry's key, + /// Sets the value of the entry with the `VacantEntry`'s key, /// and returns a mutable reference to it. /// /// # Examples @@ -1874,24 +2986,26 @@ impl<'a, K: 'a, V: 'a> VacantEntry<'a, K, V> { self.base.insert(value) } - /// Sets the value of the entry with the VacantEntry's key, - /// and returns an OccupiedEntry. + /// Sets the value of the entry with the `VacantEntry`'s key, + /// and returns an `OccupiedEntry`. /// /// # Examples /// /// ``` + /// #![feature(entry_insert)] /// use std::collections::HashMap; /// use std::collections::hash_map::Entry; /// /// let mut map: HashMap<&str, u32> = HashMap::new(); /// /// if let Entry::Vacant(o) = map.entry("poneyland") { - /// o.insert(37); + /// o.insert_entry(37); /// } /// assert_eq!(map["poneyland"], 37); /// ``` #[inline] - fn insert_entry(self, value: V) -> OccupiedEntry<'a, K, V> { + #[unstable(feature = "entry_insert", issue = "65225")] + pub fn insert_entry(self, value: V) -> OccupiedEntry<'a, K, V> { let base = self.base.insert_entry(value); OccupiedEntry { base } } @@ -1922,6 +3036,16 @@ where fn extend>(&mut self, iter: T) { self.base.extend(iter) } + + #[inline] + fn extend_one(&mut self, (k, v): (K, V)) { + self.base.insert(k, v); + } + + #[inline] + fn extend_reserve(&mut self, additional: usize) { + self.base.extend_reserve(additional); + } } #[stable(feature = "hash_extend_copy", since = "1.4.0")] @@ -1935,6 +3059,16 @@ where fn extend>(&mut self, iter: T) { self.base.extend(iter) } + + #[inline] + fn extend_one(&mut self, (&k, &v): (&'a K, &'a V)) { + self.base.insert(k, v); + } + + #[inline] + fn extend_reserve(&mut self, additional: usize) { + Extend::<(K, V)>::extend_reserve(self, additional) + } } /// `RandomState` is the default state for [`HashMap`] types. @@ -1943,9 +3077,6 @@ where /// [`Hasher`], but the hashers created by two different `RandomState` /// instances are unlikely to produce the same result for the same values. /// -/// [`HashMap`]: struct.HashMap.html -/// [`Hasher`]: ../../hash/trait.Hasher.html -/// /// # Examples /// /// ``` @@ -1976,9 +3107,29 @@ impl RandomState { #[inline] #[allow(deprecated)] // rand + #[must_use] #[stable(feature = "hashmap_build_hasher", since = "1.7.0")] pub fn new() -> RandomState { - RandomState { k0: 1, k1: 2 } + // Historically this function did not cache keys from the OS and instead + // simply always called `rand::thread_rng().gen()` twice. In #31356 it + // was discovered, however, that because we re-seed the thread-local RNG + // from the OS periodically that this can cause excessive slowdown when + // many hash maps are created on a thread. To solve this performance + // trap we cache the first set of randomly generated keys per-thread. + // + // Later in #36481 it was discovered that exposing a deterministic + // iteration order allows a form of DOS attack. To counter that we + // increment one of the seeds on every RandomState creation, giving + // every corresponding HashMap a different iteration order. + thread_local!(static KEYS: Cell<(u64, u64)> = { + Cell::new(sys::hashmap_random_keys()) + }); + + KEYS.with(|keys| { + let (k0, k1) = keys.get(); + keys.set((k0.wrapping_add(1), k1)); + RandomState { k0, k1 } + }) } } @@ -1996,9 +3147,6 @@ impl BuildHasher for RandomState { /// /// The internal algorithm is not specified, and so it and its hashes should /// not be relied upon over releases. -/// -/// [`RandomState`]: struct.RandomState.html -/// [`Hasher`]: ../../hash/trait.Hasher.html #[stable(feature = "hashmap_default_hasher", since = "1.13.0")] #[allow(deprecated)] #[derive(Clone, Debug)] @@ -2011,31 +3159,44 @@ impl DefaultHasher { /// `DefaultHasher` instances, but is the same as all other `DefaultHasher` /// instances created through `new` or `default`. #[stable(feature = "hashmap_default_hasher", since = "1.13.0")] + #[inline] #[allow(deprecated)] - pub fn new() -> DefaultHasher { + #[rustc_const_unstable(feature = "const_hash", issue = "104061")] + #[must_use] + pub const fn new() -> DefaultHasher { DefaultHasher(SipHasher13::new_with_keys(0, 0)) } } #[stable(feature = "hashmap_default_hasher", since = "1.13.0")] -impl Default for DefaultHasher { - // FIXME: here should link `new` to [DefaultHasher::new], but it occurs intra-doc link - // resolution failure when re-exporting libstd items. When #56922 fixed, - // link `new` to [DefaultHasher::new] again. - /// Creates a new `DefaultHasher` using `new`. +#[rustc_const_unstable(feature = "const_hash", issue = "104061")] +impl const Default for DefaultHasher { + /// Creates a new `DefaultHasher` using [`new`]. /// See its documentation for more. + /// + /// [`new`]: DefaultHasher::new + #[inline] fn default() -> DefaultHasher { DefaultHasher::new() } } #[stable(feature = "hashmap_default_hasher", since = "1.13.0")] -impl Hasher for DefaultHasher { +#[rustc_const_unstable(feature = "const_hash", issue = "104061")] +impl const Hasher for DefaultHasher { + // The underlying `SipHasher13` doesn't override the other + // `write_*` methods, so it's ok not to forward them here. + #[inline] fn write(&mut self, msg: &[u8]) { self.0.write(msg) } + #[inline] + fn write_str(&mut self, s: &str) { + self.0.write_str(s); + } + #[inline] fn finish(&self) -> u64 { self.0.finish() @@ -2054,7 +3215,7 @@ impl Default for RandomState { #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for RandomState { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("RandomState { .. }") + f.debug_struct("RandomState").finish_non_exhaustive() } } @@ -2067,8 +3228,25 @@ fn map_entry<'a, K: 'a, V: 'a>(raw: base::RustcEntry<'a, K, V>) -> Entry<'a, K, } #[inline] -fn map_collection_alloc_err(err: TryReserveError) -> TryReserveError { - err +pub(super) fn map_try_reserve_error(err: hashbrown::TryReserveError) -> TryReserveError { + match err { + hashbrown::TryReserveError::CapacityOverflow => { + TryReserveErrorKind::CapacityOverflow.into() + } + hashbrown::TryReserveError::AllocError { layout } => { + TryReserveErrorKind::AllocError { layout, non_exhaustive: () }.into() + } + } +} + +#[inline] +fn map_raw_entry<'a, K: 'a, V: 'a, S: 'a>( + raw: base::RawEntryMut<'a, K, V, S>, +) -> RawEntryMut<'a, K, V, S> { + match raw { + base::RawEntryMut::Occupied(base) => RawEntryMut::Occupied(RawOccupiedEntryMut { base }), + base::RawEntryMut::Vacant(base) => RawEntryMut::Vacant(RawVacantEntryMut { base }), + } } #[allow(dead_code)] @@ -2109,910 +3287,3 @@ fn assert_covariance() { d } } - -#[cfg(test)] -mod test_map { - use super::Entry::{Occupied, Vacant}; - use super::HashMap; - use super::RandomState; - use crate::cell::RefCell; - use rand::{thread_rng, Rng}; - use realstd::collections::TryReserveError::*; - use realstd::usize; - - // https://github.com/rust-lang/rust/issues/62301 - fn _assert_hashmap_is_unwind_safe() { - fn assert_unwind_safe() {} - assert_unwind_safe::>>(); - } - - #[test] - fn test_zero_capacities() { - type HM = HashMap; - - let m = HM::new(); - assert_eq!(m.capacity(), 0); - - let m = HM::default(); - assert_eq!(m.capacity(), 0); - - let m = HM::with_hasher(RandomState::new()); - assert_eq!(m.capacity(), 0); - - let m = HM::with_capacity(0); - assert_eq!(m.capacity(), 0); - - let m = HM::with_capacity_and_hasher(0, RandomState::new()); - assert_eq!(m.capacity(), 0); - - let mut m = HM::new(); - m.insert(1, 1); - m.insert(2, 2); - m.remove(&1); - m.remove(&2); - m.shrink_to_fit(); - assert_eq!(m.capacity(), 0); - - let mut m = HM::new(); - m.reserve(0); - assert_eq!(m.capacity(), 0); - } - - #[test] - fn test_create_capacity_zero() { - let mut m = HashMap::with_capacity(0); - - assert!(m.insert(1, 1).is_none()); - - assert!(m.contains_key(&1)); - assert!(!m.contains_key(&0)); - } - - #[test] - fn test_insert() { - let mut m = HashMap::new(); - assert_eq!(m.len(), 0); - assert!(m.insert(1, 2).is_none()); - assert_eq!(m.len(), 1); - assert!(m.insert(2, 4).is_none()); - assert_eq!(m.len(), 2); - assert_eq!(*m.get(&1).unwrap(), 2); - assert_eq!(*m.get(&2).unwrap(), 4); - } - - #[test] - fn test_clone() { - let mut m = HashMap::new(); - assert_eq!(m.len(), 0); - assert!(m.insert(1, 2).is_none()); - assert_eq!(m.len(), 1); - assert!(m.insert(2, 4).is_none()); - assert_eq!(m.len(), 2); - let m2 = m.clone(); - assert_eq!(*m2.get(&1).unwrap(), 2); - assert_eq!(*m2.get(&2).unwrap(), 4); - assert_eq!(m2.len(), 2); - } - - thread_local! { static DROP_VECTOR: RefCell> = RefCell::new(Vec::new()) } - - #[derive(Hash, PartialEq, Eq)] - struct Droppable { - k: usize, - } - - impl Droppable { - fn new(k: usize) -> Droppable { - DROP_VECTOR.with(|slot| { - slot.borrow_mut()[k] += 1; - }); - - Droppable { k } - } - } - - impl Drop for Droppable { - fn drop(&mut self) { - DROP_VECTOR.with(|slot| { - slot.borrow_mut()[self.k] -= 1; - }); - } - } - - impl Clone for Droppable { - fn clone(&self) -> Droppable { - Droppable::new(self.k) - } - } - - #[test] - fn test_drops() { - DROP_VECTOR.with(|slot| { - *slot.borrow_mut() = vec![0; 200]; - }); - - { - let mut m = HashMap::new(); - - DROP_VECTOR.with(|v| { - for i in 0..200 { - assert_eq!(v.borrow()[i], 0); - } - }); - - for i in 0..100 { - let d1 = Droppable::new(i); - let d2 = Droppable::new(i + 100); - m.insert(d1, d2); - } - - DROP_VECTOR.with(|v| { - for i in 0..200 { - assert_eq!(v.borrow()[i], 1); - } - }); - - for i in 0..50 { - let k = Droppable::new(i); - let v = m.remove(&k); - - assert!(v.is_some()); - - DROP_VECTOR.with(|v| { - assert_eq!(v.borrow()[i], 1); - assert_eq!(v.borrow()[i + 100], 1); - }); - } - - DROP_VECTOR.with(|v| { - for i in 0..50 { - assert_eq!(v.borrow()[i], 0); - assert_eq!(v.borrow()[i + 100], 0); - } - - for i in 50..100 { - assert_eq!(v.borrow()[i], 1); - assert_eq!(v.borrow()[i + 100], 1); - } - }); - } - - DROP_VECTOR.with(|v| { - for i in 0..200 { - assert_eq!(v.borrow()[i], 0); - } - }); - } - - #[test] - fn test_into_iter_drops() { - DROP_VECTOR.with(|v| { - *v.borrow_mut() = vec![0; 200]; - }); - - let hm = { - let mut hm = HashMap::new(); - - DROP_VECTOR.with(|v| { - for i in 0..200 { - assert_eq!(v.borrow()[i], 0); - } - }); - - for i in 0..100 { - let d1 = Droppable::new(i); - let d2 = Droppable::new(i + 100); - hm.insert(d1, d2); - } - - DROP_VECTOR.with(|v| { - for i in 0..200 { - assert_eq!(v.borrow()[i], 1); - } - }); - - hm - }; - - // By the way, ensure that cloning doesn't screw up the dropping. - drop(hm.clone()); - - { - let mut half = hm.into_iter().take(50); - - DROP_VECTOR.with(|v| { - for i in 0..200 { - assert_eq!(v.borrow()[i], 1); - } - }); - - for _ in half.by_ref() {} - - DROP_VECTOR.with(|v| { - let nk = (0..100).filter(|&i| v.borrow()[i] == 1).count(); - - let nv = (0..100).filter(|&i| v.borrow()[i + 100] == 1).count(); - - assert_eq!(nk, 50); - assert_eq!(nv, 50); - }); - }; - - DROP_VECTOR.with(|v| { - for i in 0..200 { - assert_eq!(v.borrow()[i], 0); - } - }); - } - - #[test] - fn test_empty_remove() { - let mut m: HashMap = HashMap::new(); - assert_eq!(m.remove(&0), None); - } - - #[test] - fn test_empty_entry() { - let mut m: HashMap = HashMap::new(); - match m.entry(0) { - Occupied(_) => panic!(), - Vacant(_) => {} - } - assert!(*m.entry(0).or_insert(true)); - assert_eq!(m.len(), 1); - } - - #[test] - fn test_empty_iter() { - let mut m: HashMap = HashMap::new(); - assert_eq!(m.drain().next(), None); - assert_eq!(m.keys().next(), None); - assert_eq!(m.values().next(), None); - assert_eq!(m.values_mut().next(), None); - assert_eq!(m.iter().next(), None); - assert_eq!(m.iter_mut().next(), None); - assert_eq!(m.len(), 0); - assert!(m.is_empty()); - assert_eq!(m.into_iter().next(), None); - } - - #[test] - fn test_lots_of_insertions() { - let mut m = HashMap::new(); - - // Try this a few times to make sure we never screw up the hashmap's - // internal state. - for _ in 0..10 { - assert!(m.is_empty()); - - for i in 1..1001 { - assert!(m.insert(i, i).is_none()); - - for j in 1..=i { - let r = m.get(&j); - assert_eq!(r, Some(&j)); - } - - for j in i + 1..1001 { - let r = m.get(&j); - assert_eq!(r, None); - } - } - - for i in 1001..2001 { - assert!(!m.contains_key(&i)); - } - - // remove forwards - for i in 1..1001 { - assert!(m.remove(&i).is_some()); - - for j in 1..=i { - assert!(!m.contains_key(&j)); - } - - for j in i + 1..1001 { - assert!(m.contains_key(&j)); - } - } - - for i in 1..1001 { - assert!(!m.contains_key(&i)); - } - - for i in 1..1001 { - assert!(m.insert(i, i).is_none()); - } - - // remove backwards - for i in (1..1001).rev() { - assert!(m.remove(&i).is_some()); - - for j in i..1001 { - assert!(!m.contains_key(&j)); - } - - for j in 1..i { - assert!(m.contains_key(&j)); - } - } - } - } - - #[test] - fn test_find_mut() { - let mut m = HashMap::new(); - assert!(m.insert(1, 12).is_none()); - assert!(m.insert(2, 8).is_none()); - assert!(m.insert(5, 14).is_none()); - let new = 100; - match m.get_mut(&5) { - None => panic!(), - Some(x) => *x = new, - } - assert_eq!(m.get(&5), Some(&new)); - } - - #[test] - fn test_insert_overwrite() { - let mut m = HashMap::new(); - assert!(m.insert(1, 2).is_none()); - assert_eq!(*m.get(&1).unwrap(), 2); - assert!(!m.insert(1, 3).is_none()); - assert_eq!(*m.get(&1).unwrap(), 3); - } - - #[test] - fn test_insert_conflicts() { - let mut m = HashMap::with_capacity(4); - assert!(m.insert(1, 2).is_none()); - assert!(m.insert(5, 3).is_none()); - assert!(m.insert(9, 4).is_none()); - assert_eq!(*m.get(&9).unwrap(), 4); - assert_eq!(*m.get(&5).unwrap(), 3); - assert_eq!(*m.get(&1).unwrap(), 2); - } - - #[test] - fn test_conflict_remove() { - let mut m = HashMap::with_capacity(4); - assert!(m.insert(1, 2).is_none()); - assert_eq!(*m.get(&1).unwrap(), 2); - assert!(m.insert(5, 3).is_none()); - assert_eq!(*m.get(&1).unwrap(), 2); - assert_eq!(*m.get(&5).unwrap(), 3); - assert!(m.insert(9, 4).is_none()); - assert_eq!(*m.get(&1).unwrap(), 2); - assert_eq!(*m.get(&5).unwrap(), 3); - assert_eq!(*m.get(&9).unwrap(), 4); - assert!(m.remove(&1).is_some()); - assert_eq!(*m.get(&9).unwrap(), 4); - assert_eq!(*m.get(&5).unwrap(), 3); - } - - #[test] - fn test_is_empty() { - let mut m = HashMap::with_capacity(4); - assert!(m.insert(1, 2).is_none()); - assert!(!m.is_empty()); - assert!(m.remove(&1).is_some()); - assert!(m.is_empty()); - } - - #[test] - fn test_remove() { - let mut m = HashMap::new(); - m.insert(1, 2); - assert_eq!(m.remove(&1), Some(2)); - assert_eq!(m.remove(&1), None); - } - - #[test] - fn test_remove_entry() { - let mut m = HashMap::new(); - m.insert(1, 2); - assert_eq!(m.remove_entry(&1), Some((1, 2))); - assert_eq!(m.remove(&1), None); - } - - #[test] - fn test_iterate() { - let mut m = HashMap::with_capacity(4); - for i in 0..32 { - assert!(m.insert(i, i * 2).is_none()); - } - assert_eq!(m.len(), 32); - - let mut observed: u32 = 0; - - for (k, v) in &m { - assert_eq!(*v, *k * 2); - observed |= 1 << *k; - } - assert_eq!(observed, 0xFFFF_FFFF); - } - - #[test] - fn test_keys() { - let vec = vec![(1, 'a'), (2, 'b'), (3, 'c')]; - let map: HashMap<_, _> = vec.into_iter().collect(); - let keys: Vec<_> = map.keys().cloned().collect(); - assert_eq!(keys.len(), 3); - assert!(keys.contains(&1)); - assert!(keys.contains(&2)); - assert!(keys.contains(&3)); - } - - #[test] - fn test_values() { - let vec = vec![(1, 'a'), (2, 'b'), (3, 'c')]; - let map: HashMap<_, _> = vec.into_iter().collect(); - let values: Vec<_> = map.values().cloned().collect(); - assert_eq!(values.len(), 3); - assert!(values.contains(&'a')); - assert!(values.contains(&'b')); - assert!(values.contains(&'c')); - } - - #[test] - fn test_values_mut() { - let vec = vec![(1, 1), (2, 2), (3, 3)]; - let mut map: HashMap<_, _> = vec.into_iter().collect(); - for value in map.values_mut() { - *value = (*value) * 2 - } - let values: Vec<_> = map.values().cloned().collect(); - assert_eq!(values.len(), 3); - assert!(values.contains(&2)); - assert!(values.contains(&4)); - assert!(values.contains(&6)); - } - - #[test] - fn test_find() { - let mut m = HashMap::new(); - assert!(m.get(&1).is_none()); - m.insert(1, 2); - match m.get(&1) { - None => panic!(), - Some(v) => assert_eq!(*v, 2), - } - } - - #[test] - fn test_eq() { - let mut m1 = HashMap::new(); - m1.insert(1, 2); - m1.insert(2, 3); - m1.insert(3, 4); - - let mut m2 = HashMap::new(); - m2.insert(1, 2); - m2.insert(2, 3); - - assert!(m1 != m2); - - m2.insert(3, 4); - - assert_eq!(m1, m2); - } - - #[test] - fn test_show() { - let mut map = HashMap::new(); - let empty: HashMap = HashMap::new(); - - map.insert(1, 2); - map.insert(3, 4); - - let map_str = format!("{:?}", map); - - assert!(map_str == "{1: 2, 3: 4}" || map_str == "{3: 4, 1: 2}"); - assert_eq!(format!("{:?}", empty), "{}"); - } - - #[test] - fn test_reserve_shrink_to_fit() { - let mut m = HashMap::new(); - m.insert(0, 0); - m.remove(&0); - assert!(m.capacity() >= m.len()); - for i in 0..128 { - m.insert(i, i); - } - m.reserve(256); - - let usable_cap = m.capacity(); - for i in 128..(128 + 256) { - m.insert(i, i); - assert_eq!(m.capacity(), usable_cap); - } - - for i in 100..(128 + 256) { - assert_eq!(m.remove(&i), Some(i)); - } - m.shrink_to_fit(); - - assert_eq!(m.len(), 100); - assert!(!m.is_empty()); - assert!(m.capacity() >= m.len()); - - for i in 0..100 { - assert_eq!(m.remove(&i), Some(i)); - } - m.shrink_to_fit(); - m.insert(0, 0); - - assert_eq!(m.len(), 1); - assert!(m.capacity() >= m.len()); - assert_eq!(m.remove(&0), Some(0)); - } - - #[test] - fn test_from_iter() { - let xs = [(1, 1), (2, 2), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]; - - let map: HashMap<_, _> = xs.iter().cloned().collect(); - - for &(k, v) in &xs { - assert_eq!(map.get(&k), Some(&v)); - } - - assert_eq!(map.iter().len(), xs.len() - 1); - } - - #[test] - fn test_size_hint() { - let xs = [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]; - - let map: HashMap<_, _> = xs.iter().cloned().collect(); - - let mut iter = map.iter(); - - for _ in iter.by_ref().take(3) {} - - assert_eq!(iter.size_hint(), (3, Some(3))); - } - - #[test] - fn test_iter_len() { - let xs = [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]; - - let map: HashMap<_, _> = xs.iter().cloned().collect(); - - let mut iter = map.iter(); - - for _ in iter.by_ref().take(3) {} - - assert_eq!(iter.len(), 3); - } - - #[test] - fn test_mut_size_hint() { - let xs = [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]; - - let mut map: HashMap<_, _> = xs.iter().cloned().collect(); - - let mut iter = map.iter_mut(); - - for _ in iter.by_ref().take(3) {} - - assert_eq!(iter.size_hint(), (3, Some(3))); - } - - #[test] - fn test_iter_mut_len() { - let xs = [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]; - - let mut map: HashMap<_, _> = xs.iter().cloned().collect(); - - let mut iter = map.iter_mut(); - - for _ in iter.by_ref().take(3) {} - - assert_eq!(iter.len(), 3); - } - - #[test] - fn test_index() { - let mut map = HashMap::new(); - - map.insert(1, 2); - map.insert(2, 1); - map.insert(3, 4); - - assert_eq!(map[&2], 1); - } - - #[test] - #[should_panic] - fn test_index_nonexistent() { - let mut map = HashMap::new(); - - map.insert(1, 2); - map.insert(2, 1); - map.insert(3, 4); - - map[&4]; - } - - #[test] - fn test_entry() { - let xs = [(1, 10), (2, 20), (3, 30), (4, 40), (5, 50), (6, 60)]; - - let mut map: HashMap<_, _> = xs.iter().cloned().collect(); - - // Existing key (insert) - match map.entry(1) { - Vacant(_) => unreachable!(), - Occupied(mut view) => { - assert_eq!(view.get(), &10); - assert_eq!(view.insert(100), 10); - } - } - assert_eq!(map.get(&1).unwrap(), &100); - assert_eq!(map.len(), 6); - - // Existing key (update) - match map.entry(2) { - Vacant(_) => unreachable!(), - Occupied(mut view) => { - let v = view.get_mut(); - let new_v = (*v) * 10; - *v = new_v; - } - } - assert_eq!(map.get(&2).unwrap(), &200); - assert_eq!(map.len(), 6); - - // Existing key (take) - match map.entry(3) { - Vacant(_) => unreachable!(), - Occupied(view) => { - assert_eq!(view.remove(), 30); - } - } - assert_eq!(map.get(&3), None); - assert_eq!(map.len(), 5); - - // Inexistent key (insert) - match map.entry(10) { - Occupied(_) => unreachable!(), - Vacant(view) => { - assert_eq!(*view.insert(1000), 1000); - } - } - assert_eq!(map.get(&10).unwrap(), &1000); - assert_eq!(map.len(), 6); - } - - #[test] - fn test_entry_take_doesnt_corrupt() { - #![allow(deprecated)] //rand - // Test for #19292 - fn check(m: &HashMap) { - for k in m.keys() { - assert!(m.contains_key(k), "{} is in keys() but not in the map?", k); - } - } - - let mut m = HashMap::new(); - let mut rng = thread_rng(); - - // Populate the map with some items. - for _ in 0..50 { - let x = rng.gen_range(-10, 10); - m.insert(x, ()); - } - - for _ in 0..1000 { - let x = rng.gen_range(-10, 10); - match m.entry(x) { - Vacant(_) => {} - Occupied(e) => { - e.remove(); - } - } - - check(&m); - } - } - - #[test] - fn test_extend_ref() { - let mut a = HashMap::new(); - a.insert(1, "one"); - let mut b = HashMap::new(); - b.insert(2, "two"); - b.insert(3, "three"); - - a.extend(&b); - - assert_eq!(a.len(), 3); - assert_eq!(a[&1], "one"); - assert_eq!(a[&2], "two"); - assert_eq!(a[&3], "three"); - } - - #[test] - fn test_capacity_not_less_than_len() { - let mut a = HashMap::new(); - let mut item = 0; - - for _ in 0..116 { - a.insert(item, 0); - item += 1; - } - - assert!(a.capacity() > a.len()); - - let free = a.capacity() - a.len(); - for _ in 0..free { - a.insert(item, 0); - item += 1; - } - - assert_eq!(a.len(), a.capacity()); - - // Insert at capacity should cause allocation. - a.insert(item, 0); - assert!(a.capacity() > a.len()); - } - - #[test] - fn test_occupied_entry_key() { - let mut a = HashMap::new(); - let key = "hello there"; - let value = "value goes here"; - assert!(a.is_empty()); - a.insert(key.clone(), value.clone()); - assert_eq!(a.len(), 1); - assert_eq!(a[key], value); - - match a.entry(key.clone()) { - Vacant(_) => panic!(), - Occupied(e) => assert_eq!(key, *e.key()), - } - assert_eq!(a.len(), 1); - assert_eq!(a[key], value); - } - - #[test] - fn test_vacant_entry_key() { - let mut a = HashMap::new(); - let key = "hello there"; - let value = "value goes here"; - - assert!(a.is_empty()); - match a.entry(key.clone()) { - Occupied(_) => panic!(), - Vacant(e) => { - assert_eq!(key, *e.key()); - e.insert(value.clone()); - } - } - assert_eq!(a.len(), 1); - assert_eq!(a[key], value); - } - - #[test] - fn test_retain() { - let mut map: HashMap = (0..100).map(|x| (x, x * 10)).collect(); - - map.retain(|&k, _| k % 2 == 0); - assert_eq!(map.len(), 50); - assert_eq!(map[&2], 20); - assert_eq!(map[&4], 40); - assert_eq!(map[&6], 60); - } - - #[test] - fn test_try_reserve() { - let mut empty_bytes: HashMap = HashMap::new(); - - const MAX_USIZE: usize = usize::MAX; - - if let Err(CapacityOverflow) = empty_bytes.try_reserve(MAX_USIZE) { - } else { - panic!("usize::MAX should trigger an overflow!"); - } - - if let Err(AllocError { .. }) = empty_bytes.try_reserve(MAX_USIZE / 8) { - } else { - panic!("usize::MAX / 8 should trigger an OOM!") - } - } - - #[test] - fn test_raw_entry() { - use super::RawEntryMut::{Occupied, Vacant}; - - let xs = [(1i32, 10i32), (2, 20), (3, 30), (4, 40), (5, 50), (6, 60)]; - - let mut map: HashMap<_, _> = xs.iter().cloned().collect(); - - let compute_hash = |map: &HashMap, k: i32| -> u64 { - use core::hash::{BuildHasher, Hash, Hasher}; - - let mut hasher = map.hasher().build_hasher(); - k.hash(&mut hasher); - hasher.finish() - }; - - // Existing key (insert) - match map.raw_entry_mut().from_key(&1) { - Vacant(_) => unreachable!(), - Occupied(mut view) => { - assert_eq!(view.get(), &10); - assert_eq!(view.insert(100), 10); - } - } - let hash1 = compute_hash(&map, 1); - assert_eq!(map.raw_entry().from_key(&1).unwrap(), (&1, &100)); - assert_eq!(map.raw_entry().from_hash(hash1, |k| *k == 1).unwrap(), (&1, &100)); - assert_eq!(map.raw_entry().from_key_hashed_nocheck(hash1, &1).unwrap(), (&1, &100)); - assert_eq!(map.len(), 6); - - // Existing key (update) - match map.raw_entry_mut().from_key(&2) { - Vacant(_) => unreachable!(), - Occupied(mut view) => { - let v = view.get_mut(); - let new_v = (*v) * 10; - *v = new_v; - } - } - let hash2 = compute_hash(&map, 2); - assert_eq!(map.raw_entry().from_key(&2).unwrap(), (&2, &200)); - assert_eq!(map.raw_entry().from_hash(hash2, |k| *k == 2).unwrap(), (&2, &200)); - assert_eq!(map.raw_entry().from_key_hashed_nocheck(hash2, &2).unwrap(), (&2, &200)); - assert_eq!(map.len(), 6); - - // Existing key (take) - let hash3 = compute_hash(&map, 3); - match map.raw_entry_mut().from_key_hashed_nocheck(hash3, &3) { - Vacant(_) => unreachable!(), - Occupied(view) => { - assert_eq!(view.remove_entry(), (3, 30)); - } - } - assert_eq!(map.raw_entry().from_key(&3), None); - assert_eq!(map.raw_entry().from_hash(hash3, |k| *k == 3), None); - assert_eq!(map.raw_entry().from_key_hashed_nocheck(hash3, &3), None); - assert_eq!(map.len(), 5); - - // Nonexistent key (insert) - match map.raw_entry_mut().from_key(&10) { - Occupied(_) => unreachable!(), - Vacant(view) => { - assert_eq!(view.insert(10, 1000), (&mut 10, &mut 1000)); - } - } - assert_eq!(map.raw_entry().from_key(&10).unwrap(), (&10, &1000)); - assert_eq!(map.len(), 6); - - // Ensure all lookup methods produce equivalent results. - for k in 0..12 { - let hash = compute_hash(&map, k); - let v = map.get(&k).cloned(); - let kv = v.as_ref().map(|v| (&k, v)); - - assert_eq!(map.raw_entry().from_key(&k), kv); - assert_eq!(map.raw_entry().from_hash(hash, |q| *q == k), kv); - assert_eq!(map.raw_entry().from_key_hashed_nocheck(hash, &k), kv); - - match map.raw_entry_mut().from_key(&k) { - Occupied(mut o) => assert_eq!(Some(o.get_key_value()), kv), - Vacant(_) => assert_eq!(v, None), - } - match map.raw_entry_mut().from_key_hashed_nocheck(hash, &k) { - Occupied(mut o) => assert_eq!(Some(o.get_key_value()), kv), - Vacant(_) => assert_eq!(v, None), - } - match map.raw_entry_mut().from_hash(hash, |q| *q == k) { - Occupied(mut o) => assert_eq!(Some(o.get_key_value()), kv), - Vacant(_) => assert_eq!(v, None), - } - } - } -} diff --git a/crux-mir/lib/std/src/collections/hash/map/tests.rs b/crux-mir/lib/std/src/collections/hash/map/tests.rs new file mode 100644 index 000000000..6b89518e2 --- /dev/null +++ b/crux-mir/lib/std/src/collections/hash/map/tests.rs @@ -0,0 +1,1124 @@ +use super::Entry::{Occupied, Vacant}; +use super::HashMap; +use super::RandomState; +use crate::assert_matches::assert_matches; +use crate::cell::RefCell; +use crate::test_helpers::test_rng; +use rand::Rng; +use realstd::collections::TryReserveErrorKind::*; + +// https://github.com/rust-lang/rust/issues/62301 +fn _assert_hashmap_is_unwind_safe() { + fn assert_unwind_safe() {} + assert_unwind_safe::>>(); +} + +#[test] +fn test_zero_capacities() { + type HM = HashMap; + + let m = HM::new(); + assert_eq!(m.capacity(), 0); + + let m = HM::default(); + assert_eq!(m.capacity(), 0); + + let m = HM::with_hasher(RandomState::new()); + assert_eq!(m.capacity(), 0); + + let m = HM::with_capacity(0); + assert_eq!(m.capacity(), 0); + + let m = HM::with_capacity_and_hasher(0, RandomState::new()); + assert_eq!(m.capacity(), 0); + + let mut m = HM::new(); + m.insert(1, 1); + m.insert(2, 2); + m.remove(&1); + m.remove(&2); + m.shrink_to_fit(); + assert_eq!(m.capacity(), 0); + + let mut m = HM::new(); + m.reserve(0); + assert_eq!(m.capacity(), 0); +} + +#[test] +fn test_create_capacity_zero() { + let mut m = HashMap::with_capacity(0); + + assert!(m.insert(1, 1).is_none()); + + assert!(m.contains_key(&1)); + assert!(!m.contains_key(&0)); +} + +#[test] +fn test_insert() { + let mut m = HashMap::new(); + assert_eq!(m.len(), 0); + assert!(m.insert(1, 2).is_none()); + assert_eq!(m.len(), 1); + assert!(m.insert(2, 4).is_none()); + assert_eq!(m.len(), 2); + assert_eq!(*m.get(&1).unwrap(), 2); + assert_eq!(*m.get(&2).unwrap(), 4); +} + +#[test] +fn test_clone() { + let mut m = HashMap::new(); + assert_eq!(m.len(), 0); + assert!(m.insert(1, 2).is_none()); + assert_eq!(m.len(), 1); + assert!(m.insert(2, 4).is_none()); + assert_eq!(m.len(), 2); + let m2 = m.clone(); + assert_eq!(*m2.get(&1).unwrap(), 2); + assert_eq!(*m2.get(&2).unwrap(), 4); + assert_eq!(m2.len(), 2); +} + +thread_local! { static DROP_VECTOR: RefCell> = RefCell::new(Vec::new()) } + +#[derive(Hash, PartialEq, Eq)] +struct Droppable { + k: usize, +} + +impl Droppable { + fn new(k: usize) -> Droppable { + DROP_VECTOR.with(|slot| { + slot.borrow_mut()[k] += 1; + }); + + Droppable { k } + } +} + +impl Drop for Droppable { + fn drop(&mut self) { + DROP_VECTOR.with(|slot| { + slot.borrow_mut()[self.k] -= 1; + }); + } +} + +impl Clone for Droppable { + fn clone(&self) -> Droppable { + Droppable::new(self.k) + } +} + +#[test] +fn test_drops() { + DROP_VECTOR.with(|slot| { + *slot.borrow_mut() = vec![0; 200]; + }); + + { + let mut m = HashMap::new(); + + DROP_VECTOR.with(|v| { + for i in 0..200 { + assert_eq!(v.borrow()[i], 0); + } + }); + + for i in 0..100 { + let d1 = Droppable::new(i); + let d2 = Droppable::new(i + 100); + m.insert(d1, d2); + } + + DROP_VECTOR.with(|v| { + for i in 0..200 { + assert_eq!(v.borrow()[i], 1); + } + }); + + for i in 0..50 { + let k = Droppable::new(i); + let v = m.remove(&k); + + assert!(v.is_some()); + + DROP_VECTOR.with(|v| { + assert_eq!(v.borrow()[i], 1); + assert_eq!(v.borrow()[i + 100], 1); + }); + } + + DROP_VECTOR.with(|v| { + for i in 0..50 { + assert_eq!(v.borrow()[i], 0); + assert_eq!(v.borrow()[i + 100], 0); + } + + for i in 50..100 { + assert_eq!(v.borrow()[i], 1); + assert_eq!(v.borrow()[i + 100], 1); + } + }); + } + + DROP_VECTOR.with(|v| { + for i in 0..200 { + assert_eq!(v.borrow()[i], 0); + } + }); +} + +#[test] +fn test_into_iter_drops() { + DROP_VECTOR.with(|v| { + *v.borrow_mut() = vec![0; 200]; + }); + + let hm = { + let mut hm = HashMap::new(); + + DROP_VECTOR.with(|v| { + for i in 0..200 { + assert_eq!(v.borrow()[i], 0); + } + }); + + for i in 0..100 { + let d1 = Droppable::new(i); + let d2 = Droppable::new(i + 100); + hm.insert(d1, d2); + } + + DROP_VECTOR.with(|v| { + for i in 0..200 { + assert_eq!(v.borrow()[i], 1); + } + }); + + hm + }; + + // By the way, ensure that cloning doesn't screw up the dropping. + drop(hm.clone()); + + { + let mut half = hm.into_iter().take(50); + + DROP_VECTOR.with(|v| { + for i in 0..200 { + assert_eq!(v.borrow()[i], 1); + } + }); + + for _ in half.by_ref() {} + + DROP_VECTOR.with(|v| { + let nk = (0..100).filter(|&i| v.borrow()[i] == 1).count(); + + let nv = (0..100).filter(|&i| v.borrow()[i + 100] == 1).count(); + + assert_eq!(nk, 50); + assert_eq!(nv, 50); + }); + }; + + DROP_VECTOR.with(|v| { + for i in 0..200 { + assert_eq!(v.borrow()[i], 0); + } + }); +} + +#[test] +fn test_empty_remove() { + let mut m: HashMap = HashMap::new(); + assert_eq!(m.remove(&0), None); +} + +#[test] +fn test_empty_entry() { + let mut m: HashMap = HashMap::new(); + match m.entry(0) { + Occupied(_) => panic!(), + Vacant(_) => {} + } + assert!(*m.entry(0).or_insert(true)); + assert_eq!(m.len(), 1); +} + +#[test] +fn test_empty_iter() { + let mut m: HashMap = HashMap::new(); + assert_eq!(m.drain().next(), None); + assert_eq!(m.keys().next(), None); + assert_eq!(m.values().next(), None); + assert_eq!(m.values_mut().next(), None); + assert_eq!(m.iter().next(), None); + assert_eq!(m.iter_mut().next(), None); + assert_eq!(m.len(), 0); + assert!(m.is_empty()); + assert_eq!(m.into_iter().next(), None); +} + +#[test] +fn test_lots_of_insertions() { + let mut m = HashMap::new(); + + // Try this a few times to make sure we never screw up the hashmap's + // internal state. + let loops = if cfg!(miri) { 2 } else { 10 }; + for _ in 0..loops { + assert!(m.is_empty()); + + let count = if cfg!(miri) { 101 } else { 1001 }; + + for i in 1..count { + assert!(m.insert(i, i).is_none()); + + for j in 1..=i { + let r = m.get(&j); + assert_eq!(r, Some(&j)); + } + + for j in i + 1..count { + let r = m.get(&j); + assert_eq!(r, None); + } + } + + for i in count..(2 * count) { + assert!(!m.contains_key(&i)); + } + + // remove forwards + for i in 1..count { + assert!(m.remove(&i).is_some()); + + for j in 1..=i { + assert!(!m.contains_key(&j)); + } + + for j in i + 1..count { + assert!(m.contains_key(&j)); + } + } + + for i in 1..count { + assert!(!m.contains_key(&i)); + } + + for i in 1..count { + assert!(m.insert(i, i).is_none()); + } + + // remove backwards + for i in (1..count).rev() { + assert!(m.remove(&i).is_some()); + + for j in i..count { + assert!(!m.contains_key(&j)); + } + + for j in 1..i { + assert!(m.contains_key(&j)); + } + } + } +} + +#[test] +fn test_find_mut() { + let mut m = HashMap::new(); + assert!(m.insert(1, 12).is_none()); + assert!(m.insert(2, 8).is_none()); + assert!(m.insert(5, 14).is_none()); + let new = 100; + match m.get_mut(&5) { + None => panic!(), + Some(x) => *x = new, + } + assert_eq!(m.get(&5), Some(&new)); +} + +#[test] +fn test_insert_overwrite() { + let mut m = HashMap::new(); + assert!(m.insert(1, 2).is_none()); + assert_eq!(*m.get(&1).unwrap(), 2); + assert!(!m.insert(1, 3).is_none()); + assert_eq!(*m.get(&1).unwrap(), 3); +} + +#[test] +fn test_insert_conflicts() { + let mut m = HashMap::with_capacity(4); + assert!(m.insert(1, 2).is_none()); + assert!(m.insert(5, 3).is_none()); + assert!(m.insert(9, 4).is_none()); + assert_eq!(*m.get(&9).unwrap(), 4); + assert_eq!(*m.get(&5).unwrap(), 3); + assert_eq!(*m.get(&1).unwrap(), 2); +} + +#[test] +fn test_conflict_remove() { + let mut m = HashMap::with_capacity(4); + assert!(m.insert(1, 2).is_none()); + assert_eq!(*m.get(&1).unwrap(), 2); + assert!(m.insert(5, 3).is_none()); + assert_eq!(*m.get(&1).unwrap(), 2); + assert_eq!(*m.get(&5).unwrap(), 3); + assert!(m.insert(9, 4).is_none()); + assert_eq!(*m.get(&1).unwrap(), 2); + assert_eq!(*m.get(&5).unwrap(), 3); + assert_eq!(*m.get(&9).unwrap(), 4); + assert!(m.remove(&1).is_some()); + assert_eq!(*m.get(&9).unwrap(), 4); + assert_eq!(*m.get(&5).unwrap(), 3); +} + +#[test] +fn test_is_empty() { + let mut m = HashMap::with_capacity(4); + assert!(m.insert(1, 2).is_none()); + assert!(!m.is_empty()); + assert!(m.remove(&1).is_some()); + assert!(m.is_empty()); +} + +#[test] +fn test_remove() { + let mut m = HashMap::new(); + m.insert(1, 2); + assert_eq!(m.remove(&1), Some(2)); + assert_eq!(m.remove(&1), None); +} + +#[test] +fn test_remove_entry() { + let mut m = HashMap::new(); + m.insert(1, 2); + assert_eq!(m.remove_entry(&1), Some((1, 2))); + assert_eq!(m.remove(&1), None); +} + +#[test] +fn test_iterate() { + let mut m = HashMap::with_capacity(4); + for i in 0..32 { + assert!(m.insert(i, i * 2).is_none()); + } + assert_eq!(m.len(), 32); + + let mut observed: u32 = 0; + + for (k, v) in &m { + assert_eq!(*v, *k * 2); + observed |= 1 << *k; + } + assert_eq!(observed, 0xFFFF_FFFF); +} + +#[test] +fn test_keys() { + let pairs = [(1, 'a'), (2, 'b'), (3, 'c')]; + let map: HashMap<_, _> = pairs.into_iter().collect(); + let keys: Vec<_> = map.keys().cloned().collect(); + assert_eq!(keys.len(), 3); + assert!(keys.contains(&1)); + assert!(keys.contains(&2)); + assert!(keys.contains(&3)); +} + +#[test] +fn test_values() { + let pairs = [(1, 'a'), (2, 'b'), (3, 'c')]; + let map: HashMap<_, _> = pairs.into_iter().collect(); + let values: Vec<_> = map.values().cloned().collect(); + assert_eq!(values.len(), 3); + assert!(values.contains(&'a')); + assert!(values.contains(&'b')); + assert!(values.contains(&'c')); +} + +#[test] +fn test_values_mut() { + let pairs = [(1, 1), (2, 2), (3, 3)]; + let mut map: HashMap<_, _> = pairs.into_iter().collect(); + for value in map.values_mut() { + *value = (*value) * 2 + } + let values: Vec<_> = map.values().cloned().collect(); + assert_eq!(values.len(), 3); + assert!(values.contains(&2)); + assert!(values.contains(&4)); + assert!(values.contains(&6)); +} + +#[test] +fn test_into_keys() { + let pairs = [(1, 'a'), (2, 'b'), (3, 'c')]; + let map: HashMap<_, _> = pairs.into_iter().collect(); + let keys: Vec<_> = map.into_keys().collect(); + + assert_eq!(keys.len(), 3); + assert!(keys.contains(&1)); + assert!(keys.contains(&2)); + assert!(keys.contains(&3)); +} + +#[test] +fn test_into_values() { + let pairs = [(1, 'a'), (2, 'b'), (3, 'c')]; + let map: HashMap<_, _> = pairs.into_iter().collect(); + let values: Vec<_> = map.into_values().collect(); + + assert_eq!(values.len(), 3); + assert!(values.contains(&'a')); + assert!(values.contains(&'b')); + assert!(values.contains(&'c')); +} + +#[test] +fn test_find() { + let mut m = HashMap::new(); + assert!(m.get(&1).is_none()); + m.insert(1, 2); + match m.get(&1) { + None => panic!(), + Some(v) => assert_eq!(*v, 2), + } +} + +#[test] +fn test_eq() { + let mut m1 = HashMap::new(); + m1.insert(1, 2); + m1.insert(2, 3); + m1.insert(3, 4); + + let mut m2 = HashMap::new(); + m2.insert(1, 2); + m2.insert(2, 3); + + assert!(m1 != m2); + + m2.insert(3, 4); + + assert_eq!(m1, m2); +} + +#[test] +fn test_show() { + let mut map = HashMap::new(); + let empty: HashMap = HashMap::new(); + + map.insert(1, 2); + map.insert(3, 4); + + let map_str = format!("{map:?}"); + + assert!(map_str == "{1: 2, 3: 4}" || map_str == "{3: 4, 1: 2}"); + assert_eq!(format!("{empty:?}"), "{}"); +} + +#[test] +fn test_reserve_shrink_to_fit() { + let mut m = HashMap::new(); + m.insert(0, 0); + m.remove(&0); + assert!(m.capacity() >= m.len()); + for i in 0..128 { + m.insert(i, i); + } + m.reserve(256); + + let usable_cap = m.capacity(); + for i in 128..(128 + 256) { + m.insert(i, i); + assert_eq!(m.capacity(), usable_cap); + } + + for i in 100..(128 + 256) { + assert_eq!(m.remove(&i), Some(i)); + } + m.shrink_to_fit(); + + assert_eq!(m.len(), 100); + assert!(!m.is_empty()); + assert!(m.capacity() >= m.len()); + + for i in 0..100 { + assert_eq!(m.remove(&i), Some(i)); + } + m.shrink_to_fit(); + m.insert(0, 0); + + assert_eq!(m.len(), 1); + assert!(m.capacity() >= m.len()); + assert_eq!(m.remove(&0), Some(0)); +} + +#[test] +fn test_from_iter() { + let xs = [(1, 1), (2, 2), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]; + + let map: HashMap<_, _> = xs.iter().cloned().collect(); + + for &(k, v) in &xs { + assert_eq!(map.get(&k), Some(&v)); + } + + assert_eq!(map.iter().len(), xs.len() - 1); +} + +#[test] +fn test_size_hint() { + let xs = [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]; + + let map: HashMap<_, _> = xs.iter().cloned().collect(); + + let mut iter = map.iter(); + + for _ in iter.by_ref().take(3) {} + + assert_eq!(iter.size_hint(), (3, Some(3))); +} + +#[test] +fn test_iter_len() { + let xs = [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]; + + let map: HashMap<_, _> = xs.iter().cloned().collect(); + + let mut iter = map.iter(); + + for _ in iter.by_ref().take(3) {} + + assert_eq!(iter.len(), 3); +} + +#[test] +fn test_mut_size_hint() { + let xs = [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]; + + let mut map: HashMap<_, _> = xs.iter().cloned().collect(); + + let mut iter = map.iter_mut(); + + for _ in iter.by_ref().take(3) {} + + assert_eq!(iter.size_hint(), (3, Some(3))); +} + +#[test] +fn test_iter_mut_len() { + let xs = [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]; + + let mut map: HashMap<_, _> = xs.iter().cloned().collect(); + + let mut iter = map.iter_mut(); + + for _ in iter.by_ref().take(3) {} + + assert_eq!(iter.len(), 3); +} + +#[test] +fn test_index() { + let mut map = HashMap::new(); + + map.insert(1, 2); + map.insert(2, 1); + map.insert(3, 4); + + assert_eq!(map[&2], 1); +} + +#[test] +#[should_panic] +fn test_index_nonexistent() { + let mut map = HashMap::new(); + + map.insert(1, 2); + map.insert(2, 1); + map.insert(3, 4); + + map[&4]; +} + +#[test] +fn test_entry() { + let xs = [(1, 10), (2, 20), (3, 30), (4, 40), (5, 50), (6, 60)]; + + let mut map: HashMap<_, _> = xs.iter().cloned().collect(); + + // Existing key (insert) + match map.entry(1) { + Vacant(_) => unreachable!(), + Occupied(mut view) => { + assert_eq!(view.get(), &10); + assert_eq!(view.insert(100), 10); + } + } + assert_eq!(map.get(&1).unwrap(), &100); + assert_eq!(map.len(), 6); + + // Existing key (update) + match map.entry(2) { + Vacant(_) => unreachable!(), + Occupied(mut view) => { + let v = view.get_mut(); + let new_v = (*v) * 10; + *v = new_v; + } + } + assert_eq!(map.get(&2).unwrap(), &200); + assert_eq!(map.len(), 6); + + // Existing key (take) + match map.entry(3) { + Vacant(_) => unreachable!(), + Occupied(view) => { + assert_eq!(view.remove(), 30); + } + } + assert_eq!(map.get(&3), None); + assert_eq!(map.len(), 5); + + // Inexistent key (insert) + match map.entry(10) { + Occupied(_) => unreachable!(), + Vacant(view) => { + assert_eq!(*view.insert(1000), 1000); + } + } + assert_eq!(map.get(&10).unwrap(), &1000); + assert_eq!(map.len(), 6); +} + +#[test] +fn test_entry_take_doesnt_corrupt() { + #![allow(deprecated)] //rand + // Test for #19292 + fn check(m: &HashMap) { + for k in m.keys() { + assert!(m.contains_key(k), "{k} is in keys() but not in the map?"); + } + } + + let mut m = HashMap::new(); + let mut rng = test_rng(); + + // Populate the map with some items. + for _ in 0..50 { + let x = rng.gen_range(-10..10); + m.insert(x, ()); + } + + for _ in 0..1000 { + let x = rng.gen_range(-10..10); + match m.entry(x) { + Vacant(_) => {} + Occupied(e) => { + e.remove(); + } + } + + check(&m); + } +} + +#[test] +fn test_extend_ref() { + let mut a = HashMap::new(); + a.insert(1, "one"); + let mut b = HashMap::new(); + b.insert(2, "two"); + b.insert(3, "three"); + + a.extend(&b); + + assert_eq!(a.len(), 3); + assert_eq!(a[&1], "one"); + assert_eq!(a[&2], "two"); + assert_eq!(a[&3], "three"); +} + +#[test] +fn test_capacity_not_less_than_len() { + let mut a = HashMap::new(); + let mut item = 0; + + for _ in 0..116 { + a.insert(item, 0); + item += 1; + } + + assert!(a.capacity() > a.len()); + + let free = a.capacity() - a.len(); + for _ in 0..free { + a.insert(item, 0); + item += 1; + } + + assert_eq!(a.len(), a.capacity()); + + // Insert at capacity should cause allocation. + a.insert(item, 0); + assert!(a.capacity() > a.len()); +} + +#[test] +fn test_occupied_entry_key() { + let mut a = HashMap::new(); + let key = "hello there"; + let value = "value goes here"; + assert!(a.is_empty()); + a.insert(key, value); + assert_eq!(a.len(), 1); + assert_eq!(a[key], value); + + match a.entry(key) { + Vacant(_) => panic!(), + Occupied(e) => assert_eq!(key, *e.key()), + } + assert_eq!(a.len(), 1); + assert_eq!(a[key], value); +} + +#[test] +fn test_vacant_entry_key() { + let mut a = HashMap::new(); + let key = "hello there"; + let value = "value goes here"; + + assert!(a.is_empty()); + match a.entry(key) { + Occupied(_) => panic!(), + Vacant(e) => { + assert_eq!(key, *e.key()); + e.insert(value); + } + } + assert_eq!(a.len(), 1); + assert_eq!(a[key], value); +} + +#[test] +fn test_retain() { + let mut map: HashMap = (0..100).map(|x| (x, x * 10)).collect(); + + map.retain(|&k, _| k % 2 == 0); + assert_eq!(map.len(), 50); + assert_eq!(map[&2], 20); + assert_eq!(map[&4], 40); + assert_eq!(map[&6], 60); +} + +#[test] +#[cfg_attr(miri, ignore)] // Miri does not support signalling OOM +#[cfg_attr(target_os = "android", ignore)] // Android used in CI has a broken dlmalloc +fn test_try_reserve() { + let mut empty_bytes: HashMap = HashMap::new(); + + const MAX_USIZE: usize = usize::MAX; + + assert_matches!( + empty_bytes.try_reserve(MAX_USIZE).map_err(|e| e.kind()), + Err(CapacityOverflow), + "usize::MAX should trigger an overflow!" + ); + + if let Err(AllocError { .. }) = empty_bytes.try_reserve(MAX_USIZE / 16).map_err(|e| e.kind()) { + } else { + // This may succeed if there is enough free memory. Attempt to + // allocate a few more hashmaps to ensure the allocation will fail. + let mut empty_bytes2: HashMap = HashMap::new(); + let _ = empty_bytes2.try_reserve(MAX_USIZE / 16); + let mut empty_bytes3: HashMap = HashMap::new(); + let _ = empty_bytes3.try_reserve(MAX_USIZE / 16); + let mut empty_bytes4: HashMap = HashMap::new(); + assert_matches!( + empty_bytes4.try_reserve(MAX_USIZE / 16).map_err(|e| e.kind()), + Err(AllocError { .. }), + "usize::MAX / 16 should trigger an OOM!" + ); + } +} + +#[test] +fn test_raw_entry() { + use super::RawEntryMut::{Occupied, Vacant}; + + let xs = [(1i32, 10i32), (2, 20), (3, 30), (4, 40), (5, 50), (6, 60)]; + + let mut map: HashMap<_, _> = xs.iter().cloned().collect(); + + let compute_hash = |map: &HashMap, k: i32| -> u64 { + use core::hash::{BuildHasher, Hash, Hasher}; + + let mut hasher = map.hasher().build_hasher(); + k.hash(&mut hasher); + hasher.finish() + }; + + // Existing key (insert) + match map.raw_entry_mut().from_key(&1) { + Vacant(_) => unreachable!(), + Occupied(mut view) => { + assert_eq!(view.get(), &10); + assert_eq!(view.insert(100), 10); + } + } + let hash1 = compute_hash(&map, 1); + assert_eq!(map.raw_entry().from_key(&1).unwrap(), (&1, &100)); + assert_eq!(map.raw_entry().from_hash(hash1, |k| *k == 1).unwrap(), (&1, &100)); + assert_eq!(map.raw_entry().from_key_hashed_nocheck(hash1, &1).unwrap(), (&1, &100)); + assert_eq!(map.len(), 6); + + // Existing key (update) + match map.raw_entry_mut().from_key(&2) { + Vacant(_) => unreachable!(), + Occupied(mut view) => { + let v = view.get_mut(); + let new_v = (*v) * 10; + *v = new_v; + } + } + let hash2 = compute_hash(&map, 2); + assert_eq!(map.raw_entry().from_key(&2).unwrap(), (&2, &200)); + assert_eq!(map.raw_entry().from_hash(hash2, |k| *k == 2).unwrap(), (&2, &200)); + assert_eq!(map.raw_entry().from_key_hashed_nocheck(hash2, &2).unwrap(), (&2, &200)); + assert_eq!(map.len(), 6); + + // Existing key (take) + let hash3 = compute_hash(&map, 3); + match map.raw_entry_mut().from_key_hashed_nocheck(hash3, &3) { + Vacant(_) => unreachable!(), + Occupied(view) => { + assert_eq!(view.remove_entry(), (3, 30)); + } + } + assert_eq!(map.raw_entry().from_key(&3), None); + assert_eq!(map.raw_entry().from_hash(hash3, |k| *k == 3), None); + assert_eq!(map.raw_entry().from_key_hashed_nocheck(hash3, &3), None); + assert_eq!(map.len(), 5); + + // Nonexistent key (insert) + match map.raw_entry_mut().from_key(&10) { + Occupied(_) => unreachable!(), + Vacant(view) => { + assert_eq!(view.insert(10, 1000), (&mut 10, &mut 1000)); + } + } + assert_eq!(map.raw_entry().from_key(&10).unwrap(), (&10, &1000)); + assert_eq!(map.len(), 6); + + // Ensure all lookup methods produce equivalent results. + for k in 0..12 { + let hash = compute_hash(&map, k); + let v = map.get(&k).cloned(); + let kv = v.as_ref().map(|v| (&k, v)); + + assert_eq!(map.raw_entry().from_key(&k), kv); + assert_eq!(map.raw_entry().from_hash(hash, |q| *q == k), kv); + assert_eq!(map.raw_entry().from_key_hashed_nocheck(hash, &k), kv); + + match map.raw_entry_mut().from_key(&k) { + Occupied(mut o) => assert_eq!(Some(o.get_key_value()), kv), + Vacant(_) => assert_eq!(v, None), + } + match map.raw_entry_mut().from_key_hashed_nocheck(hash, &k) { + Occupied(mut o) => assert_eq!(Some(o.get_key_value()), kv), + Vacant(_) => assert_eq!(v, None), + } + match map.raw_entry_mut().from_hash(hash, |q| *q == k) { + Occupied(mut o) => assert_eq!(Some(o.get_key_value()), kv), + Vacant(_) => assert_eq!(v, None), + } + } +} + +mod test_drain_filter { + use super::*; + + use crate::panic::{catch_unwind, AssertUnwindSafe}; + use crate::sync::atomic::{AtomicUsize, Ordering}; + + trait EqSorted: Iterator { + fn eq_sorted>(self, other: I) -> bool; + } + + impl EqSorted for T + where + T::Item: Eq + Ord, + { + fn eq_sorted>(self, other: I) -> bool { + let mut v: Vec<_> = self.collect(); + v.sort_unstable(); + v.into_iter().eq(other) + } + } + + #[test] + fn empty() { + let mut map: HashMap = HashMap::new(); + map.drain_filter(|_, _| unreachable!("there's nothing to decide on")); + assert!(map.is_empty()); + } + + #[test] + fn consuming_nothing() { + let pairs = (0..3).map(|i| (i, i)); + let mut map: HashMap<_, _> = pairs.collect(); + assert!(map.drain_filter(|_, _| false).eq_sorted(crate::iter::empty())); + assert_eq!(map.len(), 3); + } + + #[test] + fn consuming_all() { + let pairs = (0..3).map(|i| (i, i)); + let mut map: HashMap<_, _> = pairs.clone().collect(); + assert!(map.drain_filter(|_, _| true).eq_sorted(pairs)); + assert!(map.is_empty()); + } + + #[test] + fn mutating_and_keeping() { + let pairs = (0..3).map(|i| (i, i)); + let mut map: HashMap<_, _> = pairs.collect(); + assert!( + map.drain_filter(|_, v| { + *v += 6; + false + }) + .eq_sorted(crate::iter::empty()) + ); + assert!(map.keys().copied().eq_sorted(0..3)); + assert!(map.values().copied().eq_sorted(6..9)); + } + + #[test] + fn mutating_and_removing() { + let pairs = (0..3).map(|i| (i, i)); + let mut map: HashMap<_, _> = pairs.collect(); + assert!( + map.drain_filter(|_, v| { + *v += 6; + true + }) + .eq_sorted((0..3).map(|i| (i, i + 6))) + ); + assert!(map.is_empty()); + } + + #[test] + fn drop_panic_leak() { + static PREDS: AtomicUsize = AtomicUsize::new(0); + static DROPS: AtomicUsize = AtomicUsize::new(0); + + struct D; + impl Drop for D { + fn drop(&mut self) { + if DROPS.fetch_add(1, Ordering::SeqCst) == 1 { + panic!("panic in `drop`"); + } + } + } + + let mut map = (0..3).map(|i| (i, D)).collect::>(); + + catch_unwind(move || { + drop(map.drain_filter(|_, _| { + PREDS.fetch_add(1, Ordering::SeqCst); + true + })) + }) + .unwrap_err(); + + assert_eq!(PREDS.load(Ordering::SeqCst), 3); + assert_eq!(DROPS.load(Ordering::SeqCst), 3); + } + + #[test] + fn pred_panic_leak() { + static PREDS: AtomicUsize = AtomicUsize::new(0); + static DROPS: AtomicUsize = AtomicUsize::new(0); + + struct D; + impl Drop for D { + fn drop(&mut self) { + DROPS.fetch_add(1, Ordering::SeqCst); + } + } + + let mut map = (0..3).map(|i| (i, D)).collect::>(); + + catch_unwind(AssertUnwindSafe(|| { + drop(map.drain_filter(|_, _| match PREDS.fetch_add(1, Ordering::SeqCst) { + 0 => true, + _ => panic!(), + })) + })) + .unwrap_err(); + + assert_eq!(PREDS.load(Ordering::SeqCst), 2); + assert_eq!(DROPS.load(Ordering::SeqCst), 1); + assert_eq!(map.len(), 2); + } + + // Same as above, but attempt to use the iterator again after the panic in the predicate + #[test] + fn pred_panic_reuse() { + static PREDS: AtomicUsize = AtomicUsize::new(0); + static DROPS: AtomicUsize = AtomicUsize::new(0); + + struct D; + impl Drop for D { + fn drop(&mut self) { + DROPS.fetch_add(1, Ordering::SeqCst); + } + } + + let mut map = (0..3).map(|i| (i, D)).collect::>(); + + { + let mut it = map.drain_filter(|_, _| match PREDS.fetch_add(1, Ordering::SeqCst) { + 0 => true, + _ => panic!(), + }); + catch_unwind(AssertUnwindSafe(|| while it.next().is_some() {})).unwrap_err(); + // Iterator behaviour after a panic is explicitly unspecified, + // so this is just the current implementation: + let result = catch_unwind(AssertUnwindSafe(|| it.next())); + assert!(result.is_err()); + } + + assert_eq!(PREDS.load(Ordering::SeqCst), 3); + assert_eq!(DROPS.load(Ordering::SeqCst), 1); + assert_eq!(map.len(), 2); + } +} + +#[test] +fn from_array() { + let map = HashMap::from([(1, 2), (3, 4)]); + let unordered_duplicates = HashMap::from([(3, 4), (1, 2), (1, 2)]); + assert_eq!(map, unordered_duplicates); + + // This next line must infer the hasher type parameter. + // If you make a change that causes this line to no longer infer, + // that's a problem! + let _must_not_require_type_annotation = HashMap::from([(1, 2)]); +} + +#[test] +fn const_with_hasher() { + const X: HashMap<(), (), ()> = HashMap::with_hasher(()); + assert_eq!(X.len(), 0); +} diff --git a/crux-mir/lib/std/src/collections/hash/mod.rs b/crux-mir/lib/std/src/collections/hash/mod.rs index 7d37d0f72..348820af5 100644 --- a/crux-mir/lib/std/src/collections/hash/mod.rs +++ b/crux-mir/lib/std/src/collections/hash/mod.rs @@ -2,5 +2,3 @@ pub mod map; pub mod set; - -mod crucible_map; diff --git a/crux-mir/lib/std/src/collections/hash/set.rs b/crux-mir/lib/std/src/collections/hash/set.rs index ad3dc9c92..b59f89d32 100644 --- a/crux-mir/lib/std/src/collections/hash/set.rs +++ b/crux-mir/lib/std/src/collections/hash/set.rs @@ -1,11 +1,16 @@ +#[cfg(test)] +mod tests; + +use hashbrown::hash_set as base; + use crate::borrow::Borrow; use crate::collections::TryReserveError; use crate::fmt; use crate::hash::{BuildHasher, Hash}; -use crate::iter::{Chain, FromIterator, FusedIterator}; +use crate::iter::{Chain, FusedIterator}; use crate::ops::{BitAnd, BitOr, BitXor, Sub}; -use super::map::{self, HashMap, Keys, RandomState}; +use super::map::{map_try_reserve_error, RandomState}; // Future Optimization (FIXME!) // ============================ @@ -14,7 +19,7 @@ use super::map::{self, HashMap, Keys, RandomState}; // for `bucket.val` in the case of HashSet. I suppose we would need HKT // to get rid of it properly. -/// A hash set implemented as a `HashMap` where the value is `()`. +/// A [hash set] implemented as a `HashMap` where the value is `()`. /// /// As with the [`HashMap`] type, a `HashSet` requires that the elements /// implement the [`Eq`] and [`Hash`] traits. This can frequently be achieved by @@ -28,11 +33,14 @@ use super::map::{self, HashMap, Keys, RandomState}; /// In other words, if two keys are equal, their hashes must be equal. /// /// -/// It is a logic error for an item to be modified in such a way that the -/// item's hash, as determined by the [`Hash`] trait, or its equality, as -/// determined by the [`Eq`] trait, changes while it is in the set. This is -/// normally only possible through [`Cell`], [`RefCell`], global state, I/O, or -/// unsafe code. +/// It is a logic error for a key to be modified in such a way that the key's +/// hash, as determined by the [`Hash`] trait, or its equality, as determined by +/// the [`Eq`] trait, changes while it is in the map. This is normally only +/// possible through [`Cell`], [`RefCell`], global state, I/O, or unsafe code. +/// The behavior resulting from such a logic error is not specified, but will +/// be encapsulated to the `HashSet` that observed the logic error and not +/// result in undefined behavior. This could include panics, incorrect results, +/// aborts, memory leaks, and non-termination. /// /// # Examples /// @@ -59,7 +67,7 @@ use super::map::{self, HashMap, Keys, RandomState}; /// /// // Iterate over everything. /// for book in &books { -/// println!("{}", book); +/// println!("{book}"); /// } /// ``` /// @@ -84,30 +92,26 @@ use super::map::{self, HashMap, Keys, RandomState}; /// /// // Use derived implementation to print the vikings. /// for x in &vikings { -/// println!("{:?}", x); +/// println!("{x:?}"); /// } /// ``` /// -/// A `HashSet` with fixed list of elements can be initialized from an array: +/// A `HashSet` with a known list of items can be initialized from an array: /// /// ``` /// use std::collections::HashSet; /// -/// let viking_names: HashSet<&'static str> = -/// [ "Einar", "Olaf", "Harald" ].iter().cloned().collect(); -/// // use the values stored in the set +/// let viking_names = HashSet::from(["Einar", "Olaf", "Harald"]); /// ``` /// -/// [`Cell`]: ../../std/cell/struct.Cell.html -/// [`Eq`]: ../../std/cmp/trait.Eq.html -/// [`Hash`]: ../../std/hash/trait.Hash.html -/// [`HashMap`]: struct.HashMap.html -/// [`PartialEq`]: ../../std/cmp/trait.PartialEq.html -/// [`RefCell`]: ../../std/cell/struct.RefCell.html -#[derive(Clone)] +/// [hash set]: crate::collections#use-the-set-variant-of-any-of-these-maps-when +/// [`HashMap`]: crate::collections::HashMap +/// [`RefCell`]: crate::cell::RefCell +/// [`Cell`]: crate::cell::Cell +#[cfg_attr(not(test), rustc_diagnostic_item = "HashSet")] #[stable(feature = "rust1", since = "1.0.0")] pub struct HashSet { - map: HashMap, + base: base::HashSet, } impl HashSet { @@ -123,15 +127,17 @@ impl HashSet { /// let set: HashSet = HashSet::new(); /// ``` #[inline] + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn new() -> HashSet { - HashSet { map: HashMap::new() } + Default::default() } - /// Creates an empty `HashSet` with the specified capacity. + /// Creates an empty `HashSet` with at least the specified capacity. /// /// The hash set will be able to hold at least `capacity` elements without - /// reallocating. If `capacity` is 0, the hash set will not allocate. + /// reallocating. This method is allowed to allocate for more elements than + /// `capacity`. If `capacity` is 0, the hash set will not allocate. /// /// # Examples /// @@ -141,9 +147,10 @@ impl HashSet { /// assert!(set.capacity() >= 10); /// ``` #[inline] + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn with_capacity(capacity: usize) -> HashSet { - HashSet { map: HashMap::with_capacity(capacity) } + HashSet { base: base::HashSet::with_capacity_and_hasher(capacity, Default::default()) } } } @@ -160,7 +167,7 @@ impl HashSet { #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn capacity(&self) -> usize { - self.map.capacity() + self.base.capacity() } /// An iterator visiting all elements in arbitrary order. @@ -176,13 +183,19 @@ impl HashSet { /// /// // Will print in an arbitrary order. /// for x in set.iter() { - /// println!("{}", x); + /// println!("{x}"); /// } /// ``` + /// + /// # Performance + /// + /// In the current implementation, iterating over set takes O(capacity) time + /// instead of O(len) because it internally visits empty buckets too. #[inline] + #[rustc_lint_query_instability] #[stable(feature = "rust1", since = "1.0.0")] pub fn iter(&self) -> Iter<'_, T> { - Iter { iter: self.map.keys() } + Iter { base: self.base.iter() } } /// Returns the number of elements in the set. @@ -200,7 +213,7 @@ impl HashSet { #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn len(&self) -> usize { - self.map.len() + self.base.len() } /// Returns `true` if the set contains no elements. @@ -218,30 +231,106 @@ impl HashSet { #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn is_empty(&self) -> bool { - self.map.is_empty() + self.base.is_empty() } - /// Clears the set, returning all elements in an iterator. + /// Clears the set, returning all elements as an iterator. Keeps the + /// allocated memory for reuse. + /// + /// If the returned iterator is dropped before being fully consumed, it + /// drops the remaining elements. The returned iterator keeps a mutable + /// borrow on the set to optimize its implementation. /// /// # Examples /// /// ``` /// use std::collections::HashSet; /// - /// let mut set: HashSet<_> = [1, 2, 3].iter().cloned().collect(); + /// let mut set = HashSet::from([1, 2, 3]); /// assert!(!set.is_empty()); /// /// // print 1, 2, 3 in an arbitrary order /// for i in set.drain() { - /// println!("{}", i); + /// println!("{i}"); /// } /// /// assert!(set.is_empty()); /// ``` #[inline] + #[rustc_lint_query_instability] #[stable(feature = "drain", since = "1.6.0")] pub fn drain(&mut self) -> Drain<'_, T> { - Drain { iter: self.map.drain() } + Drain { base: self.base.drain() } + } + + /// Creates an iterator which uses a closure to determine if a value should be removed. + /// + /// If the closure returns true, then the value is removed and yielded. + /// If the closure returns false, the value will remain in the list and will not be yielded + /// by the iterator. + /// + /// If the iterator is only partially consumed or not consumed at all, each of the remaining + /// values will still be subjected to the closure and removed and dropped if it returns true. + /// + /// It is unspecified how many more values will be subjected to the closure + /// if a panic occurs in the closure, or if a panic occurs while dropping a value, or if the + /// `DrainFilter` itself is leaked. + /// + /// # Examples + /// + /// Splitting a set into even and odd values, reusing the original set: + /// + /// ``` + /// #![feature(hash_drain_filter)] + /// use std::collections::HashSet; + /// + /// let mut set: HashSet = (0..8).collect(); + /// let drained: HashSet = set.drain_filter(|v| v % 2 == 0).collect(); + /// + /// let mut evens = drained.into_iter().collect::>(); + /// let mut odds = set.into_iter().collect::>(); + /// evens.sort(); + /// odds.sort(); + /// + /// assert_eq!(evens, vec![0, 2, 4, 6]); + /// assert_eq!(odds, vec![1, 3, 5, 7]); + /// ``` + #[inline] + #[rustc_lint_query_instability] + #[unstable(feature = "hash_drain_filter", issue = "59618")] + pub fn drain_filter(&mut self, pred: F) -> DrainFilter<'_, T, F> + where + F: FnMut(&T) -> bool, + { + DrainFilter { base: self.base.drain_filter(pred) } + } + + /// Retains only the elements specified by the predicate. + /// + /// In other words, remove all elements `e` for which `f(&e)` returns `false`. + /// The elements are visited in unsorted (and unspecified) order. + /// + /// # Examples + /// + /// ``` + /// use std::collections::HashSet; + /// + /// let mut set = HashSet::from([1, 2, 3, 4, 5, 6]); + /// set.retain(|&k| k % 2 == 0); + /// assert_eq!(set, HashSet::from([2, 4, 6])); + /// ``` + /// + /// # Performance + /// + /// In the current implementation, this operation takes O(capacity) time + /// instead of O(len) because it internally visits empty buckets too. + #[rustc_lint_query_instability] + #[stable(feature = "retain_hash_collection", since = "1.18.0")] + pub fn retain(&mut self, f: F) + where + F: FnMut(&T) -> bool, + { + self.base.retain(f) } /// Clears the set, removing all values. @@ -259,7 +348,7 @@ impl HashSet { #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn clear(&mut self) { - self.map.clear() + self.base.clear() } /// Creates a new empty hash set which will use the given hasher to hash @@ -272,6 +361,9 @@ impl HashSet { /// cause many collisions and very poor performance. Setting it /// manually using this function can expose a DoS attack vector. /// + /// The `hash_builder` passed should implement the [`BuildHasher`] trait for + /// the HashMap to be useful, see its documentation for details. + /// /// # Examples /// /// ``` @@ -284,21 +376,26 @@ impl HashSet { /// ``` #[inline] #[stable(feature = "hashmap_build_hasher", since = "1.7.0")] - pub fn with_hasher(hasher: S) -> HashSet { - HashSet { map: HashMap::with_hasher(hasher) } + #[rustc_const_unstable(feature = "const_collections_with_hasher", issue = "102575")] + pub const fn with_hasher(hasher: S) -> HashSet { + HashSet { base: base::HashSet::with_hasher(hasher) } } - /// Creates an empty `HashSet` with the specified capacity, using + /// Creates an empty `HashSet` with at least the specified capacity, using /// `hasher` to hash the keys. /// /// The hash set will be able to hold at least `capacity` elements without - /// reallocating. If `capacity` is 0, the hash set will not allocate. + /// reallocating. This method is allowed to allocate for more elements than + /// `capacity`. If `capacity` is 0, the hash set will not allocate. /// /// Warning: `hasher` is normally randomly generated, and /// is designed to allow `HashSet`s to be resistant to attacks that /// cause many collisions and very poor performance. Setting it /// manually using this function can expose a DoS attack vector. /// + /// The `hash_builder` passed should implement the [`BuildHasher`] trait for + /// the HashMap to be useful, see its documentation for details. + /// /// # Examples /// /// ``` @@ -312,13 +409,11 @@ impl HashSet { #[inline] #[stable(feature = "hashmap_build_hasher", since = "1.7.0")] pub fn with_capacity_and_hasher(capacity: usize, hasher: S) -> HashSet { - HashSet { map: HashMap::with_capacity_and_hasher(capacity, hasher) } + HashSet { base: base::HashSet::with_capacity_and_hasher(capacity, hasher) } } /// Returns a reference to the set's [`BuildHasher`]. /// - /// [`BuildHasher`]: ../../std/hash/trait.BuildHasher.html - /// /// # Examples /// /// ``` @@ -332,7 +427,7 @@ impl HashSet { #[inline] #[stable(feature = "hashmap_public_hasher", since = "1.9.0")] pub fn hasher(&self) -> &S { - self.map.hasher() + self.base.hasher() } } @@ -342,8 +437,10 @@ where S: BuildHasher, { /// Reserves capacity for at least `additional` more elements to be inserted - /// in the `HashSet`. The collection may reserve more space to avoid - /// frequent reallocations. + /// in the `HashSet`. The collection may reserve more space to speculatively + /// avoid frequent reallocations. After calling `reserve`, + /// capacity will be greater than or equal to `self.len() + additional`. + /// Does nothing if capacity is already sufficient. /// /// # Panics /// @@ -360,12 +457,15 @@ where #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn reserve(&mut self, additional: usize) { - self.map.reserve(additional) + self.base.reserve(additional) } /// Tries to reserve capacity for at least `additional` more elements to be inserted - /// in the given `HashSet`. The collection may reserve more space to avoid - /// frequent reallocations. + /// in the `HashSet`. The collection may reserve more space to speculatively + /// avoid frequent reallocations. After calling `try_reserve`, + /// capacity will be greater than or equal to `self.len() + additional` if + /// it returns `Ok(())`. + /// Does nothing if capacity is already sufficient. /// /// # Errors /// @@ -375,15 +475,14 @@ where /// # Examples /// /// ``` - /// #![feature(try_reserve)] /// use std::collections::HashSet; /// let mut set: HashSet = HashSet::new(); - /// set.try_reserve(10).expect("why is the test harness OOMing on 10 bytes?"); + /// set.try_reserve(10).expect("why is the test harness OOMing on a handful of bytes?"); /// ``` #[inline] - #[unstable(feature = "try_reserve", reason = "new API", issue = "48043")] + #[stable(feature = "try_reserve", since = "1.57.0")] pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { - self.map.try_reserve(additional) + self.base.try_reserve(additional).map_err(map_try_reserve_error) } /// Shrinks the capacity of the set as much as possible. It will drop @@ -405,20 +504,17 @@ where #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn shrink_to_fit(&mut self) { - self.map.shrink_to_fit() + self.base.shrink_to_fit() } /// Shrinks the capacity of the set with a lower limit. It will drop /// down no lower than the supplied limit while maintaining the internal rules /// and possibly leaving some space in accordance with the resize policy. /// - /// Panics if the current capacity is smaller than the supplied - /// minimum capacity. - /// + /// If the current capacity is less than the lower limit, this is a no-op. /// # Examples /// /// ``` - /// #![feature(shrink_to)] /// use std::collections::HashSet; /// /// let mut set = HashSet::with_capacity(100); @@ -431,9 +527,9 @@ where /// assert!(set.capacity() >= 2); /// ``` #[inline] - #[unstable(feature = "shrink_to", reason = "new API", issue = "56431")] + #[stable(feature = "shrink_to", since = "1.56.0")] pub fn shrink_to(&mut self, min_capacity: usize) { - self.map.shrink_to(min_capacity) + self.base.shrink_to(min_capacity) } /// Visits the values representing the difference, @@ -443,12 +539,12 @@ where /// /// ``` /// use std::collections::HashSet; - /// let a: HashSet<_> = [1, 2, 3].iter().cloned().collect(); - /// let b: HashSet<_> = [4, 2, 3, 4].iter().cloned().collect(); + /// let a = HashSet::from([1, 2, 3]); + /// let b = HashSet::from([4, 2, 3, 4]); /// /// // Can be seen as `a - b`. /// for x in a.difference(&b) { - /// println!("{}", x); // Print 1 + /// println!("{x}"); // Print 1 /// } /// /// let diff: HashSet<_> = a.difference(&b).collect(); @@ -460,6 +556,7 @@ where /// assert_eq!(diff, [4].iter().collect()); /// ``` #[inline] + #[rustc_lint_query_instability] #[stable(feature = "rust1", since = "1.0.0")] pub fn difference<'a>(&'a self, other: &'a HashSet) -> Difference<'a, T, S> { Difference { iter: self.iter(), other } @@ -472,12 +569,12 @@ where /// /// ``` /// use std::collections::HashSet; - /// let a: HashSet<_> = [1, 2, 3].iter().cloned().collect(); - /// let b: HashSet<_> = [4, 2, 3, 4].iter().cloned().collect(); + /// let a = HashSet::from([1, 2, 3]); + /// let b = HashSet::from([4, 2, 3, 4]); /// /// // Print 1, 4 in arbitrary order. /// for x in a.symmetric_difference(&b) { - /// println!("{}", x); + /// println!("{x}"); /// } /// /// let diff1: HashSet<_> = a.symmetric_difference(&b).collect(); @@ -487,6 +584,7 @@ where /// assert_eq!(diff1, [1, 4].iter().collect()); /// ``` #[inline] + #[rustc_lint_query_instability] #[stable(feature = "rust1", since = "1.0.0")] pub fn symmetric_difference<'a>( &'a self, @@ -498,22 +596,29 @@ where /// Visits the values representing the intersection, /// i.e., the values that are both in `self` and `other`. /// + /// When an equal element is present in `self` and `other` + /// then the resulting `Intersection` may yield references to + /// one or the other. This can be relevant if `T` contains fields which + /// are not compared by its `Eq` implementation, and may hold different + /// value between the two equal copies of `T` in the two sets. + /// /// # Examples /// /// ``` /// use std::collections::HashSet; - /// let a: HashSet<_> = [1, 2, 3].iter().cloned().collect(); - /// let b: HashSet<_> = [4, 2, 3, 4].iter().cloned().collect(); + /// let a = HashSet::from([1, 2, 3]); + /// let b = HashSet::from([4, 2, 3, 4]); /// /// // Print 2, 3 in arbitrary order. /// for x in a.intersection(&b) { - /// println!("{}", x); + /// println!("{x}"); /// } /// /// let intersection: HashSet<_> = a.intersection(&b).collect(); /// assert_eq!(intersection, [2, 3].iter().collect()); /// ``` #[inline] + #[rustc_lint_query_instability] #[stable(feature = "rust1", since = "1.0.0")] pub fn intersection<'a>(&'a self, other: &'a HashSet) -> Intersection<'a, T, S> { if self.len() <= other.len() { @@ -530,18 +635,19 @@ where /// /// ``` /// use std::collections::HashSet; - /// let a: HashSet<_> = [1, 2, 3].iter().cloned().collect(); - /// let b: HashSet<_> = [4, 2, 3, 4].iter().cloned().collect(); + /// let a = HashSet::from([1, 2, 3]); + /// let b = HashSet::from([4, 2, 3, 4]); /// /// // Print 1, 2, 3, 4 in arbitrary order. /// for x in a.union(&b) { - /// println!("{}", x); + /// println!("{x}"); /// } /// /// let union: HashSet<_> = a.union(&b).collect(); /// assert_eq!(union, [1, 2, 3, 4].iter().collect()); /// ``` #[inline] + #[rustc_lint_query_instability] #[stable(feature = "rust1", since = "1.0.0")] pub fn union<'a>(&'a self, other: &'a HashSet) -> Union<'a, T, S> { if self.len() >= other.len() { @@ -562,13 +668,10 @@ where /// ``` /// use std::collections::HashSet; /// - /// let set: HashSet<_> = [1, 2, 3].iter().cloned().collect(); + /// let set = HashSet::from([1, 2, 3]); /// assert_eq!(set.contains(&1), true); /// assert_eq!(set.contains(&4), false); /// ``` - /// - /// [`Eq`]: ../../std/cmp/trait.Eq.html - /// [`Hash`]: ../../std/hash/trait.Hash.html #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn contains(&self, value: &Q) -> bool @@ -576,7 +679,7 @@ where T: Borrow, Q: Hash + Eq, { - self.map.contains_key(value) + self.base.contains(value) } /// Returns a reference to the value in the set, if any, that is equal to the given value. @@ -590,13 +693,10 @@ where /// ``` /// use std::collections::HashSet; /// - /// let set: HashSet<_> = [1, 2, 3].iter().cloned().collect(); + /// let set = HashSet::from([1, 2, 3]); /// assert_eq!(set.get(&2), Some(&2)); /// assert_eq!(set.get(&4), None); /// ``` - /// - /// [`Eq`]: ../../std/cmp/trait.Eq.html - /// [`Hash`]: ../../std/hash/trait.Hash.html #[inline] #[stable(feature = "set_recovery", since = "1.9.0")] pub fn get(&self, value: &Q) -> Option<&T> @@ -604,7 +704,96 @@ where T: Borrow, Q: Hash + Eq, { - self.map.get_key_value(value).map(|(k, _)| k) + self.base.get(value) + } + + /// Inserts the given `value` into the set if it is not present, then + /// returns a reference to the value in the set. + /// + /// # Examples + /// + /// ``` + /// #![feature(hash_set_entry)] + /// + /// use std::collections::HashSet; + /// + /// let mut set = HashSet::from([1, 2, 3]); + /// assert_eq!(set.len(), 3); + /// assert_eq!(set.get_or_insert(2), &2); + /// assert_eq!(set.get_or_insert(100), &100); + /// assert_eq!(set.len(), 4); // 100 was inserted + /// ``` + #[inline] + #[unstable(feature = "hash_set_entry", issue = "60896")] + pub fn get_or_insert(&mut self, value: T) -> &T { + // Although the raw entry gives us `&mut T`, we only return `&T` to be consistent with + // `get`. Key mutation is "raw" because you're not supposed to affect `Eq` or `Hash`. + self.base.get_or_insert(value) + } + + /// Inserts an owned copy of the given `value` into the set if it is not + /// present, then returns a reference to the value in the set. + /// + /// # Examples + /// + /// ``` + /// #![feature(hash_set_entry)] + /// + /// use std::collections::HashSet; + /// + /// let mut set: HashSet = ["cat", "dog", "horse"] + /// .iter().map(|&pet| pet.to_owned()).collect(); + /// + /// assert_eq!(set.len(), 3); + /// for &pet in &["cat", "dog", "fish"] { + /// let value = set.get_or_insert_owned(pet); + /// assert_eq!(value, pet); + /// } + /// assert_eq!(set.len(), 4); // a new "fish" was inserted + /// ``` + #[inline] + #[unstable(feature = "hash_set_entry", issue = "60896")] + pub fn get_or_insert_owned(&mut self, value: &Q) -> &T + where + T: Borrow, + Q: Hash + Eq + ToOwned, + { + // Although the raw entry gives us `&mut T`, we only return `&T` to be consistent with + // `get`. Key mutation is "raw" because you're not supposed to affect `Eq` or `Hash`. + self.base.get_or_insert_owned(value) + } + + /// Inserts a value computed from `f` into the set if the given `value` is + /// not present, then returns a reference to the value in the set. + /// + /// # Examples + /// + /// ``` + /// #![feature(hash_set_entry)] + /// + /// use std::collections::HashSet; + /// + /// let mut set: HashSet = ["cat", "dog", "horse"] + /// .iter().map(|&pet| pet.to_owned()).collect(); + /// + /// assert_eq!(set.len(), 3); + /// for &pet in &["cat", "dog", "fish"] { + /// let value = set.get_or_insert_with(pet, str::to_owned); + /// assert_eq!(value, pet); + /// } + /// assert_eq!(set.len(), 4); // a new "fish" was inserted + /// ``` + #[inline] + #[unstable(feature = "hash_set_entry", issue = "60896")] + pub fn get_or_insert_with(&mut self, value: &Q, f: F) -> &T + where + T: Borrow, + Q: Hash + Eq, + F: FnOnce(&Q) -> T, + { + // Although the raw entry gives us `&mut T`, we only return `&T` to be consistent with + // `get`. Key mutation is "raw" because you're not supposed to affect `Eq` or `Hash`. + self.base.get_or_insert_with(value, f) } /// Returns `true` if `self` has no elements in common with `other`. @@ -615,7 +804,7 @@ where /// ``` /// use std::collections::HashSet; /// - /// let a: HashSet<_> = [1, 2, 3].iter().cloned().collect(); + /// let a = HashSet::from([1, 2, 3]); /// let mut b = HashSet::new(); /// /// assert_eq!(a.is_disjoint(&b), true); @@ -641,7 +830,7 @@ where /// ``` /// use std::collections::HashSet; /// - /// let sup: HashSet<_> = [1, 2, 3].iter().cloned().collect(); + /// let sup = HashSet::from([1, 2, 3]); /// let mut set = HashSet::new(); /// /// assert_eq!(set.is_subset(&sup), true); @@ -663,7 +852,7 @@ where /// ``` /// use std::collections::HashSet; /// - /// let sub: HashSet<_> = [1, 2].iter().cloned().collect(); + /// let sub = HashSet::from([1, 2]); /// let mut set = HashSet::new(); /// /// assert_eq!(set.is_superset(&sub), false); @@ -683,9 +872,10 @@ where /// Adds a value to the set. /// - /// If the set did not have this value present, `true` is returned. + /// Returns whether the value was newly inserted. That is: /// - /// If the set did have this value present, `false` is returned. + /// - If the set did not previously contain this value, `true` is returned. + /// - If the set already contained this value, `false` is returned. /// /// # Examples /// @@ -701,7 +891,7 @@ where #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn insert(&mut self, value: T) -> bool { - self.map.insert(value, ()).is_none() + self.base.insert(value) } /// Adds a value to the set, replacing the existing value, if any, that is equal to the given @@ -722,13 +912,7 @@ where #[inline] #[stable(feature = "set_recovery", since = "1.9.0")] pub fn replace(&mut self, value: T) -> Option { - match self.map.entry(value) { - map::Entry::Occupied(occupied) => Some(occupied.replace_key()), - map::Entry::Vacant(vacant) => { - vacant.insert(()); - None - } - } + self.base.replace(value) } /// Removes a value from the set. Returns whether the value was @@ -749,9 +933,6 @@ where /// assert_eq!(set.remove(&2), true); /// assert_eq!(set.remove(&2), false); /// ``` - /// - /// [`Eq`]: ../../std/cmp/trait.Eq.html - /// [`Hash`]: ../../std/hash/trait.Hash.html #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn remove(&mut self, value: &Q) -> bool @@ -759,7 +940,7 @@ where T: Borrow, Q: Hash + Eq, { - self.map.remove(value).is_some() + self.base.remove(value) } /// Removes and returns the value in the set, if any, that is equal to the given one. @@ -773,13 +954,10 @@ where /// ``` /// use std::collections::HashSet; /// - /// let mut set: HashSet<_> = [1, 2, 3].iter().cloned().collect(); + /// let mut set = HashSet::from([1, 2, 3]); /// assert_eq!(set.take(&2), Some(2)); /// assert_eq!(set.take(&2), None); /// ``` - /// - /// [`Eq`]: ../../std/cmp/trait.Eq.html - /// [`Hash`]: ../../std/hash/trait.Hash.html #[inline] #[stable(feature = "set_recovery", since = "1.9.0")] pub fn take(&mut self, value: &Q) -> Option @@ -787,29 +965,24 @@ where T: Borrow, Q: Hash + Eq, { - self.map.remove_entry(value).map(|(k, _)| k) + self.base.take(value) } +} - /// Retains only the elements specified by the predicate. - /// - /// In other words, remove all elements `e` such that `f(&e)` returns `false`. - /// - /// # Examples - /// - /// ``` - /// use std::collections::HashSet; - /// - /// let xs = [1,2,3,4,5,6]; - /// let mut set: HashSet = xs.iter().cloned().collect(); - /// set.retain(|&k| k % 2 == 0); - /// assert_eq!(set.len(), 3); - /// ``` - #[stable(feature = "retain_hash_collection", since = "1.18.0")] - pub fn retain(&mut self, mut f: F) - where - F: FnMut(&T) -> bool, - { - self.map.retain(|k, _| f(k)); +#[stable(feature = "rust1", since = "1.0.0")] +impl Clone for HashSet +where + T: Clone, + S: Clone, +{ + #[inline] + fn clone(&self) -> Self { + Self { base: self.base.clone() } + } + + #[inline] + fn clone_from(&mut self, other: &Self) { + self.base.clone_from(&other.base); } } @@ -860,6 +1033,37 @@ where } } +#[stable(feature = "std_collections_from_array", since = "1.56.0")] +// Note: as what is currently the most convenient built-in way to construct +// a HashSet, a simple usage of this function must not *require* the user +// to provide a type annotation in order to infer the third type parameter +// (the hasher parameter, conventionally "S"). +// To that end, this impl is defined using RandomState as the concrete +// type of S, rather than being generic over `S: BuildHasher + Default`. +// It is expected that users who want to specify a hasher will manually use +// `with_capacity_and_hasher`. +// If type parameter defaults worked on impls, and if type parameter +// defaults could be mixed with const generics, then perhaps +// this could be generalized. +// See also the equivalent impl on HashMap. +impl From<[T; N]> for HashSet +where + T: Eq + Hash, +{ + /// # Examples + /// + /// ``` + /// use std::collections::HashSet; + /// + /// let set1 = HashSet::from([1, 2, 3, 4]); + /// let set2: HashSet<_> = [1, 2, 3, 4].into(); + /// assert_eq!(set1, set2); + /// ``` + fn from(arr: [T; N]) -> Self { + Self::from_iter(arr) + } +} + #[stable(feature = "rust1", since = "1.0.0")] impl Extend for HashSet where @@ -868,7 +1072,17 @@ where { #[inline] fn extend>(&mut self, iter: I) { - self.map.extend(iter.into_iter().map(|k| (k, ()))); + self.base.extend(iter); + } + + #[inline] + fn extend_one(&mut self, item: T) { + self.base.insert(item); + } + + #[inline] + fn extend_reserve(&mut self, additional: usize) { + self.base.extend_reserve(additional); } } @@ -882,6 +1096,16 @@ where fn extend>(&mut self, iter: I) { self.extend(iter.into_iter().cloned()); } + + #[inline] + fn extend_one(&mut self, &item: &'a T) { + self.base.insert(item); + } + + #[inline] + fn extend_reserve(&mut self, additional: usize) { + Extend::::extend_reserve(self, additional) + } } #[stable(feature = "rust1", since = "1.0.0")] @@ -892,7 +1116,7 @@ where /// Creates an empty `HashSet` with the `Default` value for the hasher. #[inline] fn default() -> HashSet { - HashSet { map: HashMap::default() } + HashSet { base: Default::default() } } } @@ -911,8 +1135,8 @@ where /// ``` /// use std::collections::HashSet; /// - /// let a: HashSet<_> = vec![1, 2, 3].into_iter().collect(); - /// let b: HashSet<_> = vec![3, 4, 5].into_iter().collect(); + /// let a = HashSet::from([1, 2, 3]); + /// let b = HashSet::from([3, 4, 5]); /// /// let set = &a | &b; /// @@ -944,8 +1168,8 @@ where /// ``` /// use std::collections::HashSet; /// - /// let a: HashSet<_> = vec![1, 2, 3].into_iter().collect(); - /// let b: HashSet<_> = vec![2, 3, 4].into_iter().collect(); + /// let a = HashSet::from([1, 2, 3]); + /// let b = HashSet::from([2, 3, 4]); /// /// let set = &a & &b; /// @@ -977,8 +1201,8 @@ where /// ``` /// use std::collections::HashSet; /// - /// let a: HashSet<_> = vec![1, 2, 3].into_iter().collect(); - /// let b: HashSet<_> = vec![3, 4, 5].into_iter().collect(); + /// let a = HashSet::from([1, 2, 3]); + /// let b = HashSet::from([3, 4, 5]); /// /// let set = &a ^ &b; /// @@ -1010,8 +1234,8 @@ where /// ``` /// use std::collections::HashSet; /// - /// let a: HashSet<_> = vec![1, 2, 3].into_iter().collect(); - /// let b: HashSet<_> = vec![3, 4, 5].into_iter().collect(); + /// let a = HashSet::from([1, 2, 3]); + /// let b = HashSet::from([3, 4, 5]); /// /// let set = &a - &b; /// @@ -1033,23 +1257,42 @@ where /// This `struct` is created by the [`iter`] method on [`HashSet`]. /// See its documentation for more. /// -/// [`HashSet`]: struct.HashSet.html -/// [`iter`]: struct.HashSet.html#method.iter +/// [`iter`]: HashSet::iter +/// +/// # Examples +/// +/// ``` +/// use std::collections::HashSet; +/// +/// let a = HashSet::from([1, 2, 3]); +/// +/// let mut iter = a.iter(); +/// ``` #[stable(feature = "rust1", since = "1.0.0")] pub struct Iter<'a, K: 'a> { - iter: Keys<'a, K, ()>, + base: base::Iter<'a, K>, } /// An owning iterator over the items of a `HashSet`. /// /// This `struct` is created by the [`into_iter`] method on [`HashSet`] -/// (provided by the `IntoIterator` trait). See its documentation for more. +/// (provided by the [`IntoIterator`] trait). See its documentation for more. +/// +/// [`into_iter`]: IntoIterator::into_iter +/// [`IntoIterator`]: crate::iter::IntoIterator /// -/// [`HashSet`]: struct.HashSet.html -/// [`into_iter`]: struct.HashSet.html#method.into_iter +/// # Examples +/// +/// ``` +/// use std::collections::HashSet; +/// +/// let a = HashSet::from([1, 2, 3]); +/// +/// let mut iter = a.into_iter(); +/// ``` #[stable(feature = "rust1", since = "1.0.0")] pub struct IntoIter { - iter: map::IntoIter, + base: base::IntoIter, } /// A draining iterator over the items of a `HashSet`. @@ -1057,11 +1300,45 @@ pub struct IntoIter { /// This `struct` is created by the [`drain`] method on [`HashSet`]. /// See its documentation for more. /// -/// [`HashSet`]: struct.HashSet.html -/// [`drain`]: struct.HashSet.html#method.drain +/// [`drain`]: HashSet::drain +/// +/// # Examples +/// +/// ``` +/// use std::collections::HashSet; +/// +/// let mut a = HashSet::from([1, 2, 3]); +/// +/// let mut drain = a.drain(); +/// ``` #[stable(feature = "rust1", since = "1.0.0")] pub struct Drain<'a, K: 'a> { - iter: map::Drain<'a, K, ()>, + base: base::Drain<'a, K>, +} + +/// A draining, filtering iterator over the items of a `HashSet`. +/// +/// This `struct` is created by the [`drain_filter`] method on [`HashSet`]. +/// +/// [`drain_filter`]: HashSet::drain_filter +/// +/// # Examples +/// +/// ``` +/// #![feature(hash_drain_filter)] +/// +/// use std::collections::HashSet; +/// +/// let mut a = HashSet::from([1, 2, 3]); +/// +/// let mut drain_filtered = a.drain_filter(|v| v % 2 == 0); +/// ``` +#[unstable(feature = "hash_drain_filter", issue = "59618")] +pub struct DrainFilter<'a, K, F> +where + F: FnMut(&K) -> bool, +{ + base: base::DrainFilter<'a, K, F>, } /// A lazy iterator producing elements in the intersection of `HashSet`s. @@ -1069,8 +1346,20 @@ pub struct Drain<'a, K: 'a> { /// This `struct` is created by the [`intersection`] method on [`HashSet`]. /// See its documentation for more. /// -/// [`HashSet`]: struct.HashSet.html -/// [`intersection`]: struct.HashSet.html#method.intersection +/// [`intersection`]: HashSet::intersection +/// +/// # Examples +/// +/// ``` +/// use std::collections::HashSet; +/// +/// let a = HashSet::from([1, 2, 3]); +/// let b = HashSet::from([4, 2, 3, 4]); +/// +/// let mut intersection = a.intersection(&b); +/// ``` +#[must_use = "this returns the intersection as an iterator, \ + without modifying either input set"] #[stable(feature = "rust1", since = "1.0.0")] pub struct Intersection<'a, T: 'a, S: 'a> { // iterator of the first set @@ -1084,8 +1373,20 @@ pub struct Intersection<'a, T: 'a, S: 'a> { /// This `struct` is created by the [`difference`] method on [`HashSet`]. /// See its documentation for more. /// -/// [`HashSet`]: struct.HashSet.html -/// [`difference`]: struct.HashSet.html#method.difference +/// [`difference`]: HashSet::difference +/// +/// # Examples +/// +/// ``` +/// use std::collections::HashSet; +/// +/// let a = HashSet::from([1, 2, 3]); +/// let b = HashSet::from([4, 2, 3, 4]); +/// +/// let mut difference = a.difference(&b); +/// ``` +#[must_use = "this returns the difference as an iterator, \ + without modifying either input set"] #[stable(feature = "rust1", since = "1.0.0")] pub struct Difference<'a, T: 'a, S: 'a> { // iterator of the first set @@ -1099,8 +1400,20 @@ pub struct Difference<'a, T: 'a, S: 'a> { /// This `struct` is created by the [`symmetric_difference`] method on /// [`HashSet`]. See its documentation for more. /// -/// [`HashSet`]: struct.HashSet.html -/// [`symmetric_difference`]: struct.HashSet.html#method.symmetric_difference +/// [`symmetric_difference`]: HashSet::symmetric_difference +/// +/// # Examples +/// +/// ``` +/// use std::collections::HashSet; +/// +/// let a = HashSet::from([1, 2, 3]); +/// let b = HashSet::from([4, 2, 3, 4]); +/// +/// let mut intersection = a.symmetric_difference(&b); +/// ``` +#[must_use = "this returns the difference as an iterator, \ + without modifying either input set"] #[stable(feature = "rust1", since = "1.0.0")] pub struct SymmetricDifference<'a, T: 'a, S: 'a> { iter: Chain, Difference<'a, T, S>>, @@ -1111,8 +1424,20 @@ pub struct SymmetricDifference<'a, T: 'a, S: 'a> { /// This `struct` is created by the [`union`] method on [`HashSet`]. /// See its documentation for more. /// -/// [`HashSet`]: struct.HashSet.html -/// [`union`]: struct.HashSet.html#method.union +/// [`union`]: HashSet::union +/// +/// # Examples +/// +/// ``` +/// use std::collections::HashSet; +/// +/// let a = HashSet::from([1, 2, 3]); +/// let b = HashSet::from([4, 2, 3, 4]); +/// +/// let mut union_iter = a.union(&b); +/// ``` +#[must_use = "this returns the union as an iterator, \ + without modifying either input set"] #[stable(feature = "rust1", since = "1.0.0")] pub struct Union<'a, T: 'a, S: 'a> { iter: Chain, Difference<'a, T, S>>, @@ -1124,6 +1449,7 @@ impl<'a, T, S> IntoIterator for &'a HashSet { type IntoIter = Iter<'a, T>; #[inline] + #[rustc_lint_query_instability] fn into_iter(self) -> Iter<'a, T> { self.iter() } @@ -1151,12 +1477,13 @@ impl IntoIterator for HashSet { /// /// // Will print in an arbitrary order. /// for x in &v { - /// println!("{}", x); + /// println!("{x}"); /// } /// ``` #[inline] + #[rustc_lint_query_instability] fn into_iter(self) -> IntoIter { - IntoIter { iter: self.map.into_iter() } + IntoIter { base: self.base.into_iter() } } } @@ -1164,7 +1491,7 @@ impl IntoIterator for HashSet { impl Clone for Iter<'_, K> { #[inline] fn clone(&self) -> Self { - Iter { iter: self.iter.clone() } + Iter { base: self.base.clone() } } } #[stable(feature = "rust1", since = "1.0.0")] @@ -1173,18 +1500,18 @@ impl<'a, K> Iterator for Iter<'a, K> { #[inline] fn next(&mut self) -> Option<&'a K> { - self.iter.next() + self.base.next() } #[inline] fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() + self.base.size_hint() } } #[stable(feature = "rust1", since = "1.0.0")] impl ExactSizeIterator for Iter<'_, K> { #[inline] fn len(&self) -> usize { - self.iter.len() + self.base.len() } } #[stable(feature = "fused", since = "1.26.0")] @@ -1203,18 +1530,18 @@ impl Iterator for IntoIter { #[inline] fn next(&mut self) -> Option { - self.iter.next().map(|(k, _)| k) + self.base.next() } #[inline] fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() + self.base.size_hint() } } #[stable(feature = "rust1", since = "1.0.0")] impl ExactSizeIterator for IntoIter { #[inline] fn len(&self) -> usize { - self.iter.len() + self.base.len() } } #[stable(feature = "fused", since = "1.26.0")] @@ -1223,8 +1550,7 @@ impl FusedIterator for IntoIter {} #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for IntoIter { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let entries_iter = self.iter.iter().map(|(k, _)| k); - f.debug_list().entries(entries_iter).finish() + fmt::Debug::fmt(&self.base, f) } } @@ -1234,18 +1560,18 @@ impl<'a, K> Iterator for Drain<'a, K> { #[inline] fn next(&mut self) -> Option { - self.iter.next().map(|(k, _)| k) + self.base.next() } #[inline] fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() + self.base.size_hint() } } #[stable(feature = "rust1", since = "1.0.0")] impl ExactSizeIterator for Drain<'_, K> { #[inline] fn len(&self) -> usize { - self.iter.len() + self.base.len() } } #[stable(feature = "fused", since = "1.26.0")] @@ -1254,8 +1580,37 @@ impl FusedIterator for Drain<'_, K> {} #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for Drain<'_, K> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let entries_iter = self.iter.iter().map(|(k, _)| k); - f.debug_list().entries(entries_iter).finish() + fmt::Debug::fmt(&self.base, f) + } +} + +#[unstable(feature = "hash_drain_filter", issue = "59618")] +impl Iterator for DrainFilter<'_, K, F> +where + F: FnMut(&K) -> bool, +{ + type Item = K; + + #[inline] + fn next(&mut self) -> Option { + self.base.next() + } + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.base.size_hint() + } +} + +#[unstable(feature = "hash_drain_filter", issue = "59618")] +impl FusedIterator for DrainFilter<'_, K, F> where F: FnMut(&K) -> bool {} + +#[unstable(feature = "hash_drain_filter", issue = "59618")] +impl<'a, K, F> fmt::Debug for DrainFilter<'a, K, F> +where + F: FnMut(&K) -> bool, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DrainFilter").finish_non_exhaustive() } } @@ -1488,422 +1843,3 @@ fn assert_covariance() { d } } - -#[cfg(test)] -mod test_set { - use super::super::map::RandomState; - use super::HashSet; - - #[test] - fn test_zero_capacities() { - type HS = HashSet; - - let s = HS::new(); - assert_eq!(s.capacity(), 0); - - let s = HS::default(); - assert_eq!(s.capacity(), 0); - - let s = HS::with_hasher(RandomState::new()); - assert_eq!(s.capacity(), 0); - - let s = HS::with_capacity(0); - assert_eq!(s.capacity(), 0); - - let s = HS::with_capacity_and_hasher(0, RandomState::new()); - assert_eq!(s.capacity(), 0); - - let mut s = HS::new(); - s.insert(1); - s.insert(2); - s.remove(&1); - s.remove(&2); - s.shrink_to_fit(); - assert_eq!(s.capacity(), 0); - - let mut s = HS::new(); - s.reserve(0); - assert_eq!(s.capacity(), 0); - } - - #[test] - fn test_disjoint() { - let mut xs = HashSet::new(); - let mut ys = HashSet::new(); - assert!(xs.is_disjoint(&ys)); - assert!(ys.is_disjoint(&xs)); - assert!(xs.insert(5)); - assert!(ys.insert(11)); - assert!(xs.is_disjoint(&ys)); - assert!(ys.is_disjoint(&xs)); - assert!(xs.insert(7)); - assert!(xs.insert(19)); - assert!(xs.insert(4)); - assert!(ys.insert(2)); - assert!(ys.insert(-11)); - assert!(xs.is_disjoint(&ys)); - assert!(ys.is_disjoint(&xs)); - assert!(ys.insert(7)); - assert!(!xs.is_disjoint(&ys)); - assert!(!ys.is_disjoint(&xs)); - } - - #[test] - fn test_subset_and_superset() { - let mut a = HashSet::new(); - assert!(a.insert(0)); - assert!(a.insert(5)); - assert!(a.insert(11)); - assert!(a.insert(7)); - - let mut b = HashSet::new(); - assert!(b.insert(0)); - assert!(b.insert(7)); - assert!(b.insert(19)); - assert!(b.insert(250)); - assert!(b.insert(11)); - assert!(b.insert(200)); - - assert!(!a.is_subset(&b)); - assert!(!a.is_superset(&b)); - assert!(!b.is_subset(&a)); - assert!(!b.is_superset(&a)); - - assert!(b.insert(5)); - - assert!(a.is_subset(&b)); - assert!(!a.is_superset(&b)); - assert!(!b.is_subset(&a)); - assert!(b.is_superset(&a)); - } - - #[test] - fn test_iterate() { - let mut a = HashSet::new(); - for i in 0..32 { - assert!(a.insert(i)); - } - let mut observed: u32 = 0; - for k in &a { - observed |= 1 << *k; - } - assert_eq!(observed, 0xFFFF_FFFF); - } - - #[test] - fn test_intersection() { - let mut a = HashSet::new(); - let mut b = HashSet::new(); - assert!(a.intersection(&b).next().is_none()); - - assert!(a.insert(11)); - assert!(a.insert(1)); - assert!(a.insert(3)); - assert!(a.insert(77)); - assert!(a.insert(103)); - assert!(a.insert(5)); - assert!(a.insert(-5)); - - assert!(b.insert(2)); - assert!(b.insert(11)); - assert!(b.insert(77)); - assert!(b.insert(-9)); - assert!(b.insert(-42)); - assert!(b.insert(5)); - assert!(b.insert(3)); - - let mut i = 0; - let expected = [3, 5, 11, 77]; - for x in a.intersection(&b) { - assert!(expected.contains(x)); - i += 1 - } - assert_eq!(i, expected.len()); - - assert!(a.insert(9)); // make a bigger than b - - i = 0; - for x in a.intersection(&b) { - assert!(expected.contains(x)); - i += 1 - } - assert_eq!(i, expected.len()); - - i = 0; - for x in b.intersection(&a) { - assert!(expected.contains(x)); - i += 1 - } - assert_eq!(i, expected.len()); - } - - #[test] - fn test_difference() { - let mut a = HashSet::new(); - let mut b = HashSet::new(); - - assert!(a.insert(1)); - assert!(a.insert(3)); - assert!(a.insert(5)); - assert!(a.insert(9)); - assert!(a.insert(11)); - - assert!(b.insert(3)); - assert!(b.insert(9)); - - let mut i = 0; - let expected = [1, 5, 11]; - for x in a.difference(&b) { - assert!(expected.contains(x)); - i += 1 - } - assert_eq!(i, expected.len()); - } - - #[test] - fn test_symmetric_difference() { - let mut a = HashSet::new(); - let mut b = HashSet::new(); - - assert!(a.insert(1)); - assert!(a.insert(3)); - assert!(a.insert(5)); - assert!(a.insert(9)); - assert!(a.insert(11)); - - assert!(b.insert(-2)); - assert!(b.insert(3)); - assert!(b.insert(9)); - assert!(b.insert(14)); - assert!(b.insert(22)); - - let mut i = 0; - let expected = [-2, 1, 5, 11, 14, 22]; - for x in a.symmetric_difference(&b) { - assert!(expected.contains(x)); - i += 1 - } - assert_eq!(i, expected.len()); - } - - #[test] - fn test_union() { - let mut a = HashSet::new(); - let mut b = HashSet::new(); - assert!(a.union(&b).next().is_none()); - assert!(b.union(&a).next().is_none()); - - assert!(a.insert(1)); - assert!(a.insert(3)); - assert!(a.insert(11)); - assert!(a.insert(16)); - assert!(a.insert(19)); - assert!(a.insert(24)); - - assert!(b.insert(-2)); - assert!(b.insert(1)); - assert!(b.insert(5)); - assert!(b.insert(9)); - assert!(b.insert(13)); - assert!(b.insert(19)); - - let mut i = 0; - let expected = [-2, 1, 3, 5, 9, 11, 13, 16, 19, 24]; - for x in a.union(&b) { - assert!(expected.contains(x)); - i += 1 - } - assert_eq!(i, expected.len()); - - assert!(a.insert(9)); // make a bigger than b - assert!(a.insert(5)); - - i = 0; - for x in a.union(&b) { - assert!(expected.contains(x)); - i += 1 - } - assert_eq!(i, expected.len()); - - i = 0; - for x in b.union(&a) { - assert!(expected.contains(x)); - i += 1 - } - assert_eq!(i, expected.len()); - } - - #[test] - fn test_from_iter() { - let xs = [1, 2, 2, 3, 4, 5, 6, 7, 8, 9]; - - let set: HashSet<_> = xs.iter().cloned().collect(); - - for x in &xs { - assert!(set.contains(x)); - } - - assert_eq!(set.iter().len(), xs.len() - 1); - } - - #[test] - fn test_move_iter() { - let hs = { - let mut hs = HashSet::new(); - - hs.insert('a'); - hs.insert('b'); - - hs - }; - - let v = hs.into_iter().collect::>(); - assert!(v == ['a', 'b'] || v == ['b', 'a']); - } - - #[test] - fn test_eq() { - // These constants once happened to expose a bug in insert(). - // I'm keeping them around to prevent a regression. - let mut s1 = HashSet::new(); - - s1.insert(1); - s1.insert(2); - s1.insert(3); - - let mut s2 = HashSet::new(); - - s2.insert(1); - s2.insert(2); - - assert!(s1 != s2); - - s2.insert(3); - - assert_eq!(s1, s2); - } - - #[test] - fn test_show() { - let mut set = HashSet::new(); - let empty = HashSet::::new(); - - set.insert(1); - set.insert(2); - - let set_str = format!("{:?}", set); - - assert!(set_str == "{1, 2}" || set_str == "{2, 1}"); - assert_eq!(format!("{:?}", empty), "{}"); - } - - #[test] - fn test_trivial_drain() { - let mut s = HashSet::::new(); - for _ in s.drain() {} - assert!(s.is_empty()); - drop(s); - - let mut s = HashSet::::new(); - drop(s.drain()); - assert!(s.is_empty()); - } - - #[test] - fn test_drain() { - let mut s: HashSet<_> = (1..100).collect(); - - // try this a bunch of times to make sure we don't screw up internal state. - for _ in 0..20 { - assert_eq!(s.len(), 99); - - { - let mut last_i = 0; - let mut d = s.drain(); - for (i, x) in d.by_ref().take(50).enumerate() { - last_i = i; - assert!(x != 0); - } - assert_eq!(last_i, 49); - } - - for _ in &s { - panic!("s should be empty!"); - } - - // reset to try again. - s.extend(1..100); - } - } - - #[test] - fn test_replace() { - use crate::hash; - - #[derive(Debug)] - struct Foo(&'static str, i32); - - impl PartialEq for Foo { - fn eq(&self, other: &Self) -> bool { - self.0 == other.0 - } - } - - impl Eq for Foo {} - - impl hash::Hash for Foo { - fn hash(&self, h: &mut H) { - self.0.hash(h); - } - } - - let mut s = HashSet::new(); - assert_eq!(s.replace(Foo("a", 1)), None); - assert_eq!(s.len(), 1); - assert_eq!(s.replace(Foo("a", 2)), Some(Foo("a", 1))); - assert_eq!(s.len(), 1); - - let mut it = s.iter(); - assert_eq!(it.next(), Some(&Foo("a", 2))); - assert_eq!(it.next(), None); - } - - #[test] - fn test_extend_ref() { - let mut a = HashSet::new(); - a.insert(1); - - a.extend(&[2, 3, 4]); - - assert_eq!(a.len(), 4); - assert!(a.contains(&1)); - assert!(a.contains(&2)); - assert!(a.contains(&3)); - assert!(a.contains(&4)); - - let mut b = HashSet::new(); - b.insert(5); - b.insert(6); - - a.extend(&b); - - assert_eq!(a.len(), 6); - assert!(a.contains(&1)); - assert!(a.contains(&2)); - assert!(a.contains(&3)); - assert!(a.contains(&4)); - assert!(a.contains(&5)); - assert!(a.contains(&6)); - } - - #[test] - fn test_retain() { - let xs = [1, 2, 3, 4, 5, 6]; - let mut set: HashSet = xs.iter().cloned().collect(); - set.retain(|&k| k % 2 == 0); - assert_eq!(set.len(), 3); - assert!(set.contains(&2)); - assert!(set.contains(&4)); - assert!(set.contains(&6)); - } -} diff --git a/crux-mir/lib/std/src/collections/hash/set/tests.rs b/crux-mir/lib/std/src/collections/hash/set/tests.rs new file mode 100644 index 000000000..941a0450c --- /dev/null +++ b/crux-mir/lib/std/src/collections/hash/set/tests.rs @@ -0,0 +1,504 @@ +use super::super::map::RandomState; +use super::HashSet; + +use crate::panic::{catch_unwind, AssertUnwindSafe}; +use crate::sync::atomic::{AtomicU32, Ordering}; + +#[test] +fn test_zero_capacities() { + type HS = HashSet; + + let s = HS::new(); + assert_eq!(s.capacity(), 0); + + let s = HS::default(); + assert_eq!(s.capacity(), 0); + + let s = HS::with_hasher(RandomState::new()); + assert_eq!(s.capacity(), 0); + + let s = HS::with_capacity(0); + assert_eq!(s.capacity(), 0); + + let s = HS::with_capacity_and_hasher(0, RandomState::new()); + assert_eq!(s.capacity(), 0); + + let mut s = HS::new(); + s.insert(1); + s.insert(2); + s.remove(&1); + s.remove(&2); + s.shrink_to_fit(); + assert_eq!(s.capacity(), 0); + + let mut s = HS::new(); + s.reserve(0); + assert_eq!(s.capacity(), 0); +} + +#[test] +fn test_disjoint() { + let mut xs = HashSet::new(); + let mut ys = HashSet::new(); + assert!(xs.is_disjoint(&ys)); + assert!(ys.is_disjoint(&xs)); + assert!(xs.insert(5)); + assert!(ys.insert(11)); + assert!(xs.is_disjoint(&ys)); + assert!(ys.is_disjoint(&xs)); + assert!(xs.insert(7)); + assert!(xs.insert(19)); + assert!(xs.insert(4)); + assert!(ys.insert(2)); + assert!(ys.insert(-11)); + assert!(xs.is_disjoint(&ys)); + assert!(ys.is_disjoint(&xs)); + assert!(ys.insert(7)); + assert!(!xs.is_disjoint(&ys)); + assert!(!ys.is_disjoint(&xs)); +} + +#[test] +fn test_subset_and_superset() { + let mut a = HashSet::new(); + assert!(a.insert(0)); + assert!(a.insert(5)); + assert!(a.insert(11)); + assert!(a.insert(7)); + + let mut b = HashSet::new(); + assert!(b.insert(0)); + assert!(b.insert(7)); + assert!(b.insert(19)); + assert!(b.insert(250)); + assert!(b.insert(11)); + assert!(b.insert(200)); + + assert!(!a.is_subset(&b)); + assert!(!a.is_superset(&b)); + assert!(!b.is_subset(&a)); + assert!(!b.is_superset(&a)); + + assert!(b.insert(5)); + + assert!(a.is_subset(&b)); + assert!(!a.is_superset(&b)); + assert!(!b.is_subset(&a)); + assert!(b.is_superset(&a)); +} + +#[test] +fn test_iterate() { + let mut a = HashSet::new(); + for i in 0..32 { + assert!(a.insert(i)); + } + let mut observed: u32 = 0; + for k in &a { + observed |= 1 << *k; + } + assert_eq!(observed, 0xFFFF_FFFF); +} + +#[test] +fn test_intersection() { + let mut a = HashSet::new(); + let mut b = HashSet::new(); + assert!(a.intersection(&b).next().is_none()); + + assert!(a.insert(11)); + assert!(a.insert(1)); + assert!(a.insert(3)); + assert!(a.insert(77)); + assert!(a.insert(103)); + assert!(a.insert(5)); + assert!(a.insert(-5)); + + assert!(b.insert(2)); + assert!(b.insert(11)); + assert!(b.insert(77)); + assert!(b.insert(-9)); + assert!(b.insert(-42)); + assert!(b.insert(5)); + assert!(b.insert(3)); + + let mut i = 0; + let expected = [3, 5, 11, 77]; + for x in a.intersection(&b) { + assert!(expected.contains(x)); + i += 1 + } + assert_eq!(i, expected.len()); + + assert!(a.insert(9)); // make a bigger than b + + i = 0; + for x in a.intersection(&b) { + assert!(expected.contains(x)); + i += 1 + } + assert_eq!(i, expected.len()); + + i = 0; + for x in b.intersection(&a) { + assert!(expected.contains(x)); + i += 1 + } + assert_eq!(i, expected.len()); +} + +#[test] +fn test_difference() { + let mut a = HashSet::new(); + let mut b = HashSet::new(); + + assert!(a.insert(1)); + assert!(a.insert(3)); + assert!(a.insert(5)); + assert!(a.insert(9)); + assert!(a.insert(11)); + + assert!(b.insert(3)); + assert!(b.insert(9)); + + let mut i = 0; + let expected = [1, 5, 11]; + for x in a.difference(&b) { + assert!(expected.contains(x)); + i += 1 + } + assert_eq!(i, expected.len()); +} + +#[test] +fn test_symmetric_difference() { + let mut a = HashSet::new(); + let mut b = HashSet::new(); + + assert!(a.insert(1)); + assert!(a.insert(3)); + assert!(a.insert(5)); + assert!(a.insert(9)); + assert!(a.insert(11)); + + assert!(b.insert(-2)); + assert!(b.insert(3)); + assert!(b.insert(9)); + assert!(b.insert(14)); + assert!(b.insert(22)); + + let mut i = 0; + let expected = [-2, 1, 5, 11, 14, 22]; + for x in a.symmetric_difference(&b) { + assert!(expected.contains(x)); + i += 1 + } + assert_eq!(i, expected.len()); +} + +#[test] +fn test_union() { + let mut a = HashSet::new(); + let mut b = HashSet::new(); + assert!(a.union(&b).next().is_none()); + assert!(b.union(&a).next().is_none()); + + assert!(a.insert(1)); + assert!(a.insert(3)); + assert!(a.insert(11)); + assert!(a.insert(16)); + assert!(a.insert(19)); + assert!(a.insert(24)); + + assert!(b.insert(-2)); + assert!(b.insert(1)); + assert!(b.insert(5)); + assert!(b.insert(9)); + assert!(b.insert(13)); + assert!(b.insert(19)); + + let mut i = 0; + let expected = [-2, 1, 3, 5, 9, 11, 13, 16, 19, 24]; + for x in a.union(&b) { + assert!(expected.contains(x)); + i += 1 + } + assert_eq!(i, expected.len()); + + assert!(a.insert(9)); // make a bigger than b + assert!(a.insert(5)); + + i = 0; + for x in a.union(&b) { + assert!(expected.contains(x)); + i += 1 + } + assert_eq!(i, expected.len()); + + i = 0; + for x in b.union(&a) { + assert!(expected.contains(x)); + i += 1 + } + assert_eq!(i, expected.len()); +} + +#[test] +fn test_from_iter() { + let xs = [1, 2, 2, 3, 4, 5, 6, 7, 8, 9]; + + let set: HashSet<_> = xs.iter().cloned().collect(); + + for x in &xs { + assert!(set.contains(x)); + } + + assert_eq!(set.iter().len(), xs.len() - 1); +} + +#[test] +fn test_move_iter() { + let hs = { + let mut hs = HashSet::new(); + + hs.insert('a'); + hs.insert('b'); + + hs + }; + + let v = hs.into_iter().collect::>(); + assert!(v == ['a', 'b'] || v == ['b', 'a']); +} + +#[test] +fn test_eq() { + // These constants once happened to expose a bug in insert(). + // I'm keeping them around to prevent a regression. + let mut s1 = HashSet::new(); + + s1.insert(1); + s1.insert(2); + s1.insert(3); + + let mut s2 = HashSet::new(); + + s2.insert(1); + s2.insert(2); + + assert!(s1 != s2); + + s2.insert(3); + + assert_eq!(s1, s2); +} + +#[test] +fn test_show() { + let mut set = HashSet::new(); + let empty = HashSet::::new(); + + set.insert(1); + set.insert(2); + + let set_str = format!("{set:?}"); + + assert!(set_str == "{1, 2}" || set_str == "{2, 1}"); + assert_eq!(format!("{empty:?}"), "{}"); +} + +#[test] +fn test_trivial_drain() { + let mut s = HashSet::::new(); + for _ in s.drain() {} + assert!(s.is_empty()); + drop(s); + + let mut s = HashSet::::new(); + drop(s.drain()); + assert!(s.is_empty()); +} + +#[test] +fn test_drain() { + let mut s: HashSet<_> = (1..100).collect(); + + // try this a bunch of times to make sure we don't screw up internal state. + for _ in 0..20 { + assert_eq!(s.len(), 99); + + { + let mut last_i = 0; + let mut d = s.drain(); + for (i, x) in d.by_ref().take(50).enumerate() { + last_i = i; + assert!(x != 0); + } + assert_eq!(last_i, 49); + } + + for _ in &s { + panic!("s should be empty!"); + } + + // reset to try again. + s.extend(1..100); + } +} + +#[test] +fn test_replace() { + use crate::hash; + + #[derive(Debug)] + struct Foo(&'static str, i32); + + impl PartialEq for Foo { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } + } + + impl Eq for Foo {} + + impl hash::Hash for Foo { + fn hash(&self, h: &mut H) { + self.0.hash(h); + } + } + + let mut s = HashSet::new(); + assert_eq!(s.replace(Foo("a", 1)), None); + assert_eq!(s.len(), 1); + assert_eq!(s.replace(Foo("a", 2)), Some(Foo("a", 1))); + assert_eq!(s.len(), 1); + + let mut it = s.iter(); + assert_eq!(it.next(), Some(&Foo("a", 2))); + assert_eq!(it.next(), None); +} + +#[test] +fn test_extend_ref() { + let mut a = HashSet::new(); + a.insert(1); + + a.extend(&[2, 3, 4]); + + assert_eq!(a.len(), 4); + assert!(a.contains(&1)); + assert!(a.contains(&2)); + assert!(a.contains(&3)); + assert!(a.contains(&4)); + + let mut b = HashSet::new(); + b.insert(5); + b.insert(6); + + a.extend(&b); + + assert_eq!(a.len(), 6); + assert!(a.contains(&1)); + assert!(a.contains(&2)); + assert!(a.contains(&3)); + assert!(a.contains(&4)); + assert!(a.contains(&5)); + assert!(a.contains(&6)); +} + +#[test] +fn test_retain() { + let xs = [1, 2, 3, 4, 5, 6]; + let mut set: HashSet = xs.iter().cloned().collect(); + set.retain(|&k| k % 2 == 0); + assert_eq!(set.len(), 3); + assert!(set.contains(&2)); + assert!(set.contains(&4)); + assert!(set.contains(&6)); +} + +#[test] +fn test_drain_filter() { + let mut x: HashSet<_> = [1].iter().copied().collect(); + let mut y: HashSet<_> = [1].iter().copied().collect(); + + x.drain_filter(|_| true); + y.drain_filter(|_| false); + assert_eq!(x.len(), 0); + assert_eq!(y.len(), 1); +} + +#[test] +fn test_drain_filter_drop_panic_leak() { + static PREDS: AtomicU32 = AtomicU32::new(0); + static DROPS: AtomicU32 = AtomicU32::new(0); + + #[derive(PartialEq, Eq, PartialOrd, Hash)] + struct D(i32); + impl Drop for D { + fn drop(&mut self) { + if DROPS.fetch_add(1, Ordering::SeqCst) == 1 { + panic!("panic in `drop`"); + } + } + } + + let mut set = (0..3).map(|i| D(i)).collect::>(); + + catch_unwind(move || { + drop(set.drain_filter(|_| { + PREDS.fetch_add(1, Ordering::SeqCst); + true + })) + }) + .ok(); + + assert_eq!(PREDS.load(Ordering::SeqCst), 3); + assert_eq!(DROPS.load(Ordering::SeqCst), 3); +} + +#[test] +fn test_drain_filter_pred_panic_leak() { + static PREDS: AtomicU32 = AtomicU32::new(0); + static DROPS: AtomicU32 = AtomicU32::new(0); + + #[derive(PartialEq, Eq, PartialOrd, Hash)] + struct D; + impl Drop for D { + fn drop(&mut self) { + DROPS.fetch_add(1, Ordering::SeqCst); + } + } + + let mut set: HashSet<_> = (0..3).map(|_| D).collect(); + + catch_unwind(AssertUnwindSafe(|| { + drop(set.drain_filter(|_| match PREDS.fetch_add(1, Ordering::SeqCst) { + 0 => true, + _ => panic!(), + })) + })) + .ok(); + + assert_eq!(PREDS.load(Ordering::SeqCst), 1); + assert_eq!(DROPS.load(Ordering::SeqCst), 3); + assert_eq!(set.len(), 0); +} + +#[test] +fn from_array() { + let set = HashSet::from([1, 2, 3, 4]); + let unordered_duplicates = HashSet::from([4, 1, 4, 3, 2]); + assert_eq!(set, unordered_duplicates); + + // This next line must infer the hasher type parameter. + // If you make a change that causes this line to no longer infer, + // that's a problem! + let _must_not_require_type_annotation = HashSet::from([1, 2]); +} + +#[test] +fn const_with_hasher() { + const X: HashSet<(), ()> = HashSet::with_hasher(()); + assert_eq!(X.len(), 0); +} diff --git a/crux-mir/lib/std/src/collections/mod.rs b/crux-mir/lib/std/src/collections/mod.rs index e8b9e9cb1..ae2baba09 100644 --- a/crux-mir/lib/std/src/collections/mod.rs +++ b/crux-mir/lib/std/src/collections/mod.rs @@ -86,7 +86,7 @@ //! cost are suffixed with a `~`. //! //! All amortized costs are for the potential need to resize when capacity is -//! exhausted. If a resize occurs it will take O(n) time. Our collections never +//! exhausted. If a resize occurs it will take *O*(*n*) time. Our collections never //! automatically shrink, so removal operations aren't amortized. Over a //! sufficiently large series of operations, the average cost per operation will //! deterministically equal the given cost. @@ -97,11 +97,11 @@ //! //! ## Sequences //! -//! | | get(i) | insert(i) | remove(i) | append | split_off(i) | -//! |----------------|----------------|-----------------|----------------|--------|----------------| -//! | [`Vec`] | O(1) | O(n-i)* | O(n-i) | O(m)* | O(n-i) | -//! | [`VecDeque`] | O(1) | O(min(i, n-i))* | O(min(i, n-i)) | O(m)* | O(min(i, n-i)) | -//! | [`LinkedList`] | O(min(i, n-i)) | O(min(i, n-i)) | O(min(i, n-i)) | O(1) | O(min(i, n-i)) | +//! | | get(i) | insert(i) | remove(i) | append | split_off(i) | +//! |----------------|------------------------|-------------------------|------------------------|-----------|------------------------| +//! | [`Vec`] | *O*(1) | *O*(*n*-*i*)* | *O*(*n*-*i*) | *O*(*m*)* | *O*(*n*-*i*) | +//! | [`VecDeque`] | *O*(1) | *O*(min(*i*, *n*-*i*))* | *O*(min(*i*, *n*-*i*)) | *O*(*m*)* | *O*(min(*i*, *n*-*i*)) | +//! | [`LinkedList`] | *O*(min(*i*, *n*-*i*)) | *O*(min(*i*, *n*-*i*)) | *O*(min(*i*, *n*-*i*)) | *O*(1) | *O*(min(*i*, *n*-*i*)) | //! //! Note that where ties occur, [`Vec`] is generally going to be faster than [`VecDeque`], and //! [`VecDeque`] is generally going to be faster than [`LinkedList`]. @@ -110,10 +110,10 @@ //! //! For Sets, all operations have the cost of the equivalent Map operation. //! -//! | | get | insert | remove | predecessor | append | -//! |--------------|-----------|----------|----------|-------------|--------| -//! | [`HashMap`] | O(1)~ | O(1)~* | O(1)~ | N/A | N/A | -//! | [`BTreeMap`] | O(log n) | O(log n) | O(log n) | O(log n) | O(n+m) | +//! | | get | insert | remove | range | append | +//! |--------------|---------------|---------------|---------------|---------------|--------------| +//! | [`HashMap`] | *O*(1)~ | *O*(1)~* | *O*(1)~ | N/A | N/A | +//! | [`BTreeMap`] | *O*(log(*n*)) | *O*(log(*n*)) | *O*(log(*n*)) | *O*(log(*n*)) | *O*(*n*+*m*) | //! //! # Correct and Efficient Usage of Collections //! @@ -199,7 +199,7 @@ //! ``` //! let vec = vec![1, 2, 3, 4]; //! for x in vec.iter() { -//! println!("vec contained {}", x); +//! println!("vec contained {x:?}"); //! } //! ``` //! @@ -217,7 +217,7 @@ //! contents by-value. This is great when the collection itself is no longer //! needed, and the values are needed elsewhere. Using `extend` with `into_iter` //! is the main way that contents of one collection are moved into another. -//! `extend` automatically calls `into_iter`, and takes any `T: `[`IntoIterator`]. +//! `extend` automatically calls `into_iter`, and takes any T: [IntoIterator]. //! Calling `collect` on an iterator itself is also a great way to convert one //! collection into another. Both of these methods should internally use the //! capacity management tools discussed in the previous section to do this as @@ -232,21 +232,21 @@ //! ``` //! use std::collections::VecDeque; //! -//! let vec = vec![1, 2, 3, 4]; +//! let vec = [1, 2, 3, 4]; //! let buf: VecDeque<_> = vec.into_iter().collect(); //! ``` //! //! Iterators also provide a series of *adapter* methods for performing common //! threads to sequences. Among the adapters are functional favorites like `map`, //! `fold`, `skip` and `take`. Of particular interest to collections is the -//! `rev` adapter, that reverses any iterator that supports this operation. Most +//! `rev` adapter, which reverses any iterator that supports this operation. Most //! collections provide reversible iterators as the way to iterate over them in //! reverse order. //! //! ``` //! let vec = vec![1, 2, 3, 4]; //! for x in vec.iter().rev() { -//! println!("vec contained {}", x); +//! println!("vec contained {x:?}"); //! } //! ``` //! @@ -268,7 +268,7 @@ //! not. Normally, this would require a `find` followed by an `insert`, //! effectively duplicating the search effort on each insertion. //! -//! When a user calls `map.entry(&key)`, the map will search for the key and +//! When a user calls `map.entry(key)`, the map will search for the key and //! then yield a variant of the `Entry` enum. //! //! If a `Vacant(entry)` is yielded, then the key *was not* found. In this case @@ -306,7 +306,7 @@ //! //! println!("Number of occurrences of each character"); //! for (char, count) in &count { -//! println!("{}: {}", char, count); +//! println!("{char}: {count}"); //! } //! ``` //! @@ -339,7 +339,7 @@ //! // Check if they're sober enough to have another beer. //! if person.blood_alcohol > 0.3 { //! // Too drunk... for now. -//! println!("Sorry {}, I have to cut you off", id); +//! println!("Sorry {id}, I have to cut you off"); //! } else { //! // Have another! //! person.blood_alcohol += 0.1; @@ -396,22 +396,16 @@ //! assert_eq!(map.keys().next().unwrap().b, "baz"); //! ``` //! -//! [`Vec`]: ../../std/vec/struct.Vec.html -//! [`HashMap`]: ../../std/collections/struct.HashMap.html -//! [`VecDeque`]: ../../std/collections/struct.VecDeque.html -//! [`LinkedList`]: ../../std/collections/struct.LinkedList.html -//! [`BTreeMap`]: ../../std/collections/struct.BTreeMap.html -//! [`HashSet`]: ../../std/collections/struct.HashSet.html -//! [`BTreeSet`]: ../../std/collections/struct.BTreeSet.html -//! [`BinaryHeap`]: ../../std/collections/struct.BinaryHeap.html -//! [`IntoIterator`]: ../../std/iter/trait.IntoIterator.html +//! [IntoIterator]: crate::iter::IntoIterator "iter::IntoIterator" #![stable(feature = "rust1", since = "1.0.0")] #[stable(feature = "rust1", since = "1.0.0")] -#[rustc_deprecated(reason = "moved to `std::ops::Bound`", since = "1.26.0")] +// FIXME(#82080) The deprecation here is only theoretical, and does not actually produce a warning. +#[deprecated(note = "moved to `std::ops::Bound`", since = "1.26.0")] #[doc(hidden)] pub use crate::ops::Bound; + #[stable(feature = "rust1", since = "1.0.0")] pub use alloc_crate::collections::{binary_heap, btree_map, btree_set}; #[stable(feature = "rust1", since = "1.0.0")] @@ -426,8 +420,14 @@ pub use self::hash_map::HashMap; #[stable(feature = "rust1", since = "1.0.0")] pub use self::hash_set::HashSet; -#[unstable(feature = "try_reserve", reason = "new API", issue = "48043")] +#[stable(feature = "try_reserve", since = "1.57.0")] pub use alloc_crate::collections::TryReserveError; +#[unstable( + feature = "try_reserve_kind", + reason = "Uncertain how much info should be exposed", + issue = "48043" +)] +pub use alloc_crate::collections::TryReserveErrorKind; mod hash; diff --git a/crux-mir/lib/std/src/env.rs b/crux-mir/lib/std/src/env.rs index 6aad082a9..183f9ab3b 100644 --- a/crux-mir/lib/std/src/env.rs +++ b/crux-mir/lib/std/src/env.rs @@ -7,12 +7,12 @@ //! There are several functions and structs in this module that have a //! counterpart ending in `os`. Those ending in `os` will return an [`OsString`] //! and those without will return a [`String`]. -//! -//! [`OsString`]: ../../std/ffi/struct.OsString.html -//! [`String`]: ../string/struct.String.html #![stable(feature = "env", since = "1.0.0")] +#[cfg(test)] +mod tests; + use crate::error::Error; use crate::ffi::{OsStr, OsString}; use crate::fmt; @@ -23,6 +23,13 @@ use crate::sys::os as os_imp; /// Returns the current working directory as a [`PathBuf`]. /// +/// # Platform-specific behavior +/// +/// This function [currently] corresponds to the `getcwd` function on Unix +/// and the `GetCurrentDirectoryW` function on Windows. +/// +/// [currently]: crate::io#platform-specific-behavior +/// /// # Errors /// /// Returns an [`Err`] if the current working directory value is invalid. @@ -31,9 +38,6 @@ use crate::sys::os as os_imp; /// * Current directory does not exist. /// * There are insufficient permissions to access the current directory. /// -/// [`PathBuf`]: ../../std/path/struct.PathBuf.html -/// [`Err`]: ../../std/result/enum.Result.html#method.err -/// /// # Examples /// /// ``` @@ -45,6 +49,9 @@ use crate::sys::os as os_imp; /// Ok(()) /// } /// ``` +#[doc(alias = "pwd")] +#[doc(alias = "getcwd")] +#[doc(alias = "GetCurrentDirectory")] #[stable(feature = "env", since = "1.0.0")] pub fn current_dir() -> io::Result { os_imp::getcwd() @@ -52,9 +59,14 @@ pub fn current_dir() -> io::Result { /// Changes the current working directory to the specified path. /// +/// # Platform-specific behavior +/// +/// This function [currently] corresponds to the `chdir` function on Unix +/// and the `SetCurrentDirectoryW` function on Windows. +/// /// Returns an [`Err`] if the operation fails. /// -/// [`Err`]: ../../std/result/enum.Result.html#method.err +/// [currently]: crate::io#platform-specific-behavior /// /// # Examples /// @@ -66,6 +78,7 @@ pub fn current_dir() -> io::Result { /// assert!(env::set_current_dir(&root).is_ok()); /// println!("Successfully changed working directory to {}!", root.display()); /// ``` +#[doc(alias = "chdir")] #[stable(feature = "env", since = "1.0.0")] pub fn set_current_dir>(path: P) -> io::Result<()> { os_imp::chdir(path.as_ref()) @@ -73,10 +86,9 @@ pub fn set_current_dir>(path: P) -> io::Result<()> { /// An iterator over a snapshot of the environment variables of this process. /// -/// This structure is created by the [`std::env::vars`] function. See its -/// documentation for more. +/// This structure is created by [`env::vars()`]. See its documentation for more. /// -/// [`std::env::vars`]: fn.vars.html +/// [`env::vars()`]: vars #[stable(feature = "env", since = "1.0.0")] pub struct Vars { inner: VarsOs, @@ -84,10 +96,9 @@ pub struct Vars { /// An iterator over a snapshot of the environment variables of this process. /// -/// This structure is created by the [`std::env::vars_os`] function. See -/// its documentation for more. +/// This structure is created by [`env::vars_os()`]. See its documentation for more. /// -/// [`std::env::vars_os`]: fn.vars_os.html +/// [`env::vars_os()`]: vars_os #[stable(feature = "env", since = "1.0.0")] pub struct VarsOs { inner: os_imp::Env, @@ -103,10 +114,8 @@ pub struct VarsOs { /// # Panics /// /// While iterating, the returned iterator will panic if any key or value in the -/// environment is not valid unicode. If this is not desired, consider using the -/// [`env::vars_os`] function. -/// -/// [`env::vars_os`]: fn.vars_os.html +/// environment is not valid unicode. If this is not desired, consider using +/// [`env::vars_os()`]. /// /// # Examples /// @@ -116,9 +125,12 @@ pub struct VarsOs { /// // We will iterate through the references to the element returned by /// // env::vars(); /// for (key, value) in env::vars() { -/// println!("{}: {}", key, value); +/// println!("{key}: {value}"); /// } /// ``` +/// +/// [`env::vars_os()`]: vars_os +#[must_use] #[stable(feature = "env", since = "1.0.0")] pub fn vars() -> Vars { Vars { inner: vars_os() } @@ -131,6 +143,10 @@ pub fn vars() -> Vars { /// variables at the time of this invocation. Modifications to environment /// variables afterwards will not be reflected in the returned iterator. /// +/// Note that the returned iterator will not check if the environment variables +/// are valid Unicode. If you want to panic on invalid UTF-8, +/// use the [`vars`] function instead. +/// /// # Examples /// /// ``` @@ -139,9 +155,10 @@ pub fn vars() -> Vars { /// // We will iterate through the references to the element returned by /// // env::vars_os(); /// for (key, value) in env::vars_os() { -/// println!("{:?}: {:?}", key, value); +/// println!("{key:?}: {value:?}"); /// } /// ``` +#[must_use] #[stable(feature = "env", since = "1.0.0")] pub fn vars_os() -> VarsOs { VarsOs { inner: os_imp::env() } @@ -161,7 +178,7 @@ impl Iterator for Vars { #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for Vars { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("Vars { .. }") + f.debug_struct("Vars").finish_non_exhaustive() } } @@ -179,7 +196,7 @@ impl Iterator for VarsOs { #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for VarsOs { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("VarsOs { .. }") + f.debug_struct("VarOs").finish_non_exhaustive() } } @@ -187,14 +204,13 @@ impl fmt::Debug for VarsOs { /// /// # Errors /// -/// * Environment variable is not present -/// * Environment variable is not valid unicode +/// This function will return an error if the environment variable isn't set. /// -/// # Panics +/// This function may return an error if the environment variable's name contains +/// the equal sign character (`=`) or the NUL character. /// -/// This function may panic if `key` is empty, contains an ASCII equals sign -/// `'='` or the NUL character `'\0'`, or when the value contains the NUL -/// character. +/// This function will return an error if the environment variable's value is +/// not valid Unicode. If this is not desired, consider using [`var_os`]. /// /// # Examples /// @@ -203,8 +219,8 @@ impl fmt::Debug for VarsOs { /// /// let key = "HOME"; /// match env::var(key) { -/// Ok(val) => println!("{}: {:?}", key, val), -/// Err(e) => println!("couldn't interpret {}: {}", key, e), +/// Ok(val) => println!("{key}: {val:?}"), +/// Err(e) => println!("couldn't interpret {key}: {e}"), /// } /// ``` #[stable(feature = "env", since = "1.0.0")] @@ -220,15 +236,21 @@ fn _var(key: &OsStr) -> Result { } /// Fetches the environment variable `key` from the current process, returning -/// [`None`] if the variable isn't set. +/// [`None`] if the variable isn't set or there's another error. /// -/// [`None`]: ../option/enum.Option.html#variant.None +/// Note that the method will not check if the environment variable +/// is valid Unicode. If you want to have an error on invalid UTF-8, +/// use the [`var`] function instead. /// -/// # Panics +/// # Errors /// -/// This function may panic if `key` is empty, contains an ASCII equals sign -/// `'='` or the NUL character `'\0'`, or when the value contains the NUL -/// character. +/// This function returns an error if the environment variable isn't set. +/// +/// This function may return an error if the environment variable's name contains +/// the equal sign character (`=`) or the NUL character. +/// +/// This function may return an error if the environment variable's value contains +/// the NUL character. /// /// # Examples /// @@ -237,10 +259,11 @@ fn _var(key: &OsStr) -> Result { /// /// let key = "HOME"; /// match env::var_os(key) { -/// Some(val) => println!("{}: {:?}", key, val), -/// None => println!("{} is not defined in the environment.", key) +/// Some(val) => println!("{key}: {val:?}"), +/// None => println!("{key} is not defined in the environment.") /// } /// ``` +#[must_use] #[stable(feature = "env", since = "1.0.0")] pub fn var_os>(key: K) -> Option { _var_os(key.as_ref()) @@ -248,13 +271,12 @@ pub fn var_os>(key: K) -> Option { fn _var_os(key: &OsStr) -> Option { os_imp::getenv(key) - .unwrap_or_else(|e| panic!("failed to get environment variable `{:?}`: {}", key, e)) } /// The error type for operations interacting with environment variables. -/// Possibly returned from the [`env::var`] function. +/// Possibly returned from [`env::var()`]. /// -/// [`env::var`]: fn.var.html +/// [`env::var()`]: var #[derive(Debug, PartialEq, Eq, Clone)] #[stable(feature = "env", since = "1.0.0")] pub enum VarError { @@ -293,7 +315,7 @@ impl Error for VarError { } } -/// Sets the environment variable `k` to the value `v` for the currently running +/// Sets the environment variable `key` to the value `value` for the currently running /// process. /// /// Note that while concurrent access to environment variables is safe in Rust, @@ -304,14 +326,13 @@ impl Error for VarError { /// /// Discussion of this unsafety on Unix may be found in: /// -/// - [Austin Group Bugzilla](http://austingroupbugs.net/view.php?id=188) +/// - [Austin Group Bugzilla](https://austingroupbugs.net/view.php?id=188) /// - [GNU C library Bugzilla](https://sourceware.org/bugzilla/show_bug.cgi?id=15607#c2) /// /// # Panics /// -/// This function may panic if `key` is empty, contains an ASCII equals sign -/// `'='` or the NUL character `'\0'`, or when the value contains the NUL -/// character. +/// This function may panic if `key` is empty, contains an ASCII equals sign `'='` +/// or the NUL character `'\0'`, or when `value` contains the NUL character. /// /// # Examples /// @@ -323,13 +344,13 @@ impl Error for VarError { /// assert_eq!(env::var(key), Ok("VALUE".to_string())); /// ``` #[stable(feature = "env", since = "1.0.0")] -pub fn set_var, V: AsRef>(k: K, v: V) { - _set_var(k.as_ref(), v.as_ref()) +pub fn set_var, V: AsRef>(key: K, value: V) { + _set_var(key.as_ref(), value.as_ref()) } -fn _set_var(k: &OsStr, v: &OsStr) { - os_imp::setenv(k, v).unwrap_or_else(|e| { - panic!("failed to set environment variable `{:?}` to `{:?}`: {}", k, v, e) +fn _set_var(key: &OsStr, value: &OsStr) { + os_imp::setenv(key, value).unwrap_or_else(|e| { + panic!("failed to set environment variable `{key:?}` to `{value:?}`: {e}") }) } @@ -343,7 +364,7 @@ fn _set_var(k: &OsStr, v: &OsStr) { /// /// Discussion of this unsafety on Unix may be found in: /// -/// - [Austin Group Bugzilla](http://austingroupbugs.net/view.php?id=188) +/// - [Austin Group Bugzilla](https://austingroupbugs.net/view.php?id=188) /// - [GNU C library Bugzilla](https://sourceware.org/bugzilla/show_bug.cgi?id=15607#c2) /// /// # Panics @@ -365,13 +386,13 @@ fn _set_var(k: &OsStr, v: &OsStr) { /// assert!(env::var(key).is_err()); /// ``` #[stable(feature = "env", since = "1.0.0")] -pub fn remove_var>(k: K) { - _remove_var(k.as_ref()) +pub fn remove_var>(key: K) { + _remove_var(key.as_ref()) } -fn _remove_var(k: &OsStr) { - os_imp::unsetenv(k) - .unwrap_or_else(|e| panic!("failed to remove environment variable `{:?}`: {}", k, e)) +fn _remove_var(key: &OsStr) { + os_imp::unsetenv(key) + .unwrap_or_else(|e| panic!("failed to remove environment variable `{key:?}`: {e}")) } /// An iterator that splits an environment variable into paths according to @@ -379,11 +400,11 @@ fn _remove_var(k: &OsStr) { /// /// The iterator element type is [`PathBuf`]. /// -/// This structure is created by the [`std::env::split_paths`] function. See its +/// This structure is created by [`env::split_paths()`]. See its /// documentation for more. /// -/// [`PathBuf`]: ../../std/path/struct.PathBuf.html -/// [`std::env::split_paths`]: fn.split_paths.html +/// [`env::split_paths()`]: split_paths +#[must_use = "iterators are lazy and do nothing unless consumed"] #[stable(feature = "env", since = "1.0.0")] pub struct SplitPaths<'a> { inner: os_imp::SplitPaths<'a>, @@ -407,11 +428,9 @@ pub struct SplitPaths<'a> { /// println!("'{}'", path.display()); /// } /// } -/// None => println!("{} is not defined in the environment.", key) +/// None => println!("{key} is not defined in the environment.") /// } /// ``` -/// -/// [`PathBuf`]: ../../std/path/struct.PathBuf.html #[stable(feature = "env", since = "1.0.0")] pub fn split_paths + ?Sized>(unparsed: &T) -> SplitPaths<'_> { SplitPaths { inner: os_imp::split_paths(unparsed.as_ref()) } @@ -431,14 +450,14 @@ impl<'a> Iterator for SplitPaths<'a> { #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for SplitPaths<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("SplitPaths { .. }") + f.debug_struct("SplitPaths").finish_non_exhaustive() } } /// The error type for operations on the `PATH` variable. Possibly returned from -/// the [`env::join_paths`] function. +/// [`env::join_paths()`]. /// -/// [`env::join_paths`]: fn.join_paths.html +/// [`env::join_paths()`]: join_paths #[derive(Debug)] #[stable(feature = "env", since = "1.0.0")] pub struct JoinPathsError { @@ -450,14 +469,10 @@ pub struct JoinPathsError { /// /// # Errors /// -/// Returns an [`Err`][err] (containing an error message) if one of the input +/// Returns an [`Err`] (containing an error message) if one of the input /// [`Path`]s contains an invalid character for constructing the `PATH` /// variable (a double quote on Windows or a colon on Unix). /// -/// [`Path`]: ../../std/path/struct.Path.html -/// [`OsString`]: ../../std/ffi/struct.OsString.html -/// [err]: ../../std/result/enum.Result.html#variant.Err -/// /// # Examples /// /// Joining paths on a Unix-like platform: @@ -477,7 +492,8 @@ pub struct JoinPathsError { /// } /// ``` /// -/// Joining a path containing a colon on a Unix-like platform results in an error: +/// Joining a path containing a colon on a Unix-like platform results in an +/// error: /// /// ``` /// # if cfg!(unix) { @@ -489,8 +505,8 @@ pub struct JoinPathsError { /// # } /// ``` /// -/// Using `env::join_paths` with [`env::split_paths`] to append an item to the `PATH` environment -/// variable: +/// Using `env::join_paths()` with [`env::split_paths()`] to append an item to +/// the `PATH` environment variable: /// /// ``` /// use std::env; @@ -508,7 +524,7 @@ pub struct JoinPathsError { /// } /// ``` /// -/// [`env::split_paths`]: fn.split_paths.html +/// [`env::split_paths()`]: split_paths #[stable(feature = "env", since = "1.0.0")] pub fn join_paths(paths: I) -> Result where @@ -554,6 +570,13 @@ impl Error for JoinPathsError { /// /// [msdn]: https://docs.microsoft.com/en-us/windows/win32/api/userenv/nf-userenv-getuserprofiledirectorya /// +/// # Deprecation +/// +/// This function is deprecated because the behaviour on Windows is not correct. +/// The 'HOME' environment variable is not standard on Windows, and may not produce +/// desired results; for instance, under Cygwin or Mingw it will return `/home/you` +/// when it should return `C:\Users\you`. +/// /// # Examples /// /// ``` @@ -564,11 +587,12 @@ impl Error for JoinPathsError { /// None => println!("Impossible to get your home dir!"), /// } /// ``` -#[rustc_deprecated( +#[deprecated( since = "1.29.0", - reason = "This function's behavior is unexpected and probably not what you want. \ - Consider using the home_dir function from https://crates.io/crates/dirs instead." + note = "This function's behavior may be unexpected on Windows. \ + Consider using a crate from crates.io instead." )] +#[must_use] #[stable(feature = "env", since = "1.0.0")] pub fn home_dir() -> Option { os_imp::home_dir() @@ -576,35 +600,36 @@ pub fn home_dir() -> Option { /// Returns the path of a temporary directory. /// -/// # Unix +/// The temporary directory may be shared among users, or between processes +/// with different privileges; thus, the creation of any files or directories +/// in the temporary directory must use a secure method to create a uniquely +/// named file. Creating a file or directory with a fixed or predictable name +/// may result in "insecure temporary file" security vulnerabilities. Consider +/// using a crate that securely creates temporary files or directories. +/// +/// # Platform-specific behavior /// -/// Returns the value of the `TMPDIR` environment variable if it is -/// set, otherwise for non-Android it returns `/tmp`. If Android, since there +/// On Unix, returns the value of the `TMPDIR` environment variable if it is +/// set, otherwise for non-Android it returns `/tmp`. On Android, since there /// is no global temporary folder (it is usually allocated per-app), it returns /// `/data/local/tmp`. +/// On Windows, the behavior is equivalent to that of [`GetTempPath2`][GetTempPath2] / +/// [`GetTempPath`][GetTempPath], which this function uses internally. +/// Note that, this [may change in the future][changes]. /// -/// # Windows -/// -/// Returns the value of, in order, the `TMP`, `TEMP`, -/// `USERPROFILE` environment variable if any are set and not the empty -/// string. Otherwise, `temp_dir` returns the path of the Windows directory. -/// This behavior is identical to that of [`GetTempPath`][msdn], which this -/// function uses internally. -/// -/// [msdn]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettemppatha +/// [changes]: io#platform-specific-behavior +/// [GetTempPath2]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettemppath2a +/// [GetTempPath]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettemppatha /// /// ```no_run /// use std::env; -/// use std::fs::File; /// -/// fn main() -> std::io::Result<()> { -/// let mut dir = env::temp_dir(); -/// dir.push("foo.txt"); -/// -/// let f = File::create(dir)?; -/// Ok(()) +/// fn main() { +/// let dir = env::temp_dir(); +/// println!("Temporary directory: {}", dir.display()); /// } /// ``` +#[must_use] #[stable(feature = "env", since = "1.0.0")] pub fn temp_dir() -> PathBuf { os_imp::temp_dir() @@ -618,6 +643,9 @@ pub fn temp_dir() -> PathBuf { /// return the path of the symbolic link and other platforms will return the /// path of the symbolic link’s target. /// +/// If the executable is renamed while it is running, platforms may return the +/// path at the time it was loaded instead of the new path. +/// /// # Errors /// /// Acquiring the path of the current executable is a platform-specific operation @@ -626,36 +654,23 @@ pub fn temp_dir() -> PathBuf { /// /// # Security /// -/// The output of this function should not be used in anything that might have -/// security implications. For example: +/// The output of this function should not be trusted for anything +/// that might have security implications. Basically, if users can run +/// the executable, they can change the output arbitrarily. /// -/// ``` -/// fn main() { -/// println!("{:?}", std::env::current_exe()); -/// } -/// ``` -/// -/// On Linux systems, if this is compiled as `foo`: -/// -/// ```bash -/// $ rustc foo.rs -/// $ ./foo -/// Ok("/home/alex/foo") -/// ``` +/// As an example, you can easily introduce a race condition. It goes +/// like this: /// -/// And you make a hard link of the program: +/// 1. You get the path to the current executable using `current_exe()`, and +/// store it in a variable. +/// 2. Time passes. A malicious actor removes the current executable, and +/// replaces it with a malicious one. +/// 3. You then use the stored path to re-execute the current +/// executable. /// -/// ```bash -/// $ ln foo bar -/// ``` -/// -/// When you run it, you won’t get the path of the original executable, you’ll -/// get the path of the hard link: -/// -/// ```bash -/// $ ./bar -/// Ok("/home/alex/bar") -/// ``` +/// You expected to safely execute the current executable, but you're +/// instead executing something completely different. The code you +/// just executed run with your privileges. /// /// This sort of behavior has been known to [lead to privilege escalation] when /// used incorrectly. @@ -670,7 +685,7 @@ pub fn temp_dir() -> PathBuf { /// match env::current_exe() { /// Ok(exe_path) => println!("Path of this executable is: {}", /// exe_path.display()), -/// Err(e) => println!("failed to get current exe path: {}", e), +/// Err(e) => println!("failed to get current exe path: {e}"), /// }; /// ``` #[stable(feature = "env", since = "1.0.0")] @@ -681,15 +696,15 @@ pub fn current_exe() -> io::Result { /// An iterator over the arguments of a process, yielding a [`String`] value for /// each argument. /// -/// This struct is created by the [`std::env::args`] function. See its -/// documentation for more. +/// This struct is created by [`env::args()`]. See its documentation +/// for more. /// /// The first element is traditionally the path of the executable, but it can be -/// set to arbitrary text, and may not even exist. This means this property +/// set to arbitrary text, and might not even exist. This means this property /// should not be relied upon for security purposes. /// -/// [`String`]: ../string/struct.String.html -/// [`std::env::args`]: ./fn.args.html +/// [`env::args()`]: args +#[must_use = "iterators are lazy and do nothing unless consumed"] #[stable(feature = "env", since = "1.0.0")] pub struct Args { inner: ArgsOs, @@ -698,40 +713,40 @@ pub struct Args { /// An iterator over the arguments of a process, yielding an [`OsString`] value /// for each argument. /// -/// This struct is created by the [`std::env::args_os`] function. See its -/// documentation for more. +/// This struct is created by [`env::args_os()`]. See its documentation +/// for more. /// /// The first element is traditionally the path of the executable, but it can be -/// set to arbitrary text, and may not even exist. This means this property +/// set to arbitrary text, and might not even exist. This means this property /// should not be relied upon for security purposes. /// -/// [`OsString`]: ../ffi/struct.OsString.html -/// [`std::env::args_os`]: ./fn.args_os.html +/// [`env::args_os()`]: args_os +#[must_use = "iterators are lazy and do nothing unless consumed"] #[stable(feature = "env", since = "1.0.0")] pub struct ArgsOs { inner: sys::args::Args, } -/// Returns the arguments which this program was started with (normally passed +/// Returns the arguments that this program was started with (normally passed /// via the command line). /// /// The first element is traditionally the path of the executable, but it can be -/// set to arbitrary text, and may not even exist. This means this property should +/// set to arbitrary text, and might not even exist. This means this property should /// not be relied upon for security purposes. /// -/// On Unix systems shell usually expands unquoted arguments with glob patterns +/// On Unix systems the shell usually expands unquoted arguments with glob patterns /// (such as `*` and `?`). On Windows this is not done, and such arguments are /// passed as-is. /// -/// On glibc Linux systems, arguments are retrieved by placing a function in ".init_array". -/// Glibc passes argc, argv, and envp to functions in ".init_array", as a non-standard extension. -/// This allows `std::env::args` to work even in a `cdylib` or `staticlib`, as it does on macOS -/// and Windows. +/// On glibc Linux systems, arguments are retrieved by placing a function in `.init_array`. +/// glibc passes `argc`, `argv`, and `envp` to functions in `.init_array`, as a non-standard +/// extension. This allows `std::env::args` to work even in a `cdylib` or `staticlib`, as it +/// does on macOS and Windows. /// /// # Panics /// /// The returned iterator will panic during iteration if any argument to the -/// process is not valid unicode. If this is not desired, +/// process is not valid Unicode. If this is not desired, /// use the [`args_os`] function instead. /// /// # Examples @@ -741,27 +756,33 @@ pub struct ArgsOs { /// /// // Prints each argument on a separate line /// for argument in env::args() { -/// println!("{}", argument); +/// println!("{argument}"); /// } /// ``` -/// -/// [`args_os`]: ./fn.args_os.html #[stable(feature = "env", since = "1.0.0")] pub fn args() -> Args { Args { inner: args_os() } } -/// Returns the arguments which this program was started with (normally passed +/// Returns the arguments that this program was started with (normally passed /// via the command line). /// /// The first element is traditionally the path of the executable, but it can be -/// set to arbitrary text, and it may not even exist, so this property should +/// set to arbitrary text, and might not even exist. This means this property should /// not be relied upon for security purposes. /// -/// On glibc Linux systems, arguments are retrieved by placing a function in ".init_array". -/// Glibc passes argc, argv, and envp to functions in ".init_array", as a non-standard extension. -/// This allows `std::env::args` to work even in a `cdylib` or `staticlib`, as it does on macOS -/// and Windows. +/// On Unix systems the shell usually expands unquoted arguments with glob patterns +/// (such as `*` and `?`). On Windows this is not done, and such arguments are +/// passed as-is. +/// +/// On glibc Linux systems, arguments are retrieved by placing a function in `.init_array`. +/// glibc passes `argc`, `argv`, and `envp` to functions in `.init_array`, as a non-standard +/// extension. This allows `std::env::args_os` to work even in a `cdylib` or `staticlib`, as it +/// does on macOS and Windows. +/// +/// Note that the returned iterator will not check if the arguments to the +/// process are valid Unicode. If you want to panic on invalid UTF-8, +/// use the [`args`] function instead. /// /// # Examples /// @@ -770,7 +791,7 @@ pub fn args() -> Args { /// /// // Prints each argument on a separate line /// for argument in env::args_os() { -/// println!("{:?}", argument); +/// println!("{argument:?}"); /// } /// ``` #[stable(feature = "env", since = "1.0.0")] @@ -815,7 +836,7 @@ impl DoubleEndedIterator for Args { #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for Args { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Args").field("inner", &self.inner.inner.inner_debug()).finish() + f.debug_struct("Args").field("inner", &self.inner.inner).finish() } } @@ -856,7 +877,7 @@ impl DoubleEndedIterator for ArgsOs { #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for ArgsOs { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("ArgsOs").field("inner", &self.inner.inner_debug()).finish() + f.debug_struct("ArgsOs").field("inner", &self.inner).finish() } } @@ -874,6 +895,7 @@ pub mod consts { /// - x86_64 /// - arm /// - aarch64 + /// - m68k /// - mips /// - mips64 /// - powerpc @@ -882,7 +904,7 @@ pub mod consts { /// - s390x /// - sparc64 #[stable(feature = "env", since = "1.0.0")] - pub const ARCH: &str = super::arch::ARCH; + pub const ARCH: &str = env!("STD_ENV_ARCH"); /// The family of the operating system. Example value is `unix`. /// @@ -965,187 +987,3 @@ pub mod consts { #[stable(feature = "env", since = "1.0.0")] pub const EXE_EXTENSION: &str = os::EXE_EXTENSION; } - -#[cfg(target_arch = "x86")] -mod arch { - pub const ARCH: &str = "x86"; -} - -#[cfg(target_arch = "x86_64")] -mod arch { - pub const ARCH: &str = "x86_64"; -} - -#[cfg(target_arch = "arm")] -mod arch { - pub const ARCH: &str = "arm"; -} - -#[cfg(target_arch = "aarch64")] -mod arch { - pub const ARCH: &str = "aarch64"; -} - -#[cfg(target_arch = "mips")] -mod arch { - pub const ARCH: &str = "mips"; -} - -#[cfg(target_arch = "mips64")] -mod arch { - pub const ARCH: &str = "mips64"; -} - -#[cfg(target_arch = "powerpc")] -mod arch { - pub const ARCH: &str = "powerpc"; -} - -#[cfg(target_arch = "powerpc64")] -mod arch { - pub const ARCH: &str = "powerpc64"; -} - -#[cfg(target_arch = "s390x")] -mod arch { - pub const ARCH: &str = "s390x"; -} - -#[cfg(target_arch = "sparc64")] -mod arch { - pub const ARCH: &str = "sparc64"; -} - -#[cfg(target_arch = "le32")] -mod arch { - pub const ARCH: &str = "le32"; -} - -#[cfg(target_arch = "asmjs")] -mod arch { - pub const ARCH: &str = "asmjs"; -} - -#[cfg(target_arch = "wasm32")] -mod arch { - pub const ARCH: &str = "wasm32"; -} - -#[cfg(target_arch = "hexagon")] -mod arch { - pub const ARCH: &'static str = "hexagon"; -} - -#[cfg(target_arch = "riscv64")] -mod arch { - pub const ARCH: &'static str = "riscv64"; -} - -#[cfg(test)] -mod tests { - use super::*; - - use crate::path::Path; - - #[test] - #[cfg_attr(any(target_os = "emscripten", target_env = "sgx"), ignore)] - fn test_self_exe_path() { - let path = current_exe(); - assert!(path.is_ok()); - let path = path.unwrap(); - - // Hard to test this function - assert!(path.is_absolute()); - } - - #[test] - fn test() { - assert!((!Path::new("test-path").is_absolute())); - - #[cfg(not(target_env = "sgx"))] - current_dir().unwrap(); - } - - #[test] - #[cfg(windows)] - fn split_paths_windows() { - use crate::path::PathBuf; - - fn check_parse(unparsed: &str, parsed: &[&str]) -> bool { - split_paths(unparsed).collect::>() - == parsed.iter().map(|s| PathBuf::from(*s)).collect::>() - } - - assert!(check_parse("", &mut [""])); - assert!(check_parse(r#""""#, &mut [""])); - assert!(check_parse(";;", &mut ["", "", ""])); - assert!(check_parse(r"c:\", &mut [r"c:\"])); - assert!(check_parse(r"c:\;", &mut [r"c:\", ""])); - assert!(check_parse(r"c:\;c:\Program Files\", &mut [r"c:\", r"c:\Program Files\"])); - assert!(check_parse(r#"c:\;c:\"foo"\"#, &mut [r"c:\", r"c:\foo\"])); - assert!(check_parse( - r#"c:\;c:\"foo;bar"\;c:\baz"#, - &mut [r"c:\", r"c:\foo;bar\", r"c:\baz"] - )); - } - - #[test] - #[cfg(unix)] - fn split_paths_unix() { - use crate::path::PathBuf; - - fn check_parse(unparsed: &str, parsed: &[&str]) -> bool { - split_paths(unparsed).collect::>() - == parsed.iter().map(|s| PathBuf::from(*s)).collect::>() - } - - assert!(check_parse("", &mut [""])); - assert!(check_parse("::", &mut ["", "", ""])); - assert!(check_parse("/", &mut ["/"])); - assert!(check_parse("/:", &mut ["/", ""])); - assert!(check_parse("/:/usr/local", &mut ["/", "/usr/local"])); - } - - #[test] - #[cfg(unix)] - fn join_paths_unix() { - use crate::ffi::OsStr; - - fn test_eq(input: &[&str], output: &str) -> bool { - &*join_paths(input.iter().cloned()).unwrap() == OsStr::new(output) - } - - assert!(test_eq(&[], "")); - assert!(test_eq(&["/bin", "/usr/bin", "/usr/local/bin"], "/bin:/usr/bin:/usr/local/bin")); - assert!(test_eq(&["", "/bin", "", "", "/usr/bin", ""], ":/bin:::/usr/bin:")); - assert!(join_paths(["/te:st"].iter().cloned()).is_err()); - } - - #[test] - #[cfg(windows)] - fn join_paths_windows() { - use crate::ffi::OsStr; - - fn test_eq(input: &[&str], output: &str) -> bool { - &*join_paths(input.iter().cloned()).unwrap() == OsStr::new(output) - } - - assert!(test_eq(&[], "")); - assert!(test_eq(&[r"c:\windows", r"c:\"], r"c:\windows;c:\")); - assert!(test_eq(&["", r"c:\windows", "", "", r"c:\", ""], r";c:\windows;;;c:\;")); - assert!(test_eq(&[r"c:\te;st", r"c:\"], r#""c:\te;st";c:\"#)); - assert!(join_paths([r#"c:\te"st"#].iter().cloned()).is_err()); - } - - #[test] - fn args_debug() { - assert_eq!( - format!("Args {{ inner: {:?} }}", args().collect::>()), - format!("{:?}", args()) - ); - assert_eq!( - format!("ArgsOs {{ inner: {:?} }}", args_os().collect::>()), - format!("{:?}", args_os()) - ); - } -} diff --git a/crux-mir/lib/std/src/env/tests.rs b/crux-mir/lib/std/src/env/tests.rs new file mode 100644 index 000000000..94cace03a --- /dev/null +++ b/crux-mir/lib/std/src/env/tests.rs @@ -0,0 +1,102 @@ +use super::*; + +use crate::path::Path; + +#[test] +#[cfg_attr(any(target_os = "emscripten", target_env = "sgx"), ignore)] +fn test_self_exe_path() { + let path = current_exe(); + assert!(path.is_ok()); + let path = path.unwrap(); + + // Hard to test this function + assert!(path.is_absolute()); +} + +#[test] +fn test() { + assert!((!Path::new("test-path").is_absolute())); + + #[cfg(not(target_env = "sgx"))] + current_dir().unwrap(); +} + +#[test] +#[cfg(windows)] +fn split_paths_windows() { + use crate::path::PathBuf; + + fn check_parse(unparsed: &str, parsed: &[&str]) -> bool { + split_paths(unparsed).collect::>() + == parsed.iter().map(|s| PathBuf::from(*s)).collect::>() + } + + assert!(check_parse("", &mut [""])); + assert!(check_parse(r#""""#, &mut [""])); + assert!(check_parse(";;", &mut ["", "", ""])); + assert!(check_parse(r"c:\", &mut [r"c:\"])); + assert!(check_parse(r"c:\;", &mut [r"c:\", ""])); + assert!(check_parse(r"c:\;c:\Program Files\", &mut [r"c:\", r"c:\Program Files\"])); + assert!(check_parse(r#"c:\;c:\"foo"\"#, &mut [r"c:\", r"c:\foo\"])); + assert!(check_parse(r#"c:\;c:\"foo;bar"\;c:\baz"#, &mut [r"c:\", r"c:\foo;bar\", r"c:\baz"])); +} + +#[test] +#[cfg(unix)] +fn split_paths_unix() { + use crate::path::PathBuf; + + fn check_parse(unparsed: &str, parsed: &[&str]) -> bool { + split_paths(unparsed).collect::>() + == parsed.iter().map(|s| PathBuf::from(*s)).collect::>() + } + + assert!(check_parse("", &mut [""])); + assert!(check_parse("::", &mut ["", "", ""])); + assert!(check_parse("/", &mut ["/"])); + assert!(check_parse("/:", &mut ["/", ""])); + assert!(check_parse("/:/usr/local", &mut ["/", "/usr/local"])); +} + +#[test] +#[cfg(unix)] +fn join_paths_unix() { + use crate::ffi::OsStr; + + fn test_eq(input: &[&str], output: &str) -> bool { + &*join_paths(input.iter().cloned()).unwrap() == OsStr::new(output) + } + + assert!(test_eq(&[], "")); + assert!(test_eq(&["/bin", "/usr/bin", "/usr/local/bin"], "/bin:/usr/bin:/usr/local/bin")); + assert!(test_eq(&["", "/bin", "", "", "/usr/bin", ""], ":/bin:::/usr/bin:")); + assert!(join_paths(["/te:st"].iter().cloned()).is_err()); +} + +#[test] +#[cfg(windows)] +fn join_paths_windows() { + use crate::ffi::OsStr; + + fn test_eq(input: &[&str], output: &str) -> bool { + &*join_paths(input.iter().cloned()).unwrap() == OsStr::new(output) + } + + assert!(test_eq(&[], "")); + assert!(test_eq(&[r"c:\windows", r"c:\"], r"c:\windows;c:\")); + assert!(test_eq(&["", r"c:\windows", "", "", r"c:\", ""], r";c:\windows;;;c:\;")); + assert!(test_eq(&[r"c:\te;st", r"c:\"], r#""c:\te;st";c:\"#)); + assert!(join_paths([r#"c:\te"st"#].iter().cloned()).is_err()); +} + +#[test] +fn args_debug() { + assert_eq!( + format!("Args {{ inner: {:?} }}", args().collect::>()), + format!("{:?}", args()) + ); + assert_eq!( + format!("ArgsOs {{ inner: {:?} }}", args_os().collect::>()), + format!("{:?}", args_os()) + ); +} diff --git a/crux-mir/lib/std/src/error.rs b/crux-mir/lib/std/src/error.rs index 2a370f192..05f8fd8de 100644 --- a/crux-mir/lib/std/src/error.rs +++ b/crux-mir/lib/std/src/error.rs @@ -1,155 +1,14 @@ -//! Traits for working with Errors. - +#![doc = include_str!("../../core/src/error.md")] #![stable(feature = "rust1", since = "1.0.0")] -// A note about crates and the facade: -// -// Originally, the `Error` trait was defined in libcore, and the impls -// were scattered about. However, coherence objected to this -// arrangement, because to create the blanket impls for `Box` required -// knowing that `&str: !Error`, and we have no means to deal with that -// sort of conflict just now. Therefore, for the time being, we have -// moved the `Error` trait into libstd. As we evolve a sol'n to the -// coherence challenge (e.g., specialization, neg impls, etc) we can -// reconsider what crate these items belong in. - -use core::array; +#[cfg(test)] +mod tests; -use crate::alloc::{AllocErr, CannotReallocInPlace, LayoutErr}; -use crate::any::TypeId; use crate::backtrace::Backtrace; -use crate::borrow::Cow; -use crate::cell; -use crate::char; -use crate::fmt::{self, Debug, Display}; -use crate::mem::transmute; -use crate::num; -use crate::str; -use crate::string; +use crate::fmt::{self, Write}; -/// `Error` is a trait representing the basic expectations for error values, -/// i.e., values of type `E` in [`Result`]. Errors must describe -/// themselves through the [`Display`] and [`Debug`] traits, and may provide -/// cause chain information: -/// -/// The [`source`] method is generally used when errors cross "abstraction -/// boundaries". If one module must report an error that is caused by an error -/// from a lower-level module, it can allow access to that error via the -/// [`source`] method. This makes it possible for the high-level module to -/// provide its own errors while also revealing some of the implementation for -/// debugging via [`source`] chains. -/// -/// [`Result`]: ../result/enum.Result.html -/// [`Display`]: ../fmt/trait.Display.html -/// [`Debug`]: ../fmt/trait.Debug.html -/// [`source`]: trait.Error.html#method.source #[stable(feature = "rust1", since = "1.0.0")] -pub trait Error: Debug + Display { - /// The lower-level source of this error, if any. - /// - /// # Examples - /// - /// ``` - /// use std::error::Error; - /// use std::fmt; - /// - /// #[derive(Debug)] - /// struct SuperError { - /// side: SuperErrorSideKick, - /// } - /// - /// impl fmt::Display for SuperError { - /// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - /// write!(f, "SuperError is here!") - /// } - /// } - /// - /// impl Error for SuperError { - /// fn source(&self) -> Option<&(dyn Error + 'static)> { - /// Some(&self.side) - /// } - /// } - /// - /// #[derive(Debug)] - /// struct SuperErrorSideKick; - /// - /// impl fmt::Display for SuperErrorSideKick { - /// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - /// write!(f, "SuperErrorSideKick is here!") - /// } - /// } - /// - /// impl Error for SuperErrorSideKick {} - /// - /// fn get_super_error() -> Result<(), SuperError> { - /// Err(SuperError { side: SuperErrorSideKick }) - /// } - /// - /// fn main() { - /// match get_super_error() { - /// Err(e) => { - /// println!("Error: {}", e.description()); - /// println!("Caused by: {}", e.source().unwrap()); - /// } - /// _ => println!("No error"), - /// } - /// } - /// ``` - #[stable(feature = "error_source", since = "1.30.0")] - fn source(&self) -> Option<&(dyn Error + 'static)> { - None - } - - /// Gets the `TypeId` of `self`. - #[doc(hidden)] - #[unstable( - feature = "error_type_id", - reason = "this is memory-unsafe to override in user code", - issue = "60784" - )] - fn type_id(&self, _: private::Internal) -> TypeId - where - Self: 'static, - { - TypeId::of::() - } - - /// Returns a stack backtrace, if available, of where this error occurred. - /// - /// This function allows inspecting the location, in code, of where an error - /// happened. The returned `Backtrace` contains information about the stack - /// trace of the OS thread of execution of where the error originated from. - /// - /// Note that not all errors contain a `Backtrace`. Also note that a - /// `Backtrace` may actually be empty. For more information consult the - /// `Backtrace` type itself. - #[unstable(feature = "backtrace", issue = "53487")] - fn backtrace(&self) -> Option<&Backtrace> { - None - } - - /// ``` - /// if let Err(e) = "xc".parse::() { - /// // Print `e` itself, no need for description(). - /// eprintln!("Error: {}", e); - /// } - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_deprecated(since = "1.42.0", reason = "use the Display impl or to_string()")] - fn description(&self) -> &str { - "description() is deprecated; use Display" - } - - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_deprecated( - since = "1.33.0", - reason = "replaced by Error::source, which can support downcasting" - )] - #[allow(missing_docs)] - fn cause(&self) -> Option<&dyn Error> { - self.source() - } -} +pub use core::error::Error; mod private { // This is a hack to prevent `type_id` from being overridden by `Error` @@ -159,650 +18,559 @@ mod private { pub struct Internal; } -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, E: Error + 'a> From for Box { - /// Converts a type of [`Error`] into a box of dyn [`Error`]. - /// - /// [`Error`]: ../error/trait.Error.html - /// - /// # Examples - /// - /// ``` - /// use std::error::Error; - /// use std::fmt; - /// use std::mem; - /// - /// #[derive(Debug)] - /// struct AnError; - /// - /// impl fmt::Display for AnError { - /// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - /// write!(f , "An error") - /// } - /// } - /// - /// impl Error for AnError {} - /// - /// let an_error = AnError; - /// assert!(0 == mem::size_of_val(&an_error)); - /// let a_boxed_error = Box::::from(an_error); - /// assert!(mem::size_of::>() == mem::size_of_val(&a_boxed_error)) - /// ``` - fn from(err: E) -> Box { - Box::new(err) - } +/// An error reporter that prints an error and its sources. +/// +/// Report also exposes configuration options for formatting the error sources, either entirely on a +/// single line, or in multi-line format with each source on a new line. +/// +/// `Report` only requires that the wrapped error implement `Error`. It doesn't require that the +/// wrapped error be `Send`, `Sync`, or `'static`. +/// +/// # Examples +/// +/// ```rust +/// #![feature(error_reporter)] +/// use std::error::{Error, Report}; +/// use std::fmt; +/// +/// #[derive(Debug)] +/// struct SuperError { +/// source: SuperErrorSideKick, +/// } +/// +/// impl fmt::Display for SuperError { +/// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +/// write!(f, "SuperError is here!") +/// } +/// } +/// +/// impl Error for SuperError { +/// fn source(&self) -> Option<&(dyn Error + 'static)> { +/// Some(&self.source) +/// } +/// } +/// +/// #[derive(Debug)] +/// struct SuperErrorSideKick; +/// +/// impl fmt::Display for SuperErrorSideKick { +/// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +/// write!(f, "SuperErrorSideKick is here!") +/// } +/// } +/// +/// impl Error for SuperErrorSideKick {} +/// +/// fn get_super_error() -> Result<(), SuperError> { +/// Err(SuperError { source: SuperErrorSideKick }) +/// } +/// +/// fn main() { +/// match get_super_error() { +/// Err(e) => println!("Error: {}", Report::new(e)), +/// _ => println!("No error"), +/// } +/// } +/// ``` +/// +/// This example produces the following output: +/// +/// ```console +/// Error: SuperError is here!: SuperErrorSideKick is here! +/// ``` +/// +/// ## Output consistency +/// +/// Report prints the same output via `Display` and `Debug`, so it works well with +/// [`Result::unwrap`]/[`Result::expect`] which print their `Err` variant via `Debug`: +/// +/// ```should_panic +/// #![feature(error_reporter)] +/// use std::error::Report; +/// # use std::error::Error; +/// # use std::fmt; +/// # #[derive(Debug)] +/// # struct SuperError { +/// # source: SuperErrorSideKick, +/// # } +/// # impl fmt::Display for SuperError { +/// # fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +/// # write!(f, "SuperError is here!") +/// # } +/// # } +/// # impl Error for SuperError { +/// # fn source(&self) -> Option<&(dyn Error + 'static)> { +/// # Some(&self.source) +/// # } +/// # } +/// # #[derive(Debug)] +/// # struct SuperErrorSideKick; +/// # impl fmt::Display for SuperErrorSideKick { +/// # fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +/// # write!(f, "SuperErrorSideKick is here!") +/// # } +/// # } +/// # impl Error for SuperErrorSideKick {} +/// # fn get_super_error() -> Result<(), SuperError> { +/// # Err(SuperError { source: SuperErrorSideKick }) +/// # } +/// +/// get_super_error().map_err(Report::new).unwrap(); +/// ``` +/// +/// This example produces the following output: +/// +/// ```console +/// thread 'main' panicked at 'called `Result::unwrap()` on an `Err` value: SuperError is here!: SuperErrorSideKick is here!', src/error.rs:34:40 +/// note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +/// ``` +/// +/// ## Return from `main` +/// +/// `Report` also implements `From` for all types that implement [`Error`]; this when combined with +/// the `Debug` output means `Report` is an ideal starting place for formatting errors returned +/// from `main`. +/// +/// ```should_panic +/// #![feature(error_reporter)] +/// use std::error::Report; +/// # use std::error::Error; +/// # use std::fmt; +/// # #[derive(Debug)] +/// # struct SuperError { +/// # source: SuperErrorSideKick, +/// # } +/// # impl fmt::Display for SuperError { +/// # fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +/// # write!(f, "SuperError is here!") +/// # } +/// # } +/// # impl Error for SuperError { +/// # fn source(&self) -> Option<&(dyn Error + 'static)> { +/// # Some(&self.source) +/// # } +/// # } +/// # #[derive(Debug)] +/// # struct SuperErrorSideKick; +/// # impl fmt::Display for SuperErrorSideKick { +/// # fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +/// # write!(f, "SuperErrorSideKick is here!") +/// # } +/// # } +/// # impl Error for SuperErrorSideKick {} +/// # fn get_super_error() -> Result<(), SuperError> { +/// # Err(SuperError { source: SuperErrorSideKick }) +/// # } +/// +/// fn main() -> Result<(), Report> { +/// get_super_error()?; +/// Ok(()) +/// } +/// ``` +/// +/// This example produces the following output: +/// +/// ```console +/// Error: SuperError is here!: SuperErrorSideKick is here! +/// ``` +/// +/// **Note**: `Report`s constructed via `?` and `From` will be configured to use the single line +/// output format. If you want to make sure your `Report`s are pretty printed and include backtrace +/// you will need to manually convert and enable those flags. +/// +/// ```should_panic +/// #![feature(error_reporter)] +/// use std::error::Report; +/// # use std::error::Error; +/// # use std::fmt; +/// # #[derive(Debug)] +/// # struct SuperError { +/// # source: SuperErrorSideKick, +/// # } +/// # impl fmt::Display for SuperError { +/// # fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +/// # write!(f, "SuperError is here!") +/// # } +/// # } +/// # impl Error for SuperError { +/// # fn source(&self) -> Option<&(dyn Error + 'static)> { +/// # Some(&self.source) +/// # } +/// # } +/// # #[derive(Debug)] +/// # struct SuperErrorSideKick; +/// # impl fmt::Display for SuperErrorSideKick { +/// # fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +/// # write!(f, "SuperErrorSideKick is here!") +/// # } +/// # } +/// # impl Error for SuperErrorSideKick {} +/// # fn get_super_error() -> Result<(), SuperError> { +/// # Err(SuperError { source: SuperErrorSideKick }) +/// # } +/// +/// fn main() -> Result<(), Report> { +/// get_super_error() +/// .map_err(Report::from) +/// .map_err(|r| r.pretty(true).show_backtrace(true))?; +/// Ok(()) +/// } +/// ``` +/// +/// This example produces the following output: +/// +/// ```console +/// Error: SuperError is here! +/// +/// Caused by: +/// SuperErrorSideKick is here! +/// ``` +#[unstable(feature = "error_reporter", issue = "90172")] +pub struct Report> { + /// The error being reported. + error: E, + /// Whether a backtrace should be included as part of the report. + show_backtrace: bool, + /// Whether the report should be pretty-printed. + pretty: bool, } -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, E: Error + Send + Sync + 'a> From for Box { - /// Converts a type of [`Error`] + [`Send`] + [`Sync`] into a box of - /// dyn [`Error`] + [`Send`] + [`Sync`]. - /// - /// [`Error`]: ../error/trait.Error.html - /// - /// # Examples - /// - /// ``` - /// use std::error::Error; - /// use std::fmt; - /// use std::mem; - /// - /// #[derive(Debug)] - /// struct AnError; - /// - /// impl fmt::Display for AnError { - /// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - /// write!(f , "An error") - /// } - /// } - /// - /// impl Error for AnError {} - /// - /// unsafe impl Send for AnError {} - /// - /// unsafe impl Sync for AnError {} - /// - /// let an_error = AnError; - /// assert!(0 == mem::size_of_val(&an_error)); - /// let a_boxed_error = Box::::from(an_error); - /// assert!( - /// mem::size_of::>() == mem::size_of_val(&a_boxed_error)) - /// ``` - fn from(err: E) -> Box { - Box::new(err) +impl Report +where + Report: From, +{ + /// Create a new `Report` from an input error. + #[unstable(feature = "error_reporter", issue = "90172")] + pub fn new(error: E) -> Report { + Self::from(error) } } -#[stable(feature = "rust1", since = "1.0.0")] -impl From for Box { - /// Converts a [`String`] into a box of dyn [`Error`] + [`Send`] + [`Sync`]. - /// - /// [`Error`]: ../error/trait.Error.html +impl Report { + /// Enable pretty-printing the report across multiple lines. /// /// # Examples /// + /// ```rust + /// #![feature(error_reporter)] + /// use std::error::Report; + /// # use std::error::Error; + /// # use std::fmt; + /// # #[derive(Debug)] + /// # struct SuperError { + /// # source: SuperErrorSideKick, + /// # } + /// # impl fmt::Display for SuperError { + /// # fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// # write!(f, "SuperError is here!") + /// # } + /// # } + /// # impl Error for SuperError { + /// # fn source(&self) -> Option<&(dyn Error + 'static)> { + /// # Some(&self.source) + /// # } + /// # } + /// # #[derive(Debug)] + /// # struct SuperErrorSideKick; + /// # impl fmt::Display for SuperErrorSideKick { + /// # fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// # write!(f, "SuperErrorSideKick is here!") + /// # } + /// # } + /// # impl Error for SuperErrorSideKick {} + /// + /// let error = SuperError { source: SuperErrorSideKick }; + /// let report = Report::new(error).pretty(true); + /// eprintln!("Error: {report:?}"); /// ``` - /// use std::error::Error; - /// use std::mem; - /// - /// let a_string_error = "a string error".to_string(); - /// let a_boxed_error = Box::::from(a_string_error); - /// assert!( - /// mem::size_of::>() == mem::size_of_val(&a_boxed_error)) - /// ``` - #[inline] - fn from(err: String) -> Box { - struct StringError(String); - - impl Error for StringError { - #[allow(deprecated)] - fn description(&self) -> &str { - &self.0 - } - } - - impl Display for StringError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - Display::fmt(&self.0, f) - } - } - - // Purposefully skip printing "StringError(..)" - impl Debug for StringError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - Debug::fmt(&self.0, f) - } - } - - Box::new(StringError(err)) - } -} - -#[stable(feature = "string_box_error", since = "1.6.0")] -impl From for Box { - /// Converts a [`String`] into a box of dyn [`Error`]. /// - /// [`Error`]: ../error/trait.Error.html + /// This example produces the following output: /// - /// # Examples + /// ```console + /// Error: SuperError is here! /// + /// Caused by: + /// SuperErrorSideKick is here! /// ``` - /// use std::error::Error; - /// use std::mem; /// - /// let a_string_error = "a string error".to_string(); - /// let a_boxed_error = Box::::from(a_string_error); - /// assert!(mem::size_of::>() == mem::size_of_val(&a_boxed_error)) + /// When there are multiple source errors the causes will be numbered in order of iteration + /// starting from the outermost error. + /// + /// ```rust + /// #![feature(error_reporter)] + /// use std::error::Report; + /// # use std::error::Error; + /// # use std::fmt; + /// # #[derive(Debug)] + /// # struct SuperError { + /// # source: SuperErrorSideKick, + /// # } + /// # impl fmt::Display for SuperError { + /// # fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// # write!(f, "SuperError is here!") + /// # } + /// # } + /// # impl Error for SuperError { + /// # fn source(&self) -> Option<&(dyn Error + 'static)> { + /// # Some(&self.source) + /// # } + /// # } + /// # #[derive(Debug)] + /// # struct SuperErrorSideKick { + /// # source: SuperErrorSideKickSideKick, + /// # } + /// # impl fmt::Display for SuperErrorSideKick { + /// # fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// # write!(f, "SuperErrorSideKick is here!") + /// # } + /// # } + /// # impl Error for SuperErrorSideKick { + /// # fn source(&self) -> Option<&(dyn Error + 'static)> { + /// # Some(&self.source) + /// # } + /// # } + /// # #[derive(Debug)] + /// # struct SuperErrorSideKickSideKick; + /// # impl fmt::Display for SuperErrorSideKickSideKick { + /// # fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// # write!(f, "SuperErrorSideKickSideKick is here!") + /// # } + /// # } + /// # impl Error for SuperErrorSideKickSideKick { } + /// + /// let source = SuperErrorSideKickSideKick; + /// let source = SuperErrorSideKick { source }; + /// let error = SuperError { source }; + /// let report = Report::new(error).pretty(true); + /// eprintln!("Error: {report:?}"); /// ``` - fn from(str_err: String) -> Box { - let err1: Box = From::from(str_err); - let err2: Box = err1; - err2 - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a> From<&str> for Box { - /// Converts a [`str`] into a box of dyn [`Error`] + [`Send`] + [`Sync`]. /// - /// [`Error`]: ../error/trait.Error.html + /// This example produces the following output: /// - /// # Examples + /// ```console + /// Error: SuperError is here! /// + /// Caused by: + /// 0: SuperErrorSideKick is here! + /// 1: SuperErrorSideKickSideKick is here! /// ``` - /// use std::error::Error; - /// use std::mem; - /// - /// let a_str_error = "a str error"; - /// let a_boxed_error = Box::::from(a_str_error); - /// assert!( - /// mem::size_of::>() == mem::size_of_val(&a_boxed_error)) - /// ``` - #[inline] - fn from(err: &str) -> Box { - From::from(String::from(err)) + #[unstable(feature = "error_reporter", issue = "90172")] + pub fn pretty(mut self, pretty: bool) -> Self { + self.pretty = pretty; + self } -} -#[stable(feature = "string_box_error", since = "1.6.0")] -impl From<&str> for Box { - /// Converts a [`str`] into a box of dyn [`Error`]. - /// - /// [`Error`]: ../error/trait.Error.html + /// Display backtrace if available when using pretty output format. /// /// # Examples /// - /// ``` - /// use std::error::Error; - /// use std::mem; - /// - /// let a_str_error = "a str error"; - /// let a_boxed_error = Box::::from(a_str_error); - /// assert!(mem::size_of::>() == mem::size_of_val(&a_boxed_error)) - /// ``` - fn from(err: &str) -> Box { - From::from(String::from(err)) - } -} - -#[stable(feature = "cow_box_error", since = "1.22.0")] -impl<'a, 'b> From> for Box { - /// Converts a [`Cow`] into a box of dyn [`Error`] + [`Send`] + [`Sync`]. - /// - /// [`Cow`]: ../borrow/enum.Cow.html - /// [`Error`]: ../error/trait.Error.html - /// - /// # Examples - /// - /// ``` - /// use std::error::Error; - /// use std::mem; - /// use std::borrow::Cow; - /// - /// let a_cow_str_error = Cow::from("a str error"); - /// let a_boxed_error = Box::::from(a_cow_str_error); - /// assert!( - /// mem::size_of::>() == mem::size_of_val(&a_boxed_error)) - /// ``` - fn from(err: Cow<'b, str>) -> Box { - From::from(String::from(err)) - } -} - -#[stable(feature = "cow_box_error", since = "1.22.0")] -impl<'a> From> for Box { - /// Converts a [`Cow`] into a box of dyn [`Error`]. + /// **Note**: Report will search for the first `Backtrace` it can find starting from the + /// outermost error. In this example it will display the backtrace from the second error in the + /// sources, `SuperErrorSideKick`. + /// + /// ```rust + /// #![feature(error_reporter)] + /// #![feature(provide_any)] + /// #![feature(error_generic_member_access)] + /// # use std::error::Error; + /// # use std::fmt; + /// use std::any::Demand; + /// use std::error::Report; + /// use std::backtrace::Backtrace; + /// + /// # #[derive(Debug)] + /// # struct SuperError { + /// # source: SuperErrorSideKick, + /// # } + /// # impl fmt::Display for SuperError { + /// # fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// # write!(f, "SuperError is here!") + /// # } + /// # } + /// # impl Error for SuperError { + /// # fn source(&self) -> Option<&(dyn Error + 'static)> { + /// # Some(&self.source) + /// # } + /// # } + /// #[derive(Debug)] + /// struct SuperErrorSideKick { + /// backtrace: Backtrace, + /// } /// - /// [`Cow`]: ../borrow/enum.Cow.html - /// [`Error`]: ../error/trait.Error.html + /// impl SuperErrorSideKick { + /// fn new() -> SuperErrorSideKick { + /// SuperErrorSideKick { backtrace: Backtrace::force_capture() } + /// } + /// } /// - /// # Examples + /// impl Error for SuperErrorSideKick { + /// fn provide<'a>(&'a self, demand: &mut Demand<'a>) { + /// demand.provide_ref::(&self.backtrace); + /// } + /// } /// + /// // The rest of the example is unchanged ... + /// # impl fmt::Display for SuperErrorSideKick { + /// # fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// # write!(f, "SuperErrorSideKick is here!") + /// # } + /// # } + /// + /// let source = SuperErrorSideKick::new(); + /// let error = SuperError { source }; + /// let report = Report::new(error).pretty(true).show_backtrace(true); + /// eprintln!("Error: {report:?}"); /// ``` - /// use std::error::Error; - /// use std::mem; - /// use std::borrow::Cow; /// - /// let a_cow_str_error = Cow::from("a str error"); - /// let a_boxed_error = Box::::from(a_cow_str_error); - /// assert!(mem::size_of::>() == mem::size_of_val(&a_boxed_error)) + /// This example produces something similar to the following output: + /// + /// ```console + /// Error: SuperError is here! + /// + /// Caused by: + /// SuperErrorSideKick is here! + /// + /// Stack backtrace: + /// 0: rust_out::main::_doctest_main_src_error_rs_1158_0::SuperErrorSideKick::new + /// 1: rust_out::main::_doctest_main_src_error_rs_1158_0 + /// 2: rust_out::main + /// 3: core::ops::function::FnOnce::call_once + /// 4: std::sys_common::backtrace::__rust_begin_short_backtrace + /// 5: std::rt::lang_start::{{closure}} + /// 6: std::panicking::try + /// 7: std::rt::lang_start_internal + /// 8: std::rt::lang_start + /// 9: main + /// 10: __libc_start_main + /// 11: _start /// ``` - fn from(err: Cow<'a, str>) -> Box { - From::from(String::from(err)) - } -} - -#[unstable(feature = "never_type", issue = "35121")] -impl Error for ! {} - -#[unstable( - feature = "allocator_api", - reason = "the precise API and guarantees it provides may be tweaked.", - issue = "32838" -)] -impl Error for AllocErr {} - -#[unstable( - feature = "allocator_api", - reason = "the precise API and guarantees it provides may be tweaked.", - issue = "32838" -)] -impl Error for LayoutErr {} - -#[unstable( - feature = "allocator_api", - reason = "the precise API and guarantees it provides may be tweaked.", - issue = "32838" -)] -impl Error for CannotReallocInPlace {} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Error for str::ParseBoolError { - #[allow(deprecated)] - fn description(&self) -> &str { - "failed to parse bool" - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Error for str::Utf8Error { - #[allow(deprecated)] - fn description(&self) -> &str { - "invalid utf-8: corrupt contents" - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Error for num::ParseIntError { - #[allow(deprecated)] - fn description(&self) -> &str { - self.__description() - } -} - -#[stable(feature = "try_from", since = "1.34.0")] -impl Error for num::TryFromIntError { - #[allow(deprecated)] - fn description(&self) -> &str { - self.__description() - } -} - -#[stable(feature = "try_from", since = "1.34.0")] -impl Error for array::TryFromSliceError { - #[allow(deprecated)] - fn description(&self) -> &str { - self.__description() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Error for num::ParseFloatError { - #[allow(deprecated)] - fn description(&self) -> &str { - self.__description() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Error for string::FromUtf8Error { - #[allow(deprecated)] - fn description(&self) -> &str { - "invalid utf-8" - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Error for string::FromUtf16Error { - #[allow(deprecated)] - fn description(&self) -> &str { - "invalid utf-16" - } -} - -#[stable(feature = "str_parse_error2", since = "1.8.0")] -impl Error for string::ParseError { - fn description(&self) -> &str { - match *self {} - } -} - -#[stable(feature = "decode_utf16", since = "1.9.0")] -impl Error for char::DecodeUtf16Error { - #[allow(deprecated)] - fn description(&self) -> &str { - "unpaired surrogate found" - } -} - -#[stable(feature = "box_error", since = "1.8.0")] -impl Error for Box { - #[allow(deprecated, deprecated_in_future)] - fn description(&self) -> &str { - Error::description(&**self) - } - - #[allow(deprecated)] - fn cause(&self) -> Option<&dyn Error> { - Error::cause(&**self) - } - - fn source(&self) -> Option<&(dyn Error + 'static)> { - Error::source(&**self) - } -} - -#[stable(feature = "fmt_error", since = "1.11.0")] -impl Error for fmt::Error { - #[allow(deprecated)] - fn description(&self) -> &str { - "an error occurred when formatting an argument" - } -} - -#[stable(feature = "try_borrow", since = "1.13.0")] -impl Error for cell::BorrowError { - #[allow(deprecated)] - fn description(&self) -> &str { - "already mutably borrowed" - } -} - -#[stable(feature = "try_borrow", since = "1.13.0")] -impl Error for cell::BorrowMutError { - #[allow(deprecated)] - fn description(&self) -> &str { - "already borrowed" + #[unstable(feature = "error_reporter", issue = "90172")] + pub fn show_backtrace(mut self, show_backtrace: bool) -> Self { + self.show_backtrace = show_backtrace; + self } } -#[stable(feature = "try_from", since = "1.34.0")] -impl Error for char::CharTryFromError { - #[allow(deprecated)] - fn description(&self) -> &str { - "converted integer out of range for `char`" - } -} +impl Report +where + E: Error, +{ + fn backtrace(&self) -> Option<&Backtrace> { + // have to grab the backtrace on the first error directly since that error may not be + // 'static + let backtrace = (&self.error as &dyn Error).request_ref(); + let backtrace = backtrace.or_else(|| { + self.error + .source() + .map(|source| source.sources().find_map(|source| source.request_ref())) + .flatten() + }); + backtrace + } + + /// Format the report as a single line. + #[unstable(feature = "error_reporter", issue = "90172")] + fn fmt_singleline(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.error)?; + + let sources = self.error.source().into_iter().flat_map(::sources); + + for cause in sources { + write!(f, ": {cause}")?; + } -#[stable(feature = "char_from_str", since = "1.20.0")] -impl Error for char::ParseCharError { - #[allow(deprecated)] - fn description(&self) -> &str { - self.__description() + Ok(()) } -} -#[unstable(feature = "try_reserve", reason = "new API", issue = "48043")] -impl Error for alloc::collections::TryReserveError {} + /// Format the report as multiple lines, with each error cause on its own line. + #[unstable(feature = "error_reporter", issue = "90172")] + fn fmt_multiline(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let error = &self.error; -// Copied from `any.rs`. -impl dyn Error + 'static { - /// Returns `true` if the boxed type is the same as `T` - #[stable(feature = "error_downcast", since = "1.3.0")] - #[inline] - pub fn is(&self) -> bool { - // Get `TypeId` of the type this function is instantiated with. - let t = TypeId::of::(); + write!(f, "{error}")?; - // Get `TypeId` of the type in the trait object. - let boxed = self.type_id(private::Internal); + if let Some(cause) = error.source() { + write!(f, "\n\nCaused by:")?; - // Compare both `TypeId`s on equality. - t == boxed - } + let multiple = cause.source().is_some(); - /// Returns some reference to the boxed value if it is of type `T`, or - /// `None` if it isn't. - #[stable(feature = "error_downcast", since = "1.3.0")] - #[inline] - pub fn downcast_ref(&self) -> Option<&T> { - if self.is::() { - unsafe { Some(&*(self as *const dyn Error as *const T)) } - } else { - None + for (ind, error) in cause.sources().enumerate() { + writeln!(f)?; + let mut indented = Indented { inner: f }; + if multiple { + write!(indented, "{ind: >4}: {error}")?; + } else { + write!(indented, " {error}")?; + } + } } - } - /// Returns some mutable reference to the boxed value if it is of type `T`, or - /// `None` if it isn't. - #[stable(feature = "error_downcast", since = "1.3.0")] - #[inline] - pub fn downcast_mut(&mut self) -> Option<&mut T> { - if self.is::() { - unsafe { Some(&mut *(self as *mut dyn Error as *mut T)) } - } else { - None - } - } -} + if self.show_backtrace { + let backtrace = self.backtrace(); -impl dyn Error + 'static + Send { - /// Forwards to the method defined on the type `dyn Error`. - #[stable(feature = "error_downcast", since = "1.3.0")] - #[inline] - pub fn is(&self) -> bool { - ::is::(self) - } + if let Some(backtrace) = backtrace { + let backtrace = backtrace.to_string(); - /// Forwards to the method defined on the type `dyn Error`. - #[stable(feature = "error_downcast", since = "1.3.0")] - #[inline] - pub fn downcast_ref(&self) -> Option<&T> { - ::downcast_ref::(self) - } - - /// Forwards to the method defined on the type `dyn Error`. - #[stable(feature = "error_downcast", since = "1.3.0")] - #[inline] - pub fn downcast_mut(&mut self) -> Option<&mut T> { - ::downcast_mut::(self) - } -} - -impl dyn Error + 'static + Send + Sync { - /// Forwards to the method defined on the type `dyn Error`. - #[stable(feature = "error_downcast", since = "1.3.0")] - #[inline] - pub fn is(&self) -> bool { - ::is::(self) - } - - /// Forwards to the method defined on the type `dyn Error`. - #[stable(feature = "error_downcast", since = "1.3.0")] - #[inline] - pub fn downcast_ref(&self) -> Option<&T> { - ::downcast_ref::(self) - } - - /// Forwards to the method defined on the type `dyn Error`. - #[stable(feature = "error_downcast", since = "1.3.0")] - #[inline] - pub fn downcast_mut(&mut self) -> Option<&mut T> { - ::downcast_mut::(self) - } -} - -impl dyn Error { - #[inline] - #[stable(feature = "error_downcast", since = "1.3.0")] - /// Attempts to downcast the box to a concrete type. - pub fn downcast(self: Box) -> Result, Box> { - if self.is::() { - unsafe { - let raw: *mut dyn Error = Box::into_raw(self); - Ok(Box::from_raw(raw as *mut T)) + f.write_str("\n\nStack backtrace:\n")?; + f.write_str(backtrace.trim_end())?; } - } else { - Err(self) } - } - /// Returns an iterator starting with the current error and continuing with - /// recursively calling [`source`]. - /// - /// If you want to omit the current error and only use its sources, - /// use `skip(1)`. - /// - /// # Examples - /// - /// ``` - /// #![feature(error_iter)] - /// use std::error::Error; - /// use std::fmt; - /// - /// #[derive(Debug)] - /// struct A; - /// - /// #[derive(Debug)] - /// struct B(Option>); - /// - /// impl fmt::Display for A { - /// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - /// write!(f, "A") - /// } - /// } - /// - /// impl fmt::Display for B { - /// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - /// write!(f, "B") - /// } - /// } - /// - /// impl Error for A {} - /// - /// impl Error for B { - /// fn source(&self) -> Option<&(dyn Error + 'static)> { - /// self.0.as_ref().map(|e| e.as_ref()) - /// } - /// } - /// - /// let b = B(Some(Box::new(A))); - /// - /// // let err : Box = b.into(); // or - /// let err = &b as &(dyn Error); - /// - /// let mut iter = err.chain(); - /// - /// assert_eq!("B".to_string(), iter.next().unwrap().to_string()); - /// assert_eq!("A".to_string(), iter.next().unwrap().to_string()); - /// assert!(iter.next().is_none()); - /// assert!(iter.next().is_none()); - /// ``` - /// - /// [`source`]: trait.Error.html#method.source - #[unstable(feature = "error_iter", issue = "58520")] - #[inline] - pub fn chain(&self) -> Chain<'_> { - Chain { current: Some(self) } + Ok(()) } } -/// An iterator over an [`Error`] and its sources. -/// -/// If you want to omit the initial error and only process -/// its sources, use `skip(1)`. -/// -/// [`Error`]: trait.Error.html -#[unstable(feature = "error_iter", issue = "58520")] -#[derive(Clone, Debug)] -pub struct Chain<'a> { - current: Option<&'a (dyn Error + 'static)>, -} - -#[unstable(feature = "error_iter", issue = "58520")] -impl<'a> Iterator for Chain<'a> { - type Item = &'a (dyn Error + 'static); - - fn next(&mut self) -> Option { - let current = self.current; - self.current = self.current.and_then(Error::source); - current +#[unstable(feature = "error_reporter", issue = "90172")] +impl From for Report +where + E: Error, +{ + fn from(error: E) -> Self { + Report { error, show_backtrace: false, pretty: false } } } -impl dyn Error + Send { - #[inline] - #[stable(feature = "error_downcast", since = "1.3.0")] - /// Attempts to downcast the box to a concrete type. - pub fn downcast(self: Box) -> Result, Box> { - let err: Box = self; - ::downcast(err).map_err(|s| unsafe { - // Reapply the `Send` marker. - transmute::, Box>(s) - }) +#[unstable(feature = "error_reporter", issue = "90172")] +impl fmt::Display for Report +where + E: Error, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.pretty { self.fmt_multiline(f) } else { self.fmt_singleline(f) } } } -impl dyn Error + Send + Sync { - #[inline] - #[stable(feature = "error_downcast", since = "1.3.0")] - /// Attempts to downcast the box to a concrete type. - pub fn downcast(self: Box) -> Result, Box> { - let err: Box = self; - ::downcast(err).map_err(|s| unsafe { - // Reapply the `Send + Sync` marker. - transmute::, Box>(s) - }) +// This type intentionally outputs the same format for `Display` and `Debug`for +// situations where you unwrap a `Report` or return it from main. +#[unstable(feature = "error_reporter", issue = "90172")] +impl fmt::Debug for Report +where + Report: fmt::Display, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) } } -#[cfg(test)] -mod tests { - use super::Error; - use crate::fmt; +/// Wrapper type for indenting the inner source. +struct Indented<'a, D> { + inner: &'a mut D, +} - #[derive(Debug, PartialEq)] - struct A; - #[derive(Debug, PartialEq)] - struct B; +impl Write for Indented<'_, T> +where + T: Write, +{ + fn write_str(&mut self, s: &str) -> fmt::Result { + for (i, line) in s.split('\n').enumerate() { + if i > 0 { + self.inner.write_char('\n')?; + self.inner.write_str(" ")?; + } - impl fmt::Display for A { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "A") - } - } - impl fmt::Display for B { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "B") + self.inner.write_str(line)?; } - } - - impl Error for A {} - impl Error for B {} - #[test] - fn downcasting() { - let mut a = A; - let a = &mut a as &mut (dyn Error + 'static); - assert_eq!(a.downcast_ref::(), Some(&A)); - assert_eq!(a.downcast_ref::(), None); - assert_eq!(a.downcast_mut::(), Some(&mut A)); - assert_eq!(a.downcast_mut::(), None); - - let a: Box = Box::new(A); - match a.downcast::() { - Ok(..) => panic!("expected error"), - Err(e) => assert_eq!(*e.downcast::().unwrap(), A), - } + Ok(()) } } diff --git a/crux-mir/lib/std/src/error/tests.rs b/crux-mir/lib/std/src/error/tests.rs new file mode 100644 index 000000000..ee999bd65 --- /dev/null +++ b/crux-mir/lib/std/src/error/tests.rs @@ -0,0 +1,443 @@ +use super::Error; +use crate::fmt; +use core::any::Demand; + +#[derive(Debug, PartialEq)] +struct A; +#[derive(Debug, PartialEq)] +struct B; + +impl fmt::Display for A { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "A") + } +} +impl fmt::Display for B { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "B") + } +} + +impl Error for A {} +impl Error for B {} + +#[test] +fn downcasting() { + let mut a = A; + let a = &mut a as &mut (dyn Error + 'static); + assert_eq!(a.downcast_ref::(), Some(&A)); + assert_eq!(a.downcast_ref::(), None); + assert_eq!(a.downcast_mut::(), Some(&mut A)); + assert_eq!(a.downcast_mut::(), None); + + let a: Box = Box::new(A); + match a.downcast::() { + Ok(..) => panic!("expected error"), + Err(e) => assert_eq!(*e.downcast::().unwrap(), A), + } +} + +use crate::backtrace::Backtrace; +use crate::error::Report; + +#[derive(Debug)] +struct SuperError { + source: SuperErrorSideKick, +} + +impl fmt::Display for SuperError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "SuperError is here!") + } +} + +impl Error for SuperError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + Some(&self.source) + } +} + +#[derive(Debug)] +struct SuperErrorSideKick; + +impl fmt::Display for SuperErrorSideKick { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "SuperErrorSideKick is here!") + } +} + +impl Error for SuperErrorSideKick {} + +#[test] +fn single_line_formatting() { + let error = SuperError { source: SuperErrorSideKick }; + let report = Report::new(&error); + let actual = report.to_string(); + let expected = String::from("SuperError is here!: SuperErrorSideKick is here!"); + + assert_eq!(expected, actual); +} + +#[test] +fn multi_line_formatting() { + let error = SuperError { source: SuperErrorSideKick }; + let report = Report::new(&error).pretty(true); + let actual = report.to_string(); + let expected = String::from( + "\ +SuperError is here! + +Caused by: + SuperErrorSideKick is here!", + ); + + assert_eq!(expected, actual); +} + +#[test] +fn error_with_no_sources_formats_single_line_correctly() { + let report = Report::new(SuperErrorSideKick); + let actual = report.to_string(); + let expected = String::from("SuperErrorSideKick is here!"); + + assert_eq!(expected, actual); +} + +#[test] +fn error_with_no_sources_formats_multi_line_correctly() { + let report = Report::new(SuperErrorSideKick).pretty(true); + let actual = report.to_string(); + let expected = String::from("SuperErrorSideKick is here!"); + + assert_eq!(expected, actual); +} + +#[test] +fn error_with_backtrace_outputs_correctly_with_one_source() { + let trace = Backtrace::force_capture(); + let expected = format!( + "\ +The source of the error + +Caused by: + Error with backtrace + +Stack backtrace: +{}", + trace + ); + let error = GenericError::new("Error with backtrace"); + let mut error = GenericError::new_with_source("The source of the error", error); + error.backtrace = Some(trace); + let report = Report::new(error).pretty(true).show_backtrace(true); + + println!("Error: {report}"); + assert_eq!(expected.trim_end(), report.to_string()); +} + +#[test] +fn error_with_backtrace_outputs_correctly_with_two_sources() { + let trace = Backtrace::force_capture(); + let expected = format!( + "\ +Error with two sources + +Caused by: + 0: The source of the error + 1: Error with backtrace + +Stack backtrace: +{}", + trace + ); + let mut error = GenericError::new("Error with backtrace"); + error.backtrace = Some(trace); + let error = GenericError::new_with_source("The source of the error", error); + let error = GenericError::new_with_source("Error with two sources", error); + let report = Report::new(error).pretty(true).show_backtrace(true); + + println!("Error: {report}"); + assert_eq!(expected.trim_end(), report.to_string()); +} + +#[derive(Debug)] +struct GenericError { + message: D, + backtrace: Option, + source: Option>, +} + +impl GenericError { + fn new(message: D) -> GenericError { + Self { message, backtrace: None, source: None } + } + + fn new_with_source(message: D, source: E) -> GenericError + where + E: Error + 'static, + { + let source: Box = Box::new(source); + let source = Some(source); + GenericError { message, backtrace: None, source } + } +} + +impl fmt::Display for GenericError +where + D: fmt::Display, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.message, f) + } +} + +impl Error for GenericError +where + D: fmt::Debug + fmt::Display, +{ + fn source(&self) -> Option<&(dyn Error + 'static)> { + self.source.as_deref() + } + + fn provide<'a>(&'a self, req: &mut Demand<'a>) { + self.backtrace.as_ref().map(|bt| req.provide_ref::(bt)); + } +} + +#[test] +fn error_formats_single_line_with_rude_display_impl() { + #[derive(Debug)] + struct MyMessage; + + impl fmt::Display for MyMessage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("line 1\nline 2")?; + f.write_str("\nline 3\nline 4\n")?; + f.write_str("line 5\nline 6")?; + Ok(()) + } + } + + let error = GenericError::new(MyMessage); + let error = GenericError::new_with_source(MyMessage, error); + let error = GenericError::new_with_source(MyMessage, error); + let error = GenericError::new_with_source(MyMessage, error); + let report = Report::new(error); + let expected = "\ +line 1 +line 2 +line 3 +line 4 +line 5 +line 6: line 1 +line 2 +line 3 +line 4 +line 5 +line 6: line 1 +line 2 +line 3 +line 4 +line 5 +line 6: line 1 +line 2 +line 3 +line 4 +line 5 +line 6"; + + let actual = report.to_string(); + assert_eq!(expected, actual); +} + +#[test] +fn error_formats_multi_line_with_rude_display_impl() { + #[derive(Debug)] + struct MyMessage; + + impl fmt::Display for MyMessage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("line 1\nline 2")?; + f.write_str("\nline 3\nline 4\n")?; + f.write_str("line 5\nline 6")?; + Ok(()) + } + } + + let error = GenericError::new(MyMessage); + let error = GenericError::new_with_source(MyMessage, error); + let error = GenericError::new_with_source(MyMessage, error); + let error = GenericError::new_with_source(MyMessage, error); + let report = Report::new(error).pretty(true); + let expected = "line 1 +line 2 +line 3 +line 4 +line 5 +line 6 + +Caused by: + 0: line 1 + line 2 + line 3 + line 4 + line 5 + line 6 + 1: line 1 + line 2 + line 3 + line 4 + line 5 + line 6 + 2: line 1 + line 2 + line 3 + line 4 + line 5 + line 6"; + + let actual = report.to_string(); + assert_eq!(expected, actual); +} + +#[test] +fn errors_that_start_with_newline_formats_correctly() { + #[derive(Debug)] + struct MyMessage; + + impl fmt::Display for MyMessage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("\nThe message\n") + } + } + + let error = GenericError::new(MyMessage); + let error = GenericError::new_with_source(MyMessage, error); + let error = GenericError::new_with_source(MyMessage, error); + let report = Report::new(error).pretty(true); + let expected = " +The message + + +Caused by: + 0: \ +\n The message + \ +\n 1: \ +\n The message + "; + + let actual = report.to_string(); + assert_eq!(expected, actual); +} + +#[test] +fn errors_with_multiple_writes_on_same_line_dont_insert_erroneous_newlines() { + #[derive(Debug)] + struct MyMessage; + + impl fmt::Display for MyMessage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("The message")?; + f.write_str(" goes on")?; + f.write_str(" and on.") + } + } + + let error = GenericError::new(MyMessage); + let error = GenericError::new_with_source(MyMessage, error); + let error = GenericError::new_with_source(MyMessage, error); + let report = Report::new(error).pretty(true); + let expected = "\ +The message goes on and on. + +Caused by: + 0: The message goes on and on. + 1: The message goes on and on."; + + let actual = report.to_string(); + println!("{actual}"); + assert_eq!(expected, actual); +} + +#[test] +fn errors_with_string_interpolation_formats_correctly() { + #[derive(Debug)] + struct MyMessage(usize); + + impl fmt::Display for MyMessage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Got an error code: ({}). ", self.0)?; + write!(f, "What would you like to do in response?") + } + } + + let error = GenericError::new(MyMessage(10)); + let error = GenericError::new_with_source(MyMessage(20), error); + let report = Report::new(error).pretty(true); + let expected = "\ +Got an error code: (20). What would you like to do in response? + +Caused by: + Got an error code: (10). What would you like to do in response?"; + let actual = report.to_string(); + assert_eq!(expected, actual); +} + +#[test] +fn empty_lines_mid_message() { + #[derive(Debug)] + struct MyMessage; + + impl fmt::Display for MyMessage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("line 1\n\nline 2") + } + } + + let error = GenericError::new(MyMessage); + let error = GenericError::new_with_source(MyMessage, error); + let error = GenericError::new_with_source(MyMessage, error); + let report = Report::new(error).pretty(true); + let expected = "\ +line 1 + +line 2 + +Caused by: + 0: line 1 + \ +\n line 2 + 1: line 1 + \ +\n line 2"; + + let actual = report.to_string(); + assert_eq!(expected, actual); +} + +#[test] +fn only_one_source() { + #[derive(Debug)] + struct MyMessage; + + impl fmt::Display for MyMessage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("line 1\nline 2") + } + } + + let error = GenericError::new(MyMessage); + let error = GenericError::new_with_source(MyMessage, error); + let report = Report::new(error).pretty(true); + let expected = "\ +line 1 +line 2 + +Caused by: + line 1 + line 2"; + + let actual = report.to_string(); + assert_eq!(expected, actual); +} diff --git a/crux-mir/lib/std/src/f32.rs b/crux-mir/lib/std/src/f32.rs index 20425aea8..4e3007624 100644 --- a/crux-mir/lib/std/src/f32.rs +++ b/crux-mir/lib/std/src/f32.rs @@ -1,36 +1,35 @@ -//! This module provides constants which are specific to the implementation -//! of the `f32` floating point data type. +//! Constants for the `f32` single-precision floating point type. //! -//! *[See also the `f32` primitive type](../../std/primitive.f32.html).* +//! *[See also the `f32` primitive type](primitive@f32).* //! //! Mathematically significant numbers are provided in the `consts` sub-module. //! -//! Although using these constants won’t cause compilation warnings, -//! new code should use the associated constants directly on the primitive type. +//! For the constants defined directly in this module +//! (as distinct from those defined in the `consts` sub-module), +//! new code should instead use the associated constants +//! defined directly on the `f32` type. #![stable(feature = "rust1", since = "1.0.0")] #![allow(missing_docs)] +#[cfg(test)] +mod tests; + #[cfg(not(test))] use crate::intrinsics; #[cfg(not(test))] use crate::sys::cmath; #[stable(feature = "rust1", since = "1.0.0")] -pub use core::f32::consts; -#[stable(feature = "rust1", since = "1.0.0")] -pub use core::f32::{DIGITS, EPSILON, MANTISSA_DIGITS, RADIX}; -#[stable(feature = "rust1", since = "1.0.0")] -pub use core::f32::{INFINITY, MAX_10_EXP, NAN, NEG_INFINITY}; -#[stable(feature = "rust1", since = "1.0.0")] -pub use core::f32::{MAX, MIN, MIN_POSITIVE}; -#[stable(feature = "rust1", since = "1.0.0")] -pub use core::f32::{MAX_EXP, MIN_10_EXP, MIN_EXP}; +#[allow(deprecated, deprecated_in_future)] +pub use core::f32::{ + consts, DIGITS, EPSILON, INFINITY, MANTISSA_DIGITS, MAX, MAX_10_EXP, MAX_EXP, MIN, MIN_10_EXP, + MIN_EXP, MIN_POSITIVE, NAN, NEG_INFINITY, RADIX, +}; #[cfg(not(test))] -#[lang = "f32_runtime"] impl f32 { - /// Returns the largest integer less than or equal to a number. + /// Returns the largest integer less than or equal to `self`. /// /// # Examples /// @@ -43,6 +42,7 @@ impl f32 { /// assert_eq!(g.floor(), 3.0); /// assert_eq!(h.floor(), -4.0); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -50,7 +50,7 @@ impl f32 { unsafe { intrinsics::floorf32(self) } } - /// Returns the smallest integer greater than or equal to a number. + /// Returns the smallest integer greater than or equal to `self`. /// /// # Examples /// @@ -61,6 +61,7 @@ impl f32 { /// assert_eq!(f.ceil(), 4.0); /// assert_eq!(g.ceil(), 4.0); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -68,7 +69,7 @@ impl f32 { unsafe { intrinsics::ceilf32(self) } } - /// Returns the nearest integer to a number. Round half-way cases away from + /// Returns the nearest integer to `self`. Round half-way cases away from /// `0.0`. /// /// # Examples @@ -76,10 +77,13 @@ impl f32 { /// ``` /// let f = 3.3_f32; /// let g = -3.3_f32; + /// let h = -3.7_f32; /// /// assert_eq!(f.round(), 3.0); /// assert_eq!(g.round(), -3.0); + /// assert_eq!(h.round(), -4.0); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -87,7 +91,8 @@ impl f32 { unsafe { intrinsics::roundf32(self) } } - /// Returns the integer part of a number. + /// Returns the integer part of `self`. + /// This means that non-integer numbers are always truncated towards zero. /// /// # Examples /// @@ -100,6 +105,7 @@ impl f32 { /// assert_eq!(g.trunc(), 3.0); /// assert_eq!(h.trunc(), -3.0); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -107,13 +113,11 @@ impl f32 { unsafe { intrinsics::truncf32(self) } } - /// Returns the fractional part of a number. + /// Returns the fractional part of `self`. /// /// # Examples /// /// ``` - /// use std::f32; - /// /// let x = 3.6_f32; /// let y = -3.6_f32; /// let abs_difference_x = (x.fract() - 0.6).abs(); @@ -122,6 +126,7 @@ impl f32 { /// assert!(abs_difference_x <= f32::EPSILON); /// assert!(abs_difference_y <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -129,14 +134,11 @@ impl f32 { self - self.trunc() } - /// Computes the absolute value of `self`. Returns `NAN` if the - /// number is `NAN`. + /// Computes the absolute value of `self`. /// /// # Examples /// /// ``` - /// use std::f32; - /// /// let x = 3.5_f32; /// let y = -3.5_f32; /// @@ -148,6 +150,7 @@ impl f32 { /// /// assert!(f32::NAN.abs().is_nan()); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -159,13 +162,11 @@ impl f32 { /// /// - `1.0` if the number is positive, `+0.0` or `INFINITY` /// - `-1.0` if the number is negative, `-0.0` or `NEG_INFINITY` - /// - `NAN` if the number is `NAN` + /// - NaN if the number is NaN /// /// # Examples /// /// ``` - /// use std::f32; - /// /// let f = 3.5_f32; /// /// assert_eq!(f.signum(), 1.0); @@ -173,25 +174,26 @@ impl f32 { /// /// assert!(f32::NAN.signum().is_nan()); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn signum(self) -> f32 { - if self.is_nan() { NAN } else { 1.0_f32.copysign(self) } + if self.is_nan() { Self::NAN } else { 1.0_f32.copysign(self) } } /// Returns a number composed of the magnitude of `self` and the sign of /// `sign`. /// /// Equal to `self` if the sign of `self` and `sign` are the same, otherwise - /// equal to `-self`. If `self` is a `NAN`, then a `NAN` with the sign of - /// `sign` is returned. + /// equal to `-self`. If `self` is a NaN, then a NaN with the sign bit of + /// `sign` is returned. Note, however, that conserving the sign bit on NaN + /// across arithmetical operations is not generally guaranteed. + /// See [explanation of NaN as a special value](primitive@f32) for more info. /// /// # Examples /// /// ``` - /// use std::f32; - /// /// let f = 3.5_f32; /// /// assert_eq!(f.copysign(0.42), 3.5_f32); @@ -201,6 +203,7 @@ impl f32 { /// /// assert!(f32::NAN.copysign(1.0).is_nan()); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[inline] #[stable(feature = "copysign", since = "1.35.0")] @@ -211,14 +214,14 @@ impl f32 { /// Fused multiply-add. Computes `(self * a) + b` with only one rounding /// error, yielding a more accurate result than an unfused multiply-add. /// - /// Using `mul_add` can be more performant than an unfused multiply-add if - /// the target architecture has a dedicated `fma` CPU instruction. + /// Using `mul_add` *may* be more performant than an unfused multiply-add if + /// the target architecture has a dedicated `fma` CPU instruction. However, + /// this is not always true, and will be heavily dependant on designing + /// algorithms with specific target hardware in mind. /// /// # Examples /// /// ``` - /// use std::f32; - /// /// let m = 10.0_f32; /// let x = 4.0_f32; /// let b = 60.0_f32; @@ -228,6 +231,7 @@ impl f32 { /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -252,6 +256,7 @@ impl f32 { /// assert_eq!(a.div_euclid(-b), -1.0); // 7.0 >= -4.0 * -1.0 /// assert_eq!((-a).div_euclid(-b), 2.0); // -7.0 >= -4.0 * 2.0 /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[inline] #[stable(feature = "euclidean_division", since = "1.38.0")] @@ -272,7 +277,7 @@ impl f32 { /// This result is not an element of the function's codomain, but it is the /// closest floating point number in the real numbers and thus fulfills the /// property `self == self.div_euclid(rhs) * rhs + self.rem_euclid(rhs)` - /// approximatively. + /// approximately. /// /// # Examples /// @@ -284,8 +289,9 @@ impl f32 { /// assert_eq!(a.rem_euclid(-b), 3.0); /// assert_eq!((-a).rem_euclid(-b), 1.0); /// // limitation due to round-off error - /// assert!((-std::f32::EPSILON).rem_euclid(3.0) != 0.0); + /// assert!((-f32::EPSILON).rem_euclid(3.0) != 0.0); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[inline] #[stable(feature = "euclidean_division", since = "1.38.0")] @@ -296,18 +302,19 @@ impl f32 { /// Raises a number to an integer power. /// - /// Using this function is generally faster than using `powf` + /// Using this function is generally faster than using `powf`. + /// It might have a different sequence of rounding operations than `powf`, + /// so the results are not guaranteed to agree. /// /// # Examples /// /// ``` - /// use std::f32; - /// /// let x = 2.0_f32; /// let abs_difference = (x.powi(2) - (x * x)).abs(); /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -320,13 +327,12 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// /// let x = 2.0_f32; /// let abs_difference = (x.powf(2.0) - (x * x)).abs(); /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -336,21 +342,22 @@ impl f32 { /// Returns the square root of a number. /// - /// Returns NaN if `self` is a negative number. + /// Returns NaN if `self` is a negative number other than `-0.0`. /// /// # Examples /// /// ``` - /// use std::f32; - /// /// let positive = 4.0_f32; /// let negative = -4.0_f32; + /// let negative_zero = -0.0_f32; /// /// let abs_difference = (positive.sqrt() - 2.0).abs(); /// /// assert!(abs_difference <= f32::EPSILON); /// assert!(negative.sqrt().is_nan()); + /// assert!(negative_zero.sqrt() == negative_zero); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -363,8 +370,6 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// /// let one = 1.0f32; /// // e^1 /// let e = one.exp(); @@ -374,6 +379,7 @@ impl f32 { /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -386,8 +392,6 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// /// let f = 2.0f32; /// /// // 2^2 - 4 == 0 @@ -395,6 +399,7 @@ impl f32 { /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -407,8 +412,6 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// /// let one = 1.0f32; /// // e^1 /// let e = one.exp(); @@ -418,6 +421,7 @@ impl f32 { /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -427,15 +431,13 @@ impl f32 { /// Returns the logarithm of the number with respect to an arbitrary base. /// - /// The result may not be correctly rounded owing to implementation details; + /// The result might not be correctly rounded owing to implementation details; /// `self.log2()` can produce more accurate results for base 2, and /// `self.log10()` can produce more accurate results for base 10. /// /// # Examples /// /// ``` - /// use std::f32; - /// /// let five = 5.0f32; /// /// // log5(5) - 1 == 0 @@ -443,6 +445,7 @@ impl f32 { /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -455,8 +458,6 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// /// let two = 2.0f32; /// /// // log2(2) - 1 == 0 @@ -464,6 +465,7 @@ impl f32 { /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -479,8 +481,6 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// /// let ten = 10.0f32; /// /// // log10(10) - 1 == 0 @@ -488,6 +488,7 @@ impl f32 { /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -503,8 +504,6 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// /// let x = 3.0f32; /// let y = -3.0f32; /// @@ -514,30 +513,29 @@ impl f32 { /// assert!(abs_difference_x <= f32::EPSILON); /// assert!(abs_difference_y <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] - #[rustc_deprecated( + #[deprecated( since = "1.10.0", - reason = "you probably meant `(self - other).abs()`: \ - this operation is `(self - other).max(0.0)` \ - except that `abs_sub` also propagates NaNs (also \ - known as `fdimf` in C). If you truly need the positive \ - difference, consider using that expression or the C function \ - `fdimf`, depending on how you wish to handle NaN (please consider \ - filing an issue describing your use-case too)." + note = "you probably meant `(self - other).abs()`: \ + this operation is `(self - other).max(0.0)` \ + except that `abs_sub` also propagates NaNs (also \ + known as `fdimf` in C). If you truly need the positive \ + difference, consider using that expression or the C function \ + `fdimf`, depending on how you wish to handle NaN (please consider \ + filing an issue describing your use-case too)." )] pub fn abs_sub(self, other: f32) -> f32 { unsafe { cmath::fdimf(self, other) } } - /// Returns the cubic root of a number. + /// Returns the cube root of a number. /// /// # Examples /// /// ``` - /// use std::f32; - /// /// let x = 8.0f32; /// /// // x^(1/3) - 2 == 0 @@ -545,6 +543,7 @@ impl f32 { /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -558,8 +557,6 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// /// let x = 2.0f32; /// let y = 3.0f32; /// @@ -568,6 +565,7 @@ impl f32 { /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -580,14 +578,13 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// - /// let x = f32::consts::FRAC_PI_2; + /// let x = std::f32::consts::FRAC_PI_2; /// /// let abs_difference = (x.sin() - 1.0).abs(); /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -600,14 +597,13 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// - /// let x = 2.0 * f32::consts::PI; + /// let x = 2.0 * std::f32::consts::PI; /// /// let abs_difference = (x.cos() - 1.0).abs(); /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -620,13 +616,12 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// - /// let x = f32::consts::FRAC_PI_4; + /// let x = std::f32::consts::FRAC_PI_4; /// let abs_difference = (x.tan() - 1.0).abs(); /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -641,15 +636,14 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// - /// let f = f32::consts::FRAC_PI_2; + /// let f = std::f32::consts::FRAC_PI_2; /// /// // asin(sin(pi/2)) - /// let abs_difference = (f.sin().asin() - f32::consts::FRAC_PI_2).abs(); + /// let abs_difference = (f.sin().asin() - std::f32::consts::FRAC_PI_2).abs(); /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -664,15 +658,14 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// - /// let f = f32::consts::FRAC_PI_4; + /// let f = std::f32::consts::FRAC_PI_4; /// /// // acos(cos(pi/4)) - /// let abs_difference = (f.cos().acos() - f32::consts::FRAC_PI_4).abs(); + /// let abs_difference = (f.cos().acos() - std::f32::consts::FRAC_PI_4).abs(); /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -686,8 +679,6 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// /// let f = 1.0f32; /// /// // atan(tan(1)) @@ -695,6 +686,7 @@ impl f32 { /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -712,8 +704,6 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// /// // Positive angles measured counter-clockwise /// // from positive x axis /// // -pi/4 radians (45 deg clockwise) @@ -724,12 +714,13 @@ impl f32 { /// let x2 = -3.0f32; /// let y2 = 3.0f32; /// - /// let abs_difference_1 = (y1.atan2(x1) - (-f32::consts::FRAC_PI_4)).abs(); - /// let abs_difference_2 = (y2.atan2(x2) - (3.0 * f32::consts::FRAC_PI_4)).abs(); + /// let abs_difference_1 = (y1.atan2(x1) - (-std::f32::consts::FRAC_PI_4)).abs(); + /// let abs_difference_2 = (y2.atan2(x2) - (3.0 * std::f32::consts::FRAC_PI_4)).abs(); /// /// assert!(abs_difference_1 <= f32::EPSILON); /// assert!(abs_difference_2 <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -743,9 +734,7 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// - /// let x = f32::consts::FRAC_PI_4; + /// let x = std::f32::consts::FRAC_PI_4; /// let f = x.sin_cos(); /// /// let abs_difference_0 = (f.0 - x.sin()).abs(); @@ -754,6 +743,7 @@ impl f32 { /// assert!(abs_difference_0 <= f32::EPSILON); /// assert!(abs_difference_1 <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn sin_cos(self) -> (f32, f32) { @@ -766,15 +756,15 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// - /// let x = 6.0f32; + /// let x = 1e-8_f32; /// - /// // e^(ln(6)) - 1 - /// let abs_difference = (x.ln().exp_m1() - 5.0).abs(); + /// // for very small x, e^x is approximately 1 + x + x^2 / 2 + /// let approx = x + x * x / 2.0; + /// let abs_difference = (x.exp_m1() - approx).abs(); /// - /// assert!(abs_difference <= f32::EPSILON); + /// assert!(abs_difference < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -788,15 +778,15 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; + /// let x = 1e-8_f32; /// - /// let x = f32::consts::E - 1.0; + /// // for very small x, ln(1 + x) is approximately x - x^2 / 2 + /// let approx = x - x * x / 2.0; + /// let abs_difference = (x.ln_1p() - approx).abs(); /// - /// // ln(1 + (e - 1)) == ln(e) == 1 - /// let abs_difference = (x.ln_1p() - 1.0).abs(); - /// - /// assert!(abs_difference <= f32::EPSILON); + /// assert!(abs_difference < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -809,9 +799,7 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// - /// let e = f32::consts::E; + /// let e = std::f32::consts::E; /// let x = 1.0f32; /// /// let f = x.sinh(); @@ -821,6 +809,7 @@ impl f32 { /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -833,9 +822,7 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// - /// let e = f32::consts::E; + /// let e = std::f32::consts::E; /// let x = 1.0f32; /// let f = x.cosh(); /// // Solving cosh() at 1 gives this result @@ -845,6 +832,7 @@ impl f32 { /// // Same result /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -857,9 +845,7 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// - /// let e = f32::consts::E; + /// let e = std::f32::consts::E; /// let x = 1.0f32; /// /// let f = x.tanh(); @@ -869,6 +855,7 @@ impl f32 { /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -881,8 +868,6 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// /// let x = 1.0f32; /// let f = x.sinh().asinh(); /// @@ -890,15 +875,14 @@ impl f32 { /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn asinh(self) -> f32 { - if self == NEG_INFINITY { - NEG_INFINITY - } else { - (self + ((self * self) + 1.0).sqrt()).ln().copysign(self) - } + let ax = self.abs(); + let ix = 1.0 / ax; + (ax + (ax / (Self::hypot(1.0, ix) + ix))).ln_1p().copysign(self) } /// Inverse hyperbolic cosine function. @@ -906,8 +890,6 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// /// let x = 1.0f32; /// let f = x.cosh().acosh(); /// @@ -915,11 +897,16 @@ impl f32 { /// /// assert!(abs_difference <= f32::EPSILON); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn acosh(self) -> f32 { - if self < 1.0 { crate::f32::NAN } else { (self + ((self * self) - 1.0).sqrt()).ln() } + if self < 1.0 { + Self::NAN + } else { + (self + ((self - 1.0).sqrt() * (self + 1.0).sqrt())).ln() + } } /// Inverse hyperbolic tangent function. @@ -927,674 +914,18 @@ impl f32 { /// # Examples /// /// ``` - /// use std::f32; - /// - /// let e = f32::consts::E; + /// let e = std::f32::consts::E; /// let f = e.tanh().atanh(); /// /// let abs_difference = (f - e).abs(); /// /// assert!(abs_difference <= 1e-5); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn atanh(self) -> f32 { 0.5 * ((2.0 * self) / (1.0 - self)).ln_1p() } - - /// Restrict a value to a certain interval unless it is NaN. - /// - /// Returns `max` if `self` is greater than `max`, and `min` if `self` is - /// less than `min`. Otherwise this returns `self`. - /// - /// Not that this function returns NaN if the initial value was NaN as - /// well. - /// - /// # Panics - /// - /// Panics if `min > max`, `min` is NaN, or `max` is NaN. - /// - /// # Examples - /// - /// ``` - /// #![feature(clamp)] - /// assert!((-3.0f32).clamp(-2.0, 1.0) == -2.0); - /// assert!((0.0f32).clamp(-2.0, 1.0) == 0.0); - /// assert!((2.0f32).clamp(-2.0, 1.0) == 1.0); - /// assert!((std::f32::NAN).clamp(-2.0, 1.0).is_nan()); - /// ``` - #[must_use = "method returns a new number and does not mutate the original value"] - #[unstable(feature = "clamp", issue = "44095")] - #[inline] - pub fn clamp(self, min: f32, max: f32) -> f32 { - assert!(min <= max); - let mut x = self; - if x < min { - x = min; - } - if x > max { - x = max; - } - x - } -} - -#[cfg(test)] -mod tests { - use crate::f32; - use crate::f32::*; - use crate::num::FpCategory as Fp; - use crate::num::*; - - #[test] - fn test_num_f32() { - test_num(10f32, 2f32); - } - - #[test] - fn test_min_nan() { - assert_eq!(NAN.min(2.0), 2.0); - assert_eq!(2.0f32.min(NAN), 2.0); - } - - #[test] - fn test_max_nan() { - assert_eq!(NAN.max(2.0), 2.0); - assert_eq!(2.0f32.max(NAN), 2.0); - } - - #[test] - fn test_nan() { - let nan: f32 = f32::NAN; - assert!(nan.is_nan()); - assert!(!nan.is_infinite()); - assert!(!nan.is_finite()); - assert!(!nan.is_normal()); - assert!(nan.is_sign_positive()); - assert!(!nan.is_sign_negative()); - assert_eq!(Fp::Nan, nan.classify()); - } - - #[test] - fn test_infinity() { - let inf: f32 = f32::INFINITY; - assert!(inf.is_infinite()); - assert!(!inf.is_finite()); - assert!(inf.is_sign_positive()); - assert!(!inf.is_sign_negative()); - assert!(!inf.is_nan()); - assert!(!inf.is_normal()); - assert_eq!(Fp::Infinite, inf.classify()); - } - - #[test] - fn test_neg_infinity() { - let neg_inf: f32 = f32::NEG_INFINITY; - assert!(neg_inf.is_infinite()); - assert!(!neg_inf.is_finite()); - assert!(!neg_inf.is_sign_positive()); - assert!(neg_inf.is_sign_negative()); - assert!(!neg_inf.is_nan()); - assert!(!neg_inf.is_normal()); - assert_eq!(Fp::Infinite, neg_inf.classify()); - } - - #[test] - fn test_zero() { - let zero: f32 = 0.0f32; - assert_eq!(0.0, zero); - assert!(!zero.is_infinite()); - assert!(zero.is_finite()); - assert!(zero.is_sign_positive()); - assert!(!zero.is_sign_negative()); - assert!(!zero.is_nan()); - assert!(!zero.is_normal()); - assert_eq!(Fp::Zero, zero.classify()); - } - - #[test] - fn test_neg_zero() { - let neg_zero: f32 = -0.0; - assert_eq!(0.0, neg_zero); - assert!(!neg_zero.is_infinite()); - assert!(neg_zero.is_finite()); - assert!(!neg_zero.is_sign_positive()); - assert!(neg_zero.is_sign_negative()); - assert!(!neg_zero.is_nan()); - assert!(!neg_zero.is_normal()); - assert_eq!(Fp::Zero, neg_zero.classify()); - } - - #[test] - fn test_one() { - let one: f32 = 1.0f32; - assert_eq!(1.0, one); - assert!(!one.is_infinite()); - assert!(one.is_finite()); - assert!(one.is_sign_positive()); - assert!(!one.is_sign_negative()); - assert!(!one.is_nan()); - assert!(one.is_normal()); - assert_eq!(Fp::Normal, one.classify()); - } - - #[test] - fn test_is_nan() { - let nan: f32 = f32::NAN; - let inf: f32 = f32::INFINITY; - let neg_inf: f32 = f32::NEG_INFINITY; - assert!(nan.is_nan()); - assert!(!0.0f32.is_nan()); - assert!(!5.3f32.is_nan()); - assert!(!(-10.732f32).is_nan()); - assert!(!inf.is_nan()); - assert!(!neg_inf.is_nan()); - } - - #[test] - fn test_is_infinite() { - let nan: f32 = f32::NAN; - let inf: f32 = f32::INFINITY; - let neg_inf: f32 = f32::NEG_INFINITY; - assert!(!nan.is_infinite()); - assert!(inf.is_infinite()); - assert!(neg_inf.is_infinite()); - assert!(!0.0f32.is_infinite()); - assert!(!42.8f32.is_infinite()); - assert!(!(-109.2f32).is_infinite()); - } - - #[test] - fn test_is_finite() { - let nan: f32 = f32::NAN; - let inf: f32 = f32::INFINITY; - let neg_inf: f32 = f32::NEG_INFINITY; - assert!(!nan.is_finite()); - assert!(!inf.is_finite()); - assert!(!neg_inf.is_finite()); - assert!(0.0f32.is_finite()); - assert!(42.8f32.is_finite()); - assert!((-109.2f32).is_finite()); - } - - #[test] - fn test_is_normal() { - let nan: f32 = f32::NAN; - let inf: f32 = f32::INFINITY; - let neg_inf: f32 = f32::NEG_INFINITY; - let zero: f32 = 0.0f32; - let neg_zero: f32 = -0.0; - assert!(!nan.is_normal()); - assert!(!inf.is_normal()); - assert!(!neg_inf.is_normal()); - assert!(!zero.is_normal()); - assert!(!neg_zero.is_normal()); - assert!(1f32.is_normal()); - assert!(1e-37f32.is_normal()); - assert!(!1e-38f32.is_normal()); - } - - #[test] - fn test_classify() { - let nan: f32 = f32::NAN; - let inf: f32 = f32::INFINITY; - let neg_inf: f32 = f32::NEG_INFINITY; - let zero: f32 = 0.0f32; - let neg_zero: f32 = -0.0; - assert_eq!(nan.classify(), Fp::Nan); - assert_eq!(inf.classify(), Fp::Infinite); - assert_eq!(neg_inf.classify(), Fp::Infinite); - assert_eq!(zero.classify(), Fp::Zero); - assert_eq!(neg_zero.classify(), Fp::Zero); - assert_eq!(1f32.classify(), Fp::Normal); - assert_eq!(1e-37f32.classify(), Fp::Normal); - assert_eq!(1e-38f32.classify(), Fp::Subnormal); - } - - #[test] - fn test_floor() { - assert_approx_eq!(1.0f32.floor(), 1.0f32); - assert_approx_eq!(1.3f32.floor(), 1.0f32); - assert_approx_eq!(1.5f32.floor(), 1.0f32); - assert_approx_eq!(1.7f32.floor(), 1.0f32); - assert_approx_eq!(0.0f32.floor(), 0.0f32); - assert_approx_eq!((-0.0f32).floor(), -0.0f32); - assert_approx_eq!((-1.0f32).floor(), -1.0f32); - assert_approx_eq!((-1.3f32).floor(), -2.0f32); - assert_approx_eq!((-1.5f32).floor(), -2.0f32); - assert_approx_eq!((-1.7f32).floor(), -2.0f32); - } - - #[test] - fn test_ceil() { - assert_approx_eq!(1.0f32.ceil(), 1.0f32); - assert_approx_eq!(1.3f32.ceil(), 2.0f32); - assert_approx_eq!(1.5f32.ceil(), 2.0f32); - assert_approx_eq!(1.7f32.ceil(), 2.0f32); - assert_approx_eq!(0.0f32.ceil(), 0.0f32); - assert_approx_eq!((-0.0f32).ceil(), -0.0f32); - assert_approx_eq!((-1.0f32).ceil(), -1.0f32); - assert_approx_eq!((-1.3f32).ceil(), -1.0f32); - assert_approx_eq!((-1.5f32).ceil(), -1.0f32); - assert_approx_eq!((-1.7f32).ceil(), -1.0f32); - } - - #[test] - fn test_round() { - assert_approx_eq!(1.0f32.round(), 1.0f32); - assert_approx_eq!(1.3f32.round(), 1.0f32); - assert_approx_eq!(1.5f32.round(), 2.0f32); - assert_approx_eq!(1.7f32.round(), 2.0f32); - assert_approx_eq!(0.0f32.round(), 0.0f32); - assert_approx_eq!((-0.0f32).round(), -0.0f32); - assert_approx_eq!((-1.0f32).round(), -1.0f32); - assert_approx_eq!((-1.3f32).round(), -1.0f32); - assert_approx_eq!((-1.5f32).round(), -2.0f32); - assert_approx_eq!((-1.7f32).round(), -2.0f32); - } - - #[test] - fn test_trunc() { - assert_approx_eq!(1.0f32.trunc(), 1.0f32); - assert_approx_eq!(1.3f32.trunc(), 1.0f32); - assert_approx_eq!(1.5f32.trunc(), 1.0f32); - assert_approx_eq!(1.7f32.trunc(), 1.0f32); - assert_approx_eq!(0.0f32.trunc(), 0.0f32); - assert_approx_eq!((-0.0f32).trunc(), -0.0f32); - assert_approx_eq!((-1.0f32).trunc(), -1.0f32); - assert_approx_eq!((-1.3f32).trunc(), -1.0f32); - assert_approx_eq!((-1.5f32).trunc(), -1.0f32); - assert_approx_eq!((-1.7f32).trunc(), -1.0f32); - } - - #[test] - fn test_fract() { - assert_approx_eq!(1.0f32.fract(), 0.0f32); - assert_approx_eq!(1.3f32.fract(), 0.3f32); - assert_approx_eq!(1.5f32.fract(), 0.5f32); - assert_approx_eq!(1.7f32.fract(), 0.7f32); - assert_approx_eq!(0.0f32.fract(), 0.0f32); - assert_approx_eq!((-0.0f32).fract(), -0.0f32); - assert_approx_eq!((-1.0f32).fract(), -0.0f32); - assert_approx_eq!((-1.3f32).fract(), -0.3f32); - assert_approx_eq!((-1.5f32).fract(), -0.5f32); - assert_approx_eq!((-1.7f32).fract(), -0.7f32); - } - - #[test] - fn test_abs() { - assert_eq!(INFINITY.abs(), INFINITY); - assert_eq!(1f32.abs(), 1f32); - assert_eq!(0f32.abs(), 0f32); - assert_eq!((-0f32).abs(), 0f32); - assert_eq!((-1f32).abs(), 1f32); - assert_eq!(NEG_INFINITY.abs(), INFINITY); - assert_eq!((1f32 / NEG_INFINITY).abs(), 0f32); - assert!(NAN.abs().is_nan()); - } - - #[test] - fn test_signum() { - assert_eq!(INFINITY.signum(), 1f32); - assert_eq!(1f32.signum(), 1f32); - assert_eq!(0f32.signum(), 1f32); - assert_eq!((-0f32).signum(), -1f32); - assert_eq!((-1f32).signum(), -1f32); - assert_eq!(NEG_INFINITY.signum(), -1f32); - assert_eq!((1f32 / NEG_INFINITY).signum(), -1f32); - assert!(NAN.signum().is_nan()); - } - - #[test] - fn test_is_sign_positive() { - assert!(INFINITY.is_sign_positive()); - assert!(1f32.is_sign_positive()); - assert!(0f32.is_sign_positive()); - assert!(!(-0f32).is_sign_positive()); - assert!(!(-1f32).is_sign_positive()); - assert!(!NEG_INFINITY.is_sign_positive()); - assert!(!(1f32 / NEG_INFINITY).is_sign_positive()); - assert!(NAN.is_sign_positive()); - assert!(!(-NAN).is_sign_positive()); - } - - #[test] - fn test_is_sign_negative() { - assert!(!INFINITY.is_sign_negative()); - assert!(!1f32.is_sign_negative()); - assert!(!0f32.is_sign_negative()); - assert!((-0f32).is_sign_negative()); - assert!((-1f32).is_sign_negative()); - assert!(NEG_INFINITY.is_sign_negative()); - assert!((1f32 / NEG_INFINITY).is_sign_negative()); - assert!(!NAN.is_sign_negative()); - assert!((-NAN).is_sign_negative()); - } - - #[test] - fn test_mul_add() { - let nan: f32 = f32::NAN; - let inf: f32 = f32::INFINITY; - let neg_inf: f32 = f32::NEG_INFINITY; - assert_approx_eq!(12.3f32.mul_add(4.5, 6.7), 62.05); - assert_approx_eq!((-12.3f32).mul_add(-4.5, -6.7), 48.65); - assert_approx_eq!(0.0f32.mul_add(8.9, 1.2), 1.2); - assert_approx_eq!(3.4f32.mul_add(-0.0, 5.6), 5.6); - assert!(nan.mul_add(7.8, 9.0).is_nan()); - assert_eq!(inf.mul_add(7.8, 9.0), inf); - assert_eq!(neg_inf.mul_add(7.8, 9.0), neg_inf); - assert_eq!(8.9f32.mul_add(inf, 3.2), inf); - assert_eq!((-3.2f32).mul_add(2.4, neg_inf), neg_inf); - } - - #[test] - fn test_recip() { - let nan: f32 = f32::NAN; - let inf: f32 = f32::INFINITY; - let neg_inf: f32 = f32::NEG_INFINITY; - assert_eq!(1.0f32.recip(), 1.0); - assert_eq!(2.0f32.recip(), 0.5); - assert_eq!((-0.4f32).recip(), -2.5); - assert_eq!(0.0f32.recip(), inf); - assert!(nan.recip().is_nan()); - assert_eq!(inf.recip(), 0.0); - assert_eq!(neg_inf.recip(), 0.0); - } - - #[test] - fn test_powi() { - let nan: f32 = f32::NAN; - let inf: f32 = f32::INFINITY; - let neg_inf: f32 = f32::NEG_INFINITY; - assert_eq!(1.0f32.powi(1), 1.0); - assert_approx_eq!((-3.1f32).powi(2), 9.61); - assert_approx_eq!(5.9f32.powi(-2), 0.028727); - assert_eq!(8.3f32.powi(0), 1.0); - assert!(nan.powi(2).is_nan()); - assert_eq!(inf.powi(3), inf); - assert_eq!(neg_inf.powi(2), inf); - } - - #[test] - fn test_powf() { - let nan: f32 = f32::NAN; - let inf: f32 = f32::INFINITY; - let neg_inf: f32 = f32::NEG_INFINITY; - assert_eq!(1.0f32.powf(1.0), 1.0); - assert_approx_eq!(3.4f32.powf(4.5), 246.408218); - assert_approx_eq!(2.7f32.powf(-3.2), 0.041652); - assert_approx_eq!((-3.1f32).powf(2.0), 9.61); - assert_approx_eq!(5.9f32.powf(-2.0), 0.028727); - assert_eq!(8.3f32.powf(0.0), 1.0); - assert!(nan.powf(2.0).is_nan()); - assert_eq!(inf.powf(2.0), inf); - assert_eq!(neg_inf.powf(3.0), neg_inf); - } - - #[test] - fn test_sqrt_domain() { - assert!(NAN.sqrt().is_nan()); - assert!(NEG_INFINITY.sqrt().is_nan()); - assert!((-1.0f32).sqrt().is_nan()); - assert_eq!((-0.0f32).sqrt(), -0.0); - assert_eq!(0.0f32.sqrt(), 0.0); - assert_eq!(1.0f32.sqrt(), 1.0); - assert_eq!(INFINITY.sqrt(), INFINITY); - } - - #[test] - fn test_exp() { - assert_eq!(1.0, 0.0f32.exp()); - assert_approx_eq!(2.718282, 1.0f32.exp()); - assert_approx_eq!(148.413162, 5.0f32.exp()); - - let inf: f32 = f32::INFINITY; - let neg_inf: f32 = f32::NEG_INFINITY; - let nan: f32 = f32::NAN; - assert_eq!(inf, inf.exp()); - assert_eq!(0.0, neg_inf.exp()); - assert!(nan.exp().is_nan()); - } - - #[test] - fn test_exp2() { - assert_eq!(32.0, 5.0f32.exp2()); - assert_eq!(1.0, 0.0f32.exp2()); - - let inf: f32 = f32::INFINITY; - let neg_inf: f32 = f32::NEG_INFINITY; - let nan: f32 = f32::NAN; - assert_eq!(inf, inf.exp2()); - assert_eq!(0.0, neg_inf.exp2()); - assert!(nan.exp2().is_nan()); - } - - #[test] - fn test_ln() { - let nan: f32 = f32::NAN; - let inf: f32 = f32::INFINITY; - let neg_inf: f32 = f32::NEG_INFINITY; - assert_approx_eq!(1.0f32.exp().ln(), 1.0); - assert!(nan.ln().is_nan()); - assert_eq!(inf.ln(), inf); - assert!(neg_inf.ln().is_nan()); - assert!((-2.3f32).ln().is_nan()); - assert_eq!((-0.0f32).ln(), neg_inf); - assert_eq!(0.0f32.ln(), neg_inf); - assert_approx_eq!(4.0f32.ln(), 1.386294); - } - - #[test] - fn test_log() { - let nan: f32 = f32::NAN; - let inf: f32 = f32::INFINITY; - let neg_inf: f32 = f32::NEG_INFINITY; - assert_eq!(10.0f32.log(10.0), 1.0); - assert_approx_eq!(2.3f32.log(3.5), 0.664858); - assert_eq!(1.0f32.exp().log(1.0f32.exp()), 1.0); - assert!(1.0f32.log(1.0).is_nan()); - assert!(1.0f32.log(-13.9).is_nan()); - assert!(nan.log(2.3).is_nan()); - assert_eq!(inf.log(10.0), inf); - assert!(neg_inf.log(8.8).is_nan()); - assert!((-2.3f32).log(0.1).is_nan()); - assert_eq!((-0.0f32).log(2.0), neg_inf); - assert_eq!(0.0f32.log(7.0), neg_inf); - } - - #[test] - fn test_log2() { - let nan: f32 = f32::NAN; - let inf: f32 = f32::INFINITY; - let neg_inf: f32 = f32::NEG_INFINITY; - assert_approx_eq!(10.0f32.log2(), 3.321928); - assert_approx_eq!(2.3f32.log2(), 1.201634); - assert_approx_eq!(1.0f32.exp().log2(), 1.442695); - assert!(nan.log2().is_nan()); - assert_eq!(inf.log2(), inf); - assert!(neg_inf.log2().is_nan()); - assert!((-2.3f32).log2().is_nan()); - assert_eq!((-0.0f32).log2(), neg_inf); - assert_eq!(0.0f32.log2(), neg_inf); - } - - #[test] - fn test_log10() { - let nan: f32 = f32::NAN; - let inf: f32 = f32::INFINITY; - let neg_inf: f32 = f32::NEG_INFINITY; - assert_eq!(10.0f32.log10(), 1.0); - assert_approx_eq!(2.3f32.log10(), 0.361728); - assert_approx_eq!(1.0f32.exp().log10(), 0.434294); - assert_eq!(1.0f32.log10(), 0.0); - assert!(nan.log10().is_nan()); - assert_eq!(inf.log10(), inf); - assert!(neg_inf.log10().is_nan()); - assert!((-2.3f32).log10().is_nan()); - assert_eq!((-0.0f32).log10(), neg_inf); - assert_eq!(0.0f32.log10(), neg_inf); - } - - #[test] - fn test_to_degrees() { - let pi: f32 = consts::PI; - let nan: f32 = f32::NAN; - let inf: f32 = f32::INFINITY; - let neg_inf: f32 = f32::NEG_INFINITY; - assert_eq!(0.0f32.to_degrees(), 0.0); - assert_approx_eq!((-5.8f32).to_degrees(), -332.315521); - assert_eq!(pi.to_degrees(), 180.0); - assert!(nan.to_degrees().is_nan()); - assert_eq!(inf.to_degrees(), inf); - assert_eq!(neg_inf.to_degrees(), neg_inf); - assert_eq!(1_f32.to_degrees(), 57.2957795130823208767981548141051703); - } - - #[test] - fn test_to_radians() { - let pi: f32 = consts::PI; - let nan: f32 = f32::NAN; - let inf: f32 = f32::INFINITY; - let neg_inf: f32 = f32::NEG_INFINITY; - assert_eq!(0.0f32.to_radians(), 0.0); - assert_approx_eq!(154.6f32.to_radians(), 2.698279); - assert_approx_eq!((-332.31f32).to_radians(), -5.799903); - assert_eq!(180.0f32.to_radians(), pi); - assert!(nan.to_radians().is_nan()); - assert_eq!(inf.to_radians(), inf); - assert_eq!(neg_inf.to_radians(), neg_inf); - } - - #[test] - fn test_asinh() { - assert_eq!(0.0f32.asinh(), 0.0f32); - assert_eq!((-0.0f32).asinh(), -0.0f32); - - let inf: f32 = f32::INFINITY; - let neg_inf: f32 = f32::NEG_INFINITY; - let nan: f32 = f32::NAN; - assert_eq!(inf.asinh(), inf); - assert_eq!(neg_inf.asinh(), neg_inf); - assert!(nan.asinh().is_nan()); - assert!((-0.0f32).asinh().is_sign_negative()); // issue 63271 - assert_approx_eq!(2.0f32.asinh(), 1.443635475178810342493276740273105f32); - assert_approx_eq!((-2.0f32).asinh(), -1.443635475178810342493276740273105f32); - } - - #[test] - fn test_acosh() { - assert_eq!(1.0f32.acosh(), 0.0f32); - assert!(0.999f32.acosh().is_nan()); - - let inf: f32 = f32::INFINITY; - let neg_inf: f32 = f32::NEG_INFINITY; - let nan: f32 = f32::NAN; - assert_eq!(inf.acosh(), inf); - assert!(neg_inf.acosh().is_nan()); - assert!(nan.acosh().is_nan()); - assert_approx_eq!(2.0f32.acosh(), 1.31695789692481670862504634730796844f32); - assert_approx_eq!(3.0f32.acosh(), 1.76274717403908605046521864995958461f32); - } - - #[test] - fn test_atanh() { - assert_eq!(0.0f32.atanh(), 0.0f32); - assert_eq!((-0.0f32).atanh(), -0.0f32); - - let inf32: f32 = f32::INFINITY; - let neg_inf32: f32 = f32::NEG_INFINITY; - assert_eq!(1.0f32.atanh(), inf32); - assert_eq!((-1.0f32).atanh(), neg_inf32); - - assert!(2f64.atanh().atanh().is_nan()); - assert!((-2f64).atanh().atanh().is_nan()); - - let inf64: f32 = f32::INFINITY; - let neg_inf64: f32 = f32::NEG_INFINITY; - let nan32: f32 = f32::NAN; - assert!(inf64.atanh().is_nan()); - assert!(neg_inf64.atanh().is_nan()); - assert!(nan32.atanh().is_nan()); - - assert_approx_eq!(0.5f32.atanh(), 0.54930614433405484569762261846126285f32); - assert_approx_eq!((-0.5f32).atanh(), -0.54930614433405484569762261846126285f32); - } - - #[test] - fn test_real_consts() { - use super::consts; - - let pi: f32 = consts::PI; - let frac_pi_2: f32 = consts::FRAC_PI_2; - let frac_pi_3: f32 = consts::FRAC_PI_3; - let frac_pi_4: f32 = consts::FRAC_PI_4; - let frac_pi_6: f32 = consts::FRAC_PI_6; - let frac_pi_8: f32 = consts::FRAC_PI_8; - let frac_1_pi: f32 = consts::FRAC_1_PI; - let frac_2_pi: f32 = consts::FRAC_2_PI; - let frac_2_sqrtpi: f32 = consts::FRAC_2_SQRT_PI; - let sqrt2: f32 = consts::SQRT_2; - let frac_1_sqrt2: f32 = consts::FRAC_1_SQRT_2; - let e: f32 = consts::E; - let log2_e: f32 = consts::LOG2_E; - let log10_e: f32 = consts::LOG10_E; - let ln_2: f32 = consts::LN_2; - let ln_10: f32 = consts::LN_10; - - assert_approx_eq!(frac_pi_2, pi / 2f32); - assert_approx_eq!(frac_pi_3, pi / 3f32); - assert_approx_eq!(frac_pi_4, pi / 4f32); - assert_approx_eq!(frac_pi_6, pi / 6f32); - assert_approx_eq!(frac_pi_8, pi / 8f32); - assert_approx_eq!(frac_1_pi, 1f32 / pi); - assert_approx_eq!(frac_2_pi, 2f32 / pi); - assert_approx_eq!(frac_2_sqrtpi, 2f32 / pi.sqrt()); - assert_approx_eq!(sqrt2, 2f32.sqrt()); - assert_approx_eq!(frac_1_sqrt2, 1f32 / 2f32.sqrt()); - assert_approx_eq!(log2_e, e.log2()); - assert_approx_eq!(log10_e, e.log10()); - assert_approx_eq!(ln_2, 2f32.ln()); - assert_approx_eq!(ln_10, 10f32.ln()); - } - - #[test] - fn test_float_bits_conv() { - assert_eq!((1f32).to_bits(), 0x3f800000); - assert_eq!((12.5f32).to_bits(), 0x41480000); - assert_eq!((1337f32).to_bits(), 0x44a72000); - assert_eq!((-14.25f32).to_bits(), 0xc1640000); - assert_approx_eq!(f32::from_bits(0x3f800000), 1.0); - assert_approx_eq!(f32::from_bits(0x41480000), 12.5); - assert_approx_eq!(f32::from_bits(0x44a72000), 1337.0); - assert_approx_eq!(f32::from_bits(0xc1640000), -14.25); - - // Check that NaNs roundtrip their bits regardless of signalingness - // 0xA is 0b1010; 0x5 is 0b0101 -- so these two together clobbers all the mantissa bits - let masked_nan1 = f32::NAN.to_bits() ^ 0x002A_AAAA; - let masked_nan2 = f32::NAN.to_bits() ^ 0x0055_5555; - assert!(f32::from_bits(masked_nan1).is_nan()); - assert!(f32::from_bits(masked_nan2).is_nan()); - - assert_eq!(f32::from_bits(masked_nan1).to_bits(), masked_nan1); - assert_eq!(f32::from_bits(masked_nan2).to_bits(), masked_nan2); - } - - #[test] - #[should_panic] - fn test_clamp_min_greater_than_max() { - let _ = 1.0f32.clamp(3.0, 1.0); - } - - #[test] - #[should_panic] - fn test_clamp_min_is_nan() { - let _ = 1.0f32.clamp(NAN, 1.0); - } - - #[test] - #[should_panic] - fn test_clamp_max_is_nan() { - let _ = 1.0f32.clamp(3.0, NAN); - } } diff --git a/crux-mir/lib/std/src/f32/tests.rs b/crux-mir/lib/std/src/f32/tests.rs new file mode 100644 index 000000000..6ee295de6 --- /dev/null +++ b/crux-mir/lib/std/src/f32/tests.rs @@ -0,0 +1,857 @@ +use crate::f32::consts; +use crate::num::FpCategory as Fp; +use crate::num::*; + +#[test] +fn test_num_f32() { + test_num(10f32, 2f32); +} + +#[test] +fn test_min_nan() { + assert_eq!(f32::NAN.min(2.0), 2.0); + assert_eq!(2.0f32.min(f32::NAN), 2.0); +} + +#[test] +fn test_max_nan() { + assert_eq!(f32::NAN.max(2.0), 2.0); + assert_eq!(2.0f32.max(f32::NAN), 2.0); +} + +#[test] +fn test_minimum() { + assert!(f32::NAN.minimum(2.0).is_nan()); + assert!(2.0f32.minimum(f32::NAN).is_nan()); +} + +#[test] +fn test_maximum() { + assert!(f32::NAN.maximum(2.0).is_nan()); + assert!(2.0f32.maximum(f32::NAN).is_nan()); +} + +#[test] +fn test_nan() { + let nan: f32 = f32::NAN; + assert!(nan.is_nan()); + assert!(!nan.is_infinite()); + assert!(!nan.is_finite()); + assert!(!nan.is_normal()); + assert!(nan.is_sign_positive()); + assert!(!nan.is_sign_negative()); + assert_eq!(Fp::Nan, nan.classify()); +} + +#[test] +fn test_infinity() { + let inf: f32 = f32::INFINITY; + assert!(inf.is_infinite()); + assert!(!inf.is_finite()); + assert!(inf.is_sign_positive()); + assert!(!inf.is_sign_negative()); + assert!(!inf.is_nan()); + assert!(!inf.is_normal()); + assert_eq!(Fp::Infinite, inf.classify()); +} + +#[test] +fn test_neg_infinity() { + let neg_inf: f32 = f32::NEG_INFINITY; + assert!(neg_inf.is_infinite()); + assert!(!neg_inf.is_finite()); + assert!(!neg_inf.is_sign_positive()); + assert!(neg_inf.is_sign_negative()); + assert!(!neg_inf.is_nan()); + assert!(!neg_inf.is_normal()); + assert_eq!(Fp::Infinite, neg_inf.classify()); +} + +#[test] +fn test_zero() { + let zero: f32 = 0.0f32; + assert_eq!(0.0, zero); + assert!(!zero.is_infinite()); + assert!(zero.is_finite()); + assert!(zero.is_sign_positive()); + assert!(!zero.is_sign_negative()); + assert!(!zero.is_nan()); + assert!(!zero.is_normal()); + assert_eq!(Fp::Zero, zero.classify()); +} + +#[test] +fn test_neg_zero() { + let neg_zero: f32 = -0.0; + assert_eq!(0.0, neg_zero); + assert!(!neg_zero.is_infinite()); + assert!(neg_zero.is_finite()); + assert!(!neg_zero.is_sign_positive()); + assert!(neg_zero.is_sign_negative()); + assert!(!neg_zero.is_nan()); + assert!(!neg_zero.is_normal()); + assert_eq!(Fp::Zero, neg_zero.classify()); +} + +#[test] +fn test_one() { + let one: f32 = 1.0f32; + assert_eq!(1.0, one); + assert!(!one.is_infinite()); + assert!(one.is_finite()); + assert!(one.is_sign_positive()); + assert!(!one.is_sign_negative()); + assert!(!one.is_nan()); + assert!(one.is_normal()); + assert_eq!(Fp::Normal, one.classify()); +} + +#[test] +fn test_is_nan() { + let nan: f32 = f32::NAN; + let inf: f32 = f32::INFINITY; + let neg_inf: f32 = f32::NEG_INFINITY; + assert!(nan.is_nan()); + assert!(!0.0f32.is_nan()); + assert!(!5.3f32.is_nan()); + assert!(!(-10.732f32).is_nan()); + assert!(!inf.is_nan()); + assert!(!neg_inf.is_nan()); +} + +#[test] +fn test_is_infinite() { + let nan: f32 = f32::NAN; + let inf: f32 = f32::INFINITY; + let neg_inf: f32 = f32::NEG_INFINITY; + assert!(!nan.is_infinite()); + assert!(inf.is_infinite()); + assert!(neg_inf.is_infinite()); + assert!(!0.0f32.is_infinite()); + assert!(!42.8f32.is_infinite()); + assert!(!(-109.2f32).is_infinite()); +} + +#[test] +fn test_is_finite() { + let nan: f32 = f32::NAN; + let inf: f32 = f32::INFINITY; + let neg_inf: f32 = f32::NEG_INFINITY; + assert!(!nan.is_finite()); + assert!(!inf.is_finite()); + assert!(!neg_inf.is_finite()); + assert!(0.0f32.is_finite()); + assert!(42.8f32.is_finite()); + assert!((-109.2f32).is_finite()); +} + +#[test] +fn test_is_normal() { + let nan: f32 = f32::NAN; + let inf: f32 = f32::INFINITY; + let neg_inf: f32 = f32::NEG_INFINITY; + let zero: f32 = 0.0f32; + let neg_zero: f32 = -0.0; + assert!(!nan.is_normal()); + assert!(!inf.is_normal()); + assert!(!neg_inf.is_normal()); + assert!(!zero.is_normal()); + assert!(!neg_zero.is_normal()); + assert!(1f32.is_normal()); + assert!(1e-37f32.is_normal()); + assert!(!1e-38f32.is_normal()); +} + +#[test] +fn test_classify() { + let nan: f32 = f32::NAN; + let inf: f32 = f32::INFINITY; + let neg_inf: f32 = f32::NEG_INFINITY; + let zero: f32 = 0.0f32; + let neg_zero: f32 = -0.0; + assert_eq!(nan.classify(), Fp::Nan); + assert_eq!(inf.classify(), Fp::Infinite); + assert_eq!(neg_inf.classify(), Fp::Infinite); + assert_eq!(zero.classify(), Fp::Zero); + assert_eq!(neg_zero.classify(), Fp::Zero); + assert_eq!(1f32.classify(), Fp::Normal); + assert_eq!(1e-37f32.classify(), Fp::Normal); + assert_eq!(1e-38f32.classify(), Fp::Subnormal); +} + +#[test] +fn test_floor() { + assert_approx_eq!(1.0f32.floor(), 1.0f32); + assert_approx_eq!(1.3f32.floor(), 1.0f32); + assert_approx_eq!(1.5f32.floor(), 1.0f32); + assert_approx_eq!(1.7f32.floor(), 1.0f32); + assert_approx_eq!(0.0f32.floor(), 0.0f32); + assert_approx_eq!((-0.0f32).floor(), -0.0f32); + assert_approx_eq!((-1.0f32).floor(), -1.0f32); + assert_approx_eq!((-1.3f32).floor(), -2.0f32); + assert_approx_eq!((-1.5f32).floor(), -2.0f32); + assert_approx_eq!((-1.7f32).floor(), -2.0f32); +} + +#[test] +fn test_ceil() { + assert_approx_eq!(1.0f32.ceil(), 1.0f32); + assert_approx_eq!(1.3f32.ceil(), 2.0f32); + assert_approx_eq!(1.5f32.ceil(), 2.0f32); + assert_approx_eq!(1.7f32.ceil(), 2.0f32); + assert_approx_eq!(0.0f32.ceil(), 0.0f32); + assert_approx_eq!((-0.0f32).ceil(), -0.0f32); + assert_approx_eq!((-1.0f32).ceil(), -1.0f32); + assert_approx_eq!((-1.3f32).ceil(), -1.0f32); + assert_approx_eq!((-1.5f32).ceil(), -1.0f32); + assert_approx_eq!((-1.7f32).ceil(), -1.0f32); +} + +#[test] +fn test_round() { + assert_approx_eq!(1.0f32.round(), 1.0f32); + assert_approx_eq!(1.3f32.round(), 1.0f32); + assert_approx_eq!(1.5f32.round(), 2.0f32); + assert_approx_eq!(1.7f32.round(), 2.0f32); + assert_approx_eq!(0.0f32.round(), 0.0f32); + assert_approx_eq!((-0.0f32).round(), -0.0f32); + assert_approx_eq!((-1.0f32).round(), -1.0f32); + assert_approx_eq!((-1.3f32).round(), -1.0f32); + assert_approx_eq!((-1.5f32).round(), -2.0f32); + assert_approx_eq!((-1.7f32).round(), -2.0f32); +} + +#[test] +fn test_trunc() { + assert_approx_eq!(1.0f32.trunc(), 1.0f32); + assert_approx_eq!(1.3f32.trunc(), 1.0f32); + assert_approx_eq!(1.5f32.trunc(), 1.0f32); + assert_approx_eq!(1.7f32.trunc(), 1.0f32); + assert_approx_eq!(0.0f32.trunc(), 0.0f32); + assert_approx_eq!((-0.0f32).trunc(), -0.0f32); + assert_approx_eq!((-1.0f32).trunc(), -1.0f32); + assert_approx_eq!((-1.3f32).trunc(), -1.0f32); + assert_approx_eq!((-1.5f32).trunc(), -1.0f32); + assert_approx_eq!((-1.7f32).trunc(), -1.0f32); +} + +#[test] +fn test_fract() { + assert_approx_eq!(1.0f32.fract(), 0.0f32); + assert_approx_eq!(1.3f32.fract(), 0.3f32); + assert_approx_eq!(1.5f32.fract(), 0.5f32); + assert_approx_eq!(1.7f32.fract(), 0.7f32); + assert_approx_eq!(0.0f32.fract(), 0.0f32); + assert_approx_eq!((-0.0f32).fract(), -0.0f32); + assert_approx_eq!((-1.0f32).fract(), -0.0f32); + assert_approx_eq!((-1.3f32).fract(), -0.3f32); + assert_approx_eq!((-1.5f32).fract(), -0.5f32); + assert_approx_eq!((-1.7f32).fract(), -0.7f32); +} + +#[test] +fn test_abs() { + assert_eq!(f32::INFINITY.abs(), f32::INFINITY); + assert_eq!(1f32.abs(), 1f32); + assert_eq!(0f32.abs(), 0f32); + assert_eq!((-0f32).abs(), 0f32); + assert_eq!((-1f32).abs(), 1f32); + assert_eq!(f32::NEG_INFINITY.abs(), f32::INFINITY); + assert_eq!((1f32 / f32::NEG_INFINITY).abs(), 0f32); + assert!(f32::NAN.abs().is_nan()); +} + +#[test] +fn test_signum() { + assert_eq!(f32::INFINITY.signum(), 1f32); + assert_eq!(1f32.signum(), 1f32); + assert_eq!(0f32.signum(), 1f32); + assert_eq!((-0f32).signum(), -1f32); + assert_eq!((-1f32).signum(), -1f32); + assert_eq!(f32::NEG_INFINITY.signum(), -1f32); + assert_eq!((1f32 / f32::NEG_INFINITY).signum(), -1f32); + assert!(f32::NAN.signum().is_nan()); +} + +#[test] +fn test_is_sign_positive() { + assert!(f32::INFINITY.is_sign_positive()); + assert!(1f32.is_sign_positive()); + assert!(0f32.is_sign_positive()); + assert!(!(-0f32).is_sign_positive()); + assert!(!(-1f32).is_sign_positive()); + assert!(!f32::NEG_INFINITY.is_sign_positive()); + assert!(!(1f32 / f32::NEG_INFINITY).is_sign_positive()); + assert!(f32::NAN.is_sign_positive()); + assert!(!(-f32::NAN).is_sign_positive()); +} + +#[test] +fn test_is_sign_negative() { + assert!(!f32::INFINITY.is_sign_negative()); + assert!(!1f32.is_sign_negative()); + assert!(!0f32.is_sign_negative()); + assert!((-0f32).is_sign_negative()); + assert!((-1f32).is_sign_negative()); + assert!(f32::NEG_INFINITY.is_sign_negative()); + assert!((1f32 / f32::NEG_INFINITY).is_sign_negative()); + assert!(!f32::NAN.is_sign_negative()); + assert!((-f32::NAN).is_sign_negative()); +} + +#[allow(unused_macros)] +macro_rules! assert_f32_biteq { + ($left : expr, $right : expr) => { + let l: &f32 = &$left; + let r: &f32 = &$right; + let lb = l.to_bits(); + let rb = r.to_bits(); + assert_eq!(lb, rb, "float {} ({:#x}) is not equal to {} ({:#x})", *l, lb, *r, rb); + }; +} + +// Ignore test on x87 floating point, these platforms do not guarantee NaN +// payloads are preserved and flush denormals to zero, failing the tests. +#[cfg(not(target_arch = "x86"))] +#[test] +fn test_next_up() { + let tiny = f32::from_bits(1); + let tiny_up = f32::from_bits(2); + let max_down = f32::from_bits(0x7f7f_fffe); + let largest_subnormal = f32::from_bits(0x007f_ffff); + let smallest_normal = f32::from_bits(0x0080_0000); + assert_f32_biteq!(f32::NEG_INFINITY.next_up(), f32::MIN); + assert_f32_biteq!(f32::MIN.next_up(), -max_down); + assert_f32_biteq!((-1.0 - f32::EPSILON).next_up(), -1.0); + assert_f32_biteq!((-smallest_normal).next_up(), -largest_subnormal); + assert_f32_biteq!((-tiny_up).next_up(), -tiny); + assert_f32_biteq!((-tiny).next_up(), -0.0f32); + assert_f32_biteq!((-0.0f32).next_up(), tiny); + assert_f32_biteq!(0.0f32.next_up(), tiny); + assert_f32_biteq!(tiny.next_up(), tiny_up); + assert_f32_biteq!(largest_subnormal.next_up(), smallest_normal); + assert_f32_biteq!(1.0f32.next_up(), 1.0 + f32::EPSILON); + assert_f32_biteq!(f32::MAX.next_up(), f32::INFINITY); + assert_f32_biteq!(f32::INFINITY.next_up(), f32::INFINITY); + + // Check that NaNs roundtrip. + let nan0 = f32::NAN; + let nan1 = f32::from_bits(f32::NAN.to_bits() ^ 0x002a_aaaa); + let nan2 = f32::from_bits(f32::NAN.to_bits() ^ 0x0055_5555); + assert_f32_biteq!(nan0.next_up(), nan0); + assert_f32_biteq!(nan1.next_up(), nan1); + assert_f32_biteq!(nan2.next_up(), nan2); +} + +// Ignore test on x87 floating point, these platforms do not guarantee NaN +// payloads are preserved and flush denormals to zero, failing the tests. +#[cfg(not(target_arch = "x86"))] +#[test] +fn test_next_down() { + let tiny = f32::from_bits(1); + let tiny_up = f32::from_bits(2); + let max_down = f32::from_bits(0x7f7f_fffe); + let largest_subnormal = f32::from_bits(0x007f_ffff); + let smallest_normal = f32::from_bits(0x0080_0000); + assert_f32_biteq!(f32::NEG_INFINITY.next_down(), f32::NEG_INFINITY); + assert_f32_biteq!(f32::MIN.next_down(), f32::NEG_INFINITY); + assert_f32_biteq!((-max_down).next_down(), f32::MIN); + assert_f32_biteq!((-1.0f32).next_down(), -1.0 - f32::EPSILON); + assert_f32_biteq!((-largest_subnormal).next_down(), -smallest_normal); + assert_f32_biteq!((-tiny).next_down(), -tiny_up); + assert_f32_biteq!((-0.0f32).next_down(), -tiny); + assert_f32_biteq!((0.0f32).next_down(), -tiny); + assert_f32_biteq!(tiny.next_down(), 0.0f32); + assert_f32_biteq!(tiny_up.next_down(), tiny); + assert_f32_biteq!(smallest_normal.next_down(), largest_subnormal); + assert_f32_biteq!((1.0 + f32::EPSILON).next_down(), 1.0f32); + assert_f32_biteq!(f32::MAX.next_down(), max_down); + assert_f32_biteq!(f32::INFINITY.next_down(), f32::MAX); + + // Check that NaNs roundtrip. + let nan0 = f32::NAN; + let nan1 = f32::from_bits(f32::NAN.to_bits() ^ 0x002a_aaaa); + let nan2 = f32::from_bits(f32::NAN.to_bits() ^ 0x0055_5555); + assert_f32_biteq!(nan0.next_down(), nan0); + assert_f32_biteq!(nan1.next_down(), nan1); + assert_f32_biteq!(nan2.next_down(), nan2); +} + +#[test] +fn test_mul_add() { + let nan: f32 = f32::NAN; + let inf: f32 = f32::INFINITY; + let neg_inf: f32 = f32::NEG_INFINITY; + assert_approx_eq!(12.3f32.mul_add(4.5, 6.7), 62.05); + assert_approx_eq!((-12.3f32).mul_add(-4.5, -6.7), 48.65); + assert_approx_eq!(0.0f32.mul_add(8.9, 1.2), 1.2); + assert_approx_eq!(3.4f32.mul_add(-0.0, 5.6), 5.6); + assert!(nan.mul_add(7.8, 9.0).is_nan()); + assert_eq!(inf.mul_add(7.8, 9.0), inf); + assert_eq!(neg_inf.mul_add(7.8, 9.0), neg_inf); + assert_eq!(8.9f32.mul_add(inf, 3.2), inf); + assert_eq!((-3.2f32).mul_add(2.4, neg_inf), neg_inf); +} + +#[test] +fn test_recip() { + let nan: f32 = f32::NAN; + let inf: f32 = f32::INFINITY; + let neg_inf: f32 = f32::NEG_INFINITY; + assert_eq!(1.0f32.recip(), 1.0); + assert_eq!(2.0f32.recip(), 0.5); + assert_eq!((-0.4f32).recip(), -2.5); + assert_eq!(0.0f32.recip(), inf); + assert!(nan.recip().is_nan()); + assert_eq!(inf.recip(), 0.0); + assert_eq!(neg_inf.recip(), 0.0); +} + +#[test] +fn test_powi() { + let nan: f32 = f32::NAN; + let inf: f32 = f32::INFINITY; + let neg_inf: f32 = f32::NEG_INFINITY; + assert_eq!(1.0f32.powi(1), 1.0); + assert_approx_eq!((-3.1f32).powi(2), 9.61); + assert_approx_eq!(5.9f32.powi(-2), 0.028727); + assert_eq!(8.3f32.powi(0), 1.0); + assert!(nan.powi(2).is_nan()); + assert_eq!(inf.powi(3), inf); + assert_eq!(neg_inf.powi(2), inf); +} + +#[test] +fn test_powf() { + let nan: f32 = f32::NAN; + let inf: f32 = f32::INFINITY; + let neg_inf: f32 = f32::NEG_INFINITY; + assert_eq!(1.0f32.powf(1.0), 1.0); + assert_approx_eq!(3.4f32.powf(4.5), 246.408218); + assert_approx_eq!(2.7f32.powf(-3.2), 0.041652); + assert_approx_eq!((-3.1f32).powf(2.0), 9.61); + assert_approx_eq!(5.9f32.powf(-2.0), 0.028727); + assert_eq!(8.3f32.powf(0.0), 1.0); + assert!(nan.powf(2.0).is_nan()); + assert_eq!(inf.powf(2.0), inf); + assert_eq!(neg_inf.powf(3.0), neg_inf); +} + +#[test] +fn test_sqrt_domain() { + assert!(f32::NAN.sqrt().is_nan()); + assert!(f32::NEG_INFINITY.sqrt().is_nan()); + assert!((-1.0f32).sqrt().is_nan()); + assert_eq!((-0.0f32).sqrt(), -0.0); + assert_eq!(0.0f32.sqrt(), 0.0); + assert_eq!(1.0f32.sqrt(), 1.0); + assert_eq!(f32::INFINITY.sqrt(), f32::INFINITY); +} + +#[test] +fn test_exp() { + assert_eq!(1.0, 0.0f32.exp()); + assert_approx_eq!(2.718282, 1.0f32.exp()); + assert_approx_eq!(148.413162, 5.0f32.exp()); + + let inf: f32 = f32::INFINITY; + let neg_inf: f32 = f32::NEG_INFINITY; + let nan: f32 = f32::NAN; + assert_eq!(inf, inf.exp()); + assert_eq!(0.0, neg_inf.exp()); + assert!(nan.exp().is_nan()); +} + +#[test] +fn test_exp2() { + assert_eq!(32.0, 5.0f32.exp2()); + assert_eq!(1.0, 0.0f32.exp2()); + + let inf: f32 = f32::INFINITY; + let neg_inf: f32 = f32::NEG_INFINITY; + let nan: f32 = f32::NAN; + assert_eq!(inf, inf.exp2()); + assert_eq!(0.0, neg_inf.exp2()); + assert!(nan.exp2().is_nan()); +} + +#[test] +fn test_ln() { + let nan: f32 = f32::NAN; + let inf: f32 = f32::INFINITY; + let neg_inf: f32 = f32::NEG_INFINITY; + assert_approx_eq!(1.0f32.exp().ln(), 1.0); + assert!(nan.ln().is_nan()); + assert_eq!(inf.ln(), inf); + assert!(neg_inf.ln().is_nan()); + assert!((-2.3f32).ln().is_nan()); + assert_eq!((-0.0f32).ln(), neg_inf); + assert_eq!(0.0f32.ln(), neg_inf); + assert_approx_eq!(4.0f32.ln(), 1.386294); +} + +#[test] +fn test_log() { + let nan: f32 = f32::NAN; + let inf: f32 = f32::INFINITY; + let neg_inf: f32 = f32::NEG_INFINITY; + assert_eq!(10.0f32.log(10.0), 1.0); + assert_approx_eq!(2.3f32.log(3.5), 0.664858); + assert_eq!(1.0f32.exp().log(1.0f32.exp()), 1.0); + assert!(1.0f32.log(1.0).is_nan()); + assert!(1.0f32.log(-13.9).is_nan()); + assert!(nan.log(2.3).is_nan()); + assert_eq!(inf.log(10.0), inf); + assert!(neg_inf.log(8.8).is_nan()); + assert!((-2.3f32).log(0.1).is_nan()); + assert_eq!((-0.0f32).log(2.0), neg_inf); + assert_eq!(0.0f32.log(7.0), neg_inf); +} + +#[test] +fn test_log2() { + let nan: f32 = f32::NAN; + let inf: f32 = f32::INFINITY; + let neg_inf: f32 = f32::NEG_INFINITY; + assert_approx_eq!(10.0f32.log2(), 3.321928); + assert_approx_eq!(2.3f32.log2(), 1.201634); + assert_approx_eq!(1.0f32.exp().log2(), 1.442695); + assert!(nan.log2().is_nan()); + assert_eq!(inf.log2(), inf); + assert!(neg_inf.log2().is_nan()); + assert!((-2.3f32).log2().is_nan()); + assert_eq!((-0.0f32).log2(), neg_inf); + assert_eq!(0.0f32.log2(), neg_inf); +} + +#[test] +fn test_log10() { + let nan: f32 = f32::NAN; + let inf: f32 = f32::INFINITY; + let neg_inf: f32 = f32::NEG_INFINITY; + assert_eq!(10.0f32.log10(), 1.0); + assert_approx_eq!(2.3f32.log10(), 0.361728); + assert_approx_eq!(1.0f32.exp().log10(), 0.434294); + assert_eq!(1.0f32.log10(), 0.0); + assert!(nan.log10().is_nan()); + assert_eq!(inf.log10(), inf); + assert!(neg_inf.log10().is_nan()); + assert!((-2.3f32).log10().is_nan()); + assert_eq!((-0.0f32).log10(), neg_inf); + assert_eq!(0.0f32.log10(), neg_inf); +} + +#[test] +fn test_to_degrees() { + let pi: f32 = consts::PI; + let nan: f32 = f32::NAN; + let inf: f32 = f32::INFINITY; + let neg_inf: f32 = f32::NEG_INFINITY; + assert_eq!(0.0f32.to_degrees(), 0.0); + assert_approx_eq!((-5.8f32).to_degrees(), -332.315521); + assert_eq!(pi.to_degrees(), 180.0); + assert!(nan.to_degrees().is_nan()); + assert_eq!(inf.to_degrees(), inf); + assert_eq!(neg_inf.to_degrees(), neg_inf); + assert_eq!(1_f32.to_degrees(), 57.2957795130823208767981548141051703); +} + +#[test] +fn test_to_radians() { + let pi: f32 = consts::PI; + let nan: f32 = f32::NAN; + let inf: f32 = f32::INFINITY; + let neg_inf: f32 = f32::NEG_INFINITY; + assert_eq!(0.0f32.to_radians(), 0.0); + assert_approx_eq!(154.6f32.to_radians(), 2.698279); + assert_approx_eq!((-332.31f32).to_radians(), -5.799903); + assert_eq!(180.0f32.to_radians(), pi); + assert!(nan.to_radians().is_nan()); + assert_eq!(inf.to_radians(), inf); + assert_eq!(neg_inf.to_radians(), neg_inf); +} + +#[test] +fn test_asinh() { + assert_eq!(0.0f32.asinh(), 0.0f32); + assert_eq!((-0.0f32).asinh(), -0.0f32); + + let inf: f32 = f32::INFINITY; + let neg_inf: f32 = f32::NEG_INFINITY; + let nan: f32 = f32::NAN; + assert_eq!(inf.asinh(), inf); + assert_eq!(neg_inf.asinh(), neg_inf); + assert!(nan.asinh().is_nan()); + assert!((-0.0f32).asinh().is_sign_negative()); // issue 63271 + assert_approx_eq!(2.0f32.asinh(), 1.443635475178810342493276740273105f32); + assert_approx_eq!((-2.0f32).asinh(), -1.443635475178810342493276740273105f32); + // regression test for the catastrophic cancellation fixed in 72486 + assert_approx_eq!((-3000.0f32).asinh(), -8.699514775987968673236893537700647f32); + + // test for low accuracy from issue 104548 + assert_approx_eq!(60.0f32, 60.0f32.sinh().asinh()); + // mul needed for approximate comparison to be meaningful + assert_approx_eq!(1.0f32, 1e-15f32.sinh().asinh() * 1e15f32); +} + +#[test] +fn test_acosh() { + assert_eq!(1.0f32.acosh(), 0.0f32); + assert!(0.999f32.acosh().is_nan()); + + let inf: f32 = f32::INFINITY; + let neg_inf: f32 = f32::NEG_INFINITY; + let nan: f32 = f32::NAN; + assert_eq!(inf.acosh(), inf); + assert!(neg_inf.acosh().is_nan()); + assert!(nan.acosh().is_nan()); + assert_approx_eq!(2.0f32.acosh(), 1.31695789692481670862504634730796844f32); + assert_approx_eq!(3.0f32.acosh(), 1.76274717403908605046521864995958461f32); + + // test for low accuracy from issue 104548 + assert_approx_eq!(60.0f32, 60.0f32.cosh().acosh()); +} + +#[test] +fn test_atanh() { + assert_eq!(0.0f32.atanh(), 0.0f32); + assert_eq!((-0.0f32).atanh(), -0.0f32); + + let inf32: f32 = f32::INFINITY; + let neg_inf32: f32 = f32::NEG_INFINITY; + assert_eq!(1.0f32.atanh(), inf32); + assert_eq!((-1.0f32).atanh(), neg_inf32); + + assert!(2f64.atanh().atanh().is_nan()); + assert!((-2f64).atanh().atanh().is_nan()); + + let inf64: f32 = f32::INFINITY; + let neg_inf64: f32 = f32::NEG_INFINITY; + let nan32: f32 = f32::NAN; + assert!(inf64.atanh().is_nan()); + assert!(neg_inf64.atanh().is_nan()); + assert!(nan32.atanh().is_nan()); + + assert_approx_eq!(0.5f32.atanh(), 0.54930614433405484569762261846126285f32); + assert_approx_eq!((-0.5f32).atanh(), -0.54930614433405484569762261846126285f32); +} + +#[test] +fn test_real_consts() { + use super::consts; + + let pi: f32 = consts::PI; + let frac_pi_2: f32 = consts::FRAC_PI_2; + let frac_pi_3: f32 = consts::FRAC_PI_3; + let frac_pi_4: f32 = consts::FRAC_PI_4; + let frac_pi_6: f32 = consts::FRAC_PI_6; + let frac_pi_8: f32 = consts::FRAC_PI_8; + let frac_1_pi: f32 = consts::FRAC_1_PI; + let frac_2_pi: f32 = consts::FRAC_2_PI; + let frac_2_sqrtpi: f32 = consts::FRAC_2_SQRT_PI; + let sqrt2: f32 = consts::SQRT_2; + let frac_1_sqrt2: f32 = consts::FRAC_1_SQRT_2; + let e: f32 = consts::E; + let log2_e: f32 = consts::LOG2_E; + let log10_e: f32 = consts::LOG10_E; + let ln_2: f32 = consts::LN_2; + let ln_10: f32 = consts::LN_10; + + assert_approx_eq!(frac_pi_2, pi / 2f32); + assert_approx_eq!(frac_pi_3, pi / 3f32); + assert_approx_eq!(frac_pi_4, pi / 4f32); + assert_approx_eq!(frac_pi_6, pi / 6f32); + assert_approx_eq!(frac_pi_8, pi / 8f32); + assert_approx_eq!(frac_1_pi, 1f32 / pi); + assert_approx_eq!(frac_2_pi, 2f32 / pi); + assert_approx_eq!(frac_2_sqrtpi, 2f32 / pi.sqrt()); + assert_approx_eq!(sqrt2, 2f32.sqrt()); + assert_approx_eq!(frac_1_sqrt2, 1f32 / 2f32.sqrt()); + assert_approx_eq!(log2_e, e.log2()); + assert_approx_eq!(log10_e, e.log10()); + assert_approx_eq!(ln_2, 2f32.ln()); + assert_approx_eq!(ln_10, 10f32.ln()); +} + +#[test] +fn test_float_bits_conv() { + assert_eq!((1f32).to_bits(), 0x3f800000); + assert_eq!((12.5f32).to_bits(), 0x41480000); + assert_eq!((1337f32).to_bits(), 0x44a72000); + assert_eq!((-14.25f32).to_bits(), 0xc1640000); + assert_approx_eq!(f32::from_bits(0x3f800000), 1.0); + assert_approx_eq!(f32::from_bits(0x41480000), 12.5); + assert_approx_eq!(f32::from_bits(0x44a72000), 1337.0); + assert_approx_eq!(f32::from_bits(0xc1640000), -14.25); + + // Check that NaNs roundtrip their bits regardless of signaling-ness + // 0xA is 0b1010; 0x5 is 0b0101 -- so these two together clobbers all the mantissa bits + let masked_nan1 = f32::NAN.to_bits() ^ 0x002A_AAAA; + let masked_nan2 = f32::NAN.to_bits() ^ 0x0055_5555; + assert!(f32::from_bits(masked_nan1).is_nan()); + assert!(f32::from_bits(masked_nan2).is_nan()); + + assert_eq!(f32::from_bits(masked_nan1).to_bits(), masked_nan1); + assert_eq!(f32::from_bits(masked_nan2).to_bits(), masked_nan2); +} + +#[test] +#[should_panic] +fn test_clamp_min_greater_than_max() { + let _ = 1.0f32.clamp(3.0, 1.0); +} + +#[test] +#[should_panic] +fn test_clamp_min_is_nan() { + let _ = 1.0f32.clamp(f32::NAN, 1.0); +} + +#[test] +#[should_panic] +fn test_clamp_max_is_nan() { + let _ = 1.0f32.clamp(3.0, f32::NAN); +} + +#[test] +fn test_total_cmp() { + use core::cmp::Ordering; + + fn quiet_bit_mask() -> u32 { + 1 << (f32::MANTISSA_DIGITS - 2) + } + + fn min_subnorm() -> f32 { + f32::MIN_POSITIVE / f32::powf(2.0, f32::MANTISSA_DIGITS as f32 - 1.0) + } + + fn max_subnorm() -> f32 { + f32::MIN_POSITIVE - min_subnorm() + } + + fn q_nan() -> f32 { + f32::from_bits(f32::NAN.to_bits() | quiet_bit_mask()) + } + + fn s_nan() -> f32 { + f32::from_bits((f32::NAN.to_bits() & !quiet_bit_mask()) + 42) + } + + assert_eq!(Ordering::Equal, (-q_nan()).total_cmp(&-q_nan())); + assert_eq!(Ordering::Equal, (-s_nan()).total_cmp(&-s_nan())); + assert_eq!(Ordering::Equal, (-f32::INFINITY).total_cmp(&-f32::INFINITY)); + assert_eq!(Ordering::Equal, (-f32::MAX).total_cmp(&-f32::MAX)); + assert_eq!(Ordering::Equal, (-2.5_f32).total_cmp(&-2.5)); + assert_eq!(Ordering::Equal, (-1.0_f32).total_cmp(&-1.0)); + assert_eq!(Ordering::Equal, (-1.5_f32).total_cmp(&-1.5)); + assert_eq!(Ordering::Equal, (-0.5_f32).total_cmp(&-0.5)); + assert_eq!(Ordering::Equal, (-f32::MIN_POSITIVE).total_cmp(&-f32::MIN_POSITIVE)); + assert_eq!(Ordering::Equal, (-max_subnorm()).total_cmp(&-max_subnorm())); + assert_eq!(Ordering::Equal, (-min_subnorm()).total_cmp(&-min_subnorm())); + assert_eq!(Ordering::Equal, (-0.0_f32).total_cmp(&-0.0)); + assert_eq!(Ordering::Equal, 0.0_f32.total_cmp(&0.0)); + assert_eq!(Ordering::Equal, min_subnorm().total_cmp(&min_subnorm())); + assert_eq!(Ordering::Equal, max_subnorm().total_cmp(&max_subnorm())); + assert_eq!(Ordering::Equal, f32::MIN_POSITIVE.total_cmp(&f32::MIN_POSITIVE)); + assert_eq!(Ordering::Equal, 0.5_f32.total_cmp(&0.5)); + assert_eq!(Ordering::Equal, 1.0_f32.total_cmp(&1.0)); + assert_eq!(Ordering::Equal, 1.5_f32.total_cmp(&1.5)); + assert_eq!(Ordering::Equal, 2.5_f32.total_cmp(&2.5)); + assert_eq!(Ordering::Equal, f32::MAX.total_cmp(&f32::MAX)); + assert_eq!(Ordering::Equal, f32::INFINITY.total_cmp(&f32::INFINITY)); + assert_eq!(Ordering::Equal, s_nan().total_cmp(&s_nan())); + assert_eq!(Ordering::Equal, q_nan().total_cmp(&q_nan())); + + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-s_nan())); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-f32::INFINITY)); + assert_eq!(Ordering::Less, (-f32::INFINITY).total_cmp(&-f32::MAX)); + assert_eq!(Ordering::Less, (-f32::MAX).total_cmp(&-2.5)); + assert_eq!(Ordering::Less, (-2.5_f32).total_cmp(&-1.5)); + assert_eq!(Ordering::Less, (-1.5_f32).total_cmp(&-1.0)); + assert_eq!(Ordering::Less, (-1.0_f32).total_cmp(&-0.5)); + assert_eq!(Ordering::Less, (-0.5_f32).total_cmp(&-f32::MIN_POSITIVE)); + assert_eq!(Ordering::Less, (-f32::MIN_POSITIVE).total_cmp(&-max_subnorm())); + assert_eq!(Ordering::Less, (-max_subnorm()).total_cmp(&-min_subnorm())); + assert_eq!(Ordering::Less, (-min_subnorm()).total_cmp(&-0.0)); + assert_eq!(Ordering::Less, (-0.0_f32).total_cmp(&0.0)); + assert_eq!(Ordering::Less, 0.0_f32.total_cmp(&min_subnorm())); + assert_eq!(Ordering::Less, min_subnorm().total_cmp(&max_subnorm())); + assert_eq!(Ordering::Less, max_subnorm().total_cmp(&f32::MIN_POSITIVE)); + assert_eq!(Ordering::Less, f32::MIN_POSITIVE.total_cmp(&0.5)); + assert_eq!(Ordering::Less, 0.5_f32.total_cmp(&1.0)); + assert_eq!(Ordering::Less, 1.0_f32.total_cmp(&1.5)); + assert_eq!(Ordering::Less, 1.5_f32.total_cmp(&2.5)); + assert_eq!(Ordering::Less, 2.5_f32.total_cmp(&f32::MAX)); + assert_eq!(Ordering::Less, f32::MAX.total_cmp(&f32::INFINITY)); + assert_eq!(Ordering::Less, f32::INFINITY.total_cmp(&s_nan())); + assert_eq!(Ordering::Less, s_nan().total_cmp(&q_nan())); + + assert_eq!(Ordering::Greater, (-s_nan()).total_cmp(&-q_nan())); + assert_eq!(Ordering::Greater, (-f32::INFINITY).total_cmp(&-s_nan())); + assert_eq!(Ordering::Greater, (-f32::MAX).total_cmp(&-f32::INFINITY)); + assert_eq!(Ordering::Greater, (-2.5_f32).total_cmp(&-f32::MAX)); + assert_eq!(Ordering::Greater, (-1.5_f32).total_cmp(&-2.5)); + assert_eq!(Ordering::Greater, (-1.0_f32).total_cmp(&-1.5)); + assert_eq!(Ordering::Greater, (-0.5_f32).total_cmp(&-1.0)); + assert_eq!(Ordering::Greater, (-f32::MIN_POSITIVE).total_cmp(&-0.5)); + assert_eq!(Ordering::Greater, (-max_subnorm()).total_cmp(&-f32::MIN_POSITIVE)); + assert_eq!(Ordering::Greater, (-min_subnorm()).total_cmp(&-max_subnorm())); + assert_eq!(Ordering::Greater, (-0.0_f32).total_cmp(&-min_subnorm())); + assert_eq!(Ordering::Greater, 0.0_f32.total_cmp(&-0.0)); + assert_eq!(Ordering::Greater, min_subnorm().total_cmp(&0.0)); + assert_eq!(Ordering::Greater, max_subnorm().total_cmp(&min_subnorm())); + assert_eq!(Ordering::Greater, f32::MIN_POSITIVE.total_cmp(&max_subnorm())); + assert_eq!(Ordering::Greater, 0.5_f32.total_cmp(&f32::MIN_POSITIVE)); + assert_eq!(Ordering::Greater, 1.0_f32.total_cmp(&0.5)); + assert_eq!(Ordering::Greater, 1.5_f32.total_cmp(&1.0)); + assert_eq!(Ordering::Greater, 2.5_f32.total_cmp(&1.5)); + assert_eq!(Ordering::Greater, f32::MAX.total_cmp(&2.5)); + assert_eq!(Ordering::Greater, f32::INFINITY.total_cmp(&f32::MAX)); + assert_eq!(Ordering::Greater, s_nan().total_cmp(&f32::INFINITY)); + assert_eq!(Ordering::Greater, q_nan().total_cmp(&s_nan())); + + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-s_nan())); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-f32::INFINITY)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-f32::MAX)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-2.5)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-1.5)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-1.0)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-0.5)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-f32::MIN_POSITIVE)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-max_subnorm())); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-min_subnorm())); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-0.0)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&0.0)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&min_subnorm())); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&max_subnorm())); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&f32::MIN_POSITIVE)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&0.5)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&1.0)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&1.5)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&2.5)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&f32::MAX)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&f32::INFINITY)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&s_nan())); + + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-f32::INFINITY)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-f32::MAX)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-2.5)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-1.5)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-1.0)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-0.5)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-f32::MIN_POSITIVE)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-max_subnorm())); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-min_subnorm())); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-0.0)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&0.0)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&min_subnorm())); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&max_subnorm())); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&f32::MIN_POSITIVE)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&0.5)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&1.0)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&1.5)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&2.5)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&f32::MAX)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&f32::INFINITY)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&s_nan())); +} diff --git a/crux-mir/lib/std/src/f64.rs b/crux-mir/lib/std/src/f64.rs index a1128a589..ec67fdad4 100644 --- a/crux-mir/lib/std/src/f64.rs +++ b/crux-mir/lib/std/src/f64.rs @@ -1,36 +1,35 @@ -//! This module provides constants which are specific to the implementation -//! of the `f64` floating point data type. +//! Constants for the `f64` double-precision floating point type. //! -//! *[See also the `f64` primitive type](../../std/primitive.f64.html).* +//! *[See also the `f64` primitive type](primitive@f64).* //! //! Mathematically significant numbers are provided in the `consts` sub-module. //! -//! Although using these constants won’t cause compilation warnings, -//! new code should use the associated constants directly on the primitive type. +//! For the constants defined directly in this module +//! (as distinct from those defined in the `consts` sub-module), +//! new code should instead use the associated constants +//! defined directly on the `f64` type. #![stable(feature = "rust1", since = "1.0.0")] #![allow(missing_docs)] +#[cfg(test)] +mod tests; + #[cfg(not(test))] use crate::intrinsics; #[cfg(not(test))] use crate::sys::cmath; #[stable(feature = "rust1", since = "1.0.0")] -pub use core::f64::consts; -#[stable(feature = "rust1", since = "1.0.0")] -pub use core::f64::{DIGITS, EPSILON, MANTISSA_DIGITS, RADIX}; -#[stable(feature = "rust1", since = "1.0.0")] -pub use core::f64::{INFINITY, MAX_10_EXP, NAN, NEG_INFINITY}; -#[stable(feature = "rust1", since = "1.0.0")] -pub use core::f64::{MAX, MIN, MIN_POSITIVE}; -#[stable(feature = "rust1", since = "1.0.0")] -pub use core::f64::{MAX_EXP, MIN_10_EXP, MIN_EXP}; +#[allow(deprecated, deprecated_in_future)] +pub use core::f64::{ + consts, DIGITS, EPSILON, INFINITY, MANTISSA_DIGITS, MAX, MAX_10_EXP, MAX_EXP, MIN, MIN_10_EXP, + MIN_EXP, MIN_POSITIVE, NAN, NEG_INFINITY, RADIX, +}; #[cfg(not(test))] -#[lang = "f64_runtime"] impl f64 { - /// Returns the largest integer less than or equal to a number. + /// Returns the largest integer less than or equal to `self`. /// /// # Examples /// @@ -43,6 +42,7 @@ impl f64 { /// assert_eq!(g.floor(), 3.0); /// assert_eq!(h.floor(), -4.0); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -50,7 +50,7 @@ impl f64 { unsafe { intrinsics::floorf64(self) } } - /// Returns the smallest integer greater than or equal to a number. + /// Returns the smallest integer greater than or equal to `self`. /// /// # Examples /// @@ -61,6 +61,7 @@ impl f64 { /// assert_eq!(f.ceil(), 4.0); /// assert_eq!(g.ceil(), 4.0); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -68,7 +69,7 @@ impl f64 { unsafe { intrinsics::ceilf64(self) } } - /// Returns the nearest integer to a number. Round half-way cases away from + /// Returns the nearest integer to `self`. Round half-way cases away from /// `0.0`. /// /// # Examples @@ -76,10 +77,13 @@ impl f64 { /// ``` /// let f = 3.3_f64; /// let g = -3.3_f64; + /// let h = -3.7_f64; /// /// assert_eq!(f.round(), 3.0); /// assert_eq!(g.round(), -3.0); + /// assert_eq!(h.round(), -4.0); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -87,7 +91,8 @@ impl f64 { unsafe { intrinsics::roundf64(self) } } - /// Returns the integer part of a number. + /// Returns the integer part of `self`. + /// This means that non-integer numbers are always truncated towards zero. /// /// # Examples /// @@ -100,6 +105,7 @@ impl f64 { /// assert_eq!(g.trunc(), 3.0); /// assert_eq!(h.trunc(), -3.0); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -107,7 +113,7 @@ impl f64 { unsafe { intrinsics::truncf64(self) } } - /// Returns the fractional part of a number. + /// Returns the fractional part of `self`. /// /// # Examples /// @@ -120,6 +126,7 @@ impl f64 { /// assert!(abs_difference_x < 1e-10); /// assert!(abs_difference_y < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -127,14 +134,11 @@ impl f64 { self - self.trunc() } - /// Computes the absolute value of `self`. Returns `NAN` if the - /// number is `NAN`. + /// Computes the absolute value of `self`. /// /// # Examples /// /// ``` - /// use std::f64; - /// /// let x = 3.5_f64; /// let y = -3.5_f64; /// @@ -146,6 +150,7 @@ impl f64 { /// /// assert!(f64::NAN.abs().is_nan()); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -157,13 +162,11 @@ impl f64 { /// /// - `1.0` if the number is positive, `+0.0` or `INFINITY` /// - `-1.0` if the number is negative, `-0.0` or `NEG_INFINITY` - /// - `NAN` if the number is `NAN` + /// - NaN if the number is NaN /// /// # Examples /// /// ``` - /// use std::f64; - /// /// let f = 3.5_f64; /// /// assert_eq!(f.signum(), 1.0); @@ -171,25 +174,26 @@ impl f64 { /// /// assert!(f64::NAN.signum().is_nan()); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn signum(self) -> f64 { - if self.is_nan() { NAN } else { 1.0_f64.copysign(self) } + if self.is_nan() { Self::NAN } else { 1.0_f64.copysign(self) } } /// Returns a number composed of the magnitude of `self` and the sign of /// `sign`. /// /// Equal to `self` if the sign of `self` and `sign` are the same, otherwise - /// equal to `-self`. If `self` is a `NAN`, then a `NAN` with the sign of - /// `sign` is returned. + /// equal to `-self`. If `self` is a NaN, then a NaN with the sign bit of + /// `sign` is returned. Note, however, that conserving the sign bit on NaN + /// across arithmetical operations is not generally guaranteed. + /// See [explanation of NaN as a special value](primitive@f32) for more info. /// /// # Examples /// /// ``` - /// use std::f64; - /// /// let f = 3.5_f64; /// /// assert_eq!(f.copysign(0.42), 3.5_f64); @@ -199,6 +203,7 @@ impl f64 { /// /// assert!(f64::NAN.copysign(1.0).is_nan()); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "copysign", since = "1.35.0")] #[inline] @@ -209,8 +214,10 @@ impl f64 { /// Fused multiply-add. Computes `(self * a) + b` with only one rounding /// error, yielding a more accurate result than an unfused multiply-add. /// - /// Using `mul_add` can be more performant than an unfused multiply-add if - /// the target architecture has a dedicated `fma` CPU instruction. + /// Using `mul_add` *may* be more performant than an unfused multiply-add if + /// the target architecture has a dedicated `fma` CPU instruction. However, + /// this is not always true, and will be heavily dependant on designing + /// algorithms with specific target hardware in mind. /// /// # Examples /// @@ -224,6 +231,7 @@ impl f64 { /// /// assert!(abs_difference < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -248,6 +256,7 @@ impl f64 { /// assert_eq!(a.div_euclid(-b), -1.0); // 7.0 >= -4.0 * -1.0 /// assert_eq!((-a).div_euclid(-b), 2.0); // -7.0 >= -4.0 * 2.0 /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[inline] #[stable(feature = "euclidean_division", since = "1.38.0")] @@ -268,7 +277,7 @@ impl f64 { /// This result is not an element of the function's codomain, but it is the /// closest floating point number in the real numbers and thus fulfills the /// property `self == self.div_euclid(rhs) * rhs + self.rem_euclid(rhs)` - /// approximatively. + /// approximately. /// /// # Examples /// @@ -280,8 +289,9 @@ impl f64 { /// assert_eq!(a.rem_euclid(-b), 3.0); /// assert_eq!((-a).rem_euclid(-b), 1.0); /// // limitation due to round-off error - /// assert!((-std::f64::EPSILON).rem_euclid(3.0) != 0.0); + /// assert!((-f64::EPSILON).rem_euclid(3.0) != 0.0); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[inline] #[stable(feature = "euclidean_division", since = "1.38.0")] @@ -292,7 +302,9 @@ impl f64 { /// Raises a number to an integer power. /// - /// Using this function is generally faster than using `powf` + /// Using this function is generally faster than using `powf`. + /// It might have a different sequence of rounding operations than `powf`, + /// so the results are not guaranteed to agree. /// /// # Examples /// @@ -302,6 +314,7 @@ impl f64 { /// /// assert!(abs_difference < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -319,6 +332,7 @@ impl f64 { /// /// assert!(abs_difference < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -328,19 +342,22 @@ impl f64 { /// Returns the square root of a number. /// - /// Returns NaN if `self` is a negative number. + /// Returns NaN if `self` is a negative number other than `-0.0`. /// /// # Examples /// /// ``` /// let positive = 4.0_f64; /// let negative = -4.0_f64; + /// let negative_zero = -0.0_f64; /// /// let abs_difference = (positive.sqrt() - 2.0).abs(); /// /// assert!(abs_difference < 1e-10); /// assert!(negative.sqrt().is_nan()); + /// assert!(negative_zero.sqrt() == negative_zero); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -362,6 +379,7 @@ impl f64 { /// /// assert!(abs_difference < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -381,6 +399,7 @@ impl f64 { /// /// assert!(abs_difference < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -402,6 +421,7 @@ impl f64 { /// /// assert!(abs_difference < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -411,7 +431,7 @@ impl f64 { /// Returns the logarithm of the number with respect to an arbitrary base. /// - /// The result may not be correctly rounded owing to implementation details; + /// The result might not be correctly rounded owing to implementation details; /// `self.log2()` can produce more accurate results for base 2, and /// `self.log10()` can produce more accurate results for base 10. /// @@ -425,6 +445,7 @@ impl f64 { /// /// assert!(abs_difference < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -444,6 +465,7 @@ impl f64 { /// /// assert!(abs_difference < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -468,6 +490,7 @@ impl f64 { /// /// assert!(abs_difference < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -492,24 +515,25 @@ impl f64 { /// assert!(abs_difference_x < 1e-10); /// assert!(abs_difference_y < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] - #[rustc_deprecated( + #[deprecated( since = "1.10.0", - reason = "you probably meant `(self - other).abs()`: \ - this operation is `(self - other).max(0.0)` \ - except that `abs_sub` also propagates NaNs (also \ - known as `fdim` in C). If you truly need the positive \ - difference, consider using that expression or the C function \ - `fdim`, depending on how you wish to handle NaN (please consider \ - filing an issue describing your use-case too)." + note = "you probably meant `(self - other).abs()`: \ + this operation is `(self - other).max(0.0)` \ + except that `abs_sub` also propagates NaNs (also \ + known as `fdim` in C). If you truly need the positive \ + difference, consider using that expression or the C function \ + `fdim`, depending on how you wish to handle NaN (please consider \ + filing an issue describing your use-case too)." )] pub fn abs_sub(self, other: f64) -> f64 { unsafe { cmath::fdim(self, other) } } - /// Returns the cubic root of a number. + /// Returns the cube root of a number. /// /// # Examples /// @@ -521,6 +545,7 @@ impl f64 { /// /// assert!(abs_difference < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -542,6 +567,7 @@ impl f64 { /// /// assert!(abs_difference < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -554,14 +580,13 @@ impl f64 { /// # Examples /// /// ``` - /// use std::f64; - /// - /// let x = f64::consts::FRAC_PI_2; + /// let x = std::f64::consts::FRAC_PI_2; /// /// let abs_difference = (x.sin() - 1.0).abs(); /// /// assert!(abs_difference < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -574,14 +599,13 @@ impl f64 { /// # Examples /// /// ``` - /// use std::f64; - /// - /// let x = 2.0 * f64::consts::PI; + /// let x = 2.0 * std::f64::consts::PI; /// /// let abs_difference = (x.cos() - 1.0).abs(); /// /// assert!(abs_difference < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -594,13 +618,12 @@ impl f64 { /// # Examples /// /// ``` - /// use std::f64; - /// - /// let x = f64::consts::FRAC_PI_4; + /// let x = std::f64::consts::FRAC_PI_4; /// let abs_difference = (x.tan() - 1.0).abs(); /// /// assert!(abs_difference < 1e-14); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -615,15 +638,14 @@ impl f64 { /// # Examples /// /// ``` - /// use std::f64; - /// - /// let f = f64::consts::FRAC_PI_2; + /// let f = std::f64::consts::FRAC_PI_2; /// /// // asin(sin(pi/2)) - /// let abs_difference = (f.sin().asin() - f64::consts::FRAC_PI_2).abs(); + /// let abs_difference = (f.sin().asin() - std::f64::consts::FRAC_PI_2).abs(); /// /// assert!(abs_difference < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -638,15 +660,14 @@ impl f64 { /// # Examples /// /// ``` - /// use std::f64; - /// - /// let f = f64::consts::FRAC_PI_4; + /// let f = std::f64::consts::FRAC_PI_4; /// /// // acos(cos(pi/4)) - /// let abs_difference = (f.cos().acos() - f64::consts::FRAC_PI_4).abs(); + /// let abs_difference = (f.cos().acos() - std::f64::consts::FRAC_PI_4).abs(); /// /// assert!(abs_difference < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -667,6 +688,7 @@ impl f64 { /// /// assert!(abs_difference < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -684,8 +706,6 @@ impl f64 { /// # Examples /// /// ``` - /// use std::f64; - /// /// // Positive angles measured counter-clockwise /// // from positive x axis /// // -pi/4 radians (45 deg clockwise) @@ -696,12 +716,13 @@ impl f64 { /// let x2 = -3.0_f64; /// let y2 = 3.0_f64; /// - /// let abs_difference_1 = (y1.atan2(x1) - (-f64::consts::FRAC_PI_4)).abs(); - /// let abs_difference_2 = (y2.atan2(x2) - (3.0 * f64::consts::FRAC_PI_4)).abs(); + /// let abs_difference_1 = (y1.atan2(x1) - (-std::f64::consts::FRAC_PI_4)).abs(); + /// let abs_difference_2 = (y2.atan2(x2) - (3.0 * std::f64::consts::FRAC_PI_4)).abs(); /// /// assert!(abs_difference_1 < 1e-10); /// assert!(abs_difference_2 < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -715,9 +736,7 @@ impl f64 { /// # Examples /// /// ``` - /// use std::f64; - /// - /// let x = f64::consts::FRAC_PI_4; + /// let x = std::f64::consts::FRAC_PI_4; /// let f = x.sin_cos(); /// /// let abs_difference_0 = (f.0 - x.sin()).abs(); @@ -726,6 +745,7 @@ impl f64 { /// assert!(abs_difference_0 < 1e-10); /// assert!(abs_difference_1 < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn sin_cos(self) -> (f64, f64) { @@ -738,13 +758,15 @@ impl f64 { /// # Examples /// /// ``` - /// let x = 7.0_f64; + /// let x = 1e-16_f64; /// - /// // e^(ln(7)) - 1 - /// let abs_difference = (x.ln().exp_m1() - 6.0).abs(); + /// // for very small x, e^x is approximately 1 + x + x^2 / 2 + /// let approx = x + x * x / 2.0; + /// let abs_difference = (x.exp_m1() - approx).abs(); /// - /// assert!(abs_difference < 1e-10); + /// assert!(abs_difference < 1e-20); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -758,15 +780,15 @@ impl f64 { /// # Examples /// /// ``` - /// use std::f64; + /// let x = 1e-16_f64; /// - /// let x = f64::consts::E - 1.0; + /// // for very small x, ln(1 + x) is approximately x - x^2 / 2 + /// let approx = x - x * x / 2.0; + /// let abs_difference = (x.ln_1p() - approx).abs(); /// - /// // ln(1 + (e - 1)) == ln(e) == 1 - /// let abs_difference = (x.ln_1p() - 1.0).abs(); - /// - /// assert!(abs_difference < 1e-10); + /// assert!(abs_difference < 1e-20); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -779,9 +801,7 @@ impl f64 { /// # Examples /// /// ``` - /// use std::f64; - /// - /// let e = f64::consts::E; + /// let e = std::f64::consts::E; /// let x = 1.0_f64; /// /// let f = x.sinh(); @@ -791,6 +811,7 @@ impl f64 { /// /// assert!(abs_difference < 1e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -803,9 +824,7 @@ impl f64 { /// # Examples /// /// ``` - /// use std::f64; - /// - /// let e = f64::consts::E; + /// let e = std::f64::consts::E; /// let x = 1.0_f64; /// let f = x.cosh(); /// // Solving cosh() at 1 gives this result @@ -815,6 +834,7 @@ impl f64 { /// // Same result /// assert!(abs_difference < 1.0e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -827,9 +847,7 @@ impl f64 { /// # Examples /// /// ``` - /// use std::f64; - /// - /// let e = f64::consts::E; + /// let e = std::f64::consts::E; /// let x = 1.0_f64; /// /// let f = x.tanh(); @@ -839,6 +857,7 @@ impl f64 { /// /// assert!(abs_difference < 1.0e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -858,15 +877,14 @@ impl f64 { /// /// assert!(abs_difference < 1.0e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn asinh(self) -> f64 { - if self == NEG_INFINITY { - NEG_INFINITY - } else { - (self + ((self * self) + 1.0).sqrt()).ln().copysign(self) - } + let ax = self.abs(); + let ix = 1.0 / ax; + (ax + (ax / (Self::hypot(1.0, ix) + ix))).ln_1p().copysign(self) } /// Inverse hyperbolic cosine function. @@ -881,11 +899,16 @@ impl f64 { /// /// assert!(abs_difference < 1.0e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn acosh(self) -> f64 { - if self < 1.0 { NAN } else { (self + ((self * self) - 1.0).sqrt()).ln() } + if self < 1.0 { + Self::NAN + } else { + (self + ((self - 1.0).sqrt() * (self + 1.0).sqrt())).ln() + } } /// Inverse hyperbolic tangent function. @@ -893,15 +916,14 @@ impl f64 { /// # Examples /// /// ``` - /// use std::f64; - /// - /// let e = f64::consts::E; + /// let e = std::f64::consts::E; /// let f = e.tanh().atanh(); /// /// let abs_difference = (f - e).abs(); /// /// assert!(abs_difference < 1.0e-10); /// ``` + #[rustc_allow_incoherent_impl] #[must_use = "method returns a new number and does not mutate the original value"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -909,679 +931,27 @@ impl f64 { 0.5 * ((2.0 * self) / (1.0 - self)).ln_1p() } - /// Restrict a value to a certain interval unless it is NaN. - /// - /// Returns `max` if `self` is greater than `max`, and `min` if `self` is - /// less than `min`. Otherwise this returns `self`. - /// - /// Not that this function returns NaN if the initial value was NaN as - /// well. - /// - /// # Panics - /// - /// Panics if `min > max`, `min` is NaN, or `max` is NaN. - /// - /// # Examples - /// - /// ``` - /// #![feature(clamp)] - /// assert!((-3.0f64).clamp(-2.0, 1.0) == -2.0); - /// assert!((0.0f64).clamp(-2.0, 1.0) == 0.0); - /// assert!((2.0f64).clamp(-2.0, 1.0) == 1.0); - /// assert!((std::f64::NAN).clamp(-2.0, 1.0).is_nan()); - /// ``` - #[must_use = "method returns a new number and does not mutate the original value"] - #[unstable(feature = "clamp", issue = "44095")] - #[inline] - pub fn clamp(self, min: f64, max: f64) -> f64 { - assert!(min <= max); - let mut x = self; - if x < min { - x = min; - } - if x > max { - x = max; - } - x - } - // Solaris/Illumos requires a wrapper around log, log2, and log10 functions // because of their non-standard behavior (e.g., log(-n) returns -Inf instead // of expected NaN). + #[rustc_allow_incoherent_impl] fn log_wrapper f64>(self, log_fn: F) -> f64 { - if !cfg!(target_os = "solaris") { + if !cfg!(any(target_os = "solaris", target_os = "illumos")) { log_fn(self) - } else { - if self.is_finite() { - if self > 0.0 { - log_fn(self) - } else if self == 0.0 { - NEG_INFINITY // log(0) = -Inf - } else { - NAN // log(-n) = NaN - } - } else if self.is_nan() { - self // log(NaN) = NaN - } else if self > 0.0 { - self // log(Inf) = Inf + } else if self.is_finite() { + if self > 0.0 { + log_fn(self) + } else if self == 0.0 { + Self::NEG_INFINITY // log(0) = -Inf } else { - NAN // log(-Inf) = NaN + Self::NAN // log(-n) = NaN } + } else if self.is_nan() { + self // log(NaN) = NaN + } else if self > 0.0 { + self // log(Inf) = Inf + } else { + Self::NAN // log(-Inf) = NaN } } } - -#[cfg(test)] -mod tests { - use crate::f64; - use crate::f64::*; - use crate::num::FpCategory as Fp; - use crate::num::*; - - #[test] - fn test_num_f64() { - test_num(10f64, 2f64); - } - - #[test] - fn test_min_nan() { - assert_eq!(NAN.min(2.0), 2.0); - assert_eq!(2.0f64.min(NAN), 2.0); - } - - #[test] - fn test_max_nan() { - assert_eq!(NAN.max(2.0), 2.0); - assert_eq!(2.0f64.max(NAN), 2.0); - } - - #[test] - fn test_nan() { - let nan: f64 = NAN; - assert!(nan.is_nan()); - assert!(!nan.is_infinite()); - assert!(!nan.is_finite()); - assert!(!nan.is_normal()); - assert!(nan.is_sign_positive()); - assert!(!nan.is_sign_negative()); - assert_eq!(Fp::Nan, nan.classify()); - } - - #[test] - fn test_infinity() { - let inf: f64 = INFINITY; - assert!(inf.is_infinite()); - assert!(!inf.is_finite()); - assert!(inf.is_sign_positive()); - assert!(!inf.is_sign_negative()); - assert!(!inf.is_nan()); - assert!(!inf.is_normal()); - assert_eq!(Fp::Infinite, inf.classify()); - } - - #[test] - fn test_neg_infinity() { - let neg_inf: f64 = NEG_INFINITY; - assert!(neg_inf.is_infinite()); - assert!(!neg_inf.is_finite()); - assert!(!neg_inf.is_sign_positive()); - assert!(neg_inf.is_sign_negative()); - assert!(!neg_inf.is_nan()); - assert!(!neg_inf.is_normal()); - assert_eq!(Fp::Infinite, neg_inf.classify()); - } - - #[test] - fn test_zero() { - let zero: f64 = 0.0f64; - assert_eq!(0.0, zero); - assert!(!zero.is_infinite()); - assert!(zero.is_finite()); - assert!(zero.is_sign_positive()); - assert!(!zero.is_sign_negative()); - assert!(!zero.is_nan()); - assert!(!zero.is_normal()); - assert_eq!(Fp::Zero, zero.classify()); - } - - #[test] - fn test_neg_zero() { - let neg_zero: f64 = -0.0; - assert_eq!(0.0, neg_zero); - assert!(!neg_zero.is_infinite()); - assert!(neg_zero.is_finite()); - assert!(!neg_zero.is_sign_positive()); - assert!(neg_zero.is_sign_negative()); - assert!(!neg_zero.is_nan()); - assert!(!neg_zero.is_normal()); - assert_eq!(Fp::Zero, neg_zero.classify()); - } - - #[cfg_attr(all(target_arch = "wasm32", target_os = "emscripten"), ignore)] // issue 42630 - #[test] - fn test_one() { - let one: f64 = 1.0f64; - assert_eq!(1.0, one); - assert!(!one.is_infinite()); - assert!(one.is_finite()); - assert!(one.is_sign_positive()); - assert!(!one.is_sign_negative()); - assert!(!one.is_nan()); - assert!(one.is_normal()); - assert_eq!(Fp::Normal, one.classify()); - } - - #[test] - fn test_is_nan() { - let nan: f64 = NAN; - let inf: f64 = INFINITY; - let neg_inf: f64 = NEG_INFINITY; - assert!(nan.is_nan()); - assert!(!0.0f64.is_nan()); - assert!(!5.3f64.is_nan()); - assert!(!(-10.732f64).is_nan()); - assert!(!inf.is_nan()); - assert!(!neg_inf.is_nan()); - } - - #[test] - fn test_is_infinite() { - let nan: f64 = NAN; - let inf: f64 = INFINITY; - let neg_inf: f64 = NEG_INFINITY; - assert!(!nan.is_infinite()); - assert!(inf.is_infinite()); - assert!(neg_inf.is_infinite()); - assert!(!0.0f64.is_infinite()); - assert!(!42.8f64.is_infinite()); - assert!(!(-109.2f64).is_infinite()); - } - - #[test] - fn test_is_finite() { - let nan: f64 = NAN; - let inf: f64 = INFINITY; - let neg_inf: f64 = NEG_INFINITY; - assert!(!nan.is_finite()); - assert!(!inf.is_finite()); - assert!(!neg_inf.is_finite()); - assert!(0.0f64.is_finite()); - assert!(42.8f64.is_finite()); - assert!((-109.2f64).is_finite()); - } - - #[cfg_attr(all(target_arch = "wasm32", target_os = "emscripten"), ignore)] // issue 42630 - #[test] - fn test_is_normal() { - let nan: f64 = NAN; - let inf: f64 = INFINITY; - let neg_inf: f64 = NEG_INFINITY; - let zero: f64 = 0.0f64; - let neg_zero: f64 = -0.0; - assert!(!nan.is_normal()); - assert!(!inf.is_normal()); - assert!(!neg_inf.is_normal()); - assert!(!zero.is_normal()); - assert!(!neg_zero.is_normal()); - assert!(1f64.is_normal()); - assert!(1e-307f64.is_normal()); - assert!(!1e-308f64.is_normal()); - } - - #[cfg_attr(all(target_arch = "wasm32", target_os = "emscripten"), ignore)] // issue 42630 - #[test] - fn test_classify() { - let nan: f64 = NAN; - let inf: f64 = INFINITY; - let neg_inf: f64 = NEG_INFINITY; - let zero: f64 = 0.0f64; - let neg_zero: f64 = -0.0; - assert_eq!(nan.classify(), Fp::Nan); - assert_eq!(inf.classify(), Fp::Infinite); - assert_eq!(neg_inf.classify(), Fp::Infinite); - assert_eq!(zero.classify(), Fp::Zero); - assert_eq!(neg_zero.classify(), Fp::Zero); - assert_eq!(1e-307f64.classify(), Fp::Normal); - assert_eq!(1e-308f64.classify(), Fp::Subnormal); - } - - #[test] - fn test_floor() { - assert_approx_eq!(1.0f64.floor(), 1.0f64); - assert_approx_eq!(1.3f64.floor(), 1.0f64); - assert_approx_eq!(1.5f64.floor(), 1.0f64); - assert_approx_eq!(1.7f64.floor(), 1.0f64); - assert_approx_eq!(0.0f64.floor(), 0.0f64); - assert_approx_eq!((-0.0f64).floor(), -0.0f64); - assert_approx_eq!((-1.0f64).floor(), -1.0f64); - assert_approx_eq!((-1.3f64).floor(), -2.0f64); - assert_approx_eq!((-1.5f64).floor(), -2.0f64); - assert_approx_eq!((-1.7f64).floor(), -2.0f64); - } - - #[test] - fn test_ceil() { - assert_approx_eq!(1.0f64.ceil(), 1.0f64); - assert_approx_eq!(1.3f64.ceil(), 2.0f64); - assert_approx_eq!(1.5f64.ceil(), 2.0f64); - assert_approx_eq!(1.7f64.ceil(), 2.0f64); - assert_approx_eq!(0.0f64.ceil(), 0.0f64); - assert_approx_eq!((-0.0f64).ceil(), -0.0f64); - assert_approx_eq!((-1.0f64).ceil(), -1.0f64); - assert_approx_eq!((-1.3f64).ceil(), -1.0f64); - assert_approx_eq!((-1.5f64).ceil(), -1.0f64); - assert_approx_eq!((-1.7f64).ceil(), -1.0f64); - } - - #[test] - fn test_round() { - assert_approx_eq!(1.0f64.round(), 1.0f64); - assert_approx_eq!(1.3f64.round(), 1.0f64); - assert_approx_eq!(1.5f64.round(), 2.0f64); - assert_approx_eq!(1.7f64.round(), 2.0f64); - assert_approx_eq!(0.0f64.round(), 0.0f64); - assert_approx_eq!((-0.0f64).round(), -0.0f64); - assert_approx_eq!((-1.0f64).round(), -1.0f64); - assert_approx_eq!((-1.3f64).round(), -1.0f64); - assert_approx_eq!((-1.5f64).round(), -2.0f64); - assert_approx_eq!((-1.7f64).round(), -2.0f64); - } - - #[test] - fn test_trunc() { - assert_approx_eq!(1.0f64.trunc(), 1.0f64); - assert_approx_eq!(1.3f64.trunc(), 1.0f64); - assert_approx_eq!(1.5f64.trunc(), 1.0f64); - assert_approx_eq!(1.7f64.trunc(), 1.0f64); - assert_approx_eq!(0.0f64.trunc(), 0.0f64); - assert_approx_eq!((-0.0f64).trunc(), -0.0f64); - assert_approx_eq!((-1.0f64).trunc(), -1.0f64); - assert_approx_eq!((-1.3f64).trunc(), -1.0f64); - assert_approx_eq!((-1.5f64).trunc(), -1.0f64); - assert_approx_eq!((-1.7f64).trunc(), -1.0f64); - } - - #[test] - fn test_fract() { - assert_approx_eq!(1.0f64.fract(), 0.0f64); - assert_approx_eq!(1.3f64.fract(), 0.3f64); - assert_approx_eq!(1.5f64.fract(), 0.5f64); - assert_approx_eq!(1.7f64.fract(), 0.7f64); - assert_approx_eq!(0.0f64.fract(), 0.0f64); - assert_approx_eq!((-0.0f64).fract(), -0.0f64); - assert_approx_eq!((-1.0f64).fract(), -0.0f64); - assert_approx_eq!((-1.3f64).fract(), -0.3f64); - assert_approx_eq!((-1.5f64).fract(), -0.5f64); - assert_approx_eq!((-1.7f64).fract(), -0.7f64); - } - - #[test] - fn test_abs() { - assert_eq!(INFINITY.abs(), INFINITY); - assert_eq!(1f64.abs(), 1f64); - assert_eq!(0f64.abs(), 0f64); - assert_eq!((-0f64).abs(), 0f64); - assert_eq!((-1f64).abs(), 1f64); - assert_eq!(NEG_INFINITY.abs(), INFINITY); - assert_eq!((1f64 / NEG_INFINITY).abs(), 0f64); - assert!(NAN.abs().is_nan()); - } - - #[test] - fn test_signum() { - assert_eq!(INFINITY.signum(), 1f64); - assert_eq!(1f64.signum(), 1f64); - assert_eq!(0f64.signum(), 1f64); - assert_eq!((-0f64).signum(), -1f64); - assert_eq!((-1f64).signum(), -1f64); - assert_eq!(NEG_INFINITY.signum(), -1f64); - assert_eq!((1f64 / NEG_INFINITY).signum(), -1f64); - assert!(NAN.signum().is_nan()); - } - - #[test] - fn test_is_sign_positive() { - assert!(INFINITY.is_sign_positive()); - assert!(1f64.is_sign_positive()); - assert!(0f64.is_sign_positive()); - assert!(!(-0f64).is_sign_positive()); - assert!(!(-1f64).is_sign_positive()); - assert!(!NEG_INFINITY.is_sign_positive()); - assert!(!(1f64 / NEG_INFINITY).is_sign_positive()); - assert!(NAN.is_sign_positive()); - assert!(!(-NAN).is_sign_positive()); - } - - #[test] - fn test_is_sign_negative() { - assert!(!INFINITY.is_sign_negative()); - assert!(!1f64.is_sign_negative()); - assert!(!0f64.is_sign_negative()); - assert!((-0f64).is_sign_negative()); - assert!((-1f64).is_sign_negative()); - assert!(NEG_INFINITY.is_sign_negative()); - assert!((1f64 / NEG_INFINITY).is_sign_negative()); - assert!(!NAN.is_sign_negative()); - assert!((-NAN).is_sign_negative()); - } - - #[test] - fn test_mul_add() { - let nan: f64 = NAN; - let inf: f64 = INFINITY; - let neg_inf: f64 = NEG_INFINITY; - assert_approx_eq!(12.3f64.mul_add(4.5, 6.7), 62.05); - assert_approx_eq!((-12.3f64).mul_add(-4.5, -6.7), 48.65); - assert_approx_eq!(0.0f64.mul_add(8.9, 1.2), 1.2); - assert_approx_eq!(3.4f64.mul_add(-0.0, 5.6), 5.6); - assert!(nan.mul_add(7.8, 9.0).is_nan()); - assert_eq!(inf.mul_add(7.8, 9.0), inf); - assert_eq!(neg_inf.mul_add(7.8, 9.0), neg_inf); - assert_eq!(8.9f64.mul_add(inf, 3.2), inf); - assert_eq!((-3.2f64).mul_add(2.4, neg_inf), neg_inf); - } - - #[test] - fn test_recip() { - let nan: f64 = NAN; - let inf: f64 = INFINITY; - let neg_inf: f64 = NEG_INFINITY; - assert_eq!(1.0f64.recip(), 1.0); - assert_eq!(2.0f64.recip(), 0.5); - assert_eq!((-0.4f64).recip(), -2.5); - assert_eq!(0.0f64.recip(), inf); - assert!(nan.recip().is_nan()); - assert_eq!(inf.recip(), 0.0); - assert_eq!(neg_inf.recip(), 0.0); - } - - #[test] - fn test_powi() { - let nan: f64 = NAN; - let inf: f64 = INFINITY; - let neg_inf: f64 = NEG_INFINITY; - assert_eq!(1.0f64.powi(1), 1.0); - assert_approx_eq!((-3.1f64).powi(2), 9.61); - assert_approx_eq!(5.9f64.powi(-2), 0.028727); - assert_eq!(8.3f64.powi(0), 1.0); - assert!(nan.powi(2).is_nan()); - assert_eq!(inf.powi(3), inf); - assert_eq!(neg_inf.powi(2), inf); - } - - #[test] - fn test_powf() { - let nan: f64 = NAN; - let inf: f64 = INFINITY; - let neg_inf: f64 = NEG_INFINITY; - assert_eq!(1.0f64.powf(1.0), 1.0); - assert_approx_eq!(3.4f64.powf(4.5), 246.408183); - assert_approx_eq!(2.7f64.powf(-3.2), 0.041652); - assert_approx_eq!((-3.1f64).powf(2.0), 9.61); - assert_approx_eq!(5.9f64.powf(-2.0), 0.028727); - assert_eq!(8.3f64.powf(0.0), 1.0); - assert!(nan.powf(2.0).is_nan()); - assert_eq!(inf.powf(2.0), inf); - assert_eq!(neg_inf.powf(3.0), neg_inf); - } - - #[test] - fn test_sqrt_domain() { - assert!(NAN.sqrt().is_nan()); - assert!(NEG_INFINITY.sqrt().is_nan()); - assert!((-1.0f64).sqrt().is_nan()); - assert_eq!((-0.0f64).sqrt(), -0.0); - assert_eq!(0.0f64.sqrt(), 0.0); - assert_eq!(1.0f64.sqrt(), 1.0); - assert_eq!(INFINITY.sqrt(), INFINITY); - } - - #[test] - fn test_exp() { - assert_eq!(1.0, 0.0f64.exp()); - assert_approx_eq!(2.718282, 1.0f64.exp()); - assert_approx_eq!(148.413159, 5.0f64.exp()); - - let inf: f64 = INFINITY; - let neg_inf: f64 = NEG_INFINITY; - let nan: f64 = NAN; - assert_eq!(inf, inf.exp()); - assert_eq!(0.0, neg_inf.exp()); - assert!(nan.exp().is_nan()); - } - - #[test] - fn test_exp2() { - assert_eq!(32.0, 5.0f64.exp2()); - assert_eq!(1.0, 0.0f64.exp2()); - - let inf: f64 = INFINITY; - let neg_inf: f64 = NEG_INFINITY; - let nan: f64 = NAN; - assert_eq!(inf, inf.exp2()); - assert_eq!(0.0, neg_inf.exp2()); - assert!(nan.exp2().is_nan()); - } - - #[test] - fn test_ln() { - let nan: f64 = NAN; - let inf: f64 = INFINITY; - let neg_inf: f64 = NEG_INFINITY; - assert_approx_eq!(1.0f64.exp().ln(), 1.0); - assert!(nan.ln().is_nan()); - assert_eq!(inf.ln(), inf); - assert!(neg_inf.ln().is_nan()); - assert!((-2.3f64).ln().is_nan()); - assert_eq!((-0.0f64).ln(), neg_inf); - assert_eq!(0.0f64.ln(), neg_inf); - assert_approx_eq!(4.0f64.ln(), 1.386294); - } - - #[test] - fn test_log() { - let nan: f64 = NAN; - let inf: f64 = INFINITY; - let neg_inf: f64 = NEG_INFINITY; - assert_eq!(10.0f64.log(10.0), 1.0); - assert_approx_eq!(2.3f64.log(3.5), 0.664858); - assert_eq!(1.0f64.exp().log(1.0f64.exp()), 1.0); - assert!(1.0f64.log(1.0).is_nan()); - assert!(1.0f64.log(-13.9).is_nan()); - assert!(nan.log(2.3).is_nan()); - assert_eq!(inf.log(10.0), inf); - assert!(neg_inf.log(8.8).is_nan()); - assert!((-2.3f64).log(0.1).is_nan()); - assert_eq!((-0.0f64).log(2.0), neg_inf); - assert_eq!(0.0f64.log(7.0), neg_inf); - } - - #[test] - fn test_log2() { - let nan: f64 = NAN; - let inf: f64 = INFINITY; - let neg_inf: f64 = NEG_INFINITY; - assert_approx_eq!(10.0f64.log2(), 3.321928); - assert_approx_eq!(2.3f64.log2(), 1.201634); - assert_approx_eq!(1.0f64.exp().log2(), 1.442695); - assert!(nan.log2().is_nan()); - assert_eq!(inf.log2(), inf); - assert!(neg_inf.log2().is_nan()); - assert!((-2.3f64).log2().is_nan()); - assert_eq!((-0.0f64).log2(), neg_inf); - assert_eq!(0.0f64.log2(), neg_inf); - } - - #[test] - fn test_log10() { - let nan: f64 = NAN; - let inf: f64 = INFINITY; - let neg_inf: f64 = NEG_INFINITY; - assert_eq!(10.0f64.log10(), 1.0); - assert_approx_eq!(2.3f64.log10(), 0.361728); - assert_approx_eq!(1.0f64.exp().log10(), 0.434294); - assert_eq!(1.0f64.log10(), 0.0); - assert!(nan.log10().is_nan()); - assert_eq!(inf.log10(), inf); - assert!(neg_inf.log10().is_nan()); - assert!((-2.3f64).log10().is_nan()); - assert_eq!((-0.0f64).log10(), neg_inf); - assert_eq!(0.0f64.log10(), neg_inf); - } - - #[test] - fn test_to_degrees() { - let pi: f64 = consts::PI; - let nan: f64 = NAN; - let inf: f64 = INFINITY; - let neg_inf: f64 = NEG_INFINITY; - assert_eq!(0.0f64.to_degrees(), 0.0); - assert_approx_eq!((-5.8f64).to_degrees(), -332.315521); - assert_eq!(pi.to_degrees(), 180.0); - assert!(nan.to_degrees().is_nan()); - assert_eq!(inf.to_degrees(), inf); - assert_eq!(neg_inf.to_degrees(), neg_inf); - } - - #[test] - fn test_to_radians() { - let pi: f64 = consts::PI; - let nan: f64 = NAN; - let inf: f64 = INFINITY; - let neg_inf: f64 = NEG_INFINITY; - assert_eq!(0.0f64.to_radians(), 0.0); - assert_approx_eq!(154.6f64.to_radians(), 2.698279); - assert_approx_eq!((-332.31f64).to_radians(), -5.799903); - assert_eq!(180.0f64.to_radians(), pi); - assert!(nan.to_radians().is_nan()); - assert_eq!(inf.to_radians(), inf); - assert_eq!(neg_inf.to_radians(), neg_inf); - } - - #[test] - fn test_asinh() { - assert_eq!(0.0f64.asinh(), 0.0f64); - assert_eq!((-0.0f64).asinh(), -0.0f64); - - let inf: f64 = INFINITY; - let neg_inf: f64 = NEG_INFINITY; - let nan: f64 = NAN; - assert_eq!(inf.asinh(), inf); - assert_eq!(neg_inf.asinh(), neg_inf); - assert!(nan.asinh().is_nan()); - assert!((-0.0f64).asinh().is_sign_negative()); - // issue 63271 - assert_approx_eq!(2.0f64.asinh(), 1.443635475178810342493276740273105f64); - assert_approx_eq!((-2.0f64).asinh(), -1.443635475178810342493276740273105f64); - } - - #[test] - fn test_acosh() { - assert_eq!(1.0f64.acosh(), 0.0f64); - assert!(0.999f64.acosh().is_nan()); - - let inf: f64 = INFINITY; - let neg_inf: f64 = NEG_INFINITY; - let nan: f64 = NAN; - assert_eq!(inf.acosh(), inf); - assert!(neg_inf.acosh().is_nan()); - assert!(nan.acosh().is_nan()); - assert_approx_eq!(2.0f64.acosh(), 1.31695789692481670862504634730796844f64); - assert_approx_eq!(3.0f64.acosh(), 1.76274717403908605046521864995958461f64); - } - - #[test] - fn test_atanh() { - assert_eq!(0.0f64.atanh(), 0.0f64); - assert_eq!((-0.0f64).atanh(), -0.0f64); - - let inf: f64 = INFINITY; - let neg_inf: f64 = NEG_INFINITY; - let nan: f64 = NAN; - assert_eq!(1.0f64.atanh(), inf); - assert_eq!((-1.0f64).atanh(), neg_inf); - assert!(2f64.atanh().atanh().is_nan()); - assert!((-2f64).atanh().atanh().is_nan()); - assert!(inf.atanh().is_nan()); - assert!(neg_inf.atanh().is_nan()); - assert!(nan.atanh().is_nan()); - assert_approx_eq!(0.5f64.atanh(), 0.54930614433405484569762261846126285f64); - assert_approx_eq!((-0.5f64).atanh(), -0.54930614433405484569762261846126285f64); - } - - #[test] - fn test_real_consts() { - use super::consts; - let pi: f64 = consts::PI; - let frac_pi_2: f64 = consts::FRAC_PI_2; - let frac_pi_3: f64 = consts::FRAC_PI_3; - let frac_pi_4: f64 = consts::FRAC_PI_4; - let frac_pi_6: f64 = consts::FRAC_PI_6; - let frac_pi_8: f64 = consts::FRAC_PI_8; - let frac_1_pi: f64 = consts::FRAC_1_PI; - let frac_2_pi: f64 = consts::FRAC_2_PI; - let frac_2_sqrtpi: f64 = consts::FRAC_2_SQRT_PI; - let sqrt2: f64 = consts::SQRT_2; - let frac_1_sqrt2: f64 = consts::FRAC_1_SQRT_2; - let e: f64 = consts::E; - let log2_e: f64 = consts::LOG2_E; - let log10_e: f64 = consts::LOG10_E; - let ln_2: f64 = consts::LN_2; - let ln_10: f64 = consts::LN_10; - - assert_approx_eq!(frac_pi_2, pi / 2f64); - assert_approx_eq!(frac_pi_3, pi / 3f64); - assert_approx_eq!(frac_pi_4, pi / 4f64); - assert_approx_eq!(frac_pi_6, pi / 6f64); - assert_approx_eq!(frac_pi_8, pi / 8f64); - assert_approx_eq!(frac_1_pi, 1f64 / pi); - assert_approx_eq!(frac_2_pi, 2f64 / pi); - assert_approx_eq!(frac_2_sqrtpi, 2f64 / pi.sqrt()); - assert_approx_eq!(sqrt2, 2f64.sqrt()); - assert_approx_eq!(frac_1_sqrt2, 1f64 / 2f64.sqrt()); - assert_approx_eq!(log2_e, e.log2()); - assert_approx_eq!(log10_e, e.log10()); - assert_approx_eq!(ln_2, 2f64.ln()); - assert_approx_eq!(ln_10, 10f64.ln()); - } - - #[test] - fn test_float_bits_conv() { - assert_eq!((1f64).to_bits(), 0x3ff0000000000000); - assert_eq!((12.5f64).to_bits(), 0x4029000000000000); - assert_eq!((1337f64).to_bits(), 0x4094e40000000000); - assert_eq!((-14.25f64).to_bits(), 0xc02c800000000000); - assert_approx_eq!(f64::from_bits(0x3ff0000000000000), 1.0); - assert_approx_eq!(f64::from_bits(0x4029000000000000), 12.5); - assert_approx_eq!(f64::from_bits(0x4094e40000000000), 1337.0); - assert_approx_eq!(f64::from_bits(0xc02c800000000000), -14.25); - - // Check that NaNs roundtrip their bits regardless of signalingness - // 0xA is 0b1010; 0x5 is 0b0101 -- so these two together clobbers all the mantissa bits - let masked_nan1 = f64::NAN.to_bits() ^ 0x000A_AAAA_AAAA_AAAA; - let masked_nan2 = f64::NAN.to_bits() ^ 0x0005_5555_5555_5555; - assert!(f64::from_bits(masked_nan1).is_nan()); - assert!(f64::from_bits(masked_nan2).is_nan()); - - assert_eq!(f64::from_bits(masked_nan1).to_bits(), masked_nan1); - assert_eq!(f64::from_bits(masked_nan2).to_bits(), masked_nan2); - } - - #[test] - #[should_panic] - fn test_clamp_min_greater_than_max() { - let _ = 1.0f64.clamp(3.0, 1.0); - } - - #[test] - #[should_panic] - fn test_clamp_min_is_nan() { - let _ = 1.0f64.clamp(NAN, 1.0); - } - - #[test] - #[should_panic] - fn test_clamp_max_is_nan() { - let _ = 1.0f64.clamp(3.0, NAN); - } -} diff --git a/crux-mir/lib/std/src/f64/tests.rs b/crux-mir/lib/std/src/f64/tests.rs new file mode 100644 index 000000000..5b039d445 --- /dev/null +++ b/crux-mir/lib/std/src/f64/tests.rs @@ -0,0 +1,839 @@ +use crate::f64::consts; +use crate::num::FpCategory as Fp; +use crate::num::*; + +#[test] +fn test_num_f64() { + test_num(10f64, 2f64); +} + +#[test] +fn test_min_nan() { + assert_eq!(f64::NAN.min(2.0), 2.0); + assert_eq!(2.0f64.min(f64::NAN), 2.0); +} + +#[test] +fn test_max_nan() { + assert_eq!(f64::NAN.max(2.0), 2.0); + assert_eq!(2.0f64.max(f64::NAN), 2.0); +} + +#[test] +fn test_nan() { + let nan: f64 = f64::NAN; + assert!(nan.is_nan()); + assert!(!nan.is_infinite()); + assert!(!nan.is_finite()); + assert!(!nan.is_normal()); + assert!(nan.is_sign_positive()); + assert!(!nan.is_sign_negative()); + assert_eq!(Fp::Nan, nan.classify()); +} + +#[test] +fn test_infinity() { + let inf: f64 = f64::INFINITY; + assert!(inf.is_infinite()); + assert!(!inf.is_finite()); + assert!(inf.is_sign_positive()); + assert!(!inf.is_sign_negative()); + assert!(!inf.is_nan()); + assert!(!inf.is_normal()); + assert_eq!(Fp::Infinite, inf.classify()); +} + +#[test] +fn test_neg_infinity() { + let neg_inf: f64 = f64::NEG_INFINITY; + assert!(neg_inf.is_infinite()); + assert!(!neg_inf.is_finite()); + assert!(!neg_inf.is_sign_positive()); + assert!(neg_inf.is_sign_negative()); + assert!(!neg_inf.is_nan()); + assert!(!neg_inf.is_normal()); + assert_eq!(Fp::Infinite, neg_inf.classify()); +} + +#[test] +fn test_zero() { + let zero: f64 = 0.0f64; + assert_eq!(0.0, zero); + assert!(!zero.is_infinite()); + assert!(zero.is_finite()); + assert!(zero.is_sign_positive()); + assert!(!zero.is_sign_negative()); + assert!(!zero.is_nan()); + assert!(!zero.is_normal()); + assert_eq!(Fp::Zero, zero.classify()); +} + +#[test] +fn test_neg_zero() { + let neg_zero: f64 = -0.0; + assert_eq!(0.0, neg_zero); + assert!(!neg_zero.is_infinite()); + assert!(neg_zero.is_finite()); + assert!(!neg_zero.is_sign_positive()); + assert!(neg_zero.is_sign_negative()); + assert!(!neg_zero.is_nan()); + assert!(!neg_zero.is_normal()); + assert_eq!(Fp::Zero, neg_zero.classify()); +} + +#[cfg_attr(all(target_arch = "wasm32", target_os = "emscripten"), ignore)] // issue 42630 +#[test] +fn test_one() { + let one: f64 = 1.0f64; + assert_eq!(1.0, one); + assert!(!one.is_infinite()); + assert!(one.is_finite()); + assert!(one.is_sign_positive()); + assert!(!one.is_sign_negative()); + assert!(!one.is_nan()); + assert!(one.is_normal()); + assert_eq!(Fp::Normal, one.classify()); +} + +#[test] +fn test_is_nan() { + let nan: f64 = f64::NAN; + let inf: f64 = f64::INFINITY; + let neg_inf: f64 = f64::NEG_INFINITY; + assert!(nan.is_nan()); + assert!(!0.0f64.is_nan()); + assert!(!5.3f64.is_nan()); + assert!(!(-10.732f64).is_nan()); + assert!(!inf.is_nan()); + assert!(!neg_inf.is_nan()); +} + +#[test] +fn test_is_infinite() { + let nan: f64 = f64::NAN; + let inf: f64 = f64::INFINITY; + let neg_inf: f64 = f64::NEG_INFINITY; + assert!(!nan.is_infinite()); + assert!(inf.is_infinite()); + assert!(neg_inf.is_infinite()); + assert!(!0.0f64.is_infinite()); + assert!(!42.8f64.is_infinite()); + assert!(!(-109.2f64).is_infinite()); +} + +#[test] +fn test_is_finite() { + let nan: f64 = f64::NAN; + let inf: f64 = f64::INFINITY; + let neg_inf: f64 = f64::NEG_INFINITY; + assert!(!nan.is_finite()); + assert!(!inf.is_finite()); + assert!(!neg_inf.is_finite()); + assert!(0.0f64.is_finite()); + assert!(42.8f64.is_finite()); + assert!((-109.2f64).is_finite()); +} + +#[cfg_attr(all(target_arch = "wasm32", target_os = "emscripten"), ignore)] // issue 42630 +#[test] +fn test_is_normal() { + let nan: f64 = f64::NAN; + let inf: f64 = f64::INFINITY; + let neg_inf: f64 = f64::NEG_INFINITY; + let zero: f64 = 0.0f64; + let neg_zero: f64 = -0.0; + assert!(!nan.is_normal()); + assert!(!inf.is_normal()); + assert!(!neg_inf.is_normal()); + assert!(!zero.is_normal()); + assert!(!neg_zero.is_normal()); + assert!(1f64.is_normal()); + assert!(1e-307f64.is_normal()); + assert!(!1e-308f64.is_normal()); +} + +#[cfg_attr(all(target_arch = "wasm32", target_os = "emscripten"), ignore)] // issue 42630 +#[test] +fn test_classify() { + let nan: f64 = f64::NAN; + let inf: f64 = f64::INFINITY; + let neg_inf: f64 = f64::NEG_INFINITY; + let zero: f64 = 0.0f64; + let neg_zero: f64 = -0.0; + assert_eq!(nan.classify(), Fp::Nan); + assert_eq!(inf.classify(), Fp::Infinite); + assert_eq!(neg_inf.classify(), Fp::Infinite); + assert_eq!(zero.classify(), Fp::Zero); + assert_eq!(neg_zero.classify(), Fp::Zero); + assert_eq!(1e-307f64.classify(), Fp::Normal); + assert_eq!(1e-308f64.classify(), Fp::Subnormal); +} + +#[test] +fn test_floor() { + assert_approx_eq!(1.0f64.floor(), 1.0f64); + assert_approx_eq!(1.3f64.floor(), 1.0f64); + assert_approx_eq!(1.5f64.floor(), 1.0f64); + assert_approx_eq!(1.7f64.floor(), 1.0f64); + assert_approx_eq!(0.0f64.floor(), 0.0f64); + assert_approx_eq!((-0.0f64).floor(), -0.0f64); + assert_approx_eq!((-1.0f64).floor(), -1.0f64); + assert_approx_eq!((-1.3f64).floor(), -2.0f64); + assert_approx_eq!((-1.5f64).floor(), -2.0f64); + assert_approx_eq!((-1.7f64).floor(), -2.0f64); +} + +#[test] +fn test_ceil() { + assert_approx_eq!(1.0f64.ceil(), 1.0f64); + assert_approx_eq!(1.3f64.ceil(), 2.0f64); + assert_approx_eq!(1.5f64.ceil(), 2.0f64); + assert_approx_eq!(1.7f64.ceil(), 2.0f64); + assert_approx_eq!(0.0f64.ceil(), 0.0f64); + assert_approx_eq!((-0.0f64).ceil(), -0.0f64); + assert_approx_eq!((-1.0f64).ceil(), -1.0f64); + assert_approx_eq!((-1.3f64).ceil(), -1.0f64); + assert_approx_eq!((-1.5f64).ceil(), -1.0f64); + assert_approx_eq!((-1.7f64).ceil(), -1.0f64); +} + +#[test] +fn test_round() { + assert_approx_eq!(1.0f64.round(), 1.0f64); + assert_approx_eq!(1.3f64.round(), 1.0f64); + assert_approx_eq!(1.5f64.round(), 2.0f64); + assert_approx_eq!(1.7f64.round(), 2.0f64); + assert_approx_eq!(0.0f64.round(), 0.0f64); + assert_approx_eq!((-0.0f64).round(), -0.0f64); + assert_approx_eq!((-1.0f64).round(), -1.0f64); + assert_approx_eq!((-1.3f64).round(), -1.0f64); + assert_approx_eq!((-1.5f64).round(), -2.0f64); + assert_approx_eq!((-1.7f64).round(), -2.0f64); +} + +#[test] +fn test_trunc() { + assert_approx_eq!(1.0f64.trunc(), 1.0f64); + assert_approx_eq!(1.3f64.trunc(), 1.0f64); + assert_approx_eq!(1.5f64.trunc(), 1.0f64); + assert_approx_eq!(1.7f64.trunc(), 1.0f64); + assert_approx_eq!(0.0f64.trunc(), 0.0f64); + assert_approx_eq!((-0.0f64).trunc(), -0.0f64); + assert_approx_eq!((-1.0f64).trunc(), -1.0f64); + assert_approx_eq!((-1.3f64).trunc(), -1.0f64); + assert_approx_eq!((-1.5f64).trunc(), -1.0f64); + assert_approx_eq!((-1.7f64).trunc(), -1.0f64); +} + +#[test] +fn test_fract() { + assert_approx_eq!(1.0f64.fract(), 0.0f64); + assert_approx_eq!(1.3f64.fract(), 0.3f64); + assert_approx_eq!(1.5f64.fract(), 0.5f64); + assert_approx_eq!(1.7f64.fract(), 0.7f64); + assert_approx_eq!(0.0f64.fract(), 0.0f64); + assert_approx_eq!((-0.0f64).fract(), -0.0f64); + assert_approx_eq!((-1.0f64).fract(), -0.0f64); + assert_approx_eq!((-1.3f64).fract(), -0.3f64); + assert_approx_eq!((-1.5f64).fract(), -0.5f64); + assert_approx_eq!((-1.7f64).fract(), -0.7f64); +} + +#[test] +fn test_abs() { + assert_eq!(f64::INFINITY.abs(), f64::INFINITY); + assert_eq!(1f64.abs(), 1f64); + assert_eq!(0f64.abs(), 0f64); + assert_eq!((-0f64).abs(), 0f64); + assert_eq!((-1f64).abs(), 1f64); + assert_eq!(f64::NEG_INFINITY.abs(), f64::INFINITY); + assert_eq!((1f64 / f64::NEG_INFINITY).abs(), 0f64); + assert!(f64::NAN.abs().is_nan()); +} + +#[test] +fn test_signum() { + assert_eq!(f64::INFINITY.signum(), 1f64); + assert_eq!(1f64.signum(), 1f64); + assert_eq!(0f64.signum(), 1f64); + assert_eq!((-0f64).signum(), -1f64); + assert_eq!((-1f64).signum(), -1f64); + assert_eq!(f64::NEG_INFINITY.signum(), -1f64); + assert_eq!((1f64 / f64::NEG_INFINITY).signum(), -1f64); + assert!(f64::NAN.signum().is_nan()); +} + +#[test] +fn test_is_sign_positive() { + assert!(f64::INFINITY.is_sign_positive()); + assert!(1f64.is_sign_positive()); + assert!(0f64.is_sign_positive()); + assert!(!(-0f64).is_sign_positive()); + assert!(!(-1f64).is_sign_positive()); + assert!(!f64::NEG_INFINITY.is_sign_positive()); + assert!(!(1f64 / f64::NEG_INFINITY).is_sign_positive()); + assert!(f64::NAN.is_sign_positive()); + assert!(!(-f64::NAN).is_sign_positive()); +} + +#[test] +fn test_is_sign_negative() { + assert!(!f64::INFINITY.is_sign_negative()); + assert!(!1f64.is_sign_negative()); + assert!(!0f64.is_sign_negative()); + assert!((-0f64).is_sign_negative()); + assert!((-1f64).is_sign_negative()); + assert!(f64::NEG_INFINITY.is_sign_negative()); + assert!((1f64 / f64::NEG_INFINITY).is_sign_negative()); + assert!(!f64::NAN.is_sign_negative()); + assert!((-f64::NAN).is_sign_negative()); +} + +#[allow(unused_macros)] +macro_rules! assert_f64_biteq { + ($left : expr, $right : expr) => { + let l: &f64 = &$left; + let r: &f64 = &$right; + let lb = l.to_bits(); + let rb = r.to_bits(); + assert_eq!(lb, rb, "float {} ({:#x}) is not equal to {} ({:#x})", *l, lb, *r, rb); + }; +} + +// Ignore test on x87 floating point, these platforms do not guarantee NaN +// payloads are preserved and flush denormals to zero, failing the tests. +#[cfg(not(target_arch = "x86"))] +#[test] +fn test_next_up() { + let tiny = f64::from_bits(1); + let tiny_up = f64::from_bits(2); + let max_down = f64::from_bits(0x7fef_ffff_ffff_fffe); + let largest_subnormal = f64::from_bits(0x000f_ffff_ffff_ffff); + let smallest_normal = f64::from_bits(0x0010_0000_0000_0000); + assert_f64_biteq!(f64::NEG_INFINITY.next_up(), f64::MIN); + assert_f64_biteq!(f64::MIN.next_up(), -max_down); + assert_f64_biteq!((-1.0 - f64::EPSILON).next_up(), -1.0); + assert_f64_biteq!((-smallest_normal).next_up(), -largest_subnormal); + assert_f64_biteq!((-tiny_up).next_up(), -tiny); + assert_f64_biteq!((-tiny).next_up(), -0.0f64); + assert_f64_biteq!((-0.0f64).next_up(), tiny); + assert_f64_biteq!(0.0f64.next_up(), tiny); + assert_f64_biteq!(tiny.next_up(), tiny_up); + assert_f64_biteq!(largest_subnormal.next_up(), smallest_normal); + assert_f64_biteq!(1.0f64.next_up(), 1.0 + f64::EPSILON); + assert_f64_biteq!(f64::MAX.next_up(), f64::INFINITY); + assert_f64_biteq!(f64::INFINITY.next_up(), f64::INFINITY); + + let nan0 = f64::NAN; + let nan1 = f64::from_bits(f64::NAN.to_bits() ^ 0x000a_aaaa_aaaa_aaaa); + let nan2 = f64::from_bits(f64::NAN.to_bits() ^ 0x0005_5555_5555_5555); + assert_f64_biteq!(nan0.next_up(), nan0); + assert_f64_biteq!(nan1.next_up(), nan1); + assert_f64_biteq!(nan2.next_up(), nan2); +} + +// Ignore test on x87 floating point, these platforms do not guarantee NaN +// payloads are preserved and flush denormals to zero, failing the tests. +#[cfg(not(target_arch = "x86"))] +#[test] +fn test_next_down() { + let tiny = f64::from_bits(1); + let tiny_up = f64::from_bits(2); + let max_down = f64::from_bits(0x7fef_ffff_ffff_fffe); + let largest_subnormal = f64::from_bits(0x000f_ffff_ffff_ffff); + let smallest_normal = f64::from_bits(0x0010_0000_0000_0000); + assert_f64_biteq!(f64::NEG_INFINITY.next_down(), f64::NEG_INFINITY); + assert_f64_biteq!(f64::MIN.next_down(), f64::NEG_INFINITY); + assert_f64_biteq!((-max_down).next_down(), f64::MIN); + assert_f64_biteq!((-1.0f64).next_down(), -1.0 - f64::EPSILON); + assert_f64_biteq!((-largest_subnormal).next_down(), -smallest_normal); + assert_f64_biteq!((-tiny).next_down(), -tiny_up); + assert_f64_biteq!((-0.0f64).next_down(), -tiny); + assert_f64_biteq!((0.0f64).next_down(), -tiny); + assert_f64_biteq!(tiny.next_down(), 0.0f64); + assert_f64_biteq!(tiny_up.next_down(), tiny); + assert_f64_biteq!(smallest_normal.next_down(), largest_subnormal); + assert_f64_biteq!((1.0 + f64::EPSILON).next_down(), 1.0f64); + assert_f64_biteq!(f64::MAX.next_down(), max_down); + assert_f64_biteq!(f64::INFINITY.next_down(), f64::MAX); + + let nan0 = f64::NAN; + let nan1 = f64::from_bits(f64::NAN.to_bits() ^ 0x000a_aaaa_aaaa_aaaa); + let nan2 = f64::from_bits(f64::NAN.to_bits() ^ 0x0005_5555_5555_5555); + assert_f64_biteq!(nan0.next_down(), nan0); + assert_f64_biteq!(nan1.next_down(), nan1); + assert_f64_biteq!(nan2.next_down(), nan2); +} + +#[test] +fn test_mul_add() { + let nan: f64 = f64::NAN; + let inf: f64 = f64::INFINITY; + let neg_inf: f64 = f64::NEG_INFINITY; + assert_approx_eq!(12.3f64.mul_add(4.5, 6.7), 62.05); + assert_approx_eq!((-12.3f64).mul_add(-4.5, -6.7), 48.65); + assert_approx_eq!(0.0f64.mul_add(8.9, 1.2), 1.2); + assert_approx_eq!(3.4f64.mul_add(-0.0, 5.6), 5.6); + assert!(nan.mul_add(7.8, 9.0).is_nan()); + assert_eq!(inf.mul_add(7.8, 9.0), inf); + assert_eq!(neg_inf.mul_add(7.8, 9.0), neg_inf); + assert_eq!(8.9f64.mul_add(inf, 3.2), inf); + assert_eq!((-3.2f64).mul_add(2.4, neg_inf), neg_inf); +} + +#[test] +fn test_recip() { + let nan: f64 = f64::NAN; + let inf: f64 = f64::INFINITY; + let neg_inf: f64 = f64::NEG_INFINITY; + assert_eq!(1.0f64.recip(), 1.0); + assert_eq!(2.0f64.recip(), 0.5); + assert_eq!((-0.4f64).recip(), -2.5); + assert_eq!(0.0f64.recip(), inf); + assert!(nan.recip().is_nan()); + assert_eq!(inf.recip(), 0.0); + assert_eq!(neg_inf.recip(), 0.0); +} + +#[test] +fn test_powi() { + let nan: f64 = f64::NAN; + let inf: f64 = f64::INFINITY; + let neg_inf: f64 = f64::NEG_INFINITY; + assert_eq!(1.0f64.powi(1), 1.0); + assert_approx_eq!((-3.1f64).powi(2), 9.61); + assert_approx_eq!(5.9f64.powi(-2), 0.028727); + assert_eq!(8.3f64.powi(0), 1.0); + assert!(nan.powi(2).is_nan()); + assert_eq!(inf.powi(3), inf); + assert_eq!(neg_inf.powi(2), inf); +} + +#[test] +fn test_powf() { + let nan: f64 = f64::NAN; + let inf: f64 = f64::INFINITY; + let neg_inf: f64 = f64::NEG_INFINITY; + assert_eq!(1.0f64.powf(1.0), 1.0); + assert_approx_eq!(3.4f64.powf(4.5), 246.408183); + assert_approx_eq!(2.7f64.powf(-3.2), 0.041652); + assert_approx_eq!((-3.1f64).powf(2.0), 9.61); + assert_approx_eq!(5.9f64.powf(-2.0), 0.028727); + assert_eq!(8.3f64.powf(0.0), 1.0); + assert!(nan.powf(2.0).is_nan()); + assert_eq!(inf.powf(2.0), inf); + assert_eq!(neg_inf.powf(3.0), neg_inf); +} + +#[test] +fn test_sqrt_domain() { + assert!(f64::NAN.sqrt().is_nan()); + assert!(f64::NEG_INFINITY.sqrt().is_nan()); + assert!((-1.0f64).sqrt().is_nan()); + assert_eq!((-0.0f64).sqrt(), -0.0); + assert_eq!(0.0f64.sqrt(), 0.0); + assert_eq!(1.0f64.sqrt(), 1.0); + assert_eq!(f64::INFINITY.sqrt(), f64::INFINITY); +} + +#[test] +fn test_exp() { + assert_eq!(1.0, 0.0f64.exp()); + assert_approx_eq!(2.718282, 1.0f64.exp()); + assert_approx_eq!(148.413159, 5.0f64.exp()); + + let inf: f64 = f64::INFINITY; + let neg_inf: f64 = f64::NEG_INFINITY; + let nan: f64 = f64::NAN; + assert_eq!(inf, inf.exp()); + assert_eq!(0.0, neg_inf.exp()); + assert!(nan.exp().is_nan()); +} + +#[test] +fn test_exp2() { + assert_eq!(32.0, 5.0f64.exp2()); + assert_eq!(1.0, 0.0f64.exp2()); + + let inf: f64 = f64::INFINITY; + let neg_inf: f64 = f64::NEG_INFINITY; + let nan: f64 = f64::NAN; + assert_eq!(inf, inf.exp2()); + assert_eq!(0.0, neg_inf.exp2()); + assert!(nan.exp2().is_nan()); +} + +#[test] +fn test_ln() { + let nan: f64 = f64::NAN; + let inf: f64 = f64::INFINITY; + let neg_inf: f64 = f64::NEG_INFINITY; + assert_approx_eq!(1.0f64.exp().ln(), 1.0); + assert!(nan.ln().is_nan()); + assert_eq!(inf.ln(), inf); + assert!(neg_inf.ln().is_nan()); + assert!((-2.3f64).ln().is_nan()); + assert_eq!((-0.0f64).ln(), neg_inf); + assert_eq!(0.0f64.ln(), neg_inf); + assert_approx_eq!(4.0f64.ln(), 1.386294); +} + +#[test] +fn test_log() { + let nan: f64 = f64::NAN; + let inf: f64 = f64::INFINITY; + let neg_inf: f64 = f64::NEG_INFINITY; + assert_eq!(10.0f64.log(10.0), 1.0); + assert_approx_eq!(2.3f64.log(3.5), 0.664858); + assert_eq!(1.0f64.exp().log(1.0f64.exp()), 1.0); + assert!(1.0f64.log(1.0).is_nan()); + assert!(1.0f64.log(-13.9).is_nan()); + assert!(nan.log(2.3).is_nan()); + assert_eq!(inf.log(10.0), inf); + assert!(neg_inf.log(8.8).is_nan()); + assert!((-2.3f64).log(0.1).is_nan()); + assert_eq!((-0.0f64).log(2.0), neg_inf); + assert_eq!(0.0f64.log(7.0), neg_inf); +} + +#[test] +fn test_log2() { + let nan: f64 = f64::NAN; + let inf: f64 = f64::INFINITY; + let neg_inf: f64 = f64::NEG_INFINITY; + assert_approx_eq!(10.0f64.log2(), 3.321928); + assert_approx_eq!(2.3f64.log2(), 1.201634); + assert_approx_eq!(1.0f64.exp().log2(), 1.442695); + assert!(nan.log2().is_nan()); + assert_eq!(inf.log2(), inf); + assert!(neg_inf.log2().is_nan()); + assert!((-2.3f64).log2().is_nan()); + assert_eq!((-0.0f64).log2(), neg_inf); + assert_eq!(0.0f64.log2(), neg_inf); +} + +#[test] +fn test_log10() { + let nan: f64 = f64::NAN; + let inf: f64 = f64::INFINITY; + let neg_inf: f64 = f64::NEG_INFINITY; + assert_eq!(10.0f64.log10(), 1.0); + assert_approx_eq!(2.3f64.log10(), 0.361728); + assert_approx_eq!(1.0f64.exp().log10(), 0.434294); + assert_eq!(1.0f64.log10(), 0.0); + assert!(nan.log10().is_nan()); + assert_eq!(inf.log10(), inf); + assert!(neg_inf.log10().is_nan()); + assert!((-2.3f64).log10().is_nan()); + assert_eq!((-0.0f64).log10(), neg_inf); + assert_eq!(0.0f64.log10(), neg_inf); +} + +#[test] +fn test_to_degrees() { + let pi: f64 = consts::PI; + let nan: f64 = f64::NAN; + let inf: f64 = f64::INFINITY; + let neg_inf: f64 = f64::NEG_INFINITY; + assert_eq!(0.0f64.to_degrees(), 0.0); + assert_approx_eq!((-5.8f64).to_degrees(), -332.315521); + assert_eq!(pi.to_degrees(), 180.0); + assert!(nan.to_degrees().is_nan()); + assert_eq!(inf.to_degrees(), inf); + assert_eq!(neg_inf.to_degrees(), neg_inf); +} + +#[test] +fn test_to_radians() { + let pi: f64 = consts::PI; + let nan: f64 = f64::NAN; + let inf: f64 = f64::INFINITY; + let neg_inf: f64 = f64::NEG_INFINITY; + assert_eq!(0.0f64.to_radians(), 0.0); + assert_approx_eq!(154.6f64.to_radians(), 2.698279); + assert_approx_eq!((-332.31f64).to_radians(), -5.799903); + assert_eq!(180.0f64.to_radians(), pi); + assert!(nan.to_radians().is_nan()); + assert_eq!(inf.to_radians(), inf); + assert_eq!(neg_inf.to_radians(), neg_inf); +} + +#[test] +fn test_asinh() { + assert_eq!(0.0f64.asinh(), 0.0f64); + assert_eq!((-0.0f64).asinh(), -0.0f64); + + let inf: f64 = f64::INFINITY; + let neg_inf: f64 = f64::NEG_INFINITY; + let nan: f64 = f64::NAN; + assert_eq!(inf.asinh(), inf); + assert_eq!(neg_inf.asinh(), neg_inf); + assert!(nan.asinh().is_nan()); + assert!((-0.0f64).asinh().is_sign_negative()); + // issue 63271 + assert_approx_eq!(2.0f64.asinh(), 1.443635475178810342493276740273105f64); + assert_approx_eq!((-2.0f64).asinh(), -1.443635475178810342493276740273105f64); + // regression test for the catastrophic cancellation fixed in 72486 + assert_approx_eq!((-67452098.07139316f64).asinh(), -18.72007542627454439398548429400083); + + // test for low accuracy from issue 104548 + assert_approx_eq!(60.0f64, 60.0f64.sinh().asinh()); + // mul needed for approximate comparison to be meaningful + assert_approx_eq!(1.0f64, 1e-15f64.sinh().asinh() * 1e15f64); +} + +#[test] +fn test_acosh() { + assert_eq!(1.0f64.acosh(), 0.0f64); + assert!(0.999f64.acosh().is_nan()); + + let inf: f64 = f64::INFINITY; + let neg_inf: f64 = f64::NEG_INFINITY; + let nan: f64 = f64::NAN; + assert_eq!(inf.acosh(), inf); + assert!(neg_inf.acosh().is_nan()); + assert!(nan.acosh().is_nan()); + assert_approx_eq!(2.0f64.acosh(), 1.31695789692481670862504634730796844f64); + assert_approx_eq!(3.0f64.acosh(), 1.76274717403908605046521864995958461f64); + + // test for low accuracy from issue 104548 + assert_approx_eq!(60.0f64, 60.0f64.cosh().acosh()); +} + +#[test] +fn test_atanh() { + assert_eq!(0.0f64.atanh(), 0.0f64); + assert_eq!((-0.0f64).atanh(), -0.0f64); + + let inf: f64 = f64::INFINITY; + let neg_inf: f64 = f64::NEG_INFINITY; + let nan: f64 = f64::NAN; + assert_eq!(1.0f64.atanh(), inf); + assert_eq!((-1.0f64).atanh(), neg_inf); + assert!(2f64.atanh().atanh().is_nan()); + assert!((-2f64).atanh().atanh().is_nan()); + assert!(inf.atanh().is_nan()); + assert!(neg_inf.atanh().is_nan()); + assert!(nan.atanh().is_nan()); + assert_approx_eq!(0.5f64.atanh(), 0.54930614433405484569762261846126285f64); + assert_approx_eq!((-0.5f64).atanh(), -0.54930614433405484569762261846126285f64); +} + +#[test] +fn test_real_consts() { + use super::consts; + let pi: f64 = consts::PI; + let frac_pi_2: f64 = consts::FRAC_PI_2; + let frac_pi_3: f64 = consts::FRAC_PI_3; + let frac_pi_4: f64 = consts::FRAC_PI_4; + let frac_pi_6: f64 = consts::FRAC_PI_6; + let frac_pi_8: f64 = consts::FRAC_PI_8; + let frac_1_pi: f64 = consts::FRAC_1_PI; + let frac_2_pi: f64 = consts::FRAC_2_PI; + let frac_2_sqrtpi: f64 = consts::FRAC_2_SQRT_PI; + let sqrt2: f64 = consts::SQRT_2; + let frac_1_sqrt2: f64 = consts::FRAC_1_SQRT_2; + let e: f64 = consts::E; + let log2_e: f64 = consts::LOG2_E; + let log10_e: f64 = consts::LOG10_E; + let ln_2: f64 = consts::LN_2; + let ln_10: f64 = consts::LN_10; + + assert_approx_eq!(frac_pi_2, pi / 2f64); + assert_approx_eq!(frac_pi_3, pi / 3f64); + assert_approx_eq!(frac_pi_4, pi / 4f64); + assert_approx_eq!(frac_pi_6, pi / 6f64); + assert_approx_eq!(frac_pi_8, pi / 8f64); + assert_approx_eq!(frac_1_pi, 1f64 / pi); + assert_approx_eq!(frac_2_pi, 2f64 / pi); + assert_approx_eq!(frac_2_sqrtpi, 2f64 / pi.sqrt()); + assert_approx_eq!(sqrt2, 2f64.sqrt()); + assert_approx_eq!(frac_1_sqrt2, 1f64 / 2f64.sqrt()); + assert_approx_eq!(log2_e, e.log2()); + assert_approx_eq!(log10_e, e.log10()); + assert_approx_eq!(ln_2, 2f64.ln()); + assert_approx_eq!(ln_10, 10f64.ln()); +} + +#[test] +fn test_float_bits_conv() { + assert_eq!((1f64).to_bits(), 0x3ff0000000000000); + assert_eq!((12.5f64).to_bits(), 0x4029000000000000); + assert_eq!((1337f64).to_bits(), 0x4094e40000000000); + assert_eq!((-14.25f64).to_bits(), 0xc02c800000000000); + assert_approx_eq!(f64::from_bits(0x3ff0000000000000), 1.0); + assert_approx_eq!(f64::from_bits(0x4029000000000000), 12.5); + assert_approx_eq!(f64::from_bits(0x4094e40000000000), 1337.0); + assert_approx_eq!(f64::from_bits(0xc02c800000000000), -14.25); + + // Check that NaNs roundtrip their bits regardless of signaling-ness + // 0xA is 0b1010; 0x5 is 0b0101 -- so these two together clobbers all the mantissa bits + let masked_nan1 = f64::NAN.to_bits() ^ 0x000A_AAAA_AAAA_AAAA; + let masked_nan2 = f64::NAN.to_bits() ^ 0x0005_5555_5555_5555; + assert!(f64::from_bits(masked_nan1).is_nan()); + assert!(f64::from_bits(masked_nan2).is_nan()); + + assert_eq!(f64::from_bits(masked_nan1).to_bits(), masked_nan1); + assert_eq!(f64::from_bits(masked_nan2).to_bits(), masked_nan2); +} + +#[test] +#[should_panic] +fn test_clamp_min_greater_than_max() { + let _ = 1.0f64.clamp(3.0, 1.0); +} + +#[test] +#[should_panic] +fn test_clamp_min_is_nan() { + let _ = 1.0f64.clamp(f64::NAN, 1.0); +} + +#[test] +#[should_panic] +fn test_clamp_max_is_nan() { + let _ = 1.0f64.clamp(3.0, f64::NAN); +} + +#[test] +fn test_total_cmp() { + use core::cmp::Ordering; + + fn quiet_bit_mask() -> u64 { + 1 << (f64::MANTISSA_DIGITS - 2) + } + + fn min_subnorm() -> f64 { + f64::MIN_POSITIVE / f64::powf(2.0, f64::MANTISSA_DIGITS as f64 - 1.0) + } + + fn max_subnorm() -> f64 { + f64::MIN_POSITIVE - min_subnorm() + } + + fn q_nan() -> f64 { + f64::from_bits(f64::NAN.to_bits() | quiet_bit_mask()) + } + + fn s_nan() -> f64 { + f64::from_bits((f64::NAN.to_bits() & !quiet_bit_mask()) + 42) + } + + assert_eq!(Ordering::Equal, (-q_nan()).total_cmp(&-q_nan())); + assert_eq!(Ordering::Equal, (-s_nan()).total_cmp(&-s_nan())); + assert_eq!(Ordering::Equal, (-f64::INFINITY).total_cmp(&-f64::INFINITY)); + assert_eq!(Ordering::Equal, (-f64::MAX).total_cmp(&-f64::MAX)); + assert_eq!(Ordering::Equal, (-2.5_f64).total_cmp(&-2.5)); + assert_eq!(Ordering::Equal, (-1.0_f64).total_cmp(&-1.0)); + assert_eq!(Ordering::Equal, (-1.5_f64).total_cmp(&-1.5)); + assert_eq!(Ordering::Equal, (-0.5_f64).total_cmp(&-0.5)); + assert_eq!(Ordering::Equal, (-f64::MIN_POSITIVE).total_cmp(&-f64::MIN_POSITIVE)); + assert_eq!(Ordering::Equal, (-max_subnorm()).total_cmp(&-max_subnorm())); + assert_eq!(Ordering::Equal, (-min_subnorm()).total_cmp(&-min_subnorm())); + assert_eq!(Ordering::Equal, (-0.0_f64).total_cmp(&-0.0)); + assert_eq!(Ordering::Equal, 0.0_f64.total_cmp(&0.0)); + assert_eq!(Ordering::Equal, min_subnorm().total_cmp(&min_subnorm())); + assert_eq!(Ordering::Equal, max_subnorm().total_cmp(&max_subnorm())); + assert_eq!(Ordering::Equal, f64::MIN_POSITIVE.total_cmp(&f64::MIN_POSITIVE)); + assert_eq!(Ordering::Equal, 0.5_f64.total_cmp(&0.5)); + assert_eq!(Ordering::Equal, 1.0_f64.total_cmp(&1.0)); + assert_eq!(Ordering::Equal, 1.5_f64.total_cmp(&1.5)); + assert_eq!(Ordering::Equal, 2.5_f64.total_cmp(&2.5)); + assert_eq!(Ordering::Equal, f64::MAX.total_cmp(&f64::MAX)); + assert_eq!(Ordering::Equal, f64::INFINITY.total_cmp(&f64::INFINITY)); + assert_eq!(Ordering::Equal, s_nan().total_cmp(&s_nan())); + assert_eq!(Ordering::Equal, q_nan().total_cmp(&q_nan())); + + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-s_nan())); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-f64::INFINITY)); + assert_eq!(Ordering::Less, (-f64::INFINITY).total_cmp(&-f64::MAX)); + assert_eq!(Ordering::Less, (-f64::MAX).total_cmp(&-2.5)); + assert_eq!(Ordering::Less, (-2.5_f64).total_cmp(&-1.5)); + assert_eq!(Ordering::Less, (-1.5_f64).total_cmp(&-1.0)); + assert_eq!(Ordering::Less, (-1.0_f64).total_cmp(&-0.5)); + assert_eq!(Ordering::Less, (-0.5_f64).total_cmp(&-f64::MIN_POSITIVE)); + assert_eq!(Ordering::Less, (-f64::MIN_POSITIVE).total_cmp(&-max_subnorm())); + assert_eq!(Ordering::Less, (-max_subnorm()).total_cmp(&-min_subnorm())); + assert_eq!(Ordering::Less, (-min_subnorm()).total_cmp(&-0.0)); + assert_eq!(Ordering::Less, (-0.0_f64).total_cmp(&0.0)); + assert_eq!(Ordering::Less, 0.0_f64.total_cmp(&min_subnorm())); + assert_eq!(Ordering::Less, min_subnorm().total_cmp(&max_subnorm())); + assert_eq!(Ordering::Less, max_subnorm().total_cmp(&f64::MIN_POSITIVE)); + assert_eq!(Ordering::Less, f64::MIN_POSITIVE.total_cmp(&0.5)); + assert_eq!(Ordering::Less, 0.5_f64.total_cmp(&1.0)); + assert_eq!(Ordering::Less, 1.0_f64.total_cmp(&1.5)); + assert_eq!(Ordering::Less, 1.5_f64.total_cmp(&2.5)); + assert_eq!(Ordering::Less, 2.5_f64.total_cmp(&f64::MAX)); + assert_eq!(Ordering::Less, f64::MAX.total_cmp(&f64::INFINITY)); + assert_eq!(Ordering::Less, f64::INFINITY.total_cmp(&s_nan())); + assert_eq!(Ordering::Less, s_nan().total_cmp(&q_nan())); + + assert_eq!(Ordering::Greater, (-s_nan()).total_cmp(&-q_nan())); + assert_eq!(Ordering::Greater, (-f64::INFINITY).total_cmp(&-s_nan())); + assert_eq!(Ordering::Greater, (-f64::MAX).total_cmp(&-f64::INFINITY)); + assert_eq!(Ordering::Greater, (-2.5_f64).total_cmp(&-f64::MAX)); + assert_eq!(Ordering::Greater, (-1.5_f64).total_cmp(&-2.5)); + assert_eq!(Ordering::Greater, (-1.0_f64).total_cmp(&-1.5)); + assert_eq!(Ordering::Greater, (-0.5_f64).total_cmp(&-1.0)); + assert_eq!(Ordering::Greater, (-f64::MIN_POSITIVE).total_cmp(&-0.5)); + assert_eq!(Ordering::Greater, (-max_subnorm()).total_cmp(&-f64::MIN_POSITIVE)); + assert_eq!(Ordering::Greater, (-min_subnorm()).total_cmp(&-max_subnorm())); + assert_eq!(Ordering::Greater, (-0.0_f64).total_cmp(&-min_subnorm())); + assert_eq!(Ordering::Greater, 0.0_f64.total_cmp(&-0.0)); + assert_eq!(Ordering::Greater, min_subnorm().total_cmp(&0.0)); + assert_eq!(Ordering::Greater, max_subnorm().total_cmp(&min_subnorm())); + assert_eq!(Ordering::Greater, f64::MIN_POSITIVE.total_cmp(&max_subnorm())); + assert_eq!(Ordering::Greater, 0.5_f64.total_cmp(&f64::MIN_POSITIVE)); + assert_eq!(Ordering::Greater, 1.0_f64.total_cmp(&0.5)); + assert_eq!(Ordering::Greater, 1.5_f64.total_cmp(&1.0)); + assert_eq!(Ordering::Greater, 2.5_f64.total_cmp(&1.5)); + assert_eq!(Ordering::Greater, f64::MAX.total_cmp(&2.5)); + assert_eq!(Ordering::Greater, f64::INFINITY.total_cmp(&f64::MAX)); + assert_eq!(Ordering::Greater, s_nan().total_cmp(&f64::INFINITY)); + assert_eq!(Ordering::Greater, q_nan().total_cmp(&s_nan())); + + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-s_nan())); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-f64::INFINITY)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-f64::MAX)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-2.5)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-1.5)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-1.0)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-0.5)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-f64::MIN_POSITIVE)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-max_subnorm())); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-min_subnorm())); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&-0.0)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&0.0)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&min_subnorm())); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&max_subnorm())); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&f64::MIN_POSITIVE)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&0.5)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&1.0)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&1.5)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&2.5)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&f64::MAX)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&f64::INFINITY)); + assert_eq!(Ordering::Less, (-q_nan()).total_cmp(&s_nan())); + + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-f64::INFINITY)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-f64::MAX)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-2.5)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-1.5)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-1.0)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-0.5)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-f64::MIN_POSITIVE)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-max_subnorm())); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-min_subnorm())); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&-0.0)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&0.0)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&min_subnorm())); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&max_subnorm())); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&f64::MIN_POSITIVE)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&0.5)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&1.0)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&1.5)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&2.5)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&f64::MAX)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&f64::INFINITY)); + assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&s_nan())); +} diff --git a/crux-mir/lib/std/src/ffi/c_str.rs b/crux-mir/lib/std/src/ffi/c_str.rs deleted file mode 100644 index 04eaba515..000000000 --- a/crux-mir/lib/std/src/ffi/c_str.rs +++ /dev/null @@ -1,1536 +0,0 @@ -use crate::ascii; -use crate::borrow::{Borrow, Cow}; -use crate::cmp::Ordering; -use crate::error::Error; -use crate::fmt::{self, Write}; -use crate::io; -use crate::mem; -use crate::memchr; -use crate::num::NonZeroU8; -use crate::ops; -use crate::os::raw::c_char; -use crate::ptr; -use crate::rc::Rc; -use crate::slice; -use crate::str::{self, Utf8Error}; -use crate::sync::Arc; -use crate::sys; - -/// A type representing an owned, C-compatible, nul-terminated string with no nul bytes in the -/// middle. -/// -/// This type serves the purpose of being able to safely generate a -/// C-compatible string from a Rust byte slice or vector. An instance of this -/// type is a static guarantee that the underlying bytes contain no interior 0 -/// bytes ("nul characters") and that the final byte is 0 ("nul terminator"). -/// -/// `CString` is to [`&CStr`] as [`String`] is to [`&str`]: the former -/// in each pair are owned strings; the latter are borrowed -/// references. -/// -/// # Creating a `CString` -/// -/// A `CString` is created from either a byte slice or a byte vector, -/// or anything that implements [`Into`]`<`[`Vec`]`<`[`u8`]`>>` (for -/// example, you can build a `CString` straight out of a [`String`] or -/// a [`&str`], since both implement that trait). -/// -/// The [`new`] method will actually check that the provided `&[u8]` -/// does not have 0 bytes in the middle, and return an error if it -/// finds one. -/// -/// # Extracting a raw pointer to the whole C string -/// -/// `CString` implements a [`as_ptr`] method through the [`Deref`] -/// trait. This method will give you a `*const c_char` which you can -/// feed directly to extern functions that expect a nul-terminated -/// string, like C's `strdup()`. Notice that [`as_ptr`] returns a -/// read-only pointer; if the C code writes to it, that causes -/// undefined behavior. -/// -/// # Extracting a slice of the whole C string -/// -/// Alternatively, you can obtain a `&[`[`u8`]`]` slice from a -/// `CString` with the [`as_bytes`] method. Slices produced in this -/// way do *not* contain the trailing nul terminator. This is useful -/// when you will be calling an extern function that takes a `*const -/// u8` argument which is not necessarily nul-terminated, plus another -/// argument with the length of the string — like C's `strndup()`. -/// You can of course get the slice's length with its -/// [`len`][slice.len] method. -/// -/// If you need a `&[`[`u8`]`]` slice *with* the nul terminator, you -/// can use [`as_bytes_with_nul`] instead. -/// -/// Once you have the kind of slice you need (with or without a nul -/// terminator), you can call the slice's own -/// [`as_ptr`][slice.as_ptr] method to get a read-only raw pointer to pass to -/// extern functions. See the documentation for that function for a -/// discussion on ensuring the lifetime of the raw pointer. -/// -/// [`Into`]: ../convert/trait.Into.html -/// [`Vec`]: ../vec/struct.Vec.html -/// [`String`]: ../string/struct.String.html -/// [`&str`]: ../primitive.str.html -/// [`u8`]: ../primitive.u8.html -/// [`new`]: #method.new -/// [`as_bytes`]: #method.as_bytes -/// [`as_bytes_with_nul`]: #method.as_bytes_with_nul -/// [`as_ptr`]: #method.as_ptr -/// [slice.as_ptr]: ../primitive.slice.html#method.as_ptr -/// [slice.len]: ../primitive.slice.html#method.len -/// [`Deref`]: ../ops/trait.Deref.html -/// [`CStr`]: struct.CStr.html -/// [`&CStr`]: struct.CStr.html -/// -/// # Examples -/// -/// ```ignore (extern-declaration) -/// # fn main() { -/// use std::ffi::CString; -/// use std::os::raw::c_char; -/// -/// extern { -/// fn my_printer(s: *const c_char); -/// } -/// -/// // We are certain that our string doesn't have 0 bytes in the middle, -/// // so we can .expect() -/// let c_to_print = CString::new("Hello, world!").expect("CString::new failed"); -/// unsafe { -/// my_printer(c_to_print.as_ptr()); -/// } -/// # } -/// ``` -/// -/// # Safety -/// -/// `CString` is intended for working with traditional C-style strings -/// (a sequence of non-nul bytes terminated by a single nul byte); the -/// primary use case for these kinds of strings is interoperating with C-like -/// code. Often you will need to transfer ownership to/from that external -/// code. It is strongly recommended that you thoroughly read through the -/// documentation of `CString` before use, as improper ownership management -/// of `CString` instances can lead to invalid memory accesses, memory leaks, -/// and other memory errors. - -#[derive(PartialEq, PartialOrd, Eq, Ord, Hash, Clone)] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct CString { - // Invariant 1: the slice ends with a zero byte and has a length of at least one. - // Invariant 2: the slice contains only one zero byte. - // Improper usage of unsafe function can break Invariant 2, but not Invariant 1. - inner: Box<[u8]>, -} - -/// Representation of a borrowed C string. -/// -/// This type represents a borrowed reference to a nul-terminated -/// array of bytes. It can be constructed safely from a `&[`[`u8`]`]` -/// slice, or unsafely from a raw `*const c_char`. It can then be -/// converted to a Rust [`&str`] by performing UTF-8 validation, or -/// into an owned [`CString`]. -/// -/// `&CStr` is to [`CString`] as [`&str`] is to [`String`]: the former -/// in each pair are borrowed references; the latter are owned -/// strings. -/// -/// Note that this structure is **not** `repr(C)` and is not recommended to be -/// placed in the signatures of FFI functions. Instead, safe wrappers of FFI -/// functions may leverage the unsafe [`from_ptr`] constructor to provide a safe -/// interface to other consumers. -/// -/// # Examples -/// -/// Inspecting a foreign C string: -/// -/// ```ignore (extern-declaration) -/// use std::ffi::CStr; -/// use std::os::raw::c_char; -/// -/// extern { fn my_string() -> *const c_char; } -/// -/// unsafe { -/// let slice = CStr::from_ptr(my_string()); -/// println!("string buffer size without nul terminator: {}", slice.to_bytes().len()); -/// } -/// ``` -/// -/// Passing a Rust-originating C string: -/// -/// ```ignore (extern-declaration) -/// use std::ffi::{CString, CStr}; -/// use std::os::raw::c_char; -/// -/// fn work(data: &CStr) { -/// extern { fn work_with(data: *const c_char); } -/// -/// unsafe { work_with(data.as_ptr()) } -/// } -/// -/// let s = CString::new("data data data data").expect("CString::new failed"); -/// work(&s); -/// ``` -/// -/// Converting a foreign C string into a Rust [`String`]: -/// -/// ```ignore (extern-declaration) -/// use std::ffi::CStr; -/// use std::os::raw::c_char; -/// -/// extern { fn my_string() -> *const c_char; } -/// -/// fn my_string_safe() -> String { -/// unsafe { -/// CStr::from_ptr(my_string()).to_string_lossy().into_owned() -/// } -/// } -/// -/// println!("string: {}", my_string_safe()); -/// ``` -/// -/// [`u8`]: ../primitive.u8.html -/// [`&str`]: ../primitive.str.html -/// [`String`]: ../string/struct.String.html -/// [`CString`]: struct.CString.html -/// [`from_ptr`]: #method.from_ptr -#[derive(Hash)] -#[stable(feature = "rust1", since = "1.0.0")] -// FIXME: -// `fn from` in `impl From<&CStr> for Box` current implementation relies -// on `CStr` being layout-compatible with `[u8]`. -// When attribute privacy is implemented, `CStr` should be annotated as `#[repr(transparent)]`. -// Anyway, `CStr` representation and layout are considered implementation detail, are -// not documented and must not be relied upon. -pub struct CStr { - // FIXME: this should not be represented with a DST slice but rather with - // just a raw `c_char` along with some form of marker to make - // this an unsized type. Essentially `sizeof(&CStr)` should be the - // same as `sizeof(&c_char)` but `CStr` should be an unsized type. - inner: [c_char], -} - -/// An error indicating that an interior nul byte was found. -/// -/// While Rust strings may contain nul bytes in the middle, C strings -/// can't, as that byte would effectively truncate the string. -/// -/// This error is created by the [`new`][`CString::new`] method on -/// [`CString`]. See its documentation for more. -/// -/// [`CString`]: struct.CString.html -/// [`CString::new`]: struct.CString.html#method.new -/// -/// # Examples -/// -/// ``` -/// use std::ffi::{CString, NulError}; -/// -/// let _: NulError = CString::new(b"f\0oo".to_vec()).unwrap_err(); -/// ``` -#[derive(Clone, PartialEq, Eq, Debug)] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct NulError(usize, Vec); - -/// An error indicating that a nul byte was not in the expected position. -/// -/// The slice used to create a [`CStr`] must have one and only one nul -/// byte at the end of the slice. -/// -/// This error is created by the -/// [`from_bytes_with_nul`][`CStr::from_bytes_with_nul`] method on -/// [`CStr`]. See its documentation for more. -/// -/// [`CStr`]: struct.CStr.html -/// [`CStr::from_bytes_with_nul`]: struct.CStr.html#method.from_bytes_with_nul -/// -/// # Examples -/// -/// ``` -/// use std::ffi::{CStr, FromBytesWithNulError}; -/// -/// let _: FromBytesWithNulError = CStr::from_bytes_with_nul(b"f\0oo").unwrap_err(); -/// ``` -#[derive(Clone, PartialEq, Eq, Debug)] -#[stable(feature = "cstr_from_bytes", since = "1.10.0")] -pub struct FromBytesWithNulError { - kind: FromBytesWithNulErrorKind, -} - -#[derive(Clone, PartialEq, Eq, Debug)] -enum FromBytesWithNulErrorKind { - InteriorNul(usize), - NotNulTerminated, -} - -impl FromBytesWithNulError { - fn interior_nul(pos: usize) -> FromBytesWithNulError { - FromBytesWithNulError { kind: FromBytesWithNulErrorKind::InteriorNul(pos) } - } - fn not_nul_terminated() -> FromBytesWithNulError { - FromBytesWithNulError { kind: FromBytesWithNulErrorKind::NotNulTerminated } - } -} - -/// An error indicating invalid UTF-8 when converting a [`CString`] into a [`String`]. -/// -/// `CString` is just a wrapper over a buffer of bytes with a nul -/// terminator; [`into_string`][`CString::into_string`] performs UTF-8 -/// validation on those bytes and may return this error. -/// -/// This `struct` is created by the -/// [`into_string`][`CString::into_string`] method on [`CString`]. See -/// its documentation for more. -/// -/// [`String`]: ../string/struct.String.html -/// [`CString`]: struct.CString.html -/// [`CString::into_string`]: struct.CString.html#method.into_string -#[derive(Clone, PartialEq, Eq, Debug)] -#[stable(feature = "cstring_into", since = "1.7.0")] -pub struct IntoStringError { - inner: CString, - error: Utf8Error, -} - -impl CString { - /// Creates a new C-compatible string from a container of bytes. - /// - /// This function will consume the provided data and use the - /// underlying bytes to construct a new string, ensuring that - /// there is a trailing 0 byte. This trailing 0 byte will be - /// appended by this function; the provided data should *not* - /// contain any 0 bytes in it. - /// - /// # Examples - /// - /// ```ignore (extern-declaration) - /// use std::ffi::CString; - /// use std::os::raw::c_char; - /// - /// extern { fn puts(s: *const c_char); } - /// - /// let to_print = CString::new("Hello!").expect("CString::new failed"); - /// unsafe { - /// puts(to_print.as_ptr()); - /// } - /// ``` - /// - /// # Errors - /// - /// This function will return an error if the supplied bytes contain an - /// internal 0 byte. The [`NulError`] returned will contain the bytes as well as - /// the position of the nul byte. - /// - /// [`NulError`]: struct.NulError.html - #[stable(feature = "rust1", since = "1.0.0")] - pub fn new>>(t: T) -> Result { - trait SpecIntoVec { - fn into_vec(self) -> Vec; - } - impl>> SpecIntoVec for T { - default fn into_vec(self) -> Vec { - self.into() - } - } - // Specialization for avoiding reallocation. - impl SpecIntoVec for &'_ [u8] { - fn into_vec(self) -> Vec { - let mut v = Vec::with_capacity(self.len() + 1); - v.extend(self); - v - } - } - impl SpecIntoVec for &'_ str { - fn into_vec(self) -> Vec { - let mut v = Vec::with_capacity(self.len() + 1); - v.extend(self.as_bytes()); - v - } - } - - Self::_new(SpecIntoVec::into_vec(t)) - } - - fn _new(bytes: Vec) -> Result { - match memchr::memchr(0, &bytes) { - Some(i) => Err(NulError(i, bytes)), - None => Ok(unsafe { CString::from_vec_unchecked(bytes) }), - } - } - - /// Creates a C-compatible string by consuming a byte vector, - /// without checking for interior 0 bytes. - /// - /// This method is equivalent to [`new`] except that no runtime assertion - /// is made that `v` contains no 0 bytes, and it requires an actual - /// byte vector, not anything that can be converted to one with Into. - /// - /// [`new`]: #method.new - /// - /// # Examples - /// - /// ``` - /// use std::ffi::CString; - /// - /// let raw = b"foo".to_vec(); - /// unsafe { - /// let c_string = CString::from_vec_unchecked(raw); - /// } - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub unsafe fn from_vec_unchecked(mut v: Vec) -> CString { - v.reserve_exact(1); - v.push(0); - CString { inner: v.into_boxed_slice() } - } - - /// Retakes ownership of a `CString` that was transferred to C via [`into_raw`]. - /// - /// Additionally, the length of the string will be recalculated from the pointer. - /// - /// # Safety - /// - /// This should only ever be called with a pointer that was earlier - /// obtained by calling [`into_raw`] on a `CString`. Other usage (e.g., trying to take - /// ownership of a string that was allocated by foreign code) is likely to lead - /// to undefined behavior or allocator corruption. - /// - /// > **Note:** If you need to borrow a string that was allocated by - /// > foreign code, use [`CStr`]. If you need to take ownership of - /// > a string that was allocated by foreign code, you will need to - /// > make your own provisions for freeing it appropriately, likely - /// > with the foreign code's API to do that. - /// - /// [`into_raw`]: #method.into_raw - /// [`CStr`]: struct.CStr.html - /// - /// # Examples - /// - /// Creates a `CString`, pass ownership to an `extern` function (via raw pointer), then retake - /// ownership with `from_raw`: - /// - /// ```ignore (extern-declaration) - /// use std::ffi::CString; - /// use std::os::raw::c_char; - /// - /// extern { - /// fn some_extern_function(s: *mut c_char); - /// } - /// - /// let c_string = CString::new("Hello!").expect("CString::new failed"); - /// let raw = c_string.into_raw(); - /// unsafe { - /// some_extern_function(raw); - /// let c_string = CString::from_raw(raw); - /// } - /// ``` - #[stable(feature = "cstr_memory", since = "1.4.0")] - pub unsafe fn from_raw(ptr: *mut c_char) -> CString { - let len = sys::strlen(ptr) + 1; // Including the NUL byte - let slice = slice::from_raw_parts_mut(ptr, len as usize); - CString { inner: Box::from_raw(slice as *mut [c_char] as *mut [u8]) } - } - - /// Consumes the `CString` and transfers ownership of the string to a C caller. - /// - /// The pointer which this function returns must be returned to Rust and reconstituted using - /// [`from_raw`] to be properly deallocated. Specifically, one - /// should *not* use the standard C `free()` function to deallocate - /// this string. - /// - /// Failure to call [`from_raw`] will lead to a memory leak. - /// - /// [`from_raw`]: #method.from_raw - /// - /// # Examples - /// - /// ``` - /// use std::ffi::CString; - /// - /// let c_string = CString::new("foo").expect("CString::new failed"); - /// - /// let ptr = c_string.into_raw(); - /// - /// unsafe { - /// assert_eq!(b'f', *ptr as u8); - /// assert_eq!(b'o', *ptr.offset(1) as u8); - /// assert_eq!(b'o', *ptr.offset(2) as u8); - /// assert_eq!(b'\0', *ptr.offset(3) as u8); - /// - /// // retake pointer to free memory - /// let _ = CString::from_raw(ptr); - /// } - /// ``` - #[inline] - #[stable(feature = "cstr_memory", since = "1.4.0")] - pub fn into_raw(self) -> *mut c_char { - Box::into_raw(self.into_inner()) as *mut c_char - } - - /// Converts the `CString` into a [`String`] if it contains valid UTF-8 data. - /// - /// On failure, ownership of the original `CString` is returned. - /// - /// [`String`]: ../string/struct.String.html - /// - /// # Examples - /// - /// ``` - /// use std::ffi::CString; - /// - /// let valid_utf8 = vec![b'f', b'o', b'o']; - /// let cstring = CString::new(valid_utf8).expect("CString::new failed"); - /// assert_eq!(cstring.into_string().expect("into_string() call failed"), "foo"); - /// - /// let invalid_utf8 = vec![b'f', 0xff, b'o', b'o']; - /// let cstring = CString::new(invalid_utf8).expect("CString::new failed"); - /// let err = cstring.into_string().err().expect("into_string().err() failed"); - /// assert_eq!(err.utf8_error().valid_up_to(), 1); - /// ``` - - #[stable(feature = "cstring_into", since = "1.7.0")] - pub fn into_string(self) -> Result { - String::from_utf8(self.into_bytes()).map_err(|e| IntoStringError { - error: e.utf8_error(), - inner: unsafe { CString::from_vec_unchecked(e.into_bytes()) }, - }) - } - - /// Consumes the `CString` and returns the underlying byte buffer. - /// - /// The returned buffer does **not** contain the trailing nul - /// terminator, and it is guaranteed to not have any interior nul - /// bytes. - /// - /// # Examples - /// - /// ``` - /// use std::ffi::CString; - /// - /// let c_string = CString::new("foo").expect("CString::new failed"); - /// let bytes = c_string.into_bytes(); - /// assert_eq!(bytes, vec![b'f', b'o', b'o']); - /// ``` - #[stable(feature = "cstring_into", since = "1.7.0")] - pub fn into_bytes(self) -> Vec { - let mut vec = self.into_inner().into_vec(); - let _nul = vec.pop(); - debug_assert_eq!(_nul, Some(0u8)); - vec - } - - /// Equivalent to the [`into_bytes`] function except that the returned vector - /// includes the trailing nul terminator. - /// - /// [`into_bytes`]: #method.into_bytes - /// - /// # Examples - /// - /// ``` - /// use std::ffi::CString; - /// - /// let c_string = CString::new("foo").expect("CString::new failed"); - /// let bytes = c_string.into_bytes_with_nul(); - /// assert_eq!(bytes, vec![b'f', b'o', b'o', b'\0']); - /// ``` - #[stable(feature = "cstring_into", since = "1.7.0")] - pub fn into_bytes_with_nul(self) -> Vec { - self.into_inner().into_vec() - } - - /// Returns the contents of this `CString` as a slice of bytes. - /// - /// The returned slice does **not** contain the trailing nul - /// terminator, and it is guaranteed to not have any interior nul - /// bytes. If you need the nul terminator, use - /// [`as_bytes_with_nul`] instead. - /// - /// [`as_bytes_with_nul`]: #method.as_bytes_with_nul - /// - /// # Examples - /// - /// ``` - /// use std::ffi::CString; - /// - /// let c_string = CString::new("foo").expect("CString::new failed"); - /// let bytes = c_string.as_bytes(); - /// assert_eq!(bytes, &[b'f', b'o', b'o']); - /// ``` - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn as_bytes(&self) -> &[u8] { - &self.inner[..self.inner.len() - 1] - } - - /// Equivalent to the [`as_bytes`] function except that the returned slice - /// includes the trailing nul terminator. - /// - /// [`as_bytes`]: #method.as_bytes - /// - /// # Examples - /// - /// ``` - /// use std::ffi::CString; - /// - /// let c_string = CString::new("foo").expect("CString::new failed"); - /// let bytes = c_string.as_bytes_with_nul(); - /// assert_eq!(bytes, &[b'f', b'o', b'o', b'\0']); - /// ``` - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn as_bytes_with_nul(&self) -> &[u8] { - &self.inner - } - - /// Extracts a [`CStr`] slice containing the entire string. - /// - /// [`CStr`]: struct.CStr.html - /// - /// # Examples - /// - /// ``` - /// use std::ffi::{CString, CStr}; - /// - /// let c_string = CString::new(b"foo".to_vec()).expect("CString::new failed"); - /// let cstr = c_string.as_c_str(); - /// assert_eq!(cstr, - /// CStr::from_bytes_with_nul(b"foo\0").expect("CStr::from_bytes_with_nul failed")); - /// ``` - #[inline] - #[stable(feature = "as_c_str", since = "1.20.0")] - pub fn as_c_str(&self) -> &CStr { - &*self - } - - /// Converts this `CString` into a boxed [`CStr`]. - /// - /// [`CStr`]: struct.CStr.html - /// - /// # Examples - /// - /// ``` - /// use std::ffi::{CString, CStr}; - /// - /// let c_string = CString::new(b"foo".to_vec()).expect("CString::new failed"); - /// let boxed = c_string.into_boxed_c_str(); - /// assert_eq!(&*boxed, - /// CStr::from_bytes_with_nul(b"foo\0").expect("CStr::from_bytes_with_nul failed")); - /// ``` - #[stable(feature = "into_boxed_c_str", since = "1.20.0")] - pub fn into_boxed_c_str(self) -> Box { - unsafe { Box::from_raw(Box::into_raw(self.into_inner()) as *mut CStr) } - } - - /// Bypass "move out of struct which implements [`Drop`] trait" restriction. - /// - /// [`Drop`]: ../ops/trait.Drop.html - fn into_inner(self) -> Box<[u8]> { - // Rationale: `mem::forget(self)` invalidates the previous call to `ptr::read(&self.inner)` - // so we use `ManuallyDrop` to ensure `self` is not dropped. - // Then we can return the box directly without invalidating it. - // See https://github.com/rust-lang/rust/issues/62553. - let this = mem::ManuallyDrop::new(self); - unsafe { ptr::read(&this.inner) } - } -} - -// Turns this `CString` into an empty string to prevent -// memory-unsafe code from working by accident. Inline -// to prevent LLVM from optimizing it away in debug builds. -#[stable(feature = "cstring_drop", since = "1.13.0")] -impl Drop for CString { - #[inline] - fn drop(&mut self) { - unsafe { - *self.inner.get_unchecked_mut(0) = 0; - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ops::Deref for CString { - type Target = CStr; - - #[inline] - fn deref(&self) -> &CStr { - unsafe { CStr::from_bytes_with_nul_unchecked(self.as_bytes_with_nul()) } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl fmt::Debug for CString { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&**self, f) - } -} - -#[stable(feature = "cstring_into", since = "1.7.0")] -impl From for Vec { - /// Converts a [`CString`] into a [`Vec`]``. - /// - /// The conversion consumes the [`CString`], and removes the terminating NUL byte. - /// - /// [`Vec`]: ../vec/struct.Vec.html - /// [`CString`]: ../ffi/struct.CString.html - #[inline] - fn from(s: CString) -> Vec { - s.into_bytes() - } -} - -#[stable(feature = "cstr_debug", since = "1.3.0")] -impl fmt::Debug for CStr { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "\"")?; - for byte in self.to_bytes().iter().flat_map(|&b| ascii::escape_default(b)) { - f.write_char(byte as char)?; - } - write!(f, "\"") - } -} - -#[stable(feature = "cstr_default", since = "1.10.0")] -impl Default for &CStr { - fn default() -> Self { - const SLICE: &[c_char] = &[0]; - unsafe { CStr::from_ptr(SLICE.as_ptr()) } - } -} - -#[stable(feature = "cstr_default", since = "1.10.0")] -impl Default for CString { - /// Creates an empty `CString`. - fn default() -> CString { - let a: &CStr = Default::default(); - a.to_owned() - } -} - -#[stable(feature = "cstr_borrow", since = "1.3.0")] -impl Borrow for CString { - #[inline] - fn borrow(&self) -> &CStr { - self - } -} - -#[stable(feature = "cstring_from_cow_cstr", since = "1.28.0")] -impl<'a> From> for CString { - #[inline] - fn from(s: Cow<'a, CStr>) -> Self { - s.into_owned() - } -} - -#[stable(feature = "box_from_c_str", since = "1.17.0")] -impl From<&CStr> for Box { - fn from(s: &CStr) -> Box { - let boxed: Box<[u8]> = Box::from(s.to_bytes_with_nul()); - unsafe { Box::from_raw(Box::into_raw(boxed) as *mut CStr) } - } -} - -#[stable(feature = "c_string_from_box", since = "1.18.0")] -impl From> for CString { - /// Converts a [`Box`]`` into a [`CString`] without copying or allocating. - /// - /// [`Box`]: ../boxed/struct.Box.html - /// [`CString`]: ../ffi/struct.CString.html - #[inline] - fn from(s: Box) -> CString { - s.into_c_string() - } -} - -#[stable(feature = "cstring_from_vec_of_nonzerou8", since = "1.43.0")] -impl From> for CString { - /// Converts a [`Vec`]`<`[`NonZeroU8`]`>` into a [`CString`] without - /// copying nor checking for inner null bytes. - /// - /// [`CString`]: ../ffi/struct.CString.html - /// [`NonZeroU8`]: ../num/struct.NonZeroU8.html - /// [`Vec`]: ../vec/struct.Vec.html - #[inline] - fn from(v: Vec) -> CString { - unsafe { - // Transmute `Vec` to `Vec`. - let v: Vec = { - // Safety: - // - transmuting between `NonZeroU8` and `u8` is sound; - // - `alloc::Layout == alloc::Layout`. - let (ptr, len, cap): (*mut NonZeroU8, _, _) = Vec::into_raw_parts(v); - Vec::from_raw_parts(ptr.cast::(), len, cap) - }; - // Safety: `v` cannot contain null bytes, given the type-level - // invariant of `NonZeroU8`. - CString::from_vec_unchecked(v) - } - } -} - -#[stable(feature = "more_box_slice_clone", since = "1.29.0")] -impl Clone for Box { - #[inline] - fn clone(&self) -> Self { - (**self).into() - } -} - -#[stable(feature = "box_from_c_string", since = "1.20.0")] -impl From for Box { - /// Converts a [`CString`] into a [`Box`]`` without copying or allocating. - /// - /// [`CString`]: ../ffi/struct.CString.html - /// [`Box`]: ../boxed/struct.Box.html - #[inline] - fn from(s: CString) -> Box { - s.into_boxed_c_str() - } -} - -#[stable(feature = "cow_from_cstr", since = "1.28.0")] -impl<'a> From for Cow<'a, CStr> { - #[inline] - fn from(s: CString) -> Cow<'a, CStr> { - Cow::Owned(s) - } -} - -#[stable(feature = "cow_from_cstr", since = "1.28.0")] -impl<'a> From<&'a CStr> for Cow<'a, CStr> { - #[inline] - fn from(s: &'a CStr) -> Cow<'a, CStr> { - Cow::Borrowed(s) - } -} - -#[stable(feature = "cow_from_cstr", since = "1.28.0")] -impl<'a> From<&'a CString> for Cow<'a, CStr> { - #[inline] - fn from(s: &'a CString) -> Cow<'a, CStr> { - Cow::Borrowed(s.as_c_str()) - } -} - -#[stable(feature = "shared_from_slice2", since = "1.24.0")] -impl From for Arc { - /// Converts a [`CString`] into a [`Arc`]`` without copying or allocating. - /// - /// [`CString`]: ../ffi/struct.CString.html - /// [`Arc`]: ../sync/struct.Arc.html - #[inline] - fn from(s: CString) -> Arc { - let arc: Arc<[u8]> = Arc::from(s.into_inner()); - unsafe { Arc::from_raw(Arc::into_raw(arc) as *const CStr) } - } -} - -#[stable(feature = "shared_from_slice2", since = "1.24.0")] -impl From<&CStr> for Arc { - #[inline] - fn from(s: &CStr) -> Arc { - let arc: Arc<[u8]> = Arc::from(s.to_bytes_with_nul()); - unsafe { Arc::from_raw(Arc::into_raw(arc) as *const CStr) } - } -} - -#[stable(feature = "shared_from_slice2", since = "1.24.0")] -impl From for Rc { - /// Converts a [`CString`] into a [`Rc`]`` without copying or allocating. - /// - /// [`CString`]: ../ffi/struct.CString.html - /// [`Rc`]: ../rc/struct.Rc.html - #[inline] - fn from(s: CString) -> Rc { - let rc: Rc<[u8]> = Rc::from(s.into_inner()); - unsafe { Rc::from_raw(Rc::into_raw(rc) as *const CStr) } - } -} - -#[stable(feature = "shared_from_slice2", since = "1.24.0")] -impl From<&CStr> for Rc { - #[inline] - fn from(s: &CStr) -> Rc { - let rc: Rc<[u8]> = Rc::from(s.to_bytes_with_nul()); - unsafe { Rc::from_raw(Rc::into_raw(rc) as *const CStr) } - } -} - -#[stable(feature = "default_box_extra", since = "1.17.0")] -impl Default for Box { - fn default() -> Box { - let boxed: Box<[u8]> = Box::from([0]); - unsafe { Box::from_raw(Box::into_raw(boxed) as *mut CStr) } - } -} - -impl NulError { - /// Returns the position of the nul byte in the slice that caused - /// [`CString::new`] to fail. - /// - /// [`CString::new`]: struct.CString.html#method.new - /// - /// # Examples - /// - /// ``` - /// use std::ffi::CString; - /// - /// let nul_error = CString::new("foo\0bar").unwrap_err(); - /// assert_eq!(nul_error.nul_position(), 3); - /// - /// let nul_error = CString::new("foo bar\0").unwrap_err(); - /// assert_eq!(nul_error.nul_position(), 7); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn nul_position(&self) -> usize { - self.0 - } - - /// Consumes this error, returning the underlying vector of bytes which - /// generated the error in the first place. - /// - /// # Examples - /// - /// ``` - /// use std::ffi::CString; - /// - /// let nul_error = CString::new("foo\0bar").unwrap_err(); - /// assert_eq!(nul_error.into_vec(), b"foo\0bar"); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn into_vec(self) -> Vec { - self.1 - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Error for NulError { - #[allow(deprecated)] - fn description(&self) -> &str { - "nul byte found in data" - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl fmt::Display for NulError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "nul byte found in provided data at position: {}", self.0) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl From for io::Error { - /// Converts a [`NulError`] into a [`io::Error`]. - /// - /// [`NulError`]: ../ffi/struct.NulError.html - /// [`io::Error`]: ../io/struct.Error.html - fn from(_: NulError) -> io::Error { - io::Error::new(io::ErrorKind::InvalidInput, "data provided contains a nul byte") - } -} - -#[stable(feature = "frombyteswithnulerror_impls", since = "1.17.0")] -impl Error for FromBytesWithNulError { - #[allow(deprecated)] - fn description(&self) -> &str { - match self.kind { - FromBytesWithNulErrorKind::InteriorNul(..) => { - "data provided contains an interior nul byte" - } - FromBytesWithNulErrorKind::NotNulTerminated => "data provided is not nul terminated", - } - } -} - -#[stable(feature = "frombyteswithnulerror_impls", since = "1.17.0")] -impl fmt::Display for FromBytesWithNulError { - #[allow(deprecated, deprecated_in_future)] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.description())?; - if let FromBytesWithNulErrorKind::InteriorNul(pos) = self.kind { - write!(f, " at byte pos {}", pos)?; - } - Ok(()) - } -} - -impl IntoStringError { - /// Consumes this error, returning original [`CString`] which generated the - /// error. - /// - /// [`CString`]: struct.CString.html - #[stable(feature = "cstring_into", since = "1.7.0")] - pub fn into_cstring(self) -> CString { - self.inner - } - - /// Access the underlying UTF-8 error that was the cause of this error. - #[stable(feature = "cstring_into", since = "1.7.0")] - pub fn utf8_error(&self) -> Utf8Error { - self.error - } -} - -#[stable(feature = "cstring_into", since = "1.7.0")] -impl Error for IntoStringError { - #[allow(deprecated)] - fn description(&self) -> &str { - "C string contained non-utf8 bytes" - } - - fn source(&self) -> Option<&(dyn Error + 'static)> { - Some(&self.error) - } -} - -#[stable(feature = "cstring_into", since = "1.7.0")] -impl fmt::Display for IntoStringError { - #[allow(deprecated, deprecated_in_future)] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.description().fmt(f) - } -} - -impl CStr { - /// Wraps a raw C string with a safe C string wrapper. - /// - /// This function will wrap the provided `ptr` with a `CStr` wrapper, which - /// allows inspection and interoperation of non-owned C strings. The total - /// size of the raw C string must be smaller than `isize::MAX` **bytes** - /// in memory due to calling the `slice::from_raw_parts` function. - /// This method is unsafe for a number of reasons: - /// - /// * There is no guarantee to the validity of `ptr`. - /// * The returned lifetime is not guaranteed to be the actual lifetime of - /// `ptr`. - /// * There is no guarantee that the memory pointed to by `ptr` contains a - /// valid nul terminator byte at the end of the string. - /// * It is not guaranteed that the memory pointed by `ptr` won't change - /// before the `CStr` has been destroyed. - /// - /// > **Note**: This operation is intended to be a 0-cost cast but it is - /// > currently implemented with an up-front calculation of the length of - /// > the string. This is not guaranteed to always be the case. - /// - /// # Examples - /// - /// ```ignore (extern-declaration) - /// # fn main() { - /// use std::ffi::CStr; - /// use std::os::raw::c_char; - /// - /// extern { - /// fn my_string() -> *const c_char; - /// } - /// - /// unsafe { - /// let slice = CStr::from_ptr(my_string()); - /// println!("string returned: {}", slice.to_str().unwrap()); - /// } - /// # } - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub unsafe fn from_ptr<'a>(ptr: *const c_char) -> &'a CStr { - let len = sys::strlen(ptr); - let ptr = ptr as *const u8; - CStr::from_bytes_with_nul_unchecked(slice::from_raw_parts(ptr, len as usize + 1)) - } - - /// Creates a C string wrapper from a byte slice. - /// - /// This function will cast the provided `bytes` to a `CStr` - /// wrapper after ensuring that the byte slice is nul-terminated - /// and does not contain any interior nul bytes. - /// - /// # Examples - /// - /// ``` - /// use std::ffi::CStr; - /// - /// let cstr = CStr::from_bytes_with_nul(b"hello\0"); - /// assert!(cstr.is_ok()); - /// ``` - /// - /// Creating a `CStr` without a trailing nul terminator is an error: - /// - /// ``` - /// use std::ffi::CStr; - /// - /// let cstr = CStr::from_bytes_with_nul(b"hello"); - /// assert!(cstr.is_err()); - /// ``` - /// - /// Creating a `CStr` with an interior nul byte is an error: - /// - /// ``` - /// use std::ffi::CStr; - /// - /// let cstr = CStr::from_bytes_with_nul(b"he\0llo\0"); - /// assert!(cstr.is_err()); - /// ``` - #[stable(feature = "cstr_from_bytes", since = "1.10.0")] - pub fn from_bytes_with_nul(bytes: &[u8]) -> Result<&CStr, FromBytesWithNulError> { - let nul_pos = memchr::memchr(0, bytes); - if let Some(nul_pos) = nul_pos { - if nul_pos + 1 != bytes.len() { - return Err(FromBytesWithNulError::interior_nul(nul_pos)); - } - Ok(unsafe { CStr::from_bytes_with_nul_unchecked(bytes) }) - } else { - Err(FromBytesWithNulError::not_nul_terminated()) - } - } - - /// Unsafely creates a C string wrapper from a byte slice. - /// - /// This function will cast the provided `bytes` to a `CStr` wrapper without - /// performing any sanity checks. The provided slice **must** be nul-terminated - /// and not contain any interior nul bytes. - /// - /// # Examples - /// - /// ``` - /// use std::ffi::{CStr, CString}; - /// - /// unsafe { - /// let cstring = CString::new("hello").expect("CString::new failed"); - /// let cstr = CStr::from_bytes_with_nul_unchecked(cstring.to_bytes_with_nul()); - /// assert_eq!(cstr, &*cstring); - /// } - /// ``` - #[inline] - #[stable(feature = "cstr_from_bytes", since = "1.10.0")] - #[rustc_const_unstable(feature = "const_cstr_unchecked", issue = "none")] - pub const unsafe fn from_bytes_with_nul_unchecked(bytes: &[u8]) -> &CStr { - &*(bytes as *const [u8] as *const CStr) - } - - /// Returns the inner pointer to this C string. - /// - /// The returned pointer will be valid for as long as `self` is, and points - /// to a contiguous region of memory terminated with a 0 byte to represent - /// the end of the string. - /// - /// **WARNING** - /// - /// The returned pointer is read-only; writing to it (including passing it - /// to C code that writes to it) causes undefined behavior. - /// - /// It is your responsibility to make sure that the underlying memory is not - /// freed too early. For example, the following code will cause undefined - /// behavior when `ptr` is used inside the `unsafe` block: - /// - /// ```no_run - /// # #![allow(unused_must_use)] - /// use std::ffi::CString; - /// - /// let ptr = CString::new("Hello").expect("CString::new failed").as_ptr(); - /// unsafe { - /// // `ptr` is dangling - /// *ptr; - /// } - /// ``` - /// - /// This happens because the pointer returned by `as_ptr` does not carry any - /// lifetime information and the [`CString`] is deallocated immediately after - /// the `CString::new("Hello").expect("CString::new failed").as_ptr()` expression is evaluated. - /// To fix the problem, bind the `CString` to a local variable: - /// - /// ```no_run - /// # #![allow(unused_must_use)] - /// use std::ffi::CString; - /// - /// let hello = CString::new("Hello").expect("CString::new failed"); - /// let ptr = hello.as_ptr(); - /// unsafe { - /// // `ptr` is valid because `hello` is in scope - /// *ptr; - /// } - /// ``` - /// - /// This way, the lifetime of the `CString` in `hello` encompasses - /// the lifetime of `ptr` and the `unsafe` block. - /// - /// [`CString`]: struct.CString.html - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_str_as_ptr", since = "1.32.0")] - pub const fn as_ptr(&self) -> *const c_char { - self.inner.as_ptr() - } - - /// Converts this C string to a byte slice. - /// - /// The returned slice will **not** contain the trailing nul terminator that this C - /// string has. - /// - /// > **Note**: This method is currently implemented as a constant-time - /// > cast, but it is planned to alter its definition in the future to - /// > perform the length calculation whenever this method is called. - /// - /// # Examples - /// - /// ``` - /// use std::ffi::CStr; - /// - /// let cstr = CStr::from_bytes_with_nul(b"foo\0").expect("CStr::from_bytes_with_nul failed"); - /// assert_eq!(cstr.to_bytes(), b"foo"); - /// ``` - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn to_bytes(&self) -> &[u8] { - let bytes = self.to_bytes_with_nul(); - &bytes[..bytes.len() - 1] - } - - /// Converts this C string to a byte slice containing the trailing 0 byte. - /// - /// This function is the equivalent of [`to_bytes`] except that it will retain - /// the trailing nul terminator instead of chopping it off. - /// - /// > **Note**: This method is currently implemented as a 0-cost cast, but - /// > it is planned to alter its definition in the future to perform the - /// > length calculation whenever this method is called. - /// - /// [`to_bytes`]: #method.to_bytes - /// - /// # Examples - /// - /// ``` - /// use std::ffi::CStr; - /// - /// let cstr = CStr::from_bytes_with_nul(b"foo\0").expect("CStr::from_bytes_with_nul failed"); - /// assert_eq!(cstr.to_bytes_with_nul(), b"foo\0"); - /// ``` - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn to_bytes_with_nul(&self) -> &[u8] { - unsafe { &*(&self.inner as *const [c_char] as *const [u8]) } - } - - /// Yields a [`&str`] slice if the `CStr` contains valid UTF-8. - /// - /// If the contents of the `CStr` are valid UTF-8 data, this - /// function will return the corresponding [`&str`] slice. Otherwise, - /// it will return an error with details of where UTF-8 validation failed. - /// - /// [`&str`]: ../primitive.str.html - /// - /// # Examples - /// - /// ``` - /// use std::ffi::CStr; - /// - /// let cstr = CStr::from_bytes_with_nul(b"foo\0").expect("CStr::from_bytes_with_nul failed"); - /// assert_eq!(cstr.to_str(), Ok("foo")); - /// ``` - #[stable(feature = "cstr_to_str", since = "1.4.0")] - pub fn to_str(&self) -> Result<&str, str::Utf8Error> { - // N.B., when `CStr` is changed to perform the length check in `.to_bytes()` - // instead of in `from_ptr()`, it may be worth considering if this should - // be rewritten to do the UTF-8 check inline with the length calculation - // instead of doing it afterwards. - str::from_utf8(self.to_bytes()) - } - - /// Converts a `CStr` into a [`Cow`]`<`[`str`]`>`. - /// - /// If the contents of the `CStr` are valid UTF-8 data, this - /// function will return a [`Cow`]`::`[`Borrowed`]`(`[`&str`]`)` - /// with the corresponding [`&str`] slice. Otherwise, it will - /// replace any invalid UTF-8 sequences with - /// [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD] and return a - /// [`Cow`]`::`[`Owned`]`(`[`String`]`)` with the result. - /// - /// [`Cow`]: ../borrow/enum.Cow.html - /// [`Borrowed`]: ../borrow/enum.Cow.html#variant.Borrowed - /// [`Owned`]: ../borrow/enum.Cow.html#variant.Owned - /// [`str`]: ../primitive.str.html - /// [`String`]: ../string/struct.String.html - /// [U+FFFD]: ../char/constant.REPLACEMENT_CHARACTER.html - /// - /// # Examples - /// - /// Calling `to_string_lossy` on a `CStr` containing valid UTF-8: - /// - /// ``` - /// use std::borrow::Cow; - /// use std::ffi::CStr; - /// - /// let cstr = CStr::from_bytes_with_nul(b"Hello World\0") - /// .expect("CStr::from_bytes_with_nul failed"); - /// assert_eq!(cstr.to_string_lossy(), Cow::Borrowed("Hello World")); - /// ``` - /// - /// Calling `to_string_lossy` on a `CStr` containing invalid UTF-8: - /// - /// ``` - /// use std::borrow::Cow; - /// use std::ffi::CStr; - /// - /// let cstr = CStr::from_bytes_with_nul(b"Hello \xF0\x90\x80World\0") - /// .expect("CStr::from_bytes_with_nul failed"); - /// assert_eq!( - /// cstr.to_string_lossy(), - /// Cow::Owned(String::from("Hello �World")) as Cow<'_, str> - /// ); - /// ``` - #[stable(feature = "cstr_to_str", since = "1.4.0")] - pub fn to_string_lossy(&self) -> Cow<'_, str> { - String::from_utf8_lossy(self.to_bytes()) - } - - /// Converts a [`Box`]`` into a [`CString`] without copying or allocating. - /// - /// [`Box`]: ../boxed/struct.Box.html - /// [`CString`]: struct.CString.html - /// - /// # Examples - /// - /// ``` - /// use std::ffi::CString; - /// - /// let c_string = CString::new(b"foo".to_vec()).expect("CString::new failed"); - /// let boxed = c_string.into_boxed_c_str(); - /// assert_eq!(boxed.into_c_string(), CString::new("foo").expect("CString::new failed")); - /// ``` - #[stable(feature = "into_boxed_c_str", since = "1.20.0")] - pub fn into_c_string(self: Box) -> CString { - let raw = Box::into_raw(self) as *mut [u8]; - CString { inner: unsafe { Box::from_raw(raw) } } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl PartialEq for CStr { - fn eq(&self, other: &CStr) -> bool { - self.to_bytes().eq(other.to_bytes()) - } -} -#[stable(feature = "rust1", since = "1.0.0")] -impl Eq for CStr {} -#[stable(feature = "rust1", since = "1.0.0")] -impl PartialOrd for CStr { - fn partial_cmp(&self, other: &CStr) -> Option { - self.to_bytes().partial_cmp(&other.to_bytes()) - } -} -#[stable(feature = "rust1", since = "1.0.0")] -impl Ord for CStr { - fn cmp(&self, other: &CStr) -> Ordering { - self.to_bytes().cmp(&other.to_bytes()) - } -} - -#[stable(feature = "cstr_borrow", since = "1.3.0")] -impl ToOwned for CStr { - type Owned = CString; - - fn to_owned(&self) -> CString { - CString { inner: self.to_bytes_with_nul().into() } - } -} - -#[stable(feature = "cstring_asref", since = "1.7.0")] -impl From<&CStr> for CString { - fn from(s: &CStr) -> CString { - s.to_owned() - } -} - -#[stable(feature = "cstring_asref", since = "1.7.0")] -impl ops::Index for CString { - type Output = CStr; - - #[inline] - fn index(&self, _index: ops::RangeFull) -> &CStr { - self - } -} - -#[stable(feature = "cstring_asref", since = "1.7.0")] -impl AsRef for CStr { - #[inline] - fn as_ref(&self) -> &CStr { - self - } -} - -#[stable(feature = "cstring_asref", since = "1.7.0")] -impl AsRef for CString { - #[inline] - fn as_ref(&self) -> &CStr { - self - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::borrow::Cow::{Borrowed, Owned}; - use crate::collections::hash_map::DefaultHasher; - use crate::hash::{Hash, Hasher}; - use crate::os::raw::c_char; - use crate::rc::Rc; - use crate::sync::Arc; - - #[test] - fn c_to_rust() { - let data = b"123\0"; - let ptr = data.as_ptr() as *const c_char; - unsafe { - assert_eq!(CStr::from_ptr(ptr).to_bytes(), b"123"); - assert_eq!(CStr::from_ptr(ptr).to_bytes_with_nul(), b"123\0"); - } - } - - #[test] - fn simple() { - let s = CString::new("1234").unwrap(); - assert_eq!(s.as_bytes(), b"1234"); - assert_eq!(s.as_bytes_with_nul(), b"1234\0"); - } - - #[test] - fn build_with_zero1() { - assert!(CString::new(&b"\0"[..]).is_err()); - } - #[test] - fn build_with_zero2() { - assert!(CString::new(vec![0]).is_err()); - } - - #[test] - fn build_with_zero3() { - unsafe { - let s = CString::from_vec_unchecked(vec![0]); - assert_eq!(s.as_bytes(), b"\0"); - } - } - - #[test] - fn formatted() { - let s = CString::new(&b"abc\x01\x02\n\xE2\x80\xA6\xFF"[..]).unwrap(); - assert_eq!(format!("{:?}", s), r#""abc\x01\x02\n\xe2\x80\xa6\xff""#); - } - - #[test] - fn borrowed() { - unsafe { - let s = CStr::from_ptr(b"12\0".as_ptr() as *const _); - assert_eq!(s.to_bytes(), b"12"); - assert_eq!(s.to_bytes_with_nul(), b"12\0"); - } - } - - #[test] - fn to_str() { - let data = b"123\xE2\x80\xA6\0"; - let ptr = data.as_ptr() as *const c_char; - unsafe { - assert_eq!(CStr::from_ptr(ptr).to_str(), Ok("123…")); - assert_eq!(CStr::from_ptr(ptr).to_string_lossy(), Borrowed("123…")); - } - let data = b"123\xE2\0"; - let ptr = data.as_ptr() as *const c_char; - unsafe { - assert!(CStr::from_ptr(ptr).to_str().is_err()); - assert_eq!(CStr::from_ptr(ptr).to_string_lossy(), Owned::(format!("123\u{FFFD}"))); - } - } - - #[test] - fn to_owned() { - let data = b"123\0"; - let ptr = data.as_ptr() as *const c_char; - - let owned = unsafe { CStr::from_ptr(ptr).to_owned() }; - assert_eq!(owned.as_bytes_with_nul(), data); - } - - #[test] - fn equal_hash() { - let data = b"123\xE2\xFA\xA6\0"; - let ptr = data.as_ptr() as *const c_char; - let cstr: &'static CStr = unsafe { CStr::from_ptr(ptr) }; - - let mut s = DefaultHasher::new(); - cstr.hash(&mut s); - let cstr_hash = s.finish(); - let mut s = DefaultHasher::new(); - CString::new(&data[..data.len() - 1]).unwrap().hash(&mut s); - let cstring_hash = s.finish(); - - assert_eq!(cstr_hash, cstring_hash); - } - - #[test] - fn from_bytes_with_nul() { - let data = b"123\0"; - let cstr = CStr::from_bytes_with_nul(data); - assert_eq!(cstr.map(CStr::to_bytes), Ok(&b"123"[..])); - let cstr = CStr::from_bytes_with_nul(data); - assert_eq!(cstr.map(CStr::to_bytes_with_nul), Ok(&b"123\0"[..])); - - unsafe { - let cstr = CStr::from_bytes_with_nul(data); - let cstr_unchecked = CStr::from_bytes_with_nul_unchecked(data); - assert_eq!(cstr, Ok(cstr_unchecked)); - } - } - - #[test] - fn from_bytes_with_nul_unterminated() { - let data = b"123"; - let cstr = CStr::from_bytes_with_nul(data); - assert!(cstr.is_err()); - } - - #[test] - fn from_bytes_with_nul_interior() { - let data = b"1\023\0"; - let cstr = CStr::from_bytes_with_nul(data); - assert!(cstr.is_err()); - } - - #[test] - fn into_boxed() { - let orig: &[u8] = b"Hello, world!\0"; - let cstr = CStr::from_bytes_with_nul(orig).unwrap(); - let boxed: Box = Box::from(cstr); - let cstring = cstr.to_owned().into_boxed_c_str().into_c_string(); - assert_eq!(cstr, &*boxed); - assert_eq!(&*boxed, &*cstring); - assert_eq!(&*cstring, cstr); - } - - #[test] - fn boxed_default() { - let boxed = >::default(); - assert_eq!(boxed.to_bytes_with_nul(), &[0]); - } - - #[test] - fn into_rc() { - let orig: &[u8] = b"Hello, world!\0"; - let cstr = CStr::from_bytes_with_nul(orig).unwrap(); - let rc: Rc = Rc::from(cstr); - let arc: Arc = Arc::from(cstr); - - assert_eq!(&*rc, cstr); - assert_eq!(&*arc, cstr); - - let rc2: Rc = Rc::from(cstr.to_owned()); - let arc2: Arc = Arc::from(cstr.to_owned()); - - assert_eq!(&*rc2, cstr); - assert_eq!(&*arc2, cstr); - } - - #[test] - fn cstr_const_constructor() { - const CSTR: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"Hello, world!\0") }; - - assert_eq!(CSTR.to_str().unwrap(), "Hello, world!"); - } -} diff --git a/crux-mir/lib/std/src/ffi/mod.rs b/crux-mir/lib/std/src/ffi/mod.rs index 72f7367c9..d987bf69b 100644 --- a/crux-mir/lib/std/src/ffi/mod.rs +++ b/crux-mir/lib/std/src/ffi/mod.rs @@ -43,8 +43,8 @@ //! terminator, so the buffer length is really `len+1` characters. //! Rust strings don't have a nul terminator; their length is always //! stored and does not need to be calculated. While in Rust -//! accessing a string's length is a O(1) operation (because the -//! length is stored); in C it is an O(length) operation because the +//! accessing a string's length is an *O*(1) operation (because the +//! length is stored); in C it is an *O*(*n*) operation because the //! length needs to be computed by scanning the string for the nul //! terminator. //! @@ -64,15 +64,15 @@ //! string: it is nul-terminated, and has no internal nul characters. //! Rust code can create a [`CString`] out of a normal string (provided //! that the string doesn't have nul characters in the middle), and -//! then use a variety of methods to obtain a raw `*mut `[`u8`] that can +//! then use a variety of methods to obtain a raw \*mut [u8] that can //! then be passed as an argument to functions which use the C //! conventions for strings. //! //! * **From C to Rust:** [`CStr`] represents a borrowed C string; it -//! is what you would use to wrap a raw `*const `[`u8`] that you got from +//! is what you would use to wrap a raw \*const [u8] that you got from //! a C function. A [`CStr`] is guaranteed to be a nul-terminated array //! of bytes. Once you have a [`CStr`], you can convert it to a Rust -//! [`&str`][`str`] if it's valid UTF-8, or lossily convert it by adding +//! &[str] if it's valid UTF-8, or lossily convert it by adding //! replacement characters. //! //! [`OsString`] and [`OsStr`] are useful when you need to transfer @@ -81,20 +81,20 @@ //! [`OsStr`] and Rust strings work similarly to those for [`CString`] //! and [`CStr`]. //! -//! * [`OsString`] represents an owned string in whatever -//! representation the operating system prefers. In the Rust standard -//! library, various APIs that transfer strings to/from the operating +//! * [`OsString`] losslessly represents an owned platform string. However, this +//! representation is not necessarily in a form native to the platform. +//! In the Rust standard library, various APIs that transfer strings to/from the operating //! system use [`OsString`] instead of plain strings. For example, //! [`env::var_os()`] is used to query environment variables; it -//! returns an [`Option`]`<`[`OsString`]`>`. If the environment variable -//! exists you will get a [`Some`]`(os_string)`, which you can *then* try to -//! convert to a Rust string. This yields a [`Result<>`], so that +//! returns an [Option]<[OsString]>. If the environment variable +//! exists you will get a [Some]\(os_string), which you can +//! *then* try to convert to a Rust string. This yields a [`Result`], so that //! your code can detect errors in case the environment variable did //! not in fact contain valid Unicode data. //! -//! * [`OsStr`] represents a borrowed reference to a string in a -//! format that can be passed to the operating system. It can be -//! converted into an UTF-8 Rust string slice in a similar way to +//! * [`OsStr`] losslessly represents a borrowed reference to a platform string. +//! However, this representation is not necessarily in a form native to the platform. +//! It can be converted into a UTF-8 Rust string slice in a similar way to //! [`OsString`]. //! //! # Conversions @@ -102,67 +102,64 @@ //! ## On Unix //! //! On Unix, [`OsStr`] implements the -//! `std::os::unix::ffi::`[`OsStrExt`][unix.OsStrExt] trait, which +//! std::os::unix::ffi::[OsStrExt][unix.OsStrExt] trait, which //! augments it with two methods, [`from_bytes`] and [`as_bytes`]. -//! These do inexpensive conversions from and to UTF-8 byte slices. +//! These do inexpensive conversions from and to byte slices. //! //! Additionally, on Unix [`OsString`] implements the -//! `std::os::unix::ffi::`[`OsStringExt`][unix.OsStringExt] trait, +//! std::os::unix::ffi::[OsStringExt][unix.OsStringExt] trait, //! which provides [`from_vec`] and [`into_vec`] methods that consume //! their arguments, and take or produce vectors of [`u8`]. //! //! ## On Windows //! +//! An [`OsStr`] can be losslessly converted to a native Windows string. And +//! a native Windows string can be losslessly converted to an [`OsString`]. +//! //! On Windows, [`OsStr`] implements the -//! `std::os::windows::ffi::`[`OsStrExt`][windows.OsStrExt] trait, +//! std::os::windows::ffi::[OsStrExt][windows.OsStrExt] trait, //! which provides an [`encode_wide`] method. This provides an -//! iterator that can be [`collect`]ed into a vector of [`u16`]. +//! iterator that can be [`collect`]ed into a vector of [`u16`]. After a nul +//! characters is appended, this is the same as a native Windows string. //! //! Additionally, on Windows [`OsString`] implements the -//! `std::os::windows:ffi::`[`OsStringExt`][windows.OsStringExt] -//! trait, which provides a [`from_wide`] method. The result of this -//! method is an [`OsString`] which can be round-tripped to a Windows -//! string losslessly. -//! -//! [`String`]: ../string/struct.String.html -//! [`str`]: ../primitive.str.html -//! [`char`]: ../primitive.char.html -//! [`u8`]: ../primitive.u8.html -//! [`u16`]: ../primitive.u16.html -//! [Unicode scalar value]: http://www.unicode.org/glossary/#unicode_scalar_value -//! [Unicode code point]: http://www.unicode.org/glossary/#code_point -//! [`CString`]: struct.CString.html -//! [`CStr`]: struct.CStr.html -//! [`OsString`]: struct.OsString.html -//! [`OsStr`]: struct.OsStr.html -//! [`env::set_var()`]: ../env/fn.set_var.html -//! [`env::var_os()`]: ../env/fn.var_os.html -//! [`Result<>`]: ../result/enum.Result.html -//! [unix.OsStringExt]: ../os/unix/ffi/trait.OsStringExt.html -//! [`from_vec`]: ../os/unix/ffi/trait.OsStringExt.html#tymethod.from_vec -//! [`into_vec`]: ../os/unix/ffi/trait.OsStringExt.html#tymethod.into_vec -//! [unix.OsStrExt]: ../os/unix/ffi/trait.OsStrExt.html -//! [`from_bytes`]: ../os/unix/ffi/trait.OsStrExt.html#tymethod.from_bytes -//! [`as_bytes`]: ../os/unix/ffi/trait.OsStrExt.html#tymethod.as_bytes -//! [`OsStrExt`]: ../os/unix/ffi/trait.OsStrExt.html -//! [windows.OsStrExt]: ../os/windows/ffi/trait.OsStrExt.html -//! [`encode_wide`]: ../os/windows/ffi/trait.OsStrExt.html#tymethod.encode_wide -//! [`collect`]: ../iter/trait.Iterator.html#method.collect -//! [windows.OsStringExt]: ../os/windows/ffi/trait.OsStringExt.html -//! [`from_wide`]: ../os/windows/ffi/trait.OsStringExt.html#tymethod.from_wide -//! [`Option`]: ../option/enum.Option.html -//! [`Some`]: ../option/enum.Option.html#variant.Some +//! std::os::windows:ffi::[OsStringExt][windows.OsStringExt] +//! trait, which provides a [`from_wide`] method to convert a native Windows +//! string (without the terminating nul character) to an [`OsString`]. +//! +//! [Unicode scalar value]: https://www.unicode.org/glossary/#unicode_scalar_value +//! [Unicode code point]: https://www.unicode.org/glossary/#code_point +//! [`env::set_var()`]: crate::env::set_var "env::set_var" +//! [`env::var_os()`]: crate::env::var_os "env::var_os" +//! [unix.OsStringExt]: crate::os::unix::ffi::OsStringExt "os::unix::ffi::OsStringExt" +//! [`from_vec`]: crate::os::unix::ffi::OsStringExt::from_vec "os::unix::ffi::OsStringExt::from_vec" +//! [`into_vec`]: crate::os::unix::ffi::OsStringExt::into_vec "os::unix::ffi::OsStringExt::into_vec" +//! [unix.OsStrExt]: crate::os::unix::ffi::OsStrExt "os::unix::ffi::OsStrExt" +//! [`from_bytes`]: crate::os::unix::ffi::OsStrExt::from_bytes "os::unix::ffi::OsStrExt::from_bytes" +//! [`as_bytes`]: crate::os::unix::ffi::OsStrExt::as_bytes "os::unix::ffi::OsStrExt::as_bytes" +//! [`OsStrExt`]: crate::os::unix::ffi::OsStrExt "os::unix::ffi::OsStrExt" +//! [windows.OsStrExt]: crate::os::windows::ffi::OsStrExt "os::windows::ffi::OsStrExt" +//! [`encode_wide`]: crate::os::windows::ffi::OsStrExt::encode_wide "os::windows::ffi::OsStrExt::encode_wide" +//! [`collect`]: crate::iter::Iterator::collect "iter::Iterator::collect" +//! [windows.OsStringExt]: crate::os::windows::ffi::OsStringExt "os::windows::ffi::OsStringExt" +//! [`from_wide`]: crate::os::windows::ffi::OsStringExt::from_wide "os::windows::ffi::OsStringExt::from_wide" #![stable(feature = "rust1", since = "1.0.0")] -#[stable(feature = "cstr_from_bytes", since = "1.10.0")] -pub use self::c_str::FromBytesWithNulError; -#[stable(feature = "rust1", since = "1.0.0")] -pub use self::c_str::{CStr, CString, IntoStringError, NulError}; +#[stable(feature = "alloc_c_string", since = "1.64.0")] +pub use alloc::ffi::{CString, FromVecWithNulError, IntoStringError, NulError}; +#[stable(feature = "core_c_str", since = "1.64.0")] +pub use core::ffi::{CStr, FromBytesWithNulError}; #[stable(feature = "rust1", since = "1.0.0")] pub use self::os_str::{OsStr, OsString}; +#[stable(feature = "core_ffi_c", since = "1.64.0")] +pub use core::ffi::{ + c_char, c_double, c_float, c_int, c_long, c_longlong, c_schar, c_short, c_uchar, c_uint, + c_ulong, c_ulonglong, c_ushort, +}; + #[stable(feature = "core_c_void", since = "1.30.0")] pub use core::ffi::c_void; @@ -174,5 +171,4 @@ pub use core::ffi::c_void; )] pub use core::ffi::{VaList, VaListImpl}; -mod c_str; mod os_str; diff --git a/crux-mir/lib/std/src/ffi/os_str.rs b/crux-mir/lib/std/src/ffi/os_str.rs index 77da97219..80ed34157 100644 --- a/crux-mir/lib/std/src/ffi/os_str.rs +++ b/crux-mir/lib/std/src/ffi/os_str.rs @@ -1,9 +1,15 @@ +#[cfg(test)] +mod tests; + use crate::borrow::{Borrow, Cow}; use crate::cmp; +use crate::collections::TryReserveError; use crate::fmt; use crate::hash::{Hash, Hasher}; +use crate::iter::Extend; use crate::ops; use crate::rc::Rc; +use crate::str::FromStr; use crate::sync::Arc; use crate::sys::os_str::{Buf, Slice}; @@ -28,7 +34,7 @@ use crate::sys_common::{AsInner, FromInner, IntoInner}; /// of this is that `OsString` instances are *not* `NUL` terminated; in order /// to pass to e.g., Unix system call, you should create a [`CStr`]. /// -/// `OsString` is to [`&OsStr`] as [`String`] is to [`&str`]: the former +/// `OsString` is to &[OsStr] as [`String`] is to &[str]: the former /// in each pair are owned strings; the latter are borrowed /// references. /// @@ -39,21 +45,37 @@ use crate::sys_common::{AsInner, FromInner, IntoInner}; /// values, encoded in a less-strict variant of UTF-8. This is useful to /// understand when handling capacity and length values. /// +/// # Capacity of `OsString` +/// +/// Capacity uses units of UTF-8 bytes for OS strings which were created from valid unicode, and +/// uses units of bytes in an unspecified encoding for other contents. On a given target, all +/// `OsString` and `OsStr` values use the same units for capacity, so the following will work: +/// ``` +/// use std::ffi::{OsStr, OsString}; +/// +/// fn concat_os_strings(a: &OsStr, b: &OsStr) -> OsString { +/// let mut ret = OsString::with_capacity(a.len() + b.len()); // This will allocate +/// ret.push(a); // This will not allocate further +/// ret.push(b); // This will not allocate further +/// ret +/// } +/// ``` +/// /// # Creating an `OsString` /// /// **From a Rust string**: `OsString` implements -/// [`From`]`<`[`String`]`>`, so you can use `my_string.from` to +/// [From]<[String]>, so you can use my_string.[into]\() to /// create an `OsString` from a normal Rust string. /// /// **From slices:** Just like you can start with an empty Rust -/// [`String`] and then [`push_str`][String.push_str] `&str` +/// [`String`] and then [`String::push_str`] some &[str] /// sub-string slices into it, you can create an empty `OsString` with -/// the [`new`] method and then push string slices into it with the -/// [`push`] method. +/// the [`OsString::new`] method and then push string slices into it with the +/// [`OsString::push`] method. /// /// # Extracting a borrowed reference to the whole OS string /// -/// You can use the [`as_os_str`] method to get an `&`[`OsStr`] from +/// You can use the [`OsString::as_os_str`] method to get an &[OsStr] from /// an `OsString`; this is effectively a borrowed reference to the /// whole string. /// @@ -62,51 +84,47 @@ use crate::sys_common::{AsInner, FromInner, IntoInner}; /// See the [module's toplevel documentation about conversions][conversions] for a discussion on /// the traits which `OsString` implements for [conversions] from/to native representations. /// -/// [`OsStr`]: struct.OsStr.html -/// [`&OsStr`]: struct.OsStr.html -/// [`CStr`]: struct.CStr.html -/// [`From`]: ../convert/trait.From.html -/// [`String`]: ../string/struct.String.html -/// [`&str`]: ../primitive.str.html -/// [`u8`]: ../primitive.u8.html -/// [`u16`]: ../primitive.u16.html -/// [String.push_str]: ../string/struct.String.html#method.push_str -/// [`new`]: #method.new -/// [`push`]: #method.push -/// [`as_os_str`]: #method.as_os_str -/// [conversions]: index.html#conversions -#[derive(Clone)] +/// [`CStr`]: crate::ffi::CStr +/// [conversions]: super#conversions +/// [into]: Into::into +#[cfg_attr(not(test), rustc_diagnostic_item = "OsString")] #[stable(feature = "rust1", since = "1.0.0")] pub struct OsString { inner: Buf, } +/// Allows extension traits within `std`. +#[unstable(feature = "sealed", issue = "none")] +impl crate::sealed::Sealed for OsString {} + /// Borrowed reference to an OS string (see [`OsString`]). /// /// This type represents a borrowed reference to a string in the operating system's preferred /// representation. /// -/// `&OsStr` is to [`OsString`] as [`&str`] is to [`String`]: the former in each pair are borrowed -/// references; the latter are owned strings. +/// `&OsStr` is to [`OsString`] as &[str] is to [`String`]: the +/// former in each pair are borrowed references; the latter are owned strings. /// /// See the [module's toplevel documentation about conversions][conversions] for a discussion on /// the traits which `OsStr` implements for [conversions] from/to native representations. /// -/// [`OsString`]: struct.OsString.html -/// [`&str`]: ../primitive.str.html -/// [`String`]: ../string/struct.String.html -/// [conversions]: index.html#conversions +/// [conversions]: super#conversions +#[cfg_attr(not(test), rustc_diagnostic_item = "OsStr")] #[stable(feature = "rust1", since = "1.0.0")] // FIXME: // `OsStr::from_inner` current implementation relies // on `OsStr` being layout-compatible with `Slice`. // When attribute privacy is implemented, `OsStr` should be annotated as `#[repr(transparent)]`. -// Anyway, `OsStr` representation and layout are considered implementation detail, are +// Anyway, `OsStr` representation and layout are considered implementation details, are // not documented and must not be relied upon. pub struct OsStr { inner: Slice, } +/// Allows extension traits within `std`. +#[unstable(feature = "sealed", issue = "none")] +impl crate::sealed::Sealed for OsStr {} + impl OsString { /// Constructs a new empty `OsString`. /// @@ -118,14 +136,14 @@ impl OsString { /// let os_string = OsString::new(); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + #[inline] pub fn new() -> OsString { OsString { inner: Buf::from_string(String::new()) } } /// Converts to an [`OsStr`] slice. /// - /// [`OsStr`]: struct.OsStr.html - /// /// # Examples /// /// ``` @@ -136,6 +154,8 @@ impl OsString { /// assert_eq!(os_string.as_os_str(), os_str); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + #[inline] pub fn as_os_str(&self) -> &OsStr { self } @@ -144,8 +164,6 @@ impl OsString { /// /// On failure, ownership of the original `OsString` is returned. /// - /// [`String`]: ../../std/string/struct.String.html - /// /// # Examples /// /// ``` @@ -156,13 +174,12 @@ impl OsString { /// assert_eq!(string, Ok(String::from("foo"))); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[inline] pub fn into_string(self) -> Result { self.inner.into_string().map_err(|buf| OsString { inner: buf }) } - /// Extends the string with the given [`&OsStr`] slice. - /// - /// [`&OsStr`]: struct.OsStr.html + /// Extends the string with the given &[OsStr] slice. /// /// # Examples /// @@ -174,17 +191,19 @@ impl OsString { /// assert_eq!(&os_string, "foobar"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[inline] pub fn push>(&mut self, s: T) { self.inner.push_slice(&s.as_ref().inner) } - /// Creates a new `OsString` with the given capacity. + /// Creates a new `OsString` with at least the given capacity. /// - /// The string will be able to hold exactly `capacity` length units of other - /// OS strings without reallocating. If `capacity` is 0, the string will not + /// The string will be able to hold at least `capacity` length units of other + /// OS strings without reallocating. This method is allowed to allocate for + /// more units than `capacity`. If `capacity` is 0, the string will not /// allocate. /// - /// See main `OsString` documentation information about encoding. + /// See the main `OsString` documentation information about encoding and capacity units. /// /// # Examples /// @@ -200,6 +219,8 @@ impl OsString { /// assert_eq!(capacity, os_string.capacity()); /// ``` #[stable(feature = "osstring_simple_functions", since = "1.9.0")] + #[must_use] + #[inline] pub fn with_capacity(capacity: usize) -> OsString { OsString { inner: Buf::with_capacity(capacity) } } @@ -218,13 +239,14 @@ impl OsString { /// assert_eq!(&os_string, ""); /// ``` #[stable(feature = "osstring_simple_functions", since = "1.9.0")] + #[inline] pub fn clear(&mut self) { self.inner.clear() } /// Returns the capacity this `OsString` can hold without reallocating. /// - /// See `OsString` introduction for information about encoding. + /// See the main `OsString` documentation information about encoding and capacity units. /// /// # Examples /// @@ -235,14 +257,19 @@ impl OsString { /// assert!(os_string.capacity() >= 10); /// ``` #[stable(feature = "osstring_simple_functions", since = "1.9.0")] + #[must_use] + #[inline] pub fn capacity(&self) -> usize { self.inner.capacity() } /// Reserves capacity for at least `additional` more capacity to be inserted - /// in the given `OsString`. + /// in the given `OsString`. Does nothing if the capacity is + /// already sufficient. /// - /// The collection may reserve more space to avoid frequent reallocations. + /// The collection may reserve more space to speculatively avoid frequent reallocations. + /// + /// See the main `OsString` documentation information about encoding and capacity units. /// /// # Examples /// @@ -254,17 +281,61 @@ impl OsString { /// assert!(s.capacity() >= 10); /// ``` #[stable(feature = "osstring_simple_functions", since = "1.9.0")] + #[inline] pub fn reserve(&mut self, additional: usize) { self.inner.reserve(additional) } - /// Reserves the minimum capacity for exactly `additional` more capacity to + /// Tries to reserve capacity for at least `additional` more length units + /// in the given `OsString`. The string may reserve more space to speculatively avoid + /// frequent reallocations. After calling `try_reserve`, capacity will be + /// greater than or equal to `self.len() + additional` if it returns `Ok(())`. + /// Does nothing if capacity is already sufficient. This method preserves + /// the contents even if an error occurs. + /// + /// See the main `OsString` documentation information about encoding and capacity units. + /// + /// # Errors + /// + /// If the capacity overflows, or the allocator reports a failure, then an error + /// is returned. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::{OsStr, OsString}; + /// use std::collections::TryReserveError; + /// + /// fn process_data(data: &str) -> Result { + /// let mut s = OsString::new(); + /// + /// // Pre-reserve the memory, exiting if we can't + /// s.try_reserve(OsStr::new(data).len())?; + /// + /// // Now we know this can't OOM in the middle of our complex work + /// s.push(data); + /// + /// Ok(s) + /// } + /// # process_data("123").expect("why is the test harness OOMing on 3 bytes?"); + /// ``` + #[stable(feature = "try_reserve_2", since = "1.63.0")] + #[inline] + pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.inner.try_reserve(additional) + } + + /// Reserves the minimum capacity for at least `additional` more capacity to /// be inserted in the given `OsString`. Does nothing if the capacity is /// already sufficient. /// /// Note that the allocator may give the collection more space than it /// requests. Therefore, capacity can not be relied upon to be precisely - /// minimal. Prefer reserve if future insertions are expected. + /// minimal. Prefer [`reserve`] if future insertions are expected. + /// + /// [`reserve`]: OsString::reserve + /// + /// See the main `OsString` documentation information about encoding and capacity units. /// /// # Examples /// @@ -276,12 +347,59 @@ impl OsString { /// assert!(s.capacity() >= 10); /// ``` #[stable(feature = "osstring_simple_functions", since = "1.9.0")] + #[inline] pub fn reserve_exact(&mut self, additional: usize) { self.inner.reserve_exact(additional) } + /// Tries to reserve the minimum capacity for at least `additional` + /// more length units in the given `OsString`. After calling + /// `try_reserve_exact`, capacity will be greater than or equal to + /// `self.len() + additional` if it returns `Ok(())`. + /// Does nothing if the capacity is already sufficient. + /// + /// Note that the allocator may give the `OsString` more space than it + /// requests. Therefore, capacity can not be relied upon to be precisely + /// minimal. Prefer [`try_reserve`] if future insertions are expected. + /// + /// [`try_reserve`]: OsString::try_reserve + /// + /// See the main `OsString` documentation information about encoding and capacity units. + /// + /// # Errors + /// + /// If the capacity overflows, or the allocator reports a failure, then an error + /// is returned. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::{OsStr, OsString}; + /// use std::collections::TryReserveError; + /// + /// fn process_data(data: &str) -> Result { + /// let mut s = OsString::new(); + /// + /// // Pre-reserve the memory, exiting if we can't + /// s.try_reserve_exact(OsStr::new(data).len())?; + /// + /// // Now we know this can't OOM in the middle of our complex work + /// s.push(data); + /// + /// Ok(s) + /// } + /// # process_data("123").expect("why is the test harness OOMing on 3 bytes?"); + /// ``` + #[stable(feature = "try_reserve_2", since = "1.63.0")] + #[inline] + pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.inner.try_reserve_exact(additional) + } + /// Shrinks the capacity of the `OsString` to match its length. /// + /// See the main `OsString` documentation information about encoding and capacity units. + /// /// # Examples /// /// ``` @@ -296,6 +414,7 @@ impl OsString { /// assert_eq!(3, s.capacity()); /// ``` #[stable(feature = "osstring_shrink_to_fit", since = "1.19.0")] + #[inline] pub fn shrink_to_fit(&mut self) { self.inner.shrink_to_fit() } @@ -305,13 +424,13 @@ impl OsString { /// The capacity will remain at least as large as both the length /// and the supplied value. /// - /// Panics if the current capacity is smaller than the supplied - /// minimum capacity. + /// If the current capacity is less than the lower limit, this is a no-op. + /// + /// See the main `OsString` documentation information about encoding and capacity units. /// /// # Examples /// /// ``` - /// #![feature(shrink_to)] /// use std::ffi::OsString; /// /// let mut s = OsString::from("foo"); @@ -325,15 +444,13 @@ impl OsString { /// assert!(s.capacity() >= 3); /// ``` #[inline] - #[unstable(feature = "shrink_to", reason = "new API", issue = "56431")] + #[stable(feature = "shrink_to", since = "1.56.0")] pub fn shrink_to(&mut self, min_capacity: usize) { self.inner.shrink_to(min_capacity) } /// Converts this `OsString` into a boxed [`OsStr`]. /// - /// [`OsStr`]: struct.OsStr.html - /// /// # Examples /// /// ``` @@ -343,6 +460,7 @@ impl OsString { /// /// let b: Box = s.into_boxed_os_str(); /// ``` + #[must_use = "`self` will be dropped if the result is not used"] #[stable(feature = "into_boxed_os_str", since = "1.20.0")] pub fn into_boxed_os_str(self) -> Box { let rw = Box::into_raw(self.inner.into_box()) as *mut OsStr; @@ -352,11 +470,10 @@ impl OsString { #[stable(feature = "rust1", since = "1.0.0")] impl From for OsString { - /// Converts a [`String`] into a [`OsString`]. - /// - /// The conversion copies the data, and includes an allocation on the heap. + /// Converts a [`String`] into an [`OsString`]. /// - /// [`OsString`]: ../../std/ffi/struct.OsString.html + /// This conversion does not allocate or copy memory. + #[inline] fn from(s: String) -> OsString { OsString { inner: Buf::from_string(s) } } @@ -364,6 +481,8 @@ impl From for OsString { #[stable(feature = "rust1", since = "1.0.0")] impl> From<&T> for OsString { + /// Copies any value implementing [AsRef]<[OsStr]> + /// into a newly allocated [`OsString`]. fn from(s: &T) -> OsString { s.as_ref().to_os_string() } @@ -379,6 +498,14 @@ impl ops::Index for OsString { } } +#[stable(feature = "mut_osstr", since = "1.44.0")] +impl ops::IndexMut for OsString { + #[inline] + fn index_mut(&mut self, _index: ops::RangeFull) -> &mut OsStr { + OsStr::from_inner_mut(self.inner.as_mut_slice()) + } +} + #[stable(feature = "rust1", since = "1.0.0")] impl ops::Deref for OsString { type Target = OsStr; @@ -389,6 +516,14 @@ impl ops::Deref for OsString { } } +#[stable(feature = "mut_osstr", since = "1.44.0")] +impl ops::DerefMut for OsString { + #[inline] + fn deref_mut(&mut self) -> &mut OsStr { + &mut self[..] + } +} + #[stable(feature = "osstring_default", since = "1.9.0")] impl Default for OsString { /// Constructs an empty `OsString`. @@ -398,6 +533,19 @@ impl Default for OsString { } } +#[stable(feature = "rust1", since = "1.0.0")] +impl Clone for OsString { + #[inline] + fn clone(&self) -> Self { + OsString { inner: self.inner.clone() } + } + + #[inline] + fn clone_from(&mut self, source: &Self) { + self.inner.clone_from(&source.inner) + } +} + #[stable(feature = "rust1", since = "1.0.0")] impl fmt::Debug for OsString { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -407,6 +555,7 @@ impl fmt::Debug for OsString { #[stable(feature = "rust1", since = "1.0.0")] impl PartialEq for OsString { + #[inline] fn eq(&self, other: &OsString) -> bool { &**self == &**other } @@ -414,6 +563,7 @@ impl PartialEq for OsString { #[stable(feature = "rust1", since = "1.0.0")] impl PartialEq for OsString { + #[inline] fn eq(&self, other: &str) -> bool { &**self == other } @@ -421,6 +571,7 @@ impl PartialEq for OsString { #[stable(feature = "rust1", since = "1.0.0")] impl PartialEq for str { + #[inline] fn eq(&self, other: &OsString) -> bool { &**other == self } @@ -428,6 +579,7 @@ impl PartialEq for str { #[stable(feature = "os_str_str_ref_eq", since = "1.29.0")] impl PartialEq<&str> for OsString { + #[inline] fn eq(&self, other: &&str) -> bool { **self == **other } @@ -435,6 +587,7 @@ impl PartialEq<&str> for OsString { #[stable(feature = "os_str_str_ref_eq", since = "1.29.0")] impl<'a> PartialEq for &'a str { + #[inline] fn eq(&self, other: &OsString) -> bool { **other == **self } @@ -491,6 +644,14 @@ impl Hash for OsString { } } +#[stable(feature = "os_string_fmt_write", since = "1.64.0")] +impl fmt::Write for OsString { + fn write_str(&mut self, s: &str) -> fmt::Result { + self.push(s); + Ok(()) + } +} + impl OsStr { /// Coerces into an `OsStr` slice. /// @@ -509,15 +670,24 @@ impl OsStr { #[inline] fn from_inner(inner: &Slice) -> &OsStr { + // SAFETY: OsStr is just a wrapper of Slice, + // therefore converting &Slice to &OsStr is safe. unsafe { &*(inner as *const Slice as *const OsStr) } } - /// Yields a [`&str`] slice if the `OsStr` is valid Unicode. + #[inline] + fn from_inner_mut(inner: &mut Slice) -> &mut OsStr { + // SAFETY: OsStr is just a wrapper of Slice, + // therefore converting &mut Slice to &mut OsStr is safe. + // Any method that mutates OsStr must be careful not to + // break platform-specific encoding, in particular Wtf8 on Windows. + unsafe { &mut *(inner as *mut Slice as *mut OsStr) } + } + + /// Yields a &[str] slice if the `OsStr` is valid Unicode. /// /// This conversion may entail doing a check for UTF-8 validity. /// - /// [`&str`]: ../../std/primitive.str.html - /// /// # Examples /// /// ``` @@ -527,18 +697,19 @@ impl OsStr { /// assert_eq!(os_str.to_str(), Some("foo")); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] pub fn to_str(&self) -> Option<&str> { self.inner.to_str() } - /// Converts an `OsStr` to a [`Cow`]`<`[`str`]`>`. + /// Converts an `OsStr` to a [Cow]<[str]>. /// /// Any non-Unicode sequences are replaced with /// [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. /// - /// [`Cow`]: ../../std/borrow/enum.Cow.html - /// [`str`]: ../../std/primitive.str.html - /// [U+FFFD]: ../../std/char/constant.REPLACEMENT_CHARACTER.html + /// [U+FFFD]: crate::char::REPLACEMENT_CHARACTER /// /// # Examples /// @@ -552,7 +723,7 @@ impl OsStr { /// // sequences simply through collecting user command line arguments, for /// // example. /// - /// #[cfg(any(unix, target_os = "redox"))] { + /// #[cfg(unix)] { /// use std::ffi::OsStr; /// use std::os::unix::ffi::OsStrExt; /// @@ -579,14 +750,15 @@ impl OsStr { /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] pub fn to_string_lossy(&self) -> Cow<'_, str> { self.inner.to_string_lossy() } /// Copies the slice into an owned [`OsString`]. /// - /// [`OsString`]: struct.OsString.html - /// /// # Examples /// /// ``` @@ -597,6 +769,9 @@ impl OsStr { /// assert_eq!(os_string, OsString::from("foo")); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] pub fn to_os_string(&self) -> OsString { OsString { inner: self.inner.to_owned() } } @@ -615,6 +790,7 @@ impl OsStr { /// assert!(!os_str.is_empty()); /// ``` #[stable(feature = "osstring_simple_functions", since = "1.9.0")] + #[must_use] #[inline] pub fn is_empty(&self) -> bool { self.inner.inner.is_empty() @@ -634,8 +810,7 @@ impl OsStr { /// This number is simply useful for passing to other methods, like /// [`OsString::with_capacity`] to avoid reallocations. /// - /// [`OsString`]: struct.OsString.html - /// [`OsString::with_capacity`]: struct.OsString.html#method.with_capacity + /// See the main `OsString` documentation information about encoding and capacity units. /// /// # Examples /// @@ -649,15 +824,15 @@ impl OsStr { /// assert_eq!(os_str.len(), 3); /// ``` #[stable(feature = "osstring_simple_functions", since = "1.9.0")] + #[must_use] + #[inline] pub fn len(&self) -> usize { self.inner.inner.len() } - /// Converts a [`Box`]`` into an [`OsString`] without copying or allocating. - /// - /// [`Box`]: ../boxed/struct.Box.html - /// [`OsString`]: struct.OsString.html + /// Converts a [Box]<[OsStr]> into an [`OsString`] without copying or allocating. #[stable(feature = "into_boxed_os_str", since = "1.20.0")] + #[must_use = "`self` will be dropped if the result is not used"] pub fn into_os_string(self: Box) -> OsString { let boxed = unsafe { Box::from_raw(Box::into_raw(self) as *mut Slice) }; OsString { inner: Buf::from_box(boxed) } @@ -665,29 +840,175 @@ impl OsStr { /// Gets the underlying byte representation. /// - /// Note: it is *crucial* that this API is private, to avoid + /// Note: it is *crucial* that this API is not externally public, to avoid /// revealing the internal, platform-specific encodings. #[inline] - fn bytes(&self) -> &[u8] { + pub(crate) fn bytes(&self) -> &[u8] { unsafe { &*(&self.inner as *const _ as *const [u8]) } } + + /// Converts this string to its ASCII lower case equivalent in-place. + /// + /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', + /// but non-ASCII letters are unchanged. + /// + /// To return a new lowercased value without modifying the existing one, use + /// [`OsStr::to_ascii_lowercase`]. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::OsString; + /// + /// let mut s = OsString::from("GRÜßE, JÜRGEN ❤"); + /// + /// s.make_ascii_lowercase(); + /// + /// assert_eq!("grÜße, jÜrgen ❤", s); + /// ``` + #[stable(feature = "osstring_ascii", since = "1.53.0")] + #[inline] + pub fn make_ascii_lowercase(&mut self) { + self.inner.make_ascii_lowercase() + } + + /// Converts this string to its ASCII upper case equivalent in-place. + /// + /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', + /// but non-ASCII letters are unchanged. + /// + /// To return a new uppercased value without modifying the existing one, use + /// [`OsStr::to_ascii_uppercase`]. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::OsString; + /// + /// let mut s = OsString::from("Grüße, Jürgen ❤"); + /// + /// s.make_ascii_uppercase(); + /// + /// assert_eq!("GRüßE, JüRGEN ❤", s); + /// ``` + #[stable(feature = "osstring_ascii", since = "1.53.0")] + #[inline] + pub fn make_ascii_uppercase(&mut self) { + self.inner.make_ascii_uppercase() + } + + /// Returns a copy of this string where each character is mapped to its + /// ASCII lower case equivalent. + /// + /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', + /// but non-ASCII letters are unchanged. + /// + /// To lowercase the value in-place, use [`OsStr::make_ascii_lowercase`]. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::OsString; + /// let s = OsString::from("Grüße, Jürgen ❤"); + /// + /// assert_eq!("grüße, jürgen ❤", s.to_ascii_lowercase()); + /// ``` + #[must_use = "to lowercase the value in-place, use `make_ascii_lowercase`"] + #[stable(feature = "osstring_ascii", since = "1.53.0")] + pub fn to_ascii_lowercase(&self) -> OsString { + OsString::from_inner(self.inner.to_ascii_lowercase()) + } + + /// Returns a copy of this string where each character is mapped to its + /// ASCII upper case equivalent. + /// + /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', + /// but non-ASCII letters are unchanged. + /// + /// To uppercase the value in-place, use [`OsStr::make_ascii_uppercase`]. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::OsString; + /// let s = OsString::from("Grüße, Jürgen ❤"); + /// + /// assert_eq!("GRüßE, JüRGEN ❤", s.to_ascii_uppercase()); + /// ``` + #[must_use = "to uppercase the value in-place, use `make_ascii_uppercase`"] + #[stable(feature = "osstring_ascii", since = "1.53.0")] + pub fn to_ascii_uppercase(&self) -> OsString { + OsString::from_inner(self.inner.to_ascii_uppercase()) + } + + /// Checks if all characters in this string are within the ASCII range. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::OsString; + /// + /// let ascii = OsString::from("hello!\n"); + /// let non_ascii = OsString::from("Grüße, Jürgen ❤"); + /// + /// assert!(ascii.is_ascii()); + /// assert!(!non_ascii.is_ascii()); + /// ``` + #[stable(feature = "osstring_ascii", since = "1.53.0")] + #[must_use] + #[inline] + pub fn is_ascii(&self) -> bool { + self.inner.is_ascii() + } + + /// Checks that two strings are an ASCII case-insensitive match. + /// + /// Same as `to_ascii_lowercase(a) == to_ascii_lowercase(b)`, + /// but without allocating and copying temporaries. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::OsString; + /// + /// assert!(OsString::from("Ferris").eq_ignore_ascii_case("FERRIS")); + /// assert!(OsString::from("Ferrös").eq_ignore_ascii_case("FERRöS")); + /// assert!(!OsString::from("Ferrös").eq_ignore_ascii_case("FERRÖS")); + /// ``` + #[stable(feature = "osstring_ascii", since = "1.53.0")] + pub fn eq_ignore_ascii_case>(&self, other: S) -> bool { + self.inner.eq_ignore_ascii_case(&other.as_ref().inner) + } } #[stable(feature = "box_from_os_str", since = "1.17.0")] impl From<&OsStr> for Box { + /// Copies the string into a newly allocated [Box]<[OsStr]>. + #[inline] fn from(s: &OsStr) -> Box { let rw = Box::into_raw(s.inner.into_box()) as *mut OsStr; unsafe { Box::from_raw(rw) } } } +#[stable(feature = "box_from_cow", since = "1.45.0")] +impl From> for Box { + /// Converts a `Cow<'a, OsStr>` into a [Box]<[OsStr]>, + /// by copying the contents if they are borrowed. + #[inline] + fn from(cow: Cow<'_, OsStr>) -> Box { + match cow { + Cow::Borrowed(s) => Box::from(s), + Cow::Owned(s) => Box::from(s), + } + } +} + #[stable(feature = "os_string_from_box", since = "1.18.0")] impl From> for OsString { - /// Converts a [`Box`]`<`[`OsStr`]`>` into a `OsString` without copying or + /// Converts a [Box]<[OsStr]> into an [`OsString`] without copying or /// allocating. - /// - /// [`Box`]: ../boxed/struct.Box.html - /// [`OsStr`]: ../ffi/struct.OsStr.html + #[inline] fn from(boxed: Box) -> OsString { boxed.into_os_string() } @@ -695,10 +1016,8 @@ impl From> for OsString { #[stable(feature = "box_from_os_string", since = "1.20.0")] impl From for Box { - /// Converts a [`OsString`] into a [`Box`]`` without copying or allocating. - /// - /// [`Box`]: ../boxed/struct.Box.html - /// [`OsString`]: ../ffi/struct.OsString.html + /// Converts an [`OsString`] into a [Box]<[OsStr]> without copying or allocating. + #[inline] fn from(s: OsString) -> Box { s.into_boxed_os_str() } @@ -714,10 +1033,8 @@ impl Clone for Box { #[stable(feature = "shared_from_slice2", since = "1.24.0")] impl From for Arc { - /// Converts a [`OsString`] into a [`Arc`]`` without copying or allocating. - /// - /// [`Arc`]: ../sync/struct.Arc.html - /// [`OsString`]: ../ffi/struct.OsString.html + /// Converts an [`OsString`] into an [Arc]<[OsStr]> by moving the [`OsString`] + /// data into a new [`Arc`] buffer. #[inline] fn from(s: OsString) -> Arc { let arc = s.inner.into_arc(); @@ -727,6 +1044,7 @@ impl From for Arc { #[stable(feature = "shared_from_slice2", since = "1.24.0")] impl From<&OsStr> for Arc { + /// Copies the string into a newly allocated [Arc]<[OsStr]>. #[inline] fn from(s: &OsStr) -> Arc { let arc = s.inner.into_arc(); @@ -736,10 +1054,8 @@ impl From<&OsStr> for Arc { #[stable(feature = "shared_from_slice2", since = "1.24.0")] impl From for Rc { - /// Converts a [`OsString`] into a [`Rc`]`` without copying or allocating. - /// - /// [`Rc`]: ../rc/struct.Rc.html - /// [`OsString`]: ../ffi/struct.OsString.html + /// Converts an [`OsString`] into an [Rc]<[OsStr]> by moving the [`OsString`] + /// data into a new [`Rc`] buffer. #[inline] fn from(s: OsString) -> Rc { let rc = s.inner.into_rc(); @@ -749,6 +1065,7 @@ impl From for Rc { #[stable(feature = "shared_from_slice2", since = "1.24.0")] impl From<&OsStr> for Rc { + /// Copies the string into a newly allocated [Rc]<[OsStr]>. #[inline] fn from(s: &OsStr) -> Rc { let rc = s.inner.into_rc(); @@ -758,6 +1075,7 @@ impl From<&OsStr> for Rc { #[stable(feature = "cow_from_osstr", since = "1.28.0")] impl<'a> From for Cow<'a, OsStr> { + /// Moves the string into a [`Cow::Owned`]. #[inline] fn from(s: OsString) -> Cow<'a, OsStr> { Cow::Owned(s) @@ -766,6 +1084,7 @@ impl<'a> From for Cow<'a, OsStr> { #[stable(feature = "cow_from_osstr", since = "1.28.0")] impl<'a> From<&'a OsStr> for Cow<'a, OsStr> { + /// Converts the string reference into a [`Cow::Borrowed`]. #[inline] fn from(s: &'a OsStr) -> Cow<'a, OsStr> { Cow::Borrowed(s) @@ -774,6 +1093,7 @@ impl<'a> From<&'a OsStr> for Cow<'a, OsStr> { #[stable(feature = "cow_from_osstr", since = "1.28.0")] impl<'a> From<&'a OsString> for Cow<'a, OsStr> { + /// Converts the string reference into a [`Cow::Borrowed`]. #[inline] fn from(s: &'a OsString) -> Cow<'a, OsStr> { Cow::Borrowed(s.as_os_str()) @@ -782,6 +1102,8 @@ impl<'a> From<&'a OsString> for Cow<'a, OsStr> { #[stable(feature = "osstring_from_cow_osstr", since = "1.28.0")] impl<'a> From> for OsString { + /// Converts a `Cow<'a, OsStr>` into an [`OsString`], + /// by copying the contents if they are borrowed. #[inline] fn from(s: Cow<'a, OsStr>) -> Self { s.into_owned() @@ -790,6 +1112,7 @@ impl<'a> From> for OsString { #[stable(feature = "box_default_extra", since = "1.17.0")] impl Default for Box { + #[inline] fn default() -> Box { let rw = Box::into_raw(Slice::empty_box()) as *mut OsStr; unsafe { Box::from_raw(rw) } @@ -938,8 +1261,26 @@ impl OsStr { } } +#[unstable(feature = "slice_concat_ext", issue = "27747")] +impl> alloc::slice::Join<&OsStr> for [S] { + type Output = OsString; + + fn join(slice: &Self, sep: &OsStr) -> OsString { + let Some((first, suffix)) = slice.split_first() else { + return OsString::new(); + }; + let first_owned = first.borrow().to_owned(); + suffix.iter().fold(first_owned, |mut a, b| { + a.push(sep); + a.push(b.borrow()); + a + }) + } +} + #[stable(feature = "rust1", since = "1.0.0")] impl Borrow for OsString { + #[inline] fn borrow(&self) -> &OsStr { &self[..] } @@ -948,17 +1289,19 @@ impl Borrow for OsString { #[stable(feature = "rust1", since = "1.0.0")] impl ToOwned for OsStr { type Owned = OsString; + #[inline] fn to_owned(&self) -> OsString { self.to_os_string() } + #[inline] fn clone_into(&self, target: &mut OsString) { - target.clear(); - target.push(self); + self.inner.clone_into(&mut target.inner) } } #[stable(feature = "rust1", since = "1.0.0")] impl AsRef for OsStr { + #[inline] fn as_ref(&self) -> &OsStr { self } @@ -989,12 +1332,14 @@ impl AsRef for String { } impl FromInner for OsString { + #[inline] fn from_inner(buf: Buf) -> OsString { OsString { inner: buf } } } impl IntoInner for OsString { + #[inline] fn into_inner(self) -> Buf { self.inner } @@ -1007,171 +1352,97 @@ impl AsInner for OsStr { } } -#[cfg(test)] -mod tests { - use super::*; - use crate::sys_common::{AsInner, IntoInner}; - - use crate::rc::Rc; - use crate::sync::Arc; - - #[test] - fn test_os_string_with_capacity() { - let os_string = OsString::with_capacity(0); - assert_eq!(0, os_string.inner.into_inner().capacity()); - - let os_string = OsString::with_capacity(10); - assert_eq!(10, os_string.inner.into_inner().capacity()); - - let mut os_string = OsString::with_capacity(0); - os_string.push("abc"); - assert!(os_string.inner.into_inner().capacity() >= 3); - } - - #[test] - fn test_os_string_clear() { - let mut os_string = OsString::from("abc"); - assert_eq!(3, os_string.inner.as_inner().len()); - - os_string.clear(); - assert_eq!(&os_string, ""); - assert_eq!(0, os_string.inner.as_inner().len()); - } - - #[test] - fn test_os_string_capacity() { - let os_string = OsString::with_capacity(0); - assert_eq!(0, os_string.capacity()); +#[stable(feature = "osstring_from_str", since = "1.45.0")] +impl FromStr for OsString { + type Err = core::convert::Infallible; - let os_string = OsString::with_capacity(10); - assert_eq!(10, os_string.capacity()); - - let mut os_string = OsString::with_capacity(0); - os_string.push("abc"); - assert!(os_string.capacity() >= 3); + #[inline] + fn from_str(s: &str) -> Result { + Ok(OsString::from(s)) } +} - #[test] - fn test_os_string_reserve() { - let mut os_string = OsString::new(); - assert_eq!(os_string.capacity(), 0); - - os_string.reserve(2); - assert!(os_string.capacity() >= 2); - - for _ in 0..16 { - os_string.push("a"); +#[stable(feature = "osstring_extend", since = "1.52.0")] +impl Extend for OsString { + #[inline] + fn extend>(&mut self, iter: T) { + for s in iter { + self.push(&s); } - - assert!(os_string.capacity() >= 16); - os_string.reserve(16); - assert!(os_string.capacity() >= 32); - - os_string.push("a"); - - os_string.reserve(16); - assert!(os_string.capacity() >= 33) } +} - #[test] - fn test_os_string_reserve_exact() { - let mut os_string = OsString::new(); - assert_eq!(os_string.capacity(), 0); - - os_string.reserve_exact(2); - assert!(os_string.capacity() >= 2); - - for _ in 0..16 { - os_string.push("a"); +#[stable(feature = "osstring_extend", since = "1.52.0")] +impl<'a> Extend<&'a OsStr> for OsString { + #[inline] + fn extend>(&mut self, iter: T) { + for s in iter { + self.push(s); } - - assert!(os_string.capacity() >= 16); - os_string.reserve_exact(16); - assert!(os_string.capacity() >= 32); - - os_string.push("a"); - - os_string.reserve_exact(16); - assert!(os_string.capacity() >= 33) - } - - #[test] - fn test_os_string_default() { - let os_string: OsString = Default::default(); - assert_eq!("", &os_string); - } - - #[test] - fn test_os_str_is_empty() { - let mut os_string = OsString::new(); - assert!(os_string.is_empty()); - - os_string.push("abc"); - assert!(!os_string.is_empty()); - - os_string.clear(); - assert!(os_string.is_empty()); - } - - #[test] - fn test_os_str_len() { - let mut os_string = OsString::new(); - assert_eq!(0, os_string.len()); - - os_string.push("abc"); - assert_eq!(3, os_string.len()); - - os_string.clear(); - assert_eq!(0, os_string.len()); - } - - #[test] - fn test_os_str_default() { - let os_str: &OsStr = Default::default(); - assert_eq!("", os_str); } +} - #[test] - fn into_boxed() { - let orig = "Hello, world!"; - let os_str = OsStr::new(orig); - let boxed: Box = Box::from(os_str); - let os_string = os_str.to_owned().into_boxed_os_str().into_os_string(); - assert_eq!(os_str, &*boxed); - assert_eq!(&*boxed, &*os_string); - assert_eq!(&*os_string, os_str); +#[stable(feature = "osstring_extend", since = "1.52.0")] +impl<'a> Extend> for OsString { + #[inline] + fn extend>>(&mut self, iter: T) { + for s in iter { + self.push(&s); + } } +} - #[test] - fn boxed_default() { - let boxed = >::default(); - assert!(boxed.is_empty()); +#[stable(feature = "osstring_extend", since = "1.52.0")] +impl FromIterator for OsString { + #[inline] + fn from_iter>(iter: I) -> Self { + let mut iterator = iter.into_iter(); + + // Because we're iterating over `OsString`s, we can avoid at least + // one allocation by getting the first string from the iterator + // and appending to it all the subsequent strings. + match iterator.next() { + None => OsString::new(), + Some(mut buf) => { + buf.extend(iterator); + buf + } + } } +} - #[test] - fn test_os_str_clone_into() { - let mut os_string = OsString::with_capacity(123); - os_string.push("hello"); - let os_str = OsStr::new("bonjour"); - os_str.clone_into(&mut os_string); - assert_eq!(os_str, os_string); - assert!(os_string.capacity() >= 123); +#[stable(feature = "osstring_extend", since = "1.52.0")] +impl<'a> FromIterator<&'a OsStr> for OsString { + #[inline] + fn from_iter>(iter: I) -> Self { + let mut buf = Self::new(); + for s in iter { + buf.push(s); + } + buf } +} - #[test] - fn into_rc() { - let orig = "Hello, world!"; - let os_str = OsStr::new(orig); - let rc: Rc = Rc::from(os_str); - let arc: Arc = Arc::from(os_str); - - assert_eq!(&*rc, os_str); - assert_eq!(&*arc, os_str); - - let rc2: Rc = Rc::from(os_str.to_owned()); - let arc2: Arc = Arc::from(os_str.to_owned()); - - assert_eq!(&*rc2, os_str); - assert_eq!(&*arc2, os_str); +#[stable(feature = "osstring_extend", since = "1.52.0")] +impl<'a> FromIterator> for OsString { + #[inline] + fn from_iter>>(iter: I) -> Self { + let mut iterator = iter.into_iter(); + + // Because we're iterating over `OsString`s, we can avoid at least + // one allocation by getting the first owned string from the iterator + // and appending to it all the subsequent strings. + match iterator.next() { + None => OsString::new(), + Some(Cow::Owned(mut buf)) => { + buf.extend(iterator); + buf + } + Some(Cow::Borrowed(buf)) => { + let mut buf = OsString::from(buf); + buf.extend(iterator); + buf + } + } } } diff --git a/crux-mir/lib/std/src/ffi/os_str/tests.rs b/crux-mir/lib/std/src/ffi/os_str/tests.rs new file mode 100644 index 000000000..d7926749a --- /dev/null +++ b/crux-mir/lib/std/src/ffi/os_str/tests.rs @@ -0,0 +1,179 @@ +use super::*; +use crate::sys_common::{AsInner, IntoInner}; + +use crate::rc::Rc; +use crate::sync::Arc; + +#[test] +fn test_os_string_with_capacity() { + let os_string = OsString::with_capacity(0); + assert_eq!(0, os_string.inner.into_inner().capacity()); + + let os_string = OsString::with_capacity(10); + assert_eq!(10, os_string.inner.into_inner().capacity()); + + let mut os_string = OsString::with_capacity(0); + os_string.push("abc"); + assert!(os_string.inner.into_inner().capacity() >= 3); +} + +#[test] +fn test_os_string_clear() { + let mut os_string = OsString::from("abc"); + assert_eq!(3, os_string.inner.as_inner().len()); + + os_string.clear(); + assert_eq!(&os_string, ""); + assert_eq!(0, os_string.inner.as_inner().len()); +} + +#[test] +fn test_os_string_capacity() { + let os_string = OsString::with_capacity(0); + assert_eq!(0, os_string.capacity()); + + let os_string = OsString::with_capacity(10); + assert_eq!(10, os_string.capacity()); + + let mut os_string = OsString::with_capacity(0); + os_string.push("abc"); + assert!(os_string.capacity() >= 3); +} + +#[test] +fn test_os_string_reserve() { + let mut os_string = OsString::new(); + assert_eq!(os_string.capacity(), 0); + + os_string.reserve(2); + assert!(os_string.capacity() >= 2); + + for _ in 0..16 { + os_string.push("a"); + } + + assert!(os_string.capacity() >= 16); + os_string.reserve(16); + assert!(os_string.capacity() >= 32); + + os_string.push("a"); + + os_string.reserve(16); + assert!(os_string.capacity() >= 33) +} + +#[test] +fn test_os_string_reserve_exact() { + let mut os_string = OsString::new(); + assert_eq!(os_string.capacity(), 0); + + os_string.reserve_exact(2); + assert!(os_string.capacity() >= 2); + + for _ in 0..16 { + os_string.push("a"); + } + + assert!(os_string.capacity() >= 16); + os_string.reserve_exact(16); + assert!(os_string.capacity() >= 32); + + os_string.push("a"); + + os_string.reserve_exact(16); + assert!(os_string.capacity() >= 33) +} + +#[test] +fn test_os_string_join() { + let strings = [OsStr::new("hello"), OsStr::new("dear"), OsStr::new("world")]; + assert_eq!("hello", strings[..1].join(OsStr::new(" "))); + assert_eq!("hello dear world", strings.join(OsStr::new(" "))); + assert_eq!("hellodearworld", strings.join(OsStr::new(""))); + assert_eq!("hello.\n dear.\n world", strings.join(OsStr::new(".\n "))); + + assert_eq!("dear world", strings[1..].join(&OsString::from(" "))); + + let strings_abc = [OsString::from("a"), OsString::from("b"), OsString::from("c")]; + assert_eq!("a b c", strings_abc.join(OsStr::new(" "))); +} + +#[test] +fn test_os_string_default() { + let os_string: OsString = Default::default(); + assert_eq!("", &os_string); +} + +#[test] +fn test_os_str_is_empty() { + let mut os_string = OsString::new(); + assert!(os_string.is_empty()); + + os_string.push("abc"); + assert!(!os_string.is_empty()); + + os_string.clear(); + assert!(os_string.is_empty()); +} + +#[test] +fn test_os_str_len() { + let mut os_string = OsString::new(); + assert_eq!(0, os_string.len()); + + os_string.push("abc"); + assert_eq!(3, os_string.len()); + + os_string.clear(); + assert_eq!(0, os_string.len()); +} + +#[test] +fn test_os_str_default() { + let os_str: &OsStr = Default::default(); + assert_eq!("", os_str); +} + +#[test] +fn into_boxed() { + let orig = "Hello, world!"; + let os_str = OsStr::new(orig); + let boxed: Box = Box::from(os_str); + let os_string = os_str.to_owned().into_boxed_os_str().into_os_string(); + assert_eq!(os_str, &*boxed); + assert_eq!(&*boxed, &*os_string); + assert_eq!(&*os_string, os_str); +} + +#[test] +fn boxed_default() { + let boxed = >::default(); + assert!(boxed.is_empty()); +} + +#[test] +fn test_os_str_clone_into() { + let mut os_string = OsString::with_capacity(123); + os_string.push("hello"); + let os_str = OsStr::new("bonjour"); + os_str.clone_into(&mut os_string); + assert_eq!(os_str, os_string); + assert!(os_string.capacity() >= 123); +} + +#[test] +fn into_rc() { + let orig = "Hello, world!"; + let os_str = OsStr::new(orig); + let rc: Rc = Rc::from(os_str); + let arc: Arc = Arc::from(os_str); + + assert_eq!(&*rc, os_str); + assert_eq!(&*arc, os_str); + + let rc2: Rc = Rc::from(os_str.to_owned()); + let arc2: Arc = Arc::from(os_str.to_owned()); + + assert_eq!(&*rc2, os_str); + assert_eq!(&*arc2, os_str); +} diff --git a/crux-mir/lib/std/src/fs.rs b/crux-mir/lib/std/src/fs.rs index e20fcfafa..286ad68fd 100644 --- a/crux-mir/lib/std/src/fs.rs +++ b/crux-mir/lib/std/src/fs.rs @@ -1,5 +1,3 @@ -// ignore-tidy-filelength - //! Filesystem manipulation operations. //! //! This module contains basic methods to manipulate the contents of the local @@ -8,16 +6,20 @@ //! extension traits of `std::os::$platform`. #![stable(feature = "rust1", since = "1.0.0")] +#![deny(unsafe_op_in_unsafe_fn)] + +#[cfg(all(test, not(any(target_os = "emscripten", target_env = "sgx"))))] +mod tests; use crate::ffi::OsString; use crate::fmt; -use crate::io::{self, Initializer, IoSlice, IoSliceMut, Read, Seek, SeekFrom, Write}; +use crate::io::{self, BorrowedCursor, IoSlice, IoSliceMut, Read, Seek, SeekFrom, Write}; use crate::path::{Path, PathBuf}; use crate::sys::fs as fs_imp; use crate::sys_common::{AsInner, AsInnerMut, FromInner, IntoInner}; use crate::time::SystemTime; -/// A reference to an open file on the filesystem. +/// An object providing access to an open file on the filesystem. /// /// An instance of a `File` can be read and/or written depending on what options /// it was opened with. Files also implement [`Seek`] to alter the logical cursor @@ -29,7 +31,7 @@ use crate::time::SystemTime; /// /// # Examples /// -/// Creates a new file and write bytes to it (you can also use [`write`]): +/// Creates a new file and write bytes to it (you can also use [`write()`]): /// /// ```no_run /// use std::fs::File; @@ -83,15 +85,16 @@ use crate::time::SystemTime; /// by different processes. Avoid assuming that holding a `&File` means that the /// file will not change. /// -/// [`Seek`]: ../io/trait.Seek.html -/// [`String`]: ../string/struct.String.html -/// [`Read`]: ../io/trait.Read.html -/// [`Write`]: ../io/trait.Write.html -/// [`BufReader`]: ../io/struct.BufReader.html -/// [`sync_all`]: struct.File.html#method.sync_all -/// [`read`]: fn.read.html -/// [`write`]: fn.write.html +/// # Platform-specific behavior +/// +/// On Windows, the implementation of [`Read`] and [`Write`] traits for `File` +/// perform synchronous I/O operations. Therefore the underlying file must not +/// have been opened for asynchronous I/O (e.g. by using `FILE_FLAG_OVERLAPPED`). +/// +/// [`BufReader`]: io::BufReader +/// [`sync_all`]: File::sync_all #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "File")] pub struct File { inner: fs_imp::File, } @@ -102,9 +105,6 @@ pub struct File { /// [`symlink_metadata`] function or method and represents known /// metadata about a file such as its permissions, size, modification /// times, etc. -/// -/// [`metadata`]: fn.metadata.html -/// [`symlink_metadata`]: fn.symlink_metadata.html #[stable(feature = "rust1", since = "1.0.0")] #[derive(Clone)] pub struct Metadata(fs_imp::FileAttr); @@ -112,7 +112,7 @@ pub struct Metadata(fs_imp::FileAttr); /// Iterator over the entries in a directory. /// /// This iterator is returned from the [`read_dir`] function of this module and -/// will yield instances of [`io::Result`]`<`[`DirEntry`]`>`. Through a [`DirEntry`] +/// will yield instances of [io::Result]<[DirEntry]>. Through a [`DirEntry`] /// information like the entry's path and possibly other metadata can be /// learned. /// @@ -123,22 +123,25 @@ pub struct Metadata(fs_imp::FileAttr); /// /// This [`io::Result`] will be an [`Err`] if there's some sort of intermittent /// IO error during iteration. -/// -/// [`read_dir`]: fn.read_dir.html -/// [`DirEntry`]: struct.DirEntry.html -/// [`io::Result`]: ../io/type.Result.html -/// [`Err`]: ../result/enum.Result.html#variant.Err #[stable(feature = "rust1", since = "1.0.0")] #[derive(Debug)] pub struct ReadDir(fs_imp::ReadDir); /// Entries returned by the [`ReadDir`] iterator. /// -/// [`ReadDir`]: struct.ReadDir.html -/// /// An instance of `DirEntry` represents an entry inside of a directory on the /// filesystem. Each entry can be inspected via methods to learn about the full /// path or possibly other metadata through per-platform extension traits. +/// +/// # Platform-specific behavior +/// +/// On Unix, the `DirEntry` struct contains an internal reference to the open +/// directory. Holding `DirEntry` objects will consume a file handle even +/// after the `ReadDir` iterator is dropped. +/// +/// Note that this [may change in the future][changes]. +/// +/// [changes]: io#platform-specific-behavior #[stable(feature = "rust1", since = "1.0.0")] pub struct DirEntry(fs_imp::DirEntry); @@ -149,20 +152,11 @@ pub struct DirEntry(fs_imp::DirEntry); /// [`File::create`] methods are aliases for commonly used options using this /// builder. /// -/// [`File`]: struct.File.html -/// [`File::open`]: struct.File.html#method.open -/// [`File::create`]: struct.File.html#method.create -/// -/// Generally speaking, when using `OpenOptions`, you'll first call [`new`], -/// then chain calls to methods to set each option, then call [`open`], -/// passing the path of the file you're trying to open. This will give you a -/// [`io::Result`][result] with a [`File`][file] inside that you can further -/// operate on. -/// -/// [`new`]: struct.OpenOptions.html#method.new -/// [`open`]: struct.OpenOptions.html#method.open -/// [result]: ../io/type.Result.html -/// [file]: struct.File.html +/// Generally speaking, when using `OpenOptions`, you'll first call +/// [`OpenOptions::new`], then chain calls to methods to set each option, then +/// call [`OpenOptions::open`], passing the path of the file you're trying to +/// open. This will give you a [`io::Result`] with a [`File`] inside that you +/// can further operate on. /// /// # Examples /// @@ -190,66 +184,55 @@ pub struct DirEntry(fs_imp::DirEntry); #[stable(feature = "rust1", since = "1.0.0")] pub struct OpenOptions(fs_imp::OpenOptions); +/// Representation of the various timestamps on a file. +#[derive(Copy, Clone, Debug, Default)] +#[unstable(feature = "file_set_times", issue = "98245")] +pub struct FileTimes(fs_imp::FileTimes); + /// Representation of the various permissions on a file. /// -/// This module only currently provides one bit of information, [`readonly`], -/// which is exposed on all currently supported platforms. Unix-specific -/// functionality, such as mode bits, is available through the -/// [`PermissionsExt`] trait. +/// This module only currently provides one bit of information, +/// [`Permissions::readonly`], which is exposed on all currently supported +/// platforms. Unix-specific functionality, such as mode bits, is available +/// through the [`PermissionsExt`] trait. /// -/// [`readonly`]: struct.Permissions.html#method.readonly -/// [`PermissionsExt`]: ../os/unix/fs/trait.PermissionsExt.html +/// [`PermissionsExt`]: crate::os::unix::fs::PermissionsExt #[derive(Clone, PartialEq, Eq, Debug)] #[stable(feature = "rust1", since = "1.0.0")] pub struct Permissions(fs_imp::FilePermissions); /// A structure representing a type of file with accessors for each file type. /// It is returned by [`Metadata::file_type`] method. -/// -/// [`Metadata::file_type`]: struct.Metadata.html#method.file_type #[stable(feature = "file_type", since = "1.1.0")] #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] +#[cfg_attr(not(test), rustc_diagnostic_item = "FileType")] pub struct FileType(fs_imp::FileType); /// A builder used to create directories in various manners. /// /// This builder also supports platform-specific options. #[stable(feature = "dir_builder", since = "1.6.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "DirBuilder")] #[derive(Debug)] pub struct DirBuilder { inner: fs_imp::DirBuilder, recursive: bool, } -/// Indicates how large a buffer to pre-allocate before reading the entire file. -fn initial_buffer_size(file: &File) -> usize { - // Allocate one extra byte so the buffer doesn't need to grow before the - // final `read` call at the end of the file. Don't worry about `usize` - // overflow because reading will fail regardless in that case. - file.metadata().map(|m| m.len() as usize + 1).unwrap_or(0) -} - /// Read the entire contents of a file into a bytes vector. /// /// This is a convenience function for using [`File::open`] and [`read_to_end`] -/// with fewer imports and without an intermediate variable. It pre-allocates a -/// buffer based on the file size when available, so it is generally faster than -/// reading into a vector created with `Vec::new()`. +/// with fewer imports and without an intermediate variable. /// -/// [`File::open`]: struct.File.html#method.open -/// [`read_to_end`]: ../io/trait.Read.html#method.read_to_end +/// [`read_to_end`]: Read::read_to_end /// /// # Errors /// /// This function will return an error if `path` does not already exist. /// Other errors may also be returned according to [`OpenOptions::open`]. /// -/// [`OpenOptions::open`]: struct.OpenOptions.html#method.open -/// /// It will also return an error if it encounters while reading an error -/// of a kind other than [`ErrorKind::Interrupted`]. -/// -/// [`ErrorKind::Interrupted`]: ../../std/io/enum.ErrorKind.html#variant.Interrupted +/// of a kind other than [`io::ErrorKind::Interrupted`]. /// /// # Examples /// @@ -266,8 +249,9 @@ fn initial_buffer_size(file: &File) -> usize { pub fn read>(path: P) -> io::Result> { fn inner(path: &Path) -> io::Result> { let mut file = File::open(path)?; - let mut bytes = Vec::with_capacity(initial_buffer_size(&file)); - file.read_to_end(&mut bytes)?; + let size = file.metadata().map(|m| m.len()).unwrap_or(0); + let mut bytes = Vec::with_capacity(size as usize); + io::default_read_to_end(&mut file, &mut bytes)?; Ok(bytes) } inner(path.as_ref()) @@ -276,33 +260,27 @@ pub fn read>(path: P) -> io::Result> { /// Read the entire contents of a file into a string. /// /// This is a convenience function for using [`File::open`] and [`read_to_string`] -/// with fewer imports and without an intermediate variable. It pre-allocates a -/// buffer based on the file size when available, so it is generally faster than -/// reading into a string created with `String::new()`. +/// with fewer imports and without an intermediate variable. /// -/// [`File::open`]: struct.File.html#method.open -/// [`read_to_string`]: ../io/trait.Read.html#method.read_to_string +/// [`read_to_string`]: Read::read_to_string /// /// # Errors /// /// This function will return an error if `path` does not already exist. /// Other errors may also be returned according to [`OpenOptions::open`]. /// -/// [`OpenOptions::open`]: struct.OpenOptions.html#method.open -/// /// It will also return an error if it encounters while reading an error -/// of a kind other than [`ErrorKind::Interrupted`], +/// of a kind other than [`io::ErrorKind::Interrupted`], /// or if the contents of the file are not valid UTF-8. /// -/// [`ErrorKind::Interrupted`]: ../../std/io/enum.ErrorKind.html#variant.Interrupted -/// /// # Examples /// /// ```no_run /// use std::fs; /// use std::net::SocketAddr; +/// use std::error::Error; /// -/// fn main() -> Result<(), Box> { +/// fn main() -> Result<(), Box> { /// let foo: SocketAddr = fs::read_to_string("address.txt")?.parse()?; /// Ok(()) /// } @@ -311,8 +289,9 @@ pub fn read>(path: P) -> io::Result> { pub fn read_to_string>(path: P) -> io::Result { fn inner(path: &Path) -> io::Result { let mut file = File::open(path)?; - let mut string = String::with_capacity(initial_buffer_size(&file)); - file.read_to_string(&mut string)?; + let size = file.metadata().map(|m| m.len()).unwrap_or(0); + let mut string = String::with_capacity(size as usize); + io::default_read_to_string(&mut file, &mut string)?; Ok(string) } inner(path.as_ref()) @@ -323,11 +302,13 @@ pub fn read_to_string>(path: P) -> io::Result { /// This function will create a file if it does not exist, /// and will entirely replace its contents if it does. /// +/// Depending on the platform, this function may fail if the +/// full directory path does not exist. +/// /// This is a convenience function for using [`File::create`] and [`write_all`] /// with fewer imports. /// -/// [`File::create`]: struct.File.html#method.create -/// [`write_all`]: ../io/trait.Write.html#method.write_all +/// [`write_all`]: Write::write_all /// /// # Examples /// @@ -358,8 +339,6 @@ impl File { /// This function will return an error if `path` does not already exist. /// Other errors may also be returned according to [`OpenOptions::open`]. /// - /// [`OpenOptions::open`]: struct.OpenOptions.html#method.open - /// /// # Examples /// /// ```no_run @@ -380,9 +359,10 @@ impl File { /// This function will create a file if it does not exist, /// and will truncate it if it does. /// - /// See the [`OpenOptions::open`] function for more details. + /// Depending on the platform, this function may fail if the + /// full directory path does not exist. /// - /// [`OpenOptions::open`]: struct.OpenOptions.html#method.open + /// See the [`OpenOptions::open`] function for more details. /// /// # Examples /// @@ -399,34 +379,62 @@ impl File { OpenOptions::new().write(true).create(true).truncate(true).open(path.as_ref()) } + /// Creates a new file in read-write mode; error if the file exists. + /// + /// This function will create a file if it does not exist, or return an error if it does. This + /// way, if the call succeeds, the file returned is guaranteed to be new. + /// + /// This option is useful because it is atomic. Otherwise between checking whether a file + /// exists and creating a new one, the file may have been created by another process (a TOCTOU + /// race condition / attack). + /// + /// This can also be written using + /// `File::options().read(true).write(true).create_new(true).open(...)`. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(file_create_new)] + /// + /// use std::fs::File; + /// + /// fn main() -> std::io::Result<()> { + /// let mut f = File::create_new("foo.txt")?; + /// Ok(()) + /// } + /// ``` + #[unstable(feature = "file_create_new", issue = "105135")] + pub fn create_new>(path: P) -> io::Result { + OpenOptions::new().read(true).write(true).create_new(true).open(path.as_ref()) + } + /// Returns a new OpenOptions object. /// /// This function returns a new OpenOptions object that you can use to /// open or create a file with specific options if `open()` or `create()` /// are not appropriate. /// - /// It is equivalent to `OpenOptions::new()` but allows you to write more - /// readable code. Instead of `OpenOptions::new().read(true).open("foo.txt")` - /// you can write `File::with_options().read(true).open("foo.txt")`. This + /// It is equivalent to `OpenOptions::new()`, but allows you to write more + /// readable code. Instead of + /// `OpenOptions::new().append(true).open("example.log")`, + /// you can write `File::options().append(true).open("example.log")`. This /// also avoids the need to import `OpenOptions`. /// /// See the [`OpenOptions::new`] function for more details. /// - /// [`OpenOptions::new`]: struct.OpenOptions.html#method.new - /// /// # Examples /// /// ```no_run - /// #![feature(with_options)] /// use std::fs::File; /// /// fn main() -> std::io::Result<()> { - /// let mut f = File::with_options().read(true).open("foo.txt")?; + /// let mut f = File::options().append(true).open("example.log")?; /// Ok(()) /// } /// ``` - #[unstable(feature = "with_options", issue = "65439")] - pub fn with_options() -> OpenOptions { + #[must_use] + #[stable(feature = "with_options", since = "1.58.0")] + pub fn options() -> OpenOptions { OpenOptions::new() } @@ -458,7 +466,7 @@ impl File { self.inner.fsync() } - /// This function is similar to [`sync_all`], except that it may not + /// This function is similar to [`sync_all`], except that it might not /// synchronize file metadata to the filesystem. /// /// This is intended for use cases that must synchronize content, but don't @@ -468,7 +476,7 @@ impl File { /// Note that some platforms may simply implement this in terms of /// [`sync_all`]. /// - /// [`sync_all`]: struct.File.html#method.sync_all + /// [`sync_all`]: File::sync_all /// /// # Examples /// @@ -504,8 +512,9 @@ impl File { /// # Errors /// /// This function will return an error if the file is not opened for writing. - /// Also, std::io::ErrorKind::InvalidInput will be returned if the desired - /// length would cause an overflow due to the implementation specifics. + /// Also, [`std::io::ErrorKind::InvalidInput`](crate::io::ErrorKind::InvalidInput) + /// will be returned if the desired length would cause an overflow due to + /// the implementation specifics. /// /// # Examples /// @@ -596,7 +605,7 @@ impl File { /// the `SetFileInformationByHandle` function on Windows. Note that, this /// [may change in the future][changes]. /// - /// [changes]: ../io/index.html#platform-specific-behavior + /// [changes]: io#platform-specific-behavior /// /// # Errors /// @@ -624,8 +633,66 @@ impl File { pub fn set_permissions(&self, perm: Permissions) -> io::Result<()> { self.inner.set_permissions(perm.0) } + + /// Changes the timestamps of the underlying file. + /// + /// # Platform-specific behavior + /// + /// This function currently corresponds to the `futimens` function on Unix (falling back to + /// `futimes` on macOS before 10.13) and the `SetFileTime` function on Windows. Note that this + /// [may change in the future][changes]. + /// + /// [changes]: io#platform-specific-behavior + /// + /// # Errors + /// + /// This function will return an error if the user lacks permission to change timestamps on the + /// underlying file. It may also return an error in other os-specific unspecified cases. + /// + /// This function may return an error if the operating system lacks support to change one or + /// more of the timestamps set in the `FileTimes` structure. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(file_set_times)] + /// + /// fn main() -> std::io::Result<()> { + /// use std::fs::{self, File, FileTimes}; + /// + /// let src = fs::metadata("src")?; + /// let dest = File::options().write(true).open("dest")?; + /// let times = FileTimes::new() + /// .set_accessed(src.accessed()?) + /// .set_modified(src.modified()?); + /// dest.set_times(times)?; + /// Ok(()) + /// } + /// ``` + #[unstable(feature = "file_set_times", issue = "98245")] + #[doc(alias = "futimens")] + #[doc(alias = "futimes")] + #[doc(alias = "SetFileTime")] + pub fn set_times(&self, times: FileTimes) -> io::Result<()> { + self.inner.set_times(times.0) + } + + /// Changes the modification time of the underlying file. + /// + /// This is an alias for `set_times(FileTimes::new().set_modified(time))`. + #[unstable(feature = "file_set_times", issue = "98245")] + #[inline] + pub fn set_modified(&self, time: SystemTime) -> io::Result<()> { + self.set_times(FileTimes::new().set_modified(time)) + } } +// In addition to the `impl`s here, `File` also has `impl`s for +// `AsFd`/`From`/`Into` and +// `AsRawFd`/`IntoRawFd`/`FromRawFd`, on Unix and WASI, and +// `AsHandle`/`From`/`Into` and +// `AsRawHandle`/`IntoRawHandle`/`FromRawHandle` on Windows. + impl AsInner for File { fn as_inner(&self) -> &fs_imp::File { &self.inner @@ -649,6 +716,15 @@ impl fmt::Debug for File { } } +/// Indicates how much extra capacity is needed to read the rest of the file. +fn buffer_capacity_required(mut file: &File) -> usize { + let size = file.metadata().map(|m| m.len()).unwrap_or(0); + let pos = file.stream_position().unwrap_or(0); + // Don't worry about `usize` overflow because reading will fail regardless + // in that case. + size.saturating_sub(pos) as usize +} + #[stable(feature = "rust1", since = "1.0.0")] impl Read for File { fn read(&mut self, buf: &mut [u8]) -> io::Result { @@ -659,9 +735,25 @@ impl Read for File { self.inner.read_vectored(bufs) } + fn read_buf(&mut self, cursor: BorrowedCursor<'_>) -> io::Result<()> { + self.inner.read_buf(cursor) + } + #[inline] - unsafe fn initializer(&self) -> Initializer { - Initializer::nop() + fn is_read_vectored(&self) -> bool { + self.inner.is_read_vectored() + } + + // Reserves space in the buffer based on the file size when available. + fn read_to_end(&mut self, buf: &mut Vec) -> io::Result { + buf.reserve(buffer_capacity_required(self)); + io::default_read_to_end(self, buf) + } + + // Reserves space in the buffer based on the file size when available. + fn read_to_string(&mut self, buf: &mut String) -> io::Result { + buf.reserve(buffer_capacity_required(self)); + io::default_read_to_string(self, buf) } } #[stable(feature = "rust1", since = "1.0.0")] @@ -674,6 +766,11 @@ impl Write for File { self.inner.write_vectored(bufs) } + #[inline] + fn is_write_vectored(&self) -> bool { + self.inner.is_write_vectored() + } + fn flush(&mut self) -> io::Result<()> { self.inner.flush() } @@ -690,13 +787,29 @@ impl Read for &File { self.inner.read(buf) } + fn read_buf(&mut self, cursor: BorrowedCursor<'_>) -> io::Result<()> { + self.inner.read_buf(cursor) + } + fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { self.inner.read_vectored(bufs) } #[inline] - unsafe fn initializer(&self) -> Initializer { - Initializer::nop() + fn is_read_vectored(&self) -> bool { + self.inner.is_read_vectored() + } + + // Reserves space in the buffer based on the file size when available. + fn read_to_end(&mut self, buf: &mut Vec) -> io::Result { + buf.reserve(buffer_capacity_required(self)); + io::default_read_to_end(self, buf) + } + + // Reserves space in the buffer based on the file size when available. + fn read_to_string(&mut self, buf: &mut String) -> io::Result { + buf.reserve(buffer_capacity_required(self)); + io::default_read_to_string(self, buf) } } #[stable(feature = "rust1", since = "1.0.0")] @@ -709,6 +822,11 @@ impl Write for &File { self.inner.write_vectored(bufs) } + #[inline] + fn is_write_vectored(&self) -> bool { + self.inner.is_write_vectored() + } + fn flush(&mut self) -> io::Result<()> { self.inner.flush() } @@ -734,7 +852,8 @@ impl OpenOptions { /// let file = options.read(true).open("foo.txt"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn new() -> OpenOptions { + #[must_use] + pub fn new() -> Self { OpenOptions(fs_imp::OpenOptions::new()) } @@ -751,7 +870,7 @@ impl OpenOptions { /// let file = OpenOptions::new().read(true).open("foo.txt"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn read(&mut self, read: bool) -> &mut OpenOptions { + pub fn read(&mut self, read: bool) -> &mut Self { self.0.read(read); self } @@ -772,7 +891,7 @@ impl OpenOptions { /// let file = OpenOptions::new().write(true).open("foo.txt"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn write(&mut self, write: bool) -> &mut OpenOptions { + pub fn write(&mut self, write: bool) -> &mut Self { self.0.write(write); self } @@ -797,19 +916,17 @@ impl OpenOptions { /// If a file is opened with both read and append access, beware that after /// opening, and after every write, the position for reading may be set at the /// end of the file. So, before writing, save the current position (using - /// [`seek`]`(`[`SeekFrom`]`::`[`Current`]`(0))`), and restore it before the next read. + /// [seek]\([SeekFrom]::[Current]\(0))), and restore it before the next read. /// /// ## Note /// - /// This function doesn't create the file if it doesn't exist. Use the [`create`] - /// method to do so. + /// This function doesn't create the file if it doesn't exist. Use the + /// [`OpenOptions::create`] method to do so. /// - /// [`write()`]: ../../std/fs/struct.File.html#method.write - /// [`flush()`]: ../../std/fs/struct.File.html#method.flush - /// [`seek`]: ../../std/fs/struct.File.html#method.seek - /// [`SeekFrom`]: ../../std/io/enum.SeekFrom.html - /// [`Current`]: ../../std/io/enum.SeekFrom.html#variant.Current - /// [`create`]: #method.create + /// [`write()`]: Write::write "io::Write::write" + /// [`flush()`]: Write::flush "io::Write::flush" + /// [seek]: Seek::seek "io::Seek::seek" + /// [Current]: SeekFrom::Current "io::SeekFrom::Current" /// /// # Examples /// @@ -819,7 +936,7 @@ impl OpenOptions { /// let file = OpenOptions::new().append(true).open("foo.txt"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn append(&mut self, append: bool) -> &mut OpenOptions { + pub fn append(&mut self, append: bool) -> &mut Self { self.0.append(append); self } @@ -839,18 +956,15 @@ impl OpenOptions { /// let file = OpenOptions::new().write(true).truncate(true).open("foo.txt"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn truncate(&mut self, truncate: bool) -> &mut OpenOptions { + pub fn truncate(&mut self, truncate: bool) -> &mut Self { self.0.truncate(truncate); self } /// Sets the option to create a new file, or open it if it already exists. /// - /// In order for the file to be created, [`write`] or [`append`] access must - /// be used. - /// - /// [`write`]: #method.write - /// [`append`]: #method.append + /// In order for the file to be created, [`OpenOptions::write`] or + /// [`OpenOptions::append`] access must be used. /// /// # Examples /// @@ -860,7 +974,7 @@ impl OpenOptions { /// let file = OpenOptions::new().write(true).create(true).open("foo.txt"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn create(&mut self, create: bool) -> &mut OpenOptions { + pub fn create(&mut self, create: bool) -> &mut Self { self.0.create(create); self } @@ -880,8 +994,8 @@ impl OpenOptions { /// The file must be opened with write or append access in order to create /// a new file. /// - /// [`.create()`]: #method.create - /// [`.truncate()`]: #method.truncate + /// [`.create()`]: OpenOptions::create + /// [`.truncate()`]: OpenOptions::truncate /// /// # Examples /// @@ -893,7 +1007,7 @@ impl OpenOptions { /// .open("foo.txt"); /// ``` #[stable(feature = "expand_open_options2", since = "1.9.0")] - pub fn create_new(&mut self, create_new: bool) -> &mut OpenOptions { + pub fn create_new(&mut self, create_new: bool) -> &mut Self { self.0.create_new(create_new); self } @@ -904,9 +1018,8 @@ impl OpenOptions { /// /// This function will return an error under a number of different /// circumstances. Some of these error conditions are listed here, together - /// with their [`ErrorKind`]. The mapping to [`ErrorKind`]s is not part of - /// the compatibility contract of the function, especially the `Other` kind - /// might change to more specific kinds in the future. + /// with their [`io::ErrorKind`]. The mapping to [`io::ErrorKind`]s is not + /// part of the compatibility contract of the function. /// /// * [`NotFound`]: The specified file does not exist and neither `create` /// or `create_new` is set. @@ -920,9 +1033,11 @@ impl OpenOptions { /// exists. /// * [`InvalidInput`]: Invalid combinations of open options (truncate /// without write access, no access mode set, etc.). - /// * [`Other`]: One of the directory components of the specified file path + /// + /// The following errors don't match any existing [`io::ErrorKind`] at the moment: + /// * One of the directory components of the specified file path /// was not, in fact, a directory. - /// * [`Other`]: Filesystem-level errors: full disk, write permission + /// * Filesystem-level errors: full disk, write permission /// requested on a read-only file system, exceeded disk quota, too many /// open files, too long filename, too many symbolic links in the /// specified path (Unix-like systems only), etc. @@ -935,12 +1050,10 @@ impl OpenOptions { /// let file = OpenOptions::new().read(true).open("foo.txt"); /// ``` /// - /// [`ErrorKind`]: ../io/enum.ErrorKind.html - /// [`AlreadyExists`]: ../io/enum.ErrorKind.html#variant.AlreadyExists - /// [`InvalidInput`]: ../io/enum.ErrorKind.html#variant.InvalidInput - /// [`NotFound`]: ../io/enum.ErrorKind.html#variant.NotFound - /// [`Other`]: ../io/enum.ErrorKind.html#variant.Other - /// [`PermissionDenied`]: ../io/enum.ErrorKind.html#variant.PermissionDenied + /// [`AlreadyExists`]: io::ErrorKind::AlreadyExists + /// [`InvalidInput`]: io::ErrorKind::InvalidInput + /// [`NotFound`]: io::ErrorKind::NotFound + /// [`PermissionDenied`]: io::ErrorKind::PermissionDenied #[stable(feature = "rust1", since = "1.0.0")] pub fn open>(&self, path: P) -> io::Result { self._open(path.as_ref()) @@ -978,6 +1091,7 @@ impl Metadata { /// Ok(()) /// } /// ``` + #[must_use] #[stable(feature = "file_type", since = "1.1.0")] pub fn file_type(&self) -> FileType { FileType(self.0.file_type()) @@ -985,12 +1099,9 @@ impl Metadata { /// Returns `true` if this metadata is for a directory. The /// result is mutually exclusive to the result of - /// [`is_file`], and will be false for symlink metadata + /// [`Metadata::is_file`], and will be false for symlink metadata /// obtained from [`symlink_metadata`]. /// - /// [`is_file`]: struct.Metadata.html#method.is_file - /// [`symlink_metadata`]: fn.symlink_metadata.html - /// /// # Examples /// /// ```no_run @@ -1003,6 +1114,7 @@ impl Metadata { /// Ok(()) /// } /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn is_dir(&self) -> bool { self.file_type().is_dir() @@ -1010,11 +1122,14 @@ impl Metadata { /// Returns `true` if this metadata is for a regular file. The /// result is mutually exclusive to the result of - /// [`is_dir`], and will be false for symlink metadata + /// [`Metadata::is_dir`], and will be false for symlink metadata /// obtained from [`symlink_metadata`]. /// - /// [`is_dir`]: struct.Metadata.html#method.is_dir - /// [`symlink_metadata`]: fn.symlink_metadata.html + /// When the goal is simply to read from (or write to) the source, the most + /// reliable way to test the source can be read (or written to) is to open + /// it. Only using `is_file` can break workflows like `diff <( prog_a )` on + /// a Unix-like system for example. See [`File::open`] or + /// [`OpenOptions::open`] for more information. /// /// # Examples /// @@ -1028,11 +1143,38 @@ impl Metadata { /// Ok(()) /// } /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn is_file(&self) -> bool { self.file_type().is_file() } + /// Returns `true` if this metadata is for a symbolic link. + /// + /// # Examples + /// + #[cfg_attr(unix, doc = "```no_run")] + #[cfg_attr(not(unix), doc = "```ignore")] + /// use std::fs; + /// use std::path::Path; + /// use std::os::unix::fs::symlink; + /// + /// fn main() -> std::io::Result<()> { + /// let link_path = Path::new("link"); + /// symlink("/origin_does_not_exist/", link_path)?; + /// + /// let metadata = fs::symlink_metadata(link_path)?; + /// + /// assert!(metadata.is_symlink()); + /// Ok(()) + /// } + /// ``` + #[must_use] + #[stable(feature = "is_symlink", since = "1.58.0")] + pub fn is_symlink(&self) -> bool { + self.file_type().is_symlink() + } + /// Returns the size of the file, in bytes, this metadata is for. /// /// # Examples @@ -1047,6 +1189,7 @@ impl Metadata { /// Ok(()) /// } /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn len(&self) -> u64 { self.0.size() @@ -1066,6 +1209,7 @@ impl Metadata { /// Ok(()) /// } /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn permissions(&self) -> Permissions { Permissions(self.0.perm()) @@ -1078,7 +1222,7 @@ impl Metadata { /// /// # Errors /// - /// This field may not be available on all platforms, and will return an + /// This field might not be available on all platforms, and will return an /// `Err` on platforms where it is not available. /// /// # Examples @@ -1090,7 +1234,7 @@ impl Metadata { /// let metadata = fs::metadata("foo.txt")?; /// /// if let Ok(time) = metadata.modified() { - /// println!("{:?}", time); + /// println!("{time:?}"); /// } else { /// println!("Not supported on this platform"); /// } @@ -1113,7 +1257,7 @@ impl Metadata { /// /// # Errors /// - /// This field may not be available on all platforms, and will return an + /// This field might not be available on all platforms, and will return an /// `Err` on platforms where it is not available. /// /// # Examples @@ -1125,7 +1269,7 @@ impl Metadata { /// let metadata = fs::metadata("foo.txt")?; /// /// if let Ok(time) = metadata.accessed() { - /// println!("{:?}", time); + /// println!("{time:?}"); /// } else { /// println!("Not supported on this platform"); /// } @@ -1145,7 +1289,7 @@ impl Metadata { /// /// # Errors /// - /// This field may not be available on all platforms, and will return an + /// This field might not be available on all platforms, and will return an /// `Err` on platforms or filesystems where it is not available. /// /// # Examples @@ -1157,7 +1301,7 @@ impl Metadata { /// let metadata = fs::metadata("foo.txt")?; /// /// if let Ok(time) = metadata.created() { - /// println!("{:?}", time); + /// println!("{time:?}"); /// } else { /// println!("Not supported on this platform or filesystem"); /// } @@ -1181,7 +1325,7 @@ impl fmt::Debug for Metadata { .field("modified", &self.modified()) .field("accessed", &self.accessed()) .field("created", &self.created()) - .finish() + .finish_non_exhaustive() } } @@ -1197,9 +1341,61 @@ impl FromInner for Metadata { } } +impl FileTimes { + /// Create a new `FileTimes` with no times set. + /// + /// Using the resulting `FileTimes` in [`File::set_times`] will not modify any timestamps. + #[unstable(feature = "file_set_times", issue = "98245")] + pub fn new() -> Self { + Self::default() + } + + /// Set the last access time of a file. + #[unstable(feature = "file_set_times", issue = "98245")] + pub fn set_accessed(mut self, t: SystemTime) -> Self { + self.0.set_accessed(t.into_inner()); + self + } + + /// Set the last modified time of a file. + #[unstable(feature = "file_set_times", issue = "98245")] + pub fn set_modified(mut self, t: SystemTime) -> Self { + self.0.set_modified(t.into_inner()); + self + } +} + impl Permissions { /// Returns `true` if these permissions describe a readonly (unwritable) file. /// + /// # Note + /// + /// This function does not take Access Control Lists (ACLs) or Unix group + /// membership into account. + /// + /// # Windows + /// + /// On Windows this returns [`FILE_ATTRIBUTE_READONLY`](https://docs.microsoft.com/en-us/windows/win32/fileio/file-attribute-constants). + /// If `FILE_ATTRIBUTE_READONLY` is set then writes to the file will fail + /// but the user may still have permission to change this flag. If + /// `FILE_ATTRIBUTE_READONLY` is *not* set then writes may still fail due + /// to lack of write permission. + /// The behavior of this attribute for directories depends on the Windows + /// version. + /// + /// # Unix (including macOS) + /// + /// On Unix-based platforms this checks if *any* of the owner, group or others + /// write permission bits are set. It does not check if the current + /// user is in the file's assigned group. It also does not check ACLs. + /// Therefore even if this returns true you may not be able to write to the + /// file, and vice versa. The [`PermissionsExt`] trait gives direct access + /// to the permission bits but also does not read ACLs. If you need to + /// accurately know whether or not a file is writable use the `access()` + /// function from libc. + /// + /// [`PermissionsExt`]: crate::os::unix::fs::PermissionsExt + /// /// # Examples /// /// ```no_run @@ -1213,6 +1409,7 @@ impl Permissions { /// Ok(()) /// } /// ``` + #[must_use = "call `set_readonly` to modify the readonly flag"] #[stable(feature = "rust1", since = "1.0.0")] pub fn readonly(&self) -> bool { self.0.readonly() @@ -1224,10 +1421,40 @@ impl Permissions { /// using the resulting `Permission` will update file permissions to allow /// writing. /// - /// This operation does **not** modify the filesystem. To modify the - /// filesystem use the [`fs::set_permissions`] function. + /// This operation does **not** modify the files attributes. This only + /// changes the in-memory value of these attributes for this `Permissions` + /// instance. To modify the files attributes use the [`set_permissions`] + /// function which commits these attribute changes to the file. + /// + /// # Note + /// + /// `set_readonly(false)` makes the file *world-writable* on Unix. + /// You can use the [`PermissionsExt`] trait on Unix to avoid this issue. + /// + /// It also does not take Access Control Lists (ACLs) or Unix group + /// membership into account. + /// + /// # Windows /// - /// [`fs::set_permissions`]: fn.set_permissions.html + /// On Windows this sets or clears [`FILE_ATTRIBUTE_READONLY`](https://docs.microsoft.com/en-us/windows/win32/fileio/file-attribute-constants). + /// If `FILE_ATTRIBUTE_READONLY` is set then writes to the file will fail + /// but the user may still have permission to change this flag. If + /// `FILE_ATTRIBUTE_READONLY` is *not* set then the write may still fail if + /// the user does not have permission to write to the file. + /// + /// In Windows 7 and earlier this attribute prevents deleting empty + /// directories. It does not prevent modifying the directory contents. + /// On later versions of Windows this attribute is ignored for directories. + /// + /// # Unix (including macOS) + /// + /// On Unix-based platforms this sets or clears the write access bit for + /// the owner, group *and* others, equivalent to `chmod a+w ` + /// or `chmod a-w ` respectively. The latter will grant write access + /// to all users! You can use the [`PermissionsExt`] trait on Unix + /// to avoid this issue. + /// + /// [`PermissionsExt`]: crate::os::unix::fs::PermissionsExt /// /// # Examples /// @@ -1241,7 +1468,8 @@ impl Permissions { /// /// permissions.set_readonly(true); /// - /// // filesystem doesn't change + /// // filesystem doesn't change, only the in memory state of the + /// // readonly permission /// assert_eq!(false, metadata.permissions().readonly()); /// /// // just this particular `permissions`. @@ -1261,8 +1489,8 @@ impl FileType { /// [`is_file`] and [`is_symlink`]; only zero or one of these /// tests may pass. /// - /// [`is_file`]: struct.FileType.html#method.is_file - /// [`is_symlink`]: struct.FileType.html#method.is_symlink + /// [`is_file`]: FileType::is_file + /// [`is_symlink`]: FileType::is_symlink /// /// # Examples /// @@ -1277,18 +1505,25 @@ impl FileType { /// Ok(()) /// } /// ``` + #[must_use] #[stable(feature = "file_type", since = "1.1.0")] pub fn is_dir(&self) -> bool { self.0.is_dir() } /// Tests whether this file type represents a regular file. - /// The result is mutually exclusive to the results of + /// The result is mutually exclusive to the results of /// [`is_dir`] and [`is_symlink`]; only zero or one of these /// tests may pass. /// - /// [`is_dir`]: struct.FileType.html#method.is_dir - /// [`is_symlink`]: struct.FileType.html#method.is_symlink + /// When the goal is simply to read from (or write to) the source, the most + /// reliable way to test the source can be read (or written to) is to open + /// it. Only using `is_file` can break workflows like `diff <( prog_a )` on + /// a Unix-like system for example. See [`File::open`] or + /// [`OpenOptions::open`] for more information. + /// + /// [`is_dir`]: FileType::is_dir + /// [`is_symlink`]: FileType::is_symlink /// /// # Examples /// @@ -1303,6 +1538,7 @@ impl FileType { /// Ok(()) /// } /// ``` + #[must_use] #[stable(feature = "file_type", since = "1.1.0")] pub fn is_file(&self) -> bool { self.0.is_file() @@ -1319,12 +1555,11 @@ impl FileType { /// follows symbolic links, so [`is_symlink`] would always /// return `false` for the target file. /// - /// [`Metadata`]: struct.Metadata.html - /// [`fs::metadata`]: fn.metadata.html - /// [`fs::symlink_metadata`]: fn.symlink_metadata.html - /// [`is_dir`]: struct.FileType.html#method.is_dir - /// [`is_file`]: struct.FileType.html#method.is_file - /// [`is_symlink`]: struct.FileType.html#method.is_symlink + /// [`fs::metadata`]: metadata + /// [`fs::symlink_metadata`]: symlink_metadata + /// [`is_dir`]: FileType::is_dir + /// [`is_file`]: FileType::is_file + /// [`is_symlink`]: FileType::is_symlink /// /// # Examples /// @@ -1339,6 +1574,7 @@ impl FileType { /// Ok(()) /// } /// ``` + #[must_use] #[stable(feature = "file_type", since = "1.1.0")] pub fn is_symlink(&self) -> bool { self.0.is_symlink() @@ -1401,6 +1637,7 @@ impl DirEntry { /// ``` /// /// The exact text, of course, depends on what files you have in `.`. + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn path(&self) -> PathBuf { self.0.path() @@ -1409,7 +1646,10 @@ impl DirEntry { /// Returns the metadata for the file that this entry points at. /// /// This function will not traverse symlinks if this entry points at a - /// symlink. + /// symlink. To traverse symlinks use [`fs::metadata`] or [`fs::File::metadata`]. + /// + /// [`fs::metadata`]: metadata + /// [`fs::File::metadata`]: File::metadata /// /// # Platform-specific behavior /// @@ -1493,6 +1733,7 @@ impl DirEntry { /// } /// } /// ``` + #[must_use] #[stable(feature = "dir_entry_ext", since = "1.1.0")] pub fn file_name(&self) -> OsString { self.0.file_name() @@ -1524,7 +1765,7 @@ impl AsInner for DirEntry { /// and the `DeleteFile` function on Windows. /// Note that, this [may change in the future][changes]. /// -/// [changes]: ../io/index.html#platform-specific-behavior +/// [changes]: io#platform-specific-behavior /// /// # Errors /// @@ -1532,6 +1773,7 @@ impl AsInner for DirEntry { /// limited to just these cases: /// /// * `path` points to a directory. +/// * The file doesn't exist. /// * The user lacks permissions to remove the file. /// /// # Examples @@ -1558,10 +1800,10 @@ pub fn remove_file>(path: P) -> io::Result<()> { /// # Platform-specific behavior /// /// This function currently corresponds to the `stat` function on Unix -/// and the `GetFileAttributesEx` function on Windows. +/// and the `GetFileInformationByHandle` function on Windows. /// Note that, this [may change in the future][changes]. /// -/// [changes]: ../io/index.html#platform-specific-behavior +/// [changes]: io#platform-specific-behavior /// /// # Errors /// @@ -1592,10 +1834,10 @@ pub fn metadata>(path: P) -> io::Result { /// # Platform-specific behavior /// /// This function currently corresponds to the `lstat` function on Unix -/// and the `GetFileAttributesEx` function on Windows. +/// and the `GetFileInformationByHandle` function on Windows. /// Note that, this [may change in the future][changes]. /// -/// [changes]: ../io/index.html#platform-specific-behavior +/// [changes]: io#platform-specific-behavior /// /// # Errors /// @@ -1638,7 +1880,7 @@ pub fn symlink_metadata>(path: P) -> io::Result { /// /// Note that, this [may change in the future][changes]. /// -/// [changes]: ../io/index.html#platform-specific-behavior +/// [changes]: io#platform-specific-behavior /// /// # Errors /// @@ -1676,32 +1918,35 @@ pub fn rename, Q: AsRef>(from: P, to: Q) -> io::Result<()> /// the length of the `to` file as reported by `metadata`. /// /// If you’re wanting to copy the contents of one file to another and you’re -/// working with [`File`]s, see the [`io::copy`] function. -/// -/// [`io::copy`]: ../io/fn.copy.html -/// [`File`]: ./struct.File.html +/// working with [`File`]s, see the [`io::copy()`] function. /// /// # Platform-specific behavior /// /// This function currently corresponds to the `open` function in Unix /// with `O_RDONLY` for `from` and `O_WRONLY`, `O_CREAT`, and `O_TRUNC` for `to`. /// `O_CLOEXEC` is set for returned file descriptors. +/// +/// On Linux (including Android), this function attempts to use `copy_file_range(2)`, +/// and falls back to reading and writing if that is not possible. +/// /// On Windows, this function currently corresponds to `CopyFileEx`. Alternate /// NTFS streams are copied but only the size of the main stream is returned by -/// this function. On MacOS, this function corresponds to `fclonefileat` and -/// `fcopyfile`. -/// Note that, this [may change in the future][changes]. +/// this function. +/// +/// On MacOS, this function corresponds to `fclonefileat` and `fcopyfile`. /// -/// [changes]: ../io/index.html#platform-specific-behavior +/// Note that platform-specific behavior [may change in the future][changes]. +/// +/// [changes]: io#platform-specific-behavior /// /// # Errors /// /// This function will return an error in the following situations, but is not /// limited to just these cases: /// -/// * The `from` path is not a file. -/// * The `from` file does not exist. -/// * The current process does not have the permission rights to access +/// * `from` is neither a regular file nor a symlink to a regular file. +/// * `from` does not exist. +/// * The current process does not have the permission rights to read /// `from` or write `to`. /// /// # Examples @@ -1721,23 +1966,32 @@ pub fn copy, Q: AsRef>(from: P, to: Q) -> io::Result { /// Creates a new hard link on the filesystem. /// -/// The `dst` path will be a link pointing to the `src` path. Note that systems -/// often require these two paths to both be located on the same filesystem. +/// The `link` path will be a link pointing to the `original` path. Note that +/// systems often require these two paths to both be located on the same +/// filesystem. +/// +/// If `original` names a symbolic link, it is platform-specific whether the +/// symbolic link is followed. On platforms where it's possible to not follow +/// it, it is not followed, and the created hard link points to the symbolic +/// link itself. /// /// # Platform-specific behavior /// -/// This function currently corresponds to the `link` function on Unix -/// and the `CreateHardLink` function on Windows. +/// This function currently corresponds the `CreateHardLink` function on Windows. +/// On most Unix systems, it corresponds to the `linkat` function with no flags. +/// On Android, VxWorks, and Redox, it instead corresponds to the `link` function. +/// On MacOS, it uses the `linkat` function if it is available, but on very old +/// systems where `linkat` is not available, `link` is selected at runtime instead. /// Note that, this [may change in the future][changes]. /// -/// [changes]: ../io/index.html#platform-specific-behavior +/// [changes]: io#platform-specific-behavior /// /// # Errors /// /// This function will return an error in the following situations, but is not /// limited to just these cases: /// -/// * The `src` path is not a file or doesn't exist. +/// * The `original` path is not a file or doesn't exist. /// /// # Examples /// @@ -1750,22 +2004,21 @@ pub fn copy, Q: AsRef>(from: P, to: Q) -> io::Result { /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] -pub fn hard_link, Q: AsRef>(src: P, dst: Q) -> io::Result<()> { - fs_imp::link(src.as_ref(), dst.as_ref()) +pub fn hard_link, Q: AsRef>(original: P, link: Q) -> io::Result<()> { + fs_imp::link(original.as_ref(), link.as_ref()) } /// Creates a new symbolic link on the filesystem. /// -/// The `dst` path will be a symbolic link pointing to the `src` path. +/// The `link` path will be a symbolic link pointing to the `original` path. /// On Windows, this will be a file symlink, not a directory symlink; /// for this reason, the platform-specific [`std::os::unix::fs::symlink`] /// and [`std::os::windows::fs::symlink_file`] or [`symlink_dir`] should be /// used instead to make the intent explicit. /// -/// [`std::os::unix::fs::symlink`]: ../os/unix/fs/fn.symlink.html -/// [`std::os::windows::fs::symlink_file`]: ../os/windows/fs/fn.symlink_file.html -/// [`symlink_dir`]: ../os/windows/fs/fn.symlink_dir.html -/// +/// [`std::os::unix::fs::symlink`]: crate::os::unix::fs::symlink +/// [`std::os::windows::fs::symlink_file`]: crate::os::windows::fs::symlink_file +/// [`symlink_dir`]: crate::os::windows::fs::symlink_dir /// /// # Examples /// @@ -1778,13 +2031,13 @@ pub fn hard_link, Q: AsRef>(src: P, dst: Q) -> io::Result<( /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] -#[rustc_deprecated( +#[deprecated( since = "1.1.0", - reason = "replaced with std::os::unix::fs::symlink and \ - std::os::windows::fs::{symlink_file, symlink_dir}" + note = "replaced with std::os::unix::fs::symlink and \ + std::os::windows::fs::{symlink_file, symlink_dir}" )] -pub fn soft_link, Q: AsRef>(src: P, dst: Q) -> io::Result<()> { - fs_imp::symlink(src.as_ref(), dst.as_ref()) +pub fn soft_link, Q: AsRef>(original: P, link: Q) -> io::Result<()> { + fs_imp::symlink(original.as_ref(), link.as_ref()) } /// Reads a symbolic link, returning the file that the link points to. @@ -1796,7 +2049,7 @@ pub fn soft_link, Q: AsRef>(src: P, dst: Q) -> io::Result<( /// `FILE_FLAG_BACKUP_SEMANTICS` flags on Windows. /// Note that, this [may change in the future][changes]. /// -/// [changes]: ../io/index.html#platform-specific-behavior +/// [changes]: io#platform-specific-behavior /// /// # Errors /// @@ -1836,7 +2089,7 @@ pub fn read_link>(path: P) -> io::Result { /// with other applications (if passed to the application on the command-line, /// or written to a file another application may read). /// -/// [changes]: ../io/index.html#platform-specific-behavior +/// [changes]: io#platform-specific-behavior /// [path]: https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file /// /// # Errors @@ -1857,6 +2110,8 @@ pub fn read_link>(path: P) -> io::Result { /// Ok(()) /// } /// ``` +#[doc(alias = "realpath")] +#[doc(alias = "GetFinalPathNameByHandle")] #[stable(feature = "fs_canonicalize", since = "1.5.0")] pub fn canonicalize>(path: P) -> io::Result { fs_imp::canonicalize(path.as_ref()) @@ -1870,7 +2125,7 @@ pub fn canonicalize>(path: P) -> io::Result { /// and the `CreateDirectory` function on Windows. /// Note that, this [may change in the future][changes]. /// -/// [changes]: ../io/index.html#platform-specific-behavior +/// [changes]: io#platform-specific-behavior /// /// **NOTE**: If a parent of the given path doesn't exist, this function will /// return an error. To create a directory and all its missing parents at the @@ -1887,8 +2142,6 @@ pub fn canonicalize>(path: P) -> io::Result { /// function.) /// * `path` already exists. /// -/// [`create_dir_all`]: fn.create_dir_all.html -/// /// # Examples /// /// ```no_run @@ -1899,6 +2152,7 @@ pub fn canonicalize>(path: P) -> io::Result { /// Ok(()) /// } /// ``` +#[doc(alias = "mkdir")] #[stable(feature = "rust1", since = "1.0.0")] pub fn create_dir>(path: P) -> io::Result<()> { DirBuilder::new().create(path.as_ref()) @@ -1913,7 +2167,7 @@ pub fn create_dir>(path: P) -> io::Result<()> { /// and the `CreateDirectory` function on Windows. /// Note that, this [may change in the future][changes]. /// -/// [changes]: ../io/index.html#platform-specific-behavior +/// [changes]: io#platform-specific-behavior /// /// # Errors /// @@ -1931,7 +2185,7 @@ pub fn create_dir>(path: P) -> io::Result<()> { /// concurrently from multiple threads or processes is guaranteed not to fail /// due to a race condition with itself. /// -/// [`fs::create_dir`]: fn.create_dir.html +/// [`fs::create_dir`]: create_dir /// /// # Examples /// @@ -1948,7 +2202,7 @@ pub fn create_dir_all>(path: P) -> io::Result<()> { DirBuilder::new().recursive(true).create(path.as_ref()) } -/// Removes an existing, empty directory. +/// Removes an empty directory. /// /// # Platform-specific behavior /// @@ -1956,13 +2210,15 @@ pub fn create_dir_all>(path: P) -> io::Result<()> { /// and the `RemoveDirectory` function on Windows. /// Note that, this [may change in the future][changes]. /// -/// [changes]: ../io/index.html#platform-specific-behavior +/// [changes]: io#platform-specific-behavior /// /// # Errors /// /// This function will return an error in the following situations, but is not /// limited to just these cases: /// +/// * `path` doesn't exist. +/// * `path` isn't a directory. /// * The user lacks permissions to remove the directory at the provided `path`. /// * The directory isn't empty. /// @@ -1976,6 +2232,7 @@ pub fn create_dir_all>(path: P) -> io::Result<()> { /// Ok(()) /// } /// ``` +#[doc(alias = "rmdir")] #[stable(feature = "rust1", since = "1.0.0")] pub fn remove_dir>(path: P) -> io::Result<()> { fs_imp::rmdir(path.as_ref()) @@ -1989,19 +2246,24 @@ pub fn remove_dir>(path: P) -> io::Result<()> { /// /// # Platform-specific behavior /// -/// This function currently corresponds to `opendir`, `lstat`, `rm` and `rmdir` functions on Unix -/// and the `FindFirstFile`, `GetFileAttributesEx`, `DeleteFile`, and `RemoveDirectory` functions -/// on Windows. -/// Note that, this [may change in the future][changes]. +/// This function currently corresponds to `openat`, `fdopendir`, `unlinkat` and `lstat` functions +/// on Unix (except for macOS before version 10.10 and REDOX) and the `CreateFileW`, +/// `GetFileInformationByHandleEx`, `SetFileInformationByHandle`, and `NtCreateFile` functions on +/// Windows. Note that, this [may change in the future][changes]. +/// +/// [changes]: io#platform-specific-behavior /// -/// [changes]: ../io/index.html#platform-specific-behavior +/// On macOS before version 10.10 and REDOX, as well as when running in Miri for any target, this +/// function is not protected against time-of-check to time-of-use (TOCTOU) race conditions, and +/// should not be used in security-sensitive code on those platforms. All other platforms are +/// protected. /// /// # Errors /// /// See [`fs::remove_file`] and [`fs::remove_dir`]. /// -/// [`fs::remove_file`]: fn.remove_file.html -/// [`fs::remove_dir`]: fn.remove_dir.html +/// [`fs::remove_file`]: remove_file +/// [`fs::remove_dir`]: remove_dir /// /// # Examples /// @@ -2020,11 +2282,10 @@ pub fn remove_dir_all>(path: P) -> io::Result<()> { /// Returns an iterator over the entries within a directory. /// -/// The iterator will yield instances of [`io::Result`]`<`[`DirEntry`]`>`. +/// The iterator will yield instances of [io::Result]<[DirEntry]>. /// New errors may be encountered after an iterator is initially constructed. -/// -/// [`io::Result`]: ../io/type.Result.html -/// [`DirEntry`]: struct.DirEntry.html +/// Entries for the current and parent directories (typically `.` and `..`) are +/// skipped. /// /// # Platform-specific behavior /// @@ -2033,7 +2294,7 @@ pub fn remove_dir_all>(path: P) -> io::Result<()> { /// currently corresponds to `readdir` on Unix and `FindNextFile` on Windows. /// Note that, this [may change in the future][changes]. /// -/// [changes]: ../io/index.html#platform-specific-behavior +/// [changes]: io#platform-specific-behavior /// /// The order in which this iterator returns entries is platform and filesystem /// dependent. @@ -2102,7 +2363,7 @@ pub fn read_dir>(path: P) -> io::Result { /// and the `SetFileAttributes` function on Windows. /// Note that, this [may change in the future][changes]. /// -/// [changes]: ../io/index.html#platform-specific-behavior +/// [changes]: io#platform-specific-behavior /// /// # Errors /// @@ -2141,6 +2402,7 @@ impl DirBuilder { /// let builder = DirBuilder::new(); /// ``` #[stable(feature = "dir_builder", since = "1.6.0")] + #[must_use] pub fn new() -> DirBuilder { DirBuilder { inner: fs_imp::DirBuilder::new(), recursive: false } } @@ -2206,7 +2468,10 @@ impl DirBuilder { match path.parent() { Some(p) => self.create_dir_all(p)?, None => { - return Err(io::Error::new(io::ErrorKind::Other, "failed to create whole tree")); + return Err(io::const_io_error!( + io::ErrorKind::Uncategorized, + "failed to create whole tree", + )); } } match self.inner.mkdir(path) { @@ -2223,1348 +2488,34 @@ impl AsInnerMut for DirBuilder { } } -#[cfg(all(test, not(any(target_os = "cloudabi", target_os = "emscripten", target_env = "sgx"))))] -mod tests { - use crate::io::prelude::*; - - use crate::fs::{self, File, OpenOptions}; - use crate::io::{ErrorKind, SeekFrom}; - use crate::path::Path; - use crate::str; - use crate::sys_common::io::test::{tmpdir, TempDir}; - use crate::thread; - - use rand::{rngs::StdRng, RngCore, SeedableRng}; - - #[cfg(unix)] - use crate::os::unix::fs::symlink as symlink_dir; - #[cfg(unix)] - use crate::os::unix::fs::symlink as symlink_file; - #[cfg(unix)] - use crate::os::unix::fs::symlink as symlink_junction; - #[cfg(windows)] - use crate::os::windows::fs::{symlink_dir, symlink_file}; - #[cfg(windows)] - use crate::sys::fs::symlink_junction; - - macro_rules! check { - ($e:expr) => { - match $e { - Ok(t) => t, - Err(e) => panic!("{} failed with: {}", stringify!($e), e), - } - }; - } - - #[cfg(windows)] - macro_rules! error { - ($e:expr, $s:expr) => { - match $e { - Ok(_) => panic!("Unexpected success. Should've been: {:?}", $s), - Err(ref err) => assert!( - err.raw_os_error() == Some($s), - format!("`{}` did not have a code of `{}`", err, $s) - ), - } - }; - } - - #[cfg(unix)] - macro_rules! error { - ($e:expr, $s:expr) => { - error_contains!($e, $s) - }; - } - - macro_rules! error_contains { - ($e:expr, $s:expr) => { - match $e { - Ok(_) => panic!("Unexpected success. Should've been: {:?}", $s), - Err(ref err) => assert!( - err.to_string().contains($s), - format!("`{}` did not contain `{}`", err, $s) - ), - } - }; - } - - // Several test fail on windows if the user does not have permission to - // create symlinks (the `SeCreateSymbolicLinkPrivilege`). Instead of - // disabling these test on Windows, use this function to test whether we - // have permission, and return otherwise. This way, we still don't run these - // tests most of the time, but at least we do if the user has the right - // permissions. - pub fn got_symlink_permission(tmpdir: &TempDir) -> bool { - if cfg!(unix) { - return true; - } - let link = tmpdir.join("some_hopefully_unique_link_name"); - - match symlink_file(r"nonexisting_target", link) { - Ok(_) => true, - // ERROR_PRIVILEGE_NOT_HELD = 1314 - Err(ref err) if err.raw_os_error() == Some(1314) => false, - Err(_) => true, - } - } - - #[test] - fn file_test_io_smoke_test() { - let message = "it's alright. have a good time"; - let tmpdir = tmpdir(); - let filename = &tmpdir.join("file_rt_io_file_test.txt"); - { - let mut write_stream = check!(File::create(filename)); - check!(write_stream.write(message.as_bytes())); - } - { - let mut read_stream = check!(File::open(filename)); - let mut read_buf = [0; 1028]; - let read_str = match check!(read_stream.read(&mut read_buf)) { - 0 => panic!("shouldn't happen"), - n => str::from_utf8(&read_buf[..n]).unwrap().to_string(), - }; - assert_eq!(read_str, message); - } - check!(fs::remove_file(filename)); - } - - #[test] - fn invalid_path_raises() { - let tmpdir = tmpdir(); - let filename = &tmpdir.join("file_that_does_not_exist.txt"); - let result = File::open(filename); - - #[cfg(all(unix, not(target_os = "vxworks")))] - error!(result, "No such file or directory"); - #[cfg(target_os = "vxworks")] - error!(result, "no such file or directory"); - #[cfg(windows)] - error!(result, 2); // ERROR_FILE_NOT_FOUND - } - - #[test] - fn file_test_iounlinking_invalid_path_should_raise_condition() { - let tmpdir = tmpdir(); - let filename = &tmpdir.join("file_another_file_that_does_not_exist.txt"); - - let result = fs::remove_file(filename); - - #[cfg(all(unix, not(target_os = "vxworks")))] - error!(result, "No such file or directory"); - #[cfg(target_os = "vxworks")] - error!(result, "no such file or directory"); - #[cfg(windows)] - error!(result, 2); // ERROR_FILE_NOT_FOUND - } - - #[test] - fn file_test_io_non_positional_read() { - let message: &str = "ten-four"; - let mut read_mem = [0; 8]; - let tmpdir = tmpdir(); - let filename = &tmpdir.join("file_rt_io_file_test_positional.txt"); - { - let mut rw_stream = check!(File::create(filename)); - check!(rw_stream.write(message.as_bytes())); - } - { - let mut read_stream = check!(File::open(filename)); - { - let read_buf = &mut read_mem[0..4]; - check!(read_stream.read(read_buf)); - } - { - let read_buf = &mut read_mem[4..8]; - check!(read_stream.read(read_buf)); - } - } - check!(fs::remove_file(filename)); - let read_str = str::from_utf8(&read_mem).unwrap(); - assert_eq!(read_str, message); - } - - #[test] - fn file_test_io_seek_and_tell_smoke_test() { - let message = "ten-four"; - let mut read_mem = [0; 4]; - let set_cursor = 4 as u64; - let tell_pos_pre_read; - let tell_pos_post_read; - let tmpdir = tmpdir(); - let filename = &tmpdir.join("file_rt_io_file_test_seeking.txt"); - { - let mut rw_stream = check!(File::create(filename)); - check!(rw_stream.write(message.as_bytes())); - } - { - let mut read_stream = check!(File::open(filename)); - check!(read_stream.seek(SeekFrom::Start(set_cursor))); - tell_pos_pre_read = check!(read_stream.seek(SeekFrom::Current(0))); - check!(read_stream.read(&mut read_mem)); - tell_pos_post_read = check!(read_stream.seek(SeekFrom::Current(0))); - } - check!(fs::remove_file(filename)); - let read_str = str::from_utf8(&read_mem).unwrap(); - assert_eq!(read_str, &message[4..8]); - assert_eq!(tell_pos_pre_read, set_cursor); - assert_eq!(tell_pos_post_read, message.len() as u64); - } - - #[test] - fn file_test_io_seek_and_write() { - let initial_msg = "food-is-yummy"; - let overwrite_msg = "-the-bar!!"; - let final_msg = "foo-the-bar!!"; - let seek_idx = 3; - let mut read_mem = [0; 13]; - let tmpdir = tmpdir(); - let filename = &tmpdir.join("file_rt_io_file_test_seek_and_write.txt"); - { - let mut rw_stream = check!(File::create(filename)); - check!(rw_stream.write(initial_msg.as_bytes())); - check!(rw_stream.seek(SeekFrom::Start(seek_idx))); - check!(rw_stream.write(overwrite_msg.as_bytes())); - } - { - let mut read_stream = check!(File::open(filename)); - check!(read_stream.read(&mut read_mem)); - } - check!(fs::remove_file(filename)); - let read_str = str::from_utf8(&read_mem).unwrap(); - assert!(read_str == final_msg); - } - - #[test] - fn file_test_io_seek_shakedown() { - // 01234567890123 - let initial_msg = "qwer-asdf-zxcv"; - let chunk_one: &str = "qwer"; - let chunk_two: &str = "asdf"; - let chunk_three: &str = "zxcv"; - let mut read_mem = [0; 4]; - let tmpdir = tmpdir(); - let filename = &tmpdir.join("file_rt_io_file_test_seek_shakedown.txt"); - { - let mut rw_stream = check!(File::create(filename)); - check!(rw_stream.write(initial_msg.as_bytes())); - } - { - let mut read_stream = check!(File::open(filename)); - - check!(read_stream.seek(SeekFrom::End(-4))); - check!(read_stream.read(&mut read_mem)); - assert_eq!(str::from_utf8(&read_mem).unwrap(), chunk_three); - - check!(read_stream.seek(SeekFrom::Current(-9))); - check!(read_stream.read(&mut read_mem)); - assert_eq!(str::from_utf8(&read_mem).unwrap(), chunk_two); - - check!(read_stream.seek(SeekFrom::Start(0))); - check!(read_stream.read(&mut read_mem)); - assert_eq!(str::from_utf8(&read_mem).unwrap(), chunk_one); - } - check!(fs::remove_file(filename)); - } - - #[test] - fn file_test_io_eof() { - let tmpdir = tmpdir(); - let filename = tmpdir.join("file_rt_io_file_test_eof.txt"); - let mut buf = [0; 256]; - { - let oo = OpenOptions::new().create_new(true).write(true).read(true).clone(); - let mut rw = check!(oo.open(&filename)); - assert_eq!(check!(rw.read(&mut buf)), 0); - assert_eq!(check!(rw.read(&mut buf)), 0); - } - check!(fs::remove_file(&filename)); - } - - #[test] - #[cfg(unix)] - fn file_test_io_read_write_at() { - use crate::os::unix::fs::FileExt; - - let tmpdir = tmpdir(); - let filename = tmpdir.join("file_rt_io_file_test_read_write_at.txt"); - let mut buf = [0; 256]; - let write1 = "asdf"; - let write2 = "qwer-"; - let write3 = "-zxcv"; - let content = "qwer-asdf-zxcv"; - { - let oo = OpenOptions::new().create_new(true).write(true).read(true).clone(); - let mut rw = check!(oo.open(&filename)); - assert_eq!(check!(rw.write_at(write1.as_bytes(), 5)), write1.len()); - assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 0); - assert_eq!(check!(rw.read_at(&mut buf, 5)), write1.len()); - assert_eq!(str::from_utf8(&buf[..write1.len()]), Ok(write1)); - assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 0); - assert_eq!(check!(rw.read_at(&mut buf[..write2.len()], 0)), write2.len()); - assert_eq!(str::from_utf8(&buf[..write2.len()]), Ok("\0\0\0\0\0")); - assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 0); - assert_eq!(check!(rw.write(write2.as_bytes())), write2.len()); - assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 5); - assert_eq!(check!(rw.read(&mut buf)), write1.len()); - assert_eq!(str::from_utf8(&buf[..write1.len()]), Ok(write1)); - assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 9); - assert_eq!(check!(rw.read_at(&mut buf[..write2.len()], 0)), write2.len()); - assert_eq!(str::from_utf8(&buf[..write2.len()]), Ok(write2)); - assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 9); - assert_eq!(check!(rw.write_at(write3.as_bytes(), 9)), write3.len()); - assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 9); - } - { - let mut read = check!(File::open(&filename)); - assert_eq!(check!(read.read_at(&mut buf, 0)), content.len()); - assert_eq!(str::from_utf8(&buf[..content.len()]), Ok(content)); - assert_eq!(check!(read.seek(SeekFrom::Current(0))), 0); - assert_eq!(check!(read.seek(SeekFrom::End(-5))), 9); - assert_eq!(check!(read.read_at(&mut buf, 0)), content.len()); - assert_eq!(str::from_utf8(&buf[..content.len()]), Ok(content)); - assert_eq!(check!(read.seek(SeekFrom::Current(0))), 9); - assert_eq!(check!(read.read(&mut buf)), write3.len()); - assert_eq!(str::from_utf8(&buf[..write3.len()]), Ok(write3)); - assert_eq!(check!(read.seek(SeekFrom::Current(0))), 14); - assert_eq!(check!(read.read_at(&mut buf, 0)), content.len()); - assert_eq!(str::from_utf8(&buf[..content.len()]), Ok(content)); - assert_eq!(check!(read.seek(SeekFrom::Current(0))), 14); - assert_eq!(check!(read.read_at(&mut buf, 14)), 0); - assert_eq!(check!(read.read_at(&mut buf, 15)), 0); - assert_eq!(check!(read.seek(SeekFrom::Current(0))), 14); - } - check!(fs::remove_file(&filename)); - } - - #[test] - #[cfg(unix)] - fn set_get_unix_permissions() { - use crate::os::unix::fs::PermissionsExt; - - let tmpdir = tmpdir(); - let filename = &tmpdir.join("set_get_unix_permissions"); - check!(fs::create_dir(filename)); - let mask = 0o7777; - - check!(fs::set_permissions(filename, fs::Permissions::from_mode(0))); - let metadata0 = check!(fs::metadata(filename)); - assert_eq!(mask & metadata0.permissions().mode(), 0); - - check!(fs::set_permissions(filename, fs::Permissions::from_mode(0o1777))); - let metadata1 = check!(fs::metadata(filename)); - #[cfg(all(unix, not(target_os = "vxworks")))] - assert_eq!(mask & metadata1.permissions().mode(), 0o1777); - #[cfg(target_os = "vxworks")] - assert_eq!(mask & metadata1.permissions().mode(), 0o0777); - } - - #[test] - #[cfg(windows)] - fn file_test_io_seek_read_write() { - use crate::os::windows::fs::FileExt; - - let tmpdir = tmpdir(); - let filename = tmpdir.join("file_rt_io_file_test_seek_read_write.txt"); - let mut buf = [0; 256]; - let write1 = "asdf"; - let write2 = "qwer-"; - let write3 = "-zxcv"; - let content = "qwer-asdf-zxcv"; - { - let oo = OpenOptions::new().create_new(true).write(true).read(true).clone(); - let mut rw = check!(oo.open(&filename)); - assert_eq!(check!(rw.seek_write(write1.as_bytes(), 5)), write1.len()); - assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 9); - assert_eq!(check!(rw.seek_read(&mut buf, 5)), write1.len()); - assert_eq!(str::from_utf8(&buf[..write1.len()]), Ok(write1)); - assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 9); - assert_eq!(check!(rw.seek(SeekFrom::Start(0))), 0); - assert_eq!(check!(rw.write(write2.as_bytes())), write2.len()); - assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 5); - assert_eq!(check!(rw.read(&mut buf)), write1.len()); - assert_eq!(str::from_utf8(&buf[..write1.len()]), Ok(write1)); - assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 9); - assert_eq!(check!(rw.seek_read(&mut buf[..write2.len()], 0)), write2.len()); - assert_eq!(str::from_utf8(&buf[..write2.len()]), Ok(write2)); - assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 5); - assert_eq!(check!(rw.seek_write(write3.as_bytes(), 9)), write3.len()); - assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 14); - } - { - let mut read = check!(File::open(&filename)); - assert_eq!(check!(read.seek_read(&mut buf, 0)), content.len()); - assert_eq!(str::from_utf8(&buf[..content.len()]), Ok(content)); - assert_eq!(check!(read.seek(SeekFrom::Current(0))), 14); - assert_eq!(check!(read.seek(SeekFrom::End(-5))), 9); - assert_eq!(check!(read.seek_read(&mut buf, 0)), content.len()); - assert_eq!(str::from_utf8(&buf[..content.len()]), Ok(content)); - assert_eq!(check!(read.seek(SeekFrom::Current(0))), 14); - assert_eq!(check!(read.seek(SeekFrom::End(-5))), 9); - assert_eq!(check!(read.read(&mut buf)), write3.len()); - assert_eq!(str::from_utf8(&buf[..write3.len()]), Ok(write3)); - assert_eq!(check!(read.seek(SeekFrom::Current(0))), 14); - assert_eq!(check!(read.seek_read(&mut buf, 0)), content.len()); - assert_eq!(str::from_utf8(&buf[..content.len()]), Ok(content)); - assert_eq!(check!(read.seek(SeekFrom::Current(0))), 14); - assert_eq!(check!(read.seek_read(&mut buf, 14)), 0); - assert_eq!(check!(read.seek_read(&mut buf, 15)), 0); - } - check!(fs::remove_file(&filename)); - } - - #[test] - fn file_test_stat_is_correct_on_is_file() { - let tmpdir = tmpdir(); - let filename = &tmpdir.join("file_stat_correct_on_is_file.txt"); - { - let mut opts = OpenOptions::new(); - let mut fs = check!(opts.read(true).write(true).create(true).open(filename)); - let msg = "hw"; - fs.write(msg.as_bytes()).unwrap(); - - let fstat_res = check!(fs.metadata()); - assert!(fstat_res.is_file()); - } - let stat_res_fn = check!(fs::metadata(filename)); - assert!(stat_res_fn.is_file()); - let stat_res_meth = check!(filename.metadata()); - assert!(stat_res_meth.is_file()); - check!(fs::remove_file(filename)); - } - - #[test] - fn file_test_stat_is_correct_on_is_dir() { - let tmpdir = tmpdir(); - let filename = &tmpdir.join("file_stat_correct_on_is_dir"); - check!(fs::create_dir(filename)); - let stat_res_fn = check!(fs::metadata(filename)); - assert!(stat_res_fn.is_dir()); - let stat_res_meth = check!(filename.metadata()); - assert!(stat_res_meth.is_dir()); - check!(fs::remove_dir(filename)); - } - - #[test] - fn file_test_fileinfo_false_when_checking_is_file_on_a_directory() { - let tmpdir = tmpdir(); - let dir = &tmpdir.join("fileinfo_false_on_dir"); - check!(fs::create_dir(dir)); - assert!(!dir.is_file()); - check!(fs::remove_dir(dir)); - } - - #[test] - fn file_test_fileinfo_check_exists_before_and_after_file_creation() { - let tmpdir = tmpdir(); - let file = &tmpdir.join("fileinfo_check_exists_b_and_a.txt"); - check!(check!(File::create(file)).write(b"foo")); - assert!(file.exists()); - check!(fs::remove_file(file)); - assert!(!file.exists()); - } - - #[test] - fn file_test_directoryinfo_check_exists_before_and_after_mkdir() { - let tmpdir = tmpdir(); - let dir = &tmpdir.join("before_and_after_dir"); - assert!(!dir.exists()); - check!(fs::create_dir(dir)); - assert!(dir.exists()); - assert!(dir.is_dir()); - check!(fs::remove_dir(dir)); - assert!(!dir.exists()); - } - - #[test] - fn file_test_directoryinfo_readdir() { - let tmpdir = tmpdir(); - let dir = &tmpdir.join("di_readdir"); - check!(fs::create_dir(dir)); - let prefix = "foo"; - for n in 0..3 { - let f = dir.join(&format!("{}.txt", n)); - let mut w = check!(File::create(&f)); - let msg_str = format!("{}{}", prefix, n.to_string()); - let msg = msg_str.as_bytes(); - check!(w.write(msg)); - } - let files = check!(fs::read_dir(dir)); - let mut mem = [0; 4]; - for f in files { - let f = f.unwrap().path(); - { - let n = f.file_stem().unwrap(); - check!(check!(File::open(&f)).read(&mut mem)); - let read_str = str::from_utf8(&mem).unwrap(); - let expected = format!("{}{}", prefix, n.to_str().unwrap()); - assert_eq!(expected, read_str); - } - check!(fs::remove_file(&f)); - } - check!(fs::remove_dir(dir)); - } - - #[test] - fn file_create_new_already_exists_error() { - let tmpdir = tmpdir(); - let file = &tmpdir.join("file_create_new_error_exists"); - check!(fs::File::create(file)); - let e = fs::OpenOptions::new().write(true).create_new(true).open(file).unwrap_err(); - assert_eq!(e.kind(), ErrorKind::AlreadyExists); - } - - #[test] - fn mkdir_path_already_exists_error() { - let tmpdir = tmpdir(); - let dir = &tmpdir.join("mkdir_error_twice"); - check!(fs::create_dir(dir)); - let e = fs::create_dir(dir).unwrap_err(); - assert_eq!(e.kind(), ErrorKind::AlreadyExists); - } - - #[test] - fn recursive_mkdir() { - let tmpdir = tmpdir(); - let dir = tmpdir.join("d1/d2"); - check!(fs::create_dir_all(&dir)); - assert!(dir.is_dir()) - } - - #[test] - fn recursive_mkdir_failure() { - let tmpdir = tmpdir(); - let dir = tmpdir.join("d1"); - let file = dir.join("f1"); - - check!(fs::create_dir_all(&dir)); - check!(File::create(&file)); - - let result = fs::create_dir_all(&file); - - assert!(result.is_err()); - } - - #[test] - fn concurrent_recursive_mkdir() { - for _ in 0..100 { - let dir = tmpdir(); - let mut dir = dir.join("a"); - for _ in 0..40 { - dir = dir.join("a"); - } - let mut join = vec![]; - for _ in 0..8 { - let dir = dir.clone(); - join.push(thread::spawn(move || { - check!(fs::create_dir_all(&dir)); - })) - } - - // No `Display` on result of `join()` - join.drain(..).map(|join| join.join().unwrap()).count(); - } - } - - #[test] - fn recursive_mkdir_slash() { - check!(fs::create_dir_all(Path::new("/"))); - } - - #[test] - fn recursive_mkdir_dot() { - check!(fs::create_dir_all(Path::new("."))); - } - - #[test] - fn recursive_mkdir_empty() { - check!(fs::create_dir_all(Path::new(""))); - } - - #[test] - fn recursive_rmdir() { - let tmpdir = tmpdir(); - let d1 = tmpdir.join("d1"); - let dt = d1.join("t"); - let dtt = dt.join("t"); - let d2 = tmpdir.join("d2"); - let canary = d2.join("do_not_delete"); - check!(fs::create_dir_all(&dtt)); - check!(fs::create_dir_all(&d2)); - check!(check!(File::create(&canary)).write(b"foo")); - check!(symlink_junction(&d2, &dt.join("d2"))); - let _ = symlink_file(&canary, &d1.join("canary")); - check!(fs::remove_dir_all(&d1)); - - assert!(!d1.is_dir()); - assert!(canary.exists()); - } - - #[test] - fn recursive_rmdir_of_symlink() { - // test we do not recursively delete a symlink but only dirs. - let tmpdir = tmpdir(); - let link = tmpdir.join("d1"); - let dir = tmpdir.join("d2"); - let canary = dir.join("do_not_delete"); - check!(fs::create_dir_all(&dir)); - check!(check!(File::create(&canary)).write(b"foo")); - check!(symlink_junction(&dir, &link)); - check!(fs::remove_dir_all(&link)); - - assert!(!link.is_dir()); - assert!(canary.exists()); - } - - #[test] - // only Windows makes a distinction between file and directory symlinks. - #[cfg(windows)] - fn recursive_rmdir_of_file_symlink() { - let tmpdir = tmpdir(); - if !got_symlink_permission(&tmpdir) { - return; - }; - - let f1 = tmpdir.join("f1"); - let f2 = tmpdir.join("f2"); - check!(check!(File::create(&f1)).write(b"foo")); - check!(symlink_file(&f1, &f2)); - match fs::remove_dir_all(&f2) { - Ok(..) => panic!("wanted a failure"), - Err(..) => {} - } - } - - #[test] - fn unicode_path_is_dir() { - assert!(Path::new(".").is_dir()); - assert!(!Path::new("test/stdtest/fs.rs").is_dir()); - - let tmpdir = tmpdir(); - - let mut dirpath = tmpdir.path().to_path_buf(); - dirpath.push("test-가一ー你好"); - check!(fs::create_dir(&dirpath)); - assert!(dirpath.is_dir()); - - let mut filepath = dirpath; - filepath.push("unicode-file-\u{ac00}\u{4e00}\u{30fc}\u{4f60}\u{597d}.rs"); - check!(File::create(&filepath)); // ignore return; touch only - assert!(!filepath.is_dir()); - assert!(filepath.exists()); - } - - #[test] - fn unicode_path_exists() { - assert!(Path::new(".").exists()); - assert!(!Path::new("test/nonexistent-bogus-path").exists()); - - let tmpdir = tmpdir(); - let unicode = tmpdir.path(); - let unicode = unicode.join("test-각丁ー再见"); - check!(fs::create_dir(&unicode)); - assert!(unicode.exists()); - assert!(!Path::new("test/unicode-bogus-path-각丁ー再见").exists()); - } - - #[test] - fn copy_file_does_not_exist() { - let from = Path::new("test/nonexistent-bogus-path"); - let to = Path::new("test/other-bogus-path"); - - match fs::copy(&from, &to) { - Ok(..) => panic!(), - Err(..) => { - assert!(!from.exists()); - assert!(!to.exists()); - } - } - } - - #[test] - fn copy_src_does_not_exist() { - let tmpdir = tmpdir(); - let from = Path::new("test/nonexistent-bogus-path"); - let to = tmpdir.join("out.txt"); - check!(check!(File::create(&to)).write(b"hello")); - assert!(fs::copy(&from, &to).is_err()); - assert!(!from.exists()); - let mut v = Vec::new(); - check!(check!(File::open(&to)).read_to_end(&mut v)); - assert_eq!(v, b"hello"); - } - - #[test] - fn copy_file_ok() { - let tmpdir = tmpdir(); - let input = tmpdir.join("in.txt"); - let out = tmpdir.join("out.txt"); - - check!(check!(File::create(&input)).write(b"hello")); - check!(fs::copy(&input, &out)); - let mut v = Vec::new(); - check!(check!(File::open(&out)).read_to_end(&mut v)); - assert_eq!(v, b"hello"); - - assert_eq!(check!(input.metadata()).permissions(), check!(out.metadata()).permissions()); - } - - #[test] - fn copy_file_dst_dir() { - let tmpdir = tmpdir(); - let out = tmpdir.join("out"); - - check!(File::create(&out)); - match fs::copy(&*out, tmpdir.path()) { - Ok(..) => panic!(), - Err(..) => {} - } - } - - #[test] - fn copy_file_dst_exists() { - let tmpdir = tmpdir(); - let input = tmpdir.join("in"); - let output = tmpdir.join("out"); - - check!(check!(File::create(&input)).write("foo".as_bytes())); - check!(check!(File::create(&output)).write("bar".as_bytes())); - check!(fs::copy(&input, &output)); - - let mut v = Vec::new(); - check!(check!(File::open(&output)).read_to_end(&mut v)); - assert_eq!(v, b"foo".to_vec()); - } - - #[test] - fn copy_file_src_dir() { - let tmpdir = tmpdir(); - let out = tmpdir.join("out"); - - match fs::copy(tmpdir.path(), &out) { - Ok(..) => panic!(), - Err(..) => {} - } - assert!(!out.exists()); - } - - #[test] - fn copy_file_preserves_perm_bits() { - let tmpdir = tmpdir(); - let input = tmpdir.join("in.txt"); - let out = tmpdir.join("out.txt"); - - let attr = check!(check!(File::create(&input)).metadata()); - let mut p = attr.permissions(); - p.set_readonly(true); - check!(fs::set_permissions(&input, p)); - check!(fs::copy(&input, &out)); - assert!(check!(out.metadata()).permissions().readonly()); - check!(fs::set_permissions(&input, attr.permissions())); - check!(fs::set_permissions(&out, attr.permissions())); - } - - #[test] - #[cfg(windows)] - fn copy_file_preserves_streams() { - let tmp = tmpdir(); - check!(check!(File::create(tmp.join("in.txt:bunny"))).write("carrot".as_bytes())); - assert_eq!(check!(fs::copy(tmp.join("in.txt"), tmp.join("out.txt"))), 0); - assert_eq!(check!(tmp.join("out.txt").metadata()).len(), 0); - let mut v = Vec::new(); - check!(check!(File::open(tmp.join("out.txt:bunny"))).read_to_end(&mut v)); - assert_eq!(v, b"carrot".to_vec()); - } - - #[test] - fn copy_file_returns_metadata_len() { - let tmp = tmpdir(); - let in_path = tmp.join("in.txt"); - let out_path = tmp.join("out.txt"); - check!(check!(File::create(&in_path)).write(b"lettuce")); - #[cfg(windows)] - check!(check!(File::create(tmp.join("in.txt:bunny"))).write(b"carrot")); - let copied_len = check!(fs::copy(&in_path, &out_path)); - assert_eq!(check!(out_path.metadata()).len(), copied_len); - } - - #[test] - fn copy_file_follows_dst_symlink() { - let tmp = tmpdir(); - if !got_symlink_permission(&tmp) { - return; - }; - - let in_path = tmp.join("in.txt"); - let out_path = tmp.join("out.txt"); - let out_path_symlink = tmp.join("out_symlink.txt"); - - check!(fs::write(&in_path, "foo")); - check!(fs::write(&out_path, "bar")); - check!(symlink_file(&out_path, &out_path_symlink)); - - check!(fs::copy(&in_path, &out_path_symlink)); - - assert!(check!(out_path_symlink.symlink_metadata()).file_type().is_symlink()); - assert_eq!(check!(fs::read(&out_path_symlink)), b"foo".to_vec()); - assert_eq!(check!(fs::read(&out_path)), b"foo".to_vec()); - } - - #[test] - fn symlinks_work() { - let tmpdir = tmpdir(); - if !got_symlink_permission(&tmpdir) { - return; - }; - - let input = tmpdir.join("in.txt"); - let out = tmpdir.join("out.txt"); - - check!(check!(File::create(&input)).write("foobar".as_bytes())); - check!(symlink_file(&input, &out)); - assert!(check!(out.symlink_metadata()).file_type().is_symlink()); - assert_eq!(check!(fs::metadata(&out)).len(), check!(fs::metadata(&input)).len()); - let mut v = Vec::new(); - check!(check!(File::open(&out)).read_to_end(&mut v)); - assert_eq!(v, b"foobar".to_vec()); - } - - #[test] - fn symlink_noexist() { - // Symlinks can point to things that don't exist - let tmpdir = tmpdir(); - if !got_symlink_permission(&tmpdir) { - return; - }; - - // Use a relative path for testing. Symlinks get normalized by Windows, - // so we may not get the same path back for absolute paths - check!(symlink_file(&"foo", &tmpdir.join("bar"))); - assert_eq!(check!(fs::read_link(&tmpdir.join("bar"))).to_str().unwrap(), "foo"); - } - - #[test] - fn read_link() { - if cfg!(windows) { - // directory symlink - assert_eq!( - check!(fs::read_link(r"C:\Users\All Users")).to_str().unwrap(), - r"C:\ProgramData" - ); - // junction - assert_eq!( - check!(fs::read_link(r"C:\Users\Default User")).to_str().unwrap(), - r"C:\Users\Default" - ); - // junction with special permissions - assert_eq!( - check!(fs::read_link(r"C:\Documents and Settings\")).to_str().unwrap(), - r"C:\Users" - ); - } - let tmpdir = tmpdir(); - let link = tmpdir.join("link"); - if !got_symlink_permission(&tmpdir) { - return; - }; - check!(symlink_file(&"foo", &link)); - assert_eq!(check!(fs::read_link(&link)).to_str().unwrap(), "foo"); - } - - #[test] - fn readlink_not_symlink() { - let tmpdir = tmpdir(); - match fs::read_link(tmpdir.path()) { - Ok(..) => panic!("wanted a failure"), - Err(..) => {} - } - } - - #[test] - fn links_work() { - let tmpdir = tmpdir(); - let input = tmpdir.join("in.txt"); - let out = tmpdir.join("out.txt"); - - check!(check!(File::create(&input)).write("foobar".as_bytes())); - check!(fs::hard_link(&input, &out)); - assert_eq!(check!(fs::metadata(&out)).len(), check!(fs::metadata(&input)).len()); - assert_eq!(check!(fs::metadata(&out)).len(), check!(input.metadata()).len()); - let mut v = Vec::new(); - check!(check!(File::open(&out)).read_to_end(&mut v)); - assert_eq!(v, b"foobar".to_vec()); - - // can't link to yourself - match fs::hard_link(&input, &input) { - Ok(..) => panic!("wanted a failure"), - Err(..) => {} - } - // can't link to something that doesn't exist - match fs::hard_link(&tmpdir.join("foo"), &tmpdir.join("bar")) { - Ok(..) => panic!("wanted a failure"), - Err(..) => {} - } - } - - #[test] - fn chmod_works() { - let tmpdir = tmpdir(); - let file = tmpdir.join("in.txt"); - - check!(File::create(&file)); - let attr = check!(fs::metadata(&file)); - assert!(!attr.permissions().readonly()); - let mut p = attr.permissions(); - p.set_readonly(true); - check!(fs::set_permissions(&file, p.clone())); - let attr = check!(fs::metadata(&file)); - assert!(attr.permissions().readonly()); - - match fs::set_permissions(&tmpdir.join("foo"), p.clone()) { - Ok(..) => panic!("wanted an error"), - Err(..) => {} - } - - p.set_readonly(false); - check!(fs::set_permissions(&file, p)); - } - - #[test] - fn fchmod_works() { - let tmpdir = tmpdir(); - let path = tmpdir.join("in.txt"); - - let file = check!(File::create(&path)); - let attr = check!(fs::metadata(&path)); - assert!(!attr.permissions().readonly()); - let mut p = attr.permissions(); - p.set_readonly(true); - check!(file.set_permissions(p.clone())); - let attr = check!(fs::metadata(&path)); - assert!(attr.permissions().readonly()); - - p.set_readonly(false); - check!(file.set_permissions(p)); - } - - #[test] - fn sync_doesnt_kill_anything() { - let tmpdir = tmpdir(); - let path = tmpdir.join("in.txt"); - - let mut file = check!(File::create(&path)); - check!(file.sync_all()); - check!(file.sync_data()); - check!(file.write(b"foo")); - check!(file.sync_all()); - check!(file.sync_data()); - } - - #[test] - fn truncate_works() { - let tmpdir = tmpdir(); - let path = tmpdir.join("in.txt"); - - let mut file = check!(File::create(&path)); - check!(file.write(b"foo")); - check!(file.sync_all()); - - // Do some simple things with truncation - assert_eq!(check!(file.metadata()).len(), 3); - check!(file.set_len(10)); - assert_eq!(check!(file.metadata()).len(), 10); - check!(file.write(b"bar")); - check!(file.sync_all()); - assert_eq!(check!(file.metadata()).len(), 10); - - let mut v = Vec::new(); - check!(check!(File::open(&path)).read_to_end(&mut v)); - assert_eq!(v, b"foobar\0\0\0\0".to_vec()); - - // Truncate to a smaller length, don't seek, and then write something. - // Ensure that the intermediate zeroes are all filled in (we have `seek`ed - // past the end of the file). - check!(file.set_len(2)); - assert_eq!(check!(file.metadata()).len(), 2); - check!(file.write(b"wut")); - check!(file.sync_all()); - assert_eq!(check!(file.metadata()).len(), 9); - let mut v = Vec::new(); - check!(check!(File::open(&path)).read_to_end(&mut v)); - assert_eq!(v, b"fo\0\0\0\0wut".to_vec()); - } - - #[test] - fn open_flavors() { - use crate::fs::OpenOptions as OO; - fn c(t: &T) -> T { - t.clone() - } - - let tmpdir = tmpdir(); - - let mut r = OO::new(); - r.read(true); - let mut w = OO::new(); - w.write(true); - let mut rw = OO::new(); - rw.read(true).write(true); - let mut a = OO::new(); - a.append(true); - let mut ra = OO::new(); - ra.read(true).append(true); - - #[cfg(windows)] - let invalid_options = 87; // ERROR_INVALID_PARAMETER - #[cfg(all(unix, not(target_os = "vxworks")))] - let invalid_options = "Invalid argument"; - #[cfg(target_os = "vxworks")] - let invalid_options = "invalid argument"; - - // Test various combinations of creation modes and access modes. - // - // Allowed: - // creation mode | read | write | read-write | append | read-append | - // :-----------------------|:-----:|:-----:|:----------:|:------:|:-----------:| - // not set (open existing) | X | X | X | X | X | - // create | | X | X | X | X | - // truncate | | X | X | | | - // create and truncate | | X | X | | | - // create_new | | X | X | X | X | - // - // tested in reverse order, so 'create_new' creates the file, and 'open existing' opens it. - - // write-only - check!(c(&w).create_new(true).open(&tmpdir.join("a"))); - check!(c(&w).create(true).truncate(true).open(&tmpdir.join("a"))); - check!(c(&w).truncate(true).open(&tmpdir.join("a"))); - check!(c(&w).create(true).open(&tmpdir.join("a"))); - check!(c(&w).open(&tmpdir.join("a"))); - - // read-only - error!(c(&r).create_new(true).open(&tmpdir.join("b")), invalid_options); - error!(c(&r).create(true).truncate(true).open(&tmpdir.join("b")), invalid_options); - error!(c(&r).truncate(true).open(&tmpdir.join("b")), invalid_options); - error!(c(&r).create(true).open(&tmpdir.join("b")), invalid_options); - check!(c(&r).open(&tmpdir.join("a"))); // try opening the file created with write_only - - // read-write - check!(c(&rw).create_new(true).open(&tmpdir.join("c"))); - check!(c(&rw).create(true).truncate(true).open(&tmpdir.join("c"))); - check!(c(&rw).truncate(true).open(&tmpdir.join("c"))); - check!(c(&rw).create(true).open(&tmpdir.join("c"))); - check!(c(&rw).open(&tmpdir.join("c"))); - - // append - check!(c(&a).create_new(true).open(&tmpdir.join("d"))); - error!(c(&a).create(true).truncate(true).open(&tmpdir.join("d")), invalid_options); - error!(c(&a).truncate(true).open(&tmpdir.join("d")), invalid_options); - check!(c(&a).create(true).open(&tmpdir.join("d"))); - check!(c(&a).open(&tmpdir.join("d"))); - - // read-append - check!(c(&ra).create_new(true).open(&tmpdir.join("e"))); - error!(c(&ra).create(true).truncate(true).open(&tmpdir.join("e")), invalid_options); - error!(c(&ra).truncate(true).open(&tmpdir.join("e")), invalid_options); - check!(c(&ra).create(true).open(&tmpdir.join("e"))); - check!(c(&ra).open(&tmpdir.join("e"))); - - // Test opening a file without setting an access mode - let mut blank = OO::new(); - error!(blank.create(true).open(&tmpdir.join("f")), invalid_options); - - // Test write works - check!(check!(File::create(&tmpdir.join("h"))).write("foobar".as_bytes())); - - // Test write fails for read-only - check!(r.open(&tmpdir.join("h"))); - { - let mut f = check!(r.open(&tmpdir.join("h"))); - assert!(f.write("wut".as_bytes()).is_err()); - } - - // Test write overwrites - { - let mut f = check!(c(&w).open(&tmpdir.join("h"))); - check!(f.write("baz".as_bytes())); - } - { - let mut f = check!(c(&r).open(&tmpdir.join("h"))); - let mut b = vec![0; 6]; - check!(f.read(&mut b)); - assert_eq!(b, "bazbar".as_bytes()); - } - - // Test truncate works - { - let mut f = check!(c(&w).truncate(true).open(&tmpdir.join("h"))); - check!(f.write("foo".as_bytes())); - } - assert_eq!(check!(fs::metadata(&tmpdir.join("h"))).len(), 3); - - // Test append works - assert_eq!(check!(fs::metadata(&tmpdir.join("h"))).len(), 3); - { - let mut f = check!(c(&a).open(&tmpdir.join("h"))); - check!(f.write("bar".as_bytes())); - } - assert_eq!(check!(fs::metadata(&tmpdir.join("h"))).len(), 6); - - // Test .append(true) equals .write(true).append(true) - { - let mut f = check!(c(&w).append(true).open(&tmpdir.join("h"))); - check!(f.write("baz".as_bytes())); - } - assert_eq!(check!(fs::metadata(&tmpdir.join("h"))).len(), 9); - } - - #[test] - fn _assert_send_sync() { - fn _assert_send_sync() {} - _assert_send_sync::(); - } - - #[test] - fn binary_file() { - let mut bytes = [0; 1024]; - StdRng::from_entropy().fill_bytes(&mut bytes); - - let tmpdir = tmpdir(); - - check!(check!(File::create(&tmpdir.join("test"))).write(&bytes)); - let mut v = Vec::new(); - check!(check!(File::open(&tmpdir.join("test"))).read_to_end(&mut v)); - assert!(v == &bytes[..]); - } - - #[test] - fn write_then_read() { - let mut bytes = [0; 1024]; - StdRng::from_entropy().fill_bytes(&mut bytes); - - let tmpdir = tmpdir(); - - check!(fs::write(&tmpdir.join("test"), &bytes[..])); - let v = check!(fs::read(&tmpdir.join("test"))); - assert!(v == &bytes[..]); - - check!(fs::write(&tmpdir.join("not-utf8"), &[0xFF])); - error_contains!( - fs::read_to_string(&tmpdir.join("not-utf8")), - "stream did not contain valid UTF-8" - ); - - let s = "𐁁𐀓𐀠𐀴𐀍"; - check!(fs::write(&tmpdir.join("utf8"), s.as_bytes())); - let string = check!(fs::read_to_string(&tmpdir.join("utf8"))); - assert_eq!(string, s); - } - - #[test] - fn file_try_clone() { - let tmpdir = tmpdir(); - - let mut f1 = check!( - OpenOptions::new().read(true).write(true).create(true).open(&tmpdir.join("test")) - ); - let mut f2 = check!(f1.try_clone()); - - check!(f1.write_all(b"hello world")); - check!(f1.seek(SeekFrom::Start(2))); - - let mut buf = vec![]; - check!(f2.read_to_end(&mut buf)); - assert_eq!(buf, b"llo world"); - drop(f2); - - check!(f1.write_all(b"!")); - } - - #[test] - #[cfg(not(windows))] - fn unlink_readonly() { - let tmpdir = tmpdir(); - let path = tmpdir.join("file"); - check!(File::create(&path)); - let mut perm = check!(fs::metadata(&path)).permissions(); - perm.set_readonly(true); - check!(fs::set_permissions(&path, perm)); - check!(fs::remove_file(&path)); - } - - #[test] - fn mkdir_trailing_slash() { - let tmpdir = tmpdir(); - let path = tmpdir.join("file"); - check!(fs::create_dir_all(&path.join("a/"))); - } - - #[test] - fn canonicalize_works_simple() { - let tmpdir = tmpdir(); - let tmpdir = fs::canonicalize(tmpdir.path()).unwrap(); - let file = tmpdir.join("test"); - File::create(&file).unwrap(); - assert_eq!(fs::canonicalize(&file).unwrap(), file); - } - - #[test] - fn realpath_works() { - let tmpdir = tmpdir(); - if !got_symlink_permission(&tmpdir) { - return; - }; - - let tmpdir = fs::canonicalize(tmpdir.path()).unwrap(); - let file = tmpdir.join("test"); - let dir = tmpdir.join("test2"); - let link = dir.join("link"); - let linkdir = tmpdir.join("test3"); - - File::create(&file).unwrap(); - fs::create_dir(&dir).unwrap(); - symlink_file(&file, &link).unwrap(); - symlink_dir(&dir, &linkdir).unwrap(); - - assert!(link.symlink_metadata().unwrap().file_type().is_symlink()); - - assert_eq!(fs::canonicalize(&tmpdir).unwrap(), tmpdir); - assert_eq!(fs::canonicalize(&file).unwrap(), file); - assert_eq!(fs::canonicalize(&link).unwrap(), file); - assert_eq!(fs::canonicalize(&linkdir).unwrap(), dir); - assert_eq!(fs::canonicalize(&linkdir.join("link")).unwrap(), file); - } - - #[test] - fn realpath_works_tricky() { - let tmpdir = tmpdir(); - if !got_symlink_permission(&tmpdir) { - return; - }; - - let tmpdir = fs::canonicalize(tmpdir.path()).unwrap(); - let a = tmpdir.join("a"); - let b = a.join("b"); - let c = b.join("c"); - let d = a.join("d"); - let e = d.join("e"); - let f = a.join("f"); - - fs::create_dir_all(&b).unwrap(); - fs::create_dir_all(&d).unwrap(); - File::create(&f).unwrap(); - if cfg!(not(windows)) { - symlink_file("../d/e", &c).unwrap(); - symlink_file("../f", &e).unwrap(); - } - if cfg!(windows) { - symlink_file(r"..\d\e", &c).unwrap(); - symlink_file(r"..\f", &e).unwrap(); - } - - assert_eq!(fs::canonicalize(&c).unwrap(), f); - assert_eq!(fs::canonicalize(&e).unwrap(), f); - } - - #[test] - fn dir_entry_methods() { - let tmpdir = tmpdir(); - - fs::create_dir_all(&tmpdir.join("a")).unwrap(); - File::create(&tmpdir.join("b")).unwrap(); - - for file in tmpdir.path().read_dir().unwrap().map(|f| f.unwrap()) { - let fname = file.file_name(); - match fname.to_str() { - Some("a") => { - assert!(file.file_type().unwrap().is_dir()); - assert!(file.metadata().unwrap().is_dir()); - } - Some("b") => { - assert!(file.file_type().unwrap().is_file()); - assert!(file.metadata().unwrap().is_file()); - } - f => panic!("unknown file name: {:?}", f), - } - } - } - - #[test] - fn dir_entry_debug() { - let tmpdir = tmpdir(); - File::create(&tmpdir.join("b")).unwrap(); - let mut read_dir = tmpdir.path().read_dir().unwrap(); - let dir_entry = read_dir.next().unwrap().unwrap(); - let actual = format!("{:?}", dir_entry); - let expected = format!("DirEntry({:?})", dir_entry.0.path()); - assert_eq!(actual, expected); - } - - #[test] - fn read_dir_not_found() { - let res = fs::read_dir("/path/that/does/not/exist"); - assert_eq!(res.err().unwrap().kind(), ErrorKind::NotFound); - } - - #[test] - fn create_dir_all_with_junctions() { - let tmpdir = tmpdir(); - let target = tmpdir.join("target"); - - let junction = tmpdir.join("junction"); - let b = junction.join("a/b"); - - let link = tmpdir.join("link"); - let d = link.join("c/d"); - - fs::create_dir(&target).unwrap(); - - check!(symlink_junction(&target, &junction)); - check!(fs::create_dir_all(&b)); - // the junction itself is not a directory, but `is_dir()` on a Path - // follows links - assert!(junction.is_dir()); - assert!(b.exists()); - - if !got_symlink_permission(&tmpdir) { - return; - }; - check!(symlink_dir(&target, &link)); - check!(fs::create_dir_all(&d)); - assert!(link.is_dir()); - assert!(d.exists()); - } - - #[test] - fn metadata_access_times() { - let tmpdir = tmpdir(); - - let b = tmpdir.join("b"); - File::create(&b).unwrap(); - - let a = check!(fs::metadata(&tmpdir.path())); - let b = check!(fs::metadata(&b)); - - assert_eq!(check!(a.accessed()), check!(a.accessed())); - assert_eq!(check!(a.modified()), check!(a.modified())); - assert_eq!(check!(b.accessed()), check!(b.modified())); - - if cfg!(target_os = "macos") || cfg!(target_os = "windows") { - check!(a.created()); - check!(b.created()); - } - - if cfg!(target_os = "linux") { - // Not always available - match (a.created(), b.created()) { - (Ok(t1), Ok(t2)) => assert!(t1 <= t2), - (Err(e1), Err(e2)) - if e1.kind() == ErrorKind::Other && e2.kind() == ErrorKind::Other => {} - (a, b) => panic!( - "creation time must be always supported or not supported: {:?} {:?}", - a, b, - ), - } - } - } +/// Returns `Ok(true)` if the path points at an existing entity. +/// +/// This function will traverse symbolic links to query information about the +/// destination file. In case of broken symbolic links this will return `Ok(false)`. +/// +/// As opposed to the [`Path::exists`] method, this one doesn't silently ignore errors +/// unrelated to the path not existing. (E.g. it will return `Err(_)` in case of permission +/// denied on some of the parent directories.) +/// +/// Note that while this avoids some pitfalls of the `exists()` method, it still can not +/// prevent time-of-check to time-of-use (TOCTOU) bugs. You should only use it in scenarios +/// where those bugs are not an issue. +/// +/// # Examples +/// +/// ```no_run +/// #![feature(fs_try_exists)] +/// use std::fs; +/// +/// assert!(!fs::try_exists("does_not_exist.txt").expect("Can't check existence of file does_not_exist.txt")); +/// assert!(fs::try_exists("/root/secret_file.txt").is_err()); +/// ``` +/// +/// [`Path::exists`]: crate::path::Path::exists +// FIXME: stabilization should modify documentation of `exists()` to recommend this method +// instead. +#[unstable(feature = "fs_try_exists", issue = "83186")] +#[inline] +pub fn try_exists>(path: P) -> io::Result { + fs_imp::try_exists(path.as_ref()) } diff --git a/crux-mir/lib/std/src/fs/tests.rs b/crux-mir/lib/std/src/fs/tests.rs new file mode 100644 index 000000000..eb582be01 --- /dev/null +++ b/crux-mir/lib/std/src/fs/tests.rs @@ -0,0 +1,1597 @@ +use crate::io::prelude::*; + +use crate::env; +use crate::fs::{self, File, OpenOptions}; +use crate::io::{ErrorKind, SeekFrom}; +use crate::path::Path; +use crate::str; +use crate::sync::Arc; +use crate::sys_common::io::test::{tmpdir, TempDir}; +use crate::thread; +use crate::time::{Duration, Instant}; + +use rand::RngCore; + +#[cfg(unix)] +use crate::os::unix::fs::symlink as symlink_dir; +#[cfg(unix)] +use crate::os::unix::fs::symlink as symlink_file; +#[cfg(unix)] +use crate::os::unix::fs::symlink as symlink_junction; +#[cfg(windows)] +use crate::os::windows::fs::{symlink_dir, symlink_file}; +#[cfg(windows)] +use crate::sys::fs::symlink_junction; +#[cfg(target_os = "macos")] +use crate::sys::weak::weak; +#[cfg(target_os = "macos")] +use libc::{c_char, c_int}; + +macro_rules! check { + ($e:expr) => { + match $e { + Ok(t) => t, + Err(e) => panic!("{} failed with: {e}", stringify!($e)), + } + }; +} + +#[cfg(windows)] +macro_rules! error { + ($e:expr, $s:expr) => { + match $e { + Ok(_) => panic!("Unexpected success. Should've been: {:?}", $s), + Err(ref err) => { + assert!(err.raw_os_error() == Some($s), "`{}` did not have a code of `{}`", err, $s) + } + } + }; +} + +#[cfg(unix)] +macro_rules! error { + ($e:expr, $s:expr) => { + error_contains!($e, $s) + }; +} + +macro_rules! error_contains { + ($e:expr, $s:expr) => { + match $e { + Ok(_) => panic!("Unexpected success. Should've been: {:?}", $s), + Err(ref err) => { + assert!(err.to_string().contains($s), "`{}` did not contain `{}`", err, $s) + } + } + }; +} + +// Several test fail on windows if the user does not have permission to +// create symlinks (the `SeCreateSymbolicLinkPrivilege`). Instead of +// disabling these test on Windows, use this function to test whether we +// have permission, and return otherwise. This way, we still don't run these +// tests most of the time, but at least we do if the user has the right +// permissions. +pub fn got_symlink_permission(tmpdir: &TempDir) -> bool { + if cfg!(unix) { + return true; + } + let link = tmpdir.join("some_hopefully_unique_link_name"); + + match symlink_file(r"nonexisting_target", link) { + // ERROR_PRIVILEGE_NOT_HELD = 1314 + Err(ref err) if err.raw_os_error() == Some(1314) => false, + Ok(_) | Err(_) => true, + } +} + +#[cfg(target_os = "macos")] +fn able_to_not_follow_symlinks_while_hard_linking() -> bool { + weak!(fn linkat(c_int, *const c_char, c_int, *const c_char, c_int) -> c_int); + linkat.get().is_some() +} + +#[cfg(not(target_os = "macos"))] +fn able_to_not_follow_symlinks_while_hard_linking() -> bool { + return true; +} + +#[test] +fn file_test_io_smoke_test() { + let message = "it's alright. have a good time"; + let tmpdir = tmpdir(); + let filename = &tmpdir.join("file_rt_io_file_test.txt"); + { + let mut write_stream = check!(File::create(filename)); + check!(write_stream.write(message.as_bytes())); + } + { + let mut read_stream = check!(File::open(filename)); + let mut read_buf = [0; 1028]; + let read_str = match check!(read_stream.read(&mut read_buf)) { + 0 => panic!("shouldn't happen"), + n => str::from_utf8(&read_buf[..n]).unwrap().to_string(), + }; + assert_eq!(read_str, message); + } + check!(fs::remove_file(filename)); +} + +#[test] +fn invalid_path_raises() { + let tmpdir = tmpdir(); + let filename = &tmpdir.join("file_that_does_not_exist.txt"); + let result = File::open(filename); + + #[cfg(all(unix, not(target_os = "vxworks")))] + error!(result, "No such file or directory"); + #[cfg(target_os = "vxworks")] + error!(result, "no such file or directory"); + #[cfg(windows)] + error!(result, 2); // ERROR_FILE_NOT_FOUND +} + +#[test] +fn file_test_iounlinking_invalid_path_should_raise_condition() { + let tmpdir = tmpdir(); + let filename = &tmpdir.join("file_another_file_that_does_not_exist.txt"); + + let result = fs::remove_file(filename); + + #[cfg(all(unix, not(target_os = "vxworks")))] + error!(result, "No such file or directory"); + #[cfg(target_os = "vxworks")] + error!(result, "no such file or directory"); + #[cfg(windows)] + error!(result, 2); // ERROR_FILE_NOT_FOUND +} + +#[test] +fn file_test_io_non_positional_read() { + let message: &str = "ten-four"; + let mut read_mem = [0; 8]; + let tmpdir = tmpdir(); + let filename = &tmpdir.join("file_rt_io_file_test_positional.txt"); + { + let mut rw_stream = check!(File::create(filename)); + check!(rw_stream.write(message.as_bytes())); + } + { + let mut read_stream = check!(File::open(filename)); + { + let read_buf = &mut read_mem[0..4]; + check!(read_stream.read(read_buf)); + } + { + let read_buf = &mut read_mem[4..8]; + check!(read_stream.read(read_buf)); + } + } + check!(fs::remove_file(filename)); + let read_str = str::from_utf8(&read_mem).unwrap(); + assert_eq!(read_str, message); +} + +#[test] +fn file_test_io_seek_and_tell_smoke_test() { + let message = "ten-four"; + let mut read_mem = [0; 4]; + let set_cursor = 4 as u64; + let tell_pos_pre_read; + let tell_pos_post_read; + let tmpdir = tmpdir(); + let filename = &tmpdir.join("file_rt_io_file_test_seeking.txt"); + { + let mut rw_stream = check!(File::create(filename)); + check!(rw_stream.write(message.as_bytes())); + } + { + let mut read_stream = check!(File::open(filename)); + check!(read_stream.seek(SeekFrom::Start(set_cursor))); + tell_pos_pre_read = check!(read_stream.seek(SeekFrom::Current(0))); + check!(read_stream.read(&mut read_mem)); + tell_pos_post_read = check!(read_stream.seek(SeekFrom::Current(0))); + } + check!(fs::remove_file(filename)); + let read_str = str::from_utf8(&read_mem).unwrap(); + assert_eq!(read_str, &message[4..8]); + assert_eq!(tell_pos_pre_read, set_cursor); + assert_eq!(tell_pos_post_read, message.len() as u64); +} + +#[test] +fn file_test_io_seek_and_write() { + let initial_msg = "food-is-yummy"; + let overwrite_msg = "-the-bar!!"; + let final_msg = "foo-the-bar!!"; + let seek_idx = 3; + let mut read_mem = [0; 13]; + let tmpdir = tmpdir(); + let filename = &tmpdir.join("file_rt_io_file_test_seek_and_write.txt"); + { + let mut rw_stream = check!(File::create(filename)); + check!(rw_stream.write(initial_msg.as_bytes())); + check!(rw_stream.seek(SeekFrom::Start(seek_idx))); + check!(rw_stream.write(overwrite_msg.as_bytes())); + } + { + let mut read_stream = check!(File::open(filename)); + check!(read_stream.read(&mut read_mem)); + } + check!(fs::remove_file(filename)); + let read_str = str::from_utf8(&read_mem).unwrap(); + assert!(read_str == final_msg); +} + +#[test] +fn file_test_io_seek_shakedown() { + // 01234567890123 + let initial_msg = "qwer-asdf-zxcv"; + let chunk_one: &str = "qwer"; + let chunk_two: &str = "asdf"; + let chunk_three: &str = "zxcv"; + let mut read_mem = [0; 4]; + let tmpdir = tmpdir(); + let filename = &tmpdir.join("file_rt_io_file_test_seek_shakedown.txt"); + { + let mut rw_stream = check!(File::create(filename)); + check!(rw_stream.write(initial_msg.as_bytes())); + } + { + let mut read_stream = check!(File::open(filename)); + + check!(read_stream.seek(SeekFrom::End(-4))); + check!(read_stream.read(&mut read_mem)); + assert_eq!(str::from_utf8(&read_mem).unwrap(), chunk_three); + + check!(read_stream.seek(SeekFrom::Current(-9))); + check!(read_stream.read(&mut read_mem)); + assert_eq!(str::from_utf8(&read_mem).unwrap(), chunk_two); + + check!(read_stream.seek(SeekFrom::Start(0))); + check!(read_stream.read(&mut read_mem)); + assert_eq!(str::from_utf8(&read_mem).unwrap(), chunk_one); + } + check!(fs::remove_file(filename)); +} + +#[test] +fn file_test_io_eof() { + let tmpdir = tmpdir(); + let filename = tmpdir.join("file_rt_io_file_test_eof.txt"); + let mut buf = [0; 256]; + { + let oo = OpenOptions::new().create_new(true).write(true).read(true).clone(); + let mut rw = check!(oo.open(&filename)); + assert_eq!(check!(rw.read(&mut buf)), 0); + assert_eq!(check!(rw.read(&mut buf)), 0); + } + check!(fs::remove_file(&filename)); +} + +#[test] +#[cfg(unix)] +fn file_test_io_read_write_at() { + use crate::os::unix::fs::FileExt; + + let tmpdir = tmpdir(); + let filename = tmpdir.join("file_rt_io_file_test_read_write_at.txt"); + let mut buf = [0; 256]; + let write1 = "asdf"; + let write2 = "qwer-"; + let write3 = "-zxcv"; + let content = "qwer-asdf-zxcv"; + { + let oo = OpenOptions::new().create_new(true).write(true).read(true).clone(); + let mut rw = check!(oo.open(&filename)); + assert_eq!(check!(rw.write_at(write1.as_bytes(), 5)), write1.len()); + assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 0); + assert_eq!(check!(rw.read_at(&mut buf, 5)), write1.len()); + assert_eq!(str::from_utf8(&buf[..write1.len()]), Ok(write1)); + assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 0); + assert_eq!(check!(rw.read_at(&mut buf[..write2.len()], 0)), write2.len()); + assert_eq!(str::from_utf8(&buf[..write2.len()]), Ok("\0\0\0\0\0")); + assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 0); + assert_eq!(check!(rw.write(write2.as_bytes())), write2.len()); + assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 5); + assert_eq!(check!(rw.read(&mut buf)), write1.len()); + assert_eq!(str::from_utf8(&buf[..write1.len()]), Ok(write1)); + assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 9); + assert_eq!(check!(rw.read_at(&mut buf[..write2.len()], 0)), write2.len()); + assert_eq!(str::from_utf8(&buf[..write2.len()]), Ok(write2)); + assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 9); + assert_eq!(check!(rw.write_at(write3.as_bytes(), 9)), write3.len()); + assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 9); + } + { + let mut read = check!(File::open(&filename)); + assert_eq!(check!(read.read_at(&mut buf, 0)), content.len()); + assert_eq!(str::from_utf8(&buf[..content.len()]), Ok(content)); + assert_eq!(check!(read.seek(SeekFrom::Current(0))), 0); + assert_eq!(check!(read.seek(SeekFrom::End(-5))), 9); + assert_eq!(check!(read.read_at(&mut buf, 0)), content.len()); + assert_eq!(str::from_utf8(&buf[..content.len()]), Ok(content)); + assert_eq!(check!(read.seek(SeekFrom::Current(0))), 9); + assert_eq!(check!(read.read(&mut buf)), write3.len()); + assert_eq!(str::from_utf8(&buf[..write3.len()]), Ok(write3)); + assert_eq!(check!(read.seek(SeekFrom::Current(0))), 14); + assert_eq!(check!(read.read_at(&mut buf, 0)), content.len()); + assert_eq!(str::from_utf8(&buf[..content.len()]), Ok(content)); + assert_eq!(check!(read.seek(SeekFrom::Current(0))), 14); + assert_eq!(check!(read.read_at(&mut buf, 14)), 0); + assert_eq!(check!(read.read_at(&mut buf, 15)), 0); + assert_eq!(check!(read.seek(SeekFrom::Current(0))), 14); + } + check!(fs::remove_file(&filename)); +} + +#[test] +#[cfg(unix)] +fn set_get_unix_permissions() { + use crate::os::unix::fs::PermissionsExt; + + let tmpdir = tmpdir(); + let filename = &tmpdir.join("set_get_unix_permissions"); + check!(fs::create_dir(filename)); + let mask = 0o7777; + + check!(fs::set_permissions(filename, fs::Permissions::from_mode(0))); + let metadata0 = check!(fs::metadata(filename)); + assert_eq!(mask & metadata0.permissions().mode(), 0); + + check!(fs::set_permissions(filename, fs::Permissions::from_mode(0o1777))); + let metadata1 = check!(fs::metadata(filename)); + #[cfg(all(unix, not(target_os = "vxworks")))] + assert_eq!(mask & metadata1.permissions().mode(), 0o1777); + #[cfg(target_os = "vxworks")] + assert_eq!(mask & metadata1.permissions().mode(), 0o0777); +} + +#[test] +#[cfg(windows)] +fn file_test_io_seek_read_write() { + use crate::os::windows::fs::FileExt; + + let tmpdir = tmpdir(); + let filename = tmpdir.join("file_rt_io_file_test_seek_read_write.txt"); + let mut buf = [0; 256]; + let write1 = "asdf"; + let write2 = "qwer-"; + let write3 = "-zxcv"; + let content = "qwer-asdf-zxcv"; + { + let oo = OpenOptions::new().create_new(true).write(true).read(true).clone(); + let mut rw = check!(oo.open(&filename)); + assert_eq!(check!(rw.seek_write(write1.as_bytes(), 5)), write1.len()); + assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 9); + assert_eq!(check!(rw.seek_read(&mut buf, 5)), write1.len()); + assert_eq!(str::from_utf8(&buf[..write1.len()]), Ok(write1)); + assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 9); + assert_eq!(check!(rw.seek(SeekFrom::Start(0))), 0); + assert_eq!(check!(rw.write(write2.as_bytes())), write2.len()); + assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 5); + assert_eq!(check!(rw.read(&mut buf)), write1.len()); + assert_eq!(str::from_utf8(&buf[..write1.len()]), Ok(write1)); + assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 9); + assert_eq!(check!(rw.seek_read(&mut buf[..write2.len()], 0)), write2.len()); + assert_eq!(str::from_utf8(&buf[..write2.len()]), Ok(write2)); + assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 5); + assert_eq!(check!(rw.seek_write(write3.as_bytes(), 9)), write3.len()); + assert_eq!(check!(rw.seek(SeekFrom::Current(0))), 14); + } + { + let mut read = check!(File::open(&filename)); + assert_eq!(check!(read.seek_read(&mut buf, 0)), content.len()); + assert_eq!(str::from_utf8(&buf[..content.len()]), Ok(content)); + assert_eq!(check!(read.seek(SeekFrom::Current(0))), 14); + assert_eq!(check!(read.seek(SeekFrom::End(-5))), 9); + assert_eq!(check!(read.seek_read(&mut buf, 0)), content.len()); + assert_eq!(str::from_utf8(&buf[..content.len()]), Ok(content)); + assert_eq!(check!(read.seek(SeekFrom::Current(0))), 14); + assert_eq!(check!(read.seek(SeekFrom::End(-5))), 9); + assert_eq!(check!(read.read(&mut buf)), write3.len()); + assert_eq!(str::from_utf8(&buf[..write3.len()]), Ok(write3)); + assert_eq!(check!(read.seek(SeekFrom::Current(0))), 14); + assert_eq!(check!(read.seek_read(&mut buf, 0)), content.len()); + assert_eq!(str::from_utf8(&buf[..content.len()]), Ok(content)); + assert_eq!(check!(read.seek(SeekFrom::Current(0))), 14); + assert_eq!(check!(read.seek_read(&mut buf, 14)), 0); + assert_eq!(check!(read.seek_read(&mut buf, 15)), 0); + } + check!(fs::remove_file(&filename)); +} + +#[test] +fn file_test_stat_is_correct_on_is_file() { + let tmpdir = tmpdir(); + let filename = &tmpdir.join("file_stat_correct_on_is_file.txt"); + { + let mut opts = OpenOptions::new(); + let mut fs = check!(opts.read(true).write(true).create(true).open(filename)); + let msg = "hw"; + fs.write(msg.as_bytes()).unwrap(); + + let fstat_res = check!(fs.metadata()); + assert!(fstat_res.is_file()); + } + let stat_res_fn = check!(fs::metadata(filename)); + assert!(stat_res_fn.is_file()); + let stat_res_meth = check!(filename.metadata()); + assert!(stat_res_meth.is_file()); + check!(fs::remove_file(filename)); +} + +#[test] +fn file_test_stat_is_correct_on_is_dir() { + let tmpdir = tmpdir(); + let filename = &tmpdir.join("file_stat_correct_on_is_dir"); + check!(fs::create_dir(filename)); + let stat_res_fn = check!(fs::metadata(filename)); + assert!(stat_res_fn.is_dir()); + let stat_res_meth = check!(filename.metadata()); + assert!(stat_res_meth.is_dir()); + check!(fs::remove_dir(filename)); +} + +#[test] +fn file_test_fileinfo_false_when_checking_is_file_on_a_directory() { + let tmpdir = tmpdir(); + let dir = &tmpdir.join("fileinfo_false_on_dir"); + check!(fs::create_dir(dir)); + assert!(!dir.is_file()); + check!(fs::remove_dir(dir)); +} + +#[test] +fn file_test_fileinfo_check_exists_before_and_after_file_creation() { + let tmpdir = tmpdir(); + let file = &tmpdir.join("fileinfo_check_exists_b_and_a.txt"); + check!(check!(File::create(file)).write(b"foo")); + assert!(file.exists()); + check!(fs::remove_file(file)); + assert!(!file.exists()); +} + +#[test] +fn file_test_directoryinfo_check_exists_before_and_after_mkdir() { + let tmpdir = tmpdir(); + let dir = &tmpdir.join("before_and_after_dir"); + assert!(!dir.exists()); + check!(fs::create_dir(dir)); + assert!(dir.exists()); + assert!(dir.is_dir()); + check!(fs::remove_dir(dir)); + assert!(!dir.exists()); +} + +#[test] +fn file_test_directoryinfo_readdir() { + let tmpdir = tmpdir(); + let dir = &tmpdir.join("di_readdir"); + check!(fs::create_dir(dir)); + let prefix = "foo"; + for n in 0..3 { + let f = dir.join(&format!("{n}.txt")); + let mut w = check!(File::create(&f)); + let msg_str = format!("{}{}", prefix, n.to_string()); + let msg = msg_str.as_bytes(); + check!(w.write(msg)); + } + let files = check!(fs::read_dir(dir)); + let mut mem = [0; 4]; + for f in files { + let f = f.unwrap().path(); + { + let n = f.file_stem().unwrap(); + check!(check!(File::open(&f)).read(&mut mem)); + let read_str = str::from_utf8(&mem).unwrap(); + let expected = format!("{}{}", prefix, n.to_str().unwrap()); + assert_eq!(expected, read_str); + } + check!(fs::remove_file(&f)); + } + check!(fs::remove_dir(dir)); +} + +#[test] +fn file_create_new_already_exists_error() { + let tmpdir = tmpdir(); + let file = &tmpdir.join("file_create_new_error_exists"); + check!(fs::File::create(file)); + let e = fs::OpenOptions::new().write(true).create_new(true).open(file).unwrap_err(); + assert_eq!(e.kind(), ErrorKind::AlreadyExists); +} + +#[test] +fn mkdir_path_already_exists_error() { + let tmpdir = tmpdir(); + let dir = &tmpdir.join("mkdir_error_twice"); + check!(fs::create_dir(dir)); + let e = fs::create_dir(dir).unwrap_err(); + assert_eq!(e.kind(), ErrorKind::AlreadyExists); +} + +#[test] +fn recursive_mkdir() { + let tmpdir = tmpdir(); + let dir = tmpdir.join("d1/d2"); + check!(fs::create_dir_all(&dir)); + assert!(dir.is_dir()) +} + +#[test] +fn recursive_mkdir_failure() { + let tmpdir = tmpdir(); + let dir = tmpdir.join("d1"); + let file = dir.join("f1"); + + check!(fs::create_dir_all(&dir)); + check!(File::create(&file)); + + let result = fs::create_dir_all(&file); + + assert!(result.is_err()); +} + +#[test] +fn concurrent_recursive_mkdir() { + for _ in 0..100 { + let dir = tmpdir(); + let mut dir = dir.join("a"); + for _ in 0..40 { + dir = dir.join("a"); + } + let mut join = vec![]; + for _ in 0..8 { + let dir = dir.clone(); + join.push(thread::spawn(move || { + check!(fs::create_dir_all(&dir)); + })) + } + + // No `Display` on result of `join()` + join.drain(..).map(|join| join.join().unwrap()).count(); + } +} + +#[test] +fn recursive_mkdir_slash() { + check!(fs::create_dir_all(Path::new("/"))); +} + +#[test] +fn recursive_mkdir_dot() { + check!(fs::create_dir_all(Path::new("."))); +} + +#[test] +fn recursive_mkdir_empty() { + check!(fs::create_dir_all(Path::new(""))); +} + +#[test] +fn recursive_rmdir() { + let tmpdir = tmpdir(); + let d1 = tmpdir.join("d1"); + let dt = d1.join("t"); + let dtt = dt.join("t"); + let d2 = tmpdir.join("d2"); + let canary = d2.join("do_not_delete"); + check!(fs::create_dir_all(&dtt)); + check!(fs::create_dir_all(&d2)); + check!(check!(File::create(&canary)).write(b"foo")); + check!(symlink_junction(&d2, &dt.join("d2"))); + let _ = symlink_file(&canary, &d1.join("canary")); + check!(fs::remove_dir_all(&d1)); + + assert!(!d1.is_dir()); + assert!(canary.exists()); +} + +#[test] +fn recursive_rmdir_of_symlink() { + // test we do not recursively delete a symlink but only dirs. + let tmpdir = tmpdir(); + let link = tmpdir.join("d1"); + let dir = tmpdir.join("d2"); + let canary = dir.join("do_not_delete"); + check!(fs::create_dir_all(&dir)); + check!(check!(File::create(&canary)).write(b"foo")); + check!(symlink_junction(&dir, &link)); + check!(fs::remove_dir_all(&link)); + + assert!(!link.is_dir()); + assert!(canary.exists()); +} + +#[test] +fn recursive_rmdir_of_file_fails() { + // test we do not delete a directly specified file. + let tmpdir = tmpdir(); + let canary = tmpdir.join("do_not_delete"); + check!(check!(File::create(&canary)).write(b"foo")); + let result = fs::remove_dir_all(&canary); + #[cfg(unix)] + error!(result, "Not a directory"); + #[cfg(windows)] + error!(result, 267); // ERROR_DIRECTORY - The directory name is invalid. + assert!(result.is_err()); + assert!(canary.exists()); +} + +#[test] +// only Windows makes a distinction between file and directory symlinks. +#[cfg(windows)] +fn recursive_rmdir_of_file_symlink() { + let tmpdir = tmpdir(); + if !got_symlink_permission(&tmpdir) { + return; + }; + + let f1 = tmpdir.join("f1"); + let f2 = tmpdir.join("f2"); + check!(check!(File::create(&f1)).write(b"foo")); + check!(symlink_file(&f1, &f2)); + match fs::remove_dir_all(&f2) { + Ok(..) => panic!("wanted a failure"), + Err(..) => {} + } +} + +#[test] +#[ignore] // takes too much time +fn recursive_rmdir_toctou() { + // Test for time-of-check to time-of-use issues. + // + // Scenario: + // The attacker wants to get directory contents deleted, to which they do not have access. + // They have a way to get a privileged Rust binary call `std::fs::remove_dir_all()` on a + // directory they control, e.g. in their home directory. + // + // The POC sets up the `attack_dest/attack_file` which the attacker wants to have deleted. + // The attacker repeatedly creates a directory and replaces it with a symlink from + // `victim_del` to `attack_dest` while the victim code calls `std::fs::remove_dir_all()` + // on `victim_del`. After a few seconds the attack has succeeded and + // `attack_dest/attack_file` is deleted. + let tmpdir = tmpdir(); + let victim_del_path = tmpdir.join("victim_del"); + let victim_del_path_clone = victim_del_path.clone(); + + // setup dest + let attack_dest_dir = tmpdir.join("attack_dest"); + let attack_dest_dir = attack_dest_dir.as_path(); + fs::create_dir(attack_dest_dir).unwrap(); + let attack_dest_file = tmpdir.join("attack_dest/attack_file"); + File::create(&attack_dest_file).unwrap(); + + let drop_canary_arc = Arc::new(()); + let drop_canary_weak = Arc::downgrade(&drop_canary_arc); + + eprintln!("x: {:?}", &victim_del_path); + + // victim just continuously removes `victim_del` + thread::spawn(move || { + while drop_canary_weak.upgrade().is_some() { + let _ = fs::remove_dir_all(&victim_del_path_clone); + } + }); + + // attacker (could of course be in a separate process) + let start_time = Instant::now(); + while Instant::now().duration_since(start_time) < Duration::from_secs(1000) { + if !attack_dest_file.exists() { + panic!( + "Victim deleted symlinked file outside of victim_del. Attack succeeded in {:?}.", + Instant::now().duration_since(start_time) + ); + } + let _ = fs::create_dir(&victim_del_path); + let _ = fs::remove_dir(&victim_del_path); + let _ = symlink_dir(attack_dest_dir, &victim_del_path); + } +} + +#[test] +fn unicode_path_is_dir() { + assert!(Path::new(".").is_dir()); + assert!(!Path::new("test/stdtest/fs.rs").is_dir()); + + let tmpdir = tmpdir(); + + let mut dirpath = tmpdir.path().to_path_buf(); + dirpath.push("test-가一ー你好"); + check!(fs::create_dir(&dirpath)); + assert!(dirpath.is_dir()); + + let mut filepath = dirpath; + filepath.push("unicode-file-\u{ac00}\u{4e00}\u{30fc}\u{4f60}\u{597d}.rs"); + check!(File::create(&filepath)); // ignore return; touch only + assert!(!filepath.is_dir()); + assert!(filepath.exists()); +} + +#[test] +fn unicode_path_exists() { + assert!(Path::new(".").exists()); + assert!(!Path::new("test/nonexistent-bogus-path").exists()); + + let tmpdir = tmpdir(); + let unicode = tmpdir.path(); + let unicode = unicode.join("test-각丁ー再见"); + check!(fs::create_dir(&unicode)); + assert!(unicode.exists()); + assert!(!Path::new("test/unicode-bogus-path-각丁ー再见").exists()); +} + +#[test] +fn copy_file_does_not_exist() { + let from = Path::new("test/nonexistent-bogus-path"); + let to = Path::new("test/other-bogus-path"); + + match fs::copy(&from, &to) { + Ok(..) => panic!(), + Err(..) => { + assert!(!from.exists()); + assert!(!to.exists()); + } + } +} + +#[test] +fn copy_src_does_not_exist() { + let tmpdir = tmpdir(); + let from = Path::new("test/nonexistent-bogus-path"); + let to = tmpdir.join("out.txt"); + check!(check!(File::create(&to)).write(b"hello")); + assert!(fs::copy(&from, &to).is_err()); + assert!(!from.exists()); + let mut v = Vec::new(); + check!(check!(File::open(&to)).read_to_end(&mut v)); + assert_eq!(v, b"hello"); +} + +#[test] +fn copy_file_ok() { + let tmpdir = tmpdir(); + let input = tmpdir.join("in.txt"); + let out = tmpdir.join("out.txt"); + + check!(check!(File::create(&input)).write(b"hello")); + check!(fs::copy(&input, &out)); + let mut v = Vec::new(); + check!(check!(File::open(&out)).read_to_end(&mut v)); + assert_eq!(v, b"hello"); + + assert_eq!(check!(input.metadata()).permissions(), check!(out.metadata()).permissions()); +} + +#[test] +fn copy_file_dst_dir() { + let tmpdir = tmpdir(); + let out = tmpdir.join("out"); + + check!(File::create(&out)); + match fs::copy(&*out, tmpdir.path()) { + Ok(..) => panic!(), + Err(..) => {} + } +} + +#[test] +fn copy_file_dst_exists() { + let tmpdir = tmpdir(); + let input = tmpdir.join("in"); + let output = tmpdir.join("out"); + + check!(check!(File::create(&input)).write("foo".as_bytes())); + check!(check!(File::create(&output)).write("bar".as_bytes())); + check!(fs::copy(&input, &output)); + + let mut v = Vec::new(); + check!(check!(File::open(&output)).read_to_end(&mut v)); + assert_eq!(v, b"foo".to_vec()); +} + +#[test] +fn copy_file_src_dir() { + let tmpdir = tmpdir(); + let out = tmpdir.join("out"); + + match fs::copy(tmpdir.path(), &out) { + Ok(..) => panic!(), + Err(..) => {} + } + assert!(!out.exists()); +} + +#[test] +fn copy_file_preserves_perm_bits() { + let tmpdir = tmpdir(); + let input = tmpdir.join("in.txt"); + let out = tmpdir.join("out.txt"); + + let attr = check!(check!(File::create(&input)).metadata()); + let mut p = attr.permissions(); + p.set_readonly(true); + check!(fs::set_permissions(&input, p)); + check!(fs::copy(&input, &out)); + assert!(check!(out.metadata()).permissions().readonly()); + check!(fs::set_permissions(&input, attr.permissions())); + check!(fs::set_permissions(&out, attr.permissions())); +} + +#[test] +#[cfg(windows)] +fn copy_file_preserves_streams() { + let tmp = tmpdir(); + check!(check!(File::create(tmp.join("in.txt:bunny"))).write("carrot".as_bytes())); + assert_eq!(check!(fs::copy(tmp.join("in.txt"), tmp.join("out.txt"))), 0); + assert_eq!(check!(tmp.join("out.txt").metadata()).len(), 0); + let mut v = Vec::new(); + check!(check!(File::open(tmp.join("out.txt:bunny"))).read_to_end(&mut v)); + assert_eq!(v, b"carrot".to_vec()); +} + +#[test] +fn copy_file_returns_metadata_len() { + let tmp = tmpdir(); + let in_path = tmp.join("in.txt"); + let out_path = tmp.join("out.txt"); + check!(check!(File::create(&in_path)).write(b"lettuce")); + #[cfg(windows)] + check!(check!(File::create(tmp.join("in.txt:bunny"))).write(b"carrot")); + let copied_len = check!(fs::copy(&in_path, &out_path)); + assert_eq!(check!(out_path.metadata()).len(), copied_len); +} + +#[test] +fn copy_file_follows_dst_symlink() { + let tmp = tmpdir(); + if !got_symlink_permission(&tmp) { + return; + }; + + let in_path = tmp.join("in.txt"); + let out_path = tmp.join("out.txt"); + let out_path_symlink = tmp.join("out_symlink.txt"); + + check!(fs::write(&in_path, "foo")); + check!(fs::write(&out_path, "bar")); + check!(symlink_file(&out_path, &out_path_symlink)); + + check!(fs::copy(&in_path, &out_path_symlink)); + + assert!(check!(out_path_symlink.symlink_metadata()).file_type().is_symlink()); + assert_eq!(check!(fs::read(&out_path_symlink)), b"foo".to_vec()); + assert_eq!(check!(fs::read(&out_path)), b"foo".to_vec()); +} + +#[test] +fn symlinks_work() { + let tmpdir = tmpdir(); + if !got_symlink_permission(&tmpdir) { + return; + }; + + let input = tmpdir.join("in.txt"); + let out = tmpdir.join("out.txt"); + + check!(check!(File::create(&input)).write("foobar".as_bytes())); + check!(symlink_file(&input, &out)); + assert!(check!(out.symlink_metadata()).file_type().is_symlink()); + assert_eq!(check!(fs::metadata(&out)).len(), check!(fs::metadata(&input)).len()); + let mut v = Vec::new(); + check!(check!(File::open(&out)).read_to_end(&mut v)); + assert_eq!(v, b"foobar".to_vec()); +} + +#[test] +fn symlink_noexist() { + // Symlinks can point to things that don't exist + let tmpdir = tmpdir(); + if !got_symlink_permission(&tmpdir) { + return; + }; + + // Use a relative path for testing. Symlinks get normalized by Windows, + // so we might not get the same path back for absolute paths + check!(symlink_file(&"foo", &tmpdir.join("bar"))); + assert_eq!(check!(fs::read_link(&tmpdir.join("bar"))).to_str().unwrap(), "foo"); +} + +#[test] +fn read_link() { + if cfg!(windows) { + // directory symlink + assert_eq!(check!(fs::read_link(r"C:\Users\All Users")), Path::new(r"C:\ProgramData")); + // junction + assert_eq!(check!(fs::read_link(r"C:\Users\Default User")), Path::new(r"C:\Users\Default")); + // junction with special permissions + // Since not all localized windows versions contain the folder "Documents and Settings" in english, + // we will briefly check, if it exists and otherwise skip the test. Except during CI we will always execute the test. + if Path::new(r"C:\Documents and Settings\").exists() || env::var_os("CI").is_some() { + assert_eq!( + check!(fs::read_link(r"C:\Documents and Settings\")), + Path::new(r"C:\Users") + ); + } + } + let tmpdir = tmpdir(); + let link = tmpdir.join("link"); + if !got_symlink_permission(&tmpdir) { + return; + }; + check!(symlink_file(&"foo", &link)); + assert_eq!(check!(fs::read_link(&link)).to_str().unwrap(), "foo"); +} + +#[test] +fn readlink_not_symlink() { + let tmpdir = tmpdir(); + match fs::read_link(tmpdir.path()) { + Ok(..) => panic!("wanted a failure"), + Err(..) => {} + } +} + +#[test] +fn links_work() { + let tmpdir = tmpdir(); + let input = tmpdir.join("in.txt"); + let out = tmpdir.join("out.txt"); + + check!(check!(File::create(&input)).write("foobar".as_bytes())); + check!(fs::hard_link(&input, &out)); + assert_eq!(check!(fs::metadata(&out)).len(), check!(fs::metadata(&input)).len()); + assert_eq!(check!(fs::metadata(&out)).len(), check!(input.metadata()).len()); + let mut v = Vec::new(); + check!(check!(File::open(&out)).read_to_end(&mut v)); + assert_eq!(v, b"foobar".to_vec()); + + // can't link to yourself + match fs::hard_link(&input, &input) { + Ok(..) => panic!("wanted a failure"), + Err(..) => {} + } + // can't link to something that doesn't exist + match fs::hard_link(&tmpdir.join("foo"), &tmpdir.join("bar")) { + Ok(..) => panic!("wanted a failure"), + Err(..) => {} + } +} + +#[test] +fn chmod_works() { + let tmpdir = tmpdir(); + let file = tmpdir.join("in.txt"); + + check!(File::create(&file)); + let attr = check!(fs::metadata(&file)); + assert!(!attr.permissions().readonly()); + let mut p = attr.permissions(); + p.set_readonly(true); + check!(fs::set_permissions(&file, p.clone())); + let attr = check!(fs::metadata(&file)); + assert!(attr.permissions().readonly()); + + match fs::set_permissions(&tmpdir.join("foo"), p.clone()) { + Ok(..) => panic!("wanted an error"), + Err(..) => {} + } + + p.set_readonly(false); + check!(fs::set_permissions(&file, p)); +} + +#[test] +fn fchmod_works() { + let tmpdir = tmpdir(); + let path = tmpdir.join("in.txt"); + + let file = check!(File::create(&path)); + let attr = check!(fs::metadata(&path)); + assert!(!attr.permissions().readonly()); + let mut p = attr.permissions(); + p.set_readonly(true); + check!(file.set_permissions(p.clone())); + let attr = check!(fs::metadata(&path)); + assert!(attr.permissions().readonly()); + + p.set_readonly(false); + check!(file.set_permissions(p)); +} + +#[test] +fn sync_doesnt_kill_anything() { + let tmpdir = tmpdir(); + let path = tmpdir.join("in.txt"); + + let mut file = check!(File::create(&path)); + check!(file.sync_all()); + check!(file.sync_data()); + check!(file.write(b"foo")); + check!(file.sync_all()); + check!(file.sync_data()); +} + +#[test] +fn truncate_works() { + let tmpdir = tmpdir(); + let path = tmpdir.join("in.txt"); + + let mut file = check!(File::create(&path)); + check!(file.write(b"foo")); + check!(file.sync_all()); + + // Do some simple things with truncation + assert_eq!(check!(file.metadata()).len(), 3); + check!(file.set_len(10)); + assert_eq!(check!(file.metadata()).len(), 10); + check!(file.write(b"bar")); + check!(file.sync_all()); + assert_eq!(check!(file.metadata()).len(), 10); + + let mut v = Vec::new(); + check!(check!(File::open(&path)).read_to_end(&mut v)); + assert_eq!(v, b"foobar\0\0\0\0".to_vec()); + + // Truncate to a smaller length, don't seek, and then write something. + // Ensure that the intermediate zeroes are all filled in (we have `seek`ed + // past the end of the file). + check!(file.set_len(2)); + assert_eq!(check!(file.metadata()).len(), 2); + check!(file.write(b"wut")); + check!(file.sync_all()); + assert_eq!(check!(file.metadata()).len(), 9); + let mut v = Vec::new(); + check!(check!(File::open(&path)).read_to_end(&mut v)); + assert_eq!(v, b"fo\0\0\0\0wut".to_vec()); +} + +#[test] +fn open_flavors() { + use crate::fs::OpenOptions as OO; + fn c(t: &T) -> T { + t.clone() + } + + let tmpdir = tmpdir(); + + let mut r = OO::new(); + r.read(true); + let mut w = OO::new(); + w.write(true); + let mut rw = OO::new(); + rw.read(true).write(true); + let mut a = OO::new(); + a.append(true); + let mut ra = OO::new(); + ra.read(true).append(true); + + #[cfg(windows)] + let invalid_options = 87; // ERROR_INVALID_PARAMETER + #[cfg(all(unix, not(target_os = "vxworks")))] + let invalid_options = "Invalid argument"; + #[cfg(target_os = "vxworks")] + let invalid_options = "invalid argument"; + + // Test various combinations of creation modes and access modes. + // + // Allowed: + // creation mode | read | write | read-write | append | read-append | + // :-----------------------|:-----:|:-----:|:----------:|:------:|:-----------:| + // not set (open existing) | X | X | X | X | X | + // create | | X | X | X | X | + // truncate | | X | X | | | + // create and truncate | | X | X | | | + // create_new | | X | X | X | X | + // + // tested in reverse order, so 'create_new' creates the file, and 'open existing' opens it. + + // write-only + check!(c(&w).create_new(true).open(&tmpdir.join("a"))); + check!(c(&w).create(true).truncate(true).open(&tmpdir.join("a"))); + check!(c(&w).truncate(true).open(&tmpdir.join("a"))); + check!(c(&w).create(true).open(&tmpdir.join("a"))); + check!(c(&w).open(&tmpdir.join("a"))); + + // read-only + error!(c(&r).create_new(true).open(&tmpdir.join("b")), invalid_options); + error!(c(&r).create(true).truncate(true).open(&tmpdir.join("b")), invalid_options); + error!(c(&r).truncate(true).open(&tmpdir.join("b")), invalid_options); + error!(c(&r).create(true).open(&tmpdir.join("b")), invalid_options); + check!(c(&r).open(&tmpdir.join("a"))); // try opening the file created with write_only + + // read-write + check!(c(&rw).create_new(true).open(&tmpdir.join("c"))); + check!(c(&rw).create(true).truncate(true).open(&tmpdir.join("c"))); + check!(c(&rw).truncate(true).open(&tmpdir.join("c"))); + check!(c(&rw).create(true).open(&tmpdir.join("c"))); + check!(c(&rw).open(&tmpdir.join("c"))); + + // append + check!(c(&a).create_new(true).open(&tmpdir.join("d"))); + error!(c(&a).create(true).truncate(true).open(&tmpdir.join("d")), invalid_options); + error!(c(&a).truncate(true).open(&tmpdir.join("d")), invalid_options); + check!(c(&a).create(true).open(&tmpdir.join("d"))); + check!(c(&a).open(&tmpdir.join("d"))); + + // read-append + check!(c(&ra).create_new(true).open(&tmpdir.join("e"))); + error!(c(&ra).create(true).truncate(true).open(&tmpdir.join("e")), invalid_options); + error!(c(&ra).truncate(true).open(&tmpdir.join("e")), invalid_options); + check!(c(&ra).create(true).open(&tmpdir.join("e"))); + check!(c(&ra).open(&tmpdir.join("e"))); + + // Test opening a file without setting an access mode + let mut blank = OO::new(); + error!(blank.create(true).open(&tmpdir.join("f")), invalid_options); + + // Test write works + check!(check!(File::create(&tmpdir.join("h"))).write("foobar".as_bytes())); + + // Test write fails for read-only + check!(r.open(&tmpdir.join("h"))); + { + let mut f = check!(r.open(&tmpdir.join("h"))); + assert!(f.write("wut".as_bytes()).is_err()); + } + + // Test write overwrites + { + let mut f = check!(c(&w).open(&tmpdir.join("h"))); + check!(f.write("baz".as_bytes())); + } + { + let mut f = check!(c(&r).open(&tmpdir.join("h"))); + let mut b = vec![0; 6]; + check!(f.read(&mut b)); + assert_eq!(b, "bazbar".as_bytes()); + } + + // Test truncate works + { + let mut f = check!(c(&w).truncate(true).open(&tmpdir.join("h"))); + check!(f.write("foo".as_bytes())); + } + assert_eq!(check!(fs::metadata(&tmpdir.join("h"))).len(), 3); + + // Test append works + assert_eq!(check!(fs::metadata(&tmpdir.join("h"))).len(), 3); + { + let mut f = check!(c(&a).open(&tmpdir.join("h"))); + check!(f.write("bar".as_bytes())); + } + assert_eq!(check!(fs::metadata(&tmpdir.join("h"))).len(), 6); + + // Test .append(true) equals .write(true).append(true) + { + let mut f = check!(c(&w).append(true).open(&tmpdir.join("h"))); + check!(f.write("baz".as_bytes())); + } + assert_eq!(check!(fs::metadata(&tmpdir.join("h"))).len(), 9); +} + +#[test] +fn _assert_send_sync() { + fn _assert_send_sync() {} + _assert_send_sync::(); +} + +#[test] +fn binary_file() { + let mut bytes = [0; 1024]; + crate::test_helpers::test_rng().fill_bytes(&mut bytes); + + let tmpdir = tmpdir(); + + check!(check!(File::create(&tmpdir.join("test"))).write(&bytes)); + let mut v = Vec::new(); + check!(check!(File::open(&tmpdir.join("test"))).read_to_end(&mut v)); + assert!(v == &bytes[..]); +} + +#[test] +fn write_then_read() { + let mut bytes = [0; 1024]; + crate::test_helpers::test_rng().fill_bytes(&mut bytes); + + let tmpdir = tmpdir(); + + check!(fs::write(&tmpdir.join("test"), &bytes[..])); + let v = check!(fs::read(&tmpdir.join("test"))); + assert!(v == &bytes[..]); + + check!(fs::write(&tmpdir.join("not-utf8"), &[0xFF])); + error_contains!( + fs::read_to_string(&tmpdir.join("not-utf8")), + "stream did not contain valid UTF-8" + ); + + let s = "𐁁𐀓𐀠𐀴𐀍"; + check!(fs::write(&tmpdir.join("utf8"), s.as_bytes())); + let string = check!(fs::read_to_string(&tmpdir.join("utf8"))); + assert_eq!(string, s); +} + +#[test] +fn file_try_clone() { + let tmpdir = tmpdir(); + + let mut f1 = + check!(OpenOptions::new().read(true).write(true).create(true).open(&tmpdir.join("test"))); + let mut f2 = check!(f1.try_clone()); + + check!(f1.write_all(b"hello world")); + check!(f1.seek(SeekFrom::Start(2))); + + let mut buf = vec![]; + check!(f2.read_to_end(&mut buf)); + assert_eq!(buf, b"llo world"); + drop(f2); + + check!(f1.write_all(b"!")); +} + +#[test] +#[cfg(not(windows))] +fn unlink_readonly() { + let tmpdir = tmpdir(); + let path = tmpdir.join("file"); + check!(File::create(&path)); + let mut perm = check!(fs::metadata(&path)).permissions(); + perm.set_readonly(true); + check!(fs::set_permissions(&path, perm)); + check!(fs::remove_file(&path)); +} + +#[test] +fn mkdir_trailing_slash() { + let tmpdir = tmpdir(); + let path = tmpdir.join("file"); + check!(fs::create_dir_all(&path.join("a/"))); +} + +#[test] +fn canonicalize_works_simple() { + let tmpdir = tmpdir(); + let tmpdir = fs::canonicalize(tmpdir.path()).unwrap(); + let file = tmpdir.join("test"); + File::create(&file).unwrap(); + assert_eq!(fs::canonicalize(&file).unwrap(), file); +} + +#[test] +fn realpath_works() { + let tmpdir = tmpdir(); + if !got_symlink_permission(&tmpdir) { + return; + }; + + let tmpdir = fs::canonicalize(tmpdir.path()).unwrap(); + let file = tmpdir.join("test"); + let dir = tmpdir.join("test2"); + let link = dir.join("link"); + let linkdir = tmpdir.join("test3"); + + File::create(&file).unwrap(); + fs::create_dir(&dir).unwrap(); + symlink_file(&file, &link).unwrap(); + symlink_dir(&dir, &linkdir).unwrap(); + + assert!(link.symlink_metadata().unwrap().file_type().is_symlink()); + + assert_eq!(fs::canonicalize(&tmpdir).unwrap(), tmpdir); + assert_eq!(fs::canonicalize(&file).unwrap(), file); + assert_eq!(fs::canonicalize(&link).unwrap(), file); + assert_eq!(fs::canonicalize(&linkdir).unwrap(), dir); + assert_eq!(fs::canonicalize(&linkdir.join("link")).unwrap(), file); +} + +#[test] +fn realpath_works_tricky() { + let tmpdir = tmpdir(); + if !got_symlink_permission(&tmpdir) { + return; + }; + + let tmpdir = fs::canonicalize(tmpdir.path()).unwrap(); + let a = tmpdir.join("a"); + let b = a.join("b"); + let c = b.join("c"); + let d = a.join("d"); + let e = d.join("e"); + let f = a.join("f"); + + fs::create_dir_all(&b).unwrap(); + fs::create_dir_all(&d).unwrap(); + File::create(&f).unwrap(); + if cfg!(not(windows)) { + symlink_file("../d/e", &c).unwrap(); + symlink_file("../f", &e).unwrap(); + } + if cfg!(windows) { + symlink_file(r"..\d\e", &c).unwrap(); + symlink_file(r"..\f", &e).unwrap(); + } + + assert_eq!(fs::canonicalize(&c).unwrap(), f); + assert_eq!(fs::canonicalize(&e).unwrap(), f); +} + +#[test] +fn dir_entry_methods() { + let tmpdir = tmpdir(); + + fs::create_dir_all(&tmpdir.join("a")).unwrap(); + File::create(&tmpdir.join("b")).unwrap(); + + for file in tmpdir.path().read_dir().unwrap().map(|f| f.unwrap()) { + let fname = file.file_name(); + match fname.to_str() { + Some("a") => { + assert!(file.file_type().unwrap().is_dir()); + assert!(file.metadata().unwrap().is_dir()); + } + Some("b") => { + assert!(file.file_type().unwrap().is_file()); + assert!(file.metadata().unwrap().is_file()); + } + f => panic!("unknown file name: {f:?}"), + } + } +} + +#[test] +fn dir_entry_debug() { + let tmpdir = tmpdir(); + File::create(&tmpdir.join("b")).unwrap(); + let mut read_dir = tmpdir.path().read_dir().unwrap(); + let dir_entry = read_dir.next().unwrap().unwrap(); + let actual = format!("{dir_entry:?}"); + let expected = format!("DirEntry({:?})", dir_entry.0.path()); + assert_eq!(actual, expected); +} + +#[test] +fn read_dir_not_found() { + let res = fs::read_dir("/path/that/does/not/exist"); + assert_eq!(res.err().unwrap().kind(), ErrorKind::NotFound); +} + +#[test] +fn file_open_not_found() { + let res = File::open("/path/that/does/not/exist"); + assert_eq!(res.err().unwrap().kind(), ErrorKind::NotFound); +} + +#[test] +fn create_dir_all_with_junctions() { + let tmpdir = tmpdir(); + let target = tmpdir.join("target"); + + let junction = tmpdir.join("junction"); + let b = junction.join("a/b"); + + let link = tmpdir.join("link"); + let d = link.join("c/d"); + + fs::create_dir(&target).unwrap(); + + check!(symlink_junction(&target, &junction)); + check!(fs::create_dir_all(&b)); + // the junction itself is not a directory, but `is_dir()` on a Path + // follows links + assert!(junction.is_dir()); + assert!(b.exists()); + + if !got_symlink_permission(&tmpdir) { + return; + }; + check!(symlink_dir(&target, &link)); + check!(fs::create_dir_all(&d)); + assert!(link.is_dir()); + assert!(d.exists()); +} + +#[test] +fn metadata_access_times() { + let tmpdir = tmpdir(); + + let b = tmpdir.join("b"); + File::create(&b).unwrap(); + + let a = check!(fs::metadata(&tmpdir.path())); + let b = check!(fs::metadata(&b)); + + assert_eq!(check!(a.accessed()), check!(a.accessed())); + assert_eq!(check!(a.modified()), check!(a.modified())); + assert_eq!(check!(b.accessed()), check!(b.modified())); + + if cfg!(target_os = "macos") || cfg!(target_os = "windows") { + check!(a.created()); + check!(b.created()); + } + + if cfg!(target_os = "linux") { + // Not always available + match (a.created(), b.created()) { + (Ok(t1), Ok(t2)) => assert!(t1 <= t2), + (Err(e1), Err(e2)) + if e1.kind() == ErrorKind::Uncategorized + && e2.kind() == ErrorKind::Uncategorized + || e1.kind() == ErrorKind::Unsupported + && e2.kind() == ErrorKind::Unsupported => {} + (a, b) => { + panic!("creation time must be always supported or not supported: {a:?} {b:?}") + } + } + } +} + +/// Test creating hard links to symlinks. +#[test] +fn symlink_hard_link() { + let tmpdir = tmpdir(); + if !got_symlink_permission(&tmpdir) { + return; + }; + if !able_to_not_follow_symlinks_while_hard_linking() { + return; + } + + // Create "file", a file. + check!(fs::File::create(tmpdir.join("file"))); + + // Create "symlink", a symlink to "file". + check!(symlink_file("file", tmpdir.join("symlink"))); + + // Create "hard_link", a hard link to "symlink". + check!(fs::hard_link(tmpdir.join("symlink"), tmpdir.join("hard_link"))); + + // "hard_link" should appear as a symlink. + assert!(check!(fs::symlink_metadata(tmpdir.join("hard_link"))).file_type().is_symlink()); + + // We should be able to open "file" via any of the above names. + let _ = check!(fs::File::open(tmpdir.join("file"))); + assert!(fs::File::open(tmpdir.join("file.renamed")).is_err()); + let _ = check!(fs::File::open(tmpdir.join("symlink"))); + let _ = check!(fs::File::open(tmpdir.join("hard_link"))); + + // Rename "file" to "file.renamed". + check!(fs::rename(tmpdir.join("file"), tmpdir.join("file.renamed"))); + + // Now, the symlink and the hard link should be dangling. + assert!(fs::File::open(tmpdir.join("file")).is_err()); + let _ = check!(fs::File::open(tmpdir.join("file.renamed"))); + assert!(fs::File::open(tmpdir.join("symlink")).is_err()); + assert!(fs::File::open(tmpdir.join("hard_link")).is_err()); + + // The symlink and the hard link should both still point to "file". + assert!(fs::read_link(tmpdir.join("file")).is_err()); + assert!(fs::read_link(tmpdir.join("file.renamed")).is_err()); + assert_eq!(check!(fs::read_link(tmpdir.join("symlink"))), Path::new("file")); + assert_eq!(check!(fs::read_link(tmpdir.join("hard_link"))), Path::new("file")); + + // Remove "file.renamed". + check!(fs::remove_file(tmpdir.join("file.renamed"))); + + // Now, we can't open the file by any name. + assert!(fs::File::open(tmpdir.join("file")).is_err()); + assert!(fs::File::open(tmpdir.join("file.renamed")).is_err()); + assert!(fs::File::open(tmpdir.join("symlink")).is_err()); + assert!(fs::File::open(tmpdir.join("hard_link")).is_err()); + + // "hard_link" should still appear as a symlink. + assert!(check!(fs::symlink_metadata(tmpdir.join("hard_link"))).file_type().is_symlink()); +} + +/// Ensure `fs::create_dir` works on Windows with longer paths. +#[test] +#[cfg(windows)] +fn create_dir_long_paths() { + use crate::{ffi::OsStr, iter, os::windows::ffi::OsStrExt}; + const PATH_LEN: usize = 247; + + let tmpdir = tmpdir(); + let mut path = tmpdir.path().to_path_buf(); + path.push("a"); + let mut path = path.into_os_string(); + + let utf16_len = path.encode_wide().count(); + if utf16_len >= PATH_LEN { + // Skip the test in the unlikely event the local user has a long temp directory path. + // This should not affect CI. + return; + } + // Increase the length of the path. + path.extend(iter::repeat(OsStr::new("a")).take(PATH_LEN - utf16_len)); + + // This should succeed. + fs::create_dir(&path).unwrap(); + + // This will fail if the path isn't converted to verbatim. + path.push("a"); + fs::create_dir(&path).unwrap(); + + // #90940: Ensure an empty path returns the "Not Found" error. + let path = Path::new(""); + assert_eq!(path.canonicalize().unwrap_err().kind(), crate::io::ErrorKind::NotFound); +} + +/// Ensure ReadDir works on large directories. +/// Regression test for https://github.com/rust-lang/rust/issues/93384. +#[test] +fn read_large_dir() { + let tmpdir = tmpdir(); + + let count = 32 * 1024; + for i in 0..count { + check!(fs::File::create(tmpdir.join(&i.to_string()))); + } + + for entry in fs::read_dir(tmpdir.path()).unwrap() { + entry.unwrap(); + } +} + +/// Test the fallback for getting the metadata of files like hiberfil.sys that +/// Windows holds a special lock on, preventing normal means of querying +/// metadata. See #96980. +/// +/// Note this fails in CI because `hiberfil.sys` does not actually exist there. +/// Therefore it's marked as ignored. +#[test] +#[ignore] +#[cfg(windows)] +fn hiberfil_sys() { + let hiberfil = Path::new(r"C:\hiberfil.sys"); + assert_eq!(true, hiberfil.try_exists().unwrap()); + fs::symlink_metadata(hiberfil).unwrap(); + fs::metadata(hiberfil).unwrap(); + assert_eq!(true, hiberfil.exists()); +} + +/// Test that two different ways of obtaining the FileType give the same result. +/// Cf. https://github.com/rust-lang/rust/issues/104900 +#[test] +fn test_eq_direntry_metadata() { + let tmpdir = tmpdir(); + let file_path = tmpdir.join("file"); + File::create(file_path).unwrap(); + for e in fs::read_dir(tmpdir.path()).unwrap() { + let e = e.unwrap(); + let p = e.path(); + let ft1 = e.file_type().unwrap(); + let ft2 = p.metadata().unwrap().file_type(); + assert_eq!(ft1, ft2); + } +} + +/// Regression test for https://github.com/rust-lang/rust/issues/50619. +#[test] +#[cfg(target_os = "linux")] +fn test_read_dir_infinite_loop() { + use crate::io::ErrorKind; + use crate::process::Command; + + // Create a zombie child process + let Ok(mut child) = Command::new("echo").spawn() else { return }; + + // Make sure the process is (un)dead + match child.kill() { + // InvalidInput means the child already exited + Err(e) if e.kind() != ErrorKind::InvalidInput => return, + _ => {} + } + + // open() on this path will succeed, but readdir() will fail + let id = child.id(); + let path = format!("/proc/{id}/net"); + + // Skip the test if we can't open the directory in the first place + let Ok(dir) = fs::read_dir(path) else { return }; + + // Check for duplicate errors + assert!(dir.filter(|e| e.is_err()).take(2).count() < 2); +} diff --git a/crux-mir/lib/std/src/future.rs b/crux-mir/lib/std/src/future.rs deleted file mode 100644 index c0675eeba..000000000 --- a/crux-mir/lib/std/src/future.rs +++ /dev/null @@ -1,111 +0,0 @@ -//! Asynchronous values. - -#[cfg(bootstrap)] -use core::{ - cell::Cell, - marker::Unpin, - ops::{Drop, Generator, GeneratorState}, - pin::Pin, - ptr::NonNull, - task::{Context, Poll}, -}; - -#[doc(inline)] -#[stable(feature = "futures_api", since = "1.36.0")] -pub use core::future::*; - -/// Wrap a generator in a future. -/// -/// This function returns a `GenFuture` underneath, but hides it in `impl Trait` to give -/// better error messages (`impl Future` rather than `GenFuture<[closure.....]>`). -// This is `const` to avoid extra errors after we recover from `const async fn` -#[cfg(bootstrap)] -#[doc(hidden)] -#[unstable(feature = "gen_future", issue = "50547")] -pub const fn from_generator>(x: T) -> impl Future { - GenFuture(x) -} - -/// A wrapper around generators used to implement `Future` for `async`/`await` code. -#[cfg(bootstrap)] -#[doc(hidden)] -#[unstable(feature = "gen_future", issue = "50547")] -#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] -struct GenFuture>(T); - -// We rely on the fact that async/await futures are immovable in order to create -// self-referential borrows in the underlying generator. -#[cfg(bootstrap)] -impl> !Unpin for GenFuture {} - -#[cfg(bootstrap)] -#[doc(hidden)] -#[unstable(feature = "gen_future", issue = "50547")] -impl> Future for GenFuture { - type Output = T::Return; - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - // Safe because we're !Unpin + !Drop mapping to a ?Unpin value - let gen = unsafe { Pin::map_unchecked_mut(self, |s| &mut s.0) }; - let _guard = unsafe { set_task_context(cx) }; - match gen.resume(()) { - GeneratorState::Yielded(()) => Poll::Pending, - GeneratorState::Complete(x) => Poll::Ready(x), - } - } -} - -#[cfg(bootstrap)] -thread_local! { - static TLS_CX: Cell>>> = Cell::new(None); -} - -#[cfg(bootstrap)] -struct SetOnDrop(Option>>); - -#[cfg(bootstrap)] -impl Drop for SetOnDrop { - fn drop(&mut self) { - TLS_CX.with(|tls_cx| { - tls_cx.set(self.0.take()); - }); - } -} - -// Safety: the returned guard must drop before `cx` is dropped and before -// any previous guard is dropped. -#[cfg(bootstrap)] -unsafe fn set_task_context(cx: &mut Context<'_>) -> SetOnDrop { - // transmute the context's lifetime to 'static so we can store it. - let cx = core::mem::transmute::<&mut Context<'_>, &mut Context<'static>>(cx); - let old_cx = TLS_CX.with(|tls_cx| tls_cx.replace(Some(NonNull::from(cx)))); - SetOnDrop(old_cx) -} - -#[cfg(bootstrap)] -#[doc(hidden)] -#[unstable(feature = "gen_future", issue = "50547")] -/// Polls a future in the current thread-local task waker. -pub fn poll_with_tls_context(f: Pin<&mut F>) -> Poll -where - F: Future, -{ - let cx_ptr = TLS_CX.with(|tls_cx| { - // Clear the entry so that nested `get_task_waker` calls - // will fail or set their own value. - tls_cx.replace(None) - }); - let _reset = SetOnDrop(cx_ptr); - - let mut cx_ptr = cx_ptr.expect( - "TLS Context not set. This is a rustc bug. \ - Please file an issue on https://github.com/rust-lang/rust.", - ); - - // Safety: we've ensured exclusive access to the context by - // removing the pointer from TLS, only to be replaced once - // we're done with it. - // - // The pointer that was inserted came from an `&mut Context<'_>`, - // so it is safe to treat as mutable. - unsafe { F::poll(f, cx_ptr.as_mut()) } -} diff --git a/crux-mir/lib/std/src/io/buffered.rs b/crux-mir/lib/std/src/io/buffered.rs deleted file mode 100644 index 8862226ad..000000000 --- a/crux-mir/lib/std/src/io/buffered.rs +++ /dev/null @@ -1,1702 +0,0 @@ -//! Buffering wrappers for I/O traits - -use crate::io::prelude::*; - -use crate::cmp; -use crate::error; -use crate::fmt; -use crate::io::{ - self, Error, ErrorKind, Initializer, IoSlice, IoSliceMut, SeekFrom, DEFAULT_BUF_SIZE, -}; -use crate::memchr; - -/// The `BufReader` struct adds buffering to any reader. -/// -/// It can be excessively inefficient to work directly with a [`Read`] instance. -/// For example, every call to [`read`][`TcpStream::read`] on [`TcpStream`] -/// results in a system call. A `BufReader` performs large, infrequent reads on -/// the underlying [`Read`] and maintains an in-memory buffer of the results. -/// -/// `BufReader` can improve the speed of programs that make *small* and -/// *repeated* read calls to the same file or network socket. It does not -/// help when reading very large amounts at once, or reading just one or a few -/// times. It also provides no advantage when reading from a source that is -/// already in memory, like a `Vec`. -/// -/// When the `BufReader` is dropped, the contents of its buffer will be -/// discarded. Creating multiple instances of a `BufReader` on the same -/// stream can cause data loss. Reading from the underlying reader after -/// unwrapping the `BufReader` with `BufReader::into_inner` can also cause -/// data loss. -/// -/// [`Read`]: ../../std/io/trait.Read.html -/// [`TcpStream::read`]: ../../std/net/struct.TcpStream.html#method.read -/// [`TcpStream`]: ../../std/net/struct.TcpStream.html -/// -/// # Examples -/// -/// ```no_run -/// use std::io::prelude::*; -/// use std::io::BufReader; -/// use std::fs::File; -/// -/// fn main() -> std::io::Result<()> { -/// let f = File::open("log.txt")?; -/// let mut reader = BufReader::new(f); -/// -/// let mut line = String::new(); -/// let len = reader.read_line(&mut line)?; -/// println!("First line is {} bytes long", len); -/// Ok(()) -/// } -/// ``` -#[stable(feature = "rust1", since = "1.0.0")] -pub struct BufReader { - inner: R, - buf: Box<[u8]>, - pos: usize, - cap: usize, -} - -impl BufReader { - /// Creates a new `BufReader` with a default buffer capacity. The default is currently 8 KB, - /// but may change in the future. - /// - /// # Examples - /// - /// ```no_run - /// use std::io::BufReader; - /// use std::fs::File; - /// - /// fn main() -> std::io::Result<()> { - /// let f = File::open("log.txt")?; - /// let reader = BufReader::new(f); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn new(inner: R) -> BufReader { - BufReader::with_capacity(DEFAULT_BUF_SIZE, inner) - } - - /// Creates a new `BufReader` with the specified buffer capacity. - /// - /// # Examples - /// - /// Creating a buffer with ten bytes of capacity: - /// - /// ```no_run - /// use std::io::BufReader; - /// use std::fs::File; - /// - /// fn main() -> std::io::Result<()> { - /// let f = File::open("log.txt")?; - /// let reader = BufReader::with_capacity(10, f); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn with_capacity(capacity: usize, inner: R) -> BufReader { - unsafe { - let mut buffer = Vec::with_capacity(capacity); - buffer.set_len(capacity); - inner.initializer().initialize(&mut buffer); - BufReader { inner, buf: buffer.into_boxed_slice(), pos: 0, cap: 0 } - } - } -} - -impl BufReader { - /// Gets a reference to the underlying reader. - /// - /// It is inadvisable to directly read from the underlying reader. - /// - /// # Examples - /// - /// ```no_run - /// use std::io::BufReader; - /// use std::fs::File; - /// - /// fn main() -> std::io::Result<()> { - /// let f1 = File::open("log.txt")?; - /// let reader = BufReader::new(f1); - /// - /// let f2 = reader.get_ref(); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn get_ref(&self) -> &R { - &self.inner - } - - /// Gets a mutable reference to the underlying reader. - /// - /// It is inadvisable to directly read from the underlying reader. - /// - /// # Examples - /// - /// ```no_run - /// use std::io::BufReader; - /// use std::fs::File; - /// - /// fn main() -> std::io::Result<()> { - /// let f1 = File::open("log.txt")?; - /// let mut reader = BufReader::new(f1); - /// - /// let f2 = reader.get_mut(); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn get_mut(&mut self) -> &mut R { - &mut self.inner - } - - /// Returns a reference to the internally buffered data. - /// - /// Unlike `fill_buf`, this will not attempt to fill the buffer if it is empty. - /// - /// # Examples - /// - /// ```no_run - /// use std::io::{BufReader, BufRead}; - /// use std::fs::File; - /// - /// fn main() -> std::io::Result<()> { - /// let f = File::open("log.txt")?; - /// let mut reader = BufReader::new(f); - /// assert!(reader.buffer().is_empty()); - /// - /// if reader.fill_buf()?.len() > 0 { - /// assert!(!reader.buffer().is_empty()); - /// } - /// Ok(()) - /// } - /// ``` - #[stable(feature = "bufreader_buffer", since = "1.37.0")] - pub fn buffer(&self) -> &[u8] { - &self.buf[self.pos..self.cap] - } - - /// Returns the number of bytes the internal buffer can hold at once. - /// - /// # Examples - /// - /// ```no_run - /// #![feature(buffered_io_capacity)] - /// use std::io::{BufReader, BufRead}; - /// use std::fs::File; - /// - /// fn main() -> std::io::Result<()> { - /// let f = File::open("log.txt")?; - /// let mut reader = BufReader::new(f); - /// - /// let capacity = reader.capacity(); - /// let buffer = reader.fill_buf()?; - /// assert!(buffer.len() <= capacity); - /// Ok(()) - /// } - /// ``` - #[unstable(feature = "buffered_io_capacity", issue = "68833")] - pub fn capacity(&self) -> usize { - self.buf.len() - } - - /// Unwraps this `BufReader`, returning the underlying reader. - /// - /// Note that any leftover data in the internal buffer is lost. Therefore, - /// a following read from the underlying reader may lead to data loss. - /// - /// # Examples - /// - /// ```no_run - /// use std::io::BufReader; - /// use std::fs::File; - /// - /// fn main() -> std::io::Result<()> { - /// let f1 = File::open("log.txt")?; - /// let reader = BufReader::new(f1); - /// - /// let f2 = reader.into_inner(); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn into_inner(self) -> R { - self.inner - } - - /// Invalidates all data in the internal buffer. - #[inline] - fn discard_buffer(&mut self) { - self.pos = 0; - self.cap = 0; - } -} - -impl BufReader { - /// Seeks relative to the current position. If the new position lies within the buffer, - /// the buffer will not be flushed, allowing for more efficient seeks. - /// This method does not return the location of the underlying reader, so the caller - /// must track this information themselves if it is required. - #[unstable(feature = "bufreader_seek_relative", issue = "31100")] - pub fn seek_relative(&mut self, offset: i64) -> io::Result<()> { - let pos = self.pos as u64; - if offset < 0 { - if let Some(new_pos) = pos.checked_sub((-offset) as u64) { - self.pos = new_pos as usize; - return Ok(()); - } - } else { - if let Some(new_pos) = pos.checked_add(offset as u64) { - if new_pos <= self.cap as u64 { - self.pos = new_pos as usize; - return Ok(()); - } - } - } - self.seek(SeekFrom::Current(offset)).map(drop) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Read for BufReader { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - // If we don't have any buffered data and we're doing a massive read - // (larger than our internal buffer), bypass our internal buffer - // entirely. - if self.pos == self.cap && buf.len() >= self.buf.len() { - self.discard_buffer(); - return self.inner.read(buf); - } - let nread = { - let mut rem = self.fill_buf()?; - rem.read(buf)? - }; - self.consume(nread); - Ok(nread) - } - - fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { - let total_len = bufs.iter().map(|b| b.len()).sum::(); - if self.pos == self.cap && total_len >= self.buf.len() { - self.discard_buffer(); - return self.inner.read_vectored(bufs); - } - let nread = { - let mut rem = self.fill_buf()?; - rem.read_vectored(bufs)? - }; - self.consume(nread); - Ok(nread) - } - - // we can't skip unconditionally because of the large buffer case in read. - unsafe fn initializer(&self) -> Initializer { - self.inner.initializer() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl BufRead for BufReader { - fn fill_buf(&mut self) -> io::Result<&[u8]> { - // If we've reached the end of our internal buffer then we need to fetch - // some more data from the underlying reader. - // Branch using `>=` instead of the more correct `==` - // to tell the compiler that the pos..cap slice is always valid. - if self.pos >= self.cap { - debug_assert!(self.pos == self.cap); - self.cap = self.inner.read(&mut self.buf)?; - self.pos = 0; - } - Ok(&self.buf[self.pos..self.cap]) - } - - fn consume(&mut self, amt: usize) { - self.pos = cmp::min(self.pos + amt, self.cap); - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl fmt::Debug for BufReader -where - R: fmt::Debug, -{ - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt.debug_struct("BufReader") - .field("reader", &self.inner) - .field("buffer", &format_args!("{}/{}", self.cap - self.pos, self.buf.len())) - .finish() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Seek for BufReader { - /// Seek to an offset, in bytes, in the underlying reader. - /// - /// The position used for seeking with `SeekFrom::Current(_)` is the - /// position the underlying reader would be at if the `BufReader` had no - /// internal buffer. - /// - /// Seeking always discards the internal buffer, even if the seek position - /// would otherwise fall within it. This guarantees that calling - /// `.into_inner()` immediately after a seek yields the underlying reader - /// at the same position. - /// - /// To seek without discarding the internal buffer, use [`BufReader::seek_relative`]. - /// - /// See [`std::io::Seek`] for more details. - /// - /// Note: In the edge case where you're seeking with `SeekFrom::Current(n)` - /// where `n` minus the internal buffer length overflows an `i64`, two - /// seeks will be performed instead of one. If the second seek returns - /// `Err`, the underlying reader will be left at the same position it would - /// have if you called `seek` with `SeekFrom::Current(0)`. - /// - /// [`BufReader::seek_relative`]: struct.BufReader.html#method.seek_relative - /// [`std::io::Seek`]: trait.Seek.html - fn seek(&mut self, pos: SeekFrom) -> io::Result { - let result: u64; - if let SeekFrom::Current(n) = pos { - let remainder = (self.cap - self.pos) as i64; - // it should be safe to assume that remainder fits within an i64 as the alternative - // means we managed to allocate 8 exbibytes and that's absurd. - // But it's not out of the realm of possibility for some weird underlying reader to - // support seeking by i64::min_value() so we need to handle underflow when subtracting - // remainder. - if let Some(offset) = n.checked_sub(remainder) { - result = self.inner.seek(SeekFrom::Current(offset))?; - } else { - // seek backwards by our remainder, and then by the offset - self.inner.seek(SeekFrom::Current(-remainder))?; - self.discard_buffer(); - result = self.inner.seek(SeekFrom::Current(n))?; - } - } else { - // Seeking with Start/End doesn't care about our buffer length. - result = self.inner.seek(pos)?; - } - self.discard_buffer(); - Ok(result) - } -} - -/// Wraps a writer and buffers its output. -/// -/// It can be excessively inefficient to work directly with something that -/// implements [`Write`]. For example, every call to -/// [`write`][`TcpStream::write`] on [`TcpStream`] results in a system call. A -/// `BufWriter` keeps an in-memory buffer of data and writes it to an underlying -/// writer in large, infrequent batches. -/// -/// `BufWriter` can improve the speed of programs that make *small* and -/// *repeated* write calls to the same file or network socket. It does not -/// help when writing very large amounts at once, or writing just one or a few -/// times. It also provides no advantage when writing to a destination that is -/// in memory, like a `Vec`. -/// -/// It is critical to call [`flush`] before `BufWriter` is dropped. Though -/// dropping will attempt to flush the the contents of the buffer, any errors -/// that happen in the process of dropping will be ignored. Calling [`flush`] -/// ensures that the buffer is empty and thus dropping will not even attempt -/// file operations. -/// -/// # Examples -/// -/// Let's write the numbers one through ten to a [`TcpStream`]: -/// -/// ```no_run -/// use std::io::prelude::*; -/// use std::net::TcpStream; -/// -/// let mut stream = TcpStream::connect("127.0.0.1:34254").unwrap(); -/// -/// for i in 0..10 { -/// stream.write(&[i+1]).unwrap(); -/// } -/// ``` -/// -/// Because we're not buffering, we write each one in turn, incurring the -/// overhead of a system call per byte written. We can fix this with a -/// `BufWriter`: -/// -/// ```no_run -/// use std::io::prelude::*; -/// use std::io::BufWriter; -/// use std::net::TcpStream; -/// -/// let mut stream = BufWriter::new(TcpStream::connect("127.0.0.1:34254").unwrap()); -/// -/// for i in 0..10 { -/// stream.write(&[i+1]).unwrap(); -/// } -/// stream.flush().unwrap(); -/// ``` -/// -/// By wrapping the stream with a `BufWriter`, these ten writes are all grouped -/// together by the buffer and will all be written out in one system call when -/// the `stream` is flushed. -/// -/// [`Write`]: ../../std/io/trait.Write.html -/// [`TcpStream::write`]: ../../std/net/struct.TcpStream.html#method.write -/// [`TcpStream`]: ../../std/net/struct.TcpStream.html -/// [`flush`]: #method.flush -#[stable(feature = "rust1", since = "1.0.0")] -pub struct BufWriter { - inner: Option, - buf: Vec, - // #30888: If the inner writer panics in a call to write, we don't want to - // write the buffered data a second time in BufWriter's destructor. This - // flag tells the Drop impl if it should skip the flush. - panicked: bool, -} - -/// An error returned by `into_inner` which combines an error that -/// happened while writing out the buffer, and the buffered writer object -/// which may be used to recover from the condition. -/// -/// # Examples -/// -/// ```no_run -/// use std::io::BufWriter; -/// use std::net::TcpStream; -/// -/// let mut stream = BufWriter::new(TcpStream::connect("127.0.0.1:34254").unwrap()); -/// -/// // do stuff with the stream -/// -/// // we want to get our `TcpStream` back, so let's try: -/// -/// let stream = match stream.into_inner() { -/// Ok(s) => s, -/// Err(e) => { -/// // Here, e is an IntoInnerError -/// panic!("An error occurred"); -/// } -/// }; -/// ``` -#[derive(Debug)] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct IntoInnerError(W, Error); - -impl BufWriter { - /// Creates a new `BufWriter` with a default buffer capacity. The default is currently 8 KB, - /// but may change in the future. - /// - /// # Examples - /// - /// ```no_run - /// use std::io::BufWriter; - /// use std::net::TcpStream; - /// - /// let mut buffer = BufWriter::new(TcpStream::connect("127.0.0.1:34254").unwrap()); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn new(inner: W) -> BufWriter { - BufWriter::with_capacity(DEFAULT_BUF_SIZE, inner) - } - - /// Creates a new `BufWriter` with the specified buffer capacity. - /// - /// # Examples - /// - /// Creating a buffer with a buffer of a hundred bytes. - /// - /// ```no_run - /// use std::io::BufWriter; - /// use std::net::TcpStream; - /// - /// let stream = TcpStream::connect("127.0.0.1:34254").unwrap(); - /// let mut buffer = BufWriter::with_capacity(100, stream); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn with_capacity(capacity: usize, inner: W) -> BufWriter { - BufWriter { inner: Some(inner), buf: Vec::with_capacity(capacity), panicked: false } - } - - fn flush_buf(&mut self) -> io::Result<()> { - let mut written = 0; - let len = self.buf.len(); - let mut ret = Ok(()); - while written < len { - self.panicked = true; - let r = self.inner.as_mut().unwrap().write(&self.buf[written..]); - self.panicked = false; - - match r { - Ok(0) => { - ret = - Err(Error::new(ErrorKind::WriteZero, "failed to write the buffered data")); - break; - } - Ok(n) => written += n, - Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} - Err(e) => { - ret = Err(e); - break; - } - } - } - if written > 0 { - self.buf.drain(..written); - } - ret - } - - /// Gets a reference to the underlying writer. - /// - /// # Examples - /// - /// ```no_run - /// use std::io::BufWriter; - /// use std::net::TcpStream; - /// - /// let mut buffer = BufWriter::new(TcpStream::connect("127.0.0.1:34254").unwrap()); - /// - /// // we can use reference just like buffer - /// let reference = buffer.get_ref(); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn get_ref(&self) -> &W { - self.inner.as_ref().unwrap() - } - - /// Gets a mutable reference to the underlying writer. - /// - /// It is inadvisable to directly write to the underlying writer. - /// - /// # Examples - /// - /// ```no_run - /// use std::io::BufWriter; - /// use std::net::TcpStream; - /// - /// let mut buffer = BufWriter::new(TcpStream::connect("127.0.0.1:34254").unwrap()); - /// - /// // we can use reference just like buffer - /// let reference = buffer.get_mut(); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn get_mut(&mut self) -> &mut W { - self.inner.as_mut().unwrap() - } - - /// Returns a reference to the internally buffered data. - /// - /// # Examples - /// - /// ```no_run - /// use std::io::BufWriter; - /// use std::net::TcpStream; - /// - /// let buf_writer = BufWriter::new(TcpStream::connect("127.0.0.1:34254").unwrap()); - /// - /// // See how many bytes are currently buffered - /// let bytes_buffered = buf_writer.buffer().len(); - /// ``` - #[stable(feature = "bufreader_buffer", since = "1.37.0")] - pub fn buffer(&self) -> &[u8] { - &self.buf - } - - /// Returns the number of bytes the internal buffer can hold without flushing. - /// - /// # Examples - /// - /// ```no_run - /// #![feature(buffered_io_capacity)] - /// use std::io::BufWriter; - /// use std::net::TcpStream; - /// - /// let buf_writer = BufWriter::new(TcpStream::connect("127.0.0.1:34254").unwrap()); - /// - /// // Check the capacity of the inner buffer - /// let capacity = buf_writer.capacity(); - /// // Calculate how many bytes can be written without flushing - /// let without_flush = capacity - buf_writer.buffer().len(); - /// ``` - #[unstable(feature = "buffered_io_capacity", issue = "68833")] - pub fn capacity(&self) -> usize { - self.buf.capacity() - } - - /// Unwraps this `BufWriter`, returning the underlying writer. - /// - /// The buffer is written out before returning the writer. - /// - /// # Errors - /// - /// An `Err` will be returned if an error occurs while flushing the buffer. - /// - /// # Examples - /// - /// ```no_run - /// use std::io::BufWriter; - /// use std::net::TcpStream; - /// - /// let mut buffer = BufWriter::new(TcpStream::connect("127.0.0.1:34254").unwrap()); - /// - /// // unwrap the TcpStream and flush the buffer - /// let stream = buffer.into_inner().unwrap(); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn into_inner(mut self) -> Result>> { - match self.flush_buf() { - Err(e) => Err(IntoInnerError(self, e)), - Ok(()) => Ok(self.inner.take().unwrap()), - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Write for BufWriter { - fn write(&mut self, buf: &[u8]) -> io::Result { - if self.buf.len() + buf.len() > self.buf.capacity() { - self.flush_buf()?; - } - if buf.len() >= self.buf.capacity() { - self.panicked = true; - let r = self.get_mut().write(buf); - self.panicked = false; - r - } else { - self.buf.write(buf) - } - } - - fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { - let total_len = bufs.iter().map(|b| b.len()).sum::(); - if self.buf.len() + total_len > self.buf.capacity() { - self.flush_buf()?; - } - if total_len >= self.buf.capacity() { - self.panicked = true; - let r = self.get_mut().write_vectored(bufs); - self.panicked = false; - r - } else { - self.buf.write_vectored(bufs) - } - } - - fn flush(&mut self) -> io::Result<()> { - self.flush_buf().and_then(|()| self.get_mut().flush()) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl fmt::Debug for BufWriter -where - W: fmt::Debug, -{ - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt.debug_struct("BufWriter") - .field("writer", &self.inner.as_ref().unwrap()) - .field("buffer", &format_args!("{}/{}", self.buf.len(), self.buf.capacity())) - .finish() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Seek for BufWriter { - /// Seek to the offset, in bytes, in the underlying writer. - /// - /// Seeking always writes out the internal buffer before seeking. - fn seek(&mut self, pos: SeekFrom) -> io::Result { - self.flush_buf().and_then(|_| self.get_mut().seek(pos)) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Drop for BufWriter { - fn drop(&mut self) { - if self.inner.is_some() && !self.panicked { - // dtors should not panic, so we ignore a failed flush - let _r = self.flush_buf(); - } - } -} - -impl IntoInnerError { - /// Returns the error which caused the call to `into_inner()` to fail. - /// - /// This error was returned when attempting to write the internal buffer. - /// - /// # Examples - /// - /// ```no_run - /// use std::io::BufWriter; - /// use std::net::TcpStream; - /// - /// let mut stream = BufWriter::new(TcpStream::connect("127.0.0.1:34254").unwrap()); - /// - /// // do stuff with the stream - /// - /// // we want to get our `TcpStream` back, so let's try: - /// - /// let stream = match stream.into_inner() { - /// Ok(s) => s, - /// Err(e) => { - /// // Here, e is an IntoInnerError, let's log the inner error. - /// // - /// // We'll just 'log' to stdout for this example. - /// println!("{}", e.error()); - /// - /// panic!("An unexpected error occurred."); - /// } - /// }; - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn error(&self) -> &Error { - &self.1 - } - - /// Returns the buffered writer instance which generated the error. - /// - /// The returned object can be used for error recovery, such as - /// re-inspecting the buffer. - /// - /// # Examples - /// - /// ```no_run - /// use std::io::BufWriter; - /// use std::net::TcpStream; - /// - /// let mut stream = BufWriter::new(TcpStream::connect("127.0.0.1:34254").unwrap()); - /// - /// // do stuff with the stream - /// - /// // we want to get our `TcpStream` back, so let's try: - /// - /// let stream = match stream.into_inner() { - /// Ok(s) => s, - /// Err(e) => { - /// // Here, e is an IntoInnerError, let's re-examine the buffer: - /// let buffer = e.into_inner(); - /// - /// // do stuff to try to recover - /// - /// // afterwards, let's just return the stream - /// buffer.into_inner().unwrap() - /// } - /// }; - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn into_inner(self) -> W { - self.0 - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl From> for Error { - fn from(iie: IntoInnerError) -> Error { - iie.1 - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl error::Error for IntoInnerError { - #[allow(deprecated, deprecated_in_future)] - fn description(&self) -> &str { - error::Error::description(self.error()) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl fmt::Display for IntoInnerError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.error().fmt(f) - } -} - -/// Wraps a writer and buffers output to it, flushing whenever a newline -/// (`0x0a`, `'\n'`) is detected. -/// -/// The [`BufWriter`][bufwriter] struct wraps a writer and buffers its output. -/// But it only does this batched write when it goes out of scope, or when the -/// internal buffer is full. Sometimes, you'd prefer to write each line as it's -/// completed, rather than the entire buffer at once. Enter `LineWriter`. It -/// does exactly that. -/// -/// Like [`BufWriter`][bufwriter], a `LineWriter`’s buffer will also be flushed when the -/// `LineWriter` goes out of scope or when its internal buffer is full. -/// -/// [bufwriter]: struct.BufWriter.html -/// -/// If there's still a partial line in the buffer when the `LineWriter` is -/// dropped, it will flush those contents. -/// -/// # Examples -/// -/// We can use `LineWriter` to write one line at a time, significantly -/// reducing the number of actual writes to the file. -/// -/// ```no_run -/// use std::fs::{self, File}; -/// use std::io::prelude::*; -/// use std::io::LineWriter; -/// -/// fn main() -> std::io::Result<()> { -/// let road_not_taken = b"I shall be telling this with a sigh -/// Somewhere ages and ages hence: -/// Two roads diverged in a wood, and I - -/// I took the one less traveled by, -/// And that has made all the difference."; -/// -/// let file = File::create("poem.txt")?; -/// let mut file = LineWriter::new(file); -/// -/// file.write_all(b"I shall be telling this with a sigh")?; -/// -/// // No bytes are written until a newline is encountered (or -/// // the internal buffer is filled). -/// assert_eq!(fs::read_to_string("poem.txt")?, ""); -/// file.write_all(b"\n")?; -/// assert_eq!( -/// fs::read_to_string("poem.txt")?, -/// "I shall be telling this with a sigh\n", -/// ); -/// -/// // Write the rest of the poem. -/// file.write_all(b"Somewhere ages and ages hence: -/// Two roads diverged in a wood, and I - -/// I took the one less traveled by, -/// And that has made all the difference.")?; -/// -/// // The last line of the poem doesn't end in a newline, so -/// // we have to flush or drop the `LineWriter` to finish -/// // writing. -/// file.flush()?; -/// -/// // Confirm the whole poem was written. -/// assert_eq!(fs::read("poem.txt")?, &road_not_taken[..]); -/// Ok(()) -/// } -/// ``` -#[stable(feature = "rust1", since = "1.0.0")] -pub struct LineWriter { - inner: BufWriter, - need_flush: bool, -} - -impl LineWriter { - /// Creates a new `LineWriter`. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs::File; - /// use std::io::LineWriter; - /// - /// fn main() -> std::io::Result<()> { - /// let file = File::create("poem.txt")?; - /// let file = LineWriter::new(file); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn new(inner: W) -> LineWriter { - // Lines typically aren't that long, don't use a giant buffer - LineWriter::with_capacity(1024, inner) - } - - /// Creates a new `LineWriter` with a specified capacity for the internal - /// buffer. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs::File; - /// use std::io::LineWriter; - /// - /// fn main() -> std::io::Result<()> { - /// let file = File::create("poem.txt")?; - /// let file = LineWriter::with_capacity(100, file); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn with_capacity(capacity: usize, inner: W) -> LineWriter { - LineWriter { inner: BufWriter::with_capacity(capacity, inner), need_flush: false } - } - - /// Gets a reference to the underlying writer. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs::File; - /// use std::io::LineWriter; - /// - /// fn main() -> std::io::Result<()> { - /// let file = File::create("poem.txt")?; - /// let file = LineWriter::new(file); - /// - /// let reference = file.get_ref(); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn get_ref(&self) -> &W { - self.inner.get_ref() - } - - /// Gets a mutable reference to the underlying writer. - /// - /// Caution must be taken when calling methods on the mutable reference - /// returned as extra writes could corrupt the output stream. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs::File; - /// use std::io::LineWriter; - /// - /// fn main() -> std::io::Result<()> { - /// let file = File::create("poem.txt")?; - /// let mut file = LineWriter::new(file); - /// - /// // we can use reference just like file - /// let reference = file.get_mut(); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn get_mut(&mut self) -> &mut W { - self.inner.get_mut() - } - - /// Unwraps this `LineWriter`, returning the underlying writer. - /// - /// The internal buffer is written out before returning the writer. - /// - /// # Errors - /// - /// An `Err` will be returned if an error occurs while flushing the buffer. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs::File; - /// use std::io::LineWriter; - /// - /// fn main() -> std::io::Result<()> { - /// let file = File::create("poem.txt")?; - /// - /// let writer: LineWriter = LineWriter::new(file); - /// - /// let file: File = writer.into_inner()?; - /// Ok(()) - /// } - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn into_inner(self) -> Result>> { - self.inner.into_inner().map_err(|IntoInnerError(buf, e)| { - IntoInnerError(LineWriter { inner: buf, need_flush: false }, e) - }) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Write for LineWriter { - fn write(&mut self, buf: &[u8]) -> io::Result { - if self.need_flush { - self.flush()?; - } - - // Find the last newline character in the buffer provided. If found then - // we're going to write all the data up to that point and then flush, - // otherwise we just write the whole block to the underlying writer. - let i = match memchr::memrchr(b'\n', buf) { - Some(i) => i, - None => return self.inner.write(buf), - }; - - // Ok, we're going to write a partial amount of the data given first - // followed by flushing the newline. After we've successfully written - // some data then we *must* report that we wrote that data, so future - // errors are ignored. We set our internal `need_flush` flag, though, in - // case flushing fails and we need to try it first next time. - let n = self.inner.write(&buf[..=i])?; - self.need_flush = true; - if self.flush().is_err() || n != i + 1 { - return Ok(n); - } - - // At this point we successfully wrote `i + 1` bytes and flushed it out, - // meaning that the entire line is now flushed out on the screen. While - // we can attempt to finish writing the rest of the data provided. - // Remember though that we ignore errors here as we've successfully - // written data, so we need to report that. - match self.inner.write(&buf[i + 1..]) { - Ok(i) => Ok(n + i), - Err(_) => Ok(n), - } - } - - // Vectored writes are very similar to the writes above, but adjusted for - // the list of buffers that we have to write. - fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { - if self.need_flush { - self.flush()?; - } - - // Find the last newline, and failing that write the whole buffer - let last_newline = bufs - .iter() - .enumerate() - .rev() - .filter_map(|(i, buf)| { - let pos = memchr::memrchr(b'\n', buf)?; - Some((i, pos)) - }) - .next(); - let (i, j) = match last_newline { - Some(pair) => pair, - None => return self.inner.write_vectored(bufs), - }; - let (prefix, suffix) = bufs.split_at(i); - let (buf, suffix) = suffix.split_at(1); - let buf = &buf[0]; - - // Write everything up to the last newline, flushing afterwards. Note - // that only if we finished our entire `write_vectored` do we try the - // subsequent - // `write` - let mut n = 0; - let prefix_amt = prefix.iter().map(|i| i.len()).sum(); - if prefix_amt > 0 { - n += self.inner.write_vectored(prefix)?; - self.need_flush = true; - } - if n == prefix_amt { - match self.inner.write(&buf[..=j]) { - Ok(m) => n += m, - Err(e) if n == 0 => return Err(e), - Err(_) => return Ok(n), - } - self.need_flush = true; - } - if self.flush().is_err() || n != j + 1 + prefix_amt { - return Ok(n); - } - - // ... and now write out everything remaining - match self.inner.write(&buf[j + 1..]) { - Ok(i) => n += i, - Err(_) => return Ok(n), - } - - if suffix.iter().map(|s| s.len()).sum::() == 0 { - return Ok(n); - } - match self.inner.write_vectored(suffix) { - Ok(i) => Ok(n + i), - Err(_) => Ok(n), - } - } - - fn flush(&mut self) -> io::Result<()> { - self.inner.flush()?; - self.need_flush = false; - Ok(()) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl fmt::Debug for LineWriter -where - W: fmt::Debug, -{ - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt.debug_struct("LineWriter") - .field("writer", &self.inner.inner) - .field( - "buffer", - &format_args!("{}/{}", self.inner.buf.len(), self.inner.buf.capacity()), - ) - .finish() - } -} - -#[cfg(test)] -mod tests { - use crate::io::prelude::*; - use crate::io::{self, BufReader, BufWriter, IoSlice, LineWriter, SeekFrom}; - use crate::sync::atomic::{AtomicUsize, Ordering}; - use crate::thread; - - /// A dummy reader intended at testing short-reads propagation. - pub struct ShortReader { - lengths: Vec, - } - - impl Read for ShortReader { - fn read(&mut self, _: &mut [u8]) -> io::Result { - if self.lengths.is_empty() { Ok(0) } else { Ok(self.lengths.remove(0)) } - } - } - - #[test] - fn test_buffered_reader() { - let inner: &[u8] = &[5, 6, 7, 0, 1, 2, 3, 4]; - let mut reader = BufReader::with_capacity(2, inner); - - let mut buf = [0, 0, 0]; - let nread = reader.read(&mut buf); - assert_eq!(nread.unwrap(), 3); - assert_eq!(buf, [5, 6, 7]); - assert_eq!(reader.buffer(), []); - - let mut buf = [0, 0]; - let nread = reader.read(&mut buf); - assert_eq!(nread.unwrap(), 2); - assert_eq!(buf, [0, 1]); - assert_eq!(reader.buffer(), []); - - let mut buf = [0]; - let nread = reader.read(&mut buf); - assert_eq!(nread.unwrap(), 1); - assert_eq!(buf, [2]); - assert_eq!(reader.buffer(), [3]); - - let mut buf = [0, 0, 0]; - let nread = reader.read(&mut buf); - assert_eq!(nread.unwrap(), 1); - assert_eq!(buf, [3, 0, 0]); - assert_eq!(reader.buffer(), []); - - let nread = reader.read(&mut buf); - assert_eq!(nread.unwrap(), 1); - assert_eq!(buf, [4, 0, 0]); - assert_eq!(reader.buffer(), []); - - assert_eq!(reader.read(&mut buf).unwrap(), 0); - } - - #[test] - fn test_buffered_reader_seek() { - let inner: &[u8] = &[5, 6, 7, 0, 1, 2, 3, 4]; - let mut reader = BufReader::with_capacity(2, io::Cursor::new(inner)); - - assert_eq!(reader.seek(SeekFrom::Start(3)).ok(), Some(3)); - assert_eq!(reader.fill_buf().ok(), Some(&[0, 1][..])); - assert_eq!(reader.seek(SeekFrom::Current(0)).ok(), Some(3)); - assert_eq!(reader.fill_buf().ok(), Some(&[0, 1][..])); - assert_eq!(reader.seek(SeekFrom::Current(1)).ok(), Some(4)); - assert_eq!(reader.fill_buf().ok(), Some(&[1, 2][..])); - reader.consume(1); - assert_eq!(reader.seek(SeekFrom::Current(-2)).ok(), Some(3)); - } - - #[test] - fn test_buffered_reader_seek_relative() { - let inner: &[u8] = &[5, 6, 7, 0, 1, 2, 3, 4]; - let mut reader = BufReader::with_capacity(2, io::Cursor::new(inner)); - - assert!(reader.seek_relative(3).is_ok()); - assert_eq!(reader.fill_buf().ok(), Some(&[0, 1][..])); - assert!(reader.seek_relative(0).is_ok()); - assert_eq!(reader.fill_buf().ok(), Some(&[0, 1][..])); - assert!(reader.seek_relative(1).is_ok()); - assert_eq!(reader.fill_buf().ok(), Some(&[1][..])); - assert!(reader.seek_relative(-1).is_ok()); - assert_eq!(reader.fill_buf().ok(), Some(&[0, 1][..])); - assert!(reader.seek_relative(2).is_ok()); - assert_eq!(reader.fill_buf().ok(), Some(&[2, 3][..])); - } - - #[test] - fn test_buffered_reader_invalidated_after_read() { - let inner: &[u8] = &[5, 6, 7, 0, 1, 2, 3, 4]; - let mut reader = BufReader::with_capacity(3, io::Cursor::new(inner)); - - assert_eq!(reader.fill_buf().ok(), Some(&[5, 6, 7][..])); - reader.consume(3); - - let mut buffer = [0, 0, 0, 0, 0]; - assert_eq!(reader.read(&mut buffer).ok(), Some(5)); - assert_eq!(buffer, [0, 1, 2, 3, 4]); - - assert!(reader.seek_relative(-2).is_ok()); - let mut buffer = [0, 0]; - assert_eq!(reader.read(&mut buffer).ok(), Some(2)); - assert_eq!(buffer, [3, 4]); - } - - #[test] - fn test_buffered_reader_invalidated_after_seek() { - let inner: &[u8] = &[5, 6, 7, 0, 1, 2, 3, 4]; - let mut reader = BufReader::with_capacity(3, io::Cursor::new(inner)); - - assert_eq!(reader.fill_buf().ok(), Some(&[5, 6, 7][..])); - reader.consume(3); - - assert!(reader.seek(SeekFrom::Current(5)).is_ok()); - - assert!(reader.seek_relative(-2).is_ok()); - let mut buffer = [0, 0]; - assert_eq!(reader.read(&mut buffer).ok(), Some(2)); - assert_eq!(buffer, [3, 4]); - } - - #[test] - fn test_buffered_reader_seek_underflow() { - // gimmick reader that yields its position modulo 256 for each byte - struct PositionReader { - pos: u64, - } - impl Read for PositionReader { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - let len = buf.len(); - for x in buf { - *x = self.pos as u8; - self.pos = self.pos.wrapping_add(1); - } - Ok(len) - } - } - impl Seek for PositionReader { - fn seek(&mut self, pos: SeekFrom) -> io::Result { - match pos { - SeekFrom::Start(n) => { - self.pos = n; - } - SeekFrom::Current(n) => { - self.pos = self.pos.wrapping_add(n as u64); - } - SeekFrom::End(n) => { - self.pos = u64::max_value().wrapping_add(n as u64); - } - } - Ok(self.pos) - } - } - - let mut reader = BufReader::with_capacity(5, PositionReader { pos: 0 }); - assert_eq!(reader.fill_buf().ok(), Some(&[0, 1, 2, 3, 4][..])); - assert_eq!(reader.seek(SeekFrom::End(-5)).ok(), Some(u64::max_value() - 5)); - assert_eq!(reader.fill_buf().ok().map(|s| s.len()), Some(5)); - // the following seek will require two underlying seeks - let expected = 9223372036854775802; - assert_eq!(reader.seek(SeekFrom::Current(i64::min_value())).ok(), Some(expected)); - assert_eq!(reader.fill_buf().ok().map(|s| s.len()), Some(5)); - // seeking to 0 should empty the buffer. - assert_eq!(reader.seek(SeekFrom::Current(0)).ok(), Some(expected)); - assert_eq!(reader.get_ref().pos, expected); - } - - #[test] - fn test_buffered_reader_seek_underflow_discard_buffer_between_seeks() { - // gimmick reader that returns Err after first seek - struct ErrAfterFirstSeekReader { - first_seek: bool, - } - impl Read for ErrAfterFirstSeekReader { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - for x in &mut *buf { - *x = 0; - } - Ok(buf.len()) - } - } - impl Seek for ErrAfterFirstSeekReader { - fn seek(&mut self, _: SeekFrom) -> io::Result { - if self.first_seek { - self.first_seek = false; - Ok(0) - } else { - Err(io::Error::new(io::ErrorKind::Other, "oh no!")) - } - } - } - - let mut reader = BufReader::with_capacity(5, ErrAfterFirstSeekReader { first_seek: true }); - assert_eq!(reader.fill_buf().ok(), Some(&[0, 0, 0, 0, 0][..])); - - // The following seek will require two underlying seeks. The first will - // succeed but the second will fail. This should still invalidate the - // buffer. - assert!(reader.seek(SeekFrom::Current(i64::min_value())).is_err()); - assert_eq!(reader.buffer().len(), 0); - } - - #[test] - fn test_buffered_writer() { - let inner = Vec::new(); - let mut writer = BufWriter::with_capacity(2, inner); - - writer.write(&[0, 1]).unwrap(); - assert_eq!(writer.buffer(), []); - assert_eq!(*writer.get_ref(), [0, 1]); - - writer.write(&[2]).unwrap(); - assert_eq!(writer.buffer(), [2]); - assert_eq!(*writer.get_ref(), [0, 1]); - - writer.write(&[3]).unwrap(); - assert_eq!(writer.buffer(), [2, 3]); - assert_eq!(*writer.get_ref(), [0, 1]); - - writer.flush().unwrap(); - assert_eq!(writer.buffer(), []); - assert_eq!(*writer.get_ref(), [0, 1, 2, 3]); - - writer.write(&[4]).unwrap(); - writer.write(&[5]).unwrap(); - assert_eq!(writer.buffer(), [4, 5]); - assert_eq!(*writer.get_ref(), [0, 1, 2, 3]); - - writer.write(&[6]).unwrap(); - assert_eq!(writer.buffer(), [6]); - assert_eq!(*writer.get_ref(), [0, 1, 2, 3, 4, 5]); - - writer.write(&[7, 8]).unwrap(); - assert_eq!(writer.buffer(), []); - assert_eq!(*writer.get_ref(), [0, 1, 2, 3, 4, 5, 6, 7, 8]); - - writer.write(&[9, 10, 11]).unwrap(); - assert_eq!(writer.buffer(), []); - assert_eq!(*writer.get_ref(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); - - writer.flush().unwrap(); - assert_eq!(writer.buffer(), []); - assert_eq!(*writer.get_ref(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); - } - - #[test] - fn test_buffered_writer_inner_flushes() { - let mut w = BufWriter::with_capacity(3, Vec::new()); - w.write(&[0, 1]).unwrap(); - assert_eq!(*w.get_ref(), []); - let w = w.into_inner().unwrap(); - assert_eq!(w, [0, 1]); - } - - #[test] - fn test_buffered_writer_seek() { - let mut w = BufWriter::with_capacity(3, io::Cursor::new(Vec::new())); - w.write_all(&[0, 1, 2, 3, 4, 5]).unwrap(); - w.write_all(&[6, 7]).unwrap(); - assert_eq!(w.seek(SeekFrom::Current(0)).ok(), Some(8)); - assert_eq!(&w.get_ref().get_ref()[..], &[0, 1, 2, 3, 4, 5, 6, 7][..]); - assert_eq!(w.seek(SeekFrom::Start(2)).ok(), Some(2)); - w.write_all(&[8, 9]).unwrap(); - assert_eq!(&w.into_inner().unwrap().into_inner()[..], &[0, 1, 8, 9, 4, 5, 6, 7]); - } - - #[test] - fn test_read_until() { - let inner: &[u8] = &[0, 1, 2, 1, 0]; - let mut reader = BufReader::with_capacity(2, inner); - let mut v = Vec::new(); - reader.read_until(0, &mut v).unwrap(); - assert_eq!(v, [0]); - v.truncate(0); - reader.read_until(2, &mut v).unwrap(); - assert_eq!(v, [1, 2]); - v.truncate(0); - reader.read_until(1, &mut v).unwrap(); - assert_eq!(v, [1]); - v.truncate(0); - reader.read_until(8, &mut v).unwrap(); - assert_eq!(v, [0]); - v.truncate(0); - reader.read_until(9, &mut v).unwrap(); - assert_eq!(v, []); - } - - #[test] - fn test_line_buffer_fail_flush() { - // Issue #32085 - struct FailFlushWriter<'a>(&'a mut Vec); - - impl Write for FailFlushWriter<'_> { - fn write(&mut self, buf: &[u8]) -> io::Result { - self.0.extend_from_slice(buf); - Ok(buf.len()) - } - fn flush(&mut self) -> io::Result<()> { - Err(io::Error::new(io::ErrorKind::Other, "flush failed")) - } - } - - let mut buf = Vec::new(); - { - let mut writer = LineWriter::new(FailFlushWriter(&mut buf)); - let to_write = b"abc\ndef"; - if let Ok(written) = writer.write(to_write) { - assert!(written < to_write.len(), "didn't flush on new line"); - // PASS - return; - } - } - assert!(buf.is_empty(), "write returned an error but wrote data"); - } - - #[test] - fn test_line_buffer() { - let mut writer = LineWriter::new(Vec::new()); - writer.write(&[0]).unwrap(); - assert_eq!(*writer.get_ref(), []); - writer.write(&[1]).unwrap(); - assert_eq!(*writer.get_ref(), []); - writer.flush().unwrap(); - assert_eq!(*writer.get_ref(), [0, 1]); - writer.write(&[0, b'\n', 1, b'\n', 2]).unwrap(); - assert_eq!(*writer.get_ref(), [0, 1, 0, b'\n', 1, b'\n']); - writer.flush().unwrap(); - assert_eq!(*writer.get_ref(), [0, 1, 0, b'\n', 1, b'\n', 2]); - writer.write(&[3, b'\n']).unwrap(); - assert_eq!(*writer.get_ref(), [0, 1, 0, b'\n', 1, b'\n', 2, 3, b'\n']); - } - - #[test] - fn test_read_line() { - let in_buf: &[u8] = b"a\nb\nc"; - let mut reader = BufReader::with_capacity(2, in_buf); - let mut s = String::new(); - reader.read_line(&mut s).unwrap(); - assert_eq!(s, "a\n"); - s.truncate(0); - reader.read_line(&mut s).unwrap(); - assert_eq!(s, "b\n"); - s.truncate(0); - reader.read_line(&mut s).unwrap(); - assert_eq!(s, "c"); - s.truncate(0); - reader.read_line(&mut s).unwrap(); - assert_eq!(s, ""); - } - - #[test] - fn test_lines() { - let in_buf: &[u8] = b"a\nb\nc"; - let reader = BufReader::with_capacity(2, in_buf); - let mut it = reader.lines(); - assert_eq!(it.next().unwrap().unwrap(), "a".to_string()); - assert_eq!(it.next().unwrap().unwrap(), "b".to_string()); - assert_eq!(it.next().unwrap().unwrap(), "c".to_string()); - assert!(it.next().is_none()); - } - - #[test] - fn test_short_reads() { - let inner = ShortReader { lengths: vec![0, 1, 2, 0, 1, 0] }; - let mut reader = BufReader::new(inner); - let mut buf = [0, 0]; - assert_eq!(reader.read(&mut buf).unwrap(), 0); - assert_eq!(reader.read(&mut buf).unwrap(), 1); - assert_eq!(reader.read(&mut buf).unwrap(), 2); - assert_eq!(reader.read(&mut buf).unwrap(), 0); - assert_eq!(reader.read(&mut buf).unwrap(), 1); - assert_eq!(reader.read(&mut buf).unwrap(), 0); - assert_eq!(reader.read(&mut buf).unwrap(), 0); - } - - #[test] - #[should_panic] - fn dont_panic_in_drop_on_panicked_flush() { - struct FailFlushWriter; - - impl Write for FailFlushWriter { - fn write(&mut self, buf: &[u8]) -> io::Result { - Ok(buf.len()) - } - fn flush(&mut self) -> io::Result<()> { - Err(io::Error::last_os_error()) - } - } - - let writer = FailFlushWriter; - let _writer = BufWriter::new(writer); - - // If writer panics *again* due to the flush error then the process will - // abort. - panic!(); - } - - #[test] - #[cfg_attr(target_os = "emscripten", ignore)] - fn panic_in_write_doesnt_flush_in_drop() { - static WRITES: AtomicUsize = AtomicUsize::new(0); - - struct PanicWriter; - - impl Write for PanicWriter { - fn write(&mut self, _: &[u8]) -> io::Result { - WRITES.fetch_add(1, Ordering::SeqCst); - panic!(); - } - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } - } - - thread::spawn(|| { - let mut writer = BufWriter::new(PanicWriter); - let _ = writer.write(b"hello world"); - let _ = writer.flush(); - }) - .join() - .unwrap_err(); - - assert_eq!(WRITES.load(Ordering::SeqCst), 1); - } - - #[bench] - fn bench_buffered_reader(b: &mut test::Bencher) { - b.iter(|| BufReader::new(io::empty())); - } - - #[bench] - fn bench_buffered_writer(b: &mut test::Bencher) { - b.iter(|| BufWriter::new(io::sink())); - } - - struct AcceptOneThenFail { - written: bool, - flushed: bool, - } - - impl Write for AcceptOneThenFail { - fn write(&mut self, data: &[u8]) -> io::Result { - if !self.written { - assert_eq!(data, b"a\nb\n"); - self.written = true; - Ok(data.len()) - } else { - Err(io::Error::new(io::ErrorKind::NotFound, "test")) - } - } - - fn flush(&mut self) -> io::Result<()> { - assert!(self.written); - assert!(!self.flushed); - self.flushed = true; - Err(io::Error::new(io::ErrorKind::Other, "test")) - } - } - - #[test] - fn erroneous_flush_retried() { - let a = AcceptOneThenFail { written: false, flushed: false }; - - let mut l = LineWriter::new(a); - assert_eq!(l.write(b"a\nb\na").unwrap(), 4); - assert!(l.get_ref().written); - assert!(l.get_ref().flushed); - l.get_mut().flushed = false; - - assert_eq!(l.write(b"a").unwrap_err().kind(), io::ErrorKind::Other) - } - - #[test] - fn line_vectored() { - let mut a = LineWriter::new(Vec::new()); - assert_eq!( - a.write_vectored(&[ - IoSlice::new(&[]), - IoSlice::new(b"\n"), - IoSlice::new(&[]), - IoSlice::new(b"a"), - ]) - .unwrap(), - 2, - ); - assert_eq!(a.get_ref(), b"\n"); - - assert_eq!( - a.write_vectored(&[ - IoSlice::new(&[]), - IoSlice::new(b"b"), - IoSlice::new(&[]), - IoSlice::new(b"a"), - IoSlice::new(&[]), - IoSlice::new(b"c"), - ]) - .unwrap(), - 3, - ); - assert_eq!(a.get_ref(), b"\n"); - a.flush().unwrap(); - assert_eq!(a.get_ref(), b"\nabac"); - assert_eq!(a.write_vectored(&[]).unwrap(), 0); - assert_eq!( - a.write_vectored(&[ - IoSlice::new(&[]), - IoSlice::new(&[]), - IoSlice::new(&[]), - IoSlice::new(&[]), - ]) - .unwrap(), - 0, - ); - assert_eq!(a.write_vectored(&[IoSlice::new(b"a\nb"),]).unwrap(), 3); - assert_eq!(a.get_ref(), b"\nabaca\n"); - } - - #[test] - fn line_vectored_partial_and_errors() { - enum Call { - Write { inputs: Vec<&'static [u8]>, output: io::Result }, - Flush { output: io::Result<()> }, - } - struct Writer { - calls: Vec, - } - - impl Write for Writer { - fn write(&mut self, buf: &[u8]) -> io::Result { - self.write_vectored(&[IoSlice::new(buf)]) - } - - fn write_vectored(&mut self, buf: &[IoSlice<'_>]) -> io::Result { - match self.calls.pop().unwrap() { - Call::Write { inputs, output } => { - assert_eq!(inputs, buf.iter().map(|b| &**b).collect::>()); - output - } - _ => panic!("unexpected call to write"), - } - } - - fn flush(&mut self) -> io::Result<()> { - match self.calls.pop().unwrap() { - Call::Flush { output } => output, - _ => panic!("unexpected call to flush"), - } - } - } - - impl Drop for Writer { - fn drop(&mut self) { - if !thread::panicking() { - assert_eq!(self.calls.len(), 0); - } - } - } - - // partial writes keep going - let mut a = LineWriter::new(Writer { calls: Vec::new() }); - a.write_vectored(&[IoSlice::new(&[]), IoSlice::new(b"abc")]).unwrap(); - a.get_mut().calls.push(Call::Flush { output: Ok(()) }); - a.get_mut().calls.push(Call::Write { inputs: vec![b"bcx\n"], output: Ok(4) }); - a.get_mut().calls.push(Call::Write { inputs: vec![b"abcx\n"], output: Ok(1) }); - a.write_vectored(&[IoSlice::new(b"x"), IoSlice::new(b"\n")]).unwrap(); - a.get_mut().calls.push(Call::Flush { output: Ok(()) }); - a.flush().unwrap(); - - // erroneous writes stop and don't write more - a.get_mut().calls.push(Call::Write { inputs: vec![b"x\n"], output: Err(err()) }); - assert_eq!(a.write_vectored(&[IoSlice::new(b"x"), IoSlice::new(b"\na")]).unwrap(), 2); - a.get_mut().calls.push(Call::Flush { output: Ok(()) }); - a.get_mut().calls.push(Call::Write { inputs: vec![b"x\n"], output: Ok(2) }); - a.flush().unwrap(); - - fn err() -> io::Error { - io::Error::new(io::ErrorKind::Other, "x") - } - } -} diff --git a/crux-mir/lib/std/src/io/buffered/bufreader.rs b/crux-mir/lib/std/src/io/buffered/bufreader.rs new file mode 100644 index 000000000..4f339a18a --- /dev/null +++ b/crux-mir/lib/std/src/io/buffered/bufreader.rs @@ -0,0 +1,504 @@ +mod buffer; + +use crate::fmt; +use crate::io::{ + self, BorrowedCursor, BufRead, IoSliceMut, Read, Seek, SeekFrom, SizeHint, DEFAULT_BUF_SIZE, +}; +use buffer::Buffer; + +/// The `BufReader` struct adds buffering to any reader. +/// +/// It can be excessively inefficient to work directly with a [`Read`] instance. +/// For example, every call to [`read`][`TcpStream::read`] on [`TcpStream`] +/// results in a system call. A `BufReader` performs large, infrequent reads on +/// the underlying [`Read`] and maintains an in-memory buffer of the results. +/// +/// `BufReader` can improve the speed of programs that make *small* and +/// *repeated* read calls to the same file or network socket. It does not +/// help when reading very large amounts at once, or reading just one or a few +/// times. It also provides no advantage when reading from a source that is +/// already in memory, like a [Vec]\. +/// +/// When the `BufReader` is dropped, the contents of its buffer will be +/// discarded. Creating multiple instances of a `BufReader` on the same +/// stream can cause data loss. Reading from the underlying reader after +/// unwrapping the `BufReader` with [`BufReader::into_inner`] can also cause +/// data loss. +/// +// HACK(#78696): can't use `crate` for associated items +/// [`TcpStream::read`]: super::super::super::net::TcpStream::read +/// [`TcpStream`]: crate::net::TcpStream +/// +/// # Examples +/// +/// ```no_run +/// use std::io::prelude::*; +/// use std::io::BufReader; +/// use std::fs::File; +/// +/// fn main() -> std::io::Result<()> { +/// let f = File::open("log.txt")?; +/// let mut reader = BufReader::new(f); +/// +/// let mut line = String::new(); +/// let len = reader.read_line(&mut line)?; +/// println!("First line is {len} bytes long"); +/// Ok(()) +/// } +/// ``` +#[stable(feature = "rust1", since = "1.0.0")] +pub struct BufReader { + inner: R, + buf: Buffer, +} + +impl BufReader { + /// Creates a new `BufReader` with a default buffer capacity. The default is currently 8 KB, + /// but may change in the future. + /// + /// # Examples + /// + /// ```no_run + /// use std::io::BufReader; + /// use std::fs::File; + /// + /// fn main() -> std::io::Result<()> { + /// let f = File::open("log.txt")?; + /// let reader = BufReader::new(f); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn new(inner: R) -> BufReader { + BufReader::with_capacity(DEFAULT_BUF_SIZE, inner) + } + + /// Creates a new `BufReader` with the specified buffer capacity. + /// + /// # Examples + /// + /// Creating a buffer with ten bytes of capacity: + /// + /// ```no_run + /// use std::io::BufReader; + /// use std::fs::File; + /// + /// fn main() -> std::io::Result<()> { + /// let f = File::open("log.txt")?; + /// let reader = BufReader::with_capacity(10, f); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn with_capacity(capacity: usize, inner: R) -> BufReader { + BufReader { inner, buf: Buffer::with_capacity(capacity) } + } +} + +impl BufReader { + /// Gets a reference to the underlying reader. + /// + /// It is inadvisable to directly read from the underlying reader. + /// + /// # Examples + /// + /// ```no_run + /// use std::io::BufReader; + /// use std::fs::File; + /// + /// fn main() -> std::io::Result<()> { + /// let f1 = File::open("log.txt")?; + /// let reader = BufReader::new(f1); + /// + /// let f2 = reader.get_ref(); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn get_ref(&self) -> &R { + &self.inner + } + + /// Gets a mutable reference to the underlying reader. + /// + /// It is inadvisable to directly read from the underlying reader. + /// + /// # Examples + /// + /// ```no_run + /// use std::io::BufReader; + /// use std::fs::File; + /// + /// fn main() -> std::io::Result<()> { + /// let f1 = File::open("log.txt")?; + /// let mut reader = BufReader::new(f1); + /// + /// let f2 = reader.get_mut(); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn get_mut(&mut self) -> &mut R { + &mut self.inner + } + + /// Returns a reference to the internally buffered data. + /// + /// Unlike [`fill_buf`], this will not attempt to fill the buffer if it is empty. + /// + /// [`fill_buf`]: BufRead::fill_buf + /// + /// # Examples + /// + /// ```no_run + /// use std::io::{BufReader, BufRead}; + /// use std::fs::File; + /// + /// fn main() -> std::io::Result<()> { + /// let f = File::open("log.txt")?; + /// let mut reader = BufReader::new(f); + /// assert!(reader.buffer().is_empty()); + /// + /// if reader.fill_buf()?.len() > 0 { + /// assert!(!reader.buffer().is_empty()); + /// } + /// Ok(()) + /// } + /// ``` + #[stable(feature = "bufreader_buffer", since = "1.37.0")] + pub fn buffer(&self) -> &[u8] { + self.buf.buffer() + } + + /// Returns the number of bytes the internal buffer can hold at once. + /// + /// # Examples + /// + /// ```no_run + /// use std::io::{BufReader, BufRead}; + /// use std::fs::File; + /// + /// fn main() -> std::io::Result<()> { + /// let f = File::open("log.txt")?; + /// let mut reader = BufReader::new(f); + /// + /// let capacity = reader.capacity(); + /// let buffer = reader.fill_buf()?; + /// assert!(buffer.len() <= capacity); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "buffered_io_capacity", since = "1.46.0")] + pub fn capacity(&self) -> usize { + self.buf.capacity() + } + + /// Unwraps this `BufReader`, returning the underlying reader. + /// + /// Note that any leftover data in the internal buffer is lost. Therefore, + /// a following read from the underlying reader may lead to data loss. + /// + /// # Examples + /// + /// ```no_run + /// use std::io::BufReader; + /// use std::fs::File; + /// + /// fn main() -> std::io::Result<()> { + /// let f1 = File::open("log.txt")?; + /// let reader = BufReader::new(f1); + /// + /// let f2 = reader.into_inner(); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn into_inner(self) -> R { + self.inner + } + + /// Invalidates all data in the internal buffer. + #[inline] + fn discard_buffer(&mut self) { + self.buf.discard_buffer() + } +} + +// This is only used by a test which asserts that the initialization-tracking is correct. +#[cfg(test)] +impl BufReader { + pub fn initialized(&self) -> usize { + self.buf.initialized() + } +} + +impl BufReader { + /// Seeks relative to the current position. If the new position lies within the buffer, + /// the buffer will not be flushed, allowing for more efficient seeks. + /// This method does not return the location of the underlying reader, so the caller + /// must track this information themselves if it is required. + #[stable(feature = "bufreader_seek_relative", since = "1.53.0")] + pub fn seek_relative(&mut self, offset: i64) -> io::Result<()> { + let pos = self.buf.pos() as u64; + if offset < 0 { + if let Some(_) = pos.checked_sub((-offset) as u64) { + self.buf.unconsume((-offset) as usize); + return Ok(()); + } + } else if let Some(new_pos) = pos.checked_add(offset as u64) { + if new_pos <= self.buf.filled() as u64 { + self.buf.consume(offset as usize); + return Ok(()); + } + } + + self.seek(SeekFrom::Current(offset)).map(drop) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Read for BufReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + // If we don't have any buffered data and we're doing a massive read + // (larger than our internal buffer), bypass our internal buffer + // entirely. + if self.buf.pos() == self.buf.filled() && buf.len() >= self.capacity() { + self.discard_buffer(); + return self.inner.read(buf); + } + let nread = { + let mut rem = self.fill_buf()?; + rem.read(buf)? + }; + self.consume(nread); + Ok(nread) + } + + fn read_buf(&mut self, mut cursor: BorrowedCursor<'_>) -> io::Result<()> { + // If we don't have any buffered data and we're doing a massive read + // (larger than our internal buffer), bypass our internal buffer + // entirely. + if self.buf.pos() == self.buf.filled() && cursor.capacity() >= self.capacity() { + self.discard_buffer(); + return self.inner.read_buf(cursor); + } + + let prev = cursor.written(); + + let mut rem = self.fill_buf()?; + rem.read_buf(cursor.reborrow())?; + + self.consume(cursor.written() - prev); //slice impl of read_buf known to never unfill buf + + Ok(()) + } + + // Small read_exacts from a BufReader are extremely common when used with a deserializer. + // The default implementation calls read in a loop, which results in surprisingly poor code + // generation for the common path where the buffer has enough bytes to fill the passed-in + // buffer. + fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> { + if self.buf.consume_with(buf.len(), |claimed| buf.copy_from_slice(claimed)) { + return Ok(()); + } + + crate::io::default_read_exact(self, buf) + } + + fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { + let total_len = bufs.iter().map(|b| b.len()).sum::(); + if self.buf.pos() == self.buf.filled() && total_len >= self.capacity() { + self.discard_buffer(); + return self.inner.read_vectored(bufs); + } + let nread = { + let mut rem = self.fill_buf()?; + rem.read_vectored(bufs)? + }; + self.consume(nread); + Ok(nread) + } + + fn is_read_vectored(&self) -> bool { + self.inner.is_read_vectored() + } + + // The inner reader might have an optimized `read_to_end`. Drain our buffer and then + // delegate to the inner implementation. + fn read_to_end(&mut self, buf: &mut Vec) -> io::Result { + let inner_buf = self.buffer(); + buf.extend_from_slice(inner_buf); + let nread = inner_buf.len(); + self.discard_buffer(); + Ok(nread + self.inner.read_to_end(buf)?) + } + + // The inner reader might have an optimized `read_to_end`. Drain our buffer and then + // delegate to the inner implementation. + fn read_to_string(&mut self, buf: &mut String) -> io::Result { + // In the general `else` case below we must read bytes into a side buffer, check + // that they are valid UTF-8, and then append them to `buf`. This requires a + // potentially large memcpy. + // + // If `buf` is empty--the most common case--we can leverage `append_to_string` + // to read directly into `buf`'s internal byte buffer, saving an allocation and + // a memcpy. + if buf.is_empty() { + // `append_to_string`'s safety relies on the buffer only being appended to since + // it only checks the UTF-8 validity of new data. If there were existing content in + // `buf` then an untrustworthy reader (i.e. `self.inner`) could not only append + // bytes but also modify existing bytes and render them invalid. On the other hand, + // if `buf` is empty then by definition any writes must be appends and + // `append_to_string` will validate all of the new bytes. + unsafe { crate::io::append_to_string(buf, |b| self.read_to_end(b)) } + } else { + // We cannot append our byte buffer directly onto the `buf` String as there could + // be an incomplete UTF-8 sequence that has only been partially read. We must read + // everything into a side buffer first and then call `from_utf8` on the complete + // buffer. + let mut bytes = Vec::new(); + self.read_to_end(&mut bytes)?; + let string = crate::str::from_utf8(&bytes).map_err(|_| { + io::const_io_error!( + io::ErrorKind::InvalidData, + "stream did not contain valid UTF-8", + ) + })?; + *buf += string; + Ok(string.len()) + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl BufRead for BufReader { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + self.buf.fill_buf(&mut self.inner) + } + + fn consume(&mut self, amt: usize) { + self.buf.consume(amt) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Debug for BufReader +where + R: fmt::Debug, +{ + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("BufReader") + .field("reader", &self.inner) + .field( + "buffer", + &format_args!("{}/{}", self.buf.filled() - self.buf.pos(), self.capacity()), + ) + .finish() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Seek for BufReader { + /// Seek to an offset, in bytes, in the underlying reader. + /// + /// The position used for seeking with [SeekFrom::Current]\(_) is the + /// position the underlying reader would be at if the `BufReader` had no + /// internal buffer. + /// + /// Seeking always discards the internal buffer, even if the seek position + /// would otherwise fall within it. This guarantees that calling + /// [`BufReader::into_inner()`] immediately after a seek yields the underlying reader + /// at the same position. + /// + /// To seek without discarding the internal buffer, use [`BufReader::seek_relative`]. + /// + /// See [`std::io::Seek`] for more details. + /// + /// Note: In the edge case where you're seeking with [SeekFrom::Current]\(n) + /// where `n` minus the internal buffer length overflows an `i64`, two + /// seeks will be performed instead of one. If the second seek returns + /// [`Err`], the underlying reader will be left at the same position it would + /// have if you called `seek` with [SeekFrom::Current]\(0). + /// + /// [`std::io::Seek`]: Seek + fn seek(&mut self, pos: SeekFrom) -> io::Result { + let result: u64; + if let SeekFrom::Current(n) = pos { + let remainder = (self.buf.filled() - self.buf.pos()) as i64; + // it should be safe to assume that remainder fits within an i64 as the alternative + // means we managed to allocate 8 exbibytes and that's absurd. + // But it's not out of the realm of possibility for some weird underlying reader to + // support seeking by i64::MIN so we need to handle underflow when subtracting + // remainder. + if let Some(offset) = n.checked_sub(remainder) { + result = self.inner.seek(SeekFrom::Current(offset))?; + } else { + // seek backwards by our remainder, and then by the offset + self.inner.seek(SeekFrom::Current(-remainder))?; + self.discard_buffer(); + result = self.inner.seek(SeekFrom::Current(n))?; + } + } else { + // Seeking with Start/End doesn't care about our buffer length. + result = self.inner.seek(pos)?; + } + self.discard_buffer(); + Ok(result) + } + + /// Returns the current seek position from the start of the stream. + /// + /// The value returned is equivalent to `self.seek(SeekFrom::Current(0))` + /// but does not flush the internal buffer. Due to this optimization the + /// function does not guarantee that calling `.into_inner()` immediately + /// afterwards will yield the underlying reader at the same position. Use + /// [`BufReader::seek`] instead if you require that guarantee. + /// + /// # Panics + /// + /// This function will panic if the position of the inner reader is smaller + /// than the amount of buffered data. That can happen if the inner reader + /// has an incorrect implementation of [`Seek::stream_position`], or if the + /// position has gone out of sync due to calling [`Seek::seek`] directly on + /// the underlying reader. + /// + /// # Example + /// + /// ```no_run + /// use std::{ + /// io::{self, BufRead, BufReader, Seek}, + /// fs::File, + /// }; + /// + /// fn main() -> io::Result<()> { + /// let mut f = BufReader::new(File::open("foo.txt")?); + /// + /// let before = f.stream_position()?; + /// f.read_line(&mut String::new())?; + /// let after = f.stream_position()?; + /// + /// println!("The first line was {} bytes long", after - before); + /// Ok(()) + /// } + /// ``` + fn stream_position(&mut self) -> io::Result { + let remainder = (self.buf.filled() - self.buf.pos()) as u64; + self.inner.stream_position().map(|pos| { + pos.checked_sub(remainder).expect( + "overflow when subtracting remaining buffer size from inner stream position", + ) + }) + } +} + +impl SizeHint for BufReader { + #[inline] + fn lower_bound(&self) -> usize { + SizeHint::lower_bound(self.get_ref()) + self.buffer().len() + } + + #[inline] + fn upper_bound(&self) -> Option { + SizeHint::upper_bound(self.get_ref()).and_then(|up| self.buffer().len().checked_add(up)) + } +} diff --git a/crux-mir/lib/std/src/io/buffered/bufreader/buffer.rs b/crux-mir/lib/std/src/io/buffered/bufreader/buffer.rs new file mode 100644 index 000000000..e9e29d60c --- /dev/null +++ b/crux-mir/lib/std/src/io/buffered/bufreader/buffer.rs @@ -0,0 +1,122 @@ +///! An encapsulation of `BufReader`'s buffer management logic. +/// +/// This module factors out the basic functionality of `BufReader` in order to protect two core +/// invariants: +/// * `filled` bytes of `buf` are always initialized +/// * `pos` is always <= `filled` +/// Since this module encapsulates the buffer management logic, we can ensure that the range +/// `pos..filled` is always a valid index into the initialized region of the buffer. This means +/// that user code which wants to do reads from a `BufReader` via `buffer` + `consume` can do so +/// without encountering any runtime bounds checks. +use crate::cmp; +use crate::io::{self, BorrowedBuf, Read}; +use crate::mem::MaybeUninit; + +pub struct Buffer { + // The buffer. + buf: Box<[MaybeUninit]>, + // The current seek offset into `buf`, must always be <= `filled`. + pos: usize, + // Each call to `fill_buf` sets `filled` to indicate how many bytes at the start of `buf` are + // initialized with bytes from a read. + filled: usize, + // This is the max number of bytes returned across all `fill_buf` calls. We track this so that we + // can accurately tell `read_buf` how many bytes of buf are initialized, to bypass as much of its + // defensive initialization as possible. Note that while this often the same as `filled`, it + // doesn't need to be. Calls to `fill_buf` are not required to actually fill the buffer, and + // omitting this is a huge perf regression for `Read` impls that do not. + initialized: usize, +} + +impl Buffer { + #[inline] + pub fn with_capacity(capacity: usize) -> Self { + let buf = Box::new_uninit_slice(capacity); + Self { buf, pos: 0, filled: 0, initialized: 0 } + } + + #[inline] + pub fn buffer(&self) -> &[u8] { + // SAFETY: self.pos and self.cap are valid, and self.cap => self.pos, and + // that region is initialized because those are all invariants of this type. + unsafe { MaybeUninit::slice_assume_init_ref(self.buf.get_unchecked(self.pos..self.filled)) } + } + + #[inline] + pub fn capacity(&self) -> usize { + self.buf.len() + } + + #[inline] + pub fn filled(&self) -> usize { + self.filled + } + + #[inline] + pub fn pos(&self) -> usize { + self.pos + } + + // This is only used by a test which asserts that the initialization-tracking is correct. + #[cfg(test)] + pub fn initialized(&self) -> usize { + self.initialized + } + + #[inline] + pub fn discard_buffer(&mut self) { + self.pos = 0; + self.filled = 0; + } + + #[inline] + pub fn consume(&mut self, amt: usize) { + self.pos = cmp::min(self.pos + amt, self.filled); + } + + /// If there are `amt` bytes available in the buffer, pass a slice containing those bytes to + /// `visitor` and return true. If there are not enough bytes available, return false. + #[inline] + pub fn consume_with(&mut self, amt: usize, mut visitor: V) -> bool + where + V: FnMut(&[u8]), + { + if let Some(claimed) = self.buffer().get(..amt) { + visitor(claimed); + // If the indexing into self.buffer() succeeds, amt must be a valid increment. + self.pos += amt; + true + } else { + false + } + } + + #[inline] + pub fn unconsume(&mut self, amt: usize) { + self.pos = self.pos.saturating_sub(amt); + } + + #[inline] + pub fn fill_buf(&mut self, mut reader: impl Read) -> io::Result<&[u8]> { + // If we've reached the end of our internal buffer then we need to fetch + // some more data from the reader. + // Branch using `>=` instead of the more correct `==` + // to tell the compiler that the pos..cap slice is always valid. + if self.pos >= self.filled { + debug_assert!(self.pos == self.filled); + + let mut buf = BorrowedBuf::from(&mut *self.buf); + // SAFETY: `self.filled` bytes will always have been initialized. + unsafe { + buf.set_init(self.initialized); + } + + reader.read_buf(buf.unfilled())?; + + self.pos = 0; + self.filled = buf.len(); + self.initialized = buf.init_len(); + } + Ok(self.buffer()) + } +} diff --git a/crux-mir/lib/std/src/io/buffered/bufwriter.rs b/crux-mir/lib/std/src/io/buffered/bufwriter.rs new file mode 100644 index 000000000..6acb937e7 --- /dev/null +++ b/crux-mir/lib/std/src/io/buffered/bufwriter.rs @@ -0,0 +1,674 @@ +use crate::error; +use crate::fmt; +use crate::io::{ + self, ErrorKind, IntoInnerError, IoSlice, Seek, SeekFrom, Write, DEFAULT_BUF_SIZE, +}; +use crate::mem; +use crate::ptr; + +/// Wraps a writer and buffers its output. +/// +/// It can be excessively inefficient to work directly with something that +/// implements [`Write`]. For example, every call to +/// [`write`][`TcpStream::write`] on [`TcpStream`] results in a system call. A +/// `BufWriter` keeps an in-memory buffer of data and writes it to an underlying +/// writer in large, infrequent batches. +/// +/// `BufWriter` can improve the speed of programs that make *small* and +/// *repeated* write calls to the same file or network socket. It does not +/// help when writing very large amounts at once, or writing just one or a few +/// times. It also provides no advantage when writing to a destination that is +/// in memory, like a [Vec]\. +/// +/// It is critical to call [`flush`] before `BufWriter` is dropped. Though +/// dropping will attempt to flush the contents of the buffer, any errors +/// that happen in the process of dropping will be ignored. Calling [`flush`] +/// ensures that the buffer is empty and thus dropping will not even attempt +/// file operations. +/// +/// # Examples +/// +/// Let's write the numbers one through ten to a [`TcpStream`]: +/// +/// ```no_run +/// use std::io::prelude::*; +/// use std::net::TcpStream; +/// +/// let mut stream = TcpStream::connect("127.0.0.1:34254").unwrap(); +/// +/// for i in 0..10 { +/// stream.write(&[i+1]).unwrap(); +/// } +/// ``` +/// +/// Because we're not buffering, we write each one in turn, incurring the +/// overhead of a system call per byte written. We can fix this with a +/// `BufWriter`: +/// +/// ```no_run +/// use std::io::prelude::*; +/// use std::io::BufWriter; +/// use std::net::TcpStream; +/// +/// let mut stream = BufWriter::new(TcpStream::connect("127.0.0.1:34254").unwrap()); +/// +/// for i in 0..10 { +/// stream.write(&[i+1]).unwrap(); +/// } +/// stream.flush().unwrap(); +/// ``` +/// +/// By wrapping the stream with a `BufWriter`, these ten writes are all grouped +/// together by the buffer and will all be written out in one system call when +/// the `stream` is flushed. +/// +// HACK(#78696): can't use `crate` for associated items +/// [`TcpStream::write`]: super::super::super::net::TcpStream::write +/// [`TcpStream`]: crate::net::TcpStream +/// [`flush`]: BufWriter::flush +#[stable(feature = "rust1", since = "1.0.0")] +pub struct BufWriter { + inner: W, + // The buffer. Avoid using this like a normal `Vec` in common code paths. + // That is, don't use `buf.push`, `buf.extend_from_slice`, or any other + // methods that require bounds checking or the like. This makes an enormous + // difference to performance (we may want to stop using a `Vec` entirely). + buf: Vec, + // #30888: If the inner writer panics in a call to write, we don't want to + // write the buffered data a second time in BufWriter's destructor. This + // flag tells the Drop impl if it should skip the flush. + panicked: bool, +} + +impl BufWriter { + /// Creates a new `BufWriter` with a default buffer capacity. The default is currently 8 KB, + /// but may change in the future. + /// + /// # Examples + /// + /// ```no_run + /// use std::io::BufWriter; + /// use std::net::TcpStream; + /// + /// let mut buffer = BufWriter::new(TcpStream::connect("127.0.0.1:34254").unwrap()); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn new(inner: W) -> BufWriter { + BufWriter::with_capacity(DEFAULT_BUF_SIZE, inner) + } + + /// Creates a new `BufWriter` with at least the specified buffer capacity. + /// + /// # Examples + /// + /// Creating a buffer with a buffer of at least a hundred bytes. + /// + /// ```no_run + /// use std::io::BufWriter; + /// use std::net::TcpStream; + /// + /// let stream = TcpStream::connect("127.0.0.1:34254").unwrap(); + /// let mut buffer = BufWriter::with_capacity(100, stream); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn with_capacity(capacity: usize, inner: W) -> BufWriter { + BufWriter { inner, buf: Vec::with_capacity(capacity), panicked: false } + } + + /// Send data in our local buffer into the inner writer, looping as + /// necessary until either it's all been sent or an error occurs. + /// + /// Because all the data in the buffer has been reported to our owner as + /// "successfully written" (by returning nonzero success values from + /// `write`), any 0-length writes from `inner` must be reported as i/o + /// errors from this method. + pub(in crate::io) fn flush_buf(&mut self) -> io::Result<()> { + /// Helper struct to ensure the buffer is updated after all the writes + /// are complete. It tracks the number of written bytes and drains them + /// all from the front of the buffer when dropped. + struct BufGuard<'a> { + buffer: &'a mut Vec, + written: usize, + } + + impl<'a> BufGuard<'a> { + fn new(buffer: &'a mut Vec) -> Self { + Self { buffer, written: 0 } + } + + /// The unwritten part of the buffer + fn remaining(&self) -> &[u8] { + &self.buffer[self.written..] + } + + /// Flag some bytes as removed from the front of the buffer + fn consume(&mut self, amt: usize) { + self.written += amt; + } + + /// true if all of the bytes have been written + fn done(&self) -> bool { + self.written >= self.buffer.len() + } + } + + impl Drop for BufGuard<'_> { + fn drop(&mut self) { + if self.written > 0 { + self.buffer.drain(..self.written); + } + } + } + + let mut guard = BufGuard::new(&mut self.buf); + while !guard.done() { + self.panicked = true; + let r = self.inner.write(guard.remaining()); + self.panicked = false; + + match r { + Ok(0) => { + return Err(io::const_io_error!( + ErrorKind::WriteZero, + "failed to write the buffered data", + )); + } + Ok(n) => guard.consume(n), + Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + Ok(()) + } + + /// Buffer some data without flushing it, regardless of the size of the + /// data. Writes as much as possible without exceeding capacity. Returns + /// the number of bytes written. + pub(super) fn write_to_buf(&mut self, buf: &[u8]) -> usize { + let available = self.spare_capacity(); + let amt_to_buffer = available.min(buf.len()); + + // SAFETY: `amt_to_buffer` is <= buffer's spare capacity by construction. + unsafe { + self.write_to_buffer_unchecked(&buf[..amt_to_buffer]); + } + + amt_to_buffer + } + + /// Gets a reference to the underlying writer. + /// + /// # Examples + /// + /// ```no_run + /// use std::io::BufWriter; + /// use std::net::TcpStream; + /// + /// let mut buffer = BufWriter::new(TcpStream::connect("127.0.0.1:34254").unwrap()); + /// + /// // we can use reference just like buffer + /// let reference = buffer.get_ref(); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn get_ref(&self) -> &W { + &self.inner + } + + /// Gets a mutable reference to the underlying writer. + /// + /// It is inadvisable to directly write to the underlying writer. + /// + /// # Examples + /// + /// ```no_run + /// use std::io::BufWriter; + /// use std::net::TcpStream; + /// + /// let mut buffer = BufWriter::new(TcpStream::connect("127.0.0.1:34254").unwrap()); + /// + /// // we can use reference just like buffer + /// let reference = buffer.get_mut(); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn get_mut(&mut self) -> &mut W { + &mut self.inner + } + + /// Returns a reference to the internally buffered data. + /// + /// # Examples + /// + /// ```no_run + /// use std::io::BufWriter; + /// use std::net::TcpStream; + /// + /// let buf_writer = BufWriter::new(TcpStream::connect("127.0.0.1:34254").unwrap()); + /// + /// // See how many bytes are currently buffered + /// let bytes_buffered = buf_writer.buffer().len(); + /// ``` + #[stable(feature = "bufreader_buffer", since = "1.37.0")] + pub fn buffer(&self) -> &[u8] { + &self.buf + } + + /// Returns a mutable reference to the internal buffer. + /// + /// This can be used to write data directly into the buffer without triggering writers + /// to the underlying writer. + /// + /// That the buffer is a `Vec` is an implementation detail. + /// Callers should not modify the capacity as there currently is no public API to do so + /// and thus any capacity changes would be unexpected by the user. + pub(in crate::io) fn buffer_mut(&mut self) -> &mut Vec { + &mut self.buf + } + + /// Returns the number of bytes the internal buffer can hold without flushing. + /// + /// # Examples + /// + /// ```no_run + /// use std::io::BufWriter; + /// use std::net::TcpStream; + /// + /// let buf_writer = BufWriter::new(TcpStream::connect("127.0.0.1:34254").unwrap()); + /// + /// // Check the capacity of the inner buffer + /// let capacity = buf_writer.capacity(); + /// // Calculate how many bytes can be written without flushing + /// let without_flush = capacity - buf_writer.buffer().len(); + /// ``` + #[stable(feature = "buffered_io_capacity", since = "1.46.0")] + pub fn capacity(&self) -> usize { + self.buf.capacity() + } + + /// Unwraps this `BufWriter`, returning the underlying writer. + /// + /// The buffer is written out before returning the writer. + /// + /// # Errors + /// + /// An [`Err`] will be returned if an error occurs while flushing the buffer. + /// + /// # Examples + /// + /// ```no_run + /// use std::io::BufWriter; + /// use std::net::TcpStream; + /// + /// let mut buffer = BufWriter::new(TcpStream::connect("127.0.0.1:34254").unwrap()); + /// + /// // unwrap the TcpStream and flush the buffer + /// let stream = buffer.into_inner().unwrap(); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn into_inner(mut self) -> Result>> { + match self.flush_buf() { + Err(e) => Err(IntoInnerError::new(self, e)), + Ok(()) => Ok(self.into_parts().0), + } + } + + /// Disassembles this `BufWriter`, returning the underlying writer, and any buffered but + /// unwritten data. + /// + /// If the underlying writer panicked, it is not known what portion of the data was written. + /// In this case, we return `WriterPanicked` for the buffered data (from which the buffer + /// contents can still be recovered). + /// + /// `into_parts` makes no attempt to flush data and cannot fail. + /// + /// # Examples + /// + /// ``` + /// use std::io::{BufWriter, Write}; + /// + /// let mut buffer = [0u8; 10]; + /// let mut stream = BufWriter::new(buffer.as_mut()); + /// write!(stream, "too much data").unwrap(); + /// stream.flush().expect_err("it doesn't fit"); + /// let (recovered_writer, buffered_data) = stream.into_parts(); + /// assert_eq!(recovered_writer.len(), 0); + /// assert_eq!(&buffered_data.unwrap(), b"ata"); + /// ``` + #[stable(feature = "bufwriter_into_parts", since = "1.56.0")] + pub fn into_parts(mut self) -> (W, Result, WriterPanicked>) { + let buf = mem::take(&mut self.buf); + let buf = if !self.panicked { Ok(buf) } else { Err(WriterPanicked { buf }) }; + + // SAFETY: forget(self) prevents double dropping inner + let inner = unsafe { ptr::read(&mut self.inner) }; + mem::forget(self); + + (inner, buf) + } + + // Ensure this function does not get inlined into `write`, so that it + // remains inlineable and its common path remains as short as possible. + // If this function ends up being called frequently relative to `write`, + // it's likely a sign that the client is using an improperly sized buffer + // or their write patterns are somewhat pathological. + #[cold] + #[inline(never)] + fn write_cold(&mut self, buf: &[u8]) -> io::Result { + if buf.len() > self.spare_capacity() { + self.flush_buf()?; + } + + // Why not len > capacity? To avoid a needless trip through the buffer when the input + // exactly fills it. We'd just need to flush it to the underlying writer anyway. + if buf.len() >= self.buf.capacity() { + self.panicked = true; + let r = self.get_mut().write(buf); + self.panicked = false; + r + } else { + // Write to the buffer. In this case, we write to the buffer even if it fills it + // exactly. Doing otherwise would mean flushing the buffer, then writing this + // input to the inner writer, which in many cases would be a worse strategy. + + // SAFETY: There was either enough spare capacity already, or there wasn't and we + // flushed the buffer to ensure that there is. In the latter case, we know that there + // is because flushing ensured that our entire buffer is spare capacity, and we entered + // this block because the input buffer length is less than that capacity. In either + // case, it's safe to write the input buffer to our buffer. + unsafe { + self.write_to_buffer_unchecked(buf); + } + + Ok(buf.len()) + } + } + + // Ensure this function does not get inlined into `write_all`, so that it + // remains inlineable and its common path remains as short as possible. + // If this function ends up being called frequently relative to `write_all`, + // it's likely a sign that the client is using an improperly sized buffer + // or their write patterns are somewhat pathological. + #[cold] + #[inline(never)] + fn write_all_cold(&mut self, buf: &[u8]) -> io::Result<()> { + // Normally, `write_all` just calls `write` in a loop. We can do better + // by calling `self.get_mut().write_all()` directly, which avoids + // round trips through the buffer in the event of a series of partial + // writes in some circumstances. + + if buf.len() > self.spare_capacity() { + self.flush_buf()?; + } + + // Why not len > capacity? To avoid a needless trip through the buffer when the input + // exactly fills it. We'd just need to flush it to the underlying writer anyway. + if buf.len() >= self.buf.capacity() { + self.panicked = true; + let r = self.get_mut().write_all(buf); + self.panicked = false; + r + } else { + // Write to the buffer. In this case, we write to the buffer even if it fills it + // exactly. Doing otherwise would mean flushing the buffer, then writing this + // input to the inner writer, which in many cases would be a worse strategy. + + // SAFETY: There was either enough spare capacity already, or there wasn't and we + // flushed the buffer to ensure that there is. In the latter case, we know that there + // is because flushing ensured that our entire buffer is spare capacity, and we entered + // this block because the input buffer length is less than that capacity. In either + // case, it's safe to write the input buffer to our buffer. + unsafe { + self.write_to_buffer_unchecked(buf); + } + + Ok(()) + } + } + + // SAFETY: Requires `buf.len() <= self.buf.capacity() - self.buf.len()`, + // i.e., that input buffer length is less than or equal to spare capacity. + #[inline] + unsafe fn write_to_buffer_unchecked(&mut self, buf: &[u8]) { + debug_assert!(buf.len() <= self.spare_capacity()); + let old_len = self.buf.len(); + let buf_len = buf.len(); + let src = buf.as_ptr(); + let dst = self.buf.as_mut_ptr().add(old_len); + ptr::copy_nonoverlapping(src, dst, buf_len); + self.buf.set_len(old_len + buf_len); + } + + #[inline] + fn spare_capacity(&self) -> usize { + self.buf.capacity() - self.buf.len() + } +} + +#[stable(feature = "bufwriter_into_parts", since = "1.56.0")] +/// Error returned for the buffered data from `BufWriter::into_parts`, when the underlying +/// writer has previously panicked. Contains the (possibly partly written) buffered data. +/// +/// # Example +/// +/// ``` +/// use std::io::{self, BufWriter, Write}; +/// use std::panic::{catch_unwind, AssertUnwindSafe}; +/// +/// struct PanickingWriter; +/// impl Write for PanickingWriter { +/// fn write(&mut self, buf: &[u8]) -> io::Result { panic!() } +/// fn flush(&mut self) -> io::Result<()> { panic!() } +/// } +/// +/// let mut stream = BufWriter::new(PanickingWriter); +/// write!(stream, "some data").unwrap(); +/// let result = catch_unwind(AssertUnwindSafe(|| { +/// stream.flush().unwrap() +/// })); +/// assert!(result.is_err()); +/// let (recovered_writer, buffered_data) = stream.into_parts(); +/// assert!(matches!(recovered_writer, PanickingWriter)); +/// assert_eq!(buffered_data.unwrap_err().into_inner(), b"some data"); +/// ``` +pub struct WriterPanicked { + buf: Vec, +} + +impl WriterPanicked { + /// Returns the perhaps-unwritten data. Some of this data may have been written by the + /// panicking call(s) to the underlying writer, so simply writing it again is not a good idea. + #[must_use = "`self` will be dropped if the result is not used"] + #[stable(feature = "bufwriter_into_parts", since = "1.56.0")] + pub fn into_inner(self) -> Vec { + self.buf + } + + const DESCRIPTION: &'static str = + "BufWriter inner writer panicked, what data remains unwritten is not known"; +} + +#[stable(feature = "bufwriter_into_parts", since = "1.56.0")] +impl error::Error for WriterPanicked { + #[allow(deprecated, deprecated_in_future)] + fn description(&self) -> &str { + Self::DESCRIPTION + } +} + +#[stable(feature = "bufwriter_into_parts", since = "1.56.0")] +impl fmt::Display for WriterPanicked { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", Self::DESCRIPTION) + } +} + +#[stable(feature = "bufwriter_into_parts", since = "1.56.0")] +impl fmt::Debug for WriterPanicked { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("WriterPanicked") + .field("buffer", &format_args!("{}/{}", self.buf.len(), self.buf.capacity())) + .finish() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Write for BufWriter { + #[inline] + fn write(&mut self, buf: &[u8]) -> io::Result { + // Use < instead of <= to avoid a needless trip through the buffer in some cases. + // See `write_cold` for details. + if buf.len() < self.spare_capacity() { + // SAFETY: safe by above conditional. + unsafe { + self.write_to_buffer_unchecked(buf); + } + + Ok(buf.len()) + } else { + self.write_cold(buf) + } + } + + #[inline] + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + // Use < instead of <= to avoid a needless trip through the buffer in some cases. + // See `write_all_cold` for details. + if buf.len() < self.spare_capacity() { + // SAFETY: safe by above conditional. + unsafe { + self.write_to_buffer_unchecked(buf); + } + + Ok(()) + } else { + self.write_all_cold(buf) + } + } + + fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { + // FIXME: Consider applying `#[inline]` / `#[inline(never)]` optimizations already applied + // to `write` and `write_all`. The performance benefits can be significant. See #79930. + if self.get_ref().is_write_vectored() { + // We have to handle the possibility that the total length of the buffers overflows + // `usize` (even though this can only happen if multiple `IoSlice`s reference the + // same underlying buffer, as otherwise the buffers wouldn't fit in memory). If the + // computation overflows, then surely the input cannot fit in our buffer, so we forward + // to the inner writer's `write_vectored` method to let it handle it appropriately. + let saturated_total_len = + bufs.iter().fold(0usize, |acc, b| acc.saturating_add(b.len())); + + if saturated_total_len > self.spare_capacity() { + // Flush if the total length of the input exceeds our buffer's spare capacity. + // If we would have overflowed, this condition also holds, and we need to flush. + self.flush_buf()?; + } + + if saturated_total_len >= self.buf.capacity() { + // Forward to our inner writer if the total length of the input is greater than or + // equal to our buffer capacity. If we would have overflowed, this condition also + // holds, and we punt to the inner writer. + self.panicked = true; + let r = self.get_mut().write_vectored(bufs); + self.panicked = false; + r + } else { + // `saturated_total_len < self.buf.capacity()` implies that we did not saturate. + + // SAFETY: We checked whether or not the spare capacity was large enough above. If + // it was, then we're safe already. If it wasn't, we flushed, making sufficient + // room for any input <= the buffer size, which includes this input. + unsafe { + bufs.iter().for_each(|b| self.write_to_buffer_unchecked(b)); + }; + + Ok(saturated_total_len) + } + } else { + let mut iter = bufs.iter(); + let mut total_written = if let Some(buf) = iter.by_ref().find(|&buf| !buf.is_empty()) { + // This is the first non-empty slice to write, so if it does + // not fit in the buffer, we still get to flush and proceed. + if buf.len() > self.spare_capacity() { + self.flush_buf()?; + } + if buf.len() >= self.buf.capacity() { + // The slice is at least as large as the buffering capacity, + // so it's better to write it directly, bypassing the buffer. + self.panicked = true; + let r = self.get_mut().write(buf); + self.panicked = false; + return r; + } else { + // SAFETY: We checked whether or not the spare capacity was large enough above. + // If it was, then we're safe already. If it wasn't, we flushed, making + // sufficient room for any input <= the buffer size, which includes this input. + unsafe { + self.write_to_buffer_unchecked(buf); + } + + buf.len() + } + } else { + return Ok(0); + }; + debug_assert!(total_written != 0); + for buf in iter { + if buf.len() <= self.spare_capacity() { + // SAFETY: safe by above conditional. + unsafe { + self.write_to_buffer_unchecked(buf); + } + + // This cannot overflow `usize`. If we are here, we've written all of the bytes + // so far to our buffer, and we've ensured that we never exceed the buffer's + // capacity. Therefore, `total_written` <= `self.buf.capacity()` <= `usize::MAX`. + total_written += buf.len(); + } else { + break; + } + } + Ok(total_written) + } + } + + fn is_write_vectored(&self) -> bool { + true + } + + fn flush(&mut self) -> io::Result<()> { + self.flush_buf().and_then(|()| self.get_mut().flush()) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Debug for BufWriter +where + W: fmt::Debug, +{ + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("BufWriter") + .field("writer", &self.inner) + .field("buffer", &format_args!("{}/{}", self.buf.len(), self.buf.capacity())) + .finish() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Seek for BufWriter { + /// Seek to the offset, in bytes, in the underlying writer. + /// + /// Seeking always writes out the internal buffer before seeking. + fn seek(&mut self, pos: SeekFrom) -> io::Result { + self.flush_buf()?; + self.get_mut().seek(pos) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Drop for BufWriter { + fn drop(&mut self) { + if !self.panicked { + // dtors should not panic, so we ignore a failed flush + let _r = self.flush_buf(); + } + } +} diff --git a/crux-mir/lib/std/src/io/buffered/linewriter.rs b/crux-mir/lib/std/src/io/buffered/linewriter.rs new file mode 100644 index 000000000..a26a4ab33 --- /dev/null +++ b/crux-mir/lib/std/src/io/buffered/linewriter.rs @@ -0,0 +1,232 @@ +use crate::fmt; +use crate::io::{self, buffered::LineWriterShim, BufWriter, IntoInnerError, IoSlice, Write}; + +/// Wraps a writer and buffers output to it, flushing whenever a newline +/// (`0x0a`, `'\n'`) is detected. +/// +/// The [`BufWriter`] struct wraps a writer and buffers its output. +/// But it only does this batched write when it goes out of scope, or when the +/// internal buffer is full. Sometimes, you'd prefer to write each line as it's +/// completed, rather than the entire buffer at once. Enter `LineWriter`. It +/// does exactly that. +/// +/// Like [`BufWriter`], a `LineWriter`’s buffer will also be flushed when the +/// `LineWriter` goes out of scope or when its internal buffer is full. +/// +/// If there's still a partial line in the buffer when the `LineWriter` is +/// dropped, it will flush those contents. +/// +/// # Examples +/// +/// We can use `LineWriter` to write one line at a time, significantly +/// reducing the number of actual writes to the file. +/// +/// ```no_run +/// use std::fs::{self, File}; +/// use std::io::prelude::*; +/// use std::io::LineWriter; +/// +/// fn main() -> std::io::Result<()> { +/// let road_not_taken = b"I shall be telling this with a sigh +/// Somewhere ages and ages hence: +/// Two roads diverged in a wood, and I - +/// I took the one less traveled by, +/// And that has made all the difference."; +/// +/// let file = File::create("poem.txt")?; +/// let mut file = LineWriter::new(file); +/// +/// file.write_all(b"I shall be telling this with a sigh")?; +/// +/// // No bytes are written until a newline is encountered (or +/// // the internal buffer is filled). +/// assert_eq!(fs::read_to_string("poem.txt")?, ""); +/// file.write_all(b"\n")?; +/// assert_eq!( +/// fs::read_to_string("poem.txt")?, +/// "I shall be telling this with a sigh\n", +/// ); +/// +/// // Write the rest of the poem. +/// file.write_all(b"Somewhere ages and ages hence: +/// Two roads diverged in a wood, and I - +/// I took the one less traveled by, +/// And that has made all the difference.")?; +/// +/// // The last line of the poem doesn't end in a newline, so +/// // we have to flush or drop the `LineWriter` to finish +/// // writing. +/// file.flush()?; +/// +/// // Confirm the whole poem was written. +/// assert_eq!(fs::read("poem.txt")?, &road_not_taken[..]); +/// Ok(()) +/// } +/// ``` +#[stable(feature = "rust1", since = "1.0.0")] +pub struct LineWriter { + inner: BufWriter, +} + +impl LineWriter { + /// Creates a new `LineWriter`. + /// + /// # Examples + /// + /// ```no_run + /// use std::fs::File; + /// use std::io::LineWriter; + /// + /// fn main() -> std::io::Result<()> { + /// let file = File::create("poem.txt")?; + /// let file = LineWriter::new(file); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn new(inner: W) -> LineWriter { + // Lines typically aren't that long, don't use a giant buffer + LineWriter::with_capacity(1024, inner) + } + + /// Creates a new `LineWriter` with at least the specified capacity for the + /// internal buffer. + /// + /// # Examples + /// + /// ```no_run + /// use std::fs::File; + /// use std::io::LineWriter; + /// + /// fn main() -> std::io::Result<()> { + /// let file = File::create("poem.txt")?; + /// let file = LineWriter::with_capacity(100, file); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn with_capacity(capacity: usize, inner: W) -> LineWriter { + LineWriter { inner: BufWriter::with_capacity(capacity, inner) } + } + + /// Gets a reference to the underlying writer. + /// + /// # Examples + /// + /// ```no_run + /// use std::fs::File; + /// use std::io::LineWriter; + /// + /// fn main() -> std::io::Result<()> { + /// let file = File::create("poem.txt")?; + /// let file = LineWriter::new(file); + /// + /// let reference = file.get_ref(); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn get_ref(&self) -> &W { + self.inner.get_ref() + } + + /// Gets a mutable reference to the underlying writer. + /// + /// Caution must be taken when calling methods on the mutable reference + /// returned as extra writes could corrupt the output stream. + /// + /// # Examples + /// + /// ```no_run + /// use std::fs::File; + /// use std::io::LineWriter; + /// + /// fn main() -> std::io::Result<()> { + /// let file = File::create("poem.txt")?; + /// let mut file = LineWriter::new(file); + /// + /// // we can use reference just like file + /// let reference = file.get_mut(); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn get_mut(&mut self) -> &mut W { + self.inner.get_mut() + } + + /// Unwraps this `LineWriter`, returning the underlying writer. + /// + /// The internal buffer is written out before returning the writer. + /// + /// # Errors + /// + /// An [`Err`] will be returned if an error occurs while flushing the buffer. + /// + /// # Examples + /// + /// ```no_run + /// use std::fs::File; + /// use std::io::LineWriter; + /// + /// fn main() -> std::io::Result<()> { + /// let file = File::create("poem.txt")?; + /// + /// let writer: LineWriter = LineWriter::new(file); + /// + /// let file: File = writer.into_inner()?; + /// Ok(()) + /// } + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn into_inner(self) -> Result>> { + self.inner.into_inner().map_err(|err| err.new_wrapped(|inner| LineWriter { inner })) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Write for LineWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + LineWriterShim::new(&mut self.inner).write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } + + fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { + LineWriterShim::new(&mut self.inner).write_vectored(bufs) + } + + fn is_write_vectored(&self) -> bool { + self.inner.is_write_vectored() + } + + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + LineWriterShim::new(&mut self.inner).write_all(buf) + } + + fn write_all_vectored(&mut self, bufs: &mut [IoSlice<'_>]) -> io::Result<()> { + LineWriterShim::new(&mut self.inner).write_all_vectored(bufs) + } + + fn write_fmt(&mut self, fmt: fmt::Arguments<'_>) -> io::Result<()> { + LineWriterShim::new(&mut self.inner).write_fmt(fmt) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Debug for LineWriter +where + W: fmt::Debug, +{ + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("LineWriter") + .field("writer", &self.get_ref()) + .field( + "buffer", + &format_args!("{}/{}", self.inner.buffer().len(), self.inner.capacity()), + ) + .finish_non_exhaustive() + } +} diff --git a/crux-mir/lib/std/src/io/buffered/linewritershim.rs b/crux-mir/lib/std/src/io/buffered/linewritershim.rs new file mode 100644 index 000000000..0175d2693 --- /dev/null +++ b/crux-mir/lib/std/src/io/buffered/linewritershim.rs @@ -0,0 +1,276 @@ +use crate::io::{self, BufWriter, IoSlice, Write}; +use crate::sys_common::memchr; + +/// Private helper struct for implementing the line-buffered writing logic. +/// This shim temporarily wraps a BufWriter, and uses its internals to +/// implement a line-buffered writer (specifically by using the internal +/// methods like write_to_buf and flush_buf). In this way, a more +/// efficient abstraction can be created than one that only had access to +/// `write` and `flush`, without needlessly duplicating a lot of the +/// implementation details of BufWriter. This also allows existing +/// `BufWriters` to be temporarily given line-buffering logic; this is what +/// enables Stdout to be alternately in line-buffered or block-buffered mode. +#[derive(Debug)] +pub struct LineWriterShim<'a, W: Write> { + buffer: &'a mut BufWriter, +} + +impl<'a, W: Write> LineWriterShim<'a, W> { + pub fn new(buffer: &'a mut BufWriter) -> Self { + Self { buffer } + } + + /// Get a reference to the inner writer (that is, the writer + /// wrapped by the BufWriter). + fn inner(&self) -> &W { + self.buffer.get_ref() + } + + /// Get a mutable reference to the inner writer (that is, the writer + /// wrapped by the BufWriter). Be careful with this writer, as writes to + /// it will bypass the buffer. + fn inner_mut(&mut self) -> &mut W { + self.buffer.get_mut() + } + + /// Get the content currently buffered in self.buffer + fn buffered(&self) -> &[u8] { + self.buffer.buffer() + } + + /// Flush the buffer iff the last byte is a newline (indicating that an + /// earlier write only succeeded partially, and we want to retry flushing + /// the buffered line before continuing with a subsequent write) + fn flush_if_completed_line(&mut self) -> io::Result<()> { + match self.buffered().last().copied() { + Some(b'\n') => self.buffer.flush_buf(), + _ => Ok(()), + } + } +} + +impl<'a, W: Write> Write for LineWriterShim<'a, W> { + /// Write some data into this BufReader with line buffering. This means + /// that, if any newlines are present in the data, the data up to the last + /// newline is sent directly to the underlying writer, and data after it + /// is buffered. Returns the number of bytes written. + /// + /// This function operates on a "best effort basis"; in keeping with the + /// convention of `Write::write`, it makes at most one attempt to write + /// new data to the underlying writer. If that write only reports a partial + /// success, the remaining data will be buffered. + /// + /// Because this function attempts to send completed lines to the underlying + /// writer, it will also flush the existing buffer if it ends with a + /// newline, even if the incoming data does not contain any newlines. + fn write(&mut self, buf: &[u8]) -> io::Result { + let newline_idx = match memchr::memrchr(b'\n', buf) { + // If there are no new newlines (that is, if this write is less than + // one line), just do a regular buffered write (which may flush if + // we exceed the inner buffer's size) + None => { + self.flush_if_completed_line()?; + return self.buffer.write(buf); + } + // Otherwise, arrange for the lines to be written directly to the + // inner writer. + Some(newline_idx) => newline_idx + 1, + }; + + // Flush existing content to prepare for our write. We have to do this + // before attempting to write `buf` in order to maintain consistency; + // if we add `buf` to the buffer then try to flush it all at once, + // we're obligated to return Ok(), which would mean suppressing any + // errors that occur during flush. + self.buffer.flush_buf()?; + + // This is what we're going to try to write directly to the inner + // writer. The rest will be buffered, if nothing goes wrong. + let lines = &buf[..newline_idx]; + + // Write `lines` directly to the inner writer. In keeping with the + // `write` convention, make at most one attempt to add new (unbuffered) + // data. Because this write doesn't touch the BufWriter state directly, + // and the buffer is known to be empty, we don't need to worry about + // self.buffer.panicked here. + let flushed = self.inner_mut().write(lines)?; + + // If buffer returns Ok(0), propagate that to the caller without + // doing additional buffering; otherwise we're just guaranteeing + // an "ErrorKind::WriteZero" later. + if flushed == 0 { + return Ok(0); + } + + // Now that the write has succeeded, buffer the rest (or as much of + // the rest as possible). If there were any unwritten newlines, we + // only buffer out to the last unwritten newline that fits in the + // buffer; this helps prevent flushing partial lines on subsequent + // calls to LineWriterShim::write. + + // Handle the cases in order of most-common to least-common, under + // the presumption that most writes succeed in totality, and that most + // writes are smaller than the buffer. + // - Is this a partial line (ie, no newlines left in the unwritten tail) + // - If not, does the data out to the last unwritten newline fit in + // the buffer? + // - If not, scan for the last newline that *does* fit in the buffer + let tail = if flushed >= newline_idx { + &buf[flushed..] + } else if newline_idx - flushed <= self.buffer.capacity() { + &buf[flushed..newline_idx] + } else { + let scan_area = &buf[flushed..]; + let scan_area = &scan_area[..self.buffer.capacity()]; + match memchr::memrchr(b'\n', scan_area) { + Some(newline_idx) => &scan_area[..newline_idx + 1], + None => scan_area, + } + }; + + let buffered = self.buffer.write_to_buf(tail); + Ok(flushed + buffered) + } + + fn flush(&mut self) -> io::Result<()> { + self.buffer.flush() + } + + /// Write some vectored data into this BufReader with line buffering. This + /// means that, if any newlines are present in the data, the data up to + /// and including the buffer containing the last newline is sent directly + /// to the inner writer, and the data after it is buffered. Returns the + /// number of bytes written. + /// + /// This function operates on a "best effort basis"; in keeping with the + /// convention of `Write::write`, it makes at most one attempt to write + /// new data to the underlying writer. + /// + /// Because this function attempts to send completed lines to the underlying + /// writer, it will also flush the existing buffer if it contains any + /// newlines. + /// + /// Because sorting through an array of `IoSlice` can be a bit convoluted, + /// This method differs from write in the following ways: + /// + /// - It attempts to write the full content of all the buffers up to and + /// including the one containing the last newline. This means that it + /// may attempt to write a partial line, that buffer has data past the + /// newline. + /// - If the write only reports partial success, it does not attempt to + /// find the precise location of the written bytes and buffer the rest. + /// + /// If the underlying vector doesn't support vectored writing, we instead + /// simply write the first non-empty buffer with `write`. This way, we + /// get the benefits of more granular partial-line handling without losing + /// anything in efficiency + fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { + // If there's no specialized behavior for write_vectored, just use + // write. This has the benefit of more granular partial-line handling. + if !self.is_write_vectored() { + return match bufs.iter().find(|buf| !buf.is_empty()) { + Some(buf) => self.write(buf), + None => Ok(0), + }; + } + + // Find the buffer containing the last newline + let last_newline_buf_idx = bufs + .iter() + .enumerate() + .rev() + .find_map(|(i, buf)| memchr::memchr(b'\n', buf).map(|_| i)); + + // If there are no new newlines (that is, if this write is less than + // one line), just do a regular buffered write + let last_newline_buf_idx = match last_newline_buf_idx { + // No newlines; just do a normal buffered write + None => { + self.flush_if_completed_line()?; + return self.buffer.write_vectored(bufs); + } + Some(i) => i, + }; + + // Flush existing content to prepare for our write + self.buffer.flush_buf()?; + + // This is what we're going to try to write directly to the inner + // writer. The rest will be buffered, if nothing goes wrong. + let (lines, tail) = bufs.split_at(last_newline_buf_idx + 1); + + // Write `lines` directly to the inner writer. In keeping with the + // `write` convention, make at most one attempt to add new (unbuffered) + // data. Because this write doesn't touch the BufWriter state directly, + // and the buffer is known to be empty, we don't need to worry about + // self.panicked here. + let flushed = self.inner_mut().write_vectored(lines)?; + + // If inner returns Ok(0), propagate that to the caller without + // doing additional buffering; otherwise we're just guaranteeing + // an "ErrorKind::WriteZero" later. + if flushed == 0 { + return Ok(0); + } + + // Don't try to reconstruct the exact amount written; just bail + // in the event of a partial write + let lines_len = lines.iter().map(|buf| buf.len()).sum(); + if flushed < lines_len { + return Ok(flushed); + } + + // Now that the write has succeeded, buffer the rest (or as much of the + // rest as possible) + let buffered: usize = tail + .iter() + .filter(|buf| !buf.is_empty()) + .map(|buf| self.buffer.write_to_buf(buf)) + .take_while(|&n| n > 0) + .sum(); + + Ok(flushed + buffered) + } + + fn is_write_vectored(&self) -> bool { + self.inner().is_write_vectored() + } + + /// Write some data into this BufReader with line buffering. This means + /// that, if any newlines are present in the data, the data up to the last + /// newline is sent directly to the underlying writer, and data after it + /// is buffered. + /// + /// Because this function attempts to send completed lines to the underlying + /// writer, it will also flush the existing buffer if it contains any + /// newlines, even if the incoming data does not contain any newlines. + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + match memchr::memrchr(b'\n', buf) { + // If there are no new newlines (that is, if this write is less than + // one line), just do a regular buffered write (which may flush if + // we exceed the inner buffer's size) + None => { + self.flush_if_completed_line()?; + self.buffer.write_all(buf) + } + Some(newline_idx) => { + let (lines, tail) = buf.split_at(newline_idx + 1); + + if self.buffered().is_empty() { + self.inner_mut().write_all(lines)?; + } else { + // If there is any buffered data, we add the incoming lines + // to that buffer before flushing, which saves us at least + // one write call. We can't really do this with `write`, + // since we can't do this *and* not suppress errors *and* + // report a consistent state to the caller in a return + // value, but here in write_all it's fine. + self.buffer.write_all(lines)?; + self.buffer.flush_buf()?; + } + + self.buffer.write_all(tail) + } + } + } +} diff --git a/crux-mir/lib/std/src/io/buffered/mod.rs b/crux-mir/lib/std/src/io/buffered/mod.rs new file mode 100644 index 000000000..100dab1e2 --- /dev/null +++ b/crux-mir/lib/std/src/io/buffered/mod.rs @@ -0,0 +1,196 @@ +//! Buffering wrappers for I/O traits + +mod bufreader; +mod bufwriter; +mod linewriter; +mod linewritershim; + +#[cfg(test)] +mod tests; + +use crate::error; +use crate::fmt; +use crate::io::Error; + +#[stable(feature = "rust1", since = "1.0.0")] +pub use self::{bufreader::BufReader, bufwriter::BufWriter, linewriter::LineWriter}; +use linewritershim::LineWriterShim; + +#[stable(feature = "bufwriter_into_parts", since = "1.56.0")] +pub use bufwriter::WriterPanicked; + +/// An error returned by [`BufWriter::into_inner`] which combines an error that +/// happened while writing out the buffer, and the buffered writer object +/// which may be used to recover from the condition. +/// +/// # Examples +/// +/// ```no_run +/// use std::io::BufWriter; +/// use std::net::TcpStream; +/// +/// let mut stream = BufWriter::new(TcpStream::connect("127.0.0.1:34254").unwrap()); +/// +/// // do stuff with the stream +/// +/// // we want to get our `TcpStream` back, so let's try: +/// +/// let stream = match stream.into_inner() { +/// Ok(s) => s, +/// Err(e) => { +/// // Here, e is an IntoInnerError +/// panic!("An error occurred"); +/// } +/// }; +/// ``` +#[derive(Debug)] +#[stable(feature = "rust1", since = "1.0.0")] +pub struct IntoInnerError(W, Error); + +impl IntoInnerError { + /// Construct a new IntoInnerError + fn new(writer: W, error: Error) -> Self { + Self(writer, error) + } + + /// Helper to construct a new IntoInnerError; intended to help with + /// adapters that wrap other adapters + fn new_wrapped(self, f: impl FnOnce(W) -> W2) -> IntoInnerError { + let Self(writer, error) = self; + IntoInnerError::new(f(writer), error) + } + + /// Returns the error which caused the call to [`BufWriter::into_inner()`] + /// to fail. + /// + /// This error was returned when attempting to write the internal buffer. + /// + /// # Examples + /// + /// ```no_run + /// use std::io::BufWriter; + /// use std::net::TcpStream; + /// + /// let mut stream = BufWriter::new(TcpStream::connect("127.0.0.1:34254").unwrap()); + /// + /// // do stuff with the stream + /// + /// // we want to get our `TcpStream` back, so let's try: + /// + /// let stream = match stream.into_inner() { + /// Ok(s) => s, + /// Err(e) => { + /// // Here, e is an IntoInnerError, let's log the inner error. + /// // + /// // We'll just 'log' to stdout for this example. + /// println!("{}", e.error()); + /// + /// panic!("An unexpected error occurred."); + /// } + /// }; + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn error(&self) -> &Error { + &self.1 + } + + /// Returns the buffered writer instance which generated the error. + /// + /// The returned object can be used for error recovery, such as + /// re-inspecting the buffer. + /// + /// # Examples + /// + /// ```no_run + /// use std::io::BufWriter; + /// use std::net::TcpStream; + /// + /// let mut stream = BufWriter::new(TcpStream::connect("127.0.0.1:34254").unwrap()); + /// + /// // do stuff with the stream + /// + /// // we want to get our `TcpStream` back, so let's try: + /// + /// let stream = match stream.into_inner() { + /// Ok(s) => s, + /// Err(e) => { + /// // Here, e is an IntoInnerError, let's re-examine the buffer: + /// let buffer = e.into_inner(); + /// + /// // do stuff to try to recover + /// + /// // afterwards, let's just return the stream + /// buffer.into_inner().unwrap() + /// } + /// }; + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn into_inner(self) -> W { + self.0 + } + + /// Consumes the [`IntoInnerError`] and returns the error which caused the call to + /// [`BufWriter::into_inner()`] to fail. Unlike `error`, this can be used to + /// obtain ownership of the underlying error. + /// + /// # Example + /// ``` + /// use std::io::{BufWriter, ErrorKind, Write}; + /// + /// let mut not_enough_space = [0u8; 10]; + /// let mut stream = BufWriter::new(not_enough_space.as_mut()); + /// write!(stream, "this cannot be actually written").unwrap(); + /// let into_inner_err = stream.into_inner().expect_err("now we discover it's too small"); + /// let err = into_inner_err.into_error(); + /// assert_eq!(err.kind(), ErrorKind::WriteZero); + /// ``` + #[stable(feature = "io_into_inner_error_parts", since = "1.55.0")] + pub fn into_error(self) -> Error { + self.1 + } + + /// Consumes the [`IntoInnerError`] and returns the error which caused the call to + /// [`BufWriter::into_inner()`] to fail, and the underlying writer. + /// + /// This can be used to simply obtain ownership of the underlying error; it can also be used for + /// advanced error recovery. + /// + /// # Example + /// ``` + /// use std::io::{BufWriter, ErrorKind, Write}; + /// + /// let mut not_enough_space = [0u8; 10]; + /// let mut stream = BufWriter::new(not_enough_space.as_mut()); + /// write!(stream, "this cannot be actually written").unwrap(); + /// let into_inner_err = stream.into_inner().expect_err("now we discover it's too small"); + /// let (err, recovered_writer) = into_inner_err.into_parts(); + /// assert_eq!(err.kind(), ErrorKind::WriteZero); + /// assert_eq!(recovered_writer.buffer(), b"t be actually written"); + /// ``` + #[stable(feature = "io_into_inner_error_parts", since = "1.55.0")] + pub fn into_parts(self) -> (Error, W) { + (self.1, self.0) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl From> for Error { + fn from(iie: IntoInnerError) -> Error { + iie.1 + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl error::Error for IntoInnerError { + #[allow(deprecated, deprecated_in_future)] + fn description(&self) -> &str { + error::Error::description(self.error()) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Display for IntoInnerError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.error().fmt(f) + } +} diff --git a/crux-mir/lib/std/src/io/buffered/tests.rs b/crux-mir/lib/std/src/io/buffered/tests.rs new file mode 100644 index 000000000..4c1b7d576 --- /dev/null +++ b/crux-mir/lib/std/src/io/buffered/tests.rs @@ -0,0 +1,1065 @@ +use crate::io::prelude::*; +use crate::io::{ + self, BorrowedBuf, BufReader, BufWriter, ErrorKind, IoSlice, LineWriter, SeekFrom, +}; +use crate::mem::MaybeUninit; +use crate::panic; +use crate::sync::atomic::{AtomicUsize, Ordering}; +use crate::thread; + +/// A dummy reader intended at testing short-reads propagation. +pub struct ShortReader { + lengths: Vec, +} + +// FIXME: rustfmt and tidy disagree about the correct formatting of this +// function. This leads to issues for users with editors configured to +// rustfmt-on-save. +impl Read for ShortReader { + fn read(&mut self, _: &mut [u8]) -> io::Result { + if self.lengths.is_empty() { Ok(0) } else { Ok(self.lengths.remove(0)) } + } +} + +#[test] +fn test_buffered_reader() { + let inner: &[u8] = &[5, 6, 7, 0, 1, 2, 3, 4]; + let mut reader = BufReader::with_capacity(2, inner); + + let mut buf = [0, 0, 0]; + let nread = reader.read(&mut buf); + assert_eq!(nread.unwrap(), 3); + assert_eq!(buf, [5, 6, 7]); + assert_eq!(reader.buffer(), []); + + let mut buf = [0, 0]; + let nread = reader.read(&mut buf); + assert_eq!(nread.unwrap(), 2); + assert_eq!(buf, [0, 1]); + assert_eq!(reader.buffer(), []); + + let mut buf = [0]; + let nread = reader.read(&mut buf); + assert_eq!(nread.unwrap(), 1); + assert_eq!(buf, [2]); + assert_eq!(reader.buffer(), [3]); + + let mut buf = [0, 0, 0]; + let nread = reader.read(&mut buf); + assert_eq!(nread.unwrap(), 1); + assert_eq!(buf, [3, 0, 0]); + assert_eq!(reader.buffer(), []); + + let nread = reader.read(&mut buf); + assert_eq!(nread.unwrap(), 1); + assert_eq!(buf, [4, 0, 0]); + assert_eq!(reader.buffer(), []); + + assert_eq!(reader.read(&mut buf).unwrap(), 0); +} + +#[test] +fn test_buffered_reader_read_buf() { + let inner: &[u8] = &[5, 6, 7, 0, 1, 2, 3, 4]; + let mut reader = BufReader::with_capacity(2, inner); + + let buf: &mut [_] = &mut [MaybeUninit::uninit(); 3]; + let mut buf: BorrowedBuf<'_> = buf.into(); + + reader.read_buf(buf.unfilled()).unwrap(); + + assert_eq!(buf.filled(), [5, 6, 7]); + assert_eq!(reader.buffer(), []); + + let buf: &mut [_] = &mut [MaybeUninit::uninit(); 2]; + let mut buf: BorrowedBuf<'_> = buf.into(); + + reader.read_buf(buf.unfilled()).unwrap(); + + assert_eq!(buf.filled(), [0, 1]); + assert_eq!(reader.buffer(), []); + + let buf: &mut [_] = &mut [MaybeUninit::uninit(); 1]; + let mut buf: BorrowedBuf<'_> = buf.into(); + + reader.read_buf(buf.unfilled()).unwrap(); + + assert_eq!(buf.filled(), [2]); + assert_eq!(reader.buffer(), [3]); + + let buf: &mut [_] = &mut [MaybeUninit::uninit(); 3]; + let mut buf: BorrowedBuf<'_> = buf.into(); + + reader.read_buf(buf.unfilled()).unwrap(); + + assert_eq!(buf.filled(), [3]); + assert_eq!(reader.buffer(), []); + + reader.read_buf(buf.unfilled()).unwrap(); + + assert_eq!(buf.filled(), [3, 4]); + assert_eq!(reader.buffer(), []); + + buf.clear(); + + reader.read_buf(buf.unfilled()).unwrap(); + + assert!(buf.filled().is_empty()); +} + +#[test] +fn test_buffered_reader_seek() { + let inner: &[u8] = &[5, 6, 7, 0, 1, 2, 3, 4]; + let mut reader = BufReader::with_capacity(2, io::Cursor::new(inner)); + + assert_eq!(reader.seek(SeekFrom::Start(3)).ok(), Some(3)); + assert_eq!(reader.fill_buf().ok(), Some(&[0, 1][..])); + assert_eq!(reader.seek(SeekFrom::Current(0)).ok(), Some(3)); + assert_eq!(reader.fill_buf().ok(), Some(&[0, 1][..])); + assert_eq!(reader.seek(SeekFrom::Current(1)).ok(), Some(4)); + assert_eq!(reader.fill_buf().ok(), Some(&[1, 2][..])); + reader.consume(1); + assert_eq!(reader.seek(SeekFrom::Current(-2)).ok(), Some(3)); +} + +#[test] +fn test_buffered_reader_seek_relative() { + let inner: &[u8] = &[5, 6, 7, 0, 1, 2, 3, 4]; + let mut reader = BufReader::with_capacity(2, io::Cursor::new(inner)); + + assert!(reader.seek_relative(3).is_ok()); + assert_eq!(reader.fill_buf().ok(), Some(&[0, 1][..])); + assert!(reader.seek_relative(0).is_ok()); + assert_eq!(reader.fill_buf().ok(), Some(&[0, 1][..])); + assert!(reader.seek_relative(1).is_ok()); + assert_eq!(reader.fill_buf().ok(), Some(&[1][..])); + assert!(reader.seek_relative(-1).is_ok()); + assert_eq!(reader.fill_buf().ok(), Some(&[0, 1][..])); + assert!(reader.seek_relative(2).is_ok()); + assert_eq!(reader.fill_buf().ok(), Some(&[2, 3][..])); +} + +#[test] +fn test_buffered_reader_stream_position() { + let inner: &[u8] = &[5, 6, 7, 0, 1, 2, 3, 4]; + let mut reader = BufReader::with_capacity(2, io::Cursor::new(inner)); + + assert_eq!(reader.stream_position().ok(), Some(0)); + assert_eq!(reader.seek(SeekFrom::Start(3)).ok(), Some(3)); + assert_eq!(reader.stream_position().ok(), Some(3)); + // relative seeking within the buffer and reading position should keep the buffer + assert_eq!(reader.fill_buf().ok(), Some(&[0, 1][..])); + assert!(reader.seek_relative(0).is_ok()); + assert_eq!(reader.stream_position().ok(), Some(3)); + assert_eq!(reader.buffer(), &[0, 1][..]); + assert!(reader.seek_relative(1).is_ok()); + assert_eq!(reader.stream_position().ok(), Some(4)); + assert_eq!(reader.buffer(), &[1][..]); + assert!(reader.seek_relative(-1).is_ok()); + assert_eq!(reader.stream_position().ok(), Some(3)); + assert_eq!(reader.buffer(), &[0, 1][..]); + // relative seeking outside the buffer will discard it + assert!(reader.seek_relative(2).is_ok()); + assert_eq!(reader.stream_position().ok(), Some(5)); + assert_eq!(reader.buffer(), &[][..]); +} + +#[test] +fn test_buffered_reader_stream_position_panic() { + let inner: &[u8] = &[5, 6, 7, 0, 1, 2, 3, 4]; + let mut reader = BufReader::with_capacity(4, io::Cursor::new(inner)); + + // cause internal buffer to be filled but read only partially + let mut buffer = [0, 0]; + assert!(reader.read_exact(&mut buffer).is_ok()); + // rewinding the internal reader will cause buffer to loose sync + let inner = reader.get_mut(); + assert!(inner.seek(SeekFrom::Start(0)).is_ok()); + // overflow when subtracting the remaining buffer size from current position + let result = panic::catch_unwind(panic::AssertUnwindSafe(|| reader.stream_position().ok())); + assert!(result.is_err()); +} + +#[test] +fn test_buffered_reader_invalidated_after_read() { + let inner: &[u8] = &[5, 6, 7, 0, 1, 2, 3, 4]; + let mut reader = BufReader::with_capacity(3, io::Cursor::new(inner)); + + assert_eq!(reader.fill_buf().ok(), Some(&[5, 6, 7][..])); + reader.consume(3); + + let mut buffer = [0, 0, 0, 0, 0]; + assert_eq!(reader.read(&mut buffer).ok(), Some(5)); + assert_eq!(buffer, [0, 1, 2, 3, 4]); + + assert!(reader.seek_relative(-2).is_ok()); + let mut buffer = [0, 0]; + assert_eq!(reader.read(&mut buffer).ok(), Some(2)); + assert_eq!(buffer, [3, 4]); +} + +#[test] +fn test_buffered_reader_invalidated_after_seek() { + let inner: &[u8] = &[5, 6, 7, 0, 1, 2, 3, 4]; + let mut reader = BufReader::with_capacity(3, io::Cursor::new(inner)); + + assert_eq!(reader.fill_buf().ok(), Some(&[5, 6, 7][..])); + reader.consume(3); + + assert!(reader.seek(SeekFrom::Current(5)).is_ok()); + + assert!(reader.seek_relative(-2).is_ok()); + let mut buffer = [0, 0]; + assert_eq!(reader.read(&mut buffer).ok(), Some(2)); + assert_eq!(buffer, [3, 4]); +} + +#[test] +fn test_buffered_reader_seek_underflow() { + // gimmick reader that yields its position modulo 256 for each byte + struct PositionReader { + pos: u64, + } + impl Read for PositionReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let len = buf.len(); + for x in buf { + *x = self.pos as u8; + self.pos = self.pos.wrapping_add(1); + } + Ok(len) + } + } + impl Seek for PositionReader { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + match pos { + SeekFrom::Start(n) => { + self.pos = n; + } + SeekFrom::Current(n) => { + self.pos = self.pos.wrapping_add(n as u64); + } + SeekFrom::End(n) => { + self.pos = u64::MAX.wrapping_add(n as u64); + } + } + Ok(self.pos) + } + } + + let mut reader = BufReader::with_capacity(5, PositionReader { pos: 0 }); + assert_eq!(reader.fill_buf().ok(), Some(&[0, 1, 2, 3, 4][..])); + assert_eq!(reader.seek(SeekFrom::End(-5)).ok(), Some(u64::MAX - 5)); + assert_eq!(reader.fill_buf().ok().map(|s| s.len()), Some(5)); + // the following seek will require two underlying seeks + let expected = 9223372036854775802; + assert_eq!(reader.seek(SeekFrom::Current(i64::MIN)).ok(), Some(expected)); + assert_eq!(reader.fill_buf().ok().map(|s| s.len()), Some(5)); + // seeking to 0 should empty the buffer. + assert_eq!(reader.seek(SeekFrom::Current(0)).ok(), Some(expected)); + assert_eq!(reader.get_ref().pos, expected); +} + +#[test] +fn test_buffered_reader_seek_underflow_discard_buffer_between_seeks() { + // gimmick reader that returns Err after first seek + struct ErrAfterFirstSeekReader { + first_seek: bool, + } + impl Read for ErrAfterFirstSeekReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + for x in &mut *buf { + *x = 0; + } + Ok(buf.len()) + } + } + impl Seek for ErrAfterFirstSeekReader { + fn seek(&mut self, _: SeekFrom) -> io::Result { + if self.first_seek { + self.first_seek = false; + Ok(0) + } else { + Err(io::Error::new(io::ErrorKind::Other, "oh no!")) + } + } + } + + let mut reader = BufReader::with_capacity(5, ErrAfterFirstSeekReader { first_seek: true }); + assert_eq!(reader.fill_buf().ok(), Some(&[0, 0, 0, 0, 0][..])); + + // The following seek will require two underlying seeks. The first will + // succeed but the second will fail. This should still invalidate the + // buffer. + assert!(reader.seek(SeekFrom::Current(i64::MIN)).is_err()); + assert_eq!(reader.buffer().len(), 0); +} + +#[test] +fn test_buffered_reader_read_to_end_consumes_buffer() { + let data: &[u8] = &[0, 1, 2, 3, 4, 5, 6, 7]; + let mut reader = BufReader::with_capacity(3, data); + let mut buf = Vec::new(); + assert_eq!(reader.fill_buf().ok(), Some(&[0, 1, 2][..])); + assert_eq!(reader.read_to_end(&mut buf).ok(), Some(8)); + assert_eq!(&buf, &[0, 1, 2, 3, 4, 5, 6, 7]); + assert!(reader.buffer().is_empty()); +} + +#[test] +fn test_buffered_reader_read_to_string_consumes_buffer() { + let data: &[u8] = "deadbeef".as_bytes(); + let mut reader = BufReader::with_capacity(3, data); + let mut buf = String::new(); + assert_eq!(reader.fill_buf().ok(), Some("dea".as_bytes())); + assert_eq!(reader.read_to_string(&mut buf).ok(), Some(8)); + assert_eq!(&buf, "deadbeef"); + assert!(reader.buffer().is_empty()); +} + +#[test] +fn test_buffered_writer() { + let inner = Vec::new(); + let mut writer = BufWriter::with_capacity(2, inner); + + writer.write(&[0, 1]).unwrap(); + assert_eq!(writer.buffer(), []); + assert_eq!(*writer.get_ref(), [0, 1]); + + writer.write(&[2]).unwrap(); + assert_eq!(writer.buffer(), [2]); + assert_eq!(*writer.get_ref(), [0, 1]); + + writer.write(&[3]).unwrap(); + assert_eq!(writer.buffer(), [2, 3]); + assert_eq!(*writer.get_ref(), [0, 1]); + + writer.flush().unwrap(); + assert_eq!(writer.buffer(), []); + assert_eq!(*writer.get_ref(), [0, 1, 2, 3]); + + writer.write(&[4]).unwrap(); + writer.write(&[5]).unwrap(); + assert_eq!(writer.buffer(), [4, 5]); + assert_eq!(*writer.get_ref(), [0, 1, 2, 3]); + + writer.write(&[6]).unwrap(); + assert_eq!(writer.buffer(), [6]); + assert_eq!(*writer.get_ref(), [0, 1, 2, 3, 4, 5]); + + writer.write(&[7, 8]).unwrap(); + assert_eq!(writer.buffer(), []); + assert_eq!(*writer.get_ref(), [0, 1, 2, 3, 4, 5, 6, 7, 8]); + + writer.write(&[9, 10, 11]).unwrap(); + assert_eq!(writer.buffer(), []); + assert_eq!(*writer.get_ref(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); + + writer.flush().unwrap(); + assert_eq!(writer.buffer(), []); + assert_eq!(*writer.get_ref(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); +} + +#[test] +fn test_buffered_writer_inner_flushes() { + let mut w = BufWriter::with_capacity(3, Vec::new()); + w.write(&[0, 1]).unwrap(); + assert_eq!(*w.get_ref(), []); + let w = w.into_inner().unwrap(); + assert_eq!(w, [0, 1]); +} + +#[test] +fn test_buffered_writer_seek() { + let mut w = BufWriter::with_capacity(3, io::Cursor::new(Vec::new())); + w.write_all(&[0, 1, 2, 3, 4, 5]).unwrap(); + w.write_all(&[6, 7]).unwrap(); + assert_eq!(w.seek(SeekFrom::Current(0)).ok(), Some(8)); + assert_eq!(&w.get_ref().get_ref()[..], &[0, 1, 2, 3, 4, 5, 6, 7][..]); + assert_eq!(w.seek(SeekFrom::Start(2)).ok(), Some(2)); + w.write_all(&[8, 9]).unwrap(); + assert_eq!(&w.into_inner().unwrap().into_inner()[..], &[0, 1, 8, 9, 4, 5, 6, 7]); +} + +#[test] +fn test_read_until() { + let inner: &[u8] = &[0, 1, 2, 1, 0]; + let mut reader = BufReader::with_capacity(2, inner); + let mut v = Vec::new(); + reader.read_until(0, &mut v).unwrap(); + assert_eq!(v, [0]); + v.truncate(0); + reader.read_until(2, &mut v).unwrap(); + assert_eq!(v, [1, 2]); + v.truncate(0); + reader.read_until(1, &mut v).unwrap(); + assert_eq!(v, [1]); + v.truncate(0); + reader.read_until(8, &mut v).unwrap(); + assert_eq!(v, [0]); + v.truncate(0); + reader.read_until(9, &mut v).unwrap(); + assert_eq!(v, []); +} + +#[test] +fn test_line_buffer() { + let mut writer = LineWriter::new(Vec::new()); + writer.write(&[0]).unwrap(); + assert_eq!(*writer.get_ref(), []); + writer.write(&[1]).unwrap(); + assert_eq!(*writer.get_ref(), []); + writer.flush().unwrap(); + assert_eq!(*writer.get_ref(), [0, 1]); + writer.write(&[0, b'\n', 1, b'\n', 2]).unwrap(); + assert_eq!(*writer.get_ref(), [0, 1, 0, b'\n', 1, b'\n']); + writer.flush().unwrap(); + assert_eq!(*writer.get_ref(), [0, 1, 0, b'\n', 1, b'\n', 2]); + writer.write(&[3, b'\n']).unwrap(); + assert_eq!(*writer.get_ref(), [0, 1, 0, b'\n', 1, b'\n', 2, 3, b'\n']); +} + +#[test] +fn test_read_line() { + let in_buf: &[u8] = b"a\nb\nc"; + let mut reader = BufReader::with_capacity(2, in_buf); + let mut s = String::new(); + reader.read_line(&mut s).unwrap(); + assert_eq!(s, "a\n"); + s.truncate(0); + reader.read_line(&mut s).unwrap(); + assert_eq!(s, "b\n"); + s.truncate(0); + reader.read_line(&mut s).unwrap(); + assert_eq!(s, "c"); + s.truncate(0); + reader.read_line(&mut s).unwrap(); + assert_eq!(s, ""); +} + +#[test] +fn test_lines() { + let in_buf: &[u8] = b"a\nb\nc"; + let reader = BufReader::with_capacity(2, in_buf); + let mut it = reader.lines(); + assert_eq!(it.next().unwrap().unwrap(), "a".to_string()); + assert_eq!(it.next().unwrap().unwrap(), "b".to_string()); + assert_eq!(it.next().unwrap().unwrap(), "c".to_string()); + assert!(it.next().is_none()); +} + +#[test] +fn test_short_reads() { + let inner = ShortReader { lengths: vec![0, 1, 2, 0, 1, 0] }; + let mut reader = BufReader::new(inner); + let mut buf = [0, 0]; + assert_eq!(reader.read(&mut buf).unwrap(), 0); + assert_eq!(reader.read(&mut buf).unwrap(), 1); + assert_eq!(reader.read(&mut buf).unwrap(), 2); + assert_eq!(reader.read(&mut buf).unwrap(), 0); + assert_eq!(reader.read(&mut buf).unwrap(), 1); + assert_eq!(reader.read(&mut buf).unwrap(), 0); + assert_eq!(reader.read(&mut buf).unwrap(), 0); +} + +#[test] +#[should_panic] +fn dont_panic_in_drop_on_panicked_flush() { + struct FailFlushWriter; + + impl Write for FailFlushWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + Ok(buf.len()) + } + fn flush(&mut self) -> io::Result<()> { + Err(io::Error::last_os_error()) + } + } + + let writer = FailFlushWriter; + let _writer = BufWriter::new(writer); + + // If writer panics *again* due to the flush error then the process will + // abort. + panic!(); +} + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn panic_in_write_doesnt_flush_in_drop() { + static WRITES: AtomicUsize = AtomicUsize::new(0); + + struct PanicWriter; + + impl Write for PanicWriter { + fn write(&mut self, _: &[u8]) -> io::Result { + WRITES.fetch_add(1, Ordering::SeqCst); + panic!(); + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + thread::spawn(|| { + let mut writer = BufWriter::new(PanicWriter); + let _ = writer.write(b"hello world"); + let _ = writer.flush(); + }) + .join() + .unwrap_err(); + + assert_eq!(WRITES.load(Ordering::SeqCst), 1); +} + +#[bench] +fn bench_buffered_reader(b: &mut test::Bencher) { + b.iter(|| BufReader::new(io::empty())); +} + +#[bench] +fn bench_buffered_reader_small_reads(b: &mut test::Bencher) { + let data = (0..u8::MAX).cycle().take(1024 * 4).collect::>(); + b.iter(|| { + let mut reader = BufReader::new(&data[..]); + let mut buf = [0u8; 4]; + for _ in 0..1024 { + reader.read_exact(&mut buf).unwrap(); + core::hint::black_box(&buf); + } + }); +} + +#[bench] +fn bench_buffered_writer(b: &mut test::Bencher) { + b.iter(|| BufWriter::new(io::sink())); +} + +/// A simple `Write` target, designed to be wrapped by `LineWriter` / +/// `BufWriter` / etc, that can have its `write` & `flush` behavior +/// configured +#[derive(Default, Clone)] +struct ProgrammableSink { + // Writes append to this slice + pub buffer: Vec, + + // If true, writes will always be an error + pub always_write_error: bool, + + // If true, flushes will always be an error + pub always_flush_error: bool, + + // If set, only up to this number of bytes will be written in a single + // call to `write` + pub accept_prefix: Option, + + // If set, counts down with each write, and writes return an error + // when it hits 0 + pub max_writes: Option, + + // If set, attempting to write when max_writes == Some(0) will be an + // error; otherwise, it will return Ok(0). + pub error_after_max_writes: bool, +} + +impl Write for ProgrammableSink { + fn write(&mut self, data: &[u8]) -> io::Result { + if self.always_write_error { + return Err(io::Error::new(io::ErrorKind::Other, "test - always_write_error")); + } + + match self.max_writes { + Some(0) if self.error_after_max_writes => { + return Err(io::Error::new(io::ErrorKind::Other, "test - max_writes")); + } + Some(0) => return Ok(0), + Some(ref mut count) => *count -= 1, + None => {} + } + + let len = match self.accept_prefix { + None => data.len(), + Some(prefix) => data.len().min(prefix), + }; + + let data = &data[..len]; + self.buffer.extend_from_slice(data); + + Ok(len) + } + + fn flush(&mut self) -> io::Result<()> { + if self.always_flush_error { + Err(io::Error::new(io::ErrorKind::Other, "test - always_flush_error")) + } else { + Ok(()) + } + } +} + +/// Previously the `LineWriter` could successfully write some bytes but +/// then fail to report that it has done so. Additionally, an erroneous +/// flush after a successful write was permanently ignored. +/// +/// Test that a line writer correctly reports the number of written bytes, +/// and that it attempts to flush buffered lines from previous writes +/// before processing new data +/// +/// Regression test for #37807 +#[test] +fn erroneous_flush_retried() { + let writer = ProgrammableSink { + // Only write up to 4 bytes at a time + accept_prefix: Some(4), + + // Accept the first two writes, then error the others + max_writes: Some(2), + error_after_max_writes: true, + + ..Default::default() + }; + + // This should write the first 4 bytes. The rest will be buffered, out + // to the last newline. + let mut writer = LineWriter::new(writer); + assert_eq!(writer.write(b"a\nb\nc\nd\ne").unwrap(), 8); + + // This write should attempt to flush "c\nd\n", then buffer "e". No + // errors should happen here because no further writes should be + // attempted against `writer`. + assert_eq!(writer.write(b"e").unwrap(), 1); + assert_eq!(&writer.get_ref().buffer, b"a\nb\nc\nd\n"); +} + +#[test] +fn line_vectored() { + let mut a = LineWriter::new(Vec::new()); + assert_eq!( + a.write_vectored(&[ + IoSlice::new(&[]), + IoSlice::new(b"\n"), + IoSlice::new(&[]), + IoSlice::new(b"a"), + ]) + .unwrap(), + 2, + ); + assert_eq!(a.get_ref(), b"\n"); + + assert_eq!( + a.write_vectored(&[ + IoSlice::new(&[]), + IoSlice::new(b"b"), + IoSlice::new(&[]), + IoSlice::new(b"a"), + IoSlice::new(&[]), + IoSlice::new(b"c"), + ]) + .unwrap(), + 3, + ); + assert_eq!(a.get_ref(), b"\n"); + a.flush().unwrap(); + assert_eq!(a.get_ref(), b"\nabac"); + assert_eq!(a.write_vectored(&[]).unwrap(), 0); + assert_eq!( + a.write_vectored(&[ + IoSlice::new(&[]), + IoSlice::new(&[]), + IoSlice::new(&[]), + IoSlice::new(&[]), + ]) + .unwrap(), + 0, + ); + assert_eq!(a.write_vectored(&[IoSlice::new(b"a\nb"),]).unwrap(), 3); + assert_eq!(a.get_ref(), b"\nabaca\nb"); +} + +#[test] +fn line_vectored_partial_and_errors() { + use crate::collections::VecDeque; + + enum Call { + Write { inputs: Vec<&'static [u8]>, output: io::Result }, + Flush { output: io::Result<()> }, + } + + #[derive(Default)] + struct Writer { + calls: VecDeque, + } + + impl Write for Writer { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.write_vectored(&[IoSlice::new(buf)]) + } + + fn write_vectored(&mut self, buf: &[IoSlice<'_>]) -> io::Result { + match self.calls.pop_front().expect("unexpected call to write") { + Call::Write { inputs, output } => { + assert_eq!(inputs, buf.iter().map(|b| &**b).collect::>()); + output + } + Call::Flush { .. } => panic!("unexpected call to write; expected a flush"), + } + } + + fn is_write_vectored(&self) -> bool { + true + } + + fn flush(&mut self) -> io::Result<()> { + match self.calls.pop_front().expect("Unexpected call to flush") { + Call::Flush { output } => output, + Call::Write { .. } => panic!("unexpected call to flush; expected a write"), + } + } + } + + impl Drop for Writer { + fn drop(&mut self) { + if !thread::panicking() { + assert_eq!(self.calls.len(), 0); + } + } + } + + // partial writes keep going + let mut a = LineWriter::new(Writer::default()); + a.write_vectored(&[IoSlice::new(&[]), IoSlice::new(b"abc")]).unwrap(); + + a.get_mut().calls.push_back(Call::Write { inputs: vec![b"abc"], output: Ok(1) }); + a.get_mut().calls.push_back(Call::Write { inputs: vec![b"bc"], output: Ok(2) }); + a.get_mut().calls.push_back(Call::Write { inputs: vec![b"x", b"\n"], output: Ok(2) }); + + a.write_vectored(&[IoSlice::new(b"x"), IoSlice::new(b"\n")]).unwrap(); + + a.get_mut().calls.push_back(Call::Flush { output: Ok(()) }); + a.flush().unwrap(); + + // erroneous writes stop and don't write more + a.get_mut().calls.push_back(Call::Write { inputs: vec![b"x", b"\na"], output: Err(err()) }); + a.get_mut().calls.push_back(Call::Flush { output: Ok(()) }); + assert!(a.write_vectored(&[IoSlice::new(b"x"), IoSlice::new(b"\na")]).is_err()); + a.flush().unwrap(); + + fn err() -> io::Error { + io::Error::new(io::ErrorKind::Other, "x") + } +} + +/// Test that, in cases where vectored writing is not enabled, the +/// LineWriter uses the normal `write` call, which more-correctly handles +/// partial lines +#[test] +fn line_vectored_ignored() { + let writer = ProgrammableSink::default(); + let mut writer = LineWriter::new(writer); + + let content = [ + IoSlice::new(&[]), + IoSlice::new(b"Line 1\nLine"), + IoSlice::new(b" 2\nLine 3\nL"), + IoSlice::new(&[]), + IoSlice::new(&[]), + IoSlice::new(b"ine 4"), + IoSlice::new(b"\nLine 5\n"), + ]; + + let count = writer.write_vectored(&content).unwrap(); + assert_eq!(count, 11); + assert_eq!(&writer.get_ref().buffer, b"Line 1\n"); + + let count = writer.write_vectored(&content[2..]).unwrap(); + assert_eq!(count, 11); + assert_eq!(&writer.get_ref().buffer, b"Line 1\nLine 2\nLine 3\n"); + + let count = writer.write_vectored(&content[5..]).unwrap(); + assert_eq!(count, 5); + assert_eq!(&writer.get_ref().buffer, b"Line 1\nLine 2\nLine 3\n"); + + let count = writer.write_vectored(&content[6..]).unwrap(); + assert_eq!(count, 8); + assert_eq!( + writer.get_ref().buffer.as_slice(), + b"Line 1\nLine 2\nLine 3\nLine 4\nLine 5\n".as_ref() + ); +} + +/// Test that, given this input: +/// +/// Line 1\n +/// Line 2\n +/// Line 3\n +/// Line 4 +/// +/// And given a result that only writes to midway through Line 2 +/// +/// That only up to the end of Line 3 is buffered +/// +/// This behavior is desirable because it prevents flushing partial lines +#[test] +fn partial_write_buffers_line() { + let writer = ProgrammableSink { accept_prefix: Some(13), ..Default::default() }; + let mut writer = LineWriter::new(writer); + + assert_eq!(writer.write(b"Line 1\nLine 2\nLine 3\nLine4").unwrap(), 21); + assert_eq!(&writer.get_ref().buffer, b"Line 1\nLine 2"); + + assert_eq!(writer.write(b"Line 4").unwrap(), 6); + assert_eq!(&writer.get_ref().buffer, b"Line 1\nLine 2\nLine 3\n"); +} + +/// Test that, given this input: +/// +/// Line 1\n +/// Line 2\n +/// Line 3 +/// +/// And given that the full write of lines 1 and 2 was successful +/// That data up to Line 3 is buffered +#[test] +fn partial_line_buffered_after_line_write() { + let writer = ProgrammableSink::default(); + let mut writer = LineWriter::new(writer); + + assert_eq!(writer.write(b"Line 1\nLine 2\nLine 3").unwrap(), 20); + assert_eq!(&writer.get_ref().buffer, b"Line 1\nLine 2\n"); + + assert!(writer.flush().is_ok()); + assert_eq!(&writer.get_ref().buffer, b"Line 1\nLine 2\nLine 3"); +} + +/// Test that, given a partial line that exceeds the length of +/// LineBuffer's buffer (that is, without a trailing newline), that that +/// line is written to the inner writer +#[test] +fn long_line_flushed() { + let writer = ProgrammableSink::default(); + let mut writer = LineWriter::with_capacity(5, writer); + + assert_eq!(writer.write(b"0123456789").unwrap(), 10); + assert_eq!(&writer.get_ref().buffer, b"0123456789"); +} + +/// Test that, given a very long partial line *after* successfully +/// flushing a complete line, that that line is buffered unconditionally, +/// and no additional writes take place. This assures the property that +/// `write` should make at-most-one attempt to write new data. +#[test] +fn line_long_tail_not_flushed() { + let writer = ProgrammableSink::default(); + let mut writer = LineWriter::with_capacity(5, writer); + + // Assert that Line 1\n is flushed, and 01234 is buffered + assert_eq!(writer.write(b"Line 1\n0123456789").unwrap(), 12); + assert_eq!(&writer.get_ref().buffer, b"Line 1\n"); + + // Because the buffer is full, this subsequent write will flush it + assert_eq!(writer.write(b"5").unwrap(), 1); + assert_eq!(&writer.get_ref().buffer, b"Line 1\n01234"); +} + +/// Test that, if an attempt to pre-flush buffered data returns Ok(0), +/// this is propagated as an error. +#[test] +fn line_buffer_write0_error() { + let writer = ProgrammableSink { + // Accept one write, then return Ok(0) on subsequent ones + max_writes: Some(1), + + ..Default::default() + }; + let mut writer = LineWriter::new(writer); + + // This should write "Line 1\n" and buffer "Partial" + assert_eq!(writer.write(b"Line 1\nPartial").unwrap(), 14); + assert_eq!(&writer.get_ref().buffer, b"Line 1\n"); + + // This will attempt to flush "partial", which will return Ok(0), which + // needs to be an error, because we've already informed the client + // that we accepted the write. + let err = writer.write(b" Line End\n").unwrap_err(); + assert_eq!(err.kind(), ErrorKind::WriteZero); + assert_eq!(&writer.get_ref().buffer, b"Line 1\n"); +} + +/// Test that, if a write returns Ok(0) after a successful pre-flush, this +/// is propagated as Ok(0) +#[test] +fn line_buffer_write0_normal() { + let writer = ProgrammableSink { + // Accept two writes, then return Ok(0) on subsequent ones + max_writes: Some(2), + + ..Default::default() + }; + let mut writer = LineWriter::new(writer); + + // This should write "Line 1\n" and buffer "Partial" + assert_eq!(writer.write(b"Line 1\nPartial").unwrap(), 14); + assert_eq!(&writer.get_ref().buffer, b"Line 1\n"); + + // This will flush partial, which will succeed, but then return Ok(0) + // when flushing " Line End\n" + assert_eq!(writer.write(b" Line End\n").unwrap(), 0); + assert_eq!(&writer.get_ref().buffer, b"Line 1\nPartial"); +} + +/// LineWriter has a custom `write_all`; make sure it works correctly +#[test] +fn line_write_all() { + let writer = ProgrammableSink { + // Only write 5 bytes at a time + accept_prefix: Some(5), + ..Default::default() + }; + let mut writer = LineWriter::new(writer); + + writer.write_all(b"Line 1\nLine 2\nLine 3\nLine 4\nPartial").unwrap(); + assert_eq!(&writer.get_ref().buffer, b"Line 1\nLine 2\nLine 3\nLine 4\n"); + writer.write_all(b" Line 5\n").unwrap(); + assert_eq!( + writer.get_ref().buffer.as_slice(), + b"Line 1\nLine 2\nLine 3\nLine 4\nPartial Line 5\n".as_ref(), + ); +} + +#[test] +fn line_write_all_error() { + let writer = ProgrammableSink { + // Only accept up to 3 writes of up to 5 bytes each + accept_prefix: Some(5), + max_writes: Some(3), + ..Default::default() + }; + + let mut writer = LineWriter::new(writer); + let res = writer.write_all(b"Line 1\nLine 2\nLine 3\nLine 4\nPartial"); + assert!(res.is_err()); + // An error from write_all leaves everything in an indeterminate state, + // so there's nothing else to test here +} + +/// Under certain circumstances, the old implementation of LineWriter +/// would try to buffer "to the last newline" but be forced to buffer +/// less than that, leading to inappropriate partial line writes. +/// Regression test for that issue. +#[test] +fn partial_multiline_buffering() { + let writer = ProgrammableSink { + // Write only up to 5 bytes at a time + accept_prefix: Some(5), + ..Default::default() + }; + + let mut writer = LineWriter::with_capacity(10, writer); + + let content = b"AAAAABBBBB\nCCCCDDDDDD\nEEE"; + + // When content is written, LineWriter will try to write blocks A, B, + // C, and D. Only block A will succeed. Under the old behavior, LineWriter + // would then try to buffer B, C and D, but because its capacity is 10, + // it will only be able to buffer B and C. We don't want to buffer + // partial lines concurrent with whole lines, so the correct behavior + // is to buffer only block B (out to the newline) + assert_eq!(writer.write(content).unwrap(), 11); + assert_eq!(writer.get_ref().buffer, *b"AAAAA"); + + writer.flush().unwrap(); + assert_eq!(writer.get_ref().buffer, *b"AAAAABBBBB\n"); +} + +/// Same as test_partial_multiline_buffering, but in the event NO full lines +/// fit in the buffer, just buffer as much as possible +#[test] +fn partial_multiline_buffering_without_full_line() { + let writer = ProgrammableSink { + // Write only up to 5 bytes at a time + accept_prefix: Some(5), + ..Default::default() + }; + + let mut writer = LineWriter::with_capacity(5, writer); + + let content = b"AAAAABBBBBBBBBB\nCCCCC\nDDDDD"; + + // When content is written, LineWriter will try to write blocks A, B, + // and C. Only block A will succeed. Under the old behavior, LineWriter + // would then try to buffer B and C, but because its capacity is 5, + // it will only be able to buffer part of B. Because it's not possible + // for it to buffer any complete lines, it should buffer as much of B as + // possible + assert_eq!(writer.write(content).unwrap(), 10); + assert_eq!(writer.get_ref().buffer, *b"AAAAA"); + + writer.flush().unwrap(); + assert_eq!(writer.get_ref().buffer, *b"AAAAABBBBB"); +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum RecordedEvent { + Write(String), + Flush, +} + +#[derive(Debug, Clone, Default)] +struct WriteRecorder { + pub events: Vec, +} + +impl Write for WriteRecorder { + fn write(&mut self, buf: &[u8]) -> io::Result { + use crate::str::from_utf8; + + self.events.push(RecordedEvent::Write(from_utf8(buf).unwrap().to_string())); + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.events.push(RecordedEvent::Flush); + Ok(()) + } +} + +/// Test that a normal, formatted writeln only results in a single write +/// call to the underlying writer. A naive implementation of +/// LineWriter::write_all results in two writes: one of the buffered data, +/// and another of the final substring in the formatted set +#[test] +fn single_formatted_write() { + let writer = WriteRecorder::default(); + let mut writer = LineWriter::new(writer); + + // Under a naive implementation of LineWriter, this will result in two + // writes: "hello, world" and "!\n", because write() has to flush the + // buffer before attempting to write the last "!\n". write_all shouldn't + // have this limitation. + writeln!(&mut writer, "{}, {}!", "hello", "world").unwrap(); + assert_eq!(writer.get_ref().events, [RecordedEvent::Write("hello, world!\n".to_string())]); +} + +#[test] +fn bufreader_full_initialize() { + struct OneByteReader; + impl Read for OneByteReader { + fn read(&mut self, buf: &mut [u8]) -> crate::io::Result { + if buf.len() > 0 { + buf[0] = 0; + Ok(1) + } else { + Ok(0) + } + } + } + let mut reader = BufReader::new(OneByteReader); + // Nothing is initialized yet. + assert_eq!(reader.initialized(), 0); + + let buf = reader.fill_buf().unwrap(); + // We read one byte... + assert_eq!(buf.len(), 1); + // But we initialized the whole buffer! + assert_eq!(reader.initialized(), reader.capacity()); +} diff --git a/crux-mir/lib/std/src/io/copy.rs b/crux-mir/lib/std/src/io/copy.rs new file mode 100644 index 000000000..38b98afff --- /dev/null +++ b/crux-mir/lib/std/src/io/copy.rs @@ -0,0 +1,163 @@ +use super::{BorrowedBuf, BufWriter, ErrorKind, Read, Result, Write, DEFAULT_BUF_SIZE}; +use crate::mem::MaybeUninit; + +/// Copies the entire contents of a reader into a writer. +/// +/// This function will continuously read data from `reader` and then +/// write it into `writer` in a streaming fashion until `reader` +/// returns EOF. +/// +/// On success, the total number of bytes that were copied from +/// `reader` to `writer` is returned. +/// +/// If you’re wanting to copy the contents of one file to another and you’re +/// working with filesystem paths, see the [`fs::copy`] function. +/// +/// [`fs::copy`]: crate::fs::copy +/// +/// # Errors +/// +/// This function will return an error immediately if any call to [`read`] or +/// [`write`] returns an error. All instances of [`ErrorKind::Interrupted`] are +/// handled by this function and the underlying operation is retried. +/// +/// [`read`]: Read::read +/// [`write`]: Write::write +/// +/// # Examples +/// +/// ``` +/// use std::io; +/// +/// fn main() -> io::Result<()> { +/// let mut reader: &[u8] = b"hello"; +/// let mut writer: Vec = vec![]; +/// +/// io::copy(&mut reader, &mut writer)?; +/// +/// assert_eq!(&b"hello"[..], &writer[..]); +/// Ok(()) +/// } +/// ``` +/// +/// # Platform-specific behavior +/// +/// On Linux (including Android), this function uses `copy_file_range(2)`, +/// `sendfile(2)` or `splice(2)` syscalls to move data directly between file +/// descriptors if possible. +/// +/// Note that platform-specific behavior [may change in the future][changes]. +/// +/// [changes]: crate::io#platform-specific-behavior +#[stable(feature = "rust1", since = "1.0.0")] +pub fn copy(reader: &mut R, writer: &mut W) -> Result +where + R: Read, + W: Write, +{ + cfg_if::cfg_if! { + if #[cfg(any(target_os = "linux", target_os = "android"))] { + crate::sys::kernel_copy::copy_spec(reader, writer) + } else { + generic_copy(reader, writer) + } + } +} + +/// The userspace read-write-loop implementation of `io::copy` that is used when +/// OS-specific specializations for copy offloading are not available or not applicable. +pub(crate) fn generic_copy(reader: &mut R, writer: &mut W) -> Result +where + R: Read, + W: Write, +{ + BufferedCopySpec::copy_to(reader, writer) +} + +/// Specialization of the read-write loop that either uses a stack buffer +/// or reuses the internal buffer of a BufWriter +trait BufferedCopySpec: Write { + fn copy_to(reader: &mut R, writer: &mut Self) -> Result; +} + +impl BufferedCopySpec for W { + default fn copy_to(reader: &mut R, writer: &mut Self) -> Result { + stack_buffer_copy(reader, writer) + } +} + +impl BufferedCopySpec for BufWriter { + fn copy_to(reader: &mut R, writer: &mut Self) -> Result { + if writer.capacity() < DEFAULT_BUF_SIZE { + return stack_buffer_copy(reader, writer); + } + + let mut len = 0; + let mut init = 0; + + loop { + let buf = writer.buffer_mut(); + let mut read_buf: BorrowedBuf<'_> = buf.spare_capacity_mut().into(); + + unsafe { + // SAFETY: init is either 0 or the init_len from the previous iteration. + read_buf.set_init(init); + } + + if read_buf.capacity() >= DEFAULT_BUF_SIZE { + let mut cursor = read_buf.unfilled(); + match reader.read_buf(cursor.reborrow()) { + Ok(()) => { + let bytes_read = cursor.written(); + + if bytes_read == 0 { + return Ok(len); + } + + init = read_buf.init_len() - bytes_read; + len += bytes_read as u64; + + // SAFETY: BorrowedBuf guarantees all of its filled bytes are init + unsafe { buf.set_len(buf.len() + bytes_read) }; + + // Read again if the buffer still has enough capacity, as BufWriter itself would do + // This will occur if the reader returns short reads + } + Err(ref e) if e.kind() == ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } else { + writer.flush_buf()?; + init = 0; + } + } + } +} + +fn stack_buffer_copy( + reader: &mut R, + writer: &mut W, +) -> Result { + let buf: &mut [_] = &mut [MaybeUninit::uninit(); DEFAULT_BUF_SIZE]; + let mut buf: BorrowedBuf<'_> = buf.into(); + + let mut len = 0; + + loop { + match reader.read_buf(buf.unfilled()) { + Ok(()) => {} + Err(e) if e.kind() == ErrorKind::Interrupted => continue, + Err(e) => return Err(e), + }; + + if buf.filled().is_empty() { + break; + } + + len += buf.filled().len() as u64; + writer.write_all(buf.filled())?; + buf.clear(); + } + + Ok(len) +} diff --git a/crux-mir/lib/std/src/io/cursor.rs b/crux-mir/lib/std/src/io/cursor.rs index f36aa1846..d98ab021c 100644 --- a/crux-mir/lib/std/src/io/cursor.rs +++ b/crux-mir/lib/std/src/io/cursor.rs @@ -1,21 +1,23 @@ +#[cfg(test)] +mod tests; + use crate::io::prelude::*; +use crate::alloc::Allocator; use crate::cmp; -use crate::io::{self, Error, ErrorKind, Initializer, IoSlice, IoSliceMut, SeekFrom}; - -use core::convert::TryInto; +use crate::io::{self, BorrowedCursor, ErrorKind, IoSlice, IoSliceMut, SeekFrom}; /// A `Cursor` wraps an in-memory buffer and provides it with a /// [`Seek`] implementation. /// /// `Cursor`s are used with in-memory buffers, anything implementing -/// `AsRef<[u8]>`, to allow them to implement [`Read`] and/or [`Write`], +/// [AsRef]<\[u8]>, to allow them to implement [`Read`] and/or [`Write`], /// allowing these buffers to be used anywhere you might use a reader or writer /// that does actual I/O. /// /// The standard library implements some I/O traits on various types which -/// are commonly used as a buffer, like `Cursor<`[`Vec`]`>` and -/// `Cursor<`[`&[u8]`][bytes]`>`. +/// are commonly used as a buffer, like Cursor<[Vec]\> and +/// Cursor<[&\[u8\]][bytes]>. /// /// # Examples /// @@ -23,12 +25,8 @@ use core::convert::TryInto; /// code, but use an in-memory buffer in our tests. We can do this with /// `Cursor`: /// -/// [`Seek`]: trait.Seek.html -/// [`Read`]: ../../std/io/trait.Read.html -/// [`Write`]: ../../std/io/trait.Write.html -/// [`Vec`]: ../../std/vec/struct.Vec.html -/// [bytes]: ../../std/primitive.slice.html -/// [`File`]: ../fs/struct.File.html +/// [bytes]: crate::slice "slice" +/// [`File`]: crate::fs::File /// /// ```no_run /// use std::io::prelude::*; @@ -72,7 +70,7 @@ use core::convert::TryInto; /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] -#[derive(Clone, Debug, Default, Eq, PartialEq)] +#[derive(Debug, Default, Eq, PartialEq)] pub struct Cursor { inner: T, pos: u64, @@ -81,8 +79,8 @@ pub struct Cursor { impl Cursor { /// Creates a new cursor wrapping the provided underlying in-memory buffer. /// - /// Cursor initial position is `0` even if underlying buffer (e.g., `Vec`) - /// is not empty. So writing to cursor starts with overwriting `Vec` + /// Cursor initial position is `0` even if underlying buffer (e.g., [`Vec`]) + /// is not empty. So writing to cursor starts with overwriting [`Vec`] /// content, not with appending to it. /// /// # Examples @@ -95,7 +93,8 @@ impl Cursor { /// # force_inference(&buff); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn new(inner: T) -> Cursor { + #[rustc_const_unstable(feature = "const_io_structs", issue = "78812")] + pub const fn new(inner: T) -> Cursor { Cursor { pos: 0, inner } } @@ -131,7 +130,8 @@ impl Cursor { /// let reference = buff.get_ref(); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn get_ref(&self) -> &T { + #[rustc_const_unstable(feature = "const_io_structs", issue = "78812")] + pub const fn get_ref(&self) -> &T { &self.inner } @@ -176,7 +176,8 @@ impl Cursor { /// assert_eq!(buff.position(), 1); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn position(&self) -> u64 { + #[rustc_const_unstable(feature = "const_io_structs", issue = "78812")] + pub const fn position(&self) -> u64 { self.pos } @@ -203,6 +204,79 @@ impl Cursor { } } +impl Cursor +where + T: AsRef<[u8]>, +{ + /// Returns the remaining slice. + /// + /// # Examples + /// + /// ``` + /// #![feature(cursor_remaining)] + /// use std::io::Cursor; + /// + /// let mut buff = Cursor::new(vec![1, 2, 3, 4, 5]); + /// + /// assert_eq!(buff.remaining_slice(), &[1, 2, 3, 4, 5]); + /// + /// buff.set_position(2); + /// assert_eq!(buff.remaining_slice(), &[3, 4, 5]); + /// + /// buff.set_position(4); + /// assert_eq!(buff.remaining_slice(), &[5]); + /// + /// buff.set_position(6); + /// assert_eq!(buff.remaining_slice(), &[]); + /// ``` + #[unstable(feature = "cursor_remaining", issue = "86369")] + pub fn remaining_slice(&self) -> &[u8] { + let len = self.pos.min(self.inner.as_ref().len() as u64); + &self.inner.as_ref()[(len as usize)..] + } + + /// Returns `true` if the remaining slice is empty. + /// + /// # Examples + /// + /// ``` + /// #![feature(cursor_remaining)] + /// use std::io::Cursor; + /// + /// let mut buff = Cursor::new(vec![1, 2, 3, 4, 5]); + /// + /// buff.set_position(2); + /// assert!(!buff.is_empty()); + /// + /// buff.set_position(5); + /// assert!(buff.is_empty()); + /// + /// buff.set_position(10); + /// assert!(buff.is_empty()); + /// ``` + #[unstable(feature = "cursor_remaining", issue = "86369")] + pub fn is_empty(&self) -> bool { + self.pos >= self.inner.as_ref().len() as u64 + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Clone for Cursor +where + T: Clone, +{ + #[inline] + fn clone(&self) -> Self { + Cursor { inner: self.inner.clone(), pos: self.pos } + } + + #[inline] + fn clone_from(&mut self, other: &Self) { + self.inner.clone_from(&other.inner); + self.pos = other.pos; + } +} + #[stable(feature = "rust1", since = "1.0.0")] impl io::Seek for Cursor where @@ -217,17 +291,12 @@ where SeekFrom::End(n) => (self.inner.as_ref().len() as u64, n), SeekFrom::Current(n) => (self.pos, n), }; - let new_pos = if offset >= 0 { - base_pos.checked_add(offset as u64) - } else { - base_pos.checked_sub((offset.wrapping_neg()) as u64) - }; - match new_pos { + match base_pos.checked_add_signed(offset) { Some(n) => { self.pos = n; Ok(self.pos) } - None => Err(Error::new( + None => Err(io::const_io_error!( ErrorKind::InvalidInput, "invalid seek to a negative or overflowing position", )), @@ -249,11 +318,21 @@ where T: AsRef<[u8]>, { fn read(&mut self, buf: &mut [u8]) -> io::Result { - let n = Read::read(&mut self.fill_buf()?, buf)?; + let n = Read::read(&mut self.remaining_slice(), buf)?; self.pos += n as u64; Ok(n) } + fn read_buf(&mut self, mut cursor: BorrowedCursor<'_>) -> io::Result<()> { + let prev_written = cursor.written(); + + Read::read_buf(&mut self.fill_buf()?, cursor.reborrow())?; + + self.pos += (cursor.written() - prev_written) as u64; + + Ok(()) + } + fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { let mut nread = 0; for buf in bufs { @@ -266,17 +345,16 @@ where Ok(nread) } + fn is_read_vectored(&self) -> bool { + true + } + fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> { let n = buf.len(); - Read::read_exact(&mut self.fill_buf()?, buf)?; + Read::read_exact(&mut self.remaining_slice(), buf)?; self.pos += n as u64; Ok(()) } - - #[inline] - unsafe fn initializer(&self) -> Initializer { - Initializer::nop() - } } #[stable(feature = "rust1", since = "1.0.0")] @@ -285,8 +363,7 @@ where T: AsRef<[u8]>, { fn fill_buf(&mut self) -> io::Result<&[u8]> { - let amt = cmp::min(self.pos, self.inner.as_ref().len() as u64); - Ok(&self.inner.as_ref()[(amt as usize)..]) + Ok(self.remaining_slice()) } fn consume(&mut self, amt: usize) { self.pos += amt as u64; @@ -319,45 +396,127 @@ fn slice_write_vectored( Ok(nwritten) } -// Resizing write implementation -fn vec_write(pos_mut: &mut u64, vec: &mut Vec, buf: &[u8]) -> io::Result { +/// Reserves the required space, and pads the vec with 0s if necessary. +fn reserve_and_pad( + pos_mut: &mut u64, + vec: &mut Vec, + buf_len: usize, +) -> io::Result { let pos: usize = (*pos_mut).try_into().map_err(|_| { - Error::new( + io::const_io_error!( ErrorKind::InvalidInput, "cursor position exceeds maximum possible vector length", ) })?; - // Make sure the internal buffer is as least as big as where we - // currently are - let len = vec.len(); - if len < pos { - // use `resize` so that the zero filling is as efficient as possible - vec.resize(pos, 0); - } - // Figure out what bytes will be used to overwrite what's currently - // there (left), and what will be appended on the end (right) - { - let space = vec.len() - pos; - let (left, right) = buf.split_at(cmp::min(space, buf.len())); - vec[pos..pos + left.len()].copy_from_slice(left); - vec.extend_from_slice(right); + + // For safety reasons, we don't want these numbers to overflow + // otherwise our allocation won't be enough + let desired_cap = pos.saturating_add(buf_len); + if desired_cap > vec.capacity() { + // We want our vec's total capacity + // to have room for (pos+buf_len) bytes. Reserve allocates + // based on additional elements from the length, so we need to + // reserve the difference + vec.reserve(desired_cap - vec.len()); + } + // Pad if pos is above the current len. + if pos > vec.len() { + let diff = pos - vec.len(); + // Unfortunately, `resize()` would suffice but the optimiser does not + // realise the `reserve` it does can be eliminated. So we do it manually + // to eliminate that extra branch + let spare = vec.spare_capacity_mut(); + debug_assert!(spare.len() >= diff); + // Safety: we have allocated enough capacity for this. + // And we are only writing, not reading + unsafe { + spare.get_unchecked_mut(..diff).fill(core::mem::MaybeUninit::new(0)); + vec.set_len(pos); + } } + Ok(pos) +} + +/// Writes the slice to the vec without allocating +/// # Safety: vec must have buf.len() spare capacity +unsafe fn vec_write_unchecked(pos: usize, vec: &mut Vec, buf: &[u8]) -> usize +where + A: Allocator, +{ + debug_assert!(vec.capacity() >= pos + buf.len()); + vec.as_mut_ptr().add(pos).copy_from(buf.as_ptr(), buf.len()); + pos + buf.len() +} + +/// Resizing write implementation for [`Cursor`] +/// +/// Cursor is allowed to have a pre-allocated and initialised +/// vector body, but with a position of 0. This means the [`Write`] +/// will overwrite the contents of the vec. +/// +/// This also allows for the vec body to be empty, but with a position of N. +/// This means that [`Write`] will pad the vec with 0 initially, +/// before writing anything from that point +fn vec_write(pos_mut: &mut u64, vec: &mut Vec, buf: &[u8]) -> io::Result +where + A: Allocator, +{ + let buf_len = buf.len(); + let mut pos = reserve_and_pad(pos_mut, vec, buf_len)?; + + // Write the buf then progress the vec forward if necessary + // Safety: we have ensured that the capacity is available + // and that all bytes get written up to pos + unsafe { + pos = vec_write_unchecked(pos, vec, buf); + if pos > vec.len() { + vec.set_len(pos); + } + }; + // Bump us forward - *pos_mut = (pos + buf.len()) as u64; - Ok(buf.len()) + *pos_mut += buf_len as u64; + Ok(buf_len) } -fn vec_write_vectored( +/// Resizing write_vectored implementation for [`Cursor`] +/// +/// Cursor is allowed to have a pre-allocated and initialised +/// vector body, but with a position of 0. This means the [`Write`] +/// will overwrite the contents of the vec. +/// +/// This also allows for the vec body to be empty, but with a position of N. +/// This means that [`Write`] will pad the vec with 0 initially, +/// before writing anything from that point +fn vec_write_vectored( pos_mut: &mut u64, - vec: &mut Vec, + vec: &mut Vec, bufs: &[IoSlice<'_>], -) -> io::Result { - let mut nwritten = 0; - for buf in bufs { - nwritten += vec_write(pos_mut, vec, buf)?; +) -> io::Result +where + A: Allocator, +{ + // For safety reasons, we don't want this sum to overflow ever. + // If this saturates, the reserve should panic to avoid any unsound writing. + let buf_len = bufs.iter().fold(0usize, |a, b| a.saturating_add(b.len())); + let mut pos = reserve_and_pad(pos_mut, vec, buf_len)?; + + // Write the buf then progress the vec forward if necessary + // Safety: we have ensured that the capacity is available + // and that all bytes get written up to the last pos + unsafe { + for buf in bufs { + pos = vec_write_unchecked(pos, vec, buf); + } + if pos > vec.len() { + vec.set_len(pos); + } } - Ok(nwritten) + + // Bump us forward + *pos_mut += buf_len as u64; + Ok(buf_len) } #[stable(feature = "rust1", since = "1.0.0")] @@ -372,6 +531,11 @@ impl Write for Cursor<&mut [u8]> { slice_write_vectored(&mut self.pos, self.inner, bufs) } + #[inline] + fn is_write_vectored(&self) -> bool { + true + } + #[inline] fn flush(&mut self) -> io::Result<()> { Ok(()) @@ -379,7 +543,10 @@ impl Write for Cursor<&mut [u8]> { } #[stable(feature = "cursor_mut_vec", since = "1.25.0")] -impl Write for Cursor<&mut Vec> { +impl Write for Cursor<&mut Vec> +where + A: Allocator, +{ fn write(&mut self, buf: &[u8]) -> io::Result { vec_write(&mut self.pos, self.inner, buf) } @@ -388,6 +555,11 @@ impl Write for Cursor<&mut Vec> { vec_write_vectored(&mut self.pos, self.inner, bufs) } + #[inline] + fn is_write_vectored(&self) -> bool { + true + } + #[inline] fn flush(&mut self) -> io::Result<()> { Ok(()) @@ -395,7 +567,10 @@ impl Write for Cursor<&mut Vec> { } #[stable(feature = "rust1", since = "1.0.0")] -impl Write for Cursor> { +impl Write for Cursor> +where + A: Allocator, +{ fn write(&mut self, buf: &[u8]) -> io::Result { vec_write(&mut self.pos, &mut self.inner, buf) } @@ -404,6 +579,11 @@ impl Write for Cursor> { vec_write_vectored(&mut self.pos, &mut self.inner, bufs) } + #[inline] + fn is_write_vectored(&self) -> bool { + true + } + #[inline] fn flush(&mut self) -> io::Result<()> { Ok(()) @@ -411,7 +591,10 @@ impl Write for Cursor> { } #[stable(feature = "cursor_box_slice", since = "1.5.0")] -impl Write for Cursor> { +impl Write for Cursor> +where + A: Allocator, +{ #[inline] fn write(&mut self, buf: &[u8]) -> io::Result { slice_write(&mut self.pos, &mut self.inner, buf) @@ -422,536 +605,36 @@ impl Write for Cursor> { slice_write_vectored(&mut self.pos, &mut self.inner, bufs) } + #[inline] + fn is_write_vectored(&self) -> bool { + true + } + #[inline] fn flush(&mut self) -> io::Result<()> { Ok(()) } } -#[cfg(test)] -mod tests { - use crate::io::prelude::*; - use crate::io::{Cursor, IoSlice, IoSliceMut, SeekFrom}; - - #[test] - fn test_vec_writer() { - let mut writer = Vec::new(); - assert_eq!(writer.write(&[0]).unwrap(), 1); - assert_eq!(writer.write(&[1, 2, 3]).unwrap(), 3); - assert_eq!(writer.write(&[4, 5, 6, 7]).unwrap(), 4); - assert_eq!( - writer - .write_vectored(&[IoSlice::new(&[]), IoSlice::new(&[8, 9]), IoSlice::new(&[10])],) - .unwrap(), - 3 - ); - let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - assert_eq!(writer, b); - } - - #[test] - fn test_mem_writer() { - let mut writer = Cursor::new(Vec::new()); - assert_eq!(writer.write(&[0]).unwrap(), 1); - assert_eq!(writer.write(&[1, 2, 3]).unwrap(), 3); - assert_eq!(writer.write(&[4, 5, 6, 7]).unwrap(), 4); - assert_eq!( - writer - .write_vectored(&[IoSlice::new(&[]), IoSlice::new(&[8, 9]), IoSlice::new(&[10])],) - .unwrap(), - 3 - ); - let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - assert_eq!(&writer.get_ref()[..], b); - } - - #[test] - fn test_mem_mut_writer() { - let mut vec = Vec::new(); - let mut writer = Cursor::new(&mut vec); - assert_eq!(writer.write(&[0]).unwrap(), 1); - assert_eq!(writer.write(&[1, 2, 3]).unwrap(), 3); - assert_eq!(writer.write(&[4, 5, 6, 7]).unwrap(), 4); - assert_eq!( - writer - .write_vectored(&[IoSlice::new(&[]), IoSlice::new(&[8, 9]), IoSlice::new(&[10])],) - .unwrap(), - 3 - ); - let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - assert_eq!(&writer.get_ref()[..], b); - } - - #[test] - fn test_box_slice_writer() { - let mut writer = Cursor::new(vec![0u8; 9].into_boxed_slice()); - assert_eq!(writer.position(), 0); - assert_eq!(writer.write(&[0]).unwrap(), 1); - assert_eq!(writer.position(), 1); - assert_eq!(writer.write(&[1, 2, 3]).unwrap(), 3); - assert_eq!(writer.write(&[4, 5, 6, 7]).unwrap(), 4); - assert_eq!(writer.position(), 8); - assert_eq!(writer.write(&[]).unwrap(), 0); - assert_eq!(writer.position(), 8); - - assert_eq!(writer.write(&[8, 9]).unwrap(), 1); - assert_eq!(writer.write(&[10]).unwrap(), 0); - let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8]; - assert_eq!(&**writer.get_ref(), b); - } - - #[test] - fn test_box_slice_writer_vectored() { - let mut writer = Cursor::new(vec![0u8; 9].into_boxed_slice()); - assert_eq!(writer.position(), 0); - assert_eq!(writer.write_vectored(&[IoSlice::new(&[0])]).unwrap(), 1); - assert_eq!(writer.position(), 1); - assert_eq!( - writer - .write_vectored(&[IoSlice::new(&[1, 2, 3]), IoSlice::new(&[4, 5, 6, 7]),]) - .unwrap(), - 7, - ); - assert_eq!(writer.position(), 8); - assert_eq!(writer.write_vectored(&[]).unwrap(), 0); - assert_eq!(writer.position(), 8); - - assert_eq!(writer.write_vectored(&[IoSlice::new(&[8, 9])]).unwrap(), 1); - assert_eq!(writer.write_vectored(&[IoSlice::new(&[10])]).unwrap(), 0); - let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8]; - assert_eq!(&**writer.get_ref(), b); - } - - #[test] - fn test_buf_writer() { - let mut buf = [0 as u8; 9]; - { - let mut writer = Cursor::new(&mut buf[..]); - assert_eq!(writer.position(), 0); - assert_eq!(writer.write(&[0]).unwrap(), 1); - assert_eq!(writer.position(), 1); - assert_eq!(writer.write(&[1, 2, 3]).unwrap(), 3); - assert_eq!(writer.write(&[4, 5, 6, 7]).unwrap(), 4); - assert_eq!(writer.position(), 8); - assert_eq!(writer.write(&[]).unwrap(), 0); - assert_eq!(writer.position(), 8); - - assert_eq!(writer.write(&[8, 9]).unwrap(), 1); - assert_eq!(writer.write(&[10]).unwrap(), 0); - } - let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8]; - assert_eq!(buf, b); - } - - #[test] - fn test_buf_writer_vectored() { - let mut buf = [0 as u8; 9]; - { - let mut writer = Cursor::new(&mut buf[..]); - assert_eq!(writer.position(), 0); - assert_eq!(writer.write_vectored(&[IoSlice::new(&[0])]).unwrap(), 1); - assert_eq!(writer.position(), 1); - assert_eq!( - writer - .write_vectored(&[IoSlice::new(&[1, 2, 3]), IoSlice::new(&[4, 5, 6, 7])],) - .unwrap(), - 7, - ); - assert_eq!(writer.position(), 8); - assert_eq!(writer.write_vectored(&[]).unwrap(), 0); - assert_eq!(writer.position(), 8); - - assert_eq!(writer.write_vectored(&[IoSlice::new(&[8, 9])]).unwrap(), 1); - assert_eq!(writer.write_vectored(&[IoSlice::new(&[10])]).unwrap(), 0); - } - let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8]; - assert_eq!(buf, b); - } - - #[test] - fn test_buf_writer_seek() { - let mut buf = [0 as u8; 8]; - { - let mut writer = Cursor::new(&mut buf[..]); - assert_eq!(writer.position(), 0); - assert_eq!(writer.write(&[1]).unwrap(), 1); - assert_eq!(writer.position(), 1); - - assert_eq!(writer.seek(SeekFrom::Start(2)).unwrap(), 2); - assert_eq!(writer.position(), 2); - assert_eq!(writer.write(&[2]).unwrap(), 1); - assert_eq!(writer.position(), 3); - - assert_eq!(writer.seek(SeekFrom::Current(-2)).unwrap(), 1); - assert_eq!(writer.position(), 1); - assert_eq!(writer.write(&[3]).unwrap(), 1); - assert_eq!(writer.position(), 2); - - assert_eq!(writer.seek(SeekFrom::End(-1)).unwrap(), 7); - assert_eq!(writer.position(), 7); - assert_eq!(writer.write(&[4]).unwrap(), 1); - assert_eq!(writer.position(), 8); - } - let b: &[_] = &[1, 3, 2, 0, 0, 0, 0, 4]; - assert_eq!(buf, b); - } - - #[test] - fn test_buf_writer_error() { - let mut buf = [0 as u8; 2]; - let mut writer = Cursor::new(&mut buf[..]); - assert_eq!(writer.write(&[0]).unwrap(), 1); - assert_eq!(writer.write(&[0, 0]).unwrap(), 1); - assert_eq!(writer.write(&[0, 0]).unwrap(), 0); - } - - #[test] - fn test_mem_reader() { - let mut reader = Cursor::new(vec![0, 1, 2, 3, 4, 5, 6, 7]); - let mut buf = []; - assert_eq!(reader.read(&mut buf).unwrap(), 0); - assert_eq!(reader.position(), 0); - let mut buf = [0]; - assert_eq!(reader.read(&mut buf).unwrap(), 1); - assert_eq!(reader.position(), 1); - let b: &[_] = &[0]; - assert_eq!(buf, b); - let mut buf = [0; 4]; - assert_eq!(reader.read(&mut buf).unwrap(), 4); - assert_eq!(reader.position(), 5); - let b: &[_] = &[1, 2, 3, 4]; - assert_eq!(buf, b); - assert_eq!(reader.read(&mut buf).unwrap(), 3); - let b: &[_] = &[5, 6, 7]; - assert_eq!(&buf[..3], b); - assert_eq!(reader.read(&mut buf).unwrap(), 0); - } - - #[test] - fn test_mem_reader_vectored() { - let mut reader = Cursor::new(vec![0, 1, 2, 3, 4, 5, 6, 7]); - let mut buf = []; - assert_eq!(reader.read_vectored(&mut [IoSliceMut::new(&mut buf)]).unwrap(), 0); - assert_eq!(reader.position(), 0); - let mut buf = [0]; - assert_eq!( - reader - .read_vectored(&mut [IoSliceMut::new(&mut []), IoSliceMut::new(&mut buf),]) - .unwrap(), - 1, - ); - assert_eq!(reader.position(), 1); - let b: &[_] = &[0]; - assert_eq!(buf, b); - let mut buf1 = [0; 4]; - let mut buf2 = [0; 4]; - assert_eq!( - reader - .read_vectored(&mut [IoSliceMut::new(&mut buf1), IoSliceMut::new(&mut buf2),]) - .unwrap(), - 7, - ); - let b1: &[_] = &[1, 2, 3, 4]; - let b2: &[_] = &[5, 6, 7]; - assert_eq!(buf1, b1); - assert_eq!(&buf2[..3], b2); - assert_eq!(reader.read(&mut buf).unwrap(), 0); - } - - #[test] - fn test_boxed_slice_reader() { - let mut reader = Cursor::new(vec![0, 1, 2, 3, 4, 5, 6, 7].into_boxed_slice()); - let mut buf = []; - assert_eq!(reader.read(&mut buf).unwrap(), 0); - assert_eq!(reader.position(), 0); - let mut buf = [0]; - assert_eq!(reader.read(&mut buf).unwrap(), 1); - assert_eq!(reader.position(), 1); - let b: &[_] = &[0]; - assert_eq!(buf, b); - let mut buf = [0; 4]; - assert_eq!(reader.read(&mut buf).unwrap(), 4); - assert_eq!(reader.position(), 5); - let b: &[_] = &[1, 2, 3, 4]; - assert_eq!(buf, b); - assert_eq!(reader.read(&mut buf).unwrap(), 3); - let b: &[_] = &[5, 6, 7]; - assert_eq!(&buf[..3], b); - assert_eq!(reader.read(&mut buf).unwrap(), 0); - } - - #[test] - fn test_boxed_slice_reader_vectored() { - let mut reader = Cursor::new(vec![0, 1, 2, 3, 4, 5, 6, 7].into_boxed_slice()); - let mut buf = []; - assert_eq!(reader.read_vectored(&mut [IoSliceMut::new(&mut buf)]).unwrap(), 0); - assert_eq!(reader.position(), 0); - let mut buf = [0]; - assert_eq!( - reader - .read_vectored(&mut [IoSliceMut::new(&mut []), IoSliceMut::new(&mut buf),]) - .unwrap(), - 1, - ); - assert_eq!(reader.position(), 1); - let b: &[_] = &[0]; - assert_eq!(buf, b); - let mut buf1 = [0; 4]; - let mut buf2 = [0; 4]; - assert_eq!( - reader - .read_vectored(&mut [IoSliceMut::new(&mut buf1), IoSliceMut::new(&mut buf2)],) - .unwrap(), - 7, - ); - let b1: &[_] = &[1, 2, 3, 4]; - let b2: &[_] = &[5, 6, 7]; - assert_eq!(buf1, b1); - assert_eq!(&buf2[..3], b2); - assert_eq!(reader.read(&mut buf).unwrap(), 0); - } - - #[test] - fn read_to_end() { - let mut reader = Cursor::new(vec![0, 1, 2, 3, 4, 5, 6, 7]); - let mut v = Vec::new(); - reader.read_to_end(&mut v).unwrap(); - assert_eq!(v, [0, 1, 2, 3, 4, 5, 6, 7]); - } - - #[test] - fn test_slice_reader() { - let in_buf = vec![0, 1, 2, 3, 4, 5, 6, 7]; - let reader = &mut &in_buf[..]; - let mut buf = []; - assert_eq!(reader.read(&mut buf).unwrap(), 0); - let mut buf = [0]; - assert_eq!(reader.read(&mut buf).unwrap(), 1); - assert_eq!(reader.len(), 7); - let b: &[_] = &[0]; - assert_eq!(&buf[..], b); - let mut buf = [0; 4]; - assert_eq!(reader.read(&mut buf).unwrap(), 4); - assert_eq!(reader.len(), 3); - let b: &[_] = &[1, 2, 3, 4]; - assert_eq!(&buf[..], b); - assert_eq!(reader.read(&mut buf).unwrap(), 3); - let b: &[_] = &[5, 6, 7]; - assert_eq!(&buf[..3], b); - assert_eq!(reader.read(&mut buf).unwrap(), 0); - } - - #[test] - fn test_slice_reader_vectored() { - let in_buf = vec![0, 1, 2, 3, 4, 5, 6, 7]; - let reader = &mut &in_buf[..]; - let mut buf = []; - assert_eq!(reader.read_vectored(&mut [IoSliceMut::new(&mut buf)]).unwrap(), 0); - let mut buf = [0]; - assert_eq!( - reader - .read_vectored(&mut [IoSliceMut::new(&mut []), IoSliceMut::new(&mut buf),]) - .unwrap(), - 1, - ); - assert_eq!(reader.len(), 7); - let b: &[_] = &[0]; - assert_eq!(buf, b); - let mut buf1 = [0; 4]; - let mut buf2 = [0; 4]; - assert_eq!( - reader - .read_vectored(&mut [IoSliceMut::new(&mut buf1), IoSliceMut::new(&mut buf2)],) - .unwrap(), - 7, - ); - let b1: &[_] = &[1, 2, 3, 4]; - let b2: &[_] = &[5, 6, 7]; - assert_eq!(buf1, b1); - assert_eq!(&buf2[..3], b2); - assert_eq!(reader.read(&mut buf).unwrap(), 0); - } - - #[test] - fn test_read_exact() { - let in_buf = vec![0, 1, 2, 3, 4, 5, 6, 7]; - let reader = &mut &in_buf[..]; - let mut buf = []; - assert!(reader.read_exact(&mut buf).is_ok()); - let mut buf = [8]; - assert!(reader.read_exact(&mut buf).is_ok()); - assert_eq!(buf[0], 0); - assert_eq!(reader.len(), 7); - let mut buf = [0, 0, 0, 0, 0, 0, 0]; - assert!(reader.read_exact(&mut buf).is_ok()); - assert_eq!(buf, [1, 2, 3, 4, 5, 6, 7]); - assert_eq!(reader.len(), 0); - let mut buf = [0]; - assert!(reader.read_exact(&mut buf).is_err()); - } - - #[test] - fn test_buf_reader() { - let in_buf = vec![0, 1, 2, 3, 4, 5, 6, 7]; - let mut reader = Cursor::new(&in_buf[..]); - let mut buf = []; - assert_eq!(reader.read(&mut buf).unwrap(), 0); - assert_eq!(reader.position(), 0); - let mut buf = [0]; - assert_eq!(reader.read(&mut buf).unwrap(), 1); - assert_eq!(reader.position(), 1); - let b: &[_] = &[0]; - assert_eq!(buf, b); - let mut buf = [0; 4]; - assert_eq!(reader.read(&mut buf).unwrap(), 4); - assert_eq!(reader.position(), 5); - let b: &[_] = &[1, 2, 3, 4]; - assert_eq!(buf, b); - assert_eq!(reader.read(&mut buf).unwrap(), 3); - let b: &[_] = &[5, 6, 7]; - assert_eq!(&buf[..3], b); - assert_eq!(reader.read(&mut buf).unwrap(), 0); - } - - #[test] - fn seek_past_end() { - let buf = [0xff]; - let mut r = Cursor::new(&buf[..]); - assert_eq!(r.seek(SeekFrom::Start(10)).unwrap(), 10); - assert_eq!(r.read(&mut [0]).unwrap(), 0); - - let mut r = Cursor::new(vec![10]); - assert_eq!(r.seek(SeekFrom::Start(10)).unwrap(), 10); - assert_eq!(r.read(&mut [0]).unwrap(), 0); - - let mut buf = [0]; - let mut r = Cursor::new(&mut buf[..]); - assert_eq!(r.seek(SeekFrom::Start(10)).unwrap(), 10); - assert_eq!(r.write(&[3]).unwrap(), 0); - - let mut r = Cursor::new(vec![10].into_boxed_slice()); - assert_eq!(r.seek(SeekFrom::Start(10)).unwrap(), 10); - assert_eq!(r.write(&[3]).unwrap(), 0); - } - - #[test] - fn seek_past_i64() { - let buf = [0xff]; - let mut r = Cursor::new(&buf[..]); - assert_eq!(r.seek(SeekFrom::Start(6)).unwrap(), 6); - assert_eq!(r.seek(SeekFrom::Current(0x7ffffffffffffff0)).unwrap(), 0x7ffffffffffffff6); - assert_eq!(r.seek(SeekFrom::Current(0x10)).unwrap(), 0x8000000000000006); - assert_eq!(r.seek(SeekFrom::Current(0)).unwrap(), 0x8000000000000006); - assert!(r.seek(SeekFrom::Current(0x7ffffffffffffffd)).is_err()); - assert_eq!(r.seek(SeekFrom::Current(-0x8000000000000000)).unwrap(), 6); - - let mut r = Cursor::new(vec![10]); - assert_eq!(r.seek(SeekFrom::Start(6)).unwrap(), 6); - assert_eq!(r.seek(SeekFrom::Current(0x7ffffffffffffff0)).unwrap(), 0x7ffffffffffffff6); - assert_eq!(r.seek(SeekFrom::Current(0x10)).unwrap(), 0x8000000000000006); - assert_eq!(r.seek(SeekFrom::Current(0)).unwrap(), 0x8000000000000006); - assert!(r.seek(SeekFrom::Current(0x7ffffffffffffffd)).is_err()); - assert_eq!(r.seek(SeekFrom::Current(-0x8000000000000000)).unwrap(), 6); - - let mut buf = [0]; - let mut r = Cursor::new(&mut buf[..]); - assert_eq!(r.seek(SeekFrom::Start(6)).unwrap(), 6); - assert_eq!(r.seek(SeekFrom::Current(0x7ffffffffffffff0)).unwrap(), 0x7ffffffffffffff6); - assert_eq!(r.seek(SeekFrom::Current(0x10)).unwrap(), 0x8000000000000006); - assert_eq!(r.seek(SeekFrom::Current(0)).unwrap(), 0x8000000000000006); - assert!(r.seek(SeekFrom::Current(0x7ffffffffffffffd)).is_err()); - assert_eq!(r.seek(SeekFrom::Current(-0x8000000000000000)).unwrap(), 6); - - let mut r = Cursor::new(vec![10].into_boxed_slice()); - assert_eq!(r.seek(SeekFrom::Start(6)).unwrap(), 6); - assert_eq!(r.seek(SeekFrom::Current(0x7ffffffffffffff0)).unwrap(), 0x7ffffffffffffff6); - assert_eq!(r.seek(SeekFrom::Current(0x10)).unwrap(), 0x8000000000000006); - assert_eq!(r.seek(SeekFrom::Current(0)).unwrap(), 0x8000000000000006); - assert!(r.seek(SeekFrom::Current(0x7ffffffffffffffd)).is_err()); - assert_eq!(r.seek(SeekFrom::Current(-0x8000000000000000)).unwrap(), 6); - } - - #[test] - fn seek_before_0() { - let buf = [0xff]; - let mut r = Cursor::new(&buf[..]); - assert!(r.seek(SeekFrom::End(-2)).is_err()); - - let mut r = Cursor::new(vec![10]); - assert!(r.seek(SeekFrom::End(-2)).is_err()); - - let mut buf = [0]; - let mut r = Cursor::new(&mut buf[..]); - assert!(r.seek(SeekFrom::End(-2)).is_err()); - - let mut r = Cursor::new(vec![10].into_boxed_slice()); - assert!(r.seek(SeekFrom::End(-2)).is_err()); - } - - #[test] - fn test_seekable_mem_writer() { - let mut writer = Cursor::new(Vec::::new()); - assert_eq!(writer.position(), 0); - assert_eq!(writer.write(&[0]).unwrap(), 1); - assert_eq!(writer.position(), 1); - assert_eq!(writer.write(&[1, 2, 3]).unwrap(), 3); - assert_eq!(writer.write(&[4, 5, 6, 7]).unwrap(), 4); - assert_eq!(writer.position(), 8); - let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7]; - assert_eq!(&writer.get_ref()[..], b); - - assert_eq!(writer.seek(SeekFrom::Start(0)).unwrap(), 0); - assert_eq!(writer.position(), 0); - assert_eq!(writer.write(&[3, 4]).unwrap(), 2); - let b: &[_] = &[3, 4, 2, 3, 4, 5, 6, 7]; - assert_eq!(&writer.get_ref()[..], b); - - assert_eq!(writer.seek(SeekFrom::Current(1)).unwrap(), 3); - assert_eq!(writer.write(&[0, 1]).unwrap(), 2); - let b: &[_] = &[3, 4, 2, 0, 1, 5, 6, 7]; - assert_eq!(&writer.get_ref()[..], b); - - assert_eq!(writer.seek(SeekFrom::End(-1)).unwrap(), 7); - assert_eq!(writer.write(&[1, 2]).unwrap(), 2); - let b: &[_] = &[3, 4, 2, 0, 1, 5, 6, 1, 2]; - assert_eq!(&writer.get_ref()[..], b); - - assert_eq!(writer.seek(SeekFrom::End(1)).unwrap(), 10); - assert_eq!(writer.write(&[1]).unwrap(), 1); - let b: &[_] = &[3, 4, 2, 0, 1, 5, 6, 1, 2, 0, 1]; - assert_eq!(&writer.get_ref()[..], b); - } - - #[test] - fn vec_seek_past_end() { - let mut r = Cursor::new(Vec::new()); - assert_eq!(r.seek(SeekFrom::Start(10)).unwrap(), 10); - assert_eq!(r.write(&[3]).unwrap(), 1); - } - - #[test] - fn vec_seek_before_0() { - let mut r = Cursor::new(Vec::new()); - assert!(r.seek(SeekFrom::End(-2)).is_err()); - } - - #[test] - #[cfg(target_pointer_width = "32")] - fn vec_seek_and_write_past_usize_max() { - let mut c = Cursor::new(Vec::new()); - c.set_position(::max_value() as u64 + 1); - assert!(c.write_all(&[1, 2, 3]).is_err()); - } - - #[test] - fn test_partial_eq() { - assert_eq!(Cursor::new(Vec::::new()), Cursor::new(Vec::::new())); - } - - #[test] - fn test_eq() { - struct AssertEq(pub T); - - let _: AssertEq>> = AssertEq(Cursor::new(Vec::new())); +#[stable(feature = "cursor_array", since = "1.61.0")] +impl Write for Cursor<[u8; N]> { + #[inline] + fn write(&mut self, buf: &[u8]) -> io::Result { + slice_write(&mut self.pos, &mut self.inner, buf) + } + + #[inline] + fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { + slice_write_vectored(&mut self.pos, &mut self.inner, bufs) + } + + #[inline] + fn is_write_vectored(&self) -> bool { + true + } + + #[inline] + fn flush(&mut self) -> io::Result<()> { + Ok(()) } } diff --git a/crux-mir/lib/std/src/io/cursor/tests.rs b/crux-mir/lib/std/src/io/cursor/tests.rs new file mode 100644 index 000000000..d7c203c29 --- /dev/null +++ b/crux-mir/lib/std/src/io/cursor/tests.rs @@ -0,0 +1,567 @@ +use crate::io::prelude::*; +use crate::io::{Cursor, IoSlice, IoSliceMut, SeekFrom}; + +#[test] +fn test_vec_writer() { + let mut writer = Vec::new(); + assert_eq!(writer.write(&[0]).unwrap(), 1); + assert_eq!(writer.write(&[1, 2, 3]).unwrap(), 3); + assert_eq!(writer.write(&[4, 5, 6, 7]).unwrap(), 4); + assert_eq!( + writer + .write_vectored(&[IoSlice::new(&[]), IoSlice::new(&[8, 9]), IoSlice::new(&[10])],) + .unwrap(), + 3 + ); + let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + assert_eq!(writer, b); +} + +#[test] +fn test_mem_writer() { + let mut writer = Cursor::new(Vec::new()); + writer.set_position(10); + assert_eq!(writer.write(&[0]).unwrap(), 1); + assert_eq!(writer.write(&[1, 2, 3]).unwrap(), 3); + assert_eq!(writer.write(&[4, 5, 6, 7]).unwrap(), 4); + assert_eq!( + writer + .write_vectored(&[IoSlice::new(&[]), IoSlice::new(&[8, 9]), IoSlice::new(&[10])],) + .unwrap(), + 3 + ); + let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + assert_eq!(&writer.get_ref()[..10], &[0; 10]); + assert_eq!(&writer.get_ref()[10..], b); +} + +#[test] +fn test_mem_writer_preallocated() { + let mut writer = Cursor::new(vec![0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10]); + assert_eq!(writer.write(&[0]).unwrap(), 1); + assert_eq!(writer.write(&[1, 2, 3]).unwrap(), 3); + assert_eq!(writer.write(&[4, 5, 6, 7]).unwrap(), 4); + let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + assert_eq!(&writer.get_ref()[..], b); +} + +#[test] +fn test_mem_mut_writer() { + let mut vec = Vec::new(); + let mut writer = Cursor::new(&mut vec); + assert_eq!(writer.write(&[0]).unwrap(), 1); + assert_eq!(writer.write(&[1, 2, 3]).unwrap(), 3); + assert_eq!(writer.write(&[4, 5, 6, 7]).unwrap(), 4); + assert_eq!( + writer + .write_vectored(&[IoSlice::new(&[]), IoSlice::new(&[8, 9]), IoSlice::new(&[10])],) + .unwrap(), + 3 + ); + let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + assert_eq!(&writer.get_ref()[..], b); +} + +fn test_slice_writer(writer: &mut Cursor) +where + T: AsRef<[u8]>, + Cursor: Write, +{ + assert_eq!(writer.position(), 0); + assert_eq!(writer.write(&[0]).unwrap(), 1); + assert_eq!(writer.position(), 1); + assert_eq!(writer.write(&[1, 2, 3]).unwrap(), 3); + assert_eq!(writer.write(&[4, 5, 6, 7]).unwrap(), 4); + assert_eq!(writer.position(), 8); + assert_eq!(writer.write(&[]).unwrap(), 0); + assert_eq!(writer.position(), 8); + + assert_eq!(writer.write(&[8, 9]).unwrap(), 1); + assert_eq!(writer.write(&[10]).unwrap(), 0); + let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8]; + assert_eq!(writer.get_ref().as_ref(), b); +} + +fn test_slice_writer_vectored(writer: &mut Cursor) +where + T: AsRef<[u8]>, + Cursor: Write, +{ + assert_eq!(writer.position(), 0); + assert_eq!(writer.write_vectored(&[IoSlice::new(&[0])]).unwrap(), 1); + assert_eq!(writer.position(), 1); + assert_eq!( + writer.write_vectored(&[IoSlice::new(&[1, 2, 3]), IoSlice::new(&[4, 5, 6, 7]),]).unwrap(), + 7, + ); + assert_eq!(writer.position(), 8); + assert_eq!(writer.write_vectored(&[]).unwrap(), 0); + assert_eq!(writer.position(), 8); + + assert_eq!(writer.write_vectored(&[IoSlice::new(&[8, 9])]).unwrap(), 1); + assert_eq!(writer.write_vectored(&[IoSlice::new(&[10])]).unwrap(), 0); + let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8]; + assert_eq!(writer.get_ref().as_ref(), b); +} + +#[test] +fn test_box_slice_writer() { + let mut writer = Cursor::new(vec![0u8; 9].into_boxed_slice()); + test_slice_writer(&mut writer); +} + +#[test] +fn test_box_slice_writer_vectored() { + let mut writer = Cursor::new(vec![0u8; 9].into_boxed_slice()); + test_slice_writer_vectored(&mut writer); +} + +#[test] +fn test_array_writer() { + let mut writer = Cursor::new([0u8; 9]); + test_slice_writer(&mut writer); +} + +#[test] +fn test_array_writer_vectored() { + let mut writer = Cursor::new([0u8; 9]); + test_slice_writer_vectored(&mut writer); +} + +#[test] +fn test_buf_writer() { + let mut buf = [0 as u8; 9]; + let mut writer = Cursor::new(&mut buf[..]); + test_slice_writer(&mut writer); +} + +#[test] +fn test_buf_writer_vectored() { + let mut buf = [0 as u8; 9]; + let mut writer = Cursor::new(&mut buf[..]); + test_slice_writer_vectored(&mut writer); +} + +#[test] +fn test_buf_writer_seek() { + let mut buf = [0 as u8; 8]; + { + let mut writer = Cursor::new(&mut buf[..]); + assert_eq!(writer.position(), 0); + assert_eq!(writer.write(&[1]).unwrap(), 1); + assert_eq!(writer.position(), 1); + + assert_eq!(writer.seek(SeekFrom::Start(2)).unwrap(), 2); + assert_eq!(writer.position(), 2); + assert_eq!(writer.write(&[2]).unwrap(), 1); + assert_eq!(writer.position(), 3); + + assert_eq!(writer.seek(SeekFrom::Current(-2)).unwrap(), 1); + assert_eq!(writer.position(), 1); + assert_eq!(writer.write(&[3]).unwrap(), 1); + assert_eq!(writer.position(), 2); + + assert_eq!(writer.seek(SeekFrom::End(-1)).unwrap(), 7); + assert_eq!(writer.position(), 7); + assert_eq!(writer.write(&[4]).unwrap(), 1); + assert_eq!(writer.position(), 8); + } + let b: &[_] = &[1, 3, 2, 0, 0, 0, 0, 4]; + assert_eq!(buf, b); +} + +#[test] +fn test_buf_writer_error() { + let mut buf = [0 as u8; 2]; + let mut writer = Cursor::new(&mut buf[..]); + assert_eq!(writer.write(&[0]).unwrap(), 1); + assert_eq!(writer.write(&[0, 0]).unwrap(), 1); + assert_eq!(writer.write(&[0, 0]).unwrap(), 0); +} + +#[test] +fn test_mem_reader() { + let mut reader = Cursor::new(vec![0, 1, 2, 3, 4, 5, 6, 7]); + let mut buf = []; + assert_eq!(reader.read(&mut buf).unwrap(), 0); + assert_eq!(reader.position(), 0); + let mut buf = [0]; + assert_eq!(reader.read(&mut buf).unwrap(), 1); + assert_eq!(reader.position(), 1); + let b: &[_] = &[0]; + assert_eq!(buf, b); + let mut buf = [0; 4]; + assert_eq!(reader.read(&mut buf).unwrap(), 4); + assert_eq!(reader.position(), 5); + let b: &[_] = &[1, 2, 3, 4]; + assert_eq!(buf, b); + assert_eq!(reader.read(&mut buf).unwrap(), 3); + let b: &[_] = &[5, 6, 7]; + assert_eq!(&buf[..3], b); + assert_eq!(reader.read(&mut buf).unwrap(), 0); +} + +#[test] +fn test_mem_reader_vectored() { + let mut reader = Cursor::new(vec![0, 1, 2, 3, 4, 5, 6, 7]); + let mut buf = []; + assert_eq!(reader.read_vectored(&mut [IoSliceMut::new(&mut buf)]).unwrap(), 0); + assert_eq!(reader.position(), 0); + let mut buf = [0]; + assert_eq!( + reader.read_vectored(&mut [IoSliceMut::new(&mut []), IoSliceMut::new(&mut buf),]).unwrap(), + 1, + ); + assert_eq!(reader.position(), 1); + let b: &[_] = &[0]; + assert_eq!(buf, b); + let mut buf1 = [0; 4]; + let mut buf2 = [0; 4]; + assert_eq!( + reader + .read_vectored(&mut [IoSliceMut::new(&mut buf1), IoSliceMut::new(&mut buf2),]) + .unwrap(), + 7, + ); + let b1: &[_] = &[1, 2, 3, 4]; + let b2: &[_] = &[5, 6, 7]; + assert_eq!(buf1, b1); + assert_eq!(&buf2[..3], b2); + assert_eq!(reader.read(&mut buf).unwrap(), 0); +} + +#[test] +fn test_boxed_slice_reader() { + let mut reader = Cursor::new(vec![0, 1, 2, 3, 4, 5, 6, 7].into_boxed_slice()); + let mut buf = []; + assert_eq!(reader.read(&mut buf).unwrap(), 0); + assert_eq!(reader.position(), 0); + let mut buf = [0]; + assert_eq!(reader.read(&mut buf).unwrap(), 1); + assert_eq!(reader.position(), 1); + let b: &[_] = &[0]; + assert_eq!(buf, b); + let mut buf = [0; 4]; + assert_eq!(reader.read(&mut buf).unwrap(), 4); + assert_eq!(reader.position(), 5); + let b: &[_] = &[1, 2, 3, 4]; + assert_eq!(buf, b); + assert_eq!(reader.read(&mut buf).unwrap(), 3); + let b: &[_] = &[5, 6, 7]; + assert_eq!(&buf[..3], b); + assert_eq!(reader.read(&mut buf).unwrap(), 0); +} + +#[test] +fn test_boxed_slice_reader_vectored() { + let mut reader = Cursor::new(vec![0, 1, 2, 3, 4, 5, 6, 7].into_boxed_slice()); + let mut buf = []; + assert_eq!(reader.read_vectored(&mut [IoSliceMut::new(&mut buf)]).unwrap(), 0); + assert_eq!(reader.position(), 0); + let mut buf = [0]; + assert_eq!( + reader.read_vectored(&mut [IoSliceMut::new(&mut []), IoSliceMut::new(&mut buf),]).unwrap(), + 1, + ); + assert_eq!(reader.position(), 1); + let b: &[_] = &[0]; + assert_eq!(buf, b); + let mut buf1 = [0; 4]; + let mut buf2 = [0; 4]; + assert_eq!( + reader + .read_vectored(&mut [IoSliceMut::new(&mut buf1), IoSliceMut::new(&mut buf2)],) + .unwrap(), + 7, + ); + let b1: &[_] = &[1, 2, 3, 4]; + let b2: &[_] = &[5, 6, 7]; + assert_eq!(buf1, b1); + assert_eq!(&buf2[..3], b2); + assert_eq!(reader.read(&mut buf).unwrap(), 0); +} + +#[test] +fn read_to_end() { + let mut reader = Cursor::new(vec![0, 1, 2, 3, 4, 5, 6, 7]); + let mut v = Vec::new(); + reader.read_to_end(&mut v).unwrap(); + assert_eq!(v, [0, 1, 2, 3, 4, 5, 6, 7]); +} + +#[test] +fn test_slice_reader() { + let in_buf = vec![0, 1, 2, 3, 4, 5, 6, 7]; + let reader = &mut &in_buf[..]; + let mut buf = []; + assert_eq!(reader.read(&mut buf).unwrap(), 0); + let mut buf = [0]; + assert_eq!(reader.read(&mut buf).unwrap(), 1); + assert_eq!(reader.len(), 7); + let b: &[_] = &[0]; + assert_eq!(&buf[..], b); + let mut buf = [0; 4]; + assert_eq!(reader.read(&mut buf).unwrap(), 4); + assert_eq!(reader.len(), 3); + let b: &[_] = &[1, 2, 3, 4]; + assert_eq!(&buf[..], b); + assert_eq!(reader.read(&mut buf).unwrap(), 3); + let b: &[_] = &[5, 6, 7]; + assert_eq!(&buf[..3], b); + assert_eq!(reader.read(&mut buf).unwrap(), 0); +} + +#[test] +fn test_slice_reader_vectored() { + let in_buf = vec![0, 1, 2, 3, 4, 5, 6, 7]; + let reader = &mut &in_buf[..]; + let mut buf = []; + assert_eq!(reader.read_vectored(&mut [IoSliceMut::new(&mut buf)]).unwrap(), 0); + let mut buf = [0]; + assert_eq!( + reader.read_vectored(&mut [IoSliceMut::new(&mut []), IoSliceMut::new(&mut buf),]).unwrap(), + 1, + ); + assert_eq!(reader.len(), 7); + let b: &[_] = &[0]; + assert_eq!(buf, b); + let mut buf1 = [0; 4]; + let mut buf2 = [0; 4]; + assert_eq!( + reader + .read_vectored(&mut [IoSliceMut::new(&mut buf1), IoSliceMut::new(&mut buf2)],) + .unwrap(), + 7, + ); + let b1: &[_] = &[1, 2, 3, 4]; + let b2: &[_] = &[5, 6, 7]; + assert_eq!(buf1, b1); + assert_eq!(&buf2[..3], b2); + assert_eq!(reader.read(&mut buf).unwrap(), 0); +} + +#[test] +fn test_read_exact() { + let in_buf = vec![0, 1, 2, 3, 4, 5, 6, 7]; + let reader = &mut &in_buf[..]; + let mut buf = []; + assert!(reader.read_exact(&mut buf).is_ok()); + let mut buf = [8]; + assert!(reader.read_exact(&mut buf).is_ok()); + assert_eq!(buf[0], 0); + assert_eq!(reader.len(), 7); + let mut buf = [0, 0, 0, 0, 0, 0, 0]; + assert!(reader.read_exact(&mut buf).is_ok()); + assert_eq!(buf, [1, 2, 3, 4, 5, 6, 7]); + assert_eq!(reader.len(), 0); + let mut buf = [0]; + assert!(reader.read_exact(&mut buf).is_err()); +} + +#[test] +fn test_buf_reader() { + let in_buf = vec![0, 1, 2, 3, 4, 5, 6, 7]; + let mut reader = Cursor::new(&in_buf[..]); + let mut buf = []; + assert_eq!(reader.read(&mut buf).unwrap(), 0); + assert_eq!(reader.position(), 0); + let mut buf = [0]; + assert_eq!(reader.read(&mut buf).unwrap(), 1); + assert_eq!(reader.position(), 1); + let b: &[_] = &[0]; + assert_eq!(buf, b); + let mut buf = [0; 4]; + assert_eq!(reader.read(&mut buf).unwrap(), 4); + assert_eq!(reader.position(), 5); + let b: &[_] = &[1, 2, 3, 4]; + assert_eq!(buf, b); + assert_eq!(reader.read(&mut buf).unwrap(), 3); + let b: &[_] = &[5, 6, 7]; + assert_eq!(&buf[..3], b); + assert_eq!(reader.read(&mut buf).unwrap(), 0); +} + +#[test] +fn seek_past_end() { + let buf = [0xff]; + let mut r = Cursor::new(&buf[..]); + assert_eq!(r.seek(SeekFrom::Start(10)).unwrap(), 10); + assert_eq!(r.read(&mut [0]).unwrap(), 0); + + let mut r = Cursor::new(vec![10]); + assert_eq!(r.seek(SeekFrom::Start(10)).unwrap(), 10); + assert_eq!(r.read(&mut [0]).unwrap(), 0); + + let mut buf = [0]; + let mut r = Cursor::new(&mut buf[..]); + assert_eq!(r.seek(SeekFrom::Start(10)).unwrap(), 10); + assert_eq!(r.write(&[3]).unwrap(), 0); + + let mut r = Cursor::new(vec![10].into_boxed_slice()); + assert_eq!(r.seek(SeekFrom::Start(10)).unwrap(), 10); + assert_eq!(r.write(&[3]).unwrap(), 0); +} + +#[test] +fn seek_past_i64() { + let buf = [0xff]; + let mut r = Cursor::new(&buf[..]); + assert_eq!(r.seek(SeekFrom::Start(6)).unwrap(), 6); + assert_eq!(r.seek(SeekFrom::Current(0x7ffffffffffffff0)).unwrap(), 0x7ffffffffffffff6); + assert_eq!(r.seek(SeekFrom::Current(0x10)).unwrap(), 0x8000000000000006); + assert_eq!(r.seek(SeekFrom::Current(0)).unwrap(), 0x8000000000000006); + assert!(r.seek(SeekFrom::Current(0x7ffffffffffffffd)).is_err()); + assert_eq!(r.seek(SeekFrom::Current(-0x8000000000000000)).unwrap(), 6); + + let mut r = Cursor::new(vec![10]); + assert_eq!(r.seek(SeekFrom::Start(6)).unwrap(), 6); + assert_eq!(r.seek(SeekFrom::Current(0x7ffffffffffffff0)).unwrap(), 0x7ffffffffffffff6); + assert_eq!(r.seek(SeekFrom::Current(0x10)).unwrap(), 0x8000000000000006); + assert_eq!(r.seek(SeekFrom::Current(0)).unwrap(), 0x8000000000000006); + assert!(r.seek(SeekFrom::Current(0x7ffffffffffffffd)).is_err()); + assert_eq!(r.seek(SeekFrom::Current(-0x8000000000000000)).unwrap(), 6); + + let mut buf = [0]; + let mut r = Cursor::new(&mut buf[..]); + assert_eq!(r.seek(SeekFrom::Start(6)).unwrap(), 6); + assert_eq!(r.seek(SeekFrom::Current(0x7ffffffffffffff0)).unwrap(), 0x7ffffffffffffff6); + assert_eq!(r.seek(SeekFrom::Current(0x10)).unwrap(), 0x8000000000000006); + assert_eq!(r.seek(SeekFrom::Current(0)).unwrap(), 0x8000000000000006); + assert!(r.seek(SeekFrom::Current(0x7ffffffffffffffd)).is_err()); + assert_eq!(r.seek(SeekFrom::Current(-0x8000000000000000)).unwrap(), 6); + + let mut r = Cursor::new(vec![10].into_boxed_slice()); + assert_eq!(r.seek(SeekFrom::Start(6)).unwrap(), 6); + assert_eq!(r.seek(SeekFrom::Current(0x7ffffffffffffff0)).unwrap(), 0x7ffffffffffffff6); + assert_eq!(r.seek(SeekFrom::Current(0x10)).unwrap(), 0x8000000000000006); + assert_eq!(r.seek(SeekFrom::Current(0)).unwrap(), 0x8000000000000006); + assert!(r.seek(SeekFrom::Current(0x7ffffffffffffffd)).is_err()); + assert_eq!(r.seek(SeekFrom::Current(-0x8000000000000000)).unwrap(), 6); +} + +#[test] +fn seek_before_0() { + let buf = [0xff]; + let mut r = Cursor::new(&buf[..]); + assert!(r.seek(SeekFrom::End(-2)).is_err()); + + let mut r = Cursor::new(vec![10]); + assert!(r.seek(SeekFrom::End(-2)).is_err()); + + let mut buf = [0]; + let mut r = Cursor::new(&mut buf[..]); + assert!(r.seek(SeekFrom::End(-2)).is_err()); + + let mut r = Cursor::new(vec![10].into_boxed_slice()); + assert!(r.seek(SeekFrom::End(-2)).is_err()); +} + +#[test] +fn test_seekable_mem_writer() { + let mut writer = Cursor::new(Vec::::new()); + assert_eq!(writer.position(), 0); + assert_eq!(writer.write(&[0]).unwrap(), 1); + assert_eq!(writer.position(), 1); + assert_eq!(writer.write(&[1, 2, 3]).unwrap(), 3); + assert_eq!(writer.write(&[4, 5, 6, 7]).unwrap(), 4); + assert_eq!(writer.position(), 8); + let b: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7]; + assert_eq!(&writer.get_ref()[..], b); + + assert_eq!(writer.seek(SeekFrom::Start(0)).unwrap(), 0); + assert_eq!(writer.position(), 0); + assert_eq!(writer.write(&[3, 4]).unwrap(), 2); + let b: &[_] = &[3, 4, 2, 3, 4, 5, 6, 7]; + assert_eq!(&writer.get_ref()[..], b); + + assert_eq!(writer.seek(SeekFrom::Current(1)).unwrap(), 3); + assert_eq!(writer.write(&[0, 1]).unwrap(), 2); + let b: &[_] = &[3, 4, 2, 0, 1, 5, 6, 7]; + assert_eq!(&writer.get_ref()[..], b); + + assert_eq!(writer.seek(SeekFrom::End(-1)).unwrap(), 7); + assert_eq!(writer.write(&[1, 2]).unwrap(), 2); + let b: &[_] = &[3, 4, 2, 0, 1, 5, 6, 1, 2]; + assert_eq!(&writer.get_ref()[..], b); + + assert_eq!(writer.seek(SeekFrom::End(1)).unwrap(), 10); + assert_eq!(writer.write(&[1]).unwrap(), 1); + let b: &[_] = &[3, 4, 2, 0, 1, 5, 6, 1, 2, 0, 1]; + assert_eq!(&writer.get_ref()[..], b); +} + +#[test] +fn vec_seek_past_end() { + let mut r = Cursor::new(Vec::new()); + assert_eq!(r.seek(SeekFrom::Start(10)).unwrap(), 10); + assert_eq!(r.write(&[3]).unwrap(), 1); +} + +#[test] +fn vec_seek_before_0() { + let mut r = Cursor::new(Vec::new()); + assert!(r.seek(SeekFrom::End(-2)).is_err()); +} + +#[test] +#[cfg(target_pointer_width = "32")] +fn vec_seek_and_write_past_usize_max() { + let mut c = Cursor::new(Vec::new()); + c.set_position(usize::MAX as u64 + 1); + assert!(c.write_all(&[1, 2, 3]).is_err()); +} + +#[test] +fn test_partial_eq() { + assert_eq!(Cursor::new(Vec::::new()), Cursor::new(Vec::::new())); +} + +#[test] +fn test_eq() { + struct AssertEq(pub T); + + let _: AssertEq>> = AssertEq(Cursor::new(Vec::new())); +} + +#[allow(dead_code)] +fn const_cursor() { + const CURSOR: Cursor<&[u8]> = Cursor::new(&[0]); + const _: &&[u8] = CURSOR.get_ref(); + const _: u64 = CURSOR.position(); +} + +#[bench] +fn bench_write_vec(b: &mut test::Bencher) { + let slice = &[1; 128]; + + b.iter(|| { + let mut buf = b"some random data to overwrite".to_vec(); + let mut cursor = Cursor::new(&mut buf); + + let _ = cursor.write_all(slice); + test::black_box(&cursor); + }) +} + +#[bench] +fn bench_write_vec_vectored(b: &mut test::Bencher) { + let slices = [ + IoSlice::new(&[1; 128]), + IoSlice::new(&[2; 256]), + IoSlice::new(&[3; 512]), + IoSlice::new(&[4; 1024]), + IoSlice::new(&[5; 2048]), + IoSlice::new(&[6; 4096]), + IoSlice::new(&[7; 8192]), + IoSlice::new(&[8; 8192 * 2]), + ]; + + b.iter(|| { + let mut buf = b"some random data to overwrite".to_vec(); + let mut cursor = Cursor::new(&mut buf); + + let mut slices = slices; + let _ = cursor.write_all_vectored(&mut slices); + test::black_box(&cursor); + }) +} diff --git a/crux-mir/lib/std/src/io/error.rs b/crux-mir/lib/std/src/io/error.rs index 3b55d9b90..3cabf2449 100644 --- a/crux-mir/lib/std/src/io/error.rs +++ b/crux-mir/lib/std/src/io/error.rs @@ -1,11 +1,23 @@ +#[cfg(test)] +mod tests; + +#[cfg(target_pointer_width = "64")] +mod repr_bitpacked; +#[cfg(target_pointer_width = "64")] +use repr_bitpacked::Repr; + +#[cfg(not(target_pointer_width = "64"))] +mod repr_unpacked; +#[cfg(not(target_pointer_width = "64"))] +use repr_unpacked::Repr; + use crate::convert::From; use crate::error; use crate::fmt; use crate::result; use crate::sys; -/// A specialized [`Result`](../result/enum.Result.html) type for I/O -/// operations. +/// A specialized [`Result`] type for I/O operations. /// /// This type is broadly used across [`std::io`] for any operation which may /// produce an error. @@ -16,12 +28,13 @@ use crate::sys; /// While usual Rust style is to import types directly, aliases of [`Result`] /// often are not, to make it easier to distinguish between them. [`Result`] is /// generally assumed to be [`std::result::Result`][`Result`], and so users of this alias -/// will generally use `io::Result` instead of shadowing the prelude's import +/// will generally use `io::Result` instead of shadowing the [prelude]'s import /// of [`std::result::Result`][`Result`]. /// -/// [`std::io`]: ../io/index.html -/// [`io::Error`]: ../io/struct.Error.html -/// [`Result`]: ../result/enum.Result.html +/// [`std::io`]: crate::io +/// [`io::Error`]: Error +/// [`Result`]: crate::result::Result +/// [prelude]: crate::prelude /// /// # Examples /// @@ -48,10 +61,9 @@ pub type Result = result::Result; /// `Error` can be created with crafted error messages and a particular value of /// [`ErrorKind`]. /// -/// [`Read`]: ../io/trait.Read.html -/// [`Write`]: ../io/trait.Write.html -/// [`Seek`]: ../io/trait.Seek.html -/// [`ErrorKind`]: enum.ErrorKind.html +/// [`Read`]: crate::io::Read +/// [`Write`]: crate::io::Write +/// [`Seek`]: crate::io::Seek #[stable(feature = "rust1", since = "1.0.0")] pub struct Error { repr: Repr, @@ -64,13 +76,66 @@ impl fmt::Debug for Error { } } -enum Repr { +#[stable(feature = "rust1", since = "1.0.0")] +impl From for Error { + /// Converts a [`alloc::ffi::NulError`] into a [`Error`]. + fn from(_: alloc::ffi::NulError) -> Error { + const_io_error!(ErrorKind::InvalidInput, "data provided contains a nul byte") + } +} + +// Only derive debug in tests, to make sure it +// doesn't accidentally get printed. +#[cfg_attr(test, derive(Debug))] +enum ErrorData { Os(i32), Simple(ErrorKind), - Custom(Box), + SimpleMessage(&'static SimpleMessage), + Custom(C), +} + +// `#[repr(align(4))]` is probably redundant, it should have that value or +// higher already. We include it just because repr_bitpacked.rs's encoding +// requires an alignment >= 4 (note that `#[repr(align)]` will not reduce the +// alignment required by the struct, only increase it). +// +// If we add more variants to ErrorData, this can be increased to 8, but it +// should probably be behind `#[cfg_attr(target_pointer_width = "64", ...)]` or +// whatever cfg we're using to enable the `repr_bitpacked` code, since only the +// that version needs the alignment, and 8 is higher than the alignment we'll +// have on 32 bit platforms. +// +// (For the sake of being explicit: the alignment requirement here only matters +// if `error/repr_bitpacked.rs` is in use — for the unpacked repr it doesn't +// matter at all) +#[repr(align(4))] +#[derive(Debug)] +pub(crate) struct SimpleMessage { + kind: ErrorKind, + message: &'static str, +} + +impl SimpleMessage { + pub(crate) const fn new(kind: ErrorKind, message: &'static str) -> Self { + Self { kind, message } + } } +/// Create and return an `io::Error` for a given `ErrorKind` and constant +/// message. This doesn't allocate. +pub(crate) macro const_io_error($kind:expr, $message:expr $(,)?) { + $crate::io::error::Error::from_static_message({ + const MESSAGE_DATA: $crate::io::error::SimpleMessage = + $crate::io::error::SimpleMessage::new($kind, $message); + &MESSAGE_DATA + }) +} + +// As with `SimpleMessage`: `#[repr(align(4))]` here is just because +// repr_bitpacked's encoding requires it. In practice it almost certainly be +// already be this high or higher. #[derive(Debug)] +#[repr(align(4))] struct Custom { kind: ErrorKind, error: Box, @@ -83,7 +148,20 @@ struct Custom { /// /// It is used with the [`io::Error`] type. /// -/// [`io::Error`]: struct.Error.html +/// [`io::Error`]: Error +/// +/// # Handling errors and matching on `ErrorKind` +/// +/// In application code, use `match` for the `ErrorKind` values you are +/// expecting; use `_` to match "all other errors". +/// +/// In comprehensive and thorough tests that want to verify that a test doesn't +/// return any known incorrect error kind, you may want to cut-and-paste the +/// current full list of errors from here into your test code, and then match +/// `_` as the correct case. This seems counterintuitive, but it will make your +/// tests more robust. In particular, if you want to verify that your code does +/// produce an unrecognized error kind, the robust solution is to check for all +/// the recognized error kinds and fail in those cases. #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] #[stable(feature = "rust1", since = "1.0.0")] #[allow(deprecated)] @@ -101,6 +179,12 @@ pub enum ErrorKind { /// The connection was reset by the remote server. #[stable(feature = "rust1", since = "1.0.0")] ConnectionReset, + /// The remote host is not reachable. + #[unstable(feature = "io_error_more", issue = "86442")] + HostUnreachable, + /// The network containing the remote host is not reachable. + #[unstable(feature = "io_error_more", issue = "86442")] + NetworkUnreachable, /// The connection was aborted (terminated) by the remote server. #[stable(feature = "rust1", since = "1.0.0")] ConnectionAborted, @@ -115,6 +199,9 @@ pub enum ErrorKind { /// local. #[stable(feature = "rust1", since = "1.0.0")] AddrNotAvailable, + /// The system's networking is down. + #[unstable(feature = "io_error_more", issue = "86442")] + NetworkDown, /// The operation failed because a pipe was closed. #[stable(feature = "rust1", since = "1.0.0")] BrokenPipe, @@ -125,6 +212,38 @@ pub enum ErrorKind { /// requested to not occur. #[stable(feature = "rust1", since = "1.0.0")] WouldBlock, + /// A filesystem object is, unexpectedly, not a directory. + /// + /// For example, a filesystem path was specified where one of the intermediate directory + /// components was, in fact, a plain file. + #[unstable(feature = "io_error_more", issue = "86442")] + NotADirectory, + /// The filesystem object is, unexpectedly, a directory. + /// + /// A directory was specified when a non-directory was expected. + #[unstable(feature = "io_error_more", issue = "86442")] + IsADirectory, + /// A non-empty directory was specified where an empty directory was expected. + #[unstable(feature = "io_error_more", issue = "86442")] + DirectoryNotEmpty, + /// The filesystem or storage medium is read-only, but a write operation was attempted. + #[unstable(feature = "io_error_more", issue = "86442")] + ReadOnlyFilesystem, + /// Loop in the filesystem or IO subsystem; often, too many levels of symbolic links. + /// + /// There was a loop (or excessively long chain) resolving a filesystem object + /// or file IO object. + /// + /// On Unix this is usually the result of a symbolic link loop; or, of exceeding the + /// system-specific limit on the depth of symlink traversal. + #[unstable(feature = "io_error_more", issue = "86442")] + FilesystemLoop, + /// Stale network file handle. + /// + /// With some network filesystems, notably NFS, an open file (or directory) can be invalidated + /// by problems with the network or server. + #[unstable(feature = "io_error_more", issue = "86442")] + StaleNetworkFileHandle, /// A parameter was incorrect. #[stable(feature = "rust1", since = "1.0.0")] InvalidInput, @@ -137,7 +256,7 @@ pub enum ErrorKind { /// For example, a function that reads a file into a string will error with /// `InvalidData` if the file's contents are not valid UTF-8. /// - /// [`InvalidInput`]: #variant.InvalidInput + /// [`InvalidInput`]: ErrorKind::InvalidInput #[stable(feature = "io_invalid_data", since = "1.2.0")] InvalidData, /// The I/O operation's timeout expired, causing it to be canceled. @@ -150,19 +269,80 @@ pub enum ErrorKind { /// particular number of bytes but only a smaller number of bytes could be /// written. /// - /// [`write`]: ../../std/io/trait.Write.html#tymethod.write - /// [`Ok(0)`]: ../../std/io/type.Result.html + /// [`write`]: crate::io::Write::write + /// [`Ok(0)`]: Ok #[stable(feature = "rust1", since = "1.0.0")] WriteZero, + /// The underlying storage (typically, a filesystem) is full. + /// + /// This does not include out of quota errors. + #[unstable(feature = "io_error_more", issue = "86442")] + StorageFull, + /// Seek on unseekable file. + /// + /// Seeking was attempted on an open file handle which is not suitable for seeking - for + /// example, on Unix, a named pipe opened with `File::open`. + #[unstable(feature = "io_error_more", issue = "86442")] + NotSeekable, + /// Filesystem quota was exceeded. + #[unstable(feature = "io_error_more", issue = "86442")] + FilesystemQuotaExceeded, + /// File larger than allowed or supported. + /// + /// This might arise from a hard limit of the underlying filesystem or file access API, or from + /// an administratively imposed resource limitation. Simple disk full, and out of quota, have + /// their own errors. + #[unstable(feature = "io_error_more", issue = "86442")] + FileTooLarge, + /// Resource is busy. + #[unstable(feature = "io_error_more", issue = "86442")] + ResourceBusy, + /// Executable file is busy. + /// + /// An attempt was made to write to a file which is also in use as a running program. (Not all + /// operating systems detect this situation.) + #[unstable(feature = "io_error_more", issue = "86442")] + ExecutableFileBusy, + /// Deadlock (avoided). + /// + /// A file locking operation would result in deadlock. This situation is typically detected, if + /// at all, on a best-effort basis. + #[unstable(feature = "io_error_more", issue = "86442")] + Deadlock, + /// Cross-device or cross-filesystem (hard) link or rename. + #[unstable(feature = "io_error_more", issue = "86442")] + CrossesDevices, + /// Too many (hard) links to the same filesystem object. + /// + /// The filesystem does not support making so many hardlinks to the same file. + #[unstable(feature = "io_error_more", issue = "86442")] + TooManyLinks, + /// A filename was invalid. + /// + /// This error can also cause if it exceeded the filename length limit. + #[unstable(feature = "io_error_more", issue = "86442")] + InvalidFilename, + /// Program argument list too long. + /// + /// When trying to run an external program, a system or process limit on the size of the + /// arguments would have been exceeded. + #[unstable(feature = "io_error_more", issue = "86442")] + ArgumentListTooLong, /// This operation was interrupted. /// /// Interrupted operations can typically be retried. #[stable(feature = "rust1", since = "1.0.0")] Interrupted, - /// Any I/O error not part of this list. - #[stable(feature = "rust1", since = "1.0.0")] - Other, + /// This operation is unsupported on this platform. + /// + /// This means that the operation can never succeed. + #[stable(feature = "unsupported_error", since = "1.53.0")] + Unsupported, + + // ErrorKinds which are primarily categorisations for OS error + // codes should be added above. + // /// An error returned when an operation could not be completed because an /// "end of file" was reached prematurely. /// @@ -171,30 +351,103 @@ pub enum ErrorKind { /// read. #[stable(feature = "read_exact", since = "1.6.0")] UnexpectedEof, + + /// An operation could not be completed, because it failed + /// to allocate enough memory. + #[stable(feature = "out_of_memory_error", since = "1.54.0")] + OutOfMemory, + + // "Unusual" error kinds which do not correspond simply to (sets + // of) OS error codes, should be added just above this comment. + // `Other` and `Uncategorised` should remain at the end: + // + /// A custom error that does not fall under any other I/O error kind. + /// + /// This can be used to construct your own [`Error`]s that do not match any + /// [`ErrorKind`]. + /// + /// This [`ErrorKind`] is not used by the standard library. + /// + /// Errors from the standard library that do not fall under any of the I/O + /// error kinds cannot be `match`ed on, and will only match a wildcard (`_`) pattern. + /// New [`ErrorKind`]s might be added in the future for some of those. + #[stable(feature = "rust1", since = "1.0.0")] + Other, + + /// Any I/O error from the standard library that's not part of this list. + /// + /// Errors that are `Uncategorized` now may move to a different or a new + /// [`ErrorKind`] variant in the future. It is not recommended to match + /// an error against `Uncategorized`; use a wildcard match (`_`) instead. + #[unstable(feature = "io_error_uncategorized", issue = "none")] + #[doc(hidden)] + Uncategorized, } impl ErrorKind { pub(crate) fn as_str(&self) -> &'static str { + use ErrorKind::*; + // tidy-alphabetical-start match *self { - ErrorKind::NotFound => "entity not found", - ErrorKind::PermissionDenied => "permission denied", - ErrorKind::ConnectionRefused => "connection refused", - ErrorKind::ConnectionReset => "connection reset", - ErrorKind::ConnectionAborted => "connection aborted", - ErrorKind::NotConnected => "not connected", - ErrorKind::AddrInUse => "address in use", - ErrorKind::AddrNotAvailable => "address not available", - ErrorKind::BrokenPipe => "broken pipe", - ErrorKind::AlreadyExists => "entity already exists", - ErrorKind::WouldBlock => "operation would block", - ErrorKind::InvalidInput => "invalid input parameter", - ErrorKind::InvalidData => "invalid data", - ErrorKind::TimedOut => "timed out", - ErrorKind::WriteZero => "write zero", - ErrorKind::Interrupted => "operation interrupted", - ErrorKind::Other => "other os error", - ErrorKind::UnexpectedEof => "unexpected end of file", + AddrInUse => "address in use", + AddrNotAvailable => "address not available", + AlreadyExists => "entity already exists", + ArgumentListTooLong => "argument list too long", + BrokenPipe => "broken pipe", + ConnectionAborted => "connection aborted", + ConnectionRefused => "connection refused", + ConnectionReset => "connection reset", + CrossesDevices => "cross-device link or rename", + Deadlock => "deadlock", + DirectoryNotEmpty => "directory not empty", + ExecutableFileBusy => "executable file busy", + FileTooLarge => "file too large", + FilesystemLoop => "filesystem loop or indirection limit (e.g. symlink loop)", + FilesystemQuotaExceeded => "filesystem quota exceeded", + HostUnreachable => "host unreachable", + Interrupted => "operation interrupted", + InvalidData => "invalid data", + InvalidFilename => "invalid filename", + InvalidInput => "invalid input parameter", + IsADirectory => "is a directory", + NetworkDown => "network down", + NetworkUnreachable => "network unreachable", + NotADirectory => "not a directory", + NotConnected => "not connected", + NotFound => "entity not found", + NotSeekable => "seek on unseekable file", + Other => "other error", + OutOfMemory => "out of memory", + PermissionDenied => "permission denied", + ReadOnlyFilesystem => "read-only filesystem or storage medium", + ResourceBusy => "resource busy", + StaleNetworkFileHandle => "stale network file handle", + StorageFull => "no storage space", + TimedOut => "timed out", + TooManyLinks => "too many links", + Uncategorized => "uncategorized error", + UnexpectedEof => "unexpected end of file", + Unsupported => "unsupported", + WouldBlock => "operation would block", + WriteZero => "write zero", } + // tidy-alphabetical-end + } +} + +#[stable(feature = "io_errorkind_display", since = "1.60.0")] +impl fmt::Display for ErrorKind { + /// Shows a human-readable description of the `ErrorKind`. + /// + /// This is similar to `impl Display for Error`, but doesn't require first converting to Error. + /// + /// # Examples + /// ``` + /// use std::io::ErrorKind; + /// assert_eq!("entity not found", ErrorKind::NotFound.to_string()); + /// ``` + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.write_str(self.as_str()) } } @@ -204,7 +457,7 @@ impl ErrorKind { impl From for Error { /// Converts an [`ErrorKind`] into an [`Error`]. /// - /// This conversion allocates a new error with a simple representation of error kind. + /// This conversion creates a new error with a simple representation of error kind. /// /// # Examples /// @@ -213,14 +466,11 @@ impl From for Error { /// /// let not_found = ErrorKind::NotFound; /// let error = Error::from(not_found); - /// assert_eq!("entity not found", format!("{}", error)); + /// assert_eq!("entity not found", format!("{error}")); /// ``` - /// - /// [`ErrorKind`]: ../../std/io/enum.ErrorKind.html - /// [`Error`]: ../../std/io/struct.Error.html #[inline] fn from(kind: ErrorKind) -> Error { - Error { repr: Repr::Simple(kind) } + Error { repr: Repr::new_simple(kind) } } } @@ -230,7 +480,11 @@ impl Error { /// /// This function is used to generically create I/O errors which do not /// originate from the OS itself. The `error` argument is an arbitrary - /// payload which will be contained in this `Error`. + /// payload which will be contained in this [`Error`]. + /// + /// Note that this function allocates memory on the heap. + /// If no extra payload is required, use the `From` conversion from + /// `ErrorKind`. /// /// # Examples /// @@ -242,6 +496,9 @@ impl Error { /// /// // errors can also be created from other errors /// let custom_error2 = Error::new(ErrorKind::Interrupted, custom_error); + /// + /// // creating an error without payload (and without memory allocation) + /// let eof_error = Error::from(ErrorKind::UnexpectedEof); /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn new(kind: ErrorKind, error: E) -> Error @@ -251,29 +508,81 @@ impl Error { Self::_new(kind, error.into()) } + /// Creates a new I/O error from an arbitrary error payload. + /// + /// This function is used to generically create I/O errors which do not + /// originate from the OS itself. It is a shortcut for [`Error::new`] + /// with [`ErrorKind::Other`]. + /// + /// # Examples + /// + /// ``` + /// #![feature(io_error_other)] + /// + /// use std::io::Error; + /// + /// // errors can be created from strings + /// let custom_error = Error::other("oh no!"); + /// + /// // errors can also be created from other errors + /// let custom_error2 = Error::other(custom_error); + /// ``` + #[unstable(feature = "io_error_other", issue = "91946")] + pub fn other(error: E) -> Error + where + E: Into>, + { + Self::_new(ErrorKind::Other, error.into()) + } + fn _new(kind: ErrorKind, error: Box) -> Error { - Error { repr: Repr::Custom(Box::new(Custom { kind, error })) } + Error { repr: Repr::new_custom(Box::new(Custom { kind, error })) } + } + + /// Creates a new I/O error from a known kind of error as well as a constant + /// message. + /// + /// This function does not allocate. + /// + /// You should not use this directly, and instead use the `const_io_error!` + /// macro: `io::const_io_error!(ErrorKind::Something, "some_message")`. + /// + /// This function should maybe change to `from_static_message(kind: ErrorKind)` in the future, when const generics allow that. + #[inline] + pub(crate) const fn from_static_message(msg: &'static SimpleMessage) -> Error { + Self { repr: Repr::new_simple_message(msg) } } /// Returns an error representing the last OS error which occurred. /// /// This function reads the value of `errno` for the target platform (e.g. /// `GetLastError` on Windows) and will return a corresponding instance of - /// `Error` for the error code. + /// [`Error`] for the error code. + /// + /// This should be called immediately after a call to a platform function, + /// otherwise the state of the error value is indeterminate. In particular, + /// other standard library functions may call platform functions that may + /// (or may not) reset the error value even if they succeed. /// /// # Examples /// /// ``` /// use std::io::Error; /// - /// println!("last OS error: {:?}", Error::last_os_error()); + /// let os_error = Error::last_os_error(); + /// println!("last OS error: {os_error:?}"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[doc(alias = "GetLastError")] + #[doc(alias = "errno")] + #[must_use] + #[inline] pub fn last_os_error() -> Error { Error::from_raw_os_error(sys::os::errno() as i32) } - /// Creates a new instance of an `Error` from a particular OS error code. + /// Creates a new instance of an [`Error`] from a particular OS error code. /// /// # Examples /// @@ -299,15 +608,20 @@ impl Error { /// # } /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + #[inline] pub fn from_raw_os_error(code: i32) -> Error { - Error { repr: Repr::Os(code) } + Error { repr: Repr::new_os(code) } } /// Returns the OS error that this error represents (if any). /// - /// If this `Error` was constructed via `last_os_error` or - /// `from_raw_os_error`, then this function will return `Some`, otherwise - /// it will return `None`. + /// If this [`Error`] was constructed via [`last_os_error`] or + /// [`from_raw_os_error`], then this function will return [`Some`], otherwise + /// it will return [`None`]. + /// + /// [`last_os_error`]: Error::last_os_error + /// [`from_raw_os_error`]: Error::from_raw_os_error /// /// # Examples /// @@ -316,7 +630,7 @@ impl Error { /// /// fn print_os_error(err: &Error) { /// if let Some(raw_os_err) = err.raw_os_error() { - /// println!("raw OS error: {:?}", raw_os_err); + /// println!("raw OS error: {raw_os_err:?}"); /// } else { /// println!("Not an OS error"); /// } @@ -330,18 +644,23 @@ impl Error { /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + #[inline] pub fn raw_os_error(&self) -> Option { - match self.repr { - Repr::Os(i) => Some(i), - Repr::Custom(..) => None, - Repr::Simple(..) => None, + match self.repr.data() { + ErrorData::Os(i) => Some(i), + ErrorData::Custom(..) => None, + ErrorData::Simple(..) => None, + ErrorData::SimpleMessage(..) => None, } } /// Returns a reference to the inner error wrapped by this error (if any). /// - /// If this `Error` was constructed via `new` then this function will - /// return `Some`, otherwise it will return `None`. + /// If this [`Error`] was constructed via [`new`] then this function will + /// return [`Some`], otherwise it will return [`None`]. + /// + /// [`new`]: Error::new /// /// # Examples /// @@ -350,7 +669,7 @@ impl Error { /// /// fn print_error(err: &Error) { /// if let Some(inner_err) = err.get_ref() { - /// println!("Inner error: {:?}", inner_err); + /// println!("Inner error: {inner_err:?}"); /// } else { /// println!("No inner error"); /// } @@ -364,19 +683,24 @@ impl Error { /// } /// ``` #[stable(feature = "io_error_inner", since = "1.3.0")] + #[must_use] + #[inline] pub fn get_ref(&self) -> Option<&(dyn error::Error + Send + Sync + 'static)> { - match self.repr { - Repr::Os(..) => None, - Repr::Simple(..) => None, - Repr::Custom(ref c) => Some(&*c.error), + match self.repr.data() { + ErrorData::Os(..) => None, + ErrorData::Simple(..) => None, + ErrorData::SimpleMessage(..) => None, + ErrorData::Custom(c) => Some(&*c.error), } } /// Returns a mutable reference to the inner error wrapped by this error /// (if any). /// - /// If this `Error` was constructed via `new` then this function will - /// return `Some`, otherwise it will return `None`. + /// If this [`Error`] was constructed via [`new`] then this function will + /// return [`Some`], otherwise it will return [`None`]. + /// + /// [`new`]: Error::new /// /// # Examples /// @@ -419,7 +743,7 @@ impl Error { /// /// fn print_error(err: &Error) { /// if let Some(inner_err) = err.get_ref() { - /// println!("Inner error: {}", inner_err); + /// println!("Inner error: {inner_err}"); /// } else { /// println!("No inner error"); /// } @@ -433,18 +757,23 @@ impl Error { /// } /// ``` #[stable(feature = "io_error_inner", since = "1.3.0")] + #[must_use] + #[inline] pub fn get_mut(&mut self) -> Option<&mut (dyn error::Error + Send + Sync + 'static)> { - match self.repr { - Repr::Os(..) => None, - Repr::Simple(..) => None, - Repr::Custom(ref mut c) => Some(&mut *c.error), + match self.repr.data_mut() { + ErrorData::Os(..) => None, + ErrorData::Simple(..) => None, + ErrorData::SimpleMessage(..) => None, + ErrorData::Custom(c) => Some(&mut *c.error), } } /// Consumes the `Error`, returning its inner error (if any). /// - /// If this `Error` was constructed via `new` then this function will - /// return `Some`, otherwise it will return `None`. + /// If this [`Error`] was constructed via [`new`] then this function will + /// return [`Some`], otherwise it will return [`None`]. + /// + /// [`new`]: Error::new /// /// # Examples /// @@ -453,7 +782,7 @@ impl Error { /// /// fn print_error(err: Error) { /// if let Some(inner_err) = err.into_inner() { - /// println!("Inner error: {}", inner_err); + /// println!("Inner error: {inner_err}"); /// } else { /// println!("No inner error"); /// } @@ -467,15 +796,80 @@ impl Error { /// } /// ``` #[stable(feature = "io_error_inner", since = "1.3.0")] + #[must_use = "`self` will be dropped if the result is not used"] + #[inline] pub fn into_inner(self) -> Option> { - match self.repr { - Repr::Os(..) => None, - Repr::Simple(..) => None, - Repr::Custom(c) => Some(c.error), + match self.repr.into_data() { + ErrorData::Os(..) => None, + ErrorData::Simple(..) => None, + ErrorData::SimpleMessage(..) => None, + ErrorData::Custom(c) => Some(c.error), + } + } + + /// Attempt to downgrade the inner error to `E` if any. + /// + /// If this [`Error`] was constructed via [`new`] then this function will + /// attempt to perform downgrade on it, otherwise it will return [`Err`]. + /// + /// If downgrade succeeds, it will return [`Ok`], otherwise it will also + /// return [`Err`]. + /// + /// [`new`]: Error::new + /// + /// # Examples + /// + /// ``` + /// #![feature(io_error_downcast)] + /// + /// use std::fmt; + /// use std::io; + /// use std::error::Error; + /// + /// #[derive(Debug)] + /// enum E { + /// Io(io::Error), + /// SomeOtherVariant, + /// } + /// + /// impl fmt::Display for E { + /// // ... + /// # fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// # todo!() + /// # } + /// } + /// impl Error for E {} + /// + /// impl From for E { + /// fn from(err: io::Error) -> E { + /// err.downcast::() + /// .map(|b| *b) + /// .unwrap_or_else(E::Io) + /// } + /// } + /// ``` + #[unstable(feature = "io_error_downcast", issue = "99262")] + pub fn downcast(self) -> result::Result, Self> + where + E: error::Error + Send + Sync + 'static, + { + match self.repr.into_data() { + ErrorData::Custom(b) if b.error.is::() => { + let res = (*b).error.downcast::(); + + // downcast is a really trivial and is marked as inline, so + // it's likely be inlined here. + // + // And the compiler should be able to eliminate the branch + // that produces `Err` here since b.error.is::() + // returns true. + Ok(res.unwrap()) + } + repr_data => Err(Self { repr: Repr::new(repr_data) }), } } - /// Returns the corresponding `ErrorKind` for this error. + /// Returns the corresponding [`ErrorKind`] for this error. /// /// # Examples /// @@ -487,33 +881,41 @@ impl Error { /// } /// /// fn main() { - /// // Will print "No inner error". + /// // Will print "Uncategorized". /// print_error(Error::last_os_error()); - /// // Will print "Inner error: ...". + /// // Will print "AddrInUse". /// print_error(Error::new(ErrorKind::AddrInUse, "oh no!")); /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + #[inline] pub fn kind(&self) -> ErrorKind { - match self.repr { - Repr::Os(code) => sys::decode_error_kind(code), - Repr::Custom(ref c) => c.kind, - Repr::Simple(kind) => kind, + match self.repr.data() { + ErrorData::Os(code) => sys::decode_error_kind(code), + ErrorData::Custom(c) => c.kind, + ErrorData::Simple(kind) => kind, + ErrorData::SimpleMessage(m) => m.kind, } } } impl fmt::Debug for Repr { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - match *self { - Repr::Os(code) => fmt + match self.data() { + ErrorData::Os(code) => fmt .debug_struct("Os") .field("code", &code) .field("kind", &sys::decode_error_kind(code)) .field("message", &sys::os::error_string(code)) .finish(), - Repr::Custom(ref c) => fmt::Debug::fmt(&c, fmt), - Repr::Simple(kind) => fmt.debug_tuple("Kind").field(&kind).finish(), + ErrorData::Custom(c) => fmt::Debug::fmt(&c, fmt), + ErrorData::Simple(kind) => fmt.debug_tuple("Kind").field(&kind).finish(), + ErrorData::SimpleMessage(msg) => fmt + .debug_struct("Error") + .field("kind", &msg.kind) + .field("message", &msg.message) + .finish(), } } } @@ -521,13 +923,14 @@ impl fmt::Debug for Repr { #[stable(feature = "rust1", since = "1.0.0")] impl fmt::Display for Error { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.repr { - Repr::Os(code) => { + match self.repr.data() { + ErrorData::Os(code) => { let detail = sys::os::error_string(code); - write!(fmt, "{} (os error {})", detail, code) + write!(fmt, "{detail} (os error {code})") } - Repr::Custom(ref c) => c.error.fmt(fmt), - Repr::Simple(kind) => write!(fmt, "{}", kind.as_str()), + ErrorData::Custom(ref c) => c.error.fmt(fmt), + ErrorData::Simple(kind) => write!(fmt, "{}", kind.as_str()), + ErrorData::SimpleMessage(msg) => msg.message.fmt(fmt), } } } @@ -536,26 +939,29 @@ impl fmt::Display for Error { impl error::Error for Error { #[allow(deprecated, deprecated_in_future)] fn description(&self) -> &str { - match self.repr { - Repr::Os(..) | Repr::Simple(..) => self.kind().as_str(), - Repr::Custom(ref c) => c.error.description(), + match self.repr.data() { + ErrorData::Os(..) | ErrorData::Simple(..) => self.kind().as_str(), + ErrorData::SimpleMessage(msg) => msg.message, + ErrorData::Custom(c) => c.error.description(), } } #[allow(deprecated)] fn cause(&self) -> Option<&dyn error::Error> { - match self.repr { - Repr::Os(..) => None, - Repr::Simple(..) => None, - Repr::Custom(ref c) => c.error.cause(), + match self.repr.data() { + ErrorData::Os(..) => None, + ErrorData::Simple(..) => None, + ErrorData::SimpleMessage(..) => None, + ErrorData::Custom(c) => c.error.cause(), } } fn source(&self) -> Option<&(dyn error::Error + 'static)> { - match self.repr { - Repr::Os(..) => None, - Repr::Simple(..) => None, - Repr::Custom(ref c) => c.error.source(), + match self.repr.data() { + ErrorData::Os(..) => None, + ErrorData::Simple(..) => None, + ErrorData::SimpleMessage(..) => None, + ErrorData::Custom(c) => c.error.source(), } } } @@ -564,60 +970,3 @@ fn _assert_error_is_sync_send() { fn _is_sync_send() {} _is_sync_send::(); } - -#[cfg(test)] -mod test { - use super::{Custom, Error, ErrorKind, Repr}; - use crate::error; - use crate::fmt; - use crate::sys::decode_error_kind; - use crate::sys::os::error_string; - - #[test] - fn test_debug_error() { - let code = 6; - let msg = error_string(code); - let kind = decode_error_kind(code); - let err = Error { - repr: Repr::Custom(box Custom { - kind: ErrorKind::InvalidInput, - error: box Error { repr: super::Repr::Os(code) }, - }), - }; - let expected = format!( - "Custom {{ \ - kind: InvalidInput, \ - error: Os {{ \ - code: {:?}, \ - kind: {:?}, \ - message: {:?} \ - }} \ - }}", - code, kind, msg - ); - assert_eq!(format!("{:?}", err), expected); - } - - #[test] - fn test_downcasting() { - #[derive(Debug)] - struct TestError; - - impl fmt::Display for TestError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("asdf") - } - } - - impl error::Error for TestError {} - - // we have to call all of these UFCS style right now since method - // resolution won't implicitly drop the Send+Sync bounds - let mut err = Error::new(ErrorKind::Other, TestError); - assert!(err.get_ref().unwrap().is::()); - assert_eq!("asdf", err.get_ref().unwrap().to_string()); - assert!(err.get_mut().unwrap().is::()); - let extracted = err.into_inner().unwrap(); - extracted.downcast::().unwrap(); - } -} diff --git a/crux-mir/lib/std/src/io/error/repr_bitpacked.rs b/crux-mir/lib/std/src/io/error/repr_bitpacked.rs new file mode 100644 index 000000000..358148405 --- /dev/null +++ b/crux-mir/lib/std/src/io/error/repr_bitpacked.rs @@ -0,0 +1,409 @@ +//! This is a densely packed error representation which is used on targets with +//! 64-bit pointers. +//! +//! (Note that `bitpacked` vs `unpacked` here has no relationship to +//! `#[repr(packed)]`, it just refers to attempting to use any available bits in +//! a more clever manner than `rustc`'s default layout algorithm would). +//! +//! Conceptually, it stores the same data as the "unpacked" equivalent we use on +//! other targets. Specifically, you can imagine it as an optimized version of +//! the following enum (which is roughly equivalent to what's stored by +//! `repr_unpacked::Repr`, e.g. `super::ErrorData>`): +//! +//! ```ignore (exposition-only) +//! enum ErrorData { +//! Os(i32), +//! Simple(ErrorKind), +//! SimpleMessage(&'static SimpleMessage), +//! Custom(Box), +//! } +//! ``` +//! +//! However, it packs this data into a 64bit non-zero value. +//! +//! This optimization not only allows `io::Error` to occupy a single pointer, +//! but improves `io::Result` as well, especially for situations like +//! `io::Result<()>` (which is now 64 bits) or `io::Result` (which is now +//! 128 bits), which are quite common. +//! +//! # Layout +//! Tagged values are 64 bits, with the 2 least significant bits used for the +//! tag. This means there are there are 4 "variants": +//! +//! - **Tag 0b00**: The first variant is equivalent to +//! `ErrorData::SimpleMessage`, and holds a `&'static SimpleMessage` directly. +//! +//! `SimpleMessage` has an alignment >= 4 (which is requested with +//! `#[repr(align)]` and checked statically at the bottom of this file), which +//! means every `&'static SimpleMessage` should have the both tag bits as 0, +//! meaning its tagged and untagged representation are equivalent. +//! +//! This means we can skip tagging it, which is necessary as this variant can +//! be constructed from a `const fn`, which probably cannot tag pointers (or +//! at least it would be difficult). +//! +//! - **Tag 0b01**: The other pointer variant holds the data for +//! `ErrorData::Custom` and the remaining 62 bits are used to store a +//! `Box`. `Custom` also has alignment >= 4, so the bottom two bits +//! are free to use for the tag. +//! +//! The only important thing to note is that `ptr::wrapping_add` and +//! `ptr::wrapping_sub` are used to tag the pointer, rather than bitwise +//! operations. This should preserve the pointer's provenance, which would +//! otherwise be lost. +//! +//! - **Tag 0b10**: Holds the data for `ErrorData::Os(i32)`. We store the `i32` +//! in the pointer's most significant 32 bits, and don't use the bits `2..32` +//! for anything. Using the top 32 bits is just to let us easily recover the +//! `i32` code with the correct sign. +//! +//! - **Tag 0b11**: Holds the data for `ErrorData::Simple(ErrorKind)`. This +//! stores the `ErrorKind` in the top 32 bits as well, although it doesn't +//! occupy nearly that many. Most of the bits are unused here, but it's not +//! like we need them for anything else yet. +//! +//! # Use of `NonNull<()>` +//! +//! Everything is stored in a `NonNull<()>`, which is odd, but actually serves a +//! purpose. +//! +//! Conceptually you might think of this more like: +//! +//! ```ignore (exposition-only) +//! union Repr { +//! // holds integer (Simple/Os) variants, and +//! // provides access to the tag bits. +//! bits: NonZeroU64, +//! // Tag is 0, so this is stored untagged. +//! msg: &'static SimpleMessage, +//! // Tagged (offset) `Box` pointer. +//! tagged_custom: NonNull<()>, +//! } +//! ``` +//! +//! But there are a few problems with this: +//! +//! 1. Union access is equivalent to a transmute, so this representation would +//! require we transmute between integers and pointers in at least one +//! direction, which may be UB (and even if not, it is likely harder for a +//! compiler to reason about than explicit ptr->int operations). +//! +//! 2. Even if all fields of a union have a niche, the union itself doesn't, +//! although this may change in the future. This would make things like +//! `io::Result<()>` and `io::Result` larger, which defeats part of +//! the motivation of this bitpacking. +//! +//! Storing everything in a `NonZeroUsize` (or some other integer) would be a +//! bit more traditional for pointer tagging, but it would lose provenance +//! information, couldn't be constructed from a `const fn`, and would probably +//! run into other issues as well. +//! +//! The `NonNull<()>` seems like the only alternative, even if it's fairly odd +//! to use a pointer type to store something that may hold an integer, some of +//! the time. + +use super::{Custom, ErrorData, ErrorKind, SimpleMessage}; +use alloc::boxed::Box; +use core::marker::PhantomData; +use core::mem::{align_of, size_of}; +use core::ptr::{self, NonNull}; + +// The 2 least-significant bits are used as tag. +const TAG_MASK: usize = 0b11; +const TAG_SIMPLE_MESSAGE: usize = 0b00; +const TAG_CUSTOM: usize = 0b01; +const TAG_OS: usize = 0b10; +const TAG_SIMPLE: usize = 0b11; + +/// The internal representation. +/// +/// See the module docs for more, this is just a way to hack in a check that we +/// indeed are not unwind-safe. +/// +/// ```compile_fail,E0277 +/// fn is_unwind_safe() {} +/// is_unwind_safe::(); +/// ``` +#[repr(transparent)] +pub(super) struct Repr(NonNull<()>, PhantomData>>); + +// All the types `Repr` stores internally are Send + Sync, and so is it. +unsafe impl Send for Repr {} +unsafe impl Sync for Repr {} + +impl Repr { + pub(super) fn new(dat: ErrorData>) -> Self { + match dat { + ErrorData::Os(code) => Self::new_os(code), + ErrorData::Simple(kind) => Self::new_simple(kind), + ErrorData::SimpleMessage(simple_message) => Self::new_simple_message(simple_message), + ErrorData::Custom(b) => Self::new_custom(b), + } + } + + pub(super) fn new_custom(b: Box) -> Self { + let p = Box::into_raw(b).cast::(); + // Should only be possible if an allocator handed out a pointer with + // wrong alignment. + debug_assert_eq!(p.addr() & TAG_MASK, 0); + // Note: We know `TAG_CUSTOM <= size_of::()` (static_assert at + // end of file), and both the start and end of the expression must be + // valid without address space wraparound due to `Box`'s semantics. + // + // This means it would be correct to implement this using `ptr::add` + // (rather than `ptr::wrapping_add`), but it's unclear this would give + // any benefit, so we just use `wrapping_add` instead. + let tagged = p.wrapping_add(TAG_CUSTOM).cast::<()>(); + // Safety: `TAG_CUSTOM + p` is the same as `TAG_CUSTOM | p`, + // because `p`'s alignment means it isn't allowed to have any of the + // `TAG_BITS` set (you can verify that addition and bitwise-or are the + // same when the operands have no bits in common using a truth table). + // + // Then, `TAG_CUSTOM | p` is not zero, as that would require + // `TAG_CUSTOM` and `p` both be zero, and neither is (as `p` came from a + // box, and `TAG_CUSTOM` just... isn't zero -- it's `0b01`). Therefore, + // `TAG_CUSTOM + p` isn't zero and so `tagged` can't be, and the + // `new_unchecked` is safe. + let res = Self(unsafe { NonNull::new_unchecked(tagged) }, PhantomData); + // quickly smoke-check we encoded the right thing (This generally will + // only run in std's tests, unless the user uses -Zbuild-std) + debug_assert!(matches!(res.data(), ErrorData::Custom(_)), "repr(custom) encoding failed"); + res + } + + #[inline] + pub(super) fn new_os(code: i32) -> Self { + let utagged = ((code as usize) << 32) | TAG_OS; + // Safety: `TAG_OS` is not zero, so the result of the `|` is not 0. + let res = Self(unsafe { NonNull::new_unchecked(ptr::invalid_mut(utagged)) }, PhantomData); + // quickly smoke-check we encoded the right thing (This generally will + // only run in std's tests, unless the user uses -Zbuild-std) + debug_assert!( + matches!(res.data(), ErrorData::Os(c) if c == code), + "repr(os) encoding failed for {code}" + ); + res + } + + #[inline] + pub(super) fn new_simple(kind: ErrorKind) -> Self { + let utagged = ((kind as usize) << 32) | TAG_SIMPLE; + // Safety: `TAG_SIMPLE` is not zero, so the result of the `|` is not 0. + let res = Self(unsafe { NonNull::new_unchecked(ptr::invalid_mut(utagged)) }, PhantomData); + // quickly smoke-check we encoded the right thing (This generally will + // only run in std's tests, unless the user uses -Zbuild-std) + debug_assert!( + matches!(res.data(), ErrorData::Simple(k) if k == kind), + "repr(simple) encoding failed {:?}", + kind, + ); + res + } + + #[inline] + pub(super) const fn new_simple_message(m: &'static SimpleMessage) -> Self { + // Safety: References are never null. + Self(unsafe { NonNull::new_unchecked(m as *const _ as *mut ()) }, PhantomData) + } + + #[inline] + pub(super) fn data(&self) -> ErrorData<&Custom> { + // Safety: We're a Repr, decode_repr is fine. + unsafe { decode_repr(self.0, |c| &*c) } + } + + #[inline] + pub(super) fn data_mut(&mut self) -> ErrorData<&mut Custom> { + // Safety: We're a Repr, decode_repr is fine. + unsafe { decode_repr(self.0, |c| &mut *c) } + } + + #[inline] + pub(super) fn into_data(self) -> ErrorData> { + let this = core::mem::ManuallyDrop::new(self); + // Safety: We're a Repr, decode_repr is fine. The `Box::from_raw` is + // safe because we prevent double-drop using `ManuallyDrop`. + unsafe { decode_repr(this.0, |p| Box::from_raw(p)) } + } +} + +impl Drop for Repr { + #[inline] + fn drop(&mut self) { + // Safety: We're a Repr, decode_repr is fine. The `Box::from_raw` is + // safe because we're being dropped. + unsafe { + let _ = decode_repr(self.0, |p| Box::::from_raw(p)); + } + } +} + +// Shared helper to decode a `Repr`'s internal pointer into an ErrorData. +// +// Safety: `ptr`'s bits should be encoded as described in the document at the +// top (it should `some_repr.0`) +#[inline] +unsafe fn decode_repr(ptr: NonNull<()>, make_custom: F) -> ErrorData +where + F: FnOnce(*mut Custom) -> C, +{ + let bits = ptr.as_ptr().addr(); + match bits & TAG_MASK { + TAG_OS => { + let code = ((bits as i64) >> 32) as i32; + ErrorData::Os(code) + } + TAG_SIMPLE => { + let kind_bits = (bits >> 32) as u32; + let kind = kind_from_prim(kind_bits).unwrap_or_else(|| { + debug_assert!(false, "Invalid io::error::Repr bits: `Repr({:#018x})`", bits); + // This means the `ptr` passed in was not valid, which violates + // the unsafe contract of `decode_repr`. + // + // Using this rather than unwrap meaningfully improves the code + // for callers which only care about one variant (usually + // `Custom`) + core::hint::unreachable_unchecked(); + }); + ErrorData::Simple(kind) + } + TAG_SIMPLE_MESSAGE => ErrorData::SimpleMessage(&*ptr.cast::().as_ptr()), + TAG_CUSTOM => { + // It would be correct for us to use `ptr::byte_sub` here (see the + // comment above the `wrapping_add` call in `new_custom` for why), + // but it isn't clear that it makes a difference, so we don't. + let custom = ptr.as_ptr().wrapping_byte_sub(TAG_CUSTOM).cast::(); + ErrorData::Custom(make_custom(custom)) + } + _ => { + // Can't happen, and compiler can tell + unreachable!(); + } + } +} + +// This compiles to the same code as the check+transmute, but doesn't require +// unsafe, or to hard-code max ErrorKind or its size in a way the compiler +// couldn't verify. +#[inline] +fn kind_from_prim(ek: u32) -> Option { + macro_rules! from_prim { + ($prim:expr => $Enum:ident { $($Variant:ident),* $(,)? }) => {{ + // Force a compile error if the list gets out of date. + const _: fn(e: $Enum) = |e: $Enum| match e { + $($Enum::$Variant => ()),* + }; + match $prim { + $(v if v == ($Enum::$Variant as _) => Some($Enum::$Variant),)* + _ => None, + } + }} + } + from_prim!(ek => ErrorKind { + NotFound, + PermissionDenied, + ConnectionRefused, + ConnectionReset, + HostUnreachable, + NetworkUnreachable, + ConnectionAborted, + NotConnected, + AddrInUse, + AddrNotAvailable, + NetworkDown, + BrokenPipe, + AlreadyExists, + WouldBlock, + NotADirectory, + IsADirectory, + DirectoryNotEmpty, + ReadOnlyFilesystem, + FilesystemLoop, + StaleNetworkFileHandle, + InvalidInput, + InvalidData, + TimedOut, + WriteZero, + StorageFull, + NotSeekable, + FilesystemQuotaExceeded, + FileTooLarge, + ResourceBusy, + ExecutableFileBusy, + Deadlock, + CrossesDevices, + TooManyLinks, + InvalidFilename, + ArgumentListTooLong, + Interrupted, + Other, + UnexpectedEof, + Unsupported, + OutOfMemory, + Uncategorized, + }) +} + +// Some static checking to alert us if a change breaks any of the assumptions +// that our encoding relies on for correctness and soundness. (Some of these are +// a bit overly thorough/cautious, admittedly) +// +// If any of these are hit on a platform that std supports, we should likely +// just use `repr_unpacked.rs` there instead (unless the fix is easy). +macro_rules! static_assert { + ($condition:expr) => { + const _: () = assert!($condition); + }; + (@usize_eq: $lhs:expr, $rhs:expr) => { + const _: [(); $lhs] = [(); $rhs]; + }; +} + +// The bitpacking we use requires pointers be exactly 64 bits. +static_assert!(@usize_eq: size_of::>(), 8); + +// We also require pointers and usize be the same size. +static_assert!(@usize_eq: size_of::>(), size_of::()); + +// `Custom` and `SimpleMessage` need to be thin pointers. +static_assert!(@usize_eq: size_of::<&'static SimpleMessage>(), 8); +static_assert!(@usize_eq: size_of::>(), 8); + +static_assert!((TAG_MASK + 1).is_power_of_two()); +// And they must have sufficient alignment. +static_assert!(align_of::() >= TAG_MASK + 1); +static_assert!(align_of::() >= TAG_MASK + 1); + +static_assert!(@usize_eq: TAG_MASK & TAG_SIMPLE_MESSAGE, TAG_SIMPLE_MESSAGE); +static_assert!(@usize_eq: TAG_MASK & TAG_CUSTOM, TAG_CUSTOM); +static_assert!(@usize_eq: TAG_MASK & TAG_OS, TAG_OS); +static_assert!(@usize_eq: TAG_MASK & TAG_SIMPLE, TAG_SIMPLE); + +// This is obviously true (`TAG_CUSTOM` is `0b01`), but in `Repr::new_custom` we +// offset a pointer by this value, and expect it to both be within the same +// object, and to not wrap around the address space. See the comment in that +// function for further details. +// +// Actually, at the moment we use `ptr::wrapping_add`, not `ptr::add`, so this +// check isn't needed for that one, although the assertion that we don't +// actually wrap around in that wrapping_add does simplify the safety reasoning +// elsewhere considerably. +static_assert!(size_of::() >= TAG_CUSTOM); + +// These two store a payload which is allowed to be zero, so they must be +// non-zero to preserve the `NonNull`'s range invariant. +static_assert!(TAG_OS != 0); +static_assert!(TAG_SIMPLE != 0); +// We can't tag `SimpleMessage`s, the tag must be 0. +static_assert!(@usize_eq: TAG_SIMPLE_MESSAGE, 0); + +// Check that the point of all of this still holds. +// +// We'd check against `io::Error`, but *technically* it's allowed to vary, +// as it's not `#[repr(transparent)]`/`#[repr(C)]`. We could add that, but +// the `#[repr()]` would show up in rustdoc, which might be seen as a stable +// commitment. +static_assert!(@usize_eq: size_of::(), 8); +static_assert!(@usize_eq: size_of::>(), 8); +static_assert!(@usize_eq: size_of::>(), 8); +static_assert!(@usize_eq: size_of::>(), 16); diff --git a/crux-mir/lib/std/src/io/error/repr_unpacked.rs b/crux-mir/lib/std/src/io/error/repr_unpacked.rs new file mode 100644 index 000000000..d6ad55b99 --- /dev/null +++ b/crux-mir/lib/std/src/io/error/repr_unpacked.rs @@ -0,0 +1,54 @@ +//! This is a fairly simple unpacked error representation that's used on +//! non-64bit targets, where the packed 64 bit representation wouldn't work, and +//! would have no benefit. + +use super::{Custom, ErrorData, ErrorKind, SimpleMessage}; +use alloc::boxed::Box; + +type Inner = ErrorData>; + +pub(super) struct Repr(Inner); + +impl Repr { + #[inline] + pub(super) fn new(dat: ErrorData>) -> Self { + Self(dat) + } + pub(super) fn new_custom(b: Box) -> Self { + Self(Inner::Custom(b)) + } + #[inline] + pub(super) fn new_os(code: i32) -> Self { + Self(Inner::Os(code)) + } + #[inline] + pub(super) fn new_simple(kind: ErrorKind) -> Self { + Self(Inner::Simple(kind)) + } + #[inline] + pub(super) const fn new_simple_message(m: &'static SimpleMessage) -> Self { + Self(Inner::SimpleMessage(m)) + } + #[inline] + pub(super) fn into_data(self) -> ErrorData> { + self.0 + } + #[inline] + pub(super) fn data(&self) -> ErrorData<&Custom> { + match &self.0 { + Inner::Os(c) => ErrorData::Os(*c), + Inner::Simple(k) => ErrorData::Simple(*k), + Inner::SimpleMessage(m) => ErrorData::SimpleMessage(*m), + Inner::Custom(m) => ErrorData::Custom(&*m), + } + } + #[inline] + pub(super) fn data_mut(&mut self) -> ErrorData<&mut Custom> { + match &mut self.0 { + Inner::Os(c) => ErrorData::Os(*c), + Inner::Simple(k) => ErrorData::Simple(*k), + Inner::SimpleMessage(m) => ErrorData::SimpleMessage(*m), + Inner::Custom(m) => ErrorData::Custom(&mut *m), + } + } +} diff --git a/crux-mir/lib/std/src/io/error/tests.rs b/crux-mir/lib/std/src/io/error/tests.rs new file mode 100644 index 000000000..9aea62a5b --- /dev/null +++ b/crux-mir/lib/std/src/io/error/tests.rs @@ -0,0 +1,194 @@ +use super::{const_io_error, Custom, Error, ErrorData, ErrorKind, Repr, SimpleMessage}; +use crate::assert_matches::assert_matches; +use crate::error; +use crate::fmt; +use crate::mem::size_of; +use crate::sys::decode_error_kind; +use crate::sys::os::error_string; + +#[test] +fn test_size() { + assert!(size_of::() <= size_of::<[usize; 2]>()); +} + +#[test] +fn test_debug_error() { + let code = 6; + let msg = error_string(code); + let kind = decode_error_kind(code); + let err = Error { + repr: Repr::new_custom(Box::new(Custom { + kind: ErrorKind::InvalidInput, + error: Box::new(Error { repr: super::Repr::new_os(code) }), + })), + }; + let expected = format!( + "Custom {{ \ + kind: InvalidInput, \ + error: Os {{ \ + code: {:?}, \ + kind: {:?}, \ + message: {:?} \ + }} \ + }}", + code, kind, msg + ); + assert_eq!(format!("{err:?}"), expected); +} + +#[test] +fn test_downcasting() { + #[derive(Debug)] + struct TestError; + + impl fmt::Display for TestError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("asdf") + } + } + + impl error::Error for TestError {} + + // we have to call all of these UFCS style right now since method + // resolution won't implicitly drop the Send+Sync bounds + let mut err = Error::new(ErrorKind::Other, TestError); + assert!(err.get_ref().unwrap().is::()); + assert_eq!("asdf", err.get_ref().unwrap().to_string()); + assert!(err.get_mut().unwrap().is::()); + let extracted = err.into_inner().unwrap(); + extracted.downcast::().unwrap(); +} + +#[test] +fn test_const() { + const E: Error = const_io_error!(ErrorKind::NotFound, "hello"); + + assert_eq!(E.kind(), ErrorKind::NotFound); + assert_eq!(E.to_string(), "hello"); + assert!(format!("{E:?}").contains("\"hello\"")); + assert!(format!("{E:?}").contains("NotFound")); +} + +#[test] +fn test_os_packing() { + for code in -20i32..20i32 { + let e = Error::from_raw_os_error(code); + assert_eq!(e.raw_os_error(), Some(code)); + assert_matches!( + e.repr.data(), + ErrorData::Os(c) if c == code, + ); + } +} + +#[test] +fn test_errorkind_packing() { + assert_eq!(Error::from(ErrorKind::NotFound).kind(), ErrorKind::NotFound); + assert_eq!(Error::from(ErrorKind::PermissionDenied).kind(), ErrorKind::PermissionDenied); + assert_eq!(Error::from(ErrorKind::Uncategorized).kind(), ErrorKind::Uncategorized); + // Check that the innards look like what we want. + assert_matches!( + Error::from(ErrorKind::OutOfMemory).repr.data(), + ErrorData::Simple(ErrorKind::OutOfMemory), + ); +} + +#[test] +fn test_simple_message_packing() { + use super::{ErrorKind::*, SimpleMessage}; + macro_rules! check_simple_msg { + ($err:expr, $kind:ident, $msg:literal) => {{ + let e = &$err; + // Check that the public api is right. + assert_eq!(e.kind(), $kind); + assert!(format!("{e:?}").contains($msg)); + // and we got what we expected + assert_matches!( + e.repr.data(), + ErrorData::SimpleMessage(SimpleMessage { kind: $kind, message: $msg }) + ); + }}; + } + + let not_static = const_io_error!(Uncategorized, "not a constant!"); + check_simple_msg!(not_static, Uncategorized, "not a constant!"); + + const CONST: Error = const_io_error!(NotFound, "definitely a constant!"); + check_simple_msg!(CONST, NotFound, "definitely a constant!"); + + static STATIC: Error = const_io_error!(BrokenPipe, "a constant, sort of!"); + check_simple_msg!(STATIC, BrokenPipe, "a constant, sort of!"); +} + +#[derive(Debug, PartialEq)] +struct Bojji(bool); +impl error::Error for Bojji {} +impl fmt::Display for Bojji { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "ah! {:?}", self) + } +} + +#[test] +fn test_custom_error_packing() { + use super::Custom; + let test = Error::new(ErrorKind::Uncategorized, Bojji(true)); + assert_matches!( + test.repr.data(), + ErrorData::Custom(Custom { + kind: ErrorKind::Uncategorized, + error, + }) if error.downcast_ref::().as_deref() == Some(&Bojji(true)), + ); +} + +#[derive(Debug)] +struct E; + +impl fmt::Display for E { + fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { + Ok(()) + } +} + +impl error::Error for E {} + +#[test] +fn test_std_io_error_downcast() { + // Case 1: custom error, downcast succeeds + let io_error = Error::new(ErrorKind::Other, Bojji(true)); + let e: Box = io_error.downcast().unwrap(); + assert!(e.0); + + // Case 2: custom error, downcast fails + let io_error = Error::new(ErrorKind::Other, Bojji(true)); + let io_error = io_error.downcast::().unwrap_err(); + + // ensures that the custom error is intact + assert_eq!(ErrorKind::Other, io_error.kind()); + let e: Box = io_error.downcast().unwrap(); + assert!(e.0); + + // Case 3: os error + let errno = 20; + let io_error = Error::from_raw_os_error(errno); + let io_error = io_error.downcast::().unwrap_err(); + + assert_eq!(errno, io_error.raw_os_error().unwrap()); + + // Case 4: simple + let kind = ErrorKind::OutOfMemory; + let io_error: Error = kind.into(); + let io_error = io_error.downcast::().unwrap_err(); + + assert_eq!(kind, io_error.kind()); + + // Case 5: simple message + const SIMPLE_MESSAGE: SimpleMessage = + SimpleMessage { kind: ErrorKind::Other, message: "simple message error test" }; + let io_error = Error::from_static_message(&SIMPLE_MESSAGE); + let io_error = io_error.downcast::().unwrap_err(); + + assert_eq!(SIMPLE_MESSAGE.kind, io_error.kind()); + assert_eq!(SIMPLE_MESSAGE.message, format!("{io_error}")); +} diff --git a/crux-mir/lib/std/src/io/impls.rs b/crux-mir/lib/std/src/io/impls.rs index b7f82e652..e5048dcc8 100644 --- a/crux-mir/lib/std/src/io/impls.rs +++ b/crux-mir/lib/std/src/io/impls.rs @@ -1,7 +1,12 @@ +#[cfg(test)] +mod tests; + +use crate::alloc::Allocator; use crate::cmp; +use crate::collections::VecDeque; use crate::fmt; use crate::io::{ - self, BufRead, Error, ErrorKind, Initializer, IoSlice, IoSliceMut, Read, Seek, SeekFrom, Write, + self, BorrowedCursor, BufRead, ErrorKind, IoSlice, IoSliceMut, Read, Seek, SeekFrom, Write, }; use crate::mem; @@ -15,14 +20,19 @@ impl Read for &mut R { (**self).read(buf) } + #[inline] + fn read_buf(&mut self, cursor: BorrowedCursor<'_>) -> io::Result<()> { + (**self).read_buf(cursor) + } + #[inline] fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { (**self).read_vectored(bufs) } #[inline] - unsafe fn initializer(&self) -> Initializer { - (**self).initializer() + fn is_read_vectored(&self) -> bool { + (**self).is_read_vectored() } #[inline] @@ -52,6 +62,11 @@ impl Write for &mut W { (**self).write_vectored(bufs) } + #[inline] + fn is_write_vectored(&self) -> bool { + (**self).is_write_vectored() + } + #[inline] fn flush(&mut self) -> io::Result<()> { (**self).flush() @@ -73,6 +88,11 @@ impl Seek for &mut S { fn seek(&mut self, pos: SeekFrom) -> io::Result { (**self).seek(pos) } + + #[inline] + fn stream_position(&mut self) -> io::Result { + (**self).stream_position() + } } #[stable(feature = "rust1", since = "1.0.0")] impl BufRead for &mut B { @@ -104,14 +124,19 @@ impl Read for Box { (**self).read(buf) } + #[inline] + fn read_buf(&mut self, cursor: BorrowedCursor<'_>) -> io::Result<()> { + (**self).read_buf(cursor) + } + #[inline] fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { (**self).read_vectored(bufs) } #[inline] - unsafe fn initializer(&self) -> Initializer { - (**self).initializer() + fn is_read_vectored(&self) -> bool { + (**self).is_read_vectored() } #[inline] @@ -141,6 +166,11 @@ impl Write for Box { (**self).write_vectored(bufs) } + #[inline] + fn is_write_vectored(&self) -> bool { + (**self).is_write_vectored() + } + #[inline] fn flush(&mut self) -> io::Result<()> { (**self).flush() @@ -162,6 +192,11 @@ impl Seek for Box { fn seek(&mut self, pos: SeekFrom) -> io::Result { (**self).seek(pos) } + + #[inline] + fn stream_position(&mut self) -> io::Result { + (**self).stream_position() + } } #[stable(feature = "rust1", since = "1.0.0")] impl BufRead for Box { @@ -186,20 +221,6 @@ impl BufRead for Box { } } -// Used by panicking::default_hook -#[cfg(test)] -/// This impl is only used by printing logic, so any error returned is always -/// of kind `Other`, and should be ignored. -impl Write for Box { - fn write(&mut self, buf: &[u8]) -> io::Result { - (**self).write(buf).map_err(|_| ErrorKind::Other.into()) - } - - fn flush(&mut self) -> io::Result<()> { - (**self).flush().map_err(|_| ErrorKind::Other.into()) - } -} - // ============================================================================= // In-memory buffer implementations @@ -227,6 +248,17 @@ impl Read for &[u8] { Ok(amt) } + #[inline] + fn read_buf(&mut self, mut cursor: BorrowedCursor<'_>) -> io::Result<()> { + let amt = cmp::min(cursor.capacity(), self.len()); + let (a, b) = self.split_at(amt); + + cursor.append(a); + + *self = b; + Ok(()) + } + #[inline] fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { let mut nread = 0; @@ -241,14 +273,17 @@ impl Read for &[u8] { } #[inline] - unsafe fn initializer(&self) -> Initializer { - Initializer::nop() + fn is_read_vectored(&self) -> bool { + true } #[inline] fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> { if buf.len() > self.len() { - return Err(Error::new(ErrorKind::UnexpectedEof, "failed to fill whole buffer")); + return Err(io::const_io_error!( + ErrorKind::UnexpectedEof, + "failed to fill whole buffer" + )); } let (a, b) = self.split_at(buf.len()); @@ -292,6 +327,10 @@ impl BufRead for &[u8] { /// /// Note that writing updates the slice to point to the yet unwritten part. /// The slice will be empty when it has been completely overwritten. +/// +/// If the number of bytes to be written exceeds the size of the slice, write operations will +/// return short writes: ultimately, `Ok(0)`; in this situation, `write_all` returns an error of +/// kind `ErrorKind::WriteZero`. #[stable(feature = "rust1", since = "1.0.0")] impl Write for &mut [u8] { #[inline] @@ -316,12 +355,17 @@ impl Write for &mut [u8] { Ok(nwritten) } + #[inline] + fn is_write_vectored(&self) -> bool { + true + } + #[inline] fn write_all(&mut self, data: &[u8]) -> io::Result<()> { if self.write(data)? == data.len() { Ok(()) } else { - Err(Error::new(ErrorKind::WriteZero, "failed to write whole buffer")) + Err(io::const_io_error!(ErrorKind::WriteZero, "failed to write whole buffer")) } } @@ -334,7 +378,7 @@ impl Write for &mut [u8] { /// Write is implemented for `Vec` by appending to the vector. /// The vector will grow as needed. #[stable(feature = "rust1", since = "1.0.0")] -impl Write for Vec { +impl Write for Vec { #[inline] fn write(&mut self, buf: &[u8]) -> io::Result { self.extend_from_slice(buf); @@ -351,6 +395,11 @@ impl Write for Vec { Ok(len) } + #[inline] + fn is_write_vectored(&self) -> bool { + true + } + #[inline] fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { self.extend_from_slice(buf); @@ -363,63 +412,47 @@ impl Write for Vec { } } -#[cfg(test)] -mod tests { - use crate::io::prelude::*; - - #[bench] - fn bench_read_slice(b: &mut test::Bencher) { - let buf = [5; 1024]; - let mut dst = [0; 128]; - - b.iter(|| { - let mut rd = &buf[..]; - for _ in 0..8 { - let _ = rd.read(&mut dst); - test::black_box(&dst); - } - }) +/// Read is implemented for `VecDeque` by consuming bytes from the front of the `VecDeque`. +#[stable(feature = "vecdeque_read_write", since = "1.63.0")] +impl Read for VecDeque { + /// Fill `buf` with the contents of the "front" slice as returned by + /// [`as_slices`][`VecDeque::as_slices`]. If the contained byte slices of the `VecDeque` are + /// discontiguous, multiple calls to `read` will be needed to read the entire content. + #[inline] + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let (ref mut front, _) = self.as_slices(); + let n = Read::read(front, buf)?; + self.drain(..n); + Ok(n) } - #[bench] - fn bench_write_slice(b: &mut test::Bencher) { - let mut buf = [0; 1024]; - let src = [5; 128]; - - b.iter(|| { - let mut wr = &mut buf[..]; - for _ in 0..8 { - let _ = wr.write_all(&src); - test::black_box(&wr); - } - }) + #[inline] + fn read_buf(&mut self, cursor: BorrowedCursor<'_>) -> io::Result<()> { + let (ref mut front, _) = self.as_slices(); + let n = cmp::min(cursor.capacity(), front.len()); + Read::read_buf(front, cursor)?; + self.drain(..n); + Ok(()) } +} - #[bench] - fn bench_read_vec(b: &mut test::Bencher) { - let buf = vec![5; 1024]; - let mut dst = [0; 128]; - - b.iter(|| { - let mut rd = &buf[..]; - for _ in 0..8 { - let _ = rd.read(&mut dst); - test::black_box(&dst); - } - }) +/// Write is implemented for `VecDeque` by appending to the `VecDeque`, growing it as needed. +#[stable(feature = "vecdeque_read_write", since = "1.63.0")] +impl Write for VecDeque { + #[inline] + fn write(&mut self, buf: &[u8]) -> io::Result { + self.extend(buf); + Ok(buf.len()) } - #[bench] - fn bench_write_vec(b: &mut test::Bencher) { - let mut buf = Vec::with_capacity(1024); - let src = [5; 128]; + #[inline] + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + self.extend(buf); + Ok(()) + } - b.iter(|| { - let mut wr = &mut buf[..]; - for _ in 0..8 { - let _ = wr.write_all(&src); - test::black_box(&wr); - } - }) + #[inline] + fn flush(&mut self) -> io::Result<()> { + Ok(()) } } diff --git a/crux-mir/lib/std/src/io/impls/tests.rs b/crux-mir/lib/std/src/io/impls/tests.rs new file mode 100644 index 000000000..d1cd84a67 --- /dev/null +++ b/crux-mir/lib/std/src/io/impls/tests.rs @@ -0,0 +1,57 @@ +use crate::io::prelude::*; + +#[bench] +fn bench_read_slice(b: &mut test::Bencher) { + let buf = [5; 1024]; + let mut dst = [0; 128]; + + b.iter(|| { + let mut rd = &buf[..]; + for _ in 0..8 { + let _ = rd.read(&mut dst); + test::black_box(&dst); + } + }) +} + +#[bench] +fn bench_write_slice(b: &mut test::Bencher) { + let mut buf = [0; 1024]; + let src = [5; 128]; + + b.iter(|| { + let mut wr = &mut buf[..]; + for _ in 0..8 { + let _ = wr.write_all(&src); + test::black_box(&wr); + } + }) +} + +#[bench] +fn bench_read_vec(b: &mut test::Bencher) { + let buf = vec![5; 1024]; + let mut dst = [0; 128]; + + b.iter(|| { + let mut rd = &buf[..]; + for _ in 0..8 { + let _ = rd.read(&mut dst); + test::black_box(&dst); + } + }) +} + +#[bench] +fn bench_write_vec(b: &mut test::Bencher) { + let mut buf = Vec::with_capacity(1024); + let src = [5; 128]; + + b.iter(|| { + let mut wr = &mut buf[..]; + for _ in 0..8 { + let _ = wr.write_all(&src); + test::black_box(&wr); + } + }) +} diff --git a/crux-mir/lib/std/src/io/lazy.rs b/crux-mir/lib/std/src/io/lazy.rs deleted file mode 100644 index 1968d498b..000000000 --- a/crux-mir/lib/std/src/io/lazy.rs +++ /dev/null @@ -1,63 +0,0 @@ -use crate::cell::Cell; -use crate::ptr; -use crate::sync::Arc; -use crate::sys_common; -use crate::sys_common::mutex::Mutex; - -pub struct Lazy { - // We never call `lock.init()`, so it is UB to attempt to acquire this mutex reentrantly! - lock: Mutex, - ptr: Cell<*mut Arc>, -} - -#[inline] -const fn done() -> *mut Arc { - 1_usize as *mut _ -} - -unsafe impl Sync for Lazy {} - -impl Lazy { - pub const fn new() -> Lazy { - Lazy { lock: Mutex::new(), ptr: Cell::new(ptr::null_mut()) } - } -} - -impl Lazy { - /// Safety: `init` must not call `get` on the variable that is being - /// initialized. - pub unsafe fn get(&'static self, init: fn() -> Arc) -> Option> { - let _guard = self.lock.lock(); - let ptr = self.ptr.get(); - if ptr.is_null() { - Some(self.init(init)) - } else if ptr == done() { - None - } else { - Some((*ptr).clone()) - } - } - - // Must only be called with `lock` held - unsafe fn init(&'static self, init: fn() -> Arc) -> Arc { - // If we successfully register an at exit handler, then we cache the - // `Arc` allocation in our own internal box (it will get deallocated by - // the at exit handler). Otherwise we just return the freshly allocated - // `Arc`. - let registered = sys_common::at_exit(move || { - let ptr = { - let _guard = self.lock.lock(); - self.ptr.replace(done()) - }; - drop(Box::from_raw(ptr)) - }); - // This could reentrantly call `init` again, which is a problem - // because our `lock` allows reentrancy! - // That's why `get` is unsafe and requires the caller to ensure no reentrancy happens. - let ret = init(); - if registered.is_ok() { - self.ptr.set(Box::into_raw(Box::new(ret.clone()))); - } - ret - } -} diff --git a/crux-mir/lib/std/src/io/mod.rs b/crux-mir/lib/std/src/io/mod.rs index dc831432c..de528e853 100644 --- a/crux-mir/lib/std/src/io/mod.rs +++ b/crux-mir/lib/std/src/io/mod.rs @@ -91,7 +91,7 @@ //! // read a line into buffer //! reader.read_line(&mut buffer)?; //! -//! println!("{}", buffer); +//! println!("{buffer}"); //! Ok(()) //! } //! ``` @@ -238,67 +238,66 @@ //! contract. The implementation of many of these functions are subject to change over //! time and may call fewer or more syscalls/library functions. //! -//! [`Read`]: trait.Read.html -//! [`Write`]: trait.Write.html -//! [`Seek`]: trait.Seek.html -//! [`BufRead`]: trait.BufRead.html -//! [`File`]: ../fs/struct.File.html -//! [`TcpStream`]: ../net/struct.TcpStream.html -//! [`Vec`]: ../vec/struct.Vec.html -//! [`BufReader`]: struct.BufReader.html -//! [`BufWriter`]: struct.BufWriter.html -//! [`Write::write`]: trait.Write.html#tymethod.write -//! [`io::stdout`]: fn.stdout.html -//! [`println!`]: ../macro.println.html -//! [`Lines`]: struct.Lines.html -//! [`io::Result`]: type.Result.html +//! [`File`]: crate::fs::File +//! [`TcpStream`]: crate::net::TcpStream +//! [`io::stdout`]: stdout +//! [`io::Result`]: self::Result //! [`?` operator]: ../../book/appendix-02-operators.html -//! [`Read::read`]: trait.Read.html#tymethod.read -//! [`Result`]: ../result/enum.Result.html -//! [`.unwrap()`]: ../result/enum.Result.html#method.unwrap +//! [`Result`]: crate::result::Result +//! [`.unwrap()`]: crate::result::Result::unwrap #![stable(feature = "rust1", since = "1.0.0")] +#[cfg(test)] +mod tests; + use crate::cmp; use crate::fmt; -use crate::memchr; +use crate::mem::replace; use crate::ops::{Deref, DerefMut}; -use crate::ptr; use crate::slice; use crate::str; use crate::sys; +use crate::sys_common::memchr; -#[stable(feature = "rust1", since = "1.0.0")] -pub use self::buffered::IntoInnerError; -#[stable(feature = "rust1", since = "1.0.0")] -pub use self::buffered::{BufReader, BufWriter, LineWriter}; -#[stable(feature = "rust1", since = "1.0.0")] -pub use self::cursor::Cursor; -#[stable(feature = "rust1", since = "1.0.0")] -pub use self::error::{Error, ErrorKind, Result}; -#[stable(feature = "rust1", since = "1.0.0")] -pub use self::stdio::{stderr, stdin, stdout, Stderr, Stdin, Stdout}; -#[stable(feature = "rust1", since = "1.0.0")] -pub use self::stdio::{StderrLock, StdinLock, StdoutLock}; +#[stable(feature = "bufwriter_into_parts", since = "1.56.0")] +pub use self::buffered::WriterPanicked; +pub(crate) use self::stdio::attempt_print_to_stderr; +#[unstable(feature = "internal_output_capture", issue = "none")] +#[doc(no_inline, hidden)] +pub use self::stdio::set_output_capture; +#[unstable(feature = "is_terminal", issue = "98070")] +pub use self::stdio::IsTerminal; #[unstable(feature = "print_internals", issue = "none")] pub use self::stdio::{_eprint, _print}; -#[unstable(feature = "libstd_io_internals", issue = "42788")] -#[doc(no_inline, hidden)] -pub use self::stdio::{set_panic, set_print}; #[stable(feature = "rust1", since = "1.0.0")] -pub use self::util::{copy, empty, repeat, sink, Empty, Repeat, Sink}; +pub use self::{ + buffered::{BufReader, BufWriter, IntoInnerError, LineWriter}, + copy::copy, + cursor::Cursor, + error::{Error, ErrorKind, Result}, + stdio::{stderr, stdin, stdout, Stderr, StderrLock, Stdin, StdinLock, Stdout, StdoutLock}, + util::{empty, repeat, sink, Empty, Repeat, Sink}, +}; + +#[unstable(feature = "read_buf", issue = "78485")] +pub use self::readbuf::{BorrowedBuf, BorrowedCursor}; +pub(crate) use error::const_io_error; mod buffered; +pub(crate) mod copy; mod cursor; mod error; mod impls; -mod lazy; pub mod prelude; +mod readbuf; mod stdio; mod util; const DEFAULT_BUF_SIZE: usize = crate::sys_common::io::DEFAULT_BUF_SIZE; +pub(crate) use stdio::cleanup; + struct Guard<'a> { buf: &'a mut Vec, len: usize, @@ -312,11 +311,12 @@ impl Drop for Guard<'_> { } } -// A few methods below (read_to_string, read_line) will append data into a -// `String` buffer, but we need to be pretty careful when doing this. The -// implementation will just call `.as_mut_vec()` and then delegate to a -// byte-oriented reading method, but we must ensure that when returning we never -// leave `buf` in a state such that it contains invalid UTF-8 in its bounds. +// Several `read_to_string` and `read_line` methods in the standard library will +// append data into a `String` buffer, but we need to be pretty careful when +// doing this. The implementation will just call `.as_mut_vec()` and then +// delegate to a byte-oriented reading method, but we must ensure that when +// returning we never leave `buf` in a state such that it contains invalid UTF-8 +// in its bounds. // // To this end, we use an RAII guard (to protect against panics) which updates // the length of the string when it is dropped. This guard initially truncates @@ -330,21 +330,22 @@ impl Drop for Guard<'_> { // 2. We're passing a raw buffer to the function `f`, and it is expected that // the function only *appends* bytes to the buffer. We'll get undefined // behavior if existing bytes are overwritten to have non-UTF-8 data. -fn append_to_string(buf: &mut String, f: F) -> Result +pub(crate) unsafe fn append_to_string(buf: &mut String, f: F) -> Result where F: FnOnce(&mut Vec) -> Result, { - unsafe { - let mut g = Guard { len: buf.len(), buf: buf.as_mut_vec() }; - let ret = f(g.buf); - if str::from_utf8(&g.buf[g.len..]).is_err() { - ret.and_then(|_| { - Err(Error::new(ErrorKind::InvalidData, "stream did not contain valid UTF-8")) - }) - } else { - g.len = g.buf.len(); - ret - } + let mut g = Guard { len: buf.len(), buf: buf.as_mut_vec() }; + let ret = f(g.buf); + if str::from_utf8(&g.buf[g.len..]).is_err() { + ret.and_then(|_| { + Err(error::const_io_error!( + ErrorKind::InvalidData, + "stream did not contain valid UTF-8" + )) + }) + } else { + g.len = g.buf.len(); + ret } } @@ -354,58 +355,79 @@ where // of data to return. Simply tacking on an extra DEFAULT_BUF_SIZE space every // time is 4,500 times (!) slower than a default reservation size of 32 if the // reader has a very small amount of data to return. -// -// Because we're extending the buffer with uninitialized data for trusted -// readers, we need to make sure to truncate that if any of this panics. -fn read_to_end(r: &mut R, buf: &mut Vec) -> Result { - read_to_end_with_reservation(r, buf, |_| 32) -} - -fn read_to_end_with_reservation( - r: &mut R, - buf: &mut Vec, - mut reservation_size: F, -) -> Result -where - R: Read + ?Sized, - F: FnMut(&R) -> usize, -{ +pub(crate) fn default_read_to_end(r: &mut R, buf: &mut Vec) -> Result { let start_len = buf.len(); - let mut g = Guard { len: buf.len(), buf }; - let ret; + let start_cap = buf.capacity(); + + let mut initialized = 0; // Extra initialized bytes from previous loop iteration loop { - if g.len == g.buf.len() { - unsafe { - // FIXME(danielhenrymantilla): #42788 - // - // - This creates a (mut) reference to a slice of - // _uninitialized_ integers, which is **undefined behavior** - // - // - Only the standard library gets to soundly "ignore" this, - // based on its privileged knowledge of unstable rustc - // internals; - g.buf.reserve(reservation_size(r)); - let capacity = g.buf.capacity(); - g.buf.set_len(capacity); - r.initializer().initialize(&mut g.buf[g.len..]); - } + if buf.len() == buf.capacity() { + buf.reserve(32); // buf is full, need more space } - match r.read(&mut g.buf[g.len..]) { - Ok(0) => { - ret = Ok(g.len - start_len); - break; - } - Ok(n) => g.len += n, - Err(ref e) if e.kind() == ErrorKind::Interrupted => {} - Err(e) => { - ret = Err(e); - break; + let mut read_buf: BorrowedBuf<'_> = buf.spare_capacity_mut().into(); + + // SAFETY: These bytes were initialized but not filled in the previous loop + unsafe { + read_buf.set_init(initialized); + } + + let mut cursor = read_buf.unfilled(); + match r.read_buf(cursor.reborrow()) { + Ok(()) => {} + Err(e) if e.kind() == ErrorKind::Interrupted => continue, + Err(e) => return Err(e), + } + + if cursor.written() == 0 { + return Ok(buf.len() - start_len); + } + + // store how much was initialized but not filled + initialized = cursor.init_ref().len(); + + // SAFETY: BorrowedBuf's invariants mean this much memory is initialized. + unsafe { + let new_len = read_buf.filled().len() + buf.len(); + buf.set_len(new_len); + } + + if buf.len() == buf.capacity() && buf.capacity() == start_cap { + // The buffer might be an exact fit. Let's read into a probe buffer + // and see if it returns `Ok(0)`. If so, we've avoided an + // unnecessary doubling of the capacity. But if not, append the + // probe buffer to the primary buffer and let its capacity grow. + let mut probe = [0u8; 32]; + + loop { + match r.read(&mut probe) { + Ok(0) => return Ok(buf.len() - start_len), + Ok(n) => { + buf.extend_from_slice(&probe[..n]); + break; + } + Err(ref e) if e.kind() == ErrorKind::Interrupted => continue, + Err(e) => return Err(e), + } } } } +} - ret +pub(crate) fn default_read_to_string( + r: &mut R, + buf: &mut String, +) -> Result { + // Note that we do *not* call `r.read_to_end()` here. We are passing + // `&mut Vec` (the raw contents of `buf`) into the `read_to_end` + // method to fill it up. An arbitrary implementation could overwrite the + // entire contents of the vector, not just append to it (which is what + // we are expecting). + // + // To prevent extraneously checking the UTF-8-ness of the entire buffer + // we pass it to our hardcoded `default_read_to_end` implementation which + // we know is guaranteed to only read data into the end of the buffer. + unsafe { append_to_string(buf, |b| default_read_to_end(r, b)) } } pub(crate) fn default_read_vectored(read: F, bufs: &mut [IoSliceMut<'_>]) -> Result @@ -424,6 +446,37 @@ where write(buf) } +pub(crate) fn default_read_exact(this: &mut R, mut buf: &mut [u8]) -> Result<()> { + while !buf.is_empty() { + match this.read(buf) { + Ok(0) => break, + Ok(n) => { + let tmp = buf; + buf = &mut tmp[n..]; + } + Err(ref e) if e.kind() == ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + if !buf.is_empty() { + Err(error::const_io_error!(ErrorKind::UnexpectedEof, "failed to fill whole buffer")) + } else { + Ok(()) + } +} + +pub(crate) fn default_read_buf(read: F, mut cursor: BorrowedCursor<'_>) -> Result<()> +where + F: FnOnce(&mut [u8]) -> Result, +{ + let n = read(cursor.ensure_init().init_mut())?; + unsafe { + // SAFETY: we initialised using `ensure_init` so there is no uninit data to advance to. + cursor.advance(n); + } + Ok(()) +} + /// The `Read` trait allows for reading bytes from a source. /// /// Implementors of the `Read` trait are called 'readers'. @@ -471,7 +524,7 @@ where /// } /// ``` /// -/// Read from [`&str`] because [`&[u8]`][slice] implements `Read`: +/// Read from [`&str`] because [`&[u8]`][prim@slice] implements `Read`: /// /// ```no_run /// # use std::io; @@ -489,45 +542,59 @@ where /// } /// ``` /// -/// [`read()`]: trait.Read.html#tymethod.read -/// [`std::io`]: ../../std/io/index.html -/// [`File`]: ../fs/struct.File.html -/// [`BufRead`]: trait.BufRead.html -/// [`BufReader`]: struct.BufReader.html -/// [`&str`]: ../../std/primitive.str.html -/// [slice]: ../../std/primitive.slice.html +/// [`read()`]: Read::read +/// [`&str`]: prim@str +/// [`std::io`]: self +/// [`File`]: crate::fs::File #[stable(feature = "rust1", since = "1.0.0")] +#[doc(notable_trait)] +#[cfg_attr(not(test), rustc_diagnostic_item = "IoRead")] pub trait Read { /// Pull some bytes from this source into the specified buffer, returning /// how many bytes were read. /// /// This function does not provide any guarantees about whether it blocks - /// waiting for data, but if an object needs to block for a read but cannot + /// waiting for data, but if an object needs to block for a read and cannot, /// it will typically signal this via an [`Err`] return value. /// - /// If the return value of this method is [`Ok(n)`], then it must be - /// guaranteed that `0 <= n <= buf.len()`. A nonzero `n` value indicates + /// If the return value of this method is [`Ok(n)`], then implementations must + /// guarantee that `0 <= n <= buf.len()`. A nonzero `n` value indicates /// that the buffer `buf` has been filled in with `n` bytes of data from this /// source. If `n` is `0`, then it can indicate one of two scenarios: /// /// 1. This reader has reached its "end of file" and will likely no longer /// be able to produce bytes. Note that this does not mean that the - /// reader will *always* no longer be able to produce bytes. + /// reader will *always* no longer be able to produce bytes. As an example, + /// on Linux, this method will call the `recv` syscall for a [`TcpStream`], + /// where returning zero indicates the connection was shut down correctly. While + /// for [`File`], it is possible to reach the end of file and get zero as result, + /// but if more data is appended to the file, future calls to `read` will return + /// more data. /// 2. The buffer specified was 0 bytes in length. /// + /// It is not an error if the returned value `n` is smaller than the buffer size, + /// even when the reader is not at the end of the stream yet. + /// This may happen for example because fewer bytes are actually available right now + /// (e. g. being close to end-of-file) or because read() was interrupted by a signal. + /// + /// As this trait is safe to implement, callers cannot rely on `n <= buf.len()` for safety. + /// Extra care needs to be taken when `unsafe` functions are used to access the read bytes. + /// Callers have to ensure that no unchecked out-of-bounds accesses are possible even if + /// `n > buf.len()`. + /// /// No guarantees are provided about the contents of `buf` when this - /// function is called, implementations cannot rely on any property of the + /// function is called, so implementations cannot rely on any property of the /// contents of `buf` being true. It is recommended that *implementations* /// only write data to `buf` instead of reading its contents. /// - /// Correspondingly, however, *callers* of this method may not assume any guarantees + /// Correspondingly, however, *callers* of this method must not assume any guarantees /// about how the implementation uses `buf`. The trait is safe to implement, /// so it is possible that the code that's supposed to write to the buffer might also read /// from it. It is your responsibility to make sure that `buf` is initialized /// before calling `read`. Calling `read` with an uninitialized `buf` (of the kind one /// obtains via [`MaybeUninit`]) is not safe, and can lead to undefined behavior. /// - /// [`MaybeUninit`]: ../mem/union.MaybeUninit.html + /// [`MaybeUninit`]: crate::mem::MaybeUninit /// /// # Errors /// @@ -542,10 +609,9 @@ pub trait Read { /// /// [`File`]s implement `Read`: /// - /// [`Err`]: ../../std/result/enum.Result.html#variant.Err - /// [`Ok(n)`]: ../../std/result/enum.Result.html#variant.Ok - /// [`ErrorKind::Interrupted`]: ../../std/io/enum.ErrorKind.html#variant.Interrupted - /// [`File`]: ../fs/struct.File.html + /// [`Ok(n)`]: Ok + /// [`File`]: crate::fs::File + /// [`TcpStream`]: crate::net::TcpStream /// /// ```no_run /// use std::io; @@ -569,8 +635,9 @@ pub trait Read { /// Like `read`, except that it reads into a slice of buffers. /// /// Data is copied to fill each buffer in order, with the final buffer - /// written to possibly being only partially filled. This method must behave - /// as a single call to `read` with the buffers concatenated would. + /// written to possibly being only partially filled. This method must + /// behave equivalently to a single call to `read` with concatenated + /// buffers. /// /// The default implementation calls `read` with either the first nonempty /// buffer provided, or an empty one if none exists. @@ -579,32 +646,17 @@ pub trait Read { default_read_vectored(|b| self.read(b), bufs) } - /// Determines if this `Read`er can work with buffers of uninitialized - /// memory. - /// - /// The default implementation returns an initializer which will zero - /// buffers. + /// Determines if this `Read`er has an efficient `read_vectored` + /// implementation. /// - /// If a `Read`er guarantees that it can work properly with uninitialized - /// memory, it should call [`Initializer::nop()`]. See the documentation for - /// [`Initializer`] for details. + /// If a `Read`er does not override the default `read_vectored` + /// implementation, code using it may want to avoid the method all together + /// and coalesce writes into a single buffer for higher performance. /// - /// The behavior of this method must be independent of the state of the - /// `Read`er - the method only takes `&self` so that it can be used through - /// trait objects. - /// - /// # Safety - /// - /// This method is unsafe because a `Read`er could otherwise return a - /// non-zeroing `Initializer` from another `Read` type without an `unsafe` - /// block. - /// - /// [`Initializer::nop()`]: ../../std/io/struct.Initializer.html#method.nop - /// [`Initializer`]: ../../std/io/struct.Initializer.html - #[unstable(feature = "read_initializer", issue = "42788")] - #[inline] - unsafe fn initializer(&self) -> Initializer { - Initializer::zeroing() + /// The default implementation returns `false`. + #[unstable(feature = "can_vector", issue = "69941")] + fn is_read_vectored(&self) -> bool { + false } /// Read all bytes until EOF in this source, placing them into `buf`. @@ -630,10 +682,9 @@ pub trait Read { /// /// [`File`]s implement `Read`: /// - /// [`read()`]: trait.Read.html#tymethod.read - /// [`Ok(0)`]: ../../std/result/enum.Result.html#variant.Ok - /// [`ErrorKind::Interrupted`]: ../../std/io/enum.ErrorKind.html#variant.Interrupted - /// [`File`]: ../fs/struct.File.html + /// [`read()`]: Read::read + /// [`Ok(0)`]: Ok + /// [`File`]: crate::fs::File /// /// ```no_run /// use std::io; @@ -653,10 +704,10 @@ pub trait Read { /// (See also the [`std::fs::read`] convenience function for reading from a /// file.) /// - /// [`std::fs::read`]: ../fs/fn.read.html + /// [`std::fs::read`]: crate::fs::read #[stable(feature = "rust1", since = "1.0.0")] fn read_to_end(&mut self, buf: &mut Vec) -> Result { - read_to_end(self, buf) + default_read_to_end(self, buf) } /// Read all bytes until EOF in this source, appending them to `buf`. @@ -669,15 +720,15 @@ pub trait Read { /// If the data in this stream is *not* valid UTF-8 then an error is /// returned and `buf` is unchanged. /// - /// See [`read_to_end`][readtoend] for other error semantics. + /// See [`read_to_end`] for other error semantics. /// - /// [readtoend]: #method.read_to_end + /// [`read_to_end`]: Read::read_to_end /// /// # Examples /// - /// [`File`][file]s implement `Read`: + /// [`File`]s implement `Read`: /// - /// [file]: ../fs/struct.File.html + /// [`File`]: crate::fs::File /// /// ```no_run /// use std::io; @@ -696,19 +747,10 @@ pub trait Read { /// (See also the [`std::fs::read_to_string`] convenience function for /// reading from a file.) /// - /// [`std::fs::read_to_string`]: ../fs/fn.read_to_string.html + /// [`std::fs::read_to_string`]: crate::fs::read_to_string #[stable(feature = "rust1", since = "1.0.0")] fn read_to_string(&mut self, buf: &mut String) -> Result { - // Note that we do *not* call `.read_to_end()` here. We are passing - // `&mut Vec` (the raw contents of `buf`) into the `read_to_end` - // method to fill it up. An arbitrary implementation could overwrite the - // entire contents of the vector, not just append to it (which is what - // we are expecting). - // - // To prevent extraneously checking the UTF-8-ness of the entire buffer - // we pass it to our hardcoded `read_to_end` implementation which we - // know is guaranteed to only read data into the end of the buffer. - append_to_string(buf, |b| read_to_end(self, b)) + default_read_to_string(self, buf) } /// Read the exact number of bytes required to fill `buf`. @@ -717,9 +759,11 @@ pub trait Read { /// specified buffer `buf`. /// /// No guarantees are provided about the contents of `buf` when this - /// function is called, implementations cannot rely on any property of the + /// function is called, so implementations cannot rely on any property of the /// contents of `buf` being true. It is recommended that implementations - /// only write data to `buf` instead of reading its contents. + /// only write data to `buf` instead of reading its contents. The + /// documentation on [`read`] has a more detailed explanation on this + /// subject. /// /// # Errors /// @@ -742,9 +786,8 @@ pub trait Read { /// /// [`File`]s implement `Read`: /// - /// [`File`]: ../fs/struct.File.html - /// [`ErrorKind::Interrupted`]: ../../std/io/enum.ErrorKind.html#variant.Interrupted - /// [`ErrorKind::UnexpectedEof`]: ../../std/io/enum.ErrorKind.html#variant.UnexpectedEof + /// [`read`]: Read::read + /// [`File`]: crate::fs::File /// /// ```no_run /// use std::io; @@ -761,35 +804,53 @@ pub trait Read { /// } /// ``` #[stable(feature = "read_exact", since = "1.6.0")] - fn read_exact(&mut self, mut buf: &mut [u8]) -> Result<()> { - while !buf.is_empty() { - match self.read(buf) { - Ok(0) => break, - Ok(n) => { - let tmp = buf; - buf = &mut tmp[n..]; - } - Err(ref e) if e.kind() == ErrorKind::Interrupted => {} + fn read_exact(&mut self, buf: &mut [u8]) -> Result<()> { + default_read_exact(self, buf) + } + + /// Pull some bytes from this source into the specified buffer. + /// + /// This is equivalent to the [`read`](Read::read) method, except that it is passed a [`BorrowedCursor`] rather than `[u8]` to allow use + /// with uninitialized buffers. The new data will be appended to any existing contents of `buf`. + /// + /// The default implementation delegates to `read`. + #[unstable(feature = "read_buf", issue = "78485")] + fn read_buf(&mut self, buf: BorrowedCursor<'_>) -> Result<()> { + default_read_buf(|b| self.read(b), buf) + } + + /// Read the exact number of bytes required to fill `cursor`. + /// + /// This is equivalent to the [`read_exact`](Read::read_exact) method, except that it is passed a [`BorrowedCursor`] rather than `[u8]` to + /// allow use with uninitialized buffers. + #[unstable(feature = "read_buf", issue = "78485")] + fn read_buf_exact(&mut self, mut cursor: BorrowedCursor<'_>) -> Result<()> { + while cursor.capacity() > 0 { + let prev_written = cursor.written(); + match self.read_buf(cursor.reborrow()) { + Ok(()) => {} + Err(e) if e.kind() == ErrorKind::Interrupted => continue, Err(e) => return Err(e), } + + if cursor.written() == prev_written { + return Err(Error::new(ErrorKind::UnexpectedEof, "failed to fill buffer")); + } } - if !buf.is_empty() { - Err(Error::new(ErrorKind::UnexpectedEof, "failed to fill whole buffer")) - } else { - Ok(()) - } + + Ok(()) } /// Creates a "by reference" adaptor for this instance of `Read`. /// - /// The returned adaptor also implements `Read` and will simply borrow this + /// The returned adapter also implements `Read` and will simply borrow this /// current reader. /// /// # Examples /// - /// [`File`][file]s implement `Read`: + /// [`File`]s implement `Read`: /// - /// [file]: ../fs/struct.File.html + /// [`File`]: crate::fs::File /// /// ```no_run /// use std::io; @@ -824,31 +885,32 @@ pub trait Read { /// Transforms this `Read` instance to an [`Iterator`] over its bytes. /// - /// The returned type implements [`Iterator`] where the `Item` is - /// [`Result`]`<`[`u8`]`, `[`io::Error`]`>`. + /// The returned type implements [`Iterator`] where the [`Item`] is + /// [Result]<[u8], [io::Error]>. /// The yielded item is [`Ok`] if a byte was successfully read and [`Err`] /// otherwise. EOF is mapped to returning [`None`] from this iterator. /// + /// The default implementation calls `read` for each byte, + /// which can be very inefficient for data that's not in memory, + /// such as [`File`]. Consider using a [`BufReader`] in such cases. + /// /// # Examples /// - /// [`File`][file]s implement `Read`: + /// [`File`]s implement `Read`: /// - /// [file]: ../fs/struct.File.html - /// [`Iterator`]: ../../std/iter/trait.Iterator.html - /// [`Result`]: ../../std/result/enum.Result.html - /// [`io::Error`]: ../../std/io/struct.Error.html - /// [`u8`]: ../../std/primitive.u8.html - /// [`Ok`]: ../../std/result/enum.Result.html#variant.Ok - /// [`Err`]: ../../std/result/enum.Result.html#variant.Err - /// [`None`]: ../../std/option/enum.Option.html#variant.None + /// [`Item`]: Iterator::Item + /// [`File`]: crate::fs::File "fs::File" + /// [Result]: crate::result::Result "Result" + /// [io::Error]: self::Error "io::Error" /// /// ```no_run /// use std::io; /// use std::io::prelude::*; + /// use std::io::BufReader; /// use std::fs::File; /// /// fn main() -> io::Result<()> { - /// let mut f = File::open("foo.txt")?; + /// let f = BufReader::new(File::open("foo.txt")?); /// /// for byte in f.bytes() { /// println!("{}", byte.unwrap()); @@ -864,7 +926,7 @@ pub trait Read { Bytes { inner: self } } - /// Creates an adaptor which will chain this stream with another. + /// Creates an adapter which will chain this stream with another. /// /// The returned `Read` instance will first read all bytes from this object /// until EOF is encountered. Afterwards the output is equivalent to the @@ -872,9 +934,9 @@ pub trait Read { /// /// # Examples /// - /// [`File`][file]s implement `Read`: + /// [`File`]s implement `Read`: /// - /// [file]: ../fs/struct.File.html + /// [`File`]: crate::fs::File /// /// ```no_run /// use std::io; @@ -882,8 +944,8 @@ pub trait Read { /// use std::fs::File; /// /// fn main() -> io::Result<()> { - /// let mut f1 = File::open("foo.txt")?; - /// let mut f2 = File::open("bar.txt")?; + /// let f1 = File::open("foo.txt")?; + /// let f2 = File::open("bar.txt")?; /// /// let mut handle = f1.chain(f2); /// let mut buffer = String::new(); @@ -902,7 +964,7 @@ pub trait Read { Chain { first: self, second: next, done_first: false } } - /// Creates an adaptor which will read at most `limit` bytes from it. + /// Creates an adapter which will read at most `limit` bytes from it. /// /// This function returns a new instance of `Read` which will read at most /// `limit` bytes, after which it will always return EOF ([`Ok(0)`]). Any @@ -913,9 +975,9 @@ pub trait Read { /// /// [`File`]s implement `Read`: /// - /// [`File`]: ../fs/struct.File.html - /// [`Ok(0)`]: ../../std/result/enum.Result.html#variant.Ok - /// [`read()`]: trait.Read.html#tymethod.read + /// [`File`]: crate::fs::File + /// [`Ok(0)`]: Ok + /// [`read()`]: Read::read /// /// ```no_run /// use std::io; @@ -923,7 +985,7 @@ pub trait Read { /// use std::fs::File; /// /// fn main() -> io::Result<()> { - /// let mut f = File::open("foo.txt")?; + /// let f = File::open("foo.txt")?; /// let mut buffer = [0; 5]; /// /// // read at most five bytes @@ -942,6 +1004,57 @@ pub trait Read { } } +/// Read all bytes from a [reader][Read] into a new [`String`]. +/// +/// This is a convenience function for [`Read::read_to_string`]. Using this +/// function avoids having to create a variable first and provides more type +/// safety since you can only get the buffer out if there were no errors. (If you +/// use [`Read::read_to_string`] you have to remember to check whether the read +/// succeeded because otherwise your buffer will be empty or only partially full.) +/// +/// # Performance +/// +/// The downside of this function's increased ease of use and type safety is +/// that it gives you less control over performance. For example, you can't +/// pre-allocate memory like you can using [`String::with_capacity`] and +/// [`Read::read_to_string`]. Also, you can't re-use the buffer if an error +/// occurs while reading. +/// +/// In many cases, this function's performance will be adequate and the ease of use +/// and type safety tradeoffs will be worth it. However, there are cases where you +/// need more control over performance, and in those cases you should definitely use +/// [`Read::read_to_string`] directly. +/// +/// Note that in some special cases, such as when reading files, this function will +/// pre-allocate memory based on the size of the input it is reading. In those +/// cases, the performance should be as good as if you had used +/// [`Read::read_to_string`] with a manually pre-allocated buffer. +/// +/// # Errors +/// +/// This function forces you to handle errors because the output (the `String`) +/// is wrapped in a [`Result`]. See [`Read::read_to_string`] for the errors +/// that can occur. If any error occurs, you will get an [`Err`], so you +/// don't have to worry about your buffer being empty or partially full. +/// +/// # Examples +/// +/// ```no_run +/// # use std::io; +/// fn main() -> io::Result<()> { +/// let stdin = io::read_to_string(io::stdin())?; +/// println!("Stdin was:"); +/// println!("{stdin}"); +/// Ok(()) +/// } +/// ``` +#[stable(feature = "io_read_to_string", since = "1.65.0")] +pub fn read_to_string(mut reader: R) -> Result { + let mut buf = String::new(); + reader.read_to_string(&mut buf)?; + Ok(buf) +} + /// A buffer type used with `Read::read_vectored`. /// /// It is semantically a wrapper around an `&mut [u8]`, but is guaranteed to be @@ -951,6 +1064,12 @@ pub trait Read { #[repr(transparent)] pub struct IoSliceMut<'a>(sys::io::IoSliceMut<'a>); +#[stable(feature = "iovec-send-sync", since = "1.44.0")] +unsafe impl<'a> Send for IoSliceMut<'a> {} + +#[stable(feature = "iovec-send-sync", since = "1.44.0")] +unsafe impl<'a> Sync for IoSliceMut<'a> {} + #[stable(feature = "iovec", since = "1.36.0")] impl<'a> fmt::Debug for IoSliceMut<'a> { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -972,13 +1091,46 @@ impl<'a> IoSliceMut<'a> { /// Advance the internal cursor of the slice. /// - /// # Notes + /// Also see [`IoSliceMut::advance_slices`] to advance the cursors of + /// multiple buffers. + /// + /// # Panics /// - /// Elements in the slice may be modified if the cursor is not advanced to - /// the end of the slice. For example if we have a slice of buffers with 2 - /// `IoSliceMut`s, both of length 8, and we advance the cursor by 10 bytes - /// the first `IoSliceMut` will be untouched however the second will be - /// modified to remove the first 2 bytes (10 - 8). + /// Panics when trying to advance beyond the end of the slice. + /// + /// # Examples + /// + /// ``` + /// #![feature(io_slice_advance)] + /// + /// use std::io::IoSliceMut; + /// use std::ops::Deref; + /// + /// let mut data = [1; 8]; + /// let mut buf = IoSliceMut::new(&mut data); + /// + /// // Mark 3 bytes as read. + /// buf.advance(3); + /// assert_eq!(buf.deref(), [1; 5].as_ref()); + /// ``` + #[unstable(feature = "io_slice_advance", issue = "62726")] + #[inline] + pub fn advance(&mut self, n: usize) { + self.0.advance(n) + } + + /// Advance a slice of slices. + /// + /// Shrinks the slice to remove any `IoSliceMut`s that are fully advanced over. + /// If the cursor ends up in the middle of an `IoSliceMut`, it is modified + /// to start at that cursor. + /// + /// For example, if we have a slice of two 8-byte `IoSliceMut`s, and we advance by 10 bytes, + /// the result will only include the second `IoSliceMut`, advanced by 2 bytes. + /// + /// # Panics + /// + /// Panics when trying to advance beyond the end of the slices. /// /// # Examples /// @@ -998,13 +1150,13 @@ impl<'a> IoSliceMut<'a> { /// ][..]; /// /// // Mark 10 bytes as read. - /// bufs = IoSliceMut::advance(bufs, 10); + /// IoSliceMut::advance_slices(&mut bufs, 10); /// assert_eq!(bufs[0].deref(), [2; 14].as_ref()); /// assert_eq!(bufs[1].deref(), [3; 8].as_ref()); /// ``` #[unstable(feature = "io_slice_advance", issue = "62726")] #[inline] - pub fn advance<'b>(bufs: &'b mut [IoSliceMut<'a>], n: usize) -> &'b mut [IoSliceMut<'a>] { + pub fn advance_slices(bufs: &mut &mut [IoSliceMut<'a>], n: usize) { // Number of buffers to remove. let mut remove = 0; // Total length of all the to be removed buffers. @@ -1018,11 +1170,12 @@ impl<'a> IoSliceMut<'a> { } } - let bufs = &mut bufs[remove..]; - if !bufs.is_empty() { - bufs[0].0.advance(n - accumulated_len) + *bufs = &mut replace(bufs, &mut [])[remove..]; + if bufs.is_empty() { + assert!(n == accumulated_len, "advancing io slices beyond their length"); + } else { + bufs[0].advance(n - accumulated_len) } - bufs } } @@ -1046,7 +1199,7 @@ impl<'a> DerefMut for IoSliceMut<'a> { /// A buffer type used with `Write::write_vectored`. /// -/// It is semantically a wrapper around an `&[u8]`, but is guaranteed to be +/// It is semantically a wrapper around a `&[u8]`, but is guaranteed to be /// ABI compatible with the `iovec` type on Unix platforms and `WSABUF` on /// Windows. #[stable(feature = "iovec", since = "1.36.0")] @@ -1054,6 +1207,12 @@ impl<'a> DerefMut for IoSliceMut<'a> { #[repr(transparent)] pub struct IoSlice<'a>(sys::io::IoSlice<'a>); +#[stable(feature = "iovec-send-sync", since = "1.44.0")] +unsafe impl<'a> Send for IoSlice<'a> {} + +#[stable(feature = "iovec-send-sync", since = "1.44.0")] +unsafe impl<'a> Sync for IoSlice<'a> {} + #[stable(feature = "iovec", since = "1.36.0")] impl<'a> fmt::Debug for IoSlice<'a> { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -1068,6 +1227,7 @@ impl<'a> IoSlice<'a> { /// /// Panics on Windows if the slice is larger than 4GB. #[stable(feature = "iovec", since = "1.36.0")] + #[must_use] #[inline] pub fn new(buf: &'a [u8]) -> IoSlice<'a> { IoSlice(sys::io::IoSlice::new(buf)) @@ -1075,13 +1235,46 @@ impl<'a> IoSlice<'a> { /// Advance the internal cursor of the slice. /// - /// # Notes + /// Also see [`IoSlice::advance_slices`] to advance the cursors of multiple + /// buffers. + /// + /// # Panics + /// + /// Panics when trying to advance beyond the end of the slice. + /// + /// # Examples + /// + /// ``` + /// #![feature(io_slice_advance)] /// - /// Elements in the slice may be modified if the cursor is not advanced to - /// the end of the slice. For example if we have a slice of buffers with 2 - /// `IoSlice`s, both of length 8, and we advance the cursor by 10 bytes the - /// first `IoSlice` will be untouched however the second will be modified to - /// remove the first 2 bytes (10 - 8). + /// use std::io::IoSlice; + /// use std::ops::Deref; + /// + /// let data = [1; 8]; + /// let mut buf = IoSlice::new(&data); + /// + /// // Mark 3 bytes as read. + /// buf.advance(3); + /// assert_eq!(buf.deref(), [1; 5].as_ref()); + /// ``` + #[unstable(feature = "io_slice_advance", issue = "62726")] + #[inline] + pub fn advance(&mut self, n: usize) { + self.0.advance(n) + } + + /// Advance a slice of slices. + /// + /// Shrinks the slice to remove any `IoSlice`s that are fully advanced over. + /// If the cursor ends up in the middle of an `IoSlice`, it is modified + /// to start at that cursor. + /// + /// For example, if we have a slice of two 8-byte `IoSlice`s, and we advance by 10 bytes, + /// the result will only include the second `IoSlice`, advanced by 2 bytes. + /// + /// # Panics + /// + /// Panics when trying to advance beyond the end of the slices. /// /// # Examples /// @@ -1101,12 +1294,12 @@ impl<'a> IoSlice<'a> { /// ][..]; /// /// // Mark 10 bytes as written. - /// bufs = IoSlice::advance(bufs, 10); + /// IoSlice::advance_slices(&mut bufs, 10); /// assert_eq!(bufs[0].deref(), [2; 14].as_ref()); /// assert_eq!(bufs[1].deref(), [3; 8].as_ref()); #[unstable(feature = "io_slice_advance", issue = "62726")] #[inline] - pub fn advance<'b>(bufs: &'b mut [IoSlice<'a>], n: usize) -> &'b mut [IoSlice<'a>] { + pub fn advance_slices(bufs: &mut &mut [IoSlice<'a>], n: usize) { // Number of buffers to remove. let mut remove = 0; // Total length of all the to be removed buffers. @@ -1120,11 +1313,12 @@ impl<'a> IoSlice<'a> { } } - let bufs = &mut bufs[remove..]; - if !bufs.is_empty() { - bufs[0].0.advance(n - accumulated_len) + *bufs = &mut replace(bufs, &mut [])[remove..]; + if bufs.is_empty() { + assert!(n == accumulated_len, "advancing io slices beyond their length"); + } else { + bufs[0].advance(n - accumulated_len) } - bufs } } @@ -1138,50 +1332,6 @@ impl<'a> Deref for IoSlice<'a> { } } -/// A type used to conditionally initialize buffers passed to `Read` methods. -#[unstable(feature = "read_initializer", issue = "42788")] -#[derive(Debug)] -pub struct Initializer(bool); - -impl Initializer { - /// Returns a new `Initializer` which will zero out buffers. - #[unstable(feature = "read_initializer", issue = "42788")] - #[inline] - pub fn zeroing() -> Initializer { - Initializer(true) - } - - /// Returns a new `Initializer` which will not zero out buffers. - /// - /// # Safety - /// - /// This may only be called by `Read`ers which guarantee that they will not - /// read from buffers passed to `Read` methods, and that the return value of - /// the method accurately reflects the number of bytes that have been - /// written to the head of the buffer. - #[unstable(feature = "read_initializer", issue = "42788")] - #[inline] - pub unsafe fn nop() -> Initializer { - Initializer(false) - } - - /// Indicates if a buffer should be initialized. - #[unstable(feature = "read_initializer", issue = "42788")] - #[inline] - pub fn should_initialize(&self) -> bool { - self.0 - } - - /// Initializes a buffer if necessary. - #[unstable(feature = "read_initializer", issue = "42788")] - #[inline] - pub fn initialize(&self, buf: &mut [u8]) { - if self.should_initialize() { - unsafe { ptr::write_bytes(buf.as_mut_ptr(), 0, buf.len()) } - } - } -} - /// A trait for objects which are byte-oriented sinks. /// /// Implementors of the `Write` trait are sometimes called 'writers'. @@ -1191,7 +1341,7 @@ impl Initializer { /// * The [`write`] method will attempt to write some data into the object, /// returning how many bytes were successfully written. /// -/// * The [`flush`] method is useful for adaptors and explicit buffers +/// * The [`flush`] method is useful for adapters and explicit buffers /// themselves for ensuring that all buffered data has been pushed out to the /// 'true sink'. /// @@ -1199,9 +1349,9 @@ impl Initializer { /// throughout [`std::io`] take and provide types which implement the `Write` /// trait. /// -/// [`write`]: #tymethod.write -/// [`flush`]: #tymethod.flush -/// [`std::io`]: index.html +/// [`write`]: Write::write +/// [`flush`]: Write::flush +/// [`std::io`]: self /// /// # Examples /// @@ -1226,13 +1376,15 @@ impl Initializer { /// The trait also provides convenience methods like [`write_all`], which calls /// `write` in a loop until its entire input has been written. /// -/// [`write_all`]: #method.write_all +/// [`write_all`]: Write::write_all #[stable(feature = "rust1", since = "1.0.0")] +#[doc(notable_trait)] +#[cfg_attr(not(test), rustc_diagnostic_item = "IoWrite")] pub trait Write { /// Write a buffer into this writer, returning how many bytes were written. /// /// This function will attempt to write the entire contents of `buf`, but - /// the entire write may not succeed, or the write may also generate an + /// the entire write might not succeed, or the write may also generate an /// error. A call to `write` represents *at most one* attempt to write to /// any wrapped object. /// @@ -1257,10 +1409,6 @@ pub trait Write { /// An error of the [`ErrorKind::Interrupted`] kind is non-fatal and the /// write operation should be retried if there is nothing else to do. /// - /// [`Err`]: ../../std/result/enum.Result.html#variant.Err - /// [`Ok(n)`]: ../../std/result/enum.Result.html#variant.Ok - /// [`ErrorKind::Interrupted`]: ../../std/io/enum.ErrorKind.html#variant.Interrupted - /// /// # Examples /// /// ```no_run @@ -1275,22 +1423,62 @@ pub trait Write { /// Ok(()) /// } /// ``` + /// + /// [`Ok(n)`]: Ok #[stable(feature = "rust1", since = "1.0.0")] fn write(&mut self, buf: &[u8]) -> Result; - /// Like `write`, except that it writes from a slice of buffers. + /// Like [`write`], except that it writes from a slice of buffers. /// /// Data is copied from each buffer in order, with the final buffer /// read from possibly being only partially consumed. This method must - /// behave as a call to `write` with the buffers concatenated would. + /// behave as a call to [`write`] with the buffers concatenated would. /// - /// The default implementation calls `write` with either the first nonempty + /// The default implementation calls [`write`] with either the first nonempty /// buffer provided, or an empty one if none exists. + /// + /// # Examples + /// + /// ```no_run + /// use std::io::IoSlice; + /// use std::io::prelude::*; + /// use std::fs::File; + /// + /// fn main() -> std::io::Result<()> { + /// let data1 = [1; 8]; + /// let data2 = [15; 8]; + /// let io_slice1 = IoSlice::new(&data1); + /// let io_slice2 = IoSlice::new(&data2); + /// + /// let mut buffer = File::create("foo.txt")?; + /// + /// // Writes some prefix of the byte string, not necessarily all of it. + /// buffer.write_vectored(&[io_slice1, io_slice2])?; + /// Ok(()) + /// } + /// ``` + /// + /// [`write`]: Write::write #[stable(feature = "iovec", since = "1.36.0")] fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> Result { default_write_vectored(|b| self.write(b), bufs) } + /// Determines if this `Write`r has an efficient [`write_vectored`] + /// implementation. + /// + /// If a `Write`r does not override the default [`write_vectored`] + /// implementation, code using it may want to avoid the method all together + /// and coalesce writes into a single buffer for higher performance. + /// + /// The default implementation returns `false`. + /// + /// [`write_vectored`]: Write::write_vectored + #[unstable(feature = "can_vector", issue = "69941")] + fn is_write_vectored(&self) -> bool { + false + } + /// Flush this output stream, ensuring that all intermediately buffered /// contents reach their destination. /// @@ -1333,8 +1521,7 @@ pub trait Write { /// This function will return the first error of /// non-[`ErrorKind::Interrupted`] kind that [`write`] returns. /// - /// [`ErrorKind::Interrupted`]: ../../std/io/enum.ErrorKind.html#variant.Interrupted - /// [`write`]: #tymethod.write + /// [`write`]: Write::write /// /// # Examples /// @@ -1354,7 +1541,10 @@ pub trait Write { while !buf.is_empty() { match self.write(buf) { Ok(0) => { - return Err(Error::new(ErrorKind::WriteZero, "failed to write whole buffer")); + return Err(error::const_io_error!( + ErrorKind::WriteZero, + "failed to write whole buffer", + )); } Ok(n) => buf = &buf[n..], Err(ref e) if e.kind() == ErrorKind::Interrupted => {} @@ -1364,23 +1554,88 @@ pub trait Write { Ok(()) } + /// Attempts to write multiple buffers into this writer. + /// + /// This method will continuously call [`write_vectored`] until there is no + /// more data to be written or an error of non-[`ErrorKind::Interrupted`] + /// kind is returned. This method will not return until all buffers have + /// been successfully written or such an error occurs. The first error that + /// is not of [`ErrorKind::Interrupted`] kind generated from this method + /// will be returned. + /// + /// If the buffer contains no data, this will never call [`write_vectored`]. + /// + /// # Notes + /// + /// Unlike [`write_vectored`], this takes a *mutable* reference to + /// a slice of [`IoSlice`]s, not an immutable one. That's because we need to + /// modify the slice to keep track of the bytes already written. + /// + /// Once this function returns, the contents of `bufs` are unspecified, as + /// this depends on how many calls to [`write_vectored`] were necessary. It is + /// best to understand this function as taking ownership of `bufs` and to + /// not use `bufs` afterwards. The underlying buffers, to which the + /// [`IoSlice`]s point (but not the [`IoSlice`]s themselves), are unchanged and + /// can be reused. + /// + /// [`write_vectored`]: Write::write_vectored + /// + /// # Examples + /// + /// ``` + /// #![feature(write_all_vectored)] + /// # fn main() -> std::io::Result<()> { + /// + /// use std::io::{Write, IoSlice}; + /// + /// let mut writer = Vec::new(); + /// let bufs = &mut [ + /// IoSlice::new(&[1]), + /// IoSlice::new(&[2, 3]), + /// IoSlice::new(&[4, 5, 6]), + /// ]; + /// + /// writer.write_all_vectored(bufs)?; + /// // Note: the contents of `bufs` is now undefined, see the Notes section. + /// + /// assert_eq!(writer, &[1, 2, 3, 4, 5, 6]); + /// # Ok(()) } + /// ``` + #[unstable(feature = "write_all_vectored", issue = "70436")] + fn write_all_vectored(&mut self, mut bufs: &mut [IoSlice<'_>]) -> Result<()> { + // Guarantee that bufs is empty if it contains no data, + // to avoid calling write_vectored if there is no data to be written. + IoSlice::advance_slices(&mut bufs, 0); + while !bufs.is_empty() { + match self.write_vectored(bufs) { + Ok(0) => { + return Err(error::const_io_error!( + ErrorKind::WriteZero, + "failed to write whole buffer", + )); + } + Ok(n) => IoSlice::advance_slices(&mut bufs, n), + Err(ref e) if e.kind() == ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + Ok(()) + } + /// Writes a formatted string into this writer, returning any error /// encountered. /// /// This method is primarily used to interface with the - /// [`format_args!`][formatargs] macro, but it is rare that this should - /// explicitly be called. The [`write!`][write] macro should be favored to + /// [`format_args!()`] macro, and it is rare that this should + /// explicitly be called. The [`write!()`] macro should be favored to /// invoke this method instead. /// - /// [formatargs]: ../macro.format_args.html - /// [write]: ../macro.write.html - /// - /// This function internally uses the [`write_all`][writeall] method on + /// This function internally uses the [`write_all`] method on /// this trait and hence will continuously write data so long as no errors /// are received. This also means that partial writes are not indicated in /// this signature. /// - /// [writeall]: #method.write_all + /// [`write_all`]: Write::write_all /// /// # Errors /// @@ -1406,12 +1661,12 @@ pub trait Write { fn write_fmt(&mut self, fmt: fmt::Arguments<'_>) -> Result<()> { // Create a shim which translates a Write to a fmt::Write and saves // off I/O errors. instead of discarding them - struct Adaptor<'a, T: ?Sized + 'a> { + struct Adapter<'a, T: ?Sized + 'a> { inner: &'a mut T, error: Result<()>, } - impl fmt::Write for Adaptor<'_, T> { + impl fmt::Write for Adapter<'_, T> { fn write_str(&mut self, s: &str) -> fmt::Result { match self.inner.write_all(s.as_bytes()) { Ok(()) => Ok(()), @@ -1423,7 +1678,7 @@ pub trait Write { } } - let mut output = Adaptor { inner: self, error: Ok(()) }; + let mut output = Adapter { inner: self, error: Ok(()) }; match fmt::write(&mut output, fmt) { Ok(()) => Ok(()), Err(..) => { @@ -1431,15 +1686,15 @@ pub trait Write { if output.error.is_err() { output.error } else { - Err(Error::new(ErrorKind::Other, "formatter error")) + Err(error::const_io_error!(ErrorKind::Uncategorized, "formatter error")) } } } } - /// Creates a "by reference" adaptor for this instance of `Write`. + /// Creates a "by reference" adapter for this instance of `Write`. /// - /// The returned adaptor also implements `Write` and will simply borrow this + /// The returned adapter also implements `Write` and will simply borrow this /// current writer. /// /// # Examples @@ -1475,9 +1730,9 @@ pub trait Write { /// /// # Examples /// -/// [`File`][file]s implement `Seek`: +/// [`File`]s implement `Seek`: /// -/// [file]: ../fs/struct.File.html +/// [`File`]: crate::fs::File /// /// ```no_run /// use std::io; @@ -1506,12 +1761,46 @@ pub trait Seek { /// /// # Errors /// - /// Seeking to a negative offset is considered an error. + /// Seeking can fail, for example because it might involve flushing a buffer. /// - /// [`SeekFrom::Start`]: enum.SeekFrom.html#variant.Start + /// Seeking to a negative offset is considered an error. #[stable(feature = "rust1", since = "1.0.0")] fn seek(&mut self, pos: SeekFrom) -> Result; + /// Rewind to the beginning of a stream. + /// + /// This is a convenience method, equivalent to `seek(SeekFrom::Start(0))`. + /// + /// # Errors + /// + /// Rewinding can fail, for example because it might involve flushing a buffer. + /// + /// # Example + /// + /// ```no_run + /// use std::io::{Read, Seek, Write}; + /// use std::fs::OpenOptions; + /// + /// let mut f = OpenOptions::new() + /// .write(true) + /// .read(true) + /// .create(true) + /// .open("foo.txt").unwrap(); + /// + /// let hello = "Hello!\n"; + /// write!(f, "{hello}").unwrap(); + /// f.rewind().unwrap(); + /// + /// let mut buf = String::new(); + /// f.read_to_string(&mut buf).unwrap(); + /// assert_eq!(&buf, hello); + /// ``` + #[stable(feature = "seek_rewind", since = "1.55.0")] + fn rewind(&mut self) -> Result<()> { + self.seek(SeekFrom::Start(0))?; + Ok(()) + } + /// Returns the length of this stream (in bytes). /// /// This method is implemented using up to three seek operations. If this @@ -1529,11 +1818,10 @@ pub trait Seek { /// data is appended to a file). So calling this method multiple times does /// not necessarily return the same length each time. /// - /// /// # Example /// /// ```no_run - /// #![feature(seek_convenience)] + /// #![feature(seek_stream_len)] /// use std::{ /// io::{self, Seek}, /// fs::File, @@ -1543,11 +1831,11 @@ pub trait Seek { /// let mut f = File::open("foo.txt")?; /// /// let len = f.stream_len()?; - /// println!("The file is currently {} bytes long", len); + /// println!("The file is currently {len} bytes long"); /// Ok(()) /// } /// ``` - #[unstable(feature = "seek_convenience", issue = "59359")] + #[unstable(feature = "seek_stream_len", issue = "59359")] fn stream_len(&mut self) -> Result { let old_pos = self.stream_position()?; let len = self.seek(SeekFrom::End(0))?; @@ -1565,11 +1853,9 @@ pub trait Seek { /// /// This is equivalent to `self.seek(SeekFrom::Current(0))`. /// - /// /// # Example /// /// ```no_run - /// #![feature(seek_convenience)] /// use std::{ /// io::{self, BufRead, BufReader, Seek}, /// fs::File, @@ -1586,7 +1872,7 @@ pub trait Seek { /// Ok(()) /// } /// ``` - #[unstable(feature = "seek_convenience", issue = "59359")] + #[stable(feature = "seek_convenience", since = "1.51.0")] fn stream_position(&mut self) -> Result { self.seek(SeekFrom::Current(0)) } @@ -1595,8 +1881,6 @@ pub trait Seek { /// Enumeration of possible methods to seek within an I/O object. /// /// It is used by the [`Seek`] trait. -/// -/// [`Seek`]: trait.Seek.html #[derive(Copy, PartialEq, Eq, Clone, Debug)] #[stable(feature = "rust1", since = "1.0.0")] pub enum SeekFrom { @@ -1676,11 +1960,9 @@ fn read_until(r: &mut R, delim: u8, buf: &mut Vec) -> R /// For example, [`File`] implements [`Read`], but not `BufRead`. /// [`BufReader`] to the rescue! /// -/// [`BufReader`]: struct.BufReader.html -/// [`File`]: ../fs/struct.File.html -/// [`read_line`]: #method.read_line -/// [`lines`]: #method.lines -/// [`Read`]: trait.Read.html +/// [`File`]: crate::fs::File +/// [`read_line`]: BufRead::read_line +/// [`lines`]: BufRead::lines /// /// ```no_run /// use std::io::{self, BufReader}; @@ -1698,7 +1980,6 @@ fn read_until(r: &mut R, delim: u8, buf: &mut Vec) -> R /// Ok(()) /// } /// ``` -/// #[stable(feature = "rust1", since = "1.0.0")] pub trait BufRead: Read { /// Returns the contents of the internal buffer, filling it with more data @@ -1711,7 +1992,7 @@ pub trait BufRead: Read { /// be called with the number of bytes that are consumed from this buffer to /// ensure that the bytes are never returned twice. /// - /// [`consume`]: #tymethod.consume + /// [`consume`]: BufRead::consume /// /// An empty buffer returned indicates that the stream has reached EOF. /// @@ -1734,7 +2015,7 @@ pub trait BufRead: Read { /// let buffer = stdin.fill_buf().unwrap(); /// /// // work with buffer - /// println!("{:?}", buffer); + /// println!("{buffer:?}"); /// /// // ensure the bytes we worked with aren't returned again later /// let length = buffer.len(); @@ -1761,10 +2042,41 @@ pub trait BufRead: Read { /// Since `consume()` is meant to be used with [`fill_buf`], /// that method's example includes an example of `consume()`. /// - /// [`fill_buf`]: #tymethod.fill_buf + /// [`fill_buf`]: BufRead::fill_buf #[stable(feature = "rust1", since = "1.0.0")] fn consume(&mut self, amt: usize); + /// Check if the underlying `Read` has any data left to be read. + /// + /// This function may fill the buffer to check for data, + /// so this functions returns `Result`, not `bool`. + /// + /// Default implementation calls `fill_buf` and checks that + /// returned slice is empty (which means that there is no data left, + /// since EOF is reached). + /// + /// Examples + /// + /// ``` + /// #![feature(buf_read_has_data_left)] + /// use std::io; + /// use std::io::prelude::*; + /// + /// let stdin = io::stdin(); + /// let mut stdin = stdin.lock(); + /// + /// while stdin.has_data_left().unwrap() { + /// let mut line = String::new(); + /// stdin.read_line(&mut line).unwrap(); + /// // work with line + /// println!("{line:?}"); + /// } + /// ``` + #[unstable(feature = "buf_read_has_data_left", reason = "recently added", issue = "86423")] + fn has_data_left(&mut self) -> Result { + self.fill_buf().map(|b| !b.is_empty()) + } + /// Read all bytes into `buf` until the delimiter `byte` or EOF is reached. /// /// This function will read bytes from the underlying stream until the @@ -1773,6 +2085,10 @@ pub trait BufRead: Read { /// /// If successful, this function will return the total number of bytes read. /// + /// This function is blocking and should be used carefully: it is possible for + /// an attacker to continuously send bytes without ever sending the delimiter + /// or EOF. + /// /// # Errors /// /// This function will ignore all instances of [`ErrorKind::Interrupted`] and @@ -1781,8 +2097,7 @@ pub trait BufRead: Read { /// If an I/O error is encountered then all bytes read so far will be /// present in `buf` and its length will have been adjusted appropriately. /// - /// [`fill_buf`]: #tymethod.fill_buf - /// [`ErrorKind::Interrupted`]: enum.ErrorKind.html#variant.Interrupted + /// [`fill_buf`]: BufRead::fill_buf /// /// # Examples /// @@ -1790,8 +2105,6 @@ pub trait BufRead: Read { /// this example, we use [`Cursor`] to read all the bytes in a byte slice /// in hyphen delimited segments: /// - /// [`Cursor`]: struct.Cursor.html - /// /// ``` /// use std::io::{self, BufRead}; /// @@ -1823,17 +2136,28 @@ pub trait BufRead: Read { read_until(self, byte, buf) } - /// Read all bytes until a newline (the 0xA byte) is reached, and append - /// them to the provided buffer. + /// Read all bytes until a newline (the `0xA` byte) is reached, and append + /// them to the provided `String` buffer. + /// + /// Previous content of the buffer will be preserved. To avoid appending to + /// the buffer, you need to [`clear`] it first. /// /// This function will read bytes from the underlying stream until the - /// newline delimiter (the 0xA byte) or EOF is found. Once found, all bytes + /// newline delimiter (the `0xA` byte) or EOF is found. Once found, all bytes /// up to, and including, the delimiter (if found) will be appended to /// `buf`. /// /// If successful, this function will return the total number of bytes read. /// - /// If this function returns `Ok(0)`, the stream has reached EOF. + /// If this function returns [`Ok(0)`], the stream has reached EOF. + /// + /// This function is blocking and should be used carefully: it is possible for + /// an attacker to continuously send bytes without ever sending a newline + /// or EOF. You can use [`take`] to limit the maximum number of bytes read. + /// + /// [`Ok(0)`]: Ok + /// [`clear`]: String::clear + /// [`take`]: crate::io::Read::take /// /// # Errors /// @@ -1842,15 +2166,13 @@ pub trait BufRead: Read { /// error is encountered then `buf` may contain some bytes already read in /// the event that all data read so far was valid UTF-8. /// - /// [`read_until`]: #method.read_until + /// [`read_until`]: BufRead::read_until /// /// # Examples /// /// [`std::io::Cursor`][`Cursor`] is a type that implements `BufRead`. In /// this example, we use [`Cursor`] to read all the lines in a byte slice: /// - /// [`Cursor`]: struct.Cursor.html - /// /// ``` /// use std::io::{self, BufRead}; /// @@ -1882,22 +2204,21 @@ pub trait BufRead: Read { // Note that we are not calling the `.read_until` method here, but // rather our hardcoded implementation. For more details as to why, see // the comments in `read_to_end`. - append_to_string(buf, |b| read_until(self, b'\n', b)) + unsafe { append_to_string(buf, |b| read_until(self, b'\n', b)) } } /// Returns an iterator over the contents of this reader split on the byte /// `byte`. /// /// The iterator returned from this function will return instances of - /// [`io::Result`]`<`[`Vec`]`>`. Each vector returned will *not* have + /// [io::Result]<[Vec]\>. Each vector returned will *not* have /// the delimiter byte at the end. /// /// This function will yield errors whenever [`read_until`] would have /// also yielded an error. /// - /// [`io::Result`]: type.Result.html - /// [`Vec`]: ../vec/struct.Vec.html - /// [`read_until`]: #method.read_until + /// [io::Result]: self::Result "io::Result" + /// [`read_until`]: BufRead::read_until /// /// # Examples /// @@ -1905,8 +2226,6 @@ pub trait BufRead: Read { /// this example, we use [`Cursor`] to iterate over all hyphen delimited /// segments in a byte slice /// - /// [`Cursor`]: struct.Cursor.html - /// /// ``` /// use std::io::{self, BufRead}; /// @@ -1929,11 +2248,10 @@ pub trait BufRead: Read { /// Returns an iterator over the lines of this reader. /// /// The iterator returned from this function will yield instances of - /// [`io::Result`]`<`[`String`]`>`. Each string returned will *not* have a newline - /// byte (the 0xA byte) or CRLF (0xD, 0xA bytes) at the end. + /// [io::Result]<[String]>. Each string returned will *not* have a newline + /// byte (the `0xA` byte) or `CRLF` (`0xD`, `0xA` bytes) at the end. /// - /// [`io::Result`]: type.Result.html - /// [`String`]: ../string/struct.String.html + /// [io::Result]: self::Result "io::Result" /// /// # Examples /// @@ -1941,8 +2259,6 @@ pub trait BufRead: Read { /// this example, we use [`Cursor`] to iterate over all the lines in a byte /// slice. /// - /// [`Cursor`]: struct.Cursor.html - /// /// ``` /// use std::io::{self, BufRead}; /// @@ -1958,8 +2274,6 @@ pub trait BufRead: Read { /// # Errors /// /// Each line of the iterator has the same error semantics as [`BufRead::read_line`]. - /// - /// [`BufRead::read_line`]: trait.BufRead.html#method.read_line #[stable(feature = "rust1", since = "1.0.0")] fn lines(self) -> Lines where @@ -1969,13 +2283,14 @@ pub trait BufRead: Read { } } -/// Adaptor to chain together two readers. +/// Adapter to chain together two readers. /// /// This struct is generally created by calling [`chain`] on a reader. /// Please see the documentation of [`chain`] for more details. /// -/// [`chain`]: trait.Read.html#method.chain +/// [`chain`]: Read::chain #[stable(feature = "rust1", since = "1.0.0")] +#[derive(Debug)] pub struct Chain { first: T, second: U, @@ -2057,13 +2372,6 @@ impl Chain { } } -#[stable(feature = "std_debug", since = "1.16.0")] -impl fmt::Debug for Chain { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Chain").field("t", &self.first).field("u", &self.second).finish() - } -} - #[stable(feature = "rust1", since = "1.0.0")] impl Read for Chain { fn read(&mut self, buf: &mut [u8]) -> Result { @@ -2085,11 +2393,6 @@ impl Read for Chain { } self.second.read_vectored(bufs) } - - unsafe fn initializer(&self) -> Initializer { - let initializer = self.first.initializer(); - if initializer.should_initialize() { initializer } else { self.second.initializer() } - } } #[stable(feature = "chain_bufread", since = "1.9.0")] @@ -2111,12 +2414,27 @@ impl BufRead for Chain { } } -/// Reader adaptor which limits the bytes read from an underlying reader. +impl SizeHint for Chain { + #[inline] + fn lower_bound(&self) -> usize { + SizeHint::lower_bound(&self.first) + SizeHint::lower_bound(&self.second) + } + + #[inline] + fn upper_bound(&self) -> Option { + match (SizeHint::upper_bound(&self.first), SizeHint::upper_bound(&self.second)) { + (Some(first), Some(second)) => first.checked_add(second), + _ => None, + } + } +} + +/// Reader adapter which limits the bytes read from an underlying reader. /// /// This struct is generally created by calling [`take`] on a reader. /// Please see the documentation of [`take`] for more details. /// -/// [`take`]: trait.Read.html#method.take +/// [`take`]: Read::take #[stable(feature = "rust1", since = "1.0.0")] #[derive(Debug)] pub struct Take { @@ -2133,8 +2451,6 @@ impl Take { /// This instance may reach `EOF` after reading fewer bytes than indicated by /// this method if the underlying [`Read`] instance reaches EOF. /// - /// [`Read`]: ../../std/io/trait.Read.html - /// /// # Examples /// /// ```no_run @@ -2275,19 +2591,56 @@ impl Read for Take { let max = cmp::min(buf.len() as u64, self.limit) as usize; let n = self.inner.read(&mut buf[..max])?; + assert!(n as u64 <= self.limit, "number of read bytes exceeds limit"); self.limit -= n as u64; Ok(n) } - unsafe fn initializer(&self) -> Initializer { - self.inner.initializer() - } + fn read_buf(&mut self, mut buf: BorrowedCursor<'_>) -> Result<()> { + // Don't call into inner reader at all at EOF because it may still block + if self.limit == 0 { + return Ok(()); + } - fn read_to_end(&mut self, buf: &mut Vec) -> Result { - // Pass in a reservation_size closure that respects the current value - // of limit for each read. If we hit the read limit, this prevents the - // final zero-byte read from allocating again. - read_to_end_with_reservation(self, buf, |self_| cmp::min(self_.limit, 32) as usize) + if self.limit <= buf.capacity() as u64 { + // if we just use an as cast to convert, limit may wrap around on a 32 bit target + let limit = cmp::min(self.limit, usize::MAX as u64) as usize; + + let extra_init = cmp::min(limit as usize, buf.init_ref().len()); + + // SAFETY: no uninit data is written to ibuf + let ibuf = unsafe { &mut buf.as_mut()[..limit] }; + + let mut sliced_buf: BorrowedBuf<'_> = ibuf.into(); + + // SAFETY: extra_init bytes of ibuf are known to be initialized + unsafe { + sliced_buf.set_init(extra_init); + } + + let mut cursor = sliced_buf.unfilled(); + self.inner.read_buf(cursor.reborrow())?; + + let new_init = cursor.init_ref().len(); + let filled = sliced_buf.len(); + + // cursor / sliced_buf / ibuf must drop here + + unsafe { + // SAFETY: filled bytes have been filled and therefore initialized + buf.advance(filled); + // SAFETY: new_init bytes of buf's unfilled buffer have been initialized + buf.set_init(new_init); + } + + self.limit -= filled as u64; + } else { + let written = buf.written(); + self.inner.read_buf(buf.reborrow())?; + self.limit -= (buf.written() - written) as u64; + } + + Ok(()) } } @@ -2312,12 +2665,27 @@ impl BufRead for Take { } } +impl SizeHint for Take { + #[inline] + fn lower_bound(&self) -> usize { + cmp::min(SizeHint::lower_bound(&self.inner) as u64, self.limit) as usize + } + + #[inline] + fn upper_bound(&self) -> Option { + match SizeHint::upper_bound(&self.inner) { + Some(upper_bound) => Some(cmp::min(upper_bound as u64, self.limit) as usize), + None => self.limit.try_into().ok(), + } + } +} + /// An iterator over `u8` values of a reader. /// /// This struct is generally created by calling [`bytes`] on a reader. /// Please see the documentation of [`bytes`] for more details. /// -/// [`bytes`]: trait.Read.html#method.bytes +/// [`bytes`]: Read::bytes #[stable(feature = "rust1", since = "1.0.0")] #[derive(Debug)] pub struct Bytes { @@ -2339,6 +2707,68 @@ impl Iterator for Bytes { }; } } + + fn size_hint(&self) -> (usize, Option) { + SizeHint::size_hint(&self.inner) + } +} + +trait SizeHint { + fn lower_bound(&self) -> usize; + + fn upper_bound(&self) -> Option; + + fn size_hint(&self) -> (usize, Option) { + (self.lower_bound(), self.upper_bound()) + } +} + +impl SizeHint for T { + #[inline] + default fn lower_bound(&self) -> usize { + 0 + } + + #[inline] + default fn upper_bound(&self) -> Option { + None + } +} + +impl SizeHint for &mut T { + #[inline] + fn lower_bound(&self) -> usize { + SizeHint::lower_bound(*self) + } + + #[inline] + fn upper_bound(&self) -> Option { + SizeHint::upper_bound(*self) + } +} + +impl SizeHint for Box { + #[inline] + fn lower_bound(&self) -> usize { + SizeHint::lower_bound(&**self) + } + + #[inline] + fn upper_bound(&self) -> Option { + SizeHint::upper_bound(&**self) + } +} + +impl SizeHint for &[u8] { + #[inline] + fn lower_bound(&self) -> usize { + self.len() + } + + #[inline] + fn upper_bound(&self) -> Option { + Some(self.len()) + } } /// An iterator over the contents of an instance of `BufRead` split on a @@ -2347,7 +2777,7 @@ impl Iterator for Bytes { /// This struct is generally created by calling [`split`] on a `BufRead`. /// Please see the documentation of [`split`] for more details. /// -/// [`split`]: trait.BufRead.html#method.split +/// [`split`]: BufRead::split #[stable(feature = "rust1", since = "1.0.0")] #[derive(Debug)] pub struct Split { @@ -2379,7 +2809,7 @@ impl Iterator for Split { /// This struct is generally created by calling [`lines`] on a `BufRead`. /// Please see the documentation of [`lines`] for more details. /// -/// [`lines`]: trait.BufRead.html#method.lines +/// [`lines`]: BufRead::lines #[stable(feature = "rust1", since = "1.0.0")] #[derive(Debug)] pub struct Lines { @@ -2407,397 +2837,3 @@ impl Iterator for Lines { } } } - -#[cfg(test)] -mod tests { - use super::{repeat, Cursor, SeekFrom}; - use crate::cmp; - use crate::io::prelude::*; - use crate::io::{self, IoSlice, IoSliceMut}; - use crate::ops::Deref; - - #[test] - #[cfg_attr(target_os = "emscripten", ignore)] - fn read_until() { - let mut buf = Cursor::new(&b"12"[..]); - let mut v = Vec::new(); - assert_eq!(buf.read_until(b'3', &mut v).unwrap(), 2); - assert_eq!(v, b"12"); - - let mut buf = Cursor::new(&b"1233"[..]); - let mut v = Vec::new(); - assert_eq!(buf.read_until(b'3', &mut v).unwrap(), 3); - assert_eq!(v, b"123"); - v.truncate(0); - assert_eq!(buf.read_until(b'3', &mut v).unwrap(), 1); - assert_eq!(v, b"3"); - v.truncate(0); - assert_eq!(buf.read_until(b'3', &mut v).unwrap(), 0); - assert_eq!(v, []); - } - - #[test] - fn split() { - let buf = Cursor::new(&b"12"[..]); - let mut s = buf.split(b'3'); - assert_eq!(s.next().unwrap().unwrap(), vec![b'1', b'2']); - assert!(s.next().is_none()); - - let buf = Cursor::new(&b"1233"[..]); - let mut s = buf.split(b'3'); - assert_eq!(s.next().unwrap().unwrap(), vec![b'1', b'2']); - assert_eq!(s.next().unwrap().unwrap(), vec![]); - assert!(s.next().is_none()); - } - - #[test] - fn read_line() { - let mut buf = Cursor::new(&b"12"[..]); - let mut v = String::new(); - assert_eq!(buf.read_line(&mut v).unwrap(), 2); - assert_eq!(v, "12"); - - let mut buf = Cursor::new(&b"12\n\n"[..]); - let mut v = String::new(); - assert_eq!(buf.read_line(&mut v).unwrap(), 3); - assert_eq!(v, "12\n"); - v.truncate(0); - assert_eq!(buf.read_line(&mut v).unwrap(), 1); - assert_eq!(v, "\n"); - v.truncate(0); - assert_eq!(buf.read_line(&mut v).unwrap(), 0); - assert_eq!(v, ""); - } - - #[test] - fn lines() { - let buf = Cursor::new(&b"12\r"[..]); - let mut s = buf.lines(); - assert_eq!(s.next().unwrap().unwrap(), "12\r".to_string()); - assert!(s.next().is_none()); - - let buf = Cursor::new(&b"12\r\n\n"[..]); - let mut s = buf.lines(); - assert_eq!(s.next().unwrap().unwrap(), "12".to_string()); - assert_eq!(s.next().unwrap().unwrap(), "".to_string()); - assert!(s.next().is_none()); - } - - #[test] - fn read_to_end() { - let mut c = Cursor::new(&b""[..]); - let mut v = Vec::new(); - assert_eq!(c.read_to_end(&mut v).unwrap(), 0); - assert_eq!(v, []); - - let mut c = Cursor::new(&b"1"[..]); - let mut v = Vec::new(); - assert_eq!(c.read_to_end(&mut v).unwrap(), 1); - assert_eq!(v, b"1"); - - let cap = 1024 * 1024; - let data = (0..cap).map(|i| (i / 3) as u8).collect::>(); - let mut v = Vec::new(); - let (a, b) = data.split_at(data.len() / 2); - assert_eq!(Cursor::new(a).read_to_end(&mut v).unwrap(), a.len()); - assert_eq!(Cursor::new(b).read_to_end(&mut v).unwrap(), b.len()); - assert_eq!(v, data); - } - - #[test] - fn read_to_string() { - let mut c = Cursor::new(&b""[..]); - let mut v = String::new(); - assert_eq!(c.read_to_string(&mut v).unwrap(), 0); - assert_eq!(v, ""); - - let mut c = Cursor::new(&b"1"[..]); - let mut v = String::new(); - assert_eq!(c.read_to_string(&mut v).unwrap(), 1); - assert_eq!(v, "1"); - - let mut c = Cursor::new(&b"\xff"[..]); - let mut v = String::new(); - assert!(c.read_to_string(&mut v).is_err()); - } - - #[test] - fn read_exact() { - let mut buf = [0; 4]; - - let mut c = Cursor::new(&b""[..]); - assert_eq!(c.read_exact(&mut buf).unwrap_err().kind(), io::ErrorKind::UnexpectedEof); - - let mut c = Cursor::new(&b"123"[..]).chain(Cursor::new(&b"456789"[..])); - c.read_exact(&mut buf).unwrap(); - assert_eq!(&buf, b"1234"); - c.read_exact(&mut buf).unwrap(); - assert_eq!(&buf, b"5678"); - assert_eq!(c.read_exact(&mut buf).unwrap_err().kind(), io::ErrorKind::UnexpectedEof); - } - - #[test] - fn read_exact_slice() { - let mut buf = [0; 4]; - - let mut c = &b""[..]; - assert_eq!(c.read_exact(&mut buf).unwrap_err().kind(), io::ErrorKind::UnexpectedEof); - - let mut c = &b"123"[..]; - assert_eq!(c.read_exact(&mut buf).unwrap_err().kind(), io::ErrorKind::UnexpectedEof); - // make sure the optimized (early returning) method is being used - assert_eq!(&buf, &[0; 4]); - - let mut c = &b"1234"[..]; - c.read_exact(&mut buf).unwrap(); - assert_eq!(&buf, b"1234"); - - let mut c = &b"56789"[..]; - c.read_exact(&mut buf).unwrap(); - assert_eq!(&buf, b"5678"); - assert_eq!(c, b"9"); - } - - #[test] - fn take_eof() { - struct R; - - impl Read for R { - fn read(&mut self, _: &mut [u8]) -> io::Result { - Err(io::Error::new(io::ErrorKind::Other, "")) - } - } - impl BufRead for R { - fn fill_buf(&mut self) -> io::Result<&[u8]> { - Err(io::Error::new(io::ErrorKind::Other, "")) - } - fn consume(&mut self, _amt: usize) {} - } - - let mut buf = [0; 1]; - assert_eq!(0, R.take(0).read(&mut buf).unwrap()); - assert_eq!(b"", R.take(0).fill_buf().unwrap()); - } - - fn cmp_bufread(mut br1: Br1, mut br2: Br2, exp: &[u8]) { - let mut cat = Vec::new(); - loop { - let consume = { - let buf1 = br1.fill_buf().unwrap(); - let buf2 = br2.fill_buf().unwrap(); - let minlen = if buf1.len() < buf2.len() { buf1.len() } else { buf2.len() }; - assert_eq!(buf1[..minlen], buf2[..minlen]); - cat.extend_from_slice(&buf1[..minlen]); - minlen - }; - if consume == 0 { - break; - } - br1.consume(consume); - br2.consume(consume); - } - assert_eq!(br1.fill_buf().unwrap().len(), 0); - assert_eq!(br2.fill_buf().unwrap().len(), 0); - assert_eq!(&cat[..], &exp[..]) - } - - #[test] - fn chain_bufread() { - let testdata = b"ABCDEFGHIJKL"; - let chain1 = - (&testdata[..3]).chain(&testdata[3..6]).chain(&testdata[6..9]).chain(&testdata[9..]); - let chain2 = (&testdata[..4]).chain(&testdata[4..8]).chain(&testdata[8..]); - cmp_bufread(chain1, chain2, &testdata[..]); - } - - #[test] - fn chain_zero_length_read_is_not_eof() { - let a = b"A"; - let b = b"B"; - let mut s = String::new(); - let mut chain = (&a[..]).chain(&b[..]); - chain.read(&mut []).unwrap(); - chain.read_to_string(&mut s).unwrap(); - assert_eq!("AB", s); - } - - #[bench] - #[cfg_attr(target_os = "emscripten", ignore)] - fn bench_read_to_end(b: &mut test::Bencher) { - b.iter(|| { - let mut lr = repeat(1).take(10000000); - let mut vec = Vec::with_capacity(1024); - super::read_to_end(&mut lr, &mut vec) - }); - } - - #[test] - fn seek_len() -> io::Result<()> { - let mut c = Cursor::new(vec![0; 15]); - assert_eq!(c.stream_len()?, 15); - - c.seek(SeekFrom::End(0))?; - let old_pos = c.stream_position()?; - assert_eq!(c.stream_len()?, 15); - assert_eq!(c.stream_position()?, old_pos); - - c.seek(SeekFrom::Start(7))?; - c.seek(SeekFrom::Current(2))?; - let old_pos = c.stream_position()?; - assert_eq!(c.stream_len()?, 15); - assert_eq!(c.stream_position()?, old_pos); - - Ok(()) - } - - #[test] - fn seek_position() -> io::Result<()> { - // All `asserts` are duplicated here to make sure the method does not - // change anything about the seek state. - let mut c = Cursor::new(vec![0; 15]); - assert_eq!(c.stream_position()?, 0); - assert_eq!(c.stream_position()?, 0); - - c.seek(SeekFrom::End(0))?; - assert_eq!(c.stream_position()?, 15); - assert_eq!(c.stream_position()?, 15); - - c.seek(SeekFrom::Start(7))?; - c.seek(SeekFrom::Current(2))?; - assert_eq!(c.stream_position()?, 9); - assert_eq!(c.stream_position()?, 9); - - c.seek(SeekFrom::End(-3))?; - c.seek(SeekFrom::Current(1))?; - c.seek(SeekFrom::Current(-5))?; - assert_eq!(c.stream_position()?, 8); - assert_eq!(c.stream_position()?, 8); - - Ok(()) - } - - // A simple example reader which uses the default implementation of - // read_to_end. - struct ExampleSliceReader<'a> { - slice: &'a [u8], - } - - impl<'a> Read for ExampleSliceReader<'a> { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - let len = cmp::min(self.slice.len(), buf.len()); - buf[..len].copy_from_slice(&self.slice[..len]); - self.slice = &self.slice[len..]; - Ok(len) - } - } - - #[test] - fn test_read_to_end_capacity() -> io::Result<()> { - let input = &b"foo"[..]; - - // read_to_end() generally needs to over-allocate, both for efficiency - // and so that it can distinguish EOF. Assert that this is the case - // with this simple ExampleSliceReader struct, which uses the default - // implementation of read_to_end. Even though vec1 is allocated with - // exactly enough capacity for the read, read_to_end will allocate more - // space here. - let mut vec1 = Vec::with_capacity(input.len()); - ExampleSliceReader { slice: input }.read_to_end(&mut vec1)?; - assert_eq!(vec1.len(), input.len()); - assert!(vec1.capacity() > input.len(), "allocated more"); - - // However, std::io::Take includes an implementation of read_to_end - // that will not allocate when the limit has already been reached. In - // this case, vec2 never grows. - let mut vec2 = Vec::with_capacity(input.len()); - ExampleSliceReader { slice: input }.take(input.len() as u64).read_to_end(&mut vec2)?; - assert_eq!(vec2.len(), input.len()); - assert_eq!(vec2.capacity(), input.len(), "did not allocate more"); - - Ok(()) - } - - #[test] - fn io_slice_mut_advance() { - let mut buf1 = [1; 8]; - let mut buf2 = [2; 16]; - let mut buf3 = [3; 8]; - let mut bufs = &mut [ - IoSliceMut::new(&mut buf1), - IoSliceMut::new(&mut buf2), - IoSliceMut::new(&mut buf3), - ][..]; - - // Only in a single buffer.. - bufs = IoSliceMut::advance(bufs, 1); - assert_eq!(bufs[0].deref(), [1; 7].as_ref()); - assert_eq!(bufs[1].deref(), [2; 16].as_ref()); - assert_eq!(bufs[2].deref(), [3; 8].as_ref()); - - // Removing a buffer, leaving others as is. - bufs = IoSliceMut::advance(bufs, 7); - assert_eq!(bufs[0].deref(), [2; 16].as_ref()); - assert_eq!(bufs[1].deref(), [3; 8].as_ref()); - - // Removing a buffer and removing from the next buffer. - bufs = IoSliceMut::advance(bufs, 18); - assert_eq!(bufs[0].deref(), [3; 6].as_ref()); - } - - #[test] - fn io_slice_mut_advance_empty_slice() { - let empty_bufs = &mut [][..]; - // Shouldn't panic. - IoSliceMut::advance(empty_bufs, 1); - } - - #[test] - fn io_slice_mut_advance_beyond_total_length() { - let mut buf1 = [1; 8]; - let mut bufs = &mut [IoSliceMut::new(&mut buf1)][..]; - - // Going beyond the total length should be ok. - bufs = IoSliceMut::advance(bufs, 9); - assert!(bufs.is_empty()); - } - - #[test] - fn io_slice_advance() { - let buf1 = [1; 8]; - let buf2 = [2; 16]; - let buf3 = [3; 8]; - let mut bufs = &mut [IoSlice::new(&buf1), IoSlice::new(&buf2), IoSlice::new(&buf3)][..]; - - // Only in a single buffer.. - bufs = IoSlice::advance(bufs, 1); - assert_eq!(bufs[0].deref(), [1; 7].as_ref()); - assert_eq!(bufs[1].deref(), [2; 16].as_ref()); - assert_eq!(bufs[2].deref(), [3; 8].as_ref()); - - // Removing a buffer, leaving others as is. - bufs = IoSlice::advance(bufs, 7); - assert_eq!(bufs[0].deref(), [2; 16].as_ref()); - assert_eq!(bufs[1].deref(), [3; 8].as_ref()); - - // Removing a buffer and removing from the next buffer. - bufs = IoSlice::advance(bufs, 18); - assert_eq!(bufs[0].deref(), [3; 6].as_ref()); - } - - #[test] - fn io_slice_advance_empty_slice() { - let empty_bufs = &mut [][..]; - // Shouldn't panic. - IoSlice::advance(empty_bufs, 1); - } - - #[test] - fn io_slice_advance_beyond_total_length() { - let buf1 = [1; 8]; - let mut bufs = &mut [IoSlice::new(&buf1)][..]; - - // Going beyond the total length should be ok. - bufs = IoSlice::advance(bufs, 9); - assert!(bufs.is_empty()); - } -} diff --git a/crux-mir/lib/std/src/io/prelude.rs b/crux-mir/lib/std/src/io/prelude.rs index 3baab2be3..d80643101 100644 --- a/crux-mir/lib/std/src/io/prelude.rs +++ b/crux-mir/lib/std/src/io/prelude.rs @@ -1,4 +1,4 @@ -//! The I/O Prelude +//! The I/O Prelude. //! //! The purpose of this module is to alleviate imports of many common I/O traits //! by adding a glob import to the top of I/O heavy modules: diff --git a/crux-mir/lib/std/src/io/readbuf.rs b/crux-mir/lib/std/src/io/readbuf.rs new file mode 100644 index 000000000..4800eeda0 --- /dev/null +++ b/crux-mir/lib/std/src/io/readbuf.rs @@ -0,0 +1,309 @@ +#![unstable(feature = "read_buf", issue = "78485")] + +#[cfg(test)] +mod tests; + +use crate::fmt::{self, Debug, Formatter}; +use crate::io::{Result, Write}; +use crate::mem::{self, MaybeUninit}; +use crate::{cmp, ptr}; + +/// A borrowed byte buffer which is incrementally filled and initialized. +/// +/// This type is a sort of "double cursor". It tracks three regions in the buffer: a region at the beginning of the +/// buffer that has been logically filled with data, a region that has been initialized at some point but not yet +/// logically filled, and a region at the end that is fully uninitialized. The filled region is guaranteed to be a +/// subset of the initialized region. +/// +/// In summary, the contents of the buffer can be visualized as: +/// ```not_rust +/// [ capacity ] +/// [ filled | unfilled ] +/// [ initialized | uninitialized ] +/// ``` +/// +/// A `BorrowedBuf` is created around some existing data (or capacity for data) via a unique reference +/// (`&mut`). The `BorrowedBuf` can be configured (e.g., using `clear` or `set_init`), but cannot be +/// directly written. To write into the buffer, use `unfilled` to create a `BorrowedCursor`. The cursor +/// has write-only access to the unfilled portion of the buffer (you can think of it as a +/// write-only iterator). +/// +/// The lifetime `'data` is a bound on the lifetime of the underlying data. +pub struct BorrowedBuf<'data> { + /// The buffer's underlying data. + buf: &'data mut [MaybeUninit], + /// The length of `self.buf` which is known to be filled. + filled: usize, + /// The length of `self.buf` which is known to be initialized. + init: usize, +} + +impl Debug for BorrowedBuf<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("BorrowedBuf") + .field("init", &self.init) + .field("filled", &self.filled) + .field("capacity", &self.capacity()) + .finish() + } +} + +/// Create a new `BorrowedBuf` from a fully initialized slice. +impl<'data> From<&'data mut [u8]> for BorrowedBuf<'data> { + #[inline] + fn from(slice: &'data mut [u8]) -> BorrowedBuf<'data> { + let len = slice.len(); + + BorrowedBuf { + // SAFETY: initialized data never becoming uninitialized is an invariant of BorrowedBuf + buf: unsafe { (slice as *mut [u8]).as_uninit_slice_mut().unwrap() }, + filled: 0, + init: len, + } + } +} + +/// Create a new `BorrowedBuf` from an uninitialized buffer. +/// +/// Use `set_init` if part of the buffer is known to be already initialized. +impl<'data> From<&'data mut [MaybeUninit]> for BorrowedBuf<'data> { + #[inline] + fn from(buf: &'data mut [MaybeUninit]) -> BorrowedBuf<'data> { + BorrowedBuf { buf, filled: 0, init: 0 } + } +} + +impl<'data> BorrowedBuf<'data> { + /// Returns the total capacity of the buffer. + #[inline] + pub fn capacity(&self) -> usize { + self.buf.len() + } + + /// Returns the length of the filled part of the buffer. + #[inline] + pub fn len(&self) -> usize { + self.filled + } + + /// Returns the length of the initialized part of the buffer. + #[inline] + pub fn init_len(&self) -> usize { + self.init + } + + /// Returns a shared reference to the filled portion of the buffer. + #[inline] + pub fn filled(&self) -> &[u8] { + // SAFETY: We only slice the filled part of the buffer, which is always valid + unsafe { MaybeUninit::slice_assume_init_ref(&self.buf[0..self.filled]) } + } + + /// Returns a cursor over the unfilled part of the buffer. + #[inline] + pub fn unfilled<'this>(&'this mut self) -> BorrowedCursor<'this> { + BorrowedCursor { + start: self.filled, + // SAFETY: we never assign into `BorrowedCursor::buf`, so treating its + // lifetime covariantly is safe. + buf: unsafe { + mem::transmute::<&'this mut BorrowedBuf<'data>, &'this mut BorrowedBuf<'this>>(self) + }, + } + } + + /// Clears the buffer, resetting the filled region to empty. + /// + /// The number of initialized bytes is not changed, and the contents of the buffer are not modified. + #[inline] + pub fn clear(&mut self) -> &mut Self { + self.filled = 0; + self + } + + /// Asserts that the first `n` bytes of the buffer are initialized. + /// + /// `BorrowedBuf` assumes that bytes are never de-initialized, so this method does nothing when called with fewer + /// bytes than are already known to be initialized. + /// + /// # Safety + /// + /// The caller must ensure that the first `n` unfilled bytes of the buffer have already been initialized. + #[inline] + pub unsafe fn set_init(&mut self, n: usize) -> &mut Self { + self.init = cmp::max(self.init, n); + self + } +} + +/// A writeable view of the unfilled portion of a [`BorrowedBuf`](BorrowedBuf). +/// +/// Provides access to the initialized and uninitialized parts of the underlying `BorrowedBuf`. +/// Data can be written directly to the cursor by using [`append`](BorrowedCursor::append) or +/// indirectly by getting a slice of part or all of the cursor and writing into the slice. In the +/// indirect case, the caller must call [`advance`](BorrowedCursor::advance) after writing to inform +/// the cursor how many bytes have been written. +/// +/// Once data is written to the cursor, it becomes part of the filled portion of the underlying +/// `BorrowedBuf` and can no longer be accessed or re-written by the cursor. I.e., the cursor tracks +/// the unfilled part of the underlying `BorrowedBuf`. +/// +/// The lifetime `'a` is a bound on the lifetime of the underlying buffer (which means it is a bound +/// on the data in that buffer by transitivity). +#[derive(Debug)] +pub struct BorrowedCursor<'a> { + /// The underlying buffer. + // Safety invariant: we treat the type of buf as covariant in the lifetime of `BorrowedBuf` when + // we create a `BorrowedCursor`. This is only safe if we never replace `buf` by assigning into + // it, so don't do that! + buf: &'a mut BorrowedBuf<'a>, + /// The length of the filled portion of the underlying buffer at the time of the cursor's + /// creation. + start: usize, +} + +impl<'a> BorrowedCursor<'a> { + /// Reborrow this cursor by cloning it with a smaller lifetime. + /// + /// Since a cursor maintains unique access to its underlying buffer, the borrowed cursor is + /// not accessible while the new cursor exists. + #[inline] + pub fn reborrow<'this>(&'this mut self) -> BorrowedCursor<'this> { + BorrowedCursor { + // SAFETY: we never assign into `BorrowedCursor::buf`, so treating its + // lifetime covariantly is safe. + buf: unsafe { + mem::transmute::<&'this mut BorrowedBuf<'a>, &'this mut BorrowedBuf<'this>>( + self.buf, + ) + }, + start: self.start, + } + } + + /// Returns the available space in the cursor. + #[inline] + pub fn capacity(&self) -> usize { + self.buf.capacity() - self.buf.filled + } + + /// Returns the number of bytes written to this cursor since it was created from a `BorrowedBuf`. + /// + /// Note that if this cursor is a reborrowed clone of another, then the count returned is the + /// count written via either cursor, not the count since the cursor was reborrowed. + #[inline] + pub fn written(&self) -> usize { + self.buf.filled - self.start + } + + /// Returns a shared reference to the initialized portion of the cursor. + #[inline] + pub fn init_ref(&self) -> &[u8] { + // SAFETY: We only slice the initialized part of the buffer, which is always valid + unsafe { MaybeUninit::slice_assume_init_ref(&self.buf.buf[self.buf.filled..self.buf.init]) } + } + + /// Returns a mutable reference to the initialized portion of the cursor. + #[inline] + pub fn init_mut(&mut self) -> &mut [u8] { + // SAFETY: We only slice the initialized part of the buffer, which is always valid + unsafe { + MaybeUninit::slice_assume_init_mut(&mut self.buf.buf[self.buf.filled..self.buf.init]) + } + } + + /// Returns a mutable reference to the uninitialized part of the cursor. + /// + /// It is safe to uninitialize any of these bytes. + #[inline] + pub fn uninit_mut(&mut self) -> &mut [MaybeUninit] { + &mut self.buf.buf[self.buf.init..] + } + + /// Returns a mutable reference to the whole cursor. + /// + /// # Safety + /// + /// The caller must not uninitialize any bytes in the initialized portion of the cursor. + #[inline] + pub unsafe fn as_mut(&mut self) -> &mut [MaybeUninit] { + &mut self.buf.buf[self.buf.filled..] + } + + /// Advance the cursor by asserting that `n` bytes have been filled. + /// + /// After advancing, the `n` bytes are no longer accessible via the cursor and can only be + /// accessed via the underlying buffer. I.e., the buffer's filled portion grows by `n` elements + /// and its unfilled portion (and the capacity of this cursor) shrinks by `n` elements. + /// + /// # Safety + /// + /// The caller must ensure that the first `n` bytes of the cursor have been properly + /// initialised. + #[inline] + pub unsafe fn advance(&mut self, n: usize) -> &mut Self { + self.buf.filled += n; + self.buf.init = cmp::max(self.buf.init, self.buf.filled); + self + } + + /// Initializes all bytes in the cursor. + #[inline] + pub fn ensure_init(&mut self) -> &mut Self { + let uninit = self.uninit_mut(); + // SAFETY: 0 is a valid value for MaybeUninit and the length matches the allocation + // since it is comes from a slice reference. + unsafe { + ptr::write_bytes(uninit.as_mut_ptr(), 0, uninit.len()); + } + self.buf.init = self.buf.capacity(); + + self + } + + /// Asserts that the first `n` unfilled bytes of the cursor are initialized. + /// + /// `BorrowedBuf` assumes that bytes are never de-initialized, so this method does nothing when + /// called with fewer bytes than are already known to be initialized. + /// + /// # Safety + /// + /// The caller must ensure that the first `n` bytes of the buffer have already been initialized. + #[inline] + pub unsafe fn set_init(&mut self, n: usize) -> &mut Self { + self.buf.init = cmp::max(self.buf.init, self.buf.filled + n); + self + } + + /// Appends data to the cursor, advancing position within its buffer. + /// + /// # Panics + /// + /// Panics if `self.capacity()` is less than `buf.len()`. + #[inline] + pub fn append(&mut self, buf: &[u8]) { + assert!(self.capacity() >= buf.len()); + + // SAFETY: we do not de-initialize any of the elements of the slice + unsafe { + MaybeUninit::write_slice(&mut self.as_mut()[..buf.len()], buf); + } + + // SAFETY: We just added the entire contents of buf to the filled section. + unsafe { + self.set_init(buf.len()); + } + self.buf.filled += buf.len(); + } +} + +impl<'a> Write for BorrowedCursor<'a> { + fn write(&mut self, buf: &[u8]) -> Result { + self.append(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> Result<()> { + Ok(()) + } +} diff --git a/crux-mir/lib/std/src/io/readbuf/tests.rs b/crux-mir/lib/std/src/io/readbuf/tests.rs new file mode 100644 index 000000000..cc1b423f2 --- /dev/null +++ b/crux-mir/lib/std/src/io/readbuf/tests.rs @@ -0,0 +1,175 @@ +use super::BorrowedBuf; +use crate::mem::MaybeUninit; + +/// Test that BorrowedBuf has the correct numbers when created with new +#[test] +fn new() { + let buf: &mut [_] = &mut [0; 16]; + let mut rbuf: BorrowedBuf<'_> = buf.into(); + + assert_eq!(rbuf.filled().len(), 0); + assert_eq!(rbuf.init_len(), 16); + assert_eq!(rbuf.capacity(), 16); + assert_eq!(rbuf.unfilled().capacity(), 16); +} + +/// Test that BorrowedBuf has the correct numbers when created with uninit +#[test] +fn uninit() { + let buf: &mut [_] = &mut [MaybeUninit::uninit(); 16]; + let mut rbuf: BorrowedBuf<'_> = buf.into(); + + assert_eq!(rbuf.filled().len(), 0); + assert_eq!(rbuf.init_len(), 0); + assert_eq!(rbuf.capacity(), 16); + assert_eq!(rbuf.unfilled().capacity(), 16); +} + +#[test] +fn initialize_unfilled() { + let buf: &mut [_] = &mut [MaybeUninit::uninit(); 16]; + let mut rbuf: BorrowedBuf<'_> = buf.into(); + + rbuf.unfilled().ensure_init(); + + assert_eq!(rbuf.init_len(), 16); +} + +#[test] +fn addvance_filled() { + let buf: &mut [_] = &mut [0; 16]; + let mut rbuf: BorrowedBuf<'_> = buf.into(); + + unsafe { + rbuf.unfilled().advance(1); + } + + assert_eq!(rbuf.filled().len(), 1); + assert_eq!(rbuf.unfilled().capacity(), 15); +} + +#[test] +fn clear() { + let buf: &mut [_] = &mut [255; 16]; + let mut rbuf: BorrowedBuf<'_> = buf.into(); + + unsafe { + rbuf.unfilled().advance(16); + } + + assert_eq!(rbuf.filled().len(), 16); + assert_eq!(rbuf.unfilled().capacity(), 0); + + rbuf.clear(); + + assert_eq!(rbuf.filled().len(), 0); + assert_eq!(rbuf.unfilled().capacity(), 16); + + assert_eq!(rbuf.unfilled().init_ref(), [255; 16]); +} + +#[test] +fn set_init() { + let buf: &mut [_] = &mut [MaybeUninit::uninit(); 16]; + let mut rbuf: BorrowedBuf<'_> = buf.into(); + + unsafe { + rbuf.set_init(8); + } + + assert_eq!(rbuf.init_len(), 8); + + unsafe { + rbuf.unfilled().advance(4); + } + + unsafe { + rbuf.set_init(2); + } + + assert_eq!(rbuf.init_len(), 8); + + unsafe { + rbuf.set_init(8); + } + + assert_eq!(rbuf.init_len(), 8); +} + +#[test] +fn append() { + let buf: &mut [_] = &mut [MaybeUninit::new(255); 16]; + let mut rbuf: BorrowedBuf<'_> = buf.into(); + + rbuf.unfilled().append(&[0; 8]); + + assert_eq!(rbuf.init_len(), 8); + assert_eq!(rbuf.filled().len(), 8); + assert_eq!(rbuf.filled(), [0; 8]); + + rbuf.clear(); + + rbuf.unfilled().append(&[1; 16]); + + assert_eq!(rbuf.init_len(), 16); + assert_eq!(rbuf.filled().len(), 16); + assert_eq!(rbuf.filled(), [1; 16]); +} + +#[test] +fn reborrow_written() { + let buf: &mut [_] = &mut [MaybeUninit::new(0); 32]; + let mut buf: BorrowedBuf<'_> = buf.into(); + + let mut cursor = buf.unfilled(); + cursor.append(&[1; 16]); + + let mut cursor2 = cursor.reborrow(); + cursor2.append(&[2; 16]); + + assert_eq!(cursor2.written(), 32); + assert_eq!(cursor.written(), 32); + + assert_eq!(buf.unfilled().written(), 0); + assert_eq!(buf.init_len(), 32); + assert_eq!(buf.filled().len(), 32); + let filled = buf.filled(); + assert_eq!(&filled[..16], [1; 16]); + assert_eq!(&filled[16..], [2; 16]); +} + +#[test] +fn cursor_set_init() { + let buf: &mut [_] = &mut [MaybeUninit::uninit(); 16]; + let mut rbuf: BorrowedBuf<'_> = buf.into(); + + unsafe { + rbuf.unfilled().set_init(8); + } + + assert_eq!(rbuf.init_len(), 8); + assert_eq!(rbuf.unfilled().init_ref().len(), 8); + assert_eq!(rbuf.unfilled().init_mut().len(), 8); + assert_eq!(rbuf.unfilled().uninit_mut().len(), 8); + assert_eq!(unsafe { rbuf.unfilled().as_mut() }.len(), 16); + + unsafe { + rbuf.unfilled().advance(4); + } + + unsafe { + rbuf.unfilled().set_init(2); + } + + assert_eq!(rbuf.init_len(), 8); + + unsafe { + rbuf.unfilled().set_init(8); + } + + assert_eq!(rbuf.init_len(), 12); + assert_eq!(rbuf.unfilled().init_ref().len(), 8); + assert_eq!(rbuf.unfilled().init_mut().len(), 8); + assert_eq!(rbuf.unfilled().uninit_mut().len(), 4); + assert_eq!(unsafe { rbuf.unfilled().as_mut() }.len(), 12); +} diff --git a/crux-mir/lib/std/src/io/stdio.rs b/crux-mir/lib/std/src/io/stdio.rs index 9a82ae762..14bfef4c7 100644 --- a/crux-mir/lib/std/src/io/stdio.rs +++ b/crux-mir/lib/std/src/io/stdio.rs @@ -1,30 +1,41 @@ #![cfg_attr(test, allow(unused))] +#[cfg(test)] +mod tests; + use crate::io::prelude::*; -use crate::cell::RefCell; +use crate::cell::{Cell, RefCell}; use crate::fmt; -use crate::io::lazy::Lazy; -use crate::io::{self, BufReader, Initializer, IoSlice, IoSliceMut, LineWriter}; -use crate::sync::{Arc, Mutex, MutexGuard, Once}; +use crate::fs::File; +use crate::io::{self, BufReader, IoSlice, IoSliceMut, LineWriter, Lines}; +use crate::sync::atomic::{AtomicBool, Ordering}; +use crate::sync::{Arc, Mutex, MutexGuard, OnceLock, ReentrantMutex, ReentrantMutexGuard}; use crate::sys::stdio; -use crate::sys_common::remutex::{ReentrantMutex, ReentrantMutexGuard}; -use crate::thread::LocalKey; -thread_local! { - /// Stdout used by print! and println! macros - static LOCAL_STDOUT: RefCell>> = { - RefCell::new(None) - } -} +type LocalStream = Arc>>; thread_local! { - /// Stderr used by eprint! and eprintln! macros, and panics - static LOCAL_STDERR: RefCell>> = { - RefCell::new(None) + /// Used by the test crate to capture the output of the print macros and panics. + static OUTPUT_CAPTURE: Cell> = { + Cell::new(None) } } +/// Flag to indicate OUTPUT_CAPTURE is used. +/// +/// If it is None and was never set on any thread, this flag is set to false, +/// and OUTPUT_CAPTURE can be safely ignored on all threads, saving some time +/// and memory registering an unused thread local. +/// +/// Note about memory ordering: This contains information about whether a +/// thread local variable might be in use. Although this is a global flag, the +/// memory ordering between threads does not matter: we only want this flag to +/// have a consistent order between set_output_capture and print_to *within +/// the same thread*. Within the same thread, things always have a perfectly +/// consistent order. So Ordering::Relaxed is fine. +static OUTPUT_CAPTURE_USED: AtomicBool = AtomicBool::new(false); + /// A handle to a raw instance of the standard input stream of this process. /// /// This handle is not synchronized or buffered in any fashion. Constructed via @@ -50,8 +61,9 @@ struct StderrRaw(stdio::Stderr); /// handles is **not** available to raw handles returned from this function. /// /// The returned handle has no external synchronization or buffering. -fn stdin_raw() -> io::Result { - stdio::Stdin::new().map(StdinRaw) +#[unstable(feature = "libstd_sys_internals", issue = "none")] +const fn stdin_raw() -> StdinRaw { + StdinRaw(stdio::Stdin::new()) } /// Constructs a new raw handle to the standard output stream of this process. @@ -63,8 +75,9 @@ fn stdin_raw() -> io::Result { /// /// The returned handle has no external synchronization or buffering layered on /// top. -fn stdout_raw() -> io::Result { - stdio::Stdout::new().map(StdoutRaw) +#[unstable(feature = "libstd_sys_internals", issue = "none")] +const fn stdout_raw() -> StdoutRaw { + StdoutRaw(stdio::Stdout::new()) } /// Constructs a new raw handle to the standard error stream of this process. @@ -74,93 +87,95 @@ fn stdout_raw() -> io::Result { /// /// The returned handle has no external synchronization or buffering layered on /// top. -fn stderr_raw() -> io::Result { - stdio::Stderr::new().map(StderrRaw) +#[unstable(feature = "libstd_sys_internals", issue = "none")] +const fn stderr_raw() -> StderrRaw { + StderrRaw(stdio::Stderr::new()) } impl Read for StdinRaw { fn read(&mut self, buf: &mut [u8]) -> io::Result { - self.0.read(buf) + handle_ebadf(self.0.read(buf), 0) } fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { - self.0.read_vectored(bufs) + handle_ebadf(self.0.read_vectored(bufs), 0) } #[inline] - unsafe fn initializer(&self) -> Initializer { - Initializer::nop() + fn is_read_vectored(&self) -> bool { + self.0.is_read_vectored() + } + + fn read_to_end(&mut self, buf: &mut Vec) -> io::Result { + handle_ebadf(self.0.read_to_end(buf), 0) + } + + fn read_to_string(&mut self, buf: &mut String) -> io::Result { + handle_ebadf(self.0.read_to_string(buf), 0) } } + impl Write for StdoutRaw { fn write(&mut self, buf: &[u8]) -> io::Result { - self.0.write(buf) + handle_ebadf(self.0.write(buf), buf.len()) } fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { - self.0.write_vectored(bufs) + let total = bufs.iter().map(|b| b.len()).sum(); + handle_ebadf(self.0.write_vectored(bufs), total) } - fn flush(&mut self) -> io::Result<()> { - self.0.flush() + #[inline] + fn is_write_vectored(&self) -> bool { + self.0.is_write_vectored() } -} -impl Write for StderrRaw { - fn write(&mut self, buf: &[u8]) -> io::Result { - self.0.write(buf) + + fn flush(&mut self) -> io::Result<()> { + handle_ebadf(self.0.flush(), ()) } - fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { - self.0.write_vectored(bufs) + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + handle_ebadf(self.0.write_all(buf), ()) } - fn flush(&mut self) -> io::Result<()> { - self.0.flush() + fn write_all_vectored(&mut self, bufs: &mut [IoSlice<'_>]) -> io::Result<()> { + handle_ebadf(self.0.write_all_vectored(bufs), ()) } -} -enum Maybe { - Real(T), - Fake, + fn write_fmt(&mut self, fmt: fmt::Arguments<'_>) -> io::Result<()> { + handle_ebadf(self.0.write_fmt(fmt), ()) + } } -impl io::Write for Maybe { +impl Write for StderrRaw { fn write(&mut self, buf: &[u8]) -> io::Result { - match *self { - Maybe::Real(ref mut w) => handle_ebadf(w.write(buf), buf.len()), - Maybe::Fake => Ok(buf.len()), - } + handle_ebadf(self.0.write(buf), buf.len()) } fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { let total = bufs.iter().map(|b| b.len()).sum(); - match self { - Maybe::Real(w) => handle_ebadf(w.write_vectored(bufs), total), - Maybe::Fake => Ok(total), - } + handle_ebadf(self.0.write_vectored(bufs), total) + } + + #[inline] + fn is_write_vectored(&self) -> bool { + self.0.is_write_vectored() } fn flush(&mut self) -> io::Result<()> { - match *self { - Maybe::Real(ref mut w) => handle_ebadf(w.flush(), ()), - Maybe::Fake => Ok(()), - } + handle_ebadf(self.0.flush(), ()) } -} -impl io::Read for Maybe { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - match *self { - Maybe::Real(ref mut r) => handle_ebadf(r.read(buf), 0), - Maybe::Fake => Ok(0), - } + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + handle_ebadf(self.0.write_all(buf), ()) } - fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { - match self { - Maybe::Real(r) => handle_ebadf(r.read_vectored(bufs), 0), - Maybe::Fake => Ok(0), - } + fn write_all_vectored(&mut self, bufs: &mut [IoSlice<'_>]) -> io::Result<()> { + handle_ebadf(self.0.write_all_vectored(bufs), ()) + } + + fn write_fmt(&mut self, fmt: fmt::Arguments<'_>) -> io::Result<()> { + handle_ebadf(self.0.write_fmt(fmt), ()) } } @@ -183,34 +198,73 @@ fn handle_ebadf(r: io::Result, default: T) -> io::Result { /// /// Created by the [`io::stdin`] method. /// -/// [`io::stdin`]: fn.stdin.html -/// [`BufRead`]: trait.BufRead.html +/// [`io::stdin`]: stdin +/// +/// ### Note: Windows Portability Considerations /// -/// ### Note: Windows Portability Consideration /// When operating in a console, the Windows implementation of this stream does not support /// non-UTF-8 byte sequences. Attempting to read bytes that are not valid UTF-8 will return /// an error. +/// +/// In a process with a detached console, such as one using +/// `#![windows_subsystem = "windows"]`, or in a child process spawned from such a process, +/// the contained handle will be null. In such cases, the standard library's `Read` and +/// `Write` will do nothing and silently succeed. All other I/O operations, via the +/// standard library or via raw Windows API calls, will fail. +/// +/// # Examples +/// +/// ```no_run +/// use std::io; +/// +/// fn main() -> io::Result<()> { +/// let mut buffer = String::new(); +/// let stdin = io::stdin(); // We get `Stdin` here. +/// stdin.read_line(&mut buffer)?; +/// Ok(()) +/// } +/// ``` #[stable(feature = "rust1", since = "1.0.0")] pub struct Stdin { - inner: Arc>>>, + inner: &'static Mutex>, } -/// A locked reference to the `Stdin` handle. +/// A locked reference to the [`Stdin`] handle. /// /// This handle implements both the [`Read`] and [`BufRead`] traits, and /// is constructed via the [`Stdin::lock`] method. /// -/// [`Read`]: trait.Read.html -/// [`BufRead`]: trait.BufRead.html -/// [`Stdin::lock`]: struct.Stdin.html#method.lock +/// ### Note: Windows Portability Considerations /// -/// ### Note: Windows Portability Consideration /// When operating in a console, the Windows implementation of this stream does not support /// non-UTF-8 byte sequences. Attempting to read bytes that are not valid UTF-8 will return /// an error. +/// +/// In a process with a detached console, such as one using +/// `#![windows_subsystem = "windows"]`, or in a child process spawned from such a process, +/// the contained handle will be null. In such cases, the standard library's `Read` and +/// `Write` will do nothing and silently succeed. All other I/O operations, via the +/// standard library or via raw Windows API calls, will fail. +/// +/// # Examples +/// +/// ```no_run +/// use std::io::{self, BufRead}; +/// +/// fn main() -> io::Result<()> { +/// let mut buffer = String::new(); +/// let stdin = io::stdin(); // We get `Stdin` here. +/// { +/// let mut handle = stdin.lock(); // We get `StdinLock` here. +/// handle.read_line(&mut buffer)?; +/// } // `StdinLock` is dropped here. +/// Ok(()) +/// } +/// ``` +#[must_use = "if unused stdin will immediately unlock"] #[stable(feature = "rust1", since = "1.0.0")] pub struct StdinLock<'a> { - inner: MutexGuard<'a, BufReader>>, + inner: MutexGuard<'a, BufReader>, } /// Constructs a new handle to the standard input of the current process. @@ -219,23 +273,28 @@ pub struct StdinLock<'a> { /// is synchronized via a mutex. If you need more explicit control over /// locking, see the [`Stdin::lock`] method. /// -/// [`Stdin::lock`]: struct.Stdin.html#method.lock +/// ### Note: Windows Portability Considerations /// -/// ### Note: Windows Portability Consideration /// When operating in a console, the Windows implementation of this stream does not support /// non-UTF-8 byte sequences. Attempting to read bytes that are not valid UTF-8 will return /// an error. /// +/// In a process with a detached console, such as one using +/// `#![windows_subsystem = "windows"]`, or in a child process spawned from such a process, +/// the contained handle will be null. In such cases, the standard library's `Read` and +/// `Write` will do nothing and silently succeed. All other I/O operations, via the +/// standard library or via raw Windows API calls, will fail. +/// /// # Examples /// /// Using implicit synchronization: /// /// ```no_run -/// use std::io::{self, Read}; +/// use std::io; /// /// fn main() -> io::Result<()> { /// let mut buffer = String::new(); -/// io::stdin().read_to_string(&mut buffer)?; +/// io::stdin().read_line(&mut buffer)?; /// Ok(()) /// } /// ``` @@ -243,32 +302,25 @@ pub struct StdinLock<'a> { /// Using explicit synchronization: /// /// ```no_run -/// use std::io::{self, Read}; +/// use std::io::{self, BufRead}; /// /// fn main() -> io::Result<()> { /// let mut buffer = String::new(); /// let stdin = io::stdin(); /// let mut handle = stdin.lock(); /// -/// handle.read_to_string(&mut buffer)?; +/// handle.read_line(&mut buffer)?; /// Ok(()) /// } /// ``` +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn stdin() -> Stdin { - static INSTANCE: Lazy>>> = Lazy::new(); - return Stdin { - inner: unsafe { INSTANCE.get(stdin_init).expect("cannot access stdin during shutdown") }, - }; - - fn stdin_init() -> Arc>>> { - // This must not reentrantly access `INSTANCE` - let stdin = match stdin_raw() { - Ok(stdin) => Maybe::Real(stdin), - _ => Maybe::Fake, - }; - - Arc::new(Mutex::new(BufReader::with_capacity(stdio::STDIN_BUF_SIZE, stdin))) + static INSTANCE: OnceLock>> = OnceLock::new(); + Stdin { + inner: INSTANCE.get_or_init(|| { + Mutex::new(BufReader::with_capacity(stdio::STDIN_BUF_SIZE, stdin_raw())) + }), } } @@ -280,25 +332,24 @@ impl Stdin { /// returned guard also implements the [`Read`] and [`BufRead`] traits for /// accessing the underlying data. /// - /// [`Read`]: trait.Read.html - /// [`BufRead`]: trait.BufRead.html - /// /// # Examples /// /// ```no_run - /// use std::io::{self, Read}; + /// use std::io::{self, BufRead}; /// /// fn main() -> io::Result<()> { /// let mut buffer = String::new(); /// let stdin = io::stdin(); /// let mut handle = stdin.lock(); /// - /// handle.read_to_string(&mut buffer)?; + /// handle.read_line(&mut buffer)?; /// Ok(()) /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn lock(&self) -> StdinLock<'_> { + pub fn lock(&self) -> StdinLock<'static> { + // Locks this handle with 'static lifetime. This depends on the + // implementation detail that the underlying `Mutex` is static. StdinLock { inner: self.inner.lock().unwrap_or_else(|e| e.into_inner()) } } @@ -307,8 +358,6 @@ impl Stdin { /// For detailed semantics of this method, see the documentation on /// [`BufRead::read_line`]. /// - /// [`BufRead::read_line`]: trait.BufRead.html#method.read_line - /// /// # Examples /// /// ```no_run @@ -317,10 +366,10 @@ impl Stdin { /// let mut input = String::new(); /// match io::stdin().read_line(&mut input) { /// Ok(n) => { - /// println!("{} bytes read", n); - /// println!("{}", input); + /// println!("{n} bytes read"); + /// println!("{input}"); /// } - /// Err(error) => println!("error: {}", error), + /// Err(error) => println!("error: {error}"), /// } /// ``` /// @@ -334,12 +383,33 @@ impl Stdin { pub fn read_line(&self, buf: &mut String) -> io::Result { self.lock().read_line(buf) } + + /// Consumes this handle and returns an iterator over input lines. + /// + /// For detailed semantics of this method, see the documentation on + /// [`BufRead::lines`]. + /// + /// # Examples + /// + /// ```no_run + /// use std::io; + /// + /// let lines = io::stdin().lines(); + /// for line in lines { + /// println!("got a line: {}", line.unwrap()); + /// } + /// ``` + #[must_use = "`self` will be dropped if the result is not used"] + #[stable(feature = "stdin_forwarders", since = "1.62.0")] + pub fn lines(self) -> Lines> { + self.lock().lines() + } } #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for Stdin { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("Stdin { .. }") + f.debug_struct("Stdin").finish_non_exhaustive() } } @@ -352,8 +422,8 @@ impl Read for Stdin { self.lock().read_vectored(bufs) } #[inline] - unsafe fn initializer(&self) -> Initializer { - Initializer::nop() + fn is_read_vectored(&self) -> bool { + self.lock().is_read_vectored() } fn read_to_end(&mut self, buf: &mut Vec) -> io::Result { self.lock().read_to_end(buf) @@ -366,6 +436,14 @@ impl Read for Stdin { } } +// only used by platform-dependent io::copy specializations, i.e. unused on some platforms +#[cfg(any(target_os = "linux", target_os = "android"))] +impl StdinLock<'_> { + pub(crate) fn as_mut_buf(&mut self) -> &mut BufReader { + &mut self.inner + } +} + #[stable(feature = "rust1", since = "1.0.0")] impl Read for StdinLock<'_> { fn read(&mut self, buf: &mut [u8]) -> io::Result { @@ -377,8 +455,20 @@ impl Read for StdinLock<'_> { } #[inline] - unsafe fn initializer(&self) -> Initializer { - Initializer::nop() + fn is_read_vectored(&self) -> bool { + self.inner.is_read_vectored() + } + + fn read_to_end(&mut self, buf: &mut Vec) -> io::Result { + self.inner.read_to_end(buf) + } + + fn read_to_string(&mut self, buf: &mut String) -> io::Result { + self.inner.read_to_string(buf) + } + + fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> { + self.inner.read_exact(buf) } } @@ -387,15 +477,24 @@ impl BufRead for StdinLock<'_> { fn fill_buf(&mut self) -> io::Result<&[u8]> { self.inner.fill_buf() } + fn consume(&mut self, n: usize) { self.inner.consume(n) } + + fn read_until(&mut self, byte: u8, buf: &mut Vec) -> io::Result { + self.inner.read_until(byte, buf) + } + + fn read_line(&mut self, buf: &mut String) -> io::Result { + self.inner.read_line(buf) + } } #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for StdinLock<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("StdinLock { .. }") + f.debug_struct("StdinLock").finish_non_exhaustive() } } @@ -407,51 +506,70 @@ impl fmt::Debug for StdinLock<'_> { /// /// Created by the [`io::stdout`] method. /// -/// ### Note: Windows Portability Consideration +/// ### Note: Windows Portability Considerations +/// /// When operating in a console, the Windows implementation of this stream does not support /// non-UTF-8 byte sequences. Attempting to write bytes that are not valid UTF-8 will return /// an error. /// -/// [`lock`]: #method.lock -/// [`io::stdout`]: fn.stdout.html +/// In a process with a detached console, such as one using +/// `#![windows_subsystem = "windows"]`, or in a child process spawned from such a process, +/// the contained handle will be null. In such cases, the standard library's `Read` and +/// `Write` will do nothing and silently succeed. All other I/O operations, via the +/// standard library or via raw Windows API calls, will fail. +/// +/// [`lock`]: Stdout::lock +/// [`io::stdout`]: stdout #[stable(feature = "rust1", since = "1.0.0")] pub struct Stdout { // FIXME: this should be LineWriter or BufWriter depending on the state of // stdout (tty or not). Note that if this is not line buffered it // should also flush-on-panic or some form of flush-on-abort. - inner: Arc>>>>, + inner: &'static ReentrantMutex>>, } -/// A locked reference to the `Stdout` handle. +/// A locked reference to the [`Stdout`] handle. /// /// This handle implements the [`Write`] trait, and is constructed via -/// the [`Stdout::lock`] method. +/// the [`Stdout::lock`] method. See its documentation for more. +/// +/// ### Note: Windows Portability Considerations /// -/// ### Note: Windows Portability Consideration /// When operating in a console, the Windows implementation of this stream does not support /// non-UTF-8 byte sequences. Attempting to write bytes that are not valid UTF-8 will return /// an error. /// -/// [`Write`]: trait.Write.html -/// [`Stdout::lock`]: struct.Stdout.html#method.lock +/// In a process with a detached console, such as one using +/// `#![windows_subsystem = "windows"]`, or in a child process spawned from such a process, +/// the contained handle will be null. In such cases, the standard library's `Read` and +/// `Write` will do nothing and silently succeed. All other I/O operations, via the +/// standard library or via raw Windows API calls, will fail. +#[must_use = "if unused stdout will immediately unlock"] #[stable(feature = "rust1", since = "1.0.0")] pub struct StdoutLock<'a> { - inner: ReentrantMutexGuard<'a, RefCell>>>, + inner: ReentrantMutexGuard<'a, RefCell>>, } +static STDOUT: OnceLock>>> = OnceLock::new(); + /// Constructs a new handle to the standard output of the current process. /// /// Each handle returned is a reference to a shared global buffer whose access /// is synchronized via a mutex. If you need more explicit control over /// locking, see the [`Stdout::lock`] method. /// -/// [`Stdout::lock`]: struct.Stdout.html#method.lock +/// ### Note: Windows Portability Considerations /// -/// ### Note: Windows Portability Consideration /// When operating in a console, the Windows implementation of this stream does not support /// non-UTF-8 byte sequences. Attempting to write bytes that are not valid UTF-8 will return /// an error. /// +/// In a process with a detached console, such as one using +/// `#![windows_subsystem = "windows"]`, or in a child process spawned from such a process, +/// the contained handle will be null. In such cases, the standard library's `Read` and +/// `Write` will do nothing and silently succeed. All other I/O operations, via the +/// standard library or via raw Windows API calls, will fail. +/// /// # Examples /// /// Using implicit synchronization: @@ -480,23 +598,32 @@ pub struct StdoutLock<'a> { /// Ok(()) /// } /// ``` +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn stdout() -> Stdout { - static INSTANCE: Lazy>>>> = Lazy::new(); - return Stdout { - inner: unsafe { INSTANCE.get(stdout_init).expect("cannot access stdout during shutdown") }, - }; - - fn stdout_init() -> Arc>>>> { - // This must not reentrantly access `INSTANCE` - let stdout = match stdout_raw() { - Ok(stdout) => Maybe::Real(stdout), - _ => Maybe::Fake, - }; - unsafe { - let ret = Arc::new(ReentrantMutex::new(RefCell::new(LineWriter::new(stdout)))); - ret.init(); - return ret; + Stdout { + inner: STDOUT + .get_or_init(|| ReentrantMutex::new(RefCell::new(LineWriter::new(stdout_raw())))), + } +} + +// Flush the data and disable buffering during shutdown +// by replacing the line writer by one with zero +// buffering capacity. +pub fn cleanup() { + let mut initialized = false; + let stdout = STDOUT.get_or_init(|| { + initialized = true; + ReentrantMutex::new(RefCell::new(LineWriter::with_capacity(0, stdout_raw()))) + }); + + if !initialized { + // The buffer was previously initialized, overwrite it here. + // We use try_lock() instead of lock(), because someone + // might have leaked a StdoutLock, which would + // otherwise cause a deadlock here. + if let Some(lock) = stdout.try_lock() { + *lock.borrow_mut() = LineWriter::with_capacity(0, stdout_raw()); } } } @@ -514,16 +641,18 @@ impl Stdout { /// use std::io::{self, Write}; /// /// fn main() -> io::Result<()> { - /// let stdout = io::stdout(); - /// let mut handle = stdout.lock(); + /// let mut stdout = io::stdout().lock(); /// - /// handle.write_all(b"hello world")?; + /// stdout.write_all(b"hello world")?; /// /// Ok(()) /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn lock(&self) -> StdoutLock<'_> { + pub fn lock(&self) -> StdoutLock<'static> { + // Locks this handle with 'static lifetime. This depends on the + // implementation detail that the underlying `ReentrantMutex` is + // static. StdoutLock { inner: self.inner.lock() } } } @@ -531,28 +660,62 @@ impl Stdout { #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for Stdout { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("Stdout { .. }") + f.debug_struct("Stdout").finish_non_exhaustive() } } #[stable(feature = "rust1", since = "1.0.0")] impl Write for Stdout { + fn write(&mut self, buf: &[u8]) -> io::Result { + (&*self).write(buf) + } + fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { + (&*self).write_vectored(bufs) + } + #[inline] + fn is_write_vectored(&self) -> bool { + io::Write::is_write_vectored(&&*self) + } + fn flush(&mut self) -> io::Result<()> { + (&*self).flush() + } + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + (&*self).write_all(buf) + } + fn write_all_vectored(&mut self, bufs: &mut [IoSlice<'_>]) -> io::Result<()> { + (&*self).write_all_vectored(bufs) + } + fn write_fmt(&mut self, args: fmt::Arguments<'_>) -> io::Result<()> { + (&*self).write_fmt(args) + } +} + +#[stable(feature = "write_mt", since = "1.48.0")] +impl Write for &Stdout { fn write(&mut self, buf: &[u8]) -> io::Result { self.lock().write(buf) } fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { self.lock().write_vectored(bufs) } + #[inline] + fn is_write_vectored(&self) -> bool { + self.lock().is_write_vectored() + } fn flush(&mut self) -> io::Result<()> { self.lock().flush() } fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { self.lock().write_all(buf) } + fn write_all_vectored(&mut self, bufs: &mut [IoSlice<'_>]) -> io::Result<()> { + self.lock().write_all_vectored(bufs) + } fn write_fmt(&mut self, args: fmt::Arguments<'_>) -> io::Result<()> { self.lock().write_fmt(args) } } + #[stable(feature = "rust1", since = "1.0.0")] impl Write for StdoutLock<'_> { fn write(&mut self, buf: &[u8]) -> io::Result { @@ -561,15 +724,25 @@ impl Write for StdoutLock<'_> { fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { self.inner.borrow_mut().write_vectored(bufs) } + #[inline] + fn is_write_vectored(&self) -> bool { + self.inner.borrow_mut().is_write_vectored() + } fn flush(&mut self) -> io::Result<()> { self.inner.borrow_mut().flush() } + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + self.inner.borrow_mut().write_all(buf) + } + fn write_all_vectored(&mut self, bufs: &mut [IoSlice<'_>]) -> io::Result<()> { + self.inner.borrow_mut().write_all_vectored(bufs) + } } #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for StdoutLock<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("StdoutLock { .. }") + f.debug_struct("StdoutLock").finish_non_exhaustive() } } @@ -577,42 +750,62 @@ impl fmt::Debug for StdoutLock<'_> { /// /// For more information, see the [`io::stderr`] method. /// -/// [`io::stderr`]: fn.stderr.html +/// [`io::stderr`]: stderr +/// +/// ### Note: Windows Portability Considerations /// -/// ### Note: Windows Portability Consideration /// When operating in a console, the Windows implementation of this stream does not support /// non-UTF-8 byte sequences. Attempting to write bytes that are not valid UTF-8 will return /// an error. +/// +/// In a process with a detached console, such as one using +/// `#![windows_subsystem = "windows"]`, or in a child process spawned from such a process, +/// the contained handle will be null. In such cases, the standard library's `Read` and +/// `Write` will do nothing and silently succeed. All other I/O operations, via the +/// standard library or via raw Windows API calls, will fail. #[stable(feature = "rust1", since = "1.0.0")] pub struct Stderr { - inner: &'static ReentrantMutex>>, + inner: &'static ReentrantMutex>, } -/// A locked reference to the `Stderr` handle. +/// A locked reference to the [`Stderr`] handle. /// -/// This handle implements the `Write` trait and is constructed via -/// the [`Stderr::lock`] method. +/// This handle implements the [`Write`] trait and is constructed via +/// the [`Stderr::lock`] method. See its documentation for more. /// -/// [`Stderr::lock`]: struct.Stderr.html#method.lock +/// ### Note: Windows Portability Considerations /// -/// ### Note: Windows Portability Consideration /// When operating in a console, the Windows implementation of this stream does not support /// non-UTF-8 byte sequences. Attempting to write bytes that are not valid UTF-8 will return /// an error. +/// +/// In a process with a detached console, such as one using +/// `#![windows_subsystem = "windows"]`, or in a child process spawned from such a process, +/// the contained handle will be null. In such cases, the standard library's `Read` and +/// `Write` will do nothing and silently succeed. All other I/O operations, via the +/// standard library or via raw Windows API calls, will fail. +#[must_use = "if unused stderr will immediately unlock"] #[stable(feature = "rust1", since = "1.0.0")] pub struct StderrLock<'a> { - inner: ReentrantMutexGuard<'a, RefCell>>, + inner: ReentrantMutexGuard<'a, RefCell>, } /// Constructs a new handle to the standard error of the current process. /// /// This handle is not buffered. /// -/// ### Note: Windows Portability Consideration +/// ### Note: Windows Portability Considerations +/// /// When operating in a console, the Windows implementation of this stream does not support /// non-UTF-8 byte sequences. Attempting to write bytes that are not valid UTF-8 will return /// an error. /// +/// In a process with a detached console, such as one using +/// `#![windows_subsystem = "windows"]`, or in a child process spawned from such a process, +/// the contained handle will be null. In such cases, the standard library's `Read` and +/// `Write` will do nothing and silently succeed. All other I/O operations, via the +/// standard library or via raw Windows API calls, will fail. +/// /// # Examples /// /// Using implicit synchronization: @@ -641,30 +834,16 @@ pub struct StderrLock<'a> { /// Ok(()) /// } /// ``` +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn stderr() -> Stderr { - // Note that unlike `stdout()` we don't use `Lazy` here which registers a - // destructor. Stderr is not buffered nor does the `stderr_raw` type consume - // any owned resources, so there's no need to run any destructors at some - // point in the future. - // - // This has the added benefit of allowing `stderr` to be usable during - // process shutdown as well! - static INSTANCE: ReentrantMutex>> = - unsafe { ReentrantMutex::new(RefCell::new(Maybe::Fake)) }; - - // When accessing stderr we need one-time initialization of the reentrant - // mutex, followed by one-time detection of whether we actually have a - // stderr handle or not. Afterwards we can just always use the now-filled-in - // `INSTANCE` value. - static INIT: Once = Once::new(); - INIT.call_once(|| unsafe { - INSTANCE.init(); - if let Ok(stderr) = stderr_raw() { - *INSTANCE.lock().borrow_mut() = Maybe::Real(stderr); - } - }); - return Stderr { inner: &INSTANCE }; + // Note that unlike `stdout()` we don't use `at_exit` here to register a + // destructor. Stderr is not buffered, so there's no need to run a + // destructor for flushing the buffer + static INSTANCE: ReentrantMutex> = + ReentrantMutex::new(RefCell::new(stderr_raw())); + + Stderr { inner: &INSTANCE } } impl Stderr { @@ -672,7 +851,7 @@ impl Stderr { /// guard. /// /// The lock is released when the returned lock goes out of scope. The - /// returned guard also implements the `Write` trait for writing data. + /// returned guard also implements the [`Write`] trait for writing data. /// /// # Examples /// @@ -689,7 +868,10 @@ impl Stderr { /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn lock(&self) -> StderrLock<'_> { + pub fn lock(&self) -> StderrLock<'static> { + // Locks this handle with 'static lifetime. This depends on the + // implementation detail that the underlying `ReentrantMutex` is + // static. StderrLock { inner: self.inner.lock() } } } @@ -697,28 +879,62 @@ impl Stderr { #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for Stderr { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("Stderr { .. }") + f.debug_struct("Stderr").finish_non_exhaustive() } } #[stable(feature = "rust1", since = "1.0.0")] impl Write for Stderr { + fn write(&mut self, buf: &[u8]) -> io::Result { + (&*self).write(buf) + } + fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { + (&*self).write_vectored(bufs) + } + #[inline] + fn is_write_vectored(&self) -> bool { + io::Write::is_write_vectored(&&*self) + } + fn flush(&mut self) -> io::Result<()> { + (&*self).flush() + } + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + (&*self).write_all(buf) + } + fn write_all_vectored(&mut self, bufs: &mut [IoSlice<'_>]) -> io::Result<()> { + (&*self).write_all_vectored(bufs) + } + fn write_fmt(&mut self, args: fmt::Arguments<'_>) -> io::Result<()> { + (&*self).write_fmt(args) + } +} + +#[stable(feature = "write_mt", since = "1.48.0")] +impl Write for &Stderr { fn write(&mut self, buf: &[u8]) -> io::Result { self.lock().write(buf) } fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { self.lock().write_vectored(bufs) } + #[inline] + fn is_write_vectored(&self) -> bool { + self.lock().is_write_vectored() + } fn flush(&mut self) -> io::Result<()> { self.lock().flush() } fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { self.lock().write_all(buf) } + fn write_all_vectored(&mut self, bufs: &mut [IoSlice<'_>]) -> io::Result<()> { + self.lock().write_all_vectored(bufs) + } fn write_fmt(&mut self, args: fmt::Arguments<'_>) -> io::Result<()> { self.lock().write_fmt(args) } } + #[stable(feature = "rust1", since = "1.0.0")] impl Write for StderrLock<'_> { fn write(&mut self, buf: &[u8]) -> io::Result { @@ -727,102 +943,126 @@ impl Write for StderrLock<'_> { fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { self.inner.borrow_mut().write_vectored(bufs) } + #[inline] + fn is_write_vectored(&self) -> bool { + self.inner.borrow_mut().is_write_vectored() + } fn flush(&mut self) -> io::Result<()> { self.inner.borrow_mut().flush() } + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + self.inner.borrow_mut().write_all(buf) + } + fn write_all_vectored(&mut self, bufs: &mut [IoSlice<'_>]) -> io::Result<()> { + self.inner.borrow_mut().write_all_vectored(bufs) + } } #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for StderrLock<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("StderrLock { .. }") + f.debug_struct("StderrLock").finish_non_exhaustive() } } -/// Resets the thread-local stderr handle to the specified writer -/// -/// This will replace the current thread's stderr handle, returning the old -/// handle. All future calls to `panic!` and friends will emit their output to -/// this specified handle. -/// -/// Note that this does not need to be called for all new threads; the default -/// output handle is to the process's stderr stream. -#[unstable( - feature = "set_stdio", - reason = "this function may disappear completely or be replaced \ - with a more general mechanism", - issue = "none" -)] -#[doc(hidden)] -pub fn set_panic(sink: Option>) -> Option> { - use crate::mem; - LOCAL_STDERR.with(move |slot| mem::replace(&mut *slot.borrow_mut(), sink)).and_then(|mut s| { - let _ = s.flush(); - Some(s) - }) -} - -/// Resets the thread-local stdout handle to the specified writer -/// -/// This will replace the current thread's stdout handle, returning the old -/// handle. All future calls to `print!` and friends will emit their output to -/// this specified handle. -/// -/// Note that this does not need to be called for all new threads; the default -/// output handle is to the process's stdout stream. +/// Sets the thread-local output capture buffer and returns the old one. #[unstable( - feature = "set_stdio", - reason = "this function may disappear completely or be replaced \ - with a more general mechanism", + feature = "internal_output_capture", + reason = "this function is meant for use in the test crate \ + and may disappear in the future", issue = "none" )] #[doc(hidden)] -pub fn set_print(sink: Option>) -> Option> { - use crate::mem; - LOCAL_STDOUT.with(move |slot| mem::replace(&mut *slot.borrow_mut(), sink)).and_then(|mut s| { - let _ = s.flush(); - Some(s) - }) +pub fn set_output_capture(sink: Option) -> Option { + if sink.is_none() && !OUTPUT_CAPTURE_USED.load(Ordering::Relaxed) { + // OUTPUT_CAPTURE is definitely None since OUTPUT_CAPTURE_USED is false. + return None; + } + OUTPUT_CAPTURE_USED.store(true, Ordering::Relaxed); + OUTPUT_CAPTURE.with(move |slot| slot.replace(sink)) } -/// Write `args` to output stream `local_s` if possible, `global_s` +/// Write `args` to the capture buffer if enabled and possible, or `global_s` /// otherwise. `label` identifies the stream in a panic message. /// /// This function is used to print error messages, so it takes extra -/// care to avoid causing a panic when `local_s` is unusable. -/// For instance, if the TLS key for the local stream is -/// already destroyed, or if the local stream is locked by another -/// thread, it will just fall back to the global stream. +/// care to avoid causing a panic when `OUTPUT_CAPTURE` is unusable. +/// For instance, if the TLS key for output capturing is already destroyed, or +/// if the local stream is in use by another thread, it will just fall back to +/// the global stream. /// /// However, if the actual I/O causes an error, this function does panic. -fn print_to( - args: fmt::Arguments<'_>, - local_s: &'static LocalKey>>>, - global_s: fn() -> T, - label: &str, -) where +/// +/// Writing to non-blocking stdout/stderr can cause an error, which will lead +/// this function to panic. +fn print_to(args: fmt::Arguments<'_>, global_s: fn() -> T, label: &str) +where T: Write, { - let result = local_s - .try_with(|s| { + if print_to_buffer_if_capture_used(args) { + // Successfully wrote to capture buffer. + return; + } + + if let Err(e) = global_s().write_fmt(args) { + panic!("failed printing to {label}: {e}"); + } +} + +fn print_to_buffer_if_capture_used(args: fmt::Arguments<'_>) -> bool { + OUTPUT_CAPTURE_USED.load(Ordering::Relaxed) + && OUTPUT_CAPTURE.try_with(|s| { // Note that we completely remove a local sink to write to in case // our printing recursively panics/prints, so the recursive // panic/print goes to the global sink instead of our local sink. - let prev = s.borrow_mut().take(); - if let Some(mut w) = prev { - let result = w.write_fmt(args); - *s.borrow_mut() = Some(w); - return result; - } - global_s().write_fmt(args) - }) - .unwrap_or_else(|_| global_s().write_fmt(args)); + s.take().map(|w| { + let _ = w.lock().unwrap_or_else(|e| e.into_inner()).write_fmt(args); + s.set(Some(w)); + }) + }) == Ok(Some(())) +} - if let Err(e) = result { - panic!("failed printing to {}: {}", label, e); +/// Used by impl Termination for Result to print error after `main` or a test +/// has returned. Should avoid panicking, although we can't help it if one of +/// the Display impls inside args decides to. +pub(crate) fn attempt_print_to_stderr(args: fmt::Arguments<'_>) { + if print_to_buffer_if_capture_used(args) { + return; } + + // Ignore error if the write fails, for example because stderr is already + // closed. There is not much point panicking at this point. + let _ = stderr().write_fmt(args); } +/// Trait to determine if a descriptor/handle refers to a terminal/tty. +#[unstable(feature = "is_terminal", issue = "98070")] +pub trait IsTerminal: crate::sealed::Sealed { + /// Returns `true` if the descriptor/handle refers to a terminal/tty. + /// + /// On platforms where Rust does not know how to detect a terminal yet, this will return + /// `false`. This will also return `false` if an unexpected error occurred, such as from + /// passing an invalid file descriptor. + fn is_terminal(&self) -> bool; +} + +macro_rules! impl_is_terminal { + ($($t:ty),*$(,)?) => {$( + #[unstable(feature = "sealed", issue = "none")] + impl crate::sealed::Sealed for $t {} + + #[unstable(feature = "is_terminal", issue = "98070")] + impl IsTerminal for $t { + #[inline] + fn is_terminal(&self) -> bool { + crate::sys::io::is_terminal(self) + } + } + )*} +} + +impl_is_terminal!(File, Stdin, StdinLock<'_>, Stdout, StdoutLock<'_>, Stderr, StderrLock<'_>); + #[unstable( feature = "print_internals", reason = "implementation detail which may disappear or be replaced at any time", @@ -831,7 +1071,7 @@ fn print_to( #[doc(hidden)] #[cfg(not(test))] pub fn _print(args: fmt::Arguments<'_>) { - print_to(args, &LOCAL_STDOUT, stdout, "stdout"); + print_to(args, stdout, "stdout"); } #[unstable( @@ -842,59 +1082,8 @@ pub fn _print(args: fmt::Arguments<'_>) { #[doc(hidden)] #[cfg(not(test))] pub fn _eprint(args: fmt::Arguments<'_>) { - print_to(args, &LOCAL_STDERR, stderr, "stderr"); + print_to(args, stderr, "stderr"); } #[cfg(test)] pub use realstd::io::{_eprint, _print}; - -#[cfg(test)] -mod tests { - use super::*; - use crate::panic::{RefUnwindSafe, UnwindSafe}; - use crate::thread; - - #[test] - fn stdout_unwind_safe() { - assert_unwind_safe::(); - } - #[test] - fn stdoutlock_unwind_safe() { - assert_unwind_safe::>(); - assert_unwind_safe::>(); - } - #[test] - fn stderr_unwind_safe() { - assert_unwind_safe::(); - } - #[test] - fn stderrlock_unwind_safe() { - assert_unwind_safe::>(); - assert_unwind_safe::>(); - } - - fn assert_unwind_safe() {} - - #[test] - #[cfg_attr(target_os = "emscripten", ignore)] - fn panic_doesnt_poison() { - thread::spawn(|| { - let _a = stdin(); - let _a = _a.lock(); - let _a = stdout(); - let _a = _a.lock(); - let _a = stderr(); - let _a = _a.lock(); - panic!(); - }) - .join() - .unwrap_err(); - - let _a = stdin(); - let _a = _a.lock(); - let _a = stdout(); - let _a = _a.lock(); - let _a = stderr(); - let _a = _a.lock(); - } -} diff --git a/crux-mir/lib/std/src/io/stdio/tests.rs b/crux-mir/lib/std/src/io/stdio/tests.rs new file mode 100644 index 000000000..f89fd27ce --- /dev/null +++ b/crux-mir/lib/std/src/io/stdio/tests.rs @@ -0,0 +1,166 @@ +use super::*; +use crate::panic::{RefUnwindSafe, UnwindSafe}; +use crate::sync::mpsc::sync_channel; +use crate::thread; + +#[test] +fn stdout_unwind_safe() { + assert_unwind_safe::(); +} +#[test] +fn stdoutlock_unwind_safe() { + assert_unwind_safe::>(); + assert_unwind_safe::>(); +} +#[test] +fn stderr_unwind_safe() { + assert_unwind_safe::(); +} +#[test] +fn stderrlock_unwind_safe() { + assert_unwind_safe::>(); + assert_unwind_safe::>(); +} + +fn assert_unwind_safe() {} + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn panic_doesnt_poison() { + thread::spawn(|| { + let _a = stdin(); + let _a = _a.lock(); + let _a = stdout(); + let _a = _a.lock(); + let _a = stderr(); + let _a = _a.lock(); + panic!(); + }) + .join() + .unwrap_err(); + + let _a = stdin(); + let _a = _a.lock(); + let _a = stdout(); + let _a = _a.lock(); + let _a = stderr(); + let _a = _a.lock(); +} + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn test_lock_stderr() { + test_lock(stderr, || stderr().lock()); +} +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn test_lock_stdin() { + test_lock(stdin, || stdin().lock()); +} +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn test_lock_stdout() { + test_lock(stdout, || stdout().lock()); +} + +// Helper trait to make lock testing function generic. +trait Stdio<'a>: 'static +where + Self::Lock: 'a, +{ + type Lock; + fn lock(&'a self) -> Self::Lock; +} +impl<'a> Stdio<'a> for Stderr { + type Lock = StderrLock<'a>; + fn lock(&'a self) -> StderrLock<'a> { + self.lock() + } +} +impl<'a> Stdio<'a> for Stdin { + type Lock = StdinLock<'a>; + fn lock(&'a self) -> StdinLock<'a> { + self.lock() + } +} +impl<'a> Stdio<'a> for Stdout { + type Lock = StdoutLock<'a>; + fn lock(&'a self) -> StdoutLock<'a> { + self.lock() + } +} + +// Helper trait to make lock testing function generic. +trait StdioOwnedLock: 'static {} +impl StdioOwnedLock for StderrLock<'static> {} +impl StdioOwnedLock for StdinLock<'static> {} +impl StdioOwnedLock for StdoutLock<'static> {} + +// Tests locking on stdio handles by starting two threads and checking that +// they block each other appropriately. +fn test_lock(get_handle: fn() -> T, get_locked: fn() -> U) +where + T: for<'a> Stdio<'a>, + U: StdioOwnedLock, +{ + // State enum to track different phases of the test, primarily when + // each lock is acquired and released. + #[derive(Debug, PartialEq)] + enum State { + Start1, + Acquire1, + Start2, + Release1, + Acquire2, + Release2, + } + use State::*; + // Logging vector to be checked to make sure lock acquisitions and + // releases happened in the correct order. + let log = Arc::new(Mutex::new(Vec::new())); + let ((tx1, rx1), (tx2, rx2)) = (sync_channel(0), sync_channel(0)); + let th1 = { + let (log, tx) = (Arc::clone(&log), tx1); + thread::spawn(move || { + log.lock().unwrap().push(Start1); + let handle = get_handle(); + { + let locked = handle.lock(); + log.lock().unwrap().push(Acquire1); + tx.send(Acquire1).unwrap(); // notify of acquisition + tx.send(Release1).unwrap(); // wait for release command + log.lock().unwrap().push(Release1); + } + tx.send(Acquire1).unwrap(); // wait for th2 acquire + { + let locked = handle.lock(); + log.lock().unwrap().push(Acquire1); + } + log.lock().unwrap().push(Release1); + }) + }; + let th2 = { + let (log, tx) = (Arc::clone(&log), tx2); + thread::spawn(move || { + tx.send(Start2).unwrap(); // wait for start command + let locked = get_locked(); + log.lock().unwrap().push(Acquire2); + tx.send(Acquire2).unwrap(); // notify of acquisition + tx.send(Release2).unwrap(); // wait for release command + log.lock().unwrap().push(Release2); + }) + }; + assert_eq!(rx1.recv().unwrap(), Acquire1); // wait for th1 acquire + log.lock().unwrap().push(Start2); + assert_eq!(rx2.recv().unwrap(), Start2); // block th2 + assert_eq!(rx1.recv().unwrap(), Release1); // release th1 + assert_eq!(rx2.recv().unwrap(), Acquire2); // wait for th2 acquire + assert_eq!(rx1.recv().unwrap(), Acquire1); // block th1 + assert_eq!(rx2.recv().unwrap(), Release2); // release th2 + th2.join().unwrap(); + th1.join().unwrap(); + assert_eq!( + *log.lock().unwrap(), + [Start1, Acquire1, Start2, Release1, Acquire2, Release2, Acquire1, Release1] + ); +} diff --git a/crux-mir/lib/std/src/io/tests.rs b/crux-mir/lib/std/src/io/tests.rs new file mode 100644 index 000000000..f4a886d88 --- /dev/null +++ b/crux-mir/lib/std/src/io/tests.rs @@ -0,0 +1,624 @@ +use super::{repeat, BorrowedBuf, Cursor, SeekFrom}; +use crate::cmp::{self, min}; +use crate::io::{self, IoSlice, IoSliceMut}; +use crate::io::{BufRead, BufReader, Read, Seek, Write}; +use crate::mem::MaybeUninit; +use crate::ops::Deref; + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn read_until() { + let mut buf = Cursor::new(&b"12"[..]); + let mut v = Vec::new(); + assert_eq!(buf.read_until(b'3', &mut v).unwrap(), 2); + assert_eq!(v, b"12"); + + let mut buf = Cursor::new(&b"1233"[..]); + let mut v = Vec::new(); + assert_eq!(buf.read_until(b'3', &mut v).unwrap(), 3); + assert_eq!(v, b"123"); + v.truncate(0); + assert_eq!(buf.read_until(b'3', &mut v).unwrap(), 1); + assert_eq!(v, b"3"); + v.truncate(0); + assert_eq!(buf.read_until(b'3', &mut v).unwrap(), 0); + assert_eq!(v, []); +} + +#[test] +fn split() { + let buf = Cursor::new(&b"12"[..]); + let mut s = buf.split(b'3'); + assert_eq!(s.next().unwrap().unwrap(), vec![b'1', b'2']); + assert!(s.next().is_none()); + + let buf = Cursor::new(&b"1233"[..]); + let mut s = buf.split(b'3'); + assert_eq!(s.next().unwrap().unwrap(), vec![b'1', b'2']); + assert_eq!(s.next().unwrap().unwrap(), vec![]); + assert!(s.next().is_none()); +} + +#[test] +fn read_line() { + let mut buf = Cursor::new(&b"12"[..]); + let mut v = String::new(); + assert_eq!(buf.read_line(&mut v).unwrap(), 2); + assert_eq!(v, "12"); + + let mut buf = Cursor::new(&b"12\n\n"[..]); + let mut v = String::new(); + assert_eq!(buf.read_line(&mut v).unwrap(), 3); + assert_eq!(v, "12\n"); + v.truncate(0); + assert_eq!(buf.read_line(&mut v).unwrap(), 1); + assert_eq!(v, "\n"); + v.truncate(0); + assert_eq!(buf.read_line(&mut v).unwrap(), 0); + assert_eq!(v, ""); +} + +#[test] +fn lines() { + let buf = Cursor::new(&b"12\r"[..]); + let mut s = buf.lines(); + assert_eq!(s.next().unwrap().unwrap(), "12\r".to_string()); + assert!(s.next().is_none()); + + let buf = Cursor::new(&b"12\r\n\n"[..]); + let mut s = buf.lines(); + assert_eq!(s.next().unwrap().unwrap(), "12".to_string()); + assert_eq!(s.next().unwrap().unwrap(), "".to_string()); + assert!(s.next().is_none()); +} + +#[test] +fn buf_read_has_data_left() { + let mut buf = Cursor::new(&b"abcd"[..]); + assert!(buf.has_data_left().unwrap()); + buf.read_exact(&mut [0; 2]).unwrap(); + assert!(buf.has_data_left().unwrap()); + buf.read_exact(&mut [0; 2]).unwrap(); + assert!(!buf.has_data_left().unwrap()); +} + +#[test] +fn read_to_end() { + let mut c = Cursor::new(&b""[..]); + let mut v = Vec::new(); + assert_eq!(c.read_to_end(&mut v).unwrap(), 0); + assert_eq!(v, []); + + let mut c = Cursor::new(&b"1"[..]); + let mut v = Vec::new(); + assert_eq!(c.read_to_end(&mut v).unwrap(), 1); + assert_eq!(v, b"1"); + + let cap = if cfg!(miri) { 1024 } else { 1024 * 1024 }; + let data = (0..cap).map(|i| (i / 3) as u8).collect::>(); + let mut v = Vec::new(); + let (a, b) = data.split_at(data.len() / 2); + assert_eq!(Cursor::new(a).read_to_end(&mut v).unwrap(), a.len()); + assert_eq!(Cursor::new(b).read_to_end(&mut v).unwrap(), b.len()); + assert_eq!(v, data); +} + +#[test] +fn read_to_string() { + let mut c = Cursor::new(&b""[..]); + let mut v = String::new(); + assert_eq!(c.read_to_string(&mut v).unwrap(), 0); + assert_eq!(v, ""); + + let mut c = Cursor::new(&b"1"[..]); + let mut v = String::new(); + assert_eq!(c.read_to_string(&mut v).unwrap(), 1); + assert_eq!(v, "1"); + + let mut c = Cursor::new(&b"\xff"[..]); + let mut v = String::new(); + assert!(c.read_to_string(&mut v).is_err()); +} + +#[test] +fn read_exact() { + let mut buf = [0; 4]; + + let mut c = Cursor::new(&b""[..]); + assert_eq!(c.read_exact(&mut buf).unwrap_err().kind(), io::ErrorKind::UnexpectedEof); + + let mut c = Cursor::new(&b"123"[..]).chain(Cursor::new(&b"456789"[..])); + c.read_exact(&mut buf).unwrap(); + assert_eq!(&buf, b"1234"); + c.read_exact(&mut buf).unwrap(); + assert_eq!(&buf, b"5678"); + assert_eq!(c.read_exact(&mut buf).unwrap_err().kind(), io::ErrorKind::UnexpectedEof); +} + +#[test] +fn read_exact_slice() { + let mut buf = [0; 4]; + + let mut c = &b""[..]; + assert_eq!(c.read_exact(&mut buf).unwrap_err().kind(), io::ErrorKind::UnexpectedEof); + + let mut c = &b"123"[..]; + assert_eq!(c.read_exact(&mut buf).unwrap_err().kind(), io::ErrorKind::UnexpectedEof); + // make sure the optimized (early returning) method is being used + assert_eq!(&buf, &[0; 4]); + + let mut c = &b"1234"[..]; + c.read_exact(&mut buf).unwrap(); + assert_eq!(&buf, b"1234"); + + let mut c = &b"56789"[..]; + c.read_exact(&mut buf).unwrap(); + assert_eq!(&buf, b"5678"); + assert_eq!(c, b"9"); +} + +#[test] +fn read_buf_exact() { + let buf: &mut [_] = &mut [0; 4]; + let mut buf: BorrowedBuf<'_> = buf.into(); + + let mut c = Cursor::new(&b""[..]); + assert_eq!(c.read_buf_exact(buf.unfilled()).unwrap_err().kind(), io::ErrorKind::UnexpectedEof); + + let mut c = Cursor::new(&b"123456789"[..]); + c.read_buf_exact(buf.unfilled()).unwrap(); + assert_eq!(buf.filled(), b"1234"); + + buf.clear(); + + c.read_buf_exact(buf.unfilled()).unwrap(); + assert_eq!(buf.filled(), b"5678"); + + buf.clear(); + + assert_eq!(c.read_buf_exact(buf.unfilled()).unwrap_err().kind(), io::ErrorKind::UnexpectedEof); +} + +#[test] +fn take_eof() { + struct R; + + impl Read for R { + fn read(&mut self, _: &mut [u8]) -> io::Result { + Err(io::const_io_error!(io::ErrorKind::Other, "")) + } + } + impl BufRead for R { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + Err(io::const_io_error!(io::ErrorKind::Other, "")) + } + fn consume(&mut self, _amt: usize) {} + } + + let mut buf = [0; 1]; + assert_eq!(0, R.take(0).read(&mut buf).unwrap()); + assert_eq!(b"", R.take(0).fill_buf().unwrap()); +} + +fn cmp_bufread(mut br1: Br1, mut br2: Br2, exp: &[u8]) { + let mut cat = Vec::new(); + loop { + let consume = { + let buf1 = br1.fill_buf().unwrap(); + let buf2 = br2.fill_buf().unwrap(); + let minlen = if buf1.len() < buf2.len() { buf1.len() } else { buf2.len() }; + assert_eq!(buf1[..minlen], buf2[..minlen]); + cat.extend_from_slice(&buf1[..minlen]); + minlen + }; + if consume == 0 { + break; + } + br1.consume(consume); + br2.consume(consume); + } + assert_eq!(br1.fill_buf().unwrap().len(), 0); + assert_eq!(br2.fill_buf().unwrap().len(), 0); + assert_eq!(&cat[..], &exp[..]) +} + +#[test] +fn chain_bufread() { + let testdata = b"ABCDEFGHIJKL"; + let chain1 = + (&testdata[..3]).chain(&testdata[3..6]).chain(&testdata[6..9]).chain(&testdata[9..]); + let chain2 = (&testdata[..4]).chain(&testdata[4..8]).chain(&testdata[8..]); + cmp_bufread(chain1, chain2, &testdata[..]); +} + +#[test] +fn bufreader_size_hint() { + let testdata = b"ABCDEFGHIJKL"; + let mut buf_reader = BufReader::new(&testdata[..]); + assert_eq!(buf_reader.buffer().len(), 0); + + let buffer_length = testdata.len(); + buf_reader.fill_buf().unwrap(); + + // Check that size hint matches buffer contents + let mut buffered_bytes = buf_reader.bytes(); + let (lower_bound, _upper_bound) = buffered_bytes.size_hint(); + assert_eq!(lower_bound, buffer_length); + + // Check that size hint matches buffer contents after advancing + buffered_bytes.next().unwrap().unwrap(); + let (lower_bound, _upper_bound) = buffered_bytes.size_hint(); + assert_eq!(lower_bound, buffer_length - 1); +} + +#[test] +fn empty_size_hint() { + let size_hint = io::empty().bytes().size_hint(); + assert_eq!(size_hint, (0, Some(0))); +} + +#[test] +fn slice_size_hint() { + let size_hint = (&[1, 2, 3]).bytes().size_hint(); + assert_eq!(size_hint, (3, Some(3))); +} + +#[test] +fn take_size_hint() { + let size_hint = (&[1, 2, 3]).take(2).bytes().size_hint(); + assert_eq!(size_hint, (2, Some(2))); + + let size_hint = (&[1, 2, 3]).take(4).bytes().size_hint(); + assert_eq!(size_hint, (3, Some(3))); + + let size_hint = io::repeat(0).take(3).bytes().size_hint(); + assert_eq!(size_hint, (3, Some(3))); +} + +#[test] +fn chain_empty_size_hint() { + let chain = io::empty().chain(io::empty()); + let size_hint = chain.bytes().size_hint(); + assert_eq!(size_hint, (0, Some(0))); +} + +#[test] +fn chain_size_hint() { + let testdata = b"ABCDEFGHIJKL"; + let mut buf_reader_1 = BufReader::new(&testdata[..6]); + let mut buf_reader_2 = BufReader::new(&testdata[6..]); + + buf_reader_1.fill_buf().unwrap(); + buf_reader_2.fill_buf().unwrap(); + + let chain = buf_reader_1.chain(buf_reader_2); + let size_hint = chain.bytes().size_hint(); + assert_eq!(size_hint, (testdata.len(), Some(testdata.len()))); +} + +#[test] +fn chain_zero_length_read_is_not_eof() { + let a = b"A"; + let b = b"B"; + let mut s = String::new(); + let mut chain = (&a[..]).chain(&b[..]); + chain.read(&mut []).unwrap(); + chain.read_to_string(&mut s).unwrap(); + assert_eq!("AB", s); +} + +#[bench] +#[cfg_attr(target_os = "emscripten", ignore)] +#[cfg_attr(miri, ignore)] // Miri isn't fast... +fn bench_read_to_end(b: &mut test::Bencher) { + b.iter(|| { + let mut lr = repeat(1).take(10000000); + let mut vec = Vec::with_capacity(1024); + super::default_read_to_end(&mut lr, &mut vec) + }); +} + +#[test] +fn seek_len() -> io::Result<()> { + let mut c = Cursor::new(vec![0; 15]); + assert_eq!(c.stream_len()?, 15); + + c.seek(SeekFrom::End(0))?; + let old_pos = c.stream_position()?; + assert_eq!(c.stream_len()?, 15); + assert_eq!(c.stream_position()?, old_pos); + + c.seek(SeekFrom::Start(7))?; + c.seek(SeekFrom::Current(2))?; + let old_pos = c.stream_position()?; + assert_eq!(c.stream_len()?, 15); + assert_eq!(c.stream_position()?, old_pos); + + Ok(()) +} + +#[test] +fn seek_position() -> io::Result<()> { + // All `asserts` are duplicated here to make sure the method does not + // change anything about the seek state. + let mut c = Cursor::new(vec![0; 15]); + assert_eq!(c.stream_position()?, 0); + assert_eq!(c.stream_position()?, 0); + + c.seek(SeekFrom::End(0))?; + assert_eq!(c.stream_position()?, 15); + assert_eq!(c.stream_position()?, 15); + + c.seek(SeekFrom::Start(7))?; + c.seek(SeekFrom::Current(2))?; + assert_eq!(c.stream_position()?, 9); + assert_eq!(c.stream_position()?, 9); + + c.seek(SeekFrom::End(-3))?; + c.seek(SeekFrom::Current(1))?; + c.seek(SeekFrom::Current(-5))?; + assert_eq!(c.stream_position()?, 8); + assert_eq!(c.stream_position()?, 8); + + c.rewind()?; + assert_eq!(c.stream_position()?, 0); + assert_eq!(c.stream_position()?, 0); + + Ok(()) +} + +// A simple example reader which uses the default implementation of +// read_to_end. +struct ExampleSliceReader<'a> { + slice: &'a [u8], +} + +impl<'a> Read for ExampleSliceReader<'a> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let len = cmp::min(self.slice.len(), buf.len()); + buf[..len].copy_from_slice(&self.slice[..len]); + self.slice = &self.slice[len..]; + Ok(len) + } +} + +#[test] +fn test_read_to_end_capacity() -> io::Result<()> { + let input = &b"foo"[..]; + + // read_to_end() takes care not to over-allocate when a buffer is the + // exact size needed. + let mut vec1 = Vec::with_capacity(input.len()); + ExampleSliceReader { slice: input }.read_to_end(&mut vec1)?; + assert_eq!(vec1.len(), input.len()); + assert_eq!(vec1.capacity(), input.len(), "did not allocate more"); + + Ok(()) +} + +#[test] +fn io_slice_mut_advance_slices() { + let mut buf1 = [1; 8]; + let mut buf2 = [2; 16]; + let mut buf3 = [3; 8]; + let mut bufs = &mut [ + IoSliceMut::new(&mut buf1), + IoSliceMut::new(&mut buf2), + IoSliceMut::new(&mut buf3), + ][..]; + + // Only in a single buffer.. + IoSliceMut::advance_slices(&mut bufs, 1); + assert_eq!(bufs[0].deref(), [1; 7].as_ref()); + assert_eq!(bufs[1].deref(), [2; 16].as_ref()); + assert_eq!(bufs[2].deref(), [3; 8].as_ref()); + + // Removing a buffer, leaving others as is. + IoSliceMut::advance_slices(&mut bufs, 7); + assert_eq!(bufs[0].deref(), [2; 16].as_ref()); + assert_eq!(bufs[1].deref(), [3; 8].as_ref()); + + // Removing a buffer and removing from the next buffer. + IoSliceMut::advance_slices(&mut bufs, 18); + assert_eq!(bufs[0].deref(), [3; 6].as_ref()); +} + +#[test] +#[should_panic] +fn io_slice_mut_advance_slices_empty_slice() { + let mut empty_bufs = &mut [][..]; + IoSliceMut::advance_slices(&mut empty_bufs, 1); +} + +#[test] +#[should_panic] +fn io_slice_mut_advance_slices_beyond_total_length() { + let mut buf1 = [1; 8]; + let mut bufs = &mut [IoSliceMut::new(&mut buf1)][..]; + + IoSliceMut::advance_slices(&mut bufs, 9); + assert!(bufs.is_empty()); +} + +#[test] +fn io_slice_advance_slices() { + let buf1 = [1; 8]; + let buf2 = [2; 16]; + let buf3 = [3; 8]; + let mut bufs = &mut [IoSlice::new(&buf1), IoSlice::new(&buf2), IoSlice::new(&buf3)][..]; + + // Only in a single buffer.. + IoSlice::advance_slices(&mut bufs, 1); + assert_eq!(bufs[0].deref(), [1; 7].as_ref()); + assert_eq!(bufs[1].deref(), [2; 16].as_ref()); + assert_eq!(bufs[2].deref(), [3; 8].as_ref()); + + // Removing a buffer, leaving others as is. + IoSlice::advance_slices(&mut bufs, 7); + assert_eq!(bufs[0].deref(), [2; 16].as_ref()); + assert_eq!(bufs[1].deref(), [3; 8].as_ref()); + + // Removing a buffer and removing from the next buffer. + IoSlice::advance_slices(&mut bufs, 18); + assert_eq!(bufs[0].deref(), [3; 6].as_ref()); +} + +#[test] +#[should_panic] +fn io_slice_advance_slices_empty_slice() { + let mut empty_bufs = &mut [][..]; + IoSlice::advance_slices(&mut empty_bufs, 1); +} + +#[test] +#[should_panic] +fn io_slice_advance_slices_beyond_total_length() { + let buf1 = [1; 8]; + let mut bufs = &mut [IoSlice::new(&buf1)][..]; + + IoSlice::advance_slices(&mut bufs, 9); + assert!(bufs.is_empty()); +} + +/// Create a new writer that reads from at most `n_bufs` and reads +/// `per_call` bytes (in total) per call to write. +fn test_writer(n_bufs: usize, per_call: usize) -> TestWriter { + TestWriter { n_bufs, per_call, written: Vec::new() } +} + +struct TestWriter { + n_bufs: usize, + per_call: usize, + written: Vec, +} + +impl Write for TestWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.write_vectored(&[IoSlice::new(buf)]) + } + + fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { + let mut left = self.per_call; + let mut written = 0; + for buf in bufs.iter().take(self.n_bufs) { + let n = min(left, buf.len()); + self.written.extend_from_slice(&buf[0..n]); + left -= n; + written += n; + } + Ok(written) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +#[test] +fn test_writer_read_from_one_buf() { + let mut writer = test_writer(1, 2); + + assert_eq!(writer.write(&[]).unwrap(), 0); + assert_eq!(writer.write_vectored(&[]).unwrap(), 0); + + // Read at most 2 bytes. + assert_eq!(writer.write(&[1, 1, 1]).unwrap(), 2); + let bufs = &[IoSlice::new(&[2, 2, 2])]; + assert_eq!(writer.write_vectored(bufs).unwrap(), 2); + + // Only read from first buf. + let bufs = &[IoSlice::new(&[3]), IoSlice::new(&[4, 4])]; + assert_eq!(writer.write_vectored(bufs).unwrap(), 1); + + assert_eq!(writer.written, &[1, 1, 2, 2, 3]); +} + +#[test] +fn test_writer_read_from_multiple_bufs() { + let mut writer = test_writer(3, 3); + + // Read at most 3 bytes from two buffers. + let bufs = &[IoSlice::new(&[1]), IoSlice::new(&[2, 2, 2])]; + assert_eq!(writer.write_vectored(bufs).unwrap(), 3); + + // Read at most 3 bytes from three buffers. + let bufs = &[IoSlice::new(&[3]), IoSlice::new(&[4]), IoSlice::new(&[5, 5])]; + assert_eq!(writer.write_vectored(bufs).unwrap(), 3); + + assert_eq!(writer.written, &[1, 2, 2, 3, 4, 5]); +} + +#[test] +fn test_write_all_vectored() { + #[rustfmt::skip] // Becomes unreadable otherwise. + let tests: Vec<(_, &'static [u8])> = vec![ + (vec![], &[]), + (vec![IoSlice::new(&[]), IoSlice::new(&[])], &[]), + (vec![IoSlice::new(&[1])], &[1]), + (vec![IoSlice::new(&[1, 2])], &[1, 2]), + (vec![IoSlice::new(&[1, 2, 3])], &[1, 2, 3]), + (vec![IoSlice::new(&[1, 2, 3, 4])], &[1, 2, 3, 4]), + (vec![IoSlice::new(&[1, 2, 3, 4, 5])], &[1, 2, 3, 4, 5]), + (vec![IoSlice::new(&[1]), IoSlice::new(&[2])], &[1, 2]), + (vec![IoSlice::new(&[1]), IoSlice::new(&[2, 2])], &[1, 2, 2]), + (vec![IoSlice::new(&[1, 1]), IoSlice::new(&[2, 2])], &[1, 1, 2, 2]), + (vec![IoSlice::new(&[1, 1]), IoSlice::new(&[2, 2, 2])], &[1, 1, 2, 2, 2]), + (vec![IoSlice::new(&[1, 1]), IoSlice::new(&[2, 2, 2])], &[1, 1, 2, 2, 2]), + (vec![IoSlice::new(&[1, 1, 1]), IoSlice::new(&[2, 2, 2])], &[1, 1, 1, 2, 2, 2]), + (vec![IoSlice::new(&[1, 1, 1]), IoSlice::new(&[2, 2, 2, 2])], &[1, 1, 1, 2, 2, 2, 2]), + (vec![IoSlice::new(&[1, 1, 1, 1]), IoSlice::new(&[2, 2, 2, 2])], &[1, 1, 1, 1, 2, 2, 2, 2]), + (vec![IoSlice::new(&[1]), IoSlice::new(&[2]), IoSlice::new(&[3])], &[1, 2, 3]), + (vec![IoSlice::new(&[1, 1]), IoSlice::new(&[2, 2]), IoSlice::new(&[3, 3])], &[1, 1, 2, 2, 3, 3]), + (vec![IoSlice::new(&[1]), IoSlice::new(&[2, 2]), IoSlice::new(&[3, 3, 3])], &[1, 2, 2, 3, 3, 3]), + (vec![IoSlice::new(&[1, 1, 1]), IoSlice::new(&[2, 2, 2]), IoSlice::new(&[3, 3, 3])], &[1, 1, 1, 2, 2, 2, 3, 3, 3]), + ]; + + let writer_configs = &[(1, 1), (1, 2), (1, 3), (2, 2), (2, 3), (3, 3)]; + + for (n_bufs, per_call) in writer_configs.iter().copied() { + for (mut input, wanted) in tests.clone().into_iter() { + let mut writer = test_writer(n_bufs, per_call); + assert!(writer.write_all_vectored(&mut *input).is_ok()); + assert_eq!(&*writer.written, &*wanted); + } + } +} + +// Issue 94981 +#[test] +#[should_panic = "number of read bytes exceeds limit"] +fn test_take_wrong_length() { + struct LieAboutSize(bool); + + impl Read for LieAboutSize { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + // Lie about the read size at first time of read. + if core::mem::take(&mut self.0) { Ok(buf.len() + 1) } else { Ok(buf.len()) } + } + } + + let mut buffer = vec![0; 4]; + let mut reader = LieAboutSize(true).take(4); + // Primed the `Limit` by lying about the read size. + let _ = reader.read(&mut buffer[..]); +} + +#[bench] +fn bench_take_read(b: &mut test::Bencher) { + b.iter(|| { + let mut buf = [0; 64]; + + [255; 128].take(64).read(&mut buf).unwrap(); + }); +} + +#[bench] +fn bench_take_read_buf(b: &mut test::Bencher) { + b.iter(|| { + let buf: &mut [_] = &mut [MaybeUninit::uninit(); 64]; + + let mut buf: BorrowedBuf<'_> = buf.into(); + + [255; 128].take(64).read_buf(buf.unfilled()).unwrap(); + }); +} diff --git a/crux-mir/lib/std/src/io/util.rs b/crux-mir/lib/std/src/io/util.rs index b09161b97..f076ee092 100644 --- a/crux-mir/lib/std/src/io/util.rs +++ b/crux-mir/lib/std/src/io/util.rs @@ -1,89 +1,25 @@ #![allow(missing_copy_implementations)] -use crate::fmt; -use crate::io::{self, BufRead, ErrorKind, Initializer, IoSlice, IoSliceMut, Read, Write}; -use crate::mem::MaybeUninit; - -/// Copies the entire contents of a reader into a writer. -/// -/// This function will continuously read data from `reader` and then -/// write it into `writer` in a streaming fashion until `reader` -/// returns EOF. -/// -/// On success, the total number of bytes that were copied from -/// `reader` to `writer` is returned. -/// -/// If you’re wanting to copy the contents of one file to another and you’re -/// working with filesystem paths, see the [`fs::copy`] function. -/// -/// [`fs::copy`]: ../fs/fn.copy.html -/// -/// # Errors -/// -/// This function will return an error immediately if any call to `read` or -/// `write` returns an error. All instances of `ErrorKind::Interrupted` are -/// handled by this function and the underlying operation is retried. -/// -/// # Examples -/// -/// ``` -/// use std::io; -/// -/// fn main() -> io::Result<()> { -/// let mut reader: &[u8] = b"hello"; -/// let mut writer: Vec = vec![]; -/// -/// io::copy(&mut reader, &mut writer)?; -/// -/// assert_eq!(&b"hello"[..], &writer[..]); -/// Ok(()) -/// } -/// ``` -#[stable(feature = "rust1", since = "1.0.0")] -pub fn copy(reader: &mut R, writer: &mut W) -> io::Result -where - R: Read, - W: Write, -{ - let mut buf = MaybeUninit::<[u8; super::DEFAULT_BUF_SIZE]>::uninit(); - // FIXME(#53491): This is calling `get_mut` and `get_ref` on an uninitialized - // `MaybeUninit`. Revisit this once we decided whether that is valid or not. - // This is still technically undefined behavior due to creating a reference - // to uninitialized data, but within libstd we can rely on more guarantees - // than if this code were in an external lib. - unsafe { - reader.initializer().initialize(buf.get_mut()); - } +#[cfg(test)] +mod tests; - let mut written = 0; - loop { - let len = match reader.read(unsafe { buf.get_mut() }) { - Ok(0) => return Ok(written), - Ok(len) => len, - Err(ref e) if e.kind() == ErrorKind::Interrupted => continue, - Err(e) => return Err(e), - }; - writer.write_all(unsafe { &buf.get_ref()[..len] })?; - written += len as u64; - } -} +use crate::fmt; +use crate::io::{ + self, BorrowedCursor, BufRead, IoSlice, IoSliceMut, Read, Seek, SeekFrom, SizeHint, Write, +}; /// A reader which is always at EOF. /// -/// This struct is generally created by calling [`empty`]. Please see -/// the documentation of [`empty()`][`empty`] for more details. -/// -/// [`empty`]: fn.empty.html +/// This struct is generally created by calling [`empty()`]. Please see +/// the documentation of [`empty()`] for more details. #[stable(feature = "rust1", since = "1.0.0")] -pub struct Empty { - _priv: (), -} +#[non_exhaustive] +#[derive(Copy, Clone, Default)] +pub struct Empty; /// Constructs a new handle to an empty reader. /// -/// All reads from the returned reader will return [`Ok`]`(0)`. -/// -/// [`Ok`]: ../result/enum.Result.html#variant.Ok +/// All reads from the returned reader will return [Ok]\(0). /// /// # Examples /// @@ -96,9 +32,11 @@ pub struct Empty { /// io::empty().read_to_string(&mut buffer).unwrap(); /// assert!(buffer.is_empty()); /// ``` +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] -pub fn empty() -> Empty { - Empty { _priv: () } +#[rustc_const_unstable(feature = "const_io_structs", issue = "78812")] +pub const fn empty() -> Empty { + Empty } #[stable(feature = "rust1", since = "1.0.0")] @@ -109,8 +47,8 @@ impl Read for Empty { } #[inline] - unsafe fn initializer(&self) -> Initializer { - Initializer::nop() + fn read_buf(&mut self, _cursor: BorrowedCursor<'_>) -> io::Result<()> { + Ok(()) } } #[stable(feature = "rust1", since = "1.0.0")] @@ -123,19 +61,39 @@ impl BufRead for Empty { fn consume(&mut self, _n: usize) {} } +#[stable(feature = "empty_seek", since = "1.51.0")] +impl Seek for Empty { + fn seek(&mut self, _pos: SeekFrom) -> io::Result { + Ok(0) + } + + fn stream_len(&mut self) -> io::Result { + Ok(0) + } + + fn stream_position(&mut self) -> io::Result { + Ok(0) + } +} + #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for Empty { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("Empty { .. }") + f.debug_struct("Empty").finish_non_exhaustive() + } +} + +impl SizeHint for Empty { + #[inline] + fn upper_bound(&self) -> Option { + Some(0) } } /// A reader which yields one byte over and over and over and over and over and... /// -/// This struct is generally created by calling [`repeat`][repeat]. Please -/// see the documentation of `repeat()` for more details. -/// -/// [repeat]: fn.repeat.html +/// This struct is generally created by calling [`repeat()`]. Please +/// see the documentation of [`repeat()`] for more details. #[stable(feature = "rust1", since = "1.0.0")] pub struct Repeat { byte: u8, @@ -155,8 +113,10 @@ pub struct Repeat { /// io::repeat(0b101).read_exact(&mut buffer).unwrap(); /// assert_eq!(buffer, [0b101, 0b101, 0b101]); /// ``` +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] -pub fn repeat(byte: u8) -> Repeat { +#[rustc_const_unstable(feature = "const_io_structs", issue = "78812")] +pub const fn repeat(byte: u8) -> Repeat { Repeat { byte } } @@ -170,6 +130,22 @@ impl Read for Repeat { Ok(buf.len()) } + fn read_buf(&mut self, mut buf: BorrowedCursor<'_>) -> io::Result<()> { + // SAFETY: No uninit bytes are being written + for slot in unsafe { buf.as_mut() } { + slot.write(self.byte); + } + + let remaining = buf.capacity(); + + // SAFETY: the entire unfilled portion of buf has been initialized + unsafe { + buf.advance(remaining); + } + + Ok(()) + } + #[inline] fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { let mut nwritten = 0; @@ -180,34 +156,46 @@ impl Read for Repeat { } #[inline] - unsafe fn initializer(&self) -> Initializer { - Initializer::nop() + fn is_read_vectored(&self) -> bool { + true + } +} + +impl SizeHint for Repeat { + #[inline] + fn lower_bound(&self) -> usize { + usize::MAX + } + + #[inline] + fn upper_bound(&self) -> Option { + None } } #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for Repeat { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("Repeat { .. }") + f.debug_struct("Repeat").finish_non_exhaustive() } } /// A writer which will move data into the void. /// -/// This struct is generally created by calling [`sink`][sink]. Please -/// see the documentation of `sink()` for more details. -/// -/// [sink]: fn.sink.html +/// This struct is generally created by calling [`sink`]. Please +/// see the documentation of [`sink()`] for more details. #[stable(feature = "rust1", since = "1.0.0")] -pub struct Sink { - _priv: (), -} +#[non_exhaustive] +#[derive(Copy, Clone, Default)] +pub struct Sink; /// Creates an instance of a writer which will successfully consume all data. /// -/// All calls to `write` on the returned instance will return `Ok(buf.len())` +/// All calls to [`write`] on the returned instance will return `Ok(buf.len())` /// and the contents of the buffer will not be inspected. /// +/// [`write`]: Write::write +/// /// # Examples /// /// ```rust @@ -217,9 +205,11 @@ pub struct Sink { /// let num_bytes = io::sink().write(&buffer).unwrap(); /// assert_eq!(num_bytes, 5); /// ``` +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] -pub fn sink() -> Sink { - Sink { _priv: () } +#[rustc_const_unstable(feature = "const_io_structs", issue = "78812")] +pub const fn sink() -> Sink { + Sink } #[stable(feature = "rust1", since = "1.0.0")] @@ -236,63 +226,43 @@ impl Write for Sink { } #[inline] - fn flush(&mut self) -> io::Result<()> { - Ok(()) + fn is_write_vectored(&self) -> bool { + true } -} -#[stable(feature = "std_debug", since = "1.16.0")] -impl fmt::Debug for Sink { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("Sink { .. }") + #[inline] + fn flush(&mut self) -> io::Result<()> { + Ok(()) } } -#[cfg(test)] -mod tests { - use crate::io::prelude::*; - use crate::io::{copy, empty, repeat, sink}; - - #[test] - fn copy_copies() { - let mut r = repeat(0).take(4); - let mut w = sink(); - assert_eq!(copy(&mut r, &mut w).unwrap(), 4); - - let mut r = repeat(0).take(1 << 17); - assert_eq!(copy(&mut r as &mut dyn Read, &mut w as &mut dyn Write).unwrap(), 1 << 17); +#[stable(feature = "write_mt", since = "1.48.0")] +impl Write for &Sink { + #[inline] + fn write(&mut self, buf: &[u8]) -> io::Result { + Ok(buf.len()) } - #[test] - fn sink_sinks() { - let mut s = sink(); - assert_eq!(s.write(&[]).unwrap(), 0); - assert_eq!(s.write(&[0]).unwrap(), 1); - assert_eq!(s.write(&[0; 1024]).unwrap(), 1024); - assert_eq!(s.by_ref().write(&[0; 1024]).unwrap(), 1024); + #[inline] + fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { + let total_len = bufs.iter().map(|b| b.len()).sum(); + Ok(total_len) } - #[test] - fn empty_reads() { - let mut e = empty(); - assert_eq!(e.read(&mut []).unwrap(), 0); - assert_eq!(e.read(&mut [0]).unwrap(), 0); - assert_eq!(e.read(&mut [0; 1024]).unwrap(), 0); - assert_eq!(e.by_ref().read(&mut [0; 1024]).unwrap(), 0); + #[inline] + fn is_write_vectored(&self) -> bool { + true } - #[test] - fn repeat_repeats() { - let mut r = repeat(4); - let mut b = [0; 1024]; - assert_eq!(r.read(&mut b).unwrap(), 1024); - assert!(b.iter().all(|b| *b == 4)); + #[inline] + fn flush(&mut self) -> io::Result<()> { + Ok(()) } +} - #[test] - fn take_some_bytes() { - assert_eq!(repeat(4).take(100).bytes().count(), 100); - assert_eq!(repeat(4).take(100).bytes().next().unwrap().unwrap(), 4); - assert_eq!(repeat(1).take(10).chain(repeat(2).take(10)).bytes().count(), 20); +#[stable(feature = "std_debug", since = "1.16.0")] +impl fmt::Debug for Sink { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Sink").finish_non_exhaustive() } } diff --git a/crux-mir/lib/std/src/io/util/tests.rs b/crux-mir/lib/std/src/io/util/tests.rs new file mode 100644 index 000000000..ce5e2c9da --- /dev/null +++ b/crux-mir/lib/std/src/io/util/tests.rs @@ -0,0 +1,147 @@ +use crate::cmp::{max, min}; +use crate::io::prelude::*; +use crate::io::{ + copy, empty, repeat, sink, BorrowedBuf, BufWriter, Empty, Repeat, Result, SeekFrom, Sink, + DEFAULT_BUF_SIZE, +}; + +use crate::mem::MaybeUninit; + +#[test] +fn copy_copies() { + let mut r = repeat(0).take(4); + let mut w = sink(); + assert_eq!(copy(&mut r, &mut w).unwrap(), 4); + + let mut r = repeat(0).take(1 << 17); + assert_eq!(copy(&mut r as &mut dyn Read, &mut w as &mut dyn Write).unwrap(), 1 << 17); +} + +struct ShortReader { + cap: usize, + read_size: usize, + observed_buffer: usize, +} + +impl Read for ShortReader { + fn read(&mut self, buf: &mut [u8]) -> Result { + let bytes = min(self.cap, self.read_size); + self.cap -= bytes; + self.observed_buffer = max(self.observed_buffer, buf.len()); + Ok(bytes) + } +} + +struct WriteObserver { + observed_buffer: usize, +} + +impl Write for WriteObserver { + fn write(&mut self, buf: &[u8]) -> Result { + self.observed_buffer = max(self.observed_buffer, buf.len()); + Ok(buf.len()) + } + + fn flush(&mut self) -> Result<()> { + Ok(()) + } +} + +#[test] +fn copy_specializes_bufwriter() { + let cap = 117 * 1024; + let buf_sz = 16 * 1024; + let mut r = ShortReader { cap, observed_buffer: 0, read_size: 1337 }; + let mut w = BufWriter::with_capacity(buf_sz, WriteObserver { observed_buffer: 0 }); + assert_eq!( + copy(&mut r, &mut w).unwrap(), + cap as u64, + "expected the whole capacity to be copied" + ); + assert_eq!(r.observed_buffer, buf_sz, "expected a large buffer to be provided to the reader"); + assert!(w.get_mut().observed_buffer > DEFAULT_BUF_SIZE, "expected coalesced writes"); +} + +#[test] +fn sink_sinks() { + let mut s = sink(); + assert_eq!(s.write(&[]).unwrap(), 0); + assert_eq!(s.write(&[0]).unwrap(), 1); + assert_eq!(s.write(&[0; 1024]).unwrap(), 1024); + assert_eq!(s.by_ref().write(&[0; 1024]).unwrap(), 1024); +} + +#[test] +fn empty_reads() { + let mut e = empty(); + assert_eq!(e.read(&mut []).unwrap(), 0); + assert_eq!(e.read(&mut [0]).unwrap(), 0); + assert_eq!(e.read(&mut [0; 1024]).unwrap(), 0); + assert_eq!(e.by_ref().read(&mut [0; 1024]).unwrap(), 0); + + let buf: &mut [MaybeUninit<_>] = &mut []; + let mut buf: BorrowedBuf<'_> = buf.into(); + e.read_buf(buf.unfilled()).unwrap(); + assert_eq!(buf.len(), 0); + assert_eq!(buf.init_len(), 0); + + let buf: &mut [_] = &mut [MaybeUninit::uninit()]; + let mut buf: BorrowedBuf<'_> = buf.into(); + e.read_buf(buf.unfilled()).unwrap(); + assert_eq!(buf.len(), 0); + assert_eq!(buf.init_len(), 0); + + let buf: &mut [_] = &mut [MaybeUninit::uninit(); 1024]; + let mut buf: BorrowedBuf<'_> = buf.into(); + e.read_buf(buf.unfilled()).unwrap(); + assert_eq!(buf.len(), 0); + assert_eq!(buf.init_len(), 0); + + let buf: &mut [_] = &mut [MaybeUninit::uninit(); 1024]; + let mut buf: BorrowedBuf<'_> = buf.into(); + e.by_ref().read_buf(buf.unfilled()).unwrap(); + assert_eq!(buf.len(), 0); + assert_eq!(buf.init_len(), 0); +} + +#[test] +fn empty_seeks() { + let mut e = empty(); + assert!(matches!(e.seek(SeekFrom::Start(0)), Ok(0))); + assert!(matches!(e.seek(SeekFrom::Start(1)), Ok(0))); + assert!(matches!(e.seek(SeekFrom::Start(u64::MAX)), Ok(0))); + + assert!(matches!(e.seek(SeekFrom::End(i64::MIN)), Ok(0))); + assert!(matches!(e.seek(SeekFrom::End(-1)), Ok(0))); + assert!(matches!(e.seek(SeekFrom::End(0)), Ok(0))); + assert!(matches!(e.seek(SeekFrom::End(1)), Ok(0))); + assert!(matches!(e.seek(SeekFrom::End(i64::MAX)), Ok(0))); + + assert!(matches!(e.seek(SeekFrom::Current(i64::MIN)), Ok(0))); + assert!(matches!(e.seek(SeekFrom::Current(-1)), Ok(0))); + assert!(matches!(e.seek(SeekFrom::Current(0)), Ok(0))); + assert!(matches!(e.seek(SeekFrom::Current(1)), Ok(0))); + assert!(matches!(e.seek(SeekFrom::Current(i64::MAX)), Ok(0))); +} + +#[test] +fn repeat_repeats() { + let mut r = repeat(4); + let mut b = [0; 1024]; + assert_eq!(r.read(&mut b).unwrap(), 1024); + assert!(b.iter().all(|b| *b == 4)); +} + +#[test] +fn take_some_bytes() { + assert_eq!(repeat(4).take(100).bytes().count(), 100); + assert_eq!(repeat(4).take(100).bytes().next().unwrap().unwrap(), 4); + assert_eq!(repeat(1).take(10).chain(repeat(2).take(10)).bytes().count(), 20); +} + +#[allow(dead_code)] +fn const_utils() { + const _: Empty = empty(); + const _: Repeat = repeat(b'c'); + const _: Sink = sink(); +} diff --git a/crux-mir/lib/std/src/keyword_docs.rs b/crux-mir/lib/std/src/keyword_docs.rs index 314424631..e35145c4a 100644 --- a/crux-mir/lib/std/src/keyword_docs.rs +++ b/crux-mir/lib/std/src/keyword_docs.rs @@ -15,18 +15,35 @@ /// ``` /// /// In general, any cast that can be performed via ascribing the type can also be done using `as`, -/// so instead of writing `let x: u32 = 123`, you can write `let x = 123 as u32` (Note: `let x: u32 -/// = 123` would be best in that situation). The same is not true in the other direction, however, +/// so instead of writing `let x: u32 = 123`, you can write `let x = 123 as u32` (note: `let x: u32 +/// = 123` would be best in that situation). The same is not true in the other direction, however; /// explicitly using `as` allows a few more coercions that aren't allowed implicitly, such as /// changing the type of a raw pointer or turning closures into raw pointers. /// -/// Other places `as` is used include as extra syntax for [`crate`] and `use`, to change the name -/// something is imported as. +/// `as` can be seen as the primitive for `From` and `Into`: `as` only works with primitives +/// (`u8`, `bool`, `str`, pointers, ...) whereas `From` and `Into` also works with types like +/// `String` or `Vec`. /// -/// For more information on what `as` is capable of, see the [Reference] +/// `as` can also be used with the `_` placeholder when the destination type can be inferred. Note +/// that this can cause inference breakage and usually such code should use an explicit type for +/// both clarity and stability. This is most useful when converting pointers using `as *const _` or +/// `as *mut _` though the [`cast`][const-cast] method is recommended over `as *const _` and it is +/// [the same][mut-cast] for `as *mut _`: those methods make the intent clearer. +/// +/// `as` is also used to rename imports in [`use`] and [`extern crate`][`crate`] statements: +/// +/// ``` +/// # #[allow(unused_imports)] +/// use std::{mem as memory, net as network}; +/// // Now you can use the names `memory` and `network` to refer to `std::mem` and `std::net`. +/// ``` +/// For more information on what `as` is capable of, see the [Reference]. /// /// [Reference]: ../reference/expressions/operator-expr.html#type-cast-expressions /// [`crate`]: keyword.crate.html +/// [`use`]: keyword.use.html +/// [const-cast]: pointer::cast +/// [mut-cast]: primitive.pointer.html#method.cast-1 mod as_keyword {} #[doc(keyword = "break")] @@ -47,20 +64,20 @@ mod as_keyword {} /// } /// /// assert_eq!(last, 12); -/// println!("{}", last); +/// println!("{last}"); /// ``` /// /// A break expression is normally associated with the innermost loop enclosing the /// `break` but a label can be used to specify which enclosing loop is affected. /// -///```rust +/// ```rust /// 'outer: for i in 1..=5 { -/// println!("outer iteration (i): {}", i); +/// println!("outer iteration (i): {i}"); /// /// '_inner: for j in 1..=200 { -/// println!(" inner iteration (j): {}", j); +/// println!(" inner iteration (j): {j}"); /// if j >= 3 { -/// // breaks from inner loop, let's outer loop continue. +/// // breaks from inner loop, lets outer loop continue. /// break; /// } /// if i >= 2 { @@ -70,7 +87,7 @@ mod as_keyword {} /// } /// } /// println!("Bye."); -///``` +/// ``` /// /// When associated with `loop`, a break expression may be used to return a value from that loop. /// This is only valid with `loop` and not with any other type of loop. @@ -89,7 +106,7 @@ mod as_keyword {} /// }; /// // first number in Fibonacci sequence over 10: /// assert_eq!(result, 13); -/// println!("{}", result); +/// println!("{result}"); /// ``` /// /// For more details consult the [Reference on "break expression"] and the [Reference on "break and @@ -98,17 +115,18 @@ mod as_keyword {} /// [Reference on "break expression"]: ../reference/expressions/loop-expr.html#break-expressions /// [Reference on "break and loop values"]: /// ../reference/expressions/loop-expr.html#break-and-loop-values -/// mod break_keyword {} #[doc(keyword = "const")] // -/// Compile-time constants and deterministic functions. +/// Compile-time constants, compile-time evaluable functions, and raw pointers. +/// +/// ## Compile-time constants /// /// Sometimes a certain value is used many times throughout a program, and it can become /// inconvenient to copy it over and over. What's more, it's not always possible or desirable to /// make it a variable that gets carried around to each function that needs it. In these cases, the -/// `const` keyword provides a convenient alternative to code duplication. +/// `const` keyword provides a convenient alternative to code duplication: /// /// ```rust /// const THING: u32 = 0xABAD1DEA; @@ -116,10 +134,12 @@ mod break_keyword {} /// let foo = 123 + THING; /// ``` /// -/// Constants must be explicitly typed, unlike with `let` you can't ignore its type and let the -/// compiler figure it out. Any constant value can be defined in a const, which in practice happens -/// to be most things that would be reasonable to have a constant (barring `const fn`s). For -/// example, you can't have a File as a `const`. +/// Constants must be explicitly typed; unlike with `let`, you can't ignore their type and let the +/// compiler figure it out. Any constant value can be defined in a `const`, which in practice happens +/// to be most things that would be reasonable to have in a constant (barring `const fn`s). For +/// example, you can't have a [`File`] as a `const`. +/// +/// [`File`]: crate::fs::File /// /// The only lifetime allowed in a constant is `'static`, which is the lifetime that encompasses /// all others in a Rust program. For example, if you wanted to define a constant string, it would @@ -129,7 +149,7 @@ mod break_keyword {} /// const WORDS: &'static str = "hello rust!"; /// ``` /// -/// Thanks to static lifetime elision, you usually don't have to explicitly use 'static: +/// Thanks to static lifetime elision, you usually don't have to explicitly use `'static`: /// /// ```rust /// const WORDS: &str = "hello convenience!"; @@ -137,22 +157,34 @@ mod break_keyword {} /// /// `const` items looks remarkably similar to `static` items, which introduces some confusion as /// to which one should be used at which times. To put it simply, constants are inlined wherever -/// they're used, making using them identical to simply replacing the name of the const with its -/// value. Static variables on the other hand point to a single location in memory, which all +/// they're used, making using them identical to simply replacing the name of the `const` with its +/// value. Static variables, on the other hand, point to a single location in memory, which all /// accesses share. This means that, unlike with constants, they can't have destructors, and act as /// a single value across the entire codebase. /// -/// Constants, as with statics, should always be in SCREAMING_SNAKE_CASE. +/// Constants, like statics, should always be in `SCREAMING_SNAKE_CASE`. /// -/// The `const` keyword is also used in raw pointers in combination with `mut`, as seen in `*const -/// T` and `*mut T`. More about that can be read at the [pointer] primitive part of the Rust docs. +/// For more detail on `const`, see the [Rust Book] or the [Reference]. +/// +/// ## Compile-time evaluable functions +/// +/// The other main use of the `const` keyword is in `const fn`. This marks a function as being +/// callable in the body of a `const` or `static` item and in array initializers (commonly called +/// "const contexts"). `const fn` are restricted in the set of operations they can perform, to +/// ensure that they can be evaluated at compile-time. See the [Reference][const-eval] for more +/// detail. +/// +/// Turning a `fn` into a `const fn` has no effect on run-time uses of that function. /// -/// For more detail on `const`, see the [Rust Book] or the [Reference] +/// ## Other uses of `const` /// -/// [pointer]: primitive.pointer.html -/// [Rust Book]: -/// ../book/ch03-01-variables-and-mutability.html#differences-between-variables-and-constants +/// The `const` keyword is also used in raw pointers in combination with `mut`, as seen in `*const +/// T` and `*mut T`. More about `const` as used in raw pointers can be read at the Rust docs for the [pointer primitive]. +/// +/// [pointer primitive]: pointer +/// [Rust Book]: ../book/ch03-01-variables-and-mutability.html#constants /// [Reference]: ../reference/items/constant-items.html +/// [const-eval]: ../reference/const_eval.html mod const_keyword {} #[doc(keyword = "continue")] @@ -162,20 +194,20 @@ mod const_keyword {} /// When `continue` is encountered, the current iteration is terminated, returning control to the /// loop head, typically continuing with the next iteration. /// -///```rust +/// ```rust /// // Printing odd numbers by skipping even ones /// for number in 1..=10 { /// if number % 2 == 0 { /// continue; /// } -/// println!("{}", number); +/// println!("{number}"); /// } -///``` +/// ``` /// /// Like `break`, `continue` is normally associated with the innermost enclosing loop, but labels /// may be used to specify the affected loop. /// -///```rust +/// ```rust /// // Print Odd numbers under 30 with unit <= 5 /// 'tens: for ten in 0..3 { /// '_units: for unit in 0..=9 { @@ -188,7 +220,7 @@ mod const_keyword {} /// println!("{}", ten * 10 + unit); /// } /// } -///``` +/// ``` /// /// See [continue expressions] from the reference for more details. /// @@ -330,13 +362,12 @@ mod else_keyword {} /// When data follows along with a variant, such as with rust's built-in [`Option`] type, the data /// is added as the type describes, for example `Option::Some(123)`. The same follows with /// struct-like variants, with things looking like `ComplexEnum::LotsOfThings { usual_struct_stuff: -/// true, blah: "hello!".to_string(), }`. Empty Enums are similar to () in that they cannot be +/// true, blah: "hello!".to_string(), }`. Empty Enums are similar to [`!`] in that they cannot be /// instantiated at all, and are used mainly to mess with the type system in interesting ways. /// /// For more information, take a look at the [Rust Book] or the [Reference] /// /// [ADT]: https://en.wikipedia.org/wiki/Algebraic_data_type -/// [`Option`]: option/enum.Option.html /// [Rust Book]: ../book/ch06-01-defining-an-enum.html /// [Reference]: ../reference/items/enumerations.html mod enum_keyword {} @@ -368,7 +399,7 @@ mod enum_keyword {} /// /// ```rust /// #[no_mangle] -/// pub extern fn callable_from_c(x: i32) -> bool { +/// pub extern "C" fn callable_from_c(x: i32) -> bool { /// x % 3 == 0 /// } /// ``` @@ -381,16 +412,18 @@ mod enum_keyword {} /// [Rust book]: /// ../book/ch19-01-unsafe-rust.html#using-extern-functions-to-call-external-code /// [Reference]: ../reference/items/external-blocks.html +/// [`crate`]: keyword.crate.html mod extern_keyword {} #[doc(keyword = "false")] // /// A value of type [`bool`] representing logical **false**. /// -/// The documentation for this keyword is [not yet complete]. Pull requests welcome! +/// `false` is the logical opposite of [`true`]. /// -/// [`bool`]: primitive.bool.html -/// [not yet complete]: https://github.com/rust-lang/rust/issues/34601 +/// See the documentation for [`true`] for more information. +/// +/// [`true`]: keyword.true.html mod false_keyword {} #[doc(keyword = "fn")] @@ -473,8 +506,8 @@ mod fn_keyword {} /// * `for` is also used for [higher-ranked trait bounds] as in `for<'a> &'a T: PartialEq`. /// /// for-in-loops, or to be more precise, iterator loops, are a simple syntactic sugar over a common -/// practice within Rust, which is to loop over an iterator until that iterator returns `None` (or -/// `break` is called). +/// practice within Rust, which is to loop over anything that implements [`IntoIterator`] until the +/// iterator returned by `.into_iter()` returns `None` (or the loop body uses `break`). /// /// ```rust /// for i in 0..5 { @@ -482,7 +515,7 @@ mod fn_keyword {} /// } /// /// for i in std::iter::repeat(5) { -/// println!("turns out {} never stops being 5", i); +/// println!("turns out {i} never stops being 5"); /// break; // would loop forever otherwise /// } /// @@ -514,15 +547,15 @@ mod fn_keyword {} /// # fn code() { } /// # let iterator = 0..2; /// { -/// let mut _iter = std::iter::IntoIterator::into_iter(iterator); -/// loop { -/// match _iter.next() { -/// Some(loop_variable) => { -/// code() -/// }, -/// None => break, -/// } -/// } +/// let result = match IntoIterator::into_iter(iterator) { +/// mut iter => loop { +/// match iter.next() { +/// None => break, +/// Some(loop_variable) => { code(); }, +/// }; +/// }, +/// }; +/// result /// } /// ``` /// @@ -530,10 +563,13 @@ mod fn_keyword {} /// /// For more information on for-loops, see the [Rust book] or the [Reference]. /// +/// See also, [`loop`], [`while`]. +/// /// [`in`]: keyword.in.html /// [`impl`]: keyword.impl.html +/// [`loop`]: keyword.loop.html +/// [`while`]: keyword.while.html /// [higher-ranked trait bounds]: ../reference/trait-bounds.html#higher-ranked-trait-bounds -/// [`IntoIterator`]: iter/trait.IntoIterator.html /// [Rust book]: /// ../book/ch03-05-control-flow.html#looping-through-a-collection-with-for /// [Reference]: ../reference/expressions/loop-expr.html#iterator-loops @@ -680,18 +716,32 @@ mod impl_keyword {} // /// Iterate over a series of values with [`for`]. /// -/// The expression immediately following `in` must implement the [`Iterator`] trait. +/// The expression immediately following `in` must implement the [`IntoIterator`] trait. /// /// ## Literal Examples: /// -/// * `for _ **in** 1..3 {}` - Iterate over an exclusive range up to but excluding 3. -/// * `for _ **in** 1..=3 {}` - Iterate over an inclusive range up to and includeing 3. +/// * `for _ in 1..3 {}` - Iterate over an exclusive range up to but excluding 3. +/// * `for _ in 1..=3 {}` - Iterate over an inclusive range up to and including 3. /// /// (Read more about [range patterns]) /// -/// [`Iterator`]: ../book/ch13-04-performance.html -/// [`range patterns`]: ../reference/patterns.html?highlight=range#range-patterns +/// [`IntoIterator`]: ../book/ch13-04-performance.html +/// [range patterns]: ../reference/patterns.html?highlight=range#range-patterns /// [`for`]: keyword.for.html +/// +/// The other use of `in` is with the keyword `pub`. It allows users to declare an item as visible +/// only within a given scope. +/// +/// ## Literal Example: +/// +/// * `pub(in crate::outer_mod) fn outer_mod_visible_fn() {}` - fn is visible in `outer_mod` +/// +/// Starting with the 2018 edition, paths for `pub(in path)` must start with `crate`, `self` or +/// `super`. The 2015 edition may also use paths starting with `::` or modules from the crate root. +/// +/// For more information, see the [Reference]. +/// +/// [Reference]: ../reference/visibility-and-privacy.html#pubin-path-pubcrate-pubsuper-and-pubself mod in_keyword {} #[doc(keyword = "let")] @@ -740,7 +790,7 @@ mod in_keyword {} /// let shadowing_example = true; /// let shadowing_example = 123.4; /// let shadowing_example = shadowing_example as u32; -/// let mut shadowing_example = format!("cool! {}", shadowing_example); +/// let mut shadowing_example = format!("cool! {shadowing_example}"); /// shadowing_example += " something else!"; // not shadowing /// ``` /// @@ -769,7 +819,7 @@ mod let_keyword {} /// let mut counter = 0; /// /// while counter < 10 { -/// println!("{}", counter); +/// println!("{counter}"); /// counter += 1; /// } /// ``` @@ -800,7 +850,7 @@ mod let_keyword {} /// if i == 10 { /// counter = None; /// } else { -/// println!("{}", i); +/// println!("{i}"); /// counter = Some (i + 1); /// } /// } @@ -808,6 +858,8 @@ mod let_keyword {} /// /// For more information on `while` and loops in general, see the [reference]. /// +/// See also, [`for`], [`loop`]. +/// /// [`for`]: keyword.for.html /// [`loop`]: keyword.loop.html /// [reference]: ../reference/expressions/loop-expr.html#predicate-loops @@ -828,7 +880,7 @@ mod while_keyword {} /// /// let mut i = 1; /// loop { -/// println!("i is {}", i); +/// println!("i is {i}"); /// if i > 100 { /// break; /// } @@ -856,6 +908,10 @@ mod while_keyword {} /// /// For more information on `loop` and loops in general, see the [Reference]. /// +/// See also, [`for`], [`while`]. +/// +/// [`for`]: keyword.for.html +/// [`while`]: keyword.while.html /// [Reference]: ../reference/expressions/loop-expr.html mod loop_keyword {} @@ -878,8 +934,8 @@ mod loop_keyword {} /// /// let a_number = Option::Some(10); /// match a_number { -/// Some(x) if x <= 5 => println!("0 to 5 num = {}", x), -/// Some(x @ 6..=10) => println!("6 to 10 num = {}", x), +/// Some(x) if x <= 5 => println!("0 to 5 num = {x}"), +/// Some(x @ 6..=10) => println!("6 to 10 num = {x}"), /// None => panic!(), /// // all other numbers /// _ => panic!(), @@ -898,8 +954,8 @@ mod loop_keyword {} /// /// let get_inner = Outer::Double(None, Some(String::new())); /// match get_inner { -/// Outer::Double(None, Some(st)) => println!("{}", st), -/// Outer::Single(opt) => println!("{:?}", opt), +/// Outer::Double(None, Some(st)) => println!("{st}"), +/// Outer::Single(opt) => println!("{opt:?}"), /// _ => panic!(), /// } /// ``` @@ -913,10 +969,28 @@ mod match_keyword {} // /// Organize code into [modules]. /// -/// The documentation for this keyword is [not yet complete]. Pull requests welcome! +/// Use `mod` to create new [modules] to encapsulate code, including other +/// modules: /// +/// ``` +/// mod foo { +/// mod bar { +/// type MyType = (u8, u8); +/// fn baz() {} +/// } +/// } +/// ``` +/// +/// Like [`struct`]s and [`enum`]s, a module and its content are private by +/// default, inaccessible to code outside of the module. +/// +/// To learn more about allowing access, see the documentation for the [`pub`] +/// keyword. +/// +/// [`enum`]: keyword.enum.html +/// [`pub`]: keyword.pub.html +/// [`struct`]: keyword.struct.html /// [modules]: ../reference/items/modules.html -/// [not yet complete]: https://github.com/rust-lang/rust/issues/34601 mod mod_keyword {} #[doc(keyword = "move")] @@ -924,52 +998,114 @@ mod mod_keyword {} /// Capture a [closure]'s environment by value. /// /// `move` converts any variables captured by reference or mutable reference -/// to owned by value variables. The three [`Fn` trait]'s mirror the ways to capture -/// variables, when `move` is used, the closures is represented by the `FnOnce` trait. +/// to variables captured by value. /// /// ```rust -/// let capture = "hello"; -/// let closure = move || { -/// println!("rust says {}", capture); -/// }; +/// let data = vec![1, 2, 3]; +/// let closure = move || println!("captured {data:?} by value"); +/// +/// // data is no longer available, it is owned by the closure +/// ``` +/// +/// Note: `move` closures may still implement [`Fn`] or [`FnMut`], even though +/// they capture variables by `move`. This is because the traits implemented by +/// a closure type are determined by *what* the closure does with captured +/// values, not *how* it captures them: +/// +/// ```rust +/// fn create_fn() -> impl Fn() { +/// let text = "Fn".to_owned(); +/// move || println!("This is a: {text}") +/// } +/// +/// let fn_plain = create_fn(); +/// fn_plain(); /// ``` /// /// `move` is often used when [threads] are involved. /// /// ```rust -/// let x = 5; +/// let data = vec![1, 2, 3]; /// /// std::thread::spawn(move || { -/// println!("captured {} by value", x) +/// println!("captured {data:?} by value") /// }).join().unwrap(); /// -/// // x is no longer available +/// // data was moved to the spawned thread, so we cannot use it here /// ``` /// /// `move` is also valid before an async block. /// /// ```rust -/// let capture = "hello"; +/// let capture = "hello".to_owned(); /// let block = async move { -/// println!("rust says {} from async block", capture); +/// println!("rust says {capture} from async block"); /// }; /// ``` /// -/// For more information on the `move` keyword, see the [closure]'s section -/// of the Rust book or the [threads] section +/// For more information on the `move` keyword, see the [closures][closure] section +/// of the Rust book or the [threads] section. /// -/// [`Fn` trait]: ../std/ops/trait.Fn.html /// [closure]: ../book/ch13-01-closures.html /// [threads]: ../book/ch16-01-threads.html#using-move-closures-with-threads mod move_keyword {} #[doc(keyword = "mut")] // -/// A mutable binding, reference, or pointer. +/// A mutable variable, reference, or pointer. +/// +/// `mut` can be used in several situations. The first is mutable variables, +/// which can be used anywhere you can bind a value to a variable name. Some +/// examples: +/// +/// ```rust +/// // A mutable variable in the parameter list of a function. +/// fn foo(mut x: u8, y: u8) -> u8 { +/// x += y; +/// x +/// } +/// +/// // Modifying a mutable variable. +/// # #[allow(unused_assignments)] +/// let mut a = 5; +/// a = 6; +/// +/// assert_eq!(foo(3, 4), 7); +/// assert_eq!(a, 6); +/// ``` +/// +/// The second is mutable references. They can be created from `mut` variables +/// and must be unique: no other variables can have a mutable reference, nor a +/// shared reference. +/// +/// ```rust +/// // Taking a mutable reference. +/// fn push_two(v: &mut Vec) { +/// v.push(2); +/// } +/// +/// // A mutable reference cannot be taken to a non-mutable variable. +/// let mut v = vec![0, 1]; +/// // Passing a mutable reference. +/// push_two(&mut v); +/// +/// assert_eq!(v, vec![0, 1, 2]); +/// ``` +/// +/// ```rust,compile_fail,E0502 +/// let mut v = vec![0, 1]; +/// let mut_ref_v = &mut v; +/// ##[allow(unused)] +/// let ref_v = &v; +/// mut_ref_v.push(2); +/// ``` /// -/// The documentation for this keyword is [not yet complete]. Pull requests welcome! +/// Mutable raw pointers work much like mutable references, with the added +/// possibility of not pointing to a valid object. The syntax is `*mut Type`. /// -/// [not yet complete]: https://github.com/rust-lang/rust/issues/34601 +/// More information on mutable references and pointers can be found in the [Reference]. +/// +/// [Reference]: ../reference/types/pointer.html#mutable-references-mut mod mut_keyword {} #[doc(keyword = "pub")] @@ -991,48 +1127,349 @@ mod pub_keyword {} // /// Bind by reference during pattern matching. /// -/// The documentation for this keyword is [not yet complete]. Pull requests welcome! +/// `ref` annotates pattern bindings to make them borrow rather than move. +/// It is **not** a part of the pattern as far as matching is concerned: it does +/// not affect *whether* a value is matched, only *how* it is matched. +/// +/// By default, [`match`] statements consume all they can, which can sometimes +/// be a problem, when you don't really need the value to be moved and owned: +/// +/// ```compile_fail,E0382 +/// let maybe_name = Some(String::from("Alice")); +/// // The variable 'maybe_name' is consumed here ... +/// match maybe_name { +/// Some(n) => println!("Hello, {n}"), +/// _ => println!("Hello, world"), +/// } +/// // ... and is now unavailable. +/// println!("Hello again, {}", maybe_name.unwrap_or("world".into())); +/// ``` +/// +/// Using the `ref` keyword, the value is only borrowed, not moved, making it +/// available for use after the [`match`] statement: +/// +/// ``` +/// let maybe_name = Some(String::from("Alice")); +/// // Using `ref`, the value is borrowed, not moved ... +/// match maybe_name { +/// Some(ref n) => println!("Hello, {n}"), +/// _ => println!("Hello, world"), +/// } +/// // ... so it's available here! +/// println!("Hello again, {}", maybe_name.unwrap_or("world".into())); +/// ``` +/// +/// # `&` vs `ref` +/// +/// - `&` denotes that your pattern expects a reference to an object. Hence `&` +/// is a part of said pattern: `&Foo` matches different objects than `Foo` does. +/// +/// - `ref` indicates that you want a reference to an unpacked value. It is not +/// matched against: `Foo(ref foo)` matches the same objects as `Foo(foo)`. /// -/// [not yet complete]: https://github.com/rust-lang/rust/issues/34601 +/// See also the [Reference] for more information. +/// +/// [`match`]: keyword.match.html +/// [Reference]: ../reference/patterns.html#identifier-patterns mod ref_keyword {} #[doc(keyword = "return")] // /// Return a value from a function. /// -/// The documentation for this keyword is [not yet complete]. Pull requests welcome! +/// A `return` marks the end of an execution path in a function: +/// +/// ``` +/// fn foo() -> i32 { +/// return 3; +/// } +/// assert_eq!(foo(), 3); +/// ``` +/// +/// `return` is not needed when the returned value is the last expression in the +/// function. In this case the `;` is omitted: +/// +/// ``` +/// fn foo() -> i32 { +/// 3 +/// } +/// assert_eq!(foo(), 3); +/// ``` +/// +/// `return` returns from the function immediately (an "early return"): +/// +/// ```no_run +/// use std::fs::File; +/// use std::io::{Error, ErrorKind, Read, Result}; +/// +/// fn main() -> Result<()> { +/// let mut file = match File::open("foo.txt") { +/// Ok(f) => f, +/// Err(e) => return Err(e), +/// }; /// -/// [not yet complete]: https://github.com/rust-lang/rust/issues/34601 +/// let mut contents = String::new(); +/// let size = match file.read_to_string(&mut contents) { +/// Ok(s) => s, +/// Err(e) => return Err(e), +/// }; +/// +/// if contents.contains("impossible!") { +/// return Err(Error::new(ErrorKind::Other, "oh no!")); +/// } +/// +/// if size > 9000 { +/// return Err(Error::new(ErrorKind::Other, "over 9000!")); +/// } +/// +/// assert_eq!(contents, "Hello, world!"); +/// Ok(()) +/// } +/// ``` mod return_keyword {} #[doc(keyword = "self")] // /// The receiver of a method, or the current module. /// -/// The documentation for this keyword is [not yet complete]. Pull requests welcome! +/// `self` is used in two situations: referencing the current module and marking +/// the receiver of a method. /// -/// [not yet complete]: https://github.com/rust-lang/rust/issues/34601 +/// In paths, `self` can be used to refer to the current module, either in a +/// [`use`] statement or in a path to access an element: +/// +/// ``` +/// # #![allow(unused_imports)] +/// use std::io::{self, Read}; +/// ``` +/// +/// Is functionally the same as: +/// +/// ``` +/// # #![allow(unused_imports)] +/// use std::io; +/// use std::io::Read; +/// ``` +/// +/// Using `self` to access an element in the current module: +/// +/// ``` +/// # #![allow(dead_code)] +/// # fn main() {} +/// fn foo() {} +/// fn bar() { +/// self::foo() +/// } +/// ``` +/// +/// `self` as the current receiver for a method allows to omit the parameter +/// type most of the time. With the exception of this particularity, `self` is +/// used much like any other parameter: +/// +/// ``` +/// struct Foo(i32); +/// +/// impl Foo { +/// // No `self`. +/// fn new() -> Self { +/// Self(0) +/// } +/// +/// // Consuming `self`. +/// fn consume(self) -> Self { +/// Self(self.0 + 1) +/// } +/// +/// // Borrowing `self`. +/// fn borrow(&self) -> &i32 { +/// &self.0 +/// } +/// +/// // Borrowing `self` mutably. +/// fn borrow_mut(&mut self) -> &mut i32 { +/// &mut self.0 +/// } +/// } +/// +/// // This method must be called with a `Type::` prefix. +/// let foo = Foo::new(); +/// assert_eq!(foo.0, 0); +/// +/// // Those two calls produces the same result. +/// let foo = Foo::consume(foo); +/// assert_eq!(foo.0, 1); +/// let foo = foo.consume(); +/// assert_eq!(foo.0, 2); +/// +/// // Borrowing is handled automatically with the second syntax. +/// let borrow_1 = Foo::borrow(&foo); +/// let borrow_2 = foo.borrow(); +/// assert_eq!(borrow_1, borrow_2); +/// +/// // Borrowing mutably is handled automatically too with the second syntax. +/// let mut foo = Foo::new(); +/// *Foo::borrow_mut(&mut foo) += 1; +/// assert_eq!(foo.0, 1); +/// *foo.borrow_mut() += 1; +/// assert_eq!(foo.0, 2); +/// ``` +/// +/// Note that this automatic conversion when calling `foo.method()` is not +/// limited to the examples above. See the [Reference] for more information. +/// +/// [`use`]: keyword.use.html +/// [Reference]: ../reference/items/associated-items.html#methods mod self_keyword {} -#[doc(keyword = "Self")] +// FIXME: Once rustdoc can handle URL conflicts on case insensitive file systems, we can remove the +// three next lines and put back: `#[doc(keyword = "Self")]`. +#[doc(alias = "Self")] +#[allow(rustc::existing_doc_keyword)] +#[doc(keyword = "SelfTy")] // /// The implementing type within a [`trait`] or [`impl`] block, or the current type within a type /// definition. /// -/// The documentation for this keyword is [not yet complete]. Pull requests welcome! +/// Within a type definition: +/// +/// ``` +/// # #![allow(dead_code)] +/// struct Node { +/// elem: i32, +/// // `Self` is a `Node` here. +/// next: Option>, +/// } +/// ``` +/// +/// In an [`impl`] block: +/// +/// ``` +/// struct Foo(i32); +/// +/// impl Foo { +/// fn new() -> Self { +/// Self(0) +/// } +/// } +/// +/// assert_eq!(Foo::new().0, Foo(0).0); +/// ``` +/// +/// Generic parameters are implicit with `Self`: +/// +/// ``` +/// # #![allow(dead_code)] +/// struct Wrap { +/// elem: T, +/// } +/// +/// impl Wrap { +/// fn new(elem: T) -> Self { +/// Self { elem } +/// } +/// } +/// ``` +/// +/// In a [`trait`] definition and related [`impl`] block: +/// +/// ``` +/// trait Example { +/// fn example() -> Self; +/// } +/// +/// struct Foo(i32); +/// +/// impl Example for Foo { +/// fn example() -> Self { +/// Self(42) +/// } +/// } +/// +/// assert_eq!(Foo::example().0, Foo(42).0); +/// ``` /// /// [`impl`]: keyword.impl.html /// [`trait`]: keyword.trait.html -/// [not yet complete]: https://github.com/rust-lang/rust/issues/34601 mod self_upper_keyword {} #[doc(keyword = "static")] // -/// A place that is valid for the duration of a program. +/// A static item is a value which is valid for the entire duration of your +/// program (a `'static` lifetime). +/// +/// On the surface, `static` items seem very similar to [`const`]s: both contain +/// a value, both require type annotations and both can only be initialized with +/// constant functions and values. However, `static`s are notably different in +/// that they represent a location in memory. That means that you can have +/// references to `static` items and potentially even modify them, making them +/// essentially global variables. +/// +/// Static items do not call [`drop`] at the end of the program. /// -/// The documentation for this keyword is [not yet complete]. Pull requests welcome! +/// There are two types of `static` items: those declared in association with +/// the [`mut`] keyword and those without. /// -/// [not yet complete]: https://github.com/rust-lang/rust/issues/34601 +/// Static items cannot be moved: +/// +/// ```rust,compile_fail,E0507 +/// static VEC: Vec = vec![]; +/// +/// fn move_vec(v: Vec) -> Vec { +/// v +/// } +/// +/// // This line causes an error +/// move_vec(VEC); +/// ``` +/// +/// # Simple `static`s +/// +/// Accessing non-[`mut`] `static` items is considered safe, but some +/// restrictions apply. Most notably, the type of a `static` value needs to +/// implement the [`Sync`] trait, ruling out interior mutability containers +/// like [`RefCell`]. See the [Reference] for more information. +/// +/// ```rust +/// static FOO: [i32; 5] = [1, 2, 3, 4, 5]; +/// +/// let r1 = &FOO as *const _; +/// let r2 = &FOO as *const _; +/// // With a strictly read-only static, references will have the same address +/// assert_eq!(r1, r2); +/// // A static item can be used just like a variable in many cases +/// println!("{FOO:?}"); +/// ``` +/// +/// # Mutable `static`s +/// +/// If a `static` item is declared with the [`mut`] keyword, then it is allowed +/// to be modified by the program. However, accessing mutable `static`s can +/// cause undefined behavior in a number of ways, for example due to data races +/// in a multithreaded context. As such, all accesses to mutable `static`s +/// require an [`unsafe`] block. +/// +/// Despite their unsafety, mutable `static`s are necessary in many contexts: +/// they can be used to represent global state shared by the whole program or in +/// [`extern`] blocks to bind to variables from C libraries. +/// +/// In an [`extern`] block: +/// +/// ```rust,no_run +/// # #![allow(dead_code)] +/// extern "C" { +/// static mut ERROR_MESSAGE: *mut std::os::raw::c_char; +/// } +/// ``` +/// +/// Mutable `static`s, just like simple `static`s, have some restrictions that +/// apply to them. See the [Reference] for more information. +/// +/// [`const`]: keyword.const.html +/// [`extern`]: keyword.extern.html +/// [`mut`]: keyword.mut.html +/// [`unsafe`]: keyword.unsafe.html +/// [`RefCell`]: cell::RefCell +/// [Reference]: ../reference/items/static-items.html mod static_keyword {} #[doc(keyword = "struct")] @@ -1138,7 +1575,7 @@ mod static_keyword {} /// For more information on structs, take a look at the [Rust Book][book] or the /// [Reference][reference]. /// -/// [`PhantomData`]: marker/struct.PhantomData.html +/// [`PhantomData`]: marker::PhantomData /// [book]: ../book/ch05-01-defining-structs.html /// [reference]: ../reference/items/structs.html mod struct_keyword {} @@ -1147,19 +1584,211 @@ mod struct_keyword {} // /// The parent of the current [module]. /// -/// The documentation for this keyword is [not yet complete]. Pull requests welcome! +/// ```rust +/// # #![allow(dead_code)] +/// # fn main() {} +/// mod a { +/// pub fn foo() {} +/// } +/// mod b { +/// pub fn foo() { +/// super::a::foo(); // call a's foo function +/// } +/// } +/// ``` +/// +/// It is also possible to use `super` multiple times: `super::super::foo`, +/// going up the ancestor chain. +/// +/// See the [Reference] for more information. /// /// [module]: ../reference/items/modules.html -/// [not yet complete]: https://github.com/rust-lang/rust/issues/34601 +/// [Reference]: ../reference/paths.html#super mod super_keyword {} #[doc(keyword = "trait")] // -/// A common interface for a class of types. +/// A common interface for a group of types. +/// +/// A `trait` is like an interface that data types can implement. When a type +/// implements a trait it can be treated abstractly as that trait using generics +/// or trait objects. +/// +/// Traits can be made up of three varieties of associated items: +/// +/// - functions and methods +/// - types +/// - constants +/// +/// Traits may also contain additional type parameters. Those type parameters +/// or the trait itself can be constrained by other traits. +/// +/// Traits can serve as markers or carry other logical semantics that +/// aren't expressed through their items. When a type implements that +/// trait it is promising to uphold its contract. [`Send`] and [`Sync`] are two +/// such marker traits present in the standard library. +/// +/// See the [Reference][Ref-Traits] for a lot more information on traits. +/// +/// # Examples +/// +/// Traits are declared using the `trait` keyword. Types can implement them +/// using [`impl`] `Trait` [`for`] `Type`: +/// +/// ```rust +/// trait Zero { +/// const ZERO: Self; +/// fn is_zero(&self) -> bool; +/// } +/// +/// impl Zero for i32 { +/// const ZERO: Self = 0; +/// +/// fn is_zero(&self) -> bool { +/// *self == Self::ZERO +/// } +/// } +/// +/// assert_eq!(i32::ZERO, 0); +/// assert!(i32::ZERO.is_zero()); +/// assert!(!4.is_zero()); +/// ``` +/// +/// With an associated type: +/// +/// ```rust +/// trait Builder { +/// type Built; +/// +/// fn build(&self) -> Self::Built; +/// } +/// ``` +/// +/// Traits can be generic, with constraints or without: +/// +/// ```rust +/// trait MaybeFrom { +/// fn maybe_from(value: T) -> Option +/// where +/// Self: Sized; +/// } +/// ``` +/// +/// Traits can build upon the requirements of other traits. In the example +/// below `Iterator` is a **supertrait** and `ThreeIterator` is a **subtrait**: +/// +/// ```rust +/// trait ThreeIterator: std::iter::Iterator { +/// fn next_three(&mut self) -> Option<[Self::Item; 3]>; +/// } +/// ``` +/// +/// Traits can be used in functions, as parameters: +/// +/// ```rust +/// # #![allow(dead_code)] +/// fn debug_iter(it: I) where I::Item: std::fmt::Debug { +/// for elem in it { +/// println!("{elem:#?}"); +/// } +/// } +/// +/// // u8_len_1, u8_len_2 and u8_len_3 are equivalent +/// +/// fn u8_len_1(val: impl Into>) -> usize { +/// val.into().len() +/// } +/// +/// fn u8_len_2>>(val: T) -> usize { +/// val.into().len() +/// } +/// +/// fn u8_len_3(val: T) -> usize +/// where +/// T: Into>, +/// { +/// val.into().len() +/// } +/// ``` +/// +/// Or as return types: +/// +/// ```rust +/// # #![allow(dead_code)] +/// fn from_zero_to(v: u8) -> impl Iterator { +/// (0..v).into_iter() +/// } +/// ``` +/// +/// The use of the [`impl`] keyword in this position allows the function writer +/// to hide the concrete type as an implementation detail which can change +/// without breaking user's code. +/// +/// # Trait objects +/// +/// A *trait object* is an opaque value of another type that implements a set of +/// traits. A trait object implements all specified traits as well as their +/// supertraits (if any). +/// +/// The syntax is the following: `dyn BaseTrait + AutoTrait1 + ... AutoTraitN`. +/// Only one `BaseTrait` can be used so this will not compile: +/// +/// ```rust,compile_fail,E0225 +/// trait A {} +/// trait B {} +/// +/// let _: Box; +/// ``` +/// +/// Neither will this, which is a syntax error: +/// +/// ```rust,compile_fail +/// trait A {} +/// trait B {} +/// +/// let _: Box; +/// ``` +/// +/// On the other hand, this is correct: +/// +/// ```rust +/// trait A {} +/// +/// let _: Box; +/// ``` +/// +/// The [Reference][Ref-Trait-Objects] has more information about trait objects, +/// their limitations and the differences between editions. +/// +/// # Unsafe traits +/// +/// Some traits may be unsafe to implement. Using the [`unsafe`] keyword in +/// front of the trait's declaration is used to mark this: +/// +/// ```rust +/// unsafe trait UnsafeTrait {} +/// +/// unsafe impl UnsafeTrait for i32 {} +/// ``` +/// +/// # Differences between the 2015 and 2018 editions +/// +/// In the 2015 edition the parameters pattern was not needed for traits: +/// +/// ```rust,edition2015 +/// # #![allow(anonymous_parameters)] +/// trait Tr { +/// fn f(i32); +/// } +/// ``` /// -/// The documentation for this keyword is [not yet complete]. Pull requests welcome! +/// This behavior is no longer valid in edition 2018. /// -/// [not yet complete]: https://github.com/rust-lang/rust/issues/34601 +/// [`for`]: keyword.for.html +/// [`impl`]: keyword.impl.html +/// [`unsafe`]: keyword.unsafe.html +/// [Ref-Traits]: ../reference/items/traits.html +/// [Ref-Trait-Objects]: ../reference/types/trait-object.html mod trait_keyword {} #[doc(keyword = "true")] @@ -1187,65 +1816,508 @@ mod trait_keyword {} /// [`while`]: keyword.while.html /// [`match`]: ../reference/expressions/match-expr.html#match-guards /// [`false`]: keyword.false.html -/// [`bool`]: primitive.bool.html mod true_keyword {} #[doc(keyword = "type")] // /// Define an alias for an existing type. /// -/// The documentation for this keyword is [not yet complete]. Pull requests welcome! +/// The syntax is `type Name = ExistingType;`. +/// +/// # Examples +/// +/// `type` does **not** create a new type: +/// +/// ```rust +/// type Meters = u32; +/// type Kilograms = u32; +/// +/// let m: Meters = 3; +/// let k: Kilograms = 3; +/// +/// assert_eq!(m, k); +/// ``` +/// +/// In traits, `type` is used to declare an [associated type]: +/// +/// ```rust +/// trait Iterator { +/// // associated type declaration +/// type Item; +/// fn next(&mut self) -> Option; +/// } /// -/// [not yet complete]: https://github.com/rust-lang/rust/issues/34601 +/// struct Once(Option); +/// +/// impl Iterator for Once { +/// // associated type definition +/// type Item = T; +/// fn next(&mut self) -> Option { +/// self.0.take() +/// } +/// } +/// ``` +/// +/// [`trait`]: keyword.trait.html +/// [associated type]: ../reference/items/associated-items.html#associated-types mod type_keyword {} #[doc(keyword = "unsafe")] // -/// Code or interfaces whose [memory safety] cannot be verified by the type system. +/// Code or interfaces whose [memory safety] cannot be verified by the type +/// system. +/// +/// The `unsafe` keyword has two uses: +/// - to declare the existence of contracts the compiler can't check (`unsafe fn` and `unsafe +/// trait`), +/// - and to declare that a programmer has checked that these contracts have been upheld (`unsafe +/// {}` and `unsafe impl`, but also `unsafe fn` -- see below). +/// +/// They are not mutually exclusive, as can be seen in `unsafe fn`: the body of an `unsafe fn` is, +/// by default, treated like an unsafe block. The `unsafe_op_in_unsafe_fn` lint can be enabled to +/// change that. +/// +/// # Unsafe abilities +/// +/// **No matter what, Safe Rust can't cause Undefined Behavior**. This is +/// referred to as [soundness]: a well-typed program actually has the desired +/// properties. The [Nomicon][nomicon-soundness] has a more detailed explanation +/// on the subject. +/// +/// To ensure soundness, Safe Rust is restricted enough that it can be +/// automatically checked. Sometimes, however, it is necessary to write code +/// that is correct for reasons which are too clever for the compiler to +/// understand. In those cases, you need to use Unsafe Rust. +/// +/// Here are the abilities Unsafe Rust has in addition to Safe Rust: +/// +/// - Dereference [raw pointers] +/// - Implement `unsafe` [`trait`]s +/// - Call `unsafe` functions +/// - Mutate [`static`]s (including [`extern`]al ones) +/// - Access fields of [`union`]s +/// +/// However, this extra power comes with extra responsibilities: it is now up to +/// you to ensure soundness. The `unsafe` keyword helps by clearly marking the +/// pieces of code that need to worry about this. +/// +/// ## The different meanings of `unsafe` +/// +/// Not all uses of `unsafe` are equivalent: some are here to mark the existence +/// of a contract the programmer must check, others are to say "I have checked +/// the contract, go ahead and do this". The following +/// [discussion on Rust Internals] has more in-depth explanations about this but +/// here is a summary of the main points: +/// +/// - `unsafe fn`: calling this function means abiding by a contract the +/// compiler cannot enforce. +/// - `unsafe trait`: implementing the [`trait`] means abiding by a +/// contract the compiler cannot enforce. +/// - `unsafe {}`: the contract necessary to call the operations inside the +/// block has been checked by the programmer and is guaranteed to be respected. +/// - `unsafe impl`: the contract necessary to implement the trait has been +/// checked by the programmer and is guaranteed to be respected. +/// +/// By default, `unsafe fn` also acts like an `unsafe {}` block +/// around the code inside the function. This means it is not just a signal to +/// the caller, but also promises that the preconditions for the operations +/// inside the function are upheld. Mixing these two meanings can be confusing, so the +/// `unsafe_op_in_unsafe_fn` lint can be enabled to warn against that and require explicit unsafe +/// blocks even inside `unsafe fn`. +/// +/// See the [Rustnomicon] and the [Reference] for more information. +/// +/// # Examples +/// +/// ## Marking elements as `unsafe` +/// +/// `unsafe` can be used on functions. Note that functions and statics declared +/// in [`extern`] blocks are implicitly marked as `unsafe` (but not functions +/// declared as `extern "something" fn ...`). Mutable statics are always unsafe, +/// wherever they are declared. Methods can also be declared as `unsafe`: +/// +/// ```rust +/// # #![allow(dead_code)] +/// static mut FOO: &str = "hello"; +/// +/// unsafe fn unsafe_fn() {} +/// +/// extern "C" { +/// fn unsafe_extern_fn(); +/// static BAR: *mut u32; +/// } +/// +/// trait SafeTraitWithUnsafeMethod { +/// unsafe fn unsafe_method(&self); +/// } +/// +/// struct S; +/// +/// impl S { +/// unsafe fn unsafe_method_on_struct() {} +/// } +/// ``` +/// +/// Traits can also be declared as `unsafe`: +/// +/// ```rust +/// unsafe trait UnsafeTrait {} +/// ``` +/// +/// Since `unsafe fn` and `unsafe trait` indicate that there is a safety +/// contract that the compiler cannot enforce, documenting it is important. The +/// standard library has many examples of this, like the following which is an +/// extract from [`Vec::set_len`]. The `# Safety` section explains the contract +/// that must be fulfilled to safely call the function. +/// +/// ```rust,ignore (stub-to-show-doc-example) +/// /// Forces the length of the vector to `new_len`. +/// /// +/// /// This is a low-level operation that maintains none of the normal +/// /// invariants of the type. Normally changing the length of a vector +/// /// is done using one of the safe operations instead, such as +/// /// `truncate`, `resize`, `extend`, or `clear`. +/// /// +/// /// # Safety +/// /// +/// /// - `new_len` must be less than or equal to `capacity()`. +/// /// - The elements at `old_len..new_len` must be initialized. +/// pub unsafe fn set_len(&mut self, new_len: usize) +/// ``` /// -/// The documentation for this keyword is [not yet complete]. Pull requests welcome! +/// ## Using `unsafe {}` blocks and `impl`s /// +/// Performing `unsafe` operations requires an `unsafe {}` block: +/// +/// ```rust +/// # #![allow(dead_code)] +/// #![deny(unsafe_op_in_unsafe_fn)] +/// +/// /// Dereference the given pointer. +/// /// +/// /// # Safety +/// /// +/// /// `ptr` must be aligned and must not be dangling. +/// unsafe fn deref_unchecked(ptr: *const i32) -> i32 { +/// // SAFETY: the caller is required to ensure that `ptr` is aligned and dereferenceable. +/// unsafe { *ptr } +/// } +/// +/// let a = 3; +/// let b = &a as *const _; +/// // SAFETY: `a` has not been dropped and references are always aligned, +/// // so `b` is a valid address. +/// unsafe { assert_eq!(*b, deref_unchecked(b)); }; +/// ``` +/// +/// ## `unsafe` and traits +/// +/// The interactions of `unsafe` and traits can be surprising, so let us contrast the +/// two combinations of safe `fn` in `unsafe trait` and `unsafe fn` in safe trait using two +/// examples: +/// +/// ```rust +/// /// # Safety +/// /// +/// /// `make_even` must return an even number. +/// unsafe trait MakeEven { +/// fn make_even(&self) -> i32; +/// } +/// +/// // SAFETY: Our `make_even` always returns something even. +/// unsafe impl MakeEven for i32 { +/// fn make_even(&self) -> i32 { +/// self << 1 +/// } +/// } +/// +/// fn use_make_even(x: impl MakeEven) { +/// if x.make_even() % 2 == 1 { +/// // SAFETY: this can never happen, because all `MakeEven` implementations +/// // ensure that `make_even` returns something even. +/// unsafe { std::hint::unreachable_unchecked() }; +/// } +/// } +/// ``` +/// +/// Note how the safety contract of the trait is upheld by the implementation, and is itself used to +/// uphold the safety contract of the unsafe function `unreachable_unchecked` called by +/// `use_make_even`. `make_even` itself is a safe function because its *callers* do not have to +/// worry about any contract, only the *implementation* of `MakeEven` is required to uphold a +/// certain contract. `use_make_even` is safe because it can use the promise made by `MakeEven` +/// implementations to uphold the safety contract of the `unsafe fn unreachable_unchecked` it calls. +/// +/// It is also possible to have `unsafe fn` in a regular safe `trait`: +/// +/// ```rust +/// # #![feature(never_type)] +/// #![deny(unsafe_op_in_unsafe_fn)] +/// +/// trait Indexable { +/// const LEN: usize; +/// +/// /// # Safety +/// /// +/// /// The caller must ensure that `idx < LEN`. +/// unsafe fn idx_unchecked(&self, idx: usize) -> i32; +/// } +/// +/// // The implementation for `i32` doesn't need to do any contract reasoning. +/// impl Indexable for i32 { +/// const LEN: usize = 1; +/// +/// unsafe fn idx_unchecked(&self, idx: usize) -> i32 { +/// debug_assert_eq!(idx, 0); +/// *self +/// } +/// } +/// +/// // The implementation for arrays exploits the function contract to +/// // make use of `get_unchecked` on slices and avoid a run-time check. +/// impl Indexable for [i32; 42] { +/// const LEN: usize = 42; +/// +/// unsafe fn idx_unchecked(&self, idx: usize) -> i32 { +/// // SAFETY: As per this trait's documentation, the caller ensures +/// // that `idx < 42`. +/// unsafe { *self.get_unchecked(idx) } +/// } +/// } +/// +/// // The implementation for the never type declares a length of 0, +/// // which means `idx_unchecked` can never be called. +/// impl Indexable for ! { +/// const LEN: usize = 0; +/// +/// unsafe fn idx_unchecked(&self, idx: usize) -> i32 { +/// // SAFETY: As per this trait's documentation, the caller ensures +/// // that `idx < 0`, which is impossible, so this is dead code. +/// unsafe { std::hint::unreachable_unchecked() } +/// } +/// } +/// +/// fn use_indexable(x: I, idx: usize) -> i32 { +/// if idx < I::LEN { +/// // SAFETY: We have checked that `idx < I::LEN`. +/// unsafe { x.idx_unchecked(idx) } +/// } else { +/// panic!("index out-of-bounds") +/// } +/// } +/// ``` +/// +/// This time, `use_indexable` is safe because it uses a run-time check to discharge the safety +/// contract of `idx_unchecked`. Implementing `Indexable` is safe because when writing +/// `idx_unchecked`, we don't have to worry: our *callers* need to discharge a proof obligation +/// (like `use_indexable` does), but the *implementation* of `get_unchecked` has no proof obligation +/// to contend with. Of course, the implementation of `Indexable` may choose to call other unsafe +/// operations, and then it needs an `unsafe` *block* to indicate it discharged the proof +/// obligations of its callees. (We enabled `unsafe_op_in_unsafe_fn`, so the body of `idx_unchecked` +/// is not implicitly an unsafe block.) For that purpose it can make use of the contract that all +/// its callers must uphold -- the fact that `idx < LEN`. +/// +/// Formally speaking, an `unsafe fn` in a trait is a function with *preconditions* that go beyond +/// those encoded by the argument types (such as `idx < LEN`), whereas an `unsafe trait` can declare +/// that some of its functions have *postconditions* that go beyond those encoded in the return type +/// (such as returning an even integer). If a trait needs a function with both extra precondition +/// and extra postcondition, then it needs an `unsafe fn` in an `unsafe trait`. +/// +/// [`extern`]: keyword.extern.html +/// [`trait`]: keyword.trait.html +/// [`static`]: keyword.static.html +/// [`union`]: keyword.union.html +/// [`impl`]: keyword.impl.html +/// [raw pointers]: ../reference/types/pointer.html /// [memory safety]: ../book/ch19-01-unsafe-rust.html -/// [not yet complete]: https://github.com/rust-lang/rust/issues/34601 +/// [Rustnomicon]: ../nomicon/index.html +/// [nomicon-soundness]: ../nomicon/safe-unsafe-meaning.html +/// [soundness]: https://rust-lang.github.io/unsafe-code-guidelines/glossary.html#soundness-of-code--of-a-library +/// [Reference]: ../reference/unsafety.html +/// [discussion on Rust Internals]: https://internals.rust-lang.org/t/what-does-unsafe-mean/6696 mod unsafe_keyword {} #[doc(keyword = "use")] // /// Import or rename items from other crates or modules. /// -/// The documentation for this keyword is [not yet complete]. Pull requests welcome! +/// Usually a `use` keyword is used to shorten the path required to refer to a module item. +/// The keyword may appear in modules, blocks and even functions, usually at the top. +/// +/// The most basic usage of the keyword is `use path::to::item;`, +/// though a number of convenient shortcuts are supported: +/// +/// * Simultaneously binding a list of paths with a common prefix, +/// using the glob-like brace syntax `use a::b::{c, d, e::f, g::h::i};` +/// * Simultaneously binding a list of paths with a common prefix and their common parent module, +/// using the [`self`] keyword, such as `use a::b::{self, c, d::e};` +/// * Rebinding the target name as a new local name, using the syntax `use p::q::r as x;`. +/// This can also be used with the last two features: `use a::b::{self as ab, c as abc}`. +/// * Binding all paths matching a given prefix, +/// using the asterisk wildcard syntax `use a::b::*;`. +/// * Nesting groups of the previous features multiple times, +/// such as `use a::b::{self as ab, c, d::{*, e::f}};` +/// * Reexporting with visibility modifiers such as `pub use a::b;` +/// * Importing with `_` to only import the methods of a trait without binding it to a name +/// (to avoid conflict for example): `use ::std::io::Read as _;`. +/// +/// Using path qualifiers like [`crate`], [`super`] or [`self`] is supported: `use crate::a::b;`. +/// +/// Note that when the wildcard `*` is used on a type, it does not import its methods (though +/// for `enum`s it imports the variants, as shown in the example below). +/// +/// ```compile_fail,edition2018 +/// enum ExampleEnum { +/// VariantA, +/// VariantB, +/// } +/// +/// impl ExampleEnum { +/// fn new() -> Self { +/// Self::VariantA +/// } +/// } +/// +/// use ExampleEnum::*; +/// +/// // Compiles. +/// let _ = VariantA; +/// +/// // Does not compile ! +/// let n = new(); +/// ``` /// -/// [not yet complete]: https://github.com/rust-lang/rust/issues/34601 +/// For more information on `use` and paths in general, see the [Reference]. +/// +/// The differences about paths and the `use` keyword between the 2015 and 2018 editions +/// can also be found in the [Reference]. +/// +/// [`crate`]: keyword.crate.html +/// [`self`]: keyword.self.html +/// [`super`]: keyword.super.html +/// [Reference]: ../reference/items/use-declarations.html mod use_keyword {} #[doc(keyword = "where")] // /// Add constraints that must be upheld to use an item. /// -/// The documentation for this keyword is [not yet complete]. Pull requests welcome! +/// `where` allows specifying constraints on lifetime and generic parameters. +/// The [RFC] introducing `where` contains detailed information about the +/// keyword. +/// +/// # Examples +/// +/// `where` can be used for constraints with traits: +/// +/// ```rust +/// fn new() -> T { +/// T::default() +/// } +/// +/// fn new_where() -> T +/// where +/// T: Default, +/// { +/// T::default() +/// } +/// +/// assert_eq!(0.0, new()); +/// assert_eq!(0.0, new_where()); +/// +/// assert_eq!(0, new()); +/// assert_eq!(0, new_where()); +/// ``` +/// +/// `where` can also be used for lifetimes. +/// +/// This compiles because `longer` outlives `shorter`, thus the constraint is +/// respected: /// -/// [not yet complete]: https://github.com/rust-lang/rust/issues/34601 +/// ```rust +/// fn select<'short, 'long>(s1: &'short str, s2: &'long str, second: bool) -> &'short str +/// where +/// 'long: 'short, +/// { +/// if second { s2 } else { s1 } +/// } +/// +/// let outer = String::from("Long living ref"); +/// let longer = &outer; +/// { +/// let inner = String::from("Short living ref"); +/// let shorter = &inner; +/// +/// assert_eq!(select(shorter, longer, false), shorter); +/// assert_eq!(select(shorter, longer, true), longer); +/// } +/// ``` +/// +/// On the other hand, this will not compile because the `where 'b: 'a` clause +/// is missing: the `'b` lifetime is not known to live at least as long as `'a` +/// which means this function cannot ensure it always returns a valid reference: +/// +/// ```rust,compile_fail +/// fn select<'a, 'b>(s1: &'a str, s2: &'b str, second: bool) -> &'a str +/// { +/// if second { s2 } else { s1 } +/// } +/// ``` +/// +/// `where` can also be used to express more complicated constraints that cannot +/// be written with the `` syntax: +/// +/// ```rust +/// fn first_or_default(mut i: I) -> I::Item +/// where +/// I: Iterator, +/// I::Item: Default, +/// { +/// i.next().unwrap_or_else(I::Item::default) +/// } +/// +/// assert_eq!(first_or_default([1, 2, 3].into_iter()), 1); +/// assert_eq!(first_or_default(Vec::::new().into_iter()), 0); +/// ``` +/// +/// `where` is available anywhere generic and lifetime parameters are available, +/// as can be seen with the [`Cow`](crate::borrow::Cow) type from the standard +/// library: +/// +/// ```rust +/// # #![allow(dead_code)] +/// pub enum Cow<'a, B> +/// where +/// B: 'a + ToOwned + ?Sized, +/// { +/// Borrowed(&'a B), +/// Owned(::Owned), +/// } +/// ``` +/// +/// [RFC]: https://github.com/rust-lang/rfcs/blob/master/text/0135-where.md mod where_keyword {} // 2018 Edition keywords +#[doc(alias = "promise")] #[doc(keyword = "async")] // /// Return a [`Future`] instead of blocking the current thread. /// /// Use `async` in front of `fn`, `closure`, or a `block` to turn the marked code into a `Future`. /// As such the code will not be run immediately, but will only be evaluated when the returned -/// future is `.await`ed. +/// future is [`.await`]ed. /// -/// We have written an [async book] detailing async/await and trade-offs compared to using threads. +/// We have written an [async book] detailing `async`/`await` and trade-offs compared to using threads. /// /// ## Editions /// /// `async` is a keyword from the 2018 edition onwards. /// -/// It is available for use in stable rust from version 1.39 onwards. +/// It is available for use in stable Rust from version 1.39 onwards. /// -/// [`Future`]: ./future/trait.Future.html +/// [`Future`]: future::Future +/// [`.await`]: ../std/keyword.await.html /// [async book]: https://rust-lang.github.io/async-book/ mod async_keyword {} @@ -1253,19 +2325,20 @@ mod async_keyword {} // /// Suspend execution until the result of a [`Future`] is ready. /// -/// `.await`ing a future will suspend the current function's execution until the `executor` +/// `.await`ing a future will suspend the current function's execution until the executor /// has run the future to completion. /// -/// Read the [async book] for details on how async/await and executors work. +/// Read the [async book] for details on how [`async`]/`await` and executors work. /// /// ## Editions /// /// `await` is a keyword from the 2018 edition onwards. /// -/// It is available for use in stable rust from version 1.39 onwards. +/// It is available for use in stable Rust from version 1.39 onwards. /// -/// [`Future`]: ./future/trait.Future.html +/// [`Future`]: future::Future /// [async book]: https://rust-lang.github.io/async-book/ +/// [`async`]: ../std/keyword.async.html mod await_keyword {} #[doc(keyword = "dyn")] @@ -1273,7 +2346,7 @@ mod await_keyword {} /// `dyn` is a prefix of a [trait object]'s type. /// /// The `dyn` keyword is used to highlight that calls to methods on the associated `Trait` -/// are dynamically dispatched. To use the trait this way, it must be 'object safe'. +/// are [dynamically dispatched]. To use the trait this way, it must be 'object safe'. /// /// Unlike generic parameters or `impl Trait`, the compiler does not know the concrete type that /// is being passed. That is, the type has been [erased]. @@ -1285,6 +2358,9 @@ mod await_keyword {} /// At run-time, when a method needs to be called on the `dyn Trait`, the vtable is consulted to get /// the function pointer and then that function pointer is called. /// +/// See the Reference for more information on [trait objects][ref-trait-obj] +/// and [object safety][ref-obj-safety]. +/// /// ## Trade-offs /// /// The above indirection is the additional runtime cost of calling a function on a `dyn Trait`. @@ -1293,9 +2369,10 @@ mod await_keyword {} /// However, `dyn Trait` is likely to produce smaller code than `impl Trait` / generic parameters as /// the method won't be duplicated for each concrete type. /// -/// Read more about `object safety` and [trait object]s. -/// /// [trait object]: ../book/ch17-02-trait-objects.html +/// [dynamically dispatched]: https://en.wikipedia.org/wiki/Dynamic_dispatch +/// [ref-trait-obj]: ../reference/types/trait-object.html +/// [ref-obj-safety]: ../reference/items/traits.html#object-safety /// [erased]: https://en.wikipedia.org/wiki/Type_erasure mod dyn_keyword {} @@ -1303,8 +2380,72 @@ mod dyn_keyword {} // /// The [Rust equivalent of a C-style union][union]. /// -/// The documentation for this keyword is [not yet complete]. Pull requests welcome! +/// A `union` looks like a [`struct`] in terms of declaration, but all of its +/// fields exist in the same memory, superimposed over one another. For instance, +/// if we wanted some bits in memory that we sometimes interpret as a `u32` and +/// sometimes as an `f32`, we could write: +/// +/// ```rust +/// union IntOrFloat { +/// i: u32, +/// f: f32, +/// } +/// +/// let mut u = IntOrFloat { f: 1.0 }; +/// // Reading the fields of a union is always unsafe +/// assert_eq!(unsafe { u.i }, 1065353216); +/// // Updating through any of the field will modify all of them +/// u.i = 1073741824; +/// assert_eq!(unsafe { u.f }, 2.0); +/// ``` +/// +/// # Matching on unions +/// +/// It is possible to use pattern matching on `union`s. A single field name must +/// be used and it must match the name of one of the `union`'s field. +/// Like reading from a `union`, pattern matching on a `union` requires `unsafe`. +/// +/// ```rust +/// union IntOrFloat { +/// i: u32, +/// f: f32, +/// } +/// +/// let u = IntOrFloat { f: 1.0 }; +/// +/// unsafe { +/// match u { +/// IntOrFloat { i: 10 } => println!("Found exactly ten!"), +/// // Matching the field `f` provides an `f32`. +/// IntOrFloat { f } => println!("Found f = {f} !"), +/// } +/// } +/// ``` +/// +/// # References to union fields +/// +/// All fields in a `union` are all at the same place in memory which means +/// borrowing one borrows the entire `union`, for the same lifetime: +/// +/// ```rust,compile_fail,E0502 +/// union IntOrFloat { +/// i: u32, +/// f: f32, +/// } +/// +/// let mut u = IntOrFloat { f: 1.0 }; +/// +/// let f = unsafe { &u.f }; +/// // This will not compile because the field has already been borrowed, even +/// // if only immutably +/// let i = unsafe { &mut u.i }; +/// +/// *i = 10; +/// println!("f = {f} and i = {i}"); +/// ``` +/// +/// See the [Reference][union] for more information on `union`s. /// +/// [`struct`]: keyword.struct.html /// [union]: ../reference/items/unions.html -/// [not yet complete]: https://github.com/rust-lang/rust/issues/34601 mod union_keyword {} diff --git a/crux-mir/lib/std/src/lib.rs b/crux-mir/lib/std/src/lib.rs index 0bcce1b4a..a7e13f5b8 100644 --- a/crux-mir/lib/std/src/lib.rs +++ b/crux-mir/lib/std/src/lib.rs @@ -14,7 +14,7 @@ //! # How to read this documentation //! //! If you already know the name of what you are looking for, the fastest way to -//! find it is to use the search +//! find it is to use the search //! bar at the top of the page. //! //! Otherwise, you may want to jump to one of these useful sections: @@ -22,7 +22,7 @@ //! * [`std::*` modules](#modules) //! * [Primitive types](#primitives) //! * [Standard macros](#macros) -//! * [The Rust Prelude](prelude/index.html) +//! * [The Rust Prelude] //! //! If this is your first time, the documentation for the standard library is //! written to be casually perused. Clicking on interesting things should @@ -35,8 +35,8 @@ //! development you may want to press the `[-]` button near the top of the //! page to collapse it into a more skimmable view. //! -//! While you are looking at that `[-]` button also notice the `[src]` -//! button. Rust's API documentation comes with the source code and you are +//! While you are looking at that `[-]` button also notice the `source` +//! link. Rust's API documentation comes with the source code and you are //! encouraged to read it. The standard library source is generally high //! quality and a peek behind the curtains is often enlightening. //! @@ -61,14 +61,14 @@ //! type, but not the all-important methods. //! //! So for example there is a [page for the primitive type -//! `i32`](primitive.i32.html) that lists all the methods that can be called on +//! `i32`](primitive::i32) that lists all the methods that can be called on //! 32-bit integers (very useful), and there is a [page for the module -//! `std::i32`](i32/index.html) that documents the constant values [`MIN`] and -//! [`MAX`](i32/constant.MAX.html) (rarely useful). +//! `std::i32`] that documents the constant values [`MIN`] and [`MAX`] (rarely +//! useful). //! -//! Note the documentation for the primitives [`str`] and [`[T]`][slice] (also +//! Note the documentation for the primitives [`str`] and [`[T]`][prim@slice] (also //! called 'slice'). Many method calls on [`String`] and [`Vec`] are actually -//! calls to methods on [`str`] and [`[T]`][slice] respectively, via [deref +//! calls to methods on [`str`] and [`[T]`][prim@slice] respectively, via [deref //! coercions][deref-coercions]. //! //! Third, the standard library defines [The Rust Prelude], a small collection @@ -85,13 +85,15 @@ //! # Contributing changes to the documentation //! //! Check out the rust contribution guidelines [here]( -//! https://github.com/rust-lang/rust/blob/master/CONTRIBUTING.md). -//! The source for this documentation can be found on [Github](https://github.com/rust-lang). +//! https://rustc-dev-guide.rust-lang.org/contributing.html#writing-documentation). +//! The source for this documentation can be found on +//! [GitHub](https://github.com/rust-lang/rust). //! To contribute changes, make sure you read the guidelines first, then submit //! pull-requests for your suggested changes. //! //! Contributions are appreciated! If you see a part of the docs that can be -//! improved, submit a PR, or chat with us first on irc.mozilla.org #rust-docs. +//! improved, submit a PR, or chat with us first on [Discord][rust-discord] +//! #docs. //! //! # A Tour of The Rust Standard Library //! @@ -109,8 +111,8 @@ //! regions of memory: //! //! * [`Vec`] - A heap-allocated *vector* that is resizable at runtime. -//! * [`[T; n]`][array] - An inline *array* with a fixed size at compile time. -//! * [`[T]`][slice] - A dynamically sized *slice* into any other kind of contiguous +//! * [`[T; N]`][prim@array] - An inline *array* with a fixed size at compile time. +//! * [`[T]`][prim@slice] - A dynamically sized *slice* into any other kind of contiguous //! storage, whether heap-allocated or not. //! //! Slices can only be handled through some kind of *pointer*, and as such come @@ -143,183 +145,230 @@ //! abstracting over differences in common platforms, most notably Windows and //! Unix derivatives. //! -//! Common types of I/O, including [files], [TCP], [UDP], are defined in the -//! [`io`], [`fs`], and [`net`] modules. +//! Common types of I/O, including [files], [TCP], and [UDP], are defined in +//! the [`io`], [`fs`], and [`net`] modules. //! //! The [`thread`] module contains Rust's threading abstractions. [`sync`] //! contains further primitive shared memory types, including [`atomic`] and //! [`mpsc`], which contains the channel types for message passing. //! -//! [I/O]: io/index.html -//! [`MIN`]: i32/constant.MIN.html -//! [TCP]: net/struct.TcpStream.html -//! [The Rust Prelude]: prelude/index.html -//! [UDP]: net/struct.UdpSocket.html -//! [`Arc`]: sync/struct.Arc.html -//! [owned slice]: boxed/index.html -//! [`Cell`]: cell/struct.Cell.html -//! [`FromStr`]: str/trait.FromStr.html -//! [`HashMap`]: collections/struct.HashMap.html -//! [`Iterator`]: iter/trait.Iterator.html -//! [`Mutex`]: sync/struct.Mutex.html -//! [`Option`]: option/enum.Option.html -//! [`Rc`]: rc/struct.Rc.html -//! [`RefCell`]: cell/struct.RefCell.html -//! [`Result`]: result/enum.Result.html -//! [`String`]: string/struct.String.html -//! [`Vec`]: vec/struct.Vec.html -//! [array]: primitive.array.html -//! [slice]: primitive.slice.html -//! [`atomic`]: sync/atomic/index.html -//! [`collections`]: collections/index.html +//! [I/O]: io +//! [`MIN`]: i32::MIN +//! [`MAX`]: i32::MAX +//! [page for the module `std::i32`]: crate::i32 +//! [TCP]: net::TcpStream +//! [The Rust Prelude]: prelude +//! [UDP]: net::UdpSocket +//! [`Arc`]: sync::Arc +//! [owned slice]: boxed +//! [`Cell`]: cell::Cell +//! [`FromStr`]: str::FromStr +//! [`HashMap`]: collections::HashMap +//! [`Mutex`]: sync::Mutex +//! [`Option`]: option::Option +//! [`Rc`]: rc::Rc +//! [`RefCell`]: cell::RefCell +//! [`Result`]: result::Result +//! [`Vec`]: vec::Vec +//! [`atomic`]: sync::atomic //! [`for`]: ../book/ch03-05-control-flow.html#looping-through-a-collection-with-for -//! [`format!`]: macro.format.html -//! [`fs`]: fs/index.html -//! [`io`]: io/index.html -//! [`iter`]: iter/index.html -//! [`mpsc`]: sync/mpsc/index.html -//! [`net`]: net/index.html -//! [`option`]: option/index.html -//! [`result`]: result/index.html -//! [`std::cmp`]: cmp/index.html -//! [`std::slice`]: slice/index.html -//! [`str`]: primitive.str.html -//! [`sync`]: sync/index.html -//! [`thread`]: thread/index.html +//! [`str`]: prim@str +//! [`mpsc`]: sync::mpsc +//! [`std::cmp`]: cmp +//! [`std::slice`]: mod@slice //! [`use std::env`]: env/index.html //! [`use`]: ../book/ch07-02-defining-modules-to-control-scope-and-privacy.html //! [crates.io]: https://crates.io //! [deref-coercions]: ../book/ch15-02-deref.html#implicit-deref-coercions-with-functions-and-methods -//! [files]: fs/struct.File.html -//! [multithreading]: thread/index.html +//! [files]: fs::File +//! [multithreading]: thread //! [other]: #what-is-in-the-standard-library-documentation //! [primitive types]: ../book/ch03-02-data-types.html +//! [rust-discord]: https://discord.gg/rust-lang +//! [array]: prim@array +//! [slice]: prim@slice -#![stable(feature = "rust1", since = "1.0.0")] +#![cfg_attr(not(feature = "restricted-std"), stable(feature = "rust1", since = "1.0.0"))] +#![cfg_attr(feature = "restricted-std", unstable(feature = "restricted_std", issue = "none"))] #![doc( - html_root_url = "https://doc.rust-lang.org/nightly/", html_playground_url = "https://play.rust-lang.org/", issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/", test(no_crate_inject, attr(deny(warnings))), test(attr(allow(dead_code, deprecated, unused_variables, unused_mut))) )] +#![doc(cfg_hide( + not(test), + not(any(test, bootstrap)), + no_global_oom_handling, + not(no_global_oom_handling) +))] +// To run std tests without x.py without ending up with two copies of std, Miri needs to be +// able to "empty" this crate. See . +// rustc itself never sets the feature, so this line has no affect there. +#![cfg(any(not(feature = "miri-test-libstd"), test, doctest))] +// miri-test-libstd also prefers to make std use the sysroot versions of the dependencies. +#![cfg_attr(feature = "miri-test-libstd", feature(rustc_private))] // Don't link to std. We are std. #![no_std] +// Tell the compiler to link to either panic_abort or panic_unwind +#![needs_panic_runtime] +// +// Lints: #![warn(deprecated_in_future)] #![warn(missing_docs)] #![warn(missing_debug_implementations)] -#![deny(intra_doc_link_resolution_failure)] // rustdoc is run without -D warnings #![allow(explicit_outlives_requirements)] #![allow(unused_lifetimes)] -// Tell the compiler to link to either panic_abort or panic_unwind -#![needs_panic_runtime] +#![deny(rustc::existing_doc_keyword)] +#![deny(fuzzy_provenance_casts)] +// Ensure that std can be linked against panic_abort despite compiled with `-C panic=unwind` +#![deny(ffi_unwind_calls)] // std may use features in a platform-specific way #![allow(unused_features)] -#![cfg_attr(test, feature(print_internals, set_stdio, update_panic_count))] +// +// Features: +#![cfg_attr(test, feature(internal_output_capture, print_internals, update_panic_count, rt))] #![cfg_attr( all(target_vendor = "fortanix", target_env = "sgx"), - feature(slice_index_methods, coerce_unsized, sgx_platform, ptr_wrapping_offset_from) -)] -#![cfg_attr( - all(test, target_vendor = "fortanix", target_env = "sgx"), - feature(fixed_size_array, maybe_uninit_extra) + feature(slice_index_methods, coerce_unsized, sgx_platform) )] -// std is implemented with unstable features, many of which are internal -// compiler details that will never be stable -// NB: the following list is sorted to minimize merge conflicts. +// +// Language features: #![feature(alloc_error_handler)] -#![feature(alloc_layout_extra)] -#![feature(allocator_api)] #![feature(allocator_internals)] #![feature(allow_internal_unsafe)] #![feature(allow_internal_unstable)] -#![feature(arbitrary_self_types)] -#![feature(array_error_internals)] -#![feature(asm)] -#![feature(associated_type_bounds)] -#![feature(atomic_mut_ptr)] #![feature(box_syntax)] -#![feature(c_variadic)] -#![cfg_attr(not(bootstrap), feature(cfg_accessible))] -#![feature(cfg_target_has_atomic)] +#![feature(c_unwind)] #![feature(cfg_target_thread_local)] -#![feature(char_error_internals)] -#![feature(clamp)] #![feature(concat_idents)] -#![feature(const_cstr_unchecked)] -#![feature(const_raw_ptr_deref)] -#![feature(container_error_extra)] -#![feature(core_intrinsics)] -#![feature(custom_test_frameworks)] +#![feature(const_mut_refs)] +#![feature(const_trait_impl)] #![feature(decl_macro)] -#![feature(doc_alias)] +#![feature(deprecated_suggestion)] #![feature(doc_cfg)] -#![feature(doc_keyword)] +#![feature(doc_cfg_hide)] #![feature(doc_masked)] +#![feature(doc_notable_trait)] #![feature(dropck_eyepatch)] -#![feature(duration_constants)] -#![feature(exact_size_is_empty)] #![feature(exhaustive_patterns)] -#![feature(external_doc)] -#![feature(fn_traits)] -#![feature(format_args_nl)] -#![feature(generator_trait)] -#![feature(global_asm)] -#![feature(hashmap_internals)] -#![feature(int_error_internals)] -#![feature(int_error_matching)] -#![feature(integer_atomics)] +#![feature(if_let_guard)] +#![feature(intra_doc_pointers)] +#![feature(is_terminal)] #![feature(lang_items)] -#![feature(libc)] -#![feature(link_args)] +#![feature(let_chains)] #![feature(linkage)] -#![feature(log_syntax)] -#![feature(maybe_uninit_ref)] -#![feature(maybe_uninit_slice)] +#![feature(link_cfg)] +#![feature(min_specialization)] +#![feature(must_not_suspend)] #![feature(needs_panic_runtime)] +#![feature(negative_impls)] #![feature(never_type)] -#![feature(nll)] -#![feature(optin_builtin_traits)] -#![feature(panic_info_message)] -#![feature(panic_internals)] -#![feature(panic_unwind)] +#![feature(platform_intrinsics)] #![feature(prelude_import)] -#![feature(ptr_internals)] -#![feature(raw)] -#![feature(renamed_spin_loop)] #![feature(rustc_attrs)] -#![feature(rustc_private)] -#![feature(shrink_to)] -#![feature(slice_concat_ext)] -#![feature(slice_internals)] -#![feature(slice_iter_mut_as_slice)] -#![cfg_attr(bootstrap, feature(specialization))] -#![cfg_attr(not(bootstrap), feature(min_specialization))] +#![feature(rustdoc_internals)] #![feature(staged_api)] +#![feature(thread_local)] +#![feature(try_blocks)] +#![feature(utf8_chunks)] +// +// Library features (core): +#![feature(array_error_internals)] +#![feature(atomic_mut_ptr)] +#![feature(char_error_internals)] +#![feature(char_internals)] +#![feature(core_intrinsics)] +#![feature(cstr_from_bytes_until_nul)] +#![feature(cstr_internals)] +#![feature(duration_constants)] +#![feature(error_generic_member_access)] +#![feature(error_in_core)] +#![feature(error_iter)] +#![feature(exact_size_is_empty)] +#![feature(exclusive_wrapper)] +#![feature(extend_one)] +#![feature(float_minimum_maximum)] +#![feature(float_next_up_down)] +#![feature(hasher_prefixfree_extras)] +#![feature(hashmap_internals)] +#![feature(int_error_internals)] +#![feature(is_some_and)] +#![feature(maybe_uninit_slice)] +#![feature(maybe_uninit_write_slice)] +#![feature(nonnull_slice_from_raw_parts)] +#![feature(panic_can_unwind)] +#![feature(panic_info_message)] +#![feature(panic_internals)] +#![feature(pointer_byte_offsets)] +#![feature(pointer_is_aligned)] +#![feature(portable_simd)] +#![feature(prelude_2024)] +#![feature(provide_any)] +#![feature(ptr_as_uninit)] +#![feature(raw_os_nonzero)] +#![feature(slice_internals)] +#![feature(slice_ptr_get)] #![feature(std_internals)] -#![feature(stdsimd)] -#![feature(stmt_expr_attributes)] #![feature(str_internals)] +#![feature(strict_provenance)] +#![feature(maybe_uninit_uninit_array)] +#![feature(const_maybe_uninit_uninit_array)] +#![feature(const_waker)] +// +// Library features (alloc): +#![feature(alloc_layout_extra)] +#![feature(allocator_api)] +#![feature(get_mut_unchecked)] +#![feature(map_try_insert)] +#![feature(new_uninit)] +#![feature(thin_box)] +#![feature(try_reserve_kind)] +#![feature(vec_into_raw_parts)] +#![feature(slice_concat_trait)] +// +// Library features (unwind): +#![feature(panic_unwind)] +// +// Only for re-exporting: +#![feature(assert_matches)] +#![feature(async_iterator)] +#![feature(c_variadic)] +#![feature(cfg_accessible)] +#![feature(cfg_eval)] +#![feature(concat_bytes)] +#![feature(const_format_args)] +#![feature(core_panic)] +#![feature(custom_test_frameworks)] +#![feature(edition_panic)] +#![feature(format_args_nl)] +#![feature(log_syntax)] +#![feature(once_cell)] +#![feature(saturating_int_impl)] +#![feature(stdsimd)] #![feature(test)] -#![feature(thread_local)] -#![feature(toowned_clone_into)] #![feature(trace_macros)] -#![feature(track_caller)] -#![feature(try_reserve)] -#![feature(unboxed_closures)] -#![feature(untagged_unions)] -#![feature(unwind_attributes)] -#![feature(vec_drain_as_slice)] -#![feature(vec_into_raw_parts)] -// NB: the above list is sorted to minimize merge conflicts. +#![feature(get_many_mut)] +// +// Only used in tests/benchmarks: +// +// Only for const-ness: +#![feature(const_collections_with_hasher)] +#![feature(const_hash)] +#![feature(const_io_structs)] +#![feature(const_ip)] +#![feature(const_ipv4)] +#![feature(const_ipv6)] +#![feature(const_socketaddr)] +#![feature(thread_local_internals)] +// #![default_lib_allocator] -#![feature(crucible_intrinsics)] // Explicitly import the prelude. The compiler uses this same unstable attribute // to import the prelude implicitly when building crates that depend on std. #[prelude_import] #[allow(unused)] -use prelude::v1::*; +use prelude::rust_2021::*; // Access to Bencher, etc. #[cfg(test)] @@ -337,6 +386,11 @@ extern crate libc; #[allow(unused_extern_crates)] extern crate unwind; +#[doc(masked)] +#[allow(unused_extern_crates)] +#[cfg(feature = "miniz_oxide")] +extern crate miniz_oxide; + // During testing, this crate is not actually the "real" std library, but rather // it links to the real std library, which was compiled from this same source // code. So any lang items std defines are conditionally excluded (or else they @@ -350,6 +404,11 @@ extern crate std as realstd; #[macro_use] mod macros; +// The runtime entry point and a few unstable public functions used by the +// compiler +#[macro_use] +pub mod rt; + // The Rust prelude pub mod prelude; @@ -374,11 +433,10 @@ pub use alloc_crate::string; pub use alloc_crate::vec; #[stable(feature = "rust1", since = "1.0.0")] pub use core::any; -#[stable(feature = "simd_arch", since = "1.27.0")] -#[doc(no_inline)] -pub use core::arch; #[stable(feature = "core_array", since = "1.36.0")] pub use core::array; +#[unstable(feature = "async_iterator", issue = "79024")] +pub use core::async_iter; #[stable(feature = "rust1", since = "1.0.0")] pub use core::cell; #[stable(feature = "rust1", since = "1.0.0")] @@ -391,23 +449,31 @@ pub use core::cmp; pub use core::convert; #[stable(feature = "rust1", since = "1.0.0")] pub use core::default; +#[stable(feature = "futures_api", since = "1.36.0")] +pub use core::future; #[stable(feature = "rust1", since = "1.0.0")] pub use core::hash; #[stable(feature = "core_hint", since = "1.27.0")] pub use core::hint; #[stable(feature = "i128", since = "1.26.0")] +#[allow(deprecated, deprecated_in_future)] pub use core::i128; #[stable(feature = "rust1", since = "1.0.0")] +#[allow(deprecated, deprecated_in_future)] pub use core::i16; #[stable(feature = "rust1", since = "1.0.0")] +#[allow(deprecated, deprecated_in_future)] pub use core::i32; #[stable(feature = "rust1", since = "1.0.0")] +#[allow(deprecated, deprecated_in_future)] pub use core::i64; #[stable(feature = "rust1", since = "1.0.0")] +#[allow(deprecated, deprecated_in_future)] pub use core::i8; #[stable(feature = "rust1", since = "1.0.0")] pub use core::intrinsics; #[stable(feature = "rust1", since = "1.0.0")] +#[allow(deprecated, deprecated_in_future)] pub use core::isize; #[stable(feature = "rust1", since = "1.0.0")] pub use core::iter; @@ -424,20 +490,24 @@ pub use core::pin; #[stable(feature = "rust1", since = "1.0.0")] pub use core::ptr; #[stable(feature = "rust1", since = "1.0.0")] -pub use core::raw; -#[stable(feature = "rust1", since = "1.0.0")] pub use core::result; #[stable(feature = "i128", since = "1.26.0")] +#[allow(deprecated, deprecated_in_future)] pub use core::u128; #[stable(feature = "rust1", since = "1.0.0")] +#[allow(deprecated, deprecated_in_future)] pub use core::u16; #[stable(feature = "rust1", since = "1.0.0")] +#[allow(deprecated, deprecated_in_future)] pub use core::u32; #[stable(feature = "rust1", since = "1.0.0")] +#[allow(deprecated, deprecated_in_future)] pub use core::u64; #[stable(feature = "rust1", since = "1.0.0")] +#[allow(deprecated, deprecated_in_future)] pub use core::u8; #[stable(feature = "rust1", since = "1.0.0")] +#[allow(deprecated, deprecated_in_future)] pub use core::usize; pub mod f32; @@ -462,97 +532,100 @@ pub mod process; pub mod sync; pub mod time; +// Pull in `std_float` crate into std. The contents of +// `std_float` are in a different repository: rust-lang/portable-simd. +#[path = "../../portable-simd/crates/std_float/src/lib.rs"] +#[allow(missing_debug_implementations, dead_code, unsafe_op_in_unsafe_fn, unused_unsafe)] +#[allow(rustdoc::bare_urls)] +#[unstable(feature = "portable_simd", issue = "86656")] +mod std_float; + +#[doc = include_str!("../../portable-simd/crates/core_simd/src/core_simd_docs.md")] +#[unstable(feature = "portable_simd", issue = "86656")] +pub mod simd { + #[doc(inline)] + pub use crate::std_float::StdFloat; + #[doc(inline)] + pub use core::simd::*; +} + #[stable(feature = "futures_api", since = "1.36.0")] pub mod task { //! Types and Traits for working with asynchronous tasks. + #[doc(inline)] #[stable(feature = "futures_api", since = "1.36.0")] pub use core::task::*; + + #[doc(inline)] + #[stable(feature = "wake_trait", since = "1.51.0")] + pub use alloc::task::*; } -#[stable(feature = "futures_api", since = "1.36.0")] -pub mod future; +#[doc = include_str!("../../stdarch/crates/core_arch/src/core_arch_docs.md")] +#[stable(feature = "simd_arch", since = "1.27.0")] +pub mod arch { + #[stable(feature = "simd_arch", since = "1.27.0")] + // The `no_inline`-attribute is required to make the documentation of all + // targets available. + // See https://github.com/rust-lang/rust/pull/57808#issuecomment-457390549 for + // more information. + #[doc(no_inline)] // Note (#82861): required for correct documentation + pub use core::arch::*; + + #[stable(feature = "simd_aarch64", since = "1.60.0")] + pub use std_detect::is_aarch64_feature_detected; + #[stable(feature = "simd_x86", since = "1.27.0")] + pub use std_detect::is_x86_feature_detected; + #[unstable(feature = "stdsimd", issue = "48556")] + pub use std_detect::{ + is_arm_feature_detected, is_mips64_feature_detected, is_mips_feature_detected, + is_powerpc64_feature_detected, is_powerpc_feature_detected, is_riscv_feature_detected, + }; +} + +// This was stabilized in the crate root so we have to keep it there. +#[stable(feature = "simd_x86", since = "1.27.0")] +pub use std_detect::is_x86_feature_detected; // Platform-abstraction modules -#[macro_use] -mod sys_common; mod sys; +mod sys_common; pub mod alloc; // Private support modules -mod memchr; mod panicking; +mod personality; -// The runtime entry point and a few unstable public functions used by the -// compiler -pub mod rt; +#[path = "../../backtrace/src/lib.rs"] +#[allow(dead_code, unused_attributes, fuzzy_provenance_casts)] +mod backtrace_rs; -// Pull in the `std_detect` crate directly into libstd. The contents of -// `std_detect` are in a different repository: rust-lang/stdarch. -// -// `std_detect` depends on libstd, but the contents of this module are -// set up in such a way that directly pulling it here works such that the -// crate uses the this crate as its libstd. -#[path = "../stdarch/crates/std_detect/src/mod.rs"] -#[allow(missing_debug_implementations, missing_docs, dead_code)] -#[unstable(feature = "stdsimd", issue = "48556")] -#[cfg(not(test))] -mod std_detect; - -#[doc(hidden)] -#[unstable(feature = "stdsimd", issue = "48556")] -#[cfg(not(test))] -pub use std_detect::detect; - -// Re-export macros defined in libcore. +// Re-export macros defined in core. #[stable(feature = "rust1", since = "1.0.0")] #[allow(deprecated, deprecated_in_future)] pub use core::{ - // Stable - assert_eq, - assert_ne, - debug_assert, - debug_assert_eq, - debug_assert_ne, - // Unstable - matches, - r#try, - todo, - unimplemented, - unreachable, - write, - writeln, + assert_eq, assert_ne, debug_assert, debug_assert_eq, debug_assert_ne, matches, todo, r#try, + unimplemented, unreachable, write, writeln, }; -// Re-export built-in macros defined through libcore. +// Re-export built-in macros defined through core. #[stable(feature = "builtin_macro_prelude", since = "1.38.0")] +#[allow(deprecated)] pub use core::{ - // Unstable - asm, - // Stable - assert, - cfg, - column, - compile_error, - concat, - concat_idents, - env, - file, - format_args, - format_args_nl, - global_asm, - include, - include_bytes, - include_str, - line, - log_syntax, - module_path, - option_env, - stringify, - trace_macros, + assert, assert_matches, cfg, column, compile_error, concat, concat_idents, const_format_args, + env, file, format_args, format_args_nl, include, include_bytes, include_str, line, log_syntax, + module_path, option_env, stringify, trace_macros, }; +#[unstable( + feature = "concat_bytes", + issue = "87555", + reason = "`concat_bytes` is not stable enough for use and is subject to change" +)] +pub use core::concat_bytes; + #[stable(feature = "core_primitive", since = "1.43.0")] pub use core::primitive; @@ -565,3 +638,44 @@ include!("primitive_docs.rs"); // the rustdoc documentation for the existing keywords. Using `include!` // because rustdoc only looks for these modules at the crate level. include!("keyword_docs.rs"); + +// This is required to avoid an unstable error when `restricted-std` is not +// enabled. The use of #![feature(restricted_std)] in rustc-std-workspace-std +// is unconditional, so the unstable feature needs to be defined somewhere. +#[unstable(feature = "restricted_std", issue = "none")] +mod __restricted_std_workaround {} + +mod sealed { + /// This trait being unreachable from outside the crate + /// prevents outside implementations of our extension traits. + /// This allows adding more trait methods in the future. + #[unstable(feature = "sealed", issue = "none")] + pub trait Sealed {} +} + +#[cfg(test)] +#[allow(dead_code)] // Not used in all configurations. +pub(crate) mod test_helpers { + /// Test-only replacement for `rand::thread_rng()`, which is unusable for + /// us, as we want to allow running stdlib tests on tier-3 targets which may + /// not have `getrandom` support. + /// + /// Does a bit of a song and dance to ensure that the seed is different on + /// each call (as some tests sadly rely on this), but doesn't try that hard. + /// + /// This is duplicated in the `core`, `alloc` test suites (as well as + /// `std`'s integration tests), but figuring out a mechanism to share these + /// seems far more painful than copy-pasting a 7 line function a couple + /// times, given that even under a perma-unstable feature, I don't think we + /// want to expose types from `rand` from `std`. + #[track_caller] + pub(crate) fn test_rng() -> rand_xorshift::XorShiftRng { + use core::hash::{BuildHasher, Hash, Hasher}; + let mut hasher = crate::collections::hash_map::RandomState::new().build_hasher(); + core::panic::Location::caller().hash(&mut hasher); + let hc64 = hasher.finish(); + let seed_vec = hc64.to_le_bytes().into_iter().chain(0u8..8).collect::>(); + let seed: [u8; 16] = seed_vec.as_slice().try_into().unwrap(); + rand::SeedableRng::from_seed(seed) + } +} diff --git a/crux-mir/lib/std/src/macros.rs b/crux-mir/lib/std/src/macros.rs index 7fd7de56f..fcc5cfafd 100644 --- a/crux-mir/lib/std/src/macros.rs +++ b/crux-mir/lib/std/src/macros.rs @@ -1,20 +1,22 @@ //! Standard library macros //! -//! This modules contains a set of macros which are exported from the standard +//! This module contains a set of macros which are exported from the standard //! library. Each macro is available for use when linking against the standard //! library. +// ignore-tidy-dbg -#[doc(include = "../libcore/macros/panic.md")] +#[doc = include_str!("../../core/src/macros/panic.md")] #[macro_export] +#[rustc_builtin_macro(std_panic)] #[stable(feature = "rust1", since = "1.0.0")] -#[allow_internal_unstable(libstd_sys_internals)] +#[allow_internal_unstable(edition_panic)] +#[cfg_attr(not(test), rustc_diagnostic_item = "std_panic_macro")] macro_rules! panic { - () => ({ $crate::panic!("explicit panic") }); - ($msg:expr) => ({ $crate::rt::begin_panic($msg) }); - ($msg:expr,) => ({ $crate::panic!($msg) }); - ($fmt:expr, $($arg:tt)+) => ({ - $crate::rt::begin_panic_fmt(&$crate::format_args!($fmt, $($arg)+)) - }); + // Expands to either `$crate::panic::panic_2015` or `$crate::panic::panic_2021` + // depending on the edition of the caller. + ($($arg:tt)*) => { + /* compiler built-in */ + }; } /// Prints to the standard output. @@ -26,17 +28,31 @@ macro_rules! panic { /// necessary to use [`io::stdout().flush()`][flush] to ensure the output is emitted /// immediately. /// +/// The `print!` macro will lock the standard output on each call. If you call +/// `print!` within a hot loop, this behavior may be the bottleneck of the loop. +/// To avoid this, lock stdout with [`io::stdout().lock()`][lock]: +/// ``` +/// use std::io::{stdout, Write}; +/// +/// let mut lock = stdout().lock(); +/// write!(lock, "hello world").unwrap(); +/// ``` +/// /// Use `print!` only for the primary output of your program. Use /// [`eprint!`] instead to print error and progress messages. /// -/// [`println!`]: ../std/macro.println.html -/// [flush]: ../std/io/trait.Write.html#tymethod.flush -/// [`eprint!`]: ../std/macro.eprint.html +/// [flush]: crate::io::Write::flush +/// [`println!`]: crate::println +/// [`eprint!`]: crate::eprint +/// [lock]: crate::io::Stdout /// /// # Panics /// /// Panics if writing to `io::stdout()` fails. /// +/// Writing to non-blocking stdout can cause an error, which will lead +/// this macro to panic. +/// /// # Examples /// /// ``` @@ -58,9 +74,12 @@ macro_rules! panic { /// ``` #[macro_export] #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "print_macro")] #[allow_internal_unstable(print_internals)] macro_rules! print { - ($($arg:tt)*) => ($crate::io::_print($crate::format_args!($($arg)*))); + ($($arg:tt)*) => {{ + $crate::io::_print($crate::format_args!($($arg)*)); + }}; } /// Prints to the standard output, with a newline. @@ -68,18 +87,34 @@ macro_rules! print { /// On all platforms, the newline is the LINE FEED character (`\n`/`U+000A`) alone /// (no additional CARRIAGE RETURN (`\r`/`U+000D`)). /// -/// Use the [`format!`] syntax to write data to the standard output. +/// This macro uses the same syntax as [`format!`], but writes to the standard output instead. /// See [`std::fmt`] for more information. /// +/// The `println!` macro will lock the standard output on each call. If you call +/// `println!` within a hot loop, this behavior may be the bottleneck of the loop. +/// To avoid this, lock stdout with [`io::stdout().lock()`][lock]: +/// ``` +/// use std::io::{stdout, Write}; +/// +/// let mut lock = stdout().lock(); +/// writeln!(lock, "hello world").unwrap(); +/// ``` +/// /// Use `println!` only for the primary output of your program. Use /// [`eprintln!`] instead to print error and progress messages. /// -/// [`format!`]: ../std/macro.format.html -/// [`std::fmt`]: ../std/fmt/index.html -/// [`eprintln!`]: ../std/macro.eprintln.html +/// [`std::fmt`]: crate::fmt +/// [`eprintln!`]: crate::eprintln +/// [lock]: crate::io::Stdout +/// /// # Panics /// -/// Panics if writing to `io::stdout` fails. +/// Panics if writing to [`io::stdout`] fails. +/// +/// Writing to non-blocking stdout can cause an error, which will lead +/// this macro to panic. +/// +/// [`io::stdout`]: crate::io::stdout /// /// # Examples /// @@ -87,33 +122,41 @@ macro_rules! print { /// println!(); // prints just a newline /// println!("hello there!"); /// println!("format {} arguments", "some"); +/// let local_variable = "some"; +/// println!("format {local_variable} arguments"); /// ``` #[macro_export] #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "println_macro")] #[allow_internal_unstable(print_internals, format_args_nl)] macro_rules! println { - () => ($crate::print!("\n")); - ($($arg:tt)*) => ({ + () => { + $crate::print!("\n") + }; + ($($arg:tt)*) => {{ $crate::io::_print($crate::format_args_nl!($($arg)*)); - }) + }}; } /// Prints to the standard error. /// /// Equivalent to the [`print!`] macro, except that output goes to -/// [`io::stderr`] instead of `io::stdout`. See [`print!`] for +/// [`io::stderr`] instead of [`io::stdout`]. See [`print!`] for /// example usage. /// /// Use `eprint!` only for error and progress messages. Use `print!` /// instead for the primary output of your program. /// -/// [`io::stderr`]: ../std/io/struct.Stderr.html -/// [`print!`]: ../std/macro.print.html +/// [`io::stderr`]: crate::io::stderr +/// [`io::stdout`]: crate::io::stdout /// /// # Panics /// /// Panics if writing to `io::stderr` fails. /// +/// Writing to non-blocking stdout can cause an error, which will lead +/// this macro to panic. +/// /// # Examples /// /// ``` @@ -121,27 +164,34 @@ macro_rules! println { /// ``` #[macro_export] #[stable(feature = "eprint", since = "1.19.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "eprint_macro")] #[allow_internal_unstable(print_internals)] macro_rules! eprint { - ($($arg:tt)*) => ($crate::io::_eprint($crate::format_args!($($arg)*))); + ($($arg:tt)*) => {{ + $crate::io::_eprint($crate::format_args!($($arg)*)); + }}; } /// Prints to the standard error, with a newline. /// /// Equivalent to the [`println!`] macro, except that output goes to -/// [`io::stderr`] instead of `io::stdout`. See [`println!`] for +/// [`io::stderr`] instead of [`io::stdout`]. See [`println!`] for /// example usage. /// /// Use `eprintln!` only for error and progress messages. Use `println!` /// instead for the primary output of your program. /// -/// [`io::stderr`]: ../std/io/struct.Stderr.html -/// [`println!`]: ../std/macro.println.html +/// [`io::stderr`]: crate::io::stderr +/// [`io::stdout`]: crate::io::stdout +/// [`println!`]: crate::println /// /// # Panics /// /// Panics if writing to `io::stderr` fails. /// +/// Writing to non-blocking stdout can cause an error, which will lead +/// this macro to panic. +/// /// # Examples /// /// ``` @@ -149,12 +199,15 @@ macro_rules! eprint { /// ``` #[macro_export] #[stable(feature = "eprint", since = "1.19.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "eprintln_macro")] #[allow_internal_unstable(print_internals, format_args_nl)] macro_rules! eprintln { - () => ($crate::eprint!("\n")); - ($($arg:tt)*) => ({ + () => { + $crate::eprint!("\n") + }; + ($($arg:tt)*) => {{ $crate::io::_eprint($crate::format_args_nl!($($arg)*)); - }) + }}; } /// Prints and returns the value of a given expression for quick and dirty @@ -185,9 +238,10 @@ macro_rules! eprintln { /// builds or when debugging in release mode is significantly faster. /// /// Note that the macro is intended as a debugging tool and therefore you -/// should avoid having uses of it in version control for long periods. -/// Use cases involving debug output that should be added to version control -/// are better served by macros such as [`debug!`] from the [`log`] crate. +/// should avoid having uses of it in version control for long periods +/// (other than in tests and similar). +/// Debug output from production code is better done with other facilities +/// such as the [`debug!`] macro from the [`log`] crate. /// /// # Stability /// @@ -281,12 +335,17 @@ macro_rules! eprintln { /// [`debug!`]: https://docs.rs/log/*/log/macro.debug.html /// [`log`]: https://crates.io/crates/log #[macro_export] +#[cfg_attr(not(test), rustc_diagnostic_item = "dbg_macro")] #[stable(feature = "dbg_macro", since = "1.32.0")] macro_rules! dbg { + // NOTE: We cannot use `concat!` to make a static string as a format argument + // of `eprintln!` because `file!` could contain a `{` or + // `$val` expression could be a block (`{ .. }`), in which case the `eprintln!` + // will be malformed. () => { - $crate::eprintln!("[{}:{}]", $crate::file!(), $crate::line!()); + $crate::eprintln!("[{}:{}]", $crate::file!(), $crate::line!()) }; - ($val:expr) => { + ($val:expr $(,)?) => { // Use of `match` here is intentional because it affects the lifetimes // of temporaries - https://stackoverflow.com/a/48732525/1063961 match $val { @@ -297,8 +356,6 @@ macro_rules! dbg { } } }; - // Trailing comma with single argument is ignored - ($val:expr,) => { $crate::dbg!($val) }; ($($val:expr),+ $(,)?) => { ($($crate::dbg!($val)),+,) }; diff --git a/crux-mir/lib/std/src/memchr.rs b/crux-mir/lib/std/src/memchr.rs deleted file mode 100644 index d69294b2d..000000000 --- a/crux-mir/lib/std/src/memchr.rs +++ /dev/null @@ -1,133 +0,0 @@ -// Original implementation taken from rust-memchr. -// Copyright 2015 Andrew Gallant, bluss and Nicolas Koch - -/// A safe interface to `memchr`. -/// -/// Returns the index corresponding to the first occurrence of `needle` in -/// `haystack`, or `None` if one is not found. -/// -/// memchr reduces to super-optimized machine code at around an order of -/// magnitude faster than `haystack.iter().position(|&b| b == needle)`. -/// (See benchmarks.) -/// -/// # Examples -/// -/// This shows how to find the first position of a byte in a byte string. -/// -/// ```ignore (cannot-doctest-private-modules) -/// use memchr::memchr; -/// -/// let haystack = b"the quick brown fox"; -/// assert_eq!(memchr(b'k', haystack), Some(8)); -/// ``` -#[inline] -pub fn memchr(needle: u8, haystack: &[u8]) -> Option { - crate::sys::memchr::memchr(needle, haystack) -} - -/// A safe interface to `memrchr`. -/// -/// Returns the index corresponding to the last occurrence of `needle` in -/// `haystack`, or `None` if one is not found. -/// -/// # Examples -/// -/// This shows how to find the last position of a byte in a byte string. -/// -/// ```ignore (cannot-doctest-private-modules) -/// use memchr::memrchr; -/// -/// let haystack = b"the quick brown fox"; -/// assert_eq!(memrchr(b'o', haystack), Some(17)); -/// ``` -#[inline] -pub fn memrchr(needle: u8, haystack: &[u8]) -> Option { - crate::sys::memchr::memrchr(needle, haystack) -} - -#[cfg(test)] -mod tests { - // test the implementations for the current platform - use super::{memchr, memrchr}; - - #[test] - fn matches_one() { - assert_eq!(Some(0), memchr(b'a', b"a")); - } - - #[test] - fn matches_begin() { - assert_eq!(Some(0), memchr(b'a', b"aaaa")); - } - - #[test] - fn matches_end() { - assert_eq!(Some(4), memchr(b'z', b"aaaaz")); - } - - #[test] - fn matches_nul() { - assert_eq!(Some(4), memchr(b'\x00', b"aaaa\x00")); - } - - #[test] - fn matches_past_nul() { - assert_eq!(Some(5), memchr(b'z', b"aaaa\x00z")); - } - - #[test] - fn no_match_empty() { - assert_eq!(None, memchr(b'a', b"")); - } - - #[test] - fn no_match() { - assert_eq!(None, memchr(b'a', b"xyz")); - } - - #[test] - fn matches_one_reversed() { - assert_eq!(Some(0), memrchr(b'a', b"a")); - } - - #[test] - fn matches_begin_reversed() { - assert_eq!(Some(3), memrchr(b'a', b"aaaa")); - } - - #[test] - fn matches_end_reversed() { - assert_eq!(Some(0), memrchr(b'z', b"zaaaa")); - } - - #[test] - fn matches_nul_reversed() { - assert_eq!(Some(4), memrchr(b'\x00', b"aaaa\x00")); - } - - #[test] - fn matches_past_nul_reversed() { - assert_eq!(Some(0), memrchr(b'z', b"z\x00aaaa")); - } - - #[test] - fn no_match_empty_reversed() { - assert_eq!(None, memrchr(b'a', b"")); - } - - #[test] - fn no_match_reversed() { - assert_eq!(None, memrchr(b'a', b"xyz")); - } - - #[test] - fn each_alignment() { - let mut data = [1u8; 64]; - let needle = 2; - let pos = 40; - data[pos] = needle; - for start in 0..16 { - assert_eq!(Some(pos - start), memchr(needle, &data[start..])); - } - } -} diff --git a/crux-mir/lib/std/src/net/display_buffer.rs b/crux-mir/lib/std/src/net/display_buffer.rs new file mode 100644 index 000000000..7aadf06e9 --- /dev/null +++ b/crux-mir/lib/std/src/net/display_buffer.rs @@ -0,0 +1,40 @@ +use crate::fmt; +use crate::mem::MaybeUninit; +use crate::str; + +/// Used for slow path in `Display` implementations when alignment is required. +pub struct DisplayBuffer { + buf: [MaybeUninit; SIZE], + len: usize, +} + +impl DisplayBuffer { + #[inline] + pub const fn new() -> Self { + Self { buf: MaybeUninit::uninit_array(), len: 0 } + } + + #[inline] + pub fn as_str(&self) -> &str { + // SAFETY: `buf` is only written to by the `fmt::Write::write_str` implementation + // which writes a valid UTF-8 string to `buf` and correctly sets `len`. + unsafe { + let s = MaybeUninit::slice_assume_init_ref(&self.buf[..self.len]); + str::from_utf8_unchecked(s) + } + } +} + +impl fmt::Write for DisplayBuffer { + fn write_str(&mut self, s: &str) -> fmt::Result { + let bytes = s.as_bytes(); + + if let Some(buf) = self.buf.get_mut(self.len..(self.len + bytes.len())) { + MaybeUninit::write_slice(buf, bytes); + self.len += bytes.len(); + Ok(()) + } else { + Err(fmt::Error) + } + } +} diff --git a/crux-mir/lib/std/src/net/ip.rs b/crux-mir/lib/std/src/net/ip.rs deleted file mode 100644 index edc28033c..000000000 --- a/crux-mir/lib/std/src/net/ip.rs +++ /dev/null @@ -1,2726 +0,0 @@ -#![unstable( - feature = "ip", - reason = "extra functionality has not been \ - scrutinized to the level that it should \ - be to be stable", - issue = "27709" -)] - -use crate::cmp::Ordering; -use crate::fmt; -use crate::hash; -use crate::io::Write; -use crate::sys::net::netc as c; -use crate::sys_common::{AsInner, FromInner}; - -/// An IP address, either IPv4 or IPv6. -/// -/// This enum can contain either an [`Ipv4Addr`] or an [`Ipv6Addr`], see their -/// respective documentation for more details. -/// -/// The size of an `IpAddr` instance may vary depending on the target operating -/// system. -/// -/// [`Ipv4Addr`]: ../../std/net/struct.Ipv4Addr.html -/// [`Ipv6Addr`]: ../../std/net/struct.Ipv6Addr.html -/// -/// # Examples -/// -/// ``` -/// use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; -/// -/// let localhost_v4 = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)); -/// let localhost_v6 = IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)); -/// -/// assert_eq!("127.0.0.1".parse(), Ok(localhost_v4)); -/// assert_eq!("::1".parse(), Ok(localhost_v6)); -/// -/// assert_eq!(localhost_v4.is_ipv6(), false); -/// assert_eq!(localhost_v4.is_ipv4(), true); -/// ``` -#[stable(feature = "ip_addr", since = "1.7.0")] -#[derive(Copy, Clone, Eq, PartialEq, Debug, Hash, PartialOrd, Ord)] -pub enum IpAddr { - /// An IPv4 address. - #[stable(feature = "ip_addr", since = "1.7.0")] - V4(#[stable(feature = "ip_addr", since = "1.7.0")] Ipv4Addr), - /// An IPv6 address. - #[stable(feature = "ip_addr", since = "1.7.0")] - V6(#[stable(feature = "ip_addr", since = "1.7.0")] Ipv6Addr), -} - -/// An IPv4 address. -/// -/// IPv4 addresses are defined as 32-bit integers in [IETF RFC 791]. -/// They are usually represented as four octets. -/// -/// See [`IpAddr`] for a type encompassing both IPv4 and IPv6 addresses. -/// -/// The size of an `Ipv4Addr` struct may vary depending on the target operating -/// system. -/// -/// [IETF RFC 791]: https://tools.ietf.org/html/rfc791 -/// [`IpAddr`]: ../../std/net/enum.IpAddr.html -/// -/// # Textual representation -/// -/// `Ipv4Addr` provides a [`FromStr`] implementation. The four octets are in decimal -/// notation, divided by `.` (this is called "dot-decimal notation"). -/// -/// [`FromStr`]: ../../std/str/trait.FromStr.html -/// -/// # Examples -/// -/// ``` -/// use std::net::Ipv4Addr; -/// -/// let localhost = Ipv4Addr::new(127, 0, 0, 1); -/// assert_eq!("127.0.0.1".parse(), Ok(localhost)); -/// assert_eq!(localhost.is_loopback(), true); -/// ``` -#[derive(Copy)] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct Ipv4Addr { - inner: c::in_addr, -} - -/// An IPv6 address. -/// -/// IPv6 addresses are defined as 128-bit integers in [IETF RFC 4291]. -/// They are usually represented as eight 16-bit segments. -/// -/// See [`IpAddr`] for a type encompassing both IPv4 and IPv6 addresses. -/// -/// The size of an `Ipv6Addr` struct may vary depending on the target operating -/// system. -/// -/// [IETF RFC 4291]: https://tools.ietf.org/html/rfc4291 -/// [`IpAddr`]: ../../std/net/enum.IpAddr.html -/// -/// # Textual representation -/// -/// `Ipv6Addr` provides a [`FromStr`] implementation. There are many ways to represent -/// an IPv6 address in text, but in general, each segments is written in hexadecimal -/// notation, and segments are separated by `:`. For more information, see -/// [IETF RFC 5952]. -/// -/// [`FromStr`]: ../../std/str/trait.FromStr.html -/// [IETF RFC 5952]: https://tools.ietf.org/html/rfc5952 -/// -/// # Examples -/// -/// ``` -/// use std::net::Ipv6Addr; -/// -/// let localhost = Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1); -/// assert_eq!("::1".parse(), Ok(localhost)); -/// assert_eq!(localhost.is_loopback(), true); -/// ``` -#[derive(Copy)] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct Ipv6Addr { - inner: c::in6_addr, -} - -#[allow(missing_docs)] -#[derive(Copy, PartialEq, Eq, Clone, Hash, Debug)] -pub enum Ipv6MulticastScope { - InterfaceLocal, - LinkLocal, - RealmLocal, - AdminLocal, - SiteLocal, - OrganizationLocal, - Global, -} - -impl IpAddr { - /// Returns [`true`] for the special 'unspecified' address. - /// - /// See the documentation for [`Ipv4Addr::is_unspecified`][IPv4] and - /// [`Ipv6Addr::is_unspecified`][IPv6] for more details. - /// - /// [IPv4]: ../../std/net/struct.Ipv4Addr.html#method.is_unspecified - /// [IPv6]: ../../std/net/struct.Ipv6Addr.html#method.is_unspecified - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; - /// - /// assert_eq!(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)).is_unspecified(), true); - /// assert_eq!(IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0)).is_unspecified(), true); - /// ``` - #[stable(feature = "ip_shared", since = "1.12.0")] - pub fn is_unspecified(&self) -> bool { - match self { - IpAddr::V4(ip) => ip.is_unspecified(), - IpAddr::V6(ip) => ip.is_unspecified(), - } - } - - /// Returns [`true`] if this is a loopback address. - /// - /// See the documentation for [`Ipv4Addr::is_loopback`][IPv4] and - /// [`Ipv6Addr::is_loopback`][IPv6] for more details. - /// - /// [IPv4]: ../../std/net/struct.Ipv4Addr.html#method.is_loopback - /// [IPv6]: ../../std/net/struct.Ipv6Addr.html#method.is_loopback - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; - /// - /// assert_eq!(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)).is_loopback(), true); - /// assert_eq!(IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0x1)).is_loopback(), true); - /// ``` - #[stable(feature = "ip_shared", since = "1.12.0")] - pub fn is_loopback(&self) -> bool { - match self { - IpAddr::V4(ip) => ip.is_loopback(), - IpAddr::V6(ip) => ip.is_loopback(), - } - } - - /// Returns [`true`] if the address appears to be globally routable. - /// - /// See the documentation for [`Ipv4Addr::is_global`][IPv4] and - /// [`Ipv6Addr::is_global`][IPv6] for more details. - /// - /// [IPv4]: ../../std/net/struct.Ipv4Addr.html#method.is_global - /// [IPv6]: ../../std/net/struct.Ipv6Addr.html#method.is_global - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// #![feature(ip)] - /// - /// use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; - /// - /// assert_eq!(IpAddr::V4(Ipv4Addr::new(80, 9, 12, 3)).is_global(), true); - /// assert_eq!(IpAddr::V6(Ipv6Addr::new(0, 0, 0x1c9, 0, 0, 0xafc8, 0, 0x1)).is_global(), true); - /// ``` - pub fn is_global(&self) -> bool { - match self { - IpAddr::V4(ip) => ip.is_global(), - IpAddr::V6(ip) => ip.is_global(), - } - } - - /// Returns [`true`] if this is a multicast address. - /// - /// See the documentation for [`Ipv4Addr::is_multicast`][IPv4] and - /// [`Ipv6Addr::is_multicast`][IPv6] for more details. - /// - /// [IPv4]: ../../std/net/struct.Ipv4Addr.html#method.is_multicast - /// [IPv6]: ../../std/net/struct.Ipv6Addr.html#method.is_multicast - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; - /// - /// assert_eq!(IpAddr::V4(Ipv4Addr::new(224, 254, 0, 0)).is_multicast(), true); - /// assert_eq!(IpAddr::V6(Ipv6Addr::new(0xff00, 0, 0, 0, 0, 0, 0, 0)).is_multicast(), true); - /// ``` - #[stable(feature = "ip_shared", since = "1.12.0")] - pub fn is_multicast(&self) -> bool { - match self { - IpAddr::V4(ip) => ip.is_multicast(), - IpAddr::V6(ip) => ip.is_multicast(), - } - } - - /// Returns [`true`] if this address is in a range designated for documentation. - /// - /// See the documentation for [`Ipv4Addr::is_documentation`][IPv4] and - /// [`Ipv6Addr::is_documentation`][IPv6] for more details. - /// - /// [IPv4]: ../../std/net/struct.Ipv4Addr.html#method.is_documentation - /// [IPv6]: ../../std/net/struct.Ipv6Addr.html#method.is_documentation - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// #![feature(ip)] - /// - /// use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; - /// - /// assert_eq!(IpAddr::V4(Ipv4Addr::new(203, 0, 113, 6)).is_documentation(), true); - /// assert_eq!( - /// IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 0)).is_documentation(), - /// true - /// ); - /// ``` - pub fn is_documentation(&self) -> bool { - match self { - IpAddr::V4(ip) => ip.is_documentation(), - IpAddr::V6(ip) => ip.is_documentation(), - } - } - - /// Returns [`true`] if this address is an [IPv4 address], and [`false`] otherwise. - /// - /// [`true`]: ../../std/primitive.bool.html - /// [`false`]: ../../std/primitive.bool.html - /// [IPv4 address]: #variant.V4 - /// - /// # Examples - /// - /// ``` - /// use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; - /// - /// assert_eq!(IpAddr::V4(Ipv4Addr::new(203, 0, 113, 6)).is_ipv4(), true); - /// assert_eq!(IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 0)).is_ipv4(), false); - /// ``` - #[stable(feature = "ipaddr_checker", since = "1.16.0")] - pub fn is_ipv4(&self) -> bool { - matches!(self, IpAddr::V4(_)) - } - - /// Returns [`true`] if this address is an [IPv6 address], and [`false`] otherwise. - /// - /// [`true`]: ../../std/primitive.bool.html - /// [`false`]: ../../std/primitive.bool.html - /// [IPv6 address]: #variant.V6 - /// - /// # Examples - /// - /// ``` - /// use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; - /// - /// assert_eq!(IpAddr::V4(Ipv4Addr::new(203, 0, 113, 6)).is_ipv6(), false); - /// assert_eq!(IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 0)).is_ipv6(), true); - /// ``` - #[stable(feature = "ipaddr_checker", since = "1.16.0")] - pub fn is_ipv6(&self) -> bool { - matches!(self, IpAddr::V6(_)) - } -} - -impl Ipv4Addr { - /// Creates a new IPv4 address from four eight-bit octets. - /// - /// The result will represent the IP address `a`.`b`.`c`.`d`. - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv4Addr; - /// - /// let addr = Ipv4Addr::new(127, 0, 0, 1); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_ipv4", since = "1.32.0")] - pub const fn new(a: u8, b: u8, c: u8, d: u8) -> Ipv4Addr { - // FIXME: should just be u32::from_be_bytes([a, b, c, d]), - // once that method is no longer rustc_const_unstable - Ipv4Addr { - inner: c::in_addr { - s_addr: u32::to_be( - ((a as u32) << 24) | ((b as u32) << 16) | ((c as u32) << 8) | (d as u32), - ), - }, - } - } - - /// An IPv4 address with the address pointing to localhost: 127.0.0.1. - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv4Addr; - /// - /// let addr = Ipv4Addr::LOCALHOST; - /// assert_eq!(addr, Ipv4Addr::new(127, 0, 0, 1)); - /// ``` - #[stable(feature = "ip_constructors", since = "1.30.0")] - pub const LOCALHOST: Self = Ipv4Addr::new(127, 0, 0, 1); - - /// An IPv4 address representing an unspecified address: 0.0.0.0 - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv4Addr; - /// - /// let addr = Ipv4Addr::UNSPECIFIED; - /// assert_eq!(addr, Ipv4Addr::new(0, 0, 0, 0)); - /// ``` - #[stable(feature = "ip_constructors", since = "1.30.0")] - pub const UNSPECIFIED: Self = Ipv4Addr::new(0, 0, 0, 0); - - /// An IPv4 address representing the broadcast address: 255.255.255.255 - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv4Addr; - /// - /// let addr = Ipv4Addr::BROADCAST; - /// assert_eq!(addr, Ipv4Addr::new(255, 255, 255, 255)); - /// ``` - #[stable(feature = "ip_constructors", since = "1.30.0")] - pub const BROADCAST: Self = Ipv4Addr::new(255, 255, 255, 255); - - /// Returns the four eight-bit integers that make up this address. - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv4Addr; - /// - /// let addr = Ipv4Addr::new(127, 0, 0, 1); - /// assert_eq!(addr.octets(), [127, 0, 0, 1]); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn octets(&self) -> [u8; 4] { - // This returns the order we want because s_addr is stored in big-endian. - self.inner.s_addr.to_ne_bytes() - } - - /// Returns [`true`] for the special 'unspecified' address (0.0.0.0). - /// - /// This property is defined in _UNIX Network Programming, Second Edition_, - /// W. Richard Stevens, p. 891; see also [ip7]. - /// - /// [ip7]: http://man7.org/linux/man-pages/man7/ip.7.html - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv4Addr; - /// - /// assert_eq!(Ipv4Addr::new(0, 0, 0, 0).is_unspecified(), true); - /// assert_eq!(Ipv4Addr::new(45, 22, 13, 197).is_unspecified(), false); - /// ``` - #[stable(feature = "ip_shared", since = "1.12.0")] - #[rustc_const_stable(feature = "const_ipv4", since = "1.32.0")] - pub const fn is_unspecified(&self) -> bool { - self.inner.s_addr == 0 - } - - /// Returns [`true`] if this is a loopback address (127.0.0.0/8). - /// - /// This property is defined by [IETF RFC 1122]. - /// - /// [IETF RFC 1122]: https://tools.ietf.org/html/rfc1122 - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv4Addr; - /// - /// assert_eq!(Ipv4Addr::new(127, 0, 0, 1).is_loopback(), true); - /// assert_eq!(Ipv4Addr::new(45, 22, 13, 197).is_loopback(), false); - /// ``` - #[stable(since = "1.7.0", feature = "ip_17")] - pub fn is_loopback(&self) -> bool { - self.octets()[0] == 127 - } - - /// Returns [`true`] if this is a private address. - /// - /// The private address ranges are defined in [IETF RFC 1918] and include: - /// - /// - 10.0.0.0/8 - /// - 172.16.0.0/12 - /// - 192.168.0.0/16 - /// - /// [IETF RFC 1918]: https://tools.ietf.org/html/rfc1918 - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv4Addr; - /// - /// assert_eq!(Ipv4Addr::new(10, 0, 0, 1).is_private(), true); - /// assert_eq!(Ipv4Addr::new(10, 10, 10, 10).is_private(), true); - /// assert_eq!(Ipv4Addr::new(172, 16, 10, 10).is_private(), true); - /// assert_eq!(Ipv4Addr::new(172, 29, 45, 14).is_private(), true); - /// assert_eq!(Ipv4Addr::new(172, 32, 0, 2).is_private(), false); - /// assert_eq!(Ipv4Addr::new(192, 168, 0, 2).is_private(), true); - /// assert_eq!(Ipv4Addr::new(192, 169, 0, 2).is_private(), false); - /// ``` - #[stable(since = "1.7.0", feature = "ip_17")] - pub fn is_private(&self) -> bool { - match self.octets() { - [10, ..] => true, - [172, b, ..] if b >= 16 && b <= 31 => true, - [192, 168, ..] => true, - _ => false, - } - } - - /// Returns [`true`] if the address is link-local (169.254.0.0/16). - /// - /// This property is defined by [IETF RFC 3927]. - /// - /// [IETF RFC 3927]: https://tools.ietf.org/html/rfc3927 - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv4Addr; - /// - /// assert_eq!(Ipv4Addr::new(169, 254, 0, 0).is_link_local(), true); - /// assert_eq!(Ipv4Addr::new(169, 254, 10, 65).is_link_local(), true); - /// assert_eq!(Ipv4Addr::new(16, 89, 10, 65).is_link_local(), false); - /// ``` - #[stable(since = "1.7.0", feature = "ip_17")] - pub fn is_link_local(&self) -> bool { - match self.octets() { - [169, 254, ..] => true, - _ => false, - } - } - - /// Returns [`true`] if the address appears to be globally routable. - /// See [iana-ipv4-special-registry][ipv4-sr]. - /// - /// The following return false: - /// - /// - private addresses (see [`is_private()`](#method.is_private)) - /// - the loopback address (see [`is_loopback()`](#method.is_loopback)) - /// - the link-local address (see [`is_link_local()`](#method.is_link_local)) - /// - the broadcast address (see [`is_broadcast()`](#method.is_broadcast)) - /// - addresses used for documentation (see [`is_documentation()`](#method.is_documentation)) - /// - the unspecified address (see [`is_unspecified()`](#method.is_unspecified)), and the whole - /// 0.0.0.0/8 block - /// - addresses reserved for future protocols (see - /// [`is_ietf_protocol_assignment()`](#method.is_ietf_protocol_assignment), except - /// `192.0.0.9/32` and `192.0.0.10/32` which are globally routable - /// - addresses reserved for future use (see [`is_reserved()`](#method.is_reserved) - /// - addresses reserved for networking devices benchmarking (see - /// [`is_benchmarking`](#method.is_benchmarking)) - /// - /// [ipv4-sr]: https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// #![feature(ip)] - /// - /// use std::net::Ipv4Addr; - /// - /// // private addresses are not global - /// assert_eq!(Ipv4Addr::new(10, 254, 0, 0).is_global(), false); - /// assert_eq!(Ipv4Addr::new(192, 168, 10, 65).is_global(), false); - /// assert_eq!(Ipv4Addr::new(172, 16, 10, 65).is_global(), false); - /// - /// // the 0.0.0.0/8 block is not global - /// assert_eq!(Ipv4Addr::new(0, 1, 2, 3).is_global(), false); - /// // in particular, the unspecified address is not global - /// assert_eq!(Ipv4Addr::new(0, 0, 0, 0).is_global(), false); - /// - /// // the loopback address is not global - /// assert_eq!(Ipv4Addr::new(127, 0, 0, 1).is_global(), false); - /// - /// // link local addresses are not global - /// assert_eq!(Ipv4Addr::new(169, 254, 45, 1).is_global(), false); - /// - /// // the broadcast address is not global - /// assert_eq!(Ipv4Addr::new(255, 255, 255, 255).is_global(), false); - /// - /// // the address space designated for documentation is not global - /// assert_eq!(Ipv4Addr::new(192, 0, 2, 255).is_global(), false); - /// assert_eq!(Ipv4Addr::new(198, 51, 100, 65).is_global(), false); - /// assert_eq!(Ipv4Addr::new(203, 0, 113, 6).is_global(), false); - /// - /// // shared addresses are not global - /// assert_eq!(Ipv4Addr::new(100, 100, 0, 0).is_global(), false); - /// - /// // addresses reserved for protocol assignment are not global - /// assert_eq!(Ipv4Addr::new(192, 0, 0, 0).is_global(), false); - /// assert_eq!(Ipv4Addr::new(192, 0, 0, 255).is_global(), false); - /// - /// // addresses reserved for future use are not global - /// assert_eq!(Ipv4Addr::new(250, 10, 20, 30).is_global(), false); - /// - /// // addresses reserved for network devices benchmarking are not global - /// assert_eq!(Ipv4Addr::new(198, 18, 0, 0).is_global(), false); - /// - /// // All the other addresses are global - /// assert_eq!(Ipv4Addr::new(1, 1, 1, 1).is_global(), true); - /// assert_eq!(Ipv4Addr::new(80, 9, 12, 3).is_global(), true); - /// ``` - pub fn is_global(&self) -> bool { - // check if this address is 192.0.0.9 or 192.0.0.10. These addresses are the only two - // globally routable addresses in the 192.0.0.0/24 range. - if u32::from(*self) == 0xc0000009 || u32::from(*self) == 0xc000000a { - return true; - } - !self.is_private() - && !self.is_loopback() - && !self.is_link_local() - && !self.is_broadcast() - && !self.is_documentation() - && !self.is_shared() - && !self.is_ietf_protocol_assignment() - && !self.is_reserved() - && !self.is_benchmarking() - // Make sure the address is not in 0.0.0.0/8 - && self.octets()[0] != 0 - } - - /// Returns [`true`] if this address is part of the Shared Address Space defined in - /// [IETF RFC 6598] (`100.64.0.0/10`). - /// - /// [IETF RFC 6598]: https://tools.ietf.org/html/rfc6598 - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// #![feature(ip)] - /// use std::net::Ipv4Addr; - /// - /// assert_eq!(Ipv4Addr::new(100, 64, 0, 0).is_shared(), true); - /// assert_eq!(Ipv4Addr::new(100, 127, 255, 255).is_shared(), true); - /// assert_eq!(Ipv4Addr::new(100, 128, 0, 0).is_shared(), false); - /// ``` - pub fn is_shared(&self) -> bool { - self.octets()[0] == 100 && (self.octets()[1] & 0b1100_0000 == 0b0100_0000) - } - - /// Returns [`true`] if this address is part of `192.0.0.0/24`, which is reserved to - /// IANA for IETF protocol assignments, as documented in [IETF RFC 6890]. - /// - /// Note that parts of this block are in use: - /// - /// - `192.0.0.8/32` is the "IPv4 dummy address" (see [IETF RFC 7600]) - /// - `192.0.0.9/32` is the "Port Control Protocol Anycast" (see [IETF RFC 7723]) - /// - `192.0.0.10/32` is used for NAT traversal (see [IETF RFC 8155]) - /// - /// [IETF RFC 6890]: https://tools.ietf.org/html/rfc6890 - /// [IETF RFC 7600]: https://tools.ietf.org/html/rfc7600 - /// [IETF RFC 7723]: https://tools.ietf.org/html/rfc7723 - /// [IETF RFC 8155]: https://tools.ietf.org/html/rfc8155 - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// #![feature(ip)] - /// use std::net::Ipv4Addr; - /// - /// assert_eq!(Ipv4Addr::new(192, 0, 0, 0).is_ietf_protocol_assignment(), true); - /// assert_eq!(Ipv4Addr::new(192, 0, 0, 8).is_ietf_protocol_assignment(), true); - /// assert_eq!(Ipv4Addr::new(192, 0, 0, 9).is_ietf_protocol_assignment(), true); - /// assert_eq!(Ipv4Addr::new(192, 0, 0, 255).is_ietf_protocol_assignment(), true); - /// assert_eq!(Ipv4Addr::new(192, 0, 1, 0).is_ietf_protocol_assignment(), false); - /// assert_eq!(Ipv4Addr::new(191, 255, 255, 255).is_ietf_protocol_assignment(), false); - /// ``` - pub fn is_ietf_protocol_assignment(&self) -> bool { - self.octets()[0] == 192 && self.octets()[1] == 0 && self.octets()[2] == 0 - } - - /// Returns [`true`] if this address part of the `198.18.0.0/15` range, which is reserved for - /// network devices benchmarking. This range is defined in [IETF RFC 2544] as `192.18.0.0` - /// through `198.19.255.255` but [errata 423] corrects it to `198.18.0.0/15`. - /// - /// [IETF RFC 2544]: https://tools.ietf.org/html/rfc2544 - /// [errata 423]: https://www.rfc-editor.org/errata/eid423 - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// #![feature(ip)] - /// use std::net::Ipv4Addr; - /// - /// assert_eq!(Ipv4Addr::new(198, 17, 255, 255).is_benchmarking(), false); - /// assert_eq!(Ipv4Addr::new(198, 18, 0, 0).is_benchmarking(), true); - /// assert_eq!(Ipv4Addr::new(198, 19, 255, 255).is_benchmarking(), true); - /// assert_eq!(Ipv4Addr::new(198, 20, 0, 0).is_benchmarking(), false); - /// ``` - pub fn is_benchmarking(&self) -> bool { - self.octets()[0] == 198 && (self.octets()[1] & 0xfe) == 18 - } - - /// Returns [`true`] if this address is reserved by IANA for future use. [IETF RFC 1112] - /// defines the block of reserved addresses as `240.0.0.0/4`. This range normally includes the - /// broadcast address `255.255.255.255`, but this implementation explicitly excludes it, since - /// it is obviously not reserved for future use. - /// - /// [IETF RFC 1112]: https://tools.ietf.org/html/rfc1112 - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Warning - /// - /// As IANA assigns new addresses, this method will be - /// updated. This may result in non-reserved addresses being - /// treated as reserved in code that relies on an outdated version - /// of this method. - /// - /// # Examples - /// - /// ``` - /// #![feature(ip)] - /// use std::net::Ipv4Addr; - /// - /// assert_eq!(Ipv4Addr::new(240, 0, 0, 0).is_reserved(), true); - /// assert_eq!(Ipv4Addr::new(255, 255, 255, 254).is_reserved(), true); - /// - /// assert_eq!(Ipv4Addr::new(239, 255, 255, 255).is_reserved(), false); - /// // The broadcast address is not considered as reserved for future use by this implementation - /// assert_eq!(Ipv4Addr::new(255, 255, 255, 255).is_reserved(), false); - /// ``` - pub fn is_reserved(&self) -> bool { - self.octets()[0] & 240 == 240 && !self.is_broadcast() - } - - /// Returns [`true`] if this is a multicast address (224.0.0.0/4). - /// - /// Multicast addresses have a most significant octet between 224 and 239, - /// and is defined by [IETF RFC 5771]. - /// - /// [IETF RFC 5771]: https://tools.ietf.org/html/rfc5771 - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv4Addr; - /// - /// assert_eq!(Ipv4Addr::new(224, 254, 0, 0).is_multicast(), true); - /// assert_eq!(Ipv4Addr::new(236, 168, 10, 65).is_multicast(), true); - /// assert_eq!(Ipv4Addr::new(172, 16, 10, 65).is_multicast(), false); - /// ``` - #[stable(since = "1.7.0", feature = "ip_17")] - pub fn is_multicast(&self) -> bool { - self.octets()[0] >= 224 && self.octets()[0] <= 239 - } - - /// Returns [`true`] if this is a broadcast address (255.255.255.255). - /// - /// A broadcast address has all octets set to 255 as defined in [IETF RFC 919]. - /// - /// [IETF RFC 919]: https://tools.ietf.org/html/rfc919 - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv4Addr; - /// - /// assert_eq!(Ipv4Addr::new(255, 255, 255, 255).is_broadcast(), true); - /// assert_eq!(Ipv4Addr::new(236, 168, 10, 65).is_broadcast(), false); - /// ``` - #[stable(since = "1.7.0", feature = "ip_17")] - pub fn is_broadcast(&self) -> bool { - self == &Self::BROADCAST - } - - /// Returns [`true`] if this address is in a range designated for documentation. - /// - /// This is defined in [IETF RFC 5737]: - /// - /// - 192.0.2.0/24 (TEST-NET-1) - /// - 198.51.100.0/24 (TEST-NET-2) - /// - 203.0.113.0/24 (TEST-NET-3) - /// - /// [IETF RFC 5737]: https://tools.ietf.org/html/rfc5737 - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv4Addr; - /// - /// assert_eq!(Ipv4Addr::new(192, 0, 2, 255).is_documentation(), true); - /// assert_eq!(Ipv4Addr::new(198, 51, 100, 65).is_documentation(), true); - /// assert_eq!(Ipv4Addr::new(203, 0, 113, 6).is_documentation(), true); - /// assert_eq!(Ipv4Addr::new(193, 34, 17, 19).is_documentation(), false); - /// ``` - #[stable(since = "1.7.0", feature = "ip_17")] - pub fn is_documentation(&self) -> bool { - match self.octets() { - [192, 0, 2, _] => true, - [198, 51, 100, _] => true, - [203, 0, 113, _] => true, - _ => false, - } - } - - /// Converts this address to an IPv4-compatible [IPv6 address]. - /// - /// a.b.c.d becomes ::a.b.c.d - /// - /// [IPv6 address]: ../../std/net/struct.Ipv6Addr.html - /// - /// # Examples - /// - /// ``` - /// use std::net::{Ipv4Addr, Ipv6Addr}; - /// - /// assert_eq!( - /// Ipv4Addr::new(192, 0, 2, 255).to_ipv6_compatible(), - /// Ipv6Addr::new(0, 0, 0, 0, 0, 0, 49152, 767) - /// ); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn to_ipv6_compatible(&self) -> Ipv6Addr { - let octets = self.octets(); - Ipv6Addr::from([ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, octets[0], octets[1], octets[2], octets[3], - ]) - } - - /// Converts this address to an IPv4-mapped [IPv6 address]. - /// - /// a.b.c.d becomes ::ffff:a.b.c.d - /// - /// [IPv6 address]: ../../std/net/struct.Ipv6Addr.html - /// - /// # Examples - /// - /// ``` - /// use std::net::{Ipv4Addr, Ipv6Addr}; - /// - /// assert_eq!(Ipv4Addr::new(192, 0, 2, 255).to_ipv6_mapped(), - /// Ipv6Addr::new(0, 0, 0, 0, 0, 65535, 49152, 767)); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn to_ipv6_mapped(&self) -> Ipv6Addr { - let octets = self.octets(); - Ipv6Addr::from([ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF, octets[0], octets[1], octets[2], octets[3], - ]) - } -} - -#[stable(feature = "ip_addr", since = "1.7.0")] -impl fmt::Display for IpAddr { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - IpAddr::V4(ip) => ip.fmt(fmt), - IpAddr::V6(ip) => ip.fmt(fmt), - } - } -} - -#[stable(feature = "ip_from_ip", since = "1.16.0")] -impl From for IpAddr { - /// Copies this address to a new `IpAddr::V4`. - /// - /// # Examples - /// - /// ``` - /// use std::net::{IpAddr, Ipv4Addr}; - /// - /// let addr = Ipv4Addr::new(127, 0, 0, 1); - /// - /// assert_eq!( - /// IpAddr::V4(addr), - /// IpAddr::from(addr) - /// ) - /// ``` - fn from(ipv4: Ipv4Addr) -> IpAddr { - IpAddr::V4(ipv4) - } -} - -#[stable(feature = "ip_from_ip", since = "1.16.0")] -impl From for IpAddr { - /// Copies this address to a new `IpAddr::V6`. - /// - /// # Examples - /// - /// ``` - /// use std::net::{IpAddr, Ipv6Addr}; - /// - /// let addr = Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff); - /// - /// assert_eq!( - /// IpAddr::V6(addr), - /// IpAddr::from(addr) - /// ); - /// ``` - fn from(ipv6: Ipv6Addr) -> IpAddr { - IpAddr::V6(ipv6) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl fmt::Display for Ipv4Addr { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - const IPV4_BUF_LEN: usize = 15; // Long enough for the longest possible IPv4 address - let mut buf = [0u8; IPV4_BUF_LEN]; - let mut buf_slice = &mut buf[..]; - let octets = self.octets(); - // Note: The call to write should never fail, hence the unwrap - write!(buf_slice, "{}.{}.{}.{}", octets[0], octets[1], octets[2], octets[3]).unwrap(); - let len = IPV4_BUF_LEN - buf_slice.len(); - // This unsafe is OK because we know what is being written to the buffer - let buf = unsafe { crate::str::from_utf8_unchecked(&buf[..len]) }; - fmt.pad(buf) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl fmt::Debug for Ipv4Addr { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(self, fmt) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Clone for Ipv4Addr { - fn clone(&self) -> Ipv4Addr { - *self - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl PartialEq for Ipv4Addr { - fn eq(&self, other: &Ipv4Addr) -> bool { - self.inner.s_addr == other.inner.s_addr - } -} - -#[stable(feature = "ip_cmp", since = "1.16.0")] -impl PartialEq for IpAddr { - fn eq(&self, other: &Ipv4Addr) -> bool { - match self { - IpAddr::V4(v4) => v4 == other, - IpAddr::V6(_) => false, - } - } -} - -#[stable(feature = "ip_cmp", since = "1.16.0")] -impl PartialEq for Ipv4Addr { - fn eq(&self, other: &IpAddr) -> bool { - match other { - IpAddr::V4(v4) => self == v4, - IpAddr::V6(_) => false, - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Eq for Ipv4Addr {} - -#[stable(feature = "rust1", since = "1.0.0")] -impl hash::Hash for Ipv4Addr { - fn hash(&self, s: &mut H) { - // `inner` is #[repr(packed)], so we need to copy `s_addr`. - { self.inner.s_addr }.hash(s) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl PartialOrd for Ipv4Addr { - fn partial_cmp(&self, other: &Ipv4Addr) -> Option { - Some(self.cmp(other)) - } -} - -#[stable(feature = "ip_cmp", since = "1.16.0")] -impl PartialOrd for IpAddr { - fn partial_cmp(&self, other: &Ipv4Addr) -> Option { - match self { - IpAddr::V4(v4) => v4.partial_cmp(other), - IpAddr::V6(_) => Some(Ordering::Greater), - } - } -} - -#[stable(feature = "ip_cmp", since = "1.16.0")] -impl PartialOrd for Ipv4Addr { - fn partial_cmp(&self, other: &IpAddr) -> Option { - match other { - IpAddr::V4(v4) => self.partial_cmp(v4), - IpAddr::V6(_) => Some(Ordering::Less), - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Ord for Ipv4Addr { - fn cmp(&self, other: &Ipv4Addr) -> Ordering { - u32::from_be(self.inner.s_addr).cmp(&u32::from_be(other.inner.s_addr)) - } -} - -impl AsInner for Ipv4Addr { - fn as_inner(&self) -> &c::in_addr { - &self.inner - } -} -impl FromInner for Ipv4Addr { - fn from_inner(addr: c::in_addr) -> Ipv4Addr { - Ipv4Addr { inner: addr } - } -} - -#[stable(feature = "ip_u32", since = "1.1.0")] -impl From for u32 { - /// Converts an `Ipv4Addr` into a host byte order `u32`. - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv4Addr; - /// - /// let addr = Ipv4Addr::new(13, 12, 11, 10); - /// assert_eq!(0x0d0c0b0au32, u32::from(addr)); - /// ``` - fn from(ip: Ipv4Addr) -> u32 { - let ip = ip.octets(); - u32::from_be_bytes(ip) - } -} - -#[stable(feature = "ip_u32", since = "1.1.0")] -impl From for Ipv4Addr { - /// Converts a host byte order `u32` into an `Ipv4Addr`. - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv4Addr; - /// - /// let addr = Ipv4Addr::from(0x0d0c0b0au32); - /// assert_eq!(Ipv4Addr::new(13, 12, 11, 10), addr); - /// ``` - fn from(ip: u32) -> Ipv4Addr { - Ipv4Addr::from(ip.to_be_bytes()) - } -} - -#[stable(feature = "from_slice_v4", since = "1.9.0")] -impl From<[u8; 4]> for Ipv4Addr { - /// Creates an `Ipv4Addr` from a four element byte array. - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv4Addr; - /// - /// let addr = Ipv4Addr::from([13u8, 12u8, 11u8, 10u8]); - /// assert_eq!(Ipv4Addr::new(13, 12, 11, 10), addr); - /// ``` - fn from(octets: [u8; 4]) -> Ipv4Addr { - Ipv4Addr::new(octets[0], octets[1], octets[2], octets[3]) - } -} - -#[stable(feature = "ip_from_slice", since = "1.17.0")] -impl From<[u8; 4]> for IpAddr { - /// Creates an `IpAddr::V4` from a four element byte array. - /// - /// # Examples - /// - /// ``` - /// use std::net::{IpAddr, Ipv4Addr}; - /// - /// let addr = IpAddr::from([13u8, 12u8, 11u8, 10u8]); - /// assert_eq!(IpAddr::V4(Ipv4Addr::new(13, 12, 11, 10)), addr); - /// ``` - fn from(octets: [u8; 4]) -> IpAddr { - IpAddr::V4(Ipv4Addr::from(octets)) - } -} - -impl Ipv6Addr { - /// Creates a new IPv6 address from eight 16-bit segments. - /// - /// The result will represent the IP address `a:b:c:d:e:f:g:h`. - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv6Addr; - /// - /// let addr = Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_ipv6", since = "1.32.0")] - pub const fn new(a: u16, b: u16, c: u16, d: u16, e: u16, f: u16, g: u16, h: u16) -> Ipv6Addr { - Ipv6Addr { - inner: c::in6_addr { - s6_addr: [ - (a >> 8) as u8, - a as u8, - (b >> 8) as u8, - b as u8, - (c >> 8) as u8, - c as u8, - (d >> 8) as u8, - d as u8, - (e >> 8) as u8, - e as u8, - (f >> 8) as u8, - f as u8, - (g >> 8) as u8, - g as u8, - (h >> 8) as u8, - h as u8, - ], - }, - } - } - - /// An IPv6 address representing localhost: `::1`. - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv6Addr; - /// - /// let addr = Ipv6Addr::LOCALHOST; - /// assert_eq!(addr, Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)); - /// ``` - #[stable(feature = "ip_constructors", since = "1.30.0")] - pub const LOCALHOST: Self = Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1); - - /// An IPv6 address representing the unspecified address: `::` - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv6Addr; - /// - /// let addr = Ipv6Addr::UNSPECIFIED; - /// assert_eq!(addr, Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0)); - /// ``` - #[stable(feature = "ip_constructors", since = "1.30.0")] - pub const UNSPECIFIED: Self = Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0); - - /// Returns the eight 16-bit segments that make up this address. - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv6Addr; - /// - /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).segments(), - /// [0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff]); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn segments(&self) -> [u16; 8] { - let arr = &self.inner.s6_addr; - [ - u16::from_be_bytes([arr[0], arr[1]]), - u16::from_be_bytes([arr[2], arr[3]]), - u16::from_be_bytes([arr[4], arr[5]]), - u16::from_be_bytes([arr[6], arr[7]]), - u16::from_be_bytes([arr[8], arr[9]]), - u16::from_be_bytes([arr[10], arr[11]]), - u16::from_be_bytes([arr[12], arr[13]]), - u16::from_be_bytes([arr[14], arr[15]]), - ] - } - - /// Returns [`true`] for the special 'unspecified' address (::). - /// - /// This property is defined in [IETF RFC 4291]. - /// - /// [IETF RFC 4291]: https://tools.ietf.org/html/rfc4291 - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv6Addr; - /// - /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).is_unspecified(), false); - /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0).is_unspecified(), true); - /// ``` - #[stable(since = "1.7.0", feature = "ip_17")] - pub fn is_unspecified(&self) -> bool { - self.segments() == [0, 0, 0, 0, 0, 0, 0, 0] - } - - /// Returns [`true`] if this is a loopback address (::1). - /// - /// This property is defined in [IETF RFC 4291]. - /// - /// [IETF RFC 4291]: https://tools.ietf.org/html/rfc4291 - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv6Addr; - /// - /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).is_loopback(), false); - /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0x1).is_loopback(), true); - /// ``` - #[stable(since = "1.7.0", feature = "ip_17")] - pub fn is_loopback(&self) -> bool { - self.segments() == [0, 0, 0, 0, 0, 0, 0, 1] - } - - /// Returns [`true`] if the address appears to be globally routable. - /// - /// The following return [`false`]: - /// - /// - the loopback address - /// - link-local and unique local unicast addresses - /// - interface-, link-, realm-, admin- and site-local multicast addresses - /// - /// [`true`]: ../../std/primitive.bool.html - /// [`false`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// #![feature(ip)] - /// - /// use std::net::Ipv6Addr; - /// - /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).is_global(), true); - /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0x1).is_global(), false); - /// assert_eq!(Ipv6Addr::new(0, 0, 0x1c9, 0, 0, 0xafc8, 0, 0x1).is_global(), true); - /// ``` - pub fn is_global(&self) -> bool { - match self.multicast_scope() { - Some(Ipv6MulticastScope::Global) => true, - None => self.is_unicast_global(), - _ => false, - } - } - - /// Returns [`true`] if this is a unique local address (`fc00::/7`). - /// - /// This property is defined in [IETF RFC 4193]. - /// - /// [IETF RFC 4193]: https://tools.ietf.org/html/rfc4193 - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// #![feature(ip)] - /// - /// use std::net::Ipv6Addr; - /// - /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).is_unique_local(), false); - /// assert_eq!(Ipv6Addr::new(0xfc02, 0, 0, 0, 0, 0, 0, 0).is_unique_local(), true); - /// ``` - pub fn is_unique_local(&self) -> bool { - (self.segments()[0] & 0xfe00) == 0xfc00 - } - - /// Returns [`true`] if the address is a unicast link-local address (`fe80::/64`). - /// - /// A common mis-conception is to think that "unicast link-local addresses start with - /// `fe80::`", but the [IETF RFC 4291] actually defines a stricter format for these addresses: - /// - /// ```no_rust - /// | 10 | - /// | bits | 54 bits | 64 bits | - /// +----------+-------------------------+----------------------------+ - /// |1111111010| 0 | interface ID | - /// +----------+-------------------------+----------------------------+ - /// ``` - /// - /// This method validates the format defined in the RFC and won't recognize the following - /// addresses such as `fe80:0:0:1::` or `fe81::` as unicast link-local addresses for example. - /// If you need a less strict validation use [`is_unicast_link_local()`] instead. - /// - /// # Examples - /// - /// ``` - /// #![feature(ip)] - /// - /// use std::net::Ipv6Addr; - /// - /// let ip = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 0); - /// assert!(ip.is_unicast_link_local_strict()); - /// - /// let ip = Ipv6Addr::new(0xfe80, 0, 0, 0, 0xffff, 0xffff, 0xffff, 0xffff); - /// assert!(ip.is_unicast_link_local_strict()); - /// - /// let ip = Ipv6Addr::new(0xfe80, 0, 0, 1, 0, 0, 0, 0); - /// assert!(!ip.is_unicast_link_local_strict()); - /// assert!(ip.is_unicast_link_local()); - /// - /// let ip = Ipv6Addr::new(0xfe81, 0, 0, 0, 0, 0, 0, 0); - /// assert!(!ip.is_unicast_link_local_strict()); - /// assert!(ip.is_unicast_link_local()); - /// ``` - /// - /// # See also - /// - /// - [IETF RFC 4291 section 2.5.6] - /// - [RFC 4291 errata 4406] - /// - [`is_unicast_link_local()`] - /// - /// [IETF RFC 4291]: https://tools.ietf.org/html/rfc4291 - /// [IETF RFC 4291 section 2.5.6]: https://tools.ietf.org/html/rfc4291#section-2.5.6 - /// [`true`]: ../../std/primitive.bool.html - /// [RFC 4291 errata 4406]: https://www.rfc-editor.org/errata/eid4406 - /// [`is_unicast_link_local()`]: ../../std/net/struct.Ipv6Addr.html#method.is_unicast_link_local - /// - pub fn is_unicast_link_local_strict(&self) -> bool { - (self.segments()[0] & 0xffff) == 0xfe80 - && (self.segments()[1] & 0xffff) == 0 - && (self.segments()[2] & 0xffff) == 0 - && (self.segments()[3] & 0xffff) == 0 - } - - /// Returns [`true`] if the address is a unicast link-local address (`fe80::/10`). - /// - /// This method returns [`true`] for addresses in the range reserved by [RFC 4291 section 2.4], - /// i.e. addresses with the following format: - /// - /// ```no_rust - /// | 10 | - /// | bits | 54 bits | 64 bits | - /// +----------+-------------------------+----------------------------+ - /// |1111111010| arbitratry value | interface ID | - /// +----------+-------------------------+----------------------------+ - /// ``` - /// - /// As a result, this method consider addresses such as `fe80:0:0:1::` or `fe81::` to be - /// unicast link-local addresses, whereas [`is_unicast_link_local_strict()`] does not. If you - /// need a strict validation fully compliant with the RFC, use - /// [`is_unicast_link_local_strict()`]. - /// - /// # Examples - /// - /// ``` - /// #![feature(ip)] - /// - /// use std::net::Ipv6Addr; - /// - /// let ip = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 0); - /// assert!(ip.is_unicast_link_local()); - /// - /// let ip = Ipv6Addr::new(0xfe80, 0, 0, 0, 0xffff, 0xffff, 0xffff, 0xffff); - /// assert!(ip.is_unicast_link_local()); - /// - /// let ip = Ipv6Addr::new(0xfe80, 0, 0, 1, 0, 0, 0, 0); - /// assert!(ip.is_unicast_link_local()); - /// assert!(!ip.is_unicast_link_local_strict()); - /// - /// let ip = Ipv6Addr::new(0xfe81, 0, 0, 0, 0, 0, 0, 0); - /// assert!(ip.is_unicast_link_local()); - /// assert!(!ip.is_unicast_link_local_strict()); - /// ``` - /// - /// # See also - /// - /// - [IETF RFC 4291 section 2.4] - /// - [RFC 4291 errata 4406] - /// - /// [IETF RFC 4291 section 2.4]: https://tools.ietf.org/html/rfc4291#section-2.4 - /// [`true`]: ../../std/primitive.bool.html - /// [RFC 4291 errata 4406]: https://www.rfc-editor.org/errata/eid4406 - /// [`is_unicast_link_local_strict()`]: ../../std/net/struct.Ipv6Addr.html#method.is_unicast_link_local_strict - /// - pub fn is_unicast_link_local(&self) -> bool { - (self.segments()[0] & 0xffc0) == 0xfe80 - } - - /// Returns [`true`] if this is a deprecated unicast site-local address (fec0::/10). The - /// unicast site-local address format is defined in [RFC 4291 section 2.5.7] as: - /// - /// ```no_rust - /// | 10 | - /// | bits | 54 bits | 64 bits | - /// +----------+-------------------------+----------------------------+ - /// |1111111011| subnet ID | interface ID | - /// +----------+-------------------------+----------------------------+ - /// ``` - /// - /// [`true`]: ../../std/primitive.bool.html - /// [RFC 4291 section 2.5.7]: https://tools.ietf.org/html/rfc4291#section-2.5.7 - /// - /// # Examples - /// - /// ``` - /// #![feature(ip)] - /// - /// use std::net::Ipv6Addr; - /// - /// assert_eq!( - /// Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).is_unicast_site_local(), - /// false - /// ); - /// assert_eq!(Ipv6Addr::new(0xfec2, 0, 0, 0, 0, 0, 0, 0).is_unicast_site_local(), true); - /// ``` - /// - /// # Warning - /// - /// As per [RFC 3879], the whole `FEC0::/10` prefix is - /// deprecated. New software must not support site-local - /// addresses. - /// - /// [RFC 3879]: https://tools.ietf.org/html/rfc3879 - pub fn is_unicast_site_local(&self) -> bool { - (self.segments()[0] & 0xffc0) == 0xfec0 - } - - /// Returns [`true`] if this is an address reserved for documentation - /// (2001:db8::/32). - /// - /// This property is defined in [IETF RFC 3849]. - /// - /// [IETF RFC 3849]: https://tools.ietf.org/html/rfc3849 - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// #![feature(ip)] - /// - /// use std::net::Ipv6Addr; - /// - /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).is_documentation(), false); - /// assert_eq!(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 0).is_documentation(), true); - /// ``` - pub fn is_documentation(&self) -> bool { - (self.segments()[0] == 0x2001) && (self.segments()[1] == 0xdb8) - } - - /// Returns [`true`] if the address is a globally routable unicast address. - /// - /// The following return false: - /// - /// - the loopback address - /// - the link-local addresses - /// - unique local addresses - /// - the unspecified address - /// - the address range reserved for documentation - /// - /// This method returns [`true`] for site-local addresses as per [RFC 4291 section 2.5.7] - /// - /// ```no_rust - /// The special behavior of [the site-local unicast] prefix defined in [RFC3513] must no longer - /// be supported in new implementations (i.e., new implementations must treat this prefix as - /// Global Unicast). - /// ``` - /// - /// [`true`]: ../../std/primitive.bool.html - /// [RFC 4291 section 2.5.7]: https://tools.ietf.org/html/rfc4291#section-2.5.7 - /// - /// # Examples - /// - /// ``` - /// #![feature(ip)] - /// - /// use std::net::Ipv6Addr; - /// - /// assert_eq!(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 0).is_unicast_global(), false); - /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).is_unicast_global(), true); - /// ``` - pub fn is_unicast_global(&self) -> bool { - !self.is_multicast() - && !self.is_loopback() - && !self.is_unicast_link_local() - && !self.is_unique_local() - && !self.is_unspecified() - && !self.is_documentation() - } - - /// Returns the address's multicast scope if the address is multicast. - /// - /// # Examples - /// - /// ``` - /// #![feature(ip)] - /// - /// use std::net::{Ipv6Addr, Ipv6MulticastScope}; - /// - /// assert_eq!( - /// Ipv6Addr::new(0xff0e, 0, 0, 0, 0, 0, 0, 0).multicast_scope(), - /// Some(Ipv6MulticastScope::Global) - /// ); - /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).multicast_scope(), None); - /// ``` - pub fn multicast_scope(&self) -> Option { - if self.is_multicast() { - match self.segments()[0] & 0x000f { - 1 => Some(Ipv6MulticastScope::InterfaceLocal), - 2 => Some(Ipv6MulticastScope::LinkLocal), - 3 => Some(Ipv6MulticastScope::RealmLocal), - 4 => Some(Ipv6MulticastScope::AdminLocal), - 5 => Some(Ipv6MulticastScope::SiteLocal), - 8 => Some(Ipv6MulticastScope::OrganizationLocal), - 14 => Some(Ipv6MulticastScope::Global), - _ => None, - } - } else { - None - } - } - - /// Returns [`true`] if this is a multicast address (ff00::/8). - /// - /// This property is defined by [IETF RFC 4291]. - /// - /// [IETF RFC 4291]: https://tools.ietf.org/html/rfc4291 - /// [`true`]: ../../std/primitive.bool.html - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv6Addr; - /// - /// assert_eq!(Ipv6Addr::new(0xff00, 0, 0, 0, 0, 0, 0, 0).is_multicast(), true); - /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).is_multicast(), false); - /// ``` - #[stable(since = "1.7.0", feature = "ip_17")] - pub fn is_multicast(&self) -> bool { - (self.segments()[0] & 0xff00) == 0xff00 - } - - /// Converts this address to an [IPv4 address]. Returns [`None`] if this address is - /// neither IPv4-compatible or IPv4-mapped. - /// - /// ::a.b.c.d and ::ffff:a.b.c.d become a.b.c.d - /// - /// [IPv4 address]: ../../std/net/struct.Ipv4Addr.html - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// - /// # Examples - /// - /// ``` - /// use std::net::{Ipv4Addr, Ipv6Addr}; - /// - /// assert_eq!(Ipv6Addr::new(0xff00, 0, 0, 0, 0, 0, 0, 0).to_ipv4(), None); - /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).to_ipv4(), - /// Some(Ipv4Addr::new(192, 10, 2, 255))); - /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1).to_ipv4(), - /// Some(Ipv4Addr::new(0, 0, 0, 1))); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn to_ipv4(&self) -> Option { - match self.segments() { - [0, 0, 0, 0, 0, f, g, h] if f == 0 || f == 0xffff => { - Some(Ipv4Addr::new((g >> 8) as u8, g as u8, (h >> 8) as u8, h as u8)) - } - _ => None, - } - } - - /// Returns the sixteen eight-bit integers the IPv6 address consists of. - /// - /// ``` - /// use std::net::Ipv6Addr; - /// - /// assert_eq!(Ipv6Addr::new(0xff00, 0, 0, 0, 0, 0, 0, 0).octets(), - /// [255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); - /// ``` - #[stable(feature = "ipv6_to_octets", since = "1.12.0")] - #[rustc_const_stable(feature = "const_ipv6", since = "1.32.0")] - pub const fn octets(&self) -> [u8; 16] { - self.inner.s6_addr - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl fmt::Display for Ipv6Addr { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - // Note: The calls to write should never fail, hence the unwraps in the function - // Long enough for the longest possible IPv6: 39 - const IPV6_BUF_LEN: usize = 39; - let mut buf = [0u8; IPV6_BUF_LEN]; - let mut buf_slice = &mut buf[..]; - - match self.segments() { - // We need special cases for :: and ::1, otherwise they're formatted - // as ::0.0.0.[01] - [0, 0, 0, 0, 0, 0, 0, 0] => write!(buf_slice, "::").unwrap(), - [0, 0, 0, 0, 0, 0, 0, 1] => write!(buf_slice, "::1").unwrap(), - // Ipv4 Compatible address - [0, 0, 0, 0, 0, 0, g, h] => { - write!( - buf_slice, - "::{}.{}.{}.{}", - (g >> 8) as u8, - g as u8, - (h >> 8) as u8, - h as u8 - ) - .unwrap(); - } - // Ipv4-Mapped address - [0, 0, 0, 0, 0, 0xffff, g, h] => { - write!( - buf_slice, - "::ffff:{}.{}.{}.{}", - (g >> 8) as u8, - g as u8, - (h >> 8) as u8, - h as u8 - ) - .unwrap(); - } - _ => { - fn find_zero_slice(segments: &[u16; 8]) -> (usize, usize) { - let mut longest_span_len = 0; - let mut longest_span_at = 0; - let mut cur_span_len = 0; - let mut cur_span_at = 0; - - for i in 0..8 { - if segments[i] == 0 { - if cur_span_len == 0 { - cur_span_at = i; - } - - cur_span_len += 1; - - if cur_span_len > longest_span_len { - longest_span_len = cur_span_len; - longest_span_at = cur_span_at; - } - } else { - cur_span_len = 0; - cur_span_at = 0; - } - } - - (longest_span_at, longest_span_len) - } - - let (zeros_at, zeros_len) = find_zero_slice(&self.segments()); - - if zeros_len > 1 { - fn fmt_subslice(segments: &[u16], buf: &mut &mut [u8]) { - if !segments.is_empty() { - write!(*buf, "{:x}", segments[0]).unwrap(); - for &seg in &segments[1..] { - write!(*buf, ":{:x}", seg).unwrap(); - } - } - } - - fmt_subslice(&self.segments()[..zeros_at], &mut buf_slice); - write!(buf_slice, "::").unwrap(); - fmt_subslice(&self.segments()[zeros_at + zeros_len..], &mut buf_slice); - } else { - let &[a, b, c, d, e, f, g, h] = &self.segments(); - write!( - buf_slice, - "{:x}:{:x}:{:x}:{:x}:{:x}:{:x}:{:x}:{:x}", - a, b, c, d, e, f, g, h - ) - .unwrap(); - } - } - } - let len = IPV6_BUF_LEN - buf_slice.len(); - // This is safe because we know exactly what can be in this buffer - let buf = unsafe { crate::str::from_utf8_unchecked(&buf[..len]) }; - fmt.pad(buf) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl fmt::Debug for Ipv6Addr { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(self, fmt) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Clone for Ipv6Addr { - fn clone(&self) -> Ipv6Addr { - *self - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl PartialEq for Ipv6Addr { - fn eq(&self, other: &Ipv6Addr) -> bool { - self.inner.s6_addr == other.inner.s6_addr - } -} - -#[stable(feature = "ip_cmp", since = "1.16.0")] -impl PartialEq for Ipv6Addr { - fn eq(&self, other: &IpAddr) -> bool { - match other { - IpAddr::V4(_) => false, - IpAddr::V6(v6) => self == v6, - } - } -} - -#[stable(feature = "ip_cmp", since = "1.16.0")] -impl PartialEq for IpAddr { - fn eq(&self, other: &Ipv6Addr) -> bool { - match self { - IpAddr::V4(_) => false, - IpAddr::V6(v6) => v6 == other, - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Eq for Ipv6Addr {} - -#[stable(feature = "rust1", since = "1.0.0")] -impl hash::Hash for Ipv6Addr { - fn hash(&self, s: &mut H) { - self.inner.s6_addr.hash(s) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl PartialOrd for Ipv6Addr { - fn partial_cmp(&self, other: &Ipv6Addr) -> Option { - Some(self.cmp(other)) - } -} - -#[stable(feature = "ip_cmp", since = "1.16.0")] -impl PartialOrd for IpAddr { - fn partial_cmp(&self, other: &Ipv6Addr) -> Option { - match self { - IpAddr::V4(_) => Some(Ordering::Less), - IpAddr::V6(v6) => v6.partial_cmp(other), - } - } -} - -#[stable(feature = "ip_cmp", since = "1.16.0")] -impl PartialOrd for Ipv6Addr { - fn partial_cmp(&self, other: &IpAddr) -> Option { - match other { - IpAddr::V4(_) => Some(Ordering::Greater), - IpAddr::V6(v6) => self.partial_cmp(v6), - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Ord for Ipv6Addr { - fn cmp(&self, other: &Ipv6Addr) -> Ordering { - self.segments().cmp(&other.segments()) - } -} - -impl AsInner for Ipv6Addr { - fn as_inner(&self) -> &c::in6_addr { - &self.inner - } -} -impl FromInner for Ipv6Addr { - fn from_inner(addr: c::in6_addr) -> Ipv6Addr { - Ipv6Addr { inner: addr } - } -} - -#[stable(feature = "i128", since = "1.26.0")] -impl From for u128 { - /// Convert an `Ipv6Addr` into a host byte order `u128`. - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv6Addr; - /// - /// let addr = Ipv6Addr::new( - /// 0x1020, 0x3040, 0x5060, 0x7080, - /// 0x90A0, 0xB0C0, 0xD0E0, 0xF00D, - /// ); - /// assert_eq!(0x102030405060708090A0B0C0D0E0F00D_u128, u128::from(addr)); - /// ``` - fn from(ip: Ipv6Addr) -> u128 { - let ip = ip.octets(); - u128::from_be_bytes(ip) - } -} -#[stable(feature = "i128", since = "1.26.0")] -impl From for Ipv6Addr { - /// Convert a host byte order `u128` into an `Ipv6Addr`. - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv6Addr; - /// - /// let addr = Ipv6Addr::from(0x102030405060708090A0B0C0D0E0F00D_u128); - /// assert_eq!( - /// Ipv6Addr::new( - /// 0x1020, 0x3040, 0x5060, 0x7080, - /// 0x90A0, 0xB0C0, 0xD0E0, 0xF00D, - /// ), - /// addr); - /// ``` - fn from(ip: u128) -> Ipv6Addr { - Ipv6Addr::from(ip.to_be_bytes()) - } -} - -#[stable(feature = "ipv6_from_octets", since = "1.9.0")] -impl From<[u8; 16]> for Ipv6Addr { - /// Creates an `Ipv6Addr` from a sixteen element byte array. - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv6Addr; - /// - /// let addr = Ipv6Addr::from([ - /// 25u8, 24u8, 23u8, 22u8, 21u8, 20u8, 19u8, 18u8, - /// 17u8, 16u8, 15u8, 14u8, 13u8, 12u8, 11u8, 10u8, - /// ]); - /// assert_eq!( - /// Ipv6Addr::new( - /// 0x1918, 0x1716, - /// 0x1514, 0x1312, - /// 0x1110, 0x0f0e, - /// 0x0d0c, 0x0b0a - /// ), - /// addr - /// ); - /// ``` - fn from(octets: [u8; 16]) -> Ipv6Addr { - let inner = c::in6_addr { s6_addr: octets }; - Ipv6Addr::from_inner(inner) - } -} - -#[stable(feature = "ipv6_from_segments", since = "1.16.0")] -impl From<[u16; 8]> for Ipv6Addr { - /// Creates an `Ipv6Addr` from an eight element 16-bit array. - /// - /// # Examples - /// - /// ``` - /// use std::net::Ipv6Addr; - /// - /// let addr = Ipv6Addr::from([ - /// 525u16, 524u16, 523u16, 522u16, - /// 521u16, 520u16, 519u16, 518u16, - /// ]); - /// assert_eq!( - /// Ipv6Addr::new( - /// 0x20d, 0x20c, - /// 0x20b, 0x20a, - /// 0x209, 0x208, - /// 0x207, 0x206 - /// ), - /// addr - /// ); - /// ``` - fn from(segments: [u16; 8]) -> Ipv6Addr { - let [a, b, c, d, e, f, g, h] = segments; - Ipv6Addr::new(a, b, c, d, e, f, g, h) - } -} - -#[stable(feature = "ip_from_slice", since = "1.17.0")] -impl From<[u8; 16]> for IpAddr { - /// Creates an `IpAddr::V6` from a sixteen element byte array. - /// - /// # Examples - /// - /// ``` - /// use std::net::{IpAddr, Ipv6Addr}; - /// - /// let addr = IpAddr::from([ - /// 25u8, 24u8, 23u8, 22u8, 21u8, 20u8, 19u8, 18u8, - /// 17u8, 16u8, 15u8, 14u8, 13u8, 12u8, 11u8, 10u8, - /// ]); - /// assert_eq!( - /// IpAddr::V6(Ipv6Addr::new( - /// 0x1918, 0x1716, - /// 0x1514, 0x1312, - /// 0x1110, 0x0f0e, - /// 0x0d0c, 0x0b0a - /// )), - /// addr - /// ); - /// ``` - fn from(octets: [u8; 16]) -> IpAddr { - IpAddr::V6(Ipv6Addr::from(octets)) - } -} - -#[stable(feature = "ip_from_slice", since = "1.17.0")] -impl From<[u16; 8]> for IpAddr { - /// Creates an `IpAddr::V6` from an eight element 16-bit array. - /// - /// # Examples - /// - /// ``` - /// use std::net::{IpAddr, Ipv6Addr}; - /// - /// let addr = IpAddr::from([ - /// 525u16, 524u16, 523u16, 522u16, - /// 521u16, 520u16, 519u16, 518u16, - /// ]); - /// assert_eq!( - /// IpAddr::V6(Ipv6Addr::new( - /// 0x20d, 0x20c, - /// 0x20b, 0x20a, - /// 0x209, 0x208, - /// 0x207, 0x206 - /// )), - /// addr - /// ); - /// ``` - fn from(segments: [u16; 8]) -> IpAddr { - IpAddr::V6(Ipv6Addr::from(segments)) - } -} - -// Tests for this module -#[cfg(all(test, not(target_os = "emscripten")))] -mod tests { - use crate::net::test::{sa4, sa6, tsa}; - use crate::net::*; - use crate::str::FromStr; - - #[test] - fn test_from_str_ipv4() { - assert_eq!(Ok(Ipv4Addr::new(127, 0, 0, 1)), "127.0.0.1".parse()); - assert_eq!(Ok(Ipv4Addr::new(255, 255, 255, 255)), "255.255.255.255".parse()); - assert_eq!(Ok(Ipv4Addr::new(0, 0, 0, 0)), "0.0.0.0".parse()); - - // out of range - let none: Option = "256.0.0.1".parse().ok(); - assert_eq!(None, none); - // too short - let none: Option = "255.0.0".parse().ok(); - assert_eq!(None, none); - // too long - let none: Option = "255.0.0.1.2".parse().ok(); - assert_eq!(None, none); - // no number between dots - let none: Option = "255.0..1".parse().ok(); - assert_eq!(None, none); - } - - #[test] - fn test_from_str_ipv6() { - assert_eq!(Ok(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0)), "0:0:0:0:0:0:0:0".parse()); - assert_eq!(Ok(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)), "0:0:0:0:0:0:0:1".parse()); - - assert_eq!(Ok(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)), "::1".parse()); - assert_eq!(Ok(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0)), "::".parse()); - - assert_eq!( - Ok(Ipv6Addr::new(0x2a02, 0x6b8, 0, 0, 0, 0, 0x11, 0x11)), - "2a02:6b8::11:11".parse() - ); - - // too long group - let none: Option = "::00000".parse().ok(); - assert_eq!(None, none); - // too short - let none: Option = "1:2:3:4:5:6:7".parse().ok(); - assert_eq!(None, none); - // too long - let none: Option = "1:2:3:4:5:6:7:8:9".parse().ok(); - assert_eq!(None, none); - // triple colon - let none: Option = "1:2:::6:7:8".parse().ok(); - assert_eq!(None, none); - // two double colons - let none: Option = "1:2::6::8".parse().ok(); - assert_eq!(None, none); - // `::` indicating zero groups of zeros - let none: Option = "1:2:3:4::5:6:7:8".parse().ok(); - assert_eq!(None, none); - } - - #[test] - fn test_from_str_ipv4_in_ipv6() { - assert_eq!(Ok(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 49152, 545)), "::192.0.2.33".parse()); - assert_eq!( - Ok(Ipv6Addr::new(0, 0, 0, 0, 0, 0xFFFF, 49152, 545)), - "::FFFF:192.0.2.33".parse() - ); - assert_eq!( - Ok(Ipv6Addr::new(0x64, 0xff9b, 0, 0, 0, 0, 49152, 545)), - "64:ff9b::192.0.2.33".parse() - ); - assert_eq!( - Ok(Ipv6Addr::new(0x2001, 0xdb8, 0x122, 0xc000, 0x2, 0x2100, 49152, 545)), - "2001:db8:122:c000:2:2100:192.0.2.33".parse() - ); - - // colon after v4 - let none: Option = "::127.0.0.1:".parse().ok(); - assert_eq!(None, none); - // not enough groups - let none: Option = "1.2.3.4.5:127.0.0.1".parse().ok(); - assert_eq!(None, none); - // too many groups - let none: Option = "1.2.3.4.5:6:7:127.0.0.1".parse().ok(); - assert_eq!(None, none); - } - - #[test] - fn test_from_str_socket_addr() { - assert_eq!(Ok(sa4(Ipv4Addr::new(77, 88, 21, 11), 80)), "77.88.21.11:80".parse()); - assert_eq!( - Ok(SocketAddrV4::new(Ipv4Addr::new(77, 88, 21, 11), 80)), - "77.88.21.11:80".parse() - ); - assert_eq!( - Ok(sa6(Ipv6Addr::new(0x2a02, 0x6b8, 0, 1, 0, 0, 0, 1), 53)), - "[2a02:6b8:0:1::1]:53".parse() - ); - assert_eq!( - Ok(SocketAddrV6::new(Ipv6Addr::new(0x2a02, 0x6b8, 0, 1, 0, 0, 0, 1), 53, 0, 0)), - "[2a02:6b8:0:1::1]:53".parse() - ); - assert_eq!( - Ok(sa6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0x7F00, 1), 22)), - "[::127.0.0.1]:22".parse() - ); - assert_eq!( - Ok(SocketAddrV6::new(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0x7F00, 1), 22, 0, 0)), - "[::127.0.0.1]:22".parse() - ); - - // without port - let none: Option = "127.0.0.1".parse().ok(); - assert_eq!(None, none); - // without port - let none: Option = "127.0.0.1:".parse().ok(); - assert_eq!(None, none); - // wrong brackets around v4 - let none: Option = "[127.0.0.1]:22".parse().ok(); - assert_eq!(None, none); - // port out of range - let none: Option = "127.0.0.1:123456".parse().ok(); - assert_eq!(None, none); - } - - #[test] - fn ipv4_addr_to_string() { - // Short address - assert_eq!(Ipv4Addr::new(1, 1, 1, 1).to_string(), "1.1.1.1"); - // Long address - assert_eq!(Ipv4Addr::new(127, 127, 127, 127).to_string(), "127.127.127.127"); - - // Test padding - assert_eq!(&format!("{:16}", Ipv4Addr::new(1, 1, 1, 1)), "1.1.1.1 "); - assert_eq!(&format!("{:>16}", Ipv4Addr::new(1, 1, 1, 1)), " 1.1.1.1"); - } - - #[test] - fn ipv6_addr_to_string() { - // ipv4-mapped address - let a1 = Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc000, 0x280); - assert_eq!(a1.to_string(), "::ffff:192.0.2.128"); - - // ipv4-compatible address - let a1 = Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0xc000, 0x280); - assert_eq!(a1.to_string(), "::192.0.2.128"); - - // v6 address with no zero segments - assert_eq!(Ipv6Addr::new(8, 9, 10, 11, 12, 13, 14, 15).to_string(), "8:9:a:b:c:d:e:f"); - - // longest possible IPv6 length - assert_eq!( - Ipv6Addr::new(0x1111, 0x2222, 0x3333, 0x4444, 0x5555, 0x6666, 0x7777, 0x8888) - .to_string(), - "1111:2222:3333:4444:5555:6666:7777:8888" - ); - // padding - assert_eq!( - &format!("{:20}", Ipv6Addr::new(1, 2, 3, 4, 5, 6, 7, 8)), - "1:2:3:4:5:6:7:8 " - ); - assert_eq!( - &format!("{:>20}", Ipv6Addr::new(1, 2, 3, 4, 5, 6, 7, 8)), - " 1:2:3:4:5:6:7:8" - ); - - // reduce a single run of zeros - assert_eq!( - "ae::ffff:102:304", - Ipv6Addr::new(0xae, 0, 0, 0, 0, 0xffff, 0x0102, 0x0304).to_string() - ); - - // don't reduce just a single zero segment - assert_eq!("1:2:3:4:5:6:0:8", Ipv6Addr::new(1, 2, 3, 4, 5, 6, 0, 8).to_string()); - - // 'any' address - assert_eq!("::", Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0).to_string()); - - // loopback address - assert_eq!("::1", Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1).to_string()); - - // ends in zeros - assert_eq!("1::", Ipv6Addr::new(1, 0, 0, 0, 0, 0, 0, 0).to_string()); - - // two runs of zeros, second one is longer - assert_eq!("1:0:0:4::8", Ipv6Addr::new(1, 0, 0, 4, 0, 0, 0, 8).to_string()); - - // two runs of zeros, equal length - assert_eq!("1::4:5:0:0:8", Ipv6Addr::new(1, 0, 0, 4, 5, 0, 0, 8).to_string()); - } - - #[test] - fn ipv4_to_ipv6() { - assert_eq!( - Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0x1234, 0x5678), - Ipv4Addr::new(0x12, 0x34, 0x56, 0x78).to_ipv6_mapped() - ); - assert_eq!( - Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0x1234, 0x5678), - Ipv4Addr::new(0x12, 0x34, 0x56, 0x78).to_ipv6_compatible() - ); - } - - #[test] - fn ipv6_to_ipv4() { - assert_eq!( - Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0x1234, 0x5678).to_ipv4(), - Some(Ipv4Addr::new(0x12, 0x34, 0x56, 0x78)) - ); - assert_eq!( - Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0x1234, 0x5678).to_ipv4(), - Some(Ipv4Addr::new(0x12, 0x34, 0x56, 0x78)) - ); - assert_eq!(Ipv6Addr::new(0, 0, 1, 0, 0, 0, 0x1234, 0x5678).to_ipv4(), None); - } - - #[test] - fn ip_properties() { - macro_rules! ip { - ($s:expr) => { - IpAddr::from_str($s).unwrap() - }; - } - - macro_rules! check { - ($s:expr) => { - check!($s, 0); - }; - - ($s:expr, $mask:expr) => {{ - let unspec: u8 = 1 << 0; - let loopback: u8 = 1 << 1; - let global: u8 = 1 << 2; - let multicast: u8 = 1 << 3; - let doc: u8 = 1 << 4; - - if ($mask & unspec) == unspec { - assert!(ip!($s).is_unspecified()); - } else { - assert!(!ip!($s).is_unspecified()); - } - - if ($mask & loopback) == loopback { - assert!(ip!($s).is_loopback()); - } else { - assert!(!ip!($s).is_loopback()); - } - - if ($mask & global) == global { - assert!(ip!($s).is_global()); - } else { - assert!(!ip!($s).is_global()); - } - - if ($mask & multicast) == multicast { - assert!(ip!($s).is_multicast()); - } else { - assert!(!ip!($s).is_multicast()); - } - - if ($mask & doc) == doc { - assert!(ip!($s).is_documentation()); - } else { - assert!(!ip!($s).is_documentation()); - } - }}; - } - - let unspec: u8 = 1 << 0; - let loopback: u8 = 1 << 1; - let global: u8 = 1 << 2; - let multicast: u8 = 1 << 3; - let doc: u8 = 1 << 4; - - check!("0.0.0.0", unspec); - check!("0.0.0.1"); - check!("0.1.0.0"); - check!("10.9.8.7"); - check!("127.1.2.3", loopback); - check!("172.31.254.253"); - check!("169.254.253.242"); - check!("192.0.2.183", doc); - check!("192.1.2.183", global); - check!("192.168.254.253"); - check!("198.51.100.0", doc); - check!("203.0.113.0", doc); - check!("203.2.113.0", global); - check!("224.0.0.0", global | multicast); - check!("239.255.255.255", global | multicast); - check!("255.255.255.255"); - // make sure benchmarking addresses are not global - check!("198.18.0.0"); - check!("198.18.54.2"); - check!("198.19.255.255"); - // make sure addresses reserved for protocol assignment are not global - check!("192.0.0.0"); - check!("192.0.0.255"); - check!("192.0.0.100"); - // make sure reserved addresses are not global - check!("240.0.0.0"); - check!("251.54.1.76"); - check!("254.255.255.255"); - // make sure shared addresses are not global - check!("100.64.0.0"); - check!("100.127.255.255"); - check!("100.100.100.0"); - - check!("::", unspec); - check!("::1", loopback); - check!("::0.0.0.2", global); - check!("1::", global); - check!("fc00::"); - check!("fdff:ffff::"); - check!("fe80:ffff::"); - check!("febf:ffff::"); - check!("fec0::", global); - check!("ff01::", multicast); - check!("ff02::", multicast); - check!("ff03::", multicast); - check!("ff04::", multicast); - check!("ff05::", multicast); - check!("ff08::", multicast); - check!("ff0e::", global | multicast); - check!("2001:db8:85a3::8a2e:370:7334", doc); - check!("102:304:506:708:90a:b0c:d0e:f10", global); - } - - #[test] - fn ipv4_properties() { - macro_rules! ip { - ($s:expr) => { - Ipv4Addr::from_str($s).unwrap() - }; - } - - macro_rules! check { - ($s:expr) => { - check!($s, 0); - }; - - ($s:expr, $mask:expr) => {{ - let unspec: u16 = 1 << 0; - let loopback: u16 = 1 << 1; - let private: u16 = 1 << 2; - let link_local: u16 = 1 << 3; - let global: u16 = 1 << 4; - let multicast: u16 = 1 << 5; - let broadcast: u16 = 1 << 6; - let documentation: u16 = 1 << 7; - let benchmarking: u16 = 1 << 8; - let ietf_protocol_assignment: u16 = 1 << 9; - let reserved: u16 = 1 << 10; - let shared: u16 = 1 << 11; - - if ($mask & unspec) == unspec { - assert!(ip!($s).is_unspecified()); - } else { - assert!(!ip!($s).is_unspecified()); - } - - if ($mask & loopback) == loopback { - assert!(ip!($s).is_loopback()); - } else { - assert!(!ip!($s).is_loopback()); - } - - if ($mask & private) == private { - assert!(ip!($s).is_private()); - } else { - assert!(!ip!($s).is_private()); - } - - if ($mask & link_local) == link_local { - assert!(ip!($s).is_link_local()); - } else { - assert!(!ip!($s).is_link_local()); - } - - if ($mask & global) == global { - assert!(ip!($s).is_global()); - } else { - assert!(!ip!($s).is_global()); - } - - if ($mask & multicast) == multicast { - assert!(ip!($s).is_multicast()); - } else { - assert!(!ip!($s).is_multicast()); - } - - if ($mask & broadcast) == broadcast { - assert!(ip!($s).is_broadcast()); - } else { - assert!(!ip!($s).is_broadcast()); - } - - if ($mask & documentation) == documentation { - assert!(ip!($s).is_documentation()); - } else { - assert!(!ip!($s).is_documentation()); - } - - if ($mask & benchmarking) == benchmarking { - assert!(ip!($s).is_benchmarking()); - } else { - assert!(!ip!($s).is_benchmarking()); - } - - if ($mask & ietf_protocol_assignment) == ietf_protocol_assignment { - assert!(ip!($s).is_ietf_protocol_assignment()); - } else { - assert!(!ip!($s).is_ietf_protocol_assignment()); - } - - if ($mask & reserved) == reserved { - assert!(ip!($s).is_reserved()); - } else { - assert!(!ip!($s).is_reserved()); - } - - if ($mask & shared) == shared { - assert!(ip!($s).is_shared()); - } else { - assert!(!ip!($s).is_shared()); - } - }}; - } - - let unspec: u16 = 1 << 0; - let loopback: u16 = 1 << 1; - let private: u16 = 1 << 2; - let link_local: u16 = 1 << 3; - let global: u16 = 1 << 4; - let multicast: u16 = 1 << 5; - let broadcast: u16 = 1 << 6; - let documentation: u16 = 1 << 7; - let benchmarking: u16 = 1 << 8; - let ietf_protocol_assignment: u16 = 1 << 9; - let reserved: u16 = 1 << 10; - let shared: u16 = 1 << 11; - - check!("0.0.0.0", unspec); - check!("0.0.0.1"); - check!("0.1.0.0"); - check!("10.9.8.7", private); - check!("127.1.2.3", loopback); - check!("172.31.254.253", private); - check!("169.254.253.242", link_local); - check!("192.0.2.183", documentation); - check!("192.1.2.183", global); - check!("192.168.254.253", private); - check!("198.51.100.0", documentation); - check!("203.0.113.0", documentation); - check!("203.2.113.0", global); - check!("224.0.0.0", global | multicast); - check!("239.255.255.255", global | multicast); - check!("255.255.255.255", broadcast); - check!("198.18.0.0", benchmarking); - check!("198.18.54.2", benchmarking); - check!("198.19.255.255", benchmarking); - check!("192.0.0.0", ietf_protocol_assignment); - check!("192.0.0.255", ietf_protocol_assignment); - check!("192.0.0.100", ietf_protocol_assignment); - check!("240.0.0.0", reserved); - check!("251.54.1.76", reserved); - check!("254.255.255.255", reserved); - check!("100.64.0.0", shared); - check!("100.127.255.255", shared); - check!("100.100.100.0", shared); - } - - #[test] - fn ipv6_properties() { - macro_rules! ip { - ($s:expr) => { - Ipv6Addr::from_str($s).unwrap() - }; - } - - macro_rules! check { - ($s:expr, &[$($octet:expr),*], $mask:expr) => { - assert_eq!($s, ip!($s).to_string()); - let octets = &[$($octet),*]; - assert_eq!(&ip!($s).octets(), octets); - assert_eq!(Ipv6Addr::from(*octets), ip!($s)); - - let unspecified: u16 = 1 << 0; - let loopback: u16 = 1 << 1; - let unique_local: u16 = 1 << 2; - let global: u16 = 1 << 3; - let unicast_link_local: u16 = 1 << 4; - let unicast_link_local_strict: u16 = 1 << 5; - let unicast_site_local: u16 = 1 << 6; - let unicast_global: u16 = 1 << 7; - let documentation: u16 = 1 << 8; - let multicast_interface_local: u16 = 1 << 9; - let multicast_link_local: u16 = 1 << 10; - let multicast_realm_local: u16 = 1 << 11; - let multicast_admin_local: u16 = 1 << 12; - let multicast_site_local: u16 = 1 << 13; - let multicast_organization_local: u16 = 1 << 14; - let multicast_global: u16 = 1 << 15; - let multicast: u16 = multicast_interface_local - | multicast_admin_local - | multicast_global - | multicast_link_local - | multicast_realm_local - | multicast_site_local - | multicast_organization_local; - - if ($mask & unspecified) == unspecified { - assert!(ip!($s).is_unspecified()); - } else { - assert!(!ip!($s).is_unspecified()); - } - if ($mask & loopback) == loopback { - assert!(ip!($s).is_loopback()); - } else { - assert!(!ip!($s).is_loopback()); - } - if ($mask & unique_local) == unique_local { - assert!(ip!($s).is_unique_local()); - } else { - assert!(!ip!($s).is_unique_local()); - } - if ($mask & global) == global { - assert!(ip!($s).is_global()); - } else { - assert!(!ip!($s).is_global()); - } - if ($mask & unicast_link_local) == unicast_link_local { - assert!(ip!($s).is_unicast_link_local()); - } else { - assert!(!ip!($s).is_unicast_link_local()); - } - if ($mask & unicast_link_local_strict) == unicast_link_local_strict { - assert!(ip!($s).is_unicast_link_local_strict()); - } else { - assert!(!ip!($s).is_unicast_link_local_strict()); - } - if ($mask & unicast_site_local) == unicast_site_local { - assert!(ip!($s).is_unicast_site_local()); - } else { - assert!(!ip!($s).is_unicast_site_local()); - } - if ($mask & unicast_global) == unicast_global { - assert!(ip!($s).is_unicast_global()); - } else { - assert!(!ip!($s).is_unicast_global()); - } - if ($mask & documentation) == documentation { - assert!(ip!($s).is_documentation()); - } else { - assert!(!ip!($s).is_documentation()); - } - if ($mask & multicast) != 0 { - assert!(ip!($s).multicast_scope().is_some()); - assert!(ip!($s).is_multicast()); - } else { - assert!(ip!($s).multicast_scope().is_none()); - assert!(!ip!($s).is_multicast()); - } - if ($mask & multicast_interface_local) == multicast_interface_local { - assert_eq!(ip!($s).multicast_scope().unwrap(), - Ipv6MulticastScope::InterfaceLocal); - } - if ($mask & multicast_link_local) == multicast_link_local { - assert_eq!(ip!($s).multicast_scope().unwrap(), - Ipv6MulticastScope::LinkLocal); - } - if ($mask & multicast_realm_local) == multicast_realm_local { - assert_eq!(ip!($s).multicast_scope().unwrap(), - Ipv6MulticastScope::RealmLocal); - } - if ($mask & multicast_admin_local) == multicast_admin_local { - assert_eq!(ip!($s).multicast_scope().unwrap(), - Ipv6MulticastScope::AdminLocal); - } - if ($mask & multicast_site_local) == multicast_site_local { - assert_eq!(ip!($s).multicast_scope().unwrap(), - Ipv6MulticastScope::SiteLocal); - } - if ($mask & multicast_organization_local) == multicast_organization_local { - assert_eq!(ip!($s).multicast_scope().unwrap(), - Ipv6MulticastScope::OrganizationLocal); - } - if ($mask & multicast_global) == multicast_global { - assert_eq!(ip!($s).multicast_scope().unwrap(), - Ipv6MulticastScope::Global); - } - } - } - - let unspecified: u16 = 1 << 0; - let loopback: u16 = 1 << 1; - let unique_local: u16 = 1 << 2; - let global: u16 = 1 << 3; - let unicast_link_local: u16 = 1 << 4; - let unicast_link_local_strict: u16 = 1 << 5; - let unicast_site_local: u16 = 1 << 6; - let unicast_global: u16 = 1 << 7; - let documentation: u16 = 1 << 8; - let multicast_interface_local: u16 = 1 << 9; - let multicast_link_local: u16 = 1 << 10; - let multicast_realm_local: u16 = 1 << 11; - let multicast_admin_local: u16 = 1 << 12; - let multicast_site_local: u16 = 1 << 13; - let multicast_organization_local: u16 = 1 << 14; - let multicast_global: u16 = 1 << 15; - - check!("::", &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], unspecified); - - check!("::1", &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], loopback); - - check!( - "::0.0.0.2", - &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], - global | unicast_global - ); - - check!("1::", &[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], global | unicast_global); - - check!("fc00::", &[0xfc, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], unique_local); - - check!( - "fdff:ffff::", - &[0xfd, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - unique_local - ); - - check!( - "fe80:ffff::", - &[0xfe, 0x80, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - unicast_link_local - ); - - check!( - "fe80::", - &[0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - unicast_link_local | unicast_link_local_strict - ); - - check!( - "febf:ffff::", - &[0xfe, 0xbf, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - unicast_link_local - ); - - check!( - "febf::", - &[0xfe, 0xbf, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - unicast_link_local - ); - - check!( - "febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff", - &[ - 0xfe, 0xbf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff - ], - unicast_link_local - ); - - check!( - "fe80::ffff:ffff:ffff:ffff", - &[ - 0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff - ], - unicast_link_local | unicast_link_local_strict - ); - - check!( - "fe80:0:0:1::", - &[0xfe, 0x80, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], - unicast_link_local - ); - - check!( - "fec0::", - &[0xfe, 0xc0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - unicast_site_local | unicast_global | global - ); - - check!( - "ff01::", - &[0xff, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - multicast_interface_local - ); - - check!( - "ff02::", - &[0xff, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - multicast_link_local - ); - - check!( - "ff03::", - &[0xff, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - multicast_realm_local - ); - - check!( - "ff04::", - &[0xff, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - multicast_admin_local - ); - - check!( - "ff05::", - &[0xff, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - multicast_site_local - ); - - check!( - "ff08::", - &[0xff, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - multicast_organization_local - ); - - check!( - "ff0e::", - &[0xff, 0xe, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - multicast_global | global - ); - - check!( - "2001:db8:85a3::8a2e:370:7334", - &[0x20, 1, 0xd, 0xb8, 0x85, 0xa3, 0, 0, 0, 0, 0x8a, 0x2e, 3, 0x70, 0x73, 0x34], - documentation - ); - - check!( - "102:304:506:708:90a:b0c:d0e:f10", - &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], - global | unicast_global - ); - } - - #[test] - fn to_socket_addr_socketaddr() { - let a = sa4(Ipv4Addr::new(77, 88, 21, 11), 12345); - assert_eq!(Ok(vec![a]), tsa(a)); - } - - #[test] - fn test_ipv4_to_int() { - let a = Ipv4Addr::new(0x11, 0x22, 0x33, 0x44); - assert_eq!(u32::from(a), 0x11223344); - } - - #[test] - fn test_int_to_ipv4() { - let a = Ipv4Addr::new(0x11, 0x22, 0x33, 0x44); - assert_eq!(Ipv4Addr::from(0x11223344), a); - } - - #[test] - fn test_ipv6_to_int() { - let a = Ipv6Addr::new(0x1122, 0x3344, 0x5566, 0x7788, 0x99aa, 0xbbcc, 0xddee, 0xff11); - assert_eq!(u128::from(a), 0x112233445566778899aabbccddeeff11u128); - } - - #[test] - fn test_int_to_ipv6() { - let a = Ipv6Addr::new(0x1122, 0x3344, 0x5566, 0x7788, 0x99aa, 0xbbcc, 0xddee, 0xff11); - assert_eq!(Ipv6Addr::from(0x112233445566778899aabbccddeeff11u128), a); - } - - #[test] - fn ipv4_from_constructors() { - assert_eq!(Ipv4Addr::LOCALHOST, Ipv4Addr::new(127, 0, 0, 1)); - assert!(Ipv4Addr::LOCALHOST.is_loopback()); - assert_eq!(Ipv4Addr::UNSPECIFIED, Ipv4Addr::new(0, 0, 0, 0)); - assert!(Ipv4Addr::UNSPECIFIED.is_unspecified()); - assert_eq!(Ipv4Addr::BROADCAST, Ipv4Addr::new(255, 255, 255, 255)); - assert!(Ipv4Addr::BROADCAST.is_broadcast()); - } - - #[test] - fn ipv6_from_contructors() { - assert_eq!(Ipv6Addr::LOCALHOST, Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)); - assert!(Ipv6Addr::LOCALHOST.is_loopback()); - assert_eq!(Ipv6Addr::UNSPECIFIED, Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0)); - assert!(Ipv6Addr::UNSPECIFIED.is_unspecified()); - } - - #[test] - fn ipv4_from_octets() { - assert_eq!(Ipv4Addr::from([127, 0, 0, 1]), Ipv4Addr::new(127, 0, 0, 1)) - } - - #[test] - fn ipv6_from_segments() { - let from_u16s = - Ipv6Addr::from([0x0011, 0x2233, 0x4455, 0x6677, 0x8899, 0xaabb, 0xccdd, 0xeeff]); - let new = Ipv6Addr::new(0x0011, 0x2233, 0x4455, 0x6677, 0x8899, 0xaabb, 0xccdd, 0xeeff); - assert_eq!(new, from_u16s); - } - - #[test] - fn ipv6_from_octets() { - let from_u16s = - Ipv6Addr::from([0x0011, 0x2233, 0x4455, 0x6677, 0x8899, 0xaabb, 0xccdd, 0xeeff]); - let from_u8s = Ipv6Addr::from([ - 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, - 0xee, 0xff, - ]); - assert_eq!(from_u16s, from_u8s); - } - - #[test] - fn cmp() { - let v41 = Ipv4Addr::new(100, 64, 3, 3); - let v42 = Ipv4Addr::new(192, 0, 2, 2); - let v61 = "2001:db8:f00::1002".parse::().unwrap(); - let v62 = "2001:db8:f00::2001".parse::().unwrap(); - assert!(v41 < v42); - assert!(v61 < v62); - - assert_eq!(v41, IpAddr::V4(v41)); - assert_eq!(v61, IpAddr::V6(v61)); - assert!(v41 != IpAddr::V4(v42)); - assert!(v61 != IpAddr::V6(v62)); - - assert!(v41 < IpAddr::V4(v42)); - assert!(v61 < IpAddr::V6(v62)); - assert!(IpAddr::V4(v41) < v42); - assert!(IpAddr::V6(v61) < v62); - - assert!(v41 < IpAddr::V6(v61)); - assert!(IpAddr::V4(v41) < v61); - } - - #[test] - fn is_v4() { - let ip = IpAddr::V4(Ipv4Addr::new(100, 64, 3, 3)); - assert!(ip.is_ipv4()); - assert!(!ip.is_ipv6()); - } - - #[test] - fn is_v6() { - let ip = IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0x1234, 0x5678)); - assert!(!ip.is_ipv4()); - assert!(ip.is_ipv6()); - } -} diff --git a/crux-mir/lib/std/src/net/ip_addr.rs b/crux-mir/lib/std/src/net/ip_addr.rs new file mode 100644 index 000000000..07f08c1b5 --- /dev/null +++ b/crux-mir/lib/std/src/net/ip_addr.rs @@ -0,0 +1,2102 @@ +// Tests for this module +#[cfg(all(test, not(target_os = "emscripten")))] +mod tests; + +use crate::cmp::Ordering; +use crate::fmt::{self, Write}; +use crate::mem::transmute; +use crate::sys::net::netc as c; +use crate::sys_common::{FromInner, IntoInner}; + +use super::display_buffer::DisplayBuffer; + +/// An IP address, either IPv4 or IPv6. +/// +/// This enum can contain either an [`Ipv4Addr`] or an [`Ipv6Addr`], see their +/// respective documentation for more details. +/// +/// # Examples +/// +/// ``` +/// use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +/// +/// let localhost_v4 = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)); +/// let localhost_v6 = IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)); +/// +/// assert_eq!("127.0.0.1".parse(), Ok(localhost_v4)); +/// assert_eq!("::1".parse(), Ok(localhost_v6)); +/// +/// assert_eq!(localhost_v4.is_ipv6(), false); +/// assert_eq!(localhost_v4.is_ipv4(), true); +/// ``` +#[cfg_attr(not(test), rustc_diagnostic_item = "IpAddr")] +#[stable(feature = "ip_addr", since = "1.7.0")] +#[derive(Copy, Clone, Eq, PartialEq, Hash, PartialOrd, Ord)] +pub enum IpAddr { + /// An IPv4 address. + #[stable(feature = "ip_addr", since = "1.7.0")] + V4(#[stable(feature = "ip_addr", since = "1.7.0")] Ipv4Addr), + /// An IPv6 address. + #[stable(feature = "ip_addr", since = "1.7.0")] + V6(#[stable(feature = "ip_addr", since = "1.7.0")] Ipv6Addr), +} + +/// An IPv4 address. +/// +/// IPv4 addresses are defined as 32-bit integers in [IETF RFC 791]. +/// They are usually represented as four octets. +/// +/// See [`IpAddr`] for a type encompassing both IPv4 and IPv6 addresses. +/// +/// [IETF RFC 791]: https://tools.ietf.org/html/rfc791 +/// +/// # Textual representation +/// +/// `Ipv4Addr` provides a [`FromStr`] implementation. The four octets are in decimal +/// notation, divided by `.` (this is called "dot-decimal notation"). +/// Notably, octal numbers (which are indicated with a leading `0`) and hexadecimal numbers (which +/// are indicated with a leading `0x`) are not allowed per [IETF RFC 6943]. +/// +/// [IETF RFC 6943]: https://tools.ietf.org/html/rfc6943#section-3.1.1 +/// [`FromStr`]: crate::str::FromStr +/// +/// # Examples +/// +/// ``` +/// use std::net::Ipv4Addr; +/// +/// let localhost = Ipv4Addr::new(127, 0, 0, 1); +/// assert_eq!("127.0.0.1".parse(), Ok(localhost)); +/// assert_eq!(localhost.is_loopback(), true); +/// assert!("012.004.002.000".parse::().is_err()); // all octets are in octal +/// assert!("0000000.0.0.0".parse::().is_err()); // first octet is a zero in octal +/// assert!("0xcb.0x0.0x71.0x00".parse::().is_err()); // all octets are in hex +/// ``` +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +#[stable(feature = "rust1", since = "1.0.0")] +pub struct Ipv4Addr { + octets: [u8; 4], +} + +/// An IPv6 address. +/// +/// IPv6 addresses are defined as 128-bit integers in [IETF RFC 4291]. +/// They are usually represented as eight 16-bit segments. +/// +/// [IETF RFC 4291]: https://tools.ietf.org/html/rfc4291 +/// +/// # Embedding IPv4 Addresses +/// +/// See [`IpAddr`] for a type encompassing both IPv4 and IPv6 addresses. +/// +/// To assist in the transition from IPv4 to IPv6 two types of IPv6 addresses that embed an IPv4 address were defined: +/// IPv4-compatible and IPv4-mapped addresses. Of these IPv4-compatible addresses have been officially deprecated. +/// +/// Both types of addresses are not assigned any special meaning by this implementation, +/// other than what the relevant standards prescribe. This means that an address like `::ffff:127.0.0.1`, +/// while representing an IPv4 loopback address, is not itself an IPv6 loopback address; only `::1` is. +/// To handle these so called "IPv4-in-IPv6" addresses, they have to first be converted to their canonical IPv4 address. +/// +/// ### IPv4-Compatible IPv6 Addresses +/// +/// IPv4-compatible IPv6 addresses are defined in [IETF RFC 4291 Section 2.5.5.1], and have been officially deprecated. +/// The RFC describes the format of an "IPv4-Compatible IPv6 address" as follows: +/// +/// ```text +/// | 80 bits | 16 | 32 bits | +/// +--------------------------------------+--------------------------+ +/// |0000..............................0000|0000| IPv4 address | +/// +--------------------------------------+----+---------------------+ +/// ``` +/// So `::a.b.c.d` would be an IPv4-compatible IPv6 address representing the IPv4 address `a.b.c.d`. +/// +/// To convert from an IPv4 address to an IPv4-compatible IPv6 address, use [`Ipv4Addr::to_ipv6_compatible`]. +/// Use [`Ipv6Addr::to_ipv4`] to convert an IPv4-compatible IPv6 address to the canonical IPv4 address. +/// +/// [IETF RFC 4291 Section 2.5.5.1]: https://datatracker.ietf.org/doc/html/rfc4291#section-2.5.5.1 +/// +/// ### IPv4-Mapped IPv6 Addresses +/// +/// IPv4-mapped IPv6 addresses are defined in [IETF RFC 4291 Section 2.5.5.2]. +/// The RFC describes the format of an "IPv4-Mapped IPv6 address" as follows: +/// +/// ```text +/// | 80 bits | 16 | 32 bits | +/// +--------------------------------------+--------------------------+ +/// |0000..............................0000|FFFF| IPv4 address | +/// +--------------------------------------+----+---------------------+ +/// ``` +/// So `::ffff:a.b.c.d` would be an IPv4-mapped IPv6 address representing the IPv4 address `a.b.c.d`. +/// +/// To convert from an IPv4 address to an IPv4-mapped IPv6 address, use [`Ipv4Addr::to_ipv6_mapped`]. +/// Use [`Ipv6Addr::to_ipv4`] to convert an IPv4-mapped IPv6 address to the canonical IPv4 address. +/// Note that this will also convert the IPv6 loopback address `::1` to `0.0.0.1`. Use +/// [`Ipv6Addr::to_ipv4_mapped`] to avoid this. +/// +/// [IETF RFC 4291 Section 2.5.5.2]: https://datatracker.ietf.org/doc/html/rfc4291#section-2.5.5.2 +/// +/// # Textual representation +/// +/// `Ipv6Addr` provides a [`FromStr`] implementation. There are many ways to represent +/// an IPv6 address in text, but in general, each segments is written in hexadecimal +/// notation, and segments are separated by `:`. For more information, see +/// [IETF RFC 5952]. +/// +/// [`FromStr`]: crate::str::FromStr +/// [IETF RFC 5952]: https://tools.ietf.org/html/rfc5952 +/// +/// # Examples +/// +/// ``` +/// use std::net::Ipv6Addr; +/// +/// let localhost = Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1); +/// assert_eq!("::1".parse(), Ok(localhost)); +/// assert_eq!(localhost.is_loopback(), true); +/// ``` +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +#[stable(feature = "rust1", since = "1.0.0")] +pub struct Ipv6Addr { + octets: [u8; 16], +} + +/// Scope of an [IPv6 multicast address] as defined in [IETF RFC 7346 section 2]. +/// +/// # Stability Guarantees +/// +/// Not all possible values for a multicast scope have been assigned. +/// Future RFCs may introduce new scopes, which will be added as variants to this enum; +/// because of this the enum is marked as `#[non_exhaustive]`. +/// +/// # Examples +/// ``` +/// #![feature(ip)] +/// +/// use std::net::Ipv6Addr; +/// use std::net::Ipv6MulticastScope::*; +/// +/// // An IPv6 multicast address with global scope (`ff0e::`). +/// let address = Ipv6Addr::new(0xff0e, 0, 0, 0, 0, 0, 0, 0); +/// +/// // Will print "Global scope". +/// match address.multicast_scope() { +/// Some(InterfaceLocal) => println!("Interface-Local scope"), +/// Some(LinkLocal) => println!("Link-Local scope"), +/// Some(RealmLocal) => println!("Realm-Local scope"), +/// Some(AdminLocal) => println!("Admin-Local scope"), +/// Some(SiteLocal) => println!("Site-Local scope"), +/// Some(OrganizationLocal) => println!("Organization-Local scope"), +/// Some(Global) => println!("Global scope"), +/// Some(_) => println!("Unknown scope"), +/// None => println!("Not a multicast address!") +/// } +/// +/// ``` +/// +/// [IPv6 multicast address]: Ipv6Addr +/// [IETF RFC 7346 section 2]: https://tools.ietf.org/html/rfc7346#section-2 +#[derive(Copy, PartialEq, Eq, Clone, Hash, Debug)] +#[unstable(feature = "ip", issue = "27709")] +#[non_exhaustive] +pub enum Ipv6MulticastScope { + /// Interface-Local scope. + InterfaceLocal, + /// Link-Local scope. + LinkLocal, + /// Realm-Local scope. + RealmLocal, + /// Admin-Local scope. + AdminLocal, + /// Site-Local scope. + SiteLocal, + /// Organization-Local scope. + OrganizationLocal, + /// Global scope. + Global, +} + +impl IpAddr { + /// Returns [`true`] for the special 'unspecified' address. + /// + /// See the documentation for [`Ipv4Addr::is_unspecified()`] and + /// [`Ipv6Addr::is_unspecified()`] for more details. + /// + /// # Examples + /// + /// ``` + /// use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + /// + /// assert_eq!(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)).is_unspecified(), true); + /// assert_eq!(IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0)).is_unspecified(), true); + /// ``` + #[rustc_const_stable(feature = "const_ip_50", since = "1.50.0")] + #[stable(feature = "ip_shared", since = "1.12.0")] + #[must_use] + #[inline] + pub const fn is_unspecified(&self) -> bool { + match self { + IpAddr::V4(ip) => ip.is_unspecified(), + IpAddr::V6(ip) => ip.is_unspecified(), + } + } + + /// Returns [`true`] if this is a loopback address. + /// + /// See the documentation for [`Ipv4Addr::is_loopback()`] and + /// [`Ipv6Addr::is_loopback()`] for more details. + /// + /// # Examples + /// + /// ``` + /// use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + /// + /// assert_eq!(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)).is_loopback(), true); + /// assert_eq!(IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0x1)).is_loopback(), true); + /// ``` + #[rustc_const_stable(feature = "const_ip_50", since = "1.50.0")] + #[stable(feature = "ip_shared", since = "1.12.0")] + #[must_use] + #[inline] + pub const fn is_loopback(&self) -> bool { + match self { + IpAddr::V4(ip) => ip.is_loopback(), + IpAddr::V6(ip) => ip.is_loopback(), + } + } + + /// Returns [`true`] if the address appears to be globally routable. + /// + /// See the documentation for [`Ipv4Addr::is_global()`] and + /// [`Ipv6Addr::is_global()`] for more details. + /// + /// # Examples + /// + /// ``` + /// #![feature(ip)] + /// + /// use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + /// + /// assert_eq!(IpAddr::V4(Ipv4Addr::new(80, 9, 12, 3)).is_global(), true); + /// assert_eq!(IpAddr::V6(Ipv6Addr::new(0, 0, 0x1c9, 0, 0, 0xafc8, 0, 0x1)).is_global(), true); + /// ``` + #[rustc_const_unstable(feature = "const_ip", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] + #[must_use] + #[inline] + pub const fn is_global(&self) -> bool { + match self { + IpAddr::V4(ip) => ip.is_global(), + IpAddr::V6(ip) => ip.is_global(), + } + } + + /// Returns [`true`] if this is a multicast address. + /// + /// See the documentation for [`Ipv4Addr::is_multicast()`] and + /// [`Ipv6Addr::is_multicast()`] for more details. + /// + /// # Examples + /// + /// ``` + /// use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + /// + /// assert_eq!(IpAddr::V4(Ipv4Addr::new(224, 254, 0, 0)).is_multicast(), true); + /// assert_eq!(IpAddr::V6(Ipv6Addr::new(0xff00, 0, 0, 0, 0, 0, 0, 0)).is_multicast(), true); + /// ``` + #[rustc_const_stable(feature = "const_ip_50", since = "1.50.0")] + #[stable(feature = "ip_shared", since = "1.12.0")] + #[must_use] + #[inline] + pub const fn is_multicast(&self) -> bool { + match self { + IpAddr::V4(ip) => ip.is_multicast(), + IpAddr::V6(ip) => ip.is_multicast(), + } + } + + /// Returns [`true`] if this address is in a range designated for documentation. + /// + /// See the documentation for [`Ipv4Addr::is_documentation()`] and + /// [`Ipv6Addr::is_documentation()`] for more details. + /// + /// # Examples + /// + /// ``` + /// #![feature(ip)] + /// + /// use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + /// + /// assert_eq!(IpAddr::V4(Ipv4Addr::new(203, 0, 113, 6)).is_documentation(), true); + /// assert_eq!( + /// IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 0)).is_documentation(), + /// true + /// ); + /// ``` + #[rustc_const_unstable(feature = "const_ip", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] + #[must_use] + #[inline] + pub const fn is_documentation(&self) -> bool { + match self { + IpAddr::V4(ip) => ip.is_documentation(), + IpAddr::V6(ip) => ip.is_documentation(), + } + } + + /// Returns [`true`] if this address is in a range designated for benchmarking. + /// + /// See the documentation for [`Ipv4Addr::is_benchmarking()`] and + /// [`Ipv6Addr::is_benchmarking()`] for more details. + /// + /// # Examples + /// + /// ``` + /// #![feature(ip)] + /// + /// use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + /// + /// assert_eq!(IpAddr::V4(Ipv4Addr::new(198, 19, 255, 255)).is_benchmarking(), true); + /// assert_eq!(IpAddr::V6(Ipv6Addr::new(0x2001, 0x2, 0, 0, 0, 0, 0, 0)).is_benchmarking(), true); + /// ``` + #[unstable(feature = "ip", issue = "27709")] + #[must_use] + #[inline] + pub const fn is_benchmarking(&self) -> bool { + match self { + IpAddr::V4(ip) => ip.is_benchmarking(), + IpAddr::V6(ip) => ip.is_benchmarking(), + } + } + + /// Returns [`true`] if this address is an [`IPv4` address], and [`false`] + /// otherwise. + /// + /// [`IPv4` address]: IpAddr::V4 + /// + /// # Examples + /// + /// ``` + /// use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + /// + /// assert_eq!(IpAddr::V4(Ipv4Addr::new(203, 0, 113, 6)).is_ipv4(), true); + /// assert_eq!(IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 0)).is_ipv4(), false); + /// ``` + #[rustc_const_stable(feature = "const_ip_50", since = "1.50.0")] + #[stable(feature = "ipaddr_checker", since = "1.16.0")] + #[must_use] + #[inline] + pub const fn is_ipv4(&self) -> bool { + matches!(self, IpAddr::V4(_)) + } + + /// Returns [`true`] if this address is an [`IPv6` address], and [`false`] + /// otherwise. + /// + /// [`IPv6` address]: IpAddr::V6 + /// + /// # Examples + /// + /// ``` + /// use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + /// + /// assert_eq!(IpAddr::V4(Ipv4Addr::new(203, 0, 113, 6)).is_ipv6(), false); + /// assert_eq!(IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 0)).is_ipv6(), true); + /// ``` + #[rustc_const_stable(feature = "const_ip_50", since = "1.50.0")] + #[stable(feature = "ipaddr_checker", since = "1.16.0")] + #[must_use] + #[inline] + pub const fn is_ipv6(&self) -> bool { + matches!(self, IpAddr::V6(_)) + } + + /// Converts this address to an `IpAddr::V4` if it is an IPv4-mapped IPv6 addresses, otherwise it + /// return `self` as-is. + /// + /// # Examples + /// + /// ``` + /// #![feature(ip)] + /// use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + /// + /// assert_eq!(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)).to_canonical().is_loopback(), true); + /// assert_eq!(IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0x7f00, 0x1)).is_loopback(), false); + /// assert_eq!(IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0x7f00, 0x1)).to_canonical().is_loopback(), true); + /// ``` + #[inline] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[rustc_const_unstable(feature = "const_ip", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] + pub const fn to_canonical(&self) -> IpAddr { + match self { + &v4 @ IpAddr::V4(_) => v4, + IpAddr::V6(v6) => v6.to_canonical(), + } + } +} + +impl Ipv4Addr { + /// Creates a new IPv4 address from four eight-bit octets. + /// + /// The result will represent the IP address `a`.`b`.`c`.`d`. + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv4Addr; + /// + /// let addr = Ipv4Addr::new(127, 0, 0, 1); + /// ``` + #[rustc_const_stable(feature = "const_ip_32", since = "1.32.0")] + #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + #[inline] + pub const fn new(a: u8, b: u8, c: u8, d: u8) -> Ipv4Addr { + Ipv4Addr { octets: [a, b, c, d] } + } + + /// An IPv4 address with the address pointing to localhost: `127.0.0.1` + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv4Addr; + /// + /// let addr = Ipv4Addr::LOCALHOST; + /// assert_eq!(addr, Ipv4Addr::new(127, 0, 0, 1)); + /// ``` + #[stable(feature = "ip_constructors", since = "1.30.0")] + pub const LOCALHOST: Self = Ipv4Addr::new(127, 0, 0, 1); + + /// An IPv4 address representing an unspecified address: `0.0.0.0` + /// + /// This corresponds to the constant `INADDR_ANY` in other languages. + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv4Addr; + /// + /// let addr = Ipv4Addr::UNSPECIFIED; + /// assert_eq!(addr, Ipv4Addr::new(0, 0, 0, 0)); + /// ``` + #[doc(alias = "INADDR_ANY")] + #[stable(feature = "ip_constructors", since = "1.30.0")] + pub const UNSPECIFIED: Self = Ipv4Addr::new(0, 0, 0, 0); + + /// An IPv4 address representing the broadcast address: `255.255.255.255` + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv4Addr; + /// + /// let addr = Ipv4Addr::BROADCAST; + /// assert_eq!(addr, Ipv4Addr::new(255, 255, 255, 255)); + /// ``` + #[stable(feature = "ip_constructors", since = "1.30.0")] + pub const BROADCAST: Self = Ipv4Addr::new(255, 255, 255, 255); + + /// Returns the four eight-bit integers that make up this address. + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv4Addr; + /// + /// let addr = Ipv4Addr::new(127, 0, 0, 1); + /// assert_eq!(addr.octets(), [127, 0, 0, 1]); + /// ``` + #[rustc_const_stable(feature = "const_ip_50", since = "1.50.0")] + #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + #[inline] + pub const fn octets(&self) -> [u8; 4] { + self.octets + } + + /// Returns [`true`] for the special 'unspecified' address (`0.0.0.0`). + /// + /// This property is defined in _UNIX Network Programming, Second Edition_, + /// W. Richard Stevens, p. 891; see also [ip7]. + /// + /// [ip7]: https://man7.org/linux/man-pages/man7/ip.7.html + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv4Addr; + /// + /// assert_eq!(Ipv4Addr::new(0, 0, 0, 0).is_unspecified(), true); + /// assert_eq!(Ipv4Addr::new(45, 22, 13, 197).is_unspecified(), false); + /// ``` + #[rustc_const_stable(feature = "const_ip_32", since = "1.32.0")] + #[stable(feature = "ip_shared", since = "1.12.0")] + #[must_use] + #[inline] + pub const fn is_unspecified(&self) -> bool { + u32::from_be_bytes(self.octets) == 0 + } + + /// Returns [`true`] if this is a loopback address (`127.0.0.0/8`). + /// + /// This property is defined by [IETF RFC 1122]. + /// + /// [IETF RFC 1122]: https://tools.ietf.org/html/rfc1122 + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv4Addr; + /// + /// assert_eq!(Ipv4Addr::new(127, 0, 0, 1).is_loopback(), true); + /// assert_eq!(Ipv4Addr::new(45, 22, 13, 197).is_loopback(), false); + /// ``` + #[rustc_const_stable(feature = "const_ip_50", since = "1.50.0")] + #[stable(since = "1.7.0", feature = "ip_17")] + #[must_use] + #[inline] + pub const fn is_loopback(&self) -> bool { + self.octets()[0] == 127 + } + + /// Returns [`true`] if this is a private address. + /// + /// The private address ranges are defined in [IETF RFC 1918] and include: + /// + /// - `10.0.0.0/8` + /// - `172.16.0.0/12` + /// - `192.168.0.0/16` + /// + /// [IETF RFC 1918]: https://tools.ietf.org/html/rfc1918 + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv4Addr; + /// + /// assert_eq!(Ipv4Addr::new(10, 0, 0, 1).is_private(), true); + /// assert_eq!(Ipv4Addr::new(10, 10, 10, 10).is_private(), true); + /// assert_eq!(Ipv4Addr::new(172, 16, 10, 10).is_private(), true); + /// assert_eq!(Ipv4Addr::new(172, 29, 45, 14).is_private(), true); + /// assert_eq!(Ipv4Addr::new(172, 32, 0, 2).is_private(), false); + /// assert_eq!(Ipv4Addr::new(192, 168, 0, 2).is_private(), true); + /// assert_eq!(Ipv4Addr::new(192, 169, 0, 2).is_private(), false); + /// ``` + #[rustc_const_stable(feature = "const_ip_50", since = "1.50.0")] + #[stable(since = "1.7.0", feature = "ip_17")] + #[must_use] + #[inline] + pub const fn is_private(&self) -> bool { + match self.octets() { + [10, ..] => true, + [172, b, ..] if b >= 16 && b <= 31 => true, + [192, 168, ..] => true, + _ => false, + } + } + + /// Returns [`true`] if the address is link-local (`169.254.0.0/16`). + /// + /// This property is defined by [IETF RFC 3927]. + /// + /// [IETF RFC 3927]: https://tools.ietf.org/html/rfc3927 + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv4Addr; + /// + /// assert_eq!(Ipv4Addr::new(169, 254, 0, 0).is_link_local(), true); + /// assert_eq!(Ipv4Addr::new(169, 254, 10, 65).is_link_local(), true); + /// assert_eq!(Ipv4Addr::new(16, 89, 10, 65).is_link_local(), false); + /// ``` + #[rustc_const_stable(feature = "const_ip_50", since = "1.50.0")] + #[stable(since = "1.7.0", feature = "ip_17")] + #[must_use] + #[inline] + pub const fn is_link_local(&self) -> bool { + matches!(self.octets(), [169, 254, ..]) + } + + /// Returns [`true`] if the address appears to be globally reachable + /// as specified by the [IANA IPv4 Special-Purpose Address Registry]. + /// Whether or not an address is practically reachable will depend on your network configuration. + /// + /// Most IPv4 addresses are globally reachable; + /// unless they are specifically defined as *not* globally reachable. + /// + /// Non-exhaustive list of notable addresses that are not globally reachable: + /// + /// - The [unspecified address] ([`is_unspecified`](Ipv4Addr::is_unspecified)) + /// - Addresses reserved for private use ([`is_private`](Ipv4Addr::is_private)) + /// - Addresses in the shared address space ([`is_shared`](Ipv4Addr::is_shared)) + /// - Loopback addresses ([`is_loopback`](Ipv4Addr::is_loopback)) + /// - Link-local addresses ([`is_link_local`](Ipv4Addr::is_link_local)) + /// - Addresses reserved for documentation ([`is_documentation`](Ipv4Addr::is_documentation)) + /// - Addresses reserved for benchmarking ([`is_benchmarking`](Ipv4Addr::is_benchmarking)) + /// - Reserved addresses ([`is_reserved`](Ipv4Addr::is_reserved)) + /// - The [broadcast address] ([`is_broadcast`](Ipv4Addr::is_broadcast)) + /// + /// For the complete overview of which addresses are globally reachable, see the table at the [IANA IPv4 Special-Purpose Address Registry]. + /// + /// [IANA IPv4 Special-Purpose Address Registry]: https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml + /// [unspecified address]: Ipv4Addr::UNSPECIFIED + /// [broadcast address]: Ipv4Addr::BROADCAST + + /// + /// # Examples + /// + /// ``` + /// #![feature(ip)] + /// + /// use std::net::Ipv4Addr; + /// + /// // Most IPv4 addresses are globally reachable: + /// assert_eq!(Ipv4Addr::new(80, 9, 12, 3).is_global(), true); + /// + /// // However some addresses have been assigned a special meaning + /// // that makes them not globally reachable. Some examples are: + /// + /// // The unspecified address (`0.0.0.0`) + /// assert_eq!(Ipv4Addr::UNSPECIFIED.is_global(), false); + /// + /// // Addresses reserved for private use (`10.0.0.0/8`, `172.16.0.0/12`, 192.168.0.0/16) + /// assert_eq!(Ipv4Addr::new(10, 254, 0, 0).is_global(), false); + /// assert_eq!(Ipv4Addr::new(192, 168, 10, 65).is_global(), false); + /// assert_eq!(Ipv4Addr::new(172, 16, 10, 65).is_global(), false); + /// + /// // Addresses in the shared address space (`100.64.0.0/10`) + /// assert_eq!(Ipv4Addr::new(100, 100, 0, 0).is_global(), false); + /// + /// // The loopback addresses (`127.0.0.0/8`) + /// assert_eq!(Ipv4Addr::LOCALHOST.is_global(), false); + /// + /// // Link-local addresses (`169.254.0.0/16`) + /// assert_eq!(Ipv4Addr::new(169, 254, 45, 1).is_global(), false); + /// + /// // Addresses reserved for documentation (`192.0.2.0/24`, `198.51.100.0/24`, `203.0.113.0/24`) + /// assert_eq!(Ipv4Addr::new(192, 0, 2, 255).is_global(), false); + /// assert_eq!(Ipv4Addr::new(198, 51, 100, 65).is_global(), false); + /// assert_eq!(Ipv4Addr::new(203, 0, 113, 6).is_global(), false); + /// + /// // Addresses reserved for benchmarking (`198.18.0.0/15`) + /// assert_eq!(Ipv4Addr::new(198, 18, 0, 0).is_global(), false); + /// + /// // Reserved addresses (`240.0.0.0/4`) + /// assert_eq!(Ipv4Addr::new(250, 10, 20, 30).is_global(), false); + /// + /// // The broadcast address (`255.255.255.255`) + /// assert_eq!(Ipv4Addr::BROADCAST.is_global(), false); + /// + /// // For a complete overview see the IANA IPv4 Special-Purpose Address Registry. + /// ``` + #[rustc_const_unstable(feature = "const_ipv4", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] + #[must_use] + #[inline] + pub const fn is_global(&self) -> bool { + !(self.octets()[0] == 0 // "This network" + || self.is_private() + || self.is_shared() + || self.is_loopback() + || self.is_link_local() + // addresses reserved for future protocols (`192.0.0.0/24`) + ||(self.octets()[0] == 192 && self.octets()[1] == 0 && self.octets()[2] == 0) + || self.is_documentation() + || self.is_benchmarking() + || self.is_reserved() + || self.is_broadcast()) + } + + /// Returns [`true`] if this address is part of the Shared Address Space defined in + /// [IETF RFC 6598] (`100.64.0.0/10`). + /// + /// [IETF RFC 6598]: https://tools.ietf.org/html/rfc6598 + /// + /// # Examples + /// + /// ``` + /// #![feature(ip)] + /// use std::net::Ipv4Addr; + /// + /// assert_eq!(Ipv4Addr::new(100, 64, 0, 0).is_shared(), true); + /// assert_eq!(Ipv4Addr::new(100, 127, 255, 255).is_shared(), true); + /// assert_eq!(Ipv4Addr::new(100, 128, 0, 0).is_shared(), false); + /// ``` + #[rustc_const_unstable(feature = "const_ipv4", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] + #[must_use] + #[inline] + pub const fn is_shared(&self) -> bool { + self.octets()[0] == 100 && (self.octets()[1] & 0b1100_0000 == 0b0100_0000) + } + + /// Returns [`true`] if this address part of the `198.18.0.0/15` range, which is reserved for + /// network devices benchmarking. This range is defined in [IETF RFC 2544] as `192.18.0.0` + /// through `198.19.255.255` but [errata 423] corrects it to `198.18.0.0/15`. + /// + /// [IETF RFC 2544]: https://tools.ietf.org/html/rfc2544 + /// [errata 423]: https://www.rfc-editor.org/errata/eid423 + /// + /// # Examples + /// + /// ``` + /// #![feature(ip)] + /// use std::net::Ipv4Addr; + /// + /// assert_eq!(Ipv4Addr::new(198, 17, 255, 255).is_benchmarking(), false); + /// assert_eq!(Ipv4Addr::new(198, 18, 0, 0).is_benchmarking(), true); + /// assert_eq!(Ipv4Addr::new(198, 19, 255, 255).is_benchmarking(), true); + /// assert_eq!(Ipv4Addr::new(198, 20, 0, 0).is_benchmarking(), false); + /// ``` + #[rustc_const_unstable(feature = "const_ipv4", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] + #[must_use] + #[inline] + pub const fn is_benchmarking(&self) -> bool { + self.octets()[0] == 198 && (self.octets()[1] & 0xfe) == 18 + } + + /// Returns [`true`] if this address is reserved by IANA for future use. [IETF RFC 1112] + /// defines the block of reserved addresses as `240.0.0.0/4`. This range normally includes the + /// broadcast address `255.255.255.255`, but this implementation explicitly excludes it, since + /// it is obviously not reserved for future use. + /// + /// [IETF RFC 1112]: https://tools.ietf.org/html/rfc1112 + /// + /// # Warning + /// + /// As IANA assigns new addresses, this method will be + /// updated. This may result in non-reserved addresses being + /// treated as reserved in code that relies on an outdated version + /// of this method. + /// + /// # Examples + /// + /// ``` + /// #![feature(ip)] + /// use std::net::Ipv4Addr; + /// + /// assert_eq!(Ipv4Addr::new(240, 0, 0, 0).is_reserved(), true); + /// assert_eq!(Ipv4Addr::new(255, 255, 255, 254).is_reserved(), true); + /// + /// assert_eq!(Ipv4Addr::new(239, 255, 255, 255).is_reserved(), false); + /// // The broadcast address is not considered as reserved for future use by this implementation + /// assert_eq!(Ipv4Addr::new(255, 255, 255, 255).is_reserved(), false); + /// ``` + #[rustc_const_unstable(feature = "const_ipv4", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] + #[must_use] + #[inline] + pub const fn is_reserved(&self) -> bool { + self.octets()[0] & 240 == 240 && !self.is_broadcast() + } + + /// Returns [`true`] if this is a multicast address (`224.0.0.0/4`). + /// + /// Multicast addresses have a most significant octet between `224` and `239`, + /// and is defined by [IETF RFC 5771]. + /// + /// [IETF RFC 5771]: https://tools.ietf.org/html/rfc5771 + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv4Addr; + /// + /// assert_eq!(Ipv4Addr::new(224, 254, 0, 0).is_multicast(), true); + /// assert_eq!(Ipv4Addr::new(236, 168, 10, 65).is_multicast(), true); + /// assert_eq!(Ipv4Addr::new(172, 16, 10, 65).is_multicast(), false); + /// ``` + #[rustc_const_stable(feature = "const_ip_50", since = "1.50.0")] + #[stable(since = "1.7.0", feature = "ip_17")] + #[must_use] + #[inline] + pub const fn is_multicast(&self) -> bool { + self.octets()[0] >= 224 && self.octets()[0] <= 239 + } + + /// Returns [`true`] if this is a broadcast address (`255.255.255.255`). + /// + /// A broadcast address has all octets set to `255` as defined in [IETF RFC 919]. + /// + /// [IETF RFC 919]: https://tools.ietf.org/html/rfc919 + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv4Addr; + /// + /// assert_eq!(Ipv4Addr::new(255, 255, 255, 255).is_broadcast(), true); + /// assert_eq!(Ipv4Addr::new(236, 168, 10, 65).is_broadcast(), false); + /// ``` + #[rustc_const_stable(feature = "const_ip_50", since = "1.50.0")] + #[stable(since = "1.7.0", feature = "ip_17")] + #[must_use] + #[inline] + pub const fn is_broadcast(&self) -> bool { + u32::from_be_bytes(self.octets()) == u32::from_be_bytes(Self::BROADCAST.octets()) + } + + /// Returns [`true`] if this address is in a range designated for documentation. + /// + /// This is defined in [IETF RFC 5737]: + /// + /// - `192.0.2.0/24` (TEST-NET-1) + /// - `198.51.100.0/24` (TEST-NET-2) + /// - `203.0.113.0/24` (TEST-NET-3) + /// + /// [IETF RFC 5737]: https://tools.ietf.org/html/rfc5737 + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv4Addr; + /// + /// assert_eq!(Ipv4Addr::new(192, 0, 2, 255).is_documentation(), true); + /// assert_eq!(Ipv4Addr::new(198, 51, 100, 65).is_documentation(), true); + /// assert_eq!(Ipv4Addr::new(203, 0, 113, 6).is_documentation(), true); + /// assert_eq!(Ipv4Addr::new(193, 34, 17, 19).is_documentation(), false); + /// ``` + #[rustc_const_stable(feature = "const_ip_50", since = "1.50.0")] + #[stable(since = "1.7.0", feature = "ip_17")] + #[must_use] + #[inline] + pub const fn is_documentation(&self) -> bool { + matches!(self.octets(), [192, 0, 2, _] | [198, 51, 100, _] | [203, 0, 113, _]) + } + + /// Converts this address to an [IPv4-compatible] [`IPv6` address]. + /// + /// `a.b.c.d` becomes `::a.b.c.d` + /// + /// Note that IPv4-compatible addresses have been officially deprecated. + /// If you don't explicitly need an IPv4-compatible address for legacy reasons, consider using `to_ipv6_mapped` instead. + /// + /// [IPv4-compatible]: Ipv6Addr#ipv4-compatible-ipv6-addresses + /// [`IPv6` address]: Ipv6Addr + /// + /// # Examples + /// + /// ``` + /// use std::net::{Ipv4Addr, Ipv6Addr}; + /// + /// assert_eq!( + /// Ipv4Addr::new(192, 0, 2, 255).to_ipv6_compatible(), + /// Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0xc000, 0x2ff) + /// ); + /// ``` + #[rustc_const_stable(feature = "const_ip_50", since = "1.50.0")] + #[stable(feature = "rust1", since = "1.0.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn to_ipv6_compatible(&self) -> Ipv6Addr { + let [a, b, c, d] = self.octets(); + Ipv6Addr { octets: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, a, b, c, d] } + } + + /// Converts this address to an [IPv4-mapped] [`IPv6` address]. + /// + /// `a.b.c.d` becomes `::ffff:a.b.c.d` + /// + /// [IPv4-mapped]: Ipv6Addr#ipv4-mapped-ipv6-addresses + /// [`IPv6` address]: Ipv6Addr + /// + /// # Examples + /// + /// ``` + /// use std::net::{Ipv4Addr, Ipv6Addr}; + /// + /// assert_eq!(Ipv4Addr::new(192, 0, 2, 255).to_ipv6_mapped(), + /// Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc000, 0x2ff)); + /// ``` + #[rustc_const_stable(feature = "const_ip_50", since = "1.50.0")] + #[stable(feature = "rust1", since = "1.0.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn to_ipv6_mapped(&self) -> Ipv6Addr { + let [a, b, c, d] = self.octets(); + Ipv6Addr { octets: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF, a, b, c, d] } + } +} + +#[stable(feature = "ip_addr", since = "1.7.0")] +impl fmt::Display for IpAddr { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + IpAddr::V4(ip) => ip.fmt(fmt), + IpAddr::V6(ip) => ip.fmt(fmt), + } + } +} + +#[stable(feature = "ip_addr", since = "1.7.0")] +impl fmt::Debug for IpAddr { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, fmt) + } +} + +#[stable(feature = "ip_from_ip", since = "1.16.0")] +impl From for IpAddr { + /// Copies this address to a new `IpAddr::V4`. + /// + /// # Examples + /// + /// ``` + /// use std::net::{IpAddr, Ipv4Addr}; + /// + /// let addr = Ipv4Addr::new(127, 0, 0, 1); + /// + /// assert_eq!( + /// IpAddr::V4(addr), + /// IpAddr::from(addr) + /// ) + /// ``` + #[inline] + fn from(ipv4: Ipv4Addr) -> IpAddr { + IpAddr::V4(ipv4) + } +} + +#[stable(feature = "ip_from_ip", since = "1.16.0")] +impl From for IpAddr { + /// Copies this address to a new `IpAddr::V6`. + /// + /// # Examples + /// + /// ``` + /// use std::net::{IpAddr, Ipv6Addr}; + /// + /// let addr = Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff); + /// + /// assert_eq!( + /// IpAddr::V6(addr), + /// IpAddr::from(addr) + /// ); + /// ``` + #[inline] + fn from(ipv6: Ipv6Addr) -> IpAddr { + IpAddr::V6(ipv6) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Display for Ipv4Addr { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + let octets = self.octets(); + + // If there are no alignment requirements, write the IP address directly to `f`. + // Otherwise, write it to a local buffer and then use `f.pad`. + if fmt.precision().is_none() && fmt.width().is_none() { + write!(fmt, "{}.{}.{}.{}", octets[0], octets[1], octets[2], octets[3]) + } else { + const LONGEST_IPV4_ADDR: &str = "255.255.255.255"; + + let mut buf = DisplayBuffer::<{ LONGEST_IPV4_ADDR.len() }>::new(); + // Buffer is long enough for the longest possible IPv4 address, so this should never fail. + write!(buf, "{}.{}.{}.{}", octets[0], octets[1], octets[2], octets[3]).unwrap(); + + fmt.pad(buf.as_str()) + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Debug for Ipv4Addr { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, fmt) + } +} + +#[stable(feature = "ip_cmp", since = "1.16.0")] +impl PartialEq for IpAddr { + #[inline] + fn eq(&self, other: &Ipv4Addr) -> bool { + match self { + IpAddr::V4(v4) => v4 == other, + IpAddr::V6(_) => false, + } + } +} + +#[stable(feature = "ip_cmp", since = "1.16.0")] +impl PartialEq for Ipv4Addr { + #[inline] + fn eq(&self, other: &IpAddr) -> bool { + match other { + IpAddr::V4(v4) => self == v4, + IpAddr::V6(_) => false, + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialOrd for Ipv4Addr { + #[inline] + fn partial_cmp(&self, other: &Ipv4Addr) -> Option { + Some(self.cmp(other)) + } +} + +#[stable(feature = "ip_cmp", since = "1.16.0")] +impl PartialOrd for IpAddr { + #[inline] + fn partial_cmp(&self, other: &Ipv4Addr) -> Option { + match self { + IpAddr::V4(v4) => v4.partial_cmp(other), + IpAddr::V6(_) => Some(Ordering::Greater), + } + } +} + +#[stable(feature = "ip_cmp", since = "1.16.0")] +impl PartialOrd for Ipv4Addr { + #[inline] + fn partial_cmp(&self, other: &IpAddr) -> Option { + match other { + IpAddr::V4(v4) => self.partial_cmp(v4), + IpAddr::V6(_) => Some(Ordering::Less), + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Ord for Ipv4Addr { + #[inline] + fn cmp(&self, other: &Ipv4Addr) -> Ordering { + self.octets.cmp(&other.octets) + } +} + +impl IntoInner for Ipv4Addr { + #[inline] + fn into_inner(self) -> c::in_addr { + // `s_addr` is stored as BE on all machines and the array is in BE order. + // So the native endian conversion method is used so that it's never swapped. + c::in_addr { s_addr: u32::from_ne_bytes(self.octets) } + } +} +impl FromInner for Ipv4Addr { + fn from_inner(addr: c::in_addr) -> Ipv4Addr { + Ipv4Addr { octets: addr.s_addr.to_ne_bytes() } + } +} + +#[stable(feature = "ip_u32", since = "1.1.0")] +impl From for u32 { + /// Converts an `Ipv4Addr` into a host byte order `u32`. + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv4Addr; + /// + /// let addr = Ipv4Addr::new(0x12, 0x34, 0x56, 0x78); + /// assert_eq!(0x12345678, u32::from(addr)); + /// ``` + #[inline] + fn from(ip: Ipv4Addr) -> u32 { + u32::from_be_bytes(ip.octets) + } +} + +#[stable(feature = "ip_u32", since = "1.1.0")] +impl From for Ipv4Addr { + /// Converts a host byte order `u32` into an `Ipv4Addr`. + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv4Addr; + /// + /// let addr = Ipv4Addr::from(0x12345678); + /// assert_eq!(Ipv4Addr::new(0x12, 0x34, 0x56, 0x78), addr); + /// ``` + #[inline] + fn from(ip: u32) -> Ipv4Addr { + Ipv4Addr { octets: ip.to_be_bytes() } + } +} + +#[stable(feature = "from_slice_v4", since = "1.9.0")] +impl From<[u8; 4]> for Ipv4Addr { + /// Creates an `Ipv4Addr` from a four element byte array. + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv4Addr; + /// + /// let addr = Ipv4Addr::from([13u8, 12u8, 11u8, 10u8]); + /// assert_eq!(Ipv4Addr::new(13, 12, 11, 10), addr); + /// ``` + #[inline] + fn from(octets: [u8; 4]) -> Ipv4Addr { + Ipv4Addr { octets } + } +} + +#[stable(feature = "ip_from_slice", since = "1.17.0")] +impl From<[u8; 4]> for IpAddr { + /// Creates an `IpAddr::V4` from a four element byte array. + /// + /// # Examples + /// + /// ``` + /// use std::net::{IpAddr, Ipv4Addr}; + /// + /// let addr = IpAddr::from([13u8, 12u8, 11u8, 10u8]); + /// assert_eq!(IpAddr::V4(Ipv4Addr::new(13, 12, 11, 10)), addr); + /// ``` + #[inline] + fn from(octets: [u8; 4]) -> IpAddr { + IpAddr::V4(Ipv4Addr::from(octets)) + } +} + +impl Ipv6Addr { + /// Creates a new IPv6 address from eight 16-bit segments. + /// + /// The result will represent the IP address `a:b:c:d:e:f:g:h`. + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv6Addr; + /// + /// let addr = Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff); + /// ``` + #[rustc_const_stable(feature = "const_ip_32", since = "1.32.0")] + #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + #[inline] + pub const fn new(a: u16, b: u16, c: u16, d: u16, e: u16, f: u16, g: u16, h: u16) -> Ipv6Addr { + let addr16 = [ + a.to_be(), + b.to_be(), + c.to_be(), + d.to_be(), + e.to_be(), + f.to_be(), + g.to_be(), + h.to_be(), + ]; + Ipv6Addr { + // All elements in `addr16` are big endian. + // SAFETY: `[u16; 8]` is always safe to transmute to `[u8; 16]`. + octets: unsafe { transmute::<_, [u8; 16]>(addr16) }, + } + } + + /// An IPv6 address representing localhost: `::1`. + /// + /// This corresponds to constant `IN6ADDR_LOOPBACK_INIT` or `in6addr_loopback` in other + /// languages. + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv6Addr; + /// + /// let addr = Ipv6Addr::LOCALHOST; + /// assert_eq!(addr, Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)); + /// ``` + #[doc(alias = "IN6ADDR_LOOPBACK_INIT")] + #[doc(alias = "in6addr_loopback")] + #[stable(feature = "ip_constructors", since = "1.30.0")] + pub const LOCALHOST: Self = Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1); + + /// An IPv6 address representing the unspecified address: `::` + /// + /// This corresponds to constant `IN6ADDR_ANY_INIT` or `in6addr_any` in other languages. + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv6Addr; + /// + /// let addr = Ipv6Addr::UNSPECIFIED; + /// assert_eq!(addr, Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0)); + /// ``` + #[doc(alias = "IN6ADDR_ANY_INIT")] + #[doc(alias = "in6addr_any")] + #[stable(feature = "ip_constructors", since = "1.30.0")] + pub const UNSPECIFIED: Self = Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0); + + /// Returns the eight 16-bit segments that make up this address. + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv6Addr; + /// + /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).segments(), + /// [0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff]); + /// ``` + #[rustc_const_stable(feature = "const_ip_50", since = "1.50.0")] + #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + #[inline] + pub const fn segments(&self) -> [u16; 8] { + // All elements in `self.octets` must be big endian. + // SAFETY: `[u8; 16]` is always safe to transmute to `[u16; 8]`. + let [a, b, c, d, e, f, g, h] = unsafe { transmute::<_, [u16; 8]>(self.octets) }; + // We want native endian u16 + [ + u16::from_be(a), + u16::from_be(b), + u16::from_be(c), + u16::from_be(d), + u16::from_be(e), + u16::from_be(f), + u16::from_be(g), + u16::from_be(h), + ] + } + + /// Returns [`true`] for the special 'unspecified' address (`::`). + /// + /// This property is defined in [IETF RFC 4291]. + /// + /// [IETF RFC 4291]: https://tools.ietf.org/html/rfc4291 + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv6Addr; + /// + /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).is_unspecified(), false); + /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0).is_unspecified(), true); + /// ``` + #[rustc_const_stable(feature = "const_ip_50", since = "1.50.0")] + #[stable(since = "1.7.0", feature = "ip_17")] + #[must_use] + #[inline] + pub const fn is_unspecified(&self) -> bool { + u128::from_be_bytes(self.octets()) == u128::from_be_bytes(Ipv6Addr::UNSPECIFIED.octets()) + } + + /// Returns [`true`] if this is the [loopback address] (`::1`), + /// as defined in [IETF RFC 4291 section 2.5.3]. + /// + /// Contrary to IPv4, in IPv6 there is only one loopback address. + /// + /// [loopback address]: Ipv6Addr::LOCALHOST + /// [IETF RFC 4291 section 2.5.3]: https://tools.ietf.org/html/rfc4291#section-2.5.3 + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv6Addr; + /// + /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).is_loopback(), false); + /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0x1).is_loopback(), true); + /// ``` + #[rustc_const_stable(feature = "const_ip_50", since = "1.50.0")] + #[stable(since = "1.7.0", feature = "ip_17")] + #[must_use] + #[inline] + pub const fn is_loopback(&self) -> bool { + u128::from_be_bytes(self.octets()) == u128::from_be_bytes(Ipv6Addr::LOCALHOST.octets()) + } + + /// Returns [`true`] if the address appears to be globally reachable + /// as specified by the [IANA IPv6 Special-Purpose Address Registry]. + /// Whether or not an address is practically reachable will depend on your network configuration. + /// + /// Most IPv6 addresses are globally reachable; + /// unless they are specifically defined as *not* globally reachable. + /// + /// Non-exhaustive list of notable addresses that are not globally reachable: + /// - The [unspecified address] ([`is_unspecified`](Ipv6Addr::is_unspecified)) + /// - The [loopback address] ([`is_loopback`](Ipv6Addr::is_loopback)) + /// - IPv4-mapped addresses + /// - Addresses reserved for benchmarking + /// - Addresses reserved for documentation ([`is_documentation`](Ipv6Addr::is_documentation)) + /// - Unique local addresses ([`is_unique_local`](Ipv6Addr::is_unique_local)) + /// - Unicast addresses with link-local scope ([`is_unicast_link_local`](Ipv6Addr::is_unicast_link_local)) + /// + /// For the complete overview of which addresses are globally reachable, see the table at the [IANA IPv6 Special-Purpose Address Registry]. + /// + /// Note that an address having global scope is not the same as being globally reachable, + /// and there is no direct relation between the two concepts: There exist addresses with global scope + /// that are not globally reachable (for example unique local addresses), + /// and addresses that are globally reachable without having global scope + /// (multicast addresses with non-global scope). + /// + /// [IANA IPv6 Special-Purpose Address Registry]: https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml + /// [unspecified address]: Ipv6Addr::UNSPECIFIED + /// [loopback address]: Ipv6Addr::LOCALHOST + /// + /// # Examples + /// + /// ``` + /// #![feature(ip)] + /// + /// use std::net::Ipv6Addr; + /// + /// // Most IPv6 addresses are globally reachable: + /// assert_eq!(Ipv6Addr::new(0x26, 0, 0x1c9, 0, 0, 0xafc8, 0x10, 0x1).is_global(), true); + /// + /// // However some addresses have been assigned a special meaning + /// // that makes them not globally reachable. Some examples are: + /// + /// // The unspecified address (`::`) + /// assert_eq!(Ipv6Addr::UNSPECIFIED.is_global(), false); + /// + /// // The loopback address (`::1`) + /// assert_eq!(Ipv6Addr::LOCALHOST.is_global(), false); + /// + /// // IPv4-mapped addresses (`::ffff:0:0/96`) + /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).is_global(), false); + /// + /// // Addresses reserved for benchmarking (`2001:2::/48`) + /// assert_eq!(Ipv6Addr::new(0x2001, 2, 0, 0, 0, 0, 0, 1,).is_global(), false); + /// + /// // Addresses reserved for documentation (`2001:db8::/32`) + /// assert_eq!(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1).is_global(), false); + /// + /// // Unique local addresses (`fc00::/7`) + /// assert_eq!(Ipv6Addr::new(0xfc02, 0, 0, 0, 0, 0, 0, 1).is_global(), false); + /// + /// // Unicast addresses with link-local scope (`fe80::/10`) + /// assert_eq!(Ipv6Addr::new(0xfe81, 0, 0, 0, 0, 0, 0, 1).is_global(), false); + /// + /// // For a complete overview see the IANA IPv6 Special-Purpose Address Registry. + /// ``` + #[rustc_const_unstable(feature = "const_ipv6", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] + #[must_use] + #[inline] + pub const fn is_global(&self) -> bool { + !(self.is_unspecified() + || self.is_loopback() + // IPv4-mapped Address (`::ffff:0:0/96`) + || matches!(self.segments(), [0, 0, 0, 0, 0, 0xffff, _, _]) + // IPv4-IPv6 Translat. (`64:ff9b:1::/48`) + || matches!(self.segments(), [0x64, 0xff9b, 1, _, _, _, _, _]) + // Discard-Only Address Block (`100::/64`) + || matches!(self.segments(), [0x100, 0, 0, 0, _, _, _, _]) + // IETF Protocol Assignments (`2001::/23`) + || (matches!(self.segments(), [0x2001, b, _, _, _, _, _, _] if b < 0x200) + && !( + // Port Control Protocol Anycast (`2001:1::1`) + u128::from_be_bytes(self.octets()) == 0x2001_0001_0000_0000_0000_0000_0000_0001 + // Traversal Using Relays around NAT Anycast (`2001:1::2`) + || u128::from_be_bytes(self.octets()) == 0x2001_0001_0000_0000_0000_0000_0000_0002 + // AMT (`2001:3::/32`) + || matches!(self.segments(), [0x2001, 3, _, _, _, _, _, _]) + // AS112-v6 (`2001:4:112::/48`) + || matches!(self.segments(), [0x2001, 4, 0x112, _, _, _, _, _]) + // ORCHIDv2 (`2001:20::/28`) + || matches!(self.segments(), [0x2001, b, _, _, _, _, _, _] if b >= 0x20 && b <= 0x2F) + )) + || self.is_documentation() + || self.is_unique_local() + || self.is_unicast_link_local()) + } + + /// Returns [`true`] if this is a unique local address (`fc00::/7`). + /// + /// This property is defined in [IETF RFC 4193]. + /// + /// [IETF RFC 4193]: https://tools.ietf.org/html/rfc4193 + /// + /// # Examples + /// + /// ``` + /// #![feature(ip)] + /// + /// use std::net::Ipv6Addr; + /// + /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).is_unique_local(), false); + /// assert_eq!(Ipv6Addr::new(0xfc02, 0, 0, 0, 0, 0, 0, 0).is_unique_local(), true); + /// ``` + #[rustc_const_unstable(feature = "const_ipv6", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] + #[must_use] + #[inline] + pub const fn is_unique_local(&self) -> bool { + (self.segments()[0] & 0xfe00) == 0xfc00 + } + + /// Returns [`true`] if this is a unicast address, as defined by [IETF RFC 4291]. + /// Any address that is not a [multicast address] (`ff00::/8`) is unicast. + /// + /// [IETF RFC 4291]: https://tools.ietf.org/html/rfc4291 + /// [multicast address]: Ipv6Addr::is_multicast + /// + /// # Examples + /// + /// ``` + /// #![feature(ip)] + /// + /// use std::net::Ipv6Addr; + /// + /// // The unspecified and loopback addresses are unicast. + /// assert_eq!(Ipv6Addr::UNSPECIFIED.is_unicast(), true); + /// assert_eq!(Ipv6Addr::LOCALHOST.is_unicast(), true); + /// + /// // Any address that is not a multicast address (`ff00::/8`) is unicast. + /// assert_eq!(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 0).is_unicast(), true); + /// assert_eq!(Ipv6Addr::new(0xff00, 0, 0, 0, 0, 0, 0, 0).is_unicast(), false); + /// ``` + #[rustc_const_unstable(feature = "const_ipv6", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] + #[must_use] + #[inline] + pub const fn is_unicast(&self) -> bool { + !self.is_multicast() + } + + /// Returns `true` if the address is a unicast address with link-local scope, + /// as defined in [RFC 4291]. + /// + /// A unicast address has link-local scope if it has the prefix `fe80::/10`, as per [RFC 4291 section 2.4]. + /// Note that this encompasses more addresses than those defined in [RFC 4291 section 2.5.6], + /// which describes "Link-Local IPv6 Unicast Addresses" as having the following stricter format: + /// + /// ```text + /// | 10 bits | 54 bits | 64 bits | + /// +----------+-------------------------+----------------------------+ + /// |1111111010| 0 | interface ID | + /// +----------+-------------------------+----------------------------+ + /// ``` + /// So while currently the only addresses with link-local scope an application will encounter are all in `fe80::/64`, + /// this might change in the future with the publication of new standards. More addresses in `fe80::/10` could be allocated, + /// and those addresses will have link-local scope. + /// + /// Also note that while [RFC 4291 section 2.5.3] mentions about the [loopback address] (`::1`) that "it is treated as having Link-Local scope", + /// this does not mean that the loopback address actually has link-local scope and this method will return `false` on it. + /// + /// [RFC 4291]: https://tools.ietf.org/html/rfc4291 + /// [RFC 4291 section 2.4]: https://tools.ietf.org/html/rfc4291#section-2.4 + /// [RFC 4291 section 2.5.3]: https://tools.ietf.org/html/rfc4291#section-2.5.3 + /// [RFC 4291 section 2.5.6]: https://tools.ietf.org/html/rfc4291#section-2.5.6 + /// [loopback address]: Ipv6Addr::LOCALHOST + /// + /// # Examples + /// + /// ``` + /// #![feature(ip)] + /// + /// use std::net::Ipv6Addr; + /// + /// // The loopback address (`::1`) does not actually have link-local scope. + /// assert_eq!(Ipv6Addr::LOCALHOST.is_unicast_link_local(), false); + /// + /// // Only addresses in `fe80::/10` have link-local scope. + /// assert_eq!(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 0).is_unicast_link_local(), false); + /// assert_eq!(Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 0).is_unicast_link_local(), true); + /// + /// // Addresses outside the stricter `fe80::/64` also have link-local scope. + /// assert_eq!(Ipv6Addr::new(0xfe80, 0, 0, 1, 0, 0, 0, 0).is_unicast_link_local(), true); + /// assert_eq!(Ipv6Addr::new(0xfe81, 0, 0, 0, 0, 0, 0, 0).is_unicast_link_local(), true); + /// ``` + #[rustc_const_unstable(feature = "const_ipv6", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] + #[must_use] + #[inline] + pub const fn is_unicast_link_local(&self) -> bool { + (self.segments()[0] & 0xffc0) == 0xfe80 + } + + /// Returns [`true`] if this is an address reserved for documentation + /// (`2001:db8::/32`). + /// + /// This property is defined in [IETF RFC 3849]. + /// + /// [IETF RFC 3849]: https://tools.ietf.org/html/rfc3849 + /// + /// # Examples + /// + /// ``` + /// #![feature(ip)] + /// + /// use std::net::Ipv6Addr; + /// + /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).is_documentation(), false); + /// assert_eq!(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 0).is_documentation(), true); + /// ``` + #[rustc_const_unstable(feature = "const_ipv6", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] + #[must_use] + #[inline] + pub const fn is_documentation(&self) -> bool { + (self.segments()[0] == 0x2001) && (self.segments()[1] == 0xdb8) + } + + /// Returns [`true`] if this is an address reserved for benchmarking (`2001:2::/48`). + /// + /// This property is defined in [IETF RFC 5180], where it is mistakenly specified as covering the range `2001:0200::/48`. + /// This is corrected in [IETF RFC Errata 1752] to `2001:0002::/48`. + /// + /// [IETF RFC 5180]: https://tools.ietf.org/html/rfc5180 + /// [IETF RFC Errata 1752]: https://www.rfc-editor.org/errata_search.php?eid=1752 + /// + /// ``` + /// #![feature(ip)] + /// + /// use std::net::Ipv6Addr; + /// + /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc613, 0x0).is_benchmarking(), false); + /// assert_eq!(Ipv6Addr::new(0x2001, 0x2, 0, 0, 0, 0, 0, 0).is_benchmarking(), true); + /// ``` + #[unstable(feature = "ip", issue = "27709")] + #[must_use] + #[inline] + pub const fn is_benchmarking(&self) -> bool { + (self.segments()[0] == 0x2001) && (self.segments()[1] == 0x2) && (self.segments()[2] == 0) + } + + /// Returns [`true`] if the address is a globally routable unicast address. + /// + /// The following return false: + /// + /// - the loopback address + /// - the link-local addresses + /// - unique local addresses + /// - the unspecified address + /// - the address range reserved for documentation + /// + /// This method returns [`true`] for site-local addresses as per [RFC 4291 section 2.5.7] + /// + /// ```no_rust + /// The special behavior of [the site-local unicast] prefix defined in [RFC3513] must no longer + /// be supported in new implementations (i.e., new implementations must treat this prefix as + /// Global Unicast). + /// ``` + /// + /// [RFC 4291 section 2.5.7]: https://tools.ietf.org/html/rfc4291#section-2.5.7 + /// + /// # Examples + /// + /// ``` + /// #![feature(ip)] + /// + /// use std::net::Ipv6Addr; + /// + /// assert_eq!(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 0).is_unicast_global(), false); + /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).is_unicast_global(), true); + /// ``` + #[rustc_const_unstable(feature = "const_ipv6", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] + #[must_use] + #[inline] + pub const fn is_unicast_global(&self) -> bool { + self.is_unicast() + && !self.is_loopback() + && !self.is_unicast_link_local() + && !self.is_unique_local() + && !self.is_unspecified() + && !self.is_documentation() + && !self.is_benchmarking() + } + + /// Returns the address's multicast scope if the address is multicast. + /// + /// # Examples + /// + /// ``` + /// #![feature(ip)] + /// + /// use std::net::{Ipv6Addr, Ipv6MulticastScope}; + /// + /// assert_eq!( + /// Ipv6Addr::new(0xff0e, 0, 0, 0, 0, 0, 0, 0).multicast_scope(), + /// Some(Ipv6MulticastScope::Global) + /// ); + /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).multicast_scope(), None); + /// ``` + #[rustc_const_unstable(feature = "const_ipv6", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] + #[must_use] + #[inline] + pub const fn multicast_scope(&self) -> Option { + if self.is_multicast() { + match self.segments()[0] & 0x000f { + 1 => Some(Ipv6MulticastScope::InterfaceLocal), + 2 => Some(Ipv6MulticastScope::LinkLocal), + 3 => Some(Ipv6MulticastScope::RealmLocal), + 4 => Some(Ipv6MulticastScope::AdminLocal), + 5 => Some(Ipv6MulticastScope::SiteLocal), + 8 => Some(Ipv6MulticastScope::OrganizationLocal), + 14 => Some(Ipv6MulticastScope::Global), + _ => None, + } + } else { + None + } + } + + /// Returns [`true`] if this is a multicast address (`ff00::/8`). + /// + /// This property is defined by [IETF RFC 4291]. + /// + /// [IETF RFC 4291]: https://tools.ietf.org/html/rfc4291 + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv6Addr; + /// + /// assert_eq!(Ipv6Addr::new(0xff00, 0, 0, 0, 0, 0, 0, 0).is_multicast(), true); + /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).is_multicast(), false); + /// ``` + #[rustc_const_stable(feature = "const_ip_50", since = "1.50.0")] + #[stable(since = "1.7.0", feature = "ip_17")] + #[must_use] + #[inline] + pub const fn is_multicast(&self) -> bool { + (self.segments()[0] & 0xff00) == 0xff00 + } + + /// Converts this address to an [`IPv4` address] if it's an [IPv4-mapped] address, + /// as defined in [IETF RFC 4291 section 2.5.5.2], otherwise returns [`None`]. + /// + /// `::ffff:a.b.c.d` becomes `a.b.c.d`. + /// All addresses *not* starting with `::ffff` will return `None`. + /// + /// [`IPv4` address]: Ipv4Addr + /// [IPv4-mapped]: Ipv6Addr + /// [IETF RFC 4291 section 2.5.5.2]: https://tools.ietf.org/html/rfc4291#section-2.5.5.2 + /// + /// # Examples + /// + /// ``` + /// use std::net::{Ipv4Addr, Ipv6Addr}; + /// + /// assert_eq!(Ipv6Addr::new(0xff00, 0, 0, 0, 0, 0, 0, 0).to_ipv4_mapped(), None); + /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).to_ipv4_mapped(), + /// Some(Ipv4Addr::new(192, 10, 2, 255))); + /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1).to_ipv4_mapped(), None); + /// ``` + #[rustc_const_unstable(feature = "const_ipv6", issue = "76205")] + #[stable(feature = "ipv6_to_ipv4_mapped", since = "1.63.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn to_ipv4_mapped(&self) -> Option { + match self.octets() { + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, a, b, c, d] => { + Some(Ipv4Addr::new(a, b, c, d)) + } + _ => None, + } + } + + /// Converts this address to an [`IPv4` address] if it is either + /// an [IPv4-compatible] address as defined in [IETF RFC 4291 section 2.5.5.1], + /// or an [IPv4-mapped] address as defined in [IETF RFC 4291 section 2.5.5.2], + /// otherwise returns [`None`]. + /// + /// Note that this will return an [`IPv4` address] for the IPv6 loopback address `::1`. Use + /// [`Ipv6Addr::to_ipv4_mapped`] to avoid this. + /// + /// `::a.b.c.d` and `::ffff:a.b.c.d` become `a.b.c.d`. `::1` becomes `0.0.0.1`. + /// All addresses *not* starting with either all zeroes or `::ffff` will return `None`. + /// + /// [`IPv4` address]: Ipv4Addr + /// [IPv4-compatible]: Ipv6Addr#ipv4-compatible-ipv6-addresses + /// [IPv4-mapped]: Ipv6Addr#ipv4-mapped-ipv6-addresses + /// [IETF RFC 4291 section 2.5.5.1]: https://tools.ietf.org/html/rfc4291#section-2.5.5.1 + /// [IETF RFC 4291 section 2.5.5.2]: https://tools.ietf.org/html/rfc4291#section-2.5.5.2 + /// + /// # Examples + /// + /// ``` + /// use std::net::{Ipv4Addr, Ipv6Addr}; + /// + /// assert_eq!(Ipv6Addr::new(0xff00, 0, 0, 0, 0, 0, 0, 0).to_ipv4(), None); + /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).to_ipv4(), + /// Some(Ipv4Addr::new(192, 10, 2, 255))); + /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1).to_ipv4(), + /// Some(Ipv4Addr::new(0, 0, 0, 1))); + /// ``` + #[rustc_const_stable(feature = "const_ip_50", since = "1.50.0")] + #[stable(feature = "rust1", since = "1.0.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn to_ipv4(&self) -> Option { + if let [0, 0, 0, 0, 0, 0 | 0xffff, ab, cd] = self.segments() { + let [a, b] = ab.to_be_bytes(); + let [c, d] = cd.to_be_bytes(); + Some(Ipv4Addr::new(a, b, c, d)) + } else { + None + } + } + + /// Converts this address to an `IpAddr::V4` if it is an IPv4-mapped addresses, otherwise it + /// returns self wrapped in an `IpAddr::V6`. + /// + /// # Examples + /// + /// ``` + /// #![feature(ip)] + /// use std::net::Ipv6Addr; + /// + /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0x7f00, 0x1).is_loopback(), false); + /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0x7f00, 0x1).to_canonical().is_loopback(), true); + /// ``` + #[rustc_const_unstable(feature = "const_ipv6", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] + pub const fn to_canonical(&self) -> IpAddr { + if let Some(mapped) = self.to_ipv4_mapped() { + return IpAddr::V4(mapped); + } + IpAddr::V6(*self) + } + + /// Returns the sixteen eight-bit integers the IPv6 address consists of. + /// + /// ``` + /// use std::net::Ipv6Addr; + /// + /// assert_eq!(Ipv6Addr::new(0xff00, 0, 0, 0, 0, 0, 0, 0).octets(), + /// [255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + /// ``` + #[rustc_const_stable(feature = "const_ip_32", since = "1.32.0")] + #[stable(feature = "ipv6_to_octets", since = "1.12.0")] + #[must_use] + #[inline] + pub const fn octets(&self) -> [u8; 16] { + self.octets + } +} + +/// Write an Ipv6Addr, conforming to the canonical style described by +/// [RFC 5952](https://tools.ietf.org/html/rfc5952). +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Display for Ipv6Addr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // If there are no alignment requirements, write the IP address directly to `f`. + // Otherwise, write it to a local buffer and then use `f.pad`. + if f.precision().is_none() && f.width().is_none() { + let segments = self.segments(); + + // Special case for :: and ::1; otherwise they get written with the + // IPv4 formatter + if self.is_unspecified() { + f.write_str("::") + } else if self.is_loopback() { + f.write_str("::1") + } else if let Some(ipv4) = self.to_ipv4() { + match segments[5] { + // IPv4 Compatible address + 0 => write!(f, "::{}", ipv4), + // IPv4 Mapped address + 0xffff => write!(f, "::ffff:{}", ipv4), + _ => unreachable!(), + } + } else { + #[derive(Copy, Clone, Default)] + struct Span { + start: usize, + len: usize, + } + + // Find the inner 0 span + let zeroes = { + let mut longest = Span::default(); + let mut current = Span::default(); + + for (i, &segment) in segments.iter().enumerate() { + if segment == 0 { + if current.len == 0 { + current.start = i; + } + + current.len += 1; + + if current.len > longest.len { + longest = current; + } + } else { + current = Span::default(); + } + } + + longest + }; + + /// Write a colon-separated part of the address + #[inline] + fn fmt_subslice(f: &mut fmt::Formatter<'_>, chunk: &[u16]) -> fmt::Result { + if let Some((first, tail)) = chunk.split_first() { + write!(f, "{:x}", first)?; + for segment in tail { + f.write_char(':')?; + write!(f, "{:x}", segment)?; + } + } + Ok(()) + } + + if zeroes.len > 1 { + fmt_subslice(f, &segments[..zeroes.start])?; + f.write_str("::")?; + fmt_subslice(f, &segments[zeroes.start + zeroes.len..]) + } else { + fmt_subslice(f, &segments) + } + } + } else { + const LONGEST_IPV6_ADDR: &str = "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"; + + let mut buf = DisplayBuffer::<{ LONGEST_IPV6_ADDR.len() }>::new(); + // Buffer is long enough for the longest possible IPv6 address, so this should never fail. + write!(buf, "{}", self).unwrap(); + + f.pad(buf.as_str()) + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Debug for Ipv6Addr { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, fmt) + } +} + +#[stable(feature = "ip_cmp", since = "1.16.0")] +impl PartialEq for Ipv6Addr { + #[inline] + fn eq(&self, other: &IpAddr) -> bool { + match other { + IpAddr::V4(_) => false, + IpAddr::V6(v6) => self == v6, + } + } +} + +#[stable(feature = "ip_cmp", since = "1.16.0")] +impl PartialEq for IpAddr { + #[inline] + fn eq(&self, other: &Ipv6Addr) -> bool { + match self { + IpAddr::V4(_) => false, + IpAddr::V6(v6) => v6 == other, + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialOrd for Ipv6Addr { + #[inline] + fn partial_cmp(&self, other: &Ipv6Addr) -> Option { + Some(self.cmp(other)) + } +} + +#[stable(feature = "ip_cmp", since = "1.16.0")] +impl PartialOrd for IpAddr { + #[inline] + fn partial_cmp(&self, other: &Ipv6Addr) -> Option { + match self { + IpAddr::V4(_) => Some(Ordering::Less), + IpAddr::V6(v6) => v6.partial_cmp(other), + } + } +} + +#[stable(feature = "ip_cmp", since = "1.16.0")] +impl PartialOrd for Ipv6Addr { + #[inline] + fn partial_cmp(&self, other: &IpAddr) -> Option { + match other { + IpAddr::V4(_) => Some(Ordering::Greater), + IpAddr::V6(v6) => self.partial_cmp(v6), + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Ord for Ipv6Addr { + #[inline] + fn cmp(&self, other: &Ipv6Addr) -> Ordering { + self.segments().cmp(&other.segments()) + } +} + +impl IntoInner for Ipv6Addr { + fn into_inner(self) -> c::in6_addr { + c::in6_addr { s6_addr: self.octets } + } +} +impl FromInner for Ipv6Addr { + #[inline] + fn from_inner(addr: c::in6_addr) -> Ipv6Addr { + Ipv6Addr { octets: addr.s6_addr } + } +} + +#[stable(feature = "i128", since = "1.26.0")] +impl From for u128 { + /// Convert an `Ipv6Addr` into a host byte order `u128`. + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv6Addr; + /// + /// let addr = Ipv6Addr::new( + /// 0x1020, 0x3040, 0x5060, 0x7080, + /// 0x90A0, 0xB0C0, 0xD0E0, 0xF00D, + /// ); + /// assert_eq!(0x102030405060708090A0B0C0D0E0F00D_u128, u128::from(addr)); + /// ``` + #[inline] + fn from(ip: Ipv6Addr) -> u128 { + u128::from_be_bytes(ip.octets) + } +} +#[stable(feature = "i128", since = "1.26.0")] +impl From for Ipv6Addr { + /// Convert a host byte order `u128` into an `Ipv6Addr`. + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv6Addr; + /// + /// let addr = Ipv6Addr::from(0x102030405060708090A0B0C0D0E0F00D_u128); + /// assert_eq!( + /// Ipv6Addr::new( + /// 0x1020, 0x3040, 0x5060, 0x7080, + /// 0x90A0, 0xB0C0, 0xD0E0, 0xF00D, + /// ), + /// addr); + /// ``` + #[inline] + fn from(ip: u128) -> Ipv6Addr { + Ipv6Addr::from(ip.to_be_bytes()) + } +} + +#[stable(feature = "ipv6_from_octets", since = "1.9.0")] +impl From<[u8; 16]> for Ipv6Addr { + /// Creates an `Ipv6Addr` from a sixteen element byte array. + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv6Addr; + /// + /// let addr = Ipv6Addr::from([ + /// 25u8, 24u8, 23u8, 22u8, 21u8, 20u8, 19u8, 18u8, + /// 17u8, 16u8, 15u8, 14u8, 13u8, 12u8, 11u8, 10u8, + /// ]); + /// assert_eq!( + /// Ipv6Addr::new( + /// 0x1918, 0x1716, + /// 0x1514, 0x1312, + /// 0x1110, 0x0f0e, + /// 0x0d0c, 0x0b0a + /// ), + /// addr + /// ); + /// ``` + #[inline] + fn from(octets: [u8; 16]) -> Ipv6Addr { + Ipv6Addr { octets } + } +} + +#[stable(feature = "ipv6_from_segments", since = "1.16.0")] +impl From<[u16; 8]> for Ipv6Addr { + /// Creates an `Ipv6Addr` from an eight element 16-bit array. + /// + /// # Examples + /// + /// ``` + /// use std::net::Ipv6Addr; + /// + /// let addr = Ipv6Addr::from([ + /// 525u16, 524u16, 523u16, 522u16, + /// 521u16, 520u16, 519u16, 518u16, + /// ]); + /// assert_eq!( + /// Ipv6Addr::new( + /// 0x20d, 0x20c, + /// 0x20b, 0x20a, + /// 0x209, 0x208, + /// 0x207, 0x206 + /// ), + /// addr + /// ); + /// ``` + #[inline] + fn from(segments: [u16; 8]) -> Ipv6Addr { + let [a, b, c, d, e, f, g, h] = segments; + Ipv6Addr::new(a, b, c, d, e, f, g, h) + } +} + +#[stable(feature = "ip_from_slice", since = "1.17.0")] +impl From<[u8; 16]> for IpAddr { + /// Creates an `IpAddr::V6` from a sixteen element byte array. + /// + /// # Examples + /// + /// ``` + /// use std::net::{IpAddr, Ipv6Addr}; + /// + /// let addr = IpAddr::from([ + /// 25u8, 24u8, 23u8, 22u8, 21u8, 20u8, 19u8, 18u8, + /// 17u8, 16u8, 15u8, 14u8, 13u8, 12u8, 11u8, 10u8, + /// ]); + /// assert_eq!( + /// IpAddr::V6(Ipv6Addr::new( + /// 0x1918, 0x1716, + /// 0x1514, 0x1312, + /// 0x1110, 0x0f0e, + /// 0x0d0c, 0x0b0a + /// )), + /// addr + /// ); + /// ``` + #[inline] + fn from(octets: [u8; 16]) -> IpAddr { + IpAddr::V6(Ipv6Addr::from(octets)) + } +} + +#[stable(feature = "ip_from_slice", since = "1.17.0")] +impl From<[u16; 8]> for IpAddr { + /// Creates an `IpAddr::V6` from an eight element 16-bit array. + /// + /// # Examples + /// + /// ``` + /// use std::net::{IpAddr, Ipv6Addr}; + /// + /// let addr = IpAddr::from([ + /// 525u16, 524u16, 523u16, 522u16, + /// 521u16, 520u16, 519u16, 518u16, + /// ]); + /// assert_eq!( + /// IpAddr::V6(Ipv6Addr::new( + /// 0x20d, 0x20c, + /// 0x20b, 0x20a, + /// 0x209, 0x208, + /// 0x207, 0x206 + /// )), + /// addr + /// ); + /// ``` + #[inline] + fn from(segments: [u16; 8]) -> IpAddr { + IpAddr::V6(Ipv6Addr::from(segments)) + } +} diff --git a/crux-mir/lib/std/src/net/ip_addr/tests.rs b/crux-mir/lib/std/src/net/ip_addr/tests.rs new file mode 100644 index 000000000..0eb59d45d --- /dev/null +++ b/crux-mir/lib/std/src/net/ip_addr/tests.rs @@ -0,0 +1,1039 @@ +use crate::net::test::{sa4, sa6, tsa}; +use crate::net::*; +use crate::str::FromStr; + +#[test] +fn test_from_str_ipv4() { + assert_eq!(Ok(Ipv4Addr::new(127, 0, 0, 1)), "127.0.0.1".parse()); + assert_eq!(Ok(Ipv4Addr::new(255, 255, 255, 255)), "255.255.255.255".parse()); + assert_eq!(Ok(Ipv4Addr::new(0, 0, 0, 0)), "0.0.0.0".parse()); + + // out of range + let none: Option = "256.0.0.1".parse().ok(); + assert_eq!(None, none); + // too short + let none: Option = "255.0.0".parse().ok(); + assert_eq!(None, none); + // too long + let none: Option = "255.0.0.1.2".parse().ok(); + assert_eq!(None, none); + // no number between dots + let none: Option = "255.0..1".parse().ok(); + assert_eq!(None, none); + // octal + let none: Option = "255.0.0.01".parse().ok(); + assert_eq!(None, none); + // octal zero + let none: Option = "255.0.0.00".parse().ok(); + assert_eq!(None, none); + let none: Option = "255.0.00.0".parse().ok(); + assert_eq!(None, none); +} + +#[test] +fn test_from_str_ipv6() { + assert_eq!(Ok(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0)), "0:0:0:0:0:0:0:0".parse()); + assert_eq!(Ok(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)), "0:0:0:0:0:0:0:1".parse()); + + assert_eq!(Ok(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)), "::1".parse()); + assert_eq!(Ok(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0)), "::".parse()); + + assert_eq!(Ok(Ipv6Addr::new(0x2a02, 0x6b8, 0, 0, 0, 0, 0x11, 0x11)), "2a02:6b8::11:11".parse()); + + // too long group + let none: Option = "::00000".parse().ok(); + assert_eq!(None, none); + // too short + let none: Option = "1:2:3:4:5:6:7".parse().ok(); + assert_eq!(None, none); + // too long + let none: Option = "1:2:3:4:5:6:7:8:9".parse().ok(); + assert_eq!(None, none); + // triple colon + let none: Option = "1:2:::6:7:8".parse().ok(); + assert_eq!(None, none); + // two double colons + let none: Option = "1:2::6::8".parse().ok(); + assert_eq!(None, none); + // `::` indicating zero groups of zeros + let none: Option = "1:2:3:4::5:6:7:8".parse().ok(); + assert_eq!(None, none); +} + +#[test] +fn test_from_str_ipv4_in_ipv6() { + assert_eq!(Ok(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 49152, 545)), "::192.0.2.33".parse()); + assert_eq!(Ok(Ipv6Addr::new(0, 0, 0, 0, 0, 0xFFFF, 49152, 545)), "::FFFF:192.0.2.33".parse()); + assert_eq!( + Ok(Ipv6Addr::new(0x64, 0xff9b, 0, 0, 0, 0, 49152, 545)), + "64:ff9b::192.0.2.33".parse() + ); + assert_eq!( + Ok(Ipv6Addr::new(0x2001, 0xdb8, 0x122, 0xc000, 0x2, 0x2100, 49152, 545)), + "2001:db8:122:c000:2:2100:192.0.2.33".parse() + ); + + // colon after v4 + let none: Option = "::127.0.0.1:".parse().ok(); + assert_eq!(None, none); + // not enough groups + let none: Option = "1:2:3:4:5:127.0.0.1".parse().ok(); + assert_eq!(None, none); + // too many groups + let none: Option = "1:2:3:4:5:6:7:127.0.0.1".parse().ok(); + assert_eq!(None, none); +} + +#[test] +fn test_from_str_socket_addr() { + assert_eq!(Ok(sa4(Ipv4Addr::new(77, 88, 21, 11), 80)), "77.88.21.11:80".parse()); + assert_eq!(Ok(SocketAddrV4::new(Ipv4Addr::new(77, 88, 21, 11), 80)), "77.88.21.11:80".parse()); + assert_eq!( + Ok(sa6(Ipv6Addr::new(0x2a02, 0x6b8, 0, 1, 0, 0, 0, 1), 53)), + "[2a02:6b8:0:1::1]:53".parse() + ); + assert_eq!( + Ok(SocketAddrV6::new(Ipv6Addr::new(0x2a02, 0x6b8, 0, 1, 0, 0, 0, 1), 53, 0, 0)), + "[2a02:6b8:0:1::1]:53".parse() + ); + assert_eq!(Ok(sa6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0x7F00, 1), 22)), "[::127.0.0.1]:22".parse()); + assert_eq!( + Ok(SocketAddrV6::new(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0x7F00, 1), 22, 0, 0)), + "[::127.0.0.1]:22".parse() + ); + + // without port + let none: Option = "127.0.0.1".parse().ok(); + assert_eq!(None, none); + // without port + let none: Option = "127.0.0.1:".parse().ok(); + assert_eq!(None, none); + // wrong brackets around v4 + let none: Option = "[127.0.0.1]:22".parse().ok(); + assert_eq!(None, none); + // port out of range + let none: Option = "127.0.0.1:123456".parse().ok(); + assert_eq!(None, none); +} + +#[test] +fn ipv4_addr_to_string() { + assert_eq!(Ipv4Addr::new(127, 0, 0, 1).to_string(), "127.0.0.1"); + // Short address + assert_eq!(Ipv4Addr::new(1, 1, 1, 1).to_string(), "1.1.1.1"); + // Long address + assert_eq!(Ipv4Addr::new(127, 127, 127, 127).to_string(), "127.127.127.127"); + + // Test padding + assert_eq!(format!("{:16}", Ipv4Addr::new(1, 1, 1, 1)), "1.1.1.1 "); + assert_eq!(format!("{:>16}", Ipv4Addr::new(1, 1, 1, 1)), " 1.1.1.1"); +} + +#[test] +fn ipv6_addr_to_string() { + // ipv4-mapped address + let a1 = Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc000, 0x280); + assert_eq!(a1.to_string(), "::ffff:192.0.2.128"); + + // ipv4-compatible address + let a1 = Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0xc000, 0x280); + assert_eq!(a1.to_string(), "::192.0.2.128"); + + // v6 address with no zero segments + assert_eq!(Ipv6Addr::new(8, 9, 10, 11, 12, 13, 14, 15).to_string(), "8:9:a:b:c:d:e:f"); + + // longest possible IPv6 length + assert_eq!( + Ipv6Addr::new(0x1111, 0x2222, 0x3333, 0x4444, 0x5555, 0x6666, 0x7777, 0x8888).to_string(), + "1111:2222:3333:4444:5555:6666:7777:8888" + ); + // padding + assert_eq!(format!("{:20}", Ipv6Addr::new(1, 2, 3, 4, 5, 6, 7, 8)), "1:2:3:4:5:6:7:8 "); + assert_eq!(format!("{:>20}", Ipv6Addr::new(1, 2, 3, 4, 5, 6, 7, 8)), " 1:2:3:4:5:6:7:8"); + + // reduce a single run of zeros + assert_eq!( + "ae::ffff:102:304", + Ipv6Addr::new(0xae, 0, 0, 0, 0, 0xffff, 0x0102, 0x0304).to_string() + ); + + // don't reduce just a single zero segment + assert_eq!("1:2:3:4:5:6:0:8", Ipv6Addr::new(1, 2, 3, 4, 5, 6, 0, 8).to_string()); + + // 'any' address + assert_eq!("::", Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0).to_string()); + + // loopback address + assert_eq!("::1", Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1).to_string()); + + // ends in zeros + assert_eq!("1::", Ipv6Addr::new(1, 0, 0, 0, 0, 0, 0, 0).to_string()); + + // two runs of zeros, second one is longer + assert_eq!("1:0:0:4::8", Ipv6Addr::new(1, 0, 0, 4, 0, 0, 0, 8).to_string()); + + // two runs of zeros, equal length + assert_eq!("1::4:5:0:0:8", Ipv6Addr::new(1, 0, 0, 4, 5, 0, 0, 8).to_string()); + + // don't prefix `0x` to each segment in `dbg!`. + assert_eq!("1::4:5:0:0:8", &format!("{:#?}", Ipv6Addr::new(1, 0, 0, 4, 5, 0, 0, 8))); +} + +#[test] +fn ipv4_to_ipv6() { + assert_eq!( + Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0x1234, 0x5678), + Ipv4Addr::new(0x12, 0x34, 0x56, 0x78).to_ipv6_mapped() + ); + assert_eq!( + Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0x1234, 0x5678), + Ipv4Addr::new(0x12, 0x34, 0x56, 0x78).to_ipv6_compatible() + ); +} + +#[test] +fn ipv6_to_ipv4_mapped() { + assert_eq!( + Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0x1234, 0x5678).to_ipv4_mapped(), + Some(Ipv4Addr::new(0x12, 0x34, 0x56, 0x78)) + ); + assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0x1234, 0x5678).to_ipv4_mapped(), None); +} + +#[test] +fn ipv6_to_ipv4() { + assert_eq!( + Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0x1234, 0x5678).to_ipv4(), + Some(Ipv4Addr::new(0x12, 0x34, 0x56, 0x78)) + ); + assert_eq!( + Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0x1234, 0x5678).to_ipv4(), + Some(Ipv4Addr::new(0x12, 0x34, 0x56, 0x78)) + ); + assert_eq!(Ipv6Addr::new(0, 0, 1, 0, 0, 0, 0x1234, 0x5678).to_ipv4(), None); +} + +#[test] +fn ip_properties() { + macro_rules! ip { + ($s:expr) => { + IpAddr::from_str($s).unwrap() + }; + } + + macro_rules! check { + ($s:expr) => { + check!($s, 0); + }; + + ($s:expr, $mask:expr) => {{ + let unspec: u8 = 1 << 0; + let loopback: u8 = 1 << 1; + let global: u8 = 1 << 2; + let multicast: u8 = 1 << 3; + let doc: u8 = 1 << 4; + let benchmarking: u8 = 1 << 5; + + if ($mask & unspec) == unspec { + assert!(ip!($s).is_unspecified()); + } else { + assert!(!ip!($s).is_unspecified()); + } + + if ($mask & loopback) == loopback { + assert!(ip!($s).is_loopback()); + } else { + assert!(!ip!($s).is_loopback()); + } + + if ($mask & global) == global { + assert!(ip!($s).is_global()); + } else { + assert!(!ip!($s).is_global()); + } + + if ($mask & multicast) == multicast { + assert!(ip!($s).is_multicast()); + } else { + assert!(!ip!($s).is_multicast()); + } + + if ($mask & doc) == doc { + assert!(ip!($s).is_documentation()); + } else { + assert!(!ip!($s).is_documentation()); + } + + if ($mask & benchmarking) == benchmarking { + assert!(ip!($s).is_benchmarking()); + } else { + assert!(!ip!($s).is_benchmarking()); + } + }}; + } + + let unspec: u8 = 1 << 0; + let loopback: u8 = 1 << 1; + let global: u8 = 1 << 2; + let multicast: u8 = 1 << 3; + let doc: u8 = 1 << 4; + let benchmarking: u8 = 1 << 5; + + check!("0.0.0.0", unspec); + check!("0.0.0.1"); + check!("0.1.0.0"); + check!("10.9.8.7"); + check!("127.1.2.3", loopback); + check!("172.31.254.253"); + check!("169.254.253.242"); + check!("192.0.2.183", doc); + check!("192.1.2.183", global); + check!("192.168.254.253"); + check!("198.51.100.0", doc); + check!("203.0.113.0", doc); + check!("203.2.113.0", global); + check!("224.0.0.0", global | multicast); + check!("239.255.255.255", global | multicast); + check!("255.255.255.255"); + // make sure benchmarking addresses are not global + check!("198.18.0.0", benchmarking); + check!("198.18.54.2", benchmarking); + check!("198.19.255.255", benchmarking); + // make sure addresses reserved for protocol assignment are not global + check!("192.0.0.0"); + check!("192.0.0.255"); + check!("192.0.0.100"); + // make sure reserved addresses are not global + check!("240.0.0.0"); + check!("251.54.1.76"); + check!("254.255.255.255"); + // make sure shared addresses are not global + check!("100.64.0.0"); + check!("100.127.255.255"); + check!("100.100.100.0"); + + check!("::", unspec); + check!("::1", loopback); + check!("::0.0.0.2", global); + check!("1::", global); + check!("fc00::"); + check!("fdff:ffff::"); + check!("fe80:ffff::"); + check!("febf:ffff::"); + check!("fec0::", global); + check!("ff01::", global | multicast); + check!("ff02::", global | multicast); + check!("ff03::", global | multicast); + check!("ff04::", global | multicast); + check!("ff05::", global | multicast); + check!("ff08::", global | multicast); + check!("ff0e::", global | multicast); + check!("2001:db8:85a3::8a2e:370:7334", doc); + check!("2001:2::ac32:23ff:21", benchmarking); + check!("102:304:506:708:90a:b0c:d0e:f10", global); +} + +#[test] +fn ipv4_properties() { + macro_rules! ip { + ($s:expr) => { + Ipv4Addr::from_str($s).unwrap() + }; + } + + macro_rules! check { + ($s:expr) => { + check!($s, 0); + }; + + ($s:expr, $mask:expr) => {{ + let unspec: u16 = 1 << 0; + let loopback: u16 = 1 << 1; + let private: u16 = 1 << 2; + let link_local: u16 = 1 << 3; + let global: u16 = 1 << 4; + let multicast: u16 = 1 << 5; + let broadcast: u16 = 1 << 6; + let documentation: u16 = 1 << 7; + let benchmarking: u16 = 1 << 8; + let reserved: u16 = 1 << 10; + let shared: u16 = 1 << 11; + + if ($mask & unspec) == unspec { + assert!(ip!($s).is_unspecified()); + } else { + assert!(!ip!($s).is_unspecified()); + } + + if ($mask & loopback) == loopback { + assert!(ip!($s).is_loopback()); + } else { + assert!(!ip!($s).is_loopback()); + } + + if ($mask & private) == private { + assert!(ip!($s).is_private()); + } else { + assert!(!ip!($s).is_private()); + } + + if ($mask & link_local) == link_local { + assert!(ip!($s).is_link_local()); + } else { + assert!(!ip!($s).is_link_local()); + } + + if ($mask & global) == global { + assert!(ip!($s).is_global()); + } else { + assert!(!ip!($s).is_global()); + } + + if ($mask & multicast) == multicast { + assert!(ip!($s).is_multicast()); + } else { + assert!(!ip!($s).is_multicast()); + } + + if ($mask & broadcast) == broadcast { + assert!(ip!($s).is_broadcast()); + } else { + assert!(!ip!($s).is_broadcast()); + } + + if ($mask & documentation) == documentation { + assert!(ip!($s).is_documentation()); + } else { + assert!(!ip!($s).is_documentation()); + } + + if ($mask & benchmarking) == benchmarking { + assert!(ip!($s).is_benchmarking()); + } else { + assert!(!ip!($s).is_benchmarking()); + } + + if ($mask & reserved) == reserved { + assert!(ip!($s).is_reserved()); + } else { + assert!(!ip!($s).is_reserved()); + } + + if ($mask & shared) == shared { + assert!(ip!($s).is_shared()); + } else { + assert!(!ip!($s).is_shared()); + } + }}; + } + + let unspec: u16 = 1 << 0; + let loopback: u16 = 1 << 1; + let private: u16 = 1 << 2; + let link_local: u16 = 1 << 3; + let global: u16 = 1 << 4; + let multicast: u16 = 1 << 5; + let broadcast: u16 = 1 << 6; + let documentation: u16 = 1 << 7; + let benchmarking: u16 = 1 << 8; + let reserved: u16 = 1 << 10; + let shared: u16 = 1 << 11; + + check!("0.0.0.0", unspec); + check!("0.0.0.1"); + check!("0.1.0.0"); + check!("10.9.8.7", private); + check!("127.1.2.3", loopback); + check!("172.31.254.253", private); + check!("169.254.253.242", link_local); + check!("192.0.2.183", documentation); + check!("192.1.2.183", global); + check!("192.168.254.253", private); + check!("198.51.100.0", documentation); + check!("203.0.113.0", documentation); + check!("203.2.113.0", global); + check!("224.0.0.0", global | multicast); + check!("239.255.255.255", global | multicast); + check!("255.255.255.255", broadcast); + check!("198.18.0.0", benchmarking); + check!("198.18.54.2", benchmarking); + check!("198.19.255.255", benchmarking); + check!("192.0.0.0"); + check!("192.0.0.255"); + check!("192.0.0.100"); + check!("240.0.0.0", reserved); + check!("251.54.1.76", reserved); + check!("254.255.255.255", reserved); + check!("100.64.0.0", shared); + check!("100.127.255.255", shared); + check!("100.100.100.0", shared); +} + +#[test] +fn ipv6_properties() { + macro_rules! ip { + ($s:expr) => { + Ipv6Addr::from_str($s).unwrap() + }; + } + + macro_rules! check { + ($s:expr, &[$($octet:expr),*], $mask:expr) => { + assert_eq!($s, ip!($s).to_string()); + let octets = &[$($octet),*]; + assert_eq!(&ip!($s).octets(), octets); + assert_eq!(Ipv6Addr::from(*octets), ip!($s)); + + let unspecified: u32 = 1 << 0; + let loopback: u32 = 1 << 1; + let unique_local: u32 = 1 << 2; + let global: u32 = 1 << 3; + let unicast_link_local: u32 = 1 << 4; + let unicast_global: u32 = 1 << 7; + let documentation: u32 = 1 << 8; + let benchmarking: u32 = 1 << 16; + let multicast_interface_local: u32 = 1 << 9; + let multicast_link_local: u32 = 1 << 10; + let multicast_realm_local: u32 = 1 << 11; + let multicast_admin_local: u32 = 1 << 12; + let multicast_site_local: u32 = 1 << 13; + let multicast_organization_local: u32 = 1 << 14; + let multicast_global: u32 = 1 << 15; + let multicast: u32 = multicast_interface_local + | multicast_admin_local + | multicast_global + | multicast_link_local + | multicast_realm_local + | multicast_site_local + | multicast_organization_local; + + if ($mask & unspecified) == unspecified { + assert!(ip!($s).is_unspecified()); + } else { + assert!(!ip!($s).is_unspecified()); + } + if ($mask & loopback) == loopback { + assert!(ip!($s).is_loopback()); + } else { + assert!(!ip!($s).is_loopback()); + } + if ($mask & unique_local) == unique_local { + assert!(ip!($s).is_unique_local()); + } else { + assert!(!ip!($s).is_unique_local()); + } + if ($mask & global) == global { + assert!(ip!($s).is_global()); + } else { + assert!(!ip!($s).is_global()); + } + if ($mask & unicast_link_local) == unicast_link_local { + assert!(ip!($s).is_unicast_link_local()); + } else { + assert!(!ip!($s).is_unicast_link_local()); + } + if ($mask & unicast_global) == unicast_global { + assert!(ip!($s).is_unicast_global()); + } else { + assert!(!ip!($s).is_unicast_global()); + } + if ($mask & documentation) == documentation { + assert!(ip!($s).is_documentation()); + } else { + assert!(!ip!($s).is_documentation()); + } + if ($mask & benchmarking) == benchmarking { + assert!(ip!($s).is_benchmarking()); + } else { + assert!(!ip!($s).is_benchmarking()); + } + if ($mask & multicast) != 0 { + assert!(ip!($s).multicast_scope().is_some()); + assert!(ip!($s).is_multicast()); + } else { + assert!(ip!($s).multicast_scope().is_none()); + assert!(!ip!($s).is_multicast()); + } + if ($mask & multicast_interface_local) == multicast_interface_local { + assert_eq!(ip!($s).multicast_scope().unwrap(), + Ipv6MulticastScope::InterfaceLocal); + } + if ($mask & multicast_link_local) == multicast_link_local { + assert_eq!(ip!($s).multicast_scope().unwrap(), + Ipv6MulticastScope::LinkLocal); + } + if ($mask & multicast_realm_local) == multicast_realm_local { + assert_eq!(ip!($s).multicast_scope().unwrap(), + Ipv6MulticastScope::RealmLocal); + } + if ($mask & multicast_admin_local) == multicast_admin_local { + assert_eq!(ip!($s).multicast_scope().unwrap(), + Ipv6MulticastScope::AdminLocal); + } + if ($mask & multicast_site_local) == multicast_site_local { + assert_eq!(ip!($s).multicast_scope().unwrap(), + Ipv6MulticastScope::SiteLocal); + } + if ($mask & multicast_organization_local) == multicast_organization_local { + assert_eq!(ip!($s).multicast_scope().unwrap(), + Ipv6MulticastScope::OrganizationLocal); + } + if ($mask & multicast_global) == multicast_global { + assert_eq!(ip!($s).multicast_scope().unwrap(), + Ipv6MulticastScope::Global); + } + } + } + + let unspecified: u32 = 1 << 0; + let loopback: u32 = 1 << 1; + let unique_local: u32 = 1 << 2; + let global: u32 = 1 << 3; + let unicast_link_local: u32 = 1 << 4; + let unicast_global: u32 = 1 << 7; + let documentation: u32 = 1 << 8; + let benchmarking: u32 = 1 << 16; + let multicast_interface_local: u32 = 1 << 9; + let multicast_link_local: u32 = 1 << 10; + let multicast_realm_local: u32 = 1 << 11; + let multicast_admin_local: u32 = 1 << 12; + let multicast_site_local: u32 = 1 << 13; + let multicast_organization_local: u32 = 1 << 14; + let multicast_global: u32 = 1 << 15; + + check!("::", &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], unspecified); + + check!("::1", &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], loopback); + + check!("::0.0.0.2", &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], global | unicast_global); + + check!("1::", &[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], global | unicast_global); + + check!( + "::ffff:127.0.0.1", + &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0x7f, 0, 0, 1], + unicast_global + ); + + check!( + "64:ff9b:1::", + &[0, 0x64, 0xff, 0x9b, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + unicast_global + ); + + check!("100::", &[0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], unicast_global); + + check!("2001::", &[0x20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], unicast_global); + + check!( + "2001:1::1", + &[0x20, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], + global | unicast_global + ); + + check!( + "2001:1::2", + &[0x20, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], + global | unicast_global + ); + + check!( + "2001:3::", + &[0x20, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + global | unicast_global + ); + + check!( + "2001:4:112::", + &[0x20, 1, 0, 4, 1, 0x12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + global | unicast_global + ); + + check!( + "2001:20::", + &[0x20, 1, 0, 0x20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + global | unicast_global + ); + + check!("2001:30::", &[0x20, 1, 0, 0x30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], unicast_global); + + check!( + "2001:200::", + &[0x20, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + global | unicast_global + ); + + check!("fc00::", &[0xfc, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], unique_local); + + check!( + "fdff:ffff::", + &[0xfd, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + unique_local + ); + + check!( + "fe80:ffff::", + &[0xfe, 0x80, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + unicast_link_local + ); + + check!("fe80::", &[0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], unicast_link_local); + + check!( + "febf:ffff::", + &[0xfe, 0xbf, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + unicast_link_local + ); + + check!("febf::", &[0xfe, 0xbf, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], unicast_link_local); + + check!( + "febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + &[ + 0xfe, 0xbf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff + ], + unicast_link_local + ); + + check!( + "fe80::ffff:ffff:ffff:ffff", + &[ + 0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff + ], + unicast_link_local + ); + + check!( + "fe80:0:0:1::", + &[0xfe, 0x80, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], + unicast_link_local + ); + + check!( + "fec0::", + &[0xfe, 0xc0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + unicast_global | global + ); + + check!( + "ff01::", + &[0xff, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + multicast_interface_local | global + ); + + check!( + "ff02::", + &[0xff, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + multicast_link_local | global + ); + + check!( + "ff03::", + &[0xff, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + multicast_realm_local | global + ); + + check!( + "ff04::", + &[0xff, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + multicast_admin_local | global + ); + + check!( + "ff05::", + &[0xff, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + multicast_site_local | global + ); + + check!( + "ff08::", + &[0xff, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + multicast_organization_local | global + ); + + check!( + "ff0e::", + &[0xff, 0xe, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + multicast_global | global + ); + + check!( + "2001:db8:85a3::8a2e:370:7334", + &[0x20, 1, 0xd, 0xb8, 0x85, 0xa3, 0, 0, 0, 0, 0x8a, 0x2e, 3, 0x70, 0x73, 0x34], + documentation + ); + + check!( + "2001:2::ac32:23ff:21", + &[0x20, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0xac, 0x32, 0x23, 0xff, 0, 0x21], + benchmarking + ); + + check!( + "102:304:506:708:90a:b0c:d0e:f10", + &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + global | unicast_global + ); +} + +#[test] +fn to_socket_addr_socketaddr() { + let a = sa4(Ipv4Addr::new(77, 88, 21, 11), 12345); + assert_eq!(Ok(vec![a]), tsa(a)); +} + +#[test] +fn test_ipv4_to_int() { + let a = Ipv4Addr::new(0x11, 0x22, 0x33, 0x44); + assert_eq!(u32::from(a), 0x11223344); +} + +#[test] +fn test_int_to_ipv4() { + let a = Ipv4Addr::new(0x11, 0x22, 0x33, 0x44); + assert_eq!(Ipv4Addr::from(0x11223344), a); +} + +#[test] +fn test_ipv6_to_int() { + let a = Ipv6Addr::new(0x1122, 0x3344, 0x5566, 0x7788, 0x99aa, 0xbbcc, 0xddee, 0xff11); + assert_eq!(u128::from(a), 0x112233445566778899aabbccddeeff11u128); +} + +#[test] +fn test_int_to_ipv6() { + let a = Ipv6Addr::new(0x1122, 0x3344, 0x5566, 0x7788, 0x99aa, 0xbbcc, 0xddee, 0xff11); + assert_eq!(Ipv6Addr::from(0x112233445566778899aabbccddeeff11u128), a); +} + +#[test] +fn ipv4_from_constructors() { + assert_eq!(Ipv4Addr::LOCALHOST, Ipv4Addr::new(127, 0, 0, 1)); + assert!(Ipv4Addr::LOCALHOST.is_loopback()); + assert_eq!(Ipv4Addr::UNSPECIFIED, Ipv4Addr::new(0, 0, 0, 0)); + assert!(Ipv4Addr::UNSPECIFIED.is_unspecified()); + assert_eq!(Ipv4Addr::BROADCAST, Ipv4Addr::new(255, 255, 255, 255)); + assert!(Ipv4Addr::BROADCAST.is_broadcast()); +} + +#[test] +fn ipv6_from_constructors() { + assert_eq!(Ipv6Addr::LOCALHOST, Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)); + assert!(Ipv6Addr::LOCALHOST.is_loopback()); + assert_eq!(Ipv6Addr::UNSPECIFIED, Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0)); + assert!(Ipv6Addr::UNSPECIFIED.is_unspecified()); +} + +#[test] +fn ipv4_from_octets() { + assert_eq!(Ipv4Addr::from([127, 0, 0, 1]), Ipv4Addr::new(127, 0, 0, 1)) +} + +#[test] +fn ipv6_from_segments() { + let from_u16s = + Ipv6Addr::from([0x0011, 0x2233, 0x4455, 0x6677, 0x8899, 0xaabb, 0xccdd, 0xeeff]); + let new = Ipv6Addr::new(0x0011, 0x2233, 0x4455, 0x6677, 0x8899, 0xaabb, 0xccdd, 0xeeff); + assert_eq!(new, from_u16s); +} + +#[test] +fn ipv6_from_octets() { + let from_u16s = + Ipv6Addr::from([0x0011, 0x2233, 0x4455, 0x6677, 0x8899, 0xaabb, 0xccdd, 0xeeff]); + let from_u8s = Ipv6Addr::from([ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]); + assert_eq!(from_u16s, from_u8s); +} + +#[test] +fn cmp() { + let v41 = Ipv4Addr::new(100, 64, 3, 3); + let v42 = Ipv4Addr::new(192, 0, 2, 2); + let v61 = "2001:db8:f00::1002".parse::().unwrap(); + let v62 = "2001:db8:f00::2001".parse::().unwrap(); + assert!(v41 < v42); + assert!(v61 < v62); + + assert_eq!(v41, IpAddr::V4(v41)); + assert_eq!(v61, IpAddr::V6(v61)); + assert!(v41 != IpAddr::V4(v42)); + assert!(v61 != IpAddr::V6(v62)); + + assert!(v41 < IpAddr::V4(v42)); + assert!(v61 < IpAddr::V6(v62)); + assert!(IpAddr::V4(v41) < v42); + assert!(IpAddr::V6(v61) < v62); + + assert!(v41 < IpAddr::V6(v61)); + assert!(IpAddr::V4(v41) < v61); +} + +#[test] +fn is_v4() { + let ip = IpAddr::V4(Ipv4Addr::new(100, 64, 3, 3)); + assert!(ip.is_ipv4()); + assert!(!ip.is_ipv6()); +} + +#[test] +fn is_v6() { + let ip = IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0x1234, 0x5678)); + assert!(!ip.is_ipv4()); + assert!(ip.is_ipv6()); +} + +#[test] +fn ipv4_const() { + // test that the methods of `Ipv4Addr` are usable in a const context + + const IP_ADDRESS: Ipv4Addr = Ipv4Addr::new(127, 0, 0, 1); + assert_eq!(IP_ADDRESS, Ipv4Addr::LOCALHOST); + + const OCTETS: [u8; 4] = IP_ADDRESS.octets(); + assert_eq!(OCTETS, [127, 0, 0, 1]); + + const IS_UNSPECIFIED: bool = IP_ADDRESS.is_unspecified(); + assert!(!IS_UNSPECIFIED); + + const IS_LOOPBACK: bool = IP_ADDRESS.is_loopback(); + assert!(IS_LOOPBACK); + + const IS_PRIVATE: bool = IP_ADDRESS.is_private(); + assert!(!IS_PRIVATE); + + const IS_LINK_LOCAL: bool = IP_ADDRESS.is_link_local(); + assert!(!IS_LINK_LOCAL); + + const IS_GLOBAL: bool = IP_ADDRESS.is_global(); + assert!(!IS_GLOBAL); + + const IS_SHARED: bool = IP_ADDRESS.is_shared(); + assert!(!IS_SHARED); + + const IS_BENCHMARKING: bool = IP_ADDRESS.is_benchmarking(); + assert!(!IS_BENCHMARKING); + + const IS_RESERVED: bool = IP_ADDRESS.is_reserved(); + assert!(!IS_RESERVED); + + const IS_MULTICAST: bool = IP_ADDRESS.is_multicast(); + assert!(!IS_MULTICAST); + + const IS_BROADCAST: bool = IP_ADDRESS.is_broadcast(); + assert!(!IS_BROADCAST); + + const IS_DOCUMENTATION: bool = IP_ADDRESS.is_documentation(); + assert!(!IS_DOCUMENTATION); + + const IP_V6_COMPATIBLE: Ipv6Addr = IP_ADDRESS.to_ipv6_compatible(); + assert_eq!( + IP_V6_COMPATIBLE, + Ipv6Addr::from([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 1]) + ); + + const IP_V6_MAPPED: Ipv6Addr = IP_ADDRESS.to_ipv6_mapped(); + assert_eq!( + IP_V6_MAPPED, + Ipv6Addr::from([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 127, 0, 0, 1]) + ); +} + +#[test] +fn ipv6_const() { + // test that the methods of `Ipv6Addr` are usable in a const context + + const IP_ADDRESS: Ipv6Addr = Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1); + assert_eq!(IP_ADDRESS, Ipv6Addr::LOCALHOST); + + const SEGMENTS: [u16; 8] = IP_ADDRESS.segments(); + assert_eq!(SEGMENTS, [0, 0, 0, 0, 0, 0, 0, 1]); + + const OCTETS: [u8; 16] = IP_ADDRESS.octets(); + assert_eq!(OCTETS, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]); + + const IS_UNSPECIFIED: bool = IP_ADDRESS.is_unspecified(); + assert!(!IS_UNSPECIFIED); + + const IS_LOOPBACK: bool = IP_ADDRESS.is_loopback(); + assert!(IS_LOOPBACK); + + const IS_GLOBAL: bool = IP_ADDRESS.is_global(); + assert!(!IS_GLOBAL); + + const IS_UNIQUE_LOCAL: bool = IP_ADDRESS.is_unique_local(); + assert!(!IS_UNIQUE_LOCAL); + + const IS_UNICAST_LINK_LOCAL: bool = IP_ADDRESS.is_unicast_link_local(); + assert!(!IS_UNICAST_LINK_LOCAL); + + const IS_DOCUMENTATION: bool = IP_ADDRESS.is_documentation(); + assert!(!IS_DOCUMENTATION); + + const IS_BENCHMARKING: bool = IP_ADDRESS.is_benchmarking(); + assert!(!IS_BENCHMARKING); + + const IS_UNICAST_GLOBAL: bool = IP_ADDRESS.is_unicast_global(); + assert!(!IS_UNICAST_GLOBAL); + + const MULTICAST_SCOPE: Option = IP_ADDRESS.multicast_scope(); + assert_eq!(MULTICAST_SCOPE, None); + + const IS_MULTICAST: bool = IP_ADDRESS.is_multicast(); + assert!(!IS_MULTICAST); + + const IP_V4: Option = IP_ADDRESS.to_ipv4(); + assert_eq!(IP_V4.unwrap(), Ipv4Addr::new(0, 0, 0, 1)); +} + +#[test] +fn ip_const() { + // test that the methods of `IpAddr` are usable in a const context + + const IP_ADDRESS: IpAddr = IpAddr::V4(Ipv4Addr::LOCALHOST); + + const IS_UNSPECIFIED: bool = IP_ADDRESS.is_unspecified(); + assert!(!IS_UNSPECIFIED); + + const IS_LOOPBACK: bool = IP_ADDRESS.is_loopback(); + assert!(IS_LOOPBACK); + + const IS_GLOBAL: bool = IP_ADDRESS.is_global(); + assert!(!IS_GLOBAL); + + const IS_MULTICAST: bool = IP_ADDRESS.is_multicast(); + assert!(!IS_MULTICAST); + + const IS_IP_V4: bool = IP_ADDRESS.is_ipv4(); + assert!(IS_IP_V4); + + const IS_IP_V6: bool = IP_ADDRESS.is_ipv6(); + assert!(!IS_IP_V6); +} + +#[test] +fn structural_match() { + // test that all IP types can be structurally matched upon + + const IPV4: Ipv4Addr = Ipv4Addr::LOCALHOST; + match IPV4 { + Ipv4Addr::LOCALHOST => {} + _ => unreachable!(), + } + + const IPV6: Ipv6Addr = Ipv6Addr::LOCALHOST; + match IPV6 { + Ipv6Addr::LOCALHOST => {} + _ => unreachable!(), + } + + const IP: IpAddr = IpAddr::V4(Ipv4Addr::LOCALHOST); + match IP { + IpAddr::V4(Ipv4Addr::LOCALHOST) => {} + _ => unreachable!(), + } +} diff --git a/crux-mir/lib/std/src/net/mod.rs b/crux-mir/lib/std/src/net/mod.rs index c87e0661d..19d90e7ec 100644 --- a/crux-mir/lib/std/src/net/mod.rs +++ b/crux-mir/lib/std/src/net/mod.rs @@ -11,89 +11,65 @@ //! [`Ipv6Addr`] are respectively IPv4 and IPv6 addresses //! * [`SocketAddr`] represents socket addresses of either IPv4 or IPv6; [`SocketAddrV4`] //! and [`SocketAddrV6`] are respectively IPv4 and IPv6 socket addresses -//! * [`ToSocketAddrs`] is a trait that used for generic address resolution when interacting +//! * [`ToSocketAddrs`] is a trait that is used for generic address resolution when interacting //! with networking objects like [`TcpListener`], [`TcpStream`] or [`UdpSocket`] //! * Other types are return or parameter types for various methods in this module //! -//! [`IpAddr`]: ../../std/net/enum.IpAddr.html -//! [`Ipv4Addr`]: ../../std/net/struct.Ipv4Addr.html -//! [`Ipv6Addr`]: ../../std/net/struct.Ipv6Addr.html -//! [`SocketAddr`]: ../../std/net/enum.SocketAddr.html -//! [`SocketAddrV4`]: ../../std/net/struct.SocketAddrV4.html -//! [`SocketAddrV6`]: ../../std/net/struct.SocketAddrV6.html -//! [`TcpListener`]: ../../std/net/struct.TcpListener.html -//! [`TcpStream`]: ../../std/net/struct.TcpStream.html -//! [`ToSocketAddrs`]: ../../std/net/trait.ToSocketAddrs.html -//! [`UdpSocket`]: ../../std/net/struct.UdpSocket.html +//! Rust disables inheritance of socket objects to child processes by default when possible. For +//! example, through the use of the `CLOEXEC` flag in UNIX systems or the `HANDLE_FLAG_INHERIT` +//! flag on Windows. #![stable(feature = "rust1", since = "1.0.0")] -use crate::io::{self, Error, ErrorKind}; +use crate::io::{self, ErrorKind}; #[stable(feature = "rust1", since = "1.0.0")] -pub use self::addr::{SocketAddr, SocketAddrV4, SocketAddrV6, ToSocketAddrs}; -#[stable(feature = "rust1", since = "1.0.0")] -pub use self::ip::{IpAddr, Ipv4Addr, Ipv6Addr, Ipv6MulticastScope}; +pub use self::ip_addr::{IpAddr, Ipv4Addr, Ipv6Addr, Ipv6MulticastScope}; #[stable(feature = "rust1", since = "1.0.0")] pub use self::parser::AddrParseError; #[stable(feature = "rust1", since = "1.0.0")] +pub use self::socket_addr::{SocketAddr, SocketAddrV4, SocketAddrV6, ToSocketAddrs}; +#[unstable(feature = "tcplistener_into_incoming", issue = "88339")] +pub use self::tcp::IntoIncoming; +#[stable(feature = "rust1", since = "1.0.0")] pub use self::tcp::{Incoming, TcpListener, TcpStream}; #[stable(feature = "rust1", since = "1.0.0")] pub use self::udp::UdpSocket; -mod addr; -mod ip; +mod display_buffer; +mod ip_addr; mod parser; +mod socket_addr; mod tcp; #[cfg(test)] -mod test; +pub(crate) mod test; mod udp; -/// Possible values which can be passed to the [`shutdown`] method of -/// [`TcpStream`]. -/// -/// [`shutdown`]: struct.TcpStream.html#method.shutdown -/// [`TcpStream`]: struct.TcpStream.html +/// Possible values which can be passed to the [`TcpStream::shutdown`] method. #[derive(Copy, Clone, PartialEq, Eq, Debug)] #[stable(feature = "rust1", since = "1.0.0")] pub enum Shutdown { /// The reading portion of the [`TcpStream`] should be shut down. /// - /// All currently blocked and future [reads] will return [`Ok(0)`]. + /// All currently blocked and future [reads] will return [Ok]\(0). /// - /// [`TcpStream`]: ../../std/net/struct.TcpStream.html - /// [reads]: ../../std/io/trait.Read.html - /// [`Ok(0)`]: ../../std/result/enum.Result.html#variant.Ok + /// [reads]: crate::io::Read "io::Read" #[stable(feature = "rust1", since = "1.0.0")] Read, /// The writing portion of the [`TcpStream`] should be shut down. /// /// All currently blocked and future [writes] will return an error. /// - /// [`TcpStream`]: ../../std/net/struct.TcpStream.html - /// [writes]: ../../std/io/trait.Write.html + /// [writes]: crate::io::Write "io::Write" #[stable(feature = "rust1", since = "1.0.0")] Write, /// Both the reading and the writing portions of the [`TcpStream`] should be shut down. /// /// See [`Shutdown::Read`] and [`Shutdown::Write`] for more information. - /// - /// [`TcpStream`]: ../../std/net/struct.TcpStream.html - /// [`Shutdown::Read`]: #variant.Read - /// [`Shutdown::Write`]: #variant.Write #[stable(feature = "rust1", since = "1.0.0")] Both, } -#[inline] -const fn htons(i: u16) -> u16 { - i.to_be() -} -#[inline] -const fn ntohs(i: u16) -> u16 { - u16::from_be(i) -} - fn each_addr(addr: A, mut f: F) -> io::Result where F: FnMut(io::Result<&SocketAddr>) -> io::Result, @@ -110,6 +86,6 @@ where } } Err(last_err.unwrap_or_else(|| { - Error::new(ErrorKind::InvalidInput, "could not resolve to any addresses") + io::const_io_error!(ErrorKind::InvalidInput, "could not resolve to any addresses") })) } diff --git a/crux-mir/lib/std/src/net/parser.rs b/crux-mir/lib/std/src/net/parser.rs index 3f38ee547..a38031c48 100644 --- a/crux-mir/lib/std/src/net/parser.rs +++ b/crux-mir/lib/std/src/net/parser.rs @@ -3,255 +3,269 @@ //! This module is "publicly exported" through the `FromStr` implementations //! below. +#[cfg(test)] +mod tests; + use crate::error::Error; use crate::fmt; use crate::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}; use crate::str::FromStr; +trait ReadNumberHelper: crate::marker::Sized { + const ZERO: Self; + fn checked_mul(&self, other: u32) -> Option; + fn checked_add(&self, other: u32) -> Option; +} + +macro_rules! impl_helper { + ($($t:ty)*) => ($(impl ReadNumberHelper for $t { + const ZERO: Self = 0; + #[inline] + fn checked_mul(&self, other: u32) -> Option { + Self::checked_mul(*self, other.try_into().ok()?) + } + #[inline] + fn checked_add(&self, other: u32) -> Option { + Self::checked_add(*self, other.try_into().ok()?) + } + })*) +} + +impl_helper! { u8 u16 u32 } + struct Parser<'a> { - // parsing as ASCII, so can use byte array - s: &'a [u8], - pos: usize, + // Parsing as ASCII, so can use byte array. + state: &'a [u8], } impl<'a> Parser<'a> { - fn new(s: &'a str) -> Parser<'a> { - Parser { s: s.as_bytes(), pos: 0 } - } - - fn is_eof(&self) -> bool { - self.pos == self.s.len() + fn new(input: &'a [u8]) -> Parser<'a> { + Parser { state: input } } - // Commit only if parser returns Some - fn read_atomically(&mut self, cb: F) -> Option + /// Run a parser, and restore the pre-parse state if it fails. + fn read_atomically(&mut self, inner: F) -> Option where F: FnOnce(&mut Parser<'_>) -> Option, { - let pos = self.pos; - let r = cb(self); - if r.is_none() { - self.pos = pos; + let state = self.state; + let result = inner(self); + if result.is_none() { + self.state = state; } - r + result } - // Commit only if parser read till EOF - fn read_till_eof(&mut self, cb: F) -> Option + /// Run a parser, but fail if the entire input wasn't consumed. + /// Doesn't run atomically. + fn parse_with(&mut self, inner: F, kind: AddrKind) -> Result where F: FnOnce(&mut Parser<'_>) -> Option, { - self.read_atomically(move |p| cb(p).filter(|_| p.is_eof())) + let result = inner(self); + if self.state.is_empty() { result } else { None }.ok_or(AddrParseError(kind)) } - // Apply 3 parsers sequentially - fn read_seq_3(&mut self, pa: PA, pb: PB, pc: PC) -> Option<(A, B, C)> - where - PA: FnOnce(&mut Parser<'_>) -> Option, - PB: FnOnce(&mut Parser<'_>) -> Option, - PC: FnOnce(&mut Parser<'_>) -> Option, - { - self.read_atomically(move |p| { - let a = pa(p); - let b = if a.is_some() { pb(p) } else { None }; - let c = if b.is_some() { pc(p) } else { None }; - match (a, b, c) { - (Some(a), Some(b), Some(c)) => Some((a, b, c)), - _ => None, - } - }) + /// Peek the next character from the input + fn peek_char(&self) -> Option { + self.state.first().map(|&b| char::from(b)) } - // Read next char + /// Read the next character from the input fn read_char(&mut self) -> Option { - if self.is_eof() { - None - } else { - let r = self.s[self.pos] as char; - self.pos += 1; - Some(r) - } + self.state.split_first().map(|(&b, tail)| { + self.state = tail; + char::from(b) + }) } - // Return char and advance iff next char is equal to requested - fn read_given_char(&mut self, c: char) -> Option { - self.read_atomically(|p| match p.read_char() { - Some(next) if next == c => Some(next), - _ => None, + #[must_use] + /// Read the next character from the input if it matches the target. + fn read_given_char(&mut self, target: char) -> Option<()> { + self.read_atomically(|p| { + p.read_char().and_then(|c| if c == target { Some(()) } else { None }) }) } - // Read digit - fn read_digit(&mut self, radix: u8) -> Option { - fn parse_digit(c: char, radix: u8) -> Option { - let c = c as u8; - // assuming radix is either 10 or 16 - if c >= b'0' && c <= b'9' { - Some(c - b'0') - } else if radix > 10 && c >= b'a' && c < b'a' + (radix - 10) { - Some(c - b'a' + 10) - } else if radix > 10 && c >= b'A' && c < b'A' + (radix - 10) { - Some(c - b'A' + 10) - } else { - None + /// Helper for reading separators in an indexed loop. Reads the separator + /// character iff index > 0, then runs the parser. When used in a loop, + /// the separator character will only be read on index > 0 (see + /// read_ipv4_addr for an example) + fn read_separator(&mut self, sep: char, index: usize, inner: F) -> Option + where + F: FnOnce(&mut Parser<'_>) -> Option, + { + self.read_atomically(move |p| { + if index > 0 { + p.read_given_char(sep)?; } - } - - self.read_atomically(|p| p.read_char().and_then(|c| parse_digit(c, radix))) + inner(p) + }) } - fn read_number_impl(&mut self, radix: u8, max_digits: u32, upto: u32) -> Option { - let mut r = 0; - let mut digit_count = 0; - loop { - match self.read_digit(radix) { - Some(d) => { - r = r * (radix as u32) + (d as u32); - digit_count += 1; - if digit_count > max_digits || r >= upto { - return None; - } - } - None => { - if digit_count == 0 { + // Read a number off the front of the input in the given radix, stopping + // at the first non-digit character or eof. Fails if the number has more + // digits than max_digits or if there is no number. + fn read_number( + &mut self, + radix: u32, + max_digits: Option, + allow_zero_prefix: bool, + ) -> Option { + self.read_atomically(move |p| { + let mut result = T::ZERO; + let mut digit_count = 0; + let has_leading_zero = p.peek_char() == Some('0'); + + while let Some(digit) = p.read_atomically(|p| p.read_char()?.to_digit(radix)) { + result = result.checked_mul(radix)?; + result = result.checked_add(digit)?; + digit_count += 1; + if let Some(max_digits) = max_digits { + if digit_count > max_digits { return None; - } else { - return Some(r); } } - }; - } - } - - // Read number, failing if max_digits of number value exceeded - fn read_number(&mut self, radix: u8, max_digits: u32, upto: u32) -> Option { - self.read_atomically(|p| p.read_number_impl(radix, max_digits, upto)) - } - - fn read_ipv4_addr_impl(&mut self) -> Option { - let mut bs = [0; 4]; - let mut i = 0; - while i < 4 { - if i != 0 && self.read_given_char('.').is_none() { - return None; } - bs[i] = self.read_number(10, 3, 0x100).map(|n| n as u8)?; - i += 1; - } - Some(Ipv4Addr::new(bs[0], bs[1], bs[2], bs[3])) + if digit_count == 0 { + None + } else if !allow_zero_prefix && has_leading_zero && digit_count > 1 { + None + } else { + Some(result) + } + }) } - // Read IPv4 address + /// Read an IPv4 address. fn read_ipv4_addr(&mut self) -> Option { - self.read_atomically(|p| p.read_ipv4_addr_impl()) - } + self.read_atomically(|p| { + let mut groups = [0; 4]; + + for (i, slot) in groups.iter_mut().enumerate() { + *slot = p.read_separator('.', i, |p| { + // Disallow octal number in IP string. + // https://tools.ietf.org/html/rfc6943#section-3.1.1 + p.read_number(10, Some(3), false) + })?; + } - fn read_ipv6_addr_impl(&mut self) -> Option { - fn ipv6_addr_from_head_tail(head: &[u16], tail: &[u16]) -> Ipv6Addr { - assert!(head.len() + tail.len() <= 8); - let mut gs = [0; 8]; - gs[..head.len()].copy_from_slice(head); - gs[(8 - tail.len())..8].copy_from_slice(tail); - Ipv6Addr::new(gs[0], gs[1], gs[2], gs[3], gs[4], gs[5], gs[6], gs[7]) - } + Some(groups.into()) + }) + } - fn read_groups(p: &mut Parser<'_>, groups: &mut [u16; 8], limit: usize) -> (usize, bool) { - let mut i = 0; - while i < limit { + /// Read an IPv6 Address. + fn read_ipv6_addr(&mut self) -> Option { + /// Read a chunk of an IPv6 address into `groups`. Returns the number + /// of groups read, along with a bool indicating if an embedded + /// trailing IPv4 address was read. Specifically, read a series of + /// colon-separated IPv6 groups (0x0000 - 0xFFFF), with an optional + /// trailing embedded IPv4 address. + fn read_groups(p: &mut Parser<'_>, groups: &mut [u16]) -> (usize, bool) { + let limit = groups.len(); + + for (i, slot) in groups.iter_mut().enumerate() { + // Try to read a trailing embedded IPv4 address. There must be + // at least two groups left. if i < limit - 1 { - let ipv4 = p.read_atomically(|p| { - if i == 0 || p.read_given_char(':').is_some() { - p.read_ipv4_addr() - } else { - None - } - }); + let ipv4 = p.read_separator(':', i, |p| p.read_ipv4_addr()); + if let Some(v4_addr) = ipv4 { - let octets = v4_addr.octets(); - groups[i + 0] = ((octets[0] as u16) << 8) | (octets[1] as u16); - groups[i + 1] = ((octets[2] as u16) << 8) | (octets[3] as u16); + let [one, two, three, four] = v4_addr.octets(); + groups[i + 0] = u16::from_be_bytes([one, two]); + groups[i + 1] = u16::from_be_bytes([three, four]); return (i + 2, true); } } - let group = p.read_atomically(|p| { - if i == 0 || p.read_given_char(':').is_some() { - p.read_number(16, 4, 0x10000).map(|n| n as u16) - } else { - None - } - }); + let group = p.read_separator(':', i, |p| p.read_number(16, Some(4), true)); + match group { - Some(g) => groups[i] = g, + Some(g) => *slot = g, None => return (i, false), } - i += 1; } - (i, false) + (groups.len(), false) } - let mut head = [0; 8]; - let (head_size, head_ipv4) = read_groups(self, &mut head, 8); + self.read_atomically(|p| { + // Read the front part of the address; either the whole thing, or up + // to the first :: + let mut head = [0; 8]; + let (head_size, head_ipv4) = read_groups(p, &mut head); - if head_size == 8 { - return Some(Ipv6Addr::new( - head[0], head[1], head[2], head[3], head[4], head[5], head[6], head[7], - )); - } + if head_size == 8 { + return Some(head.into()); + } - // IPv4 part is not allowed before `::` - if head_ipv4 { - return None; - } + // IPv4 part is not allowed before `::` + if head_ipv4 { + return None; + } - // read `::` if previous code parsed less than 8 groups - if self.read_given_char(':').is_none() || self.read_given_char(':').is_none() { - return None; - } + // Read `::` if previous code parsed less than 8 groups. + // `::` indicates one or more groups of 16 bits of zeros. + p.read_given_char(':')?; + p.read_given_char(':')?; - let mut tail = [0; 8]; - // `::` indicates one or more groups of 16 bits of zeros - let limit = 8 - (head_size + 1); - let (tail_size, _) = read_groups(self, &mut tail, limit); - Some(ipv6_addr_from_head_tail(&head[..head_size], &tail[..tail_size])) - } + // Read the back part of the address. The :: must contain at least one + // set of zeroes, so our max length is 7. + let mut tail = [0; 7]; + let limit = 8 - (head_size + 1); + let (tail_size, _) = read_groups(p, &mut tail[..limit]); - fn read_ipv6_addr(&mut self) -> Option { - self.read_atomically(|p| p.read_ipv6_addr_impl()) + // Concat the head and tail of the IP address + head[(8 - tail_size)..8].copy_from_slice(&tail[..tail_size]); + + Some(head.into()) + }) } + /// Read an IP Address, either IPv4 or IPv6. fn read_ip_addr(&mut self) -> Option { - self.read_ipv4_addr().map(IpAddr::V4).or_else(|| self.read_ipv6_addr().map(IpAddr::V6)) + self.read_ipv4_addr().map(IpAddr::V4).or_else(move || self.read_ipv6_addr().map(IpAddr::V6)) } - fn read_socket_addr_v4(&mut self) -> Option { - let ip_addr = |p: &mut Parser<'_>| p.read_ipv4_addr(); - let colon = |p: &mut Parser<'_>| p.read_given_char(':'); - let port = |p: &mut Parser<'_>| p.read_number(10, 5, 0x10000).map(|n| n as u16); + /// Read a `:` followed by a port in base 10. + fn read_port(&mut self) -> Option { + self.read_atomically(|p| { + p.read_given_char(':')?; + p.read_number(10, None, true) + }) + } + + /// Read a `%` followed by a scope ID in base 10. + fn read_scope_id(&mut self) -> Option { + self.read_atomically(|p| { + p.read_given_char('%')?; + p.read_number(10, None, true) + }) + } - self.read_seq_3(ip_addr, colon, port).map(|t| { - let (ip, _, port): (Ipv4Addr, char, u16) = t; - SocketAddrV4::new(ip, port) + /// Read an IPv4 address with a port. + fn read_socket_addr_v4(&mut self) -> Option { + self.read_atomically(|p| { + let ip = p.read_ipv4_addr()?; + let port = p.read_port()?; + Some(SocketAddrV4::new(ip, port)) }) } + /// Read an IPv6 address with a port. fn read_socket_addr_v6(&mut self) -> Option { - let ip_addr = |p: &mut Parser<'_>| { - let open_br = |p: &mut Parser<'_>| p.read_given_char('['); - let ip_addr = |p: &mut Parser<'_>| p.read_ipv6_addr(); - let clos_br = |p: &mut Parser<'_>| p.read_given_char(']'); - p.read_seq_3(open_br, ip_addr, clos_br).map(|t| t.1) - }; - let colon = |p: &mut Parser<'_>| p.read_given_char(':'); - let port = |p: &mut Parser<'_>| p.read_number(10, 5, 0x10000).map(|n| n as u16); - - self.read_seq_3(ip_addr, colon, port).map(|t| { - let (ip, _, port): (Ipv6Addr, char, u16) = t; - SocketAddrV6::new(ip, port, 0, 0) + self.read_atomically(|p| { + p.read_given_char('[')?; + let ip = p.read_ipv6_addr()?; + let scope_id = p.read_scope_id().unwrap_or(0); + p.read_given_char(']')?; + + let port = p.read_port()?; + Some(SocketAddrV6::new(ip, port, 0, scope_id)) }) } + /// Read an IP address with a port fn read_socket_addr(&mut self) -> Option { self.read_socket_addr_v4() .map(SocketAddr::V4) @@ -259,13 +273,53 @@ impl<'a> Parser<'a> { } } +impl IpAddr { + /// Parse an IP address from a slice of bytes. + /// + /// ``` + /// #![feature(addr_parse_ascii)] + /// + /// use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + /// + /// let localhost_v4 = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)); + /// let localhost_v6 = IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)); + /// + /// assert_eq!(IpAddr::parse_ascii(b"127.0.0.1"), Ok(localhost_v4)); + /// assert_eq!(IpAddr::parse_ascii(b"::1"), Ok(localhost_v6)); + /// ``` + #[unstable(feature = "addr_parse_ascii", issue = "101035")] + pub fn parse_ascii(b: &[u8]) -> Result { + Parser::new(b).parse_with(|p| p.read_ip_addr(), AddrKind::Ip) + } +} + #[stable(feature = "ip_addr", since = "1.7.0")] impl FromStr for IpAddr { type Err = AddrParseError; fn from_str(s: &str) -> Result { - match Parser::new(s).read_till_eof(|p| p.read_ip_addr()) { - Some(s) => Ok(s), - None => Err(AddrParseError(())), + Self::parse_ascii(s.as_bytes()) + } +} + +impl Ipv4Addr { + /// Parse an IPv4 address from a slice of bytes. + /// + /// ``` + /// #![feature(addr_parse_ascii)] + /// + /// use std::net::Ipv4Addr; + /// + /// let localhost = Ipv4Addr::new(127, 0, 0, 1); + /// + /// assert_eq!(Ipv4Addr::parse_ascii(b"127.0.0.1"), Ok(localhost)); + /// ``` + #[unstable(feature = "addr_parse_ascii", issue = "101035")] + pub fn parse_ascii(b: &[u8]) -> Result { + // don't try to parse if too long + if b.len() > 15 { + Err(AddrParseError(AddrKind::Ipv4)) + } else { + Parser::new(b).parse_with(|p| p.read_ipv4_addr(), AddrKind::Ipv4) } } } @@ -274,10 +328,25 @@ impl FromStr for IpAddr { impl FromStr for Ipv4Addr { type Err = AddrParseError; fn from_str(s: &str) -> Result { - match Parser::new(s).read_till_eof(|p| p.read_ipv4_addr()) { - Some(s) => Ok(s), - None => Err(AddrParseError(())), - } + Self::parse_ascii(s.as_bytes()) + } +} + +impl Ipv6Addr { + /// Parse an IPv6 address from a slice of bytes. + /// + /// ``` + /// #![feature(addr_parse_ascii)] + /// + /// use std::net::Ipv6Addr; + /// + /// let localhost = Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1); + /// + /// assert_eq!(Ipv6Addr::parse_ascii(b"::1"), Ok(localhost)); + /// ``` + #[unstable(feature = "addr_parse_ascii", issue = "101035")] + pub fn parse_ascii(b: &[u8]) -> Result { + Parser::new(b).parse_with(|p| p.read_ipv6_addr(), AddrKind::Ipv6) } } @@ -285,10 +354,25 @@ impl FromStr for Ipv4Addr { impl FromStr for Ipv6Addr { type Err = AddrParseError; fn from_str(s: &str) -> Result { - match Parser::new(s).read_till_eof(|p| p.read_ipv6_addr()) { - Some(s) => Ok(s), - None => Err(AddrParseError(())), - } + Self::parse_ascii(s.as_bytes()) + } +} + +impl SocketAddrV4 { + /// Parse an IPv4 socket address from a slice of bytes. + /// + /// ``` + /// #![feature(addr_parse_ascii)] + /// + /// use std::net::{Ipv4Addr, SocketAddrV4}; + /// + /// let socket = SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), 8080); + /// + /// assert_eq!(SocketAddrV4::parse_ascii(b"127.0.0.1:8080"), Ok(socket)); + /// ``` + #[unstable(feature = "addr_parse_ascii", issue = "101035")] + pub fn parse_ascii(b: &[u8]) -> Result { + Parser::new(b).parse_with(|p| p.read_socket_addr_v4(), AddrKind::SocketV4) } } @@ -296,10 +380,25 @@ impl FromStr for Ipv6Addr { impl FromStr for SocketAddrV4 { type Err = AddrParseError; fn from_str(s: &str) -> Result { - match Parser::new(s).read_till_eof(|p| p.read_socket_addr_v4()) { - Some(s) => Ok(s), - None => Err(AddrParseError(())), - } + Self::parse_ascii(s.as_bytes()) + } +} + +impl SocketAddrV6 { + /// Parse an IPv6 socket address from a slice of bytes. + /// + /// ``` + /// #![feature(addr_parse_ascii)] + /// + /// use std::net::{Ipv6Addr, SocketAddrV6}; + /// + /// let socket = SocketAddrV6::new(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1), 8080, 0, 0); + /// + /// assert_eq!(SocketAddrV6::parse_ascii(b"[2001:db8::1]:8080"), Ok(socket)); + /// ``` + #[unstable(feature = "addr_parse_ascii", issue = "101035")] + pub fn parse_ascii(b: &[u8]) -> Result { + Parser::new(b).parse_with(|p| p.read_socket_addr_v6(), AddrKind::SocketV6) } } @@ -307,10 +406,27 @@ impl FromStr for SocketAddrV4 { impl FromStr for SocketAddrV6 { type Err = AddrParseError; fn from_str(s: &str) -> Result { - match Parser::new(s).read_till_eof(|p| p.read_socket_addr_v6()) { - Some(s) => Ok(s), - None => Err(AddrParseError(())), - } + Self::parse_ascii(s.as_bytes()) + } +} + +impl SocketAddr { + /// Parse a socket address from a slice of bytes. + /// + /// ``` + /// #![feature(addr_parse_ascii)] + /// + /// use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; + /// + /// let socket_v4 = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); + /// let socket_v6 = SocketAddr::new(IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)), 8080); + /// + /// assert_eq!(SocketAddr::parse_ascii(b"127.0.0.1:8080"), Ok(socket_v4)); + /// assert_eq!(SocketAddr::parse_ascii(b"[::1]:8080"), Ok(socket_v6)); + /// ``` + #[unstable(feature = "addr_parse_ascii", issue = "101035")] + pub fn parse_ascii(b: &[u8]) -> Result { + Parser::new(b).parse_with(|p| p.read_socket_addr(), AddrKind::Socket) } } @@ -318,13 +434,20 @@ impl FromStr for SocketAddrV6 { impl FromStr for SocketAddr { type Err = AddrParseError; fn from_str(s: &str) -> Result { - match Parser::new(s).read_till_eof(|p| p.read_socket_addr()) { - Some(s) => Ok(s), - None => Err(AddrParseError(())), - } + Self::parse_ascii(s.as_bytes()) } } +#[derive(Debug, Clone, PartialEq, Eq)] +enum AddrKind { + Ip, + Ipv4, + Ipv6, + Socket, + SocketV4, + SocketV6, +} + /// An error which can be returned when parsing an IP address or a socket address. /// /// This error is used as the error type for the [`FromStr`] implementation for @@ -349,17 +472,9 @@ impl FromStr for SocketAddr { /// // No problem, the `panic!` message has disappeared. /// let _foo: SocketAddr = "127.0.0.1:8080".parse().expect("unreachable panic"); /// ``` -/// -/// [`FromStr`]: ../../std/str/trait.FromStr.html -/// [`IpAddr`]: ../../std/net/enum.IpAddr.html -/// [`Ipv4Addr`]: ../../std/net/struct.Ipv4Addr.html -/// [`Ipv6Addr`]: ../../std/net/struct.Ipv6Addr.html -/// [`SocketAddr`]: ../../std/net/enum.SocketAddr.html -/// [`SocketAddrV4`]: ../../std/net/struct.SocketAddrV4.html -/// [`SocketAddrV6`]: ../../std/net/struct.SocketAddrV6.html #[stable(feature = "rust1", since = "1.0.0")] #[derive(Debug, Clone, PartialEq, Eq)] -pub struct AddrParseError(()); +pub struct AddrParseError(AddrKind); #[stable(feature = "addr_parse_error_error", since = "1.4.0")] impl fmt::Display for AddrParseError { @@ -373,6 +488,13 @@ impl fmt::Display for AddrParseError { impl Error for AddrParseError { #[allow(deprecated)] fn description(&self) -> &str { - "invalid IP address syntax" + match self.0 { + AddrKind::Ip => "invalid IP address syntax", + AddrKind::Ipv4 => "invalid IPv4 address syntax", + AddrKind::Ipv6 => "invalid IPv6 address syntax", + AddrKind::Socket => "invalid socket address syntax", + AddrKind::SocketV4 => "invalid IPv4 socket address syntax", + AddrKind::SocketV6 => "invalid IPv6 socket address syntax", + } } } diff --git a/crux-mir/lib/std/src/net/parser/tests.rs b/crux-mir/lib/std/src/net/parser/tests.rs new file mode 100644 index 000000000..6d2d48eca --- /dev/null +++ b/crux-mir/lib/std/src/net/parser/tests.rs @@ -0,0 +1,149 @@ +// FIXME: These tests are all excellent candidates for AFL fuzz testing +use crate::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}; +use crate::str::FromStr; + +const PORT: u16 = 8080; +const SCOPE_ID: u32 = 1337; + +const IPV4: Ipv4Addr = Ipv4Addr::new(192, 168, 0, 1); +const IPV4_STR: &str = "192.168.0.1"; +const IPV4_STR_PORT: &str = "192.168.0.1:8080"; +const IPV4_STR_WITH_OCTAL: &str = "0127.0.0.1"; +const IPV4_STR_WITH_HEX: &str = "0x10.0.0.1"; + +const IPV6: Ipv6Addr = Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0xc0a8, 0x1); +const IPV6_STR_FULL: &str = "2001:db8:0:0:0:0:c0a8:1"; +const IPV6_STR_COMPRESS: &str = "2001:db8::c0a8:1"; +const IPV6_STR_V4: &str = "2001:db8::192.168.0.1"; +const IPV6_STR_V4_WITH_OCTAL: &str = "2001:db8::0127.0.0.1"; +const IPV6_STR_V4_WITH_HEX: &str = "2001:db8::0x10.0.0.1"; +const IPV6_STR_PORT: &str = "[2001:db8::c0a8:1]:8080"; +const IPV6_STR_PORT_SCOPE_ID: &str = "[2001:db8::c0a8:1%1337]:8080"; + +#[test] +fn parse_ipv4() { + let result: Ipv4Addr = IPV4_STR.parse().unwrap(); + assert_eq!(result, IPV4); + + assert!(Ipv4Addr::from_str(IPV4_STR_PORT).is_err()); + assert!(Ipv4Addr::from_str(IPV4_STR_WITH_OCTAL).is_err()); + assert!(Ipv4Addr::from_str(IPV4_STR_WITH_HEX).is_err()); + assert!(Ipv4Addr::from_str(IPV6_STR_FULL).is_err()); + assert!(Ipv4Addr::from_str(IPV6_STR_COMPRESS).is_err()); + assert!(Ipv4Addr::from_str(IPV6_STR_V4).is_err()); + assert!(Ipv4Addr::from_str(IPV6_STR_PORT).is_err()); +} + +#[test] +fn parse_ipv6() { + let result: Ipv6Addr = IPV6_STR_FULL.parse().unwrap(); + assert_eq!(result, IPV6); + + let result: Ipv6Addr = IPV6_STR_COMPRESS.parse().unwrap(); + assert_eq!(result, IPV6); + + let result: Ipv6Addr = IPV6_STR_V4.parse().unwrap(); + assert_eq!(result, IPV6); + + assert!(Ipv6Addr::from_str(IPV6_STR_V4_WITH_OCTAL).is_err()); + assert!(Ipv6Addr::from_str(IPV6_STR_V4_WITH_HEX).is_err()); + assert!(Ipv6Addr::from_str(IPV4_STR).is_err()); + assert!(Ipv6Addr::from_str(IPV4_STR_PORT).is_err()); + assert!(Ipv6Addr::from_str(IPV6_STR_PORT).is_err()); +} + +#[test] +fn parse_ip() { + let result: IpAddr = IPV4_STR.parse().unwrap(); + assert_eq!(result, IpAddr::from(IPV4)); + + let result: IpAddr = IPV6_STR_FULL.parse().unwrap(); + assert_eq!(result, IpAddr::from(IPV6)); + + let result: IpAddr = IPV6_STR_COMPRESS.parse().unwrap(); + assert_eq!(result, IpAddr::from(IPV6)); + + let result: IpAddr = IPV6_STR_V4.parse().unwrap(); + assert_eq!(result, IpAddr::from(IPV6)); + + assert!(IpAddr::from_str(IPV4_STR_PORT).is_err()); + assert!(IpAddr::from_str(IPV6_STR_PORT).is_err()); +} + +#[test] +fn parse_socket_v4() { + let result: SocketAddrV4 = IPV4_STR_PORT.parse().unwrap(); + assert_eq!(result, SocketAddrV4::new(IPV4, PORT)); + + assert!(SocketAddrV4::from_str(IPV4_STR).is_err()); + assert!(SocketAddrV4::from_str(IPV6_STR_FULL).is_err()); + assert!(SocketAddrV4::from_str(IPV6_STR_COMPRESS).is_err()); + assert!(SocketAddrV4::from_str(IPV6_STR_V4).is_err()); + assert!(SocketAddrV4::from_str(IPV6_STR_PORT).is_err()); +} + +#[test] +fn parse_socket_v6() { + assert_eq!(IPV6_STR_PORT.parse(), Ok(SocketAddrV6::new(IPV6, PORT, 0, 0))); + assert_eq!(IPV6_STR_PORT_SCOPE_ID.parse(), Ok(SocketAddrV6::new(IPV6, PORT, 0, SCOPE_ID))); + + assert!(SocketAddrV6::from_str(IPV4_STR).is_err()); + assert!(SocketAddrV6::from_str(IPV4_STR_PORT).is_err()); + assert!(SocketAddrV6::from_str(IPV6_STR_FULL).is_err()); + assert!(SocketAddrV6::from_str(IPV6_STR_COMPRESS).is_err()); + assert!(SocketAddrV6::from_str(IPV6_STR_V4).is_err()); +} + +#[test] +fn parse_socket() { + let result: SocketAddr = IPV4_STR_PORT.parse().unwrap(); + assert_eq!(result, SocketAddr::from((IPV4, PORT))); + + let result: SocketAddr = IPV6_STR_PORT.parse().unwrap(); + assert_eq!(result, SocketAddr::from((IPV6, PORT))); + + assert!(SocketAddr::from_str(IPV4_STR).is_err()); + assert!(SocketAddr::from_str(IPV6_STR_FULL).is_err()); + assert!(SocketAddr::from_str(IPV6_STR_COMPRESS).is_err()); + assert!(SocketAddr::from_str(IPV6_STR_V4).is_err()); +} + +#[test] +fn ipv6_corner_cases() { + let result: Ipv6Addr = "1::".parse().unwrap(); + assert_eq!(result, Ipv6Addr::new(1, 0, 0, 0, 0, 0, 0, 0)); + + let result: Ipv6Addr = "1:1::".parse().unwrap(); + assert_eq!(result, Ipv6Addr::new(1, 1, 0, 0, 0, 0, 0, 0)); + + let result: Ipv6Addr = "::1".parse().unwrap(); + assert_eq!(result, Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)); + + let result: Ipv6Addr = "::1:1".parse().unwrap(); + assert_eq!(result, Ipv6Addr::new(0, 0, 0, 0, 0, 0, 1, 1)); + + let result: Ipv6Addr = "::".parse().unwrap(); + assert_eq!(result, Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0)); + + let result: Ipv6Addr = "::192.168.0.1".parse().unwrap(); + assert_eq!(result, Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0xc0a8, 0x1)); + + let result: Ipv6Addr = "::1:192.168.0.1".parse().unwrap(); + assert_eq!(result, Ipv6Addr::new(0, 0, 0, 0, 0, 1, 0xc0a8, 0x1)); + + let result: Ipv6Addr = "1:1:1:1:1:1:192.168.0.1".parse().unwrap(); + assert_eq!(result, Ipv6Addr::new(1, 1, 1, 1, 1, 1, 0xc0a8, 0x1)); +} + +// Things that might not seem like failures but are +#[test] +fn ipv6_corner_failures() { + // No IP address before the :: + assert!(Ipv6Addr::from_str("1:192.168.0.1::").is_err()); + + // :: must have at least 1 set of zeroes + assert!(Ipv6Addr::from_str("1:1:1:1::1:1:1:1").is_err()); + + // Need brackets for a port + assert!(SocketAddrV6::from_str("1:1:1:1:1:1:1:1:8080").is_err()); +} diff --git a/crux-mir/lib/std/src/net/addr.rs b/crux-mir/lib/std/src/net/socket_addr.rs similarity index 66% rename from crux-mir/lib/std/src/net/addr.rs rename to crux-mir/lib/std/src/net/socket_addr.rs index 57cba6b1f..33b0dfa03 100644 --- a/crux-mir/lib/std/src/net/addr.rs +++ b/crux-mir/lib/std/src/net/socket_addr.rs @@ -1,17 +1,22 @@ -use crate::convert::TryInto; -use crate::fmt; +#[cfg(all(test, not(target_os = "emscripten")))] +mod tests; + +use crate::cmp::Ordering; +use crate::fmt::{self, Write}; use crate::hash; use crate::io; use crate::iter; use crate::mem; -use crate::net::{htons, ntohs, IpAddr, Ipv4Addr, Ipv6Addr}; +use crate::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use crate::option; use crate::slice; use crate::sys::net::netc as c; use crate::sys_common::net::LookupHost; -use crate::sys_common::{AsInner, FromInner, IntoInner}; +use crate::sys_common::{FromInner, IntoInner}; use crate::vec; +use super::display_buffer::DisplayBuffer; + /// An internet socket address, either IPv4 or IPv6. /// /// Internet socket addresses consist of an [IP address], a 16-bit port number, as well @@ -21,9 +26,7 @@ use crate::vec; /// The size of a `SocketAddr` instance may vary depending on the target operating /// system. /// -/// [IP address]: ../../std/net/enum.IpAddr.html -/// [`SocketAddrV4`]: ../../std/net/struct.SocketAddrV4.html -/// [`SocketAddrV6`]: ../../std/net/struct.SocketAddrV6.html +/// [IP address]: IpAddr /// /// # Examples /// @@ -36,7 +39,7 @@ use crate::vec; /// assert_eq!(socket.port(), 8080); /// assert_eq!(socket.is_ipv4(), true); /// ``` -#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] +#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] #[stable(feature = "rust1", since = "1.0.0")] pub enum SocketAddr { /// An IPv4 socket address. @@ -49,17 +52,17 @@ pub enum SocketAddr { /// An IPv4 socket address. /// -/// IPv4 socket addresses consist of an [IPv4 address] and a 16-bit port number, as +/// IPv4 socket addresses consist of an [`IPv4` address] and a 16-bit port number, as /// stated in [IETF RFC 793]. /// /// See [`SocketAddr`] for a type encompassing both IPv4 and IPv6 socket addresses. /// /// The size of a `SocketAddrV4` struct may vary depending on the target operating -/// system. +/// system. Do not assume that this type has the same memory layout as the underlying +/// system representation. /// /// [IETF RFC 793]: https://tools.ietf.org/html/rfc793 -/// [IPv4 address]: ../../std/net/struct.Ipv4Addr.html -/// [`SocketAddr`]: ../../std/net/enum.SocketAddr.html +/// [`IPv4` address]: Ipv4Addr /// /// # Examples /// @@ -72,26 +75,27 @@ pub enum SocketAddr { /// assert_eq!(socket.ip(), &Ipv4Addr::new(127, 0, 0, 1)); /// assert_eq!(socket.port(), 8080); /// ``` -#[derive(Copy)] +#[derive(Copy, Clone, Eq, PartialEq)] #[stable(feature = "rust1", since = "1.0.0")] pub struct SocketAddrV4 { - inner: c::sockaddr_in, + ip: Ipv4Addr, + port: u16, } /// An IPv6 socket address. /// -/// IPv6 socket addresses consist of an [Ipv6 address], a 16-bit port number, as well +/// IPv6 socket addresses consist of an [`IPv6` address], a 16-bit port number, as well /// as fields containing the traffic class, the flow label, and a scope identifier /// (see [IETF RFC 2553, Section 3.3] for more details). /// /// See [`SocketAddr`] for a type encompassing both IPv4 and IPv6 socket addresses. /// /// The size of a `SocketAddrV6` struct may vary depending on the target operating -/// system. +/// system. Do not assume that this type has the same memory layout as the underlying +/// system representation. /// /// [IETF RFC 2553, Section 3.3]: https://tools.ietf.org/html/rfc2553#section-3.3 -/// [IPv6 address]: ../../std/net/struct.Ipv6Addr.html -/// [`SocketAddr`]: ../../std/net/enum.SocketAddr.html +/// [`IPv6` address]: Ipv6Addr /// /// # Examples /// @@ -104,16 +108,19 @@ pub struct SocketAddrV4 { /// assert_eq!(socket.ip(), &Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1)); /// assert_eq!(socket.port(), 8080); /// ``` -#[derive(Copy)] +#[derive(Copy, Clone, Eq, PartialEq)] #[stable(feature = "rust1", since = "1.0.0")] pub struct SocketAddrV6 { - inner: c::sockaddr_in6, + ip: Ipv6Addr, + port: u16, + flowinfo: u32, + scope_id: u32, } impl SocketAddr { /// Creates a new socket address from an [IP address] and a port number. /// - /// [IP address]: ../../std/net/enum.IpAddr.html + /// [IP address]: IpAddr /// /// # Examples /// @@ -125,7 +132,9 @@ impl SocketAddr { /// assert_eq!(socket.port(), 8080); /// ``` #[stable(feature = "ip_addr", since = "1.7.0")] - pub fn new(ip: IpAddr, port: u16) -> SocketAddr { + #[must_use] + #[rustc_const_unstable(feature = "const_socketaddr", issue = "82485")] + pub const fn new(ip: IpAddr, port: u16) -> SocketAddr { match ip { IpAddr::V4(a) => SocketAddr::V4(SocketAddrV4::new(a, port)), IpAddr::V6(a) => SocketAddr::V6(SocketAddrV6::new(a, port, 0, 0)), @@ -142,8 +151,10 @@ impl SocketAddr { /// let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); /// assert_eq!(socket.ip(), IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))); /// ``` + #[must_use] #[stable(feature = "ip_addr", since = "1.7.0")] - pub fn ip(&self) -> IpAddr { + #[rustc_const_unstable(feature = "const_socketaddr", issue = "82485")] + pub const fn ip(&self) -> IpAddr { match *self { SocketAddr::V4(ref a) => IpAddr::V4(*a.ip()), SocketAddr::V6(ref a) => IpAddr::V6(*a.ip()), @@ -181,8 +192,10 @@ impl SocketAddr { /// let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); /// assert_eq!(socket.port(), 8080); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] - pub fn port(&self) -> u16 { + #[rustc_const_unstable(feature = "const_socketaddr", issue = "82485")] + pub const fn port(&self) -> u16 { match *self { SocketAddr::V4(ref a) => a.port(), SocketAddr::V6(ref a) => a.port(), @@ -209,12 +222,10 @@ impl SocketAddr { } /// Returns [`true`] if the [IP address] in this `SocketAddr` is an - /// [IPv4 address], and [`false`] otherwise. + /// [`IPv4` address], and [`false`] otherwise. /// - /// [`true`]: ../../std/primitive.bool.html - /// [`false`]: ../../std/primitive.bool.html - /// [IP address]: ../../std/net/enum.IpAddr.html - /// [IPv4 address]: ../../std/net/enum.IpAddr.html#variant.V4 + /// [IP address]: IpAddr + /// [`IPv4` address]: IpAddr::V4 /// /// # Examples /// @@ -225,18 +236,18 @@ impl SocketAddr { /// assert_eq!(socket.is_ipv4(), true); /// assert_eq!(socket.is_ipv6(), false); /// ``` + #[must_use] #[stable(feature = "sockaddr_checker", since = "1.16.0")] - pub fn is_ipv4(&self) -> bool { + #[rustc_const_unstable(feature = "const_socketaddr", issue = "82485")] + pub const fn is_ipv4(&self) -> bool { matches!(*self, SocketAddr::V4(_)) } /// Returns [`true`] if the [IP address] in this `SocketAddr` is an - /// [IPv6 address], and [`false`] otherwise. + /// [`IPv6` address], and [`false`] otherwise. /// - /// [`true`]: ../../std/primitive.bool.html - /// [`false`]: ../../std/primitive.bool.html - /// [IP address]: ../../std/net/enum.IpAddr.html - /// [IPv6 address]: ../../std/net/enum.IpAddr.html#variant.V6 + /// [IP address]: IpAddr + /// [`IPv6` address]: IpAddr::V6 /// /// # Examples /// @@ -247,16 +258,18 @@ impl SocketAddr { /// assert_eq!(socket.is_ipv4(), false); /// assert_eq!(socket.is_ipv6(), true); /// ``` + #[must_use] #[stable(feature = "sockaddr_checker", since = "1.16.0")] - pub fn is_ipv6(&self) -> bool { + #[rustc_const_unstable(feature = "const_socketaddr", issue = "82485")] + pub const fn is_ipv6(&self) -> bool { matches!(*self, SocketAddr::V6(_)) } } impl SocketAddrV4 { - /// Creates a new socket address from an [IPv4 address] and a port number. + /// Creates a new socket address from an [`IPv4` address] and a port number. /// - /// [IPv4 address]: ../../std/net/struct.Ipv4Addr.html + /// [`IPv4` address]: Ipv4Addr /// /// # Examples /// @@ -266,15 +279,10 @@ impl SocketAddrV4 { /// let socket = SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), 8080); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn new(ip: Ipv4Addr, port: u16) -> SocketAddrV4 { - SocketAddrV4 { - inner: c::sockaddr_in { - sin_family: c::AF_INET as c::sa_family_t, - sin_port: htons(port), - sin_addr: *ip.as_inner(), - ..unsafe { mem::zeroed() } - }, - } + #[must_use] + #[rustc_const_unstable(feature = "const_socketaddr", issue = "82485")] + pub const fn new(ip: Ipv4Addr, port: u16) -> SocketAddrV4 { + SocketAddrV4 { ip, port } } /// Returns the IP address associated with this socket address. @@ -287,9 +295,11 @@ impl SocketAddrV4 { /// let socket = SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), 8080); /// assert_eq!(socket.ip(), &Ipv4Addr::new(127, 0, 0, 1)); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] - pub fn ip(&self) -> &Ipv4Addr { - unsafe { &*(&self.inner.sin_addr as *const c::in_addr as *const Ipv4Addr) } + #[rustc_const_unstable(feature = "const_socketaddr", issue = "82485")] + pub const fn ip(&self) -> &Ipv4Addr { + &self.ip } /// Changes the IP address associated with this socket address. @@ -305,7 +315,7 @@ impl SocketAddrV4 { /// ``` #[stable(feature = "sockaddr_setters", since = "1.9.0")] pub fn set_ip(&mut self, new_ip: Ipv4Addr) { - self.inner.sin_addr = *new_ip.as_inner() + self.ip = new_ip; } /// Returns the port number associated with this socket address. @@ -318,9 +328,11 @@ impl SocketAddrV4 { /// let socket = SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), 8080); /// assert_eq!(socket.port(), 8080); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] - pub fn port(&self) -> u16 { - ntohs(self.inner.sin_port) + #[rustc_const_unstable(feature = "const_socketaddr", issue = "82485")] + pub const fn port(&self) -> u16 { + self.port } /// Changes the port number associated with this socket address. @@ -336,19 +348,19 @@ impl SocketAddrV4 { /// ``` #[stable(feature = "sockaddr_setters", since = "1.9.0")] pub fn set_port(&mut self, new_port: u16) { - self.inner.sin_port = htons(new_port); + self.port = new_port; } } impl SocketAddrV6 { - /// Creates a new socket address from an [IPv6 address], a 16-bit port number, + /// Creates a new socket address from an [`IPv6` address], a 16-bit port number, /// and the `flowinfo` and `scope_id` fields. /// /// For more information on the meaning and layout of the `flowinfo` and `scope_id` /// parameters, see [IETF RFC 2553, Section 3.3]. /// /// [IETF RFC 2553, Section 3.3]: https://tools.ietf.org/html/rfc2553#section-3.3 - /// [IPv6 address]: ../../std/net/struct.Ipv6Addr.html + /// [`IPv6` address]: Ipv6Addr /// /// # Examples /// @@ -358,17 +370,10 @@ impl SocketAddrV6 { /// let socket = SocketAddrV6::new(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1), 8080, 0, 0); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn new(ip: Ipv6Addr, port: u16, flowinfo: u32, scope_id: u32) -> SocketAddrV6 { - SocketAddrV6 { - inner: c::sockaddr_in6 { - sin6_family: c::AF_INET6 as c::sa_family_t, - sin6_port: htons(port), - sin6_addr: *ip.as_inner(), - sin6_flowinfo: flowinfo, - sin6_scope_id: scope_id, - ..unsafe { mem::zeroed() } - }, - } + #[must_use] + #[rustc_const_unstable(feature = "const_socketaddr", issue = "82485")] + pub const fn new(ip: Ipv6Addr, port: u16, flowinfo: u32, scope_id: u32) -> SocketAddrV6 { + SocketAddrV6 { ip, port, flowinfo, scope_id } } /// Returns the IP address associated with this socket address. @@ -381,9 +386,11 @@ impl SocketAddrV6 { /// let socket = SocketAddrV6::new(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1), 8080, 0, 0); /// assert_eq!(socket.ip(), &Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] - pub fn ip(&self) -> &Ipv6Addr { - unsafe { &*(&self.inner.sin6_addr as *const c::in6_addr as *const Ipv6Addr) } + #[rustc_const_unstable(feature = "const_socketaddr", issue = "82485")] + pub const fn ip(&self) -> &Ipv6Addr { + &self.ip } /// Changes the IP address associated with this socket address. @@ -399,7 +406,7 @@ impl SocketAddrV6 { /// ``` #[stable(feature = "sockaddr_setters", since = "1.9.0")] pub fn set_ip(&mut self, new_ip: Ipv6Addr) { - self.inner.sin6_addr = *new_ip.as_inner() + self.ip = new_ip; } /// Returns the port number associated with this socket address. @@ -412,9 +419,11 @@ impl SocketAddrV6 { /// let socket = SocketAddrV6::new(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1), 8080, 0, 0); /// assert_eq!(socket.port(), 8080); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] - pub fn port(&self) -> u16 { - ntohs(self.inner.sin6_port) + #[rustc_const_unstable(feature = "const_socketaddr", issue = "82485")] + pub const fn port(&self) -> u16 { + self.port } /// Changes the port number associated with this socket address. @@ -430,7 +439,7 @@ impl SocketAddrV6 { /// ``` #[stable(feature = "sockaddr_setters", since = "1.9.0")] pub fn set_port(&mut self, new_port: u16) { - self.inner.sin6_port = htons(new_port); + self.port = new_port; } /// Returns the flow information associated with this address. @@ -453,16 +462,16 @@ impl SocketAddrV6 { /// let socket = SocketAddrV6::new(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1), 8080, 10, 0); /// assert_eq!(socket.flowinfo(), 10); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] - pub fn flowinfo(&self) -> u32 { - self.inner.sin6_flowinfo + #[rustc_const_unstable(feature = "const_socketaddr", issue = "82485")] + pub const fn flowinfo(&self) -> u32 { + self.flowinfo } /// Changes the flow information associated with this socket address. /// - /// See the [`flowinfo`] method's documentation for more details. - /// - /// [`flowinfo`]: #method.flowinfo + /// See [`SocketAddrV6::flowinfo`]'s documentation for more details. /// /// # Examples /// @@ -475,7 +484,7 @@ impl SocketAddrV6 { /// ``` #[stable(feature = "sockaddr_setters", since = "1.9.0")] pub fn set_flowinfo(&mut self, new_flowinfo: u32) { - self.inner.sin6_flowinfo = new_flowinfo; + self.flowinfo = new_flowinfo; } /// Returns the scope ID associated with this address. @@ -493,16 +502,16 @@ impl SocketAddrV6 { /// let socket = SocketAddrV6::new(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1), 8080, 0, 78); /// assert_eq!(socket.scope_id(), 78); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] - pub fn scope_id(&self) -> u32 { - self.inner.sin6_scope_id + #[rustc_const_unstable(feature = "const_socketaddr", issue = "82485")] + pub const fn scope_id(&self) -> u32 { + self.scope_id } /// Changes the scope ID associated with this socket address. /// - /// See the [`scope_id`] method's documentation for more details. - /// - /// [`scope_id`]: #method.scope_id + /// See [`SocketAddrV6::scope_id`]'s documentation for more details. /// /// # Examples /// @@ -515,28 +524,54 @@ impl SocketAddrV6 { /// ``` #[stable(feature = "sockaddr_setters", since = "1.9.0")] pub fn set_scope_id(&mut self, new_scope_id: u32) { - self.inner.sin6_scope_id = new_scope_id; + self.scope_id = new_scope_id; } } impl FromInner for SocketAddrV4 { fn from_inner(addr: c::sockaddr_in) -> SocketAddrV4 { - SocketAddrV4 { inner: addr } + SocketAddrV4 { ip: Ipv4Addr::from_inner(addr.sin_addr), port: u16::from_be(addr.sin_port) } } } impl FromInner for SocketAddrV6 { fn from_inner(addr: c::sockaddr_in6) -> SocketAddrV6 { - SocketAddrV6 { inner: addr } + SocketAddrV6 { + ip: Ipv6Addr::from_inner(addr.sin6_addr), + port: u16::from_be(addr.sin6_port), + flowinfo: addr.sin6_flowinfo, + scope_id: addr.sin6_scope_id, + } + } +} + +impl IntoInner for SocketAddrV4 { + fn into_inner(self) -> c::sockaddr_in { + c::sockaddr_in { + sin_family: c::AF_INET as c::sa_family_t, + sin_port: self.port.to_be(), + sin_addr: self.ip.into_inner(), + ..unsafe { mem::zeroed() } + } + } +} + +impl IntoInner for SocketAddrV6 { + fn into_inner(self) -> c::sockaddr_in6 { + c::sockaddr_in6 { + sin6_family: c::AF_INET6 as c::sa_family_t, + sin6_port: self.port.to_be(), + sin6_addr: self.ip.into_inner(), + sin6_flowinfo: self.flowinfo, + sin6_scope_id: self.scope_id, + ..unsafe { mem::zeroed() } + } } } #[stable(feature = "ip_from_ip", since = "1.16.0")] impl From for SocketAddr { /// Converts a [`SocketAddrV4`] into a [`SocketAddr::V4`]. - /// - /// [`SocketAddrV4`]: ../../std/net/struct.SocketAddrV4.html - /// [`SocketAddr::V4`]: ../../std/net/enum.SocketAddr.html#variant.V4 fn from(sock4: SocketAddrV4) -> SocketAddr { SocketAddr::V4(sock4) } @@ -545,9 +580,6 @@ impl From for SocketAddr { #[stable(feature = "ip_from_ip", since = "1.16.0")] impl From for SocketAddr { /// Converts a [`SocketAddrV6`] into a [`SocketAddr::V6`]. - /// - /// [`SocketAddrV6`]: ../../std/net/struct.SocketAddrV6.html - /// [`SocketAddr::V6`]: ../../std/net/enum.SocketAddr.html#variant.V6 fn from(sock6: SocketAddrV6) -> SocketAddr { SocketAddr::V6(sock6) } @@ -557,35 +589,15 @@ impl From for SocketAddr { impl> From<(I, u16)> for SocketAddr { /// Converts a tuple struct (Into<[`IpAddr`]>, `u16`) into a [`SocketAddr`]. /// - /// This conversion creates a [`SocketAddr::V4`] for a [`IpAddr::V4`] - /// and creates a [`SocketAddr::V6`] for a [`IpAddr::V6`]. + /// This conversion creates a [`SocketAddr::V4`] for an [`IpAddr::V4`] + /// and creates a [`SocketAddr::V6`] for an [`IpAddr::V6`]. /// /// `u16` is treated as port of the newly created [`SocketAddr`]. - /// - /// [`IpAddr`]: ../../std/net/enum.IpAddr.html - /// [`IpAddr::V4`]: ../../std/net/enum.IpAddr.html#variant.V4 - /// [`IpAddr::V6`]: ../../std/net/enum.IpAddr.html#variant.V6 - /// [`SocketAddr`]: ../../std/net/enum.SocketAddr.html - /// [`SocketAddr::V4`]: ../../std/net/enum.SocketAddr.html#variant.V4 - /// [`SocketAddr::V6`]: ../../std/net/enum.SocketAddr.html#variant.V6 fn from(pieces: (I, u16)) -> SocketAddr { SocketAddr::new(pieces.0.into(), pieces.1) } } -impl<'a> IntoInner<(*const c::sockaddr, c::socklen_t)> for &'a SocketAddr { - fn into_inner(self) -> (*const c::sockaddr, c::socklen_t) { - match *self { - SocketAddr::V4(ref a) => { - (a as *const _ as *const _, mem::size_of_val(a) as c::socklen_t) - } - SocketAddr::V6(ref a) => { - (a as *const _ as *const _, mem::size_of_val(a) as c::socklen_t) - } - } - } -} - #[stable(feature = "rust1", since = "1.0.0")] impl fmt::Display for SocketAddr { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -596,10 +608,29 @@ impl fmt::Display for SocketAddr { } } +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Debug for SocketAddr { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, fmt) + } +} + #[stable(feature = "rust1", since = "1.0.0")] impl fmt::Display for SocketAddrV4 { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}:{}", self.ip(), self.port()) + // If there are no alignment requirements, write the socket address directly to `f`. + // Otherwise, write it to a local buffer and then use `f.pad`. + if f.precision().is_none() && f.width().is_none() { + write!(f, "{}:{}", self.ip(), self.port()) + } else { + const LONGEST_IPV4_SOCKET_ADDR: &str = "255.255.255.255:65536"; + + let mut buf = DisplayBuffer::<{ LONGEST_IPV4_SOCKET_ADDR.len() }>::new(); + // Buffer is long enough for the longest possible IPv4 socket address, so this should never fail. + write!(buf, "{}:{}", self.ip(), self.port()).unwrap(); + + f.pad(buf.as_str()) + } } } @@ -613,7 +644,27 @@ impl fmt::Debug for SocketAddrV4 { #[stable(feature = "rust1", since = "1.0.0")] impl fmt::Display for SocketAddrV6 { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "[{}]:{}", self.ip(), self.port()) + // If there are no alignment requirements, write the socket address directly to `f`. + // Otherwise, write it to a local buffer and then use `f.pad`. + if f.precision().is_none() && f.width().is_none() { + match self.scope_id() { + 0 => write!(f, "[{}]:{}", self.ip(), self.port()), + scope_id => write!(f, "[{}%{}]:{}", self.ip(), scope_id, self.port()), + } + } else { + const LONGEST_IPV6_SOCKET_ADDR: &str = + "[ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%4294967296]:65536"; + + let mut buf = DisplayBuffer::<{ LONGEST_IPV6_SOCKET_ADDR.len() }>::new(); + match self.scope_id() { + 0 => write!(buf, "[{}]:{}", self.ip(), self.port()), + scope_id => write!(buf, "[{}%{}]:{}", self.ip(), scope_id, self.port()), + } + // Buffer is long enough for the longest possible IPv6 socket address, so this should never fail. + .unwrap(); + + f.pad(buf.as_str()) + } } } @@ -624,56 +675,44 @@ impl fmt::Debug for SocketAddrV6 { } } -#[stable(feature = "rust1", since = "1.0.0")] -impl Clone for SocketAddrV4 { - fn clone(&self) -> SocketAddrV4 { - *self +#[stable(feature = "socketaddr_ordering", since = "1.45.0")] +impl PartialOrd for SocketAddrV4 { + fn partial_cmp(&self, other: &SocketAddrV4) -> Option { + Some(self.cmp(other)) } } -#[stable(feature = "rust1", since = "1.0.0")] -impl Clone for SocketAddrV6 { - fn clone(&self) -> SocketAddrV6 { - *self + +#[stable(feature = "socketaddr_ordering", since = "1.45.0")] +impl PartialOrd for SocketAddrV6 { + fn partial_cmp(&self, other: &SocketAddrV6) -> Option { + Some(self.cmp(other)) } } -#[stable(feature = "rust1", since = "1.0.0")] -impl PartialEq for SocketAddrV4 { - fn eq(&self, other: &SocketAddrV4) -> bool { - self.inner.sin_port == other.inner.sin_port - && self.inner.sin_addr.s_addr == other.inner.sin_addr.s_addr +#[stable(feature = "socketaddr_ordering", since = "1.45.0")] +impl Ord for SocketAddrV4 { + fn cmp(&self, other: &SocketAddrV4) -> Ordering { + self.ip().cmp(other.ip()).then(self.port().cmp(&other.port())) } } -#[stable(feature = "rust1", since = "1.0.0")] -impl PartialEq for SocketAddrV6 { - fn eq(&self, other: &SocketAddrV6) -> bool { - self.inner.sin6_port == other.inner.sin6_port - && self.inner.sin6_addr.s6_addr == other.inner.sin6_addr.s6_addr - && self.inner.sin6_flowinfo == other.inner.sin6_flowinfo - && self.inner.sin6_scope_id == other.inner.sin6_scope_id + +#[stable(feature = "socketaddr_ordering", since = "1.45.0")] +impl Ord for SocketAddrV6 { + fn cmp(&self, other: &SocketAddrV6) -> Ordering { + self.ip().cmp(other.ip()).then(self.port().cmp(&other.port())) } } -#[stable(feature = "rust1", since = "1.0.0")] -impl Eq for SocketAddrV4 {} -#[stable(feature = "rust1", since = "1.0.0")] -impl Eq for SocketAddrV6 {} #[stable(feature = "rust1", since = "1.0.0")] impl hash::Hash for SocketAddrV4 { fn hash(&self, s: &mut H) { - (self.inner.sin_port, self.inner.sin_addr.s_addr).hash(s) + (self.port, self.ip).hash(s) } } #[stable(feature = "rust1", since = "1.0.0")] impl hash::Hash for SocketAddrV6 { fn hash(&self, s: &mut H) { - ( - self.inner.sin6_port, - &self.inner.sin6_addr.s6_addr, - self.inner.sin6_flowinfo, - self.inner.sin6_scope_id, - ) - .hash(s) + (self.port, &self.ip, self.flowinfo, self.scope_id).hash(s) } } @@ -685,15 +724,15 @@ impl hash::Hash for SocketAddrV6 { /// /// * [`SocketAddr`]: [`to_socket_addrs`] is the identity function. /// -/// * [`SocketAddrV4`], [`SocketAddrV6`], `(`[`IpAddr`]`, `[`u16`]`)`, -/// `(`[`Ipv4Addr`]`, `[`u16`]`)`, `(`[`Ipv6Addr`]`, `[`u16`]`)`: +/// * [`SocketAddrV4`], [`SocketAddrV6`], ([IpAddr], [u16]), +/// ([Ipv4Addr], [u16]), ([Ipv6Addr], [u16]): /// [`to_socket_addrs`] constructs a [`SocketAddr`] trivially. /// -/// * `(`[`&str`]`, `[`u16`]`)`: the string should be either a string representation +/// * (&[str], [u16]): &[str] should be either a string representation /// of an [`IpAddr`] address as expected by [`FromStr`] implementation or a host -/// name. +/// name. [`u16`] is the port number. /// -/// * [`&str`]: the string should be either a string representation of a +/// * &[str]: the string should be either a string representation of a /// [`SocketAddr`] as expected by its [`FromStr`] implementation or a string like /// `:` pair where `` is a [`u16`] value. /// @@ -709,18 +748,10 @@ impl hash::Hash for SocketAddrV6 { /// Addresses returned by the operating system that are not IP addresses are /// silently ignored. /// -/// [`FromStr`]: ../../std/str/trait.FromStr.html -/// [`IpAddr`]: ../../std/net/enum.IpAddr.html -/// [`Ipv4Addr`]: ../../std/net/struct.Ipv4Addr.html -/// [`Ipv6Addr`]: ../../std/net/struct.Ipv6Addr.html -/// [`SocketAddr`]: ../../std/net/enum.SocketAddr.html -/// [`SocketAddrV4`]: ../../std/net/struct.SocketAddrV4.html -/// [`SocketAddrV6`]: ../../std/net/struct.SocketAddrV6.html -/// [`&str`]: ../../std/primitive.str.html -/// [`TcpStream`]: ../../std/net/struct.TcpStream.html -/// [`to_socket_addrs`]: #tymethod.to_socket_addrs -/// [`UdpSocket`]: ../../std/net/struct.UdpSocket.html -/// [`u16`]: ../../std/primitive.u16.html +/// [`FromStr`]: crate::str::FromStr "std::str::FromStr" +/// [`TcpStream`]: crate::net::TcpStream "net::TcpStream" +/// [`to_socket_addrs`]: ToSocketAddrs::to_socket_addrs +/// [`UdpSocket`]: crate::net::UdpSocket "net::UdpSocket" /// /// # Examples /// @@ -791,7 +822,7 @@ impl hash::Hash for SocketAddrV6 { /// let stream = TcpStream::connect((Ipv4Addr::new(127, 0, 0, 1), 443)); /// ``` /// -/// [`TcpStream::connect`]: ../../std/net/struct.TcpStream.html#method.connect +/// [`TcpStream::connect`]: crate::net::TcpStream::connect #[stable(feature = "rust1", since = "1.0.0")] pub trait ToSocketAddrs { /// Returned iterator over socket addresses which this type may correspond @@ -799,9 +830,9 @@ pub trait ToSocketAddrs { #[stable(feature = "rust1", since = "1.0.0")] type Iter: Iterator; - /// Converts this object to an iterator of resolved `SocketAddr`s. + /// Converts this object to an iterator of resolved [`SocketAddr`]s. /// - /// The returned iterator may not actually yield any values depending on the + /// The returned iterator might not actually yield any values depending on the /// outcome of any resolution performed. /// /// Note that this function may block the current thread while resolution is @@ -895,6 +926,14 @@ impl ToSocketAddrs for (&str, u16) { } } +#[stable(feature = "string_u16_to_socket_addrs", since = "1.46.0")] +impl ToSocketAddrs for (String, u16) { + type Iter = vec::IntoIter; + fn to_socket_addrs(&self) -> io::Result> { + (&*self.0, self.1).to_socket_addrs() + } +} + // accepts strings like 'localhost:12345' #[stable(feature = "rust1", since = "1.0.0")] impl ToSocketAddrs for str { @@ -933,158 +972,3 @@ impl ToSocketAddrs for String { (&**self).to_socket_addrs() } } - -#[cfg(all(test, not(target_os = "emscripten")))] -mod tests { - use crate::net::test::{sa4, sa6, tsa}; - use crate::net::*; - - #[test] - fn to_socket_addr_ipaddr_u16() { - let a = Ipv4Addr::new(77, 88, 21, 11); - let p = 12345; - let e = SocketAddr::V4(SocketAddrV4::new(a, p)); - assert_eq!(Ok(vec![e]), tsa((a, p))); - } - - #[test] - fn to_socket_addr_str_u16() { - let a = sa4(Ipv4Addr::new(77, 88, 21, 11), 24352); - assert_eq!(Ok(vec![a]), tsa(("77.88.21.11", 24352))); - - let a = sa6(Ipv6Addr::new(0x2a02, 0x6b8, 0, 1, 0, 0, 0, 1), 53); - assert_eq!(Ok(vec![a]), tsa(("2a02:6b8:0:1::1", 53))); - - let a = sa4(Ipv4Addr::new(127, 0, 0, 1), 23924); - #[cfg(not(target_env = "sgx"))] - assert!(tsa(("localhost", 23924)).unwrap().contains(&a)); - #[cfg(target_env = "sgx")] - let _ = a; - } - - #[test] - fn to_socket_addr_str() { - let a = sa4(Ipv4Addr::new(77, 88, 21, 11), 24352); - assert_eq!(Ok(vec![a]), tsa("77.88.21.11:24352")); - - let a = sa6(Ipv6Addr::new(0x2a02, 0x6b8, 0, 1, 0, 0, 0, 1), 53); - assert_eq!(Ok(vec![a]), tsa("[2a02:6b8:0:1::1]:53")); - - let a = sa4(Ipv4Addr::new(127, 0, 0, 1), 23924); - #[cfg(not(target_env = "sgx"))] - assert!(tsa("localhost:23924").unwrap().contains(&a)); - #[cfg(target_env = "sgx")] - let _ = a; - } - - #[test] - fn to_socket_addr_string() { - let a = sa4(Ipv4Addr::new(77, 88, 21, 11), 24352); - assert_eq!(Ok(vec![a]), tsa(&*format!("{}:{}", "77.88.21.11", "24352"))); - assert_eq!(Ok(vec![a]), tsa(&format!("{}:{}", "77.88.21.11", "24352"))); - assert_eq!(Ok(vec![a]), tsa(format!("{}:{}", "77.88.21.11", "24352"))); - - let s = format!("{}:{}", "77.88.21.11", "24352"); - assert_eq!(Ok(vec![a]), tsa(s)); - // s has been moved into the tsa call - } - - // FIXME: figure out why this fails on openbsd and fix it - #[test] - #[cfg(not(any(windows, target_os = "openbsd")))] - fn to_socket_addr_str_bad() { - assert!(tsa("1200::AB00:1234::2552:7777:1313:34300").is_err()); - } - - #[test] - fn set_ip() { - fn ip4(low: u8) -> Ipv4Addr { - Ipv4Addr::new(77, 88, 21, low) - } - fn ip6(low: u16) -> Ipv6Addr { - Ipv6Addr::new(0x2a02, 0x6b8, 0, 1, 0, 0, 0, low) - } - - let mut v4 = SocketAddrV4::new(ip4(11), 80); - assert_eq!(v4.ip(), &ip4(11)); - v4.set_ip(ip4(12)); - assert_eq!(v4.ip(), &ip4(12)); - - let mut addr = SocketAddr::V4(v4); - assert_eq!(addr.ip(), IpAddr::V4(ip4(12))); - addr.set_ip(IpAddr::V4(ip4(13))); - assert_eq!(addr.ip(), IpAddr::V4(ip4(13))); - addr.set_ip(IpAddr::V6(ip6(14))); - assert_eq!(addr.ip(), IpAddr::V6(ip6(14))); - - let mut v6 = SocketAddrV6::new(ip6(1), 80, 0, 0); - assert_eq!(v6.ip(), &ip6(1)); - v6.set_ip(ip6(2)); - assert_eq!(v6.ip(), &ip6(2)); - - let mut addr = SocketAddr::V6(v6); - assert_eq!(addr.ip(), IpAddr::V6(ip6(2))); - addr.set_ip(IpAddr::V6(ip6(3))); - assert_eq!(addr.ip(), IpAddr::V6(ip6(3))); - addr.set_ip(IpAddr::V4(ip4(4))); - assert_eq!(addr.ip(), IpAddr::V4(ip4(4))); - } - - #[test] - fn set_port() { - let mut v4 = SocketAddrV4::new(Ipv4Addr::new(77, 88, 21, 11), 80); - assert_eq!(v4.port(), 80); - v4.set_port(443); - assert_eq!(v4.port(), 443); - - let mut addr = SocketAddr::V4(v4); - assert_eq!(addr.port(), 443); - addr.set_port(8080); - assert_eq!(addr.port(), 8080); - - let mut v6 = SocketAddrV6::new(Ipv6Addr::new(0x2a02, 0x6b8, 0, 1, 0, 0, 0, 1), 80, 0, 0); - assert_eq!(v6.port(), 80); - v6.set_port(443); - assert_eq!(v6.port(), 443); - - let mut addr = SocketAddr::V6(v6); - assert_eq!(addr.port(), 443); - addr.set_port(8080); - assert_eq!(addr.port(), 8080); - } - - #[test] - fn set_flowinfo() { - let mut v6 = SocketAddrV6::new(Ipv6Addr::new(0x2a02, 0x6b8, 0, 1, 0, 0, 0, 1), 80, 10, 0); - assert_eq!(v6.flowinfo(), 10); - v6.set_flowinfo(20); - assert_eq!(v6.flowinfo(), 20); - } - - #[test] - fn set_scope_id() { - let mut v6 = SocketAddrV6::new(Ipv6Addr::new(0x2a02, 0x6b8, 0, 1, 0, 0, 0, 1), 80, 0, 10); - assert_eq!(v6.scope_id(), 10); - v6.set_scope_id(20); - assert_eq!(v6.scope_id(), 20); - } - - #[test] - fn is_v4() { - let v4 = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(77, 88, 21, 11), 80)); - assert!(v4.is_ipv4()); - assert!(!v4.is_ipv6()); - } - - #[test] - fn is_v6() { - let v6 = SocketAddr::V6(SocketAddrV6::new( - Ipv6Addr::new(0x2a02, 0x6b8, 0, 1, 0, 0, 0, 1), - 80, - 10, - 0, - )); - assert!(!v6.is_ipv4()); - assert!(v6.is_ipv6()); - } -} diff --git a/crux-mir/lib/std/src/net/socket_addr/tests.rs b/crux-mir/lib/std/src/net/socket_addr/tests.rs new file mode 100644 index 000000000..dfc6dabbe --- /dev/null +++ b/crux-mir/lib/std/src/net/socket_addr/tests.rs @@ -0,0 +1,306 @@ +use crate::net::test::{sa4, sa6, tsa}; +use crate::net::*; + +#[test] +fn to_socket_addr_ipaddr_u16() { + let a = Ipv4Addr::new(77, 88, 21, 11); + let p = 12345; + let e = SocketAddr::V4(SocketAddrV4::new(a, p)); + assert_eq!(Ok(vec![e]), tsa((a, p))); +} + +#[test] +fn to_socket_addr_str_u16() { + let a = sa4(Ipv4Addr::new(77, 88, 21, 11), 24352); + assert_eq!(Ok(vec![a]), tsa(("77.88.21.11", 24352))); + + let a = sa6(Ipv6Addr::new(0x2a02, 0x6b8, 0, 1, 0, 0, 0, 1), 53); + assert_eq!(Ok(vec![a]), tsa(("2a02:6b8:0:1::1", 53))); + + let a = sa4(Ipv4Addr::new(127, 0, 0, 1), 23924); + #[cfg(not(target_env = "sgx"))] + assert!(tsa(("localhost", 23924)).unwrap().contains(&a)); + #[cfg(target_env = "sgx")] + let _ = a; +} + +#[test] +fn to_socket_addr_str() { + let a = sa4(Ipv4Addr::new(77, 88, 21, 11), 24352); + assert_eq!(Ok(vec![a]), tsa("77.88.21.11:24352")); + + let a = sa6(Ipv6Addr::new(0x2a02, 0x6b8, 0, 1, 0, 0, 0, 1), 53); + assert_eq!(Ok(vec![a]), tsa("[2a02:6b8:0:1::1]:53")); + + let a = sa4(Ipv4Addr::new(127, 0, 0, 1), 23924); + #[cfg(not(target_env = "sgx"))] + assert!(tsa("localhost:23924").unwrap().contains(&a)); + #[cfg(target_env = "sgx")] + let _ = a; +} + +#[test] +fn to_socket_addr_string() { + let a = sa4(Ipv4Addr::new(77, 88, 21, 11), 24352); + assert_eq!(Ok(vec![a]), tsa(&*format!("{}:{}", "77.88.21.11", "24352"))); + assert_eq!(Ok(vec![a]), tsa(&format!("{}:{}", "77.88.21.11", "24352"))); + assert_eq!(Ok(vec![a]), tsa(format!("{}:{}", "77.88.21.11", "24352"))); + + let s = format!("{}:{}", "77.88.21.11", "24352"); + assert_eq!(Ok(vec![a]), tsa(s)); + // s has been moved into the tsa call +} + +#[test] +fn ipv4_socket_addr_to_string() { + // Shortest possible IPv4 length. + assert_eq!(SocketAddrV4::new(Ipv4Addr::new(0, 0, 0, 0), 0).to_string(), "0.0.0.0:0"); + + // Longest possible IPv4 length. + assert_eq!( + SocketAddrV4::new(Ipv4Addr::new(255, 255, 255, 255), u16::MAX).to_string(), + "255.255.255.255:65535" + ); + + // Test padding. + assert_eq!( + format!("{:16}", SocketAddrV4::new(Ipv4Addr::new(1, 1, 1, 1), 53)), + "1.1.1.1:53 " + ); + assert_eq!( + format!("{:>16}", SocketAddrV4::new(Ipv4Addr::new(1, 1, 1, 1), 53)), + " 1.1.1.1:53" + ); +} + +#[test] +fn ipv6_socket_addr_to_string() { + // IPv4-mapped address. + assert_eq!( + SocketAddrV6::new(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc000, 0x280), 8080, 0, 0) + .to_string(), + "[::ffff:192.0.2.128]:8080" + ); + + // IPv4-compatible address. + assert_eq!( + SocketAddrV6::new(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0xc000, 0x280), 8080, 0, 0).to_string(), + "[::192.0.2.128]:8080" + ); + + // IPv6 address with no zero segments. + assert_eq!( + SocketAddrV6::new(Ipv6Addr::new(8, 9, 10, 11, 12, 13, 14, 15), 80, 0, 0).to_string(), + "[8:9:a:b:c:d:e:f]:80" + ); + + // Shortest possible IPv6 length. + assert_eq!(SocketAddrV6::new(Ipv6Addr::UNSPECIFIED, 0, 0, 0).to_string(), "[::]:0"); + + // Longest possible IPv6 length. + assert_eq!( + SocketAddrV6::new( + Ipv6Addr::new(0x1111, 0x2222, 0x3333, 0x4444, 0x5555, 0x6666, 0x7777, 0x8888), + u16::MAX, + u32::MAX, + u32::MAX, + ) + .to_string(), + "[1111:2222:3333:4444:5555:6666:7777:8888%4294967295]:65535" + ); + + // Test padding. + assert_eq!( + format!("{:22}", SocketAddrV6::new(Ipv6Addr::new(1, 2, 3, 4, 5, 6, 7, 8), 9, 0, 0)), + "[1:2:3:4:5:6:7:8]:9 " + ); + assert_eq!( + format!("{:>22}", SocketAddrV6::new(Ipv6Addr::new(1, 2, 3, 4, 5, 6, 7, 8), 9, 0, 0)), + " [1:2:3:4:5:6:7:8]:9" + ); +} + +#[test] +fn bind_udp_socket_bad() { + // rust-lang/rust#53957: This is a regression test for a parsing problem + // discovered as part of issue rust-lang/rust#23076, where we were + // incorrectly parsing invalid input and then that would result in a + // successful `UdpSocket` binding when we would expect failure. + // + // At one time, this test was written as a call to `tsa` with + // INPUT_23076. However, that structure yields an unreliable test, + // because it ends up passing junk input to the DNS server, and some DNS + // servers will respond with `Ok` to such input, with the ip address of + // the DNS server itself. + // + // This form of the test is more robust: even when the DNS server + // returns its own address, it is still an error to bind a UDP socket to + // a non-local address, and so we still get an error here in that case. + + const INPUT_23076: &str = "1200::AB00:1234::2552:7777:1313:34300"; + + assert!(crate::net::UdpSocket::bind(INPUT_23076).is_err()) +} + +#[test] +fn set_ip() { + fn ip4(low: u8) -> Ipv4Addr { + Ipv4Addr::new(77, 88, 21, low) + } + fn ip6(low: u16) -> Ipv6Addr { + Ipv6Addr::new(0x2a02, 0x6b8, 0, 1, 0, 0, 0, low) + } + + let mut v4 = SocketAddrV4::new(ip4(11), 80); + assert_eq!(v4.ip(), &ip4(11)); + v4.set_ip(ip4(12)); + assert_eq!(v4.ip(), &ip4(12)); + + let mut addr = SocketAddr::V4(v4); + assert_eq!(addr.ip(), IpAddr::V4(ip4(12))); + addr.set_ip(IpAddr::V4(ip4(13))); + assert_eq!(addr.ip(), IpAddr::V4(ip4(13))); + addr.set_ip(IpAddr::V6(ip6(14))); + assert_eq!(addr.ip(), IpAddr::V6(ip6(14))); + + let mut v6 = SocketAddrV6::new(ip6(1), 80, 0, 0); + assert_eq!(v6.ip(), &ip6(1)); + v6.set_ip(ip6(2)); + assert_eq!(v6.ip(), &ip6(2)); + + let mut addr = SocketAddr::V6(v6); + assert_eq!(addr.ip(), IpAddr::V6(ip6(2))); + addr.set_ip(IpAddr::V6(ip6(3))); + assert_eq!(addr.ip(), IpAddr::V6(ip6(3))); + addr.set_ip(IpAddr::V4(ip4(4))); + assert_eq!(addr.ip(), IpAddr::V4(ip4(4))); +} + +#[test] +fn set_port() { + let mut v4 = SocketAddrV4::new(Ipv4Addr::new(77, 88, 21, 11), 80); + assert_eq!(v4.port(), 80); + v4.set_port(443); + assert_eq!(v4.port(), 443); + + let mut addr = SocketAddr::V4(v4); + assert_eq!(addr.port(), 443); + addr.set_port(8080); + assert_eq!(addr.port(), 8080); + + let mut v6 = SocketAddrV6::new(Ipv6Addr::new(0x2a02, 0x6b8, 0, 1, 0, 0, 0, 1), 80, 0, 0); + assert_eq!(v6.port(), 80); + v6.set_port(443); + assert_eq!(v6.port(), 443); + + let mut addr = SocketAddr::V6(v6); + assert_eq!(addr.port(), 443); + addr.set_port(8080); + assert_eq!(addr.port(), 8080); +} + +#[test] +fn set_flowinfo() { + let mut v6 = SocketAddrV6::new(Ipv6Addr::new(0x2a02, 0x6b8, 0, 1, 0, 0, 0, 1), 80, 10, 0); + assert_eq!(v6.flowinfo(), 10); + v6.set_flowinfo(20); + assert_eq!(v6.flowinfo(), 20); +} + +#[test] +fn set_scope_id() { + let mut v6 = SocketAddrV6::new(Ipv6Addr::new(0x2a02, 0x6b8, 0, 1, 0, 0, 0, 1), 80, 0, 10); + assert_eq!(v6.scope_id(), 10); + v6.set_scope_id(20); + assert_eq!(v6.scope_id(), 20); +} + +#[test] +fn is_v4() { + let v4 = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(77, 88, 21, 11), 80)); + assert!(v4.is_ipv4()); + assert!(!v4.is_ipv6()); +} + +#[test] +fn is_v6() { + let v6 = SocketAddr::V6(SocketAddrV6::new( + Ipv6Addr::new(0x2a02, 0x6b8, 0, 1, 0, 0, 0, 1), + 80, + 10, + 0, + )); + assert!(!v6.is_ipv4()); + assert!(v6.is_ipv6()); +} + +#[test] +fn socket_v4_to_str() { + let socket = SocketAddrV4::new(Ipv4Addr::new(192, 168, 0, 1), 8080); + + assert_eq!(format!("{socket}"), "192.168.0.1:8080"); + assert_eq!(format!("{socket:<20}"), "192.168.0.1:8080 "); + assert_eq!(format!("{socket:>20}"), " 192.168.0.1:8080"); + assert_eq!(format!("{socket:^20}"), " 192.168.0.1:8080 "); + assert_eq!(format!("{socket:.10}"), "192.168.0."); +} + +#[test] +fn socket_v6_to_str() { + let mut socket = SocketAddrV6::new(Ipv6Addr::new(0x2a02, 0x6b8, 0, 1, 0, 0, 0, 1), 53, 0, 0); + + assert_eq!(format!("{socket}"), "[2a02:6b8:0:1::1]:53"); + assert_eq!(format!("{socket:<24}"), "[2a02:6b8:0:1::1]:53 "); + assert_eq!(format!("{socket:>24}"), " [2a02:6b8:0:1::1]:53"); + assert_eq!(format!("{socket:^24}"), " [2a02:6b8:0:1::1]:53 "); + assert_eq!(format!("{socket:.15}"), "[2a02:6b8:0:1::"); + + socket.set_scope_id(5); + + assert_eq!(format!("{socket}"), "[2a02:6b8:0:1::1%5]:53"); + assert_eq!(format!("{socket:<24}"), "[2a02:6b8:0:1::1%5]:53 "); + assert_eq!(format!("{socket:>24}"), " [2a02:6b8:0:1::1%5]:53"); + assert_eq!(format!("{socket:^24}"), " [2a02:6b8:0:1::1%5]:53 "); + assert_eq!(format!("{socket:.18}"), "[2a02:6b8:0:1::1%5"); +} + +#[test] +fn compare() { + let v4_1 = "224.120.45.1:23456".parse::().unwrap(); + let v4_2 = "224.210.103.5:12345".parse::().unwrap(); + let v4_3 = "224.210.103.5:23456".parse::().unwrap(); + let v6_1 = "[2001:db8:f00::1002]:23456".parse::().unwrap(); + let v6_2 = "[2001:db8:f00::2001]:12345".parse::().unwrap(); + let v6_3 = "[2001:db8:f00::2001]:23456".parse::().unwrap(); + + // equality + assert_eq!(v4_1, v4_1); + assert_eq!(v6_1, v6_1); + assert_eq!(SocketAddr::V4(v4_1), SocketAddr::V4(v4_1)); + assert_eq!(SocketAddr::V6(v6_1), SocketAddr::V6(v6_1)); + assert!(v4_1 != v4_2); + assert!(v6_1 != v6_2); + + // compare different addresses + assert!(v4_1 < v4_2); + assert!(v6_1 < v6_2); + assert!(v4_2 > v4_1); + assert!(v6_2 > v6_1); + + // compare the same address with different ports + assert!(v4_2 < v4_3); + assert!(v6_2 < v6_3); + assert!(v4_3 > v4_2); + assert!(v6_3 > v6_2); + + // compare different addresses with the same port + assert!(v4_1 < v4_3); + assert!(v6_1 < v6_3); + assert!(v4_3 > v4_1); + assert!(v6_3 > v6_1); + + // compare with an inferred right-hand side + assert_eq!(v4_1, "224.120.45.1:23456".parse().unwrap()); + assert_eq!(v6_1, "[2001:db8:f00::1002]:23456".parse().unwrap()); + assert_eq!(SocketAddr::V4(v4_1), "224.120.45.1:23456".parse().unwrap()); +} diff --git a/crux-mir/lib/std/src/net/tcp.rs b/crux-mir/lib/std/src/net/tcp.rs index 5023d6924..69b72a81c 100644 --- a/crux-mir/lib/std/src/net/tcp.rs +++ b/crux-mir/lib/std/src/net/tcp.rs @@ -1,7 +1,13 @@ +#![deny(unsafe_op_in_unsafe_fn)] + +#[cfg(all(test, not(target_os = "emscripten")))] +mod tests; + use crate::io::prelude::*; use crate::fmt; -use crate::io::{self, Initializer, IoSlice, IoSliceMut}; +use crate::io::{self, IoSlice, IoSliceMut}; +use crate::iter::FusedIterator; use crate::net::{Shutdown, SocketAddr, ToSocketAddrs}; use crate::sys_common::net as net_imp; use crate::sys_common::{AsInner, FromInner, IntoInner}; @@ -19,13 +25,12 @@ use crate::time::Duration; /// /// The Transmission Control Protocol is specified in [IETF RFC 793]. /// -/// [`accept`]: ../../std/net/struct.TcpListener.html#method.accept -/// [`connect`]: #method.connect +/// [`accept`]: TcpListener::accept +/// [`connect`]: TcpStream::connect /// [IETF RFC 793]: https://tools.ietf.org/html/rfc793 -/// [reading]: ../../std/io/trait.Read.html -/// [`shutdown`]: #method.shutdown -/// [`TcpListener`]: ../../std/net/struct.TcpListener.html -/// [writing]: ../../std/io/trait.Write.html +/// [reading]: Read +/// [`shutdown`]: TcpStream::shutdown +/// [writing]: Write /// /// # Examples /// @@ -54,11 +59,9 @@ pub struct TcpStream(net_imp::TcpStream); /// /// The Transmission Control Protocol is specified in [IETF RFC 793]. /// -/// [`accept`]: #method.accept -/// [`bind`]: #method.bind +/// [`accept`]: TcpListener::accept +/// [`bind`]: TcpListener::bind /// [IETF RFC 793]: https://tools.ietf.org/html/rfc793 -/// [`Incoming`]: ../../std/net/struct.Incoming.html -/// [`TcpListener::incoming`]: #method.incoming /// /// # Examples /// @@ -84,18 +87,29 @@ pub struct TcpListener(net_imp::TcpListener); /// An iterator that infinitely [`accept`]s connections on a [`TcpListener`]. /// -/// This `struct` is created by the [`incoming`] method on [`TcpListener`]. +/// This `struct` is created by the [`TcpListener::incoming`] method. /// See its documentation for more. /// -/// [`accept`]: ../../std/net/struct.TcpListener.html#method.accept -/// [`incoming`]: ../../std/net/struct.TcpListener.html#method.incoming -/// [`TcpListener`]: ../../std/net/struct.TcpListener.html +/// [`accept`]: TcpListener::accept +#[must_use = "iterators are lazy and do nothing unless consumed"] #[stable(feature = "rust1", since = "1.0.0")] #[derive(Debug)] pub struct Incoming<'a> { listener: &'a TcpListener, } +/// An iterator that infinitely [`accept`]s connections on a [`TcpListener`]. +/// +/// This `struct` is created by the [`TcpListener::into_incoming`] method. +/// See its documentation for more. +/// +/// [`accept`]: TcpListener::accept +#[derive(Debug)] +#[unstable(feature = "tcplistener_into_incoming", issue = "88339")] +pub struct IntoIncoming { + listener: TcpListener, +} + impl TcpStream { /// Opens a TCP connection to a remote host. /// @@ -108,8 +122,6 @@ impl TcpStream { /// the addresses result in a successful connection, the error returned from /// the last connection attempt (the last address) is returned. /// - /// [`ToSocketAddrs`]: ../../std/net/trait.ToSocketAddrs.html - /// /// # Examples /// /// Open a TCP connection to `127.0.0.1:8080`: @@ -156,8 +168,6 @@ impl TcpStream { /// single system call. It instead calls `connect` in nonblocking mode and /// then uses an OS-specific mechanism to await the completion of the /// connection request. - /// - /// [`SocketAddr`]: ../../std/net/enum.SocketAddr.html #[stable(feature = "tcpstream_connect_timeout", since = "1.21.0")] pub fn connect_timeout(addr: &SocketAddr, timeout: Duration) -> io::Result { net_imp::TcpStream::connect_timeout(addr, timeout).map(TcpStream) @@ -203,8 +213,6 @@ impl TcpStream { /// portions to return immediately with an appropriate value (see the /// documentation of [`Shutdown`]). /// - /// [`Shutdown`]: ../../std/net/enum.Shutdown.html - /// /// # Platform-specific behavior /// /// Calling this function multiple times may result in different behavior, @@ -259,12 +267,9 @@ impl TcpStream { /// a result of setting this option. For example Unix typically returns an /// error of the kind [`WouldBlock`], but Windows may return [`TimedOut`]. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// [`Err`]: ../../std/result/enum.Result.html#variant.Err - /// [`read`]: ../../std/io/trait.Read.html#tymethod.read - /// [`WouldBlock`]: ../../std/io/enum.ErrorKind.html#variant.WouldBlock - /// [`TimedOut`]: ../../std/io/enum.ErrorKind.html#variant.TimedOut - /// [`Duration`]: ../../std/time/struct.Duration.html + /// [`read`]: Read::read + /// [`WouldBlock`]: io::ErrorKind::WouldBlock + /// [`TimedOut`]: io::ErrorKind::TimedOut /// /// # Examples /// @@ -306,12 +311,9 @@ impl TcpStream { /// as a result of setting this option. For example Unix typically returns /// an error of the kind [`WouldBlock`], but Windows may return [`TimedOut`]. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// [`Err`]: ../../std/result/enum.Result.html#variant.Err - /// [`write`]: ../../std/io/trait.Write.html#tymethod.write - /// [`Duration`]: ../../std/time/struct.Duration.html - /// [`WouldBlock`]: ../../std/io/enum.ErrorKind.html#variant.WouldBlock - /// [`TimedOut`]: ../../std/io/enum.ErrorKind.html#variant.TimedOut + /// [`write`]: Write::write + /// [`WouldBlock`]: io::ErrorKind::WouldBlock + /// [`TimedOut`]: io::ErrorKind::TimedOut /// /// # Examples /// @@ -349,8 +351,7 @@ impl TcpStream { /// /// Some platforms do not provide access to the current timeout. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// [`read`]: ../../std/io/trait.Read.html#tymethod.read + /// [`read`]: Read::read /// /// # Examples /// @@ -375,8 +376,7 @@ impl TcpStream { /// /// Some platforms do not provide access to the current timeout. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// [`write`]: ../../std/io/trait.Write.html#tymethod.write + /// [`write`]: Write::write /// /// # Examples /// @@ -406,7 +406,7 @@ impl TcpStream { /// use std::net::TcpStream; /// /// let stream = TcpStream::connect("127.0.0.1:8000") - /// .expect("couldn't bind to address"); + /// .expect("Couldn't connect to the server..."); /// let mut buf = [0; 10]; /// let len = stream.peek(&mut buf).expect("peek failed"); /// ``` @@ -415,6 +415,53 @@ impl TcpStream { self.0.peek(buf) } + /// Sets the value of the `SO_LINGER` option on this socket. + /// + /// This value controls how the socket is closed when data remains + /// to be sent. If `SO_LINGER` is set, the socket will remain open + /// for the specified duration as the system attempts to send pending data. + /// Otherwise, the system may close the socket immediately, or wait for a + /// default timeout. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(tcp_linger)] + /// + /// use std::net::TcpStream; + /// use std::time::Duration; + /// + /// let stream = TcpStream::connect("127.0.0.1:8080") + /// .expect("Couldn't connect to the server..."); + /// stream.set_linger(Some(Duration::from_secs(0))).expect("set_linger call failed"); + /// ``` + #[unstable(feature = "tcp_linger", issue = "88494")] + pub fn set_linger(&self, linger: Option) -> io::Result<()> { + self.0.set_linger(linger) + } + + /// Gets the value of the `SO_LINGER` option on this socket. + /// + /// For more information about this option, see [`TcpStream::set_linger`]. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(tcp_linger)] + /// + /// use std::net::TcpStream; + /// use std::time::Duration; + /// + /// let stream = TcpStream::connect("127.0.0.1:8080") + /// .expect("Couldn't connect to the server..."); + /// stream.set_linger(Some(Duration::from_secs(0))).expect("set_linger call failed"); + /// assert_eq!(stream.linger().unwrap(), Some(Duration::from_secs(0))); + /// ``` + #[unstable(feature = "tcp_linger", issue = "88494")] + pub fn linger(&self) -> io::Result> { + self.0.linger() + } + /// Sets the value of the `TCP_NODELAY` option on this socket. /// /// If set, this option disables the Nagle algorithm. This means that @@ -439,9 +486,7 @@ impl TcpStream { /// Gets the value of the `TCP_NODELAY` option on this socket. /// - /// For more information about this option, see [`set_nodelay`][link]. - /// - /// [link]: #method.set_nodelay + /// For more information about this option, see [`TcpStream::set_nodelay`]. /// /// # Examples /// @@ -479,9 +524,7 @@ impl TcpStream { /// Gets the value of the `IP_TTL` option for this socket. /// - /// For more information about this option, see [`set_ttl`][link]. - /// - /// [link]: #method.set_ttl + /// For more information about this option, see [`TcpStream::set_ttl`]. /// /// # Examples /// @@ -553,19 +596,23 @@ impl TcpStream { /// // via platform-specific APIs such as epoll or IOCP /// wait_for_fd(); /// } - /// Err(e) => panic!("encountered IO error: {}", e), + /// Err(e) => panic!("encountered IO error: {e}"), /// }; /// }; - /// println!("bytes: {:?}", buf); + /// println!("bytes: {buf:?}"); /// ``` - /// - /// [`io::ErrorKind::WouldBlock`]: ../io/enum.ErrorKind.html#variant.WouldBlock #[stable(feature = "net2_mutators", since = "1.9.0")] pub fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()> { self.0.set_nonblocking(nonblocking) } } +// In addition to the `impl`s here, `TcpStream` also has `impl`s for +// `AsFd`/`From`/`Into` and +// `AsRawFd`/`IntoRawFd`/`FromRawFd`, on Unix and WASI, and +// `AsSocket`/`From`/`Into` and +// `AsRawSocket`/`IntoRawSocket`/`FromRawSocket` on Windows. + #[stable(feature = "rust1", since = "1.0.0")] impl Read for TcpStream { fn read(&mut self, buf: &mut [u8]) -> io::Result { @@ -577,8 +624,8 @@ impl Read for TcpStream { } #[inline] - unsafe fn initializer(&self) -> Initializer { - Initializer::nop() + fn is_read_vectored(&self) -> bool { + self.0.is_read_vectored() } } #[stable(feature = "rust1", since = "1.0.0")] @@ -591,6 +638,11 @@ impl Write for TcpStream { self.0.write_vectored(bufs) } + #[inline] + fn is_write_vectored(&self) -> bool { + self.0.is_write_vectored() + } + fn flush(&mut self) -> io::Result<()> { Ok(()) } @@ -606,8 +658,8 @@ impl Read for &TcpStream { } #[inline] - unsafe fn initializer(&self) -> Initializer { - Initializer::nop() + fn is_read_vectored(&self) -> bool { + self.0.is_read_vectored() } } #[stable(feature = "rust1", since = "1.0.0")] @@ -620,6 +672,11 @@ impl Write for &TcpStream { self.0.write_vectored(bufs) } + #[inline] + fn is_write_vectored(&self) -> bool { + self.0.is_write_vectored() + } + fn flush(&mut self) -> io::Result<()> { Ok(()) } @@ -658,7 +715,7 @@ impl TcpListener { /// /// Binding with a port number of 0 will request that the OS assigns a port /// to this listener. The port allocated can be queried via the - /// [`local_addr`] method. + /// [`TcpListener::local_addr`] method. /// /// The address type can be any implementor of [`ToSocketAddrs`] trait. See /// its documentation for concrete examples. @@ -668,9 +725,6 @@ impl TcpListener { /// none of the addresses succeed in creating a listener, the error returned /// from the last attempt (the last address) is returned. /// - /// [`local_addr`]: #method.local_addr - /// [`ToSocketAddrs`]: ../../std/net/trait.ToSocketAddrs.html - /// /// # Examples /// /// Creates a TCP listener bound to `127.0.0.1:80`: @@ -720,8 +774,6 @@ impl TcpListener { /// object references. Both handles can be used to accept incoming /// connections and options set on one listener will affect the other. /// - /// [`TcpListener`]: ../../std/net/struct.TcpListener.html - /// /// # Examples /// /// ```no_run @@ -741,8 +793,6 @@ impl TcpListener { /// is established. When established, the corresponding [`TcpStream`] and the /// remote peer's address will be returned. /// - /// [`TcpStream`]: ../../std/net/struct.TcpStream.html - /// /// # Examples /// /// ```no_run @@ -750,8 +800,8 @@ impl TcpListener { /// /// let listener = TcpListener::bind("127.0.0.1:8080").unwrap(); /// match listener.accept() { - /// Ok((_socket, addr)) => println!("new client: {:?}", addr), - /// Err(e) => println!("couldn't get client: {:?}", e), + /// Ok((_socket, addr)) => println!("new client: {addr:?}"), + /// Err(e) => println!("couldn't get client: {e:?}"), /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] @@ -767,26 +817,29 @@ impl TcpListener { /// /// The returned iterator will never return [`None`] and will also not yield /// the peer's [`SocketAddr`] structure. Iterating over it is equivalent to - /// calling [`accept`] in a loop. - /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// [`SocketAddr`]: ../../std/net/enum.SocketAddr.html - /// [`accept`]: #method.accept + /// calling [`TcpListener::accept`] in a loop. /// /// # Examples /// /// ```no_run - /// use std::net::TcpListener; + /// use std::net::{TcpListener, TcpStream}; /// - /// let listener = TcpListener::bind("127.0.0.1:80").unwrap(); + /// fn handle_connection(stream: TcpStream) { + /// //... + /// } /// - /// for stream in listener.incoming() { - /// match stream { - /// Ok(stream) => { - /// println!("new client!"); + /// fn main() -> std::io::Result<()> { + /// let listener = TcpListener::bind("127.0.0.1:80").unwrap(); + /// + /// for stream in listener.incoming() { + /// match stream { + /// Ok(stream) => { + /// handle_connection(stream); + /// } + /// Err(e) => { /* connection failed */ } /// } - /// Err(e) => { /* connection failed */ } /// } + /// Ok(()) /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] @@ -794,6 +847,38 @@ impl TcpListener { Incoming { listener: self } } + /// Turn this into an iterator over the connections being received on this + /// listener. + /// + /// The returned iterator will never return [`None`] and will also not yield + /// the peer's [`SocketAddr`] structure. Iterating over it is equivalent to + /// calling [`TcpListener::accept`] in a loop. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(tcplistener_into_incoming)] + /// use std::net::{TcpListener, TcpStream}; + /// + /// fn listen_on(port: u16) -> impl Iterator { + /// let listener = TcpListener::bind("127.0.0.1:80").unwrap(); + /// listener.into_incoming() + /// .filter_map(Result::ok) /* Ignore failed connections */ + /// } + /// + /// fn main() -> std::io::Result<()> { + /// for stream in listen_on(80) { + /// /* handle the connection here */ + /// } + /// Ok(()) + /// } + /// ``` + #[must_use = "`self` will be dropped if the result is not used"] + #[unstable(feature = "tcplistener_into_incoming", issue = "88339")] + pub fn into_incoming(self) -> IntoIncoming { + IntoIncoming { listener: self } + } + /// Sets the value for the `IP_TTL` option on this socket. /// /// This value sets the time-to-live field that is used in every packet sent @@ -814,9 +899,7 @@ impl TcpListener { /// Gets the value of the `IP_TTL` option for this socket. /// - /// For more information about this option, see [`set_ttl`][link]. - /// - /// [link]: #method.set_ttl + /// For more information about this option, see [`TcpListener::set_ttl`]. /// /// # Examples /// @@ -833,20 +916,14 @@ impl TcpListener { } #[stable(feature = "net2_mutators", since = "1.9.0")] - #[rustc_deprecated( - since = "1.16.0", - reason = "this option can only be set before the socket is bound" - )] + #[deprecated(since = "1.16.0", note = "this option can only be set before the socket is bound")] #[allow(missing_docs)] pub fn set_only_v6(&self, only_v6: bool) -> io::Result<()> { self.0.set_only_v6(only_v6) } #[stable(feature = "net2_mutators", since = "1.9.0")] - #[rustc_deprecated( - since = "1.16.0", - reason = "this option can only be set before the socket is bound" - )] + #[deprecated(since = "1.16.0", note = "this option can only be set before the socket is bound")] #[allow(missing_docs)] pub fn only_v6(&self) -> io::Result { self.0.only_v6() @@ -909,18 +986,22 @@ impl TcpListener { /// wait_for_fd(); /// continue; /// } - /// Err(e) => panic!("encountered IO error: {}", e), + /// Err(e) => panic!("encountered IO error: {e}"), /// } /// } /// ``` - /// - /// [`io::ErrorKind::WouldBlock`]: ../io/enum.ErrorKind.html#variant.WouldBlock #[stable(feature = "net2_mutators", since = "1.9.0")] pub fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()> { self.0.set_nonblocking(nonblocking) } } +// In addition to the `impl`s here, `TcpListener` also has `impl`s for +// `AsFd`/`From`/`Into` and +// `AsRawFd`/`IntoRawFd`/`FromRawFd`, on Unix and WASI, and +// `AsSocket`/`From`/`Into` and +// `AsRawSocket`/`IntoRawSocket`/`FromRawSocket` on Windows. + #[stable(feature = "rust1", since = "1.0.0")] impl<'a> Iterator for Incoming<'a> { type Item = io::Result; @@ -929,6 +1010,20 @@ impl<'a> Iterator for Incoming<'a> { } } +#[stable(feature = "tcp_listener_incoming_fused_iterator", since = "1.64.0")] +impl FusedIterator for Incoming<'_> {} + +#[unstable(feature = "tcplistener_into_incoming", issue = "88339")] +impl Iterator for IntoIncoming { + type Item = io::Result; + fn next(&mut self) -> Option> { + Some(self.listener.accept().map(|p| p.0)) + } +} + +#[unstable(feature = "tcplistener_into_incoming", issue = "88339")] +impl FusedIterator for IntoIncoming {} + impl AsInner for TcpListener { fn as_inner(&self) -> &net_imp::TcpListener { &self.0 @@ -953,869 +1048,3 @@ impl fmt::Debug for TcpListener { self.0.fmt(f) } } - -#[cfg(all(test, not(any(target_os = "cloudabi", target_os = "emscripten"))))] -mod tests { - use crate::fmt; - use crate::io::prelude::*; - use crate::io::{ErrorKind, IoSlice, IoSliceMut}; - use crate::net::test::{next_test_ip4, next_test_ip6}; - use crate::net::*; - use crate::sync::mpsc::channel; - use crate::thread; - use crate::time::{Duration, Instant}; - - fn each_ip(f: &mut dyn FnMut(SocketAddr)) { - f(next_test_ip4()); - f(next_test_ip6()); - } - - macro_rules! t { - ($e:expr) => { - match $e { - Ok(t) => t, - Err(e) => panic!("received error for `{}`: {}", stringify!($e), e), - } - }; - } - - #[test] - fn bind_error() { - match TcpListener::bind("1.1.1.1:9999") { - Ok(..) => panic!(), - Err(e) => assert_eq!(e.kind(), ErrorKind::AddrNotAvailable), - } - } - - #[test] - fn connect_error() { - match TcpStream::connect("0.0.0.0:1") { - Ok(..) => panic!(), - Err(e) => assert!( - e.kind() == ErrorKind::ConnectionRefused - || e.kind() == ErrorKind::InvalidInput - || e.kind() == ErrorKind::AddrInUse - || e.kind() == ErrorKind::AddrNotAvailable, - "bad error: {} {:?}", - e, - e.kind() - ), - } - } - - #[test] - fn listen_localhost() { - let socket_addr = next_test_ip4(); - let listener = t!(TcpListener::bind(&socket_addr)); - - let _t = thread::spawn(move || { - let mut stream = t!(TcpStream::connect(&("localhost", socket_addr.port()))); - t!(stream.write(&[144])); - }); - - let mut stream = t!(listener.accept()).0; - let mut buf = [0]; - t!(stream.read(&mut buf)); - assert!(buf[0] == 144); - } - - #[test] - fn connect_loopback() { - each_ip(&mut |addr| { - let acceptor = t!(TcpListener::bind(&addr)); - - let _t = thread::spawn(move || { - let host = match addr { - SocketAddr::V4(..) => "127.0.0.1", - SocketAddr::V6(..) => "::1", - }; - let mut stream = t!(TcpStream::connect(&(host, addr.port()))); - t!(stream.write(&[66])); - }); - - let mut stream = t!(acceptor.accept()).0; - let mut buf = [0]; - t!(stream.read(&mut buf)); - assert!(buf[0] == 66); - }) - } - - #[test] - fn smoke_test() { - each_ip(&mut |addr| { - let acceptor = t!(TcpListener::bind(&addr)); - - let (tx, rx) = channel(); - let _t = thread::spawn(move || { - let mut stream = t!(TcpStream::connect(&addr)); - t!(stream.write(&[99])); - tx.send(t!(stream.local_addr())).unwrap(); - }); - - let (mut stream, addr) = t!(acceptor.accept()); - let mut buf = [0]; - t!(stream.read(&mut buf)); - assert!(buf[0] == 99); - assert_eq!(addr, t!(rx.recv())); - }) - } - - #[test] - fn read_eof() { - each_ip(&mut |addr| { - let acceptor = t!(TcpListener::bind(&addr)); - - let _t = thread::spawn(move || { - let _stream = t!(TcpStream::connect(&addr)); - // Close - }); - - let mut stream = t!(acceptor.accept()).0; - let mut buf = [0]; - let nread = t!(stream.read(&mut buf)); - assert_eq!(nread, 0); - let nread = t!(stream.read(&mut buf)); - assert_eq!(nread, 0); - }) - } - - #[test] - fn write_close() { - each_ip(&mut |addr| { - let acceptor = t!(TcpListener::bind(&addr)); - - let (tx, rx) = channel(); - let _t = thread::spawn(move || { - drop(t!(TcpStream::connect(&addr))); - tx.send(()).unwrap(); - }); - - let mut stream = t!(acceptor.accept()).0; - rx.recv().unwrap(); - let buf = [0]; - match stream.write(&buf) { - Ok(..) => {} - Err(e) => { - assert!( - e.kind() == ErrorKind::ConnectionReset - || e.kind() == ErrorKind::BrokenPipe - || e.kind() == ErrorKind::ConnectionAborted, - "unknown error: {}", - e - ); - } - } - }) - } - - #[test] - fn multiple_connect_serial() { - each_ip(&mut |addr| { - let max = 10; - let acceptor = t!(TcpListener::bind(&addr)); - - let _t = thread::spawn(move || { - for _ in 0..max { - let mut stream = t!(TcpStream::connect(&addr)); - t!(stream.write(&[99])); - } - }); - - for stream in acceptor.incoming().take(max) { - let mut stream = t!(stream); - let mut buf = [0]; - t!(stream.read(&mut buf)); - assert_eq!(buf[0], 99); - } - }) - } - - #[test] - fn multiple_connect_interleaved_greedy_schedule() { - const MAX: usize = 10; - each_ip(&mut |addr| { - let acceptor = t!(TcpListener::bind(&addr)); - - let _t = thread::spawn(move || { - let acceptor = acceptor; - for (i, stream) in acceptor.incoming().enumerate().take(MAX) { - // Start another thread to handle the connection - let _t = thread::spawn(move || { - let mut stream = t!(stream); - let mut buf = [0]; - t!(stream.read(&mut buf)); - assert!(buf[0] == i as u8); - }); - } - }); - - connect(0, addr); - }); - - fn connect(i: usize, addr: SocketAddr) { - if i == MAX { - return; - } - - let t = thread::spawn(move || { - let mut stream = t!(TcpStream::connect(&addr)); - // Connect again before writing - connect(i + 1, addr); - t!(stream.write(&[i as u8])); - }); - t.join().ok().expect("thread panicked"); - } - } - - #[test] - fn multiple_connect_interleaved_lazy_schedule() { - const MAX: usize = 10; - each_ip(&mut |addr| { - let acceptor = t!(TcpListener::bind(&addr)); - - let _t = thread::spawn(move || { - for stream in acceptor.incoming().take(MAX) { - // Start another thread to handle the connection - let _t = thread::spawn(move || { - let mut stream = t!(stream); - let mut buf = [0]; - t!(stream.read(&mut buf)); - assert!(buf[0] == 99); - }); - } - }); - - connect(0, addr); - }); - - fn connect(i: usize, addr: SocketAddr) { - if i == MAX { - return; - } - - let t = thread::spawn(move || { - let mut stream = t!(TcpStream::connect(&addr)); - connect(i + 1, addr); - t!(stream.write(&[99])); - }); - t.join().ok().expect("thread panicked"); - } - } - - #[test] - fn socket_and_peer_name() { - each_ip(&mut |addr| { - let listener = t!(TcpListener::bind(&addr)); - let so_name = t!(listener.local_addr()); - assert_eq!(addr, so_name); - let _t = thread::spawn(move || { - t!(listener.accept()); - }); - - let stream = t!(TcpStream::connect(&addr)); - assert_eq!(addr, t!(stream.peer_addr())); - }) - } - - #[test] - fn partial_read() { - each_ip(&mut |addr| { - let (tx, rx) = channel(); - let srv = t!(TcpListener::bind(&addr)); - let _t = thread::spawn(move || { - let mut cl = t!(srv.accept()).0; - cl.write(&[10]).unwrap(); - let mut b = [0]; - t!(cl.read(&mut b)); - tx.send(()).unwrap(); - }); - - let mut c = t!(TcpStream::connect(&addr)); - let mut b = [0; 10]; - assert_eq!(c.read(&mut b).unwrap(), 1); - t!(c.write(&[1])); - rx.recv().unwrap(); - }) - } - - #[test] - fn read_vectored() { - each_ip(&mut |addr| { - let srv = t!(TcpListener::bind(&addr)); - let mut s1 = t!(TcpStream::connect(&addr)); - let mut s2 = t!(srv.accept()).0; - - let len = s1.write(&[10, 11, 12]).unwrap(); - assert_eq!(len, 3); - - let mut a = []; - let mut b = [0]; - let mut c = [0; 3]; - let len = t!(s2.read_vectored(&mut [ - IoSliceMut::new(&mut a), - IoSliceMut::new(&mut b), - IoSliceMut::new(&mut c) - ],)); - assert!(len > 0); - assert_eq!(b, [10]); - // some implementations don't support readv, so we may only fill the first buffer - assert!(len == 1 || c == [11, 12, 0]); - }) - } - - #[test] - fn write_vectored() { - each_ip(&mut |addr| { - let srv = t!(TcpListener::bind(&addr)); - let mut s1 = t!(TcpStream::connect(&addr)); - let mut s2 = t!(srv.accept()).0; - - let a = []; - let b = [10]; - let c = [11, 12]; - t!(s1.write_vectored(&[IoSlice::new(&a), IoSlice::new(&b), IoSlice::new(&c)])); - - let mut buf = [0; 4]; - let len = t!(s2.read(&mut buf)); - // some implementations don't support writev, so we may only write the first buffer - if len == 1 { - assert_eq!(buf, [10, 0, 0, 0]); - } else { - assert_eq!(len, 3); - assert_eq!(buf, [10, 11, 12, 0]); - } - }) - } - - #[test] - fn double_bind() { - each_ip(&mut |addr| { - let listener1 = t!(TcpListener::bind(&addr)); - match TcpListener::bind(&addr) { - Ok(listener2) => panic!( - "This system (perhaps due to options set by TcpListener::bind) \ - permits double binding: {:?} and {:?}", - listener1, listener2 - ), - Err(e) => { - assert!( - e.kind() == ErrorKind::ConnectionRefused - || e.kind() == ErrorKind::Other - || e.kind() == ErrorKind::AddrInUse, - "unknown error: {} {:?}", - e, - e.kind() - ); - } - } - }) - } - - #[test] - fn tcp_clone_smoke() { - each_ip(&mut |addr| { - let acceptor = t!(TcpListener::bind(&addr)); - - let _t = thread::spawn(move || { - let mut s = t!(TcpStream::connect(&addr)); - let mut buf = [0, 0]; - assert_eq!(s.read(&mut buf).unwrap(), 1); - assert_eq!(buf[0], 1); - t!(s.write(&[2])); - }); - - let mut s1 = t!(acceptor.accept()).0; - let s2 = t!(s1.try_clone()); - - let (tx1, rx1) = channel(); - let (tx2, rx2) = channel(); - let _t = thread::spawn(move || { - let mut s2 = s2; - rx1.recv().unwrap(); - t!(s2.write(&[1])); - tx2.send(()).unwrap(); - }); - tx1.send(()).unwrap(); - let mut buf = [0, 0]; - assert_eq!(s1.read(&mut buf).unwrap(), 1); - rx2.recv().unwrap(); - }) - } - - #[test] - fn tcp_clone_two_read() { - each_ip(&mut |addr| { - let acceptor = t!(TcpListener::bind(&addr)); - let (tx1, rx) = channel(); - let tx2 = tx1.clone(); - - let _t = thread::spawn(move || { - let mut s = t!(TcpStream::connect(&addr)); - t!(s.write(&[1])); - rx.recv().unwrap(); - t!(s.write(&[2])); - rx.recv().unwrap(); - }); - - let mut s1 = t!(acceptor.accept()).0; - let s2 = t!(s1.try_clone()); - - let (done, rx) = channel(); - let _t = thread::spawn(move || { - let mut s2 = s2; - let mut buf = [0, 0]; - t!(s2.read(&mut buf)); - tx2.send(()).unwrap(); - done.send(()).unwrap(); - }); - let mut buf = [0, 0]; - t!(s1.read(&mut buf)); - tx1.send(()).unwrap(); - - rx.recv().unwrap(); - }) - } - - #[test] - fn tcp_clone_two_write() { - each_ip(&mut |addr| { - let acceptor = t!(TcpListener::bind(&addr)); - - let _t = thread::spawn(move || { - let mut s = t!(TcpStream::connect(&addr)); - let mut buf = [0, 1]; - t!(s.read(&mut buf)); - t!(s.read(&mut buf)); - }); - - let mut s1 = t!(acceptor.accept()).0; - let s2 = t!(s1.try_clone()); - - let (done, rx) = channel(); - let _t = thread::spawn(move || { - let mut s2 = s2; - t!(s2.write(&[1])); - done.send(()).unwrap(); - }); - t!(s1.write(&[2])); - - rx.recv().unwrap(); - }) - } - - #[test] - // FIXME: https://github.com/fortanix/rust-sgx/issues/110 - #[cfg_attr(target_env = "sgx", ignore)] - fn shutdown_smoke() { - each_ip(&mut |addr| { - let a = t!(TcpListener::bind(&addr)); - let _t = thread::spawn(move || { - let mut c = t!(a.accept()).0; - let mut b = [0]; - assert_eq!(c.read(&mut b).unwrap(), 0); - t!(c.write(&[1])); - }); - - let mut s = t!(TcpStream::connect(&addr)); - t!(s.shutdown(Shutdown::Write)); - assert!(s.write(&[1]).is_err()); - let mut b = [0, 0]; - assert_eq!(t!(s.read(&mut b)), 1); - assert_eq!(b[0], 1); - }) - } - - #[test] - // FIXME: https://github.com/fortanix/rust-sgx/issues/110 - #[cfg_attr(target_env = "sgx", ignore)] - fn close_readwrite_smoke() { - each_ip(&mut |addr| { - let a = t!(TcpListener::bind(&addr)); - let (tx, rx) = channel::<()>(); - let _t = thread::spawn(move || { - let _s = t!(a.accept()); - let _ = rx.recv(); - }); - - let mut b = [0]; - let mut s = t!(TcpStream::connect(&addr)); - let mut s2 = t!(s.try_clone()); - - // closing should prevent reads/writes - t!(s.shutdown(Shutdown::Write)); - assert!(s.write(&[0]).is_err()); - t!(s.shutdown(Shutdown::Read)); - assert_eq!(s.read(&mut b).unwrap(), 0); - - // closing should affect previous handles - assert!(s2.write(&[0]).is_err()); - assert_eq!(s2.read(&mut b).unwrap(), 0); - - // closing should affect new handles - let mut s3 = t!(s.try_clone()); - assert!(s3.write(&[0]).is_err()); - assert_eq!(s3.read(&mut b).unwrap(), 0); - - // make sure these don't die - let _ = s2.shutdown(Shutdown::Read); - let _ = s2.shutdown(Shutdown::Write); - let _ = s3.shutdown(Shutdown::Read); - let _ = s3.shutdown(Shutdown::Write); - drop(tx); - }) - } - - #[test] - #[cfg(unix)] // test doesn't work on Windows, see #31657 - fn close_read_wakes_up() { - each_ip(&mut |addr| { - let a = t!(TcpListener::bind(&addr)); - let (tx1, rx) = channel::<()>(); - let _t = thread::spawn(move || { - let _s = t!(a.accept()); - let _ = rx.recv(); - }); - - let s = t!(TcpStream::connect(&addr)); - let s2 = t!(s.try_clone()); - let (tx, rx) = channel(); - let _t = thread::spawn(move || { - let mut s2 = s2; - assert_eq!(t!(s2.read(&mut [0])), 0); - tx.send(()).unwrap(); - }); - // this should wake up the child thread - t!(s.shutdown(Shutdown::Read)); - - // this test will never finish if the child doesn't wake up - rx.recv().unwrap(); - drop(tx1); - }) - } - - #[test] - fn clone_while_reading() { - each_ip(&mut |addr| { - let accept = t!(TcpListener::bind(&addr)); - - // Enqueue a thread to write to a socket - let (tx, rx) = channel(); - let (txdone, rxdone) = channel(); - let txdone2 = txdone.clone(); - let _t = thread::spawn(move || { - let mut tcp = t!(TcpStream::connect(&addr)); - rx.recv().unwrap(); - t!(tcp.write(&[0])); - txdone2.send(()).unwrap(); - }); - - // Spawn off a reading clone - let tcp = t!(accept.accept()).0; - let tcp2 = t!(tcp.try_clone()); - let txdone3 = txdone.clone(); - let _t = thread::spawn(move || { - let mut tcp2 = tcp2; - t!(tcp2.read(&mut [0])); - txdone3.send(()).unwrap(); - }); - - // Try to ensure that the reading clone is indeed reading - for _ in 0..50 { - thread::yield_now(); - } - - // clone the handle again while it's reading, then let it finish the - // read. - let _ = t!(tcp.try_clone()); - tx.send(()).unwrap(); - rxdone.recv().unwrap(); - rxdone.recv().unwrap(); - }) - } - - #[test] - fn clone_accept_smoke() { - each_ip(&mut |addr| { - let a = t!(TcpListener::bind(&addr)); - let a2 = t!(a.try_clone()); - - let _t = thread::spawn(move || { - let _ = TcpStream::connect(&addr); - }); - let _t = thread::spawn(move || { - let _ = TcpStream::connect(&addr); - }); - - t!(a.accept()); - t!(a2.accept()); - }) - } - - #[test] - fn clone_accept_concurrent() { - each_ip(&mut |addr| { - let a = t!(TcpListener::bind(&addr)); - let a2 = t!(a.try_clone()); - - let (tx, rx) = channel(); - let tx2 = tx.clone(); - - let _t = thread::spawn(move || { - tx.send(t!(a.accept())).unwrap(); - }); - let _t = thread::spawn(move || { - tx2.send(t!(a2.accept())).unwrap(); - }); - - let _t = thread::spawn(move || { - let _ = TcpStream::connect(&addr); - }); - let _t = thread::spawn(move || { - let _ = TcpStream::connect(&addr); - }); - - rx.recv().unwrap(); - rx.recv().unwrap(); - }) - } - - #[test] - fn debug() { - #[cfg(not(target_env = "sgx"))] - fn render_socket_addr<'a>(addr: &'a SocketAddr) -> impl fmt::Debug + 'a { - addr - } - #[cfg(target_env = "sgx")] - fn render_socket_addr<'a>(addr: &'a SocketAddr) -> impl fmt::Debug + 'a { - addr.to_string() - } - - #[cfg(target_env = "sgx")] - use crate::os::fortanix_sgx::io::AsRawFd; - #[cfg(unix)] - use crate::os::unix::io::AsRawFd; - #[cfg(not(windows))] - fn render_inner(addr: &dyn AsRawFd) -> impl fmt::Debug { - addr.as_raw_fd() - } - #[cfg(windows)] - fn render_inner(addr: &dyn crate::os::windows::io::AsRawSocket) -> impl fmt::Debug { - addr.as_raw_socket() - } - - let inner_name = if cfg!(windows) { "socket" } else { "fd" }; - let socket_addr = next_test_ip4(); - - let listener = t!(TcpListener::bind(&socket_addr)); - let compare = format!( - "TcpListener {{ addr: {:?}, {}: {:?} }}", - render_socket_addr(&socket_addr), - inner_name, - render_inner(&listener) - ); - assert_eq!(format!("{:?}", listener), compare); - - let stream = t!(TcpStream::connect(&("localhost", socket_addr.port()))); - let compare = format!( - "TcpStream {{ addr: {:?}, peer: {:?}, {}: {:?} }}", - render_socket_addr(&stream.local_addr().unwrap()), - render_socket_addr(&stream.peer_addr().unwrap()), - inner_name, - render_inner(&stream) - ); - assert_eq!(format!("{:?}", stream), compare); - } - - // FIXME: re-enabled openbsd tests once their socket timeout code - // no longer has rounding errors. - // VxWorks ignores SO_SNDTIMEO. - #[cfg_attr(any(target_os = "netbsd", target_os = "openbsd", target_os = "vxworks"), ignore)] - #[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - #[test] - fn timeouts() { - let addr = next_test_ip4(); - let listener = t!(TcpListener::bind(&addr)); - - let stream = t!(TcpStream::connect(&("localhost", addr.port()))); - let dur = Duration::new(15410, 0); - - assert_eq!(None, t!(stream.read_timeout())); - - t!(stream.set_read_timeout(Some(dur))); - assert_eq!(Some(dur), t!(stream.read_timeout())); - - assert_eq!(None, t!(stream.write_timeout())); - - t!(stream.set_write_timeout(Some(dur))); - assert_eq!(Some(dur), t!(stream.write_timeout())); - - t!(stream.set_read_timeout(None)); - assert_eq!(None, t!(stream.read_timeout())); - - t!(stream.set_write_timeout(None)); - assert_eq!(None, t!(stream.write_timeout())); - drop(listener); - } - - #[test] - #[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - fn test_read_timeout() { - let addr = next_test_ip4(); - let listener = t!(TcpListener::bind(&addr)); - - let mut stream = t!(TcpStream::connect(&("localhost", addr.port()))); - t!(stream.set_read_timeout(Some(Duration::from_millis(1000)))); - - let mut buf = [0; 10]; - let start = Instant::now(); - let kind = stream.read_exact(&mut buf).err().expect("expected error").kind(); - assert!( - kind == ErrorKind::WouldBlock || kind == ErrorKind::TimedOut, - "unexpected_error: {:?}", - kind - ); - assert!(start.elapsed() > Duration::from_millis(400)); - drop(listener); - } - - #[test] - #[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - fn test_read_with_timeout() { - let addr = next_test_ip4(); - let listener = t!(TcpListener::bind(&addr)); - - let mut stream = t!(TcpStream::connect(&("localhost", addr.port()))); - t!(stream.set_read_timeout(Some(Duration::from_millis(1000)))); - - let mut other_end = t!(listener.accept()).0; - t!(other_end.write_all(b"hello world")); - - let mut buf = [0; 11]; - t!(stream.read(&mut buf)); - assert_eq!(b"hello world", &buf[..]); - - let start = Instant::now(); - let kind = stream.read_exact(&mut buf).err().expect("expected error").kind(); - assert!( - kind == ErrorKind::WouldBlock || kind == ErrorKind::TimedOut, - "unexpected_error: {:?}", - kind - ); - assert!(start.elapsed() > Duration::from_millis(400)); - drop(listener); - } - - // Ensure the `set_read_timeout` and `set_write_timeout` calls return errors - // when passed zero Durations - #[test] - fn test_timeout_zero_duration() { - let addr = next_test_ip4(); - - let listener = t!(TcpListener::bind(&addr)); - let stream = t!(TcpStream::connect(&addr)); - - let result = stream.set_write_timeout(Some(Duration::new(0, 0))); - let err = result.unwrap_err(); - assert_eq!(err.kind(), ErrorKind::InvalidInput); - - let result = stream.set_read_timeout(Some(Duration::new(0, 0))); - let err = result.unwrap_err(); - assert_eq!(err.kind(), ErrorKind::InvalidInput); - - drop(listener); - } - - #[test] - #[cfg_attr(target_env = "sgx", ignore)] - fn nodelay() { - let addr = next_test_ip4(); - let _listener = t!(TcpListener::bind(&addr)); - - let stream = t!(TcpStream::connect(&("localhost", addr.port()))); - - assert_eq!(false, t!(stream.nodelay())); - t!(stream.set_nodelay(true)); - assert_eq!(true, t!(stream.nodelay())); - t!(stream.set_nodelay(false)); - assert_eq!(false, t!(stream.nodelay())); - } - - #[test] - #[cfg_attr(target_env = "sgx", ignore)] - fn ttl() { - let ttl = 100; - - let addr = next_test_ip4(); - let listener = t!(TcpListener::bind(&addr)); - - t!(listener.set_ttl(ttl)); - assert_eq!(ttl, t!(listener.ttl())); - - let stream = t!(TcpStream::connect(&("localhost", addr.port()))); - - t!(stream.set_ttl(ttl)); - assert_eq!(ttl, t!(stream.ttl())); - } - - #[test] - #[cfg_attr(target_env = "sgx", ignore)] - fn set_nonblocking() { - let addr = next_test_ip4(); - let listener = t!(TcpListener::bind(&addr)); - - t!(listener.set_nonblocking(true)); - t!(listener.set_nonblocking(false)); - - let mut stream = t!(TcpStream::connect(&("localhost", addr.port()))); - - t!(stream.set_nonblocking(false)); - t!(stream.set_nonblocking(true)); - - let mut buf = [0]; - match stream.read(&mut buf) { - Ok(_) => panic!("expected error"), - Err(ref e) if e.kind() == ErrorKind::WouldBlock => {} - Err(e) => panic!("unexpected error {}", e), - } - } - - #[test] - #[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - fn peek() { - each_ip(&mut |addr| { - let (txdone, rxdone) = channel(); - - let srv = t!(TcpListener::bind(&addr)); - let _t = thread::spawn(move || { - let mut cl = t!(srv.accept()).0; - cl.write(&[1, 3, 3, 7]).unwrap(); - t!(rxdone.recv()); - }); - - let mut c = t!(TcpStream::connect(&addr)); - let mut b = [0; 10]; - for _ in 1..3 { - let len = c.peek(&mut b).unwrap(); - assert_eq!(len, 4); - } - let len = c.read(&mut b).unwrap(); - assert_eq!(len, 4); - - t!(c.set_nonblocking(true)); - match c.peek(&mut b) { - Ok(_) => panic!("expected error"), - Err(ref e) if e.kind() == ErrorKind::WouldBlock => {} - Err(e) => panic!("unexpected error {}", e), - } - t!(txdone.send(())); - }) - } - - #[test] - #[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - fn connect_timeout_valid() { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let addr = listener.local_addr().unwrap(); - TcpStream::connect_timeout(&addr, Duration::from_secs(2)).unwrap(); - } -} diff --git a/crux-mir/lib/std/src/net/tcp/tests.rs b/crux-mir/lib/std/src/net/tcp/tests.rs new file mode 100644 index 000000000..8c0adcfb0 --- /dev/null +++ b/crux-mir/lib/std/src/net/tcp/tests.rs @@ -0,0 +1,876 @@ +use crate::fmt; +use crate::io::prelude::*; +use crate::io::{ErrorKind, IoSlice, IoSliceMut}; +use crate::net::test::{next_test_ip4, next_test_ip6}; +use crate::net::*; +use crate::sync::mpsc::channel; +use crate::thread; +use crate::time::{Duration, Instant}; + +fn each_ip(f: &mut dyn FnMut(SocketAddr)) { + f(next_test_ip4()); + f(next_test_ip6()); +} + +macro_rules! t { + ($e:expr) => { + match $e { + Ok(t) => t, + Err(e) => panic!("received error for `{}`: {}", stringify!($e), e), + } + }; +} + +#[test] +fn bind_error() { + match TcpListener::bind("1.1.1.1:9999") { + Ok(..) => panic!(), + Err(e) => assert_eq!(e.kind(), ErrorKind::AddrNotAvailable), + } +} + +#[test] +fn connect_error() { + match TcpStream::connect("0.0.0.0:1") { + Ok(..) => panic!(), + Err(e) => assert!( + e.kind() == ErrorKind::ConnectionRefused + || e.kind() == ErrorKind::InvalidInput + || e.kind() == ErrorKind::AddrInUse + || e.kind() == ErrorKind::AddrNotAvailable, + "bad error: {} {:?}", + e, + e.kind() + ), + } +} + +#[test] +fn listen_localhost() { + let socket_addr = next_test_ip4(); + let listener = t!(TcpListener::bind(&socket_addr)); + + let _t = thread::spawn(move || { + let mut stream = t!(TcpStream::connect(&("localhost", socket_addr.port()))); + t!(stream.write(&[144])); + }); + + let mut stream = t!(listener.accept()).0; + let mut buf = [0]; + t!(stream.read(&mut buf)); + assert!(buf[0] == 144); +} + +#[test] +fn connect_loopback() { + each_ip(&mut |addr| { + let acceptor = t!(TcpListener::bind(&addr)); + + let _t = thread::spawn(move || { + let host = match addr { + SocketAddr::V4(..) => "127.0.0.1", + SocketAddr::V6(..) => "::1", + }; + let mut stream = t!(TcpStream::connect(&(host, addr.port()))); + t!(stream.write(&[66])); + }); + + let mut stream = t!(acceptor.accept()).0; + let mut buf = [0]; + t!(stream.read(&mut buf)); + assert!(buf[0] == 66); + }) +} + +#[test] +fn smoke_test() { + each_ip(&mut |addr| { + let acceptor = t!(TcpListener::bind(&addr)); + + let (tx, rx) = channel(); + let _t = thread::spawn(move || { + let mut stream = t!(TcpStream::connect(&addr)); + t!(stream.write(&[99])); + tx.send(t!(stream.local_addr())).unwrap(); + }); + + let (mut stream, addr) = t!(acceptor.accept()); + let mut buf = [0]; + t!(stream.read(&mut buf)); + assert!(buf[0] == 99); + assert_eq!(addr, t!(rx.recv())); + }) +} + +#[test] +fn read_eof() { + each_ip(&mut |addr| { + let acceptor = t!(TcpListener::bind(&addr)); + + let _t = thread::spawn(move || { + let _stream = t!(TcpStream::connect(&addr)); + // Close + }); + + let mut stream = t!(acceptor.accept()).0; + let mut buf = [0]; + let nread = t!(stream.read(&mut buf)); + assert_eq!(nread, 0); + let nread = t!(stream.read(&mut buf)); + assert_eq!(nread, 0); + }) +} + +#[test] +fn write_close() { + each_ip(&mut |addr| { + let acceptor = t!(TcpListener::bind(&addr)); + + let (tx, rx) = channel(); + let _t = thread::spawn(move || { + drop(t!(TcpStream::connect(&addr))); + tx.send(()).unwrap(); + }); + + let mut stream = t!(acceptor.accept()).0; + rx.recv().unwrap(); + let buf = [0]; + match stream.write(&buf) { + Ok(..) => {} + Err(e) => { + assert!( + e.kind() == ErrorKind::ConnectionReset + || e.kind() == ErrorKind::BrokenPipe + || e.kind() == ErrorKind::ConnectionAborted, + "unknown error: {e}" + ); + } + } + }) +} + +#[test] +fn multiple_connect_serial() { + each_ip(&mut |addr| { + let max = 10; + let acceptor = t!(TcpListener::bind(&addr)); + + let _t = thread::spawn(move || { + for _ in 0..max { + let mut stream = t!(TcpStream::connect(&addr)); + t!(stream.write(&[99])); + } + }); + + for stream in acceptor.incoming().take(max) { + let mut stream = t!(stream); + let mut buf = [0]; + t!(stream.read(&mut buf)); + assert_eq!(buf[0], 99); + } + }) +} + +#[test] +fn multiple_connect_interleaved_greedy_schedule() { + const MAX: usize = 10; + each_ip(&mut |addr| { + let acceptor = t!(TcpListener::bind(&addr)); + + let _t = thread::spawn(move || { + let acceptor = acceptor; + for (i, stream) in acceptor.incoming().enumerate().take(MAX) { + // Start another thread to handle the connection + let _t = thread::spawn(move || { + let mut stream = t!(stream); + let mut buf = [0]; + t!(stream.read(&mut buf)); + assert!(buf[0] == i as u8); + }); + } + }); + + connect(0, addr); + }); + + fn connect(i: usize, addr: SocketAddr) { + if i == MAX { + return; + } + + let t = thread::spawn(move || { + let mut stream = t!(TcpStream::connect(&addr)); + // Connect again before writing + connect(i + 1, addr); + t!(stream.write(&[i as u8])); + }); + t.join().ok().expect("thread panicked"); + } +} + +#[test] +fn multiple_connect_interleaved_lazy_schedule() { + const MAX: usize = 10; + each_ip(&mut |addr| { + let acceptor = t!(TcpListener::bind(&addr)); + + let _t = thread::spawn(move || { + for stream in acceptor.incoming().take(MAX) { + // Start another thread to handle the connection + let _t = thread::spawn(move || { + let mut stream = t!(stream); + let mut buf = [0]; + t!(stream.read(&mut buf)); + assert!(buf[0] == 99); + }); + } + }); + + connect(0, addr); + }); + + fn connect(i: usize, addr: SocketAddr) { + if i == MAX { + return; + } + + let t = thread::spawn(move || { + let mut stream = t!(TcpStream::connect(&addr)); + connect(i + 1, addr); + t!(stream.write(&[99])); + }); + t.join().ok().expect("thread panicked"); + } +} + +#[test] +fn socket_and_peer_name() { + each_ip(&mut |addr| { + let listener = t!(TcpListener::bind(&addr)); + let so_name = t!(listener.local_addr()); + assert_eq!(addr, so_name); + let _t = thread::spawn(move || { + t!(listener.accept()); + }); + + let stream = t!(TcpStream::connect(&addr)); + assert_eq!(addr, t!(stream.peer_addr())); + }) +} + +#[test] +fn partial_read() { + each_ip(&mut |addr| { + let (tx, rx) = channel(); + let srv = t!(TcpListener::bind(&addr)); + let _t = thread::spawn(move || { + let mut cl = t!(srv.accept()).0; + cl.write(&[10]).unwrap(); + let mut b = [0]; + t!(cl.read(&mut b)); + tx.send(()).unwrap(); + }); + + let mut c = t!(TcpStream::connect(&addr)); + let mut b = [0; 10]; + assert_eq!(c.read(&mut b).unwrap(), 1); + t!(c.write(&[1])); + rx.recv().unwrap(); + }) +} + +#[test] +fn read_vectored() { + each_ip(&mut |addr| { + let srv = t!(TcpListener::bind(&addr)); + let mut s1 = t!(TcpStream::connect(&addr)); + let mut s2 = t!(srv.accept()).0; + + let len = s1.write(&[10, 11, 12]).unwrap(); + assert_eq!(len, 3); + + let mut a = []; + let mut b = [0]; + let mut c = [0; 3]; + let len = t!(s2.read_vectored(&mut [ + IoSliceMut::new(&mut a), + IoSliceMut::new(&mut b), + IoSliceMut::new(&mut c) + ],)); + assert!(len > 0); + assert_eq!(b, [10]); + // some implementations don't support readv, so we may only fill the first buffer + assert!(len == 1 || c == [11, 12, 0]); + }) +} + +#[test] +fn write_vectored() { + each_ip(&mut |addr| { + let srv = t!(TcpListener::bind(&addr)); + let mut s1 = t!(TcpStream::connect(&addr)); + let mut s2 = t!(srv.accept()).0; + + let a = []; + let b = [10]; + let c = [11, 12]; + t!(s1.write_vectored(&[IoSlice::new(&a), IoSlice::new(&b), IoSlice::new(&c)])); + + let mut buf = [0; 4]; + let len = t!(s2.read(&mut buf)); + // some implementations don't support writev, so we may only write the first buffer + if len == 1 { + assert_eq!(buf, [10, 0, 0, 0]); + } else { + assert_eq!(len, 3); + assert_eq!(buf, [10, 11, 12, 0]); + } + }) +} + +#[test] +fn double_bind() { + each_ip(&mut |addr| { + let listener1 = t!(TcpListener::bind(&addr)); + match TcpListener::bind(&addr) { + Ok(listener2) => panic!( + "This system (perhaps due to options set by TcpListener::bind) \ + permits double binding: {:?} and {:?}", + listener1, listener2 + ), + Err(e) => { + assert!( + e.kind() == ErrorKind::ConnectionRefused + || e.kind() == ErrorKind::Uncategorized + || e.kind() == ErrorKind::AddrInUse, + "unknown error: {} {:?}", + e, + e.kind() + ); + } + } + }) +} + +#[test] +fn tcp_clone_smoke() { + each_ip(&mut |addr| { + let acceptor = t!(TcpListener::bind(&addr)); + + let _t = thread::spawn(move || { + let mut s = t!(TcpStream::connect(&addr)); + let mut buf = [0, 0]; + assert_eq!(s.read(&mut buf).unwrap(), 1); + assert_eq!(buf[0], 1); + t!(s.write(&[2])); + }); + + let mut s1 = t!(acceptor.accept()).0; + let s2 = t!(s1.try_clone()); + + let (tx1, rx1) = channel(); + let (tx2, rx2) = channel(); + let _t = thread::spawn(move || { + let mut s2 = s2; + rx1.recv().unwrap(); + t!(s2.write(&[1])); + tx2.send(()).unwrap(); + }); + tx1.send(()).unwrap(); + let mut buf = [0, 0]; + assert_eq!(s1.read(&mut buf).unwrap(), 1); + rx2.recv().unwrap(); + }) +} + +#[test] +fn tcp_clone_two_read() { + each_ip(&mut |addr| { + let acceptor = t!(TcpListener::bind(&addr)); + let (tx1, rx) = channel(); + let tx2 = tx1.clone(); + + let _t = thread::spawn(move || { + let mut s = t!(TcpStream::connect(&addr)); + t!(s.write(&[1])); + rx.recv().unwrap(); + t!(s.write(&[2])); + rx.recv().unwrap(); + }); + + let mut s1 = t!(acceptor.accept()).0; + let s2 = t!(s1.try_clone()); + + let (done, rx) = channel(); + let _t = thread::spawn(move || { + let mut s2 = s2; + let mut buf = [0, 0]; + t!(s2.read(&mut buf)); + tx2.send(()).unwrap(); + done.send(()).unwrap(); + }); + let mut buf = [0, 0]; + t!(s1.read(&mut buf)); + tx1.send(()).unwrap(); + + rx.recv().unwrap(); + }) +} + +#[test] +fn tcp_clone_two_write() { + each_ip(&mut |addr| { + let acceptor = t!(TcpListener::bind(&addr)); + + let _t = thread::spawn(move || { + let mut s = t!(TcpStream::connect(&addr)); + let mut buf = [0, 1]; + t!(s.read(&mut buf)); + t!(s.read(&mut buf)); + }); + + let mut s1 = t!(acceptor.accept()).0; + let s2 = t!(s1.try_clone()); + + let (done, rx) = channel(); + let _t = thread::spawn(move || { + let mut s2 = s2; + t!(s2.write(&[1])); + done.send(()).unwrap(); + }); + t!(s1.write(&[2])); + + rx.recv().unwrap(); + }) +} + +#[test] +// FIXME: https://github.com/fortanix/rust-sgx/issues/110 +#[cfg_attr(target_env = "sgx", ignore)] +fn shutdown_smoke() { + each_ip(&mut |addr| { + let a = t!(TcpListener::bind(&addr)); + let _t = thread::spawn(move || { + let mut c = t!(a.accept()).0; + let mut b = [0]; + assert_eq!(c.read(&mut b).unwrap(), 0); + t!(c.write(&[1])); + }); + + let mut s = t!(TcpStream::connect(&addr)); + t!(s.shutdown(Shutdown::Write)); + assert!(s.write(&[1]).is_err()); + let mut b = [0, 0]; + assert_eq!(t!(s.read(&mut b)), 1); + assert_eq!(b[0], 1); + }) +} + +#[test] +// FIXME: https://github.com/fortanix/rust-sgx/issues/110 +#[cfg_attr(target_env = "sgx", ignore)] +fn close_readwrite_smoke() { + each_ip(&mut |addr| { + let a = t!(TcpListener::bind(&addr)); + let (tx, rx) = channel::<()>(); + let _t = thread::spawn(move || { + let _s = t!(a.accept()); + let _ = rx.recv(); + }); + + let mut b = [0]; + let mut s = t!(TcpStream::connect(&addr)); + let mut s2 = t!(s.try_clone()); + + // closing should prevent reads/writes + t!(s.shutdown(Shutdown::Write)); + assert!(s.write(&[0]).is_err()); + t!(s.shutdown(Shutdown::Read)); + assert_eq!(s.read(&mut b).unwrap(), 0); + + // closing should affect previous handles + assert!(s2.write(&[0]).is_err()); + assert_eq!(s2.read(&mut b).unwrap(), 0); + + // closing should affect new handles + let mut s3 = t!(s.try_clone()); + assert!(s3.write(&[0]).is_err()); + assert_eq!(s3.read(&mut b).unwrap(), 0); + + // make sure these don't die + let _ = s2.shutdown(Shutdown::Read); + let _ = s2.shutdown(Shutdown::Write); + let _ = s3.shutdown(Shutdown::Read); + let _ = s3.shutdown(Shutdown::Write); + drop(tx); + }) +} + +#[test] +#[cfg_attr(target_env = "sgx", ignore)] +fn close_read_wakes_up() { + each_ip(&mut |addr| { + let a = t!(TcpListener::bind(&addr)); + let (tx1, rx) = channel::<()>(); + let _t = thread::spawn(move || { + let _s = t!(a.accept()); + let _ = rx.recv(); + }); + + let s = t!(TcpStream::connect(&addr)); + let s2 = t!(s.try_clone()); + let (tx, rx) = channel(); + let _t = thread::spawn(move || { + let mut s2 = s2; + assert_eq!(t!(s2.read(&mut [0])), 0); + tx.send(()).unwrap(); + }); + // this should wake up the child thread + t!(s.shutdown(Shutdown::Read)); + + // this test will never finish if the child doesn't wake up + rx.recv().unwrap(); + drop(tx1); + }) +} + +#[test] +fn clone_while_reading() { + each_ip(&mut |addr| { + let accept = t!(TcpListener::bind(&addr)); + + // Enqueue a thread to write to a socket + let (tx, rx) = channel(); + let (txdone, rxdone) = channel(); + let txdone2 = txdone.clone(); + let _t = thread::spawn(move || { + let mut tcp = t!(TcpStream::connect(&addr)); + rx.recv().unwrap(); + t!(tcp.write(&[0])); + txdone2.send(()).unwrap(); + }); + + // Spawn off a reading clone + let tcp = t!(accept.accept()).0; + let tcp2 = t!(tcp.try_clone()); + let txdone3 = txdone.clone(); + let _t = thread::spawn(move || { + let mut tcp2 = tcp2; + t!(tcp2.read(&mut [0])); + txdone3.send(()).unwrap(); + }); + + // Try to ensure that the reading clone is indeed reading + for _ in 0..50 { + thread::yield_now(); + } + + // clone the handle again while it's reading, then let it finish the + // read. + let _ = t!(tcp.try_clone()); + tx.send(()).unwrap(); + rxdone.recv().unwrap(); + rxdone.recv().unwrap(); + }) +} + +#[test] +fn clone_accept_smoke() { + each_ip(&mut |addr| { + let a = t!(TcpListener::bind(&addr)); + let a2 = t!(a.try_clone()); + + let _t = thread::spawn(move || { + let _ = TcpStream::connect(&addr); + }); + let _t = thread::spawn(move || { + let _ = TcpStream::connect(&addr); + }); + + t!(a.accept()); + t!(a2.accept()); + }) +} + +#[test] +fn clone_accept_concurrent() { + each_ip(&mut |addr| { + let a = t!(TcpListener::bind(&addr)); + let a2 = t!(a.try_clone()); + + let (tx, rx) = channel(); + let tx2 = tx.clone(); + + let _t = thread::spawn(move || { + tx.send(t!(a.accept())).unwrap(); + }); + let _t = thread::spawn(move || { + tx2.send(t!(a2.accept())).unwrap(); + }); + + let _t = thread::spawn(move || { + let _ = TcpStream::connect(&addr); + }); + let _t = thread::spawn(move || { + let _ = TcpStream::connect(&addr); + }); + + rx.recv().unwrap(); + rx.recv().unwrap(); + }) +} + +#[test] +fn debug() { + #[cfg(not(target_env = "sgx"))] + fn render_socket_addr<'a>(addr: &'a SocketAddr) -> impl fmt::Debug + 'a { + addr + } + #[cfg(target_env = "sgx")] + fn render_socket_addr<'a>(addr: &'a SocketAddr) -> impl fmt::Debug + 'a { + addr.to_string() + } + + #[cfg(target_env = "sgx")] + use crate::os::fortanix_sgx::io::AsRawFd; + #[cfg(unix)] + use crate::os::unix::io::AsRawFd; + #[cfg(not(windows))] + fn render_inner(addr: &dyn AsRawFd) -> impl fmt::Debug { + addr.as_raw_fd() + } + #[cfg(windows)] + fn render_inner(addr: &dyn crate::os::windows::io::AsRawSocket) -> impl fmt::Debug { + addr.as_raw_socket() + } + + let inner_name = if cfg!(windows) { "socket" } else { "fd" }; + let socket_addr = next_test_ip4(); + + let listener = t!(TcpListener::bind(&socket_addr)); + let compare = format!( + "TcpListener {{ addr: {:?}, {}: {:?} }}", + render_socket_addr(&socket_addr), + inner_name, + render_inner(&listener) + ); + assert_eq!(format!("{listener:?}"), compare); + + let stream = t!(TcpStream::connect(&("localhost", socket_addr.port()))); + let compare = format!( + "TcpStream {{ addr: {:?}, peer: {:?}, {}: {:?} }}", + render_socket_addr(&stream.local_addr().unwrap()), + render_socket_addr(&stream.peer_addr().unwrap()), + inner_name, + render_inner(&stream) + ); + assert_eq!(format!("{stream:?}"), compare); +} + +// FIXME: re-enabled openbsd tests once their socket timeout code +// no longer has rounding errors. +// VxWorks ignores SO_SNDTIMEO. +#[cfg_attr(any(target_os = "netbsd", target_os = "openbsd", target_os = "vxworks"), ignore)] +#[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 +#[test] +fn timeouts() { + let addr = next_test_ip4(); + let listener = t!(TcpListener::bind(&addr)); + + let stream = t!(TcpStream::connect(&("localhost", addr.port()))); + let dur = Duration::new(15410, 0); + + assert_eq!(None, t!(stream.read_timeout())); + + t!(stream.set_read_timeout(Some(dur))); + assert_eq!(Some(dur), t!(stream.read_timeout())); + + assert_eq!(None, t!(stream.write_timeout())); + + t!(stream.set_write_timeout(Some(dur))); + assert_eq!(Some(dur), t!(stream.write_timeout())); + + t!(stream.set_read_timeout(None)); + assert_eq!(None, t!(stream.read_timeout())); + + t!(stream.set_write_timeout(None)); + assert_eq!(None, t!(stream.write_timeout())); + drop(listener); +} + +#[test] +#[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 +fn test_read_timeout() { + let addr = next_test_ip4(); + let listener = t!(TcpListener::bind(&addr)); + + let mut stream = t!(TcpStream::connect(&("localhost", addr.port()))); + t!(stream.set_read_timeout(Some(Duration::from_millis(1000)))); + + let mut buf = [0; 10]; + let start = Instant::now(); + let kind = stream.read_exact(&mut buf).err().expect("expected error").kind(); + assert!( + kind == ErrorKind::WouldBlock || kind == ErrorKind::TimedOut, + "unexpected_error: {:?}", + kind + ); + assert!(start.elapsed() > Duration::from_millis(400)); + drop(listener); +} + +#[test] +#[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 +fn test_read_with_timeout() { + let addr = next_test_ip4(); + let listener = t!(TcpListener::bind(&addr)); + + let mut stream = t!(TcpStream::connect(&("localhost", addr.port()))); + t!(stream.set_read_timeout(Some(Duration::from_millis(1000)))); + + let mut other_end = t!(listener.accept()).0; + t!(other_end.write_all(b"hello world")); + + let mut buf = [0; 11]; + t!(stream.read(&mut buf)); + assert_eq!(b"hello world", &buf[..]); + + let start = Instant::now(); + let kind = stream.read_exact(&mut buf).err().expect("expected error").kind(); + assert!( + kind == ErrorKind::WouldBlock || kind == ErrorKind::TimedOut, + "unexpected_error: {:?}", + kind + ); + assert!(start.elapsed() > Duration::from_millis(400)); + drop(listener); +} + +// Ensure the `set_read_timeout` and `set_write_timeout` calls return errors +// when passed zero Durations +#[test] +fn test_timeout_zero_duration() { + let addr = next_test_ip4(); + + let listener = t!(TcpListener::bind(&addr)); + let stream = t!(TcpStream::connect(&addr)); + + let result = stream.set_write_timeout(Some(Duration::new(0, 0))); + let err = result.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidInput); + + let result = stream.set_read_timeout(Some(Duration::new(0, 0))); + let err = result.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidInput); + + drop(listener); +} + +#[test] +#[cfg_attr(target_env = "sgx", ignore)] +fn linger() { + let addr = next_test_ip4(); + let _listener = t!(TcpListener::bind(&addr)); + + let stream = t!(TcpStream::connect(&("localhost", addr.port()))); + + assert_eq!(None, t!(stream.linger())); + t!(stream.set_linger(Some(Duration::from_secs(1)))); + assert_eq!(Some(Duration::from_secs(1)), t!(stream.linger())); + t!(stream.set_linger(None)); + assert_eq!(None, t!(stream.linger())); +} + +#[test] +#[cfg_attr(target_env = "sgx", ignore)] +fn nodelay() { + let addr = next_test_ip4(); + let _listener = t!(TcpListener::bind(&addr)); + + let stream = t!(TcpStream::connect(&("localhost", addr.port()))); + + assert_eq!(false, t!(stream.nodelay())); + t!(stream.set_nodelay(true)); + assert_eq!(true, t!(stream.nodelay())); + t!(stream.set_nodelay(false)); + assert_eq!(false, t!(stream.nodelay())); +} + +#[test] +#[cfg_attr(target_env = "sgx", ignore)] +fn ttl() { + let ttl = 100; + + let addr = next_test_ip4(); + let listener = t!(TcpListener::bind(&addr)); + + t!(listener.set_ttl(ttl)); + assert_eq!(ttl, t!(listener.ttl())); + + let stream = t!(TcpStream::connect(&("localhost", addr.port()))); + + t!(stream.set_ttl(ttl)); + assert_eq!(ttl, t!(stream.ttl())); +} + +#[test] +#[cfg_attr(target_env = "sgx", ignore)] +fn set_nonblocking() { + let addr = next_test_ip4(); + let listener = t!(TcpListener::bind(&addr)); + + t!(listener.set_nonblocking(true)); + t!(listener.set_nonblocking(false)); + + let mut stream = t!(TcpStream::connect(&("localhost", addr.port()))); + + t!(stream.set_nonblocking(false)); + t!(stream.set_nonblocking(true)); + + let mut buf = [0]; + match stream.read(&mut buf) { + Ok(_) => panic!("expected error"), + Err(ref e) if e.kind() == ErrorKind::WouldBlock => {} + Err(e) => panic!("unexpected error {e}"), + } +} + +#[test] +#[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 +fn peek() { + each_ip(&mut |addr| { + let (txdone, rxdone) = channel(); + + let srv = t!(TcpListener::bind(&addr)); + let _t = thread::spawn(move || { + let mut cl = t!(srv.accept()).0; + cl.write(&[1, 3, 3, 7]).unwrap(); + t!(rxdone.recv()); + }); + + let mut c = t!(TcpStream::connect(&addr)); + let mut b = [0; 10]; + for _ in 1..3 { + let len = c.peek(&mut b).unwrap(); + assert_eq!(len, 4); + } + let len = c.read(&mut b).unwrap(); + assert_eq!(len, 4); + + t!(c.set_nonblocking(true)); + match c.peek(&mut b) { + Ok(_) => panic!("expected error"), + Err(ref e) if e.kind() == ErrorKind::WouldBlock => {} + Err(e) => panic!("unexpected error {e}"), + } + t!(txdone.send(())); + }) +} + +#[test] +#[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 +fn connect_timeout_valid() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + TcpStream::connect_timeout(&addr, Duration::from_secs(2)).unwrap(); +} diff --git a/crux-mir/lib/std/src/net/udp.rs b/crux-mir/lib/std/src/net/udp.rs index 0096b827c..864e1b0f3 100644 --- a/crux-mir/lib/std/src/net/udp.rs +++ b/crux-mir/lib/std/src/net/udp.rs @@ -1,5 +1,8 @@ +#[cfg(all(test, not(any(target_os = "emscripten", target_env = "sgx"))))] +mod tests; + use crate::fmt; -use crate::io::{self, Error, ErrorKind}; +use crate::io::{self, ErrorKind}; use crate::net::{Ipv4Addr, Ipv6Addr, SocketAddr, ToSocketAddrs}; use crate::sys_common::net as net_imp; use crate::sys_common::{AsInner, FromInner, IntoInner}; @@ -19,15 +22,15 @@ use crate::time::Duration; /// an unordered, unreliable protocol; refer to [`TcpListener`] and [`TcpStream`] for TCP /// primitives. /// -/// [`bind`]: #method.bind -/// [`connect`]: #method.connect +/// [`bind`]: UdpSocket::bind +/// [`connect`]: UdpSocket::connect /// [IETF RFC 768]: https://tools.ietf.org/html/rfc768 -/// [`recv`]: #method.recv -/// [received from]: #method.recv_from -/// [`send`]: #method.send -/// [sent to]: #method.send_to -/// [`TcpListener`]: ../../std/net/struct.TcpListener.html -/// [`TcpStream`]: ../../std/net/struct.TcpStream.html +/// [`recv`]: UdpSocket::recv +/// [received from]: UdpSocket::recv_from +/// [`send`]: UdpSocket::send +/// [sent to]: UdpSocket::send_to +/// [`TcpListener`]: crate::net::TcpListener +/// [`TcpStream`]: crate::net::TcpStream /// /// # Examples /// @@ -36,7 +39,7 @@ use crate::time::Duration; /// /// fn main() -> std::io::Result<()> { /// { -/// let mut socket = UdpSocket::bind("127.0.0.1:34254")?; +/// let socket = UdpSocket::bind("127.0.0.1:34254")?; /// /// // Receives a single datagram message on the socket. If `buf` is too small to hold /// // the message, it will be cut off. @@ -65,8 +68,6 @@ impl UdpSocket { /// of the addresses succeed in creating a socket, the error returned from /// the last attempt (the last address) is returned. /// - /// [`ToSocketAddrs`]: ../../std/net/trait.ToSocketAddrs.html - /// /// # Examples /// /// Creates a UDP socket bound to `127.0.0.1:3400`: @@ -158,9 +159,7 @@ impl UdpSocket { /// This will return an error when the IP version of the local socket /// does not match that returned from [`ToSocketAddrs`]. /// - /// See issue #34202 for more details. - /// - /// [`ToSocketAddrs`]: ../../std/net/trait.ToSocketAddrs.html + /// See [Issue #34202] for more details. /// /// # Examples /// @@ -170,11 +169,15 @@ impl UdpSocket { /// let socket = UdpSocket::bind("127.0.0.1:34254").expect("couldn't bind to address"); /// socket.send_to(&[0; 10], "127.0.0.1:4242").expect("couldn't send data"); /// ``` + /// + /// [Issue #34202]: https://github.com/rust-lang/rust/issues/34202 #[stable(feature = "rust1", since = "1.0.0")] pub fn send_to(&self, buf: &[u8], addr: A) -> io::Result { match addr.to_socket_addrs()?.next() { Some(addr) => self.0.send_to(buf, &addr), - None => Err(Error::new(ErrorKind::InvalidInput, "no addresses to send data to")), + None => { + Err(io::const_io_error!(ErrorKind::InvalidInput, "no addresses to send data to")) + } } } @@ -193,7 +196,7 @@ impl UdpSocket { /// /// If the socket isn't connected, it will return a [`NotConnected`] error. /// - /// [`NotConnected`]: ../../std/io/enum.ErrorKind.html#variant.NotConnected + /// [`NotConnected`]: io::ErrorKind::NotConnected /// /// ```no_run /// use std::net::UdpSocket; @@ -254,12 +257,9 @@ impl UdpSocket { /// a result of setting this option. For example Unix typically returns an /// error of the kind [`WouldBlock`], but Windows may return [`TimedOut`]. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// [`Err`]: ../../std/result/enum.Result.html#variant.Err - /// [`read`]: ../../std/io/trait.Read.html#tymethod.read - /// [`Duration`]: ../../std/time/struct.Duration.html - /// [`WouldBlock`]: ../../std/io/enum.ErrorKind.html#variant.WouldBlock - /// [`TimedOut`]: ../../std/io/enum.ErrorKind.html#variant.TimedOut + /// [`read`]: io::Read::read + /// [`WouldBlock`]: io::ErrorKind::WouldBlock + /// [`TimedOut`]: io::ErrorKind::TimedOut /// /// # Examples /// @@ -300,12 +300,9 @@ impl UdpSocket { /// as a result of setting this option. For example Unix typically returns /// an error of the kind [`WouldBlock`], but Windows may return [`TimedOut`]. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// [`Err`]: ../../std/result/enum.Result.html#variant.Err - /// [`write`]: ../../std/io/trait.Write.html#tymethod.write - /// [`Duration`]: ../../std/time/struct.Duration.html - /// [`WouldBlock`]: ../../std/io/enum.ErrorKind.html#variant.WouldBlock - /// [`TimedOut`]: ../../std/io/enum.ErrorKind.html#variant.TimedOut + /// [`write`]: io::Write::write + /// [`WouldBlock`]: io::ErrorKind::WouldBlock + /// [`TimedOut`]: io::ErrorKind::TimedOut /// /// # Examples /// @@ -338,8 +335,7 @@ impl UdpSocket { /// /// If the timeout is [`None`], then [`read`] calls will block indefinitely. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// [`read`]: ../../std/io/trait.Read.html#tymethod.read + /// [`read`]: io::Read::read /// /// # Examples /// @@ -359,8 +355,7 @@ impl UdpSocket { /// /// If the timeout is [`None`], then [`write`] calls will block indefinitely. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// [`write`]: ../../std/io/trait.Write.html#tymethod.write + /// [`write`]: io::Write::write /// /// # Examples /// @@ -396,10 +391,7 @@ impl UdpSocket { /// Gets the value of the `SO_BROADCAST` option for this socket. /// - /// For more information about this option, see - /// [`set_broadcast`][link]. - /// - /// [link]: #method.set_broadcast + /// For more information about this option, see [`UdpSocket::set_broadcast`]. /// /// # Examples /// @@ -418,7 +410,7 @@ impl UdpSocket { /// Sets the value of the `IP_MULTICAST_LOOP` option for this socket. /// /// If enabled, multicast packets will be looped back to the local socket. - /// Note that this may not have any effect on IPv6 sockets. + /// Note that this might not have any effect on IPv6 sockets. /// /// # Examples /// @@ -435,10 +427,7 @@ impl UdpSocket { /// Gets the value of the `IP_MULTICAST_LOOP` option for this socket. /// - /// For more information about this option, see - /// [`set_multicast_loop_v4`][link]. - /// - /// [link]: #method.set_multicast_loop_v4 + /// For more information about this option, see [`UdpSocket::set_multicast_loop_v4`]. /// /// # Examples /// @@ -460,7 +449,7 @@ impl UdpSocket { /// this socket. The default value is 1 which means that multicast packets /// don't leave the local network unless explicitly requested. /// - /// Note that this may not have any effect on IPv6 sockets. + /// Note that this might not have any effect on IPv6 sockets. /// /// # Examples /// @@ -477,10 +466,7 @@ impl UdpSocket { /// Gets the value of the `IP_MULTICAST_TTL` option for this socket. /// - /// For more information about this option, see - /// [`set_multicast_ttl_v4`][link]. - /// - /// [link]: #method.set_multicast_ttl_v4 + /// For more information about this option, see [`UdpSocket::set_multicast_ttl_v4`]. /// /// # Examples /// @@ -499,7 +485,7 @@ impl UdpSocket { /// Sets the value of the `IPV6_MULTICAST_LOOP` option for this socket. /// /// Controls whether this socket sees the multicast packets it sends itself. - /// Note that this may not have any affect on IPv4 sockets. + /// Note that this might not have any affect on IPv4 sockets. /// /// # Examples /// @@ -516,10 +502,7 @@ impl UdpSocket { /// Gets the value of the `IPV6_MULTICAST_LOOP` option for this socket. /// - /// For more information about this option, see - /// [`set_multicast_loop_v6`][link]. - /// - /// [link]: #method.set_multicast_loop_v6 + /// For more information about this option, see [`UdpSocket::set_multicast_loop_v6`]. /// /// # Examples /// @@ -555,9 +538,7 @@ impl UdpSocket { /// Gets the value of the `IP_TTL` option for this socket. /// - /// For more information about this option, see [`set_ttl`][link]. - /// - /// [link]: #method.set_ttl + /// For more information about this option, see [`UdpSocket::set_ttl`]. /// /// # Examples /// @@ -597,10 +578,7 @@ impl UdpSocket { /// Executes an operation of the `IP_DROP_MEMBERSHIP` type. /// - /// For more information about this option, see - /// [`join_multicast_v4`][link]. - /// - /// [link]: #method.join_multicast_v4 + /// For more information about this option, see [`UdpSocket::join_multicast_v4`]. #[stable(feature = "net2_mutators", since = "1.9.0")] pub fn leave_multicast_v4(&self, multiaddr: &Ipv4Addr, interface: &Ipv4Addr) -> io::Result<()> { self.0.leave_multicast_v4(multiaddr, interface) @@ -608,10 +586,7 @@ impl UdpSocket { /// Executes an operation of the `IPV6_DROP_MEMBERSHIP` type. /// - /// For more information about this option, see - /// [`join_multicast_v6`][link]. - /// - /// [link]: #method.join_multicast_v6 + /// For more information about this option, see [`UdpSocket::join_multicast_v6`]. #[stable(feature = "net2_mutators", since = "1.9.0")] pub fn leave_multicast_v6(&self, multiaddr: &Ipv6Addr, interface: u32) -> io::Result<()> { self.0.leave_multicast_v6(multiaddr, interface) @@ -630,9 +605,9 @@ impl UdpSocket { /// /// let socket = UdpSocket::bind("127.0.0.1:34254").expect("couldn't bind to address"); /// match socket.take_error() { - /// Ok(Some(error)) => println!("UdpSocket error: {:?}", error), + /// Ok(Some(error)) => println!("UdpSocket error: {error:?}"), /// Ok(None) => println!("No error"), - /// Err(error) => println!("UdpSocket.take_error failed: {:?}", error), + /// Err(error) => println!("UdpSocket.take_error failed: {error:?}"), /// } /// ``` #[stable(feature = "net2_mutators", since = "1.9.0")] @@ -675,11 +650,9 @@ impl UdpSocket { /// Sends data on the socket to the remote address to which it is connected. /// - /// The [`connect`] method will connect this socket to a remote address. This + /// [`UdpSocket::connect`] will connect this socket to a remote address. This /// method will fail if the socket is not connected. /// - /// [`connect`]: #method.connect - /// /// # Examples /// /// ```no_run @@ -701,11 +674,9 @@ impl UdpSocket { /// hold the message bytes. If a message is too long to fit in the supplied buffer, /// excess bytes may be discarded. /// - /// The [`connect`] method will connect this socket to a remote address. This + /// [`UdpSocket::connect`] will connect this socket to a remote address. This /// method will fail if the socket is not connected. /// - /// [`connect`]: #method.connect - /// /// # Examples /// /// ```no_run @@ -715,8 +686,8 @@ impl UdpSocket { /// socket.connect("127.0.0.1:8080").expect("connect function failed"); /// let mut buf = [0; 10]; /// match socket.recv(&mut buf) { - /// Ok(received) => println!("received {} bytes {:?}", received, &buf[..received]), - /// Err(e) => println!("recv function failed: {:?}", e), + /// Ok(received) => println!("received {received} bytes {:?}", &buf[..received]), + /// Err(e) => println!("recv function failed: {e:?}"), /// } /// ``` #[stable(feature = "net2_mutators", since = "1.9.0")] @@ -738,11 +709,9 @@ impl UdpSocket { /// Do not use this function to implement busy waiting, instead use `libc::poll` to /// synchronize IO events on one or more sockets. /// - /// The [`connect`] method will connect this socket to a remote address. This + /// [`UdpSocket::connect`] will connect this socket to a remote address. This /// method will fail if the socket is not connected. /// - /// [`connect`]: #method.connect - /// /// # Errors /// /// This method will fail if the socket is not connected. The `connect` method @@ -757,8 +726,8 @@ impl UdpSocket { /// socket.connect("127.0.0.1:8080").expect("connect function failed"); /// let mut buf = [0; 10]; /// match socket.peek(&mut buf) { - /// Ok(received) => println!("received {} bytes", received), - /// Err(e) => println!("peek function failed: {:?}", e), + /// Ok(received) => println!("received {received} bytes"), + /// Err(e) => println!("peek function failed: {e:?}"), /// } /// ``` #[stable(feature = "peek", since = "1.18.0")] @@ -779,8 +748,6 @@ impl UdpSocket { /// `FIONBIO`. On Windows calling this method corresponds to calling /// `ioctlsocket` `FIONBIO`. /// - /// [`io::ErrorKind::WouldBlock`]: ../io/enum.ErrorKind.html#variant.WouldBlock - /// /// # Examples /// /// Creates a UDP socket bound to `127.0.0.1:7878` and read bytes in @@ -803,7 +770,7 @@ impl UdpSocket { /// // via platform-specific APIs such as epoll or IOCP /// wait_for_fd(); /// } - /// Err(e) => panic!("encountered IO error: {}", e), + /// Err(e) => panic!("encountered IO error: {e}"), /// } /// }; /// println!("bytes: {:?}", &buf[..num_bytes_read]); @@ -814,6 +781,12 @@ impl UdpSocket { } } +// In addition to the `impl`s here, `UdpSocket` also has `impl`s for +// `AsFd`/`From`/`Into` and +// `AsRawFd`/`IntoRawFd`/`FromRawFd`, on Unix and WASI, and +// `AsSocket`/`From`/`Into` and +// `AsRawSocket`/`IntoRawSocket`/`FromRawSocket` on Windows. + impl AsInner for UdpSocket { fn as_inner(&self) -> &net_imp::UdpSocket { &self.0 @@ -838,380 +811,3 @@ impl fmt::Debug for UdpSocket { self.0.fmt(f) } } - -#[cfg(all(test, not(any(target_os = "cloudabi", target_os = "emscripten", target_env = "sgx"))))] -mod tests { - use crate::io::ErrorKind; - use crate::net::test::{next_test_ip4, next_test_ip6}; - use crate::net::*; - use crate::sync::mpsc::channel; - use crate::sys_common::AsInner; - use crate::thread; - use crate::time::{Duration, Instant}; - - fn each_ip(f: &mut dyn FnMut(SocketAddr, SocketAddr)) { - f(next_test_ip4(), next_test_ip4()); - f(next_test_ip6(), next_test_ip6()); - } - - macro_rules! t { - ($e:expr) => { - match $e { - Ok(t) => t, - Err(e) => panic!("received error for `{}`: {}", stringify!($e), e), - } - }; - } - - #[test] - fn bind_error() { - match UdpSocket::bind("1.1.1.1:9999") { - Ok(..) => panic!(), - Err(e) => assert_eq!(e.kind(), ErrorKind::AddrNotAvailable), - } - } - - #[test] - fn socket_smoke_test_ip4() { - each_ip(&mut |server_ip, client_ip| { - let (tx1, rx1) = channel(); - let (tx2, rx2) = channel(); - - let _t = thread::spawn(move || { - let client = t!(UdpSocket::bind(&client_ip)); - rx1.recv().unwrap(); - t!(client.send_to(&[99], &server_ip)); - tx2.send(()).unwrap(); - }); - - let server = t!(UdpSocket::bind(&server_ip)); - tx1.send(()).unwrap(); - let mut buf = [0]; - let (nread, src) = t!(server.recv_from(&mut buf)); - assert_eq!(nread, 1); - assert_eq!(buf[0], 99); - assert_eq!(src, client_ip); - rx2.recv().unwrap(); - }) - } - - #[test] - fn socket_name() { - each_ip(&mut |addr, _| { - let server = t!(UdpSocket::bind(&addr)); - assert_eq!(addr, t!(server.local_addr())); - }) - } - - #[test] - fn socket_peer() { - each_ip(&mut |addr1, addr2| { - let server = t!(UdpSocket::bind(&addr1)); - assert_eq!(server.peer_addr().unwrap_err().kind(), ErrorKind::NotConnected); - t!(server.connect(&addr2)); - assert_eq!(addr2, t!(server.peer_addr())); - }) - } - - #[test] - fn udp_clone_smoke() { - each_ip(&mut |addr1, addr2| { - let sock1 = t!(UdpSocket::bind(&addr1)); - let sock2 = t!(UdpSocket::bind(&addr2)); - - let _t = thread::spawn(move || { - let mut buf = [0, 0]; - assert_eq!(sock2.recv_from(&mut buf).unwrap(), (1, addr1)); - assert_eq!(buf[0], 1); - t!(sock2.send_to(&[2], &addr1)); - }); - - let sock3 = t!(sock1.try_clone()); - - let (tx1, rx1) = channel(); - let (tx2, rx2) = channel(); - let _t = thread::spawn(move || { - rx1.recv().unwrap(); - t!(sock3.send_to(&[1], &addr2)); - tx2.send(()).unwrap(); - }); - tx1.send(()).unwrap(); - let mut buf = [0, 0]; - assert_eq!(sock1.recv_from(&mut buf).unwrap(), (1, addr2)); - rx2.recv().unwrap(); - }) - } - - #[test] - fn udp_clone_two_read() { - each_ip(&mut |addr1, addr2| { - let sock1 = t!(UdpSocket::bind(&addr1)); - let sock2 = t!(UdpSocket::bind(&addr2)); - let (tx1, rx) = channel(); - let tx2 = tx1.clone(); - - let _t = thread::spawn(move || { - t!(sock2.send_to(&[1], &addr1)); - rx.recv().unwrap(); - t!(sock2.send_to(&[2], &addr1)); - rx.recv().unwrap(); - }); - - let sock3 = t!(sock1.try_clone()); - - let (done, rx) = channel(); - let _t = thread::spawn(move || { - let mut buf = [0, 0]; - t!(sock3.recv_from(&mut buf)); - tx2.send(()).unwrap(); - done.send(()).unwrap(); - }); - let mut buf = [0, 0]; - t!(sock1.recv_from(&mut buf)); - tx1.send(()).unwrap(); - - rx.recv().unwrap(); - }) - } - - #[test] - fn udp_clone_two_write() { - each_ip(&mut |addr1, addr2| { - let sock1 = t!(UdpSocket::bind(&addr1)); - let sock2 = t!(UdpSocket::bind(&addr2)); - - let (tx, rx) = channel(); - let (serv_tx, serv_rx) = channel(); - - let _t = thread::spawn(move || { - let mut buf = [0, 1]; - rx.recv().unwrap(); - t!(sock2.recv_from(&mut buf)); - serv_tx.send(()).unwrap(); - }); - - let sock3 = t!(sock1.try_clone()); - - let (done, rx) = channel(); - let tx2 = tx.clone(); - let _t = thread::spawn(move || { - match sock3.send_to(&[1], &addr2) { - Ok(..) => { - let _ = tx2.send(()); - } - Err(..) => {} - } - done.send(()).unwrap(); - }); - match sock1.send_to(&[2], &addr2) { - Ok(..) => { - let _ = tx.send(()); - } - Err(..) => {} - } - drop(tx); - - rx.recv().unwrap(); - serv_rx.recv().unwrap(); - }) - } - - #[test] - fn debug() { - let name = if cfg!(windows) { "socket" } else { "fd" }; - let socket_addr = next_test_ip4(); - - let udpsock = t!(UdpSocket::bind(&socket_addr)); - let udpsock_inner = udpsock.0.socket().as_inner(); - let compare = - format!("UdpSocket {{ addr: {:?}, {}: {:?} }}", socket_addr, name, udpsock_inner); - assert_eq!(format!("{:?}", udpsock), compare); - } - - // FIXME: re-enabled openbsd/netbsd tests once their socket timeout code - // no longer has rounding errors. - // VxWorks ignores SO_SNDTIMEO. - #[cfg_attr(any(target_os = "netbsd", target_os = "openbsd", target_os = "vxworks"), ignore)] - #[test] - fn timeouts() { - let addr = next_test_ip4(); - - let stream = t!(UdpSocket::bind(&addr)); - let dur = Duration::new(15410, 0); - - assert_eq!(None, t!(stream.read_timeout())); - - t!(stream.set_read_timeout(Some(dur))); - assert_eq!(Some(dur), t!(stream.read_timeout())); - - assert_eq!(None, t!(stream.write_timeout())); - - t!(stream.set_write_timeout(Some(dur))); - assert_eq!(Some(dur), t!(stream.write_timeout())); - - t!(stream.set_read_timeout(None)); - assert_eq!(None, t!(stream.read_timeout())); - - t!(stream.set_write_timeout(None)); - assert_eq!(None, t!(stream.write_timeout())); - } - - #[test] - fn test_read_timeout() { - let addr = next_test_ip4(); - - let stream = t!(UdpSocket::bind(&addr)); - t!(stream.set_read_timeout(Some(Duration::from_millis(1000)))); - - let mut buf = [0; 10]; - - let start = Instant::now(); - loop { - let kind = stream.recv_from(&mut buf).err().expect("expected error").kind(); - if kind != ErrorKind::Interrupted { - assert!( - kind == ErrorKind::WouldBlock || kind == ErrorKind::TimedOut, - "unexpected_error: {:?}", - kind - ); - break; - } - } - assert!(start.elapsed() > Duration::from_millis(400)); - } - - #[test] - fn test_read_with_timeout() { - let addr = next_test_ip4(); - - let stream = t!(UdpSocket::bind(&addr)); - t!(stream.set_read_timeout(Some(Duration::from_millis(1000)))); - - t!(stream.send_to(b"hello world", &addr)); - - let mut buf = [0; 11]; - t!(stream.recv_from(&mut buf)); - assert_eq!(b"hello world", &buf[..]); - - let start = Instant::now(); - loop { - let kind = stream.recv_from(&mut buf).err().expect("expected error").kind(); - if kind != ErrorKind::Interrupted { - assert!( - kind == ErrorKind::WouldBlock || kind == ErrorKind::TimedOut, - "unexpected_error: {:?}", - kind - ); - break; - } - } - assert!(start.elapsed() > Duration::from_millis(400)); - } - - // Ensure the `set_read_timeout` and `set_write_timeout` calls return errors - // when passed zero Durations - #[test] - fn test_timeout_zero_duration() { - let addr = next_test_ip4(); - - let socket = t!(UdpSocket::bind(&addr)); - - let result = socket.set_write_timeout(Some(Duration::new(0, 0))); - let err = result.unwrap_err(); - assert_eq!(err.kind(), ErrorKind::InvalidInput); - - let result = socket.set_read_timeout(Some(Duration::new(0, 0))); - let err = result.unwrap_err(); - assert_eq!(err.kind(), ErrorKind::InvalidInput); - } - - #[test] - fn connect_send_recv() { - let addr = next_test_ip4(); - - let socket = t!(UdpSocket::bind(&addr)); - t!(socket.connect(addr)); - - t!(socket.send(b"hello world")); - - let mut buf = [0; 11]; - t!(socket.recv(&mut buf)); - assert_eq!(b"hello world", &buf[..]); - } - - #[test] - fn connect_send_peek_recv() { - each_ip(&mut |addr, _| { - let socket = t!(UdpSocket::bind(&addr)); - t!(socket.connect(addr)); - - t!(socket.send(b"hello world")); - - for _ in 1..3 { - let mut buf = [0; 11]; - let size = t!(socket.peek(&mut buf)); - assert_eq!(b"hello world", &buf[..]); - assert_eq!(size, 11); - } - - let mut buf = [0; 11]; - let size = t!(socket.recv(&mut buf)); - assert_eq!(b"hello world", &buf[..]); - assert_eq!(size, 11); - }) - } - - #[test] - fn peek_from() { - each_ip(&mut |addr, _| { - let socket = t!(UdpSocket::bind(&addr)); - t!(socket.send_to(b"hello world", &addr)); - - for _ in 1..3 { - let mut buf = [0; 11]; - let (size, _) = t!(socket.peek_from(&mut buf)); - assert_eq!(b"hello world", &buf[..]); - assert_eq!(size, 11); - } - - let mut buf = [0; 11]; - let (size, _) = t!(socket.recv_from(&mut buf)); - assert_eq!(b"hello world", &buf[..]); - assert_eq!(size, 11); - }) - } - - #[test] - fn ttl() { - let ttl = 100; - - let addr = next_test_ip4(); - - let stream = t!(UdpSocket::bind(&addr)); - - t!(stream.set_ttl(ttl)); - assert_eq!(ttl, t!(stream.ttl())); - } - - #[test] - fn set_nonblocking() { - each_ip(&mut |addr, _| { - let socket = t!(UdpSocket::bind(&addr)); - - t!(socket.set_nonblocking(true)); - t!(socket.set_nonblocking(false)); - - t!(socket.connect(addr)); - - t!(socket.set_nonblocking(false)); - t!(socket.set_nonblocking(true)); - - let mut buf = [0]; - match socket.recv(&mut buf) { - Ok(_) => panic!("expected error"), - Err(ref e) if e.kind() == ErrorKind::WouldBlock => {} - Err(e) => panic!("unexpected error {}", e), - } - }) - } -} diff --git a/crux-mir/lib/std/src/net/udp/tests.rs b/crux-mir/lib/std/src/net/udp/tests.rs new file mode 100644 index 000000000..f82904ffb --- /dev/null +++ b/crux-mir/lib/std/src/net/udp/tests.rs @@ -0,0 +1,365 @@ +use crate::io::ErrorKind; +use crate::net::test::{next_test_ip4, next_test_ip6}; +use crate::net::*; +use crate::sync::mpsc::channel; +use crate::thread; +use crate::time::{Duration, Instant}; + +fn each_ip(f: &mut dyn FnMut(SocketAddr, SocketAddr)) { + f(next_test_ip4(), next_test_ip4()); + f(next_test_ip6(), next_test_ip6()); +} + +macro_rules! t { + ($e:expr) => { + match $e { + Ok(t) => t, + Err(e) => panic!("received error for `{}`: {}", stringify!($e), e), + } + }; +} + +#[test] +fn bind_error() { + match UdpSocket::bind("1.1.1.1:9999") { + Ok(..) => panic!(), + Err(e) => assert_eq!(e.kind(), ErrorKind::AddrNotAvailable), + } +} + +#[test] +fn socket_smoke_test_ip4() { + each_ip(&mut |server_ip, client_ip| { + let (tx1, rx1) = channel(); + let (tx2, rx2) = channel(); + + let _t = thread::spawn(move || { + let client = t!(UdpSocket::bind(&client_ip)); + rx1.recv().unwrap(); + t!(client.send_to(&[99], &server_ip)); + tx2.send(()).unwrap(); + }); + + let server = t!(UdpSocket::bind(&server_ip)); + tx1.send(()).unwrap(); + let mut buf = [0]; + let (nread, src) = t!(server.recv_from(&mut buf)); + assert_eq!(nread, 1); + assert_eq!(buf[0], 99); + assert_eq!(src, client_ip); + rx2.recv().unwrap(); + }) +} + +#[test] +fn socket_name() { + each_ip(&mut |addr, _| { + let server = t!(UdpSocket::bind(&addr)); + assert_eq!(addr, t!(server.local_addr())); + }) +} + +#[test] +fn socket_peer() { + each_ip(&mut |addr1, addr2| { + let server = t!(UdpSocket::bind(&addr1)); + assert_eq!(server.peer_addr().unwrap_err().kind(), ErrorKind::NotConnected); + t!(server.connect(&addr2)); + assert_eq!(addr2, t!(server.peer_addr())); + }) +} + +#[test] +fn udp_clone_smoke() { + each_ip(&mut |addr1, addr2| { + let sock1 = t!(UdpSocket::bind(&addr1)); + let sock2 = t!(UdpSocket::bind(&addr2)); + + let _t = thread::spawn(move || { + let mut buf = [0, 0]; + assert_eq!(sock2.recv_from(&mut buf).unwrap(), (1, addr1)); + assert_eq!(buf[0], 1); + t!(sock2.send_to(&[2], &addr1)); + }); + + let sock3 = t!(sock1.try_clone()); + + let (tx1, rx1) = channel(); + let (tx2, rx2) = channel(); + let _t = thread::spawn(move || { + rx1.recv().unwrap(); + t!(sock3.send_to(&[1], &addr2)); + tx2.send(()).unwrap(); + }); + tx1.send(()).unwrap(); + let mut buf = [0, 0]; + assert_eq!(sock1.recv_from(&mut buf).unwrap(), (1, addr2)); + rx2.recv().unwrap(); + }) +} + +#[test] +fn udp_clone_two_read() { + each_ip(&mut |addr1, addr2| { + let sock1 = t!(UdpSocket::bind(&addr1)); + let sock2 = t!(UdpSocket::bind(&addr2)); + let (tx1, rx) = channel(); + let tx2 = tx1.clone(); + + let _t = thread::spawn(move || { + t!(sock2.send_to(&[1], &addr1)); + rx.recv().unwrap(); + t!(sock2.send_to(&[2], &addr1)); + rx.recv().unwrap(); + }); + + let sock3 = t!(sock1.try_clone()); + + let (done, rx) = channel(); + let _t = thread::spawn(move || { + let mut buf = [0, 0]; + t!(sock3.recv_from(&mut buf)); + tx2.send(()).unwrap(); + done.send(()).unwrap(); + }); + let mut buf = [0, 0]; + t!(sock1.recv_from(&mut buf)); + tx1.send(()).unwrap(); + + rx.recv().unwrap(); + }) +} + +#[test] +fn udp_clone_two_write() { + each_ip(&mut |addr1, addr2| { + let sock1 = t!(UdpSocket::bind(&addr1)); + let sock2 = t!(UdpSocket::bind(&addr2)); + + let (tx, rx) = channel(); + let (serv_tx, serv_rx) = channel(); + + let _t = thread::spawn(move || { + let mut buf = [0, 1]; + rx.recv().unwrap(); + t!(sock2.recv_from(&mut buf)); + serv_tx.send(()).unwrap(); + }); + + let sock3 = t!(sock1.try_clone()); + + let (done, rx) = channel(); + let tx2 = tx.clone(); + let _t = thread::spawn(move || { + if sock3.send_to(&[1], &addr2).is_ok() { + let _ = tx2.send(()); + } + done.send(()).unwrap(); + }); + if sock1.send_to(&[2], &addr2).is_ok() { + let _ = tx.send(()); + } + drop(tx); + + rx.recv().unwrap(); + serv_rx.recv().unwrap(); + }) +} + +#[test] +fn debug() { + let name = if cfg!(windows) { "socket" } else { "fd" }; + let socket_addr = next_test_ip4(); + + let udpsock = t!(UdpSocket::bind(&socket_addr)); + let udpsock_inner = udpsock.0.socket().as_raw(); + let compare = format!("UdpSocket {{ addr: {socket_addr:?}, {name}: {udpsock_inner:?} }}"); + assert_eq!(format!("{udpsock:?}"), compare); +} + +// FIXME: re-enabled openbsd/netbsd tests once their socket timeout code +// no longer has rounding errors. +// VxWorks ignores SO_SNDTIMEO. +#[cfg_attr(any(target_os = "netbsd", target_os = "openbsd", target_os = "vxworks"), ignore)] +#[test] +fn timeouts() { + let addr = next_test_ip4(); + + let stream = t!(UdpSocket::bind(&addr)); + let dur = Duration::new(15410, 0); + + assert_eq!(None, t!(stream.read_timeout())); + + t!(stream.set_read_timeout(Some(dur))); + assert_eq!(Some(dur), t!(stream.read_timeout())); + + assert_eq!(None, t!(stream.write_timeout())); + + t!(stream.set_write_timeout(Some(dur))); + assert_eq!(Some(dur), t!(stream.write_timeout())); + + t!(stream.set_read_timeout(None)); + assert_eq!(None, t!(stream.read_timeout())); + + t!(stream.set_write_timeout(None)); + assert_eq!(None, t!(stream.write_timeout())); +} + +#[test] +fn test_read_timeout() { + let addr = next_test_ip4(); + + let stream = t!(UdpSocket::bind(&addr)); + t!(stream.set_read_timeout(Some(Duration::from_millis(1000)))); + + let mut buf = [0; 10]; + + let start = Instant::now(); + loop { + let kind = stream.recv_from(&mut buf).err().expect("expected error").kind(); + if kind != ErrorKind::Interrupted { + assert!( + kind == ErrorKind::WouldBlock || kind == ErrorKind::TimedOut, + "unexpected_error: {:?}", + kind + ); + break; + } + } + assert!(start.elapsed() > Duration::from_millis(400)); +} + +#[test] +fn test_read_with_timeout() { + let addr = next_test_ip4(); + + let stream = t!(UdpSocket::bind(&addr)); + t!(stream.set_read_timeout(Some(Duration::from_millis(1000)))); + + t!(stream.send_to(b"hello world", &addr)); + + let mut buf = [0; 11]; + t!(stream.recv_from(&mut buf)); + assert_eq!(b"hello world", &buf[..]); + + let start = Instant::now(); + loop { + let kind = stream.recv_from(&mut buf).err().expect("expected error").kind(); + if kind != ErrorKind::Interrupted { + assert!( + kind == ErrorKind::WouldBlock || kind == ErrorKind::TimedOut, + "unexpected_error: {:?}", + kind + ); + break; + } + } + assert!(start.elapsed() > Duration::from_millis(400)); +} + +// Ensure the `set_read_timeout` and `set_write_timeout` calls return errors +// when passed zero Durations +#[test] +fn test_timeout_zero_duration() { + let addr = next_test_ip4(); + + let socket = t!(UdpSocket::bind(&addr)); + + let result = socket.set_write_timeout(Some(Duration::new(0, 0))); + let err = result.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidInput); + + let result = socket.set_read_timeout(Some(Duration::new(0, 0))); + let err = result.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidInput); +} + +#[test] +fn connect_send_recv() { + let addr = next_test_ip4(); + + let socket = t!(UdpSocket::bind(&addr)); + t!(socket.connect(addr)); + + t!(socket.send(b"hello world")); + + let mut buf = [0; 11]; + t!(socket.recv(&mut buf)); + assert_eq!(b"hello world", &buf[..]); +} + +#[test] +fn connect_send_peek_recv() { + each_ip(&mut |addr, _| { + let socket = t!(UdpSocket::bind(&addr)); + t!(socket.connect(addr)); + + t!(socket.send(b"hello world")); + + for _ in 1..3 { + let mut buf = [0; 11]; + let size = t!(socket.peek(&mut buf)); + assert_eq!(b"hello world", &buf[..]); + assert_eq!(size, 11); + } + + let mut buf = [0; 11]; + let size = t!(socket.recv(&mut buf)); + assert_eq!(b"hello world", &buf[..]); + assert_eq!(size, 11); + }) +} + +#[test] +fn peek_from() { + each_ip(&mut |addr, _| { + let socket = t!(UdpSocket::bind(&addr)); + t!(socket.send_to(b"hello world", &addr)); + + for _ in 1..3 { + let mut buf = [0; 11]; + let (size, _) = t!(socket.peek_from(&mut buf)); + assert_eq!(b"hello world", &buf[..]); + assert_eq!(size, 11); + } + + let mut buf = [0; 11]; + let (size, _) = t!(socket.recv_from(&mut buf)); + assert_eq!(b"hello world", &buf[..]); + assert_eq!(size, 11); + }) +} + +#[test] +fn ttl() { + let ttl = 100; + + let addr = next_test_ip4(); + + let stream = t!(UdpSocket::bind(&addr)); + + t!(stream.set_ttl(ttl)); + assert_eq!(ttl, t!(stream.ttl())); +} + +#[test] +fn set_nonblocking() { + each_ip(&mut |addr, _| { + let socket = t!(UdpSocket::bind(&addr)); + + t!(socket.set_nonblocking(true)); + t!(socket.set_nonblocking(false)); + + t!(socket.connect(addr)); + + t!(socket.set_nonblocking(false)); + t!(socket.set_nonblocking(true)); + + let mut buf = [0]; + match socket.recv(&mut buf) { + Ok(_) => panic!("expected error"), + Err(ref e) if e.kind() == ErrorKind::WouldBlock => {} + Err(e) => panic!("unexpected error {e}"), + } + }) +} diff --git a/crux-mir/lib/std/src/num.rs b/crux-mir/lib/std/src/num.rs index de8acf8d9..46064bd28 100644 --- a/crux-mir/lib/std/src/num.rs +++ b/crux-mir/lib/std/src/num.rs @@ -6,6 +6,14 @@ #![stable(feature = "rust1", since = "1.0.0")] #![allow(missing_docs)] +#[cfg(test)] +mod tests; + +#[cfg(test)] +mod benches; + +#[unstable(feature = "saturating_int_impl", issue = "87920")] +pub use core::num::Saturating; #[stable(feature = "rust1", since = "1.0.0")] pub use core::num::Wrapping; #[stable(feature = "rust1", since = "1.0.0")] @@ -16,12 +24,7 @@ pub use core::num::{NonZeroI128, NonZeroI16, NonZeroI32, NonZeroI64, NonZeroI8, #[stable(feature = "nonzero", since = "1.28.0")] pub use core::num::{NonZeroU128, NonZeroU16, NonZeroU32, NonZeroU64, NonZeroU8, NonZeroUsize}; -#[unstable( - feature = "int_error_matching", - reason = "it can be useful to match errors when making error messages \ - for integer parsing", - issue = "22639" -)] +#[stable(feature = "int_error_matching", since = "1.55.0")] pub use core::num::IntErrorKind; #[cfg(test)] @@ -48,259 +51,3 @@ where assert_eq!(ten.div(two), ten / two); assert_eq!(ten.rem(two), ten % two); } - -#[cfg(test)] -mod tests { - use crate::ops::Mul; - use crate::u16; - use crate::u32; - use crate::u64; - use crate::u8; - use crate::usize; - - #[test] - fn test_saturating_add_uint() { - use crate::usize::MAX; - assert_eq!(3_usize.saturating_add(5_usize), 8_usize); - assert_eq!(3_usize.saturating_add(MAX - 1), MAX); - assert_eq!(MAX.saturating_add(MAX), MAX); - assert_eq!((MAX - 2).saturating_add(1), MAX - 1); - } - - #[test] - fn test_saturating_sub_uint() { - use crate::usize::MAX; - assert_eq!(5_usize.saturating_sub(3_usize), 2_usize); - assert_eq!(3_usize.saturating_sub(5_usize), 0_usize); - assert_eq!(0_usize.saturating_sub(1_usize), 0_usize); - assert_eq!((MAX - 1).saturating_sub(MAX), 0); - } - - #[test] - fn test_saturating_add_int() { - use crate::isize::{MAX, MIN}; - assert_eq!(3i32.saturating_add(5), 8); - assert_eq!(3isize.saturating_add(MAX - 1), MAX); - assert_eq!(MAX.saturating_add(MAX), MAX); - assert_eq!((MAX - 2).saturating_add(1), MAX - 1); - assert_eq!(3i32.saturating_add(-5), -2); - assert_eq!(MIN.saturating_add(-1), MIN); - assert_eq!((-2isize).saturating_add(-MAX), MIN); - } - - #[test] - fn test_saturating_sub_int() { - use crate::isize::{MAX, MIN}; - assert_eq!(3i32.saturating_sub(5), -2); - assert_eq!(MIN.saturating_sub(1), MIN); - assert_eq!((-2isize).saturating_sub(MAX), MIN); - assert_eq!(3i32.saturating_sub(-5), 8); - assert_eq!(3isize.saturating_sub(-(MAX - 1)), MAX); - assert_eq!(MAX.saturating_sub(-MAX), MAX); - assert_eq!((MAX - 2).saturating_sub(-1), MAX - 1); - } - - #[test] - fn test_checked_add() { - let five_less = usize::MAX - 5; - assert_eq!(five_less.checked_add(0), Some(usize::MAX - 5)); - assert_eq!(five_less.checked_add(1), Some(usize::MAX - 4)); - assert_eq!(five_less.checked_add(2), Some(usize::MAX - 3)); - assert_eq!(five_less.checked_add(3), Some(usize::MAX - 2)); - assert_eq!(five_less.checked_add(4), Some(usize::MAX - 1)); - assert_eq!(five_less.checked_add(5), Some(usize::MAX)); - assert_eq!(five_less.checked_add(6), None); - assert_eq!(five_less.checked_add(7), None); - } - - #[test] - fn test_checked_sub() { - assert_eq!(5_usize.checked_sub(0), Some(5)); - assert_eq!(5_usize.checked_sub(1), Some(4)); - assert_eq!(5_usize.checked_sub(2), Some(3)); - assert_eq!(5_usize.checked_sub(3), Some(2)); - assert_eq!(5_usize.checked_sub(4), Some(1)); - assert_eq!(5_usize.checked_sub(5), Some(0)); - assert_eq!(5_usize.checked_sub(6), None); - assert_eq!(5_usize.checked_sub(7), None); - } - - #[test] - fn test_checked_mul() { - let third = usize::MAX / 3; - assert_eq!(third.checked_mul(0), Some(0)); - assert_eq!(third.checked_mul(1), Some(third)); - assert_eq!(third.checked_mul(2), Some(third * 2)); - assert_eq!(third.checked_mul(3), Some(third * 3)); - assert_eq!(third.checked_mul(4), None); - } - - macro_rules! test_is_power_of_two { - ($test_name:ident, $T:ident) => { - fn $test_name() { - #![test] - assert_eq!((0 as $T).is_power_of_two(), false); - assert_eq!((1 as $T).is_power_of_two(), true); - assert_eq!((2 as $T).is_power_of_two(), true); - assert_eq!((3 as $T).is_power_of_two(), false); - assert_eq!((4 as $T).is_power_of_two(), true); - assert_eq!((5 as $T).is_power_of_two(), false); - assert_eq!(($T::MAX / 2 + 1).is_power_of_two(), true); - } - }; - } - - test_is_power_of_two! { test_is_power_of_two_u8, u8 } - test_is_power_of_two! { test_is_power_of_two_u16, u16 } - test_is_power_of_two! { test_is_power_of_two_u32, u32 } - test_is_power_of_two! { test_is_power_of_two_u64, u64 } - test_is_power_of_two! { test_is_power_of_two_uint, usize } - - macro_rules! test_next_power_of_two { - ($test_name:ident, $T:ident) => { - fn $test_name() { - #![test] - assert_eq!((0 as $T).next_power_of_two(), 1); - let mut next_power = 1; - for i in 1 as $T..40 { - assert_eq!(i.next_power_of_two(), next_power); - if i == next_power { - next_power *= 2 - } - } - } - }; - } - - test_next_power_of_two! { test_next_power_of_two_u8, u8 } - test_next_power_of_two! { test_next_power_of_two_u16, u16 } - test_next_power_of_two! { test_next_power_of_two_u32, u32 } - test_next_power_of_two! { test_next_power_of_two_u64, u64 } - test_next_power_of_two! { test_next_power_of_two_uint, usize } - - macro_rules! test_checked_next_power_of_two { - ($test_name:ident, $T:ident) => { - fn $test_name() { - #![test] - assert_eq!((0 as $T).checked_next_power_of_two(), Some(1)); - let smax = $T::MAX >> 1; - assert_eq!(smax.checked_next_power_of_two(), Some(smax + 1)); - assert_eq!((smax + 1).checked_next_power_of_two(), Some(smax + 1)); - assert_eq!((smax + 2).checked_next_power_of_two(), None); - assert_eq!(($T::MAX - 1).checked_next_power_of_two(), None); - assert_eq!($T::MAX.checked_next_power_of_two(), None); - let mut next_power = 1; - for i in 1 as $T..40 { - assert_eq!(i.checked_next_power_of_two(), Some(next_power)); - if i == next_power { - next_power *= 2 - } - } - } - }; - } - - test_checked_next_power_of_two! { test_checked_next_power_of_two_u8, u8 } - test_checked_next_power_of_two! { test_checked_next_power_of_two_u16, u16 } - test_checked_next_power_of_two! { test_checked_next_power_of_two_u32, u32 } - test_checked_next_power_of_two! { test_checked_next_power_of_two_u64, u64 } - test_checked_next_power_of_two! { test_checked_next_power_of_two_uint, usize } - - #[test] - fn test_pow() { - fn naive_pow + Copy>(one: T, base: T, exp: usize) -> T { - (0..exp).fold(one, |acc, _| acc * base) - } - macro_rules! assert_pow { - (($num:expr, $exp:expr) => $expected:expr) => {{ - let result = $num.pow($exp); - assert_eq!(result, $expected); - assert_eq!(result, naive_pow(1, $num, $exp)); - }}; - } - assert_pow!((3u32, 0 ) => 1); - assert_pow!((5u32, 1 ) => 5); - assert_pow!((-4i32, 2 ) => 16); - assert_pow!((8u32, 3 ) => 512); - assert_pow!((2u64, 50) => 1125899906842624); - } - - #[test] - fn test_uint_to_str_overflow() { - let mut u8_val: u8 = 255; - assert_eq!(u8_val.to_string(), "255"); - - u8_val = u8_val.wrapping_add(1); - assert_eq!(u8_val.to_string(), "0"); - - let mut u16_val: u16 = 65_535; - assert_eq!(u16_val.to_string(), "65535"); - - u16_val = u16_val.wrapping_add(1); - assert_eq!(u16_val.to_string(), "0"); - - let mut u32_val: u32 = 4_294_967_295; - assert_eq!(u32_val.to_string(), "4294967295"); - - u32_val = u32_val.wrapping_add(1); - assert_eq!(u32_val.to_string(), "0"); - - let mut u64_val: u64 = 18_446_744_073_709_551_615; - assert_eq!(u64_val.to_string(), "18446744073709551615"); - - u64_val = u64_val.wrapping_add(1); - assert_eq!(u64_val.to_string(), "0"); - } - - fn from_str(t: &str) -> Option { - crate::str::FromStr::from_str(t).ok() - } - - #[test] - fn test_uint_from_str_overflow() { - let mut u8_val: u8 = 255; - assert_eq!(from_str::("255"), Some(u8_val)); - assert_eq!(from_str::("256"), None); - - u8_val = u8_val.wrapping_add(1); - assert_eq!(from_str::("0"), Some(u8_val)); - assert_eq!(from_str::("-1"), None); - - let mut u16_val: u16 = 65_535; - assert_eq!(from_str::("65535"), Some(u16_val)); - assert_eq!(from_str::("65536"), None); - - u16_val = u16_val.wrapping_add(1); - assert_eq!(from_str::("0"), Some(u16_val)); - assert_eq!(from_str::("-1"), None); - - let mut u32_val: u32 = 4_294_967_295; - assert_eq!(from_str::("4294967295"), Some(u32_val)); - assert_eq!(from_str::("4294967296"), None); - - u32_val = u32_val.wrapping_add(1); - assert_eq!(from_str::("0"), Some(u32_val)); - assert_eq!(from_str::("-1"), None); - - let mut u64_val: u64 = 18_446_744_073_709_551_615; - assert_eq!(from_str::("18446744073709551615"), Some(u64_val)); - assert_eq!(from_str::("18446744073709551616"), None); - - u64_val = u64_val.wrapping_add(1); - assert_eq!(from_str::("0"), Some(u64_val)); - assert_eq!(from_str::("-1"), None); - } -} - -#[cfg(test)] -mod bench { - use test::Bencher; - - #[bench] - fn bench_pow_function(b: &mut Bencher) { - let v = (0..1024).collect::>(); - b.iter(|| { - v.iter().fold(0u32, |old, new| old.pow(*new as u32)); - }); - } -} diff --git a/crux-mir/lib/std/src/num/benches.rs b/crux-mir/lib/std/src/num/benches.rs new file mode 100644 index 000000000..233ea0506 --- /dev/null +++ b/crux-mir/lib/std/src/num/benches.rs @@ -0,0 +1,9 @@ +use test::Bencher; + +#[bench] +fn bench_pow_function(b: &mut Bencher) { + let v = (0..1024).collect::>(); + b.iter(|| { + v.iter().fold(0u32, |old, new| old.pow(*new as u32)); + }); +} diff --git a/crux-mir/lib/std/src/num/tests.rs b/crux-mir/lib/std/src/num/tests.rs new file mode 100644 index 000000000..df0df3f23 --- /dev/null +++ b/crux-mir/lib/std/src/num/tests.rs @@ -0,0 +1,230 @@ +use crate::ops::Mul; + +#[test] +fn test_saturating_add_uint() { + assert_eq!(3_usize.saturating_add(5_usize), 8_usize); + assert_eq!(3_usize.saturating_add(usize::MAX - 1), usize::MAX); + assert_eq!(usize::MAX.saturating_add(usize::MAX), usize::MAX); + assert_eq!((usize::MAX - 2).saturating_add(1), usize::MAX - 1); +} + +#[test] +fn test_saturating_sub_uint() { + assert_eq!(5_usize.saturating_sub(3_usize), 2_usize); + assert_eq!(3_usize.saturating_sub(5_usize), 0_usize); + assert_eq!(0_usize.saturating_sub(1_usize), 0_usize); + assert_eq!((usize::MAX - 1).saturating_sub(usize::MAX), 0); +} + +#[test] +fn test_saturating_add_int() { + assert_eq!(3i32.saturating_add(5), 8); + assert_eq!(3isize.saturating_add(isize::MAX - 1), isize::MAX); + assert_eq!(isize::MAX.saturating_add(isize::MAX), isize::MAX); + assert_eq!((isize::MAX - 2).saturating_add(1), isize::MAX - 1); + assert_eq!(3i32.saturating_add(-5), -2); + assert_eq!(isize::MIN.saturating_add(-1), isize::MIN); + assert_eq!((-2isize).saturating_add(-isize::MAX), isize::MIN); +} + +#[test] +fn test_saturating_sub_int() { + assert_eq!(3i32.saturating_sub(5), -2); + assert_eq!(isize::MIN.saturating_sub(1), isize::MIN); + assert_eq!((-2isize).saturating_sub(isize::MAX), isize::MIN); + assert_eq!(3i32.saturating_sub(-5), 8); + assert_eq!(3isize.saturating_sub(-(isize::MAX - 1)), isize::MAX); + assert_eq!(isize::MAX.saturating_sub(-isize::MAX), isize::MAX); + assert_eq!((isize::MAX - 2).saturating_sub(-1), isize::MAX - 1); +} + +#[test] +fn test_checked_add() { + let five_less = usize::MAX - 5; + assert_eq!(five_less.checked_add(0), Some(usize::MAX - 5)); + assert_eq!(five_less.checked_add(1), Some(usize::MAX - 4)); + assert_eq!(five_less.checked_add(2), Some(usize::MAX - 3)); + assert_eq!(five_less.checked_add(3), Some(usize::MAX - 2)); + assert_eq!(five_less.checked_add(4), Some(usize::MAX - 1)); + assert_eq!(five_less.checked_add(5), Some(usize::MAX)); + assert_eq!(five_less.checked_add(6), None); + assert_eq!(five_less.checked_add(7), None); +} + +#[test] +fn test_checked_sub() { + assert_eq!(5_usize.checked_sub(0), Some(5)); + assert_eq!(5_usize.checked_sub(1), Some(4)); + assert_eq!(5_usize.checked_sub(2), Some(3)); + assert_eq!(5_usize.checked_sub(3), Some(2)); + assert_eq!(5_usize.checked_sub(4), Some(1)); + assert_eq!(5_usize.checked_sub(5), Some(0)); + assert_eq!(5_usize.checked_sub(6), None); + assert_eq!(5_usize.checked_sub(7), None); +} + +#[test] +fn test_checked_mul() { + let third = usize::MAX / 3; + assert_eq!(third.checked_mul(0), Some(0)); + assert_eq!(third.checked_mul(1), Some(third)); + assert_eq!(third.checked_mul(2), Some(third * 2)); + assert_eq!(third.checked_mul(3), Some(third * 3)); + assert_eq!(third.checked_mul(4), None); +} + +macro_rules! test_is_power_of_two { + ($test_name:ident, $T:ident) => { + #[test] + fn $test_name() { + assert_eq!((0 as $T).is_power_of_two(), false); + assert_eq!((1 as $T).is_power_of_two(), true); + assert_eq!((2 as $T).is_power_of_two(), true); + assert_eq!((3 as $T).is_power_of_two(), false); + assert_eq!((4 as $T).is_power_of_two(), true); + assert_eq!((5 as $T).is_power_of_two(), false); + assert_eq!(($T::MAX / 2 + 1).is_power_of_two(), true); + } + }; +} + +test_is_power_of_two! { test_is_power_of_two_u8, u8 } +test_is_power_of_two! { test_is_power_of_two_u16, u16 } +test_is_power_of_two! { test_is_power_of_two_u32, u32 } +test_is_power_of_two! { test_is_power_of_two_u64, u64 } +test_is_power_of_two! { test_is_power_of_two_uint, usize } + +macro_rules! test_next_power_of_two { + ($test_name:ident, $T:ident) => { + #[test] + fn $test_name() { + assert_eq!((0 as $T).next_power_of_two(), 1); + let mut next_power = 1; + for i in 1 as $T..40 { + assert_eq!(i.next_power_of_two(), next_power); + if i == next_power { + next_power *= 2 + } + } + } + }; +} + +test_next_power_of_two! { test_next_power_of_two_u8, u8 } +test_next_power_of_two! { test_next_power_of_two_u16, u16 } +test_next_power_of_two! { test_next_power_of_two_u32, u32 } +test_next_power_of_two! { test_next_power_of_two_u64, u64 } +test_next_power_of_two! { test_next_power_of_two_uint, usize } + +macro_rules! test_checked_next_power_of_two { + ($test_name:ident, $T:ident) => { + #[test] + fn $test_name() { + assert_eq!((0 as $T).checked_next_power_of_two(), Some(1)); + let smax = $T::MAX >> 1; + assert_eq!(smax.checked_next_power_of_two(), Some(smax + 1)); + assert_eq!((smax + 1).checked_next_power_of_two(), Some(smax + 1)); + assert_eq!((smax + 2).checked_next_power_of_two(), None); + assert_eq!(($T::MAX - 1).checked_next_power_of_two(), None); + assert_eq!($T::MAX.checked_next_power_of_two(), None); + let mut next_power = 1; + for i in 1 as $T..40 { + assert_eq!(i.checked_next_power_of_two(), Some(next_power)); + if i == next_power { + next_power *= 2 + } + } + } + }; +} + +test_checked_next_power_of_two! { test_checked_next_power_of_two_u8, u8 } +test_checked_next_power_of_two! { test_checked_next_power_of_two_u16, u16 } +test_checked_next_power_of_two! { test_checked_next_power_of_two_u32, u32 } +test_checked_next_power_of_two! { test_checked_next_power_of_two_u64, u64 } +test_checked_next_power_of_two! { test_checked_next_power_of_two_uint, usize } + +#[test] +fn test_pow() { + fn naive_pow + Copy>(one: T, base: T, exp: usize) -> T { + (0..exp).fold(one, |acc, _| acc * base) + } + macro_rules! assert_pow { + (($num:expr, $exp:expr) => $expected:expr) => {{ + let result = $num.pow($exp); + assert_eq!(result, $expected); + assert_eq!(result, naive_pow(1, $num, $exp)); + }}; + } + assert_pow!((3u32, 0 ) => 1); + assert_pow!((5u32, 1 ) => 5); + assert_pow!((-4i32, 2 ) => 16); + assert_pow!((8u32, 3 ) => 512); + assert_pow!((2u64, 50) => 1125899906842624); +} + +#[test] +fn test_uint_to_str_overflow() { + let mut u8_val: u8 = 255; + assert_eq!(u8_val.to_string(), "255"); + + u8_val = u8_val.wrapping_add(1); + assert_eq!(u8_val.to_string(), "0"); + + let mut u16_val: u16 = 65_535; + assert_eq!(u16_val.to_string(), "65535"); + + u16_val = u16_val.wrapping_add(1); + assert_eq!(u16_val.to_string(), "0"); + + let mut u32_val: u32 = 4_294_967_295; + assert_eq!(u32_val.to_string(), "4294967295"); + + u32_val = u32_val.wrapping_add(1); + assert_eq!(u32_val.to_string(), "0"); + + let mut u64_val: u64 = 18_446_744_073_709_551_615; + assert_eq!(u64_val.to_string(), "18446744073709551615"); + + u64_val = u64_val.wrapping_add(1); + assert_eq!(u64_val.to_string(), "0"); +} + +fn from_str(t: &str) -> Option { + crate::str::FromStr::from_str(t).ok() +} + +#[test] +fn test_uint_from_str_overflow() { + let mut u8_val: u8 = 255; + assert_eq!(from_str::("255"), Some(u8_val)); + assert_eq!(from_str::("256"), None); + + u8_val = u8_val.wrapping_add(1); + assert_eq!(from_str::("0"), Some(u8_val)); + assert_eq!(from_str::("-1"), None); + + let mut u16_val: u16 = 65_535; + assert_eq!(from_str::("65535"), Some(u16_val)); + assert_eq!(from_str::("65536"), None); + + u16_val = u16_val.wrapping_add(1); + assert_eq!(from_str::("0"), Some(u16_val)); + assert_eq!(from_str::("-1"), None); + + let mut u32_val: u32 = 4_294_967_295; + assert_eq!(from_str::("4294967295"), Some(u32_val)); + assert_eq!(from_str::("4294967296"), None); + + u32_val = u32_val.wrapping_add(1); + assert_eq!(from_str::("0"), Some(u32_val)); + assert_eq!(from_str::("-1"), None); + + let mut u64_val: u64 = 18_446_744_073_709_551_615; + assert_eq!(from_str::("18446744073709551615"), Some(u64_val)); + assert_eq!(from_str::("18446744073709551616"), None); + + u64_val = u64_val.wrapping_add(1); + assert_eq!(from_str::("0"), Some(u64_val)); + assert_eq!(from_str::("-1"), None); +} diff --git a/crux-mir/lib/std/src/os/android/fs.rs b/crux-mir/lib/std/src/os/android/fs.rs index 9356e607c..1beb3cf6e 100644 --- a/crux-mir/lib/std/src/os/android/fs.rs +++ b/crux-mir/lib/std/src/os/android/fs.rs @@ -8,7 +8,7 @@ use crate::os::android::raw; /// OS-specific extensions to [`fs::Metadata`]. /// -/// [`fs::Metadata`]: ../../../../std/fs/struct.Metadata.html +/// [`fs::Metadata`]: crate::fs::Metadata #[stable(feature = "metadata_ext", since = "1.1.0")] pub trait MetadataExt { /// Gain a reference to the underlying `stat` structure which contains @@ -18,10 +18,10 @@ pub trait MetadataExt { /// Unix platforms. The `os::unix::fs::MetadataExt` trait contains the /// cross-Unix abstractions contained within the raw stat. #[stable(feature = "metadata_ext", since = "1.1.0")] - #[rustc_deprecated( + #[deprecated( since = "1.8.0", - reason = "deprecated in favor of the accessor \ - methods of this trait" + note = "deprecated in favor of the accessor \ + methods of this trait" )] #[allow(deprecated)] fn as_raw_stat(&self) -> &raw::stat; diff --git a/crux-mir/lib/std/src/os/android/mod.rs b/crux-mir/lib/std/src/os/android/mod.rs index dbb0127f3..5adcb82b6 100644 --- a/crux-mir/lib/std/src/os/android/mod.rs +++ b/crux-mir/lib/std/src/os/android/mod.rs @@ -3,4 +3,5 @@ #![stable(feature = "raw_ext", since = "1.1.0")] pub mod fs; +pub mod net; pub mod raw; diff --git a/crux-mir/lib/std/src/os/android/net.rs b/crux-mir/lib/std/src/os/android/net.rs new file mode 100644 index 000000000..7cecd1bbf --- /dev/null +++ b/crux-mir/lib/std/src/os/android/net.rs @@ -0,0 +1,9 @@ +//! Android-specific networking functionality. + +#![unstable(feature = "tcp_quickack", issue = "96256")] + +#[unstable(feature = "unix_socket_abstract", issue = "85410")] +pub use crate::os::net::linux_ext::addr::SocketAddrExt; + +#[unstable(feature = "tcp_quickack", issue = "96256")] +pub use crate::os::net::linux_ext::tcp::TcpStreamExt; diff --git a/crux-mir/lib/std/src/os/android/raw.rs b/crux-mir/lib/std/src/os/android/raw.rs index 2b8ade8a8..a255d0320 100644 --- a/crux-mir/lib/std/src/os/android/raw.rs +++ b/crux-mir/lib/std/src/os/android/raw.rs @@ -1,12 +1,12 @@ //! Android-specific raw type definitions #![stable(feature = "raw_ext", since = "1.1.0")] -#![rustc_deprecated( +#![deprecated( since = "1.8.0", - reason = "these type aliases are no longer supported by \ - the standard library, the `libc` crate on \ - crates.io should be used instead for the correct \ - definitions" + note = "these type aliases are no longer supported by \ + the standard library, the `libc` crate on \ + crates.io should be used instead for the correct \ + definitions" )] #![allow(deprecated)] diff --git a/crux-mir/lib/std/src/os/dragonfly/fs.rs b/crux-mir/lib/std/src/os/dragonfly/fs.rs index 8552abb1c..1424fc4c6 100644 --- a/crux-mir/lib/std/src/os/dragonfly/fs.rs +++ b/crux-mir/lib/std/src/os/dragonfly/fs.rs @@ -8,7 +8,7 @@ use crate::os::dragonfly::raw; /// OS-specific extensions to [`fs::Metadata`]. /// -/// [`fs::Metadata`]: ../../../../std/fs/struct.Metadata.html +/// [`fs::Metadata`]: crate::fs::Metadata #[stable(feature = "metadata_ext", since = "1.1.0")] pub trait MetadataExt { /// Gain a reference to the underlying `stat` structure which contains @@ -18,10 +18,10 @@ pub trait MetadataExt { /// Unix platforms. The `os::unix::fs::MetadataExt` trait contains the /// cross-Unix abstractions contained within the raw stat. #[stable(feature = "metadata_ext", since = "1.1.0")] - #[rustc_deprecated( + #[deprecated( since = "1.8.0", - reason = "deprecated in favor of the accessor \ - methods of this trait" + note = "deprecated in favor of the accessor \ + methods of this trait" )] #[allow(deprecated)] fn as_raw_stat(&self) -> &raw::stat; diff --git a/crux-mir/lib/std/src/os/dragonfly/raw.rs b/crux-mir/lib/std/src/os/dragonfly/raw.rs index 2a2d29043..071bf6199 100644 --- a/crux-mir/lib/std/src/os/dragonfly/raw.rs +++ b/crux-mir/lib/std/src/os/dragonfly/raw.rs @@ -1,12 +1,12 @@ //! Dragonfly-specific raw type definitions #![stable(feature = "raw_ext", since = "1.1.0")] -#![rustc_deprecated( +#![deprecated( since = "1.8.0", - reason = "these type aliases are no longer supported by \ - the standard library, the `libc` crate on \ - crates.io should be used instead for the correct \ - definitions" + note = "these type aliases are no longer supported by \ + the standard library, the `libc` crate on \ + crates.io should be used instead for the correct \ + definitions" )] #![allow(deprecated)] diff --git a/crux-mir/lib/std/src/os/emscripten/fs.rs b/crux-mir/lib/std/src/os/emscripten/fs.rs index f5e30dc8e..d5ec8e03c 100644 --- a/crux-mir/lib/std/src/os/emscripten/fs.rs +++ b/crux-mir/lib/std/src/os/emscripten/fs.rs @@ -8,7 +8,7 @@ use crate::os::emscripten::raw; /// OS-specific extensions to [`fs::Metadata`]. /// -/// [`fs::Metadata`]: ../../../../std/fs/struct.Metadata.html +/// [`fs::Metadata`]: crate::fs::Metadata #[stable(feature = "metadata_ext", since = "1.1.0")] pub trait MetadataExt { /// Gain a reference to the underlying `stat` structure which contains @@ -18,10 +18,10 @@ pub trait MetadataExt { /// Unix platforms. The `os::unix::fs::MetadataExt` trait contains the /// cross-Unix abstractions contained within the raw stat. #[stable(feature = "metadata_ext", since = "1.1.0")] - #[rustc_deprecated( + #[deprecated( since = "1.8.0", - reason = "deprecated in favor of the accessor \ - methods of this trait" + note = "deprecated in favor of the accessor \ + methods of this trait" )] #[allow(deprecated)] fn as_raw_stat(&self) -> &raw::stat; diff --git a/crux-mir/lib/std/src/os/emscripten/raw.rs b/crux-mir/lib/std/src/os/emscripten/raw.rs index dda7c8252..d23011c73 100644 --- a/crux-mir/lib/std/src/os/emscripten/raw.rs +++ b/crux-mir/lib/std/src/os/emscripten/raw.rs @@ -3,12 +3,12 @@ //! except using the musl-specific stat64 structure in liblibc. #![stable(feature = "raw_ext", since = "1.1.0")] -#![rustc_deprecated( +#![deprecated( since = "1.8.0", - reason = "these type aliases are no longer supported by \ - the standard library, the `libc` crate on \ - crates.io should be used instead for the correct \ - definitions" + note = "these type aliases are no longer supported by \ + the standard library, the `libc` crate on \ + crates.io should be used instead for the correct \ + definitions" )] #![allow(deprecated)] @@ -22,7 +22,6 @@ pub type mode_t = u32; #[stable(feature = "pthread_t", since = "1.8.0")] pub type pthread_t = c_ulong; -#[doc(inline)] #[stable(feature = "raw_ext", since = "1.1.0")] pub type blkcnt_t = u64; #[stable(feature = "raw_ext", since = "1.1.0")] diff --git a/crux-mir/lib/std/src/os/espidf/fs.rs b/crux-mir/lib/std/src/os/espidf/fs.rs new file mode 100644 index 000000000..88701dafe --- /dev/null +++ b/crux-mir/lib/std/src/os/espidf/fs.rs @@ -0,0 +1,117 @@ +#![stable(feature = "metadata_ext", since = "1.1.0")] + +use crate::fs::Metadata; +use crate::sys_common::AsInner; + +#[allow(deprecated)] +use crate::os::espidf::raw; + +/// OS-specific extensions to [`fs::Metadata`]. +/// +/// [`fs::Metadata`]: crate::fs::Metadata +#[stable(feature = "metadata_ext", since = "1.1.0")] +pub trait MetadataExt { + #[stable(feature = "metadata_ext", since = "1.1.0")] + #[deprecated( + since = "1.8.0", + note = "deprecated in favor of the accessor \ + methods of this trait" + )] + #[allow(deprecated)] + fn as_raw_stat(&self) -> &raw::stat; + + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_dev(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_ino(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_mode(&self) -> u32; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_nlink(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_uid(&self) -> u32; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_gid(&self) -> u32; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_rdev(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_size(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_atime(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_atime_nsec(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_mtime(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_mtime_nsec(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_ctime(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_ctime_nsec(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_blksize(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_blocks(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_spare4(&self) -> [u32; 2]; +} + +#[stable(feature = "metadata_ext", since = "1.1.0")] +impl MetadataExt for Metadata { + #[allow(deprecated)] + fn as_raw_stat(&self) -> &raw::stat { + unsafe { &*(self.as_inner().as_inner() as *const libc::stat as *const raw::stat) } + } + fn st_dev(&self) -> u64 { + self.as_inner().as_inner().st_dev as u64 + } + fn st_ino(&self) -> u64 { + self.as_inner().as_inner().st_ino as u64 + } + fn st_mode(&self) -> u32 { + self.as_inner().as_inner().st_mode as u32 + } + fn st_nlink(&self) -> u64 { + self.as_inner().as_inner().st_nlink as u64 + } + fn st_uid(&self) -> u32 { + self.as_inner().as_inner().st_uid as u32 + } + fn st_gid(&self) -> u32 { + self.as_inner().as_inner().st_gid as u32 + } + fn st_rdev(&self) -> u64 { + self.as_inner().as_inner().st_rdev as u64 + } + fn st_size(&self) -> u64 { + self.as_inner().as_inner().st_size as u64 + } + fn st_atime(&self) -> i64 { + self.as_inner().as_inner().st_atime as i64 + } + fn st_atime_nsec(&self) -> i64 { + 0 + } + fn st_mtime(&self) -> i64 { + self.as_inner().as_inner().st_mtime as i64 + } + fn st_mtime_nsec(&self) -> i64 { + 0 + } + fn st_ctime(&self) -> i64 { + self.as_inner().as_inner().st_ctime as i64 + } + fn st_ctime_nsec(&self) -> i64 { + 0 + } + fn st_blksize(&self) -> u64 { + self.as_inner().as_inner().st_blksize as u64 + } + fn st_blocks(&self) -> u64 { + self.as_inner().as_inner().st_blocks as u64 + } + fn st_spare4(&self) -> [u32; 2] { + let spare4 = self.as_inner().as_inner().st_spare4; + [spare4[0] as u32, spare4[1] as u32] + } +} diff --git a/crux-mir/lib/std/src/os/espidf/mod.rs b/crux-mir/lib/std/src/os/espidf/mod.rs new file mode 100644 index 000000000..a9cef9709 --- /dev/null +++ b/crux-mir/lib/std/src/os/espidf/mod.rs @@ -0,0 +1,6 @@ +//! Definitions for the ESP-IDF framework. + +#![stable(feature = "raw_ext", since = "1.1.0")] + +pub mod fs; +pub mod raw; diff --git a/crux-mir/lib/std/src/os/espidf/raw.rs b/crux-mir/lib/std/src/os/espidf/raw.rs new file mode 100644 index 000000000..7df0e74b2 --- /dev/null +++ b/crux-mir/lib/std/src/os/espidf/raw.rs @@ -0,0 +1,69 @@ +//! Raw type definitions for the ESP-IDF framework. + +#![stable(feature = "raw_ext", since = "1.1.0")] +#![deprecated( + since = "1.8.0", + note = "these type aliases are no longer supported by \ + the standard library, the `libc` crate on \ + crates.io should be used instead for the correct \ + definitions" +)] + +use crate::os::raw::c_long; +use crate::os::unix::raw::{gid_t, uid_t}; + +#[stable(feature = "pthread_t", since = "1.8.0")] +pub type pthread_t = libc::pthread_t; + +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type blkcnt_t = libc::blkcnt_t; + +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type blksize_t = libc::blksize_t; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type dev_t = libc::dev_t; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type ino_t = libc::ino_t; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type mode_t = libc::mode_t; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type nlink_t = libc::nlink_t; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type off_t = libc::off_t; + +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type time_t = libc::time_t; + +#[repr(C)] +#[derive(Clone)] +#[stable(feature = "raw_ext", since = "1.1.0")] +pub struct stat { + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_dev: dev_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ino: ino_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mode: mode_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_nlink: nlink_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_uid: uid_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_gid: gid_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_rdev: dev_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_size: off_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_atime: time_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mtime: time_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ctime: time_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_blksize: blksize_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_blocks: blkcnt_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_spare4: [c_long; 2usize], +} diff --git a/crux-mir/lib/std/src/os/fd/mod.rs b/crux-mir/lib/std/src/os/fd/mod.rs new file mode 100644 index 000000000..35de4860f --- /dev/null +++ b/crux-mir/lib/std/src/os/fd/mod.rs @@ -0,0 +1,25 @@ +//! Owned and borrowed Unix-like file descriptors. +//! +//! This module is supported on Unix platforms and WASI, which both use a +//! similar file descriptor system for referencing OS resources. + +#![stable(feature = "os_fd", since = "1.66.0")] +#![deny(unsafe_op_in_unsafe_fn)] + +// `RawFd`, `AsRawFd`, etc. +mod raw; + +// `OwnedFd`, `AsFd`, etc. +mod owned; + +// Implementations for `AsRawFd` etc. for network types. +mod net; + +#[cfg(test)] +mod tests; + +// Export the types and traits for the public API. +#[stable(feature = "os_fd", since = "1.66.0")] +pub use owned::*; +#[stable(feature = "os_fd", since = "1.66.0")] +pub use raw::*; diff --git a/crux-mir/lib/std/src/os/fd/net.rs b/crux-mir/lib/std/src/os/fd/net.rs new file mode 100644 index 000000000..843f45f7f --- /dev/null +++ b/crux-mir/lib/std/src/os/fd/net.rs @@ -0,0 +1,46 @@ +use crate::os::fd::owned::OwnedFd; +use crate::os::fd::raw::{AsRawFd, FromRawFd, IntoRawFd, RawFd}; +use crate::sys_common::{self, AsInner, FromInner, IntoInner}; +use crate::{net, sys}; + +macro_rules! impl_as_raw_fd { + ($($t:ident)*) => {$( + #[stable(feature = "rust1", since = "1.0.0")] + impl AsRawFd for net::$t { + #[inline] + fn as_raw_fd(&self) -> RawFd { + self.as_inner().socket().as_raw_fd() + } + } + )*}; +} +impl_as_raw_fd! { TcpStream TcpListener UdpSocket } + +macro_rules! impl_from_raw_fd { + ($($t:ident)*) => {$( + #[stable(feature = "from_raw_os", since = "1.1.0")] + impl FromRawFd for net::$t { + #[inline] + unsafe fn from_raw_fd(fd: RawFd) -> net::$t { + unsafe { + let socket = sys::net::Socket::from_inner(FromInner::from_inner(OwnedFd::from_raw_fd(fd))); + net::$t::from_inner(sys_common::net::$t::from_inner(socket)) + } + } + } + )*}; +} +impl_from_raw_fd! { TcpStream TcpListener UdpSocket } + +macro_rules! impl_into_raw_fd { + ($($t:ident)*) => {$( + #[stable(feature = "into_raw_os", since = "1.4.0")] + impl IntoRawFd for net::$t { + #[inline] + fn into_raw_fd(self) -> RawFd { + self.into_inner().into_socket().into_inner().into_inner().into_raw_fd() + } + } + )*}; +} +impl_into_raw_fd! { TcpStream TcpListener UdpSocket } diff --git a/crux-mir/lib/std/src/os/fd/owned.rs b/crux-mir/lib/std/src/os/fd/owned.rs new file mode 100644 index 000000000..c41e093a7 --- /dev/null +++ b/crux-mir/lib/std/src/os/fd/owned.rs @@ -0,0 +1,456 @@ +//! Owned and borrowed Unix-like file descriptors. + +#![stable(feature = "io_safety", since = "1.63.0")] +#![deny(unsafe_op_in_unsafe_fn)] + +use super::raw::{AsRawFd, FromRawFd, IntoRawFd, RawFd}; +use crate::fmt; +use crate::fs; +use crate::io; +use crate::marker::PhantomData; +use crate::mem::forget; +#[cfg(not(any(target_arch = "wasm32", target_env = "sgx")))] +use crate::sys::cvt; +use crate::sys_common::{AsInner, FromInner, IntoInner}; + +/// A borrowed file descriptor. +/// +/// This has a lifetime parameter to tie it to the lifetime of something that +/// owns the file descriptor. +/// +/// This uses `repr(transparent)` and has the representation of a host file +/// descriptor, so it can be used in FFI in places where a file descriptor is +/// passed as an argument, it is not captured or consumed, and it never has the +/// value `-1`. +/// +/// This type's `.to_owned()` implementation returns another `BorrowedFd` +/// rather than an `OwnedFd`. It just makes a trivial copy of the raw file +/// descriptor, which is then borrowed under the same lifetime. +#[derive(Copy, Clone)] +#[repr(transparent)] +#[rustc_layout_scalar_valid_range_start(0)] +// libstd/os/raw/mod.rs assures me that every libstd-supported platform has a +// 32-bit c_int. Below is -2, in two's complement, but that only works out +// because c_int is 32 bits. +#[rustc_layout_scalar_valid_range_end(0xFF_FF_FF_FE)] +#[rustc_nonnull_optimization_guaranteed] +#[stable(feature = "io_safety", since = "1.63.0")] +pub struct BorrowedFd<'fd> { + fd: RawFd, + _phantom: PhantomData<&'fd OwnedFd>, +} + +/// An owned file descriptor. +/// +/// This closes the file descriptor on drop. +/// +/// This uses `repr(transparent)` and has the representation of a host file +/// descriptor, so it can be used in FFI in places where a file descriptor is +/// passed as a consumed argument or returned as an owned value, and it never +/// has the value `-1`. +#[repr(transparent)] +#[rustc_layout_scalar_valid_range_start(0)] +// libstd/os/raw/mod.rs assures me that every libstd-supported platform has a +// 32-bit c_int. Below is -2, in two's complement, but that only works out +// because c_int is 32 bits. +#[rustc_layout_scalar_valid_range_end(0xFF_FF_FF_FE)] +#[rustc_nonnull_optimization_guaranteed] +#[stable(feature = "io_safety", since = "1.63.0")] +pub struct OwnedFd { + fd: RawFd, +} + +impl BorrowedFd<'_> { + /// Return a `BorrowedFd` holding the given raw file descriptor. + /// + /// # Safety + /// + /// The resource pointed to by `fd` must remain open for the duration of + /// the returned `BorrowedFd`, and it must not have the value `-1`. + #[inline] + #[rustc_const_stable(feature = "io_safety", since = "1.63.0")] + #[stable(feature = "io_safety", since = "1.63.0")] + pub const unsafe fn borrow_raw(fd: RawFd) -> Self { + assert!(fd != u32::MAX as RawFd); + // SAFETY: we just asserted that the value is in the valid range and isn't `-1` (the only value bigger than `0xFF_FF_FF_FE` unsigned) + unsafe { Self { fd, _phantom: PhantomData } } + } +} + +impl OwnedFd { + /// Creates a new `OwnedFd` instance that shares the same underlying file + /// description as the existing `OwnedFd` instance. + #[stable(feature = "io_safety", since = "1.63.0")] + pub fn try_clone(&self) -> crate::io::Result { + self.as_fd().try_clone_to_owned() + } +} + +impl BorrowedFd<'_> { + /// Creates a new `OwnedFd` instance that shares the same underlying file + /// description as the existing `BorrowedFd` instance. + #[cfg(not(target_arch = "wasm32"))] + #[stable(feature = "io_safety", since = "1.63.0")] + pub fn try_clone_to_owned(&self) -> crate::io::Result { + // We want to atomically duplicate this file descriptor and set the + // CLOEXEC flag, and currently that's done via F_DUPFD_CLOEXEC. This + // is a POSIX flag that was added to Linux in 2.6.24. + #[cfg(not(target_os = "espidf"))] + let cmd = libc::F_DUPFD_CLOEXEC; + + // For ESP-IDF, F_DUPFD is used instead, because the CLOEXEC semantics + // will never be supported, as this is a bare metal framework with + // no capabilities for multi-process execution. While F_DUPFD is also + // not supported yet, it might be (currently it returns ENOSYS). + #[cfg(target_os = "espidf")] + let cmd = libc::F_DUPFD; + + // Avoid using file descriptors below 3 as they are used for stdio + let fd = cvt(unsafe { libc::fcntl(self.as_raw_fd(), cmd, 3) })?; + Ok(unsafe { OwnedFd::from_raw_fd(fd) }) + } + + /// Creates a new `OwnedFd` instance that shares the same underlying file + /// description as the existing `BorrowedFd` instance. + #[cfg(target_arch = "wasm32")] + #[stable(feature = "io_safety", since = "1.63.0")] + pub fn try_clone_to_owned(&self) -> crate::io::Result { + Err(crate::io::const_io_error!( + crate::io::ErrorKind::Unsupported, + "operation not supported on WASI yet", + )) + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsRawFd for BorrowedFd<'_> { + #[inline] + fn as_raw_fd(&self) -> RawFd { + self.fd + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsRawFd for OwnedFd { + #[inline] + fn as_raw_fd(&self) -> RawFd { + self.fd + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl IntoRawFd for OwnedFd { + #[inline] + fn into_raw_fd(self) -> RawFd { + let fd = self.fd; + forget(self); + fd + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl FromRawFd for OwnedFd { + /// Constructs a new instance of `Self` from the given raw file descriptor. + /// + /// # Safety + /// + /// The resource pointed to by `fd` must be open and suitable for assuming + /// ownership. The resource must not require any cleanup other than `close`. + #[inline] + unsafe fn from_raw_fd(fd: RawFd) -> Self { + assert_ne!(fd, u32::MAX as RawFd); + // SAFETY: we just asserted that the value is in the valid range and isn't `-1` (the only value bigger than `0xFF_FF_FF_FE` unsigned) + unsafe { Self { fd } } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl Drop for OwnedFd { + #[inline] + fn drop(&mut self) { + unsafe { + // Note that errors are ignored when closing a file descriptor. The + // reason for this is that if an error occurs we don't actually know if + // the file descriptor was closed or not, and if we retried (for + // something like EINTR), we might close another valid file descriptor + // opened after we closed ours. + let _ = libc::close(self.fd); + } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl fmt::Debug for BorrowedFd<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("BorrowedFd").field("fd", &self.fd).finish() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl fmt::Debug for OwnedFd { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OwnedFd").field("fd", &self.fd).finish() + } +} + +macro_rules! impl_is_terminal { + ($($t:ty),*$(,)?) => {$( + #[unstable(feature = "sealed", issue = "none")] + impl crate::sealed::Sealed for $t {} + + #[unstable(feature = "is_terminal", issue = "98070")] + impl crate::io::IsTerminal for $t { + #[inline] + fn is_terminal(&self) -> bool { + crate::sys::io::is_terminal(self) + } + } + )*} +} + +impl_is_terminal!(BorrowedFd<'_>, OwnedFd); + +/// A trait to borrow the file descriptor from an underlying object. +/// +/// This is only available on unix platforms and must be imported in order to +/// call the method. Windows platforms have a corresponding `AsHandle` and +/// `AsSocket` set of traits. +#[stable(feature = "io_safety", since = "1.63.0")] +pub trait AsFd { + /// Borrows the file descriptor. + /// + /// # Example + /// + /// ```rust,no_run + /// use std::fs::File; + /// # use std::io; + /// # #[cfg(any(unix, target_os = "wasi"))] + /// # use std::os::fd::{AsFd, BorrowedFd}; + /// + /// let mut f = File::open("foo.txt")?; + /// # #[cfg(any(unix, target_os = "wasi"))] + /// let borrowed_fd: BorrowedFd<'_> = f.as_fd(); + /// # Ok::<(), io::Error>(()) + /// ``` + #[stable(feature = "io_safety", since = "1.63.0")] + fn as_fd(&self) -> BorrowedFd<'_>; +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsFd for &T { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + T::as_fd(self) + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsFd for &mut T { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + T::as_fd(self) + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsFd for BorrowedFd<'_> { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + *self + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsFd for OwnedFd { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + // Safety: `OwnedFd` and `BorrowedFd` have the same validity + // invariants, and the `BorrowdFd` is bounded by the lifetime + // of `&self`. + unsafe { BorrowedFd::borrow_raw(self.as_raw_fd()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsFd for fs::File { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + self.as_inner().as_fd() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for OwnedFd { + #[inline] + fn from(file: fs::File) -> OwnedFd { + file.into_inner().into_inner().into_inner() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for fs::File { + #[inline] + fn from(owned_fd: OwnedFd) -> Self { + Self::from_inner(FromInner::from_inner(FromInner::from_inner(owned_fd))) + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsFd for crate::net::TcpStream { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + self.as_inner().socket().as_fd() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for OwnedFd { + #[inline] + fn from(tcp_stream: crate::net::TcpStream) -> OwnedFd { + tcp_stream.into_inner().into_socket().into_inner().into_inner().into() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for crate::net::TcpStream { + #[inline] + fn from(owned_fd: OwnedFd) -> Self { + Self::from_inner(FromInner::from_inner(FromInner::from_inner(FromInner::from_inner( + owned_fd, + )))) + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsFd for crate::net::TcpListener { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + self.as_inner().socket().as_fd() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for OwnedFd { + #[inline] + fn from(tcp_listener: crate::net::TcpListener) -> OwnedFd { + tcp_listener.into_inner().into_socket().into_inner().into_inner().into() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for crate::net::TcpListener { + #[inline] + fn from(owned_fd: OwnedFd) -> Self { + Self::from_inner(FromInner::from_inner(FromInner::from_inner(FromInner::from_inner( + owned_fd, + )))) + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsFd for crate::net::UdpSocket { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + self.as_inner().socket().as_fd() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for OwnedFd { + #[inline] + fn from(udp_socket: crate::net::UdpSocket) -> OwnedFd { + udp_socket.into_inner().into_socket().into_inner().into_inner().into() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for crate::net::UdpSocket { + #[inline] + fn from(owned_fd: OwnedFd) -> Self { + Self::from_inner(FromInner::from_inner(FromInner::from_inner(FromInner::from_inner( + owned_fd, + )))) + } +} + +#[stable(feature = "asfd_ptrs", since = "1.64.0")] +/// This impl allows implementing traits that require `AsFd` on Arc. +/// ``` +/// # #[cfg(any(unix, target_os = "wasi"))] mod group_cfg { +/// # #[cfg(target_os = "wasi")] +/// # use std::os::wasi::io::AsFd; +/// # #[cfg(unix)] +/// # use std::os::unix::io::AsFd; +/// use std::net::UdpSocket; +/// use std::sync::Arc; +/// +/// trait MyTrait: AsFd {} +/// impl MyTrait for Arc {} +/// impl MyTrait for Box {} +/// # } +/// ``` +impl AsFd for crate::sync::Arc { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + (**self).as_fd() + } +} + +#[stable(feature = "asfd_ptrs", since = "1.64.0")] +impl AsFd for Box { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + (**self).as_fd() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsFd for io::Stdin { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(0) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl<'a> AsFd for io::StdinLock<'a> { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + // SAFETY: user code should not close stdin out from under the standard library + unsafe { BorrowedFd::borrow_raw(0) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsFd for io::Stdout { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(1) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl<'a> AsFd for io::StdoutLock<'a> { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + // SAFETY: user code should not close stdout out from under the standard library + unsafe { BorrowedFd::borrow_raw(1) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsFd for io::Stderr { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + unsafe { BorrowedFd::borrow_raw(2) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl<'a> AsFd for io::StderrLock<'a> { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + // SAFETY: user code should not close stderr out from under the standard library + unsafe { BorrowedFd::borrow_raw(2) } + } +} diff --git a/crux-mir/lib/std/src/os/fd/raw.rs b/crux-mir/lib/std/src/os/fd/raw.rs new file mode 100644 index 000000000..f92a05066 --- /dev/null +++ b/crux-mir/lib/std/src/os/fd/raw.rs @@ -0,0 +1,253 @@ +//! Raw Unix-like file descriptors. + +#![stable(feature = "rust1", since = "1.0.0")] + +use crate::fs; +use crate::io; +use crate::os::raw; +#[cfg(all(doc, not(target_arch = "wasm32")))] +use crate::os::unix::io::AsFd; +#[cfg(unix)] +use crate::os::unix::io::OwnedFd; +#[cfg(target_os = "wasi")] +use crate::os::wasi::io::OwnedFd; +use crate::sys_common::{AsInner, IntoInner}; + +/// Raw file descriptors. +#[rustc_allowed_through_unstable_modules] +#[stable(feature = "rust1", since = "1.0.0")] +pub type RawFd = raw::c_int; + +/// A trait to extract the raw file descriptor from an underlying object. +/// +/// This is only available on unix and WASI platforms and must be imported in +/// order to call the method. Windows platforms have a corresponding +/// `AsRawHandle` and `AsRawSocket` set of traits. +#[rustc_allowed_through_unstable_modules] +#[stable(feature = "rust1", since = "1.0.0")] +pub trait AsRawFd { + /// Extracts the raw file descriptor. + /// + /// This function is typically used to **borrow** an owned file descriptor. + /// When used in this way, this method does **not** pass ownership of the + /// raw file descriptor to the caller, and the file descriptor is only + /// guaranteed to be valid while the original object has not yet been + /// destroyed. + /// + /// However, borrowing is not strictly required. See [`AsFd::as_fd`] + /// for an API which strictly borrows a file descriptor. + /// + /// # Example + /// + /// ```no_run + /// use std::fs::File; + /// # use std::io; + /// #[cfg(any(unix, target_os = "wasi"))] + /// use std::os::fd::{AsRawFd, RawFd}; + /// + /// let mut f = File::open("foo.txt")?; + /// // Note that `raw_fd` is only valid as long as `f` exists. + /// #[cfg(any(unix, target_os = "wasi"))] + /// let raw_fd: RawFd = f.as_raw_fd(); + /// # Ok::<(), io::Error>(()) + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + fn as_raw_fd(&self) -> RawFd; +} + +/// A trait to express the ability to construct an object from a raw file +/// descriptor. +#[rustc_allowed_through_unstable_modules] +#[stable(feature = "from_raw_os", since = "1.1.0")] +pub trait FromRawFd { + /// Constructs a new instance of `Self` from the given raw file + /// descriptor. + /// + /// This function is typically used to **consume ownership** of the + /// specified file descriptor. When used in this way, the returned object + /// will take responsibility for closing it when the object goes out of + /// scope. + /// + /// However, consuming ownership is not strictly required. Use a + /// [`From::from`] implementation for an API which strictly + /// consumes ownership. + /// + /// # Safety + /// + /// The `fd` passed in must be a valid and open file descriptor. + /// + /// # Example + /// + /// ```no_run + /// use std::fs::File; + /// # use std::io; + /// #[cfg(any(unix, target_os = "wasi"))] + /// use std::os::fd::{FromRawFd, IntoRawFd, RawFd}; + /// + /// let f = File::open("foo.txt")?; + /// # #[cfg(any(unix, target_os = "wasi"))] + /// let raw_fd: RawFd = f.into_raw_fd(); + /// // SAFETY: no other functions should call `from_raw_fd`, so there + /// // is only one owner for the file descriptor. + /// # #[cfg(any(unix, target_os = "wasi"))] + /// let f = unsafe { File::from_raw_fd(raw_fd) }; + /// # Ok::<(), io::Error>(()) + /// ``` + #[stable(feature = "from_raw_os", since = "1.1.0")] + unsafe fn from_raw_fd(fd: RawFd) -> Self; +} + +/// A trait to express the ability to consume an object and acquire ownership of +/// its raw file descriptor. +#[rustc_allowed_through_unstable_modules] +#[stable(feature = "into_raw_os", since = "1.4.0")] +pub trait IntoRawFd { + /// Consumes this object, returning the raw underlying file descriptor. + /// + /// This function is typically used to **transfer ownership** of the underlying + /// file descriptor to the caller. When used in this way, callers are then the unique + /// owners of the file descriptor and must close it once it's no longer needed. + /// + /// However, transferring ownership is not strictly required. Use a + /// [`Into::into`] implementation for an API which strictly + /// transfers ownership. + /// + /// # Example + /// + /// ```no_run + /// use std::fs::File; + /// # use std::io; + /// #[cfg(any(unix, target_os = "wasi"))] + /// use std::os::fd::{IntoRawFd, RawFd}; + /// + /// let f = File::open("foo.txt")?; + /// #[cfg(any(unix, target_os = "wasi"))] + /// let raw_fd: RawFd = f.into_raw_fd(); + /// # Ok::<(), io::Error>(()) + /// ``` + #[stable(feature = "into_raw_os", since = "1.4.0")] + fn into_raw_fd(self) -> RawFd; +} + +#[stable(feature = "raw_fd_reflexive_traits", since = "1.48.0")] +impl AsRawFd for RawFd { + #[inline] + fn as_raw_fd(&self) -> RawFd { + *self + } +} +#[stable(feature = "raw_fd_reflexive_traits", since = "1.48.0")] +impl IntoRawFd for RawFd { + #[inline] + fn into_raw_fd(self) -> RawFd { + self + } +} +#[stable(feature = "raw_fd_reflexive_traits", since = "1.48.0")] +impl FromRawFd for RawFd { + #[inline] + unsafe fn from_raw_fd(fd: RawFd) -> RawFd { + fd + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl AsRawFd for fs::File { + #[inline] + fn as_raw_fd(&self) -> RawFd { + self.as_inner().as_raw_fd() + } +} +#[stable(feature = "from_raw_os", since = "1.1.0")] +impl FromRawFd for fs::File { + #[inline] + unsafe fn from_raw_fd(fd: RawFd) -> fs::File { + unsafe { fs::File::from(OwnedFd::from_raw_fd(fd)) } + } +} +#[stable(feature = "into_raw_os", since = "1.4.0")] +impl IntoRawFd for fs::File { + #[inline] + fn into_raw_fd(self) -> RawFd { + self.into_inner().into_inner().into_raw_fd() + } +} + +#[stable(feature = "asraw_stdio", since = "1.21.0")] +impl AsRawFd for io::Stdin { + #[inline] + fn as_raw_fd(&self) -> RawFd { + libc::STDIN_FILENO + } +} + +#[stable(feature = "asraw_stdio", since = "1.21.0")] +impl AsRawFd for io::Stdout { + #[inline] + fn as_raw_fd(&self) -> RawFd { + libc::STDOUT_FILENO + } +} + +#[stable(feature = "asraw_stdio", since = "1.21.0")] +impl AsRawFd for io::Stderr { + #[inline] + fn as_raw_fd(&self) -> RawFd { + libc::STDERR_FILENO + } +} + +#[stable(feature = "asraw_stdio_locks", since = "1.35.0")] +impl<'a> AsRawFd for io::StdinLock<'a> { + #[inline] + fn as_raw_fd(&self) -> RawFd { + libc::STDIN_FILENO + } +} + +#[stable(feature = "asraw_stdio_locks", since = "1.35.0")] +impl<'a> AsRawFd for io::StdoutLock<'a> { + #[inline] + fn as_raw_fd(&self) -> RawFd { + libc::STDOUT_FILENO + } +} + +#[stable(feature = "asraw_stdio_locks", since = "1.35.0")] +impl<'a> AsRawFd for io::StderrLock<'a> { + #[inline] + fn as_raw_fd(&self) -> RawFd { + libc::STDERR_FILENO + } +} + +/// This impl allows implementing traits that require `AsRawFd` on Arc. +/// ``` +/// # #[cfg(any(unix, target_os = "wasi"))] mod group_cfg { +/// # #[cfg(target_os = "wasi")] +/// # use std::os::wasi::io::AsRawFd; +/// # #[cfg(unix)] +/// # use std::os::unix::io::AsRawFd; +/// use std::net::UdpSocket; +/// use std::sync::Arc; +/// trait MyTrait: AsRawFd { +/// } +/// impl MyTrait for Arc {} +/// impl MyTrait for Box {} +/// # } +/// ``` +#[stable(feature = "asrawfd_ptrs", since = "1.63.0")] +impl AsRawFd for crate::sync::Arc { + #[inline] + fn as_raw_fd(&self) -> RawFd { + (**self).as_raw_fd() + } +} + +#[stable(feature = "asrawfd_ptrs", since = "1.63.0")] +impl AsRawFd for Box { + #[inline] + fn as_raw_fd(&self) -> RawFd { + (**self).as_raw_fd() + } +} diff --git a/crux-mir/lib/std/src/os/fd/tests.rs b/crux-mir/lib/std/src/os/fd/tests.rs new file mode 100644 index 000000000..b39863644 --- /dev/null +++ b/crux-mir/lib/std/src/os/fd/tests.rs @@ -0,0 +1,53 @@ +#[cfg(any(unix, target_os = "wasi"))] +#[test] +fn test_raw_fd() { + #[cfg(unix)] + use crate::os::unix::io::{AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, RawFd}; + #[cfg(target_os = "wasi")] + use crate::os::wasi::io::{AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, RawFd}; + + let raw_fd: RawFd = crate::io::stdin().as_raw_fd(); + + let stdin_as_file = unsafe { crate::fs::File::from_raw_fd(raw_fd) }; + assert_eq!(stdin_as_file.as_raw_fd(), raw_fd); + assert_eq!(unsafe { BorrowedFd::borrow_raw(raw_fd).as_raw_fd() }, raw_fd); + assert_eq!(stdin_as_file.into_raw_fd(), 0); +} + +#[cfg(any(unix, target_os = "wasi"))] +#[test] +fn test_fd() { + #[cfg(unix)] + use crate::os::unix::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; + #[cfg(target_os = "wasi")] + use crate::os::wasi::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; + + let stdin = crate::io::stdin(); + let fd: BorrowedFd<'_> = stdin.as_fd(); + let raw_fd: RawFd = fd.as_raw_fd(); + let owned_fd: OwnedFd = unsafe { OwnedFd::from_raw_fd(raw_fd) }; + + let stdin_as_file = crate::fs::File::from(owned_fd); + + assert_eq!(stdin_as_file.as_fd().as_raw_fd(), raw_fd); + assert_eq!(Into::::into(stdin_as_file).into_raw_fd(), raw_fd); +} + +#[cfg(any(unix, target_os = "wasi"))] +#[test] +fn test_niche_optimizations() { + use crate::mem::size_of; + #[cfg(unix)] + use crate::os::unix::io::{BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; + #[cfg(target_os = "wasi")] + use crate::os::wasi::io::{BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; + + assert_eq!(size_of::>(), size_of::()); + assert_eq!(size_of::>>(), size_of::()); + unsafe { + assert_eq!(OwnedFd::from_raw_fd(RawFd::MIN).into_raw_fd(), RawFd::MIN); + assert_eq!(OwnedFd::from_raw_fd(RawFd::MAX).into_raw_fd(), RawFd::MAX); + assert_eq!(Some(OwnedFd::from_raw_fd(RawFd::MIN)).unwrap().into_raw_fd(), RawFd::MIN); + assert_eq!(Some(OwnedFd::from_raw_fd(RawFd::MAX)).unwrap().into_raw_fd(), RawFd::MAX); + } +} diff --git a/crux-mir/lib/std/src/sys/sgx/ext/arch.rs b/crux-mir/lib/std/src/os/fortanix_sgx/arch.rs similarity index 72% rename from crux-mir/lib/std/src/sys/sgx/ext/arch.rs rename to crux-mir/lib/std/src/os/fortanix_sgx/arch.rs index 5056e3881..8358cb9e8 100644 --- a/crux-mir/lib/std/src/sys/sgx/ext/arch.rs +++ b/crux-mir/lib/std/src/os/fortanix_sgx/arch.rs @@ -5,6 +5,7 @@ #![unstable(feature = "sgx_platform", issue = "56975")] use crate::mem::MaybeUninit; +use core::arch::asm; /// Wrapper struct to force 16-byte alignment. #[repr(align(16))] @@ -32,12 +33,14 @@ pub fn egetkey(request: &Align512<[u8; 512]>) -> Result, u32> let error; asm!( - "enclu" - : "={eax}"(error) - : "{eax}"(ENCLU_EGETKEY), - "{rbx}"(request), - "{rcx}"(out.as_mut_ptr()) - : "flags" + // rbx is reserved by LLVM + "xchg %rbx, {0}", + "enclu", + "mov {0}, %rbx", + inout(reg) request => _, + inlateout("eax") ENCLU_EGETKEY => error, + in("rcx") out.as_mut_ptr(), + options(att_syntax, nostack), ); match error { @@ -61,12 +64,15 @@ pub fn ereport( let mut report = MaybeUninit::uninit(); asm!( - "enclu" - : /* no output registers */ - : "{eax}"(ENCLU_EREPORT), - "{rbx}"(targetinfo), - "{rcx}"(reportdata), - "{rdx}"(report.as_mut_ptr()) + // rbx is reserved by LLVM + "xchg %rbx, {0}", + "enclu", + "mov {0}, %rbx", + inout(reg) targetinfo => _, + in("eax") ENCLU_EREPORT, + in("rcx") reportdata, + in("rdx") report.as_mut_ptr(), + options(att_syntax, preserves_flags, nostack), ); report.assume_init() diff --git a/crux-mir/lib/std/src/sys/sgx/ext/ffi.rs b/crux-mir/lib/std/src/os/fortanix_sgx/ffi.rs similarity index 90% rename from crux-mir/lib/std/src/sys/sgx/ext/ffi.rs rename to crux-mir/lib/std/src/os/fortanix_sgx/ffi.rs index 63fc5ff28..ac1db0e5e 100644 --- a/crux-mir/lib/std/src/sys/sgx/ext/ffi.rs +++ b/crux-mir/lib/std/src/os/fortanix_sgx/ffi.rs @@ -34,5 +34,8 @@ #![unstable(feature = "sgx_platform", issue = "56975")] +#[path = "../unix/ffi/os_str.rs"] +mod os_str; + #[unstable(feature = "sgx_platform", issue = "56975")] -pub use crate::sys_common::os_str_bytes::*; +pub use self::os_str::{OsStrExt, OsStringExt}; diff --git a/crux-mir/lib/std/src/sys/sgx/ext/io.rs b/crux-mir/lib/std/src/os/fortanix_sgx/io.rs similarity index 75% rename from crux-mir/lib/std/src/sys/sgx/ext/io.rs rename to crux-mir/lib/std/src/os/fortanix_sgx/io.rs index 8aa84a550..7223ade68 100644 --- a/crux-mir/lib/std/src/sys/sgx/ext/io.rs +++ b/crux-mir/lib/std/src/os/fortanix_sgx/io.rs @@ -1,7 +1,7 @@ //! SGX-specific extensions to general I/O primitives //! //! SGX file descriptors behave differently from Unix file descriptors. See the -//! description of [`TryIntoRawFd`](trait.TryIntoRawFd.html) for more details. +//! description of [`TryIntoRawFd`] for more details. #![unstable(feature = "sgx_platform", issue = "56975")] use crate::net; @@ -25,8 +25,11 @@ pub trait AsRawFd { /// descriptor. #[unstable(feature = "sgx_platform", issue = "56975")] pub trait FromRawFd { + /// An associated type that contains relevant metadata for `Self`. + type Metadata: Default; + /// Constructs a new instance of `Self` from the given raw file - /// descriptor. + /// descriptor and metadata. /// /// This function **consumes ownership** of the specified file /// descriptor. The returned object will take responsibility for closing @@ -38,7 +41,7 @@ pub trait FromRawFd { /// accidentally allow violating this contract which can cause memory /// unsafety in code that relies on it being true. #[unstable(feature = "sgx_platform", issue = "56975")] - unsafe fn from_raw_fd(fd: RawFd) -> Self; + unsafe fn from_raw_fd(fd: RawFd, metadata: Self::Metadata) -> Self; } /// A trait to express the ability to consume an object and acquire ownership of @@ -60,34 +63,61 @@ pub trait TryIntoRawFd: Sized { } impl AsRawFd for net::TcpStream { + #[inline] fn as_raw_fd(&self) -> RawFd { *self.as_inner().as_inner().as_inner().as_inner() } } impl AsRawFd for net::TcpListener { + #[inline] fn as_raw_fd(&self) -> RawFd { *self.as_inner().as_inner().as_inner().as_inner() } } +/// Metadata for `TcpStream`. +#[derive(Debug, Clone, Default)] +#[unstable(feature = "sgx_platform", issue = "56975")] +pub struct TcpStreamMetadata { + /// Local address of the TCP stream + pub local_addr: Option, + /// Peer address of the TCP stream + pub peer_addr: Option, +} + impl FromRawFd for net::TcpStream { - unsafe fn from_raw_fd(fd: RawFd) -> net::TcpStream { + type Metadata = TcpStreamMetadata; + + #[inline] + unsafe fn from_raw_fd(fd: RawFd, metadata: Self::Metadata) -> net::TcpStream { let fd = sys::fd::FileDesc::from_inner(fd); - let socket = sys::net::Socket::from_inner(fd); - net::TcpStream::from_inner(sys::net::TcpStream::from_inner((socket, None))) + let socket = sys::net::Socket::from_inner((fd, metadata.local_addr)); + net::TcpStream::from_inner(sys::net::TcpStream::from_inner((socket, metadata.peer_addr))) } } +/// Metadata for `TcpListener`. +#[derive(Debug, Clone, Default)] +#[unstable(feature = "sgx_platform", issue = "56975")] +pub struct TcpListenerMetadata { + /// Local address of the TCP listener + pub local_addr: Option, +} + impl FromRawFd for net::TcpListener { - unsafe fn from_raw_fd(fd: RawFd) -> net::TcpListener { + type Metadata = TcpListenerMetadata; + + #[inline] + unsafe fn from_raw_fd(fd: RawFd, metadata: Self::Metadata) -> net::TcpListener { let fd = sys::fd::FileDesc::from_inner(fd); - let socket = sys::net::Socket::from_inner(fd); + let socket = sys::net::Socket::from_inner((fd, metadata.local_addr)); net::TcpListener::from_inner(sys::net::TcpListener::from_inner(socket)) } } impl TryIntoRawFd for net::TcpStream { + #[inline] fn try_into_raw_fd(self) -> Result { let (socket, peer_addr) = self.into_inner().into_inner(); match socket.try_into_inner() { @@ -101,6 +131,7 @@ impl TryIntoRawFd for net::TcpStream { } impl TryIntoRawFd for net::TcpListener { + #[inline] fn try_into_raw_fd(self) -> Result { match self.into_inner().into_inner().try_into_inner() { Ok(fd) => Ok(fd.into_inner()), diff --git a/crux-mir/lib/std/src/os/fortanix_sgx/mod.rs b/crux-mir/lib/std/src/os/fortanix_sgx/mod.rs index 69923268e..39a42f4e1 100644 --- a/crux-mir/lib/std/src/os/fortanix_sgx/mod.rs +++ b/crux-mir/lib/std/src/os/fortanix_sgx/mod.rs @@ -3,7 +3,7 @@ //! This includes functions to deal with memory isolation, usercalls, and the //! SGX instruction set. -#![deny(missing_docs, missing_debug_implementations)] +#![deny(missing_docs)] #![unstable(feature = "sgx_platform", issue = "56975")] /// Low-level interfaces to usercalls. See the [ABI documentation] for more @@ -26,10 +26,13 @@ pub mod usercalls { free, insecure_time, launch_thread, read, read_alloc, send, wait, write, }; pub use crate::sys::abi::usercalls::raw::{do_usercall, Usercalls as UsercallNrs}; + pub use crate::sys::abi::usercalls::raw::{Register, RegisterArgument, ReturnValue}; // fortanix-sgx-abi re-exports pub use crate::sys::abi::usercalls::raw::Error; - pub use crate::sys::abi::usercalls::raw::{ByteBuffer, FifoDescriptor, Return, Usercall}; + pub use crate::sys::abi::usercalls::raw::{ + ByteBuffer, Cancel, FifoDescriptor, Return, Usercall, + }; pub use crate::sys::abi::usercalls::raw::{Fd, Result, Tcs}; pub use crate::sys::abi::usercalls::raw::{ EV_RETURNQ_NOT_EMPTY, EV_UNPARK, EV_USERCALLQ_NOT_FULL, FD_STDERR, FD_STDIN, FD_STDOUT, @@ -43,7 +46,9 @@ pub mod mem { pub use crate::sys::abi::mem::*; } -pub use crate::sys::ext::{arch, ffi, io}; +pub mod arch; +pub mod ffi; +pub mod io; /// Functions for querying thread-related information. pub mod thread { diff --git a/crux-mir/lib/std/src/os/freebsd/fs.rs b/crux-mir/lib/std/src/os/freebsd/fs.rs index 6798e0d8f..8db3a950c 100644 --- a/crux-mir/lib/std/src/os/freebsd/fs.rs +++ b/crux-mir/lib/std/src/os/freebsd/fs.rs @@ -8,7 +8,7 @@ use crate::os::freebsd::raw; /// OS-specific extensions to [`fs::Metadata`]. /// -/// [`fs::Metadata`]: ../../../../std/fs/struct.Metadata.html +/// [`fs::Metadata`]: crate::fs::Metadata #[stable(feature = "metadata_ext", since = "1.1.0")] pub trait MetadataExt { /// Gain a reference to the underlying `stat` structure which contains @@ -18,10 +18,10 @@ pub trait MetadataExt { /// Unix platforms. The `os::unix::fs::MetadataExt` trait contains the /// cross-Unix abstractions contained within the raw stat. #[stable(feature = "metadata_ext", since = "1.1.0")] - #[rustc_deprecated( + #[deprecated( since = "1.8.0", - reason = "deprecated in favor of the accessor \ - methods of this trait" + note = "deprecated in favor of the accessor \ + methods of this trait" )] #[allow(deprecated)] fn as_raw_stat(&self) -> &raw::stat; @@ -74,7 +74,14 @@ pub trait MetadataExt { impl MetadataExt for Metadata { #[allow(deprecated)] fn as_raw_stat(&self) -> &raw::stat { - unsafe { &*(self.as_inner().as_inner() as *const libc::stat as *const raw::stat) } + // The methods below use libc::stat, so they work fine when libc is built with FreeBSD 12 ABI. + // This method would just return nonsense. + #[cfg(freebsd12)] + panic!("as_raw_stat not supported with FreeBSD 12 ABI"); + #[cfg(not(freebsd12))] + unsafe { + &*(self.as_inner().as_inner() as *const libc::stat as *const raw::stat) + } } fn st_dev(&self) -> u64 { self.as_inner().as_inner().st_dev as u64 @@ -136,6 +143,11 @@ impl MetadataExt for Metadata { fn st_flags(&self) -> u32 { self.as_inner().as_inner().st_flags as u32 } + #[cfg(freebsd12)] + fn st_lspare(&self) -> u32 { + panic!("st_lspare not supported with FreeBSD 12 ABI"); + } + #[cfg(not(freebsd12))] fn st_lspare(&self) -> u32 { self.as_inner().as_inner().st_lspare as u32 } diff --git a/crux-mir/lib/std/src/os/freebsd/raw.rs b/crux-mir/lib/std/src/os/freebsd/raw.rs index aeae08fc6..ab0bf7923 100644 --- a/crux-mir/lib/std/src/os/freebsd/raw.rs +++ b/crux-mir/lib/std/src/os/freebsd/raw.rs @@ -1,12 +1,12 @@ //! FreeBSD-specific raw type definitions #![stable(feature = "raw_ext", since = "1.1.0")] -#![rustc_deprecated( +#![deprecated( since = "1.8.0", - reason = "these type aliases are no longer supported by \ - the standard library, the `libc` crate on \ - crates.io should be used instead for the correct \ - definitions" + note = "these type aliases are no longer supported by \ + the standard library, the `libc` crate on \ + crates.io should be used instead for the correct \ + definitions" )] #![allow(deprecated)] diff --git a/crux-mir/lib/std/src/os/fuchsia/fs.rs b/crux-mir/lib/std/src/os/fuchsia/fs.rs index 1544bdfbe..b48a46f91 100644 --- a/crux-mir/lib/std/src/os/fuchsia/fs.rs +++ b/crux-mir/lib/std/src/os/fuchsia/fs.rs @@ -5,7 +5,7 @@ use crate::sys_common::AsInner; /// OS-specific extensions to [`fs::Metadata`]. /// -/// [`fs::Metadata`]: ../../../../std/fs/struct.Metadata.html +/// [`fs::Metadata`]: crate::fs::Metadata #[stable(feature = "metadata_ext", since = "1.1.0")] pub trait MetadataExt { #[stable(feature = "metadata_ext2", since = "1.8.0")] diff --git a/crux-mir/lib/std/src/os/fuchsia/raw.rs b/crux-mir/lib/std/src/os/fuchsia/raw.rs index f94659cd5..060d6e86b 100644 --- a/crux-mir/lib/std/src/os/fuchsia/raw.rs +++ b/crux-mir/lib/std/src/os/fuchsia/raw.rs @@ -1,12 +1,12 @@ //! Fuchsia-specific raw type definitions #![stable(feature = "raw_ext", since = "1.1.0")] -#![rustc_deprecated( +#![deprecated( since = "1.8.0", - reason = "these type aliases are no longer supported by \ - the standard library, the `libc` crate on \ - crates.io should be used instead for the correct \ - definitions" + note = "these type aliases are no longer supported by \ + the standard library, the `libc` crate on \ + crates.io should be used instead for the correct \ + definitions" )] #![allow(deprecated)] diff --git a/crux-mir/lib/std/src/os/haiku/fs.rs b/crux-mir/lib/std/src/os/haiku/fs.rs index 13a4a92ae..a23a2af8f 100644 --- a/crux-mir/lib/std/src/os/haiku/fs.rs +++ b/crux-mir/lib/std/src/os/haiku/fs.rs @@ -8,7 +8,7 @@ use crate::os::haiku::raw; /// OS-specific extensions to [`fs::Metadata`]. /// -/// [`fs::Metadata`]: ../../../../std/fs/struct.Metadata.html +/// [`fs::Metadata`]: crate::fs::Metadata #[stable(feature = "metadata_ext", since = "1.1.0")] pub trait MetadataExt { /// Gain a reference to the underlying `stat` structure which contains @@ -18,10 +18,10 @@ pub trait MetadataExt { /// Unix platforms. The `os::unix::fs::MetadataExt` trait contains the /// cross-Unix abstractions contained within the raw stat. #[stable(feature = "metadata_ext", since = "1.1.0")] - #[rustc_deprecated( + #[deprecated( since = "1.8.0", - reason = "deprecated in favor of the accessor \ - methods of this trait" + note = "deprecated in favor of the accessor \ + methods of this trait" )] #[allow(deprecated)] fn as_raw_stat(&self) -> &raw::stat; diff --git a/crux-mir/lib/std/src/os/haiku/raw.rs b/crux-mir/lib/std/src/os/haiku/raw.rs index 0d7e70b6b..afbb66ccb 100644 --- a/crux-mir/lib/std/src/os/haiku/raw.rs +++ b/crux-mir/lib/std/src/os/haiku/raw.rs @@ -1,6 +1,13 @@ //! Haiku-specific raw type definitions #![stable(feature = "raw_ext", since = "1.1.0")] +#![deprecated( + since = "1.53.0", + note = "these type aliases are no longer supported by \ + the standard library, the `libc` crate on \ + crates.io should be used instead for the correct \ + definitions" +)] #![allow(deprecated)] use crate::os::raw::c_long; diff --git a/crux-mir/lib/std/src/os/hermit/ffi.rs b/crux-mir/lib/std/src/os/hermit/ffi.rs new file mode 100644 index 000000000..19761fd99 --- /dev/null +++ b/crux-mir/lib/std/src/os/hermit/ffi.rs @@ -0,0 +1,41 @@ +//! HermitCore-specific extension to the primitives in the `std::ffi` module +//! +//! # Examples +//! +//! ``` +//! use std::ffi::OsString; +//! use std::os::hermit::ffi::OsStringExt; +//! +//! let bytes = b"foo".to_vec(); +//! +//! // OsStringExt::from_vec +//! let os_string = OsString::from_vec(bytes); +//! assert_eq!(os_string.to_str(), Some("foo")); +//! +//! // OsStringExt::into_vec +//! let bytes = os_string.into_vec(); +//! assert_eq!(bytes, b"foo"); +//! ``` +//! +//! ``` +//! use std::ffi::OsStr; +//! use std::os::hermit::ffi::OsStrExt; +//! +//! let bytes = b"foo"; +//! +//! // OsStrExt::from_bytes +//! let os_str = OsStr::from_bytes(bytes); +//! assert_eq!(os_str.to_str(), Some("foo")); +//! +//! // OsStrExt::as_bytes +//! let bytes = os_str.as_bytes(); +//! assert_eq!(bytes, b"foo"); +//! ``` + +#![stable(feature = "rust1", since = "1.0.0")] + +#[path = "../unix/ffi/os_str.rs"] +mod os_str; + +#[stable(feature = "rust1", since = "1.0.0")] +pub use self::os_str::{OsStrExt, OsStringExt}; diff --git a/crux-mir/lib/std/src/os/hermit/mod.rs b/crux-mir/lib/std/src/os/hermit/mod.rs new file mode 100644 index 000000000..4657b545a --- /dev/null +++ b/crux-mir/lib/std/src/os/hermit/mod.rs @@ -0,0 +1,13 @@ +#![stable(feature = "rust1", since = "1.0.0")] + +pub mod ffi; + +/// A prelude for conveniently writing platform-specific code. +/// +/// Includes all extension traits, and some important type definitions. +#[stable(feature = "rust1", since = "1.0.0")] +pub mod prelude { + #[doc(no_inline)] + #[stable(feature = "rust1", since = "1.0.0")] + pub use super::ffi::{OsStrExt, OsStringExt}; +} diff --git a/crux-mir/lib/std/src/os/horizon/fs.rs b/crux-mir/lib/std/src/os/horizon/fs.rs new file mode 100644 index 000000000..132552210 --- /dev/null +++ b/crux-mir/lib/std/src/os/horizon/fs.rs @@ -0,0 +1,95 @@ +#![stable(feature = "metadata_ext", since = "1.1.0")] + +use crate::fs::Metadata; +use crate::sys_common::AsInner; + +/// OS-specific extensions to [`fs::Metadata`]. +/// +/// [`fs::Metadata`]: crate::fs::Metadata +#[stable(feature = "metadata_ext", since = "1.1.0")] +pub trait MetadataExt { + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_dev(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_ino(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_mode(&self) -> u32; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_nlink(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_uid(&self) -> u32; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_gid(&self) -> u32; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_rdev(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_size(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_atime(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_atime_nsec(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_mtime(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_mtime_nsec(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_ctime(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_ctime_nsec(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_blksize(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_blocks(&self) -> u64; +} + +#[stable(feature = "metadata_ext", since = "1.1.0")] +impl MetadataExt for Metadata { + fn st_dev(&self) -> u64 { + self.as_inner().as_inner().st_dev as u64 + } + fn st_ino(&self) -> u64 { + self.as_inner().as_inner().st_ino as u64 + } + fn st_mode(&self) -> u32 { + self.as_inner().as_inner().st_mode as u32 + } + fn st_nlink(&self) -> u64 { + self.as_inner().as_inner().st_nlink as u64 + } + fn st_uid(&self) -> u32 { + self.as_inner().as_inner().st_uid as u32 + } + fn st_gid(&self) -> u32 { + self.as_inner().as_inner().st_gid as u32 + } + fn st_rdev(&self) -> u64 { + self.as_inner().as_inner().st_rdev as u64 + } + fn st_size(&self) -> u64 { + self.as_inner().as_inner().st_size as u64 + } + fn st_atime(&self) -> i64 { + self.as_inner().as_inner().st_atim.tv_sec + } + fn st_atime_nsec(&self) -> i64 { + self.as_inner().as_inner().st_atim.tv_nsec as i64 + } + fn st_mtime(&self) -> i64 { + self.as_inner().as_inner().st_mtim.tv_sec + } + fn st_mtime_nsec(&self) -> i64 { + self.as_inner().as_inner().st_mtim.tv_nsec as i64 + } + fn st_ctime(&self) -> i64 { + self.as_inner().as_inner().st_ctim.tv_sec + } + fn st_ctime_nsec(&self) -> i64 { + self.as_inner().as_inner().st_ctim.tv_nsec as i64 + } + fn st_blksize(&self) -> u64 { + self.as_inner().as_inner().st_blksize as u64 + } + fn st_blocks(&self) -> u64 { + self.as_inner().as_inner().st_blocks as u64 + } +} diff --git a/crux-mir/lib/std/src/os/horizon/mod.rs b/crux-mir/lib/std/src/os/horizon/mod.rs new file mode 100644 index 000000000..326d0ae9c --- /dev/null +++ b/crux-mir/lib/std/src/os/horizon/mod.rs @@ -0,0 +1,6 @@ +//! Definitions for Horizon OS + +#![stable(feature = "raw_ext", since = "1.1.0")] + +pub mod fs; +pub(crate) mod raw; diff --git a/crux-mir/lib/std/src/os/horizon/raw.rs b/crux-mir/lib/std/src/os/horizon/raw.rs new file mode 100644 index 000000000..929fa7db1 --- /dev/null +++ b/crux-mir/lib/std/src/os/horizon/raw.rs @@ -0,0 +1,70 @@ +//! Horizon OS raw type definitions + +#![stable(feature = "raw_ext", since = "1.1.0")] +#![deprecated( + since = "1.8.0", + note = "these type aliases are no longer supported by \ + the standard library, the `libc` crate on \ + crates.io should be used instead for the correct \ + definitions" +)] +#![allow(deprecated)] + +use crate::os::raw::c_long; +use crate::os::unix::raw::{gid_t, uid_t}; + +#[stable(feature = "pthread_t", since = "1.8.0")] +pub type pthread_t = libc::pthread_t; + +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type blkcnt_t = libc::blkcnt_t; + +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type blksize_t = libc::blksize_t; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type dev_t = libc::dev_t; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type ino_t = libc::ino_t; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type mode_t = libc::mode_t; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type nlink_t = libc::nlink_t; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type off_t = libc::off_t; + +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type time_t = libc::time_t; + +#[repr(C)] +#[derive(Clone)] +#[stable(feature = "raw_ext", since = "1.1.0")] +pub struct stat { + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_dev: dev_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ino: ino_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mode: mode_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_nlink: nlink_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_uid: uid_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_gid: gid_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_rdev: dev_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_size: off_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_atime: time_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mtime: time_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ctime: time_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_blksize: blksize_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_blocks: blkcnt_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_spare4: [c_long; 2usize], +} diff --git a/crux-mir/lib/std/src/os/illumos/fs.rs b/crux-mir/lib/std/src/os/illumos/fs.rs new file mode 100644 index 000000000..63be48b81 --- /dev/null +++ b/crux-mir/lib/std/src/os/illumos/fs.rs @@ -0,0 +1,116 @@ +#![stable(feature = "metadata_ext", since = "1.1.0")] + +use crate::fs::Metadata; +use crate::sys_common::AsInner; + +#[allow(deprecated)] +use crate::os::illumos::raw; + +/// OS-specific extensions to [`fs::Metadata`]. +/// +/// [`fs::Metadata`]: crate::fs::Metadata +#[stable(feature = "metadata_ext", since = "1.1.0")] +pub trait MetadataExt { + /// Gain a reference to the underlying `stat` structure which contains + /// the raw information returned by the OS. + /// + /// The contents of the returned `stat` are **not** consistent across + /// Unix platforms. The `os::unix::fs::MetadataExt` trait contains the + /// cross-Unix abstractions contained within the raw stat. + #[stable(feature = "metadata_ext", since = "1.1.0")] + #[deprecated( + since = "1.8.0", + note = "deprecated in favor of the accessor methods of this trait" + )] + #[allow(deprecated)] + fn as_raw_stat(&self) -> &raw::stat; + + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_dev(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_ino(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_mode(&self) -> u32; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_nlink(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_uid(&self) -> u32; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_gid(&self) -> u32; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_rdev(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_size(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_atime(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_atime_nsec(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_mtime(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_mtime_nsec(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_ctime(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_ctime_nsec(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_blksize(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_blocks(&self) -> u64; +} + +#[stable(feature = "metadata_ext", since = "1.1.0")] +impl MetadataExt for Metadata { + #[allow(deprecated)] + fn as_raw_stat(&self) -> &raw::stat { + unsafe { &*(self.as_inner().as_inner() as *const libc::stat as *const raw::stat) } + } + fn st_dev(&self) -> u64 { + self.as_inner().as_inner().st_dev as u64 + } + fn st_ino(&self) -> u64 { + self.as_inner().as_inner().st_ino as u64 + } + fn st_mode(&self) -> u32 { + self.as_inner().as_inner().st_mode as u32 + } + fn st_nlink(&self) -> u64 { + self.as_inner().as_inner().st_nlink as u64 + } + fn st_uid(&self) -> u32 { + self.as_inner().as_inner().st_uid as u32 + } + fn st_gid(&self) -> u32 { + self.as_inner().as_inner().st_gid as u32 + } + fn st_rdev(&self) -> u64 { + self.as_inner().as_inner().st_rdev as u64 + } + fn st_size(&self) -> u64 { + self.as_inner().as_inner().st_size as u64 + } + fn st_atime(&self) -> i64 { + self.as_inner().as_inner().st_atime as i64 + } + fn st_atime_nsec(&self) -> i64 { + self.as_inner().as_inner().st_atime_nsec as i64 + } + fn st_mtime(&self) -> i64 { + self.as_inner().as_inner().st_mtime as i64 + } + fn st_mtime_nsec(&self) -> i64 { + self.as_inner().as_inner().st_mtime_nsec as i64 + } + fn st_ctime(&self) -> i64 { + self.as_inner().as_inner().st_ctime as i64 + } + fn st_ctime_nsec(&self) -> i64 { + self.as_inner().as_inner().st_ctime_nsec as i64 + } + fn st_blksize(&self) -> u64 { + self.as_inner().as_inner().st_blksize as u64 + } + fn st_blocks(&self) -> u64 { + self.as_inner().as_inner().st_blocks as u64 + } +} diff --git a/crux-mir/lib/std/src/os/illumos/mod.rs b/crux-mir/lib/std/src/os/illumos/mod.rs new file mode 100644 index 000000000..e61926f89 --- /dev/null +++ b/crux-mir/lib/std/src/os/illumos/mod.rs @@ -0,0 +1,6 @@ +//! illumos-specific definitions + +#![stable(feature = "raw_ext", since = "1.1.0")] + +pub mod fs; +pub mod raw; diff --git a/crux-mir/lib/std/src/os/illumos/raw.rs b/crux-mir/lib/std/src/os/illumos/raw.rs new file mode 100644 index 000000000..2bea9ebb3 --- /dev/null +++ b/crux-mir/lib/std/src/os/illumos/raw.rs @@ -0,0 +1,74 @@ +//! illumos-specific raw type definitions + +#![stable(feature = "raw_ext", since = "1.1.0")] +#![deprecated( + since = "1.8.0", + note = "these type aliases are no longer supported by the standard library, the `libc` \ + crate on crates.io should be used instead for the correct definitions" +)] +#![allow(deprecated)] + +use crate::os::raw::c_long; +use crate::os::unix::raw::{gid_t, uid_t}; + +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type blkcnt_t = u64; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type blksize_t = u64; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type dev_t = u64; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type fflags_t = u32; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type ino_t = u64; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type mode_t = u32; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type nlink_t = u64; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type off_t = u64; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type time_t = i64; + +#[stable(feature = "pthread_t", since = "1.8.0")] +pub type pthread_t = u32; + +#[repr(C)] +#[derive(Clone)] +#[stable(feature = "raw_ext", since = "1.1.0")] +pub struct stat { + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_dev: dev_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ino: ino_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mode: mode_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_nlink: nlink_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_uid: uid_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_gid: gid_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_rdev: dev_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_size: off_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_atime: time_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_atime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mtime: time_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mtime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ctime: time_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ctime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_blksize: blksize_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_blocks: blkcnt_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub __unused: [u8; 16], +} diff --git a/crux-mir/lib/std/src/os/ios/fs.rs b/crux-mir/lib/std/src/os/ios/fs.rs index 08d3e4bce..4a4637ce0 100644 --- a/crux-mir/lib/std/src/os/ios/fs.rs +++ b/crux-mir/lib/std/src/os/ios/fs.rs @@ -8,7 +8,7 @@ use crate::os::ios::raw; /// OS-specific extensions to [`fs::Metadata`]. /// -/// [`fs::Metadata`]: ../../../../std/fs/struct.Metadata.html +/// [`fs::Metadata`]: crate::fs::Metadata #[stable(feature = "metadata_ext", since = "1.1.0")] pub trait MetadataExt { /// Gain a reference to the underlying `stat` structure which contains @@ -18,10 +18,10 @@ pub trait MetadataExt { /// Unix platforms. The `os::unix::fs::MetadataExt` trait contains the /// cross-Unix abstractions contained within the raw stat. #[stable(feature = "metadata_ext", since = "1.1.0")] - #[rustc_deprecated( + #[deprecated( since = "1.8.0", - reason = "deprecated in favor of the accessor \ - methods of this trait" + note = "deprecated in favor of the accessor \ + methods of this trait" )] #[allow(deprecated)] fn as_raw_stat(&self) -> &raw::stat; diff --git a/crux-mir/lib/std/src/os/ios/raw.rs b/crux-mir/lib/std/src/os/ios/raw.rs index 97b0a96b0..af12aeebe 100644 --- a/crux-mir/lib/std/src/os/ios/raw.rs +++ b/crux-mir/lib/std/src/os/ios/raw.rs @@ -1,12 +1,12 @@ //! iOS-specific raw type definitions #![stable(feature = "raw_ext", since = "1.1.0")] -#![rustc_deprecated( +#![deprecated( since = "1.8.0", - reason = "these type aliases are no longer supported by \ - the standard library, the `libc` crate on \ - crates.io should be used instead for the correct \ - definitions" + note = "these type aliases are no longer supported by \ + the standard library, the `libc` crate on \ + crates.io should be used instead for the correct \ + definitions" )] #![allow(deprecated)] diff --git a/crux-mir/lib/std/src/os/l4re/fs.rs b/crux-mir/lib/std/src/os/l4re/fs.rs new file mode 100644 index 000000000..6d6a535b1 --- /dev/null +++ b/crux-mir/lib/std/src/os/l4re/fs.rs @@ -0,0 +1,382 @@ +//! L4Re-specific extensions to primitives in the [`std::fs`] module. +//! +//! [`std::fs`]: crate::fs + +#![stable(feature = "metadata_ext", since = "1.1.0")] + +use crate::fs::Metadata; +use crate::sys_common::AsInner; + +#[allow(deprecated)] +use crate::os::l4re::raw; + +/// OS-specific extensions to [`fs::Metadata`]. +/// +/// [`fs::Metadata`]: crate::fs::Metadata +#[stable(feature = "metadata_ext", since = "1.1.0")] +pub trait MetadataExt { + /// Gain a reference to the underlying `stat` structure which contains + /// the raw information returned by the OS. + /// + /// The contents of the returned [`stat`] are **not** consistent across + /// Unix platforms. The `os::unix::fs::MetadataExt` trait contains the + /// cross-Unix abstractions contained within the raw stat. + /// + /// [`stat`]: struct@crate::os::linux::raw::stat + /// + /// # Examples + /// + /// ```no_run + /// use std::fs; + /// use std::io; + /// use std::os::linux::fs::MetadataExt; + /// + /// fn main() -> io::Result<()> { + /// let meta = fs::metadata("some_file")?; + /// let stat = meta.as_raw_stat(); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "metadata_ext", since = "1.1.0")] + #[deprecated(since = "1.8.0", note = "other methods of this trait are now preferred")] + #[allow(deprecated)] + fn as_raw_stat(&self) -> &raw::stat; + + /// Returns the device ID on which this file resides. + /// + /// # Examples + /// + /// ```no_run + /// use std::fs; + /// use std::io; + /// use std::os::linux::fs::MetadataExt; + /// + /// fn main() -> io::Result<()> { + /// let meta = fs::metadata("some_file")?; + /// println!("{}", meta.st_dev()); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_dev(&self) -> u64; + /// Returns the inode number. + /// + /// # Examples + /// + /// ```no_run + /// use std::fs; + /// use std::io; + /// use std::os::linux::fs::MetadataExt; + /// + /// fn main() -> io::Result<()> { + /// let meta = fs::metadata("some_file")?; + /// println!("{}", meta.st_ino()); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_ino(&self) -> u64; + /// Returns the file type and mode. + /// + /// # Examples + /// + /// ```no_run + /// use std::fs; + /// use std::io; + /// use std::os::linux::fs::MetadataExt; + /// + /// fn main() -> io::Result<()> { + /// let meta = fs::metadata("some_file")?; + /// println!("{}", meta.st_mode()); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_mode(&self) -> u32; + /// Returns the number of hard links to file. + /// + /// # Examples + /// + /// ```no_run + /// use std::fs; + /// use std::io; + /// use std::os::linux::fs::MetadataExt; + /// + /// fn main() -> io::Result<()> { + /// let meta = fs::metadata("some_file")?; + /// println!("{}", meta.st_nlink()); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_nlink(&self) -> u64; + /// Returns the user ID of the file owner. + /// + /// # Examples + /// + /// ```no_run + /// use std::fs; + /// use std::io; + /// use std::os::linux::fs::MetadataExt; + /// + /// fn main() -> io::Result<()> { + /// let meta = fs::metadata("some_file")?; + /// println!("{}", meta.st_uid()); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_uid(&self) -> u32; + /// Returns the group ID of the file owner. + /// + /// # Examples + /// + /// ```no_run + /// use std::fs; + /// use std::io; + /// use std::os::linux::fs::MetadataExt; + /// + /// fn main() -> io::Result<()> { + /// let meta = fs::metadata("some_file")?; + /// println!("{}", meta.st_gid()); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_gid(&self) -> u32; + /// Returns the device ID that this file represents. Only relevant for special file. + /// + /// # Examples + /// + /// ```no_run + /// use std::fs; + /// use std::io; + /// use std::os::linux::fs::MetadataExt; + /// + /// fn main() -> io::Result<()> { + /// let meta = fs::metadata("some_file")?; + /// println!("{}", meta.st_rdev()); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_rdev(&self) -> u64; + /// Returns the size of the file (if it is a regular file or a symbolic link) in bytes. + /// + /// The size of a symbolic link is the length of the pathname it contains, + /// without a terminating null byte. + /// + /// # Examples + /// + /// ```no_run + /// use std::fs; + /// use std::io; + /// use std::os::linux::fs::MetadataExt; + /// + /// fn main() -> io::Result<()> { + /// let meta = fs::metadata("some_file")?; + /// println!("{}", meta.st_size()); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_size(&self) -> u64; + /// Returns the last access time of the file, in seconds since Unix Epoch. + /// + /// # Examples + /// + /// ```no_run + /// use std::fs; + /// use std::io; + /// use std::os::linux::fs::MetadataExt; + /// + /// fn main() -> io::Result<()> { + /// let meta = fs::metadata("some_file")?; + /// println!("{}", meta.st_atime()); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_atime(&self) -> i64; + /// Returns the last access time of the file, in nanoseconds since [`st_atime`]. + /// + /// [`st_atime`]: Self::st_atime + /// + /// # Examples + /// + /// ```no_run + /// use std::fs; + /// use std::io; + /// use std::os::linux::fs::MetadataExt; + /// + /// fn main() -> io::Result<()> { + /// let meta = fs::metadata("some_file")?; + /// println!("{}", meta.st_atime_nsec()); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_atime_nsec(&self) -> i64; + /// Returns the last modification time of the file, in seconds since Unix Epoch. + /// + /// # Examples + /// + /// ```no_run + /// use std::fs; + /// use std::io; + /// use std::os::linux::fs::MetadataExt; + /// + /// fn main() -> io::Result<()> { + /// let meta = fs::metadata("some_file")?; + /// println!("{}", meta.st_mtime()); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_mtime(&self) -> i64; + /// Returns the last modification time of the file, in nanoseconds since [`st_mtime`]. + /// + /// [`st_mtime`]: Self::st_mtime + /// + /// # Examples + /// + /// ```no_run + /// use std::fs; + /// use std::io; + /// use std::os::linux::fs::MetadataExt; + /// + /// fn main() -> io::Result<()> { + /// let meta = fs::metadata("some_file")?; + /// println!("{}", meta.st_mtime_nsec()); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_mtime_nsec(&self) -> i64; + /// Returns the last status change time of the file, in seconds since Unix Epoch. + /// + /// # Examples + /// + /// ```no_run + /// use std::fs; + /// use std::io; + /// use std::os::linux::fs::MetadataExt; + /// + /// fn main() -> io::Result<()> { + /// let meta = fs::metadata("some_file")?; + /// println!("{}", meta.st_ctime()); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_ctime(&self) -> i64; + /// Returns the last status change time of the file, in nanoseconds since [`st_ctime`]. + /// + /// [`st_ctime`]: Self::st_ctime + /// + /// # Examples + /// + /// ```no_run + /// use std::fs; + /// use std::io; + /// use std::os::linux::fs::MetadataExt; + /// + /// fn main() -> io::Result<()> { + /// let meta = fs::metadata("some_file")?; + /// println!("{}", meta.st_ctime_nsec()); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_ctime_nsec(&self) -> i64; + /// Returns the "preferred" block size for efficient filesystem I/O. + /// + /// # Examples + /// + /// ```no_run + /// use std::fs; + /// use std::io; + /// use std::os::linux::fs::MetadataExt; + /// + /// fn main() -> io::Result<()> { + /// let meta = fs::metadata("some_file")?; + /// println!("{}", meta.st_blksize()); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_blksize(&self) -> u64; + /// Returns the number of blocks allocated to the file, 512-byte units. + /// + /// # Examples + /// + /// ```no_run + /// use std::fs; + /// use std::io; + /// use std::os::linux::fs::MetadataExt; + /// + /// fn main() -> io::Result<()> { + /// let meta = fs::metadata("some_file")?; + /// println!("{}", meta.st_blocks()); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_blocks(&self) -> u64; +} + +#[stable(feature = "metadata_ext", since = "1.1.0")] +impl MetadataExt for Metadata { + #[allow(deprecated)] + fn as_raw_stat(&self) -> &raw::stat { + unsafe { &*(self.as_inner().as_inner() as *const libc::stat64 as *const raw::stat) } + } + fn st_dev(&self) -> u64 { + self.as_inner().as_inner().st_dev as u64 + } + fn st_ino(&self) -> u64 { + self.as_inner().as_inner().st_ino as u64 + } + fn st_mode(&self) -> u32 { + self.as_inner().as_inner().st_mode as u32 + } + fn st_nlink(&self) -> u64 { + self.as_inner().as_inner().st_nlink as u64 + } + fn st_uid(&self) -> u32 { + self.as_inner().as_inner().st_uid as u32 + } + fn st_gid(&self) -> u32 { + self.as_inner().as_inner().st_gid as u32 + } + fn st_rdev(&self) -> u64 { + self.as_inner().as_inner().st_rdev as u64 + } + fn st_size(&self) -> u64 { + self.as_inner().as_inner().st_size as u64 + } + fn st_atime(&self) -> i64 { + self.as_inner().as_inner().st_atime as i64 + } + fn st_atime_nsec(&self) -> i64 { + self.as_inner().as_inner().st_atime_nsec as i64 + } + fn st_mtime(&self) -> i64 { + self.as_inner().as_inner().st_mtime as i64 + } + fn st_mtime_nsec(&self) -> i64 { + self.as_inner().as_inner().st_mtime_nsec as i64 + } + fn st_ctime(&self) -> i64 { + self.as_inner().as_inner().st_ctime as i64 + } + fn st_ctime_nsec(&self) -> i64 { + self.as_inner().as_inner().st_ctime_nsec as i64 + } + fn st_blksize(&self) -> u64 { + self.as_inner().as_inner().st_blksize as u64 + } + fn st_blocks(&self) -> u64 { + self.as_inner().as_inner().st_blocks as u64 + } +} diff --git a/crux-mir/lib/std/src/os/l4re/mod.rs b/crux-mir/lib/std/src/os/l4re/mod.rs new file mode 100644 index 000000000..14c2425c1 --- /dev/null +++ b/crux-mir/lib/std/src/os/l4re/mod.rs @@ -0,0 +1,7 @@ +//! L4Re-specific definitions. + +#![stable(feature = "raw_ext", since = "1.1.0")] +#![doc(cfg(target_os = "l4re"))] + +pub mod fs; +pub mod raw; diff --git a/crux-mir/lib/std/src/os/l4re/raw.rs b/crux-mir/lib/std/src/os/l4re/raw.rs new file mode 100644 index 000000000..699e8be33 --- /dev/null +++ b/crux-mir/lib/std/src/os/l4re/raw.rs @@ -0,0 +1,365 @@ +//! L4Re-specific raw type definitions. + +#![stable(feature = "raw_ext", since = "1.1.0")] +#![deprecated( + since = "1.8.0", + note = "these type aliases are no longer supported by \ + the standard library, the `libc` crate on \ + crates.io should be used instead for the correct \ + definitions" +)] +#![allow(deprecated)] + +use crate::os::raw::c_ulong; + +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type dev_t = u64; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type mode_t = u32; + +#[stable(feature = "pthread_t", since = "1.8.0")] +pub type pthread_t = c_ulong; + +#[doc(inline)] +#[stable(feature = "raw_ext", since = "1.1.0")] +pub use self::arch::{blkcnt_t, blksize_t, ino_t, nlink_t, off_t, stat, time_t}; + +#[cfg(any( + target_arch = "x86", + target_arch = "le32", + target_arch = "m68k", + target_arch = "powerpc", + target_arch = "sparc", + target_arch = "arm", + target_arch = "asmjs", + target_arch = "wasm32" +))] +mod arch { + use crate::os::raw::{c_long, c_short, c_uint}; + + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type blkcnt_t = u64; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type blksize_t = u64; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type ino_t = u64; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type nlink_t = u64; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type off_t = u64; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type time_t = i64; + + #[repr(C)] + #[derive(Clone)] + #[stable(feature = "raw_ext", since = "1.1.0")] + pub struct stat { + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_dev: u64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub __pad1: c_short, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub __st_ino: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mode: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_nlink: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_uid: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_gid: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_rdev: u64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub __pad2: c_uint, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_size: i64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_blksize: i32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_blocks: i64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_atime: i32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_atime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mtime: i32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mtime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ctime: i32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ctime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ino: u64, + } +} + +#[cfg(target_arch = "mips")] +mod arch { + use crate::os::raw::{c_long, c_ulong}; + + #[cfg(target_env = "musl")] + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type blkcnt_t = i64; + #[cfg(not(target_env = "musl"))] + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type blkcnt_t = u64; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type blksize_t = u64; + #[cfg(target_env = "musl")] + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type ino_t = u64; + #[cfg(not(target_env = "musl"))] + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type ino_t = u64; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type nlink_t = u64; + #[cfg(target_env = "musl")] + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type off_t = u64; + #[cfg(not(target_env = "musl"))] + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type off_t = u64; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type time_t = i64; + + #[repr(C)] + #[derive(Clone)] + #[stable(feature = "raw_ext", since = "1.1.0")] + pub struct stat { + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_dev: c_ulong, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_pad1: [c_long; 3], + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ino: u64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mode: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_nlink: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_uid: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_gid: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_rdev: c_ulong, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_pad2: [c_long; 2], + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_size: i64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_atime: i32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_atime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mtime: i32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mtime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ctime: i32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ctime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_blksize: i32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_blocks: i64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_pad5: [c_long; 14], + } +} + +#[cfg(target_arch = "hexagon")] +mod arch { + use crate::os::raw::{c_int, c_long, c_uint}; + + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type blkcnt_t = i64; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type blksize_t = c_long; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type ino_t = u64; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type nlink_t = c_uint; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type off_t = i64; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type time_t = i64; + + #[repr(C)] + #[derive(Clone)] + #[stable(feature = "raw_ext", since = "1.1.0")] + pub struct stat { + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_dev: u64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ino: u64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mode: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_nlink: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_uid: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_gid: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_rdev: u64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub __pad1: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_size: i64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_blksize: i32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub __pad2: i32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_blocks: i64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_atime: i64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_atime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mtime: i64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mtime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ctime: i64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ctime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub __pad3: [c_int; 2], + } +} + +#[cfg(any( + target_arch = "mips64", + target_arch = "s390x", + target_arch = "sparc64", + target_arch = "riscv64", + target_arch = "riscv32" +))] +mod arch { + pub use libc::{blkcnt_t, blksize_t, ino_t, nlink_t, off_t, stat, time_t}; +} + +#[cfg(target_arch = "aarch64")] +mod arch { + use crate::os::raw::{c_int, c_long}; + + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type blkcnt_t = i64; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type blksize_t = i32; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type ino_t = u64; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type nlink_t = u32; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type off_t = i64; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type time_t = c_long; + + #[repr(C)] + #[derive(Clone)] + #[stable(feature = "raw_ext", since = "1.1.0")] + pub struct stat { + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_dev: u64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ino: u64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mode: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_nlink: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_uid: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_gid: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_rdev: u64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub __pad1: u64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_size: i64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_blksize: i32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub __pad2: c_int, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_blocks: i64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_atime: time_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_atime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mtime: time_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mtime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ctime: time_t, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ctime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub __unused: [c_int; 2], + } +} + +#[cfg(any(target_arch = "x86_64", target_arch = "powerpc64"))] +mod arch { + use crate::os::raw::{c_int, c_long}; + + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type blkcnt_t = u64; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type blksize_t = u64; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type ino_t = u64; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type nlink_t = u64; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type off_t = u64; + #[stable(feature = "raw_ext", since = "1.1.0")] + pub type time_t = i64; + + #[repr(C)] + #[derive(Clone)] + #[stable(feature = "raw_ext", since = "1.1.0")] + pub struct stat { + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_dev: u64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ino: u64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_nlink: u64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mode: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_uid: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_gid: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub __pad0: c_int, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_rdev: u64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_size: i64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_blksize: i64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_blocks: i64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_atime: i64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_atime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mtime: i64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mtime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ctime: i64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ctime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub __unused: [c_long; 3], + } +} diff --git a/crux-mir/lib/std/src/os/linux/fs.rs b/crux-mir/lib/std/src/os/linux/fs.rs index d22b44a06..479bbcc17 100644 --- a/crux-mir/lib/std/src/os/linux/fs.rs +++ b/crux-mir/lib/std/src/os/linux/fs.rs @@ -1,3 +1,7 @@ +//! Linux-specific extensions to primitives in the [`std::fs`] module. +//! +//! [`std::fs`]: crate::fs + #![stable(feature = "metadata_ext", since = "1.1.0")] use crate::fs::Metadata; @@ -8,7 +12,7 @@ use crate::os::linux::raw; /// OS-specific extensions to [`fs::Metadata`]. /// -/// [`fs::Metadata`]: ../../../../std/fs/struct.Metadata.html +/// [`fs::Metadata`]: crate::fs::Metadata #[stable(feature = "metadata_ext", since = "1.1.0")] pub trait MetadataExt { /// Gain a reference to the underlying `stat` structure which contains @@ -18,7 +22,7 @@ pub trait MetadataExt { /// Unix platforms. The `os::unix::fs::MetadataExt` trait contains the /// cross-Unix abstractions contained within the raw stat. /// - /// [`stat`]: ../../../../std/os/linux/raw/struct.stat.html + /// [`stat`]: struct@crate::os::linux::raw::stat /// /// # Examples /// @@ -34,7 +38,7 @@ pub trait MetadataExt { /// } /// ``` #[stable(feature = "metadata_ext", since = "1.1.0")] - #[rustc_deprecated(since = "1.8.0", reason = "other methods of this trait are now preferred")] + #[deprecated(since = "1.8.0", note = "other methods of this trait are now preferred")] #[allow(deprecated)] fn as_raw_stat(&self) -> &raw::stat; @@ -196,7 +200,7 @@ pub trait MetadataExt { fn st_atime(&self) -> i64; /// Returns the last access time of the file, in nanoseconds since [`st_atime`]. /// - /// [`st_atime`]: #tymethod.st_atime + /// [`st_atime`]: Self::st_atime /// /// # Examples /// @@ -232,7 +236,7 @@ pub trait MetadataExt { fn st_mtime(&self) -> i64; /// Returns the last modification time of the file, in nanoseconds since [`st_mtime`]. /// - /// [`st_mtime`]: #tymethod.st_mtime + /// [`st_mtime`]: Self::st_mtime /// /// # Examples /// @@ -268,7 +272,7 @@ pub trait MetadataExt { fn st_ctime(&self) -> i64; /// Returns the last status change time of the file, in nanoseconds since [`st_ctime`]. /// - /// [`st_ctime`]: #tymethod.st_ctime + /// [`st_ctime`]: Self::st_ctime /// /// # Examples /// @@ -285,7 +289,7 @@ pub trait MetadataExt { /// ``` #[stable(feature = "metadata_ext2", since = "1.8.0")] fn st_ctime_nsec(&self) -> i64; - /// Returns the "preferred" blocksize for efficient filesystem I/O. + /// Returns the "preferred" block size for efficient filesystem I/O. /// /// # Examples /// @@ -352,19 +356,34 @@ impl MetadataExt for Metadata { self.as_inner().as_inner().st_size as u64 } fn st_atime(&self) -> i64 { - self.as_inner().as_inner().st_atime as i64 + let file_attr = self.as_inner(); + #[cfg(all(target_env = "gnu", target_pointer_width = "32"))] + if let Some(atime) = file_attr.stx_atime() { + return atime.tv_sec; + } + file_attr.as_inner().st_atime as i64 } fn st_atime_nsec(&self) -> i64 { self.as_inner().as_inner().st_atime_nsec as i64 } fn st_mtime(&self) -> i64 { - self.as_inner().as_inner().st_mtime as i64 + let file_attr = self.as_inner(); + #[cfg(all(target_env = "gnu", target_pointer_width = "32"))] + if let Some(mtime) = file_attr.stx_mtime() { + return mtime.tv_sec; + } + file_attr.as_inner().st_mtime as i64 } fn st_mtime_nsec(&self) -> i64 { self.as_inner().as_inner().st_mtime_nsec as i64 } fn st_ctime(&self) -> i64 { - self.as_inner().as_inner().st_ctime as i64 + let file_attr = self.as_inner(); + #[cfg(all(target_env = "gnu", target_pointer_width = "32"))] + if let Some(ctime) = file_attr.stx_ctime() { + return ctime.tv_sec; + } + file_attr.as_inner().st_ctime as i64 } fn st_ctime_nsec(&self) -> i64 { self.as_inner().as_inner().st_ctime_nsec as i64 diff --git a/crux-mir/lib/std/src/os/linux/mod.rs b/crux-mir/lib/std/src/os/linux/mod.rs index d35307162..c17053011 100644 --- a/crux-mir/lib/std/src/os/linux/mod.rs +++ b/crux-mir/lib/std/src/os/linux/mod.rs @@ -1,6 +1,9 @@ -//! Linux-specific definitions +//! Linux-specific definitions. #![stable(feature = "raw_ext", since = "1.1.0")] +#![doc(cfg(target_os = "linux"))] pub mod fs; +pub mod net; +pub mod process; pub mod raw; diff --git a/crux-mir/lib/std/src/os/linux/net.rs b/crux-mir/lib/std/src/os/linux/net.rs new file mode 100644 index 000000000..94081c8dd --- /dev/null +++ b/crux-mir/lib/std/src/os/linux/net.rs @@ -0,0 +1,9 @@ +//! Linux-specific networking functionality. + +#![unstable(feature = "tcp_quickack", issue = "96256")] + +#[unstable(feature = "unix_socket_abstract", issue = "85410")] +pub use crate::os::net::linux_ext::addr::SocketAddrExt; + +#[unstable(feature = "tcp_quickack", issue = "96256")] +pub use crate::os::net::linux_ext::tcp::TcpStreamExt; diff --git a/crux-mir/lib/std/src/os/linux/process.rs b/crux-mir/lib/std/src/os/linux/process.rs new file mode 100644 index 000000000..540363c03 --- /dev/null +++ b/crux-mir/lib/std/src/os/linux/process.rs @@ -0,0 +1,165 @@ +//! Linux-specific extensions to primitives in the [`std::process`] module. +//! +//! [`std::process`]: crate::process + +#![unstable(feature = "linux_pidfd", issue = "82971")] + +use crate::io::Result; +use crate::os::unix::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; +use crate::process; +use crate::sealed::Sealed; +#[cfg(not(doc))] +use crate::sys::fd::FileDesc; +use crate::sys_common::{AsInner, AsInnerMut, FromInner, IntoInner}; + +#[cfg(doc)] +struct FileDesc; + +/// This type represents a file descriptor that refers to a process. +/// +/// A `PidFd` can be obtained by setting the corresponding option on [`Command`] +/// with [`create_pidfd`]. Subsequently, the created pidfd can be retrieved +/// from the [`Child`] by calling [`pidfd`] or [`take_pidfd`]. +/// +/// Example: +/// ```no_run +/// #![feature(linux_pidfd)] +/// use std::os::linux::process::{CommandExt, ChildExt}; +/// use std::process::Command; +/// +/// let mut child = Command::new("echo") +/// .create_pidfd(true) +/// .spawn() +/// .expect("Failed to spawn child"); +/// +/// let pidfd = child +/// .take_pidfd() +/// .expect("Failed to retrieve pidfd"); +/// +/// // The file descriptor will be closed when `pidfd` is dropped. +/// ``` +/// Refer to the man page of [`pidfd_open(2)`] for further details. +/// +/// [`Command`]: process::Command +/// [`create_pidfd`]: CommandExt::create_pidfd +/// [`Child`]: process::Child +/// [`pidfd`]: fn@ChildExt::pidfd +/// [`take_pidfd`]: ChildExt::take_pidfd +/// [`pidfd_open(2)`]: https://man7.org/linux/man-pages/man2/pidfd_open.2.html +#[derive(Debug)] +pub struct PidFd { + inner: FileDesc, +} + +impl AsInner for PidFd { + fn as_inner(&self) -> &FileDesc { + &self.inner + } +} + +impl FromInner for PidFd { + fn from_inner(inner: FileDesc) -> PidFd { + PidFd { inner } + } +} + +impl IntoInner for PidFd { + fn into_inner(self) -> FileDesc { + self.inner + } +} + +impl AsRawFd for PidFd { + fn as_raw_fd(&self) -> RawFd { + self.as_inner().as_raw_fd() + } +} + +impl FromRawFd for PidFd { + unsafe fn from_raw_fd(fd: RawFd) -> Self { + Self::from_inner(FileDesc::from_raw_fd(fd)) + } +} + +impl IntoRawFd for PidFd { + fn into_raw_fd(self) -> RawFd { + self.into_inner().into_raw_fd() + } +} + +impl AsFd for PidFd { + fn as_fd(&self) -> BorrowedFd<'_> { + self.as_inner().as_fd() + } +} + +impl From for PidFd { + fn from(fd: OwnedFd) -> Self { + Self::from_inner(FileDesc::from_inner(fd)) + } +} + +impl From for OwnedFd { + fn from(pid_fd: PidFd) -> Self { + pid_fd.into_inner().into_inner() + } +} + +/// Os-specific extensions for [`Child`] +/// +/// [`Child`]: process::Child +pub trait ChildExt: Sealed { + /// Obtains a reference to the [`PidFd`] created for this [`Child`], if available. + /// + /// A pidfd will only be available if its creation was requested with + /// [`create_pidfd`] when the corresponding [`Command`] was created. + /// + /// Even if requested, a pidfd may not be available due to an older + /// version of Linux being in use, or if some other error occurred. + /// + /// [`Command`]: process::Command + /// [`create_pidfd`]: CommandExt::create_pidfd + /// [`Child`]: process::Child + fn pidfd(&self) -> Result<&PidFd>; + + /// Takes ownership of the [`PidFd`] created for this [`Child`], if available. + /// + /// A pidfd will only be available if its creation was requested with + /// [`create_pidfd`] when the corresponding [`Command`] was created. + /// + /// Even if requested, a pidfd may not be available due to an older + /// version of Linux being in use, or if some other error occurred. + /// + /// [`Command`]: process::Command + /// [`create_pidfd`]: CommandExt::create_pidfd + /// [`Child`]: process::Child + fn take_pidfd(&mut self) -> Result; +} + +/// Os-specific extensions for [`Command`] +/// +/// [`Command`]: process::Command +pub trait CommandExt: Sealed { + /// Sets whether a [`PidFd`](struct@PidFd) should be created for the [`Child`] + /// spawned by this [`Command`]. + /// By default, no pidfd will be created. + /// + /// The pidfd can be retrieved from the child with [`pidfd`] or [`take_pidfd`]. + /// + /// A pidfd will only be created if it is possible to do so + /// in a guaranteed race-free manner (e.g. if the `clone3` system call + /// is supported). Otherwise, [`pidfd`] will return an error. + /// + /// [`Command`]: process::Command + /// [`Child`]: process::Child + /// [`pidfd`]: fn@ChildExt::pidfd + /// [`take_pidfd`]: ChildExt::take_pidfd + fn create_pidfd(&mut self, val: bool) -> &mut process::Command; +} + +impl CommandExt for process::Command { + fn create_pidfd(&mut self, val: bool) -> &mut process::Command { + self.as_inner_mut().create_pidfd(val); + self + } +} diff --git a/crux-mir/lib/std/src/os/linux/raw.rs b/crux-mir/lib/std/src/os/linux/raw.rs index 0caec97bb..c73791d14 100644 --- a/crux-mir/lib/std/src/os/linux/raw.rs +++ b/crux-mir/lib/std/src/os/linux/raw.rs @@ -1,15 +1,14 @@ -//! Linux-specific raw type definitions +//! Linux-specific raw type definitions. #![stable(feature = "raw_ext", since = "1.1.0")] -#![rustc_deprecated( +#![deprecated( since = "1.8.0", - reason = "these type aliases are no longer supported by \ - the standard library, the `libc` crate on \ - crates.io should be used instead for the correct \ - definitions" + note = "these type aliases are no longer supported by \ + the standard library, the `libc` crate on \ + crates.io should be used instead for the correct \ + definitions" )] #![allow(deprecated)] -#![allow(missing_debug_implementations)] use crate::os::raw::c_ulong; @@ -28,7 +27,9 @@ pub use self::arch::{blkcnt_t, blksize_t, ino_t, nlink_t, off_t, stat, time_t}; #[cfg(any( target_arch = "x86", target_arch = "le32", + target_arch = "m68k", target_arch = "powerpc", + target_arch = "sparc", target_arch = "arm", target_arch = "asmjs", target_arch = "wasm32" @@ -170,63 +171,63 @@ mod arch { #[cfg(target_arch = "hexagon")] mod arch { - use crate::os::raw::{c_int, c_long, c_longlong, culonglong}; + use crate::os::raw::{c_int, c_long, c_uint}; #[stable(feature = "raw_ext", since = "1.1.0")] - pub type blkcnt_t = c_longlong; + pub type blkcnt_t = i64; #[stable(feature = "raw_ext", since = "1.1.0")] pub type blksize_t = c_long; #[stable(feature = "raw_ext", since = "1.1.0")] - pub type ino_t = c_ulonglong; + pub type ino_t = u64; #[stable(feature = "raw_ext", since = "1.1.0")] pub type nlink_t = c_uint; #[stable(feature = "raw_ext", since = "1.1.0")] - pub type off_t = c_longlong; + pub type off_t = i64; #[stable(feature = "raw_ext", since = "1.1.0")] - pub type time_t = c_long; + pub type time_t = i64; #[repr(C)] #[derive(Clone)] #[stable(feature = "raw_ext", since = "1.1.0")] pub struct stat { #[stable(feature = "raw_ext", since = "1.1.0")] - pub st_dev: ::dev_t, + pub st_dev: u64, #[stable(feature = "raw_ext", since = "1.1.0")] - pub st_ino: ::c_ulonglong, + pub st_ino: u64, #[stable(feature = "raw_ext", since = "1.1.0")] - pub st_mode: ::c_uint, + pub st_mode: u32, #[stable(feature = "raw_ext", since = "1.1.0")] - pub st_nlink: ::c_uint, + pub st_nlink: u32, #[stable(feature = "raw_ext", since = "1.1.0")] - pub st_uid: ::c_uint, + pub st_uid: u32, #[stable(feature = "raw_ext", since = "1.1.0")] - pub st_gid: ::c_uint, + pub st_gid: u32, #[stable(feature = "raw_ext", since = "1.1.0")] - pub st_rdev: ::c_ulonglong, + pub st_rdev: u64, #[stable(feature = "raw_ext", since = "1.1.0")] - pub __pad1: ::c_ulong, + pub __pad1: u32, #[stable(feature = "raw_ext", since = "1.1.0")] - pub st_size: ::c_longlong, + pub st_size: i64, #[stable(feature = "raw_ext", since = "1.1.0")] - pub st_blksize: ::blksize_t, + pub st_blksize: i32, #[stable(feature = "raw_ext", since = "1.1.0")] - pub __pad2: ::c_int, + pub __pad2: i32, #[stable(feature = "raw_ext", since = "1.1.0")] - pub st_blocks: ::blkcnt_t, + pub st_blocks: i64, #[stable(feature = "raw_ext", since = "1.1.0")] - pub st_atime: ::time_t, + pub st_atime: i64, #[stable(feature = "raw_ext", since = "1.1.0")] - pub st_atime_nsec: ::c_long, + pub st_atime_nsec: c_long, #[stable(feature = "raw_ext", since = "1.1.0")] - pub st_mtime: ::time_t, + pub st_mtime: i64, #[stable(feature = "raw_ext", since = "1.1.0")] - pub st_mtime_nsec: ::c_long, + pub st_mtime_nsec: c_long, #[stable(feature = "raw_ext", since = "1.1.0")] - pub st_ctime: ::time_t, + pub st_ctime: i64, #[stable(feature = "raw_ext", since = "1.1.0")] - pub st_ctime_nsec: ::c_long, + pub st_ctime_nsec: c_long, #[stable(feature = "raw_ext", since = "1.1.0")] - pub __pad3: [::c_int; 2], + pub __pad3: [c_int; 2], } } @@ -234,9 +235,11 @@ mod arch { target_arch = "mips64", target_arch = "s390x", target_arch = "sparc64", - target_arch = "riscv64" + target_arch = "riscv64", + target_arch = "riscv32" ))] mod arch { + #[stable(feature = "raw_ext", since = "1.1.0")] pub use libc::{blkcnt_t, blksize_t, ino_t, nlink_t, off_t, stat, time_t}; } @@ -245,17 +248,17 @@ mod arch { use crate::os::raw::{c_int, c_long}; #[stable(feature = "raw_ext", since = "1.1.0")] - pub type blkcnt_t = u64; + pub type blkcnt_t = i64; #[stable(feature = "raw_ext", since = "1.1.0")] - pub type blksize_t = u64; + pub type blksize_t = i32; #[stable(feature = "raw_ext", since = "1.1.0")] pub type ino_t = u64; #[stable(feature = "raw_ext", since = "1.1.0")] - pub type nlink_t = u64; + pub type nlink_t = u32; #[stable(feature = "raw_ext", since = "1.1.0")] - pub type off_t = u64; + pub type off_t = i64; #[stable(feature = "raw_ext", since = "1.1.0")] - pub type time_t = i64; + pub type time_t = c_long; #[repr(C)] #[derive(Clone)] @@ -286,15 +289,15 @@ mod arch { #[stable(feature = "raw_ext", since = "1.1.0")] pub st_blocks: i64, #[stable(feature = "raw_ext", since = "1.1.0")] - pub st_atime: i64, + pub st_atime: time_t, #[stable(feature = "raw_ext", since = "1.1.0")] pub st_atime_nsec: c_long, #[stable(feature = "raw_ext", since = "1.1.0")] - pub st_mtime: i64, + pub st_mtime: time_t, #[stable(feature = "raw_ext", since = "1.1.0")] pub st_mtime_nsec: c_long, #[stable(feature = "raw_ext", since = "1.1.0")] - pub st_ctime: i64, + pub st_ctime: time_t, #[stable(feature = "raw_ext", since = "1.1.0")] pub st_ctime_nsec: c_long, #[stable(feature = "raw_ext", since = "1.1.0")] diff --git a/crux-mir/lib/std/src/os/macos/fs.rs b/crux-mir/lib/std/src/os/macos/fs.rs index ad313a124..91915da6a 100644 --- a/crux-mir/lib/std/src/os/macos/fs.rs +++ b/crux-mir/lib/std/src/os/macos/fs.rs @@ -8,7 +8,7 @@ use crate::os::macos::raw; /// OS-specific extensions to [`fs::Metadata`]. /// -/// [`fs::Metadata`]: ../../../../std/fs/struct.Metadata.html +/// [`fs::Metadata`]: crate::fs::Metadata #[stable(feature = "metadata_ext", since = "1.1.0")] pub trait MetadataExt { /// Gain a reference to the underlying `stat` structure which contains @@ -18,10 +18,10 @@ pub trait MetadataExt { /// Unix platforms. The `os::unix::fs::MetadataExt` trait contains the /// cross-Unix abstractions contained within the raw stat. #[stable(feature = "metadata_ext", since = "1.1.0")] - #[rustc_deprecated( + #[deprecated( since = "1.8.0", - reason = "deprecated in favor of the accessor \ - methods of this trait" + note = "deprecated in favor of the accessor \ + methods of this trait" )] #[allow(deprecated)] fn as_raw_stat(&self) -> &raw::stat; diff --git a/crux-mir/lib/std/src/os/macos/raw.rs b/crux-mir/lib/std/src/os/macos/raw.rs index 708261d86..0b21f6ee5 100644 --- a/crux-mir/lib/std/src/os/macos/raw.rs +++ b/crux-mir/lib/std/src/os/macos/raw.rs @@ -1,12 +1,12 @@ //! macOS-specific raw type definitions #![stable(feature = "raw_ext", since = "1.1.0")] -#![rustc_deprecated( +#![deprecated( since = "1.8.0", - reason = "these type aliases are no longer supported by \ - the standard library, the `libc` crate on \ - crates.io should be used instead for the correct \ - definitions" + note = "these type aliases are no longer supported by \ + the standard library, the `libc` crate on \ + crates.io should be used instead for the correct \ + definitions" )] #![allow(deprecated)] diff --git a/crux-mir/lib/std/src/os/mod.rs b/crux-mir/lib/std/src/os/mod.rs index 91e37ed83..42773805c 100644 --- a/crux-mir/lib/std/src/os/mod.rs +++ b/crux-mir/lib/std/src/os/mod.rs @@ -3,47 +3,118 @@ #![stable(feature = "os", since = "1.0.0")] #![allow(missing_docs, nonstandard_style, missing_debug_implementations)] -cfg_if::cfg_if! { - if #[cfg(doc)] { - - // When documenting libstd we want to show unix/windows/linux modules as - // these are the "main modules" that are used across platforms. This - // should help show platform-specific functionality in a hopefully - // cross-platform way in the documentation - - #[stable(feature = "rust1", since = "1.0.0")] - pub use crate::sys::unix_ext as unix; +pub mod raw; - #[stable(feature = "rust1", since = "1.0.0")] - pub use crate::sys::windows_ext as windows; +// The code below could be written clearer using `cfg_if!`. However, the items below are +// publicly exported by `std` and external tools can have trouble analysing them because of the use +// of a macro that is not vendored by Rust and included in the toolchain. +// See https://github.com/rust-analyzer/rust-analyzer/issues/6038. - #[doc(cfg(target_os = "linux"))] - pub mod linux; - } else { +// On certain platforms right now the "main modules" modules that are +// documented don't compile (missing things in `libc` which is empty), +// so just omit them with an empty module and add the "unstable" attribute. - // If we're not documenting libstd then we just expose the main modules - // as we otherwise would. +// Unix, linux, wasi and windows are handled a bit differently. +#[cfg(all( + doc, + any( + all(target_arch = "wasm32", not(target_os = "wasi")), + all(target_vendor = "fortanix", target_env = "sgx") + ) +))] +#[unstable(issue = "none", feature = "std_internals")] +pub mod unix {} +#[cfg(all( + doc, + any( + all(target_arch = "wasm32", not(target_os = "wasi")), + all(target_vendor = "fortanix", target_env = "sgx") + ) +))] +#[unstable(issue = "none", feature = "std_internals")] +pub mod linux {} +#[cfg(all( + doc, + any( + all(target_arch = "wasm32", not(target_os = "wasi")), + all(target_vendor = "fortanix", target_env = "sgx") + ) +))] +#[unstable(issue = "none", feature = "std_internals")] +pub mod wasi {} +#[cfg(all( + doc, + any( + all(target_arch = "wasm32", not(target_os = "wasi")), + all(target_vendor = "fortanix", target_env = "sgx") + ) +))] +#[unstable(issue = "none", feature = "std_internals")] +pub mod windows {} - #[cfg(any(target_os = "redox", unix, target_os = "vxworks"))] - #[stable(feature = "rust1", since = "1.0.0")] - pub use crate::sys::ext as unix; +// unix +#[cfg(not(all( + doc, + any( + all(target_arch = "wasm32", not(target_os = "wasi")), + all(target_vendor = "fortanix", target_env = "sgx") + ) +)))] +#[cfg(target_os = "hermit")] +#[path = "hermit/mod.rs"] +pub mod unix; +#[cfg(not(all( + doc, + any( + all(target_arch = "wasm32", not(target_os = "wasi")), + all(target_vendor = "fortanix", target_env = "sgx") + ) +)))] +#[cfg(all(not(target_os = "hermit"), any(unix, doc)))] +pub mod unix; - #[cfg(windows)] - #[stable(feature = "rust1", since = "1.0.0")] - pub use crate::sys::ext as windows; +// linux +#[cfg(not(all( + doc, + any( + all(target_arch = "wasm32", not(target_os = "wasi")), + all(target_vendor = "fortanix", target_env = "sgx") + ) +)))] +#[cfg(any(target_os = "linux", doc))] +pub mod linux; - #[cfg(any(target_os = "linux", target_os = "l4re"))] - pub mod linux; +// wasi +#[cfg(not(all( + doc, + any( + all(target_arch = "wasm32", not(target_os = "wasi")), + all(target_vendor = "fortanix", target_env = "sgx") + ) +)))] +#[cfg(any(target_os = "wasi", doc))] +pub mod wasi; - } -} +// windows +#[cfg(not(all( + doc, + any( + all(target_arch = "wasm32", not(target_os = "wasi")), + all(target_vendor = "fortanix", target_env = "sgx") + ) +)))] +#[cfg(any(windows, doc))] +pub mod windows; +// Others. #[cfg(target_os = "android")] pub mod android; #[cfg(target_os = "dragonfly")] pub mod dragonfly; #[cfg(target_os = "emscripten")] pub mod emscripten; +#[cfg(target_os = "espidf")] +pub mod espidf; #[cfg(all(target_vendor = "fortanix", target_env = "sgx"))] pub mod fortanix_sgx; #[cfg(target_os = "freebsd")] @@ -52,8 +123,14 @@ pub mod freebsd; pub mod fuchsia; #[cfg(target_os = "haiku")] pub mod haiku; +#[cfg(target_os = "horizon")] +pub mod horizon; +#[cfg(target_os = "illumos")] +pub mod illumos; #[cfg(target_os = "ios")] pub mod ios; +#[cfg(target_os = "l4re")] +pub mod l4re; #[cfg(target_os = "macos")] pub mod macos; #[cfg(target_os = "netbsd")] @@ -64,9 +141,15 @@ pub mod openbsd; pub mod redox; #[cfg(target_os = "solaris")] pub mod solaris; +#[cfg(target_os = "solid_asp3")] +pub mod solid; #[cfg(target_os = "vxworks")] pub mod vxworks; -#[cfg(target_os = "wasi")] -pub mod wasi; +#[cfg(target_os = "watchos")] +pub(crate) mod watchos; -pub mod raw; +#[cfg(any(unix, target_os = "wasi", doc))] +pub mod fd; + +#[cfg(any(target_os = "linux", target_os = "android", doc))] +mod net; diff --git a/crux-mir/lib/std/src/os/net/linux_ext/addr.rs b/crux-mir/lib/std/src/os/net/linux_ext/addr.rs new file mode 100644 index 000000000..85065984f --- /dev/null +++ b/crux-mir/lib/std/src/os/net/linux_ext/addr.rs @@ -0,0 +1,64 @@ +//! Linux and Android-specific extensions to socket addresses. + +use crate::os::unix::net::SocketAddr; +use crate::sealed::Sealed; + +/// Platform-specific extensions to [`SocketAddr`]. +#[unstable(feature = "unix_socket_abstract", issue = "85410")] +pub trait SocketAddrExt: Sealed { + /// Creates a Unix socket address in the abstract namespace. + /// + /// The abstract namespace is a Linux-specific extension that allows Unix + /// sockets to be bound without creating an entry in the filesystem. + /// Abstract sockets are unaffected by filesystem layout or permissions, + /// and no cleanup is necessary when the socket is closed. + /// + /// An abstract socket address name may contain any bytes, including zero. + /// + /// # Errors + /// + /// Returns an error if the name is longer than `SUN_LEN - 1`. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(unix_socket_abstract)] + /// use std::os::unix::net::{UnixListener, SocketAddr}; + /// use std::os::linux::net::SocketAddrExt; + /// + /// fn main() -> std::io::Result<()> { + /// let addr = SocketAddr::from_abstract_name(b"hidden")?; + /// let listener = match UnixListener::bind_addr(&addr) { + /// Ok(sock) => sock, + /// Err(err) => { + /// println!("Couldn't bind: {err:?}"); + /// return Err(err); + /// } + /// }; + /// Ok(()) + /// } + /// ``` + fn from_abstract_name(name: N) -> crate::io::Result + where + N: AsRef<[u8]>; + + /// Returns the contents of this address if it is in the abstract namespace. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(unix_socket_abstract)] + /// use std::os::unix::net::{UnixListener, SocketAddr}; + /// use std::os::linux::net::SocketAddrExt; + /// + /// fn main() -> std::io::Result<()> { + /// let name = b"hidden"; + /// let name_addr = SocketAddr::from_abstract_name(name)?; + /// let socket = UnixListener::bind_addr(&name_addr)?; + /// let local_addr = socket.local_addr().expect("Couldn't get local address"); + /// assert_eq!(local_addr.as_abstract_name(), Some(&name[..])); + /// Ok(()) + /// } + /// ``` + fn as_abstract_name(&self) -> Option<&[u8]>; +} diff --git a/crux-mir/lib/std/src/os/net/linux_ext/mod.rs b/crux-mir/lib/std/src/os/net/linux_ext/mod.rs new file mode 100644 index 000000000..318ebacfd --- /dev/null +++ b/crux-mir/lib/std/src/os/net/linux_ext/mod.rs @@ -0,0 +1,12 @@ +//! Linux and Android-specific networking functionality. + +#![doc(cfg(any(target_os = "linux", target_os = "android")))] + +#[unstable(feature = "unix_socket_abstract", issue = "85410")] +pub(crate) mod addr; + +#[unstable(feature = "tcp_quickack", issue = "96256")] +pub(crate) mod tcp; + +#[cfg(test)] +mod tests; diff --git a/crux-mir/lib/std/src/os/net/linux_ext/tcp.rs b/crux-mir/lib/std/src/os/net/linux_ext/tcp.rs new file mode 100644 index 000000000..5e9ee65a4 --- /dev/null +++ b/crux-mir/lib/std/src/os/net/linux_ext/tcp.rs @@ -0,0 +1,70 @@ +//! Linux and Android-specific tcp extensions to primitives in the [`std::net`] module. +//! +//! [`std::net`]: crate::net + +use crate::io; +use crate::net; +use crate::sealed::Sealed; +use crate::sys_common::AsInner; + +/// Os-specific extensions for [`TcpStream`] +/// +/// [`TcpStream`]: net::TcpStream +#[unstable(feature = "tcp_quickack", issue = "96256")] +pub trait TcpStreamExt: Sealed { + /// Enable or disable `TCP_QUICKACK`. + /// + /// This flag causes Linux to eagerly send ACKs rather than delaying them. + /// Linux may reset this flag after further operations on the socket. + /// + /// See [`man 7 tcp`](https://man7.org/linux/man-pages/man7/tcp.7.html) and + /// [TCP delayed acknowledgement](https://en.wikipedia.org/wiki/TCP_delayed_acknowledgment) + /// for more information. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(tcp_quickack)] + /// use std::net::TcpStream; + /// use std::os::linux::net::TcpStreamExt; + /// + /// let stream = TcpStream::connect("127.0.0.1:8080") + /// .expect("Couldn't connect to the server..."); + /// stream.set_quickack(true).expect("set_quickack call failed"); + /// ``` + #[unstable(feature = "tcp_quickack", issue = "96256")] + fn set_quickack(&self, quickack: bool) -> io::Result<()>; + + /// Gets the value of the `TCP_QUICKACK` option on this socket. + /// + /// For more information about this option, see [`TcpStreamExt::set_quickack`]. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(tcp_quickack)] + /// use std::net::TcpStream; + /// use std::os::linux::net::TcpStreamExt; + /// + /// let stream = TcpStream::connect("127.0.0.1:8080") + /// .expect("Couldn't connect to the server..."); + /// stream.set_quickack(true).expect("set_quickack call failed"); + /// assert_eq!(stream.quickack().unwrap_or(false), true); + /// ``` + #[unstable(feature = "tcp_quickack", issue = "96256")] + fn quickack(&self) -> io::Result; +} + +#[unstable(feature = "tcp_quickack", issue = "96256")] +impl Sealed for net::TcpStream {} + +#[unstable(feature = "tcp_quickack", issue = "96256")] +impl TcpStreamExt for net::TcpStream { + fn set_quickack(&self, quickack: bool) -> io::Result<()> { + self.as_inner().as_inner().set_quickack(quickack) + } + + fn quickack(&self) -> io::Result { + self.as_inner().as_inner().quickack() + } +} diff --git a/crux-mir/lib/std/src/os/net/linux_ext/tests.rs b/crux-mir/lib/std/src/os/net/linux_ext/tests.rs new file mode 100644 index 000000000..2db4deed0 --- /dev/null +++ b/crux-mir/lib/std/src/os/net/linux_ext/tests.rs @@ -0,0 +1,28 @@ +#[test] +fn quickack() { + use crate::{ + net::{test::next_test_ip4, TcpListener, TcpStream}, + os::net::linux_ext::tcp::TcpStreamExt, + }; + + macro_rules! t { + ($e:expr) => { + match $e { + Ok(t) => t, + Err(e) => panic!("received error for `{}`: {}", stringify!($e), e), + } + }; + } + + let addr = next_test_ip4(); + let _listener = t!(TcpListener::bind(&addr)); + + let stream = t!(TcpStream::connect(&("localhost", addr.port()))); + + t!(stream.set_quickack(false)); + assert_eq!(false, t!(stream.quickack())); + t!(stream.set_quickack(true)); + assert_eq!(true, t!(stream.quickack())); + t!(stream.set_quickack(false)); + assert_eq!(false, t!(stream.quickack())); +} diff --git a/crux-mir/lib/std/src/os/net/mod.rs b/crux-mir/lib/std/src/os/net/mod.rs new file mode 100644 index 000000000..5ec267c41 --- /dev/null +++ b/crux-mir/lib/std/src/os/net/mod.rs @@ -0,0 +1,4 @@ +//! OS-specific networking functionality. + +#[cfg(any(target_os = "linux", target_os = "android", doc))] +pub(super) mod linux_ext; diff --git a/crux-mir/lib/std/src/os/netbsd/fs.rs b/crux-mir/lib/std/src/os/netbsd/fs.rs index 90980fdce..fe0be069e 100644 --- a/crux-mir/lib/std/src/os/netbsd/fs.rs +++ b/crux-mir/lib/std/src/os/netbsd/fs.rs @@ -8,7 +8,7 @@ use crate::os::netbsd::raw; /// OS-specific extensions to [`fs::Metadata`]. /// -/// [`fs::Metadata`]: ../../../../std/fs/struct.Metadata.html +/// [`fs::Metadata`]: crate::fs::Metadata #[stable(feature = "metadata_ext", since = "1.1.0")] pub trait MetadataExt { /// Gain a reference to the underlying `stat` structure which contains @@ -18,10 +18,10 @@ pub trait MetadataExt { /// Unix platforms. The `os::unix::fs::MetadataExt` trait contains the /// cross-Unix abstractions contained within the raw stat. #[stable(feature = "metadata_ext", since = "1.1.0")] - #[rustc_deprecated( + #[deprecated( since = "1.8.0", - reason = "deprecated in favor of the accessor \ - methods of this trait" + note = "deprecated in favor of the accessor \ + methods of this trait" )] #[allow(deprecated)] fn as_raw_stat(&self) -> &raw::stat; diff --git a/crux-mir/lib/std/src/os/netbsd/raw.rs b/crux-mir/lib/std/src/os/netbsd/raw.rs index 475fcdcc4..18057291f 100644 --- a/crux-mir/lib/std/src/os/netbsd/raw.rs +++ b/crux-mir/lib/std/src/os/netbsd/raw.rs @@ -1,12 +1,12 @@ //! NetBSD-specific raw type definitions #![stable(feature = "raw_ext", since = "1.1.0")] -#![rustc_deprecated( +#![deprecated( since = "1.8.0", - reason = "these type aliases are no longer supported by \ - the standard library, the `libc` crate on \ - crates.io should be used instead for the correct \ - definitions" + note = "these type aliases are no longer supported by \ + the standard library, the `libc` crate on \ + crates.io should be used instead for the correct \ + definitions" )] #![allow(deprecated)] diff --git a/crux-mir/lib/std/src/os/openbsd/fs.rs b/crux-mir/lib/std/src/os/openbsd/fs.rs index 47da00ae2..b8d8d31c5 100644 --- a/crux-mir/lib/std/src/os/openbsd/fs.rs +++ b/crux-mir/lib/std/src/os/openbsd/fs.rs @@ -8,7 +8,7 @@ use crate::os::openbsd::raw; /// OS-specific extensions to [`fs::Metadata`]. /// -/// [`fs::Metadata`]: ../../../../std/fs/struct.Metadata.html +/// [`fs::Metadata`]: crate::fs::Metadata #[stable(feature = "metadata_ext", since = "1.1.0")] pub trait MetadataExt { /// Gain a reference to the underlying `stat` structure which contains @@ -18,10 +18,10 @@ pub trait MetadataExt { /// Unix platforms. The `os::unix::fs::MetadataExt` trait contains the /// cross-Unix abstractions contained within the raw stat. #[stable(feature = "metadata_ext", since = "1.1.0")] - #[rustc_deprecated( + #[deprecated( since = "1.8.0", - reason = "deprecated in favor of the accessor \ - methods of this trait" + note = "deprecated in favor of the accessor \ + methods of this trait" )] #[allow(deprecated)] fn as_raw_stat(&self) -> &raw::stat; diff --git a/crux-mir/lib/std/src/os/openbsd/raw.rs b/crux-mir/lib/std/src/os/openbsd/raw.rs index 8e34e5483..6711fb51b 100644 --- a/crux-mir/lib/std/src/os/openbsd/raw.rs +++ b/crux-mir/lib/std/src/os/openbsd/raw.rs @@ -1,12 +1,12 @@ //! OpenBSD-specific raw type definitions #![stable(feature = "raw_ext", since = "1.1.0")] -#![rustc_deprecated( +#![deprecated( since = "1.8.0", - reason = "these type aliases are no longer supported by \ - the standard library, the `libc` crate on \ - crates.io should be used instead for the correct \ - definitions" + note = "these type aliases are no longer supported by \ + the standard library, the `libc` crate on \ + crates.io should be used instead for the correct \ + definitions" )] #![allow(deprecated)] diff --git a/crux-mir/lib/std/src/os/raw/mod.rs b/crux-mir/lib/std/src/os/raw/mod.rs index 47daf0cce..19d0ffb2e 100644 --- a/crux-mir/lib/std/src/os/raw/mod.rs +++ b/crux-mir/lib/std/src/os/raw/mod.rs @@ -1,167 +1,31 @@ -//! Platform-specific types, as defined by C. -//! -//! Code that interacts via FFI will almost certainly be using the -//! base types provided by C, which aren't nearly as nicely defined -//! as Rust's primitive types. This module provides types which will -//! match those defined by C, so that code that interacts with C will -//! refer to the correct types. +//! Compatibility module for C platform-specific types. Use [`core::ffi`] instead. #![stable(feature = "raw_os", since = "1.1.0")] -#[doc(include = "char.md")] -#[cfg(any( - all( - target_os = "linux", - any( - target_arch = "aarch64", - target_arch = "arm", - target_arch = "hexagon", - target_arch = "powerpc", - target_arch = "powerpc64", - target_arch = "s390x", - target_arch = "riscv64" - ) - ), - all(target_os = "android", any(target_arch = "aarch64", target_arch = "arm")), - all(target_os = "l4re", target_arch = "x86_64"), - all( - target_os = "freebsd", - any( - target_arch = "aarch64", - target_arch = "arm", - target_arch = "powerpc", - target_arch = "powerpc64" - ) - ), - all( - target_os = "netbsd", - any(target_arch = "aarch64", target_arch = "arm", target_arch = "powerpc") - ), - all(target_os = "openbsd", target_arch = "aarch64"), - all( - target_os = "vxworks", - any( - target_arch = "aarch64", - target_arch = "arm", - target_arch = "powerpc64", - target_arch = "powerpc" - ) - ), - all(target_os = "fuchsia", target_arch = "aarch64") -))] -#[stable(feature = "raw_os", since = "1.1.0")] -pub type c_char = u8; -#[doc(include = "char.md")] -#[cfg(not(any( - all( - target_os = "linux", - any( - target_arch = "aarch64", - target_arch = "arm", - target_arch = "hexagon", - target_arch = "powerpc", - target_arch = "powerpc64", - target_arch = "s390x", - target_arch = "riscv64" - ) - ), - all(target_os = "android", any(target_arch = "aarch64", target_arch = "arm")), - all(target_os = "l4re", target_arch = "x86_64"), - all( - target_os = "freebsd", - any( - target_arch = "aarch64", - target_arch = "arm", - target_arch = "powerpc", - target_arch = "powerpc64" - ) - ), - all( - target_os = "netbsd", - any(target_arch = "aarch64", target_arch = "arm", target_arch = "powerpc") - ), - all(target_os = "openbsd", target_arch = "aarch64"), - all( - target_os = "vxworks", - any( - target_arch = "aarch64", - target_arch = "arm", - target_arch = "powerpc64", - target_arch = "powerpc" - ) - ), - all(target_os = "fuchsia", target_arch = "aarch64") -)))] -#[stable(feature = "raw_os", since = "1.1.0")] -pub type c_char = i8; -#[doc(include = "schar.md")] -#[stable(feature = "raw_os", since = "1.1.0")] -pub type c_schar = i8; -#[doc(include = "uchar.md")] -#[stable(feature = "raw_os", since = "1.1.0")] -pub type c_uchar = u8; -#[doc(include = "short.md")] -#[stable(feature = "raw_os", since = "1.1.0")] -pub type c_short = i16; -#[doc(include = "ushort.md")] -#[stable(feature = "raw_os", since = "1.1.0")] -pub type c_ushort = u16; -#[doc(include = "int.md")] -#[stable(feature = "raw_os", since = "1.1.0")] -pub type c_int = i32; -#[doc(include = "uint.md")] -#[stable(feature = "raw_os", since = "1.1.0")] -pub type c_uint = u32; -#[doc(include = "long.md")] -#[cfg(any(target_pointer_width = "32", windows))] -#[stable(feature = "raw_os", since = "1.1.0")] -pub type c_long = i32; -#[doc(include = "ulong.md")] -#[cfg(any(target_pointer_width = "32", windows))] -#[stable(feature = "raw_os", since = "1.1.0")] -pub type c_ulong = u32; -#[doc(include = "long.md")] -#[cfg(all(target_pointer_width = "64", not(windows)))] -#[stable(feature = "raw_os", since = "1.1.0")] -pub type c_long = i64; -#[doc(include = "ulong.md")] -#[cfg(all(target_pointer_width = "64", not(windows)))] -#[stable(feature = "raw_os", since = "1.1.0")] -pub type c_ulong = u64; -#[doc(include = "longlong.md")] -#[stable(feature = "raw_os", since = "1.1.0")] -pub type c_longlong = i64; -#[doc(include = "ulonglong.md")] -#[stable(feature = "raw_os", since = "1.1.0")] -pub type c_ulonglong = u64; -#[doc(include = "float.md")] -#[stable(feature = "raw_os", since = "1.1.0")] -pub type c_float = f32; -#[doc(include = "double.md")] -#[stable(feature = "raw_os", since = "1.1.0")] -pub type c_double = f64; - -#[stable(feature = "raw_os", since = "1.1.0")] -#[doc(no_inline)] -pub use core::ffi::c_void; - #[cfg(test)] -#[allow(unused_imports)] -mod tests { - use crate::any::TypeId; - use crate::mem; +mod tests; - macro_rules! ok { - ($($t:ident)*) => {$( - assert!(TypeId::of::() == TypeId::of::(), - "{} is wrong", stringify!($t)); - )*} - } +macro_rules! alias_core_ffi { + ($($t:ident)*) => {$( + #[stable(feature = "raw_os", since = "1.1.0")] + #[doc = include_str!(concat!("../../../../core/src/ffi/", stringify!($t), ".md"))] + // Make this type alias appear cfg-dependent so that Clippy does not suggest + // replacing expressions like `0 as c_char` with `0_i8`/`0_u8`. This #[cfg(all())] can be + // removed after the false positive in https://github.com/rust-lang/rust-clippy/issues/8093 + // is fixed. + #[cfg(all())] + #[doc(cfg(all()))] + pub type $t = core::ffi::$t; + )*} +} - #[test] - fn same() { - use crate::os::raw; - ok!(c_char c_schar c_uchar c_short c_ushort c_int c_uint c_long c_ulong - c_longlong c_ulonglong c_float c_double); - } +alias_core_ffi! { + c_char c_schar c_uchar + c_short c_ushort + c_int c_uint + c_long c_ulong + c_longlong c_ulonglong + c_float + c_double + c_void } diff --git a/crux-mir/lib/std/src/os/raw/tests.rs b/crux-mir/lib/std/src/os/raw/tests.rs new file mode 100644 index 000000000..e7bb7d7e7 --- /dev/null +++ b/crux-mir/lib/std/src/os/raw/tests.rs @@ -0,0 +1,15 @@ +use crate::any::TypeId; + +macro_rules! ok { + ($($t:ident)*) => {$( + assert!(TypeId::of::() == TypeId::of::(), + "{} is wrong", stringify!($t)); + )*} +} + +#[test] +fn same() { + use crate::os::raw; + ok!(c_char c_schar c_uchar c_short c_ushort c_int c_uint c_long c_ulong + c_longlong c_ulonglong c_float c_double); +} diff --git a/crux-mir/lib/std/src/os/redox/fs.rs b/crux-mir/lib/std/src/os/redox/fs.rs index 6c87df534..682ca6a2c 100644 --- a/crux-mir/lib/std/src/os/redox/fs.rs +++ b/crux-mir/lib/std/src/os/redox/fs.rs @@ -8,7 +8,7 @@ use crate::os::redox::raw; /// OS-specific extensions to [`fs::Metadata`]. /// -/// [`fs::Metadata`]: ../../../../std/fs/struct.Metadata.html +/// [`fs::Metadata`]: crate::fs::Metadata #[stable(feature = "metadata_ext", since = "1.1.0")] pub trait MetadataExt { /// Gain a reference to the underlying `stat` structure which contains @@ -18,7 +18,7 @@ pub trait MetadataExt { /// Unix platforms. The `os::unix::fs::MetadataExt` trait contains the /// cross-Unix abstractions contained within the raw stat. /// - /// [`stat`]: ../../../../std/os/redox/raw/struct.stat.html + /// [`stat`]: crate::os::redox::raw::stat /// /// # Examples /// @@ -34,10 +34,10 @@ pub trait MetadataExt { /// } /// ``` #[stable(feature = "metadata_ext", since = "1.1.0")] - #[rustc_deprecated( + #[deprecated( since = "1.8.0", - reason = "deprecated in favor of the accessor \ - methods of this trait" + note = "deprecated in favor of the accessor \ + methods of this trait" )] #[allow(deprecated)] fn as_raw_stat(&self) -> &raw::stat; @@ -200,7 +200,7 @@ pub trait MetadataExt { fn st_atime(&self) -> i64; /// Returns the last access time of the file, in nanoseconds since [`st_atime`]. /// - /// [`st_atime`]: #tymethod.st_atime + /// [`st_atime`]: Self::st_atime /// /// # Examples /// @@ -236,7 +236,7 @@ pub trait MetadataExt { fn st_mtime(&self) -> i64; /// Returns the last modification time of the file, in nanoseconds since [`st_mtime`]. /// - /// [`st_mtime`]: #tymethod.st_mtime + /// [`st_mtime`]: Self::st_mtime /// /// # Examples /// @@ -272,7 +272,7 @@ pub trait MetadataExt { fn st_ctime(&self) -> i64; /// Returns the last status change time of the file, in nanoseconds since [`st_ctime`]. /// - /// [`st_ctime`]: #tymethod.st_ctime + /// [`st_ctime`]: Self::st_ctime /// /// # Examples /// @@ -289,7 +289,7 @@ pub trait MetadataExt { /// ``` #[stable(feature = "metadata_ext2", since = "1.8.0")] fn st_ctime_nsec(&self) -> i64; - /// Returns the "preferred" blocksize for efficient filesystem I/O. + /// Returns the "preferred" block size for efficient filesystem I/O. /// /// # Examples /// diff --git a/crux-mir/lib/std/src/os/redox/raw.rs b/crux-mir/lib/std/src/os/redox/raw.rs index abe6dfc6b..7b1cd8ae8 100644 --- a/crux-mir/lib/std/src/os/redox/raw.rs +++ b/crux-mir/lib/std/src/os/redox/raw.rs @@ -1,15 +1,14 @@ //! Redox-specific raw type definitions #![stable(feature = "raw_ext", since = "1.1.0")] -#![rustc_deprecated( +#![deprecated( since = "1.8.0", - reason = "these type aliases are no longer supported by \ - the standard library, the `libc` crate on \ - crates.io should be used instead for the correct \ - definitions" + note = "these type aliases are no longer supported by \ + the standard library, the `libc` crate on \ + crates.io should be used instead for the correct \ + definitions" )] #![allow(deprecated)] -#![allow(missing_debug_implementations)] use crate::os::raw::{c_char, c_int, c_long, c_ulong, c_void}; diff --git a/crux-mir/lib/std/src/os/solaris/fs.rs b/crux-mir/lib/std/src/os/solaris/fs.rs index 549d3d756..093143737 100644 --- a/crux-mir/lib/std/src/os/solaris/fs.rs +++ b/crux-mir/lib/std/src/os/solaris/fs.rs @@ -8,7 +8,7 @@ use crate::os::solaris::raw; /// OS-specific extensions to [`fs::Metadata`]. /// -/// [`fs::Metadata`]: ../../../../std/fs/struct.Metadata.html +/// [`fs::Metadata`]: crate::fs::Metadata #[stable(feature = "metadata_ext", since = "1.1.0")] pub trait MetadataExt { /// Gain a reference to the underlying `stat` structure which contains @@ -18,10 +18,10 @@ pub trait MetadataExt { /// Unix platforms. The `os::unix::fs::MetadataExt` trait contains the /// cross-Unix abstractions contained within the raw stat. #[stable(feature = "metadata_ext", since = "1.1.0")] - #[rustc_deprecated( + #[deprecated( since = "1.8.0", - reason = "deprecated in favor of the accessor \ - methods of this trait" + note = "deprecated in favor of the accessor \ + methods of this trait" )] #[allow(deprecated)] fn as_raw_stat(&self) -> &raw::stat; diff --git a/crux-mir/lib/std/src/os/solaris/raw.rs b/crux-mir/lib/std/src/os/solaris/raw.rs index e78f9992b..63426c969 100644 --- a/crux-mir/lib/std/src/os/solaris/raw.rs +++ b/crux-mir/lib/std/src/os/solaris/raw.rs @@ -1,12 +1,12 @@ //! Solaris-specific raw type definitions #![stable(feature = "raw_ext", since = "1.1.0")] -#![rustc_deprecated( +#![deprecated( since = "1.8.0", - reason = "these type aliases are no longer supported by \ - the standard library, the `libc` crate on \ - crates.io should be used instead for the correct \ - definitions" + note = "these type aliases are no longer supported by \ + the standard library, the `libc` crate on \ + crates.io should be used instead for the correct \ + definitions" )] #![allow(deprecated)] diff --git a/crux-mir/lib/std/src/sys/vxworks/ext/ffi.rs b/crux-mir/lib/std/src/os/solid/ffi.rs similarity index 74% rename from crux-mir/lib/std/src/sys/vxworks/ext/ffi.rs rename to crux-mir/lib/std/src/os/solid/ffi.rs index 76b34a6b5..aaa2070a6 100644 --- a/crux-mir/lib/std/src/sys/vxworks/ext/ffi.rs +++ b/crux-mir/lib/std/src/os/solid/ffi.rs @@ -1,10 +1,10 @@ -//! Unix-specific extension to the primitives in the `std::ffi` module +//! SOLID-specific extension to the primitives in the `std::ffi` module //! //! # Examples //! //! ``` //! use std::ffi::OsString; -//! use std::os::unix::ffi::OsStringExt; +//! use std::os::solid::ffi::OsStringExt; //! //! let bytes = b"foo".to_vec(); //! @@ -19,7 +19,7 @@ //! //! ``` //! use std::ffi::OsStr; -//! use std::os::unix::ffi::OsStrExt; +//! use std::os::solid::ffi::OsStrExt; //! //! let bytes = b"foo"; //! @@ -34,5 +34,8 @@ #![stable(feature = "rust1", since = "1.0.0")] +#[path = "../unix/ffi/os_str.rs"] +mod os_str; + #[stable(feature = "rust1", since = "1.0.0")] -pub use crate::sys_common::os_str_bytes::*; +pub use self::os_str::{OsStrExt, OsStringExt}; diff --git a/crux-mir/lib/std/src/os/solid/io.rs b/crux-mir/lib/std/src/os/solid/io.rs new file mode 100644 index 000000000..33cc5a015 --- /dev/null +++ b/crux-mir/lib/std/src/os/solid/io.rs @@ -0,0 +1,113 @@ +//! SOLID-specific extensions to general I/O primitives + +#![deny(unsafe_op_in_unsafe_fn)] +#![unstable(feature = "solid_ext", issue = "none")] + +use crate::net; +use crate::sys; +use crate::sys_common::{self, AsInner, FromInner, IntoInner}; + +/// Raw file descriptors. +pub type RawFd = i32; + +/// A trait to extract the raw SOLID Sockets file descriptor from an underlying +/// object. +pub trait AsRawFd { + /// Extracts the raw file descriptor. + /// + /// This method does **not** pass ownership of the raw file descriptor + /// to the caller. The descriptor is only guaranteed to be valid while + /// the original object has not yet been destroyed. + fn as_raw_fd(&self) -> RawFd; +} + +/// A trait to express the ability to construct an object from a raw file +/// descriptor. +pub trait FromRawFd { + /// Constructs a new instance of `Self` from the given raw file + /// descriptor. + /// + /// This function **consumes ownership** of the specified file + /// descriptor. The returned object will take responsibility for closing + /// it when the object goes out of scope. + /// + /// This function is also unsafe as the primitives currently returned + /// have the contract that they are the sole owner of the file + /// descriptor they are wrapping. Usage of this function could + /// accidentally allow violating this contract which can cause memory + /// unsafety in code that relies on it being true. + unsafe fn from_raw_fd(fd: RawFd) -> Self; +} + +/// A trait to express the ability to consume an object and acquire ownership of +/// its raw file descriptor. +pub trait IntoRawFd { + /// Consumes this object, returning the raw underlying file descriptor. + /// + /// This function **transfers ownership** of the underlying file descriptor + /// to the caller. Callers are then the unique owners of the file descriptor + /// and must close the descriptor once it's no longer needed. + fn into_raw_fd(self) -> RawFd; +} + +#[stable(feature = "raw_fd_reflexive_traits", since = "1.48.0")] +impl AsRawFd for RawFd { + #[inline] + fn as_raw_fd(&self) -> RawFd { + *self + } +} +#[stable(feature = "raw_fd_reflexive_traits", since = "1.48.0")] +impl IntoRawFd for RawFd { + #[inline] + fn into_raw_fd(self) -> RawFd { + self + } +} +#[stable(feature = "raw_fd_reflexive_traits", since = "1.48.0")] +impl FromRawFd for RawFd { + #[inline] + unsafe fn from_raw_fd(fd: RawFd) -> RawFd { + fd + } +} + +macro_rules! impl_as_raw_fd { + ($($t:ident)*) => {$( + #[stable(feature = "rust1", since = "1.0.0")] + impl AsRawFd for net::$t { + #[inline] + fn as_raw_fd(&self) -> RawFd { + *self.as_inner().socket().as_inner() + } + } + )*}; +} +impl_as_raw_fd! { TcpStream TcpListener UdpSocket } + +macro_rules! impl_from_raw_fd { + ($($t:ident)*) => {$( + #[stable(feature = "from_raw_os", since = "1.1.0")] + impl FromRawFd for net::$t { + #[inline] + unsafe fn from_raw_fd(fd: RawFd) -> net::$t { + let socket = sys::net::Socket::from_inner(fd); + net::$t::from_inner(sys_common::net::$t::from_inner(socket)) + } + } + )*}; +} +impl_from_raw_fd! { TcpStream TcpListener UdpSocket } + +macro_rules! impl_into_raw_fd { + ($($t:ident)*) => {$( + #[stable(feature = "into_raw_os", since = "1.4.0")] + impl IntoRawFd for net::$t { + #[inline] + fn into_raw_fd(self) -> RawFd { + self.into_inner().into_socket().into_inner() + } + } + )*}; +} +impl_into_raw_fd! { TcpStream TcpListener UdpSocket } diff --git a/crux-mir/lib/std/src/sys/vxworks/ext/mod.rs b/crux-mir/lib/std/src/os/solid/mod.rs similarity index 54% rename from crux-mir/lib/std/src/sys/vxworks/ext/mod.rs rename to crux-mir/lib/std/src/os/solid/mod.rs index 8fa9bd9d1..4328ba7c3 100644 --- a/crux-mir/lib/std/src/sys/vxworks/ext/mod.rs +++ b/crux-mir/lib/std/src/os/solid/mod.rs @@ -1,12 +1,11 @@ #![stable(feature = "rust1", since = "1.0.0")] -#![allow(missing_docs)] pub mod ffi; -pub mod fs; pub mod io; -pub mod process; -pub mod raw; +/// A prelude for conveniently writing platform-specific code. +/// +/// Includes all extension traits, and some important type definitions. #[stable(feature = "rust1", since = "1.0.0")] pub mod prelude { #[doc(no_inline)] @@ -14,11 +13,5 @@ pub mod prelude { pub use super::ffi::{OsStrExt, OsStringExt}; #[doc(no_inline)] #[stable(feature = "rust1", since = "1.0.0")] - pub use super::fs::{FileTypeExt, MetadataExt, OpenOptionsExt, PermissionsExt}; - #[doc(no_inline)] - #[stable(feature = "rust1", since = "1.0.0")] pub use super::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd}; - #[doc(no_inline)] - #[stable(feature = "rust1", since = "1.0.0")] - pub use super::process::ExitStatusExt; } diff --git a/crux-mir/lib/std/src/sys/unix/ext/ffi.rs b/crux-mir/lib/std/src/os/unix/ffi/mod.rs similarity index 82% rename from crux-mir/lib/std/src/sys/unix/ext/ffi.rs rename to crux-mir/lib/std/src/os/unix/ffi/mod.rs index 76b34a6b5..5b49f5076 100644 --- a/crux-mir/lib/std/src/sys/unix/ext/ffi.rs +++ b/crux-mir/lib/std/src/os/unix/ffi/mod.rs @@ -1,4 +1,4 @@ -//! Unix-specific extension to the primitives in the `std::ffi` module +//! Unix-specific extensions to primitives in the [`std::ffi`] module. //! //! # Examples //! @@ -31,8 +31,12 @@ //! let bytes = os_str.as_bytes(); //! assert_eq!(bytes, b"foo"); //! ``` +//! +//! [`std::ffi`]: crate::ffi #![stable(feature = "rust1", since = "1.0.0")] +mod os_str; + #[stable(feature = "rust1", since = "1.0.0")] -pub use crate::sys_common::os_str_bytes::*; +pub use self::os_str::{OsStrExt, OsStringExt}; diff --git a/crux-mir/lib/std/src/os/unix/ffi/os_str.rs b/crux-mir/lib/std/src/os/unix/ffi/os_str.rs new file mode 100644 index 000000000..650f712bc --- /dev/null +++ b/crux-mir/lib/std/src/os/unix/ffi/os_str.rs @@ -0,0 +1,70 @@ +use crate::ffi::{OsStr, OsString}; +use crate::mem; +use crate::sealed::Sealed; +use crate::sys::os_str::Buf; +use crate::sys_common::{AsInner, FromInner, IntoInner}; + +// Note: this file is currently reused in other `std::os::{platform}::ffi` modules to reduce duplication. +// Keep this in mind when applying changes to this file that only apply to `unix`. + +/// Platform-specific extensions to [`OsString`]. +/// +/// This trait is sealed: it cannot be implemented outside the standard library. +/// This is so that future additional methods are not breaking changes. +#[stable(feature = "rust1", since = "1.0.0")] +pub trait OsStringExt: Sealed { + /// Creates an [`OsString`] from a byte vector. + /// + /// See the module documentation for an example. + #[stable(feature = "rust1", since = "1.0.0")] + fn from_vec(vec: Vec) -> Self; + + /// Yields the underlying byte vector of this [`OsString`]. + /// + /// See the module documentation for an example. + #[stable(feature = "rust1", since = "1.0.0")] + fn into_vec(self) -> Vec; +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl OsStringExt for OsString { + #[inline] + fn from_vec(vec: Vec) -> OsString { + FromInner::from_inner(Buf { inner: vec }) + } + #[inline] + fn into_vec(self) -> Vec { + self.into_inner().inner + } +} + +/// Platform-specific extensions to [`OsStr`]. +/// +/// This trait is sealed: it cannot be implemented outside the standard library. +/// This is so that future additional methods are not breaking changes. +#[stable(feature = "rust1", since = "1.0.0")] +pub trait OsStrExt: Sealed { + #[stable(feature = "rust1", since = "1.0.0")] + /// Creates an [`OsStr`] from a byte slice. + /// + /// See the module documentation for an example. + fn from_bytes(slice: &[u8]) -> &Self; + + /// Gets the underlying byte view of the [`OsStr`] slice. + /// + /// See the module documentation for an example. + #[stable(feature = "rust1", since = "1.0.0")] + fn as_bytes(&self) -> &[u8]; +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl OsStrExt for OsStr { + #[inline] + fn from_bytes(slice: &[u8]) -> &OsStr { + unsafe { mem::transmute(slice) } + } + #[inline] + fn as_bytes(&self) -> &[u8] { + &self.as_inner().inner + } +} diff --git a/crux-mir/lib/std/src/sys/unix/ext/fs.rs b/crux-mir/lib/std/src/os/unix/fs.rs similarity index 80% rename from crux-mir/lib/std/src/sys/unix/ext/fs.rs rename to crux-mir/lib/std/src/os/unix/fs.rs index 732cd677a..3fc6cc44c 100644 --- a/crux-mir/lib/std/src/sys/unix/ext/fs.rs +++ b/crux-mir/lib/std/src/os/unix/fs.rs @@ -1,17 +1,23 @@ -//! Unix-specific extensions to primitives in the `std::fs` module. +//! Unix-specific extensions to primitives in the [`std::fs`] module. +//! +//! [`std::fs`]: crate::fs #![stable(feature = "rust1", since = "1.0.0")] +use super::platform::fs::MetadataExt as _; use crate::fs::{self, OpenOptions, Permissions}; use crate::io; +use crate::os::unix::io::{AsFd, AsRawFd}; use crate::path::Path; use crate::sys; -use crate::sys::platform::fs::MetadataExt as UnixMetadataExt; use crate::sys_common::{AsInner, AsInnerMut, FromInner}; +// Used for `File::read` on intra-doc links +use crate::ffi::OsStr; +use crate::sealed::Sealed; +#[allow(unused_imports)] +use io::{Read, Write}; -/// Unix-specific extensions to [`File`]. -/// -/// [`File`]: ../../../../std/fs/struct.File.html +/// Unix-specific extensions to [`fs::File`]. #[stable(feature = "file_offset", since = "1.15.0")] pub trait FileExt { /// Reads a number of bytes starting from a given offset. @@ -26,7 +32,7 @@ pub trait FileExt { /// Note that similar to [`File::read`], it is not an error to return with a /// short read. /// - /// [`File::read`]: ../../../../std/fs/struct.File.html#method.read + /// [`File::read`]: fs::File::read /// /// # Examples /// @@ -41,7 +47,7 @@ pub trait FileExt { /// /// // We now read 8 bytes from the offset 10. /// let num_bytes_read = file.read_at(&mut buf, 10)?; - /// println!("read {} bytes: {:?}", num_bytes_read, buf); + /// println!("read {num_bytes_read} bytes: {buf:?}"); /// Ok(()) /// } /// ``` @@ -55,19 +61,18 @@ pub trait FileExt { /// /// The current file cursor is not affected by this function. /// - /// Similar to [`Read::read_exact`] but uses [`read_at`] instead of `read`. + /// Similar to [`io::Read::read_exact`] but uses [`read_at`] instead of `read`. /// - /// [`Read::read_exact`]: ../../../../std/io/trait.Read.html#method.read_exact - /// [`read_at`]: #tymethod.read_at + /// [`read_at`]: FileExt::read_at /// /// # Errors /// /// If this function encounters an error of the kind - /// [`ErrorKind::Interrupted`] then the error is ignored and the operation + /// [`io::ErrorKind::Interrupted`] then the error is ignored and the operation /// will continue. /// /// If this function encounters an "end of file" before completely filling - /// the buffer, it returns an error of the kind [`ErrorKind::UnexpectedEof`]. + /// the buffer, it returns an error of the kind [`io::ErrorKind::UnexpectedEof`]. /// The contents of `buf` are unspecified in this case. /// /// If any other read error is encountered then this function immediately @@ -77,9 +82,6 @@ pub trait FileExt { /// has read, but it will never read more than would be necessary to /// completely fill the buffer. /// - /// [`ErrorKind::Interrupted`]: ../../../../std/io/enum.ErrorKind.html#variant.Interrupted - /// [`ErrorKind::UnexpectedEof`]: ../../../../std/io/enum.ErrorKind.html#variant.UnexpectedEof - /// /// # Examples /// /// ```no_run @@ -112,7 +114,7 @@ pub trait FileExt { } } if !buf.is_empty() { - Err(io::Error::new(io::ErrorKind::UnexpectedEof, "failed to fill whole buffer")) + Err(io::const_io_error!(io::ErrorKind::UnexpectedEof, "failed to fill whole buffer",)) } else { Ok(()) } @@ -133,7 +135,7 @@ pub trait FileExt { /// Note that similar to [`File::write`], it is not an error to return a /// short write. /// - /// [`File::write`]: ../../../../std/fs/struct.File.html#write.v + /// [`File::write`]: fs::File::write /// /// # Examples /// @@ -161,19 +163,18 @@ pub trait FileExt { /// The current file cursor is not affected by this function. /// /// This method will continuously call [`write_at`] until there is no more data - /// to be written or an error of non-[`ErrorKind::Interrupted`] kind is + /// to be written or an error of non-[`io::ErrorKind::Interrupted`] kind is /// returned. This method will not return until the entire buffer has been /// successfully written or such an error occurs. The first error that is - /// not of [`ErrorKind::Interrupted`] kind generated from this method will be + /// not of [`io::ErrorKind::Interrupted`] kind generated from this method will be /// returned. /// /// # Errors /// /// This function will return the first error of - /// non-[`ErrorKind::Interrupted`] kind that [`write_at`] returns. + /// non-[`io::ErrorKind::Interrupted`] kind that [`write_at`] returns. /// - /// [`ErrorKind::Interrupted`]: ../../../../std/io/enum.ErrorKind.html#variant.Interrupted - /// [`write_at`]: #tymethod.write_at + /// [`write_at`]: FileExt::write_at /// /// # Examples /// @@ -195,7 +196,7 @@ pub trait FileExt { while !buf.is_empty() { match self.write_at(buf, offset) { Ok(0) => { - return Err(io::Error::new( + return Err(io::const_io_error!( io::ErrorKind::WriteZero, "failed to write whole buffer", )); @@ -223,8 +224,6 @@ impl FileExt for fs::File { } /// Unix-specific extensions to [`fs::Permissions`]. -/// -/// [`fs::Permissions`]: ../../../../std/fs/struct.Permissions.html #[stable(feature = "fs_ext", since = "1.1.0")] pub trait PermissionsExt { /// Returns the underlying raw `st_mode` bits that contain the standard @@ -242,7 +241,8 @@ pub trait PermissionsExt { /// let permissions = metadata.permissions(); /// /// println!("permissions: {:o}", permissions.mode()); - /// Ok(()) } + /// Ok(()) + /// } /// ``` #[stable(feature = "fs_ext", since = "1.1.0")] fn mode(&self) -> u32; @@ -262,7 +262,8 @@ pub trait PermissionsExt { /// /// permissions.set_mode(0o644); // Read/write for owner and read for others. /// assert_eq!(permissions.mode(), 0o644); - /// Ok(()) } + /// Ok(()) + /// } /// ``` #[stable(feature = "fs_ext", since = "1.1.0")] fn set_mode(&mut self, mode: u32); @@ -300,8 +301,6 @@ impl PermissionsExt for Permissions { } /// Unix-specific extensions to [`fs::OpenOptions`]. -/// -/// [`fs::OpenOptions`]: ../../../../std/fs/struct.OpenOptions.html #[stable(feature = "fs_ext", since = "1.1.0")] pub trait OpenOptionsExt { /// Sets the mode bits that a new file will be created with. @@ -370,8 +369,6 @@ impl OpenOptionsExt for OpenOptions { } /// Unix-specific extensions to [`fs::Metadata`]. -/// -/// [`fs::Metadata`]: ../../../../std/fs/struct.Metadata.html #[stable(feature = "metadata_ext", since = "1.1.0")] pub trait MetadataExt { /// Returns the ID of the device containing the file. @@ -436,7 +433,7 @@ pub trait MetadataExt { /// ```no_run /// use std::fs; /// use std::os::unix::fs::MetadataExt; - /// use std::io; + /// use std::io; /// /// fn main() -> io::Result<()> { /// let meta = fs::metadata("some_file")?; @@ -533,7 +530,7 @@ pub trait MetadataExt { fn atime(&self) -> i64; /// Returns the last access time of the file, in nanoseconds since [`atime`]. /// - /// [`atime`]: #tymethod.atime + /// [`atime`]: MetadataExt::atime /// /// # Examples /// @@ -569,7 +566,7 @@ pub trait MetadataExt { fn mtime(&self) -> i64; /// Returns the last modification time of the file, in nanoseconds since [`mtime`]. /// - /// [`mtime`]: #tymethod.mtime + /// [`mtime`]: MetadataExt::mtime /// /// # Examples /// @@ -605,7 +602,7 @@ pub trait MetadataExt { fn ctime(&self) -> i64; /// Returns the last status change time of the file, in nanoseconds since [`ctime`]. /// - /// [`ctime`]: #tymethod.ctime + /// [`ctime`]: MetadataExt::ctime /// /// # Examples /// @@ -622,7 +619,7 @@ pub trait MetadataExt { /// ``` #[stable(feature = "metadata_ext", since = "1.1.0")] fn ctime_nsec(&self) -> i64; - /// Returns the blocksize for filesystem I/O. + /// Returns the block size for filesystem I/O. /// /// # Examples /// @@ -633,7 +630,7 @@ pub trait MetadataExt { /// /// fn main() -> io::Result<()> { /// let meta = fs::metadata("some_file")?; - /// let blocksize = meta.blksize(); + /// let block_size = meta.blksize(); /// Ok(()) /// } /// ``` @@ -658,6 +655,9 @@ pub trait MetadataExt { /// ``` #[stable(feature = "metadata_ext", since = "1.1.0")] fn blocks(&self) -> u64; + #[cfg(target_os = "vxworks")] + #[stable(feature = "metadata_ext", since = "1.1.0")] + fn attrib(&self) -> u8; } #[stable(feature = "metadata_ext", since = "1.1.0")] @@ -710,14 +710,16 @@ impl MetadataExt for fs::Metadata { fn blocks(&self) -> u64 { self.st_blocks() } + #[cfg(target_os = "vxworks")] + fn attrib(&self) -> u8 { + self.st_attrib() + } } -/// Unix-specific extensions for [`FileType`]. +/// Unix-specific extensions for [`fs::FileType`]. /// /// Adds support for special Unix file types such as block/character devices, /// pipes, and sockets. -/// -/// [`FileType`]: ../../../../std/fs/struct.FileType.html #[stable(feature = "file_type_ext", since = "1.5.0")] pub trait FileTypeExt { /// Returns `true` if this file type is a block device. @@ -811,8 +813,6 @@ impl FileTypeExt for fs::FileType { } /// Unix-specific extension methods for [`fs::DirEntry`]. -/// -/// [`fs::DirEntry`]: ../../../../std/fs/struct.DirEntry.html #[stable(feature = "dir_entry_ext", since = "1.1.0")] pub trait DirEntryExt { /// Returns the underlying `d_ino` field in the contained `dirent` @@ -844,18 +844,46 @@ impl DirEntryExt for fs::DirEntry { } } +/// Sealed Unix-specific extension methods for [`fs::DirEntry`]. +#[unstable(feature = "dir_entry_ext2", issue = "85573")] +pub trait DirEntryExt2: Sealed { + /// Returns a reference to the underlying `OsStr` of this entry's filename. + /// + /// # Examples + /// + /// ``` + /// #![feature(dir_entry_ext2)] + /// use std::os::unix::fs::DirEntryExt2; + /// use std::{fs, io}; + /// + /// fn main() -> io::Result<()> { + /// let mut entries = fs::read_dir(".")?.collect::, io::Error>>()?; + /// entries.sort_unstable_by(|a, b| a.file_name_ref().cmp(b.file_name_ref())); + /// + /// for p in entries { + /// println!("{p:?}"); + /// } + /// + /// Ok(()) + /// } + /// ``` + fn file_name_ref(&self) -> &OsStr; +} + +/// Allows extension traits within `std`. +#[unstable(feature = "sealed", issue = "none")] +impl Sealed for fs::DirEntry {} + +#[unstable(feature = "dir_entry_ext2", issue = "85573")] +impl DirEntryExt2 for fs::DirEntry { + fn file_name_ref(&self) -> &OsStr { + self.as_inner().file_name_os_str() + } +} + /// Creates a new symbolic link on the filesystem. /// -/// The `dst` path will be a symbolic link pointing to the `src` path. -/// -/// # Note -/// -/// On Windows, you must specify whether a symbolic link points to a file -/// or directory. Use `os::windows::fs::symlink_file` to create a -/// symbolic link to a file, or `os::windows::fs::symlink_dir` to create a -/// symbolic link to a directory. Additionally, the process must have -/// `SeCreateSymbolicLinkPrivilege` in order to be able to create a -/// symbolic link. +/// The `link` path will be a symbolic link pointing to the `original` path. /// /// # Examples /// @@ -868,13 +896,11 @@ impl DirEntryExt for fs::DirEntry { /// } /// ``` #[stable(feature = "symlink", since = "1.1.0")] -pub fn symlink, Q: AsRef>(src: P, dst: Q) -> io::Result<()> { - sys::fs::symlink(src.as_ref(), dst.as_ref()) +pub fn symlink, Q: AsRef>(original: P, link: Q) -> io::Result<()> { + sys::fs::symlink(original.as_ref(), link.as_ref()) } /// Unix-specific extensions to [`fs::DirBuilder`]. -/// -/// [`fs::DirBuilder`]: ../../../../std/fs/struct.DirBuilder.html #[stable(feature = "dir_builder", since = "1.6.0")] pub trait DirBuilderExt { /// Sets the mode to create new directories with. This option defaults to @@ -900,3 +926,97 @@ impl DirBuilderExt for fs::DirBuilder { self } } + +/// Change the owner and group of the specified path. +/// +/// Specifying either the uid or gid as `None` will leave it unchanged. +/// +/// Changing the owner typically requires privileges, such as root or a specific capability. +/// Changing the group typically requires either being the owner and a member of the group, or +/// having privileges. +/// +/// If called on a symbolic link, this will change the owner and group of the link target. To +/// change the owner and group of the link itself, see [`lchown`]. +/// +/// # Examples +/// +/// ```no_run +/// #![feature(unix_chown)] +/// use std::os::unix::fs; +/// +/// fn main() -> std::io::Result<()> { +/// fs::chown("/sandbox", Some(0), Some(0))?; +/// Ok(()) +/// } +/// ``` +#[unstable(feature = "unix_chown", issue = "88989")] +pub fn chown>(dir: P, uid: Option, gid: Option) -> io::Result<()> { + sys::fs::chown(dir.as_ref(), uid.unwrap_or(u32::MAX), gid.unwrap_or(u32::MAX)) +} + +/// Change the owner and group of the file referenced by the specified open file descriptor. +/// +/// For semantics and required privileges, see [`chown`]. +/// +/// # Examples +/// +/// ```no_run +/// #![feature(unix_chown)] +/// use std::os::unix::fs; +/// +/// fn main() -> std::io::Result<()> { +/// let f = std::fs::File::open("/file")?; +/// fs::fchown(&f, Some(0), Some(0))?; +/// Ok(()) +/// } +/// ``` +#[unstable(feature = "unix_chown", issue = "88989")] +pub fn fchown(fd: F, uid: Option, gid: Option) -> io::Result<()> { + sys::fs::fchown(fd.as_fd().as_raw_fd(), uid.unwrap_or(u32::MAX), gid.unwrap_or(u32::MAX)) +} + +/// Change the owner and group of the specified path, without dereferencing symbolic links. +/// +/// Identical to [`chown`], except that if called on a symbolic link, this will change the owner +/// and group of the link itself rather than the owner and group of the link target. +/// +/// # Examples +/// +/// ```no_run +/// #![feature(unix_chown)] +/// use std::os::unix::fs; +/// +/// fn main() -> std::io::Result<()> { +/// fs::lchown("/symlink", Some(0), Some(0))?; +/// Ok(()) +/// } +/// ``` +#[unstable(feature = "unix_chown", issue = "88989")] +pub fn lchown>(dir: P, uid: Option, gid: Option) -> io::Result<()> { + sys::fs::lchown(dir.as_ref(), uid.unwrap_or(u32::MAX), gid.unwrap_or(u32::MAX)) +} + +/// Change the root directory of the current process to the specified path. +/// +/// This typically requires privileges, such as root or a specific capability. +/// +/// This does not change the current working directory; you should call +/// [`std::env::set_current_dir`][`crate::env::set_current_dir`] afterwards. +/// +/// # Examples +/// +/// ```no_run +/// use std::os::unix::fs; +/// +/// fn main() -> std::io::Result<()> { +/// fs::chroot("/sandbox")?; +/// std::env::set_current_dir("/")?; +/// // continue working in sandbox +/// Ok(()) +/// } +/// ``` +#[stable(feature = "unix_chroot", since = "1.56.0")] +#[cfg(not(any(target_os = "fuchsia", target_os = "vxworks")))] +pub fn chroot>(dir: P) -> io::Result<()> { + sys::fs::chroot(dir.as_ref()) +} diff --git a/crux-mir/lib/std/src/os/unix/io/mod.rs b/crux-mir/lib/std/src/os/unix/io/mod.rs new file mode 100644 index 000000000..25b5dbff1 --- /dev/null +++ b/crux-mir/lib/std/src/os/unix/io/mod.rs @@ -0,0 +1,85 @@ +//! Unix-specific extensions to general I/O primitives. +//! +//! Just like raw pointers, raw file descriptors point to resources with +//! dynamic lifetimes, and they can dangle if they outlive their resources +//! or be forged if they're created from invalid values. +//! +//! This module provides three types for representing file descriptors, +//! with different ownership properties: raw, borrowed, and owned, which are +//! analogous to types used for representing pointers: +//! +//! | Type | Analogous to | +//! | ------------------ | ------------ | +//! | [`RawFd`] | `*const _` | +//! | [`BorrowedFd<'a>`] | `&'a _` | +//! | [`OwnedFd`] | `Box<_>` | +//! +//! Like raw pointers, `RawFd` values are primitive values. And in new code, +//! they should be considered unsafe to do I/O on (analogous to dereferencing +//! them). Rust did not always provide this guidance, so existing code in the +//! Rust ecosystem often doesn't mark `RawFd` usage as unsafe. Once the +//! `io_safety` feature is stable, libraries will be encouraged to migrate, +//! either by adding `unsafe` to APIs that dereference `RawFd` values, or by +//! using to `BorrowedFd` or `OwnedFd` instead. +//! +//! Like references, `BorrowedFd` values are tied to a lifetime, to ensure +//! that they don't outlive the resource they point to. These are safe to +//! use. `BorrowedFd` values may be used in APIs which provide safe access to +//! any system call except for: +//! +//! - `close`, because that would end the dynamic lifetime of the resource +//! without ending the lifetime of the file descriptor. +//! +//! - `dup2`/`dup3`, in the second argument, because this argument is +//! closed and assigned a new resource, which may break the assumptions +//! other code using that file descriptor. +//! +//! `BorrowedFd` values may be used in APIs which provide safe access to `dup` +//! system calls, so types implementing `AsFd` or `From` should not +//! assume they always have exclusive access to the underlying file +//! description. +//! +//! `BorrowedFd` values may also be used with `mmap`, since `mmap` uses the +//! provided file descriptor in a manner similar to `dup` and does not require +//! the `BorrowedFd` passed to it to live for the lifetime of the resulting +//! mapping. That said, `mmap` is unsafe for other reasons: it operates on raw +//! pointers, and it can have undefined behavior if the underlying storage is +//! mutated. Mutations may come from other processes, or from the same process +//! if the API provides `BorrowedFd` access, since as mentioned earlier, +//! `BorrowedFd` values may be used in APIs which provide safe access to any +//! system call. Consequently, code using `mmap` and presenting a safe API must +//! take full responsibility for ensuring that safe Rust code cannot evoke +//! undefined behavior through it. +//! +//! Like boxes, `OwnedFd` values conceptually own the resource they point to, +//! and free (close) it when they are dropped. +//! +//! ## `/proc/self/mem` and similar OS features +//! +//! Some platforms have special files, such as `/proc/self/mem`, which +//! provide read and write access to the process's memory. Such reads +//! and writes happen outside the control of the Rust compiler, so they do not +//! uphold Rust's memory safety guarantees. +//! +//! This does not mean that all APIs that might allow `/proc/self/mem` +//! to be opened and read from or written must be `unsafe`. Rust's safety guarantees +//! only cover what the program itself can do, and not what entities outside +//! the program can do to it. `/proc/self/mem` is considered to be such an +//! external entity, along with debugging interfaces, and people with physical access to +//! the hardware. This is true even in cases where the program is controlling +//! the external entity. +//! +//! If you desire to comprehensively prevent programs from reaching out and +//! causing external entities to reach back in and violate memory safety, it's +//! necessary to use *sandboxing*, which is outside the scope of `std`. +//! +//! [`BorrowedFd<'a>`]: crate::os::unix::io::BorrowedFd + +#![stable(feature = "rust1", since = "1.0.0")] + +#[stable(feature = "rust1", since = "1.0.0")] +pub use crate::os::fd::*; + +// Tests for this module +#[cfg(test)] +mod tests; diff --git a/crux-mir/lib/std/src/os/unix/io/tests.rs b/crux-mir/lib/std/src/os/unix/io/tests.rs new file mode 100644 index 000000000..84d2a7a1a --- /dev/null +++ b/crux-mir/lib/std/src/os/unix/io/tests.rs @@ -0,0 +1,11 @@ +use crate::mem::size_of; +use crate::os::unix::io::RawFd; + +#[test] +fn test_raw_fd_layout() { + // `OwnedFd` and `BorrowedFd` use `rustc_layout_scalar_valid_range_start` + // and `rustc_layout_scalar_valid_range_end`, with values that depend on + // the bit width of `RawFd`. If this ever changes, those values will need + // to be updated. + assert_eq!(size_of::(), 4); +} diff --git a/crux-mir/lib/std/src/os/unix/mod.rs b/crux-mir/lib/std/src/os/unix/mod.rs new file mode 100644 index 000000000..f97fa0fb0 --- /dev/null +++ b/crux-mir/lib/std/src/os/unix/mod.rs @@ -0,0 +1,128 @@ +//! Platform-specific extensions to `std` for Unix platforms. +//! +//! Provides access to platform-level information on Unix platforms, and +//! exposes Unix-specific functions that would otherwise be inappropriate as +//! part of the core `std` library. +//! +//! It exposes more ways to deal with platform-specific strings ([`OsStr`], +//! [`OsString`]), allows to set permissions more granularly, extract low-level +//! file descriptors from files and sockets, and has platform-specific helpers +//! for spawning processes. +//! +//! # Examples +//! +//! ```no_run +//! use std::fs::File; +//! use std::os::unix::prelude::*; +//! +//! fn main() -> std::io::Result<()> { +//! let f = File::create("foo.txt")?; +//! let fd = f.as_raw_fd(); +//! +//! // use fd with native unix bindings +//! +//! Ok(()) +//! } +//! ``` +//! +//! [`OsStr`]: crate::ffi::OsStr +//! [`OsString`]: crate::ffi::OsString + +#![stable(feature = "rust1", since = "1.0.0")] +#![doc(cfg(unix))] + +// Use linux as the default platform when documenting on other platforms like Windows +#[cfg(doc)] +use crate::os::linux as platform; + +#[cfg(not(doc))] +mod platform { + #[cfg(target_os = "android")] + pub use crate::os::android::*; + #[cfg(target_os = "dragonfly")] + pub use crate::os::dragonfly::*; + #[cfg(target_os = "emscripten")] + pub use crate::os::emscripten::*; + #[cfg(target_os = "espidf")] + pub use crate::os::espidf::*; + #[cfg(target_os = "freebsd")] + pub use crate::os::freebsd::*; + #[cfg(target_os = "fuchsia")] + pub use crate::os::fuchsia::*; + #[cfg(target_os = "haiku")] + pub use crate::os::haiku::*; + #[cfg(target_os = "horizon")] + pub use crate::os::horizon::*; + #[cfg(target_os = "illumos")] + pub use crate::os::illumos::*; + #[cfg(target_os = "ios")] + pub use crate::os::ios::*; + #[cfg(target_os = "l4re")] + pub use crate::os::l4re::*; + #[cfg(target_os = "linux")] + pub use crate::os::linux::*; + #[cfg(target_os = "macos")] + pub use crate::os::macos::*; + #[cfg(target_os = "netbsd")] + pub use crate::os::netbsd::*; + #[cfg(target_os = "openbsd")] + pub use crate::os::openbsd::*; + #[cfg(target_os = "redox")] + pub use crate::os::redox::*; + #[cfg(target_os = "solaris")] + pub use crate::os::solaris::*; + #[cfg(target_os = "vxworks")] + pub use crate::os::vxworks::*; + #[cfg(target_os = "watchos")] + pub use crate::os::watchos::*; +} + +pub mod ffi; +pub mod fs; +pub mod io; +pub mod net; +pub mod process; +pub mod raw; +pub mod thread; + +#[unstable(feature = "peer_credentials_unix_socket", issue = "42839", reason = "unstable")] +#[cfg(any( + target_os = "android", + target_os = "linux", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "ios", + target_os = "watchos", + target_os = "macos", + target_os = "netbsd", + target_os = "openbsd" +))] +pub mod ucred; + +/// A prelude for conveniently writing platform-specific code. +/// +/// Includes all extension traits, and some important type definitions. +#[stable(feature = "rust1", since = "1.0.0")] +pub mod prelude { + #[doc(no_inline)] + #[stable(feature = "rust1", since = "1.0.0")] + pub use super::ffi::{OsStrExt, OsStringExt}; + #[doc(no_inline)] + #[stable(feature = "rust1", since = "1.0.0")] + pub use super::fs::DirEntryExt; + #[doc(no_inline)] + #[stable(feature = "file_offset", since = "1.15.0")] + pub use super::fs::FileExt; + #[doc(no_inline)] + #[stable(feature = "rust1", since = "1.0.0")] + pub use super::fs::{FileTypeExt, MetadataExt, OpenOptionsExt, PermissionsExt}; + #[doc(no_inline)] + #[stable(feature = "rust1", since = "1.0.0")] + pub use super::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; + #[doc(no_inline)] + #[stable(feature = "rust1", since = "1.0.0")] + pub use super::process::{CommandExt, ExitStatusExt}; + #[doc(no_inline)] + #[stable(feature = "rust1", since = "1.0.0")] + pub use super::thread::JoinHandleExt; +} diff --git a/crux-mir/lib/std/src/os/unix/net/addr.rs b/crux-mir/lib/std/src/os/unix/net/addr.rs new file mode 100644 index 000000000..ece2b33bd --- /dev/null +++ b/crux-mir/lib/std/src/os/unix/net/addr.rs @@ -0,0 +1,295 @@ +use crate::ffi::OsStr; +#[cfg(any(doc, target_os = "android", target_os = "linux"))] +use crate::os::net::linux_ext; +use crate::os::unix::ffi::OsStrExt; +use crate::path::Path; +use crate::sealed::Sealed; +use crate::sys::cvt; +use crate::{fmt, io, mem, ptr}; + +// FIXME(#43348): Make libc adapt #[doc(cfg(...))] so we don't need these fake definitions here? +#[cfg(not(unix))] +#[allow(non_camel_case_types)] +mod libc { + pub use libc::c_int; + pub type socklen_t = u32; + pub struct sockaddr; + #[derive(Clone)] + pub struct sockaddr_un; +} + +fn sun_path_offset(addr: &libc::sockaddr_un) -> usize { + // Work with an actual instance of the type since using a null pointer is UB + let base = (addr as *const libc::sockaddr_un).addr(); + let path = (&addr.sun_path as *const libc::c_char).addr(); + path - base +} + +pub(super) fn sockaddr_un(path: &Path) -> io::Result<(libc::sockaddr_un, libc::socklen_t)> { + // SAFETY: All zeros is a valid representation for `sockaddr_un`. + let mut addr: libc::sockaddr_un = unsafe { mem::zeroed() }; + addr.sun_family = libc::AF_UNIX as libc::sa_family_t; + + let bytes = path.as_os_str().as_bytes(); + + if bytes.contains(&0) { + return Err(io::const_io_error!( + io::ErrorKind::InvalidInput, + "paths must not contain interior null bytes", + )); + } + + if bytes.len() >= addr.sun_path.len() { + return Err(io::const_io_error!( + io::ErrorKind::InvalidInput, + "path must be shorter than SUN_LEN", + )); + } + // SAFETY: `bytes` and `addr.sun_path` are not overlapping and + // both point to valid memory. + // NOTE: We zeroed the memory above, so the path is already null + // terminated. + unsafe { + ptr::copy_nonoverlapping(bytes.as_ptr(), addr.sun_path.as_mut_ptr().cast(), bytes.len()) + }; + + let mut len = sun_path_offset(&addr) + bytes.len(); + match bytes.get(0) { + Some(&0) | None => {} + Some(_) => len += 1, + } + Ok((addr, len as libc::socklen_t)) +} + +enum AddressKind<'a> { + Unnamed, + Pathname(&'a Path), + Abstract(&'a [u8]), +} + +/// An address associated with a Unix socket. +/// +/// # Examples +/// +/// ``` +/// use std::os::unix::net::UnixListener; +/// +/// let socket = match UnixListener::bind("/tmp/sock") { +/// Ok(sock) => sock, +/// Err(e) => { +/// println!("Couldn't bind: {e:?}"); +/// return +/// } +/// }; +/// let addr = socket.local_addr().expect("Couldn't get local address"); +/// ``` +#[derive(Clone)] +#[stable(feature = "unix_socket", since = "1.10.0")] +pub struct SocketAddr { + pub(super) addr: libc::sockaddr_un, + pub(super) len: libc::socklen_t, +} + +impl SocketAddr { + pub(super) fn new(f: F) -> io::Result + where + F: FnOnce(*mut libc::sockaddr, *mut libc::socklen_t) -> libc::c_int, + { + unsafe { + let mut addr: libc::sockaddr_un = mem::zeroed(); + let mut len = mem::size_of::() as libc::socklen_t; + cvt(f(&mut addr as *mut _ as *mut _, &mut len))?; + SocketAddr::from_parts(addr, len) + } + } + + pub(super) fn from_parts( + addr: libc::sockaddr_un, + mut len: libc::socklen_t, + ) -> io::Result { + if len == 0 { + // When there is a datagram from unnamed unix socket + // linux returns zero bytes of address + len = sun_path_offset(&addr) as libc::socklen_t; // i.e., zero-length address + } else if addr.sun_family != libc::AF_UNIX as libc::sa_family_t { + return Err(io::const_io_error!( + io::ErrorKind::InvalidInput, + "file descriptor did not correspond to a Unix socket", + )); + } + + Ok(SocketAddr { addr, len }) + } + + /// Constructs a `SockAddr` with the family `AF_UNIX` and the provided path. + /// + /// # Errors + /// + /// Returns an error if the path is longer than `SUN_LEN` or if it contains + /// NULL bytes. + /// + /// # Examples + /// + /// ``` + /// use std::os::unix::net::SocketAddr; + /// use std::path::Path; + /// + /// # fn main() -> std::io::Result<()> { + /// let address = SocketAddr::from_pathname("/path/to/socket")?; + /// assert_eq!(address.as_pathname(), Some(Path::new("/path/to/socket"))); + /// # Ok(()) + /// # } + /// ``` + /// + /// Creating a `SocketAddr` with a NULL byte results in an error. + /// + /// ``` + /// use std::os::unix::net::SocketAddr; + /// + /// assert!(SocketAddr::from_pathname("/path/with/\0/bytes").is_err()); + /// ``` + #[stable(feature = "unix_socket_creation", since = "1.61.0")] + pub fn from_pathname

(path: P) -> io::Result + where + P: AsRef, + { + sockaddr_un(path.as_ref()).map(|(addr, len)| SocketAddr { addr, len }) + } + + /// Returns `true` if the address is unnamed. + /// + /// # Examples + /// + /// A named address: + /// + /// ```no_run + /// use std::os::unix::net::UnixListener; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixListener::bind("/tmp/sock")?; + /// let addr = socket.local_addr().expect("Couldn't get local address"); + /// assert_eq!(addr.is_unnamed(), false); + /// Ok(()) + /// } + /// ``` + /// + /// An unnamed address: + /// + /// ``` + /// use std::os::unix::net::UnixDatagram; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixDatagram::unbound()?; + /// let addr = socket.local_addr().expect("Couldn't get local address"); + /// assert_eq!(addr.is_unnamed(), true); + /// Ok(()) + /// } + /// ``` + #[must_use] + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn is_unnamed(&self) -> bool { + matches!(self.address(), AddressKind::Unnamed) + } + + /// Returns the contents of this address if it is a `pathname` address. + /// + /// # Examples + /// + /// With a pathname: + /// + /// ```no_run + /// use std::os::unix::net::UnixListener; + /// use std::path::Path; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixListener::bind("/tmp/sock")?; + /// let addr = socket.local_addr().expect("Couldn't get local address"); + /// assert_eq!(addr.as_pathname(), Some(Path::new("/tmp/sock"))); + /// Ok(()) + /// } + /// ``` + /// + /// Without a pathname: + /// + /// ``` + /// use std::os::unix::net::UnixDatagram; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixDatagram::unbound()?; + /// let addr = socket.local_addr().expect("Couldn't get local address"); + /// assert_eq!(addr.as_pathname(), None); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + #[must_use] + pub fn as_pathname(&self) -> Option<&Path> { + if let AddressKind::Pathname(path) = self.address() { Some(path) } else { None } + } + + fn address(&self) -> AddressKind<'_> { + let len = self.len as usize - sun_path_offset(&self.addr); + let path = unsafe { mem::transmute::<&[libc::c_char], &[u8]>(&self.addr.sun_path) }; + + // macOS seems to return a len of 16 and a zeroed sun_path for unnamed addresses + if len == 0 + || (cfg!(not(any(target_os = "linux", target_os = "android"))) + && self.addr.sun_path[0] == 0) + { + AddressKind::Unnamed + } else if self.addr.sun_path[0] == 0 { + AddressKind::Abstract(&path[1..len]) + } else { + AddressKind::Pathname(OsStr::from_bytes(&path[..len - 1]).as_ref()) + } + } +} + +#[unstable(feature = "unix_socket_abstract", issue = "85410")] +impl Sealed for SocketAddr {} + +#[doc(cfg(any(target_os = "android", target_os = "linux")))] +#[cfg(any(doc, target_os = "android", target_os = "linux"))] +#[unstable(feature = "unix_socket_abstract", issue = "85410")] +impl linux_ext::addr::SocketAddrExt for SocketAddr { + fn as_abstract_name(&self) -> Option<&[u8]> { + if let AddressKind::Abstract(name) = self.address() { Some(name) } else { None } + } + + fn from_abstract_name(name: N) -> crate::io::Result + where + N: AsRef<[u8]>, + { + let name = name.as_ref(); + unsafe { + let mut addr: libc::sockaddr_un = mem::zeroed(); + addr.sun_family = libc::AF_UNIX as libc::sa_family_t; + + if name.len() + 1 > addr.sun_path.len() { + return Err(io::const_io_error!( + io::ErrorKind::InvalidInput, + "abstract socket name must be shorter than SUN_LEN", + )); + } + + crate::ptr::copy_nonoverlapping( + name.as_ptr(), + addr.sun_path.as_mut_ptr().add(1) as *mut u8, + name.len(), + ); + let len = (sun_path_offset(&addr) + 1 + name.len()) as libc::socklen_t; + SocketAddr::from_parts(addr, len) + } + } +} + +#[stable(feature = "unix_socket", since = "1.10.0")] +impl fmt::Debug for SocketAddr { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.address() { + AddressKind::Unnamed => write!(fmt, "(unnamed)"), + AddressKind::Abstract(name) => write!(fmt, "\"{}\" (abstract)", name.escape_ascii()), + AddressKind::Pathname(path) => write!(fmt, "{path:?} (pathname)"), + } + } +} diff --git a/crux-mir/lib/std/src/os/unix/net/ancillary.rs b/crux-mir/lib/std/src/os/unix/net/ancillary.rs new file mode 100644 index 000000000..7cc901a79 --- /dev/null +++ b/crux-mir/lib/std/src/os/unix/net/ancillary.rs @@ -0,0 +1,674 @@ +// FIXME: This is currently disabled on *BSD. + +use super::{sockaddr_un, SocketAddr}; +use crate::io::{self, IoSlice, IoSliceMut}; +use crate::marker::PhantomData; +use crate::mem::{size_of, zeroed}; +use crate::os::unix::io::RawFd; +use crate::path::Path; +use crate::ptr::{eq, read_unaligned}; +use crate::slice::from_raw_parts; +use crate::sys::net::Socket; + +// FIXME(#43348): Make libc adapt #[doc(cfg(...))] so we don't need these fake definitions here? +#[cfg(all(doc, not(target_os = "linux"), not(target_os = "android"), not(target_os = "netbsd")))] +#[allow(non_camel_case_types)] +mod libc { + pub use libc::c_int; + pub struct ucred; + pub struct cmsghdr; + pub type pid_t = i32; + pub type gid_t = u32; + pub type uid_t = u32; +} + +pub(super) fn recv_vectored_with_ancillary_from( + socket: &Socket, + bufs: &mut [IoSliceMut<'_>], + ancillary: &mut SocketAncillary<'_>, +) -> io::Result<(usize, bool, io::Result)> { + unsafe { + let mut msg_name: libc::sockaddr_un = zeroed(); + let mut msg: libc::msghdr = zeroed(); + msg.msg_name = &mut msg_name as *mut _ as *mut _; + msg.msg_namelen = size_of::() as libc::socklen_t; + msg.msg_iov = bufs.as_mut_ptr().cast(); + msg.msg_iovlen = bufs.len() as _; + msg.msg_controllen = ancillary.buffer.len() as _; + // macos requires that the control pointer is null when the len is 0. + if msg.msg_controllen > 0 { + msg.msg_control = ancillary.buffer.as_mut_ptr().cast(); + } + + let count = socket.recv_msg(&mut msg)?; + + ancillary.length = msg.msg_controllen as usize; + ancillary.truncated = msg.msg_flags & libc::MSG_CTRUNC == libc::MSG_CTRUNC; + + let truncated = msg.msg_flags & libc::MSG_TRUNC == libc::MSG_TRUNC; + let addr = SocketAddr::from_parts(msg_name, msg.msg_namelen); + + Ok((count, truncated, addr)) + } +} + +pub(super) fn send_vectored_with_ancillary_to( + socket: &Socket, + path: Option<&Path>, + bufs: &[IoSlice<'_>], + ancillary: &mut SocketAncillary<'_>, +) -> io::Result { + unsafe { + let (mut msg_name, msg_namelen) = + if let Some(path) = path { sockaddr_un(path)? } else { (zeroed(), 0) }; + + let mut msg: libc::msghdr = zeroed(); + msg.msg_name = &mut msg_name as *mut _ as *mut _; + msg.msg_namelen = msg_namelen; + msg.msg_iov = bufs.as_ptr() as *mut _; + msg.msg_iovlen = bufs.len() as _; + msg.msg_controllen = ancillary.length as _; + // macos requires that the control pointer is null when the len is 0. + if msg.msg_controllen > 0 { + msg.msg_control = ancillary.buffer.as_mut_ptr().cast(); + } + + ancillary.truncated = false; + + socket.send_msg(&mut msg) + } +} + +fn add_to_ancillary_data( + buffer: &mut [u8], + length: &mut usize, + source: &[T], + cmsg_level: libc::c_int, + cmsg_type: libc::c_int, +) -> bool { + let source_len = if let Some(source_len) = source.len().checked_mul(size_of::()) { + if let Ok(source_len) = u32::try_from(source_len) { + source_len + } else { + return false; + } + } else { + return false; + }; + + unsafe { + let additional_space = libc::CMSG_SPACE(source_len) as usize; + + let new_length = if let Some(new_length) = additional_space.checked_add(*length) { + new_length + } else { + return false; + }; + + if new_length > buffer.len() { + return false; + } + + buffer[*length..new_length].fill(0); + + *length = new_length; + + let mut msg: libc::msghdr = zeroed(); + msg.msg_control = buffer.as_mut_ptr().cast(); + msg.msg_controllen = *length as _; + + let mut cmsg = libc::CMSG_FIRSTHDR(&msg); + let mut previous_cmsg = cmsg; + while !cmsg.is_null() { + previous_cmsg = cmsg; + cmsg = libc::CMSG_NXTHDR(&msg, cmsg); + + // Most operating systems, but not Linux or emscripten, return the previous pointer + // when its length is zero. Therefore, check if the previous pointer is the same as + // the current one. + if eq(cmsg, previous_cmsg) { + break; + } + } + + if previous_cmsg.is_null() { + return false; + } + + (*previous_cmsg).cmsg_level = cmsg_level; + (*previous_cmsg).cmsg_type = cmsg_type; + (*previous_cmsg).cmsg_len = libc::CMSG_LEN(source_len) as _; + + let data = libc::CMSG_DATA(previous_cmsg).cast(); + + libc::memcpy(data, source.as_ptr().cast(), source_len as usize); + } + true +} + +struct AncillaryDataIter<'a, T> { + data: &'a [u8], + phantom: PhantomData, +} + +impl<'a, T> AncillaryDataIter<'a, T> { + /// Create `AncillaryDataIter` struct to iterate through the data unit in the control message. + /// + /// # Safety + /// + /// `data` must contain a valid control message. + unsafe fn new(data: &'a [u8]) -> AncillaryDataIter<'a, T> { + AncillaryDataIter { data, phantom: PhantomData } + } +} + +impl<'a, T> Iterator for AncillaryDataIter<'a, T> { + type Item = T; + + fn next(&mut self) -> Option { + if size_of::() <= self.data.len() { + unsafe { + let unit = read_unaligned(self.data.as_ptr().cast()); + self.data = &self.data[size_of::()..]; + Some(unit) + } + } else { + None + } + } +} + +#[cfg(all(doc, not(target_os = "android"), not(target_os = "linux"), not(target_os = "netbsd")))] +#[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] +#[derive(Clone)] +pub struct SocketCred(()); + +/// Unix credential. +#[cfg(any(target_os = "android", target_os = "linux",))] +#[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] +#[derive(Clone)] +pub struct SocketCred(libc::ucred); + +#[cfg(target_os = "netbsd")] +#[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] +#[derive(Clone)] +pub struct SocketCred(libc::sockcred); + +#[doc(cfg(any(target_os = "android", target_os = "linux")))] +#[cfg(any(target_os = "android", target_os = "linux"))] +impl SocketCred { + /// Create a Unix credential struct. + /// + /// PID, UID and GID is set to 0. + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + #[must_use] + pub fn new() -> SocketCred { + SocketCred(libc::ucred { pid: 0, uid: 0, gid: 0 }) + } + + /// Set the PID. + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn set_pid(&mut self, pid: libc::pid_t) { + self.0.pid = pid; + } + + /// Get the current PID. + #[must_use] + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn get_pid(&self) -> libc::pid_t { + self.0.pid + } + + /// Set the UID. + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn set_uid(&mut self, uid: libc::uid_t) { + self.0.uid = uid; + } + + /// Get the current UID. + #[must_use] + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn get_uid(&self) -> libc::uid_t { + self.0.uid + } + + /// Set the GID. + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn set_gid(&mut self, gid: libc::gid_t) { + self.0.gid = gid; + } + + /// Get the current GID. + #[must_use] + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn get_gid(&self) -> libc::gid_t { + self.0.gid + } +} + +#[cfg(target_os = "netbsd")] +impl SocketCred { + /// Create a Unix credential struct. + /// + /// PID, UID and GID is set to 0. + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn new() -> SocketCred { + SocketCred(libc::sockcred { + sc_pid: 0, + sc_uid: 0, + sc_euid: 0, + sc_gid: 0, + sc_egid: 0, + sc_ngroups: 0, + sc_groups: [0u32; 1], + }) + } + + /// Set the PID. + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn set_pid(&mut self, pid: libc::pid_t) { + self.0.sc_pid = pid; + } + + /// Get the current PID. + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn get_pid(&self) -> libc::pid_t { + self.0.sc_pid + } + + /// Set the UID. + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn set_uid(&mut self, uid: libc::uid_t) { + self.0.sc_uid = uid; + } + + /// Get the current UID. + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn get_uid(&self) -> libc::uid_t { + self.0.sc_uid + } + + /// Set the GID. + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn set_gid(&mut self, gid: libc::gid_t) { + self.0.sc_gid = gid; + } + + /// Get the current GID. + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn get_gid(&self) -> libc::gid_t { + self.0.sc_gid + } +} + +/// This control message contains file descriptors. +/// +/// The level is equal to `SOL_SOCKET` and the type is equal to `SCM_RIGHTS`. +#[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] +pub struct ScmRights<'a>(AncillaryDataIter<'a, RawFd>); + +#[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] +impl<'a> Iterator for ScmRights<'a> { + type Item = RawFd; + + fn next(&mut self) -> Option { + self.0.next() + } +} + +#[cfg(all(doc, not(target_os = "android"), not(target_os = "linux"), not(target_os = "netbsd")))] +#[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] +pub struct ScmCredentials<'a>(AncillaryDataIter<'a, ()>); + +/// This control message contains unix credentials. +/// +/// The level is equal to `SOL_SOCKET` and the type is equal to `SCM_CREDENTIALS` or `SCM_CREDS`. +#[cfg(any(target_os = "android", target_os = "linux",))] +#[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] +pub struct ScmCredentials<'a>(AncillaryDataIter<'a, libc::ucred>); + +#[cfg(target_os = "netbsd")] +#[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] +pub struct ScmCredentials<'a>(AncillaryDataIter<'a, libc::sockcred>); + +#[cfg(any(doc, target_os = "android", target_os = "linux", target_os = "netbsd",))] +#[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] +impl<'a> Iterator for ScmCredentials<'a> { + type Item = SocketCred; + + fn next(&mut self) -> Option { + Some(SocketCred(self.0.next()?)) + } +} + +/// The error type which is returned from parsing the type a control message. +#[non_exhaustive] +#[derive(Debug)] +#[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] +pub enum AncillaryError { + Unknown { cmsg_level: i32, cmsg_type: i32 }, +} + +/// This enum represent one control message of variable type. +#[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] +pub enum AncillaryData<'a> { + ScmRights(ScmRights<'a>), + #[cfg(any(doc, target_os = "android", target_os = "linux", target_os = "netbsd",))] + ScmCredentials(ScmCredentials<'a>), +} + +impl<'a> AncillaryData<'a> { + /// Create an `AncillaryData::ScmRights` variant. + /// + /// # Safety + /// + /// `data` must contain a valid control message and the control message must be type of + /// `SOL_SOCKET` and level of `SCM_RIGHTS`. + unsafe fn as_rights(data: &'a [u8]) -> Self { + let ancillary_data_iter = AncillaryDataIter::new(data); + let scm_rights = ScmRights(ancillary_data_iter); + AncillaryData::ScmRights(scm_rights) + } + + /// Create an `AncillaryData::ScmCredentials` variant. + /// + /// # Safety + /// + /// `data` must contain a valid control message and the control message must be type of + /// `SOL_SOCKET` and level of `SCM_CREDENTIALS` or `SCM_CREDS`. + #[cfg(any(doc, target_os = "android", target_os = "linux", target_os = "netbsd",))] + unsafe fn as_credentials(data: &'a [u8]) -> Self { + let ancillary_data_iter = AncillaryDataIter::new(data); + let scm_credentials = ScmCredentials(ancillary_data_iter); + AncillaryData::ScmCredentials(scm_credentials) + } + + fn try_from_cmsghdr(cmsg: &'a libc::cmsghdr) -> Result { + unsafe { + let cmsg_len_zero = libc::CMSG_LEN(0) as usize; + let data_len = (*cmsg).cmsg_len as usize - cmsg_len_zero; + let data = libc::CMSG_DATA(cmsg).cast(); + let data = from_raw_parts(data, data_len); + + match (*cmsg).cmsg_level { + libc::SOL_SOCKET => match (*cmsg).cmsg_type { + libc::SCM_RIGHTS => Ok(AncillaryData::as_rights(data)), + #[cfg(any(target_os = "android", target_os = "linux",))] + libc::SCM_CREDENTIALS => Ok(AncillaryData::as_credentials(data)), + #[cfg(target_os = "netbsd")] + libc::SCM_CREDS => Ok(AncillaryData::as_credentials(data)), + cmsg_type => { + Err(AncillaryError::Unknown { cmsg_level: libc::SOL_SOCKET, cmsg_type }) + } + }, + cmsg_level => { + Err(AncillaryError::Unknown { cmsg_level, cmsg_type: (*cmsg).cmsg_type }) + } + } + } + } +} + +/// This struct is used to iterate through the control messages. +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] +pub struct Messages<'a> { + buffer: &'a [u8], + current: Option<&'a libc::cmsghdr>, +} + +#[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] +impl<'a> Iterator for Messages<'a> { + type Item = Result, AncillaryError>; + + fn next(&mut self) -> Option { + unsafe { + let mut msg: libc::msghdr = zeroed(); + msg.msg_control = self.buffer.as_ptr() as *mut _; + msg.msg_controllen = self.buffer.len() as _; + + let cmsg = if let Some(current) = self.current { + libc::CMSG_NXTHDR(&msg, current) + } else { + libc::CMSG_FIRSTHDR(&msg) + }; + + let cmsg = cmsg.as_ref()?; + + // Most operating systems, but not Linux or emscripten, return the previous pointer + // when its length is zero. Therefore, check if the previous pointer is the same as + // the current one. + if let Some(current) = self.current { + if eq(current, cmsg) { + return None; + } + } + + self.current = Some(cmsg); + let ancillary_result = AncillaryData::try_from_cmsghdr(cmsg); + Some(ancillary_result) + } + } +} + +/// A Unix socket Ancillary data struct. +/// +/// # Example +/// ```no_run +/// #![feature(unix_socket_ancillary_data)] +/// use std::os::unix::net::{UnixStream, SocketAncillary, AncillaryData}; +/// use std::io::IoSliceMut; +/// +/// fn main() -> std::io::Result<()> { +/// let sock = UnixStream::connect("/tmp/sock")?; +/// +/// let mut fds = [0; 8]; +/// let mut ancillary_buffer = [0; 128]; +/// let mut ancillary = SocketAncillary::new(&mut ancillary_buffer[..]); +/// +/// let mut buf = [1; 8]; +/// let mut bufs = &mut [IoSliceMut::new(&mut buf[..])][..]; +/// sock.recv_vectored_with_ancillary(bufs, &mut ancillary)?; +/// +/// for ancillary_result in ancillary.messages() { +/// if let AncillaryData::ScmRights(scm_rights) = ancillary_result.unwrap() { +/// for fd in scm_rights { +/// println!("receive file descriptor: {fd}"); +/// } +/// } +/// } +/// Ok(()) +/// } +/// ``` +#[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] +#[derive(Debug)] +pub struct SocketAncillary<'a> { + buffer: &'a mut [u8], + length: usize, + truncated: bool, +} + +impl<'a> SocketAncillary<'a> { + /// Create an ancillary data with the given buffer. + /// + /// # Example + /// + /// ```no_run + /// # #![allow(unused_mut)] + /// #![feature(unix_socket_ancillary_data)] + /// use std::os::unix::net::SocketAncillary; + /// let mut ancillary_buffer = [0; 128]; + /// let mut ancillary = SocketAncillary::new(&mut ancillary_buffer[..]); + /// ``` + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn new(buffer: &'a mut [u8]) -> Self { + SocketAncillary { buffer, length: 0, truncated: false } + } + + /// Returns the capacity of the buffer. + #[must_use] + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn capacity(&self) -> usize { + self.buffer.len() + } + + /// Returns `true` if the ancillary data is empty. + #[must_use] + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn is_empty(&self) -> bool { + self.length == 0 + } + + /// Returns the number of used bytes. + #[must_use] + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn len(&self) -> usize { + self.length + } + + /// Returns the iterator of the control messages. + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn messages(&self) -> Messages<'_> { + Messages { buffer: &self.buffer[..self.length], current: None } + } + + /// Is `true` if during a recv operation the ancillary was truncated. + /// + /// # Example + /// + /// ```no_run + /// #![feature(unix_socket_ancillary_data)] + /// use std::os::unix::net::{UnixStream, SocketAncillary}; + /// use std::io::IoSliceMut; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixStream::connect("/tmp/sock")?; + /// + /// let mut ancillary_buffer = [0; 128]; + /// let mut ancillary = SocketAncillary::new(&mut ancillary_buffer[..]); + /// + /// let mut buf = [1; 8]; + /// let mut bufs = &mut [IoSliceMut::new(&mut buf[..])][..]; + /// sock.recv_vectored_with_ancillary(bufs, &mut ancillary)?; + /// + /// println!("Is truncated: {}", ancillary.truncated()); + /// Ok(()) + /// } + /// ``` + #[must_use] + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn truncated(&self) -> bool { + self.truncated + } + + /// Add file descriptors to the ancillary data. + /// + /// The function returns `true` if there was enough space in the buffer. + /// If there was not enough space then no file descriptors was appended. + /// Technically, that means this operation adds a control message with the level `SOL_SOCKET` + /// and type `SCM_RIGHTS`. + /// + /// # Example + /// + /// ```no_run + /// #![feature(unix_socket_ancillary_data)] + /// use std::os::unix::net::{UnixStream, SocketAncillary}; + /// use std::os::unix::io::AsRawFd; + /// use std::io::IoSlice; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixStream::connect("/tmp/sock")?; + /// + /// let mut ancillary_buffer = [0; 128]; + /// let mut ancillary = SocketAncillary::new(&mut ancillary_buffer[..]); + /// ancillary.add_fds(&[sock.as_raw_fd()][..]); + /// + /// let buf = [1; 8]; + /// let mut bufs = &mut [IoSlice::new(&buf[..])][..]; + /// sock.send_vectored_with_ancillary(bufs, &mut ancillary)?; + /// Ok(()) + /// } + /// ``` + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn add_fds(&mut self, fds: &[RawFd]) -> bool { + self.truncated = false; + add_to_ancillary_data( + &mut self.buffer, + &mut self.length, + fds, + libc::SOL_SOCKET, + libc::SCM_RIGHTS, + ) + } + + /// Add credentials to the ancillary data. + /// + /// The function returns `true` if there was enough space in the buffer. + /// If there was not enough space then no credentials was appended. + /// Technically, that means this operation adds a control message with the level `SOL_SOCKET` + /// and type `SCM_CREDENTIALS` or `SCM_CREDS`. + /// + #[cfg(any(doc, target_os = "android", target_os = "linux", target_os = "netbsd",))] + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn add_creds(&mut self, creds: &[SocketCred]) -> bool { + self.truncated = false; + add_to_ancillary_data( + &mut self.buffer, + &mut self.length, + creds, + libc::SOL_SOCKET, + #[cfg(not(target_os = "netbsd"))] + libc::SCM_CREDENTIALS, + #[cfg(target_os = "netbsd")] + libc::SCM_CREDS, + ) + } + + /// Clears the ancillary data, removing all values. + /// + /// # Example + /// + /// ```no_run + /// #![feature(unix_socket_ancillary_data)] + /// use std::os::unix::net::{UnixStream, SocketAncillary, AncillaryData}; + /// use std::io::IoSliceMut; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixStream::connect("/tmp/sock")?; + /// + /// let mut fds1 = [0; 8]; + /// let mut fds2 = [0; 8]; + /// let mut ancillary_buffer = [0; 128]; + /// let mut ancillary = SocketAncillary::new(&mut ancillary_buffer[..]); + /// + /// let mut buf = [1; 8]; + /// let mut bufs = &mut [IoSliceMut::new(&mut buf[..])][..]; + /// + /// sock.recv_vectored_with_ancillary(bufs, &mut ancillary)?; + /// for ancillary_result in ancillary.messages() { + /// if let AncillaryData::ScmRights(scm_rights) = ancillary_result.unwrap() { + /// for fd in scm_rights { + /// println!("receive file descriptor: {fd}"); + /// } + /// } + /// } + /// + /// ancillary.clear(); + /// + /// sock.recv_vectored_with_ancillary(bufs, &mut ancillary)?; + /// for ancillary_result in ancillary.messages() { + /// if let AncillaryData::ScmRights(scm_rights) = ancillary_result.unwrap() { + /// for fd in scm_rights { + /// println!("receive file descriptor: {fd}"); + /// } + /// } + /// } + /// Ok(()) + /// } + /// ``` + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn clear(&mut self) { + self.length = 0; + self.truncated = false; + } +} diff --git a/crux-mir/lib/std/src/os/unix/net/datagram.rs b/crux-mir/lib/std/src/os/unix/net/datagram.rs new file mode 100644 index 000000000..f758f88d0 --- /dev/null +++ b/crux-mir/lib/std/src/os/unix/net/datagram.rs @@ -0,0 +1,1012 @@ +#[cfg(any(doc, target_os = "android", target_os = "linux"))] +use super::{recv_vectored_with_ancillary_from, send_vectored_with_ancillary_to, SocketAncillary}; +use super::{sockaddr_un, SocketAddr}; +#[cfg(any(doc, target_os = "android", target_os = "linux"))] +use crate::io::{IoSlice, IoSliceMut}; +use crate::net::Shutdown; +use crate::os::unix::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; +use crate::path::Path; +use crate::sys::cvt; +use crate::sys::net::Socket; +use crate::sys_common::{AsInner, FromInner, IntoInner}; +use crate::time::Duration; +use crate::{fmt, io}; + +#[cfg(any( + target_os = "linux", + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "openbsd", + target_os = "netbsd", + target_os = "haiku" +))] +use libc::MSG_NOSIGNAL; +#[cfg(not(any( + target_os = "linux", + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "openbsd", + target_os = "netbsd", + target_os = "haiku" +)))] +const MSG_NOSIGNAL: libc::c_int = 0x0; + +/// A Unix datagram socket. +/// +/// # Examples +/// +/// ```no_run +/// use std::os::unix::net::UnixDatagram; +/// +/// fn main() -> std::io::Result<()> { +/// let socket = UnixDatagram::bind("/path/to/my/socket")?; +/// socket.send_to(b"hello world", "/path/to/other/socket")?; +/// let mut buf = [0; 100]; +/// let (count, address) = socket.recv_from(&mut buf)?; +/// println!("socket {:?} sent {:?}", address, &buf[..count]); +/// Ok(()) +/// } +/// ``` +#[stable(feature = "unix_socket", since = "1.10.0")] +pub struct UnixDatagram(Socket); + +#[stable(feature = "unix_socket", since = "1.10.0")] +impl fmt::Debug for UnixDatagram { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut builder = fmt.debug_struct("UnixDatagram"); + builder.field("fd", self.0.as_inner()); + if let Ok(addr) = self.local_addr() { + builder.field("local", &addr); + } + if let Ok(addr) = self.peer_addr() { + builder.field("peer", &addr); + } + builder.finish() + } +} + +impl UnixDatagram { + /// Creates a Unix datagram socket bound to the given path. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixDatagram; + /// + /// let sock = match UnixDatagram::bind("/path/to/the/socket") { + /// Ok(sock) => sock, + /// Err(e) => { + /// println!("Couldn't bind: {e:?}"); + /// return + /// } + /// }; + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn bind>(path: P) -> io::Result { + unsafe { + let socket = UnixDatagram::unbound()?; + let (addr, len) = sockaddr_un(path.as_ref())?; + + cvt(libc::bind(socket.as_raw_fd(), &addr as *const _ as *const _, len as _))?; + + Ok(socket) + } + } + + /// Creates a Unix datagram socket bound to an address. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(unix_socket_abstract)] + /// use std::os::unix::net::{UnixDatagram}; + /// + /// fn main() -> std::io::Result<()> { + /// let sock1 = UnixDatagram::bind("path/to/socket")?; + /// let addr = sock1.local_addr()?; + /// + /// let sock2 = match UnixDatagram::bind_addr(&addr) { + /// Ok(sock) => sock, + /// Err(err) => { + /// println!("Couldn't bind: {err:?}"); + /// return Err(err); + /// } + /// }; + /// Ok(()) + /// } + /// ``` + #[unstable(feature = "unix_socket_abstract", issue = "85410")] + pub fn bind_addr(socket_addr: &SocketAddr) -> io::Result { + unsafe { + let socket = UnixDatagram::unbound()?; + cvt(libc::bind( + socket.as_raw_fd(), + &socket_addr.addr as *const _ as *const _, + socket_addr.len as _, + ))?; + Ok(socket) + } + } + + /// Creates a Unix Datagram socket which is not bound to any address. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixDatagram; + /// + /// let sock = match UnixDatagram::unbound() { + /// Ok(sock) => sock, + /// Err(e) => { + /// println!("Couldn't unbound: {e:?}"); + /// return + /// } + /// }; + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn unbound() -> io::Result { + let inner = Socket::new_raw(libc::AF_UNIX, libc::SOCK_DGRAM)?; + Ok(UnixDatagram(inner)) + } + + /// Creates an unnamed pair of connected sockets. + /// + /// Returns two `UnixDatagrams`s which are connected to each other. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixDatagram; + /// + /// let (sock1, sock2) = match UnixDatagram::pair() { + /// Ok((sock1, sock2)) => (sock1, sock2), + /// Err(e) => { + /// println!("Couldn't unbound: {e:?}"); + /// return + /// } + /// }; + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn pair() -> io::Result<(UnixDatagram, UnixDatagram)> { + let (i1, i2) = Socket::new_pair(libc::AF_UNIX, libc::SOCK_DGRAM)?; + Ok((UnixDatagram(i1), UnixDatagram(i2))) + } + + /// Connects the socket to the specified path address. + /// + /// The [`send`] method may be used to send data to the specified address. + /// [`recv`] and [`recv_from`] will only receive data from that address. + /// + /// [`send`]: UnixDatagram::send + /// [`recv`]: UnixDatagram::recv + /// [`recv_from`]: UnixDatagram::recv_from + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixDatagram; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixDatagram::unbound()?; + /// match sock.connect("/path/to/the/socket") { + /// Ok(sock) => sock, + /// Err(e) => { + /// println!("Couldn't connect: {e:?}"); + /// return Err(e) + /// } + /// }; + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn connect>(&self, path: P) -> io::Result<()> { + unsafe { + let (addr, len) = sockaddr_un(path.as_ref())?; + + cvt(libc::connect(self.as_raw_fd(), &addr as *const _ as *const _, len))?; + } + Ok(()) + } + + /// Connects the socket to an address. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(unix_socket_abstract)] + /// use std::os::unix::net::{UnixDatagram}; + /// + /// fn main() -> std::io::Result<()> { + /// let bound = UnixDatagram::bind("/path/to/socket")?; + /// let addr = bound.local_addr()?; + /// + /// let sock = UnixDatagram::unbound()?; + /// match sock.connect_addr(&addr) { + /// Ok(sock) => sock, + /// Err(e) => { + /// println!("Couldn't connect: {e:?}"); + /// return Err(e) + /// } + /// }; + /// Ok(()) + /// } + /// ``` + #[unstable(feature = "unix_socket_abstract", issue = "85410")] + pub fn connect_addr(&self, socket_addr: &SocketAddr) -> io::Result<()> { + unsafe { + cvt(libc::connect( + self.as_raw_fd(), + &socket_addr.addr as *const _ as *const _, + socket_addr.len, + ))?; + } + Ok(()) + } + + /// Creates a new independently owned handle to the underlying socket. + /// + /// The returned `UnixDatagram` is a reference to the same socket that this + /// object references. Both handles can be used to accept incoming + /// connections and options set on one side will affect the other. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixDatagram; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixDatagram::bind("/path/to/the/socket")?; + /// let sock_copy = sock.try_clone().expect("try_clone failed"); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn try_clone(&self) -> io::Result { + self.0.duplicate().map(UnixDatagram) + } + + /// Returns the address of this socket. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixDatagram; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixDatagram::bind("/path/to/the/socket")?; + /// let addr = sock.local_addr().expect("Couldn't get local address"); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn local_addr(&self) -> io::Result { + SocketAddr::new(|addr, len| unsafe { libc::getsockname(self.as_raw_fd(), addr, len) }) + } + + /// Returns the address of this socket's peer. + /// + /// The [`connect`] method will connect the socket to a peer. + /// + /// [`connect`]: UnixDatagram::connect + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixDatagram; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixDatagram::unbound()?; + /// sock.connect("/path/to/the/socket")?; + /// + /// let addr = sock.peer_addr().expect("Couldn't get peer address"); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn peer_addr(&self) -> io::Result { + SocketAddr::new(|addr, len| unsafe { libc::getpeername(self.as_raw_fd(), addr, len) }) + } + + fn recv_from_flags( + &self, + buf: &mut [u8], + flags: libc::c_int, + ) -> io::Result<(usize, SocketAddr)> { + let mut count = 0; + let addr = SocketAddr::new(|addr, len| unsafe { + count = libc::recvfrom( + self.as_raw_fd(), + buf.as_mut_ptr() as *mut _, + buf.len(), + flags, + addr, + len, + ); + if count > 0 { + 1 + } else if count == 0 { + 0 + } else { + -1 + } + })?; + + Ok((count as usize, addr)) + } + + /// Receives data from the socket. + /// + /// On success, returns the number of bytes read and the address from + /// whence the data came. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixDatagram; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixDatagram::unbound()?; + /// let mut buf = vec![0; 10]; + /// let (size, sender) = sock.recv_from(buf.as_mut_slice())?; + /// println!("received {size} bytes from {sender:?}"); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn recv_from(&self, buf: &mut [u8]) -> io::Result<(usize, SocketAddr)> { + self.recv_from_flags(buf, 0) + } + + /// Receives data from the socket. + /// + /// On success, returns the number of bytes read. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixDatagram; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixDatagram::bind("/path/to/the/socket")?; + /// let mut buf = vec![0; 10]; + /// sock.recv(buf.as_mut_slice()).expect("recv function failed"); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn recv(&self, buf: &mut [u8]) -> io::Result { + self.0.read(buf) + } + + /// Receives data and ancillary data from socket. + /// + /// On success, returns the number of bytes read, if the data was truncated and the address from whence the msg came. + /// + /// # Examples + /// + #[cfg_attr(any(target_os = "android", target_os = "linux"), doc = "```no_run")] + #[cfg_attr(not(any(target_os = "android", target_os = "linux")), doc = "```ignore")] + /// #![feature(unix_socket_ancillary_data)] + /// use std::os::unix::net::{UnixDatagram, SocketAncillary, AncillaryData}; + /// use std::io::IoSliceMut; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixDatagram::unbound()?; + /// let mut buf1 = [1; 8]; + /// let mut buf2 = [2; 16]; + /// let mut buf3 = [3; 8]; + /// let mut bufs = &mut [ + /// IoSliceMut::new(&mut buf1), + /// IoSliceMut::new(&mut buf2), + /// IoSliceMut::new(&mut buf3), + /// ][..]; + /// let mut fds = [0; 8]; + /// let mut ancillary_buffer = [0; 128]; + /// let mut ancillary = SocketAncillary::new(&mut ancillary_buffer[..]); + /// let (size, _truncated, sender) = sock.recv_vectored_with_ancillary_from(bufs, &mut ancillary)?; + /// println!("received {size}"); + /// for ancillary_result in ancillary.messages() { + /// if let AncillaryData::ScmRights(scm_rights) = ancillary_result.unwrap() { + /// for fd in scm_rights { + /// println!("receive file descriptor: {fd}"); + /// } + /// } + /// } + /// Ok(()) + /// } + /// ``` + #[cfg(any(doc, target_os = "android", target_os = "linux"))] + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn recv_vectored_with_ancillary_from( + &self, + bufs: &mut [IoSliceMut<'_>], + ancillary: &mut SocketAncillary<'_>, + ) -> io::Result<(usize, bool, SocketAddr)> { + let (count, truncated, addr) = recv_vectored_with_ancillary_from(&self.0, bufs, ancillary)?; + let addr = addr?; + + Ok((count, truncated, addr)) + } + + /// Receives data and ancillary data from socket. + /// + /// On success, returns the number of bytes read and if the data was truncated. + /// + /// # Examples + /// + #[cfg_attr(any(target_os = "android", target_os = "linux"), doc = "```no_run")] + #[cfg_attr(not(any(target_os = "android", target_os = "linux")), doc = "```ignore")] + /// #![feature(unix_socket_ancillary_data)] + /// use std::os::unix::net::{UnixDatagram, SocketAncillary, AncillaryData}; + /// use std::io::IoSliceMut; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixDatagram::unbound()?; + /// let mut buf1 = [1; 8]; + /// let mut buf2 = [2; 16]; + /// let mut buf3 = [3; 8]; + /// let mut bufs = &mut [ + /// IoSliceMut::new(&mut buf1), + /// IoSliceMut::new(&mut buf2), + /// IoSliceMut::new(&mut buf3), + /// ][..]; + /// let mut fds = [0; 8]; + /// let mut ancillary_buffer = [0; 128]; + /// let mut ancillary = SocketAncillary::new(&mut ancillary_buffer[..]); + /// let (size, _truncated) = sock.recv_vectored_with_ancillary(bufs, &mut ancillary)?; + /// println!("received {size}"); + /// for ancillary_result in ancillary.messages() { + /// if let AncillaryData::ScmRights(scm_rights) = ancillary_result.unwrap() { + /// for fd in scm_rights { + /// println!("receive file descriptor: {fd}"); + /// } + /// } + /// } + /// Ok(()) + /// } + /// ``` + #[cfg(any(doc, target_os = "android", target_os = "linux"))] + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn recv_vectored_with_ancillary( + &self, + bufs: &mut [IoSliceMut<'_>], + ancillary: &mut SocketAncillary<'_>, + ) -> io::Result<(usize, bool)> { + let (count, truncated, addr) = recv_vectored_with_ancillary_from(&self.0, bufs, ancillary)?; + addr?; + + Ok((count, truncated)) + } + + /// Sends data on the socket to the specified address. + /// + /// On success, returns the number of bytes written. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixDatagram; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixDatagram::unbound()?; + /// sock.send_to(b"omelette au fromage", "/some/sock").expect("send_to function failed"); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn send_to>(&self, buf: &[u8], path: P) -> io::Result { + unsafe { + let (addr, len) = sockaddr_un(path.as_ref())?; + + let count = cvt(libc::sendto( + self.as_raw_fd(), + buf.as_ptr() as *const _, + buf.len(), + MSG_NOSIGNAL, + &addr as *const _ as *const _, + len, + ))?; + Ok(count as usize) + } + } + + /// Sends data on the socket to the specified [SocketAddr]. + /// + /// On success, returns the number of bytes written. + /// + /// [SocketAddr]: crate::os::unix::net::SocketAddr + /// + /// # Examples + /// + /// ```no_run + /// #![feature(unix_socket_abstract)] + /// use std::os::unix::net::{UnixDatagram}; + /// + /// fn main() -> std::io::Result<()> { + /// let bound = UnixDatagram::bind("/path/to/socket")?; + /// let addr = bound.local_addr()?; + /// + /// let sock = UnixDatagram::unbound()?; + /// sock.send_to_addr(b"bacon egg and cheese", &addr).expect("send_to_addr function failed"); + /// Ok(()) + /// } + /// ``` + #[unstable(feature = "unix_socket_abstract", issue = "85410")] + pub fn send_to_addr(&self, buf: &[u8], socket_addr: &SocketAddr) -> io::Result { + unsafe { + let count = cvt(libc::sendto( + self.as_raw_fd(), + buf.as_ptr() as *const _, + buf.len(), + MSG_NOSIGNAL, + &socket_addr.addr as *const _ as *const _, + socket_addr.len, + ))?; + Ok(count as usize) + } + } + + /// Sends data on the socket to the socket's peer. + /// + /// The peer address may be set by the `connect` method, and this method + /// will return an error if the socket has not already been connected. + /// + /// On success, returns the number of bytes written. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixDatagram; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixDatagram::unbound()?; + /// sock.connect("/some/sock").expect("Couldn't connect"); + /// sock.send(b"omelette au fromage").expect("send_to function failed"); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn send(&self, buf: &[u8]) -> io::Result { + self.0.write(buf) + } + + /// Sends data and ancillary data on the socket to the specified address. + /// + /// On success, returns the number of bytes written. + /// + /// # Examples + /// + #[cfg_attr(any(target_os = "android", target_os = "linux"), doc = "```no_run")] + #[cfg_attr(not(any(target_os = "android", target_os = "linux")), doc = "```ignore")] + /// #![feature(unix_socket_ancillary_data)] + /// use std::os::unix::net::{UnixDatagram, SocketAncillary}; + /// use std::io::IoSlice; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixDatagram::unbound()?; + /// let buf1 = [1; 8]; + /// let buf2 = [2; 16]; + /// let buf3 = [3; 8]; + /// let bufs = &[ + /// IoSlice::new(&buf1), + /// IoSlice::new(&buf2), + /// IoSlice::new(&buf3), + /// ][..]; + /// let fds = [0, 1, 2]; + /// let mut ancillary_buffer = [0; 128]; + /// let mut ancillary = SocketAncillary::new(&mut ancillary_buffer[..]); + /// ancillary.add_fds(&fds[..]); + /// sock.send_vectored_with_ancillary_to(bufs, &mut ancillary, "/some/sock") + /// .expect("send_vectored_with_ancillary_to function failed"); + /// Ok(()) + /// } + /// ``` + #[cfg(any(doc, target_os = "android", target_os = "linux"))] + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn send_vectored_with_ancillary_to>( + &self, + bufs: &[IoSlice<'_>], + ancillary: &mut SocketAncillary<'_>, + path: P, + ) -> io::Result { + send_vectored_with_ancillary_to(&self.0, Some(path.as_ref()), bufs, ancillary) + } + + /// Sends data and ancillary data on the socket. + /// + /// On success, returns the number of bytes written. + /// + /// # Examples + /// + #[cfg_attr(any(target_os = "android", target_os = "linux"), doc = "```no_run")] + #[cfg_attr(not(any(target_os = "android", target_os = "linux")), doc = "```ignore")] + /// #![feature(unix_socket_ancillary_data)] + /// use std::os::unix::net::{UnixDatagram, SocketAncillary}; + /// use std::io::IoSlice; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixDatagram::unbound()?; + /// let buf1 = [1; 8]; + /// let buf2 = [2; 16]; + /// let buf3 = [3; 8]; + /// let bufs = &[ + /// IoSlice::new(&buf1), + /// IoSlice::new(&buf2), + /// IoSlice::new(&buf3), + /// ][..]; + /// let fds = [0, 1, 2]; + /// let mut ancillary_buffer = [0; 128]; + /// let mut ancillary = SocketAncillary::new(&mut ancillary_buffer[..]); + /// ancillary.add_fds(&fds[..]); + /// sock.send_vectored_with_ancillary(bufs, &mut ancillary) + /// .expect("send_vectored_with_ancillary function failed"); + /// Ok(()) + /// } + /// ``` + #[cfg(any(doc, target_os = "android", target_os = "linux"))] + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn send_vectored_with_ancillary( + &self, + bufs: &[IoSlice<'_>], + ancillary: &mut SocketAncillary<'_>, + ) -> io::Result { + send_vectored_with_ancillary_to(&self.0, None, bufs, ancillary) + } + + /// Sets the read timeout for the socket. + /// + /// If the provided value is [`None`], then [`recv`] and [`recv_from`] calls will + /// block indefinitely. An [`Err`] is returned if the zero [`Duration`] + /// is passed to this method. + /// + /// [`recv`]: UnixDatagram::recv + /// [`recv_from`]: UnixDatagram::recv_from + /// + /// # Examples + /// + /// ``` + /// use std::os::unix::net::UnixDatagram; + /// use std::time::Duration; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixDatagram::unbound()?; + /// sock.set_read_timeout(Some(Duration::new(1, 0))) + /// .expect("set_read_timeout function failed"); + /// Ok(()) + /// } + /// ``` + /// + /// An [`Err`] is returned if the zero [`Duration`] is passed to this + /// method: + /// + /// ```no_run + /// use std::io; + /// use std::os::unix::net::UnixDatagram; + /// use std::time::Duration; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixDatagram::unbound()?; + /// let result = socket.set_read_timeout(Some(Duration::new(0, 0))); + /// let err = result.unwrap_err(); + /// assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn set_read_timeout(&self, timeout: Option) -> io::Result<()> { + self.0.set_timeout(timeout, libc::SO_RCVTIMEO) + } + + /// Sets the write timeout for the socket. + /// + /// If the provided value is [`None`], then [`send`] and [`send_to`] calls will + /// block indefinitely. An [`Err`] is returned if the zero [`Duration`] is passed to this + /// method. + /// + /// [`send`]: UnixDatagram::send + /// [`send_to`]: UnixDatagram::send_to + /// + /// # Examples + /// + /// ``` + /// use std::os::unix::net::UnixDatagram; + /// use std::time::Duration; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixDatagram::unbound()?; + /// sock.set_write_timeout(Some(Duration::new(1, 0))) + /// .expect("set_write_timeout function failed"); + /// Ok(()) + /// } + /// ``` + /// + /// An [`Err`] is returned if the zero [`Duration`] is passed to this + /// method: + /// + /// ```no_run + /// use std::io; + /// use std::os::unix::net::UnixDatagram; + /// use std::time::Duration; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixDatagram::unbound()?; + /// let result = socket.set_write_timeout(Some(Duration::new(0, 0))); + /// let err = result.unwrap_err(); + /// assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn set_write_timeout(&self, timeout: Option) -> io::Result<()> { + self.0.set_timeout(timeout, libc::SO_SNDTIMEO) + } + + /// Returns the read timeout of this socket. + /// + /// # Examples + /// + /// ``` + /// use std::os::unix::net::UnixDatagram; + /// use std::time::Duration; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixDatagram::unbound()?; + /// sock.set_read_timeout(Some(Duration::new(1, 0))) + /// .expect("set_read_timeout function failed"); + /// assert_eq!(sock.read_timeout()?, Some(Duration::new(1, 0))); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn read_timeout(&self) -> io::Result> { + self.0.timeout(libc::SO_RCVTIMEO) + } + + /// Returns the write timeout of this socket. + /// + /// # Examples + /// + /// ``` + /// use std::os::unix::net::UnixDatagram; + /// use std::time::Duration; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixDatagram::unbound()?; + /// sock.set_write_timeout(Some(Duration::new(1, 0))) + /// .expect("set_write_timeout function failed"); + /// assert_eq!(sock.write_timeout()?, Some(Duration::new(1, 0))); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn write_timeout(&self) -> io::Result> { + self.0.timeout(libc::SO_SNDTIMEO) + } + + /// Moves the socket into or out of nonblocking mode. + /// + /// # Examples + /// + /// ``` + /// use std::os::unix::net::UnixDatagram; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixDatagram::unbound()?; + /// sock.set_nonblocking(true).expect("set_nonblocking function failed"); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()> { + self.0.set_nonblocking(nonblocking) + } + + /// Moves the socket to pass unix credentials as control message in [`SocketAncillary`]. + /// + /// Set the socket option `SO_PASSCRED`. + /// + /// # Examples + /// + #[cfg_attr(any(target_os = "android", target_os = "linux"), doc = "```no_run")] + #[cfg_attr(not(any(target_os = "android", target_os = "linux")), doc = "```ignore")] + /// #![feature(unix_socket_ancillary_data)] + /// use std::os::unix::net::UnixDatagram; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixDatagram::unbound()?; + /// sock.set_passcred(true).expect("set_passcred function failed"); + /// Ok(()) + /// } + /// ``` + #[cfg(any(doc, target_os = "android", target_os = "linux", target_os = "netbsd",))] + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn set_passcred(&self, passcred: bool) -> io::Result<()> { + self.0.set_passcred(passcred) + } + + /// Get the current value of the socket for passing unix credentials in [`SocketAncillary`]. + /// This value can be change by [`set_passcred`]. + /// + /// Get the socket option `SO_PASSCRED`. + /// + /// [`set_passcred`]: UnixDatagram::set_passcred + #[cfg(any(doc, target_os = "android", target_os = "linux", target_os = "netbsd",))] + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn passcred(&self) -> io::Result { + self.0.passcred() + } + + /// Set the id of the socket for network filtering purpose + /// + #[cfg_attr( + any(target_os = "linux", target_os = "freebsd", target_os = "openbsd"), + doc = "```no_run" + )] + #[cfg_attr( + not(any(target_os = "linux", target_os = "freebsd", target_os = "openbsd")), + doc = "```ignore" + )] + /// #![feature(unix_set_mark)] + /// use std::os::unix::net::UnixDatagram; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixDatagram::unbound()?; + /// sock.set_mark(32)?; + /// Ok(()) + /// } + /// ``` + #[cfg(any(doc, target_os = "linux", target_os = "freebsd", target_os = "openbsd",))] + #[unstable(feature = "unix_set_mark", issue = "96467")] + pub fn set_mark(&self, mark: u32) -> io::Result<()> { + self.0.set_mark(mark) + } + + /// Returns the value of the `SO_ERROR` option. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixDatagram; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixDatagram::unbound()?; + /// if let Ok(Some(err)) = sock.take_error() { + /// println!("Got error: {err:?}"); + /// } + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn take_error(&self) -> io::Result> { + self.0.take_error() + } + + /// Shut down the read, write, or both halves of this connection. + /// + /// This function will cause all pending and future I/O calls on the + /// specified portions to immediately return with an appropriate value + /// (see the documentation of [`Shutdown`]). + /// + /// ```no_run + /// use std::os::unix::net::UnixDatagram; + /// use std::net::Shutdown; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixDatagram::unbound()?; + /// sock.shutdown(Shutdown::Both).expect("shutdown function failed"); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn shutdown(&self, how: Shutdown) -> io::Result<()> { + self.0.shutdown(how) + } + + /// Receives data on the socket from the remote address to which it is + /// connected, without removing that data from the queue. On success, + /// returns the number of bytes peeked. + /// + /// Successive calls return the same data. This is accomplished by passing + /// `MSG_PEEK` as a flag to the underlying `recv` system call. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(unix_socket_peek)] + /// + /// use std::os::unix::net::UnixDatagram; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixDatagram::bind("/tmp/sock")?; + /// let mut buf = [0; 10]; + /// let len = socket.peek(&mut buf).expect("peek failed"); + /// Ok(()) + /// } + /// ``` + #[unstable(feature = "unix_socket_peek", issue = "76923")] + pub fn peek(&self, buf: &mut [u8]) -> io::Result { + self.0.peek(buf) + } + + /// Receives a single datagram message on the socket, without removing it from the + /// queue. On success, returns the number of bytes read and the origin. + /// + /// The function must be called with valid byte array `buf` of sufficient size to + /// hold the message bytes. If a message is too long to fit in the supplied buffer, + /// excess bytes may be discarded. + /// + /// Successive calls return the same data. This is accomplished by passing + /// `MSG_PEEK` as a flag to the underlying `recvfrom` system call. + /// + /// Do not use this function to implement busy waiting, instead use `libc::poll` to + /// synchronize IO events on one or more sockets. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(unix_socket_peek)] + /// + /// use std::os::unix::net::UnixDatagram; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixDatagram::bind("/tmp/sock")?; + /// let mut buf = [0; 10]; + /// let (len, addr) = socket.peek_from(&mut buf).expect("peek failed"); + /// Ok(()) + /// } + /// ``` + #[unstable(feature = "unix_socket_peek", issue = "76923")] + pub fn peek_from(&self, buf: &mut [u8]) -> io::Result<(usize, SocketAddr)> { + self.recv_from_flags(buf, libc::MSG_PEEK) + } +} + +#[stable(feature = "unix_socket", since = "1.10.0")] +impl AsRawFd for UnixDatagram { + #[inline] + fn as_raw_fd(&self) -> RawFd { + self.0.as_inner().as_raw_fd() + } +} + +#[stable(feature = "unix_socket", since = "1.10.0")] +impl FromRawFd for UnixDatagram { + #[inline] + unsafe fn from_raw_fd(fd: RawFd) -> UnixDatagram { + UnixDatagram(Socket::from_inner(FromInner::from_inner(OwnedFd::from_raw_fd(fd)))) + } +} + +#[stable(feature = "unix_socket", since = "1.10.0")] +impl IntoRawFd for UnixDatagram { + #[inline] + fn into_raw_fd(self) -> RawFd { + self.0.into_inner().into_inner().into_raw_fd() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsFd for UnixDatagram { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + self.0.as_inner().as_fd() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for OwnedFd { + #[inline] + fn from(unix_datagram: UnixDatagram) -> OwnedFd { + unsafe { OwnedFd::from_raw_fd(unix_datagram.into_raw_fd()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for UnixDatagram { + #[inline] + fn from(owned: OwnedFd) -> Self { + unsafe { Self::from_raw_fd(owned.into_raw_fd()) } + } +} diff --git a/crux-mir/lib/std/src/os/unix/net/listener.rs b/crux-mir/lib/std/src/os/unix/net/listener.rs new file mode 100644 index 000000000..02090afc8 --- /dev/null +++ b/crux-mir/lib/std/src/os/unix/net/listener.rs @@ -0,0 +1,391 @@ +use super::{sockaddr_un, SocketAddr, UnixStream}; +use crate::os::unix::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; +use crate::path::Path; +use crate::sys::cvt; +use crate::sys::net::Socket; +use crate::sys_common::{AsInner, FromInner, IntoInner}; +use crate::{fmt, io, mem}; + +/// A structure representing a Unix domain socket server. +/// +/// # Examples +/// +/// ```no_run +/// use std::thread; +/// use std::os::unix::net::{UnixStream, UnixListener}; +/// +/// fn handle_client(stream: UnixStream) { +/// // ... +/// } +/// +/// fn main() -> std::io::Result<()> { +/// let listener = UnixListener::bind("/path/to/the/socket")?; +/// +/// // accept connections and process them, spawning a new thread for each one +/// for stream in listener.incoming() { +/// match stream { +/// Ok(stream) => { +/// /* connection succeeded */ +/// thread::spawn(|| handle_client(stream)); +/// } +/// Err(err) => { +/// /* connection failed */ +/// break; +/// } +/// } +/// } +/// Ok(()) +/// } +/// ``` +#[stable(feature = "unix_socket", since = "1.10.0")] +pub struct UnixListener(Socket); + +#[stable(feature = "unix_socket", since = "1.10.0")] +impl fmt::Debug for UnixListener { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut builder = fmt.debug_struct("UnixListener"); + builder.field("fd", self.0.as_inner()); + if let Ok(addr) = self.local_addr() { + builder.field("local", &addr); + } + builder.finish() + } +} + +impl UnixListener { + /// Creates a new `UnixListener` bound to the specified socket. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixListener; + /// + /// let listener = match UnixListener::bind("/path/to/the/socket") { + /// Ok(sock) => sock, + /// Err(e) => { + /// println!("Couldn't connect: {e:?}"); + /// return + /// } + /// }; + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn bind>(path: P) -> io::Result { + unsafe { + let inner = Socket::new_raw(libc::AF_UNIX, libc::SOCK_STREAM)?; + let (addr, len) = sockaddr_un(path.as_ref())?; + const backlog: libc::c_int = + if cfg!(any(target_os = "linux", target_os = "freebsd")) { -1 } else { 128 }; + + cvt(libc::bind(inner.as_inner().as_raw_fd(), &addr as *const _ as *const _, len as _))?; + cvt(libc::listen(inner.as_inner().as_raw_fd(), backlog))?; + + Ok(UnixListener(inner)) + } + } + + /// Creates a new `UnixListener` bound to the specified [`socket address`]. + /// + /// [`socket address`]: crate::os::unix::net::SocketAddr + /// + /// # Examples + /// + /// ```no_run + /// #![feature(unix_socket_abstract)] + /// use std::os::unix::net::{UnixListener}; + /// + /// fn main() -> std::io::Result<()> { + /// let listener1 = UnixListener::bind("path/to/socket")?; + /// let addr = listener1.local_addr()?; + /// + /// let listener2 = match UnixListener::bind_addr(&addr) { + /// Ok(sock) => sock, + /// Err(err) => { + /// println!("Couldn't bind: {err:?}"); + /// return Err(err); + /// } + /// }; + /// Ok(()) + /// } + /// ``` + #[unstable(feature = "unix_socket_abstract", issue = "85410")] + pub fn bind_addr(socket_addr: &SocketAddr) -> io::Result { + unsafe { + let inner = Socket::new_raw(libc::AF_UNIX, libc::SOCK_STREAM)?; + #[cfg(target_os = "linux")] + const backlog: libc::c_int = -1; + #[cfg(not(target_os = "linux"))] + const backlog: libc::c_int = 128; + cvt(libc::bind( + inner.as_raw_fd(), + &socket_addr.addr as *const _ as *const _, + socket_addr.len as _, + ))?; + cvt(libc::listen(inner.as_raw_fd(), backlog))?; + Ok(UnixListener(inner)) + } + } + + /// Accepts a new incoming connection to this listener. + /// + /// This function will block the calling thread until a new Unix connection + /// is established. When established, the corresponding [`UnixStream`] and + /// the remote peer's address will be returned. + /// + /// [`UnixStream`]: crate::os::unix::net::UnixStream + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixListener; + /// + /// fn main() -> std::io::Result<()> { + /// let listener = UnixListener::bind("/path/to/the/socket")?; + /// + /// match listener.accept() { + /// Ok((socket, addr)) => println!("Got a client: {addr:?}"), + /// Err(e) => println!("accept function failed: {e:?}"), + /// } + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn accept(&self) -> io::Result<(UnixStream, SocketAddr)> { + let mut storage: libc::sockaddr_un = unsafe { mem::zeroed() }; + let mut len = mem::size_of_val(&storage) as libc::socklen_t; + let sock = self.0.accept(&mut storage as *mut _ as *mut _, &mut len)?; + let addr = SocketAddr::from_parts(storage, len)?; + Ok((UnixStream(sock), addr)) + } + + /// Creates a new independently owned handle to the underlying socket. + /// + /// The returned `UnixListener` is a reference to the same socket that this + /// object references. Both handles can be used to accept incoming + /// connections and options set on one listener will affect the other. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixListener; + /// + /// fn main() -> std::io::Result<()> { + /// let listener = UnixListener::bind("/path/to/the/socket")?; + /// let listener_copy = listener.try_clone().expect("try_clone failed"); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn try_clone(&self) -> io::Result { + self.0.duplicate().map(UnixListener) + } + + /// Returns the local socket address of this listener. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixListener; + /// + /// fn main() -> std::io::Result<()> { + /// let listener = UnixListener::bind("/path/to/the/socket")?; + /// let addr = listener.local_addr().expect("Couldn't get local address"); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn local_addr(&self) -> io::Result { + SocketAddr::new(|addr, len| unsafe { libc::getsockname(self.as_raw_fd(), addr, len) }) + } + + /// Moves the socket into or out of nonblocking mode. + /// + /// This will result in the `accept` operation becoming nonblocking, + /// i.e., immediately returning from their calls. If the IO operation is + /// successful, `Ok` is returned and no further action is required. If the + /// IO operation could not be completed and needs to be retried, an error + /// with kind [`io::ErrorKind::WouldBlock`] is returned. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixListener; + /// + /// fn main() -> std::io::Result<()> { + /// let listener = UnixListener::bind("/path/to/the/socket")?; + /// listener.set_nonblocking(true).expect("Couldn't set non blocking"); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()> { + self.0.set_nonblocking(nonblocking) + } + + /// Returns the value of the `SO_ERROR` option. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixListener; + /// + /// fn main() -> std::io::Result<()> { + /// let listener = UnixListener::bind("/tmp/sock")?; + /// + /// if let Ok(Some(err)) = listener.take_error() { + /// println!("Got error: {err:?}"); + /// } + /// Ok(()) + /// } + /// ``` + /// + /// # Platform specific + /// On Redox this always returns `None`. + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn take_error(&self) -> io::Result> { + self.0.take_error() + } + + /// Returns an iterator over incoming connections. + /// + /// The iterator will never return [`None`] and will also not yield the + /// peer's [`SocketAddr`] structure. + /// + /// # Examples + /// + /// ```no_run + /// use std::thread; + /// use std::os::unix::net::{UnixStream, UnixListener}; + /// + /// fn handle_client(stream: UnixStream) { + /// // ... + /// } + /// + /// fn main() -> std::io::Result<()> { + /// let listener = UnixListener::bind("/path/to/the/socket")?; + /// + /// for stream in listener.incoming() { + /// match stream { + /// Ok(stream) => { + /// thread::spawn(|| handle_client(stream)); + /// } + /// Err(err) => { + /// break; + /// } + /// } + /// } + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn incoming(&self) -> Incoming<'_> { + Incoming { listener: self } + } +} + +#[stable(feature = "unix_socket", since = "1.10.0")] +impl AsRawFd for UnixListener { + #[inline] + fn as_raw_fd(&self) -> RawFd { + self.0.as_inner().as_raw_fd() + } +} + +#[stable(feature = "unix_socket", since = "1.10.0")] +impl FromRawFd for UnixListener { + #[inline] + unsafe fn from_raw_fd(fd: RawFd) -> UnixListener { + UnixListener(Socket::from_inner(FromInner::from_inner(OwnedFd::from_raw_fd(fd)))) + } +} + +#[stable(feature = "unix_socket", since = "1.10.0")] +impl IntoRawFd for UnixListener { + #[inline] + fn into_raw_fd(self) -> RawFd { + self.0.into_inner().into_inner().into_raw_fd() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsFd for UnixListener { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + self.0.as_inner().as_fd() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for UnixListener { + #[inline] + fn from(fd: OwnedFd) -> UnixListener { + UnixListener(Socket::from_inner(FromInner::from_inner(fd))) + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for OwnedFd { + #[inline] + fn from(listener: UnixListener) -> OwnedFd { + listener.0.into_inner().into_inner() + } +} + +#[stable(feature = "unix_socket", since = "1.10.0")] +impl<'a> IntoIterator for &'a UnixListener { + type Item = io::Result; + type IntoIter = Incoming<'a>; + + fn into_iter(self) -> Incoming<'a> { + self.incoming() + } +} + +/// An iterator over incoming connections to a [`UnixListener`]. +/// +/// It will never return [`None`]. +/// +/// # Examples +/// +/// ```no_run +/// use std::thread; +/// use std::os::unix::net::{UnixStream, UnixListener}; +/// +/// fn handle_client(stream: UnixStream) { +/// // ... +/// } +/// +/// fn main() -> std::io::Result<()> { +/// let listener = UnixListener::bind("/path/to/the/socket")?; +/// +/// for stream in listener.incoming() { +/// match stream { +/// Ok(stream) => { +/// thread::spawn(|| handle_client(stream)); +/// } +/// Err(err) => { +/// break; +/// } +/// } +/// } +/// Ok(()) +/// } +/// ``` +#[derive(Debug)] +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "unix_socket", since = "1.10.0")] +pub struct Incoming<'a> { + listener: &'a UnixListener, +} + +#[stable(feature = "unix_socket", since = "1.10.0")] +impl<'a> Iterator for Incoming<'a> { + type Item = io::Result; + + fn next(&mut self) -> Option> { + Some(self.listener.accept().map(|s| s.0)) + } + + fn size_hint(&self) -> (usize, Option) { + (usize::MAX, None) + } +} diff --git a/crux-mir/lib/std/src/os/unix/net/mod.rs b/crux-mir/lib/std/src/os/unix/net/mod.rs new file mode 100644 index 000000000..6da3e350b --- /dev/null +++ b/crux-mir/lib/std/src/os/unix/net/mod.rs @@ -0,0 +1,26 @@ +//! Unix-specific networking functionality. + +#![allow(irrefutable_let_patterns)] +#![stable(feature = "unix_socket", since = "1.10.0")] + +mod addr; +#[doc(cfg(any(target_os = "android", target_os = "linux")))] +#[cfg(any(doc, target_os = "android", target_os = "linux"))] +mod ancillary; +mod datagram; +mod listener; +mod stream; +#[cfg(all(test, not(target_os = "emscripten")))] +mod tests; + +#[stable(feature = "unix_socket", since = "1.10.0")] +pub use self::addr::*; +#[cfg(any(doc, target_os = "android", target_os = "linux"))] +#[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] +pub use self::ancillary::*; +#[stable(feature = "unix_socket", since = "1.10.0")] +pub use self::datagram::*; +#[stable(feature = "unix_socket", since = "1.10.0")] +pub use self::listener::*; +#[stable(feature = "unix_socket", since = "1.10.0")] +pub use self::stream::*; diff --git a/crux-mir/lib/std/src/os/unix/net/stream.rs b/crux-mir/lib/std/src/os/unix/net/stream.rs new file mode 100644 index 000000000..dff8f6e85 --- /dev/null +++ b/crux-mir/lib/std/src/os/unix/net/stream.rs @@ -0,0 +1,736 @@ +#[cfg(any(doc, target_os = "android", target_os = "linux"))] +use super::{recv_vectored_with_ancillary_from, send_vectored_with_ancillary_to, SocketAncillary}; +use super::{sockaddr_un, SocketAddr}; +use crate::fmt; +use crate::io::{self, IoSlice, IoSliceMut}; +use crate::net::Shutdown; +use crate::os::unix::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; +#[cfg(any( + target_os = "android", + target_os = "linux", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "ios", + target_os = "macos", + target_os = "watchos", + target_os = "netbsd", + target_os = "openbsd" +))] +use crate::os::unix::ucred; +use crate::path::Path; +use crate::sys::cvt; +use crate::sys::net::Socket; +use crate::sys_common::{AsInner, FromInner}; +use crate::time::Duration; + +#[unstable(feature = "peer_credentials_unix_socket", issue = "42839", reason = "unstable")] +#[cfg(any( + target_os = "android", + target_os = "linux", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "ios", + target_os = "macos", + target_os = "watchos", + target_os = "netbsd", + target_os = "openbsd" +))] +pub use ucred::UCred; + +/// A Unix stream socket. +/// +/// # Examples +/// +/// ```no_run +/// use std::os::unix::net::UnixStream; +/// use std::io::prelude::*; +/// +/// fn main() -> std::io::Result<()> { +/// let mut stream = UnixStream::connect("/path/to/my/socket")?; +/// stream.write_all(b"hello world")?; +/// let mut response = String::new(); +/// stream.read_to_string(&mut response)?; +/// println!("{response}"); +/// Ok(()) +/// } +/// ``` +#[stable(feature = "unix_socket", since = "1.10.0")] +pub struct UnixStream(pub(super) Socket); + +#[stable(feature = "unix_socket", since = "1.10.0")] +impl fmt::Debug for UnixStream { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut builder = fmt.debug_struct("UnixStream"); + builder.field("fd", self.0.as_inner()); + if let Ok(addr) = self.local_addr() { + builder.field("local", &addr); + } + if let Ok(addr) = self.peer_addr() { + builder.field("peer", &addr); + } + builder.finish() + } +} + +impl UnixStream { + /// Connects to the socket named by `path`. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixStream; + /// + /// let socket = match UnixStream::connect("/tmp/sock") { + /// Ok(sock) => sock, + /// Err(e) => { + /// println!("Couldn't connect: {e:?}"); + /// return + /// } + /// }; + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn connect>(path: P) -> io::Result { + unsafe { + let inner = Socket::new_raw(libc::AF_UNIX, libc::SOCK_STREAM)?; + let (addr, len) = sockaddr_un(path.as_ref())?; + + cvt(libc::connect(inner.as_raw_fd(), &addr as *const _ as *const _, len))?; + Ok(UnixStream(inner)) + } + } + + /// Connects to the socket specified by [`address`]. + /// + /// [`address`]: crate::os::unix::net::SocketAddr + /// + /// # Examples + /// + /// ```no_run + /// #![feature(unix_socket_abstract)] + /// use std::os::unix::net::{UnixListener, UnixStream}; + /// + /// fn main() -> std::io::Result<()> { + /// let listener = UnixListener::bind("/path/to/the/socket")?; + /// let addr = listener.local_addr()?; + /// + /// let sock = match UnixStream::connect_addr(&addr) { + /// Ok(sock) => sock, + /// Err(e) => { + /// println!("Couldn't connect: {e:?}"); + /// return Err(e) + /// } + /// }; + /// Ok(()) + /// } + /// ```` + #[unstable(feature = "unix_socket_abstract", issue = "85410")] + pub fn connect_addr(socket_addr: &SocketAddr) -> io::Result { + unsafe { + let inner = Socket::new_raw(libc::AF_UNIX, libc::SOCK_STREAM)?; + cvt(libc::connect( + inner.as_raw_fd(), + &socket_addr.addr as *const _ as *const _, + socket_addr.len, + ))?; + Ok(UnixStream(inner)) + } + } + + /// Creates an unnamed pair of connected sockets. + /// + /// Returns two `UnixStream`s which are connected to each other. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixStream; + /// + /// let (sock1, sock2) = match UnixStream::pair() { + /// Ok((sock1, sock2)) => (sock1, sock2), + /// Err(e) => { + /// println!("Couldn't create a pair of sockets: {e:?}"); + /// return + /// } + /// }; + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn pair() -> io::Result<(UnixStream, UnixStream)> { + let (i1, i2) = Socket::new_pair(libc::AF_UNIX, libc::SOCK_STREAM)?; + Ok((UnixStream(i1), UnixStream(i2))) + } + + /// Creates a new independently owned handle to the underlying socket. + /// + /// The returned `UnixStream` is a reference to the same stream that this + /// object references. Both handles will read and write the same stream of + /// data, and options set on one stream will be propagated to the other + /// stream. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixStream; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixStream::connect("/tmp/sock")?; + /// let sock_copy = socket.try_clone().expect("Couldn't clone socket"); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn try_clone(&self) -> io::Result { + self.0.duplicate().map(UnixStream) + } + + /// Returns the socket address of the local half of this connection. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixStream; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixStream::connect("/tmp/sock")?; + /// let addr = socket.local_addr().expect("Couldn't get local address"); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn local_addr(&self) -> io::Result { + SocketAddr::new(|addr, len| unsafe { libc::getsockname(self.as_raw_fd(), addr, len) }) + } + + /// Returns the socket address of the remote half of this connection. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixStream; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixStream::connect("/tmp/sock")?; + /// let addr = socket.peer_addr().expect("Couldn't get peer address"); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn peer_addr(&self) -> io::Result { + SocketAddr::new(|addr, len| unsafe { libc::getpeername(self.as_raw_fd(), addr, len) }) + } + + /// Gets the peer credentials for this Unix domain socket. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(peer_credentials_unix_socket)] + /// use std::os::unix::net::UnixStream; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixStream::connect("/tmp/sock")?; + /// let peer_cred = socket.peer_cred().expect("Couldn't get peer credentials"); + /// Ok(()) + /// } + /// ``` + #[unstable(feature = "peer_credentials_unix_socket", issue = "42839", reason = "unstable")] + #[cfg(any( + target_os = "android", + target_os = "linux", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "ios", + target_os = "macos", + target_os = "watchos", + target_os = "netbsd", + target_os = "openbsd" + ))] + pub fn peer_cred(&self) -> io::Result { + ucred::peer_cred(self) + } + + /// Sets the read timeout for the socket. + /// + /// If the provided value is [`None`], then [`read`] calls will block + /// indefinitely. An [`Err`] is returned if the zero [`Duration`] is passed to this + /// method. + /// + /// [`read`]: io::Read::read + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixStream; + /// use std::time::Duration; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixStream::connect("/tmp/sock")?; + /// socket.set_read_timeout(Some(Duration::new(1, 0))).expect("Couldn't set read timeout"); + /// Ok(()) + /// } + /// ``` + /// + /// An [`Err`] is returned if the zero [`Duration`] is passed to this + /// method: + /// + /// ```no_run + /// use std::io; + /// use std::os::unix::net::UnixStream; + /// use std::time::Duration; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixStream::connect("/tmp/sock")?; + /// let result = socket.set_read_timeout(Some(Duration::new(0, 0))); + /// let err = result.unwrap_err(); + /// assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn set_read_timeout(&self, timeout: Option) -> io::Result<()> { + self.0.set_timeout(timeout, libc::SO_RCVTIMEO) + } + + /// Sets the write timeout for the socket. + /// + /// If the provided value is [`None`], then [`write`] calls will block + /// indefinitely. An [`Err`] is returned if the zero [`Duration`] is + /// passed to this method. + /// + /// [`read`]: io::Read::read + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixStream; + /// use std::time::Duration; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixStream::connect("/tmp/sock")?; + /// socket.set_write_timeout(Some(Duration::new(1, 0))) + /// .expect("Couldn't set write timeout"); + /// Ok(()) + /// } + /// ``` + /// + /// An [`Err`] is returned if the zero [`Duration`] is passed to this + /// method: + /// + /// ```no_run + /// use std::io; + /// use std::net::UdpSocket; + /// use std::time::Duration; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UdpSocket::bind("127.0.0.1:34254")?; + /// let result = socket.set_write_timeout(Some(Duration::new(0, 0))); + /// let err = result.unwrap_err(); + /// assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn set_write_timeout(&self, timeout: Option) -> io::Result<()> { + self.0.set_timeout(timeout, libc::SO_SNDTIMEO) + } + + /// Returns the read timeout of this socket. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixStream; + /// use std::time::Duration; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixStream::connect("/tmp/sock")?; + /// socket.set_read_timeout(Some(Duration::new(1, 0))).expect("Couldn't set read timeout"); + /// assert_eq!(socket.read_timeout()?, Some(Duration::new(1, 0))); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn read_timeout(&self) -> io::Result> { + self.0.timeout(libc::SO_RCVTIMEO) + } + + /// Returns the write timeout of this socket. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixStream; + /// use std::time::Duration; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixStream::connect("/tmp/sock")?; + /// socket.set_write_timeout(Some(Duration::new(1, 0))) + /// .expect("Couldn't set write timeout"); + /// assert_eq!(socket.write_timeout()?, Some(Duration::new(1, 0))); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn write_timeout(&self) -> io::Result> { + self.0.timeout(libc::SO_SNDTIMEO) + } + + /// Moves the socket into or out of nonblocking mode. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixStream; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixStream::connect("/tmp/sock")?; + /// socket.set_nonblocking(true).expect("Couldn't set nonblocking"); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()> { + self.0.set_nonblocking(nonblocking) + } + + /// Moves the socket to pass unix credentials as control message in [`SocketAncillary`]. + /// + /// Set the socket option `SO_PASSCRED`. + /// + /// # Examples + /// + #[cfg_attr(any(target_os = "android", target_os = "linux"), doc = "```no_run")] + #[cfg_attr(not(any(target_os = "android", target_os = "linux")), doc = "```ignore")] + /// #![feature(unix_socket_ancillary_data)] + /// use std::os::unix::net::UnixStream; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixStream::connect("/tmp/sock")?; + /// socket.set_passcred(true).expect("Couldn't set passcred"); + /// Ok(()) + /// } + /// ``` + #[cfg(any(doc, target_os = "android", target_os = "linux", target_os = "netbsd",))] + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn set_passcred(&self, passcred: bool) -> io::Result<()> { + self.0.set_passcred(passcred) + } + + /// Get the current value of the socket for passing unix credentials in [`SocketAncillary`]. + /// This value can be change by [`set_passcred`]. + /// + /// Get the socket option `SO_PASSCRED`. + /// + /// [`set_passcred`]: UnixStream::set_passcred + #[cfg(any(doc, target_os = "android", target_os = "linux", target_os = "netbsd",))] + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn passcred(&self) -> io::Result { + self.0.passcred() + } + + /// Set the id of the socket for network filtering purpose + /// + #[cfg_attr( + any(target_os = "linux", target_os = "freebsd", target_os = "openbsd"), + doc = "```no_run" + )] + #[cfg_attr( + not(any(target_os = "linux", target_os = "freebsd", target_os = "openbsd")), + doc = "```ignore" + )] + /// #![feature(unix_set_mark)] + /// use std::os::unix::net::UnixStream; + /// + /// fn main() -> std::io::Result<()> { + /// let sock = UnixStream::connect("/tmp/sock")?; + /// sock.set_mark(32)?; + /// Ok(()) + /// } + /// ``` + #[cfg(any(doc, target_os = "linux", target_os = "freebsd", target_os = "openbsd",))] + #[unstable(feature = "unix_set_mark", issue = "96467")] + pub fn set_mark(&self, mark: u32) -> io::Result<()> { + self.0.set_mark(mark) + } + + /// Returns the value of the `SO_ERROR` option. + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixStream; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixStream::connect("/tmp/sock")?; + /// if let Ok(Some(err)) = socket.take_error() { + /// println!("Got error: {err:?}"); + /// } + /// Ok(()) + /// } + /// ``` + /// + /// # Platform specific + /// On Redox this always returns `None`. + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn take_error(&self) -> io::Result> { + self.0.take_error() + } + + /// Shuts down the read, write, or both halves of this connection. + /// + /// This function will cause all pending and future I/O calls on the + /// specified portions to immediately return with an appropriate value + /// (see the documentation of [`Shutdown`]). + /// + /// # Examples + /// + /// ```no_run + /// use std::os::unix::net::UnixStream; + /// use std::net::Shutdown; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixStream::connect("/tmp/sock")?; + /// socket.shutdown(Shutdown::Both).expect("shutdown function failed"); + /// Ok(()) + /// } + /// ``` + #[stable(feature = "unix_socket", since = "1.10.0")] + pub fn shutdown(&self, how: Shutdown) -> io::Result<()> { + self.0.shutdown(how) + } + + /// Receives data on the socket from the remote address to which it is + /// connected, without removing that data from the queue. On success, + /// returns the number of bytes peeked. + /// + /// Successive calls return the same data. This is accomplished by passing + /// `MSG_PEEK` as a flag to the underlying `recv` system call. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(unix_socket_peek)] + /// + /// use std::os::unix::net::UnixStream; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixStream::connect("/tmp/sock")?; + /// let mut buf = [0; 10]; + /// let len = socket.peek(&mut buf).expect("peek failed"); + /// Ok(()) + /// } + /// ``` + #[unstable(feature = "unix_socket_peek", issue = "76923")] + pub fn peek(&self, buf: &mut [u8]) -> io::Result { + self.0.peek(buf) + } + + /// Receives data and ancillary data from socket. + /// + /// On success, returns the number of bytes read. + /// + /// # Examples + /// + #[cfg_attr(any(target_os = "android", target_os = "linux"), doc = "```no_run")] + #[cfg_attr(not(any(target_os = "android", target_os = "linux")), doc = "```ignore")] + /// #![feature(unix_socket_ancillary_data)] + /// use std::os::unix::net::{UnixStream, SocketAncillary, AncillaryData}; + /// use std::io::IoSliceMut; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixStream::connect("/tmp/sock")?; + /// let mut buf1 = [1; 8]; + /// let mut buf2 = [2; 16]; + /// let mut buf3 = [3; 8]; + /// let mut bufs = &mut [ + /// IoSliceMut::new(&mut buf1), + /// IoSliceMut::new(&mut buf2), + /// IoSliceMut::new(&mut buf3), + /// ][..]; + /// let mut fds = [0; 8]; + /// let mut ancillary_buffer = [0; 128]; + /// let mut ancillary = SocketAncillary::new(&mut ancillary_buffer[..]); + /// let size = socket.recv_vectored_with_ancillary(bufs, &mut ancillary)?; + /// println!("received {size}"); + /// for ancillary_result in ancillary.messages() { + /// if let AncillaryData::ScmRights(scm_rights) = ancillary_result.unwrap() { + /// for fd in scm_rights { + /// println!("receive file descriptor: {fd}"); + /// } + /// } + /// } + /// Ok(()) + /// } + /// ``` + #[cfg(any(doc, target_os = "android", target_os = "linux"))] + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn recv_vectored_with_ancillary( + &self, + bufs: &mut [IoSliceMut<'_>], + ancillary: &mut SocketAncillary<'_>, + ) -> io::Result { + let (count, _, _) = recv_vectored_with_ancillary_from(&self.0, bufs, ancillary)?; + + Ok(count) + } + + /// Sends data and ancillary data on the socket. + /// + /// On success, returns the number of bytes written. + /// + /// # Examples + /// + #[cfg_attr(any(target_os = "android", target_os = "linux"), doc = "```no_run")] + #[cfg_attr(not(any(target_os = "android", target_os = "linux")), doc = "```ignore")] + /// #![feature(unix_socket_ancillary_data)] + /// use std::os::unix::net::{UnixStream, SocketAncillary}; + /// use std::io::IoSlice; + /// + /// fn main() -> std::io::Result<()> { + /// let socket = UnixStream::connect("/tmp/sock")?; + /// let buf1 = [1; 8]; + /// let buf2 = [2; 16]; + /// let buf3 = [3; 8]; + /// let bufs = &[ + /// IoSlice::new(&buf1), + /// IoSlice::new(&buf2), + /// IoSlice::new(&buf3), + /// ][..]; + /// let fds = [0, 1, 2]; + /// let mut ancillary_buffer = [0; 128]; + /// let mut ancillary = SocketAncillary::new(&mut ancillary_buffer[..]); + /// ancillary.add_fds(&fds[..]); + /// socket.send_vectored_with_ancillary(bufs, &mut ancillary) + /// .expect("send_vectored_with_ancillary function failed"); + /// Ok(()) + /// } + /// ``` + #[cfg(any(doc, target_os = "android", target_os = "linux"))] + #[unstable(feature = "unix_socket_ancillary_data", issue = "76915")] + pub fn send_vectored_with_ancillary( + &self, + bufs: &[IoSlice<'_>], + ancillary: &mut SocketAncillary<'_>, + ) -> io::Result { + send_vectored_with_ancillary_to(&self.0, None, bufs, ancillary) + } +} + +#[stable(feature = "unix_socket", since = "1.10.0")] +impl io::Read for UnixStream { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + io::Read::read(&mut &*self, buf) + } + + fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { + io::Read::read_vectored(&mut &*self, bufs) + } + + #[inline] + fn is_read_vectored(&self) -> bool { + io::Read::is_read_vectored(&&*self) + } +} + +#[stable(feature = "unix_socket", since = "1.10.0")] +impl<'a> io::Read for &'a UnixStream { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.0.read(buf) + } + + fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { + self.0.read_vectored(bufs) + } + + #[inline] + fn is_read_vectored(&self) -> bool { + self.0.is_read_vectored() + } +} + +#[stable(feature = "unix_socket", since = "1.10.0")] +impl io::Write for UnixStream { + fn write(&mut self, buf: &[u8]) -> io::Result { + io::Write::write(&mut &*self, buf) + } + + fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { + io::Write::write_vectored(&mut &*self, bufs) + } + + #[inline] + fn is_write_vectored(&self) -> bool { + io::Write::is_write_vectored(&&*self) + } + + fn flush(&mut self) -> io::Result<()> { + io::Write::flush(&mut &*self) + } +} + +#[stable(feature = "unix_socket", since = "1.10.0")] +impl<'a> io::Write for &'a UnixStream { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.0.write(buf) + } + + fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { + self.0.write_vectored(bufs) + } + + #[inline] + fn is_write_vectored(&self) -> bool { + self.0.is_write_vectored() + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +#[stable(feature = "unix_socket", since = "1.10.0")] +impl AsRawFd for UnixStream { + #[inline] + fn as_raw_fd(&self) -> RawFd { + self.0.as_raw_fd() + } +} + +#[stable(feature = "unix_socket", since = "1.10.0")] +impl FromRawFd for UnixStream { + #[inline] + unsafe fn from_raw_fd(fd: RawFd) -> UnixStream { + UnixStream(Socket::from_inner(FromInner::from_inner(OwnedFd::from_raw_fd(fd)))) + } +} + +#[stable(feature = "unix_socket", since = "1.10.0")] +impl IntoRawFd for UnixStream { + #[inline] + fn into_raw_fd(self) -> RawFd { + self.0.into_raw_fd() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsFd for UnixStream { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + self.0.as_fd() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for OwnedFd { + #[inline] + fn from(unix_stream: UnixStream) -> OwnedFd { + unsafe { OwnedFd::from_raw_fd(unix_stream.into_raw_fd()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for UnixStream { + #[inline] + fn from(owned: OwnedFd) -> Self { + unsafe { Self::from_raw_fd(owned.into_raw_fd()) } + } +} diff --git a/crux-mir/lib/std/src/os/unix/net/tests.rs b/crux-mir/lib/std/src/os/unix/net/tests.rs new file mode 100644 index 000000000..37fcfa844 --- /dev/null +++ b/crux-mir/lib/std/src/os/unix/net/tests.rs @@ -0,0 +1,759 @@ +use super::*; +use crate::io::prelude::*; +use crate::io::{self, ErrorKind, IoSlice, IoSliceMut}; +#[cfg(any(target_os = "android", target_os = "linux"))] +use crate::os::unix::io::AsRawFd; +use crate::sys_common::io::test::tmpdir; +use crate::thread; +use crate::time::Duration; + +#[cfg(target_os = "android")] +use crate::os::android::net::SocketAddrExt; + +#[cfg(target_os = "linux")] +use crate::os::linux::net::SocketAddrExt; + +macro_rules! or_panic { + ($e:expr) => { + match $e { + Ok(e) => e, + Err(e) => panic!("{e}"), + } + }; +} + +#[test] +fn basic() { + let dir = tmpdir(); + let socket_path = dir.path().join("sock"); + let msg1 = b"hello"; + let msg2 = b"world!"; + + let listener = or_panic!(UnixListener::bind(&socket_path)); + let thread = thread::spawn(move || { + let mut stream = or_panic!(listener.accept()).0; + let mut buf = [0; 5]; + or_panic!(stream.read(&mut buf)); + assert_eq!(&msg1[..], &buf[..]); + or_panic!(stream.write_all(msg2)); + }); + + let mut stream = or_panic!(UnixStream::connect(&socket_path)); + assert_eq!(Some(&*socket_path), stream.peer_addr().unwrap().as_pathname()); + or_panic!(stream.write_all(msg1)); + let mut buf = vec![]; + or_panic!(stream.read_to_end(&mut buf)); + assert_eq!(&msg2[..], &buf[..]); + drop(stream); + + thread.join().unwrap(); +} + +#[test] +fn vectored() { + let (mut s1, mut s2) = or_panic!(UnixStream::pair()); + + let len = or_panic!(s1.write_vectored(&[ + IoSlice::new(b"hello"), + IoSlice::new(b" "), + IoSlice::new(b"world!") + ],)); + assert_eq!(len, 12); + + let mut buf1 = [0; 6]; + let mut buf2 = [0; 7]; + let len = + or_panic!(s2.read_vectored(&mut [IoSliceMut::new(&mut buf1), IoSliceMut::new(&mut buf2)],)); + assert_eq!(len, 12); + assert_eq!(&buf1, b"hello "); + assert_eq!(&buf2, b"world!\0"); +} + +#[test] +fn pair() { + let msg1 = b"hello"; + let msg2 = b"world!"; + + let (mut s1, mut s2) = or_panic!(UnixStream::pair()); + let thread = thread::spawn(move || { + // s1 must be moved in or the test will hang! + let mut buf = [0; 5]; + or_panic!(s1.read(&mut buf)); + assert_eq!(&msg1[..], &buf[..]); + or_panic!(s1.write_all(msg2)); + }); + + or_panic!(s2.write_all(msg1)); + let mut buf = vec![]; + or_panic!(s2.read_to_end(&mut buf)); + assert_eq!(&msg2[..], &buf[..]); + drop(s2); + + thread.join().unwrap(); +} + +#[test] +fn try_clone() { + let dir = tmpdir(); + let socket_path = dir.path().join("sock"); + let msg1 = b"hello"; + let msg2 = b"world"; + + let listener = or_panic!(UnixListener::bind(&socket_path)); + let thread = thread::spawn(move || { + let mut stream = or_panic!(listener.accept()).0; + or_panic!(stream.write_all(msg1)); + or_panic!(stream.write_all(msg2)); + }); + + let mut stream = or_panic!(UnixStream::connect(&socket_path)); + let mut stream2 = or_panic!(stream.try_clone()); + + let mut buf = [0; 5]; + or_panic!(stream.read(&mut buf)); + assert_eq!(&msg1[..], &buf[..]); + or_panic!(stream2.read(&mut buf)); + assert_eq!(&msg2[..], &buf[..]); + + thread.join().unwrap(); +} + +#[test] +fn iter() { + let dir = tmpdir(); + let socket_path = dir.path().join("sock"); + + let listener = or_panic!(UnixListener::bind(&socket_path)); + let thread = thread::spawn(move || { + for stream in listener.incoming().take(2) { + let mut stream = or_panic!(stream); + let mut buf = [0]; + or_panic!(stream.read(&mut buf)); + } + }); + + for _ in 0..2 { + let mut stream = or_panic!(UnixStream::connect(&socket_path)); + or_panic!(stream.write_all(&[0])); + } + + thread.join().unwrap(); +} + +#[test] +fn long_path() { + let dir = tmpdir(); + let socket_path = dir.path().join( + "asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfa\ + sasdfasdfasdasdfasdfasdfadfasdfasdfasdfasdfasdf", + ); + match UnixStream::connect(&socket_path) { + Err(ref e) if e.kind() == io::ErrorKind::InvalidInput => {} + Err(e) => panic!("unexpected error {e}"), + Ok(_) => panic!("unexpected success"), + } + + match UnixListener::bind(&socket_path) { + Err(ref e) if e.kind() == io::ErrorKind::InvalidInput => {} + Err(e) => panic!("unexpected error {e}"), + Ok(_) => panic!("unexpected success"), + } + + match UnixDatagram::bind(&socket_path) { + Err(ref e) if e.kind() == io::ErrorKind::InvalidInput => {} + Err(e) => panic!("unexpected error {e}"), + Ok(_) => panic!("unexpected success"), + } +} + +#[test] +fn timeouts() { + let dir = tmpdir(); + let socket_path = dir.path().join("sock"); + + let _listener = or_panic!(UnixListener::bind(&socket_path)); + + let stream = or_panic!(UnixStream::connect(&socket_path)); + let dur = Duration::new(15410, 0); + + assert_eq!(None, or_panic!(stream.read_timeout())); + + or_panic!(stream.set_read_timeout(Some(dur))); + assert_eq!(Some(dur), or_panic!(stream.read_timeout())); + + assert_eq!(None, or_panic!(stream.write_timeout())); + + or_panic!(stream.set_write_timeout(Some(dur))); + assert_eq!(Some(dur), or_panic!(stream.write_timeout())); + + or_panic!(stream.set_read_timeout(None)); + assert_eq!(None, or_panic!(stream.read_timeout())); + + or_panic!(stream.set_write_timeout(None)); + assert_eq!(None, or_panic!(stream.write_timeout())); +} + +#[test] +fn test_read_timeout() { + let dir = tmpdir(); + let socket_path = dir.path().join("sock"); + + let _listener = or_panic!(UnixListener::bind(&socket_path)); + + let mut stream = or_panic!(UnixStream::connect(&socket_path)); + or_panic!(stream.set_read_timeout(Some(Duration::from_millis(1000)))); + + let mut buf = [0; 10]; + let kind = stream.read_exact(&mut buf).err().expect("expected error").kind(); + assert!( + kind == ErrorKind::WouldBlock || kind == ErrorKind::TimedOut, + "unexpected_error: {:?}", + kind + ); +} + +#[test] +fn test_read_with_timeout() { + let dir = tmpdir(); + let socket_path = dir.path().join("sock"); + + let listener = or_panic!(UnixListener::bind(&socket_path)); + + let mut stream = or_panic!(UnixStream::connect(&socket_path)); + or_panic!(stream.set_read_timeout(Some(Duration::from_millis(1000)))); + + let mut other_end = or_panic!(listener.accept()).0; + or_panic!(other_end.write_all(b"hello world")); + + let mut buf = [0; 11]; + or_panic!(stream.read(&mut buf)); + assert_eq!(b"hello world", &buf[..]); + + let kind = stream.read_exact(&mut buf).err().expect("expected error").kind(); + assert!( + kind == ErrorKind::WouldBlock || kind == ErrorKind::TimedOut, + "unexpected_error: {:?}", + kind + ); +} + +// Ensure the `set_read_timeout` and `set_write_timeout` calls return errors +// when passed zero Durations +#[test] +fn test_unix_stream_timeout_zero_duration() { + let dir = tmpdir(); + let socket_path = dir.path().join("sock"); + + let listener = or_panic!(UnixListener::bind(&socket_path)); + let stream = or_panic!(UnixStream::connect(&socket_path)); + + let result = stream.set_write_timeout(Some(Duration::new(0, 0))); + let err = result.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidInput); + + let result = stream.set_read_timeout(Some(Duration::new(0, 0))); + let err = result.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidInput); + + drop(listener); +} + +#[test] +fn test_unix_datagram() { + let dir = tmpdir(); + let path1 = dir.path().join("sock1"); + let path2 = dir.path().join("sock2"); + + let sock1 = or_panic!(UnixDatagram::bind(&path1)); + let sock2 = or_panic!(UnixDatagram::bind(&path2)); + + let msg = b"hello world"; + or_panic!(sock1.send_to(msg, &path2)); + let mut buf = [0; 11]; + or_panic!(sock2.recv_from(&mut buf)); + assert_eq!(msg, &buf[..]); +} + +#[test] +fn test_unnamed_unix_datagram() { + let dir = tmpdir(); + let path1 = dir.path().join("sock1"); + + let sock1 = or_panic!(UnixDatagram::bind(&path1)); + let sock2 = or_panic!(UnixDatagram::unbound()); + + let msg = b"hello world"; + or_panic!(sock2.send_to(msg, &path1)); + let mut buf = [0; 11]; + let (usize, addr) = or_panic!(sock1.recv_from(&mut buf)); + assert_eq!(usize, 11); + assert!(addr.is_unnamed()); + assert_eq!(msg, &buf[..]); +} + +#[test] +fn test_unix_datagram_connect_to_recv_addr() { + let dir = tmpdir(); + let path1 = dir.path().join("sock1"); + let path2 = dir.path().join("sock2"); + + let sock1 = or_panic!(UnixDatagram::bind(&path1)); + let sock2 = or_panic!(UnixDatagram::bind(&path2)); + + let msg = b"hello world"; + let sock1_addr = or_panic!(sock1.local_addr()); + or_panic!(sock2.send_to_addr(msg, &sock1_addr)); + let mut buf = [0; 11]; + let (_, addr) = or_panic!(sock1.recv_from(&mut buf)); + + let new_msg = b"hello back"; + let mut new_buf = [0; 10]; + or_panic!(sock2.connect_addr(&addr)); + or_panic!(sock2.send(new_msg)); // set by connect_addr + let usize = or_panic!(sock2.recv(&mut new_buf)); + assert_eq!(usize, 10); + assert_eq!(new_msg, &new_buf[..]); +} + +#[test] +fn test_connect_unix_datagram() { + let dir = tmpdir(); + let path1 = dir.path().join("sock1"); + let path2 = dir.path().join("sock2"); + + let bsock1 = or_panic!(UnixDatagram::bind(&path1)); + let bsock2 = or_panic!(UnixDatagram::bind(&path2)); + let sock = or_panic!(UnixDatagram::unbound()); + or_panic!(sock.connect(&path1)); + + // Check send() + let msg = b"hello there"; + or_panic!(sock.send(msg)); + let mut buf = [0; 11]; + let (usize, addr) = or_panic!(bsock1.recv_from(&mut buf)); + assert_eq!(usize, 11); + assert!(addr.is_unnamed()); + assert_eq!(msg, &buf[..]); + + // Changing default socket works too + or_panic!(sock.connect(&path2)); + or_panic!(sock.send(msg)); + or_panic!(bsock2.recv_from(&mut buf)); +} + +#[test] +fn test_unix_datagram_recv() { + let dir = tmpdir(); + let path1 = dir.path().join("sock1"); + + let sock1 = or_panic!(UnixDatagram::bind(&path1)); + let sock2 = or_panic!(UnixDatagram::unbound()); + or_panic!(sock2.connect(&path1)); + + let msg = b"hello world"; + or_panic!(sock2.send(msg)); + let mut buf = [0; 11]; + let size = or_panic!(sock1.recv(&mut buf)); + assert_eq!(size, 11); + assert_eq!(msg, &buf[..]); +} + +#[test] +fn datagram_pair() { + let msg1 = b"hello"; + let msg2 = b"world!"; + + let (s1, s2) = or_panic!(UnixDatagram::pair()); + let thread = thread::spawn(move || { + // s1 must be moved in or the test will hang! + let mut buf = [0; 5]; + or_panic!(s1.recv(&mut buf)); + assert_eq!(&msg1[..], &buf[..]); + or_panic!(s1.send(msg2)); + }); + + or_panic!(s2.send(msg1)); + let mut buf = [0; 6]; + or_panic!(s2.recv(&mut buf)); + assert_eq!(&msg2[..], &buf[..]); + drop(s2); + + thread.join().unwrap(); +} + +// Ensure the `set_read_timeout` and `set_write_timeout` calls return errors +// when passed zero Durations +#[test] +fn test_unix_datagram_timeout_zero_duration() { + let dir = tmpdir(); + let path = dir.path().join("sock"); + + let datagram = or_panic!(UnixDatagram::bind(&path)); + + let result = datagram.set_write_timeout(Some(Duration::new(0, 0))); + let err = result.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidInput); + + let result = datagram.set_read_timeout(Some(Duration::new(0, 0))); + let err = result.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidInput); +} + +#[test] +fn abstract_namespace_not_allowed_connect() { + assert!(UnixStream::connect("\0asdf").is_err()); +} + +#[cfg(any(target_os = "android", target_os = "linux"))] +#[test] +fn test_abstract_stream_connect() { + let msg1 = b"hello"; + let msg2 = b"world"; + + let socket_addr = or_panic!(SocketAddr::from_abstract_name(b"name")); + let listener = or_panic!(UnixListener::bind_addr(&socket_addr)); + + let thread = thread::spawn(move || { + let mut stream = or_panic!(listener.accept()).0; + let mut buf = [0; 5]; + or_panic!(stream.read(&mut buf)); + assert_eq!(&msg1[..], &buf[..]); + or_panic!(stream.write_all(msg2)); + }); + + let mut stream = or_panic!(UnixStream::connect_addr(&socket_addr)); + + let peer = or_panic!(stream.peer_addr()); + assert_eq!(peer.as_abstract_name().unwrap(), b"name"); + + or_panic!(stream.write_all(msg1)); + let mut buf = vec![]; + or_panic!(stream.read_to_end(&mut buf)); + assert_eq!(&msg2[..], &buf[..]); + drop(stream); + + thread.join().unwrap(); +} + +#[cfg(any(target_os = "android", target_os = "linux"))] +#[test] +fn test_abstract_stream_iter() { + let addr = or_panic!(SocketAddr::from_abstract_name(b"hidden")); + let listener = or_panic!(UnixListener::bind_addr(&addr)); + + let thread = thread::spawn(move || { + for stream in listener.incoming().take(2) { + let mut stream = or_panic!(stream); + let mut buf = [0]; + or_panic!(stream.read(&mut buf)); + } + }); + + for _ in 0..2 { + let mut stream = or_panic!(UnixStream::connect_addr(&addr)); + or_panic!(stream.write_all(&[0])); + } + + thread.join().unwrap(); +} + +#[cfg(any(target_os = "android", target_os = "linux"))] +#[test] +fn test_abstract_datagram_bind_send_to_addr() { + let addr1 = or_panic!(SocketAddr::from_abstract_name(b"ns1")); + let sock1 = or_panic!(UnixDatagram::bind_addr(&addr1)); + + let local = or_panic!(sock1.local_addr()); + assert_eq!(local.as_abstract_name().unwrap(), b"ns1"); + + let addr2 = or_panic!(SocketAddr::from_abstract_name(b"ns2")); + let sock2 = or_panic!(UnixDatagram::bind_addr(&addr2)); + + let msg = b"hello world"; + or_panic!(sock1.send_to_addr(msg, &addr2)); + let mut buf = [0; 11]; + let (len, addr) = or_panic!(sock2.recv_from(&mut buf)); + assert_eq!(msg, &buf[..]); + assert_eq!(len, 11); + assert_eq!(addr.as_abstract_name().unwrap(), b"ns1"); +} + +#[cfg(any(target_os = "android", target_os = "linux"))] +#[test] +fn test_abstract_datagram_connect_addr() { + let addr1 = or_panic!(SocketAddr::from_abstract_name(b"ns3")); + let bsock1 = or_panic!(UnixDatagram::bind_addr(&addr1)); + + let sock = or_panic!(UnixDatagram::unbound()); + or_panic!(sock.connect_addr(&addr1)); + + let msg = b"hello world"; + or_panic!(sock.send(msg)); + let mut buf = [0; 11]; + let (len, addr) = or_panic!(bsock1.recv_from(&mut buf)); + assert_eq!(len, 11); + assert_eq!(addr.is_unnamed(), true); + assert_eq!(msg, &buf[..]); + + let addr2 = or_panic!(SocketAddr::from_abstract_name(b"ns4")); + let bsock2 = or_panic!(UnixDatagram::bind_addr(&addr2)); + + or_panic!(sock.connect_addr(&addr2)); + or_panic!(sock.send(msg)); + or_panic!(bsock2.recv_from(&mut buf)); +} + +#[cfg(any(target_os = "android", target_os = "linux"))] +#[test] +fn test_abstract_name_too_long() { + match SocketAddr::from_abstract_name( + b"abcdefghijklmnopqrstuvwxyzabcdefghijklmn\ + opqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghi\ + jklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz", + ) { + Err(ref e) if e.kind() == io::ErrorKind::InvalidInput => {} + Err(e) => panic!("unexpected error {e}"), + Ok(_) => panic!("unexpected success"), + } +} + +#[cfg(any(target_os = "android", target_os = "linux"))] +#[test] +fn test_abstract_no_pathname_and_not_unnamed() { + let name = b"local"; + let addr = or_panic!(SocketAddr::from_abstract_name(name)); + assert_eq!(addr.as_pathname(), None); + assert_eq!(addr.as_abstract_name(), Some(&name[..])); + assert_eq!(addr.is_unnamed(), false); +} + +#[test] +fn test_unix_stream_peek() { + let (txdone, rxdone) = crate::sync::mpsc::channel(); + + let dir = tmpdir(); + let path = dir.path().join("sock"); + + let listener = or_panic!(UnixListener::bind(&path)); + let thread = thread::spawn(move || { + let mut stream = or_panic!(listener.accept()).0; + or_panic!(stream.write_all(&[1, 3, 3, 7])); + or_panic!(rxdone.recv()); + }); + + let mut stream = or_panic!(UnixStream::connect(&path)); + let mut buf = [0; 10]; + for _ in 0..2 { + assert_eq!(or_panic!(stream.peek(&mut buf)), 4); + } + assert_eq!(or_panic!(stream.read(&mut buf)), 4); + + or_panic!(stream.set_nonblocking(true)); + match stream.peek(&mut buf) { + Ok(_) => panic!("expected error"), + Err(ref e) if e.kind() == ErrorKind::WouldBlock => {} + Err(e) => panic!("unexpected error: {e}"), + } + + or_panic!(txdone.send(())); + thread.join().unwrap(); +} + +#[test] +fn test_unix_datagram_peek() { + let dir = tmpdir(); + let path1 = dir.path().join("sock"); + + let sock1 = or_panic!(UnixDatagram::bind(&path1)); + let sock2 = or_panic!(UnixDatagram::unbound()); + or_panic!(sock2.connect(&path1)); + + let msg = b"hello world"; + or_panic!(sock2.send(msg)); + for _ in 0..2 { + let mut buf = [0; 11]; + let size = or_panic!(sock1.peek(&mut buf)); + assert_eq!(size, 11); + assert_eq!(msg, &buf[..]); + } + + let mut buf = [0; 11]; + let size = or_panic!(sock1.recv(&mut buf)); + assert_eq!(size, 11); + assert_eq!(msg, &buf[..]); +} + +#[test] +fn test_unix_datagram_peek_from() { + let dir = tmpdir(); + let path1 = dir.path().join("sock"); + + let sock1 = or_panic!(UnixDatagram::bind(&path1)); + let sock2 = or_panic!(UnixDatagram::unbound()); + or_panic!(sock2.connect(&path1)); + + let msg = b"hello world"; + or_panic!(sock2.send(msg)); + for _ in 0..2 { + let mut buf = [0; 11]; + let (size, _) = or_panic!(sock1.peek_from(&mut buf)); + assert_eq!(size, 11); + assert_eq!(msg, &buf[..]); + } + + let mut buf = [0; 11]; + let size = or_panic!(sock1.recv(&mut buf)); + assert_eq!(size, 11); + assert_eq!(msg, &buf[..]); +} + +#[cfg(any(target_os = "android", target_os = "linux"))] +#[test] +fn test_send_vectored_fds_unix_stream() { + let (s1, s2) = or_panic!(UnixStream::pair()); + + let buf1 = [1; 8]; + let bufs_send = &[IoSlice::new(&buf1[..])][..]; + + let mut ancillary1_buffer = [0; 128]; + let mut ancillary1 = SocketAncillary::new(&mut ancillary1_buffer[..]); + assert!(ancillary1.add_fds(&[s1.as_raw_fd()][..])); + + let usize = or_panic!(s1.send_vectored_with_ancillary(&bufs_send, &mut ancillary1)); + assert_eq!(usize, 8); + + let mut buf2 = [0; 8]; + let mut bufs_recv = &mut [IoSliceMut::new(&mut buf2[..])][..]; + + let mut ancillary2_buffer = [0; 128]; + let mut ancillary2 = SocketAncillary::new(&mut ancillary2_buffer[..]); + + let usize = or_panic!(s2.recv_vectored_with_ancillary(&mut bufs_recv, &mut ancillary2)); + assert_eq!(usize, 8); + assert_eq!(buf1, buf2); + + let mut ancillary_data_vec = Vec::from_iter(ancillary2.messages()); + assert_eq!(ancillary_data_vec.len(), 1); + if let AncillaryData::ScmRights(scm_rights) = ancillary_data_vec.pop().unwrap().unwrap() { + let fd_vec = Vec::from_iter(scm_rights); + assert_eq!(fd_vec.len(), 1); + unsafe { + libc::close(fd_vec[0]); + } + } else { + unreachable!("must be ScmRights"); + } +} + +#[cfg(any(target_os = "android", target_os = "linux",))] +#[test] +fn test_send_vectored_with_ancillary_to_unix_datagram() { + fn getpid() -> libc::pid_t { + unsafe { libc::getpid() } + } + + fn getuid() -> libc::uid_t { + unsafe { libc::getuid() } + } + + fn getgid() -> libc::gid_t { + unsafe { libc::getgid() } + } + + let dir = tmpdir(); + let path1 = dir.path().join("sock1"); + let path2 = dir.path().join("sock2"); + + let bsock1 = or_panic!(UnixDatagram::bind(&path1)); + let bsock2 = or_panic!(UnixDatagram::bind(&path2)); + + or_panic!(bsock2.set_passcred(true)); + + let buf1 = [1; 8]; + let bufs_send = &[IoSlice::new(&buf1[..])][..]; + + let mut ancillary1_buffer = [0; 128]; + let mut ancillary1 = SocketAncillary::new(&mut ancillary1_buffer[..]); + let mut cred1 = SocketCred::new(); + cred1.set_pid(getpid()); + cred1.set_uid(getuid()); + cred1.set_gid(getgid()); + assert!(ancillary1.add_creds(&[cred1.clone()][..])); + + let usize = + or_panic!(bsock1.send_vectored_with_ancillary_to(&bufs_send, &mut ancillary1, &path2)); + assert_eq!(usize, 8); + + let mut buf2 = [0; 8]; + let mut bufs_recv = &mut [IoSliceMut::new(&mut buf2[..])][..]; + + let mut ancillary2_buffer = [0; 128]; + let mut ancillary2 = SocketAncillary::new(&mut ancillary2_buffer[..]); + + let (usize, truncated, _addr) = + or_panic!(bsock2.recv_vectored_with_ancillary_from(&mut bufs_recv, &mut ancillary2)); + assert_eq!(ancillary2.truncated(), false); + assert_eq!(usize, 8); + assert_eq!(truncated, false); + assert_eq!(buf1, buf2); + + let mut ancillary_data_vec = Vec::from_iter(ancillary2.messages()); + assert_eq!(ancillary_data_vec.len(), 1); + if let AncillaryData::ScmCredentials(scm_credentials) = + ancillary_data_vec.pop().unwrap().unwrap() + { + let cred_vec = Vec::from_iter(scm_credentials); + assert_eq!(cred_vec.len(), 1); + assert_eq!(cred1.get_pid(), cred_vec[0].get_pid()); + assert_eq!(cred1.get_uid(), cred_vec[0].get_uid()); + assert_eq!(cred1.get_gid(), cred_vec[0].get_gid()); + } else { + unreachable!("must be ScmCredentials"); + } +} + +#[cfg(any(target_os = "android", target_os = "linux"))] +#[test] +fn test_send_vectored_with_ancillary_unix_datagram() { + let dir = tmpdir(); + let path1 = dir.path().join("sock1"); + let path2 = dir.path().join("sock2"); + + let bsock1 = or_panic!(UnixDatagram::bind(&path1)); + let bsock2 = or_panic!(UnixDatagram::bind(&path2)); + + let buf1 = [1; 8]; + let bufs_send = &[IoSlice::new(&buf1[..])][..]; + + let mut ancillary1_buffer = [0; 128]; + let mut ancillary1 = SocketAncillary::new(&mut ancillary1_buffer[..]); + assert!(ancillary1.add_fds(&[bsock1.as_raw_fd()][..])); + + or_panic!(bsock1.connect(&path2)); + let usize = or_panic!(bsock1.send_vectored_with_ancillary(&bufs_send, &mut ancillary1)); + assert_eq!(usize, 8); + + let mut buf2 = [0; 8]; + let mut bufs_recv = &mut [IoSliceMut::new(&mut buf2[..])][..]; + + let mut ancillary2_buffer = [0; 128]; + let mut ancillary2 = SocketAncillary::new(&mut ancillary2_buffer[..]); + + let (usize, truncated) = + or_panic!(bsock2.recv_vectored_with_ancillary(&mut bufs_recv, &mut ancillary2)); + assert_eq!(usize, 8); + assert_eq!(truncated, false); + assert_eq!(buf1, buf2); + + let mut ancillary_data_vec = Vec::from_iter(ancillary2.messages()); + assert_eq!(ancillary_data_vec.len(), 1); + if let AncillaryData::ScmRights(scm_rights) = ancillary_data_vec.pop().unwrap().unwrap() { + let fd_vec = Vec::from_iter(scm_rights); + assert_eq!(fd_vec.len(), 1); + unsafe { + libc::close(fd_vec[0]); + } + } else { + unreachable!("must be ScmRights"); + } +} diff --git a/crux-mir/lib/std/src/os/unix/process.rs b/crux-mir/lib/std/src/os/unix/process.rs new file mode 100644 index 000000000..09b2bfe39 --- /dev/null +++ b/crux-mir/lib/std/src/os/unix/process.rs @@ -0,0 +1,466 @@ +//! Unix-specific extensions to primitives in the [`std::process`] module. +//! +//! [`std::process`]: crate::process + +#![stable(feature = "rust1", since = "1.0.0")] + +use crate::ffi::OsStr; +use crate::io; +use crate::os::unix::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; +use crate::process; +use crate::sealed::Sealed; +use crate::sys; +use crate::sys_common::{AsInner, AsInnerMut, FromInner, IntoInner}; + +#[cfg(not(any(target_os = "vxworks", target_os = "espidf", target_os = "horizon")))] +type UserId = u32; +#[cfg(not(any(target_os = "vxworks", target_os = "espidf", target_os = "horizon")))] +type GroupId = u32; + +#[cfg(any(target_os = "vxworks", target_os = "espidf", target_os = "horizon"))] +type UserId = u16; +#[cfg(any(target_os = "vxworks", target_os = "espidf", target_os = "horizon"))] +type GroupId = u16; + +/// Unix-specific extensions to the [`process::Command`] builder. +/// +/// This trait is sealed: it cannot be implemented outside the standard library. +/// This is so that future additional methods are not breaking changes. +#[stable(feature = "rust1", since = "1.0.0")] +pub trait CommandExt: Sealed { + /// Sets the child process's user ID. This translates to a + /// `setuid` call in the child process. Failure in the `setuid` + /// call will cause the spawn to fail. + #[stable(feature = "rust1", since = "1.0.0")] + fn uid(&mut self, id: UserId) -> &mut process::Command; + + /// Similar to `uid`, but sets the group ID of the child process. This has + /// the same semantics as the `uid` field. + #[stable(feature = "rust1", since = "1.0.0")] + fn gid(&mut self, id: GroupId) -> &mut process::Command; + + /// Sets the supplementary group IDs for the calling process. Translates to + /// a `setgroups` call in the child process. + #[unstable(feature = "setgroups", issue = "90747")] + fn groups(&mut self, groups: &[GroupId]) -> &mut process::Command; + + /// Schedules a closure to be run just before the `exec` function is + /// invoked. + /// + /// The closure is allowed to return an I/O error whose OS error code will + /// be communicated back to the parent and returned as an error from when + /// the spawn was requested. + /// + /// Multiple closures can be registered and they will be called in order of + /// their registration. If a closure returns `Err` then no further closures + /// will be called and the spawn operation will immediately return with a + /// failure. + /// + /// # Notes and Safety + /// + /// This closure will be run in the context of the child process after a + /// `fork`. This primarily means that any modifications made to memory on + /// behalf of this closure will **not** be visible to the parent process. + /// This is often a very constrained environment where normal operations + /// like `malloc`, accessing environment variables through [`std::env`] + /// or acquiring a mutex are not guaranteed to work (due to + /// other threads perhaps still running when the `fork` was run). + /// + /// For further details refer to the [POSIX fork() specification] + /// and the equivalent documentation for any targeted + /// platform, especially the requirements around *async-signal-safety*. + /// + /// This also means that all resources such as file descriptors and + /// memory-mapped regions got duplicated. It is your responsibility to make + /// sure that the closure does not violate library invariants by making + /// invalid use of these duplicates. + /// + /// Panicking in the closure is safe only if all the format arguments for the + /// panic message can be safely formatted; this is because although + /// `Command` calls [`std::panic::always_abort`](crate::panic::always_abort) + /// before calling the pre_exec hook, panic will still try to format the + /// panic message. + /// + /// When this closure is run, aspects such as the stdio file descriptors and + /// working directory have successfully been changed, so output to these + /// locations might not appear where intended. + /// + /// [POSIX fork() specification]: + /// https://pubs.opengroup.org/onlinepubs/9699919799/functions/fork.html + /// [`std::env`]: mod@crate::env + #[stable(feature = "process_pre_exec", since = "1.34.0")] + unsafe fn pre_exec(&mut self, f: F) -> &mut process::Command + where + F: FnMut() -> io::Result<()> + Send + Sync + 'static; + + /// Schedules a closure to be run just before the `exec` function is + /// invoked. + /// + /// This method is stable and usable, but it should be unsafe. To fix + /// that, it got deprecated in favor of the unsafe [`pre_exec`]. + /// + /// [`pre_exec`]: CommandExt::pre_exec + #[stable(feature = "process_exec", since = "1.15.0")] + #[deprecated(since = "1.37.0", note = "should be unsafe, use `pre_exec` instead")] + fn before_exec(&mut self, f: F) -> &mut process::Command + where + F: FnMut() -> io::Result<()> + Send + Sync + 'static, + { + unsafe { self.pre_exec(f) } + } + + /// Performs all the required setup by this `Command`, followed by calling + /// the `execvp` syscall. + /// + /// On success this function will not return, and otherwise it will return + /// an error indicating why the exec (or another part of the setup of the + /// `Command`) failed. + /// + /// `exec` not returning has the same implications as calling + /// [`process::exit`] – no destructors on the current stack or any other + /// thread’s stack will be run. Therefore, it is recommended to only call + /// `exec` at a point where it is fine to not run any destructors. Note, + /// that the `execvp` syscall independently guarantees that all memory is + /// freed and all file descriptors with the `CLOEXEC` option (set by default + /// on all file descriptors opened by the standard library) are closed. + /// + /// This function, unlike `spawn`, will **not** `fork` the process to create + /// a new child. Like spawn, however, the default behavior for the stdio + /// descriptors will be to inherited from the current process. + /// + /// # Notes + /// + /// The process may be in a "broken state" if this function returns in + /// error. For example the working directory, environment variables, signal + /// handling settings, various user/group information, or aspects of stdio + /// file descriptors may have changed. If a "transactional spawn" is + /// required to gracefully handle errors it is recommended to use the + /// cross-platform `spawn` instead. + #[stable(feature = "process_exec2", since = "1.9.0")] + fn exec(&mut self) -> io::Error; + + /// Set executable argument + /// + /// Set the first process argument, `argv[0]`, to something other than the + /// default executable path. + #[stable(feature = "process_set_argv0", since = "1.45.0")] + fn arg0(&mut self, arg: S) -> &mut process::Command + where + S: AsRef; + + /// Sets the process group ID (PGID) of the child process. Equivalent to a + /// `setpgid` call in the child process, but may be more efficient. + /// + /// Process groups determine which processes receive signals. + /// + /// # Examples + /// + /// Pressing Ctrl-C in a terminal will send SIGINT to all processes in + /// the current foreground process group. By spawning the `sleep` + /// subprocess in a new process group, it will not receive SIGINT from the + /// terminal. + /// + /// The parent process could install a signal handler and manage the + /// subprocess on its own terms. + /// + /// A process group ID of 0 will use the process ID as the PGID. + /// + /// ```no_run + /// use std::process::Command; + /// use std::os::unix::process::CommandExt; + /// + /// Command::new("sleep") + /// .arg("10") + /// .process_group(0) + /// .spawn()? + /// .wait()?; + /// # + /// # Ok::<_, Box>(()) + /// ``` + #[stable(feature = "process_set_process_group", since = "1.64.0")] + fn process_group(&mut self, pgroup: i32) -> &mut process::Command; +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl CommandExt for process::Command { + fn uid(&mut self, id: UserId) -> &mut process::Command { + self.as_inner_mut().uid(id); + self + } + + fn gid(&mut self, id: GroupId) -> &mut process::Command { + self.as_inner_mut().gid(id); + self + } + + fn groups(&mut self, groups: &[GroupId]) -> &mut process::Command { + self.as_inner_mut().groups(groups); + self + } + + unsafe fn pre_exec(&mut self, f: F) -> &mut process::Command + where + F: FnMut() -> io::Result<()> + Send + Sync + 'static, + { + self.as_inner_mut().pre_exec(Box::new(f)); + self + } + + fn exec(&mut self) -> io::Error { + // NOTE: This may *not* be safe to call after `libc::fork`, because it + // may allocate. That may be worth fixing at some point in the future. + self.as_inner_mut().exec(sys::process::Stdio::Inherit) + } + + fn arg0(&mut self, arg: S) -> &mut process::Command + where + S: AsRef, + { + self.as_inner_mut().set_arg_0(arg.as_ref()); + self + } + + fn process_group(&mut self, pgroup: i32) -> &mut process::Command { + self.as_inner_mut().pgroup(pgroup); + self + } +} + +/// Unix-specific extensions to [`process::ExitStatus`] and +/// [`ExitStatusError`](process::ExitStatusError). +/// +/// On Unix, `ExitStatus` **does not necessarily represent an exit status**, as +/// passed to the `_exit` system call or returned by +/// [`ExitStatus::code()`](crate::process::ExitStatus::code). It represents **any wait status** +/// as returned by one of the `wait` family of system +/// calls. +/// +/// A Unix wait status (a Rust `ExitStatus`) can represent a Unix exit status, but can also +/// represent other kinds of process event. +/// +/// This trait is sealed: it cannot be implemented outside the standard library. +/// This is so that future additional methods are not breaking changes. +#[stable(feature = "rust1", since = "1.0.0")] +pub trait ExitStatusExt: Sealed { + /// Creates a new `ExitStatus` or `ExitStatusError` from the raw underlying integer status + /// value from `wait` + /// + /// The value should be a **wait status, not an exit status**. + /// + /// # Panics + /// + /// Panics on an attempt to make an `ExitStatusError` from a wait status of `0`. + /// + /// Making an `ExitStatus` always succeeds and never panics. + #[stable(feature = "exit_status_from", since = "1.12.0")] + fn from_raw(raw: i32) -> Self; + + /// If the process was terminated by a signal, returns that signal. + /// + /// In other words, if `WIFSIGNALED`, this returns `WTERMSIG`. + #[stable(feature = "rust1", since = "1.0.0")] + fn signal(&self) -> Option; + + /// If the process was terminated by a signal, says whether it dumped core. + #[stable(feature = "unix_process_wait_more", since = "1.58.0")] + fn core_dumped(&self) -> bool; + + /// If the process was stopped by a signal, returns that signal. + /// + /// In other words, if `WIFSTOPPED`, this returns `WSTOPSIG`. This is only possible if the status came from + /// a `wait` system call which was passed `WUNTRACED`, and was then converted into an `ExitStatus`. + #[stable(feature = "unix_process_wait_more", since = "1.58.0")] + fn stopped_signal(&self) -> Option; + + /// Whether the process was continued from a stopped status. + /// + /// Ie, `WIFCONTINUED`. This is only possible if the status came from a `wait` system call + /// which was passed `WCONTINUED`, and was then converted into an `ExitStatus`. + #[stable(feature = "unix_process_wait_more", since = "1.58.0")] + fn continued(&self) -> bool; + + /// Returns the underlying raw `wait` status. + /// + /// The returned integer is a **wait status, not an exit status**. + #[stable(feature = "unix_process_wait_more", since = "1.58.0")] + fn into_raw(self) -> i32; +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ExitStatusExt for process::ExitStatus { + fn from_raw(raw: i32) -> Self { + process::ExitStatus::from_inner(From::from(raw)) + } + + fn signal(&self) -> Option { + self.as_inner().signal() + } + + fn core_dumped(&self) -> bool { + self.as_inner().core_dumped() + } + + fn stopped_signal(&self) -> Option { + self.as_inner().stopped_signal() + } + + fn continued(&self) -> bool { + self.as_inner().continued() + } + + fn into_raw(self) -> i32 { + self.as_inner().into_raw().into() + } +} + +#[unstable(feature = "exit_status_error", issue = "84908")] +impl ExitStatusExt for process::ExitStatusError { + fn from_raw(raw: i32) -> Self { + process::ExitStatus::from_raw(raw) + .exit_ok() + .expect_err("::from_raw(0) but zero is not an error") + } + + fn signal(&self) -> Option { + self.into_status().signal() + } + + fn core_dumped(&self) -> bool { + self.into_status().core_dumped() + } + + fn stopped_signal(&self) -> Option { + self.into_status().stopped_signal() + } + + fn continued(&self) -> bool { + self.into_status().continued() + } + + fn into_raw(self) -> i32 { + self.into_status().into_raw() + } +} + +#[stable(feature = "process_extensions", since = "1.2.0")] +impl FromRawFd for process::Stdio { + #[inline] + unsafe fn from_raw_fd(fd: RawFd) -> process::Stdio { + let fd = sys::fd::FileDesc::from_raw_fd(fd); + let io = sys::process::Stdio::Fd(fd); + process::Stdio::from_inner(io) + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for process::Stdio { + #[inline] + fn from(fd: OwnedFd) -> process::Stdio { + let fd = sys::fd::FileDesc::from_inner(fd); + let io = sys::process::Stdio::Fd(fd); + process::Stdio::from_inner(io) + } +} + +#[stable(feature = "process_extensions", since = "1.2.0")] +impl AsRawFd for process::ChildStdin { + #[inline] + fn as_raw_fd(&self) -> RawFd { + self.as_inner().as_raw_fd() + } +} + +#[stable(feature = "process_extensions", since = "1.2.0")] +impl AsRawFd for process::ChildStdout { + #[inline] + fn as_raw_fd(&self) -> RawFd { + self.as_inner().as_raw_fd() + } +} + +#[stable(feature = "process_extensions", since = "1.2.0")] +impl AsRawFd for process::ChildStderr { + #[inline] + fn as_raw_fd(&self) -> RawFd { + self.as_inner().as_raw_fd() + } +} + +#[stable(feature = "into_raw_os", since = "1.4.0")] +impl IntoRawFd for process::ChildStdin { + #[inline] + fn into_raw_fd(self) -> RawFd { + self.into_inner().into_inner().into_raw_fd() + } +} + +#[stable(feature = "into_raw_os", since = "1.4.0")] +impl IntoRawFd for process::ChildStdout { + #[inline] + fn into_raw_fd(self) -> RawFd { + self.into_inner().into_inner().into_raw_fd() + } +} + +#[stable(feature = "into_raw_os", since = "1.4.0")] +impl IntoRawFd for process::ChildStderr { + #[inline] + fn into_raw_fd(self) -> RawFd { + self.into_inner().into_inner().into_raw_fd() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsFd for crate::process::ChildStdin { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + self.as_inner().as_fd() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for OwnedFd { + #[inline] + fn from(child_stdin: crate::process::ChildStdin) -> OwnedFd { + child_stdin.into_inner().into_inner().into_inner() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsFd for crate::process::ChildStdout { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + self.as_inner().as_fd() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for OwnedFd { + #[inline] + fn from(child_stdout: crate::process::ChildStdout) -> OwnedFd { + child_stdout.into_inner().into_inner().into_inner() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsFd for crate::process::ChildStderr { + #[inline] + fn as_fd(&self) -> BorrowedFd<'_> { + self.as_inner().as_fd() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for OwnedFd { + #[inline] + fn from(child_stderr: crate::process::ChildStderr) -> OwnedFd { + child_stderr.into_inner().into_inner().into_inner() + } +} + +/// Returns the OS-assigned process identifier associated with this process's parent. +#[must_use] +#[stable(feature = "unix_ppid", since = "1.27.0")] +pub fn parent_id() -> u32 { + crate::sys::os::getppid() +} diff --git a/crux-mir/lib/std/src/os/unix/raw.rs b/crux-mir/lib/std/src/os/unix/raw.rs new file mode 100644 index 000000000..fe761627b --- /dev/null +++ b/crux-mir/lib/std/src/os/unix/raw.rs @@ -0,0 +1,33 @@ +//! Unix-specific primitives available on all unix platforms. + +#![stable(feature = "raw_ext", since = "1.1.0")] +#![deprecated( + since = "1.8.0", + note = "these type aliases are no longer supported by \ + the standard library, the `libc` crate on \ + crates.io should be used instead for the correct \ + definitions" +)] +#![allow(deprecated)] + +#[stable(feature = "raw_ext", since = "1.1.0")] +#[allow(non_camel_case_types)] +pub type uid_t = u32; + +#[stable(feature = "raw_ext", since = "1.1.0")] +#[allow(non_camel_case_types)] +pub type gid_t = u32; + +#[stable(feature = "raw_ext", since = "1.1.0")] +#[allow(non_camel_case_types)] +pub type pid_t = i32; + +#[doc(inline)] +#[stable(feature = "pthread_t", since = "1.8.0")] +pub use super::platform::raw::pthread_t; +#[doc(inline)] +#[stable(feature = "raw_ext", since = "1.1.0")] +pub use super::platform::raw::{blkcnt_t, time_t}; +#[doc(inline)] +#[stable(feature = "raw_ext", since = "1.1.0")] +pub use super::platform::raw::{blksize_t, dev_t, ino_t, mode_t, nlink_t, off_t}; diff --git a/crux-mir/lib/std/src/sys/unix/ext/thread.rs b/crux-mir/lib/std/src/os/unix/thread.rs similarity index 85% rename from crux-mir/lib/std/src/sys/unix/ext/thread.rs rename to crux-mir/lib/std/src/os/unix/thread.rs index 759ef6236..03dcc3a4f 100644 --- a/crux-mir/lib/std/src/sys/unix/ext/thread.rs +++ b/crux-mir/lib/std/src/os/unix/thread.rs @@ -1,4 +1,6 @@ -//! Unix-specific extensions to primitives in the `std::thread` module. +//! Unix-specific extensions to primitives in the [`std::thread`] module. +//! +//! [`std::thread`]: crate::thread #![stable(feature = "thread_extensions", since = "1.9.0")] @@ -11,9 +13,7 @@ use crate::thread::JoinHandle; #[allow(deprecated)] pub type RawPthread = pthread_t; -/// Unix-specific extensions to [`thread::JoinHandle`]. -/// -/// [`thread::JoinHandle`]: ../../../../std/thread/struct.JoinHandle.html +/// Unix-specific extensions to [`JoinHandle`]. #[stable(feature = "thread_extensions", since = "1.9.0")] pub trait JoinHandleExt { /// Extracts the raw pthread_t without taking ownership diff --git a/crux-mir/lib/std/src/os/unix/ucred.rs b/crux-mir/lib/std/src/os/unix/ucred.rs new file mode 100644 index 000000000..ae4faf27b --- /dev/null +++ b/crux-mir/lib/std/src/os/unix/ucred.rs @@ -0,0 +1,136 @@ +//! Unix peer credentials. + +// NOTE: Code in this file is heavily based on work done in PR 13 from the tokio-uds repository on +// GitHub. +// +// For reference, the link is here: https://github.com/tokio-rs/tokio-uds/pull/13 +// Credit to Martin Habovštiak (GitHub username Kixunil) and contributors for this work. + +use libc::{gid_t, pid_t, uid_t}; + +/// Credentials for a UNIX process for credentials passing. +#[unstable(feature = "peer_credentials_unix_socket", issue = "42839", reason = "unstable")] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct UCred { + /// The UID part of the peer credential. This is the effective UID of the process at the domain + /// socket's endpoint. + pub uid: uid_t, + /// The GID part of the peer credential. This is the effective GID of the process at the domain + /// socket's endpoint. + pub gid: gid_t, + /// The PID part of the peer credential. This field is optional because the PID part of the + /// peer credentials is not supported on every platform. On platforms where the mechanism to + /// discover the PID exists, this field will be populated to the PID of the process at the + /// domain socket's endpoint. Otherwise, it will be set to None. + pub pid: Option, +} + +#[cfg(any(target_os = "android", target_os = "linux"))] +pub use self::impl_linux::peer_cred; + +#[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "openbsd", + target_os = "netbsd" +))] +pub use self::impl_bsd::peer_cred; + +#[cfg(any(target_os = "macos", target_os = "ios", target_os = "watchos"))] +pub use self::impl_mac::peer_cred; + +#[cfg(any(target_os = "linux", target_os = "android"))] +pub mod impl_linux { + use super::UCred; + use crate::os::unix::io::AsRawFd; + use crate::os::unix::net::UnixStream; + use crate::{io, mem}; + use libc::{c_void, getsockopt, socklen_t, ucred, SOL_SOCKET, SO_PEERCRED}; + + pub fn peer_cred(socket: &UnixStream) -> io::Result { + let ucred_size = mem::size_of::(); + + // Trivial sanity checks. + assert!(mem::size_of::() <= mem::size_of::()); + assert!(ucred_size <= u32::MAX as usize); + + let mut ucred_size = ucred_size as socklen_t; + let mut ucred: ucred = ucred { pid: 1, uid: 1, gid: 1 }; + + unsafe { + let ret = getsockopt( + socket.as_raw_fd(), + SOL_SOCKET, + SO_PEERCRED, + &mut ucred as *mut ucred as *mut c_void, + &mut ucred_size, + ); + + if ret == 0 && ucred_size as usize == mem::size_of::() { + Ok(UCred { uid: ucred.uid, gid: ucred.gid, pid: Some(ucred.pid) }) + } else { + Err(io::Error::last_os_error()) + } + } + } +} + +#[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "openbsd", + target_os = "netbsd" +))] +pub mod impl_bsd { + use super::UCred; + use crate::io; + use crate::os::unix::io::AsRawFd; + use crate::os::unix::net::UnixStream; + + pub fn peer_cred(socket: &UnixStream) -> io::Result { + let mut cred = UCred { uid: 1, gid: 1, pid: None }; + unsafe { + let ret = libc::getpeereid(socket.as_raw_fd(), &mut cred.uid, &mut cred.gid); + + if ret == 0 { Ok(cred) } else { Err(io::Error::last_os_error()) } + } + } +} + +#[cfg(any(target_os = "macos", target_os = "ios", target_os = "watchos"))] +pub mod impl_mac { + use super::UCred; + use crate::os::unix::io::AsRawFd; + use crate::os::unix::net::UnixStream; + use crate::{io, mem}; + use libc::{c_void, getpeereid, getsockopt, pid_t, socklen_t, LOCAL_PEERPID, SOL_LOCAL}; + + pub fn peer_cred(socket: &UnixStream) -> io::Result { + let mut cred = UCred { uid: 1, gid: 1, pid: None }; + unsafe { + let ret = getpeereid(socket.as_raw_fd(), &mut cred.uid, &mut cred.gid); + + if ret != 0 { + return Err(io::Error::last_os_error()); + } + + let mut pid: pid_t = 1; + let mut pid_size = mem::size_of::() as socklen_t; + + let ret = getsockopt( + socket.as_raw_fd(), + SOL_LOCAL, + LOCAL_PEERPID, + &mut pid as *mut pid_t as *mut c_void, + &mut pid_size, + ); + + if ret == 0 && pid_size as usize == mem::size_of::() { + cred.pid = Some(pid); + Ok(cred) + } else { + Err(io::Error::last_os_error()) + } + } + } +} diff --git a/crux-mir/lib/std/src/os/unix/ucred/tests.rs b/crux-mir/lib/std/src/os/unix/ucred/tests.rs new file mode 100644 index 000000000..e63a2fc24 --- /dev/null +++ b/crux-mir/lib/std/src/os/unix/ucred/tests.rs @@ -0,0 +1,39 @@ +use crate::os::unix::net::UnixStream; +use libc::{getegid, geteuid, getpid}; + +#[test] +#[cfg(any( + target_os = "android", + target_os = "linux", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "ios", + target_os = "macos", + target_os = "watchos", + target_os = "openbsd" +))] +fn test_socket_pair() { + // Create two connected sockets and get their peer credentials. They should be equal. + let (sock_a, sock_b) = UnixStream::pair().unwrap(); + let (cred_a, cred_b) = (sock_a.peer_cred().unwrap(), sock_b.peer_cred().unwrap()); + assert_eq!(cred_a, cred_b); + + // Check that the UID and GIDs match up. + let uid = unsafe { geteuid() }; + let gid = unsafe { getegid() }; + assert_eq!(cred_a.uid, uid); + assert_eq!(cred_a.gid, gid); +} + +#[test] +#[cfg(any(target_os = "linux", target_os = "ios", target_os = "macos", target_os = "watchos"))] +fn test_socket_pair_pids(arg: Type) -> RetType { + // Create two connected sockets and get their peer credentials. + let (sock_a, sock_b) = UnixStream::pair().unwrap(); + let (cred_a, cred_b) = (sock_a.peer_cred().unwrap(), sock_b.peer_cred().unwrap()); + + // On supported platforms (see the cfg above), the credentials should always include the PID. + let pid = unsafe { getpid() }; + assert_eq!(cred_a.pid, Some(pid)); + assert_eq!(cred_b.pid, Some(pid)); +} diff --git a/crux-mir/lib/std/src/os/vxworks/fs.rs b/crux-mir/lib/std/src/os/vxworks/fs.rs index 57ab4fb94..77e6238ca 100644 --- a/crux-mir/lib/std/src/os/vxworks/fs.rs +++ b/crux-mir/lib/std/src/os/vxworks/fs.rs @@ -4,7 +4,7 @@ use crate::fs::Metadata; use crate::sys_common::AsInner; /// -/// [`fs::Metadata`]: ../../../../std/fs/struct.Metadata.html +/// [`fs::Metadata`]: crate::fs::Metadata #[stable(feature = "metadata_ext", since = "1.1.0")] pub trait MetadataExt { #[stable(feature = "metadata_ext2", since = "1.8.0")] @@ -26,10 +26,16 @@ pub trait MetadataExt { #[stable(feature = "metadata_ext2", since = "1.8.0")] fn st_atime(&self) -> i64; #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_atime_nsec(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] fn st_mtime(&self) -> i64; #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_mtime_nsec(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] fn st_ctime(&self) -> i64; #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_ctime_nsec(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] fn st_blksize(&self) -> u64; #[stable(feature = "metadata_ext2", since = "1.8.0")] fn st_blocks(&self) -> u64; @@ -66,12 +72,21 @@ impl MetadataExt for Metadata { fn st_atime(&self) -> i64 { self.as_inner().as_inner().st_atime as i64 } + fn st_atime_nsec(&self) -> i64 { + 0 + } fn st_mtime(&self) -> i64 { self.as_inner().as_inner().st_mtime as i64 } + fn st_mtime_nsec(&self) -> i64 { + 0 + } fn st_ctime(&self) -> i64 { self.as_inner().as_inner().st_ctime as i64 } + fn st_ctime_nsec(&self) -> i64 { + 0 + } fn st_blksize(&self) -> u64 { self.as_inner().as_inner().st_blksize as u64 } diff --git a/crux-mir/lib/std/src/os/vxworks/raw.rs b/crux-mir/lib/std/src/os/vxworks/raw.rs index 29a0af564..cb41ddfe2 100644 --- a/crux-mir/lib/std/src/os/vxworks/raw.rs +++ b/crux-mir/lib/std/src/os/vxworks/raw.rs @@ -5,3 +5,6 @@ use crate::os::raw::c_ulong; #[stable(feature = "pthread_t", since = "1.8.0")] pub type pthread_t = c_ulong; + +#[stable(feature = "raw_ext", since = "1.1.0")] +pub use libc::{blkcnt_t, blksize_t, dev_t, ino_t, mode_t, nlink_t, off_t, time_t}; diff --git a/crux-mir/lib/std/src/os/wasi.rs b/crux-mir/lib/std/src/os/wasi.rs deleted file mode 100644 index d25b8d39e..000000000 --- a/crux-mir/lib/std/src/os/wasi.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! WASI-specific definitions - -#![stable(feature = "raw_ext", since = "1.1.0")] - -#[stable(feature = "rust1", since = "1.0.0")] -pub use crate::sys::ext::*; diff --git a/crux-mir/lib/std/src/os/wasi/ffi.rs b/crux-mir/lib/std/src/os/wasi/ffi.rs new file mode 100644 index 000000000..41dd8702e --- /dev/null +++ b/crux-mir/lib/std/src/os/wasi/ffi.rs @@ -0,0 +1,11 @@ +//! WASI-specific extensions to primitives in the [`std::ffi`] module +//! +//! [`std::ffi`]: crate::ffi + +#![stable(feature = "rust1", since = "1.0.0")] + +#[path = "../unix/ffi/os_str.rs"] +mod os_str; + +#[stable(feature = "rust1", since = "1.0.0")] +pub use self::os_str::{OsStrExt, OsStringExt}; diff --git a/crux-mir/lib/std/src/sys/wasi/ext/fs.rs b/crux-mir/lib/std/src/os/wasi/fs.rs similarity index 60% rename from crux-mir/lib/std/src/sys/wasi/ext/fs.rs rename to crux-mir/lib/std/src/os/wasi/fs.rs index 6696efa88..160c8f1ec 100644 --- a/crux-mir/lib/std/src/sys/wasi/ext/fs.rs +++ b/crux-mir/lib/std/src/os/wasi/fs.rs @@ -1,17 +1,37 @@ -//! WASI-specific extensions to primitives in the `std::fs` module. +//! WASI-specific extensions to primitives in the [`std::fs`] module. +//! +//! [`std::fs`]: crate::fs -#![unstable(feature = "wasi_ext", issue = "none")] +#![deny(unsafe_op_in_unsafe_fn)] +#![unstable(feature = "wasi_ext", issue = "71213")] +use crate::ffi::OsStr; use crate::fs::{self, File, Metadata, OpenOptions}; use crate::io::{self, IoSlice, IoSliceMut}; use crate::path::{Path, PathBuf}; -use crate::sys::fs::osstr2str; use crate::sys_common::{AsInner, AsInnerMut, FromInner}; +// Used for `File::read` on intra-doc links +#[allow(unused_imports)] +use io::{Read, Write}; /// WASI-specific extensions to [`File`]. -/// -/// [`File`]: ../../../../std/fs/struct.File.html pub trait FileExt { + /// Reads a number of bytes starting from a given offset. + /// + /// Returns the number of bytes read. + /// + /// The offset is relative to the start of the file and thus independent + /// from the current cursor. + /// + /// The current file cursor is not affected by this function. + /// + /// Note that similar to [`File::read`], it is not an error to return with a + /// short read. + fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result { + let bufs = &mut [IoSliceMut::new(buf)]; + self.read_vectored_at(bufs, offset) + } + /// Reads a number of bytes starting from a given offset. /// /// Returns the number of bytes read. @@ -23,9 +43,74 @@ pub trait FileExt { /// /// Note that similar to [`File::read_vectored`], it is not an error to /// return with a short read. + fn read_vectored_at(&self, bufs: &mut [IoSliceMut<'_>], offset: u64) -> io::Result; + + /// Reads the exact number of byte required to fill `buf` from the given offset. /// - /// [`File::read`]: ../../../../std/fs/struct.File.html#method.read_vectored - fn read_at(&self, bufs: &mut [IoSliceMut<'_>], offset: u64) -> io::Result; + /// The offset is relative to the start of the file and thus independent + /// from the current cursor. + /// + /// The current file cursor is not affected by this function. + /// + /// Similar to [`Read::read_exact`] but uses [`read_at`] instead of `read`. + /// + /// [`read_at`]: FileExt::read_at + /// + /// # Errors + /// + /// If this function encounters an error of the kind + /// [`io::ErrorKind::Interrupted`] then the error is ignored and the operation + /// will continue. + /// + /// If this function encounters an "end of file" before completely filling + /// the buffer, it returns an error of the kind [`io::ErrorKind::UnexpectedEof`]. + /// The contents of `buf` are unspecified in this case. + /// + /// If any other read error is encountered then this function immediately + /// returns. The contents of `buf` are unspecified in this case. + /// + /// If this function returns an error, it is unspecified how many bytes it + /// has read, but it will never read more than would be necessary to + /// completely fill the buffer. + #[stable(feature = "rw_exact_all_at", since = "1.33.0")] + fn read_exact_at(&self, mut buf: &mut [u8], mut offset: u64) -> io::Result<()> { + while !buf.is_empty() { + match self.read_at(buf, offset) { + Ok(0) => break, + Ok(n) => { + let tmp = buf; + buf = &mut tmp[n..]; + offset += n as u64; + } + Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + if !buf.is_empty() { + Err(io::const_io_error!(io::ErrorKind::UnexpectedEof, "failed to fill whole buffer")) + } else { + Ok(()) + } + } + + /// Writes a number of bytes starting from a given offset. + /// + /// Returns the number of bytes written. + /// + /// The offset is relative to the start of the file and thus independent + /// from the current cursor. + /// + /// The current file cursor is not affected by this function. + /// + /// When writing beyond the end of the file, the file is appropriately + /// extended and the intermediate bytes are initialized with the value 0. + /// + /// Note that similar to [`File::write`], it is not an error to return a + /// short write. + fn write_at(&self, buf: &[u8], offset: u64) -> io::Result { + let bufs = &[IoSlice::new(buf)]; + self.write_vectored_at(bufs, offset) + } /// Writes a number of bytes starting from a given offset. /// @@ -41,9 +126,48 @@ pub trait FileExt { /// /// Note that similar to [`File::write_vectored`], it is not an error to return a /// short write. + fn write_vectored_at(&self, bufs: &[IoSlice<'_>], offset: u64) -> io::Result; + + /// Attempts to write an entire buffer starting from a given offset. + /// + /// The offset is relative to the start of the file and thus independent + /// from the current cursor. + /// + /// The current file cursor is not affected by this function. /// - /// [`File::write`]: ../../../../std/fs/struct.File.html#method.write_vectored - fn write_at(&self, bufs: &[IoSlice<'_>], offset: u64) -> io::Result; + /// This method will continuously call [`write_at`] until there is no more data + /// to be written or an error of non-[`io::ErrorKind::Interrupted`] kind is + /// returned. This method will not return until the entire buffer has been + /// successfully written or such an error occurs. The first error that is + /// not of [`io::ErrorKind::Interrupted`] kind generated from this method will be + /// returned. + /// + /// # Errors + /// + /// This function will return the first error of + /// non-[`io::ErrorKind::Interrupted`] kind that [`write_at`] returns. + /// + /// [`write_at`]: FileExt::write_at + #[stable(feature = "rw_exact_all_at", since = "1.33.0")] + fn write_all_at(&self, mut buf: &[u8], mut offset: u64) -> io::Result<()> { + while !buf.is_empty() { + match self.write_at(buf, offset) { + Ok(0) => { + return Err(io::const_io_error!( + io::ErrorKind::WriteZero, + "failed to write whole buffer", + )); + } + Ok(n) => { + buf = &buf[n..]; + offset += n as u64 + } + Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + Ok(()) + } /// Returns the current position within the file. /// @@ -105,36 +229,51 @@ pub trait FileExt { // FIXME: bind random_get maybe? - on crates.io for unix impl FileExt for fs::File { - fn read_at(&self, bufs: &mut [IoSliceMut<'_>], offset: u64) -> io::Result { - self.as_inner().fd().pread(bufs, offset) + fn read_vectored_at(&self, bufs: &mut [IoSliceMut<'_>], offset: u64) -> io::Result { + self.as_inner().as_inner().pread(bufs, offset) } - fn write_at(&self, bufs: &[IoSlice<'_>], offset: u64) -> io::Result { - self.as_inner().fd().pwrite(bufs, offset) + fn write_vectored_at(&self, bufs: &[IoSlice<'_>], offset: u64) -> io::Result { + self.as_inner().as_inner().pwrite(bufs, offset) } fn tell(&self) -> io::Result { - self.as_inner().fd().tell() + self.as_inner().as_inner().tell() } fn fdstat_set_flags(&self, flags: u16) -> io::Result<()> { - self.as_inner().fd().set_flags(flags) + self.as_inner().as_inner().set_flags(flags) } fn fdstat_set_rights(&self, rights: u64, inheriting: u64) -> io::Result<()> { - self.as_inner().fd().set_rights(rights, inheriting) + self.as_inner().as_inner().set_rights(rights, inheriting) } fn advise(&self, offset: u64, len: u64, advice: u8) -> io::Result<()> { - self.as_inner().fd().advise(offset, len, advice) + let advice = match advice { + a if a == wasi::ADVICE_NORMAL.raw() => wasi::ADVICE_NORMAL, + a if a == wasi::ADVICE_SEQUENTIAL.raw() => wasi::ADVICE_SEQUENTIAL, + a if a == wasi::ADVICE_RANDOM.raw() => wasi::ADVICE_RANDOM, + a if a == wasi::ADVICE_WILLNEED.raw() => wasi::ADVICE_WILLNEED, + a if a == wasi::ADVICE_DONTNEED.raw() => wasi::ADVICE_DONTNEED, + a if a == wasi::ADVICE_NOREUSE.raw() => wasi::ADVICE_NOREUSE, + _ => { + return Err(io::const_io_error!( + io::ErrorKind::InvalidInput, + "invalid parameter 'advice'", + )); + } + }; + + self.as_inner().as_inner().advise(offset, len, advice) } fn allocate(&self, offset: u64, len: u64) -> io::Result<()> { - self.as_inner().fd().allocate(offset, len) + self.as_inner().as_inner().allocate(offset, len) } fn create_directory>(&self, dir: P) -> io::Result<()> { - self.as_inner().fd().create_directory(osstr2str(dir.as_ref().as_ref())?) + self.as_inner().as_inner().create_directory(osstr2str(dir.as_ref().as_ref())?) } fn read_link>(&self, path: P) -> io::Result { @@ -147,17 +286,15 @@ impl FileExt for fs::File { } fn remove_file>(&self, path: P) -> io::Result<()> { - self.as_inner().fd().unlink_file(osstr2str(path.as_ref().as_ref())?) + self.as_inner().as_inner().unlink_file(osstr2str(path.as_ref().as_ref())?) } fn remove_directory>(&self, path: P) -> io::Result<()> { - self.as_inner().fd().remove_directory(osstr2str(path.as_ref().as_ref())?) + self.as_inner().as_inner().remove_directory(osstr2str(path.as_ref().as_ref())?) } } /// WASI-specific extensions to [`fs::OpenOptions`]. -/// -/// [`fs::OpenOptions`]: ../../../../std/fs/struct.OpenOptions.html pub trait OpenOptionsExt { /// Pass custom `dirflags` argument to `path_open`. /// @@ -273,8 +410,6 @@ impl OpenOptionsExt for OpenOptions { } /// WASI-specific extensions to [`fs::Metadata`]. -/// -/// [`fs::Metadata`]: ../../../../std/fs/struct.Metadata.html pub trait MetadataExt { /// Returns the `st_dev` field of the internal `filestat_t` fn dev(&self) -> u64; @@ -282,6 +417,8 @@ pub trait MetadataExt { fn ino(&self) -> u64; /// Returns the `st_nlink` field of the internal `filestat_t` fn nlink(&self) -> u64; + /// Returns the `st_size` field of the internal `filestat_t` + fn size(&self) -> u64; /// Returns the `st_atim` field of the internal `filestat_t` fn atim(&self) -> u64; /// Returns the `st_mtim` field of the internal `filestat_t` @@ -300,6 +437,9 @@ impl MetadataExt for fs::Metadata { fn nlink(&self) -> u64 { self.as_inner().as_wasi().nlink } + fn size(&self) -> u64 { + self.as_inner().as_wasi().size + } fn atim(&self) -> u64 { self.as_inner().as_wasi().atim } @@ -311,28 +451,30 @@ impl MetadataExt for fs::Metadata { } } -/// WASI-specific extensions for [`FileType`]. +/// WASI-specific extensions for [`fs::FileType`]. /// /// Adds support for special WASI file types such as block/character devices, /// pipes, and sockets. -/// -/// [`FileType`]: ../../../../std/fs/struct.FileType.html pub trait FileTypeExt { /// Returns `true` if this file type is a block device. fn is_block_device(&self) -> bool; /// Returns `true` if this file type is a character device. - fn is_character_device(&self) -> bool; + fn is_char_device(&self) -> bool; /// Returns `true` if this file type is a socket datagram. fn is_socket_dgram(&self) -> bool; /// Returns `true` if this file type is a socket stream. fn is_socket_stream(&self) -> bool; + /// Returns `true` if this file type is any type of socket. + fn is_socket(&self) -> bool { + self.is_socket_stream() || self.is_socket_dgram() + } } impl FileTypeExt for fs::FileType { fn is_block_device(&self) -> bool { self.as_inner().bits() == wasi::FILETYPE_BLOCK_DEVICE } - fn is_character_device(&self) -> bool { + fn is_char_device(&self) -> bool { self.as_inner().bits() == wasi::FILETYPE_CHARACTER_DEVICE } fn is_socket_dgram(&self) -> bool { @@ -344,8 +486,6 @@ impl FileTypeExt for fs::FileType { } /// WASI-specific extension methods for [`fs::DirEntry`]. -/// -/// [`fs::DirEntry`]: ../../../../std/fs/struct.DirEntry.html pub trait DirEntryExt { /// Returns the underlying `d_ino` field of the `dirent_t` fn ino(&self) -> u64; @@ -367,10 +507,10 @@ pub fn link, U: AsRef>( new_fd: &File, new_path: U, ) -> io::Result<()> { - old_fd.as_inner().fd().link( + old_fd.as_inner().as_inner().link( old_flags, osstr2str(old_path.as_ref().as_ref())?, - new_fd.as_inner().fd(), + new_fd.as_inner().as_inner(), osstr2str(new_path.as_ref().as_ref())?, ) } @@ -384,9 +524,9 @@ pub fn rename, U: AsRef>( new_fd: &File, new_path: U, ) -> io::Result<()> { - old_fd.as_inner().fd().rename( + old_fd.as_inner().as_inner().rename( osstr2str(old_path.as_ref().as_ref())?, - new_fd.as_inner().fd(), + new_fd.as_inner().as_inner(), osstr2str(new_path.as_ref().as_ref())?, ) } @@ -400,6 +540,19 @@ pub fn symlink, U: AsRef>( new_path: U, ) -> io::Result<()> { fd.as_inner() - .fd() + .as_inner() .symlink(osstr2str(old_path.as_ref().as_ref())?, osstr2str(new_path.as_ref().as_ref())?) } + +/// Create a symbolic link. +/// +/// This is a convenience API similar to `std::os::unix::fs::symlink` and +/// `std::os::windows::fs::symlink_file` and `std::os::windows::fs::symlink_dir`. +pub fn symlink_path, U: AsRef>(old_path: P, new_path: U) -> io::Result<()> { + crate::sys::fs::symlink(old_path.as_ref(), new_path.as_ref()) +} + +fn osstr2str(f: &OsStr) -> io::Result<&str> { + f.to_str() + .ok_or_else(|| io::const_io_error!(io::ErrorKind::Uncategorized, "input must be utf-8")) +} diff --git a/crux-mir/lib/std/src/os/wasi/io/fd.rs b/crux-mir/lib/std/src/os/wasi/io/fd.rs new file mode 100644 index 000000000..930aca887 --- /dev/null +++ b/crux-mir/lib/std/src/os/wasi/io/fd.rs @@ -0,0 +1,9 @@ +//! Owned and borrowed file descriptors. + +#![unstable(feature = "wasi_ext", issue = "71213")] + +// Tests for this module +#[cfg(test)] +mod tests; + +pub use crate::os::fd::owned::*; diff --git a/crux-mir/lib/std/src/os/wasi/io/fd/tests.rs b/crux-mir/lib/std/src/os/wasi/io/fd/tests.rs new file mode 100644 index 000000000..418274752 --- /dev/null +++ b/crux-mir/lib/std/src/os/wasi/io/fd/tests.rs @@ -0,0 +1,11 @@ +use crate::mem::size_of; +use crate::os::wasi::io::RawFd; + +#[test] +fn test_raw_fd_layout() { + // `OwnedFd` and `BorrowedFd` use `rustc_layout_scalar_valid_range_start` + // and `rustc_layout_scalar_valid_range_end`, with values that depend on + // the bit width of `RawFd`. If this ever changes, those values will need + // to be updated. + assert_eq!(size_of::(), 4); +} diff --git a/crux-mir/lib/std/src/os/wasi/io/mod.rs b/crux-mir/lib/std/src/os/wasi/io/mod.rs new file mode 100644 index 000000000..4e123a1ee --- /dev/null +++ b/crux-mir/lib/std/src/os/wasi/io/mod.rs @@ -0,0 +1,6 @@ +//! WASI-specific extensions to general I/O primitives. + +#![stable(feature = "io_safety_wasi", since = "1.65.0")] + +#[stable(feature = "io_safety_wasi", since = "1.65.0")] +pub use crate::os::fd::*; diff --git a/crux-mir/lib/std/src/os/wasi/io/raw.rs b/crux-mir/lib/std/src/os/wasi/io/raw.rs new file mode 100644 index 000000000..da3b36ada --- /dev/null +++ b/crux-mir/lib/std/src/os/wasi/io/raw.rs @@ -0,0 +1,20 @@ +//! WASI-specific extensions to general I/O primitives. + +#![unstable(feature = "wasi_ext", issue = "71213")] + +// NOTE: despite the fact that this module is unstable, +// stable Rust had the capability to access the stable +// re-exported items from os::fd::raw through this +// unstable module. +// In PR #95956 the stability checker was changed to check +// all path segments of an item rather than just the last, +// which caused the aforementioned stable usage to regress +// (see issue #99502). +// As a result, the items in os::fd::raw were given the +// rustc_allowed_through_unstable_modules attribute. +// No regression tests were added to ensure this property, +// as CI is not configured to test wasm32-wasi. +// If this module is stabilized, +// you may want to remove those attributes +// (assuming no other unstable modules need them). +pub use crate::os::fd::raw::*; diff --git a/crux-mir/lib/std/src/sys/unix/ext/mod.rs b/crux-mir/lib/std/src/os/wasi/mod.rs similarity index 55% rename from crux-mir/lib/std/src/sys/unix/ext/mod.rs rename to crux-mir/lib/std/src/os/wasi/mod.rs index cbdb1c100..bbaf328f4 100644 --- a/crux-mir/lib/std/src/sys/unix/ext/mod.rs +++ b/crux-mir/lib/std/src/os/wasi/mod.rs @@ -1,7 +1,7 @@ -//! Platform-specific extensions to `std` for Unix platforms. +//! Platform-specific extensions to `std` for the WebAssembly System Interface (WASI). //! -//! Provides access to platform-level information on Unix platforms, and -//! exposes Unix-specific functions that would otherwise be inappropriate as +//! Provides access to platform-level information on WASI, and exposes +//! WASI-specific functions that would otherwise be inappropriate as //! part of the core `std` library. //! //! It exposes more ways to deal with platform-specific strings (`OsStr`, @@ -13,29 +13,29 @@ //! //! ```no_run //! use std::fs::File; -//! use std::os::unix::prelude::*; +//! use std::os::wasi::prelude::*; //! //! fn main() -> std::io::Result<()> { //! let f = File::create("foo.txt")?; //! let fd = f.as_raw_fd(); //! -//! // use fd with native unix bindings +//! // use fd with native WASI bindings //! //! Ok(()) //! } //! ``` +//! +//! [`OsStr`]: crate::ffi::OsStr +//! [`OsString`]: crate::ffi::OsString #![stable(feature = "rust1", since = "1.0.0")] -#![doc(cfg(unix))] -#![allow(missing_docs)] +#![deny(unsafe_op_in_unsafe_fn)] +#![doc(cfg(target_os = "wasi"))] pub mod ffi; pub mod fs; pub mod io; pub mod net; -pub mod process; -pub mod raw; -pub mod thread; /// A prelude for conveniently writing platform-specific code. /// @@ -47,20 +47,11 @@ pub mod prelude { pub use super::ffi::{OsStrExt, OsStringExt}; #[doc(no_inline)] #[stable(feature = "rust1", since = "1.0.0")] - pub use super::fs::DirEntryExt; - #[doc(no_inline)] - #[stable(feature = "file_offset", since = "1.15.0")] - pub use super::fs::FileExt; - #[doc(no_inline)] - #[stable(feature = "rust1", since = "1.0.0")] - pub use super::fs::{FileTypeExt, MetadataExt, OpenOptionsExt, PermissionsExt}; - #[doc(no_inline)] - #[stable(feature = "rust1", since = "1.0.0")] - pub use super::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd}; + pub use super::fs::FileTypeExt; #[doc(no_inline)] #[stable(feature = "rust1", since = "1.0.0")] - pub use super::process::{CommandExt, ExitStatusExt}; + pub use super::fs::{DirEntryExt, FileExt, MetadataExt, OpenOptionsExt}; #[doc(no_inline)] #[stable(feature = "rust1", since = "1.0.0")] - pub use super::thread::JoinHandleExt; + pub use super::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; } diff --git a/crux-mir/lib/std/src/os/wasi/net/mod.rs b/crux-mir/lib/std/src/os/wasi/net/mod.rs new file mode 100644 index 000000000..73c097d4a --- /dev/null +++ b/crux-mir/lib/std/src/os/wasi/net/mod.rs @@ -0,0 +1,23 @@ +//! WASI-specific networking functionality + +#![unstable(feature = "wasi_ext", issue = "71213")] + +use crate::io; +use crate::net; +use crate::sys_common::AsInner; + +/// WASI-specific extensions to [`std::net::TcpListener`]. +/// +/// [`std::net::TcpListener`]: crate::net::TcpListener +pub trait TcpListenerExt { + /// Accept a socket. + /// + /// This corresponds to the `sock_accept` syscall. + fn sock_accept(&self, flags: u16) -> io::Result; +} + +impl TcpListenerExt for net::TcpListener { + fn sock_accept(&self, flags: u16) -> io::Result { + self.as_inner().as_inner().as_inner().sock_accept(flags) + } +} diff --git a/crux-mir/lib/std/src/os/watchos/fs.rs b/crux-mir/lib/std/src/os/watchos/fs.rs new file mode 100644 index 000000000..a14fe35a7 --- /dev/null +++ b/crux-mir/lib/std/src/os/watchos/fs.rs @@ -0,0 +1,142 @@ +#![stable(feature = "metadata_ext", since = "1.1.0")] + +use crate::fs::Metadata; +use crate::sys_common::AsInner; + +#[allow(deprecated)] +use crate::os::watchos::raw; + +/// OS-specific extensions to [`fs::Metadata`]. +/// +/// [`fs::Metadata`]: crate::fs::Metadata +#[stable(feature = "metadata_ext", since = "1.1.0")] +pub trait MetadataExt { + /// Gain a reference to the underlying `stat` structure which contains + /// the raw information returned by the OS. + /// + /// The contents of the returned `stat` are **not** consistent across + /// Unix platforms. The `os::unix::fs::MetadataExt` trait contains the + /// cross-Unix abstractions contained within the raw stat. + #[stable(feature = "metadata_ext", since = "1.1.0")] + #[deprecated( + since = "1.8.0", + note = "deprecated in favor of the accessor \ + methods of this trait" + )] + #[allow(deprecated)] + fn as_raw_stat(&self) -> &raw::stat; + + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_dev(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_ino(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_mode(&self) -> u32; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_nlink(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_uid(&self) -> u32; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_gid(&self) -> u32; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_rdev(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_size(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_atime(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_atime_nsec(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_mtime(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_mtime_nsec(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_ctime(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_ctime_nsec(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_birthtime(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_birthtime_nsec(&self) -> i64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_blksize(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_blocks(&self) -> u64; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_flags(&self) -> u32; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_gen(&self) -> u32; + #[stable(feature = "metadata_ext2", since = "1.8.0")] + fn st_lspare(&self) -> u32; +} + +#[stable(feature = "metadata_ext", since = "1.1.0")] +impl MetadataExt for Metadata { + #[allow(deprecated)] + fn as_raw_stat(&self) -> &raw::stat { + unsafe { &*(self.as_inner().as_inner() as *const libc::stat as *const raw::stat) } + } + fn st_dev(&self) -> u64 { + self.as_inner().as_inner().st_dev as u64 + } + fn st_ino(&self) -> u64 { + self.as_inner().as_inner().st_ino as u64 + } + fn st_mode(&self) -> u32 { + self.as_inner().as_inner().st_mode as u32 + } + fn st_nlink(&self) -> u64 { + self.as_inner().as_inner().st_nlink as u64 + } + fn st_uid(&self) -> u32 { + self.as_inner().as_inner().st_uid as u32 + } + fn st_gid(&self) -> u32 { + self.as_inner().as_inner().st_gid as u32 + } + fn st_rdev(&self) -> u64 { + self.as_inner().as_inner().st_rdev as u64 + } + fn st_size(&self) -> u64 { + self.as_inner().as_inner().st_size as u64 + } + fn st_atime(&self) -> i64 { + self.as_inner().as_inner().st_atime as i64 + } + fn st_atime_nsec(&self) -> i64 { + self.as_inner().as_inner().st_atime_nsec as i64 + } + fn st_mtime(&self) -> i64 { + self.as_inner().as_inner().st_mtime as i64 + } + fn st_mtime_nsec(&self) -> i64 { + self.as_inner().as_inner().st_mtime_nsec as i64 + } + fn st_ctime(&self) -> i64 { + self.as_inner().as_inner().st_ctime as i64 + } + fn st_ctime_nsec(&self) -> i64 { + self.as_inner().as_inner().st_ctime_nsec as i64 + } + fn st_birthtime(&self) -> i64 { + self.as_inner().as_inner().st_birthtime as i64 + } + fn st_birthtime_nsec(&self) -> i64 { + self.as_inner().as_inner().st_birthtime_nsec as i64 + } + fn st_blksize(&self) -> u64 { + self.as_inner().as_inner().st_blksize as u64 + } + fn st_blocks(&self) -> u64 { + self.as_inner().as_inner().st_blocks as u64 + } + fn st_gen(&self) -> u32 { + self.as_inner().as_inner().st_gen as u32 + } + fn st_flags(&self) -> u32 { + self.as_inner().as_inner().st_flags as u32 + } + fn st_lspare(&self) -> u32 { + self.as_inner().as_inner().st_lspare as u32 + } +} diff --git a/crux-mir/lib/std/src/os/watchos/mod.rs b/crux-mir/lib/std/src/os/watchos/mod.rs new file mode 100644 index 000000000..cd6454ebb --- /dev/null +++ b/crux-mir/lib/std/src/os/watchos/mod.rs @@ -0,0 +1,6 @@ +//! watchOS-specific definitions + +#![stable(feature = "raw_ext", since = "1.1.0")] + +pub mod fs; +pub mod raw; diff --git a/crux-mir/lib/std/src/os/watchos/raw.rs b/crux-mir/lib/std/src/os/watchos/raw.rs new file mode 100644 index 000000000..630a533d9 --- /dev/null +++ b/crux-mir/lib/std/src/os/watchos/raw.rs @@ -0,0 +1,83 @@ +//! watchOS-specific raw type definitions + +#![stable(feature = "raw_ext", since = "1.1.0")] +#![deprecated( + since = "1.8.0", + note = "these type aliases are no longer supported by \ + the standard library, the `libc` crate on \ + crates.io should be used instead for the correct \ + definitions" +)] +#![allow(deprecated)] + +use crate::os::raw::c_long; + +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type blkcnt_t = u64; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type blksize_t = u64; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type dev_t = u64; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type ino_t = u64; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type mode_t = u32; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type nlink_t = u64; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type off_t = u64; +#[stable(feature = "raw_ext", since = "1.1.0")] +pub type time_t = i64; + +#[stable(feature = "pthread_t", since = "1.8.0")] +pub type pthread_t = usize; + +#[repr(C)] +#[derive(Clone)] +#[stable(feature = "raw_ext", since = "1.1.0")] +pub struct stat { + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_dev: i32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mode: u16, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_nlink: u16, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ino: u64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_uid: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_gid: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_rdev: i32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_atime: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_atime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mtime: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_mtime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ctime: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_ctime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_birthtime: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_birthtime_nsec: c_long, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_size: i64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_blocks: i64, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_blksize: i32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_flags: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_gen: u32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_lspare: i32, + #[stable(feature = "raw_ext", since = "1.1.0")] + pub st_qspare: [i64; 2], +} diff --git a/crux-mir/lib/std/src/sys/windows/ext/ffi.rs b/crux-mir/lib/std/src/os/windows/ffi.rs similarity index 80% rename from crux-mir/lib/std/src/sys/windows/ext/ffi.rs rename to crux-mir/lib/std/src/os/windows/ffi.rs index 6e7811938..96bab59d3 100644 --- a/crux-mir/lib/std/src/sys/windows/ext/ffi.rs +++ b/crux-mir/lib/std/src/os/windows/ffi.rs @@ -1,4 +1,4 @@ -//! Windows-specific extensions to the primitives in the `std::ffi` module. +//! Windows-specific extensions to primitives in the [`std::ffi`] module. //! //! # Overview //! @@ -30,13 +30,13 @@ //! [`OsString`] is the Rust wrapper for owned strings in the //! preferred representation of the operating system. On Windows, //! this struct gets augmented with an implementation of the -//! [`OsStringExt`] trait, which has a [`from_wide`] method. This +//! [`OsStringExt`] trait, which has an [`OsStringExt::from_wide`] method. This //! lets you create an [`OsString`] from a `&[u16]` slice; presumably //! you get such a slice out of a `WCHAR` Windows API. //! //! Similarly, [`OsStr`] is the Rust wrapper for borrowed strings from //! preferred representation of the operating system. On Windows, the -//! [`OsStrExt`] trait provides the [`encode_wide`] method, which +//! [`OsStrExt`] trait provides the [`OsStrExt::encode_wide`] method, which //! outputs an [`EncodeWide`] iterator. You can [`collect`] this //! iterator, for example, to obtain a `Vec`; you can later get a //! pointer to this vector's contents and feed it to Windows APIs. @@ -47,19 +47,14 @@ //! ill-formed UTF-16. //! //! [ill-formed-utf-16]: https://simonsapin.github.io/wtf-8/#ill-formed-utf-16 -//! [`OsString`]: ../../../ffi/struct.OsString.html -//! [`OsStr`]: ../../../ffi/struct.OsStr.html -//! [`OsStringExt`]: trait.OsStringExt.html -//! [`OsStrExt`]: trait.OsStrExt.html -//! [`EncodeWide`]: struct.EncodeWide.html -//! [`from_wide`]: trait.OsStringExt.html#tymethod.from_wide -//! [`encode_wide`]: trait.OsStrExt.html#tymethod.encode_wide -//! [`collect`]: ../../../iter/trait.Iterator.html#method.collect -//! [U+FFFD]: ../../../char/constant.REPLACEMENT_CHARACTER.html +//! [`collect`]: crate::iter::Iterator::collect +//! [U+FFFD]: crate::char::REPLACEMENT_CHARACTER +//! [`std::ffi`]: crate::ffi #![stable(feature = "rust1", since = "1.0.0")] use crate::ffi::{OsStr, OsString}; +use crate::sealed::Sealed; use crate::sys::os_str::Buf; use crate::sys_common::wtf8::Wtf8Buf; use crate::sys_common::{AsInner, FromInner}; @@ -69,13 +64,14 @@ pub use crate::sys_common::wtf8::EncodeWide; /// Windows-specific extensions to [`OsString`]. /// -/// [`OsString`]: ../../../../std/ffi/struct.OsString.html +/// This trait is sealed: it cannot be implemented outside the standard library. +/// This is so that future additional methods are not breaking changes. #[stable(feature = "rust1", since = "1.0.0")] -pub trait OsStringExt { +pub trait OsStringExt: Sealed { /// Creates an `OsString` from a potentially ill-formed UTF-16 slice of /// 16-bit code units. /// - /// This is lossless: calling [`encode_wide`] on the resulting string + /// This is lossless: calling [`OsStrExt::encode_wide`] on the resulting string /// will always return the original code units. /// /// # Examples @@ -89,8 +85,6 @@ pub trait OsStringExt { /// /// let string = OsString::from_wide(&source[..]); /// ``` - /// - /// [`encode_wide`]: ./trait.OsStrExt.html#tymethod.encode_wide #[stable(feature = "rust1", since = "1.0.0")] fn from_wide(wide: &[u16]) -> Self; } @@ -104,13 +98,14 @@ impl OsStringExt for OsString { /// Windows-specific extensions to [`OsStr`]. /// -/// [`OsStr`]: ../../../../std/ffi/struct.OsStr.html +/// This trait is sealed: it cannot be implemented outside the standard library. +/// This is so that future additional methods are not breaking changes. #[stable(feature = "rust1", since = "1.0.0")] -pub trait OsStrExt { +pub trait OsStrExt: Sealed { /// Re-encodes an `OsStr` as a wide character sequence, i.e., potentially /// ill-formed UTF-16. /// - /// This is lossless: calling [`OsString::from_wide`] and then + /// This is lossless: calling [`OsStringExt::from_wide`] and then /// `encode_wide` on the result will yield the original code units. /// Note that the encoding does not add a final null terminator. /// @@ -128,14 +123,13 @@ pub trait OsStrExt { /// let result: Vec = string.encode_wide().collect(); /// assert_eq!(&source[..], &result[..]); /// ``` - /// - /// [`OsString::from_wide`]: ./trait.OsStringExt.html#tymethod.from_wide #[stable(feature = "rust1", since = "1.0.0")] fn encode_wide(&self) -> EncodeWide<'_>; } #[stable(feature = "rust1", since = "1.0.0")] impl OsStrExt for OsStr { + #[inline] fn encode_wide(&self) -> EncodeWide<'_> { self.as_inner().inner.encode_wide() } diff --git a/crux-mir/lib/std/src/sys/windows/ext/fs.rs b/crux-mir/lib/std/src/os/windows/fs.rs similarity index 84% rename from crux-mir/lib/std/src/sys/windows/ext/fs.rs rename to crux-mir/lib/std/src/os/windows/fs.rs index d508a3334..a091f06dd 100644 --- a/crux-mir/lib/std/src/sys/windows/ext/fs.rs +++ b/crux-mir/lib/std/src/os/windows/fs.rs @@ -1,16 +1,17 @@ -//! Windows-specific extensions for the primitives in the `std::fs` module. +//! Windows-specific extensions to primitives in the [`std::fs`] module. +//! +//! [`std::fs`]: crate::fs #![stable(feature = "rust1", since = "1.0.0")] use crate::fs::{self, Metadata, OpenOptions}; use crate::io; use crate::path::Path; +use crate::sealed::Sealed; use crate::sys; use crate::sys_common::{AsInner, AsInnerMut}; -/// Windows-specific extensions to [`File`]. -/// -/// [`File`]: ../../../fs/struct.File.html +/// Windows-specific extensions to [`fs::File`]. #[stable(feature = "file_offset", since = "1.15.0")] pub trait FileExt { /// Seeks to a given position and reads a number of bytes. @@ -94,8 +95,6 @@ impl FileExt for fs::File { } /// Windows-specific extensions to [`fs::OpenOptions`]. -/// -/// [`fs::OpenOptions`]: ../../../../std/fs/struct.OpenOptions.html #[stable(feature = "open_options_ext", since = "1.10.0")] pub trait OpenOptionsExt { /// Overrides the `dwDesiredAccess` argument to the call to [`CreateFile`] @@ -160,6 +159,7 @@ pub trait OpenOptionsExt { /// # Examples /// /// ```no_run + /// # #![allow(unexpected_cfgs)] /// # #[cfg(for_demonstration_only)] /// extern crate winapi; /// # mod winapi { pub const FILE_FLAG_DELETE_ON_CLOSE: u32 = 0x04000000; } @@ -197,6 +197,7 @@ pub trait OpenOptionsExt { /// # Examples /// /// ```no_run + /// # #![allow(unexpected_cfgs)] /// # #[cfg(for_demonstration_only)] /// extern crate winapi; /// # mod winapi { pub const FILE_ATTRIBUTE_HIDDEN: u32 = 2; } @@ -224,7 +225,7 @@ pub trait OpenOptionsExt { /// opening a named pipe, to control to which degree a server process can /// act on behalf of a client process (security impersonation level). /// - /// When `security_qos_flags` is not set a malicious program can gain the + /// When `security_qos_flags` is not set, a malicious program can gain the /// elevated privileges of a privileged Rust process when it allows opening /// user-specified paths, by tricking it into opening a named pipe. So /// arguably `security_qos_flags` should also be set when opening arbitrary @@ -238,6 +239,7 @@ pub trait OpenOptionsExt { /// # Examples /// /// ```no_run + /// # #![allow(unexpected_cfgs)] /// # #[cfg(for_demonstration_only)] /// extern crate winapi; /// # mod winapi { pub const SECURITY_IDENTIFICATION: u32 = 0; } @@ -259,7 +261,7 @@ pub trait OpenOptionsExt { /// [Impersonation Levels]: /// https://docs.microsoft.com/en-us/windows/win32/api/winnt/ne-winnt-security_impersonation_level #[stable(feature = "open_options_ext", since = "1.10.0")] - fn security_qos_flags(&mut self, flags: u32) -> &mut OpenOptions; + fn security_qos_flags(&mut self, flags: u32) -> &mut Self; } #[stable(feature = "open_options_ext", since = "1.10.0")] @@ -295,7 +297,6 @@ impl OpenOptionsExt for OpenOptions { /// The data members that this trait exposes correspond to the members /// of the [`BY_HANDLE_FILE_INFORMATION`] structure. /// -/// [`fs::Metadata`]: ../../../../std/fs/struct.Metadata.html /// [`BY_HANDLE_FILE_INFORMATION`]: /// https://docs.microsoft.com/en-us/windows/win32/api/fileapi/ns-fileapi-by_handle_file_information #[stable(feature = "metadata_ext", since = "1.1.0")] @@ -499,22 +500,23 @@ impl MetadataExt for Metadata { } } -/// Windows-specific extensions to [`FileType`]. +/// Windows-specific extensions to [`fs::FileType`]. /// /// On Windows, a symbolic link knows whether it is a file or directory. -/// -/// [`FileType`]: ../../../../std/fs/struct.FileType.html -#[unstable(feature = "windows_file_type_ext", issue = "none")] -pub trait FileTypeExt { +#[stable(feature = "windows_file_type_ext", since = "1.64.0")] +pub trait FileTypeExt: Sealed { /// Returns `true` if this file type is a symbolic link that is also a directory. - #[unstable(feature = "windows_file_type_ext", issue = "none")] + #[stable(feature = "windows_file_type_ext", since = "1.64.0")] fn is_symlink_dir(&self) -> bool; /// Returns `true` if this file type is a symbolic link that is also a file. - #[unstable(feature = "windows_file_type_ext", issue = "none")] + #[stable(feature = "windows_file_type_ext", since = "1.64.0")] fn is_symlink_file(&self) -> bool; } -#[unstable(feature = "windows_file_type_ext", issue = "none")] +#[stable(feature = "windows_file_type_ext", since = "1.64.0")] +impl Sealed for fs::FileType {} + +#[stable(feature = "windows_file_type_ext", since = "1.64.0")] impl FileTypeExt for fs::FileType { fn is_symlink_dir(&self) -> bool { self.as_inner().is_symlink_dir() @@ -524,11 +526,20 @@ impl FileTypeExt for fs::FileType { } } -/// Creates a new file symbolic link on the filesystem. +/// Creates a new symlink to a non-directory file on the filesystem. /// -/// The `dst` path will be a file symbolic link pointing to the `src` +/// The `link` path will be a file symbolic link pointing to the `original` /// path. /// +/// The `original` path should not be a directory or a symlink to a directory, +/// otherwise the symlink will be broken. Use [`symlink_dir`] for directories. +/// +/// This function currently corresponds to [`CreateSymbolicLinkW`][CreateSymbolicLinkW]. +/// Note that this [may change in the future][changes]. +/// +/// [CreateSymbolicLinkW]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createsymboliclinkw +/// [changes]: io#platform-specific-behavior +/// /// # Examples /// /// ```no_run @@ -539,16 +550,35 @@ impl FileTypeExt for fs::FileType { /// Ok(()) /// } /// ``` +/// +/// # Limitations +/// +/// Windows treats symlink creation as a [privileged action][symlink-security], +/// therefore this function is likely to fail unless the user makes changes to +/// their system to permit symlink creation. Users can try enabling Developer +/// Mode, granting the `SeCreateSymbolicLinkPrivilege` privilege, or running +/// the process as an administrator. +/// +/// [symlink-security]: https://docs.microsoft.com/en-us/windows/security/threat-protection/security-policy-settings/create-symbolic-links #[stable(feature = "symlink", since = "1.1.0")] -pub fn symlink_file, Q: AsRef>(src: P, dst: Q) -> io::Result<()> { - sys::fs::symlink_inner(src.as_ref(), dst.as_ref(), false) +pub fn symlink_file, Q: AsRef>(original: P, link: Q) -> io::Result<()> { + sys::fs::symlink_inner(original.as_ref(), link.as_ref(), false) } -/// Creates a new directory symlink on the filesystem. +/// Creates a new symlink to a directory on the filesystem. /// -/// The `dst` path will be a directory symbolic link pointing to the `src` +/// The `link` path will be a directory symbolic link pointing to the `original` /// path. /// +/// The `original` path must be a directory or a symlink to a directory, +/// otherwise the symlink will be broken. Use [`symlink_file`] for other files. +/// +/// This function currently corresponds to [`CreateSymbolicLinkW`][CreateSymbolicLinkW]. +/// Note that this [may change in the future][changes]. +/// +/// [CreateSymbolicLinkW]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createsymboliclinkw +/// [changes]: io#platform-specific-behavior +/// /// # Examples /// /// ```no_run @@ -559,7 +589,17 @@ pub fn symlink_file, Q: AsRef>(src: P, dst: Q) -> io::Resul /// Ok(()) /// } /// ``` +/// +/// # Limitations +/// +/// Windows treats symlink creation as a [privileged action][symlink-security], +/// therefore this function is likely to fail unless the user makes changes to +/// their system to permit symlink creation. Users can try enabling Developer +/// Mode, granting the `SeCreateSymbolicLinkPrivilege` privilege, or running +/// the process as an administrator. +/// +/// [symlink-security]: https://docs.microsoft.com/en-us/windows/security/threat-protection/security-policy-settings/create-symbolic-links #[stable(feature = "symlink", since = "1.1.0")] -pub fn symlink_dir, Q: AsRef>(src: P, dst: Q) -> io::Result<()> { - sys::fs::symlink_inner(src.as_ref(), dst.as_ref(), true) +pub fn symlink_dir, Q: AsRef>(original: P, link: Q) -> io::Result<()> { + sys::fs::symlink_inner(original.as_ref(), link.as_ref(), true) } diff --git a/crux-mir/lib/std/src/os/windows/io/handle.rs b/crux-mir/lib/std/src/os/windows/io/handle.rs new file mode 100644 index 000000000..1dfecc573 --- /dev/null +++ b/crux-mir/lib/std/src/os/windows/io/handle.rs @@ -0,0 +1,593 @@ +//! Owned and borrowed OS handles. + +#![stable(feature = "io_safety", since = "1.63.0")] + +use super::raw::{AsRawHandle, FromRawHandle, IntoRawHandle, RawHandle}; +use crate::fmt; +use crate::fs; +use crate::io; +use crate::marker::PhantomData; +use crate::mem::forget; +use crate::ptr; +use crate::sys::c; +use crate::sys::cvt; +use crate::sys_common::{AsInner, FromInner, IntoInner}; + +/// A borrowed handle. +/// +/// This has a lifetime parameter to tie it to the lifetime of something that +/// owns the handle. +/// +/// This uses `repr(transparent)` and has the representation of a host handle, +/// so it can be used in FFI in places where a handle is passed as an argument, +/// it is not captured or consumed. +/// +/// Note that it *may* have the value `-1`, which in `BorrowedHandle` always +/// represents a valid handle value, such as [the current process handle], and +/// not `INVALID_HANDLE_VALUE`, despite the two having the same value. See +/// [here] for the full story. +/// +/// And, it *may* have the value `NULL` (0), which can occur when consoles are +/// detached from processes, or when `windows_subsystem` is used. +/// +/// This type's `.to_owned()` implementation returns another `BorrowedHandle` +/// rather than an `OwnedHandle`. It just makes a trivial copy of the raw +/// handle, which is then borrowed under the same lifetime. +/// +/// [here]: https://devblogs.microsoft.com/oldnewthing/20040302-00/?p=40443 +/// [the current process handle]: https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-getcurrentprocess#remarks +#[derive(Copy, Clone)] +#[repr(transparent)] +#[stable(feature = "io_safety", since = "1.63.0")] +pub struct BorrowedHandle<'handle> { + handle: RawHandle, + _phantom: PhantomData<&'handle OwnedHandle>, +} + +/// An owned handle. +/// +/// This closes the handle on drop. +/// +/// Note that it *may* have the value `-1`, which in `OwnedHandle` always +/// represents a valid handle value, such as [the current process handle], and +/// not `INVALID_HANDLE_VALUE`, despite the two having the same value. See +/// [here] for the full story. +/// +/// And, it *may* have the value `NULL` (0), which can occur when consoles are +/// detached from processes, or when `windows_subsystem` is used. +/// +/// `OwnedHandle` uses [`CloseHandle`] to close its handle on drop. As such, +/// it must not be used with handles to open registry keys which need to be +/// closed with [`RegCloseKey`] instead. +/// +/// [`CloseHandle`]: https://docs.microsoft.com/en-us/windows/win32/api/handleapi/nf-handleapi-closehandle +/// [`RegCloseKey`]: https://docs.microsoft.com/en-us/windows/win32/api/winreg/nf-winreg-regclosekey +/// +/// [here]: https://devblogs.microsoft.com/oldnewthing/20040302-00/?p=40443 +/// [the current process handle]: https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-getcurrentprocess#remarks +#[repr(transparent)] +#[stable(feature = "io_safety", since = "1.63.0")] +pub struct OwnedHandle { + handle: RawHandle, +} + +/// FFI type for handles in return values or out parameters, where `NULL` is used +/// as a sentry value to indicate errors, such as in the return value of `CreateThread`. This uses +/// `repr(transparent)` and has the representation of a host handle, so that it can be used in such +/// FFI declarations. +/// +/// The only thing you can usefully do with a `HandleOrNull` is to convert it into an +/// `OwnedHandle` using its [`TryFrom`] implementation; this conversion takes care of the check for +/// `NULL`. This ensures that such FFI calls cannot start using the handle without +/// checking for `NULL` first. +/// +/// This type may hold any handle value that [`OwnedHandle`] may hold. As with `OwnedHandle`, when +/// it holds `-1`, that value is interpreted as a valid handle value, such as +/// [the current process handle], and not `INVALID_HANDLE_VALUE`. +/// +/// If this holds a non-null handle, it will close the handle on drop. +/// +/// [the current process handle]: https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-getcurrentprocess#remarks +#[repr(transparent)] +#[stable(feature = "io_safety", since = "1.63.0")] +#[derive(Debug)] +pub struct HandleOrNull(OwnedHandle); + +/// FFI type for handles in return values or out parameters, where `INVALID_HANDLE_VALUE` is used +/// as a sentry value to indicate errors, such as in the return value of `CreateFileW`. This uses +/// `repr(transparent)` and has the representation of a host handle, so that it can be used in such +/// FFI declarations. +/// +/// The only thing you can usefully do with a `HandleOrInvalid` is to convert it into an +/// `OwnedHandle` using its [`TryFrom`] implementation; this conversion takes care of the check for +/// `INVALID_HANDLE_VALUE`. This ensures that such FFI calls cannot start using the handle without +/// checking for `INVALID_HANDLE_VALUE` first. +/// +/// This type may hold any handle value that [`OwnedHandle`] may hold, except that when it holds +/// `-1`, that value is interpreted to mean `INVALID_HANDLE_VALUE`. +/// +/// If holds a handle other than `INVALID_HANDLE_VALUE`, it will close the handle on drop. +#[repr(transparent)] +#[stable(feature = "io_safety", since = "1.63.0")] +#[derive(Debug)] +pub struct HandleOrInvalid(OwnedHandle); + +// The Windows [`HANDLE`] type may be transferred across and shared between +// thread boundaries (despite containing a `*mut void`, which in general isn't +// `Send` or `Sync`). +// +// [`HANDLE`]: std::os::windows::raw::HANDLE +#[stable(feature = "io_safety", since = "1.63.0")] +unsafe impl Send for OwnedHandle {} +#[stable(feature = "io_safety", since = "1.63.0")] +unsafe impl Send for HandleOrNull {} +#[stable(feature = "io_safety", since = "1.63.0")] +unsafe impl Send for HandleOrInvalid {} +#[stable(feature = "io_safety", since = "1.63.0")] +unsafe impl Send for BorrowedHandle<'_> {} +#[stable(feature = "io_safety", since = "1.63.0")] +unsafe impl Sync for OwnedHandle {} +#[stable(feature = "io_safety", since = "1.63.0")] +unsafe impl Sync for HandleOrNull {} +#[stable(feature = "io_safety", since = "1.63.0")] +unsafe impl Sync for HandleOrInvalid {} +#[stable(feature = "io_safety", since = "1.63.0")] +unsafe impl Sync for BorrowedHandle<'_> {} + +impl BorrowedHandle<'_> { + /// Return a `BorrowedHandle` holding the given raw handle. + /// + /// # Safety + /// + /// The resource pointed to by `handle` must be a valid open handle, it + /// must remain open for the duration of the returned `BorrowedHandle`. + /// + /// Note that it *may* have the value `INVALID_HANDLE_VALUE` (-1), which is + /// sometimes a valid handle value. See [here] for the full story. + /// + /// And, it *may* have the value `NULL` (0), which can occur when consoles are + /// detached from processes, or when `windows_subsystem` is used. + /// + /// [here]: https://devblogs.microsoft.com/oldnewthing/20040302-00/?p=40443 + #[inline] + #[rustc_const_stable(feature = "io_safety", since = "1.63.0")] + #[stable(feature = "io_safety", since = "1.63.0")] + pub const unsafe fn borrow_raw(handle: RawHandle) -> Self { + Self { handle, _phantom: PhantomData } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl TryFrom for OwnedHandle { + type Error = NullHandleError; + + #[inline] + fn try_from(handle_or_null: HandleOrNull) -> Result { + let owned_handle = handle_or_null.0; + if owned_handle.handle.is_null() { + // Don't call `CloseHandle`; it'd be harmless, except that it could + // overwrite the `GetLastError` error. + forget(owned_handle); + + Err(NullHandleError(())) + } else { + Ok(owned_handle) + } + } +} + +impl OwnedHandle { + /// Creates a new `OwnedHandle` instance that shares the same underlying + /// object as the existing `OwnedHandle` instance. + #[stable(feature = "io_safety", since = "1.63.0")] + pub fn try_clone(&self) -> crate::io::Result { + self.as_handle().try_clone_to_owned() + } +} + +impl BorrowedHandle<'_> { + /// Creates a new `OwnedHandle` instance that shares the same underlying + /// object as the existing `BorrowedHandle` instance. + #[stable(feature = "io_safety", since = "1.63.0")] + pub fn try_clone_to_owned(&self) -> crate::io::Result { + self.duplicate(0, false, c::DUPLICATE_SAME_ACCESS) + } + + pub(crate) fn duplicate( + &self, + access: c::DWORD, + inherit: bool, + options: c::DWORD, + ) -> io::Result { + let handle = self.as_raw_handle(); + + // `Stdin`, `Stdout`, and `Stderr` can all hold null handles, such as + // in a process with a detached console. `DuplicateHandle` would fail + // if we passed it a null handle, but we can treat null as a valid + // handle which doesn't do any I/O, and allow it to be duplicated. + if handle.is_null() { + return unsafe { Ok(OwnedHandle::from_raw_handle(handle)) }; + } + + let mut ret = ptr::null_mut(); + cvt(unsafe { + let cur_proc = c::GetCurrentProcess(); + c::DuplicateHandle( + cur_proc, + handle, + cur_proc, + &mut ret, + access, + inherit as c::BOOL, + options, + ) + })?; + unsafe { Ok(OwnedHandle::from_raw_handle(ret)) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl TryFrom for OwnedHandle { + type Error = InvalidHandleError; + + #[inline] + fn try_from(handle_or_invalid: HandleOrInvalid) -> Result { + let owned_handle = handle_or_invalid.0; + if owned_handle.handle == c::INVALID_HANDLE_VALUE { + // Don't call `CloseHandle`; it'd be harmless, except that it could + // overwrite the `GetLastError` error. + forget(owned_handle); + + Err(InvalidHandleError(())) + } else { + Ok(owned_handle) + } + } +} + +/// This is the error type used by [`HandleOrNull`] when attempting to convert +/// into a handle, to indicate that the value is null. +// The empty field prevents constructing this, and allows extending it in the future. +#[stable(feature = "io_safety", since = "1.63.0")] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NullHandleError(()); + +#[stable(feature = "io_safety", since = "1.63.0")] +impl fmt::Display for NullHandleError { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + "A HandleOrNull could not be converted to a handle because it was null".fmt(fmt) + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl crate::error::Error for NullHandleError {} + +/// This is the error type used by [`HandleOrInvalid`] when attempting to +/// convert into a handle, to indicate that the value is +/// `INVALID_HANDLE_VALUE`. +// The empty field prevents constructing this, and allows extending it in the future. +#[stable(feature = "io_safety", since = "1.63.0")] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InvalidHandleError(()); + +#[stable(feature = "io_safety", since = "1.63.0")] +impl fmt::Display for InvalidHandleError { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + "A HandleOrInvalid could not be converted to a handle because it was INVALID_HANDLE_VALUE" + .fmt(fmt) + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl crate::error::Error for InvalidHandleError {} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsRawHandle for BorrowedHandle<'_> { + #[inline] + fn as_raw_handle(&self) -> RawHandle { + self.handle + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsRawHandle for OwnedHandle { + #[inline] + fn as_raw_handle(&self) -> RawHandle { + self.handle + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl IntoRawHandle for OwnedHandle { + #[inline] + fn into_raw_handle(self) -> RawHandle { + let handle = self.handle; + forget(self); + handle + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl FromRawHandle for OwnedHandle { + #[inline] + unsafe fn from_raw_handle(handle: RawHandle) -> Self { + Self { handle } + } +} + +impl HandleOrNull { + /// Constructs a new instance of `Self` from the given `RawHandle` returned + /// from a Windows API that uses null to indicate failure, such as + /// `CreateThread`. + /// + /// Use `HandleOrInvalid` instead of `HandleOrNull` for APIs that + /// use `INVALID_HANDLE_VALUE` to indicate failure. + /// + /// # Safety + /// + /// The passed `handle` value must either satisfy the safety requirements + /// of [`FromRawHandle::from_raw_handle`], or be null. Note that not all + /// Windows APIs use null for errors; see [here] for the full story. + /// + /// [here]: https://devblogs.microsoft.com/oldnewthing/20040302-00/?p=40443 + #[stable(feature = "io_safety", since = "1.63.0")] + #[inline] + pub unsafe fn from_raw_handle(handle: RawHandle) -> Self { + Self(OwnedHandle::from_raw_handle(handle)) + } +} + +impl HandleOrInvalid { + /// Constructs a new instance of `Self` from the given `RawHandle` returned + /// from a Windows API that uses `INVALID_HANDLE_VALUE` to indicate + /// failure, such as `CreateFileW`. + /// + /// Use `HandleOrNull` instead of `HandleOrInvalid` for APIs that + /// use null to indicate failure. + /// + /// # Safety + /// + /// The passed `handle` value must either satisfy the safety requirements + /// of [`FromRawHandle::from_raw_handle`], or be + /// `INVALID_HANDLE_VALUE` (-1). Note that not all Windows APIs use + /// `INVALID_HANDLE_VALUE` for errors; see [here] for the full story. + /// + /// [here]: https://devblogs.microsoft.com/oldnewthing/20040302-00/?p=40443 + #[stable(feature = "io_safety", since = "1.63.0")] + #[inline] + pub unsafe fn from_raw_handle(handle: RawHandle) -> Self { + Self(OwnedHandle::from_raw_handle(handle)) + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl Drop for OwnedHandle { + #[inline] + fn drop(&mut self) { + unsafe { + let _ = c::CloseHandle(self.handle); + } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl fmt::Debug for BorrowedHandle<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("BorrowedHandle").field("handle", &self.handle).finish() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl fmt::Debug for OwnedHandle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OwnedHandle").field("handle", &self.handle).finish() + } +} + +macro_rules! impl_is_terminal { + ($($t:ty),*$(,)?) => {$( + #[unstable(feature = "sealed", issue = "none")] + impl crate::sealed::Sealed for $t {} + + #[unstable(feature = "is_terminal", issue = "98070")] + impl crate::io::IsTerminal for $t { + #[inline] + fn is_terminal(&self) -> bool { + crate::sys::io::is_terminal(self) + } + } + )*} +} + +impl_is_terminal!(BorrowedHandle<'_>, OwnedHandle); + +/// A trait to borrow the handle from an underlying object. +#[stable(feature = "io_safety", since = "1.63.0")] +pub trait AsHandle { + /// Borrows the handle. + /// + /// # Example + /// + /// ```rust,no_run + /// use std::fs::File; + /// # use std::io; + /// use std::os::windows::io::{AsHandle, BorrowedHandle}; + /// + /// let mut f = File::open("foo.txt")?; + /// let borrowed_handle: BorrowedHandle<'_> = f.as_handle(); + /// # Ok::<(), io::Error>(()) + /// ``` + #[stable(feature = "io_safety", since = "1.63.0")] + fn as_handle(&self) -> BorrowedHandle<'_>; +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsHandle for &T { + #[inline] + fn as_handle(&self) -> BorrowedHandle<'_> { + T::as_handle(self) + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsHandle for &mut T { + #[inline] + fn as_handle(&self) -> BorrowedHandle<'_> { + T::as_handle(self) + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsHandle for BorrowedHandle<'_> { + #[inline] + fn as_handle(&self) -> BorrowedHandle<'_> { + *self + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsHandle for OwnedHandle { + #[inline] + fn as_handle(&self) -> BorrowedHandle<'_> { + // Safety: `OwnedHandle` and `BorrowedHandle` have the same validity + // invariants, and the `BorrowdHandle` is bounded by the lifetime + // of `&self`. + unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsHandle for fs::File { + #[inline] + fn as_handle(&self) -> BorrowedHandle<'_> { + self.as_inner().as_handle() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for OwnedHandle { + #[inline] + fn from(file: fs::File) -> OwnedHandle { + file.into_inner().into_inner().into_inner().into() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for fs::File { + #[inline] + fn from(owned: OwnedHandle) -> Self { + Self::from_inner(FromInner::from_inner(FromInner::from_inner(owned))) + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsHandle for crate::io::Stdin { + #[inline] + fn as_handle(&self) -> BorrowedHandle<'_> { + unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl<'a> AsHandle for crate::io::StdinLock<'a> { + #[inline] + fn as_handle(&self) -> BorrowedHandle<'_> { + unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsHandle for crate::io::Stdout { + #[inline] + fn as_handle(&self) -> BorrowedHandle<'_> { + unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl<'a> AsHandle for crate::io::StdoutLock<'a> { + #[inline] + fn as_handle(&self) -> BorrowedHandle<'_> { + unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsHandle for crate::io::Stderr { + #[inline] + fn as_handle(&self) -> BorrowedHandle<'_> { + unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl<'a> AsHandle for crate::io::StderrLock<'a> { + #[inline] + fn as_handle(&self) -> BorrowedHandle<'_> { + unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsHandle for crate::process::ChildStdin { + #[inline] + fn as_handle(&self) -> BorrowedHandle<'_> { + unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for OwnedHandle { + #[inline] + fn from(child_stdin: crate::process::ChildStdin) -> OwnedHandle { + unsafe { OwnedHandle::from_raw_handle(child_stdin.into_raw_handle()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsHandle for crate::process::ChildStdout { + #[inline] + fn as_handle(&self) -> BorrowedHandle<'_> { + unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for OwnedHandle { + #[inline] + fn from(child_stdout: crate::process::ChildStdout) -> OwnedHandle { + unsafe { OwnedHandle::from_raw_handle(child_stdout.into_raw_handle()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsHandle for crate::process::ChildStderr { + #[inline] + fn as_handle(&self) -> BorrowedHandle<'_> { + unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for OwnedHandle { + #[inline] + fn from(child_stderr: crate::process::ChildStderr) -> OwnedHandle { + unsafe { OwnedHandle::from_raw_handle(child_stderr.into_raw_handle()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsHandle for crate::thread::JoinHandle { + #[inline] + fn as_handle(&self) -> BorrowedHandle<'_> { + unsafe { BorrowedHandle::borrow_raw(self.as_raw_handle()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From> for OwnedHandle { + #[inline] + fn from(join_handle: crate::thread::JoinHandle) -> OwnedHandle { + join_handle.into_inner().into_handle().into_inner() + } +} diff --git a/crux-mir/lib/std/src/os/windows/io/mod.rs b/crux-mir/lib/std/src/os/windows/io/mod.rs new file mode 100644 index 000000000..e2a401fb6 --- /dev/null +++ b/crux-mir/lib/std/src/os/windows/io/mod.rs @@ -0,0 +1,65 @@ +//! Windows-specific extensions to general I/O primitives. +//! +//! Just like raw pointers, raw Windows handles and sockets point to resources +//! with dynamic lifetimes, and they can dangle if they outlive their resources +//! or be forged if they're created from invalid values. +//! +//! This module provides three types for representing raw handles and sockets +//! with different ownership properties: raw, borrowed, and owned, which are +//! analogous to types used for representing pointers: +//! +//! | Type | Analogous to | +//! | ---------------------- | ------------ | +//! | [`RawHandle`] | `*const _` | +//! | [`RawSocket`] | `*const _` | +//! | | | +//! | [`BorrowedHandle<'a>`] | `&'a _` | +//! | [`BorrowedSocket<'a>`] | `&'a _` | +//! | | | +//! | [`OwnedHandle`] | `Box<_>` | +//! | [`OwnedSocket`] | `Box<_>` | +//! +//! Like raw pointers, `RawHandle` and `RawSocket` values are primitive values. +//! And in new code, they should be considered unsafe to do I/O on (analogous +//! to dereferencing them). Rust did not always provide this guidance, so +//! existing code in the Rust ecosystem often doesn't mark `RawHandle` and +//! `RawSocket` usage as unsafe. Once the `io_safety` feature is stable, +//! libraries will be encouraged to migrate, either by adding `unsafe` to APIs +//! that dereference `RawHandle` and `RawSocket` values, or by using to +//! `BorrowedHandle`, `BorrowedSocket`, `OwnedHandle`, or `OwnedSocket`. +//! +//! Like references, `BorrowedHandle` and `BorrowedSocket` values are tied to a +//! lifetime, to ensure that they don't outlive the resource they point to. +//! These are safe to use. `BorrowedHandle` and `BorrowedSocket` values may be +//! used in APIs which provide safe access to any system call except for +//! `CloseHandle`, `closesocket`, or any other call that would end the +//! dynamic lifetime of the resource without ending the lifetime of the +//! handle or socket. +//! +//! `BorrowedHandle` and `BorrowedSocket` values may be used in APIs which +//! provide safe access to `DuplicateHandle` and `WSADuplicateSocketW` and +//! related functions, so types implementing `AsHandle`, `AsSocket`, +//! `From`, or `From` should not assume they always +//! have exclusive access to the underlying object. +//! +//! Like boxes, `OwnedHandle` and `OwnedSocket` values conceptually own the +//! resource they point to, and free (close) it when they are dropped. +//! +//! [`BorrowedHandle<'a>`]: crate::os::windows::io::BorrowedHandle +//! [`BorrowedSocket<'a>`]: crate::os::windows::io::BorrowedSocket + +#![stable(feature = "rust1", since = "1.0.0")] + +mod handle; +mod raw; +mod socket; + +#[stable(feature = "io_safety", since = "1.63.0")] +pub use handle::*; +#[stable(feature = "rust1", since = "1.0.0")] +pub use raw::*; +#[stable(feature = "io_safety", since = "1.63.0")] +pub use socket::*; + +#[cfg(test)] +mod tests; diff --git a/crux-mir/lib/std/src/os/windows/io/raw.rs b/crux-mir/lib/std/src/os/windows/io/raw.rs new file mode 100644 index 000000000..49e4f304f --- /dev/null +++ b/crux-mir/lib/std/src/os/windows/io/raw.rs @@ -0,0 +1,305 @@ +//! Windows-specific extensions to general I/O primitives. + +#![stable(feature = "rust1", since = "1.0.0")] + +use crate::fs; +use crate::io; +use crate::net; +#[cfg(doc)] +use crate::os::windows::io::{AsHandle, AsSocket}; +use crate::os::windows::io::{OwnedHandle, OwnedSocket}; +use crate::os::windows::raw; +use crate::ptr; +use crate::sys; +use crate::sys::c; +use crate::sys_common::{self, AsInner, FromInner, IntoInner}; + +/// Raw HANDLEs. +#[stable(feature = "rust1", since = "1.0.0")] +pub type RawHandle = raw::HANDLE; + +/// Raw SOCKETs. +#[stable(feature = "rust1", since = "1.0.0")] +pub type RawSocket = raw::SOCKET; + +/// Extracts raw handles. +#[stable(feature = "rust1", since = "1.0.0")] +pub trait AsRawHandle { + /// Extracts the raw handle. + /// + /// This function is typically used to **borrow** an owned handle. + /// When used in this way, this method does **not** pass ownership of the + /// raw handle to the caller, and the handle is only guaranteed + /// to be valid while the original object has not yet been destroyed. + /// + /// This function may return null, such as when called on [`Stdin`], + /// [`Stdout`], or [`Stderr`] when the console is detached. + /// + /// However, borrowing is not strictly required. See [`AsHandle::as_handle`] + /// for an API which strictly borrows a handle. + /// + /// [`Stdin`]: io::Stdin + /// [`Stdout`]: io::Stdout + /// [`Stderr`]: io::Stderr + #[stable(feature = "rust1", since = "1.0.0")] + fn as_raw_handle(&self) -> RawHandle; +} + +/// Construct I/O objects from raw handles. +#[stable(feature = "from_raw_os", since = "1.1.0")] +pub trait FromRawHandle { + /// Constructs a new I/O object from the specified raw handle. + /// + /// This function is typically used to **consume ownership** of the handle + /// given, passing responsibility for closing the handle to the returned + /// object. When used in this way, the returned object + /// will take responsibility for closing it when the object goes out of + /// scope. + /// + /// However, consuming ownership is not strictly required. Use a + /// `From::from` implementation for an API which strictly + /// consumes ownership. + /// + /// # Safety + /// + /// The `handle` passed in must: + /// - be a valid an open handle, + /// - be a handle for a resource that may be freed via [`CloseHandle`] + /// (as opposed to `RegCloseKey` or other close functions). + /// + /// Note that the handle *may* have the value `INVALID_HANDLE_VALUE` (-1), + /// which is sometimes a valid handle value. See [here] for the full story. + /// + /// [`CloseHandle`]: https://docs.microsoft.com/en-us/windows/win32/api/handleapi/nf-handleapi-closehandle + /// [here]: https://devblogs.microsoft.com/oldnewthing/20040302-00/?p=40443 + #[stable(feature = "from_raw_os", since = "1.1.0")] + unsafe fn from_raw_handle(handle: RawHandle) -> Self; +} + +/// A trait to express the ability to consume an object and acquire ownership of +/// its raw `HANDLE`. +#[stable(feature = "into_raw_os", since = "1.4.0")] +pub trait IntoRawHandle { + /// Consumes this object, returning the raw underlying handle. + /// + /// This function is typically used to **transfer ownership** of the underlying + /// handle to the caller. When used in this way, callers are then the unique + /// owners of the handle and must close it once it's no longer needed. + /// + /// However, transferring ownership is not strictly required. Use a + /// `Into::into` implementation for an API which strictly + /// transfers ownership. + #[stable(feature = "into_raw_os", since = "1.4.0")] + fn into_raw_handle(self) -> RawHandle; +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl AsRawHandle for fs::File { + #[inline] + fn as_raw_handle(&self) -> RawHandle { + self.as_inner().as_raw_handle() as RawHandle + } +} + +#[stable(feature = "asraw_stdio", since = "1.21.0")] +impl AsRawHandle for io::Stdin { + fn as_raw_handle(&self) -> RawHandle { + stdio_handle(unsafe { c::GetStdHandle(c::STD_INPUT_HANDLE) as RawHandle }) + } +} + +#[stable(feature = "asraw_stdio", since = "1.21.0")] +impl AsRawHandle for io::Stdout { + fn as_raw_handle(&self) -> RawHandle { + stdio_handle(unsafe { c::GetStdHandle(c::STD_OUTPUT_HANDLE) as RawHandle }) + } +} + +#[stable(feature = "asraw_stdio", since = "1.21.0")] +impl AsRawHandle for io::Stderr { + fn as_raw_handle(&self) -> RawHandle { + stdio_handle(unsafe { c::GetStdHandle(c::STD_ERROR_HANDLE) as RawHandle }) + } +} + +#[stable(feature = "asraw_stdio_locks", since = "1.35.0")] +impl<'a> AsRawHandle for io::StdinLock<'a> { + fn as_raw_handle(&self) -> RawHandle { + stdio_handle(unsafe { c::GetStdHandle(c::STD_INPUT_HANDLE) as RawHandle }) + } +} + +#[stable(feature = "asraw_stdio_locks", since = "1.35.0")] +impl<'a> AsRawHandle for io::StdoutLock<'a> { + fn as_raw_handle(&self) -> RawHandle { + stdio_handle(unsafe { c::GetStdHandle(c::STD_OUTPUT_HANDLE) as RawHandle }) + } +} + +#[stable(feature = "asraw_stdio_locks", since = "1.35.0")] +impl<'a> AsRawHandle for io::StderrLock<'a> { + fn as_raw_handle(&self) -> RawHandle { + stdio_handle(unsafe { c::GetStdHandle(c::STD_ERROR_HANDLE) as RawHandle }) + } +} + +// Translate a handle returned from `GetStdHandle` into a handle to return to +// the user. +fn stdio_handle(raw: RawHandle) -> RawHandle { + // `GetStdHandle` isn't expected to actually fail, so when it returns + // `INVALID_HANDLE_VALUE`, it means we were launched from a parent which + // didn't provide us with stdio handles, such as a parent with a detached + // console. In that case, return null to the user, which is consistent + // with what they'd get in the parent, and which avoids the problem that + // `INVALID_HANDLE_VALUE` aliases the current process handle. + if raw == c::INVALID_HANDLE_VALUE { ptr::null_mut() } else { raw } +} + +#[stable(feature = "from_raw_os", since = "1.1.0")] +impl FromRawHandle for fs::File { + #[inline] + unsafe fn from_raw_handle(handle: RawHandle) -> fs::File { + let handle = handle as c::HANDLE; + fs::File::from_inner(sys::fs::File::from_inner(FromInner::from_inner( + OwnedHandle::from_raw_handle(handle), + ))) + } +} + +#[stable(feature = "into_raw_os", since = "1.4.0")] +impl IntoRawHandle for fs::File { + #[inline] + fn into_raw_handle(self) -> RawHandle { + self.into_inner().into_raw_handle() as *mut _ + } +} + +/// Extracts raw sockets. +#[stable(feature = "rust1", since = "1.0.0")] +pub trait AsRawSocket { + /// Extracts the raw socket. + /// + /// This function is typically used to **borrow** an owned socket. + /// When used in this way, this method does **not** pass ownership of the + /// raw socket to the caller, and the socket is only guaranteed + /// to be valid while the original object has not yet been destroyed. + /// + /// However, borrowing is not strictly required. See [`AsSocket::as_socket`] + /// for an API which strictly borrows a socket. + #[stable(feature = "rust1", since = "1.0.0")] + fn as_raw_socket(&self) -> RawSocket; +} + +/// Creates I/O objects from raw sockets. +#[stable(feature = "from_raw_os", since = "1.1.0")] +pub trait FromRawSocket { + /// Constructs a new I/O object from the specified raw socket. + /// + /// This function is typically used to **consume ownership** of the socket + /// given, passing responsibility for closing the socket to the returned + /// object. When used in this way, the returned object + /// will take responsibility for closing it when the object goes out of + /// scope. + /// + /// However, consuming ownership is not strictly required. Use a + /// `From::from` implementation for an API which strictly + /// consumes ownership. + /// + /// # Safety + /// + /// The `socket` passed in must: + /// - be a valid an open socket, + /// - be a socket that may be freed via [`closesocket`]. + /// + /// [`closesocket`]: https://docs.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-closesocket + #[stable(feature = "from_raw_os", since = "1.1.0")] + unsafe fn from_raw_socket(sock: RawSocket) -> Self; +} + +/// A trait to express the ability to consume an object and acquire ownership of +/// its raw `SOCKET`. +#[stable(feature = "into_raw_os", since = "1.4.0")] +pub trait IntoRawSocket { + /// Consumes this object, returning the raw underlying socket. + /// + /// This function is typically used to **transfer ownership** of the underlying + /// socket to the caller. When used in this way, callers are then the unique + /// owners of the socket and must close it once it's no longer needed. + /// + /// However, transferring ownership is not strictly required. Use a + /// `Into::into` implementation for an API which strictly + /// transfers ownership. + #[stable(feature = "into_raw_os", since = "1.4.0")] + fn into_raw_socket(self) -> RawSocket; +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl AsRawSocket for net::TcpStream { + #[inline] + fn as_raw_socket(&self) -> RawSocket { + self.as_inner().socket().as_raw_socket() + } +} +#[stable(feature = "rust1", since = "1.0.0")] +impl AsRawSocket for net::TcpListener { + #[inline] + fn as_raw_socket(&self) -> RawSocket { + self.as_inner().socket().as_raw_socket() + } +} +#[stable(feature = "rust1", since = "1.0.0")] +impl AsRawSocket for net::UdpSocket { + #[inline] + fn as_raw_socket(&self) -> RawSocket { + self.as_inner().socket().as_raw_socket() + } +} + +#[stable(feature = "from_raw_os", since = "1.1.0")] +impl FromRawSocket for net::TcpStream { + #[inline] + unsafe fn from_raw_socket(sock: RawSocket) -> net::TcpStream { + let sock = sys::net::Socket::from_inner(OwnedSocket::from_raw_socket(sock)); + net::TcpStream::from_inner(sys_common::net::TcpStream::from_inner(sock)) + } +} +#[stable(feature = "from_raw_os", since = "1.1.0")] +impl FromRawSocket for net::TcpListener { + #[inline] + unsafe fn from_raw_socket(sock: RawSocket) -> net::TcpListener { + let sock = sys::net::Socket::from_inner(OwnedSocket::from_raw_socket(sock)); + net::TcpListener::from_inner(sys_common::net::TcpListener::from_inner(sock)) + } +} +#[stable(feature = "from_raw_os", since = "1.1.0")] +impl FromRawSocket for net::UdpSocket { + #[inline] + unsafe fn from_raw_socket(sock: RawSocket) -> net::UdpSocket { + let sock = sys::net::Socket::from_inner(OwnedSocket::from_raw_socket(sock)); + net::UdpSocket::from_inner(sys_common::net::UdpSocket::from_inner(sock)) + } +} + +#[stable(feature = "into_raw_os", since = "1.4.0")] +impl IntoRawSocket for net::TcpStream { + #[inline] + fn into_raw_socket(self) -> RawSocket { + self.into_inner().into_socket().into_inner().into_raw_socket() + } +} + +#[stable(feature = "into_raw_os", since = "1.4.0")] +impl IntoRawSocket for net::TcpListener { + #[inline] + fn into_raw_socket(self) -> RawSocket { + self.into_inner().into_socket().into_inner().into_raw_socket() + } +} + +#[stable(feature = "into_raw_os", since = "1.4.0")] +impl IntoRawSocket for net::UdpSocket { + #[inline] + fn into_raw_socket(self) -> RawSocket { + self.into_inner().into_socket().into_inner().into_raw_socket() + } +} diff --git a/crux-mir/lib/std/src/os/windows/io/socket.rs b/crux-mir/lib/std/src/os/windows/io/socket.rs new file mode 100644 index 000000000..5c1634084 --- /dev/null +++ b/crux-mir/lib/std/src/os/windows/io/socket.rs @@ -0,0 +1,339 @@ +//! Owned and borrowed OS sockets. + +#![stable(feature = "io_safety", since = "1.63.0")] + +use super::raw::{AsRawSocket, FromRawSocket, IntoRawSocket, RawSocket}; +use crate::fmt; +use crate::io; +use crate::marker::PhantomData; +use crate::mem; +use crate::mem::forget; +use crate::sys; +use crate::sys::c; +#[cfg(not(target_vendor = "uwp"))] +use crate::sys::cvt; + +/// A borrowed socket. +/// +/// This has a lifetime parameter to tie it to the lifetime of something that +/// owns the socket. +/// +/// This uses `repr(transparent)` and has the representation of a host socket, +/// so it can be used in FFI in places where a socket is passed as an argument, +/// it is not captured or consumed, and it never has the value +/// `INVALID_SOCKET`. +/// +/// This type's `.to_owned()` implementation returns another `BorrowedSocket` +/// rather than an `OwnedSocket`. It just makes a trivial copy of the raw +/// socket, which is then borrowed under the same lifetime. +#[derive(Copy, Clone)] +#[repr(transparent)] +#[rustc_layout_scalar_valid_range_start(0)] +// This is -2, in two's complement. -1 is `INVALID_SOCKET`. +#[cfg_attr(target_pointer_width = "32", rustc_layout_scalar_valid_range_end(0xFF_FF_FF_FE))] +#[cfg_attr( + target_pointer_width = "64", + rustc_layout_scalar_valid_range_end(0xFF_FF_FF_FF_FF_FF_FF_FE) +)] +#[rustc_nonnull_optimization_guaranteed] +#[stable(feature = "io_safety", since = "1.63.0")] +pub struct BorrowedSocket<'socket> { + socket: RawSocket, + _phantom: PhantomData<&'socket OwnedSocket>, +} + +/// An owned socket. +/// +/// This closes the socket on drop. +/// +/// This uses `repr(transparent)` and has the representation of a host socket, +/// so it can be used in FFI in places where a socket is passed as a consumed +/// argument or returned as an owned value, and it never has the value +/// `INVALID_SOCKET`. +#[repr(transparent)] +#[rustc_layout_scalar_valid_range_start(0)] +// This is -2, in two's complement. -1 is `INVALID_SOCKET`. +#[cfg_attr(target_pointer_width = "32", rustc_layout_scalar_valid_range_end(0xFF_FF_FF_FE))] +#[cfg_attr( + target_pointer_width = "64", + rustc_layout_scalar_valid_range_end(0xFF_FF_FF_FF_FF_FF_FF_FE) +)] +#[rustc_nonnull_optimization_guaranteed] +#[stable(feature = "io_safety", since = "1.63.0")] +pub struct OwnedSocket { + socket: RawSocket, +} + +impl BorrowedSocket<'_> { + /// Return a `BorrowedSocket` holding the given raw socket. + /// + /// # Safety + /// + /// The resource pointed to by `raw` must remain open for the duration of + /// the returned `BorrowedSocket`, and it must not have the value + /// `INVALID_SOCKET`. + #[inline] + #[rustc_const_stable(feature = "io_safety", since = "1.63.0")] + #[stable(feature = "io_safety", since = "1.63.0")] + pub const unsafe fn borrow_raw(socket: RawSocket) -> Self { + assert!(socket != c::INVALID_SOCKET as RawSocket); + Self { socket, _phantom: PhantomData } + } +} + +impl OwnedSocket { + /// Creates a new `OwnedSocket` instance that shares the same underlying + /// object as the existing `OwnedSocket` instance. + #[stable(feature = "io_safety", since = "1.63.0")] + pub fn try_clone(&self) -> io::Result { + self.as_socket().try_clone_to_owned() + } + + // FIXME(strict_provenance_magic): we defined RawSocket to be a u64 ;-; + #[allow(fuzzy_provenance_casts)] + #[cfg(not(target_vendor = "uwp"))] + pub(crate) fn set_no_inherit(&self) -> io::Result<()> { + cvt(unsafe { + c::SetHandleInformation(self.as_raw_socket() as c::HANDLE, c::HANDLE_FLAG_INHERIT, 0) + }) + .map(drop) + } + + #[cfg(target_vendor = "uwp")] + pub(crate) fn set_no_inherit(&self) -> io::Result<()> { + Err(io::const_io_error!(io::ErrorKind::Unsupported, "Unavailable on UWP")) + } +} + +impl BorrowedSocket<'_> { + /// Creates a new `OwnedSocket` instance that shares the same underlying + /// object as the existing `BorrowedSocket` instance. + #[stable(feature = "io_safety", since = "1.63.0")] + pub fn try_clone_to_owned(&self) -> io::Result { + let mut info = unsafe { mem::zeroed::() }; + let result = unsafe { + c::WSADuplicateSocketW(self.as_raw_socket(), c::GetCurrentProcessId(), &mut info) + }; + sys::net::cvt(result)?; + let socket = unsafe { + c::WSASocketW( + info.iAddressFamily, + info.iSocketType, + info.iProtocol, + &mut info, + 0, + c::WSA_FLAG_OVERLAPPED | c::WSA_FLAG_NO_HANDLE_INHERIT, + ) + }; + + if socket != c::INVALID_SOCKET { + unsafe { Ok(OwnedSocket::from_raw_socket(socket)) } + } else { + let error = unsafe { c::WSAGetLastError() }; + + if error != c::WSAEPROTOTYPE && error != c::WSAEINVAL { + return Err(io::Error::from_raw_os_error(error)); + } + + let socket = unsafe { + c::WSASocketW( + info.iAddressFamily, + info.iSocketType, + info.iProtocol, + &mut info, + 0, + c::WSA_FLAG_OVERLAPPED, + ) + }; + + if socket == c::INVALID_SOCKET { + return Err(last_error()); + } + + unsafe { + let socket = OwnedSocket::from_raw_socket(socket); + socket.set_no_inherit()?; + Ok(socket) + } + } + } +} + +/// Returns the last error from the Windows socket interface. +fn last_error() -> io::Error { + io::Error::from_raw_os_error(unsafe { c::WSAGetLastError() }) +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsRawSocket for BorrowedSocket<'_> { + #[inline] + fn as_raw_socket(&self) -> RawSocket { + self.socket + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsRawSocket for OwnedSocket { + #[inline] + fn as_raw_socket(&self) -> RawSocket { + self.socket + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl IntoRawSocket for OwnedSocket { + #[inline] + fn into_raw_socket(self) -> RawSocket { + let socket = self.socket; + forget(self); + socket + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl FromRawSocket for OwnedSocket { + #[inline] + unsafe fn from_raw_socket(socket: RawSocket) -> Self { + debug_assert_ne!(socket, c::INVALID_SOCKET as RawSocket); + Self { socket } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl Drop for OwnedSocket { + #[inline] + fn drop(&mut self) { + unsafe { + let _ = c::closesocket(self.socket); + } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl fmt::Debug for BorrowedSocket<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("BorrowedSocket").field("socket", &self.socket).finish() + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl fmt::Debug for OwnedSocket { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OwnedSocket").field("socket", &self.socket).finish() + } +} + +/// A trait to borrow the socket from an underlying object. +#[stable(feature = "io_safety", since = "1.63.0")] +pub trait AsSocket { + /// Borrows the socket. + #[stable(feature = "io_safety", since = "1.63.0")] + fn as_socket(&self) -> BorrowedSocket<'_>; +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsSocket for &T { + #[inline] + fn as_socket(&self) -> BorrowedSocket<'_> { + T::as_socket(self) + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsSocket for &mut T { + #[inline] + fn as_socket(&self) -> BorrowedSocket<'_> { + T::as_socket(self) + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsSocket for BorrowedSocket<'_> { + #[inline] + fn as_socket(&self) -> BorrowedSocket<'_> { + *self + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsSocket for OwnedSocket { + #[inline] + fn as_socket(&self) -> BorrowedSocket<'_> { + // Safety: `OwnedSocket` and `BorrowedSocket` have the same validity + // invariants, and the `BorrowdSocket` is bounded by the lifetime + // of `&self`. + unsafe { BorrowedSocket::borrow_raw(self.as_raw_socket()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsSocket for crate::net::TcpStream { + #[inline] + fn as_socket(&self) -> BorrowedSocket<'_> { + unsafe { BorrowedSocket::borrow_raw(self.as_raw_socket()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for OwnedSocket { + #[inline] + fn from(tcp_stream: crate::net::TcpStream) -> OwnedSocket { + unsafe { OwnedSocket::from_raw_socket(tcp_stream.into_raw_socket()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for crate::net::TcpStream { + #[inline] + fn from(owned: OwnedSocket) -> Self { + unsafe { Self::from_raw_socket(owned.into_raw_socket()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsSocket for crate::net::TcpListener { + #[inline] + fn as_socket(&self) -> BorrowedSocket<'_> { + unsafe { BorrowedSocket::borrow_raw(self.as_raw_socket()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for OwnedSocket { + #[inline] + fn from(tcp_listener: crate::net::TcpListener) -> OwnedSocket { + unsafe { OwnedSocket::from_raw_socket(tcp_listener.into_raw_socket()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for crate::net::TcpListener { + #[inline] + fn from(owned: OwnedSocket) -> Self { + unsafe { Self::from_raw_socket(owned.into_raw_socket()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsSocket for crate::net::UdpSocket { + #[inline] + fn as_socket(&self) -> BorrowedSocket<'_> { + unsafe { BorrowedSocket::borrow_raw(self.as_raw_socket()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for OwnedSocket { + #[inline] + fn from(udp_socket: crate::net::UdpSocket) -> OwnedSocket { + unsafe { OwnedSocket::from_raw_socket(udp_socket.into_raw_socket()) } + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for crate::net::UdpSocket { + #[inline] + fn from(owned: OwnedSocket) -> Self { + unsafe { Self::from_raw_socket(owned.into_raw_socket()) } + } +} diff --git a/crux-mir/lib/std/src/os/windows/io/tests.rs b/crux-mir/lib/std/src/os/windows/io/tests.rs new file mode 100644 index 000000000..41734e52e --- /dev/null +++ b/crux-mir/lib/std/src/os/windows/io/tests.rs @@ -0,0 +1,21 @@ +#[test] +fn test_niche_optimizations_socket() { + use crate::mem::size_of; + use crate::os::windows::io::{ + BorrowedSocket, FromRawSocket, IntoRawSocket, OwnedSocket, RawSocket, + }; + + assert_eq!(size_of::>(), size_of::()); + assert_eq!(size_of::>>(), size_of::(),); + unsafe { + #[cfg(target_pointer_width = "32")] + let (min, max) = (i32::MIN as u32, i32::MAX as u32); + #[cfg(target_pointer_width = "64")] + let (min, max) = (i64::MIN as u64, i64::MAX as u64); + + assert_eq!(OwnedSocket::from_raw_socket(min).into_raw_socket(), min); + assert_eq!(OwnedSocket::from_raw_socket(max).into_raw_socket(), max); + assert_eq!(Some(OwnedSocket::from_raw_socket(min)).unwrap().into_raw_socket(), min); + assert_eq!(Some(OwnedSocket::from_raw_socket(max)).unwrap().into_raw_socket(), max); + } +} diff --git a/crux-mir/lib/std/src/sys/windows/ext/mod.rs b/crux-mir/lib/std/src/os/windows/mod.rs similarity index 72% rename from crux-mir/lib/std/src/sys/windows/ext/mod.rs rename to crux-mir/lib/std/src/os/windows/mod.rs index 613d3dc18..52eb3b7c0 100644 --- a/crux-mir/lib/std/src/sys/windows/ext/mod.rs +++ b/crux-mir/lib/std/src/os/windows/mod.rs @@ -5,10 +5,25 @@ //! the core `std` library. These extensions allow developers to use //! `std` types and idioms with Windows in a way that the normal //! platform-agnostic idioms would not normally support. +//! +//! # Examples +//! +//! ```no_run +//! use std::fs::File; +//! use std::os::windows::prelude::*; +//! +//! fn main() -> std::io::Result<()> { +//! let f = File::create("foo.txt")?; +//! let handle = f.as_raw_handle(); +//! +//! // use handle with native windows bindings +//! +//! Ok(()) +//! } +//! ``` #![stable(feature = "rust1", since = "1.0.0")] #![doc(cfg(windows))] -#![allow(missing_docs)] pub mod ffi; pub mod fs; @@ -33,8 +48,11 @@ pub mod prelude { pub use super::fs::{MetadataExt, OpenOptionsExt}; #[doc(no_inline)] #[stable(feature = "rust1", since = "1.0.0")] - pub use super::io::{AsRawHandle, AsRawSocket, RawHandle, RawSocket}; + pub use super::io::{ + AsHandle, AsSocket, BorrowedHandle, BorrowedSocket, FromRawHandle, FromRawSocket, + HandleOrInvalid, IntoRawHandle, IntoRawSocket, OwnedHandle, OwnedSocket, + }; #[doc(no_inline)] #[stable(feature = "rust1", since = "1.0.0")] - pub use super::io::{FromRawHandle, FromRawSocket, IntoRawHandle, IntoRawSocket}; + pub use super::io::{AsRawHandle, AsRawSocket, RawHandle, RawSocket}; } diff --git a/crux-mir/lib/std/src/os/windows/process.rs b/crux-mir/lib/std/src/os/windows/process.rs new file mode 100644 index 000000000..073168cf2 --- /dev/null +++ b/crux-mir/lib/std/src/os/windows/process.rs @@ -0,0 +1,259 @@ +//! Windows-specific extensions to primitives in the [`std::process`] module. +//! +//! [`std::process`]: crate::process + +#![stable(feature = "process_extensions", since = "1.2.0")] + +use crate::ffi::OsStr; +use crate::os::windows::io::{ + AsHandle, AsRawHandle, BorrowedHandle, FromRawHandle, IntoRawHandle, OwnedHandle, RawHandle, +}; +use crate::process; +use crate::sealed::Sealed; +use crate::sys; +use crate::sys_common::{AsInner, AsInnerMut, FromInner, IntoInner}; + +#[stable(feature = "process_extensions", since = "1.2.0")] +impl FromRawHandle for process::Stdio { + unsafe fn from_raw_handle(handle: RawHandle) -> process::Stdio { + let handle = sys::handle::Handle::from_raw_handle(handle as *mut _); + let io = sys::process::Stdio::Handle(handle); + process::Stdio::from_inner(io) + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for process::Stdio { + fn from(handle: OwnedHandle) -> process::Stdio { + let handle = sys::handle::Handle::from_inner(handle); + let io = sys::process::Stdio::Handle(handle); + process::Stdio::from_inner(io) + } +} + +#[stable(feature = "process_extensions", since = "1.2.0")] +impl AsRawHandle for process::Child { + #[inline] + fn as_raw_handle(&self) -> RawHandle { + self.as_inner().handle().as_raw_handle() as *mut _ + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl AsHandle for process::Child { + #[inline] + fn as_handle(&self) -> BorrowedHandle<'_> { + self.as_inner().handle().as_handle() + } +} + +#[stable(feature = "into_raw_os", since = "1.4.0")] +impl IntoRawHandle for process::Child { + fn into_raw_handle(self) -> RawHandle { + self.into_inner().into_handle().into_raw_handle() as *mut _ + } +} + +#[stable(feature = "io_safety", since = "1.63.0")] +impl From for OwnedHandle { + fn from(child: process::Child) -> OwnedHandle { + child.into_inner().into_handle().into_inner() + } +} + +#[stable(feature = "process_extensions", since = "1.2.0")] +impl AsRawHandle for process::ChildStdin { + #[inline] + fn as_raw_handle(&self) -> RawHandle { + self.as_inner().handle().as_raw_handle() as *mut _ + } +} + +#[stable(feature = "process_extensions", since = "1.2.0")] +impl AsRawHandle for process::ChildStdout { + #[inline] + fn as_raw_handle(&self) -> RawHandle { + self.as_inner().handle().as_raw_handle() as *mut _ + } +} + +#[stable(feature = "process_extensions", since = "1.2.0")] +impl AsRawHandle for process::ChildStderr { + #[inline] + fn as_raw_handle(&self) -> RawHandle { + self.as_inner().handle().as_raw_handle() as *mut _ + } +} + +#[stable(feature = "into_raw_os", since = "1.4.0")] +impl IntoRawHandle for process::ChildStdin { + fn into_raw_handle(self) -> RawHandle { + self.into_inner().into_handle().into_raw_handle() as *mut _ + } +} + +#[stable(feature = "into_raw_os", since = "1.4.0")] +impl IntoRawHandle for process::ChildStdout { + fn into_raw_handle(self) -> RawHandle { + self.into_inner().into_handle().into_raw_handle() as *mut _ + } +} + +#[stable(feature = "into_raw_os", since = "1.4.0")] +impl IntoRawHandle for process::ChildStderr { + fn into_raw_handle(self) -> RawHandle { + self.into_inner().into_handle().into_raw_handle() as *mut _ + } +} + +/// Windows-specific extensions to [`process::ExitStatus`]. +/// +/// This trait is sealed: it cannot be implemented outside the standard library. +/// This is so that future additional methods are not breaking changes. +#[stable(feature = "exit_status_from", since = "1.12.0")] +pub trait ExitStatusExt: Sealed { + /// Creates a new `ExitStatus` from the raw underlying `u32` return value of + /// a process. + #[stable(feature = "exit_status_from", since = "1.12.0")] + fn from_raw(raw: u32) -> Self; +} + +#[stable(feature = "exit_status_from", since = "1.12.0")] +impl ExitStatusExt for process::ExitStatus { + fn from_raw(raw: u32) -> Self { + process::ExitStatus::from_inner(From::from(raw)) + } +} + +/// Windows-specific extensions to the [`process::Command`] builder. +/// +/// This trait is sealed: it cannot be implemented outside the standard library. +/// This is so that future additional methods are not breaking changes. +#[stable(feature = "windows_process_extensions", since = "1.16.0")] +pub trait CommandExt: Sealed { + /// Sets the [process creation flags][1] to be passed to `CreateProcess`. + /// + /// These will always be ORed with `CREATE_UNICODE_ENVIRONMENT`. + /// + /// [1]: https://docs.microsoft.com/en-us/windows/win32/procthread/process-creation-flags + #[stable(feature = "windows_process_extensions", since = "1.16.0")] + fn creation_flags(&mut self, flags: u32) -> &mut process::Command; + + /// Forces all arguments to be wrapped in quote (`"`) characters. + /// + /// This is useful for passing arguments to [MSYS2/Cygwin][1] based + /// executables: these programs will expand unquoted arguments containing + /// wildcard characters (`?` and `*`) by searching for any file paths + /// matching the wildcard pattern. + /// + /// Adding quotes has no effect when passing arguments to programs + /// that use [msvcrt][2]. This includes programs built with both + /// MinGW and MSVC. + /// + /// [1]: + /// [2]: + #[unstable(feature = "windows_process_extensions_force_quotes", issue = "82227")] + fn force_quotes(&mut self, enabled: bool) -> &mut process::Command; + + /// Append literal text to the command line without any quoting or escaping. + /// + /// This is useful for passing arguments to `cmd.exe /c`, which doesn't follow + /// `CommandLineToArgvW` escaping rules. + #[stable(feature = "windows_process_extensions_raw_arg", since = "1.62.0")] + fn raw_arg>(&mut self, text_to_append_as_is: S) -> &mut process::Command; + + /// When [`process::Command`] creates pipes, request that our side is always async. + /// + /// By default [`process::Command`] may choose to use pipes where both ends + /// are opened for synchronous read or write operations. By using + /// `async_pipes(true)`, this behavior is overridden so that our side is + /// always async. + /// + /// This is important because if doing async I/O a pipe or a file has to be + /// opened for async access. + /// + /// The end of the pipe sent to the child process will always be synchronous + /// regardless of this option. + /// + /// # Example + /// + /// ``` + /// #![feature(windows_process_extensions_async_pipes)] + /// use std::os::windows::process::CommandExt; + /// use std::process::{Command, Stdio}; + /// + /// # let program = ""; + /// + /// Command::new(program) + /// .async_pipes(true) + /// .stdin(Stdio::piped()) + /// .stdout(Stdio::piped()) + /// .stderr(Stdio::piped()); + /// ``` + #[unstable(feature = "windows_process_extensions_async_pipes", issue = "98289")] + fn async_pipes(&mut self, always_async: bool) -> &mut process::Command; +} + +#[stable(feature = "windows_process_extensions", since = "1.16.0")] +impl CommandExt for process::Command { + fn creation_flags(&mut self, flags: u32) -> &mut process::Command { + self.as_inner_mut().creation_flags(flags); + self + } + + fn force_quotes(&mut self, enabled: bool) -> &mut process::Command { + self.as_inner_mut().force_quotes(enabled); + self + } + + fn raw_arg>(&mut self, raw_text: S) -> &mut process::Command { + self.as_inner_mut().raw_arg(raw_text.as_ref()); + self + } + + fn async_pipes(&mut self, always_async: bool) -> &mut process::Command { + // FIXME: This currently has an intentional no-op implementation. + // For the time being our side of the pipes will always be async. + // Once the ecosystem has adjusted, we may then be able to start making + // use of synchronous pipes within the standard library. + let _ = always_async; + self + } +} + +#[unstable(feature = "windows_process_extensions_main_thread_handle", issue = "96723")] +pub trait ChildExt: Sealed { + /// Extracts the main thread raw handle, without taking ownership + #[unstable(feature = "windows_process_extensions_main_thread_handle", issue = "96723")] + fn main_thread_handle(&self) -> BorrowedHandle<'_>; +} + +#[unstable(feature = "windows_process_extensions_main_thread_handle", issue = "96723")] +impl ChildExt for process::Child { + fn main_thread_handle(&self) -> BorrowedHandle<'_> { + self.handle.main_thread_handle() + } +} + +/// Windows-specific extensions to [`process::ExitCode`]. +/// +/// This trait is sealed: it cannot be implemented outside the standard library. +/// This is so that future additional methods are not breaking changes. +#[unstable(feature = "windows_process_exit_code_from", issue = "none")] +pub trait ExitCodeExt: Sealed { + /// Creates a new `ExitCode` from the raw underlying `u32` return value of + /// a process. + /// + /// The exit code should not be 259, as this conflicts with the `STILL_ACTIVE` + /// macro returned from the `GetExitCodeProcess` function to signal that the + /// process has yet to run to completion. + #[unstable(feature = "windows_process_exit_code_from", issue = "none")] + fn from_raw(raw: u32) -> Self; +} + +#[unstable(feature = "windows_process_exit_code_from", issue = "none")] +impl ExitCodeExt for process::ExitCode { + fn from_raw(raw: u32) -> Self { + process::ExitCode::from_inner(From::from(raw)) + } +} diff --git a/crux-mir/lib/std/src/sys/windows/ext/raw.rs b/crux-mir/lib/std/src/os/windows/raw.rs similarity index 84% rename from crux-mir/lib/std/src/sys/windows/ext/raw.rs rename to crux-mir/lib/std/src/os/windows/raw.rs index 7f2a28778..0ef3adade 100644 --- a/crux-mir/lib/std/src/sys/windows/ext/raw.rs +++ b/crux-mir/lib/std/src/os/windows/raw.rs @@ -1,4 +1,4 @@ -//! Windows-specific primitives +//! Windows-specific primitives. #![stable(feature = "raw_ext", since = "1.1.0")] @@ -7,8 +7,10 @@ use crate::os::raw::c_void; #[stable(feature = "raw_ext", since = "1.1.0")] pub type HANDLE = *mut c_void; #[cfg(target_pointer_width = "32")] +#[doc(cfg(all()))] #[stable(feature = "raw_ext", since = "1.1.0")] pub type SOCKET = u32; #[cfg(target_pointer_width = "64")] +#[doc(cfg(all()))] #[stable(feature = "raw_ext", since = "1.1.0")] pub type SOCKET = u64; diff --git a/crux-mir/lib/std/src/sys/windows/ext/thread.rs b/crux-mir/lib/std/src/os/windows/thread.rs similarity index 65% rename from crux-mir/lib/std/src/sys/windows/ext/thread.rs rename to crux-mir/lib/std/src/os/windows/thread.rs index 41c29f5b9..d81d6d0ac 100644 --- a/crux-mir/lib/std/src/sys/windows/ext/thread.rs +++ b/crux-mir/lib/std/src/os/windows/thread.rs @@ -1,4 +1,6 @@ -//! Extensions to `std::thread` for Windows. +//! Windows-specific extensions to primitives in the [`std::thread`] module. +//! +//! [`std::thread`]: crate::thread #![stable(feature = "thread_extensions", since = "1.9.0")] @@ -8,14 +10,16 @@ use crate::thread; #[stable(feature = "thread_extensions", since = "1.9.0")] impl AsRawHandle for thread::JoinHandle { + #[inline] fn as_raw_handle(&self) -> RawHandle { - self.as_inner().handle().raw() as *mut _ + self.as_inner().handle().as_raw_handle() as *mut _ } } #[stable(feature = "thread_extensions", since = "1.9.0")] impl IntoRawHandle for thread::JoinHandle { + #[inline] fn into_raw_handle(self) -> RawHandle { - self.into_inner().into_handle().into_raw() as *mut _ + self.into_inner().into_handle().into_raw_handle() as *mut _ } } diff --git a/crux-mir/lib/std/src/panic.rs b/crux-mir/lib/std/src/panic.rs index 6ad5519d3..9fa8f5702 100644 --- a/crux-mir/lib/std/src/panic.rs +++ b/crux-mir/lib/std/src/panic.rs @@ -3,287 +3,74 @@ #![stable(feature = "std_panic", since = "1.9.0")] use crate::any::Any; -use crate::cell::UnsafeCell; use crate::collections; -use crate::fmt; -use crate::future::Future; -use crate::ops::{Deref, DerefMut}; use crate::panicking; -use crate::pin::Pin; -use crate::ptr::{NonNull, Unique}; -use crate::rc::Rc; -use crate::sync::atomic; -use crate::sync::{Arc, Mutex, RwLock}; -use crate::task::{Context, Poll}; +use crate::sync::atomic::{AtomicUsize, Ordering}; +use crate::sync::{Mutex, RwLock}; use crate::thread::Result; +#[doc(hidden)] +#[unstable(feature = "edition_panic", issue = "none", reason = "use panic!() instead")] +#[allow_internal_unstable(libstd_sys_internals, const_format_args, core_panic, rt)] +#[cfg_attr(not(test), rustc_diagnostic_item = "std_panic_2015_macro")] +#[rustc_macro_transparency = "semitransparent"] +pub macro panic_2015 { + () => ({ + $crate::rt::begin_panic("explicit panic") + }), + ($msg:expr $(,)?) => ({ + $crate::rt::begin_panic($msg) + }), + // Special-case the single-argument case for const_panic. + ("{}", $arg:expr $(,)?) => ({ + $crate::rt::panic_display(&$arg) + }), + ($fmt:expr, $($arg:tt)+) => ({ + $crate::rt::panic_fmt($crate::const_format_args!($fmt, $($arg)+)) + }), +} + +#[doc(hidden)] +#[unstable(feature = "edition_panic", issue = "none", reason = "use panic!() instead")] +pub use core::panic::panic_2021; + #[stable(feature = "panic_hooks", since = "1.10.0")] pub use crate::panicking::{set_hook, take_hook}; +#[unstable(feature = "panic_update_hook", issue = "92649")] +pub use crate::panicking::update_hook; + #[stable(feature = "panic_hooks", since = "1.10.0")] pub use core::panic::{Location, PanicInfo}; -/// A marker trait which represents "panic safe" types in Rust. -/// -/// This trait is implemented by default for many types and behaves similarly in -/// terms of inference of implementation to the [`Send`] and [`Sync`] traits. The -/// purpose of this trait is to encode what types are safe to cross a [`catch_unwind`] -/// boundary with no fear of unwind safety. -/// -/// [`Send`]: ../marker/trait.Send.html -/// [`Sync`]: ../marker/trait.Sync.html -/// [`catch_unwind`]: ./fn.catch_unwind.html -/// -/// ## What is unwind safety? -/// -/// In Rust a function can "return" early if it either panics or calls a -/// function which transitively panics. This sort of control flow is not always -/// anticipated, and has the possibility of causing subtle bugs through a -/// combination of two critical components: -/// -/// 1. A data structure is in a temporarily invalid state when the thread -/// panics. -/// 2. This broken invariant is then later observed. -/// -/// Typically in Rust, it is difficult to perform step (2) because catching a -/// panic involves either spawning a thread (which in turns makes it difficult -/// to later witness broken invariants) or using the `catch_unwind` function in this -/// module. Additionally, even if an invariant is witnessed, it typically isn't a -/// problem in Rust because there are no uninitialized values (like in C or C++). -/// -/// It is possible, however, for **logical** invariants to be broken in Rust, -/// which can end up causing behavioral bugs. Another key aspect of unwind safety -/// in Rust is that, in the absence of `unsafe` code, a panic cannot lead to -/// memory unsafety. -/// -/// That was a bit of a whirlwind tour of unwind safety, but for more information -/// about unwind safety and how it applies to Rust, see an [associated RFC][rfc]. -/// -/// [rfc]: https://github.com/rust-lang/rfcs/blob/master/text/1236-stabilize-catch-panic.md -/// -/// ## What is `UnwindSafe`? -/// -/// Now that we've got an idea of what unwind safety is in Rust, it's also -/// important to understand what this trait represents. As mentioned above, one -/// way to witness broken invariants is through the `catch_unwind` function in this -/// module as it allows catching a panic and then re-using the environment of -/// the closure. -/// -/// Simply put, a type `T` implements `UnwindSafe` if it cannot easily allow -/// witnessing a broken invariant through the use of `catch_unwind` (catching a -/// panic). This trait is an auto trait, so it is automatically implemented for -/// many types, and it is also structurally composed (e.g., a struct is unwind -/// safe if all of its components are unwind safe). -/// -/// Note, however, that this is not an unsafe trait, so there is not a succinct -/// contract that this trait is providing. Instead it is intended as more of a -/// "speed bump" to alert users of `catch_unwind` that broken invariants may be -/// witnessed and may need to be accounted for. -/// -/// ## Who implements `UnwindSafe`? -/// -/// Types such as `&mut T` and `&RefCell` are examples which are **not** -/// unwind safe. The general idea is that any mutable state which can be shared -/// across `catch_unwind` is not unwind safe by default. This is because it is very -/// easy to witness a broken invariant outside of `catch_unwind` as the data is -/// simply accessed as usual. -/// -/// Types like `&Mutex`, however, are unwind safe because they implement -/// poisoning by default. They still allow witnessing a broken invariant, but -/// they already provide their own "speed bumps" to do so. -/// -/// ## When should `UnwindSafe` be used? -/// -/// It is not intended that most types or functions need to worry about this trait. -/// It is only used as a bound on the `catch_unwind` function and as mentioned -/// above, the lack of `unsafe` means it is mostly an advisory. The -/// [`AssertUnwindSafe`] wrapper struct can be used to force this trait to be -/// implemented for any closed over variables passed to `catch_unwind`. -/// -/// [`AssertUnwindSafe`]: ./struct.AssertUnwindSafe.html -#[stable(feature = "catch_unwind", since = "1.9.0")] -#[rustc_on_unimplemented( - message = "the type `{Self}` may not be safely transferred across an unwind boundary", - label = "`{Self}` may not be safely transferred across an unwind boundary" -)] -pub auto trait UnwindSafe {} - -/// A marker trait representing types where a shared reference is considered -/// unwind safe. -/// -/// This trait is namely not implemented by [`UnsafeCell`], the root of all -/// interior mutability. -/// -/// This is a "helper marker trait" used to provide impl blocks for the -/// [`UnwindSafe`] trait, for more information see that documentation. -/// -/// [`UnsafeCell`]: ../cell/struct.UnsafeCell.html -/// [`UnwindSafe`]: ./trait.UnwindSafe.html #[stable(feature = "catch_unwind", since = "1.9.0")] -#[rustc_on_unimplemented( - message = "the type `{Self}` may contain interior mutability and a reference may not be safely \ - transferrable across a catch_unwind boundary", - label = "`{Self}` may contain interior mutability and a reference may not be safely \ - transferrable across a catch_unwind boundary" -)] -pub auto trait RefUnwindSafe {} +pub use core::panic::{AssertUnwindSafe, RefUnwindSafe, UnwindSafe}; -/// A simple wrapper around a type to assert that it is unwind safe. -/// -/// When using [`catch_unwind`] it may be the case that some of the closed over -/// variables are not unwind safe. For example if `&mut T` is captured the -/// compiler will generate a warning indicating that it is not unwind safe. It -/// may not be the case, however, that this is actually a problem due to the -/// specific usage of [`catch_unwind`] if unwind safety is specifically taken into -/// account. This wrapper struct is useful for a quick and lightweight -/// annotation that a variable is indeed unwind safe. +/// Panic the current thread with the given message as the panic payload. /// -/// [`catch_unwind`]: ./fn.catch_unwind.html -/// # Examples -/// -/// One way to use `AssertUnwindSafe` is to assert that the entire closure -/// itself is unwind safe, bypassing all checks for all variables: -/// -/// ``` -/// use std::panic::{self, AssertUnwindSafe}; +/// The message can be of any (`Any + Send`) type, not just strings. /// -/// let mut variable = 4; -/// -/// // This code will not compile because the closure captures `&mut variable` -/// // which is not considered unwind safe by default. -/// -/// // panic::catch_unwind(|| { -/// // variable += 3; -/// // }); -/// -/// // This, however, will compile due to the `AssertUnwindSafe` wrapper -/// let result = panic::catch_unwind(AssertUnwindSafe(|| { -/// variable += 3; -/// })); -/// // ... -/// ``` +/// The message is wrapped in a `Box<'static + Any + Send>`, which can be +/// accessed later using [`PanicInfo::payload`]. /// -/// Wrapping the entire closure amounts to a blanket assertion that all captured -/// variables are unwind safe. This has the downside that if new captures are -/// added in the future, they will also be considered unwind safe. Therefore, -/// you may prefer to just wrap individual captures, as shown below. This is -/// more annotation, but it ensures that if a new capture is added which is not -/// unwind safe, you will get a compilation error at that time, which will -/// allow you to consider whether that new capture in fact represent a bug or -/// not. -/// -/// ``` -/// use std::panic::{self, AssertUnwindSafe}; -/// -/// let mut variable = 4; -/// let other_capture = 3; -/// -/// let result = { -/// let mut wrapper = AssertUnwindSafe(&mut variable); -/// panic::catch_unwind(move || { -/// **wrapper += other_capture; -/// }) -/// }; -/// // ... -/// ``` -#[stable(feature = "catch_unwind", since = "1.9.0")] -pub struct AssertUnwindSafe(#[stable(feature = "catch_unwind", since = "1.9.0")] pub T); - -// Implementations of the `UnwindSafe` trait: -// -// * By default everything is unwind safe -// * pointers T contains mutability of some form are not unwind safe -// * Unique, an owning pointer, lifts an implementation -// * Types like Mutex/RwLock which are explicitly poisoned are unwind safe -// * Our custom AssertUnwindSafe wrapper is indeed unwind safe +/// See the [`panic!`] macro for more information about panicking. +#[stable(feature = "panic_any", since = "1.51.0")] +#[inline] +#[track_caller] +pub fn panic_any(msg: M) -> ! { + crate::panicking::begin_panic(msg); +} -#[stable(feature = "catch_unwind", since = "1.9.0")] -impl !UnwindSafe for &mut T {} -#[stable(feature = "catch_unwind", since = "1.9.0")] -impl UnwindSafe for &T {} -#[stable(feature = "catch_unwind", since = "1.9.0")] -impl UnwindSafe for *const T {} -#[stable(feature = "catch_unwind", since = "1.9.0")] -impl UnwindSafe for *mut T {} -#[unstable(feature = "ptr_internals", issue = "none")] -impl UnwindSafe for Unique {} -#[stable(feature = "nonnull", since = "1.25.0")] -impl UnwindSafe for NonNull {} #[stable(feature = "catch_unwind", since = "1.9.0")] impl UnwindSafe for Mutex {} #[stable(feature = "catch_unwind", since = "1.9.0")] impl UnwindSafe for RwLock {} -#[stable(feature = "catch_unwind", since = "1.9.0")] -impl UnwindSafe for AssertUnwindSafe {} - -// not covered via the Shared impl above b/c the inner contents use -// Cell/AtomicUsize, but the usage here is unwind safe so we can lift the -// impl up one level to Arc/Rc itself -#[stable(feature = "catch_unwind", since = "1.9.0")] -impl UnwindSafe for Rc {} -#[stable(feature = "catch_unwind", since = "1.9.0")] -impl UnwindSafe for Arc {} - -// Pretty simple implementations for the `RefUnwindSafe` marker trait, -// basically just saying that `UnsafeCell` is the -// only thing which doesn't implement it (which then transitively applies to -// everything else). -#[stable(feature = "catch_unwind", since = "1.9.0")] -impl !RefUnwindSafe for UnsafeCell {} -#[stable(feature = "catch_unwind", since = "1.9.0")] -impl RefUnwindSafe for AssertUnwindSafe {} #[stable(feature = "unwind_safe_lock_refs", since = "1.12.0")] impl RefUnwindSafe for Mutex {} #[stable(feature = "unwind_safe_lock_refs", since = "1.12.0")] impl RefUnwindSafe for RwLock {} -#[cfg(target_has_atomic_load_store = "ptr")] -#[stable(feature = "unwind_safe_atomic_refs", since = "1.14.0")] -impl RefUnwindSafe for atomic::AtomicIsize {} -#[cfg(target_has_atomic_load_store = "8")] -#[unstable(feature = "integer_atomics", issue = "32976")] -impl RefUnwindSafe for atomic::AtomicI8 {} -#[cfg(target_has_atomic_load_store = "16")] -#[unstable(feature = "integer_atomics", issue = "32976")] -impl RefUnwindSafe for atomic::AtomicI16 {} -#[cfg(target_has_atomic_load_store = "32")] -#[unstable(feature = "integer_atomics", issue = "32976")] -impl RefUnwindSafe for atomic::AtomicI32 {} -#[cfg(target_has_atomic_load_store = "64")] -#[unstable(feature = "integer_atomics", issue = "32976")] -impl RefUnwindSafe for atomic::AtomicI64 {} -#[cfg(target_has_atomic_load_store = "128")] -#[unstable(feature = "integer_atomics", issue = "32976")] -impl RefUnwindSafe for atomic::AtomicI128 {} - -#[cfg(target_has_atomic_load_store = "ptr")] -#[stable(feature = "unwind_safe_atomic_refs", since = "1.14.0")] -impl RefUnwindSafe for atomic::AtomicUsize {} -#[cfg(target_has_atomic_load_store = "8")] -#[unstable(feature = "integer_atomics", issue = "32976")] -impl RefUnwindSafe for atomic::AtomicU8 {} -#[cfg(target_has_atomic_load_store = "16")] -#[unstable(feature = "integer_atomics", issue = "32976")] -impl RefUnwindSafe for atomic::AtomicU16 {} -#[cfg(target_has_atomic_load_store = "32")] -#[unstable(feature = "integer_atomics", issue = "32976")] -impl RefUnwindSafe for atomic::AtomicU32 {} -#[cfg(target_has_atomic_load_store = "64")] -#[unstable(feature = "integer_atomics", issue = "32976")] -impl RefUnwindSafe for atomic::AtomicU64 {} -#[cfg(target_has_atomic_load_store = "128")] -#[unstable(feature = "integer_atomics", issue = "32976")] -impl RefUnwindSafe for atomic::AtomicU128 {} - -#[cfg(target_has_atomic_load_store = "8")] -#[stable(feature = "unwind_safe_atomic_refs", since = "1.14.0")] -impl RefUnwindSafe for atomic::AtomicBool {} - -#[cfg(target_has_atomic_load_store = "ptr")] -#[stable(feature = "unwind_safe_atomic_refs", since = "1.14.0")] -impl RefUnwindSafe for atomic::AtomicPtr {} - // https://github.com/rust-lang/rust/issues/62301 #[stable(feature = "hashbrown", since = "1.36.0")] impl UnwindSafe for collections::HashMap @@ -294,48 +81,6 @@ where { } -#[stable(feature = "catch_unwind", since = "1.9.0")] -impl Deref for AssertUnwindSafe { - type Target = T; - - fn deref(&self) -> &T { - &self.0 - } -} - -#[stable(feature = "catch_unwind", since = "1.9.0")] -impl DerefMut for AssertUnwindSafe { - fn deref_mut(&mut self) -> &mut T { - &mut self.0 - } -} - -#[stable(feature = "catch_unwind", since = "1.9.0")] -impl R> FnOnce<()> for AssertUnwindSafe { - type Output = R; - - extern "rust-call" fn call_once(self, _args: ()) -> R { - (self.0)() - } -} - -#[stable(feature = "std_debug", since = "1.16.0")] -impl fmt::Debug for AssertUnwindSafe { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("AssertUnwindSafe").field(&self.0).finish() - } -} - -#[stable(feature = "futures_api", since = "1.36.0")] -impl Future for AssertUnwindSafe { - type Output = F::Output; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let pinned_field = unsafe { Pin::map_unchecked_mut(self, |x| &mut x.0) }; - F::poll(pinned_field, cx) - } -} - /// Invokes a closure, capturing the cause of an unwinding panic if one occurs. /// /// This function will return `Ok` with the closure's result if the closure @@ -352,8 +97,6 @@ impl Future for AssertUnwindSafe { /// can fail on a regular basis. Additionally, this function is not guaranteed /// to catch all panics, see the "Notes" section below. /// -/// [`Result`]: ../result/enum.Result.html -/// /// The closure provided is required to adhere to the [`UnwindSafe`] trait to ensure /// that all captured variables are safe to cross this boundary. The purpose of /// this bound is to encode the concept of [exception safety][rfc] in the type @@ -362,18 +105,21 @@ impl Future for AssertUnwindSafe { /// becomes a problem the [`AssertUnwindSafe`] wrapper struct can be used to quickly /// assert that the usage here is indeed unwind safe. /// -/// [`AssertUnwindSafe`]: ./struct.AssertUnwindSafe.html -/// [`UnwindSafe`]: ./trait.UnwindSafe.html -/// /// [rfc]: https://github.com/rust-lang/rfcs/blob/master/text/1236-stabilize-catch-panic.md /// /// # Notes /// -/// Note that this function **may not catch all panics** in Rust. A panic in +/// Note that this function **might not catch all panics** in Rust. A panic in /// Rust is not always implemented via unwinding, but can be implemented by /// aborting the process as well. This function *only* catches unwinding panics, /// not those that abort the process. /// +/// Note that if a custom panic hook has been set, it will be invoked before +/// the panic is caught, before unwinding. +/// +/// Also note that unwinding into Rust code with a foreign exception (e.g. +/// an exception thrown from C++ code) is undefined behavior. +/// /// # Examples /// /// ``` @@ -399,8 +145,6 @@ pub fn catch_unwind R + UnwindSafe, R>(f: F) -> Result { /// This is designed to be used in conjunction with [`catch_unwind`] to, for /// example, carry a panic across a layer of C code. /// -/// [`catch_unwind`]: ./fn.catch_unwind.html -/// /// # Notes /// /// Note that panics in Rust are not always implemented via unwinding, but they @@ -425,3 +169,154 @@ pub fn catch_unwind R + UnwindSafe, R>(f: F) -> Result { pub fn resume_unwind(payload: Box) -> ! { panicking::rust_panic_without_hook(payload) } + +/// Make all future panics abort directly without running the panic hook or unwinding. +/// +/// There is no way to undo this; the effect lasts until the process exits or +/// execs (or the equivalent). +/// +/// # Use after fork +/// +/// This function is particularly useful for calling after `libc::fork`. After `fork`, in a +/// multithreaded program it is (on many platforms) not safe to call the allocator. It is also +/// generally highly undesirable for an unwind to unwind past the `fork`, because that results in +/// the unwind propagating to code that was only ever expecting to run in the parent. +/// +/// `panic::always_abort()` helps avoid both of these. It directly avoids any further unwinding, +/// and if there is a panic, the abort will occur without allocating provided that the arguments to +/// panic can be formatted without allocating. +/// +/// Examples +/// +/// ```no_run +/// #![feature(panic_always_abort)] +/// use std::panic; +/// +/// panic::always_abort(); +/// +/// let _ = panic::catch_unwind(|| { +/// panic!("inside the catch"); +/// }); +/// +/// // We will have aborted already, due to the panic. +/// unreachable!(); +/// ``` +#[unstable(feature = "panic_always_abort", issue = "84438")] +pub fn always_abort() { + crate::panicking::panic_count::set_always_abort(); +} + +/// The configuration for whether and how the default panic hook will capture +/// and display the backtrace. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[unstable(feature = "panic_backtrace_config", issue = "93346")] +#[non_exhaustive] +pub enum BacktraceStyle { + /// Prints a terser backtrace which ideally only contains relevant + /// information. + Short, + /// Prints a backtrace with all possible information. + Full, + /// Disable collecting and displaying backtraces. + Off, +} + +impl BacktraceStyle { + pub(crate) fn full() -> Option { + if cfg!(feature = "backtrace") { Some(BacktraceStyle::Full) } else { None } + } + + fn as_usize(self) -> usize { + match self { + BacktraceStyle::Short => 1, + BacktraceStyle::Full => 2, + BacktraceStyle::Off => 3, + } + } + + fn from_usize(s: usize) -> Option { + Some(match s { + 0 => return None, + 1 => BacktraceStyle::Short, + 2 => BacktraceStyle::Full, + 3 => BacktraceStyle::Off, + _ => unreachable!(), + }) + } +} + +// Tracks whether we should/can capture a backtrace, and how we should display +// that backtrace. +// +// Internally stores equivalent of an Option. +static SHOULD_CAPTURE: AtomicUsize = AtomicUsize::new(0); + +/// Configure whether the default panic hook will capture and display a +/// backtrace. +/// +/// The default value for this setting may be set by the `RUST_BACKTRACE` +/// environment variable; see the details in [`get_backtrace_style`]. +#[unstable(feature = "panic_backtrace_config", issue = "93346")] +pub fn set_backtrace_style(style: BacktraceStyle) { + if !cfg!(feature = "backtrace") { + // If the `backtrace` feature of this crate isn't enabled, skip setting. + return; + } + SHOULD_CAPTURE.store(style.as_usize(), Ordering::Release); +} + +/// Checks whether the standard library's panic hook will capture and print a +/// backtrace. +/// +/// This function will, if a backtrace style has not been set via +/// [`set_backtrace_style`], read the environment variable `RUST_BACKTRACE` to +/// determine a default value for the backtrace formatting: +/// +/// The first call to `get_backtrace_style` may read the `RUST_BACKTRACE` +/// environment variable if `set_backtrace_style` has not been called to +/// override the default value. After a call to `set_backtrace_style` or +/// `get_backtrace_style`, any changes to `RUST_BACKTRACE` will have no effect. +/// +/// `RUST_BACKTRACE` is read according to these rules: +/// +/// * `0` for `BacktraceStyle::Off` +/// * `full` for `BacktraceStyle::Full` +/// * `1` for `BacktraceStyle::Short` +/// * Other values are currently `BacktraceStyle::Short`, but this may change in +/// the future +/// +/// Returns `None` if backtraces aren't currently supported. +#[unstable(feature = "panic_backtrace_config", issue = "93346")] +pub fn get_backtrace_style() -> Option { + if !cfg!(feature = "backtrace") { + // If the `backtrace` feature of this crate isn't enabled quickly return + // `Unsupported` so this can be constant propagated all over the place + // to optimize away callers. + return None; + } + if let Some(style) = BacktraceStyle::from_usize(SHOULD_CAPTURE.load(Ordering::Acquire)) { + return Some(style); + } + + let format = crate::env::var_os("RUST_BACKTRACE") + .map(|x| { + if &x == "0" { + BacktraceStyle::Off + } else if &x == "full" { + BacktraceStyle::Full + } else { + BacktraceStyle::Short + } + }) + .unwrap_or(if cfg!(target_os = "fuchsia") { + // Fuchsia components default to full backtrace. + BacktraceStyle::Full + } else { + BacktraceStyle::Off + }); + set_backtrace_style(format); + Some(format) +} + +#[cfg(test)] +mod tests; diff --git a/crux-mir/lib/std/src/panic/tests.rs b/crux-mir/lib/std/src/panic/tests.rs new file mode 100644 index 000000000..b37d74011 --- /dev/null +++ b/crux-mir/lib/std/src/panic/tests.rs @@ -0,0 +1,56 @@ +#![allow(dead_code)] + +use crate::cell::RefCell; +use crate::panic::{AssertUnwindSafe, UnwindSafe}; +use crate::rc::Rc; +use crate::sync::{Arc, Mutex, RwLock}; + +struct Foo { + a: i32, +} + +fn assert() {} + +#[test] +fn panic_safety_traits() { + assert::(); + assert::<&i32>(); + assert::<*mut i32>(); + assert::<*const i32>(); + assert::(); + assert::(); + assert::<&str>(); + assert::(); + assert::<&Foo>(); + assert::>(); + assert::(); + assert::>(); + assert::>(); + assert::>(); + assert::>(); + assert::<&Mutex>(); + assert::<&RwLock>(); + assert::>(); + assert::>(); + assert::>(); + + { + trait Trait: UnwindSafe {} + assert::>(); + } + + fn bar() { + assert::>(); + assert::>(); + } + + fn baz() { + assert::>(); + assert::>(); + assert::>(); + assert::>(); + assert::<&AssertUnwindSafe>(); + assert::>>(); + assert::>>(); + } +} diff --git a/crux-mir/lib/std/src/panicking.rs b/crux-mir/lib/std/src/panicking.rs index 10078bd4a..b0db3112e 100644 --- a/crux-mir/lib/std/src/panicking.rs +++ b/crux-mir/lib/std/src/panicking.rs @@ -7,6 +7,9 @@ //! * Executing a panic up to doing the actual implementation //! * Shims around "try" +#![deny(unsafe_op_in_unsafe_fn)] + +use crate::panic::BacktraceStyle; use core::panic::{BoxMeUp, Location, PanicInfo}; use crate::any::Any; @@ -15,18 +18,18 @@ use crate::intrinsics; use crate::mem::{self, ManuallyDrop}; use crate::process; use crate::sync::atomic::{AtomicBool, Ordering}; +use crate::sync::{PoisonError, RwLock}; use crate::sys::stdio::panic_output; -use crate::sys_common::backtrace::{self, RustBacktrace}; -use crate::sys_common::rwlock::RWLock; -use crate::sys_common::{thread_info, util}; +use crate::sys_common::backtrace; +use crate::sys_common::thread_info; use crate::thread; #[cfg(not(test))] -use crate::io::set_panic; +use crate::io::set_output_capture; // make sure to use the stderr output configured // by libtest in the real copy of std #[cfg(test)] -use realstd::io::set_panic; +use realstd::io::set_output_capture; // Binary interface to the panic runtime that the standard library depends on. // @@ -41,12 +44,14 @@ use realstd::io::set_panic; #[allow(improper_ctypes)] extern "C" { fn __rust_panic_cleanup(payload: *mut u8) -> *mut (dyn Any + Send + 'static); +} - /// `payload` is actually a `*mut &mut dyn BoxMeUp` but that would cause FFI warnings. - /// It cannot be `Box` because the other end of this call does not depend - /// on liballoc, and thus cannot use `Box`. - #[unwind(allowed)] - fn __rust_start_panic(payload: usize) -> u32; +#[allow(improper_ctypes)] +extern "Rust" { + /// `payload` is passed through another layer of raw pointers as `&mut dyn Trait` is not + /// FFI-safe. `BoxMeUp` lazily performs allocation only when needed (this avoids allocations + /// when using the "abort" panic runtime). + fn __rust_start_panic(payload: *mut &mut dyn BoxMeUp) -> u32; } /// This function is called by the panic runtime if FFI code catches a Rust @@ -58,14 +63,37 @@ extern "C" fn __rust_drop_panic() -> ! { rtabort!("Rust panics must be rethrown"); } -#[derive(Copy, Clone)] +/// This function is called by the panic runtime if it catches an exception +/// object which does not correspond to a Rust panic. +#[cfg(not(test))] +#[rustc_std_internal_symbol] +extern "C" fn __rust_foreign_exception() -> ! { + rtabort!("Rust cannot catch foreign exceptions"); +} + enum Hook { Default, - Custom(*mut (dyn Fn(&PanicInfo<'_>) + 'static + Sync + Send)), + Custom(Box) + 'static + Sync + Send>), } -static HOOK_LOCK: RWLock = RWLock::new(); -static mut HOOK: Hook = Hook::Default; +impl Hook { + #[inline] + fn into_box(self) -> Box) + 'static + Sync + Send> { + match self { + Hook::Default => Box::new(default_hook), + Hook::Custom(hook) => hook, + } + } +} + +impl Default for Hook { + #[inline] + fn default() -> Hook { + Hook::Default + } +} + +static HOOK: RwLock = RwLock::new(Hook::Default); /// Registers a custom panic hook, replacing any that was previously registered. /// @@ -106,19 +134,13 @@ pub fn set_hook(hook: Box) + 'static + Sync + Send>) { panic!("cannot modify the panic hook from a panicking thread"); } - unsafe { - HOOK_LOCK.write(); - let old_hook = HOOK; - HOOK = Hook::Custom(Box::into_raw(hook)); - HOOK_LOCK.write_unlock(); - - if let Hook::Custom(ptr) = old_hook { - #[allow(unused_must_use)] - { - Box::from_raw(ptr); - } - } - } + let new = Hook::Custom(hook); + let mut hook = HOOK.write().unwrap_or_else(PoisonError::into_inner); + let old = mem::replace(&mut *hook, new); + drop(hook); + // Only drop the old hook after releasing the lock to avoid deadlocking + // if its destructor panics. + drop(old); } /// Unregisters the current panic hook, returning it. @@ -148,32 +170,75 @@ pub fn set_hook(hook: Box) + 'static + Sync + Send>) { /// /// panic!("Normal panic"); /// ``` +#[must_use] #[stable(feature = "panic_hooks", since = "1.10.0")] pub fn take_hook() -> Box) + 'static + Sync + Send> { if thread::panicking() { panic!("cannot modify the panic hook from a panicking thread"); } - unsafe { - HOOK_LOCK.write(); - let hook = HOOK; - HOOK = Hook::Default; - HOOK_LOCK.write_unlock(); + let mut hook = HOOK.write().unwrap_or_else(PoisonError::into_inner); + let old_hook = mem::take(&mut *hook); + drop(hook); - match hook { - Hook::Default => Box::new(default_hook), - Hook::Custom(ptr) => Box::from_raw(ptr), - } + old_hook.into_box() +} + +/// Atomic combination of [`take_hook`] and [`set_hook`]. Use this to replace the panic handler with +/// a new panic handler that does something and then executes the old handler. +/// +/// [`take_hook`]: ./fn.take_hook.html +/// [`set_hook`]: ./fn.set_hook.html +/// +/// # Panics +/// +/// Panics if called from a panicking thread. +/// +/// # Examples +/// +/// The following will print the custom message, and then the normal output of panic. +/// +/// ```should_panic +/// #![feature(panic_update_hook)] +/// use std::panic; +/// +/// // Equivalent to +/// // let prev = panic::take_hook(); +/// // panic::set_hook(move |info| { +/// // println!("..."); +/// // prev(info); +/// // ); +/// panic::update_hook(move |prev, info| { +/// println!("Print custom message and execute panic handler as usual"); +/// prev(info); +/// }); +/// +/// panic!("Custom and then normal"); +/// ``` +#[unstable(feature = "panic_update_hook", issue = "92649")] +pub fn update_hook(hook_fn: F) +where + F: Fn(&(dyn Fn(&PanicInfo<'_>) + Send + Sync + 'static), &PanicInfo<'_>) + + Sync + + Send + + 'static, +{ + if thread::panicking() { + panic!("cannot modify the panic hook from a panicking thread"); } + + let mut hook = HOOK.write().unwrap_or_else(PoisonError::into_inner); + let prev = mem::take(&mut *hook).into_box(); + *hook = Hook::Custom(Box::new(move |info| hook_fn(&prev, info))); } fn default_hook(info: &PanicInfo<'_>) { // If this is a double panic, make sure that we print a backtrace // for this panic. Otherwise only print it if logging is enabled. - let backtrace_env = if update_panic_count(0) >= 2 { - RustBacktrace::Print(backtrace_rs::PrintFmt::Full) + let backtrace = if panic_count::get_count() >= 2 { + BacktraceStyle::full() } else { - backtrace::rust_backtrace_env() + crate::panic::get_backtrace_style() }; // The current implementation always returns `Some`. @@ -183,37 +248,40 @@ fn default_hook(info: &PanicInfo<'_>) { Some(s) => *s, None => match info.payload().downcast_ref::() { Some(s) => &s[..], - None => "Box", + None => "Box", }, }; let thread = thread_info::current_thread(); let name = thread.as_ref().and_then(|t| t.name()).unwrap_or(""); let write = |err: &mut dyn crate::io::Write| { - let _ = writeln!(err, "thread '{}' panicked at '{}', {}", name, msg, location); + let _ = writeln!(err, "thread '{name}' panicked at '{msg}', {location}"); static FIRST_PANIC: AtomicBool = AtomicBool::new(true); - match backtrace_env { - RustBacktrace::Print(format) => drop(backtrace::print(err, format)), - RustBacktrace::Disabled => {} - RustBacktrace::RuntimeDisabled => { + match backtrace { + Some(BacktraceStyle::Short) => { + drop(backtrace::print(err, crate::backtrace_rs::PrintFmt::Short)) + } + Some(BacktraceStyle::Full) => { + drop(backtrace::print(err, crate::backtrace_rs::PrintFmt::Full)) + } + Some(BacktraceStyle::Off) => { if FIRST_PANIC.swap(false, Ordering::SeqCst) { let _ = writeln!( err, - "note: run with `RUST_BACKTRACE=1` \ - environment variable to display a backtrace" + "note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace" ); } } + // If backtraces aren't supported, do nothing. + None => {} } }; - if let Some(mut local) = set_panic(None) { - // NB. In `cfg(test)` this uses the forwarding impl - // for `Box`. - write(&mut local); - set_panic(Some(local)); + if let Some(local) = set_output_capture(None) { + write(&mut *local.lock().unwrap_or_else(|e| e.into_inner())); + set_output_capture(Some(local)); } else if let Some(mut out) = panic_output() { write(&mut out); } @@ -222,19 +290,114 @@ fn default_hook(info: &PanicInfo<'_>) { #[cfg(not(test))] #[doc(hidden)] #[unstable(feature = "update_panic_count", issue = "none")] -pub fn update_panic_count(amt: isize) -> usize { +pub mod panic_count { use crate::cell::Cell; - thread_local! { static PANIC_COUNT: Cell = Cell::new(0) } + use crate::sync::atomic::{AtomicUsize, Ordering}; - PANIC_COUNT.with(|c| { - let next = (c.get() as isize + amt) as usize; - c.set(next); - next - }) + pub const ALWAYS_ABORT_FLAG: usize = 1 << (usize::BITS - 1); + + // Panic count for the current thread. + thread_local! { static LOCAL_PANIC_COUNT: Cell = const { Cell::new(0) } } + + // Sum of panic counts from all threads. The purpose of this is to have + // a fast path in `count_is_zero` (which is used by `panicking`). In any particular + // thread, if that thread currently views `GLOBAL_PANIC_COUNT` as being zero, + // then `LOCAL_PANIC_COUNT` in that thread is zero. This invariant holds before + // and after increase and decrease, but not necessarily during their execution. + // + // Additionally, the top bit of GLOBAL_PANIC_COUNT (GLOBAL_ALWAYS_ABORT_FLAG) + // records whether panic::always_abort() has been called. This can only be + // set, never cleared. + // panic::always_abort() is usually called to prevent memory allocations done by + // the panic handling in the child created by `libc::fork`. + // Memory allocations performed in a child created with `libc::fork` are undefined + // behavior in most operating systems. + // Accessing LOCAL_PANIC_COUNT in a child created by `libc::fork` would lead to a memory + // allocation. Only GLOBAL_PANIC_COUNT can be accessed in this situation. This is + // sufficient because a child process will always have exactly one thread only. + // See also #85261 for details. + // + // This could be viewed as a struct containing a single bit and an n-1-bit + // value, but if we wrote it like that it would be more than a single word, + // and even a newtype around usize would be clumsy because we need atomics. + // But we use such a tuple for the return type of increase(). + // + // Stealing a bit is fine because it just amounts to assuming that each + // panicking thread consumes at least 2 bytes of address space. + static GLOBAL_PANIC_COUNT: AtomicUsize = AtomicUsize::new(0); + + // Return the state of the ALWAYS_ABORT_FLAG and number of panics. + // + // If ALWAYS_ABORT_FLAG is not set, the number is determined on a per-thread + // base (stored in LOCAL_PANIC_COUNT), i.e. it is the amount of recursive calls + // of the calling thread. + // If ALWAYS_ABORT_FLAG is set, the number equals the *global* number of panic + // calls. See above why LOCAL_PANIC_COUNT is not used. + pub fn increase() -> (bool, usize) { + let global_count = GLOBAL_PANIC_COUNT.fetch_add(1, Ordering::Relaxed); + let must_abort = global_count & ALWAYS_ABORT_FLAG != 0; + let panics = if must_abort { + global_count & !ALWAYS_ABORT_FLAG + } else { + LOCAL_PANIC_COUNT.with(|c| { + let next = c.get() + 1; + c.set(next); + next + }) + }; + (must_abort, panics) + } + + pub fn decrease() { + GLOBAL_PANIC_COUNT.fetch_sub(1, Ordering::Relaxed); + LOCAL_PANIC_COUNT.with(|c| { + let next = c.get() - 1; + c.set(next); + next + }); + } + + pub fn set_always_abort() { + GLOBAL_PANIC_COUNT.fetch_or(ALWAYS_ABORT_FLAG, Ordering::Relaxed); + } + + // Disregards ALWAYS_ABORT_FLAG + #[must_use] + pub fn get_count() -> usize { + LOCAL_PANIC_COUNT.with(|c| c.get()) + } + + // Disregards ALWAYS_ABORT_FLAG + #[must_use] + #[inline] + pub fn count_is_zero() -> bool { + if GLOBAL_PANIC_COUNT.load(Ordering::Relaxed) & !ALWAYS_ABORT_FLAG == 0 { + // Fast path: if `GLOBAL_PANIC_COUNT` is zero, all threads + // (including the current one) will have `LOCAL_PANIC_COUNT` + // equal to zero, so TLS access can be avoided. + // + // In terms of performance, a relaxed atomic load is similar to a normal + // aligned memory read (e.g., a mov instruction in x86), but with some + // compiler optimization restrictions. On the other hand, a TLS access + // might require calling a non-inlinable function (such as `__tls_get_addr` + // when using the GD TLS model). + true + } else { + is_zero_slow_path() + } + } + + // Slow path is in a separate function to reduce the amount of code + // inlined from `count_is_zero`. + #[inline(never)] + #[cold] + fn is_zero_slow_path() -> bool { + LOCAL_PANIC_COUNT.with(|c| c.get() == 0) + } } #[cfg(test)] -pub use realstd::rt::update_panic_count; +pub use realstd::rt::panic_count; /// Invoke a closure, capturing the cause of an unwinding panic if one occurs. pub unsafe fn r#try R>(f: F) -> Result> { @@ -271,42 +434,21 @@ pub unsafe fn r#try R>(f: F) -> Result> let mut data = Data { f: ManuallyDrop::new(f) }; let data_ptr = &mut data as *mut _ as *mut u8; - return if do_try(do_call::, data_ptr, do_catch::) == 0 { - Ok(ManuallyDrop::into_inner(data.r)) - } else { - Err(ManuallyDrop::into_inner(data.p)) - }; - - // Compatibility wrapper around the try intrinsic for bootstrap. + // SAFETY: // - // We also need to mark it #[inline(never)] to work around a bug on MinGW - // targets: the unwinding implementation was relying on UB, but this only - // becomes a problem in practice if inlining is involved. - #[cfg(not(bootstrap))] - use intrinsics::r#try as do_try; - #[cfg(bootstrap)] - #[inline(never)] - unsafe fn do_try(try_fn: fn(*mut u8), data: *mut u8, catch_fn: fn(*mut u8, *mut u8)) -> i32 { - use crate::mem::MaybeUninit; - #[cfg(target_env = "msvc")] - type TryPayload = [u64; 2]; - #[cfg(not(target_env = "msvc"))] - type TryPayload = *mut u8; - - let mut payload: MaybeUninit = MaybeUninit::uninit(); - let payload_ptr = payload.as_mut_ptr() as *mut u8; - let r = intrinsics::r#try(try_fn, data, payload_ptr); - if r != 0 { - #[cfg(target_env = "msvc")] - { - catch_fn(data, payload_ptr) - } - #[cfg(not(target_env = "msvc"))] - { - catch_fn(data, payload.assume_init()) - } - } - r + // Access to the union's fields: this is `std` and we know that the `r#try` + // intrinsic fills in the `r` or `p` union field based on its return value. + // + // The call to `intrinsics::r#try` is made safe by: + // - `do_call`, the first argument, can be called with the initial `data_ptr`. + // - `do_catch`, the second argument, can be called with the `data_ptr` as well. + // See their safety preconditions for more information + unsafe { + return if intrinsics::r#try(do_call::, data_ptr, do_catch::) == 0 { + Ok(ManuallyDrop::into_inner(data.r)) + } else { + Err(ManuallyDrop::into_inner(data.p)) + }; } // We consider unwinding to be rare, so mark this function as cold. However, @@ -315,15 +457,25 @@ pub unsafe fn r#try R>(f: F) -> Result> // non-cold function, though, as of the writing of this comment). #[cold] unsafe fn cleanup(payload: *mut u8) -> Box { - let obj = Box::from_raw(__rust_panic_cleanup(payload)); - update_panic_count(-1); + // SAFETY: The whole unsafe block hinges on a correct implementation of + // the panic handler `__rust_panic_cleanup`. As such we can only + // assume it returns the correct thing for `Box::from_raw` to work + // without undefined behavior. + let obj = unsafe { Box::from_raw(__rust_panic_cleanup(payload)) }; + panic_count::decrease(); obj } - // See comment on do_try above for why #[inline(never)] is needed on bootstrap. - #[cfg_attr(bootstrap, inline(never))] - #[cfg_attr(not(bootstrap), inline)] + // SAFETY: + // data must be non-NUL, correctly aligned, and a pointer to a `Data` + // Its must contains a valid `f` (type: F) value that can be use to fill + // `data.r`. + // + // This function cannot be marked as `unsafe` because `intrinsics::r#try` + // expects normal function pointers. + #[inline] fn do_call R, R>(data: *mut u8) { + // SAFETY: this is the responsibility of the caller, see above. unsafe { let data = data as *mut Data; let data = &mut (*data); @@ -335,8 +487,21 @@ pub unsafe fn r#try R>(f: F) -> Result> // We *do* want this part of the catch to be inlined: this allows the // compiler to properly track accesses to the Data union and optimize it // away most of the time. + // + // SAFETY: + // data must be non-NUL, correctly aligned, and a pointer to a `Data` + // Since this uses `cleanup` it also hinges on a correct implementation of + // `__rustc_panic_cleanup`. + // + // This function cannot be marked as `unsafe` because `intrinsics::r#try` + // expects normal function pointers. #[inline] fn do_catch R, R>(data: *mut u8, payload: *mut u8) { + // SAFETY: this is the responsibility of the caller, see above. + // + // When `__rustc_panic_cleaner` is correctly implemented we can rely + // on `obj` being the correct thing to pass to `data.p` (after wrapping + // in `ManuallyDrop`). unsafe { let data = data as *mut Data; let data = &mut (*data); @@ -347,35 +512,14 @@ pub unsafe fn r#try R>(f: F) -> Result> } /// Determines whether the current thread is unwinding because of panic. +#[inline] pub fn panicking() -> bool { - update_panic_count(0) != 0 -} - -/// The entry point for panicking with a formatted message. -/// -/// This is designed to reduce the amount of code required at the call -/// site as much as possible (so that `panic!()` has as low an impact -/// on (e.g.) the inlining of other functions as possible), by moving -/// the actual formatting into this shared place. -#[unstable(feature = "libstd_sys_internals", reason = "used by the panic! macro", issue = "none")] -#[cold] -// If panic_immediate_abort, inline the abort call, -// otherwise avoid inlining because of it is cold path. -#[cfg_attr(not(feature = "panic_immediate_abort"), track_caller)] -#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never))] -#[cfg_attr(feature = "panic_immediate_abort", inline)] -pub fn begin_panic_fmt(msg: &fmt::Arguments<'_>) -> ! { - if cfg!(feature = "panic_immediate_abort") { - unsafe { intrinsics::abort() } - } - - let info = PanicInfo::internal_constructor(Some(msg), Location::caller()); - begin_panic_handler(&info) + !panic_count::count_is_zero() } -/// Entry point of panics from the libcore crate (`panic_impl` lang item). -#[cfg_attr(not(test), panic_handler)] -#[unwind(allowed)] +/// Entry point of panics from the core crate (`panic_impl` lang item). +#[cfg(not(test))] +#[panic_handler] pub fn begin_panic_handler(info: &PanicInfo<'_>) -> ! { struct PanicPayload<'a> { inner: &'a fmt::Arguments<'a>, @@ -414,9 +558,32 @@ pub fn begin_panic_handler(info: &PanicInfo<'_>) -> ! { } } + struct StrPanicPayload(&'static str); + + unsafe impl BoxMeUp for StrPanicPayload { + fn take_box(&mut self) -> *mut (dyn Any + Send) { + Box::into_raw(Box::new(self.0)) + } + + fn get(&mut self) -> &(dyn Any + Send) { + &self.0 + } + } + let loc = info.location().unwrap(); // The current implementation always returns Some let msg = info.message().unwrap(); // The current implementation always returns Some - rust_panic_with_hook(&mut PanicPayload::new(msg), info.message(), loc); + crate::sys_common::backtrace::__rust_end_short_backtrace(move || { + if let Some(msg) = msg.as_str() { + rust_panic_with_hook(&mut StrPanicPayload(msg), info.message(), loc, info.can_unwind()); + } else { + rust_panic_with_hook( + &mut PanicPayload::new(msg), + info.message(), + loc, + info.can_unwind(), + ); + } + }) } /// This is the entry point of panicking for the non-format-string variants of @@ -427,15 +594,19 @@ pub fn begin_panic_handler(info: &PanicInfo<'_>) -> ! { // lang item for CTFE panic support // never inline unless panic_immediate_abort to avoid code // bloat at the call sites as much as possible -#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never))] -#[cold] +#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never), cold)] +#[cfg_attr(feature = "panic_immediate_abort", inline)] #[track_caller] -pub fn begin_panic(msg: M) -> ! { +#[rustc_do_not_const_check] // hooked by const-eval +pub const fn begin_panic(msg: M) -> ! { if cfg!(feature = "panic_immediate_abort") { - unsafe { intrinsics::abort() } + intrinsics::abort() } - rust_panic_with_hook(&mut PanicPayload::new(msg), None, Location::caller()); + let loc = Location::caller(); + return crate::sys_common::backtrace::__rust_end_short_backtrace(move || { + rust_panic_with_hook(&mut PanicPayload::new(msg), None, loc, true) + }); struct PanicPayload { inner: Option, @@ -479,55 +650,61 @@ fn rust_panic_with_hook( payload: &mut dyn BoxMeUp, message: Option<&fmt::Arguments<'_>>, location: &Location<'_>, + can_unwind: bool, ) -> ! { - let panics = update_panic_count(1); + let (must_abort, panics) = panic_count::increase(); // If this is the third nested call (e.g., panics == 2, this is 0-indexed), // the panic hook probably triggered the last panic, otherwise the // double-panic check would have aborted the process. In this case abort the // process real quickly as we don't want to try calling it again as it'll // probably just panic again. - if panics > 2 { - util::dumb_print(format_args!( - "thread panicked while processing \ - panic. aborting.\n" - )); - unsafe { intrinsics::abort() } + if must_abort || panics > 2 { + if panics > 2 { + // Don't try to print the message in this case + // - perhaps that is causing the recursive panics. + rtprintpanic!("thread panicked while processing panic. aborting.\n"); + } else { + // Unfortunately, this does not print a backtrace, because creating + // a `Backtrace` will allocate, which we must to avoid here. + let panicinfo = PanicInfo::internal_constructor(message, location, can_unwind); + rtprintpanic!("{panicinfo}\npanicked after panic::always_abort(), aborting.\n"); + } + crate::sys::abort_internal(); } - unsafe { - let mut info = PanicInfo::internal_constructor(message, location); - HOOK_LOCK.read(); - match HOOK { - // Some platforms (like wasm) know that printing to stderr won't ever actually - // print anything, and if that's the case we can skip the default - // hook. Since string formatting happens lazily when calling `payload` - // methods, this means we avoid formatting the string at all! - // (The panic runtime might still call `payload.take_box()` though and trigger - // formatting.) - Hook::Default if panic_output().is_none() => {} - Hook::Default => { - info.set_payload(payload.get()); - default_hook(&info); - } - Hook::Custom(ptr) => { - info.set_payload(payload.get()); - (*ptr)(&info); - } - }; - HOOK_LOCK.read_unlock(); - } + let mut info = PanicInfo::internal_constructor(message, location, can_unwind); + let hook = HOOK.read().unwrap_or_else(PoisonError::into_inner); + match *hook { + // Some platforms (like wasm) know that printing to stderr won't ever actually + // print anything, and if that's the case we can skip the default + // hook. Since string formatting happens lazily when calling `payload` + // methods, this means we avoid formatting the string at all! + // (The panic runtime might still call `payload.take_box()` though and trigger + // formatting.) + Hook::Default if panic_output().is_none() => {} + Hook::Default => { + info.set_payload(payload.get()); + default_hook(&info); + } + Hook::Custom(ref hook) => { + info.set_payload(payload.get()); + hook(&info); + } + }; + drop(hook); - if panics > 1 { + if panics > 1 || !can_unwind { // If a thread panics while it's already unwinding then we // have limited options. Currently our preference is to // just abort. In the future we may consider resuming // unwinding or otherwise exiting the thread cleanly. - util::dumb_print(format_args!( - "thread panicked while panicking. \ - aborting.\n" - )); - unsafe { intrinsics::abort() } + if !can_unwind { + rtprintpanic!("thread caused non-unwinding panic. aborting.\n"); + } else { + rtprintpanic!("thread panicked while panicking. aborting.\n"); + } + crate::sys::abort_internal(); } rust_panic(payload) @@ -536,7 +713,7 @@ fn rust_panic_with_hook( /// This is the entry point for `resume_unwind`. /// It just forwards the payload to the panic runtime. pub fn rust_panic_without_hook(payload: Box) -> ! { - update_panic_count(1); + panic_count::increase(); struct RewrapBox(Box); @@ -560,7 +737,7 @@ pub fn rust_panic_without_hook(payload: Box) -> ! { fn rust_panic(mut msg: &mut dyn BoxMeUp) -> ! { let code = unsafe { let obj = &mut msg as *mut &mut dyn BoxMeUp; - __rust_start_panic(obj as usize) + __rust_start_panic(obj) }; - rtabort!("failed to initiate panic, error {}", code) + rtabort!("failed to initiate panic, error {code}") } diff --git a/crux-mir/lib/std/src/path.rs b/crux-mir/lib/std/src/path.rs index b8361d3e8..2f53cf839 100644 --- a/crux-mir/lib/std/src/path.rs +++ b/crux-mir/lib/std/src/path.rs @@ -1,5 +1,3 @@ -// ignore-tidy-filelength - //! Cross-platform path manipulation. //! //! This module provides two types, [`PathBuf`] and [`Path`] (akin to [`String`] @@ -14,6 +12,13 @@ //! [`PathBuf`]; note that the paths may differ syntactically by the //! normalization described in the documentation for the [`components`] method. //! +//! ## Case sensitivity +//! +//! Unless otherwise indicated path methods that do not access the filesystem, +//! such as [`Path::starts_with`] and [`Path::ends_with`], are case sensitive no +//! matter the platform or filesystem. An exception to this is made for Windows +//! drive letters. +//! //! ## Simple usage //! //! Path manipulation includes both parsing components from slices and building @@ -56,21 +61,18 @@ //! let path: PathBuf = ["c:\\", "windows", "system32.dll"].iter().collect(); //! ``` //! -//! [`Component`]: ../../std/path/enum.Component.html -//! [`components`]: ../../std/path/struct.Path.html#method.components -//! [`PathBuf`]: ../../std/path/struct.PathBuf.html -//! [`Path`]: ../../std/path/struct.Path.html -//! [`push`]: ../../std/path/struct.PathBuf.html#method.push -//! [`String`]: ../../std/string/struct.String.html -//! -//! [`str`]: ../../std/primitive.str.html -//! [`OsString`]: ../../std/ffi/struct.OsString.html -//! [`OsStr`]: ../../std/ffi/struct.OsStr.html +//! [`components`]: Path::components +//! [`push`]: PathBuf::push #![stable(feature = "rust1", since = "1.0.0")] +#![deny(unsafe_op_in_unsafe_fn)] + +#[cfg(test)] +mod tests; use crate::borrow::{Borrow, Cow}; use crate::cmp; +use crate::collections::TryReserveError; use crate::error::Error; use crate::fmt; use crate::fs; @@ -83,7 +85,7 @@ use crate::str::FromStr; use crate::sync::Arc; use crate::ffi::{OsStr, OsString}; - +use crate::sys; use crate::sys::path::{is_sep_byte, is_verbatim_sep, parse_prefix, MAIN_SEP_STR}; //////////////////////////////////////////////////////////////////////////////// @@ -157,17 +159,17 @@ pub enum Prefix<'a> { #[stable(feature = "rust1", since = "1.0.0")] &'a OsStr, ), - /// Verbatim disk prefix, e.g., `\\?\C:\`. + /// Verbatim disk prefix, e.g., `\\?\C:`. /// /// Verbatim disk prefixes consist of `\\?\` immediately followed by the - /// drive letter and `:\`. + /// drive letter and `:`. #[stable(feature = "rust1", since = "1.0.0")] VerbatimDisk(#[stable(feature = "rust1", since = "1.0.0")] u8), /// Device namespace prefix, e.g., `\\.\COM42`. /// - /// Device namespace prefixes consist of `\\.\` immediately followed by the - /// device name. + /// Device namespace prefixes consist of `\\.\` (possibly using `/` + /// instead of `\`), immediately followed by the device name. #[stable(feature = "rust1", since = "1.0.0")] DeviceNS(#[stable(feature = "rust1", since = "1.0.0")] &'a OsStr), @@ -191,7 +193,7 @@ impl<'a> Prefix<'a> { fn len(&self) -> usize { use self::Prefix::*; fn os_str_len(s: &OsStr) -> usize { - os_str_as_u8_slice(s).len() + s.bytes().len() } match *self { Verbatim(x) => 4 + os_str_len(x), @@ -221,6 +223,7 @@ impl<'a> Prefix<'a> { /// assert!(!Disk(b'C').is_verbatim()); /// ``` #[inline] + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn is_verbatim(&self) -> bool { use self::Prefix::*; @@ -253,6 +256,7 @@ impl<'a> Prefix<'a> { /// assert!(path::is_separator('/')); // '/' works for both Unix and Windows /// assert!(!path::is_separator('❤')); /// ``` +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn is_separator(c: char) -> bool { c.is_ascii() && is_sep_byte(c as u8) @@ -264,6 +268,12 @@ pub fn is_separator(c: char) -> bool { #[stable(feature = "rust1", since = "1.0.0")] pub const MAIN_SEPARATOR: char = crate::sys::path::MAIN_SEP; +/// The primary separator of path components for the current platform. +/// +/// For example, `/` on Unix and `\` on Windows. +#[stable(feature = "main_separator_str", since = "CURRENT_RUSTC_VERSION")] +pub const MAIN_SEPARATOR_STR: &str = crate::sys::path::MAIN_SEP_STR; + //////////////////////////////////////////////////////////////////////////////// // Misc helpers //////////////////////////////////////////////////////////////////////////////// @@ -289,19 +299,18 @@ where } } -// See note at the top of this module to understand why these are used: -// -// These casts are safe as OsStr is internally a wrapper around [u8] on all -// platforms. -// -// Note that currently this relies on the special knowledge that libstd has; -// these types are single-element structs but are not marked repr(transparent) -// or repr(C) which would make these casts allowable outside std. -fn os_str_as_u8_slice(s: &OsStr) -> &[u8] { - unsafe { &*(s as *const OsStr as *const [u8]) } -} unsafe fn u8_slice_as_os_str(s: &[u8]) -> &OsStr { - &*(s as *const [u8] as *const OsStr) + // SAFETY: See note at the top of this module to understand why this and + // `OsStr::bytes` are used: + // + // This casts are safe as OsStr is internally a wrapper around [u8] on all + // platforms. + // + // Note that currently this relies on the special knowledge that std has; + // these types are single-element structs but are not marked + // repr(transparent) or repr(C) which would make these casts not allowable + // outside std. + unsafe { &*(s as *const [u8] as *const OsStr) } } // Detect scheme on Redox @@ -320,26 +329,42 @@ fn has_physical_root(s: &[u8], prefix: Option>) -> bool { } // basic workhorse for splitting stem and extension -fn split_file_at_dot(file: &OsStr) -> (Option<&OsStr>, Option<&OsStr>) { - unsafe { - if os_str_as_u8_slice(file) == b".." { - return (Some(file), None); - } - - // The unsafety here stems from converting between &OsStr and &[u8] - // and back. This is safe to do because (1) we only look at ASCII - // contents of the encoding and (2) new &OsStr values are produced - // only from ASCII-bounded slices of existing &OsStr values. - - let mut iter = os_str_as_u8_slice(file).rsplitn(2, |b| *b == b'.'); - let after = iter.next(); - let before = iter.next(); - if before == Some(b"") { - (Some(file), None) - } else { - (before.map(|s| u8_slice_as_os_str(s)), after.map(|s| u8_slice_as_os_str(s))) - } - } +fn rsplit_file_at_dot(file: &OsStr) -> (Option<&OsStr>, Option<&OsStr>) { + if file.bytes() == b".." { + return (Some(file), None); + } + + // The unsafety here stems from converting between &OsStr and &[u8] + // and back. This is safe to do because (1) we only look at ASCII + // contents of the encoding and (2) new &OsStr values are produced + // only from ASCII-bounded slices of existing &OsStr values. + let mut iter = file.bytes().rsplitn(2, |b| *b == b'.'); + let after = iter.next(); + let before = iter.next(); + if before == Some(b"") { + (Some(file), None) + } else { + unsafe { (before.map(|s| u8_slice_as_os_str(s)), after.map(|s| u8_slice_as_os_str(s))) } + } +} + +fn split_file_at_dot(file: &OsStr) -> (&OsStr, Option<&OsStr>) { + let slice = file.bytes(); + if slice == b".." { + return (file, None); + } + + // The unsafety here stems from converting between &OsStr and &[u8] + // and back. This is safe to do because (1) we only look at ASCII + // contents of the encoding and (2) new &OsStr values are produced + // only from ASCII-bounded slices of existing &OsStr values. + let i = match slice[1..].iter().position(|b| *b == b'.') { + Some(i) => i + 1, + None => return (file, None), + }; + let before = &slice[..i]; + let after = &slice[i + 1..]; + unsafe { (u8_slice_as_os_str(before), Some(u8_slice_as_os_str(after))) } } //////////////////////////////////////////////////////////////////////////////// @@ -390,12 +415,9 @@ enum State { /// # } /// ``` /// -/// [`as_os_str`]: #method.as_os_str -/// [`Component`]: enum.Component.html -/// [`kind`]: #method.kind -/// [`OsStr`]: ../../std/ffi/struct.OsStr.html -/// [`Prefix` variant]: enum.Component.html#variant.Prefix -/// [`Prefix`]: enum.Prefix.html +/// [`as_os_str`]: PrefixComponent::as_os_str +/// [`kind`]: PrefixComponent::kind +/// [`Prefix` variant]: Component::Prefix #[stable(feature = "rust1", since = "1.0.0")] #[derive(Copy, Clone, Eq, Debug)] pub struct PrefixComponent<'a> { @@ -411,17 +433,17 @@ impl<'a> PrefixComponent<'a> { /// /// See [`Prefix`]'s documentation for more information on the different /// kinds of prefixes. - /// - /// [`Prefix`]: enum.Prefix.html #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + #[inline] pub fn kind(&self) -> Prefix<'a> { self.parsed } /// Returns the raw [`OsStr`] slice for this prefix. - /// - /// [`OsStr`]: ../../std/ffi/struct.OsStr.html #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + #[inline] pub fn as_os_str(&self) -> &'a OsStr { self.raw } @@ -429,6 +451,7 @@ impl<'a> PrefixComponent<'a> { #[stable(feature = "rust1", since = "1.0.0")] impl<'a> cmp::PartialEq for PrefixComponent<'a> { + #[inline] fn eq(&self, other: &PrefixComponent<'a>) -> bool { cmp::PartialEq::eq(&self.parsed, &other.parsed) } @@ -436,6 +459,7 @@ impl<'a> cmp::PartialEq for PrefixComponent<'a> { #[stable(feature = "rust1", since = "1.0.0")] impl<'a> cmp::PartialOrd for PrefixComponent<'a> { + #[inline] fn partial_cmp(&self, other: &PrefixComponent<'a>) -> Option { cmp::PartialOrd::partial_cmp(&self.parsed, &other.parsed) } @@ -443,6 +467,7 @@ impl<'a> cmp::PartialOrd for PrefixComponent<'a> { #[stable(feature = "rust1", since = "1.0.0")] impl cmp::Ord for PrefixComponent<'_> { + #[inline] fn cmp(&self, other: &Self) -> cmp::Ordering { cmp::Ord::cmp(&self.parsed, &other.parsed) } @@ -461,7 +486,7 @@ impl Hash for PrefixComponent<'_> { /// (`/` or `\`). /// /// This `enum` is created by iterating over [`Components`], which in turn is -/// created by the [`components`][`Path::components`] method on [`Path`]. +/// created by the [`components`](Path::components) method on [`Path`]. /// /// # Examples /// @@ -477,10 +502,6 @@ impl Hash for PrefixComponent<'_> { /// Component::Normal("bar.txt".as_ref()), /// ]); /// ``` -/// -/// [`Components`]: struct.Components.html -/// [`Path`]: struct.Path.html -/// [`Path::components`]: struct.Path.html#method.components #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] #[stable(feature = "rust1", since = "1.0.0")] pub enum Component<'a> { @@ -490,8 +511,6 @@ pub enum Component<'a> { /// for more. /// /// Does not occur on Unix. - /// - /// [`Prefix`]: enum.Prefix.html #[stable(feature = "rust1", since = "1.0.0")] Prefix(#[stable(feature = "rust1", since = "1.0.0")] PrefixComponent<'a>), @@ -529,8 +548,7 @@ impl<'a> Component<'a> { /// let components: Vec<_> = path.components().map(|comp| comp.as_os_str()).collect(); /// assert_eq!(&components, &[".", "tmp", "foo", "bar.txt"]); /// ``` - /// - /// [`OsStr`]: ../../std/ffi/struct.OsStr.html + #[must_use = "`self` will be dropped if the result is not used"] #[stable(feature = "rust1", since = "1.0.0")] pub fn as_os_str(self) -> &'a OsStr { match self { @@ -545,6 +563,7 @@ impl<'a> Component<'a> { #[stable(feature = "rust1", since = "1.0.0")] impl AsRef for Component<'_> { + #[inline] fn as_ref(&self) -> &OsStr { self.as_os_str() } @@ -552,6 +571,7 @@ impl AsRef for Component<'_> { #[stable(feature = "path_component_asref", since = "1.25.0")] impl AsRef for Component<'_> { + #[inline] fn as_ref(&self) -> &Path { self.as_os_str().as_ref() } @@ -570,14 +590,13 @@ impl AsRef for Component<'_> { /// let path = Path::new("/tmp/foo/bar.txt"); /// /// for component in path.components() { -/// println!("{:?}", component); +/// println!("{component:?}"); /// } /// ``` /// -/// [`Component`]: enum.Component.html -/// [`components`]: struct.Path.html#method.components -/// [`Path`]: struct.Path.html +/// [`components`]: Path::components #[derive(Clone)] +#[must_use = "iterators are lazy and do nothing unless consumed"] #[stable(feature = "rust1", since = "1.0.0")] pub struct Components<'a> { // The path left to parse components from @@ -587,8 +606,8 @@ pub struct Components<'a> { prefix: Option>, // true if path *physically* has a root separator; for most Windows - // prefixes, it may have a "logical" rootseparator for the purposes of - // normalization, e.g., \\server\share == \\server\share\. + // prefixes, it may have a "logical" root separator for the purposes of + // normalization, e.g., \\server\share == \\server\share\. has_physical_root: bool, // The iterator is double-ended, and these two states keep track of what has @@ -602,11 +621,9 @@ pub struct Components<'a> { /// This `struct` is created by the [`iter`] method on [`Path`]. /// See its documentation for more. /// -/// [`Component`]: enum.Component.html -/// [`iter`]: struct.Path.html#method.iter -/// [`OsStr`]: ../../std/ffi/struct.OsStr.html -/// [`Path`]: struct.Path.html +/// [`iter`]: Path::iter #[derive(Clone)] +#[must_use = "iterators are lazy and do nothing unless consumed"] #[stable(feature = "rust1", since = "1.0.0")] pub struct Iter<'a> { inner: Components<'a>, @@ -677,6 +694,7 @@ impl<'a> Components<'a> { /// /// assert_eq!(Path::new("foo/bar.txt"), components.as_path()); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn as_path(&self) -> &'a Path { let mut comps = self.clone(); @@ -707,7 +725,7 @@ impl<'a> Components<'a> { if self.has_root() { return false; } - let mut iter = self.path[self.prefix_len()..].iter(); + let mut iter = self.path[self.prefix_remaining()..].iter(); match (iter.next(), iter.next()) { (Some(&b'.'), None) => true, (Some(&b'.'), Some(&b)) => self.is_sep_byte(b), @@ -778,6 +796,7 @@ impl<'a> Components<'a> { #[stable(feature = "rust1", since = "1.0.0")] impl AsRef for Components<'_> { + #[inline] fn as_ref(&self) -> &Path { self.as_path() } @@ -785,6 +804,7 @@ impl AsRef for Components<'_> { #[stable(feature = "rust1", since = "1.0.0")] impl AsRef for Components<'_> { + #[inline] fn as_ref(&self) -> &OsStr { self.as_path().as_os_str() } @@ -820,6 +840,8 @@ impl<'a> Iter<'a> { /// assert_eq!(Path::new("foo/bar.txt"), iter.as_path()); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + #[inline] pub fn as_path(&self) -> &'a Path { self.inner.as_path() } @@ -827,6 +849,7 @@ impl<'a> Iter<'a> { #[stable(feature = "rust1", since = "1.0.0")] impl AsRef for Iter<'_> { + #[inline] fn as_ref(&self) -> &Path { self.as_path() } @@ -834,6 +857,7 @@ impl AsRef for Iter<'_> { #[stable(feature = "rust1", since = "1.0.0")] impl AsRef for Iter<'_> { + #[inline] fn as_ref(&self) -> &OsStr { self.as_path().as_os_str() } @@ -843,6 +867,7 @@ impl AsRef for Iter<'_> { impl<'a> Iterator for Iter<'a> { type Item = &'a OsStr; + #[inline] fn next(&mut self) -> Option<&'a OsStr> { self.inner.next().map(Component::as_os_str) } @@ -850,6 +875,7 @@ impl<'a> Iterator for Iter<'a> { #[stable(feature = "rust1", since = "1.0.0")] impl<'a> DoubleEndedIterator for Iter<'a> { + #[inline] fn next_back(&mut self) -> Option<&'a OsStr> { self.inner.next_back().map(Component::as_os_str) } @@ -963,8 +989,28 @@ impl FusedIterator for Components<'_> {} #[stable(feature = "rust1", since = "1.0.0")] impl<'a> cmp::PartialEq for Components<'a> { + #[inline] fn eq(&self, other: &Components<'a>) -> bool { - Iterator::eq(self.clone(), other.clone()) + let Components { path: _, front: _, back: _, has_physical_root: _, prefix: _ } = self; + + // Fast path for exact matches, e.g. for hashmap lookups. + // Don't explicitly compare the prefix or has_physical_root fields since they'll + // either be covered by the `path` buffer or are only relevant for `prefix_verbatim()`. + if self.path.len() == other.path.len() + && self.front == other.front + && self.back == State::Body + && other.back == State::Body + && self.prefix_verbatim() == other.prefix_verbatim() + { + // possible future improvement: this could bail out earlier if there were a + // reverse memcmp/bcmp comparing back to front + if self.path == other.path { + return true; + } + } + + // compare back to front since absolute paths often share long prefixes + Iterator::eq(self.clone().rev(), other.clone().rev()) } } @@ -973,16 +1019,50 @@ impl cmp::Eq for Components<'_> {} #[stable(feature = "rust1", since = "1.0.0")] impl<'a> cmp::PartialOrd for Components<'a> { + #[inline] fn partial_cmp(&self, other: &Components<'a>) -> Option { - Iterator::partial_cmp(self.clone(), other.clone()) + Some(compare_components(self.clone(), other.clone())) } } #[stable(feature = "rust1", since = "1.0.0")] impl cmp::Ord for Components<'_> { + #[inline] fn cmp(&self, other: &Self) -> cmp::Ordering { - Iterator::cmp(self.clone(), other.clone()) + compare_components(self.clone(), other.clone()) + } +} + +fn compare_components(mut left: Components<'_>, mut right: Components<'_>) -> cmp::Ordering { + // Fast path for long shared prefixes + // + // - compare raw bytes to find first mismatch + // - backtrack to find separator before mismatch to avoid ambiguous parsings of '.' or '..' characters + // - if found update state to only do a component-wise comparison on the remainder, + // otherwise do it on the full path + // + // The fast path isn't taken for paths with a PrefixComponent to avoid backtracking into + // the middle of one + if left.prefix.is_none() && right.prefix.is_none() && left.front == right.front { + // possible future improvement: a [u8]::first_mismatch simd implementation + let first_difference = match left.path.iter().zip(right.path).position(|(&a, &b)| a != b) { + None if left.path.len() == right.path.len() => return cmp::Ordering::Equal, + None => left.path.len().min(right.path.len()), + Some(diff) => diff, + }; + + if let Some(previous_sep) = + left.path[..first_difference].iter().rposition(|&b| left.is_sep_byte(b)) + { + let mismatched_component_start = previous_sep + 1; + left.path = &left.path[mismatched_component_start..]; + left.front = State::Body; + right.path = &right.path[mismatched_component_start..]; + right.front = State::Body; + } } + + Iterator::cmp(left, right) } /// An iterator over [`Path`] and its ancestors. @@ -1002,9 +1082,9 @@ impl cmp::Ord for Components<'_> { /// } /// ``` /// -/// [`ancestors`]: struct.Path.html#method.ancestors -/// [`Path`]: struct.Path.html +/// [`ancestors`]: Path::ancestors #[derive(Copy, Clone, Debug)] +#[must_use = "iterators are lazy and do nothing unless consumed"] #[stable(feature = "path_ancestors", since = "1.28.0")] pub struct Ancestors<'a> { next: Option<&'a Path>, @@ -1014,6 +1094,7 @@ pub struct Ancestors<'a> { impl<'a> Iterator for Ancestors<'a> { type Item = &'a Path; + #[inline] fn next(&mut self) -> Option { let next = self.next; self.next = next.and_then(Path::parent); @@ -1034,14 +1115,11 @@ impl FusedIterator for Ancestors<'_> {} /// the path in place. It also implements [`Deref`] to [`Path`], meaning that /// all methods on [`Path`] slices are available on `PathBuf` values as well. /// -/// [`String`]: ../string/struct.String.html -/// [`Path`]: struct.Path.html -/// [`push`]: struct.PathBuf.html#method.push -/// [`set_extension`]: struct.PathBuf.html#method.set_extension -/// [`Deref`]: ../ops/trait.Deref.html +/// [`push`]: PathBuf::push +/// [`set_extension`]: PathBuf::set_extension /// /// More details about the overall approach can be found in -/// the [module documentation](index.html). +/// the [module documentation](self). /// /// # Examples /// @@ -1079,7 +1157,7 @@ impl FusedIterator for Ancestors<'_> {} /// ``` /// /// Which method works best depends on what kind of situation you're in. -#[derive(Clone)] +#[cfg_attr(not(test), rustc_diagnostic_item = "PathBuf")] #[stable(feature = "rust1", since = "1.0.0")] // FIXME: // `PathBuf::as_mut_vec` current implementation relies @@ -1092,6 +1170,7 @@ pub struct PathBuf { } impl PathBuf { + #[inline] fn as_mut_vec(&mut self) -> &mut Vec { unsafe { &mut *(self as *mut PathBuf as *mut Vec) } } @@ -1106,6 +1185,8 @@ impl PathBuf { /// let path = PathBuf::new(); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + #[inline] pub fn new() -> PathBuf { PathBuf { inner: OsString::new() } } @@ -1116,7 +1197,6 @@ impl PathBuf { /// # Examples /// /// ``` - /// #![feature(path_buf_capacity)] /// use std::path::PathBuf; /// /// let mut path = PathBuf::with_capacity(10); @@ -1128,17 +1208,16 @@ impl PathBuf { /// assert_eq!(capacity, path.capacity()); /// ``` /// - /// [`with_capacity`]: ../ffi/struct.OsString.html#method.with_capacity - /// [`OsString`]: ../ffi/struct.OsString.html - #[unstable(feature = "path_buf_capacity", issue = "58234")] + /// [`with_capacity`]: OsString::with_capacity + #[stable(feature = "path_buf_capacity", since = "1.44.0")] + #[must_use] + #[inline] pub fn with_capacity(capacity: usize) -> PathBuf { PathBuf { inner: OsString::with_capacity(capacity) } } /// Coerces to a [`Path`] slice. /// - /// [`Path`]: struct.Path.html - /// /// # Examples /// /// ``` @@ -1148,6 +1227,8 @@ impl PathBuf { /// assert_eq!(Path::new("/test"), p.as_path()); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + #[inline] pub fn as_path(&self) -> &Path { self } @@ -1161,6 +1242,12 @@ impl PathBuf { /// * if `path` has a root but no prefix (e.g., `\windows`), it /// replaces everything except for the prefix (if any) of `self`. /// * if `path` has a prefix but no root, it replaces `self`. + /// * if `self` has a verbatim prefix (e.g. `\\?\C:\windows`) + /// and `path` is not empty, the new path is normalized: all references + /// to `.` and `..` are removed. + /// + /// Consider using [`Path::join`] if you need a new `PathBuf` instead of + /// using this function on a cloned `PathBuf`. /// /// # Examples /// @@ -1193,20 +1280,59 @@ impl PathBuf { let mut need_sep = self.as_mut_vec().last().map(|c| !is_sep_byte(*c)).unwrap_or(false); // in the special case of `C:` on Windows, do *not* add a separator + let comps = self.components(); + + if comps.prefix_len() > 0 + && comps.prefix_len() == comps.path.len() + && comps.prefix.unwrap().is_drive() { - let comps = self.components(); - if comps.prefix_len() > 0 - && comps.prefix_len() == comps.path.len() - && comps.prefix.unwrap().is_drive() - { - need_sep = false - } + need_sep = false } // absolute `path` replaces `self` if path.is_absolute() || path.prefix().is_some() { self.as_mut_vec().truncate(0); + // verbatim paths need . and .. removed + } else if comps.prefix_verbatim() && !path.inner.is_empty() { + let mut buf: Vec<_> = comps.collect(); + for c in path.components() { + match c { + Component::RootDir => { + buf.truncate(1); + buf.push(c); + } + Component::CurDir => (), + Component::ParentDir => { + if let Some(Component::Normal(_)) = buf.last() { + buf.pop(); + } + } + _ => buf.push(c), + } + } + + let mut res = OsString::new(); + let mut need_sep = false; + + for c in buf { + if need_sep && c != Component::RootDir { + res.push(MAIN_SEP_STR); + } + res.push(c.as_os_str()); + + need_sep = match c { + Component::RootDir => false, + Component::Prefix(prefix) => { + !prefix.parsed.is_drive() && prefix.parsed.len() > 0 + } + _ => true, + } + } + + self.inner = res; + return; + // `path` has a root but no prefix, e.g., `\windows` (Windows only) } else if path.has_root() { let prefix_len = self.components().prefix_remaining(); @@ -1225,18 +1351,17 @@ impl PathBuf { /// Returns `false` and does nothing if [`self.parent`] is [`None`]. /// Otherwise, returns `true`. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// [`self.parent`]: struct.PathBuf.html#method.parent + /// [`self.parent`]: Path::parent /// /// # Examples /// /// ``` /// use std::path::{Path, PathBuf}; /// - /// let mut p = PathBuf::from("/test/test.rs"); + /// let mut p = PathBuf::from("/spirited/away.rs"); /// /// p.pop(); - /// assert_eq!(Path::new("/test"), p); + /// assert_eq!(Path::new("/spirited"), p); /// p.pop(); /// assert_eq!(Path::new("/"), p); /// ``` @@ -1260,9 +1385,8 @@ impl PathBuf { /// `file_name`. The new path will be a sibling of the original path. /// (That is, it will have the same parent.) /// - /// [`self.file_name`]: struct.PathBuf.html#method.file_name - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// [`pop`]: struct.PathBuf.html#method.pop + /// [`self.file_name`]: Path::file_name + /// [`pop`]: PathBuf::pop /// /// # Examples /// @@ -1290,7 +1414,8 @@ impl PathBuf { self.push(file_name); } - /// Updates [`self.extension`] to `extension`. + /// Updates [`self.extension`] to `Some(extension)` or to `None` if + /// `extension` is empty. /// /// Returns `false` and does nothing if [`self.file_name`] is [`None`], /// returns `true` and updates the extension otherwise. @@ -1298,9 +1423,22 @@ impl PathBuf { /// If [`self.extension`] is [`None`], the extension is added; otherwise /// it is replaced. /// - /// [`self.file_name`]: struct.PathBuf.html#method.file_name - /// [`self.extension`]: struct.PathBuf.html#method.extension - /// [`None`]: ../../std/option/enum.Option.html#variant.None + /// If `extension` is the empty string, [`self.extension`] will be [`None`] + /// afterwards, not `Some("")`. + /// + /// # Caveats + /// + /// The new `extension` may contain dots and will be used in its entirety, + /// but only the part after the final dot will be reflected in + /// [`self.extension`]. + /// + /// If the file stem contains internal dots and `extension` is empty, part + /// of the old file stem will be considered the new [`self.extension`]. + /// + /// See the examples below. + /// + /// [`self.file_name`]: Path::file_name + /// [`self.extension`]: Path::extension /// /// # Examples /// @@ -1312,8 +1450,20 @@ impl PathBuf { /// p.set_extension("force"); /// assert_eq!(Path::new("/feel/the.force"), p.as_path()); /// - /// p.set_extension("dark_side"); - /// assert_eq!(Path::new("/feel/the.dark_side"), p.as_path()); + /// p.set_extension("dark.side"); + /// assert_eq!(Path::new("/feel/the.dark.side"), p.as_path()); + /// + /// p.set_extension("cookie"); + /// assert_eq!(Path::new("/feel/the.dark.cookie"), p.as_path()); + /// + /// p.set_extension(""); + /// assert_eq!(Path::new("/feel/the.dark"), p.as_path()); + /// + /// p.set_extension(""); + /// assert_eq!(Path::new("/feel/the"), p.as_path()); + /// + /// p.set_extension(""); + /// assert_eq!(Path::new("/feel/the"), p.as_path()); /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn set_extension>(&mut self, extension: S) -> bool { @@ -1323,17 +1473,17 @@ impl PathBuf { fn _set_extension(&mut self, extension: &OsStr) -> bool { let file_stem = match self.file_stem() { None => return false, - Some(f) => os_str_as_u8_slice(f), + Some(f) => f.bytes(), }; // truncate until right after the file stem - let end_file_stem = file_stem[file_stem.len()..].as_ptr() as usize; - let start = os_str_as_u8_slice(&self.inner).as_ptr() as usize; + let end_file_stem = file_stem[file_stem.len()..].as_ptr().addr(); + let start = self.inner.bytes().as_ptr().addr(); let v = self.as_mut_vec(); v.truncate(end_file_stem.wrapping_sub(start)); // add the new extension, if any - let new = os_str_as_u8_slice(extension); + let new = extension.bytes(); if !new.is_empty() { v.reserve_exact(new.len() + 1); v.push(b'.'); @@ -1343,9 +1493,31 @@ impl PathBuf { true } - /// Consumes the `PathBuf`, yielding its internal [`OsString`] storage. + /// Yields a mutable reference to the underlying [`OsString`] instance. + /// + /// # Examples /// - /// [`OsString`]: ../ffi/struct.OsString.html + /// ``` + /// #![feature(path_as_mut_os_str)] + /// use std::path::{Path, PathBuf}; + /// + /// let mut path = PathBuf::from("/foo"); + /// + /// path.push("bar"); + /// assert_eq!(path, Path::new("/foo/bar")); + /// + /// // OsString's `push` does not add a separator. + /// path.as_mut_os_string().push("baz"); + /// assert_eq!(path, Path::new("/foo/barbaz")); + /// ``` + #[unstable(feature = "path_as_mut_os_str", issue = "105021")] + #[must_use] + #[inline] + pub fn as_mut_os_string(&mut self) -> &mut OsString { + &mut self.inner + } + + /// Consumes the `PathBuf`, yielding its internal [`OsString`] storage. /// /// # Examples /// @@ -1356,15 +1528,16 @@ impl PathBuf { /// let os_str = p.into_os_string(); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use = "`self` will be dropped if the result is not used"] + #[inline] pub fn into_os_string(self) -> OsString { self.inner } - /// Converts this `PathBuf` into a [boxed][`Box`] [`Path`]. - /// - /// [`Box`]: ../../std/boxed/struct.Box.html - /// [`Path`]: struct.Path.html + /// Converts this `PathBuf` into a [boxed](Box) [`Path`]. #[stable(feature = "into_boxed_path", since = "1.20.0")] + #[must_use = "`self` will be dropped if the result is not used"] + #[inline] pub fn into_boxed_path(self) -> Box { let rw = Box::into_raw(self.inner.into_boxed_os_str()) as *mut Path; unsafe { Box::from_raw(rw) } @@ -1372,61 +1545,96 @@ impl PathBuf { /// Invokes [`capacity`] on the underlying instance of [`OsString`]. /// - /// [`capacity`]: ../ffi/struct.OsString.html#method.capacity - /// [`OsString`]: ../ffi/struct.OsString.html - #[unstable(feature = "path_buf_capacity", issue = "58234")] + /// [`capacity`]: OsString::capacity + #[stable(feature = "path_buf_capacity", since = "1.44.0")] + #[must_use] + #[inline] pub fn capacity(&self) -> usize { self.inner.capacity() } /// Invokes [`clear`] on the underlying instance of [`OsString`]. /// - /// [`clear`]: ../ffi/struct.OsString.html#method.clear - /// [`OsString`]: ../ffi/struct.OsString.html - #[unstable(feature = "path_buf_capacity", issue = "58234")] + /// [`clear`]: OsString::clear + #[stable(feature = "path_buf_capacity", since = "1.44.0")] + #[inline] pub fn clear(&mut self) { self.inner.clear() } /// Invokes [`reserve`] on the underlying instance of [`OsString`]. /// - /// [`reserve`]: ../ffi/struct.OsString.html#method.reserve - /// [`OsString`]: ../ffi/struct.OsString.html - #[unstable(feature = "path_buf_capacity", issue = "58234")] + /// [`reserve`]: OsString::reserve + #[stable(feature = "path_buf_capacity", since = "1.44.0")] + #[inline] pub fn reserve(&mut self, additional: usize) { self.inner.reserve(additional) } + /// Invokes [`try_reserve`] on the underlying instance of [`OsString`]. + /// + /// [`try_reserve`]: OsString::try_reserve + #[stable(feature = "try_reserve_2", since = "1.63.0")] + #[inline] + pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.inner.try_reserve(additional) + } + /// Invokes [`reserve_exact`] on the underlying instance of [`OsString`]. /// - /// [`reserve_exact`]: ../ffi/struct.OsString.html#method.reserve_exact - /// [`OsString`]: ../ffi/struct.OsString.html - #[unstable(feature = "path_buf_capacity", issue = "58234")] + /// [`reserve_exact`]: OsString::reserve_exact + #[stable(feature = "path_buf_capacity", since = "1.44.0")] + #[inline] pub fn reserve_exact(&mut self, additional: usize) { self.inner.reserve_exact(additional) } + /// Invokes [`try_reserve_exact`] on the underlying instance of [`OsString`]. + /// + /// [`try_reserve_exact`]: OsString::try_reserve_exact + #[stable(feature = "try_reserve_2", since = "1.63.0")] + #[inline] + pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.inner.try_reserve_exact(additional) + } + /// Invokes [`shrink_to_fit`] on the underlying instance of [`OsString`]. /// - /// [`shrink_to_fit`]: ../ffi/struct.OsString.html#method.shrink_to_fit - /// [`OsString`]: ../ffi/struct.OsString.html - #[unstable(feature = "path_buf_capacity", issue = "58234")] + /// [`shrink_to_fit`]: OsString::shrink_to_fit + #[stable(feature = "path_buf_capacity", since = "1.44.0")] + #[inline] pub fn shrink_to_fit(&mut self) { self.inner.shrink_to_fit() } /// Invokes [`shrink_to`] on the underlying instance of [`OsString`]. /// - /// [`shrink_to`]: ../ffi/struct.OsString.html#method.shrink_to - /// [`OsString`]: ../ffi/struct.OsString.html - #[unstable(feature = "path_buf_capacity", issue = "58234")] + /// [`shrink_to`]: OsString::shrink_to + #[stable(feature = "shrink_to", since = "1.56.0")] + #[inline] pub fn shrink_to(&mut self, min_capacity: usize) { self.inner.shrink_to(min_capacity) } } +#[stable(feature = "rust1", since = "1.0.0")] +impl Clone for PathBuf { + #[inline] + fn clone(&self) -> Self { + PathBuf { inner: self.inner.clone() } + } + + #[inline] + fn clone_from(&mut self, source: &Self) { + self.inner.clone_from(&source.inner) + } +} + #[stable(feature = "box_from_path", since = "1.17.0")] impl From<&Path> for Box { + /// Creates a boxed [`Path`] from a reference. + /// + /// This will allocate and clone `path` to it. fn from(path: &Path) -> Box { let boxed: Box = path.inner.into(); let rw = Box::into_raw(boxed) as *mut Path; @@ -1434,11 +1642,26 @@ impl From<&Path> for Box { } } +#[stable(feature = "box_from_cow", since = "1.45.0")] +impl From> for Box { + /// Creates a boxed [`Path`] from a clone-on-write pointer. + /// + /// Converting from a `Cow::Owned` does not clone or allocate. + #[inline] + fn from(cow: Cow<'_, Path>) -> Box { + match cow { + Cow::Borrowed(path) => Box::from(path), + Cow::Owned(path) => Box::from(path), + } + } +} + #[stable(feature = "path_buf_from_box", since = "1.18.0")] impl From> for PathBuf { - /// Converts a `Box` into a `PathBuf` + /// Converts a [Box]<[Path]> into a [`PathBuf`]. /// /// This conversion does not allocate or copy memory. + #[inline] fn from(boxed: Box) -> PathBuf { boxed.into_path_buf() } @@ -1446,10 +1669,11 @@ impl From> for PathBuf { #[stable(feature = "box_from_path_buf", since = "1.20.0")] impl From for Box { - /// Converts a `PathBuf` into a `Box` + /// Converts a [`PathBuf`] into a [Box]<[Path]>. /// /// This conversion currently should not allocate memory, /// but this behavior is not guaranteed on all platforms or in all future versions. + #[inline] fn from(p: PathBuf) -> Box { p.into_boxed_path() } @@ -1465,6 +1689,10 @@ impl Clone for Box { #[stable(feature = "rust1", since = "1.0.0")] impl> From<&T> for PathBuf { + /// Converts a borrowed [`OsStr`] to a [`PathBuf`]. + /// + /// Allocates a [`PathBuf`] and copies the data into it. + #[inline] fn from(s: &T) -> PathBuf { PathBuf::from(s.as_ref().to_os_string()) } @@ -1472,7 +1700,7 @@ impl> From<&T> for PathBuf { #[stable(feature = "rust1", since = "1.0.0")] impl From for PathBuf { - /// Converts a `OsString` into a `PathBuf` + /// Converts an [`OsString`] into a [`PathBuf`] /// /// This conversion does not allocate or copy memory. #[inline] @@ -1483,9 +1711,10 @@ impl From for PathBuf { #[stable(feature = "from_path_buf_for_os_string", since = "1.14.0")] impl From for OsString { - /// Converts a `PathBuf` into a `OsString` + /// Converts a [`PathBuf`] into an [`OsString`] /// /// This conversion does not allocate or copy memory. + #[inline] fn from(path_buf: PathBuf) -> OsString { path_buf.inner } @@ -1493,9 +1722,10 @@ impl From for OsString { #[stable(feature = "rust1", since = "1.0.0")] impl From for PathBuf { - /// Converts a `String` into a `PathBuf` + /// Converts a [`String`] into a [`PathBuf`] /// /// This conversion does not allocate or copy memory. + #[inline] fn from(s: String) -> PathBuf { PathBuf::from(OsString::from(s)) } @@ -1505,6 +1735,7 @@ impl From for PathBuf { impl FromStr for PathBuf { type Err = core::convert::Infallible; + #[inline] fn from_str(s: &str) -> Result { Ok(PathBuf::from(s)) } @@ -1524,6 +1755,11 @@ impl> iter::Extend

for PathBuf { fn extend>(&mut self, iter: I) { iter.into_iter().for_each(move |p| self.push(p.as_ref())); } + + #[inline] + fn extend_one(&mut self, p: P) { + self.push(p.as_ref()); + } } #[stable(feature = "rust1", since = "1.0.0")] @@ -1542,8 +1778,17 @@ impl ops::Deref for PathBuf { } } +#[stable(feature = "path_buf_deref_mut", since = "CURRENT_RUSTC_VERSION")] +impl ops::DerefMut for PathBuf { + #[inline] + fn deref_mut(&mut self) -> &mut Path { + Path::from_inner_mut(&mut self.inner) + } +} + #[stable(feature = "rust1", since = "1.0.0")] impl Borrow for PathBuf { + #[inline] fn borrow(&self) -> &Path { self.deref() } @@ -1551,6 +1796,7 @@ impl Borrow for PathBuf { #[stable(feature = "default_for_pathbuf", since = "1.17.0")] impl Default for PathBuf { + #[inline] fn default() -> Self { PathBuf::new() } @@ -1558,6 +1804,10 @@ impl Default for PathBuf { #[stable(feature = "cow_from_path", since = "1.6.0")] impl<'a> From<&'a Path> for Cow<'a, Path> { + /// Creates a clone-on-write pointer from a reference to + /// [`Path`]. + /// + /// This conversion does not clone or allocate. #[inline] fn from(s: &'a Path) -> Cow<'a, Path> { Cow::Borrowed(s) @@ -1566,6 +1816,10 @@ impl<'a> From<&'a Path> for Cow<'a, Path> { #[stable(feature = "cow_from_path", since = "1.6.0")] impl<'a> From for Cow<'a, Path> { + /// Creates a clone-on-write pointer from an owned + /// instance of [`PathBuf`]. + /// + /// This conversion does not clone or allocate. #[inline] fn from(s: PathBuf) -> Cow<'a, Path> { Cow::Owned(s) @@ -1574,6 +1828,10 @@ impl<'a> From for Cow<'a, Path> { #[stable(feature = "cow_from_pathbuf_ref", since = "1.28.0")] impl<'a> From<&'a PathBuf> for Cow<'a, Path> { + /// Creates a clone-on-write pointer from a reference to + /// [`PathBuf`]. + /// + /// This conversion does not clone or allocate. #[inline] fn from(p: &'a PathBuf) -> Cow<'a, Path> { Cow::Borrowed(p.as_path()) @@ -1582,6 +1840,9 @@ impl<'a> From<&'a PathBuf> for Cow<'a, Path> { #[stable(feature = "pathbuf_from_cow_path", since = "1.28.0")] impl<'a> From> for PathBuf { + /// Converts a clone-on-write pointer to an owned path. + /// + /// Converting from a `Cow::Owned` does not clone or allocate. #[inline] fn from(p: Cow<'a, Path>) -> Self { p.into_owned() @@ -1590,7 +1851,8 @@ impl<'a> From> for PathBuf { #[stable(feature = "shared_from_slice2", since = "1.24.0")] impl From for Arc { - /// Converts a `PathBuf` into an `Arc` by moving the `PathBuf` data into a new `Arc` buffer. + /// Converts a [`PathBuf`] into an [Arc]<[Path]> by moving the [`PathBuf`] data + /// into a new [`Arc`] buffer. #[inline] fn from(s: PathBuf) -> Arc { let arc: Arc = Arc::from(s.into_os_string()); @@ -1600,7 +1862,7 @@ impl From for Arc { #[stable(feature = "shared_from_slice2", since = "1.24.0")] impl From<&Path> for Arc { - /// Converts a `Path` into an `Arc` by copying the `Path` data into a new `Arc` buffer. + /// Converts a [`Path`] into an [`Arc`] by copying the [`Path`] data into a new [`Arc`] buffer. #[inline] fn from(s: &Path) -> Arc { let arc: Arc = Arc::from(s.as_os_str()); @@ -1610,7 +1872,8 @@ impl From<&Path> for Arc { #[stable(feature = "shared_from_slice2", since = "1.24.0")] impl From for Rc { - /// Converts a `PathBuf` into an `Rc` by moving the `PathBuf` data into a new `Rc` buffer. + /// Converts a [`PathBuf`] into an [Rc]<[Path]> by moving the [`PathBuf`] data into + /// a new [`Rc`] buffer. #[inline] fn from(s: PathBuf) -> Rc { let rc: Rc = Rc::from(s.into_os_string()); @@ -1620,7 +1883,7 @@ impl From for Rc { #[stable(feature = "shared_from_slice2", since = "1.24.0")] impl From<&Path> for Rc { - /// Converts a `Path` into an `Rc` by copying the `Path` data into a new `Rc` buffer. + /// Converts a [`Path`] into an [`Rc`] by copying the [`Path`] data into a new [`Rc`] buffer. #[inline] fn from(s: &Path) -> Rc { let rc: Rc = Rc::from(s.as_os_str()); @@ -1631,9 +1894,11 @@ impl From<&Path> for Rc { #[stable(feature = "rust1", since = "1.0.0")] impl ToOwned for Path { type Owned = PathBuf; + #[inline] fn to_owned(&self) -> PathBuf { self.to_path_buf() } + #[inline] fn clone_into(&self, target: &mut PathBuf) { self.inner.clone_into(&mut target.inner); } @@ -1641,6 +1906,7 @@ impl ToOwned for Path { #[stable(feature = "rust1", since = "1.0.0")] impl cmp::PartialEq for PathBuf { + #[inline] fn eq(&self, other: &PathBuf) -> bool { self.components() == other.components() } @@ -1658,20 +1924,23 @@ impl cmp::Eq for PathBuf {} #[stable(feature = "rust1", since = "1.0.0")] impl cmp::PartialOrd for PathBuf { + #[inline] fn partial_cmp(&self, other: &PathBuf) -> Option { - self.components().partial_cmp(other.components()) + Some(compare_components(self.components(), other.components())) } } #[stable(feature = "rust1", since = "1.0.0")] impl cmp::Ord for PathBuf { + #[inline] fn cmp(&self, other: &PathBuf) -> cmp::Ordering { - self.components().cmp(other.components()) + compare_components(self.components(), other.components()) } } #[stable(feature = "rust1", since = "1.0.0")] impl AsRef for PathBuf { + #[inline] fn as_ref(&self) -> &OsStr { &self.inner[..] } @@ -1688,12 +1957,8 @@ impl AsRef for PathBuf { /// pointer like `&` or [`Box`]. For an owned version of this type, /// see [`PathBuf`]. /// -/// [`str`]: ../primitive.str.html -/// [`Box`]: ../boxed/struct.Box.html -/// [`PathBuf`]: struct.PathBuf.html -/// /// More details about the overall approach can be found in -/// the [module documentation](index.html). +/// the [module documentation](self). /// /// # Examples /// @@ -1713,6 +1978,7 @@ impl AsRef for PathBuf { /// let extension = path.extension(); /// assert_eq!(extension, Some(OsStr::new("txt"))); /// ``` +#[cfg_attr(not(test), rustc_diagnostic_item = "Path")] #[stable(feature = "rust1", since = "1.0.0")] // FIXME: // `Path::new` current implementation relies @@ -1724,14 +1990,12 @@ pub struct Path { inner: OsStr, } -/// An error returned from [`Path::strip_prefix`][`strip_prefix`] if the prefix -/// was not found. +/// An error returned from [`Path::strip_prefix`] if the prefix was not found. /// /// This `struct` is created by the [`strip_prefix`] method on [`Path`]. /// See its documentation for more. /// -/// [`strip_prefix`]: struct.Path.html#method.strip_prefix -/// [`Path`]: struct.Path.html +/// [`strip_prefix`]: Path::strip_prefix #[derive(Debug, Clone, PartialEq, Eq)] #[stable(since = "1.7.0", feature = "strip_prefix")] pub struct StripPrefixError(()); @@ -1740,11 +2004,11 @@ impl Path { // The following (private!) function allows construction of a path from a u8 // slice, which is only safe when it is known to follow the OsStr encoding. unsafe fn from_u8_slice(s: &[u8]) -> &Path { - Path::new(u8_slice_as_os_str(s)) + unsafe { Path::new(u8_slice_as_os_str(s)) } } // The following (private!) function reveals the byte encoding used for OsStr. fn as_u8_slice(&self) -> &[u8] { - os_str_as_u8_slice(&self.inner) + self.inner.bytes() } /// Directly wraps a string slice as a `Path` slice. @@ -1774,10 +2038,14 @@ impl Path { unsafe { &*(s.as_ref() as *const OsStr as *const Path) } } + fn from_inner_mut(inner: &mut OsStr) -> &mut Path { + // SAFETY: Path is just a wrapper around OsStr, + // therefore converting &mut OsStr to &mut Path is safe. + unsafe { &mut *(inner as *mut OsStr as *mut Path) } + } + /// Yields the underlying [`OsStr`] slice. /// - /// [`OsStr`]: ../ffi/struct.OsStr.html - /// /// # Examples /// /// ``` @@ -1787,17 +2055,41 @@ impl Path { /// assert_eq!(os_str, std::ffi::OsStr::new("foo.txt")); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + #[inline] pub fn as_os_str(&self) -> &OsStr { &self.inner } + /// Yields a mutable reference to the underlying [`OsStr`] slice. + /// + /// # Examples + /// + /// ``` + /// #![feature(path_as_mut_os_str)] + /// use std::path::{Path, PathBuf}; + /// + /// let mut path = PathBuf::from("Foo.TXT"); + /// + /// assert_ne!(path, Path::new("foo.txt")); + /// + /// path.as_mut_os_str().make_ascii_lowercase(); + /// assert_eq!(path, Path::new("foo.txt")); + /// ``` + #[unstable(feature = "path_as_mut_os_str", issue = "105021")] + #[must_use] + #[inline] + pub fn as_mut_os_str(&mut self) -> &mut OsStr { + &mut self.inner + } + /// Yields a [`&str`] slice if the `Path` is valid unicode. /// /// This conversion may entail doing a check for UTF-8 validity. /// Note that validation is performed because non-UTF-8 strings are /// perfectly valid for some OS. /// - /// [`&str`]: ../primitive.str.html + /// [`&str`]: str /// /// # Examples /// @@ -1808,6 +2100,9 @@ impl Path { /// assert_eq!(path.to_str(), Some("foo.txt")); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] pub fn to_str(&self) -> Option<&str> { self.inner.to_str() } @@ -1817,8 +2112,7 @@ impl Path { /// Any non-Unicode sequences are replaced with /// [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. /// - /// [`Cow`]: ../borrow/enum.Cow.html - /// [U+FFFD]: ../char/constant.REPLACEMENT_CHARACTER.html + /// [U+FFFD]: super::char::REPLACEMENT_CHARACTER /// /// # Examples /// @@ -1834,14 +2128,15 @@ impl Path { /// Had `path` contained invalid unicode, the `to_string_lossy` call might /// have returned `"fo�.txt"`. #[stable(feature = "rust1", since = "1.0.0")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline] pub fn to_string_lossy(&self) -> Cow<'_, str> { self.inner.to_string_lossy() } /// Converts a `Path` to an owned [`PathBuf`]. /// - /// [`PathBuf`]: struct.PathBuf.html - /// /// # Examples /// /// ``` @@ -1851,6 +2146,8 @@ impl Path { /// assert_eq!(path_buf, std::path::PathBuf::from("foo.txt")); /// ``` #[rustc_conversion_suggestion] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] #[stable(feature = "rust1", since = "1.0.0")] pub fn to_path_buf(&self) -> PathBuf { PathBuf::from(self.inner.to_os_string()) @@ -1873,15 +2170,16 @@ impl Path { /// assert!(!Path::new("foo.txt").is_absolute()); /// ``` /// - /// [`has_root`]: #method.has_root + /// [`has_root`]: Path::has_root #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] #[allow(deprecated)] pub fn is_absolute(&self) -> bool { if cfg!(target_os = "redox") { // FIXME: Allow Redox prefixes self.has_root() || has_redox_scheme(self.as_u8_slice()) } else { - self.has_root() && (cfg!(unix) || self.prefix().is_some()) + self.has_root() && (cfg!(any(unix, target_os = "wasi")) || self.prefix().is_some()) } } @@ -1897,8 +2195,10 @@ impl Path { /// assert!(Path::new("foo.txt").is_relative()); /// ``` /// - /// [`is_absolute`]: #method.is_absolute + /// [`is_absolute`]: Path::is_absolute #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + #[inline] pub fn is_relative(&self) -> bool { !self.is_absolute() } @@ -1924,15 +2224,18 @@ impl Path { /// assert!(Path::new("/etc/passwd").has_root()); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + #[inline] pub fn has_root(&self) -> bool { self.components().has_root() } /// Returns the `Path` without its final component, if there is one. /// - /// Returns [`None`] if the path terminates in a root or prefix. + /// This means it returns `Some("")` for relative paths with one component. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None + /// Returns [`None`] if the path terminates in a root or prefix, or if it's + /// the empty string. /// /// # Examples /// @@ -1946,8 +2249,18 @@ impl Path { /// let grand_parent = parent.parent().unwrap(); /// assert_eq!(grand_parent, Path::new("/")); /// assert_eq!(grand_parent.parent(), None); + /// + /// let relative_path = Path::new("foo/bar"); + /// let parent = relative_path.parent(); + /// assert_eq!(parent, Some(Path::new("foo"))); + /// let grand_parent = parent.and_then(Path::parent); + /// assert_eq!(grand_parent, Some(Path::new(""))); + /// let great_grand_parent = grand_parent.and_then(Path::parent); + /// assert_eq!(great_grand_parent, None); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[doc(alias = "dirname")] + #[must_use] pub fn parent(&self) -> Option<&Path> { let mut comps = self.components(); let comp = comps.next_back(); @@ -1977,11 +2290,18 @@ impl Path { /// assert_eq!(ancestors.next(), Some(Path::new("/foo"))); /// assert_eq!(ancestors.next(), Some(Path::new("/"))); /// assert_eq!(ancestors.next(), None); + /// + /// let mut ancestors = Path::new("../foo/bar").ancestors(); + /// assert_eq!(ancestors.next(), Some(Path::new("../foo/bar"))); + /// assert_eq!(ancestors.next(), Some(Path::new("../foo"))); + /// assert_eq!(ancestors.next(), Some(Path::new(".."))); + /// assert_eq!(ancestors.next(), Some(Path::new(""))); + /// assert_eq!(ancestors.next(), None); /// ``` /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// [`parent`]: struct.Path.html#method.parent + /// [`parent`]: Path::parent #[stable(feature = "path_ancestors", since = "1.28.0")] + #[inline] pub fn ancestors(&self) -> Ancestors<'_> { Ancestors { next: Some(&self) } } @@ -1993,8 +2313,6 @@ impl Path { /// /// Returns [`None`] if the path terminates in `..`. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// /// # Examples /// /// ``` @@ -2009,6 +2327,8 @@ impl Path { /// assert_eq!(None, Path::new("/").file_name()); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[doc(alias = "basename")] + #[must_use] pub fn file_name(&self) -> Option<&OsStr> { self.components().next_back().and_then(|p| match p { Component::Normal(p) => Some(p), @@ -2023,8 +2343,7 @@ impl Path { /// If `base` is not a prefix of `self` (i.e., [`starts_with`] /// returns `false`), returns [`Err`]. /// - /// [`starts_with`]: #method.starts_with - /// [`Err`]: ../../std/result/enum.Result.html#variant.Err + /// [`starts_with`]: Path::starts_with /// /// # Examples /// @@ -2038,8 +2357,9 @@ impl Path { /// assert_eq!(path.strip_prefix("/test/"), Ok(Path::new("haha/foo.txt"))); /// assert_eq!(path.strip_prefix("/test/haha/foo.txt"), Ok(Path::new(""))); /// assert_eq!(path.strip_prefix("/test/haha/foo.txt/"), Ok(Path::new(""))); - /// assert_eq!(path.strip_prefix("test").is_ok(), false); - /// assert_eq!(path.strip_prefix("/haha").is_ok(), false); + /// + /// assert!(path.strip_prefix("test").is_err()); + /// assert!(path.strip_prefix("/haha").is_err()); /// /// let prefix = PathBuf::from("/test/"); /// assert_eq!(path.strip_prefix(prefix), Ok(Path::new("haha/foo.txt"))); @@ -2072,11 +2392,16 @@ impl Path { /// assert!(path.starts_with("/etc")); /// assert!(path.starts_with("/etc/")); /// assert!(path.starts_with("/etc/passwd")); - /// assert!(path.starts_with("/etc/passwd/")); + /// assert!(path.starts_with("/etc/passwd/")); // extra slash is okay + /// assert!(path.starts_with("/etc/passwd///")); // multiple extra slashes are okay /// /// assert!(!path.starts_with("/e")); + /// assert!(!path.starts_with("/etc/passwd.txt")); + /// + /// assert!(!Path::new("/etc/foo.rs").starts_with("/etc/foo")); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] pub fn starts_with>(&self, base: P) -> bool { self._starts_with(base.as_ref()) } @@ -2094,11 +2419,17 @@ impl Path { /// ``` /// use std::path::Path; /// - /// let path = Path::new("/etc/passwd"); + /// let path = Path::new("/etc/resolv.conf"); + /// + /// assert!(path.ends_with("resolv.conf")); + /// assert!(path.ends_with("etc/resolv.conf")); + /// assert!(path.ends_with("/etc/resolv.conf")); /// - /// assert!(path.ends_with("passwd")); + /// assert!(!path.ends_with("/resolv.conf")); + /// assert!(!path.ends_with("conf")); // use .extension() instead /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] pub fn ends_with>(&self, child: P) -> bool { self._ends_with(child.as_ref()) } @@ -2109,7 +2440,7 @@ impl Path { /// Extracts the stem (non-extension) portion of [`self.file_name`]. /// - /// [`self.file_name`]: struct.Path.html#method.file_name + /// [`self.file_name`]: Path::file_name /// /// The stem is: /// @@ -2118,23 +2449,62 @@ impl Path { /// * The entire file name if the file name begins with `.` and has no other `.`s within; /// * Otherwise, the portion of the file name before the final `.` /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// /// # Examples /// /// ``` /// use std::path::Path; /// - /// let path = Path::new("foo.rs"); - /// - /// assert_eq!("foo", path.file_stem().unwrap()); + /// assert_eq!("foo", Path::new("foo.rs").file_stem().unwrap()); + /// assert_eq!("foo.tar", Path::new("foo.tar.gz").file_stem().unwrap()); /// ``` + /// + /// # See Also + /// This method is similar to [`Path::file_prefix`], which extracts the portion of the file name + /// before the *first* `.` + /// + /// [`Path::file_prefix`]: Path::file_prefix + /// #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] pub fn file_stem(&self) -> Option<&OsStr> { - self.file_name().map(split_file_at_dot).and_then(|(before, after)| before.or(after)) + self.file_name().map(rsplit_file_at_dot).and_then(|(before, after)| before.or(after)) } - /// Extracts the extension of [`self.file_name`], if possible. + /// Extracts the prefix of [`self.file_name`]. + /// + /// The prefix is: + /// + /// * [`None`], if there is no file name; + /// * The entire file name if there is no embedded `.`; + /// * The portion of the file name before the first non-beginning `.`; + /// * The entire file name if the file name begins with `.` and has no other `.`s within; + /// * The portion of the file name before the second `.` if the file name begins with `.` + /// + /// [`self.file_name`]: Path::file_name + /// + /// # Examples + /// + /// ``` + /// # #![feature(path_file_prefix)] + /// use std::path::Path; + /// + /// assert_eq!("foo", Path::new("foo.rs").file_prefix().unwrap()); + /// assert_eq!("foo", Path::new("foo.tar.gz").file_prefix().unwrap()); + /// ``` + /// + /// # See Also + /// This method is similar to [`Path::file_stem`], which extracts the portion of the file name + /// before the *last* `.` + /// + /// [`Path::file_stem`]: Path::file_stem + /// + #[unstable(feature = "path_file_prefix", issue = "86319")] + #[must_use] + pub fn file_prefix(&self) -> Option<&OsStr> { + self.file_name().map(split_file_at_dot).and_then(|(before, _after)| Some(before)) + } + + /// Extracts the extension (without the leading dot) of [`self.file_name`], if possible. /// /// The extension is: /// @@ -2143,29 +2513,27 @@ impl Path { /// * [`None`], if the file name begins with `.` and has no other `.`s within; /// * Otherwise, the portion of the file name after the final `.` /// - /// [`self.file_name`]: struct.Path.html#method.file_name - /// [`None`]: ../../std/option/enum.Option.html#variant.None + /// [`self.file_name`]: Path::file_name /// /// # Examples /// /// ``` /// use std::path::Path; /// - /// let path = Path::new("foo.rs"); - /// - /// assert_eq!("rs", path.extension().unwrap()); + /// assert_eq!("rs", Path::new("foo.rs").extension().unwrap()); + /// assert_eq!("gz", Path::new("foo.tar.gz").extension().unwrap()); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] pub fn extension(&self) -> Option<&OsStr> { - self.file_name().map(split_file_at_dot).and_then(|(before, after)| before.and(after)) + self.file_name().map(rsplit_file_at_dot).and_then(|(before, after)| before.and(after)) } /// Creates an owned [`PathBuf`] with `path` adjoined to `self`. /// - /// See [`PathBuf::push`] for more details on what it means to adjoin a path. + /// If `path` is absolute, it replaces the current path. /// - /// [`PathBuf`]: struct.PathBuf.html - /// [`PathBuf::push`]: struct.PathBuf.html#method.push + /// See [`PathBuf::push`] for more details on what it means to adjoin a path. /// /// # Examples /// @@ -2173,6 +2541,7 @@ impl Path { /// use std::path::{Path, PathBuf}; /// /// assert_eq!(Path::new("/etc").join("passwd"), PathBuf::from("/etc/passwd")); + /// assert_eq!(Path::new("/etc").join("/bin/sh"), PathBuf::from("/bin/sh")); /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[must_use] @@ -2190,9 +2559,6 @@ impl Path { /// /// See [`PathBuf::set_file_name`] for more details. /// - /// [`PathBuf`]: struct.PathBuf.html - /// [`PathBuf::set_file_name`]: struct.PathBuf.html#method.set_file_name - /// /// # Examples /// /// ``` @@ -2205,6 +2571,7 @@ impl Path { /// assert_eq!(path.with_file_name("var"), PathBuf::from("/var")); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] pub fn with_file_name>(&self, file_name: S) -> PathBuf { self._with_file_name(file_name.as_ref()) } @@ -2219,9 +2586,6 @@ impl Path { /// /// See [`PathBuf::set_extension`] for more details. /// - /// [`PathBuf`]: struct.PathBuf.html - /// [`PathBuf::set_extension`]: struct.PathBuf.html#method.set_extension - /// /// # Examples /// /// ``` @@ -2229,6 +2593,11 @@ impl Path { /// /// let path = Path::new("foo.rs"); /// assert_eq!(path.with_extension("txt"), PathBuf::from("foo.txt")); + /// + /// let path = Path::new("foo.tar.gz"); + /// assert_eq!(path.with_extension(""), PathBuf::from("foo.tar")); + /// assert_eq!(path.with_extension("xz"), PathBuf::from("foo.tar.xz")); + /// assert_eq!(path.with_extension("").with_extension("txt"), PathBuf::from("foo.txt")); /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn with_extension>(&self, extension: S) -> PathBuf { @@ -2273,8 +2642,7 @@ impl Path { /// assert_eq!(components.next(), None) /// ``` /// - /// [`Component`]: enum.Component.html - /// [`CurDir`]: enum.Component.html#variant.CurDir + /// [`CurDir`]: Component::CurDir #[stable(feature = "rust1", since = "1.0.0")] pub fn components(&self) -> Components<'_> { let prefix = parse_prefix(self.as_os_str()); @@ -2294,8 +2662,7 @@ impl Path { /// For more information about the particulars of how the path is separated /// into components, see [`components`]. /// - /// [`components`]: #method.components - /// [`OsStr`]: ../ffi/struct.OsStr.html + /// [`components`]: Path::components /// /// # Examples /// @@ -2310,14 +2677,17 @@ impl Path { /// assert_eq!(it.next(), None) /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[inline] pub fn iter(&self) -> Iter<'_> { Iter { inner: self.components() } } /// Returns an object that implements [`Display`] for safely printing paths - /// that may contain non-Unicode data. + /// that may contain non-Unicode data. This may perform lossy conversion, + /// depending on the platform. If you would like an implementation which + /// escapes the path please use [`Debug`] instead. /// - /// [`Display`]: ../fmt/trait.Display.html + /// [`Display`]: fmt::Display /// /// # Examples /// @@ -2329,6 +2699,9 @@ impl Path { /// println!("{}", path.display()); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use = "this does not display the path, \ + it returns an object that can be displayed"] + #[inline] pub fn display(&self) -> Display<'_> { Display { path: self } } @@ -2340,8 +2713,6 @@ impl Path { /// /// This is an alias to [`fs::metadata`]. /// - /// [`fs::metadata`]: ../fs/fn.metadata.html - /// /// # Examples /// /// ```no_run @@ -2352,6 +2723,7 @@ impl Path { /// println!("{:?}", metadata.file_type()); /// ``` #[stable(feature = "path_ext", since = "1.5.0")] + #[inline] pub fn metadata(&self) -> io::Result { fs::metadata(self) } @@ -2360,8 +2732,6 @@ impl Path { /// /// This is an alias to [`fs::symlink_metadata`]. /// - /// [`fs::symlink_metadata`]: ../fs/fn.symlink_metadata.html - /// /// # Examples /// /// ```no_run @@ -2372,6 +2742,7 @@ impl Path { /// println!("{:?}", metadata.file_type()); /// ``` #[stable(feature = "path_ext", since = "1.5.0")] + #[inline] pub fn symlink_metadata(&self) -> io::Result { fs::symlink_metadata(self) } @@ -2381,8 +2752,6 @@ impl Path { /// /// This is an alias to [`fs::canonicalize`]. /// - /// [`fs::canonicalize`]: ../fs/fn.canonicalize.html - /// /// # Examples /// /// ```no_run @@ -2392,6 +2761,7 @@ impl Path { /// assert_eq!(path.canonicalize().unwrap(), PathBuf::from("/foo/test/bar.rs")); /// ``` #[stable(feature = "path_ext", since = "1.5.0")] + #[inline] pub fn canonicalize(&self) -> io::Result { fs::canonicalize(self) } @@ -2400,8 +2770,6 @@ impl Path { /// /// This is an alias to [`fs::read_link`]. /// - /// [`fs::read_link`]: ../fs/fn.read_link.html - /// /// # Examples /// /// ```no_run @@ -2411,21 +2779,18 @@ impl Path { /// let path_link = path.read_link().expect("read_link call failed"); /// ``` #[stable(feature = "path_ext", since = "1.5.0")] + #[inline] pub fn read_link(&self) -> io::Result { fs::read_link(self) } /// Returns an iterator over the entries within a directory. /// - /// The iterator will yield instances of [`io::Result`]`<`[`DirEntry`]`>`. New + /// The iterator will yield instances of [io::Result]<[fs::DirEntry]>. New /// errors may be encountered after an iterator is initially constructed. /// /// This is an alias to [`fs::read_dir`]. /// - /// [`io::Result`]: ../io/type.Result.html - /// [`DirEntry`]: ../fs/struct.DirEntry.html - /// [`fs::read_dir`]: ../fs/fn.read_dir.html - /// /// # Examples /// /// ```no_run @@ -2439,43 +2804,77 @@ impl Path { /// } /// ``` #[stable(feature = "path_ext", since = "1.5.0")] + #[inline] pub fn read_dir(&self) -> io::Result { fs::read_dir(self) } /// Returns `true` if the path points at an existing entity. /// + /// Warning: this method may be error-prone, consider using [`try_exists()`] instead! + /// It also has a risk of introducing time-of-check to time-of-use (TOCTOU) bugs. + /// /// This function will traverse symbolic links to query information about the - /// destination file. In case of broken symbolic links this will return `false`. + /// destination file. /// - /// If you cannot access the directory containing the file, e.g., because of a - /// permission error, this will return `false`. + /// If you cannot access the metadata of the file, e.g. because of a + /// permission error or broken symbolic links, this will return `false`. /// /// # Examples /// /// ```no_run /// use std::path::Path; - /// assert_eq!(Path::new("does_not_exist.txt").exists(), false); + /// assert!(!Path::new("does_not_exist.txt").exists()); /// ``` /// /// # See Also /// /// This is a convenience function that coerces errors to false. If you want to - /// check errors, call [fs::metadata]. + /// check errors, call [`Path::try_exists`]. /// - /// [fs::metadata]: ../../std/fs/fn.metadata.html + /// [`try_exists()`]: Self::try_exists #[stable(feature = "path_ext", since = "1.5.0")] + #[must_use] + #[inline] pub fn exists(&self) -> bool { fs::metadata(self).is_ok() } + /// Returns `Ok(true)` if the path points at an existing entity. + /// + /// This function will traverse symbolic links to query information about the + /// destination file. In case of broken symbolic links this will return `Ok(false)`. + /// + /// As opposed to the [`exists()`] method, this one doesn't silently ignore errors + /// unrelated to the path not existing. (E.g. it will return `Err(_)` in case of permission + /// denied on some of the parent directories.) + /// + /// Note that while this avoids some pitfalls of the `exists()` method, it still can not + /// prevent time-of-check to time-of-use (TOCTOU) bugs. You should only use it in scenarios + /// where those bugs are not an issue. + /// + /// # Examples + /// + /// ```no_run + /// use std::path::Path; + /// assert!(!Path::new("does_not_exist.txt").try_exists().expect("Can't check existence of file does_not_exist.txt")); + /// assert!(Path::new("/root/secret_file.txt").try_exists().is_err()); + /// ``` + /// + /// [`exists()`]: Self::exists + #[stable(feature = "path_try_exists", since = "1.63.0")] + #[inline] + pub fn try_exists(&self) -> io::Result { + fs::try_exists(self) + } + /// Returns `true` if the path exists on disk and is pointing at a regular file. /// /// This function will traverse symbolic links to query information about the - /// destination file. In case of broken symbolic links this will return `false`. + /// destination file. /// - /// If you cannot access the directory containing the file, e.g., because of a - /// permission error, this will return `false`. + /// If you cannot access the metadata of the file, e.g. because of a + /// permission error or broken symbolic links, this will return `false`. /// /// # Examples /// @@ -2488,12 +2887,16 @@ impl Path { /// # See Also /// /// This is a convenience function that coerces errors to false. If you want to - /// check errors, call [fs::metadata] and handle its Result. Then call - /// [fs::Metadata::is_file] if it was Ok. - /// - /// [fs::metadata]: ../../std/fs/fn.metadata.html - /// [fs::Metadata::is_file]: ../../std/fs/struct.Metadata.html#method.is_file + /// check errors, call [`fs::metadata`] and handle its [`Result`]. Then call + /// [`fs::Metadata::is_file`] if it was [`Ok`]. + /// + /// When the goal is simply to read from (or write to) the source, the most + /// reliable way to test the source can be read (or written to) is to open + /// it. Only using `is_file` can break workflows like `diff <( prog_a )` on + /// a Unix-like system for example. See [`fs::File::open`] or + /// [`fs::OpenOptions::open`] for more information. #[stable(feature = "path_ext", since = "1.5.0")] + #[must_use] pub fn is_file(&self) -> bool { fs::metadata(self).map(|m| m.is_file()).unwrap_or(false) } @@ -2501,10 +2904,10 @@ impl Path { /// Returns `true` if the path exists on disk and is pointing at a directory. /// /// This function will traverse symbolic links to query information about the - /// destination file. In case of broken symbolic links this will return `false`. + /// destination file. /// - /// If you cannot access the directory containing the file, e.g., because of a - /// permission error, this will return `false`. + /// If you cannot access the metadata of the file, e.g. because of a + /// permission error or broken symbolic links, this will return `false`. /// /// # Examples /// @@ -2517,22 +2920,50 @@ impl Path { /// # See Also /// /// This is a convenience function that coerces errors to false. If you want to - /// check errors, call [fs::metadata] and handle its Result. Then call - /// [fs::Metadata::is_dir] if it was Ok. - /// - /// [fs::metadata]: ../../std/fs/fn.metadata.html - /// [fs::Metadata::is_dir]: ../../std/fs/struct.Metadata.html#method.is_dir + /// check errors, call [`fs::metadata`] and handle its [`Result`]. Then call + /// [`fs::Metadata::is_dir`] if it was [`Ok`]. #[stable(feature = "path_ext", since = "1.5.0")] + #[must_use] pub fn is_dir(&self) -> bool { fs::metadata(self).map(|m| m.is_dir()).unwrap_or(false) } - /// Converts a [`Box`][`Box`] into a [`PathBuf`] without copying or - /// allocating. + /// Returns `true` if the path exists on disk and is pointing at a symbolic link. + /// + /// This function will not traverse symbolic links. + /// In case of a broken symbolic link this will also return true. + /// + /// If you cannot access the directory containing the file, e.g., because of a + /// permission error, this will return false. + /// + /// # Examples + /// + #[cfg_attr(unix, doc = "```no_run")] + #[cfg_attr(not(unix), doc = "```ignore")] + /// use std::path::Path; + /// use std::os::unix::fs::symlink; + /// + /// let link_path = Path::new("link"); + /// symlink("/origin_does_not_exist/", link_path).unwrap(); + /// assert_eq!(link_path.is_symlink(), true); + /// assert_eq!(link_path.exists(), false); + /// ``` /// - /// [`Box`]: ../../std/boxed/struct.Box.html - /// [`PathBuf`]: struct.PathBuf.html + /// # See Also + /// + /// This is a convenience function that coerces errors to false. If you want to + /// check errors, call [`fs::symlink_metadata`] and handle its [`Result`]. Then call + /// [`fs::Metadata::is_symlink`] if it was [`Ok`]. + #[must_use] + #[stable(feature = "is_symlink", since = "1.58.0")] + pub fn is_symlink(&self) -> bool { + fs::symlink_metadata(self).map(|m| m.is_symlink()).unwrap_or(false) + } + + /// Converts a [`Box`](Box) into a [`PathBuf`] without copying or + /// allocating. #[stable(feature = "into_boxed_path", since = "1.20.0")] + #[must_use = "`self` will be dropped if the result is not used"] pub fn into_path_buf(self: Box) -> PathBuf { let rw = Box::into_raw(self) as *mut OsStr; let inner = unsafe { Box::from_raw(rw) }; @@ -2542,6 +2973,7 @@ impl Path { #[stable(feature = "rust1", since = "1.0.0")] impl AsRef for Path { + #[inline] fn as_ref(&self) -> &OsStr { &self.inner } @@ -2558,7 +2990,9 @@ impl fmt::Debug for Path { /// /// A [`Path`] might contain non-Unicode data. This `struct` implements the /// [`Display`] trait in a way that mitigates that. It is created by the -/// [`display`][`Path::display`] method on [`Path`]. +/// [`display`](Path::display) method on [`Path`]. This may perform lossy +/// conversion, depending on the platform. If you would like an implementation +/// which escapes the path please use [`Debug`] instead. /// /// # Examples /// @@ -2570,10 +3004,8 @@ impl fmt::Debug for Path { /// println!("{}", path.display()); /// ``` /// -/// [`Display`]: ../../std/fmt/trait.Display.html -/// [`format!`]: ../../std/macro.format.html -/// [`Path`]: struct.Path.html -/// [`Path::display`]: struct.Path.html#method.display +/// [`Display`]: fmt::Display +/// [`format!`]: crate::format #[stable(feature = "rust1", since = "1.0.0")] pub struct Display<'a> { path: &'a Path, @@ -2595,17 +3027,60 @@ impl fmt::Display for Display<'_> { #[stable(feature = "rust1", since = "1.0.0")] impl cmp::PartialEq for Path { + #[inline] fn eq(&self, other: &Path) -> bool { - self.components().eq(other.components()) + self.components() == other.components() } } #[stable(feature = "rust1", since = "1.0.0")] impl Hash for Path { fn hash(&self, h: &mut H) { - for component in self.components() { - component.hash(h); + let bytes = self.as_u8_slice(); + let (prefix_len, verbatim) = match parse_prefix(&self.inner) { + Some(prefix) => { + prefix.hash(h); + (prefix.len(), prefix.is_verbatim()) + } + None => (0, false), + }; + let bytes = &bytes[prefix_len..]; + + let mut component_start = 0; + let mut bytes_hashed = 0; + + for i in 0..bytes.len() { + let is_sep = if verbatim { is_verbatim_sep(bytes[i]) } else { is_sep_byte(bytes[i]) }; + if is_sep { + if i > component_start { + let to_hash = &bytes[component_start..i]; + h.write(to_hash); + bytes_hashed += to_hash.len(); + } + + // skip over separator and optionally a following CurDir item + // since components() would normalize these away. + component_start = i + 1; + + let tail = &bytes[component_start..]; + + if !verbatim { + component_start += match tail { + [b'.'] => 1, + [b'.', sep @ _, ..] if is_sep_byte(*sep) => 1, + _ => 0, + }; + } + } } + + if component_start < bytes.len() { + let to_hash = &bytes[component_start..]; + h.write(to_hash); + bytes_hashed += to_hash.len(); + } + + h.write_usize(bytes_hashed); } } @@ -2614,20 +3089,23 @@ impl cmp::Eq for Path {} #[stable(feature = "rust1", since = "1.0.0")] impl cmp::PartialOrd for Path { + #[inline] fn partial_cmp(&self, other: &Path) -> Option { - self.components().partial_cmp(other.components()) + Some(compare_components(self.components(), other.components())) } } #[stable(feature = "rust1", since = "1.0.0")] impl cmp::Ord for Path { + #[inline] fn cmp(&self, other: &Path) -> cmp::Ordering { - self.components().cmp(other.components()) + compare_components(self.components(), other.components()) } } #[stable(feature = "rust1", since = "1.0.0")] impl AsRef for Path { + #[inline] fn as_ref(&self) -> &Path { self } @@ -2635,6 +3113,7 @@ impl AsRef for Path { #[stable(feature = "rust1", since = "1.0.0")] impl AsRef for OsStr { + #[inline] fn as_ref(&self) -> &Path { Path::new(self) } @@ -2642,6 +3121,7 @@ impl AsRef for OsStr { #[stable(feature = "cow_os_str_as_ref_path", since = "1.8.0")] impl AsRef for Cow<'_, OsStr> { + #[inline] fn as_ref(&self) -> &Path { Path::new(self) } @@ -2649,6 +3129,7 @@ impl AsRef for Cow<'_, OsStr> { #[stable(feature = "rust1", since = "1.0.0")] impl AsRef for OsString { + #[inline] fn as_ref(&self) -> &Path { Path::new(self) } @@ -2664,6 +3145,7 @@ impl AsRef for str { #[stable(feature = "rust1", since = "1.0.0")] impl AsRef for String { + #[inline] fn as_ref(&self) -> &Path { Path::new(self) } @@ -2681,6 +3163,7 @@ impl AsRef for PathBuf { impl<'a> IntoIterator for &'a PathBuf { type Item = &'a OsStr; type IntoIter = Iter<'a>; + #[inline] fn into_iter(self) -> Iter<'a> { self.iter() } @@ -2690,15 +3173,16 @@ impl<'a> IntoIterator for &'a PathBuf { impl<'a> IntoIterator for &'a Path { type Item = &'a OsStr; type IntoIter = Iter<'a>; + #[inline] fn into_iter(self) -> Iter<'a> { self.iter() } } macro_rules! impl_cmp { - ($lhs:ty, $rhs: ty) => { + (<$($life:lifetime),*> $lhs:ty, $rhs: ty) => { #[stable(feature = "partialeq_path", since = "1.6.0")] - impl<'a, 'b> PartialEq<$rhs> for $lhs { + impl<$($life),*> PartialEq<$rhs> for $lhs { #[inline] fn eq(&self, other: &$rhs) -> bool { ::eq(self, other) @@ -2706,7 +3190,7 @@ macro_rules! impl_cmp { } #[stable(feature = "partialeq_path", since = "1.6.0")] - impl<'a, 'b> PartialEq<$lhs> for $rhs { + impl<$($life),*> PartialEq<$lhs> for $rhs { #[inline] fn eq(&self, other: &$lhs) -> bool { ::eq(self, other) @@ -2714,7 +3198,7 @@ macro_rules! impl_cmp { } #[stable(feature = "cmp_path", since = "1.8.0")] - impl<'a, 'b> PartialOrd<$rhs> for $lhs { + impl<$($life),*> PartialOrd<$rhs> for $lhs { #[inline] fn partial_cmp(&self, other: &$rhs) -> Option { ::partial_cmp(self, other) @@ -2722,7 +3206,7 @@ macro_rules! impl_cmp { } #[stable(feature = "cmp_path", since = "1.8.0")] - impl<'a, 'b> PartialOrd<$lhs> for $rhs { + impl<$($life),*> PartialOrd<$lhs> for $rhs { #[inline] fn partial_cmp(&self, other: &$lhs) -> Option { ::partial_cmp(self, other) @@ -2731,16 +3215,16 @@ macro_rules! impl_cmp { }; } -impl_cmp!(PathBuf, Path); -impl_cmp!(PathBuf, &'a Path); -impl_cmp!(Cow<'a, Path>, Path); -impl_cmp!(Cow<'a, Path>, &'b Path); -impl_cmp!(Cow<'a, Path>, PathBuf); +impl_cmp!(<> PathBuf, Path); +impl_cmp!(<'a> PathBuf, &'a Path); +impl_cmp!(<'a> Cow<'a, Path>, Path); +impl_cmp!(<'a, 'b> Cow<'a, Path>, &'b Path); +impl_cmp!(<'a> Cow<'a, Path>, PathBuf); macro_rules! impl_cmp_os_str { - ($lhs:ty, $rhs: ty) => { + (<$($life:lifetime),*> $lhs:ty, $rhs: ty) => { #[stable(feature = "cmp_path", since = "1.8.0")] - impl<'a, 'b> PartialEq<$rhs> for $lhs { + impl<$($life),*> PartialEq<$rhs> for $lhs { #[inline] fn eq(&self, other: &$rhs) -> bool { ::eq(self, other.as_ref()) @@ -2748,7 +3232,7 @@ macro_rules! impl_cmp_os_str { } #[stable(feature = "cmp_path", since = "1.8.0")] - impl<'a, 'b> PartialEq<$lhs> for $rhs { + impl<$($life),*> PartialEq<$lhs> for $rhs { #[inline] fn eq(&self, other: &$lhs) -> bool { ::eq(self.as_ref(), other) @@ -2756,7 +3240,7 @@ macro_rules! impl_cmp_os_str { } #[stable(feature = "cmp_path", since = "1.8.0")] - impl<'a, 'b> PartialOrd<$rhs> for $lhs { + impl<$($life),*> PartialOrd<$rhs> for $lhs { #[inline] fn partial_cmp(&self, other: &$rhs) -> Option { ::partial_cmp(self, other.as_ref()) @@ -2764,7 +3248,7 @@ macro_rules! impl_cmp_os_str { } #[stable(feature = "cmp_path", since = "1.8.0")] - impl<'a, 'b> PartialOrd<$lhs> for $rhs { + impl<$($life),*> PartialOrd<$lhs> for $rhs { #[inline] fn partial_cmp(&self, other: &$lhs) -> Option { ::partial_cmp(self.as_ref(), other) @@ -2773,20 +3257,20 @@ macro_rules! impl_cmp_os_str { }; } -impl_cmp_os_str!(PathBuf, OsStr); -impl_cmp_os_str!(PathBuf, &'a OsStr); -impl_cmp_os_str!(PathBuf, Cow<'a, OsStr>); -impl_cmp_os_str!(PathBuf, OsString); -impl_cmp_os_str!(Path, OsStr); -impl_cmp_os_str!(Path, &'a OsStr); -impl_cmp_os_str!(Path, Cow<'a, OsStr>); -impl_cmp_os_str!(Path, OsString); -impl_cmp_os_str!(&'a Path, OsStr); -impl_cmp_os_str!(&'a Path, Cow<'b, OsStr>); -impl_cmp_os_str!(&'a Path, OsString); -impl_cmp_os_str!(Cow<'a, Path>, OsStr); -impl_cmp_os_str!(Cow<'a, Path>, &'b OsStr); -impl_cmp_os_str!(Cow<'a, Path>, OsString); +impl_cmp_os_str!(<> PathBuf, OsStr); +impl_cmp_os_str!(<'a> PathBuf, &'a OsStr); +impl_cmp_os_str!(<'a> PathBuf, Cow<'a, OsStr>); +impl_cmp_os_str!(<> PathBuf, OsString); +impl_cmp_os_str!(<> Path, OsStr); +impl_cmp_os_str!(<'a> Path, &'a OsStr); +impl_cmp_os_str!(<'a> Path, Cow<'a, OsStr>); +impl_cmp_os_str!(<> Path, OsString); +impl_cmp_os_str!(<'a> &'a Path, OsStr); +impl_cmp_os_str!(<'a, 'b> &'a Path, Cow<'b, OsStr>); +impl_cmp_os_str!(<'a> &'a Path, OsString); +impl_cmp_os_str!(<'a> Cow<'a, Path>, OsStr); +impl_cmp_os_str!(<'a, 'b> Cow<'a, Path>, &'b OsStr); +impl_cmp_os_str!(<'a> Cow<'a, Path>, OsString); #[stable(since = "1.7.0", feature = "strip_prefix")] impl fmt::Display for StripPrefixError { @@ -2804,1400 +3288,78 @@ impl Error for StripPrefixError { } } -#[cfg(test)] -mod tests { - use super::*; - - use crate::rc::Rc; - use crate::sync::Arc; - - macro_rules! t( - ($path:expr, iter: $iter:expr) => ( - { - let path = Path::new($path); - - // Forward iteration - let comps = path.iter() - .map(|p| p.to_string_lossy().into_owned()) - .collect::>(); - let exp: &[&str] = &$iter; - let exps = exp.iter().map(|s| s.to_string()).collect::>(); - assert!(comps == exps, "iter: Expected {:?}, found {:?}", - exps, comps); - - // Reverse iteration - let comps = Path::new($path).iter().rev() - .map(|p| p.to_string_lossy().into_owned()) - .collect::>(); - let exps = exps.into_iter().rev().collect::>(); - assert!(comps == exps, "iter().rev(): Expected {:?}, found {:?}", - exps, comps); - } - ); - - ($path:expr, has_root: $has_root:expr, is_absolute: $is_absolute:expr) => ( - { - let path = Path::new($path); - - let act_root = path.has_root(); - assert!(act_root == $has_root, "has_root: Expected {:?}, found {:?}", - $has_root, act_root); - - let act_abs = path.is_absolute(); - assert!(act_abs == $is_absolute, "is_absolute: Expected {:?}, found {:?}", - $is_absolute, act_abs); - } - ); - - ($path:expr, parent: $parent:expr, file_name: $file:expr) => ( - { - let path = Path::new($path); - - let parent = path.parent().map(|p| p.to_str().unwrap()); - let exp_parent: Option<&str> = $parent; - assert!(parent == exp_parent, "parent: Expected {:?}, found {:?}", - exp_parent, parent); - - let file = path.file_name().map(|p| p.to_str().unwrap()); - let exp_file: Option<&str> = $file; - assert!(file == exp_file, "file_name: Expected {:?}, found {:?}", - exp_file, file); - } - ); - - ($path:expr, file_stem: $file_stem:expr, extension: $extension:expr) => ( - { - let path = Path::new($path); - - let stem = path.file_stem().map(|p| p.to_str().unwrap()); - let exp_stem: Option<&str> = $file_stem; - assert!(stem == exp_stem, "file_stem: Expected {:?}, found {:?}", - exp_stem, stem); - - let ext = path.extension().map(|p| p.to_str().unwrap()); - let exp_ext: Option<&str> = $extension; - assert!(ext == exp_ext, "extension: Expected {:?}, found {:?}", - exp_ext, ext); - } - ); - - ($path:expr, iter: $iter:expr, - has_root: $has_root:expr, is_absolute: $is_absolute:expr, - parent: $parent:expr, file_name: $file:expr, - file_stem: $file_stem:expr, extension: $extension:expr) => ( - { - t!($path, iter: $iter); - t!($path, has_root: $has_root, is_absolute: $is_absolute); - t!($path, parent: $parent, file_name: $file); - t!($path, file_stem: $file_stem, extension: $extension); - } - ); - ); - - #[test] - fn into() { - use crate::borrow::Cow; - - let static_path = Path::new("/home/foo"); - let static_cow_path: Cow<'static, Path> = static_path.into(); - let pathbuf = PathBuf::from("/home/foo"); - - { - let path: &Path = &pathbuf; - let borrowed_cow_path: Cow<'_, Path> = path.into(); - - assert_eq!(static_cow_path, borrowed_cow_path); - } - - let owned_cow_path: Cow<'static, Path> = pathbuf.into(); - - assert_eq!(static_cow_path, owned_cow_path); - } - - #[test] - #[cfg(unix)] - pub fn test_decompositions_unix() { - t!("", - iter: [], - has_root: false, - is_absolute: false, - parent: None, - file_name: None, - file_stem: None, - extension: None - ); - - t!("foo", - iter: ["foo"], - has_root: false, - is_absolute: false, - parent: Some(""), - file_name: Some("foo"), - file_stem: Some("foo"), - extension: None - ); - - t!("/", - iter: ["/"], - has_root: true, - is_absolute: true, - parent: None, - file_name: None, - file_stem: None, - extension: None - ); - - t!("/foo", - iter: ["/", "foo"], - has_root: true, - is_absolute: true, - parent: Some("/"), - file_name: Some("foo"), - file_stem: Some("foo"), - extension: None - ); - - t!("foo/", - iter: ["foo"], - has_root: false, - is_absolute: false, - parent: Some(""), - file_name: Some("foo"), - file_stem: Some("foo"), - extension: None - ); - - t!("/foo/", - iter: ["/", "foo"], - has_root: true, - is_absolute: true, - parent: Some("/"), - file_name: Some("foo"), - file_stem: Some("foo"), - extension: None - ); - - t!("foo/bar", - iter: ["foo", "bar"], - has_root: false, - is_absolute: false, - parent: Some("foo"), - file_name: Some("bar"), - file_stem: Some("bar"), - extension: None - ); - - t!("/foo/bar", - iter: ["/", "foo", "bar"], - has_root: true, - is_absolute: true, - parent: Some("/foo"), - file_name: Some("bar"), - file_stem: Some("bar"), - extension: None - ); - - t!("///foo///", - iter: ["/", "foo"], - has_root: true, - is_absolute: true, - parent: Some("/"), - file_name: Some("foo"), - file_stem: Some("foo"), - extension: None - ); - - t!("///foo///bar", - iter: ["/", "foo", "bar"], - has_root: true, - is_absolute: true, - parent: Some("///foo"), - file_name: Some("bar"), - file_stem: Some("bar"), - extension: None - ); - - t!("./.", - iter: ["."], - has_root: false, - is_absolute: false, - parent: Some(""), - file_name: None, - file_stem: None, - extension: None - ); - - t!("/..", - iter: ["/", ".."], - has_root: true, - is_absolute: true, - parent: Some("/"), - file_name: None, - file_stem: None, - extension: None - ); - - t!("../", - iter: [".."], - has_root: false, - is_absolute: false, - parent: Some(""), - file_name: None, - file_stem: None, - extension: None - ); - - t!("foo/.", - iter: ["foo"], - has_root: false, - is_absolute: false, - parent: Some(""), - file_name: Some("foo"), - file_stem: Some("foo"), - extension: None - ); - - t!("foo/..", - iter: ["foo", ".."], - has_root: false, - is_absolute: false, - parent: Some("foo"), - file_name: None, - file_stem: None, - extension: None - ); - - t!("foo/./", - iter: ["foo"], - has_root: false, - is_absolute: false, - parent: Some(""), - file_name: Some("foo"), - file_stem: Some("foo"), - extension: None - ); - - t!("foo/./bar", - iter: ["foo", "bar"], - has_root: false, - is_absolute: false, - parent: Some("foo"), - file_name: Some("bar"), - file_stem: Some("bar"), - extension: None - ); - - t!("foo/../", - iter: ["foo", ".."], - has_root: false, - is_absolute: false, - parent: Some("foo"), - file_name: None, - file_stem: None, - extension: None - ); - - t!("foo/../bar", - iter: ["foo", "..", "bar"], - has_root: false, - is_absolute: false, - parent: Some("foo/.."), - file_name: Some("bar"), - file_stem: Some("bar"), - extension: None - ); - - t!("./a", - iter: [".", "a"], - has_root: false, - is_absolute: false, - parent: Some("."), - file_name: Some("a"), - file_stem: Some("a"), - extension: None - ); - - t!(".", - iter: ["."], - has_root: false, - is_absolute: false, - parent: Some(""), - file_name: None, - file_stem: None, - extension: None - ); - - t!("./", - iter: ["."], - has_root: false, - is_absolute: false, - parent: Some(""), - file_name: None, - file_stem: None, - extension: None - ); - - t!("a/b", - iter: ["a", "b"], - has_root: false, - is_absolute: false, - parent: Some("a"), - file_name: Some("b"), - file_stem: Some("b"), - extension: None - ); - - t!("a//b", - iter: ["a", "b"], - has_root: false, - is_absolute: false, - parent: Some("a"), - file_name: Some("b"), - file_stem: Some("b"), - extension: None - ); - - t!("a/./b", - iter: ["a", "b"], - has_root: false, - is_absolute: false, - parent: Some("a"), - file_name: Some("b"), - file_stem: Some("b"), - extension: None - ); - - t!("a/b/c", - iter: ["a", "b", "c"], - has_root: false, - is_absolute: false, - parent: Some("a/b"), - file_name: Some("c"), - file_stem: Some("c"), - extension: None - ); - - t!(".foo", - iter: [".foo"], - has_root: false, - is_absolute: false, - parent: Some(""), - file_name: Some(".foo"), - file_stem: Some(".foo"), - extension: None - ); - } - - #[test] - #[cfg(windows)] - pub fn test_decompositions_windows() { - t!("", - iter: [], - has_root: false, - is_absolute: false, - parent: None, - file_name: None, - file_stem: None, - extension: None - ); - - t!("foo", - iter: ["foo"], - has_root: false, - is_absolute: false, - parent: Some(""), - file_name: Some("foo"), - file_stem: Some("foo"), - extension: None - ); - - t!("/", - iter: ["\\"], - has_root: true, - is_absolute: false, - parent: None, - file_name: None, - file_stem: None, - extension: None - ); - - t!("\\", - iter: ["\\"], - has_root: true, - is_absolute: false, - parent: None, - file_name: None, - file_stem: None, - extension: None - ); - - t!("c:", - iter: ["c:"], - has_root: false, - is_absolute: false, - parent: None, - file_name: None, - file_stem: None, - extension: None - ); - - t!("c:\\", - iter: ["c:", "\\"], - has_root: true, - is_absolute: true, - parent: None, - file_name: None, - file_stem: None, - extension: None - ); - - t!("c:/", - iter: ["c:", "\\"], - has_root: true, - is_absolute: true, - parent: None, - file_name: None, - file_stem: None, - extension: None - ); - - t!("/foo", - iter: ["\\", "foo"], - has_root: true, - is_absolute: false, - parent: Some("/"), - file_name: Some("foo"), - file_stem: Some("foo"), - extension: None - ); - - t!("foo/", - iter: ["foo"], - has_root: false, - is_absolute: false, - parent: Some(""), - file_name: Some("foo"), - file_stem: Some("foo"), - extension: None - ); - - t!("/foo/", - iter: ["\\", "foo"], - has_root: true, - is_absolute: false, - parent: Some("/"), - file_name: Some("foo"), - file_stem: Some("foo"), - extension: None - ); - - t!("foo/bar", - iter: ["foo", "bar"], - has_root: false, - is_absolute: false, - parent: Some("foo"), - file_name: Some("bar"), - file_stem: Some("bar"), - extension: None - ); - - t!("/foo/bar", - iter: ["\\", "foo", "bar"], - has_root: true, - is_absolute: false, - parent: Some("/foo"), - file_name: Some("bar"), - file_stem: Some("bar"), - extension: None - ); - - t!("///foo///", - iter: ["\\", "foo"], - has_root: true, - is_absolute: false, - parent: Some("/"), - file_name: Some("foo"), - file_stem: Some("foo"), - extension: None - ); - - t!("///foo///bar", - iter: ["\\", "foo", "bar"], - has_root: true, - is_absolute: false, - parent: Some("///foo"), - file_name: Some("bar"), - file_stem: Some("bar"), - extension: None - ); - - t!("./.", - iter: ["."], - has_root: false, - is_absolute: false, - parent: Some(""), - file_name: None, - file_stem: None, - extension: None - ); - - t!("/..", - iter: ["\\", ".."], - has_root: true, - is_absolute: false, - parent: Some("/"), - file_name: None, - file_stem: None, - extension: None - ); - - t!("../", - iter: [".."], - has_root: false, - is_absolute: false, - parent: Some(""), - file_name: None, - file_stem: None, - extension: None - ); - - t!("foo/.", - iter: ["foo"], - has_root: false, - is_absolute: false, - parent: Some(""), - file_name: Some("foo"), - file_stem: Some("foo"), - extension: None - ); - - t!("foo/..", - iter: ["foo", ".."], - has_root: false, - is_absolute: false, - parent: Some("foo"), - file_name: None, - file_stem: None, - extension: None - ); - - t!("foo/./", - iter: ["foo"], - has_root: false, - is_absolute: false, - parent: Some(""), - file_name: Some("foo"), - file_stem: Some("foo"), - extension: None - ); - - t!("foo/./bar", - iter: ["foo", "bar"], - has_root: false, - is_absolute: false, - parent: Some("foo"), - file_name: Some("bar"), - file_stem: Some("bar"), - extension: None - ); - - t!("foo/../", - iter: ["foo", ".."], - has_root: false, - is_absolute: false, - parent: Some("foo"), - file_name: None, - file_stem: None, - extension: None - ); - - t!("foo/../bar", - iter: ["foo", "..", "bar"], - has_root: false, - is_absolute: false, - parent: Some("foo/.."), - file_name: Some("bar"), - file_stem: Some("bar"), - extension: None - ); - - t!("./a", - iter: [".", "a"], - has_root: false, - is_absolute: false, - parent: Some("."), - file_name: Some("a"), - file_stem: Some("a"), - extension: None - ); - - t!(".", - iter: ["."], - has_root: false, - is_absolute: false, - parent: Some(""), - file_name: None, - file_stem: None, - extension: None - ); - - t!("./", - iter: ["."], - has_root: false, - is_absolute: false, - parent: Some(""), - file_name: None, - file_stem: None, - extension: None - ); - - t!("a/b", - iter: ["a", "b"], - has_root: false, - is_absolute: false, - parent: Some("a"), - file_name: Some("b"), - file_stem: Some("b"), - extension: None - ); - - t!("a//b", - iter: ["a", "b"], - has_root: false, - is_absolute: false, - parent: Some("a"), - file_name: Some("b"), - file_stem: Some("b"), - extension: None - ); - - t!("a/./b", - iter: ["a", "b"], - has_root: false, - is_absolute: false, - parent: Some("a"), - file_name: Some("b"), - file_stem: Some("b"), - extension: None - ); - - t!("a/b/c", - iter: ["a", "b", "c"], - has_root: false, - is_absolute: false, - parent: Some("a/b"), - file_name: Some("c"), - file_stem: Some("c"), - extension: None); - - t!("a\\b\\c", - iter: ["a", "b", "c"], - has_root: false, - is_absolute: false, - parent: Some("a\\b"), - file_name: Some("c"), - file_stem: Some("c"), - extension: None - ); - - t!("\\a", - iter: ["\\", "a"], - has_root: true, - is_absolute: false, - parent: Some("\\"), - file_name: Some("a"), - file_stem: Some("a"), - extension: None - ); - - t!("c:\\foo.txt", - iter: ["c:", "\\", "foo.txt"], - has_root: true, - is_absolute: true, - parent: Some("c:\\"), - file_name: Some("foo.txt"), - file_stem: Some("foo"), - extension: Some("txt") - ); - - t!("\\\\server\\share\\foo.txt", - iter: ["\\\\server\\share", "\\", "foo.txt"], - has_root: true, - is_absolute: true, - parent: Some("\\\\server\\share\\"), - file_name: Some("foo.txt"), - file_stem: Some("foo"), - extension: Some("txt") - ); - - t!("\\\\server\\share", - iter: ["\\\\server\\share", "\\"], - has_root: true, - is_absolute: true, - parent: None, - file_name: None, - file_stem: None, - extension: None - ); - - t!("\\\\server", - iter: ["\\", "server"], - has_root: true, - is_absolute: false, - parent: Some("\\"), - file_name: Some("server"), - file_stem: Some("server"), - extension: None - ); - - t!("\\\\?\\bar\\foo.txt", - iter: ["\\\\?\\bar", "\\", "foo.txt"], - has_root: true, - is_absolute: true, - parent: Some("\\\\?\\bar\\"), - file_name: Some("foo.txt"), - file_stem: Some("foo"), - extension: Some("txt") - ); - - t!("\\\\?\\bar", - iter: ["\\\\?\\bar"], - has_root: true, - is_absolute: true, - parent: None, - file_name: None, - file_stem: None, - extension: None - ); - - t!("\\\\?\\", - iter: ["\\\\?\\"], - has_root: true, - is_absolute: true, - parent: None, - file_name: None, - file_stem: None, - extension: None - ); - - t!("\\\\?\\UNC\\server\\share\\foo.txt", - iter: ["\\\\?\\UNC\\server\\share", "\\", "foo.txt"], - has_root: true, - is_absolute: true, - parent: Some("\\\\?\\UNC\\server\\share\\"), - file_name: Some("foo.txt"), - file_stem: Some("foo"), - extension: Some("txt") - ); - - t!("\\\\?\\UNC\\server", - iter: ["\\\\?\\UNC\\server"], - has_root: true, - is_absolute: true, - parent: None, - file_name: None, - file_stem: None, - extension: None - ); - - t!("\\\\?\\UNC\\", - iter: ["\\\\?\\UNC\\"], - has_root: true, - is_absolute: true, - parent: None, - file_name: None, - file_stem: None, - extension: None - ); - - t!("\\\\?\\C:\\foo.txt", - iter: ["\\\\?\\C:", "\\", "foo.txt"], - has_root: true, - is_absolute: true, - parent: Some("\\\\?\\C:\\"), - file_name: Some("foo.txt"), - file_stem: Some("foo"), - extension: Some("txt") - ); - - t!("\\\\?\\C:\\", - iter: ["\\\\?\\C:", "\\"], - has_root: true, - is_absolute: true, - parent: None, - file_name: None, - file_stem: None, - extension: None - ); - - t!("\\\\?\\C:", - iter: ["\\\\?\\C:"], - has_root: true, - is_absolute: true, - parent: None, - file_name: None, - file_stem: None, - extension: None - ); - - t!("\\\\?\\foo/bar", - iter: ["\\\\?\\foo/bar"], - has_root: true, - is_absolute: true, - parent: None, - file_name: None, - file_stem: None, - extension: None - ); - - t!("\\\\?\\C:/foo", - iter: ["\\\\?\\C:/foo"], - has_root: true, - is_absolute: true, - parent: None, - file_name: None, - file_stem: None, - extension: None - ); - - t!("\\\\.\\foo\\bar", - iter: ["\\\\.\\foo", "\\", "bar"], - has_root: true, - is_absolute: true, - parent: Some("\\\\.\\foo\\"), - file_name: Some("bar"), - file_stem: Some("bar"), - extension: None - ); - - t!("\\\\.\\foo", - iter: ["\\\\.\\foo", "\\"], - has_root: true, - is_absolute: true, - parent: None, - file_name: None, - file_stem: None, - extension: None - ); - - t!("\\\\.\\foo/bar", - iter: ["\\\\.\\foo/bar", "\\"], - has_root: true, - is_absolute: true, - parent: None, - file_name: None, - file_stem: None, - extension: None - ); - - t!("\\\\.\\foo\\bar/baz", - iter: ["\\\\.\\foo", "\\", "bar", "baz"], - has_root: true, - is_absolute: true, - parent: Some("\\\\.\\foo\\bar"), - file_name: Some("baz"), - file_stem: Some("baz"), - extension: None - ); - - t!("\\\\.\\", - iter: ["\\\\.\\", "\\"], - has_root: true, - is_absolute: true, - parent: None, - file_name: None, - file_stem: None, - extension: None - ); - - t!("\\\\?\\a\\b\\", - iter: ["\\\\?\\a", "\\", "b"], - has_root: true, - is_absolute: true, - parent: Some("\\\\?\\a\\"), - file_name: Some("b"), - file_stem: Some("b"), - extension: None - ); - } - - #[test] - pub fn test_stem_ext() { - t!("foo", - file_stem: Some("foo"), - extension: None - ); - - t!("foo.", - file_stem: Some("foo"), - extension: Some("") - ); - - t!(".foo", - file_stem: Some(".foo"), - extension: None - ); - - t!("foo.txt", - file_stem: Some("foo"), - extension: Some("txt") - ); - - t!("foo.bar.txt", - file_stem: Some("foo.bar"), - extension: Some("txt") - ); - - t!("foo.bar.", - file_stem: Some("foo.bar"), - extension: Some("") - ); - - t!(".", file_stem: None, extension: None); - - t!("..", file_stem: None, extension: None); - - t!("", file_stem: None, extension: None); - } - - #[test] - pub fn test_push() { - macro_rules! tp( - ($path:expr, $push:expr, $expected:expr) => ( { - let mut actual = PathBuf::from($path); - actual.push($push); - assert!(actual.to_str() == Some($expected), - "pushing {:?} onto {:?}: Expected {:?}, got {:?}", - $push, $path, $expected, actual.to_str().unwrap()); - }); - ); - - if cfg!(unix) || cfg!(all(target_env = "sgx", target_vendor = "fortanix")) { - tp!("", "foo", "foo"); - tp!("foo", "bar", "foo/bar"); - tp!("foo/", "bar", "foo/bar"); - tp!("foo//", "bar", "foo//bar"); - tp!("foo/.", "bar", "foo/./bar"); - tp!("foo./.", "bar", "foo././bar"); - tp!("foo", "", "foo/"); - tp!("foo", ".", "foo/."); - tp!("foo", "..", "foo/.."); - tp!("foo", "/", "/"); - tp!("/foo/bar", "/", "/"); - tp!("/foo/bar", "/baz", "/baz"); - tp!("/foo/bar", "./baz", "/foo/bar/./baz"); - } else { - tp!("", "foo", "foo"); - tp!("foo", "bar", r"foo\bar"); - tp!("foo/", "bar", r"foo/bar"); - tp!(r"foo\", "bar", r"foo\bar"); - tp!("foo//", "bar", r"foo//bar"); - tp!(r"foo\\", "bar", r"foo\\bar"); - tp!("foo/.", "bar", r"foo/.\bar"); - tp!("foo./.", "bar", r"foo./.\bar"); - tp!(r"foo\.", "bar", r"foo\.\bar"); - tp!(r"foo.\.", "bar", r"foo.\.\bar"); - tp!("foo", "", "foo\\"); - tp!("foo", ".", r"foo\."); - tp!("foo", "..", r"foo\.."); - tp!("foo", "/", "/"); - tp!("foo", r"\", r"\"); - tp!("/foo/bar", "/", "/"); - tp!(r"\foo\bar", r"\", r"\"); - tp!("/foo/bar", "/baz", "/baz"); - tp!("/foo/bar", r"\baz", r"\baz"); - tp!("/foo/bar", "./baz", r"/foo/bar\./baz"); - tp!("/foo/bar", r".\baz", r"/foo/bar\.\baz"); - - tp!("c:\\", "windows", "c:\\windows"); - tp!("c:", "windows", "c:windows"); - - tp!("a\\b\\c", "d", "a\\b\\c\\d"); - tp!("\\a\\b\\c", "d", "\\a\\b\\c\\d"); - tp!("a\\b", "c\\d", "a\\b\\c\\d"); - tp!("a\\b", "\\c\\d", "\\c\\d"); - tp!("a\\b", ".", "a\\b\\."); - tp!("a\\b", "..\\c", "a\\b\\..\\c"); - tp!("a\\b", "C:a.txt", "C:a.txt"); - tp!("a\\b", "C:\\a.txt", "C:\\a.txt"); - tp!("C:\\a", "C:\\b.txt", "C:\\b.txt"); - tp!("C:\\a\\b\\c", "C:d", "C:d"); - tp!("C:a\\b\\c", "C:d", "C:d"); - tp!("C:", r"a\b\c", r"C:a\b\c"); - tp!("C:", r"..\a", r"C:..\a"); - tp!("\\\\server\\share\\foo", "bar", "\\\\server\\share\\foo\\bar"); - tp!("\\\\server\\share\\foo", "C:baz", "C:baz"); - tp!("\\\\?\\C:\\a\\b", "C:c\\d", "C:c\\d"); - tp!("\\\\?\\C:a\\b", "C:c\\d", "C:c\\d"); - tp!("\\\\?\\C:\\a\\b", "C:\\c\\d", "C:\\c\\d"); - tp!("\\\\?\\foo\\bar", "baz", "\\\\?\\foo\\bar\\baz"); - tp!("\\\\?\\UNC\\server\\share\\foo", "bar", "\\\\?\\UNC\\server\\share\\foo\\bar"); - tp!("\\\\?\\UNC\\server\\share", "C:\\a", "C:\\a"); - tp!("\\\\?\\UNC\\server\\share", "C:a", "C:a"); - - // Note: modified from old path API - tp!("\\\\?\\UNC\\server", "foo", "\\\\?\\UNC\\server\\foo"); - - tp!("C:\\a", "\\\\?\\UNC\\server\\share", "\\\\?\\UNC\\server\\share"); - tp!("\\\\.\\foo\\bar", "baz", "\\\\.\\foo\\bar\\baz"); - tp!("\\\\.\\foo\\bar", "C:a", "C:a"); - // again, not sure about the following, but I'm assuming \\.\ should be verbatim - tp!("\\\\.\\foo", "..\\bar", "\\\\.\\foo\\..\\bar"); - - tp!("\\\\?\\C:", "foo", "\\\\?\\C:\\foo"); // this is a weird one - } - } - - #[test] - pub fn test_pop() { - macro_rules! tp( - ($path:expr, $expected:expr, $output:expr) => ( { - let mut actual = PathBuf::from($path); - let output = actual.pop(); - assert!(actual.to_str() == Some($expected) && output == $output, - "popping from {:?}: Expected {:?}/{:?}, got {:?}/{:?}", - $path, $expected, $output, - actual.to_str().unwrap(), output); - }); - ); - - tp!("", "", false); - tp!("/", "/", false); - tp!("foo", "", true); - tp!(".", "", true); - tp!("/foo", "/", true); - tp!("/foo/bar", "/foo", true); - tp!("foo/bar", "foo", true); - tp!("foo/.", "", true); - tp!("foo//bar", "foo", true); - - if cfg!(windows) { - tp!("a\\b\\c", "a\\b", true); - tp!("\\a", "\\", true); - tp!("\\", "\\", false); - - tp!("C:\\a\\b", "C:\\a", true); - tp!("C:\\a", "C:\\", true); - tp!("C:\\", "C:\\", false); - tp!("C:a\\b", "C:a", true); - tp!("C:a", "C:", true); - tp!("C:", "C:", false); - tp!("\\\\server\\share\\a\\b", "\\\\server\\share\\a", true); - tp!("\\\\server\\share\\a", "\\\\server\\share\\", true); - tp!("\\\\server\\share", "\\\\server\\share", false); - tp!("\\\\?\\a\\b\\c", "\\\\?\\a\\b", true); - tp!("\\\\?\\a\\b", "\\\\?\\a\\", true); - tp!("\\\\?\\a", "\\\\?\\a", false); - tp!("\\\\?\\C:\\a\\b", "\\\\?\\C:\\a", true); - tp!("\\\\?\\C:\\a", "\\\\?\\C:\\", true); - tp!("\\\\?\\C:\\", "\\\\?\\C:\\", false); - tp!("\\\\?\\UNC\\server\\share\\a\\b", "\\\\?\\UNC\\server\\share\\a", true); - tp!("\\\\?\\UNC\\server\\share\\a", "\\\\?\\UNC\\server\\share\\", true); - tp!("\\\\?\\UNC\\server\\share", "\\\\?\\UNC\\server\\share", false); - tp!("\\\\.\\a\\b\\c", "\\\\.\\a\\b", true); - tp!("\\\\.\\a\\b", "\\\\.\\a\\", true); - tp!("\\\\.\\a", "\\\\.\\a", false); - - tp!("\\\\?\\a\\b\\", "\\\\?\\a\\", true); - } - } - - #[test] - pub fn test_set_file_name() { - macro_rules! tfn( - ($path:expr, $file:expr, $expected:expr) => ( { - let mut p = PathBuf::from($path); - p.set_file_name($file); - assert!(p.to_str() == Some($expected), - "setting file name of {:?} to {:?}: Expected {:?}, got {:?}", - $path, $file, $expected, - p.to_str().unwrap()); - }); - ); - - tfn!("foo", "foo", "foo"); - tfn!("foo", "bar", "bar"); - tfn!("foo", "", ""); - tfn!("", "foo", "foo"); - if cfg!(unix) || cfg!(all(target_env = "sgx", target_vendor = "fortanix")) { - tfn!(".", "foo", "./foo"); - tfn!("foo/", "bar", "bar"); - tfn!("foo/.", "bar", "bar"); - tfn!("..", "foo", "../foo"); - tfn!("foo/..", "bar", "foo/../bar"); - tfn!("/", "foo", "/foo"); - } else { - tfn!(".", "foo", r".\foo"); - tfn!(r"foo\", "bar", r"bar"); - tfn!(r"foo\.", "bar", r"bar"); - tfn!("..", "foo", r"..\foo"); - tfn!(r"foo\..", "bar", r"foo\..\bar"); - tfn!(r"\", "foo", r"\foo"); - } - } - - #[test] - pub fn test_set_extension() { - macro_rules! tfe( - ($path:expr, $ext:expr, $expected:expr, $output:expr) => ( { - let mut p = PathBuf::from($path); - let output = p.set_extension($ext); - assert!(p.to_str() == Some($expected) && output == $output, - "setting extension of {:?} to {:?}: Expected {:?}/{:?}, got {:?}/{:?}", - $path, $ext, $expected, $output, - p.to_str().unwrap(), output); - }); - ); - - tfe!("foo", "txt", "foo.txt", true); - tfe!("foo.bar", "txt", "foo.txt", true); - tfe!("foo.bar.baz", "txt", "foo.bar.txt", true); - tfe!(".test", "txt", ".test.txt", true); - tfe!("foo.txt", "", "foo", true); - tfe!("foo", "", "foo", true); - tfe!("", "foo", "", false); - tfe!(".", "foo", ".", false); - tfe!("foo/", "bar", "foo.bar", true); - tfe!("foo/.", "bar", "foo.bar", true); - tfe!("..", "foo", "..", false); - tfe!("foo/..", "bar", "foo/..", false); - tfe!("/", "foo", "/", false); - } - - #[test] - fn test_eq_receivers() { - use crate::borrow::Cow; - - let borrowed: &Path = Path::new("foo/bar"); - let mut owned: PathBuf = PathBuf::new(); - owned.push("foo"); - owned.push("bar"); - let borrowed_cow: Cow<'_, Path> = borrowed.into(); - let owned_cow: Cow<'_, Path> = owned.clone().into(); - - macro_rules! t { - ($($current:expr),+) => { - $( - assert_eq!($current, borrowed); - assert_eq!($current, owned); - assert_eq!($current, borrowed_cow); - assert_eq!($current, owned_cow); - )+ - } - } - - t!(borrowed, owned, borrowed_cow, owned_cow); - } - - #[test] - pub fn test_compare() { - use crate::collections::hash_map::DefaultHasher; - use crate::hash::{Hash, Hasher}; - - fn hash(t: T) -> u64 { - let mut s = DefaultHasher::new(); - t.hash(&mut s); - s.finish() - } - - macro_rules! tc( - ($path1:expr, $path2:expr, eq: $eq:expr, - starts_with: $starts_with:expr, ends_with: $ends_with:expr, - relative_from: $relative_from:expr) => ({ - let path1 = Path::new($path1); - let path2 = Path::new($path2); - - let eq = path1 == path2; - assert!(eq == $eq, "{:?} == {:?}, expected {:?}, got {:?}", - $path1, $path2, $eq, eq); - assert!($eq == (hash(path1) == hash(path2)), - "{:?} == {:?}, expected {:?}, got {} and {}", - $path1, $path2, $eq, hash(path1), hash(path2)); - - let starts_with = path1.starts_with(path2); - assert!(starts_with == $starts_with, - "{:?}.starts_with({:?}), expected {:?}, got {:?}", $path1, $path2, - $starts_with, starts_with); - - let ends_with = path1.ends_with(path2); - assert!(ends_with == $ends_with, - "{:?}.ends_with({:?}), expected {:?}, got {:?}", $path1, $path2, - $ends_with, ends_with); - - let relative_from = path1.strip_prefix(path2) - .map(|p| p.to_str().unwrap()) - .ok(); - let exp: Option<&str> = $relative_from; - assert!(relative_from == exp, - "{:?}.strip_prefix({:?}), expected {:?}, got {:?}", - $path1, $path2, exp, relative_from); - }); - ); - - tc!("", "", - eq: true, - starts_with: true, - ends_with: true, - relative_from: Some("") - ); - - tc!("foo", "", - eq: false, - starts_with: true, - ends_with: true, - relative_from: Some("foo") - ); - - tc!("", "foo", - eq: false, - starts_with: false, - ends_with: false, - relative_from: None - ); - - tc!("foo", "foo", - eq: true, - starts_with: true, - ends_with: true, - relative_from: Some("") - ); - - tc!("foo/", "foo", - eq: true, - starts_with: true, - ends_with: true, - relative_from: Some("") - ); - - tc!("foo/bar", "foo", - eq: false, - starts_with: true, - ends_with: false, - relative_from: Some("bar") - ); - - tc!("foo/bar/baz", "foo/bar", - eq: false, - starts_with: true, - ends_with: false, - relative_from: Some("baz") - ); - - tc!("foo/bar", "foo/bar/baz", - eq: false, - starts_with: false, - ends_with: false, - relative_from: None - ); - - tc!("./foo/bar/", ".", - eq: false, - starts_with: true, - ends_with: false, - relative_from: Some("foo/bar") - ); - - if cfg!(windows) { - tc!(r"C:\src\rust\cargo-test\test\Cargo.toml", - r"c:\src\rust\cargo-test\test", - eq: false, - starts_with: true, - ends_with: false, - relative_from: Some("Cargo.toml") - ); - - tc!(r"c:\foo", r"C:\foo", - eq: true, - starts_with: true, - ends_with: true, - relative_from: Some("") - ); - } - } - - #[test] - fn test_components_debug() { - let path = Path::new("/tmp"); - - let mut components = path.components(); - - let expected = "Components([RootDir, Normal(\"tmp\")])"; - let actual = format!("{:?}", components); - assert_eq!(expected, actual); - - let _ = components.next().unwrap(); - let expected = "Components([Normal(\"tmp\")])"; - let actual = format!("{:?}", components); - assert_eq!(expected, actual); - - let _ = components.next().unwrap(); - let expected = "Components([])"; - let actual = format!("{:?}", components); - assert_eq!(expected, actual); - } - - #[cfg(unix)] - #[test] - fn test_iter_debug() { - let path = Path::new("/tmp"); - - let mut iter = path.iter(); - - let expected = "Iter([\"/\", \"tmp\"])"; - let actual = format!("{:?}", iter); - assert_eq!(expected, actual); - - let _ = iter.next().unwrap(); - let expected = "Iter([\"tmp\"])"; - let actual = format!("{:?}", iter); - assert_eq!(expected, actual); - - let _ = iter.next().unwrap(); - let expected = "Iter([])"; - let actual = format!("{:?}", iter); - assert_eq!(expected, actual); - } - - #[test] - fn into_boxed() { - let orig: &str = "some/sort/of/path"; - let path = Path::new(orig); - let boxed: Box = Box::from(path); - let path_buf = path.to_owned().into_boxed_path().into_path_buf(); - assert_eq!(path, &*boxed); - assert_eq!(&*boxed, &*path_buf); - assert_eq!(&*path_buf, path); - } - - #[test] - fn test_clone_into() { - let mut path_buf = PathBuf::from("supercalifragilisticexpialidocious"); - let path = Path::new("short"); - path.clone_into(&mut path_buf); - assert_eq!(path, path_buf); - assert!(path_buf.into_os_string().capacity() >= 15); - } - - #[test] - fn display_format_flags() { - assert_eq!(format!("a{:#<5}b", Path::new("").display()), "a#####b"); - assert_eq!(format!("a{:#<5}b", Path::new("a").display()), "aa####b"); - } - - #[test] - fn into_rc() { - let orig = "hello/world"; - let path = Path::new(orig); - let rc: Rc = Rc::from(path); - let arc: Arc = Arc::from(path); - - assert_eq!(&*rc, path); - assert_eq!(&*arc, path); - - let rc2: Rc = Rc::from(path.to_owned()); - let arc2: Arc = Arc::from(path.to_owned()); - - assert_eq!(&*rc2, path); - assert_eq!(&*arc2, path); +/// Makes the path absolute without accessing the filesystem. +/// +/// If the path is relative, the current directory is used as the base directory. +/// All intermediate components will be resolved according to platforms-specific +/// rules but unlike [`canonicalize`][crate::fs::canonicalize] this does not +/// resolve symlinks and may succeed even if the path does not exist. +/// +/// If the `path` is empty or getting the +/// [current directory][crate::env::current_dir] fails then an error will be +/// returned. +/// +/// # Examples +/// +/// ## Posix paths +/// +/// ``` +/// #![feature(absolute_path)] +/// # #[cfg(unix)] +/// fn main() -> std::io::Result<()> { +/// use std::path::{self, Path}; +/// +/// // Relative to absolute +/// let absolute = path::absolute("foo/./bar")?; +/// assert!(absolute.ends_with("foo/bar")); +/// +/// // Absolute to absolute +/// let absolute = path::absolute("/foo//test/.././bar.rs")?; +/// assert_eq!(absolute, Path::new("/foo/test/../bar.rs")); +/// Ok(()) +/// } +/// # #[cfg(not(unix))] +/// # fn main() {} +/// ``` +/// +/// The path is resolved using [POSIX semantics][posix-semantics] except that +/// it stops short of resolving symlinks. This means it will keep `..` +/// components and trailing slashes. +/// +/// ## Windows paths +/// +/// ``` +/// #![feature(absolute_path)] +/// # #[cfg(windows)] +/// fn main() -> std::io::Result<()> { +/// use std::path::{self, Path}; +/// +/// // Relative to absolute +/// let absolute = path::absolute("foo/./bar")?; +/// assert!(absolute.ends_with(r"foo\bar")); +/// +/// // Absolute to absolute +/// let absolute = path::absolute(r"C:\foo//test\..\./bar.rs")?; +/// +/// assert_eq!(absolute, Path::new(r"C:\foo\bar.rs")); +/// Ok(()) +/// } +/// # #[cfg(not(windows))] +/// # fn main() {} +/// ``` +/// +/// For verbatim paths this will simply return the path as given. For other +/// paths this is currently equivalent to calling [`GetFullPathNameW`][windows-path] +/// This may change in the future. +/// +/// [posix-semantics]: https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 +/// [windows-path]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getfullpathnamew +#[unstable(feature = "absolute_path", issue = "92750")] +pub fn absolute>(path: P) -> io::Result { + let path = path.as_ref(); + if path.as_os_str().is_empty() { + Err(io::const_io_error!(io::ErrorKind::InvalidInput, "cannot make an empty path absolute",)) + } else { + sys::path::absolute(path) } } diff --git a/crux-mir/lib/std/src/path/tests.rs b/crux-mir/lib/std/src/path/tests.rs new file mode 100644 index 000000000..dd307022c --- /dev/null +++ b/crux-mir/lib/std/src/path/tests.rs @@ -0,0 +1,1878 @@ +use super::*; + +use crate::collections::hash_map::DefaultHasher; +use crate::collections::{BTreeSet, HashSet}; +use crate::hash::Hasher; +use crate::rc::Rc; +use crate::sync::Arc; +use core::hint::black_box; + +#[allow(unknown_lints, unused_macro_rules)] +macro_rules! t ( + ($path:expr, iter: $iter:expr) => ( + { + let path = Path::new($path); + + // Forward iteration + let comps = path.iter() + .map(|p| p.to_string_lossy().into_owned()) + .collect::>(); + let exp: &[&str] = &$iter; + let exps = exp.iter().map(|s| s.to_string()).collect::>(); + assert!(comps == exps, "iter: Expected {:?}, found {:?}", + exps, comps); + + // Reverse iteration + let comps = Path::new($path).iter().rev() + .map(|p| p.to_string_lossy().into_owned()) + .collect::>(); + let exps = exps.into_iter().rev().collect::>(); + assert!(comps == exps, "iter().rev(): Expected {:?}, found {:?}", + exps, comps); + } + ); + + ($path:expr, has_root: $has_root:expr, is_absolute: $is_absolute:expr) => ( + { + let path = Path::new($path); + + let act_root = path.has_root(); + assert!(act_root == $has_root, "has_root: Expected {:?}, found {:?}", + $has_root, act_root); + + let act_abs = path.is_absolute(); + assert!(act_abs == $is_absolute, "is_absolute: Expected {:?}, found {:?}", + $is_absolute, act_abs); + } + ); + + ($path:expr, parent: $parent:expr, file_name: $file:expr) => ( + { + let path = Path::new($path); + + let parent = path.parent().map(|p| p.to_str().unwrap()); + let exp_parent: Option<&str> = $parent; + assert!(parent == exp_parent, "parent: Expected {:?}, found {:?}", + exp_parent, parent); + + let file = path.file_name().map(|p| p.to_str().unwrap()); + let exp_file: Option<&str> = $file; + assert!(file == exp_file, "file_name: Expected {:?}, found {:?}", + exp_file, file); + } + ); + + ($path:expr, file_stem: $file_stem:expr, extension: $extension:expr) => ( + { + let path = Path::new($path); + + let stem = path.file_stem().map(|p| p.to_str().unwrap()); + let exp_stem: Option<&str> = $file_stem; + assert!(stem == exp_stem, "file_stem: Expected {:?}, found {:?}", + exp_stem, stem); + + let ext = path.extension().map(|p| p.to_str().unwrap()); + let exp_ext: Option<&str> = $extension; + assert!(ext == exp_ext, "extension: Expected {:?}, found {:?}", + exp_ext, ext); + } + ); + + ($path:expr, file_prefix: $file_prefix:expr, extension: $extension:expr) => ( + { + let path = Path::new($path); + + let prefix = path.file_prefix().map(|p| p.to_str().unwrap()); + let exp_prefix: Option<&str> = $file_prefix; + assert!(prefix == exp_prefix, "file_prefix: Expected {:?}, found {:?}", + exp_prefix, prefix); + + let ext = path.extension().map(|p| p.to_str().unwrap()); + let exp_ext: Option<&str> = $extension; + assert!(ext == exp_ext, "extension: Expected {:?}, found {:?}", + exp_ext, ext); + } + ); + + ($path:expr, iter: $iter:expr, + has_root: $has_root:expr, is_absolute: $is_absolute:expr, + parent: $parent:expr, file_name: $file:expr, + file_stem: $file_stem:expr, extension: $extension:expr, + file_prefix: $file_prefix:expr) => ( + { + t!($path, iter: $iter); + t!($path, has_root: $has_root, is_absolute: $is_absolute); + t!($path, parent: $parent, file_name: $file); + t!($path, file_stem: $file_stem, extension: $extension); + t!($path, file_prefix: $file_prefix, extension: $extension); + } + ); +); + +#[test] +fn into() { + use crate::borrow::Cow; + + let static_path = Path::new("/home/foo"); + let static_cow_path: Cow<'static, Path> = static_path.into(); + let pathbuf = PathBuf::from("/home/foo"); + + { + let path: &Path = &pathbuf; + let borrowed_cow_path: Cow<'_, Path> = path.into(); + + assert_eq!(static_cow_path, borrowed_cow_path); + } + + let owned_cow_path: Cow<'static, Path> = pathbuf.into(); + + assert_eq!(static_cow_path, owned_cow_path); +} + +#[test] +#[cfg(unix)] +pub fn test_decompositions_unix() { + t!("", + iter: [], + has_root: false, + is_absolute: false, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("foo", + iter: ["foo"], + has_root: false, + is_absolute: false, + parent: Some(""), + file_name: Some("foo"), + file_stem: Some("foo"), + extension: None, + file_prefix: Some("foo") + ); + + t!("/", + iter: ["/"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("/foo", + iter: ["/", "foo"], + has_root: true, + is_absolute: true, + parent: Some("/"), + file_name: Some("foo"), + file_stem: Some("foo"), + extension: None, + file_prefix: Some("foo") + ); + + t!("foo/", + iter: ["foo"], + has_root: false, + is_absolute: false, + parent: Some(""), + file_name: Some("foo"), + file_stem: Some("foo"), + extension: None, + file_prefix: Some("foo") + ); + + t!("/foo/", + iter: ["/", "foo"], + has_root: true, + is_absolute: true, + parent: Some("/"), + file_name: Some("foo"), + file_stem: Some("foo"), + extension: None, + file_prefix: Some("foo") + ); + + t!("foo/bar", + iter: ["foo", "bar"], + has_root: false, + is_absolute: false, + parent: Some("foo"), + file_name: Some("bar"), + file_stem: Some("bar"), + extension: None, + file_prefix: Some("bar") + ); + + t!("/foo/bar", + iter: ["/", "foo", "bar"], + has_root: true, + is_absolute: true, + parent: Some("/foo"), + file_name: Some("bar"), + file_stem: Some("bar"), + extension: None, + file_prefix: Some("bar") + ); + + t!("///foo///", + iter: ["/", "foo"], + has_root: true, + is_absolute: true, + parent: Some("/"), + file_name: Some("foo"), + file_stem: Some("foo"), + extension: None, + file_prefix: Some("foo") + ); + + t!("///foo///bar", + iter: ["/", "foo", "bar"], + has_root: true, + is_absolute: true, + parent: Some("///foo"), + file_name: Some("bar"), + file_stem: Some("bar"), + extension: None, + file_prefix: Some("bar") + ); + + t!("./.", + iter: ["."], + has_root: false, + is_absolute: false, + parent: Some(""), + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("/..", + iter: ["/", ".."], + has_root: true, + is_absolute: true, + parent: Some("/"), + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("../", + iter: [".."], + has_root: false, + is_absolute: false, + parent: Some(""), + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("foo/.", + iter: ["foo"], + has_root: false, + is_absolute: false, + parent: Some(""), + file_name: Some("foo"), + file_stem: Some("foo"), + extension: None, + file_prefix: Some("foo") + ); + + t!("foo/..", + iter: ["foo", ".."], + has_root: false, + is_absolute: false, + parent: Some("foo"), + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("foo/./", + iter: ["foo"], + has_root: false, + is_absolute: false, + parent: Some(""), + file_name: Some("foo"), + file_stem: Some("foo"), + extension: None, + file_prefix: Some("foo") + ); + + t!("foo/./bar", + iter: ["foo", "bar"], + has_root: false, + is_absolute: false, + parent: Some("foo"), + file_name: Some("bar"), + file_stem: Some("bar"), + extension: None, + file_prefix: Some("bar") + ); + + t!("foo/../", + iter: ["foo", ".."], + has_root: false, + is_absolute: false, + parent: Some("foo"), + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("foo/../bar", + iter: ["foo", "..", "bar"], + has_root: false, + is_absolute: false, + parent: Some("foo/.."), + file_name: Some("bar"), + file_stem: Some("bar"), + extension: None, + file_prefix: Some("bar") + ); + + t!("./a", + iter: [".", "a"], + has_root: false, + is_absolute: false, + parent: Some("."), + file_name: Some("a"), + file_stem: Some("a"), + extension: None, + file_prefix: Some("a") + ); + + t!(".", + iter: ["."], + has_root: false, + is_absolute: false, + parent: Some(""), + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("./", + iter: ["."], + has_root: false, + is_absolute: false, + parent: Some(""), + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("a/b", + iter: ["a", "b"], + has_root: false, + is_absolute: false, + parent: Some("a"), + file_name: Some("b"), + file_stem: Some("b"), + extension: None, + file_prefix: Some("b") + ); + + t!("a//b", + iter: ["a", "b"], + has_root: false, + is_absolute: false, + parent: Some("a"), + file_name: Some("b"), + file_stem: Some("b"), + extension: None, + file_prefix: Some("b") + ); + + t!("a/./b", + iter: ["a", "b"], + has_root: false, + is_absolute: false, + parent: Some("a"), + file_name: Some("b"), + file_stem: Some("b"), + extension: None, + file_prefix: Some("b") + ); + + t!("a/b/c", + iter: ["a", "b", "c"], + has_root: false, + is_absolute: false, + parent: Some("a/b"), + file_name: Some("c"), + file_stem: Some("c"), + extension: None, + file_prefix: Some("c") + ); + + t!(".foo", + iter: [".foo"], + has_root: false, + is_absolute: false, + parent: Some(""), + file_name: Some(".foo"), + file_stem: Some(".foo"), + extension: None, + file_prefix: Some(".foo") + ); + + t!("a/.foo", + iter: ["a", ".foo"], + has_root: false, + is_absolute: false, + parent: Some("a"), + file_name: Some(".foo"), + file_stem: Some(".foo"), + extension: None, + file_prefix: Some(".foo") + ); + + t!("a/.rustfmt.toml", + iter: ["a", ".rustfmt.toml"], + has_root: false, + is_absolute: false, + parent: Some("a"), + file_name: Some(".rustfmt.toml"), + file_stem: Some(".rustfmt"), + extension: Some("toml"), + file_prefix: Some(".rustfmt") + ); + + t!("a/.x.y.z", + iter: ["a", ".x.y.z"], + has_root: false, + is_absolute: false, + parent: Some("a"), + file_name: Some(".x.y.z"), + file_stem: Some(".x.y"), + extension: Some("z"), + file_prefix: Some(".x") + ); +} + +#[test] +#[cfg(windows)] +pub fn test_decompositions_windows() { + t!("", + iter: [], + has_root: false, + is_absolute: false, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("foo", + iter: ["foo"], + has_root: false, + is_absolute: false, + parent: Some(""), + file_name: Some("foo"), + file_stem: Some("foo"), + extension: None, + file_prefix: Some("foo") + ); + + t!("/", + iter: ["\\"], + has_root: true, + is_absolute: false, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("\\", + iter: ["\\"], + has_root: true, + is_absolute: false, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("c:", + iter: ["c:"], + has_root: false, + is_absolute: false, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("c:\\", + iter: ["c:", "\\"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("c:/", + iter: ["c:", "\\"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("/foo", + iter: ["\\", "foo"], + has_root: true, + is_absolute: false, + parent: Some("/"), + file_name: Some("foo"), + file_stem: Some("foo"), + extension: None, + file_prefix: Some("foo") + ); + + t!("foo/", + iter: ["foo"], + has_root: false, + is_absolute: false, + parent: Some(""), + file_name: Some("foo"), + file_stem: Some("foo"), + extension: None, + file_prefix: Some("foo") + ); + + t!("/foo/", + iter: ["\\", "foo"], + has_root: true, + is_absolute: false, + parent: Some("/"), + file_name: Some("foo"), + file_stem: Some("foo"), + extension: None, + file_prefix: Some("foo") + ); + + t!("foo/bar", + iter: ["foo", "bar"], + has_root: false, + is_absolute: false, + parent: Some("foo"), + file_name: Some("bar"), + file_stem: Some("bar"), + extension: None, + file_prefix: Some("bar") + ); + + t!("/foo/bar", + iter: ["\\", "foo", "bar"], + has_root: true, + is_absolute: false, + parent: Some("/foo"), + file_name: Some("bar"), + file_stem: Some("bar"), + extension: None, + file_prefix: Some("bar") + ); + + t!("///foo///", + iter: ["\\", "foo"], + has_root: true, + is_absolute: false, + parent: Some("/"), + file_name: Some("foo"), + file_stem: Some("foo"), + extension: None, + file_prefix: Some("foo") + ); + + t!("///foo///bar", + iter: ["\\", "foo", "bar"], + has_root: true, + is_absolute: false, + parent: Some("///foo"), + file_name: Some("bar"), + file_stem: Some("bar"), + extension: None, + file_prefix: Some("bar") + ); + + t!("./.", + iter: ["."], + has_root: false, + is_absolute: false, + parent: Some(""), + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("/..", + iter: ["\\", ".."], + has_root: true, + is_absolute: false, + parent: Some("/"), + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("../", + iter: [".."], + has_root: false, + is_absolute: false, + parent: Some(""), + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("foo/.", + iter: ["foo"], + has_root: false, + is_absolute: false, + parent: Some(""), + file_name: Some("foo"), + file_stem: Some("foo"), + extension: None, + file_prefix: Some("foo") + ); + + t!("foo/..", + iter: ["foo", ".."], + has_root: false, + is_absolute: false, + parent: Some("foo"), + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("foo/./", + iter: ["foo"], + has_root: false, + is_absolute: false, + parent: Some(""), + file_name: Some("foo"), + file_stem: Some("foo"), + extension: None, + file_prefix: Some("foo") + ); + + t!("foo/./bar", + iter: ["foo", "bar"], + has_root: false, + is_absolute: false, + parent: Some("foo"), + file_name: Some("bar"), + file_stem: Some("bar"), + extension: None, + file_prefix: Some("bar") + ); + + t!("foo/../", + iter: ["foo", ".."], + has_root: false, + is_absolute: false, + parent: Some("foo"), + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("foo/../bar", + iter: ["foo", "..", "bar"], + has_root: false, + is_absolute: false, + parent: Some("foo/.."), + file_name: Some("bar"), + file_stem: Some("bar"), + extension: None, + file_prefix: Some("bar") + ); + + t!("./a", + iter: [".", "a"], + has_root: false, + is_absolute: false, + parent: Some("."), + file_name: Some("a"), + file_stem: Some("a"), + extension: None, + file_prefix: Some("a") + ); + + t!(".", + iter: ["."], + has_root: false, + is_absolute: false, + parent: Some(""), + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("./", + iter: ["."], + has_root: false, + is_absolute: false, + parent: Some(""), + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("a/b", + iter: ["a", "b"], + has_root: false, + is_absolute: false, + parent: Some("a"), + file_name: Some("b"), + file_stem: Some("b"), + extension: None, + file_prefix: Some("b") + ); + + t!("a//b", + iter: ["a", "b"], + has_root: false, + is_absolute: false, + parent: Some("a"), + file_name: Some("b"), + file_stem: Some("b"), + extension: None, + file_prefix: Some("b") + ); + + t!("a/./b", + iter: ["a", "b"], + has_root: false, + is_absolute: false, + parent: Some("a"), + file_name: Some("b"), + file_stem: Some("b"), + extension: None, + file_prefix: Some("b") + ); + + t!("a/b/c", + iter: ["a", "b", "c"], + has_root: false, + is_absolute: false, + parent: Some("a/b"), + file_name: Some("c"), + file_stem: Some("c"), + extension: None, + file_prefix: Some("c") + ); + + t!("a\\b\\c", + iter: ["a", "b", "c"], + has_root: false, + is_absolute: false, + parent: Some("a\\b"), + file_name: Some("c"), + file_stem: Some("c"), + extension: None, + file_prefix: Some("c") + ); + + t!("\\a", + iter: ["\\", "a"], + has_root: true, + is_absolute: false, + parent: Some("\\"), + file_name: Some("a"), + file_stem: Some("a"), + extension: None, + file_prefix: Some("a") + ); + + t!("c:\\foo.txt", + iter: ["c:", "\\", "foo.txt"], + has_root: true, + is_absolute: true, + parent: Some("c:\\"), + file_name: Some("foo.txt"), + file_stem: Some("foo"), + extension: Some("txt"), + file_prefix: Some("foo") + ); + + t!("\\\\server\\share\\foo.txt", + iter: ["\\\\server\\share", "\\", "foo.txt"], + has_root: true, + is_absolute: true, + parent: Some("\\\\server\\share\\"), + file_name: Some("foo.txt"), + file_stem: Some("foo"), + extension: Some("txt"), + file_prefix: Some("foo") + ); + + t!("\\\\server\\share", + iter: ["\\\\server\\share", "\\"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("\\\\server", + iter: ["\\", "server"], + has_root: true, + is_absolute: false, + parent: Some("\\"), + file_name: Some("server"), + file_stem: Some("server"), + extension: None, + file_prefix: Some("server") + ); + + t!("\\\\?\\bar\\foo.txt", + iter: ["\\\\?\\bar", "\\", "foo.txt"], + has_root: true, + is_absolute: true, + parent: Some("\\\\?\\bar\\"), + file_name: Some("foo.txt"), + file_stem: Some("foo"), + extension: Some("txt"), + file_prefix: Some("foo") + ); + + t!("\\\\?\\bar", + iter: ["\\\\?\\bar"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("\\\\?\\", + iter: ["\\\\?\\"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("\\\\?\\UNC\\server\\share\\foo.txt", + iter: ["\\\\?\\UNC\\server\\share", "\\", "foo.txt"], + has_root: true, + is_absolute: true, + parent: Some("\\\\?\\UNC\\server\\share\\"), + file_name: Some("foo.txt"), + file_stem: Some("foo"), + extension: Some("txt"), + file_prefix: Some("foo") + ); + + t!("\\\\?\\UNC\\server", + iter: ["\\\\?\\UNC\\server"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("\\\\?\\UNC\\", + iter: ["\\\\?\\UNC\\"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("\\\\?\\C:\\foo.txt", + iter: ["\\\\?\\C:", "\\", "foo.txt"], + has_root: true, + is_absolute: true, + parent: Some("\\\\?\\C:\\"), + file_name: Some("foo.txt"), + file_stem: Some("foo"), + extension: Some("txt"), + file_prefix: Some("foo") + ); + + t!("\\\\?\\C:\\", + iter: ["\\\\?\\C:", "\\"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("\\\\?\\C:", + iter: ["\\\\?\\C:"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("\\\\?\\foo/bar", + iter: ["\\\\?\\foo/bar"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("\\\\?\\C:/foo/bar", + iter: ["\\\\?\\C:", "\\", "foo/bar"], + has_root: true, + is_absolute: true, + parent: Some("\\\\?\\C:/"), + file_name: Some("foo/bar"), + file_stem: Some("foo/bar"), + extension: None, + file_prefix: Some("foo/bar") + ); + + t!("\\\\.\\foo\\bar", + iter: ["\\\\.\\foo", "\\", "bar"], + has_root: true, + is_absolute: true, + parent: Some("\\\\.\\foo\\"), + file_name: Some("bar"), + file_stem: Some("bar"), + extension: None, + file_prefix: Some("bar") + ); + + t!("\\\\.\\foo", + iter: ["\\\\.\\foo", "\\"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("\\\\.\\foo/bar", + iter: ["\\\\.\\foo", "\\", "bar"], + has_root: true, + is_absolute: true, + parent: Some("\\\\.\\foo/"), + file_name: Some("bar"), + file_stem: Some("bar"), + extension: None, + file_prefix: Some("bar") + ); + + t!("\\\\.\\foo\\bar/baz", + iter: ["\\\\.\\foo", "\\", "bar", "baz"], + has_root: true, + is_absolute: true, + parent: Some("\\\\.\\foo\\bar"), + file_name: Some("baz"), + file_stem: Some("baz"), + extension: None, + file_prefix: Some("baz") + ); + + t!("\\\\.\\", + iter: ["\\\\.\\", "\\"], + has_root: true, + is_absolute: true, + parent: None, + file_name: None, + file_stem: None, + extension: None, + file_prefix: None + ); + + t!("\\\\?\\a\\b\\", + iter: ["\\\\?\\a", "\\", "b"], + has_root: true, + is_absolute: true, + parent: Some("\\\\?\\a\\"), + file_name: Some("b"), + file_stem: Some("b"), + extension: None, + file_prefix: Some("b") + ); + + t!("\\\\?\\C:\\foo.txt.zip", + iter: ["\\\\?\\C:", "\\", "foo.txt.zip"], + has_root: true, + is_absolute: true, + parent: Some("\\\\?\\C:\\"), + file_name: Some("foo.txt.zip"), + file_stem: Some("foo.txt"), + extension: Some("zip"), + file_prefix: Some("foo") + ); + + t!("\\\\?\\C:\\.foo.txt.zip", + iter: ["\\\\?\\C:", "\\", ".foo.txt.zip"], + has_root: true, + is_absolute: true, + parent: Some("\\\\?\\C:\\"), + file_name: Some(".foo.txt.zip"), + file_stem: Some(".foo.txt"), + extension: Some("zip"), + file_prefix: Some(".foo") + ); + + t!("\\\\?\\C:\\.foo", + iter: ["\\\\?\\C:", "\\", ".foo"], + has_root: true, + is_absolute: true, + parent: Some("\\\\?\\C:\\"), + file_name: Some(".foo"), + file_stem: Some(".foo"), + extension: None, + file_prefix: Some(".foo") + ); + + t!("a/.x.y.z", + iter: ["a", ".x.y.z"], + has_root: false, + is_absolute: false, + parent: Some("a"), + file_name: Some(".x.y.z"), + file_stem: Some(".x.y"), + extension: Some("z"), + file_prefix: Some(".x") + ); +} + +#[test] +pub fn test_stem_ext() { + t!("foo", + file_stem: Some("foo"), + extension: None + ); + + t!("foo.", + file_stem: Some("foo"), + extension: Some("") + ); + + t!(".foo", + file_stem: Some(".foo"), + extension: None + ); + + t!("foo.txt", + file_stem: Some("foo"), + extension: Some("txt") + ); + + t!("foo.bar.txt", + file_stem: Some("foo.bar"), + extension: Some("txt") + ); + + t!("foo.bar.", + file_stem: Some("foo.bar"), + extension: Some("") + ); + + t!(".", file_stem: None, extension: None); + + t!("..", file_stem: None, extension: None); + + t!(".x.y.z", file_stem: Some(".x.y"), extension: Some("z")); + + t!("..x.y.z", file_stem: Some("..x.y"), extension: Some("z")); + + t!("", file_stem: None, extension: None); +} + +#[test] +pub fn test_prefix_ext() { + t!("foo", + file_prefix: Some("foo"), + extension: None + ); + + t!("foo.", + file_prefix: Some("foo"), + extension: Some("") + ); + + t!(".foo", + file_prefix: Some(".foo"), + extension: None + ); + + t!("foo.txt", + file_prefix: Some("foo"), + extension: Some("txt") + ); + + t!("foo.bar.txt", + file_prefix: Some("foo"), + extension: Some("txt") + ); + + t!("foo.bar.", + file_prefix: Some("foo"), + extension: Some("") + ); + + t!(".", file_prefix: None, extension: None); + + t!("..", file_prefix: None, extension: None); + + t!(".x.y.z", file_prefix: Some(".x"), extension: Some("z")); + + t!("..x.y.z", file_prefix: Some("."), extension: Some("z")); + + t!("", file_prefix: None, extension: None); +} + +#[test] +pub fn test_push() { + macro_rules! tp ( + ($path:expr, $push:expr, $expected:expr) => ( { + let mut actual = PathBuf::from($path); + actual.push($push); + assert!(actual.to_str() == Some($expected), + "pushing {:?} onto {:?}: Expected {:?}, got {:?}", + $push, $path, $expected, actual.to_str().unwrap()); + }); + ); + + if cfg!(unix) || cfg!(all(target_env = "sgx", target_vendor = "fortanix")) { + tp!("", "foo", "foo"); + tp!("foo", "bar", "foo/bar"); + tp!("foo/", "bar", "foo/bar"); + tp!("foo//", "bar", "foo//bar"); + tp!("foo/.", "bar", "foo/./bar"); + tp!("foo./.", "bar", "foo././bar"); + tp!("foo", "", "foo/"); + tp!("foo", ".", "foo/."); + tp!("foo", "..", "foo/.."); + tp!("foo", "/", "/"); + tp!("/foo/bar", "/", "/"); + tp!("/foo/bar", "/baz", "/baz"); + tp!("/foo/bar", "./baz", "/foo/bar/./baz"); + } else { + tp!("", "foo", "foo"); + tp!("foo", "bar", r"foo\bar"); + tp!("foo/", "bar", r"foo/bar"); + tp!(r"foo\", "bar", r"foo\bar"); + tp!("foo//", "bar", r"foo//bar"); + tp!(r"foo\\", "bar", r"foo\\bar"); + tp!("foo/.", "bar", r"foo/.\bar"); + tp!("foo./.", "bar", r"foo./.\bar"); + tp!(r"foo\.", "bar", r"foo\.\bar"); + tp!(r"foo.\.", "bar", r"foo.\.\bar"); + tp!("foo", "", "foo\\"); + tp!("foo", ".", r"foo\."); + tp!("foo", "..", r"foo\.."); + tp!("foo", "/", "/"); + tp!("foo", r"\", r"\"); + tp!("/foo/bar", "/", "/"); + tp!(r"\foo\bar", r"\", r"\"); + tp!("/foo/bar", "/baz", "/baz"); + tp!("/foo/bar", r"\baz", r"\baz"); + tp!("/foo/bar", "./baz", r"/foo/bar\./baz"); + tp!("/foo/bar", r".\baz", r"/foo/bar\.\baz"); + + tp!("c:\\", "windows", "c:\\windows"); + tp!("c:", "windows", "c:windows"); + + tp!("a\\b\\c", "d", "a\\b\\c\\d"); + tp!("\\a\\b\\c", "d", "\\a\\b\\c\\d"); + tp!("a\\b", "c\\d", "a\\b\\c\\d"); + tp!("a\\b", "\\c\\d", "\\c\\d"); + tp!("a\\b", ".", "a\\b\\."); + tp!("a\\b", "..\\c", "a\\b\\..\\c"); + tp!("a\\b", "C:a.txt", "C:a.txt"); + tp!("a\\b", "C:\\a.txt", "C:\\a.txt"); + tp!("C:\\a", "C:\\b.txt", "C:\\b.txt"); + tp!("C:\\a\\b\\c", "C:d", "C:d"); + tp!("C:a\\b\\c", "C:d", "C:d"); + tp!("C:", r"a\b\c", r"C:a\b\c"); + tp!("C:", r"..\a", r"C:..\a"); + tp!("\\\\server\\share\\foo", "bar", "\\\\server\\share\\foo\\bar"); + tp!("\\\\server\\share\\foo", "C:baz", "C:baz"); + tp!("\\\\?\\C:\\a\\b", "C:c\\d", "C:c\\d"); + tp!("\\\\?\\C:a\\b", "C:c\\d", "C:c\\d"); + tp!("\\\\?\\C:\\a\\b", "C:\\c\\d", "C:\\c\\d"); + tp!("\\\\?\\foo\\bar", "baz", "\\\\?\\foo\\bar\\baz"); + tp!("\\\\?\\UNC\\server\\share\\foo", "bar", "\\\\?\\UNC\\server\\share\\foo\\bar"); + tp!("\\\\?\\UNC\\server\\share", "C:\\a", "C:\\a"); + tp!("\\\\?\\UNC\\server\\share", "C:a", "C:a"); + + // Note: modified from old path API + tp!("\\\\?\\UNC\\server", "foo", "\\\\?\\UNC\\server\\foo"); + + tp!("C:\\a", "\\\\?\\UNC\\server\\share", "\\\\?\\UNC\\server\\share"); + tp!("\\\\.\\foo\\bar", "baz", "\\\\.\\foo\\bar\\baz"); + tp!("\\\\.\\foo\\bar", "C:a", "C:a"); + // again, not sure about the following, but I'm assuming \\.\ should be verbatim + tp!("\\\\.\\foo", "..\\bar", "\\\\.\\foo\\..\\bar"); + + tp!("\\\\?\\C:", "foo", "\\\\?\\C:\\foo"); // this is a weird one + + tp!(r"\\?\C:\bar", "../foo", r"\\?\C:\foo"); + tp!(r"\\?\C:\bar", "../../foo", r"\\?\C:\foo"); + tp!(r"\\?\C:\", "../foo", r"\\?\C:\foo"); + tp!(r"\\?\C:", r"D:\foo/./", r"D:\foo/./"); + tp!(r"\\?\C:", r"\\?\D:\foo\.\", r"\\?\D:\foo\.\"); + tp!(r"\\?\A:\x\y", "/foo", r"\\?\A:\foo"); + tp!(r"\\?\A:", r"..\foo\.", r"\\?\A:\foo"); + tp!(r"\\?\A:\x\y", r".\foo\.", r"\\?\A:\x\y\foo"); + tp!(r"\\?\A:\x\y", r"", r"\\?\A:\x\y\"); + } +} + +#[test] +pub fn test_pop() { + macro_rules! tp ( + ($path:expr, $expected:expr, $output:expr) => ( { + let mut actual = PathBuf::from($path); + let output = actual.pop(); + assert!(actual.to_str() == Some($expected) && output == $output, + "popping from {:?}: Expected {:?}/{:?}, got {:?}/{:?}", + $path, $expected, $output, + actual.to_str().unwrap(), output); + }); + ); + + tp!("", "", false); + tp!("/", "/", false); + tp!("foo", "", true); + tp!(".", "", true); + tp!("/foo", "/", true); + tp!("/foo/bar", "/foo", true); + tp!("foo/bar", "foo", true); + tp!("foo/.", "", true); + tp!("foo//bar", "foo", true); + + if cfg!(windows) { + tp!("a\\b\\c", "a\\b", true); + tp!("\\a", "\\", true); + tp!("\\", "\\", false); + + tp!("C:\\a\\b", "C:\\a", true); + tp!("C:\\a", "C:\\", true); + tp!("C:\\", "C:\\", false); + tp!("C:a\\b", "C:a", true); + tp!("C:a", "C:", true); + tp!("C:", "C:", false); + tp!("\\\\server\\share\\a\\b", "\\\\server\\share\\a", true); + tp!("\\\\server\\share\\a", "\\\\server\\share\\", true); + tp!("\\\\server\\share", "\\\\server\\share", false); + tp!("\\\\?\\a\\b\\c", "\\\\?\\a\\b", true); + tp!("\\\\?\\a\\b", "\\\\?\\a\\", true); + tp!("\\\\?\\a", "\\\\?\\a", false); + tp!("\\\\?\\C:\\a\\b", "\\\\?\\C:\\a", true); + tp!("\\\\?\\C:\\a", "\\\\?\\C:\\", true); + tp!("\\\\?\\C:\\", "\\\\?\\C:\\", false); + tp!("\\\\?\\UNC\\server\\share\\a\\b", "\\\\?\\UNC\\server\\share\\a", true); + tp!("\\\\?\\UNC\\server\\share\\a", "\\\\?\\UNC\\server\\share\\", true); + tp!("\\\\?\\UNC\\server\\share", "\\\\?\\UNC\\server\\share", false); + tp!("\\\\.\\a\\b\\c", "\\\\.\\a\\b", true); + tp!("\\\\.\\a\\b", "\\\\.\\a\\", true); + tp!("\\\\.\\a", "\\\\.\\a", false); + + tp!("\\\\?\\a\\b\\", "\\\\?\\a\\", true); + } +} + +#[test] +pub fn test_set_file_name() { + macro_rules! tfn ( + ($path:expr, $file:expr, $expected:expr) => ( { + let mut p = PathBuf::from($path); + p.set_file_name($file); + assert!(p.to_str() == Some($expected), + "setting file name of {:?} to {:?}: Expected {:?}, got {:?}", + $path, $file, $expected, + p.to_str().unwrap()); + }); + ); + + tfn!("foo", "foo", "foo"); + tfn!("foo", "bar", "bar"); + tfn!("foo", "", ""); + tfn!("", "foo", "foo"); + if cfg!(unix) || cfg!(all(target_env = "sgx", target_vendor = "fortanix")) { + tfn!(".", "foo", "./foo"); + tfn!("foo/", "bar", "bar"); + tfn!("foo/.", "bar", "bar"); + tfn!("..", "foo", "../foo"); + tfn!("foo/..", "bar", "foo/../bar"); + tfn!("/", "foo", "/foo"); + } else { + tfn!(".", "foo", r".\foo"); + tfn!(r"foo\", "bar", r"bar"); + tfn!(r"foo\.", "bar", r"bar"); + tfn!("..", "foo", r"..\foo"); + tfn!(r"foo\..", "bar", r"foo\..\bar"); + tfn!(r"\", "foo", r"\foo"); + } +} + +#[test] +pub fn test_set_extension() { + macro_rules! tfe ( + ($path:expr, $ext:expr, $expected:expr, $output:expr) => ( { + let mut p = PathBuf::from($path); + let output = p.set_extension($ext); + assert!(p.to_str() == Some($expected) && output == $output, + "setting extension of {:?} to {:?}: Expected {:?}/{:?}, got {:?}/{:?}", + $path, $ext, $expected, $output, + p.to_str().unwrap(), output); + }); + ); + + tfe!("foo", "txt", "foo.txt", true); + tfe!("foo.bar", "txt", "foo.txt", true); + tfe!("foo.bar.baz", "txt", "foo.bar.txt", true); + tfe!(".test", "txt", ".test.txt", true); + tfe!("foo.txt", "", "foo", true); + tfe!("foo", "", "foo", true); + tfe!("", "foo", "", false); + tfe!(".", "foo", ".", false); + tfe!("foo/", "bar", "foo.bar", true); + tfe!("foo/.", "bar", "foo.bar", true); + tfe!("..", "foo", "..", false); + tfe!("foo/..", "bar", "foo/..", false); + tfe!("/", "foo", "/", false); +} + +#[test] +fn test_eq_receivers() { + use crate::borrow::Cow; + + let borrowed: &Path = Path::new("foo/bar"); + let mut owned: PathBuf = PathBuf::new(); + owned.push("foo"); + owned.push("bar"); + let borrowed_cow: Cow<'_, Path> = borrowed.into(); + let owned_cow: Cow<'_, Path> = owned.clone().into(); + + macro_rules! t { + ($($current:expr),+) => { + $( + assert_eq!($current, borrowed); + assert_eq!($current, owned); + assert_eq!($current, borrowed_cow); + assert_eq!($current, owned_cow); + )+ + } + } + + t!(borrowed, owned, borrowed_cow, owned_cow); +} + +#[test] +pub fn test_compare() { + use crate::collections::hash_map::DefaultHasher; + use crate::hash::{Hash, Hasher}; + + fn hash(t: T) -> u64 { + let mut s = DefaultHasher::new(); + t.hash(&mut s); + s.finish() + } + + macro_rules! tc ( + ($path1:expr, $path2:expr, eq: $eq:expr, + starts_with: $starts_with:expr, ends_with: $ends_with:expr, + relative_from: $relative_from:expr) => ({ + let path1 = Path::new($path1); + let path2 = Path::new($path2); + + let eq = path1 == path2; + assert!(eq == $eq, "{:?} == {:?}, expected {:?}, got {:?}", + $path1, $path2, $eq, eq); + assert!($eq == (hash(path1) == hash(path2)), + "{:?} == {:?}, expected {:?}, got {} and {}", + $path1, $path2, $eq, hash(path1), hash(path2)); + + let starts_with = path1.starts_with(path2); + assert!(starts_with == $starts_with, + "{:?}.starts_with({:?}), expected {:?}, got {:?}", $path1, $path2, + $starts_with, starts_with); + + let ends_with = path1.ends_with(path2); + assert!(ends_with == $ends_with, + "{:?}.ends_with({:?}), expected {:?}, got {:?}", $path1, $path2, + $ends_with, ends_with); + + let relative_from = path1.strip_prefix(path2) + .map(|p| p.to_str().unwrap()) + .ok(); + let exp: Option<&str> = $relative_from; + assert!(relative_from == exp, + "{:?}.strip_prefix({:?}), expected {:?}, got {:?}", + $path1, $path2, exp, relative_from); + }); + ); + + tc!("", "", + eq: true, + starts_with: true, + ends_with: true, + relative_from: Some("") + ); + + tc!("foo", "", + eq: false, + starts_with: true, + ends_with: true, + relative_from: Some("foo") + ); + + tc!("", "foo", + eq: false, + starts_with: false, + ends_with: false, + relative_from: None + ); + + tc!("foo", "foo", + eq: true, + starts_with: true, + ends_with: true, + relative_from: Some("") + ); + + tc!("foo/", "foo", + eq: true, + starts_with: true, + ends_with: true, + relative_from: Some("") + ); + + tc!("foo/.", "foo", + eq: true, + starts_with: true, + ends_with: true, + relative_from: Some("") + ); + + tc!("foo/./bar", "foo/bar", + eq: true, + starts_with: true, + ends_with: true, + relative_from: Some("") + ); + + tc!("foo/bar", "foo", + eq: false, + starts_with: true, + ends_with: false, + relative_from: Some("bar") + ); + + tc!("foo/bar/baz", "foo/bar", + eq: false, + starts_with: true, + ends_with: false, + relative_from: Some("baz") + ); + + tc!("foo/bar", "foo/bar/baz", + eq: false, + starts_with: false, + ends_with: false, + relative_from: None + ); + + tc!("./foo/bar/", ".", + eq: false, + starts_with: true, + ends_with: false, + relative_from: Some("foo/bar") + ); + + if cfg!(windows) { + tc!(r"C:\src\rust\cargo-test\test\Cargo.toml", + r"c:\src\rust\cargo-test\test", + eq: false, + starts_with: true, + ends_with: false, + relative_from: Some("Cargo.toml") + ); + + tc!(r"c:\foo", r"C:\foo", + eq: true, + starts_with: true, + ends_with: true, + relative_from: Some("") + ); + + tc!(r"C:\foo\.\bar.txt", r"C:\foo\bar.txt", + eq: true, + starts_with: true, + ends_with: true, + relative_from: Some("") + ); + + tc!(r"C:\foo\.", r"C:\foo", + eq: true, + starts_with: true, + ends_with: true, + relative_from: Some("") + ); + + tc!(r"\\?\C:\foo\.\bar.txt", r"\\?\C:\foo\bar.txt", + eq: false, + starts_with: false, + ends_with: false, + relative_from: None + ); + } +} + +#[test] +fn test_components_debug() { + let path = Path::new("/tmp"); + + let mut components = path.components(); + + let expected = "Components([RootDir, Normal(\"tmp\")])"; + let actual = format!("{components:?}"); + assert_eq!(expected, actual); + + let _ = components.next().unwrap(); + let expected = "Components([Normal(\"tmp\")])"; + let actual = format!("{components:?}"); + assert_eq!(expected, actual); + + let _ = components.next().unwrap(); + let expected = "Components([])"; + let actual = format!("{components:?}"); + assert_eq!(expected, actual); +} + +#[cfg(unix)] +#[test] +fn test_iter_debug() { + let path = Path::new("/tmp"); + + let mut iter = path.iter(); + + let expected = "Iter([\"/\", \"tmp\"])"; + let actual = format!("{iter:?}"); + assert_eq!(expected, actual); + + let _ = iter.next().unwrap(); + let expected = "Iter([\"tmp\"])"; + let actual = format!("{iter:?}"); + assert_eq!(expected, actual); + + let _ = iter.next().unwrap(); + let expected = "Iter([])"; + let actual = format!("{iter:?}"); + assert_eq!(expected, actual); +} + +#[test] +fn into_boxed() { + let orig: &str = "some/sort/of/path"; + let path = Path::new(orig); + let boxed: Box = Box::from(path); + let path_buf = path.to_owned().into_boxed_path().into_path_buf(); + assert_eq!(path, &*boxed); + assert_eq!(&*boxed, &*path_buf); + assert_eq!(&*path_buf, path); +} + +#[test] +fn test_clone_into() { + let mut path_buf = PathBuf::from("supercalifragilisticexpialidocious"); + let path = Path::new("short"); + path.clone_into(&mut path_buf); + assert_eq!(path, path_buf); + assert!(path_buf.into_os_string().capacity() >= 15); +} + +#[test] +fn display_format_flags() { + assert_eq!(format!("a{:#<5}b", Path::new("").display()), "a#####b"); + assert_eq!(format!("a{:#<5}b", Path::new("a").display()), "aa####b"); +} + +#[test] +fn into_rc() { + let orig = "hello/world"; + let path = Path::new(orig); + let rc: Rc = Rc::from(path); + let arc: Arc = Arc::from(path); + + assert_eq!(&*rc, path); + assert_eq!(&*arc, path); + + let rc2: Rc = Rc::from(path.to_owned()); + let arc2: Arc = Arc::from(path.to_owned()); + + assert_eq!(&*rc2, path); + assert_eq!(&*arc2, path); +} + +#[test] +fn test_ord() { + macro_rules! ord( + ($ord:ident, $left:expr, $right:expr) => ( { + use core::cmp::Ordering; + + let left = Path::new($left); + let right = Path::new($right); + assert_eq!(left.cmp(&right), Ordering::$ord); + if (core::cmp::Ordering::$ord == Ordering::Equal) { + assert_eq!(left, right); + + let mut hasher = DefaultHasher::new(); + left.hash(&mut hasher); + let left_hash = hasher.finish(); + hasher = DefaultHasher::new(); + right.hash(&mut hasher); + let right_hash = hasher.finish(); + + assert_eq!(left_hash, right_hash, "hashes for {:?} and {:?} must match", left, right); + } else { + assert_ne!(left, right); + } + }); + ); + + ord!(Less, "1", "2"); + ord!(Less, "/foo/bar", "/foo./bar"); + ord!(Less, "foo/bar", "foo/bar."); + ord!(Equal, "foo/./bar", "foo/bar/"); + ord!(Equal, "foo/bar", "foo/bar/"); + ord!(Equal, "foo/bar", "foo/bar/."); + ord!(Equal, "foo/bar", "foo/bar//"); +} + +#[test] +#[cfg(unix)] +fn test_unix_absolute() { + use crate::path::absolute; + + assert!(absolute("").is_err()); + + let relative = "a/b"; + let mut expected = crate::env::current_dir().unwrap(); + expected.push(relative); + assert_eq!(absolute(relative).unwrap().as_os_str(), expected.as_os_str()); + + // Test how components are collected. + assert_eq!(absolute("/a/b/c").unwrap().as_os_str(), Path::new("/a/b/c").as_os_str()); + assert_eq!(absolute("/a//b/c").unwrap().as_os_str(), Path::new("/a/b/c").as_os_str()); + assert_eq!(absolute("//a/b/c").unwrap().as_os_str(), Path::new("//a/b/c").as_os_str()); + assert_eq!(absolute("///a/b/c").unwrap().as_os_str(), Path::new("/a/b/c").as_os_str()); + assert_eq!(absolute("/a/b/c/").unwrap().as_os_str(), Path::new("/a/b/c/").as_os_str()); + assert_eq!( + absolute("/a/./b/../c/.././..").unwrap().as_os_str(), + Path::new("/a/b/../c/../..").as_os_str() + ); + + // Test leading `.` and `..` components + let curdir = crate::env::current_dir().unwrap(); + assert_eq!(absolute("./a").unwrap().as_os_str(), curdir.join("a").as_os_str()); + assert_eq!(absolute("../a").unwrap().as_os_str(), curdir.join("../a").as_os_str()); // return /pwd/../a +} + +#[test] +#[cfg(windows)] +fn test_windows_absolute() { + use crate::path::absolute; + // An empty path is an error. + assert!(absolute("").is_err()); + + let relative = r"a\b"; + let mut expected = crate::env::current_dir().unwrap(); + expected.push(relative); + assert_eq!(absolute(relative).unwrap().as_os_str(), expected.as_os_str()); + + macro_rules! unchanged( + ($path:expr) => { + assert_eq!(absolute($path).unwrap().as_os_str(), Path::new($path).as_os_str()); + } + ); + + unchanged!(r"C:\path\to\file"); + unchanged!(r"C:\path\to\file\"); + unchanged!(r"\\server\share\to\file"); + unchanged!(r"\\server.\share.\to\file"); + unchanged!(r"\\.\PIPE\name"); + unchanged!(r"\\.\C:\path\to\COM1"); + unchanged!(r"\\?\C:\path\to\file"); + unchanged!(r"\\?\UNC\server\share\to\file"); + unchanged!(r"\\?\PIPE\name"); + // Verbatim paths are always unchanged, no matter what. + unchanged!(r"\\?\path.\to/file.."); + + assert_eq!( + absolute(r"C:\path..\to.\file.").unwrap().as_os_str(), + Path::new(r"C:\path..\to\file").as_os_str() + ); + assert_eq!(absolute(r"COM1").unwrap().as_os_str(), Path::new(r"\\.\COM1").as_os_str()); +} + +#[bench] +#[cfg_attr(miri, ignore)] // Miri isn't fast... +fn bench_path_cmp_fast_path_buf_sort(b: &mut test::Bencher) { + let prefix = "my/home"; + let mut paths: Vec<_> = + (0..1000).map(|num| PathBuf::from(prefix).join(format!("file {num}.rs"))).collect(); + + paths.sort(); + + b.iter(|| { + black_box(paths.as_mut_slice()).sort_unstable(); + }); +} + +#[bench] +#[cfg_attr(miri, ignore)] // Miri isn't fast... +fn bench_path_cmp_fast_path_long(b: &mut test::Bencher) { + let prefix = "/my/home/is/my/castle/and/my/castle/has/a/rusty/workbench/"; + let paths: Vec<_> = + (0..1000).map(|num| PathBuf::from(prefix).join(format!("file {num}.rs"))).collect(); + + let mut set = BTreeSet::new(); + + paths.iter().for_each(|p| { + set.insert(p.as_path()); + }); + + b.iter(|| { + set.remove(paths[500].as_path()); + set.insert(paths[500].as_path()); + }); +} + +#[bench] +#[cfg_attr(miri, ignore)] // Miri isn't fast... +fn bench_path_cmp_fast_path_short(b: &mut test::Bencher) { + let prefix = "my/home"; + let paths: Vec<_> = + (0..1000).map(|num| PathBuf::from(prefix).join(format!("file {num}.rs"))).collect(); + + let mut set = BTreeSet::new(); + + paths.iter().for_each(|p| { + set.insert(p.as_path()); + }); + + b.iter(|| { + set.remove(paths[500].as_path()); + set.insert(paths[500].as_path()); + }); +} + +#[bench] +#[cfg_attr(miri, ignore)] // Miri isn't fast... +fn bench_path_hashset(b: &mut test::Bencher) { + let prefix = "/my/home/is/my/castle/and/my/castle/has/a/rusty/workbench/"; + let paths: Vec<_> = + (0..1000).map(|num| PathBuf::from(prefix).join(format!("file {num}.rs"))).collect(); + + let mut set = HashSet::new(); + + paths.iter().for_each(|p| { + set.insert(p.as_path()); + }); + + b.iter(|| { + set.remove(paths[500].as_path()); + set.insert(black_box(paths[500].as_path())) + }); +} + +#[bench] +#[cfg_attr(miri, ignore)] // Miri isn't fast... +fn bench_path_hashset_miss(b: &mut test::Bencher) { + let prefix = "/my/home/is/my/castle/and/my/castle/has/a/rusty/workbench/"; + let paths: Vec<_> = + (0..1000).map(|num| PathBuf::from(prefix).join(format!("file {num}.rs"))).collect(); + + let mut set = HashSet::new(); + + paths.iter().for_each(|p| { + set.insert(p.as_path()); + }); + + let probe = PathBuf::from(prefix).join("other"); + + b.iter(|| set.remove(black_box(probe.as_path()))); +} + +#[bench] +fn bench_hash_path_short(b: &mut test::Bencher) { + let mut hasher = DefaultHasher::new(); + let path = Path::new("explorer.exe"); + + b.iter(|| black_box(path).hash(&mut hasher)); + + black_box(hasher.finish()); +} + +#[bench] +fn bench_hash_path_long(b: &mut test::Bencher) { + let mut hasher = DefaultHasher::new(); + let path = + Path::new("/aaaaa/aaaaaa/./../aaaaaaaa/bbbbbbbbbbbbb/ccccccccccc/ddddddddd/eeeeeee.fff"); + + b.iter(|| black_box(path).hash(&mut hasher)); + + black_box(hasher.finish()); +} diff --git a/crux-mir/lib/std/src/personality.rs b/crux-mir/lib/std/src/personality.rs new file mode 100644 index 000000000..63f0ad4f1 --- /dev/null +++ b/crux-mir/lib/std/src/personality.rs @@ -0,0 +1,46 @@ +//! This module contains the implementation of the `eh_personality` lang item. +//! +//! The actual implementation is heavily dependent on the target since Rust +//! tries to use the native stack unwinding mechanism whenever possible. +//! +//! This personality function is still required with `-C panic=abort` because +//! it is used to catch foreign exceptions from `extern "C-unwind"` and turn +//! them into aborts. +//! +//! Additionally, ARM EHABI uses the personality function when generating +//! backtraces. + +mod dwarf; + +#[cfg(not(test))] +cfg_if::cfg_if! { + if #[cfg(target_os = "emscripten")] { + mod emcc; + } else if #[cfg(target_env = "msvc")] { + // This is required by the compiler to exist (e.g., it's a lang item), + // but it's never actually called by the compiler because + // _CxxFrameHandler3 is the personality function that is always used. + // Hence this is just an aborting stub. + #[lang = "eh_personality"] + fn rust_eh_personality() { + core::intrinsics::abort() + } + } else if #[cfg(any( + all(target_family = "windows", target_env = "gnu"), + target_os = "psp", + target_os = "solid_asp3", + all(target_family = "unix", not(target_os = "espidf")), + all(target_vendor = "fortanix", target_env = "sgx"), + ))] { + mod gcc; + } else { + // Targets that don't support unwinding. + // - family=wasm + // - os=none ("bare metal" targets) + // - os=uefi + // - os=espidf + // - os=hermit + // - nvptx64-nvidia-cuda + // - arch=avr + } +} diff --git a/crux-mir/lib/panic_unwind/dwarf/eh.rs b/crux-mir/lib/std/src/personality/dwarf/eh.rs similarity index 74% rename from crux-mir/lib/panic_unwind/dwarf/eh.rs rename to crux-mir/lib/std/src/personality/dwarf/eh.rs index 302478cfa..87585a8fc 100644 --- a/crux-mir/lib/panic_unwind/dwarf/eh.rs +++ b/crux-mir/lib/std/src/personality/dwarf/eh.rs @@ -1,9 +1,9 @@ //! Parsing of GCC-style Language-Specific Data Area (LSDA) //! For details see: -//! http://refspecs.linuxfoundation.org/LSB_3.0.0/LSB-PDA/LSB-PDA/ehframechpt.html -//! http://mentorembedded.github.io/cxx-abi/exceptions.pdf -//! http://www.airs.com/blog/archives/460 -//! http://www.airs.com/blog/archives/464 +//! * +//! * +//! * +//! * //! //! A reference implementation may be found in the GCC source tree //! (`/libgcc/unwind-c.c` as of this writing). @@ -11,8 +11,9 @@ #![allow(non_upper_case_globals)] #![allow(unused)] -use crate::dwarf::DwarfReader; +use super::DwarfReader; use core::mem; +use core::ptr; pub const DW_EH_PE_omit: u8 = 0xFF; pub const DW_EH_PE_absptr: u8 = 0x00; @@ -51,11 +52,7 @@ pub enum EHAction { pub const USING_SJLJ_EXCEPTIONS: bool = cfg!(all(target_os = "ios", target_arch = "arm")); -pub unsafe fn find_eh_action( - lsda: *const u8, - context: &EHContext<'_>, - foreign_exception: bool, -) -> Result { +pub unsafe fn find_eh_action(lsda: *const u8, context: &EHContext<'_>) -> Result { if lsda.is_null() { return Ok(EHAction::None); } @@ -79,7 +76,7 @@ pub unsafe fn find_eh_action( let call_site_encoding = reader.read::(); let call_site_table_length = reader.read_uleb128(); - let action_table = reader.ptr.offset(call_site_table_length as isize); + let action_table = reader.ptr.add(call_site_table_length as usize); let ip = context.ip; if !USING_SJLJ_EXCEPTIONS { @@ -87,7 +84,7 @@ pub unsafe fn find_eh_action( let cs_start = read_encoded_pointer(&mut reader, context, call_site_encoding)?; let cs_len = read_encoded_pointer(&mut reader, context, call_site_encoding)?; let cs_lpad = read_encoded_pointer(&mut reader, context, call_site_encoding)?; - let cs_action = reader.read_uleb128(); + let cs_action_entry = reader.read_uleb128(); // Callsite table is sorted by cs_start, so if we've passed the ip, we // may stop searching. if ip < func_start + cs_start { @@ -98,13 +95,12 @@ pub unsafe fn find_eh_action( return Ok(EHAction::None); } else { let lpad = lpad_base + cs_lpad; - return Ok(interpret_cs_action(cs_action, lpad, foreign_exception)); + return Ok(interpret_cs_action(action_table as *mut u8, cs_action_entry, lpad)); } } } - // Ip is not present in the table. This should not happen... but it does: issue #35011. - // So rather than returning EHAction::Terminate, we do this. - Ok(EHAction::None) + // Ip is not present in the table. This indicates a nounwind call. + Ok(EHAction::Terminate) } else { // SjLj version: // The "IP" is an index into the call-site table, with two exceptions: @@ -117,30 +113,39 @@ pub unsafe fn find_eh_action( let mut idx = ip; loop { let cs_lpad = reader.read_uleb128(); - let cs_action = reader.read_uleb128(); + let cs_action_entry = reader.read_uleb128(); idx -= 1; if idx == 0 { // Can never have null landing pad for sjlj -- that would have // been indicated by a -1 call site index. let lpad = (cs_lpad + 1) as usize; - return Ok(interpret_cs_action(cs_action, lpad, foreign_exception)); + return Ok(interpret_cs_action(action_table as *mut u8, cs_action_entry, lpad)); } } } } -fn interpret_cs_action(cs_action: u64, lpad: usize, foreign_exception: bool) -> EHAction { - if cs_action == 0 { - // If cs_action is 0 then this is a cleanup (Drop::drop). We run these +unsafe fn interpret_cs_action( + action_table: *mut u8, + cs_action_entry: u64, + lpad: usize, +) -> EHAction { + if cs_action_entry == 0 { + // If cs_action_entry is 0 then this is a cleanup (Drop::drop). We run these // for both Rust panics and foreign exceptions. EHAction::Cleanup(lpad) - } else if foreign_exception { - // catch_unwind should not catch foreign exceptions, only Rust panics. - // Instead just continue unwinding. - EHAction::None } else { - // Stop unwinding Rust panics at catch_unwind. - EHAction::Catch(lpad) + // If lpad != 0 and cs_action_entry != 0, we have to check ttype_index. + // If ttype_index == 0 under the condition, we take cleanup action. + let action_record = (action_table as *mut u8).offset(cs_action_entry as isize - 1); + let mut action_reader = DwarfReader::new(action_record); + let ttype_index = action_reader.read_sleb128(); + if ttype_index == 0 { + EHAction::Cleanup(lpad) + } else { + // Stop unwinding Rust panics at catch_unwind. + EHAction::Catch(lpad) + } } } @@ -160,7 +165,7 @@ unsafe fn read_encoded_pointer( // DW_EH_PE_aligned implies it's an absolute pointer value if encoding == DW_EH_PE_aligned { - reader.ptr = round_up(reader.ptr as usize, mem::size_of::())? as *const u8; + reader.ptr = reader.ptr.with_addr(round_up(reader.ptr.addr(), mem::size_of::())?); return Ok(reader.read::()); } @@ -180,7 +185,7 @@ unsafe fn read_encoded_pointer( result += match encoding & 0x70 { DW_EH_PE_absptr => 0, // relative to address of the encoded value, despite the name - DW_EH_PE_pcrel => reader.ptr as usize, + DW_EH_PE_pcrel => reader.ptr.expose_addr(), DW_EH_PE_funcrel => { if context.func_start == 0 { return Err(()); @@ -193,7 +198,7 @@ unsafe fn read_encoded_pointer( }; if encoding & DW_EH_PE_indirect != 0 { - result = *(result as *const usize); + result = *ptr::from_exposed_addr::(result); } Ok(result) diff --git a/crux-mir/lib/panic_unwind/dwarf/mod.rs b/crux-mir/lib/std/src/personality/dwarf/mod.rs similarity index 94% rename from crux-mir/lib/panic_unwind/dwarf/mod.rs rename to crux-mir/lib/std/src/personality/dwarf/mod.rs index 649bbce52..652fbe95a 100644 --- a/crux-mir/lib/panic_unwind/dwarf/mod.rs +++ b/crux-mir/lib/std/src/personality/dwarf/mod.rs @@ -53,7 +53,7 @@ impl DwarfReader { } pub unsafe fn read_sleb128(&mut self) -> i64 { - let mut shift: usize = 0; + let mut shift: u32 = 0; let mut result: u64 = 0; let mut byte: u8; loop { @@ -65,7 +65,7 @@ impl DwarfReader { } } // sign-extend - if shift < 8 * mem::size_of::() && (byte & 0x40) != 0 { + if shift < u64::BITS && (byte & 0x40) != 0 { result |= (!0 as u64) << shift; } result as i64 diff --git a/crux-mir/lib/panic_unwind/dwarf/tests.rs b/crux-mir/lib/std/src/personality/dwarf/tests.rs similarity index 100% rename from crux-mir/lib/panic_unwind/dwarf/tests.rs rename to crux-mir/lib/std/src/personality/dwarf/tests.rs diff --git a/crux-mir/lib/std/src/personality/emcc.rs b/crux-mir/lib/std/src/personality/emcc.rs new file mode 100644 index 000000000..f942bdf18 --- /dev/null +++ b/crux-mir/lib/std/src/personality/emcc.rs @@ -0,0 +1,20 @@ +//! On Emscripten Rust panics are wrapped in C++ exceptions, so we just forward +//! to `__gxx_personality_v0` which is provided by Emscripten. + +use libc::c_int; +use unwind as uw; + +// This is required by the compiler to exist (e.g., it's a lang item), but it's +// never actually called by the compiler. Emscripten EH doesn't use a +// personality function at all, it instead uses __cxa_find_matching_catch. +// Wasm error handling would use __gxx_personality_wasm0. +#[lang = "eh_personality"] +unsafe extern "C" fn rust_eh_personality( + _version: c_int, + _actions: uw::_Unwind_Action, + _exception_class: uw::_Unwind_Exception_Class, + _exception_object: *mut uw::_Unwind_Exception, + _context: *mut uw::_Unwind_Context, +) -> uw::_Unwind_Reason_Code { + core::intrinsics::abort() +} diff --git a/crux-mir/lib/panic_unwind/gcc.rs b/crux-mir/lib/std/src/personality/gcc.rs similarity index 52% rename from crux-mir/lib/panic_unwind/gcc.rs rename to crux-mir/lib/std/src/personality/gcc.rs index 1622442a5..5fc1b91a1 100644 --- a/crux-mir/lib/panic_unwind/gcc.rs +++ b/crux-mir/lib/std/src/personality/gcc.rs @@ -4,9 +4,9 @@ //! "Exception Handling in LLVM" (llvm.org/docs/ExceptionHandling.html) and //! documents linked from it. //! These are also good reads: -//! https://itanium-cxx-abi.github.io/cxx-abi/abi-eh.html -//! http://monoinfinito.wordpress.com/series/exception-handling-in-c/ -//! http://www.airs.com/blog/index.php?s=exception+frames +//! * +//! * +//! * //! //! ## A brief summary //! @@ -36,59 +36,15 @@ //! Once stack has been unwound down to the handler frame level, unwinding stops //! and the last personality routine transfers control to the catch block. -use alloc::boxed::Box; -use core::any::Any; - -use crate::dwarf::eh::{self, EHAction, EHContext}; +use super::dwarf::eh::{self, EHAction, EHContext}; use libc::{c_int, uintptr_t}; use unwind as uw; -#[repr(C)] -struct Exception { - _uwe: uw::_Unwind_Exception, - cause: Box, -} - -pub unsafe fn panic(data: Box) -> u32 { - let exception = Box::new(Exception { - _uwe: uw::_Unwind_Exception { - exception_class: rust_exception_class(), - exception_cleanup, - private: [0; uw::unwinder_private_data_size], - }, - cause: data, - }); - let exception_param = Box::into_raw(exception) as *mut uw::_Unwind_Exception; - return uw::_Unwind_RaiseException(exception_param) as u32; - - extern "C" fn exception_cleanup( - _unwind_code: uw::_Unwind_Reason_Code, - exception: *mut uw::_Unwind_Exception, - ) { - unsafe { - let _: Box = Box::from_raw(exception as *mut Exception); - super::__rust_drop_panic(); - } - } -} - -pub unsafe fn cleanup(ptr: *mut u8) -> Box { - let exception = Box::from_raw(ptr as *mut Exception); - exception.cause -} - -// Rust's exception class identifier. This is used by personality routines to -// determine whether the exception was thrown by their own runtime. -fn rust_exception_class() -> uw::_Unwind_Exception_Class { - // M O Z \0 R U S T -- vendor, language - 0x4d4f5a_00_52555354 -} - // Register ids were lifted from LLVM's TargetLowering::getExceptionPointerRegister() // and TargetLowering::getExceptionSelectorRegister() for each architecture, // then mapped to DWARF register numbers via register definition tables // (typically RegisterInfo.td, search for "DwarfRegNum"). -// See also http://llvm.org/docs/WritingAnLLVMBackend.html#defining-a-register. +// See also https://llvm.org/docs/WritingAnLLVMBackend.html#defining-a-register. #[cfg(target_arch = "x86")] const UNWIND_DATA_REG: (i32, i32) = (0, 2); // EAX, EDX @@ -99,6 +55,9 @@ const UNWIND_DATA_REG: (i32, i32) = (0, 1); // RAX, RDX #[cfg(any(target_arch = "arm", target_arch = "aarch64"))] const UNWIND_DATA_REG: (i32, i32) = (0, 1); // R0, R1 / X0, X1 +#[cfg(target_arch = "m68k")] +const UNWIND_DATA_REG: (i32, i32) = (0, 1); // D0, D1 + #[cfg(any(target_arch = "mips", target_arch = "mips64"))] const UNWIND_DATA_REG: (i32, i32) = (4, 5); // A0, A1 @@ -108,13 +67,13 @@ const UNWIND_DATA_REG: (i32, i32) = (3, 4); // R3, R4 / X3, X4 #[cfg(target_arch = "s390x")] const UNWIND_DATA_REG: (i32, i32) = (6, 7); // R6, R7 -#[cfg(target_arch = "sparc64")] +#[cfg(any(target_arch = "sparc", target_arch = "sparc64"))] const UNWIND_DATA_REG: (i32, i32) = (24, 25); // I0, I1 #[cfg(target_arch = "hexagon")] const UNWIND_DATA_REG: (i32, i32) = (0, 1); // R0, R1 -#[cfg(target_arch = "riscv64")] +#[cfg(any(target_arch = "riscv64", target_arch = "riscv32"))] const UNWIND_DATA_REG: (i32, i32) = (10, 11); // x10, x11 // The following code is based on GCC's C and C++ personality routines. For reference, see: @@ -122,16 +81,17 @@ const UNWIND_DATA_REG: (i32, i32) = (10, 11); // x10, x11 // https://github.com/gcc-mirror/gcc/blob/trunk/libgcc/unwind-c.c cfg_if::cfg_if! { - if #[cfg(all(target_arch = "arm", not(target_os = "ios"), not(target_os = "netbsd")))] { + if #[cfg(all(target_arch = "arm", not(target_os = "ios"), not(target_os = "watchos"), not(target_os = "netbsd")))] { // ARM EHABI personality routine. - // http://infocenter.arm.com/help/topic/com.arm.doc.ihi0038b/IHI0038B_ehabi.pdf + // https://infocenter.arm.com/help/topic/com.arm.doc.ihi0038b/IHI0038B_ehabi.pdf // // iOS uses the default routine instead since it uses SjLj unwinding. #[lang = "eh_personality"] - unsafe extern "C" fn rust_eh_personality(state: uw::_Unwind_State, - exception_object: *mut uw::_Unwind_Exception, - context: *mut uw::_Unwind_Context) - -> uw::_Unwind_Reason_Code { + unsafe extern "C" fn rust_eh_personality( + state: uw::_Unwind_State, + exception_object: *mut uw::_Unwind_Exception, + context: *mut uw::_Unwind_Context, + ) -> uw::_Unwind_Reason_Code { let state = state as c_int; let action = state & uw::_US_ACTION_MASK as c_int; let search_phase = if action == uw::_US_VIRTUAL_UNWIND_FRAME as c_int { @@ -157,23 +117,20 @@ cfg_if::cfg_if! { // take only the context pointer, GCC personality routines stash a pointer to // exception_object in the context, using location reserved for ARM's // "scratch register" (r12). - uw::_Unwind_SetGR(context, - uw::UNWIND_POINTER_REG, - exception_object as uw::_Unwind_Ptr); + uw::_Unwind_SetGR(context, uw::UNWIND_POINTER_REG, exception_object as uw::_Unwind_Ptr); // ...A more principled approach would be to provide the full definition of ARM's // _Unwind_Context in our libunwind bindings and fetch the required data from there // directly, bypassing DWARF compatibility functions. - let exception_class = (*exception_object).exception_class; - let foreign_exception = exception_class != rust_exception_class(); - let eh_action = match find_eh_action(context, foreign_exception) { + let eh_action = match find_eh_action(context) { Ok(action) => action, Err(_) => return uw::_URC_FAILURE, }; if search_phase { match eh_action { - EHAction::None | - EHAction::Cleanup(_) => return continue_unwind(exception_object, context), + EHAction::None | EHAction::Cleanup(_) => { + return continue_unwind(exception_object, context); + } EHAction::Catch(_) => { // EHABI requires the personality routine to update the // SP value in the barrier cache of the exception object. @@ -186,10 +143,12 @@ cfg_if::cfg_if! { } else { match eh_action { EHAction::None => return continue_unwind(exception_object, context), - EHAction::Cleanup(lpad) | - EHAction::Catch(lpad) => { - uw::_Unwind_SetGR(context, UNWIND_DATA_REG.0, - exception_object as uintptr_t); + EHAction::Cleanup(lpad) | EHAction::Catch(lpad) => { + uw::_Unwind_SetGR( + context, + UNWIND_DATA_REG.0, + exception_object as uintptr_t, + ); uw::_Unwind_SetGR(context, UNWIND_DATA_REG.1, 0); uw::_Unwind_SetIP(context, lpad); return uw::_URC_INSTALL_CONTEXT; @@ -200,9 +159,10 @@ cfg_if::cfg_if! { // On ARM EHABI the personality routine is responsible for actually // unwinding a single stack frame before returning (ARM EHABI Sec. 6.1). - unsafe fn continue_unwind(exception_object: *mut uw::_Unwind_Exception, - context: *mut uw::_Unwind_Context) - -> uw::_Unwind_Reason_Code { + unsafe fn continue_unwind( + exception_object: *mut uw::_Unwind_Exception, + context: *mut uw::_Unwind_Context, + ) -> uw::_Unwind_Reason_Code { if __gnu_unwind_frame(exception_object, context) == uw::_URC_NO_REASON { uw::_URC_CONTINUE_UNWIND } else { @@ -211,42 +171,44 @@ cfg_if::cfg_if! { } // defined in libgcc extern "C" { - fn __gnu_unwind_frame(exception_object: *mut uw::_Unwind_Exception, - context: *mut uw::_Unwind_Context) - -> uw::_Unwind_Reason_Code; + fn __gnu_unwind_frame( + exception_object: *mut uw::_Unwind_Exception, + context: *mut uw::_Unwind_Context, + ) -> uw::_Unwind_Reason_Code; } } } else { // Default personality routine, which is used directly on most targets // and indirectly on Windows x86_64 via SEH. - unsafe extern "C" fn rust_eh_personality_impl(version: c_int, - actions: uw::_Unwind_Action, - exception_class: uw::_Unwind_Exception_Class, - exception_object: *mut uw::_Unwind_Exception, - context: *mut uw::_Unwind_Context) - -> uw::_Unwind_Reason_Code { + unsafe extern "C" fn rust_eh_personality_impl( + version: c_int, + actions: uw::_Unwind_Action, + _exception_class: uw::_Unwind_Exception_Class, + exception_object: *mut uw::_Unwind_Exception, + context: *mut uw::_Unwind_Context, + ) -> uw::_Unwind_Reason_Code { if version != 1 { return uw::_URC_FATAL_PHASE1_ERROR; } - let foreign_exception = exception_class != rust_exception_class(); - let eh_action = match find_eh_action(context, foreign_exception) { + let eh_action = match find_eh_action(context) { Ok(action) => action, Err(_) => return uw::_URC_FATAL_PHASE1_ERROR, }; if actions as i32 & uw::_UA_SEARCH_PHASE as i32 != 0 { match eh_action { - EHAction::None | - EHAction::Cleanup(_) => uw::_URC_CONTINUE_UNWIND, + EHAction::None | EHAction::Cleanup(_) => uw::_URC_CONTINUE_UNWIND, EHAction::Catch(_) => uw::_URC_HANDLER_FOUND, EHAction::Terminate => uw::_URC_FATAL_PHASE1_ERROR, } } else { match eh_action { EHAction::None => uw::_URC_CONTINUE_UNWIND, - EHAction::Cleanup(lpad) | - EHAction::Catch(lpad) => { - uw::_Unwind_SetGR(context, UNWIND_DATA_REG.0, - exception_object as uintptr_t); + EHAction::Cleanup(lpad) | EHAction::Catch(lpad) => { + uw::_Unwind_SetGR( + context, + UNWIND_DATA_REG.0, + exception_object as uintptr_t, + ); uw::_Unwind_SetGR(context, UNWIND_DATA_REG.1, 0); uw::_Unwind_SetIP(context, lpad); uw::_URC_INSTALL_CONTEXT @@ -257,102 +219,61 @@ cfg_if::cfg_if! { } cfg_if::cfg_if! { - if #[cfg(all(windows, target_arch = "x86_64", target_env = "gnu"))] { + if #[cfg(all(windows, any(target_arch = "aarch64", target_arch = "x86_64"), target_env = "gnu"))] { // On x86_64 MinGW targets, the unwinding mechanism is SEH however the unwind // handler data (aka LSDA) uses GCC-compatible encoding. #[lang = "eh_personality"] #[allow(nonstandard_style)] - unsafe extern "C" fn rust_eh_personality(exceptionRecord: *mut uw::EXCEPTION_RECORD, - establisherFrame: uw::LPVOID, - contextRecord: *mut uw::CONTEXT, - dispatcherContext: *mut uw::DISPATCHER_CONTEXT) - -> uw::EXCEPTION_DISPOSITION { - uw::_GCC_specific_handler(exceptionRecord, - establisherFrame, - contextRecord, - dispatcherContext, - rust_eh_personality_impl) + unsafe extern "C" fn rust_eh_personality( + exceptionRecord: *mut uw::EXCEPTION_RECORD, + establisherFrame: uw::LPVOID, + contextRecord: *mut uw::CONTEXT, + dispatcherContext: *mut uw::DISPATCHER_CONTEXT, + ) -> uw::EXCEPTION_DISPOSITION { + uw::_GCC_specific_handler( + exceptionRecord, + establisherFrame, + contextRecord, + dispatcherContext, + rust_eh_personality_impl, + ) } } else { // The personality routine for most of our targets. #[lang = "eh_personality"] - unsafe extern "C" fn rust_eh_personality(version: c_int, - actions: uw::_Unwind_Action, - exception_class: uw::_Unwind_Exception_Class, - exception_object: *mut uw::_Unwind_Exception, - context: *mut uw::_Unwind_Context) - -> uw::_Unwind_Reason_Code { - rust_eh_personality_impl(version, - actions, - exception_class, - exception_object, - context) + unsafe extern "C" fn rust_eh_personality( + version: c_int, + actions: uw::_Unwind_Action, + exception_class: uw::_Unwind_Exception_Class, + exception_object: *mut uw::_Unwind_Exception, + context: *mut uw::_Unwind_Context, + ) -> uw::_Unwind_Reason_Code { + rust_eh_personality_impl( + version, + actions, + exception_class, + exception_object, + context, + ) } } } } } -unsafe fn find_eh_action( - context: *mut uw::_Unwind_Context, - foreign_exception: bool, -) -> Result { +unsafe fn find_eh_action(context: *mut uw::_Unwind_Context) -> Result { let lsda = uw::_Unwind_GetLanguageSpecificData(context) as *const u8; let mut ip_before_instr: c_int = 0; let ip = uw::_Unwind_GetIPInfo(context, &mut ip_before_instr); let eh_context = EHContext { // The return address points 1 byte past the call instruction, // which could be in the next IP range in LSDA range table. - ip: if ip_before_instr != 0 { ip } else { ip - 1 }, + // + // `ip = -1` has special meaning, so use wrapping sub to allow for that + ip: if ip_before_instr != 0 { ip } else { ip.wrapping_sub(1) }, func_start: uw::_Unwind_GetRegionStart(context), get_text_start: &|| uw::_Unwind_GetTextRelBase(context), get_data_start: &|| uw::_Unwind_GetDataRelBase(context), }; - eh::find_eh_action(lsda, &eh_context, foreign_exception) -} - -#[cfg(all( - bootstrap, - target_os = "windows", - any(target_arch = "x86", target_arch = "x86_64"), - target_env = "gnu" -))] -#[lang = "eh_unwind_resume"] -#[unwind(allowed)] -unsafe extern "C" fn rust_eh_unwind_resume(panic_ctx: *mut u8) -> ! { - uw::_Unwind_Resume(panic_ctx as *mut uw::_Unwind_Exception); -} - -// Frame unwind info registration -// -// Each module's image contains a frame unwind info section (usually -// ".eh_frame"). When a module is loaded/unloaded into the process, the -// unwinder must be informed about the location of this section in memory. The -// methods of achieving that vary by the platform. On some (e.g., Linux), the -// unwinder can discover unwind info sections on its own (by dynamically -// enumerating currently loaded modules via the dl_iterate_phdr() API and -// finding their ".eh_frame" sections); Others, like Windows, require modules -// to actively register their unwind info sections via unwinder API. -// -// This module defines two symbols which are referenced and called from -// rsbegin.rs to register our information with the GCC runtime. The -// implementation of stack unwinding is (for now) deferred to libgcc_eh, however -// Rust crates use these Rust-specific entry points to avoid potential clashes -// with any GCC runtime. -#[cfg(all(target_os = "windows", target_arch = "x86", target_env = "gnu"))] -pub mod eh_frame_registry { - extern "C" { - fn __register_frame_info(eh_frame_begin: *const u8, object: *mut u8); - fn __deregister_frame_info(eh_frame_begin: *const u8, object: *mut u8); - } - - #[rustc_std_internal_symbol] - pub unsafe extern "C" fn rust_eh_register_frames(eh_frame_begin: *const u8, object: *mut u8) { - __register_frame_info(eh_frame_begin, object); - } - - #[rustc_std_internal_symbol] - pub unsafe extern "C" fn rust_eh_unregister_frames(eh_frame_begin: *const u8, object: *mut u8) { - __deregister_frame_info(eh_frame_begin, object); - } + eh::find_eh_action(lsda, &eh_context) } diff --git a/crux-mir/lib/std/src/prelude/mod.rs b/crux-mir/lib/std/src/prelude/mod.rs index 3085c3d82..c314bbbb6 100644 --- a/crux-mir/lib/std/src/prelude/mod.rs +++ b/crux-mir/lib/std/src/prelude/mod.rs @@ -1,4 +1,4 @@ -//! The Rust Prelude. +//! # The Rust Prelude //! //! Rust comes with a variety of things in its standard library. However, if //! you had to manually import every single thing that you used, it would be @@ -10,22 +10,6 @@ //! things, particularly traits, which are used in almost every single Rust //! program. //! -//! On a technical level, Rust inserts -//! -//! ``` -//! # #[allow(unused_extern_crates)] -//! extern crate std; -//! ``` -//! -//! into the crate root of every crate, and -//! -//! ``` -//! # #[allow(unused_imports)] -//! use std::prelude::v1::*; -//! ``` -//! -//! into every module. -//! //! # Other preludes //! //! Preludes can be seen as a pattern to make using multiple types more @@ -33,7 +17,7 @@ //! such as [`std::io::prelude`]. Various libraries in the Rust ecosystem may //! also define their own preludes. //! -//! [`std::io::prelude`]: ../io/prelude/index.html +//! [`std::io::prelude`]: crate::io::prelude //! //! The difference between 'the prelude' and these other preludes is that they //! are not automatically `use`'d, and must be imported manually. This is still @@ -41,91 +25,71 @@ //! //! # Prelude contents //! -//! The current version of the prelude (version 1) lives in -//! [`std::prelude::v1`], and re-exports the following. +//! The first version of the prelude is used in Rust 2015 and Rust 2018, +//! and lives in [`std::prelude::v1`]. +//! [`std::prelude::rust_2015`] and [`std::prelude::rust_2018`] re-export this prelude. +//! It re-exports the following: //! -//! * [`std::marker`]::{[`Copy`], [`Send`], [`Sized`], [`Sync`], [`Unpin`]}. The -//! marker traits indicate fundamental properties of types. -//! * [`std::ops`]::{[`Drop`], [`Fn`], [`FnMut`], [`FnOnce`]}. Various +//! * [std::marker]::{[Copy], [Send], [Sized], [Sync], [Unpin]}, +//! marker traits that indicate fundamental properties of types. +//! * [std::ops]::{[Drop], [Fn], [FnMut], [FnOnce]}, various //! operations for both destructors and overloading `()`. -//! * [`std::mem`]::[`drop`][`mem::drop`], a convenience function for explicitly +//! * [std::mem]::[drop][mem::drop], a convenience function for explicitly //! dropping a value. -//! * [`std::boxed`]::[`Box`], a way to allocate values on the heap. -//! * [`std::borrow`]::[`ToOwned`], The conversion trait that defines +//! * [std::boxed]::[Box], a way to allocate values on the heap. +//! * [std::borrow]::[ToOwned], the conversion trait that defines //! [`to_owned`], the generic method for creating an owned type from a //! borrowed type. -//! * [`std::clone`]::[`Clone`], the ubiquitous trait that defines -//! [`clone`][`Clone::clone`], the method for producing a copy of a value. -//! * [`std::cmp`]::{[`PartialEq`], [`PartialOrd`], [`Eq`], [`Ord`] }. The +//! * [std::clone]::[Clone], the ubiquitous trait that defines +//! [`clone`][Clone::clone], the method for producing a copy of a value. +//! * [std::cmp]::{[PartialEq], [PartialOrd], [Eq], [Ord]}, the //! comparison traits, which implement the comparison operators and are often //! seen in trait bounds. -//! * [`std::convert`]::{[`AsRef`], [`AsMut`], [`Into`], [`From`]}. Generic +//! * [std::convert]::{[AsRef], [AsMut], [Into], [From]}, generic //! conversions, used by savvy API authors to create overloaded methods. -//! * [`std::default`]::[`Default`], types that have default values. -//! * [`std::iter`]::{[`Iterator`], [`Extend`], [`IntoIterator`], -//! [`DoubleEndedIterator`], [`ExactSizeIterator`]}. Iterators of various +//! * [std::default]::[Default], types that have default values. +//! * [std::iter]::{[Iterator], [Extend], [IntoIterator], [DoubleEndedIterator], [ExactSizeIterator]}, +//! iterators of various //! kinds. -//! * [`std::option`]::[`Option`]::{`self`, `Some`, `None`}. A type which -//! expresses the presence or absence of a value. This type is so commonly -//! used, its variants are also exported. -//! * [`std::result`]::[`Result`]::{`self`, `Ok`, `Err`}. A type for functions -//! that may succeed or fail. Like [`Option`], its variants are exported as -//! well. -//! * [`std::string`]::{[`String`], [`ToString`]}, heap allocated strings. -//! * [`std::vec`]::[`Vec`](../vec/struct.Vec.html), a growable, heap-allocated -//! vector. +//! * [std::option]::[Option]::{[self][Option], [Some], [None]}, a +//! type which expresses the presence or absence of a value. This type is so +//! commonly used, its variants are also exported. +//! * [std::result]::[Result]::{[self][Result], [Ok], [Err]}, a type +//! for functions that may succeed or fail. Like [`Option`], its variants are +//! exported as well. +//! * [std::string]::{[String], [ToString]}, heap-allocated strings. +//! * [std::vec]::[Vec], a growable, heap-allocated vector. +//! +//! The prelude used in Rust 2021, [`std::prelude::rust_2021`], includes all of the above, +//! and in addition re-exports: +//! +//! * [std::convert]::{[TryFrom], [TryInto]}, +//! * [std::iter]::[FromIterator]. //! -//! [`AsMut`]: ../convert/trait.AsMut.html -//! [`AsRef`]: ../convert/trait.AsRef.html -//! [`Box`]: ../boxed/struct.Box.html -//! [`Clone`]: ../clone/trait.Clone.html -//! [`Copy`]: ../marker/trait.Copy.html -//! [`Default`]: ../default/trait.Default.html -//! [`DoubleEndedIterator`]: ../iter/trait.DoubleEndedIterator.html -//! [`Drop`]: ../ops/trait.Drop.html -//! [`Eq`]: ../cmp/trait.Eq.html -//! [`ExactSizeIterator`]: ../iter/trait.ExactSizeIterator.html -//! [`Extend`]: ../iter/trait.Extend.html -//! [`FnMut`]: ../ops/trait.FnMut.html -//! [`FnOnce`]: ../ops/trait.FnOnce.html -//! [`Fn`]: ../ops/trait.Fn.html -//! [`From`]: ../convert/trait.From.html -//! [`IntoIterator`]: ../iter/trait.IntoIterator.html -//! [`Into`]: ../convert/trait.Into.html -//! [`Iterator`]: ../iter/trait.Iterator.html -//! [`Option`]: ../option/enum.Option.html -//! [`Ord`]: ../cmp/trait.Ord.html -//! [`PartialEq`]: ../cmp/trait.PartialEq.html -//! [`PartialOrd`]: ../cmp/trait.PartialOrd.html -//! [`Result`]: ../result/enum.Result.html -//! [`Send`]: ../marker/trait.Send.html -//! [`Sized`]: ../marker/trait.Sized.html -//! [`SliceConcatExt`]: ../slice/trait.SliceConcatExt.html -//! [`String`]: ../string/struct.String.html -//! [`Sync`]: ../marker/trait.Sync.html -//! [`ToOwned`]: ../borrow/trait.ToOwned.html -//! [`ToString`]: ../string/trait.ToString.html -//! [`Unpin`]: ../marker/trait.Unpin.html -//! [`Vec`]: ../vec/struct.Vec.html -//! [`Clone::clone`]: ../clone/trait.Clone.html#tymethod.clone -//! [`mem::drop`]: ../mem/fn.drop.html -//! [`std::borrow`]: ../borrow/index.html -//! [`std::boxed`]: ../boxed/index.html -//! [`std::clone`]: ../clone/index.html -//! [`std::cmp`]: ../cmp/index.html -//! [`std::convert`]: ../convert/index.html -//! [`std::default`]: ../default/index.html -//! [`std::iter`]: ../iter/index.html -//! [`std::marker`]: ../marker/index.html -//! [`std::mem`]: ../mem/index.html -//! [`std::ops`]: ../ops/index.html -//! [`std::option`]: ../option/index.html -//! [`std::prelude::v1`]: v1/index.html -//! [`std::result`]: ../result/index.html -//! [`std::slice`]: ../slice/index.html -//! [`std::string`]: ../string/index.html -//! [`std::vec`]: ../vec/index.html -//! [`to_owned`]: ../borrow/trait.ToOwned.html#tymethod.to_owned +//! [mem::drop]: crate::mem::drop +//! [std::borrow]: crate::borrow +//! [std::boxed]: crate::boxed +//! [std::clone]: crate::clone +//! [std::cmp]: crate::cmp +//! [std::convert]: crate::convert +//! [std::default]: crate::default +//! [std::iter]: crate::iter +//! [std::marker]: crate::marker +//! [std::mem]: crate::mem +//! [std::ops]: crate::ops +//! [std::option]: crate::option +//! [`std::prelude::v1`]: v1 +//! [`std::prelude::rust_2015`]: rust_2015 +//! [`std::prelude::rust_2018`]: rust_2018 +//! [`std::prelude::rust_2021`]: rust_2021 +//! [std::result]: crate::result +//! [std::slice]: crate::slice +//! [std::string]: crate::string +//! [std::vec]: mod@crate::vec +//! [TryFrom]: crate::convert::TryFrom +//! [TryInto]: crate::convert::TryInto +//! [FromIterator]: crate::iter::FromIterator +//! [`to_owned`]: crate::borrow::ToOwned::to_owned //! [book-closures]: ../../book/ch13-01-closures.html //! [book-dtor]: ../../book/ch15-03-drop.html //! [book-enums]: ../../book/ch06-01-defining-an-enum.html @@ -134,3 +98,51 @@ #![stable(feature = "rust1", since = "1.0.0")] pub mod v1; + +/// The 2015 version of the prelude of The Rust Standard Library. +/// +/// See the [module-level documentation](self) for more. +#[stable(feature = "prelude_2015", since = "1.55.0")] +pub mod rust_2015 { + #[stable(feature = "prelude_2015", since = "1.55.0")] + #[doc(no_inline)] + pub use super::v1::*; +} + +/// The 2018 version of the prelude of The Rust Standard Library. +/// +/// See the [module-level documentation](self) for more. +#[stable(feature = "prelude_2018", since = "1.55.0")] +pub mod rust_2018 { + #[stable(feature = "prelude_2018", since = "1.55.0")] + #[doc(no_inline)] + pub use super::v1::*; +} + +/// The 2021 version of the prelude of The Rust Standard Library. +/// +/// See the [module-level documentation](self) for more. +#[stable(feature = "prelude_2021", since = "1.55.0")] +pub mod rust_2021 { + #[stable(feature = "prelude_2021", since = "1.55.0")] + #[doc(no_inline)] + pub use super::v1::*; + + #[stable(feature = "prelude_2021", since = "1.55.0")] + #[doc(no_inline)] + pub use core::prelude::rust_2021::*; +} + +/// The 2024 version of the prelude of The Rust Standard Library. +/// +/// See the [module-level documentation](self) for more. +#[unstable(feature = "prelude_2024", issue = "none")] +pub mod rust_2024 { + #[unstable(feature = "prelude_2024", issue = "none")] + #[doc(no_inline)] + pub use super::v1::*; + + #[unstable(feature = "prelude_2024", issue = "none")] + #[doc(no_inline)] + pub use core::prelude::rust_2024::*; +} diff --git a/crux-mir/lib/std/src/prelude/v1.rs b/crux-mir/lib/std/src/prelude/v1.rs index 6712f5ba5..2aefd7c51 100644 --- a/crux-mir/lib/std/src/prelude/v1.rs +++ b/crux-mir/lib/std/src/prelude/v1.rs @@ -1,6 +1,6 @@ //! The first version of the prelude of The Rust Standard Library. //! -//! See the [module-level documentation](../index.html) for more. +//! See the [module-level documentation](super) for more. #![stable(feature = "rust1", since = "1.0.0")] @@ -36,32 +36,61 @@ pub use crate::result::Result::{self, Err, Ok}; // Re-exported built-in macros #[stable(feature = "builtin_macro_prelude", since = "1.38.0")] +#[allow(deprecated)] #[doc(no_inline)] pub use core::prelude::v1::{ - asm, assert, cfg, column, compile_error, concat, concat_idents, env, file, format_args, - format_args_nl, global_asm, include, include_bytes, include_str, line, log_syntax, module_path, - option_env, stringify, trace_macros, + assert, cfg, column, compile_error, concat, concat_idents, env, file, format_args, + format_args_nl, include, include_bytes, include_str, line, log_syntax, module_path, option_env, + stringify, trace_macros, Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, }; -// FIXME: Attribute and derive macros are not documented because for them rustdoc generates -// dead links which fail link checker testing. +#[unstable( + feature = "concat_bytes", + issue = "87555", + reason = "`concat_bytes` is not stable enough for use and is subject to change" +)] +#[doc(no_inline)] +pub use core::prelude::v1::concat_bytes; + +// Do not `doc(inline)` these `doc(hidden)` items. #[stable(feature = "builtin_macro_prelude", since = "1.38.0")] #[allow(deprecated)] -#[doc(hidden)] +pub use core::prelude::v1::{RustcDecodable, RustcEncodable}; + +// Do not `doc(no_inline)` so that they become doc items on their own +// (no public module for them to be re-exported from). +#[stable(feature = "builtin_macro_prelude", since = "1.38.0")] pub use core::prelude::v1::{ - bench, global_allocator, test, test_case, Clone, Copy, Debug, Default, Eq, Hash, Ord, - PartialEq, PartialOrd, RustcDecodable, RustcEncodable, + alloc_error_handler, bench, derive, global_allocator, test, test_case, }; -#[cfg(not(bootstrap))] +#[unstable(feature = "derive_const", issue = "none")] +pub use core::prelude::v1::derive_const; + +// Do not `doc(no_inline)` either. #[unstable( feature = "cfg_accessible", issue = "64797", reason = "`cfg_accessible` is not fully implemented" )] -#[doc(hidden)] pub use core::prelude::v1::cfg_accessible; +// Do not `doc(no_inline)` either. +#[unstable( + feature = "cfg_eval", + issue = "82679", + reason = "`cfg_eval` is a recently implemented feature" +)] +pub use core::prelude::v1::cfg_eval; + +// Do not `doc(no_inline)` either. +#[unstable( + feature = "type_ascription", + issue = "23416", + reason = "placeholder syntax for type ascription" +)] +pub use core::prelude::v1::type_ascribe; + // The file so far is equivalent to src/libcore/prelude/v1.rs, // and below to src/liballoc/prelude.rs. // Those files are duplicated rather than using glob imports diff --git a/crux-mir/lib/std/src/primitive_docs.rs b/crux-mir/lib/std/src/primitive_docs.rs index adad90f56..d6e9da187 100644 --- a/crux-mir/lib/std/src/primitive_docs.rs +++ b/crux-mir/lib/std/src/primitive_docs.rs @@ -1,33 +1,38 @@ +// `library/{std,core}/src/primitive_docs.rs` should have the same contents. +// These are different files so that relative links work properly without +// having to have `CARGO_PKG_NAME` set, but conceptually they should always be the same. #[doc(primitive = "bool")] #[doc(alias = "true")] #[doc(alias = "false")] -// /// The boolean type. /// -/// The `bool` represents a value, which could only be either `true` or `false`. If you cast -/// a `bool` into an integer, `true` will be 1 and `false` will be 0. +/// The `bool` represents a value, which could only be either [`true`] or [`false`]. If you cast +/// a `bool` into an integer, [`true`] will be 1 and [`false`] will be 0. /// /// # Basic usage /// /// `bool` implements various traits, such as [`BitAnd`], [`BitOr`], [`Not`], etc., /// which allow us to perform boolean operations using `&`, `|` and `!`. /// -/// `if` always demands a `bool` value. [`assert!`], being an important macro in testing, -/// checks whether an expression returns `true`. +/// [`if`] requires a `bool` value as its conditional. [`assert!`], which is an +/// important macro in testing, checks whether an expression is [`true`] and panics +/// if it isn't. /// /// ``` /// let bool_val = true & false | false; /// assert!(!bool_val); /// ``` /// -/// [`assert!`]: macro.assert.html -/// [`BitAnd`]: ops/trait.BitAnd.html -/// [`BitOr`]: ops/trait.BitOr.html -/// [`Not`]: ops/trait.Not.html +/// [`true`]: ../std/keyword.true.html +/// [`false`]: ../std/keyword.false.html +/// [`BitAnd`]: ops::BitAnd +/// [`BitOr`]: ops::BitOr +/// [`Not`]: ops::Not +/// [`if`]: ../std/keyword.if.html /// /// # Examples /// -/// A trivial example of the usage of `bool`, +/// A trivial example of the usage of `bool`: /// /// ``` /// let praise_the_borrow_checker = true; @@ -46,7 +51,7 @@ /// } /// ``` /// -/// Also, since `bool` implements the [`Copy`](marker/trait.Copy.html) trait, we don't +/// Also, since `bool` implements the [`Copy`] trait, we don't /// have to worry about the move semantics (just like the integer and float primitives). /// /// Now an example of `bool` cast to integer type: @@ -100,8 +105,8 @@ mod prim_bool {} /// at all we know it can never produce a value which isn't a [`u32`]. This illustrates another /// behaviour of the `!` type - expressions with type `!` will coerce into any other type. /// -/// [`u32`]: primitive.str.html -/// [`exit`]: process/fn.exit.html +/// [`u32`]: prim@u32 +#[doc = concat!("[`exit`]: ", include_str!("../primitive_docs/process_exit.md"))] /// /// # `!` and generics /// @@ -124,9 +129,9 @@ mod prim_bool {} /// `!`, if we have to call [`String::from_str`] for some reason the result will be a /// [`Result`] which we can unpack like this: /// -/// ```ignore (string-from-str-error-type-is-not-never-yet) -/// #[feature(exhaustive_patterns)] -/// // NOTE: this does not work today! +/// ``` +/// #![feature(exhaustive_patterns)] +/// use std::str::FromStr; /// let Ok(s) = String::from_str("hello"); /// ``` /// @@ -185,26 +190,55 @@ mod prim_bool {} /// ever stops, it means that an error occurred. We don't even have to wrap the loop in an `Ok` /// because `!` coerces to `Result` automatically. /// -/// [`String::from_str`]: str/trait.FromStr.html#tymethod.from_str -/// [`Result`]: result/enum.Result.html -/// [`Result`]: result/enum.Result.html -/// [`Result`]: result/enum.Result.html -/// [`Ok`]: result/enum.Result.html#variant.Ok -/// [`String`]: string/struct.String.html -/// [`Err`]: result/enum.Result.html#variant.Err -/// [`FromStr`]: str/trait.FromStr.html +/// [`String::from_str`]: str::FromStr::from_str +#[doc = concat!("[`String`]: ", include_str!("../primitive_docs/string_string.md"))] +/// [`FromStr`]: str::FromStr /// /// # `!` and traits /// /// When writing your own traits, `!` should have an `impl` whenever there is an obvious `impl` -/// which doesn't `panic!`. As it turns out, most traits can have an `impl` for `!`. Take [`Debug`] +/// which doesn't `panic!`. The reason is that functions returning an `impl Trait` where `!` +/// does not have an `impl` of `Trait` cannot diverge as their only possible code path. In other +/// words, they can't return `!` from every code path. As an example, this code doesn't compile: +/// +/// ```compile_fail +/// use std::ops::Add; +/// +/// fn foo() -> impl Add { +/// unimplemented!() +/// } +/// ``` +/// +/// But this code does: +/// +/// ``` +/// use std::ops::Add; +/// +/// fn foo() -> impl Add { +/// if true { +/// unimplemented!() +/// } else { +/// 0 +/// } +/// } +/// ``` +/// +/// The reason is that, in the first example, there are many possible types that `!` could coerce +/// to, because many types implement `Add`. However, in the second example, +/// the `else` branch returns a `0`, which the compiler infers from the return type to be of type +/// `u32`. Since `u32` is a concrete type, `!` can and will be coerced to it. See issue [#36375] +/// for more information on this quirk of `!`. +/// +/// [#36375]: https://github.com/rust-lang/rust/issues/36375 +/// +/// As it turns out, though, most traits can have an `impl` for `!`. Take [`Debug`] /// for example: /// /// ``` /// #![feature(never_type)] /// # use std::fmt; /// # trait Debug { -/// # fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result; +/// # fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result; /// # } /// impl Debug for ! { /// fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -233,31 +267,77 @@ mod prim_bool {} /// `impl` for this which simply panics, but the same is true for any type (we could `impl /// Default` for (eg.) [`File`] by just making [`default()`] panic.) /// -/// [`fmt::Result`]: fmt/type.Result.html -/// [`File`]: fs/struct.File.html -/// [`Debug`]: fmt/trait.Debug.html -/// [`Default`]: default/trait.Default.html -/// [`default()`]: default/trait.Default.html#tymethod.default +#[doc = concat!("[`File`]: ", include_str!("../primitive_docs/fs_file.md"))] +/// [`Debug`]: fmt::Debug +/// [`default()`]: Default::default /// #[unstable(feature = "never_type", issue = "35121")] mod prim_never {} #[doc(primitive = "char")] -// +#[allow(rustdoc::invalid_rust_codeblocks)] /// A character type. /// /// The `char` type represents a single character. More specifically, since /// 'character' isn't a well-defined concept in Unicode, `char` is a '[Unicode -/// scalar value]', which is similar to, but not the same as, a '[Unicode code -/// point]'. -/// -/// [Unicode scalar value]: http://www.unicode.org/glossary/#unicode_scalar_value -/// [Unicode code point]: http://www.unicode.org/glossary/#code_point +/// scalar value]'. /// /// This documentation describes a number of methods and trait implementations on the /// `char` type. For technical reasons, there is additional, separate /// documentation in [the `std::char` module](char/index.html) as well. /// +/// # Validity +/// +/// A `char` is a '[Unicode scalar value]', which is any '[Unicode code point]' +/// other than a [surrogate code point]. This has a fixed numerical definition: +/// code points are in the range 0 to 0x10FFFF, inclusive. +/// Surrogate code points, used by UTF-16, are in the range 0xD800 to 0xDFFF. +/// +/// No `char` may be constructed, whether as a literal or at runtime, that is not a +/// Unicode scalar value: +/// +/// ```compile_fail +/// // Each of these is a compiler error +/// ['\u{D800}', '\u{DFFF}', '\u{110000}']; +/// ``` +/// +/// ```should_panic +/// // Panics; from_u32 returns None. +/// char::from_u32(0xDE01).unwrap(); +/// ``` +/// +/// ```no_run +/// // Undefined behaviour +/// unsafe { char::from_u32_unchecked(0x110000) }; +/// ``` +/// +/// USVs are also the exact set of values that may be encoded in UTF-8. Because +/// `char` values are USVs and `str` values are valid UTF-8, it is safe to store +/// any `char` in a `str` or read any character from a `str` as a `char`. +/// +/// The gap in valid `char` values is understood by the compiler, so in the +/// below example the two ranges are understood to cover the whole range of +/// possible `char` values and there is no error for a [non-exhaustive match]. +/// +/// ``` +/// let c: char = 'a'; +/// match c { +/// '\0' ..= '\u{D7FF}' => false, +/// '\u{E000}' ..= '\u{10FFFF}' => true, +/// }; +/// ``` +/// +/// All USVs are valid `char` values, but not all of them represent a real +/// character. Many USVs are not currently assigned to a character, but may be +/// in the future ("reserved"); some will never be a character +/// ("noncharacters"); and some may be given different meanings by different +/// users ("private use"). +/// +/// [Unicode code point]: https://www.unicode.org/glossary/#code_point +/// [Unicode scalar value]: https://www.unicode.org/glossary/#unicode_scalar_value +/// [non-exhaustive match]: ../book/ch06-02-match.html#matches-are-exhaustive +/// [surrogate code point]: https://www.unicode.org/glossary/#surrogate_code_point +/// /// # Representation /// /// `char` is always four bytes in size. This is a different representation than @@ -275,9 +355,9 @@ mod prim_never {} /// assert_eq!(5, s.len() * std::mem::size_of::()); /// ``` /// -/// [`String`]: string/struct.String.html +#[doc = concat!("[`String`]: ", include_str!("../primitive_docs/string_string.md"))] /// -/// As always, remember that a human intuition for 'character' may not map to +/// As always, remember that a human intuition for 'character' might not map to /// Unicode's definitions. For example, despite looking similar, the 'é' /// character is one Unicode code point while 'é' is two Unicode code points: /// @@ -319,8 +399,11 @@ mod prim_never {} mod prim_char {} #[doc(primitive = "unit")] +#[doc(alias = "(")] +#[doc(alias = ")")] +#[doc(alias = "()")] // -/// The `()` type, sometimes called "unit" or "nil". +/// The `()` type, also called "unit". /// /// The `()` type has exactly one value `()`, and is used when there /// is no other meaningful value that could be returned. `()` is most @@ -356,11 +439,36 @@ mod prim_char {} #[stable(feature = "rust1", since = "1.0.0")] mod prim_unit {} +// Required to make auto trait impls render. +// See src/librustdoc/passes/collect_trait_impls.rs:collect_trait_impls +#[doc(hidden)] +impl () {} + +// Fake impl that's only really used for docs. +#[cfg(doc)] +#[stable(feature = "rust1", since = "1.0.0")] +impl Clone for () { + fn clone(&self) -> Self { + loop {} + } +} + +// Fake impl that's only really used for docs. +#[cfg(doc)] +#[stable(feature = "rust1", since = "1.0.0")] +impl Copy for () { + // empty +} + #[doc(primitive = "pointer")] +#[doc(alias = "ptr")] +#[doc(alias = "*")] +#[doc(alias = "*const")] +#[doc(alias = "*mut")] // /// Raw, unsafe pointers, `*const T`, and `*mut T`. /// -/// *[See also the `std::ptr` module](ptr/index.html).* +/// *[See also the `std::ptr` module](ptr).* /// /// Working with raw pointers in Rust is uncommon, typically limited to a few patterns. /// Raw pointers can be unaligned or [`null`]. However, when a raw pointer is @@ -418,7 +526,27 @@ mod prim_unit {} /// Note that here the call to [`drop`] is for clarity - it indicates /// that we are done with the given value and it should be destroyed. /// -/// ## 3. Get it from C. +/// ## 3. Create it using `ptr::addr_of!` +/// +/// Instead of coercing a reference to a raw pointer, you can use the macros +/// [`ptr::addr_of!`] (for `*const T`) and [`ptr::addr_of_mut!`] (for `*mut T`). +/// These macros allow you to create raw pointers to fields to which you cannot +/// create a reference (without causing undefined behaviour), such as an +/// unaligned field. This might be necessary if packed structs or uninitialized +/// memory is involved. +/// +/// ``` +/// #[derive(Debug, Default, Copy, Clone)] +/// #[repr(C, packed)] +/// struct S { +/// aligned: u8, +/// unaligned: u32, +/// } +/// let s = S::default(); +/// let p = std::ptr::addr_of!(s.unaligned); // not allowed with coercion +/// ``` +/// +/// ## 4. Get it from C. /// /// ``` /// # #![feature(rustc_private)] @@ -439,18 +567,20 @@ mod prim_unit {} /// but C APIs hand out a lot of pointers generally, so are a common source /// of raw pointers in Rust. /// -/// [`null`]: ../std/ptr/fn.null.html -/// [`null_mut`]: ../std/ptr/fn.null_mut.html -/// [`is_null`]: ../std/primitive.pointer.html#method.is_null -/// [`offset`]: ../std/primitive.pointer.html#method.offset -/// [`into_raw`]: ../std/boxed/struct.Box.html#method.into_raw -/// [`drop`]: ../std/mem/fn.drop.html -/// [`write`]: ../std/ptr/fn.write.html +/// [`null`]: ptr::null +/// [`null_mut`]: ptr::null_mut +/// [`is_null`]: pointer::is_null +/// [`offset`]: pointer::offset +#[doc = concat!("[`into_raw`]: ", include_str!("../primitive_docs/box_into_raw.md"))] +/// [`drop`]: mem::drop +/// [`write`]: ptr::write #[stable(feature = "rust1", since = "1.0.0")] mod prim_pointer {} #[doc(primitive = "array")] -// +#[doc(alias = "[]")] +#[doc(alias = "[T;N]")] // unfortunately, rustdoc doesn't have fuzzy search for aliases +#[doc(alias = "[T; N]")] /// A fixed-size array, denoted `[T; N]`, for the element type, `T`, and the /// non-negative compile-time constant size, `N`. /// @@ -458,36 +588,45 @@ mod prim_pointer {} /// /// * A list with each element, i.e., `[x, y, z]`. /// * A repeat expression `[x; N]`, which produces an array with `N` copies of `x`. -/// The type of `x` must be [`Copy`][copy]. -/// -/// Arrays of sizes from 0 to 32 (inclusive) implement the following traits if -/// the element type allows it: -/// -/// - [`Debug`][debug] -/// - [`IntoIterator`][intoiterator] (implemented for `&[T; N]` and `&mut [T; N]`) -/// - [`PartialEq`][partialeq], [`PartialOrd`][partialord], [`Eq`][eq], [`Ord`][ord] -/// - [`Hash`][hash] -/// - [`AsRef`][asref], [`AsMut`][asmut] -/// - [`Borrow`][borrow], [`BorrowMut`][borrowmut] -/// - [`Default`][default] -/// -/// This limitation on the size `N` exists because Rust does not yet support -/// code that is generic over the size of an array type. `[Foo; 3]` and `[Bar; 3]` -/// are instances of same generic type `[T; 3]`, but `[Foo; 3]` and `[Foo; 5]` are -/// entirely different types. As a stopgap, trait implementations are -/// statically generated up to size 32. +/// The type of `x` must be [`Copy`]. +/// +/// Note that `[expr; 0]` is allowed, and produces an empty array. +/// This will still evaluate `expr`, however, and immediately drop the resulting value, so +/// be mindful of side effects. /// -/// Arrays of *any* size are [`Copy`][copy] if the element type is [`Copy`][copy] -/// and [`Clone`][clone] if the element type is [`Clone`][clone]. This works -/// because [`Copy`][copy] and [`Clone`][clone] traits are specially known -/// to the compiler. +/// Arrays of *any* size implement the following traits if the element type allows it: +/// +/// - [`Copy`] +/// - [`Clone`] +/// - [`Debug`] +/// - [`IntoIterator`] (implemented for `[T; N]`, `&[T; N]` and `&mut [T; N]`) +/// - [`PartialEq`], [`PartialOrd`], [`Eq`], [`Ord`] +/// - [`Hash`] +/// - [`AsRef`], [`AsMut`] +/// - [`Borrow`], [`BorrowMut`] +/// +/// Arrays of sizes from 0 to 32 (inclusive) implement the [`Default`] trait +/// if the element type allows it. As a stopgap, trait implementations are +/// statically generated up to size 32. /// /// Arrays coerce to [slices (`[T]`)][slice], so a slice method may be called on /// an array. Indeed, this provides most of the API for working with arrays. -/// Slices have a dynamic size and do not coerce to arrays. /// -/// You can move elements out of an array with a slice pattern. If you want -/// one element, see [`mem::replace`][replace]. +/// Slices have a dynamic size and do not coerce to arrays. Instead, use +/// `slice.try_into().unwrap()` or `::try_from(slice).unwrap()`. +/// +/// Array's `try_from(slice)` implementations (and the corresponding `slice.try_into()` +/// array implementations) succeed if the input slice length is the same as the result +/// array length. They optimize especially well when the optimizer can easily determine +/// the slice length, e.g. `<[u8; 4]>::try_from(&slice[4..8]).unwrap()`. Array implements +/// [TryFrom](crate::convert::TryFrom) returning: +/// +/// - `[T; N]` copies from the slice's elements +/// - `&[T; N]` references the original slice's elements +/// - `&mut [T; N]` references the original slice's elements +/// +/// You can move elements out of an array with a [slice pattern]. If you want +/// one element, see [`mem::replace`]. /// /// # Examples /// @@ -500,36 +639,29 @@ mod prim_pointer {} /// assert_eq!([1, 2], &array[1..]); /// /// // This loop prints: 0 1 2 -/// for x in &array { -/// print!("{} ", x); +/// for x in array { +/// print!("{x} "); /// } /// ``` /// -/// An array itself is not iterable: +/// You can also iterate over reference to the array's elements: /// -/// ```compile_fail,E0277 -/// let array: [i32; 3] = [0; 3]; -/// -/// for x in array { } -/// // error: the trait bound `[i32; 3]: std::iter::Iterator` is not satisfied /// ``` +/// let array: [i32; 3] = [0; 3]; /// -/// The solution is to coerce the array to a slice by calling a slice method: -/// -/// ``` -/// # let array: [i32; 3] = [0; 3]; -/// for x in array.iter() { } +/// for x in &array { } /// ``` /// -/// If the array has 32 or fewer elements (see above), you can also use the -/// array reference's [`IntoIterator`] implementation: +/// You can use `::try_from(slice)` or `slice.try_into()` to get an array from +/// a slice: /// /// ``` -/// # let array: [i32; 3] = [0; 3]; -/// for x in &array { } +/// let bytes: [u8; 3] = [1, 0, 2]; +/// assert_eq!(1, u16::from_le_bytes(<[u8; 2]>::try_from(&bytes[0..2]).unwrap())); +/// assert_eq!(512, u16::from_le_bytes(bytes[1..3].try_into().unwrap())); /// ``` /// -/// You can use a slice pattern to move elements out of an array: +/// You can use a [slice pattern] to move elements out of an array: /// /// ``` /// fn move_away(_: String) { /* Do interesting things. */ } @@ -539,24 +671,108 @@ mod prim_pointer {} /// move_away(roa); /// ``` /// -/// [slice]: primitive.slice.html -/// [copy]: marker/trait.Copy.html -/// [clone]: clone/trait.Clone.html -/// [debug]: fmt/trait.Debug.html -/// [intoiterator]: iter/trait.IntoIterator.html -/// [partialeq]: cmp/trait.PartialEq.html -/// [partialord]: cmp/trait.PartialOrd.html -/// [eq]: cmp/trait.Eq.html -/// [ord]: cmp/trait.Ord.html -/// [hash]: hash/trait.Hash.html -/// [asref]: convert/trait.AsRef.html -/// [asmut]: convert/trait.AsMut.html -/// [borrow]: borrow/trait.Borrow.html -/// [borrowmut]: borrow/trait.BorrowMut.html -/// [default]: default/trait.Default.html -/// [replace]: mem/fn.replace.html -/// [`IntoIterator`]: iter/trait.IntoIterator.html +/// # Editions +/// +/// Prior to Rust 1.53, arrays did not implement [`IntoIterator`] by value, so the method call +/// `array.into_iter()` auto-referenced into a [slice iterator](slice::iter). Right now, the old +/// behavior is preserved in the 2015 and 2018 editions of Rust for compatibility, ignoring +/// [`IntoIterator`] by value. In the future, the behavior on the 2015 and 2018 edition +/// might be made consistent to the behavior of later editions. +/// +/// ```rust,edition2018 +/// // Rust 2015 and 2018: +/// +/// # #![allow(array_into_iter)] // override our `deny(warnings)` +/// let array: [i32; 3] = [0; 3]; +/// +/// // This creates a slice iterator, producing references to each value. +/// for item in array.into_iter().enumerate() { +/// let (i, x): (usize, &i32) = item; +/// println!("array[{i}] = {x}"); +/// } +/// +/// // The `array_into_iter` lint suggests this change for future compatibility: +/// for item in array.iter().enumerate() { +/// let (i, x): (usize, &i32) = item; +/// println!("array[{i}] = {x}"); +/// } +/// +/// // You can explicitly iterate an array by value using `IntoIterator::into_iter` +/// for item in IntoIterator::into_iter(array).enumerate() { +/// let (i, x): (usize, i32) = item; +/// println!("array[{i}] = {x}"); +/// } +/// ``` +/// +/// Starting in the 2021 edition, `array.into_iter()` uses `IntoIterator` normally to iterate +/// by value, and `iter()` should be used to iterate by reference like previous editions. +/// +/// ```rust,edition2021 +/// // Rust 2021: +/// +/// let array: [i32; 3] = [0; 3]; +/// +/// // This iterates by reference: +/// for item in array.iter().enumerate() { +/// let (i, x): (usize, &i32) = item; +/// println!("array[{i}] = {x}"); +/// } +/// +/// // This iterates by value: +/// for item in array.into_iter().enumerate() { +/// let (i, x): (usize, i32) = item; +/// println!("array[{i}] = {x}"); +/// } +/// ``` +/// +/// Future language versions might start treating the `array.into_iter()` +/// syntax on editions 2015 and 2018 the same as on edition 2021. So code using +/// those older editions should still be written with this change in mind, to +/// prevent breakage in the future. The safest way to accomplish this is to +/// avoid the `into_iter` syntax on those editions. If an edition update is not +/// viable/desired, there are multiple alternatives: +/// * use `iter`, equivalent to the old behavior, creating references +/// * use [`IntoIterator::into_iter`], equivalent to the post-2021 behavior (Rust 1.53+) +/// * replace `for ... in array.into_iter() {` with `for ... in array {`, +/// equivalent to the post-2021 behavior (Rust 1.53+) +/// +/// ```rust,edition2018 +/// // Rust 2015 and 2018: +/// +/// let array: [i32; 3] = [0; 3]; +/// +/// // This iterates by reference: +/// for item in array.iter() { +/// let x: &i32 = item; +/// println!("{x}"); +/// } +/// +/// // This iterates by value: +/// for item in IntoIterator::into_iter(array) { +/// let x: i32 = item; +/// println!("{x}"); +/// } /// +/// // This iterates by value: +/// for item in array { +/// let x: i32 = item; +/// println!("{x}"); +/// } +/// +/// // IntoIter can also start a chain. +/// // This iterates by value: +/// for item in IntoIterator::into_iter(array).enumerate() { +/// let (i, x): (usize, i32) = item; +/// println!("array[{i}] = {x}"); +/// } +/// ``` +/// +/// [slice]: prim@slice +/// [`Debug`]: fmt::Debug +/// [`Hash`]: hash::Hash +/// [`Borrow`]: borrow::Borrow +/// [`BorrowMut`]: borrow::BorrowMut +/// [slice pattern]: ../reference/patterns.html#slice-patterns #[stable(feature = "rust1", since = "1.0.0")] mod prim_array {} @@ -568,7 +784,7 @@ mod prim_array {} /// means that elements are laid out so that every element is the same /// distance from its neighbors. /// -/// *[See also the `std::slice` module](slice/index.html).* +/// *[See also the `std::slice` module](crate::slice).* /// /// Slices are a view into a block of memory represented as a pointer and a /// length. @@ -592,14 +808,70 @@ mod prim_array {} /// x[1] = 7; /// assert_eq!(x, &[1, 7, 3]); /// ``` +/// +/// As slices store the length of the sequence they refer to, they have twice +/// the size of pointers to [`Sized`](marker/trait.Sized.html) types. +/// Also see the reference on +/// [dynamically sized types](../reference/dynamically-sized-types.html). +/// +/// ``` +/// # use std::rc::Rc; +/// let pointer_size = std::mem::size_of::<&u8>(); +/// assert_eq!(2 * pointer_size, std::mem::size_of::<&[u8]>()); +/// assert_eq!(2 * pointer_size, std::mem::size_of::<*const [u8]>()); +/// assert_eq!(2 * pointer_size, std::mem::size_of::>()); +/// assert_eq!(2 * pointer_size, std::mem::size_of::>()); +/// ``` +/// +/// ## Trait Implementations +/// +/// Some traits are implemented for slices if the element type implements +/// that trait. This includes [`Eq`], [`Hash`] and [`Ord`]. +/// +/// ## Iteration +/// +/// The slices implement `IntoIterator`. The iterator yields references to the +/// slice elements. +/// +/// ``` +/// let numbers: &[i32] = &[0, 1, 2]; +/// for n in numbers { +/// println!("{n} is a number!"); +/// } +/// ``` +/// +/// The mutable slice yields mutable references to the elements: +/// +/// ``` +/// let mut scores: &mut [i32] = &mut [7, 8, 9]; +/// for score in scores { +/// *score += 1; +/// } +/// ``` +/// +/// This iterator yields mutable references to the slice's elements, so while +/// the element type of the slice is `i32`, the element type of the iterator is +/// `&mut i32`. +/// +/// * [`.iter`] and [`.iter_mut`] are the explicit methods to return the default +/// iterators. +/// * Further methods that return iterators are [`.split`], [`.splitn`], +/// [`.chunks`], [`.windows`] and more. +/// +/// [`Hash`]: core::hash::Hash +/// [`.iter`]: slice::iter +/// [`.iter_mut`]: slice::iter_mut +/// [`.split`]: slice::split +/// [`.splitn`]: slice::splitn +/// [`.chunks`]: slice::chunks +/// [`.windows`]: slice::windows #[stable(feature = "rust1", since = "1.0.0")] mod prim_slice {} #[doc(primitive = "str")] -// /// String slices. /// -/// *[See also the `std::str` module](str/index.html).* +/// *[See also the `std::str` module](crate::str).* /// /// The `str` type, also called a 'string slice', is the most primitive string /// type. It is usually seen in its borrowed form, `&str`. It is also the type @@ -607,19 +879,22 @@ mod prim_slice {} /// /// String slices are always valid UTF-8. /// -/// # Examples +/// # Basic Usage /// /// String literals are string slices: /// /// ``` -/// let hello = "Hello, world!"; -/// -/// // with an explicit type annotation -/// let hello: &'static str = "Hello, world!"; +/// let hello_world = "Hello, World!"; /// ``` /// -/// They are `'static` because they're stored directly in the final binary, and -/// so will be valid for the `'static` duration. +/// Here we have declared a string slice initialized with a string literal. +/// String literals have a static lifetime, which means the string `hello_world` +/// is guaranteed to be valid for the duration of the entire program. +/// We can explicitly specify `hello_world`'s lifetime as well: +/// +/// ``` +/// let hello_world: &'static str = "Hello, world!"; +/// ``` /// /// # Representation /// @@ -651,8 +926,8 @@ mod prim_slice {} /// assert_eq!(s, Ok(story)); /// ``` /// -/// [`as_ptr`]: #method.as_ptr -/// [`len`]: #method.len +/// [`as_ptr`]: str::as_ptr +/// [`len`]: str::len /// /// Note: This example shows the internals of `&str`. `unsafe` should not be /// used to get a string slice under normal circumstances. Use `as_str` @@ -700,18 +975,23 @@ mod prim_str {} /// ``` /// /// The sequential nature of the tuple applies to its implementations of various -/// traits. For example, in `PartialOrd` and `Ord`, the elements are compared +/// traits. For example, in [`PartialOrd`] and [`Ord`], the elements are compared /// sequentially until the first non-equal set is found. /// /// For more about tuples, see [the book](../book/ch03-02-data-types.html#the-tuple-type). /// +// Hardcoded anchor in src/librustdoc/html/format.rs +// linked to as `#trait-implementations-1` /// # Trait implementations /// -/// If every type inside a tuple implements one of the following traits, then a -/// tuple itself also implements it. +/// In this documentation the shorthand `(T₁, T₂, …, Tₙ)` is used to represent tuples of varying +/// length. When that is used, any trait bound expressed on `T` applies to each element of the +/// tuple independently. Note that this is a convenience notation to avoid repetitive +/// documentation, not valid Rust syntax. +/// +/// Due to a temporary restriction in Rust’s type system, the following traits are only +/// implemented on tuples of arity 12 or less. In the future, this may change: /// -/// * [`Clone`] -/// * [`Copy`] /// * [`PartialEq`] /// * [`Eq`] /// * [`PartialOrd`] @@ -720,18 +1000,24 @@ mod prim_str {} /// * [`Default`] /// * [`Hash`] /// -/// [`Clone`]: clone/trait.Clone.html -/// [`Copy`]: marker/trait.Copy.html -/// [`PartialEq`]: cmp/trait.PartialEq.html -/// [`Eq`]: cmp/trait.Eq.html -/// [`PartialOrd`]: cmp/trait.PartialOrd.html -/// [`Ord`]: cmp/trait.Ord.html -/// [`Debug`]: fmt/trait.Debug.html -/// [`Default`]: default/trait.Default.html -/// [`Hash`]: hash/trait.Hash.html +/// [`Debug`]: fmt::Debug +/// [`Hash`]: hash::Hash /// -/// Due to a temporary restriction in Rust's type system, these traits are only -/// implemented on tuples of arity 12 or less. In the future, this may change. +/// The following traits are implemented for tuples of any length. These traits have +/// implementations that are automatically generated by the compiler, so are not limited by +/// missing language features. +/// +/// * [`Clone`] +/// * [`Copy`] +/// * [`Send`] +/// * [`Sync`] +/// * [`Unpin`] +/// * [`UnwindSafe`] +/// * [`RefUnwindSafe`] +/// +/// [`Unpin`]: marker::Unpin +/// [`UnwindSafe`]: panic::UnwindSafe +/// [`RefUnwindSafe`]: panic::RefUnwindSafe /// /// # Examples /// @@ -768,20 +1054,109 @@ mod prim_str {} #[stable(feature = "rust1", since = "1.0.0")] mod prim_tuple {} +// Required to make auto trait impls render. +// See src/librustdoc/passes/collect_trait_impls.rs:collect_trait_impls +#[doc(hidden)] +impl (T,) {} + +// Fake impl that's only really used for docs. +#[cfg(doc)] +#[stable(feature = "rust1", since = "1.0.0")] +#[doc(fake_variadic)] +/// This trait is implemented on arbitrary-length tuples. +impl Clone for (T,) { + fn clone(&self) -> Self { + loop {} + } +} + +// Fake impl that's only really used for docs. +#[cfg(doc)] +#[stable(feature = "rust1", since = "1.0.0")] +#[doc(fake_variadic)] +/// This trait is implemented on arbitrary-length tuples. +impl Copy for (T,) { + // empty +} + #[doc(primitive = "f32")] -/// The 32-bit floating point type. -/// -/// *[See also the `std::f32::consts` module](f32/consts/index.html).* -/// +/// A 32-bit floating point type (specifically, the "binary32" type defined in IEEE 754-2008). +/// +/// This type can represent a wide range of decimal numbers, like `3.5`, `27`, +/// `-113.75`, `0.0078125`, `34359738368`, `0`, `-1`. So unlike integer types +/// (such as `i32`), floating point types can represent non-integer numbers, +/// too. +/// +/// However, being able to represent this wide range of numbers comes at the +/// cost of precision: floats can only represent some of the real numbers and +/// calculation with floats round to a nearby representable number. For example, +/// `5.0` and `1.0` can be exactly represented as `f32`, but `1.0 / 5.0` results +/// in `0.20000000298023223876953125` since `0.2` cannot be exactly represented +/// as `f32`. Note, however, that printing floats with `println` and friends will +/// often discard insignificant digits: `println!("{}", 1.0f32 / 5.0f32)` will +/// print `0.2`. +/// +/// Additionally, `f32` can represent some special values: +/// +/// - −0.0: IEEE 754 floating point numbers have a bit that indicates their sign, so −0.0 is a +/// possible value. For comparison −0.0 = +0.0, but floating point operations can carry +/// the sign bit through arithmetic operations. This means −0.0 × +0.0 produces −0.0 and +/// a negative number rounded to a value smaller than a float can represent also produces −0.0. +/// - [∞](#associatedconstant.INFINITY) and +/// [−∞](#associatedconstant.NEG_INFINITY): these result from calculations +/// like `1.0 / 0.0`. +/// - [NaN (not a number)](#associatedconstant.NAN): this value results from +/// calculations like `(-1.0).sqrt()`. NaN has some potentially unexpected +/// behavior: +/// - It is unequal to any float, including itself! This is the reason `f32` +/// doesn't implement the `Eq` trait. +/// - It is also neither smaller nor greater than any float, making it +/// impossible to sort by the default comparison operation, which is the +/// reason `f32` doesn't implement the `Ord` trait. +/// - It is also considered *infectious* as almost all calculations where one +/// of the operands is NaN will also result in NaN. The explanations on this +/// page only explicitly document behavior on NaN operands if this default +/// is deviated from. +/// - Lastly, there are multiple bit patterns that are considered NaN. +/// Rust does not currently guarantee that the bit patterns of NaN are +/// preserved over arithmetic operations, and they are not guaranteed to be +/// portable or even fully deterministic! This means that there may be some +/// surprising results upon inspecting the bit patterns, +/// as the same calculations might produce NaNs with different bit patterns. +/// +/// When the number resulting from a primitive operation (addition, +/// subtraction, multiplication, or division) on this type is not exactly +/// representable as `f32`, it is rounded according to the roundTiesToEven +/// direction defined in IEEE 754-2008. That means: +/// +/// - The result is the representable value closest to the true value, if there +/// is a unique closest representable value. +/// - If the true value is exactly half-way between two representable values, +/// the result is the one with an even least-significant binary digit. +/// - If the true value's magnitude is ≥ `f32::MAX` + 2(`f32::MAX_EXP` − +/// `f32::MANTISSA_DIGITS` − 1), the result is ∞ or −∞ (preserving the +/// true value's sign). +/// +/// For more information on floating point numbers, see [Wikipedia][wikipedia]. +/// +/// *[See also the `std::f32::consts` module](crate::f32::consts).* +/// +/// [wikipedia]: https://en.wikipedia.org/wiki/Single-precision_floating-point_format #[stable(feature = "rust1", since = "1.0.0")] mod prim_f32 {} #[doc(primitive = "f64")] -// -/// The 64-bit floating point type. +/// A 64-bit floating point type (specifically, the "binary64" type defined in IEEE 754-2008). +/// +/// This type is very similar to [`f32`], but has increased +/// precision by using twice as many bits. Please see [the documentation for +/// `f32`][`f32`] or [Wikipedia on double precision +/// values][wikipedia] for more information. /// -/// *[See also the `std::f64::consts` module](f64/consts/index.html).* +/// *[See also the `std::f64::consts` module](crate::f64::consts).* /// +/// [`f32`]: prim@f32 +/// [wikipedia]: https://en.wikipedia.org/wiki/Double-precision_floating-point_format #[stable(feature = "rust1", since = "1.0.0")] mod prim_f64 {} @@ -867,18 +1242,21 @@ mod prim_usize {} #[doc(primitive = "reference")] #[doc(alias = "&")] +#[doc(alias = "&mut")] // -/// References, both shared and mutable. +/// References, `&T` and `&mut T`. /// /// A reference represents a borrow of some owned value. You can get one by using the `&` or `&mut` -/// operators on a value, or by using a `ref` or `ref mut` pattern. +/// operators on a value, or by using a [`ref`](../std/keyword.ref.html) or +/// [ref](../std/keyword.ref.html) [mut](../std/keyword.mut.html) pattern. /// /// For those familiar with pointers, a reference is just a pointer that is assumed to be /// aligned, not null, and pointing to memory containing a valid value of `T` - for example, -/// `&bool` can only point to an allocation containing the integer values `1` (`true`) or `0` -/// (`false`), but creating a `&bool` that points to an allocation containing +/// &[bool] can only point to an allocation containing the integer values `1` +/// ([`true`](../std/keyword.true.html)) or `0` ([`false`](../std/keyword.false.html)), but +/// creating a &[bool] that points to an allocation containing /// the value `3` causes undefined behaviour. -/// In fact, `Option<&T>` has the same memory representation as a +/// In fact, [Option]\<&T> has the same memory representation as a /// nullable but aligned pointer, and can be passed across FFI boundaries as such. /// /// In most cases, references can be used much like the original value. Field access, method @@ -900,9 +1278,6 @@ mod prim_usize {} /// implicit reference-pointer coercion and raw pointer equality via [`ptr::eq`], while /// [`PartialEq`] compares values. /// -/// [`ptr::eq`]: ptr/fn.eq.html -/// [`PartialEq`]: cmp/trait.PartialEq.html -/// /// ``` /// use std::ptr; /// @@ -932,13 +1307,10 @@ mod prim_usize {} /// * [`Clone`] \(Note that this will not defer to `T`'s `Clone` implementation if it exists!) /// * [`Deref`] /// * [`Borrow`] -/// * [`Pointer`] +/// * [`fmt::Pointer`] /// -/// [`Copy`]: marker/trait.Copy.html -/// [`Clone`]: clone/trait.Clone.html -/// [`Deref`]: ops/trait.Deref.html -/// [`Borrow`]: borrow/trait.Borrow.html -/// [`Pointer`]: fmt/trait.Pointer.html +/// [`Deref`]: ops::Deref +/// [`Borrow`]: borrow::Borrow /// /// `&mut T` references get all of the above except `Copy` and `Clone` (to prevent creating /// multiple simultaneous mutable borrows), plus the following, regardless of the type of its @@ -947,13 +1319,14 @@ mod prim_usize {} /// * [`DerefMut`] /// * [`BorrowMut`] /// -/// [`DerefMut`]: ops/trait.DerefMut.html -/// [`BorrowMut`]: borrow/trait.BorrowMut.html +/// [`DerefMut`]: ops::DerefMut +/// [`BorrowMut`]: borrow::BorrowMut +/// [bool]: prim@bool /// /// The following traits are implemented on `&T` references if the underlying `T` also implements /// that trait: /// -/// * All the traits in [`std::fmt`] except [`Pointer`] and [`fmt::Write`] +/// * All the traits in [`std::fmt`] except [`fmt::Pointer`] (which is implemented regardless of the type of its referent) and [`fmt::Write`] /// * [`PartialOrd`] /// * [`Ord`] /// * [`PartialEq`] @@ -962,19 +1335,11 @@ mod prim_usize {} /// * [`Fn`] \(in addition, `&T` references get [`FnMut`] and [`FnOnce`] if `T: Fn`) /// * [`Hash`] /// * [`ToSocketAddrs`] +/// * [`Send`] \(`&T` references also require T: [Sync]) /// -/// [`std::fmt`]: fmt/index.html -/// [`fmt::Write`]: fmt/trait.Write.html -/// [`PartialOrd`]: cmp/trait.PartialOrd.html -/// [`Ord`]: cmp/trait.Ord.html -/// [`PartialEq`]: cmp/trait.PartialEq.html -/// [`Eq`]: cmp/trait.Eq.html -/// [`AsRef`]: convert/trait.AsRef.html -/// [`Fn`]: ops/trait.Fn.html -/// [`FnMut`]: ops/trait.FnMut.html -/// [`FnOnce`]: ops/trait.FnOnce.html -/// [`Hash`]: hash/trait.Hash.html -/// [`ToSocketAddrs`]: net/trait.ToSocketAddrs.html +/// [`std::fmt`]: fmt +/// [`Hash`]: hash::Hash +#[doc = concat!("[`ToSocketAddrs`]: ", include_str!("../primitive_docs/net_tosocketaddrs.md"))] /// /// `&mut T` references get all of the above except `ToSocketAddrs`, plus the following, if `T` /// implements that trait: @@ -987,23 +1352,17 @@ mod prim_usize {} /// * [`ExactSizeIterator`] /// * [`FusedIterator`] /// * [`TrustedLen`] -/// * [`Send`] \(note that `&T` references only get `Send` if `T: Sync`) /// * [`io::Write`] /// * [`Read`] /// * [`Seek`] /// * [`BufRead`] /// -/// [`AsMut`]: convert/trait.AsMut.html -/// [`Iterator`]: iter/trait.Iterator.html -/// [`DoubleEndedIterator`]: iter/trait.DoubleEndedIterator.html -/// [`ExactSizeIterator`]: iter/trait.ExactSizeIterator.html -/// [`FusedIterator`]: iter/trait.FusedIterator.html -/// [`TrustedLen`]: iter/trait.TrustedLen.html -/// [`Send`]: marker/trait.Send.html -/// [`io::Write`]: io/trait.Write.html -/// [`Read`]: io/trait.Read.html -/// [`Seek`]: io/trait.Seek.html -/// [`BufRead`]: io/trait.BufRead.html +/// [`FusedIterator`]: iter::FusedIterator +/// [`TrustedLen`]: iter::TrustedLen +#[doc = concat!("[`Seek`]: ", include_str!("../primitive_docs/io_seek.md"))] +#[doc = concat!("[`BufRead`]: ", include_str!("../primitive_docs/io_bufread.md"))] +#[doc = concat!("[`Read`]: ", include_str!("../primitive_docs/io_read.md"))] +#[doc = concat!("[`io::Write`]: ", include_str!("../primitive_docs/io_write.md"))] /// /// Note that due to method call deref coercion, simply calling a trait method will act like they /// work on references as well as they do on owned values! The implementations described here are @@ -1018,14 +1377,17 @@ mod prim_ref {} /// /// *See also the traits [`Fn`], [`FnMut`], and [`FnOnce`].* /// -/// [`Fn`]: ops/trait.Fn.html -/// [`FnMut`]: ops/trait.FnMut.html -/// [`FnOnce`]: ops/trait.FnOnce.html +/// [`Fn`]: ops::Fn +/// [`FnMut`]: ops::FnMut +/// [`FnOnce`]: ops::FnOnce /// /// Function pointers are pointers that point to *code*, not data. They can be called /// just like functions. Like references, function pointers are, among other things, assumed to /// not be null, so if you want to pass a function pointer over FFI and be able to accommodate null -/// pointers, make your type `Option` with your required signature. +/// pointers, make your type [`Option`](core::option#options-and-pointers-nullable-pointers) +/// with your required signature. +/// +/// ### Safety /// /// Plain function pointers are obtained by casting either plain functions, or closures that don't /// capture an environment: @@ -1064,27 +1426,99 @@ mod prim_ref {} /// let really_safe_ptr: unsafe fn(usize) -> usize = add_one; /// ``` /// -/// On top of that, function pointers can vary based on what ABI they use. This is achieved by -/// adding the `extern` keyword to the type name, followed by the ABI in question. For example, -/// `fn()` is different from `extern "C" fn()`, which itself is different from `extern "stdcall" -/// fn()`, and so on for the various ABIs that Rust supports. Non-`extern` functions have an ABI -/// of `"Rust"`, and `extern` functions without an explicit ABI have an ABI of `"C"`. For more -/// information, see [the nomicon's section on foreign calling conventions][nomicon-abi]. +/// ### ABI +/// +/// On top of that, function pointers can vary based on what ABI they use. This +/// is achieved by adding the `extern` keyword before the type, followed by the +/// ABI in question. The default ABI is "Rust", i.e., `fn()` is the exact same +/// type as `extern "Rust" fn()`. A pointer to a function with C ABI would have +/// type `extern "C" fn()`. +/// +/// `extern "ABI" { ... }` blocks declare functions with ABI "ABI". The default +/// here is "C", i.e., functions declared in an `extern {...}` block have "C" +/// ABI. +/// +/// For more information and a list of supported ABIs, see [the nomicon's +/// section on foreign calling conventions][nomicon-abi]. /// /// [nomicon-abi]: ../nomicon/ffi.html#foreign-calling-conventions /// +/// ### Variadic functions +/// /// Extern function declarations with the "C" or "cdecl" ABIs can also be *variadic*, allowing them -/// to be called with a variable number of arguments. Normal rust functions, even those with an +/// to be called with a variable number of arguments. Normal Rust functions, even those with an /// `extern "ABI"`, cannot be variadic. For more information, see [the nomicon's section on /// variadic functions][nomicon-variadic]. /// /// [nomicon-variadic]: ../nomicon/ffi.html#variadic-functions /// -/// These markers can be combined, so `unsafe extern "stdcall" fn()` is a valid type. +/// ### Creating function pointers /// -/// Function pointers implement the following traits: +/// When `bar` is the name of a function, then the expression `bar` is *not* a +/// function pointer. Rather, it denotes a value of an unnameable type that +/// uniquely identifies the function `bar`. The value is zero-sized because the +/// type already identifies the function. This has the advantage that "calling" +/// the value (it implements the `Fn*` traits) does not require dynamic +/// dispatch. +/// +/// This zero-sized type *coerces* to a regular function pointer. For example: +/// +/// ```rust +/// use std::mem; +/// +/// fn bar(x: i32) {} +/// +/// let not_bar_ptr = bar; // `not_bar_ptr` is zero-sized, uniquely identifying `bar` +/// assert_eq!(mem::size_of_val(¬_bar_ptr), 0); +/// +/// let bar_ptr: fn(i32) = not_bar_ptr; // force coercion to function pointer +/// assert_eq!(mem::size_of_val(&bar_ptr), mem::size_of::()); +/// +/// let footgun = &bar; // this is a shared reference to the zero-sized type identifying `bar` +/// ``` +/// +/// The last line shows that `&bar` is not a function pointer either. Rather, it +/// is a reference to the function-specific ZST. `&bar` is basically never what you +/// want when `bar` is a function. +/// +/// ### Casting to and from integers +/// +/// You cast function pointers directly to integers: +/// +/// ```rust +/// let fnptr: fn(i32) -> i32 = |x| x+2; +/// let fnptr_addr = fnptr as usize; +/// ``` +/// +/// However, a direct cast back is not possible. You need to use `transmute`: +/// +/// ```rust +/// # #[cfg(not(miri))] { // FIXME: use strict provenance APIs once they are stable, then remove this `cfg` +/// # let fnptr: fn(i32) -> i32 = |x| x+2; +/// # let fnptr_addr = fnptr as usize; +/// let fnptr = fnptr_addr as *const (); +/// let fnptr: fn(i32) -> i32 = unsafe { std::mem::transmute(fnptr) }; +/// assert_eq!(fnptr(40), 42); +/// # } +/// ``` +/// +/// Crucially, we `as`-cast to a raw pointer before `transmute`ing to a function pointer. +/// This avoids an integer-to-pointer `transmute`, which can be problematic. +/// Transmuting between raw pointers and function pointers (i.e., two pointer types) is fine. +/// +/// Note that all of this is not portable to platforms where function pointers and data pointers +/// have different sizes. +/// +/// ### Trait implementations +/// +/// In this documentation the shorthand `fn (T₁, T₂, …, Tₙ)` is used to represent non-variadic +/// function pointers of varying length. Note that this is a convenience notation to avoid +/// repetitive documentation, not valid Rust syntax. +/// +/// Due to a temporary restriction in Rust's type system, these traits are only implemented on +/// functions that take 12 arguments or less, with the `"Rust"` and `"C"` ABIs. In the future, this +/// may change: /// -/// * [`Clone`] /// * [`PartialEq`] /// * [`Eq`] /// * [`PartialOrd`] @@ -1093,23 +1527,49 @@ mod prim_ref {} /// * [`Pointer`] /// * [`Debug`] /// -/// [`Clone`]: clone/trait.Clone.html -/// [`PartialEq`]: cmp/trait.PartialEq.html -/// [`Eq`]: cmp/trait.Eq.html -/// [`PartialOrd`]: cmp/trait.PartialOrd.html -/// [`Ord`]: cmp/trait.Ord.html -/// [`Hash`]: hash/trait.Hash.html -/// [`Pointer`]: fmt/trait.Pointer.html -/// [`Debug`]: fmt/trait.Debug.html +/// The following traits are implemented for function pointers with any number of arguments and +/// any ABI. These traits have implementations that are automatically generated by the compiler, +/// so are not limited by missing language features: /// -/// Due to a temporary restriction in Rust's type system, these traits are only implemented on -/// functions that take 12 arguments or less, with the `"Rust"` and `"C"` ABIs. In the future, this -/// may change. -/// -/// In addition, function pointers of *any* signature, ABI, or safety are [`Copy`], and all *safe* -/// function pointers implement [`Fn`], [`FnMut`], and [`FnOnce`]. This works because these traits -/// are specially known to the compiler. -/// -/// [`Copy`]: marker/trait.Copy.html +/// * [`Clone`] +/// * [`Copy`] +/// * [`Send`] +/// * [`Sync`] +/// * [`Unpin`] +/// * [`UnwindSafe`] +/// * [`RefUnwindSafe`] +/// +/// [`Hash`]: hash::Hash +/// [`Pointer`]: fmt::Pointer +/// [`UnwindSafe`]: panic::UnwindSafe +/// [`RefUnwindSafe`]: panic::RefUnwindSafe +/// +/// In addition, all *safe* function pointers implement [`Fn`], [`FnMut`], and [`FnOnce`], because +/// these traits are specially known to the compiler. #[stable(feature = "rust1", since = "1.0.0")] mod prim_fn {} + +// Required to make auto trait impls render. +// See src/librustdoc/passes/collect_trait_impls.rs:collect_trait_impls +#[doc(hidden)] +impl fn(T) -> Ret {} + +// Fake impl that's only really used for docs. +#[cfg(doc)] +#[stable(feature = "rust1", since = "1.0.0")] +#[doc(fake_variadic)] +/// This trait is implemented on function pointers with any number of arguments. +impl Clone for fn(T) -> Ret { + fn clone(&self) -> Self { + loop {} + } +} + +// Fake impl that's only really used for docs. +#[cfg(doc)] +#[stable(feature = "rust1", since = "1.0.0")] +#[doc(fake_variadic)] +/// This trait is implemented on function pointers with any number of arguments. +impl Copy for fn(T) -> Ret { + // empty +} diff --git a/crux-mir/lib/std/src/process.rs b/crux-mir/lib/std/src/process.rs index 3eee45d00..62ce2cb33 100644 --- a/crux-mir/lib/std/src/process.rs +++ b/crux-mir/lib/std/src/process.rs @@ -71,11 +71,15 @@ //! .spawn() //! .expect("failed to execute child"); //! -//! { -//! // limited borrow of stdin -//! let stdin = child.stdin.as_mut().expect("failed to get stdin"); +//! // If the child process fills its stdout buffer, it may end up +//! // waiting until the parent reads the stdout, and not be able to +//! // read stdin in the meantime, causing a deadlock. +//! // Writing from another thread ensures that stdout is being read +//! // at the same time, avoiding the problem. +//! let mut stdin = child.stdin.take().expect("failed to get stdin"); +//! std::thread::spawn(move || { //! stdin.write_all(b"test").expect("failed to write to stdin"); -//! } +//! }); //! //! let output = child //! .wait_with_output() @@ -84,38 +88,36 @@ //! assert_eq!(b"test", output.stdout.as_slice()); //! ``` //! -//! [`abort`]: fn.abort.html -//! [`exit`]: fn.exit.html +//! [`spawn`]: Command::spawn +//! [`output`]: Command::output //! -//! [`Command`]: struct.Command.html -//! [`spawn`]: struct.Command.html#method.spawn -//! [`output`]: struct.Command.html#method.output +//! [`stdout`]: Command::stdout +//! [`stdin`]: Command::stdin +//! [`stderr`]: Command::stderr //! -//! [`Child`]: struct.Child.html -//! [`ChildStdin`]: struct.ChildStdin.html -//! [`ChildStdout`]: struct.ChildStdout.html -//! [`ChildStderr`]: struct.ChildStderr.html -//! [`Stdio`]: struct.Stdio.html -//! -//! [`stdout`]: struct.Command.html#method.stdout -//! [`stdin`]: struct.Command.html#method.stdin -//! [`stderr`]: struct.Command.html#method.stderr -//! -//! [`Write`]: ../io/trait.Write.html -//! [`Read`]: ../io/trait.Read.html +//! [`Write`]: io::Write +//! [`Read`]: io::Read #![stable(feature = "process", since = "1.0.0")] +#![deny(unsafe_op_in_unsafe_fn)] + +#[cfg(all(test, not(any(target_os = "emscripten", target_env = "sgx"))))] +mod tests; use crate::io::prelude::*; +use crate::convert::Infallible; use crate::ffi::OsStr; use crate::fmt; use crate::fs; -use crate::io::{self, Initializer, IoSlice, IoSliceMut}; +use crate::io::{self, IoSlice, IoSliceMut}; +use crate::num::NonZeroI32; use crate::path::Path; use crate::str; use crate::sys::pipe::{read2, AnonPipe}; use crate::sys::process as imp; +#[stable(feature = "command_access", since = "1.57.0")] +pub use crate::sys_common::process::CommandEnvs; use crate::sys_common::{AsInner, AsInnerMut, FromInner, IntoInner}; /// Representation of a running or exited child process. @@ -130,13 +132,13 @@ use crate::sys_common::{AsInner, AsInnerMut, FromInner, IntoInner}; /// run, even after the `Child` handle to the child process has gone out of /// scope. /// -/// Calling [`wait`](#method.wait) (or other functions that wrap around it) will make +/// Calling [`wait`] (or other functions that wrap around it) will make /// the parent process wait until the child has actually exited before /// continuing. /// /// # Warning /// -/// On some system, calling [`wait`] or similar is necessary for the OS to +/// On some systems, calling [`wait`] or similar is necessary for the OS to /// release resources. A process that terminated but has not been waited on is /// still around as a "zombie". Leaving too many zombies around may exhaust /// global resources (for example process IDs). @@ -162,29 +164,52 @@ use crate::sys_common::{AsInner, AsInnerMut, FromInner, IntoInner}; /// assert!(ecode.success()); /// ``` /// -/// [`Command`]: struct.Command.html -/// [`Drop`]: ../../core/ops/trait.Drop.html -/// [`wait`]: #method.wait +/// [`wait`]: Child::wait #[stable(feature = "process", since = "1.0.0")] pub struct Child { - handle: imp::Process, + pub(crate) handle: imp::Process, - /// The handle for writing to the child's standard input (stdin), if it has - /// been captured. + /// The handle for writing to the child's standard input (stdin), if it + /// has been captured. You might find it helpful to do + /// + /// ```compile_fail,E0425 + /// let stdin = child.stdin.take().unwrap(); + /// ``` + /// + /// to avoid partially moving the `child` and thus blocking yourself from calling + /// functions on `child` while using `stdin`. #[stable(feature = "process", since = "1.0.0")] pub stdin: Option, /// The handle for reading from the child's standard output (stdout), if it - /// has been captured. + /// has been captured. You might find it helpful to do + /// + /// ```compile_fail,E0425 + /// let stdout = child.stdout.take().unwrap(); + /// ``` + /// + /// to avoid partially moving the `child` and thus blocking yourself from calling + /// functions on `child` while using `stdout`. #[stable(feature = "process", since = "1.0.0")] pub stdout: Option, /// The handle for reading from the child's standard error (stderr), if it - /// has been captured. + /// has been captured. You might find it helpful to do + /// + /// ```compile_fail,E0425 + /// let stderr = child.stderr.take().unwrap(); + /// ``` + /// + /// to avoid partially moving the `child` and thus blocking yourself from calling + /// functions on `child` while using `stderr`. #[stable(feature = "process", since = "1.0.0")] pub stderr: Option, } +/// Allows extension traits within `std`. +#[unstable(feature = "sealed", issue = "none")] +impl crate::sealed::Sealed for Child {} + impl AsInner for Child { fn as_inner(&self) -> &imp::Process { &self.handle @@ -215,7 +240,7 @@ impl fmt::Debug for Child { .field("stdin", &self.stdin) .field("stdout", &self.stdout) .field("stderr", &self.stderr) - .finish() + .finish_non_exhaustive() } } @@ -227,16 +252,40 @@ impl fmt::Debug for Child { /// file handle will be closed. If the child process was blocked on input prior /// to being dropped, it will become unblocked after dropping. /// -/// [`Child`]: struct.Child.html -/// [`stdin`]: struct.Child.html#structfield.stdin -/// [dropped]: ../ops/trait.Drop.html +/// [`stdin`]: Child::stdin +/// [dropped]: Drop #[stable(feature = "process", since = "1.0.0")] pub struct ChildStdin { inner: AnonPipe, } +// In addition to the `impl`s here, `ChildStdin` also has `impl`s for +// `AsFd`/`From`/`Into` and +// `AsRawFd`/`IntoRawFd`/`FromRawFd`, on Unix and WASI, and +// `AsHandle`/`From`/`Into` and +// `AsRawHandle`/`IntoRawHandle`/`FromRawHandle` on Windows. + #[stable(feature = "process", since = "1.0.0")] impl Write for ChildStdin { + fn write(&mut self, buf: &[u8]) -> io::Result { + (&*self).write(buf) + } + + fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { + (&*self).write_vectored(bufs) + } + + fn is_write_vectored(&self) -> bool { + io::Write::is_write_vectored(&&*self) + } + + fn flush(&mut self) -> io::Result<()> { + (&*self).flush() + } +} + +#[stable(feature = "write_mt", since = "1.48.0")] +impl Write for &ChildStdin { fn write(&mut self, buf: &[u8]) -> io::Result { self.inner.write(buf) } @@ -245,6 +294,10 @@ impl Write for ChildStdin { self.inner.write_vectored(bufs) } + fn is_write_vectored(&self) -> bool { + self.inner.is_write_vectored() + } + fn flush(&mut self) -> io::Result<()> { Ok(()) } @@ -271,7 +324,7 @@ impl FromInner for ChildStdin { #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for ChildStdin { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("ChildStdin { .. }") + f.debug_struct("ChildStdin").finish_non_exhaustive() } } @@ -282,14 +335,19 @@ impl fmt::Debug for ChildStdin { /// When an instance of `ChildStdout` is [dropped], the `ChildStdout`'s /// underlying file handle will be closed. /// -/// [`Child`]: struct.Child.html -/// [`stdout`]: struct.Child.html#structfield.stdout -/// [dropped]: ../ops/trait.Drop.html +/// [`stdout`]: Child::stdout +/// [dropped]: Drop #[stable(feature = "process", since = "1.0.0")] pub struct ChildStdout { inner: AnonPipe, } +// In addition to the `impl`s here, `ChildStdout` also has `impl`s for +// `AsFd`/`From`/`Into` and +// `AsRawFd`/`IntoRawFd`/`FromRawFd`, on Unix and WASI, and +// `AsHandle`/`From`/`Into` and +// `AsRawHandle`/`IntoRawHandle`/`FromRawHandle` on Windows. + #[stable(feature = "process", since = "1.0.0")] impl Read for ChildStdout { fn read(&mut self, buf: &mut [u8]) -> io::Result { @@ -301,8 +359,12 @@ impl Read for ChildStdout { } #[inline] - unsafe fn initializer(&self) -> Initializer { - Initializer::nop() + fn is_read_vectored(&self) -> bool { + self.inner.is_read_vectored() + } + + fn read_to_end(&mut self, buf: &mut Vec) -> io::Result { + self.inner.read_to_end(buf) } } @@ -327,7 +389,7 @@ impl FromInner for ChildStdout { #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for ChildStdout { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("ChildStdout { .. }") + f.debug_struct("ChildStdout").finish_non_exhaustive() } } @@ -338,14 +400,19 @@ impl fmt::Debug for ChildStdout { /// When an instance of `ChildStderr` is [dropped], the `ChildStderr`'s /// underlying file handle will be closed. /// -/// [`Child`]: struct.Child.html -/// [`stderr`]: struct.Child.html#structfield.stderr -/// [dropped]: ../ops/trait.Drop.html +/// [`stderr`]: Child::stderr +/// [dropped]: Drop #[stable(feature = "process", since = "1.0.0")] pub struct ChildStderr { inner: AnonPipe, } +// In addition to the `impl`s here, `ChildStderr` also has `impl`s for +// `AsFd`/`From`/`Into` and +// `AsRawFd`/`IntoRawFd`/`FromRawFd`, on Unix and WASI, and +// `AsHandle`/`From`/`Into` and +// `AsRawHandle`/`IntoRawHandle`/`FromRawHandle` on Windows. + #[stable(feature = "process", since = "1.0.0")] impl Read for ChildStderr { fn read(&mut self, buf: &mut [u8]) -> io::Result { @@ -357,8 +424,8 @@ impl Read for ChildStderr { } #[inline] - unsafe fn initializer(&self) -> Initializer { - Initializer::nop() + fn is_read_vectored(&self) -> bool { + self.inner.is_read_vectored() } } @@ -383,7 +450,7 @@ impl FromInner for ChildStderr { #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for ChildStderr { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("ChildStderr { .. }") + f.debug_struct("ChildStderr").finish_non_exhaustive() } } @@ -400,7 +467,7 @@ impl fmt::Debug for ChildStderr { /// /// let output = if cfg!(target_os = "windows") { /// Command::new("cmd") -/// .args(&["/C", "echo hello"]) +/// .args(["/C", "echo hello"]) /// .output() /// .expect("failed to execute process") /// } else { @@ -451,6 +518,10 @@ pub struct Command { inner: imp::Command, } +/// Allows extension traits within `std`. +#[unstable(feature = "sealed", issue = "none")] +impl crate::sealed::Sealed for Command {} + impl Command { /// Constructs a new `Command` for launching the program at /// path `program`, with the following default configuration: @@ -458,7 +529,11 @@ impl Command { /// * No arguments to the program /// * Inherit the current process's environment /// * Inherit the current process's working directory - /// * Inherit stdin/stdout/stderr for `spawn` or `status`, but create pipes for `output` + /// * Inherit stdin/stdout/stderr for [`spawn`] or [`status`], but create pipes for [`output`] + /// + /// [`spawn`]: Self::spawn + /// [`status`]: Self::status + /// [`output`]: Self::output /// /// Builder methods are provided to change these defaults and /// otherwise configure the process. @@ -508,7 +583,12 @@ impl Command { /// /// To pass multiple arguments see [`args`]. /// - /// [`args`]: #method.args + /// [`args`]: Command::args + /// + /// Note that the argument is not passed through a shell, but given + /// literally to the program. This means that shell syntax like quotes, + /// escaped characters, word splitting, glob patterns, substitution, etc. + /// have no effect. /// /// # Examples /// @@ -533,7 +613,12 @@ impl Command { /// /// To pass a single argument see [`arg`]. /// - /// [`arg`]: #method.arg + /// [`arg`]: Command::arg + /// + /// Note that the arguments are not passed through a shell, but given + /// literally to the program. This means that shell syntax like quotes, + /// escaped characters, word splitting, glob patterns, substitution, etc. + /// have no effect. /// /// # Examples /// @@ -543,7 +628,7 @@ impl Command { /// use std::process::Command; /// /// Command::new("ls") - /// .args(&["-l", "-a"]) + /// .args(["-l", "-a"]) /// .spawn() /// .expect("ls command failed to start"); /// ``` @@ -686,7 +771,7 @@ impl Command { /// .expect("ls command failed to start"); /// ``` /// - /// [`canonicalize`]: ../fs/fn.canonicalize.html + /// [`canonicalize`]: crate::fs::canonicalize #[stable(feature = "process", since = "1.0.0")] pub fn current_dir>(&mut self, dir: P) -> &mut Command { self.inner.cwd(dir.as_ref().as_ref()); @@ -695,11 +780,14 @@ impl Command { /// Configuration for the child process's standard input (stdin) handle. /// - /// Defaults to [`inherit`] when used with `spawn` or `status`, and - /// defaults to [`piped`] when used with `output`. + /// Defaults to [`inherit`] when used with [`spawn`] or [`status`], and + /// defaults to [`piped`] when used with [`output`]. /// - /// [`inherit`]: struct.Stdio.html#method.inherit - /// [`piped`]: struct.Stdio.html#method.piped + /// [`inherit`]: Stdio::inherit + /// [`piped`]: Stdio::piped + /// [`spawn`]: Self::spawn + /// [`status`]: Self::status + /// [`output`]: Self::output /// /// # Examples /// @@ -721,11 +809,14 @@ impl Command { /// Configuration for the child process's standard output (stdout) handle. /// - /// Defaults to [`inherit`] when used with `spawn` or `status`, and - /// defaults to [`piped`] when used with `output`. + /// Defaults to [`inherit`] when used with [`spawn`] or [`status`], and + /// defaults to [`piped`] when used with [`output`]. /// - /// [`inherit`]: struct.Stdio.html#method.inherit - /// [`piped`]: struct.Stdio.html#method.piped + /// [`inherit`]: Stdio::inherit + /// [`piped`]: Stdio::piped + /// [`spawn`]: Self::spawn + /// [`status`]: Self::status + /// [`output`]: Self::output /// /// # Examples /// @@ -747,11 +838,14 @@ impl Command { /// Configuration for the child process's standard error (stderr) handle. /// - /// Defaults to [`inherit`] when used with `spawn` or `status`, and - /// defaults to [`piped`] when used with `output`. + /// Defaults to [`inherit`] when used with [`spawn`] or [`status`], and + /// defaults to [`piped`] when used with [`output`]. /// - /// [`inherit`]: struct.Stdio.html#method.inherit - /// [`piped`]: struct.Stdio.html#method.piped + /// [`inherit`]: Stdio::inherit + /// [`piped`]: Stdio::piped + /// [`spawn`]: Self::spawn + /// [`status`]: Self::status + /// [`output`]: Self::output /// /// # Examples /// @@ -817,14 +911,12 @@ impl Command { /// ``` #[stable(feature = "process", since = "1.0.0")] pub fn output(&mut self) -> io::Result { - self.inner - .spawn(imp::Stdio::MakePipe, false) - .map(Child::from_inner) - .and_then(|p| p.wait_with_output()) + let (status, stdout, stderr) = self.inner.output()?; + Ok(Output { status: ExitStatus(status), stdout, stderr }) } /// Executes a command as a child process, waiting for it to finish and - /// collecting its exit status. + /// collecting its status. /// /// By default, stdin, stdout and stderr are inherited from the parent. /// @@ -838,7 +930,7 @@ impl Command { /// .status() /// .expect("failed to execute process"); /// - /// println!("process exited with: {}", status); + /// println!("process finished with: {status}"); /// /// assert!(status.success()); /// ``` @@ -849,6 +941,96 @@ impl Command { .map(Child::from_inner) .and_then(|mut p| p.wait()) } + + /// Returns the path to the program that was given to [`Command::new`]. + /// + /// # Examples + /// + /// ``` + /// use std::process::Command; + /// + /// let cmd = Command::new("echo"); + /// assert_eq!(cmd.get_program(), "echo"); + /// ``` + #[must_use] + #[stable(feature = "command_access", since = "1.57.0")] + pub fn get_program(&self) -> &OsStr { + self.inner.get_program() + } + + /// Returns an iterator of the arguments that will be passed to the program. + /// + /// This does not include the path to the program as the first argument; + /// it only includes the arguments specified with [`Command::arg`] and + /// [`Command::args`]. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::OsStr; + /// use std::process::Command; + /// + /// let mut cmd = Command::new("echo"); + /// cmd.arg("first").arg("second"); + /// let args: Vec<&OsStr> = cmd.get_args().collect(); + /// assert_eq!(args, &["first", "second"]); + /// ``` + #[stable(feature = "command_access", since = "1.57.0")] + pub fn get_args(&self) -> CommandArgs<'_> { + CommandArgs { inner: self.inner.get_args() } + } + + /// Returns an iterator of the environment variables that will be set when + /// the process is spawned. + /// + /// Each element is a tuple `(&OsStr, Option<&OsStr>)`, where the first + /// value is the key, and the second is the value, which is [`None`] if + /// the environment variable is to be explicitly removed. + /// + /// This only includes environment variables explicitly set with + /// [`Command::env`], [`Command::envs`], and [`Command::env_remove`]. It + /// does not include environment variables that will be inherited by the + /// child process. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::OsStr; + /// use std::process::Command; + /// + /// let mut cmd = Command::new("ls"); + /// cmd.env("TERM", "dumb").env_remove("TZ"); + /// let envs: Vec<(&OsStr, Option<&OsStr>)> = cmd.get_envs().collect(); + /// assert_eq!(envs, &[ + /// (OsStr::new("TERM"), Some(OsStr::new("dumb"))), + /// (OsStr::new("TZ"), None) + /// ]); + /// ``` + #[stable(feature = "command_access", since = "1.57.0")] + pub fn get_envs(&self) -> CommandEnvs<'_> { + self.inner.get_envs() + } + + /// Returns the working directory for the child process. + /// + /// This returns [`None`] if the working directory will not be changed. + /// + /// # Examples + /// + /// ``` + /// use std::path::Path; + /// use std::process::Command; + /// + /// let mut cmd = Command::new("ls"); + /// assert_eq!(cmd.get_current_dir(), None); + /// cmd.current_dir("/bin"); + /// assert_eq!(cmd.get_current_dir(), Some(Path::new("/bin"))); + /// ``` + #[must_use] + #[stable(feature = "command_access", since = "1.57.0")] + pub fn get_current_dir(&self) -> Option<&Path> { + self.inner.get_current_dir() + } } #[stable(feature = "rust1", since = "1.0.0")] @@ -856,6 +1038,15 @@ impl fmt::Debug for Command { /// Format the program and arguments of a Command for display. Any /// non-utf8 data is lossily converted using the utf8 replacement /// character. + /// + /// The default format approximates a shell invocation of the program along with its + /// arguments. It does not include most of the other command properties. The output is not guaranteed to work + /// (e.g. due to lack of shell-escaping or differences in path resolution) + /// On some platforms you can use [the alternate syntax] to show more fields. + /// + /// Note that the debug implementation is platform-specific. + /// + /// [the alternate syntax]: fmt#sign0 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.inner.fmt(f) } @@ -873,16 +1064,46 @@ impl AsInnerMut for Command { } } +/// An iterator over the command arguments. +/// +/// This struct is created by [`Command::get_args`]. See its documentation for +/// more. +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "command_access", since = "1.57.0")] +#[derive(Debug)] +pub struct CommandArgs<'a> { + inner: imp::CommandArgs<'a>, +} + +#[stable(feature = "command_access", since = "1.57.0")] +impl<'a> Iterator for CommandArgs<'a> { + type Item = &'a OsStr; + fn next(&mut self) -> Option<&'a OsStr> { + self.inner.next() + } + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} + +#[stable(feature = "command_access", since = "1.57.0")] +impl<'a> ExactSizeIterator for CommandArgs<'a> { + fn len(&self) -> usize { + self.inner.len() + } + fn is_empty(&self) -> bool { + self.inner.is_empty() + } +} + /// The output of a finished process. /// /// This is returned in a Result by either the [`output`] method of a /// [`Command`], or the [`wait_with_output`] method of a [`Child`] /// process. /// -/// [`Command`]: struct.Command.html -/// [`Child`]: struct.Child.html -/// [`output`]: struct.Command.html#method.output -/// [`wait_with_output`]: struct.Child.html#method.wait_with_output +/// [`output`]: Command::output +/// [`wait_with_output`]: Child::wait_with_output #[derive(PartialEq, Eq, Clone)] #[stable(feature = "process", since = "1.0.0")] pub struct Output { @@ -925,10 +1146,9 @@ impl fmt::Debug for Output { /// Describes what to do with a standard I/O stream for a child process when /// passed to the [`stdin`], [`stdout`], and [`stderr`] methods of [`Command`]. /// -/// [`stdin`]: struct.Command.html#method.stdin -/// [`stdout`]: struct.Command.html#method.stdout -/// [`stderr`]: struct.Command.html#method.stderr -/// [`Command`]: struct.Command.html +/// [`stdin`]: Command::stdin +/// [`stdout`]: Command::stdout +/// [`stderr`]: Command::stderr #[stable(feature = "process", since = "1.0.0")] pub struct Stdio(imp::Stdio); @@ -964,14 +1184,22 @@ impl Stdio { /// .spawn() /// .expect("Failed to spawn child process"); /// - /// { - /// let stdin = child.stdin.as_mut().expect("Failed to open stdin"); + /// let mut stdin = child.stdin.take().expect("Failed to open stdin"); + /// std::thread::spawn(move || { /// stdin.write_all("Hello, world!".as_bytes()).expect("Failed to write to stdin"); - /// } + /// }); /// /// let output = child.wait_with_output().expect("Failed to read stdout"); /// assert_eq!(String::from_utf8_lossy(&output.stdout), "!dlrow ,olleH"); /// ``` + /// + /// Writing more than a pipe buffer's worth of input to stdin without also reading + /// stdout and stderr at the same time may cause a deadlock. + /// This is an issue when running any program that doesn't guarantee that it reads + /// its entire stdin before writing more than a pipe buffer's worth of output. + /// The size of a pipe buffer varies on different targets. + /// + #[must_use] #[stable(feature = "process", since = "1.0.0")] pub fn piped() -> Stdio { Stdio(imp::Stdio::MakePipe) @@ -1011,13 +1239,14 @@ impl Stdio { /// print!("You piped in the reverse of: "); /// io::stdout().write_all(&output.stdout).unwrap(); /// ``` + #[must_use] #[stable(feature = "process", since = "1.0.0")] pub fn inherit() -> Stdio { Stdio(imp::Stdio::Inherit) } /// This stream will be ignored. This is the equivalent of attaching the - /// stream to `/dev/null` + /// stream to `/dev/null`. /// /// # Examples /// @@ -1050,10 +1279,27 @@ impl Stdio { /// assert_eq!(String::from_utf8_lossy(&output.stdout), ""); /// // Ignores any piped-in input /// ``` + #[must_use] #[stable(feature = "process", since = "1.0.0")] pub fn null() -> Stdio { Stdio(imp::Stdio::Null) } + + /// Returns `true` if this requires [`Command`] to create a new pipe. + /// + /// # Example + /// + /// ``` + /// #![feature(stdio_makes_pipe)] + /// use std::process::Stdio; + /// + /// let io = Stdio::piped(); + /// assert_eq!(io.makes_pipe(), true); + /// ``` + #[unstable(feature = "stdio_makes_pipe", issue = "98288")] + pub fn makes_pipe(&self) -> bool { + matches!(self.0, imp::Stdio::MakePipe) + } } impl FromInner for Stdio { @@ -1065,13 +1311,13 @@ impl FromInner for Stdio { #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for Stdio { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("Stdio { .. }") + f.debug_struct("Stdio").finish_non_exhaustive() } } #[stable(feature = "stdio_from", since = "1.20.0")] impl From for Stdio { - /// Converts a `ChildStdin` into a `Stdio` + /// Converts a [`ChildStdin`] into a [`Stdio`]. /// /// # Examples /// @@ -1100,7 +1346,7 @@ impl From for Stdio { #[stable(feature = "stdio_from", since = "1.20.0")] impl From for Stdio { - /// Converts a `ChildStdout` into a `Stdio` + /// Converts a [`ChildStdout`] into a [`Stdio`]. /// /// # Examples /// @@ -1129,7 +1375,7 @@ impl From for Stdio { #[stable(feature = "stdio_from", since = "1.20.0")] impl From for Stdio { - /// Converts a `ChildStderr` into a `Stdio` + /// Converts a [`ChildStderr`] into a [`Stdio`]. /// /// # Examples /// @@ -1160,7 +1406,7 @@ impl From for Stdio { #[stable(feature = "stdio_from", since = "1.20.0")] impl From for Stdio { - /// Converts a `File` into a `Stdio` + /// Converts a [`File`](fs::File) into a [`Stdio`]. /// /// # Examples /// @@ -1187,20 +1433,65 @@ impl From for Stdio { /// Describes the result of a process after it has terminated. /// -/// This `struct` is used to represent the exit status of a child process. +/// This `struct` is used to represent the exit status or other termination of a child process. /// Child processes are created via the [`Command`] struct and their exit /// status is exposed through the [`status`] method, or the [`wait`] method /// of a [`Child`] process. /// -/// [`Command`]: struct.Command.html -/// [`Child`]: struct.Child.html -/// [`status`]: struct.Command.html#method.status -/// [`wait`]: struct.Child.html#method.wait +/// An `ExitStatus` represents every possible disposition of a process. On Unix this +/// is the **wait status**. It is *not* simply an *exit status* (a value passed to `exit`). +/// +/// For proper error reporting of failed processes, print the value of `ExitStatus` or +/// `ExitStatusError` using their implementations of [`Display`](crate::fmt::Display). +/// +/// # Differences from `ExitCode` +/// +/// [`ExitCode`] is intended for terminating the currently running process, via +/// the `Termination` trait, in contrast to `ExitStatus`, which represents the +/// termination of a child process. These APIs are separate due to platform +/// compatibility differences and their expected usage; it is not generally +/// possible to exactly reproduce an `ExitStatus` from a child for the current +/// process after the fact. +/// +/// [`status`]: Command::status +/// [`wait`]: Child::wait +// +// We speak slightly loosely (here and in various other places in the stdlib docs) about `exit` +// vs `_exit`. Naming of Unix system calls is not standardised across Unices, so terminology is a +// matter of convention and tradition. For clarity we usually speak of `exit`, even when we might +// mean an underlying system call such as `_exit`. #[derive(PartialEq, Eq, Clone, Copy, Debug)] #[stable(feature = "process", since = "1.0.0")] pub struct ExitStatus(imp::ExitStatus); +/// Allows extension traits within `std`. +#[unstable(feature = "sealed", issue = "none")] +impl crate::sealed::Sealed for ExitStatus {} + impl ExitStatus { + /// Was termination successful? Returns a `Result`. + /// + /// # Examples + /// + /// ``` + /// #![feature(exit_status_error)] + /// # if cfg!(unix) { + /// use std::process::Command; + /// + /// let status = Command::new("ls") + /// .arg("/dev/nonexistent") + /// .status() + /// .expect("ls could not be executed"); + /// + /// println!("ls: {status}"); + /// status.exit_ok().expect_err("/dev/nonexistent could be listed!"); + /// # } // cfg!(unix) + /// ``` + #[unstable(feature = "exit_status_error", issue = "84908")] + pub fn exit_ok(&self) -> Result<(), ExitStatusError> { + self.0.exit_ok().map_err(ExitStatusError) + } + /// Was termination successful? Signal termination is not considered a /// success, and success is defined as a zero exit status. /// @@ -1217,19 +1508,25 @@ impl ExitStatus { /// if status.success() { /// println!("'projects/' directory created"); /// } else { - /// println!("failed to create 'projects/' directory"); + /// println!("failed to create 'projects/' directory: {status}"); /// } /// ``` + #[must_use] #[stable(feature = "process", since = "1.0.0")] pub fn success(&self) -> bool { - self.0.success() + self.0.exit_ok().is_ok() } /// Returns the exit code of the process, if any. /// - /// On Unix, this will return `None` if the process was terminated - /// by a signal; `std::os::unix` provides an extension trait for - /// extracting the signal and other details from the `ExitStatus`. + /// In Unix terms the return value is the **exit status**: the value passed to `exit`, if the + /// process finished by calling `exit`. Note that on Unix the exit status is truncated to 8 + /// bits, and that values that didn't come from a program's call to `exit` may be invented by the + /// runtime system (often, for example, 255, 254, 127 or 126). + /// + /// On Unix, this will return `None` if the process was terminated by a signal. + /// [`ExitStatusExt`](crate::os::unix::process::ExitStatusExt) is an + /// extension trait for extracting any such signal, and other details, from the `ExitStatus`. /// /// # Examples /// @@ -1242,10 +1539,11 @@ impl ExitStatus { /// .expect("failed to execute mkdir"); /// /// match status.code() { - /// Some(code) => println!("Exited with status code: {}", code), + /// Some(code) => println!("Exited with status code: {code}"), /// None => println!("Process terminated by signal") /// } /// ``` + #[must_use] #[stable(feature = "process", since = "1.0.0")] pub fn code(&self) -> Option { self.0.code() @@ -1271,8 +1569,132 @@ impl fmt::Display for ExitStatus { } } -/// This type represents the status code a process can return to its -/// parent under normal termination. +/// Allows extension traits within `std`. +#[unstable(feature = "sealed", issue = "none")] +impl crate::sealed::Sealed for ExitStatusError {} + +/// Describes the result of a process after it has failed +/// +/// Produced by the [`.exit_ok`](ExitStatus::exit_ok) method on [`ExitStatus`]. +/// +/// # Examples +/// +/// ``` +/// #![feature(exit_status_error)] +/// # if cfg!(unix) { +/// use std::process::{Command, ExitStatusError}; +/// +/// fn run(cmd: &str) -> Result<(),ExitStatusError> { +/// Command::new(cmd).status().unwrap().exit_ok()?; +/// Ok(()) +/// } +/// +/// run("true").unwrap(); +/// run("false").unwrap_err(); +/// # } // cfg!(unix) +/// ``` +#[derive(PartialEq, Eq, Clone, Copy, Debug)] +#[unstable(feature = "exit_status_error", issue = "84908")] +// The definition of imp::ExitStatusError should ideally be such that +// Result<(), imp::ExitStatusError> has an identical representation to imp::ExitStatus. +pub struct ExitStatusError(imp::ExitStatusError); + +#[unstable(feature = "exit_status_error", issue = "84908")] +impl ExitStatusError { + /// Reports the exit code, if applicable, from an `ExitStatusError`. + /// + /// In Unix terms the return value is the **exit status**: the value passed to `exit`, if the + /// process finished by calling `exit`. Note that on Unix the exit status is truncated to 8 + /// bits, and that values that didn't come from a program's call to `exit` may be invented by the + /// runtime system (often, for example, 255, 254, 127 or 126). + /// + /// On Unix, this will return `None` if the process was terminated by a signal. If you want to + /// handle such situations specially, consider using methods from + /// [`ExitStatusExt`](crate::os::unix::process::ExitStatusExt). + /// + /// If the process finished by calling `exit` with a nonzero value, this will return + /// that exit status. + /// + /// If the error was something else, it will return `None`. + /// + /// If the process exited successfully (ie, by calling `exit(0)`), there is no + /// `ExitStatusError`. So the return value from `ExitStatusError::code()` is always nonzero. + /// + /// # Examples + /// + /// ``` + /// #![feature(exit_status_error)] + /// # #[cfg(unix)] { + /// use std::process::Command; + /// + /// let bad = Command::new("false").status().unwrap().exit_ok().unwrap_err(); + /// assert_eq!(bad.code(), Some(1)); + /// # } // #[cfg(unix)] + /// ``` + #[must_use] + pub fn code(&self) -> Option { + self.code_nonzero().map(Into::into) + } + + /// Reports the exit code, if applicable, from an `ExitStatusError`, as a `NonZero` + /// + /// This is exactly like [`code()`](Self::code), except that it returns a `NonZeroI32`. + /// + /// Plain `code`, returning a plain integer, is provided because it is often more convenient. + /// The returned value from `code()` is indeed also nonzero; use `code_nonzero()` when you want + /// a type-level guarantee of nonzeroness. + /// + /// # Examples + /// + /// ``` + /// #![feature(exit_status_error)] + /// # if cfg!(unix) { + /// use std::num::NonZeroI32; + /// use std::process::Command; + /// + /// let bad = Command::new("false").status().unwrap().exit_ok().unwrap_err(); + /// assert_eq!(bad.code_nonzero().unwrap(), NonZeroI32::try_from(1).unwrap()); + /// # } // cfg!(unix) + /// ``` + #[must_use] + pub fn code_nonzero(&self) -> Option { + self.0.code() + } + + /// Converts an `ExitStatusError` (back) to an `ExitStatus`. + #[must_use] + pub fn into_status(&self) -> ExitStatus { + ExitStatus(self.0.into()) + } +} + +#[unstable(feature = "exit_status_error", issue = "84908")] +impl Into for ExitStatusError { + fn into(self) -> ExitStatus { + ExitStatus(self.0.into()) + } +} + +#[unstable(feature = "exit_status_error", issue = "84908")] +impl fmt::Display for ExitStatusError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "process exited unsuccessfully: {}", self.into_status()) + } +} + +#[unstable(feature = "exit_status_error", issue = "84908")] +impl crate::error::Error for ExitStatusError {} + +/// This type represents the status code the current process can return +/// to its parent under normal termination. +/// +/// `ExitCode` is intended to be consumed only by the standard library (via +/// [`Termination::report()`]), and intentionally does not provide accessors like +/// `PartialEq`, `Eq`, or `Hash`. Instead the standard library provides the +/// canonical `SUCCESS` and `FAILURE` exit codes as well as `From for +/// ExitCode` for constructing other arbitrary exit codes. +/// +/// # Portability /// /// Numeric values used in this type don't have portable meanings, and /// different platforms may mask different amounts of them. @@ -1280,43 +1702,150 @@ impl fmt::Display for ExitStatus { /// For the platform's canonical successful and unsuccessful codes, see /// the [`SUCCESS`] and [`FAILURE`] associated items. /// -/// [`SUCCESS`]: #associatedconstant.SUCCESS -/// [`FAILURE`]: #associatedconstant.FAILURE +/// [`SUCCESS`]: ExitCode::SUCCESS +/// [`FAILURE`]: ExitCode::FAILURE +/// +/// # Differences from `ExitStatus` +/// +/// `ExitCode` is intended for terminating the currently running process, via +/// the `Termination` trait, in contrast to [`ExitStatus`], which represents the +/// termination of a child process. These APIs are separate due to platform +/// compatibility differences and their expected usage; it is not generally +/// possible to exactly reproduce an `ExitStatus` from a child for the current +/// process after the fact. +/// +/// # Examples +/// +/// `ExitCode` can be returned from the `main` function of a crate, as it implements +/// [`Termination`]: +/// +/// ``` +/// use std::process::ExitCode; +/// # fn check_foo() -> bool { true } /// -/// **Warning**: While various forms of this were discussed in [RFC #1937], -/// it was ultimately cut from that RFC, and thus this type is more subject -/// to change even than the usual unstable item churn. +/// fn main() -> ExitCode { +/// if !check_foo() { +/// return ExitCode::from(42); +/// } /// -/// [RFC #1937]: https://github.com/rust-lang/rfcs/pull/1937 +/// ExitCode::SUCCESS +/// } +/// ``` #[derive(Clone, Copy, Debug)] -#[unstable(feature = "process_exitcode_placeholder", issue = "48711")] +#[stable(feature = "process_exitcode", since = "1.61.0")] pub struct ExitCode(imp::ExitCode); -#[unstable(feature = "process_exitcode_placeholder", issue = "48711")] +/// Allows extension traits within `std`. +#[unstable(feature = "sealed", issue = "none")] +impl crate::sealed::Sealed for ExitCode {} + +#[stable(feature = "process_exitcode", since = "1.61.0")] impl ExitCode { - /// The canonical ExitCode for successful termination on this platform. + /// The canonical `ExitCode` for successful termination on this platform. /// /// Note that a `()`-returning `main` implicitly results in a successful /// termination, so there's no need to return this from `main` unless /// you're also returning other possible codes. - #[unstable(feature = "process_exitcode_placeholder", issue = "48711")] + #[stable(feature = "process_exitcode", since = "1.61.0")] pub const SUCCESS: ExitCode = ExitCode(imp::ExitCode::SUCCESS); - /// The canonical ExitCode for unsuccessful termination on this platform. + /// The canonical `ExitCode` for unsuccessful termination on this platform. /// /// If you're only returning this and `SUCCESS` from `main`, consider /// instead returning `Err(_)` and `Ok(())` respectively, which will /// return the same codes (but will also `eprintln!` the error). - #[unstable(feature = "process_exitcode_placeholder", issue = "48711")] + #[stable(feature = "process_exitcode", since = "1.61.0")] pub const FAILURE: ExitCode = ExitCode(imp::ExitCode::FAILURE); + + /// Exit the current process with the given `ExitCode`. + /// + /// Note that this has the same caveats as [`process::exit()`][exit], namely that this function + /// terminates the process immediately, so no destructors on the current stack or any other + /// thread's stack will be run. If a clean shutdown is needed, it is recommended to simply + /// return this ExitCode from the `main` function, as demonstrated in the [type + /// documentation](#examples). + /// + /// # Differences from `process::exit()` + /// + /// `process::exit()` accepts any `i32` value as the exit code for the process; however, there + /// are platforms that only use a subset of that value (see [`process::exit` platform-specific + /// behavior][exit#platform-specific-behavior]). `ExitCode` exists because of this; only + /// `ExitCode`s that are supported by a majority of our platforms can be created, so those + /// problems don't exist (as much) with this method. + /// + /// # Examples + /// + /// ``` + /// #![feature(exitcode_exit_method)] + /// # use std::process::ExitCode; + /// # use std::fmt; + /// # enum UhOhError { GenericProblem, Specific, WithCode { exit_code: ExitCode, _x: () } } + /// # impl fmt::Display for UhOhError { + /// # fn fmt(&self, _: &mut fmt::Formatter) -> fmt::Result { unimplemented!() } + /// # } + /// // there's no way to gracefully recover from an UhOhError, so we just + /// // print a message and exit + /// fn handle_unrecoverable_error(err: UhOhError) -> ! { + /// eprintln!("UH OH! {err}"); + /// let code = match err { + /// UhOhError::GenericProblem => ExitCode::FAILURE, + /// UhOhError::Specific => ExitCode::from(3), + /// UhOhError::WithCode { exit_code, .. } => exit_code, + /// }; + /// code.exit_process() + /// } + /// ``` + #[unstable(feature = "exitcode_exit_method", issue = "97100")] + pub fn exit_process(self) -> ! { + exit(self.to_i32()) + } +} + +impl ExitCode { + // This is private/perma-unstable because ExitCode is opaque; we don't know that i32 will serve + // all usecases, for example windows seems to use u32, unix uses the 8-15th bits of an i32, we + // likely want to isolate users anything that could restrict the platform specific + // representation of an ExitCode + // + // More info: https://internals.rust-lang.org/t/mini-pre-rfc-redesigning-process-exitstatus/5426 + /// Convert an `ExitCode` into an i32 + #[unstable( + feature = "process_exitcode_internals", + reason = "exposed only for libstd", + issue = "none" + )] + #[inline] + #[doc(hidden)] + pub fn to_i32(self) -> i32 { + self.0.as_i32() + } +} + +#[stable(feature = "process_exitcode", since = "1.61.0")] +impl From for ExitCode { + /// Construct an `ExitCode` from an arbitrary u8 value. + fn from(code: u8) -> Self { + ExitCode(imp::ExitCode::from(code)) + } +} + +impl AsInner for ExitCode { + fn as_inner(&self) -> &imp::ExitCode { + &self.0 + } +} + +impl FromInner for ExitCode { + fn from_inner(s: imp::ExitCode) -> ExitCode { + ExitCode(s) + } } impl Child { /// Forces the child process to exit. If the child has already exited, an [`InvalidInput`] /// error is returned. /// - /// The mapping to [`ErrorKind`]s is not part of the compatibility contract of the function, - /// especially the [`Other`] kind might change to more specific kinds in the future. + /// The mapping to [`ErrorKind`]s is not part of the compatibility contract of the function. /// /// This is equivalent to sending a SIGKILL on Unix platforms. /// @@ -1335,9 +1864,8 @@ impl Child { /// } /// ``` /// - /// [`ErrorKind`]: ../io/enum.ErrorKind.html - /// [`InvalidInput`]: ../io/enum.ErrorKind.html#variant.InvalidInput - /// [`Other`]: ../io/enum.ErrorKind.html#variant.Other + /// [`ErrorKind`]: io::ErrorKind + /// [`InvalidInput`]: io::ErrorKind::InvalidInput #[stable(feature = "process", since = "1.0.0")] pub fn kill(&mut self) -> io::Result<()> { self.handle.kill() @@ -1359,6 +1887,7 @@ impl Child { /// println!("ls command didn't start"); /// } /// ``` + #[must_use] #[stable(feature = "process_id", since = "1.3.0")] pub fn id(&self) -> u32 { self.handle.id() @@ -1419,13 +1948,13 @@ impl Child { /// let mut child = Command::new("ls").spawn().unwrap(); /// /// match child.try_wait() { - /// Ok(Some(status)) => println!("exited with: {}", status), + /// Ok(Some(status)) => println!("exited with: {status}"), /// Ok(None) => { /// println!("status not ready yet, let's really wait"); /// let res = child.wait(); - /// println!("result: {:?}", res); + /// println!("result: {res:?}"); /// } - /// Err(e) => println!("error attempting to wait: {}", e), + /// Err(e) => println!("error attempting to wait: {e}"), /// } /// ``` #[stable(feature = "process_try_wait", since = "1.18.0")] @@ -1501,7 +2030,17 @@ impl Child { /// process, no destructors on the current stack or any other thread's stack /// will be run. If a clean shutdown is needed it is recommended to only call /// this function at a known point where there are no more destructors left -/// to run. +/// to run; or, preferably, simply return a type implementing [`Termination`] +/// (such as [`ExitCode`] or `Result`) from the `main` function and avoid this +/// function altogether: +/// +/// ``` +/// # use std::io::Error as MyError; +/// fn main() -> Result<(), MyError> { +/// // ... +/// Ok(()) +/// } +/// ``` /// /// ## Platform-specific behavior /// @@ -1509,42 +2048,17 @@ impl Child { /// will be visible to a parent process inspecting the exit code. On most /// Unix-like platforms, only the eight least-significant bits are considered. /// -/// # Examples -/// -/// Due to this function’s behavior regarding destructors, a conventional way -/// to use the function is to extract the actual computation to another -/// function and compute the exit code from its return value: -/// -/// ``` -/// fn run_app() -> Result<(), ()> { -/// // Application logic here -/// Ok(()) -/// } -/// -/// fn main() { -/// std::process::exit(match run_app() { -/// Ok(_) => 0, -/// Err(err) => { -/// eprintln!("error: {:?}", err); -/// 1 -/// } -/// }); -/// } -/// ``` -/// -/// Due to [platform-specific behavior], the exit code for this example will be -/// `0` on Linux, but `256` on Windows: +/// For example, the exit code for this example will be `0` on Linux, but `256` +/// on Windows: /// /// ```no_run /// use std::process; /// /// process::exit(0x0100); /// ``` -/// -/// [platform-specific behavior]: #platform-specific-behavior #[stable(feature = "rust1", since = "1.0.0")] pub fn exit(code: i32) -> ! { - crate::sys_common::cleanup(); + crate::rt::cleanup(); crate::sys::os::exit(code) } @@ -1557,6 +2071,9 @@ pub fn exit(code: i32) -> ! { /// process, no destructors on the current stack or any other thread's stack /// will be run. /// +/// Rust IO buffers (eg, from `BufWriter`) will not be flushed. +/// Likewise, C stdio buffers will (on most platforms) not be flushed. +/// /// This is in contrast to the default behaviour of [`panic!`] which unwinds /// the current thread's stack and calls all destructors. /// When `panic="abort"` is set, either as an argument to `rustc` or in a @@ -1567,6 +2084,10 @@ pub fn exit(code: i32) -> ! { /// this function at a known point where there are no more destructors left /// to run. /// +/// The process's termination will be similar to that from the C `abort()` +/// function. On Unix, the process will terminate with signal `SIGABRT`, which +/// typically means that the shell prints "Aborted". +/// /// # Examples /// /// ```no_run @@ -1602,11 +2123,11 @@ pub fn exit(code: i32) -> ! { /// } /// ``` /// -/// [`panic!`]: ../../std/macro.panic.html -/// [panic hook]: ../../std/panic/fn.set_hook.html +/// [panic hook]: crate::panic::set_hook #[stable(feature = "process_abort", since = "1.17.0")] +#[cold] pub fn abort() -> ! { - unsafe { crate::sys::abort_internal() }; + crate::sys::abort_internal(); } /// Returns the OS-assigned process identifier associated with this process. @@ -1622,6 +2143,7 @@ pub fn abort() -> ! { /// ``` /// /// +#[must_use] #[stable(feature = "getpid", since = "1.26.0")] pub fn id() -> u32 { crate::sys::os::getpid() @@ -1629,470 +2151,70 @@ pub fn id() -> u32 { /// A trait for implementing arbitrary return types in the `main` function. /// -/// The C-main function only supports to return integers as return type. +/// The C-main function only supports returning integers. /// So, every type implementing the `Termination` trait has to be converted /// to an integer. /// /// The default implementations are returning `libc::EXIT_SUCCESS` to indicate /// a successful execution. In case of a failure, `libc::EXIT_FAILURE` is returned. +/// +/// Because different runtimes have different specifications on the return value +/// of the `main` function, this trait is likely to be available only on +/// standard library's runtime for convenience. Other runtimes are not required +/// to provide similar functionality. #[cfg_attr(not(test), lang = "termination")] -#[unstable(feature = "termination_trait_lib", issue = "43301")] -#[rustc_on_unimplemented( +#[stable(feature = "termination_trait_lib", since = "1.61.0")] +#[rustc_on_unimplemented(on( + cause = "MainFunctionType", message = "`main` has invalid return type `{Self}`", label = "`main` can only return types that implement `{Termination}`" -)] +))] pub trait Termination { /// Is called to get the representation of the value as status code. /// This status code is returned to the operating system. - fn report(self) -> i32; + #[stable(feature = "termination_trait_lib", since = "1.61.0")] + fn report(self) -> ExitCode; } -#[unstable(feature = "termination_trait_lib", issue = "43301")] +#[stable(feature = "termination_trait_lib", since = "1.61.0")] impl Termination for () { #[inline] - fn report(self) -> i32 { - ExitCode::SUCCESS.report() - } -} - -#[unstable(feature = "termination_trait_lib", issue = "43301")] -impl Termination for Result<(), E> { - fn report(self) -> i32 { - match self { - Ok(()) => ().report(), - Err(err) => Err::(err).report(), - } + fn report(self) -> ExitCode { + ExitCode::SUCCESS } } -#[unstable(feature = "termination_trait_lib", issue = "43301")] +#[stable(feature = "termination_trait_lib", since = "1.61.0")] impl Termination for ! { - fn report(self) -> i32 { + fn report(self) -> ExitCode { self } } -#[unstable(feature = "termination_trait_lib", issue = "43301")] -impl Termination for Result { - fn report(self) -> i32 { - let Err(err) = self; - eprintln!("Error: {:?}", err); - ExitCode::FAILURE.report() +#[stable(feature = "termination_trait_lib", since = "1.61.0")] +impl Termination for Infallible { + fn report(self) -> ExitCode { + match self {} } } -#[unstable(feature = "termination_trait_lib", issue = "43301")] +#[stable(feature = "termination_trait_lib", since = "1.61.0")] impl Termination for ExitCode { #[inline] - fn report(self) -> i32 { - self.0.as_i32() + fn report(self) -> ExitCode { + self } } -#[cfg(all(test, not(any(target_os = "cloudabi", target_os = "emscripten", target_env = "sgx"))))] -mod tests { - use crate::io::prelude::*; - - use super::{Command, Output, Stdio}; - use crate::io::ErrorKind; - use crate::str; - - // FIXME(#10380) these tests should not all be ignored on android. - - #[test] - #[cfg_attr(any(target_os = "vxworks", target_os = "android"), ignore)] - fn smoke() { - let p = if cfg!(target_os = "windows") { - Command::new("cmd").args(&["/C", "exit 0"]).spawn() - } else { - Command::new("true").spawn() - }; - assert!(p.is_ok()); - let mut p = p.unwrap(); - assert!(p.wait().unwrap().success()); - } - - #[test] - #[cfg_attr(target_os = "android", ignore)] - fn smoke_failure() { - match Command::new("if-this-is-a-binary-then-the-world-has-ended").spawn() { - Ok(..) => panic!(), - Err(..) => {} - } - } - - #[test] - #[cfg_attr(any(target_os = "vxworks", target_os = "android"), ignore)] - fn exit_reported_right() { - let p = if cfg!(target_os = "windows") { - Command::new("cmd").args(&["/C", "exit 1"]).spawn() - } else { - Command::new("false").spawn() - }; - assert!(p.is_ok()); - let mut p = p.unwrap(); - assert!(p.wait().unwrap().code() == Some(1)); - drop(p.wait()); - } - - #[test] - #[cfg(unix)] - #[cfg_attr(any(target_os = "vxworks", target_os = "android"), ignore)] - fn signal_reported_right() { - use crate::os::unix::process::ExitStatusExt; - - let mut p = - Command::new("/bin/sh").arg("-c").arg("read a").stdin(Stdio::piped()).spawn().unwrap(); - p.kill().unwrap(); - match p.wait().unwrap().signal() { - Some(9) => {} - result => panic!("not terminated by signal 9 (instead, {:?})", result), - } - } - - pub fn run_output(mut cmd: Command) -> String { - let p = cmd.spawn(); - assert!(p.is_ok()); - let mut p = p.unwrap(); - assert!(p.stdout.is_some()); - let mut ret = String::new(); - p.stdout.as_mut().unwrap().read_to_string(&mut ret).unwrap(); - assert!(p.wait().unwrap().success()); - return ret; - } - - #[test] - #[cfg_attr(any(target_os = "vxworks", target_os = "android"), ignore)] - fn stdout_works() { - if cfg!(target_os = "windows") { - let mut cmd = Command::new("cmd"); - cmd.args(&["/C", "echo foobar"]).stdout(Stdio::piped()); - assert_eq!(run_output(cmd), "foobar\r\n"); - } else { - let mut cmd = Command::new("echo"); - cmd.arg("foobar").stdout(Stdio::piped()); - assert_eq!(run_output(cmd), "foobar\n"); - } - } - - #[test] - #[cfg_attr(any(windows, target_os = "android", target_os = "vxworks"), ignore)] - fn set_current_dir_works() { - let mut cmd = Command::new("/bin/sh"); - cmd.arg("-c").arg("pwd").current_dir("/").stdout(Stdio::piped()); - assert_eq!(run_output(cmd), "/\n"); - } - - #[test] - #[cfg_attr(any(windows, target_os = "android", target_os = "vxworks"), ignore)] - fn stdin_works() { - let mut p = Command::new("/bin/sh") - .arg("-c") - .arg("read line; echo $line") - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .spawn() - .unwrap(); - p.stdin.as_mut().unwrap().write("foobar".as_bytes()).unwrap(); - drop(p.stdin.take()); - let mut out = String::new(); - p.stdout.as_mut().unwrap().read_to_string(&mut out).unwrap(); - assert!(p.wait().unwrap().success()); - assert_eq!(out, "foobar\n"); - } - - #[test] - #[cfg_attr(any(target_os = "vxworks", target_os = "android"), ignore)] - fn test_process_status() { - let mut status = if cfg!(target_os = "windows") { - Command::new("cmd").args(&["/C", "exit 1"]).status().unwrap() - } else { - Command::new("false").status().unwrap() - }; - assert!(status.code() == Some(1)); - - status = if cfg!(target_os = "windows") { - Command::new("cmd").args(&["/C", "exit 0"]).status().unwrap() - } else { - Command::new("true").status().unwrap() - }; - assert!(status.success()); - } - - #[test] - fn test_process_output_fail_to_start() { - match Command::new("/no-binary-by-this-name-should-exist").output() { - Err(e) => assert_eq!(e.kind(), ErrorKind::NotFound), - Ok(..) => panic!(), - } - } - - #[test] - #[cfg_attr(any(target_os = "vxworks", target_os = "android"), ignore)] - fn test_process_output_output() { - let Output { status, stdout, stderr } = if cfg!(target_os = "windows") { - Command::new("cmd").args(&["/C", "echo hello"]).output().unwrap() - } else { - Command::new("echo").arg("hello").output().unwrap() - }; - let output_str = str::from_utf8(&stdout).unwrap(); - - assert!(status.success()); - assert_eq!(output_str.trim().to_string(), "hello"); - assert_eq!(stderr, Vec::new()); - } - - #[test] - #[cfg_attr(any(target_os = "vxworks", target_os = "android"), ignore)] - fn test_process_output_error() { - let Output { status, stdout, stderr } = if cfg!(target_os = "windows") { - Command::new("cmd").args(&["/C", "mkdir ."]).output().unwrap() - } else { - Command::new("mkdir").arg("./").output().unwrap() - }; - - assert!(status.code() == Some(1)); - assert_eq!(stdout, Vec::new()); - assert!(!stderr.is_empty()); - } - - #[test] - #[cfg_attr(any(target_os = "vxworks", target_os = "android"), ignore)] - fn test_finish_once() { - let mut prog = if cfg!(target_os = "windows") { - Command::new("cmd").args(&["/C", "exit 1"]).spawn().unwrap() - } else { - Command::new("false").spawn().unwrap() - }; - assert!(prog.wait().unwrap().code() == Some(1)); - } - - #[test] - #[cfg_attr(any(target_os = "vxworks", target_os = "android"), ignore)] - fn test_finish_twice() { - let mut prog = if cfg!(target_os = "windows") { - Command::new("cmd").args(&["/C", "exit 1"]).spawn().unwrap() - } else { - Command::new("false").spawn().unwrap() - }; - assert!(prog.wait().unwrap().code() == Some(1)); - assert!(prog.wait().unwrap().code() == Some(1)); - } - - #[test] - #[cfg_attr(any(target_os = "vxworks", target_os = "android"), ignore)] - fn test_wait_with_output_once() { - let prog = if cfg!(target_os = "windows") { - Command::new("cmd").args(&["/C", "echo hello"]).stdout(Stdio::piped()).spawn().unwrap() - } else { - Command::new("echo").arg("hello").stdout(Stdio::piped()).spawn().unwrap() - }; - - let Output { status, stdout, stderr } = prog.wait_with_output().unwrap(); - let output_str = str::from_utf8(&stdout).unwrap(); - - assert!(status.success()); - assert_eq!(output_str.trim().to_string(), "hello"); - assert_eq!(stderr, Vec::new()); - } - - #[cfg(all(unix, not(target_os = "android")))] - pub fn env_cmd() -> Command { - Command::new("env") - } - #[cfg(target_os = "android")] - pub fn env_cmd() -> Command { - let mut cmd = Command::new("/system/bin/sh"); - cmd.arg("-c").arg("set"); - cmd - } - - #[cfg(windows)] - pub fn env_cmd() -> Command { - let mut cmd = Command::new("cmd"); - cmd.arg("/c").arg("set"); - cmd - } - - #[test] - #[cfg_attr(target_os = "vxworks", ignore)] - fn test_override_env() { - use crate::env; - - // In some build environments (such as chrooted Nix builds), `env` can - // only be found in the explicitly-provided PATH env variable, not in - // default places such as /bin or /usr/bin. So we need to pass through - // PATH to our sub-process. - let mut cmd = env_cmd(); - cmd.env_clear().env("RUN_TEST_NEW_ENV", "123"); - if let Some(p) = env::var_os("PATH") { - cmd.env("PATH", &p); - } - let result = cmd.output().unwrap(); - let output = String::from_utf8_lossy(&result.stdout).to_string(); - - assert!( - output.contains("RUN_TEST_NEW_ENV=123"), - "didn't find RUN_TEST_NEW_ENV inside of:\n\n{}", - output - ); - } - - #[test] - #[cfg_attr(target_os = "vxworks", ignore)] - fn test_add_to_env() { - let result = env_cmd().env("RUN_TEST_NEW_ENV", "123").output().unwrap(); - let output = String::from_utf8_lossy(&result.stdout).to_string(); - - assert!( - output.contains("RUN_TEST_NEW_ENV=123"), - "didn't find RUN_TEST_NEW_ENV inside of:\n\n{}", - output - ); - } - - #[test] - #[cfg_attr(target_os = "vxworks", ignore)] - fn test_capture_env_at_spawn() { - use crate::env; - - let mut cmd = env_cmd(); - cmd.env("RUN_TEST_NEW_ENV1", "123"); - - // This variable will not be present if the environment has already - // been captured above. - env::set_var("RUN_TEST_NEW_ENV2", "456"); - let result = cmd.output().unwrap(); - env::remove_var("RUN_TEST_NEW_ENV2"); - - let output = String::from_utf8_lossy(&result.stdout).to_string(); - - assert!( - output.contains("RUN_TEST_NEW_ENV1=123"), - "didn't find RUN_TEST_NEW_ENV1 inside of:\n\n{}", - output - ); - assert!( - output.contains("RUN_TEST_NEW_ENV2=456"), - "didn't find RUN_TEST_NEW_ENV2 inside of:\n\n{}", - output - ); - } - - // Regression tests for #30858. - #[test] - fn test_interior_nul_in_progname_is_error() { - match Command::new("has-some-\0\0s-inside").spawn() { - Err(e) => assert_eq!(e.kind(), ErrorKind::InvalidInput), - Ok(_) => panic!(), - } - } - - #[test] - fn test_interior_nul_in_arg_is_error() { - match Command::new("echo").arg("has-some-\0\0s-inside").spawn() { - Err(e) => assert_eq!(e.kind(), ErrorKind::InvalidInput), - Ok(_) => panic!(), - } - } - - #[test] - fn test_interior_nul_in_args_is_error() { - match Command::new("echo").args(&["has-some-\0\0s-inside"]).spawn() { - Err(e) => assert_eq!(e.kind(), ErrorKind::InvalidInput), - Ok(_) => panic!(), - } - } - - #[test] - fn test_interior_nul_in_current_dir_is_error() { - match Command::new("echo").current_dir("has-some-\0\0s-inside").spawn() { - Err(e) => assert_eq!(e.kind(), ErrorKind::InvalidInput), - Ok(_) => panic!(), - } - } - - // Regression tests for #30862. - #[test] - #[cfg_attr(target_os = "vxworks", ignore)] - fn test_interior_nul_in_env_key_is_error() { - match env_cmd().env("has-some-\0\0s-inside", "value").spawn() { - Err(e) => assert_eq!(e.kind(), ErrorKind::InvalidInput), - Ok(_) => panic!(), - } - } - - #[test] - #[cfg_attr(target_os = "vxworks", ignore)] - fn test_interior_nul_in_env_value_is_error() { - match env_cmd().env("key", "has-some-\0\0s-inside").spawn() { - Err(e) => assert_eq!(e.kind(), ErrorKind::InvalidInput), - Ok(_) => panic!(), - } - } - - /// Tests that process creation flags work by debugging a process. - /// Other creation flags make it hard or impossible to detect - /// behavioral changes in the process. - #[test] - #[cfg(windows)] - fn test_creation_flags() { - use crate::os::windows::process::CommandExt; - use crate::sys::c::{BOOL, DWORD, INFINITE}; - #[repr(C, packed)] - struct DEBUG_EVENT { - pub event_code: DWORD, - pub process_id: DWORD, - pub thread_id: DWORD, - // This is a union in the real struct, but we don't - // need this data for the purposes of this test. - pub _junk: [u8; 164], - } - - extern "system" { - fn WaitForDebugEvent(lpDebugEvent: *mut DEBUG_EVENT, dwMilliseconds: DWORD) -> BOOL; - fn ContinueDebugEvent( - dwProcessId: DWORD, - dwThreadId: DWORD, - dwContinueStatus: DWORD, - ) -> BOOL; - } - - const DEBUG_PROCESS: DWORD = 1; - const EXIT_PROCESS_DEBUG_EVENT: DWORD = 5; - const DBG_EXCEPTION_NOT_HANDLED: DWORD = 0x80010001; - - let mut child = Command::new("cmd") - .creation_flags(DEBUG_PROCESS) - .stdin(Stdio::piped()) - .spawn() - .unwrap(); - child.stdin.take().unwrap().write_all(b"exit\r\n").unwrap(); - let mut events = 0; - let mut event = DEBUG_EVENT { event_code: 0, process_id: 0, thread_id: 0, _junk: [0; 164] }; - loop { - if unsafe { WaitForDebugEvent(&mut event as *mut DEBUG_EVENT, INFINITE) } == 0 { - panic!("WaitForDebugEvent failed!"); - } - events += 1; - - if event.event_code == EXIT_PROCESS_DEBUG_EVENT { - break; - } - - if unsafe { - ContinueDebugEvent(event.process_id, event.thread_id, DBG_EXCEPTION_NOT_HANDLED) - } == 0 - { - panic!("ContinueDebugEvent failed!"); +#[stable(feature = "termination_trait_lib", since = "1.61.0")] +impl Termination for Result { + fn report(self) -> ExitCode { + match self { + Ok(val) => val.report(), + Err(err) => { + io::attempt_print_to_stderr(format_args_nl!("Error: {err:?}")); + ExitCode::FAILURE } } - assert!(events > 0); - } - - #[test] - fn test_command_implements_send() { - fn take_send_type(_: T) {} - take_send_type(Command::new("")) } } diff --git a/crux-mir/lib/std/src/process/tests.rs b/crux-mir/lib/std/src/process/tests.rs new file mode 100644 index 000000000..b4f6cc2da --- /dev/null +++ b/crux-mir/lib/std/src/process/tests.rs @@ -0,0 +1,552 @@ +use crate::io::prelude::*; + +use super::{Command, Output, Stdio}; +use crate::io::ErrorKind; +use crate::str; + +fn known_command() -> Command { + if cfg!(windows) { Command::new("help") } else { Command::new("echo") } +} + +#[cfg(target_os = "android")] +fn shell_cmd() -> Command { + Command::new("/system/bin/sh") +} + +#[cfg(not(target_os = "android"))] +fn shell_cmd() -> Command { + Command::new("/bin/sh") +} + +#[test] +#[cfg_attr(any(target_os = "vxworks"), ignore)] +fn smoke() { + let p = if cfg!(target_os = "windows") { + Command::new("cmd").args(&["/C", "exit 0"]).spawn() + } else { + shell_cmd().arg("-c").arg("true").spawn() + }; + assert!(p.is_ok()); + let mut p = p.unwrap(); + assert!(p.wait().unwrap().success()); +} + +#[test] +#[cfg_attr(target_os = "android", ignore)] +fn smoke_failure() { + match Command::new("if-this-is-a-binary-then-the-world-has-ended").spawn() { + Ok(..) => panic!(), + Err(..) => {} + } +} + +#[test] +#[cfg_attr(any(target_os = "vxworks"), ignore)] +fn exit_reported_right() { + let p = if cfg!(target_os = "windows") { + Command::new("cmd").args(&["/C", "exit 1"]).spawn() + } else { + shell_cmd().arg("-c").arg("false").spawn() + }; + assert!(p.is_ok()); + let mut p = p.unwrap(); + assert!(p.wait().unwrap().code() == Some(1)); + drop(p.wait()); +} + +#[test] +#[cfg(unix)] +#[cfg_attr(any(target_os = "vxworks"), ignore)] +fn signal_reported_right() { + use crate::os::unix::process::ExitStatusExt; + + let mut p = shell_cmd().arg("-c").arg("read a").stdin(Stdio::piped()).spawn().unwrap(); + p.kill().unwrap(); + match p.wait().unwrap().signal() { + Some(9) => {} + result => panic!("not terminated by signal 9 (instead, {result:?})"), + } +} + +pub fn run_output(mut cmd: Command) -> String { + let p = cmd.spawn(); + assert!(p.is_ok()); + let mut p = p.unwrap(); + assert!(p.stdout.is_some()); + let mut ret = String::new(); + p.stdout.as_mut().unwrap().read_to_string(&mut ret).unwrap(); + assert!(p.wait().unwrap().success()); + return ret; +} + +#[test] +#[cfg_attr(any(target_os = "vxworks"), ignore)] +fn stdout_works() { + if cfg!(target_os = "windows") { + let mut cmd = Command::new("cmd"); + cmd.args(&["/C", "echo foobar"]).stdout(Stdio::piped()); + assert_eq!(run_output(cmd), "foobar\r\n"); + } else { + let mut cmd = shell_cmd(); + cmd.arg("-c").arg("echo foobar").stdout(Stdio::piped()); + assert_eq!(run_output(cmd), "foobar\n"); + } +} + +#[test] +#[cfg_attr(any(windows, target_os = "vxworks"), ignore)] +fn set_current_dir_works() { + let mut cmd = shell_cmd(); + cmd.arg("-c").arg("pwd").current_dir("/").stdout(Stdio::piped()); + assert_eq!(run_output(cmd), "/\n"); +} + +#[test] +#[cfg_attr(any(windows, target_os = "vxworks"), ignore)] +fn stdin_works() { + let mut p = shell_cmd() + .arg("-c") + .arg("read line; echo $line") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + p.stdin.as_mut().unwrap().write("foobar".as_bytes()).unwrap(); + drop(p.stdin.take()); + let mut out = String::new(); + p.stdout.as_mut().unwrap().read_to_string(&mut out).unwrap(); + assert!(p.wait().unwrap().success()); + assert_eq!(out, "foobar\n"); +} + +#[test] +#[cfg_attr(any(target_os = "vxworks"), ignore)] +fn test_process_status() { + let mut status = if cfg!(target_os = "windows") { + Command::new("cmd").args(&["/C", "exit 1"]).status().unwrap() + } else { + shell_cmd().arg("-c").arg("false").status().unwrap() + }; + assert!(status.code() == Some(1)); + + status = if cfg!(target_os = "windows") { + Command::new("cmd").args(&["/C", "exit 0"]).status().unwrap() + } else { + shell_cmd().arg("-c").arg("true").status().unwrap() + }; + assert!(status.success()); +} + +#[test] +fn test_process_output_fail_to_start() { + match Command::new("/no-binary-by-this-name-should-exist").output() { + Err(e) => assert_eq!(e.kind(), ErrorKind::NotFound), + Ok(..) => panic!(), + } +} + +#[test] +#[cfg_attr(any(target_os = "vxworks"), ignore)] +fn test_process_output_output() { + let Output { status, stdout, stderr } = if cfg!(target_os = "windows") { + Command::new("cmd").args(&["/C", "echo hello"]).output().unwrap() + } else { + shell_cmd().arg("-c").arg("echo hello").output().unwrap() + }; + let output_str = str::from_utf8(&stdout).unwrap(); + + assert!(status.success()); + assert_eq!(output_str.trim().to_string(), "hello"); + assert_eq!(stderr, Vec::new()); +} + +#[test] +#[cfg_attr(any(target_os = "vxworks"), ignore)] +fn test_process_output_error() { + let Output { status, stdout, stderr } = if cfg!(target_os = "windows") { + Command::new("cmd").args(&["/C", "mkdir ."]).output().unwrap() + } else { + Command::new("mkdir").arg("./").output().unwrap() + }; + + assert!(status.code().is_some()); + assert!(status.code() != Some(0)); + assert_eq!(stdout, Vec::new()); + assert!(!stderr.is_empty()); +} + +#[test] +#[cfg_attr(any(target_os = "vxworks"), ignore)] +fn test_finish_once() { + let mut prog = if cfg!(target_os = "windows") { + Command::new("cmd").args(&["/C", "exit 1"]).spawn().unwrap() + } else { + shell_cmd().arg("-c").arg("false").spawn().unwrap() + }; + assert!(prog.wait().unwrap().code() == Some(1)); +} + +#[test] +#[cfg_attr(any(target_os = "vxworks"), ignore)] +fn test_finish_twice() { + let mut prog = if cfg!(target_os = "windows") { + Command::new("cmd").args(&["/C", "exit 1"]).spawn().unwrap() + } else { + shell_cmd().arg("-c").arg("false").spawn().unwrap() + }; + assert!(prog.wait().unwrap().code() == Some(1)); + assert!(prog.wait().unwrap().code() == Some(1)); +} + +#[test] +#[cfg_attr(any(target_os = "vxworks"), ignore)] +fn test_wait_with_output_once() { + let prog = if cfg!(target_os = "windows") { + Command::new("cmd").args(&["/C", "echo hello"]).stdout(Stdio::piped()).spawn().unwrap() + } else { + shell_cmd().arg("-c").arg("echo hello").stdout(Stdio::piped()).spawn().unwrap() + }; + + let Output { status, stdout, stderr } = prog.wait_with_output().unwrap(); + let output_str = str::from_utf8(&stdout).unwrap(); + + assert!(status.success()); + assert_eq!(output_str.trim().to_string(), "hello"); + assert_eq!(stderr, Vec::new()); +} + +#[cfg(all(unix, not(target_os = "android")))] +pub fn env_cmd() -> Command { + Command::new("env") +} +#[cfg(target_os = "android")] +pub fn env_cmd() -> Command { + let mut cmd = Command::new("/system/bin/sh"); + cmd.arg("-c").arg("set"); + cmd +} + +#[cfg(windows)] +pub fn env_cmd() -> Command { + let mut cmd = Command::new("cmd"); + cmd.arg("/c").arg("set"); + cmd +} + +#[test] +#[cfg_attr(target_os = "vxworks", ignore)] +fn test_override_env() { + use crate::env; + + // In some build environments (such as chrooted Nix builds), `env` can + // only be found in the explicitly-provided PATH env variable, not in + // default places such as /bin or /usr/bin. So we need to pass through + // PATH to our sub-process. + let mut cmd = env_cmd(); + cmd.env_clear().env("RUN_TEST_NEW_ENV", "123"); + if let Some(p) = env::var_os("PATH") { + cmd.env("PATH", &p); + } + let result = cmd.output().unwrap(); + let output = String::from_utf8_lossy(&result.stdout).to_string(); + + assert!( + output.contains("RUN_TEST_NEW_ENV=123"), + "didn't find RUN_TEST_NEW_ENV inside of:\n\n{output}", + ); +} + +#[test] +#[cfg_attr(target_os = "vxworks", ignore)] +fn test_add_to_env() { + let result = env_cmd().env("RUN_TEST_NEW_ENV", "123").output().unwrap(); + let output = String::from_utf8_lossy(&result.stdout).to_string(); + + assert!( + output.contains("RUN_TEST_NEW_ENV=123"), + "didn't find RUN_TEST_NEW_ENV inside of:\n\n{output}" + ); +} + +#[test] +#[cfg_attr(target_os = "vxworks", ignore)] +fn test_capture_env_at_spawn() { + use crate::env; + + let mut cmd = env_cmd(); + cmd.env("RUN_TEST_NEW_ENV1", "123"); + + // This variable will not be present if the environment has already + // been captured above. + env::set_var("RUN_TEST_NEW_ENV2", "456"); + let result = cmd.output().unwrap(); + env::remove_var("RUN_TEST_NEW_ENV2"); + + let output = String::from_utf8_lossy(&result.stdout).to_string(); + + assert!( + output.contains("RUN_TEST_NEW_ENV1=123"), + "didn't find RUN_TEST_NEW_ENV1 inside of:\n\n{output}" + ); + assert!( + output.contains("RUN_TEST_NEW_ENV2=456"), + "didn't find RUN_TEST_NEW_ENV2 inside of:\n\n{output}" + ); +} + +// Regression tests for #30858. +#[test] +fn test_interior_nul_in_progname_is_error() { + match Command::new("has-some-\0\0s-inside").spawn() { + Err(e) => assert_eq!(e.kind(), ErrorKind::InvalidInput), + Ok(_) => panic!(), + } +} + +#[test] +fn test_interior_nul_in_arg_is_error() { + match known_command().arg("has-some-\0\0s-inside").spawn() { + Err(e) => assert_eq!(e.kind(), ErrorKind::InvalidInput), + Ok(_) => panic!(), + } +} + +#[test] +fn test_interior_nul_in_args_is_error() { + match known_command().args(&["has-some-\0\0s-inside"]).spawn() { + Err(e) => assert_eq!(e.kind(), ErrorKind::InvalidInput), + Ok(_) => panic!(), + } +} + +#[test] +fn test_interior_nul_in_current_dir_is_error() { + match known_command().current_dir("has-some-\0\0s-inside").spawn() { + Err(e) => assert_eq!(e.kind(), ErrorKind::InvalidInput), + Ok(_) => panic!(), + } +} + +// Regression tests for #30862. +#[test] +#[cfg_attr(target_os = "vxworks", ignore)] +fn test_interior_nul_in_env_key_is_error() { + match env_cmd().env("has-some-\0\0s-inside", "value").spawn() { + Err(e) => assert_eq!(e.kind(), ErrorKind::InvalidInput), + Ok(_) => panic!(), + } +} + +#[test] +#[cfg_attr(target_os = "vxworks", ignore)] +fn test_interior_nul_in_env_value_is_error() { + match env_cmd().env("key", "has-some-\0\0s-inside").spawn() { + Err(e) => assert_eq!(e.kind(), ErrorKind::InvalidInput), + Ok(_) => panic!(), + } +} + +/// Tests that process creation flags work by debugging a process. +/// Other creation flags make it hard or impossible to detect +/// behavioral changes in the process. +#[test] +#[cfg(windows)] +fn test_creation_flags() { + use crate::os::windows::process::CommandExt; + use crate::sys::c::{BOOL, DWORD, INFINITE}; + #[repr(C, packed)] + struct DEBUG_EVENT { + pub event_code: DWORD, + pub process_id: DWORD, + pub thread_id: DWORD, + // This is a union in the real struct, but we don't + // need this data for the purposes of this test. + pub _junk: [u8; 164], + } + + extern "system" { + fn WaitForDebugEvent(lpDebugEvent: *mut DEBUG_EVENT, dwMilliseconds: DWORD) -> BOOL; + fn ContinueDebugEvent( + dwProcessId: DWORD, + dwThreadId: DWORD, + dwContinueStatus: DWORD, + ) -> BOOL; + } + + const DEBUG_PROCESS: DWORD = 1; + const EXIT_PROCESS_DEBUG_EVENT: DWORD = 5; + const DBG_EXCEPTION_NOT_HANDLED: DWORD = 0x80010001; + + let mut child = + Command::new("cmd").creation_flags(DEBUG_PROCESS).stdin(Stdio::piped()).spawn().unwrap(); + child.stdin.take().unwrap().write_all(b"exit\r\n").unwrap(); + let mut events = 0; + let mut event = DEBUG_EVENT { event_code: 0, process_id: 0, thread_id: 0, _junk: [0; 164] }; + loop { + if unsafe { WaitForDebugEvent(&mut event as *mut DEBUG_EVENT, INFINITE) } == 0 { + panic!("WaitForDebugEvent failed!"); + } + events += 1; + + if event.event_code == EXIT_PROCESS_DEBUG_EVENT { + break; + } + + if unsafe { + ContinueDebugEvent(event.process_id, event.thread_id, DBG_EXCEPTION_NOT_HANDLED) + } == 0 + { + panic!("ContinueDebugEvent failed!"); + } + } + assert!(events > 0); +} + +#[test] +fn test_command_implements_send_sync() { + fn take_send_sync_type(_: T) {} + take_send_sync_type(Command::new("")) +} + +// Ensure that starting a process with no environment variables works on Windows. +// This will fail if the environment block is ill-formed. +#[test] +#[cfg(windows)] +fn env_empty() { + let p = Command::new("cmd").args(&["/C", "exit 0"]).env_clear().spawn(); + assert!(p.is_ok()); +} + +#[test] +#[cfg(not(windows))] +#[cfg_attr(any(target_os = "emscripten", target_env = "sgx"), ignore)] +fn main() { + const PIDFD: &'static str = + if cfg!(target_os = "linux") { " create_pidfd: false,\n" } else { "" }; + + let mut command = Command::new("some-boring-name"); + + assert_eq!(format!("{command:?}"), format!(r#""some-boring-name""#)); + + assert_eq!( + format!("{command:#?}"), + format!( + r#"Command {{ + program: "some-boring-name", + args: [ + "some-boring-name", + ], +{PIDFD}}}"# + ) + ); + + command.args(&["1", "2", "3"]); + + assert_eq!(format!("{command:?}"), format!(r#""some-boring-name" "1" "2" "3""#)); + + assert_eq!( + format!("{command:#?}"), + format!( + r#"Command {{ + program: "some-boring-name", + args: [ + "some-boring-name", + "1", + "2", + "3", + ], +{PIDFD}}}"# + ) + ); + + crate::os::unix::process::CommandExt::arg0(&mut command, "exciting-name"); + + assert_eq!( + format!("{command:?}"), + format!(r#"["some-boring-name"] "exciting-name" "1" "2" "3""#) + ); + + assert_eq!( + format!("{command:#?}"), + format!( + r#"Command {{ + program: "some-boring-name", + args: [ + "exciting-name", + "1", + "2", + "3", + ], +{PIDFD}}}"# + ) + ); + + let mut command_with_env_and_cwd = Command::new("boring-name"); + command_with_env_and_cwd.current_dir("/some/path").env("FOO", "bar"); + assert_eq!( + format!("{command_with_env_and_cwd:?}"), + r#"cd "/some/path" && FOO="bar" "boring-name""# + ); + assert_eq!( + format!("{command_with_env_and_cwd:#?}"), + format!( + r#"Command {{ + program: "boring-name", + args: [ + "boring-name", + ], + env: CommandEnv {{ + clear: false, + vars: {{ + "FOO": Some( + "bar", + ), + }}, + }}, + cwd: Some( + "/some/path", + ), +{PIDFD}}}"# + ) + ); +} + +// See issue #91991 +#[test] +#[cfg(windows)] +fn run_bat_script() { + let tempdir = crate::sys_common::io::test::tmpdir(); + let script_path = tempdir.join("hello.cmd"); + + crate::fs::write(&script_path, "@echo Hello, %~1!").unwrap(); + let output = Command::new(&script_path) + .arg("fellow Rustaceans") + .stdout(crate::process::Stdio::piped()) + .spawn() + .unwrap() + .wait_with_output() + .unwrap(); + assert!(output.status.success()); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "Hello, fellow Rustaceans!"); +} + +// See issue #95178 +#[test] +#[cfg(windows)] +fn run_canonical_bat_script() { + let tempdir = crate::sys_common::io::test::tmpdir(); + let script_path = tempdir.join("hello.cmd"); + + crate::fs::write(&script_path, "@echo Hello, %~1!").unwrap(); + + // Try using a canonical path + let output = Command::new(&script_path.canonicalize().unwrap()) + .arg("fellow Rustaceans") + .stdout(crate::process::Stdio::piped()) + .spawn() + .unwrap() + .wait_with_output() + .unwrap(); + assert!(output.status.success()); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "Hello, fellow Rustaceans!"); +} diff --git a/crux-mir/lib/std/src/rt.rs b/crux-mir/lib/std/src/rt.rs index 2426b2dea..f1eeb75be 100644 --- a/crux-mir/lib/std/src/rt.rs +++ b/crux-mir/lib/std/src/rt.rs @@ -13,48 +13,145 @@ issue = "none" )] #![doc(hidden)] +#![deny(unsafe_op_in_unsafe_fn)] +#![allow(unused_macros)] + +use crate::ffi::CString; // Re-export some of our utilities which are expected by other crates. -pub use crate::panicking::{begin_panic, begin_panic_fmt, update_panic_count}; +pub use crate::panicking::{begin_panic, panic_count}; +pub use core::panicking::{panic_display, panic_fmt}; -// To reduce the generated code of the new `lang_start`, this function is doing -// the real work. -#[cfg(not(test))] -fn lang_start_internal( - main: &(dyn Fn() -> i32 + Sync + crate::panic::RefUnwindSafe), - argc: isize, - argv: *const *const u8, -) -> isize { - use crate::panic; - use crate::sys; - use crate::sys_common; - use crate::sys_common::thread_info; - use crate::thread::Thread; +use crate::sync::Once; +use crate::sys; +use crate::sys_common::thread_info; +use crate::thread::Thread; + +// Prints to the "panic output", depending on the platform this may be: +// - the standard error output +// - some dedicated platform specific output +// - nothing (so this macro is a no-op) +macro_rules! rtprintpanic { + ($($t:tt)*) => { + if let Some(mut out) = crate::sys::stdio::panic_output() { + let _ = crate::io::Write::write_fmt(&mut out, format_args!($($t)*)); + } + } +} + +macro_rules! rtabort { + ($($t:tt)*) => { + { + rtprintpanic!("fatal runtime error: {}\n", format_args!($($t)*)); + crate::sys::abort_internal(); + } + } +} + +macro_rules! rtassert { + ($e:expr) => { + if !$e { + rtabort!(concat!("assertion failed: ", stringify!($e))); + } + }; +} - sys::init(); +macro_rules! rtunwrap { + ($ok:ident, $e:expr) => { + match $e { + $ok(v) => v, + ref err => { + let err = err.as_ref().map(drop); // map Ok/Some which might not be Debug + rtabort!(concat!("unwrap failed: ", stringify!($e), " = {:?}"), err) + } + } + }; +} +// One-time runtime initialization. +// Runs before `main`. +// SAFETY: must be called only once during runtime initialization. +// NOTE: this is not guaranteed to run, for example when Rust code is called externally. +// +// # The `sigpipe` parameter +// +// Since 2014, the Rust runtime on Unix has set the `SIGPIPE` handler to +// `SIG_IGN`. Applications have good reasons to want a different behavior +// though, so there is a `#[unix_sigpipe = "..."]` attribute on `fn main()` that +// can be used to select how `SIGPIPE` shall be setup (if changed at all) before +// `fn main()` is called. See +// for more info. +// +// The `sigpipe` parameter to this function gets its value via the code that +// rustc generates to invoke `fn lang_start()`. The reason we have `sigpipe` for +// all platforms and not only Unix, is because std is not allowed to have `cfg` +// directives as this high level. See the module docs in +// `src/tools/tidy/src/pal.rs` for more info. On all other platforms, `sigpipe` +// has a value, but its value is ignored. +// +// Even though it is an `u8`, it only ever has 4 values. These are documented in +// `compiler/rustc_session/src/config/sigpipe.rs`. +#[cfg_attr(test, allow(dead_code))] +unsafe fn init(argc: isize, argv: *const *const u8, sigpipe: u8) { unsafe { - let main_guard = sys::thread::guard::init(); - sys::stack_overflow::init(); + sys::init(argc, argv, sigpipe); + let main_guard = sys::thread::guard::init(); // Next, set up the current Thread with the guard information we just // created. Note that this isn't necessary in general for new threads, // but we just do this to name the main thread and to give it correct // info about the stack bounds. - let thread = Thread::new(Some("main".to_owned())); + let thread = Thread::new(Some(rtunwrap!(Ok, CString::new("main")))); thread_info::set(main_guard, thread); + } +} - // Store our args if necessary in a squirreled away location - sys::args::init(argc, argv); +// One-time runtime cleanup. +// Runs after `main` or at program exit. +// NOTE: this is not guaranteed to run, for example when the program aborts. +pub(crate) fn cleanup() { + static CLEANUP: Once = Once::new(); + CLEANUP.call_once(|| unsafe { + // Flush stdout and disable buffering. + crate::io::cleanup(); + // SAFETY: Only called once during runtime cleanup. + sys::cleanup(); + }); +} - // Let's run some code! - let exit_code = panic::catch_unwind(|| { - sys_common::backtrace::__rust_begin_short_backtrace(move || main()) +// To reduce the generated code of the new `lang_start`, this function is doing +// the real work. +#[cfg(not(test))] +fn lang_start_internal( + main: &(dyn Fn() -> i32 + Sync + crate::panic::RefUnwindSafe), + argc: isize, + argv: *const *const u8, + sigpipe: u8, +) -> Result { + use crate::{mem, panic}; + let rt_abort = move |e| { + mem::forget(e); + rtabort!("initialization or cleanup bug"); + }; + // Guard against the code called by this function from unwinding outside of the Rust-controlled + // code, which is UB. This is a requirement imposed by a combination of how the + // `#[lang="start"]` attribute is implemented as well as by the implementation of the panicking + // mechanism itself. + // + // There are a couple of instances where unwinding can begin. First is inside of the + // `rt::init`, `rt::cleanup` and similar functions controlled by bstd. In those instances a + // panic is a std implementation bug. A quite likely one too, as there isn't any way to + // prevent std from accidentally introducing a panic to these functions. Another is from + // user code from `main` or, more nefariously, as described in e.g. issue #86030. + // SAFETY: Only called once during runtime initialization. + panic::catch_unwind(move || unsafe { init(argc, argv, sigpipe) }).map_err(rt_abort)?; + let ret_code = panic::catch_unwind(move || panic::catch_unwind(main).unwrap_or(101) as isize) + .map_err(move |e| { + mem::forget(e); + rtabort!("drop of the panic payload panicked"); }); - - sys_common::cleanup(); - exit_code.unwrap_or(101) as isize - } + panic::catch_unwind(cleanup).map_err(rt_abort)?; + ret_code } #[cfg(not(test))] @@ -63,6 +160,13 @@ fn lang_start( main: fn() -> T, argc: isize, argv: *const *const u8, + sigpipe: u8, ) -> isize { - lang_start_internal(&move || main().report(), argc, argv) + let Ok(v) = lang_start_internal( + &move || crate::sys_common::backtrace::__rust_begin_short_backtrace(main).report().to_i32(), + argc, + argv, + sigpipe, + ); + v } diff --git a/crux-mir/lib/std/src/sync/barrier.rs b/crux-mir/lib/std/src/sync/barrier.rs index 01314370c..11836b7b6 100644 --- a/crux-mir/lib/std/src/sync/barrier.rs +++ b/crux-mir/lib/std/src/sync/barrier.rs @@ -1,3 +1,6 @@ +#[cfg(test)] +mod tests; + use crate::fmt; use crate::sync::{Condvar, Mutex}; @@ -13,7 +16,7 @@ use crate::sync::{Condvar, Mutex}; /// let mut handles = Vec::with_capacity(10); /// let barrier = Arc::new(Barrier::new(10)); /// for _ in 0..10 { -/// let c = barrier.clone(); +/// let c = Arc::clone(&barrier); /// // The same messages will be printed together. /// // You will NOT see any interleaving. /// handles.push(thread::spawn(move|| { @@ -40,11 +43,8 @@ struct BarrierState { generation_id: usize, } -/// A `BarrierWaitResult` is returned by [`wait`] when all threads in the [`Barrier`] -/// have rendezvoused. -/// -/// [`wait`]: struct.Barrier.html#method.wait -/// [`Barrier`]: struct.Barrier.html +/// A `BarrierWaitResult` is returned by [`Barrier::wait()`] when all threads +/// in the [`Barrier`] have rendezvoused. /// /// # Examples /// @@ -60,17 +60,17 @@ pub struct BarrierWaitResult(bool); #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for Barrier { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("Barrier { .. }") + f.debug_struct("Barrier").finish_non_exhaustive() } } impl Barrier { /// Creates a new barrier that can block a given number of threads. /// - /// A barrier will block `n`-1 threads which call [`wait`] and then wake up - /// all threads at once when the `n`th thread calls [`wait`]. + /// A barrier will block `n`-1 threads which call [`wait()`] and then wake + /// up all threads at once when the `n`th thread calls [`wait()`]. /// - /// [`wait`]: #method.wait + /// [`wait()`]: Barrier::wait /// /// # Examples /// @@ -80,6 +80,7 @@ impl Barrier { /// let barrier = Barrier::new(10); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] pub fn new(n: usize) -> Barrier { Barrier { lock: Mutex::new(BarrierState { count: 0, generation_id: 0 }), @@ -94,12 +95,9 @@ impl Barrier { /// be used continuously. /// /// A single (arbitrary) thread will receive a [`BarrierWaitResult`] that - /// returns `true` from [`is_leader`] when returning from this function, and - /// all other threads will receive a result that will return `false` from - /// [`is_leader`]. - /// - /// [`BarrierWaitResult`]: struct.BarrierWaitResult.html - /// [`is_leader`]: struct.BarrierWaitResult.html#method.is_leader + /// returns `true` from [`BarrierWaitResult::is_leader()`] when returning + /// from this function, and all other threads will receive a result that + /// will return `false` from [`BarrierWaitResult::is_leader()`]. /// /// # Examples /// @@ -110,7 +108,7 @@ impl Barrier { /// let mut handles = Vec::with_capacity(10); /// let barrier = Arc::new(Barrier::new(10)); /// for _ in 0..10 { - /// let c = barrier.clone(); + /// let c = Arc::clone(&barrier); /// // The same messages will be printed together. /// // You will NOT see any interleaving. /// handles.push(thread::spawn(move|| { @@ -131,8 +129,8 @@ impl Barrier { lock.count += 1; if lock.count < self.num_threads { // We need a while loop to guard against spurious wakeups. - // http://en.wikipedia.org/wiki/Spurious_wakeup - while local_gen == lock.generation_id && lock.count < self.num_threads { + // https://en.wikipedia.org/wiki/Spurious_wakeup + while local_gen == lock.generation_id { lock = self.cvar.wait(lock).unwrap(); } BarrierWaitResult(false) @@ -153,13 +151,12 @@ impl fmt::Debug for BarrierWaitResult { } impl BarrierWaitResult { - /// Returns `true` if this thread from [`wait`] is the "leader thread". + /// Returns `true` if this thread is the "leader thread" for the call to + /// [`Barrier::wait()`]. /// /// Only one thread will have `true` returned from their result, all other /// threads will have `false` returned. /// - /// [`wait`]: struct.Barrier.html#method.wait - /// /// # Examples /// /// ``` @@ -170,46 +167,8 @@ impl BarrierWaitResult { /// println!("{:?}", barrier_wait_result.is_leader()); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] pub fn is_leader(&self) -> bool { self.0 } } - -#[cfg(test)] -mod tests { - use crate::sync::mpsc::{channel, TryRecvError}; - use crate::sync::{Arc, Barrier}; - use crate::thread; - - #[test] - #[cfg_attr(target_os = "emscripten", ignore)] - fn test_barrier() { - const N: usize = 10; - - let barrier = Arc::new(Barrier::new(N)); - let (tx, rx) = channel(); - - for _ in 0..N - 1 { - let c = barrier.clone(); - let tx = tx.clone(); - thread::spawn(move || { - tx.send(c.wait().is_leader()).unwrap(); - }); - } - - // At this point, all spawned threads should be blocked, - // so we shouldn't get anything from the port - assert!(matches!(rx.try_recv(), Err(TryRecvError::Empty))); - - let mut leader_found = barrier.wait().is_leader(); - - // Now, the barrier is cleared and we should get data. - for _ in 0..N - 1 { - if rx.recv().unwrap() { - assert!(!leader_found); - leader_found = true; - } - } - assert!(leader_found); - } -} diff --git a/crux-mir/lib/std/src/sync/barrier/tests.rs b/crux-mir/lib/std/src/sync/barrier/tests.rs new file mode 100644 index 000000000..834a3e751 --- /dev/null +++ b/crux-mir/lib/std/src/sync/barrier/tests.rs @@ -0,0 +1,35 @@ +use crate::sync::mpsc::{channel, TryRecvError}; +use crate::sync::{Arc, Barrier}; +use crate::thread; + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn test_barrier() { + const N: usize = 10; + + let barrier = Arc::new(Barrier::new(N)); + let (tx, rx) = channel(); + + for _ in 0..N - 1 { + let c = barrier.clone(); + let tx = tx.clone(); + thread::spawn(move || { + tx.send(c.wait().is_leader()).unwrap(); + }); + } + + // At this point, all spawned threads should be blocked, + // so we shouldn't get anything from the port + assert!(matches!(rx.try_recv(), Err(TryRecvError::Empty))); + + let mut leader_found = barrier.wait().is_leader(); + + // Now, the barrier is cleared and we should get data. + for _ in 0..N - 1 { + if rx.recv().unwrap() { + assert!(!leader_found); + leader_found = true; + } + } + assert!(leader_found); +} diff --git a/crux-mir/lib/std/src/sync/condvar.rs b/crux-mir/lib/std/src/sync/condvar.rs index 77e521eae..76a1b4a2a 100644 --- a/crux-mir/lib/std/src/sync/condvar.rs +++ b/crux-mir/lib/std/src/sync/condvar.rs @@ -1,9 +1,9 @@ +#[cfg(test)] +mod tests; + use crate::fmt; -use crate::sync::atomic::{AtomicUsize, Ordering}; -use crate::sync::{mutex, MutexGuard, PoisonError}; -use crate::sys_common::condvar as sys; -use crate::sys_common::mutex as sys_mutex; -use crate::sys_common::poison::{self, LockResult}; +use crate::sync::{mutex, poison, LockResult, MutexGuard, PoisonError}; +use crate::sys::locks as sys; use crate::time::{Duration, Instant}; /// A type indicating whether a timed wait on a condition variable returned @@ -11,7 +11,7 @@ use crate::time::{Duration, Instant}; /// /// It is returned by the [`wait_timeout`] method. /// -/// [`wait_timeout`]: struct.Condvar.html#method.wait_timeout +/// [`wait_timeout`]: Condvar::wait_timeout #[derive(Debug, PartialEq, Eq, Copy, Clone)] #[stable(feature = "wait_timeout", since = "1.5.0")] pub struct WaitTimeoutResult(bool); @@ -33,7 +33,7 @@ impl WaitTimeoutResult { /// use std::time::Duration; /// /// let pair = Arc::new((Mutex::new(false), Condvar::new())); - /// let pair2 = pair.clone(); + /// let pair2 = Arc::clone(&pair); /// /// thread::spawn(move || { /// let (lock, cvar) = &*pair2; @@ -61,6 +61,7 @@ impl WaitTimeoutResult { /// } /// } /// ``` + #[must_use] #[stable(feature = "wait_timeout", since = "1.5.0")] pub fn timed_out(&self) -> bool { self.0 @@ -75,13 +76,9 @@ impl WaitTimeoutResult { /// and a mutex. The predicate is always verified inside of the mutex before /// determining that a thread must block. /// -/// Functions in this module will block the current **thread** of execution and -/// are bindings to system-provided condition variables where possible. Note -/// that this module places one additional restriction over the system condition -/// variables: each condvar can be used with precisely one mutex at runtime. Any -/// attempt to use multiple mutexes on the same condition variable will result -/// in a runtime panic. If this is not desired, then the unsafe primitives in -/// `sys` do not have this restriction but may result in undefined behavior. +/// Functions in this module will block the current **thread** of execution. +/// Note that any attempt to use multiple mutexes on the same condition +/// variable may result in a runtime panic. /// /// # Examples /// @@ -90,7 +87,7 @@ impl WaitTimeoutResult { /// use std::thread; /// /// let pair = Arc::new((Mutex::new(false), Condvar::new())); -/// let pair2 = pair.clone(); +/// let pair2 = Arc::clone(&pair); /// /// // Inside of our lock, spawn a new thread, and then wait for it to start. /// thread::spawn(move|| { @@ -110,8 +107,7 @@ impl WaitTimeoutResult { /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub struct Condvar { - inner: Box, - mutex: AtomicUsize, + inner: sys::Condvar, } impl Condvar { @@ -126,12 +122,11 @@ impl Condvar { /// let condvar = Condvar::new(); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn new() -> Condvar { - let mut c = Condvar { inner: box sys::Condvar::new(), mutex: AtomicUsize::new(0) }; - unsafe { - c.inner.init(); - } - c + #[rustc_const_stable(feature = "const_locks", since = "1.63.0")] + #[must_use] + #[inline] + pub const fn new() -> Condvar { + Condvar { inner: sys::Condvar::new() } } /// Blocks the current thread until this condition variable receives a @@ -156,16 +151,13 @@ impl Condvar { /// /// # Panics /// - /// This function will [`panic!`] if it is used with more than one mutex - /// over time. Each condition variable is dynamically bound to exactly one - /// mutex to ensure defined behavior across platforms. If this functionality - /// is not desired, then unsafe primitives in `sys` are provided. + /// This function may [`panic!`] if it is used with more than one mutex + /// over time. /// - /// [`notify_one`]: #method.notify_one - /// [`notify_all`]: #method.notify_all - /// [poisoning]: ../sync/struct.Mutex.html#poisoning - /// [`Mutex`]: ../sync/struct.Mutex.html - /// [`panic!`]: ../../std/macro.panic.html + /// [`notify_one`]: Self::notify_one + /// [`notify_all`]: Self::notify_all + /// [poisoning]: super::Mutex#poisoning + /// [`Mutex`]: super::Mutex /// /// # Examples /// @@ -174,7 +166,7 @@ impl Condvar { /// use std::thread; /// /// let pair = Arc::new((Mutex::new(false), Condvar::new())); - /// let pair2 = pair.clone(); + /// let pair2 = Arc::clone(&pair); /// /// thread::spawn(move|| { /// let (lock, cvar) = &*pair2; @@ -196,7 +188,6 @@ impl Condvar { pub fn wait<'a, T>(&self, guard: MutexGuard<'a, T>) -> LockResult> { let poisoned = unsafe { let lock = mutex::guard_lock(&guard); - self.verify(lock); self.inner.wait(lock); mutex::guard_poison(&guard).get() }; @@ -218,10 +209,10 @@ impl Condvar { /// poisoned when this thread re-acquires the lock. For more information, /// see information about [poisoning] on the [`Mutex`] type. /// - /// [`notify_one`]: #method.notify_one - /// [`notify_all`]: #method.notify_all - /// [poisoning]: ../sync/struct.Mutex.html#poisoning - /// [`Mutex`]: ../sync/struct.Mutex.html + /// [`notify_one`]: Self::notify_one + /// [`notify_all`]: Self::notify_all + /// [poisoning]: super::Mutex#poisoning + /// [`Mutex`]: super::Mutex /// /// # Examples /// @@ -230,7 +221,7 @@ impl Condvar { /// use std::thread; /// /// let pair = Arc::new((Mutex::new(true), Condvar::new())); - /// let pair2 = pair.clone(); + /// let pair2 = Arc::clone(&pair); /// /// thread::spawn(move|| { /// let (lock, cvar) = &*pair2; @@ -267,7 +258,7 @@ impl Condvar { /// except that the thread will be blocked for roughly no longer /// than `ms` milliseconds. This method should not be used for /// precise timing due to anomalies such as preemption or platform - /// differences that may not cause the maximum amount of time + /// differences that might not cause the maximum amount of time /// waited to be precisely `ms`. /// /// Note that the best effort is made to ensure that the time waited is @@ -280,7 +271,7 @@ impl Condvar { /// Like [`wait`], the lock specified will be re-acquired when this function /// returns, regardless of whether the timeout elapsed or not. /// - /// [`wait`]: #method.wait + /// [`wait`]: Self::wait /// /// # Examples /// @@ -289,7 +280,7 @@ impl Condvar { /// use std::thread; /// /// let pair = Arc::new((Mutex::new(false), Condvar::new())); - /// let pair2 = pair.clone(); + /// let pair2 = Arc::clone(&pair); /// /// thread::spawn(move|| { /// let (lock, cvar) = &*pair2; @@ -314,7 +305,7 @@ impl Condvar { /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_deprecated(since = "1.6.0", reason = "replaced by `std::sync::Condvar::wait_timeout`")] + #[deprecated(since = "1.6.0", note = "replaced by `std::sync::Condvar::wait_timeout`")] pub fn wait_timeout_ms<'a, T>( &self, guard: MutexGuard<'a, T>, @@ -330,7 +321,7 @@ impl Condvar { /// The semantics of this function are equivalent to [`wait`] except that /// the thread will be blocked for roughly no longer than `dur`. This /// method should not be used for precise timing due to anomalies such as - /// preemption or platform differences that may not cause the maximum + /// preemption or platform differences that might not cause the maximum /// amount of time waited to be precisely `dur`. /// /// Note that the best effort is made to ensure that the time waited is @@ -350,9 +341,8 @@ impl Condvar { /// Like [`wait`], the lock specified will be re-acquired when this function /// returns, regardless of whether the timeout elapsed or not. /// - /// [`wait`]: #method.wait - /// [`wait_timeout_while`]: #method.wait_timeout_while - /// [`WaitTimeoutResult`]: struct.WaitTimeoutResult.html + /// [`wait`]: Self::wait + /// [`wait_timeout_while`]: Self::wait_timeout_while /// /// # Examples /// @@ -362,7 +352,7 @@ impl Condvar { /// use std::time::Duration; /// /// let pair = Arc::new((Mutex::new(false), Condvar::new())); - /// let pair2 = pair.clone(); + /// let pair2 = Arc::clone(&pair); /// /// thread::spawn(move|| { /// let (lock, cvar) = &*pair2; @@ -394,7 +384,6 @@ impl Condvar { ) -> LockResult<(MutexGuard<'a, T>, WaitTimeoutResult)> { let (poisoned, result) = unsafe { let lock = mutex::guard_lock(&guard); - self.verify(lock); let success = self.inner.wait_timeout(lock, dur); (mutex::guard_poison(&guard).get(), WaitTimeoutResult(!success)) }; @@ -407,7 +396,7 @@ impl Condvar { /// The semantics of this function are equivalent to [`wait_while`] except /// that the thread will be blocked for roughly no longer than `dur`. This /// method should not be used for precise timing due to anomalies such as - /// preemption or platform differences that may not cause the maximum + /// preemption or platform differences that might not cause the maximum /// amount of time waited to be precisely `dur`. /// /// Note that the best effort is made to ensure that the time waited is @@ -420,9 +409,8 @@ impl Condvar { /// Like [`wait_while`], the lock specified will be re-acquired when this /// function returns, regardless of whether the timeout elapsed or not. /// - /// [`wait_while`]: #method.wait_while - /// [`wait_timeout`]: #method.wait_timeout - /// [`WaitTimeoutResult`]: struct.WaitTimeoutResult.html + /// [`wait_while`]: Self::wait_while + /// [`wait_timeout`]: Self::wait_timeout /// /// # Examples /// @@ -432,7 +420,7 @@ impl Condvar { /// use std::time::Duration; /// /// let pair = Arc::new((Mutex::new(true), Condvar::new())); - /// let pair2 = pair.clone(); + /// let pair2 = Arc::clone(&pair); /// /// thread::spawn(move|| { /// let (lock, cvar) = &*pair2; @@ -485,9 +473,9 @@ impl Condvar { /// /// To wake up all threads, see [`notify_all`]. /// - /// [`wait`]: #method.wait - /// [`wait_timeout`]: #method.wait_timeout - /// [`notify_all`]: #method.notify_all + /// [`wait`]: Self::wait + /// [`wait_timeout`]: Self::wait_timeout + /// [`notify_all`]: Self::notify_all /// /// # Examples /// @@ -496,7 +484,7 @@ impl Condvar { /// use std::thread; /// /// let pair = Arc::new((Mutex::new(false), Condvar::new())); - /// let pair2 = pair.clone(); + /// let pair2 = Arc::clone(&pair); /// /// thread::spawn(move|| { /// let (lock, cvar) = &*pair2; @@ -516,7 +504,7 @@ impl Condvar { /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn notify_one(&self) { - unsafe { self.inner.notify_one() } + self.inner.notify_one() } /// Wakes up all blocked threads on this condvar. @@ -527,7 +515,7 @@ impl Condvar { /// /// To wake up only one thread, see [`notify_one`]. /// - /// [`notify_one`]: #method.notify_one + /// [`notify_one`]: Self::notify_one /// /// # Examples /// @@ -536,7 +524,7 @@ impl Condvar { /// use std::thread; /// /// let pair = Arc::new((Mutex::new(false), Condvar::new())); - /// let pair2 = pair.clone(); + /// let pair2 = Arc::clone(&pair); /// /// thread::spawn(move|| { /// let (lock, cvar) = &*pair2; @@ -556,34 +544,14 @@ impl Condvar { /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn notify_all(&self) { - unsafe { self.inner.notify_all() } - } - - fn verify(&self, mutex: &sys_mutex::Mutex) { - let addr = mutex as *const _ as usize; - match self.mutex.compare_and_swap(0, addr, Ordering::SeqCst) { - // If we got out 0, then we have successfully bound the mutex to - // this cvar. - 0 => {} - - // If we get out a value that's the same as `addr`, then someone - // already beat us to the punch. - n if n == addr => {} - - // Anything else and we're using more than one mutex on this cvar, - // which is currently disallowed. - _ => panic!( - "attempted to use a condition variable with two \ - mutexes" - ), - } + self.inner.notify_all() } } #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for Condvar { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("Condvar { .. }") + f.debug_struct("Condvar").finish_non_exhaustive() } } @@ -594,230 +562,3 @@ impl Default for Condvar { Condvar::new() } } - -#[stable(feature = "rust1", since = "1.0.0")] -impl Drop for Condvar { - fn drop(&mut self) { - unsafe { self.inner.destroy() } - } -} - -#[cfg(test)] -mod tests { - use crate::sync::atomic::{AtomicBool, Ordering}; - use crate::sync::mpsc::channel; - use crate::sync::{Arc, Condvar, Mutex}; - use crate::thread; - use crate::time::Duration; - use crate::u64; - - #[test] - fn smoke() { - let c = Condvar::new(); - c.notify_one(); - c.notify_all(); - } - - #[test] - #[cfg_attr(target_os = "emscripten", ignore)] - fn notify_one() { - let m = Arc::new(Mutex::new(())); - let m2 = m.clone(); - let c = Arc::new(Condvar::new()); - let c2 = c.clone(); - - let g = m.lock().unwrap(); - let _t = thread::spawn(move || { - let _g = m2.lock().unwrap(); - c2.notify_one(); - }); - let g = c.wait(g).unwrap(); - drop(g); - } - - #[test] - #[cfg_attr(target_os = "emscripten", ignore)] - fn notify_all() { - const N: usize = 10; - - let data = Arc::new((Mutex::new(0), Condvar::new())); - let (tx, rx) = channel(); - for _ in 0..N { - let data = data.clone(); - let tx = tx.clone(); - thread::spawn(move || { - let &(ref lock, ref cond) = &*data; - let mut cnt = lock.lock().unwrap(); - *cnt += 1; - if *cnt == N { - tx.send(()).unwrap(); - } - while *cnt != 0 { - cnt = cond.wait(cnt).unwrap(); - } - tx.send(()).unwrap(); - }); - } - drop(tx); - - let &(ref lock, ref cond) = &*data; - rx.recv().unwrap(); - let mut cnt = lock.lock().unwrap(); - *cnt = 0; - cond.notify_all(); - drop(cnt); - - for _ in 0..N { - rx.recv().unwrap(); - } - } - - #[test] - #[cfg_attr(target_os = "emscripten", ignore)] - fn wait_while() { - let pair = Arc::new((Mutex::new(false), Condvar::new())); - let pair2 = pair.clone(); - - // Inside of our lock, spawn a new thread, and then wait for it to start. - thread::spawn(move || { - let &(ref lock, ref cvar) = &*pair2; - let mut started = lock.lock().unwrap(); - *started = true; - // We notify the condvar that the value has changed. - cvar.notify_one(); - }); - - // Wait for the thread to start up. - let &(ref lock, ref cvar) = &*pair; - let guard = cvar.wait_while(lock.lock().unwrap(), |started| !*started); - assert!(*guard.unwrap()); - } - - #[test] - #[cfg_attr(target_os = "emscripten", ignore)] - #[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - fn wait_timeout_wait() { - let m = Arc::new(Mutex::new(())); - let c = Arc::new(Condvar::new()); - - loop { - let g = m.lock().unwrap(); - let (_g, no_timeout) = c.wait_timeout(g, Duration::from_millis(1)).unwrap(); - // spurious wakeups mean this isn't necessarily true - // so execute test again, if not timeout - if !no_timeout.timed_out() { - continue; - } - - break; - } - } - - #[test] - #[cfg_attr(target_os = "emscripten", ignore)] - #[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - fn wait_timeout_while_wait() { - let m = Arc::new(Mutex::new(())); - let c = Arc::new(Condvar::new()); - - let g = m.lock().unwrap(); - let (_g, wait) = c.wait_timeout_while(g, Duration::from_millis(1), |_| true).unwrap(); - // no spurious wakeups. ensure it timed-out - assert!(wait.timed_out()); - } - - #[test] - #[cfg_attr(target_os = "emscripten", ignore)] - fn wait_timeout_while_instant_satisfy() { - let m = Arc::new(Mutex::new(())); - let c = Arc::new(Condvar::new()); - - let g = m.lock().unwrap(); - let (_g, wait) = c.wait_timeout_while(g, Duration::from_millis(0), |_| false).unwrap(); - // ensure it didn't time-out even if we were not given any time. - assert!(!wait.timed_out()); - } - - #[test] - #[cfg_attr(target_os = "emscripten", ignore)] - #[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - fn wait_timeout_while_wake() { - let pair = Arc::new((Mutex::new(false), Condvar::new())); - let pair_copy = pair.clone(); - - let &(ref m, ref c) = &*pair; - let g = m.lock().unwrap(); - let _t = thread::spawn(move || { - let &(ref lock, ref cvar) = &*pair_copy; - let mut started = lock.lock().unwrap(); - thread::sleep(Duration::from_millis(1)); - *started = true; - cvar.notify_one(); - }); - let (g2, wait) = c - .wait_timeout_while(g, Duration::from_millis(u64::MAX), |&mut notified| !notified) - .unwrap(); - // ensure it didn't time-out even if we were not given any time. - assert!(!wait.timed_out()); - assert!(*g2); - } - - #[test] - #[cfg_attr(target_os = "emscripten", ignore)] - #[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - fn wait_timeout_wake() { - let m = Arc::new(Mutex::new(())); - let c = Arc::new(Condvar::new()); - - loop { - let g = m.lock().unwrap(); - - let c2 = c.clone(); - let m2 = m.clone(); - - let notified = Arc::new(AtomicBool::new(false)); - let notified_copy = notified.clone(); - - let t = thread::spawn(move || { - let _g = m2.lock().unwrap(); - thread::sleep(Duration::from_millis(1)); - notified_copy.store(true, Ordering::SeqCst); - c2.notify_one(); - }); - let (g, timeout_res) = c.wait_timeout(g, Duration::from_millis(u64::MAX)).unwrap(); - assert!(!timeout_res.timed_out()); - // spurious wakeups mean this isn't necessarily true - // so execute test again, if not notified - if !notified.load(Ordering::SeqCst) { - t.join().unwrap(); - continue; - } - drop(g); - - t.join().unwrap(); - - break; - } - } - - #[test] - #[should_panic] - #[cfg_attr(target_os = "emscripten", ignore)] - fn two_mutexes() { - let m = Arc::new(Mutex::new(())); - let m2 = m.clone(); - let c = Arc::new(Condvar::new()); - let c2 = c.clone(); - - let mut g = m.lock().unwrap(); - let _t = thread::spawn(move || { - let _g = m2.lock().unwrap(); - c2.notify_one(); - }); - g = c.wait(g).unwrap(); - drop(g); - - let m = Mutex::new(()); - let _ = c.wait(m.lock().unwrap()).unwrap(); - } -} diff --git a/crux-mir/lib/std/src/sync/condvar/tests.rs b/crux-mir/lib/std/src/sync/condvar/tests.rs new file mode 100644 index 000000000..24f467f0b --- /dev/null +++ b/crux-mir/lib/std/src/sync/condvar/tests.rs @@ -0,0 +1,190 @@ +use crate::sync::atomic::{AtomicBool, Ordering}; +use crate::sync::mpsc::channel; +use crate::sync::{Arc, Condvar, Mutex}; +use crate::thread; +use crate::time::Duration; + +#[test] +fn smoke() { + let c = Condvar::new(); + c.notify_one(); + c.notify_all(); +} + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn notify_one() { + let m = Arc::new(Mutex::new(())); + let m2 = m.clone(); + let c = Arc::new(Condvar::new()); + let c2 = c.clone(); + + let g = m.lock().unwrap(); + let _t = thread::spawn(move || { + let _g = m2.lock().unwrap(); + c2.notify_one(); + }); + let g = c.wait(g).unwrap(); + drop(g); +} + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn notify_all() { + const N: usize = 10; + + let data = Arc::new((Mutex::new(0), Condvar::new())); + let (tx, rx) = channel(); + for _ in 0..N { + let data = data.clone(); + let tx = tx.clone(); + thread::spawn(move || { + let &(ref lock, ref cond) = &*data; + let mut cnt = lock.lock().unwrap(); + *cnt += 1; + if *cnt == N { + tx.send(()).unwrap(); + } + while *cnt != 0 { + cnt = cond.wait(cnt).unwrap(); + } + tx.send(()).unwrap(); + }); + } + drop(tx); + + let &(ref lock, ref cond) = &*data; + rx.recv().unwrap(); + let mut cnt = lock.lock().unwrap(); + *cnt = 0; + cond.notify_all(); + drop(cnt); + + for _ in 0..N { + rx.recv().unwrap(); + } +} + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn wait_while() { + let pair = Arc::new((Mutex::new(false), Condvar::new())); + let pair2 = pair.clone(); + + // Inside of our lock, spawn a new thread, and then wait for it to start. + thread::spawn(move || { + let &(ref lock, ref cvar) = &*pair2; + let mut started = lock.lock().unwrap(); + *started = true; + // We notify the condvar that the value has changed. + cvar.notify_one(); + }); + + // Wait for the thread to start up. + let &(ref lock, ref cvar) = &*pair; + let guard = cvar.wait_while(lock.lock().unwrap(), |started| !*started); + assert!(*guard.unwrap()); +} + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn wait_timeout_wait() { + let m = Arc::new(Mutex::new(())); + let c = Arc::new(Condvar::new()); + + loop { + let g = m.lock().unwrap(); + let (_g, no_timeout) = c.wait_timeout(g, Duration::from_millis(1)).unwrap(); + // spurious wakeups mean this isn't necessarily true + // so execute test again, if not timeout + if !no_timeout.timed_out() { + continue; + } + + break; + } +} + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn wait_timeout_while_wait() { + let m = Arc::new(Mutex::new(())); + let c = Arc::new(Condvar::new()); + + let g = m.lock().unwrap(); + let (_g, wait) = c.wait_timeout_while(g, Duration::from_millis(1), |_| true).unwrap(); + // no spurious wakeups. ensure it timed-out + assert!(wait.timed_out()); +} + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn wait_timeout_while_instant_satisfy() { + let m = Arc::new(Mutex::new(())); + let c = Arc::new(Condvar::new()); + + let g = m.lock().unwrap(); + let (_g, wait) = c.wait_timeout_while(g, Duration::from_millis(0), |_| false).unwrap(); + // ensure it didn't time-out even if we were not given any time. + assert!(!wait.timed_out()); +} + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn wait_timeout_while_wake() { + let pair = Arc::new((Mutex::new(false), Condvar::new())); + let pair_copy = pair.clone(); + + let &(ref m, ref c) = &*pair; + let g = m.lock().unwrap(); + let _t = thread::spawn(move || { + let &(ref lock, ref cvar) = &*pair_copy; + let mut started = lock.lock().unwrap(); + thread::sleep(Duration::from_millis(1)); + *started = true; + cvar.notify_one(); + }); + let (g2, wait) = c + .wait_timeout_while(g, Duration::from_millis(u64::MAX), |&mut notified| !notified) + .unwrap(); + // ensure it didn't time-out even if we were not given any time. + assert!(!wait.timed_out()); + assert!(*g2); +} + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn wait_timeout_wake() { + let m = Arc::new(Mutex::new(())); + let c = Arc::new(Condvar::new()); + + loop { + let g = m.lock().unwrap(); + + let c2 = c.clone(); + let m2 = m.clone(); + + let notified = Arc::new(AtomicBool::new(false)); + let notified_copy = notified.clone(); + + let t = thread::spawn(move || { + let _g = m2.lock().unwrap(); + thread::sleep(Duration::from_millis(1)); + notified_copy.store(true, Ordering::SeqCst); + c2.notify_one(); + }); + let (g, timeout_res) = c.wait_timeout(g, Duration::from_millis(u64::MAX)).unwrap(); + assert!(!timeout_res.timed_out()); + // spurious wakeups mean this isn't necessarily true + // so execute test again, if not notified + if !notified.load(Ordering::SeqCst) { + t.join().unwrap(); + continue; + } + drop(g); + + t.join().unwrap(); + + break; + } +} diff --git a/crux-mir/lib/std/src/sync/lazy_lock.rs b/crux-mir/lib/std/src/sync/lazy_lock.rs new file mode 100644 index 000000000..4a1530530 --- /dev/null +++ b/crux-mir/lib/std/src/sync/lazy_lock.rs @@ -0,0 +1,125 @@ +use crate::cell::Cell; +use crate::fmt; +use crate::ops::Deref; +use crate::panic::{RefUnwindSafe, UnwindSafe}; +use crate::sync::OnceLock; + +/// A value which is initialized on the first access. +/// +/// This type is a thread-safe [`LazyCell`], and can be used in statics. +/// +/// [`LazyCell`]: crate::cell::LazyCell +/// +/// # Examples +/// +/// ``` +/// #![feature(once_cell)] +/// +/// use std::collections::HashMap; +/// +/// use std::sync::LazyLock; +/// +/// static HASHMAP: LazyLock> = LazyLock::new(|| { +/// println!("initializing"); +/// let mut m = HashMap::new(); +/// m.insert(13, "Spica".to_string()); +/// m.insert(74, "Hoyten".to_string()); +/// m +/// }); +/// +/// fn main() { +/// println!("ready"); +/// std::thread::spawn(|| { +/// println!("{:?}", HASHMAP.get(&13)); +/// }).join().unwrap(); +/// println!("{:?}", HASHMAP.get(&74)); +/// +/// // Prints: +/// // ready +/// // initializing +/// // Some("Spica") +/// // Some("Hoyten") +/// } +/// ``` +#[unstable(feature = "once_cell", issue = "74465")] +pub struct LazyLock T> { + cell: OnceLock, + init: Cell>, +} +impl T> LazyLock { + /// Creates a new lazy value with the given initializing + /// function. + #[inline] + #[unstable(feature = "once_cell", issue = "74465")] + pub const fn new(f: F) -> LazyLock { + LazyLock { cell: OnceLock::new(), init: Cell::new(Some(f)) } + } + + /// Forces the evaluation of this lazy value and + /// returns a reference to result. This is equivalent + /// to the `Deref` impl, but is explicit. + /// + /// # Examples + /// + /// ``` + /// #![feature(once_cell)] + /// + /// use std::sync::LazyLock; + /// + /// let lazy = LazyLock::new(|| 92); + /// + /// assert_eq!(LazyLock::force(&lazy), &92); + /// assert_eq!(&*lazy, &92); + /// ``` + #[inline] + #[unstable(feature = "once_cell", issue = "74465")] + pub fn force(this: &LazyLock) -> &T { + this.cell.get_or_init(|| match this.init.take() { + Some(f) => f(), + None => panic!("Lazy instance has previously been poisoned"), + }) + } +} + +#[unstable(feature = "once_cell", issue = "74465")] +impl T> Deref for LazyLock { + type Target = T; + + #[inline] + fn deref(&self) -> &T { + LazyLock::force(self) + } +} + +#[unstable(feature = "once_cell", issue = "74465")] +impl Default for LazyLock { + /// Creates a new lazy value using `Default` as the initializing function. + #[inline] + fn default() -> LazyLock { + LazyLock::new(T::default) + } +} + +#[unstable(feature = "once_cell", issue = "74465")] +impl fmt::Debug for LazyLock { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Lazy").field("cell", &self.cell).finish_non_exhaustive() + } +} + +// We never create a `&F` from a `&LazyLock` so it is fine +// to not impl `Sync` for `F` +// we do create a `&mut Option` in `force`, but this is +// properly synchronized, so it only happens once +// so it also does not contribute to this impl. +#[unstable(feature = "once_cell", issue = "74465")] +unsafe impl Sync for LazyLock where OnceLock: Sync {} +// auto-derived `Send` impl is OK. + +#[unstable(feature = "once_cell", issue = "74465")] +impl RefUnwindSafe for LazyLock where OnceLock: RefUnwindSafe {} +#[unstable(feature = "once_cell", issue = "74465")] +impl UnwindSafe for LazyLock where OnceLock: UnwindSafe {} + +#[cfg(test)] +mod tests; diff --git a/crux-mir/lib/std/src/sync/lazy_lock/tests.rs b/crux-mir/lib/std/src/sync/lazy_lock/tests.rs new file mode 100644 index 000000000..a5d4e25c5 --- /dev/null +++ b/crux-mir/lib/std/src/sync/lazy_lock/tests.rs @@ -0,0 +1,149 @@ +use crate::{ + cell::LazyCell, + panic, + sync::{ + atomic::{AtomicUsize, Ordering::SeqCst}, + Mutex, + }, + sync::{LazyLock, OnceLock}, + thread, +}; + +fn spawn_and_wait(f: impl FnOnce() -> R + Send + 'static) -> R { + thread::spawn(f).join().unwrap() +} + +#[test] +fn lazy_default() { + static CALLED: AtomicUsize = AtomicUsize::new(0); + + struct Foo(u8); + impl Default for Foo { + fn default() -> Self { + CALLED.fetch_add(1, SeqCst); + Foo(42) + } + } + + let lazy: LazyCell> = <_>::default(); + + assert_eq!(CALLED.load(SeqCst), 0); + + assert_eq!(lazy.lock().unwrap().0, 42); + assert_eq!(CALLED.load(SeqCst), 1); + + lazy.lock().unwrap().0 = 21; + + assert_eq!(lazy.lock().unwrap().0, 21); + assert_eq!(CALLED.load(SeqCst), 1); +} + +#[test] +fn lazy_poisoning() { + let x: LazyCell = LazyCell::new(|| panic!("kaboom")); + for _ in 0..2 { + let res = panic::catch_unwind(panic::AssertUnwindSafe(|| x.len())); + assert!(res.is_err()); + } +} + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn sync_lazy_new() { + static CALLED: AtomicUsize = AtomicUsize::new(0); + static SYNC_LAZY: LazyLock = LazyLock::new(|| { + CALLED.fetch_add(1, SeqCst); + 92 + }); + + assert_eq!(CALLED.load(SeqCst), 0); + + spawn_and_wait(|| { + let y = *SYNC_LAZY - 30; + assert_eq!(y, 62); + assert_eq!(CALLED.load(SeqCst), 1); + }); + + let y = *SYNC_LAZY - 30; + assert_eq!(y, 62); + assert_eq!(CALLED.load(SeqCst), 1); +} + +#[test] +fn sync_lazy_default() { + static CALLED: AtomicUsize = AtomicUsize::new(0); + + struct Foo(u8); + impl Default for Foo { + fn default() -> Self { + CALLED.fetch_add(1, SeqCst); + Foo(42) + } + } + + let lazy: LazyLock> = <_>::default(); + + assert_eq!(CALLED.load(SeqCst), 0); + + assert_eq!(lazy.lock().unwrap().0, 42); + assert_eq!(CALLED.load(SeqCst), 1); + + lazy.lock().unwrap().0 = 21; + + assert_eq!(lazy.lock().unwrap().0, 21); + assert_eq!(CALLED.load(SeqCst), 1); +} + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn static_sync_lazy() { + static XS: LazyLock> = LazyLock::new(|| { + let mut xs = Vec::new(); + xs.push(1); + xs.push(2); + xs.push(3); + xs + }); + + spawn_and_wait(|| { + assert_eq!(&*XS, &vec![1, 2, 3]); + }); + + assert_eq!(&*XS, &vec![1, 2, 3]); +} + +#[test] +fn static_sync_lazy_via_fn() { + fn xs() -> &'static Vec { + static XS: OnceLock> = OnceLock::new(); + XS.get_or_init(|| { + let mut xs = Vec::new(); + xs.push(1); + xs.push(2); + xs.push(3); + xs + }) + } + assert_eq!(xs(), &vec![1, 2, 3]); +} + +#[test] +fn sync_lazy_poisoning() { + let x: LazyLock = LazyLock::new(|| panic!("kaboom")); + for _ in 0..2 { + let res = panic::catch_unwind(|| x.len()); + assert!(res.is_err()); + } +} + +// Check that we can infer `T` from closure's type. +#[test] +fn lazy_type_inference() { + let _ = LazyCell::new(|| ()); +} + +#[test] +fn is_sync_send() { + fn assert_traits() {} + assert_traits::>(); +} diff --git a/crux-mir/lib/std/src/sync/mod.rs b/crux-mir/lib/std/src/sync/mod.rs index b6699910b..ba20bab87 100644 --- a/crux-mir/lib/std/src/sync/mod.rs +++ b/crux-mir/lib/std/src/sync/mod.rs @@ -19,7 +19,7 @@ //! B = 4; //! A = A + B; //! C = B; -//! println!("{} {} {}", A, B, C); +//! println!("{A} {B} {C}"); //! C = A; //! } //! } @@ -155,6 +155,8 @@ pub use alloc_crate::sync::{Arc, Weak}; #[stable(feature = "rust1", since = "1.0.0")] pub use core::sync::atomic; +#[unstable(feature = "exclusive_wrapper", issue = "98407")] +pub use core::sync::Exclusive; #[stable(feature = "rust1", since = "1.0.0")] pub use self::barrier::{Barrier, BarrierWaitResult}; @@ -166,14 +168,26 @@ pub use self::mutex::{Mutex, MutexGuard}; #[allow(deprecated)] pub use self::once::{Once, OnceState, ONCE_INIT}; #[stable(feature = "rust1", since = "1.0.0")] -pub use self::rwlock::{RwLock, RwLockReadGuard, RwLockWriteGuard}; +pub use self::poison::{LockResult, PoisonError, TryLockError, TryLockResult}; #[stable(feature = "rust1", since = "1.0.0")] -pub use crate::sys_common::poison::{LockResult, PoisonError, TryLockError, TryLockResult}; +pub use self::rwlock::{RwLock, RwLockReadGuard, RwLockWriteGuard}; + +#[unstable(feature = "once_cell", issue = "74465")] +pub use self::lazy_lock::LazyLock; +#[unstable(feature = "once_cell", issue = "74465")] +pub use self::once_lock::OnceLock; + +pub(crate) use self::remutex::{ReentrantMutex, ReentrantMutexGuard}; pub mod mpsc; mod barrier; mod condvar; +mod lazy_lock; +mod mpmc; mod mutex; mod once; +mod once_lock; +mod poison; +mod remutex; mod rwlock; diff --git a/crux-mir/lib/std/src/sync/mpmc/array.rs b/crux-mir/lib/std/src/sync/mpmc/array.rs new file mode 100644 index 000000000..c1e3e48b0 --- /dev/null +++ b/crux-mir/lib/std/src/sync/mpmc/array.rs @@ -0,0 +1,513 @@ +//! Bounded channel based on a preallocated array. +//! +//! This flavor has a fixed, positive capacity. +//! +//! The implementation is based on Dmitry Vyukov's bounded MPMC queue. +//! +//! Source: +//! - +//! - + +use super::context::Context; +use super::error::*; +use super::select::{Operation, Selected, Token}; +use super::utils::{Backoff, CachePadded}; +use super::waker::SyncWaker; + +use crate::cell::UnsafeCell; +use crate::mem::MaybeUninit; +use crate::ptr; +use crate::sync::atomic::{self, AtomicUsize, Ordering}; +use crate::time::Instant; + +/// A slot in a channel. +struct Slot { + /// The current stamp. + stamp: AtomicUsize, + + /// The message in this slot. + msg: UnsafeCell>, +} + +/// The token type for the array flavor. +#[derive(Debug)] +pub(crate) struct ArrayToken { + /// Slot to read from or write to. + slot: *const u8, + + /// Stamp to store into the slot after reading or writing. + stamp: usize, +} + +impl Default for ArrayToken { + #[inline] + fn default() -> Self { + ArrayToken { slot: ptr::null(), stamp: 0 } + } +} + +/// Bounded channel based on a preallocated array. +pub(crate) struct Channel { + /// The head of the channel. + /// + /// This value is a "stamp" consisting of an index into the buffer, a mark bit, and a lap, but + /// packed into a single `usize`. The lower bits represent the index, while the upper bits + /// represent the lap. The mark bit in the head is always zero. + /// + /// Messages are popped from the head of the channel. + head: CachePadded, + + /// The tail of the channel. + /// + /// This value is a "stamp" consisting of an index into the buffer, a mark bit, and a lap, but + /// packed into a single `usize`. The lower bits represent the index, while the upper bits + /// represent the lap. The mark bit indicates that the channel is disconnected. + /// + /// Messages are pushed into the tail of the channel. + tail: CachePadded, + + /// The buffer holding slots. + buffer: Box<[Slot]>, + + /// The channel capacity. + cap: usize, + + /// A stamp with the value of `{ lap: 1, mark: 0, index: 0 }`. + one_lap: usize, + + /// If this bit is set in the tail, that means the channel is disconnected. + mark_bit: usize, + + /// Senders waiting while the channel is full. + senders: SyncWaker, + + /// Receivers waiting while the channel is empty and not disconnected. + receivers: SyncWaker, +} + +impl Channel { + /// Creates a bounded channel of capacity `cap`. + pub(crate) fn with_capacity(cap: usize) -> Self { + assert!(cap > 0, "capacity must be positive"); + + // Compute constants `mark_bit` and `one_lap`. + let mark_bit = (cap + 1).next_power_of_two(); + let one_lap = mark_bit * 2; + + // Head is initialized to `{ lap: 0, mark: 0, index: 0 }`. + let head = 0; + // Tail is initialized to `{ lap: 0, mark: 0, index: 0 }`. + let tail = 0; + + // Allocate a buffer of `cap` slots initialized + // with stamps. + let buffer: Box<[Slot]> = (0..cap) + .map(|i| { + // Set the stamp to `{ lap: 0, mark: 0, index: i }`. + Slot { stamp: AtomicUsize::new(i), msg: UnsafeCell::new(MaybeUninit::uninit()) } + }) + .collect(); + + Channel { + buffer, + cap, + one_lap, + mark_bit, + head: CachePadded::new(AtomicUsize::new(head)), + tail: CachePadded::new(AtomicUsize::new(tail)), + senders: SyncWaker::new(), + receivers: SyncWaker::new(), + } + } + + /// Attempts to reserve a slot for sending a message. + fn start_send(&self, token: &mut Token) -> bool { + let backoff = Backoff::new(); + let mut tail = self.tail.load(Ordering::Relaxed); + + loop { + // Check if the channel is disconnected. + if tail & self.mark_bit != 0 { + token.array.slot = ptr::null(); + token.array.stamp = 0; + return true; + } + + // Deconstruct the tail. + let index = tail & (self.mark_bit - 1); + let lap = tail & !(self.one_lap - 1); + + // Inspect the corresponding slot. + debug_assert!(index < self.buffer.len()); + let slot = unsafe { self.buffer.get_unchecked(index) }; + let stamp = slot.stamp.load(Ordering::Acquire); + + // If the tail and the stamp match, we may attempt to push. + if tail == stamp { + let new_tail = if index + 1 < self.cap { + // Same lap, incremented index. + // Set to `{ lap: lap, mark: 0, index: index + 1 }`. + tail + 1 + } else { + // One lap forward, index wraps around to zero. + // Set to `{ lap: lap.wrapping_add(1), mark: 0, index: 0 }`. + lap.wrapping_add(self.one_lap) + }; + + // Try moving the tail. + match self.tail.compare_exchange_weak( + tail, + new_tail, + Ordering::SeqCst, + Ordering::Relaxed, + ) { + Ok(_) => { + // Prepare the token for the follow-up call to `write`. + token.array.slot = slot as *const Slot as *const u8; + token.array.stamp = tail + 1; + return true; + } + Err(_) => { + backoff.spin_light(); + tail = self.tail.load(Ordering::Relaxed); + } + } + } else if stamp.wrapping_add(self.one_lap) == tail + 1 { + atomic::fence(Ordering::SeqCst); + let head = self.head.load(Ordering::Relaxed); + + // If the head lags one lap behind the tail as well... + if head.wrapping_add(self.one_lap) == tail { + // ...then the channel is full. + return false; + } + + backoff.spin_light(); + tail = self.tail.load(Ordering::Relaxed); + } else { + // Snooze because we need to wait for the stamp to get updated. + backoff.spin_heavy(); + tail = self.tail.load(Ordering::Relaxed); + } + } + } + + /// Writes a message into the channel. + pub(crate) unsafe fn write(&self, token: &mut Token, msg: T) -> Result<(), T> { + // If there is no slot, the channel is disconnected. + if token.array.slot.is_null() { + return Err(msg); + } + + let slot: &Slot = &*(token.array.slot as *const Slot); + + // Write the message into the slot and update the stamp. + slot.msg.get().write(MaybeUninit::new(msg)); + slot.stamp.store(token.array.stamp, Ordering::Release); + + // Wake a sleeping receiver. + self.receivers.notify(); + Ok(()) + } + + /// Attempts to reserve a slot for receiving a message. + fn start_recv(&self, token: &mut Token) -> bool { + let backoff = Backoff::new(); + let mut head = self.head.load(Ordering::Relaxed); + + loop { + // Deconstruct the head. + let index = head & (self.mark_bit - 1); + let lap = head & !(self.one_lap - 1); + + // Inspect the corresponding slot. + debug_assert!(index < self.buffer.len()); + let slot = unsafe { self.buffer.get_unchecked(index) }; + let stamp = slot.stamp.load(Ordering::Acquire); + + // If the stamp is ahead of the head by 1, we may attempt to pop. + if head + 1 == stamp { + let new = if index + 1 < self.cap { + // Same lap, incremented index. + // Set to `{ lap: lap, mark: 0, index: index + 1 }`. + head + 1 + } else { + // One lap forward, index wraps around to zero. + // Set to `{ lap: lap.wrapping_add(1), mark: 0, index: 0 }`. + lap.wrapping_add(self.one_lap) + }; + + // Try moving the head. + match self.head.compare_exchange_weak( + head, + new, + Ordering::SeqCst, + Ordering::Relaxed, + ) { + Ok(_) => { + // Prepare the token for the follow-up call to `read`. + token.array.slot = slot as *const Slot as *const u8; + token.array.stamp = head.wrapping_add(self.one_lap); + return true; + } + Err(_) => { + backoff.spin_light(); + head = self.head.load(Ordering::Relaxed); + } + } + } else if stamp == head { + atomic::fence(Ordering::SeqCst); + let tail = self.tail.load(Ordering::Relaxed); + + // If the tail equals the head, that means the channel is empty. + if (tail & !self.mark_bit) == head { + // If the channel is disconnected... + if tail & self.mark_bit != 0 { + // ...then receive an error. + token.array.slot = ptr::null(); + token.array.stamp = 0; + return true; + } else { + // Otherwise, the receive operation is not ready. + return false; + } + } + + backoff.spin_light(); + head = self.head.load(Ordering::Relaxed); + } else { + // Snooze because we need to wait for the stamp to get updated. + backoff.spin_heavy(); + head = self.head.load(Ordering::Relaxed); + } + } + } + + /// Reads a message from the channel. + pub(crate) unsafe fn read(&self, token: &mut Token) -> Result { + if token.array.slot.is_null() { + // The channel is disconnected. + return Err(()); + } + + let slot: &Slot = &*(token.array.slot as *const Slot); + + // Read the message from the slot and update the stamp. + let msg = slot.msg.get().read().assume_init(); + slot.stamp.store(token.array.stamp, Ordering::Release); + + // Wake a sleeping sender. + self.senders.notify(); + Ok(msg) + } + + /// Attempts to send a message into the channel. + pub(crate) fn try_send(&self, msg: T) -> Result<(), TrySendError> { + let token = &mut Token::default(); + if self.start_send(token) { + unsafe { self.write(token, msg).map_err(TrySendError::Disconnected) } + } else { + Err(TrySendError::Full(msg)) + } + } + + /// Sends a message into the channel. + pub(crate) fn send( + &self, + msg: T, + deadline: Option, + ) -> Result<(), SendTimeoutError> { + let token = &mut Token::default(); + loop { + // Try sending a message several times. + let backoff = Backoff::new(); + loop { + if self.start_send(token) { + let res = unsafe { self.write(token, msg) }; + return res.map_err(SendTimeoutError::Disconnected); + } + + if backoff.is_completed() { + break; + } else { + backoff.spin_light(); + } + } + + if let Some(d) = deadline { + if Instant::now() >= d { + return Err(SendTimeoutError::Timeout(msg)); + } + } + + Context::with(|cx| { + // Prepare for blocking until a receiver wakes us up. + let oper = Operation::hook(token); + self.senders.register(oper, cx); + + // Has the channel become ready just now? + if !self.is_full() || self.is_disconnected() { + let _ = cx.try_select(Selected::Aborted); + } + + // Block the current thread. + let sel = cx.wait_until(deadline); + + match sel { + Selected::Waiting => unreachable!(), + Selected::Aborted | Selected::Disconnected => { + self.senders.unregister(oper).unwrap(); + } + Selected::Operation(_) => {} + } + }); + } + } + + /// Attempts to receive a message without blocking. + pub(crate) fn try_recv(&self) -> Result { + let token = &mut Token::default(); + + if self.start_recv(token) { + unsafe { self.read(token).map_err(|_| TryRecvError::Disconnected) } + } else { + Err(TryRecvError::Empty) + } + } + + /// Receives a message from the channel. + pub(crate) fn recv(&self, deadline: Option) -> Result { + let token = &mut Token::default(); + loop { + if self.start_recv(token) { + let res = unsafe { self.read(token) }; + return res.map_err(|_| RecvTimeoutError::Disconnected); + } + + if let Some(d) = deadline { + if Instant::now() >= d { + return Err(RecvTimeoutError::Timeout); + } + } + + Context::with(|cx| { + // Prepare for blocking until a sender wakes us up. + let oper = Operation::hook(token); + self.receivers.register(oper, cx); + + // Has the channel become ready just now? + if !self.is_empty() || self.is_disconnected() { + let _ = cx.try_select(Selected::Aborted); + } + + // Block the current thread. + let sel = cx.wait_until(deadline); + + match sel { + Selected::Waiting => unreachable!(), + Selected::Aborted | Selected::Disconnected => { + self.receivers.unregister(oper).unwrap(); + // If the channel was disconnected, we still have to check for remaining + // messages. + } + Selected::Operation(_) => {} + } + }); + } + } + + /// Returns the current number of messages inside the channel. + pub(crate) fn len(&self) -> usize { + loop { + // Load the tail, then load the head. + let tail = self.tail.load(Ordering::SeqCst); + let head = self.head.load(Ordering::SeqCst); + + // If the tail didn't change, we've got consistent values to work with. + if self.tail.load(Ordering::SeqCst) == tail { + let hix = head & (self.mark_bit - 1); + let tix = tail & (self.mark_bit - 1); + + return if hix < tix { + tix - hix + } else if hix > tix { + self.cap - hix + tix + } else if (tail & !self.mark_bit) == head { + 0 + } else { + self.cap + }; + } + } + } + + /// Returns the capacity of the channel. + #[allow(clippy::unnecessary_wraps)] // This is intentional. + pub(crate) fn capacity(&self) -> Option { + Some(self.cap) + } + + /// Disconnects the channel and wakes up all blocked senders and receivers. + /// + /// Returns `true` if this call disconnected the channel. + pub(crate) fn disconnect(&self) -> bool { + let tail = self.tail.fetch_or(self.mark_bit, Ordering::SeqCst); + + if tail & self.mark_bit == 0 { + self.senders.disconnect(); + self.receivers.disconnect(); + true + } else { + false + } + } + + /// Returns `true` if the channel is disconnected. + pub(crate) fn is_disconnected(&self) -> bool { + self.tail.load(Ordering::SeqCst) & self.mark_bit != 0 + } + + /// Returns `true` if the channel is empty. + pub(crate) fn is_empty(&self) -> bool { + let head = self.head.load(Ordering::SeqCst); + let tail = self.tail.load(Ordering::SeqCst); + + // Is the tail equal to the head? + // + // Note: If the head changes just before we load the tail, that means there was a moment + // when the channel was not empty, so it is safe to just return `false`. + (tail & !self.mark_bit) == head + } + + /// Returns `true` if the channel is full. + pub(crate) fn is_full(&self) -> bool { + let tail = self.tail.load(Ordering::SeqCst); + let head = self.head.load(Ordering::SeqCst); + + // Is the head lagging one lap behind tail? + // + // Note: If the tail changes just before we load the head, that means there was a moment + // when the channel was not full, so it is safe to just return `false`. + head.wrapping_add(self.one_lap) == tail & !self.mark_bit + } +} + +impl Drop for Channel { + fn drop(&mut self) { + // Get the index of the head. + let hix = self.head.load(Ordering::Relaxed) & (self.mark_bit - 1); + + // Loop over all slots that hold a message and drop them. + for i in 0..self.len() { + // Compute the index of the next slot holding a message. + let index = if hix + i < self.cap { hix + i } else { hix + i - self.cap }; + + unsafe { + debug_assert!(index < self.buffer.len()); + let slot = self.buffer.get_unchecked_mut(index); + let msg = &mut *slot.msg.get(); + msg.as_mut_ptr().drop_in_place(); + } + } + } +} diff --git a/crux-mir/lib/std/src/sync/mpmc/context.rs b/crux-mir/lib/std/src/sync/mpmc/context.rs new file mode 100644 index 000000000..bbfc6ce00 --- /dev/null +++ b/crux-mir/lib/std/src/sync/mpmc/context.rs @@ -0,0 +1,155 @@ +//! Thread-local channel context. + +use super::select::Selected; +use super::waker::current_thread_id; + +use crate::cell::Cell; +use crate::ptr; +use crate::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; +use crate::sync::Arc; +use crate::thread::{self, Thread}; +use crate::time::Instant; + +/// Thread-local context. +#[derive(Debug, Clone)] +pub struct Context { + inner: Arc, +} + +/// Inner representation of `Context`. +#[derive(Debug)] +struct Inner { + /// Selected operation. + select: AtomicUsize, + + /// A slot into which another thread may store a pointer to its `Packet`. + packet: AtomicPtr<()>, + + /// Thread handle. + thread: Thread, + + /// Thread id. + thread_id: usize, +} + +impl Context { + /// Creates a new context for the duration of the closure. + #[inline] + pub fn with(f: F) -> R + where + F: FnOnce(&Context) -> R, + { + thread_local! { + /// Cached thread-local context. + static CONTEXT: Cell> = Cell::new(Some(Context::new())); + } + + let mut f = Some(f); + let mut f = |cx: &Context| -> R { + let f = f.take().unwrap(); + f(cx) + }; + + CONTEXT + .try_with(|cell| match cell.take() { + None => f(&Context::new()), + Some(cx) => { + cx.reset(); + let res = f(&cx); + cell.set(Some(cx)); + res + } + }) + .unwrap_or_else(|_| f(&Context::new())) + } + + /// Creates a new `Context`. + #[cold] + fn new() -> Context { + Context { + inner: Arc::new(Inner { + select: AtomicUsize::new(Selected::Waiting.into()), + packet: AtomicPtr::new(ptr::null_mut()), + thread: thread::current(), + thread_id: current_thread_id(), + }), + } + } + + /// Resets `select` and `packet`. + #[inline] + fn reset(&self) { + self.inner.select.store(Selected::Waiting.into(), Ordering::Release); + self.inner.packet.store(ptr::null_mut(), Ordering::Release); + } + + /// Attempts to select an operation. + /// + /// On failure, the previously selected operation is returned. + #[inline] + pub fn try_select(&self, select: Selected) -> Result<(), Selected> { + self.inner + .select + .compare_exchange( + Selected::Waiting.into(), + select.into(), + Ordering::AcqRel, + Ordering::Acquire, + ) + .map(|_| ()) + .map_err(|e| e.into()) + } + + /// Stores a packet. + /// + /// This method must be called after `try_select` succeeds and there is a packet to provide. + #[inline] + pub fn store_packet(&self, packet: *mut ()) { + if !packet.is_null() { + self.inner.packet.store(packet, Ordering::Release); + } + } + + /// Waits until an operation is selected and returns it. + /// + /// If the deadline is reached, `Selected::Aborted` will be selected. + #[inline] + pub fn wait_until(&self, deadline: Option) -> Selected { + loop { + // Check whether an operation has been selected. + let sel = Selected::from(self.inner.select.load(Ordering::Acquire)); + if sel != Selected::Waiting { + return sel; + } + + // If there's a deadline, park the current thread until the deadline is reached. + if let Some(end) = deadline { + let now = Instant::now(); + + if now < end { + thread::park_timeout(end - now); + } else { + // The deadline has been reached. Try aborting select. + return match self.try_select(Selected::Aborted) { + Ok(()) => Selected::Aborted, + Err(s) => s, + }; + } + } else { + thread::park(); + } + } + } + + /// Unparks the thread this context belongs to. + #[inline] + pub fn unpark(&self) { + self.inner.thread.unpark(); + } + + /// Returns the id of the thread this context belongs to. + #[inline] + pub fn thread_id(&self) -> usize { + self.inner.thread_id + } +} diff --git a/crux-mir/lib/std/src/sync/mpmc/counter.rs b/crux-mir/lib/std/src/sync/mpmc/counter.rs new file mode 100644 index 000000000..a5a6bdc67 --- /dev/null +++ b/crux-mir/lib/std/src/sync/mpmc/counter.rs @@ -0,0 +1,137 @@ +use crate::ops; +use crate::process; +use crate::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +/// Reference counter internals. +struct Counter { + /// The number of senders associated with the channel. + senders: AtomicUsize, + + /// The number of receivers associated with the channel. + receivers: AtomicUsize, + + /// Set to `true` if the last sender or the last receiver reference deallocates the channel. + destroy: AtomicBool, + + /// The internal channel. + chan: C, +} + +/// Wraps a channel into the reference counter. +pub(crate) fn new(chan: C) -> (Sender, Receiver) { + let counter = Box::into_raw(Box::new(Counter { + senders: AtomicUsize::new(1), + receivers: AtomicUsize::new(1), + destroy: AtomicBool::new(false), + chan, + })); + let s = Sender { counter }; + let r = Receiver { counter }; + (s, r) +} + +/// The sending side. +pub(crate) struct Sender { + counter: *mut Counter, +} + +impl Sender { + /// Returns the internal `Counter`. + fn counter(&self) -> &Counter { + unsafe { &*self.counter } + } + + /// Acquires another sender reference. + pub(crate) fn acquire(&self) -> Sender { + let count = self.counter().senders.fetch_add(1, Ordering::Relaxed); + + // Cloning senders and calling `mem::forget` on the clones could potentially overflow the + // counter. It's very difficult to recover sensibly from such degenerate scenarios so we + // just abort when the count becomes very large. + if count > isize::MAX as usize { + process::abort(); + } + + Sender { counter: self.counter } + } + + /// Releases the sender reference. + /// + /// Function `disconnect` will be called if this is the last sender reference. + pub(crate) unsafe fn release bool>(&self, disconnect: F) { + if self.counter().senders.fetch_sub(1, Ordering::AcqRel) == 1 { + disconnect(&self.counter().chan); + + if self.counter().destroy.swap(true, Ordering::AcqRel) { + drop(Box::from_raw(self.counter)); + } + } + } +} + +impl ops::Deref for Sender { + type Target = C; + + fn deref(&self) -> &C { + &self.counter().chan + } +} + +impl PartialEq for Sender { + fn eq(&self, other: &Sender) -> bool { + self.counter == other.counter + } +} + +/// The receiving side. +pub(crate) struct Receiver { + counter: *mut Counter, +} + +impl Receiver { + /// Returns the internal `Counter`. + fn counter(&self) -> &Counter { + unsafe { &*self.counter } + } + + /// Acquires another receiver reference. + pub(crate) fn acquire(&self) -> Receiver { + let count = self.counter().receivers.fetch_add(1, Ordering::Relaxed); + + // Cloning receivers and calling `mem::forget` on the clones could potentially overflow the + // counter. It's very difficult to recover sensibly from such degenerate scenarios so we + // just abort when the count becomes very large. + if count > isize::MAX as usize { + process::abort(); + } + + Receiver { counter: self.counter } + } + + /// Releases the receiver reference. + /// + /// Function `disconnect` will be called if this is the last receiver reference. + pub(crate) unsafe fn release bool>(&self, disconnect: F) { + if self.counter().receivers.fetch_sub(1, Ordering::AcqRel) == 1 { + disconnect(&self.counter().chan); + + if self.counter().destroy.swap(true, Ordering::AcqRel) { + drop(Box::from_raw(self.counter)); + } + } + } +} + +impl ops::Deref for Receiver { + type Target = C; + + fn deref(&self) -> &C { + &self.counter().chan + } +} + +impl PartialEq for Receiver { + fn eq(&self, other: &Receiver) -> bool { + self.counter == other.counter + } +} diff --git a/crux-mir/lib/std/src/sync/mpmc/error.rs b/crux-mir/lib/std/src/sync/mpmc/error.rs new file mode 100644 index 000000000..1b8a1f387 --- /dev/null +++ b/crux-mir/lib/std/src/sync/mpmc/error.rs @@ -0,0 +1,46 @@ +use crate::error; +use crate::fmt; + +pub use crate::sync::mpsc::{RecvError, RecvTimeoutError, SendError, TryRecvError, TrySendError}; + +/// An error returned from the [`send_timeout`] method. +/// +/// The error contains the message being sent so it can be recovered. +/// +/// [`send_timeout`]: super::Sender::send_timeout +#[derive(PartialEq, Eq, Clone, Copy)] +pub enum SendTimeoutError { + /// The message could not be sent because the channel is full and the operation timed out. + /// + /// If this is a zero-capacity channel, then the error indicates that there was no receiver + /// available to receive the message and the operation timed out. + Timeout(T), + + /// The message could not be sent because the channel is disconnected. + Disconnected(T), +} + +impl fmt::Debug for SendTimeoutError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + "SendTimeoutError(..)".fmt(f) + } +} + +impl fmt::Display for SendTimeoutError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + SendTimeoutError::Timeout(..) => "timed out waiting on send operation".fmt(f), + SendTimeoutError::Disconnected(..) => "sending on a disconnected channel".fmt(f), + } + } +} + +impl error::Error for SendTimeoutError {} + +impl From> for SendTimeoutError { + fn from(err: SendError) -> SendTimeoutError { + match err { + SendError(e) => SendTimeoutError::Disconnected(e), + } + } +} diff --git a/crux-mir/lib/std/src/sync/mpmc/list.rs b/crux-mir/lib/std/src/sync/mpmc/list.rs new file mode 100644 index 000000000..ec6c0726a --- /dev/null +++ b/crux-mir/lib/std/src/sync/mpmc/list.rs @@ -0,0 +1,638 @@ +//! Unbounded channel implemented as a linked list. + +use super::context::Context; +use super::error::*; +use super::select::{Operation, Selected, Token}; +use super::utils::{Backoff, CachePadded}; +use super::waker::SyncWaker; + +use crate::cell::UnsafeCell; +use crate::marker::PhantomData; +use crate::mem::MaybeUninit; +use crate::ptr; +use crate::sync::atomic::{self, AtomicPtr, AtomicUsize, Ordering}; +use crate::time::Instant; + +// Bits indicating the state of a slot: +// * If a message has been written into the slot, `WRITE` is set. +// * If a message has been read from the slot, `READ` is set. +// * If the block is being destroyed, `DESTROY` is set. +const WRITE: usize = 1; +const READ: usize = 2; +const DESTROY: usize = 4; + +// Each block covers one "lap" of indices. +const LAP: usize = 32; +// The maximum number of messages a block can hold. +const BLOCK_CAP: usize = LAP - 1; +// How many lower bits are reserved for metadata. +const SHIFT: usize = 1; +// Has two different purposes: +// * If set in head, indicates that the block is not the last one. +// * If set in tail, indicates that the channel is disconnected. +const MARK_BIT: usize = 1; + +/// A slot in a block. +struct Slot { + /// The message. + msg: UnsafeCell>, + + /// The state of the slot. + state: AtomicUsize, +} + +impl Slot { + /// Waits until a message is written into the slot. + fn wait_write(&self) { + let backoff = Backoff::new(); + while self.state.load(Ordering::Acquire) & WRITE == 0 { + backoff.spin_heavy(); + } + } +} + +/// A block in a linked list. +/// +/// Each block in the list can hold up to `BLOCK_CAP` messages. +struct Block { + /// The next block in the linked list. + next: AtomicPtr>, + + /// Slots for messages. + slots: [Slot; BLOCK_CAP], +} + +impl Block { + /// Creates an empty block. + fn new() -> Block { + // SAFETY: This is safe because: + // [1] `Block::next` (AtomicPtr) may be safely zero initialized. + // [2] `Block::slots` (Array) may be safely zero initialized because of [3, 4]. + // [3] `Slot::msg` (UnsafeCell) may be safely zero initialized because it + // holds a MaybeUninit. + // [4] `Slot::state` (AtomicUsize) may be safely zero initialized. + unsafe { MaybeUninit::zeroed().assume_init() } + } + + /// Waits until the next pointer is set. + fn wait_next(&self) -> *mut Block { + let backoff = Backoff::new(); + loop { + let next = self.next.load(Ordering::Acquire); + if !next.is_null() { + return next; + } + backoff.spin_heavy(); + } + } + + /// Sets the `DESTROY` bit in slots starting from `start` and destroys the block. + unsafe fn destroy(this: *mut Block, start: usize) { + // It is not necessary to set the `DESTROY` bit in the last slot because that slot has + // begun destruction of the block. + for i in start..BLOCK_CAP - 1 { + let slot = (*this).slots.get_unchecked(i); + + // Mark the `DESTROY` bit if a thread is still using the slot. + if slot.state.load(Ordering::Acquire) & READ == 0 + && slot.state.fetch_or(DESTROY, Ordering::AcqRel) & READ == 0 + { + // If a thread is still using the slot, it will continue destruction of the block. + return; + } + } + + // No thread is using the block, now it is safe to destroy it. + drop(Box::from_raw(this)); + } +} + +/// A position in a channel. +#[derive(Debug)] +struct Position { + /// The index in the channel. + index: AtomicUsize, + + /// The block in the linked list. + block: AtomicPtr>, +} + +/// The token type for the list flavor. +#[derive(Debug)] +pub(crate) struct ListToken { + /// The block of slots. + block: *const u8, + + /// The offset into the block. + offset: usize, +} + +impl Default for ListToken { + #[inline] + fn default() -> Self { + ListToken { block: ptr::null(), offset: 0 } + } +} + +/// Unbounded channel implemented as a linked list. +/// +/// Each message sent into the channel is assigned a sequence number, i.e. an index. Indices are +/// represented as numbers of type `usize` and wrap on overflow. +/// +/// Consecutive messages are grouped into blocks in order to put less pressure on the allocator and +/// improve cache efficiency. +pub(crate) struct Channel { + /// The head of the channel. + head: CachePadded>, + + /// The tail of the channel. + tail: CachePadded>, + + /// Receivers waiting while the channel is empty and not disconnected. + receivers: SyncWaker, + + /// Indicates that dropping a `Channel` may drop messages of type `T`. + _marker: PhantomData, +} + +impl Channel { + /// Creates a new unbounded channel. + pub(crate) fn new() -> Self { + Channel { + head: CachePadded::new(Position { + block: AtomicPtr::new(ptr::null_mut()), + index: AtomicUsize::new(0), + }), + tail: CachePadded::new(Position { + block: AtomicPtr::new(ptr::null_mut()), + index: AtomicUsize::new(0), + }), + receivers: SyncWaker::new(), + _marker: PhantomData, + } + } + + /// Attempts to reserve a slot for sending a message. + fn start_send(&self, token: &mut Token) -> bool { + let backoff = Backoff::new(); + let mut tail = self.tail.index.load(Ordering::Acquire); + let mut block = self.tail.block.load(Ordering::Acquire); + let mut next_block = None; + + loop { + // Check if the channel is disconnected. + if tail & MARK_BIT != 0 { + token.list.block = ptr::null(); + return true; + } + + // Calculate the offset of the index into the block. + let offset = (tail >> SHIFT) % LAP; + + // If we reached the end of the block, wait until the next one is installed. + if offset == BLOCK_CAP { + backoff.spin_heavy(); + tail = self.tail.index.load(Ordering::Acquire); + block = self.tail.block.load(Ordering::Acquire); + continue; + } + + // If we're going to have to install the next block, allocate it in advance in order to + // make the wait for other threads as short as possible. + if offset + 1 == BLOCK_CAP && next_block.is_none() { + next_block = Some(Box::new(Block::::new())); + } + + // If this is the first message to be sent into the channel, we need to allocate the + // first block and install it. + if block.is_null() { + let new = Box::into_raw(Box::new(Block::::new())); + + if self + .tail + .block + .compare_exchange(block, new, Ordering::Release, Ordering::Relaxed) + .is_ok() + { + self.head.block.store(new, Ordering::Release); + block = new; + } else { + next_block = unsafe { Some(Box::from_raw(new)) }; + tail = self.tail.index.load(Ordering::Acquire); + block = self.tail.block.load(Ordering::Acquire); + continue; + } + } + + let new_tail = tail + (1 << SHIFT); + + // Try advancing the tail forward. + match self.tail.index.compare_exchange_weak( + tail, + new_tail, + Ordering::SeqCst, + Ordering::Acquire, + ) { + Ok(_) => unsafe { + // If we've reached the end of the block, install the next one. + if offset + 1 == BLOCK_CAP { + let next_block = Box::into_raw(next_block.unwrap()); + self.tail.block.store(next_block, Ordering::Release); + self.tail.index.fetch_add(1 << SHIFT, Ordering::Release); + (*block).next.store(next_block, Ordering::Release); + } + + token.list.block = block as *const u8; + token.list.offset = offset; + return true; + }, + Err(_) => { + backoff.spin_light(); + tail = self.tail.index.load(Ordering::Acquire); + block = self.tail.block.load(Ordering::Acquire); + } + } + } + } + + /// Writes a message into the channel. + pub(crate) unsafe fn write(&self, token: &mut Token, msg: T) -> Result<(), T> { + // If there is no slot, the channel is disconnected. + if token.list.block.is_null() { + return Err(msg); + } + + // Write the message into the slot. + let block = token.list.block as *mut Block; + let offset = token.list.offset; + let slot = (*block).slots.get_unchecked(offset); + slot.msg.get().write(MaybeUninit::new(msg)); + slot.state.fetch_or(WRITE, Ordering::Release); + + // Wake a sleeping receiver. + self.receivers.notify(); + Ok(()) + } + + /// Attempts to reserve a slot for receiving a message. + fn start_recv(&self, token: &mut Token) -> bool { + let backoff = Backoff::new(); + let mut head = self.head.index.load(Ordering::Acquire); + let mut block = self.head.block.load(Ordering::Acquire); + + loop { + // Calculate the offset of the index into the block. + let offset = (head >> SHIFT) % LAP; + + // If we reached the end of the block, wait until the next one is installed. + if offset == BLOCK_CAP { + backoff.spin_heavy(); + head = self.head.index.load(Ordering::Acquire); + block = self.head.block.load(Ordering::Acquire); + continue; + } + + let mut new_head = head + (1 << SHIFT); + + if new_head & MARK_BIT == 0 { + atomic::fence(Ordering::SeqCst); + let tail = self.tail.index.load(Ordering::Relaxed); + + // If the tail equals the head, that means the channel is empty. + if head >> SHIFT == tail >> SHIFT { + // If the channel is disconnected... + if tail & MARK_BIT != 0 { + // ...then receive an error. + token.list.block = ptr::null(); + return true; + } else { + // Otherwise, the receive operation is not ready. + return false; + } + } + + // If head and tail are not in the same block, set `MARK_BIT` in head. + if (head >> SHIFT) / LAP != (tail >> SHIFT) / LAP { + new_head |= MARK_BIT; + } + } + + // The block can be null here only if the first message is being sent into the channel. + // In that case, just wait until it gets initialized. + if block.is_null() { + backoff.spin_heavy(); + head = self.head.index.load(Ordering::Acquire); + block = self.head.block.load(Ordering::Acquire); + continue; + } + + // Try moving the head index forward. + match self.head.index.compare_exchange_weak( + head, + new_head, + Ordering::SeqCst, + Ordering::Acquire, + ) { + Ok(_) => unsafe { + // If we've reached the end of the block, move to the next one. + if offset + 1 == BLOCK_CAP { + let next = (*block).wait_next(); + let mut next_index = (new_head & !MARK_BIT).wrapping_add(1 << SHIFT); + if !(*next).next.load(Ordering::Relaxed).is_null() { + next_index |= MARK_BIT; + } + + self.head.block.store(next, Ordering::Release); + self.head.index.store(next_index, Ordering::Release); + } + + token.list.block = block as *const u8; + token.list.offset = offset; + return true; + }, + Err(_) => { + backoff.spin_light(); + head = self.head.index.load(Ordering::Acquire); + block = self.head.block.load(Ordering::Acquire); + } + } + } + } + + /// Reads a message from the channel. + pub(crate) unsafe fn read(&self, token: &mut Token) -> Result { + if token.list.block.is_null() { + // The channel is disconnected. + return Err(()); + } + + // Read the message. + let block = token.list.block as *mut Block; + let offset = token.list.offset; + let slot = (*block).slots.get_unchecked(offset); + slot.wait_write(); + let msg = slot.msg.get().read().assume_init(); + + // Destroy the block if we've reached the end, or if another thread wanted to destroy but + // couldn't because we were busy reading from the slot. + if offset + 1 == BLOCK_CAP { + Block::destroy(block, 0); + } else if slot.state.fetch_or(READ, Ordering::AcqRel) & DESTROY != 0 { + Block::destroy(block, offset + 1); + } + + Ok(msg) + } + + /// Attempts to send a message into the channel. + pub(crate) fn try_send(&self, msg: T) -> Result<(), TrySendError> { + self.send(msg, None).map_err(|err| match err { + SendTimeoutError::Disconnected(msg) => TrySendError::Disconnected(msg), + SendTimeoutError::Timeout(_) => unreachable!(), + }) + } + + /// Sends a message into the channel. + pub(crate) fn send( + &self, + msg: T, + _deadline: Option, + ) -> Result<(), SendTimeoutError> { + let token = &mut Token::default(); + assert!(self.start_send(token)); + unsafe { self.write(token, msg).map_err(SendTimeoutError::Disconnected) } + } + + /// Attempts to receive a message without blocking. + pub(crate) fn try_recv(&self) -> Result { + let token = &mut Token::default(); + + if self.start_recv(token) { + unsafe { self.read(token).map_err(|_| TryRecvError::Disconnected) } + } else { + Err(TryRecvError::Empty) + } + } + + /// Receives a message from the channel. + pub(crate) fn recv(&self, deadline: Option) -> Result { + let token = &mut Token::default(); + loop { + if self.start_recv(token) { + unsafe { + return self.read(token).map_err(|_| RecvTimeoutError::Disconnected); + } + } + + if let Some(d) = deadline { + if Instant::now() >= d { + return Err(RecvTimeoutError::Timeout); + } + } + + // Prepare for blocking until a sender wakes us up. + Context::with(|cx| { + let oper = Operation::hook(token); + self.receivers.register(oper, cx); + + // Has the channel become ready just now? + if !self.is_empty() || self.is_disconnected() { + let _ = cx.try_select(Selected::Aborted); + } + + // Block the current thread. + let sel = cx.wait_until(deadline); + + match sel { + Selected::Waiting => unreachable!(), + Selected::Aborted | Selected::Disconnected => { + self.receivers.unregister(oper).unwrap(); + // If the channel was disconnected, we still have to check for remaining + // messages. + } + Selected::Operation(_) => {} + } + }); + } + } + + /// Returns the current number of messages inside the channel. + pub(crate) fn len(&self) -> usize { + loop { + // Load the tail index, then load the head index. + let mut tail = self.tail.index.load(Ordering::SeqCst); + let mut head = self.head.index.load(Ordering::SeqCst); + + // If the tail index didn't change, we've got consistent indices to work with. + if self.tail.index.load(Ordering::SeqCst) == tail { + // Erase the lower bits. + tail &= !((1 << SHIFT) - 1); + head &= !((1 << SHIFT) - 1); + + // Fix up indices if they fall onto block ends. + if (tail >> SHIFT) & (LAP - 1) == LAP - 1 { + tail = tail.wrapping_add(1 << SHIFT); + } + if (head >> SHIFT) & (LAP - 1) == LAP - 1 { + head = head.wrapping_add(1 << SHIFT); + } + + // Rotate indices so that head falls into the first block. + let lap = (head >> SHIFT) / LAP; + tail = tail.wrapping_sub((lap * LAP) << SHIFT); + head = head.wrapping_sub((lap * LAP) << SHIFT); + + // Remove the lower bits. + tail >>= SHIFT; + head >>= SHIFT; + + // Return the difference minus the number of blocks between tail and head. + return tail - head - tail / LAP; + } + } + } + + /// Returns the capacity of the channel. + pub(crate) fn capacity(&self) -> Option { + None + } + + /// Disconnects senders and wakes up all blocked receivers. + /// + /// Returns `true` if this call disconnected the channel. + pub(crate) fn disconnect_senders(&self) -> bool { + let tail = self.tail.index.fetch_or(MARK_BIT, Ordering::SeqCst); + + if tail & MARK_BIT == 0 { + self.receivers.disconnect(); + true + } else { + false + } + } + + /// Disconnects receivers. + /// + /// Returns `true` if this call disconnected the channel. + pub(crate) fn disconnect_receivers(&self) -> bool { + let tail = self.tail.index.fetch_or(MARK_BIT, Ordering::SeqCst); + + if tail & MARK_BIT == 0 { + // If receivers are dropped first, discard all messages to free + // memory eagerly. + self.discard_all_messages(); + true + } else { + false + } + } + + /// Discards all messages. + /// + /// This method should only be called when all receivers are dropped. + fn discard_all_messages(&self) { + let backoff = Backoff::new(); + let mut tail = self.tail.index.load(Ordering::Acquire); + loop { + let offset = (tail >> SHIFT) % LAP; + if offset != BLOCK_CAP { + break; + } + + // New updates to tail will be rejected by MARK_BIT and aborted unless it's + // at boundary. We need to wait for the updates take affect otherwise there + // can be memory leaks. + backoff.spin_heavy(); + tail = self.tail.index.load(Ordering::Acquire); + } + + let mut head = self.head.index.load(Ordering::Acquire); + let mut block = self.head.block.load(Ordering::Acquire); + + unsafe { + // Drop all messages between head and tail and deallocate the heap-allocated blocks. + while head >> SHIFT != tail >> SHIFT { + let offset = (head >> SHIFT) % LAP; + + if offset < BLOCK_CAP { + // Drop the message in the slot. + let slot = (*block).slots.get_unchecked(offset); + slot.wait_write(); + let p = &mut *slot.msg.get(); + p.as_mut_ptr().drop_in_place(); + } else { + (*block).wait_next(); + // Deallocate the block and move to the next one. + let next = (*block).next.load(Ordering::Acquire); + drop(Box::from_raw(block)); + block = next; + } + + head = head.wrapping_add(1 << SHIFT); + } + + // Deallocate the last remaining block. + if !block.is_null() { + drop(Box::from_raw(block)); + } + } + head &= !MARK_BIT; + self.head.block.store(ptr::null_mut(), Ordering::Release); + self.head.index.store(head, Ordering::Release); + } + + /// Returns `true` if the channel is disconnected. + pub(crate) fn is_disconnected(&self) -> bool { + self.tail.index.load(Ordering::SeqCst) & MARK_BIT != 0 + } + + /// Returns `true` if the channel is empty. + pub(crate) fn is_empty(&self) -> bool { + let head = self.head.index.load(Ordering::SeqCst); + let tail = self.tail.index.load(Ordering::SeqCst); + head >> SHIFT == tail >> SHIFT + } + + /// Returns `true` if the channel is full. + pub(crate) fn is_full(&self) -> bool { + false + } +} + +impl Drop for Channel { + fn drop(&mut self) { + let mut head = self.head.index.load(Ordering::Relaxed); + let mut tail = self.tail.index.load(Ordering::Relaxed); + let mut block = self.head.block.load(Ordering::Relaxed); + + // Erase the lower bits. + head &= !((1 << SHIFT) - 1); + tail &= !((1 << SHIFT) - 1); + + unsafe { + // Drop all messages between head and tail and deallocate the heap-allocated blocks. + while head != tail { + let offset = (head >> SHIFT) % LAP; + + if offset < BLOCK_CAP { + // Drop the message in the slot. + let slot = (*block).slots.get_unchecked(offset); + let p = &mut *slot.msg.get(); + p.as_mut_ptr().drop_in_place(); + } else { + // Deallocate the block and move to the next one. + let next = (*block).next.load(Ordering::Relaxed); + drop(Box::from_raw(block)); + block = next; + } + + head = head.wrapping_add(1 << SHIFT); + } + + // Deallocate the last remaining block. + if !block.is_null() { + drop(Box::from_raw(block)); + } + } + } +} diff --git a/crux-mir/lib/std/src/sync/mpmc/mod.rs b/crux-mir/lib/std/src/sync/mpmc/mod.rs new file mode 100644 index 000000000..7a602cecd --- /dev/null +++ b/crux-mir/lib/std/src/sync/mpmc/mod.rs @@ -0,0 +1,430 @@ +//! Multi-producer multi-consumer channels. + +// This module is not currently exposed publicly, but is used +// as the implementation for the channels in `sync::mpsc`. The +// implementation comes from the crossbeam-channel crate: +// +// Copyright (c) 2019 The Crossbeam Project Developers +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +mod array; +mod context; +mod counter; +mod error; +mod list; +mod select; +mod utils; +mod waker; +mod zero; + +use crate::fmt; +use crate::panic::{RefUnwindSafe, UnwindSafe}; +use crate::time::{Duration, Instant}; +pub use error::*; + +/// Creates a channel of unbounded capacity. +/// +/// This channel has a growable buffer that can hold any number of messages at a time. +pub fn channel() -> (Sender, Receiver) { + let (s, r) = counter::new(list::Channel::new()); + let s = Sender { flavor: SenderFlavor::List(s) }; + let r = Receiver { flavor: ReceiverFlavor::List(r) }; + (s, r) +} + +/// Creates a channel of bounded capacity. +/// +/// This channel has a buffer that can hold at most `cap` messages at a time. +/// +/// A special case is zero-capacity channel, which cannot hold any messages. Instead, send and +/// receive operations must appear at the same time in order to pair up and pass the message over. +pub fn sync_channel(cap: usize) -> (Sender, Receiver) { + if cap == 0 { + let (s, r) = counter::new(zero::Channel::new()); + let s = Sender { flavor: SenderFlavor::Zero(s) }; + let r = Receiver { flavor: ReceiverFlavor::Zero(r) }; + (s, r) + } else { + let (s, r) = counter::new(array::Channel::with_capacity(cap)); + let s = Sender { flavor: SenderFlavor::Array(s) }; + let r = Receiver { flavor: ReceiverFlavor::Array(r) }; + (s, r) + } +} + +/// The sending side of a channel. +pub struct Sender { + flavor: SenderFlavor, +} + +/// Sender flavors. +enum SenderFlavor { + /// Bounded channel based on a preallocated array. + Array(counter::Sender>), + + /// Unbounded channel implemented as a linked list. + List(counter::Sender>), + + /// Zero-capacity channel. + Zero(counter::Sender>), +} + +unsafe impl Send for Sender {} +unsafe impl Sync for Sender {} + +impl UnwindSafe for Sender {} +impl RefUnwindSafe for Sender {} + +impl Sender { + /// Attempts to send a message into the channel without blocking. + /// + /// This method will either send a message into the channel immediately or return an error if + /// the channel is full or disconnected. The returned error contains the original message. + /// + /// If called on a zero-capacity channel, this method will send the message only if there + /// happens to be a receive operation on the other side of the channel at the same time. + pub fn try_send(&self, msg: T) -> Result<(), TrySendError> { + match &self.flavor { + SenderFlavor::Array(chan) => chan.try_send(msg), + SenderFlavor::List(chan) => chan.try_send(msg), + SenderFlavor::Zero(chan) => chan.try_send(msg), + } + } + + /// Blocks the current thread until a message is sent or the channel is disconnected. + /// + /// If the channel is full and not disconnected, this call will block until the send operation + /// can proceed. If the channel becomes disconnected, this call will wake up and return an + /// error. The returned error contains the original message. + /// + /// If called on a zero-capacity channel, this method will wait for a receive operation to + /// appear on the other side of the channel. + pub fn send(&self, msg: T) -> Result<(), SendError> { + match &self.flavor { + SenderFlavor::Array(chan) => chan.send(msg, None), + SenderFlavor::List(chan) => chan.send(msg, None), + SenderFlavor::Zero(chan) => chan.send(msg, None), + } + .map_err(|err| match err { + SendTimeoutError::Disconnected(msg) => SendError(msg), + SendTimeoutError::Timeout(_) => unreachable!(), + }) + } +} + +// The methods below are not used by `sync::mpsc`, but +// are useful and we'll likely want to expose them +// eventually +#[allow(unused)] +impl Sender { + /// Waits for a message to be sent into the channel, but only for a limited time. + /// + /// If the channel is full and not disconnected, this call will block until the send operation + /// can proceed or the operation times out. If the channel becomes disconnected, this call will + /// wake up and return an error. The returned error contains the original message. + /// + /// If called on a zero-capacity channel, this method will wait for a receive operation to + /// appear on the other side of the channel. + pub fn send_timeout(&self, msg: T, timeout: Duration) -> Result<(), SendTimeoutError> { + match Instant::now().checked_add(timeout) { + Some(deadline) => self.send_deadline(msg, deadline), + // So far in the future that it's practically the same as waiting indefinitely. + None => self.send(msg).map_err(SendTimeoutError::from), + } + } + + /// Waits for a message to be sent into the channel, but only until a given deadline. + /// + /// If the channel is full and not disconnected, this call will block until the send operation + /// can proceed or the operation times out. If the channel becomes disconnected, this call will + /// wake up and return an error. The returned error contains the original message. + /// + /// If called on a zero-capacity channel, this method will wait for a receive operation to + /// appear on the other side of the channel. + pub fn send_deadline(&self, msg: T, deadline: Instant) -> Result<(), SendTimeoutError> { + match &self.flavor { + SenderFlavor::Array(chan) => chan.send(msg, Some(deadline)), + SenderFlavor::List(chan) => chan.send(msg, Some(deadline)), + SenderFlavor::Zero(chan) => chan.send(msg, Some(deadline)), + } + } + + /// Returns `true` if the channel is empty. + /// + /// Note: Zero-capacity channels are always empty. + pub fn is_empty(&self) -> bool { + match &self.flavor { + SenderFlavor::Array(chan) => chan.is_empty(), + SenderFlavor::List(chan) => chan.is_empty(), + SenderFlavor::Zero(chan) => chan.is_empty(), + } + } + + /// Returns `true` if the channel is full. + /// + /// Note: Zero-capacity channels are always full. + pub fn is_full(&self) -> bool { + match &self.flavor { + SenderFlavor::Array(chan) => chan.is_full(), + SenderFlavor::List(chan) => chan.is_full(), + SenderFlavor::Zero(chan) => chan.is_full(), + } + } + + /// Returns the number of messages in the channel. + pub fn len(&self) -> usize { + match &self.flavor { + SenderFlavor::Array(chan) => chan.len(), + SenderFlavor::List(chan) => chan.len(), + SenderFlavor::Zero(chan) => chan.len(), + } + } + + /// If the channel is bounded, returns its capacity. + pub fn capacity(&self) -> Option { + match &self.flavor { + SenderFlavor::Array(chan) => chan.capacity(), + SenderFlavor::List(chan) => chan.capacity(), + SenderFlavor::Zero(chan) => chan.capacity(), + } + } + + /// Returns `true` if senders belong to the same channel. + pub fn same_channel(&self, other: &Sender) -> bool { + match (&self.flavor, &other.flavor) { + (SenderFlavor::Array(ref a), SenderFlavor::Array(ref b)) => a == b, + (SenderFlavor::List(ref a), SenderFlavor::List(ref b)) => a == b, + (SenderFlavor::Zero(ref a), SenderFlavor::Zero(ref b)) => a == b, + _ => false, + } + } +} + +impl Drop for Sender { + fn drop(&mut self) { + unsafe { + match &self.flavor { + SenderFlavor::Array(chan) => chan.release(|c| c.disconnect()), + SenderFlavor::List(chan) => chan.release(|c| c.disconnect_senders()), + SenderFlavor::Zero(chan) => chan.release(|c| c.disconnect()), + } + } + } +} + +impl Clone for Sender { + fn clone(&self) -> Self { + let flavor = match &self.flavor { + SenderFlavor::Array(chan) => SenderFlavor::Array(chan.acquire()), + SenderFlavor::List(chan) => SenderFlavor::List(chan.acquire()), + SenderFlavor::Zero(chan) => SenderFlavor::Zero(chan.acquire()), + }; + + Sender { flavor } + } +} + +impl fmt::Debug for Sender { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("Sender { .. }") + } +} + +/// The receiving side of a channel. +pub struct Receiver { + flavor: ReceiverFlavor, +} + +/// Receiver flavors. +enum ReceiverFlavor { + /// Bounded channel based on a preallocated array. + Array(counter::Receiver>), + + /// Unbounded channel implemented as a linked list. + List(counter::Receiver>), + + /// Zero-capacity channel. + Zero(counter::Receiver>), +} + +unsafe impl Send for Receiver {} +unsafe impl Sync for Receiver {} + +impl UnwindSafe for Receiver {} +impl RefUnwindSafe for Receiver {} + +impl Receiver { + /// Attempts to receive a message from the channel without blocking. + /// + /// This method will either receive a message from the channel immediately or return an error + /// if the channel is empty. + /// + /// If called on a zero-capacity channel, this method will receive a message only if there + /// happens to be a send operation on the other side of the channel at the same time. + pub fn try_recv(&self) -> Result { + match &self.flavor { + ReceiverFlavor::Array(chan) => chan.try_recv(), + ReceiverFlavor::List(chan) => chan.try_recv(), + ReceiverFlavor::Zero(chan) => chan.try_recv(), + } + } + + /// Blocks the current thread until a message is received or the channel is empty and + /// disconnected. + /// + /// If the channel is empty and not disconnected, this call will block until the receive + /// operation can proceed. If the channel is empty and becomes disconnected, this call will + /// wake up and return an error. + /// + /// If called on a zero-capacity channel, this method will wait for a send operation to appear + /// on the other side of the channel. + pub fn recv(&self) -> Result { + match &self.flavor { + ReceiverFlavor::Array(chan) => chan.recv(None), + ReceiverFlavor::List(chan) => chan.recv(None), + ReceiverFlavor::Zero(chan) => chan.recv(None), + } + .map_err(|_| RecvError) + } + + /// Waits for a message to be received from the channel, but only for a limited time. + /// + /// If the channel is empty and not disconnected, this call will block until the receive + /// operation can proceed or the operation times out. If the channel is empty and becomes + /// disconnected, this call will wake up and return an error. + /// + /// If called on a zero-capacity channel, this method will wait for a send operation to appear + /// on the other side of the channel. + pub fn recv_timeout(&self, timeout: Duration) -> Result { + match Instant::now().checked_add(timeout) { + Some(deadline) => self.recv_deadline(deadline), + // So far in the future that it's practically the same as waiting indefinitely. + None => self.recv().map_err(RecvTimeoutError::from), + } + } + + /// Waits for a message to be received from the channel, but only for a limited time. + /// + /// If the channel is empty and not disconnected, this call will block until the receive + /// operation can proceed or the operation times out. If the channel is empty and becomes + /// disconnected, this call will wake up and return an error. + /// + /// If called on a zero-capacity channel, this method will wait for a send operation to appear + /// on the other side of the channel. + pub fn recv_deadline(&self, deadline: Instant) -> Result { + match &self.flavor { + ReceiverFlavor::Array(chan) => chan.recv(Some(deadline)), + ReceiverFlavor::List(chan) => chan.recv(Some(deadline)), + ReceiverFlavor::Zero(chan) => chan.recv(Some(deadline)), + } + } +} + +// The methods below are not used by `sync::mpsc`, but +// are useful and we'll likely want to expose them +// eventually +#[allow(unused)] +impl Receiver { + /// Returns `true` if the channel is empty. + /// + /// Note: Zero-capacity channels are always empty. + pub fn is_empty(&self) -> bool { + match &self.flavor { + ReceiverFlavor::Array(chan) => chan.is_empty(), + ReceiverFlavor::List(chan) => chan.is_empty(), + ReceiverFlavor::Zero(chan) => chan.is_empty(), + } + } + + /// Returns `true` if the channel is full. + /// + /// Note: Zero-capacity channels are always full. + pub fn is_full(&self) -> bool { + match &self.flavor { + ReceiverFlavor::Array(chan) => chan.is_full(), + ReceiverFlavor::List(chan) => chan.is_full(), + ReceiverFlavor::Zero(chan) => chan.is_full(), + } + } + + /// Returns the number of messages in the channel. + pub fn len(&self) -> usize { + match &self.flavor { + ReceiverFlavor::Array(chan) => chan.len(), + ReceiverFlavor::List(chan) => chan.len(), + ReceiverFlavor::Zero(chan) => chan.len(), + } + } + + /// If the channel is bounded, returns its capacity. + pub fn capacity(&self) -> Option { + match &self.flavor { + ReceiverFlavor::Array(chan) => chan.capacity(), + ReceiverFlavor::List(chan) => chan.capacity(), + ReceiverFlavor::Zero(chan) => chan.capacity(), + } + } + + /// Returns `true` if receivers belong to the same channel. + pub fn same_channel(&self, other: &Receiver) -> bool { + match (&self.flavor, &other.flavor) { + (ReceiverFlavor::Array(a), ReceiverFlavor::Array(b)) => a == b, + (ReceiverFlavor::List(a), ReceiverFlavor::List(b)) => a == b, + (ReceiverFlavor::Zero(a), ReceiverFlavor::Zero(b)) => a == b, + _ => false, + } + } +} + +impl Drop for Receiver { + fn drop(&mut self) { + unsafe { + match &self.flavor { + ReceiverFlavor::Array(chan) => chan.release(|c| c.disconnect()), + ReceiverFlavor::List(chan) => chan.release(|c| c.disconnect_receivers()), + ReceiverFlavor::Zero(chan) => chan.release(|c| c.disconnect()), + } + } + } +} + +impl Clone for Receiver { + fn clone(&self) -> Self { + let flavor = match &self.flavor { + ReceiverFlavor::Array(chan) => ReceiverFlavor::Array(chan.acquire()), + ReceiverFlavor::List(chan) => ReceiverFlavor::List(chan.acquire()), + ReceiverFlavor::Zero(chan) => ReceiverFlavor::Zero(chan.acquire()), + }; + + Receiver { flavor } + } +} + +impl fmt::Debug for Receiver { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("Receiver { .. }") + } +} diff --git a/crux-mir/lib/std/src/sync/mpmc/select.rs b/crux-mir/lib/std/src/sync/mpmc/select.rs new file mode 100644 index 000000000..56a83fee2 --- /dev/null +++ b/crux-mir/lib/std/src/sync/mpmc/select.rs @@ -0,0 +1,71 @@ +/// Temporary data that gets initialized during a blocking operation, and is consumed by +/// `read` or `write`. +/// +/// Each field contains data associated with a specific channel flavor. +#[derive(Debug, Default)] +pub struct Token { + pub(crate) array: super::array::ArrayToken, + pub(crate) list: super::list::ListToken, + #[allow(dead_code)] + pub(crate) zero: super::zero::ZeroToken, +} + +/// Identifier associated with an operation by a specific thread on a specific channel. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Operation(usize); + +impl Operation { + /// Creates an operation identifier from a mutable reference. + /// + /// This function essentially just turns the address of the reference into a number. The + /// reference should point to a variable that is specific to the thread and the operation, + /// and is alive for the entire duration of a blocking operation. + #[inline] + pub fn hook(r: &mut T) -> Operation { + let val = r as *mut T as usize; + // Make sure that the pointer address doesn't equal the numerical representation of + // `Selected::{Waiting, Aborted, Disconnected}`. + assert!(val > 2); + Operation(val) + } +} + +/// Current state of a blocking operation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Selected { + /// Still waiting for an operation. + Waiting, + + /// The attempt to block the current thread has been aborted. + Aborted, + + /// An operation became ready because a channel is disconnected. + Disconnected, + + /// An operation became ready because a message can be sent or received. + Operation(Operation), +} + +impl From for Selected { + #[inline] + fn from(val: usize) -> Selected { + match val { + 0 => Selected::Waiting, + 1 => Selected::Aborted, + 2 => Selected::Disconnected, + oper => Selected::Operation(Operation(oper)), + } + } +} + +impl Into for Selected { + #[inline] + fn into(self) -> usize { + match self { + Selected::Waiting => 0, + Selected::Aborted => 1, + Selected::Disconnected => 2, + Selected::Operation(Operation(val)) => val, + } + } +} diff --git a/crux-mir/lib/std/src/sync/mpmc/utils.rs b/crux-mir/lib/std/src/sync/mpmc/utils.rs new file mode 100644 index 000000000..cfe42750d --- /dev/null +++ b/crux-mir/lib/std/src/sync/mpmc/utils.rs @@ -0,0 +1,143 @@ +use crate::cell::Cell; +use crate::ops::{Deref, DerefMut}; + +/// Pads and aligns a value to the length of a cache line. +#[derive(Clone, Copy, Default, Hash, PartialEq, Eq)] +// Starting from Intel's Sandy Bridge, spatial prefetcher is now pulling pairs of 64-byte cache +// lines at a time, so we have to align to 128 bytes rather than 64. +// +// Sources: +// - https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf +// - https://github.com/facebook/folly/blob/1b5288e6eea6df074758f877c849b6e73bbb9fbb/folly/lang/Align.h#L107 +// +// ARM's big.LITTLE architecture has asymmetric cores and "big" cores have 128-byte cache line size. +// +// Sources: +// - https://www.mono-project.com/news/2016/09/12/arm64-icache/ +// +// powerpc64 has 128-byte cache line size. +// +// Sources: +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_ppc64x.go#L9 +#[cfg_attr( + any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "powerpc64",), + repr(align(128)) +)] +// arm, mips, mips64, and riscv64 have 32-byte cache line size. +// +// Sources: +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_arm.go#L7 +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips.go#L7 +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mipsle.go#L7 +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips64x.go#L9 +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_riscv64.go#L7 +#[cfg_attr( + any( + target_arch = "arm", + target_arch = "mips", + target_arch = "mips64", + target_arch = "riscv64", + ), + repr(align(32)) +)] +// s390x has 256-byte cache line size. +// +// Sources: +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_s390x.go#L7 +#[cfg_attr(target_arch = "s390x", repr(align(256)))] +// x86 and wasm have 64-byte cache line size. +// +// Sources: +// - https://github.com/golang/go/blob/dda2991c2ea0c5914714469c4defc2562a907230/src/internal/cpu/cpu_x86.go#L9 +// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_wasm.go#L7 +// +// All others are assumed to have 64-byte cache line size. +#[cfg_attr( + not(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "arm", + target_arch = "mips", + target_arch = "mips64", + target_arch = "riscv64", + target_arch = "s390x", + )), + repr(align(64)) +)] +pub struct CachePadded { + value: T, +} + +impl CachePadded { + /// Pads and aligns a value to the length of a cache line. + pub fn new(value: T) -> CachePadded { + CachePadded:: { value } + } +} + +impl Deref for CachePadded { + type Target = T; + + fn deref(&self) -> &T { + &self.value + } +} + +impl DerefMut for CachePadded { + fn deref_mut(&mut self) -> &mut T { + &mut self.value + } +} + +const SPIN_LIMIT: u32 = 6; + +/// Performs quadratic backoff in spin loops. +pub struct Backoff { + step: Cell, +} + +impl Backoff { + /// Creates a new `Backoff`. + pub fn new() -> Self { + Backoff { step: Cell::new(0) } + } + + /// Backs off using lightweight spinning. + /// + /// This method should be used for: + /// - Retrying an operation because another thread made progress. i.e. on CAS failure. + /// - Waiting for an operation to complete by spinning optimistically for a few iterations + /// before falling back to parking the thread (see `Backoff::is_completed`). + #[inline] + pub fn spin_light(&self) { + let step = self.step.get().min(SPIN_LIMIT); + for _ in 0..step.pow(2) { + crate::hint::spin_loop(); + } + + self.step.set(self.step.get() + 1); + } + + /// Backs off using heavyweight spinning. + /// + /// This method should be used in blocking loops where parking the thread is not an option. + #[inline] + pub fn spin_heavy(&self) { + if self.step.get() <= SPIN_LIMIT { + for _ in 0..self.step.get().pow(2) { + crate::hint::spin_loop() + } + } else { + crate::thread::yield_now(); + } + + self.step.set(self.step.get() + 1); + } + + /// Returns `true` if quadratic backoff has completed and parking the thread is advised. + #[inline] + pub fn is_completed(&self) -> bool { + self.step.get() > SPIN_LIMIT + } +} diff --git a/crux-mir/lib/std/src/sync/mpmc/waker.rs b/crux-mir/lib/std/src/sync/mpmc/waker.rs new file mode 100644 index 000000000..4912ca4f8 --- /dev/null +++ b/crux-mir/lib/std/src/sync/mpmc/waker.rs @@ -0,0 +1,204 @@ +//! Waking mechanism for threads blocked on channel operations. + +use super::context::Context; +use super::select::{Operation, Selected}; + +use crate::ptr; +use crate::sync::atomic::{AtomicBool, Ordering}; +use crate::sync::Mutex; + +/// Represents a thread blocked on a specific channel operation. +pub(crate) struct Entry { + /// The operation. + pub(crate) oper: Operation, + + /// Optional packet. + pub(crate) packet: *mut (), + + /// Context associated with the thread owning this operation. + pub(crate) cx: Context, +} + +/// A queue of threads blocked on channel operations. +/// +/// This data structure is used by threads to register blocking operations and get woken up once +/// an operation becomes ready. +pub(crate) struct Waker { + /// A list of select operations. + selectors: Vec, + + /// A list of operations waiting to be ready. + observers: Vec, +} + +impl Waker { + /// Creates a new `Waker`. + #[inline] + pub(crate) fn new() -> Self { + Waker { selectors: Vec::new(), observers: Vec::new() } + } + + /// Registers a select operation. + #[inline] + pub(crate) fn register(&mut self, oper: Operation, cx: &Context) { + self.register_with_packet(oper, ptr::null_mut(), cx); + } + + /// Registers a select operation and a packet. + #[inline] + pub(crate) fn register_with_packet(&mut self, oper: Operation, packet: *mut (), cx: &Context) { + self.selectors.push(Entry { oper, packet, cx: cx.clone() }); + } + + /// Unregisters a select operation. + #[inline] + pub(crate) fn unregister(&mut self, oper: Operation) -> Option { + if let Some((i, _)) = + self.selectors.iter().enumerate().find(|&(_, entry)| entry.oper == oper) + { + let entry = self.selectors.remove(i); + Some(entry) + } else { + None + } + } + + /// Attempts to find another thread's entry, select the operation, and wake it up. + #[inline] + pub(crate) fn try_select(&mut self) -> Option { + self.selectors + .iter() + .position(|selector| { + // Does the entry belong to a different thread? + selector.cx.thread_id() != current_thread_id() + && selector // Try selecting this operation. + .cx + .try_select(Selected::Operation(selector.oper)) + .is_ok() + && { + // Provide the packet. + selector.cx.store_packet(selector.packet); + // Wake the thread up. + selector.cx.unpark(); + true + } + }) + // Remove the entry from the queue to keep it clean and improve + // performance. + .map(|pos| self.selectors.remove(pos)) + } + + /// Notifies all operations waiting to be ready. + #[inline] + pub(crate) fn notify(&mut self) { + for entry in self.observers.drain(..) { + if entry.cx.try_select(Selected::Operation(entry.oper)).is_ok() { + entry.cx.unpark(); + } + } + } + + /// Notifies all registered operations that the channel is disconnected. + #[inline] + pub(crate) fn disconnect(&mut self) { + for entry in self.selectors.iter() { + if entry.cx.try_select(Selected::Disconnected).is_ok() { + // Wake the thread up. + // + // Here we don't remove the entry from the queue. Registered threads must + // unregister from the waker by themselves. They might also want to recover the + // packet value and destroy it, if necessary. + entry.cx.unpark(); + } + } + + self.notify(); + } +} + +impl Drop for Waker { + #[inline] + fn drop(&mut self) { + debug_assert_eq!(self.selectors.len(), 0); + debug_assert_eq!(self.observers.len(), 0); + } +} + +/// A waker that can be shared among threads without locking. +/// +/// This is a simple wrapper around `Waker` that internally uses a mutex for synchronization. +pub(crate) struct SyncWaker { + /// The inner `Waker`. + inner: Mutex, + + /// `true` if the waker is empty. + is_empty: AtomicBool, +} + +impl SyncWaker { + /// Creates a new `SyncWaker`. + #[inline] + pub(crate) fn new() -> Self { + SyncWaker { inner: Mutex::new(Waker::new()), is_empty: AtomicBool::new(true) } + } + + /// Registers the current thread with an operation. + #[inline] + pub(crate) fn register(&self, oper: Operation, cx: &Context) { + let mut inner = self.inner.lock().unwrap(); + inner.register(oper, cx); + self.is_empty + .store(inner.selectors.is_empty() && inner.observers.is_empty(), Ordering::SeqCst); + } + + /// Unregisters an operation previously registered by the current thread. + #[inline] + pub(crate) fn unregister(&self, oper: Operation) -> Option { + let mut inner = self.inner.lock().unwrap(); + let entry = inner.unregister(oper); + self.is_empty + .store(inner.selectors.is_empty() && inner.observers.is_empty(), Ordering::SeqCst); + entry + } + + /// Attempts to find one thread (not the current one), select its operation, and wake it up. + #[inline] + pub(crate) fn notify(&self) { + if !self.is_empty.load(Ordering::SeqCst) { + let mut inner = self.inner.lock().unwrap(); + if !self.is_empty.load(Ordering::SeqCst) { + inner.try_select(); + inner.notify(); + self.is_empty.store( + inner.selectors.is_empty() && inner.observers.is_empty(), + Ordering::SeqCst, + ); + } + } + } + + /// Notifies all threads that the channel is disconnected. + #[inline] + pub(crate) fn disconnect(&self) { + let mut inner = self.inner.lock().unwrap(); + inner.disconnect(); + self.is_empty + .store(inner.selectors.is_empty() && inner.observers.is_empty(), Ordering::SeqCst); + } +} + +impl Drop for SyncWaker { + #[inline] + fn drop(&mut self) { + debug_assert!(self.is_empty.load(Ordering::SeqCst)); + } +} + +/// Returns a unique id for the current thread. +#[inline] +pub fn current_thread_id() -> usize { + // `u8` is not drop so this variable will be available during thread destruction, + // whereas `thread::current()` would not be + thread_local! { static DUMMY: u8 = 0 } + DUMMY.with(|x| (x as *const u8).addr()) +} diff --git a/crux-mir/lib/std/src/sync/mpmc/zero.rs b/crux-mir/lib/std/src/sync/mpmc/zero.rs new file mode 100644 index 000000000..33f768dcb --- /dev/null +++ b/crux-mir/lib/std/src/sync/mpmc/zero.rs @@ -0,0 +1,318 @@ +//! Zero-capacity channel. +//! +//! This kind of channel is also known as *rendezvous* channel. + +use super::context::Context; +use super::error::*; +use super::select::{Operation, Selected, Token}; +use super::utils::Backoff; +use super::waker::Waker; + +use crate::cell::UnsafeCell; +use crate::marker::PhantomData; +use crate::sync::atomic::{AtomicBool, Ordering}; +use crate::sync::Mutex; +use crate::time::Instant; +use crate::{fmt, ptr}; + +/// A pointer to a packet. +pub(crate) struct ZeroToken(*mut ()); + +impl Default for ZeroToken { + fn default() -> Self { + Self(ptr::null_mut()) + } +} + +impl fmt::Debug for ZeroToken { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&(self.0 as usize), f) + } +} + +/// A slot for passing one message from a sender to a receiver. +struct Packet { + /// Equals `true` if the packet is allocated on the stack. + on_stack: bool, + + /// Equals `true` once the packet is ready for reading or writing. + ready: AtomicBool, + + /// The message. + msg: UnsafeCell>, +} + +impl Packet { + /// Creates an empty packet on the stack. + fn empty_on_stack() -> Packet { + Packet { on_stack: true, ready: AtomicBool::new(false), msg: UnsafeCell::new(None) } + } + + /// Creates a packet on the stack, containing a message. + fn message_on_stack(msg: T) -> Packet { + Packet { on_stack: true, ready: AtomicBool::new(false), msg: UnsafeCell::new(Some(msg)) } + } + + /// Waits until the packet becomes ready for reading or writing. + fn wait_ready(&self) { + let backoff = Backoff::new(); + while !self.ready.load(Ordering::Acquire) { + backoff.spin_heavy(); + } + } +} + +/// Inner representation of a zero-capacity channel. +struct Inner { + /// Senders waiting to pair up with a receive operation. + senders: Waker, + + /// Receivers waiting to pair up with a send operation. + receivers: Waker, + + /// Equals `true` when the channel is disconnected. + is_disconnected: bool, +} + +/// Zero-capacity channel. +pub(crate) struct Channel { + /// Inner representation of the channel. + inner: Mutex, + + /// Indicates that dropping a `Channel` may drop values of type `T`. + _marker: PhantomData, +} + +impl Channel { + /// Constructs a new zero-capacity channel. + pub(crate) fn new() -> Self { + Channel { + inner: Mutex::new(Inner { + senders: Waker::new(), + receivers: Waker::new(), + is_disconnected: false, + }), + _marker: PhantomData, + } + } + + /// Writes a message into the packet. + pub(crate) unsafe fn write(&self, token: &mut Token, msg: T) -> Result<(), T> { + // If there is no packet, the channel is disconnected. + if token.zero.0.is_null() { + return Err(msg); + } + + let packet = &*(token.zero.0 as *const Packet); + packet.msg.get().write(Some(msg)); + packet.ready.store(true, Ordering::Release); + Ok(()) + } + + /// Reads a message from the packet. + pub(crate) unsafe fn read(&self, token: &mut Token) -> Result { + // If there is no packet, the channel is disconnected. + if token.zero.0.is_null() { + return Err(()); + } + + let packet = &*(token.zero.0 as *const Packet); + + if packet.on_stack { + // The message has been in the packet from the beginning, so there is no need to wait + // for it. However, after reading the message, we need to set `ready` to `true` in + // order to signal that the packet can be destroyed. + let msg = packet.msg.get().replace(None).unwrap(); + packet.ready.store(true, Ordering::Release); + Ok(msg) + } else { + // Wait until the message becomes available, then read it and destroy the + // heap-allocated packet. + packet.wait_ready(); + let msg = packet.msg.get().replace(None).unwrap(); + drop(Box::from_raw(token.zero.0 as *mut Packet)); + Ok(msg) + } + } + + /// Attempts to send a message into the channel. + pub(crate) fn try_send(&self, msg: T) -> Result<(), TrySendError> { + let token = &mut Token::default(); + let mut inner = self.inner.lock().unwrap(); + + // If there's a waiting receiver, pair up with it. + if let Some(operation) = inner.receivers.try_select() { + token.zero.0 = operation.packet; + drop(inner); + unsafe { + self.write(token, msg).ok().unwrap(); + } + Ok(()) + } else if inner.is_disconnected { + Err(TrySendError::Disconnected(msg)) + } else { + Err(TrySendError::Full(msg)) + } + } + + /// Sends a message into the channel. + pub(crate) fn send( + &self, + msg: T, + deadline: Option, + ) -> Result<(), SendTimeoutError> { + let token = &mut Token::default(); + let mut inner = self.inner.lock().unwrap(); + + // If there's a waiting receiver, pair up with it. + if let Some(operation) = inner.receivers.try_select() { + token.zero.0 = operation.packet; + drop(inner); + unsafe { + self.write(token, msg).ok().unwrap(); + } + return Ok(()); + } + + if inner.is_disconnected { + return Err(SendTimeoutError::Disconnected(msg)); + } + + Context::with(|cx| { + // Prepare for blocking until a receiver wakes us up. + let oper = Operation::hook(token); + let mut packet = Packet::::message_on_stack(msg); + inner.senders.register_with_packet(oper, &mut packet as *mut Packet as *mut (), cx); + inner.receivers.notify(); + drop(inner); + + // Block the current thread. + let sel = cx.wait_until(deadline); + + match sel { + Selected::Waiting => unreachable!(), + Selected::Aborted => { + self.inner.lock().unwrap().senders.unregister(oper).unwrap(); + let msg = unsafe { packet.msg.get().replace(None).unwrap() }; + Err(SendTimeoutError::Timeout(msg)) + } + Selected::Disconnected => { + self.inner.lock().unwrap().senders.unregister(oper).unwrap(); + let msg = unsafe { packet.msg.get().replace(None).unwrap() }; + Err(SendTimeoutError::Disconnected(msg)) + } + Selected::Operation(_) => { + // Wait until the message is read, then drop the packet. + packet.wait_ready(); + Ok(()) + } + } + }) + } + + /// Attempts to receive a message without blocking. + pub(crate) fn try_recv(&self) -> Result { + let token = &mut Token::default(); + let mut inner = self.inner.lock().unwrap(); + + // If there's a waiting sender, pair up with it. + if let Some(operation) = inner.senders.try_select() { + token.zero.0 = operation.packet; + drop(inner); + unsafe { self.read(token).map_err(|_| TryRecvError::Disconnected) } + } else if inner.is_disconnected { + Err(TryRecvError::Disconnected) + } else { + Err(TryRecvError::Empty) + } + } + + /// Receives a message from the channel. + pub(crate) fn recv(&self, deadline: Option) -> Result { + let token = &mut Token::default(); + let mut inner = self.inner.lock().unwrap(); + + // If there's a waiting sender, pair up with it. + if let Some(operation) = inner.senders.try_select() { + token.zero.0 = operation.packet; + drop(inner); + unsafe { + return self.read(token).map_err(|_| RecvTimeoutError::Disconnected); + } + } + + if inner.is_disconnected { + return Err(RecvTimeoutError::Disconnected); + } + + Context::with(|cx| { + // Prepare for blocking until a sender wakes us up. + let oper = Operation::hook(token); + let mut packet = Packet::::empty_on_stack(); + inner.receivers.register_with_packet( + oper, + &mut packet as *mut Packet as *mut (), + cx, + ); + inner.senders.notify(); + drop(inner); + + // Block the current thread. + let sel = cx.wait_until(deadline); + + match sel { + Selected::Waiting => unreachable!(), + Selected::Aborted => { + self.inner.lock().unwrap().receivers.unregister(oper).unwrap(); + Err(RecvTimeoutError::Timeout) + } + Selected::Disconnected => { + self.inner.lock().unwrap().receivers.unregister(oper).unwrap(); + Err(RecvTimeoutError::Disconnected) + } + Selected::Operation(_) => { + // Wait until the message is provided, then read it. + packet.wait_ready(); + unsafe { Ok(packet.msg.get().replace(None).unwrap()) } + } + } + }) + } + + /// Disconnects the channel and wakes up all blocked senders and receivers. + /// + /// Returns `true` if this call disconnected the channel. + pub(crate) fn disconnect(&self) -> bool { + let mut inner = self.inner.lock().unwrap(); + + if !inner.is_disconnected { + inner.is_disconnected = true; + inner.senders.disconnect(); + inner.receivers.disconnect(); + true + } else { + false + } + } + + /// Returns the current number of messages inside the channel. + pub(crate) fn len(&self) -> usize { + 0 + } + + /// Returns the capacity of the channel. + #[allow(clippy::unnecessary_wraps)] // This is intentional. + pub(crate) fn capacity(&self) -> Option { + Some(0) + } + + /// Returns `true` if the channel is empty. + pub(crate) fn is_empty(&self) -> bool { + true + } + + /// Returns `true` if the channel is full. + pub(crate) fn is_full(&self) -> bool { + true + } +} diff --git a/crux-mir/lib/std/src/sync/mpsc/blocking.rs b/crux-mir/lib/std/src/sync/mpsc/blocking.rs deleted file mode 100644 index d34de6a4f..000000000 --- a/crux-mir/lib/std/src/sync/mpsc/blocking.rs +++ /dev/null @@ -1,79 +0,0 @@ -//! Generic support for building blocking abstractions. - -use crate::mem; -use crate::sync::atomic::{AtomicBool, Ordering}; -use crate::sync::Arc; -use crate::thread::{self, Thread}; -use crate::time::Instant; - -struct Inner { - thread: Thread, - woken: AtomicBool, -} - -unsafe impl Send for Inner {} -unsafe impl Sync for Inner {} - -#[derive(Clone)] -pub struct SignalToken { - inner: Arc, -} - -pub struct WaitToken { - inner: Arc, -} - -impl !Send for WaitToken {} - -impl !Sync for WaitToken {} - -pub fn tokens() -> (WaitToken, SignalToken) { - let inner = Arc::new(Inner { thread: thread::current(), woken: AtomicBool::new(false) }); - let wait_token = WaitToken { inner: inner.clone() }; - let signal_token = SignalToken { inner }; - (wait_token, signal_token) -} - -impl SignalToken { - pub fn signal(&self) -> bool { - let wake = !self.inner.woken.compare_and_swap(false, true, Ordering::SeqCst); - if wake { - self.inner.thread.unpark(); - } - wake - } - - /// Converts to an unsafe usize value. Useful for storing in a pipe's state - /// flag. - #[inline] - pub unsafe fn cast_to_usize(self) -> usize { - mem::transmute(self.inner) - } - - /// Converts from an unsafe usize value. Useful for retrieving a pipe's state - /// flag. - #[inline] - pub unsafe fn cast_from_usize(signal_ptr: usize) -> SignalToken { - SignalToken { inner: mem::transmute(signal_ptr) } - } -} - -impl WaitToken { - pub fn wait(self) { - while !self.inner.woken.load(Ordering::SeqCst) { - thread::park() - } - } - - /// Returns `true` if we wake up normally. - pub fn wait_max_until(self, end: Instant) -> bool { - while !self.inner.woken.load(Ordering::SeqCst) { - let now = Instant::now(); - if now >= end { - return false; - } - thread::park_timeout(end - now) - } - true - } -} diff --git a/crux-mir/lib/std/src/sync/mpsc/cache_aligned.rs b/crux-mir/lib/std/src/sync/mpsc/cache_aligned.rs deleted file mode 100644 index b08421443..000000000 --- a/crux-mir/lib/std/src/sync/mpsc/cache_aligned.rs +++ /dev/null @@ -1,27 +0,0 @@ -use crate::ops::{Deref, DerefMut}; - -#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[repr(align(64))] -pub(super) struct Aligner; - -#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub(super) struct CacheAligned(pub T, pub Aligner); - -impl Deref for CacheAligned { - type Target = T; - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl DerefMut for CacheAligned { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -impl CacheAligned { - pub(super) fn new(t: T) -> Self { - CacheAligned(t, Aligner) - } -} diff --git a/crux-mir/lib/std/src/sync/mpsc/mod.rs b/crux-mir/lib/std/src/sync/mpsc/mod.rs index e70204d68..6e3c28f10 100644 --- a/crux-mir/lib/std/src/sync/mpsc/mod.rs +++ b/crux-mir/lib/std/src/sync/mpsc/mod.rs @@ -1,5 +1,3 @@ -// ignore-tidy-filelength - //! Multi-producer, single-consumer FIFO queue communication primitives. //! //! This module provides message-based communication over channels, concretely @@ -27,12 +25,7 @@ //! that a bound of 0 is allowed, causing the channel to become a "rendezvous" //! channel where each sender atomically hands off a message to a receiver. //! -//! [`Sender`]: ../../../std/sync/mpsc/struct.Sender.html -//! [`SyncSender`]: ../../../std/sync/mpsc/struct.SyncSender.html -//! [`Receiver`]: ../../../std/sync/mpsc/struct.Receiver.html -//! [`send`]: ../../../std/sync/mpsc/struct.Sender.html#method.send -//! [`channel`]: ../../../std/sync/mpsc/fn.channel.html -//! [`sync_channel`]: ../../../std/sync/mpsc/fn.sync_channel.html +//! [`send`]: Sender::send //! //! ## Disconnection //! @@ -46,9 +39,7 @@ //! will continue to [`unwrap`] the results returned from this module, //! instigating a propagation of failure among threads if one unexpectedly dies. //! -//! [`Result`]: ../../../std/result/enum.Result.html -//! [`Err`]: ../../../std/result/enum.Result.html#variant.Err -//! [`unwrap`]: ../../../std/result/enum.Result.html#method.unwrap +//! [`unwrap`]: Result::unwrap //! //! # Examples //! @@ -114,186 +105,60 @@ //! }); //! rx.recv().unwrap(); //! ``` +//! +//! Unbounded receive loop: +//! +//! ``` +//! use std::sync::mpsc::sync_channel; +//! use std::thread; +//! +//! let (tx, rx) = sync_channel(3); +//! +//! for _ in 0..3 { +//! // It would be the same without thread and clone here +//! // since there will still be one `tx` left. +//! let tx = tx.clone(); +//! // cloned tx dropped within thread +//! thread::spawn(move || tx.send("ok").unwrap()); +//! } +//! +//! // Drop the last sender to stop `rx` waiting for message. +//! // The program will not complete if we comment this out. +//! // **All** `tx` needs to be dropped for `rx` to have `Err`. +//! drop(tx); +//! +//! // Unbounded receiver waiting for all senders to complete. +//! while let Ok(msg) = rx.recv() { +//! println!("{msg}"); +//! } +//! +//! println!("completed"); +//! ``` #![stable(feature = "rust1", since = "1.0.0")] -// A description of how Rust's channel implementation works -// -// Channels are supposed to be the basic building block for all other -// concurrent primitives that are used in Rust. As a result, the channel type -// needs to be highly optimized, flexible, and broad enough for use everywhere. -// -// The choice of implementation of all channels is to be built on lock-free data -// structures. The channels themselves are then consequently also lock-free data -// structures. As always with lock-free code, this is a very "here be dragons" -// territory, especially because I'm unaware of any academic papers that have -// gone into great length about channels of these flavors. -// -// ## Flavors of channels -// -// From the perspective of a consumer of this library, there is only one flavor -// of channel. This channel can be used as a stream and cloned to allow multiple -// senders. Under the hood, however, there are actually three flavors of -// channels in play. -// -// * Flavor::Oneshots - these channels are highly optimized for the one-send use -// case. They contain as few atomics as possible and -// involve one and exactly one allocation. -// * Streams - these channels are optimized for the non-shared use case. They -// use a different concurrent queue that is more tailored for this -// use case. The initial allocation of this flavor of channel is not -// optimized. -// * Shared - this is the most general form of channel that this module offers, -// a channel with multiple senders. This type is as optimized as it -// can be, but the previous two types mentioned are much faster for -// their use-cases. -// -// ## Concurrent queues -// -// The basic idea of Rust's Sender/Receiver types is that send() never blocks, -// but recv() obviously blocks. This means that under the hood there must be -// some shared and concurrent queue holding all of the actual data. -// -// With two flavors of channels, two flavors of queues are also used. We have -// chosen to use queues from a well-known author that are abbreviated as SPSC -// and MPSC (single producer, single consumer and multiple producer, single -// consumer). SPSC queues are used for streams while MPSC queues are used for -// shared channels. -// -// ### SPSC optimizations -// -// The SPSC queue found online is essentially a linked list of nodes where one -// half of the nodes are the "queue of data" and the other half of nodes are a -// cache of unused nodes. The unused nodes are used such that an allocation is -// not required on every push() and a free doesn't need to happen on every -// pop(). -// -// As found online, however, the cache of nodes is of an infinite size. This -// means that if a channel at one point in its life had 50k items in the queue, -// then the queue will always have the capacity for 50k items. I believed that -// this was an unnecessary limitation of the implementation, so I have altered -// the queue to optionally have a bound on the cache size. -// -// By default, streams will have an unbounded SPSC queue with a small-ish cache -// size. The hope is that the cache is still large enough to have very fast -// send() operations while not too large such that millions of channels can -// coexist at once. -// -// ### MPSC optimizations -// -// Right now the MPSC queue has not been optimized. Like the SPSC queue, it uses -// a linked list under the hood to earn its unboundedness, but I have not put -// forth much effort into having a cache of nodes similar to the SPSC queue. -// -// For now, I believe that this is "ok" because shared channels are not the most -// common type, but soon we may wish to revisit this queue choice and determine -// another candidate for backend storage of shared channels. -// -// ## Overview of the Implementation -// -// Now that there's a little background on the concurrent queues used, it's -// worth going into much more detail about the channels themselves. The basic -// pseudocode for a send/recv are: -// -// -// send(t) recv() -// queue.push(t) return if queue.pop() -// if increment() == -1 deschedule { -// wakeup() if decrement() > 0 -// cancel_deschedule() -// } -// queue.pop() -// -// As mentioned before, there are no locks in this implementation, only atomic -// instructions are used. -// -// ### The internal atomic counter -// -// Every channel has a shared counter with each half to keep track of the size -// of the queue. This counter is used to abort descheduling by the receiver and -// to know when to wake up on the sending side. -// -// As seen in the pseudocode, senders will increment this count and receivers -// will decrement the count. The theory behind this is that if a sender sees a -// -1 count, it will wake up the receiver, and if the receiver sees a 1+ count, -// then it doesn't need to block. -// -// The recv() method has a beginning call to pop(), and if successful, it needs -// to decrement the count. It is a crucial implementation detail that this -// decrement does *not* happen to the shared counter. If this were the case, -// then it would be possible for the counter to be very negative when there were -// no receivers waiting, in which case the senders would have to determine when -// it was actually appropriate to wake up a receiver. -// -// Instead, the "steal count" is kept track of separately (not atomically -// because it's only used by receivers), and then the decrement() call when -// descheduling will lump in all of the recent steals into one large decrement. -// -// The implication of this is that if a sender sees a -1 count, then there's -// guaranteed to be a waiter waiting! -// -// ## Native Implementation -// -// A major goal of these channels is to work seamlessly on and off the runtime. -// All of the previous race conditions have been worded in terms of -// scheduler-isms (which is obviously not available without the runtime). -// -// For now, native usage of channels (off the runtime) will fall back onto -// mutexes/cond vars for descheduling/atomic decisions. The no-contention path -// is still entirely lock-free, the "deschedule" blocks above are surrounded by -// a mutex and the "wakeup" blocks involve grabbing a mutex and signaling on a -// condition variable. -// -// ## Select -// -// Being able to support selection over channels has greatly influenced this -// design, and not only does selection need to work inside the runtime, but also -// outside the runtime. -// -// The implementation is fairly straightforward. The goal of select() is not to -// return some data, but only to return which channel can receive data without -// blocking. The implementation is essentially the entire blocking procedure -// followed by an increment as soon as its woken up. The cancellation procedure -// involves an increment and swapping out of to_wake to acquire ownership of the -// thread to unblock. -// -// Sadly this current implementation requires multiple allocations, so I have -// seen the throughput of select() be much worse than it should be. I do not -// believe that there is anything fundamental that needs to change about these -// channels, however, in order to support a more efficient select(). -// -// FIXME: Select is now removed, so these factors are ready to be cleaned up! -// -// # Conclusion -// -// And now that you've seen all the races that I found and attempted to fix, -// here's the code for you to find some more! - -use crate::cell::UnsafeCell; +#[cfg(all(test, not(target_os = "emscripten")))] +mod tests; + +#[cfg(all(test, not(target_os = "emscripten")))] +mod sync_tests; + +// MPSC channels are built as a wrapper around MPMC channels, which +// were ported from the `crossbeam-channel` crate. MPMC channels are +// not exposed publicly, but if you are curious about the implementation, +// that's where everything is. + use crate::error; use crate::fmt; -use crate::mem; -use crate::sync::Arc; +use crate::sync::mpmc; use crate::time::{Duration, Instant}; -mod blocking; -mod mpsc_queue; -mod oneshot; -mod shared; -mod spsc_queue; -mod stream; -mod sync; - -mod cache_aligned; - /// The receiving half of Rust's [`channel`] (or [`sync_channel`]) type. /// This half can only be owned by one thread. /// /// Messages sent to the channel can be retrieved using [`recv`]. /// -/// [`channel`]: fn.channel.html -/// [`sync_channel`]: fn.sync_channel.html -/// [`recv`]: struct.Receiver.html#method.recv +/// [`recv`]: Receiver::recv /// /// # Examples /// @@ -315,8 +180,9 @@ mod cache_aligned; /// println!("{}", recv.recv().unwrap()); // Received after 2 seconds /// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "Receiver")] pub struct Receiver { - inner: UnsafeCell>, + inner: mpmc::Receiver, } // The receiver port can be sent from place to place, so long as it @@ -333,10 +199,8 @@ impl !Sync for Receiver {} /// waiting for a new message, and [`None`] will be returned /// when the corresponding channel has hung up. /// -/// [`iter`]: struct.Receiver.html#method.iter -/// [`Receiver`]: struct.Receiver.html -/// [`next`]: ../../../std/iter/trait.Iterator.html#tymethod.next -/// [`None`]: ../../../std/option/enum.Option.html#variant.None +/// [`iter`]: Receiver::iter +/// [`next`]: Iterator::next /// /// # Examples /// @@ -353,7 +217,7 @@ impl !Sync for Receiver {} /// }); /// /// for x in recv.iter() { -/// println!("Got: {}", x); +/// println!("Got: {x}"); /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] @@ -371,9 +235,7 @@ pub struct Iter<'a, T: 'a> { /// This iterator will never block the caller in order to wait for data to /// become available. Instead, it will return [`None`]. /// -/// [`Receiver`]: struct.Receiver.html -/// [`try_iter`]: struct.Receiver.html#method.try_iter -/// [`None`]: ../../../std/option/enum.Option.html#variant.None +/// [`try_iter`]: Receiver::try_iter /// /// # Examples /// @@ -398,7 +260,7 @@ pub struct Iter<'a, T: 'a> { /// thread::sleep(Duration::from_secs(2)); // block for two seconds /// /// for x in receiver.try_iter() { -/// println!("Got: {}", x); +/// println!("Got: {x}"); /// } /// ``` #[stable(feature = "receiver_try_iter", since = "1.15.0")] @@ -408,15 +270,14 @@ pub struct TryIter<'a, T: 'a> { } /// An owning iterator over messages on a [`Receiver`], -/// created by **Receiver::into_iter**. +/// created by [`into_iter`]. /// /// This iterator will block whenever [`next`] /// is called, waiting for a new message, and [`None`] will be /// returned if the corresponding channel has hung up. /// -/// [`Receiver`]: struct.Receiver.html -/// [`next`]: ../../../std/iter/trait.Iterator.html#tymethod.next -/// [`None`]: ../../../std/option/enum.Option.html#variant.None +/// [`into_iter`]: Receiver::into_iter +/// [`next`]: Iterator::next /// /// # Examples /// @@ -433,7 +294,7 @@ pub struct TryIter<'a, T: 'a> { /// }); /// /// for x in recv.into_iter() { -/// println!("Got: {}", x); +/// println!("Got: {x}"); /// } /// ``` #[stable(feature = "receiver_into_iter", since = "1.1.0")] @@ -447,8 +308,10 @@ pub struct IntoIter { /// /// Messages can be sent through this channel with [`send`]. /// -/// [`channel`]: fn.channel.html -/// [`send`]: struct.Sender.html#method.send +/// Note: all senders (the original and the clones) need to be dropped for the receiver +/// to stop blocking to receive messages with [`Receiver::recv`]. +/// +/// [`send`]: Sender::send /// /// # Examples /// @@ -476,7 +339,7 @@ pub struct IntoIter { /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub struct Sender { - inner: UnsafeCell>, + inner: mpmc::Sender, } // The send port can be sent from place to place, so long as it @@ -493,9 +356,8 @@ impl !Sync for Sender {} /// /// [`send`] will block if there is no space in the internal buffer. /// -/// [`sync_channel`]: fn.sync_channel.html -/// [`send`]: struct.SyncSender.html#method.send -/// [`try_send`]: struct.SyncSender.html#method.try_send +/// [`send`]: SyncSender::send +/// [`try_send`]: SyncSender::try_send /// /// # Examples /// @@ -523,20 +385,20 @@ impl !Sync for Sender {} /// let mut msg; /// /// msg = receiver.recv().unwrap(); -/// println!("message {} received", msg); +/// println!("message {msg} received"); /// /// // "Thread unblocked!" will be printed now /// /// msg = receiver.recv().unwrap(); -/// println!("message {} received", msg); +/// println!("message {msg} received"); /// /// msg = receiver.recv().unwrap(); /// -/// println!("message {} received", msg); +/// println!("message {msg} received"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub struct SyncSender { - inner: Arc>, + inner: mpmc::Sender, } #[stable(feature = "rust1", since = "1.0.0")] @@ -548,9 +410,6 @@ unsafe impl Send for SyncSender {} /// A **send** operation can only fail if the receiving end of a channel is /// disconnected, implying that the data could never be received. The error /// contains the data being sent as a payload so it can be recovered. -/// -/// [`Sender::send`]: struct.Sender.html#method.send -/// [`SyncSender::send`]: struct.SyncSender.html#method.send #[stable(feature = "rust1", since = "1.0.0")] #[derive(PartialEq, Eq, Clone, Copy)] pub struct SendError(#[stable(feature = "rust1", since = "1.0.0")] pub T); @@ -561,10 +420,7 @@ pub struct SendError(#[stable(feature = "rust1", since = "1.0.0")] pub T); /// [`channel`] (or [`sync_channel`]) is disconnected, implying that no further /// messages will ever be received. /// -/// [`recv`]: struct.Receiver.html#method.recv -/// [`Receiver`]: struct.Receiver.html -/// [`channel`]: fn.channel.html -/// [`sync_channel`]: fn.sync_channel.html +/// [`recv`]: Receiver::recv #[derive(PartialEq, Eq, Clone, Copy, Debug)] #[stable(feature = "rust1", since = "1.0.0")] pub struct RecvError; @@ -573,9 +429,7 @@ pub struct RecvError; /// not return data when called. This can occur with both a [`channel`] and /// a [`sync_channel`]. /// -/// [`try_recv`]: struct.Receiver.html#method.try_recv -/// [`channel`]: fn.channel.html -/// [`sync_channel`]: fn.sync_channel.html +/// [`try_recv`]: Receiver::try_recv #[derive(PartialEq, Eq, Clone, Copy, Debug)] #[stable(feature = "rust1", since = "1.0.0")] pub enum TryRecvError { @@ -594,9 +448,7 @@ pub enum TryRecvError { /// unable to return data when called. This can occur with both a [`channel`] and /// a [`sync_channel`]. /// -/// [`recv_timeout`]: struct.Receiver.html#method.recv_timeout -/// [`channel`]: fn.channel.html -/// [`sync_channel`]: fn.sync_channel.html +/// [`recv_timeout`]: Receiver::recv_timeout #[derive(PartialEq, Eq, Clone, Copy, Debug)] #[stable(feature = "mpsc_recv_timeout", since = "1.12.0")] pub enum RecvTimeoutError { @@ -613,7 +465,7 @@ pub enum RecvTimeoutError { /// This enumeration is the list of the possible error outcomes for the /// [`try_send`] method. /// -/// [`try_send`]: struct.SyncSender.html#method.try_send +/// [`try_send`]: SyncSender::try_send #[stable(feature = "rust1", since = "1.0.0")] #[derive(PartialEq, Eq, Clone, Copy)] pub enum TrySendError { @@ -623,54 +475,21 @@ pub enum TrySendError { /// If this is a buffered channel, then the buffer is full at this time. If /// this is not a buffered channel, then there is no [`Receiver`] available to /// acquire the data. - /// - /// [`sync_channel`]: fn.sync_channel.html - /// [`Receiver`]: struct.Receiver.html #[stable(feature = "rust1", since = "1.0.0")] Full(#[stable(feature = "rust1", since = "1.0.0")] T), /// This [`sync_channel`]'s receiving half has disconnected, so the data could not be /// sent. The data is returned back to the callee in this case. - /// - /// [`sync_channel`]: fn.sync_channel.html #[stable(feature = "rust1", since = "1.0.0")] Disconnected(#[stable(feature = "rust1", since = "1.0.0")] T), } -enum Flavor { - Oneshot(Arc>), - Stream(Arc>), - Shared(Arc>), - Sync(Arc>), -} - -#[doc(hidden)] -trait UnsafeFlavor { - fn inner_unsafe(&self) -> &UnsafeCell>; - unsafe fn inner_mut(&self) -> &mut Flavor { - &mut *self.inner_unsafe().get() - } - unsafe fn inner(&self) -> &Flavor { - &*self.inner_unsafe().get() - } -} -impl UnsafeFlavor for Sender { - fn inner_unsafe(&self) -> &UnsafeCell> { - &self.inner - } -} -impl UnsafeFlavor for Receiver { - fn inner_unsafe(&self) -> &UnsafeCell> { - &self.inner - } -} - /// Creates a new asynchronous channel, returning the sender/receiver halves. /// All data sent on the [`Sender`] will become available on the [`Receiver`] in /// the same order as it was sent, and no [`send`] will block the calling thread /// (this channel has an "infinite buffer", unlike [`sync_channel`], which will /// block after its buffer limit is reached). [`recv`] will block until a message -/// is available. +/// is available while there is at least one [`Sender`] alive (including clones). /// /// The [`Sender`] can be cloned to [`send`] to the same channel multiple times, but /// only one [`Receiver`] is supported. @@ -680,13 +499,8 @@ impl UnsafeFlavor for Receiver { /// [`Sender`] is disconnected while trying to [`recv`], the [`recv`] method will /// return a [`RecvError`]. /// -/// [`send`]: struct.Sender.html#method.send -/// [`recv`]: struct.Receiver.html#method.recv -/// [`Sender`]: struct.Sender.html -/// [`Receiver`]: struct.Receiver.html -/// [`sync_channel`]: fn.sync_channel.html -/// [`SendError`]: struct.SendError.html -/// [`RecvError`]: struct.RecvError.html +/// [`send`]: Sender::send +/// [`recv`]: Receiver::recv /// /// # Examples /// @@ -707,10 +521,11 @@ impl UnsafeFlavor for Receiver { /// // Let's see what that answer was /// println!("{:?}", receiver.recv().unwrap()); /// ``` +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn channel() -> (Sender, Receiver) { - let a = Arc::new(oneshot::Packet::new()); - (Sender::new(Flavor::Oneshot(a.clone())), Receiver::new(Flavor::Oneshot(a))) + let (tx, rx) = mpmc::channel(); + (Sender { inner: tx }, Receiver { inner: rx }) } /// Creates a new synchronous, bounded channel. @@ -733,13 +548,8 @@ pub fn channel() -> (Sender, Receiver) { /// [`SendError`]. Similarly, If the [`SyncSender`] is disconnected while trying /// to [`recv`], the [`recv`] method will return a [`RecvError`]. /// -/// [`channel`]: fn.channel.html -/// [`send`]: struct.SyncSender.html#method.send -/// [`recv`]: struct.Receiver.html#method.recv -/// [`SyncSender`]: struct.SyncSender.html -/// [`Receiver`]: struct.Receiver.html -/// [`SendError`]: struct.SendError.html -/// [`RecvError`]: struct.RecvError.html +/// [`send`]: SyncSender::send +/// [`recv`]: Receiver::recv /// /// # Examples /// @@ -760,10 +570,11 @@ pub fn channel() -> (Sender, Receiver) { /// assert_eq!(receiver.recv().unwrap(), 1); /// assert_eq!(receiver.recv().unwrap(), 2); /// ``` +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn sync_channel(bound: usize) -> (SyncSender, Receiver) { - let a = Arc::new(sync::Packet::new(bound)); - (SyncSender::new(a.clone()), Receiver::new(Flavor::Sync(a))) + let (tx, rx) = mpmc::sync_channel(bound); + (SyncSender { inner: tx }, Receiver { inner: rx }) } //////////////////////////////////////////////////////////////////////////////// @@ -771,10 +582,6 @@ pub fn sync_channel(bound: usize) -> (SyncSender, Receiver) { //////////////////////////////////////////////////////////////////////////////// impl Sender { - fn new(inner: Flavor) -> Sender { - Sender { inner: UnsafeCell::new(inner) } - } - /// Attempts to send a value on this channel, returning it back if it could /// not be sent. /// @@ -786,9 +593,6 @@ impl Sender { /// will be received. It is possible for the corresponding receiver to /// hang up immediately after this function returns [`Ok`]. /// - /// [`Err`]: ../../../std/result/enum.Result.html#variant.Err - /// [`Ok`]: ../../../std/result/enum.Result.html#variant.Ok - /// /// This method will never block the current thread. /// /// # Examples @@ -807,104 +611,31 @@ impl Sender { /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn send(&self, t: T) -> Result<(), SendError> { - let (new_inner, ret) = match *unsafe { self.inner() } { - Flavor::Oneshot(ref p) => { - if !p.sent() { - return p.send(t).map_err(SendError); - } else { - let a = Arc::new(stream::Packet::new()); - let rx = Receiver::new(Flavor::Stream(a.clone())); - match p.upgrade(rx) { - oneshot::UpSuccess => { - let ret = a.send(t); - (a, ret) - } - oneshot::UpDisconnected => (a, Err(t)), - oneshot::UpWoke(token) => { - // This send cannot panic because the thread is - // asleep (we're looking at it), so the receiver - // can't go away. - a.send(t).ok().unwrap(); - token.signal(); - (a, Ok(())) - } - } - } - } - Flavor::Stream(ref p) => return p.send(t).map_err(SendError), - Flavor::Shared(ref p) => return p.send(t).map_err(SendError), - Flavor::Sync(..) => unreachable!(), - }; - - unsafe { - let tmp = Sender::new(Flavor::Stream(new_inner)); - mem::swap(self.inner_mut(), tmp.inner_mut()); - } - ret.map_err(SendError) + self.inner.send(t) } } #[stable(feature = "rust1", since = "1.0.0")] impl Clone for Sender { + /// Clone a sender to send to other threads. + /// + /// Note, be aware of the lifetime of the sender because all senders + /// (including the original) need to be dropped in order for + /// [`Receiver::recv`] to stop blocking. fn clone(&self) -> Sender { - let packet = match *unsafe { self.inner() } { - Flavor::Oneshot(ref p) => { - let a = Arc::new(shared::Packet::new()); - { - let guard = a.postinit_lock(); - let rx = Receiver::new(Flavor::Shared(a.clone())); - let sleeper = match p.upgrade(rx) { - oneshot::UpSuccess | oneshot::UpDisconnected => None, - oneshot::UpWoke(task) => Some(task), - }; - a.inherit_blocker(sleeper, guard); - } - a - } - Flavor::Stream(ref p) => { - let a = Arc::new(shared::Packet::new()); - { - let guard = a.postinit_lock(); - let rx = Receiver::new(Flavor::Shared(a.clone())); - let sleeper = match p.upgrade(rx) { - stream::UpSuccess | stream::UpDisconnected => None, - stream::UpWoke(task) => Some(task), - }; - a.inherit_blocker(sleeper, guard); - } - a - } - Flavor::Shared(ref p) => { - p.clone_chan(); - return Sender::new(Flavor::Shared(p.clone())); - } - Flavor::Sync(..) => unreachable!(), - }; - - unsafe { - let tmp = Sender::new(Flavor::Shared(packet.clone())); - mem::swap(self.inner_mut(), tmp.inner_mut()); - } - Sender::new(Flavor::Shared(packet)) + Sender { inner: self.inner.clone() } } } #[stable(feature = "rust1", since = "1.0.0")] impl Drop for Sender { - fn drop(&mut self) { - match *unsafe { self.inner() } { - Flavor::Oneshot(ref p) => p.drop_chan(), - Flavor::Stream(ref p) => p.drop_chan(), - Flavor::Shared(ref p) => p.drop_chan(), - Flavor::Sync(..) => unreachable!(), - } - } + fn drop(&mut self) {} } #[stable(feature = "mpsc_debug", since = "1.8.0")] impl fmt::Debug for Sender { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Sender").finish() + f.debug_struct("Sender").finish_non_exhaustive() } } @@ -913,10 +644,6 @@ impl fmt::Debug for Sender { //////////////////////////////////////////////////////////////////////////////// impl SyncSender { - fn new(inner: Arc>) -> SyncSender { - SyncSender { inner } - } - /// Sends a value on this synchronous channel. /// /// This function will *block* until space in the internal buffer becomes @@ -933,9 +660,6 @@ impl SyncSender { /// [`Receiver`] has disconnected and is no longer able to receive /// information. /// - /// [`Err`]: ../../../std/result/enum.Result.html#variant.Err - /// [`Receiver`]: ../../../std/sync/mpsc/struct.Receiver.html - /// /// # Examples /// /// ```rust @@ -958,7 +682,7 @@ impl SyncSender { /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn send(&self, t: T) -> Result<(), SendError> { - self.inner.send(t).map_err(SendError) + self.inner.send(t) } /// Attempts to send a value on this channel without blocking. @@ -971,7 +695,7 @@ impl SyncSender { /// See [`send`] for notes about guarantees of whether the /// receiver has received the data or not if this function is successful. /// - /// [`send`]: ../../../std/sync/mpsc/struct.SyncSender.html#method.send + /// [`send`]: Self::send /// /// # Examples /// @@ -999,14 +723,14 @@ impl SyncSender { /// /// let mut msg; /// msg = receiver.recv().unwrap(); - /// println!("message {} received", msg); + /// println!("message {msg} received"); /// /// msg = receiver.recv().unwrap(); - /// println!("message {} received", msg); + /// println!("message {msg} received"); /// /// // Third message may have never been sent /// match receiver.try_recv() { - /// Ok(msg) => println!("message {} received", msg), + /// Ok(msg) => println!("message {msg} received"), /// Err(_) => println!("the third message was never sent"), /// } /// ``` @@ -1014,27 +738,33 @@ impl SyncSender { pub fn try_send(&self, t: T) -> Result<(), TrySendError> { self.inner.try_send(t) } + + // Attempts to send for a value on this receiver, returning an error if the + // corresponding channel has hung up, or if it waits more than `timeout`. + // + // This method is currently private and only used for tests. + #[allow(unused)] + fn send_timeout(&self, t: T, timeout: Duration) -> Result<(), mpmc::SendTimeoutError> { + self.inner.send_timeout(t, timeout) + } } #[stable(feature = "rust1", since = "1.0.0")] impl Clone for SyncSender { fn clone(&self) -> SyncSender { - self.inner.clone_chan(); - SyncSender::new(self.inner.clone()) + SyncSender { inner: self.inner.clone() } } } #[stable(feature = "rust1", since = "1.0.0")] impl Drop for SyncSender { - fn drop(&mut self) { - self.inner.drop_chan(); - } + fn drop(&mut self) {} } #[stable(feature = "mpsc_debug", since = "1.8.0")] impl fmt::Debug for SyncSender { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SyncSender").finish() + f.debug_struct("SyncSender").finish_non_exhaustive() } } @@ -1043,10 +773,6 @@ impl fmt::Debug for SyncSender { //////////////////////////////////////////////////////////////////////////////// impl Receiver { - fn new(inner: Flavor) -> Receiver { - Receiver { inner: UnsafeCell::new(inner) } - } - /// Attempts to return a pending value on this receiver without blocking. /// /// This method will never block the caller in order to wait for data to @@ -1059,7 +785,7 @@ impl Receiver { /// Compared with [`recv`], this function has two failure cases instead of one /// (one for disconnection, one for an empty buffer). /// - /// [`recv`]: struct.Receiver.html#method.recv + /// [`recv`]: Self::recv /// /// # Examples /// @@ -1072,44 +798,17 @@ impl Receiver { /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn try_recv(&self) -> Result { - loop { - let new_port = match *unsafe { self.inner() } { - Flavor::Oneshot(ref p) => match p.try_recv() { - Ok(t) => return Ok(t), - Err(oneshot::Empty) => return Err(TryRecvError::Empty), - Err(oneshot::Disconnected) => return Err(TryRecvError::Disconnected), - Err(oneshot::Upgraded(rx)) => rx, - }, - Flavor::Stream(ref p) => match p.try_recv() { - Ok(t) => return Ok(t), - Err(stream::Empty) => return Err(TryRecvError::Empty), - Err(stream::Disconnected) => return Err(TryRecvError::Disconnected), - Err(stream::Upgraded(rx)) => rx, - }, - Flavor::Shared(ref p) => match p.try_recv() { - Ok(t) => return Ok(t), - Err(shared::Empty) => return Err(TryRecvError::Empty), - Err(shared::Disconnected) => return Err(TryRecvError::Disconnected), - }, - Flavor::Sync(ref p) => match p.try_recv() { - Ok(t) => return Ok(t), - Err(sync::Empty) => return Err(TryRecvError::Empty), - Err(sync::Disconnected) => return Err(TryRecvError::Disconnected), - }, - }; - unsafe { - mem::swap(self.inner_mut(), new_port.inner_mut()); - } - } + self.inner.try_recv() } /// Attempts to wait for a value on this receiver, returning an error if the /// corresponding channel has hung up. /// /// This function will always block the current thread if there is no data - /// available and it's possible for more data to be sent. Once a message is - /// sent to the corresponding [`Sender`] (or [`SyncSender`]), then this - /// receiver will wake up and return that message. + /// available and it's possible for more data to be sent (at least one sender + /// still exists). Once a message is sent to the corresponding [`Sender`] + /// (or [`SyncSender`]), this receiver will wake up and return that + /// message. /// /// If the corresponding [`Sender`] has disconnected, or it disconnects while /// this call is blocking, this call will wake up and return [`Err`] to @@ -1117,10 +816,6 @@ impl Receiver { /// However, since channels are buffered, messages sent before the disconnect /// will still be properly received. /// - /// [`Sender`]: struct.Sender.html - /// [`SyncSender`]: struct.SyncSender.html - /// [`Err`]: ../../../std/result/enum.Result.html#variant.Err - /// /// # Examples /// /// ``` @@ -1162,40 +857,17 @@ impl Receiver { /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn recv(&self) -> Result { - loop { - let new_port = match *unsafe { self.inner() } { - Flavor::Oneshot(ref p) => match p.recv(None) { - Ok(t) => return Ok(t), - Err(oneshot::Disconnected) => return Err(RecvError), - Err(oneshot::Upgraded(rx)) => rx, - Err(oneshot::Empty) => unreachable!(), - }, - Flavor::Stream(ref p) => match p.recv(None) { - Ok(t) => return Ok(t), - Err(stream::Disconnected) => return Err(RecvError), - Err(stream::Upgraded(rx)) => rx, - Err(stream::Empty) => unreachable!(), - }, - Flavor::Shared(ref p) => match p.recv(None) { - Ok(t) => return Ok(t), - Err(shared::Disconnected) => return Err(RecvError), - Err(shared::Empty) => unreachable!(), - }, - Flavor::Sync(ref p) => return p.recv(None).map_err(|_| RecvError), - }; - unsafe { - mem::swap(self.inner_mut(), new_port.inner_mut()); - } - } + self.inner.recv() } /// Attempts to wait for a value on this receiver, returning an error if the /// corresponding channel has hung up, or if it waits more than `timeout`. /// /// This function will always block the current thread if there is no data - /// available and it's possible for more data to be sent. Once a message is - /// sent to the corresponding [`Sender`] (or [`SyncSender`]), then this - /// receiver will wake up and return that message. + /// available and it's possible for more data to be sent (at least one sender + /// still exists). Once a message is sent to the corresponding [`Sender`] + /// (or [`SyncSender`]), this receiver will wake up and return that + /// message. /// /// If the corresponding [`Sender`] has disconnected, or it disconnects while /// this call is blocking, this call will wake up and return [`Err`] to @@ -1203,38 +875,6 @@ impl Receiver { /// However, since channels are buffered, messages sent before the disconnect /// will still be properly received. /// - /// [`Sender`]: struct.Sender.html - /// [`SyncSender`]: struct.SyncSender.html - /// [`Err`]: ../../../std/result/enum.Result.html#variant.Err - /// - /// # Known Issues - /// - /// There is currently a known issue (see [`#39364`]) that causes `recv_timeout` - /// to panic unexpectedly with the following example: - /// - /// ```no_run - /// use std::sync::mpsc::channel; - /// use std::thread; - /// use std::time::Duration; - /// - /// let (tx, rx) = channel::(); - /// - /// thread::spawn(move || { - /// let d = Duration::from_millis(10); - /// loop { - /// println!("recv"); - /// let _r = rx.recv_timeout(d); - /// } - /// }); - /// - /// thread::sleep(Duration::from_millis(100)); - /// let _c1 = tx.clone(); - /// - /// thread::sleep(Duration::from_secs(1)); - /// ``` - /// - /// [`#39364`]: https://github.com/rust-lang/rust/issues/39364 - /// /// # Examples /// /// Successfully receiving value before encountering timeout: @@ -1277,17 +917,7 @@ impl Receiver { /// ``` #[stable(feature = "mpsc_recv_timeout", since = "1.12.0")] pub fn recv_timeout(&self, timeout: Duration) -> Result { - // Do an optimistic try_recv to avoid the performance impact of - // Instant::now() in the full-channel case. - match self.try_recv() { - Ok(result) => Ok(result), - Err(TryRecvError::Disconnected) => Err(RecvTimeoutError::Disconnected), - Err(TryRecvError::Empty) => match Instant::now().checked_add(timeout) { - Some(deadline) => self.recv_deadline(deadline), - // So far in the future that it's practically the same as waiting indefinitely. - None => self.recv().map_err(RecvTimeoutError::from), - }, - } + self.inner.recv_timeout(timeout) } /// Attempts to wait for a value on this receiver, returning an error if the @@ -1304,10 +934,6 @@ impl Receiver { /// However, since channels are buffered, messages sent before the disconnect /// will still be properly received. /// - /// [`Sender`]: struct.Sender.html - /// [`SyncSender`]: struct.SyncSender.html - /// [`Err`]: ../../../std/result/enum.Result.html#variant.Err - /// /// # Examples /// /// Successfully receiving value before reaching deadline: @@ -1352,54 +978,12 @@ impl Receiver { /// ``` #[unstable(feature = "deadline_api", issue = "46316")] pub fn recv_deadline(&self, deadline: Instant) -> Result { - use self::RecvTimeoutError::*; - - loop { - let port_or_empty = match *unsafe { self.inner() } { - Flavor::Oneshot(ref p) => match p.recv(Some(deadline)) { - Ok(t) => return Ok(t), - Err(oneshot::Disconnected) => return Err(Disconnected), - Err(oneshot::Upgraded(rx)) => Some(rx), - Err(oneshot::Empty) => None, - }, - Flavor::Stream(ref p) => match p.recv(Some(deadline)) { - Ok(t) => return Ok(t), - Err(stream::Disconnected) => return Err(Disconnected), - Err(stream::Upgraded(rx)) => Some(rx), - Err(stream::Empty) => None, - }, - Flavor::Shared(ref p) => match p.recv(Some(deadline)) { - Ok(t) => return Ok(t), - Err(shared::Disconnected) => return Err(Disconnected), - Err(shared::Empty) => None, - }, - Flavor::Sync(ref p) => match p.recv(Some(deadline)) { - Ok(t) => return Ok(t), - Err(sync::Disconnected) => return Err(Disconnected), - Err(sync::Empty) => None, - }, - }; - - if let Some(new_port) = port_or_empty { - unsafe { - mem::swap(self.inner_mut(), new_port.inner_mut()); - } - } - - // If we're already passed the deadline, and we're here without - // data, return a timeout, else try again. - if Instant::now() >= deadline { - return Err(Timeout); - } - } + self.inner.recv_deadline(deadline) } /// Returns an iterator that will block waiting for messages, but never /// [`panic!`]. It will return [`None`] when the channel has hung up. /// - /// [`panic!`]: ../../../std/macro.panic.html - /// [`None`]: ../../../std/option/enum.Option.html#variant.None - /// /// # Examples /// /// ```rust @@ -1430,8 +1014,6 @@ impl Receiver { /// channel has hung up. The iterator will never [`panic!`] or block the /// user by waiting for values. /// - /// [`panic!`]: ../../../std/macro.panic.html - /// /// # Examples /// /// ```no_run @@ -1517,27 +1099,20 @@ impl IntoIterator for Receiver { #[stable(feature = "rust1", since = "1.0.0")] impl Drop for Receiver { - fn drop(&mut self) { - match *unsafe { self.inner() } { - Flavor::Oneshot(ref p) => p.drop_port(), - Flavor::Stream(ref p) => p.drop_port(), - Flavor::Shared(ref p) => p.drop_port(), - Flavor::Sync(ref p) => p.drop_port(), - } - } + fn drop(&mut self) {} } #[stable(feature = "mpsc_debug", since = "1.8.0")] impl fmt::Debug for Receiver { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Receiver").finish() + f.debug_struct("Receiver").finish_non_exhaustive() } } #[stable(feature = "rust1", since = "1.0.0")] impl fmt::Debug for SendError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - "SendError(..)".fmt(f) + f.debug_struct("SendError").finish_non_exhaustive() } } @@ -1589,6 +1164,11 @@ impl error::Error for TrySendError { #[stable(feature = "mpsc_error_conversions", since = "1.24.0")] impl From> for TrySendError { + /// Converts a `SendError` into a `TrySendError`. + /// + /// This conversion always returns a `TrySendError::Disconnected` containing the data in the `SendError`. + /// + /// No data is allocated on the heap. fn from(err: SendError) -> TrySendError { match err { SendError(t) => TrySendError::Disconnected(t), @@ -1634,6 +1214,11 @@ impl error::Error for TryRecvError { #[stable(feature = "mpsc_error_conversions", since = "1.24.0")] impl From for TryRecvError { + /// Converts a `RecvError` into a `TryRecvError`. + /// + /// This conversion always returns `TryRecvError::Disconnected`. + /// + /// No data is allocated on the heap. fn from(err: RecvError) -> TryRecvError { match err { RecvError => TryRecvError::Disconnected, @@ -1664,1380 +1249,14 @@ impl error::Error for RecvTimeoutError { #[stable(feature = "mpsc_error_conversions", since = "1.24.0")] impl From for RecvTimeoutError { + /// Converts a `RecvError` into a `RecvTimeoutError`. + /// + /// This conversion always returns `RecvTimeoutError::Disconnected`. + /// + /// No data is allocated on the heap. fn from(err: RecvError) -> RecvTimeoutError { match err { RecvError => RecvTimeoutError::Disconnected, } } } - -#[cfg(all(test, not(target_os = "emscripten")))] -mod tests { - use super::*; - use crate::env; - use crate::thread; - use crate::time::{Duration, Instant}; - - pub fn stress_factor() -> usize { - match env::var("RUST_TEST_STRESS") { - Ok(val) => val.parse().unwrap(), - Err(..) => 1, - } - } - - #[test] - fn smoke() { - let (tx, rx) = channel::(); - tx.send(1).unwrap(); - assert_eq!(rx.recv().unwrap(), 1); - } - - #[test] - fn drop_full() { - let (tx, _rx) = channel::>(); - tx.send(box 1).unwrap(); - } - - #[test] - fn drop_full_shared() { - let (tx, _rx) = channel::>(); - drop(tx.clone()); - drop(tx.clone()); - tx.send(box 1).unwrap(); - } - - #[test] - fn smoke_shared() { - let (tx, rx) = channel::(); - tx.send(1).unwrap(); - assert_eq!(rx.recv().unwrap(), 1); - let tx = tx.clone(); - tx.send(1).unwrap(); - assert_eq!(rx.recv().unwrap(), 1); - } - - #[test] - fn smoke_threads() { - let (tx, rx) = channel::(); - let _t = thread::spawn(move || { - tx.send(1).unwrap(); - }); - assert_eq!(rx.recv().unwrap(), 1); - } - - #[test] - fn smoke_port_gone() { - let (tx, rx) = channel::(); - drop(rx); - assert!(tx.send(1).is_err()); - } - - #[test] - fn smoke_shared_port_gone() { - let (tx, rx) = channel::(); - drop(rx); - assert!(tx.send(1).is_err()) - } - - #[test] - fn smoke_shared_port_gone2() { - let (tx, rx) = channel::(); - drop(rx); - let tx2 = tx.clone(); - drop(tx); - assert!(tx2.send(1).is_err()); - } - - #[test] - fn port_gone_concurrent() { - let (tx, rx) = channel::(); - let _t = thread::spawn(move || { - rx.recv().unwrap(); - }); - while tx.send(1).is_ok() {} - } - - #[test] - fn port_gone_concurrent_shared() { - let (tx, rx) = channel::(); - let tx2 = tx.clone(); - let _t = thread::spawn(move || { - rx.recv().unwrap(); - }); - while tx.send(1).is_ok() && tx2.send(1).is_ok() {} - } - - #[test] - fn smoke_chan_gone() { - let (tx, rx) = channel::(); - drop(tx); - assert!(rx.recv().is_err()); - } - - #[test] - fn smoke_chan_gone_shared() { - let (tx, rx) = channel::<()>(); - let tx2 = tx.clone(); - drop(tx); - drop(tx2); - assert!(rx.recv().is_err()); - } - - #[test] - fn chan_gone_concurrent() { - let (tx, rx) = channel::(); - let _t = thread::spawn(move || { - tx.send(1).unwrap(); - tx.send(1).unwrap(); - }); - while rx.recv().is_ok() {} - } - - #[test] - fn stress() { - let (tx, rx) = channel::(); - let t = thread::spawn(move || { - for _ in 0..10000 { - tx.send(1).unwrap(); - } - }); - for _ in 0..10000 { - assert_eq!(rx.recv().unwrap(), 1); - } - t.join().ok().expect("thread panicked"); - } - - #[test] - fn stress_shared() { - const AMT: u32 = 10000; - const NTHREADS: u32 = 8; - let (tx, rx) = channel::(); - - let t = thread::spawn(move || { - for _ in 0..AMT * NTHREADS { - assert_eq!(rx.recv().unwrap(), 1); - } - match rx.try_recv() { - Ok(..) => panic!(), - _ => {} - } - }); - - for _ in 0..NTHREADS { - let tx = tx.clone(); - thread::spawn(move || { - for _ in 0..AMT { - tx.send(1).unwrap(); - } - }); - } - drop(tx); - t.join().ok().expect("thread panicked"); - } - - #[test] - fn send_from_outside_runtime() { - let (tx1, rx1) = channel::<()>(); - let (tx2, rx2) = channel::(); - let t1 = thread::spawn(move || { - tx1.send(()).unwrap(); - for _ in 0..40 { - assert_eq!(rx2.recv().unwrap(), 1); - } - }); - rx1.recv().unwrap(); - let t2 = thread::spawn(move || { - for _ in 0..40 { - tx2.send(1).unwrap(); - } - }); - t1.join().ok().expect("thread panicked"); - t2.join().ok().expect("thread panicked"); - } - - #[test] - fn recv_from_outside_runtime() { - let (tx, rx) = channel::(); - let t = thread::spawn(move || { - for _ in 0..40 { - assert_eq!(rx.recv().unwrap(), 1); - } - }); - for _ in 0..40 { - tx.send(1).unwrap(); - } - t.join().ok().expect("thread panicked"); - } - - #[test] - fn no_runtime() { - let (tx1, rx1) = channel::(); - let (tx2, rx2) = channel::(); - let t1 = thread::spawn(move || { - assert_eq!(rx1.recv().unwrap(), 1); - tx2.send(2).unwrap(); - }); - let t2 = thread::spawn(move || { - tx1.send(1).unwrap(); - assert_eq!(rx2.recv().unwrap(), 2); - }); - t1.join().ok().expect("thread panicked"); - t2.join().ok().expect("thread panicked"); - } - - #[test] - fn oneshot_single_thread_close_port_first() { - // Simple test of closing without sending - let (_tx, rx) = channel::(); - drop(rx); - } - - #[test] - fn oneshot_single_thread_close_chan_first() { - // Simple test of closing without sending - let (tx, _rx) = channel::(); - drop(tx); - } - - #[test] - fn oneshot_single_thread_send_port_close() { - // Testing that the sender cleans up the payload if receiver is closed - let (tx, rx) = channel::>(); - drop(rx); - assert!(tx.send(box 0).is_err()); - } - - #[test] - fn oneshot_single_thread_recv_chan_close() { - // Receiving on a closed chan will panic - let res = thread::spawn(move || { - let (tx, rx) = channel::(); - drop(tx); - rx.recv().unwrap(); - }) - .join(); - // What is our res? - assert!(res.is_err()); - } - - #[test] - fn oneshot_single_thread_send_then_recv() { - let (tx, rx) = channel::>(); - tx.send(box 10).unwrap(); - assert!(*rx.recv().unwrap() == 10); - } - - #[test] - fn oneshot_single_thread_try_send_open() { - let (tx, rx) = channel::(); - assert!(tx.send(10).is_ok()); - assert!(rx.recv().unwrap() == 10); - } - - #[test] - fn oneshot_single_thread_try_send_closed() { - let (tx, rx) = channel::(); - drop(rx); - assert!(tx.send(10).is_err()); - } - - #[test] - fn oneshot_single_thread_try_recv_open() { - let (tx, rx) = channel::(); - tx.send(10).unwrap(); - assert!(rx.recv() == Ok(10)); - } - - #[test] - fn oneshot_single_thread_try_recv_closed() { - let (tx, rx) = channel::(); - drop(tx); - assert!(rx.recv().is_err()); - } - - #[test] - fn oneshot_single_thread_peek_data() { - let (tx, rx) = channel::(); - assert_eq!(rx.try_recv(), Err(TryRecvError::Empty)); - tx.send(10).unwrap(); - assert_eq!(rx.try_recv(), Ok(10)); - } - - #[test] - fn oneshot_single_thread_peek_close() { - let (tx, rx) = channel::(); - drop(tx); - assert_eq!(rx.try_recv(), Err(TryRecvError::Disconnected)); - assert_eq!(rx.try_recv(), Err(TryRecvError::Disconnected)); - } - - #[test] - fn oneshot_single_thread_peek_open() { - let (_tx, rx) = channel::(); - assert_eq!(rx.try_recv(), Err(TryRecvError::Empty)); - } - - #[test] - fn oneshot_multi_task_recv_then_send() { - let (tx, rx) = channel::>(); - let _t = thread::spawn(move || { - assert!(*rx.recv().unwrap() == 10); - }); - - tx.send(box 10).unwrap(); - } - - #[test] - fn oneshot_multi_task_recv_then_close() { - let (tx, rx) = channel::>(); - let _t = thread::spawn(move || { - drop(tx); - }); - let res = thread::spawn(move || { - assert!(*rx.recv().unwrap() == 10); - }) - .join(); - assert!(res.is_err()); - } - - #[test] - fn oneshot_multi_thread_close_stress() { - for _ in 0..stress_factor() { - let (tx, rx) = channel::(); - let _t = thread::spawn(move || { - drop(rx); - }); - drop(tx); - } - } - - #[test] - fn oneshot_multi_thread_send_close_stress() { - for _ in 0..stress_factor() { - let (tx, rx) = channel::(); - let _t = thread::spawn(move || { - drop(rx); - }); - let _ = thread::spawn(move || { - tx.send(1).unwrap(); - }) - .join(); - } - } - - #[test] - fn oneshot_multi_thread_recv_close_stress() { - for _ in 0..stress_factor() { - let (tx, rx) = channel::(); - thread::spawn(move || { - let res = thread::spawn(move || { - rx.recv().unwrap(); - }) - .join(); - assert!(res.is_err()); - }); - let _t = thread::spawn(move || { - thread::spawn(move || { - drop(tx); - }); - }); - } - } - - #[test] - fn oneshot_multi_thread_send_recv_stress() { - for _ in 0..stress_factor() { - let (tx, rx) = channel::>(); - let _t = thread::spawn(move || { - tx.send(box 10).unwrap(); - }); - assert!(*rx.recv().unwrap() == 10); - } - } - - #[test] - fn stream_send_recv_stress() { - for _ in 0..stress_factor() { - let (tx, rx) = channel(); - - send(tx, 0); - recv(rx, 0); - - fn send(tx: Sender>, i: i32) { - if i == 10 { - return; - } - - thread::spawn(move || { - tx.send(box i).unwrap(); - send(tx, i + 1); - }); - } - - fn recv(rx: Receiver>, i: i32) { - if i == 10 { - return; - } - - thread::spawn(move || { - assert!(*rx.recv().unwrap() == i); - recv(rx, i + 1); - }); - } - } - } - - #[test] - #[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - fn oneshot_single_thread_recv_timeout() { - let (tx, rx) = channel(); - tx.send(()).unwrap(); - assert_eq!(rx.recv_timeout(Duration::from_millis(1)), Ok(())); - assert_eq!(rx.recv_timeout(Duration::from_millis(1)), Err(RecvTimeoutError::Timeout)); - tx.send(()).unwrap(); - assert_eq!(rx.recv_timeout(Duration::from_millis(1)), Ok(())); - } - - #[test] - #[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - fn stress_recv_timeout_two_threads() { - let (tx, rx) = channel(); - let stress = stress_factor() + 100; - let timeout = Duration::from_millis(100); - - thread::spawn(move || { - for i in 0..stress { - if i % 2 == 0 { - thread::sleep(timeout * 2); - } - tx.send(1usize).unwrap(); - } - }); - - let mut recv_count = 0; - loop { - match rx.recv_timeout(timeout) { - Ok(n) => { - assert_eq!(n, 1usize); - recv_count += 1; - } - Err(RecvTimeoutError::Timeout) => continue, - Err(RecvTimeoutError::Disconnected) => break, - } - } - - assert_eq!(recv_count, stress); - } - - #[test] - #[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - fn recv_timeout_upgrade() { - let (tx, rx) = channel::<()>(); - let timeout = Duration::from_millis(1); - let _tx_clone = tx.clone(); - - let start = Instant::now(); - assert_eq!(rx.recv_timeout(timeout), Err(RecvTimeoutError::Timeout)); - assert!(Instant::now() >= start + timeout); - } - - #[test] - #[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - fn stress_recv_timeout_shared() { - let (tx, rx) = channel(); - let stress = stress_factor() + 100; - - for i in 0..stress { - let tx = tx.clone(); - thread::spawn(move || { - thread::sleep(Duration::from_millis(i as u64 * 10)); - tx.send(1usize).unwrap(); - }); - } - - drop(tx); - - let mut recv_count = 0; - loop { - match rx.recv_timeout(Duration::from_millis(10)) { - Ok(n) => { - assert_eq!(n, 1usize); - recv_count += 1; - } - Err(RecvTimeoutError::Timeout) => continue, - Err(RecvTimeoutError::Disconnected) => break, - } - } - - assert_eq!(recv_count, stress); - } - - #[test] - #[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - fn very_long_recv_timeout_wont_panic() { - let (tx, rx) = channel::<()>(); - let join_handle = - thread::spawn(move || rx.recv_timeout(Duration::from_secs(u64::max_value()))); - thread::sleep(Duration::from_secs(1)); - assert!(tx.send(()).is_ok()); - assert_eq!(join_handle.join().unwrap(), Ok(())); - } - - #[test] - fn recv_a_lot() { - // Regression test that we don't run out of stack in scheduler context - let (tx, rx) = channel(); - for _ in 0..10000 { - tx.send(()).unwrap(); - } - for _ in 0..10000 { - rx.recv().unwrap(); - } - } - - #[test] - #[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - fn shared_recv_timeout() { - let (tx, rx) = channel(); - let total = 5; - for _ in 0..total { - let tx = tx.clone(); - thread::spawn(move || { - tx.send(()).unwrap(); - }); - } - - for _ in 0..total { - rx.recv().unwrap(); - } - - assert_eq!(rx.recv_timeout(Duration::from_millis(1)), Err(RecvTimeoutError::Timeout)); - tx.send(()).unwrap(); - assert_eq!(rx.recv_timeout(Duration::from_millis(1)), Ok(())); - } - - #[test] - fn shared_chan_stress() { - let (tx, rx) = channel(); - let total = stress_factor() + 100; - for _ in 0..total { - let tx = tx.clone(); - thread::spawn(move || { - tx.send(()).unwrap(); - }); - } - - for _ in 0..total { - rx.recv().unwrap(); - } - } - - #[test] - fn test_nested_recv_iter() { - let (tx, rx) = channel::(); - let (total_tx, total_rx) = channel::(); - - let _t = thread::spawn(move || { - let mut acc = 0; - for x in rx.iter() { - acc += x; - } - total_tx.send(acc).unwrap(); - }); - - tx.send(3).unwrap(); - tx.send(1).unwrap(); - tx.send(2).unwrap(); - drop(tx); - assert_eq!(total_rx.recv().unwrap(), 6); - } - - #[test] - fn test_recv_iter_break() { - let (tx, rx) = channel::(); - let (count_tx, count_rx) = channel(); - - let _t = thread::spawn(move || { - let mut count = 0; - for x in rx.iter() { - if count >= 3 { - break; - } else { - count += x; - } - } - count_tx.send(count).unwrap(); - }); - - tx.send(2).unwrap(); - tx.send(2).unwrap(); - tx.send(2).unwrap(); - let _ = tx.send(2); - drop(tx); - assert_eq!(count_rx.recv().unwrap(), 4); - } - - #[test] - fn test_recv_try_iter() { - let (request_tx, request_rx) = channel(); - let (response_tx, response_rx) = channel(); - - // Request `x`s until we have `6`. - let t = thread::spawn(move || { - let mut count = 0; - loop { - for x in response_rx.try_iter() { - count += x; - if count == 6 { - return count; - } - } - request_tx.send(()).unwrap(); - } - }); - - for _ in request_rx.iter() { - if response_tx.send(2).is_err() { - break; - } - } - - assert_eq!(t.join().unwrap(), 6); - } - - #[test] - fn test_recv_into_iter_owned() { - let mut iter = { - let (tx, rx) = channel::(); - tx.send(1).unwrap(); - tx.send(2).unwrap(); - - rx.into_iter() - }; - assert_eq!(iter.next().unwrap(), 1); - assert_eq!(iter.next().unwrap(), 2); - assert_eq!(iter.next().is_none(), true); - } - - #[test] - fn test_recv_into_iter_borrowed() { - let (tx, rx) = channel::(); - tx.send(1).unwrap(); - tx.send(2).unwrap(); - drop(tx); - let mut iter = (&rx).into_iter(); - assert_eq!(iter.next().unwrap(), 1); - assert_eq!(iter.next().unwrap(), 2); - assert_eq!(iter.next().is_none(), true); - } - - #[test] - fn try_recv_states() { - let (tx1, rx1) = channel::(); - let (tx2, rx2) = channel::<()>(); - let (tx3, rx3) = channel::<()>(); - let _t = thread::spawn(move || { - rx2.recv().unwrap(); - tx1.send(1).unwrap(); - tx3.send(()).unwrap(); - rx2.recv().unwrap(); - drop(tx1); - tx3.send(()).unwrap(); - }); - - assert_eq!(rx1.try_recv(), Err(TryRecvError::Empty)); - tx2.send(()).unwrap(); - rx3.recv().unwrap(); - assert_eq!(rx1.try_recv(), Ok(1)); - assert_eq!(rx1.try_recv(), Err(TryRecvError::Empty)); - tx2.send(()).unwrap(); - rx3.recv().unwrap(); - assert_eq!(rx1.try_recv(), Err(TryRecvError::Disconnected)); - } - - // This bug used to end up in a livelock inside of the Receiver destructor - // because the internal state of the Shared packet was corrupted - #[test] - fn destroy_upgraded_shared_port_when_sender_still_active() { - let (tx, rx) = channel(); - let (tx2, rx2) = channel(); - let _t = thread::spawn(move || { - rx.recv().unwrap(); // wait on a oneshot - drop(rx); // destroy a shared - tx2.send(()).unwrap(); - }); - // make sure the other thread has gone to sleep - for _ in 0..5000 { - thread::yield_now(); - } - - // upgrade to a shared chan and send a message - let t = tx.clone(); - drop(tx); - t.send(()).unwrap(); - - // wait for the child thread to exit before we exit - rx2.recv().unwrap(); - } - - #[test] - fn issue_32114() { - let (tx, _) = channel(); - let _ = tx.send(123); - assert_eq!(tx.send(123), Err(SendError(123))); - } -} - -#[cfg(all(test, not(target_os = "emscripten")))] -mod sync_tests { - use super::*; - use crate::env; - use crate::thread; - use crate::time::Duration; - - pub fn stress_factor() -> usize { - match env::var("RUST_TEST_STRESS") { - Ok(val) => val.parse().unwrap(), - Err(..) => 1, - } - } - - #[test] - fn smoke() { - let (tx, rx) = sync_channel::(1); - tx.send(1).unwrap(); - assert_eq!(rx.recv().unwrap(), 1); - } - - #[test] - fn drop_full() { - let (tx, _rx) = sync_channel::>(1); - tx.send(box 1).unwrap(); - } - - #[test] - fn smoke_shared() { - let (tx, rx) = sync_channel::(1); - tx.send(1).unwrap(); - assert_eq!(rx.recv().unwrap(), 1); - let tx = tx.clone(); - tx.send(1).unwrap(); - assert_eq!(rx.recv().unwrap(), 1); - } - - #[test] - #[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - fn recv_timeout() { - let (tx, rx) = sync_channel::(1); - assert_eq!(rx.recv_timeout(Duration::from_millis(1)), Err(RecvTimeoutError::Timeout)); - tx.send(1).unwrap(); - assert_eq!(rx.recv_timeout(Duration::from_millis(1)), Ok(1)); - } - - #[test] - fn smoke_threads() { - let (tx, rx) = sync_channel::(0); - let _t = thread::spawn(move || { - tx.send(1).unwrap(); - }); - assert_eq!(rx.recv().unwrap(), 1); - } - - #[test] - fn smoke_port_gone() { - let (tx, rx) = sync_channel::(0); - drop(rx); - assert!(tx.send(1).is_err()); - } - - #[test] - fn smoke_shared_port_gone2() { - let (tx, rx) = sync_channel::(0); - drop(rx); - let tx2 = tx.clone(); - drop(tx); - assert!(tx2.send(1).is_err()); - } - - #[test] - fn port_gone_concurrent() { - let (tx, rx) = sync_channel::(0); - let _t = thread::spawn(move || { - rx.recv().unwrap(); - }); - while tx.send(1).is_ok() {} - } - - #[test] - fn port_gone_concurrent_shared() { - let (tx, rx) = sync_channel::(0); - let tx2 = tx.clone(); - let _t = thread::spawn(move || { - rx.recv().unwrap(); - }); - while tx.send(1).is_ok() && tx2.send(1).is_ok() {} - } - - #[test] - fn smoke_chan_gone() { - let (tx, rx) = sync_channel::(0); - drop(tx); - assert!(rx.recv().is_err()); - } - - #[test] - fn smoke_chan_gone_shared() { - let (tx, rx) = sync_channel::<()>(0); - let tx2 = tx.clone(); - drop(tx); - drop(tx2); - assert!(rx.recv().is_err()); - } - - #[test] - fn chan_gone_concurrent() { - let (tx, rx) = sync_channel::(0); - thread::spawn(move || { - tx.send(1).unwrap(); - tx.send(1).unwrap(); - }); - while rx.recv().is_ok() {} - } - - #[test] - fn stress() { - let (tx, rx) = sync_channel::(0); - thread::spawn(move || { - for _ in 0..10000 { - tx.send(1).unwrap(); - } - }); - for _ in 0..10000 { - assert_eq!(rx.recv().unwrap(), 1); - } - } - - #[test] - #[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - fn stress_recv_timeout_two_threads() { - let (tx, rx) = sync_channel::(0); - - thread::spawn(move || { - for _ in 0..10000 { - tx.send(1).unwrap(); - } - }); - - let mut recv_count = 0; - loop { - match rx.recv_timeout(Duration::from_millis(1)) { - Ok(v) => { - assert_eq!(v, 1); - recv_count += 1; - } - Err(RecvTimeoutError::Timeout) => continue, - Err(RecvTimeoutError::Disconnected) => break, - } - } - - assert_eq!(recv_count, 10000); - } - - #[test] - #[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - fn stress_recv_timeout_shared() { - const AMT: u32 = 1000; - const NTHREADS: u32 = 8; - let (tx, rx) = sync_channel::(0); - let (dtx, drx) = sync_channel::<()>(0); - - thread::spawn(move || { - let mut recv_count = 0; - loop { - match rx.recv_timeout(Duration::from_millis(10)) { - Ok(v) => { - assert_eq!(v, 1); - recv_count += 1; - } - Err(RecvTimeoutError::Timeout) => continue, - Err(RecvTimeoutError::Disconnected) => break, - } - } - - assert_eq!(recv_count, AMT * NTHREADS); - assert!(rx.try_recv().is_err()); - - dtx.send(()).unwrap(); - }); - - for _ in 0..NTHREADS { - let tx = tx.clone(); - thread::spawn(move || { - for _ in 0..AMT { - tx.send(1).unwrap(); - } - }); - } - - drop(tx); - - drx.recv().unwrap(); - } - - #[test] - fn stress_shared() { - const AMT: u32 = 1000; - const NTHREADS: u32 = 8; - let (tx, rx) = sync_channel::(0); - let (dtx, drx) = sync_channel::<()>(0); - - thread::spawn(move || { - for _ in 0..AMT * NTHREADS { - assert_eq!(rx.recv().unwrap(), 1); - } - match rx.try_recv() { - Ok(..) => panic!(), - _ => {} - } - dtx.send(()).unwrap(); - }); - - for _ in 0..NTHREADS { - let tx = tx.clone(); - thread::spawn(move || { - for _ in 0..AMT { - tx.send(1).unwrap(); - } - }); - } - drop(tx); - drx.recv().unwrap(); - } - - #[test] - fn oneshot_single_thread_close_port_first() { - // Simple test of closing without sending - let (_tx, rx) = sync_channel::(0); - drop(rx); - } - - #[test] - fn oneshot_single_thread_close_chan_first() { - // Simple test of closing without sending - let (tx, _rx) = sync_channel::(0); - drop(tx); - } - - #[test] - fn oneshot_single_thread_send_port_close() { - // Testing that the sender cleans up the payload if receiver is closed - let (tx, rx) = sync_channel::>(0); - drop(rx); - assert!(tx.send(box 0).is_err()); - } - - #[test] - fn oneshot_single_thread_recv_chan_close() { - // Receiving on a closed chan will panic - let res = thread::spawn(move || { - let (tx, rx) = sync_channel::(0); - drop(tx); - rx.recv().unwrap(); - }) - .join(); - // What is our res? - assert!(res.is_err()); - } - - #[test] - fn oneshot_single_thread_send_then_recv() { - let (tx, rx) = sync_channel::>(1); - tx.send(box 10).unwrap(); - assert!(*rx.recv().unwrap() == 10); - } - - #[test] - fn oneshot_single_thread_try_send_open() { - let (tx, rx) = sync_channel::(1); - assert_eq!(tx.try_send(10), Ok(())); - assert!(rx.recv().unwrap() == 10); - } - - #[test] - fn oneshot_single_thread_try_send_closed() { - let (tx, rx) = sync_channel::(0); - drop(rx); - assert_eq!(tx.try_send(10), Err(TrySendError::Disconnected(10))); - } - - #[test] - fn oneshot_single_thread_try_send_closed2() { - let (tx, _rx) = sync_channel::(0); - assert_eq!(tx.try_send(10), Err(TrySendError::Full(10))); - } - - #[test] - fn oneshot_single_thread_try_recv_open() { - let (tx, rx) = sync_channel::(1); - tx.send(10).unwrap(); - assert!(rx.recv() == Ok(10)); - } - - #[test] - fn oneshot_single_thread_try_recv_closed() { - let (tx, rx) = sync_channel::(0); - drop(tx); - assert!(rx.recv().is_err()); - } - - #[test] - fn oneshot_single_thread_try_recv_closed_with_data() { - let (tx, rx) = sync_channel::(1); - tx.send(10).unwrap(); - drop(tx); - assert_eq!(rx.try_recv(), Ok(10)); - assert_eq!(rx.try_recv(), Err(TryRecvError::Disconnected)); - } - - #[test] - fn oneshot_single_thread_peek_data() { - let (tx, rx) = sync_channel::(1); - assert_eq!(rx.try_recv(), Err(TryRecvError::Empty)); - tx.send(10).unwrap(); - assert_eq!(rx.try_recv(), Ok(10)); - } - - #[test] - fn oneshot_single_thread_peek_close() { - let (tx, rx) = sync_channel::(0); - drop(tx); - assert_eq!(rx.try_recv(), Err(TryRecvError::Disconnected)); - assert_eq!(rx.try_recv(), Err(TryRecvError::Disconnected)); - } - - #[test] - fn oneshot_single_thread_peek_open() { - let (_tx, rx) = sync_channel::(0); - assert_eq!(rx.try_recv(), Err(TryRecvError::Empty)); - } - - #[test] - fn oneshot_multi_task_recv_then_send() { - let (tx, rx) = sync_channel::>(0); - let _t = thread::spawn(move || { - assert!(*rx.recv().unwrap() == 10); - }); - - tx.send(box 10).unwrap(); - } - - #[test] - fn oneshot_multi_task_recv_then_close() { - let (tx, rx) = sync_channel::>(0); - let _t = thread::spawn(move || { - drop(tx); - }); - let res = thread::spawn(move || { - assert!(*rx.recv().unwrap() == 10); - }) - .join(); - assert!(res.is_err()); - } - - #[test] - fn oneshot_multi_thread_close_stress() { - for _ in 0..stress_factor() { - let (tx, rx) = sync_channel::(0); - let _t = thread::spawn(move || { - drop(rx); - }); - drop(tx); - } - } - - #[test] - fn oneshot_multi_thread_send_close_stress() { - for _ in 0..stress_factor() { - let (tx, rx) = sync_channel::(0); - let _t = thread::spawn(move || { - drop(rx); - }); - let _ = thread::spawn(move || { - tx.send(1).unwrap(); - }) - .join(); - } - } - - #[test] - fn oneshot_multi_thread_recv_close_stress() { - for _ in 0..stress_factor() { - let (tx, rx) = sync_channel::(0); - let _t = thread::spawn(move || { - let res = thread::spawn(move || { - rx.recv().unwrap(); - }) - .join(); - assert!(res.is_err()); - }); - let _t = thread::spawn(move || { - thread::spawn(move || { - drop(tx); - }); - }); - } - } - - #[test] - fn oneshot_multi_thread_send_recv_stress() { - for _ in 0..stress_factor() { - let (tx, rx) = sync_channel::>(0); - let _t = thread::spawn(move || { - tx.send(box 10).unwrap(); - }); - assert!(*rx.recv().unwrap() == 10); - } - } - - #[test] - fn stream_send_recv_stress() { - for _ in 0..stress_factor() { - let (tx, rx) = sync_channel::>(0); - - send(tx, 0); - recv(rx, 0); - - fn send(tx: SyncSender>, i: i32) { - if i == 10 { - return; - } - - thread::spawn(move || { - tx.send(box i).unwrap(); - send(tx, i + 1); - }); - } - - fn recv(rx: Receiver>, i: i32) { - if i == 10 { - return; - } - - thread::spawn(move || { - assert!(*rx.recv().unwrap() == i); - recv(rx, i + 1); - }); - } - } - } - - #[test] - fn recv_a_lot() { - // Regression test that we don't run out of stack in scheduler context - let (tx, rx) = sync_channel(10000); - for _ in 0..10000 { - tx.send(()).unwrap(); - } - for _ in 0..10000 { - rx.recv().unwrap(); - } - } - - #[test] - fn shared_chan_stress() { - let (tx, rx) = sync_channel(0); - let total = stress_factor() + 100; - for _ in 0..total { - let tx = tx.clone(); - thread::spawn(move || { - tx.send(()).unwrap(); - }); - } - - for _ in 0..total { - rx.recv().unwrap(); - } - } - - #[test] - fn test_nested_recv_iter() { - let (tx, rx) = sync_channel::(0); - let (total_tx, total_rx) = sync_channel::(0); - - let _t = thread::spawn(move || { - let mut acc = 0; - for x in rx.iter() { - acc += x; - } - total_tx.send(acc).unwrap(); - }); - - tx.send(3).unwrap(); - tx.send(1).unwrap(); - tx.send(2).unwrap(); - drop(tx); - assert_eq!(total_rx.recv().unwrap(), 6); - } - - #[test] - fn test_recv_iter_break() { - let (tx, rx) = sync_channel::(0); - let (count_tx, count_rx) = sync_channel(0); - - let _t = thread::spawn(move || { - let mut count = 0; - for x in rx.iter() { - if count >= 3 { - break; - } else { - count += x; - } - } - count_tx.send(count).unwrap(); - }); - - tx.send(2).unwrap(); - tx.send(2).unwrap(); - tx.send(2).unwrap(); - let _ = tx.try_send(2); - drop(tx); - assert_eq!(count_rx.recv().unwrap(), 4); - } - - #[test] - fn try_recv_states() { - let (tx1, rx1) = sync_channel::(1); - let (tx2, rx2) = sync_channel::<()>(1); - let (tx3, rx3) = sync_channel::<()>(1); - let _t = thread::spawn(move || { - rx2.recv().unwrap(); - tx1.send(1).unwrap(); - tx3.send(()).unwrap(); - rx2.recv().unwrap(); - drop(tx1); - tx3.send(()).unwrap(); - }); - - assert_eq!(rx1.try_recv(), Err(TryRecvError::Empty)); - tx2.send(()).unwrap(); - rx3.recv().unwrap(); - assert_eq!(rx1.try_recv(), Ok(1)); - assert_eq!(rx1.try_recv(), Err(TryRecvError::Empty)); - tx2.send(()).unwrap(); - rx3.recv().unwrap(); - assert_eq!(rx1.try_recv(), Err(TryRecvError::Disconnected)); - } - - // This bug used to end up in a livelock inside of the Receiver destructor - // because the internal state of the Shared packet was corrupted - #[test] - fn destroy_upgraded_shared_port_when_sender_still_active() { - let (tx, rx) = sync_channel::<()>(0); - let (tx2, rx2) = sync_channel::<()>(0); - let _t = thread::spawn(move || { - rx.recv().unwrap(); // wait on a oneshot - drop(rx); // destroy a shared - tx2.send(()).unwrap(); - }); - // make sure the other thread has gone to sleep - for _ in 0..5000 { - thread::yield_now(); - } - - // upgrade to a shared chan and send a message - let t = tx.clone(); - drop(tx); - t.send(()).unwrap(); - - // wait for the child thread to exit before we exit - rx2.recv().unwrap(); - } - - #[test] - fn send1() { - let (tx, rx) = sync_channel::(0); - let _t = thread::spawn(move || { - rx.recv().unwrap(); - }); - assert_eq!(tx.send(1), Ok(())); - } - - #[test] - fn send2() { - let (tx, rx) = sync_channel::(0); - let _t = thread::spawn(move || { - drop(rx); - }); - assert!(tx.send(1).is_err()); - } - - #[test] - fn send3() { - let (tx, rx) = sync_channel::(1); - assert_eq!(tx.send(1), Ok(())); - let _t = thread::spawn(move || { - drop(rx); - }); - assert!(tx.send(1).is_err()); - } - - #[test] - fn send4() { - let (tx, rx) = sync_channel::(0); - let tx2 = tx.clone(); - let (done, donerx) = channel(); - let done2 = done.clone(); - let _t = thread::spawn(move || { - assert!(tx.send(1).is_err()); - done.send(()).unwrap(); - }); - let _t = thread::spawn(move || { - assert!(tx2.send(2).is_err()); - done2.send(()).unwrap(); - }); - drop(rx); - donerx.recv().unwrap(); - donerx.recv().unwrap(); - } - - #[test] - fn try_send1() { - let (tx, _rx) = sync_channel::(0); - assert_eq!(tx.try_send(1), Err(TrySendError::Full(1))); - } - - #[test] - fn try_send2() { - let (tx, _rx) = sync_channel::(1); - assert_eq!(tx.try_send(1), Ok(())); - assert_eq!(tx.try_send(1), Err(TrySendError::Full(1))); - } - - #[test] - fn try_send3() { - let (tx, rx) = sync_channel::(1); - assert_eq!(tx.try_send(1), Ok(())); - drop(rx); - assert_eq!(tx.try_send(1), Err(TrySendError::Disconnected(1))); - } - - #[test] - fn issue_15761() { - fn repro() { - let (tx1, rx1) = sync_channel::<()>(3); - let (tx2, rx2) = sync_channel::<()>(3); - - let _t = thread::spawn(move || { - rx1.recv().unwrap(); - tx2.try_send(()).unwrap(); - }); - - tx1.try_send(()).unwrap(); - rx2.recv().unwrap(); - } - - for _ in 0..100 { - repro() - } - } -} diff --git a/crux-mir/lib/std/src/sync/mpsc/mpsc_queue.rs b/crux-mir/lib/std/src/sync/mpsc/mpsc_queue.rs deleted file mode 100644 index 6e7a7be44..000000000 --- a/crux-mir/lib/std/src/sync/mpsc/mpsc_queue.rs +++ /dev/null @@ -1,165 +0,0 @@ -//! A mostly lock-free multi-producer, single consumer queue. -//! -//! This module contains an implementation of a concurrent MPSC queue. This -//! queue can be used to share data between threads, and is also used as the -//! building block of channels in rust. -//! -//! Note that the current implementation of this queue has a caveat of the `pop` -//! method, and see the method for more information about it. Due to this -//! caveat, this queue may not be appropriate for all use-cases. - -// http://www.1024cores.net/home/lock-free-algorithms -// /queues/non-intrusive-mpsc-node-based-queue - -pub use self::PopResult::*; - -use core::cell::UnsafeCell; -use core::ptr; - -use crate::boxed::Box; -use crate::sync::atomic::{AtomicPtr, Ordering}; - -/// A result of the `pop` function. -pub enum PopResult { - /// Some data has been popped - Data(T), - /// The queue is empty - Empty, - /// The queue is in an inconsistent state. Popping data should succeed, but - /// some pushers have yet to make enough progress in order allow a pop to - /// succeed. It is recommended that a pop() occur "in the near future" in - /// order to see if the sender has made progress or not - Inconsistent, -} - -struct Node { - next: AtomicPtr>, - value: Option, -} - -/// The multi-producer single-consumer structure. This is not cloneable, but it -/// may be safely shared so long as it is guaranteed that there is only one -/// popper at a time (many pushers are allowed). -pub struct Queue { - head: AtomicPtr>, - tail: UnsafeCell<*mut Node>, -} - -unsafe impl Send for Queue {} -unsafe impl Sync for Queue {} - -impl Node { - unsafe fn new(v: Option) -> *mut Node { - Box::into_raw(box Node { next: AtomicPtr::new(ptr::null_mut()), value: v }) - } -} - -impl Queue { - /// Creates a new queue that is safe to share among multiple producers and - /// one consumer. - pub fn new() -> Queue { - let stub = unsafe { Node::new(None) }; - Queue { head: AtomicPtr::new(stub), tail: UnsafeCell::new(stub) } - } - - /// Pushes a new value onto this queue. - pub fn push(&self, t: T) { - unsafe { - let n = Node::new(Some(t)); - let prev = self.head.swap(n, Ordering::AcqRel); - (*prev).next.store(n, Ordering::Release); - } - } - - /// Pops some data from this queue. - /// - /// Note that the current implementation means that this function cannot - /// return `Option`. It is possible for this queue to be in an - /// inconsistent state where many pushes have succeeded and completely - /// finished, but pops cannot return `Some(t)`. This inconsistent state - /// happens when a pusher is pre-empted at an inopportune moment. - /// - /// This inconsistent state means that this queue does indeed have data, but - /// it does not currently have access to it at this time. - pub fn pop(&self) -> PopResult { - unsafe { - let tail = *self.tail.get(); - let next = (*tail).next.load(Ordering::Acquire); - - if !next.is_null() { - *self.tail.get() = next; - assert!((*tail).value.is_none()); - assert!((*next).value.is_some()); - let ret = (*next).value.take().unwrap(); - let _: Box> = Box::from_raw(tail); - return Data(ret); - } - - if self.head.load(Ordering::Acquire) == tail { Empty } else { Inconsistent } - } - } -} - -impl Drop for Queue { - fn drop(&mut self) { - unsafe { - let mut cur = *self.tail.get(); - while !cur.is_null() { - let next = (*cur).next.load(Ordering::Relaxed); - let _: Box> = Box::from_raw(cur); - cur = next; - } - } - } -} - -#[cfg(all(test, not(target_os = "emscripten")))] -mod tests { - use super::{Data, Empty, Inconsistent, Queue}; - use crate::sync::mpsc::channel; - use crate::sync::Arc; - use crate::thread; - - #[test] - fn test_full() { - let q: Queue> = Queue::new(); - q.push(box 1); - q.push(box 2); - } - - #[test] - fn test() { - let nthreads = 8; - let nmsgs = 1000; - let q = Queue::new(); - match q.pop() { - Empty => {} - Inconsistent | Data(..) => panic!(), - } - let (tx, rx) = channel(); - let q = Arc::new(q); - - for _ in 0..nthreads { - let tx = tx.clone(); - let q = q.clone(); - thread::spawn(move || { - for i in 0..nmsgs { - q.push(i); - } - tx.send(()).unwrap(); - }); - } - - let mut i = 0; - while i < nthreads * nmsgs { - match q.pop() { - Empty | Inconsistent => {} - Data(_) => i += 1, - } - } - drop(tx); - for _ in 0..nthreads { - rx.recv().unwrap(); - } - } -} diff --git a/crux-mir/lib/std/src/sync/mpsc/oneshot.rs b/crux-mir/lib/std/src/sync/mpsc/oneshot.rs deleted file mode 100644 index 5b41525e0..000000000 --- a/crux-mir/lib/std/src/sync/mpsc/oneshot.rs +++ /dev/null @@ -1,307 +0,0 @@ -/// Oneshot channels/ports -/// -/// This is the initial flavor of channels/ports used for comm module. This is -/// an optimization for the one-use case of a channel. The major optimization of -/// this type is to have one and exactly one allocation when the chan/port pair -/// is created. -/// -/// Another possible optimization would be to not use an Arc box because -/// in theory we know when the shared packet can be deallocated (no real need -/// for the atomic reference counting), but I was having trouble how to destroy -/// the data early in a drop of a Port. -/// -/// # Implementation -/// -/// Oneshots are implemented around one atomic usize variable. This variable -/// indicates both the state of the port/chan but also contains any threads -/// blocked on the port. All atomic operations happen on this one word. -/// -/// In order to upgrade a oneshot channel, an upgrade is considered a disconnect -/// on behalf of the channel side of things (it can be mentally thought of as -/// consuming the port). This upgrade is then also stored in the shared packet. -/// The one caveat to consider is that when a port sees a disconnected channel -/// it must check for data because there is no "data plus upgrade" state. -pub use self::Failure::*; -use self::MyUpgrade::*; -pub use self::UpgradeResult::*; - -use crate::cell::UnsafeCell; -use crate::ptr; -use crate::sync::atomic::{AtomicUsize, Ordering}; -use crate::sync::mpsc::blocking::{self, SignalToken}; -use crate::sync::mpsc::Receiver; -use crate::time::Instant; - -// Various states you can find a port in. -const EMPTY: usize = 0; // initial state: no data, no blocked receiver -const DATA: usize = 1; // data ready for receiver to take -const DISCONNECTED: usize = 2; // channel is disconnected OR upgraded -// Any other value represents a pointer to a SignalToken value. The -// protocol ensures that when the state moves *to* a pointer, -// ownership of the token is given to the packet, and when the state -// moves *from* a pointer, ownership of the token is transferred to -// whoever changed the state. - -pub struct Packet { - // Internal state of the chan/port pair (stores the blocked thread as well) - state: AtomicUsize, - // One-shot data slot location - data: UnsafeCell>, - // when used for the second time, a oneshot channel must be upgraded, and - // this contains the slot for the upgrade - upgrade: UnsafeCell>, -} - -pub enum Failure { - Empty, - Disconnected, - Upgraded(Receiver), -} - -pub enum UpgradeResult { - UpSuccess, - UpDisconnected, - UpWoke(SignalToken), -} - -enum MyUpgrade { - NothingSent, - SendUsed, - GoUp(Receiver), -} - -impl Packet { - pub fn new() -> Packet { - Packet { - data: UnsafeCell::new(None), - upgrade: UnsafeCell::new(NothingSent), - state: AtomicUsize::new(EMPTY), - } - } - - pub fn send(&self, t: T) -> Result<(), T> { - unsafe { - // Sanity check - match *self.upgrade.get() { - NothingSent => {} - _ => panic!("sending on a oneshot that's already sent on "), - } - assert!((*self.data.get()).is_none()); - ptr::write(self.data.get(), Some(t)); - ptr::write(self.upgrade.get(), SendUsed); - - match self.state.swap(DATA, Ordering::SeqCst) { - // Sent the data, no one was waiting - EMPTY => Ok(()), - - // Couldn't send the data, the port hung up first. Return the data - // back up the stack. - DISCONNECTED => { - self.state.swap(DISCONNECTED, Ordering::SeqCst); - ptr::write(self.upgrade.get(), NothingSent); - Err((&mut *self.data.get()).take().unwrap()) - } - - // Not possible, these are one-use channels - DATA => unreachable!(), - - // There is a thread waiting on the other end. We leave the 'DATA' - // state inside so it'll pick it up on the other end. - ptr => { - SignalToken::cast_from_usize(ptr).signal(); - Ok(()) - } - } - } - } - - // Just tests whether this channel has been sent on or not, this is only - // safe to use from the sender. - pub fn sent(&self) -> bool { - unsafe { !matches!(*self.upgrade.get(), NothingSent) } - } - - pub fn recv(&self, deadline: Option) -> Result> { - // Attempt to not block the thread (it's a little expensive). If it looks - // like we're not empty, then immediately go through to `try_recv`. - if self.state.load(Ordering::SeqCst) == EMPTY { - let (wait_token, signal_token) = blocking::tokens(); - let ptr = unsafe { signal_token.cast_to_usize() }; - - // race with senders to enter the blocking state - if self.state.compare_and_swap(EMPTY, ptr, Ordering::SeqCst) == EMPTY { - if let Some(deadline) = deadline { - let timed_out = !wait_token.wait_max_until(deadline); - // Try to reset the state - if timed_out { - self.abort_selection().map_err(Upgraded)?; - } - } else { - wait_token.wait(); - debug_assert!(self.state.load(Ordering::SeqCst) != EMPTY); - } - } else { - // drop the signal token, since we never blocked - drop(unsafe { SignalToken::cast_from_usize(ptr) }); - } - } - - self.try_recv() - } - - pub fn try_recv(&self) -> Result> { - unsafe { - match self.state.load(Ordering::SeqCst) { - EMPTY => Err(Empty), - - // We saw some data on the channel, but the channel can be used - // again to send us an upgrade. As a result, we need to re-insert - // into the channel that there's no data available (otherwise we'll - // just see DATA next time). This is done as a cmpxchg because if - // the state changes under our feet we'd rather just see that state - // change. - DATA => { - self.state.compare_and_swap(DATA, EMPTY, Ordering::SeqCst); - match (&mut *self.data.get()).take() { - Some(data) => Ok(data), - None => unreachable!(), - } - } - - // There's no guarantee that we receive before an upgrade happens, - // and an upgrade flags the channel as disconnected, so when we see - // this we first need to check if there's data available and *then* - // we go through and process the upgrade. - DISCONNECTED => match (&mut *self.data.get()).take() { - Some(data) => Ok(data), - None => match ptr::replace(self.upgrade.get(), SendUsed) { - SendUsed | NothingSent => Err(Disconnected), - GoUp(upgrade) => Err(Upgraded(upgrade)), - }, - }, - - // We are the sole receiver; there cannot be a blocking - // receiver already. - _ => unreachable!(), - } - } - } - - // Returns whether the upgrade was completed. If the upgrade wasn't - // completed, then the port couldn't get sent to the other half (it will - // never receive it). - pub fn upgrade(&self, up: Receiver) -> UpgradeResult { - unsafe { - let prev = match *self.upgrade.get() { - NothingSent => NothingSent, - SendUsed => SendUsed, - _ => panic!("upgrading again"), - }; - ptr::write(self.upgrade.get(), GoUp(up)); - - match self.state.swap(DISCONNECTED, Ordering::SeqCst) { - // If the channel is empty or has data on it, then we're good to go. - // Senders will check the data before the upgrade (in case we - // plastered over the DATA state). - DATA | EMPTY => UpSuccess, - - // If the other end is already disconnected, then we failed the - // upgrade. Be sure to trash the port we were given. - DISCONNECTED => { - ptr::replace(self.upgrade.get(), prev); - UpDisconnected - } - - // If someone's waiting, we gotta wake them up - ptr => UpWoke(SignalToken::cast_from_usize(ptr)), - } - } - } - - pub fn drop_chan(&self) { - match self.state.swap(DISCONNECTED, Ordering::SeqCst) { - DATA | DISCONNECTED | EMPTY => {} - - // If someone's waiting, we gotta wake them up - ptr => unsafe { - SignalToken::cast_from_usize(ptr).signal(); - }, - } - } - - pub fn drop_port(&self) { - match self.state.swap(DISCONNECTED, Ordering::SeqCst) { - // An empty channel has nothing to do, and a remotely disconnected - // channel also has nothing to do b/c we're about to run the drop - // glue - DISCONNECTED | EMPTY => {} - - // There's data on the channel, so make sure we destroy it promptly. - // This is why not using an arc is a little difficult (need the box - // to stay valid while we take the data). - DATA => unsafe { - (&mut *self.data.get()).take().unwrap(); - }, - - // We're the only ones that can block on this port - _ => unreachable!(), - } - } - - //////////////////////////////////////////////////////////////////////////// - // select implementation - //////////////////////////////////////////////////////////////////////////// - - // Remove a previous selecting thread from this port. This ensures that the - // blocked thread will no longer be visible to any other threads. - // - // The return value indicates whether there's data on this port. - pub fn abort_selection(&self) -> Result> { - let state = match self.state.load(Ordering::SeqCst) { - // Each of these states means that no further activity will happen - // with regard to abortion selection - s @ EMPTY | s @ DATA | s @ DISCONNECTED => s, - - // If we've got a blocked thread, then use an atomic to gain ownership - // of it (may fail) - ptr => self.state.compare_and_swap(ptr, EMPTY, Ordering::SeqCst), - }; - - // Now that we've got ownership of our state, figure out what to do - // about it. - match state { - EMPTY => unreachable!(), - // our thread used for select was stolen - DATA => Ok(true), - - // If the other end has hung up, then we have complete ownership - // of the port. First, check if there was data waiting for us. This - // is possible if the other end sent something and then hung up. - // - // We then need to check to see if there was an upgrade requested, - // and if so, the upgraded port needs to have its selection aborted. - DISCONNECTED => unsafe { - if (*self.data.get()).is_some() { - Ok(true) - } else { - match ptr::replace(self.upgrade.get(), SendUsed) { - GoUp(port) => Err(port), - _ => Ok(true), - } - } - }, - - // We woke ourselves up from select. - ptr => unsafe { - drop(SignalToken::cast_from_usize(ptr)); - Ok(false) - }, - } - } -} - -impl Drop for Packet { - fn drop(&mut self) { - assert_eq!(self.state.load(Ordering::SeqCst), DISCONNECTED); - } -} diff --git a/crux-mir/lib/std/src/sync/mpsc/shared.rs b/crux-mir/lib/std/src/sync/mpsc/shared.rs deleted file mode 100644 index 2b0393573..000000000 --- a/crux-mir/lib/std/src/sync/mpsc/shared.rs +++ /dev/null @@ -1,492 +0,0 @@ -/// Shared channels. -/// -/// This is the flavor of channels which are not necessarily optimized for any -/// particular use case, but are the most general in how they are used. Shared -/// channels are cloneable allowing for multiple senders. -/// -/// High level implementation details can be found in the comment of the parent -/// module. You'll also note that the implementation of the shared and stream -/// channels are quite similar, and this is no coincidence! -pub use self::Failure::*; -use self::StartResult::*; - -use core::cmp; -use core::intrinsics::abort; -use core::isize; - -use crate::cell::UnsafeCell; -use crate::ptr; -use crate::sync::atomic::{AtomicBool, AtomicIsize, AtomicUsize, Ordering}; -use crate::sync::mpsc::blocking::{self, SignalToken}; -use crate::sync::mpsc::mpsc_queue as mpsc; -use crate::sync::{Mutex, MutexGuard}; -use crate::thread; -use crate::time::Instant; - -const DISCONNECTED: isize = isize::MIN; -const FUDGE: isize = 1024; -const MAX_REFCOUNT: usize = (isize::MAX) as usize; -#[cfg(test)] -const MAX_STEALS: isize = 5; -#[cfg(not(test))] -const MAX_STEALS: isize = 1 << 20; - -pub struct Packet { - queue: mpsc::Queue, - cnt: AtomicIsize, // How many items are on this channel - steals: UnsafeCell, // How many times has a port received without blocking? - to_wake: AtomicUsize, // SignalToken for wake up - - // The number of channels which are currently using this packet. - channels: AtomicUsize, - - // See the discussion in Port::drop and the channel send methods for what - // these are used for - port_dropped: AtomicBool, - sender_drain: AtomicIsize, - - // this lock protects various portions of this implementation during - // select() - select_lock: Mutex<()>, -} - -pub enum Failure { - Empty, - Disconnected, -} - -#[derive(PartialEq, Eq)] -enum StartResult { - Installed, - Abort, -} - -impl Packet { - // Creation of a packet *must* be followed by a call to postinit_lock - // and later by inherit_blocker - pub fn new() -> Packet { - Packet { - queue: mpsc::Queue::new(), - cnt: AtomicIsize::new(0), - steals: UnsafeCell::new(0), - to_wake: AtomicUsize::new(0), - channels: AtomicUsize::new(2), - port_dropped: AtomicBool::new(false), - sender_drain: AtomicIsize::new(0), - select_lock: Mutex::new(()), - } - } - - // This function should be used after newly created Packet - // was wrapped with an Arc - // In other case mutex data will be duplicated while cloning - // and that could cause problems on platforms where it is - // represented by opaque data structure - pub fn postinit_lock(&self) -> MutexGuard<'_, ()> { - self.select_lock.lock().unwrap() - } - - // This function is used at the creation of a shared packet to inherit a - // previously blocked thread. This is done to prevent spurious wakeups of - // threads in select(). - // - // This can only be called at channel-creation time - pub fn inherit_blocker(&self, token: Option, guard: MutexGuard<'_, ()>) { - token.map(|token| { - assert_eq!(self.cnt.load(Ordering::SeqCst), 0); - assert_eq!(self.to_wake.load(Ordering::SeqCst), 0); - self.to_wake.store(unsafe { token.cast_to_usize() }, Ordering::SeqCst); - self.cnt.store(-1, Ordering::SeqCst); - - // This store is a little sketchy. What's happening here is that - // we're transferring a blocker from a oneshot or stream channel to - // this shared channel. In doing so, we never spuriously wake them - // up and rather only wake them up at the appropriate time. This - // implementation of shared channels assumes that any blocking - // recv() will undo the increment of steals performed in try_recv() - // once the recv is complete. This thread that we're inheriting, - // however, is not in the middle of recv. Hence, the first time we - // wake them up, they're going to wake up from their old port, move - // on to the upgraded port, and then call the block recv() function. - // - // When calling this function, they'll find there's data immediately - // available, counting it as a steal. This in fact wasn't a steal - // because we appropriately blocked them waiting for data. - // - // To offset this bad increment, we initially set the steal count to - // -1. You'll find some special code in abort_selection() as well to - // ensure that this -1 steal count doesn't escape too far. - unsafe { - *self.steals.get() = -1; - } - }); - - // When the shared packet is constructed, we grabbed this lock. The - // purpose of this lock is to ensure that abort_selection() doesn't - // interfere with this method. After we unlock this lock, we're - // signifying that we're done modifying self.cnt and self.to_wake and - // the port is ready for the world to continue using it. - drop(guard); - } - - pub fn send(&self, t: T) -> Result<(), T> { - // See Port::drop for what's going on - if self.port_dropped.load(Ordering::SeqCst) { - return Err(t); - } - - // Note that the multiple sender case is a little trickier - // semantically than the single sender case. The logic for - // incrementing is "add and if disconnected store disconnected". - // This could end up leading some senders to believe that there - // wasn't a disconnect if in fact there was a disconnect. This means - // that while one thread is attempting to re-store the disconnected - // states, other threads could walk through merrily incrementing - // this very-negative disconnected count. To prevent senders from - // spuriously attempting to send when the channels is actually - // disconnected, the count has a ranged check here. - // - // This is also done for another reason. Remember that the return - // value of this function is: - // - // `true` == the data *may* be received, this essentially has no - // meaning - // `false` == the data will *never* be received, this has a lot of - // meaning - // - // In the SPSC case, we have a check of 'queue.is_empty()' to see - // whether the data was actually received, but this same condition - // means nothing in a multi-producer context. As a result, this - // preflight check serves as the definitive "this will never be - // received". Once we get beyond this check, we have permanently - // entered the realm of "this may be received" - if self.cnt.load(Ordering::SeqCst) < DISCONNECTED + FUDGE { - return Err(t); - } - - self.queue.push(t); - match self.cnt.fetch_add(1, Ordering::SeqCst) { - -1 => { - self.take_to_wake().signal(); - } - - // In this case, we have possibly failed to send our data, and - // we need to consider re-popping the data in order to fully - // destroy it. We must arbitrate among the multiple senders, - // however, because the queues that we're using are - // single-consumer queues. In order to do this, all exiting - // pushers will use an atomic count in order to count those - // flowing through. Pushers who see 0 are required to drain as - // much as possible, and then can only exit when they are the - // only pusher (otherwise they must try again). - n if n < DISCONNECTED + FUDGE => { - // see the comment in 'try' for a shared channel for why this - // window of "not disconnected" is ok. - self.cnt.store(DISCONNECTED, Ordering::SeqCst); - - if self.sender_drain.fetch_add(1, Ordering::SeqCst) == 0 { - loop { - // drain the queue, for info on the thread yield see the - // discussion in try_recv - loop { - match self.queue.pop() { - mpsc::Data(..) => {} - mpsc::Empty => break, - mpsc::Inconsistent => thread::yield_now(), - } - } - // maybe we're done, if we're not the last ones - // here, then we need to go try again. - if self.sender_drain.fetch_sub(1, Ordering::SeqCst) == 1 { - break; - } - } - - // At this point, there may still be data on the queue, - // but only if the count hasn't been incremented and - // some other sender hasn't finished pushing data just - // yet. That sender in question will drain its own data. - } - } - - // Can't make any assumptions about this case like in the SPSC case. - _ => {} - } - - Ok(()) - } - - pub fn recv(&self, deadline: Option) -> Result { - // This code is essentially the exact same as that found in the stream - // case (see stream.rs) - match self.try_recv() { - Err(Empty) => {} - data => return data, - } - - let (wait_token, signal_token) = blocking::tokens(); - if self.decrement(signal_token) == Installed { - if let Some(deadline) = deadline { - let timed_out = !wait_token.wait_max_until(deadline); - if timed_out { - self.abort_selection(false); - } - } else { - wait_token.wait(); - } - } - - match self.try_recv() { - data @ Ok(..) => unsafe { - *self.steals.get() -= 1; - data - }, - data => data, - } - } - - // Essentially the exact same thing as the stream decrement function. - // Returns true if blocking should proceed. - fn decrement(&self, token: SignalToken) -> StartResult { - unsafe { - assert_eq!(self.to_wake.load(Ordering::SeqCst), 0); - let ptr = token.cast_to_usize(); - self.to_wake.store(ptr, Ordering::SeqCst); - - let steals = ptr::replace(self.steals.get(), 0); - - match self.cnt.fetch_sub(1 + steals, Ordering::SeqCst) { - DISCONNECTED => { - self.cnt.store(DISCONNECTED, Ordering::SeqCst); - } - // If we factor in our steals and notice that the channel has no - // data, we successfully sleep - n => { - assert!(n >= 0); - if n - steals <= 0 { - return Installed; - } - } - } - - self.to_wake.store(0, Ordering::SeqCst); - drop(SignalToken::cast_from_usize(ptr)); - Abort - } - } - - pub fn try_recv(&self) -> Result { - let ret = match self.queue.pop() { - mpsc::Data(t) => Some(t), - mpsc::Empty => None, - - // This is a bit of an interesting case. The channel is reported as - // having data available, but our pop() has failed due to the queue - // being in an inconsistent state. This means that there is some - // pusher somewhere which has yet to complete, but we are guaranteed - // that a pop will eventually succeed. In this case, we spin in a - // yield loop because the remote sender should finish their enqueue - // operation "very quickly". - // - // Avoiding this yield loop would require a different queue - // abstraction which provides the guarantee that after M pushes have - // succeeded, at least M pops will succeed. The current queues - // guarantee that if there are N active pushes, you can pop N times - // once all N have finished. - mpsc::Inconsistent => { - let data; - loop { - thread::yield_now(); - match self.queue.pop() { - mpsc::Data(t) => { - data = t; - break; - } - mpsc::Empty => panic!("inconsistent => empty"), - mpsc::Inconsistent => {} - } - } - Some(data) - } - }; - match ret { - // See the discussion in the stream implementation for why we - // might decrement steals. - Some(data) => unsafe { - if *self.steals.get() > MAX_STEALS { - match self.cnt.swap(0, Ordering::SeqCst) { - DISCONNECTED => { - self.cnt.store(DISCONNECTED, Ordering::SeqCst); - } - n => { - let m = cmp::min(n, *self.steals.get()); - *self.steals.get() -= m; - self.bump(n - m); - } - } - assert!(*self.steals.get() >= 0); - } - *self.steals.get() += 1; - Ok(data) - }, - - // See the discussion in the stream implementation for why we try - // again. - None => { - match self.cnt.load(Ordering::SeqCst) { - n if n != DISCONNECTED => Err(Empty), - _ => { - match self.queue.pop() { - mpsc::Data(t) => Ok(t), - mpsc::Empty => Err(Disconnected), - // with no senders, an inconsistency is impossible. - mpsc::Inconsistent => unreachable!(), - } - } - } - } - } - } - - // Prepares this shared packet for a channel clone, essentially just bumping - // a refcount. - pub fn clone_chan(&self) { - let old_count = self.channels.fetch_add(1, Ordering::SeqCst); - - // See comments on Arc::clone() on why we do this (for `mem::forget`). - if old_count > MAX_REFCOUNT { - unsafe { - abort(); - } - } - } - - // Decrement the reference count on a channel. This is called whenever a - // Chan is dropped and may end up waking up a receiver. It's the receiver's - // responsibility on the other end to figure out that we've disconnected. - pub fn drop_chan(&self) { - match self.channels.fetch_sub(1, Ordering::SeqCst) { - 1 => {} - n if n > 1 => return, - n => panic!("bad number of channels left {}", n), - } - - match self.cnt.swap(DISCONNECTED, Ordering::SeqCst) { - -1 => { - self.take_to_wake().signal(); - } - DISCONNECTED => {} - n => { - assert!(n >= 0); - } - } - } - - // See the long discussion inside of stream.rs for why the queue is drained, - // and why it is done in this fashion. - pub fn drop_port(&self) { - self.port_dropped.store(true, Ordering::SeqCst); - let mut steals = unsafe { *self.steals.get() }; - while { - let cnt = self.cnt.compare_and_swap(steals, DISCONNECTED, Ordering::SeqCst); - cnt != DISCONNECTED && cnt != steals - } { - // See the discussion in 'try_recv' for why we yield - // control of this thread. - loop { - match self.queue.pop() { - mpsc::Data(..) => { - steals += 1; - } - mpsc::Empty | mpsc::Inconsistent => break, - } - } - } - } - - // Consumes ownership of the 'to_wake' field. - fn take_to_wake(&self) -> SignalToken { - let ptr = self.to_wake.load(Ordering::SeqCst); - self.to_wake.store(0, Ordering::SeqCst); - assert!(ptr != 0); - unsafe { SignalToken::cast_from_usize(ptr) } - } - - //////////////////////////////////////////////////////////////////////////// - // select implementation - //////////////////////////////////////////////////////////////////////////// - - // increment the count on the channel (used for selection) - fn bump(&self, amt: isize) -> isize { - match self.cnt.fetch_add(amt, Ordering::SeqCst) { - DISCONNECTED => { - self.cnt.store(DISCONNECTED, Ordering::SeqCst); - DISCONNECTED - } - n => n, - } - } - - // Cancels a previous thread waiting on this port, returning whether there's - // data on the port. - // - // This is similar to the stream implementation (hence fewer comments), but - // uses a different value for the "steals" variable. - pub fn abort_selection(&self, _was_upgrade: bool) -> bool { - // Before we do anything else, we bounce on this lock. The reason for - // doing this is to ensure that any upgrade-in-progress is gone and - // done with. Without this bounce, we can race with inherit_blocker - // about looking at and dealing with to_wake. Once we have acquired the - // lock, we are guaranteed that inherit_blocker is done. - { - let _guard = self.select_lock.lock().unwrap(); - } - - // Like the stream implementation, we want to make sure that the count - // on the channel goes non-negative. We don't know how negative the - // stream currently is, so instead of using a steal value of 1, we load - // the channel count and figure out what we should do to make it - // positive. - let steals = { - let cnt = self.cnt.load(Ordering::SeqCst); - if cnt < 0 && cnt != DISCONNECTED { -cnt } else { 0 } - }; - let prev = self.bump(steals + 1); - - if prev == DISCONNECTED { - assert_eq!(self.to_wake.load(Ordering::SeqCst), 0); - true - } else { - let cur = prev + steals + 1; - assert!(cur >= 0); - if prev < 0 { - drop(self.take_to_wake()); - } else { - while self.to_wake.load(Ordering::SeqCst) != 0 { - thread::yield_now(); - } - } - unsafe { - // if the number of steals is -1, it was the pre-emptive -1 steal - // count from when we inherited a blocker. This is fine because - // we're just going to overwrite it with a real value. - let old = self.steals.get(); - assert!(*old == 0 || *old == -1); - *old = steals; - prev >= 0 - } - } - } -} - -impl Drop for Packet { - fn drop(&mut self) { - // Note that this load is not only an assert for correctness about - // disconnection, but also a proper fence before the read of - // `to_wake`, so this assert cannot be removed with also removing - // the `to_wake` assert. - assert_eq!(self.cnt.load(Ordering::SeqCst), DISCONNECTED); - assert_eq!(self.to_wake.load(Ordering::SeqCst), 0); - assert_eq!(self.channels.load(Ordering::SeqCst), 0); - } -} diff --git a/crux-mir/lib/std/src/sync/mpsc/spsc_queue.rs b/crux-mir/lib/std/src/sync/mpsc/spsc_queue.rs deleted file mode 100644 index 0274268f6..000000000 --- a/crux-mir/lib/std/src/sync/mpsc/spsc_queue.rs +++ /dev/null @@ -1,338 +0,0 @@ -//! A single-producer single-consumer concurrent queue -//! -//! This module contains the implementation of an SPSC queue which can be used -//! concurrently between two threads. This data structure is safe to use and -//! enforces the semantics that there is one pusher and one popper. - -// http://www.1024cores.net/home/lock-free-algorithms/queues/unbounded-spsc-queue - -use core::cell::UnsafeCell; -use core::ptr; - -use crate::boxed::Box; -use crate::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; - -use super::cache_aligned::CacheAligned; - -// Node within the linked list queue of messages to send -struct Node { - // FIXME: this could be an uninitialized T if we're careful enough, and - // that would reduce memory usage (and be a bit faster). - // is it worth it? - value: Option, // nullable for re-use of nodes - cached: bool, // This node goes into the node cache - next: AtomicPtr>, // next node in the queue -} - -/// The single-producer single-consumer queue. This structure is not cloneable, -/// but it can be safely shared in an Arc if it is guaranteed that there -/// is only one popper and one pusher touching the queue at any one point in -/// time. -pub struct Queue { - // consumer fields - consumer: CacheAligned>, - - // producer fields - producer: CacheAligned>, -} - -struct Consumer { - tail: UnsafeCell<*mut Node>, // where to pop from - tail_prev: AtomicPtr>, // where to pop from - cache_bound: usize, // maximum cache size - cached_nodes: AtomicUsize, // number of nodes marked as cacheable - addition: Addition, -} - -struct Producer { - head: UnsafeCell<*mut Node>, // where to push to - first: UnsafeCell<*mut Node>, // where to get new nodes from - tail_copy: UnsafeCell<*mut Node>, // between first/tail - addition: Addition, -} - -unsafe impl Send for Queue {} - -unsafe impl Sync for Queue {} - -impl Node { - fn new() -> *mut Node { - Box::into_raw(box Node { - value: None, - cached: false, - next: AtomicPtr::new(ptr::null_mut::>()), - }) - } -} - -impl Queue { - /// Creates a new queue. With given additional elements in the producer and - /// consumer portions of the queue. - /// - /// Due to the performance implications of cache-contention, - /// we wish to keep fields used mainly by the producer on a separate cache - /// line than those used by the consumer. - /// Since cache lines are usually 64 bytes, it is unreasonably expensive to - /// allocate one for small fields, so we allow users to insert additional - /// fields into the cache lines already allocated by this for the producer - /// and consumer. - /// - /// This is unsafe as the type system doesn't enforce a single - /// consumer-producer relationship. It also allows the consumer to `pop` - /// items while there is a `peek` active due to all methods having a - /// non-mutable receiver. - /// - /// # Arguments - /// - /// * `bound` - This queue implementation is implemented with a linked - /// list, and this means that a push is always a malloc. In - /// order to amortize this cost, an internal cache of nodes is - /// maintained to prevent a malloc from always being - /// necessary. This bound is the limit on the size of the - /// cache (if desired). If the value is 0, then the cache has - /// no bound. Otherwise, the cache will never grow larger than - /// `bound` (although the queue itself could be much larger. - pub unsafe fn with_additions( - bound: usize, - producer_addition: ProducerAddition, - consumer_addition: ConsumerAddition, - ) -> Self { - let n1 = Node::new(); - let n2 = Node::new(); - (*n1).next.store(n2, Ordering::Relaxed); - Queue { - consumer: CacheAligned::new(Consumer { - tail: UnsafeCell::new(n2), - tail_prev: AtomicPtr::new(n1), - cache_bound: bound, - cached_nodes: AtomicUsize::new(0), - addition: consumer_addition, - }), - producer: CacheAligned::new(Producer { - head: UnsafeCell::new(n2), - first: UnsafeCell::new(n1), - tail_copy: UnsafeCell::new(n1), - addition: producer_addition, - }), - } - } - - /// Pushes a new value onto this queue. Note that to use this function - /// safely, it must be externally guaranteed that there is only one pusher. - pub fn push(&self, t: T) { - unsafe { - // Acquire a node (which either uses a cached one or allocates a new - // one), and then append this to the 'head' node. - let n = self.alloc(); - assert!((*n).value.is_none()); - (*n).value = Some(t); - (*n).next.store(ptr::null_mut(), Ordering::Relaxed); - (**self.producer.head.get()).next.store(n, Ordering::Release); - *(&self.producer.head).get() = n; - } - } - - unsafe fn alloc(&self) -> *mut Node { - // First try to see if we can consume the 'first' node for our uses. - if *self.producer.first.get() != *self.producer.tail_copy.get() { - let ret = *self.producer.first.get(); - *self.producer.0.first.get() = (*ret).next.load(Ordering::Relaxed); - return ret; - } - // If the above fails, then update our copy of the tail and try - // again. - *self.producer.0.tail_copy.get() = self.consumer.tail_prev.load(Ordering::Acquire); - if *self.producer.first.get() != *self.producer.tail_copy.get() { - let ret = *self.producer.first.get(); - *self.producer.0.first.get() = (*ret).next.load(Ordering::Relaxed); - return ret; - } - // If all of that fails, then we have to allocate a new node - // (there's nothing in the node cache). - Node::new() - } - - /// Attempts to pop a value from this queue. Remember that to use this type - /// safely you must ensure that there is only one popper at a time. - pub fn pop(&self) -> Option { - unsafe { - // The `tail` node is not actually a used node, but rather a - // sentinel from where we should start popping from. Hence, look at - // tail's next field and see if we can use it. If we do a pop, then - // the current tail node is a candidate for going into the cache. - let tail = *self.consumer.tail.get(); - let next = (*tail).next.load(Ordering::Acquire); - if next.is_null() { - return None; - } - assert!((*next).value.is_some()); - let ret = (*next).value.take(); - - *self.consumer.0.tail.get() = next; - if self.consumer.cache_bound == 0 { - self.consumer.tail_prev.store(tail, Ordering::Release); - } else { - let cached_nodes = self.consumer.cached_nodes.load(Ordering::Relaxed); - if cached_nodes < self.consumer.cache_bound && !(*tail).cached { - self.consumer.cached_nodes.store(cached_nodes, Ordering::Relaxed); - (*tail).cached = true; - } - - if (*tail).cached { - self.consumer.tail_prev.store(tail, Ordering::Release); - } else { - (*self.consumer.tail_prev.load(Ordering::Relaxed)) - .next - .store(next, Ordering::Relaxed); - // We have successfully erased all references to 'tail', so - // now we can safely drop it. - let _: Box> = Box::from_raw(tail); - } - } - ret - } - } - - /// Attempts to peek at the head of the queue, returning `None` if the queue - /// has no data currently - /// - /// # Warning - /// The reference returned is invalid if it is not used before the consumer - /// pops the value off the queue. If the producer then pushes another value - /// onto the queue, it will overwrite the value pointed to by the reference. - pub fn peek(&self) -> Option<&mut T> { - // This is essentially the same as above with all the popping bits - // stripped out. - unsafe { - let tail = *self.consumer.tail.get(); - let next = (*tail).next.load(Ordering::Acquire); - if next.is_null() { None } else { (*next).value.as_mut() } - } - } - - pub fn producer_addition(&self) -> &ProducerAddition { - &self.producer.addition - } - - pub fn consumer_addition(&self) -> &ConsumerAddition { - &self.consumer.addition - } -} - -impl Drop for Queue { - fn drop(&mut self) { - unsafe { - let mut cur = *self.producer.first.get(); - while !cur.is_null() { - let next = (*cur).next.load(Ordering::Relaxed); - let _n: Box> = Box::from_raw(cur); - cur = next; - } - } - } -} - -#[cfg(all(test, not(target_os = "emscripten")))] -mod tests { - use super::Queue; - use crate::sync::mpsc::channel; - use crate::sync::Arc; - use crate::thread; - - #[test] - fn smoke() { - unsafe { - let queue = Queue::with_additions(0, (), ()); - queue.push(1); - queue.push(2); - assert_eq!(queue.pop(), Some(1)); - assert_eq!(queue.pop(), Some(2)); - assert_eq!(queue.pop(), None); - queue.push(3); - queue.push(4); - assert_eq!(queue.pop(), Some(3)); - assert_eq!(queue.pop(), Some(4)); - assert_eq!(queue.pop(), None); - } - } - - #[test] - fn peek() { - unsafe { - let queue = Queue::with_additions(0, (), ()); - queue.push(vec![1]); - - // Ensure the borrowchecker works - match queue.peek() { - Some(vec) => { - assert_eq!(&*vec, &[1]); - } - None => unreachable!(), - } - - match queue.pop() { - Some(vec) => { - assert_eq!(&*vec, &[1]); - } - None => unreachable!(), - } - } - } - - #[test] - fn drop_full() { - unsafe { - let q: Queue> = Queue::with_additions(0, (), ()); - q.push(box 1); - q.push(box 2); - } - } - - #[test] - fn smoke_bound() { - unsafe { - let q = Queue::with_additions(0, (), ()); - q.push(1); - q.push(2); - assert_eq!(q.pop(), Some(1)); - assert_eq!(q.pop(), Some(2)); - assert_eq!(q.pop(), None); - q.push(3); - q.push(4); - assert_eq!(q.pop(), Some(3)); - assert_eq!(q.pop(), Some(4)); - assert_eq!(q.pop(), None); - } - } - - #[test] - fn stress() { - unsafe { - stress_bound(0); - stress_bound(1); - } - - unsafe fn stress_bound(bound: usize) { - let q = Arc::new(Queue::with_additions(bound, (), ())); - - let (tx, rx) = channel(); - let q2 = q.clone(); - let _t = thread::spawn(move || { - for _ in 0..100000 { - loop { - match q2.pop() { - Some(1) => break, - Some(_) => panic!(), - None => {} - } - } - } - tx.send(()).unwrap(); - }); - for _ in 0..100000 { - q.push(1); - } - rx.recv().unwrap(); - } - } -} diff --git a/crux-mir/lib/std/src/sync/mpsc/stream.rs b/crux-mir/lib/std/src/sync/mpsc/stream.rs deleted file mode 100644 index 2e3270e81..000000000 --- a/crux-mir/lib/std/src/sync/mpsc/stream.rs +++ /dev/null @@ -1,454 +0,0 @@ -/// Stream channels -/// -/// This is the flavor of channels which are optimized for one sender and one -/// receiver. The sender will be upgraded to a shared channel if the channel is -/// cloned. -/// -/// High level implementation details can be found in the comment of the parent -/// module. -pub use self::Failure::*; -use self::Message::*; -pub use self::UpgradeResult::*; - -use core::cmp; -use core::isize; - -use crate::cell::UnsafeCell; -use crate::ptr; -use crate::thread; -use crate::time::Instant; - -use crate::sync::atomic::{AtomicBool, AtomicIsize, AtomicUsize, Ordering}; -use crate::sync::mpsc::blocking::{self, SignalToken}; -use crate::sync::mpsc::spsc_queue as spsc; -use crate::sync::mpsc::Receiver; - -const DISCONNECTED: isize = isize::MIN; -#[cfg(test)] -const MAX_STEALS: isize = 5; -#[cfg(not(test))] -const MAX_STEALS: isize = 1 << 20; - -pub struct Packet { - // internal queue for all messages - queue: spsc::Queue, ProducerAddition, ConsumerAddition>, -} - -struct ProducerAddition { - cnt: AtomicIsize, // How many items are on this channel - to_wake: AtomicUsize, // SignalToken for the blocked thread to wake up - - port_dropped: AtomicBool, // flag if the channel has been destroyed. -} - -struct ConsumerAddition { - steals: UnsafeCell, // How many times has a port received without blocking? -} - -pub enum Failure { - Empty, - Disconnected, - Upgraded(Receiver), -} - -pub enum UpgradeResult { - UpSuccess, - UpDisconnected, - UpWoke(SignalToken), -} - -// Any message could contain an "upgrade request" to a new shared port, so the -// internal queue it's a queue of T, but rather Message -enum Message { - Data(T), - GoUp(Receiver), -} - -impl Packet { - pub fn new() -> Packet { - Packet { - queue: unsafe { - spsc::Queue::with_additions( - 128, - ProducerAddition { - cnt: AtomicIsize::new(0), - to_wake: AtomicUsize::new(0), - - port_dropped: AtomicBool::new(false), - }, - ConsumerAddition { steals: UnsafeCell::new(0) }, - ) - }, - } - } - - pub fn send(&self, t: T) -> Result<(), T> { - // If the other port has deterministically gone away, then definitely - // must return the data back up the stack. Otherwise, the data is - // considered as being sent. - if self.queue.producer_addition().port_dropped.load(Ordering::SeqCst) { - return Err(t); - } - - match self.do_send(Data(t)) { - UpSuccess | UpDisconnected => {} - UpWoke(token) => { - token.signal(); - } - } - Ok(()) - } - - pub fn upgrade(&self, up: Receiver) -> UpgradeResult { - // If the port has gone away, then there's no need to proceed any - // further. - if self.queue.producer_addition().port_dropped.load(Ordering::SeqCst) { - return UpDisconnected; - } - - self.do_send(GoUp(up)) - } - - fn do_send(&self, t: Message) -> UpgradeResult { - self.queue.push(t); - match self.queue.producer_addition().cnt.fetch_add(1, Ordering::SeqCst) { - // As described in the mod's doc comment, -1 == wakeup - -1 => UpWoke(self.take_to_wake()), - // As as described before, SPSC queues must be >= -2 - -2 => UpSuccess, - - // Be sure to preserve the disconnected state, and the return value - // in this case is going to be whether our data was received or not. - // This manifests itself on whether we have an empty queue or not. - // - // Primarily, are required to drain the queue here because the port - // will never remove this data. We can only have at most one item to - // drain (the port drains the rest). - DISCONNECTED => { - self.queue.producer_addition().cnt.store(DISCONNECTED, Ordering::SeqCst); - let first = self.queue.pop(); - let second = self.queue.pop(); - assert!(second.is_none()); - - match first { - Some(..) => UpSuccess, // we failed to send the data - None => UpDisconnected, // we successfully sent data - } - } - - // Otherwise we just sent some data on a non-waiting queue, so just - // make sure the world is sane and carry on! - n => { - assert!(n >= 0); - UpSuccess - } - } - } - - // Consumes ownership of the 'to_wake' field. - fn take_to_wake(&self) -> SignalToken { - let ptr = self.queue.producer_addition().to_wake.load(Ordering::SeqCst); - self.queue.producer_addition().to_wake.store(0, Ordering::SeqCst); - assert!(ptr != 0); - unsafe { SignalToken::cast_from_usize(ptr) } - } - - // Decrements the count on the channel for a sleeper, returning the sleeper - // back if it shouldn't sleep. Note that this is the location where we take - // steals into account. - fn decrement(&self, token: SignalToken) -> Result<(), SignalToken> { - assert_eq!(self.queue.producer_addition().to_wake.load(Ordering::SeqCst), 0); - let ptr = unsafe { token.cast_to_usize() }; - self.queue.producer_addition().to_wake.store(ptr, Ordering::SeqCst); - - let steals = unsafe { ptr::replace(self.queue.consumer_addition().steals.get(), 0) }; - - match self.queue.producer_addition().cnt.fetch_sub(1 + steals, Ordering::SeqCst) { - DISCONNECTED => { - self.queue.producer_addition().cnt.store(DISCONNECTED, Ordering::SeqCst); - } - // If we factor in our steals and notice that the channel has no - // data, we successfully sleep - n => { - assert!(n >= 0); - if n - steals <= 0 { - return Ok(()); - } - } - } - - self.queue.producer_addition().to_wake.store(0, Ordering::SeqCst); - Err(unsafe { SignalToken::cast_from_usize(ptr) }) - } - - pub fn recv(&self, deadline: Option) -> Result> { - // Optimistic preflight check (scheduling is expensive). - match self.try_recv() { - Err(Empty) => {} - data => return data, - } - - // Welp, our channel has no data. Deschedule the current thread and - // initiate the blocking protocol. - let (wait_token, signal_token) = blocking::tokens(); - if self.decrement(signal_token).is_ok() { - if let Some(deadline) = deadline { - let timed_out = !wait_token.wait_max_until(deadline); - if timed_out { - self.abort_selection(/* was_upgrade = */ false).map_err(Upgraded)?; - } - } else { - wait_token.wait(); - } - } - - match self.try_recv() { - // Messages which actually popped from the queue shouldn't count as - // a steal, so offset the decrement here (we already have our - // "steal" factored into the channel count above). - data @ Ok(..) | data @ Err(Upgraded(..)) => unsafe { - *self.queue.consumer_addition().steals.get() -= 1; - data - }, - - data => data, - } - } - - pub fn try_recv(&self) -> Result> { - match self.queue.pop() { - // If we stole some data, record to that effect (this will be - // factored into cnt later on). - // - // Note that we don't allow steals to grow without bound in order to - // prevent eventual overflow of either steals or cnt as an overflow - // would have catastrophic results. Sometimes, steals > cnt, but - // other times cnt > steals, so we don't know the relation between - // steals and cnt. This code path is executed only rarely, so we do - // a pretty slow operation, of swapping 0 into cnt, taking steals - // down as much as possible (without going negative), and then - // adding back in whatever we couldn't factor into steals. - Some(data) => unsafe { - if *self.queue.consumer_addition().steals.get() > MAX_STEALS { - match self.queue.producer_addition().cnt.swap(0, Ordering::SeqCst) { - DISCONNECTED => { - self.queue - .producer_addition() - .cnt - .store(DISCONNECTED, Ordering::SeqCst); - } - n => { - let m = cmp::min(n, *self.queue.consumer_addition().steals.get()); - *self.queue.consumer_addition().steals.get() -= m; - self.bump(n - m); - } - } - assert!(*self.queue.consumer_addition().steals.get() >= 0); - } - *self.queue.consumer_addition().steals.get() += 1; - match data { - Data(t) => Ok(t), - GoUp(up) => Err(Upgraded(up)), - } - }, - - None => { - match self.queue.producer_addition().cnt.load(Ordering::SeqCst) { - n if n != DISCONNECTED => Err(Empty), - - // This is a little bit of a tricky case. We failed to pop - // data above, and then we have viewed that the channel is - // disconnected. In this window more data could have been - // sent on the channel. It doesn't really make sense to - // return that the channel is disconnected when there's - // actually data on it, so be extra sure there's no data by - // popping one more time. - // - // We can ignore steals because the other end is - // disconnected and we'll never need to really factor in our - // steals again. - _ => match self.queue.pop() { - Some(Data(t)) => Ok(t), - Some(GoUp(up)) => Err(Upgraded(up)), - None => Err(Disconnected), - }, - } - } - } - } - - pub fn drop_chan(&self) { - // Dropping a channel is pretty simple, we just flag it as disconnected - // and then wakeup a blocker if there is one. - match self.queue.producer_addition().cnt.swap(DISCONNECTED, Ordering::SeqCst) { - -1 => { - self.take_to_wake().signal(); - } - DISCONNECTED => {} - n => { - assert!(n >= 0); - } - } - } - - pub fn drop_port(&self) { - // Dropping a port seems like a fairly trivial thing. In theory all we - // need to do is flag that we're disconnected and then everything else - // can take over (we don't have anyone to wake up). - // - // The catch for Ports is that we want to drop the entire contents of - // the queue. There are multiple reasons for having this property, the - // largest of which is that if another chan is waiting in this channel - // (but not received yet), then waiting on that port will cause a - // deadlock. - // - // So if we accept that we must now destroy the entire contents of the - // queue, this code may make a bit more sense. The tricky part is that - // we can't let any in-flight sends go un-dropped, we have to make sure - // *everything* is dropped and nothing new will come onto the channel. - - // The first thing we do is set a flag saying that we're done for. All - // sends are gated on this flag, so we're immediately guaranteed that - // there are a bounded number of active sends that we'll have to deal - // with. - self.queue.producer_addition().port_dropped.store(true, Ordering::SeqCst); - - // Now that we're guaranteed to deal with a bounded number of senders, - // we need to drain the queue. This draining process happens atomically - // with respect to the "count" of the channel. If the count is nonzero - // (with steals taken into account), then there must be data on the - // channel. In this case we drain everything and then try again. We will - // continue to fail while active senders send data while we're dropping - // data, but eventually we're guaranteed to break out of this loop - // (because there is a bounded number of senders). - let mut steals = unsafe { *self.queue.consumer_addition().steals.get() }; - while { - let cnt = self.queue.producer_addition().cnt.compare_and_swap( - steals, - DISCONNECTED, - Ordering::SeqCst, - ); - cnt != DISCONNECTED && cnt != steals - } { - while let Some(_) = self.queue.pop() { - steals += 1; - } - } - - // At this point in time, we have gated all future senders from sending, - // and we have flagged the channel as being disconnected. The senders - // still have some responsibility, however, because some sends may not - // complete until after we flag the disconnection. There are more - // details in the sending methods that see DISCONNECTED - } - - //////////////////////////////////////////////////////////////////////////// - // select implementation - //////////////////////////////////////////////////////////////////////////// - - // increment the count on the channel (used for selection) - fn bump(&self, amt: isize) -> isize { - match self.queue.producer_addition().cnt.fetch_add(amt, Ordering::SeqCst) { - DISCONNECTED => { - self.queue.producer_addition().cnt.store(DISCONNECTED, Ordering::SeqCst); - DISCONNECTED - } - n => n, - } - } - - // Removes a previous thread from being blocked in this port - pub fn abort_selection(&self, was_upgrade: bool) -> Result> { - // If we're aborting selection after upgrading from a oneshot, then - // we're guarantee that no one is waiting. The only way that we could - // have seen the upgrade is if data was actually sent on the channel - // half again. For us, this means that there is guaranteed to be data on - // this channel. Furthermore, we're guaranteed that there was no - // start_selection previously, so there's no need to modify `self.cnt` - // at all. - // - // Hence, because of these invariants, we immediately return `Ok(true)`. - // Note that the data may not actually be sent on the channel just yet. - // The other end could have flagged the upgrade but not sent data to - // this end. This is fine because we know it's a small bounded windows - // of time until the data is actually sent. - if was_upgrade { - assert_eq!(unsafe { *self.queue.consumer_addition().steals.get() }, 0); - assert_eq!(self.queue.producer_addition().to_wake.load(Ordering::SeqCst), 0); - return Ok(true); - } - - // We want to make sure that the count on the channel goes non-negative, - // and in the stream case we can have at most one steal, so just assume - // that we had one steal. - let steals = 1; - let prev = self.bump(steals + 1); - - // If we were previously disconnected, then we know for sure that there - // is no thread in to_wake, so just keep going - let has_data = if prev == DISCONNECTED { - assert_eq!(self.queue.producer_addition().to_wake.load(Ordering::SeqCst), 0); - true // there is data, that data is that we're disconnected - } else { - let cur = prev + steals + 1; - assert!(cur >= 0); - - // If the previous count was negative, then we just made things go - // positive, hence we passed the -1 boundary and we're responsible - // for removing the to_wake() field and trashing it. - // - // If the previous count was positive then we're in a tougher - // situation. A possible race is that a sender just incremented - // through -1 (meaning it's going to try to wake a thread up), but it - // hasn't yet read the to_wake. In order to prevent a future recv() - // from waking up too early (this sender picking up the plastered - // over to_wake), we spin loop here waiting for to_wake to be 0. - // Note that this entire select() implementation needs an overhaul, - // and this is *not* the worst part of it, so this is not done as a - // final solution but rather out of necessity for now to get - // something working. - if prev < 0 { - drop(self.take_to_wake()); - } else { - while self.queue.producer_addition().to_wake.load(Ordering::SeqCst) != 0 { - thread::yield_now(); - } - } - unsafe { - assert_eq!(*self.queue.consumer_addition().steals.get(), 0); - *self.queue.consumer_addition().steals.get() = steals; - } - - // if we were previously positive, then there's surely data to - // receive - prev >= 0 - }; - - // Now that we've determined that this queue "has data", we peek at the - // queue to see if the data is an upgrade or not. If it's an upgrade, - // then we need to destroy this port and abort selection on the - // upgraded port. - if has_data { - match self.queue.peek() { - Some(&mut GoUp(..)) => match self.queue.pop() { - Some(GoUp(port)) => Err(port), - _ => unreachable!(), - }, - _ => Ok(true), - } - } else { - Ok(false) - } - } -} - -impl Drop for Packet { - fn drop(&mut self) { - // Note that this load is not only an assert for correctness about - // disconnection, but also a proper fence before the read of - // `to_wake`, so this assert cannot be removed with also removing - // the `to_wake` assert. - assert_eq!(self.queue.producer_addition().cnt.load(Ordering::SeqCst), DISCONNECTED); - assert_eq!(self.queue.producer_addition().to_wake.load(Ordering::SeqCst), 0); - } -} diff --git a/crux-mir/lib/std/src/sync/mpsc/sync.rs b/crux-mir/lib/std/src/sync/mpsc/sync.rs deleted file mode 100644 index 79e868171..000000000 --- a/crux-mir/lib/std/src/sync/mpsc/sync.rs +++ /dev/null @@ -1,492 +0,0 @@ -use self::Blocker::*; -/// Synchronous channels/ports -/// -/// This channel implementation differs significantly from the asynchronous -/// implementations found next to it (oneshot/stream/share). This is an -/// implementation of a synchronous, bounded buffer channel. -/// -/// Each channel is created with some amount of backing buffer, and sends will -/// *block* until buffer space becomes available. A buffer size of 0 is valid, -/// which means that every successful send is paired with a successful recv. -/// -/// This flavor of channels defines a new `send_opt` method for channels which -/// is the method by which a message is sent but the thread does not panic if it -/// cannot be delivered. -/// -/// Another major difference is that send() will *always* return back the data -/// if it couldn't be sent. This is because it is deterministically known when -/// the data is received and when it is not received. -/// -/// Implementation-wise, it can all be summed up with "use a mutex plus some -/// logic". The mutex used here is an OS native mutex, meaning that no user code -/// is run inside of the mutex (to prevent context switching). This -/// implementation shares almost all code for the buffered and unbuffered cases -/// of a synchronous channel. There are a few branches for the unbuffered case, -/// but they're mostly just relevant to blocking senders. -pub use self::Failure::*; - -use core::intrinsics::abort; -use core::isize; -use core::mem; -use core::ptr; - -use crate::sync::atomic::{AtomicUsize, Ordering}; -use crate::sync::mpsc::blocking::{self, SignalToken, WaitToken}; -use crate::sync::{Mutex, MutexGuard}; -use crate::time::Instant; - -const MAX_REFCOUNT: usize = (isize::MAX) as usize; - -pub struct Packet { - /// Only field outside of the mutex. Just done for kicks, but mainly because - /// the other shared channel already had the code implemented - channels: AtomicUsize, - - lock: Mutex>, -} - -unsafe impl Send for Packet {} - -unsafe impl Sync for Packet {} - -struct State { - disconnected: bool, // Is the channel disconnected yet? - queue: Queue, // queue of senders waiting to send data - blocker: Blocker, // currently blocked thread on this channel - buf: Buffer, // storage for buffered messages - cap: usize, // capacity of this channel - - /// A curious flag used to indicate whether a sender failed or succeeded in - /// blocking. This is used to transmit information back to the thread that it - /// must dequeue its message from the buffer because it was not received. - /// This is only relevant in the 0-buffer case. This obviously cannot be - /// safely constructed, but it's guaranteed to always have a valid pointer - /// value. - canceled: Option<&'static mut bool>, -} - -unsafe impl Send for State {} - -/// Possible flavors of threads who can be blocked on this channel. -enum Blocker { - BlockedSender(SignalToken), - BlockedReceiver(SignalToken), - NoneBlocked, -} - -/// Simple queue for threading threads together. Nodes are stack-allocated, so -/// this structure is not safe at all -struct Queue { - head: *mut Node, - tail: *mut Node, -} - -struct Node { - token: Option, - next: *mut Node, -} - -unsafe impl Send for Node {} - -/// A simple ring-buffer -struct Buffer { - buf: Vec>, - start: usize, - size: usize, -} - -#[derive(Debug)] -pub enum Failure { - Empty, - Disconnected, -} - -/// Atomically blocks the current thread, placing it into `slot`, unlocking `lock` -/// in the meantime. This re-locks the mutex upon returning. -fn wait<'a, 'b, T>( - lock: &'a Mutex>, - mut guard: MutexGuard<'b, State>, - f: fn(SignalToken) -> Blocker, -) -> MutexGuard<'a, State> { - let (wait_token, signal_token) = blocking::tokens(); - match mem::replace(&mut guard.blocker, f(signal_token)) { - NoneBlocked => {} - _ => unreachable!(), - } - drop(guard); // unlock - wait_token.wait(); // block - lock.lock().unwrap() // relock -} - -/// Same as wait, but waiting at most until `deadline`. -fn wait_timeout_receiver<'a, 'b, T>( - lock: &'a Mutex>, - deadline: Instant, - mut guard: MutexGuard<'b, State>, - success: &mut bool, -) -> MutexGuard<'a, State> { - let (wait_token, signal_token) = blocking::tokens(); - match mem::replace(&mut guard.blocker, BlockedReceiver(signal_token)) { - NoneBlocked => {} - _ => unreachable!(), - } - drop(guard); // unlock - *success = wait_token.wait_max_until(deadline); // block - let mut new_guard = lock.lock().unwrap(); // relock - if !*success { - abort_selection(&mut new_guard); - } - new_guard -} - -fn abort_selection(guard: &mut MutexGuard<'_, State>) -> bool { - match mem::replace(&mut guard.blocker, NoneBlocked) { - NoneBlocked => true, - BlockedSender(token) => { - guard.blocker = BlockedSender(token); - true - } - BlockedReceiver(token) => { - drop(token); - false - } - } -} - -/// Wakes up a thread, dropping the lock at the correct time -fn wakeup(token: SignalToken, guard: MutexGuard<'_, State>) { - // We need to be careful to wake up the waiting thread *outside* of the mutex - // in case it incurs a context switch. - drop(guard); - token.signal(); -} - -impl Packet { - pub fn new(capacity: usize) -> Packet { - Packet { - channels: AtomicUsize::new(1), - lock: Mutex::new(State { - disconnected: false, - blocker: NoneBlocked, - cap: capacity, - canceled: None, - queue: Queue { head: ptr::null_mut(), tail: ptr::null_mut() }, - buf: Buffer { - buf: (0..capacity + if capacity == 0 { 1 } else { 0 }).map(|_| None).collect(), - start: 0, - size: 0, - }, - }), - } - } - - // wait until a send slot is available, returning locked access to - // the channel state. - fn acquire_send_slot(&self) -> MutexGuard<'_, State> { - let mut node = Node { token: None, next: ptr::null_mut() }; - loop { - let mut guard = self.lock.lock().unwrap(); - // are we ready to go? - if guard.disconnected || guard.buf.size() < guard.buf.capacity() { - return guard; - } - // no room; actually block - let wait_token = guard.queue.enqueue(&mut node); - drop(guard); - wait_token.wait(); - } - } - - pub fn send(&self, t: T) -> Result<(), T> { - let mut guard = self.acquire_send_slot(); - if guard.disconnected { - return Err(t); - } - guard.buf.enqueue(t); - - match mem::replace(&mut guard.blocker, NoneBlocked) { - // if our capacity is 0, then we need to wait for a receiver to be - // available to take our data. After waiting, we check again to make - // sure the port didn't go away in the meantime. If it did, we need - // to hand back our data. - NoneBlocked if guard.cap == 0 => { - let mut canceled = false; - assert!(guard.canceled.is_none()); - guard.canceled = Some(unsafe { mem::transmute(&mut canceled) }); - let mut guard = wait(&self.lock, guard, BlockedSender); - if canceled { Err(guard.buf.dequeue()) } else { Ok(()) } - } - - // success, we buffered some data - NoneBlocked => Ok(()), - - // success, someone's about to receive our buffered data. - BlockedReceiver(token) => { - wakeup(token, guard); - Ok(()) - } - - BlockedSender(..) => panic!("lolwut"), - } - } - - pub fn try_send(&self, t: T) -> Result<(), super::TrySendError> { - let mut guard = self.lock.lock().unwrap(); - if guard.disconnected { - Err(super::TrySendError::Disconnected(t)) - } else if guard.buf.size() == guard.buf.capacity() { - Err(super::TrySendError::Full(t)) - } else if guard.cap == 0 { - // With capacity 0, even though we have buffer space we can't - // transfer the data unless there's a receiver waiting. - match mem::replace(&mut guard.blocker, NoneBlocked) { - NoneBlocked => Err(super::TrySendError::Full(t)), - BlockedSender(..) => unreachable!(), - BlockedReceiver(token) => { - guard.buf.enqueue(t); - wakeup(token, guard); - Ok(()) - } - } - } else { - // If the buffer has some space and the capacity isn't 0, then we - // just enqueue the data for later retrieval, ensuring to wake up - // any blocked receiver if there is one. - assert!(guard.buf.size() < guard.buf.capacity()); - guard.buf.enqueue(t); - match mem::replace(&mut guard.blocker, NoneBlocked) { - BlockedReceiver(token) => wakeup(token, guard), - NoneBlocked => {} - BlockedSender(..) => unreachable!(), - } - Ok(()) - } - } - - // Receives a message from this channel - // - // When reading this, remember that there can only ever be one receiver at - // time. - pub fn recv(&self, deadline: Option) -> Result { - let mut guard = self.lock.lock().unwrap(); - - let mut woke_up_after_waiting = false; - // Wait for the buffer to have something in it. No need for a - // while loop because we're the only receiver. - if !guard.disconnected && guard.buf.size() == 0 { - if let Some(deadline) = deadline { - guard = - wait_timeout_receiver(&self.lock, deadline, guard, &mut woke_up_after_waiting); - } else { - guard = wait(&self.lock, guard, BlockedReceiver); - woke_up_after_waiting = true; - } - } - - // N.B., channel could be disconnected while waiting, so the order of - // these conditionals is important. - if guard.disconnected && guard.buf.size() == 0 { - return Err(Disconnected); - } - - // Pick up the data, wake up our neighbors, and carry on - assert!(guard.buf.size() > 0 || (deadline.is_some() && !woke_up_after_waiting)); - - if guard.buf.size() == 0 { - return Err(Empty); - } - - let ret = guard.buf.dequeue(); - self.wakeup_senders(woke_up_after_waiting, guard); - Ok(ret) - } - - pub fn try_recv(&self) -> Result { - let mut guard = self.lock.lock().unwrap(); - - // Easy cases first - if guard.disconnected && guard.buf.size() == 0 { - return Err(Disconnected); - } - if guard.buf.size() == 0 { - return Err(Empty); - } - - // Be sure to wake up neighbors - let ret = Ok(guard.buf.dequeue()); - self.wakeup_senders(false, guard); - ret - } - - // Wake up pending senders after some data has been received - // - // * `waited` - flag if the receiver blocked to receive some data, or if it - // just picked up some data on the way out - // * `guard` - the lock guard that is held over this channel's lock - fn wakeup_senders(&self, waited: bool, mut guard: MutexGuard<'_, State>) { - let pending_sender1: Option = guard.queue.dequeue(); - - // If this is a no-buffer channel (cap == 0), then if we didn't wait we - // need to ACK the sender. If we waited, then the sender waking us up - // was already the ACK. - let pending_sender2 = if guard.cap == 0 && !waited { - match mem::replace(&mut guard.blocker, NoneBlocked) { - NoneBlocked => None, - BlockedReceiver(..) => unreachable!(), - BlockedSender(token) => { - guard.canceled.take(); - Some(token) - } - } - } else { - None - }; - mem::drop(guard); - - // only outside of the lock do we wake up the pending threads - pending_sender1.map(|t| t.signal()); - pending_sender2.map(|t| t.signal()); - } - - // Prepares this shared packet for a channel clone, essentially just bumping - // a refcount. - pub fn clone_chan(&self) { - let old_count = self.channels.fetch_add(1, Ordering::SeqCst); - - // See comments on Arc::clone() on why we do this (for `mem::forget`). - if old_count > MAX_REFCOUNT { - unsafe { - abort(); - } - } - } - - pub fn drop_chan(&self) { - // Only flag the channel as disconnected if we're the last channel - match self.channels.fetch_sub(1, Ordering::SeqCst) { - 1 => {} - _ => return, - } - - // Not much to do other than wake up a receiver if one's there - let mut guard = self.lock.lock().unwrap(); - if guard.disconnected { - return; - } - guard.disconnected = true; - match mem::replace(&mut guard.blocker, NoneBlocked) { - NoneBlocked => {} - BlockedSender(..) => unreachable!(), - BlockedReceiver(token) => wakeup(token, guard), - } - } - - pub fn drop_port(&self) { - let mut guard = self.lock.lock().unwrap(); - - if guard.disconnected { - return; - } - guard.disconnected = true; - - // If the capacity is 0, then the sender may want its data back after - // we're disconnected. Otherwise it's now our responsibility to destroy - // the buffered data. As with many other portions of this code, this - // needs to be careful to destroy the data *outside* of the lock to - // prevent deadlock. - let _data = if guard.cap != 0 { mem::take(&mut guard.buf.buf) } else { Vec::new() }; - let mut queue = - mem::replace(&mut guard.queue, Queue { head: ptr::null_mut(), tail: ptr::null_mut() }); - - let waiter = match mem::replace(&mut guard.blocker, NoneBlocked) { - NoneBlocked => None, - BlockedSender(token) => { - *guard.canceled.take().unwrap() = true; - Some(token) - } - BlockedReceiver(..) => unreachable!(), - }; - mem::drop(guard); - - while let Some(token) = queue.dequeue() { - token.signal(); - } - waiter.map(|t| t.signal()); - } -} - -impl Drop for Packet { - fn drop(&mut self) { - assert_eq!(self.channels.load(Ordering::SeqCst), 0); - let mut guard = self.lock.lock().unwrap(); - assert!(guard.queue.dequeue().is_none()); - assert!(guard.canceled.is_none()); - } -} - -//////////////////////////////////////////////////////////////////////////////// -// Buffer, a simple ring buffer backed by Vec -//////////////////////////////////////////////////////////////////////////////// - -impl Buffer { - fn enqueue(&mut self, t: T) { - let pos = (self.start + self.size) % self.buf.len(); - self.size += 1; - let prev = mem::replace(&mut self.buf[pos], Some(t)); - assert!(prev.is_none()); - } - - fn dequeue(&mut self) -> T { - let start = self.start; - self.size -= 1; - self.start = (self.start + 1) % self.buf.len(); - let result = &mut self.buf[start]; - result.take().unwrap() - } - - fn size(&self) -> usize { - self.size - } - fn capacity(&self) -> usize { - self.buf.len() - } -} - -//////////////////////////////////////////////////////////////////////////////// -// Queue, a simple queue to enqueue threads with (stack-allocated nodes) -//////////////////////////////////////////////////////////////////////////////// - -impl Queue { - fn enqueue(&mut self, node: &mut Node) -> WaitToken { - let (wait_token, signal_token) = blocking::tokens(); - node.token = Some(signal_token); - node.next = ptr::null_mut(); - - if self.tail.is_null() { - self.head = node as *mut Node; - self.tail = node as *mut Node; - } else { - unsafe { - (*self.tail).next = node as *mut Node; - self.tail = node as *mut Node; - } - } - - wait_token - } - - fn dequeue(&mut self) -> Option { - if self.head.is_null() { - return None; - } - let node = self.head; - self.head = unsafe { (*node).next }; - if self.head.is_null() { - self.tail = ptr::null_mut(); - } - unsafe { - (*node).next = ptr::null_mut(); - Some((*node).token.take().unwrap()) - } - } -} diff --git a/crux-mir/lib/std/src/sync/mpsc/sync_tests.rs b/crux-mir/lib/std/src/sync/mpsc/sync_tests.rs new file mode 100644 index 000000000..9d2f92ffc --- /dev/null +++ b/crux-mir/lib/std/src/sync/mpsc/sync_tests.rs @@ -0,0 +1,658 @@ +use super::*; +use crate::env; +use crate::sync::mpmc::SendTimeoutError; +use crate::thread; +use crate::time::Duration; + +pub fn stress_factor() -> usize { + match env::var("RUST_TEST_STRESS") { + Ok(val) => val.parse().unwrap(), + Err(..) => 1, + } +} + +#[test] +fn smoke() { + let (tx, rx) = sync_channel::(1); + tx.send(1).unwrap(); + assert_eq!(rx.recv().unwrap(), 1); +} + +#[test] +fn drop_full() { + let (tx, _rx) = sync_channel::>(1); + tx.send(Box::new(1)).unwrap(); +} + +#[test] +fn smoke_shared() { + let (tx, rx) = sync_channel::(1); + tx.send(1).unwrap(); + assert_eq!(rx.recv().unwrap(), 1); + let tx = tx.clone(); + tx.send(1).unwrap(); + assert_eq!(rx.recv().unwrap(), 1); +} + +#[test] +fn recv_timeout() { + let (tx, rx) = sync_channel::(1); + assert_eq!(rx.recv_timeout(Duration::from_millis(1)), Err(RecvTimeoutError::Timeout)); + tx.send(1).unwrap(); + assert_eq!(rx.recv_timeout(Duration::from_millis(1)), Ok(1)); +} + +#[test] +fn send_timeout() { + let (tx, _rx) = sync_channel::(1); + assert_eq!(tx.send_timeout(1, Duration::from_millis(1)), Ok(())); + assert_eq!(tx.send_timeout(1, Duration::from_millis(1)), Err(SendTimeoutError::Timeout(1))); +} + +#[test] +fn smoke_threads() { + let (tx, rx) = sync_channel::(0); + let _t = thread::spawn(move || { + tx.send(1).unwrap(); + }); + assert_eq!(rx.recv().unwrap(), 1); +} + +#[test] +fn smoke_port_gone() { + let (tx, rx) = sync_channel::(0); + drop(rx); + assert!(tx.send(1).is_err()); +} + +#[test] +fn smoke_shared_port_gone2() { + let (tx, rx) = sync_channel::(0); + drop(rx); + let tx2 = tx.clone(); + drop(tx); + assert!(tx2.send(1).is_err()); +} + +#[test] +fn port_gone_concurrent() { + let (tx, rx) = sync_channel::(0); + let _t = thread::spawn(move || { + rx.recv().unwrap(); + }); + while tx.send(1).is_ok() {} +} + +#[test] +fn port_gone_concurrent_shared() { + let (tx, rx) = sync_channel::(0); + let tx2 = tx.clone(); + let _t = thread::spawn(move || { + rx.recv().unwrap(); + }); + while tx.send(1).is_ok() && tx2.send(1).is_ok() {} +} + +#[test] +fn smoke_chan_gone() { + let (tx, rx) = sync_channel::(0); + drop(tx); + assert!(rx.recv().is_err()); +} + +#[test] +fn smoke_chan_gone_shared() { + let (tx, rx) = sync_channel::<()>(0); + let tx2 = tx.clone(); + drop(tx); + drop(tx2); + assert!(rx.recv().is_err()); +} + +#[test] +fn chan_gone_concurrent() { + let (tx, rx) = sync_channel::(0); + thread::spawn(move || { + tx.send(1).unwrap(); + tx.send(1).unwrap(); + }); + while rx.recv().is_ok() {} +} + +#[test] +fn stress() { + let count = if cfg!(miri) { 100 } else { 10000 }; + let (tx, rx) = sync_channel::(0); + thread::spawn(move || { + for _ in 0..count { + tx.send(1).unwrap(); + } + }); + for _ in 0..count { + assert_eq!(rx.recv().unwrap(), 1); + } +} + +#[test] +fn stress_recv_timeout_two_threads() { + let count = if cfg!(miri) { 100 } else { 10000 }; + let (tx, rx) = sync_channel::(0); + + thread::spawn(move || { + for _ in 0..count { + tx.send(1).unwrap(); + } + }); + + let mut recv_count = 0; + loop { + match rx.recv_timeout(Duration::from_millis(1)) { + Ok(v) => { + assert_eq!(v, 1); + recv_count += 1; + } + Err(RecvTimeoutError::Timeout) => continue, + Err(RecvTimeoutError::Disconnected) => break, + } + } + + assert_eq!(recv_count, count); +} + +#[test] +fn stress_recv_timeout_shared() { + const AMT: u32 = if cfg!(miri) { 100 } else { 1000 }; + const NTHREADS: u32 = 8; + let (tx, rx) = sync_channel::(0); + let (dtx, drx) = sync_channel::<()>(0); + + thread::spawn(move || { + let mut recv_count = 0; + loop { + match rx.recv_timeout(Duration::from_millis(10)) { + Ok(v) => { + assert_eq!(v, 1); + recv_count += 1; + } + Err(RecvTimeoutError::Timeout) => continue, + Err(RecvTimeoutError::Disconnected) => break, + } + } + + assert_eq!(recv_count, AMT * NTHREADS); + assert!(rx.try_recv().is_err()); + + dtx.send(()).unwrap(); + }); + + for _ in 0..NTHREADS { + let tx = tx.clone(); + thread::spawn(move || { + for _ in 0..AMT { + tx.send(1).unwrap(); + } + }); + } + + drop(tx); + + drx.recv().unwrap(); +} + +#[test] +fn stress_shared() { + const AMT: u32 = if cfg!(miri) { 100 } else { 1000 }; + const NTHREADS: u32 = 8; + let (tx, rx) = sync_channel::(0); + let (dtx, drx) = sync_channel::<()>(0); + + thread::spawn(move || { + for _ in 0..AMT * NTHREADS { + assert_eq!(rx.recv().unwrap(), 1); + } + match rx.try_recv() { + Ok(..) => panic!(), + _ => {} + } + dtx.send(()).unwrap(); + }); + + for _ in 0..NTHREADS { + let tx = tx.clone(); + thread::spawn(move || { + for _ in 0..AMT { + tx.send(1).unwrap(); + } + }); + } + drop(tx); + drx.recv().unwrap(); +} + +#[test] +fn oneshot_single_thread_close_port_first() { + // Simple test of closing without sending + let (_tx, rx) = sync_channel::(0); + drop(rx); +} + +#[test] +fn oneshot_single_thread_close_chan_first() { + // Simple test of closing without sending + let (tx, _rx) = sync_channel::(0); + drop(tx); +} + +#[test] +fn oneshot_single_thread_send_port_close() { + // Testing that the sender cleans up the payload if receiver is closed + let (tx, rx) = sync_channel::>(0); + drop(rx); + assert!(tx.send(Box::new(0)).is_err()); +} + +#[test] +fn oneshot_single_thread_recv_chan_close() { + // Receiving on a closed chan will panic + let res = thread::spawn(move || { + let (tx, rx) = sync_channel::(0); + drop(tx); + rx.recv().unwrap(); + }) + .join(); + // What is our res? + assert!(res.is_err()); +} + +#[test] +fn oneshot_single_thread_send_then_recv() { + let (tx, rx) = sync_channel::>(1); + tx.send(Box::new(10)).unwrap(); + assert!(*rx.recv().unwrap() == 10); +} + +#[test] +fn oneshot_single_thread_try_send_open() { + let (tx, rx) = sync_channel::(1); + assert_eq!(tx.try_send(10), Ok(())); + assert!(rx.recv().unwrap() == 10); +} + +#[test] +fn oneshot_single_thread_try_send_closed() { + let (tx, rx) = sync_channel::(0); + drop(rx); + assert_eq!(tx.try_send(10), Err(TrySendError::Disconnected(10))); +} + +#[test] +fn oneshot_single_thread_try_send_closed2() { + let (tx, _rx) = sync_channel::(0); + assert_eq!(tx.try_send(10), Err(TrySendError::Full(10))); +} + +#[test] +fn oneshot_single_thread_try_recv_open() { + let (tx, rx) = sync_channel::(1); + tx.send(10).unwrap(); + assert!(rx.recv() == Ok(10)); +} + +#[test] +fn oneshot_single_thread_try_recv_closed() { + let (tx, rx) = sync_channel::(0); + drop(tx); + assert!(rx.recv().is_err()); +} + +#[test] +fn oneshot_single_thread_try_recv_closed_with_data() { + let (tx, rx) = sync_channel::(1); + tx.send(10).unwrap(); + drop(tx); + assert_eq!(rx.try_recv(), Ok(10)); + assert_eq!(rx.try_recv(), Err(TryRecvError::Disconnected)); +} + +#[test] +fn oneshot_single_thread_peek_data() { + let (tx, rx) = sync_channel::(1); + assert_eq!(rx.try_recv(), Err(TryRecvError::Empty)); + tx.send(10).unwrap(); + assert_eq!(rx.try_recv(), Ok(10)); +} + +#[test] +fn oneshot_single_thread_peek_close() { + let (tx, rx) = sync_channel::(0); + drop(tx); + assert_eq!(rx.try_recv(), Err(TryRecvError::Disconnected)); + assert_eq!(rx.try_recv(), Err(TryRecvError::Disconnected)); +} + +#[test] +fn oneshot_single_thread_peek_open() { + let (_tx, rx) = sync_channel::(0); + assert_eq!(rx.try_recv(), Err(TryRecvError::Empty)); +} + +#[test] +fn oneshot_multi_task_recv_then_send() { + let (tx, rx) = sync_channel::>(0); + let _t = thread::spawn(move || { + assert!(*rx.recv().unwrap() == 10); + }); + + tx.send(Box::new(10)).unwrap(); +} + +#[test] +fn oneshot_multi_task_recv_then_close() { + let (tx, rx) = sync_channel::>(0); + let _t = thread::spawn(move || { + drop(tx); + }); + let res = thread::spawn(move || { + assert!(*rx.recv().unwrap() == 10); + }) + .join(); + assert!(res.is_err()); +} + +#[test] +fn oneshot_multi_thread_close_stress() { + for _ in 0..stress_factor() { + let (tx, rx) = sync_channel::(0); + let _t = thread::spawn(move || { + drop(rx); + }); + drop(tx); + } +} + +#[test] +fn oneshot_multi_thread_send_close_stress() { + for _ in 0..stress_factor() { + let (tx, rx) = sync_channel::(0); + let _t = thread::spawn(move || { + drop(rx); + }); + let _ = thread::spawn(move || { + tx.send(1).unwrap(); + }) + .join(); + } +} + +#[test] +fn oneshot_multi_thread_recv_close_stress() { + for _ in 0..stress_factor() { + let (tx, rx) = sync_channel::(0); + let _t = thread::spawn(move || { + let res = thread::spawn(move || { + rx.recv().unwrap(); + }) + .join(); + assert!(res.is_err()); + }); + let _t = thread::spawn(move || { + thread::spawn(move || { + drop(tx); + }); + }); + } +} + +#[test] +fn oneshot_multi_thread_send_recv_stress() { + for _ in 0..stress_factor() { + let (tx, rx) = sync_channel::>(0); + let _t = thread::spawn(move || { + tx.send(Box::new(10)).unwrap(); + }); + assert!(*rx.recv().unwrap() == 10); + } +} + +#[test] +fn stream_send_recv_stress() { + for _ in 0..stress_factor() { + let (tx, rx) = sync_channel::>(0); + + send(tx, 0); + recv(rx, 0); + + fn send(tx: SyncSender>, i: i32) { + if i == 10 { + return; + } + + thread::spawn(move || { + tx.send(Box::new(i)).unwrap(); + send(tx, i + 1); + }); + } + + fn recv(rx: Receiver>, i: i32) { + if i == 10 { + return; + } + + thread::spawn(move || { + assert!(*rx.recv().unwrap() == i); + recv(rx, i + 1); + }); + } + } +} + +#[test] +fn recv_a_lot() { + let count = if cfg!(miri) { 1000 } else { 10000 }; + // Regression test that we don't run out of stack in scheduler context + let (tx, rx) = sync_channel(count); + for _ in 0..count { + tx.send(()).unwrap(); + } + for _ in 0..count { + rx.recv().unwrap(); + } +} + +#[test] +fn shared_chan_stress() { + let (tx, rx) = sync_channel(0); + let total = stress_factor() + 100; + for _ in 0..total { + let tx = tx.clone(); + thread::spawn(move || { + tx.send(()).unwrap(); + }); + } + + for _ in 0..total { + rx.recv().unwrap(); + } +} + +#[test] +fn test_nested_recv_iter() { + let (tx, rx) = sync_channel::(0); + let (total_tx, total_rx) = sync_channel::(0); + + let _t = thread::spawn(move || { + let mut acc = 0; + for x in rx.iter() { + acc += x; + } + total_tx.send(acc).unwrap(); + }); + + tx.send(3).unwrap(); + tx.send(1).unwrap(); + tx.send(2).unwrap(); + drop(tx); + assert_eq!(total_rx.recv().unwrap(), 6); +} + +#[test] +fn test_recv_iter_break() { + let (tx, rx) = sync_channel::(0); + let (count_tx, count_rx) = sync_channel(0); + + let _t = thread::spawn(move || { + let mut count = 0; + for x in rx.iter() { + if count >= 3 { + break; + } else { + count += x; + } + } + count_tx.send(count).unwrap(); + }); + + tx.send(2).unwrap(); + tx.send(2).unwrap(); + tx.send(2).unwrap(); + let _ = tx.try_send(2); + drop(tx); + assert_eq!(count_rx.recv().unwrap(), 4); +} + +#[test] +fn try_recv_states() { + let (tx1, rx1) = sync_channel::(1); + let (tx2, rx2) = sync_channel::<()>(1); + let (tx3, rx3) = sync_channel::<()>(1); + let _t = thread::spawn(move || { + rx2.recv().unwrap(); + tx1.send(1).unwrap(); + tx3.send(()).unwrap(); + rx2.recv().unwrap(); + drop(tx1); + tx3.send(()).unwrap(); + }); + + assert_eq!(rx1.try_recv(), Err(TryRecvError::Empty)); + tx2.send(()).unwrap(); + rx3.recv().unwrap(); + assert_eq!(rx1.try_recv(), Ok(1)); + assert_eq!(rx1.try_recv(), Err(TryRecvError::Empty)); + tx2.send(()).unwrap(); + rx3.recv().unwrap(); + assert_eq!(rx1.try_recv(), Err(TryRecvError::Disconnected)); +} + +// This bug used to end up in a livelock inside of the Receiver destructor +// because the internal state of the Shared packet was corrupted +#[test] +fn destroy_upgraded_shared_port_when_sender_still_active() { + let (tx, rx) = sync_channel::<()>(0); + let (tx2, rx2) = sync_channel::<()>(0); + let _t = thread::spawn(move || { + rx.recv().unwrap(); // wait on a oneshot + drop(rx); // destroy a shared + tx2.send(()).unwrap(); + }); + // make sure the other thread has gone to sleep + for _ in 0..5000 { + thread::yield_now(); + } + + // upgrade to a shared chan and send a message + let t = tx.clone(); + drop(tx); + t.send(()).unwrap(); + + // wait for the child thread to exit before we exit + rx2.recv().unwrap(); +} + +#[test] +fn send1() { + let (tx, rx) = sync_channel::(0); + let _t = thread::spawn(move || { + rx.recv().unwrap(); + }); + assert_eq!(tx.send(1), Ok(())); +} + +#[test] +fn send2() { + let (tx, rx) = sync_channel::(0); + let _t = thread::spawn(move || { + drop(rx); + }); + assert!(tx.send(1).is_err()); +} + +#[test] +fn send3() { + let (tx, rx) = sync_channel::(1); + assert_eq!(tx.send(1), Ok(())); + let _t = thread::spawn(move || { + drop(rx); + }); + assert!(tx.send(1).is_err()); +} + +#[test] +fn send4() { + let (tx, rx) = sync_channel::(0); + let tx2 = tx.clone(); + let (done, donerx) = channel(); + let done2 = done.clone(); + let _t = thread::spawn(move || { + assert!(tx.send(1).is_err()); + done.send(()).unwrap(); + }); + let _t = thread::spawn(move || { + assert!(tx2.send(2).is_err()); + done2.send(()).unwrap(); + }); + drop(rx); + donerx.recv().unwrap(); + donerx.recv().unwrap(); +} + +#[test] +fn try_send1() { + let (tx, _rx) = sync_channel::(0); + assert_eq!(tx.try_send(1), Err(TrySendError::Full(1))); +} + +#[test] +fn try_send2() { + let (tx, _rx) = sync_channel::(1); + assert_eq!(tx.try_send(1), Ok(())); + assert_eq!(tx.try_send(1), Err(TrySendError::Full(1))); +} + +#[test] +fn try_send3() { + let (tx, rx) = sync_channel::(1); + assert_eq!(tx.try_send(1), Ok(())); + drop(rx); + assert_eq!(tx.try_send(1), Err(TrySendError::Disconnected(1))); +} + +#[test] +fn issue_15761() { + fn repro() { + let (tx1, rx1) = sync_channel::<()>(3); + let (tx2, rx2) = sync_channel::<()>(3); + + let _t = thread::spawn(move || { + rx1.recv().unwrap(); + tx2.try_send(()).unwrap(); + }); + + tx1.try_send(()).unwrap(); + rx2.recv().unwrap(); + } + + for _ in 0..100 { + repro() + } +} diff --git a/crux-mir/lib/std/src/sync/mpsc/tests.rs b/crux-mir/lib/std/src/sync/mpsc/tests.rs new file mode 100644 index 000000000..1e52a4a70 --- /dev/null +++ b/crux-mir/lib/std/src/sync/mpsc/tests.rs @@ -0,0 +1,723 @@ +use super::*; +use crate::env; +use crate::thread; +use crate::time::{Duration, Instant}; + +pub fn stress_factor() -> usize { + match env::var("RUST_TEST_STRESS") { + Ok(val) => val.parse().unwrap(), + Err(..) => 1, + } +} + +#[test] +fn smoke() { + let (tx, rx) = channel::(); + tx.send(1).unwrap(); + assert_eq!(rx.recv().unwrap(), 1); +} + +#[test] +fn drop_full() { + let (tx, _rx) = channel::>(); + tx.send(Box::new(1)).unwrap(); +} + +#[test] +fn drop_full_shared() { + let (tx, _rx) = channel::>(); + drop(tx.clone()); + drop(tx.clone()); + tx.send(Box::new(1)).unwrap(); +} + +#[test] +fn smoke_shared() { + let (tx, rx) = channel::(); + tx.send(1).unwrap(); + assert_eq!(rx.recv().unwrap(), 1); + let tx = tx.clone(); + tx.send(1).unwrap(); + assert_eq!(rx.recv().unwrap(), 1); +} + +#[test] +fn smoke_threads() { + let (tx, rx) = channel::(); + let _t = thread::spawn(move || { + tx.send(1).unwrap(); + }); + assert_eq!(rx.recv().unwrap(), 1); +} + +#[test] +fn smoke_port_gone() { + let (tx, rx) = channel::(); + drop(rx); + assert!(tx.send(1).is_err()); +} + +#[test] +fn smoke_shared_port_gone() { + let (tx, rx) = channel::(); + drop(rx); + assert!(tx.send(1).is_err()) +} + +#[test] +fn smoke_shared_port_gone2() { + let (tx, rx) = channel::(); + drop(rx); + let tx2 = tx.clone(); + drop(tx); + assert!(tx2.send(1).is_err()); +} + +#[test] +fn port_gone_concurrent() { + let (tx, rx) = channel::(); + let _t = thread::spawn(move || { + rx.recv().unwrap(); + }); + while tx.send(1).is_ok() {} +} + +#[test] +fn port_gone_concurrent_shared() { + let (tx, rx) = channel::(); + let tx2 = tx.clone(); + let _t = thread::spawn(move || { + rx.recv().unwrap(); + }); + while tx.send(1).is_ok() && tx2.send(1).is_ok() {} +} + +#[test] +fn smoke_chan_gone() { + let (tx, rx) = channel::(); + drop(tx); + assert!(rx.recv().is_err()); +} + +#[test] +fn smoke_chan_gone_shared() { + let (tx, rx) = channel::<()>(); + let tx2 = tx.clone(); + drop(tx); + drop(tx2); + assert!(rx.recv().is_err()); +} + +#[test] +fn chan_gone_concurrent() { + let (tx, rx) = channel::(); + let _t = thread::spawn(move || { + tx.send(1).unwrap(); + tx.send(1).unwrap(); + }); + while rx.recv().is_ok() {} +} + +#[test] +fn stress() { + let count = if cfg!(miri) { 100 } else { 10000 }; + let (tx, rx) = channel::(); + let t = thread::spawn(move || { + for _ in 0..count { + tx.send(1).unwrap(); + } + }); + for _ in 0..count { + assert_eq!(rx.recv().unwrap(), 1); + } + t.join().ok().expect("thread panicked"); +} + +#[test] +fn stress_shared() { + const AMT: u32 = if cfg!(miri) { 100 } else { 10000 }; + const NTHREADS: u32 = 8; + let (tx, rx) = channel::(); + + let t = thread::spawn(move || { + for _ in 0..AMT * NTHREADS { + assert_eq!(rx.recv().unwrap(), 1); + } + match rx.try_recv() { + Ok(..) => panic!(), + _ => {} + } + }); + + for _ in 0..NTHREADS { + let tx = tx.clone(); + thread::spawn(move || { + for _ in 0..AMT { + tx.send(1).unwrap(); + } + }); + } + drop(tx); + t.join().ok().expect("thread panicked"); +} + +#[test] +fn send_from_outside_runtime() { + let (tx1, rx1) = channel::<()>(); + let (tx2, rx2) = channel::(); + let t1 = thread::spawn(move || { + tx1.send(()).unwrap(); + for _ in 0..40 { + assert_eq!(rx2.recv().unwrap(), 1); + } + }); + rx1.recv().unwrap(); + let t2 = thread::spawn(move || { + for _ in 0..40 { + tx2.send(1).unwrap(); + } + }); + t1.join().ok().expect("thread panicked"); + t2.join().ok().expect("thread panicked"); +} + +#[test] +fn recv_from_outside_runtime() { + let (tx, rx) = channel::(); + let t = thread::spawn(move || { + for _ in 0..40 { + assert_eq!(rx.recv().unwrap(), 1); + } + }); + for _ in 0..40 { + tx.send(1).unwrap(); + } + t.join().ok().expect("thread panicked"); +} + +#[test] +fn no_runtime() { + let (tx1, rx1) = channel::(); + let (tx2, rx2) = channel::(); + let t1 = thread::spawn(move || { + assert_eq!(rx1.recv().unwrap(), 1); + tx2.send(2).unwrap(); + }); + let t2 = thread::spawn(move || { + tx1.send(1).unwrap(); + assert_eq!(rx2.recv().unwrap(), 2); + }); + t1.join().ok().expect("thread panicked"); + t2.join().ok().expect("thread panicked"); +} + +#[test] +fn oneshot_single_thread_close_port_first() { + // Simple test of closing without sending + let (_tx, rx) = channel::(); + drop(rx); +} + +#[test] +fn oneshot_single_thread_close_chan_first() { + // Simple test of closing without sending + let (tx, _rx) = channel::(); + drop(tx); +} + +#[test] +fn oneshot_single_thread_send_port_close() { + // Testing that the sender cleans up the payload if receiver is closed + let (tx, rx) = channel::>(); + drop(rx); + assert!(tx.send(Box::new(0)).is_err()); +} + +#[test] +fn oneshot_single_thread_recv_chan_close() { + // Receiving on a closed chan will panic + let res = thread::spawn(move || { + let (tx, rx) = channel::(); + drop(tx); + rx.recv().unwrap(); + }) + .join(); + // What is our res? + assert!(res.is_err()); +} + +#[test] +fn oneshot_single_thread_send_then_recv() { + let (tx, rx) = channel::>(); + tx.send(Box::new(10)).unwrap(); + assert!(*rx.recv().unwrap() == 10); +} + +#[test] +fn oneshot_single_thread_try_send_open() { + let (tx, rx) = channel::(); + assert!(tx.send(10).is_ok()); + assert!(rx.recv().unwrap() == 10); +} + +#[test] +fn oneshot_single_thread_try_send_closed() { + let (tx, rx) = channel::(); + drop(rx); + assert!(tx.send(10).is_err()); +} + +#[test] +fn oneshot_single_thread_try_recv_open() { + let (tx, rx) = channel::(); + tx.send(10).unwrap(); + assert!(rx.recv() == Ok(10)); +} + +#[test] +fn oneshot_single_thread_try_recv_closed() { + let (tx, rx) = channel::(); + drop(tx); + assert!(rx.recv().is_err()); +} + +#[test] +fn oneshot_single_thread_peek_data() { + let (tx, rx) = channel::(); + assert_eq!(rx.try_recv(), Err(TryRecvError::Empty)); + tx.send(10).unwrap(); + assert_eq!(rx.try_recv(), Ok(10)); +} + +#[test] +fn oneshot_single_thread_peek_close() { + let (tx, rx) = channel::(); + drop(tx); + assert_eq!(rx.try_recv(), Err(TryRecvError::Disconnected)); + assert_eq!(rx.try_recv(), Err(TryRecvError::Disconnected)); +} + +#[test] +fn oneshot_single_thread_peek_open() { + let (_tx, rx) = channel::(); + assert_eq!(rx.try_recv(), Err(TryRecvError::Empty)); +} + +#[test] +fn oneshot_multi_task_recv_then_send() { + let (tx, rx) = channel::>(); + let _t = thread::spawn(move || { + assert!(*rx.recv().unwrap() == 10); + }); + + tx.send(Box::new(10)).unwrap(); +} + +#[test] +fn oneshot_multi_task_recv_then_close() { + let (tx, rx) = channel::>(); + let _t = thread::spawn(move || { + drop(tx); + }); + let res = thread::spawn(move || { + assert!(*rx.recv().unwrap() == 10); + }) + .join(); + assert!(res.is_err()); +} + +#[test] +fn oneshot_multi_thread_close_stress() { + for _ in 0..stress_factor() { + let (tx, rx) = channel::(); + let _t = thread::spawn(move || { + drop(rx); + }); + drop(tx); + } +} + +#[test] +fn oneshot_multi_thread_send_close_stress() { + for _ in 0..stress_factor() { + let (tx, rx) = channel::(); + let _t = thread::spawn(move || { + drop(rx); + }); + let _ = thread::spawn(move || { + tx.send(1).unwrap(); + }) + .join(); + } +} + +#[test] +fn oneshot_multi_thread_recv_close_stress() { + for _ in 0..stress_factor() { + let (tx, rx) = channel::(); + thread::spawn(move || { + let res = thread::spawn(move || { + rx.recv().unwrap(); + }) + .join(); + assert!(res.is_err()); + }); + let _t = thread::spawn(move || { + thread::spawn(move || { + drop(tx); + }); + }); + } +} + +#[test] +fn oneshot_multi_thread_send_recv_stress() { + for _ in 0..stress_factor() { + let (tx, rx) = channel::>(); + let _t = thread::spawn(move || { + tx.send(Box::new(10)).unwrap(); + }); + assert!(*rx.recv().unwrap() == 10); + } +} + +#[test] +fn stream_send_recv_stress() { + for _ in 0..stress_factor() { + let (tx, rx) = channel(); + + send(tx, 0); + recv(rx, 0); + + fn send(tx: Sender>, i: i32) { + if i == 10 { + return; + } + + thread::spawn(move || { + tx.send(Box::new(i)).unwrap(); + send(tx, i + 1); + }); + } + + fn recv(rx: Receiver>, i: i32) { + if i == 10 { + return; + } + + thread::spawn(move || { + assert!(*rx.recv().unwrap() == i); + recv(rx, i + 1); + }); + } + } +} + +#[test] +fn oneshot_single_thread_recv_timeout() { + let (tx, rx) = channel(); + tx.send(()).unwrap(); + assert_eq!(rx.recv_timeout(Duration::from_millis(1)), Ok(())); + assert_eq!(rx.recv_timeout(Duration::from_millis(1)), Err(RecvTimeoutError::Timeout)); + tx.send(()).unwrap(); + assert_eq!(rx.recv_timeout(Duration::from_millis(1)), Ok(())); +} + +#[test] +fn stress_recv_timeout_two_threads() { + let (tx, rx) = channel(); + let stress = stress_factor() + 100; + let timeout = Duration::from_millis(100); + + thread::spawn(move || { + for i in 0..stress { + if i % 2 == 0 { + thread::sleep(timeout * 2); + } + tx.send(1usize).unwrap(); + } + }); + + let mut recv_count = 0; + loop { + match rx.recv_timeout(timeout) { + Ok(n) => { + assert_eq!(n, 1usize); + recv_count += 1; + } + Err(RecvTimeoutError::Timeout) => continue, + Err(RecvTimeoutError::Disconnected) => break, + } + } + + assert_eq!(recv_count, stress); +} + +#[test] +fn recv_timeout_upgrade() { + let (tx, rx) = channel::<()>(); + let timeout = Duration::from_millis(1); + let _tx_clone = tx.clone(); + + let start = Instant::now(); + assert_eq!(rx.recv_timeout(timeout), Err(RecvTimeoutError::Timeout)); + assert!(Instant::now() >= start + timeout); +} + +#[test] +fn stress_recv_timeout_shared() { + let (tx, rx) = channel(); + let stress = stress_factor() + 100; + + for i in 0..stress { + let tx = tx.clone(); + thread::spawn(move || { + thread::sleep(Duration::from_millis(i as u64 * 10)); + tx.send(1usize).unwrap(); + }); + } + + drop(tx); + + let mut recv_count = 0; + loop { + match rx.recv_timeout(Duration::from_millis(10)) { + Ok(n) => { + assert_eq!(n, 1usize); + recv_count += 1; + } + Err(RecvTimeoutError::Timeout) => continue, + Err(RecvTimeoutError::Disconnected) => break, + } + } + + assert_eq!(recv_count, stress); +} + +#[test] +fn very_long_recv_timeout_wont_panic() { + let (tx, rx) = channel::<()>(); + let join_handle = thread::spawn(move || rx.recv_timeout(Duration::from_secs(u64::MAX))); + thread::sleep(Duration::from_secs(1)); + assert!(tx.send(()).is_ok()); + assert_eq!(join_handle.join().unwrap(), Ok(())); +} + +#[test] +fn recv_a_lot() { + let count = if cfg!(miri) { 1000 } else { 10000 }; + // Regression test that we don't run out of stack in scheduler context + let (tx, rx) = channel(); + for _ in 0..count { + tx.send(()).unwrap(); + } + for _ in 0..count { + rx.recv().unwrap(); + } +} + +#[test] +fn shared_recv_timeout() { + let (tx, rx) = channel(); + let total = 5; + for _ in 0..total { + let tx = tx.clone(); + thread::spawn(move || { + tx.send(()).unwrap(); + }); + } + + for _ in 0..total { + rx.recv().unwrap(); + } + + assert_eq!(rx.recv_timeout(Duration::from_millis(1)), Err(RecvTimeoutError::Timeout)); + tx.send(()).unwrap(); + assert_eq!(rx.recv_timeout(Duration::from_millis(1)), Ok(())); +} + +#[test] +fn shared_chan_stress() { + let (tx, rx) = channel(); + let total = stress_factor() + 100; + for _ in 0..total { + let tx = tx.clone(); + thread::spawn(move || { + tx.send(()).unwrap(); + }); + } + + for _ in 0..total { + rx.recv().unwrap(); + } +} + +#[test] +fn test_nested_recv_iter() { + let (tx, rx) = channel::(); + let (total_tx, total_rx) = channel::(); + + let _t = thread::spawn(move || { + let mut acc = 0; + for x in rx.iter() { + acc += x; + } + total_tx.send(acc).unwrap(); + }); + + tx.send(3).unwrap(); + tx.send(1).unwrap(); + tx.send(2).unwrap(); + drop(tx); + assert_eq!(total_rx.recv().unwrap(), 6); +} + +#[test] +fn test_recv_iter_break() { + let (tx, rx) = channel::(); + let (count_tx, count_rx) = channel(); + + let _t = thread::spawn(move || { + let mut count = 0; + for x in rx.iter() { + if count >= 3 { + break; + } else { + count += x; + } + } + count_tx.send(count).unwrap(); + }); + + tx.send(2).unwrap(); + tx.send(2).unwrap(); + tx.send(2).unwrap(); + let _ = tx.send(2); + drop(tx); + assert_eq!(count_rx.recv().unwrap(), 4); +} + +#[test] +fn test_recv_try_iter() { + let (request_tx, request_rx) = channel(); + let (response_tx, response_rx) = channel(); + + // Request `x`s until we have `6`. + let t = thread::spawn(move || { + let mut count = 0; + loop { + for x in response_rx.try_iter() { + count += x; + if count == 6 { + return count; + } + } + request_tx.send(()).unwrap(); + } + }); + + for _ in request_rx.iter() { + if response_tx.send(2).is_err() { + break; + } + } + + assert_eq!(t.join().unwrap(), 6); +} + +#[test] +fn test_recv_into_iter_owned() { + let mut iter = { + let (tx, rx) = channel::(); + tx.send(1).unwrap(); + tx.send(2).unwrap(); + + rx.into_iter() + }; + assert_eq!(iter.next().unwrap(), 1); + assert_eq!(iter.next().unwrap(), 2); + assert_eq!(iter.next().is_none(), true); +} + +#[test] +fn test_recv_into_iter_borrowed() { + let (tx, rx) = channel::(); + tx.send(1).unwrap(); + tx.send(2).unwrap(); + drop(tx); + let mut iter = (&rx).into_iter(); + assert_eq!(iter.next().unwrap(), 1); + assert_eq!(iter.next().unwrap(), 2); + assert_eq!(iter.next().is_none(), true); +} + +#[test] +fn try_recv_states() { + let (tx1, rx1) = channel::(); + let (tx2, rx2) = channel::<()>(); + let (tx3, rx3) = channel::<()>(); + let _t = thread::spawn(move || { + rx2.recv().unwrap(); + tx1.send(1).unwrap(); + tx3.send(()).unwrap(); + rx2.recv().unwrap(); + drop(tx1); + tx3.send(()).unwrap(); + }); + + assert_eq!(rx1.try_recv(), Err(TryRecvError::Empty)); + tx2.send(()).unwrap(); + rx3.recv().unwrap(); + assert_eq!(rx1.try_recv(), Ok(1)); + assert_eq!(rx1.try_recv(), Err(TryRecvError::Empty)); + tx2.send(()).unwrap(); + rx3.recv().unwrap(); + assert_eq!(rx1.try_recv(), Err(TryRecvError::Disconnected)); +} + +// This bug used to end up in a livelock inside of the Receiver destructor +// because the internal state of the Shared packet was corrupted +#[test] +fn destroy_upgraded_shared_port_when_sender_still_active() { + let (tx, rx) = channel(); + let (tx2, rx2) = channel(); + let _t = thread::spawn(move || { + rx.recv().unwrap(); // wait on a oneshot + drop(rx); // destroy a shared + tx2.send(()).unwrap(); + }); + // make sure the other thread has gone to sleep + for _ in 0..5000 { + thread::yield_now(); + } + + // upgrade to a shared chan and send a message + let t = tx.clone(); + drop(tx); + t.send(()).unwrap(); + + // wait for the child thread to exit before we exit + rx2.recv().unwrap(); +} + +#[test] +fn issue_32114() { + let (tx, _) = channel(); + let _ = tx.send(123); + assert_eq!(tx.send(123), Err(SendError(123))); +} + +#[test] +fn issue_39364() { + let (tx, rx) = channel::<()>(); + let t = thread::spawn(move || { + thread::sleep(Duration::from_millis(300)); + let _ = tx.clone(); + // Don't drop; hand back to caller. + tx + }); + + let _ = rx.recv_timeout(Duration::from_millis(500)); + let _tx = t.join().unwrap(); // delay dropping until end of test + let _ = rx.recv_timeout(Duration::from_millis(500)); +} diff --git a/crux-mir/lib/std/src/sync/mutex.rs b/crux-mir/lib/std/src/sync/mutex.rs index 1c2351b08..065045f44 100644 --- a/crux-mir/lib/std/src/sync/mutex.rs +++ b/crux-mir/lib/std/src/sync/mutex.rs @@ -1,19 +1,19 @@ +#[cfg(all(test, not(target_os = "emscripten")))] +mod tests; + use crate::cell::UnsafeCell; use crate::fmt; -use crate::mem; use crate::ops::{Deref, DerefMut}; -use crate::ptr; -use crate::sys_common::mutex as sys; -use crate::sys_common::poison::{self, LockResult, TryLockError, TryLockResult}; +use crate::sync::{poison, LockResult, TryLockError, TryLockResult}; +use crate::sys::locks as sys; /// A mutual exclusion primitive useful for protecting shared data /// /// This mutex will block threads waiting for the lock to become available. The -/// mutex can also be statically initialized or created via a [`new`] -/// constructor. Each mutex has a type parameter which represents the data that -/// it is protecting. The data can only be accessed through the RAII guards -/// returned from [`lock`] and [`try_lock`], which guarantees that the data is only -/// ever accessed when the mutex is locked. +/// mutex can be created via a [`new`] constructor. Each mutex has a type parameter +/// which represents the data that it is protecting. The data can only be accessed +/// through the RAII guards returned from [`lock`] and [`try_lock`], which +/// guarantees that the data is only ever accessed when the mutex is locked. /// /// # Poisoning /// @@ -33,13 +33,12 @@ use crate::sys_common::poison::{self, LockResult, TryLockError, TryLockResult}; /// the guard that would have otherwise been returned on a successful lock. This /// allows access to the data, despite the lock being poisoned. /// -/// [`new`]: #method.new -/// [`lock`]: #method.lock -/// [`try_lock`]: #method.try_lock -/// [`Result`]: ../../std/result/enum.Result.html -/// [`unwrap()`]: ../../std/result/enum.Result.html#method.unwrap -/// [`PoisonError`]: ../../std/sync/struct.PoisonError.html -/// [`into_inner`]: ../../std/sync/struct.PoisonError.html#method.into_inner +/// [`new`]: Self::new +/// [`lock`]: Self::lock +/// [`try_lock`]: Self::try_lock +/// [`unwrap()`]: Result::unwrap +/// [`PoisonError`]: super::PoisonError +/// [`into_inner`]: super::PoisonError::into_inner /// /// # Examples /// @@ -86,7 +85,7 @@ use crate::sys_common::poison::{self, LockResult, TryLockError, TryLockResult}; /// use std::thread; /// /// let lock = Arc::new(Mutex::new(0_u32)); -/// let lock2 = lock.clone(); +/// let lock2 = Arc::clone(&lock); /// /// let _ = thread::spawn(move || -> () { /// // This thread will acquire the mutex first, unwrapping the result of @@ -107,14 +106,64 @@ use crate::sys_common::poison::{self, LockResult, TryLockError, TryLockResult}; /// /// *guard += 1; /// ``` +/// +/// It is sometimes necessary to manually drop the mutex guard to unlock it +/// sooner than the end of the enclosing scope. +/// +/// ``` +/// use std::sync::{Arc, Mutex}; +/// use std::thread; +/// +/// const N: usize = 3; +/// +/// let data_mutex = Arc::new(Mutex::new(vec![1, 2, 3, 4])); +/// let res_mutex = Arc::new(Mutex::new(0)); +/// +/// let mut threads = Vec::with_capacity(N); +/// (0..N).for_each(|_| { +/// let data_mutex_clone = Arc::clone(&data_mutex); +/// let res_mutex_clone = Arc::clone(&res_mutex); +/// +/// threads.push(thread::spawn(move || { +/// let mut data = data_mutex_clone.lock().unwrap(); +/// // This is the result of some important and long-ish work. +/// let result = data.iter().fold(0, |acc, x| acc + x * 2); +/// data.push(result); +/// drop(data); +/// *res_mutex_clone.lock().unwrap() += result; +/// })); +/// }); +/// +/// let mut data = data_mutex.lock().unwrap(); +/// // This is the result of some important and long-ish work. +/// let result = data.iter().fold(0, |acc, x| acc + x * 2); +/// data.push(result); +/// // We drop the `data` explicitly because it's not necessary anymore and the +/// // thread still has work to do. This allow other threads to start working on +/// // the data immediately, without waiting for the rest of the unrelated work +/// // to be done here. +/// // +/// // It's even more important here than in the threads because we `.join` the +/// // threads after that. If we had not dropped the mutex guard, a thread could +/// // be waiting forever for it, causing a deadlock. +/// drop(data); +/// // Here the mutex guard is not assigned to a variable and so, even if the +/// // scope does not end after this line, the mutex is still released: there is +/// // no deadlock. +/// *res_mutex.lock().unwrap() += result; +/// +/// threads.into_iter().for_each(|thread| { +/// thread +/// .join() +/// .expect("The thread creating or execution failed !") +/// }); +/// +/// assert_eq!(*res_mutex.lock().unwrap(), 800); +/// ``` #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "Mutex")] pub struct Mutex { - // Note that this mutex is in a *box*, not inlined into the struct itself. - // Once a native mutex has been used once, its address can never change (it - // can't be moved). This mutex type can be safely moved at any time, so to - // ensure that the native mutex is used correctly we box the inner mutex to - // give it a constant address. - inner: Box, + inner: sys::Mutex, poison: poison::Flag, data: UnsafeCell, } @@ -135,13 +184,15 @@ unsafe impl Sync for Mutex {} /// This structure is created by the [`lock`] and [`try_lock`] methods on /// [`Mutex`]. /// -/// [`Deref`]: ../../std/ops/trait.Deref.html -/// [`DerefMut`]: ../../std/ops/trait.DerefMut.html -/// [`lock`]: struct.Mutex.html#method.lock -/// [`try_lock`]: struct.Mutex.html#method.try_lock -/// [`Mutex`]: struct.Mutex.html +/// [`lock`]: Mutex::lock +/// [`try_lock`]: Mutex::try_lock #[must_use = "if unused the Mutex will immediately unlock"] +#[must_not_suspend = "holding a MutexGuard across suspend \ + points can cause deadlocks, delays, \ + and cause Futures to not implement `Send`"] #[stable(feature = "rust1", since = "1.0.0")] +#[clippy::has_significant_drop] +#[cfg_attr(not(test), rustc_diagnostic_item = "MutexGuard")] pub struct MutexGuard<'a, T: ?Sized + 'a> { lock: &'a Mutex, poison: poison::Guard, @@ -163,20 +214,10 @@ impl Mutex { /// let mutex = Mutex::new(0); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn new(t: T) -> Mutex { - let mut m = Mutex { - inner: box sys::Mutex::new(), - poison: poison::Flag::new(), - data: UnsafeCell::new(t), - }; - unsafe { - m.inner.init(); - } - m - } - #[stable(feature = "rust1", since = "1.0.0")] - pub fn crucible_TEMP_unlock(&self) { - unsafe { sys::raw(&self.inner).unlock() } + #[rustc_const_stable(feature = "const_locks", since = "1.63.0")] + #[inline] + pub const fn new(t: T) -> Mutex { + Mutex { inner: sys::Mutex::new(), poison: poison::Flag::new(), data: UnsafeCell::new(t) } } } @@ -209,7 +250,7 @@ impl Mutex { /// use std::thread; /// /// let mutex = Arc::new(Mutex::new(0)); - /// let c_mutex = mutex.clone(); + /// let c_mutex = Arc::clone(&mutex); /// /// thread::spawn(move || { /// *c_mutex.lock().unwrap() = 10; @@ -219,7 +260,7 @@ impl Mutex { #[stable(feature = "rust1", since = "1.0.0")] pub fn lock(&self) -> LockResult> { unsafe { - self.inner.raw_lock(); + self.inner.lock(); MutexGuard::new(self) } } @@ -235,10 +276,14 @@ impl Mutex { /// # Errors /// /// If another user of this mutex panicked while holding the mutex, then - /// this call will return failure if the mutex would otherwise be - /// acquired. + /// this call will return the [`Poisoned`] error if the mutex would + /// otherwise be acquired. + /// + /// If the mutex could not be acquired because it is already locked, then + /// this call will return the [`WouldBlock`] error. /// - /// [`Err`]: ../../std/result/enum.Result.html#variant.Err + /// [`Poisoned`]: TryLockError::Poisoned + /// [`WouldBlock`]: TryLockError::WouldBlock /// /// # Examples /// @@ -247,7 +292,7 @@ impl Mutex { /// use std::thread; /// /// let mutex = Arc::new(Mutex::new(0)); - /// let c_mutex = mutex.clone(); + /// let c_mutex = Arc::clone(&mutex); /// /// thread::spawn(move || { /// let mut lock = c_mutex.try_lock(); @@ -270,6 +315,26 @@ impl Mutex { } } + /// Immediately drops the guard, and consequently unlocks the mutex. + /// + /// This function is equivalent to calling [`drop`] on the guard but is more self-documenting. + /// Alternately, the guard will be automatically dropped when it goes out of scope. + /// + /// ``` + /// #![feature(mutex_unlock)] + /// + /// use std::sync::Mutex; + /// let mutex = Mutex::new(0); + /// + /// let mut guard = mutex.lock().unwrap(); + /// *guard += 20; + /// Mutex::unlock(guard); + /// ``` + #[unstable(feature = "mutex_unlock", issue = "81872")] + pub fn unlock(guard: MutexGuard<'_, T>) { + drop(guard); + } + /// Determines whether the mutex is poisoned. /// /// If another thread is active, the mutex can still become poisoned at any @@ -283,7 +348,7 @@ impl Mutex { /// use std::thread; /// /// let mutex = Arc::new(Mutex::new(0)); - /// let c_mutex = mutex.clone(); + /// let c_mutex = Arc::clone(&mutex); /// /// let _ = thread::spawn(move || { /// let _lock = c_mutex.lock().unwrap(); @@ -297,6 +362,45 @@ impl Mutex { self.poison.get() } + /// Clear the poisoned state from a mutex + /// + /// If the mutex is poisoned, it will remain poisoned until this function is called. This + /// allows recovering from a poisoned state and marking that it has recovered. For example, if + /// the value is overwritten by a known-good value, then the mutex can be marked as + /// un-poisoned. Or possibly, the value could be inspected to determine if it is in a + /// consistent state, and if so the poison is removed. + /// + /// # Examples + /// + /// ``` + /// #![feature(mutex_unpoison)] + /// + /// use std::sync::{Arc, Mutex}; + /// use std::thread; + /// + /// let mutex = Arc::new(Mutex::new(0)); + /// let c_mutex = Arc::clone(&mutex); + /// + /// let _ = thread::spawn(move || { + /// let _lock = c_mutex.lock().unwrap(); + /// panic!(); // the mutex gets poisoned + /// }).join(); + /// + /// assert_eq!(mutex.is_poisoned(), true); + /// let x = mutex.lock().unwrap_or_else(|mut e| { + /// **e.get_mut() = 1; + /// mutex.clear_poison(); + /// e.into_inner() + /// }); + /// assert_eq!(mutex.is_poisoned(), false); + /// assert_eq!(*x, 1); + /// ``` + #[inline] + #[unstable(feature = "mutex_unpoison", issue = "96469")] + pub fn clear_poison(&self) { + self.poison.clear(); + } + /// Consumes this mutex, returning the underlying data. /// /// # Errors @@ -317,24 +421,8 @@ impl Mutex { where T: Sized, { - // We know statically that there are no outstanding references to - // `self` so there's no need to lock the inner mutex. - // - // To get the inner value, we'd like to call `data.into_inner()`, - // but because `Mutex` impl-s `Drop`, we can't move out of it, so - // we'll have to destructure it manually instead. - unsafe { - // Like `let Mutex { inner, poison, data } = self`. - let (inner, poison, data) = { - let Mutex { ref inner, ref poison, ref data } = self; - (ptr::read(inner), ptr::read(poison), ptr::read(data)) - }; - mem::forget(self); - inner.destroy(); // Keep in sync with the `Drop` impl. - drop(inner); - - poison::map_result(poison.borrow(), |_| data.into_inner()) - } + let data = self.data.into_inner(); + poison::map_result(self.poison.borrow(), |()| data) } /// Returns a mutable reference to the underlying data. @@ -358,22 +446,8 @@ impl Mutex { /// ``` #[stable(feature = "mutex_get_mut", since = "1.6.0")] pub fn get_mut(&mut self) -> LockResult<&mut T> { - // We know statically that there are no other references to `self`, so - // there's no need to lock the inner mutex. - let data = unsafe { &mut *self.data.get() }; - poison::map_result(self.poison.borrow(), |_| data) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -unsafe impl<#[may_dangle] T: ?Sized> Drop for Mutex { - fn drop(&mut self) { - // This is actually safe b/c we know that there is no further usage of - // this mutex (it's up to the user to arrange for a mutex to get - // dropped, that's not our job) - // - // IMPORTANT: This code must be kept in sync with `Mutex::into_inner`. - unsafe { self.inner.destroy() } + let data = self.data.get_mut(); + poison::map_result(self.poison.borrow(), |()| data) } } @@ -381,8 +455,6 @@ unsafe impl<#[may_dangle] T: ?Sized> Drop for Mutex { impl From for Mutex { /// Creates a new mutex in an unlocked state ready for use. /// This is equivalent to [`Mutex::new`]. - /// - /// [`Mutex::new`]: ../../std/sync/struct.Mutex.html#method.new fn from(t: T) -> Self { Mutex::new(t) } @@ -399,10 +471,13 @@ impl Default for Mutex { #[stable(feature = "rust1", since = "1.0.0")] impl fmt::Debug for Mutex { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut d = f.debug_struct("Mutex"); match self.try_lock() { - Ok(guard) => f.debug_struct("Mutex").field("data", &&*guard).finish(), + Ok(guard) => { + d.field("data", &&*guard); + } Err(TryLockError::Poisoned(err)) => { - f.debug_struct("Mutex").field("data", &&**err.get_ref()).finish() + d.field("data", &&**err.get_ref()); } Err(TryLockError::WouldBlock) => { struct LockedPlaceholder; @@ -411,16 +486,17 @@ impl fmt::Debug for Mutex { f.write_str("") } } - - f.debug_struct("Mutex").field("data", &LockedPlaceholder).finish() + d.field("data", &LockedPlaceholder); } } + d.field("poisoned", &self.poison.get()); + d.finish_non_exhaustive() } } impl<'mutex, T: ?Sized> MutexGuard<'mutex, T> { unsafe fn new(lock: &'mutex Mutex) -> LockResult> { - poison::map_result(lock.poison.borrow(), |guard| MutexGuard { lock, poison: guard }) + poison::map_result(lock.poison.guard(), |guard| MutexGuard { lock, poison: guard }) } } @@ -446,7 +522,7 @@ impl Drop for MutexGuard<'_, T> { fn drop(&mut self) { unsafe { self.lock.poison.done(&self.poison); - self.lock.inner.raw_unlock(); + self.lock.inner.unlock(); } } } @@ -472,245 +548,3 @@ pub fn guard_lock<'a, T: ?Sized>(guard: &MutexGuard<'a, T>) -> &'a sys::Mutex { pub fn guard_poison<'a, T: ?Sized>(guard: &MutexGuard<'a, T>) -> &'a poison::Flag { &guard.lock.poison } - -#[cfg(all(test, not(target_os = "emscripten")))] -mod tests { - use crate::sync::atomic::{AtomicUsize, Ordering}; - use crate::sync::mpsc::channel; - use crate::sync::{Arc, Condvar, Mutex}; - use crate::thread; - - struct Packet(Arc<(Mutex, Condvar)>); - - #[derive(Eq, PartialEq, Debug)] - struct NonCopy(i32); - - #[test] - fn smoke() { - let m = Mutex::new(()); - drop(m.lock().unwrap()); - drop(m.lock().unwrap()); - } - - #[test] - fn lots_and_lots() { - const J: u32 = 1000; - const K: u32 = 3; - - let m = Arc::new(Mutex::new(0)); - - fn inc(m: &Mutex) { - for _ in 0..J { - *m.lock().unwrap() += 1; - } - } - - let (tx, rx) = channel(); - for _ in 0..K { - let tx2 = tx.clone(); - let m2 = m.clone(); - thread::spawn(move || { - inc(&m2); - tx2.send(()).unwrap(); - }); - let tx2 = tx.clone(); - let m2 = m.clone(); - thread::spawn(move || { - inc(&m2); - tx2.send(()).unwrap(); - }); - } - - drop(tx); - for _ in 0..2 * K { - rx.recv().unwrap(); - } - assert_eq!(*m.lock().unwrap(), J * K * 2); - } - - #[test] - fn try_lock() { - let m = Mutex::new(()); - *m.try_lock().unwrap() = (); - } - - #[test] - fn test_into_inner() { - let m = Mutex::new(NonCopy(10)); - assert_eq!(m.into_inner().unwrap(), NonCopy(10)); - } - - #[test] - fn test_into_inner_drop() { - struct Foo(Arc); - impl Drop for Foo { - fn drop(&mut self) { - self.0.fetch_add(1, Ordering::SeqCst); - } - } - let num_drops = Arc::new(AtomicUsize::new(0)); - let m = Mutex::new(Foo(num_drops.clone())); - assert_eq!(num_drops.load(Ordering::SeqCst), 0); - { - let _inner = m.into_inner().unwrap(); - assert_eq!(num_drops.load(Ordering::SeqCst), 0); - } - assert_eq!(num_drops.load(Ordering::SeqCst), 1); - } - - #[test] - fn test_into_inner_poison() { - let m = Arc::new(Mutex::new(NonCopy(10))); - let m2 = m.clone(); - let _ = thread::spawn(move || { - let _lock = m2.lock().unwrap(); - panic!("test panic in inner thread to poison mutex"); - }) - .join(); - - assert!(m.is_poisoned()); - match Arc::try_unwrap(m).unwrap().into_inner() { - Err(e) => assert_eq!(e.into_inner(), NonCopy(10)), - Ok(x) => panic!("into_inner of poisoned Mutex is Ok: {:?}", x), - } - } - - #[test] - fn test_get_mut() { - let mut m = Mutex::new(NonCopy(10)); - *m.get_mut().unwrap() = NonCopy(20); - assert_eq!(m.into_inner().unwrap(), NonCopy(20)); - } - - #[test] - fn test_get_mut_poison() { - let m = Arc::new(Mutex::new(NonCopy(10))); - let m2 = m.clone(); - let _ = thread::spawn(move || { - let _lock = m2.lock().unwrap(); - panic!("test panic in inner thread to poison mutex"); - }) - .join(); - - assert!(m.is_poisoned()); - match Arc::try_unwrap(m).unwrap().get_mut() { - Err(e) => assert_eq!(*e.into_inner(), NonCopy(10)), - Ok(x) => panic!("get_mut of poisoned Mutex is Ok: {:?}", x), - } - } - - #[test] - fn test_mutex_arc_condvar() { - let packet = Packet(Arc::new((Mutex::new(false), Condvar::new()))); - let packet2 = Packet(packet.0.clone()); - let (tx, rx) = channel(); - let _t = thread::spawn(move || { - // wait until parent gets in - rx.recv().unwrap(); - let &(ref lock, ref cvar) = &*packet2.0; - let mut lock = lock.lock().unwrap(); - *lock = true; - cvar.notify_one(); - }); - - let &(ref lock, ref cvar) = &*packet.0; - let mut lock = lock.lock().unwrap(); - tx.send(()).unwrap(); - assert!(!*lock); - while !*lock { - lock = cvar.wait(lock).unwrap(); - } - } - - #[test] - fn test_arc_condvar_poison() { - let packet = Packet(Arc::new((Mutex::new(1), Condvar::new()))); - let packet2 = Packet(packet.0.clone()); - let (tx, rx) = channel(); - - let _t = thread::spawn(move || -> () { - rx.recv().unwrap(); - let &(ref lock, ref cvar) = &*packet2.0; - let _g = lock.lock().unwrap(); - cvar.notify_one(); - // Parent should fail when it wakes up. - panic!(); - }); - - let &(ref lock, ref cvar) = &*packet.0; - let mut lock = lock.lock().unwrap(); - tx.send(()).unwrap(); - while *lock == 1 { - match cvar.wait(lock) { - Ok(l) => { - lock = l; - assert_eq!(*lock, 1); - } - Err(..) => break, - } - } - } - - #[test] - fn test_mutex_arc_poison() { - let arc = Arc::new(Mutex::new(1)); - assert!(!arc.is_poisoned()); - let arc2 = arc.clone(); - let _ = thread::spawn(move || { - let lock = arc2.lock().unwrap(); - assert_eq!(*lock, 2); - }) - .join(); - assert!(arc.lock().is_err()); - assert!(arc.is_poisoned()); - } - - #[test] - fn test_mutex_arc_nested() { - // Tests nested mutexes and access - // to underlying data. - let arc = Arc::new(Mutex::new(1)); - let arc2 = Arc::new(Mutex::new(arc)); - let (tx, rx) = channel(); - let _t = thread::spawn(move || { - let lock = arc2.lock().unwrap(); - let lock2 = lock.lock().unwrap(); - assert_eq!(*lock2, 1); - tx.send(()).unwrap(); - }); - rx.recv().unwrap(); - } - - #[test] - fn test_mutex_arc_access_in_unwind() { - let arc = Arc::new(Mutex::new(1)); - let arc2 = arc.clone(); - let _ = thread::spawn(move || -> () { - struct Unwinder { - i: Arc>, - } - impl Drop for Unwinder { - fn drop(&mut self) { - *self.i.lock().unwrap() += 1; - } - } - let _u = Unwinder { i: arc2 }; - panic!(); - }) - .join(); - let lock = arc.lock().unwrap(); - assert_eq!(*lock, 2); - } - - #[test] - fn test_mutex_unsized() { - let mutex: &Mutex<[i32]> = &Mutex::new([1, 2, 3]); - { - let b = &mut *mutex.lock().unwrap(); - b[0] = 4; - b[2] = 5; - } - let comp: &[i32] = &[4, 2, 5]; - assert_eq!(&*mutex.lock().unwrap(), comp); - } -} diff --git a/crux-mir/lib/std/src/sync/mutex/tests.rs b/crux-mir/lib/std/src/sync/mutex/tests.rs new file mode 100644 index 000000000..1786a3c09 --- /dev/null +++ b/crux-mir/lib/std/src/sync/mutex/tests.rs @@ -0,0 +1,238 @@ +use crate::sync::atomic::{AtomicUsize, Ordering}; +use crate::sync::mpsc::channel; +use crate::sync::{Arc, Condvar, Mutex}; +use crate::thread; + +struct Packet(Arc<(Mutex, Condvar)>); + +#[derive(Eq, PartialEq, Debug)] +struct NonCopy(i32); + +#[test] +fn smoke() { + let m = Mutex::new(()); + drop(m.lock().unwrap()); + drop(m.lock().unwrap()); +} + +#[test] +fn lots_and_lots() { + const J: u32 = 1000; + const K: u32 = 3; + + let m = Arc::new(Mutex::new(0)); + + fn inc(m: &Mutex) { + for _ in 0..J { + *m.lock().unwrap() += 1; + } + } + + let (tx, rx) = channel(); + for _ in 0..K { + let tx2 = tx.clone(); + let m2 = m.clone(); + thread::spawn(move || { + inc(&m2); + tx2.send(()).unwrap(); + }); + let tx2 = tx.clone(); + let m2 = m.clone(); + thread::spawn(move || { + inc(&m2); + tx2.send(()).unwrap(); + }); + } + + drop(tx); + for _ in 0..2 * K { + rx.recv().unwrap(); + } + assert_eq!(*m.lock().unwrap(), J * K * 2); +} + +#[test] +fn try_lock() { + let m = Mutex::new(()); + *m.try_lock().unwrap() = (); +} + +#[test] +fn test_into_inner() { + let m = Mutex::new(NonCopy(10)); + assert_eq!(m.into_inner().unwrap(), NonCopy(10)); +} + +#[test] +fn test_into_inner_drop() { + struct Foo(Arc); + impl Drop for Foo { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + let num_drops = Arc::new(AtomicUsize::new(0)); + let m = Mutex::new(Foo(num_drops.clone())); + assert_eq!(num_drops.load(Ordering::SeqCst), 0); + { + let _inner = m.into_inner().unwrap(); + assert_eq!(num_drops.load(Ordering::SeqCst), 0); + } + assert_eq!(num_drops.load(Ordering::SeqCst), 1); +} + +#[test] +fn test_into_inner_poison() { + let m = Arc::new(Mutex::new(NonCopy(10))); + let m2 = m.clone(); + let _ = thread::spawn(move || { + let _lock = m2.lock().unwrap(); + panic!("test panic in inner thread to poison mutex"); + }) + .join(); + + assert!(m.is_poisoned()); + match Arc::try_unwrap(m).unwrap().into_inner() { + Err(e) => assert_eq!(e.into_inner(), NonCopy(10)), + Ok(x) => panic!("into_inner of poisoned Mutex is Ok: {x:?}"), + } +} + +#[test] +fn test_get_mut() { + let mut m = Mutex::new(NonCopy(10)); + *m.get_mut().unwrap() = NonCopy(20); + assert_eq!(m.into_inner().unwrap(), NonCopy(20)); +} + +#[test] +fn test_get_mut_poison() { + let m = Arc::new(Mutex::new(NonCopy(10))); + let m2 = m.clone(); + let _ = thread::spawn(move || { + let _lock = m2.lock().unwrap(); + panic!("test panic in inner thread to poison mutex"); + }) + .join(); + + assert!(m.is_poisoned()); + match Arc::try_unwrap(m).unwrap().get_mut() { + Err(e) => assert_eq!(*e.into_inner(), NonCopy(10)), + Ok(x) => panic!("get_mut of poisoned Mutex is Ok: {x:?}"), + } +} + +#[test] +fn test_mutex_arc_condvar() { + let packet = Packet(Arc::new((Mutex::new(false), Condvar::new()))); + let packet2 = Packet(packet.0.clone()); + let (tx, rx) = channel(); + let _t = thread::spawn(move || { + // wait until parent gets in + rx.recv().unwrap(); + let &(ref lock, ref cvar) = &*packet2.0; + let mut lock = lock.lock().unwrap(); + *lock = true; + cvar.notify_one(); + }); + + let &(ref lock, ref cvar) = &*packet.0; + let mut lock = lock.lock().unwrap(); + tx.send(()).unwrap(); + assert!(!*lock); + while !*lock { + lock = cvar.wait(lock).unwrap(); + } +} + +#[test] +fn test_arc_condvar_poison() { + let packet = Packet(Arc::new((Mutex::new(1), Condvar::new()))); + let packet2 = Packet(packet.0.clone()); + let (tx, rx) = channel(); + + let _t = thread::spawn(move || -> () { + rx.recv().unwrap(); + let &(ref lock, ref cvar) = &*packet2.0; + let _g = lock.lock().unwrap(); + cvar.notify_one(); + // Parent should fail when it wakes up. + panic!(); + }); + + let &(ref lock, ref cvar) = &*packet.0; + let mut lock = lock.lock().unwrap(); + tx.send(()).unwrap(); + while *lock == 1 { + match cvar.wait(lock) { + Ok(l) => { + lock = l; + assert_eq!(*lock, 1); + } + Err(..) => break, + } + } +} + +#[test] +fn test_mutex_arc_poison() { + let arc = Arc::new(Mutex::new(1)); + assert!(!arc.is_poisoned()); + let arc2 = arc.clone(); + let _ = thread::spawn(move || { + let lock = arc2.lock().unwrap(); + assert_eq!(*lock, 2); // deliberate assertion failure to poison the mutex + }) + .join(); + assert!(arc.lock().is_err()); + assert!(arc.is_poisoned()); +} + +#[test] +fn test_mutex_arc_nested() { + // Tests nested mutexes and access + // to underlying data. + let arc = Arc::new(Mutex::new(1)); + let arc2 = Arc::new(Mutex::new(arc)); + let (tx, rx) = channel(); + let _t = thread::spawn(move || { + let lock = arc2.lock().unwrap(); + let lock2 = lock.lock().unwrap(); + assert_eq!(*lock2, 1); + tx.send(()).unwrap(); + }); + rx.recv().unwrap(); +} + +#[test] +fn test_mutex_arc_access_in_unwind() { + let arc = Arc::new(Mutex::new(1)); + let arc2 = arc.clone(); + let _ = thread::spawn(move || -> () { + struct Unwinder { + i: Arc>, + } + impl Drop for Unwinder { + fn drop(&mut self) { + *self.i.lock().unwrap() += 1; + } + } + let _u = Unwinder { i: arc2 }; + panic!(); + }) + .join(); + let lock = arc.lock().unwrap(); + assert_eq!(*lock, 2); +} + +#[test] +fn test_mutex_unsized() { + let mutex: &Mutex<[i32]> = &Mutex::new([1, 2, 3]); + { + let b = &mut *mutex.lock().unwrap(); + b[0] = 4; + b[2] = 5; + } + let comp: &[i32] = &[4, 2, 5]; + assert_eq!(&*mutex.lock().unwrap(), comp); +} diff --git a/crux-mir/lib/std/src/sync/once.rs b/crux-mir/lib/std/src/sync/once.rs index 1e6b6c430..0f25417d6 100644 --- a/crux-mir/lib/std/src/sync/once.rs +++ b/crux-mir/lib/std/src/sync/once.rs @@ -3,99 +3,16 @@ //! This primitive is meant to be used to run one-time initialization. An //! example use case would be for initializing an FFI library. -// A "once" is a relatively simple primitive, and it's also typically provided -// by the OS as well (see `pthread_once` or `InitOnceExecuteOnce`). The OS -// primitives, however, tend to have surprising restrictions, such as the Unix -// one doesn't allow an argument to be passed to the function. -// -// As a result, we end up implementing it ourselves in the standard library. -// This also gives us the opportunity to optimize the implementation a bit which -// should help the fast path on call sites. Consequently, let's explain how this -// primitive works now! -// -// So to recap, the guarantees of a Once are that it will call the -// initialization closure at most once, and it will never return until the one -// that's running has finished running. This means that we need some form of -// blocking here while the custom callback is running at the very least. -// Additionally, we add on the restriction of **poisoning**. Whenever an -// initialization closure panics, the Once enters a "poisoned" state which means -// that all future calls will immediately panic as well. -// -// So to implement this, one might first reach for a `Mutex`, but those cannot -// be put into a `static`. It also gets a lot harder with poisoning to figure -// out when the mutex needs to be deallocated because it's not after the closure -// finishes, but after the first successful closure finishes. -// -// All in all, this is instead implemented with atomics and lock-free -// operations! Whee! Each `Once` has one word of atomic state, and this state is -// CAS'd on to determine what to do. There are four possible state of a `Once`: -// -// * Incomplete - no initialization has run yet, and no thread is currently -// using the Once. -// * Poisoned - some thread has previously attempted to initialize the Once, but -// it panicked, so the Once is now poisoned. There are no other -// threads currently accessing this Once. -// * Running - some thread is currently attempting to run initialization. It may -// succeed, so all future threads need to wait for it to finish. -// Note that this state is accompanied with a payload, described -// below. -// * Complete - initialization has completed and all future calls should finish -// immediately. -// -// With 4 states we need 2 bits to encode this, and we use the remaining bits -// in the word we have allocated as a queue of threads waiting for the thread -// responsible for entering the RUNNING state. This queue is just a linked list -// of Waiter nodes which is monotonically increasing in size. Each node is -// allocated on the stack, and whenever the running closure finishes it will -// consume the entire queue and notify all waiters they should try again. -// -// You'll find a few more details in the implementation, but that's the gist of -// it! -// -// Atomic orderings: -// When running `Once` we deal with multiple atomics: -// `Once.state_and_queue` and an unknown number of `Waiter.signaled`. -// * `state_and_queue` is used (1) as a state flag, (2) for synchronizing the -// result of the `Once`, and (3) for synchronizing `Waiter` nodes. -// - At the end of the `call_inner` function we have to make sure the result -// of the `Once` is acquired. So every load which can be the only one to -// load COMPLETED must have at least Acquire ordering, which means all -// three of them. -// - `WaiterQueue::Drop` is the only place that may store COMPLETED, and -// must do so with Release ordering to make the result available. -// - `wait` inserts `Waiter` nodes as a pointer in `state_and_queue`, and -// needs to make the nodes available with Release ordering. The load in -// its `compare_and_swap` can be Relaxed because it only has to compare -// the atomic, not to read other data. -// - `WaiterQueue::Drop` must see the `Waiter` nodes, so it must load -// `state_and_queue` with Acquire ordering. -// - There is just one store where `state_and_queue` is used only as a -// state flag, without having to synchronize data: switching the state -// from INCOMPLETE to RUNNING in `call_inner`. This store can be Relaxed, -// but the read has to be Acquire because of the requirements mentioned -// above. -// * `Waiter.signaled` is both used as a flag, and to protect a field with -// interior mutability in `Waiter`. `Waiter.thread` is changed in -// `WaiterQueue::Drop` which then sets `signaled` with Release ordering. -// After `wait` loads `signaled` with Acquire and sees it is true, it needs to -// see the changes to drop the `Waiter` struct correctly. -// * There is one place where the two atomics `Once.state_and_queue` and -// `Waiter.signaled` come together, and might be reordered by the compiler or -// processor. Because both use Aquire ordering such a reordering is not -// allowed, so no need for SeqCst. +#[cfg(all(test, not(target_os = "emscripten")))] +mod tests; -use crate::cell::Cell; use crate::fmt; -use crate::marker; -use crate::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use crate::thread::{self, Thread}; +use crate::panic::{RefUnwindSafe, UnwindSafe}; +use crate::sys_common::once as sys; /// A synchronization primitive which can be used to run a one-time global /// initialization. Useful for one-time initialization for FFI or related -/// functionality. This type can only be constructed with the [`Once::new`] -/// constructor. -/// -/// [`Once::new`]: struct.Once.html#method.new +/// functionality. This type can only be constructed with [`Once::new()`]. /// /// # Examples /// @@ -110,34 +27,24 @@ use crate::thread::{self, Thread}; /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub struct Once { - // `state_and_queue` is actually an a pointer to a `Waiter` with extra state - // bits, so we add the `PhantomData` appropriately. - state_and_queue: AtomicUsize, - _marker: marker::PhantomData<*const Waiter>, + inner: sys::Once, } -// The `PhantomData` of a raw pointer removes these two auto traits, but we -// enforce both below in the implementation so this should be safe to add. -#[stable(feature = "rust1", since = "1.0.0")] -unsafe impl Sync for Once {} -#[stable(feature = "rust1", since = "1.0.0")] -unsafe impl Send for Once {} +#[stable(feature = "sync_once_unwind_safe", since = "1.59.0")] +impl UnwindSafe for Once {} -/// State yielded to [`call_once_force`]’s closure parameter. The state can be -/// used to query the poison status of the [`Once`]. -/// -/// [`call_once_force`]: struct.Once.html#method.call_once_force -/// [`Once`]: struct.Once.html -#[unstable(feature = "once_poison", issue = "33577")] -#[derive(Debug)] +#[stable(feature = "sync_once_unwind_safe", since = "1.59.0")] +impl RefUnwindSafe for Once {} + +/// State yielded to [`Once::call_once_force()`]’s closure parameter. The state +/// can be used to query the poison status of the [`Once`]. +#[stable(feature = "once_poison", since = "1.51.0")] pub struct OnceState { - poisoned: bool, + pub(crate) inner: sys::OnceState, } /// Initialization value for static [`Once`] values. /// -/// [`Once`]: struct.Once.html -/// /// # Examples /// /// ``` @@ -146,51 +53,21 @@ pub struct OnceState { /// static START: Once = ONCE_INIT; /// ``` #[stable(feature = "rust1", since = "1.0.0")] -#[rustc_deprecated( +#[deprecated( since = "1.38.0", - reason = "the `new` function is now preferred", + note = "the `new` function is now preferred", suggestion = "Once::new()" )] pub const ONCE_INIT: Once = Once::new(); -// Four states that a Once can be in, encoded into the lower bits of -// `state_and_queue` in the Once structure. -const INCOMPLETE: usize = 0x0; -const POISONED: usize = 0x1; -const RUNNING: usize = 0x2; -const COMPLETE: usize = 0x3; - -// Mask to learn about the state. All other bits are the queue of waiters if -// this is in the RUNNING state. -const STATE_MASK: usize = 0x3; - -// Representation of a node in the linked list of waiters, used while in the -// RUNNING state. -// Note: `Waiter` can't hold a mutable pointer to the next thread, because then -// `wait` would both hand out a mutable reference to its `Waiter` node, and keep -// a shared reference to check `signaled`. Instead we hold shared references and -// use interior mutability. -#[repr(align(4))] // Ensure the two lower bits are free to use as state bits. -struct Waiter { - thread: Cell>, - signaled: AtomicBool, - next: *const Waiter, -} - -// Head of a linked list of waiters. -// Every node is a struct on the stack of a waiting thread. -// Will wake up the waiters when it gets dropped, i.e. also on panic. -struct WaiterQueue<'a> { - state_and_queue: &'a AtomicUsize, - set_state_on_drop_to: usize, -} - impl Once { /// Creates a new `Once` value. + #[inline] #[stable(feature = "once_new", since = "1.2.0")] #[rustc_const_stable(feature = "const_once_new", since = "1.32.0")] + #[must_use] pub const fn new() -> Once { - Once { state_and_queue: AtomicUsize::new(INCOMPLETE), _marker: marker::PhantomData } + Once { inner: sys::Once::new() } } /// Performs an initialization routine once and only once. The given closure @@ -201,13 +78,13 @@ impl Once { /// routine is currently running. /// /// When this function returns, it is guaranteed that some initialization - /// has run and completed (it may not be the closure specified). It is also + /// has run and completed (it might not be the closure specified). It is also /// guaranteed that any memory writes performed by the executed closure can /// be reliably observed by other threads at this point (there is a /// happens-before relation between the closure and code executing after the /// return). /// - /// If the given closure recursively invokes `call_once` on the same `Once` + /// If the given closure recursively invokes `call_once` on the same [`Once`] /// instance the exact behavior is not specified, allowed outcomes are /// a panic or a deadlock. /// @@ -244,47 +121,47 @@ impl Once { /// /// The closure `f` will only be executed once if this is called /// concurrently amongst many threads. If that closure panics, however, then - /// it will *poison* this `Once` instance, causing all future invocations of + /// it will *poison* this [`Once`] instance, causing all future invocations of /// `call_once` to also panic. /// /// This is similar to [poisoning with mutexes][poison]. /// /// [poison]: struct.Mutex.html#poisoning + #[inline] #[stable(feature = "rust1", since = "1.0.0")] + #[track_caller] pub fn call_once(&self, f: F) where F: FnOnce(), { // Fast path check - if self.is_completed() { + if self.inner.is_completed() { return; } let mut f = Some(f); - self.call_inner(false, &mut |_| f.take().unwrap()()); + self.inner.call(false, &mut |_| f.take().unwrap()()); } - /// Performs the same function as [`call_once`] except ignores poisoning. + /// Performs the same function as [`call_once()`] except ignores poisoning. /// - /// Unlike [`call_once`], if this `Once` has been poisoned (i.e., a previous - /// call to `call_once` or `call_once_force` caused a panic), calling - /// `call_once_force` will still invoke the closure `f` and will _not_ - /// result in an immediate panic. If `f` panics, the `Once` will remain - /// in a poison state. If `f` does _not_ panic, the `Once` will no - /// longer be in a poison state and all future calls to `call_once` or - /// `call_one_force` will be no-ops. + /// Unlike [`call_once()`], if this [`Once`] has been poisoned (i.e., a previous + /// call to [`call_once()`] or [`call_once_force()`] caused a panic), calling + /// [`call_once_force()`] will still invoke the closure `f` and will _not_ + /// result in an immediate panic. If `f` panics, the [`Once`] will remain + /// in a poison state. If `f` does _not_ panic, the [`Once`] will no + /// longer be in a poison state and all future calls to [`call_once()`] or + /// [`call_once_force()`] will be no-ops. /// /// The closure `f` is yielded a [`OnceState`] structure which can be used - /// to query the poison status of the `Once`. + /// to query the poison status of the [`Once`]. /// - /// [`call_once`]: struct.Once.html#method.call_once - /// [`OnceState`]: struct.OnceState.html + /// [`call_once()`]: Once::call_once + /// [`call_once_force()`]: Once::call_once_force /// /// # Examples /// /// ``` - /// #![feature(once_poison)] - /// /// use std::sync::Once; /// use std::thread; /// @@ -304,38 +181,41 @@ impl Once { /// /// // call_once_force will still run and reset the poisoned state /// INIT.call_once_force(|state| { - /// assert!(state.poisoned()); + /// assert!(state.is_poisoned()); /// }); /// /// // once any success happens, we stop propagating the poison /// INIT.call_once(|| {}); /// ``` - #[unstable(feature = "once_poison", issue = "33577")] + #[inline] + #[stable(feature = "once_poison", since = "1.51.0")] pub fn call_once_force(&self, f: F) where F: FnOnce(&OnceState), { // Fast path check - if self.is_completed() { + if self.inner.is_completed() { return; } let mut f = Some(f); - self.call_inner(true, &mut |p| f.take().unwrap()(&OnceState { poisoned: p })); + self.inner.call(true, &mut |p| f.take().unwrap()(p)); } - /// Returns `true` if some `call_once` call has completed + /// Returns `true` if some [`call_once()`] call has completed /// successfully. Specifically, `is_completed` will return false in /// the following situations: - /// * `call_once` was not called at all, - /// * `call_once` was called, but has not yet completed, - /// * the `Once` instance is poisoned + /// * [`call_once()`] was not called at all, + /// * [`call_once()`] was called, but has not yet completed, + /// * the [`Once`] instance is poisoned /// - /// This function returning `false` does not mean that `Once` has not been + /// This function returning `false` does not mean that [`Once`] has not been /// executed. For example, it may have been executed in the time between /// when `is_completed` starts executing and when it returns, in which case /// the `false` return value would be stale (but still permissible). /// + /// [`call_once()`]: Once::call_once + /// /// # Examples /// /// ``` @@ -366,162 +246,26 @@ impl Once { #[stable(feature = "once_is_completed", since = "1.43.0")] #[inline] pub fn is_completed(&self) -> bool { - // An `Acquire` load is enough because that makes all the initialization - // operations visible to us, and, this being a fast path, weaker - // ordering helps with performance. This `Acquire` synchronizes with - // `Release` operations on the slow path. - self.state_and_queue.load(Ordering::Acquire) == COMPLETE - } - - // This is a non-generic function to reduce the monomorphization cost of - // using `call_once` (this isn't exactly a trivial or small implementation). - // - // Additionally, this is tagged with `#[cold]` as it should indeed be cold - // and it helps let LLVM know that calls to this function should be off the - // fast path. Essentially, this should help generate more straight line code - // in LLVM. - // - // Finally, this takes an `FnMut` instead of a `FnOnce` because there's - // currently no way to take an `FnOnce` and call it via virtual dispatch - // without some allocation overhead. - #[cold] - fn call_inner(&self, ignore_poisoning: bool, init: &mut dyn FnMut(bool)) { - let mut state_and_queue = self.state_and_queue.load(Ordering::Acquire); - loop { - match state_and_queue { - COMPLETE => break, - POISONED if !ignore_poisoning => { - // Panic to propagate the poison. - panic!("Once instance has previously been poisoned"); - } - POISONED | INCOMPLETE => { - // Try to register this thread as the one RUNNING. - let old = self.state_and_queue.compare_and_swap( - state_and_queue, - RUNNING, - Ordering::Acquire, - ); - if old != state_and_queue { - state_and_queue = old; - continue; - } - // `waiter_queue` will manage other waiting threads, and - // wake them up on drop. - let mut waiter_queue = WaiterQueue { - state_and_queue: &self.state_and_queue, - set_state_on_drop_to: POISONED, - }; - // Run the initialization function, letting it know if we're - // poisoned or not. - init(state_and_queue == POISONED); - waiter_queue.set_state_on_drop_to = COMPLETE; - break; - } - _ => { - // All other values must be RUNNING with possibly a - // pointer to the waiter queue in the more significant bits. - assert!(state_and_queue & STATE_MASK == RUNNING); - wait(&self.state_and_queue, state_and_queue); - state_and_queue = self.state_and_queue.load(Ordering::Acquire); - } - } - } - } -} - -fn wait(state_and_queue: &AtomicUsize, mut current_state: usize) { - // Note: the following code was carefully written to avoid creating a - // mutable reference to `node` that gets aliased. - loop { - // Don't queue this thread if the status is no longer running, - // otherwise we will not be woken up. - if current_state & STATE_MASK != RUNNING { - return; - } - - // Create the node for our current thread. - let node = Waiter { - thread: Cell::new(Some(thread::current())), - signaled: AtomicBool::new(false), - next: (current_state & !STATE_MASK) as *const Waiter, - }; - let me = &node as *const Waiter as usize; - - // Try to slide in the node at the head of the linked list, making sure - // that another thread didn't just replace the head of the linked list. - let old = state_and_queue.compare_and_swap(current_state, me | RUNNING, Ordering::Release); - if old != current_state { - current_state = old; - continue; - } - - // We have enqueued ourselves, now lets wait. - // It is important not to return before being signaled, otherwise we - // would drop our `Waiter` node and leave a hole in the linked list - // (and a dangling reference). Guard against spurious wakeups by - // reparking ourselves until we are signaled. - while !node.signaled.load(Ordering::Acquire) { - // If the managing thread happens to signal and unpark us before we - // can park ourselves, the result could be this thread never gets - // unparked. Luckily `park` comes with the guarantee that if it got - // an `unpark` just before on an unparked thread is does not park. - thread::park(); - } - break; + self.inner.is_completed() } } #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for Once { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("Once { .. }") - } -} - -impl Drop for WaiterQueue<'_> { - fn drop(&mut self) { - // Swap out our state with however we finished. - let state_and_queue = - self.state_and_queue.swap(self.set_state_on_drop_to, Ordering::AcqRel); - - // We should only ever see an old state which was RUNNING. - assert_eq!(state_and_queue & STATE_MASK, RUNNING); - - // Walk the entire linked list of waiters and wake them up (in lifo - // order, last to register is first to wake up). - unsafe { - // Right after setting `node.signaled = true` the other thread may - // free `node` if there happens to be has a spurious wakeup. - // So we have to take out the `thread` field and copy the pointer to - // `next` first. - let mut queue = (state_and_queue & !STATE_MASK) as *const Waiter; - while !queue.is_null() { - let next = (*queue).next; - let thread = (*queue).thread.replace(None).unwrap(); - (*queue).signaled.store(true, Ordering::Release); - // ^- FIXME (maybe): This is another case of issue #55005 - // `store()` has a potentially dangling ref to `signaled`. - queue = next; - thread.unpark(); - } - } + f.debug_struct("Once").finish_non_exhaustive() } } impl OnceState { /// Returns `true` if the associated [`Once`] was poisoned prior to the - /// invocation of the closure passed to [`call_once_force`]. - /// - /// [`call_once_force`]: struct.Once.html#method.call_once_force - /// [`Once`]: struct.Once.html + /// invocation of the closure passed to [`Once::call_once_force()`]. /// /// # Examples /// - /// A poisoned `Once`: + /// A poisoned [`Once`]: /// /// ``` - /// #![feature(once_poison)] - /// /// use std::sync::Once; /// use std::thread; /// @@ -534,144 +278,37 @@ impl OnceState { /// assert!(handle.join().is_err()); /// /// INIT.call_once_force(|state| { - /// assert!(state.poisoned()); + /// assert!(state.is_poisoned()); /// }); /// ``` /// - /// An unpoisoned `Once`: + /// An unpoisoned [`Once`]: /// /// ``` - /// #![feature(once_poison)] - /// /// use std::sync::Once; /// /// static INIT: Once = Once::new(); /// /// INIT.call_once_force(|state| { - /// assert!(!state.poisoned()); + /// assert!(!state.is_poisoned()); /// }); - #[unstable(feature = "once_poison", issue = "33577")] - pub fn poisoned(&self) -> bool { - self.poisoned - } -} - -#[cfg(all(test, not(target_os = "emscripten")))] -mod tests { - use super::Once; - use crate::panic; - use crate::sync::mpsc::channel; - use crate::thread; - - #[test] - fn smoke_once() { - static O: Once = Once::new(); - let mut a = 0; - O.call_once(|| a += 1); - assert_eq!(a, 1); - O.call_once(|| a += 1); - assert_eq!(a, 1); - } - - #[test] - fn stampede_once() { - static O: Once = Once::new(); - static mut RUN: bool = false; - - let (tx, rx) = channel(); - for _ in 0..10 { - let tx = tx.clone(); - thread::spawn(move || { - for _ in 0..4 { - thread::yield_now() - } - unsafe { - O.call_once(|| { - assert!(!RUN); - RUN = true; - }); - assert!(RUN); - } - tx.send(()).unwrap(); - }); - } - - unsafe { - O.call_once(|| { - assert!(!RUN); - RUN = true; - }); - assert!(RUN); - } - - for _ in 0..10 { - rx.recv().unwrap(); - } + #[stable(feature = "once_poison", since = "1.51.0")] + #[inline] + pub fn is_poisoned(&self) -> bool { + self.inner.is_poisoned() } - #[test] - fn poison_bad() { - static O: Once = Once::new(); - - // poison the once - let t = panic::catch_unwind(|| { - O.call_once(|| panic!()); - }); - assert!(t.is_err()); - - // poisoning propagates - let t = panic::catch_unwind(|| { - O.call_once(|| {}); - }); - assert!(t.is_err()); - - // we can subvert poisoning, however - let mut called = false; - O.call_once_force(|p| { - called = true; - assert!(p.poisoned()) - }); - assert!(called); - - // once any success happens, we stop propagating the poison - O.call_once(|| {}); + /// Poison the associated [`Once`] without explicitly panicking. + // NOTE: This is currently only exposed for `OnceLock`. + #[inline] + pub(crate) fn poison(&self) { + self.inner.poison(); } +} - #[test] - fn wait_for_force_to_finish() { - static O: Once = Once::new(); - - // poison the once - let t = panic::catch_unwind(|| { - O.call_once(|| panic!()); - }); - assert!(t.is_err()); - - // make sure someone's waiting inside the once via a force - let (tx1, rx1) = channel(); - let (tx2, rx2) = channel(); - let t1 = thread::spawn(move || { - O.call_once_force(|p| { - assert!(p.poisoned()); - tx1.send(()).unwrap(); - rx2.recv().unwrap(); - }); - }); - - rx1.recv().unwrap(); - - // put another waiter on the once - let t2 = thread::spawn(|| { - let mut called = false; - O.call_once(|| { - called = true; - }); - assert!(!called); - }); - - tx2.send(()).unwrap(); - - assert!(t1.join().is_ok()); - assert!(t2.join().is_ok()); +#[stable(feature = "std_debug", since = "1.16.0")] +impl fmt::Debug for OnceState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OnceState").field("poisoned", &self.is_poisoned()).finish() } } diff --git a/crux-mir/lib/std/src/sync/once/tests.rs b/crux-mir/lib/std/src/sync/once/tests.rs new file mode 100644 index 000000000..0c35597e1 --- /dev/null +++ b/crux-mir/lib/std/src/sync/once/tests.rs @@ -0,0 +1,116 @@ +use super::Once; +use crate::panic; +use crate::sync::mpsc::channel; +use crate::thread; + +#[test] +fn smoke_once() { + static O: Once = Once::new(); + let mut a = 0; + O.call_once(|| a += 1); + assert_eq!(a, 1); + O.call_once(|| a += 1); + assert_eq!(a, 1); +} + +#[test] +fn stampede_once() { + static O: Once = Once::new(); + static mut RUN: bool = false; + + let (tx, rx) = channel(); + for _ in 0..10 { + let tx = tx.clone(); + thread::spawn(move || { + for _ in 0..4 { + thread::yield_now() + } + unsafe { + O.call_once(|| { + assert!(!RUN); + RUN = true; + }); + assert!(RUN); + } + tx.send(()).unwrap(); + }); + } + + unsafe { + O.call_once(|| { + assert!(!RUN); + RUN = true; + }); + assert!(RUN); + } + + for _ in 0..10 { + rx.recv().unwrap(); + } +} + +#[test] +fn poison_bad() { + static O: Once = Once::new(); + + // poison the once + let t = panic::catch_unwind(|| { + O.call_once(|| panic!()); + }); + assert!(t.is_err()); + + // poisoning propagates + let t = panic::catch_unwind(|| { + O.call_once(|| {}); + }); + assert!(t.is_err()); + + // we can subvert poisoning, however + let mut called = false; + O.call_once_force(|p| { + called = true; + assert!(p.is_poisoned()) + }); + assert!(called); + + // once any success happens, we stop propagating the poison + O.call_once(|| {}); +} + +#[test] +fn wait_for_force_to_finish() { + static O: Once = Once::new(); + + // poison the once + let t = panic::catch_unwind(|| { + O.call_once(|| panic!()); + }); + assert!(t.is_err()); + + // make sure someone's waiting inside the once via a force + let (tx1, rx1) = channel(); + let (tx2, rx2) = channel(); + let t1 = thread::spawn(move || { + O.call_once_force(|p| { + assert!(p.is_poisoned()); + tx1.send(()).unwrap(); + rx2.recv().unwrap(); + }); + }); + + rx1.recv().unwrap(); + + // put another waiter on the once + let t2 = thread::spawn(|| { + let mut called = false; + O.call_once(|| { + called = true; + }); + assert!(!called); + }); + + tx2.send(()).unwrap(); + + assert!(t1.join().is_ok()); + assert!(t2.join().is_ok()); +} diff --git a/crux-mir/lib/std/src/sync/once_lock.rs b/crux-mir/lib/std/src/sync/once_lock.rs new file mode 100644 index 000000000..ed339ca5d --- /dev/null +++ b/crux-mir/lib/std/src/sync/once_lock.rs @@ -0,0 +1,458 @@ +use crate::cell::UnsafeCell; +use crate::fmt; +use crate::marker::PhantomData; +use crate::mem::MaybeUninit; +use crate::panic::{RefUnwindSafe, UnwindSafe}; +use crate::sync::Once; + +/// A synchronization primitive which can be written to only once. +/// +/// This type is a thread-safe [`OnceCell`], and can be used in statics. +/// +/// [`OnceCell`]: crate::cell::OnceCell +/// +/// # Examples +/// +/// ``` +/// #![feature(once_cell)] +/// +/// use std::sync::OnceLock; +/// +/// static CELL: OnceLock = OnceLock::new(); +/// assert!(CELL.get().is_none()); +/// +/// std::thread::spawn(|| { +/// let value: &String = CELL.get_or_init(|| { +/// "Hello, World!".to_string() +/// }); +/// assert_eq!(value, "Hello, World!"); +/// }).join().unwrap(); +/// +/// let value: Option<&String> = CELL.get(); +/// assert!(value.is_some()); +/// assert_eq!(value.unwrap().as_str(), "Hello, World!"); +/// ``` +#[unstable(feature = "once_cell", issue = "74465")] +pub struct OnceLock { + once: Once, + // Whether or not the value is initialized is tracked by `once.is_completed()`. + value: UnsafeCell>, + /// `PhantomData` to make sure dropck understands we're dropping T in our Drop impl. + /// + /// ```compile_fail,E0597 + /// #![feature(once_cell)] + /// + /// use std::sync::OnceLock; + /// + /// struct A<'a>(&'a str); + /// + /// impl<'a> Drop for A<'a> { + /// fn drop(&mut self) {} + /// } + /// + /// let cell = OnceLock::new(); + /// { + /// let s = String::new(); + /// let _ = cell.set(A(&s)); + /// } + /// ``` + _marker: PhantomData, +} + +impl OnceLock { + /// Creates a new empty cell. + #[inline] + #[must_use] + #[unstable(feature = "once_cell", issue = "74465")] + pub const fn new() -> OnceLock { + OnceLock { + once: Once::new(), + value: UnsafeCell::new(MaybeUninit::uninit()), + _marker: PhantomData, + } + } + + /// Gets the reference to the underlying value. + /// + /// Returns `None` if the cell is empty, or being initialized. This + /// method never blocks. + #[inline] + #[unstable(feature = "once_cell", issue = "74465")] + pub fn get(&self) -> Option<&T> { + if self.is_initialized() { + // Safe b/c checked is_initialized + Some(unsafe { self.get_unchecked() }) + } else { + None + } + } + + /// Gets the mutable reference to the underlying value. + /// + /// Returns `None` if the cell is empty. This method never blocks. + #[inline] + #[unstable(feature = "once_cell", issue = "74465")] + pub fn get_mut(&mut self) -> Option<&mut T> { + if self.is_initialized() { + // Safe b/c checked is_initialized and we have a unique access + Some(unsafe { self.get_unchecked_mut() }) + } else { + None + } + } + + /// Sets the contents of this cell to `value`. + /// + /// May block if another thread is currently attempting to initialize the cell. The cell is + /// guaranteed to contain a value when set returns, though not necessarily the one provided. + /// + /// Returns `Ok(())` if the cell's value was set by this call. + /// + /// # Examples + /// + /// ``` + /// #![feature(once_cell)] + /// + /// use std::sync::OnceLock; + /// + /// static CELL: OnceLock = OnceLock::new(); + /// + /// fn main() { + /// assert!(CELL.get().is_none()); + /// + /// std::thread::spawn(|| { + /// assert_eq!(CELL.set(92), Ok(())); + /// }).join().unwrap(); + /// + /// assert_eq!(CELL.set(62), Err(62)); + /// assert_eq!(CELL.get(), Some(&92)); + /// } + /// ``` + #[inline] + #[unstable(feature = "once_cell", issue = "74465")] + pub fn set(&self, value: T) -> Result<(), T> { + let mut value = Some(value); + self.get_or_init(|| value.take().unwrap()); + match value { + None => Ok(()), + Some(value) => Err(value), + } + } + + /// Gets the contents of the cell, initializing it with `f` if the cell + /// was empty. + /// + /// Many threads may call `get_or_init` concurrently with different + /// initializing functions, but it is guaranteed that only one function + /// will be executed. + /// + /// # Panics + /// + /// If `f` panics, the panic is propagated to the caller, and the cell + /// remains uninitialized. + /// + /// It is an error to reentrantly initialize the cell from `f`. The + /// exact outcome is unspecified. Current implementation deadlocks, but + /// this may be changed to a panic in the future. + /// + /// # Examples + /// + /// ``` + /// #![feature(once_cell)] + /// + /// use std::sync::OnceLock; + /// + /// let cell = OnceLock::new(); + /// let value = cell.get_or_init(|| 92); + /// assert_eq!(value, &92); + /// let value = cell.get_or_init(|| unreachable!()); + /// assert_eq!(value, &92); + /// ``` + #[inline] + #[unstable(feature = "once_cell", issue = "74465")] + pub fn get_or_init(&self, f: F) -> &T + where + F: FnOnce() -> T, + { + match self.get_or_try_init(|| Ok::(f())) { + Ok(val) => val, + } + } + + /// Gets the contents of the cell, initializing it with `f` if + /// the cell was empty. If the cell was empty and `f` failed, an + /// error is returned. + /// + /// # Panics + /// + /// If `f` panics, the panic is propagated to the caller, and + /// the cell remains uninitialized. + /// + /// It is an error to reentrantly initialize the cell from `f`. + /// The exact outcome is unspecified. Current implementation + /// deadlocks, but this may be changed to a panic in the future. + /// + /// # Examples + /// + /// ``` + /// #![feature(once_cell)] + /// + /// use std::sync::OnceLock; + /// + /// let cell = OnceLock::new(); + /// assert_eq!(cell.get_or_try_init(|| Err(())), Err(())); + /// assert!(cell.get().is_none()); + /// let value = cell.get_or_try_init(|| -> Result { + /// Ok(92) + /// }); + /// assert_eq!(value, Ok(&92)); + /// assert_eq!(cell.get(), Some(&92)) + /// ``` + #[inline] + #[unstable(feature = "once_cell", issue = "74465")] + pub fn get_or_try_init(&self, f: F) -> Result<&T, E> + where + F: FnOnce() -> Result, + { + // Fast path check + // NOTE: We need to perform an acquire on the state in this method + // in order to correctly synchronize `LazyLock::force`. This is + // currently done by calling `self.get()`, which in turn calls + // `self.is_initialized()`, which in turn performs the acquire. + if let Some(value) = self.get() { + return Ok(value); + } + self.initialize(f)?; + + debug_assert!(self.is_initialized()); + + // SAFETY: The inner value has been initialized + Ok(unsafe { self.get_unchecked() }) + } + + /// Consumes the `OnceLock`, returning the wrapped value. Returns + /// `None` if the cell was empty. + /// + /// # Examples + /// + /// ``` + /// #![feature(once_cell)] + /// + /// use std::sync::OnceLock; + /// + /// let cell: OnceLock = OnceLock::new(); + /// assert_eq!(cell.into_inner(), None); + /// + /// let cell = OnceLock::new(); + /// cell.set("hello".to_string()).unwrap(); + /// assert_eq!(cell.into_inner(), Some("hello".to_string())); + /// ``` + #[inline] + #[unstable(feature = "once_cell", issue = "74465")] + pub fn into_inner(mut self) -> Option { + self.take() + } + + /// Takes the value out of this `OnceLock`, moving it back to an uninitialized state. + /// + /// Has no effect and returns `None` if the `OnceLock` hasn't been initialized. + /// + /// Safety is guaranteed by requiring a mutable reference. + /// + /// # Examples + /// + /// ``` + /// #![feature(once_cell)] + /// + /// use std::sync::OnceLock; + /// + /// let mut cell: OnceLock = OnceLock::new(); + /// assert_eq!(cell.take(), None); + /// + /// let mut cell = OnceLock::new(); + /// cell.set("hello".to_string()).unwrap(); + /// assert_eq!(cell.take(), Some("hello".to_string())); + /// assert_eq!(cell.get(), None); + /// ``` + #[inline] + #[unstable(feature = "once_cell", issue = "74465")] + pub fn take(&mut self) -> Option { + if self.is_initialized() { + self.once = Once::new(); + // SAFETY: `self.value` is initialized and contains a valid `T`. + // `self.once` is reset, so `is_initialized()` will be false again + // which prevents the value from being read twice. + unsafe { Some((&mut *self.value.get()).assume_init_read()) } + } else { + None + } + } + + #[inline] + fn is_initialized(&self) -> bool { + self.once.is_completed() + } + + #[cold] + fn initialize(&self, f: F) -> Result<(), E> + where + F: FnOnce() -> Result, + { + let mut res: Result<(), E> = Ok(()); + let slot = &self.value; + + // Ignore poisoning from other threads + // If another thread panics, then we'll be able to run our closure + self.once.call_once_force(|p| { + match f() { + Ok(value) => { + unsafe { (&mut *slot.get()).write(value) }; + } + Err(e) => { + res = Err(e); + + // Treat the underlying `Once` as poisoned since we + // failed to initialize our value. Calls + p.poison(); + } + } + }); + res + } + + /// # Safety + /// + /// The value must be initialized + #[inline] + unsafe fn get_unchecked(&self) -> &T { + debug_assert!(self.is_initialized()); + (&*self.value.get()).assume_init_ref() + } + + /// # Safety + /// + /// The value must be initialized + #[inline] + unsafe fn get_unchecked_mut(&mut self) -> &mut T { + debug_assert!(self.is_initialized()); + (&mut *self.value.get()).assume_init_mut() + } +} + +// Why do we need `T: Send`? +// Thread A creates a `OnceLock` and shares it with +// scoped thread B, which fills the cell, which is +// then destroyed by A. That is, destructor observes +// a sent value. +#[unstable(feature = "once_cell", issue = "74465")] +unsafe impl Sync for OnceLock {} +#[unstable(feature = "once_cell", issue = "74465")] +unsafe impl Send for OnceLock {} + +#[unstable(feature = "once_cell", issue = "74465")] +impl RefUnwindSafe for OnceLock {} +#[unstable(feature = "once_cell", issue = "74465")] +impl UnwindSafe for OnceLock {} + +#[unstable(feature = "once_cell", issue = "74465")] +#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] +impl const Default for OnceLock { + /// Creates a new empty cell. + /// + /// # Example + /// + /// ``` + /// #![feature(once_cell)] + /// + /// use std::sync::OnceLock; + /// + /// fn main() { + /// assert_eq!(OnceLock::<()>::new(), OnceLock::default()); + /// } + /// ``` + #[inline] + fn default() -> OnceLock { + OnceLock::new() + } +} + +#[unstable(feature = "once_cell", issue = "74465")] +impl fmt::Debug for OnceLock { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.get() { + Some(v) => f.debug_tuple("Once").field(v).finish(), + None => f.write_str("Once(Uninit)"), + } + } +} + +#[unstable(feature = "once_cell", issue = "74465")] +impl Clone for OnceLock { + #[inline] + fn clone(&self) -> OnceLock { + let cell = Self::new(); + if let Some(value) = self.get() { + match cell.set(value.clone()) { + Ok(()) => (), + Err(_) => unreachable!(), + } + } + cell + } +} + +#[unstable(feature = "once_cell", issue = "74465")] +impl From for OnceLock { + /// Create a new cell with its contents set to `value`. + /// + /// # Example + /// + /// ``` + /// #![feature(once_cell)] + /// + /// use std::sync::OnceLock; + /// + /// # fn main() -> Result<(), i32> { + /// let a = OnceLock::from(3); + /// let b = OnceLock::new(); + /// b.set(3)?; + /// assert_eq!(a, b); + /// Ok(()) + /// # } + /// ``` + #[inline] + fn from(value: T) -> Self { + let cell = Self::new(); + match cell.set(value) { + Ok(()) => cell, + Err(_) => unreachable!(), + } + } +} + +#[unstable(feature = "once_cell", issue = "74465")] +impl PartialEq for OnceLock { + #[inline] + fn eq(&self, other: &OnceLock) -> bool { + self.get() == other.get() + } +} + +#[unstable(feature = "once_cell", issue = "74465")] +impl Eq for OnceLock {} + +#[unstable(feature = "once_cell", issue = "74465")] +unsafe impl<#[may_dangle] T> Drop for OnceLock { + #[inline] + fn drop(&mut self) { + if self.is_initialized() { + // SAFETY: The cell is initialized and being dropped, so it can't + // be accessed again. We also don't touch the `T` other than + // dropping it, which validates our usage of #[may_dangle]. + unsafe { (&mut *self.value.get()).assume_init_drop() }; + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crux-mir/lib/std/src/sync/once_lock/tests.rs b/crux-mir/lib/std/src/sync/once_lock/tests.rs new file mode 100644 index 000000000..46695225b --- /dev/null +++ b/crux-mir/lib/std/src/sync/once_lock/tests.rs @@ -0,0 +1,203 @@ +use crate::{ + panic, + sync::OnceLock, + sync::{ + atomic::{AtomicUsize, Ordering::SeqCst}, + mpsc::channel, + }, + thread, +}; + +fn spawn_and_wait(f: impl FnOnce() -> R + Send + 'static) -> R { + thread::spawn(f).join().unwrap() +} + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn sync_once_cell() { + static ONCE_CELL: OnceLock = OnceLock::new(); + + assert!(ONCE_CELL.get().is_none()); + + spawn_and_wait(|| { + ONCE_CELL.get_or_init(|| 92); + assert_eq!(ONCE_CELL.get(), Some(&92)); + }); + + ONCE_CELL.get_or_init(|| panic!("Kabom!")); + assert_eq!(ONCE_CELL.get(), Some(&92)); +} + +#[test] +fn sync_once_cell_get_mut() { + let mut c = OnceLock::new(); + assert!(c.get_mut().is_none()); + c.set(90).unwrap(); + *c.get_mut().unwrap() += 2; + assert_eq!(c.get_mut(), Some(&mut 92)); +} + +#[test] +fn sync_once_cell_get_unchecked() { + let c = OnceLock::new(); + c.set(92).unwrap(); + unsafe { + assert_eq!(c.get_unchecked(), &92); + } +} + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn sync_once_cell_drop() { + static DROP_CNT: AtomicUsize = AtomicUsize::new(0); + struct Dropper; + impl Drop for Dropper { + fn drop(&mut self) { + DROP_CNT.fetch_add(1, SeqCst); + } + } + + let x = OnceLock::new(); + spawn_and_wait(move || { + x.get_or_init(|| Dropper); + assert_eq!(DROP_CNT.load(SeqCst), 0); + drop(x); + }); + + assert_eq!(DROP_CNT.load(SeqCst), 1); +} + +#[test] +fn sync_once_cell_drop_empty() { + let x = OnceLock::::new(); + drop(x); +} + +#[test] +fn clone() { + let s = OnceLock::new(); + let c = s.clone(); + assert!(c.get().is_none()); + + s.set("hello".to_string()).unwrap(); + let c = s.clone(); + assert_eq!(c.get().map(String::as_str), Some("hello")); +} + +#[test] +fn get_or_try_init() { + let cell: OnceLock = OnceLock::new(); + assert!(cell.get().is_none()); + + let res = panic::catch_unwind(|| cell.get_or_try_init(|| -> Result<_, ()> { panic!() })); + assert!(res.is_err()); + assert!(!cell.is_initialized()); + assert!(cell.get().is_none()); + + assert_eq!(cell.get_or_try_init(|| Err(())), Err(())); + + assert_eq!(cell.get_or_try_init(|| Ok::<_, ()>("hello".to_string())), Ok(&"hello".to_string())); + assert_eq!(cell.get(), Some(&"hello".to_string())); +} + +#[test] +fn from_impl() { + assert_eq!(OnceLock::from("value").get(), Some(&"value")); + assert_ne!(OnceLock::from("foo").get(), Some(&"bar")); +} + +#[test] +fn partialeq_impl() { + assert!(OnceLock::from("value") == OnceLock::from("value")); + assert!(OnceLock::from("foo") != OnceLock::from("bar")); + + assert!(OnceLock::::new() == OnceLock::new()); + assert!(OnceLock::::new() != OnceLock::from("value".to_owned())); +} + +#[test] +fn into_inner() { + let cell: OnceLock = OnceLock::new(); + assert_eq!(cell.into_inner(), None); + let cell = OnceLock::new(); + cell.set("hello".to_string()).unwrap(); + assert_eq!(cell.into_inner(), Some("hello".to_string())); +} + +#[test] +fn is_sync_send() { + fn assert_traits() {} + assert_traits::>(); +} + +#[test] +fn eval_once_macro() { + macro_rules! eval_once { + (|| -> $ty:ty { + $($body:tt)* + }) => {{ + static ONCE_CELL: OnceLock<$ty> = OnceLock::new(); + fn init() -> $ty { + $($body)* + } + ONCE_CELL.get_or_init(init) + }}; + } + + let fib: &'static Vec = eval_once! { + || -> Vec { + let mut res = vec![1, 1]; + for i in 0..10 { + let next = res[i] + res[i + 1]; + res.push(next); + } + res + } + }; + assert_eq!(fib[5], 8) +} + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn sync_once_cell_does_not_leak_partially_constructed_boxes() { + static ONCE_CELL: OnceLock = OnceLock::new(); + + let n_readers = 10; + let n_writers = 3; + const MSG: &str = "Hello, World"; + + let (tx, rx) = channel(); + + for _ in 0..n_readers { + let tx = tx.clone(); + thread::spawn(move || { + loop { + if let Some(msg) = ONCE_CELL.get() { + tx.send(msg).unwrap(); + break; + } + #[cfg(target_env = "sgx")] + crate::thread::yield_now(); + } + }); + } + for _ in 0..n_writers { + thread::spawn(move || { + let _ = ONCE_CELL.set(MSG.to_owned()); + }); + } + + for _ in 0..n_readers { + let msg = rx.recv().unwrap(); + assert_eq!(msg, MSG); + } +} + +#[test] +fn dropck() { + let cell = OnceLock::new(); + { + let s = String::new(); + cell.set(&s).unwrap(); + } +} diff --git a/crux-mir/lib/std/src/sys_common/poison.rs b/crux-mir/lib/std/src/sync/poison.rs similarity index 85% rename from crux-mir/lib/std/src/sys_common/poison.rs rename to crux-mir/lib/std/src/sync/poison.rs index a98c1a581..741312d55 100644 --- a/crux-mir/lib/std/src/sys_common/poison.rs +++ b/crux-mir/lib/std/src/sync/poison.rs @@ -19,16 +19,20 @@ pub struct Flag { // all cases. impl Flag { + #[inline] pub const fn new() -> Flag { - #[cfg(feature = "opaque-poison-atomics")] - { Flag { failed: AtomicBool::new_unmodeled(false) } } + Flag { failed: AtomicBool::new(false) } + } - #[cfg(not(feature = "opaque-poison-atomics"))] - { Flag { failed: AtomicBool::new(false) } } + /// Check the flag for an unguarded borrow, where we only care about existing poison. + #[inline] + pub fn borrow(&self) -> LockResult<()> { + if self.get() { Err(PoisonError::new(())) } else { Ok(()) } } + /// Check the flag for a guarded borrow, where we may also set poison when `done`. #[inline] - pub fn borrow(&self) -> LockResult { + pub fn guard(&self) -> LockResult { let ret = Guard { panicking: thread::panicking() }; if self.get() { Err(PoisonError::new(ret)) } else { Ok(ret) } } @@ -44,6 +48,11 @@ impl Flag { pub fn get(&self) -> bool { self.failed.load(Ordering::Relaxed) } + + #[inline] + pub fn clear(&self) { + self.failed.store(false, Ordering::Relaxed) + } } pub struct Guard { @@ -66,7 +75,7 @@ pub struct Guard { /// let mutex = Arc::new(Mutex::new(1)); /// /// // poison the mutex -/// let c_mutex = mutex.clone(); +/// let c_mutex = Arc::clone(&mutex); /// let _ = thread::spawn(move || { /// let mut data = c_mutex.lock().unwrap(); /// *data = 2; @@ -77,13 +86,12 @@ pub struct Guard { /// Ok(_) => unreachable!(), /// Err(p_err) => { /// let data = p_err.get_ref(); -/// println!("recovered: {}", data); +/// println!("recovered: {data}"); /// } /// }; /// ``` -/// -/// [`Mutex`]: ../../std/sync/struct.Mutex.html -/// [`RwLock`]: ../../std/sync/struct.RwLock.html +/// [`Mutex`]: crate::sync::Mutex +/// [`RwLock`]: crate::sync::RwLock #[stable(feature = "rust1", since = "1.0.0")] pub struct PoisonError { guard: T, @@ -93,12 +101,11 @@ pub struct PoisonError { /// can occur while trying to acquire a lock, from the [`try_lock`] method on a /// [`Mutex`] or the [`try_read`] and [`try_write`] methods on an [`RwLock`]. /// -/// [`Mutex`]: struct.Mutex.html -/// [`RwLock`]: struct.RwLock.html -/// [`TryLockResult`]: type.TryLockResult.html -/// [`try_lock`]: struct.Mutex.html#method.try_lock -/// [`try_read`]: struct.RwLock.html#method.try_read -/// [`try_write`]: struct.RwLock.html#method.try_write +/// [`try_lock`]: crate::sync::Mutex::try_lock +/// [`try_read`]: crate::sync::RwLock::try_read +/// [`try_write`]: crate::sync::RwLock::try_write +/// [`Mutex`]: crate::sync::Mutex +/// [`RwLock`]: crate::sync::RwLock #[stable(feature = "rust1", since = "1.0.0")] pub enum TryLockError { /// The lock could not be acquired because another thread failed while holding @@ -119,27 +126,22 @@ pub enum TryLockError { /// the associated guard, and it can be acquired through the [`into_inner`] /// method. /// -/// [`Ok`]: ../../std/result/enum.Result.html#variant.Ok -/// [`Err`]: ../../std/result/enum.Result.html#variant.Err -/// [`into_inner`]: ../../std/sync/struct.PoisonError.html#method.into_inner +/// [`into_inner`]: PoisonError::into_inner #[stable(feature = "rust1", since = "1.0.0")] pub type LockResult = Result>; /// A type alias for the result of a nonblocking locking method. /// /// For more information, see [`LockResult`]. A `TryLockResult` doesn't -/// necessarily hold the associated guard in the [`Err`] type as the lock may not +/// necessarily hold the associated guard in the [`Err`] type as the lock might not /// have been acquired for other reasons. -/// -/// [`LockResult`]: ../../std/sync/type.LockResult.html -/// [`Err`]: ../../std/result/enum.Result.html#variant.Err #[stable(feature = "rust1", since = "1.0.0")] pub type TryLockResult = Result>; #[stable(feature = "rust1", since = "1.0.0")] impl fmt::Debug for PoisonError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - "PoisonError { inner: .. }".fmt(f) + f.debug_struct("PoisonError").finish_non_exhaustive() } } @@ -161,10 +163,8 @@ impl Error for PoisonError { impl PoisonError { /// Creates a `PoisonError`. /// - /// This is generally created by methods like [`Mutex::lock`] or [`RwLock::read`]. - /// - /// [`Mutex::lock`]: ../../std/sync/struct.Mutex.html#method.lock - /// [`RwLock::read`]: ../../std/sync/struct.RwLock.html#method.read + /// This is generally created by methods like [`Mutex::lock`](crate::sync::Mutex::lock) + /// or [`RwLock::read`](crate::sync::RwLock::read). #[stable(feature = "sync_poison", since = "1.2.0")] pub fn new(guard: T) -> PoisonError { PoisonError { guard } @@ -183,7 +183,7 @@ impl PoisonError { /// let mutex = Arc::new(Mutex::new(HashSet::new())); /// /// // poison the mutex - /// let c_mutex = mutex.clone(); + /// let c_mutex = Arc::clone(&mutex); /// let _ = thread::spawn(move || { /// let mut data = c_mutex.lock().unwrap(); /// data.insert(10); diff --git a/crux-mir/lib/std/src/sync/remutex.rs b/crux-mir/lib/std/src/sync/remutex.rs new file mode 100644 index 000000000..4c054da64 --- /dev/null +++ b/crux-mir/lib/std/src/sync/remutex.rs @@ -0,0 +1,178 @@ +#[cfg(all(test, not(target_os = "emscripten")))] +mod tests; + +use crate::cell::UnsafeCell; +use crate::ops::Deref; +use crate::panic::{RefUnwindSafe, UnwindSafe}; +use crate::sync::atomic::{AtomicUsize, Ordering::Relaxed}; +use crate::sys::locks as sys; + +/// A re-entrant mutual exclusion +/// +/// This mutex will block *other* threads waiting for the lock to become +/// available. The thread which has already locked the mutex can lock it +/// multiple times without blocking, preventing a common source of deadlocks. +/// +/// This is used by stdout().lock() and friends. +/// +/// ## Implementation details +/// +/// The 'owner' field tracks which thread has locked the mutex. +/// +/// We use current_thread_unique_ptr() as the thread identifier, +/// which is just the address of a thread local variable. +/// +/// If `owner` is set to the identifier of the current thread, +/// we assume the mutex is already locked and instead of locking it again, +/// we increment `lock_count`. +/// +/// When unlocking, we decrement `lock_count`, and only unlock the mutex when +/// it reaches zero. +/// +/// `lock_count` is protected by the mutex and only accessed by the thread that has +/// locked the mutex, so needs no synchronization. +/// +/// `owner` can be checked by other threads that want to see if they already +/// hold the lock, so needs to be atomic. If it compares equal, we're on the +/// same thread that holds the mutex and memory access can use relaxed ordering +/// since we're not dealing with multiple threads. If it compares unequal, +/// synchronization is left to the mutex, making relaxed memory ordering for +/// the `owner` field fine in all cases. +pub struct ReentrantMutex { + mutex: sys::Mutex, + owner: AtomicUsize, + lock_count: UnsafeCell, + data: T, +} + +unsafe impl Send for ReentrantMutex {} +unsafe impl Sync for ReentrantMutex {} + +impl UnwindSafe for ReentrantMutex {} +impl RefUnwindSafe for ReentrantMutex {} + +/// An RAII implementation of a "scoped lock" of a mutex. When this structure is +/// dropped (falls out of scope), the lock will be unlocked. +/// +/// The data protected by the mutex can be accessed through this guard via its +/// Deref implementation. +/// +/// # Mutability +/// +/// Unlike `MutexGuard`, `ReentrantMutexGuard` does not implement `DerefMut`, +/// because implementation of the trait would violate Rust’s reference aliasing +/// rules. Use interior mutability (usually `RefCell`) in order to mutate the +/// guarded data. +#[must_use = "if unused the ReentrantMutex will immediately unlock"] +pub struct ReentrantMutexGuard<'a, T: 'a> { + lock: &'a ReentrantMutex, +} + +impl !Send for ReentrantMutexGuard<'_, T> {} + +impl ReentrantMutex { + /// Creates a new reentrant mutex in an unlocked state. + pub const fn new(t: T) -> ReentrantMutex { + ReentrantMutex { + mutex: sys::Mutex::new(), + owner: AtomicUsize::new(0), + lock_count: UnsafeCell::new(0), + data: t, + } + } + + /// Acquires a mutex, blocking the current thread until it is able to do so. + /// + /// This function will block the caller until it is available to acquire the mutex. + /// Upon returning, the thread is the only thread with the mutex held. When the thread + /// calling this method already holds the lock, the call shall succeed without + /// blocking. + /// + /// # Errors + /// + /// If another user of this mutex panicked while holding the mutex, then + /// this call will return failure if the mutex would otherwise be + /// acquired. + pub fn lock(&self) -> ReentrantMutexGuard<'_, T> { + let this_thread = current_thread_unique_ptr(); + // Safety: We only touch lock_count when we own the lock. + unsafe { + if self.owner.load(Relaxed) == this_thread { + self.increment_lock_count(); + } else { + self.mutex.lock(); + self.owner.store(this_thread, Relaxed); + debug_assert_eq!(*self.lock_count.get(), 0); + *self.lock_count.get() = 1; + } + } + ReentrantMutexGuard { lock: self } + } + + /// Attempts to acquire this lock. + /// + /// If the lock could not be acquired at this time, then `Err` is returned. + /// Otherwise, an RAII guard is returned. + /// + /// This function does not block. + /// + /// # Errors + /// + /// If another user of this mutex panicked while holding the mutex, then + /// this call will return failure if the mutex would otherwise be + /// acquired. + pub fn try_lock(&self) -> Option> { + let this_thread = current_thread_unique_ptr(); + // Safety: We only touch lock_count when we own the lock. + unsafe { + if self.owner.load(Relaxed) == this_thread { + self.increment_lock_count(); + Some(ReentrantMutexGuard { lock: self }) + } else if self.mutex.try_lock() { + self.owner.store(this_thread, Relaxed); + debug_assert_eq!(*self.lock_count.get(), 0); + *self.lock_count.get() = 1; + Some(ReentrantMutexGuard { lock: self }) + } else { + None + } + } + } + + unsafe fn increment_lock_count(&self) { + *self.lock_count.get() = (*self.lock_count.get()) + .checked_add(1) + .expect("lock count overflow in reentrant mutex"); + } +} + +impl Deref for ReentrantMutexGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &T { + &self.lock.data + } +} + +impl Drop for ReentrantMutexGuard<'_, T> { + #[inline] + fn drop(&mut self) { + // Safety: We own the lock. + unsafe { + *self.lock.lock_count.get() -= 1; + if *self.lock.lock_count.get() == 0 { + self.lock.owner.store(0, Relaxed); + self.lock.mutex.unlock(); + } + } + } +} + +/// Get an address that is unique per running thread. +/// +/// This can be used as a non-null usize-sized ID. +pub fn current_thread_unique_ptr() -> usize { + // Use a non-drop type to make sure it's still available during thread destruction. + thread_local! { static X: u8 = const { 0 } } + X.with(|x| <*const _>::addr(x)) +} diff --git a/crux-mir/lib/std/src/sync/remutex/tests.rs b/crux-mir/lib/std/src/sync/remutex/tests.rs new file mode 100644 index 000000000..fc553081d --- /dev/null +++ b/crux-mir/lib/std/src/sync/remutex/tests.rs @@ -0,0 +1,60 @@ +use super::{ReentrantMutex, ReentrantMutexGuard}; +use crate::cell::RefCell; +use crate::sync::Arc; +use crate::thread; + +#[test] +fn smoke() { + let m = ReentrantMutex::new(()); + { + let a = m.lock(); + { + let b = m.lock(); + { + let c = m.lock(); + assert_eq!(*c, ()); + } + assert_eq!(*b, ()); + } + assert_eq!(*a, ()); + } +} + +#[test] +fn is_mutex() { + let m = Arc::new(ReentrantMutex::new(RefCell::new(0))); + let m2 = m.clone(); + let lock = m.lock(); + let child = thread::spawn(move || { + let lock = m2.lock(); + assert_eq!(*lock.borrow(), 4950); + }); + for i in 0..100 { + let lock = m.lock(); + *lock.borrow_mut() += i; + } + drop(lock); + child.join().unwrap(); +} + +#[test] +fn trylock_works() { + let m = Arc::new(ReentrantMutex::new(())); + let m2 = m.clone(); + let _lock = m.try_lock(); + let _lock2 = m.try_lock(); + thread::spawn(move || { + let lock = m2.try_lock(); + assert!(lock.is_none()); + }) + .join() + .unwrap(); + let _lock3 = m.try_lock(); +} + +pub struct Answer<'a>(pub ReentrantMutexGuard<'a, RefCell>); +impl Drop for Answer<'_> { + fn drop(&mut self) { + *self.0.borrow_mut() = 42; + } +} diff --git a/crux-mir/lib/std/src/sync/rwlock.rs b/crux-mir/lib/std/src/sync/rwlock.rs index 50f54dbf1..7c409cb3e 100644 --- a/crux-mir/lib/std/src/sync/rwlock.rs +++ b/crux-mir/lib/std/src/sync/rwlock.rs @@ -1,10 +1,12 @@ +#[cfg(all(test, not(target_os = "emscripten")))] +mod tests; + use crate::cell::UnsafeCell; use crate::fmt; -use crate::mem; use crate::ops::{Deref, DerefMut}; -use crate::ptr; -use crate::sys_common::poison::{self, LockResult, TryLockError, TryLockResult}; -use crate::sys_common::rwlock as sys; +use crate::ptr::NonNull; +use crate::sync::{poison, LockResult, TryLockError, TryLockResult}; +use crate::sys::locks as sys; /// A reader-writer lock /// @@ -20,7 +22,21 @@ use crate::sys_common::rwlock as sys; /// /// The priority policy of the lock is dependent on the underlying operating /// system's implementation, and this type does not guarantee that any -/// particular policy will be used. +/// particular policy will be used. In particular, a writer which is waiting to +/// acquire the lock in `write` might or might not block concurrent calls to +/// `read`, e.g.: +/// +///

Potential deadlock example +/// +/// ```text +/// // Thread 1 | // Thread 2 +/// let _rg = lock.read(); | +/// | // will block +/// | let _wg = lock.write(); +/// // may deadlock | +/// let _rg = lock.read(); | +/// ``` +///
/// /// The type parameter `T` represents the data that this lock protects. It is /// required that `T` satisfies [`Send`] to be shared across threads and @@ -58,14 +74,11 @@ use crate::sys_common::rwlock as sys; /// } // write lock is dropped here /// ``` /// -/// [`Deref`]: ../../std/ops/trait.Deref.html -/// [`DerefMut`]: ../../std/ops/trait.DerefMut.html -/// [`Send`]: ../../std/marker/trait.Send.html -/// [`Sync`]: ../../std/marker/trait.Sync.html -/// [`Mutex`]: struct.Mutex.html +/// [`Mutex`]: super::Mutex #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "RwLock")] pub struct RwLock { - inner: Box, + inner: sys::RwLock, poison: poison::Flag, data: UnsafeCell, } @@ -81,13 +94,22 @@ unsafe impl Sync for RwLock {} /// This structure is created by the [`read`] and [`try_read`] methods on /// [`RwLock`]. /// -/// [`read`]: struct.RwLock.html#method.read -/// [`try_read`]: struct.RwLock.html#method.try_read -/// [`RwLock`]: struct.RwLock.html +/// [`read`]: RwLock::read +/// [`try_read`]: RwLock::try_read #[must_use = "if unused the RwLock will immediately unlock"] +#[must_not_suspend = "holding a RwLockReadGuard across suspend \ + points can cause deadlocks, delays, \ + and cause Futures to not implement `Send`"] #[stable(feature = "rust1", since = "1.0.0")] +#[clippy::has_significant_drop] +#[cfg_attr(not(test), rustc_diagnostic_item = "RwLockReadGuard")] pub struct RwLockReadGuard<'a, T: ?Sized + 'a> { - lock: &'a RwLock, + // NB: we use a pointer instead of `&'a T` to avoid `noalias` violations, because a + // `Ref` argument doesn't hold immutability for its whole scope, only until it drops. + // `NonNull` is also covariant over `T`, just like we would have with `&T`. `NonNull` + // is preferable over `const* T` to allow for niche optimization. + data: NonNull, + inner_lock: &'a sys::RwLock, } #[stable(feature = "rust1", since = "1.0.0")] @@ -102,11 +124,15 @@ unsafe impl Sync for RwLockReadGuard<'_, T> {} /// This structure is created by the [`write`] and [`try_write`] methods /// on [`RwLock`]. /// -/// [`write`]: struct.RwLock.html#method.write -/// [`try_write`]: struct.RwLock.html#method.try_write -/// [`RwLock`]: struct.RwLock.html +/// [`write`]: RwLock::write +/// [`try_write`]: RwLock::try_write #[must_use = "if unused the RwLock will immediately unlock"] +#[must_not_suspend = "holding a RwLockWriteGuard across suspend \ + points can cause deadlocks, delays, \ + and cause Future's to not implement `Send`"] #[stable(feature = "rust1", since = "1.0.0")] +#[clippy::has_significant_drop] +#[cfg_attr(not(test), rustc_diagnostic_item = "RwLockWriteGuard")] pub struct RwLockWriteGuard<'a, T: ?Sized + 'a> { lock: &'a RwLock, poison: poison::Guard, @@ -129,17 +155,15 @@ impl RwLock { /// let lock = RwLock::new(5); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn new(t: T) -> RwLock { - RwLock { - inner: box sys::RWLock::new(), - poison: poison::Flag::new(), - data: UnsafeCell::new(t), - } + #[rustc_const_stable(feature = "const_locks", since = "1.63.0")] + #[inline] + pub const fn new(t: T) -> RwLock { + RwLock { inner: sys::RwLock::new(), poison: poison::Flag::new(), data: UnsafeCell::new(t) } } } impl RwLock { - /// Locks this rwlock with shared read access, blocking the current thread + /// Locks this `RwLock` with shared read access, blocking the current thread /// until it can be acquired. /// /// The calling thread will be blocked until there are no more writers which @@ -153,9 +177,10 @@ impl RwLock { /// /// # Errors /// - /// This function will return an error if the RwLock is poisoned. An RwLock - /// is poisoned whenever a writer panics while holding an exclusive lock. - /// The failure will occur immediately after the lock has been acquired. + /// This function will return an error if the `RwLock` is poisoned. An + /// `RwLock` is poisoned whenever a writer panics while holding an exclusive + /// lock. The failure will occur immediately after the lock has been + /// acquired. /// /// # Panics /// @@ -168,7 +193,7 @@ impl RwLock { /// use std::thread; /// /// let lock = Arc::new(RwLock::new(1)); - /// let c_lock = lock.clone(); + /// let c_lock = Arc::clone(&lock); /// /// let n = lock.read().unwrap(); /// assert_eq!(*n, 1); @@ -187,7 +212,7 @@ impl RwLock { } } - /// Attempts to acquire this rwlock with shared read access. + /// Attempts to acquire this `RwLock` with shared read access. /// /// If the access could not be granted at this time, then `Err` is returned. /// Otherwise, an RAII guard is returned which will release the shared access @@ -200,10 +225,16 @@ impl RwLock { /// /// # Errors /// - /// This function will return an error if the RwLock is poisoned. An RwLock - /// is poisoned whenever a writer panics while holding an exclusive lock. An - /// error will only be returned if the lock would have otherwise been - /// acquired. + /// This function will return the [`Poisoned`] error if the `RwLock` is + /// poisoned. An `RwLock` is poisoned whenever a writer panics while holding + /// an exclusive lock. `Poisoned` will only be returned if the lock would + /// have otherwise been acquired. + /// + /// This function will return the [`WouldBlock`] error if the `RwLock` could + /// not be acquired because it was already locked exclusively. + /// + /// [`Poisoned`]: TryLockError::Poisoned + /// [`WouldBlock`]: TryLockError::WouldBlock /// /// # Examples /// @@ -229,20 +260,20 @@ impl RwLock { } } - /// Locks this rwlock with exclusive write access, blocking the current + /// Locks this `RwLock` with exclusive write access, blocking the current /// thread until it can be acquired. /// /// This function will not return while other writers or other readers /// currently have access to the lock. /// - /// Returns an RAII guard which will drop the write access of this rwlock + /// Returns an RAII guard which will drop the write access of this `RwLock` /// when dropped. /// /// # Errors /// - /// This function will return an error if the RwLock is poisoned. An RwLock - /// is poisoned whenever a writer panics while holding an exclusive lock. - /// An error will be returned when the lock is acquired. + /// This function will return an error if the `RwLock` is poisoned. An + /// `RwLock` is poisoned whenever a writer panics while holding an exclusive + /// lock. An error will be returned when the lock is acquired. /// /// # Panics /// @@ -269,7 +300,7 @@ impl RwLock { } } - /// Attempts to lock this rwlock with exclusive write access. + /// Attempts to lock this `RwLock` with exclusive write access. /// /// If the lock could not be acquired at this time, then `Err` is returned. /// Otherwise, an RAII guard is returned which will release the lock when @@ -282,10 +313,17 @@ impl RwLock { /// /// # Errors /// - /// This function will return an error if the RwLock is poisoned. An RwLock - /// is poisoned whenever a writer panics while holding an exclusive lock. An - /// error will only be returned if the lock would have otherwise been - /// acquired. + /// This function will return the [`Poisoned`] error if the `RwLock` is + /// poisoned. An `RwLock` is poisoned whenever a writer panics while holding + /// an exclusive lock. `Poisoned` will only be returned if the lock would + /// have otherwise been acquired. + /// + /// This function will return the [`WouldBlock`] error if the `RwLock` could + /// not be acquired because it was already locked exclusively. + /// + /// [`Poisoned`]: TryLockError::Poisoned + /// [`WouldBlock`]: TryLockError::WouldBlock + /// /// /// # Examples /// @@ -324,7 +362,7 @@ impl RwLock { /// use std::thread; /// /// let lock = Arc::new(RwLock::new(0)); - /// let c_lock = lock.clone(); + /// let c_lock = Arc::clone(&lock); /// /// let _ = thread::spawn(move || { /// let _lock = c_lock.write().unwrap(); @@ -338,14 +376,53 @@ impl RwLock { self.poison.get() } + /// Clear the poisoned state from a lock + /// + /// If the lock is poisoned, it will remain poisoned until this function is called. This allows + /// recovering from a poisoned state and marking that it has recovered. For example, if the + /// value is overwritten by a known-good value, then the mutex can be marked as un-poisoned. Or + /// possibly, the value could be inspected to determine if it is in a consistent state, and if + /// so the poison is removed. + /// + /// # Examples + /// + /// ``` + /// #![feature(mutex_unpoison)] + /// + /// use std::sync::{Arc, RwLock}; + /// use std::thread; + /// + /// let lock = Arc::new(RwLock::new(0)); + /// let c_lock = Arc::clone(&lock); + /// + /// let _ = thread::spawn(move || { + /// let _lock = c_lock.write().unwrap(); + /// panic!(); // the mutex gets poisoned + /// }).join(); + /// + /// assert_eq!(lock.is_poisoned(), true); + /// let guard = lock.write().unwrap_or_else(|mut e| { + /// **e.get_mut() = 1; + /// lock.clear_poison(); + /// e.into_inner() + /// }); + /// assert_eq!(lock.is_poisoned(), false); + /// assert_eq!(*guard, 1); + /// ``` + #[inline] + #[unstable(feature = "mutex_unpoison", issue = "96469")] + pub fn clear_poison(&self) { + self.poison.clear(); + } + /// Consumes this `RwLock`, returning the underlying data. /// /// # Errors /// - /// This function will return an error if the RwLock is poisoned. An RwLock - /// is poisoned whenever a writer panics while holding an exclusive lock. An - /// error will only be returned if the lock would have otherwise been - /// acquired. + /// This function will return an error if the `RwLock` is poisoned. An + /// `RwLock` is poisoned whenever a writer panics while holding an exclusive + /// lock. An error will only be returned if the lock would have otherwise + /// been acquired. /// /// # Examples /// @@ -364,24 +441,8 @@ impl RwLock { where T: Sized, { - // We know statically that there are no outstanding references to - // `self` so there's no need to lock the inner lock. - // - // To get the inner value, we'd like to call `data.into_inner()`, - // but because `RwLock` impl-s `Drop`, we can't move out of it, so - // we'll have to destructure it manually instead. - unsafe { - // Like `let RwLock { inner, poison, data } = self`. - let (inner, poison, data) = { - let RwLock { ref inner, ref poison, ref data } = self; - (ptr::read(inner), ptr::read(poison), ptr::read(data)) - }; - mem::forget(self); - inner.destroy(); // Keep in sync with the `Drop` impl. - drop(inner); - - poison::map_result(poison.borrow(), |_| data.into_inner()) - } + let data = self.data.into_inner(); + poison::map_result(self.poison.borrow(), |()| data) } /// Returns a mutable reference to the underlying data. @@ -391,10 +452,10 @@ impl RwLock { /// /// # Errors /// - /// This function will return an error if the RwLock is poisoned. An RwLock - /// is poisoned whenever a writer panics while holding an exclusive lock. An - /// error will only be returned if the lock would have otherwise been - /// acquired. + /// This function will return an error if the `RwLock` is poisoned. An + /// `RwLock` is poisoned whenever a writer panics while holding an exclusive + /// lock. An error will only be returned if the lock would have otherwise + /// been acquired. /// /// # Examples /// @@ -407,28 +468,21 @@ impl RwLock { /// ``` #[stable(feature = "rwlock_get_mut", since = "1.6.0")] pub fn get_mut(&mut self) -> LockResult<&mut T> { - // We know statically that there are no other references to `self`, so - // there's no need to lock the inner lock. - let data = unsafe { &mut *self.data.get() }; - poison::map_result(self.poison.borrow(), |_| data) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -unsafe impl<#[may_dangle] T: ?Sized> Drop for RwLock { - fn drop(&mut self) { - // IMPORTANT: This code needs to be kept in sync with `RwLock::into_inner`. - unsafe { self.inner.destroy() } + let data = self.data.get_mut(); + poison::map_result(self.poison.borrow(), |()| data) } } #[stable(feature = "rust1", since = "1.0.0")] impl fmt::Debug for RwLock { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut d = f.debug_struct("RwLock"); match self.try_read() { - Ok(guard) => f.debug_struct("RwLock").field("data", &&*guard).finish(), + Ok(guard) => { + d.field("data", &&*guard); + } Err(TryLockError::Poisoned(err)) => { - f.debug_struct("RwLock").field("data", &&**err.get_ref()).finish() + d.field("data", &&**err.get_ref()); } Err(TryLockError::WouldBlock) => { struct LockedPlaceholder; @@ -437,10 +491,11 @@ impl fmt::Debug for RwLock { f.write_str("") } } - - f.debug_struct("RwLock").field("data", &LockedPlaceholder).finish() + d.field("data", &LockedPlaceholder); } } + d.field("poisoned", &self.poison.get()); + d.finish_non_exhaustive() } } @@ -456,29 +511,36 @@ impl Default for RwLock { impl From for RwLock { /// Creates a new instance of an `RwLock` which is unlocked. /// This is equivalent to [`RwLock::new`]. - /// - /// [`RwLock::new`]: ../../std/sync/struct.RwLock.html#method.new fn from(t: T) -> Self { RwLock::new(t) } } impl<'rwlock, T: ?Sized> RwLockReadGuard<'rwlock, T> { + /// Create a new instance of `RwLockReadGuard` from a `RwLock`. + // SAFETY: if and only if `lock.inner.read()` (or `lock.inner.try_read()`) has been + // successfully called from the same thread before instantiating this object. unsafe fn new(lock: &'rwlock RwLock) -> LockResult> { - poison::map_result(lock.poison.borrow(), |_| RwLockReadGuard { lock }) + poison::map_result(lock.poison.borrow(), |()| RwLockReadGuard { + data: NonNull::new_unchecked(lock.data.get()), + inner_lock: &lock.inner, + }) } } impl<'rwlock, T: ?Sized> RwLockWriteGuard<'rwlock, T> { + /// Create a new instance of `RwLockWriteGuard` from a `RwLock`. + // SAFETY: if and only if `lock.inner.write()` (or `lock.inner.try_write()`) has been + // successfully called from the same thread before instantiating this object. unsafe fn new(lock: &'rwlock RwLock) -> LockResult> { - poison::map_result(lock.poison.borrow(), |guard| RwLockWriteGuard { lock, poison: guard }) + poison::map_result(lock.poison.guard(), |guard| RwLockWriteGuard { lock, poison: guard }) } } #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for RwLockReadGuard<'_, T> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("RwLockReadGuard").field("lock", &self.lock).finish() + (**self).fmt(f) } } @@ -492,7 +554,7 @@ impl fmt::Display for RwLockReadGuard<'_, T> { #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for RwLockWriteGuard<'_, T> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("RwLockWriteGuard").field("lock", &self.lock).finish() + (**self).fmt(f) } } @@ -508,7 +570,8 @@ impl Deref for RwLockReadGuard<'_, T> { type Target = T; fn deref(&self) -> &T { - unsafe { &*self.lock.data.get() } + // SAFETY: the conditions of `RwLockGuard::new` were satisfied when created. + unsafe { self.data.as_ref() } } } @@ -517,6 +580,7 @@ impl Deref for RwLockWriteGuard<'_, T> { type Target = T; fn deref(&self) -> &T { + // SAFETY: the conditions of `RwLockWriteGuard::new` were satisfied when created. unsafe { &*self.lock.data.get() } } } @@ -524,6 +588,7 @@ impl Deref for RwLockWriteGuard<'_, T> { #[stable(feature = "rust1", since = "1.0.0")] impl DerefMut for RwLockWriteGuard<'_, T> { fn deref_mut(&mut self) -> &mut T { + // SAFETY: the conditions of `RwLockWriteGuard::new` were satisfied when created. unsafe { &mut *self.lock.data.get() } } } @@ -531,8 +596,9 @@ impl DerefMut for RwLockWriteGuard<'_, T> { #[stable(feature = "rust1", since = "1.0.0")] impl Drop for RwLockReadGuard<'_, T> { fn drop(&mut self) { + // SAFETY: the conditions of `RwLockReadGuard::new` were satisfied when created. unsafe { - self.lock.inner.read_unlock(); + self.inner_lock.read_unlock(); } } } @@ -541,259 +607,9 @@ impl Drop for RwLockReadGuard<'_, T> { impl Drop for RwLockWriteGuard<'_, T> { fn drop(&mut self) { self.lock.poison.done(&self.poison); + // SAFETY: the conditions of `RwLockWriteGuard::new` were satisfied when created. unsafe { self.lock.inner.write_unlock(); } } } - -#[cfg(all(test, not(target_os = "emscripten")))] -mod tests { - use crate::sync::atomic::{AtomicUsize, Ordering}; - use crate::sync::mpsc::channel; - use crate::sync::{Arc, RwLock, TryLockError}; - use crate::thread; - use rand::{self, Rng}; - - #[derive(Eq, PartialEq, Debug)] - struct NonCopy(i32); - - #[test] - fn smoke() { - let l = RwLock::new(()); - drop(l.read().unwrap()); - drop(l.write().unwrap()); - drop((l.read().unwrap(), l.read().unwrap())); - drop(l.write().unwrap()); - } - - #[test] - fn frob() { - const N: u32 = 10; - const M: usize = 1000; - - let r = Arc::new(RwLock::new(())); - - let (tx, rx) = channel::<()>(); - for _ in 0..N { - let tx = tx.clone(); - let r = r.clone(); - thread::spawn(move || { - let mut rng = rand::thread_rng(); - for _ in 0..M { - if rng.gen_bool(1.0 / (N as f64)) { - drop(r.write().unwrap()); - } else { - drop(r.read().unwrap()); - } - } - drop(tx); - }); - } - drop(tx); - let _ = rx.recv(); - } - - #[test] - fn test_rw_arc_poison_wr() { - let arc = Arc::new(RwLock::new(1)); - let arc2 = arc.clone(); - let _: Result<(), _> = thread::spawn(move || { - let _lock = arc2.write().unwrap(); - panic!(); - }) - .join(); - assert!(arc.read().is_err()); - } - - #[test] - fn test_rw_arc_poison_ww() { - let arc = Arc::new(RwLock::new(1)); - assert!(!arc.is_poisoned()); - let arc2 = arc.clone(); - let _: Result<(), _> = thread::spawn(move || { - let _lock = arc2.write().unwrap(); - panic!(); - }) - .join(); - assert!(arc.write().is_err()); - assert!(arc.is_poisoned()); - } - - #[test] - fn test_rw_arc_no_poison_rr() { - let arc = Arc::new(RwLock::new(1)); - let arc2 = arc.clone(); - let _: Result<(), _> = thread::spawn(move || { - let _lock = arc2.read().unwrap(); - panic!(); - }) - .join(); - let lock = arc.read().unwrap(); - assert_eq!(*lock, 1); - } - #[test] - fn test_rw_arc_no_poison_rw() { - let arc = Arc::new(RwLock::new(1)); - let arc2 = arc.clone(); - let _: Result<(), _> = thread::spawn(move || { - let _lock = arc2.read().unwrap(); - panic!() - }) - .join(); - let lock = arc.write().unwrap(); - assert_eq!(*lock, 1); - } - - #[test] - fn test_rw_arc() { - let arc = Arc::new(RwLock::new(0)); - let arc2 = arc.clone(); - let (tx, rx) = channel(); - - thread::spawn(move || { - let mut lock = arc2.write().unwrap(); - for _ in 0..10 { - let tmp = *lock; - *lock = -1; - thread::yield_now(); - *lock = tmp + 1; - } - tx.send(()).unwrap(); - }); - - // Readers try to catch the writer in the act - let mut children = Vec::new(); - for _ in 0..5 { - let arc3 = arc.clone(); - children.push(thread::spawn(move || { - let lock = arc3.read().unwrap(); - assert!(*lock >= 0); - })); - } - - // Wait for children to pass their asserts - for r in children { - assert!(r.join().is_ok()); - } - - // Wait for writer to finish - rx.recv().unwrap(); - let lock = arc.read().unwrap(); - assert_eq!(*lock, 10); - } - - #[test] - fn test_rw_arc_access_in_unwind() { - let arc = Arc::new(RwLock::new(1)); - let arc2 = arc.clone(); - let _ = thread::spawn(move || -> () { - struct Unwinder { - i: Arc>, - } - impl Drop for Unwinder { - fn drop(&mut self) { - let mut lock = self.i.write().unwrap(); - *lock += 1; - } - } - let _u = Unwinder { i: arc2 }; - panic!(); - }) - .join(); - let lock = arc.read().unwrap(); - assert_eq!(*lock, 2); - } - - #[test] - fn test_rwlock_unsized() { - let rw: &RwLock<[i32]> = &RwLock::new([1, 2, 3]); - { - let b = &mut *rw.write().unwrap(); - b[0] = 4; - b[2] = 5; - } - let comp: &[i32] = &[4, 2, 5]; - assert_eq!(&*rw.read().unwrap(), comp); - } - - #[test] - fn test_rwlock_try_write() { - let lock = RwLock::new(0isize); - let read_guard = lock.read().unwrap(); - - let write_result = lock.try_write(); - match write_result { - Err(TryLockError::WouldBlock) => (), - Ok(_) => assert!(false, "try_write should not succeed while read_guard is in scope"), - Err(_) => assert!(false, "unexpected error"), - } - - drop(read_guard); - } - - #[test] - fn test_into_inner() { - let m = RwLock::new(NonCopy(10)); - assert_eq!(m.into_inner().unwrap(), NonCopy(10)); - } - - #[test] - fn test_into_inner_drop() { - struct Foo(Arc); - impl Drop for Foo { - fn drop(&mut self) { - self.0.fetch_add(1, Ordering::SeqCst); - } - } - let num_drops = Arc::new(AtomicUsize::new(0)); - let m = RwLock::new(Foo(num_drops.clone())); - assert_eq!(num_drops.load(Ordering::SeqCst), 0); - { - let _inner = m.into_inner().unwrap(); - assert_eq!(num_drops.load(Ordering::SeqCst), 0); - } - assert_eq!(num_drops.load(Ordering::SeqCst), 1); - } - - #[test] - fn test_into_inner_poison() { - let m = Arc::new(RwLock::new(NonCopy(10))); - let m2 = m.clone(); - let _ = thread::spawn(move || { - let _lock = m2.write().unwrap(); - panic!("test panic in inner thread to poison RwLock"); - }) - .join(); - - assert!(m.is_poisoned()); - match Arc::try_unwrap(m).unwrap().into_inner() { - Err(e) => assert_eq!(e.into_inner(), NonCopy(10)), - Ok(x) => panic!("into_inner of poisoned RwLock is Ok: {:?}", x), - } - } - - #[test] - fn test_get_mut() { - let mut m = RwLock::new(NonCopy(10)); - *m.get_mut().unwrap() = NonCopy(20); - assert_eq!(m.into_inner().unwrap(), NonCopy(20)); - } - - #[test] - fn test_get_mut_poison() { - let m = Arc::new(RwLock::new(NonCopy(10))); - let m2 = m.clone(); - let _ = thread::spawn(move || { - let _lock = m2.write().unwrap(); - panic!("test panic in inner thread to poison RwLock"); - }) - .join(); - - assert!(m.is_poisoned()); - match Arc::try_unwrap(m).unwrap().get_mut() { - Err(e) => assert_eq!(*e.into_inner(), NonCopy(10)), - Ok(x) => panic!("get_mut of poisoned RwLock is Ok: {:?}", x), - } - } -} diff --git a/crux-mir/lib/std/src/sync/rwlock/tests.rs b/crux-mir/lib/std/src/sync/rwlock/tests.rs new file mode 100644 index 000000000..1a9d3d3f1 --- /dev/null +++ b/crux-mir/lib/std/src/sync/rwlock/tests.rs @@ -0,0 +1,259 @@ +use crate::sync::atomic::{AtomicUsize, Ordering}; +use crate::sync::mpsc::channel; +use crate::sync::{Arc, RwLock, RwLockReadGuard, TryLockError}; +use crate::thread; +use rand::Rng; + +#[derive(Eq, PartialEq, Debug)] +struct NonCopy(i32); + +#[test] +fn smoke() { + let l = RwLock::new(()); + drop(l.read().unwrap()); + drop(l.write().unwrap()); + drop((l.read().unwrap(), l.read().unwrap())); + drop(l.write().unwrap()); +} + +#[test] +fn frob() { + const N: u32 = 10; + const M: usize = if cfg!(miri) { 100 } else { 1000 }; + + let r = Arc::new(RwLock::new(())); + + let (tx, rx) = channel::<()>(); + for _ in 0..N { + let tx = tx.clone(); + let r = r.clone(); + thread::spawn(move || { + let mut rng = crate::test_helpers::test_rng(); + for _ in 0..M { + if rng.gen_bool(1.0 / (N as f64)) { + drop(r.write().unwrap()); + } else { + drop(r.read().unwrap()); + } + } + drop(tx); + }); + } + drop(tx); + let _ = rx.recv(); +} + +#[test] +fn test_rw_arc_poison_wr() { + let arc = Arc::new(RwLock::new(1)); + let arc2 = arc.clone(); + let _: Result<(), _> = thread::spawn(move || { + let _lock = arc2.write().unwrap(); + panic!(); + }) + .join(); + assert!(arc.read().is_err()); +} + +#[test] +fn test_rw_arc_poison_ww() { + let arc = Arc::new(RwLock::new(1)); + assert!(!arc.is_poisoned()); + let arc2 = arc.clone(); + let _: Result<(), _> = thread::spawn(move || { + let _lock = arc2.write().unwrap(); + panic!(); + }) + .join(); + assert!(arc.write().is_err()); + assert!(arc.is_poisoned()); +} + +#[test] +fn test_rw_arc_no_poison_rr() { + let arc = Arc::new(RwLock::new(1)); + let arc2 = arc.clone(); + let _: Result<(), _> = thread::spawn(move || { + let _lock = arc2.read().unwrap(); + panic!(); + }) + .join(); + let lock = arc.read().unwrap(); + assert_eq!(*lock, 1); +} +#[test] +fn test_rw_arc_no_poison_rw() { + let arc = Arc::new(RwLock::new(1)); + let arc2 = arc.clone(); + let _: Result<(), _> = thread::spawn(move || { + let _lock = arc2.read().unwrap(); + panic!() + }) + .join(); + let lock = arc.write().unwrap(); + assert_eq!(*lock, 1); +} + +#[test] +fn test_rw_arc() { + let arc = Arc::new(RwLock::new(0)); + let arc2 = arc.clone(); + let (tx, rx) = channel(); + + thread::spawn(move || { + let mut lock = arc2.write().unwrap(); + for _ in 0..10 { + let tmp = *lock; + *lock = -1; + thread::yield_now(); + *lock = tmp + 1; + } + tx.send(()).unwrap(); + }); + + // Readers try to catch the writer in the act + let mut children = Vec::new(); + for _ in 0..5 { + let arc3 = arc.clone(); + children.push(thread::spawn(move || { + let lock = arc3.read().unwrap(); + assert!(*lock >= 0); + })); + } + + // Wait for children to pass their asserts + for r in children { + assert!(r.join().is_ok()); + } + + // Wait for writer to finish + rx.recv().unwrap(); + let lock = arc.read().unwrap(); + assert_eq!(*lock, 10); +} + +#[test] +fn test_rw_arc_access_in_unwind() { + let arc = Arc::new(RwLock::new(1)); + let arc2 = arc.clone(); + let _ = thread::spawn(move || -> () { + struct Unwinder { + i: Arc>, + } + impl Drop for Unwinder { + fn drop(&mut self) { + let mut lock = self.i.write().unwrap(); + *lock += 1; + } + } + let _u = Unwinder { i: arc2 }; + panic!(); + }) + .join(); + let lock = arc.read().unwrap(); + assert_eq!(*lock, 2); +} + +#[test] +fn test_rwlock_unsized() { + let rw: &RwLock<[i32]> = &RwLock::new([1, 2, 3]); + { + let b = &mut *rw.write().unwrap(); + b[0] = 4; + b[2] = 5; + } + let comp: &[i32] = &[4, 2, 5]; + assert_eq!(&*rw.read().unwrap(), comp); +} + +#[test] +fn test_rwlock_try_write() { + let lock = RwLock::new(0isize); + let read_guard = lock.read().unwrap(); + + let write_result = lock.try_write(); + match write_result { + Err(TryLockError::WouldBlock) => (), + Ok(_) => assert!(false, "try_write should not succeed while read_guard is in scope"), + Err(_) => assert!(false, "unexpected error"), + } + + drop(read_guard); +} + +#[test] +fn test_into_inner() { + let m = RwLock::new(NonCopy(10)); + assert_eq!(m.into_inner().unwrap(), NonCopy(10)); +} + +#[test] +fn test_into_inner_drop() { + struct Foo(Arc); + impl Drop for Foo { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + let num_drops = Arc::new(AtomicUsize::new(0)); + let m = RwLock::new(Foo(num_drops.clone())); + assert_eq!(num_drops.load(Ordering::SeqCst), 0); + { + let _inner = m.into_inner().unwrap(); + assert_eq!(num_drops.load(Ordering::SeqCst), 0); + } + assert_eq!(num_drops.load(Ordering::SeqCst), 1); +} + +#[test] +fn test_into_inner_poison() { + let m = Arc::new(RwLock::new(NonCopy(10))); + let m2 = m.clone(); + let _ = thread::spawn(move || { + let _lock = m2.write().unwrap(); + panic!("test panic in inner thread to poison RwLock"); + }) + .join(); + + assert!(m.is_poisoned()); + match Arc::try_unwrap(m).unwrap().into_inner() { + Err(e) => assert_eq!(e.into_inner(), NonCopy(10)), + Ok(x) => panic!("into_inner of poisoned RwLock is Ok: {x:?}"), + } +} + +#[test] +fn test_get_mut() { + let mut m = RwLock::new(NonCopy(10)); + *m.get_mut().unwrap() = NonCopy(20); + assert_eq!(m.into_inner().unwrap(), NonCopy(20)); +} + +#[test] +fn test_get_mut_poison() { + let m = Arc::new(RwLock::new(NonCopy(10))); + let m2 = m.clone(); + let _ = thread::spawn(move || { + let _lock = m2.write().unwrap(); + panic!("test panic in inner thread to poison RwLock"); + }) + .join(); + + assert!(m.is_poisoned()); + match Arc::try_unwrap(m).unwrap().get_mut() { + Err(e) => assert_eq!(*e.into_inner(), NonCopy(10)), + Ok(x) => panic!("get_mut of poisoned RwLock is Ok: {x:?}"), + } +} + +#[test] +fn test_read_guard_covariance() { + fn do_stuff<'a>(_: RwLockReadGuard<'_, &'a i32>, _: &'a i32) {} + let j: i32 = 5; + let lock = RwLock::new(&j); + { + let i = 6; + do_stuff(lock.read().unwrap(), &i); + } + drop(lock); +} diff --git a/crux-mir/lib/std/src/sys/cloudabi/abi/bitflags.rs b/crux-mir/lib/std/src/sys/cloudabi/abi/bitflags.rs deleted file mode 100644 index 2383277ad..000000000 --- a/crux-mir/lib/std/src/sys/cloudabi/abi/bitflags.rs +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) 2018 Nuxi (https://nuxi.nl/) and contributors. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -// ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -// OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -// SUCH DAMAGE. - -#[cfg(feature = "bitflags")] -use bitflags::bitflags; - -// Minimal implementation of bitflags! in case we can't depend on the bitflags -// crate. Only implements `bits()` and a `from_bits_truncate()` that doesn't -// actually truncate. -#[cfg(not(feature = "bitflags"))] -macro_rules! bitflags { - ( - $(#[$attr:meta])* - pub struct $name:ident: $type:ty { - $($(#[$const_attr:meta])* const $const:ident = $val:expr;)* - } - ) => { - $(#[$attr])* - #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] - pub struct $name { bits: $type } - impl $name { - $($(#[$const_attr])* pub const $const: $name = $name{ bits: $val };)* - pub fn bits(&self) -> $type { self.bits } - pub fn from_bits_truncate(bits: $type) -> Self { $name{ bits } } - } - } -} diff --git a/crux-mir/lib/std/src/sys/cloudabi/abi/cloudabi.rs b/crux-mir/lib/std/src/sys/cloudabi/abi/cloudabi.rs deleted file mode 100644 index b02faf183..000000000 --- a/crux-mir/lib/std/src/sys/cloudabi/abi/cloudabi.rs +++ /dev/null @@ -1,2947 +0,0 @@ -// Copyright (c) 2016-2017 Nuxi and contributors. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -// ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -// OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -// SUCH DAMAGE. -// -// This file is automatically generated. Do not edit. -// -// Source: https://github.com/NuxiNL/cloudabi - -// Appease Rust's tidy. -// ignore-tidy-linelength - -//! **PLEASE NOTE: This entire crate including this -//! documentation is automatically generated from -//! [`cloudabi.txt`](https://github.com/NuxiNL/cloudabi/blob/master/cloudabi.txt)** -//! -//! # Nuxi CloudABI -//! -//! CloudABI is what you get if you take POSIX, add capability-based -//! security, and remove everything that's incompatible with that. The -//! result is a minimal ABI consisting of only 49 syscalls. -//! -//! CloudABI doesn't have its own kernel, but instead is implemented in existing -//! kernels: FreeBSD has CloudABI support for x86-64 and arm64, and [a patch-set -//! for NetBSD](https://github.com/NuxiNL/netbsd) and [a patch-set for -//! Linux](https://github.com/NuxiNL/linux) are available as well. This means that -//! CloudABI binaries can be executed on different operating systems, without any -//! modification. -//! -//! ## Capability-Based Security -//! -//! Capability-based security means that processes can only perform -//! actions that have no global impact. Processes cannot open files by -//! their absolute path, cannot open network connections, and cannot -//! observe global system state such as the process table. -//! -//! The capabilities of a process are fully determined by its set of open -//! file descriptors (fds). For example, files can only be opened if the -//! process already has a file descriptor to a directory the file is in. -//! -//! Unlike in POSIX, where processes are normally started with file -//! descriptors 0, 1, and 2 reserved for standard input, output, and -//! error, CloudABI does not reserve any file descriptor numbers for -//! specific purposes. -//! -//! In CloudABI, a process depends on its parent process to launch it with -//! the right set of resources, since the process will not be able to open -//! any new resources. For example, a simple static web server would need -//! to be started with a file descriptor to a [TCP -//! listener](https://github.com/NuxiNL/flower), and a file descriptor to -//! the directory for which to serve files. The web server will then be -//! unable to do anything other than reading files in that directory, and -//! process incoming network connections. -//! -//! So, unknown CloudABI binaries can safely be executed without the need -//! for containers, virtual machines, or other sandboxing technologies. -//! -//! Watch [Ed Schouten's Talk at -//! 32C3](https://www.youtube.com/watch?v=3N29vrPoDv8) for more -//! information about what capability-based security for UNIX means. -//! -//! ## Cloudlibc -//! -//! [Cloudlibc](https://github.com/NuxiNL/cloudlibc) is an implementation -//! of the C standard library, without all CloudABI-incompatible -//! functions. For example, Cloudlibc does not have `printf`, but does -//! have `fprintf`. It does not have `open`, but does have `openat`. -//! -//! ## CloudABI-Ports -//! -//! [CloudABI-Ports](https://github.com/NuxiNL/cloudabi-ports) is a -//! collection of ports of commonly used libraries and applications to -//! CloudABI. It contains software such as `zlib`, `libpng`, `boost`, -//! `memcached`, and much more. The software is patched to not depend on -//! any global state, such as files in `/etc` or `/dev`, using `open()`, -//! etc. -//! -//! ## Using CloudABI -//! -//! Instructions for using CloudABI (including kernel modules/patches, -//! toolchain, and ports) are available for several operating systems: -//! -//! - [Arch Linux](https://nuxi.nl/cloudabi/archlinux/) -//! - [Debian, Ubuntu, and other Debian derivatives](https://nuxi.nl/cloudabi/debian/) -//! - [FreeBSD, PC-BSD and DragonFly BSD](https://nuxi.nl/cloudabi/freebsd/) -//! - [Mac OS X](https://nuxi.nl/cloudabi/mac/) -//! - [NetBSD](https://nuxi.nl/cloudabi/netbsd/) -//! -//! ## Specification of the ABI -//! -//! The entire ABI is specified in a file called -//! [`cloudabi.txt`](https://github.com/NuxiNL/cloudabi/blob/master/cloudabi.txt), -//! from which all -//! [headers](https://github.com/NuxiNL/cloudabi/tree/master/headers) -//! and documentation (including the one you're reading now) is generated. - -#![no_std] -#![allow(non_camel_case_types)] -#![allow(deprecated)] // FIXME: using `mem::uninitialized()` - -include!("bitflags.rs"); - -/// File or memory access pattern advisory information. -#[repr(u8)] -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] -#[non_exhaustive] -pub enum advice { - /// The application expects that it will not access the - /// specified data in the near future. - DONTNEED = 1, - /// The application expects to access the specified data - /// once and then not reuse it thereafter. - NOREUSE = 2, - /// The application has no advice to give on its behavior - /// with respect to the specified data. - NORMAL = 3, - /// The application expects to access the specified data - /// in a random order. - RANDOM = 4, - /// The application expects to access the specified data - /// sequentially from lower offsets to higher offsets. - SEQUENTIAL = 5, - /// The application expects to access the specified data - /// in the near future. - WILLNEED = 6, -} - -/// Enumeration describing the kind of value stored in [`auxv`](struct.auxv.html). -#[repr(u32)] -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] -#[non_exhaustive] -pub enum auxtype { - /// Base address of the binary argument data provided to - /// [`proc_exec()`](fn.proc_exec.html). - ARGDATA = 256, - /// Length of the binary argument data provided to - /// [`proc_exec()`](fn.proc_exec.html). - ARGDATALEN = 257, - /// Base address at which the executable is placed in - /// memory. - BASE = 7, - /// Base address of a buffer of random data that may be - /// used for non-cryptographic purposes, for example as a - /// canary for stack smashing protection. - CANARY = 258, - /// Length of a buffer of random data that may be used - /// for non-cryptographic purposes, for example as a - /// canary for stack smashing protection. - CANARYLEN = 259, - /// Number of CPUs that the system this process is running - /// on has. - NCPUS = 260, - /// Terminator of the auxiliary vector. - NULL = 0, - /// Smallest memory object size for which individual - /// memory protection controls can be configured. - PAGESZ = 6, - /// Address of the first ELF program header of the - /// executable. - PHDR = 3, - /// Number of ELF program headers of the executable. - PHNUM = 4, - /// Identifier of the process. - /// - /// This environment does not provide any simple numerical - /// process identifiers, for the reason that these are not - /// useful in distributed contexts. Instead, processes are - /// identified by a UUID. - /// - /// This record should point to sixteen bytes of binary - /// data, containing a version 4 UUID (fully random). - PID = 263, - /// Address of the ELF header of the vDSO. - /// - /// The vDSO is a shared library that is mapped in the - /// address space of the process. It provides entry points - /// for every system call supported by the environment, - /// all having a corresponding symbol that is prefixed - /// with `cloudabi_sys_`. System calls should be invoked - /// through these entry points. - /// - /// The first advantage of letting processes call into a - /// vDSO to perform system calls instead of raising - /// hardware traps is that it allows for easy emulation of - /// executables on top of existing operating systems. The - /// second advantage is that in cases where an operating - /// system provides native support for CloudABI executables, - /// it may still implement partial userspace - /// implementations of these system calls to improve - /// performance (e.g., [`clock_time_get()`](fn.clock_time_get.html)). It also provides - /// a more dynamic way of adding, removing or replacing - /// system calls. - SYSINFO_EHDR = 262, - /// Thread ID of the initial thread of the process. - TID = 261, -} - -/// Identifiers for clocks. -#[repr(u32)] -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] -#[non_exhaustive] -pub enum clockid { - /// The system-wide monotonic clock, which is defined as a - /// clock measuring real time, whose value cannot be - /// adjusted and which cannot have negative clock jumps. - /// - /// The epoch of this clock is undefined. The absolute - /// time value of this clock therefore has no meaning. - MONOTONIC = 1, - /// The CPU-time clock associated with the current - /// process. - PROCESS_CPUTIME_ID = 2, - /// The system-wide clock measuring real time. Time value - /// zero corresponds with 1970-01-01T00:00:00Z. - REALTIME = 3, - /// The CPU-time clock associated with the current thread. - THREAD_CPUTIME_ID = 4, -} - -/// A userspace condition variable. -#[repr(C)] -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] -pub struct condvar(pub u32); -/// The condition variable is in its initial state. There -/// are no threads waiting to be woken up. If the -/// condition variable has any other value, the kernel -/// must be called to wake up any sleeping threads. -pub const CONDVAR_HAS_NO_WAITERS: condvar = condvar(0); - -/// Identifier for a device containing a file system. Can be used -/// in combination with [`inode`](struct.inode.html) to uniquely identify a file on the -/// local system. -#[repr(C)] -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] -pub struct device(pub u64); - -/// A reference to the offset of a directory entry. -#[repr(C)] -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] -pub struct dircookie(pub u64); -/// Permanent reference to the first directory entry -/// within a directory. -pub const DIRCOOKIE_START: dircookie = dircookie(0); - -/// Error codes returned by system calls. -/// -/// Not all of these error codes are returned by the system calls -/// provided by this environment, but are either used in userspace -/// exclusively or merely provided for alignment with POSIX. -#[repr(u16)] -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] -#[non_exhaustive] -pub enum errno { - /// No error occurred. System call completed successfully. - SUCCESS = 0, - /// Argument list too long. - TOOBIG = 1, - /// Permission denied. - ACCES = 2, - /// Address in use. - ADDRINUSE = 3, - /// Address not available. - ADDRNOTAVAIL = 4, - /// Address family not supported. - AFNOSUPPORT = 5, - /// Resource unavailable, or operation would block. - AGAIN = 6, - /// Connection already in progress. - ALREADY = 7, - /// Bad file descriptor. - BADF = 8, - /// Bad message. - BADMSG = 9, - /// Device or resource busy. - BUSY = 10, - /// Operation canceled. - CANCELED = 11, - /// No child processes. - CHILD = 12, - /// Connection aborted. - CONNABORTED = 13, - /// Connection refused. - CONNREFUSED = 14, - /// Connection reset. - CONNRESET = 15, - /// Resource deadlock would occur. - DEADLK = 16, - /// Destination address required. - DESTADDRREQ = 17, - /// Mathematics argument out of domain of function. - DOM = 18, - /// Reserved. - DQUOT = 19, - /// File exists. - EXIST = 20, - /// Bad address. - FAULT = 21, - /// File too large. - FBIG = 22, - /// Host is unreachable. - HOSTUNREACH = 23, - /// Identifier removed. - IDRM = 24, - /// Illegal byte sequence. - ILSEQ = 25, - /// Operation in progress. - INPROGRESS = 26, - /// Interrupted function. - INTR = 27, - /// Invalid argument. - INVAL = 28, - /// I/O error. - IO = 29, - /// Socket is connected. - ISCONN = 30, - /// Is a directory. - ISDIR = 31, - /// Too many levels of symbolic links. - LOOP = 32, - /// File descriptor value too large. - MFILE = 33, - /// Too many links. - MLINK = 34, - /// Message too large. - MSGSIZE = 35, - /// Reserved. - MULTIHOP = 36, - /// Filename too long. - NAMETOOLONG = 37, - /// Network is down. - NETDOWN = 38, - /// Connection aborted by network. - NETRESET = 39, - /// Network unreachable. - NETUNREACH = 40, - /// Too many files open in system. - NFILE = 41, - /// No buffer space available. - NOBUFS = 42, - /// No such device. - NODEV = 43, - /// No such file or directory. - NOENT = 44, - /// Executable file format error. - NOEXEC = 45, - /// No locks available. - NOLCK = 46, - /// Reserved. - NOLINK = 47, - /// Not enough space. - NOMEM = 48, - /// No message of the desired type. - NOMSG = 49, - /// Protocol not available. - NOPROTOOPT = 50, - /// No space left on device. - NOSPC = 51, - /// Function not supported. - NOSYS = 52, - /// The socket is not connected. - NOTCONN = 53, - /// Not a directory or a symbolic link to a directory. - NOTDIR = 54, - /// Directory not empty. - NOTEMPTY = 55, - /// State not recoverable. - NOTRECOVERABLE = 56, - /// Not a socket. - NOTSOCK = 57, - /// Not supported, or operation not supported on socket. - NOTSUP = 58, - /// Inappropriate I/O control operation. - NOTTY = 59, - /// No such device or address. - NXIO = 60, - /// Value too large to be stored in data type. - OVERFLOW = 61, - /// Previous owner died. - OWNERDEAD = 62, - /// Operation not permitted. - PERM = 63, - /// Broken pipe. - PIPE = 64, - /// Protocol error. - PROTO = 65, - /// Protocol not supported. - PROTONOSUPPORT = 66, - /// Protocol wrong type for socket. - PROTOTYPE = 67, - /// Result too large. - RANGE = 68, - /// Read-only file system. - ROFS = 69, - /// Invalid seek. - SPIPE = 70, - /// No such process. - SRCH = 71, - /// Reserved. - STALE = 72, - /// Connection timed out. - TIMEDOUT = 73, - /// Text file busy. - TXTBSY = 74, - /// Cross-device link. - XDEV = 75, - /// Extension: Capabilities insufficient. - NOTCAPABLE = 76, -} - -bitflags! { - /// The state of the file descriptor subscribed to with - /// [`FD_READ`](enum.eventtype.html#variant.FD_READ) or [`FD_WRITE`](enum.eventtype.html#variant.FD_WRITE). - #[repr(C)] - pub struct eventrwflags: u16 { - /// The peer of this socket has closed or disconnected. - const HANGUP = 0x0001; - } -} - -/// Type of a subscription to an event or its occurrence. -#[repr(u8)] -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] -#[non_exhaustive] -pub enum eventtype { - /// The time value of clock [`subscription.union.clock.clock_id`](struct.subscription_clock.html#structfield.clock_id) - /// has reached timestamp [`subscription.union.clock.timeout`](struct.subscription_clock.html#structfield.timeout). - CLOCK = 1, - /// Condition variable [`subscription.union.condvar.condvar`](struct.subscription_condvar.html#structfield.condvar) has - /// been woken up and [`subscription.union.condvar.lock`](struct.subscription_condvar.html#structfield.lock) has been - /// acquired for writing. - CONDVAR = 2, - /// File descriptor [`subscription.union.fd_readwrite.fd`](struct.subscription_fd_readwrite.html#structfield.fd) has - /// data available for reading. This event always triggers - /// for regular files. - FD_READ = 3, - /// File descriptor [`subscription.union.fd_readwrite.fd`](struct.subscription_fd_readwrite.html#structfield.fd) has - /// capacity available for writing. This event always - /// triggers for regular files. - FD_WRITE = 4, - /// Lock [`subscription.union.lock.lock`](struct.subscription_lock.html#structfield.lock) has been acquired for - /// reading. - LOCK_RDLOCK = 5, - /// Lock [`subscription.union.lock.lock`](struct.subscription_lock.html#structfield.lock) has been acquired for - /// writing. - LOCK_WRLOCK = 6, - /// The process associated with process descriptor - /// [`subscription.union.proc_terminate.fd`](struct.subscription_proc_terminate.html#structfield.fd) has terminated. - PROC_TERMINATE = 7, -} - -/// Exit code generated by a process when exiting. -pub type exitcode = u32; - -/// A file descriptor number. -/// -/// Unlike on POSIX-compliant systems, none of the file descriptor -/// numbers are reserved for a purpose (e.g., stdin, stdout, -/// stderr). Operating systems are not required to allocate new -/// file descriptors in ascending order. -#[repr(C)] -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] -pub struct fd(pub u32); -/// Returned to the child process by [`proc_fork()`](fn.proc_fork.html). -pub const PROCESS_CHILD: fd = fd(0xffffffff); -/// Passed to [`mem_map()`](fn.mem_map.html) when creating a mapping to -/// anonymous memory. -pub const MAP_ANON_FD: fd = fd(0xffffffff); - -bitflags! { - /// File descriptor flags. - #[repr(C)] - pub struct fdflags: u16 { - /// Append mode: Data written to the file is always - /// appended to the file's end. - const APPEND = 0x0001; - /// Write according to synchronized I/O data integrity - /// completion. Only the data stored in the file is - /// synchronized. - const DSYNC = 0x0002; - /// Non-blocking mode. - const NONBLOCK = 0x0004; - /// Synchronized read I/O operations. - const RSYNC = 0x0008; - /// Write according to synchronized I/O file integrity - /// completion. In addition to synchronizing the data - /// stored in the file, the system may also synchronously - /// update the file's metadata. - const SYNC = 0x0010; - } -} - -bitflags! { - /// Which file descriptor attributes to adjust. - #[repr(C)] - pub struct fdsflags: u16 { - /// Adjust the file descriptor flags stored in - /// [`fdstat.fs_flags`](struct.fdstat.html#structfield.fs_flags). - const FLAGS = 0x0001; - /// Restrict the rights of the file descriptor to the - /// rights stored in [`fdstat.fs_rights_base`](struct.fdstat.html#structfield.fs_rights_base) and - /// [`fdstat.fs_rights_inheriting`](struct.fdstat.html#structfield.fs_rights_inheriting). - const RIGHTS = 0x0002; - } -} - -/// Relative offset within a file. -pub type filedelta = i64; - -/// Non-negative file size or length of a region within a file. -pub type filesize = u64; - -/// The type of a file descriptor or file. -#[repr(u8)] -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] -#[non_exhaustive] -pub enum filetype { - /// The type of the file descriptor or file is unknown or - /// is different from any of the other types specified. - UNKNOWN = 0, - /// The file descriptor or file refers to a block device - /// inode. - BLOCK_DEVICE = 16, - /// The file descriptor or file refers to a character - /// device inode. - CHARACTER_DEVICE = 17, - /// The file descriptor or file refers to a directory - /// inode. - DIRECTORY = 32, - /// The file descriptor refers to a process handle. - PROCESS = 80, - /// The file descriptor or file refers to a regular file - /// inode. - REGULAR_FILE = 96, - /// The file descriptor refers to a shared memory object. - SHARED_MEMORY = 112, - /// The file descriptor or file refers to a datagram - /// socket. - SOCKET_DGRAM = 128, - /// The file descriptor or file refers to a byte-stream - /// socket. - SOCKET_STREAM = 130, - /// The file refers to a symbolic link inode. - SYMBOLIC_LINK = 144, -} - -bitflags! { - /// Which file attributes to adjust. - #[repr(C)] - pub struct fsflags: u16 { - /// Adjust the last data access timestamp to the value - /// stored in [`filestat.st_atim`](struct.filestat.html#structfield.st_atim). - const ATIM = 0x0001; - /// Adjust the last data access timestamp to the time - /// of clock [`REALTIME`](enum.clockid.html#variant.REALTIME). - const ATIM_NOW = 0x0002; - /// Adjust the last data modification timestamp to the - /// value stored in [`filestat.st_mtim`](struct.filestat.html#structfield.st_mtim). - const MTIM = 0x0004; - /// Adjust the last data modification timestamp to the - /// time of clock [`REALTIME`](enum.clockid.html#variant.REALTIME). - const MTIM_NOW = 0x0008; - /// Truncate or extend the file to the size stored in - /// [`filestat.st_size`](struct.filestat.html#structfield.st_size). - const SIZE = 0x0010; - } -} - -/// File serial number that is unique within its file system. -#[repr(C)] -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] -pub struct inode(pub u64); - -/// Number of hard links to an inode. -pub type linkcount = u32; - -/// A userspace read-recursive readers-writer lock, similar to a -/// Linux futex or a FreeBSD umtx. -#[repr(C)] -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] -pub struct lock(pub u32); -/// Value indicating that the lock is in its initial -/// unlocked state. -pub const LOCK_UNLOCKED: lock = lock(0x00000000); -/// Bitmask indicating that the lock is write-locked. If -/// set, the lower 30 bits of the lock contain the -/// identifier of the thread that owns the write lock. -/// Otherwise, the lower 30 bits of the lock contain the -/// number of acquired read locks. -pub const LOCK_WRLOCKED: lock = lock(0x40000000); -/// Bitmask indicating that the lock is either read locked -/// or write locked, and that one or more threads have -/// their execution suspended, waiting to acquire the -/// lock. The last owner of the lock must call the -/// kernel to unlock. -/// -/// When the lock is acquired for reading and this bit is -/// set, it means that one or more threads are attempting -/// to acquire this lock for writing. In that case, other -/// threads should only acquire additional read locks if -/// suspending execution would cause a deadlock. It is -/// preferred to suspend execution, as this prevents -/// starvation of writers. -pub const LOCK_KERNEL_MANAGED: lock = lock(0x80000000); -/// Value indicating that the lock is in an incorrect -/// state. A lock cannot be in its initial unlocked state, -/// while also managed by the kernel. -pub const LOCK_BOGUS: lock = lock(0x80000000); - -bitflags! { - /// Flags determining the method of how paths are resolved. - #[repr(C)] - pub struct lookupflags: u32 { - /// As long as the resolved path corresponds to a symbolic - /// link, it is expanded. - const SYMLINK_FOLLOW = 0x00000001; - } -} - -bitflags! { - /// Memory mapping flags. - #[repr(C)] - pub struct mflags: u8 { - /// Instead of mapping the contents of the file provided, - /// create a mapping to anonymous memory. The file - /// descriptor argument must be set to [`MAP_ANON_FD`](constant.MAP_ANON_FD.html), - /// and the offset must be set to zero. - const ANON = 0x01; - /// Require that the mapping is performed at the base - /// address provided. - const FIXED = 0x02; - /// Changes are private. - const PRIVATE = 0x04; - /// Changes are shared. - const SHARED = 0x08; - } -} - -bitflags! { - /// Memory page protection options. - /// - /// This implementation enforces the `W^X` property: Pages cannot be - /// mapped for execution while also mapped for writing. - #[repr(C)] - pub struct mprot: u8 { - /// Page can be executed. - const EXEC = 0x01; - /// Page can be written. - const WRITE = 0x02; - /// Page can be read. - const READ = 0x04; - } -} - -bitflags! { - /// Methods of synchronizing memory with physical storage. - #[repr(C)] - pub struct msflags: u8 { - /// Performs asynchronous writes. - const ASYNC = 0x01; - /// Invalidates cached data. - const INVALIDATE = 0x02; - /// Performs synchronous writes. - const SYNC = 0x04; - } -} - -/// Specifies the number of threads sleeping on a condition -/// variable that should be woken up. -pub type nthreads = u32; - -bitflags! { - /// Open flags used by [`file_open()`](fn.file_open.html). - #[repr(C)] - pub struct oflags: u16 { - /// Create file if it does not exist. - const CREAT = 0x0001; - /// Fail if not a directory. - const DIRECTORY = 0x0002; - /// Fail if file already exists. - const EXCL = 0x0004; - /// Truncate file to size 0. - const TRUNC = 0x0008; - } -} - -bitflags! { - /// Flags provided to [`sock_recv()`](fn.sock_recv.html). - #[repr(C)] - pub struct riflags: u16 { - /// Returns the message without removing it from the - /// socket's receive queue. - const PEEK = 0x0004; - /// On byte-stream sockets, block until the full amount - /// of data can be returned. - const WAITALL = 0x0010; - } -} - -bitflags! { - /// File descriptor rights, determining which actions may be - /// performed. - #[repr(C)] - pub struct rights: u64 { - /// The right to invoke [`fd_datasync()`](fn.fd_datasync.html). - /// - /// If [`FILE_OPEN`](struct.rights.html#associatedconstant.FILE_OPEN) is set, includes the right to - /// invoke [`file_open()`](fn.file_open.html) with [`DSYNC`](struct.fdflags.html#associatedconstant.DSYNC). - const FD_DATASYNC = 0x0000000000000001; - /// The right to invoke [`fd_read()`](fn.fd_read.html) and [`sock_recv()`](fn.sock_recv.html). - /// - /// If [`MEM_MAP`](struct.rights.html#associatedconstant.MEM_MAP) is set, includes the right to - /// invoke [`mem_map()`](fn.mem_map.html) with memory protection option - /// [`READ`](struct.mprot.html#associatedconstant.READ). - /// - /// If [`FD_SEEK`](struct.rights.html#associatedconstant.FD_SEEK) is set, includes the right to invoke - /// [`fd_pread()`](fn.fd_pread.html). - const FD_READ = 0x0000000000000002; - /// The right to invoke [`fd_seek()`](fn.fd_seek.html). This flag implies - /// [`FD_TELL`](struct.rights.html#associatedconstant.FD_TELL). - const FD_SEEK = 0x0000000000000004; - /// The right to invoke [`fd_stat_put()`](fn.fd_stat_put.html) with - /// [`FLAGS`](struct.fdsflags.html#associatedconstant.FLAGS). - const FD_STAT_PUT_FLAGS = 0x0000000000000008; - /// The right to invoke [`fd_sync()`](fn.fd_sync.html). - /// - /// If [`FILE_OPEN`](struct.rights.html#associatedconstant.FILE_OPEN) is set, includes the right to - /// invoke [`file_open()`](fn.file_open.html) with [`RSYNC`](struct.fdflags.html#associatedconstant.RSYNC) and - /// [`DSYNC`](struct.fdflags.html#associatedconstant.DSYNC). - const FD_SYNC = 0x0000000000000010; - /// The right to invoke [`fd_seek()`](fn.fd_seek.html) in such a way that the - /// file offset remains unaltered (i.e., [`CUR`](enum.whence.html#variant.CUR) with - /// offset zero). - const FD_TELL = 0x0000000000000020; - /// The right to invoke [`fd_write()`](fn.fd_write.html) and [`sock_send()`](fn.sock_send.html). - /// - /// If [`MEM_MAP`](struct.rights.html#associatedconstant.MEM_MAP) is set, includes the right to - /// invoke [`mem_map()`](fn.mem_map.html) with memory protection option - /// [`WRITE`](struct.mprot.html#associatedconstant.WRITE). - /// - /// If [`FD_SEEK`](struct.rights.html#associatedconstant.FD_SEEK) is set, includes the right to - /// invoke [`fd_pwrite()`](fn.fd_pwrite.html). - const FD_WRITE = 0x0000000000000040; - /// The right to invoke [`file_advise()`](fn.file_advise.html). - const FILE_ADVISE = 0x0000000000000080; - /// The right to invoke [`file_allocate()`](fn.file_allocate.html). - const FILE_ALLOCATE = 0x0000000000000100; - /// The right to invoke [`file_create()`](fn.file_create.html) with - /// [`DIRECTORY`](enum.filetype.html#variant.DIRECTORY). - const FILE_CREATE_DIRECTORY = 0x0000000000000200; - /// If [`FILE_OPEN`](struct.rights.html#associatedconstant.FILE_OPEN) is set, the right to invoke - /// [`file_open()`](fn.file_open.html) with [`CREAT`](struct.oflags.html#associatedconstant.CREAT). - const FILE_CREATE_FILE = 0x0000000000000400; - /// The right to invoke [`file_link()`](fn.file_link.html) with the file - /// descriptor as the source directory. - const FILE_LINK_SOURCE = 0x0000000000001000; - /// The right to invoke [`file_link()`](fn.file_link.html) with the file - /// descriptor as the target directory. - const FILE_LINK_TARGET = 0x0000000000002000; - /// The right to invoke [`file_open()`](fn.file_open.html). - const FILE_OPEN = 0x0000000000004000; - /// The right to invoke [`file_readdir()`](fn.file_readdir.html). - const FILE_READDIR = 0x0000000000008000; - /// The right to invoke [`file_readlink()`](fn.file_readlink.html). - const FILE_READLINK = 0x0000000000010000; - /// The right to invoke [`file_rename()`](fn.file_rename.html) with the file - /// descriptor as the source directory. - const FILE_RENAME_SOURCE = 0x0000000000020000; - /// The right to invoke [`file_rename()`](fn.file_rename.html) with the file - /// descriptor as the target directory. - const FILE_RENAME_TARGET = 0x0000000000040000; - /// The right to invoke [`file_stat_fget()`](fn.file_stat_fget.html). - const FILE_STAT_FGET = 0x0000000000080000; - /// The right to invoke [`file_stat_fput()`](fn.file_stat_fput.html) with - /// [`SIZE`](struct.fsflags.html#associatedconstant.SIZE). - /// - /// If [`FILE_OPEN`](struct.rights.html#associatedconstant.FILE_OPEN) is set, includes the right to - /// invoke [`file_open()`](fn.file_open.html) with [`TRUNC`](struct.oflags.html#associatedconstant.TRUNC). - const FILE_STAT_FPUT_SIZE = 0x0000000000100000; - /// The right to invoke [`file_stat_fput()`](fn.file_stat_fput.html) with - /// [`ATIM`](struct.fsflags.html#associatedconstant.ATIM), [`ATIM_NOW`](struct.fsflags.html#associatedconstant.ATIM_NOW), [`MTIM`](struct.fsflags.html#associatedconstant.MTIM), - /// and [`MTIM_NOW`](struct.fsflags.html#associatedconstant.MTIM_NOW). - const FILE_STAT_FPUT_TIMES = 0x0000000000200000; - /// The right to invoke [`file_stat_get()`](fn.file_stat_get.html). - const FILE_STAT_GET = 0x0000000000400000; - /// The right to invoke [`file_stat_put()`](fn.file_stat_put.html) with - /// [`ATIM`](struct.fsflags.html#associatedconstant.ATIM), [`ATIM_NOW`](struct.fsflags.html#associatedconstant.ATIM_NOW), [`MTIM`](struct.fsflags.html#associatedconstant.MTIM), - /// and [`MTIM_NOW`](struct.fsflags.html#associatedconstant.MTIM_NOW). - const FILE_STAT_PUT_TIMES = 0x0000000000800000; - /// The right to invoke [`file_symlink()`](fn.file_symlink.html). - const FILE_SYMLINK = 0x0000000001000000; - /// The right to invoke [`file_unlink()`](fn.file_unlink.html). - const FILE_UNLINK = 0x0000000002000000; - /// The right to invoke [`mem_map()`](fn.mem_map.html) with [`mprot`](struct.mprot.html) set to - /// zero. - const MEM_MAP = 0x0000000004000000; - /// If [`MEM_MAP`](struct.rights.html#associatedconstant.MEM_MAP) is set, the right to invoke - /// [`mem_map()`](fn.mem_map.html) with [`EXEC`](struct.mprot.html#associatedconstant.EXEC). - const MEM_MAP_EXEC = 0x0000000008000000; - /// If [`FD_READ`](struct.rights.html#associatedconstant.FD_READ) is set, includes the right to - /// invoke [`poll()`](fn.poll.html) to subscribe to [`FD_READ`](enum.eventtype.html#variant.FD_READ). - /// - /// If [`FD_WRITE`](struct.rights.html#associatedconstant.FD_WRITE) is set, includes the right to - /// invoke [`poll()`](fn.poll.html) to subscribe to [`FD_WRITE`](enum.eventtype.html#variant.FD_WRITE). - const POLL_FD_READWRITE = 0x0000000010000000; - /// The right to invoke [`poll()`](fn.poll.html) to subscribe to - /// [`PROC_TERMINATE`](enum.eventtype.html#variant.PROC_TERMINATE). - const POLL_PROC_TERMINATE = 0x0000000040000000; - /// The right to invoke [`proc_exec()`](fn.proc_exec.html). - const PROC_EXEC = 0x0000000100000000; - /// The right to invoke [`sock_shutdown()`](fn.sock_shutdown.html). - const SOCK_SHUTDOWN = 0x0000008000000000; - } -} - -bitflags! { - /// Flags returned by [`sock_recv()`](fn.sock_recv.html). - #[repr(C)] - pub struct roflags: u16 { - /// Returned by [`sock_recv()`](fn.sock_recv.html): List of file descriptors - /// has been truncated. - const FDS_TRUNCATED = 0x0001; - /// Returned by [`sock_recv()`](fn.sock_recv.html): Message data has been - /// truncated. - const DATA_TRUNCATED = 0x0008; - } -} - -/// Indicates whether an object is stored in private or shared -/// memory. -#[repr(u8)] -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] -#[non_exhaustive] -pub enum scope { - /// The object is stored in private memory. - PRIVATE = 4, - /// The object is stored in shared memory. - SHARED = 8, -} - -bitflags! { - /// Which channels on a socket need to be shut down. - #[repr(C)] - pub struct sdflags: u8 { - /// Disables further receive operations. - const RD = 0x01; - /// Disables further send operations. - const WR = 0x02; - } -} - -bitflags! { - /// Flags provided to [`sock_send()`](fn.sock_send.html). As there are currently no flags - /// defined, it must be set to zero. - #[repr(C)] - pub struct siflags: u16 { - const DEFAULT = 0; - } -} - -/// Signal condition. -#[repr(u8)] -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] -#[non_exhaustive] -pub enum signal { - /// Process abort signal. - /// - /// Action: Terminates the process. - ABRT = 1, - /// Alarm clock. - /// - /// Action: Terminates the process. - ALRM = 2, - /// Access to an undefined portion of a memory object. - /// - /// Action: Terminates the process. - BUS = 3, - /// Child process terminated, stopped, or continued. - /// - /// Action: Ignored. - CHLD = 4, - /// Continue executing, if stopped. - /// - /// Action: Continues executing, if stopped. - CONT = 5, - /// Erroneous arithmetic operation. - /// - /// Action: Terminates the process. - FPE = 6, - /// Hangup. - /// - /// Action: Terminates the process. - HUP = 7, - /// Illegal instruction. - /// - /// Action: Terminates the process. - ILL = 8, - /// Terminate interrupt signal. - /// - /// Action: Terminates the process. - INT = 9, - /// Kill. - /// - /// Action: Terminates the process. - KILL = 10, - /// Write on a pipe with no one to read it. - /// - /// Action: Ignored. - PIPE = 11, - /// Terminal quit signal. - /// - /// Action: Terminates the process. - QUIT = 12, - /// Invalid memory reference. - /// - /// Action: Terminates the process. - SEGV = 13, - /// Stop executing. - /// - /// Action: Stops executing. - STOP = 14, - /// Bad system call. - /// - /// Action: Terminates the process. - SYS = 15, - /// Termination signal. - /// - /// Action: Terminates the process. - TERM = 16, - /// Trace/breakpoint trap. - /// - /// Action: Terminates the process. - TRAP = 17, - /// Terminal stop signal. - /// - /// Action: Stops executing. - TSTP = 18, - /// Background process attempting read. - /// - /// Action: Stops executing. - TTIN = 19, - /// Background process attempting write. - /// - /// Action: Stops executing. - TTOU = 20, - /// High bandwidth data is available at a socket. - /// - /// Action: Ignored. - URG = 21, - /// User-defined signal 1. - /// - /// Action: Terminates the process. - USR1 = 22, - /// User-defined signal 2. - /// - /// Action: Terminates the process. - USR2 = 23, - /// Virtual timer expired. - /// - /// Action: Terminates the process. - VTALRM = 24, - /// CPU time limit exceeded. - /// - /// Action: Terminates the process. - XCPU = 25, - /// File size limit exceeded. - /// - /// Action: Terminates the process. - XFSZ = 26, -} - -bitflags! { - /// Flags determining how the timestamp provided in - /// [`subscription.union.clock.timeout`](struct.subscription_clock.html#structfield.timeout) should be interpreted. - #[repr(C)] - pub struct subclockflags: u16 { - /// If set, treat the timestamp provided in - /// [`subscription.union.clock.timeout`](struct.subscription_clock.html#structfield.timeout) as an absolute timestamp - /// of clock [`subscription.union.clock.clock_id`](struct.subscription_clock.html#structfield.clock_id). - /// - /// If clear, treat the timestamp provided in - /// [`subscription.union.clock.timeout`](struct.subscription_clock.html#structfield.timeout) relative to the current - /// time value of clock [`subscription.union.clock.clock_id`](struct.subscription_clock.html#structfield.clock_id). - const ABSTIME = 0x0001; - } -} - -bitflags! { - /// Flags influencing the method of polling for read or writing on - /// a file descriptor. - #[repr(C)] - pub struct subrwflags: u16 { - /// Deprecated. Must be set by callers and ignored by - /// implementations. - const POLL = 0x0001; - } -} - -/// Unique system-local identifier of a thread. This identifier is -/// only valid during the lifetime of the thread. -/// -/// Threads must be aware of their thread identifier, as it is -/// written it into locks when acquiring them for writing. It is -/// not advised to use these identifiers for any other purpose. -/// -/// As the thread identifier is also stored in [`lock`](struct.lock.html) when -/// [`LOCK_WRLOCKED`](constant.LOCK_WRLOCKED.html) is set, the top two bits of the thread -/// must always be set to zero. -#[repr(C)] -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] -pub struct tid(pub u32); - -/// Timestamp in nanoseconds. -pub type timestamp = u64; - -bitflags! { - /// Specifies whether files are unlinked or directories are - /// removed. - #[repr(C)] - pub struct ulflags: u8 { - /// If set, removes a directory. Otherwise, unlinks any - /// non-directory file. - const REMOVEDIR = 0x01; - } -} - -/// User-provided value that can be attached to objects that is -/// retained when extracted from the kernel. -pub type userdata = u64; - -/// Relative to which position the offset of the file descriptor -/// should be set. -#[repr(u8)] -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] -#[non_exhaustive] -pub enum whence { - /// Seek relative to current position. - CUR = 1, - /// Seek relative to end-of-file. - END = 2, - /// Seek relative to start-of-file. - SET = 3, -} - -/// Auxiliary vector entry. -/// -/// The auxiliary vector is a list of key-value pairs that is -/// provided to the process on startup. Unlike structures, it is -/// extensible, as it is possible to add new records later on. -/// The auxiliary vector is always terminated by an entry having -/// type [`NULL`](enum.auxtype.html#variant.NULL). -/// -/// The auxiliary vector is part of the x86-64 ABI, but is used by -/// this environment on all architectures. -#[repr(C)] -#[derive(Copy, Clone)] -pub struct auxv { - /// The type of the auxiliary vector entry. - pub a_type: auxtype, - pub union: auxv_union, -} -/// A union inside `auxv`. -#[repr(C)] -#[derive(Copy, Clone)] -pub union auxv_union { - /// Used when `a_type` is [`ARGDATALEN`](enum.auxtype.html#variant.ARGDATALEN), [`CANARYLEN`](enum.auxtype.html#variant.CANARYLEN), [`NCPUS`](enum.auxtype.html#variant.NCPUS), [`PAGESZ`](enum.auxtype.html#variant.PAGESZ), [`PHNUM`](enum.auxtype.html#variant.PHNUM), or [`TID`](enum.auxtype.html#variant.TID). - /// A numerical value. - pub a_val: usize, - /// Used when `a_type` is [`ARGDATA`](enum.auxtype.html#variant.ARGDATA), [`BASE`](enum.auxtype.html#variant.BASE), [`CANARY`](enum.auxtype.html#variant.CANARY), [`PHDR`](enum.auxtype.html#variant.PHDR), [`PID`](enum.auxtype.html#variant.PID), or [`SYSINFO_EHDR`](enum.auxtype.html#variant.SYSINFO_EHDR). - /// A pointer value. - pub a_ptr: *mut (), -} -#[test] -#[cfg(target_pointer_width = "32")] -fn auxv_layout_test_32() { - assert_eq!(core::mem::size_of::(), 8); - assert_eq!(core::mem::align_of::(), 4); - unsafe { - let obj: auxv = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.a_type as *const _ as usize - base, 0); - assert_eq!(&obj.union.a_val as *const _ as usize - base, 4); - assert_eq!(&obj.union.a_ptr as *const _ as usize - base, 4); - } -} -#[test] -#[cfg(target_pointer_width = "64")] -fn auxv_layout_test_64() { - assert_eq!(core::mem::size_of::(), 16); - assert_eq!(core::mem::align_of::(), 8); - unsafe { - let obj: auxv = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.a_type as *const _ as usize - base, 0); - assert_eq!(&obj.union.a_val as *const _ as usize - base, 8); - assert_eq!(&obj.union.a_ptr as *const _ as usize - base, 8); - } -} - -/// A region of memory for scatter/gather writes. -#[repr(C)] -#[derive(Copy, Clone)] -pub struct ciovec { - /// The address and length of the buffer to be written. - pub buf: (*const (), usize), -} -#[test] -#[cfg(target_pointer_width = "32")] -fn ciovec_layout_test_32() { - assert_eq!(core::mem::size_of::(), 8); - assert_eq!(core::mem::align_of::(), 4); - unsafe { - let obj: ciovec = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.buf.0 as *const _ as usize - base, 0); - assert_eq!(&obj.buf.1 as *const _ as usize - base, 4); - } -} -#[test] -#[cfg(target_pointer_width = "64")] -fn ciovec_layout_test_64() { - assert_eq!(core::mem::size_of::(), 16); - assert_eq!(core::mem::align_of::(), 8); - unsafe { - let obj: ciovec = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.buf.0 as *const _ as usize - base, 0); - assert_eq!(&obj.buf.1 as *const _ as usize - base, 8); - } -} - -/// A directory entry. -#[repr(C)] -#[derive(Copy, Clone)] -pub struct dirent { - /// The offset of the next directory entry stored in this - /// directory. - pub d_next: dircookie, - /// The serial number of the file referred to by this - /// directory entry. - pub d_ino: inode, - /// The length of the name of the directory entry. - pub d_namlen: u32, - /// The type of the file referred to by this directory - /// entry. - pub d_type: filetype, -} -#[test] -fn dirent_layout_test() { - assert_eq!(core::mem::size_of::(), 24); - assert_eq!(core::mem::align_of::(), 8); - unsafe { - let obj: dirent = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.d_next as *const _ as usize - base, 0); - assert_eq!(&obj.d_ino as *const _ as usize - base, 8); - assert_eq!(&obj.d_namlen as *const _ as usize - base, 16); - assert_eq!(&obj.d_type as *const _ as usize - base, 20); - } -} - -/// An event that occurred. -#[repr(C)] -#[derive(Copy, Clone)] -pub struct event { - /// User-provided value that got attached to - /// [`subscription.userdata`](struct.subscription.html#structfield.userdata). - pub userdata: userdata, - /// If non-zero, an error that occurred while processing - /// the subscription request. - pub error: errno, - /// The type of the event that occurred. - pub type_: eventtype, - pub union: event_union, -} -/// A union inside `event`. -#[repr(C)] -#[derive(Copy, Clone)] -pub union event_union { - /// Used when `type_` is [`FD_READ`](enum.eventtype.html#variant.FD_READ) or [`FD_WRITE`](enum.eventtype.html#variant.FD_WRITE). - pub fd_readwrite: event_fd_readwrite, - /// Used when `type_` is [`PROC_TERMINATE`](enum.eventtype.html#variant.PROC_TERMINATE). - pub proc_terminate: event_proc_terminate, -} -#[repr(C)] -#[derive(Copy, Clone)] -pub struct event_fd_readwrite { - /// The number of bytes available - /// for reading or writing. - pub nbytes: filesize, - /// Obsolete. - pub unused: [u8; 4], - /// The state of the file - /// descriptor. - pub flags: eventrwflags, -} -#[repr(C)] -#[derive(Copy, Clone)] -pub struct event_proc_terminate { - /// Obsolete. - pub unused: [u8; 4], - /// If zero, the process has - /// exited. - /// Otherwise, the signal - /// condition causing it to - /// terminated. - pub signal: signal, - /// If exited, the exit code of - /// the process. - pub exitcode: exitcode, -} -#[test] -fn event_layout_test() { - assert_eq!(core::mem::size_of::(), 32); - assert_eq!(core::mem::align_of::(), 8); - unsafe { - let obj: event = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.userdata as *const _ as usize - base, 0); - assert_eq!(&obj.error as *const _ as usize - base, 8); - assert_eq!(&obj.type_ as *const _ as usize - base, 10); - assert_eq!(&obj.union.fd_readwrite.nbytes as *const _ as usize - base, 16); - assert_eq!(&obj.union.fd_readwrite.unused as *const _ as usize - base, 24); - assert_eq!(&obj.union.fd_readwrite.flags as *const _ as usize - base, 28); - assert_eq!(&obj.union.proc_terminate.unused as *const _ as usize - base, 16); - assert_eq!(&obj.union.proc_terminate.signal as *const _ as usize - base, 20); - assert_eq!(&obj.union.proc_terminate.exitcode as *const _ as usize - base, 24); - } -} - -/// File descriptor attributes. -#[repr(C)] -#[derive(Copy, Clone)] -pub struct fdstat { - /// File type. - pub fs_filetype: filetype, - /// File descriptor flags. - pub fs_flags: fdflags, - /// Rights that apply to this file descriptor. - pub fs_rights_base: rights, - /// Maximum set of rights that can be installed on new - /// file descriptors that are created through this file - /// descriptor, e.g., through [`file_open()`](fn.file_open.html). - pub fs_rights_inheriting: rights, -} -#[test] -fn fdstat_layout_test() { - assert_eq!(core::mem::size_of::(), 24); - assert_eq!(core::mem::align_of::(), 8); - unsafe { - let obj: fdstat = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.fs_filetype as *const _ as usize - base, 0); - assert_eq!(&obj.fs_flags as *const _ as usize - base, 2); - assert_eq!(&obj.fs_rights_base as *const _ as usize - base, 8); - assert_eq!(&obj.fs_rights_inheriting as *const _ as usize - base, 16); - } -} - -/// File attributes. -#[repr(C)] -#[derive(Copy, Clone)] -pub struct filestat { - /// Device ID of device containing the file. - pub st_dev: device, - /// File serial number. - pub st_ino: inode, - /// File type. - pub st_filetype: filetype, - /// Number of hard links to the file. - pub st_nlink: linkcount, - /// For regular files, the file size in bytes. For - /// symbolic links, the length in bytes of the pathname - /// contained in the symbolic link. - pub st_size: filesize, - /// Last data access timestamp. - pub st_atim: timestamp, - /// Last data modification timestamp. - pub st_mtim: timestamp, - /// Last file status change timestamp. - pub st_ctim: timestamp, -} -#[test] -fn filestat_layout_test() { - assert_eq!(core::mem::size_of::(), 56); - assert_eq!(core::mem::align_of::(), 8); - unsafe { - let obj: filestat = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.st_dev as *const _ as usize - base, 0); - assert_eq!(&obj.st_ino as *const _ as usize - base, 8); - assert_eq!(&obj.st_filetype as *const _ as usize - base, 16); - assert_eq!(&obj.st_nlink as *const _ as usize - base, 20); - assert_eq!(&obj.st_size as *const _ as usize - base, 24); - assert_eq!(&obj.st_atim as *const _ as usize - base, 32); - assert_eq!(&obj.st_mtim as *const _ as usize - base, 40); - assert_eq!(&obj.st_ctim as *const _ as usize - base, 48); - } -} - -/// A region of memory for scatter/gather reads. -#[repr(C)] -#[derive(Copy, Clone)] -pub struct iovec { - /// The address and length of the buffer to be filled. - pub buf: (*mut (), usize), -} -#[test] -#[cfg(target_pointer_width = "32")] -fn iovec_layout_test_32() { - assert_eq!(core::mem::size_of::(), 8); - assert_eq!(core::mem::align_of::(), 4); - unsafe { - let obj: iovec = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.buf.0 as *const _ as usize - base, 0); - assert_eq!(&obj.buf.1 as *const _ as usize - base, 4); - } -} -#[test] -#[cfg(target_pointer_width = "64")] -fn iovec_layout_test_64() { - assert_eq!(core::mem::size_of::(), 16); - assert_eq!(core::mem::align_of::(), 8); - unsafe { - let obj: iovec = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.buf.0 as *const _ as usize - base, 0); - assert_eq!(&obj.buf.1 as *const _ as usize - base, 8); - } -} - -/// Path lookup properties. -#[repr(C)] -#[derive(Copy, Clone)] -pub struct lookup { - /// The working directory at which the resolution of the - /// path starts. - pub fd: fd, - /// Flags determining the method of how the path is - /// resolved. - pub flags: lookupflags, -} -#[test] -fn lookup_layout_test() { - assert_eq!(core::mem::size_of::(), 8); - assert_eq!(core::mem::align_of::(), 4); - unsafe { - let obj: lookup = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.fd as *const _ as usize - base, 0); - assert_eq!(&obj.flags as *const _ as usize - base, 4); - } -} - -/// Entry point for a process (`_start`). -/// -/// **auxv**: -/// The auxiliary vector. See [`auxv`](struct.auxv.html). -pub type processentry = unsafe extern "C" fn(auxv: *const auxv) -> (); - -/// Arguments of [`sock_recv()`](fn.sock_recv.html). -#[repr(C)] -#[derive(Copy, Clone)] -pub struct recv_in { - /// List of scatter/gather vectors where message data - /// should be stored. - pub ri_data: (*const iovec, usize), - /// Buffer where numbers of incoming file descriptors - /// should be stored. - pub ri_fds: (*mut fd, usize), - /// Message flags. - pub ri_flags: riflags, -} -#[test] -#[cfg(target_pointer_width = "32")] -fn recv_in_layout_test_32() { - assert_eq!(core::mem::size_of::(), 20); - assert_eq!(core::mem::align_of::(), 4); - unsafe { - let obj: recv_in = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.ri_data.0 as *const _ as usize - base, 0); - assert_eq!(&obj.ri_data.1 as *const _ as usize - base, 4); - assert_eq!(&obj.ri_fds.0 as *const _ as usize - base, 8); - assert_eq!(&obj.ri_fds.1 as *const _ as usize - base, 12); - assert_eq!(&obj.ri_flags as *const _ as usize - base, 16); - } -} -#[test] -#[cfg(target_pointer_width = "64")] -fn recv_in_layout_test_64() { - assert_eq!(core::mem::size_of::(), 40); - assert_eq!(core::mem::align_of::(), 8); - unsafe { - let obj: recv_in = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.ri_data.0 as *const _ as usize - base, 0); - assert_eq!(&obj.ri_data.1 as *const _ as usize - base, 8); - assert_eq!(&obj.ri_fds.0 as *const _ as usize - base, 16); - assert_eq!(&obj.ri_fds.1 as *const _ as usize - base, 24); - assert_eq!(&obj.ri_flags as *const _ as usize - base, 32); - } -} - -/// Results of [`sock_recv()`](fn.sock_recv.html). -#[repr(C)] -#[derive(Copy, Clone)] -pub struct recv_out { - /// Number of bytes stored in [`recv_in.ri_data`](struct.recv_in.html#structfield.ri_data). - pub ro_datalen: usize, - /// Number of file descriptors stored in [`recv_in.ri_fds`](struct.recv_in.html#structfield.ri_fds). - pub ro_fdslen: usize, - /// Fields that were used by previous implementations. - pub ro_unused: [u8; 40], - /// Message flags. - pub ro_flags: roflags, -} -#[test] -#[cfg(target_pointer_width = "32")] -fn recv_out_layout_test_32() { - assert_eq!(core::mem::size_of::(), 52); - assert_eq!(core::mem::align_of::(), 4); - unsafe { - let obj: recv_out = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.ro_datalen as *const _ as usize - base, 0); - assert_eq!(&obj.ro_fdslen as *const _ as usize - base, 4); - assert_eq!(&obj.ro_unused as *const _ as usize - base, 8); - assert_eq!(&obj.ro_flags as *const _ as usize - base, 48); - } -} -#[test] -#[cfg(target_pointer_width = "64")] -fn recv_out_layout_test_64() { - assert_eq!(core::mem::size_of::(), 64); - assert_eq!(core::mem::align_of::(), 8); - unsafe { - let obj: recv_out = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.ro_datalen as *const _ as usize - base, 0); - assert_eq!(&obj.ro_fdslen as *const _ as usize - base, 8); - assert_eq!(&obj.ro_unused as *const _ as usize - base, 16); - assert_eq!(&obj.ro_flags as *const _ as usize - base, 56); - } -} - -/// Arguments of [`sock_send()`](fn.sock_send.html). -#[repr(C)] -#[derive(Copy, Clone)] -pub struct send_in { - /// List of scatter/gather vectors where message data - /// should be retrieved. - pub si_data: (*const ciovec, usize), - /// File descriptors that need to be attached to the - /// message. - pub si_fds: (*const fd, usize), - /// Message flags. - pub si_flags: siflags, -} -#[test] -#[cfg(target_pointer_width = "32")] -fn send_in_layout_test_32() { - assert_eq!(core::mem::size_of::(), 20); - assert_eq!(core::mem::align_of::(), 4); - unsafe { - let obj: send_in = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.si_data.0 as *const _ as usize - base, 0); - assert_eq!(&obj.si_data.1 as *const _ as usize - base, 4); - assert_eq!(&obj.si_fds.0 as *const _ as usize - base, 8); - assert_eq!(&obj.si_fds.1 as *const _ as usize - base, 12); - assert_eq!(&obj.si_flags as *const _ as usize - base, 16); - } -} -#[test] -#[cfg(target_pointer_width = "64")] -fn send_in_layout_test_64() { - assert_eq!(core::mem::size_of::(), 40); - assert_eq!(core::mem::align_of::(), 8); - unsafe { - let obj: send_in = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.si_data.0 as *const _ as usize - base, 0); - assert_eq!(&obj.si_data.1 as *const _ as usize - base, 8); - assert_eq!(&obj.si_fds.0 as *const _ as usize - base, 16); - assert_eq!(&obj.si_fds.1 as *const _ as usize - base, 24); - assert_eq!(&obj.si_flags as *const _ as usize - base, 32); - } -} - -/// Results of [`sock_send()`](fn.sock_send.html). -#[repr(C)] -#[derive(Copy, Clone)] -pub struct send_out { - /// Number of bytes transmitted. - pub so_datalen: usize, -} -#[test] -#[cfg(target_pointer_width = "32")] -fn send_out_layout_test_32() { - assert_eq!(core::mem::size_of::(), 4); - assert_eq!(core::mem::align_of::(), 4); - unsafe { - let obj: send_out = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.so_datalen as *const _ as usize - base, 0); - } -} -#[test] -#[cfg(target_pointer_width = "64")] -fn send_out_layout_test_64() { - assert_eq!(core::mem::size_of::(), 8); - assert_eq!(core::mem::align_of::(), 8); - unsafe { - let obj: send_out = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.so_datalen as *const _ as usize - base, 0); - } -} - -/// Subscription to an event. -#[repr(C)] -#[derive(Copy, Clone)] -pub struct subscription { - /// User-provided value that is attached to the - /// subscription in the kernel and returned through - /// [`event.userdata`](struct.event.html#structfield.userdata). - pub userdata: userdata, - /// Used by previous implementations. Ignored. - pub unused: u16, - /// The type of the event to which to subscribe. - /// - /// Currently, [`CONDVAR`](enum.eventtype.html#variant.CONDVAR), - /// [`LOCK_RDLOCK`](enum.eventtype.html#variant.LOCK_RDLOCK), and [`LOCK_WRLOCK`](enum.eventtype.html#variant.LOCK_WRLOCK) - /// must be provided as the first subscription and may - /// only be followed by up to one other subscription, - /// having type [`CLOCK`](enum.eventtype.html#variant.CLOCK). - pub type_: eventtype, - pub union: subscription_union, -} -/// A union inside `subscription`. -#[repr(C)] -#[derive(Copy, Clone)] -pub union subscription_union { - /// Used when `type_` is [`CLOCK`](enum.eventtype.html#variant.CLOCK). - pub clock: subscription_clock, - /// Used when `type_` is [`CONDVAR`](enum.eventtype.html#variant.CONDVAR). - pub condvar: subscription_condvar, - /// Used when `type_` is [`FD_READ`](enum.eventtype.html#variant.FD_READ) or [`FD_WRITE`](enum.eventtype.html#variant.FD_WRITE). - pub fd_readwrite: subscription_fd_readwrite, - /// Used when `type_` is [`LOCK_RDLOCK`](enum.eventtype.html#variant.LOCK_RDLOCK) or [`LOCK_WRLOCK`](enum.eventtype.html#variant.LOCK_WRLOCK). - pub lock: subscription_lock, - /// Used when `type_` is [`PROC_TERMINATE`](enum.eventtype.html#variant.PROC_TERMINATE). - pub proc_terminate: subscription_proc_terminate, -} -#[repr(C)] -#[derive(Copy, Clone)] -pub struct subscription_clock { - /// The user-defined unique - /// identifier of the clock. - pub identifier: userdata, - /// The clock against which the - /// timestamp should be compared. - pub clock_id: clockid, - /// The absolute or relative - /// timestamp. - pub timeout: timestamp, - /// The amount of time that the - /// kernel may wait additionally - /// to coalesce with other events. - pub precision: timestamp, - /// Flags specifying whether the - /// timeout is absolute or - /// relative. - pub flags: subclockflags, -} -#[repr(C)] -#[derive(Copy, Clone)] -pub struct subscription_condvar { - /// The condition variable on - /// which to wait to be woken up. - pub condvar: *mut condvar, - /// The lock that will be - /// released while waiting. - /// - /// The lock will be reacquired - /// for writing when the condition - /// variable triggers. - pub lock: *mut lock, - /// Whether the condition variable - /// is stored in private or shared - /// memory. - pub condvar_scope: scope, - /// Whether the lock is stored in - /// private or shared memory. - pub lock_scope: scope, -} -#[repr(C)] -#[derive(Copy, Clone)] -pub struct subscription_fd_readwrite { - /// The file descriptor on which - /// to wait for it to become ready - /// for reading or writing. - pub fd: fd, - /// Under which conditions to - /// trigger. - pub flags: subrwflags, -} -#[repr(C)] -#[derive(Copy, Clone)] -pub struct subscription_lock { - /// The lock that will be acquired - /// for reading or writing. - pub lock: *mut lock, - /// Whether the lock is stored in - /// private or shared memory. - pub lock_scope: scope, -} -#[repr(C)] -#[derive(Copy, Clone)] -pub struct subscription_proc_terminate { - /// The process descriptor on - /// which to wait for process - /// termination. - pub fd: fd, -} -#[test] -#[cfg(target_pointer_width = "32")] -fn subscription_layout_test_32() { - assert_eq!(core::mem::size_of::(), 56); - assert_eq!(core::mem::align_of::(), 8); - unsafe { - let obj: subscription = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.userdata as *const _ as usize - base, 0); - assert_eq!(&obj.unused as *const _ as usize - base, 8); - assert_eq!(&obj.type_ as *const _ as usize - base, 10); - assert_eq!(&obj.union.clock.identifier as *const _ as usize - base, 16); - assert_eq!(&obj.union.clock.clock_id as *const _ as usize - base, 24); - assert_eq!(&obj.union.clock.timeout as *const _ as usize - base, 32); - assert_eq!(&obj.union.clock.precision as *const _ as usize - base, 40); - assert_eq!(&obj.union.clock.flags as *const _ as usize - base, 48); - assert_eq!(&obj.union.condvar.condvar as *const _ as usize - base, 16); - assert_eq!(&obj.union.condvar.lock as *const _ as usize - base, 20); - assert_eq!(&obj.union.condvar.condvar_scope as *const _ as usize - base, 24); - assert_eq!(&obj.union.condvar.lock_scope as *const _ as usize - base, 25); - assert_eq!(&obj.union.fd_readwrite.fd as *const _ as usize - base, 16); - assert_eq!(&obj.union.fd_readwrite.flags as *const _ as usize - base, 20); - assert_eq!(&obj.union.lock.lock as *const _ as usize - base, 16); - assert_eq!(&obj.union.lock.lock_scope as *const _ as usize - base, 20); - assert_eq!(&obj.union.proc_terminate.fd as *const _ as usize - base, 16); - } -} -#[test] -#[cfg(target_pointer_width = "64")] -fn subscription_layout_test_64() { - assert_eq!(core::mem::size_of::(), 56); - assert_eq!(core::mem::align_of::(), 8); - unsafe { - let obj: subscription = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.userdata as *const _ as usize - base, 0); - assert_eq!(&obj.unused as *const _ as usize - base, 8); - assert_eq!(&obj.type_ as *const _ as usize - base, 10); - assert_eq!(&obj.union.clock.identifier as *const _ as usize - base, 16); - assert_eq!(&obj.union.clock.clock_id as *const _ as usize - base, 24); - assert_eq!(&obj.union.clock.timeout as *const _ as usize - base, 32); - assert_eq!(&obj.union.clock.precision as *const _ as usize - base, 40); - assert_eq!(&obj.union.clock.flags as *const _ as usize - base, 48); - assert_eq!(&obj.union.condvar.condvar as *const _ as usize - base, 16); - assert_eq!(&obj.union.condvar.lock as *const _ as usize - base, 24); - assert_eq!(&obj.union.condvar.condvar_scope as *const _ as usize - base, 32); - assert_eq!(&obj.union.condvar.lock_scope as *const _ as usize - base, 33); - assert_eq!(&obj.union.fd_readwrite.fd as *const _ as usize - base, 16); - assert_eq!(&obj.union.fd_readwrite.flags as *const _ as usize - base, 20); - assert_eq!(&obj.union.lock.lock as *const _ as usize - base, 16); - assert_eq!(&obj.union.lock.lock_scope as *const _ as usize - base, 24); - assert_eq!(&obj.union.proc_terminate.fd as *const _ as usize - base, 16); - } -} - -/// The Thread Control Block (TCB). -/// -/// After a thread begins execution (at program startup or when -/// created through [`thread_create()`](fn.thread_create.html)), the CPU's registers -/// controlling Thread-Local Storage (TLS) will already be -/// initialized. They will point to an area only containing the -/// TCB. -/// -/// If the thread needs space for storing thread-specific -/// variables, the thread may allocate a larger area and adjust -/// the CPU's registers to point to that area instead. However, it -/// does need to make sure that the TCB is copied over to the new -/// TLS area. -/// -/// The purpose of the TCB is that it allows light-weight -/// emulators to store information related to individual threads. -/// For example, it may be used to store a copy of the CPU -/// registers prior emulation, so that TLS for the host system -/// can be restored if needed. -#[repr(C)] -#[derive(Copy, Clone)] -pub struct tcb { - /// Pointer that may be freely assigned by the system. Its - /// value cannot be interpreted by the application. - pub parent: *mut (), -} -#[test] -#[cfg(target_pointer_width = "32")] -fn tcb_layout_test_32() { - assert_eq!(core::mem::size_of::(), 4); - assert_eq!(core::mem::align_of::(), 4); - unsafe { - let obj: tcb = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.parent as *const _ as usize - base, 0); - } -} -#[test] -#[cfg(target_pointer_width = "64")] -fn tcb_layout_test_64() { - assert_eq!(core::mem::size_of::(), 8); - assert_eq!(core::mem::align_of::(), 8); - unsafe { - let obj: tcb = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.parent as *const _ as usize - base, 0); - } -} - -/// Entry point for additionally created threads. -/// -/// `tid`: thread ID of the current thread. -/// -/// `aux`: copy of the value stored in -/// [`threadattr.argument`](struct.threadattr.html#structfield.argument). -pub type threadentry = unsafe extern "C" fn(tid: tid, aux: *mut ()) -> (); - -/// Attributes for thread creation. -#[repr(C)] -#[derive(Copy, Clone)] -pub struct threadattr { - /// Initial program counter value. - pub entry_point: threadentry, - /// Region allocated to serve as stack space. - pub stack: (*mut (), usize), - /// Argument to be forwarded to the entry point function. - pub argument: *mut (), -} -#[test] -#[cfg(target_pointer_width = "32")] -fn threadattr_layout_test_32() { - assert_eq!(core::mem::size_of::(), 16); - assert_eq!(core::mem::align_of::(), 4); - unsafe { - let obj: threadattr = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.entry_point as *const _ as usize - base, 0); - assert_eq!(&obj.stack.0 as *const _ as usize - base, 4); - assert_eq!(&obj.stack.1 as *const _ as usize - base, 8); - assert_eq!(&obj.argument as *const _ as usize - base, 12); - } -} -#[test] -#[cfg(target_pointer_width = "64")] -fn threadattr_layout_test_64() { - assert_eq!(core::mem::size_of::(), 32); - assert_eq!(core::mem::align_of::(), 8); - unsafe { - let obj: threadattr = core::mem::uninitialized(); - let base = &obj as *const _ as usize; - assert_eq!(&obj.entry_point as *const _ as usize - base, 0); - assert_eq!(&obj.stack.0 as *const _ as usize - base, 8); - assert_eq!(&obj.stack.1 as *const _ as usize - base, 16); - assert_eq!(&obj.argument as *const _ as usize - base, 24); - } -} - -/// The table with pointers to all syscall implementations. -#[allow(improper_ctypes)] -extern "C" { - fn cloudabi_sys_clock_res_get(_: clockid, _: *mut timestamp) -> errno; - fn cloudabi_sys_clock_time_get(_: clockid, _: timestamp, _: *mut timestamp) -> errno; - fn cloudabi_sys_condvar_signal(_: *mut condvar, _: scope, _: nthreads) -> errno; - fn cloudabi_sys_fd_close(_: fd) -> errno; - fn cloudabi_sys_fd_create1(_: filetype, _: *mut fd) -> errno; - fn cloudabi_sys_fd_create2(_: filetype, _: *mut fd, _: *mut fd) -> errno; - fn cloudabi_sys_fd_datasync(_: fd) -> errno; - fn cloudabi_sys_fd_dup(_: fd, _: *mut fd) -> errno; - fn cloudabi_sys_fd_pread(_: fd, _: *const iovec, _: usize, _: filesize, _: *mut usize) - -> errno; - fn cloudabi_sys_fd_pwrite( - _: fd, - _: *const ciovec, - _: usize, - _: filesize, - _: *mut usize, - ) -> errno; - fn cloudabi_sys_fd_read(_: fd, _: *const iovec, _: usize, _: *mut usize) -> errno; - fn cloudabi_sys_fd_replace(_: fd, _: fd) -> errno; - fn cloudabi_sys_fd_seek(_: fd, _: filedelta, _: whence, _: *mut filesize) -> errno; - fn cloudabi_sys_fd_stat_get(_: fd, _: *mut fdstat) -> errno; - fn cloudabi_sys_fd_stat_put(_: fd, _: *const fdstat, _: fdsflags) -> errno; - fn cloudabi_sys_fd_sync(_: fd) -> errno; - fn cloudabi_sys_fd_write(_: fd, _: *const ciovec, _: usize, _: *mut usize) -> errno; - fn cloudabi_sys_file_advise(_: fd, _: filesize, _: filesize, _: advice) -> errno; - fn cloudabi_sys_file_allocate(_: fd, _: filesize, _: filesize) -> errno; - fn cloudabi_sys_file_create(_: fd, _: *const u8, _: usize, _: filetype) -> errno; - fn cloudabi_sys_file_link( - _: lookup, - _: *const u8, - _: usize, - _: fd, - _: *const u8, - _: usize, - ) -> errno; - fn cloudabi_sys_file_open( - _: lookup, - _: *const u8, - _: usize, - _: oflags, - _: *const fdstat, - _: *mut fd, - ) -> errno; - fn cloudabi_sys_file_readdir(_: fd, _: *mut (), _: usize, _: dircookie, _: *mut usize) - -> errno; - fn cloudabi_sys_file_readlink( - _: fd, - _: *const u8, - _: usize, - _: *mut u8, - _: usize, - _: *mut usize, - ) -> errno; - fn cloudabi_sys_file_rename( - _: fd, - _: *const u8, - _: usize, - _: fd, - _: *const u8, - _: usize, - ) -> errno; - fn cloudabi_sys_file_stat_fget(_: fd, _: *mut filestat) -> errno; - fn cloudabi_sys_file_stat_fput(_: fd, _: *const filestat, _: fsflags) -> errno; - fn cloudabi_sys_file_stat_get(_: lookup, _: *const u8, _: usize, _: *mut filestat) -> errno; - fn cloudabi_sys_file_stat_put( - _: lookup, - _: *const u8, - _: usize, - _: *const filestat, - _: fsflags, - ) -> errno; - fn cloudabi_sys_file_symlink(_: *const u8, _: usize, _: fd, _: *const u8, _: usize) -> errno; - fn cloudabi_sys_file_unlink(_: fd, _: *const u8, _: usize, _: ulflags) -> errno; - fn cloudabi_sys_lock_unlock(_: *mut lock, _: scope) -> errno; - fn cloudabi_sys_mem_advise(_: *mut (), _: usize, _: advice) -> errno; - fn cloudabi_sys_mem_map( - _: *mut (), - _: usize, - _: mprot, - _: mflags, - _: fd, - _: filesize, - _: *mut *mut (), - ) -> errno; - fn cloudabi_sys_mem_protect(_: *mut (), _: usize, _: mprot) -> errno; - fn cloudabi_sys_mem_sync(_: *mut (), _: usize, _: msflags) -> errno; - fn cloudabi_sys_mem_unmap(_: *mut (), _: usize) -> errno; - fn cloudabi_sys_poll(_: *const subscription, _: *mut event, _: usize, _: *mut usize) -> errno; - fn cloudabi_sys_proc_exec(_: fd, _: *const (), _: usize, _: *const fd, _: usize) -> errno; - fn cloudabi_sys_proc_exit(_: exitcode) -> !; - fn cloudabi_sys_proc_fork(_: *mut fd, _: *mut tid) -> errno; - fn cloudabi_sys_proc_raise(_: signal) -> errno; - fn cloudabi_sys_random_get(_: *mut (), _: usize) -> errno; - fn cloudabi_sys_sock_recv(_: fd, _: *const recv_in, _: *mut recv_out) -> errno; - fn cloudabi_sys_sock_send(_: fd, _: *const send_in, _: *mut send_out) -> errno; - fn cloudabi_sys_sock_shutdown(_: fd, _: sdflags) -> errno; - fn cloudabi_sys_thread_create(_: *mut threadattr, _: *mut tid) -> errno; - fn cloudabi_sys_thread_exit(_: *mut lock, _: scope) -> !; - fn cloudabi_sys_thread_yield() -> errno; -} - -/// Obtains the resolution of a clock. -/// -/// ## Parameters -/// -/// **clock_id**: -/// The clock for which the resolution needs to be -/// returned. -/// -/// **resolution**: -/// The resolution of the clock. -#[inline] -pub unsafe fn clock_res_get(clock_id_: clockid, resolution_: &mut timestamp) -> errno { - cloudabi_sys_clock_res_get(clock_id_, resolution_) -} - -/// Obtains the time value of a clock. -/// -/// ## Parameters -/// -/// **clock_id**: -/// The clock for which the time needs to be -/// returned. -/// -/// **precision**: -/// The maximum lag (exclusive) that the returned -/// time value may have, compared to its actual -/// value. -/// -/// **time**: -/// The time value of the clock. -#[inline] -pub unsafe fn clock_time_get( - clock_id_: clockid, - precision_: timestamp, - time_: *mut timestamp, -) -> errno { - cloudabi_sys_clock_time_get(clock_id_, precision_, time_) -} - -/// Wakes up threads waiting on a userspace condition variable. -/// -/// If an invocation of this system call causes all waiting -/// threads to be woken up, the value of the condition variable -/// is set to [`CONDVAR_HAS_NO_WAITERS`](constant.CONDVAR_HAS_NO_WAITERS.html). As long as the condition -/// variable is set to this value, it is not needed to invoke this -/// system call. -/// -/// ## Parameters -/// -/// **condvar**: -/// The userspace condition variable that has -/// waiting threads. -/// -/// **scope**: -/// Whether the condition variable is stored in -/// private or shared memory. -/// -/// **nwaiters**: -/// The number of threads that need to be woken -/// up. If it exceeds the number of waiting -/// threads, all threads are woken up. -#[inline] -pub unsafe fn condvar_signal(condvar_: *mut condvar, scope_: scope, nwaiters_: nthreads) -> errno { - cloudabi_sys_condvar_signal(condvar_, scope_, nwaiters_) -} - -/// Closes a file descriptor. -/// -/// ## Parameters -/// -/// **fd**: -/// The file descriptor that needs to be closed. -#[inline] -pub unsafe fn fd_close(fd_: fd) -> errno { - cloudabi_sys_fd_close(fd_) -} - -/// Creates a file descriptor. -/// -/// ## Parameters -/// -/// **type**: -/// Possible values: -/// -/// - [`SHARED_MEMORY`](enum.filetype.html#variant.SHARED_MEMORY): -/// Creates an anonymous shared memory -/// object. -/// -/// **fd**: -/// The file descriptor that has been created. -#[inline] -pub unsafe fn fd_create1(type_: filetype, fd_: &mut fd) -> errno { - cloudabi_sys_fd_create1(type_, fd_) -} - -/// Creates a pair of file descriptors. -/// -/// ## Parameters -/// -/// **type**: -/// Possible values: -/// -/// - [`SOCKET_DGRAM`](enum.filetype.html#variant.SOCKET_DGRAM): -/// Creates a UNIX datagram socket pair. -/// - [`SOCKET_STREAM`](enum.filetype.html#variant.SOCKET_STREAM): -/// Creates a UNIX byte-stream socket -/// pair. -/// -/// **fd1**: -/// The first file descriptor of the pair. -/// -/// **fd2**: -/// The second file descriptor of the pair. -#[inline] -pub unsafe fn fd_create2(type_: filetype, fd1_: &mut fd, fd2_: &mut fd) -> errno { - cloudabi_sys_fd_create2(type_, fd1_, fd2_) -} - -/// Synchronizes the data of a file to disk. -/// -/// ## Parameters -/// -/// **fd**: -/// The file descriptor of the file whose data -/// needs to be synchronized to disk. -#[inline] -pub unsafe fn fd_datasync(fd_: fd) -> errno { - cloudabi_sys_fd_datasync(fd_) -} - -/// Duplicates a file descriptor. -/// -/// ## Parameters -/// -/// **from**: -/// The file descriptor that needs to be -/// duplicated. -/// -/// **fd**: -/// The new file descriptor. -#[inline] -pub unsafe fn fd_dup(from_: fd, fd_: &mut fd) -> errno { - cloudabi_sys_fd_dup(from_, fd_) -} - -/// Reads from a file descriptor, without using and updating the -/// file descriptor's offset. -/// -/// ## Parameters -/// -/// **fd**: -/// The file descriptor from which data should be -/// read. -/// -/// **iovs**: -/// List of scatter/gather vectors where data -/// should be stored. -/// -/// **offset**: -/// The offset within the file at which reading -/// should start. -/// -/// **nread**: -/// The number of bytes read. -#[inline] -pub unsafe fn fd_pread(fd_: fd, iovs_: &[iovec], offset_: filesize, nread_: &mut usize) -> errno { - cloudabi_sys_fd_pread(fd_, iovs_.as_ptr(), iovs_.len(), offset_, nread_) -} - -/// Writes to a file descriptor, without using and updating the -/// file descriptor's offset. -/// -/// ## Parameters -/// -/// **fd**: -/// The file descriptor to which data should be -/// written. -/// -/// **iovs**: -/// List of scatter/gather vectors where data -/// should be retrieved. -/// -/// **offset**: -/// The offset within the file at which writing -/// should start. -/// -/// **nwritten**: -/// The number of bytes written. -#[inline] -pub unsafe fn fd_pwrite( - fd_: fd, - iovs_: &[ciovec], - offset_: filesize, - nwritten_: &mut usize, -) -> errno { - cloudabi_sys_fd_pwrite(fd_, iovs_.as_ptr(), iovs_.len(), offset_, nwritten_) -} - -/// Reads from a file descriptor. -/// -/// ## Parameters -/// -/// **fd**: -/// The file descriptor from which data should be -/// read. -/// -/// **iovs**: -/// List of scatter/gather vectors where data -/// should be stored. -/// -/// **nread**: -/// The number of bytes read. -#[inline] -pub unsafe fn fd_read(fd_: fd, iovs_: &[iovec], nread_: &mut usize) -> errno { - cloudabi_sys_fd_read(fd_, iovs_.as_ptr(), iovs_.len(), nread_) -} - -/// Atomically replaces a file descriptor by a copy of another -/// file descriptor. -/// -/// Due to the strong focus on thread safety, this environment -/// does not provide a mechanism to duplicate a file descriptor to -/// an arbitrary number, like dup2(). This would be prone to race -/// conditions, as an actual file descriptor with the same number -/// could be allocated by a different thread at the same time. -/// -/// This system call provides a way to atomically replace file -/// descriptors, which would disappear if dup2() were to be -/// removed entirely. -/// -/// ## Parameters -/// -/// **from**: -/// The file descriptor that needs to be copied. -/// -/// **to**: -/// The file descriptor that needs to be -/// overwritten. -#[inline] -pub unsafe fn fd_replace(from_: fd, to_: fd) -> errno { - cloudabi_sys_fd_replace(from_, to_) -} - -/// Moves the offset of the file descriptor. -/// -/// ## Parameters -/// -/// **fd**: -/// The file descriptor whose offset has to be -/// moved. -/// -/// **offset**: -/// The number of bytes to move. -/// -/// **whence**: -/// Relative to which position the move should -/// take place. -/// -/// **newoffset**: -/// The new offset of the file descriptor, -/// relative to the start of the file. -#[inline] -pub unsafe fn fd_seek( - fd_: fd, - offset_: filedelta, - whence_: whence, - newoffset_: &mut filesize, -) -> errno { - cloudabi_sys_fd_seek(fd_, offset_, whence_, newoffset_) -} - -/// Gets attributes of a file descriptor. -/// -/// ## Parameters -/// -/// **fd**: -/// The file descriptor whose attributes have to -/// be obtained. -/// -/// **buf**: -/// The buffer where the file descriptor's -/// attributes are stored. -#[inline] -pub unsafe fn fd_stat_get(fd_: fd, buf_: *mut fdstat) -> errno { - cloudabi_sys_fd_stat_get(fd_, buf_) -} - -/// Adjusts attributes of a file descriptor. -/// -/// ## Parameters -/// -/// **fd**: -/// The file descriptor whose attributes have to -/// be adjusted. -/// -/// **buf**: -/// The desired values of the file descriptor -/// attributes that are adjusted. -/// -/// **flags**: -/// A bitmask indicating which attributes have to -/// be adjusted. -#[inline] -pub unsafe fn fd_stat_put(fd_: fd, buf_: *const fdstat, flags_: fdsflags) -> errno { - cloudabi_sys_fd_stat_put(fd_, buf_, flags_) -} - -/// Synchronizes the data and metadata of a file to disk. -/// -/// ## Parameters -/// -/// **fd**: -/// The file descriptor of the file whose data -/// and metadata needs to be synchronized to disk. -#[inline] -pub unsafe fn fd_sync(fd_: fd) -> errno { - cloudabi_sys_fd_sync(fd_) -} - -/// Writes to a file descriptor. -/// -/// ## Parameters -/// -/// **fd**: -/// The file descriptor to which data should be -/// written. -/// -/// **iovs**: -/// List of scatter/gather vectors where data -/// should be retrieved. -/// -/// **nwritten**: -/// The number of bytes written. -#[inline] -pub unsafe fn fd_write(fd_: fd, iovs_: &[ciovec], nwritten_: &mut usize) -> errno { - cloudabi_sys_fd_write(fd_, iovs_.as_ptr(), iovs_.len(), nwritten_) -} - -/// Provides file advisory information on a file descriptor. -/// -/// ## Parameters -/// -/// **fd**: -/// The file descriptor for which to provide file -/// advisory information. -/// -/// **offset**: -/// The offset within the file to which the -/// advisory applies. -/// -/// **len**: -/// The length of the region to which the advisory -/// applies. -/// -/// **advice**: -/// The advice. -#[inline] -pub unsafe fn file_advise(fd_: fd, offset_: filesize, len_: filesize, advice_: advice) -> errno { - cloudabi_sys_file_advise(fd_, offset_, len_, advice_) -} - -/// Forces the allocation of space in a file. -/// -/// ## Parameters -/// -/// **fd**: -/// The file in which the space should be -/// allocated. -/// -/// **offset**: -/// The offset at which the allocation should -/// start. -/// -/// **len**: -/// The length of the area that is allocated. -#[inline] -pub unsafe fn file_allocate(fd_: fd, offset_: filesize, len_: filesize) -> errno { - cloudabi_sys_file_allocate(fd_, offset_, len_) -} - -/// Creates a file of a specified type. -/// -/// ## Parameters -/// -/// **fd**: -/// The working directory at which the resolution -/// of the file to be created starts. -/// -/// **path**: -/// The path at which the file should be created. -/// -/// **type**: -/// Possible values: -/// -/// - [`DIRECTORY`](enum.filetype.html#variant.DIRECTORY): -/// Creates a directory. -#[inline] -pub unsafe fn file_create(fd_: fd, path_: &[u8], type_: filetype) -> errno { - cloudabi_sys_file_create(fd_, path_.as_ptr(), path_.len(), type_) -} - -/// Creates a hard link. -/// -/// ## Parameters -/// -/// **fd1**: -/// The working directory at which the resolution -/// of the source path starts. -/// -/// **path1**: -/// The source path of the file that should be -/// hard linked. -/// -/// **fd2**: -/// The working directory at which the resolution -/// of the destination path starts. -/// -/// **path2**: -/// The destination path at which the hard link -/// should be created. -#[inline] -pub unsafe fn file_link(fd1_: lookup, path1_: &[u8], fd2_: fd, path2_: &[u8]) -> errno { - cloudabi_sys_file_link(fd1_, path1_.as_ptr(), path1_.len(), fd2_, path2_.as_ptr(), path2_.len()) -} - -/// Opens a file. -/// -/// ## Parameters -/// -/// **dirfd**: -/// The working directory at which the resolution -/// of the file to be opened starts. -/// -/// **path**: -/// The path of the file that should be opened. -/// -/// **oflags**: -/// The method at which the file should be opened. -/// -/// **fds**: -/// [`fdstat.fs_rights_base`](struct.fdstat.html#structfield.fs_rights_base) and -/// [`fdstat.fs_rights_inheriting`](struct.fdstat.html#structfield.fs_rights_inheriting) specify the -/// initial rights of the newly created file -/// descriptor. The operating system is allowed to -/// return a file descriptor with fewer rights -/// than specified, if and only if those rights do -/// not apply to the type of file being opened. -/// -/// [`fdstat.fs_flags`](struct.fdstat.html#structfield.fs_flags) specifies the initial flags -/// of the file descriptor. -/// -/// [`fdstat.fs_filetype`](struct.fdstat.html#structfield.fs_filetype) is ignored. -/// -/// **fd**: -/// The file descriptor of the file that has been -/// opened. -#[inline] -pub unsafe fn file_open( - dirfd_: lookup, - path_: &[u8], - oflags_: oflags, - fds_: *const fdstat, - fd_: &mut fd, -) -> errno { - cloudabi_sys_file_open(dirfd_, path_.as_ptr(), path_.len(), oflags_, fds_, fd_) -} - -/// Reads directory entries from a directory. -/// -/// When successful, the contents of the output buffer consist of -/// a sequence of directory entries. Each directory entry consists -/// of a [`dirent`](struct.dirent.html) object, followed by [`dirent.d_namlen`](struct.dirent.html#structfield.d_namlen) bytes -/// holding the name of the directory entry. -/// -/// This system call fills the output buffer as much as possible, -/// potentially truncating the last directory entry. This allows -/// the caller to grow its read buffer size in case it's too small -/// to fit a single large directory entry, or skip the oversized -/// directory entry. -/// -/// ## Parameters -/// -/// **fd**: -/// The directory from which to read the directory -/// entries. -/// -/// **buf**: -/// The buffer where directory entries are stored. -/// -/// **cookie**: -/// The location within the directory to start -/// reading. -/// -/// **bufused**: -/// The number of bytes stored in the read buffer. -/// If less than the size of the read buffer, the -/// end of the directory has been reached. -#[inline] -pub unsafe fn file_readdir( - fd_: fd, - buf_: &mut [u8], - cookie_: dircookie, - bufused_: &mut usize, -) -> errno { - cloudabi_sys_file_readdir(fd_, buf_.as_mut_ptr() as *mut (), buf_.len(), cookie_, bufused_) -} - -/// Reads the contents of a symbolic link. -/// -/// ## Parameters -/// -/// **fd**: -/// The working directory at which the resolution -/// of the path of the symbolic starts. -/// -/// **path**: -/// The path of the symbolic link whose contents -/// should be read. -/// -/// **buf**: -/// The buffer where the contents of the symbolic -/// link should be stored. -/// -/// **bufused**: -/// The number of bytes placed in the buffer. -#[inline] -pub unsafe fn file_readlink(fd_: fd, path_: &[u8], buf_: &mut [u8], bufused_: &mut usize) -> errno { - cloudabi_sys_file_readlink( - fd_, - path_.as_ptr(), - path_.len(), - buf_.as_mut_ptr(), - buf_.len(), - bufused_, - ) -} - -/// Renames a file. -/// -/// ## Parameters -/// -/// **fd1**: -/// The working directory at which the resolution -/// of the source path starts. -/// -/// **path1**: -/// The source path of the file that should be -/// renamed. -/// -/// **fd2**: -/// The working directory at which the resolution -/// of the destination path starts. -/// -/// **path2**: -/// The destination path to which the file should -/// be renamed. -#[inline] -pub unsafe fn file_rename(fd1_: fd, path1_: &[u8], fd2_: fd, path2_: &[u8]) -> errno { - cloudabi_sys_file_rename( - fd1_, - path1_.as_ptr(), - path1_.len(), - fd2_, - path2_.as_ptr(), - path2_.len(), - ) -} - -/// Gets attributes of a file by file descriptor. -/// -/// ## Parameters -/// -/// **fd**: -/// The file descriptor whose attributes have to -/// be obtained. -/// -/// **buf**: -/// The buffer where the file's attributes are -/// stored. -#[inline] -pub unsafe fn file_stat_fget(fd_: fd, buf_: *mut filestat) -> errno { - cloudabi_sys_file_stat_fget(fd_, buf_) -} - -/// Adjusts attributes of a file by file descriptor. -/// -/// ## Parameters -/// -/// **fd**: -/// The file descriptor whose attributes have to -/// be adjusted. -/// -/// **buf**: -/// The desired values of the file attributes that -/// are adjusted. -/// -/// **flags**: -/// A bitmask indicating which attributes have to -/// be adjusted. -#[inline] -pub unsafe fn file_stat_fput(fd_: fd, buf_: *const filestat, flags_: fsflags) -> errno { - cloudabi_sys_file_stat_fput(fd_, buf_, flags_) -} - -/// Gets attributes of a file by path. -/// -/// ## Parameters -/// -/// **fd**: -/// The working directory at which the resolution -/// of the path whose attributes have to be -/// obtained starts. -/// -/// **path**: -/// The path of the file whose attributes have to -/// be obtained. -/// -/// **buf**: -/// The buffer where the file's attributes are -/// stored. -#[inline] -pub unsafe fn file_stat_get(fd_: lookup, path_: &[u8], buf_: *mut filestat) -> errno { - cloudabi_sys_file_stat_get(fd_, path_.as_ptr(), path_.len(), buf_) -} - -/// Adjusts attributes of a file by path. -/// -/// ## Parameters -/// -/// **fd**: -/// The working directory at which the resolution -/// of the path whose attributes have to be -/// adjusted starts. -/// -/// **path**: -/// The path of the file whose attributes have to -/// be adjusted. -/// -/// **buf**: -/// The desired values of the file attributes that -/// are adjusted. -/// -/// **flags**: -/// A bitmask indicating which attributes have to -/// be adjusted. -#[inline] -pub unsafe fn file_stat_put( - fd_: lookup, - path_: &[u8], - buf_: *const filestat, - flags_: fsflags, -) -> errno { - cloudabi_sys_file_stat_put(fd_, path_.as_ptr(), path_.len(), buf_, flags_) -} - -/// Creates a symbolic link. -/// -/// ## Parameters -/// -/// **path1**: -/// The contents of the symbolic link. -/// -/// **fd**: -/// The working directory at which the resolution -/// of the destination path starts. -/// -/// **path2**: -/// The destination path at which the symbolic -/// link should be created. -#[inline] -pub unsafe fn file_symlink(path1_: &[u8], fd_: fd, path2_: &[u8]) -> errno { - cloudabi_sys_file_symlink(path1_.as_ptr(), path1_.len(), fd_, path2_.as_ptr(), path2_.len()) -} - -/// Unlinks a file, or removes a directory. -/// -/// ## Parameters -/// -/// **fd**: -/// The working directory at which the resolution -/// of the path starts. -/// -/// **path**: -/// The path that needs to be unlinked or removed. -/// -/// **flags**: -/// Possible values: -/// -/// - [`REMOVEDIR`](struct.ulflags.html#associatedconstant.REMOVEDIR): -/// If set, attempt to remove a directory. -/// Otherwise, unlink a file. -#[inline] -pub unsafe fn file_unlink(fd_: fd, path_: &[u8], flags_: ulflags) -> errno { - cloudabi_sys_file_unlink(fd_, path_.as_ptr(), path_.len(), flags_) -} - -/// Unlocks a write-locked userspace lock. -/// -/// If a userspace lock is unlocked while having its -/// [`LOCK_KERNEL_MANAGED`](constant.LOCK_KERNEL_MANAGED.html) flag set, the lock cannot be unlocked in -/// userspace directly. This system call needs to be performed -/// instead, so that any waiting threads can be woken up. -/// -/// To prevent spurious invocations of this system call, the lock -/// must be locked for writing. This prevents other threads from -/// acquiring additional read locks while the system call is in -/// progress. If the lock is acquired for reading, it must first -/// be upgraded to a write lock. -/// -/// ## Parameters -/// -/// **lock**: -/// The userspace lock that is locked for writing -/// by the calling thread. -/// -/// **scope**: -/// Whether the lock is stored in private or -/// shared memory. -#[inline] -pub unsafe fn lock_unlock(lock_: *mut lock, scope_: scope) -> errno { - cloudabi_sys_lock_unlock(lock_, scope_) -} - -/// Provides memory advisory information on a region of memory. -/// -/// ## Parameters -/// -/// **mapping**: -/// The pages for which to provide memory advisory -/// information. -/// -/// **advice**: -/// The advice. -#[inline] -pub unsafe fn mem_advise(mapping_: &mut [u8], advice_: advice) -> errno { - cloudabi_sys_mem_advise(mapping_.as_mut_ptr() as *mut (), mapping_.len(), advice_) -} - -/// Creates a memory mapping, making the contents of a file -/// accessible through memory. -/// -/// ## Parameters -/// -/// **addr**: -/// If [`FIXED`](struct.mflags.html#associatedconstant.FIXED) is set, specifies to which -/// address the file region is mapped. Otherwise, -/// the mapping is performed at an unused -/// location. -/// -/// **len**: -/// The length of the memory mapping to be -/// created. -/// -/// **prot**: -/// Initial memory protection options for the -/// memory mapping. -/// -/// **flags**: -/// Memory mapping flags. -/// -/// **fd**: -/// If [`ANON`](struct.mflags.html#associatedconstant.ANON) is set, this argument must be -/// [`MAP_ANON_FD`](constant.MAP_ANON_FD.html). Otherwise, this argument -/// specifies the file whose contents need to be -/// mapped. -/// -/// **off**: -/// If [`ANON`](struct.mflags.html#associatedconstant.ANON) is set, this argument must be -/// zero. Otherwise, this argument specifies the -/// offset within the file at which the mapping -/// starts. -/// -/// **mem**: -/// The starting address of the memory mapping. -#[inline] -pub unsafe fn mem_map( - addr_: *mut (), - len_: usize, - prot_: mprot, - flags_: mflags, - fd_: fd, - off_: filesize, - mem_: &mut *mut (), -) -> errno { - cloudabi_sys_mem_map(addr_, len_, prot_, flags_, fd_, off_, mem_) -} - -/// Changes the protection of a memory mapping. -/// -/// ## Parameters -/// -/// **mapping**: -/// The pages that need their protection changed. -/// -/// **prot**: -/// New protection options. -#[inline] -pub unsafe fn mem_protect(mapping_: &mut [u8], prot_: mprot) -> errno { - cloudabi_sys_mem_protect(mapping_.as_mut_ptr() as *mut (), mapping_.len(), prot_) -} - -/// Synchronizes a region of memory with its physical storage. -/// -/// ## Parameters -/// -/// **mapping**: -/// The pages that need to be synchronized. -/// -/// **flags**: -/// The method of synchronization. -#[inline] -pub unsafe fn mem_sync(mapping_: &mut [u8], flags_: msflags) -> errno { - cloudabi_sys_mem_sync(mapping_.as_mut_ptr() as *mut (), mapping_.len(), flags_) -} - -/// Unmaps a region of memory. -/// -/// ## Parameters -/// -/// **mapping**: -/// The pages that needs to be unmapped. -#[inline] -pub unsafe fn mem_unmap(mapping_: &mut [u8]) -> errno { - cloudabi_sys_mem_unmap(mapping_.as_mut_ptr() as *mut (), mapping_.len()) -} - -/// Concurrently polls for the occurrence of a set of events. -/// -/// ## Parameters -/// -/// **in**: -/// The events to which to subscribe. -/// -/// **out**: -/// The events that have occurred. -/// -/// **nsubscriptions**: -/// Both the number of subscriptions and events. -/// -/// **nevents**: -/// The number of events stored. -#[inline] -pub unsafe fn poll( - in_: *const subscription, - out_: *mut event, - nsubscriptions_: usize, - nevents_: *mut usize, -) -> errno { - cloudabi_sys_poll(in_, out_, nsubscriptions_, nevents_) -} - -/// Replaces the process by a new executable. -/// -/// Process execution in CloudABI differs from POSIX in two ways: -/// handling of arguments and inheritance of file descriptors. -/// -/// CloudABI does not use string command line arguments. Instead, -/// a buffer with binary data is copied into the address space of -/// the new executable. The kernel does not enforce any specific -/// structure to this data, although CloudABI's C library uses it -/// to store a tree structure that is semantically identical to -/// YAML. -/// -/// Due to the strong focus on thread safety, file descriptors -/// aren't inherited through close-on-exec flags. An explicit -/// list of file descriptors that need to be retained needs to be -/// provided. After execution, file descriptors are placed in the -/// order in which they are stored in the array. This not only -/// makes the execution process deterministic. It also prevents -/// potential information disclosures about the layout of the -/// original process. -/// -/// ## Parameters -/// -/// **fd**: -/// A file descriptor of the new executable. -/// -/// **data**: -/// Binary argument data that is passed on to the -/// new executable. -/// -/// **fds**: -/// The layout of the file descriptor table after -/// execution. -#[inline] -pub unsafe fn proc_exec(fd_: fd, data_: &[u8], fds_: &[fd]) -> errno { - cloudabi_sys_proc_exec(fd_, data_.as_ptr() as *const (), data_.len(), fds_.as_ptr(), fds_.len()) -} - -/// Terminates the process normally. -/// -/// ## Parameters -/// -/// **rval**: -/// The exit code returned by the process. The -/// exit code can be obtained by other processes -/// through [`event.union.proc_terminate.exitcode`](struct.event_proc_terminate.html#structfield.exitcode). -#[inline] -pub unsafe fn proc_exit(rval_: exitcode) -> ! { - cloudabi_sys_proc_exit(rval_) -} - -/// Forks the process of the calling thread. -/// -/// After forking, a new process shall be created, having only a -/// copy of the calling thread. The parent process will obtain a -/// process descriptor. When closed, the child process is -/// automatically signaled with [`KILL`](enum.signal.html#variant.KILL). -/// -/// ## Parameters -/// -/// **fd**: -/// In the parent process: the file descriptor -/// number of the process descriptor. -/// -/// In the child process: [`PROCESS_CHILD`](constant.PROCESS_CHILD.html). -/// -/// **tid**: -/// In the parent process: undefined. -/// -/// In the child process: the thread ID of the -/// initial thread of the child process. -#[inline] -pub unsafe fn proc_fork(fd_: &mut fd, tid_: &mut tid) -> errno { - cloudabi_sys_proc_fork(fd_, tid_) -} - -/// Sends a signal to the process of the calling thread. -/// -/// ## Parameters -/// -/// **sig**: -/// The signal condition that should be triggered. -/// If the signal causes the process to terminate, -/// its condition can be obtained by other -/// processes through -/// [`event.union.proc_terminate.signal`](struct.event_proc_terminate.html#structfield.signal). -#[inline] -pub unsafe fn proc_raise(sig_: signal) -> errno { - cloudabi_sys_proc_raise(sig_) -} - -/// Obtains random data from the kernel random number generator. -/// -/// As this interface is not guaranteed to be fast, it is advised -/// that the random data obtained through this system call is used -/// as the seed for a userspace pseudo-random number generator. -/// -/// ## Parameters -/// -/// **buf**: -/// The buffer that needs to be filled with random -/// data. -#[inline] -pub unsafe fn random_get(buf_: &mut [u8]) -> errno { - cloudabi_sys_random_get(buf_.as_mut_ptr() as *mut (), buf_.len()) -} - -/// Receives a message on a socket. -/// -/// ## Parameters -/// -/// **sock**: -/// The socket on which a message should be -/// received. -/// -/// **in**: -/// Input parameters. -/// -/// **out**: -/// Output parameters. -#[inline] -pub unsafe fn sock_recv(sock_: fd, in_: *const recv_in, out_: *mut recv_out) -> errno { - cloudabi_sys_sock_recv(sock_, in_, out_) -} - -/// Sends a message on a socket. -/// -/// ## Parameters -/// -/// **sock**: -/// The socket on which a message should be sent. -/// -/// **in**: -/// Input parameters. -/// -/// **out**: -/// Output parameters. -#[inline] -pub unsafe fn sock_send(sock_: fd, in_: *const send_in, out_: *mut send_out) -> errno { - cloudabi_sys_sock_send(sock_, in_, out_) -} - -/// Shuts down socket send and receive channels. -/// -/// ## Parameters -/// -/// **sock**: -/// The socket that needs its channels shut down. -/// -/// **how**: -/// Which channels on the socket need to be shut -/// down. -#[inline] -pub unsafe fn sock_shutdown(sock_: fd, how_: sdflags) -> errno { - cloudabi_sys_sock_shutdown(sock_, how_) -} - -/// Creates a new thread within the current process. -/// -/// ## Parameters -/// -/// **attr**: -/// The desired attributes of the new thread. -/// -/// **tid**: -/// The thread ID of the new thread. -#[inline] -pub unsafe fn thread_create(attr_: *mut threadattr, tid_: &mut tid) -> errno { - cloudabi_sys_thread_create(attr_, tid_) -} - -/// Terminates the calling thread. -/// -/// This system call can also unlock a single userspace lock -/// after termination, which can be used to implement thread -/// joining. -/// -/// ## Parameters -/// -/// **lock**: -/// Userspace lock that is locked for writing by -/// the calling thread. -/// -/// **scope**: -/// Whether the lock is stored in private or -/// shared memory. -#[inline] -pub unsafe fn thread_exit(lock_: *mut lock, scope_: scope) -> ! { - cloudabi_sys_thread_exit(lock_, scope_) -} - -/// Temporarily yields execution of the calling thread. -#[inline] -pub unsafe fn thread_yield() -> errno { - cloudabi_sys_thread_yield() -} diff --git a/crux-mir/lib/std/src/sys/cloudabi/abi/mod.rs b/crux-mir/lib/std/src/sys/cloudabi/abi/mod.rs deleted file mode 100644 index 9d01d24ea..000000000 --- a/crux-mir/lib/std/src/sys/cloudabi/abi/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -#[allow(warnings)] -mod cloudabi; -pub use self::cloudabi::*; diff --git a/crux-mir/lib/std/src/sys/cloudabi/args.rs b/crux-mir/lib/std/src/sys/cloudabi/args.rs deleted file mode 100644 index dea562aba..000000000 --- a/crux-mir/lib/std/src/sys/cloudabi/args.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub use crate::sys::cloudabi::shims::args::*; - -#[allow(dead_code)] -pub fn init(_: isize, _: *const *const u8) {} - -#[allow(dead_code)] -pub fn cleanup() {} diff --git a/crux-mir/lib/std/src/sys/cloudabi/condvar.rs b/crux-mir/lib/std/src/sys/cloudabi/condvar.rs deleted file mode 100644 index 3ba51d774..000000000 --- a/crux-mir/lib/std/src/sys/cloudabi/condvar.rs +++ /dev/null @@ -1,152 +0,0 @@ -use crate::cell::UnsafeCell; -use crate::mem; -use crate::sync::atomic::{AtomicU32, Ordering}; -use crate::sys::cloudabi::abi; -use crate::sys::mutex::{self, Mutex}; -use crate::sys::time::checked_dur2intervals; -use crate::time::Duration; - -extern "C" { - #[thread_local] - static __pthread_thread_id: abi::tid; -} - -pub struct Condvar { - condvar: UnsafeCell, -} - -unsafe impl Send for Condvar {} -unsafe impl Sync for Condvar {} - -const NEW: Condvar = - Condvar { condvar: UnsafeCell::new(AtomicU32::new(abi::CONDVAR_HAS_NO_WAITERS.0)) }; - -impl Condvar { - pub const fn new() -> Condvar { - NEW - } - - pub unsafe fn init(&mut self) {} - - pub unsafe fn notify_one(&self) { - let condvar = self.condvar.get(); - if (*condvar).load(Ordering::Relaxed) != abi::CONDVAR_HAS_NO_WAITERS.0 { - let ret = abi::condvar_signal(condvar as *mut abi::condvar, abi::scope::PRIVATE, 1); - assert_eq!(ret, abi::errno::SUCCESS, "Failed to signal on condition variable"); - } - } - - pub unsafe fn notify_all(&self) { - let condvar = self.condvar.get(); - if (*condvar).load(Ordering::Relaxed) != abi::CONDVAR_HAS_NO_WAITERS.0 { - let ret = abi::condvar_signal( - condvar as *mut abi::condvar, - abi::scope::PRIVATE, - abi::nthreads::max_value(), - ); - assert_eq!(ret, abi::errno::SUCCESS, "Failed to broadcast on condition variable"); - } - } - - pub unsafe fn wait(&self, mutex: &Mutex) { - let mutex = mutex::raw(mutex); - assert_eq!( - (*mutex).load(Ordering::Relaxed) & !abi::LOCK_KERNEL_MANAGED.0, - __pthread_thread_id.0 | abi::LOCK_WRLOCKED.0, - "This lock is not write-locked by this thread" - ); - - // Call into the kernel to wait on the condition variable. - let condvar = self.condvar.get(); - let subscription = abi::subscription { - type_: abi::eventtype::CONDVAR, - union: abi::subscription_union { - condvar: abi::subscription_condvar { - condvar: condvar as *mut abi::condvar, - condvar_scope: abi::scope::PRIVATE, - lock: mutex as *mut abi::lock, - lock_scope: abi::scope::PRIVATE, - }, - }, - ..mem::zeroed() - }; - let mut event: mem::MaybeUninit = mem::MaybeUninit::uninit(); - let mut nevents: mem::MaybeUninit = mem::MaybeUninit::uninit(); - let ret = abi::poll(&subscription, event.as_mut_ptr(), 1, nevents.as_mut_ptr()); - assert_eq!(ret, abi::errno::SUCCESS, "Failed to wait on condition variable"); - assert_eq!( - event.assume_init().error, - abi::errno::SUCCESS, - "Failed to wait on condition variable" - ); - } - - pub unsafe fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool { - let mutex = mutex::raw(mutex); - assert_eq!( - (*mutex).load(Ordering::Relaxed) & !abi::LOCK_KERNEL_MANAGED.0, - __pthread_thread_id.0 | abi::LOCK_WRLOCKED.0, - "This lock is not write-locked by this thread" - ); - - // Call into the kernel to wait on the condition variable. - let condvar = self.condvar.get(); - let timeout = - checked_dur2intervals(&dur).expect("overflow converting duration to nanoseconds"); - let subscriptions = [ - abi::subscription { - type_: abi::eventtype::CONDVAR, - union: abi::subscription_union { - condvar: abi::subscription_condvar { - condvar: condvar as *mut abi::condvar, - condvar_scope: abi::scope::PRIVATE, - lock: mutex as *mut abi::lock, - lock_scope: abi::scope::PRIVATE, - }, - }, - ..mem::zeroed() - }, - abi::subscription { - type_: abi::eventtype::CLOCK, - union: abi::subscription_union { - clock: abi::subscription_clock { - clock_id: abi::clockid::MONOTONIC, - timeout, - ..mem::zeroed() - }, - }, - ..mem::zeroed() - }, - ]; - let mut events: [mem::MaybeUninit; 2] = [mem::MaybeUninit::uninit(); 2]; - let mut nevents: mem::MaybeUninit = mem::MaybeUninit::uninit(); - let ret = abi::poll( - subscriptions.as_ptr(), - mem::MaybeUninit::first_ptr_mut(&mut events), - 2, - nevents.as_mut_ptr(), - ); - assert_eq!(ret, abi::errno::SUCCESS, "Failed to wait on condition variable"); - let nevents = nevents.assume_init(); - for i in 0..nevents { - assert_eq!( - events[i].assume_init().error, - abi::errno::SUCCESS, - "Failed to wait on condition variable" - ); - if events[i].assume_init().type_ == abi::eventtype::CONDVAR { - return true; - } - } - false - } - - pub unsafe fn destroy(&self) { - let condvar = self.condvar.get(); - assert_eq!( - (*condvar).load(Ordering::Relaxed), - abi::CONDVAR_HAS_NO_WAITERS.0, - "Attempted to destroy a condition variable with blocked threads" - ); - } -} diff --git a/crux-mir/lib/std/src/sys/cloudabi/io.rs b/crux-mir/lib/std/src/sys/cloudabi/io.rs deleted file mode 100644 index d5f475b43..000000000 --- a/crux-mir/lib/std/src/sys/cloudabi/io.rs +++ /dev/null @@ -1,47 +0,0 @@ -use crate::mem; - -#[derive(Copy, Clone)] -pub struct IoSlice<'a>(&'a [u8]); - -impl<'a> IoSlice<'a> { - #[inline] - pub fn new(buf: &'a [u8]) -> IoSlice<'a> { - IoSlice(buf) - } - - #[inline] - pub fn advance(&mut self, n: usize) { - self.0 = &self.0[n..] - } - - #[inline] - pub fn as_slice(&self) -> &[u8] { - self.0 - } -} - -pub struct IoSliceMut<'a>(&'a mut [u8]); - -impl<'a> IoSliceMut<'a> { - #[inline] - pub fn new(buf: &'a mut [u8]) -> IoSliceMut<'a> { - IoSliceMut(buf) - } - - #[inline] - pub fn advance(&mut self, n: usize) { - let slice = mem::replace(&mut self.0, &mut []); - let (_, remaining) = slice.split_at_mut(n); - self.0 = remaining; - } - - #[inline] - pub fn as_slice(&self) -> &[u8] { - self.0 - } - - #[inline] - pub fn as_mut_slice(&mut self) -> &mut [u8] { - self.0 - } -} diff --git a/crux-mir/lib/std/src/sys/cloudabi/mod.rs b/crux-mir/lib/std/src/sys/cloudabi/mod.rs deleted file mode 100644 index e5f1dd984..000000000 --- a/crux-mir/lib/std/src/sys/cloudabi/mod.rs +++ /dev/null @@ -1,66 +0,0 @@ -use crate::io::ErrorKind; -use crate::mem; - -#[path = "../unix/alloc.rs"] -pub mod alloc; -pub mod args; -#[path = "../unix/cmath.rs"] -pub mod cmath; -pub mod condvar; -pub mod io; -#[path = "../unix/memchr.rs"] -pub mod memchr; -pub mod mutex; -pub mod os; -pub mod rwlock; -pub mod stack_overflow; -pub mod stdio; -pub mod thread; -#[path = "../unix/thread_local.rs"] -pub mod thread_local; -pub mod time; - -pub use crate::sys_common::os_str_bytes as os_str; - -mod abi; - -mod shims; -pub use self::shims::*; - -#[allow(dead_code)] -pub fn init() {} - -pub fn decode_error_kind(errno: i32) -> ErrorKind { - match errno { - x if x == abi::errno::ACCES as i32 => ErrorKind::PermissionDenied, - x if x == abi::errno::ADDRINUSE as i32 => ErrorKind::AddrInUse, - x if x == abi::errno::ADDRNOTAVAIL as i32 => ErrorKind::AddrNotAvailable, - x if x == abi::errno::AGAIN as i32 => ErrorKind::WouldBlock, - x if x == abi::errno::CONNABORTED as i32 => ErrorKind::ConnectionAborted, - x if x == abi::errno::CONNREFUSED as i32 => ErrorKind::ConnectionRefused, - x if x == abi::errno::CONNRESET as i32 => ErrorKind::ConnectionReset, - x if x == abi::errno::EXIST as i32 => ErrorKind::AlreadyExists, - x if x == abi::errno::INTR as i32 => ErrorKind::Interrupted, - x if x == abi::errno::INVAL as i32 => ErrorKind::InvalidInput, - x if x == abi::errno::NOENT as i32 => ErrorKind::NotFound, - x if x == abi::errno::NOTCONN as i32 => ErrorKind::NotConnected, - x if x == abi::errno::PERM as i32 => ErrorKind::PermissionDenied, - x if x == abi::errno::PIPE as i32 => ErrorKind::BrokenPipe, - x if x == abi::errno::TIMEDOUT as i32 => ErrorKind::TimedOut, - _ => ErrorKind::Other, - } -} - -pub unsafe fn abort_internal() -> ! { - core::intrinsics::abort(); -} - -pub use libc::strlen; - -pub fn hashmap_random_keys() -> (u64, u64) { - unsafe { - let mut v: mem::MaybeUninit<(u64, u64)> = mem::MaybeUninit::uninit(); - libc::arc4random_buf(v.as_mut_ptr() as *mut libc::c_void, mem::size_of_val(&v)); - v.assume_init() - } -} diff --git a/crux-mir/lib/std/src/sys/cloudabi/mutex.rs b/crux-mir/lib/std/src/sys/cloudabi/mutex.rs deleted file mode 100644 index 580ab0e8a..000000000 --- a/crux-mir/lib/std/src/sys/cloudabi/mutex.rs +++ /dev/null @@ -1,153 +0,0 @@ -use crate::cell::UnsafeCell; -use crate::mem; -use crate::mem::MaybeUninit; -use crate::sync::atomic::{AtomicU32, Ordering}; -use crate::sys::cloudabi::abi; -use crate::sys::rwlock::{self, RWLock}; - -extern "C" { - #[thread_local] - static __pthread_thread_id: abi::tid; -} - -// Implement Mutex using an RWLock. This doesn't introduce any -// performance overhead in this environment, as the operations would be -// implemented identically. -pub struct Mutex(RWLock); - -pub unsafe fn raw(m: &Mutex) -> *mut AtomicU32 { - rwlock::raw(&m.0) -} - -impl Mutex { - pub const fn new() -> Mutex { - Mutex(RWLock::new()) - } - - pub unsafe fn init(&mut self) { - // This function should normally reinitialize the mutex after - // moving it to a different memory address. This implementation - // does not require adjustments after moving. - } - - pub unsafe fn try_lock(&self) -> bool { - self.0.try_write() - } - - pub unsafe fn lock(&self) { - self.0.write() - } - - pub unsafe fn unlock(&self) { - self.0.write_unlock() - } - - pub unsafe fn destroy(&self) { - self.0.destroy() - } -} - -pub struct ReentrantMutex { - lock: UnsafeCell>, - recursion: UnsafeCell>, -} - -impl ReentrantMutex { - pub const unsafe fn uninitialized() -> ReentrantMutex { - ReentrantMutex { - lock: UnsafeCell::new(MaybeUninit::uninit()), - recursion: UnsafeCell::new(MaybeUninit::uninit()), - } - } - - pub unsafe fn init(&self) { - *self.lock.get() = MaybeUninit::new(AtomicU32::new(abi::LOCK_UNLOCKED.0)); - *self.recursion.get() = MaybeUninit::new(0); - } - - pub unsafe fn try_lock(&self) -> bool { - // Attempt to acquire the lock. - let lock = (*self.lock.get()).as_mut_ptr(); - let recursion = (*self.recursion.get()).as_mut_ptr(); - if let Err(old) = (*lock).compare_exchange( - abi::LOCK_UNLOCKED.0, - __pthread_thread_id.0 | abi::LOCK_WRLOCKED.0, - Ordering::Acquire, - Ordering::Relaxed, - ) { - // If we fail to acquire the lock, it may be the case - // that we've already acquired it and may need to recurse. - if old & !abi::LOCK_KERNEL_MANAGED.0 == __pthread_thread_id.0 | abi::LOCK_WRLOCKED.0 { - *recursion += 1; - true - } else { - false - } - } else { - // Success. - assert_eq!(*recursion, 0, "Mutex has invalid recursion count"); - true - } - } - - pub unsafe fn lock(&self) { - if !self.try_lock() { - // Call into the kernel to acquire a write lock. - let lock = self.lock.get(); - let subscription = abi::subscription { - type_: abi::eventtype::LOCK_WRLOCK, - union: abi::subscription_union { - lock: abi::subscription_lock { - lock: lock as *mut abi::lock, - lock_scope: abi::scope::PRIVATE, - }, - }, - ..mem::zeroed() - }; - let mut event = MaybeUninit::::uninit(); - let mut nevents = MaybeUninit::::uninit(); - let ret = abi::poll(&subscription, event.as_mut_ptr(), 1, nevents.as_mut_ptr()); - assert_eq!(ret, abi::errno::SUCCESS, "Failed to acquire mutex"); - let event = event.assume_init(); - assert_eq!(event.error, abi::errno::SUCCESS, "Failed to acquire mutex"); - } - } - - pub unsafe fn unlock(&self) { - let lock = (*self.lock.get()).as_mut_ptr(); - let recursion = (*self.recursion.get()).as_mut_ptr(); - assert_eq!( - (*lock).load(Ordering::Relaxed) & !abi::LOCK_KERNEL_MANAGED.0, - __pthread_thread_id.0 | abi::LOCK_WRLOCKED.0, - "This mutex is locked by a different thread" - ); - - if *recursion > 0 { - *recursion -= 1; - } else if !(*lock) - .compare_exchange( - __pthread_thread_id.0 | abi::LOCK_WRLOCKED.0, - abi::LOCK_UNLOCKED.0, - Ordering::Release, - Ordering::Relaxed, - ) - .is_ok() - { - // Lock is managed by kernelspace. Call into the kernel - // to unblock waiting threads. - let ret = abi::lock_unlock(lock as *mut abi::lock, abi::scope::PRIVATE); - assert_eq!(ret, abi::errno::SUCCESS, "Failed to unlock a mutex"); - } - } - - pub unsafe fn destroy(&self) { - let lock = (*self.lock.get()).as_mut_ptr(); - let recursion = (*self.recursion.get()).as_mut_ptr(); - assert_eq!( - (*lock).load(Ordering::Relaxed), - abi::LOCK_UNLOCKED.0, - "Attempted to destroy locked mutex" - ); - assert_eq!(*recursion, 0, "Recursion counter invalid"); - } -} diff --git a/crux-mir/lib/std/src/sys/cloudabi/os.rs b/crux-mir/lib/std/src/sys/cloudabi/os.rs deleted file mode 100644 index 326faaa85..000000000 --- a/crux-mir/lib/std/src/sys/cloudabi/os.rs +++ /dev/null @@ -1,26 +0,0 @@ -use crate::ffi::CStr; -use crate::str; - -use libc::c_int; - -pub use crate::sys::cloudabi::shims::os::*; - -pub fn errno() -> i32 { - extern "C" { - #[thread_local] - static errno: c_int; - } - - unsafe { errno as i32 } -} - -/// Gets a detailed string description for the given error number. -pub fn error_string(errno: i32) -> String { - // cloudlibc's strerror() is guaranteed to be thread-safe. There is - // thus no need to use strerror_r(). - str::from_utf8(unsafe { CStr::from_ptr(libc::strerror(errno)) }.to_bytes()).unwrap().to_owned() -} - -pub fn exit(code: i32) -> ! { - unsafe { libc::exit(code as c_int) } -} diff --git a/crux-mir/lib/std/src/sys/cloudabi/rwlock.rs b/crux-mir/lib/std/src/sys/cloudabi/rwlock.rs deleted file mode 100644 index b8af5af1d..000000000 --- a/crux-mir/lib/std/src/sys/cloudabi/rwlock.rs +++ /dev/null @@ -1,218 +0,0 @@ -use crate::cell::UnsafeCell; -use crate::mem; -use crate::mem::MaybeUninit; -use crate::sync::atomic::{AtomicU32, Ordering}; -use crate::sys::cloudabi::abi; - -extern "C" { - #[thread_local] - static __pthread_thread_id: abi::tid; -} - -#[thread_local] -static mut RDLOCKS_ACQUIRED: u32 = 0; - -pub struct RWLock { - lock: UnsafeCell, -} - -pub unsafe fn raw(r: &RWLock) -> *mut AtomicU32 { - r.lock.get() -} - -unsafe impl Send for RWLock {} -unsafe impl Sync for RWLock {} - -const NEW: RWLock = RWLock { lock: UnsafeCell::new(AtomicU32::new(abi::LOCK_UNLOCKED.0)) }; - -impl RWLock { - pub const fn new() -> RWLock { - NEW - } - - pub unsafe fn try_read(&self) -> bool { - let lock = self.lock.get(); - let mut old = abi::LOCK_UNLOCKED.0; - while let Err(cur) = - (*lock).compare_exchange_weak(old, old + 1, Ordering::Acquire, Ordering::Relaxed) - { - if (cur & abi::LOCK_WRLOCKED.0) != 0 { - // Another thread already has a write lock. - assert_ne!( - old & !abi::LOCK_KERNEL_MANAGED.0, - __pthread_thread_id.0 | abi::LOCK_WRLOCKED.0, - "Attempted to acquire a read lock while holding a write lock" - ); - return false; - } else if (old & abi::LOCK_KERNEL_MANAGED.0) != 0 && RDLOCKS_ACQUIRED == 0 { - // Lock has threads waiting for the lock. Only acquire - // the lock if we have already acquired read locks. In - // that case, it is justified to acquire this lock to - // prevent a deadlock. - return false; - } - old = cur; - } - - RDLOCKS_ACQUIRED += 1; - true - } - - pub unsafe fn read(&self) { - if !self.try_read() { - // Call into the kernel to acquire a read lock. - let lock = self.lock.get(); - let subscription = abi::subscription { - type_: abi::eventtype::LOCK_RDLOCK, - union: abi::subscription_union { - lock: abi::subscription_lock { - lock: lock as *mut abi::lock, - lock_scope: abi::scope::PRIVATE, - }, - }, - ..mem::zeroed() - }; - let mut event = MaybeUninit::::uninit(); - let mut nevents = MaybeUninit::::uninit(); - let ret = abi::poll(&subscription, event.as_mut_ptr(), 1, nevents.as_mut_ptr()); - assert_eq!(ret, abi::errno::SUCCESS, "Failed to acquire read lock"); - let event = event.assume_init(); - assert_eq!(event.error, abi::errno::SUCCESS, "Failed to acquire read lock"); - - RDLOCKS_ACQUIRED += 1; - } - } - - pub unsafe fn read_unlock(&self) { - // Perform a read unlock. We can do this in userspace, except when - // other threads are blocked and we are performing the last unlock. - // In that case, call into the kernel. - // - // Other threads may attempt to increment the read lock count, - // meaning that the call into the kernel could be spurious. To - // prevent this from happening, upgrade to a write lock first. This - // allows us to call into the kernel, having the guarantee that the - // lock value will not change in the meantime. - assert!(RDLOCKS_ACQUIRED > 0, "Bad lock count"); - let mut old = 1; - loop { - let lock = self.lock.get(); - if old == 1 | abi::LOCK_KERNEL_MANAGED.0 { - // Last read lock while threads are waiting. Attempt to upgrade - // to a write lock before calling into the kernel to unlock. - if let Err(cur) = (*lock).compare_exchange_weak( - old, - __pthread_thread_id.0 | abi::LOCK_WRLOCKED.0 | abi::LOCK_KERNEL_MANAGED.0, - Ordering::Acquire, - Ordering::Relaxed, - ) { - old = cur; - } else { - // Call into the kernel to unlock. - let ret = abi::lock_unlock(lock as *mut abi::lock, abi::scope::PRIVATE); - assert_eq!(ret, abi::errno::SUCCESS, "Failed to write unlock a rwlock"); - break; - } - } else { - // No threads waiting or not the last read lock. Just decrement - // the read lock count. - assert_ne!(old & !abi::LOCK_KERNEL_MANAGED.0, 0, "This rwlock is not locked"); - assert_eq!( - old & abi::LOCK_WRLOCKED.0, - 0, - "Attempted to read-unlock a write-locked rwlock" - ); - if let Err(cur) = (*lock).compare_exchange_weak( - old, - old - 1, - Ordering::Acquire, - Ordering::Relaxed, - ) { - old = cur; - } else { - break; - } - } - } - - RDLOCKS_ACQUIRED -= 1; - } - - pub unsafe fn try_write(&self) -> bool { - // Attempt to acquire the lock. - let lock = self.lock.get(); - if let Err(old) = (*lock).compare_exchange( - abi::LOCK_UNLOCKED.0, - __pthread_thread_id.0 | abi::LOCK_WRLOCKED.0, - Ordering::Acquire, - Ordering::Relaxed, - ) { - // Failure. Crash upon recursive acquisition. - assert_ne!( - old & !abi::LOCK_KERNEL_MANAGED.0, - __pthread_thread_id.0 | abi::LOCK_WRLOCKED.0, - "Attempted to recursive write-lock a rwlock", - ); - false - } else { - // Success. - true - } - } - - pub unsafe fn write(&self) { - if !self.try_write() { - // Call into the kernel to acquire a write lock. - let lock = self.lock.get(); - let subscription = abi::subscription { - type_: abi::eventtype::LOCK_WRLOCK, - union: abi::subscription_union { - lock: abi::subscription_lock { - lock: lock as *mut abi::lock, - lock_scope: abi::scope::PRIVATE, - }, - }, - ..mem::zeroed() - }; - let mut event = MaybeUninit::::uninit(); - let mut nevents = MaybeUninit::::uninit(); - let ret = abi::poll(&subscription, event.as_mut_ptr(), 1, nevents.as_mut_ptr()); - assert_eq!(ret, abi::errno::SUCCESS, "Failed to acquire write lock"); - let event = event.assume_init(); - assert_eq!(event.error, abi::errno::SUCCESS, "Failed to acquire write lock"); - } - } - - pub unsafe fn write_unlock(&self) { - let lock = self.lock.get(); - assert_eq!( - (*lock).load(Ordering::Relaxed) & !abi::LOCK_KERNEL_MANAGED.0, - __pthread_thread_id.0 | abi::LOCK_WRLOCKED.0, - "This rwlock is not write-locked by this thread" - ); - - if !(*lock) - .compare_exchange( - __pthread_thread_id.0 | abi::LOCK_WRLOCKED.0, - abi::LOCK_UNLOCKED.0, - Ordering::Release, - Ordering::Relaxed, - ) - .is_ok() - { - // Lock is managed by kernelspace. Call into the kernel - // to unblock waiting threads. - let ret = abi::lock_unlock(lock as *mut abi::lock, abi::scope::PRIVATE); - assert_eq!(ret, abi::errno::SUCCESS, "Failed to write unlock a rwlock"); - } - } - - pub unsafe fn destroy(&self) { - let lock = self.lock.get(); - assert_eq!( - (*lock).load(Ordering::Relaxed), - abi::LOCK_UNLOCKED.0, - "Attempted to destroy locked rwlock" - ); - } -} diff --git a/crux-mir/lib/std/src/sys/cloudabi/shims/env.rs b/crux-mir/lib/std/src/sys/cloudabi/shims/env.rs deleted file mode 100644 index de165a864..000000000 --- a/crux-mir/lib/std/src/sys/cloudabi/shims/env.rs +++ /dev/null @@ -1,9 +0,0 @@ -pub mod os { - pub const FAMILY: &str = "cloudabi"; - pub const OS: &str = "cloudabi"; - pub const DLL_PREFIX: &str = "lib"; - pub const DLL_SUFFIX: &str = ".so"; - pub const DLL_EXTENSION: &str = "so"; - pub const EXE_SUFFIX: &str = ""; - pub const EXE_EXTENSION: &str = ""; -} diff --git a/crux-mir/lib/std/src/sys/cloudabi/shims/fs.rs b/crux-mir/lib/std/src/sys/cloudabi/shims/fs.rs deleted file mode 100644 index e6160d145..000000000 --- a/crux-mir/lib/std/src/sys/cloudabi/shims/fs.rs +++ /dev/null @@ -1,300 +0,0 @@ -use crate::ffi::OsString; -use crate::fmt; -use crate::hash::{Hash, Hasher}; -use crate::io::{self, IoSlice, IoSliceMut, SeekFrom}; -use crate::path::{Path, PathBuf}; -use crate::sys::time::SystemTime; -use crate::sys::{unsupported, Void}; - -pub struct File(Void); - -pub struct FileAttr(Void); - -pub struct ReadDir(Void); - -pub struct DirEntry(Void); - -#[derive(Clone, Debug)] -pub struct OpenOptions {} - -pub struct FilePermissions(Void); - -pub struct FileType(Void); - -#[derive(Debug)] -pub struct DirBuilder {} - -impl FileAttr { - pub fn size(&self) -> u64 { - match self.0 {} - } - - pub fn perm(&self) -> FilePermissions { - match self.0 {} - } - - pub fn file_type(&self) -> FileType { - match self.0 {} - } - - pub fn modified(&self) -> io::Result { - match self.0 {} - } - - pub fn accessed(&self) -> io::Result { - match self.0 {} - } - - pub fn created(&self) -> io::Result { - match self.0 {} - } -} - -impl Clone for FileAttr { - fn clone(&self) -> FileAttr { - match self.0 {} - } -} - -impl FilePermissions { - pub fn readonly(&self) -> bool { - match self.0 {} - } - - pub fn set_readonly(&mut self, _readonly: bool) { - match self.0 {} - } -} - -impl Clone for FilePermissions { - fn clone(&self) -> FilePermissions { - match self.0 {} - } -} - -impl PartialEq for FilePermissions { - fn eq(&self, _other: &FilePermissions) -> bool { - match self.0 {} - } -} - -impl Eq for FilePermissions {} - -impl fmt::Debug for FilePermissions { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} - } -} - -impl FileType { - pub fn is_dir(&self) -> bool { - match self.0 {} - } - - pub fn is_file(&self) -> bool { - match self.0 {} - } - - pub fn is_symlink(&self) -> bool { - match self.0 {} - } -} - -impl Clone for FileType { - fn clone(&self) -> FileType { - match self.0 {} - } -} - -impl Copy for FileType {} - -impl PartialEq for FileType { - fn eq(&self, _other: &FileType) -> bool { - match self.0 {} - } -} - -impl Eq for FileType {} - -impl Hash for FileType { - fn hash(&self, _h: &mut H) { - match self.0 {} - } -} - -impl fmt::Debug for FileType { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} - } -} - -impl fmt::Debug for ReadDir { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} - } -} - -impl Iterator for ReadDir { - type Item = io::Result; - - fn next(&mut self) -> Option> { - match self.0 {} - } -} - -impl DirEntry { - pub fn path(&self) -> PathBuf { - match self.0 {} - } - - pub fn file_name(&self) -> OsString { - match self.0 {} - } - - pub fn metadata(&self) -> io::Result { - match self.0 {} - } - - pub fn file_type(&self) -> io::Result { - match self.0 {} - } -} - -impl OpenOptions { - pub fn new() -> OpenOptions { - OpenOptions {} - } - - pub fn read(&mut self, _read: bool) {} - pub fn write(&mut self, _write: bool) {} - pub fn append(&mut self, _append: bool) {} - pub fn truncate(&mut self, _truncate: bool) {} - pub fn create(&mut self, _create: bool) {} - pub fn create_new(&mut self, _create_new: bool) {} -} - -impl File { - pub fn open(_path: &Path, _opts: &OpenOptions) -> io::Result { - unsupported() - } - - pub fn file_attr(&self) -> io::Result { - match self.0 {} - } - - pub fn fsync(&self) -> io::Result<()> { - match self.0 {} - } - - pub fn datasync(&self) -> io::Result<()> { - match self.0 {} - } - - pub fn truncate(&self, _size: u64) -> io::Result<()> { - match self.0 {} - } - - pub fn read(&self, _buf: &mut [u8]) -> io::Result { - match self.0 {} - } - - pub fn read_vectored(&self, _bufs: &mut [IoSliceMut<'_>]) -> io::Result { - match self.0 {} - } - - pub fn write(&self, _buf: &[u8]) -> io::Result { - match self.0 {} - } - - pub fn write_vectored(&self, _bufs: &[IoSlice<'_>]) -> io::Result { - match self.0 {} - } - - pub fn flush(&self) -> io::Result<()> { - match self.0 {} - } - - pub fn seek(&self, _pos: SeekFrom) -> io::Result { - match self.0 {} - } - - pub fn duplicate(&self) -> io::Result { - match self.0 {} - } - - pub fn set_permissions(&self, _perm: FilePermissions) -> io::Result<()> { - match self.0 {} - } - - pub fn diverge(&self) -> ! { - match self.0 {} - } -} - -impl DirBuilder { - pub fn new() -> DirBuilder { - DirBuilder {} - } - - pub fn mkdir(&self, _p: &Path) -> io::Result<()> { - unsupported() - } -} - -impl fmt::Debug for File { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} - } -} - -pub fn readdir(_p: &Path) -> io::Result { - unsupported() -} - -pub fn unlink(_p: &Path) -> io::Result<()> { - unsupported() -} - -pub fn rename(_old: &Path, _new: &Path) -> io::Result<()> { - unsupported() -} - -pub fn set_perm(_p: &Path, perm: FilePermissions) -> io::Result<()> { - match perm.0 {} -} - -pub fn rmdir(_p: &Path) -> io::Result<()> { - unsupported() -} - -pub fn remove_dir_all(_path: &Path) -> io::Result<()> { - unsupported() -} - -pub fn readlink(_p: &Path) -> io::Result { - unsupported() -} - -pub fn symlink(_src: &Path, _dst: &Path) -> io::Result<()> { - unsupported() -} - -pub fn link(_src: &Path, _dst: &Path) -> io::Result<()> { - unsupported() -} - -pub fn stat(_p: &Path) -> io::Result { - unsupported() -} - -pub fn lstat(_p: &Path) -> io::Result { - unsupported() -} - -pub fn canonicalize(_p: &Path) -> io::Result { - unsupported() -} - -pub fn copy(_from: &Path, _to: &Path) -> io::Result { - unsupported() -} diff --git a/crux-mir/lib/std/src/sys/cloudabi/shims/mod.rs b/crux-mir/lib/std/src/sys/cloudabi/shims/mod.rs deleted file mode 100644 index b1b5f142f..000000000 --- a/crux-mir/lib/std/src/sys/cloudabi/shims/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -use crate::io; - -pub mod args; -pub mod env; -pub mod fs; -pub mod net; -pub mod os; -#[path = "../../unix/path.rs"] -pub mod path; -pub mod pipe; -pub mod process; - -// This enum is used as the storage for a bunch of types which can't actually exist. -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] -pub enum Void {} - -pub fn unsupported() -> io::Result { - Err(io::Error::new(io::ErrorKind::Other, "This function is not available on CloudABI.")) -} diff --git a/crux-mir/lib/std/src/sys/cloudabi/shims/net.rs b/crux-mir/lib/std/src/sys/cloudabi/shims/net.rs deleted file mode 100644 index 67c436fa7..000000000 --- a/crux-mir/lib/std/src/sys/cloudabi/shims/net.rs +++ /dev/null @@ -1,318 +0,0 @@ -use crate::convert::TryFrom; -use crate::fmt; -use crate::io::{self, IoSlice, IoSliceMut}; -use crate::net::{Ipv4Addr, Ipv6Addr, Shutdown, SocketAddr}; -use crate::sys::{unsupported, Void}; -use crate::time::Duration; - -#[allow(unused_extern_crates)] -pub extern crate libc as netc; - -pub struct TcpStream(Void); - -impl TcpStream { - pub fn connect(_: io::Result<&SocketAddr>) -> io::Result { - unsupported() - } - - pub fn connect_timeout(_: &SocketAddr, _: Duration) -> io::Result { - unsupported() - } - - pub fn set_read_timeout(&self, _: Option) -> io::Result<()> { - match self.0 {} - } - - pub fn set_write_timeout(&self, _: Option) -> io::Result<()> { - match self.0 {} - } - - pub fn read_timeout(&self) -> io::Result> { - match self.0 {} - } - - pub fn write_timeout(&self) -> io::Result> { - match self.0 {} - } - - pub fn peek(&self, _: &mut [u8]) -> io::Result { - match self.0 {} - } - - pub fn read(&self, _: &mut [u8]) -> io::Result { - match self.0 {} - } - - pub fn read_vectored(&self, _: &mut [IoSliceMut<'_>]) -> io::Result { - match self.0 {} - } - - pub fn write(&self, _: &[u8]) -> io::Result { - match self.0 {} - } - - pub fn write_vectored(&self, _: &[IoSlice<'_>]) -> io::Result { - match self.0 {} - } - - pub fn peer_addr(&self) -> io::Result { - match self.0 {} - } - - pub fn socket_addr(&self) -> io::Result { - match self.0 {} - } - - pub fn shutdown(&self, _: Shutdown) -> io::Result<()> { - match self.0 {} - } - - pub fn duplicate(&self) -> io::Result { - match self.0 {} - } - - pub fn set_nodelay(&self, _: bool) -> io::Result<()> { - match self.0 {} - } - - pub fn nodelay(&self) -> io::Result { - match self.0 {} - } - - pub fn set_ttl(&self, _: u32) -> io::Result<()> { - match self.0 {} - } - - pub fn ttl(&self) -> io::Result { - match self.0 {} - } - - pub fn take_error(&self) -> io::Result> { - match self.0 {} - } - - pub fn set_nonblocking(&self, _: bool) -> io::Result<()> { - match self.0 {} - } -} - -impl fmt::Debug for TcpStream { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} - } -} - -pub struct TcpListener(Void); - -impl TcpListener { - pub fn bind(_: io::Result<&SocketAddr>) -> io::Result { - unsupported() - } - - pub fn socket_addr(&self) -> io::Result { - match self.0 {} - } - - pub fn accept(&self) -> io::Result<(TcpStream, SocketAddr)> { - match self.0 {} - } - - pub fn duplicate(&self) -> io::Result { - match self.0 {} - } - - pub fn set_ttl(&self, _: u32) -> io::Result<()> { - match self.0 {} - } - - pub fn ttl(&self) -> io::Result { - match self.0 {} - } - - pub fn set_only_v6(&self, _: bool) -> io::Result<()> { - match self.0 {} - } - - pub fn only_v6(&self) -> io::Result { - match self.0 {} - } - - pub fn take_error(&self) -> io::Result> { - match self.0 {} - } - - pub fn set_nonblocking(&self, _: bool) -> io::Result<()> { - match self.0 {} - } -} - -impl fmt::Debug for TcpListener { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} - } -} - -pub struct UdpSocket(Void); - -impl UdpSocket { - pub fn bind(_: io::Result<&SocketAddr>) -> io::Result { - unsupported() - } - - pub fn peer_addr(&self) -> io::Result { - match self.0 {} - } - - pub fn socket_addr(&self) -> io::Result { - match self.0 {} - } - - pub fn recv_from(&self, _: &mut [u8]) -> io::Result<(usize, SocketAddr)> { - match self.0 {} - } - - pub fn peek_from(&self, _: &mut [u8]) -> io::Result<(usize, SocketAddr)> { - match self.0 {} - } - - pub fn send_to(&self, _: &[u8], _: &SocketAddr) -> io::Result { - match self.0 {} - } - - pub fn duplicate(&self) -> io::Result { - match self.0 {} - } - - pub fn set_read_timeout(&self, _: Option) -> io::Result<()> { - match self.0 {} - } - - pub fn set_write_timeout(&self, _: Option) -> io::Result<()> { - match self.0 {} - } - - pub fn read_timeout(&self) -> io::Result> { - match self.0 {} - } - - pub fn write_timeout(&self) -> io::Result> { - match self.0 {} - } - - pub fn set_broadcast(&self, _: bool) -> io::Result<()> { - match self.0 {} - } - - pub fn broadcast(&self) -> io::Result { - match self.0 {} - } - - pub fn set_multicast_loop_v4(&self, _: bool) -> io::Result<()> { - match self.0 {} - } - - pub fn multicast_loop_v4(&self) -> io::Result { - match self.0 {} - } - - pub fn set_multicast_ttl_v4(&self, _: u32) -> io::Result<()> { - match self.0 {} - } - - pub fn multicast_ttl_v4(&self) -> io::Result { - match self.0 {} - } - - pub fn set_multicast_loop_v6(&self, _: bool) -> io::Result<()> { - match self.0 {} - } - - pub fn multicast_loop_v6(&self) -> io::Result { - match self.0 {} - } - - pub fn join_multicast_v4(&self, _: &Ipv4Addr, _: &Ipv4Addr) -> io::Result<()> { - match self.0 {} - } - - pub fn join_multicast_v6(&self, _: &Ipv6Addr, _: u32) -> io::Result<()> { - match self.0 {} - } - - pub fn leave_multicast_v4(&self, _: &Ipv4Addr, _: &Ipv4Addr) -> io::Result<()> { - match self.0 {} - } - - pub fn leave_multicast_v6(&self, _: &Ipv6Addr, _: u32) -> io::Result<()> { - match self.0 {} - } - - pub fn set_ttl(&self, _: u32) -> io::Result<()> { - match self.0 {} - } - - pub fn ttl(&self) -> io::Result { - match self.0 {} - } - - pub fn take_error(&self) -> io::Result> { - match self.0 {} - } - - pub fn set_nonblocking(&self, _: bool) -> io::Result<()> { - match self.0 {} - } - - pub fn recv(&self, _: &mut [u8]) -> io::Result { - match self.0 {} - } - - pub fn peek(&self, _: &mut [u8]) -> io::Result { - match self.0 {} - } - - pub fn send(&self, _: &[u8]) -> io::Result { - match self.0 {} - } - - pub fn connect(&self, _: io::Result<&SocketAddr>) -> io::Result<()> { - match self.0 {} - } -} - -impl fmt::Debug for UdpSocket { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} - } -} - -pub struct LookupHost(Void); - -impl LookupHost { - pub fn port(&self) -> u16 { - match self.0 {} - } -} - -impl Iterator for LookupHost { - type Item = SocketAddr; - fn next(&mut self) -> Option { - match self.0 {} - } -} - -impl TryFrom<&str> for LookupHost { - type Error = io::Error; - - fn try_from(_v: &str) -> io::Result { - unsupported() - } -} - -impl<'a> TryFrom<(&'a str, u16)> for LookupHost { - type Error = io::Error; - - fn try_from(_v: (&'a str, u16)) -> io::Result { - unsupported() - } -} diff --git a/crux-mir/lib/std/src/sys/cloudabi/shims/os.rs b/crux-mir/lib/std/src/sys/cloudabi/shims/os.rs deleted file mode 100644 index 779e6d54b..000000000 --- a/crux-mir/lib/std/src/sys/cloudabi/shims/os.rs +++ /dev/null @@ -1,86 +0,0 @@ -use crate::error::Error as StdError; -use crate::ffi::{OsStr, OsString}; -use crate::fmt; -use crate::io; -use crate::iter; -use crate::path::{self, PathBuf}; -use crate::sys::{unsupported, Void}; - -pub fn getcwd() -> io::Result { - unsupported() -} - -pub fn chdir(_: &path::Path) -> io::Result<()> { - unsupported() -} - -pub type Env = iter::Empty<(OsString, OsString)>; - -pub fn env() -> Env { - iter::empty() -} - -pub fn getenv(_: &OsStr) -> io::Result> { - Ok(None) -} - -pub fn setenv(_: &OsStr, _: &OsStr) -> io::Result<()> { - unsupported() -} - -pub fn unsetenv(_: &OsStr) -> io::Result<()> { - unsupported() -} - -pub struct SplitPaths<'a>(&'a Void); - -pub fn split_paths(_unparsed: &OsStr) -> SplitPaths<'_> { - panic!("unsupported") -} - -impl<'a> Iterator for SplitPaths<'a> { - type Item = PathBuf; - fn next(&mut self) -> Option { - match *self.0 {} - } -} - -#[derive(Debug)] -pub struct JoinPathsError; - -pub fn join_paths(_paths: I) -> Result -where - I: Iterator, - T: AsRef, -{ - Err(JoinPathsError) -} - -impl fmt::Display for JoinPathsError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - "not supported on CloudABI yet".fmt(f) - } -} - -impl StdError for JoinPathsError { - #[allow(deprecated)] - fn description(&self) -> &str { - "not supported on CloudABI yet" - } -} - -pub fn home_dir() -> Option { - None -} - -pub fn temp_dir() -> PathBuf { - PathBuf::from("/tmp") -} - -pub fn current_exe() -> io::Result { - unsupported() -} - -pub fn getpid() -> u32 { - 1 -} diff --git a/crux-mir/lib/std/src/sys/cloudabi/shims/pipe.rs b/crux-mir/lib/std/src/sys/cloudabi/shims/pipe.rs deleted file mode 100644 index fb14dc591..000000000 --- a/crux-mir/lib/std/src/sys/cloudabi/shims/pipe.rs +++ /dev/null @@ -1,30 +0,0 @@ -use crate::io::{self, IoSlice, IoSliceMut}; -use crate::sys::Void; - -pub struct AnonPipe(Void); - -impl AnonPipe { - pub fn read(&self, _buf: &mut [u8]) -> io::Result { - match self.0 {} - } - - pub fn read_vectored(&self, _bufs: &mut [IoSliceMut<'_>]) -> io::Result { - match self.0 {} - } - - pub fn write(&self, _buf: &[u8]) -> io::Result { - match self.0 {} - } - - pub fn write_vectored(&self, _bufs: &[IoSlice<'_>]) -> io::Result { - match self.0 {} - } - - pub fn diverge(&self) -> ! { - match self.0 {} - } -} - -pub fn read2(p1: AnonPipe, _v1: &mut Vec, _p2: AnonPipe, _v2: &mut Vec) -> io::Result<()> { - match p1.0 {} -} diff --git a/crux-mir/lib/std/src/sys/cloudabi/shims/process.rs b/crux-mir/lib/std/src/sys/cloudabi/shims/process.rs deleted file mode 100644 index 4702e5c54..000000000 --- a/crux-mir/lib/std/src/sys/cloudabi/shims/process.rs +++ /dev/null @@ -1,149 +0,0 @@ -use crate::ffi::OsStr; -use crate::fmt; -use crate::io; -use crate::sys::fs::File; -use crate::sys::pipe::AnonPipe; -use crate::sys::{unsupported, Void}; -use crate::sys_common::process::CommandEnv; - -pub use crate::ffi::OsString as EnvKey; - -//////////////////////////////////////////////////////////////////////////////// -// Command -//////////////////////////////////////////////////////////////////////////////// - -pub struct Command { - env: CommandEnv, -} - -// passed back to std::process with the pipes connected to the child, if any -// were requested -pub struct StdioPipes { - pub stdin: Option, - pub stdout: Option, - pub stderr: Option, -} - -pub enum Stdio { - Inherit, - Null, - MakePipe, -} - -impl Command { - pub fn new(_program: &OsStr) -> Command { - Command { env: Default::default() } - } - - pub fn arg(&mut self, _arg: &OsStr) {} - - pub fn env_mut(&mut self) -> &mut CommandEnv { - &mut self.env - } - - pub fn cwd(&mut self, _dir: &OsStr) {} - - pub fn stdin(&mut self, _stdin: Stdio) {} - - pub fn stdout(&mut self, _stdout: Stdio) {} - - pub fn stderr(&mut self, _stderr: Stdio) {} - - pub fn spawn( - &mut self, - _default: Stdio, - _needs_stdin: bool, - ) -> io::Result<(Process, StdioPipes)> { - unsupported() - } -} - -impl From for Stdio { - fn from(pipe: AnonPipe) -> Stdio { - pipe.diverge() - } -} - -impl From for Stdio { - fn from(file: File) -> Stdio { - file.diverge() - } -} - -impl fmt::Debug for Command { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - Ok(()) - } -} - -pub struct ExitStatus(Void); - -impl ExitStatus { - pub fn success(&self) -> bool { - match self.0 {} - } - - pub fn code(&self) -> Option { - match self.0 {} - } -} - -impl Clone for ExitStatus { - fn clone(&self) -> ExitStatus { - match self.0 {} - } -} - -impl Copy for ExitStatus {} - -impl PartialEq for ExitStatus { - fn eq(&self, _other: &ExitStatus) -> bool { - match self.0 {} - } -} - -impl Eq for ExitStatus {} - -impl fmt::Debug for ExitStatus { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} - } -} - -impl fmt::Display for ExitStatus { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} - } -} - -#[derive(PartialEq, Eq, Clone, Copy, Debug)] -pub struct ExitCode(bool); - -impl ExitCode { - pub const SUCCESS: ExitCode = ExitCode(false); - pub const FAILURE: ExitCode = ExitCode(true); - - pub fn as_i32(&self) -> i32 { - self.0 as i32 - } -} - -pub struct Process(Void); - -impl Process { - pub fn id(&self) -> u32 { - match self.0 {} - } - - pub fn kill(&mut self) -> io::Result<()> { - match self.0 {} - } - - pub fn wait(&mut self) -> io::Result { - match self.0 {} - } - - pub fn try_wait(&mut self) -> io::Result> { - match self.0 {} - } -} diff --git a/crux-mir/lib/std/src/sys/cloudabi/stack_overflow.rs b/crux-mir/lib/std/src/sys/cloudabi/stack_overflow.rs deleted file mode 100644 index e97831b2c..000000000 --- a/crux-mir/lib/std/src/sys/cloudabi/stack_overflow.rs +++ /dev/null @@ -1,13 +0,0 @@ -#![cfg_attr(test, allow(dead_code))] - -pub struct Handler; - -impl Handler { - pub unsafe fn new() -> Handler { - Handler - } -} - -pub unsafe fn init() {} - -pub unsafe fn cleanup() {} diff --git a/crux-mir/lib/std/src/sys/cloudabi/stdio.rs b/crux-mir/lib/std/src/sys/cloudabi/stdio.rs deleted file mode 100644 index 601563c5b..000000000 --- a/crux-mir/lib/std/src/sys/cloudabi/stdio.rs +++ /dev/null @@ -1,66 +0,0 @@ -use crate::io; -use crate::sys::cloudabi::abi; - -pub struct Stdin(()); -pub struct Stdout(()); -pub struct Stderr(()); - -impl Stdin { - pub fn new() -> io::Result { - Ok(Stdin(())) - } -} - -impl io::Read for Stdin { - fn read(&mut self, _buf: &mut [u8]) -> io::Result { - Ok(0) - } -} - -impl Stdout { - pub fn new() -> io::Result { - Ok(Stdout(())) - } -} - -impl io::Write for Stdout { - fn write(&mut self, _buf: &[u8]) -> io::Result { - Err(io::Error::new( - io::ErrorKind::BrokenPipe, - "Stdout is not connected to any output in this environment", - )) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -impl Stderr { - pub fn new() -> io::Result { - Ok(Stderr(())) - } -} - -impl io::Write for Stderr { - fn write(&mut self, _buf: &[u8]) -> io::Result { - Err(io::Error::new( - io::ErrorKind::BrokenPipe, - "Stderr is not connected to any output in this environment", - )) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -pub fn is_ebadf(err: &io::Error) -> bool { - err.raw_os_error() == Some(abi::errno::BADF as i32) -} - -pub const STDIN_BUF_SIZE: usize = crate::sys_common::io::DEFAULT_BUF_SIZE; - -pub fn panic_output() -> Option { - Stderr::new().ok() -} diff --git a/crux-mir/lib/std/src/sys/cloudabi/thread.rs b/crux-mir/lib/std/src/sys/cloudabi/thread.rs deleted file mode 100644 index 3afcae7ae..000000000 --- a/crux-mir/lib/std/src/sys/cloudabi/thread.rs +++ /dev/null @@ -1,113 +0,0 @@ -use crate::cmp; -use crate::ffi::CStr; -use crate::io; -use crate::mem; -use crate::ptr; -use crate::sys::cloudabi::abi; -use crate::sys::time::checked_dur2intervals; -use crate::sys_common::thread::*; -use crate::time::Duration; - -pub const DEFAULT_MIN_STACK_SIZE: usize = 2 * 1024 * 1024; - -pub struct Thread { - id: libc::pthread_t, -} - -// CloudABI has pthread_t as a pointer in which case we still want -// a thread to be Send/Sync -unsafe impl Send for Thread {} -unsafe impl Sync for Thread {} - -impl Thread { - // unsafe: see thread::Builder::spawn_unchecked for safety requirements - pub unsafe fn new(stack: usize, p: Box) -> io::Result { - let p = box p; - let mut native: libc::pthread_t = mem::zeroed(); - let mut attr: libc::pthread_attr_t = mem::zeroed(); - assert_eq!(libc::pthread_attr_init(&mut attr), 0); - - let stack_size = cmp::max(stack, min_stack_size(&attr)); - assert_eq!(libc::pthread_attr_setstacksize(&mut attr, stack_size), 0); - - let ret = libc::pthread_create(&mut native, &attr, thread_start, &*p as *const _ as *mut _); - assert_eq!(libc::pthread_attr_destroy(&mut attr), 0); - - return if ret != 0 { - Err(io::Error::from_raw_os_error(ret)) - } else { - mem::forget(p); // ownership passed to pthread_create - Ok(Thread { id: native }) - }; - - extern "C" fn thread_start(main: *mut libc::c_void) -> *mut libc::c_void { - unsafe { - start_thread(main as *mut u8); - } - ptr::null_mut() - } - } - - pub fn yield_now() { - let ret = unsafe { abi::thread_yield() }; - debug_assert_eq!(ret, abi::errno::SUCCESS); - } - - pub fn set_name(_name: &CStr) { - // CloudABI has no way to set a thread name. - } - - pub fn sleep(dur: Duration) { - let timeout = - checked_dur2intervals(&dur).expect("overflow converting duration to nanoseconds"); - unsafe { - let subscription = abi::subscription { - type_: abi::eventtype::CLOCK, - union: abi::subscription_union { - clock: abi::subscription_clock { - clock_id: abi::clockid::MONOTONIC, - timeout, - ..mem::zeroed() - }, - }, - ..mem::zeroed() - }; - let mut event = mem::MaybeUninit::::uninit(); - let mut nevents = mem::MaybeUninit::::uninit(); - let ret = abi::poll(&subscription, event.as_mut_ptr(), 1, nevents.as_mut_ptr()); - assert_eq!(ret, abi::errno::SUCCESS); - let event = event.assume_init(); - assert_eq!(event.error, abi::errno::SUCCESS); - } - } - - pub fn join(self) { - unsafe { - let ret = libc::pthread_join(self.id, ptr::null_mut()); - mem::forget(self); - assert!(ret == 0, "failed to join thread: {}", io::Error::from_raw_os_error(ret)); - } - } -} - -impl Drop for Thread { - fn drop(&mut self) { - let ret = unsafe { libc::pthread_detach(self.id) }; - debug_assert_eq!(ret, 0); - } -} - -#[cfg_attr(test, allow(dead_code))] -pub mod guard { - pub type Guard = !; - pub unsafe fn current() -> Option { - None - } - pub unsafe fn init() -> Option { - None - } -} - -fn min_stack_size(_: *const libc::pthread_attr_t) -> usize { - libc::PTHREAD_STACK_MIN -} diff --git a/crux-mir/lib/std/src/sys/cloudabi/time.rs b/crux-mir/lib/std/src/sys/cloudabi/time.rs deleted file mode 100644 index c209231cf..000000000 --- a/crux-mir/lib/std/src/sys/cloudabi/time.rs +++ /dev/null @@ -1,82 +0,0 @@ -use crate::mem; -use crate::sys::cloudabi::abi; -use crate::time::Duration; - -const NSEC_PER_SEC: abi::timestamp = 1_000_000_000; - -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] -pub struct Instant { - t: abi::timestamp, -} - -pub fn checked_dur2intervals(dur: &Duration) -> Option { - dur.as_secs().checked_mul(NSEC_PER_SEC)?.checked_add(dur.subsec_nanos() as abi::timestamp) -} - -impl Instant { - pub fn now() -> Instant { - unsafe { - let mut t: mem::MaybeUninit = mem::MaybeUninit::uninit(); - let ret = abi::clock_time_get(abi::clockid::MONOTONIC, 0, t.as_mut_ptr()); - assert_eq!(ret, abi::errno::SUCCESS); - Instant { t: t.assume_init() } - } - } - - pub fn actually_monotonic() -> bool { - true - } - - pub const fn zero() -> Instant { - Instant { t: 0 } - } - - pub fn checked_sub_instant(&self, other: &Instant) -> Option { - let diff = self.t.checked_sub(other.t)?; - Some(Duration::new(diff / NSEC_PER_SEC, (diff % NSEC_PER_SEC) as u32)) - } - - pub fn checked_add_duration(&self, other: &Duration) -> Option { - Some(Instant { t: self.t.checked_add(checked_dur2intervals(other)?)? }) - } - - pub fn checked_sub_duration(&self, other: &Duration) -> Option { - Some(Instant { t: self.t.checked_sub(checked_dur2intervals(other)?)? }) - } -} - -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] -pub struct SystemTime { - t: abi::timestamp, -} - -impl SystemTime { - pub fn now() -> SystemTime { - unsafe { - let mut t: mem::MaybeUninit = mem::MaybeUninit::uninit(); - let ret = abi::clock_time_get(abi::clockid::REALTIME, 0, t.as_mut_ptr()); - assert_eq!(ret, abi::errno::SUCCESS); - SystemTime { t: t.assume_init() } - } - } - - pub fn sub_time(&self, other: &SystemTime) -> Result { - if self.t >= other.t { - let diff = self.t - other.t; - Ok(Duration::new(diff / NSEC_PER_SEC, (diff % NSEC_PER_SEC) as u32)) - } else { - let diff = other.t - self.t; - Err(Duration::new(diff / NSEC_PER_SEC, (diff % NSEC_PER_SEC) as u32)) - } - } - - pub fn checked_add_duration(&self, other: &Duration) -> Option { - Some(SystemTime { t: self.t.checked_add(checked_dur2intervals(other)?)? }) - } - - pub fn checked_sub_duration(&self, other: &Duration) -> Option { - Some(SystemTime { t: self.t.checked_sub(checked_dur2intervals(other)?)? }) - } -} - -pub const UNIX_EPOCH: SystemTime = SystemTime { t: 0 }; diff --git a/crux-mir/lib/std/src/sys_common/alloc.rs b/crux-mir/lib/std/src/sys/common/alloc.rs similarity index 70% rename from crux-mir/lib/std/src/sys_common/alloc.rs rename to crux-mir/lib/std/src/sys/common/alloc.rs index c66941007..3edbe7280 100644 --- a/crux-mir/lib/std/src/sys_common/alloc.rs +++ b/crux-mir/lib/std/src/sys/common/alloc.rs @@ -1,31 +1,39 @@ -#![allow(dead_code)] - use crate::alloc::{GlobalAlloc, Layout, System}; use crate::cmp; use crate::ptr; // The minimum alignment guaranteed by the architecture. This value is used to // add fast paths for low alignment values. -#[cfg(all(any( +#[cfg(any( target_arch = "x86", target_arch = "arm", target_arch = "mips", target_arch = "powerpc", target_arch = "powerpc64", + target_arch = "sparc", target_arch = "asmjs", target_arch = "wasm32", - target_arch = "hexagon" -)))] + target_arch = "hexagon", + all(target_arch = "riscv32", not(target_os = "espidf")), + all(target_arch = "xtensa", not(target_os = "espidf")), +))] pub const MIN_ALIGN: usize = 8; -#[cfg(all(any( +#[cfg(any( target_arch = "x86_64", target_arch = "aarch64", target_arch = "mips64", target_arch = "s390x", target_arch = "sparc64", - target_arch = "riscv64" -)))] + target_arch = "riscv64", + target_arch = "wasm64", +))] pub const MIN_ALIGN: usize = 16; +// The allocator on the esp-idf platform guarantees 4 byte alignment. +#[cfg(any( + all(target_arch = "riscv32", target_os = "espidf"), + all(target_arch = "xtensa", target_os = "espidf"), +))] +pub const MIN_ALIGN: usize = 4; pub unsafe fn realloc_fallback( alloc: &System, diff --git a/crux-mir/lib/std/src/sys/common/mod.rs b/crux-mir/lib/std/src/sys/common/mod.rs new file mode 100644 index 000000000..29fc0835d --- /dev/null +++ b/crux-mir/lib/std/src/sys/common/mod.rs @@ -0,0 +1,17 @@ +// This module contains code that is shared between all platforms, mostly utility or fallback code. +// This explicitly does not include code that is shared between only a few platforms, +// such as when reusing an implementation from `unix` or `unsupported`. +// In those cases the desired code should be included directly using the #[path] attribute, +// not moved to this module. +// +// Currently `sys_common` contains a lot of code that should live in this module, +// ideally `sys_common` would only contain platform-independent abstractions on top of `sys`. +// Progress on this is tracked in #84187. + +#![allow(dead_code)] + +pub mod alloc; +pub mod small_c_string; + +#[cfg(test)] +mod tests; diff --git a/crux-mir/lib/std/src/sys/common/small_c_string.rs b/crux-mir/lib/std/src/sys/common/small_c_string.rs new file mode 100644 index 000000000..01acd5191 --- /dev/null +++ b/crux-mir/lib/std/src/sys/common/small_c_string.rs @@ -0,0 +1,58 @@ +use crate::ffi::{CStr, CString}; +use crate::mem::MaybeUninit; +use crate::path::Path; +use crate::slice; +use crate::{io, ptr}; + +// Make sure to stay under 4096 so the compiler doesn't insert a probe frame: +// https://docs.rs/compiler_builtins/latest/compiler_builtins/probestack/index.html +#[cfg(not(target_os = "espidf"))] +const MAX_STACK_ALLOCATION: usize = 384; +#[cfg(target_os = "espidf")] +const MAX_STACK_ALLOCATION: usize = 32; + +const NUL_ERR: io::Error = + io::const_io_error!(io::ErrorKind::InvalidInput, "file name contained an unexpected NUL byte"); + +#[inline] +pub fn run_path_with_cstr(path: &Path, f: F) -> io::Result +where + F: FnOnce(&CStr) -> io::Result, +{ + run_with_cstr(path.as_os_str().bytes(), f) +} + +#[inline] +pub fn run_with_cstr(bytes: &[u8], f: F) -> io::Result +where + F: FnOnce(&CStr) -> io::Result, +{ + if bytes.len() >= MAX_STACK_ALLOCATION { + return run_with_cstr_allocating(bytes, f); + } + + let mut buf = MaybeUninit::<[u8; MAX_STACK_ALLOCATION]>::uninit(); + let buf_ptr = buf.as_mut_ptr() as *mut u8; + + unsafe { + ptr::copy_nonoverlapping(bytes.as_ptr(), buf_ptr, bytes.len()); + buf_ptr.add(bytes.len()).write(0); + } + + match CStr::from_bytes_with_nul(unsafe { slice::from_raw_parts(buf_ptr, bytes.len() + 1) }) { + Ok(s) => f(s), + Err(_) => Err(NUL_ERR), + } +} + +#[cold] +#[inline(never)] +fn run_with_cstr_allocating(bytes: &[u8], f: F) -> io::Result +where + F: FnOnce(&CStr) -> io::Result, +{ + match CString::new(bytes) { + Ok(s) => f(&s), + Err(_) => Err(NUL_ERR), + } +} diff --git a/crux-mir/lib/std/src/sys/common/tests.rs b/crux-mir/lib/std/src/sys/common/tests.rs new file mode 100644 index 000000000..fb6f5d6af --- /dev/null +++ b/crux-mir/lib/std/src/sys/common/tests.rs @@ -0,0 +1,66 @@ +use crate::ffi::CString; +use crate::hint::black_box; +use crate::path::Path; +use crate::sys::common::small_c_string::run_path_with_cstr; +use core::iter::repeat; + +#[test] +fn stack_allocation_works() { + let path = Path::new("abc"); + let result = run_path_with_cstr(path, |p| { + assert_eq!(p, &*CString::new(path.as_os_str().bytes()).unwrap()); + Ok(42) + }); + assert_eq!(result.unwrap(), 42); +} + +#[test] +fn stack_allocation_fails() { + let path = Path::new("ab\0"); + assert!(run_path_with_cstr::<(), _>(path, |_| unreachable!()).is_err()); +} + +#[test] +fn heap_allocation_works() { + let path = repeat("a").take(384).collect::(); + let path = Path::new(&path); + let result = run_path_with_cstr(path, |p| { + assert_eq!(p, &*CString::new(path.as_os_str().bytes()).unwrap()); + Ok(42) + }); + assert_eq!(result.unwrap(), 42); +} + +#[test] +fn heap_allocation_fails() { + let mut path = repeat("a").take(384).collect::(); + path.push('\0'); + let path = Path::new(&path); + assert!(run_path_with_cstr::<(), _>(path, |_| unreachable!()).is_err()); +} + +#[bench] +fn bench_stack_path_alloc(b: &mut test::Bencher) { + let path = repeat("a").take(383).collect::(); + let p = Path::new(&path); + b.iter(|| { + run_path_with_cstr(p, |cstr| { + black_box(cstr); + Ok(()) + }) + .unwrap(); + }); +} + +#[bench] +fn bench_heap_path_alloc(b: &mut test::Bencher) { + let path = repeat("a").take(384).collect::(); + let p = Path::new(&path); + b.iter(|| { + run_path_with_cstr(p, |cstr| { + black_box(cstr); + Ok(()) + }) + .unwrap(); + }); +} diff --git a/crux-mir/lib/std/src/sys/crux/condvar.rs b/crux-mir/lib/std/src/sys/crux/condvar.rs deleted file mode 100644 index 8b7e42468..000000000 --- a/crux-mir/lib/std/src/sys/crux/condvar.rs +++ /dev/null @@ -1,44 +0,0 @@ -use crate::sys::crux::mutex::Mutex; -use crate::time::Duration; - -pub struct Condvar { -} - -unsafe impl Send for Condvar {} -unsafe impl Sync for Condvar {} - -impl Condvar { - pub const fn new() -> Condvar { - Condvar {} - } - - pub unsafe fn init(&mut self) { - // No-op - } - - #[inline] - pub unsafe fn notify_one(&self) { - // No-op - } - - #[inline] - pub unsafe fn notify_all(&self) { - // No-op - } - - #[inline] - pub unsafe fn wait(&self, mutex: &Mutex) { - panic!("deadlock: single-threaded program is waiting on a condvar"); - } - - #[inline] - pub unsafe fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool { - // Pretend to have timed out, as there's no other way to end a wait. - false - } - - #[inline] - pub unsafe fn destroy(&self) { - // No-op - } -} diff --git a/crux-mir/lib/std/src/sys/crux/mod.rs b/crux-mir/lib/std/src/sys/crux/mod.rs deleted file mode 100644 index 08110995f..000000000 --- a/crux-mir/lib/std/src/sys/crux/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod condvar; -pub mod mutex; -pub mod rwlock; -pub mod time; diff --git a/crux-mir/lib/std/src/sys/crux/mutex.rs b/crux-mir/lib/std/src/sys/crux/mutex.rs deleted file mode 100644 index 792ddef0d..000000000 --- a/crux-mir/lib/std/src/sys/crux/mutex.rs +++ /dev/null @@ -1,94 +0,0 @@ -#![allow(dead_code)] -use crate::cell::Cell; -use crate::mem; -use core::crucible::concurrency; - -pub struct Mutex { - locked: Cell, -} - -unsafe impl Send for Mutex {} -unsafe impl Sync for Mutex {} - -impl Mutex { - pub const fn new() -> Mutex { - Mutex { - locked: Cell::new(false), - } - } - #[inline] - pub unsafe fn init(&mut self) { - // No-op - } - #[inline] - pub unsafe fn lock(&self) { - // TODO: Currently, we never run `drop` code, so locks are never released. Once we support - // drops, we can enable this assertion to check for (invalid) reentrant locking. - // assert!(!self.locked.get()); - concurrency::mutex_lock(self); - self.locked.set(true); - } - #[inline] - pub unsafe fn unlock(&self) { - // TODO: Currently, we never run `drop` code, so locks are never released. Once we support - // drops, we can enable this assertion to check for invalid usage. - concurrency::mutex_unlock(self); - // assert!(self.locked.get()); - self.locked.set(false); - } - #[inline] - pub unsafe fn try_lock(&self) -> bool { - if self.locked.get() { - false - } else { - self.locked.set(true); - true - } - } - #[inline] - pub unsafe fn destroy(&self) { - // No-op - } -} - -pub struct ReentrantMutex { - lock_count: Cell, -} - -unsafe impl Send for ReentrantMutex {} -unsafe impl Sync for ReentrantMutex {} - -impl ReentrantMutex { - pub const unsafe fn uninitialized() -> ReentrantMutex { - ReentrantMutex { - lock_count: Cell::new(0), - } - } - - #[inline] - pub unsafe fn init(&self) { - // No-op - } - - #[inline] - pub unsafe fn lock(&self) { - self.lock_count.set(self.lock_count.get() + 1); - } - - #[inline] - pub unsafe fn try_lock(&self) -> bool { - // There is only one thread, so locking cannot fail. - self.lock(); - true - } - - #[inline] - pub unsafe fn unlock(&self) { - self.lock_count.set(self.lock_count.get() - 1); - } - - #[inline] - pub unsafe fn destroy(&self) { - // No-op - } -} diff --git a/crux-mir/lib/std/src/sys/crux/rwlock.rs b/crux-mir/lib/std/src/sys/crux/rwlock.rs deleted file mode 100644 index 99b060290..000000000 --- a/crux-mir/lib/std/src/sys/crux/rwlock.rs +++ /dev/null @@ -1,70 +0,0 @@ -use crate::cell::Cell; - -pub struct RWLock { - num_readers: Cell, - write_locked: Cell, -} - -unsafe impl Send for RWLock {} -unsafe impl Sync for RWLock {} - -impl RWLock { - pub const fn new() -> RWLock { - RWLock { - num_readers: Cell::new(0), - write_locked: Cell::new(false), - } - } - #[inline] - pub unsafe fn read(&self) { - // TODO: Currently, we never run `drop` code, so locks are never released. Once we support - // drops, we can enable this assertion to check for invalid usage. - //assert!(!self.write_locked.get()); - self.num_readers.set(self.num_readers.get() + 1); - } - - #[inline] - pub unsafe fn try_read(&self) -> bool { - if self.write_locked.get() { - false - } else { - self.read(); - true - } - } - - #[inline] - pub unsafe fn write(&self) { - // TODO: Currently, we never run `drop` code, so locks are never released. Once we support - // drops, we can enable this assertion to check for invalid usage. - //assert!(!self.write_locked.get()); - //assert!(self.num_readers.get() == 0); - self.write_locked.set(true); - } - - #[inline] - pub unsafe fn try_write(&self) -> bool { - if self.write_locked.get() || self.num_readers.get() > 0 { - false - } else { - self.write(); - true - } - } - - #[inline] - pub unsafe fn read_unlock(&self) { - self.num_readers.set(self.num_readers.get() - 1); - } - - #[inline] - pub unsafe fn write_unlock(&self) { - assert!(self.write_locked.get()); - self.write_locked.set(false); - } - - #[inline] - pub unsafe fn destroy(&self) { - // No-op - } -} diff --git a/crux-mir/lib/std/src/sys/crux/time.rs b/crux-mir/lib/std/src/sys/crux/time.rs deleted file mode 100644 index 47df2dcf1..000000000 --- a/crux-mir/lib/std/src/sys/crux/time.rs +++ /dev/null @@ -1,70 +0,0 @@ -use core::convert::TryInto; -use core::time::Duration; -use crate::sys; -use crate::sys_common::FromInner; - - -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] -pub struct Instant(u64); - -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] -pub struct SystemTime(i64); - -pub const UNIX_EPOCH: SystemTime = SystemTime(0); - -impl Instant { - pub fn now() -> Instant { - Instant(0) - } - - pub const fn zero() -> Instant { - Instant(0) - } - - pub fn actually_monotonic() -> bool { - true - } - - pub fn checked_sub_instant(&self, other: &Instant) -> Option { - Some(Duration::from_nanos(self.0.checked_sub(other.0)?)) - } - - pub fn checked_add_duration(&self, other: &Duration) -> Option { - Some(Instant(self.0.checked_add(other.as_nanos().try_into().ok()?)?)) - } - - pub fn checked_sub_duration(&self, other: &Duration) -> Option { - Some(Instant(self.0.checked_sub(other.as_nanos().try_into().ok()?)?)) - } -} - -impl SystemTime { - pub fn now() -> SystemTime { - SystemTime(0) - } - - pub fn sub_time(&self, other: &SystemTime) -> Result { - if self.0 >= other.0 { - Ok(Duration::from_nanos((self.0 - other.0) as u64)) - } else { - Err(Duration::from_nanos((other.0 - self.0) as u64)) - } - } - - pub fn checked_add_duration(&self, other: &Duration) -> Option { - Some(SystemTime(self.0.checked_add(other.as_nanos().try_into().ok()?)?)) - } - - pub fn checked_sub_duration(&self, other: &Duration) -> Option { - Some(SystemTime(self.0.checked_sub(other.as_nanos().try_into().ok()?)?)) - } -} - -impl FromInner for SystemTime { - fn from_inner(time: sys::time::SystemTime) -> SystemTime { - match time.sub_time(&sys::time::UNIX_EPOCH) { - Ok(pos) => UNIX_EPOCH.checked_add_duration(&pos).unwrap(), - Err(neg) => UNIX_EPOCH.checked_sub_duration(&neg).unwrap(), - } - } -} diff --git a/crux-mir/lib/std/src/sys/hermit/args.rs b/crux-mir/lib/std/src/sys/hermit/args.rs index 72c1b8511..afcae6c90 100644 --- a/crux-mir/lib/std/src/sys/hermit/args.rs +++ b/crux-mir/lib/std/src/sys/hermit/args.rs @@ -1,33 +1,52 @@ -use crate::ffi::OsString; -use crate::marker::PhantomData; +use crate::ffi::{c_char, CStr, OsString}; +use crate::fmt; +use crate::os::unix::ffi::OsStringExt; +use crate::ptr; +use crate::sync::atomic::{ + AtomicIsize, AtomicPtr, + Ordering::{Acquire, Relaxed, Release}, +}; use crate::vec; +static ARGC: AtomicIsize = AtomicIsize::new(0); +static ARGV: AtomicPtr<*const u8> = AtomicPtr::new(ptr::null_mut()); + /// One-time global initialization. pub unsafe fn init(argc: isize, argv: *const *const u8) { - imp::init(argc, argv) -} - -/// One-time global cleanup. -pub unsafe fn cleanup() { - imp::cleanup() + ARGC.store(argc, Relaxed); + // Use release ordering here to broadcast writes by the OS. + ARGV.store(argv as *mut *const u8, Release); } /// Returns the command line arguments pub fn args() -> Args { - imp::args() + // Synchronize with the store above. + let argv = ARGV.load(Acquire); + // If argv has not been initialized yet, do not return any arguments. + let argc = if argv.is_null() { 0 } else { ARGC.load(Relaxed) }; + let args: Vec = (0..argc) + .map(|i| unsafe { + let cstr = CStr::from_ptr(*argv.offset(i) as *const c_char); + OsStringExt::from_vec(cstr.to_bytes().to_vec()) + }) + .collect(); + + Args { iter: args.into_iter() } } pub struct Args { iter: vec::IntoIter, - _dont_send_or_sync_me: PhantomData<*mut ()>, } -impl Args { - pub fn inner_debug(&self) -> &[OsString] { - self.iter.as_slice() +impl fmt::Debug for Args { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.iter.as_slice().fmt(f) } } +impl !Send for Args {} +impl !Sync for Args {} + impl Iterator for Args { type Item = OsString; fn next(&mut self) -> Option { @@ -49,45 +68,3 @@ impl DoubleEndedIterator for Args { self.iter.next_back() } } - -mod imp { - use super::Args; - use crate::ffi::{CStr, OsString}; - use crate::marker::PhantomData; - use crate::ptr; - use crate::sys_common::os_str_bytes::*; - - use crate::sys_common::mutex::Mutex; - - static mut ARGC: isize = 0; - static mut ARGV: *const *const u8 = ptr::null(); - static LOCK: Mutex = Mutex::new(); - - pub unsafe fn init(argc: isize, argv: *const *const u8) { - let _guard = LOCK.lock(); - ARGC = argc; - ARGV = argv; - } - - pub unsafe fn cleanup() { - let _guard = LOCK.lock(); - ARGC = 0; - ARGV = ptr::null(); - } - - pub fn args() -> Args { - Args { iter: clone().into_iter(), _dont_send_or_sync_me: PhantomData } - } - - fn clone() -> Vec { - unsafe { - let _guard = LOCK.lock(); - (0..ARGC) - .map(|i| { - let cstr = CStr::from_ptr(*ARGV.offset(i) as *const i8); - OsStringExt::from_vec(cstr.to_bytes().to_vec()) - }) - .collect() - } - } -} diff --git a/crux-mir/lib/std/src/sys/hermit/cmath.rs b/crux-mir/lib/std/src/sys/hermit/cmath.rs deleted file mode 100644 index 304cf906b..000000000 --- a/crux-mir/lib/std/src/sys/hermit/cmath.rs +++ /dev/null @@ -1,29 +0,0 @@ -// These symbols are all defined in `compiler-builtins` -extern "C" { - pub fn acos(n: f64) -> f64; - pub fn acosf(n: f32) -> f32; - pub fn asin(n: f64) -> f64; - pub fn asinf(n: f32) -> f32; - pub fn atan(n: f64) -> f64; - pub fn atan2(a: f64, b: f64) -> f64; - pub fn atan2f(a: f32, b: f32) -> f32; - pub fn atanf(n: f32) -> f32; - pub fn cbrt(n: f64) -> f64; - pub fn cbrtf(n: f32) -> f32; - pub fn cosh(n: f64) -> f64; - pub fn coshf(n: f32) -> f32; - pub fn expm1(n: f64) -> f64; - pub fn expm1f(n: f32) -> f32; - pub fn fdim(a: f64, b: f64) -> f64; - pub fn fdimf(a: f32, b: f32) -> f32; - pub fn hypot(x: f64, y: f64) -> f64; - pub fn hypotf(x: f32, y: f32) -> f32; - pub fn log1p(n: f64) -> f64; - pub fn log1pf(n: f32) -> f32; - pub fn sinh(n: f64) -> f64; - pub fn sinhf(n: f32) -> f32; - pub fn tan(n: f64) -> f64; - pub fn tanf(n: f32) -> f32; - pub fn tanh(n: f64) -> f64; - pub fn tanhf(n: f32) -> f32; -} diff --git a/crux-mir/lib/std/src/sys/hermit/condvar.rs b/crux-mir/lib/std/src/sys/hermit/condvar.rs deleted file mode 100644 index 5b7f16ce5..000000000 --- a/crux-mir/lib/std/src/sys/hermit/condvar.rs +++ /dev/null @@ -1,62 +0,0 @@ -use crate::cmp; -use crate::sys::hermit::abi; -use crate::sys::mutex::Mutex; -use crate::time::Duration; - -pub struct Condvar { - identifier: usize, -} - -impl Condvar { - pub const fn new() -> Condvar { - Condvar { identifier: 0 } - } - - #[inline] - pub unsafe fn init(&mut self) { - // nothing to do - } - - pub unsafe fn notify_one(&self) { - let _ = abi::notify(self.id(), 1); - } - - #[inline] - pub unsafe fn notify_all(&self) { - let _ = abi::notify(self.id(), -1 /* =all */); - } - - pub unsafe fn wait(&self, mutex: &Mutex) { - // add current task to the wait queue - let _ = abi::add_queue(self.id(), -1 /* no timeout */); - mutex.unlock(); - let _ = abi::wait(self.id()); - mutex.lock(); - } - - pub unsafe fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool { - let nanos = dur.as_nanos(); - let nanos = cmp::min(i64::max_value() as u128, nanos); - - // add current task to the wait queue - let _ = abi::add_queue(self.id(), nanos as i64); - - mutex.unlock(); - // If the return value is !0 then a timeout happened, so we return - // `false` as we weren't actually notified. - let ret = abi::wait(self.id()) == 0; - mutex.lock(); - - ret - } - - #[inline] - pub unsafe fn destroy(&self) { - let _ = abi::destroy_queue(self.id()); - } - - #[inline] - fn id(&self) -> usize { - &self.identifier as *const usize as usize - } -} diff --git a/crux-mir/lib/std/src/sys/hermit/fast_thread_local.rs b/crux-mir/lib/std/src/sys/hermit/fast_thread_local.rs deleted file mode 100644 index 1108e2545..000000000 --- a/crux-mir/lib/std/src/sys/hermit/fast_thread_local.rs +++ /dev/null @@ -1,4 +0,0 @@ -#![cfg(target_thread_local)] -#![unstable(feature = "thread_local_internals", issue = "none")] - -pub use crate::sys_common::thread_local::register_dtor_fallback as register_dtor; diff --git a/crux-mir/lib/std/src/sys/hermit/fd.rs b/crux-mir/lib/std/src/sys/hermit/fd.rs index 97d1a38b4..c400f5f2c 100644 --- a/crux-mir/lib/std/src/sys/hermit/fd.rs +++ b/crux-mir/lib/std/src/sys/hermit/fd.rs @@ -1,9 +1,10 @@ #![unstable(reason = "not public", issue = "none", feature = "fd")] -use crate::io::{self, ErrorKind, Read}; +use crate::io::{self, Read}; use crate::mem; use crate::sys::cvt; use crate::sys::hermit::abi; +use crate::sys::unsupported; use crate::sys_common::AsInner; #[derive(Debug)] @@ -46,7 +47,7 @@ impl FileDesc { self.duplicate_path(&[]) } pub fn duplicate_path(&self, _path: &[u8]) -> io::Result { - Err(io::Error::new(ErrorKind::Other, "duplicate isn't supported")) + unsupported() } pub fn nonblocking(&self) -> io::Result { @@ -54,11 +55,11 @@ impl FileDesc { } pub fn set_cloexec(&self) -> io::Result<()> { - Err(io::Error::new(ErrorKind::Other, "cloexec isn't supported")) + unsupported() } pub fn set_nonblocking(&self, _nonblocking: bool) -> io::Result<()> { - Err(io::Error::new(ErrorKind::Other, "nonblocking isn't supported")) + unsupported() } } diff --git a/crux-mir/lib/std/src/sys/hermit/fs.rs b/crux-mir/lib/std/src/sys/hermit/fs.rs index 37ac5984e..6fb92c037 100644 --- a/crux-mir/lib/std/src/sys/hermit/fs.rs +++ b/crux-mir/lib/std/src/sys/hermit/fs.rs @@ -1,39 +1,28 @@ -use crate::ffi::{CStr, CString, OsString}; +use crate::ffi::{CStr, OsString}; use crate::fmt; use crate::hash::{Hash, Hasher}; use crate::io::{self, Error, ErrorKind}; -use crate::io::{IoSlice, IoSliceMut, SeekFrom}; +use crate::io::{BorrowedCursor, IoSlice, IoSliceMut, SeekFrom}; use crate::path::{Path, PathBuf}; +use crate::sys::common::small_c_string::run_path_with_cstr; use crate::sys::cvt; use crate::sys::hermit::abi; +use crate::sys::hermit::abi::{O_APPEND, O_CREAT, O_EXCL, O_RDONLY, O_RDWR, O_TRUNC, O_WRONLY}; use crate::sys::hermit::fd::FileDesc; use crate::sys::time::SystemTime; -use crate::sys::{unsupported, Void}; -use crate::sys_common::os_str_bytes::OsStrExt; +use crate::sys::unsupported; -pub use crate::sys_common::fs::copy; +pub use crate::sys_common::fs::{copy, try_exists}; //pub use crate::sys_common::fs::remove_dir_all; -fn cstr(path: &Path) -> io::Result { - Ok(CString::new(path.as_os_str().as_bytes())?) -} -//const O_ACCMODE: i32 = 00000003; -const O_RDONLY: i32 = 00000000; -const O_WRONLY: i32 = 00000001; -const O_RDWR: i32 = 00000002; -const O_CREAT: i32 = 00000100; -const O_EXCL: i32 = 00000200; -const O_TRUNC: i32 = 00001000; -const O_APPEND: i32 = 00002000; - #[derive(Debug)] pub struct File(FileDesc); -pub struct FileAttr(Void); +pub struct FileAttr(!); -pub struct ReadDir(Void); +pub struct ReadDir(!); -pub struct DirEntry(Void); +pub struct DirEntry(!); #[derive(Clone, Debug)] pub struct OpenOptions { @@ -48,64 +37,67 @@ pub struct OpenOptions { mode: i32, } -pub struct FilePermissions(Void); +#[derive(Copy, Clone, Debug, Default)] +pub struct FileTimes {} -pub struct FileType(Void); +pub struct FilePermissions(!); + +pub struct FileType(!); #[derive(Debug)] pub struct DirBuilder {} impl FileAttr { pub fn size(&self) -> u64 { - match self.0 {} + self.0 } pub fn perm(&self) -> FilePermissions { - match self.0 {} + self.0 } pub fn file_type(&self) -> FileType { - match self.0 {} + self.0 } pub fn modified(&self) -> io::Result { - match self.0 {} + self.0 } pub fn accessed(&self) -> io::Result { - match self.0 {} + self.0 } pub fn created(&self) -> io::Result { - match self.0 {} + self.0 } } impl Clone for FileAttr { fn clone(&self) -> FileAttr { - match self.0 {} + self.0 } } impl FilePermissions { pub fn readonly(&self) -> bool { - match self.0 {} + self.0 } pub fn set_readonly(&mut self, _readonly: bool) { - match self.0 {} + self.0 } } impl Clone for FilePermissions { fn clone(&self) -> FilePermissions { - match self.0 {} + self.0 } } impl PartialEq for FilePermissions { fn eq(&self, _other: &FilePermissions) -> bool { - match self.0 {} + self.0 } } @@ -113,27 +105,32 @@ impl Eq for FilePermissions {} impl fmt::Debug for FilePermissions { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} + self.0 } } +impl FileTimes { + pub fn set_accessed(&mut self, _t: SystemTime) {} + pub fn set_modified(&mut self, _t: SystemTime) {} +} + impl FileType { pub fn is_dir(&self) -> bool { - match self.0 {} + self.0 } pub fn is_file(&self) -> bool { - match self.0 {} + self.0 } pub fn is_symlink(&self) -> bool { - match self.0 {} + self.0 } } impl Clone for FileType { fn clone(&self) -> FileType { - match self.0 {} + self.0 } } @@ -141,7 +138,7 @@ impl Copy for FileType {} impl PartialEq for FileType { fn eq(&self, _other: &FileType) -> bool { - match self.0 {} + self.0 } } @@ -149,19 +146,19 @@ impl Eq for FileType {} impl Hash for FileType { fn hash(&self, _h: &mut H) { - match self.0 {} + self.0 } } impl fmt::Debug for FileType { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} + self.0 } } impl fmt::Debug for ReadDir { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} + self.0 } } @@ -169,25 +166,25 @@ impl Iterator for ReadDir { type Item = io::Result; fn next(&mut self) -> Option> { - match self.0 {} + self.0 } } impl DirEntry { pub fn path(&self) -> PathBuf { - match self.0 {} + self.0 } pub fn file_name(&self) -> OsString { - match self.0 {} + self.0 } pub fn metadata(&self) -> io::Result { - match self.0 {} + self.0 } pub fn file_type(&self) -> io::Result { - match self.0 {} + self.0 } } @@ -233,7 +230,7 @@ impl OpenOptions { (false, _, true) => Ok(O_WRONLY | O_APPEND), (true, _, true) => Ok(O_RDWR | O_APPEND), (false, false, false) => { - Err(io::Error::new(ErrorKind::InvalidInput, "invalid access mode")) + Err(io::const_io_error!(ErrorKind::InvalidInput, "invalid access mode")) } } } @@ -243,12 +240,18 @@ impl OpenOptions { (true, false) => {} (false, false) => { if self.truncate || self.create || self.create_new { - return Err(io::Error::new(ErrorKind::InvalidInput, "invalid creation mode")); + return Err(io::const_io_error!( + ErrorKind::InvalidInput, + "invalid creation mode", + )); } } (_, true) => { if self.truncate && !self.create_new { - return Err(io::Error::new(ErrorKind::InvalidInput, "invalid creation mode")); + return Err(io::const_io_error!( + ErrorKind::InvalidInput, + "invalid creation mode", + )); } } } @@ -265,8 +268,7 @@ impl OpenOptions { impl File { pub fn open(path: &Path, opts: &OpenOptions) -> io::Result { - let path = cstr(path)?; - File::open_c(&path, opts) + run_path_with_cstr(path, |path| File::open_c(&path, opts)) } pub fn open_c(path: &CStr, opts: &OpenOptions) -> io::Result { @@ -308,6 +310,15 @@ impl File { crate::io::default_read_vectored(|buf| self.read(buf), bufs) } + #[inline] + pub fn is_read_vectored(&self) -> bool { + false + } + + pub fn read_buf(&self, cursor: BorrowedCursor<'_>) -> io::Result<()> { + crate::io::default_read_buf(|buf| self.read(buf), cursor) + } + pub fn write(&self, buf: &[u8]) -> io::Result { self.0.write(buf) } @@ -316,6 +327,11 @@ impl File { crate::io::default_write_vectored(|buf| self.write(buf), bufs) } + #[inline] + pub fn is_write_vectored(&self) -> bool { + false + } + pub fn flush(&self) -> io::Result<()> { Ok(()) } @@ -332,8 +348,8 @@ impl File { Err(Error::from_raw_os_error(22)) } - pub fn diverge(&self) -> ! { - loop {} + pub fn set_times(&self, _times: FileTimes) -> io::Result<()> { + Err(Error::from_raw_os_error(22)) } } @@ -352,9 +368,7 @@ pub fn readdir(_p: &Path) -> io::Result { } pub fn unlink(path: &Path) -> io::Result<()> { - let name = cstr(path)?; - let _ = unsafe { cvt(abi::unlink(name.as_ptr()))? }; - Ok(()) + run_path_with_cstr(path, |path| cvt(unsafe { abi::unlink(path.as_ptr()) }).map(|_| ())) } pub fn rename(_old: &Path, _new: &Path) -> io::Result<()> { @@ -378,11 +392,11 @@ pub fn readlink(_p: &Path) -> io::Result { unsupported() } -pub fn symlink(_src: &Path, _dst: &Path) -> io::Result<()> { +pub fn symlink(_original: &Path, _link: &Path) -> io::Result<()> { unsupported() } -pub fn link(_src: &Path, _dst: &Path) -> io::Result<()> { +pub fn link(_original: &Path, _link: &Path) -> io::Result<()> { unsupported() } diff --git a/crux-mir/lib/std/src/sys/hermit/futex.rs b/crux-mir/lib/std/src/sys/hermit/futex.rs new file mode 100644 index 000000000..b64c174b0 --- /dev/null +++ b/crux-mir/lib/std/src/sys/hermit/futex.rs @@ -0,0 +1,39 @@ +use super::abi; +use crate::ptr::null; +use crate::sync::atomic::AtomicU32; +use crate::time::Duration; + +pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option) -> bool { + // Calculate the timeout as a relative timespec. + // + // Overflows are rounded up to an infinite timeout (None). + let timespec = timeout.and_then(|dur| { + Some(abi::timespec { + tv_sec: dur.as_secs().try_into().ok()?, + tv_nsec: dur.subsec_nanos().into(), + }) + }); + + let r = unsafe { + abi::futex_wait( + futex.as_mut_ptr(), + expected, + timespec.as_ref().map_or(null(), |t| t as *const abi::timespec), + abi::FUTEX_RELATIVE_TIMEOUT, + ) + }; + + r != -abi::errno::ETIMEDOUT +} + +#[inline] +pub fn futex_wake(futex: &AtomicU32) -> bool { + unsafe { abi::futex_wake(futex.as_mut_ptr(), 1) > 0 } +} + +#[inline] +pub fn futex_wake_all(futex: &AtomicU32) { + unsafe { + abi::futex_wake(futex.as_mut_ptr(), i32::MAX); + } +} diff --git a/crux-mir/lib/std/src/sys/hermit/mod.rs b/crux-mir/lib/std/src/sys/hermit/mod.rs index 1e4a53abd..6811fadb0 100644 --- a/crux-mir/lib/std/src/sys/hermit/mod.rs +++ b/crux-mir/lib/std/src/sys/hermit/mod.rs @@ -13,34 +13,50 @@ //! compiling for wasm. That way it's a compile time error for something that's //! guaranteed to be a runtime error! +#![allow(unsafe_op_in_unsafe_fn)] + use crate::intrinsics; use crate::os::raw::c_char; pub mod alloc; pub mod args; +#[path = "../unix/cmath.rs"] pub mod cmath; -pub mod condvar; pub mod env; -pub mod fast_thread_local; pub mod fd; pub mod fs; +pub mod futex; +#[path = "../unsupported/io.rs"] pub mod io; pub mod memchr; -pub mod mutex; pub mod net; pub mod os; +#[path = "../unix/os_str.rs"] +pub mod os_str; +#[path = "../unix/path.rs"] pub mod path; +#[path = "../unsupported/pipe.rs"] pub mod pipe; +#[path = "../unsupported/process.rs"] pub mod process; -pub mod rwlock; -pub mod stack_overflow; pub mod stdio; pub mod thread; -pub mod thread_local; +pub mod thread_local_dtor; +#[path = "../unsupported/thread_local_key.rs"] +pub mod thread_local_key; pub mod time; +#[path = "../unix/locks"] +pub mod locks { + mod futex_condvar; + mod futex_mutex; + mod futex_rwlock; + pub(crate) use futex_condvar::Condvar; + pub(crate) use futex_mutex::Mutex; + pub(crate) use futex_rwlock::RwLock; +} + use crate::io::ErrorKind; -pub use crate::sys_common::os_str_bytes as os_str; #[allow(unused_extern_crates)] pub extern crate hermit_abi as abi; @@ -50,22 +66,10 @@ pub fn unsupported() -> crate::io::Result { } pub fn unsupported_err() -> crate::io::Error { - crate::io::Error::new(crate::io::ErrorKind::Other, "operation not supported on HermitCore yet") -} - -// This enum is used as the storage for a bunch of types which can't actually -// exist. -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] -pub enum Void {} - -pub unsafe fn strlen(start: *const c_char) -> usize { - let mut str = start; - - while *str != 0 { - str = str.offset(1); - } - - (str as usize) - (start as usize) + crate::io::const_io_error!( + crate::io::ErrorKind::Unsupported, + "operation not supported on HermitCore yet", + ) } #[no_mangle] @@ -73,8 +77,10 @@ pub extern "C" fn floor(x: f64) -> f64 { unsafe { intrinsics::floorf64(x) } } -pub unsafe fn abort_internal() -> ! { - abi::abort(); +pub fn abort_internal() -> ! { + unsafe { + abi::abort(); + } } // FIXME: just a workaround to test the system @@ -87,17 +93,21 @@ pub fn hashmap_random_keys() -> (u64, u64) { #[cfg(not(test))] #[no_mangle] // NB. used by both libunwind and libpanic_abort -pub unsafe extern "C" fn __rust_abort() { +pub extern "C" fn __rust_abort() { abort_internal(); } -#[cfg(not(test))] -pub fn init() { - unsafe { - let _ = net::init(); - } +// SAFETY: must be called only once during runtime initialization. +// NOTE: this is not guaranteed to run, for example when Rust code is called externally. +pub unsafe fn init(argc: isize, argv: *const *const u8, _sigpipe: u8) { + let _ = net::init(); + args::init(argc, argv); } +// SAFETY: must be called only once during runtime cleanup. +// NOTE: this is not guaranteed to run, for example when the program aborts. +pub unsafe fn cleanup() {} + #[cfg(not(test))] #[no_mangle] pub unsafe extern "C" fn runtime_entry( @@ -105,6 +115,7 @@ pub unsafe extern "C" fn runtime_entry( argv: *const *const c_char, env: *const *const c_char, ) -> ! { + use crate::sys::hermit::thread_local_dtor::run_dtors; extern "C" { fn main(argc: isize, argv: *const *const c_char) -> i32; } @@ -114,6 +125,7 @@ pub unsafe extern "C" fn runtime_entry( let result = main(argc as isize, argv); + run_dtors(); abi::exit(result); } @@ -134,7 +146,7 @@ pub fn decode_error_kind(errno: i32) -> ErrorKind { x if x == 1 as i32 => ErrorKind::PermissionDenied, x if x == 32 as i32 => ErrorKind::BrokenPipe, x if x == 110 as i32 => ErrorKind::TimedOut, - _ => ErrorKind::Other, + _ => ErrorKind::Uncategorized, } } diff --git a/crux-mir/lib/std/src/sys/hermit/mutex.rs b/crux-mir/lib/std/src/sys/hermit/mutex.rs deleted file mode 100644 index 3d4813209..000000000 --- a/crux-mir/lib/std/src/sys/hermit/mutex.rs +++ /dev/null @@ -1,77 +0,0 @@ -use crate::ffi::c_void; -use crate::ptr; -use crate::sys::hermit::abi; - -pub struct Mutex { - inner: *const c_void, -} - -unsafe impl Send for Mutex {} -unsafe impl Sync for Mutex {} - -impl Mutex { - pub const fn new() -> Mutex { - Mutex { inner: ptr::null() } - } - - #[inline] - pub unsafe fn init(&mut self) { - let _ = abi::sem_init(&mut self.inner as *mut *const c_void, 1); - } - - #[inline] - pub unsafe fn lock(&self) { - let _ = abi::sem_timedwait(self.inner, 0); - } - - #[inline] - pub unsafe fn unlock(&self) { - let _ = abi::sem_post(self.inner); - } - - #[inline] - pub unsafe fn try_lock(&self) -> bool { - let result = abi::sem_trywait(self.inner); - result == 0 - } - - #[inline] - pub unsafe fn destroy(&self) { - let _ = abi::sem_destroy(self.inner); - } -} - -pub struct ReentrantMutex { - inner: *const c_void, -} - -impl ReentrantMutex { - pub const unsafe fn uninitialized() -> ReentrantMutex { - ReentrantMutex { inner: ptr::null() } - } - - #[inline] - pub unsafe fn init(&self) { - let _ = abi::recmutex_init(&self.inner as *const *const c_void as *mut _); - } - - #[inline] - pub unsafe fn lock(&self) { - let _ = abi::recmutex_lock(self.inner); - } - - #[inline] - pub unsafe fn try_lock(&self) -> bool { - true - } - - #[inline] - pub unsafe fn unlock(&self) { - let _ = abi::recmutex_unlock(self.inner); - } - - #[inline] - pub unsafe fn destroy(&self) { - let _ = abi::recmutex_destroy(self.inner); - } -} diff --git a/crux-mir/lib/std/src/sys/hermit/net.rs b/crux-mir/lib/std/src/sys/hermit/net.rs index 82917e71b..8a13879d8 100644 --- a/crux-mir/lib/std/src/sys/hermit/net.rs +++ b/crux-mir/lib/std/src/sys/hermit/net.rs @@ -1,161 +1,294 @@ -use crate::convert::TryFrom; use crate::fmt; -use crate::io::{self, IoSlice, IoSliceMut}; -use crate::net::{Ipv4Addr, Ipv6Addr, Shutdown, SocketAddr}; +use crate::io::{self, ErrorKind, IoSlice, IoSliceMut}; +use crate::net::{IpAddr, Ipv4Addr, Ipv6Addr, Shutdown, SocketAddr}; use crate::str; -use crate::sys::{unsupported, Void}; +use crate::sync::Arc; +use crate::sys::hermit::abi; +use crate::sys::hermit::abi::IpAddress::{Ipv4, Ipv6}; +use crate::sys::unsupported; +use crate::sys_common::AsInner; use crate::time::Duration; -//// Iinitializes HermitCore's network stack -pub unsafe fn init() -> io::Result<()> { +/// Checks whether the HermitCore's socket interface has been started already, and +/// if not, starts it. +pub fn init() -> io::Result<()> { + if abi::network_init() < 0 { + return Err(io::const_io_error!( + ErrorKind::Uncategorized, + "Unable to initialize network interface", + )); + } + Ok(()) } -pub struct TcpStream(Void); +#[derive(Debug, Clone)] +pub struct Socket(abi::Handle); -impl TcpStream { - pub fn connect(_: io::Result<&SocketAddr>) -> io::Result { - unsupported() +impl AsInner for Socket { + fn as_inner(&self) -> &abi::Handle { + &self.0 } +} - pub fn connect_timeout(_: &SocketAddr, _: Duration) -> io::Result { - unsupported() +impl Drop for Socket { + fn drop(&mut self) { + let _ = abi::tcpstream::close(self.0); } +} - pub fn set_read_timeout(&self, _: Option) -> io::Result<()> { - match self.0 {} - } +// Arc is used to count the number of used sockets. +// Only if all sockets are released, the drop +// method will close the socket. +#[derive(Clone)] +pub struct TcpStream(Arc); - pub fn set_write_timeout(&self, _: Option) -> io::Result<()> { - match self.0 {} +impl TcpStream { + pub fn connect(addr: io::Result<&SocketAddr>) -> io::Result { + let addr = addr?; + + match abi::tcpstream::connect(addr.ip().to_string().as_bytes(), addr.port(), None) { + Ok(handle) => Ok(TcpStream(Arc::new(Socket(handle)))), + _ => Err(io::const_io_error!( + ErrorKind::Uncategorized, + "Unable to initiate a connection on a socket", + )), + } + } + + pub fn connect_timeout(saddr: &SocketAddr, duration: Duration) -> io::Result { + match abi::tcpstream::connect( + saddr.ip().to_string().as_bytes(), + saddr.port(), + Some(duration.as_millis() as u64), + ) { + Ok(handle) => Ok(TcpStream(Arc::new(Socket(handle)))), + _ => Err(io::const_io_error!( + ErrorKind::Uncategorized, + "Unable to initiate a connection on a socket", + )), + } + } + + pub fn set_read_timeout(&self, duration: Option) -> io::Result<()> { + abi::tcpstream::set_read_timeout(*self.0.as_inner(), duration.map(|d| d.as_millis() as u64)) + .map_err(|_| { + io::const_io_error!(ErrorKind::Uncategorized, "Unable to set timeout value") + }) + } + + pub fn set_write_timeout(&self, duration: Option) -> io::Result<()> { + abi::tcpstream::set_write_timeout( + *self.0.as_inner(), + duration.map(|d| d.as_millis() as u64), + ) + .map_err(|_| io::const_io_error!(ErrorKind::Uncategorized, "Unable to set timeout value")) } pub fn read_timeout(&self) -> io::Result> { - match self.0 {} + let duration = abi::tcpstream::get_read_timeout(*self.0.as_inner()).map_err(|_| { + io::const_io_error!(ErrorKind::Uncategorized, "Unable to determine timeout value") + })?; + + Ok(duration.map(|d| Duration::from_millis(d))) } pub fn write_timeout(&self) -> io::Result> { - match self.0 {} + let duration = abi::tcpstream::get_write_timeout(*self.0.as_inner()).map_err(|_| { + io::const_io_error!(ErrorKind::Uncategorized, "Unable to determine timeout value") + })?; + + Ok(duration.map(|d| Duration::from_millis(d))) } - pub fn peek(&self, _: &mut [u8]) -> io::Result { - match self.0 {} + pub fn peek(&self, buf: &mut [u8]) -> io::Result { + abi::tcpstream::peek(*self.0.as_inner(), buf) + .map_err(|_| io::const_io_error!(ErrorKind::Uncategorized, "peek failed")) + } + + pub fn read(&self, buffer: &mut [u8]) -> io::Result { + self.read_vectored(&mut [IoSliceMut::new(buffer)]) + } + + pub fn read_vectored(&self, ioslice: &mut [IoSliceMut<'_>]) -> io::Result { + let mut size: usize = 0; + + for i in ioslice.iter_mut() { + let ret = abi::tcpstream::read(*self.0.as_inner(), &mut i[0..]).map_err(|_| { + io::const_io_error!(ErrorKind::Uncategorized, "Unable to read on socket") + })?; + + if ret != 0 { + size += ret; + } + } + + Ok(size) } - pub fn read(&self, _: &mut [u8]) -> io::Result { - match self.0 {} + #[inline] + pub fn is_read_vectored(&self) -> bool { + true } - pub fn read_vectored(&self, _: &mut [IoSliceMut<'_>]) -> io::Result { - match self.0 {} + pub fn write(&self, buffer: &[u8]) -> io::Result { + self.write_vectored(&[IoSlice::new(buffer)]) } - pub fn write(&self, _: &[u8]) -> io::Result { - match self.0 {} + pub fn write_vectored(&self, ioslice: &[IoSlice<'_>]) -> io::Result { + let mut size: usize = 0; + + for i in ioslice.iter() { + size += abi::tcpstream::write(*self.0.as_inner(), i).map_err(|_| { + io::const_io_error!(ErrorKind::Uncategorized, "Unable to write on socket") + })?; + } + + Ok(size) } - pub fn write_vectored(&self, _: &[IoSlice<'_>]) -> io::Result { - match self.0 {} + #[inline] + pub fn is_write_vectored(&self) -> bool { + true } pub fn peer_addr(&self) -> io::Result { - match self.0 {} + let (ipaddr, port) = abi::tcpstream::peer_addr(*self.0.as_inner()) + .map_err(|_| io::const_io_error!(ErrorKind::Uncategorized, "peer_addr failed"))?; + + let saddr = match ipaddr { + Ipv4(ref addr) => SocketAddr::new(IpAddr::V4(Ipv4Addr::from(addr.0)), port), + Ipv6(ref addr) => SocketAddr::new(IpAddr::V6(Ipv6Addr::from(addr.0)), port), + _ => { + return Err(io::const_io_error!(ErrorKind::Uncategorized, "peer_addr failed")); + } + }; + + Ok(saddr) } pub fn socket_addr(&self) -> io::Result { - match self.0 {} + unsupported() } - pub fn shutdown(&self, _: Shutdown) -> io::Result<()> { - match self.0 {} + pub fn shutdown(&self, how: Shutdown) -> io::Result<()> { + abi::tcpstream::shutdown(*self.0.as_inner(), how as i32) + .map_err(|_| io::const_io_error!(ErrorKind::Uncategorized, "unable to shutdown socket")) } pub fn duplicate(&self) -> io::Result { - match self.0 {} + Ok(self.clone()) } - pub fn set_nodelay(&self, _: bool) -> io::Result<()> { - match self.0 {} + pub fn set_linger(&self, _linger: Option) -> io::Result<()> { + unsupported() + } + + pub fn linger(&self) -> io::Result> { + unsupported() + } + + pub fn set_nodelay(&self, mode: bool) -> io::Result<()> { + abi::tcpstream::set_nodelay(*self.0.as_inner(), mode) + .map_err(|_| io::const_io_error!(ErrorKind::Uncategorized, "set_nodelay failed")) } pub fn nodelay(&self) -> io::Result { - match self.0 {} + abi::tcpstream::nodelay(*self.0.as_inner()) + .map_err(|_| io::const_io_error!(ErrorKind::Uncategorized, "nodelay failed")) } - pub fn set_ttl(&self, _: u32) -> io::Result<()> { - match self.0 {} + pub fn set_ttl(&self, tll: u32) -> io::Result<()> { + abi::tcpstream::set_tll(*self.0.as_inner(), tll) + .map_err(|_| io::const_io_error!(ErrorKind::Uncategorized, "unable to set TTL")) } pub fn ttl(&self) -> io::Result { - match self.0 {} + abi::tcpstream::get_tll(*self.0.as_inner()) + .map_err(|_| io::const_io_error!(ErrorKind::Uncategorized, "unable to get TTL")) } pub fn take_error(&self) -> io::Result> { - match self.0 {} + unsupported() } - pub fn set_nonblocking(&self, _: bool) -> io::Result<()> { - match self.0 {} + pub fn set_nonblocking(&self, mode: bool) -> io::Result<()> { + abi::tcpstream::set_nonblocking(*self.0.as_inner(), mode).map_err(|_| { + io::const_io_error!(ErrorKind::Uncategorized, "unable to set blocking mode") + }) } } impl fmt::Debug for TcpStream { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} + Ok(()) } } -pub struct TcpListener(Void); +#[derive(Clone)] +pub struct TcpListener(SocketAddr); impl TcpListener { - pub fn bind(_: io::Result<&SocketAddr>) -> io::Result { - unsupported() + pub fn bind(addr: io::Result<&SocketAddr>) -> io::Result { + let addr = addr?; + + Ok(TcpListener(*addr)) } pub fn socket_addr(&self) -> io::Result { - match self.0 {} + Ok(self.0) } pub fn accept(&self) -> io::Result<(TcpStream, SocketAddr)> { - match self.0 {} + let (handle, ipaddr, port) = abi::tcplistener::accept(self.0.port()) + .map_err(|_| io::const_io_error!(ErrorKind::Uncategorized, "accept failed"))?; + let saddr = match ipaddr { + Ipv4(ref addr) => SocketAddr::new(IpAddr::V4(Ipv4Addr::from(addr.0)), port), + Ipv6(ref addr) => SocketAddr::new(IpAddr::V6(Ipv6Addr::from(addr.0)), port), + _ => { + return Err(io::const_io_error!(ErrorKind::Uncategorized, "accept failed")); + } + }; + + Ok((TcpStream(Arc::new(Socket(handle))), saddr)) } pub fn duplicate(&self) -> io::Result { - match self.0 {} + Ok(self.clone()) } pub fn set_ttl(&self, _: u32) -> io::Result<()> { - match self.0 {} + unsupported() } pub fn ttl(&self) -> io::Result { - match self.0 {} + unsupported() } pub fn set_only_v6(&self, _: bool) -> io::Result<()> { - match self.0 {} + unsupported() } pub fn only_v6(&self) -> io::Result { - match self.0 {} + unsupported() } pub fn take_error(&self) -> io::Result> { - match self.0 {} + unsupported() } pub fn set_nonblocking(&self, _: bool) -> io::Result<()> { - match self.0 {} + unsupported() } } impl fmt::Debug for TcpListener { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} + Ok(()) } } -pub struct UdpSocket(Void); +pub struct UdpSocket(abi::Handle); impl UdpSocket { pub fn bind(_: io::Result<&SocketAddr>) -> io::Result { @@ -163,144 +296,144 @@ impl UdpSocket { } pub fn peer_addr(&self) -> io::Result { - match self.0 {} + unsupported() } pub fn socket_addr(&self) -> io::Result { - match self.0 {} + unsupported() } pub fn recv_from(&self, _: &mut [u8]) -> io::Result<(usize, SocketAddr)> { - match self.0 {} + unsupported() } pub fn peek_from(&self, _: &mut [u8]) -> io::Result<(usize, SocketAddr)> { - match self.0 {} + unsupported() } pub fn send_to(&self, _: &[u8], _: &SocketAddr) -> io::Result { - match self.0 {} + unsupported() } pub fn duplicate(&self) -> io::Result { - match self.0 {} + unsupported() } pub fn set_read_timeout(&self, _: Option) -> io::Result<()> { - match self.0 {} + unsupported() } pub fn set_write_timeout(&self, _: Option) -> io::Result<()> { - match self.0 {} + unsupported() } pub fn read_timeout(&self) -> io::Result> { - match self.0 {} + unsupported() } pub fn write_timeout(&self) -> io::Result> { - match self.0 {} + unsupported() } pub fn set_broadcast(&self, _: bool) -> io::Result<()> { - match self.0 {} + unsupported() } pub fn broadcast(&self) -> io::Result { - match self.0 {} + unsupported() } pub fn set_multicast_loop_v4(&self, _: bool) -> io::Result<()> { - match self.0 {} + unsupported() } pub fn multicast_loop_v4(&self) -> io::Result { - match self.0 {} + unsupported() } pub fn set_multicast_ttl_v4(&self, _: u32) -> io::Result<()> { - match self.0 {} + unsupported() } pub fn multicast_ttl_v4(&self) -> io::Result { - match self.0 {} + unsupported() } pub fn set_multicast_loop_v6(&self, _: bool) -> io::Result<()> { - match self.0 {} + unsupported() } pub fn multicast_loop_v6(&self) -> io::Result { - match self.0 {} + unsupported() } pub fn join_multicast_v4(&self, _: &Ipv4Addr, _: &Ipv4Addr) -> io::Result<()> { - match self.0 {} + unsupported() } pub fn join_multicast_v6(&self, _: &Ipv6Addr, _: u32) -> io::Result<()> { - match self.0 {} + unsupported() } pub fn leave_multicast_v4(&self, _: &Ipv4Addr, _: &Ipv4Addr) -> io::Result<()> { - match self.0 {} + unsupported() } pub fn leave_multicast_v6(&self, _: &Ipv6Addr, _: u32) -> io::Result<()> { - match self.0 {} + unsupported() } pub fn set_ttl(&self, _: u32) -> io::Result<()> { - match self.0 {} + unsupported() } pub fn ttl(&self) -> io::Result { - match self.0 {} + unsupported() } pub fn take_error(&self) -> io::Result> { - match self.0 {} + unsupported() } pub fn set_nonblocking(&self, _: bool) -> io::Result<()> { - match self.0 {} + unsupported() } pub fn recv(&self, _: &mut [u8]) -> io::Result { - match self.0 {} + unsupported() } pub fn peek(&self, _: &mut [u8]) -> io::Result { - match self.0 {} + unsupported() } pub fn send(&self, _: &[u8]) -> io::Result { - match self.0 {} + unsupported() } pub fn connect(&self, _: io::Result<&SocketAddr>) -> io::Result<()> { - match self.0 {} + unsupported() } } impl fmt::Debug for UdpSocket { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} + Ok(()) } } -pub struct LookupHost(Void); +pub struct LookupHost(!); impl LookupHost { pub fn port(&self) -> u16 { - match self.0 {} + self.0 } } impl Iterator for LookupHost { type Item = SocketAddr; fn next(&mut self) -> Option { - match self.0 {} + self.0 } } @@ -354,6 +487,4 @@ pub mod netc { #[derive(Copy, Clone)] pub struct sockaddr {} - - pub type socklen_t = usize; } diff --git a/crux-mir/lib/std/src/sys/hermit/os.rs b/crux-mir/lib/std/src/sys/hermit/os.rs index 78eabf8f8..8f927df85 100644 --- a/crux-mir/lib/std/src/sys/hermit/os.rs +++ b/crux-mir/lib/std/src/sys/hermit/os.rs @@ -4,13 +4,13 @@ use crate::ffi::{CStr, OsStr, OsString}; use crate::fmt; use crate::io; use crate::marker::PhantomData; -use crate::memchr; +use crate::os::unix::ffi::OsStringExt; use crate::path::{self, PathBuf}; use crate::str; use crate::sync::Mutex; use crate::sys::hermit::abi; -use crate::sys::{unsupported, Void}; -use crate::sys_common::os_str_bytes::*; +use crate::sys::memchr; +use crate::sys::unsupported; use crate::vec; pub fn errno() -> i32 { @@ -29,7 +29,7 @@ pub fn chdir(_: &path::Path) -> io::Result<()> { unsupported() } -pub struct SplitPaths<'a>(&'a Void); +pub struct SplitPaths<'a>(!, PhantomData<&'a ()>); pub fn split_paths(_unparsed: &OsStr) -> SplitPaths<'_> { panic!("unsupported") @@ -38,7 +38,7 @@ pub fn split_paths(_unparsed: &OsStr) -> SplitPaths<'_> { impl<'a> Iterator for SplitPaths<'a> { type Item = PathBuf; fn next(&mut self) -> Option { - match *self.0 {} + self.0 } } @@ -110,9 +110,11 @@ pub fn init_environment(env: *const *const i8) { pub struct Env { iter: vec::IntoIter<(OsString, OsString)>, - _dont_send_or_sync_me: PhantomData<*mut ()>, } +impl !Send for Env {} +impl !Sync for Env {} + impl Iterator for Env { type Item = (OsString, OsString); fn next(&mut self) -> Option<(OsString, OsString)> { @@ -134,17 +136,12 @@ pub fn env() -> Env { result.push((key.clone(), value.clone())); } - return Env { iter: result.into_iter(), _dont_send_or_sync_me: PhantomData }; + return Env { iter: result.into_iter() }; } } -pub fn getenv(k: &OsStr) -> io::Result> { - unsafe { - match ENV.as_ref().unwrap().lock().unwrap().get_mut(k) { - Some(value) => Ok(Some(value.clone())), - None => Ok(None), - } - } +pub fn getenv(k: &OsStr) -> Option { + unsafe { ENV.as_ref().unwrap().lock().unwrap().get_mut(k).cloned() } } pub fn setenv(k: &OsStr, v: &OsStr) -> io::Result<()> { diff --git a/crux-mir/lib/std/src/sys/hermit/path.rs b/crux-mir/lib/std/src/sys/hermit/path.rs deleted file mode 100644 index 840a7ae04..000000000 --- a/crux-mir/lib/std/src/sys/hermit/path.rs +++ /dev/null @@ -1,19 +0,0 @@ -use crate::ffi::OsStr; -use crate::path::Prefix; - -#[inline] -pub fn is_sep_byte(b: u8) -> bool { - b == b'/' -} - -#[inline] -pub fn is_verbatim_sep(b: u8) -> bool { - b == b'/' -} - -pub fn parse_prefix(_: &OsStr) -> Option> { - None -} - -pub const MAIN_SEP_STR: &str = "/"; -pub const MAIN_SEP: char = '/'; diff --git a/crux-mir/lib/std/src/sys/hermit/pipe.rs b/crux-mir/lib/std/src/sys/hermit/pipe.rs deleted file mode 100644 index fb14dc591..000000000 --- a/crux-mir/lib/std/src/sys/hermit/pipe.rs +++ /dev/null @@ -1,30 +0,0 @@ -use crate::io::{self, IoSlice, IoSliceMut}; -use crate::sys::Void; - -pub struct AnonPipe(Void); - -impl AnonPipe { - pub fn read(&self, _buf: &mut [u8]) -> io::Result { - match self.0 {} - } - - pub fn read_vectored(&self, _bufs: &mut [IoSliceMut<'_>]) -> io::Result { - match self.0 {} - } - - pub fn write(&self, _buf: &[u8]) -> io::Result { - match self.0 {} - } - - pub fn write_vectored(&self, _bufs: &[IoSlice<'_>]) -> io::Result { - match self.0 {} - } - - pub fn diverge(&self) -> ! { - match self.0 {} - } -} - -pub fn read2(p1: AnonPipe, _v1: &mut Vec, _p2: AnonPipe, _v2: &mut Vec) -> io::Result<()> { - match p1.0 {} -} diff --git a/crux-mir/lib/std/src/sys/hermit/process.rs b/crux-mir/lib/std/src/sys/hermit/process.rs deleted file mode 100644 index 4702e5c54..000000000 --- a/crux-mir/lib/std/src/sys/hermit/process.rs +++ /dev/null @@ -1,149 +0,0 @@ -use crate::ffi::OsStr; -use crate::fmt; -use crate::io; -use crate::sys::fs::File; -use crate::sys::pipe::AnonPipe; -use crate::sys::{unsupported, Void}; -use crate::sys_common::process::CommandEnv; - -pub use crate::ffi::OsString as EnvKey; - -//////////////////////////////////////////////////////////////////////////////// -// Command -//////////////////////////////////////////////////////////////////////////////// - -pub struct Command { - env: CommandEnv, -} - -// passed back to std::process with the pipes connected to the child, if any -// were requested -pub struct StdioPipes { - pub stdin: Option, - pub stdout: Option, - pub stderr: Option, -} - -pub enum Stdio { - Inherit, - Null, - MakePipe, -} - -impl Command { - pub fn new(_program: &OsStr) -> Command { - Command { env: Default::default() } - } - - pub fn arg(&mut self, _arg: &OsStr) {} - - pub fn env_mut(&mut self) -> &mut CommandEnv { - &mut self.env - } - - pub fn cwd(&mut self, _dir: &OsStr) {} - - pub fn stdin(&mut self, _stdin: Stdio) {} - - pub fn stdout(&mut self, _stdout: Stdio) {} - - pub fn stderr(&mut self, _stderr: Stdio) {} - - pub fn spawn( - &mut self, - _default: Stdio, - _needs_stdin: bool, - ) -> io::Result<(Process, StdioPipes)> { - unsupported() - } -} - -impl From for Stdio { - fn from(pipe: AnonPipe) -> Stdio { - pipe.diverge() - } -} - -impl From for Stdio { - fn from(file: File) -> Stdio { - file.diverge() - } -} - -impl fmt::Debug for Command { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - Ok(()) - } -} - -pub struct ExitStatus(Void); - -impl ExitStatus { - pub fn success(&self) -> bool { - match self.0 {} - } - - pub fn code(&self) -> Option { - match self.0 {} - } -} - -impl Clone for ExitStatus { - fn clone(&self) -> ExitStatus { - match self.0 {} - } -} - -impl Copy for ExitStatus {} - -impl PartialEq for ExitStatus { - fn eq(&self, _other: &ExitStatus) -> bool { - match self.0 {} - } -} - -impl Eq for ExitStatus {} - -impl fmt::Debug for ExitStatus { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} - } -} - -impl fmt::Display for ExitStatus { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} - } -} - -#[derive(PartialEq, Eq, Clone, Copy, Debug)] -pub struct ExitCode(bool); - -impl ExitCode { - pub const SUCCESS: ExitCode = ExitCode(false); - pub const FAILURE: ExitCode = ExitCode(true); - - pub fn as_i32(&self) -> i32 { - self.0 as i32 - } -} - -pub struct Process(Void); - -impl Process { - pub fn id(&self) -> u32 { - match self.0 {} - } - - pub fn kill(&mut self) -> io::Result<()> { - match self.0 {} - } - - pub fn wait(&mut self) -> io::Result { - match self.0 {} - } - - pub fn try_wait(&mut self) -> io::Result> { - match self.0 {} - } -} diff --git a/crux-mir/lib/std/src/sys/hermit/rwlock.rs b/crux-mir/lib/std/src/sys/hermit/rwlock.rs deleted file mode 100644 index c19799af3..000000000 --- a/crux-mir/lib/std/src/sys/hermit/rwlock.rs +++ /dev/null @@ -1,49 +0,0 @@ -use super::mutex::Mutex; - -pub struct RWLock { - mutex: Mutex, -} - -unsafe impl Send for RWLock {} -unsafe impl Sync for RWLock {} - -impl RWLock { - pub const fn new() -> RWLock { - RWLock { mutex: Mutex::new() } - } - - #[inline] - pub unsafe fn read(&self) { - self.mutex.lock(); - } - - #[inline] - pub unsafe fn try_read(&self) -> bool { - self.mutex.try_lock() - } - - #[inline] - pub unsafe fn write(&self) { - self.mutex.lock(); - } - - #[inline] - pub unsafe fn try_write(&self) -> bool { - self.mutex.try_lock() - } - - #[inline] - pub unsafe fn read_unlock(&self) { - self.mutex.unlock(); - } - - #[inline] - pub unsafe fn write_unlock(&self) { - self.mutex.unlock(); - } - - #[inline] - pub unsafe fn destroy(&self) { - self.mutex.destroy(); - } -} diff --git a/crux-mir/lib/std/src/sys/hermit/stack_overflow.rs b/crux-mir/lib/std/src/sys/hermit/stack_overflow.rs deleted file mode 100644 index 65a1b17ac..000000000 --- a/crux-mir/lib/std/src/sys/hermit/stack_overflow.rs +++ /dev/null @@ -1,13 +0,0 @@ -pub struct Handler; - -impl Handler { - pub unsafe fn new() -> Handler { - Handler - } -} - -#[inline] -pub unsafe fn init() {} - -#[inline] -pub unsafe fn cleanup() {} diff --git a/crux-mir/lib/std/src/sys/hermit/stdio.rs b/crux-mir/lib/std/src/sys/hermit/stdio.rs index 2eb011ccb..514de1df6 100644 --- a/crux-mir/lib/std/src/sys/hermit/stdio.rs +++ b/crux-mir/lib/std/src/sys/hermit/stdio.rs @@ -7,95 +7,105 @@ pub struct Stdout; pub struct Stderr; impl Stdin { - pub fn new() -> io::Result { - Ok(Stdin) + pub const fn new() -> Stdin { + Stdin } +} - pub fn read(&self, data: &mut [u8]) -> io::Result { +impl io::Read for Stdin { + fn read(&mut self, data: &mut [u8]) -> io::Result { self.read_vectored(&mut [IoSliceMut::new(data)]) } - pub fn read_vectored(&self, _data: &mut [IoSliceMut<'_>]) -> io::Result { - //ManuallyDrop::new(unsafe { WasiFd::from_raw(libc::STDIN_FILENO as u32) }) - // .read(data) + fn read_vectored(&mut self, _data: &mut [IoSliceMut<'_>]) -> io::Result { Ok(0) } + + #[inline] + fn is_read_vectored(&self) -> bool { + true + } } impl Stdout { - pub fn new() -> io::Result { - Ok(Stdout) + pub const fn new() -> Stdout { + Stdout } +} - pub fn write(&self, data: &[u8]) -> io::Result { +impl io::Write for Stdout { + fn write(&mut self, data: &[u8]) -> io::Result { let len; unsafe { len = abi::write(1, data.as_ptr() as *const u8, data.len()) } if len < 0 { - Err(io::Error::new(io::ErrorKind::Other, "Stdout is not able to print")) + Err(io::const_io_error!(io::ErrorKind::Uncategorized, "Stdout is not able to print")) } else { Ok(len as usize) } } - pub fn write_vectored(&self, data: &[IoSlice<'_>]) -> io::Result { + fn write_vectored(&mut self, data: &[IoSlice<'_>]) -> io::Result { let len; unsafe { len = abi::write(1, data.as_ptr() as *const u8, data.len()) } if len < 0 { - Err(io::Error::new(io::ErrorKind::Other, "Stdout is not able to print")) + Err(io::const_io_error!(io::ErrorKind::Uncategorized, "Stdout is not able to print")) } else { Ok(len as usize) } } - pub fn flush(&self) -> io::Result<()> { + #[inline] + fn is_write_vectored(&self) -> bool { + true + } + + fn flush(&mut self) -> io::Result<()> { Ok(()) } } impl Stderr { - pub fn new() -> io::Result { - Ok(Stderr) + pub const fn new() -> Stderr { + Stderr } +} - pub fn write(&self, data: &[u8]) -> io::Result { +impl io::Write for Stderr { + fn write(&mut self, data: &[u8]) -> io::Result { let len; unsafe { len = abi::write(2, data.as_ptr() as *const u8, data.len()) } if len < 0 { - Err(io::Error::new(io::ErrorKind::Other, "Stderr is not able to print")) + Err(io::const_io_error!(io::ErrorKind::Uncategorized, "Stderr is not able to print")) } else { Ok(len as usize) } } - pub fn write_vectored(&self, data: &[IoSlice<'_>]) -> io::Result { + fn write_vectored(&mut self, data: &[IoSlice<'_>]) -> io::Result { let len; unsafe { len = abi::write(2, data.as_ptr() as *const u8, data.len()) } if len < 0 { - Err(io::Error::new(io::ErrorKind::Other, "Stderr is not able to print")) + Err(io::const_io_error!(io::ErrorKind::Uncategorized, "Stderr is not able to print")) } else { Ok(len as usize) } } - pub fn flush(&self) -> io::Result<()> { - Ok(()) + #[inline] + fn is_write_vectored(&self) -> bool { + true } -} -impl io::Write for Stderr { - fn write(&mut self, data: &[u8]) -> io::Result { - (&*self).write(data) - } fn flush(&mut self) -> io::Result<()> { - (&*self).flush() + Ok(()) } } @@ -106,5 +116,5 @@ pub fn is_ebadf(_err: &io::Error) -> bool { } pub fn panic_output() -> Option { - Stderr::new().ok() + Some(Stderr::new()) } diff --git a/crux-mir/lib/std/src/sys/hermit/thread.rs b/crux-mir/lib/std/src/sys/hermit/thread.rs index c3c29c938..8f65544a9 100644 --- a/crux-mir/lib/std/src/sys/hermit/thread.rs +++ b/crux-mir/lib/std/src/sys/hermit/thread.rs @@ -1,39 +1,17 @@ #![allow(dead_code)] +use super::unsupported; use crate::ffi::CStr; -use crate::fmt; use crate::io; use crate::mem; +use crate::num::NonZeroUsize; +use crate::ptr; use crate::sys::hermit::abi; +use crate::sys::hermit::thread_local_dtor::run_dtors; use crate::time::Duration; -use core::u32; - -use crate::sys_common::thread::*; pub type Tid = abi::Tid; -/// Priority of a task -#[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)] -pub struct Priority(u8); - -impl Priority { - pub const fn into(self) -> u8 { - self.0 - } - - pub const fn from(x: u8) -> Self { - Priority(x) - } -} - -impl fmt::Display for Priority { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.0) - } -} - -pub const NORMAL_PRIO: Priority = Priority::from(2); - pub struct Thread { tid: Tid, } @@ -41,34 +19,39 @@ pub struct Thread { unsafe impl Send for Thread {} unsafe impl Sync for Thread {} -pub const DEFAULT_MIN_STACK_SIZE: usize = 262144; +pub const DEFAULT_MIN_STACK_SIZE: usize = 1 << 20; impl Thread { pub unsafe fn new_with_coreid( - _stack: usize, + stack: usize, p: Box, core_id: isize, ) -> io::Result { - let p = box p; - let mut tid: Tid = u32::MAX; - let ret = abi::spawn( - &mut tid as *mut Tid, + let p = Box::into_raw(box p); + let tid = abi::spawn2( thread_start, - &*p as *const _ as *const u8 as usize, - Priority::into(NORMAL_PRIO), + p as usize, + abi::Priority::into(abi::NORMAL_PRIO), + stack, core_id, ); - return if ret == 0 { - mem::forget(p); // ownership passed to pthread_create - Ok(Thread { tid: tid }) + return if tid == 0 { + // The thread failed to start and as a result p was not consumed. Therefore, it is + // safe to reconstruct the box so that it gets deallocated. + drop(Box::from_raw(p)); + Err(io::const_io_error!(io::ErrorKind::Uncategorized, "Unable to create thread!")) } else { - Err(io::Error::new(io::ErrorKind::Other, "Unable to create thread!")) + Ok(Thread { tid: tid }) }; extern "C" fn thread_start(main: usize) { unsafe { - start_thread(main as *mut u8); + // Finally, let's run some code. + Box::from_raw(ptr::from_exposed_addr::>(main).cast_mut())(); + + // run all destructors + run_dtors(); } } } @@ -115,6 +98,10 @@ impl Thread { } } +pub fn available_parallelism() -> io::Result { + unsupported() +} + pub mod guard { pub type Guard = !; pub unsafe fn current() -> Option { diff --git a/crux-mir/lib/std/src/sys/hermit/thread_local.rs b/crux-mir/lib/std/src/sys/hermit/thread_local.rs deleted file mode 100644 index c6f8adb21..000000000 --- a/crux-mir/lib/std/src/sys/hermit/thread_local.rs +++ /dev/null @@ -1,60 +0,0 @@ -#![allow(dead_code)] // not used on all platforms - -use crate::collections::BTreeMap; -use crate::ptr; -use crate::sync::atomic::{AtomicUsize, Ordering}; -use crate::sys_common::mutex::Mutex; - -pub type Key = usize; - -type Dtor = unsafe extern "C" fn(*mut u8); - -static NEXT_KEY: AtomicUsize = AtomicUsize::new(0); - -static mut KEYS: *mut BTreeMap> = ptr::null_mut(); -static KEYS_LOCK: Mutex = Mutex::new(); - -#[thread_local] -static mut LOCALS: *mut BTreeMap = ptr::null_mut(); - -unsafe fn keys() -> &'static mut BTreeMap> { - if KEYS.is_null() { - KEYS = Box::into_raw(Box::new(BTreeMap::new())); - } - &mut *KEYS -} - -unsafe fn locals() -> &'static mut BTreeMap { - if LOCALS.is_null() { - LOCALS = Box::into_raw(Box::new(BTreeMap::new())); - } - &mut *LOCALS -} - -#[inline] -pub unsafe fn create(dtor: Option) -> Key { - let key = NEXT_KEY.fetch_add(1, Ordering::SeqCst); - let _guard = KEYS_LOCK.lock(); - keys().insert(key, dtor); - key -} - -#[inline] -pub unsafe fn get(key: Key) -> *mut u8 { - if let Some(&entry) = locals().get(&key) { entry } else { ptr::null_mut() } -} - -#[inline] -pub unsafe fn set(key: Key, value: *mut u8) { - locals().insert(key, value); -} - -#[inline] -pub unsafe fn destroy(key: Key) { - keys().remove(&key); -} - -#[inline] -pub fn requires_synchronized_create() -> bool { - false -} diff --git a/crux-mir/lib/std/src/sys/hermit/thread_local_dtor.rs b/crux-mir/lib/std/src/sys/hermit/thread_local_dtor.rs new file mode 100644 index 000000000..9b683fce1 --- /dev/null +++ b/crux-mir/lib/std/src/sys/hermit/thread_local_dtor.rs @@ -0,0 +1,36 @@ +#![cfg(target_thread_local)] +#![unstable(feature = "thread_local_internals", issue = "none")] + +// Simplify dtor registration by using a list of destructors. +// The this solution works like the implementation of macOS and +// doesn't additional OS support + +use crate::cell::Cell; +use crate::ptr; + +#[thread_local] +static DTORS: Cell<*mut List> = Cell::new(ptr::null_mut()); + +type List = Vec<(*mut u8, unsafe extern "C" fn(*mut u8))>; + +pub unsafe fn register_dtor(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { + if DTORS.get().is_null() { + let v: Box = box Vec::new(); + DTORS.set(Box::into_raw(v)); + } + + let list: &mut List = &mut *DTORS.get(); + list.push((t, dtor)); +} + +// every thread call this function to run through all possible destructors +pub unsafe fn run_dtors() { + let mut ptr = DTORS.replace(ptr::null_mut()); + while !ptr.is_null() { + let list = Box::from_raw(ptr); + for (ptr, dtor) in list.into_iter() { + dtor(ptr); + } + ptr = DTORS.replace(ptr::null_mut()); + } +} diff --git a/crux-mir/lib/std/src/sys/hermit/time.rs b/crux-mir/lib/std/src/sys/hermit/time.rs index c02de17c1..c17e6c8af 100644 --- a/crux-mir/lib/std/src/sys/hermit/time.rs +++ b/crux-mir/lib/std/src/sys/hermit/time.rs @@ -1,7 +1,6 @@ #![allow(dead_code)] use crate::cmp::Ordering; -use crate::convert::TryInto; use crate::sys::hermit::abi; use crate::sys::hermit::abi::timespec; use crate::sys::hermit::abi::{CLOCK_MONOTONIC, CLOCK_REALTIME, NSEC_PER_SEC}; @@ -115,14 +114,6 @@ impl Instant { Instant { t: time } } - pub const fn zero() -> Instant { - Instant { t: Timespec::zero() } - } - - pub fn actually_monotonic() -> bool { - true - } - pub fn checked_sub_instant(&self, other: &Instant) -> Option { self.t.sub_timespec(&other.t).ok() } diff --git a/crux-mir/lib/std/src/sys/itron/abi.rs b/crux-mir/lib/std/src/sys/itron/abi.rs new file mode 100644 index 000000000..5eb14bb7e --- /dev/null +++ b/crux-mir/lib/std/src/sys/itron/abi.rs @@ -0,0 +1,197 @@ +//! ABI for μITRON derivatives +pub type int_t = crate::os::raw::c_int; +pub type uint_t = crate::os::raw::c_uint; +pub type bool_t = int_t; + +/// Kernel object ID +pub type ID = int_t; + +/// The current task. +pub const TSK_SELF: ID = 0; + +/// Relative time +pub type RELTIM = u32; + +/// Timeout (a valid `RELTIM` value or `TMO_FEVR`) +pub type TMO = u32; + +/// The infinite timeout value +pub const TMO_FEVR: TMO = TMO::MAX; + +/// The maximum valid value of `RELTIM` +pub const TMAX_RELTIM: RELTIM = 4_000_000_000; + +/// System time +pub type SYSTIM = u64; + +/// Error code type +pub type ER = int_t; + +/// Error code type, `ID` on success +pub type ER_ID = int_t; + +/// Service call operational mode +pub type MODE = uint_t; + +/// OR waiting condition for an eventflag +pub const TWF_ORW: MODE = 0x01; + +/// Object attributes +pub type ATR = uint_t; + +/// FIFO wait order +pub const TA_FIFO: ATR = 0; +/// Only one task is allowed to be in the waiting state for the eventflag +pub const TA_WSGL: ATR = 0; +/// The eventflag’s bit pattern is cleared when a task is released from the +/// waiting state for that eventflag. +pub const TA_CLR: ATR = 0x04; + +/// Bit pattern of an eventflag +pub type FLGPTN = uint_t; + +/// Task or interrupt priority +pub type PRI = int_t; + +/// The special value of `PRI` representing the current task's priority. +pub const TPRI_SELF: PRI = 0; + +/// Use the priority inheritance protocol +#[cfg(target_os = "solid_asp3")] +pub const TA_INHERIT: ATR = 0x02; + +/// Activate the task on creation +pub const TA_ACT: ATR = 0x01; + +/// The maximum count of a semaphore +pub const TMAX_MAXSEM: uint_t = uint_t::MAX; + +/// Callback parameter +pub type EXINF = isize; + +/// Task entrypoint +pub type TASK = Option; + +// Error codes +pub const E_OK: ER = 0; +pub const E_SYS: ER = -5; +pub const E_NOSPT: ER = -9; +pub const E_RSFN: ER = -10; +pub const E_RSATR: ER = -11; +pub const E_PAR: ER = -17; +pub const E_ID: ER = -18; +pub const E_CTX: ER = -25; +pub const E_MACV: ER = -26; +pub const E_OACV: ER = -27; +pub const E_ILUSE: ER = -28; +pub const E_NOMEM: ER = -33; +pub const E_NOID: ER = -34; +pub const E_NORES: ER = -35; +pub const E_OBJ: ER = -41; +pub const E_NOEXS: ER = -42; +pub const E_QOVR: ER = -43; +pub const E_RLWAI: ER = -49; +pub const E_TMOUT: ER = -50; +pub const E_DLT: ER = -51; +pub const E_CLS: ER = -52; +pub const E_RASTER: ER = -53; +pub const E_WBLK: ER = -57; +pub const E_BOVR: ER = -58; +pub const E_COMM: ER = -65; + +#[derive(Clone, Copy)] +#[repr(C)] +pub struct T_CSEM { + pub sematr: ATR, + pub isemcnt: uint_t, + pub maxsem: uint_t, +} + +#[derive(Clone, Copy)] +#[repr(C)] +pub struct T_CFLG { + pub flgatr: ATR, + pub iflgptn: FLGPTN, +} + +#[derive(Clone, Copy)] +#[repr(C)] +pub struct T_CMTX { + pub mtxatr: ATR, + pub ceilpri: PRI, +} + +#[derive(Clone, Copy)] +#[repr(C)] +pub struct T_CTSK { + pub tskatr: ATR, + pub exinf: EXINF, + pub task: TASK, + pub itskpri: PRI, + pub stksz: usize, + pub stk: *mut u8, +} + +extern "C" { + #[link_name = "__asp3_acre_tsk"] + pub fn acre_tsk(pk_ctsk: *const T_CTSK) -> ER_ID; + #[link_name = "__asp3_get_tid"] + pub fn get_tid(p_tskid: *mut ID) -> ER; + #[link_name = "__asp3_dly_tsk"] + pub fn dly_tsk(dlytim: RELTIM) -> ER; + #[link_name = "__asp3_ter_tsk"] + pub fn ter_tsk(tskid: ID) -> ER; + #[link_name = "__asp3_del_tsk"] + pub fn del_tsk(tskid: ID) -> ER; + #[link_name = "__asp3_get_pri"] + pub fn get_pri(tskid: ID, p_tskpri: *mut PRI) -> ER; + #[link_name = "__asp3_rot_rdq"] + pub fn rot_rdq(tskpri: PRI) -> ER; + #[link_name = "__asp3_slp_tsk"] + pub fn slp_tsk() -> ER; + #[link_name = "__asp3_tslp_tsk"] + pub fn tslp_tsk(tmout: TMO) -> ER; + #[link_name = "__asp3_wup_tsk"] + pub fn wup_tsk(tskid: ID) -> ER; + #[link_name = "__asp3_unl_cpu"] + pub fn unl_cpu() -> ER; + #[link_name = "__asp3_dis_dsp"] + pub fn dis_dsp() -> ER; + #[link_name = "__asp3_ena_dsp"] + pub fn ena_dsp() -> ER; + #[link_name = "__asp3_sns_dsp"] + pub fn sns_dsp() -> bool_t; + #[link_name = "__asp3_get_tim"] + pub fn get_tim(p_systim: *mut SYSTIM) -> ER; + #[link_name = "__asp3_acre_flg"] + pub fn acre_flg(pk_cflg: *const T_CFLG) -> ER_ID; + #[link_name = "__asp3_del_flg"] + pub fn del_flg(flgid: ID) -> ER; + #[link_name = "__asp3_set_flg"] + pub fn set_flg(flgid: ID, setptn: FLGPTN) -> ER; + #[link_name = "__asp3_clr_flg"] + pub fn clr_flg(flgid: ID, clrptn: FLGPTN) -> ER; + #[link_name = "__asp3_wai_flg"] + pub fn wai_flg(flgid: ID, waiptn: FLGPTN, wfmode: MODE, p_flgptn: *mut FLGPTN) -> ER; + #[link_name = "__asp3_twai_flg"] + pub fn twai_flg( + flgid: ID, + waiptn: FLGPTN, + wfmode: MODE, + p_flgptn: *mut FLGPTN, + tmout: TMO, + ) -> ER; + #[link_name = "__asp3_acre_mtx"] + pub fn acre_mtx(pk_cmtx: *const T_CMTX) -> ER_ID; + #[link_name = "__asp3_del_mtx"] + pub fn del_mtx(tskid: ID) -> ER; + #[link_name = "__asp3_loc_mtx"] + pub fn loc_mtx(mtxid: ID) -> ER; + #[link_name = "__asp3_ploc_mtx"] + pub fn ploc_mtx(mtxid: ID) -> ER; + #[link_name = "__asp3_tloc_mtx"] + pub fn tloc_mtx(mtxid: ID, tmout: TMO) -> ER; + #[link_name = "__asp3_unl_mtx"] + pub fn unl_mtx(mtxid: ID) -> ER; + pub fn exd_tsk() -> ER; +} diff --git a/crux-mir/lib/std/src/sys/itron/condvar.rs b/crux-mir/lib/std/src/sys/itron/condvar.rs new file mode 100644 index 000000000..7a47cc669 --- /dev/null +++ b/crux-mir/lib/std/src/sys/itron/condvar.rs @@ -0,0 +1,292 @@ +//! POSIX conditional variable implementation based on user-space wait queues. +use super::{abi, error::expect_success_aborting, spin::SpinMutex, task, time::with_tmos_strong}; +use crate::{mem::replace, ptr::NonNull, sys::locks::Mutex, time::Duration}; + +// The implementation is inspired by the queue-based implementation shown in +// Andrew D. Birrell's paper "Implementing Condition Variables with Semaphores" + +pub struct Condvar { + waiters: SpinMutex, +} + +unsafe impl Send for Condvar {} +unsafe impl Sync for Condvar {} + +impl Condvar { + #[inline] + pub const fn new() -> Condvar { + Condvar { waiters: SpinMutex::new(waiter_queue::WaiterQueue::new()) } + } + + pub fn notify_one(&self) { + self.waiters.with_locked(|waiters| { + if let Some(task) = waiters.pop_front() { + // Unpark the task + match unsafe { abi::wup_tsk(task) } { + // The task already has a token. + abi::E_QOVR => {} + // Can't undo the effect; abort the program on failure + er => { + expect_success_aborting(er, &"wup_tsk"); + } + } + } + }); + } + + pub fn notify_all(&self) { + self.waiters.with_locked(|waiters| { + while let Some(task) = waiters.pop_front() { + // Unpark the task + match unsafe { abi::wup_tsk(task) } { + // The task already has a token. + abi::E_QOVR => {} + // Can't undo the effect; abort the program on failure + er => { + expect_success_aborting(er, &"wup_tsk"); + } + } + } + }); + } + + pub unsafe fn wait(&self, mutex: &Mutex) { + // Construct `Waiter`. + let mut waiter = waiter_queue::Waiter::new(); + let waiter = NonNull::from(&mut waiter); + + self.waiters.with_locked(|waiters| unsafe { + waiters.insert(waiter); + }); + + unsafe { mutex.unlock() }; + + // Wait until `waiter` is removed from the queue + loop { + // Park the current task + expect_success_aborting(unsafe { abi::slp_tsk() }, &"slp_tsk"); + + if !self.waiters.with_locked(|waiters| unsafe { waiters.is_queued(waiter) }) { + break; + } + } + + mutex.lock(); + } + + pub unsafe fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool { + // Construct and pin `Waiter` + let mut waiter = waiter_queue::Waiter::new(); + let waiter = NonNull::from(&mut waiter); + + self.waiters.with_locked(|waiters| unsafe { + waiters.insert(waiter); + }); + + unsafe { mutex.unlock() }; + + // Park the current task and do not wake up until the timeout elapses + // or the task gets woken up by `notify_*` + match with_tmos_strong(dur, |tmo| { + let er = unsafe { abi::tslp_tsk(tmo) }; + if er == 0 { + // We were unparked. Are we really dequeued? + if self.waiters.with_locked(|waiters| unsafe { waiters.is_queued(waiter) }) { + // No we are not. Continue waiting. + return abi::E_TMOUT; + } + } + er + }) { + abi::E_TMOUT => {} + er => { + expect_success_aborting(er, &"tslp_tsk"); + } + } + + // Remove `waiter` from `self.waiters`. If `waiter` is still in + // `waiters`, it means we woke up because of a timeout. Otherwise, + // we woke up because of `notify_*`. + let success = self.waiters.with_locked(|waiters| unsafe { !waiters.remove(waiter) }); + + mutex.lock(); + success + } +} + +mod waiter_queue { + use super::*; + + pub struct WaiterQueue { + head: Option, + } + + #[derive(Copy, Clone)] + struct ListHead { + first: NonNull, + last: NonNull, + } + + unsafe impl Send for ListHead {} + unsafe impl Sync for ListHead {} + + pub struct Waiter { + // These fields are only accessed through `&[mut] WaiterQueue`. + /// The waiting task's ID. Will be zeroed when the task is woken up + /// and removed from a queue. + task: abi::ID, + priority: abi::PRI, + prev: Option>, + next: Option>, + } + + unsafe impl Send for Waiter {} + unsafe impl Sync for Waiter {} + + impl Waiter { + #[inline] + pub fn new() -> Self { + let task = task::current_task_id(); + let priority = task::task_priority(abi::TSK_SELF); + + // Zeroness of `Waiter::task` indicates whether the `Waiter` is + // linked to a queue or not. This invariant is important for + // the correctness. + debug_assert_ne!(task, 0); + + Self { task, priority, prev: None, next: None } + } + } + + impl WaiterQueue { + #[inline] + pub const fn new() -> Self { + Self { head: None } + } + + /// # Safety + /// + /// - The caller must own `*waiter_ptr`. The caller will lose the + /// ownership until `*waiter_ptr` is removed from `self`. + /// + /// - `*waiter_ptr` must be valid until it's removed from the queue. + /// + /// - `*waiter_ptr` must not have been previously inserted to a `WaiterQueue`. + /// + pub unsafe fn insert(&mut self, mut waiter_ptr: NonNull) { + unsafe { + let waiter = waiter_ptr.as_mut(); + + debug_assert!(waiter.prev.is_none()); + debug_assert!(waiter.next.is_none()); + + if let Some(head) = &mut self.head { + // Find the insertion position and insert `waiter` + let insert_after = { + let mut cursor = head.last; + loop { + if waiter.priority >= cursor.as_ref().priority { + // `cursor` and all previous waiters have the same or higher + // priority than `current_task_priority`. Insert the new + // waiter right after `cursor`. + break Some(cursor); + } + cursor = if let Some(prev) = cursor.as_ref().prev { + prev + } else { + break None; + }; + } + }; + + if let Some(mut insert_after) = insert_after { + // Insert `waiter` after `insert_after` + let insert_before = insert_after.as_ref().next; + + waiter.prev = Some(insert_after); + insert_after.as_mut().next = Some(waiter_ptr); + + waiter.next = insert_before; + if let Some(mut insert_before) = insert_before { + insert_before.as_mut().prev = Some(waiter_ptr); + } else { + head.last = waiter_ptr; + } + } else { + // Insert `waiter` to the front + waiter.next = Some(head.first); + head.first.as_mut().prev = Some(waiter_ptr); + head.first = waiter_ptr; + } + } else { + // `waiter` is the only element + self.head = Some(ListHead { first: waiter_ptr, last: waiter_ptr }); + } + } + } + + /// Given a `Waiter` that was previously inserted to `self`, remove + /// it from `self` if it's still there. + #[inline] + pub unsafe fn remove(&mut self, mut waiter_ptr: NonNull) -> bool { + unsafe { + let waiter = waiter_ptr.as_mut(); + if waiter.task != 0 { + let head = self.head.as_mut().unwrap(); + + match (waiter.prev, waiter.next) { + (Some(mut prev), Some(mut next)) => { + prev.as_mut().next = Some(next); + next.as_mut().prev = Some(prev); + } + (None, Some(mut next)) => { + head.first = next; + next.as_mut().prev = None; + } + (Some(mut prev), None) => { + prev.as_mut().next = None; + head.last = prev; + } + (None, None) => { + self.head = None; + } + } + + waiter.task = 0; + + true + } else { + false + } + } + } + + /// Given a `Waiter` that was previously inserted to `self`, return a + /// flag indicating whether it's still in `self`. + #[inline] + pub unsafe fn is_queued(&self, waiter: NonNull) -> bool { + unsafe { waiter.as_ref().task != 0 } + } + + #[inline] + pub fn pop_front(&mut self) -> Option { + unsafe { + let head = self.head.as_mut()?; + let waiter = head.first.as_mut(); + + // Get the ID + let id = replace(&mut waiter.task, 0); + + // Unlink the waiter + if let Some(mut next) = waiter.next { + head.first = next; + next.as_mut().prev = None; + } else { + self.head = None; + } + + Some(id) + } + } + } +} diff --git a/crux-mir/lib/std/src/sys/itron/error.rs b/crux-mir/lib/std/src/sys/itron/error.rs new file mode 100644 index 000000000..830c60d32 --- /dev/null +++ b/crux-mir/lib/std/src/sys/itron/error.rs @@ -0,0 +1,159 @@ +use crate::{fmt, io::ErrorKind}; + +use super::abi; + +/// Wraps a μITRON error code. +#[derive(Debug, Copy, Clone)] +pub struct ItronError { + er: abi::ER, +} + +impl ItronError { + /// Construct `ItronError` from the specified error code. Returns `None` if the + /// error code does not represent a failure or warning. + #[inline] + pub fn new(er: abi::ER) -> Option { + if er < 0 { Some(Self { er }) } else { None } + } + + /// Returns `Ok(er)` if `er` represents a success or `Err(_)` otherwise. + #[inline] + pub fn err_if_negative(er: abi::ER) -> Result { + if let Some(error) = Self::new(er) { Err(error) } else { Ok(er) } + } + + /// Get the raw error code. + #[inline] + pub fn as_raw(&self) -> abi::ER { + self.er + } +} + +impl fmt::Display for ItronError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Allow the platforms to extend `error_name` + if let Some(name) = crate::sys::error::error_name(self.er) { + write!(f, "{} ({})", name, self.er) + } else { + write!(f, "{}", self.er) + } + } +} + +/// Describe the specified μITRON error code. Returns `None` if it's an +/// undefined error code. +pub fn error_name(er: abi::ER) -> Option<&'static str> { + match er { + // Success + er if er >= 0 => None, + + // μITRON 4.0 + abi::E_SYS => Some("system error"), + abi::E_NOSPT => Some("unsupported function"), + abi::E_RSFN => Some("reserved function code"), + abi::E_RSATR => Some("reserved attribute"), + abi::E_PAR => Some("parameter error"), + abi::E_ID => Some("invalid ID number"), + abi::E_CTX => Some("context error"), + abi::E_MACV => Some("memory access violation"), + abi::E_OACV => Some("object access violation"), + abi::E_ILUSE => Some("illegal service call use"), + abi::E_NOMEM => Some("insufficient memory"), + abi::E_NOID => Some("no ID number available"), + abi::E_OBJ => Some("object state error"), + abi::E_NOEXS => Some("non-existent object"), + abi::E_QOVR => Some("queue overflow"), + abi::E_RLWAI => Some("forced release from waiting"), + abi::E_TMOUT => Some("polling failure or timeout"), + abi::E_DLT => Some("waiting object deleted"), + abi::E_CLS => Some("waiting object state changed"), + abi::E_WBLK => Some("non-blocking code accepted"), + abi::E_BOVR => Some("buffer overflow"), + + // The TOPPERS third generation kernels + abi::E_NORES => Some("insufficient system resources"), + abi::E_RASTER => Some("termination request raised"), + abi::E_COMM => Some("communication failure"), + + _ => None, + } +} + +pub fn decode_error_kind(er: abi::ER) -> ErrorKind { + match er { + // Success + er if er >= 0 => ErrorKind::Uncategorized, + + // μITRON 4.0 + // abi::E_SYS + abi::E_NOSPT => ErrorKind::Unsupported, // Some("unsupported function"), + abi::E_RSFN => ErrorKind::InvalidInput, // Some("reserved function code"), + abi::E_RSATR => ErrorKind::InvalidInput, // Some("reserved attribute"), + abi::E_PAR => ErrorKind::InvalidInput, // Some("parameter error"), + abi::E_ID => ErrorKind::NotFound, // Some("invalid ID number"), + // abi::E_CTX + abi::E_MACV => ErrorKind::PermissionDenied, // Some("memory access violation"), + abi::E_OACV => ErrorKind::PermissionDenied, // Some("object access violation"), + // abi::E_ILUSE + abi::E_NOMEM => ErrorKind::OutOfMemory, // Some("insufficient memory"), + abi::E_NOID => ErrorKind::OutOfMemory, // Some("no ID number available"), + // abi::E_OBJ + abi::E_NOEXS => ErrorKind::NotFound, // Some("non-existent object"), + // abi::E_QOVR + abi::E_RLWAI => ErrorKind::Interrupted, // Some("forced release from waiting"), + abi::E_TMOUT => ErrorKind::TimedOut, // Some("polling failure or timeout"), + // abi::E_DLT + // abi::E_CLS + // abi::E_WBLK + // abi::E_BOVR + + // The TOPPERS third generation kernels + abi::E_NORES => ErrorKind::OutOfMemory, // Some("insufficient system resources"), + // abi::E_RASTER + // abi::E_COMM + _ => ErrorKind::Uncategorized, + } +} + +/// Similar to `ItronError::err_if_negative(er).expect()` except that, while +/// panicking, it prints the message to `panic_output` and aborts the program +/// instead. This ensures the error message is not obscured by double +/// panicking. +/// +/// This is useful for diagnosing creation failures of synchronization +/// primitives that are used by `std`'s internal mechanisms. Such failures +/// are common when the system is mis-configured to provide a too-small pool for +/// kernel objects. +#[inline] +pub fn expect_success(er: abi::ER, msg: &&str) -> abi::ER { + match ItronError::err_if_negative(er) { + Ok(x) => x, + Err(e) => fail(e, msg), + } +} + +/// Similar to `ItronError::err_if_negative(er).expect()` but aborts instead. +/// +/// Use this where panicking is not allowed or the effect of the failure +/// would be persistent. +#[inline] +pub fn expect_success_aborting(er: abi::ER, msg: &&str) -> abi::ER { + match ItronError::err_if_negative(er) { + Ok(x) => x, + Err(e) => fail_aborting(e, msg), + } +} + +#[cold] +pub fn fail(e: impl fmt::Display, msg: &&str) -> ! { + if crate::thread::panicking() { + fail_aborting(e, msg) + } else { + panic!("{} failed: {}", *msg, e) + } +} + +#[cold] +pub fn fail_aborting(e: impl fmt::Display, msg: &&str) -> ! { + rtabort!("{} failed: {}", *msg, e) +} diff --git a/crux-mir/lib/std/src/sys/itron/mutex.rs b/crux-mir/lib/std/src/sys/itron/mutex.rs new file mode 100644 index 000000000..1f6cc4194 --- /dev/null +++ b/crux-mir/lib/std/src/sys/itron/mutex.rs @@ -0,0 +1,85 @@ +//! Mutex implementation backed by μITRON mutexes. Assumes `acre_mtx` and +//! `TA_INHERIT` are available. +use super::{ + abi, + error::{expect_success, expect_success_aborting, fail, ItronError}, + spin::SpinIdOnceCell, +}; + +pub struct Mutex { + /// The ID of the underlying mutex object + mtx: SpinIdOnceCell<()>, +} + +/// Create a mutex object. This function never panics. +fn new_mtx() -> Result { + ItronError::err_if_negative(unsafe { + abi::acre_mtx(&abi::T_CMTX { + // Priority inheritance mutex + mtxatr: abi::TA_INHERIT, + // Unused + ceilpri: 0, + }) + }) +} + +impl Mutex { + #[inline] + pub const fn new() -> Mutex { + Mutex { mtx: SpinIdOnceCell::new() } + } + + /// Get the inner mutex's ID, which is lazily created. + fn raw(&self) -> abi::ID { + match self.mtx.get_or_try_init(|| new_mtx().map(|id| (id, ()))) { + Ok((id, ())) => id, + Err(e) => fail(e, &"acre_mtx"), + } + } + + pub fn lock(&self) { + let mtx = self.raw(); + expect_success(unsafe { abi::loc_mtx(mtx) }, &"loc_mtx"); + } + + pub unsafe fn unlock(&self) { + let mtx = unsafe { self.mtx.get_unchecked().0 }; + expect_success_aborting(unsafe { abi::unl_mtx(mtx) }, &"unl_mtx"); + } + + pub fn try_lock(&self) -> bool { + let mtx = self.raw(); + match unsafe { abi::ploc_mtx(mtx) } { + abi::E_TMOUT => false, + er => { + expect_success(er, &"ploc_mtx"); + true + } + } + } +} + +impl Drop for Mutex { + fn drop(&mut self) { + if let Some(mtx) = self.mtx.get().map(|x| x.0) { + expect_success_aborting(unsafe { abi::del_mtx(mtx) }, &"del_mtx"); + } + } +} + +pub(super) struct MutexGuard<'a>(&'a Mutex); + +impl<'a> MutexGuard<'a> { + #[inline] + pub(super) fn lock(x: &'a Mutex) -> Self { + x.lock(); + Self(x) + } +} + +impl Drop for MutexGuard<'_> { + #[inline] + fn drop(&mut self) { + unsafe { self.0.unlock() }; + } +} diff --git a/crux-mir/lib/std/src/sys/itron/spin.rs b/crux-mir/lib/std/src/sys/itron/spin.rs new file mode 100644 index 000000000..44d409444 --- /dev/null +++ b/crux-mir/lib/std/src/sys/itron/spin.rs @@ -0,0 +1,163 @@ +use super::abi; +use crate::{ + cell::UnsafeCell, + mem::MaybeUninit, + sync::atomic::{AtomicBool, AtomicUsize, Ordering}, +}; + +/// A mutex implemented by `dis_dsp` (for intra-core synchronization) and a +/// spinlock (for inter-core synchronization). +pub struct SpinMutex { + locked: AtomicBool, + data: UnsafeCell, +} + +impl SpinMutex { + #[inline] + pub const fn new(x: T) -> Self { + Self { locked: AtomicBool::new(false), data: UnsafeCell::new(x) } + } + + /// Acquire a lock. + #[inline] + pub fn with_locked(&self, f: impl FnOnce(&mut T) -> R) -> R { + struct SpinMutexGuard<'a>(&'a AtomicBool); + + impl Drop for SpinMutexGuard<'_> { + #[inline] + fn drop(&mut self) { + self.0.store(false, Ordering::Release); + unsafe { abi::ena_dsp() }; + } + } + + let _guard; + if unsafe { abi::sns_dsp() } == 0 { + let er = unsafe { abi::dis_dsp() }; + debug_assert!(er >= 0); + + // Wait until the current processor acquires a lock. + while self.locked.swap(true, Ordering::Acquire) {} + + _guard = SpinMutexGuard(&self.locked); + } + + f(unsafe { &mut *self.data.get() }) + } +} + +/// `OnceCell<(abi::ID, T)>` implemented by `dis_dsp` (for intra-core +/// synchronization) and a spinlock (for inter-core synchronization). +/// +/// It's assumed that `0` is not a valid ID, and all kernel +/// object IDs fall into range `1..=usize::MAX`. +pub struct SpinIdOnceCell { + id: AtomicUsize, + spin: SpinMutex<()>, + extra: UnsafeCell>, +} + +const ID_UNINIT: usize = 0; + +impl SpinIdOnceCell { + #[inline] + pub const fn new() -> Self { + Self { + id: AtomicUsize::new(ID_UNINIT), + extra: UnsafeCell::new(MaybeUninit::uninit()), + spin: SpinMutex::new(()), + } + } + + #[inline] + pub fn get(&self) -> Option<(abi::ID, &T)> { + match self.id.load(Ordering::Acquire) { + ID_UNINIT => None, + id => Some((id as abi::ID, unsafe { (&*self.extra.get()).assume_init_ref() })), + } + } + + #[inline] + pub fn get_mut(&mut self) -> Option<(abi::ID, &mut T)> { + match *self.id.get_mut() { + ID_UNINIT => None, + id => Some((id as abi::ID, unsafe { (&mut *self.extra.get()).assume_init_mut() })), + } + } + + #[inline] + pub unsafe fn get_unchecked(&self) -> (abi::ID, &T) { + (self.id.load(Ordering::Acquire) as abi::ID, unsafe { + (&*self.extra.get()).assume_init_ref() + }) + } + + /// Assign the content without checking if it's already initialized or + /// being initialized. + pub unsafe fn set_unchecked(&self, (id, extra): (abi::ID, T)) { + debug_assert!(self.get().is_none()); + + // Assumption: A positive `abi::ID` fits in `usize`. + debug_assert!(id >= 0); + debug_assert!(usize::try_from(id).is_ok()); + let id = id as usize; + + unsafe { *self.extra.get() = MaybeUninit::new(extra) }; + self.id.store(id, Ordering::Release); + } + + /// Gets the contents of the cell, initializing it with `f` if + /// the cell was empty. If the cell was empty and `f` failed, an + /// error is returned. + /// + /// Warning: `f` must not perform a blocking operation, which + /// includes panicking. + #[inline] + pub fn get_or_try_init(&self, f: F) -> Result<(abi::ID, &T), E> + where + F: FnOnce() -> Result<(abi::ID, T), E>, + { + // Fast path + if let Some(x) = self.get() { + return Ok(x); + } + + self.initialize(f)?; + + debug_assert!(self.get().is_some()); + + // Safety: The inner value has been initialized + Ok(unsafe { self.get_unchecked() }) + } + + fn initialize(&self, f: F) -> Result<(), E> + where + F: FnOnce() -> Result<(abi::ID, T), E>, + { + self.spin.with_locked(|_| { + if self.id.load(Ordering::Relaxed) == ID_UNINIT { + let (initialized_id, initialized_extra) = f()?; + + // Assumption: A positive `abi::ID` fits in `usize`. + debug_assert!(initialized_id >= 0); + debug_assert!(usize::try_from(initialized_id).is_ok()); + let initialized_id = initialized_id as usize; + + // Store the initialized contents. Use the release ordering to + // make sure the write is visible to the callers of `get`. + unsafe { *self.extra.get() = MaybeUninit::new(initialized_extra) }; + self.id.store(initialized_id, Ordering::Release); + } + Ok(()) + }) + } +} + +impl Drop for SpinIdOnceCell { + #[inline] + fn drop(&mut self) { + if self.get_mut().is_some() { + unsafe { (&mut *self.extra.get()).assume_init_drop() }; + } + } +} diff --git a/crux-mir/lib/std/src/sys/itron/task.rs b/crux-mir/lib/std/src/sys/itron/task.rs new file mode 100644 index 000000000..94beb50a2 --- /dev/null +++ b/crux-mir/lib/std/src/sys/itron/task.rs @@ -0,0 +1,44 @@ +use super::{ + abi, + error::{fail, fail_aborting, ItronError}, +}; + +use crate::mem::MaybeUninit; + +/// Get the ID of the task in Running state. Panics on failure. +#[inline] +pub fn current_task_id() -> abi::ID { + try_current_task_id().unwrap_or_else(|e| fail(e, &"get_tid")) +} + +/// Get the ID of the task in Running state. Aborts on failure. +#[inline] +pub fn current_task_id_aborting() -> abi::ID { + try_current_task_id().unwrap_or_else(|e| fail_aborting(e, &"get_tid")) +} + +/// Get the ID of the task in Running state. +#[inline] +pub fn try_current_task_id() -> Result { + unsafe { + let mut out = MaybeUninit::uninit(); + ItronError::err_if_negative(abi::get_tid(out.as_mut_ptr()))?; + Ok(out.assume_init()) + } +} + +/// Get the specified task's priority. Panics on failure. +#[inline] +pub fn task_priority(task: abi::ID) -> abi::PRI { + try_task_priority(task).unwrap_or_else(|e| fail(e, &"get_pri")) +} + +/// Get the specified task's priority. +#[inline] +pub fn try_task_priority(task: abi::ID) -> Result { + unsafe { + let mut out = MaybeUninit::uninit(); + ItronError::err_if_negative(abi::get_pri(task, out.as_mut_ptr()))?; + Ok(out.assume_init()) + } +} diff --git a/crux-mir/lib/std/src/sys/itron/thread.rs b/crux-mir/lib/std/src/sys/itron/thread.rs new file mode 100644 index 000000000..19350b83f --- /dev/null +++ b/crux-mir/lib/std/src/sys/itron/thread.rs @@ -0,0 +1,368 @@ +//! Thread implementation backed by μITRON tasks. Assumes `acre_tsk` and +//! `exd_tsk` are available. +use super::{ + abi, + error::{expect_success, expect_success_aborting, ItronError}, + task, + time::dur2reltims, +}; +use crate::{ + cell::UnsafeCell, + ffi::CStr, + hint, io, + mem::ManuallyDrop, + ptr::NonNull, + sync::atomic::{AtomicUsize, Ordering}, + sys::thread_local_dtor::run_dtors, + time::Duration, +}; + +pub struct Thread { + p_inner: NonNull, + + /// The ID of the underlying task. + task: abi::ID, +} + +// Safety: There's nothing in `Thread` that ties it to the original creator. It +// can be dropped by any threads. +unsafe impl Send for Thread {} +// Safety: `Thread` provides no methods that take `&self`. +unsafe impl Sync for Thread {} + +/// State data shared between a parent thread and child thread. It's dropped on +/// a transition to one of the final states. +struct ThreadInner { + /// This field is used on thread creation to pass a closure from + /// `Thread::new` to the created task. + start: UnsafeCell>>, + + /// A state machine. Each transition is annotated with `[...]` in the + /// source code. + /// + /// ```text + /// + ///

: parent, : child, (?): don't-care + /// + /// DETACHED (-1) --------------------> EXITED (?) + /// finish/exd_tsk + /// ^ + /// | + /// |

detach + /// | + /// + /// INIT (0) -----------------------> FINISHED (-1) + /// finish + /// | | + /// |

join/slp_tsk |

join/del_tsk + /// | |

detach/del_tsk + /// v v + /// + /// JOINING JOINED (?) + /// (parent_tid) + /// ^ + /// \ / + /// \ finish/wup_tsk /

slp_tsk-complete/ter_tsk + /// \ / & del_tsk + /// \ / + /// '--> JOIN_FINALIZE ---' + /// (-1) + /// + lifecycle: AtomicUsize, +} + +// Safety: The only `!Sync` field, `ThreadInner::start`, is only touched by +// the task represented by `ThreadInner`. +unsafe impl Sync for ThreadInner {} + +const LIFECYCLE_INIT: usize = 0; +const LIFECYCLE_FINISHED: usize = usize::MAX; +const LIFECYCLE_DETACHED: usize = usize::MAX; +const LIFECYCLE_JOIN_FINALIZE: usize = usize::MAX; +const LIFECYCLE_DETACHED_OR_JOINED: usize = usize::MAX; +const LIFECYCLE_EXITED_OR_FINISHED_OR_JOIN_FINALIZE: usize = usize::MAX; +// there's no single value for `JOINING` + +// 64KiB for 32-bit ISAs, 128KiB for 64-bit ISAs. +pub const DEFAULT_MIN_STACK_SIZE: usize = 0x4000 * crate::mem::size_of::(); + +impl Thread { + /// # Safety + /// + /// See `thread::Builder::spawn_unchecked` for safety requirements. + pub unsafe fn new(stack: usize, p: Box) -> io::Result { + let inner = Box::new(ThreadInner { + start: UnsafeCell::new(ManuallyDrop::new(p)), + lifecycle: AtomicUsize::new(LIFECYCLE_INIT), + }); + + unsafe extern "C" fn trampoline(exinf: isize) { + let p_inner: *mut ThreadInner = crate::ptr::from_exposed_addr_mut(exinf as usize); + // Safety: `ThreadInner` is alive at this point + let inner = unsafe { &*p_inner }; + + // Safety: Since `trampoline` is called only once for each + // `ThreadInner` and only `trampoline` touches `start`, + // `start` contains contents and is safe to mutably borrow. + let p = unsafe { ManuallyDrop::take(&mut *inner.start.get()) }; + p(); + + // Fix the current thread's state just in case, so that the + // destructors won't abort + // Safety: Not really unsafe + let _ = unsafe { abi::unl_cpu() }; + let _ = unsafe { abi::ena_dsp() }; + + // Run TLS destructors now because they are not + // called automatically for terminated tasks. + unsafe { run_dtors() }; + + let old_lifecycle = inner + .lifecycle + .swap(LIFECYCLE_EXITED_OR_FINISHED_OR_JOIN_FINALIZE, Ordering::AcqRel); + + match old_lifecycle { + LIFECYCLE_DETACHED => { + // [DETACHED → EXITED] + // No one will ever join, so we'll ask the collector task to + // delete the task. + + // In this case, `*p_inner`'s ownership has been moved to + // us, and we are responsible for dropping it. The acquire + // ordering ensures that the swap operation that wrote + // `LIFECYCLE_DETACHED` happens-before `Box::from_raw( + // p_inner)`. + // Safety: See above. + let _ = unsafe { Box::from_raw(p_inner) }; + + // Safety: There are no pinned references to the stack + unsafe { terminate_and_delete_current_task() }; + } + LIFECYCLE_INIT => { + // [INIT → FINISHED] + // The parent hasn't decided whether to join or detach this + // thread yet. Whichever option the parent chooses, + // it'll have to delete this task. + // Since the parent might drop `*inner` as soon as it sees + // `FINISHED`, the release ordering must be used in the + // above `swap` call. + } + parent_tid => { + // Since the parent might drop `*inner` and terminate us as + // soon as it sees `JOIN_FINALIZE`, the release ordering + // must be used in the above `swap` call. + // + // To make the task referred to by `parent_tid` visible, we + // must use the acquire ordering in the above `swap` call. + + // [JOINING → JOIN_FINALIZE] + // Wake up the parent task. + expect_success( + unsafe { + let mut er = abi::wup_tsk(parent_tid as _); + if er == abi::E_QOVR { + // `E_QOVR` indicates there's already + // a parking token + er = abi::E_OK; + } + er + }, + &"wup_tsk", + ); + } + } + } + + // Safety: `Box::into_raw` returns a non-null pointer + let p_inner = unsafe { NonNull::new_unchecked(Box::into_raw(inner)) }; + + let new_task = ItronError::err_if_negative(unsafe { + abi::acre_tsk(&abi::T_CTSK { + // Activate this task immediately + tskatr: abi::TA_ACT, + exinf: p_inner.as_ptr().expose_addr() as abi::EXINF, + // The entry point + task: Some(trampoline), + // Inherit the calling task's base priority + itskpri: abi::TPRI_SELF, + stksz: stack, + // Let the kernel allocate the stack, + stk: crate::ptr::null_mut(), + }) + }) + .map_err(|e| e.as_io_error())?; + + Ok(Self { p_inner, task: new_task }) + } + + pub fn yield_now() { + expect_success(unsafe { abi::rot_rdq(abi::TPRI_SELF) }, &"rot_rdq"); + } + + pub fn set_name(_name: &CStr) { + // nope + } + + pub fn sleep(dur: Duration) { + for timeout in dur2reltims(dur) { + expect_success(unsafe { abi::dly_tsk(timeout) }, &"dly_tsk"); + } + } + + pub fn join(self) { + // Safety: `ThreadInner` is alive at this point + let inner = unsafe { self.p_inner.as_ref() }; + // Get the current task ID. Panicking here would cause a resource leak, + // so just abort on failure. + let current_task = task::current_task_id_aborting(); + debug_assert!(usize::try_from(current_task).is_ok()); + debug_assert_ne!(current_task as usize, LIFECYCLE_INIT); + debug_assert_ne!(current_task as usize, LIFECYCLE_DETACHED); + + let current_task = current_task as usize; + + match inner.lifecycle.swap(current_task, Ordering::AcqRel) { + LIFECYCLE_INIT => { + // [INIT → JOINING] + // The child task will transition the state to `JOIN_FINALIZE` + // and wake us up. + // + // To make the task referred to by `current_task` visible from + // the child task's point of view, we must use the release + // ordering in the above `swap` call. + loop { + expect_success_aborting(unsafe { abi::slp_tsk() }, &"slp_tsk"); + // To synchronize with the child task's memory accesses to + // `inner` up to the point of the assignment of + // `JOIN_FINALIZE`, `Ordering::Acquire` must be used for the + // `load`. + if inner.lifecycle.load(Ordering::Acquire) == LIFECYCLE_JOIN_FINALIZE { + break; + } + } + + // [JOIN_FINALIZE → JOINED] + } + LIFECYCLE_FINISHED => { + // [FINISHED → JOINED] + // To synchronize with the child task's memory accesses to + // `inner` up to the point of the assignment of `FINISHED`, + // `Ordering::Acquire` must be used for the above `swap` call`. + } + _ => unsafe { hint::unreachable_unchecked() }, + } + + // Terminate and delete the task + // Safety: `self.task` still represents a task we own (because this + // method or `detach_inner` is called only once for each + // `Thread`). The task indicated that it's safe to delete by + // entering the `FINISHED` or `JOIN_FINALIZE` state. + unsafe { terminate_and_delete_task(self.task) }; + + // In either case, we are responsible for dropping `inner`. + // Safety: The contents of `*p_inner` will not be accessed hereafter + let _inner = unsafe { Box::from_raw(self.p_inner.as_ptr()) }; + + // Skip the destructor (because it would attempt to detach the thread) + crate::mem::forget(self); + } +} + +impl Drop for Thread { + fn drop(&mut self) { + // Safety: `ThreadInner` is alive at this point + let inner = unsafe { self.p_inner.as_ref() }; + + // Detach the thread. + match inner.lifecycle.swap(LIFECYCLE_DETACHED_OR_JOINED, Ordering::AcqRel) { + LIFECYCLE_INIT => { + // [INIT → DETACHED] + // When the time comes, the child will figure out that no + // one will ever join it. + // The ownership of `*p_inner` is moved to the child thread. + // The release ordering ensures that the above swap operation on + // `lifecycle` happens-before the child thread's + // `Box::from_raw(p_inner)`. + } + LIFECYCLE_FINISHED => { + // [FINISHED → JOINED] + // The task has already decided that we should delete the task. + // To synchronize with the child task's memory accesses to + // `inner` up to the point of the assignment of `FINISHED`, + // the acquire ordering is required for the above `swap` call. + + // Terminate and delete the task + // Safety: `self.task` still represents a task we own (because + // this method or `join_inner` is called only once for + // each `Thread`). The task indicated that it's safe to + // delete by entering the `FINISHED` state. + unsafe { terminate_and_delete_task(self.task) }; + + // Wwe are responsible for dropping `*p_inner`. + // Safety: The contents of `*p_inner` will not be accessed hereafter + let _ = unsafe { Box::from_raw(self.p_inner.as_ptr()) }; + } + _ => unsafe { hint::unreachable_unchecked() }, + } + } +} + +pub mod guard { + pub type Guard = !; + pub unsafe fn current() -> Option { + None + } + pub unsafe fn init() -> Option { + None + } +} + +/// Terminate and delete the specified task. +/// +/// This function will abort if `deleted_task` refers to the calling task. +/// +/// It is assumed that the specified task is solely managed by the caller - +/// i.e., other threads must not "resuscitate" the specified task or delete it +/// prematurely while this function is still in progress. It is allowed for the +/// specified task to exit by its own. +/// +/// # Safety +/// +/// The task must be safe to terminate. This is in general not true +/// because there might be pinned references to the task's stack. +unsafe fn terminate_and_delete_task(deleted_task: abi::ID) { + // Terminate the task + // Safety: Upheld by the caller + match unsafe { abi::ter_tsk(deleted_task) } { + // Indicates the task is already dormant, ignore it + abi::E_OBJ => {} + er => { + expect_success_aborting(er, &"ter_tsk"); + } + } + + // Delete the task + // Safety: Upheld by the caller + expect_success_aborting(unsafe { abi::del_tsk(deleted_task) }, &"del_tsk"); +} + +/// Terminate and delete the calling task. +/// +/// Atomicity is not required - i.e., it can be assumed that other threads won't +/// `ter_tsk` the calling task while this function is still in progress. (This +/// property makes it easy to implement this operation on μITRON-derived kernels +/// that don't support `exd_tsk`.) +/// +/// # Safety +/// +/// The task must be safe to terminate. This is in general not true +/// because there might be pinned references to the task's stack. +unsafe fn terminate_and_delete_current_task() -> ! { + expect_success_aborting(unsafe { abi::exd_tsk() }, &"exd_tsk"); + // Safety: `exd_tsk` never returns on success + unsafe { crate::hint::unreachable_unchecked() }; +} + +pub fn available_parallelism() -> io::Result { + super::unsupported() +} diff --git a/crux-mir/lib/std/src/sys/itron/time.rs b/crux-mir/lib/std/src/sys/itron/time.rs new file mode 100644 index 000000000..427ea0d80 --- /dev/null +++ b/crux-mir/lib/std/src/sys/itron/time.rs @@ -0,0 +1,114 @@ +use super::{abi, error::expect_success}; +use crate::{mem::MaybeUninit, time::Duration}; + +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] +pub struct Instant(abi::SYSTIM); + +impl Instant { + pub fn now() -> Instant { + // Safety: The provided pointer is valid + unsafe { + let mut out = MaybeUninit::uninit(); + expect_success(abi::get_tim(out.as_mut_ptr()), &"get_tim"); + Instant(out.assume_init()) + } + } + + pub fn checked_sub_instant(&self, other: &Instant) -> Option { + self.0.checked_sub(other.0).map(|ticks| { + // `SYSTIM` is measured in microseconds + Duration::from_micros(ticks) + }) + } + + pub fn checked_add_duration(&self, other: &Duration) -> Option { + // `SYSTIM` is measured in microseconds + let ticks = other.as_micros(); + + Some(Instant(self.0.checked_add(ticks.try_into().ok()?)?)) + } + + pub fn checked_sub_duration(&self, other: &Duration) -> Option { + // `SYSTIM` is measured in microseconds + let ticks = other.as_micros(); + + Some(Instant(self.0.checked_sub(ticks.try_into().ok()?)?)) + } +} + +/// Split `Duration` into zero or more `RELTIM`s. +#[inline] +pub fn dur2reltims(dur: Duration) -> impl Iterator { + // `RELTIM` is microseconds + let mut ticks = dur.as_micros(); + + crate::iter::from_fn(move || { + if ticks == 0 { + None + } else if ticks <= abi::TMAX_RELTIM as u128 { + Some(crate::mem::replace(&mut ticks, 0) as abi::RELTIM) + } else { + ticks -= abi::TMAX_RELTIM as u128; + Some(abi::TMAX_RELTIM) + } + }) +} + +/// Split `Duration` into one or more `TMO`s. +#[inline] +fn dur2tmos(dur: Duration) -> impl Iterator { + // `TMO` is microseconds + let mut ticks = dur.as_micros(); + let mut end = false; + + crate::iter::from_fn(move || { + if end { + None + } else if ticks <= abi::TMAX_RELTIM as u128 { + end = true; + Some(crate::mem::replace(&mut ticks, 0) as abi::TMO) + } else { + ticks -= abi::TMAX_RELTIM as u128; + Some(abi::TMAX_RELTIM) + } + }) +} + +/// Split `Duration` into one or more API calls with timeout. +#[inline] +pub fn with_tmos(dur: Duration, mut f: impl FnMut(abi::TMO) -> abi::ER) -> abi::ER { + let mut er = abi::E_TMOUT; + for tmo in dur2tmos(dur) { + er = f(tmo); + if er != abi::E_TMOUT { + break; + } + } + er +} + +/// Split `Duration` into one or more API calls with timeout. This function can +/// handle spurious wakeups. +#[inline] +pub fn with_tmos_strong(dur: Duration, mut f: impl FnMut(abi::TMO) -> abi::ER) -> abi::ER { + // `TMO` and `SYSTIM` are microseconds. + // Clamp at `SYSTIM::MAX` for performance reasons. This shouldn't cause + // a problem in practice. (`u64::MAX` μs ≈ 584942 years) + let ticks = dur.as_micros().min(abi::SYSTIM::MAX as u128) as abi::SYSTIM; + + let start = Instant::now().0; + let mut elapsed = 0; + let mut er = abi::E_TMOUT; + while elapsed <= ticks { + er = f(elapsed.min(abi::TMAX_RELTIM as abi::SYSTIM) as abi::TMO); + if er != abi::E_TMOUT { + break; + } + elapsed = Instant::now().0.wrapping_sub(start); + } + + er +} + +#[cfg(test)] +mod tests; diff --git a/crux-mir/lib/std/src/sys/itron/time/tests.rs b/crux-mir/lib/std/src/sys/itron/time/tests.rs new file mode 100644 index 000000000..d14035d9d --- /dev/null +++ b/crux-mir/lib/std/src/sys/itron/time/tests.rs @@ -0,0 +1,33 @@ +use super::*; + +fn reltim2dur(t: u64) -> Duration { + Duration::from_micros(t) +} + +#[test] +fn test_dur2reltims() { + assert_eq!(dur2reltims(reltim2dur(0)).collect::>(), vec![]); + assert_eq!(dur2reltims(reltim2dur(42)).collect::>(), vec![42]); + assert_eq!( + dur2reltims(reltim2dur(abi::TMAX_RELTIM as u64)).collect::>(), + vec![abi::TMAX_RELTIM] + ); + assert_eq!( + dur2reltims(reltim2dur(abi::TMAX_RELTIM as u64 + 10000)).collect::>(), + vec![abi::TMAX_RELTIM, 10000] + ); +} + +#[test] +fn test_dur2tmos() { + assert_eq!(dur2tmos(reltim2dur(0)).collect::>(), vec![0]); + assert_eq!(dur2tmos(reltim2dur(42)).collect::>(), vec![42]); + assert_eq!( + dur2tmos(reltim2dur(abi::TMAX_RELTIM as u64)).collect::>(), + vec![abi::TMAX_RELTIM] + ); + assert_eq!( + dur2tmos(reltim2dur(abi::TMAX_RELTIM as u64 + 10000)).collect::>(), + vec![abi::TMAX_RELTIM, 10000] + ); +} diff --git a/crux-mir/lib/std/src/sys/itron/wait_flag.rs b/crux-mir/lib/std/src/sys/itron/wait_flag.rs new file mode 100644 index 000000000..e432edd20 --- /dev/null +++ b/crux-mir/lib/std/src/sys/itron/wait_flag.rs @@ -0,0 +1,72 @@ +use crate::mem::MaybeUninit; +use crate::time::Duration; + +use super::{ + abi, + error::{expect_success, fail}, + time::with_tmos, +}; + +const CLEAR: abi::FLGPTN = 0; +const RAISED: abi::FLGPTN = 1; + +/// A thread parking primitive that is not susceptible to race conditions, +/// but provides no atomic ordering guarantees and allows only one `raise` per wait. +pub struct WaitFlag { + flag: abi::ID, +} + +impl WaitFlag { + /// Creates a new wait flag. + pub fn new() -> WaitFlag { + let flag = expect_success( + unsafe { + abi::acre_flg(&abi::T_CFLG { + flgatr: abi::TA_FIFO | abi::TA_WSGL | abi::TA_CLR, + iflgptn: CLEAR, + }) + }, + &"acre_flg", + ); + + WaitFlag { flag } + } + + /// Wait for the wait flag to be raised. + pub fn wait(&self) { + let mut token = MaybeUninit::uninit(); + expect_success( + unsafe { abi::wai_flg(self.flag, RAISED, abi::TWF_ORW, token.as_mut_ptr()) }, + &"wai_flg", + ); + } + + /// Wait for the wait flag to be raised or the timeout to occur. + /// + /// Returns whether the flag was raised (`true`) or the operation timed out (`false`). + pub fn wait_timeout(&self, dur: Duration) -> bool { + let mut token = MaybeUninit::uninit(); + let res = with_tmos(dur, |tmout| unsafe { + abi::twai_flg(self.flag, RAISED, abi::TWF_ORW, token.as_mut_ptr(), tmout) + }); + + match res { + abi::E_OK => true, + abi::E_TMOUT => false, + error => fail(error, &"twai_flg"), + } + } + + /// Raise the wait flag. + /// + /// Calls to this function should be balanced with the number of successful waits. + pub fn raise(&self) { + expect_success(unsafe { abi::set_flg(self.flag, RAISED) }, &"set_flg"); + } +} + +impl Drop for WaitFlag { + fn drop(&mut self) { + expect_success(unsafe { abi::del_flg(self.flag) }, &"del_flg"); + } +} diff --git a/crux-mir/lib/std/src/sys/mod.rs b/crux-mir/lib/std/src/sys/mod.rs index ec40662d5..c080c176a 100644 --- a/crux-mir/lib/std/src/sys/mod.rs +++ b/crux-mir/lib/std/src/sys/mod.rs @@ -22,92 +22,57 @@ #![allow(missing_debug_implementations)] +pub mod common; + cfg_if::cfg_if! { - if #[cfg(target_os = "vxworks")] { - mod vxworks; - pub use self::vxworks::*; - } else if #[cfg(unix)] { + if #[cfg(unix)] { mod unix; pub use self::unix::*; } else if #[cfg(windows)] { mod windows; pub use self::windows::*; - } else if #[cfg(target_os = "cloudabi")] { - mod cloudabi; - pub use self::cloudabi::*; + } else if #[cfg(target_os = "solid_asp3")] { + mod solid; + pub use self::solid::*; } else if #[cfg(target_os = "hermit")] { mod hermit; pub use self::hermit::*; } else if #[cfg(target_os = "wasi")] { mod wasi; pub use self::wasi::*; - } else if #[cfg(target_arch = "wasm32")] { + } else if #[cfg(target_family = "wasm")] { mod wasm; pub use self::wasm::*; } else if #[cfg(all(target_vendor = "fortanix", target_env = "sgx"))] { mod sgx; pub use self::sgx::*; } else { - compile_error!("libstd doesn't compile for this platform yet"); + mod unsupported; + pub use self::unsupported::*; } } -pub mod crux; - -// Import essential modules from both platforms when documenting. These are -// then later used in the `std::os` module when documenting, for example, -// Windows when we're compiling for Linux. +// Import essential modules from platforms used in `std::os` when documenting. +// +// Note that on some platforms those modules don't compile +// (missing things in `libc` which is empty), so they are not included in `std::os` and can be +// omitted here as well. #[cfg(doc)] +#[cfg(not(any( + all(target_arch = "wasm32", not(target_os = "wasi")), + all(target_vendor = "fortanix", target_env = "sgx") +)))] cfg_if::cfg_if! { - if #[cfg(unix)] { - // On unix we'll document what's already available - #[stable(feature = "rust1", since = "1.0.0")] - pub use self::ext as unix_ext; - } else if #[cfg(any(target_os = "cloudabi", - target_os = "hermit", - target_arch = "wasm32", - all(target_vendor = "fortanix", target_env = "sgx")))] { - // On CloudABI and wasm right now the module below doesn't compile - // (missing things in `libc` which is empty) so just omit everything - // with an empty module - #[unstable(issue = "none", feature = "std_internals")] - #[allow(missing_docs)] - pub mod unix_ext {} - } else { - // On other platforms like Windows document the bare bones of unix - use crate::os::linux as platform; - #[path = "unix/ext/mod.rs"] - pub mod unix_ext; - } -} - -#[cfg(doc)] -cfg_if::cfg_if! { - if #[cfg(windows)] { - // On windows we'll just be documenting what's already available - #[allow(missing_docs)] - #[stable(feature = "rust1", since = "1.0.0")] - pub use self::ext as windows_ext; - } else if #[cfg(any(target_os = "cloudabi", - target_arch = "wasm32", - all(target_vendor = "fortanix", target_env = "sgx")))] { - // On CloudABI and wasm right now the shim below doesn't compile, so - // just omit it - #[unstable(issue = "none", feature = "std_internals")] - #[allow(missing_docs)] - pub mod windows_ext {} - } else { - // On all other platforms (aka linux/osx/etc) then pull in a "minimal" + if #[cfg(not(windows))] { + // On non-Windows platforms (aka linux/osx/etc) pull in a "minimal" // amount of windows goop which ends up compiling + #[macro_use] #[path = "windows/compat.rs"] - mod compat; + pub mod compat; #[path = "windows/c.rs"] - mod c; - - #[path = "windows/ext/mod.rs"] - pub mod windows_ext; + pub mod c; } } diff --git a/crux-mir/lib/std/src/sys/sgx/abi/entry.S b/crux-mir/lib/std/src/sys/sgx/abi/entry.S index 1f06c9da3..f61bcf06f 100644 --- a/crux-mir/lib/std/src/sys/sgx/abi/entry.S +++ b/crux-mir/lib/std/src/sys/sgx/abi/entry.S @@ -11,7 +11,7 @@ IMAGE_BASE: .long 1 /* type = NT_VERSION */ 0: .asciz "toolchain-version" /* name */ 1: .align 4 -2: .long 0 /* desc - toolchain version number, 32-bit LE */ +2: .long 1 /* desc - toolchain version number, 32-bit LE */ 3: .align 4 .section .rodata @@ -26,18 +26,10 @@ IMAGE_BASE: .Lxsave_clear: .org .+24 .Lxsave_mxcsr: - .int 0 + .short 0x1f80 /* We can store a bunch of data in the gap between MXCSR and the XSAVE header */ -/* MXCSR initialization value for ABI */ -.Lmxcsr_init: - .int 0x1f80 - -/* x87 FPU control word initialization value for ABI */ -.Lfpucw_init: - .int 0x037f - /* The following symbols point at read-only data that will be filled in by the */ /* post-linker. */ @@ -68,10 +60,14 @@ IMAGE_BASE: globvar TEXT_BASE 8 /* The size in bytes of enclacve text section */ globvar TEXT_SIZE 8 - /* The base address (relative to enclave start) of the enclave EH_FRM_HDR section */ - globvar EH_FRM_HDR_BASE 8 - /* The size in bytes of enclacve EH_FRM_HDR section */ - globvar EH_FRM_HDR_SIZE 8 + /* The base address (relative to enclave start) of the enclave .eh_frame_hdr section */ + globvar EH_FRM_HDR_OFFSET 8 + /* The size in bytes of enclave .eh_frame_hdr section */ + globvar EH_FRM_HDR_LEN 8 + /* The base address (relative to enclave start) of the enclave .eh_frame section */ + globvar EH_FRM_OFFSET 8 + /* The size in bytes of enclacve .eh_frame section */ + globvar EH_FRM_LEN 8 .org .Lxsave_clear+512 .Lxsave_header: @@ -177,13 +173,17 @@ sgx_entry: jz .Lskip_debug_init mov %r10,%gs:tcsls_debug_panic_buf_ptr .Lskip_debug_init: +/* reset cpu state */ + mov %rdx, %r10 + mov $-1, %rax + mov $-1, %rdx + xrstor .Lxsave_clear(%rip) + mov %r10, %rdx + /* check if returning from usercall */ mov %gs:tcsls_last_rsp,%r11 test %r11,%r11 jnz .Lusercall_ret -/* reset user state */ - ldmxcsr .Lmxcsr_init(%rip) - fldcw .Lfpucw_init(%rip) /* setup stack */ mov %gs:tcsls_tos,%rsp /* initially, RSP is not set to the correct value */ /* here. This is fixed below under "adjust stack". */ @@ -324,7 +324,9 @@ usercall: /* return */ mov %rsi,%rax /* RAX = return value */ /* NOP: mov %rdx,%rdx */ /* RDX = return value */ - ret + pop %r11 + lfence + jmp *%r11 /* The following functions need to be defined externally: @@ -343,20 +345,28 @@ extern "C" fn entry(p1: u64, p2: u64, p3: u64, secondary: bool, p4: u64, p5: u64 .global get_tcs_addr get_tcs_addr: mov %gs:tcsls_tcs_addr,%rax - ret + pop %r11 + lfence + jmp *%r11 .global get_tls_ptr get_tls_ptr: mov %gs:tcsls_tls_ptr,%rax - ret + pop %r11 + lfence + jmp *%r11 .global set_tls_ptr set_tls_ptr: mov %rdi,%gs:tcsls_tls_ptr - ret + pop %r11 + lfence + jmp *%r11 .global take_debug_panic_buf_ptr take_debug_panic_buf_ptr: xor %rax,%rax xchg %gs:tcsls_debug_panic_buf_ptr,%rax - ret + pop %r11 + lfence + jmp *%r11 diff --git a/crux-mir/lib/std/src/sys/sgx/abi/mem.rs b/crux-mir/lib/std/src/sys/sgx/abi/mem.rs index 500e62b1c..18e6d5b3f 100644 --- a/crux-mir/lib/std/src/sys/sgx/abi/mem.rs +++ b/crux-mir/lib/std/src/sys/sgx/abi/mem.rs @@ -1,3 +1,5 @@ +use core::arch::asm; + // Do not remove inline: will result in relocation failure #[inline(always)] pub(crate) unsafe fn rel_ptr(offset: u64) -> *const T { @@ -12,6 +14,18 @@ pub(crate) unsafe fn rel_ptr_mut(offset: u64) -> *mut T { extern "C" { static ENCLAVE_SIZE: usize; + static HEAP_BASE: u64; + static HEAP_SIZE: usize; +} + +/// Returns the base memory address of the heap +pub(crate) fn heap_base() -> *const u8 { + unsafe { rel_ptr_mut(HEAP_BASE) } +} + +/// Returns the size of the heap +pub(crate) fn heap_size() -> usize { + unsafe { HEAP_SIZE } } // Do not remove inline: will result in relocation failure @@ -21,27 +35,59 @@ extern "C" { #[inline(always)] #[unstable(feature = "sgx_platform", issue = "56975")] pub fn image_base() -> u64 { - let base; - unsafe { asm!("lea IMAGE_BASE(%rip),$0":"=r"(base)) }; + let base: u64; + unsafe { + asm!( + "lea IMAGE_BASE(%rip), {}", + lateout(reg) base, + options(att_syntax, nostack, preserves_flags, nomem, pure), + ) + }; base } /// Returns `true` if the specified memory range is in the enclave. /// -/// `p + len` must not overflow. +/// For safety, this function also checks whether the range given overflows, +/// returning `false` if so. #[unstable(feature = "sgx_platform", issue = "56975")] pub fn is_enclave_range(p: *const u8, len: usize) -> bool { - let start = p as u64; - let end = start + (len as u64); - start >= image_base() && end <= image_base() + (unsafe { ENCLAVE_SIZE } as u64) // unsafe ok: link-time constant + let start = p as usize; + + // Subtract one from `len` when calculating `end` in case `p + len` is + // exactly at the end of addressable memory (`p + len` would overflow, but + // the range is still valid). + let end = if len == 0 { + start + } else if let Some(end) = start.checked_add(len - 1) { + end + } else { + return false; + }; + + let base = image_base() as usize; + start >= base && end <= base + (unsafe { ENCLAVE_SIZE } - 1) // unsafe ok: link-time constant } /// Returns `true` if the specified memory range is in userspace. /// -/// `p + len` must not overflow. +/// For safety, this function also checks whether the range given overflows, +/// returning `false` if so. #[unstable(feature = "sgx_platform", issue = "56975")] pub fn is_user_range(p: *const u8, len: usize) -> bool { - let start = p as u64; - let end = start + (len as u64); - end <= image_base() || start >= image_base() + (unsafe { ENCLAVE_SIZE } as u64) // unsafe ok: link-time constant + let start = p as usize; + + // Subtract one from `len` when calculating `end` in case `p + len` is + // exactly at the end of addressable memory (`p + len` would overflow, but + // the range is still valid). + let end = if len == 0 { + start + } else if let Some(end) = start.checked_add(len - 1) { + end + } else { + return false; + }; + + let base = image_base() as usize; + end < base || start > base + (unsafe { ENCLAVE_SIZE } - 1) // unsafe ok: link-time constant } diff --git a/crux-mir/lib/std/src/sys/sgx/abi/mod.rs b/crux-mir/lib/std/src/sys/sgx/abi/mod.rs index 87e7a5da2..9508c3874 100644 --- a/crux-mir/lib/std/src/sys/sgx/abi/mod.rs +++ b/crux-mir/lib/std/src/sys/sgx/abi/mod.rs @@ -1,6 +1,7 @@ #![cfg_attr(test, allow(unused))] // RT initialization logic is not compiled for test use crate::io::Write; +use core::arch::global_asm; use core::sync::atomic::{AtomicUsize, Ordering}; // runtime features @@ -15,7 +16,10 @@ pub mod tls; pub mod usercalls; #[cfg(not(test))] -global_asm!(include_str!("entry.S")); +global_asm!(include_str!("entry.S"), options(att_syntax)); + +#[repr(C)] +struct EntryReturn(u64, u64); #[cfg(not(test))] #[no_mangle] @@ -33,20 +37,20 @@ unsafe extern "C" fn tcs_init(secondary: bool) { } // Try to atomically swap UNINIT with BUSY. The returned state can be: - match RELOC_STATE.compare_and_swap(UNINIT, BUSY, Ordering::Acquire) { + match RELOC_STATE.compare_exchange(UNINIT, BUSY, Ordering::Acquire, Ordering::Acquire) { // This thread just obtained the lock and other threads will observe BUSY - UNINIT => { + Ok(_) => { reloc::relocate_elf_rela(); RELOC_STATE.store(DONE, Ordering::Release); } // We need to wait until the initialization is done. - BUSY => { + Err(BUSY) => { while RELOC_STATE.load(Ordering::Acquire) == BUSY { - core::arch::x86_64::_mm_pause() + core::hint::spin_loop(); } } // Initialization is done. - DONE => {} + Err(DONE) => {} _ => unreachable!(), } } @@ -56,15 +60,17 @@ unsafe extern "C" fn tcs_init(secondary: bool) { // able to specify this #[cfg(not(test))] #[no_mangle] -extern "C" fn entry(p1: u64, p2: u64, p3: u64, secondary: bool, p4: u64, p5: u64) -> (u64, u64) { +extern "C" fn entry(p1: u64, p2: u64, p3: u64, secondary: bool, p4: u64, p5: u64) -> EntryReturn { // FIXME: how to support TLS in library mode? let tls = Box::new(tls::Tls::new()); - let _tls_guard = unsafe { tls.activate() }; + let tls_guard = unsafe { tls.activate() }; if secondary { - super::thread::Thread::entry(); + let join_notifier = super::thread::Thread::entry(); + drop(tls_guard); + drop(join_notifier); - (0, 0) + EntryReturn(0, 0) } else { extern "C" { fn main(argc: isize, argv: *const *const u8) -> isize; @@ -89,7 +95,7 @@ extern "C" fn entry(p1: u64, p2: u64, p3: u64, secondary: bool, p4: u64, p5: u64 pub(super) fn exit_with_code(code: isize) -> ! { if code != 0 { if let Some(mut out) = panic::SgxPanicOutput::new() { - let _ = write!(out, "Exited with status code {}", code); + let _ = write!(out, "Exited with status code {code}"); } } usercalls::exit(code != 0); diff --git a/crux-mir/lib/std/src/sys/sgx/abi/thread.rs b/crux-mir/lib/std/src/sys/sgx/abi/thread.rs index ef55b821a..2b23e368c 100644 --- a/crux-mir/lib/std/src/sys/sgx/abi/thread.rs +++ b/crux-mir/lib/std/src/sys/sgx/abi/thread.rs @@ -7,7 +7,11 @@ use fortanix_sgx_abi::Tcs; #[unstable(feature = "sgx_platform", issue = "56975")] pub fn current() -> Tcs { extern "C" { - fn get_tcs_addr() -> Tcs; + fn get_tcs_addr() -> *mut u8; + } + let addr = unsafe { get_tcs_addr() }; + match Tcs::new(addr) { + Some(tcs) => tcs, + None => rtabort!("TCS must not be placed at address zero (this is a linker error)"), } - unsafe { get_tcs_addr() } } diff --git a/crux-mir/lib/std/src/sys/sgx/abi/tls.rs b/crux-mir/lib/std/src/sys/sgx/abi/tls.rs deleted file mode 100644 index 2b0485c4f..000000000 --- a/crux-mir/lib/std/src/sys/sgx/abi/tls.rs +++ /dev/null @@ -1,241 +0,0 @@ -use self::sync_bitset::*; -use crate::cell::Cell; -use crate::mem; -use crate::num::NonZeroUsize; -use crate::ptr; -use crate::sync::atomic::{AtomicUsize, Ordering}; - -#[cfg(target_pointer_width = "64")] -const USIZE_BITS: usize = 64; -const TLS_KEYS: usize = 128; // Same as POSIX minimum -const TLS_KEYS_BITSET_SIZE: usize = (TLS_KEYS + (USIZE_BITS - 1)) / USIZE_BITS; - -#[cfg_attr(test, linkage = "available_externally")] -#[export_name = "_ZN16__rust_internals3std3sys3sgx3abi3tls14TLS_KEY_IN_USEE"] -static TLS_KEY_IN_USE: SyncBitset = SYNC_BITSET_INIT; -macro_rules! dup { - ((* $($exp:tt)*) $($val:tt)*) => (dup!( ($($exp)*) $($val)* $($val)* )); - (() $($val:tt)*) => ([$($val),*]) -} -#[cfg_attr(test, linkage = "available_externally")] -#[export_name = "_ZN16__rust_internals3std3sys3sgx3abi3tls14TLS_DESTRUCTORE"] -static TLS_DESTRUCTOR: [AtomicUsize; TLS_KEYS] = dup!((* * * * * * *) (AtomicUsize::new(0))); - -extern "C" { - fn get_tls_ptr() -> *const u8; - fn set_tls_ptr(tls: *const u8); -} - -#[derive(Copy, Clone)] -#[repr(C)] -pub struct Key(NonZeroUsize); - -impl Key { - fn to_index(self) -> usize { - self.0.get() - 1 - } - - fn from_index(index: usize) -> Self { - Key(NonZeroUsize::new(index + 1).unwrap()) - } - - pub fn as_usize(self) -> usize { - self.0.get() - } - - pub fn from_usize(index: usize) -> Self { - Key(NonZeroUsize::new(index).unwrap()) - } -} - -#[repr(C)] -pub struct Tls { - data: [Cell<*mut u8>; TLS_KEYS], -} - -pub struct ActiveTls<'a> { - tls: &'a Tls, -} - -impl<'a> Drop for ActiveTls<'a> { - fn drop(&mut self) { - let value_with_destructor = |key: usize| { - let ptr = TLS_DESTRUCTOR[key].load(Ordering::Relaxed); - unsafe { mem::transmute::<_, Option>(ptr) } - .map(|dtor| (&self.tls.data[key], dtor)) - }; - - let mut any_non_null_dtor = true; - while any_non_null_dtor { - any_non_null_dtor = false; - for (value, dtor) in TLS_KEY_IN_USE.iter().filter_map(&value_with_destructor) { - let value = value.replace(ptr::null_mut()); - if !value.is_null() { - any_non_null_dtor = true; - unsafe { dtor(value) } - } - } - } - } -} - -impl Tls { - pub fn new() -> Tls { - Tls { data: dup!((* * * * * * *) (Cell::new(ptr::null_mut()))) } - } - - pub unsafe fn activate(&self) -> ActiveTls<'_> { - set_tls_ptr(self as *const Tls as _); - ActiveTls { tls: self } - } - - #[allow(unused)] - pub unsafe fn activate_persistent(self: Box) { - set_tls_ptr((&*self) as *const Tls as _); - mem::forget(self); - } - - unsafe fn current<'a>() -> &'a Tls { - &*(get_tls_ptr() as *const Tls) - } - - pub fn create(dtor: Option) -> Key { - let index = if let Some(index) = TLS_KEY_IN_USE.set() { - index - } else { - rtabort!("TLS limit exceeded") - }; - TLS_DESTRUCTOR[index].store(dtor.map_or(0, |f| f as usize), Ordering::Relaxed); - Key::from_index(index) - } - - pub fn set(key: Key, value: *mut u8) { - let index = key.to_index(); - rtassert!(TLS_KEY_IN_USE.get(index)); - unsafe { Self::current() }.data[index].set(value); - } - - pub fn get(key: Key) -> *mut u8 { - let index = key.to_index(); - rtassert!(TLS_KEY_IN_USE.get(index)); - unsafe { Self::current() }.data[index].get() - } - - pub fn destroy(key: Key) { - TLS_KEY_IN_USE.clear(key.to_index()); - } -} - -mod sync_bitset { - use super::{TLS_KEYS_BITSET_SIZE, USIZE_BITS}; - use crate::iter::{Enumerate, Peekable}; - use crate::slice::Iter; - use crate::sync::atomic::{AtomicUsize, Ordering}; - - /// A bitset that can be used synchronously. - pub(super) struct SyncBitset([AtomicUsize; TLS_KEYS_BITSET_SIZE]); - - pub(super) const SYNC_BITSET_INIT: SyncBitset = - SyncBitset([AtomicUsize::new(0), AtomicUsize::new(0)]); - - impl SyncBitset { - pub fn get(&self, index: usize) -> bool { - let (hi, lo) = Self::split(index); - (self.0[hi].load(Ordering::Relaxed) & lo) != 0 - } - - /// Not atomic. - pub fn iter(&self) -> SyncBitsetIter<'_> { - SyncBitsetIter { iter: self.0.iter().enumerate().peekable(), elem_idx: 0 } - } - - pub fn clear(&self, index: usize) { - let (hi, lo) = Self::split(index); - self.0[hi].fetch_and(!lo, Ordering::Relaxed); - } - - /// Sets any unset bit. Not atomic. Returns `None` if all bits were - /// observed to be set. - pub fn set(&self) -> Option { - 'elems: for (idx, elem) in self.0.iter().enumerate() { - let mut current = elem.load(Ordering::Relaxed); - loop { - if 0 == !current { - continue 'elems; - } - let trailing_ones = (!current).trailing_zeros() as usize; - match elem.compare_exchange( - current, - current | (1 << trailing_ones), - Ordering::AcqRel, - Ordering::Relaxed, - ) { - Ok(_) => return Some(idx * USIZE_BITS + trailing_ones), - Err(previous) => current = previous, - } - } - } - None - } - - fn split(index: usize) -> (usize, usize) { - (index / USIZE_BITS, 1 << (index % USIZE_BITS)) - } - } - - pub(super) struct SyncBitsetIter<'a> { - iter: Peekable>>, - elem_idx: usize, - } - - impl<'a> Iterator for SyncBitsetIter<'a> { - type Item = usize; - - fn next(&mut self) -> Option { - self.iter.peek().cloned().and_then(|(idx, elem)| { - let elem = elem.load(Ordering::Relaxed); - let low_mask = (1 << self.elem_idx) - 1; - let next = elem & !low_mask; - let next_idx = next.trailing_zeros() as usize; - self.elem_idx = next_idx + 1; - if self.elem_idx >= 64 { - self.elem_idx = 0; - self.iter.next(); - } - match next_idx { - 64 => self.next(), - _ => Some(idx * USIZE_BITS + next_idx), - } - }) - } - } - - #[cfg(test)] - mod tests { - use super::*; - - fn test_data(bitset: [usize; 2], bit_indices: &[usize]) { - let set = SyncBitset([AtomicUsize::new(bitset[0]), AtomicUsize::new(bitset[1])]); - assert_eq!(set.iter().collect::>(), bit_indices); - for &i in bit_indices { - assert!(set.get(i)); - } - } - - #[test] - fn iter() { - test_data([0b0110_1001, 0], &[0, 3, 5, 6]); - test_data([0x8000_0000_0000_0000, 0x8000_0000_0000_0001], &[63, 64, 127]); - test_data([0, 0], &[]); - } - - #[test] - fn set_get_clear() { - let set = SYNC_BITSET_INIT; - let key = set.set().unwrap(); - assert!(set.get(key)); - set.clear(key); - assert!(!set.get(key)); - } - } -} diff --git a/crux-mir/lib/std/src/sys/sgx/abi/tls/mod.rs b/crux-mir/lib/std/src/sys/sgx/abi/tls/mod.rs new file mode 100644 index 000000000..09c4ab3d3 --- /dev/null +++ b/crux-mir/lib/std/src/sys/sgx/abi/tls/mod.rs @@ -0,0 +1,133 @@ +mod sync_bitset; + +use self::sync_bitset::*; +use crate::cell::Cell; +use crate::mem; +use crate::num::NonZeroUsize; +use crate::ptr; +use crate::sync::atomic::{AtomicUsize, Ordering}; + +#[cfg(target_pointer_width = "64")] +const USIZE_BITS: usize = 64; +const TLS_KEYS: usize = 128; // Same as POSIX minimum +const TLS_KEYS_BITSET_SIZE: usize = (TLS_KEYS + (USIZE_BITS - 1)) / USIZE_BITS; + +#[cfg_attr(test, linkage = "available_externally")] +#[export_name = "_ZN16__rust_internals3std3sys3sgx3abi3tls14TLS_KEY_IN_USEE"] +static TLS_KEY_IN_USE: SyncBitset = SYNC_BITSET_INIT; +macro_rules! dup { + ((* $($exp:tt)*) $($val:tt)*) => (dup!( ($($exp)*) $($val)* $($val)* )); + (() $($val:tt)*) => ([$($val),*]) +} +#[cfg_attr(test, linkage = "available_externally")] +#[export_name = "_ZN16__rust_internals3std3sys3sgx3abi3tls14TLS_DESTRUCTORE"] +static TLS_DESTRUCTOR: [AtomicUsize; TLS_KEYS] = dup!((* * * * * * *) (AtomicUsize::new(0))); + +extern "C" { + fn get_tls_ptr() -> *const u8; + fn set_tls_ptr(tls: *const u8); +} + +#[derive(Copy, Clone)] +#[repr(C)] +pub struct Key(NonZeroUsize); + +impl Key { + fn to_index(self) -> usize { + self.0.get() - 1 + } + + fn from_index(index: usize) -> Self { + Key(NonZeroUsize::new(index + 1).unwrap()) + } + + pub fn as_usize(self) -> usize { + self.0.get() + } + + pub fn from_usize(index: usize) -> Self { + Key(NonZeroUsize::new(index).unwrap()) + } +} + +#[repr(C)] +pub struct Tls { + data: [Cell<*mut u8>; TLS_KEYS], +} + +pub struct ActiveTls<'a> { + tls: &'a Tls, +} + +impl<'a> Drop for ActiveTls<'a> { + fn drop(&mut self) { + let value_with_destructor = |key: usize| { + let ptr = TLS_DESTRUCTOR[key].load(Ordering::Relaxed); + unsafe { mem::transmute::<_, Option>(ptr) } + .map(|dtor| (&self.tls.data[key], dtor)) + }; + + let mut any_non_null_dtor = true; + while any_non_null_dtor { + any_non_null_dtor = false; + for (value, dtor) in TLS_KEY_IN_USE.iter().filter_map(&value_with_destructor) { + let value = value.replace(ptr::null_mut()); + if !value.is_null() { + any_non_null_dtor = true; + unsafe { dtor(value) } + } + } + } + } +} + +impl Tls { + pub fn new() -> Tls { + Tls { data: dup!((* * * * * * *) (Cell::new(ptr::null_mut()))) } + } + + pub unsafe fn activate(&self) -> ActiveTls<'_> { + // FIXME: Needs safety information. See entry.S for `set_tls_ptr` definition. + unsafe { set_tls_ptr(self as *const Tls as _) }; + ActiveTls { tls: self } + } + + #[allow(unused)] + pub unsafe fn activate_persistent(self: Box) { + // FIXME: Needs safety information. See entry.S for `set_tls_ptr` definition. + unsafe { set_tls_ptr((&*self) as *const Tls as _) }; + mem::forget(self); + } + + unsafe fn current<'a>() -> &'a Tls { + // FIXME: Needs safety information. See entry.S for `set_tls_ptr` definition. + unsafe { &*(get_tls_ptr() as *const Tls) } + } + + pub fn create(dtor: Option) -> Key { + let index = if let Some(index) = TLS_KEY_IN_USE.set() { + index + } else { + rtabort!("TLS limit exceeded") + }; + TLS_DESTRUCTOR[index].store(dtor.map_or(0, |f| f as usize), Ordering::Relaxed); + unsafe { Self::current() }.data[index].set(ptr::null_mut()); + Key::from_index(index) + } + + pub fn set(key: Key, value: *mut u8) { + let index = key.to_index(); + rtassert!(TLS_KEY_IN_USE.get(index)); + unsafe { Self::current() }.data[index].set(value); + } + + pub fn get(key: Key) -> *mut u8 { + let index = key.to_index(); + rtassert!(TLS_KEY_IN_USE.get(index)); + unsafe { Self::current() }.data[index].get() + } + + pub fn destroy(key: Key) { + TLS_KEY_IN_USE.clear(key.to_index()); + } +} diff --git a/crux-mir/lib/std/src/sys/sgx/abi/tls/sync_bitset.rs b/crux-mir/lib/std/src/sys/sgx/abi/tls/sync_bitset.rs new file mode 100644 index 000000000..4eeff8f6e --- /dev/null +++ b/crux-mir/lib/std/src/sys/sgx/abi/tls/sync_bitset.rs @@ -0,0 +1,85 @@ +#[cfg(test)] +mod tests; + +use super::{TLS_KEYS_BITSET_SIZE, USIZE_BITS}; +use crate::iter::{Enumerate, Peekable}; +use crate::slice::Iter; +use crate::sync::atomic::{AtomicUsize, Ordering}; + +/// A bitset that can be used synchronously. +pub(super) struct SyncBitset([AtomicUsize; TLS_KEYS_BITSET_SIZE]); + +pub(super) const SYNC_BITSET_INIT: SyncBitset = + SyncBitset([AtomicUsize::new(0), AtomicUsize::new(0)]); + +impl SyncBitset { + pub fn get(&self, index: usize) -> bool { + let (hi, lo) = Self::split(index); + (self.0[hi].load(Ordering::Relaxed) & lo) != 0 + } + + /// Not atomic. + pub fn iter(&self) -> SyncBitsetIter<'_> { + SyncBitsetIter { iter: self.0.iter().enumerate().peekable(), elem_idx: 0 } + } + + pub fn clear(&self, index: usize) { + let (hi, lo) = Self::split(index); + self.0[hi].fetch_and(!lo, Ordering::Relaxed); + } + + /// Sets any unset bit. Not atomic. Returns `None` if all bits were + /// observed to be set. + pub fn set(&self) -> Option { + 'elems: for (idx, elem) in self.0.iter().enumerate() { + let mut current = elem.load(Ordering::Relaxed); + loop { + if 0 == !current { + continue 'elems; + } + let trailing_ones = (!current).trailing_zeros() as usize; + match elem.compare_exchange( + current, + current | (1 << trailing_ones), + Ordering::AcqRel, + Ordering::Relaxed, + ) { + Ok(_) => return Some(idx * USIZE_BITS + trailing_ones), + Err(previous) => current = previous, + } + } + } + None + } + + fn split(index: usize) -> (usize, usize) { + (index / USIZE_BITS, 1 << (index % USIZE_BITS)) + } +} + +pub(super) struct SyncBitsetIter<'a> { + iter: Peekable>>, + elem_idx: usize, +} + +impl<'a> Iterator for SyncBitsetIter<'a> { + type Item = usize; + + fn next(&mut self) -> Option { + self.iter.peek().cloned().and_then(|(idx, elem)| { + let elem = elem.load(Ordering::Relaxed); + let low_mask = (1 << self.elem_idx) - 1; + let next = elem & !low_mask; + let next_idx = next.trailing_zeros() as usize; + self.elem_idx = next_idx + 1; + if self.elem_idx >= 64 { + self.elem_idx = 0; + self.iter.next(); + } + match next_idx { + 64 => self.next(), + _ => Some(idx * USIZE_BITS + next_idx), + } + }) + } +} diff --git a/crux-mir/lib/std/src/sys/sgx/abi/tls/sync_bitset/tests.rs b/crux-mir/lib/std/src/sys/sgx/abi/tls/sync_bitset/tests.rs new file mode 100644 index 000000000..d7eb2e139 --- /dev/null +++ b/crux-mir/lib/std/src/sys/sgx/abi/tls/sync_bitset/tests.rs @@ -0,0 +1,25 @@ +use super::*; + +fn test_data(bitset: [usize; 2], bit_indices: &[usize]) { + let set = SyncBitset([AtomicUsize::new(bitset[0]), AtomicUsize::new(bitset[1])]); + assert_eq!(set.iter().collect::>(), bit_indices); + for &i in bit_indices { + assert!(set.get(i)); + } +} + +#[test] +fn iter() { + test_data([0b0110_1001, 0], &[0, 3, 5, 6]); + test_data([0x8000_0000_0000_0000, 0x8000_0000_0000_0001], &[63, 64, 127]); + test_data([0, 0], &[]); +} + +#[test] +fn set_get_clear() { + let set = SYNC_BITSET_INIT; + let key = set.set().unwrap(); + assert!(set.get(key)); + set.clear(key); + assert!(!set.get(key)); +} diff --git a/crux-mir/lib/std/src/sys/sgx/abi/usercalls/alloc.rs b/crux-mir/lib/std/src/sys/sgx/abi/usercalls/alloc.rs index 76a9b427b..0d934318c 100644 --- a/crux-mir/lib/std/src/sys/sgx/abi/usercalls/alloc.rs +++ b/crux-mir/lib/std/src/sys/sgx/abi/usercalls/alloc.rs @@ -1,13 +1,16 @@ #![allow(unused)] +use crate::arch::asm; use crate::cell::UnsafeCell; +use crate::cmp; +use crate::convert::TryInto; use crate::mem; use crate::ops::{CoerceUnsized, Deref, DerefMut, Index, IndexMut}; use crate::ptr::{self, NonNull}; use crate::slice; use crate::slice::SliceIndex; -use super::super::mem::is_user_range; +use super::super::mem::{is_enclave_range, is_user_range}; use fortanix_sgx_abi::*; /// A type that can be safely read from or written to userspace. @@ -53,6 +56,8 @@ unsafe impl UserSafeSized for Usercall {} #[unstable(feature = "sgx_platform", issue = "56975")] unsafe impl UserSafeSized for Return {} #[unstable(feature = "sgx_platform", issue = "56975")] +unsafe impl UserSafeSized for Cancel {} +#[unstable(feature = "sgx_platform", issue = "56975")] unsafe impl UserSafeSized for [T; 2] {} /// A type that can be represented in memory as one or more `UserSafeSized`s. @@ -89,9 +94,12 @@ pub unsafe trait UserSafe { /// * the pointed-to range is not in user memory. unsafe fn from_raw_sized(ptr: *mut u8, size: usize) -> NonNull { assert!(ptr.wrapping_add(size) >= ptr); - let ret = Self::from_raw_sized_unchecked(ptr, size); - Self::check_ptr(ret); - NonNull::new_unchecked(ret as _) + // SAFETY: The caller has guaranteed the pointer is valid + let ret = unsafe { Self::from_raw_sized_unchecked(ptr, size) }; + unsafe { + Self::check_ptr(ret); + NonNull::new_unchecked(ret as _) + } } /// Checks if a pointer may point to `Self` in user memory. @@ -109,10 +117,10 @@ pub unsafe trait UserSafe { /// * the pointer is null. /// * the pointed-to range is not in user memory. unsafe fn check_ptr(ptr: *const Self) { - let is_aligned = |p| -> bool { 0 == (p as usize) & (Self::align_of() - 1) }; + let is_aligned = |p: *const u8| -> bool { p.is_aligned_to(Self::align_of()) }; assert!(is_aligned(ptr as *const u8)); - assert!(is_user_range(ptr as _, mem::size_of_val(&*ptr))); + assert!(is_user_range(ptr as _, mem::size_of_val(unsafe { &*ptr }))); assert!(!ptr.is_null()); } } @@ -135,11 +143,23 @@ unsafe impl UserSafe for [T] { mem::align_of::() } + /// # Safety + /// Behavior is undefined if any of these conditions are violated: + /// * `ptr` must be [valid] for writes of `size` many bytes, and it must be + /// properly aligned. + /// + /// [valid]: core::ptr#safety + /// # Panics + /// + /// This function panics if: + /// + /// * the element size is not a factor of the size unsafe fn from_raw_sized_unchecked(ptr: *mut u8, size: usize) -> *mut Self { let elem_size = mem::size_of::(); assert_eq!(size % elem_size, 0); let len = size / elem_size; - slice::from_raw_parts_mut(ptr as _, len) + // SAFETY: The caller must uphold the safety contract for `from_raw_sized_unchecked` + unsafe { slice::from_raw_parts_mut(ptr as _, len) } } } @@ -170,13 +190,15 @@ trait NewUserRef { impl NewUserRef<*mut T> for NonNull> { unsafe fn new_userref(v: *mut T) -> Self { - NonNull::new_unchecked(v as _) + // SAFETY: The caller has guaranteed the pointer is valid + unsafe { NonNull::new_unchecked(v as _) } } } impl NewUserRef> for NonNull> { unsafe fn new_userref(v: NonNull) -> Self { - NonNull::new_userref(v.as_ptr()) + // SAFETY: The caller has guaranteed the pointer is valid + unsafe { NonNull::new_userref(v.as_ptr()) } } } @@ -193,7 +215,9 @@ where unsafe { // Mustn't call alloc with size 0. let ptr = if size > 0 { - rtunwrap!(Ok, super::alloc(size, T::align_of())) as _ + // `copy_to_userspace` is more efficient when data is 8-byte aligned + let alignment = cmp::max(T::align_of(), 8); + rtunwrap!(Ok, super::alloc(size, alignment)) as _ } else { T::align_of() as _ // dangling pointer ok for size 0 }; @@ -208,13 +232,9 @@ where /// Copies `val` into freshly allocated space in user memory. pub fn new_from_enclave(val: &T) -> Self { unsafe { - let ret = Self::new_uninit_bytes(mem::size_of_val(val)); - ptr::copy( - val as *const T as *const u8, - ret.0.as_ptr() as *mut u8, - mem::size_of_val(val), - ); - ret + let mut user = Self::new_uninit_bytes(mem::size_of_val(val)); + user.copy_from_enclave(val); + user } } @@ -231,8 +251,9 @@ where /// * The pointer is null /// * The pointed-to range is not in user memory pub unsafe fn from_raw(ptr: *mut T) -> Self { - T::check_ptr(ptr); - User(NonNull::new_userref(ptr)) + // SAFETY: the caller must uphold the safety contract for `from_raw`. + unsafe { T::check_ptr(ptr) }; + User(unsafe { NonNull::new_userref(ptr) }) } /// Converts this value into a raw pointer. The value will no longer be @@ -280,7 +301,221 @@ where /// * The pointed-to range does not fit in the address space /// * The pointed-to range is not in user memory pub unsafe fn from_raw_parts(ptr: *mut T, len: usize) -> Self { - User(NonNull::new_userref(<[T]>::from_raw_sized(ptr as _, len * mem::size_of::()))) + User(unsafe { + NonNull::new_userref(<[T]>::from_raw_sized(ptr as _, len * mem::size_of::())) + }) + } +} + +// Split a memory region ptr..ptr + len into three parts: +// +--------+ +// | small0 | Chunk smaller than 8 bytes +// +--------+ +// | big | Chunk 8-byte aligned, and size a multiple of 8 bytes +// +--------+ +// | small1 | Chunk smaller than 8 bytes +// +--------+ +fn region_as_aligned_chunks(ptr: *const u8, len: usize) -> (usize, usize, usize) { + let small0_size = if ptr.is_aligned_to(8) { 0 } else { 8 - ptr.addr() % 8 }; + let small1_size = (len - small0_size) % 8; + let big_size = len - small0_size - small1_size; + + (small0_size, big_size, small1_size) +} + +unsafe fn copy_quadwords(src: *const u8, dst: *mut u8, len: usize) { + unsafe { + asm!( + "rep movsq (%rsi), (%rdi)", + inout("rcx") len / 8 => _, + inout("rdi") dst => _, + inout("rsi") src => _, + options(att_syntax, nostack, preserves_flags) + ); + } +} + +/// Copies `len` bytes of data from enclave pointer `src` to userspace `dst` +/// +/// This function mitigates stale data vulnerabilities by ensuring all writes to untrusted memory are either: +/// - preceded by the VERW instruction and followed by the MFENCE; LFENCE instruction sequence +/// - or are in multiples of 8 bytes, aligned to an 8-byte boundary +/// +/// # Panics +/// This function panics if: +/// +/// * The `src` pointer is null +/// * The `dst` pointer is null +/// * The `src` memory range is not in enclave memory +/// * The `dst` memory range is not in user memory +/// +/// # References +/// - https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00615.html +/// - https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/technical-documentation/processor-mmio-stale-data-vulnerabilities.html#inpage-nav-3-2-2 +pub(crate) unsafe fn copy_to_userspace(src: *const u8, dst: *mut u8, len: usize) { + unsafe fn copy_bytewise_to_userspace(src: *const u8, dst: *mut u8, len: usize) { + unsafe { + let mut seg_sel: u16 = 0; + for off in 0..len { + asm!(" + mov %ds, ({seg_sel}) + verw ({seg_sel}) + movb {val}, ({dst}) + mfence + lfence + ", + val = in(reg_byte) *src.add(off), + dst = in(reg) dst.add(off), + seg_sel = in(reg) &mut seg_sel, + options(nostack, att_syntax) + ); + } + } + } + + assert!(!src.is_null()); + assert!(!dst.is_null()); + assert!(is_enclave_range(src, len)); + assert!(is_user_range(dst, len)); + assert!(len < isize::MAX as usize); + assert!(!src.addr().overflowing_add(len).1); + assert!(!dst.addr().overflowing_add(len).1); + + if len < 8 { + // Can't align on 8 byte boundary: copy safely byte per byte + unsafe { + copy_bytewise_to_userspace(src, dst, len); + } + } else if len % 8 == 0 && dst.is_aligned_to(8) { + // Copying 8-byte aligned quadwords: copy quad word per quad word + unsafe { + copy_quadwords(src, dst, len); + } + } else { + // Split copies into three parts: + // +--------+ + // | small0 | Chunk smaller than 8 bytes + // +--------+ + // | big | Chunk 8-byte aligned, and size a multiple of 8 bytes + // +--------+ + // | small1 | Chunk smaller than 8 bytes + // +--------+ + let (small0_size, big_size, small1_size) = region_as_aligned_chunks(dst, len); + + unsafe { + // Copy small0 + copy_bytewise_to_userspace(src, dst, small0_size); + + // Copy big + let big_src = src.add(small0_size); + let big_dst = dst.add(small0_size); + copy_quadwords(big_src, big_dst, big_size); + + // Copy small1 + let small1_src = src.add(big_size + small0_size); + let small1_dst = dst.add(big_size + small0_size); + copy_bytewise_to_userspace(small1_src, small1_dst, small1_size); + } + } +} + +/// Copies `len` bytes of data from userspace pointer `src` to enclave pointer `dst` +/// +/// This function mitigates AEPIC leak vulnerabilities by ensuring all reads from untrusted memory are 8-byte aligned +/// +/// # Panics +/// This function panics if: +/// +/// * The `src` pointer is null +/// * The `dst` pointer is null +/// * The `src` memory range is not in user memory +/// * The `dst` memory range is not in enclave memory +/// +/// # References +/// - https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00657.html +/// - https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/advisory-guidance/stale-data-read-from-xapic.html +pub(crate) unsafe fn copy_from_userspace(src: *const u8, dst: *mut u8, len: usize) { + // Copies memory region `src..src + len` to the enclave at `dst`. The source memory region + // is: + // - strictly less than 8 bytes in size and may be + // - located at a misaligned memory location + fn copy_misaligned_chunk_to_enclave(src: *const u8, dst: *mut u8, len: usize) { + let mut tmp_buff = [0u8; 16]; + + unsafe { + // Compute an aligned memory region to read from + // +--------+ <-- aligned_src + aligned_len (8B-aligned) + // | pad1 | + // +--------+ <-- src + len (misaligned) + // | | + // | | + // | | + // +--------+ <-- src (misaligned) + // | pad0 | + // +--------+ <-- aligned_src (8B-aligned) + let pad0_size = src as usize % 8; + let aligned_src = src.sub(pad0_size); + + let pad1_size = 8 - (src.add(len) as usize % 8); + let aligned_len = pad0_size + len + pad1_size; + + debug_assert!(len < 8); + debug_assert_eq!(aligned_src as usize % 8, 0); + debug_assert_eq!(aligned_len % 8, 0); + debug_assert!(aligned_len <= 16); + + // Copy the aligned buffer to a temporary buffer + // Note: copying from a slightly different memory location is a bit odd. In this case it + // can't lead to page faults or inadvertent copying from the enclave as we only ensured + // that the `src` pointer is aligned at an 8 byte boundary. As pages are 4096 bytes + // aligned, `aligned_src` must be on the same page as `src`. A similar argument can be made + // for `src + len` + copy_quadwords(aligned_src as _, tmp_buff.as_mut_ptr(), aligned_len); + + // Copy the correct parts of the temporary buffer to the destination + ptr::copy(tmp_buff.as_ptr().add(pad0_size), dst, len); + } + } + + assert!(!src.is_null()); + assert!(!dst.is_null()); + assert!(is_user_range(src, len)); + assert!(is_enclave_range(dst, len)); + assert!(!(src as usize).overflowing_add(len + 8).1); + assert!(!(dst as usize).overflowing_add(len + 8).1); + + if len < 8 { + copy_misaligned_chunk_to_enclave(src, dst, len); + } else if len % 8 == 0 && src as usize % 8 == 0 { + // Copying 8-byte aligned quadwords: copy quad word per quad word + unsafe { + copy_quadwords(src, dst, len); + } + } else { + // Split copies into three parts: + // +--------+ + // | small0 | Chunk smaller than 8 bytes + // +--------+ + // | big | Chunk 8-byte aligned, and size a multiple of 8 bytes + // +--------+ + // | small1 | Chunk smaller than 8 bytes + // +--------+ + let (small0_size, big_size, small1_size) = region_as_aligned_chunks(dst, len); + + unsafe { + // Copy small0 + copy_misaligned_chunk_to_enclave(src, dst, small0_size); + + // Copy big + let big_src = src.add(small0_size); + let big_dst = dst.add(small0_size); + copy_quadwords(big_src, big_dst, big_size); + + // Copy small1 + let small1_src = src.add(big_size + small0_size); + let small1_dst = dst.add(big_size + small0_size); + copy_misaligned_chunk_to_enclave(small1_src, small1_dst, small1_size); + } } } @@ -301,8 +536,9 @@ where /// * The pointer is null /// * The pointed-to range is not in user memory pub unsafe fn from_ptr<'a>(ptr: *const T) -> &'a Self { - T::check_ptr(ptr); - &*(ptr as *const Self) + // SAFETY: The caller must uphold the safety contract for `from_ptr`. + unsafe { T::check_ptr(ptr) }; + unsafe { &*(ptr as *const Self) } } /// Creates a `&mut UserRef<[T]>` from a raw pointer. See the struct @@ -318,8 +554,9 @@ where /// * The pointer is null /// * The pointed-to range is not in user memory pub unsafe fn from_mut_ptr<'a>(ptr: *mut T) -> &'a mut Self { - T::check_ptr(ptr); - &mut *(ptr as *mut Self) + // SAFETY: The caller must uphold the safety contract for `from_mut_ptr`. + unsafe { T::check_ptr(ptr) }; + unsafe { &mut *(ptr as *mut Self) } } /// Copies `val` into user memory. @@ -330,7 +567,7 @@ where pub fn copy_from_enclave(&mut self, val: &T) { unsafe { assert_eq!(mem::size_of_val(val), mem::size_of_val(&*self.0.get())); - ptr::copy( + copy_to_userspace( val as *const T as *const u8, self.0.get() as *mut T as *mut u8, mem::size_of_val(val), @@ -346,7 +583,7 @@ where pub fn copy_to_enclave(&self, dest: &mut T) { unsafe { assert_eq!(mem::size_of_val(dest), mem::size_of_val(&*self.0.get())); - ptr::copy( + copy_from_userspace( self.0.get() as *const T as *const u8, dest as *mut T as *mut u8, mem::size_of_val(dest), @@ -372,7 +609,11 @@ where { /// Copies the value from user memory into enclave memory. pub fn to_enclave(&self) -> T { - unsafe { ptr::read(self.0.get()) } + unsafe { + let mut data: T = mem::MaybeUninit::uninit().assume_init(); + copy_from_userspace(self.0.get() as _, &mut data as *mut T as _, mem::size_of::()); + data + } } } @@ -394,7 +635,10 @@ where /// * The pointed-to range does not fit in the address space /// * The pointed-to range is not in user memory pub unsafe fn from_raw_parts<'a>(ptr: *const T, len: usize) -> &'a Self { - &*(<[T]>::from_raw_sized(ptr as _, len * mem::size_of::()).as_ptr() as *const Self) + // SAFETY: The caller must uphold the safety contract for `from_raw_parts`. + unsafe { + &*(<[T]>::from_raw_sized(ptr as _, len * mem::size_of::()).as_ptr() as *const Self) + } } /// Creates a `&mut UserRef<[T]>` from a raw thin pointer and a slice length. @@ -412,7 +656,10 @@ where /// * The pointed-to range does not fit in the address space /// * The pointed-to range is not in user memory pub unsafe fn from_raw_parts_mut<'a>(ptr: *mut T, len: usize) -> &'a mut Self { - &mut *(<[T]>::from_raw_sized(ptr as _, len * mem::size_of::()).as_ptr() as *mut Self) + // SAFETY: The caller must uphold the safety contract for `from_raw_parts_mut`. + unsafe { + &mut *(<[T]>::from_raw_sized(ptr as _, len * mem::size_of::()).as_ptr() as *mut Self) + } } /// Obtain a raw pointer to the first element of this user slice. @@ -437,13 +684,12 @@ where /// This function panics if the destination doesn't have the same size as /// the source. This can happen for dynamically-sized types such as slices. pub fn copy_to_enclave_vec(&self, dest: &mut Vec) { - unsafe { - if let Some(missing) = self.len().checked_sub(dest.capacity()) { - dest.reserve(missing) - } - dest.set_len(self.len()); - self.copy_to_enclave(&mut dest[..]); + if let Some(missing) = self.len().checked_sub(dest.capacity()) { + dest.reserve(missing) } + // SAFETY: We reserve enough space above. + unsafe { dest.set_len(self.len()) }; + self.copy_to_enclave(&mut dest[..]); } /// Copies the value from user memory into a vector in enclave memory. @@ -544,7 +790,8 @@ impl, U> CoerceUnsized> for UserRef {} impl Index for UserRef<[T]> where [T]: UserSafe, - I: SliceIndex<[T], Output: UserSafe>, + I: SliceIndex<[T]>, + I::Output: UserSafe, { type Output = UserRef; @@ -564,7 +811,8 @@ where impl IndexMut for UserRef<[T]> where [T]: UserSafe, - I: SliceIndex<[T], Output: UserSafe>, + I: SliceIndex<[T]>, + I::Output: UserSafe, { #[inline] fn index_mut(&mut self, index: I) -> &mut UserRef { diff --git a/crux-mir/lib/std/src/sys/sgx/abi/usercalls/mod.rs b/crux-mir/lib/std/src/sys/sgx/abi/usercalls/mod.rs index ae803ee47..e19e84326 100644 --- a/crux-mir/lib/std/src/sys/sgx/abi/usercalls/mod.rs +++ b/crux-mir/lib/std/src/sys/sgx/abi/usercalls/mod.rs @@ -1,10 +1,13 @@ use crate::cmp; -use crate::io::{Error as IoError, IoSlice, IoSliceMut, Result as IoResult}; -use crate::time::Duration; +use crate::io::{Error as IoError, ErrorKind, IoSlice, IoSliceMut, Result as IoResult}; +use crate::sys::rand::rdrand64; +use crate::time::{Duration, Instant}; pub(crate) mod alloc; #[macro_use] pub(crate) mod raw; +#[cfg(test)] +mod tests; use self::raw::*; @@ -81,7 +84,7 @@ pub fn close(fd: Fd) { fn string_from_bytebuffer(buf: &alloc::UserRef, usercall: &str, arg: &str) -> String { String::from_utf8(buf.copy_user_buffer()) - .unwrap_or_else(|_| rtabort!("Usercall {}: expected {} to be valid UTF-8", usercall, arg)) + .unwrap_or_else(|_| rtabort!("Usercall {usercall}: expected {arg} to be valid UTF-8")) } /// Usercall `bind_stream`. See the ABI documentation for more information. @@ -138,7 +141,8 @@ pub fn connect_stream(addr: &str) -> IoResult<(Fd, String, String)> { /// Usercall `launch_thread`. See the ABI documentation for more information. #[unstable(feature = "sgx_platform", issue = "56975")] pub unsafe fn launch_thread() -> IoResult<()> { - raw::launch_thread().from_sgx_result() + // SAFETY: The caller must uphold the safety contract for `launch_thread`. + unsafe { raw::launch_thread().from_sgx_result() } } /// Usercall `exit`. See the ABI documentation for more information. @@ -149,10 +153,94 @@ pub fn exit(panic: bool) -> ! { /// Usercall `wait`. See the ABI documentation for more information. #[unstable(feature = "sgx_platform", issue = "56975")] -pub fn wait(event_mask: u64, timeout: u64) -> IoResult { +pub fn wait(event_mask: u64, mut timeout: u64) -> IoResult { + if timeout != WAIT_NO && timeout != WAIT_INDEFINITE { + // We don't want people to rely on accuracy of timeouts to make + // security decisions in an SGX enclave. That's why we add a random + // amount not exceeding +/- 10% to the timeout value to discourage + // people from relying on accuracy of timeouts while providing a way + // to make things work in other cases. Note that in the SGX threat + // model the enclave runner which is serving the wait usercall is not + // trusted to ensure accurate timeouts. + if let Ok(timeout_signed) = i64::try_from(timeout) { + let tenth = timeout_signed / 10; + let deviation = (rdrand64() as i64).checked_rem(tenth).unwrap_or(0); + timeout = timeout_signed.saturating_add(deviation) as _; + } + } unsafe { raw::wait(event_mask, timeout).from_sgx_result() } } +/// This function makes an effort to wait for a non-spurious event at least as +/// long as `duration`. Note that in general there is no guarantee about accuracy +/// of time and timeouts in SGX model. The enclave runner serving usercalls may +/// lie about current time and/or ignore timeout values. +/// +/// Once the event is observed, `should_wake_up` will be used to determine +/// whether or not the event was spurious. +#[unstable(feature = "sgx_platform", issue = "56975")] +pub fn wait_timeout(event_mask: u64, duration: Duration, should_wake_up: F) +where + F: Fn() -> bool, +{ + // Calls the wait usercall and checks the result. Returns true if event was + // returned, and false if WouldBlock/TimedOut was returned. + // If duration is None, it will use WAIT_NO. + fn wait_checked(event_mask: u64, duration: Option) -> bool { + let timeout = duration.map_or(raw::WAIT_NO, |duration| { + cmp::min((u64::MAX - 1) as u128, duration.as_nanos()) as u64 + }); + match wait(event_mask, timeout) { + Ok(eventset) => { + if event_mask == 0 { + rtabort!("expected wait() to return Err, found Ok."); + } + rtassert!(eventset != 0 && eventset & !event_mask == 0); + true + } + Err(e) => { + rtassert!(e.kind() == ErrorKind::TimedOut || e.kind() == ErrorKind::WouldBlock); + false + } + } + } + + match wait_checked(event_mask, Some(duration)) { + false => return, // timed out + true if should_wake_up() => return, // woken up + true => {} // spurious event + } + + // Drain all cached events. + // Note that `event_mask != 0` is implied if we get here. + loop { + match wait_checked(event_mask, None) { + false => break, // no more cached events + true if should_wake_up() => return, // woken up + true => {} // spurious event + } + } + + // Continue waiting, but take note of time spent waiting so we don't wait + // forever. We intentionally don't call `Instant::now()` before this point + // to avoid the cost of the `insecure_time` usercall in case there are no + // spurious wakeups. + + let start = Instant::now(); + let mut remaining = duration; + loop { + match wait_checked(event_mask, Some(remaining)) { + false => return, // timed out + true if should_wake_up() => return, // woken up + true => {} // spurious event + } + remaining = match duration.checked_sub(start.elapsed()) { + Some(remaining) => remaining, + None => break, + } + } +} + /// Usercall `send`. See the ABI documentation for more information. #[unstable(feature = "sgx_platform", issue = "56975")] pub fn send(event_set: u64, tcs: Option) -> IoResult<()> { @@ -200,16 +288,21 @@ fn check_os_error(err: Result) -> i32 { { err } else { - rtabort!("Usercall: returned invalid error value {}", err) + rtabort!("Usercall: returned invalid error value {err}") } } -trait FromSgxResult { +/// Translate the raw result of an SGX usercall. +#[unstable(feature = "sgx_platform", issue = "56975")] +pub trait FromSgxResult { + /// Return type type Return; + /// Translate the raw result of an SGX usercall. fn from_sgx_result(self) -> IoResult; } +#[unstable(feature = "sgx_platform", issue = "56975")] impl FromSgxResult for (Result, T) { type Return = T; @@ -222,6 +315,7 @@ impl FromSgxResult for (Result, T) { } } +#[unstable(feature = "sgx_platform", issue = "56975")] impl FromSgxResult for Result { type Return = (); diff --git a/crux-mir/lib/std/src/sys/sgx/abi/usercalls/raw.rs b/crux-mir/lib/std/src/sys/sgx/abi/usercalls/raw.rs index e0ebf8606..10c1456d4 100644 --- a/crux-mir/lib/std/src/sys/sgx/abi/usercalls/raw.rs +++ b/crux-mir/lib/std/src/sys/sgx/abi/usercalls/raw.rs @@ -33,18 +33,27 @@ pub unsafe fn do_usercall( p4: u64, abort: bool, ) -> (u64, u64) { - let UsercallReturn(a, b) = usercall(nr, p1, p2, abort as _, p3, p4); + let UsercallReturn(a, b) = unsafe { usercall(nr, p1, p2, abort as _, p3, p4) }; (a, b) } -type Register = u64; +/// A value passed or returned in a CPU register. +#[unstable(feature = "sgx_platform", issue = "56975")] +pub type Register = u64; -trait RegisterArgument { +/// Translate a type from/to Register to be used as an argument. +#[unstable(feature = "sgx_platform", issue = "56975")] +pub trait RegisterArgument { + /// Translate a Register to Self. fn from_register(_: Register) -> Self; + /// Translate self to a Register. fn into_register(self) -> Register; } -trait ReturnValue { +/// Translate a pair of Registers to the raw usercall return value. +#[unstable(feature = "sgx_platform", issue = "56975")] +pub trait ReturnValue { + /// Translate a pair of Registers to the raw usercall return value. fn from_registers(call: &'static str, regs: (Register, Register)) -> Self; } @@ -68,6 +77,7 @@ macro_rules! define_usercalls { macro_rules! define_ra { (< $i:ident > $t:ty) => { + #[unstable(feature = "sgx_platform", issue = "56975")] impl<$i> RegisterArgument for $t { fn from_register(a: Register) -> Self { a as _ @@ -78,6 +88,7 @@ macro_rules! define_ra { } }; ($i:ty as $t:ty) => { + #[unstable(feature = "sgx_platform", issue = "56975")] impl RegisterArgument for $t { fn from_register(a: Register) -> Self { a as $i as _ @@ -88,6 +99,7 @@ macro_rules! define_ra { } }; ($t:ty) => { + #[unstable(feature = "sgx_platform", issue = "56975")] impl RegisterArgument for $t { fn from_register(a: Register) -> Self { a as _ @@ -112,6 +124,7 @@ define_ra!(usize as isize); define_ra!( *const T); define_ra!( *mut T); +#[unstable(feature = "sgx_platform", issue = "56975")] impl RegisterArgument for bool { fn from_register(a: Register) -> bool { if a != 0 { true } else { false } @@ -121,6 +134,7 @@ impl RegisterArgument for bool { } } +#[unstable(feature = "sgx_platform", issue = "56975")] impl RegisterArgument for Option> { fn from_register(a: Register) -> Option> { NonNull::new(a as _) @@ -130,12 +144,14 @@ impl RegisterArgument for Option> { } } +#[unstable(feature = "sgx_platform", issue = "56975")] impl ReturnValue for ! { fn from_registers(call: &'static str, _regs: (Register, Register)) -> Self { - rtabort!("Usercall {}: did not expect to be re-entered", call); + rtabort!("Usercall {call}: did not expect to be re-entered"); } } +#[unstable(feature = "sgx_platform", issue = "56975")] impl ReturnValue for () { fn from_registers(call: &'static str, usercall_retval: (Register, Register)) -> Self { rtassert!(usercall_retval.0 == 0); @@ -144,6 +160,7 @@ impl ReturnValue for () { } } +#[unstable(feature = "sgx_platform", issue = "56975")] impl ReturnValue for T { fn from_registers(call: &'static str, usercall_retval: (Register, Register)) -> Self { rtassert!(usercall_retval.1 == 0); @@ -151,6 +168,7 @@ impl ReturnValue for T { } } +#[unstable(feature = "sgx_platform", issue = "56975")] impl ReturnValue for (T, U) { fn from_registers(_call: &'static str, regs: (Register, Register)) -> Self { (T::from_register(regs.0), U::from_register(regs.1)) @@ -175,14 +193,14 @@ macro_rules! enclave_usercalls_internal_define_usercalls { #[unstable(feature = "sgx_platform", issue = "56975")] #[inline(always)] pub unsafe fn $f($n1: $t1, $n2: $t2, $n3: $t3, $n4: $t4) -> $r { - ReturnValue::from_registers(stringify!($f), do_usercall( - rtunwrap!(Some, NonZeroU64::new(Usercalls::$f as Register)), - RegisterArgument::into_register($n1), - RegisterArgument::into_register($n2), - RegisterArgument::into_register($n3), - RegisterArgument::into_register($n4), - return_type_is_abort!($r) - )) + ReturnValue::from_registers(stringify!($f), unsafe { do_usercall( + rtunwrap!(Some, NonZeroU64::new(Usercalls::$f as Register)), + RegisterArgument::into_register($n1), + RegisterArgument::into_register($n2), + RegisterArgument::into_register($n3), + RegisterArgument::into_register($n4), + return_type_is_abort!($r) + ) }) } ); (def fn $f:ident($n1:ident: $t1:ty, $n2:ident: $t2:ty, $n3:ident: $t3:ty) -> $r:tt) => ( @@ -191,14 +209,14 @@ macro_rules! enclave_usercalls_internal_define_usercalls { #[unstable(feature = "sgx_platform", issue = "56975")] #[inline(always)] pub unsafe fn $f($n1: $t1, $n2: $t2, $n3: $t3) -> $r { - ReturnValue::from_registers(stringify!($f), do_usercall( - rtunwrap!(Some, NonZeroU64::new(Usercalls::$f as Register)), - RegisterArgument::into_register($n1), - RegisterArgument::into_register($n2), - RegisterArgument::into_register($n3), - 0, - return_type_is_abort!($r) - )) + ReturnValue::from_registers(stringify!($f), unsafe { do_usercall( + rtunwrap!(Some, NonZeroU64::new(Usercalls::$f as Register)), + RegisterArgument::into_register($n1), + RegisterArgument::into_register($n2), + RegisterArgument::into_register($n3), + 0, + return_type_is_abort!($r) + ) }) } ); (def fn $f:ident($n1:ident: $t1:ty, $n2:ident: $t2:ty) -> $r:tt) => ( @@ -207,13 +225,13 @@ macro_rules! enclave_usercalls_internal_define_usercalls { #[unstable(feature = "sgx_platform", issue = "56975")] #[inline(always)] pub unsafe fn $f($n1: $t1, $n2: $t2) -> $r { - ReturnValue::from_registers(stringify!($f), do_usercall( - rtunwrap!(Some, NonZeroU64::new(Usercalls::$f as Register)), - RegisterArgument::into_register($n1), - RegisterArgument::into_register($n2), - 0,0, - return_type_is_abort!($r) - )) + ReturnValue::from_registers(stringify!($f), unsafe { do_usercall( + rtunwrap!(Some, NonZeroU64::new(Usercalls::$f as Register)), + RegisterArgument::into_register($n1), + RegisterArgument::into_register($n2), + 0,0, + return_type_is_abort!($r) + ) }) } ); (def fn $f:ident($n1:ident: $t1:ty) -> $r:tt) => ( @@ -222,12 +240,12 @@ macro_rules! enclave_usercalls_internal_define_usercalls { #[unstable(feature = "sgx_platform", issue = "56975")] #[inline(always)] pub unsafe fn $f($n1: $t1) -> $r { - ReturnValue::from_registers(stringify!($f), do_usercall( - rtunwrap!(Some, NonZeroU64::new(Usercalls::$f as Register)), - RegisterArgument::into_register($n1), - 0,0,0, - return_type_is_abort!($r) - )) + ReturnValue::from_registers(stringify!($f), unsafe { do_usercall( + rtunwrap!(Some, NonZeroU64::new(Usercalls::$f as Register)), + RegisterArgument::into_register($n1), + 0,0,0, + return_type_is_abort!($r) + ) }) } ); (def fn $f:ident() -> $r:tt) => ( @@ -236,11 +254,11 @@ macro_rules! enclave_usercalls_internal_define_usercalls { #[unstable(feature = "sgx_platform", issue = "56975")] #[inline(always)] pub unsafe fn $f() -> $r { - ReturnValue::from_registers(stringify!($f), do_usercall( - rtunwrap!(Some, NonZeroU64::new(Usercalls::$f as Register)), - 0,0,0,0, - return_type_is_abort!($r) - )) + ReturnValue::from_registers(stringify!($f), unsafe { do_usercall( + rtunwrap!(Some, NonZeroU64::new(Usercalls::$f as Register)), + 0,0,0,0, + return_type_is_abort!($r) + ) }) } ); (def fn $f:ident($($n:ident: $t:ty),*)) => ( diff --git a/crux-mir/lib/std/src/sys/sgx/abi/usercalls/tests.rs b/crux-mir/lib/std/src/sys/sgx/abi/usercalls/tests.rs new file mode 100644 index 000000000..58b8eb215 --- /dev/null +++ b/crux-mir/lib/std/src/sys/sgx/abi/usercalls/tests.rs @@ -0,0 +1,56 @@ +use super::alloc::User; +use super::alloc::{copy_from_userspace, copy_to_userspace}; + +#[test] +fn test_copy_to_userspace_function() { + let mut src = [0u8; 100]; + let mut dst = User::<[u8]>::uninitialized(100); + + for i in 0..src.len() { + src[i] = i as _; + } + + for size in 0..48 { + // For all possible alignment + for offset in 0..8 { + // overwrite complete dst + dst.copy_from_enclave(&[0u8; 100]); + + // Copy src[0..size] to dst + offset + unsafe { copy_to_userspace(src.as_ptr(), dst.as_mut_ptr().add(offset), size) }; + + // Verify copy + for byte in 0..size { + unsafe { + assert_eq!(*dst.as_ptr().add(offset + byte), src[byte as usize]); + } + } + } + } +} + +#[test] +fn test_copy_from_userspace_function() { + let mut dst = [0u8; 100]; + let mut src = User::<[u8]>::uninitialized(100); + + src.copy_from_enclave(&[0u8; 100]); + + for size in 0..48 { + // For all possible alignment + for offset in 0..8 { + // overwrite complete dst + dst = [0u8; 100]; + + // Copy src[0..size] to dst + offset + unsafe { copy_from_userspace(src.as_ptr().offset(offset), dst.as_mut_ptr(), size) }; + + // Verify copy + for byte in 0..size { + unsafe { + assert_eq!(dst[byte as usize], *src.as_ptr().offset(offset + byte as isize)); + } + } + } + } +} diff --git a/crux-mir/lib/std/src/sys/sgx/alloc.rs b/crux-mir/lib/std/src/sys/sgx/alloc.rs index 40daec758..4aea28cb8 100644 --- a/crux-mir/lib/std/src/sys/sgx/alloc.rs +++ b/crux-mir/lib/std/src/sys/sgx/alloc.rs @@ -1,33 +1,85 @@ use crate::alloc::{GlobalAlloc, Layout, System}; +use crate::ptr; +use crate::sys::sgx::abi::mem as sgx_mem; +use core::sync::atomic::{AtomicBool, Ordering}; use super::waitqueue::SpinMutex; // Using a SpinMutex because we never want to exit the enclave waiting for the // allocator. +// +// The current allocator here is the `dlmalloc` crate which we've got included +// in the rust-lang/rust repository as a submodule. The crate is a port of +// dlmalloc.c from C to Rust. #[cfg_attr(test, linkage = "available_externally")] #[export_name = "_ZN16__rust_internals3std3sys3sgx5alloc8DLMALLOCE"] -static DLMALLOC: SpinMutex = SpinMutex::new(dlmalloc::DLMALLOC_INIT); +static DLMALLOC: SpinMutex> = + SpinMutex::new(dlmalloc::Dlmalloc::new_with_allocator(Sgx {})); + +struct Sgx; + +unsafe impl dlmalloc::Allocator for Sgx { + /// Allocs system resources + fn alloc(&self, _size: usize) -> (*mut u8, usize, u32) { + static INIT: AtomicBool = AtomicBool::new(false); + + // No ordering requirement since this function is protected by the global lock. + if !INIT.swap(true, Ordering::Relaxed) { + (sgx_mem::heap_base() as _, sgx_mem::heap_size(), 0) + } else { + (ptr::null_mut(), 0, 0) + } + } + + fn remap(&self, _ptr: *mut u8, _oldsize: usize, _newsize: usize, _can_move: bool) -> *mut u8 { + ptr::null_mut() + } + + fn free_part(&self, _ptr: *mut u8, _oldsize: usize, _newsize: usize) -> bool { + false + } + + fn free(&self, _ptr: *mut u8, _size: usize) -> bool { + return false; + } + + fn can_release_part(&self, _flags: u32) -> bool { + false + } + + fn allocates_zeros(&self) -> bool { + false + } + + fn page_size(&self) -> usize { + 0x1000 + } +} #[stable(feature = "alloc_system_type", since = "1.28.0")] unsafe impl GlobalAlloc for System { #[inline] unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - DLMALLOC.lock().malloc(layout.size(), layout.align()) + // SAFETY: the caller must uphold the safety contract for `malloc` + unsafe { DLMALLOC.lock().malloc(layout.size(), layout.align()) } } #[inline] unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { - DLMALLOC.lock().calloc(layout.size(), layout.align()) + // SAFETY: the caller must uphold the safety contract for `malloc` + unsafe { DLMALLOC.lock().calloc(layout.size(), layout.align()) } } #[inline] unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - DLMALLOC.lock().free(ptr, layout.size(), layout.align()) + // SAFETY: the caller must uphold the safety contract for `malloc` + unsafe { DLMALLOC.lock().free(ptr, layout.size(), layout.align()) } } #[inline] unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - DLMALLOC.lock().realloc(ptr, layout.size(), layout.align(), new_size) + // SAFETY: the caller must uphold the safety contract for `malloc` + unsafe { DLMALLOC.lock().realloc(ptr, layout.size(), layout.align(), new_size) } } } @@ -36,11 +88,11 @@ unsafe impl GlobalAlloc for System { #[cfg(not(test))] #[no_mangle] pub unsafe extern "C" fn __rust_c_alloc(size: usize, align: usize) -> *mut u8 { - crate::alloc::alloc(Layout::from_size_align_unchecked(size, align)) + unsafe { crate::alloc::alloc(Layout::from_size_align_unchecked(size, align)) } } #[cfg(not(test))] #[no_mangle] pub unsafe extern "C" fn __rust_c_dealloc(ptr: *mut u8, size: usize, align: usize) { - crate::alloc::dealloc(ptr, Layout::from_size_align_unchecked(size, align)) + unsafe { crate::alloc::dealloc(ptr, Layout::from_size_align_unchecked(size, align)) } } diff --git a/crux-mir/lib/std/src/sys/sgx/args.rs b/crux-mir/lib/std/src/sys/sgx/args.rs index 5a53695a8..ef4176c4a 100644 --- a/crux-mir/lib/std/src/sys/sgx/args.rs +++ b/crux-mir/lib/std/src/sys/sgx/args.rs @@ -1,5 +1,6 @@ use super::abi::usercalls::{alloc, raw::ByteBuffer}; use crate::ffi::OsString; +use crate::fmt; use crate::slice; use crate::sync::atomic::{AtomicUsize, Ordering}; use crate::sys::os_str::Buf; @@ -13,7 +14,7 @@ type ArgsStore = Vec; #[cfg_attr(test, allow(dead_code))] pub unsafe fn init(argc: isize, argv: *const *const u8) { if argc != 0 { - let args = alloc::User::<[ByteBuffer]>::from_raw_parts(argv as _, argc as _); + let args = unsafe { alloc::User::<[ByteBuffer]>::from_raw_parts(argv as _, argc as _) }; let args = args .iter() .map(|a| OsString::from_inner(Buf { inner: a.copy_user_buffer() })) @@ -22,8 +23,6 @@ pub unsafe fn init(argc: isize, argv: *const *const u8) { } } -pub unsafe fn cleanup() {} - pub fn args() -> Args { let args = unsafe { (ARGS.load(Ordering::Relaxed) as *const ArgsStore).as_ref() }; if let Some(args) = args { Args(args.iter()) } else { Args([].iter()) } @@ -31,9 +30,9 @@ pub fn args() -> Args { pub struct Args(slice::Iter<'static, OsString>); -impl Args { - pub fn inner_debug(&self) -> &[OsString] { - self.0.as_slice() +impl fmt::Debug for Args { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.as_slice().fmt(f) } } diff --git a/crux-mir/lib/std/src/sys/sgx/cmath.rs b/crux-mir/lib/std/src/sys/sgx/cmath.rs deleted file mode 100644 index b89238f1d..000000000 --- a/crux-mir/lib/std/src/sys/sgx/cmath.rs +++ /dev/null @@ -1,31 +0,0 @@ -#![cfg(not(test))] - -// These symbols are all defined in `compiler-builtins` -extern "C" { - pub fn acos(n: f64) -> f64; - pub fn acosf(n: f32) -> f32; - pub fn asin(n: f64) -> f64; - pub fn asinf(n: f32) -> f32; - pub fn atan(n: f64) -> f64; - pub fn atan2(a: f64, b: f64) -> f64; - pub fn atan2f(a: f32, b: f32) -> f32; - pub fn atanf(n: f32) -> f32; - pub fn cbrt(n: f64) -> f64; - pub fn cbrtf(n: f32) -> f32; - pub fn cosh(n: f64) -> f64; - pub fn coshf(n: f32) -> f32; - pub fn expm1(n: f64) -> f64; - pub fn expm1f(n: f32) -> f32; - pub fn fdim(a: f64, b: f64) -> f64; - pub fn fdimf(a: f32, b: f32) -> f32; - pub fn hypot(x: f64, y: f64) -> f64; - pub fn hypotf(x: f32, y: f32) -> f32; - pub fn log1p(n: f64) -> f64; - pub fn log1pf(n: f32) -> f32; - pub fn sinh(n: f64) -> f64; - pub fn sinhf(n: f32) -> f32; - pub fn tan(n: f64) -> f64; - pub fn tanf(n: f32) -> f32; - pub fn tanh(n: f64) -> f64; - pub fn tanhf(n: f32) -> f32; -} diff --git a/crux-mir/lib/std/src/sys/sgx/condvar.rs b/crux-mir/lib/std/src/sys/sgx/condvar.rs index 9c5c08618..aa1174664 100644 --- a/crux-mir/lib/std/src/sys/sgx/condvar.rs +++ b/crux-mir/lib/std/src/sys/sgx/condvar.rs @@ -1,40 +1,46 @@ -use crate::sys::mutex::Mutex; +use crate::sys::locks::Mutex; +use crate::sys_common::lazy_box::{LazyBox, LazyInit}; use crate::time::Duration; use super::waitqueue::{SpinMutex, WaitQueue, WaitVariable}; +/// FIXME: `UnsafeList` is not movable. +struct AllocatedCondvar(SpinMutex>); + pub struct Condvar { - inner: SpinMutex>, + inner: LazyBox, +} + +impl LazyInit for AllocatedCondvar { + fn init() -> Box { + Box::new(AllocatedCondvar(SpinMutex::new(WaitVariable::new(())))) + } } impl Condvar { pub const fn new() -> Condvar { - Condvar { inner: SpinMutex::new(WaitVariable::new(())) } + Condvar { inner: LazyBox::new() } } #[inline] - pub unsafe fn init(&mut self) {} - - #[inline] - pub unsafe fn notify_one(&self) { - let _ = WaitQueue::notify_one(self.inner.lock()); + pub fn notify_one(&self) { + let _ = WaitQueue::notify_one(self.inner.0.lock()); } #[inline] - pub unsafe fn notify_all(&self) { - let _ = WaitQueue::notify_all(self.inner.lock()); + pub fn notify_all(&self) { + let _ = WaitQueue::notify_all(self.inner.0.lock()); } pub unsafe fn wait(&self, mutex: &Mutex) { - let guard = self.inner.lock(); - WaitQueue::wait(guard, || mutex.unlock()); + let guard = self.inner.0.lock(); + WaitQueue::wait(guard, || unsafe { mutex.unlock() }); mutex.lock() } - pub unsafe fn wait_timeout(&self, _mutex: &Mutex, _dur: Duration) -> bool { - rtabort!("timeout not supported in SGX"); + pub unsafe fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool { + let success = WaitQueue::wait_timeout(&self.inner.0, dur, || unsafe { mutex.unlock() }); + mutex.lock(); + success } - - #[inline] - pub unsafe fn destroy(&self) {} } diff --git a/crux-mir/lib/std/src/sys/sgx/env.rs b/crux-mir/lib/std/src/sys/sgx/env.rs index 6fa0ed7bc..8043b7c52 100644 --- a/crux-mir/lib/std/src/sys/sgx/env.rs +++ b/crux-mir/lib/std/src/sys/sgx/env.rs @@ -1,9 +1,9 @@ pub mod os { - pub const FAMILY: &'static str = ""; - pub const OS: &'static str = ""; - pub const DLL_PREFIX: &'static str = ""; - pub const DLL_SUFFIX: &'static str = ".sgxs"; - pub const DLL_EXTENSION: &'static str = "sgxs"; - pub const EXE_SUFFIX: &'static str = ".sgxs"; - pub const EXE_EXTENSION: &'static str = "sgxs"; + pub const FAMILY: &str = ""; + pub const OS: &str = ""; + pub const DLL_PREFIX: &str = ""; + pub const DLL_SUFFIX: &str = ".sgxs"; + pub const DLL_EXTENSION: &str = "sgxs"; + pub const EXE_SUFFIX: &str = ".sgxs"; + pub const EXE_EXTENSION: &str = "sgxs"; } diff --git a/crux-mir/lib/std/src/sys/sgx/ext/mod.rs b/crux-mir/lib/std/src/sys/sgx/ext/mod.rs deleted file mode 100644 index 258ad3cd2..000000000 --- a/crux-mir/lib/std/src/sys/sgx/ext/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -#![unstable(feature = "sgx_platform", issue = "56975")] - -pub mod arch; -pub mod ffi; -pub mod io; diff --git a/crux-mir/lib/std/src/sys/sgx/fd.rs b/crux-mir/lib/std/src/sys/sgx/fd.rs index 7da2424a6..e5dc5b5ad 100644 --- a/crux-mir/lib/std/src/sys/sgx/fd.rs +++ b/crux-mir/lib/std/src/sys/sgx/fd.rs @@ -19,7 +19,7 @@ impl FileDesc { self.fd } - /// Extracts the actual filedescriptor without closing it. + /// Extracts the actual file descriptor without closing it. pub fn into_raw(self) -> Fd { let fd = self.fd; mem::forget(self); @@ -34,6 +34,11 @@ impl FileDesc { usercalls::read(self.fd, bufs) } + #[inline] + pub fn is_read_vectored(&self) -> bool { + true + } + pub fn write(&self, buf: &[u8]) -> io::Result { usercalls::write(self.fd, &[IoSlice::new(buf)]) } @@ -42,6 +47,11 @@ impl FileDesc { usercalls::write(self.fd, bufs) } + #[inline] + pub fn is_write_vectored(&self) -> bool { + true + } + pub fn flush(&self) -> io::Result<()> { usercalls::flush(self.fd) } diff --git a/crux-mir/lib/std/src/sys/sgx/fs.rs b/crux-mir/lib/std/src/sys/sgx/fs.rs deleted file mode 100644 index e6160d145..000000000 --- a/crux-mir/lib/std/src/sys/sgx/fs.rs +++ /dev/null @@ -1,300 +0,0 @@ -use crate::ffi::OsString; -use crate::fmt; -use crate::hash::{Hash, Hasher}; -use crate::io::{self, IoSlice, IoSliceMut, SeekFrom}; -use crate::path::{Path, PathBuf}; -use crate::sys::time::SystemTime; -use crate::sys::{unsupported, Void}; - -pub struct File(Void); - -pub struct FileAttr(Void); - -pub struct ReadDir(Void); - -pub struct DirEntry(Void); - -#[derive(Clone, Debug)] -pub struct OpenOptions {} - -pub struct FilePermissions(Void); - -pub struct FileType(Void); - -#[derive(Debug)] -pub struct DirBuilder {} - -impl FileAttr { - pub fn size(&self) -> u64 { - match self.0 {} - } - - pub fn perm(&self) -> FilePermissions { - match self.0 {} - } - - pub fn file_type(&self) -> FileType { - match self.0 {} - } - - pub fn modified(&self) -> io::Result { - match self.0 {} - } - - pub fn accessed(&self) -> io::Result { - match self.0 {} - } - - pub fn created(&self) -> io::Result { - match self.0 {} - } -} - -impl Clone for FileAttr { - fn clone(&self) -> FileAttr { - match self.0 {} - } -} - -impl FilePermissions { - pub fn readonly(&self) -> bool { - match self.0 {} - } - - pub fn set_readonly(&mut self, _readonly: bool) { - match self.0 {} - } -} - -impl Clone for FilePermissions { - fn clone(&self) -> FilePermissions { - match self.0 {} - } -} - -impl PartialEq for FilePermissions { - fn eq(&self, _other: &FilePermissions) -> bool { - match self.0 {} - } -} - -impl Eq for FilePermissions {} - -impl fmt::Debug for FilePermissions { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} - } -} - -impl FileType { - pub fn is_dir(&self) -> bool { - match self.0 {} - } - - pub fn is_file(&self) -> bool { - match self.0 {} - } - - pub fn is_symlink(&self) -> bool { - match self.0 {} - } -} - -impl Clone for FileType { - fn clone(&self) -> FileType { - match self.0 {} - } -} - -impl Copy for FileType {} - -impl PartialEq for FileType { - fn eq(&self, _other: &FileType) -> bool { - match self.0 {} - } -} - -impl Eq for FileType {} - -impl Hash for FileType { - fn hash(&self, _h: &mut H) { - match self.0 {} - } -} - -impl fmt::Debug for FileType { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} - } -} - -impl fmt::Debug for ReadDir { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} - } -} - -impl Iterator for ReadDir { - type Item = io::Result; - - fn next(&mut self) -> Option> { - match self.0 {} - } -} - -impl DirEntry { - pub fn path(&self) -> PathBuf { - match self.0 {} - } - - pub fn file_name(&self) -> OsString { - match self.0 {} - } - - pub fn metadata(&self) -> io::Result { - match self.0 {} - } - - pub fn file_type(&self) -> io::Result { - match self.0 {} - } -} - -impl OpenOptions { - pub fn new() -> OpenOptions { - OpenOptions {} - } - - pub fn read(&mut self, _read: bool) {} - pub fn write(&mut self, _write: bool) {} - pub fn append(&mut self, _append: bool) {} - pub fn truncate(&mut self, _truncate: bool) {} - pub fn create(&mut self, _create: bool) {} - pub fn create_new(&mut self, _create_new: bool) {} -} - -impl File { - pub fn open(_path: &Path, _opts: &OpenOptions) -> io::Result { - unsupported() - } - - pub fn file_attr(&self) -> io::Result { - match self.0 {} - } - - pub fn fsync(&self) -> io::Result<()> { - match self.0 {} - } - - pub fn datasync(&self) -> io::Result<()> { - match self.0 {} - } - - pub fn truncate(&self, _size: u64) -> io::Result<()> { - match self.0 {} - } - - pub fn read(&self, _buf: &mut [u8]) -> io::Result { - match self.0 {} - } - - pub fn read_vectored(&self, _bufs: &mut [IoSliceMut<'_>]) -> io::Result { - match self.0 {} - } - - pub fn write(&self, _buf: &[u8]) -> io::Result { - match self.0 {} - } - - pub fn write_vectored(&self, _bufs: &[IoSlice<'_>]) -> io::Result { - match self.0 {} - } - - pub fn flush(&self) -> io::Result<()> { - match self.0 {} - } - - pub fn seek(&self, _pos: SeekFrom) -> io::Result { - match self.0 {} - } - - pub fn duplicate(&self) -> io::Result { - match self.0 {} - } - - pub fn set_permissions(&self, _perm: FilePermissions) -> io::Result<()> { - match self.0 {} - } - - pub fn diverge(&self) -> ! { - match self.0 {} - } -} - -impl DirBuilder { - pub fn new() -> DirBuilder { - DirBuilder {} - } - - pub fn mkdir(&self, _p: &Path) -> io::Result<()> { - unsupported() - } -} - -impl fmt::Debug for File { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} - } -} - -pub fn readdir(_p: &Path) -> io::Result { - unsupported() -} - -pub fn unlink(_p: &Path) -> io::Result<()> { - unsupported() -} - -pub fn rename(_old: &Path, _new: &Path) -> io::Result<()> { - unsupported() -} - -pub fn set_perm(_p: &Path, perm: FilePermissions) -> io::Result<()> { - match perm.0 {} -} - -pub fn rmdir(_p: &Path) -> io::Result<()> { - unsupported() -} - -pub fn remove_dir_all(_path: &Path) -> io::Result<()> { - unsupported() -} - -pub fn readlink(_p: &Path) -> io::Result { - unsupported() -} - -pub fn symlink(_src: &Path, _dst: &Path) -> io::Result<()> { - unsupported() -} - -pub fn link(_src: &Path, _dst: &Path) -> io::Result<()> { - unsupported() -} - -pub fn stat(_p: &Path) -> io::Result { - unsupported() -} - -pub fn lstat(_p: &Path) -> io::Result { - unsupported() -} - -pub fn canonicalize(_p: &Path) -> io::Result { - unsupported() -} - -pub fn copy(_from: &Path, _to: &Path) -> io::Result { - unsupported() -} diff --git a/crux-mir/lib/std/src/sys/sgx/io.rs b/crux-mir/lib/std/src/sys/sgx/io.rs deleted file mode 100644 index d5f475b43..000000000 --- a/crux-mir/lib/std/src/sys/sgx/io.rs +++ /dev/null @@ -1,47 +0,0 @@ -use crate::mem; - -#[derive(Copy, Clone)] -pub struct IoSlice<'a>(&'a [u8]); - -impl<'a> IoSlice<'a> { - #[inline] - pub fn new(buf: &'a [u8]) -> IoSlice<'a> { - IoSlice(buf) - } - - #[inline] - pub fn advance(&mut self, n: usize) { - self.0 = &self.0[n..] - } - - #[inline] - pub fn as_slice(&self) -> &[u8] { - self.0 - } -} - -pub struct IoSliceMut<'a>(&'a mut [u8]); - -impl<'a> IoSliceMut<'a> { - #[inline] - pub fn new(buf: &'a mut [u8]) -> IoSliceMut<'a> { - IoSliceMut(buf) - } - - #[inline] - pub fn advance(&mut self, n: usize) { - let slice = mem::replace(&mut self.0, &mut []); - let (_, remaining) = slice.split_at_mut(n); - self.0 = remaining; - } - - #[inline] - pub fn as_slice(&self) -> &[u8] { - self.0 - } - - #[inline] - pub fn as_mut_slice(&mut self) -> &mut [u8] { - self.0 - } -} diff --git a/crux-mir/lib/std/src/sys/sgx/mod.rs b/crux-mir/lib/std/src/sys/sgx/mod.rs index 83cee0cf3..9865a945b 100644 --- a/crux-mir/lib/std/src/sys/sgx/mod.rs +++ b/crux-mir/lib/std/src/sys/sgx/mod.rs @@ -2,9 +2,10 @@ //! //! This module contains the facade (aka platform-specific) implementations of //! OS level functionality for Fortanix SGX. +#![deny(unsafe_op_in_unsafe_fn)] +#![allow(fuzzy_provenance_casts)] // FIXME: this entire module systematically confuses pointers and integers use crate::io::ErrorKind; -use crate::os::raw::c_char; use crate::sync::atomic::{AtomicBool, Ordering}; pub mod abi; @@ -12,31 +13,51 @@ mod waitqueue; pub mod alloc; pub mod args; +#[path = "../unix/cmath.rs"] pub mod cmath; -pub mod condvar; pub mod env; -pub mod ext; pub mod fd; +#[path = "../unsupported/fs.rs"] pub mod fs; +#[path = "../unsupported/io.rs"] pub mod io; pub mod memchr; -pub mod mutex; pub mod net; pub mod os; +#[path = "../unix/os_str.rs"] +pub mod os_str; pub mod path; +#[path = "../unsupported/pipe.rs"] pub mod pipe; +#[path = "../unsupported/process.rs"] pub mod process; -pub mod rwlock; -pub mod stack_overflow; pub mod stdio; pub mod thread; -pub mod thread_local; +pub mod thread_local_key; +pub mod thread_parking; pub mod time; -pub use crate::sys_common::os_str_bytes as os_str; +mod condvar; +mod mutex; +mod rwlock; -#[cfg(not(test))] -pub fn init() {} +pub mod locks { + pub use super::condvar::*; + pub use super::mutex::*; + pub use super::rwlock::*; +} + +// SAFETY: must be called only once during runtime initialization. +// NOTE: this is not guaranteed to run, for example when Rust code is called externally. +pub unsafe fn init(argc: isize, argv: *const *const u8, _sigpipe: u8) { + unsafe { + args::init(argc, argv); + } +} + +// SAFETY: must be called only once during runtime cleanup. +// NOTE: this is not guaranteed to run, for example when the program aborts. +pub unsafe fn cleanup() {} /// This function is used to implement functionality that simply doesn't exist. /// Programs relying on this functionality will need to deal with the error. @@ -45,7 +66,7 @@ pub fn unsupported() -> crate::io::Result { } pub fn unsupported_err() -> crate::io::Error { - crate::io::Error::new(ErrorKind::Other, "operation not supported on SGX yet") + crate::io::const_io_error!(ErrorKind::Unsupported, "operation not supported on SGX yet") } /// This function is used to implement various functions that doesn't exist, @@ -56,8 +77,8 @@ pub fn unsupported_err() -> crate::io::Error { pub fn sgx_ineffective(v: T) -> crate::io::Result { static SGX_INEFFECTIVE_ERROR: AtomicBool = AtomicBool::new(false); if SGX_INEFFECTIVE_ERROR.load(Ordering::Relaxed) { - Err(crate::io::Error::new( - ErrorKind::Other, + Err(crate::io::const_io_error!( + ErrorKind::Uncategorized, "operation can't be trusted to have any effect on SGX", )) } else { @@ -102,29 +123,15 @@ pub fn decode_error_kind(code: i32) -> ErrorKind { } else if code == Error::Interrupted as _ { ErrorKind::Interrupted } else if code == Error::Other as _ { - ErrorKind::Other + ErrorKind::Uncategorized } else if code == Error::UnexpectedEof as _ { ErrorKind::UnexpectedEof } else { - ErrorKind::Other - } -} - -// This enum is used as the storage for a bunch of types which can't actually -// exist. -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] -pub enum Void {} - -pub unsafe fn strlen(mut s: *const c_char) -> usize { - let mut n = 0; - while *s != 0 { - n += 1; - s = s.offset(1); + ErrorKind::Uncategorized } - return n; } -pub unsafe fn abort_internal() -> ! { +pub fn abort_internal() -> ! { abi::usercalls::exit(true) } @@ -133,12 +140,12 @@ pub unsafe fn abort_internal() -> ! { #[cfg(not(test))] #[no_mangle] // NB. used by both libunwind and libpanic_abort -pub unsafe extern "C" fn __rust_abort() { +pub extern "C" fn __rust_abort() { abort_internal(); } -pub fn hashmap_random_keys() -> (u64, u64) { - fn rdrand64() -> u64 { +pub mod rand { + pub fn rdrand64() -> u64 { unsafe { let mut ret: u64 = 0; for _ in 0..10 { @@ -149,7 +156,10 @@ pub fn hashmap_random_keys() -> (u64, u64) { rtabort!("Failed to obtain random data"); } } - (rdrand64(), rdrand64()) +} + +pub fn hashmap_random_keys() -> (u64, u64) { + (self::rand::rdrand64(), self::rand::rdrand64()) } pub use crate::sys_common::{AsInner, FromInner, IntoInner}; diff --git a/crux-mir/lib/std/src/sys/sgx/mutex.rs b/crux-mir/lib/std/src/sys/sgx/mutex.rs index 4911c2f53..0dbf020eb 100644 --- a/crux-mir/lib/std/src/sys/sgx/mutex.rs +++ b/crux-mir/lib/std/src/sys/sgx/mutex.rs @@ -1,25 +1,28 @@ -use fortanix_sgx_abi::Tcs; +use super::waitqueue::{try_lock_or_false, SpinMutex, WaitQueue, WaitVariable}; +use crate::sys_common::lazy_box::{LazyBox, LazyInit}; -use super::abi::thread; - -use super::waitqueue::{try_lock_or_false, NotifiedTcs, SpinMutex, WaitQueue, WaitVariable}; +/// FIXME: `UnsafeList` is not movable. +struct AllocatedMutex(SpinMutex>); pub struct Mutex { - inner: SpinMutex>, + inner: LazyBox, +} + +impl LazyInit for AllocatedMutex { + fn init() -> Box { + Box::new(AllocatedMutex(SpinMutex::new(WaitVariable::new(false)))) + } } // Implementation according to “Operating Systems: Three Easy Pieces”, chapter 28 impl Mutex { pub const fn new() -> Mutex { - Mutex { inner: SpinMutex::new(WaitVariable::new(false)) } + Mutex { inner: LazyBox::new() } } #[inline] - pub unsafe fn init(&mut self) {} - - #[inline] - pub unsafe fn lock(&self) { - let mut guard = self.inner.lock(); + pub fn lock(&self) { + let mut guard = self.inner.0.lock(); if *guard.lock_var() { // Another thread has the lock, wait WaitQueue::wait(guard, || {}) @@ -32,7 +35,7 @@ impl Mutex { #[inline] pub unsafe fn unlock(&self) { - let guard = self.inner.lock(); + let guard = self.inner.0.lock(); if let Err(mut guard) = WaitQueue::notify_one(guard) { // No other waiters, unlock *guard.lock_var_mut() = false; @@ -42,8 +45,8 @@ impl Mutex { } #[inline] - pub unsafe fn try_lock(&self) -> bool { - let mut guard = try_lock_or_false!(self.inner); + pub fn try_lock(&self) -> bool { + let mut guard = try_lock_or_false!(self.inner.0); if *guard.lock_var() { // Another thread has the lock false @@ -53,88 +56,4 @@ impl Mutex { true } } - - #[inline] - pub unsafe fn destroy(&self) {} -} - -struct ReentrantLock { - owner: Option, - count: usize, -} - -pub struct ReentrantMutex { - inner: SpinMutex>, -} - -impl ReentrantMutex { - pub const fn uninitialized() -> ReentrantMutex { - ReentrantMutex { - inner: SpinMutex::new(WaitVariable::new(ReentrantLock { owner: None, count: 0 })), - } - } - - #[inline] - pub unsafe fn init(&self) {} - - #[inline] - pub unsafe fn lock(&self) { - let mut guard = self.inner.lock(); - match guard.lock_var().owner { - Some(tcs) if tcs != thread::current() => { - // Another thread has the lock, wait - WaitQueue::wait(guard, || {}); - // Another thread has passed the lock to us - } - _ => { - // We are just now obtaining the lock - guard.lock_var_mut().owner = Some(thread::current()); - guard.lock_var_mut().count += 1; - } - } - } - - #[inline] - pub unsafe fn unlock(&self) { - let mut guard = self.inner.lock(); - if guard.lock_var().count > 1 { - guard.lock_var_mut().count -= 1; - } else { - match WaitQueue::notify_one(guard) { - Err(mut guard) => { - // No other waiters, unlock - guard.lock_var_mut().count = 0; - guard.lock_var_mut().owner = None; - } - Ok(mut guard) => { - // There was a thread waiting, just pass the lock - if let NotifiedTcs::Single(tcs) = guard.notified_tcs() { - guard.lock_var_mut().owner = Some(tcs) - } else { - unreachable!() // called notify_one - } - } - } - } - } - - #[inline] - pub unsafe fn try_lock(&self) -> bool { - let mut guard = try_lock_or_false!(self.inner); - match guard.lock_var().owner { - Some(tcs) if tcs != thread::current() => { - // Another thread has the lock - false - } - _ => { - // We are just now obtaining the lock - guard.lock_var_mut().owner = Some(thread::current()); - guard.lock_var_mut().count += 1; - true - } - } - } - - #[inline] - pub unsafe fn destroy(&self) {} } diff --git a/crux-mir/lib/std/src/sys/sgx/net.rs b/crux-mir/lib/std/src/sys/sgx/net.rs index bd0652ab4..4c4cd7d1d 100644 --- a/crux-mir/lib/std/src/sys/sgx/net.rs +++ b/crux-mir/lib/std/src/sys/sgx/net.rs @@ -1,11 +1,10 @@ -use crate::convert::TryFrom; use crate::error; use crate::fmt; use crate::io::{self, IoSlice, IoSliceMut}; use crate::net::{Ipv4Addr, Ipv6Addr, Shutdown, SocketAddr, ToSocketAddrs}; use crate::sync::Arc; use crate::sys::fd::FileDesc; -use crate::sys::{sgx_ineffective, unsupported, AsInner, FromInner, IntoInner, TryIntoInner, Void}; +use crate::sys::{sgx_ineffective, unsupported, AsInner, FromInner, IntoInner, TryIntoInner}; use crate::time::Duration; use super::abi::usercalls; @@ -37,9 +36,9 @@ impl TryIntoInner for Socket { } } -impl FromInner for Socket { - fn from_inner(inner: FileDesc) -> Socket { - Socket { inner: Arc::new(inner), local_addr: None } +impl FromInner<(FileDesc, Option)> for Socket { + fn from_inner((inner, local_addr): (FileDesc, Option)) -> Socket { + Socket { inner: Arc::new(inner), local_addr } } } @@ -97,7 +96,7 @@ impl TcpStream { pub fn connect_timeout(addr: &SocketAddr, dur: Duration) -> io::Result { if dur == Duration::default() { - return Err(io::Error::new( + return Err(io::const_io_error!( io::ErrorKind::InvalidInput, "cannot set a 0 duration timeout", )); @@ -108,7 +107,7 @@ impl TcpStream { pub fn set_read_timeout(&self, dur: Option) -> io::Result<()> { match dur { Some(dur) if dur == Duration::default() => { - return Err(io::Error::new( + return Err(io::const_io_error!( io::ErrorKind::InvalidInput, "cannot set a 0 duration timeout", )); @@ -120,7 +119,7 @@ impl TcpStream { pub fn set_write_timeout(&self, dur: Option) -> io::Result<()> { match dur { Some(dur) if dur == Duration::default() => { - return Err(io::Error::new( + return Err(io::const_io_error!( io::ErrorKind::InvalidInput, "cannot set a 0 duration timeout", )); @@ -149,6 +148,11 @@ impl TcpStream { self.inner.inner.read_vectored(bufs) } + #[inline] + pub fn is_read_vectored(&self) -> bool { + self.inner.inner.is_read_vectored() + } + pub fn write(&self, buf: &[u8]) -> io::Result { self.inner.inner.write(buf) } @@ -157,6 +161,11 @@ impl TcpStream { self.inner.inner.write_vectored(bufs) } + #[inline] + pub fn is_write_vectored(&self) -> bool { + self.inner.inner.is_write_vectored() + } + pub fn peer_addr(&self) -> io::Result { addr_to_sockaddr(&self.peer_addr) } @@ -173,6 +182,14 @@ impl TcpStream { Ok(self.clone()) } + pub fn set_linger(&self, _: Option) -> io::Result<()> { + sgx_ineffective(()) + } + + pub fn linger(&self) -> io::Result> { + sgx_ineffective(None) + } + pub fn set_nodelay(&self, _: bool) -> io::Result<()> { sgx_ineffective(()) } @@ -300,7 +317,7 @@ impl FromInner for TcpListener { } } -pub struct UdpSocket(Void); +pub struct UdpSocket(!); impl UdpSocket { pub fn bind(_: io::Result<&SocketAddr>) -> io::Result { @@ -308,129 +325,129 @@ impl UdpSocket { } pub fn peer_addr(&self) -> io::Result { - match self.0 {} + self.0 } pub fn socket_addr(&self) -> io::Result { - match self.0 {} + self.0 } pub fn recv_from(&self, _: &mut [u8]) -> io::Result<(usize, SocketAddr)> { - match self.0 {} + self.0 } pub fn peek_from(&self, _: &mut [u8]) -> io::Result<(usize, SocketAddr)> { - match self.0 {} + self.0 } pub fn send_to(&self, _: &[u8], _: &SocketAddr) -> io::Result { - match self.0 {} + self.0 } pub fn duplicate(&self) -> io::Result { - match self.0 {} + self.0 } pub fn set_read_timeout(&self, _: Option) -> io::Result<()> { - match self.0 {} + self.0 } pub fn set_write_timeout(&self, _: Option) -> io::Result<()> { - match self.0 {} + self.0 } pub fn read_timeout(&self) -> io::Result> { - match self.0 {} + self.0 } pub fn write_timeout(&self) -> io::Result> { - match self.0 {} + self.0 } pub fn set_broadcast(&self, _: bool) -> io::Result<()> { - match self.0 {} + self.0 } pub fn broadcast(&self) -> io::Result { - match self.0 {} + self.0 } pub fn set_multicast_loop_v4(&self, _: bool) -> io::Result<()> { - match self.0 {} + self.0 } pub fn multicast_loop_v4(&self) -> io::Result { - match self.0 {} + self.0 } pub fn set_multicast_ttl_v4(&self, _: u32) -> io::Result<()> { - match self.0 {} + self.0 } pub fn multicast_ttl_v4(&self) -> io::Result { - match self.0 {} + self.0 } pub fn set_multicast_loop_v6(&self, _: bool) -> io::Result<()> { - match self.0 {} + self.0 } pub fn multicast_loop_v6(&self) -> io::Result { - match self.0 {} + self.0 } pub fn join_multicast_v4(&self, _: &Ipv4Addr, _: &Ipv4Addr) -> io::Result<()> { - match self.0 {} + self.0 } pub fn join_multicast_v6(&self, _: &Ipv6Addr, _: u32) -> io::Result<()> { - match self.0 {} + self.0 } pub fn leave_multicast_v4(&self, _: &Ipv4Addr, _: &Ipv4Addr) -> io::Result<()> { - match self.0 {} + self.0 } pub fn leave_multicast_v6(&self, _: &Ipv6Addr, _: u32) -> io::Result<()> { - match self.0 {} + self.0 } pub fn set_ttl(&self, _: u32) -> io::Result<()> { - match self.0 {} + self.0 } pub fn ttl(&self) -> io::Result { - match self.0 {} + self.0 } pub fn take_error(&self) -> io::Result> { - match self.0 {} + self.0 } pub fn set_nonblocking(&self, _: bool) -> io::Result<()> { - match self.0 {} + self.0 } pub fn recv(&self, _: &mut [u8]) -> io::Result { - match self.0 {} + self.0 } pub fn peek(&self, _: &mut [u8]) -> io::Result { - match self.0 {} + self.0 } pub fn send(&self, _: &[u8]) -> io::Result { - match self.0 {} + self.0 } pub fn connect(&self, _: io::Result<&SocketAddr>) -> io::Result<()> { - match self.0 {} + self.0 } } impl fmt::Debug for UdpSocket { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} + self.0 } } @@ -452,22 +469,22 @@ impl fmt::Display for NonIpSockAddr { } } -pub struct LookupHost(Void); +pub struct LookupHost(!); impl LookupHost { fn new(host: String) -> io::Result { - Err(io::Error::new(io::ErrorKind::Other, NonIpSockAddr { host })) + Err(io::Error::new(io::ErrorKind::Uncategorized, NonIpSockAddr { host })) } pub fn port(&self) -> u16 { - match self.0 {} + self.0 } } impl Iterator for LookupHost { type Item = SocketAddr; fn next(&mut self) -> Option { - match self.0 {} + self.0 } } @@ -483,7 +500,7 @@ impl<'a> TryFrom<(&'a str, u16)> for LookupHost { type Error = io::Error; fn try_from((host, port): (&'a str, u16)) -> io::Result { - LookupHost::new(format!("{}:{}", host, port)) + LookupHost::new(format!("{host}:{port}")) } } @@ -521,6 +538,4 @@ pub mod netc { #[derive(Copy, Clone)] pub struct sockaddr {} - - pub type socklen_t = usize; } diff --git a/crux-mir/lib/std/src/sys/sgx/os.rs b/crux-mir/lib/std/src/sys/sgx/os.rs index 56fc84b4a..5da0257f3 100644 --- a/crux-mir/lib/std/src/sys/sgx/os.rs +++ b/crux-mir/lib/std/src/sys/sgx/os.rs @@ -5,12 +5,13 @@ use crate::error::Error as StdError; use crate::ffi::{OsStr, OsString}; use crate::fmt; use crate::io; +use crate::marker::PhantomData; use crate::path::{self, PathBuf}; use crate::str; use crate::sync::atomic::{AtomicUsize, Ordering}; use crate::sync::Mutex; use crate::sync::Once; -use crate::sys::{decode_error_kind, sgx_ineffective, unsupported, Void}; +use crate::sys::{decode_error_kind, sgx_ineffective, unsupported}; use crate::vec; pub fn errno() -> i32 { @@ -21,7 +22,7 @@ pub fn error_string(errno: i32) -> String { if errno == RESULT_SUCCESS { "operation successful".into() } else if ((Error::UserRangeStart as _)..=(Error::UserRangeEnd as _)).contains(&errno) { - format!("user-specified error {:08x}", errno) + format!("user-specified error {errno:08x}") } else { decode_error_kind(errno).as_str().into() } @@ -35,7 +36,7 @@ pub fn chdir(_: &path::Path) -> io::Result<()> { sgx_ineffective(()) } -pub struct SplitPaths<'a>(&'a Void); +pub struct SplitPaths<'a>(!, PhantomData<&'a ()>); pub fn split_paths(_unparsed: &OsStr) -> SplitPaths<'_> { panic!("unsupported") @@ -44,7 +45,7 @@ pub fn split_paths(_unparsed: &OsStr) -> SplitPaths<'_> { impl<'a> Iterator for SplitPaths<'a> { type Item = PathBuf; fn next(&mut self) -> Option { - match *self.0 {} + self.0 } } @@ -105,8 +106,8 @@ pub fn env() -> Env { get_env_store().map(|env| clone_to_vec(&env.lock().unwrap())).unwrap_or_default().into_iter() } -pub fn getenv(k: &OsStr) -> io::Result> { - Ok(get_env_store().and_then(|s| s.lock().unwrap().get(k).cloned())) +pub fn getenv(k: &OsStr) -> Option { + get_env_store().and_then(|s| s.lock().unwrap().get(k).cloned()) } pub fn setenv(k: &OsStr, v: &OsStr) -> io::Result<()> { diff --git a/crux-mir/lib/std/src/sys/sgx/path.rs b/crux-mir/lib/std/src/sys/sgx/path.rs index 06c9df3ff..c805c15e7 100644 --- a/crux-mir/lib/std/src/sys/sgx/path.rs +++ b/crux-mir/lib/std/src/sys/sgx/path.rs @@ -1,5 +1,7 @@ use crate::ffi::OsStr; -use crate::path::Prefix; +use crate::io; +use crate::path::{Path, PathBuf, Prefix}; +use crate::sys::unsupported; #[inline] pub fn is_sep_byte(b: u8) -> bool { @@ -15,5 +17,9 @@ pub fn parse_prefix(_: &OsStr) -> Option> { None } -pub const MAIN_SEP_STR: &'static str = "/"; +pub const MAIN_SEP_STR: &str = "/"; pub const MAIN_SEP: char = '/'; + +pub(crate) fn absolute(_path: &Path) -> io::Result { + unsupported() +} diff --git a/crux-mir/lib/std/src/sys/sgx/process.rs b/crux-mir/lib/std/src/sys/sgx/process.rs deleted file mode 100644 index 4702e5c54..000000000 --- a/crux-mir/lib/std/src/sys/sgx/process.rs +++ /dev/null @@ -1,149 +0,0 @@ -use crate::ffi::OsStr; -use crate::fmt; -use crate::io; -use crate::sys::fs::File; -use crate::sys::pipe::AnonPipe; -use crate::sys::{unsupported, Void}; -use crate::sys_common::process::CommandEnv; - -pub use crate::ffi::OsString as EnvKey; - -//////////////////////////////////////////////////////////////////////////////// -// Command -//////////////////////////////////////////////////////////////////////////////// - -pub struct Command { - env: CommandEnv, -} - -// passed back to std::process with the pipes connected to the child, if any -// were requested -pub struct StdioPipes { - pub stdin: Option, - pub stdout: Option, - pub stderr: Option, -} - -pub enum Stdio { - Inherit, - Null, - MakePipe, -} - -impl Command { - pub fn new(_program: &OsStr) -> Command { - Command { env: Default::default() } - } - - pub fn arg(&mut self, _arg: &OsStr) {} - - pub fn env_mut(&mut self) -> &mut CommandEnv { - &mut self.env - } - - pub fn cwd(&mut self, _dir: &OsStr) {} - - pub fn stdin(&mut self, _stdin: Stdio) {} - - pub fn stdout(&mut self, _stdout: Stdio) {} - - pub fn stderr(&mut self, _stderr: Stdio) {} - - pub fn spawn( - &mut self, - _default: Stdio, - _needs_stdin: bool, - ) -> io::Result<(Process, StdioPipes)> { - unsupported() - } -} - -impl From for Stdio { - fn from(pipe: AnonPipe) -> Stdio { - pipe.diverge() - } -} - -impl From for Stdio { - fn from(file: File) -> Stdio { - file.diverge() - } -} - -impl fmt::Debug for Command { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - Ok(()) - } -} - -pub struct ExitStatus(Void); - -impl ExitStatus { - pub fn success(&self) -> bool { - match self.0 {} - } - - pub fn code(&self) -> Option { - match self.0 {} - } -} - -impl Clone for ExitStatus { - fn clone(&self) -> ExitStatus { - match self.0 {} - } -} - -impl Copy for ExitStatus {} - -impl PartialEq for ExitStatus { - fn eq(&self, _other: &ExitStatus) -> bool { - match self.0 {} - } -} - -impl Eq for ExitStatus {} - -impl fmt::Debug for ExitStatus { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} - } -} - -impl fmt::Display for ExitStatus { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} - } -} - -#[derive(PartialEq, Eq, Clone, Copy, Debug)] -pub struct ExitCode(bool); - -impl ExitCode { - pub const SUCCESS: ExitCode = ExitCode(false); - pub const FAILURE: ExitCode = ExitCode(true); - - pub fn as_i32(&self) -> i32 { - self.0 as i32 - } -} - -pub struct Process(Void); - -impl Process { - pub fn id(&self) -> u32 { - match self.0 {} - } - - pub fn kill(&mut self) -> io::Result<()> { - match self.0 {} - } - - pub fn wait(&mut self) -> io::Result { - match self.0 {} - } - - pub fn try_wait(&mut self) -> io::Result> { - match self.0 {} - } -} diff --git a/crux-mir/lib/std/src/sys/sgx/rwlock.rs b/crux-mir/lib/std/src/sys/sgx/rwlock.rs index 722b4f5e0..d89de18ca 100644 --- a/crux-mir/lib/std/src/sys/sgx/rwlock.rs +++ b/crux-mir/lib/std/src/sys/sgx/rwlock.rs @@ -1,33 +1,51 @@ +#[cfg(test)] +mod tests; + use crate::num::NonZeroUsize; +use crate::sys_common::lazy_box::{LazyBox, LazyInit}; use super::waitqueue::{ try_lock_or_false, NotifiedTcs, SpinMutex, SpinMutexGuard, WaitQueue, WaitVariable, }; -use crate::mem; +use crate::alloc::Layout; -pub struct RWLock { +struct AllocatedRwLock { readers: SpinMutex>>, writer: SpinMutex>, } -// Check at compile time that RWLock size matches C definition (see test_c_rwlock_initializer below) -#[allow(dead_code)] -unsafe fn rw_lock_size_assert(r: RWLock) { - mem::transmute::(r); +pub struct RwLock { + inner: LazyBox, } -impl RWLock { - pub const fn new() -> RWLock { - RWLock { +impl LazyInit for AllocatedRwLock { + fn init() -> Box { + Box::new(AllocatedRwLock { readers: SpinMutex::new(WaitVariable::new(None)), writer: SpinMutex::new(WaitVariable::new(false)), - } + }) + } +} + +// Check at compile time that RwLock's size and alignment matches the C definition +// in libunwind (see also `test_c_rwlock_initializer` in `tests`). +const _: () = { + let rust = Layout::new::(); + let c = Layout::new::<*mut ()>(); + assert!(rust.size() == c.size()); + assert!(rust.align() == c.align()); +}; + +impl RwLock { + pub const fn new() -> RwLock { + RwLock { inner: LazyBox::new() } } #[inline] - pub unsafe fn read(&self) { - let mut rguard = self.readers.lock(); - let wguard = self.writer.lock(); + pub fn read(&self) { + let lock = &*self.inner; + let mut rguard = lock.readers.lock(); + let wguard = lock.writer.lock(); if *wguard.lock_var() || !wguard.queue_empty() { // Another thread has or is waiting for the write lock, wait drop(wguard); @@ -42,8 +60,9 @@ impl RWLock { #[inline] pub unsafe fn try_read(&self) -> bool { - let mut rguard = try_lock_or_false!(self.readers); - let wguard = try_lock_or_false!(self.writer); + let lock = &*self.inner; + let mut rguard = try_lock_or_false!(lock.readers); + let wguard = try_lock_or_false!(lock.writer); if *wguard.lock_var() || !wguard.queue_empty() { // Another thread has or is waiting for the write lock false @@ -56,9 +75,10 @@ impl RWLock { } #[inline] - pub unsafe fn write(&self) { - let rguard = self.readers.lock(); - let mut wguard = self.writer.lock(); + pub fn write(&self) { + let lock = &*self.inner; + let rguard = lock.readers.lock(); + let mut wguard = lock.writer.lock(); if *wguard.lock_var() || rguard.lock_var().is_some() { // Another thread has the lock, wait drop(rguard); @@ -71,9 +91,10 @@ impl RWLock { } #[inline] - pub unsafe fn try_write(&self) -> bool { - let rguard = try_lock_or_false!(self.readers); - let mut wguard = try_lock_or_false!(self.writer); + pub fn try_write(&self) -> bool { + let lock = &*self.inner; + let rguard = try_lock_or_false!(lock.readers); + let mut wguard = try_lock_or_false!(lock.writer); if *wguard.lock_var() || rguard.lock_var().is_some() { // Another thread has the lock false @@ -107,9 +128,10 @@ impl RWLock { #[inline] pub unsafe fn read_unlock(&self) { - let rguard = self.readers.lock(); - let wguard = self.writer.lock(); - self.__read_unlock(rguard, wguard); + let lock = &*self.inner; + let rguard = lock.readers.lock(); + let wguard = lock.writer.lock(); + unsafe { self.__read_unlock(rguard, wguard) }; } #[inline] @@ -143,26 +165,25 @@ impl RWLock { #[inline] pub unsafe fn write_unlock(&self) { - let rguard = self.readers.lock(); - let wguard = self.writer.lock(); - self.__write_unlock(rguard, wguard); + let lock = &*self.inner; + let rguard = lock.readers.lock(); + let wguard = lock.writer.lock(); + unsafe { self.__write_unlock(rguard, wguard) }; } // only used by __rust_rwlock_unlock below #[inline] #[cfg_attr(test, allow(dead_code))] unsafe fn unlock(&self) { - let rguard = self.readers.lock(); - let wguard = self.writer.lock(); + let lock = &*self.inner; + let rguard = lock.readers.lock(); + let wguard = lock.writer.lock(); if *wguard.lock_var() == true { - self.__write_unlock(rguard, wguard); + unsafe { self.__write_unlock(rguard, wguard) }; } else { - self.__read_unlock(rguard, wguard); + unsafe { self.__read_unlock(rguard, wguard) }; } } - - #[inline] - pub unsafe fn destroy(&self) {} } // The following functions are needed by libunwind. These symbols are named @@ -172,76 +193,30 @@ const EINVAL: i32 = 22; #[cfg(not(test))] #[no_mangle] -pub unsafe extern "C" fn __rust_rwlock_rdlock(p: *mut RWLock) -> i32 { +pub unsafe extern "C" fn __rust_rwlock_rdlock(p: *mut RwLock) -> i32 { if p.is_null() { return EINVAL; } - (*p).read(); + unsafe { (*p).read() }; return 0; } #[cfg(not(test))] #[no_mangle] -pub unsafe extern "C" fn __rust_rwlock_wrlock(p: *mut RWLock) -> i32 { +pub unsafe extern "C" fn __rust_rwlock_wrlock(p: *mut RwLock) -> i32 { if p.is_null() { return EINVAL; } - (*p).write(); + unsafe { (*p).write() }; return 0; } + #[cfg(not(test))] #[no_mangle] -pub unsafe extern "C" fn __rust_rwlock_unlock(p: *mut RWLock) -> i32 { +pub unsafe extern "C" fn __rust_rwlock_unlock(p: *mut RwLock) -> i32 { if p.is_null() { return EINVAL; } - (*p).unlock(); + unsafe { (*p).unlock() }; return 0; } - -#[cfg(test)] -mod tests { - use super::*; - use crate::mem::{self, MaybeUninit}; - use core::array::FixedSizeArray; - - // Verify that the bytes of initialized RWLock are the same as in - // libunwind. If they change, `src/UnwindRustSgx.h` in libunwind needs to - // be changed too. - #[test] - fn test_c_rwlock_initializer() { - #[rustfmt::skip] - const RWLOCK_INIT: &[u8] = &[ - /* 0x00 */ 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - /* 0x10 */ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - /* 0x20 */ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - /* 0x30 */ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - /* 0x40 */ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - /* 0x50 */ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - /* 0x60 */ 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - /* 0x70 */ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - /* 0x80 */ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - ]; - - #[inline(never)] - fn zero_stack() { - test::black_box(MaybeUninit::<[RWLock; 16]>::zeroed()); - } - - #[inline(never)] - unsafe fn rwlock_new(init: &mut MaybeUninit) { - init.write(RWLock::new()); - } - - unsafe { - // try hard to make sure that the padding/unused bytes in RWLock - // get initialized as 0. If the assertion below fails, that might - // just be an issue with the test code and not with the value of - // RWLOCK_INIT. - zero_stack(); - let mut init = MaybeUninit::::zeroed(); - rwlock_new(&mut init); - assert_eq!(mem::transmute::<_, [u8; 144]>(init.assume_init()).as_slice(), RWLOCK_INIT) - }; - } -} diff --git a/crux-mir/lib/std/src/sys/sgx/rwlock/tests.rs b/crux-mir/lib/std/src/sys/sgx/rwlock/tests.rs new file mode 100644 index 000000000..5fd6670af --- /dev/null +++ b/crux-mir/lib/std/src/sys/sgx/rwlock/tests.rs @@ -0,0 +1,21 @@ +use super::*; +use crate::ptr; + +// Verify that the byte pattern libunwind uses to initialize an RwLock is +// equivalent to the value of RwLock::new(). If the value changes, +// `src/UnwindRustSgx.h` in libunwind needs to be changed too. +#[test] +fn test_c_rwlock_initializer() { + const C_RWLOCK_INIT: *mut () = ptr::null_mut(); + + // For the test to work, we need the padding/unused bytes in RwLock to be + // initialized as 0. In practice, this is the case with statics. + static RUST_RWLOCK_INIT: RwLock = RwLock::new(); + + unsafe { + // If the assertion fails, that not necessarily an issue with the value + // of C_RWLOCK_INIT. It might just be an issue with the way padding + // bytes are initialized in the test code. + assert_eq!(crate::mem::transmute_copy::<_, *mut ()>(&RUST_RWLOCK_INIT), C_RWLOCK_INIT); + }; +} diff --git a/crux-mir/lib/std/src/sys/sgx/stack_overflow.rs b/crux-mir/lib/std/src/sys/sgx/stack_overflow.rs deleted file mode 100644 index a2d13d118..000000000 --- a/crux-mir/lib/std/src/sys/sgx/stack_overflow.rs +++ /dev/null @@ -1,12 +0,0 @@ -pub struct Handler; - -impl Handler { - pub unsafe fn new() -> Handler { - Handler - } -} - -#[cfg_attr(test, allow(dead_code))] -pub unsafe fn init() {} - -pub unsafe fn cleanup() {} diff --git a/crux-mir/lib/std/src/sys/sgx/stdio.rs b/crux-mir/lib/std/src/sys/sgx/stdio.rs index 716c174bd..2e680e740 100644 --- a/crux-mir/lib/std/src/sys/sgx/stdio.rs +++ b/crux-mir/lib/std/src/sys/sgx/stdio.rs @@ -19,8 +19,8 @@ fn with_std_fd R, R>(fd: abi::Fd, f: F) -> R { } impl Stdin { - pub fn new() -> io::Result { - Ok(Stdin(())) + pub const fn new() -> Stdin { + Stdin(()) } } @@ -31,8 +31,8 @@ impl io::Read for Stdin { } impl Stdout { - pub fn new() -> io::Result { - Ok(Stdout(())) + pub const fn new() -> Stdout { + Stdout(()) } } @@ -47,8 +47,8 @@ impl io::Write for Stdout { } impl Stderr { - pub fn new() -> io::Result { - Ok(Stderr(())) + pub const fn new() -> Stderr { + Stderr(()) } } @@ -65,7 +65,7 @@ impl io::Write for Stderr { pub const STDIN_BUF_SIZE: usize = crate::sys_common::io::DEFAULT_BUF_SIZE; pub fn is_ebadf(err: &io::Error) -> bool { - // FIXME: Rust normally maps Unix EBADF to `Other` + // FIXME: Rust normally maps Unix EBADF to `Uncategorized` err.raw_os_error() == Some(abi::Error::BrokenPipe as _) } @@ -81,8 +81,8 @@ pub unsafe extern "C" fn __rust_print_err(m: *mut u8, s: i32) { if s < 0 { return; } - let buf = slice::from_raw_parts(m as *const u8, s as _); + let buf = unsafe { slice::from_raw_parts(m as *const u8, s as _) }; if let Ok(s) = str::from_utf8(&buf[..buf.iter().position(|&b| b == 0).unwrap_or(buf.len())]) { - eprint!("{}", s); + eprint!("{s}"); } } diff --git a/crux-mir/lib/std/src/sys/sgx/thread.rs b/crux-mir/lib/std/src/sys/sgx/thread.rs index 9b515eb82..1608b8cb6 100644 --- a/crux-mir/lib/std/src/sys/sgx/thread.rs +++ b/crux-mir/lib/std/src/sys/sgx/thread.rs @@ -1,6 +1,8 @@ #![cfg_attr(test, allow(dead_code))] // why is this necessary? +use super::unsupported; use crate::ffi::CStr; use crate::io; +use crate::num::NonZeroUsize; use crate::time::Duration; use super::abi::usercalls; @@ -9,26 +11,37 @@ pub struct Thread(task_queue::JoinHandle); pub const DEFAULT_MIN_STACK_SIZE: usize = 4096; +pub use self::task_queue::JoinNotifier; + mod task_queue { - use crate::sync::mpsc; + use super::wait_notify; use crate::sync::{Mutex, MutexGuard, Once}; - pub type JoinHandle = mpsc::Receiver<()>; + pub type JoinHandle = wait_notify::Waiter; + + pub struct JoinNotifier(Option); + + impl Drop for JoinNotifier { + fn drop(&mut self) { + self.0.take().unwrap().notify(); + } + } pub(super) struct Task { p: Box, - done: mpsc::Sender<()>, + done: JoinNotifier, } impl Task { pub(super) fn new(p: Box) -> (Task, JoinHandle) { - let (done, recv) = mpsc::channel(); + let (done, recv) = wait_notify::new(); + let done = JoinNotifier(Some(done)); (Task { p, done }, recv) } - pub(super) fn run(self) { + pub(super) fn run(self) -> JoinNotifier { (self.p)(); - let _ = self.done.send(()); + self.done } } @@ -47,17 +60,56 @@ mod task_queue { } } +/// This module provides a synchronization primitive that does not use thread +/// local variables. This is needed for signaling that a thread has finished +/// execution. The signal is sent once all TLS destructors have finished at +/// which point no new thread locals should be created. +pub mod wait_notify { + use crate::pin::Pin; + use crate::sync::Arc; + use crate::sys_common::thread_parking::Parker; + + pub struct Notifier(Arc); + + impl Notifier { + /// Notify the waiter. The waiter is either notified right away (if + /// currently blocked in `Waiter::wait()`) or later when it calls the + /// `Waiter::wait()` method. + pub fn notify(self) { + Pin::new(&*self.0).unpark() + } + } + + pub struct Waiter(Arc); + + impl Waiter { + /// Wait for a notification. If `Notifier::notify()` has already been + /// called, this will return immediately, otherwise the current thread + /// is blocked until notified. + pub fn wait(self) { + // SAFETY: + // This is only ever called on one thread. + unsafe { Pin::new(&*self.0).park() } + } + } + + pub fn new() -> (Notifier, Waiter) { + let inner = Arc::new(Parker::new()); + (Notifier(inner.clone()), Waiter(inner)) + } +} + impl Thread { // unsafe: see thread::Builder::spawn_unchecked for safety requirements pub unsafe fn new(_stack: usize, p: Box) -> io::Result { let mut queue_lock = task_queue::lock(); - usercalls::launch_thread()?; + unsafe { usercalls::launch_thread()? }; let (task, handle) = task_queue::Task::new(p); queue_lock.push(task); Ok(Thread(handle)) } - pub(super) fn entry() { + pub(super) fn entry() -> JoinNotifier { let mut pending_tasks = task_queue::lock(); let task = rtunwrap!(Some, pending_tasks.pop()); drop(pending_tasks); // make sure to not hold the task queue lock longer than necessary @@ -73,15 +125,19 @@ impl Thread { // FIXME: could store this pointer in TLS somewhere } - pub fn sleep(_dur: Duration) { - rtabort!("can't sleep"); // FIXME + pub fn sleep(dur: Duration) { + usercalls::wait_timeout(0, dur, || true); } pub fn join(self) { - let _ = self.0.recv(); + self.0.wait(); } } +pub fn available_parallelism() -> io::Result { + unsupported() +} + pub mod guard { pub type Guard = !; pub unsafe fn current() -> Option { diff --git a/crux-mir/lib/std/src/sys/sgx/thread_local.rs b/crux-mir/lib/std/src/sys/sgx/thread_local_key.rs similarity index 86% rename from crux-mir/lib/std/src/sys/sgx/thread_local.rs rename to crux-mir/lib/std/src/sys/sgx/thread_local_key.rs index b21784475..c7a57d3a3 100644 --- a/crux-mir/lib/std/src/sys/sgx/thread_local.rs +++ b/crux-mir/lib/std/src/sys/sgx/thread_local_key.rs @@ -21,8 +21,3 @@ pub unsafe fn get(key: Key) -> *mut u8 { pub unsafe fn destroy(key: Key) { Tls::destroy(AbiKey::from_usize(key)) } - -#[inline] -pub fn requires_synchronized_create() -> bool { - false -} diff --git a/crux-mir/lib/std/src/sys/sgx/thread_parking.rs b/crux-mir/lib/std/src/sys/sgx/thread_parking.rs new file mode 100644 index 000000000..0006cd4f1 --- /dev/null +++ b/crux-mir/lib/std/src/sys/sgx/thread_parking.rs @@ -0,0 +1,23 @@ +use super::abi::usercalls; +use crate::io::ErrorKind; +use crate::time::Duration; +use fortanix_sgx_abi::{EV_UNPARK, WAIT_INDEFINITE}; + +pub type ThreadId = fortanix_sgx_abi::Tcs; + +pub use super::abi::thread::current; + +pub fn park(_hint: usize) { + usercalls::wait(EV_UNPARK, WAIT_INDEFINITE).unwrap(); +} + +pub fn park_timeout(dur: Duration, _hint: usize) { + let timeout = u128::min(dur.as_nanos(), WAIT_INDEFINITE as u128 - 1) as u64; + if let Err(e) = usercalls::wait(EV_UNPARK, timeout) { + assert!(matches!(e.kind(), ErrorKind::TimedOut | ErrorKind::WouldBlock)) + } +} + +pub fn unpark(tid: ThreadId, _hint: usize) { + let _ = usercalls::send(EV_UNPARK, Some(tid)); +} diff --git a/crux-mir/lib/std/src/sys/sgx/time.rs b/crux-mir/lib/std/src/sys/sgx/time.rs index e2f6e6dba..db4cf2804 100644 --- a/crux-mir/lib/std/src/sys/sgx/time.rs +++ b/crux-mir/lib/std/src/sys/sgx/time.rs @@ -25,14 +25,6 @@ impl Instant { pub fn checked_sub_duration(&self, other: &Duration) -> Option { Some(Instant(self.0.checked_sub(*other)?)) } - - pub fn actually_monotonic() -> bool { - false - } - - pub const fn zero() -> Instant { - Instant(Duration::from_secs(0)) - } } impl SystemTime { diff --git a/crux-mir/lib/std/src/sys/sgx/waitqueue.rs b/crux-mir/lib/std/src/sys/sgx/waitqueue.rs deleted file mode 100644 index 6e50f161b..000000000 --- a/crux-mir/lib/std/src/sys/sgx/waitqueue.rs +++ /dev/null @@ -1,524 +0,0 @@ -use crate::num::NonZeroUsize; -/// A simple queue implementation for synchronization primitives. -/// -/// This queue is used to implement condition variable and mutexes. -/// -/// Users of this API are expected to use the `WaitVariable` type. Since -/// that type is not `Sync`, it needs to be protected by e.g., a `SpinMutex` to -/// allow shared access. -/// -/// Since userspace may send spurious wake-ups, the wakeup event state is -/// recorded in the enclave. The wakeup event state is protected by a spinlock. -/// The queue and associated wait state are stored in a `WaitVariable`. -use crate::ops::{Deref, DerefMut}; - -use super::abi::thread; -use super::abi::usercalls; -use fortanix_sgx_abi::{Tcs, EV_UNPARK, WAIT_INDEFINITE}; - -pub use self::spin_mutex::{try_lock_or_false, SpinMutex, SpinMutexGuard}; -use self::unsafe_list::{UnsafeList, UnsafeListEntry}; - -/// An queue entry in a `WaitQueue`. -struct WaitEntry { - /// TCS address of the thread that is waiting - tcs: Tcs, - /// Whether this thread has been notified to be awoken - wake: bool, -} - -/// Data stored with a `WaitQueue` alongside it. This ensures accesses to the -/// queue and the data are synchronized, since the type itself is not `Sync`. -/// -/// Consumers of this API should use a synchronization primitive for shared -/// access, such as `SpinMutex`. -#[derive(Default)] -pub struct WaitVariable { - queue: WaitQueue, - lock: T, -} - -impl WaitVariable { - pub const fn new(var: T) -> Self { - WaitVariable { queue: WaitQueue::new(), lock: var } - } - - pub fn queue_empty(&self) -> bool { - self.queue.is_empty() - } - - pub fn lock_var(&self) -> &T { - &self.lock - } - - pub fn lock_var_mut(&mut self) -> &mut T { - &mut self.lock - } -} - -#[derive(Copy, Clone)] -pub enum NotifiedTcs { - Single(Tcs), - All { count: NonZeroUsize }, -} - -/// An RAII guard that will notify a set of target threads as well as unlock -/// a mutex on drop. -pub struct WaitGuard<'a, T: 'a> { - mutex_guard: Option>>, - notified_tcs: NotifiedTcs, -} - -/// A queue of threads that are waiting on some synchronization primitive. -/// -/// `UnsafeList` entries are allocated on the waiting thread's stack. This -/// avoids any global locking that might happen in the heap allocator. This is -/// safe because the waiting thread will not return from that stack frame until -/// after it is notified. The notifying thread ensures to clean up any -/// references to the list entries before sending the wakeup event. -pub struct WaitQueue { - // We use an inner Mutex here to protect the data in the face of spurious - // wakeups. - inner: UnsafeList>, -} -unsafe impl Send for WaitQueue {} - -impl Default for WaitQueue { - fn default() -> Self { - Self::new() - } -} - -impl<'a, T> WaitGuard<'a, T> { - /// Returns which TCSes will be notified when this guard drops. - pub fn notified_tcs(&self) -> NotifiedTcs { - self.notified_tcs - } - - /// Drop this `WaitGuard`, after dropping another `guard`. - pub fn drop_after(self, guard: U) { - drop(guard); - drop(self); - } -} - -impl<'a, T> Deref for WaitGuard<'a, T> { - type Target = SpinMutexGuard<'a, WaitVariable>; - - fn deref(&self) -> &Self::Target { - self.mutex_guard.as_ref().unwrap() - } -} - -impl<'a, T> DerefMut for WaitGuard<'a, T> { - fn deref_mut(&mut self) -> &mut Self::Target { - self.mutex_guard.as_mut().unwrap() - } -} - -impl<'a, T> Drop for WaitGuard<'a, T> { - fn drop(&mut self) { - drop(self.mutex_guard.take()); - let target_tcs = match self.notified_tcs { - NotifiedTcs::Single(tcs) => Some(tcs), - NotifiedTcs::All { .. } => None, - }; - rtunwrap!(Ok, usercalls::send(EV_UNPARK, target_tcs)); - } -} - -impl WaitQueue { - pub const fn new() -> Self { - WaitQueue { inner: UnsafeList::new() } - } - - pub fn is_empty(&self) -> bool { - self.inner.is_empty() - } - - /// Adds the calling thread to the `WaitVariable`'s wait queue, then wait - /// until a wakeup event. - /// - /// This function does not return until this thread has been awoken. - pub fn wait(mut guard: SpinMutexGuard<'_, WaitVariable>, before_wait: F) { - // very unsafe: check requirements of UnsafeList::push - unsafe { - let mut entry = UnsafeListEntry::new(SpinMutex::new(WaitEntry { - tcs: thread::current(), - wake: false, - })); - let entry = guard.queue.inner.push(&mut entry); - drop(guard); - before_wait(); - while !entry.lock().wake { - // don't panic, this would invalidate `entry` during unwinding - let eventset = rtunwrap!(Ok, usercalls::wait(EV_UNPARK, WAIT_INDEFINITE)); - rtassert!(eventset & EV_UNPARK == EV_UNPARK); - } - } - } - - /// Either find the next waiter on the wait queue, or return the mutex - /// guard unchanged. - /// - /// If a waiter is found, a `WaitGuard` is returned which will notify the - /// waiter when it is dropped. - pub fn notify_one( - mut guard: SpinMutexGuard<'_, WaitVariable>, - ) -> Result, SpinMutexGuard<'_, WaitVariable>> { - unsafe { - if let Some(entry) = guard.queue.inner.pop() { - let mut entry_guard = entry.lock(); - let tcs = entry_guard.tcs; - entry_guard.wake = true; - drop(entry); - Ok(WaitGuard { mutex_guard: Some(guard), notified_tcs: NotifiedTcs::Single(tcs) }) - } else { - Err(guard) - } - } - } - - /// Either find any and all waiters on the wait queue, or return the mutex - /// guard unchanged. - /// - /// If at least one waiter is found, a `WaitGuard` is returned which will - /// notify all waiters when it is dropped. - pub fn notify_all( - mut guard: SpinMutexGuard<'_, WaitVariable>, - ) -> Result, SpinMutexGuard<'_, WaitVariable>> { - unsafe { - let mut count = 0; - while let Some(entry) = guard.queue.inner.pop() { - count += 1; - let mut entry_guard = entry.lock(); - entry_guard.wake = true; - } - if let Some(count) = NonZeroUsize::new(count) { - Ok(WaitGuard { mutex_guard: Some(guard), notified_tcs: NotifiedTcs::All { count } }) - } else { - Err(guard) - } - } - } -} - -/// A doubly-linked list where callers are in charge of memory allocation -/// of the nodes in the list. -mod unsafe_list { - use crate::mem; - use crate::ptr::NonNull; - - pub struct UnsafeListEntry { - next: NonNull>, - prev: NonNull>, - value: Option, - } - - impl UnsafeListEntry { - fn dummy() -> Self { - UnsafeListEntry { next: NonNull::dangling(), prev: NonNull::dangling(), value: None } - } - - pub fn new(value: T) -> Self { - UnsafeListEntry { value: Some(value), ..Self::dummy() } - } - } - - pub struct UnsafeList { - head_tail: NonNull>, - head_tail_entry: Option>, - } - - impl UnsafeList { - pub const fn new() -> Self { - unsafe { - UnsafeList { head_tail: NonNull::new_unchecked(1 as _), head_tail_entry: None } - } - } - - unsafe fn init(&mut self) { - if self.head_tail_entry.is_none() { - self.head_tail_entry = Some(UnsafeListEntry::dummy()); - self.head_tail = NonNull::new_unchecked(self.head_tail_entry.as_mut().unwrap()); - self.head_tail.as_mut().next = self.head_tail; - self.head_tail.as_mut().prev = self.head_tail; - } - } - - pub fn is_empty(&self) -> bool { - unsafe { - if self.head_tail_entry.is_some() { - let first = self.head_tail.as_ref().next; - if first == self.head_tail { - // ,-------> /---------\ next ---, - // | |head_tail| | - // `--- prev \---------/ <-------` - rtassert!(self.head_tail.as_ref().prev == first); - true - } else { - false - } - } else { - true - } - } - } - - /// Pushes an entry onto the back of the list. - /// - /// # Safety - /// - /// The entry must remain allocated until the entry is removed from the - /// list AND the caller who popped is done using the entry. Special - /// care must be taken in the caller of `push` to ensure unwinding does - /// not destroy the stack frame containing the entry. - pub unsafe fn push<'a>(&mut self, entry: &'a mut UnsafeListEntry) -> &'a T { - self.init(); - - // BEFORE: - // /---------\ next ---> /---------\ - // ... |prev_tail| |head_tail| ... - // \---------/ <--- prev \---------/ - // - // AFTER: - // /---------\ next ---> /-----\ next ---> /---------\ - // ... |prev_tail| |entry| |head_tail| ... - // \---------/ <--- prev \-----/ <--- prev \---------/ - let mut entry = NonNull::new_unchecked(entry); - let mut prev_tail = mem::replace(&mut self.head_tail.as_mut().prev, entry); - entry.as_mut().prev = prev_tail; - entry.as_mut().next = self.head_tail; - prev_tail.as_mut().next = entry; - // unwrap ok: always `Some` on non-dummy entries - (*entry.as_ptr()).value.as_ref().unwrap() - } - - /// Pops an entry from the front of the list. - /// - /// # Safety - /// - /// The caller must make sure to synchronize ending the borrow of the - /// return value and deallocation of the containing entry. - pub unsafe fn pop<'a>(&mut self) -> Option<&'a T> { - self.init(); - - if self.is_empty() { - None - } else { - // BEFORE: - // /---------\ next ---> /-----\ next ---> /------\ - // ... |head_tail| |first| |second| ... - // \---------/ <--- prev \-----/ <--- prev \------/ - // - // AFTER: - // /---------\ next ---> /------\ - // ... |head_tail| |second| ... - // \---------/ <--- prev \------/ - let mut first = self.head_tail.as_mut().next; - let mut second = first.as_mut().next; - self.head_tail.as_mut().next = second; - second.as_mut().prev = self.head_tail; - first.as_mut().next = NonNull::dangling(); - first.as_mut().prev = NonNull::dangling(); - // unwrap ok: always `Some` on non-dummy entries - Some((*first.as_ptr()).value.as_ref().unwrap()) - } - } - } - - #[cfg(test)] - mod tests { - use super::*; - use crate::cell::Cell; - - unsafe fn assert_empty(list: &mut UnsafeList) { - assert!(list.pop().is_none(), "assertion failed: list is not empty"); - } - - #[test] - fn init_empty() { - unsafe { - assert_empty(&mut UnsafeList::::new()); - } - } - - #[test] - fn push_pop() { - unsafe { - let mut node = UnsafeListEntry::new(1234); - let mut list = UnsafeList::new(); - assert_eq!(list.push(&mut node), &1234); - assert_eq!(list.pop().unwrap(), &1234); - assert_empty(&mut list); - } - } - - #[test] - fn complex_pushes_pops() { - unsafe { - let mut node1 = UnsafeListEntry::new(1234); - let mut node2 = UnsafeListEntry::new(4567); - let mut node3 = UnsafeListEntry::new(9999); - let mut node4 = UnsafeListEntry::new(8642); - let mut list = UnsafeList::new(); - list.push(&mut node1); - list.push(&mut node2); - assert_eq!(list.pop().unwrap(), &1234); - list.push(&mut node3); - assert_eq!(list.pop().unwrap(), &4567); - assert_eq!(list.pop().unwrap(), &9999); - assert_empty(&mut list); - list.push(&mut node4); - assert_eq!(list.pop().unwrap(), &8642); - assert_empty(&mut list); - } - } - - #[test] - fn cell() { - unsafe { - let mut node = UnsafeListEntry::new(Cell::new(0)); - let mut list = UnsafeList::new(); - let noderef = list.push(&mut node); - assert_eq!(noderef.get(), 0); - list.pop().unwrap().set(1); - assert_empty(&mut list); - assert_eq!(noderef.get(), 1); - } - } - } -} - -/// Trivial spinlock-based implementation of `sync::Mutex`. -// FIXME: Perhaps use Intel TSX to avoid locking? -mod spin_mutex { - use crate::cell::UnsafeCell; - use crate::ops::{Deref, DerefMut}; - use crate::sync::atomic::{spin_loop_hint, AtomicBool, Ordering}; - - #[derive(Default)] - pub struct SpinMutex { - value: UnsafeCell, - lock: AtomicBool, - } - - unsafe impl Send for SpinMutex {} - unsafe impl Sync for SpinMutex {} - - pub struct SpinMutexGuard<'a, T: 'a> { - mutex: &'a SpinMutex, - } - - impl<'a, T> !Send for SpinMutexGuard<'a, T> {} - unsafe impl<'a, T: Sync> Sync for SpinMutexGuard<'a, T> {} - - impl SpinMutex { - pub const fn new(value: T) -> Self { - SpinMutex { value: UnsafeCell::new(value), lock: AtomicBool::new(false) } - } - - #[inline(always)] - pub fn lock(&self) -> SpinMutexGuard<'_, T> { - loop { - match self.try_lock() { - None => { - while self.lock.load(Ordering::Relaxed) { - spin_loop_hint() - } - } - Some(guard) => return guard, - } - } - } - - #[inline(always)] - pub fn try_lock(&self) -> Option> { - if !self.lock.compare_and_swap(false, true, Ordering::Acquire) { - Some(SpinMutexGuard { mutex: self }) - } else { - None - } - } - } - - /// Lock the Mutex or return false. - pub macro try_lock_or_false($e:expr) { - if let Some(v) = $e.try_lock() { v } else { return false } - } - - impl<'a, T> Deref for SpinMutexGuard<'a, T> { - type Target = T; - - fn deref(&self) -> &T { - unsafe { &*self.mutex.value.get() } - } - } - - impl<'a, T> DerefMut for SpinMutexGuard<'a, T> { - fn deref_mut(&mut self) -> &mut T { - unsafe { &mut *self.mutex.value.get() } - } - } - - impl<'a, T> Drop for SpinMutexGuard<'a, T> { - fn drop(&mut self) { - self.mutex.lock.store(false, Ordering::Release) - } - } - - #[cfg(test)] - mod tests { - #![allow(deprecated)] - - use super::*; - use crate::sync::Arc; - use crate::thread; - use crate::time::{Duration, SystemTime}; - - #[test] - fn sleep() { - let mutex = Arc::new(SpinMutex::::default()); - let mutex2 = mutex.clone(); - let guard = mutex.lock(); - let t1 = thread::spawn(move || { - *mutex2.lock() = 1; - }); - - // "sleep" for 50ms - // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - let start = SystemTime::now(); - let max = Duration::from_millis(50); - while start.elapsed().unwrap() < max {} - - assert_eq!(*guard, 0); - drop(guard); - t1.join().unwrap(); - assert_eq!(*mutex.lock(), 1); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::sync::Arc; - use crate::thread; - - #[test] - fn queue() { - let wq = Arc::new(SpinMutex::>::default()); - let wq2 = wq.clone(); - - let locked = wq.lock(); - - let t1 = thread::spawn(move || { - // if we obtain the lock, the main thread should be waiting - assert!(WaitQueue::notify_one(wq2.lock()).is_ok()); - }); - - WaitQueue::wait(locked, || {}); - - t1.join().unwrap(); - } -} diff --git a/crux-mir/lib/std/src/sys/sgx/waitqueue/mod.rs b/crux-mir/lib/std/src/sys/sgx/waitqueue/mod.rs new file mode 100644 index 000000000..61bb11d9a --- /dev/null +++ b/crux-mir/lib/std/src/sys/sgx/waitqueue/mod.rs @@ -0,0 +1,240 @@ +//! A simple queue implementation for synchronization primitives. +//! +//! This queue is used to implement condition variable and mutexes. +//! +//! Users of this API are expected to use the `WaitVariable` type. Since +//! that type is not `Sync`, it needs to be protected by e.g., a `SpinMutex` to +//! allow shared access. +//! +//! Since userspace may send spurious wake-ups, the wakeup event state is +//! recorded in the enclave. The wakeup event state is protected by a spinlock. +//! The queue and associated wait state are stored in a `WaitVariable`. + +#[cfg(test)] +mod tests; + +mod spin_mutex; +mod unsafe_list; + +use crate::num::NonZeroUsize; +use crate::ops::{Deref, DerefMut}; +use crate::time::Duration; + +use super::abi::thread; +use super::abi::usercalls; +use fortanix_sgx_abi::{Tcs, EV_UNPARK, WAIT_INDEFINITE}; + +pub use self::spin_mutex::{try_lock_or_false, SpinMutex, SpinMutexGuard}; +use self::unsafe_list::{UnsafeList, UnsafeListEntry}; + +/// An queue entry in a `WaitQueue`. +struct WaitEntry { + /// TCS address of the thread that is waiting + tcs: Tcs, + /// Whether this thread has been notified to be awoken + wake: bool, +} + +/// Data stored with a `WaitQueue` alongside it. This ensures accesses to the +/// queue and the data are synchronized, since the type itself is not `Sync`. +/// +/// Consumers of this API should use a synchronization primitive for shared +/// access, such as `SpinMutex`. +#[derive(Default)] +pub struct WaitVariable { + queue: WaitQueue, + lock: T, +} + +impl WaitVariable { + pub const fn new(var: T) -> Self { + WaitVariable { queue: WaitQueue::new(), lock: var } + } + + pub fn queue_empty(&self) -> bool { + self.queue.is_empty() + } + + pub fn lock_var(&self) -> &T { + &self.lock + } + + pub fn lock_var_mut(&mut self) -> &mut T { + &mut self.lock + } +} + +#[derive(Copy, Clone)] +pub enum NotifiedTcs { + Single(Tcs), + All { count: NonZeroUsize }, +} + +/// An RAII guard that will notify a set of target threads as well as unlock +/// a mutex on drop. +pub struct WaitGuard<'a, T: 'a> { + mutex_guard: Option>>, + notified_tcs: NotifiedTcs, +} + +/// A queue of threads that are waiting on some synchronization primitive. +/// +/// `UnsafeList` entries are allocated on the waiting thread's stack. This +/// avoids any global locking that might happen in the heap allocator. This is +/// safe because the waiting thread will not return from that stack frame until +/// after it is notified. The notifying thread ensures to clean up any +/// references to the list entries before sending the wakeup event. +pub struct WaitQueue { + // We use an inner Mutex here to protect the data in the face of spurious + // wakeups. + inner: UnsafeList>, +} +unsafe impl Send for WaitQueue {} + +impl Default for WaitQueue { + fn default() -> Self { + Self::new() + } +} + +impl<'a, T> WaitGuard<'a, T> { + /// Returns which TCSes will be notified when this guard drops. + pub fn notified_tcs(&self) -> NotifiedTcs { + self.notified_tcs + } + + /// Drop this `WaitGuard`, after dropping another `guard`. + pub fn drop_after(self, guard: U) { + drop(guard); + drop(self); + } +} + +impl<'a, T> Deref for WaitGuard<'a, T> { + type Target = SpinMutexGuard<'a, WaitVariable>; + + fn deref(&self) -> &Self::Target { + self.mutex_guard.as_ref().unwrap() + } +} + +impl<'a, T> DerefMut for WaitGuard<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.mutex_guard.as_mut().unwrap() + } +} + +impl<'a, T> Drop for WaitGuard<'a, T> { + fn drop(&mut self) { + drop(self.mutex_guard.take()); + let target_tcs = match self.notified_tcs { + NotifiedTcs::Single(tcs) => Some(tcs), + NotifiedTcs::All { .. } => None, + }; + rtunwrap!(Ok, usercalls::send(EV_UNPARK, target_tcs)); + } +} + +impl WaitQueue { + pub const fn new() -> Self { + WaitQueue { inner: UnsafeList::new() } + } + + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + /// Adds the calling thread to the `WaitVariable`'s wait queue, then wait + /// until a wakeup event. + /// + /// This function does not return until this thread has been awoken. + pub fn wait(mut guard: SpinMutexGuard<'_, WaitVariable>, before_wait: F) { + // very unsafe: check requirements of UnsafeList::push + unsafe { + let mut entry = UnsafeListEntry::new(SpinMutex::new(WaitEntry { + tcs: thread::current(), + wake: false, + })); + let entry = guard.queue.inner.push(&mut entry); + drop(guard); + before_wait(); + while !entry.lock().wake { + // don't panic, this would invalidate `entry` during unwinding + let eventset = rtunwrap!(Ok, usercalls::wait(EV_UNPARK, WAIT_INDEFINITE)); + rtassert!(eventset & EV_UNPARK == EV_UNPARK); + } + } + } + + /// Adds the calling thread to the `WaitVariable`'s wait queue, then wait + /// until a wakeup event or timeout. If event was observed, returns true. + /// If not, it will remove the calling thread from the wait queue. + pub fn wait_timeout( + lock: &SpinMutex>, + timeout: Duration, + before_wait: F, + ) -> bool { + // very unsafe: check requirements of UnsafeList::push + unsafe { + let mut entry = UnsafeListEntry::new(SpinMutex::new(WaitEntry { + tcs: thread::current(), + wake: false, + })); + let entry_lock = lock.lock().queue.inner.push(&mut entry); + before_wait(); + usercalls::wait_timeout(EV_UNPARK, timeout, || entry_lock.lock().wake); + // acquire the wait queue's lock first to avoid deadlock. + let mut guard = lock.lock(); + let success = entry_lock.lock().wake; + if !success { + // nobody is waking us up, so remove our entry from the wait queue. + guard.queue.inner.remove(&mut entry); + } + success + } + } + + /// Either find the next waiter on the wait queue, or return the mutex + /// guard unchanged. + /// + /// If a waiter is found, a `WaitGuard` is returned which will notify the + /// waiter when it is dropped. + pub fn notify_one( + mut guard: SpinMutexGuard<'_, WaitVariable>, + ) -> Result, SpinMutexGuard<'_, WaitVariable>> { + unsafe { + if let Some(entry) = guard.queue.inner.pop() { + let mut entry_guard = entry.lock(); + let tcs = entry_guard.tcs; + entry_guard.wake = true; + drop(entry); + Ok(WaitGuard { mutex_guard: Some(guard), notified_tcs: NotifiedTcs::Single(tcs) }) + } else { + Err(guard) + } + } + } + + /// Either find any and all waiters on the wait queue, or return the mutex + /// guard unchanged. + /// + /// If at least one waiter is found, a `WaitGuard` is returned which will + /// notify all waiters when it is dropped. + pub fn notify_all( + mut guard: SpinMutexGuard<'_, WaitVariable>, + ) -> Result, SpinMutexGuard<'_, WaitVariable>> { + unsafe { + let mut count = 0; + while let Some(entry) = guard.queue.inner.pop() { + count += 1; + let mut entry_guard = entry.lock(); + entry_guard.wake = true; + } + if let Some(count) = NonZeroUsize::new(count) { + Ok(WaitGuard { mutex_guard: Some(guard), notified_tcs: NotifiedTcs::All { count } }) + } else { + Err(guard) + } + } + } +} diff --git a/crux-mir/lib/std/src/sys/sgx/waitqueue/spin_mutex.rs b/crux-mir/lib/std/src/sys/sgx/waitqueue/spin_mutex.rs new file mode 100644 index 000000000..f6e851cca --- /dev/null +++ b/crux-mir/lib/std/src/sys/sgx/waitqueue/spin_mutex.rs @@ -0,0 +1,80 @@ +//! Trivial spinlock-based implementation of `sync::Mutex`. +// FIXME: Perhaps use Intel TSX to avoid locking? + +#[cfg(test)] +mod tests; + +use crate::cell::UnsafeCell; +use crate::hint; +use crate::ops::{Deref, DerefMut}; +use crate::sync::atomic::{AtomicBool, Ordering}; + +#[derive(Default)] +pub struct SpinMutex { + value: UnsafeCell, + lock: AtomicBool, +} + +unsafe impl Send for SpinMutex {} +unsafe impl Sync for SpinMutex {} + +pub struct SpinMutexGuard<'a, T: 'a> { + mutex: &'a SpinMutex, +} + +impl<'a, T> !Send for SpinMutexGuard<'a, T> {} +unsafe impl<'a, T: Sync> Sync for SpinMutexGuard<'a, T> {} + +impl SpinMutex { + pub const fn new(value: T) -> Self { + SpinMutex { value: UnsafeCell::new(value), lock: AtomicBool::new(false) } + } + + #[inline(always)] + pub fn lock(&self) -> SpinMutexGuard<'_, T> { + loop { + match self.try_lock() { + None => { + while self.lock.load(Ordering::Relaxed) { + hint::spin_loop() + } + } + Some(guard) => return guard, + } + } + } + + #[inline(always)] + pub fn try_lock(&self) -> Option> { + if self.lock.compare_exchange(false, true, Ordering::Acquire, Ordering::Acquire).is_ok() { + Some(SpinMutexGuard { mutex: self }) + } else { + None + } + } +} + +/// Lock the Mutex or return false. +pub macro try_lock_or_false($e:expr) { + if let Some(v) = $e.try_lock() { v } else { return false } +} + +impl<'a, T> Deref for SpinMutexGuard<'a, T> { + type Target = T; + + fn deref(&self) -> &T { + unsafe { &*self.mutex.value.get() } + } +} + +impl<'a, T> DerefMut for SpinMutexGuard<'a, T> { + fn deref_mut(&mut self) -> &mut T { + unsafe { &mut *self.mutex.value.get() } + } +} + +impl<'a, T> Drop for SpinMutexGuard<'a, T> { + fn drop(&mut self) { + self.mutex.lock.store(false, Ordering::Release) + } +} diff --git a/crux-mir/lib/std/src/sys/sgx/waitqueue/spin_mutex/tests.rs b/crux-mir/lib/std/src/sys/sgx/waitqueue/spin_mutex/tests.rs new file mode 100644 index 000000000..4c5994bea --- /dev/null +++ b/crux-mir/lib/std/src/sys/sgx/waitqueue/spin_mutex/tests.rs @@ -0,0 +1,23 @@ +#![allow(deprecated)] + +use super::*; +use crate::sync::Arc; +use crate::thread; +use crate::time::Duration; + +#[test] +fn sleep() { + let mutex = Arc::new(SpinMutex::::default()); + let mutex2 = mutex.clone(); + let guard = mutex.lock(); + let t1 = thread::spawn(move || { + *mutex2.lock() = 1; + }); + + thread::sleep(Duration::from_millis(50)); + + assert_eq!(*guard, 0); + drop(guard); + t1.join().unwrap(); + assert_eq!(*mutex.lock(), 1); +} diff --git a/crux-mir/lib/std/src/sys/sgx/waitqueue/tests.rs b/crux-mir/lib/std/src/sys/sgx/waitqueue/tests.rs new file mode 100644 index 000000000..bf91fdd08 --- /dev/null +++ b/crux-mir/lib/std/src/sys/sgx/waitqueue/tests.rs @@ -0,0 +1,20 @@ +use super::*; +use crate::sync::Arc; +use crate::thread; + +#[test] +fn queue() { + let wq = Arc::new(SpinMutex::>::default()); + let wq2 = wq.clone(); + + let locked = wq.lock(); + + let t1 = thread::spawn(move || { + // if we obtain the lock, the main thread should be waiting + assert!(WaitQueue::notify_one(wq2.lock()).is_ok()); + }); + + WaitQueue::wait(locked, || {}); + + t1.join().unwrap(); +} diff --git a/crux-mir/lib/std/src/sys/sgx/waitqueue/unsafe_list.rs b/crux-mir/lib/std/src/sys/sgx/waitqueue/unsafe_list.rs new file mode 100644 index 000000000..c736cab57 --- /dev/null +++ b/crux-mir/lib/std/src/sys/sgx/waitqueue/unsafe_list.rs @@ -0,0 +1,156 @@ +//! A doubly-linked list where callers are in charge of memory allocation +//! of the nodes in the list. + +#[cfg(test)] +mod tests; + +use crate::mem; +use crate::ptr::NonNull; + +pub struct UnsafeListEntry { + next: NonNull>, + prev: NonNull>, + value: Option, +} + +impl UnsafeListEntry { + fn dummy() -> Self { + UnsafeListEntry { next: NonNull::dangling(), prev: NonNull::dangling(), value: None } + } + + pub fn new(value: T) -> Self { + UnsafeListEntry { value: Some(value), ..Self::dummy() } + } +} + +// WARNING: self-referential struct! +pub struct UnsafeList { + head_tail: NonNull>, + head_tail_entry: Option>, +} + +impl UnsafeList { + pub const fn new() -> Self { + unsafe { UnsafeList { head_tail: NonNull::new_unchecked(1 as _), head_tail_entry: None } } + } + + /// # Safety + unsafe fn init(&mut self) { + if self.head_tail_entry.is_none() { + self.head_tail_entry = Some(UnsafeListEntry::dummy()); + // SAFETY: `head_tail_entry` must be non-null, which it is because we assign it above. + self.head_tail = + unsafe { NonNull::new_unchecked(self.head_tail_entry.as_mut().unwrap()) }; + // SAFETY: `self.head_tail` must meet all requirements for a mutable reference. + unsafe { self.head_tail.as_mut() }.next = self.head_tail; + unsafe { self.head_tail.as_mut() }.prev = self.head_tail; + } + } + + pub fn is_empty(&self) -> bool { + if self.head_tail_entry.is_some() { + let first = unsafe { self.head_tail.as_ref() }.next; + if first == self.head_tail { + // ,-------> /---------\ next ---, + // | |head_tail| | + // `--- prev \---------/ <-------` + // SAFETY: `self.head_tail` must meet all requirements for a reference. + unsafe { rtassert!(self.head_tail.as_ref().prev == first) }; + true + } else { + false + } + } else { + true + } + } + + /// Pushes an entry onto the back of the list. + /// + /// # Safety + /// + /// The entry must remain allocated until the entry is removed from the + /// list AND the caller who popped is done using the entry. Special + /// care must be taken in the caller of `push` to ensure unwinding does + /// not destroy the stack frame containing the entry. + pub unsafe fn push<'a>(&mut self, entry: &'a mut UnsafeListEntry) -> &'a T { + unsafe { self.init() }; + + // BEFORE: + // /---------\ next ---> /---------\ + // ... |prev_tail| |head_tail| ... + // \---------/ <--- prev \---------/ + // + // AFTER: + // /---------\ next ---> /-----\ next ---> /---------\ + // ... |prev_tail| |entry| |head_tail| ... + // \---------/ <--- prev \-----/ <--- prev \---------/ + let mut entry = unsafe { NonNull::new_unchecked(entry) }; + let mut prev_tail = mem::replace(&mut unsafe { self.head_tail.as_mut() }.prev, entry); + // SAFETY: `entry` must meet all requirements for a mutable reference. + unsafe { entry.as_mut() }.prev = prev_tail; + unsafe { entry.as_mut() }.next = self.head_tail; + // SAFETY: `prev_tail` must meet all requirements for a mutable reference. + unsafe { prev_tail.as_mut() }.next = entry; + // unwrap ok: always `Some` on non-dummy entries + unsafe { (*entry.as_ptr()).value.as_ref() }.unwrap() + } + + /// Pops an entry from the front of the list. + /// + /// # Safety + /// + /// The caller must make sure to synchronize ending the borrow of the + /// return value and deallocation of the containing entry. + pub unsafe fn pop<'a>(&mut self) -> Option<&'a T> { + unsafe { self.init() }; + + if self.is_empty() { + None + } else { + // BEFORE: + // /---------\ next ---> /-----\ next ---> /------\ + // ... |head_tail| |first| |second| ... + // \---------/ <--- prev \-----/ <--- prev \------/ + // + // AFTER: + // /---------\ next ---> /------\ + // ... |head_tail| |second| ... + // \---------/ <--- prev \------/ + let mut first = unsafe { self.head_tail.as_mut() }.next; + let mut second = unsafe { first.as_mut() }.next; + unsafe { self.head_tail.as_mut() }.next = second; + unsafe { second.as_mut() }.prev = self.head_tail; + unsafe { first.as_mut() }.next = NonNull::dangling(); + unsafe { first.as_mut() }.prev = NonNull::dangling(); + // unwrap ok: always `Some` on non-dummy entries + Some(unsafe { (*first.as_ptr()).value.as_ref() }.unwrap()) + } + } + + /// Removes an entry from the list. + /// + /// # Safety + /// + /// The caller must ensure that `entry` has been pushed onto `self` + /// prior to this call and has not moved since then. + pub unsafe fn remove(&mut self, entry: &mut UnsafeListEntry) { + rtassert!(!self.is_empty()); + // BEFORE: + // /----\ next ---> /-----\ next ---> /----\ + // ... |prev| |entry| |next| ... + // \----/ <--- prev \-----/ <--- prev \----/ + // + // AFTER: + // /----\ next ---> /----\ + // ... |prev| |next| ... + // \----/ <--- prev \----/ + let mut prev = entry.prev; + let mut next = entry.next; + // SAFETY: `prev` and `next` must meet all requirements for a mutable reference.entry + unsafe { prev.as_mut() }.next = next; + unsafe { next.as_mut() }.prev = prev; + entry.next = NonNull::dangling(); + entry.prev = NonNull::dangling(); + } +} diff --git a/crux-mir/lib/std/src/sys/sgx/waitqueue/unsafe_list/tests.rs b/crux-mir/lib/std/src/sys/sgx/waitqueue/unsafe_list/tests.rs new file mode 100644 index 000000000..c653dee17 --- /dev/null +++ b/crux-mir/lib/std/src/sys/sgx/waitqueue/unsafe_list/tests.rs @@ -0,0 +1,105 @@ +use super::*; +use crate::cell::Cell; + +/// # Safety +/// List must be valid. +unsafe fn assert_empty(list: &mut UnsafeList) { + assert!(unsafe { list.pop() }.is_none(), "assertion failed: list is not empty"); +} + +#[test] +fn init_empty() { + unsafe { + assert_empty(&mut UnsafeList::::new()); + } +} + +#[test] +fn push_pop() { + unsafe { + let mut node = UnsafeListEntry::new(1234); + let mut list = UnsafeList::new(); + assert_eq!(list.push(&mut node), &1234); + assert_eq!(list.pop().unwrap(), &1234); + assert_empty(&mut list); + } +} + +#[test] +fn push_remove() { + unsafe { + let mut node = UnsafeListEntry::new(1234); + let mut list = UnsafeList::new(); + assert_eq!(list.push(&mut node), &1234); + list.remove(&mut node); + assert_empty(&mut list); + } +} + +#[test] +fn push_remove_pop() { + unsafe { + let mut node1 = UnsafeListEntry::new(11); + let mut node2 = UnsafeListEntry::new(12); + let mut node3 = UnsafeListEntry::new(13); + let mut node4 = UnsafeListEntry::new(14); + let mut node5 = UnsafeListEntry::new(15); + let mut list = UnsafeList::new(); + assert_eq!(list.push(&mut node1), &11); + assert_eq!(list.push(&mut node2), &12); + assert_eq!(list.push(&mut node3), &13); + assert_eq!(list.push(&mut node4), &14); + assert_eq!(list.push(&mut node5), &15); + + list.remove(&mut node1); + assert_eq!(list.pop().unwrap(), &12); + list.remove(&mut node3); + assert_eq!(list.pop().unwrap(), &14); + list.remove(&mut node5); + assert_empty(&mut list); + + assert_eq!(list.push(&mut node1), &11); + assert_eq!(list.pop().unwrap(), &11); + assert_empty(&mut list); + + assert_eq!(list.push(&mut node3), &13); + assert_eq!(list.push(&mut node4), &14); + list.remove(&mut node3); + list.remove(&mut node4); + assert_empty(&mut list); + } +} + +#[test] +fn complex_pushes_pops() { + unsafe { + let mut node1 = UnsafeListEntry::new(1234); + let mut node2 = UnsafeListEntry::new(4567); + let mut node3 = UnsafeListEntry::new(9999); + let mut node4 = UnsafeListEntry::new(8642); + let mut list = UnsafeList::new(); + list.push(&mut node1); + list.push(&mut node2); + assert_eq!(list.pop().unwrap(), &1234); + list.push(&mut node3); + assert_eq!(list.pop().unwrap(), &4567); + assert_eq!(list.pop().unwrap(), &9999); + assert_empty(&mut list); + list.push(&mut node4); + assert_eq!(list.pop().unwrap(), &8642); + assert_empty(&mut list); + } +} + +#[test] +fn cell() { + unsafe { + let mut node = UnsafeListEntry::new(Cell::new(0)); + let mut list = UnsafeList::new(); + let noderef = list.push(&mut node); + assert_eq!(noderef.get(), 0); + list.pop().unwrap().set(1); + assert_empty(&mut list); + assert_eq!(noderef.get(), 1); + } +} diff --git a/crux-mir/lib/std/src/sys/solid/abi/fs.rs b/crux-mir/lib/std/src/sys/solid/abi/fs.rs new file mode 100644 index 000000000..32800bd9a --- /dev/null +++ b/crux-mir/lib/std/src/sys/solid/abi/fs.rs @@ -0,0 +1,53 @@ +//! `solid_fs.h` +use crate::os::raw::{c_char, c_int, c_uchar}; +pub use libc::{ + blksize_t, dev_t, ino_t, off_t, stat, time_t, O_APPEND, O_CREAT, O_EXCL, O_RDONLY, O_RDWR, + O_TRUNC, O_WRONLY, SEEK_CUR, SEEK_END, SEEK_SET, S_IEXEC, S_IFBLK, S_IFCHR, S_IFDIR, S_IFIFO, + S_IFMT, S_IFREG, S_IREAD, S_IWRITE, +}; + +pub const O_ACCMODE: c_int = 0x3; + +pub const SOLID_MAX_PATH: usize = 256; + +#[repr(C)] +#[derive(Copy, Clone)] +pub struct dirent { + pub d_ino: ino_t, + pub d_type: c_uchar, + pub d_name: [c_char; 256usize], +} + +pub const DT_UNKNOWN: c_uchar = 0; +pub const DT_FIFO: c_uchar = 1; +pub const DT_CHR: c_uchar = 2; +pub const DT_DIR: c_uchar = 4; +pub const DT_BLK: c_uchar = 6; +pub const DT_REG: c_uchar = 8; +pub const DT_LNK: c_uchar = 10; +pub const DT_SOCK: c_uchar = 12; +pub const DT_WHT: c_uchar = 14; + +pub type S_DIR = c_int; + +extern "C" { + pub fn SOLID_FS_Open(fd: *mut c_int, path: *const c_char, mode: c_int) -> c_int; + pub fn SOLID_FS_Close(fd: c_int) -> c_int; + pub fn SOLID_FS_Read(fd: c_int, buf: *mut u8, size: usize, result: *mut usize) -> c_int; + pub fn SOLID_FS_Write(fd: c_int, buf: *const u8, size: usize, result: *mut usize) -> c_int; + pub fn SOLID_FS_Lseek(fd: c_int, offset: off_t, whence: c_int) -> c_int; + pub fn SOLID_FS_Sync(fd: c_int) -> c_int; + pub fn SOLID_FS_Ftell(fd: c_int, result: *mut off_t) -> c_int; + pub fn SOLID_FS_Feof(fd: c_int, result: *mut c_int) -> c_int; + pub fn SOLID_FS_Fsize(fd: c_int, result: *mut usize) -> c_int; + pub fn SOLID_FS_Truncate(path: *const c_char, size: off_t) -> c_int; + pub fn SOLID_FS_OpenDir(path: *const c_char, pDir: *mut S_DIR) -> c_int; + pub fn SOLID_FS_CloseDir(dir: S_DIR) -> c_int; + pub fn SOLID_FS_ReadDir(dir: S_DIR, dirp: *mut dirent) -> c_int; + pub fn SOLID_FS_Stat(path: *const c_char, buf: *mut stat) -> c_int; + pub fn SOLID_FS_Unlink(path: *const c_char) -> c_int; + pub fn SOLID_FS_Rename(oldpath: *const c_char, newpath: *const c_char) -> c_int; + pub fn SOLID_FS_Chmod(path: *const c_char, mode: c_int) -> c_int; + pub fn SOLID_FS_Utime(path: *const c_char, time: time_t) -> c_int; + pub fn SOLID_FS_Mkdir(path: *const c_char) -> c_int; +} diff --git a/crux-mir/lib/std/src/sys/solid/abi/mod.rs b/crux-mir/lib/std/src/sys/solid/abi/mod.rs new file mode 100644 index 000000000..8440d572c --- /dev/null +++ b/crux-mir/lib/std/src/sys/solid/abi/mod.rs @@ -0,0 +1,65 @@ +use crate::os::raw::c_int; + +mod fs; +pub mod sockets; +pub use self::fs::*; + +// `solid_types.h` +pub use super::itron::abi::{ER, ER_ID, E_TMOUT, ID}; + +pub const SOLID_ERR_NOTFOUND: ER = -1000; +pub const SOLID_ERR_NOTSUPPORTED: ER = -1001; +pub const SOLID_ERR_EBADF: ER = -1002; +pub const SOLID_ERR_INVALIDCONTENT: ER = -1003; +pub const SOLID_ERR_NOTUSED: ER = -1004; +pub const SOLID_ERR_ALREADYUSED: ER = -1005; +pub const SOLID_ERR_OUTOFBOUND: ER = -1006; +pub const SOLID_ERR_BADSEQUENCE: ER = -1007; +pub const SOLID_ERR_UNKNOWNDEVICE: ER = -1008; +pub const SOLID_ERR_BUSY: ER = -1009; +pub const SOLID_ERR_TIMEOUT: ER = -1010; +pub const SOLID_ERR_INVALIDACCESS: ER = -1011; +pub const SOLID_ERR_NOTREADY: ER = -1012; + +// `solid_rtc.h` +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct SOLID_RTC_TIME { + pub tm_sec: c_int, + pub tm_min: c_int, + pub tm_hour: c_int, + pub tm_mday: c_int, + pub tm_mon: c_int, + pub tm_year: c_int, + pub tm_wday: c_int, +} + +extern "C" { + pub fn SOLID_RTC_ReadTime(time: *mut SOLID_RTC_TIME) -> c_int; +} + +// `solid_log.h` +extern "C" { + pub fn SOLID_LOG_write(s: *const u8, l: usize); +} + +// `solid_mem.h` +extern "C" { + pub fn SOLID_TLS_AddDestructor(id: i32, dtor: unsafe extern "C" fn(*mut u8)); +} + +// `solid_rng.h` +extern "C" { + pub fn SOLID_RNG_SampleRandomBytes(buffer: *mut u8, length: usize) -> c_int; +} + +// `rwlock.h` +extern "C" { + pub fn rwl_loc_rdl(id: ID) -> ER; + pub fn rwl_loc_wrl(id: ID) -> ER; + pub fn rwl_ploc_rdl(id: ID) -> ER; + pub fn rwl_ploc_wrl(id: ID) -> ER; + pub fn rwl_unl_rwl(id: ID) -> ER; + pub fn rwl_acre_rwl() -> ER_ID; + pub fn rwl_del_rwl(id: ID) -> ER; +} diff --git a/crux-mir/lib/std/src/sys/solid/abi/sockets.rs b/crux-mir/lib/std/src/sys/solid/abi/sockets.rs new file mode 100644 index 000000000..eb06a6dd9 --- /dev/null +++ b/crux-mir/lib/std/src/sys/solid/abi/sockets.rs @@ -0,0 +1,277 @@ +use crate::os::raw::{c_char, c_uint, c_void}; +pub use libc::{c_int, c_long, size_t, ssize_t, suseconds_t, time_t, timeval}; + +pub const SOLID_NET_ERR_BASE: c_int = -2000; +pub const EINPROGRESS: c_int = SOLID_NET_ERR_BASE - libc::EINPROGRESS; + +pub const AF_INET6: i32 = 10; +pub const AF_INET: i32 = 2; +pub const IPPROTO_IP: i32 = 0; +pub const IPPROTO_IPV6: i32 = 41; +pub const IPPROTO_TCP: i32 = 6; +pub const IPV6_ADD_MEMBERSHIP: i32 = 12; +pub const IPV6_DROP_MEMBERSHIP: i32 = 13; +pub const IPV6_MULTICAST_LOOP: i32 = 19; +pub const IPV6_V6ONLY: i32 = 27; +pub const IP_TTL: i32 = 2; +pub const IP_MULTICAST_TTL: i32 = 5; +pub const IP_MULTICAST_LOOP: i32 = 7; +pub const IP_ADD_MEMBERSHIP: i32 = 3; +pub const IP_DROP_MEMBERSHIP: i32 = 4; +pub const SHUT_RD: i32 = 0; +pub const SHUT_RDWR: i32 = 2; +pub const SHUT_WR: i32 = 1; +pub const SOCK_DGRAM: i32 = 2; +pub const SOCK_STREAM: i32 = 1; +pub const SOL_SOCKET: i32 = 4095; +pub const SO_BROADCAST: i32 = 32; +pub const SO_ERROR: i32 = 4103; +pub const SO_RCVTIMEO: i32 = 4102; +pub const SO_REUSEADDR: i32 = 4; +pub const SO_SNDTIMEO: i32 = 4101; +pub const SO_LINGER: i32 = 128; +pub const TCP_NODELAY: i32 = 1; +pub const MSG_PEEK: c_int = 1; +pub const FIONBIO: c_long = 0x8008667eu32 as c_long; +pub const EAI_NONAME: i32 = -2200; +pub const EAI_SERVICE: i32 = -2201; +pub const EAI_FAIL: i32 = -2202; +pub const EAI_MEMORY: i32 = -2203; +pub const EAI_FAMILY: i32 = -2204; + +pub type sa_family_t = u8; +pub type socklen_t = u32; +pub type in_addr_t = u32; +pub type in_port_t = u16; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct in_addr { + pub s_addr: in_addr_t, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct in6_addr { + pub s6_addr: [u8; 16], +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct ip_mreq { + pub imr_multiaddr: in_addr, + pub imr_interface: in_addr, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct ipv6_mreq { + pub ipv6mr_multiaddr: in6_addr, + pub ipv6mr_interface: c_uint, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct msghdr { + pub msg_name: *mut c_void, + pub msg_namelen: socklen_t, + pub msg_iov: *mut iovec, + pub msg_iovlen: c_int, + pub msg_control: *mut c_void, + pub msg_controllen: socklen_t, + pub msg_flags: c_int, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct sockaddr { + pub sa_len: u8, + pub sa_family: sa_family_t, + pub sa_data: [c_char; 14usize], +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct sockaddr_in { + pub sin_len: u8, + pub sin_family: sa_family_t, + pub sin_port: in_port_t, + pub sin_addr: in_addr, + pub sin_zero: [c_char; 8usize], +} + +#[repr(C)] +#[derive(Copy, Clone)] +pub struct sockaddr_in6 { + pub sin6_len: u8, + pub sin6_family: sa_family_t, + pub sin6_port: in_port_t, + pub sin6_flowinfo: u32, + pub sin6_addr: in6_addr, + pub sin6_scope_id: u32, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct sockaddr_storage { + pub s2_len: u8, + pub ss_family: sa_family_t, + pub s2_data1: [c_char; 2usize], + pub s2_data2: [u32; 3usize], +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct addrinfo { + pub ai_flags: c_int, + pub ai_family: c_int, + pub ai_socktype: c_int, + pub ai_protocol: c_int, + pub ai_addrlen: socklen_t, + pub ai_addr: *mut sockaddr, + pub ai_canonname: *mut c_char, + pub ai_next: *mut addrinfo, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct linger { + pub l_onoff: c_int, + pub l_linger: c_int, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct iovec { + pub iov_base: *mut c_void, + pub iov_len: usize, +} + +/// This value can be chosen by an application +pub const SOLID_NET_FD_SETSIZE: usize = 1; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct fd_set { + pub num_fds: usize, + pub fds: [c_int; SOLID_NET_FD_SETSIZE], +} + +extern "C" { + #[link_name = "SOLID_NET_StrError"] + pub fn strerror(errnum: c_int) -> *const c_char; + + pub fn SOLID_NET_GetLastError() -> c_int; + + #[link_name = "SOLID_NET_Accept"] + pub fn accept(s: c_int, addr: *mut sockaddr, addrlen: *mut socklen_t) -> c_int; + + #[link_name = "SOLID_NET_Bind"] + pub fn bind(s: c_int, name: *const sockaddr, namelen: socklen_t) -> c_int; + + #[link_name = "SOLID_NET_Connect"] + pub fn connect(s: c_int, name: *const sockaddr, namelen: socklen_t) -> c_int; + + #[link_name = "SOLID_NET_Close"] + pub fn close(s: c_int) -> c_int; + + #[link_name = "SOLID_NET_Dup"] + pub fn dup(s: c_int) -> c_int; + + #[link_name = "SOLID_NET_GetPeerName"] + pub fn getpeername(s: c_int, name: *mut sockaddr, namelen: *mut socklen_t) -> c_int; + + #[link_name = "SOLID_NET_GetSockName"] + pub fn getsockname(s: c_int, name: *mut sockaddr, namelen: *mut socklen_t) -> c_int; + + #[link_name = "SOLID_NET_GetSockOpt"] + pub fn getsockopt( + s: c_int, + level: c_int, + optname: c_int, + optval: *mut c_void, + optlen: *mut socklen_t, + ) -> c_int; + + #[link_name = "SOLID_NET_SetSockOpt"] + pub fn setsockopt( + s: c_int, + level: c_int, + optname: c_int, + optval: *const c_void, + optlen: socklen_t, + ) -> c_int; + + #[link_name = "SOLID_NET_Ioctl"] + pub fn ioctl(s: c_int, cmd: c_long, argp: *mut c_void) -> c_int; + + #[link_name = "SOLID_NET_Listen"] + pub fn listen(s: c_int, backlog: c_int) -> c_int; + + #[link_name = "SOLID_NET_Recv"] + pub fn recv(s: c_int, mem: *mut c_void, len: size_t, flags: c_int) -> ssize_t; + + #[link_name = "SOLID_NET_Read"] + pub fn read(s: c_int, mem: *mut c_void, len: size_t) -> ssize_t; + + #[link_name = "SOLID_NET_Readv"] + pub fn readv(s: c_int, bufs: *const iovec, bufcnt: c_int) -> ssize_t; + + #[link_name = "SOLID_NET_RecvFrom"] + pub fn recvfrom( + s: c_int, + mem: *mut c_void, + len: size_t, + flags: c_int, + from: *mut sockaddr, + fromlen: *mut socklen_t, + ) -> ssize_t; + + #[link_name = "SOLID_NET_Send"] + pub fn send(s: c_int, mem: *const c_void, len: size_t, flags: c_int) -> ssize_t; + + #[link_name = "SOLID_NET_SendMsg"] + pub fn sendmsg(s: c_int, message: *const msghdr, flags: c_int) -> ssize_t; + + #[link_name = "SOLID_NET_SendTo"] + pub fn sendto( + s: c_int, + mem: *const c_void, + len: size_t, + flags: c_int, + to: *const sockaddr, + tolen: socklen_t, + ) -> ssize_t; + + #[link_name = "SOLID_NET_Shutdown"] + pub fn shutdown(s: c_int, how: c_int) -> c_int; + + #[link_name = "SOLID_NET_Socket"] + pub fn socket(domain: c_int, type_: c_int, protocol: c_int) -> c_int; + + #[link_name = "SOLID_NET_Write"] + pub fn write(s: c_int, mem: *const c_void, len: size_t) -> ssize_t; + + #[link_name = "SOLID_NET_Writev"] + pub fn writev(s: c_int, bufs: *const iovec, bufcnt: c_int) -> ssize_t; + + #[link_name = "SOLID_NET_FreeAddrInfo"] + pub fn freeaddrinfo(ai: *mut addrinfo); + + #[link_name = "SOLID_NET_GetAddrInfo"] + pub fn getaddrinfo( + nodename: *const c_char, + servname: *const c_char, + hints: *const addrinfo, + res: *mut *mut addrinfo, + ) -> c_int; + + #[link_name = "SOLID_NET_Select"] + pub fn select( + maxfdp1: c_int, + readset: *mut fd_set, + writeset: *mut fd_set, + exceptset: *mut fd_set, + timeout: *mut timeval, + ) -> c_int; +} diff --git a/crux-mir/lib/std/src/sys/solid/alloc.rs b/crux-mir/lib/std/src/sys/solid/alloc.rs new file mode 100644 index 000000000..d013bd876 --- /dev/null +++ b/crux-mir/lib/std/src/sys/solid/alloc.rs @@ -0,0 +1,32 @@ +use crate::{ + alloc::{GlobalAlloc, Layout, System}, + sys::common::alloc::{realloc_fallback, MIN_ALIGN}, +}; + +#[stable(feature = "alloc_system_type", since = "1.28.0")] +unsafe impl GlobalAlloc for System { + #[inline] + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + if layout.align() <= MIN_ALIGN && layout.align() <= layout.size() { + unsafe { libc::malloc(layout.size()) as *mut u8 } + } else { + unsafe { libc::memalign(layout.align(), layout.size()) as *mut u8 } + } + } + + #[inline] + unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { + unsafe { libc::free(ptr as *mut libc::c_void) } + } + + #[inline] + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + unsafe { + if layout.align() <= MIN_ALIGN && layout.align() <= new_size { + libc::realloc(ptr as *mut libc::c_void, new_size) as *mut u8 + } else { + realloc_fallback(self, ptr, layout, new_size) + } + } + } +} diff --git a/crux-mir/lib/std/src/sys/vxworks/env.rs b/crux-mir/lib/std/src/sys/solid/env.rs similarity index 60% rename from crux-mir/lib/std/src/sys/vxworks/env.rs rename to crux-mir/lib/std/src/sys/solid/env.rs index fe1aedd58..6855c113b 100644 --- a/crux-mir/lib/std/src/sys/vxworks/env.rs +++ b/crux-mir/lib/std/src/sys/solid/env.rs @@ -1,7 +1,7 @@ pub mod os { - pub const FAMILY: &str = "vxworks"; - pub const OS: &str = "vxworks"; - pub const DLL_PREFIX: &str = "lib"; + pub const FAMILY: &str = "itron"; + pub const OS: &str = "solid"; + pub const DLL_PREFIX: &str = ""; pub const DLL_SUFFIX: &str = ".so"; pub const DLL_EXTENSION: &str = "so"; pub const EXE_SUFFIX: &str = ""; diff --git a/crux-mir/lib/std/src/sys/solid/error.rs b/crux-mir/lib/std/src/sys/solid/error.rs new file mode 100644 index 000000000..547b4f3a9 --- /dev/null +++ b/crux-mir/lib/std/src/sys/solid/error.rs @@ -0,0 +1,55 @@ +use super::{abi, itron, net}; +use crate::io::ErrorKind; + +pub use self::itron::error::{expect_success, ItronError as SolidError}; + +/// Describe the specified SOLID error code. Returns `None` if it's an +/// undefined error code. +/// +/// The SOLID error codes are a superset of μITRON error codes. +pub fn error_name(er: abi::ER) -> Option<&'static str> { + match er { + // Success + er if er >= 0 => None, + er if er < abi::sockets::SOLID_NET_ERR_BASE => net::error_name(er), + + abi::SOLID_ERR_NOTFOUND => Some("not found"), + abi::SOLID_ERR_NOTSUPPORTED => Some("not supported"), + abi::SOLID_ERR_EBADF => Some("bad flags"), + abi::SOLID_ERR_INVALIDCONTENT => Some("invalid content"), + abi::SOLID_ERR_NOTUSED => Some("not used"), + abi::SOLID_ERR_ALREADYUSED => Some("already used"), + abi::SOLID_ERR_OUTOFBOUND => Some("out of bounds"), + abi::SOLID_ERR_BADSEQUENCE => Some("bad sequence"), + abi::SOLID_ERR_UNKNOWNDEVICE => Some("unknown device"), + abi::SOLID_ERR_BUSY => Some("busy"), + abi::SOLID_ERR_TIMEOUT => Some("operation timed out"), + abi::SOLID_ERR_INVALIDACCESS => Some("invalid access"), + abi::SOLID_ERR_NOTREADY => Some("not ready"), + + _ => itron::error::error_name(er), + } +} + +pub fn decode_error_kind(er: abi::ER) -> ErrorKind { + match er { + // Success + er if er >= 0 => ErrorKind::Uncategorized, + er if er < abi::sockets::SOLID_NET_ERR_BASE => net::decode_error_kind(er), + + abi::SOLID_ERR_NOTFOUND => ErrorKind::NotFound, + abi::SOLID_ERR_NOTSUPPORTED => ErrorKind::Unsupported, + abi::SOLID_ERR_EBADF => ErrorKind::InvalidInput, + abi::SOLID_ERR_INVALIDCONTENT => ErrorKind::InvalidData, + // abi::SOLID_ERR_NOTUSED + // abi::SOLID_ERR_ALREADYUSED + abi::SOLID_ERR_OUTOFBOUND => ErrorKind::InvalidInput, + // abi::SOLID_ERR_BADSEQUENCE + abi::SOLID_ERR_UNKNOWNDEVICE => ErrorKind::NotFound, + // abi::SOLID_ERR_BUSY + abi::SOLID_ERR_TIMEOUT => ErrorKind::TimedOut, + // abi::SOLID_ERR_INVALIDACCESS + // abi::SOLID_ERR_NOTREADY + _ => itron::error::decode_error_kind(er), + } +} diff --git a/crux-mir/lib/std/src/sys/solid/fs.rs b/crux-mir/lib/std/src/sys/solid/fs.rs new file mode 100644 index 000000000..6c66b93a3 --- /dev/null +++ b/crux-mir/lib/std/src/sys/solid/fs.rs @@ -0,0 +1,588 @@ +use super::{abi, error}; +use crate::{ + ffi::{CStr, CString, OsStr, OsString}, + fmt, + io::{self, BorrowedCursor, IoSlice, IoSliceMut, SeekFrom}, + mem::MaybeUninit, + os::raw::{c_int, c_short}, + os::solid::ffi::OsStrExt, + path::{Path, PathBuf}, + sync::Arc, + sys::time::SystemTime, + sys::unsupported, +}; + +pub use crate::sys_common::fs::try_exists; + +/// A file descriptor. +#[derive(Clone, Copy)] +#[rustc_layout_scalar_valid_range_start(0)] +// libstd/os/raw/mod.rs assures me that every libstd-supported platform has a +// 32-bit c_int. Below is -2, in two's complement, but that only works out +// because c_int is 32 bits. +#[rustc_layout_scalar_valid_range_end(0xFF_FF_FF_FE)] +struct FileDesc { + fd: c_int, +} + +impl FileDesc { + #[inline] + fn new(fd: c_int) -> FileDesc { + assert_ne!(fd, -1i32); + // Safety: we just asserted that the value is in the valid range and + // isn't `-1` (the only value bigger than `0xFF_FF_FF_FE` unsigned) + unsafe { FileDesc { fd } } + } + + #[inline] + fn raw(&self) -> c_int { + self.fd + } +} + +pub struct File { + fd: FileDesc, +} + +#[derive(Clone)] +pub struct FileAttr { + stat: abi::stat, +} + +// all DirEntry's will have a reference to this struct +struct InnerReadDir { + dirp: abi::S_DIR, + root: PathBuf, +} + +pub struct ReadDir { + inner: Arc, +} + +pub struct DirEntry { + entry: abi::dirent, + inner: Arc, +} + +#[derive(Clone, Debug)] +pub struct OpenOptions { + // generic + read: bool, + write: bool, + append: bool, + truncate: bool, + create: bool, + create_new: bool, + // system-specific + custom_flags: i32, +} + +#[derive(Copy, Clone, Debug, Default)] +pub struct FileTimes {} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct FilePermissions(c_short); + +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] +pub struct FileType(c_short); + +#[derive(Debug)] +pub struct DirBuilder {} + +impl FileAttr { + pub fn size(&self) -> u64 { + self.stat.st_size as u64 + } + + pub fn perm(&self) -> FilePermissions { + FilePermissions(self.stat.st_mode) + } + + pub fn file_type(&self) -> FileType { + FileType(self.stat.st_mode) + } + + pub fn modified(&self) -> io::Result { + Ok(SystemTime::from_time_t(self.stat.st_mtime)) + } + + pub fn accessed(&self) -> io::Result { + Ok(SystemTime::from_time_t(self.stat.st_atime)) + } + + pub fn created(&self) -> io::Result { + Ok(SystemTime::from_time_t(self.stat.st_ctime)) + } +} + +impl FilePermissions { + pub fn readonly(&self) -> bool { + (self.0 & abi::S_IWRITE) == 0 + } + + pub fn set_readonly(&mut self, readonly: bool) { + if readonly { + self.0 &= !abi::S_IWRITE; + } else { + self.0 |= abi::S_IWRITE; + } + } +} + +impl FileTimes { + pub fn set_accessed(&mut self, _t: SystemTime) {} + pub fn set_modified(&mut self, _t: SystemTime) {} +} + +impl FileType { + pub fn is_dir(&self) -> bool { + self.is(abi::S_IFDIR) + } + pub fn is_file(&self) -> bool { + self.is(abi::S_IFREG) + } + pub fn is_symlink(&self) -> bool { + false + } + + pub fn is(&self, mode: c_short) -> bool { + self.0 & abi::S_IFMT == mode + } +} + +pub fn readdir(p: &Path) -> io::Result { + unsafe { + let mut dir = MaybeUninit::uninit(); + error::SolidError::err_if_negative(abi::SOLID_FS_OpenDir( + cstr(p)?.as_ptr(), + dir.as_mut_ptr(), + )) + .map_err(|e| e.as_io_error())?; + let inner = Arc::new(InnerReadDir { dirp: dir.assume_init(), root: p.to_owned() }); + Ok(ReadDir { inner }) + } +} + +impl fmt::Debug for ReadDir { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // This will only be called from std::fs::ReadDir, which will add a "ReadDir()" frame. + // Thus the result will be e g 'ReadDir("/home")' + fmt::Debug::fmt(&*self.inner.root, f) + } +} + +impl Iterator for ReadDir { + type Item = io::Result; + + fn next(&mut self) -> Option> { + let entry = unsafe { + let mut out_entry = MaybeUninit::uninit(); + match error::SolidError::err_if_negative(abi::SOLID_FS_ReadDir( + self.inner.dirp, + out_entry.as_mut_ptr(), + )) { + Ok(_) => out_entry.assume_init(), + Err(e) if e.as_raw() == abi::SOLID_ERR_NOTFOUND => return None, + Err(e) => return Some(Err(e.as_io_error())), + } + }; + + (entry.d_name[0] != 0).then(|| Ok(DirEntry { entry, inner: Arc::clone(&self.inner) })) + } +} + +impl Drop for InnerReadDir { + fn drop(&mut self) { + unsafe { abi::SOLID_FS_CloseDir(self.dirp) }; + } +} + +impl DirEntry { + pub fn path(&self) -> PathBuf { + self.inner.root.join(OsStr::from_bytes( + unsafe { CStr::from_ptr(self.entry.d_name.as_ptr()) }.to_bytes(), + )) + } + + pub fn file_name(&self) -> OsString { + OsStr::from_bytes(unsafe { CStr::from_ptr(self.entry.d_name.as_ptr()) }.to_bytes()) + .to_os_string() + } + + pub fn metadata(&self) -> io::Result { + lstat(&self.path()) + } + + pub fn file_type(&self) -> io::Result { + match self.entry.d_type { + abi::DT_CHR => Ok(FileType(abi::S_IFCHR)), + abi::DT_FIFO => Ok(FileType(abi::S_IFIFO)), + abi::DT_REG => Ok(FileType(abi::S_IFREG)), + abi::DT_DIR => Ok(FileType(abi::S_IFDIR)), + abi::DT_BLK => Ok(FileType(abi::S_IFBLK)), + _ => lstat(&self.path()).map(|m| m.file_type()), + } + } +} + +impl OpenOptions { + pub fn new() -> OpenOptions { + OpenOptions { + // generic + read: false, + write: false, + append: false, + truncate: false, + create: false, + create_new: false, + // system-specific + custom_flags: 0, + } + } + + pub fn read(&mut self, read: bool) { + self.read = read; + } + pub fn write(&mut self, write: bool) { + self.write = write; + } + pub fn append(&mut self, append: bool) { + self.append = append; + } + pub fn truncate(&mut self, truncate: bool) { + self.truncate = truncate; + } + pub fn create(&mut self, create: bool) { + self.create = create; + } + pub fn create_new(&mut self, create_new: bool) { + self.create_new = create_new; + } + + pub fn custom_flags(&mut self, flags: i32) { + self.custom_flags = flags; + } + pub fn mode(&mut self, _mode: u32) {} + + fn get_access_mode(&self) -> io::Result { + match (self.read, self.write, self.append) { + (true, false, false) => Ok(abi::O_RDONLY), + (false, true, false) => Ok(abi::O_WRONLY), + (true, true, false) => Ok(abi::O_RDWR), + (false, _, true) => Ok(abi::O_WRONLY | abi::O_APPEND), + (true, _, true) => Ok(abi::O_RDWR | abi::O_APPEND), + (false, false, false) => Err(io::Error::from_raw_os_error(libc::EINVAL)), + } + } + + fn get_creation_mode(&self) -> io::Result { + match (self.write, self.append) { + (true, false) => {} + (false, false) => { + if self.truncate || self.create || self.create_new { + return Err(io::Error::from_raw_os_error(libc::EINVAL)); + } + } + (_, true) => { + if self.truncate && !self.create_new { + return Err(io::Error::from_raw_os_error(libc::EINVAL)); + } + } + } + + Ok(match (self.create, self.truncate, self.create_new) { + (false, false, false) => 0, + (true, false, false) => abi::O_CREAT, + (false, true, false) => abi::O_TRUNC, + (true, true, false) => abi::O_CREAT | abi::O_TRUNC, + (_, _, true) => abi::O_CREAT | abi::O_EXCL, + }) + } +} + +fn cstr(path: &Path) -> io::Result { + let path = path.as_os_str().as_bytes(); + + if !path.starts_with(br"\") { + // Relative paths aren't supported + return Err(crate::io::const_io_error!( + crate::io::ErrorKind::Unsupported, + "relative path is not supported on this platform", + )); + } + + // Apply the thread-safety wrapper + const SAFE_PREFIX: &[u8] = br"\TS"; + let wrapped_path = [SAFE_PREFIX, &path, &[0]].concat(); + + CString::from_vec_with_nul(wrapped_path).map_err(|_| { + crate::io::const_io_error!( + io::ErrorKind::InvalidInput, + "path provided contains a nul byte", + ) + }) +} + +impl File { + pub fn open(path: &Path, opts: &OpenOptions) -> io::Result { + let flags = opts.get_access_mode()? + | opts.get_creation_mode()? + | (opts.custom_flags as c_int & !abi::O_ACCMODE); + unsafe { + let mut fd = MaybeUninit::uninit(); + error::SolidError::err_if_negative(abi::SOLID_FS_Open( + fd.as_mut_ptr(), + cstr(path)?.as_ptr(), + flags, + )) + .map_err(|e| e.as_io_error())?; + Ok(File { fd: FileDesc::new(fd.assume_init()) }) + } + } + + pub fn file_attr(&self) -> io::Result { + unsupported() + } + + pub fn fsync(&self) -> io::Result<()> { + self.flush() + } + + pub fn datasync(&self) -> io::Result<()> { + self.flush() + } + + pub fn truncate(&self, _size: u64) -> io::Result<()> { + unsupported() + } + + pub fn read(&self, buf: &mut [u8]) -> io::Result { + unsafe { + let mut out_num_bytes = MaybeUninit::uninit(); + error::SolidError::err_if_negative(abi::SOLID_FS_Read( + self.fd.raw(), + buf.as_mut_ptr(), + buf.len(), + out_num_bytes.as_mut_ptr(), + )) + .map_err(|e| e.as_io_error())?; + Ok(out_num_bytes.assume_init()) + } + } + + pub fn read_buf(&self, mut cursor: BorrowedCursor<'_>) -> io::Result<()> { + unsafe { + let len = cursor.capacity(); + let mut out_num_bytes = MaybeUninit::uninit(); + error::SolidError::err_if_negative(abi::SOLID_FS_Read( + self.fd.raw(), + cursor.as_mut().as_mut_ptr() as *mut u8, + len, + out_num_bytes.as_mut_ptr(), + )) + .map_err(|e| e.as_io_error())?; + + // Safety: `out_num_bytes` is filled by the successful call to + // `SOLID_FS_Read` + let num_bytes_read = out_num_bytes.assume_init(); + + // Safety: `num_bytes_read` bytes were written to the unfilled + // portion of the buffer + cursor.advance(num_bytes_read); + + Ok(()) + } + } + + pub fn read_vectored(&self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { + crate::io::default_read_vectored(|buf| self.read(buf), bufs) + } + + pub fn is_read_vectored(&self) -> bool { + false + } + + pub fn write(&self, buf: &[u8]) -> io::Result { + unsafe { + let mut out_num_bytes = MaybeUninit::uninit(); + error::SolidError::err_if_negative(abi::SOLID_FS_Write( + self.fd.raw(), + buf.as_ptr(), + buf.len(), + out_num_bytes.as_mut_ptr(), + )) + .map_err(|e| e.as_io_error())?; + Ok(out_num_bytes.assume_init()) + } + } + + pub fn write_vectored(&self, bufs: &[IoSlice<'_>]) -> io::Result { + crate::io::default_write_vectored(|buf| self.write(buf), bufs) + } + + pub fn is_write_vectored(&self) -> bool { + false + } + + pub fn flush(&self) -> io::Result<()> { + error::SolidError::err_if_negative(unsafe { abi::SOLID_FS_Sync(self.fd.raw()) }) + .map_err(|e| e.as_io_error())?; + Ok(()) + } + + pub fn seek(&self, pos: SeekFrom) -> io::Result { + let (whence, pos) = match pos { + // Casting to `i64` is fine, too large values will end up as + // negative which will cause an error in `SOLID_FS_Lseek`. + SeekFrom::Start(off) => (abi::SEEK_SET, off as i64), + SeekFrom::End(off) => (abi::SEEK_END, off), + SeekFrom::Current(off) => (abi::SEEK_CUR, off), + }; + error::SolidError::err_if_negative(unsafe { + abi::SOLID_FS_Lseek(self.fd.raw(), pos, whence) + }) + .map_err(|e| e.as_io_error())?; + + // Get the new offset + unsafe { + let mut out_offset = MaybeUninit::uninit(); + error::SolidError::err_if_negative(abi::SOLID_FS_Ftell( + self.fd.raw(), + out_offset.as_mut_ptr(), + )) + .map_err(|e| e.as_io_error())?; + Ok(out_offset.assume_init() as u64) + } + } + + pub fn duplicate(&self) -> io::Result { + unsupported() + } + + pub fn set_permissions(&self, _perm: FilePermissions) -> io::Result<()> { + unsupported() + } + + pub fn set_times(&self, _times: FileTimes) -> io::Result<()> { + unsupported() + } +} + +impl Drop for File { + fn drop(&mut self) { + unsafe { abi::SOLID_FS_Close(self.fd.raw()) }; + } +} + +impl DirBuilder { + pub fn new() -> DirBuilder { + DirBuilder {} + } + + pub fn mkdir(&self, p: &Path) -> io::Result<()> { + error::SolidError::err_if_negative(unsafe { abi::SOLID_FS_Mkdir(cstr(p)?.as_ptr()) }) + .map_err(|e| e.as_io_error())?; + Ok(()) + } +} + +impl fmt::Debug for File { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("File").field("fd", &self.fd.raw()).finish() + } +} + +pub fn unlink(p: &Path) -> io::Result<()> { + if stat(p)?.file_type().is_dir() { + Err(io::const_io_error!(io::ErrorKind::IsADirectory, "is a directory")) + } else { + error::SolidError::err_if_negative(unsafe { abi::SOLID_FS_Unlink(cstr(p)?.as_ptr()) }) + .map_err(|e| e.as_io_error())?; + Ok(()) + } +} + +pub fn rename(old: &Path, new: &Path) -> io::Result<()> { + error::SolidError::err_if_negative(unsafe { + abi::SOLID_FS_Rename(cstr(old)?.as_ptr(), cstr(new)?.as_ptr()) + }) + .map_err(|e| e.as_io_error())?; + Ok(()) +} + +pub fn set_perm(p: &Path, perm: FilePermissions) -> io::Result<()> { + error::SolidError::err_if_negative(unsafe { + abi::SOLID_FS_Chmod(cstr(p)?.as_ptr(), perm.0.into()) + }) + .map_err(|e| e.as_io_error())?; + Ok(()) +} + +pub fn rmdir(p: &Path) -> io::Result<()> { + if stat(p)?.file_type().is_dir() { + error::SolidError::err_if_negative(unsafe { abi::SOLID_FS_Unlink(cstr(p)?.as_ptr()) }) + .map_err(|e| e.as_io_error())?; + Ok(()) + } else { + Err(io::const_io_error!(io::ErrorKind::NotADirectory, "not a directory")) + } +} + +pub fn remove_dir_all(path: &Path) -> io::Result<()> { + for child in readdir(path)? { + let child = child?; + let child_type = child.file_type()?; + if child_type.is_dir() { + remove_dir_all(&child.path())?; + } else { + unlink(&child.path())?; + } + } + rmdir(path) +} + +pub fn readlink(p: &Path) -> io::Result { + // This target doesn't support symlinks + stat(p)?; + Err(io::const_io_error!(io::ErrorKind::InvalidInput, "not a symbolic link")) +} + +pub fn symlink(_original: &Path, _link: &Path) -> io::Result<()> { + // This target doesn't support symlinks + unsupported() +} + +pub fn link(_src: &Path, _dst: &Path) -> io::Result<()> { + // This target doesn't support symlinks + unsupported() +} + +pub fn stat(p: &Path) -> io::Result { + // This target doesn't support symlinks + lstat(p) +} + +pub fn lstat(p: &Path) -> io::Result { + unsafe { + let mut out_stat = MaybeUninit::uninit(); + error::SolidError::err_if_negative(abi::SOLID_FS_Stat( + cstr(p)?.as_ptr(), + out_stat.as_mut_ptr(), + )) + .map_err(|e| e.as_io_error())?; + Ok(FileAttr { stat: out_stat.assume_init() }) + } +} + +pub fn canonicalize(_p: &Path) -> io::Result { + unsupported() +} + +pub fn copy(from: &Path, to: &Path) -> io::Result { + use crate::fs::File; + + let mut reader = File::open(from)?; + let mut writer = File::create(to)?; + + io::copy(&mut reader, &mut writer) +} diff --git a/crux-mir/lib/std/src/sys/vxworks/io.rs b/crux-mir/lib/std/src/sys/solid/io.rs similarity index 93% rename from crux-mir/lib/std/src/sys/vxworks/io.rs rename to crux-mir/lib/std/src/sys/solid/io.rs index 0f68ebf8d..a862bb787 100644 --- a/crux-mir/lib/std/src/sys/vxworks/io.rs +++ b/crux-mir/lib/std/src/sys/solid/io.rs @@ -1,7 +1,8 @@ use crate::marker::PhantomData; use crate::slice; -use libc::{c_void, iovec}; +use super::abi::sockets::iovec; +use libc::c_void; #[derive(Copy, Clone)] #[repr(transparent)] @@ -37,6 +38,7 @@ impl<'a> IoSlice<'a> { } } +#[repr(transparent)] pub struct IoSliceMut<'a> { vec: iovec, _p: PhantomData<&'a mut [u8]>, @@ -73,3 +75,7 @@ impl<'a> IoSliceMut<'a> { unsafe { slice::from_raw_parts_mut(self.vec.iov_base as *mut u8, self.vec.iov_len) } } } + +pub fn is_terminal(_: &T) -> bool { + false +} diff --git a/crux-mir/lib/std/src/sys/vxworks/memchr.rs b/crux-mir/lib/std/src/sys/solid/memchr.rs similarity index 58% rename from crux-mir/lib/std/src/sys/vxworks/memchr.rs rename to crux-mir/lib/std/src/sys/solid/memchr.rs index 928100c92..452b7a3de 100644 --- a/crux-mir/lib/std/src/sys/vxworks/memchr.rs +++ b/crux-mir/lib/std/src/sys/solid/memchr.rs @@ -1,6 +1,3 @@ -// Original implementation taken from rust-memchr. -// Copyright 2015 Andrew Gallant, bluss and Nicolas Koch - pub fn memchr(needle: u8, haystack: &[u8]) -> Option { let p = unsafe { libc::memchr( @@ -13,9 +10,12 @@ pub fn memchr(needle: u8, haystack: &[u8]) -> Option { } pub fn memrchr(needle: u8, haystack: &[u8]) -> Option { - fn memrchr_specific(needle: u8, haystack: &[u8]) -> Option { - core::slice::memchr::memrchr(needle, haystack) - } - - memrchr_specific(needle, haystack) + let p = unsafe { + libc::memrchr( + haystack.as_ptr() as *const libc::c_void, + needle as libc::c_int, + haystack.len(), + ) + }; + if p.is_null() { None } else { Some(p as usize - (haystack.as_ptr() as usize)) } } diff --git a/crux-mir/lib/std/src/sys/solid/mod.rs b/crux-mir/lib/std/src/sys/solid/mod.rs new file mode 100644 index 000000000..5867979a2 --- /dev/null +++ b/crux-mir/lib/std/src/sys/solid/mod.rs @@ -0,0 +1,92 @@ +#![allow(dead_code)] +#![allow(missing_docs, nonstandard_style)] +#![deny(unsafe_op_in_unsafe_fn)] + +mod abi; + +#[path = "../itron"] +mod itron { + pub(super) mod abi; + pub mod condvar; + pub(super) mod error; + pub mod mutex; + pub(super) mod spin; + pub(super) mod task; + pub mod thread; + pub(super) mod time; + use super::unsupported; + pub mod wait_flag; +} + +pub mod alloc; +#[path = "../unsupported/args.rs"] +pub mod args; +#[path = "../unix/cmath.rs"] +pub mod cmath; +pub mod env; +// `error` is `pub(crate)` so that it can be accessed by `itron/error.rs` as +// `crate::sys::error` +pub(crate) mod error; +pub mod fs; +pub mod io; +pub mod net; +pub mod os; +#[path = "../unix/os_str.rs"] +pub mod os_str; +pub mod path; +#[path = "../unsupported/pipe.rs"] +pub mod pipe; +#[path = "../unsupported/process.rs"] +pub mod process; +pub mod stdio; +pub use self::itron::thread; +pub mod memchr; +pub mod thread_local_dtor; +pub mod thread_local_key; +pub mod time; +pub use self::itron::wait_flag; + +mod rwlock; + +pub mod locks { + pub use super::itron::condvar::*; + pub use super::itron::mutex::*; + pub use super::rwlock::*; +} + +// SAFETY: must be called only once during runtime initialization. +// NOTE: this is not guaranteed to run, for example when Rust code is called externally. +pub unsafe fn init(_argc: isize, _argv: *const *const u8, _sigpipe: u8) {} + +// SAFETY: must be called only once during runtime cleanup. +pub unsafe fn cleanup() {} + +pub fn unsupported() -> crate::io::Result { + Err(unsupported_err()) +} + +pub fn unsupported_err() -> crate::io::Error { + crate::io::const_io_error!( + crate::io::ErrorKind::Unsupported, + "operation not supported on this platform", + ) +} + +pub fn decode_error_kind(code: i32) -> crate::io::ErrorKind { + error::decode_error_kind(code) +} + +#[inline] +pub fn abort_internal() -> ! { + unsafe { libc::abort() } +} + +pub fn hashmap_random_keys() -> (u64, u64) { + unsafe { + let mut out = crate::mem::MaybeUninit::<[u64; 2]>::uninit(); + let result = abi::SOLID_RNG_SampleRandomBytes(out.as_mut_ptr() as *mut u8, 16); + assert_eq!(result, 0, "SOLID_RNG_SampleRandomBytes failed: {result}"); + let [x1, x2] = out.assume_init(); + (x1, x2) + } +} diff --git a/crux-mir/lib/std/src/sys/solid/net.rs b/crux-mir/lib/std/src/sys/solid/net.rs new file mode 100644 index 000000000..1b98ef993 --- /dev/null +++ b/crux-mir/lib/std/src/sys/solid/net.rs @@ -0,0 +1,469 @@ +use super::abi; +use crate::{ + cmp, + ffi::CStr, + io::{self, ErrorKind, IoSlice, IoSliceMut}, + mem, + net::{Shutdown, SocketAddr}, + ptr, str, + sys_common::net::{getsockopt, setsockopt, sockaddr_to_addr}, + sys_common::{AsInner, FromInner, IntoInner}, + time::Duration, +}; + +use self::netc::{sockaddr, socklen_t, MSG_PEEK}; +use libc::{c_int, c_void, size_t}; + +pub mod netc { + pub use super::super::abi::sockets::*; +} + +pub type wrlen_t = size_t; + +const READ_LIMIT: usize = libc::ssize_t::MAX as usize; + +const fn max_iov() -> usize { + // Judging by the source code, it's unlimited, but specify a lower + // value just in case. + 1024 +} + +/// A file descriptor. +#[rustc_layout_scalar_valid_range_start(0)] +// libstd/os/raw/mod.rs assures me that every libstd-supported platform has a +// 32-bit c_int. Below is -2, in two's complement, but that only works out +// because c_int is 32 bits. +#[rustc_layout_scalar_valid_range_end(0xFF_FF_FF_FE)] +struct FileDesc { + fd: c_int, +} + +impl FileDesc { + #[inline] + fn new(fd: c_int) -> FileDesc { + assert_ne!(fd, -1i32); + // Safety: we just asserted that the value is in the valid range and + // isn't `-1` (the only value bigger than `0xFF_FF_FF_FE` unsigned) + unsafe { FileDesc { fd } } + } + + #[inline] + fn raw(&self) -> c_int { + self.fd + } + + /// Extracts the actual file descriptor without closing it. + #[inline] + fn into_raw(self) -> c_int { + let fd = self.fd; + mem::forget(self); + fd + } + + fn read(&self, buf: &mut [u8]) -> io::Result { + let ret = cvt(unsafe { + netc::read(self.fd, buf.as_mut_ptr() as *mut c_void, cmp::min(buf.len(), READ_LIMIT)) + })?; + Ok(ret as usize) + } + + fn read_vectored(&self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { + let ret = cvt(unsafe { + netc::readv( + self.fd, + bufs.as_ptr() as *const netc::iovec, + cmp::min(bufs.len(), max_iov()) as c_int, + ) + })?; + Ok(ret as usize) + } + + #[inline] + fn is_read_vectored(&self) -> bool { + true + } + + fn write(&self, buf: &[u8]) -> io::Result { + let ret = cvt(unsafe { + netc::write(self.fd, buf.as_ptr() as *const c_void, cmp::min(buf.len(), READ_LIMIT)) + })?; + Ok(ret as usize) + } + + fn write_vectored(&self, bufs: &[IoSlice<'_>]) -> io::Result { + let ret = cvt(unsafe { + netc::writev( + self.fd, + bufs.as_ptr() as *const netc::iovec, + cmp::min(bufs.len(), max_iov()) as c_int, + ) + })?; + Ok(ret as usize) + } + + #[inline] + fn is_write_vectored(&self) -> bool { + true + } + + fn duplicate(&self) -> io::Result { + cvt(unsafe { netc::dup(self.fd) }).map(Self::new) + } +} + +impl AsInner for FileDesc { + fn as_inner(&self) -> &c_int { + &self.fd + } +} + +impl Drop for FileDesc { + fn drop(&mut self) { + unsafe { netc::close(self.fd) }; + } +} + +#[doc(hidden)] +pub trait IsMinusOne { + fn is_minus_one(&self) -> bool; +} + +macro_rules! impl_is_minus_one { + ($($t:ident)*) => ($(impl IsMinusOne for $t { + fn is_minus_one(&self) -> bool { + *self == -1 + } + })*) +} + +impl_is_minus_one! { i8 i16 i32 i64 isize } + +pub fn cvt(t: T) -> io::Result { + if t.is_minus_one() { Err(last_error()) } else { Ok(t) } +} + +/// A variant of `cvt` for `getaddrinfo` which return 0 for a success. +pub fn cvt_gai(err: c_int) -> io::Result<()> { + if err == 0 { + Ok(()) + } else { + let msg: &dyn crate::fmt::Display = match err { + netc::EAI_NONAME => &"name or service not known", + netc::EAI_SERVICE => &"service not supported", + netc::EAI_FAIL => &"non-recoverable failure in name resolution", + netc::EAI_MEMORY => &"memory allocation failure", + netc::EAI_FAMILY => &"family not supported", + _ => &err, + }; + Err(io::Error::new( + io::ErrorKind::Uncategorized, + &format!("failed to lookup address information: {msg}")[..], + )) + } +} + +/// Just to provide the same interface as sys/unix/net.rs +pub fn cvt_r(mut f: F) -> io::Result +where + T: IsMinusOne, + F: FnMut() -> T, +{ + cvt(f()) +} + +/// Returns the last error from the network subsystem. +fn last_error() -> io::Error { + io::Error::from_raw_os_error(unsafe { netc::SOLID_NET_GetLastError() }) +} + +pub(super) fn error_name(er: abi::ER) -> Option<&'static str> { + unsafe { CStr::from_ptr(netc::strerror(er)) }.to_str().ok() +} + +pub(super) fn decode_error_kind(er: abi::ER) -> ErrorKind { + let errno = netc::SOLID_NET_ERR_BASE - er; + match errno as libc::c_int { + libc::ECONNREFUSED => ErrorKind::ConnectionRefused, + libc::ECONNRESET => ErrorKind::ConnectionReset, + libc::EPERM | libc::EACCES => ErrorKind::PermissionDenied, + libc::EPIPE => ErrorKind::BrokenPipe, + libc::ENOTCONN => ErrorKind::NotConnected, + libc::ECONNABORTED => ErrorKind::ConnectionAborted, + libc::EADDRNOTAVAIL => ErrorKind::AddrNotAvailable, + libc::EADDRINUSE => ErrorKind::AddrInUse, + libc::ENOENT => ErrorKind::NotFound, + libc::EINTR => ErrorKind::Interrupted, + libc::EINVAL => ErrorKind::InvalidInput, + libc::ETIMEDOUT => ErrorKind::TimedOut, + libc::EEXIST => ErrorKind::AlreadyExists, + libc::ENOSYS => ErrorKind::Unsupported, + libc::ENOMEM => ErrorKind::OutOfMemory, + libc::EAGAIN => ErrorKind::WouldBlock, + + _ => ErrorKind::Uncategorized, + } +} + +pub fn init() {} + +pub struct Socket(FileDesc); + +impl Socket { + pub fn new(addr: &SocketAddr, ty: c_int) -> io::Result { + let fam = match *addr { + SocketAddr::V4(..) => netc::AF_INET, + SocketAddr::V6(..) => netc::AF_INET6, + }; + Socket::new_raw(fam, ty) + } + + pub fn new_raw(fam: c_int, ty: c_int) -> io::Result { + unsafe { + let fd = cvt(netc::socket(fam, ty, 0))?; + let fd = FileDesc::new(fd); + let socket = Socket(fd); + + Ok(socket) + } + } + + pub fn connect_timeout(&self, addr: &SocketAddr, timeout: Duration) -> io::Result<()> { + self.set_nonblocking(true)?; + let r = unsafe { + let (addr, len) = addr.into_inner(); + cvt(netc::connect(self.0.raw(), addr.as_ptr(), len)) + }; + self.set_nonblocking(false)?; + + match r { + Ok(_) => return Ok(()), + // there's no ErrorKind for EINPROGRESS + Err(ref e) if e.raw_os_error() == Some(netc::EINPROGRESS) => {} + Err(e) => return Err(e), + } + + if timeout.as_secs() == 0 && timeout.subsec_nanos() == 0 { + return Err(io::const_io_error!( + io::ErrorKind::InvalidInput, + "cannot set a 0 duration timeout", + )); + } + + let mut timeout = + netc::timeval { tv_sec: timeout.as_secs() as _, tv_usec: timeout.subsec_micros() as _ }; + if timeout.tv_sec == 0 && timeout.tv_usec == 0 { + timeout.tv_usec = 1; + } + + let fds = netc::fd_set { num_fds: 1, fds: [self.0.raw()] }; + + let mut writefds = fds; + let mut errorfds = fds; + + let n = unsafe { + cvt(netc::select( + self.0.raw() + 1, + ptr::null_mut(), + &mut writefds, + &mut errorfds, + &mut timeout, + ))? + }; + + match n { + 0 => Err(io::const_io_error!(io::ErrorKind::TimedOut, "connection timed out")), + _ => { + let can_write = writefds.num_fds != 0; + if !can_write { + if let Some(e) = self.take_error()? { + return Err(e); + } + } + Ok(()) + } + } + } + + pub fn accept(&self, storage: *mut sockaddr, len: *mut socklen_t) -> io::Result { + let fd = cvt_r(|| unsafe { netc::accept(self.0.raw(), storage, len) })?; + let fd = FileDesc::new(fd); + Ok(Socket(fd)) + } + + pub fn duplicate(&self) -> io::Result { + self.0.duplicate().map(Socket) + } + + fn recv_with_flags(&self, buf: &mut [u8], flags: c_int) -> io::Result { + let ret = cvt(unsafe { + netc::recv(self.0.raw(), buf.as_mut_ptr() as *mut c_void, buf.len(), flags) + })?; + Ok(ret as usize) + } + + pub fn read(&self, buf: &mut [u8]) -> io::Result { + self.recv_with_flags(buf, 0) + } + + pub fn peek(&self, buf: &mut [u8]) -> io::Result { + self.recv_with_flags(buf, MSG_PEEK) + } + + pub fn read_vectored(&self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { + self.0.read_vectored(bufs) + } + + #[inline] + pub fn is_read_vectored(&self) -> bool { + self.0.is_read_vectored() + } + + fn recv_from_with_flags( + &self, + buf: &mut [u8], + flags: c_int, + ) -> io::Result<(usize, SocketAddr)> { + let mut storage: netc::sockaddr_storage = unsafe { mem::zeroed() }; + let mut addrlen = mem::size_of_val(&storage) as netc::socklen_t; + + let n = cvt(unsafe { + netc::recvfrom( + self.0.raw(), + buf.as_mut_ptr() as *mut c_void, + buf.len(), + flags, + &mut storage as *mut _ as *mut _, + &mut addrlen, + ) + })?; + Ok((n as usize, sockaddr_to_addr(&storage, addrlen as usize)?)) + } + + pub fn recv_from(&self, buf: &mut [u8]) -> io::Result<(usize, SocketAddr)> { + self.recv_from_with_flags(buf, 0) + } + + pub fn peek_from(&self, buf: &mut [u8]) -> io::Result<(usize, SocketAddr)> { + self.recv_from_with_flags(buf, MSG_PEEK) + } + + pub fn write(&self, buf: &[u8]) -> io::Result { + self.0.write(buf) + } + + pub fn write_vectored(&self, bufs: &[IoSlice<'_>]) -> io::Result { + self.0.write_vectored(bufs) + } + + #[inline] + pub fn is_write_vectored(&self) -> bool { + self.0.is_write_vectored() + } + + pub fn set_timeout(&self, dur: Option, kind: c_int) -> io::Result<()> { + let timeout = match dur { + Some(dur) => { + if dur.as_secs() == 0 && dur.subsec_nanos() == 0 { + return Err(io::const_io_error!( + io::ErrorKind::InvalidInput, + "cannot set a 0 duration timeout", + )); + } + + let secs = if dur.as_secs() > netc::c_long::MAX as u64 { + netc::c_long::MAX + } else { + dur.as_secs() as netc::c_long + }; + let mut timeout = netc::timeval { tv_sec: secs, tv_usec: dur.subsec_micros() as _ }; + if timeout.tv_sec == 0 && timeout.tv_usec == 0 { + timeout.tv_usec = 1; + } + timeout + } + None => netc::timeval { tv_sec: 0, tv_usec: 0 }, + }; + setsockopt(self, netc::SOL_SOCKET, kind, timeout) + } + + pub fn timeout(&self, kind: c_int) -> io::Result> { + let raw: netc::timeval = getsockopt(self, netc::SOL_SOCKET, kind)?; + if raw.tv_sec == 0 && raw.tv_usec == 0 { + Ok(None) + } else { + let sec = raw.tv_sec as u64; + let nsec = (raw.tv_usec as u32) * 1000; + Ok(Some(Duration::new(sec, nsec))) + } + } + + pub fn shutdown(&self, how: Shutdown) -> io::Result<()> { + let how = match how { + Shutdown::Write => netc::SHUT_WR, + Shutdown::Read => netc::SHUT_RD, + Shutdown::Both => netc::SHUT_RDWR, + }; + cvt(unsafe { netc::shutdown(self.0.raw(), how) })?; + Ok(()) + } + + pub fn set_linger(&self, linger: Option) -> io::Result<()> { + let linger = netc::linger { + l_onoff: linger.is_some() as netc::c_int, + l_linger: linger.unwrap_or_default().as_secs() as netc::c_int, + }; + + setsockopt(self, netc::SOL_SOCKET, netc::SO_LINGER, linger) + } + + pub fn linger(&self) -> io::Result> { + let val: netc::linger = getsockopt(self, netc::SOL_SOCKET, netc::SO_LINGER)?; + + Ok((val.l_onoff != 0).then(|| Duration::from_secs(val.l_linger as u64))) + } + + pub fn set_nodelay(&self, nodelay: bool) -> io::Result<()> { + setsockopt(self, netc::IPPROTO_TCP, netc::TCP_NODELAY, nodelay as c_int) + } + + pub fn nodelay(&self) -> io::Result { + let raw: c_int = getsockopt(self, netc::IPPROTO_TCP, netc::TCP_NODELAY)?; + Ok(raw != 0) + } + + pub fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()> { + let mut nonblocking = nonblocking as c_int; + cvt(unsafe { + netc::ioctl(*self.as_inner(), netc::FIONBIO, (&mut nonblocking) as *mut c_int as _) + }) + .map(drop) + } + + pub fn take_error(&self) -> io::Result> { + let raw: c_int = getsockopt(self, netc::SOL_SOCKET, netc::SO_ERROR)?; + if raw == 0 { Ok(None) } else { Ok(Some(io::Error::from_raw_os_error(raw as i32))) } + } + + // This method is used by sys_common code to abstract over targets. + pub fn as_raw(&self) -> c_int { + *self.as_inner() + } +} + +impl AsInner for Socket { + fn as_inner(&self) -> &c_int { + self.0.as_inner() + } +} + +impl FromInner for Socket { + fn from_inner(fd: c_int) -> Socket { + Socket(FileDesc::new(fd)) + } +} + +impl IntoInner for Socket { + fn into_inner(self) -> c_int { + self.0.into_raw() + } +} diff --git a/crux-mir/lib/std/src/sys/solid/os.rs b/crux-mir/lib/std/src/sys/solid/os.rs new file mode 100644 index 000000000..6135921f0 --- /dev/null +++ b/crux-mir/lib/std/src/sys/solid/os.rs @@ -0,0 +1,192 @@ +use super::unsupported; +use crate::error::Error as StdError; +use crate::ffi::{CStr, OsStr, OsString}; +use crate::fmt; +use crate::io; +use crate::os::{ + raw::{c_char, c_int}, + solid::ffi::{OsStrExt, OsStringExt}, +}; +use crate::path::{self, PathBuf}; +use crate::sync::RwLock; +use crate::sys::common::small_c_string::run_with_cstr; +use crate::vec; + +use super::{error, itron, memchr}; + +// `solid` directly maps `errno`s to μITRON error codes. +impl itron::error::ItronError { + #[inline] + pub(crate) fn as_io_error(self) -> crate::io::Error { + crate::io::Error::from_raw_os_error(self.as_raw()) + } +} + +pub fn errno() -> i32 { + 0 +} + +pub fn error_string(errno: i32) -> String { + if let Some(name) = error::error_name(errno) { name.to_owned() } else { format!("{errno}") } +} + +pub fn getcwd() -> io::Result { + unsupported() +} + +pub fn chdir(_: &path::Path) -> io::Result<()> { + unsupported() +} + +pub struct SplitPaths<'a>(&'a !); + +pub fn split_paths(_unparsed: &OsStr) -> SplitPaths<'_> { + panic!("unsupported") +} + +impl<'a> Iterator for SplitPaths<'a> { + type Item = PathBuf; + fn next(&mut self) -> Option { + *self.0 + } +} + +#[derive(Debug)] +pub struct JoinPathsError; + +pub fn join_paths(_paths: I) -> Result +where + I: Iterator, + T: AsRef, +{ + Err(JoinPathsError) +} + +impl fmt::Display for JoinPathsError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + "not supported on this platform yet".fmt(f) + } +} + +impl StdError for JoinPathsError { + #[allow(deprecated)] + fn description(&self) -> &str { + "not supported on this platform yet" + } +} + +pub fn current_exe() -> io::Result { + unsupported() +} + +static ENV_LOCK: RwLock<()> = RwLock::new(()); + +pub struct Env { + iter: vec::IntoIter<(OsString, OsString)>, +} + +impl !Send for Env {} +impl !Sync for Env {} + +impl Iterator for Env { + type Item = (OsString, OsString); + fn next(&mut self) -> Option<(OsString, OsString)> { + self.iter.next() + } + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } +} + +/// Returns a vector of (variable, value) byte-vector pairs for all the +/// environment variables of the current process. +pub fn env() -> Env { + extern "C" { + static mut environ: *const *const c_char; + } + + unsafe { + let _guard = ENV_LOCK.read(); + let mut result = Vec::new(); + if !environ.is_null() { + while !(*environ).is_null() { + if let Some(key_value) = parse(CStr::from_ptr(*environ).to_bytes()) { + result.push(key_value); + } + environ = environ.add(1); + } + } + return Env { iter: result.into_iter() }; + } + + fn parse(input: &[u8]) -> Option<(OsString, OsString)> { + // Strategy (copied from glibc): Variable name and value are separated + // by an ASCII equals sign '='. Since a variable name must not be + // empty, allow variable names starting with an equals sign. Skip all + // malformed lines. + if input.is_empty() { + return None; + } + let pos = memchr::memchr(b'=', &input[1..]).map(|p| p + 1); + pos.map(|p| { + ( + OsStringExt::from_vec(input[..p].to_vec()), + OsStringExt::from_vec(input[p + 1..].to_vec()), + ) + }) + } +} + +pub fn getenv(k: &OsStr) -> Option { + // environment variables with a nul byte can't be set, so their value is + // always None as well + let s = run_with_cstr(k.as_bytes(), |k| { + let _guard = ENV_LOCK.read(); + Ok(unsafe { libc::getenv(k.as_ptr()) } as *const libc::c_char) + }) + .ok()?; + + if s.is_null() { + None + } else { + Some(OsStringExt::from_vec(unsafe { CStr::from_ptr(s) }.to_bytes().to_vec())) + } +} + +pub fn setenv(k: &OsStr, v: &OsStr) -> io::Result<()> { + run_with_cstr(k.as_bytes(), |k| { + run_with_cstr(v.as_bytes(), |v| { + let _guard = ENV_LOCK.write(); + cvt_env(unsafe { libc::setenv(k.as_ptr(), v.as_ptr(), 1) }).map(drop) + }) + }) +} + +pub fn unsetenv(n: &OsStr) -> io::Result<()> { + run_with_cstr(n.as_bytes(), |nbuf| { + let _guard = ENV_LOCK.write(); + cvt_env(unsafe { libc::unsetenv(nbuf.as_ptr()) }).map(drop) + }) +} + +/// In kmclib, `setenv` and `unsetenv` don't always set `errno`, so this +/// function just returns a generic error. +fn cvt_env(t: c_int) -> io::Result { + if t == -1 { Err(io::const_io_error!(io::ErrorKind::Uncategorized, "failure")) } else { Ok(t) } +} + +pub fn temp_dir() -> PathBuf { + panic!("no standard temporary directory on this platform") +} + +pub fn home_dir() -> Option { + None +} + +pub fn exit(code: i32) -> ! { + rtabort!("exit({}) called", code); +} + +pub fn getpid() -> u32 { + panic!("no pids on this platform") +} diff --git a/crux-mir/lib/std/src/sys/solid/path.rs b/crux-mir/lib/std/src/sys/solid/path.rs new file mode 100644 index 000000000..7045c9be2 --- /dev/null +++ b/crux-mir/lib/std/src/sys/solid/path.rs @@ -0,0 +1,25 @@ +use crate::ffi::OsStr; +use crate::io; +use crate::path::{Path, PathBuf, Prefix}; +use crate::sys::unsupported; + +#[inline] +pub fn is_sep_byte(b: u8) -> bool { + b == b'\\' +} + +#[inline] +pub fn is_verbatim_sep(b: u8) -> bool { + b == b'\\' +} + +pub fn parse_prefix(_: &OsStr) -> Option> { + None +} + +pub const MAIN_SEP_STR: &str = "\\"; +pub const MAIN_SEP: char = '\\'; + +pub(crate) fn absolute(_path: &Path) -> io::Result { + unsupported() +} diff --git a/crux-mir/lib/std/src/sys/solid/rwlock.rs b/crux-mir/lib/std/src/sys/solid/rwlock.rs new file mode 100644 index 000000000..ecb4eb83b --- /dev/null +++ b/crux-mir/lib/std/src/sys/solid/rwlock.rs @@ -0,0 +1,93 @@ +//! A readers-writer lock implementation backed by the SOLID kernel extension. +use super::{ + abi, + itron::{ + error::{expect_success, expect_success_aborting, fail, ItronError}, + spin::SpinIdOnceCell, + }, +}; + +pub struct RwLock { + /// The ID of the underlying mutex object + rwl: SpinIdOnceCell<()>, +} + +// Safety: `num_readers` is protected by `mtx_num_readers` +unsafe impl Send for RwLock {} +unsafe impl Sync for RwLock {} + +fn new_rwl() -> Result { + ItronError::err_if_negative(unsafe { abi::rwl_acre_rwl() }) +} + +impl RwLock { + #[inline] + pub const fn new() -> RwLock { + RwLock { rwl: SpinIdOnceCell::new() } + } + + /// Get the inner mutex's ID, which is lazily created. + fn raw(&self) -> abi::ID { + match self.rwl.get_or_try_init(|| new_rwl().map(|id| (id, ()))) { + Ok((id, ())) => id, + Err(e) => fail(e, &"rwl_acre_rwl"), + } + } + + #[inline] + pub fn read(&self) { + let rwl = self.raw(); + expect_success(unsafe { abi::rwl_loc_rdl(rwl) }, &"rwl_loc_rdl"); + } + + #[inline] + pub fn try_read(&self) -> bool { + let rwl = self.raw(); + match unsafe { abi::rwl_ploc_rdl(rwl) } { + abi::E_TMOUT => false, + er => { + expect_success(er, &"rwl_ploc_rdl"); + true + } + } + } + + #[inline] + pub fn write(&self) { + let rwl = self.raw(); + expect_success(unsafe { abi::rwl_loc_wrl(rwl) }, &"rwl_loc_wrl"); + } + + #[inline] + pub fn try_write(&self) -> bool { + let rwl = self.raw(); + match unsafe { abi::rwl_ploc_wrl(rwl) } { + abi::E_TMOUT => false, + er => { + expect_success(er, &"rwl_ploc_wrl"); + true + } + } + } + + #[inline] + pub unsafe fn read_unlock(&self) { + let rwl = self.raw(); + expect_success_aborting(unsafe { abi::rwl_unl_rwl(rwl) }, &"rwl_unl_rwl"); + } + + #[inline] + pub unsafe fn write_unlock(&self) { + let rwl = self.raw(); + expect_success_aborting(unsafe { abi::rwl_unl_rwl(rwl) }, &"rwl_unl_rwl"); + } +} + +impl Drop for RwLock { + #[inline] + fn drop(&mut self) { + if let Some(rwl) = self.rwl.get().map(|x| x.0) { + expect_success_aborting(unsafe { abi::rwl_del_rwl(rwl) }, &"rwl_del_rwl"); + } + } +} diff --git a/crux-mir/lib/std/src/sys/solid/stdio.rs b/crux-mir/lib/std/src/sys/solid/stdio.rs new file mode 100644 index 000000000..50f017696 --- /dev/null +++ b/crux-mir/lib/std/src/sys/solid/stdio.rs @@ -0,0 +1,80 @@ +use super::abi; +use crate::io; + +pub struct Stdin; +pub struct Stdout; +pub struct Stderr; +struct PanicOutput; + +impl Stdin { + pub const fn new() -> Stdin { + Stdin + } +} + +impl io::Read for Stdin { + fn read(&mut self, _buf: &mut [u8]) -> io::Result { + Ok(0) + } +} + +impl Stdout { + pub const fn new() -> Stdout { + Stdout + } +} + +impl io::Write for Stdout { + fn write(&mut self, buf: &[u8]) -> io::Result { + unsafe { abi::SOLID_LOG_write(buf.as_ptr(), buf.len()) }; + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl Stderr { + pub const fn new() -> Stderr { + Stderr + } +} + +impl io::Write for Stderr { + fn write(&mut self, buf: &[u8]) -> io::Result { + unsafe { abi::SOLID_LOG_write(buf.as_ptr(), buf.len()) }; + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl PanicOutput { + pub const fn new() -> PanicOutput { + PanicOutput + } +} + +impl io::Write for PanicOutput { + fn write(&mut self, buf: &[u8]) -> io::Result { + unsafe { abi::SOLID_LOG_write(buf.as_ptr(), buf.len()) }; + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +pub const STDIN_BUF_SIZE: usize = 0; + +pub fn is_ebadf(_err: &io::Error) -> bool { + true +} + +pub fn panic_output() -> Option { + Some(PanicOutput::new()) +} diff --git a/crux-mir/lib/std/src/sys/solid/thread_local_dtor.rs b/crux-mir/lib/std/src/sys/solid/thread_local_dtor.rs new file mode 100644 index 000000000..973564570 --- /dev/null +++ b/crux-mir/lib/std/src/sys/solid/thread_local_dtor.rs @@ -0,0 +1,50 @@ +#![cfg(target_thread_local)] +#![unstable(feature = "thread_local_internals", issue = "none")] + +// Simplify dtor registration by using a list of destructors. + +use super::{abi, itron::task}; +use crate::cell::Cell; +use crate::ptr; + +#[thread_local] +static DTORS: Cell<*mut List> = Cell::new(ptr::null_mut()); + +type List = Vec<(*mut u8, unsafe extern "C" fn(*mut u8))>; + +pub unsafe fn register_dtor(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { + if DTORS.get().is_null() { + let tid = task::current_task_id_aborting(); + let v: Box = box Vec::new(); + DTORS.set(Box::into_raw(v)); + + // Register `tls_dtor` to make sure the TLS destructors are called + // for tasks created by other means than `std::thread` + unsafe { abi::SOLID_TLS_AddDestructor(tid as i32, tls_dtor) }; + } + + let list: &mut List = unsafe { &mut *DTORS.get() }; + list.push((t, dtor)); +} + +pub unsafe fn run_dtors() { + let ptr = DTORS.get(); + if !ptr.is_null() { + // Swap the destructor list, call all registered destructors, + // and repeat this until the list becomes permanently empty. + while let Some(list) = Some(crate::mem::replace(unsafe { &mut *ptr }, Vec::new())) + .filter(|list| !list.is_empty()) + { + for (ptr, dtor) in list.into_iter() { + unsafe { dtor(ptr) }; + } + } + + // Drop the destructor list + unsafe { Box::from_raw(DTORS.replace(ptr::null_mut())) }; + } +} + +unsafe extern "C" fn tls_dtor(_unused: *mut u8) { + unsafe { run_dtors() }; +} diff --git a/crux-mir/lib/std/src/sys/solid/thread_local_key.rs b/crux-mir/lib/std/src/sys/solid/thread_local_key.rs new file mode 100644 index 000000000..b37bf9996 --- /dev/null +++ b/crux-mir/lib/std/src/sys/solid/thread_local_key.rs @@ -0,0 +1,21 @@ +pub type Key = usize; + +#[inline] +pub unsafe fn create(_dtor: Option) -> Key { + panic!("should not be used on the solid target"); +} + +#[inline] +pub unsafe fn set(_key: Key, _value: *mut u8) { + panic!("should not be used on the solid target"); +} + +#[inline] +pub unsafe fn get(_key: Key) -> *mut u8 { + panic!("should not be used on the solid target"); +} + +#[inline] +pub unsafe fn destroy(_key: Key) { + panic!("should not be used on the solid target"); +} diff --git a/crux-mir/lib/std/src/sys/solid/time.rs b/crux-mir/lib/std/src/sys/solid/time.rs new file mode 100644 index 000000000..ce31cb45a --- /dev/null +++ b/crux-mir/lib/std/src/sys/solid/time.rs @@ -0,0 +1,56 @@ +use super::{abi, error::expect_success}; +use crate::{mem::MaybeUninit, time::Duration}; + +pub use super::itron::time::Instant; + +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] +pub struct SystemTime(abi::time_t); + +pub const UNIX_EPOCH: SystemTime = SystemTime(0); + +impl SystemTime { + pub fn now() -> SystemTime { + let rtc = unsafe { + let mut out = MaybeUninit::zeroed(); + expect_success(abi::SOLID_RTC_ReadTime(out.as_mut_ptr()), &"SOLID_RTC_ReadTime"); + out.assume_init() + }; + let t = unsafe { + libc::mktime(&mut libc::tm { + tm_sec: rtc.tm_sec, + tm_min: rtc.tm_min, + tm_hour: rtc.tm_hour, + tm_mday: rtc.tm_mday, + tm_mon: rtc.tm_mon - 1, + tm_year: rtc.tm_year, + tm_wday: rtc.tm_wday, + tm_yday: 0, + tm_isdst: 0, + tm_gmtoff: 0, + tm_zone: crate::ptr::null_mut(), + }) + }; + assert_ne!(t, -1, "mktime failed"); + SystemTime(t) + } + + pub(super) fn from_time_t(t: abi::time_t) -> Self { + Self(t) + } + + pub fn sub_time(&self, other: &SystemTime) -> Result { + if self.0 >= other.0 { + Ok(Duration::from_secs((self.0 as u64).wrapping_sub(other.0 as u64))) + } else { + Err(Duration::from_secs((other.0 as u64).wrapping_sub(self.0 as u64))) + } + } + + pub fn checked_add_duration(&self, other: &Duration) -> Option { + Some(SystemTime(self.0.checked_add(other.as_secs().try_into().ok()?)?)) + } + + pub fn checked_sub_duration(&self, other: &Duration) -> Option { + Some(SystemTime(self.0.checked_sub(other.as_secs().try_into().ok()?)?)) + } +} diff --git a/crux-mir/lib/std/src/sys/unix/alloc.rs b/crux-mir/lib/std/src/sys/unix/alloc.rs index 77417e413..9d6567c9f 100644 --- a/crux-mir/lib/std/src/sys/unix/alloc.rs +++ b/crux-mir/lib/std/src/sys/unix/alloc.rs @@ -1,6 +1,6 @@ use crate::alloc::{GlobalAlloc, Layout, System}; use crate::ptr; -use crate::sys_common::alloc::{realloc_fallback, MIN_ALIGN}; +use crate::sys::common::alloc::{realloc_fallback, MIN_ALIGN}; #[stable(feature = "alloc_system_type", since = "1.28.0")] unsafe impl GlobalAlloc for System { @@ -52,36 +52,50 @@ unsafe impl GlobalAlloc for System { } } -#[cfg(any(target_os = "android", target_os = "redox", target_os = "solaris"))] -#[inline] -unsafe fn aligned_malloc(layout: &Layout) -> *mut u8 { - // On android we currently target API level 9 which unfortunately - // doesn't have the `posix_memalign` API used below. Instead we use - // `memalign`, but this unfortunately has the property on some systems - // where the memory returned cannot be deallocated by `free`! - // - // Upon closer inspection, however, this appears to work just fine with - // Android, so for this platform we should be fine to call `memalign` - // (which is present in API level 9). Some helpful references could - // possibly be chromium using memalign [1], attempts at documenting that - // memalign + free is ok [2] [3], or the current source of chromium - // which still uses memalign on android [4]. - // - // [1]: https://codereview.chromium.org/10796020/ - // [2]: https://code.google.com/p/android/issues/detail?id=35391 - // [3]: https://bugs.chromium.org/p/chromium/issues/detail?id=138579 - // [4]: https://chromium.googlesource.com/chromium/src/base/+/master/ - // /memory/aligned_memory.cc - libc::memalign(layout.align(), layout.size()) as *mut u8 -} - -#[cfg(not(any(target_os = "android", target_os = "redox", target_os = "solaris")))] -#[inline] -unsafe fn aligned_malloc(layout: &Layout) -> *mut u8 { - let mut out = ptr::null_mut(); - // posix_memalign requires that the alignment be a multiple of `sizeof(void*)`. - // Since these are all powers of 2, we can just use max. - let align = layout.align().max(crate::mem::size_of::()); - let ret = libc::posix_memalign(&mut out, align, layout.size()); - if ret != 0 { ptr::null_mut() } else { out as *mut u8 } +cfg_if::cfg_if! { + if #[cfg(any( + target_os = "android", + target_os = "illumos", + target_os = "redox", + target_os = "solaris", + target_os = "espidf", + target_os = "horizon" + ))] { + #[inline] + unsafe fn aligned_malloc(layout: &Layout) -> *mut u8 { + // On android we currently target API level 9 which unfortunately + // doesn't have the `posix_memalign` API used below. Instead we use + // `memalign`, but this unfortunately has the property on some systems + // where the memory returned cannot be deallocated by `free`! + // + // Upon closer inspection, however, this appears to work just fine with + // Android, so for this platform we should be fine to call `memalign` + // (which is present in API level 9). Some helpful references could + // possibly be chromium using memalign [1], attempts at documenting that + // memalign + free is ok [2] [3], or the current source of chromium + // which still uses memalign on android [4]. + // + // [1]: https://codereview.chromium.org/10796020/ + // [2]: https://code.google.com/p/android/issues/detail?id=35391 + // [3]: https://bugs.chromium.org/p/chromium/issues/detail?id=138579 + // [4]: https://chromium.googlesource.com/chromium/src/base/+/master/ + // /memory/aligned_memory.cc + libc::memalign(layout.align(), layout.size()) as *mut u8 + } + } else if #[cfg(target_os = "wasi")] { + #[inline] + unsafe fn aligned_malloc(layout: &Layout) -> *mut u8 { + libc::aligned_alloc(layout.align(), layout.size()) as *mut u8 + } + } else { + #[inline] + unsafe fn aligned_malloc(layout: &Layout) -> *mut u8 { + let mut out = ptr::null_mut(); + // posix_memalign requires that the alignment be a multiple of `sizeof(void*)`. + // Since these are all powers of 2, we can just use max. + let align = layout.align().max(crate::mem::size_of::()); + let ret = libc::posix_memalign(&mut out, align, layout.size()); + if ret != 0 { ptr::null_mut() } else { out as *mut u8 } + } + } } diff --git a/crux-mir/lib/std/src/sys/unix/android.rs b/crux-mir/lib/std/src/sys/unix/android.rs index 8fc2599f0..0f704994f 100644 --- a/crux-mir/lib/std/src/sys/unix/android.rs +++ b/crux-mir/lib/std/src/sys/unix/android.rs @@ -1,7 +1,7 @@ //! Android ABI-compatibility module //! -//! The ABI of Android has changed quite a bit over time, and libstd attempts to -//! be both forwards and backwards compatible as much as possible. We want to +//! The ABI of Android has changed quite a bit over time, and std attempts to be +//! both forwards and backwards compatible as much as possible. We want to //! always work with the most recent version of Android, but we also want to //! work with older versions of Android for whenever projects need to. //! @@ -18,11 +18,9 @@ #![cfg(target_os = "android")] -use libc::{c_int, c_void, sighandler_t, size_t, ssize_t}; -use libc::{ftruncate, pread, pwrite}; +use libc::{c_int, sighandler_t}; -use super::{cvt, cvt_r}; -use crate::io; +use super::weak::weak; // The `log2` and `log2f` functions apparently appeared in android-18, or at // least you can see they're not present in the android-17 header [1] and they @@ -81,87 +79,3 @@ pub unsafe fn signal(signum: c_int, handler: sighandler_t) -> sighandler_t { let f = f.expect("neither `signal` nor `bsd_signal` symbols found"); f(signum, handler) } - -// The `ftruncate64` symbol apparently appeared in android-12, so we do some -// dynamic detection to see if we can figure out whether `ftruncate64` exists. -// -// If it doesn't we just fall back to `ftruncate`, generating an error for -// too-large values. -#[cfg(target_pointer_width = "32")] -pub fn ftruncate64(fd: c_int, size: u64) -> io::Result<()> { - weak!(fn ftruncate64(c_int, i64) -> c_int); - - unsafe { - match ftruncate64.get() { - Some(f) => cvt_r(|| f(fd, size as i64)).map(drop), - None => { - if size > i32::max_value() as u64 { - Err(io::Error::new(io::ErrorKind::InvalidInput, "cannot truncate >2GB")) - } else { - cvt_r(|| ftruncate(fd, size as i32)).map(drop) - } - } - } - } -} - -#[cfg(target_pointer_width = "64")] -pub fn ftruncate64(fd: c_int, size: u64) -> io::Result<()> { - unsafe { cvt_r(|| ftruncate(fd, size as i64)).map(drop) } -} - -#[cfg(target_pointer_width = "32")] -pub unsafe fn cvt_pread64( - fd: c_int, - buf: *mut c_void, - count: size_t, - offset: i64, -) -> io::Result { - use crate::convert::TryInto; - weak!(fn pread64(c_int, *mut c_void, size_t, i64) -> ssize_t); - pread64.get().map(|f| cvt(f(fd, buf, count, offset))).unwrap_or_else(|| { - if let Ok(o) = offset.try_into() { - cvt(pread(fd, buf, count, o)) - } else { - Err(io::Error::new(io::ErrorKind::InvalidInput, "cannot pread >2GB")) - } - }) -} - -#[cfg(target_pointer_width = "32")] -pub unsafe fn cvt_pwrite64( - fd: c_int, - buf: *const c_void, - count: size_t, - offset: i64, -) -> io::Result { - use crate::convert::TryInto; - weak!(fn pwrite64(c_int, *const c_void, size_t, i64) -> ssize_t); - pwrite64.get().map(|f| cvt(f(fd, buf, count, offset))).unwrap_or_else(|| { - if let Ok(o) = offset.try_into() { - cvt(pwrite(fd, buf, count, o)) - } else { - Err(io::Error::new(io::ErrorKind::InvalidInput, "cannot pwrite >2GB")) - } - }) -} - -#[cfg(target_pointer_width = "64")] -pub unsafe fn cvt_pread64( - fd: c_int, - buf: *mut c_void, - count: size_t, - offset: i64, -) -> io::Result { - cvt(pread(fd, buf, count, offset)) -} - -#[cfg(target_pointer_width = "64")] -pub unsafe fn cvt_pwrite64( - fd: c_int, - buf: *const c_void, - count: size_t, - offset: i64, -) -> io::Result { - cvt(pwrite(fd, buf, count, offset)) -} diff --git a/crux-mir/lib/std/src/sys/unix/args.rs b/crux-mir/lib/std/src/sys/unix/args.rs index 09acc3f6e..a342f0f5e 100644 --- a/crux-mir/lib/std/src/sys/unix/args.rs +++ b/crux-mir/lib/std/src/sys/unix/args.rs @@ -6,7 +6,7 @@ #![allow(dead_code)] // runtime init functions not used during testing use crate::ffi::OsString; -use crate::marker::PhantomData; +use crate::fmt; use crate::vec; /// One-time global initialization. @@ -14,11 +14,6 @@ pub unsafe fn init(argc: isize, argv: *const *const u8) { imp::init(argc, argv) } -/// One-time global cleanup. -pub unsafe fn cleanup() { - imp::cleanup() -} - /// Returns the command line arguments pub fn args() -> Args { imp::args() @@ -26,12 +21,14 @@ pub fn args() -> Args { pub struct Args { iter: vec::IntoIter, - _dont_send_or_sync_me: PhantomData<*mut ()>, } -impl Args { - pub fn inner_debug(&self) -> &[OsString] { - self.iter.as_slice() +impl !Send for Args {} +impl !Sync for Args {} + +impl fmt::Debug for Args { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.iter.as_slice().fmt(f) } } @@ -65,31 +62,36 @@ impl DoubleEndedIterator for Args { target_os = "netbsd", target_os = "openbsd", target_os = "solaris", + target_os = "illumos", target_os = "emscripten", target_os = "haiku", target_os = "l4re", target_os = "fuchsia", - target_os = "redox" + target_os = "redox", + target_os = "vxworks", + target_os = "horizon" ))] mod imp { use super::Args; use crate::ffi::{CStr, OsString}; - use crate::marker::PhantomData; use crate::os::unix::prelude::*; use crate::ptr; + use crate::sync::atomic::{AtomicIsize, AtomicPtr, Ordering}; - use crate::sys_common::mutex::Mutex; - - static mut ARGC: isize = 0; - static mut ARGV: *const *const u8 = ptr::null(); - // We never call `ENV_LOCK.init()`, so it is UB to attempt to - // acquire this mutex reentrantly! - static LOCK: Mutex = Mutex::new(); + // The system-provided argc and argv, which we store in static memory + // here so that we can defer the work of parsing them until its actually + // needed. + // + // Note that we never mutate argv/argc, the argv array, or the argv + // strings, which allows the code in this file to be very simple. + static ARGC: AtomicIsize = AtomicIsize::new(0); + static ARGV: AtomicPtr<*const u8> = AtomicPtr::new(ptr::null_mut()); unsafe fn really_init(argc: isize, argv: *const *const u8) { - let _guard = LOCK.lock(); - ARGC = argc; - ARGV = argv; + // These don't need to be ordered with each other or other stores, + // because they only hold the unmodified system-provide argv/argc. + ARGC.store(argc, Ordering::Relaxed); + ARGV.store(argv as *mut _, Ordering::Relaxed); } #[inline(always)] @@ -123,22 +125,25 @@ mod imp { init_wrapper }; - pub unsafe fn cleanup() { - let _guard = LOCK.lock(); - ARGC = 0; - ARGV = ptr::null(); - } - pub fn args() -> Args { - Args { iter: clone().into_iter(), _dont_send_or_sync_me: PhantomData } + Args { iter: clone().into_iter() } } fn clone() -> Vec { unsafe { - let _guard = LOCK.lock(); - (0..ARGC) + // Load ARGC and ARGV, which hold the unmodified system-provided + // argc/argv, so we can read the pointed-to memory without atomics + // or synchronization. + // + // If either ARGC or ARGV is still zero or null, then either there + // really are no arguments, or someone is asking for `args()` + // before initialization has completed, and we return an empty + // list. + let argv = ARGV.load(Ordering::Relaxed); + let argc = if argv.is_null() { 0 } else { ARGC.load(Ordering::Relaxed) }; + (0..argc) .map(|i| { - let cstr = CStr::from_ptr(*ARGV.offset(i) as *const libc::c_char); + let cstr = CStr::from_ptr(*argv.offset(i) as *const libc::c_char); OsStringExt::from_vec(cstr.to_bytes().to_vec()) }) .collect() @@ -146,16 +151,13 @@ mod imp { } } -#[cfg(any(target_os = "macos", target_os = "ios"))] +#[cfg(any(target_os = "macos", target_os = "ios", target_os = "watchos"))] mod imp { use super::Args; use crate::ffi::CStr; - use crate::marker::PhantomData; pub unsafe fn init(_argc: isize, _argv: *const *const u8) {} - pub fn cleanup() {} - #[cfg(target_os = "macos")] pub fn args() -> Args { use crate::os::unix::prelude::*; @@ -175,7 +177,7 @@ mod imp { }) .collect::>() }; - Args { iter: vec.into_iter(), _dont_send_or_sync_me: PhantomData } + Args { iter: vec.into_iter() } } // As _NSGetArgc and _NSGetArgv aren't mentioned in iOS docs @@ -190,7 +192,7 @@ mod imp { // for i in (0..[args count]) // res.push([args objectAtIndex:i]) // res - #[cfg(target_os = "ios")] + #[cfg(any(target_os = "ios", target_os = "watchos"))] pub fn args() -> Args { use crate::ffi::OsString; use crate::mem; @@ -204,6 +206,7 @@ mod imp { #[cfg(target_arch = "aarch64")] extern "C" { fn objc_msgSend(obj: NsId, sel: Sel) -> NsId; + #[allow(clashing_extern_declarations)] #[link_name = "objc_msgSend"] fn objc_msgSend_ul(obj: NsId, sel: Sel, i: libc::c_ulong) -> NsId; } @@ -211,6 +214,7 @@ mod imp { #[cfg(not(target_arch = "aarch64"))] extern "C" { fn objc_msgSend(obj: NsId, sel: Sel, ...) -> NsId; + #[allow(clashing_extern_declarations)] #[link_name = "objc_msgSend"] fn objc_msgSend_ul(obj: NsId, sel: Sel, ...) -> NsId; } @@ -240,6 +244,18 @@ mod imp { } } - Args { iter: res.into_iter(), _dont_send_or_sync_me: PhantomData } + Args { iter: res.into_iter() } + } +} + +#[cfg(target_os = "espidf")] +mod imp { + use super::Args; + + #[inline(always)] + pub unsafe fn init(_argc: isize, _argv: *const *const u8) {} + + pub fn args() -> Args { + Args { iter: Vec::new().into_iter() } } } diff --git a/crux-mir/lib/std/src/sys/unix/cmath.rs b/crux-mir/lib/std/src/sys/unix/cmath.rs index f327b69fc..2bf80d7a4 100644 --- a/crux-mir/lib/std/src/sys/unix/cmath.rs +++ b/crux-mir/lib/std/src/sys/unix/cmath.rs @@ -1,32 +1,33 @@ #![cfg(not(test))] -use libc::{c_double, c_float}; +// These symbols are all defined by `libm`, +// or by `compiler-builtins` on unsupported platforms. extern "C" { - pub fn acos(n: c_double) -> c_double; - pub fn acosf(n: c_float) -> c_float; - pub fn asin(n: c_double) -> c_double; - pub fn asinf(n: c_float) -> c_float; - pub fn atan(n: c_double) -> c_double; - pub fn atan2(a: c_double, b: c_double) -> c_double; - pub fn atan2f(a: c_float, b: c_float) -> c_float; - pub fn atanf(n: c_float) -> c_float; - pub fn cbrt(n: c_double) -> c_double; - pub fn cbrtf(n: c_float) -> c_float; - pub fn cosh(n: c_double) -> c_double; - pub fn coshf(n: c_float) -> c_float; - pub fn expm1(n: c_double) -> c_double; - pub fn expm1f(n: c_float) -> c_float; - pub fn fdim(a: c_double, b: c_double) -> c_double; - pub fn fdimf(a: c_float, b: c_float) -> c_float; - pub fn hypot(x: c_double, y: c_double) -> c_double; - pub fn hypotf(x: c_float, y: c_float) -> c_float; - pub fn log1p(n: c_double) -> c_double; - pub fn log1pf(n: c_float) -> c_float; - pub fn sinh(n: c_double) -> c_double; - pub fn sinhf(n: c_float) -> c_float; - pub fn tan(n: c_double) -> c_double; - pub fn tanf(n: c_float) -> c_float; - pub fn tanh(n: c_double) -> c_double; - pub fn tanhf(n: c_float) -> c_float; + pub fn acos(n: f64) -> f64; + pub fn acosf(n: f32) -> f32; + pub fn asin(n: f64) -> f64; + pub fn asinf(n: f32) -> f32; + pub fn atan(n: f64) -> f64; + pub fn atan2(a: f64, b: f64) -> f64; + pub fn atan2f(a: f32, b: f32) -> f32; + pub fn atanf(n: f32) -> f32; + pub fn cbrt(n: f64) -> f64; + pub fn cbrtf(n: f32) -> f32; + pub fn cosh(n: f64) -> f64; + pub fn coshf(n: f32) -> f32; + pub fn expm1(n: f64) -> f64; + pub fn expm1f(n: f32) -> f32; + pub fn fdim(a: f64, b: f64) -> f64; + pub fn fdimf(a: f32, b: f32) -> f32; + pub fn hypot(x: f64, y: f64) -> f64; + pub fn hypotf(x: f32, y: f32) -> f32; + pub fn log1p(n: f64) -> f64; + pub fn log1pf(n: f32) -> f32; + pub fn sinh(n: f64) -> f64; + pub fn sinhf(n: f32) -> f32; + pub fn tan(n: f64) -> f64; + pub fn tanf(n: f32) -> f32; + pub fn tanh(n: f64) -> f64; + pub fn tanhf(n: f32) -> f32; } diff --git a/crux-mir/lib/std/src/sys/unix/condvar.rs b/crux-mir/lib/std/src/sys/unix/condvar.rs deleted file mode 100644 index b4896b7ad..000000000 --- a/crux-mir/lib/std/src/sys/unix/condvar.rs +++ /dev/null @@ -1,178 +0,0 @@ -use crate::cell::UnsafeCell; -use crate::sys::mutex::{self, Mutex}; -use crate::time::Duration; - -pub struct Condvar { - inner: UnsafeCell, -} - -unsafe impl Send for Condvar {} -unsafe impl Sync for Condvar {} - -const TIMESPEC_MAX: libc::timespec = - libc::timespec { tv_sec: ::max_value(), tv_nsec: 1_000_000_000 - 1 }; - -fn saturating_cast_to_time_t(value: u64) -> libc::time_t { - if value > ::max_value() as u64 { - ::max_value() - } else { - value as libc::time_t - } -} - -impl Condvar { - pub const fn new() -> Condvar { - // Might be moved and address is changing it is better to avoid - // initialization of potentially opaque OS data before it landed - Condvar { inner: UnsafeCell::new(libc::PTHREAD_COND_INITIALIZER) } - } - - #[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "l4re", - target_os = "android", - target_os = "redox" - ))] - pub unsafe fn init(&mut self) {} - - #[cfg(not(any( - target_os = "macos", - target_os = "ios", - target_os = "l4re", - target_os = "android", - target_os = "redox" - )))] - pub unsafe fn init(&mut self) { - use crate::mem::MaybeUninit; - let mut attr = MaybeUninit::::uninit(); - let r = libc::pthread_condattr_init(attr.as_mut_ptr()); - assert_eq!(r, 0); - let r = libc::pthread_condattr_setclock(attr.as_mut_ptr(), libc::CLOCK_MONOTONIC); - assert_eq!(r, 0); - let r = libc::pthread_cond_init(self.inner.get(), attr.as_ptr()); - assert_eq!(r, 0); - let r = libc::pthread_condattr_destroy(attr.as_mut_ptr()); - assert_eq!(r, 0); - } - - #[inline] - pub unsafe fn notify_one(&self) { - let r = libc::pthread_cond_signal(self.inner.get()); - debug_assert_eq!(r, 0); - } - - #[inline] - pub unsafe fn notify_all(&self) { - let r = libc::pthread_cond_broadcast(self.inner.get()); - debug_assert_eq!(r, 0); - } - - #[inline] - pub unsafe fn wait(&self, mutex: &Mutex) { - let r = libc::pthread_cond_wait(self.inner.get(), mutex::raw(mutex)); - debug_assert_eq!(r, 0); - } - - // This implementation is used on systems that support pthread_condattr_setclock - // where we configure condition variable to use monotonic clock (instead of - // default system clock). This approach avoids all problems that result - // from changes made to the system time. - #[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "android")))] - pub unsafe fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool { - use crate::mem; - - let mut now: libc::timespec = mem::zeroed(); - let r = libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut now); - assert_eq!(r, 0); - - // Nanosecond calculations can't overflow because both values are below 1e9. - let nsec = dur.subsec_nanos() + now.tv_nsec as u32; - - let sec = saturating_cast_to_time_t(dur.as_secs()) - .checked_add((nsec / 1_000_000_000) as libc::time_t) - .and_then(|s| s.checked_add(now.tv_sec)); - let nsec = nsec % 1_000_000_000; - - let timeout = - sec.map(|s| libc::timespec { tv_sec: s, tv_nsec: nsec as _ }).unwrap_or(TIMESPEC_MAX); - - let r = libc::pthread_cond_timedwait(self.inner.get(), mutex::raw(mutex), &timeout); - assert!(r == libc::ETIMEDOUT || r == 0); - r == 0 - } - - // This implementation is modeled after libcxx's condition_variable - // https://github.com/llvm-mirror/libcxx/blob/release_35/src/condition_variable.cpp#L46 - // https://github.com/llvm-mirror/libcxx/blob/release_35/include/__mutex_base#L367 - #[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))] - pub unsafe fn wait_timeout(&self, mutex: &Mutex, mut dur: Duration) -> bool { - use crate::ptr; - use crate::time::Instant; - - // 1000 years - let max_dur = Duration::from_secs(1000 * 365 * 86400); - - if dur > max_dur { - // OSX implementation of `pthread_cond_timedwait` is buggy - // with super long durations. When duration is greater than - // 0x100_0000_0000_0000 seconds, `pthread_cond_timedwait` - // in macOS Sierra return error 316. - // - // This program demonstrates the issue: - // https://gist.github.com/stepancheg/198db4623a20aad2ad7cddb8fda4a63c - // - // To work around this issue, and possible bugs of other OSes, timeout - // is clamped to 1000 years, which is allowable per the API of `wait_timeout` - // because of spurious wakeups. - - dur = max_dur; - } - - // First, figure out what time it currently is, in both system and - // stable time. pthread_cond_timedwait uses system time, but we want to - // report timeout based on stable time. - let mut sys_now = libc::timeval { tv_sec: 0, tv_usec: 0 }; - let stable_now = Instant::now(); - let r = libc::gettimeofday(&mut sys_now, ptr::null_mut()); - debug_assert_eq!(r, 0); - - let nsec = dur.subsec_nanos() as libc::c_long + (sys_now.tv_usec * 1000) as libc::c_long; - let extra = (nsec / 1_000_000_000) as libc::time_t; - let nsec = nsec % 1_000_000_000; - let seconds = saturating_cast_to_time_t(dur.as_secs()); - - let timeout = sys_now - .tv_sec - .checked_add(extra) - .and_then(|s| s.checked_add(seconds)) - .map(|s| libc::timespec { tv_sec: s, tv_nsec: nsec }) - .unwrap_or(TIMESPEC_MAX); - - // And wait! - let r = libc::pthread_cond_timedwait(self.inner.get(), mutex::raw(mutex), &timeout); - debug_assert!(r == libc::ETIMEDOUT || r == 0); - - // ETIMEDOUT is not a totally reliable method of determining timeout due - // to clock shifts, so do the check ourselves - stable_now.elapsed() < dur - } - - #[inline] - #[cfg(not(target_os = "dragonfly"))] - pub unsafe fn destroy(&self) { - let r = libc::pthread_cond_destroy(self.inner.get()); - debug_assert_eq!(r, 0); - } - - #[inline] - #[cfg(target_os = "dragonfly")] - pub unsafe fn destroy(&self) { - let r = libc::pthread_cond_destroy(self.inner.get()); - // On DragonFly pthread_cond_destroy() returns EINVAL if called on - // a condvar that was just initialized with - // libc::PTHREAD_COND_INITIALIZER. Once it is used or - // pthread_cond_init() is called, this behaviour no longer occurs. - debug_assert!(r == 0 || r == libc::EINVAL); - } -} diff --git a/crux-mir/lib/std/src/sys/unix/env.rs b/crux-mir/lib/std/src/sys/unix/env.rs index 984bcfa45..c9ba661c8 100644 --- a/crux-mir/lib/std/src/sys/unix/env.rs +++ b/crux-mir/lib/std/src/sys/unix/env.rs @@ -31,6 +31,17 @@ pub mod os { pub const EXE_EXTENSION: &str = ""; } +#[cfg(target_os = "watchos")] +pub mod os { + pub const FAMILY: &str = "unix"; + pub const OS: &str = "watchos"; + pub const DLL_PREFIX: &str = "lib"; + pub const DLL_SUFFIX: &str = ".dylib"; + pub const DLL_EXTENSION: &str = "dylib"; + pub const EXE_SUFFIX: &str = ""; + pub const EXE_EXTENSION: &str = ""; +} + #[cfg(target_os = "freebsd")] pub mod os { pub const FAMILY: &str = "unix"; @@ -97,6 +108,17 @@ pub mod os { pub const EXE_EXTENSION: &str = ""; } +#[cfg(target_os = "illumos")] +pub mod os { + pub const FAMILY: &str = "unix"; + pub const OS: &str = "illumos"; + pub const DLL_PREFIX: &str = "lib"; + pub const DLL_SUFFIX: &str = ".so"; + pub const DLL_EXTENSION: &str = "so"; + pub const EXE_SUFFIX: &str = ""; + pub const EXE_EXTENSION: &str = ""; +} + #[cfg(target_os = "haiku")] pub mod os { pub const FAMILY: &str = "unix"; @@ -108,6 +130,17 @@ pub mod os { pub const EXE_EXTENSION: &str = ""; } +#[cfg(target_os = "horizon")] +pub mod os { + pub const FAMILY: &str = "unix"; + pub const OS: &str = "horizon"; + pub const DLL_PREFIX: &str = "lib"; + pub const DLL_SUFFIX: &str = ".so"; + pub const DLL_EXTENSION: &str = "so"; + pub const EXE_SUFFIX: &str = ".elf"; + pub const EXE_EXTENSION: &str = "elf"; +} + #[cfg(all(target_os = "emscripten", target_arch = "asmjs"))] pub mod os { pub const FAMILY: &str = "unix"; @@ -162,3 +195,25 @@ pub mod os { pub const EXE_SUFFIX: &str = ""; pub const EXE_EXTENSION: &str = ""; } + +#[cfg(target_os = "vxworks")] +pub mod os { + pub const FAMILY: &str = "unix"; + pub const OS: &str = "vxworks"; + pub const DLL_PREFIX: &str = "lib"; + pub const DLL_SUFFIX: &str = ".so"; + pub const DLL_EXTENSION: &str = "so"; + pub const EXE_SUFFIX: &str = ""; + pub const EXE_EXTENSION: &str = ""; +} + +#[cfg(target_os = "espidf")] +pub mod os { + pub const FAMILY: &str = "unix"; + pub const OS: &str = "espidf"; + pub const DLL_PREFIX: &str = "lib"; + pub const DLL_SUFFIX: &str = ".so"; + pub const DLL_EXTENSION: &str = "so"; + pub const EXE_SUFFIX: &str = ""; + pub const EXE_EXTENSION: &str = ""; +} diff --git a/crux-mir/lib/std/src/sys/unix/ext/io.rs b/crux-mir/lib/std/src/sys/unix/ext/io.rs deleted file mode 100644 index 5077e2e28..000000000 --- a/crux-mir/lib/std/src/sys/unix/ext/io.rs +++ /dev/null @@ -1,124 +0,0 @@ -//! Unix-specific extensions to general I/O primitives - -#![stable(feature = "rust1", since = "1.0.0")] - -use crate::fs; -use crate::io; -use crate::os::raw; -use crate::sys; -use crate::sys_common::{AsInner, FromInner, IntoInner}; - -/// Raw file descriptors. -#[stable(feature = "rust1", since = "1.0.0")] -pub type RawFd = raw::c_int; - -/// A trait to extract the raw unix file descriptor from an underlying -/// object. -/// -/// This is only available on unix platforms and must be imported in order -/// to call the method. Windows platforms have a corresponding `AsRawHandle` -/// and `AsRawSocket` set of traits. -#[stable(feature = "rust1", since = "1.0.0")] -pub trait AsRawFd { - /// Extracts the raw file descriptor. - /// - /// This method does **not** pass ownership of the raw file descriptor - /// to the caller. The descriptor is only guaranteed to be valid while - /// the original object has not yet been destroyed. - #[stable(feature = "rust1", since = "1.0.0")] - fn as_raw_fd(&self) -> RawFd; -} - -/// A trait to express the ability to construct an object from a raw file -/// descriptor. -#[stable(feature = "from_raw_os", since = "1.1.0")] -pub trait FromRawFd { - /// Constructs a new instance of `Self` from the given raw file - /// descriptor. - /// - /// This function **consumes ownership** of the specified file - /// descriptor. The returned object will take responsibility for closing - /// it when the object goes out of scope. - /// - /// This function is also unsafe as the primitives currently returned - /// have the contract that they are the sole owner of the file - /// descriptor they are wrapping. Usage of this function could - /// accidentally allow violating this contract which can cause memory - /// unsafety in code that relies on it being true. - #[stable(feature = "from_raw_os", since = "1.1.0")] - unsafe fn from_raw_fd(fd: RawFd) -> Self; -} - -/// A trait to express the ability to consume an object and acquire ownership of -/// its raw file descriptor. -#[stable(feature = "into_raw_os", since = "1.4.0")] -pub trait IntoRawFd { - /// Consumes this object, returning the raw underlying file descriptor. - /// - /// This function **transfers ownership** of the underlying file descriptor - /// to the caller. Callers are then the unique owners of the file descriptor - /// and must close the descriptor once it's no longer needed. - #[stable(feature = "into_raw_os", since = "1.4.0")] - fn into_raw_fd(self) -> RawFd; -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl AsRawFd for fs::File { - fn as_raw_fd(&self) -> RawFd { - self.as_inner().fd().raw() - } -} -#[stable(feature = "from_raw_os", since = "1.1.0")] -impl FromRawFd for fs::File { - unsafe fn from_raw_fd(fd: RawFd) -> fs::File { - fs::File::from_inner(sys::fs::File::from_inner(fd)) - } -} -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawFd for fs::File { - fn into_raw_fd(self) -> RawFd { - self.into_inner().into_fd().into_raw() - } -} - -#[stable(feature = "asraw_stdio", since = "1.21.0")] -impl AsRawFd for io::Stdin { - fn as_raw_fd(&self) -> RawFd { - libc::STDIN_FILENO - } -} - -#[stable(feature = "asraw_stdio", since = "1.21.0")] -impl AsRawFd for io::Stdout { - fn as_raw_fd(&self) -> RawFd { - libc::STDOUT_FILENO - } -} - -#[stable(feature = "asraw_stdio", since = "1.21.0")] -impl AsRawFd for io::Stderr { - fn as_raw_fd(&self) -> RawFd { - libc::STDERR_FILENO - } -} - -#[stable(feature = "asraw_stdio_locks", since = "1.35.0")] -impl<'a> AsRawFd for io::StdinLock<'a> { - fn as_raw_fd(&self) -> RawFd { - libc::STDIN_FILENO - } -} - -#[stable(feature = "asraw_stdio_locks", since = "1.35.0")] -impl<'a> AsRawFd for io::StdoutLock<'a> { - fn as_raw_fd(&self) -> RawFd { - libc::STDOUT_FILENO - } -} - -#[stable(feature = "asraw_stdio_locks", since = "1.35.0")] -impl<'a> AsRawFd for io::StderrLock<'a> { - fn as_raw_fd(&self) -> RawFd { - libc::STDERR_FILENO - } -} diff --git a/crux-mir/lib/std/src/sys/unix/ext/net.rs b/crux-mir/lib/std/src/sys/unix/ext/net.rs deleted file mode 100644 index 4c3cb67c9..000000000 --- a/crux-mir/lib/std/src/sys/unix/ext/net.rs +++ /dev/null @@ -1,2006 +0,0 @@ -#![stable(feature = "unix_socket", since = "1.10.0")] - -//! Unix-specific networking functionality - -#[cfg(unix)] -use libc; - -// FIXME(#43348): Make libc adapt #[doc(cfg(...))] so we don't need these fake definitions here? -#[cfg(not(unix))] -mod libc { - pub use libc::c_int; - pub type socklen_t = u32; - pub struct sockaddr; - #[derive(Clone)] - pub struct sockaddr_un; -} - -use crate::ascii; -use crate::ffi::OsStr; -use crate::fmt; -use crate::io::{self, Initializer, IoSlice, IoSliceMut}; -use crate::mem; -use crate::net::{self, Shutdown}; -use crate::os::unix::ffi::OsStrExt; -use crate::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd}; -use crate::path::Path; -use crate::sys::net::Socket; -use crate::sys::{self, cvt}; -use crate::sys_common::{self, AsInner, FromInner, IntoInner}; -use crate::time::Duration; - -#[cfg(any( - target_os = "linux", - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "openbsd", - target_os = "netbsd", - target_os = "haiku" -))] -use libc::MSG_NOSIGNAL; -#[cfg(not(any( - target_os = "linux", - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "openbsd", - target_os = "netbsd", - target_os = "haiku" -)))] -const MSG_NOSIGNAL: libc::c_int = 0x0; - -fn sun_path_offset(addr: &libc::sockaddr_un) -> usize { - // Work with an actual instance of the type since using a null pointer is UB - let base = addr as *const _ as usize; - let path = &addr.sun_path as *const _ as usize; - path - base -} - -unsafe fn sockaddr_un(path: &Path) -> io::Result<(libc::sockaddr_un, libc::socklen_t)> { - let mut addr: libc::sockaddr_un = mem::zeroed(); - addr.sun_family = libc::AF_UNIX as libc::sa_family_t; - - let bytes = path.as_os_str().as_bytes(); - - if bytes.contains(&0) { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "paths may not contain interior null bytes", - )); - } - - if bytes.len() >= addr.sun_path.len() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "path must be shorter than SUN_LEN", - )); - } - for (dst, src) in addr.sun_path.iter_mut().zip(bytes.iter()) { - *dst = *src as libc::c_char; - } - // null byte for pathname addresses is already there because we zeroed the - // struct - - let mut len = sun_path_offset(&addr) + bytes.len(); - match bytes.get(0) { - Some(&0) | None => {} - Some(_) => len += 1, - } - Ok((addr, len as libc::socklen_t)) -} - -enum AddressKind<'a> { - Unnamed, - Pathname(&'a Path), - Abstract(&'a [u8]), -} - -/// An address associated with a Unix socket. -/// -/// # Examples -/// -/// ``` -/// use std::os::unix::net::UnixListener; -/// -/// let socket = match UnixListener::bind("/tmp/sock") { -/// Ok(sock) => sock, -/// Err(e) => { -/// println!("Couldn't bind: {:?}", e); -/// return -/// } -/// }; -/// let addr = socket.local_addr().expect("Couldn't get local address"); -/// ``` -#[derive(Clone)] -#[stable(feature = "unix_socket", since = "1.10.0")] -pub struct SocketAddr { - addr: libc::sockaddr_un, - len: libc::socklen_t, -} - -impl SocketAddr { - fn new(f: F) -> io::Result - where - F: FnOnce(*mut libc::sockaddr, *mut libc::socklen_t) -> libc::c_int, - { - unsafe { - let mut addr: libc::sockaddr_un = mem::zeroed(); - let mut len = mem::size_of::() as libc::socklen_t; - cvt(f(&mut addr as *mut _ as *mut _, &mut len))?; - SocketAddr::from_parts(addr, len) - } - } - - fn from_parts(addr: libc::sockaddr_un, mut len: libc::socklen_t) -> io::Result { - if len == 0 { - // When there is a datagram from unnamed unix socket - // linux returns zero bytes of address - len = sun_path_offset(&addr) as libc::socklen_t; // i.e., zero-length address - } else if addr.sun_family != libc::AF_UNIX as libc::sa_family_t { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "file descriptor did not correspond to a Unix socket", - )); - } - - Ok(SocketAddr { addr, len }) - } - - /// Returns `true` if the address is unnamed. - /// - /// # Examples - /// - /// A named address: - /// - /// ```no_run - /// use std::os::unix::net::UnixListener; - /// - /// fn main() -> std::io::Result<()> { - /// let socket = UnixListener::bind("/tmp/sock")?; - /// let addr = socket.local_addr().expect("Couldn't get local address"); - /// assert_eq!(addr.is_unnamed(), false); - /// Ok(()) - /// } - /// ``` - /// - /// An unnamed address: - /// - /// ``` - /// use std::os::unix::net::UnixDatagram; - /// - /// fn main() -> std::io::Result<()> { - /// let socket = UnixDatagram::unbound()?; - /// let addr = socket.local_addr().expect("Couldn't get local address"); - /// assert_eq!(addr.is_unnamed(), true); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn is_unnamed(&self) -> bool { - if let AddressKind::Unnamed = self.address() { true } else { false } - } - - /// Returns the contents of this address if it is a `pathname` address. - /// - /// # Examples - /// - /// With a pathname: - /// - /// ```no_run - /// use std::os::unix::net::UnixListener; - /// use std::path::Path; - /// - /// fn main() -> std::io::Result<()> { - /// let socket = UnixListener::bind("/tmp/sock")?; - /// let addr = socket.local_addr().expect("Couldn't get local address"); - /// assert_eq!(addr.as_pathname(), Some(Path::new("/tmp/sock"))); - /// Ok(()) - /// } - /// ``` - /// - /// Without a pathname: - /// - /// ``` - /// use std::os::unix::net::UnixDatagram; - /// - /// fn main() -> std::io::Result<()> { - /// let socket = UnixDatagram::unbound()?; - /// let addr = socket.local_addr().expect("Couldn't get local address"); - /// assert_eq!(addr.as_pathname(), None); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn as_pathname(&self) -> Option<&Path> { - if let AddressKind::Pathname(path) = self.address() { Some(path) } else { None } - } - - fn address(&self) -> AddressKind<'_> { - let len = self.len as usize - sun_path_offset(&self.addr); - let path = unsafe { mem::transmute::<&[libc::c_char], &[u8]>(&self.addr.sun_path) }; - - // macOS seems to return a len of 16 and a zeroed sun_path for unnamed addresses - if len == 0 - || (cfg!(not(any(target_os = "linux", target_os = "android"))) - && self.addr.sun_path[0] == 0) - { - AddressKind::Unnamed - } else if self.addr.sun_path[0] == 0 { - AddressKind::Abstract(&path[1..len]) - } else { - AddressKind::Pathname(OsStr::from_bytes(&path[..len - 1]).as_ref()) - } - } -} - -#[stable(feature = "unix_socket", since = "1.10.0")] -impl fmt::Debug for SocketAddr { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.address() { - AddressKind::Unnamed => write!(fmt, "(unnamed)"), - AddressKind::Abstract(name) => write!(fmt, "{} (abstract)", AsciiEscaped(name)), - AddressKind::Pathname(path) => write!(fmt, "{:?} (pathname)", path), - } - } -} - -struct AsciiEscaped<'a>(&'a [u8]); - -impl<'a> fmt::Display for AsciiEscaped<'a> { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(fmt, "\"")?; - for byte in self.0.iter().cloned().flat_map(ascii::escape_default) { - write!(fmt, "{}", byte as char)?; - } - write!(fmt, "\"") - } -} - -/// A Unix stream socket. -/// -/// # Examples -/// -/// ```no_run -/// use std::os::unix::net::UnixStream; -/// use std::io::prelude::*; -/// -/// fn main() -> std::io::Result<()> { -/// let mut stream = UnixStream::connect("/path/to/my/socket")?; -/// stream.write_all(b"hello world")?; -/// let mut response = String::new(); -/// stream.read_to_string(&mut response)?; -/// println!("{}", response); -/// Ok(()) -/// } -/// ``` -#[stable(feature = "unix_socket", since = "1.10.0")] -pub struct UnixStream(Socket); - -#[stable(feature = "unix_socket", since = "1.10.0")] -impl fmt::Debug for UnixStream { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - let mut builder = fmt.debug_struct("UnixStream"); - builder.field("fd", self.0.as_inner()); - if let Ok(addr) = self.local_addr() { - builder.field("local", &addr); - } - if let Ok(addr) = self.peer_addr() { - builder.field("peer", &addr); - } - builder.finish() - } -} - -impl UnixStream { - /// Connects to the socket named by `path`. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixStream; - /// - /// let socket = match UnixStream::connect("/tmp/sock") { - /// Ok(sock) => sock, - /// Err(e) => { - /// println!("Couldn't connect: {:?}", e); - /// return - /// } - /// }; - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn connect>(path: P) -> io::Result { - fn inner(path: &Path) -> io::Result { - unsafe { - let inner = Socket::new_raw(libc::AF_UNIX, libc::SOCK_STREAM)?; - let (addr, len) = sockaddr_un(path)?; - - cvt(libc::connect(*inner.as_inner(), &addr as *const _ as *const _, len))?; - Ok(UnixStream(inner)) - } - } - inner(path.as_ref()) - } - - /// Creates an unnamed pair of connected sockets. - /// - /// Returns two `UnixStream`s which are connected to each other. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixStream; - /// - /// let (sock1, sock2) = match UnixStream::pair() { - /// Ok((sock1, sock2)) => (sock1, sock2), - /// Err(e) => { - /// println!("Couldn't create a pair of sockets: {:?}", e); - /// return - /// } - /// }; - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn pair() -> io::Result<(UnixStream, UnixStream)> { - let (i1, i2) = Socket::new_pair(libc::AF_UNIX, libc::SOCK_STREAM)?; - Ok((UnixStream(i1), UnixStream(i2))) - } - - /// Creates a new independently owned handle to the underlying socket. - /// - /// The returned `UnixStream` is a reference to the same stream that this - /// object references. Both handles will read and write the same stream of - /// data, and options set on one stream will be propagated to the other - /// stream. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixStream; - /// - /// fn main() -> std::io::Result<()> { - /// let socket = UnixStream::connect("/tmp/sock")?; - /// let sock_copy = socket.try_clone().expect("Couldn't clone socket"); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn try_clone(&self) -> io::Result { - self.0.duplicate().map(UnixStream) - } - - /// Returns the socket address of the local half of this connection. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixStream; - /// - /// fn main() -> std::io::Result<()> { - /// let socket = UnixStream::connect("/tmp/sock")?; - /// let addr = socket.local_addr().expect("Couldn't get local address"); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn local_addr(&self) -> io::Result { - SocketAddr::new(|addr, len| unsafe { libc::getsockname(*self.0.as_inner(), addr, len) }) - } - - /// Returns the socket address of the remote half of this connection. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixStream; - /// - /// fn main() -> std::io::Result<()> { - /// let socket = UnixStream::connect("/tmp/sock")?; - /// let addr = socket.peer_addr().expect("Couldn't get peer address"); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn peer_addr(&self) -> io::Result { - SocketAddr::new(|addr, len| unsafe { libc::getpeername(*self.0.as_inner(), addr, len) }) - } - - /// Sets the read timeout for the socket. - /// - /// If the provided value is [`None`], then [`read`] calls will block - /// indefinitely. An [`Err`] is returned if the zero [`Duration`] is passed to this - /// method. - /// - /// [`None`]: ../../../../std/option/enum.Option.html#variant.None - /// [`Err`]: ../../../../std/result/enum.Result.html#variant.Err - /// [`read`]: ../../../../std/io/trait.Read.html#tymethod.read - /// [`Duration`]: ../../../../std/time/struct.Duration.html - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixStream; - /// use std::time::Duration; - /// - /// fn main() -> std::io::Result<()> { - /// let socket = UnixStream::connect("/tmp/sock")?; - /// socket.set_read_timeout(Some(Duration::new(1, 0))).expect("Couldn't set read timeout"); - /// Ok(()) - /// } - /// ``` - /// - /// An [`Err`] is returned if the zero [`Duration`] is passed to this - /// method: - /// - /// ```no_run - /// use std::io; - /// use std::os::unix::net::UnixStream; - /// use std::time::Duration; - /// - /// fn main() -> std::io::Result<()> { - /// let socket = UnixStream::connect("/tmp/sock")?; - /// let result = socket.set_read_timeout(Some(Duration::new(0, 0))); - /// let err = result.unwrap_err(); - /// assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn set_read_timeout(&self, timeout: Option) -> io::Result<()> { - self.0.set_timeout(timeout, libc::SO_RCVTIMEO) - } - - /// Sets the write timeout for the socket. - /// - /// If the provided value is [`None`], then [`write`] calls will block - /// indefinitely. An [`Err`] is returned if the zero [`Duration`] is - /// passed to this method. - /// - /// [`None`]: ../../../../std/option/enum.Option.html#variant.None - /// [`Err`]: ../../../../std/result/enum.Result.html#variant.Err - /// [`write`]: ../../../../std/io/trait.Write.html#tymethod.write - /// [`Duration`]: ../../../../std/time/struct.Duration.html - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixStream; - /// use std::time::Duration; - /// - /// fn main() -> std::io::Result<()> { - /// let socket = UnixStream::connect("/tmp/sock")?; - /// socket.set_write_timeout(Some(Duration::new(1, 0))) - /// .expect("Couldn't set write timeout"); - /// Ok(()) - /// } - /// ``` - /// - /// An [`Err`] is returned if the zero [`Duration`] is passed to this - /// method: - /// - /// ```no_run - /// use std::io; - /// use std::net::UdpSocket; - /// use std::time::Duration; - /// - /// fn main() -> std::io::Result<()> { - /// let socket = UdpSocket::bind("127.0.0.1:34254")?; - /// let result = socket.set_write_timeout(Some(Duration::new(0, 0))); - /// let err = result.unwrap_err(); - /// assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn set_write_timeout(&self, timeout: Option) -> io::Result<()> { - self.0.set_timeout(timeout, libc::SO_SNDTIMEO) - } - - /// Returns the read timeout of this socket. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixStream; - /// use std::time::Duration; - /// - /// fn main() -> std::io::Result<()> { - /// let socket = UnixStream::connect("/tmp/sock")?; - /// socket.set_read_timeout(Some(Duration::new(1, 0))).expect("Couldn't set read timeout"); - /// assert_eq!(socket.read_timeout()?, Some(Duration::new(1, 0))); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn read_timeout(&self) -> io::Result> { - self.0.timeout(libc::SO_RCVTIMEO) - } - - /// Returns the write timeout of this socket. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixStream; - /// use std::time::Duration; - /// - /// fn main() -> std::io::Result<()> { - /// let socket = UnixStream::connect("/tmp/sock")?; - /// socket.set_write_timeout(Some(Duration::new(1, 0))) - /// .expect("Couldn't set write timeout"); - /// assert_eq!(socket.write_timeout()?, Some(Duration::new(1, 0))); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn write_timeout(&self) -> io::Result> { - self.0.timeout(libc::SO_SNDTIMEO) - } - - /// Moves the socket into or out of nonblocking mode. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixStream; - /// - /// fn main() -> std::io::Result<()> { - /// let socket = UnixStream::connect("/tmp/sock")?; - /// socket.set_nonblocking(true).expect("Couldn't set nonblocking"); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()> { - self.0.set_nonblocking(nonblocking) - } - - /// Returns the value of the `SO_ERROR` option. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixStream; - /// - /// fn main() -> std::io::Result<()> { - /// let socket = UnixStream::connect("/tmp/sock")?; - /// if let Ok(Some(err)) = socket.take_error() { - /// println!("Got error: {:?}", err); - /// } - /// Ok(()) - /// } - /// ``` - /// - /// # Platform specific - /// On Redox this always returns `None`. - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn take_error(&self) -> io::Result> { - self.0.take_error() - } - - /// Shuts down the read, write, or both halves of this connection. - /// - /// This function will cause all pending and future I/O calls on the - /// specified portions to immediately return with an appropriate value - /// (see the documentation of [`Shutdown`]). - /// - /// [`Shutdown`]: ../../../../std/net/enum.Shutdown.html - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixStream; - /// use std::net::Shutdown; - /// - /// fn main() -> std::io::Result<()> { - /// let socket = UnixStream::connect("/tmp/sock")?; - /// socket.shutdown(Shutdown::Both).expect("shutdown function failed"); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn shutdown(&self, how: Shutdown) -> io::Result<()> { - self.0.shutdown(how) - } -} - -#[stable(feature = "unix_socket", since = "1.10.0")] -impl io::Read for UnixStream { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - io::Read::read(&mut &*self, buf) - } - - fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { - io::Read::read_vectored(&mut &*self, bufs) - } - - #[inline] - unsafe fn initializer(&self) -> Initializer { - Initializer::nop() - } -} - -#[stable(feature = "unix_socket", since = "1.10.0")] -impl<'a> io::Read for &'a UnixStream { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - self.0.read(buf) - } - - fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { - self.0.read_vectored(bufs) - } - - #[inline] - unsafe fn initializer(&self) -> Initializer { - Initializer::nop() - } -} - -#[stable(feature = "unix_socket", since = "1.10.0")] -impl io::Write for UnixStream { - fn write(&mut self, buf: &[u8]) -> io::Result { - io::Write::write(&mut &*self, buf) - } - - fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { - io::Write::write_vectored(&mut &*self, bufs) - } - - fn flush(&mut self) -> io::Result<()> { - io::Write::flush(&mut &*self) - } -} - -#[stable(feature = "unix_socket", since = "1.10.0")] -impl<'a> io::Write for &'a UnixStream { - fn write(&mut self, buf: &[u8]) -> io::Result { - self.0.write(buf) - } - - fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { - self.0.write_vectored(bufs) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -#[stable(feature = "unix_socket", since = "1.10.0")] -impl AsRawFd for UnixStream { - fn as_raw_fd(&self) -> RawFd { - *self.0.as_inner() - } -} - -#[stable(feature = "unix_socket", since = "1.10.0")] -impl FromRawFd for UnixStream { - unsafe fn from_raw_fd(fd: RawFd) -> UnixStream { - UnixStream(Socket::from_inner(fd)) - } -} - -#[stable(feature = "unix_socket", since = "1.10.0")] -impl IntoRawFd for UnixStream { - fn into_raw_fd(self) -> RawFd { - self.0.into_inner() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl AsRawFd for net::TcpStream { - fn as_raw_fd(&self) -> RawFd { - *self.as_inner().socket().as_inner() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl AsRawFd for net::TcpListener { - fn as_raw_fd(&self) -> RawFd { - *self.as_inner().socket().as_inner() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl AsRawFd for net::UdpSocket { - fn as_raw_fd(&self) -> RawFd { - *self.as_inner().socket().as_inner() - } -} - -#[stable(feature = "from_raw_os", since = "1.1.0")] -impl FromRawFd for net::TcpStream { - unsafe fn from_raw_fd(fd: RawFd) -> net::TcpStream { - let socket = sys::net::Socket::from_inner(fd); - net::TcpStream::from_inner(sys_common::net::TcpStream::from_inner(socket)) - } -} - -#[stable(feature = "from_raw_os", since = "1.1.0")] -impl FromRawFd for net::TcpListener { - unsafe fn from_raw_fd(fd: RawFd) -> net::TcpListener { - let socket = sys::net::Socket::from_inner(fd); - net::TcpListener::from_inner(sys_common::net::TcpListener::from_inner(socket)) - } -} - -#[stable(feature = "from_raw_os", since = "1.1.0")] -impl FromRawFd for net::UdpSocket { - unsafe fn from_raw_fd(fd: RawFd) -> net::UdpSocket { - let socket = sys::net::Socket::from_inner(fd); - net::UdpSocket::from_inner(sys_common::net::UdpSocket::from_inner(socket)) - } -} - -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawFd for net::TcpStream { - fn into_raw_fd(self) -> RawFd { - self.into_inner().into_socket().into_inner() - } -} -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawFd for net::TcpListener { - fn into_raw_fd(self) -> RawFd { - self.into_inner().into_socket().into_inner() - } -} -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawFd for net::UdpSocket { - fn into_raw_fd(self) -> RawFd { - self.into_inner().into_socket().into_inner() - } -} - -/// A structure representing a Unix domain socket server. -/// -/// # Examples -/// -/// ```no_run -/// use std::thread; -/// use std::os::unix::net::{UnixStream, UnixListener}; -/// -/// fn handle_client(stream: UnixStream) { -/// // ... -/// } -/// -/// fn main() -> std::io::Result<()> { -/// let listener = UnixListener::bind("/path/to/the/socket")?; -/// -/// // accept connections and process them, spawning a new thread for each one -/// for stream in listener.incoming() { -/// match stream { -/// Ok(stream) => { -/// /* connection succeeded */ -/// thread::spawn(|| handle_client(stream)); -/// } -/// Err(err) => { -/// /* connection failed */ -/// break; -/// } -/// } -/// } -/// Ok(()) -/// } -/// ``` -#[stable(feature = "unix_socket", since = "1.10.0")] -pub struct UnixListener(Socket); - -#[stable(feature = "unix_socket", since = "1.10.0")] -impl fmt::Debug for UnixListener { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - let mut builder = fmt.debug_struct("UnixListener"); - builder.field("fd", self.0.as_inner()); - if let Ok(addr) = self.local_addr() { - builder.field("local", &addr); - } - builder.finish() - } -} - -impl UnixListener { - /// Creates a new `UnixListener` bound to the specified socket. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixListener; - /// - /// let listener = match UnixListener::bind("/path/to/the/socket") { - /// Ok(sock) => sock, - /// Err(e) => { - /// println!("Couldn't connect: {:?}", e); - /// return - /// } - /// }; - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn bind>(path: P) -> io::Result { - fn inner(path: &Path) -> io::Result { - unsafe { - let inner = Socket::new_raw(libc::AF_UNIX, libc::SOCK_STREAM)?; - let (addr, len) = sockaddr_un(path)?; - - cvt(libc::bind(*inner.as_inner(), &addr as *const _ as *const _, len as _))?; - cvt(libc::listen(*inner.as_inner(), 128))?; - - Ok(UnixListener(inner)) - } - } - inner(path.as_ref()) - } - - /// Accepts a new incoming connection to this listener. - /// - /// This function will block the calling thread until a new Unix connection - /// is established. When established, the corresponding [`UnixStream`] and - /// the remote peer's address will be returned. - /// - /// [`UnixStream`]: ../../../../std/os/unix/net/struct.UnixStream.html - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixListener; - /// - /// fn main() -> std::io::Result<()> { - /// let listener = UnixListener::bind("/path/to/the/socket")?; - /// - /// match listener.accept() { - /// Ok((socket, addr)) => println!("Got a client: {:?}", addr), - /// Err(e) => println!("accept function failed: {:?}", e), - /// } - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn accept(&self) -> io::Result<(UnixStream, SocketAddr)> { - let mut storage: libc::sockaddr_un = unsafe { mem::zeroed() }; - let mut len = mem::size_of_val(&storage) as libc::socklen_t; - let sock = self.0.accept(&mut storage as *mut _ as *mut _, &mut len)?; - let addr = SocketAddr::from_parts(storage, len)?; - Ok((UnixStream(sock), addr)) - } - - /// Creates a new independently owned handle to the underlying socket. - /// - /// The returned `UnixListener` is a reference to the same socket that this - /// object references. Both handles can be used to accept incoming - /// connections and options set on one listener will affect the other. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixListener; - /// - /// fn main() -> std::io::Result<()> { - /// let listener = UnixListener::bind("/path/to/the/socket")?; - /// let listener_copy = listener.try_clone().expect("try_clone failed"); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn try_clone(&self) -> io::Result { - self.0.duplicate().map(UnixListener) - } - - /// Returns the local socket address of this listener. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixListener; - /// - /// fn main() -> std::io::Result<()> { - /// let listener = UnixListener::bind("/path/to/the/socket")?; - /// let addr = listener.local_addr().expect("Couldn't get local address"); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn local_addr(&self) -> io::Result { - SocketAddr::new(|addr, len| unsafe { libc::getsockname(*self.0.as_inner(), addr, len) }) - } - - /// Moves the socket into or out of nonblocking mode. - /// - /// This will result in the `accept` operation becoming nonblocking, - /// i.e., immediately returning from their calls. If the IO operation is - /// successful, `Ok` is returned and no further action is required. If the - /// IO operation could not be completed and needs to be retried, an error - /// with kind [`io::ErrorKind::WouldBlock`] is returned. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixListener; - /// - /// fn main() -> std::io::Result<()> { - /// let listener = UnixListener::bind("/path/to/the/socket")?; - /// listener.set_nonblocking(true).expect("Couldn't set non blocking"); - /// Ok(()) - /// } - /// ``` - /// - /// [`io::ErrorKind::WouldBlock`]: ../../../io/enum.ErrorKind.html#variant.WouldBlock - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()> { - self.0.set_nonblocking(nonblocking) - } - - /// Returns the value of the `SO_ERROR` option. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixListener; - /// - /// fn main() -> std::io::Result<()> { - /// let listener = UnixListener::bind("/tmp/sock")?; - /// - /// if let Ok(Some(err)) = listener.take_error() { - /// println!("Got error: {:?}", err); - /// } - /// Ok(()) - /// } - /// ``` - /// - /// # Platform specific - /// On Redox this always returns `None`. - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn take_error(&self) -> io::Result> { - self.0.take_error() - } - - /// Returns an iterator over incoming connections. - /// - /// The iterator will never return [`None`] and will also not yield the - /// peer's [`SocketAddr`] structure. - /// - /// [`None`]: ../../../../std/option/enum.Option.html#variant.None - /// [`SocketAddr`]: struct.SocketAddr.html - /// - /// # Examples - /// - /// ```no_run - /// use std::thread; - /// use std::os::unix::net::{UnixStream, UnixListener}; - /// - /// fn handle_client(stream: UnixStream) { - /// // ... - /// } - /// - /// fn main() -> std::io::Result<()> { - /// let listener = UnixListener::bind("/path/to/the/socket")?; - /// - /// for stream in listener.incoming() { - /// match stream { - /// Ok(stream) => { - /// thread::spawn(|| handle_client(stream)); - /// } - /// Err(err) => { - /// break; - /// } - /// } - /// } - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn incoming(&self) -> Incoming<'_> { - Incoming { listener: self } - } -} - -#[stable(feature = "unix_socket", since = "1.10.0")] -impl AsRawFd for UnixListener { - fn as_raw_fd(&self) -> RawFd { - *self.0.as_inner() - } -} - -#[stable(feature = "unix_socket", since = "1.10.0")] -impl FromRawFd for UnixListener { - unsafe fn from_raw_fd(fd: RawFd) -> UnixListener { - UnixListener(Socket::from_inner(fd)) - } -} - -#[stable(feature = "unix_socket", since = "1.10.0")] -impl IntoRawFd for UnixListener { - fn into_raw_fd(self) -> RawFd { - self.0.into_inner() - } -} - -#[stable(feature = "unix_socket", since = "1.10.0")] -impl<'a> IntoIterator for &'a UnixListener { - type Item = io::Result; - type IntoIter = Incoming<'a>; - - fn into_iter(self) -> Incoming<'a> { - self.incoming() - } -} - -/// An iterator over incoming connections to a [`UnixListener`]. -/// -/// It will never return [`None`]. -/// -/// [`None`]: ../../../../std/option/enum.Option.html#variant.None -/// [`UnixListener`]: struct.UnixListener.html -/// -/// # Examples -/// -/// ```no_run -/// use std::thread; -/// use std::os::unix::net::{UnixStream, UnixListener}; -/// -/// fn handle_client(stream: UnixStream) { -/// // ... -/// } -/// -/// fn main() -> std::io::Result<()> { -/// let listener = UnixListener::bind("/path/to/the/socket")?; -/// -/// for stream in listener.incoming() { -/// match stream { -/// Ok(stream) => { -/// thread::spawn(|| handle_client(stream)); -/// } -/// Err(err) => { -/// break; -/// } -/// } -/// } -/// Ok(()) -/// } -/// ``` -#[derive(Debug)] -#[stable(feature = "unix_socket", since = "1.10.0")] -pub struct Incoming<'a> { - listener: &'a UnixListener, -} - -#[stable(feature = "unix_socket", since = "1.10.0")] -impl<'a> Iterator for Incoming<'a> { - type Item = io::Result; - - fn next(&mut self) -> Option> { - Some(self.listener.accept().map(|s| s.0)) - } - - fn size_hint(&self) -> (usize, Option) { - (usize::max_value(), None) - } -} - -/// A Unix datagram socket. -/// -/// # Examples -/// -/// ```no_run -/// use std::os::unix::net::UnixDatagram; -/// -/// fn main() -> std::io::Result<()> { -/// let socket = UnixDatagram::bind("/path/to/my/socket")?; -/// socket.send_to(b"hello world", "/path/to/other/socket")?; -/// let mut buf = [0; 100]; -/// let (count, address) = socket.recv_from(&mut buf)?; -/// println!("socket {:?} sent {:?}", address, &buf[..count]); -/// Ok(()) -/// } -/// ``` -#[stable(feature = "unix_socket", since = "1.10.0")] -pub struct UnixDatagram(Socket); - -#[stable(feature = "unix_socket", since = "1.10.0")] -impl fmt::Debug for UnixDatagram { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - let mut builder = fmt.debug_struct("UnixDatagram"); - builder.field("fd", self.0.as_inner()); - if let Ok(addr) = self.local_addr() { - builder.field("local", &addr); - } - if let Ok(addr) = self.peer_addr() { - builder.field("peer", &addr); - } - builder.finish() - } -} - -impl UnixDatagram { - /// Creates a Unix datagram socket bound to the given path. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixDatagram; - /// - /// let sock = match UnixDatagram::bind("/path/to/the/socket") { - /// Ok(sock) => sock, - /// Err(e) => { - /// println!("Couldn't bind: {:?}", e); - /// return - /// } - /// }; - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn bind>(path: P) -> io::Result { - fn inner(path: &Path) -> io::Result { - unsafe { - let socket = UnixDatagram::unbound()?; - let (addr, len) = sockaddr_un(path)?; - - cvt(libc::bind(*socket.0.as_inner(), &addr as *const _ as *const _, len as _))?; - - Ok(socket) - } - } - inner(path.as_ref()) - } - - /// Creates a Unix Datagram socket which is not bound to any address. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixDatagram; - /// - /// let sock = match UnixDatagram::unbound() { - /// Ok(sock) => sock, - /// Err(e) => { - /// println!("Couldn't unbound: {:?}", e); - /// return - /// } - /// }; - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn unbound() -> io::Result { - let inner = Socket::new_raw(libc::AF_UNIX, libc::SOCK_DGRAM)?; - Ok(UnixDatagram(inner)) - } - - /// Creates an unnamed pair of connected sockets. - /// - /// Returns two `UnixDatagrams`s which are connected to each other. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixDatagram; - /// - /// let (sock1, sock2) = match UnixDatagram::pair() { - /// Ok((sock1, sock2)) => (sock1, sock2), - /// Err(e) => { - /// println!("Couldn't unbound: {:?}", e); - /// return - /// } - /// }; - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn pair() -> io::Result<(UnixDatagram, UnixDatagram)> { - let (i1, i2) = Socket::new_pair(libc::AF_UNIX, libc::SOCK_DGRAM)?; - Ok((UnixDatagram(i1), UnixDatagram(i2))) - } - - /// Connects the socket to the specified address. - /// - /// The [`send`] method may be used to send data to the specified address. - /// [`recv`] and [`recv_from`] will only receive data from that address. - /// - /// [`send`]: #method.send - /// [`recv`]: #method.recv - /// [`recv_from`]: #method.recv_from - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixDatagram; - /// - /// fn main() -> std::io::Result<()> { - /// let sock = UnixDatagram::unbound()?; - /// match sock.connect("/path/to/the/socket") { - /// Ok(sock) => sock, - /// Err(e) => { - /// println!("Couldn't connect: {:?}", e); - /// return Err(e) - /// } - /// }; - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn connect>(&self, path: P) -> io::Result<()> { - fn inner(d: &UnixDatagram, path: &Path) -> io::Result<()> { - unsafe { - let (addr, len) = sockaddr_un(path)?; - - cvt(libc::connect(*d.0.as_inner(), &addr as *const _ as *const _, len))?; - - Ok(()) - } - } - inner(self, path.as_ref()) - } - - /// Creates a new independently owned handle to the underlying socket. - /// - /// The returned `UnixDatagram` is a reference to the same socket that this - /// object references. Both handles can be used to accept incoming - /// connections and options set on one side will affect the other. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixDatagram; - /// - /// fn main() -> std::io::Result<()> { - /// let sock = UnixDatagram::bind("/path/to/the/socket")?; - /// let sock_copy = sock.try_clone().expect("try_clone failed"); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn try_clone(&self) -> io::Result { - self.0.duplicate().map(UnixDatagram) - } - - /// Returns the address of this socket. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixDatagram; - /// - /// fn main() -> std::io::Result<()> { - /// let sock = UnixDatagram::bind("/path/to/the/socket")?; - /// let addr = sock.local_addr().expect("Couldn't get local address"); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn local_addr(&self) -> io::Result { - SocketAddr::new(|addr, len| unsafe { libc::getsockname(*self.0.as_inner(), addr, len) }) - } - - /// Returns the address of this socket's peer. - /// - /// The [`connect`] method will connect the socket to a peer. - /// - /// [`connect`]: #method.connect - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixDatagram; - /// - /// fn main() -> std::io::Result<()> { - /// let sock = UnixDatagram::unbound()?; - /// sock.connect("/path/to/the/socket")?; - /// - /// let addr = sock.peer_addr().expect("Couldn't get peer address"); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn peer_addr(&self) -> io::Result { - SocketAddr::new(|addr, len| unsafe { libc::getpeername(*self.0.as_inner(), addr, len) }) - } - - /// Receives data from the socket. - /// - /// On success, returns the number of bytes read and the address from - /// whence the data came. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixDatagram; - /// - /// fn main() -> std::io::Result<()> { - /// let sock = UnixDatagram::unbound()?; - /// let mut buf = vec![0; 10]; - /// let (size, sender) = sock.recv_from(buf.as_mut_slice())?; - /// println!("received {} bytes from {:?}", size, sender); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn recv_from(&self, buf: &mut [u8]) -> io::Result<(usize, SocketAddr)> { - let mut count = 0; - let addr = SocketAddr::new(|addr, len| unsafe { - count = libc::recvfrom( - *self.0.as_inner(), - buf.as_mut_ptr() as *mut _, - buf.len(), - 0, - addr, - len, - ); - if count > 0 { - 1 - } else if count == 0 { - 0 - } else { - -1 - } - })?; - - Ok((count as usize, addr)) - } - - /// Receives data from the socket. - /// - /// On success, returns the number of bytes read. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixDatagram; - /// - /// fn main() -> std::io::Result<()> { - /// let sock = UnixDatagram::bind("/path/to/the/socket")?; - /// let mut buf = vec![0; 10]; - /// sock.recv(buf.as_mut_slice()).expect("recv function failed"); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn recv(&self, buf: &mut [u8]) -> io::Result { - self.0.read(buf) - } - - /// Sends data on the socket to the specified address. - /// - /// On success, returns the number of bytes written. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixDatagram; - /// - /// fn main() -> std::io::Result<()> { - /// let sock = UnixDatagram::unbound()?; - /// sock.send_to(b"omelette au fromage", "/some/sock").expect("send_to function failed"); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn send_to>(&self, buf: &[u8], path: P) -> io::Result { - fn inner(d: &UnixDatagram, buf: &[u8], path: &Path) -> io::Result { - unsafe { - let (addr, len) = sockaddr_un(path)?; - - let count = cvt(libc::sendto( - *d.0.as_inner(), - buf.as_ptr() as *const _, - buf.len(), - MSG_NOSIGNAL, - &addr as *const _ as *const _, - len, - ))?; - Ok(count as usize) - } - } - inner(self, buf, path.as_ref()) - } - - /// Sends data on the socket to the socket's peer. - /// - /// The peer address may be set by the `connect` method, and this method - /// will return an error if the socket has not already been connected. - /// - /// On success, returns the number of bytes written. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixDatagram; - /// - /// fn main() -> std::io::Result<()> { - /// let sock = UnixDatagram::unbound()?; - /// sock.connect("/some/sock").expect("Couldn't connect"); - /// sock.send(b"omelette au fromage").expect("send_to function failed"); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn send(&self, buf: &[u8]) -> io::Result { - self.0.write(buf) - } - - /// Sets the read timeout for the socket. - /// - /// If the provided value is [`None`], then [`recv`] and [`recv_from`] calls will - /// block indefinitely. An [`Err`] is returned if the zero [`Duration`] - /// is passed to this method. - /// - /// [`None`]: ../../../../std/option/enum.Option.html#variant.None - /// [`Err`]: ../../../../std/result/enum.Result.html#variant.Err - /// [`recv`]: #method.recv - /// [`recv_from`]: #method.recv_from - /// [`Duration`]: ../../../../std/time/struct.Duration.html - /// - /// # Examples - /// - /// ``` - /// use std::os::unix::net::UnixDatagram; - /// use std::time::Duration; - /// - /// fn main() -> std::io::Result<()> { - /// let sock = UnixDatagram::unbound()?; - /// sock.set_read_timeout(Some(Duration::new(1, 0))) - /// .expect("set_read_timeout function failed"); - /// Ok(()) - /// } - /// ``` - /// - /// An [`Err`] is returned if the zero [`Duration`] is passed to this - /// method: - /// - /// ```no_run - /// use std::io; - /// use std::os::unix::net::UnixDatagram; - /// use std::time::Duration; - /// - /// fn main() -> std::io::Result<()> { - /// let socket = UnixDatagram::unbound()?; - /// let result = socket.set_read_timeout(Some(Duration::new(0, 0))); - /// let err = result.unwrap_err(); - /// assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn set_read_timeout(&self, timeout: Option) -> io::Result<()> { - self.0.set_timeout(timeout, libc::SO_RCVTIMEO) - } - - /// Sets the write timeout for the socket. - /// - /// If the provided value is [`None`], then [`send`] and [`send_to`] calls will - /// block indefinitely. An [`Err`] is returned if the zero [`Duration`] is passed to this - /// method. - /// - /// [`None`]: ../../../../std/option/enum.Option.html#variant.None - /// [`send`]: #method.send - /// [`send_to`]: #method.send_to - /// [`Duration`]: ../../../../std/time/struct.Duration.html - /// - /// # Examples - /// - /// ``` - /// use std::os::unix::net::UnixDatagram; - /// use std::time::Duration; - /// - /// fn main() -> std::io::Result<()> { - /// let sock = UnixDatagram::unbound()?; - /// sock.set_write_timeout(Some(Duration::new(1, 0))) - /// .expect("set_write_timeout function failed"); - /// Ok(()) - /// } - /// ``` - /// - /// An [`Err`] is returned if the zero [`Duration`] is passed to this - /// method: - /// - /// ```no_run - /// use std::io; - /// use std::os::unix::net::UnixDatagram; - /// use std::time::Duration; - /// - /// fn main() -> std::io::Result<()> { - /// let socket = UnixDatagram::unbound()?; - /// let result = socket.set_write_timeout(Some(Duration::new(0, 0))); - /// let err = result.unwrap_err(); - /// assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn set_write_timeout(&self, timeout: Option) -> io::Result<()> { - self.0.set_timeout(timeout, libc::SO_SNDTIMEO) - } - - /// Returns the read timeout of this socket. - /// - /// # Examples - /// - /// ``` - /// use std::os::unix::net::UnixDatagram; - /// use std::time::Duration; - /// - /// fn main() -> std::io::Result<()> { - /// let sock = UnixDatagram::unbound()?; - /// sock.set_read_timeout(Some(Duration::new(1, 0))) - /// .expect("set_read_timeout function failed"); - /// assert_eq!(sock.read_timeout()?, Some(Duration::new(1, 0))); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn read_timeout(&self) -> io::Result> { - self.0.timeout(libc::SO_RCVTIMEO) - } - - /// Returns the write timeout of this socket. - /// - /// # Examples - /// - /// ``` - /// use std::os::unix::net::UnixDatagram; - /// use std::time::Duration; - /// - /// fn main() -> std::io::Result<()> { - /// let sock = UnixDatagram::unbound()?; - /// sock.set_write_timeout(Some(Duration::new(1, 0))) - /// .expect("set_write_timeout function failed"); - /// assert_eq!(sock.write_timeout()?, Some(Duration::new(1, 0))); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn write_timeout(&self) -> io::Result> { - self.0.timeout(libc::SO_SNDTIMEO) - } - - /// Moves the socket into or out of nonblocking mode. - /// - /// # Examples - /// - /// ``` - /// use std::os::unix::net::UnixDatagram; - /// - /// fn main() -> std::io::Result<()> { - /// let sock = UnixDatagram::unbound()?; - /// sock.set_nonblocking(true).expect("set_nonblocking function failed"); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()> { - self.0.set_nonblocking(nonblocking) - } - - /// Returns the value of the `SO_ERROR` option. - /// - /// # Examples - /// - /// ```no_run - /// use std::os::unix::net::UnixDatagram; - /// - /// fn main() -> std::io::Result<()> { - /// let sock = UnixDatagram::unbound()?; - /// if let Ok(Some(err)) = sock.take_error() { - /// println!("Got error: {:?}", err); - /// } - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn take_error(&self) -> io::Result> { - self.0.take_error() - } - - /// Shut down the read, write, or both halves of this connection. - /// - /// This function will cause all pending and future I/O calls on the - /// specified portions to immediately return with an appropriate value - /// (see the documentation of [`Shutdown`]). - /// - /// [`Shutdown`]: ../../../../std/net/enum.Shutdown.html - /// - /// ```no_run - /// use std::os::unix::net::UnixDatagram; - /// use std::net::Shutdown; - /// - /// fn main() -> std::io::Result<()> { - /// let sock = UnixDatagram::unbound()?; - /// sock.shutdown(Shutdown::Both).expect("shutdown function failed"); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "unix_socket", since = "1.10.0")] - pub fn shutdown(&self, how: Shutdown) -> io::Result<()> { - self.0.shutdown(how) - } -} - -#[stable(feature = "unix_socket", since = "1.10.0")] -impl AsRawFd for UnixDatagram { - fn as_raw_fd(&self) -> RawFd { - *self.0.as_inner() - } -} - -#[stable(feature = "unix_socket", since = "1.10.0")] -impl FromRawFd for UnixDatagram { - unsafe fn from_raw_fd(fd: RawFd) -> UnixDatagram { - UnixDatagram(Socket::from_inner(fd)) - } -} - -#[stable(feature = "unix_socket", since = "1.10.0")] -impl IntoRawFd for UnixDatagram { - fn into_raw_fd(self) -> RawFd { - self.0.into_inner() - } -} - -#[cfg(all(test, not(target_os = "emscripten")))] -mod test { - use crate::io::prelude::*; - use crate::io::{self, ErrorKind}; - use crate::sys_common::io::test::tmpdir; - use crate::thread; - use crate::time::Duration; - - use super::*; - - macro_rules! or_panic { - ($e:expr) => { - match $e { - Ok(e) => e, - Err(e) => panic!("{}", e), - } - }; - } - - #[test] - fn basic() { - let dir = tmpdir(); - let socket_path = dir.path().join("sock"); - let msg1 = b"hello"; - let msg2 = b"world!"; - - let listener = or_panic!(UnixListener::bind(&socket_path)); - let thread = thread::spawn(move || { - let mut stream = or_panic!(listener.accept()).0; - let mut buf = [0; 5]; - or_panic!(stream.read(&mut buf)); - assert_eq!(&msg1[..], &buf[..]); - or_panic!(stream.write_all(msg2)); - }); - - let mut stream = or_panic!(UnixStream::connect(&socket_path)); - assert_eq!(Some(&*socket_path), stream.peer_addr().unwrap().as_pathname()); - or_panic!(stream.write_all(msg1)); - let mut buf = vec![]; - or_panic!(stream.read_to_end(&mut buf)); - assert_eq!(&msg2[..], &buf[..]); - drop(stream); - - thread.join().unwrap(); - } - - #[test] - fn vectored() { - let (mut s1, mut s2) = or_panic!(UnixStream::pair()); - - let len = or_panic!(s1.write_vectored(&[ - IoSlice::new(b"hello"), - IoSlice::new(b" "), - IoSlice::new(b"world!") - ],)); - assert_eq!(len, 12); - - let mut buf1 = [0; 6]; - let mut buf2 = [0; 7]; - let len = or_panic!( - s2.read_vectored(&mut [IoSliceMut::new(&mut buf1), IoSliceMut::new(&mut buf2)],) - ); - assert_eq!(len, 12); - assert_eq!(&buf1, b"hello "); - assert_eq!(&buf2, b"world!\0"); - } - - #[test] - fn pair() { - let msg1 = b"hello"; - let msg2 = b"world!"; - - let (mut s1, mut s2) = or_panic!(UnixStream::pair()); - let thread = thread::spawn(move || { - // s1 must be moved in or the test will hang! - let mut buf = [0; 5]; - or_panic!(s1.read(&mut buf)); - assert_eq!(&msg1[..], &buf[..]); - or_panic!(s1.write_all(msg2)); - }); - - or_panic!(s2.write_all(msg1)); - let mut buf = vec![]; - or_panic!(s2.read_to_end(&mut buf)); - assert_eq!(&msg2[..], &buf[..]); - drop(s2); - - thread.join().unwrap(); - } - - #[test] - fn try_clone() { - let dir = tmpdir(); - let socket_path = dir.path().join("sock"); - let msg1 = b"hello"; - let msg2 = b"world"; - - let listener = or_panic!(UnixListener::bind(&socket_path)); - let thread = thread::spawn(move || { - let mut stream = or_panic!(listener.accept()).0; - or_panic!(stream.write_all(msg1)); - or_panic!(stream.write_all(msg2)); - }); - - let mut stream = or_panic!(UnixStream::connect(&socket_path)); - let mut stream2 = or_panic!(stream.try_clone()); - - let mut buf = [0; 5]; - or_panic!(stream.read(&mut buf)); - assert_eq!(&msg1[..], &buf[..]); - or_panic!(stream2.read(&mut buf)); - assert_eq!(&msg2[..], &buf[..]); - - thread.join().unwrap(); - } - - #[test] - fn iter() { - let dir = tmpdir(); - let socket_path = dir.path().join("sock"); - - let listener = or_panic!(UnixListener::bind(&socket_path)); - let thread = thread::spawn(move || { - for stream in listener.incoming().take(2) { - let mut stream = or_panic!(stream); - let mut buf = [0]; - or_panic!(stream.read(&mut buf)); - } - }); - - for _ in 0..2 { - let mut stream = or_panic!(UnixStream::connect(&socket_path)); - or_panic!(stream.write_all(&[0])); - } - - thread.join().unwrap(); - } - - #[test] - fn long_path() { - let dir = tmpdir(); - let socket_path = dir.path().join( - "asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfa\ - sasdfasdfasdasdfasdfasdfadfasdfasdfasdfasdfasdf", - ); - match UnixStream::connect(&socket_path) { - Err(ref e) if e.kind() == io::ErrorKind::InvalidInput => {} - Err(e) => panic!("unexpected error {}", e), - Ok(_) => panic!("unexpected success"), - } - - match UnixListener::bind(&socket_path) { - Err(ref e) if e.kind() == io::ErrorKind::InvalidInput => {} - Err(e) => panic!("unexpected error {}", e), - Ok(_) => panic!("unexpected success"), - } - - match UnixDatagram::bind(&socket_path) { - Err(ref e) if e.kind() == io::ErrorKind::InvalidInput => {} - Err(e) => panic!("unexpected error {}", e), - Ok(_) => panic!("unexpected success"), - } - } - - #[test] - fn timeouts() { - let dir = tmpdir(); - let socket_path = dir.path().join("sock"); - - let _listener = or_panic!(UnixListener::bind(&socket_path)); - - let stream = or_panic!(UnixStream::connect(&socket_path)); - let dur = Duration::new(15410, 0); - - assert_eq!(None, or_panic!(stream.read_timeout())); - - or_panic!(stream.set_read_timeout(Some(dur))); - assert_eq!(Some(dur), or_panic!(stream.read_timeout())); - - assert_eq!(None, or_panic!(stream.write_timeout())); - - or_panic!(stream.set_write_timeout(Some(dur))); - assert_eq!(Some(dur), or_panic!(stream.write_timeout())); - - or_panic!(stream.set_read_timeout(None)); - assert_eq!(None, or_panic!(stream.read_timeout())); - - or_panic!(stream.set_write_timeout(None)); - assert_eq!(None, or_panic!(stream.write_timeout())); - } - - #[test] - fn test_read_timeout() { - let dir = tmpdir(); - let socket_path = dir.path().join("sock"); - - let _listener = or_panic!(UnixListener::bind(&socket_path)); - - let mut stream = or_panic!(UnixStream::connect(&socket_path)); - or_panic!(stream.set_read_timeout(Some(Duration::from_millis(1000)))); - - let mut buf = [0; 10]; - let kind = stream.read_exact(&mut buf).err().expect("expected error").kind(); - assert!( - kind == ErrorKind::WouldBlock || kind == ErrorKind::TimedOut, - "unexpected_error: {:?}", - kind - ); - } - - #[test] - fn test_read_with_timeout() { - let dir = tmpdir(); - let socket_path = dir.path().join("sock"); - - let listener = or_panic!(UnixListener::bind(&socket_path)); - - let mut stream = or_panic!(UnixStream::connect(&socket_path)); - or_panic!(stream.set_read_timeout(Some(Duration::from_millis(1000)))); - - let mut other_end = or_panic!(listener.accept()).0; - or_panic!(other_end.write_all(b"hello world")); - - let mut buf = [0; 11]; - or_panic!(stream.read(&mut buf)); - assert_eq!(b"hello world", &buf[..]); - - let kind = stream.read_exact(&mut buf).err().expect("expected error").kind(); - assert!( - kind == ErrorKind::WouldBlock || kind == ErrorKind::TimedOut, - "unexpected_error: {:?}", - kind - ); - } - - // Ensure the `set_read_timeout` and `set_write_timeout` calls return errors - // when passed zero Durations - #[test] - fn test_unix_stream_timeout_zero_duration() { - let dir = tmpdir(); - let socket_path = dir.path().join("sock"); - - let listener = or_panic!(UnixListener::bind(&socket_path)); - let stream = or_panic!(UnixStream::connect(&socket_path)); - - let result = stream.set_write_timeout(Some(Duration::new(0, 0))); - let err = result.unwrap_err(); - assert_eq!(err.kind(), ErrorKind::InvalidInput); - - let result = stream.set_read_timeout(Some(Duration::new(0, 0))); - let err = result.unwrap_err(); - assert_eq!(err.kind(), ErrorKind::InvalidInput); - - drop(listener); - } - - #[test] - fn test_unix_datagram() { - let dir = tmpdir(); - let path1 = dir.path().join("sock1"); - let path2 = dir.path().join("sock2"); - - let sock1 = or_panic!(UnixDatagram::bind(&path1)); - let sock2 = or_panic!(UnixDatagram::bind(&path2)); - - let msg = b"hello world"; - or_panic!(sock1.send_to(msg, &path2)); - let mut buf = [0; 11]; - or_panic!(sock2.recv_from(&mut buf)); - assert_eq!(msg, &buf[..]); - } - - #[test] - fn test_unnamed_unix_datagram() { - let dir = tmpdir(); - let path1 = dir.path().join("sock1"); - - let sock1 = or_panic!(UnixDatagram::bind(&path1)); - let sock2 = or_panic!(UnixDatagram::unbound()); - - let msg = b"hello world"; - or_panic!(sock2.send_to(msg, &path1)); - let mut buf = [0; 11]; - let (usize, addr) = or_panic!(sock1.recv_from(&mut buf)); - assert_eq!(usize, 11); - assert!(addr.is_unnamed()); - assert_eq!(msg, &buf[..]); - } - - #[test] - fn test_connect_unix_datagram() { - let dir = tmpdir(); - let path1 = dir.path().join("sock1"); - let path2 = dir.path().join("sock2"); - - let bsock1 = or_panic!(UnixDatagram::bind(&path1)); - let bsock2 = or_panic!(UnixDatagram::bind(&path2)); - let sock = or_panic!(UnixDatagram::unbound()); - or_panic!(sock.connect(&path1)); - - // Check send() - let msg = b"hello there"; - or_panic!(sock.send(msg)); - let mut buf = [0; 11]; - let (usize, addr) = or_panic!(bsock1.recv_from(&mut buf)); - assert_eq!(usize, 11); - assert!(addr.is_unnamed()); - assert_eq!(msg, &buf[..]); - - // Changing default socket works too - or_panic!(sock.connect(&path2)); - or_panic!(sock.send(msg)); - or_panic!(bsock2.recv_from(&mut buf)); - } - - #[test] - fn test_unix_datagram_recv() { - let dir = tmpdir(); - let path1 = dir.path().join("sock1"); - - let sock1 = or_panic!(UnixDatagram::bind(&path1)); - let sock2 = or_panic!(UnixDatagram::unbound()); - or_panic!(sock2.connect(&path1)); - - let msg = b"hello world"; - or_panic!(sock2.send(msg)); - let mut buf = [0; 11]; - let size = or_panic!(sock1.recv(&mut buf)); - assert_eq!(size, 11); - assert_eq!(msg, &buf[..]); - } - - #[test] - fn datagram_pair() { - let msg1 = b"hello"; - let msg2 = b"world!"; - - let (s1, s2) = or_panic!(UnixDatagram::pair()); - let thread = thread::spawn(move || { - // s1 must be moved in or the test will hang! - let mut buf = [0; 5]; - or_panic!(s1.recv(&mut buf)); - assert_eq!(&msg1[..], &buf[..]); - or_panic!(s1.send(msg2)); - }); - - or_panic!(s2.send(msg1)); - let mut buf = [0; 6]; - or_panic!(s2.recv(&mut buf)); - assert_eq!(&msg2[..], &buf[..]); - drop(s2); - - thread.join().unwrap(); - } - - // Ensure the `set_read_timeout` and `set_write_timeout` calls return errors - // when passed zero Durations - #[test] - fn test_unix_datagram_timeout_zero_duration() { - let dir = tmpdir(); - let path = dir.path().join("sock"); - - let datagram = or_panic!(UnixDatagram::bind(&path)); - - let result = datagram.set_write_timeout(Some(Duration::new(0, 0))); - let err = result.unwrap_err(); - assert_eq!(err.kind(), ErrorKind::InvalidInput); - - let result = datagram.set_read_timeout(Some(Duration::new(0, 0))); - let err = result.unwrap_err(); - assert_eq!(err.kind(), ErrorKind::InvalidInput); - } - - #[test] - fn abstract_namespace_not_allowed() { - assert!(UnixStream::connect("\0asdf").is_err()); - } -} diff --git a/crux-mir/lib/std/src/sys/unix/ext/process.rs b/crux-mir/lib/std/src/sys/unix/ext/process.rs deleted file mode 100644 index fa8670b4a..000000000 --- a/crux-mir/lib/std/src/sys/unix/ext/process.rs +++ /dev/null @@ -1,234 +0,0 @@ -//! Unix-specific extensions to primitives in the `std::process` module. - -#![stable(feature = "rust1", since = "1.0.0")] - -use crate::ffi::OsStr; -use crate::io; -use crate::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd}; -use crate::process; -use crate::sys; -use crate::sys_common::{AsInner, AsInnerMut, FromInner, IntoInner}; - -/// Unix-specific extensions to the [`process::Command`] builder. -/// -/// [`process::Command`]: ../../../../std/process/struct.Command.html -#[stable(feature = "rust1", since = "1.0.0")] -pub trait CommandExt { - /// Sets the child process's user ID. This translates to a - /// `setuid` call in the child process. Failure in the `setuid` - /// call will cause the spawn to fail. - #[stable(feature = "rust1", since = "1.0.0")] - fn uid(&mut self, id: u32) -> &mut process::Command; - - /// Similar to `uid`, but sets the group ID of the child process. This has - /// the same semantics as the `uid` field. - #[stable(feature = "rust1", since = "1.0.0")] - fn gid(&mut self, id: u32) -> &mut process::Command; - - /// Schedules a closure to be run just before the `exec` function is - /// invoked. - /// - /// The closure is allowed to return an I/O error whose OS error code will - /// be communicated back to the parent and returned as an error from when - /// the spawn was requested. - /// - /// Multiple closures can be registered and they will be called in order of - /// their registration. If a closure returns `Err` then no further closures - /// will be called and the spawn operation will immediately return with a - /// failure. - /// - /// # Notes and Safety - /// - /// This closure will be run in the context of the child process after a - /// `fork`. This primarily means that any modifications made to memory on - /// behalf of this closure will **not** be visible to the parent process. - /// This is often a very constrained environment where normal operations - /// like `malloc` or acquiring a mutex are not guaranteed to work (due to - /// other threads perhaps still running when the `fork` was run). - /// - /// This also means that all resources such as file descriptors and - /// memory-mapped regions got duplicated. It is your responsibility to make - /// sure that the closure does not violate library invariants by making - /// invalid use of these duplicates. - /// - /// When this closure is run, aspects such as the stdio file descriptors and - /// working directory have successfully been changed, so output to these - /// locations may not appear where intended. - #[stable(feature = "process_pre_exec", since = "1.34.0")] - unsafe fn pre_exec(&mut self, f: F) -> &mut process::Command - where - F: FnMut() -> io::Result<()> + Send + Sync + 'static; - - /// Schedules a closure to be run just before the `exec` function is - /// invoked. - /// - /// This method is stable and usable, but it should be unsafe. To fix - /// that, it got deprecated in favor of the unsafe [`pre_exec`]. - /// - /// [`pre_exec`]: #tymethod.pre_exec - #[stable(feature = "process_exec", since = "1.15.0")] - #[rustc_deprecated(since = "1.37.0", reason = "should be unsafe, use `pre_exec` instead")] - fn before_exec(&mut self, f: F) -> &mut process::Command - where - F: FnMut() -> io::Result<()> + Send + Sync + 'static, - { - unsafe { self.pre_exec(f) } - } - - /// Performs all the required setup by this `Command`, followed by calling - /// the `execvp` syscall. - /// - /// On success this function will not return, and otherwise it will return - /// an error indicating why the exec (or another part of the setup of the - /// `Command`) failed. - /// - /// `exec` not returning has the same implications as calling - /// [`process::exit`] – no destructors on the current stack or any other - /// thread’s stack will be run. Therefore, it is recommended to only call - /// `exec` at a point where it is fine to not run any destructors. Note, - /// that the `execvp` syscall independently guarantees that all memory is - /// freed and all file descriptors with the `CLOEXEC` option (set by default - /// on all file descriptors opened by the standard library) are closed. - /// - /// This function, unlike `spawn`, will **not** `fork` the process to create - /// a new child. Like spawn, however, the default behavior for the stdio - /// descriptors will be to inherited from the current process. - /// - /// [`process::exit`]: ../../../process/fn.exit.html - /// - /// # Notes - /// - /// The process may be in a "broken state" if this function returns in - /// error. For example the working directory, environment variables, signal - /// handling settings, various user/group information, or aspects of stdio - /// file descriptors may have changed. If a "transactional spawn" is - /// required to gracefully handle errors it is recommended to use the - /// cross-platform `spawn` instead. - #[stable(feature = "process_exec2", since = "1.9.0")] - fn exec(&mut self) -> io::Error; - - /// Set executable argument - /// - /// Set the first process argument, `argv[0]`, to something other than the - /// default executable path. - #[unstable(feature = "process_set_argv0", issue = "66510")] - fn arg0(&mut self, arg: S) -> &mut process::Command - where - S: AsRef; -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl CommandExt for process::Command { - fn uid(&mut self, id: u32) -> &mut process::Command { - self.as_inner_mut().uid(id); - self - } - - fn gid(&mut self, id: u32) -> &mut process::Command { - self.as_inner_mut().gid(id); - self - } - - unsafe fn pre_exec(&mut self, f: F) -> &mut process::Command - where - F: FnMut() -> io::Result<()> + Send + Sync + 'static, - { - self.as_inner_mut().pre_exec(Box::new(f)); - self - } - - fn exec(&mut self) -> io::Error { - self.as_inner_mut().exec(sys::process::Stdio::Inherit) - } - - fn arg0(&mut self, arg: S) -> &mut process::Command - where - S: AsRef, - { - self.as_inner_mut().set_arg_0(arg.as_ref()); - self - } -} - -/// Unix-specific extensions to [`process::ExitStatus`]. -/// -/// [`process::ExitStatus`]: ../../../../std/process/struct.ExitStatus.html -#[stable(feature = "rust1", since = "1.0.0")] -pub trait ExitStatusExt { - /// Creates a new `ExitStatus` from the raw underlying `i32` return value of - /// a process. - #[stable(feature = "exit_status_from", since = "1.12.0")] - fn from_raw(raw: i32) -> Self; - - /// If the process was terminated by a signal, returns that signal. - #[stable(feature = "rust1", since = "1.0.0")] - fn signal(&self) -> Option; -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ExitStatusExt for process::ExitStatus { - fn from_raw(raw: i32) -> Self { - process::ExitStatus::from_inner(From::from(raw)) - } - - fn signal(&self) -> Option { - self.as_inner().signal() - } -} - -#[stable(feature = "process_extensions", since = "1.2.0")] -impl FromRawFd for process::Stdio { - unsafe fn from_raw_fd(fd: RawFd) -> process::Stdio { - let fd = sys::fd::FileDesc::new(fd); - let io = sys::process::Stdio::Fd(fd); - process::Stdio::from_inner(io) - } -} - -#[stable(feature = "process_extensions", since = "1.2.0")] -impl AsRawFd for process::ChildStdin { - fn as_raw_fd(&self) -> RawFd { - self.as_inner().fd().raw() - } -} - -#[stable(feature = "process_extensions", since = "1.2.0")] -impl AsRawFd for process::ChildStdout { - fn as_raw_fd(&self) -> RawFd { - self.as_inner().fd().raw() - } -} - -#[stable(feature = "process_extensions", since = "1.2.0")] -impl AsRawFd for process::ChildStderr { - fn as_raw_fd(&self) -> RawFd { - self.as_inner().fd().raw() - } -} - -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawFd for process::ChildStdin { - fn into_raw_fd(self) -> RawFd { - self.into_inner().into_fd().into_raw() - } -} - -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawFd for process::ChildStdout { - fn into_raw_fd(self) -> RawFd { - self.into_inner().into_fd().into_raw() - } -} - -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawFd for process::ChildStderr { - fn into_raw_fd(self) -> RawFd { - self.into_inner().into_fd().into_raw() - } -} - -/// Returns the OS-assigned process identifier associated with this process's parent. -#[stable(feature = "unix_ppid", since = "1.27.0")] -pub fn parent_id() -> u32 { - crate::sys::os::getppid() -} diff --git a/crux-mir/lib/std/src/sys/unix/ext/raw.rs b/crux-mir/lib/std/src/sys/unix/ext/raw.rs deleted file mode 100644 index d81368a18..000000000 --- a/crux-mir/lib/std/src/sys/unix/ext/raw.rs +++ /dev/null @@ -1,28 +0,0 @@ -//! Unix-specific primitives available on all unix platforms - -#![stable(feature = "raw_ext", since = "1.1.0")] -#![rustc_deprecated( - since = "1.8.0", - reason = "these type aliases are no longer supported by \ - the standard library, the `libc` crate on \ - crates.io should be used instead for the correct \ - definitions" -)] -#![allow(deprecated)] - -#[stable(feature = "raw_ext", since = "1.1.0")] -pub type uid_t = u32; -#[stable(feature = "raw_ext", since = "1.1.0")] -pub type gid_t = u32; -#[stable(feature = "raw_ext", since = "1.1.0")] -pub type pid_t = i32; - -#[doc(inline)] -#[stable(feature = "pthread_t", since = "1.8.0")] -pub use crate::sys::platform::raw::pthread_t; -#[doc(inline)] -#[stable(feature = "raw_ext", since = "1.1.0")] -pub use crate::sys::platform::raw::{blkcnt_t, time_t}; -#[doc(inline)] -#[stable(feature = "raw_ext", since = "1.1.0")] -pub use crate::sys::platform::raw::{blksize_t, dev_t, ino_t, mode_t, nlink_t, off_t}; diff --git a/crux-mir/lib/std/src/sys/unix/fd.rs b/crux-mir/lib/std/src/sys/unix/fd.rs index 8a9983691..dbaa3c33e 100644 --- a/crux-mir/lib/std/src/sys/unix/fd.rs +++ b/crux-mir/lib/std/src/sys/unix/fd.rs @@ -1,197 +1,259 @@ #![unstable(reason = "not public", issue = "none", feature = "fd")] +#[cfg(test)] +mod tests; + use crate::cmp; -use crate::io::{self, Initializer, IoSlice, IoSliceMut, Read}; -use crate::mem; -use crate::sync::atomic::{AtomicBool, Ordering}; +use crate::io::{self, BorrowedCursor, IoSlice, IoSliceMut, Read}; +use crate::os::unix::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; use crate::sys::cvt; -use crate::sys_common::AsInner; +use crate::sys_common::{AsInner, FromInner, IntoInner}; -use libc::{c_int, c_void, ssize_t}; +#[cfg(any( + target_os = "android", + target_os = "linux", + target_os = "emscripten", + target_os = "l4re" +))] +use libc::off64_t; +#[cfg(not(any( + target_os = "linux", + target_os = "emscripten", + target_os = "l4re", + target_os = "android" +)))] +use libc::off_t as off64_t; #[derive(Debug)] -pub struct FileDesc { - fd: c_int, -} +pub struct FileDesc(OwnedFd); -fn max_len() -> usize { - // The maximum read limit on most posix-like systems is `SSIZE_MAX`, - // with the man page quoting that if the count of bytes to read is - // greater than `SSIZE_MAX` the result is "unspecified". - // - // On macOS, however, apparently the 64-bit libc is either buggy or - // intentionally showing odd behavior by rejecting any read with a size - // larger than or equal to INT_MAX. To handle both of these the read - // size is capped on both platforms. - if cfg!(target_os = "macos") { - ::max_value() as usize - 1 - } else { - ::max_value() as usize - } -} +// The maximum read limit on most POSIX-like systems is `SSIZE_MAX`, +// with the man page quoting that if the count of bytes to read is +// greater than `SSIZE_MAX` the result is "unspecified". +// +// On macOS, however, apparently the 64-bit libc is either buggy or +// intentionally showing odd behavior by rejecting any read with a size +// larger than or equal to INT_MAX. To handle both of these the read +// size is capped on both platforms. +#[cfg(target_os = "macos")] +const READ_LIMIT: usize = libc::c_int::MAX as usize - 1; +#[cfg(not(target_os = "macos"))] +const READ_LIMIT: usize = libc::ssize_t::MAX as usize; -impl FileDesc { - pub fn new(fd: c_int) -> FileDesc { - FileDesc { fd } - } +#[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "ios", + target_os = "macos", + target_os = "netbsd", + target_os = "openbsd", + target_os = "watchos", +))] +const fn max_iov() -> usize { + libc::IOV_MAX as usize +} - pub fn raw(&self) -> c_int { - self.fd - } +#[cfg(any(target_os = "android", target_os = "emscripten", target_os = "linux"))] +const fn max_iov() -> usize { + libc::UIO_MAXIOV as usize +} - /// Extracts the actual file descriptor without closing it. - pub fn into_raw(self) -> c_int { - let fd = self.fd; - mem::forget(self); - fd - } +#[cfg(not(any( + target_os = "android", + target_os = "dragonfly", + target_os = "emscripten", + target_os = "freebsd", + target_os = "ios", + target_os = "linux", + target_os = "macos", + target_os = "netbsd", + target_os = "openbsd", + target_os = "horizon", + target_os = "watchos", +)))] +const fn max_iov() -> usize { + 16 // The minimum value required by POSIX. +} +impl FileDesc { pub fn read(&self, buf: &mut [u8]) -> io::Result { let ret = cvt(unsafe { - libc::read(self.fd, buf.as_mut_ptr() as *mut c_void, cmp::min(buf.len(), max_len())) + libc::read( + self.as_raw_fd(), + buf.as_mut_ptr() as *mut libc::c_void, + cmp::min(buf.len(), READ_LIMIT), + ) })?; Ok(ret as usize) } + #[cfg(not(any(target_os = "espidf", target_os = "horizon")))] pub fn read_vectored(&self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { let ret = cvt(unsafe { libc::readv( - self.fd, + self.as_raw_fd(), bufs.as_ptr() as *const libc::iovec, - cmp::min(bufs.len(), c_int::max_value() as usize) as c_int, + cmp::min(bufs.len(), max_iov()) as libc::c_int, ) })?; Ok(ret as usize) } + #[cfg(any(target_os = "espidf", target_os = "horizon"))] + pub fn read_vectored(&self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { + return crate::io::default_read_vectored(|b| self.read(b), bufs); + } + + #[inline] + pub fn is_read_vectored(&self) -> bool { + cfg!(not(any(target_os = "espidf", target_os = "horizon"))) + } + pub fn read_to_end(&self, buf: &mut Vec) -> io::Result { let mut me = self; (&mut me).read_to_end(buf) } pub fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result { - #[cfg(target_os = "android")] - use super::android::cvt_pread64; - - #[cfg(not(target_os = "android"))] - unsafe fn cvt_pread64( - fd: c_int, - buf: *mut c_void, - count: usize, - offset: i64, - ) -> io::Result { - #[cfg(not(target_os = "linux"))] - use libc::pread as pread64; - #[cfg(target_os = "linux")] - use libc::pread64; - cvt(pread64(fd, buf, count, offset)) - } + #[cfg(not(any(target_os = "linux", target_os = "android")))] + use libc::pread as pread64; + #[cfg(any(target_os = "linux", target_os = "android"))] + use libc::pread64; unsafe { - cvt_pread64( - self.fd, - buf.as_mut_ptr() as *mut c_void, - cmp::min(buf.len(), max_len()), - offset as i64, - ) + cvt(pread64( + self.as_raw_fd(), + buf.as_mut_ptr() as *mut libc::c_void, + cmp::min(buf.len(), READ_LIMIT), + offset as off64_t, + )) .map(|n| n as usize) } } + pub fn read_buf(&self, mut cursor: BorrowedCursor<'_>) -> io::Result<()> { + let ret = cvt(unsafe { + libc::read( + self.as_raw_fd(), + cursor.as_mut().as_mut_ptr() as *mut libc::c_void, + cmp::min(cursor.capacity(), READ_LIMIT), + ) + })?; + + // Safety: `ret` bytes were written to the initialized portion of the buffer + unsafe { + cursor.advance(ret as usize); + } + Ok(()) + } + pub fn write(&self, buf: &[u8]) -> io::Result { let ret = cvt(unsafe { - libc::write(self.fd, buf.as_ptr() as *const c_void, cmp::min(buf.len(), max_len())) + libc::write( + self.as_raw_fd(), + buf.as_ptr() as *const libc::c_void, + cmp::min(buf.len(), READ_LIMIT), + ) })?; Ok(ret as usize) } + #[cfg(not(any(target_os = "espidf", target_os = "horizon")))] pub fn write_vectored(&self, bufs: &[IoSlice<'_>]) -> io::Result { let ret = cvt(unsafe { libc::writev( - self.fd, + self.as_raw_fd(), bufs.as_ptr() as *const libc::iovec, - cmp::min(bufs.len(), c_int::max_value() as usize) as c_int, + cmp::min(bufs.len(), max_iov()) as libc::c_int, ) })?; Ok(ret as usize) } + #[cfg(any(target_os = "espidf", target_os = "horizon"))] + pub fn write_vectored(&self, bufs: &[IoSlice<'_>]) -> io::Result { + return crate::io::default_write_vectored(|b| self.write(b), bufs); + } + + #[inline] + pub fn is_write_vectored(&self) -> bool { + cfg!(not(any(target_os = "espidf", target_os = "horizon"))) + } + pub fn write_at(&self, buf: &[u8], offset: u64) -> io::Result { - #[cfg(target_os = "android")] - use super::android::cvt_pwrite64; - - #[cfg(not(target_os = "android"))] - unsafe fn cvt_pwrite64( - fd: c_int, - buf: *const c_void, - count: usize, - offset: i64, - ) -> io::Result { - #[cfg(not(target_os = "linux"))] - use libc::pwrite as pwrite64; - #[cfg(target_os = "linux")] - use libc::pwrite64; - cvt(pwrite64(fd, buf, count, offset)) - } + #[cfg(not(any(target_os = "linux", target_os = "android")))] + use libc::pwrite as pwrite64; + #[cfg(any(target_os = "linux", target_os = "android"))] + use libc::pwrite64; unsafe { - cvt_pwrite64( - self.fd, - buf.as_ptr() as *const c_void, - cmp::min(buf.len(), max_len()), - offset as i64, - ) + cvt(pwrite64( + self.as_raw_fd(), + buf.as_ptr() as *const libc::c_void, + cmp::min(buf.len(), READ_LIMIT), + offset as off64_t, + )) .map(|n| n as usize) } } #[cfg(target_os = "linux")] pub fn get_cloexec(&self) -> io::Result { - unsafe { Ok((cvt(libc::fcntl(self.fd, libc::F_GETFD))? & libc::FD_CLOEXEC) != 0) } + unsafe { Ok((cvt(libc::fcntl(self.as_raw_fd(), libc::F_GETFD))? & libc::FD_CLOEXEC) != 0) } } #[cfg(not(any( target_env = "newlib", target_os = "solaris", + target_os = "illumos", target_os = "emscripten", target_os = "fuchsia", target_os = "l4re", target_os = "linux", target_os = "haiku", - target_os = "redox" + target_os = "redox", + target_os = "vxworks" )))] pub fn set_cloexec(&self) -> io::Result<()> { unsafe { - cvt(libc::ioctl(self.fd, libc::FIOCLEX))?; + cvt(libc::ioctl(self.as_raw_fd(), libc::FIOCLEX))?; Ok(()) } } #[cfg(any( - target_env = "newlib", + all(target_env = "newlib", not(any(target_os = "espidf", target_os = "horizon"))), target_os = "solaris", + target_os = "illumos", target_os = "emscripten", target_os = "fuchsia", target_os = "l4re", target_os = "linux", target_os = "haiku", - target_os = "redox" + target_os = "redox", + target_os = "vxworks" ))] pub fn set_cloexec(&self) -> io::Result<()> { unsafe { - let previous = cvt(libc::fcntl(self.fd, libc::F_GETFD))?; + let previous = cvt(libc::fcntl(self.as_raw_fd(), libc::F_GETFD))?; let new = previous | libc::FD_CLOEXEC; if new != previous { - cvt(libc::fcntl(self.fd, libc::F_SETFD, new))?; + cvt(libc::fcntl(self.as_raw_fd(), libc::F_SETFD, new))?; } Ok(()) } } + #[cfg(any(target_os = "espidf", target_os = "horizon"))] + pub fn set_cloexec(&self) -> io::Result<()> { + // FD_CLOEXEC is not supported in ESP-IDF and Horizon OS but there's no need to, + // because neither supports spawning processes. + Ok(()) + } #[cfg(target_os = "linux")] pub fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()> { unsafe { - let v = nonblocking as c_int; - cvt(libc::ioctl(self.fd, libc::FIONBIO, &v))?; + let v = nonblocking as libc::c_int; + cvt(libc::ioctl(self.as_raw_fd(), libc::FIONBIO, &v))?; Ok(()) } } @@ -199,66 +261,22 @@ impl FileDesc { #[cfg(not(target_os = "linux"))] pub fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()> { unsafe { - let previous = cvt(libc::fcntl(self.fd, libc::F_GETFL))?; + let previous = cvt(libc::fcntl(self.as_raw_fd(), libc::F_GETFL))?; let new = if nonblocking { previous | libc::O_NONBLOCK } else { previous & !libc::O_NONBLOCK }; if new != previous { - cvt(libc::fcntl(self.fd, libc::F_SETFL, new))?; + cvt(libc::fcntl(self.as_raw_fd(), libc::F_SETFL, new))?; } Ok(()) } } + #[inline] pub fn duplicate(&self) -> io::Result { - // We want to atomically duplicate this file descriptor and set the - // CLOEXEC flag, and currently that's done via F_DUPFD_CLOEXEC. This - // flag, however, isn't supported on older Linux kernels (earlier than - // 2.6.24). - // - // To detect this and ensure that CLOEXEC is still set, we - // follow a strategy similar to musl [1] where if passing - // F_DUPFD_CLOEXEC causes `fcntl` to return EINVAL it means it's not - // supported (the third parameter, 0, is always valid), so we stop - // trying that. - // - // Also note that Android doesn't have F_DUPFD_CLOEXEC, but get it to - // resolve so we at least compile this. - // - // [1]: http://comments.gmane.org/gmane.linux.lib.musl.general/2963 - #[cfg(any(target_os = "android", target_os = "haiku"))] - use libc::F_DUPFD as F_DUPFD_CLOEXEC; - #[cfg(not(any(target_os = "android", target_os = "haiku")))] - use libc::F_DUPFD_CLOEXEC; - - let make_filedesc = |fd| { - let fd = FileDesc::new(fd); - fd.set_cloexec()?; - Ok(fd) - }; - static TRY_CLOEXEC: AtomicBool = AtomicBool::new(!cfg!(target_os = "android")); - let fd = self.raw(); - if TRY_CLOEXEC.load(Ordering::Relaxed) { - match cvt(unsafe { libc::fcntl(fd, F_DUPFD_CLOEXEC, 0) }) { - // We *still* call the `set_cloexec` method as apparently some - // linux kernel at some point stopped setting CLOEXEC even - // though it reported doing so on F_DUPFD_CLOEXEC. - Ok(fd) => { - return Ok(if cfg!(target_os = "linux") { - make_filedesc(fd)? - } else { - FileDesc::new(fd) - }); - } - Err(ref e) if e.raw_os_error() == Some(libc::EINVAL) => { - TRY_CLOEXEC.store(false, Ordering::Relaxed); - } - Err(e) => return Err(e), - } - } - cvt(unsafe { libc::fcntl(fd, libc::F_DUPFD, 0) }).and_then(make_filedesc) + Ok(Self(self.0.try_clone()?)) } } @@ -266,26 +284,46 @@ impl<'a> Read for &'a FileDesc { fn read(&mut self, buf: &mut [u8]) -> io::Result { (**self).read(buf) } +} - #[inline] - unsafe fn initializer(&self) -> Initializer { - Initializer::nop() +impl AsInner for FileDesc { + fn as_inner(&self) -> &OwnedFd { + &self.0 + } +} + +impl IntoInner for FileDesc { + fn into_inner(self) -> OwnedFd { + self.0 + } +} + +impl FromInner for FileDesc { + fn from_inner(owned_fd: OwnedFd) -> Self { + Self(owned_fd) + } +} + +impl AsFd for FileDesc { + fn as_fd(&self) -> BorrowedFd<'_> { + self.0.as_fd() + } +} + +impl AsRawFd for FileDesc { + fn as_raw_fd(&self) -> RawFd { + self.0.as_raw_fd() } } -impl AsInner for FileDesc { - fn as_inner(&self) -> &c_int { - &self.fd +impl IntoRawFd for FileDesc { + fn into_raw_fd(self) -> RawFd { + self.0.into_raw_fd() } } -impl Drop for FileDesc { - fn drop(&mut self) { - // Note that errors are ignored when closing a file descriptor. The - // reason for this is that if an error occurs we don't actually know if - // the file descriptor was closed or not, and if we retried (for - // something like EINTR), we might close another valid file descriptor - // opened after we closed ours. - let _ = unsafe { libc::close(self.fd) }; +impl FromRawFd for FileDesc { + unsafe fn from_raw_fd(raw_fd: RawFd) -> Self { + Self(FromRawFd::from_raw_fd(raw_fd)) } } diff --git a/crux-mir/lib/std/src/sys/unix/fd/tests.rs b/crux-mir/lib/std/src/sys/unix/fd/tests.rs new file mode 100644 index 000000000..5d17e4678 --- /dev/null +++ b/crux-mir/lib/std/src/sys/unix/fd/tests.rs @@ -0,0 +1,10 @@ +use super::{FileDesc, IoSlice}; +use crate::os::unix::io::FromRawFd; +use core::mem::ManuallyDrop; + +#[test] +fn limit_vector_count() { + let stdout = ManuallyDrop::new(unsafe { FileDesc::from_raw_fd(1) }); + let bufs = (0..1500).map(|_| IoSlice::new(&[])).collect::>(); + assert!(stdout.write_vectored(&bufs).is_ok()); +} diff --git a/crux-mir/lib/std/src/sys/unix/fs.rs b/crux-mir/lib/std/src/sys/unix/fs.rs index ab2a871b9..8e1f35d6c 100644 --- a/crux-mir/lib/std/src/sys/unix/fs.rs +++ b/crux-mir/lib/std/src/sys/unix/fs.rs @@ -1,27 +1,72 @@ +// miri has some special hacks here that make things unused. +#![cfg_attr(miri, allow(unused))] + use crate::os::unix::prelude::*; -use crate::ffi::{CStr, CString, OsStr, OsString}; +use crate::ffi::{CStr, OsStr, OsString}; use crate::fmt; -use crate::io::{self, Error, ErrorKind, IoSlice, IoSliceMut, SeekFrom}; +use crate::io::{self, BorrowedCursor, Error, IoSlice, IoSliceMut, SeekFrom}; use crate::mem; +#[cfg(any( + target_os = "android", + target_os = "linux", + target_os = "solaris", + target_os = "fuchsia", + target_os = "redox", + target_os = "illumos" +))] +use crate::mem::MaybeUninit; +use crate::os::unix::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd}; use crate::path::{Path, PathBuf}; use crate::ptr; use crate::sync::Arc; +use crate::sys::common::small_c_string::run_path_with_cstr; use crate::sys::fd::FileDesc; use crate::sys::time::SystemTime; use crate::sys::{cvt, cvt_r}; -use crate::sys_common::{AsInner, FromInner}; +use crate::sys_common::{AsInner, AsInnerMut, FromInner, IntoInner}; + +#[cfg(any( + all(target_os = "linux", target_env = "gnu"), + target_os = "macos", + target_os = "ios", + target_os = "watchos", +))] +use crate::sys::weak::syscall; +#[cfg(any(target_os = "android", target_os = "macos"))] +use crate::sys::weak::weak; use libc::{c_int, mode_t}; +#[cfg(any( + target_os = "macos", + target_os = "ios", + target_os = "watchos", + all(target_os = "linux", target_env = "gnu") +))] +use libc::c_char; #[cfg(any(target_os = "linux", target_os = "emscripten", target_os = "android"))] use libc::dirfd; #[cfg(any(target_os = "linux", target_os = "emscripten"))] use libc::fstatat64; +#[cfg(any( + target_os = "android", + target_os = "solaris", + target_os = "fuchsia", + target_os = "redox", + target_os = "illumos" +))] +use libc::readdir as readdir64; +#[cfg(target_os = "linux")] +use libc::readdir64; +#[cfg(any(target_os = "emscripten", target_os = "l4re"))] +use libc::readdir64_r; #[cfg(not(any( + target_os = "android", target_os = "linux", target_os = "emscripten", target_os = "solaris", + target_os = "illumos", target_os = "l4re", target_os = "fuchsia", target_os = "redox" @@ -29,8 +74,8 @@ use libc::fstatat64; use libc::readdir_r as readdir64_r; #[cfg(target_os = "android")] use libc::{ - dirent as dirent64, fstat as fstat64, fstatat as fstatat64, lseek64, lstat as lstat64, - open as open64, stat as stat64, + dirent as dirent64, fstat as fstat64, fstatat as fstatat64, ftruncate64, lseek64, + lstat as lstat64, off64_t, open as open64, stat as stat64, }; #[cfg(not(any( target_os = "linux", @@ -43,11 +88,9 @@ use libc::{ lstat as lstat64, off_t as off64_t, open as open64, stat as stat64, }; #[cfg(any(target_os = "linux", target_os = "emscripten", target_os = "l4re"))] -use libc::{ - dirent64, fstat64, ftruncate64, lseek64, lstat64, off64_t, open64, readdir64_r, stat64, -}; +use libc::{dirent64, fstat64, ftruncate64, lseek64, lstat64, off64_t, open64, stat64}; -pub use crate::sys_common::fs::remove_dir_all; +pub use crate::sys_common::fs::try_exists; pub struct File(FileDesc); @@ -85,59 +128,82 @@ cfg_has_statx! {{ // This is needed to check if btime is supported by the filesystem. stx_mask: u32, stx_btime: libc::statx_timestamp, + // With statx, we can overcome 32-bit `time_t` too. + #[cfg(target_pointer_width = "32")] + stx_atime: libc::statx_timestamp, + #[cfg(target_pointer_width = "32")] + stx_ctime: libc::statx_timestamp, + #[cfg(target_pointer_width = "32")] + stx_mtime: libc::statx_timestamp, + } - // We prefer `statx` on Linux if available, which contains file creation time. - // Default `stat64` contains no creation time. + // We prefer `statx` on Linux if available, which contains file creation time, + // as well as 64-bit timestamps of all kinds. + // Default `stat64` contains no creation time and may have 32-bit `time_t`. unsafe fn try_statx( fd: c_int, - path: *const libc::c_char, + path: *const c_char, flags: i32, mask: u32, ) -> Option> { use crate::sync::atomic::{AtomicU8, Ordering}; - // Linux kernel prior to 4.11 or glibc prior to glibc 2.28 don't support `statx` - // We store the availability in global to avoid unnecessary syscalls. - // 0: Unknown - // 1: Not available - // 2: Available - static STATX_STATE: AtomicU8 = AtomicU8::new(0); + // Linux kernel prior to 4.11 or glibc prior to glibc 2.28 don't support `statx`. + // We check for it on first failure and remember availability to avoid having to + // do it again. + #[repr(u8)] + enum STATX_STATE{ Unknown = 0, Present, Unavailable } + static STATX_SAVED_STATE: AtomicU8 = AtomicU8::new(STATX_STATE::Unknown as u8); + syscall! { fn statx( fd: c_int, - pathname: *const libc::c_char, + pathname: *const c_char, flags: c_int, mask: libc::c_uint, statxbuf: *mut libc::statx ) -> c_int } - match STATX_STATE.load(Ordering::Relaxed) { - 0 => { - // It is a trick to call `statx` with NULL pointers to check if the syscall - // is available. According to the manual, it is expected to fail with EFAULT. - // We do this mainly for performance, since it is nearly hundreds times - // faster than a normal successful call. - let err = cvt(statx(0, ptr::null(), 0, libc::STATX_ALL, ptr::null_mut())) - .err() - .and_then(|e| e.raw_os_error()); - // We don't check `err == Some(libc::ENOSYS)` because the syscall may be limited - // and returns `EPERM`. Listing all possible errors seems not a good idea. - // See: https://github.com/rust-lang/rust/issues/65662 - if err != Some(libc::EFAULT) { - STATX_STATE.store(1, Ordering::Relaxed); - return None; - } - STATX_STATE.store(2, Ordering::Relaxed); - } - 1 => return None, - _ => {} + if STATX_SAVED_STATE.load(Ordering::Relaxed) == STATX_STATE::Unavailable as u8 { + return None; } let mut buf: libc::statx = mem::zeroed(); if let Err(err) = cvt(statx(fd, path, flags, mask, &mut buf)) { - return Some(Err(err)); + if STATX_SAVED_STATE.load(Ordering::Relaxed) == STATX_STATE::Present as u8 { + return Some(Err(err)); + } + + // Availability not checked yet. + // + // First try the cheap way. + if err.raw_os_error() == Some(libc::ENOSYS) { + STATX_SAVED_STATE.store(STATX_STATE::Unavailable as u8, Ordering::Relaxed); + return None; + } + + // Error other than `ENOSYS` is not a good enough indicator -- it is + // known that `EPERM` can be returned as a result of using seccomp to + // block the syscall. + // Availability is checked by performing a call which expects `EFAULT` + // if the syscall is usable. + // See: https://github.com/rust-lang/rust/issues/65662 + // FIXME this can probably just do the call if `EPERM` was received, but + // previous iteration of the code checked it for all errors and for now + // this is retained. + // FIXME what about transient conditions like `ENOMEM`? + let err2 = cvt(statx(0, ptr::null(), 0, libc::STATX_ALL, ptr::null_mut())) + .err() + .and_then(|e| e.raw_os_error()); + if err2 == Some(libc::EFAULT) { + STATX_SAVED_STATE.store(STATX_STATE::Present as u8, Ordering::Relaxed); + return Some(Err(err)); + } else { + STATX_SAVED_STATE.store(STATX_STATE::Unavailable as u8, Ordering::Relaxed); + return None; + } } // We cannot fill `stat64` exhaustively because of private padding fields. @@ -164,6 +230,13 @@ cfg_has_statx! {{ let extra = StatxExtraFields { stx_mask: buf.stx_mask, stx_btime: buf.stx_btime, + // Store full times to avoid 32-bit `time_t` truncation. + #[cfg(target_pointer_width = "32")] + stx_atime: buf.stx_atime, + #[cfg(target_pointer_width = "32")] + stx_ctime: buf.stx_ctime, + #[cfg(target_pointer_width = "32")] + stx_mtime: buf.stx_mtime, }; Some(Ok(FileAttr { stat, statx_extra_fields: Some(extra) })) @@ -182,26 +255,68 @@ struct InnerReadDir { root: PathBuf, } -#[derive(Clone)] pub struct ReadDir { inner: Arc, end_of_stream: bool, } +impl ReadDir { + fn new(inner: InnerReadDir) -> Self { + Self { inner: Arc::new(inner), end_of_stream: false } + } +} + struct Dir(*mut libc::DIR); unsafe impl Send for Dir {} unsafe impl Sync for Dir {} +#[cfg(any( + target_os = "android", + target_os = "linux", + target_os = "solaris", + target_os = "illumos", + target_os = "fuchsia", + target_os = "redox" +))] pub struct DirEntry { + dir: Arc, + entry: dirent64_min, + // We need to store an owned copy of the entry name on platforms that use + // readdir() (not readdir_r()), because a) struct dirent may use a flexible + // array to store the name, b) it lives only until the next readdir() call. + name: crate::ffi::CString, +} + +// Define a minimal subset of fields we need from `dirent64`, especially since +// we're not using the immediate `d_name` on these targets. Keeping this as an +// `entry` field in `DirEntry` helps reduce the `cfg` boilerplate elsewhere. +#[cfg(any( + target_os = "android", + target_os = "linux", + target_os = "solaris", + target_os = "illumos", + target_os = "fuchsia", + target_os = "redox" +))] +struct dirent64_min { + d_ino: u64, + #[cfg(not(any(target_os = "solaris", target_os = "illumos")))] + d_type: u8, +} + +#[cfg(not(any( + target_os = "android", + target_os = "linux", + target_os = "solaris", + target_os = "illumos", + target_os = "fuchsia", + target_os = "redox" +)))] +pub struct DirEntry { + dir: Arc, + // The full entry includes a fixed-length `d_name`. entry: dirent64, - dir: ReadDir, - // We need to store an owned copy of the entry name - // on Solaris and Fuchsia because a) it uses a zero-length - // array to store the name, b) its lifetime between readdir - // calls is not guaranteed. - #[cfg(any(target_os = "solaris", target_os = "fuchsia", target_os = "redox"))] - name: Box<[u8]>, } #[derive(Clone, Debug)] @@ -223,11 +338,29 @@ pub struct FilePermissions { mode: mode_t, } -#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] +#[derive(Copy, Clone, Debug, Default)] +pub struct FileTimes { + accessed: Option, + modified: Option, +} + +#[derive(Copy, Clone, Eq, Debug)] pub struct FileType { mode: mode_t, } +impl PartialEq for FileType { + fn eq(&self, other: &Self) -> bool { + self.masked() == other.masked() + } +} + +impl core::hash::Hash for FileType { + fn hash(&self, state: &mut H) { + self.masked().hash(state); + } +} + #[derive(Debug)] pub struct DirBuilder { mode: mode_t, @@ -238,6 +371,36 @@ cfg_has_statx! {{ fn from_stat64(stat: stat64) -> Self { Self { stat, statx_extra_fields: None } } + + #[cfg(target_pointer_width = "32")] + pub fn stx_mtime(&self) -> Option<&libc::statx_timestamp> { + if let Some(ext) = &self.statx_extra_fields { + if (ext.stx_mask & libc::STATX_MTIME) != 0 { + return Some(&ext.stx_mtime); + } + } + None + } + + #[cfg(target_pointer_width = "32")] + pub fn stx_atime(&self) -> Option<&libc::statx_timestamp> { + if let Some(ext) = &self.statx_extra_fields { + if (ext.stx_mask & libc::STATX_ATIME) != 0 { + return Some(&ext.stx_atime); + } + } + None + } + + #[cfg(target_pointer_width = "32")] + pub fn stx_ctime(&self) -> Option<&libc::statx_timestamp> { + if let Some(ext) = &self.statx_extra_fields { + if (ext.stx_mask & libc::STATX_CTIME) != 0 { + return Some(&ext.stx_ctime); + } + } + None + } } } else { impl FileAttr { @@ -263,81 +426,98 @@ impl FileAttr { #[cfg(target_os = "netbsd")] impl FileAttr { pub fn modified(&self) -> io::Result { - Ok(SystemTime::from(libc::timespec { - tv_sec: self.stat.st_mtime as libc::time_t, - tv_nsec: self.stat.st_mtimensec as libc::c_long, - })) + Ok(SystemTime::new(self.stat.st_mtime as i64, self.stat.st_mtimensec as i64)) } pub fn accessed(&self) -> io::Result { - Ok(SystemTime::from(libc::timespec { - tv_sec: self.stat.st_atime as libc::time_t, - tv_nsec: self.stat.st_atimensec as libc::c_long, - })) + Ok(SystemTime::new(self.stat.st_atime as i64, self.stat.st_atimensec as i64)) } pub fn created(&self) -> io::Result { - Ok(SystemTime::from(libc::timespec { - tv_sec: self.stat.st_birthtime as libc::time_t, - tv_nsec: self.stat.st_birthtimensec as libc::c_long, - })) + Ok(SystemTime::new(self.stat.st_birthtime as i64, self.stat.st_birthtimensec as i64)) } } #[cfg(not(target_os = "netbsd"))] impl FileAttr { + #[cfg(not(any(target_os = "vxworks", target_os = "espidf", target_os = "horizon")))] + pub fn modified(&self) -> io::Result { + #[cfg(target_pointer_width = "32")] + cfg_has_statx! { + if let Some(mtime) = self.stx_mtime() { + return Ok(SystemTime::new(mtime.tv_sec, mtime.tv_nsec as i64)); + } + } + + Ok(SystemTime::new(self.stat.st_mtime as i64, self.stat.st_mtime_nsec as i64)) + } + + #[cfg(any(target_os = "vxworks", target_os = "espidf"))] + pub fn modified(&self) -> io::Result { + Ok(SystemTime::new(self.stat.st_mtime as i64, 0)) + } + + #[cfg(target_os = "horizon")] pub fn modified(&self) -> io::Result { - Ok(SystemTime::from(libc::timespec { - tv_sec: self.stat.st_mtime as libc::time_t, - tv_nsec: self.stat.st_mtime_nsec as _, - })) + Ok(SystemTime::from(self.stat.st_mtim)) + } + + #[cfg(not(any(target_os = "vxworks", target_os = "espidf", target_os = "horizon")))] + pub fn accessed(&self) -> io::Result { + #[cfg(target_pointer_width = "32")] + cfg_has_statx! { + if let Some(atime) = self.stx_atime() { + return Ok(SystemTime::new(atime.tv_sec, atime.tv_nsec as i64)); + } + } + + Ok(SystemTime::new(self.stat.st_atime as i64, self.stat.st_atime_nsec as i64)) + } + + #[cfg(any(target_os = "vxworks", target_os = "espidf"))] + pub fn accessed(&self) -> io::Result { + Ok(SystemTime::new(self.stat.st_atime as i64, 0)) } + #[cfg(target_os = "horizon")] pub fn accessed(&self) -> io::Result { - Ok(SystemTime::from(libc::timespec { - tv_sec: self.stat.st_atime as libc::time_t, - tv_nsec: self.stat.st_atime_nsec as _, - })) + Ok(SystemTime::from(self.stat.st_atim)) } #[cfg(any( target_os = "freebsd", target_os = "openbsd", target_os = "macos", - target_os = "ios" + target_os = "ios", + target_os = "watchos", ))] pub fn created(&self) -> io::Result { - Ok(SystemTime::from(libc::timespec { - tv_sec: self.stat.st_birthtime as libc::time_t, - tv_nsec: self.stat.st_birthtime_nsec as libc::c_long, - })) + Ok(SystemTime::new(self.stat.st_birthtime as i64, self.stat.st_birthtime_nsec as i64)) } #[cfg(not(any( target_os = "freebsd", target_os = "openbsd", target_os = "macos", - target_os = "ios" + target_os = "ios", + target_os = "watchos", )))] pub fn created(&self) -> io::Result { cfg_has_statx! { if let Some(ext) = &self.statx_extra_fields { return if (ext.stx_mask & libc::STATX_BTIME) != 0 { - Ok(SystemTime::from(libc::timespec { - tv_sec: ext.stx_btime.tv_sec as libc::time_t, - tv_nsec: ext.stx_btime.tv_nsec as _, - })) + Ok(SystemTime::new(ext.stx_btime.tv_sec, ext.stx_btime.tv_nsec as i64)) } else { - Err(io::Error::new( - io::ErrorKind::Other, + Err(io::const_io_error!( + io::ErrorKind::Uncategorized, "creation time is not available for the filesystem", )) }; } } - Err(io::Error::new( - io::ErrorKind::Other, + Err(io::const_io_error!( + io::ErrorKind::Unsupported, "creation time is not available on this platform \ currently", )) @@ -370,6 +550,16 @@ impl FilePermissions { } } +impl FileTimes { + pub fn set_accessed(&mut self, t: SystemTime) { + self.accessed = Some(t); + } + + pub fn set_modified(&mut self, t: SystemTime) { + self.modified = Some(t); + } +} + impl FileType { pub fn is_dir(&self) -> bool { self.is(libc::S_IFDIR) @@ -382,7 +572,11 @@ impl FileType { } pub fn is(&self, mode: mode_t) -> bool { - self.mode & libc::S_IFMT == mode + self.masked() == mode + } + + fn masked(&self) -> mode_t { + self.mode & libc::S_IFMT } } @@ -403,55 +597,127 @@ impl fmt::Debug for ReadDir { impl Iterator for ReadDir { type Item = io::Result; - #[cfg(any(target_os = "solaris", target_os = "fuchsia", target_os = "redox"))] + #[cfg(any( + target_os = "android", + target_os = "linux", + target_os = "solaris", + target_os = "fuchsia", + target_os = "redox", + target_os = "illumos" + ))] fn next(&mut self) -> Option> { - use crate::slice; + if self.end_of_stream { + return None; + } unsafe { loop { - // Although readdir_r(3) would be a correct function to use here because - // of the thread safety, on Illumos and Fuchsia the readdir(3C) function - // is safe to use in threaded applications and it is generally preferred - // over the readdir_r(3C) function. + // As of POSIX.1-2017, readdir() is not required to be thread safe; only + // readdir_r() is. However, readdir_r() cannot correctly handle platforms + // with unlimited or variable NAME_MAX. Many modern platforms guarantee + // thread safety for readdir() as long an individual DIR* is not accessed + // concurrently, which is sufficient for Rust. super::os::set_errno(0); - let entry_ptr = libc::readdir(self.inner.dirp.0); + let entry_ptr = readdir64(self.inner.dirp.0); if entry_ptr.is_null() { - // NULL can mean either the end is reached or an error occurred. - // So we had to clear errno beforehand to check for an error now. + // We either encountered an error, or reached the end. Either way, + // the next call to next() should return None. + self.end_of_stream = true; + + // To distinguish between errors and end-of-directory, we had to clear + // errno beforehand to check for an error now. return match super::os::errno() { 0 => None, e => Some(Err(Error::from_raw_os_error(e))), }; } - let name = (*entry_ptr).d_name.as_ptr(); - let namelen = libc::strlen(name) as usize; + // The dirent64 struct is a weird imaginary thing that isn't ever supposed + // to be worked with by value. Its trailing d_name field is declared + // variously as [c_char; 256] or [c_char; 1] on different systems but + // either way that size is meaningless; only the offset of d_name is + // meaningful. The dirent64 pointers that libc returns from readdir64 are + // allowed to point to allocations smaller _or_ LARGER than implied by the + // definition of the struct. + // + // As such, we need to be even more careful with dirent64 than if its + // contents were "simply" partially initialized data. + // + // Like for uninitialized contents, converting entry_ptr to `&dirent64` + // would not be legal. However, unique to dirent64 is that we don't even + // get to use `addr_of!((*entry_ptr).d_name)` because that operation + // requires the full extent of *entry_ptr to be in bounds of the same + // allocation, which is not necessarily the case here. + // + // Absent any other way to obtain a pointer to `(*entry_ptr).d_name` + // legally in Rust analogously to how it would be done in C, we instead + // need to make our own non-libc allocation that conforms to the weird + // imaginary definition of dirent64, and use that for a field offset + // computation. + macro_rules! offset_ptr { + ($entry_ptr:expr, $field:ident) => {{ + const OFFSET: isize = { + let delusion = MaybeUninit::::uninit(); + let entry_ptr = delusion.as_ptr(); + unsafe { + ptr::addr_of!((*entry_ptr).$field) + .cast::() + .offset_from(entry_ptr.cast::()) + } + }; + if true { + // Cast to the same type determined by the else branch. + $entry_ptr.byte_offset(OFFSET).cast::<_>() + } else { + #[allow(deref_nullptr)] + { + ptr::addr_of!((*ptr::null::()).$field) + } + } + }}; + } - let ret = DirEntry { - entry: *entry_ptr, - name: slice::from_raw_parts(name as *const u8, namelen as usize) - .to_owned() - .into_boxed_slice(), - dir: self.clone(), - }; - if ret.name_bytes() != b"." && ret.name_bytes() != b".." { - return Some(Ok(ret)); + // d_name is guaranteed to be null-terminated. + let name = CStr::from_ptr(offset_ptr!(entry_ptr, d_name).cast()); + let name_bytes = name.to_bytes(); + if name_bytes == b"." || name_bytes == b".." { + continue; } + + let entry = dirent64_min { + d_ino: *offset_ptr!(entry_ptr, d_ino) as u64, + #[cfg(not(any(target_os = "solaris", target_os = "illumos")))] + d_type: *offset_ptr!(entry_ptr, d_type) as u8, + }; + + return Some(Ok(DirEntry { + entry, + name: name.to_owned(), + dir: Arc::clone(&self.inner), + })); } } } - #[cfg(not(any(target_os = "solaris", target_os = "fuchsia", target_os = "redox")))] + #[cfg(not(any( + target_os = "android", + target_os = "linux", + target_os = "solaris", + target_os = "fuchsia", + target_os = "redox", + target_os = "illumos" + )))] fn next(&mut self) -> Option> { if self.end_of_stream { return None; } unsafe { - let mut ret = DirEntry { entry: mem::zeroed(), dir: self.clone() }; + let mut ret = DirEntry { entry: mem::zeroed(), dir: Arc::clone(&self.inner) }; let mut entry_ptr = ptr::null_mut(); loop { - if readdir64_r(self.inner.dirp.0, &mut ret.entry, &mut entry_ptr) != 0 { + let err = readdir64_r(self.inner.dirp.0, &mut ret.entry, &mut entry_ptr); + if err != 0 { if entry_ptr.is_null() { // We encountered an error (which will be returned in this iteration), but // we also reached the end of the directory stream. The `end_of_stream` @@ -459,7 +725,7 @@ impl Iterator for ReadDir { // (instead of looping forever) self.end_of_stream = true; } - return Some(Err(Error::last_os_error())); + return Some(Err(Error::from_raw_os_error(err))); } if entry_ptr.is_null() { return None; @@ -475,23 +741,30 @@ impl Iterator for ReadDir { impl Drop for Dir { fn drop(&mut self) { let r = unsafe { libc::closedir(self.0) }; - debug_assert_eq!(r, 0); + assert!( + r == 0 || crate::io::Error::last_os_error().kind() == crate::io::ErrorKind::Interrupted, + "unexpected error during closedir: {:?}", + crate::io::Error::last_os_error() + ); } } impl DirEntry { pub fn path(&self) -> PathBuf { - self.dir.inner.root.join(OsStr::from_bytes(self.name_bytes())) + self.dir.root.join(self.file_name_os_str()) } pub fn file_name(&self) -> OsString { - OsStr::from_bytes(self.name_bytes()).to_os_string() + self.file_name_os_str().to_os_string() } - #[cfg(any(target_os = "linux", target_os = "emscripten", target_os = "android"))] + #[cfg(all( + any(target_os = "linux", target_os = "emscripten", target_os = "android"), + not(miri) + ))] pub fn metadata(&self) -> io::Result { - let fd = cvt(unsafe { dirfd(self.dir.inner.dirp.0) })?; - let name = self.entry.d_name.as_ptr(); + let fd = cvt(unsafe { dirfd(self.dir.dirp.0) })?; + let name = self.name_cstr().as_ptr(); cfg_has_statx! { if let Some(ret) = unsafe { try_statx( @@ -509,17 +782,30 @@ impl DirEntry { Ok(FileAttr::from_stat64(stat)) } - #[cfg(not(any(target_os = "linux", target_os = "emscripten", target_os = "android")))] + #[cfg(any( + not(any(target_os = "linux", target_os = "emscripten", target_os = "android")), + miri + ))] pub fn metadata(&self) -> io::Result { lstat(&self.path()) } - #[cfg(any(target_os = "solaris", target_os = "haiku"))] + #[cfg(any( + target_os = "solaris", + target_os = "illumos", + target_os = "haiku", + target_os = "vxworks" + ))] pub fn file_type(&self) -> io::Result { - lstat(&self.path()).map(|m| m.file_type()) + self.metadata().map(|m| m.file_type()) } - #[cfg(not(any(target_os = "solaris", target_os = "haiku")))] + #[cfg(not(any( + target_os = "solaris", + target_os = "illumos", + target_os = "haiku", + target_os = "vxworks" + )))] pub fn file_type(&self) -> io::Result { match self.entry.d_type { libc::DT_CHR => Ok(FileType { mode: libc::S_IFCHR }), @@ -529,21 +815,26 @@ impl DirEntry { libc::DT_SOCK => Ok(FileType { mode: libc::S_IFSOCK }), libc::DT_DIR => Ok(FileType { mode: libc::S_IFDIR }), libc::DT_BLK => Ok(FileType { mode: libc::S_IFBLK }), - _ => lstat(&self.path()).map(|m| m.file_type()), + _ => self.metadata().map(|m| m.file_type()), } } #[cfg(any( target_os = "macos", target_os = "ios", + target_os = "watchos", target_os = "linux", target_os = "emscripten", target_os = "android", target_os = "solaris", + target_os = "illumos", target_os = "haiku", target_os = "l4re", target_os = "fuchsia", - target_os = "redox" + target_os = "redox", + target_os = "vxworks", + target_os = "espidf", + target_os = "horizon" ))] pub fn ino(&self) -> u64 { self.entry.d_ino as u64 @@ -562,6 +853,7 @@ impl DirEntry { #[cfg(any( target_os = "macos", target_os = "ios", + target_os = "watchos", target_os = "netbsd", target_os = "openbsd", target_os = "freebsd", @@ -576,19 +868,44 @@ impl DirEntry { ) } } + #[cfg(not(any( + target_os = "macos", + target_os = "ios", + target_os = "watchos", + target_os = "netbsd", + target_os = "openbsd", + target_os = "freebsd", + target_os = "dragonfly" + )))] + fn name_bytes(&self) -> &[u8] { + self.name_cstr().to_bytes() + } + + #[cfg(not(any( + target_os = "android", + target_os = "linux", + target_os = "solaris", + target_os = "illumos", + target_os = "fuchsia", + target_os = "redox" + )))] + fn name_cstr(&self) -> &CStr { + unsafe { CStr::from_ptr(self.entry.d_name.as_ptr()) } + } #[cfg(any( target_os = "android", target_os = "linux", - target_os = "emscripten", - target_os = "l4re", - target_os = "haiku" + target_os = "solaris", + target_os = "illumos", + target_os = "fuchsia", + target_os = "redox" ))] - fn name_bytes(&self) -> &[u8] { - unsafe { CStr::from_ptr(self.entry.d_name.as_ptr()).to_bytes() } + fn name_cstr(&self) -> &CStr { + &self.name } - #[cfg(any(target_os = "solaris", target_os = "fuchsia", target_os = "redox"))] - fn name_bytes(&self) -> &[u8] { - &*self.name + + pub fn file_name_os_str(&self) -> &OsStr { + OsStr::from_bytes(self.name_bytes()) } } @@ -672,8 +989,7 @@ impl OpenOptions { impl File { pub fn open(path: &Path, opts: &OpenOptions) -> io::Result { - let path = cstr(path)?; - File::open_c(&path, opts) + run_path_with_cstr(path, |path| File::open_c(path, opts)) } pub fn open_c(path: &CStr, opts: &OpenOptions) -> io::Result { @@ -681,66 +997,21 @@ impl File { | opts.get_access_mode()? | opts.get_creation_mode()? | (opts.custom_flags as c_int & !libc::O_ACCMODE); + // The third argument of `open64` is documented to have type `mode_t`. On + // some platforms (like macOS, where `open64` is actually `open`), `mode_t` is `u16`. + // However, since this is a variadic function, C integer promotion rules mean that on + // the ABI level, this still gets passed as `c_int` (aka `u32` on Unix platforms). let fd = cvt_r(|| unsafe { open64(path.as_ptr(), flags, opts.mode as c_int) })?; - let fd = FileDesc::new(fd); - - // Currently the standard library supports Linux 2.6.18 which did not - // have the O_CLOEXEC flag (passed above). If we're running on an older - // Linux kernel then the flag is just ignored by the OS. After we open - // the first file, we check whether it has CLOEXEC set. If it doesn't, - // we will explicitly ask for a CLOEXEC fd for every further file we - // open, if it does, we will skip that step. - // - // The CLOEXEC flag, however, is supported on versions of macOS/BSD/etc - // that we support, so we only do this on Linux currently. - #[cfg(target_os = "linux")] - fn ensure_cloexec(fd: &FileDesc) -> io::Result<()> { - use crate::sync::atomic::{AtomicUsize, Ordering}; - - const OPEN_CLOEXEC_UNKNOWN: usize = 0; - const OPEN_CLOEXEC_SUPPORTED: usize = 1; - const OPEN_CLOEXEC_NOTSUPPORTED: usize = 2; - static OPEN_CLOEXEC: AtomicUsize = AtomicUsize::new(OPEN_CLOEXEC_UNKNOWN); - - let need_to_set; - match OPEN_CLOEXEC.load(Ordering::Relaxed) { - OPEN_CLOEXEC_UNKNOWN => { - need_to_set = !fd.get_cloexec()?; - OPEN_CLOEXEC.store( - if need_to_set { - OPEN_CLOEXEC_NOTSUPPORTED - } else { - OPEN_CLOEXEC_SUPPORTED - }, - Ordering::Relaxed, - ); - } - OPEN_CLOEXEC_SUPPORTED => need_to_set = false, - OPEN_CLOEXEC_NOTSUPPORTED => need_to_set = true, - _ => unreachable!(), - } - if need_to_set { - fd.set_cloexec()?; - } - Ok(()) - } - - #[cfg(not(target_os = "linux"))] - fn ensure_cloexec(_: &FileDesc) -> io::Result<()> { - Ok(()) - } - - ensure_cloexec(&fd)?; - Ok(File(fd)) + Ok(File(unsafe { FileDesc::from_raw_fd(fd) })) } pub fn file_attr(&self) -> io::Result { - let fd = self.0.raw(); + let fd = self.as_raw_fd(); cfg_has_statx! { if let Some(ret) = unsafe { try_statx( fd, - b"\0" as *const _ as *const libc::c_char, + b"\0" as *const _ as *const c_char, libc::AT_EMPTY_PATH | libc::AT_STATX_SYNC_AS_STAT, libc::STATX_ALL, ) } { @@ -754,48 +1025,56 @@ impl File { } pub fn fsync(&self) -> io::Result<()> { - cvt_r(|| unsafe { os_fsync(self.0.raw()) })?; + cvt_r(|| unsafe { os_fsync(self.as_raw_fd()) })?; return Ok(()); - #[cfg(any(target_os = "macos", target_os = "ios"))] + #[cfg(any(target_os = "macos", target_os = "ios", target_os = "watchos"))] unsafe fn os_fsync(fd: c_int) -> c_int { libc::fcntl(fd, libc::F_FULLFSYNC) } - #[cfg(not(any(target_os = "macos", target_os = "ios")))] + #[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "watchos")))] unsafe fn os_fsync(fd: c_int) -> c_int { libc::fsync(fd) } } pub fn datasync(&self) -> io::Result<()> { - cvt_r(|| unsafe { os_datasync(self.0.raw()) })?; + cvt_r(|| unsafe { os_datasync(self.as_raw_fd()) })?; return Ok(()); - #[cfg(any(target_os = "macos", target_os = "ios"))] + #[cfg(any(target_os = "macos", target_os = "ios", target_os = "watchos"))] unsafe fn os_datasync(fd: c_int) -> c_int { libc::fcntl(fd, libc::F_FULLFSYNC) } - #[cfg(target_os = "linux")] + #[cfg(any( + target_os = "freebsd", + target_os = "linux", + target_os = "android", + target_os = "netbsd", + target_os = "openbsd" + ))] unsafe fn os_datasync(fd: c_int) -> c_int { libc::fdatasync(fd) } - #[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "linux")))] + #[cfg(not(any( + target_os = "android", + target_os = "freebsd", + target_os = "ios", + target_os = "linux", + target_os = "macos", + target_os = "netbsd", + target_os = "openbsd", + target_os = "watchos", + )))] unsafe fn os_datasync(fd: c_int) -> c_int { libc::fsync(fd) } } pub fn truncate(&self, size: u64) -> io::Result<()> { - #[cfg(target_os = "android")] - return crate::sys::android::ftruncate64(self.0.raw(), size); - - #[cfg(not(target_os = "android"))] - { - use crate::convert::TryInto; - let size: off64_t = - size.try_into().map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?; - cvt_r(|| unsafe { ftruncate64(self.0.raw(), size) }).map(drop) - } + let size: off64_t = + size.try_into().map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?; + cvt_r(|| unsafe { ftruncate64(self.as_raw_fd(), size) }).map(drop) } pub fn read(&self, buf: &mut [u8]) -> io::Result { @@ -806,10 +1085,19 @@ impl File { self.0.read_vectored(bufs) } + #[inline] + pub fn is_read_vectored(&self) -> bool { + self.0.is_read_vectored() + } + pub fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result { self.0.read_at(buf, offset) } + pub fn read_buf(&self, cursor: BorrowedCursor<'_>) -> io::Result<()> { + self.0.read_buf(cursor) + } + pub fn write(&self, buf: &[u8]) -> io::Result { self.0.write(buf) } @@ -818,6 +1106,11 @@ impl File { self.0.write_vectored(bufs) } + #[inline] + pub fn is_write_vectored(&self) -> bool { + self.0.is_write_vectored() + } + pub fn write_at(&self, buf: &[u8], offset: u64) -> io::Result { self.0.write_at(buf, offset) } @@ -834,7 +1127,7 @@ impl File { SeekFrom::End(off) => (libc::SEEK_END, off), SeekFrom::Current(off) => (libc::SEEK_CUR, off), }; - let n = cvt(unsafe { lseek64(self.0.raw(), pos, whence) })?; + let n = cvt(unsafe { lseek64(self.as_raw_fd(), pos as off64_t, whence) })?; Ok(n as u64) } @@ -842,18 +1135,65 @@ impl File { self.0.duplicate().map(File) } - pub fn fd(&self) -> &FileDesc { - &self.0 - } - - pub fn into_fd(self) -> FileDesc { - self.0 - } - pub fn set_permissions(&self, perm: FilePermissions) -> io::Result<()> { - cvt_r(|| unsafe { libc::fchmod(self.0.raw(), perm.mode) })?; + cvt_r(|| unsafe { libc::fchmod(self.as_raw_fd(), perm.mode) })?; Ok(()) } + + pub fn set_times(&self, times: FileTimes) -> io::Result<()> { + #[cfg(not(any(target_os = "redox", target_os = "espidf", target_os = "horizon")))] + let to_timespec = |time: Option| { + match time { + Some(time) if let Some(ts) = time.t.to_timespec() => Ok(ts), + Some(time) if time > crate::sys::time::UNIX_EPOCH => Err(io::const_io_error!(io::ErrorKind::InvalidInput, "timestamp is too large to set as a file time")), + Some(_) => Err(io::const_io_error!(io::ErrorKind::InvalidInput, "timestamp is too small to set as a file time")), + None => Ok(libc::timespec { tv_sec: 0, tv_nsec: libc::UTIME_OMIT as _ }), + } + }; + #[cfg(not(any(target_os = "redox", target_os = "espidf", target_os = "horizon")))] + let times = [to_timespec(times.accessed)?, to_timespec(times.modified)?]; + cfg_if::cfg_if! { + if #[cfg(any(target_os = "redox", target_os = "espidf", target_os = "horizon"))] { + // Redox doesn't appear to support `UTIME_OMIT`. + // ESP-IDF and HorizonOS do not support `futimens` at all and the behavior for those OS is therefore + // the same as for Redox. + drop(times); + Err(io::const_io_error!( + io::ErrorKind::Unsupported, + "setting file times not supported", + )) + } else if #[cfg(any(target_os = "android", target_os = "macos"))] { + // futimens requires macOS 10.13, and Android API level 19 + cvt(unsafe { + weak!(fn futimens(c_int, *const libc::timespec) -> c_int); + match futimens.get() { + Some(futimens) => futimens(self.as_raw_fd(), times.as_ptr()), + #[cfg(target_os = "macos")] + None => { + fn ts_to_tv(ts: &libc::timespec) -> libc::timeval { + libc::timeval { + tv_sec: ts.tv_sec, + tv_usec: (ts.tv_nsec / 1000) as _ + } + } + let timevals = [ts_to_tv(×[0]), ts_to_tv(×[1])]; + libc::futimes(self.as_raw_fd(), timevals.as_ptr()) + } + // futimes requires even newer Android. + #[cfg(target_os = "android")] + None => return Err(io::const_io_error!( + io::ErrorKind::Unsupported, + "setting file times requires Android API level >= 19", + )), + } + })?; + Ok(()) + } else { + cvt(unsafe { libc::futimens(self.as_raw_fd(), times.as_ptr()) })?; + Ok(()) + } + } + } } impl DirBuilder { @@ -862,9 +1202,7 @@ impl DirBuilder { } pub fn mkdir(&self, p: &Path) -> io::Result<()> { - let p = cstr(p)?; - cvt(unsafe { libc::mkdir(p.as_ptr(), self.mode) })?; - Ok(()) + run_path_with_cstr(p, |p| cvt(unsafe { libc::mkdir(p.as_ptr(), self.mode) }).map(|_| ())) } pub fn set_mode(&mut self, mode: u32) { @@ -872,19 +1210,62 @@ impl DirBuilder { } } -fn cstr(path: &Path) -> io::Result { - Ok(CString::new(path.as_os_str().as_bytes())?) +impl AsInner for File { + fn as_inner(&self) -> &FileDesc { + &self.0 + } +} + +impl AsInnerMut for File { + fn as_inner_mut(&mut self) -> &mut FileDesc { + &mut self.0 + } +} + +impl IntoInner for File { + fn into_inner(self) -> FileDesc { + self.0 + } +} + +impl FromInner for File { + fn from_inner(file_desc: FileDesc) -> Self { + Self(file_desc) + } +} + +impl AsFd for File { + fn as_fd(&self) -> BorrowedFd<'_> { + self.0.as_fd() + } +} + +impl AsRawFd for File { + fn as_raw_fd(&self) -> RawFd { + self.0.as_raw_fd() + } +} + +impl IntoRawFd for File { + fn into_raw_fd(self) -> RawFd { + self.0.into_raw_fd() + } } -impl FromInner for File { - fn from_inner(fd: c_int) -> File { - File(FileDesc::new(fd)) +impl FromRawFd for File { + unsafe fn from_raw_fd(raw_fd: RawFd) -> Self { + Self(FromRawFd::from_raw_fd(raw_fd)) } } impl fmt::Debug for File { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - #[cfg(target_os = "linux")] + #[cfg(any( + target_os = "linux", + target_os = "netbsd", + target_os = "illumos", + target_os = "solaris" + ))] fn get_path(fd: c_int) -> Option { let mut p = PathBuf::from("/proc/self/fd"); p.push(&fd.to_string()); @@ -909,13 +1290,53 @@ impl fmt::Debug for File { Some(PathBuf::from(OsString::from_vec(buf))) } - #[cfg(not(any(target_os = "linux", target_os = "macos")))] + #[cfg(all(target_os = "freebsd", target_arch = "x86_64"))] + fn get_path(fd: c_int) -> Option { + let info = Box::::new_zeroed(); + let mut info = unsafe { info.assume_init() }; + info.kf_structsize = mem::size_of::() as libc::c_int; + let n = unsafe { libc::fcntl(fd, libc::F_KINFO, &mut *info) }; + if n == -1 { + return None; + } + let buf = unsafe { CStr::from_ptr(info.kf_path.as_mut_ptr()).to_bytes().to_vec() }; + Some(PathBuf::from(OsString::from_vec(buf))) + } + + #[cfg(target_os = "vxworks")] + fn get_path(fd: c_int) -> Option { + let mut buf = vec![0; libc::PATH_MAX as usize]; + let n = unsafe { libc::ioctl(fd, libc::FIOGETNAME, buf.as_ptr()) }; + if n == -1 { + return None; + } + let l = buf.iter().position(|&c| c == 0).unwrap(); + buf.truncate(l as usize); + Some(PathBuf::from(OsString::from_vec(buf))) + } + + #[cfg(not(any( + target_os = "linux", + target_os = "macos", + target_os = "vxworks", + all(target_os = "freebsd", target_arch = "x86_64"), + target_os = "netbsd", + target_os = "illumos", + target_os = "solaris" + )))] fn get_path(_fd: c_int) -> Option { // FIXME(#24570): implement this for other Unix platforms None } - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(any( + target_os = "linux", + target_os = "macos", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_os = "vxworks" + ))] fn get_mode(fd: c_int) -> Option<(bool, bool)> { let mode = unsafe { libc::fcntl(fd, libc::F_GETFL) }; if mode == -1 { @@ -929,13 +1350,20 @@ impl fmt::Debug for File { } } - #[cfg(not(any(target_os = "linux", target_os = "macos")))] + #[cfg(not(any( + target_os = "linux", + target_os = "macos", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_os = "vxworks" + )))] fn get_mode(_fd: c_int) -> Option<(bool, bool)> { // FIXME(#24570): implement this for other Unix platforms None } - let fd = self.0.raw(); + let fd = self.as_raw_fd(); let mut b = f.debug_struct("File"); b.field("fd", &fd); if let Some(path) = get_path(fd) { @@ -948,152 +1376,185 @@ impl fmt::Debug for File { } } -pub fn readdir(p: &Path) -> io::Result { - let root = p.to_path_buf(); - let p = cstr(p)?; - unsafe { - let ptr = libc::opendir(p.as_ptr()); - if ptr.is_null() { - Err(Error::last_os_error()) - } else { - let inner = InnerReadDir { dirp: Dir(ptr), root }; - Ok(ReadDir { inner: Arc::new(inner), end_of_stream: false }) - } +pub fn readdir(path: &Path) -> io::Result { + let ptr = run_path_with_cstr(path, |p| unsafe { Ok(libc::opendir(p.as_ptr())) })?; + if ptr.is_null() { + Err(Error::last_os_error()) + } else { + let root = path.to_path_buf(); + let inner = InnerReadDir { dirp: Dir(ptr), root }; + Ok(ReadDir::new(inner)) } } pub fn unlink(p: &Path) -> io::Result<()> { - let p = cstr(p)?; - cvt(unsafe { libc::unlink(p.as_ptr()) })?; - Ok(()) + run_path_with_cstr(p, |p| cvt(unsafe { libc::unlink(p.as_ptr()) }).map(|_| ())) } pub fn rename(old: &Path, new: &Path) -> io::Result<()> { - let old = cstr(old)?; - let new = cstr(new)?; - cvt(unsafe { libc::rename(old.as_ptr(), new.as_ptr()) })?; - Ok(()) + run_path_with_cstr(old, |old| { + run_path_with_cstr(new, |new| { + cvt(unsafe { libc::rename(old.as_ptr(), new.as_ptr()) }).map(|_| ()) + }) + }) } pub fn set_perm(p: &Path, perm: FilePermissions) -> io::Result<()> { - let p = cstr(p)?; - cvt_r(|| unsafe { libc::chmod(p.as_ptr(), perm.mode) })?; - Ok(()) + run_path_with_cstr(p, |p| cvt_r(|| unsafe { libc::chmod(p.as_ptr(), perm.mode) }).map(|_| ())) } pub fn rmdir(p: &Path) -> io::Result<()> { - let p = cstr(p)?; - cvt(unsafe { libc::rmdir(p.as_ptr()) })?; - Ok(()) + run_path_with_cstr(p, |p| cvt(unsafe { libc::rmdir(p.as_ptr()) }).map(|_| ())) } pub fn readlink(p: &Path) -> io::Result { - let c_path = cstr(p)?; - let p = c_path.as_ptr(); + run_path_with_cstr(p, |c_path| { + let p = c_path.as_ptr(); - let mut buf = Vec::with_capacity(256); + let mut buf = Vec::with_capacity(256); - loop { - let buf_read = - cvt(unsafe { libc::readlink(p, buf.as_mut_ptr() as *mut _, buf.capacity()) })? as usize; + loop { + let buf_read = + cvt(unsafe { libc::readlink(p, buf.as_mut_ptr() as *mut _, buf.capacity()) })? + as usize; - unsafe { - buf.set_len(buf_read); - } + unsafe { + buf.set_len(buf_read); + } - if buf_read != buf.capacity() { - buf.shrink_to_fit(); + if buf_read != buf.capacity() { + buf.shrink_to_fit(); - return Ok(PathBuf::from(OsString::from_vec(buf))); - } + return Ok(PathBuf::from(OsString::from_vec(buf))); + } - // Trigger the internal buffer resizing logic of `Vec` by requiring - // more space than the current capacity. The length is guaranteed to be - // the same as the capacity due to the if statement above. - buf.reserve(1); - } + // Trigger the internal buffer resizing logic of `Vec` by requiring + // more space than the current capacity. The length is guaranteed to be + // the same as the capacity due to the if statement above. + buf.reserve(1); + } + }) } -pub fn symlink(src: &Path, dst: &Path) -> io::Result<()> { - let src = cstr(src)?; - let dst = cstr(dst)?; - cvt(unsafe { libc::symlink(src.as_ptr(), dst.as_ptr()) })?; - Ok(()) +pub fn symlink(original: &Path, link: &Path) -> io::Result<()> { + run_path_with_cstr(original, |original| { + run_path_with_cstr(link, |link| { + cvt(unsafe { libc::symlink(original.as_ptr(), link.as_ptr()) }).map(|_| ()) + }) + }) } -pub fn link(src: &Path, dst: &Path) -> io::Result<()> { - let src = cstr(src)?; - let dst = cstr(dst)?; - cvt(unsafe { libc::link(src.as_ptr(), dst.as_ptr()) })?; - Ok(()) +pub fn link(original: &Path, link: &Path) -> io::Result<()> { + run_path_with_cstr(original, |original| { + run_path_with_cstr(link, |link| { + cfg_if::cfg_if! { + if #[cfg(any(target_os = "vxworks", target_os = "redox", target_os = "android", target_os = "espidf", target_os = "horizon"))] { + // VxWorks, Redox and ESP-IDF lack `linkat`, so use `link` instead. POSIX leaves + // it implementation-defined whether `link` follows symlinks, so rely on the + // `symlink_hard_link` test in library/std/src/fs/tests.rs to check the behavior. + // Android has `linkat` on newer versions, but we happen to know `link` + // always has the correct behavior, so it's here as well. + cvt(unsafe { libc::link(original.as_ptr(), link.as_ptr()) })?; + } else if #[cfg(target_os = "macos")] { + // On MacOS, older versions (<=10.9) lack support for linkat while newer + // versions have it. We want to use linkat if it is available, so we use weak! + // to check. `linkat` is preferable to `link` because it gives us a flag to + // specify how symlinks should be handled. We pass 0 as the flags argument, + // meaning it shouldn't follow symlinks. + weak!(fn linkat(c_int, *const c_char, c_int, *const c_char, c_int) -> c_int); + + if let Some(f) = linkat.get() { + cvt(unsafe { f(libc::AT_FDCWD, original.as_ptr(), libc::AT_FDCWD, link.as_ptr(), 0) })?; + } else { + cvt(unsafe { libc::link(original.as_ptr(), link.as_ptr()) })?; + }; + } else { + // Where we can, use `linkat` instead of `link`; see the comment above + // this one for details on why. + cvt(unsafe { libc::linkat(libc::AT_FDCWD, original.as_ptr(), libc::AT_FDCWD, link.as_ptr(), 0) })?; + } + } + Ok(()) + }) + }) } pub fn stat(p: &Path) -> io::Result { - let p = cstr(p)?; - - cfg_has_statx! { - if let Some(ret) = unsafe { try_statx( - libc::AT_FDCWD, - p.as_ptr(), - libc::AT_STATX_SYNC_AS_STAT, - libc::STATX_ALL, - ) } { - return ret; + run_path_with_cstr(p, |p| { + cfg_has_statx! { + if let Some(ret) = unsafe { try_statx( + libc::AT_FDCWD, + p.as_ptr(), + libc::AT_STATX_SYNC_AS_STAT, + libc::STATX_ALL, + ) } { + return ret; + } } - } - let mut stat: stat64 = unsafe { mem::zeroed() }; - cvt(unsafe { stat64(p.as_ptr(), &mut stat) })?; - Ok(FileAttr::from_stat64(stat)) + let mut stat: stat64 = unsafe { mem::zeroed() }; + cvt(unsafe { stat64(p.as_ptr(), &mut stat) })?; + Ok(FileAttr::from_stat64(stat)) + }) } pub fn lstat(p: &Path) -> io::Result { - let p = cstr(p)?; - - cfg_has_statx! { - if let Some(ret) = unsafe { try_statx( - libc::AT_FDCWD, - p.as_ptr(), - libc::AT_SYMLINK_NOFOLLOW | libc::AT_STATX_SYNC_AS_STAT, - libc::STATX_ALL, - ) } { - return ret; + run_path_with_cstr(p, |p| { + cfg_has_statx! { + if let Some(ret) = unsafe { try_statx( + libc::AT_FDCWD, + p.as_ptr(), + libc::AT_SYMLINK_NOFOLLOW | libc::AT_STATX_SYNC_AS_STAT, + libc::STATX_ALL, + ) } { + return ret; + } } - } - let mut stat: stat64 = unsafe { mem::zeroed() }; - cvt(unsafe { lstat64(p.as_ptr(), &mut stat) })?; - Ok(FileAttr::from_stat64(stat)) + let mut stat: stat64 = unsafe { mem::zeroed() }; + cvt(unsafe { lstat64(p.as_ptr(), &mut stat) })?; + Ok(FileAttr::from_stat64(stat)) + }) } pub fn canonicalize(p: &Path) -> io::Result { - let path = CString::new(p.as_os_str().as_bytes())?; - let buf; - unsafe { - let r = libc::realpath(path.as_ptr(), ptr::null_mut()); - if r.is_null() { - return Err(io::Error::last_os_error()); - } - buf = CStr::from_ptr(r).to_bytes().to_vec(); - libc::free(r as *mut _); + let r = run_path_with_cstr(p, |path| unsafe { + Ok(libc::realpath(path.as_ptr(), ptr::null_mut())) + })?; + if r.is_null() { + return Err(io::Error::last_os_error()); } - Ok(PathBuf::from(OsString::from_vec(buf))) + Ok(PathBuf::from(OsString::from_vec(unsafe { + let buf = CStr::from_ptr(r).to_bytes().to_vec(); + libc::free(r as *mut _); + buf + }))) } fn open_from(from: &Path) -> io::Result<(crate::fs::File, crate::fs::Metadata)> { use crate::fs::File; + use crate::sys_common::fs::NOT_FILE_ERROR; let reader = File::open(from)?; let metadata = reader.metadata()?; if !metadata.is_file() { - return Err(Error::new( - ErrorKind::InvalidInput, - "the source path is not an existing regular file", - )); + return Err(NOT_FILE_ERROR); } Ok((reader, metadata)) } +#[cfg(target_os = "espidf")] +fn open_to_and_set_permissions( + to: &Path, + reader_metadata: crate::fs::Metadata, +) -> io::Result<(crate::fs::File, crate::fs::Metadata)> { + use crate::fs::OpenOptions; + let writer = OpenOptions::new().open(to)?; + let writer_metadata = writer.metadata()?; + Ok((writer, writer_metadata)) +} + +#[cfg(not(target_os = "espidf"))] fn open_to_and_set_permissions( to: &Path, reader_metadata: crate::fs::Metadata, @@ -1123,7 +1584,8 @@ fn open_to_and_set_permissions( target_os = "linux", target_os = "android", target_os = "macos", - target_os = "ios" + target_os = "ios", + target_os = "watchos", )))] pub fn copy(from: &Path, to: &Path) -> io::Result { let (mut reader, reader_metadata) = open_from(from)?; @@ -1134,84 +1596,23 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { #[cfg(any(target_os = "linux", target_os = "android"))] pub fn copy(from: &Path, to: &Path) -> io::Result { - use crate::cmp; - use crate::sync::atomic::{AtomicBool, Ordering}; - - // Kernel prior to 4.5 don't have copy_file_range - // We store the availability in a global to avoid unnecessary syscalls - static HAS_COPY_FILE_RANGE: AtomicBool = AtomicBool::new(true); - - unsafe fn copy_file_range( - fd_in: libc::c_int, - off_in: *mut libc::loff_t, - fd_out: libc::c_int, - off_out: *mut libc::loff_t, - len: libc::size_t, - flags: libc::c_uint, - ) -> libc::c_long { - libc::syscall(libc::SYS_copy_file_range, fd_in, off_in, fd_out, off_out, len, flags) - } - let (mut reader, reader_metadata) = open_from(from)?; - let len = reader_metadata.len(); + let max_len = u64::MAX; let (mut writer, _) = open_to_and_set_permissions(to, reader_metadata)?; - let has_copy_file_range = HAS_COPY_FILE_RANGE.load(Ordering::Relaxed); - let mut written = 0u64; - while written < len { - let copy_result = if has_copy_file_range { - let bytes_to_copy = cmp::min(len - written, usize::max_value() as u64) as usize; - let copy_result = unsafe { - // We actually don't have to adjust the offsets, - // because copy_file_range adjusts the file offset automatically - cvt(copy_file_range( - reader.as_raw_fd(), - ptr::null_mut(), - writer.as_raw_fd(), - ptr::null_mut(), - bytes_to_copy, - 0, - )) - }; - if let Err(ref copy_err) = copy_result { - match copy_err.raw_os_error() { - Some(libc::ENOSYS) | Some(libc::EPERM) => { - HAS_COPY_FILE_RANGE.store(false, Ordering::Relaxed); - } - _ => {} - } - } - copy_result - } else { - Err(io::Error::from_raw_os_error(libc::ENOSYS)) - }; - match copy_result { - Ok(ret) => written += ret as u64, - Err(err) => { - match err.raw_os_error() { - Some(os_err) - if os_err == libc::ENOSYS - || os_err == libc::EXDEV - || os_err == libc::EINVAL - || os_err == libc::EPERM => - { - // Try fallback io::copy if either: - // - Kernel version is < 4.5 (ENOSYS) - // - Files are mounted on different fs (EXDEV) - // - copy_file_range is disallowed, for example by seccomp (EPERM) - // - copy_file_range cannot be used with pipes or device nodes (EINVAL) - assert_eq!(written, 0); - return io::copy(&mut reader, &mut writer); - } - _ => return Err(err), - } - } - } + use super::kernel_copy::{copy_regular_files, CopyResult}; + + match copy_regular_files(reader.as_raw_fd(), writer.as_raw_fd(), max_len) { + CopyResult::Ended(bytes) => Ok(bytes), + CopyResult::Error(e, _) => Err(e), + CopyResult::Fallback(written) => match io::copy::generic_copy(&mut reader, &mut writer) { + Ok(bytes) => Ok(bytes + written), + Err(e) => Err(e), + }, } - Ok(written) } -#[cfg(any(target_os = "macos", target_os = "ios"))] +#[cfg(any(target_os = "macos", target_os = "ios", target_os = "watchos"))] pub fn copy(from: &Path, to: &Path) -> io::Result { use crate::sync::atomic::{AtomicBool, Ordering}; @@ -1267,7 +1668,7 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { fn fclonefileat( srcfd: libc::c_int, dst_dirfd: libc::c_int, - dst: *const libc::c_char, + dst: *const c_char, flags: libc::c_int ) -> libc::c_int } @@ -1277,9 +1678,9 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { // Opportunistically attempt to create a copy-on-write clone of `from` // using `fclonefileat`. if HAS_FCLONEFILEAT.load(Ordering::Relaxed) { - let to = cstr(to)?; - let clonefile_result = - cvt(unsafe { fclonefileat(reader.as_raw_fd(), libc::AT_FDCWD, to.as_ptr(), 0) }); + let clonefile_result = run_path_with_cstr(to, |to| { + cvt(unsafe { fclonefileat(reader.as_raw_fd(), libc::AT_FDCWD, to.as_ptr(), 0) }) + }); match clonefile_result { Ok(_) => return Ok(reader_metadata.len()), Err(err) => match err.raw_os_error() { @@ -1321,3 +1722,224 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { })?; Ok(bytes_copied as u64) } + +pub fn chown(path: &Path, uid: u32, gid: u32) -> io::Result<()> { + run_path_with_cstr(path, |path| { + cvt(unsafe { libc::chown(path.as_ptr(), uid as libc::uid_t, gid as libc::gid_t) }) + .map(|_| ()) + }) +} + +pub fn fchown(fd: c_int, uid: u32, gid: u32) -> io::Result<()> { + cvt(unsafe { libc::fchown(fd, uid as libc::uid_t, gid as libc::gid_t) })?; + Ok(()) +} + +pub fn lchown(path: &Path, uid: u32, gid: u32) -> io::Result<()> { + run_path_with_cstr(path, |path| { + cvt(unsafe { libc::lchown(path.as_ptr(), uid as libc::uid_t, gid as libc::gid_t) }) + .map(|_| ()) + }) +} + +#[cfg(not(any(target_os = "fuchsia", target_os = "vxworks")))] +pub fn chroot(dir: &Path) -> io::Result<()> { + run_path_with_cstr(dir, |dir| cvt(unsafe { libc::chroot(dir.as_ptr()) }).map(|_| ())) +} + +pub use remove_dir_impl::remove_dir_all; + +// Fallback for REDOX, ESP-ID, Horizon, and Miri +#[cfg(any(target_os = "redox", target_os = "espidf", target_os = "horizon", miri))] +mod remove_dir_impl { + pub use crate::sys_common::fs::remove_dir_all; +} + +// Modern implementation using openat(), unlinkat() and fdopendir() +#[cfg(not(any(target_os = "redox", target_os = "espidf", target_os = "horizon", miri)))] +mod remove_dir_impl { + use super::{lstat, Dir, DirEntry, InnerReadDir, ReadDir}; + use crate::ffi::CStr; + use crate::io; + use crate::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd}; + use crate::os::unix::prelude::{OwnedFd, RawFd}; + use crate::path::{Path, PathBuf}; + use crate::sys::common::small_c_string::run_path_with_cstr; + use crate::sys::{cvt, cvt_r}; + + #[cfg(not(any( + all(target_os = "linux", target_env = "gnu"), + all(target_os = "macos", not(target_arch = "aarch64")) + )))] + use libc::{fdopendir, openat, unlinkat}; + #[cfg(all(target_os = "linux", target_env = "gnu"))] + use libc::{fdopendir, openat64 as openat, unlinkat}; + #[cfg(all(target_os = "macos", not(target_arch = "aarch64")))] + use macos_weak::{fdopendir, openat, unlinkat}; + + #[cfg(all(target_os = "macos", not(target_arch = "aarch64")))] + mod macos_weak { + use crate::sys::weak::weak; + use libc::{c_char, c_int, DIR}; + + fn get_openat_fn() -> Option c_int> { + weak!(fn openat(c_int, *const c_char, c_int) -> c_int); + openat.get() + } + + pub fn has_openat() -> bool { + get_openat_fn().is_some() + } + + pub unsafe fn openat(dirfd: c_int, pathname: *const c_char, flags: c_int) -> c_int { + get_openat_fn().map(|openat| openat(dirfd, pathname, flags)).unwrap_or_else(|| { + crate::sys::unix::os::set_errno(libc::ENOSYS); + -1 + }) + } + + pub unsafe fn fdopendir(fd: c_int) -> *mut DIR { + #[cfg(all(target_os = "macos", target_arch = "x86"))] + weak!(fn fdopendir(c_int) -> *mut DIR, "fdopendir$INODE64$UNIX2003"); + #[cfg(all(target_os = "macos", target_arch = "x86_64"))] + weak!(fn fdopendir(c_int) -> *mut DIR, "fdopendir$INODE64"); + fdopendir.get().map(|fdopendir| fdopendir(fd)).unwrap_or_else(|| { + crate::sys::unix::os::set_errno(libc::ENOSYS); + crate::ptr::null_mut() + }) + } + + pub unsafe fn unlinkat(dirfd: c_int, pathname: *const c_char, flags: c_int) -> c_int { + weak!(fn unlinkat(c_int, *const c_char, c_int) -> c_int); + unlinkat.get().map(|unlinkat| unlinkat(dirfd, pathname, flags)).unwrap_or_else(|| { + crate::sys::unix::os::set_errno(libc::ENOSYS); + -1 + }) + } + } + + pub fn openat_nofollow_dironly(parent_fd: Option, p: &CStr) -> io::Result { + let fd = cvt_r(|| unsafe { + openat( + parent_fd.unwrap_or(libc::AT_FDCWD), + p.as_ptr(), + libc::O_CLOEXEC | libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_DIRECTORY, + ) + })?; + Ok(unsafe { OwnedFd::from_raw_fd(fd) }) + } + + fn fdreaddir(dir_fd: OwnedFd) -> io::Result<(ReadDir, RawFd)> { + let ptr = unsafe { fdopendir(dir_fd.as_raw_fd()) }; + if ptr.is_null() { + return Err(io::Error::last_os_error()); + } + let dirp = Dir(ptr); + // file descriptor is automatically closed by libc::closedir() now, so give up ownership + let new_parent_fd = dir_fd.into_raw_fd(); + // a valid root is not needed because we do not call any functions involving the full path + // of the DirEntrys. + let dummy_root = PathBuf::new(); + let inner = InnerReadDir { dirp, root: dummy_root }; + Ok((ReadDir::new(inner), new_parent_fd)) + } + + #[cfg(any( + target_os = "solaris", + target_os = "illumos", + target_os = "haiku", + target_os = "vxworks", + ))] + fn is_dir(_ent: &DirEntry) -> Option { + None + } + + #[cfg(not(any( + target_os = "solaris", + target_os = "illumos", + target_os = "haiku", + target_os = "vxworks", + )))] + fn is_dir(ent: &DirEntry) -> Option { + match ent.entry.d_type { + libc::DT_UNKNOWN => None, + libc::DT_DIR => Some(true), + _ => Some(false), + } + } + + fn remove_dir_all_recursive(parent_fd: Option, path: &CStr) -> io::Result<()> { + // try opening as directory + let fd = match openat_nofollow_dironly(parent_fd, &path) { + Err(err) if matches!(err.raw_os_error(), Some(libc::ENOTDIR | libc::ELOOP)) => { + // not a directory - don't traverse further + // (for symlinks, older Linux kernels may return ELOOP instead of ENOTDIR) + return match parent_fd { + // unlink... + Some(parent_fd) => { + cvt(unsafe { unlinkat(parent_fd, path.as_ptr(), 0) }).map(drop) + } + // ...unless this was supposed to be the deletion root directory + None => Err(err), + }; + } + result => result?, + }; + + // open the directory passing ownership of the fd + let (dir, fd) = fdreaddir(fd)?; + for child in dir { + let child = child?; + let child_name = child.name_cstr(); + match is_dir(&child) { + Some(true) => { + remove_dir_all_recursive(Some(fd), child_name)?; + } + Some(false) => { + cvt(unsafe { unlinkat(fd, child_name.as_ptr(), 0) })?; + } + None => { + // POSIX specifies that calling unlink()/unlinkat(..., 0) on a directory can succeed + // if the process has the appropriate privileges. This however can causing orphaned + // directories requiring an fsck e.g. on Solaris and Illumos. So we try recursing + // into it first instead of trying to unlink() it. + remove_dir_all_recursive(Some(fd), child_name)?; + } + } + } + + // unlink the directory after removing its contents + cvt(unsafe { + unlinkat(parent_fd.unwrap_or(libc::AT_FDCWD), path.as_ptr(), libc::AT_REMOVEDIR) + })?; + Ok(()) + } + + fn remove_dir_all_modern(p: &Path) -> io::Result<()> { + // We cannot just call remove_dir_all_recursive() here because that would not delete a passed + // symlink. No need to worry about races, because remove_dir_all_recursive() does not recurse + // into symlinks. + let attr = lstat(p)?; + if attr.file_type().is_symlink() { + crate::fs::remove_file(p) + } else { + run_path_with_cstr(p, |p| remove_dir_all_recursive(None, &p)) + } + } + + #[cfg(not(all(target_os = "macos", not(target_arch = "aarch64"))))] + pub fn remove_dir_all(p: &Path) -> io::Result<()> { + remove_dir_all_modern(p) + } + + #[cfg(all(target_os = "macos", not(target_arch = "aarch64")))] + pub fn remove_dir_all(p: &Path) -> io::Result<()> { + if macos_weak::has_openat() { + // openat() is available with macOS 10.10+, just like unlinkat() and fdopendir() + remove_dir_all_modern(p) + } else { + // fall back to classic implementation + crate::sys_common::fs::remove_dir_all(p) + } + } +} diff --git a/crux-mir/lib/std/src/sys/unix/futex.rs b/crux-mir/lib/std/src/sys/unix/futex.rs new file mode 100644 index 000000000..8d5b54021 --- /dev/null +++ b/crux-mir/lib/std/src/sys/unix/futex.rs @@ -0,0 +1,303 @@ +#![cfg(any( + target_os = "linux", + target_os = "android", + all(target_os = "emscripten", target_feature = "atomics"), + target_os = "freebsd", + target_os = "openbsd", + target_os = "dragonfly", + target_os = "fuchsia", +))] + +use crate::sync::atomic::AtomicU32; +use crate::time::Duration; + +/// Wait for a futex_wake operation to wake us. +/// +/// Returns directly if the futex doesn't hold the expected value. +/// +/// Returns false on timeout, and true in all other cases. +#[cfg(any(target_os = "linux", target_os = "android", target_os = "freebsd"))] +pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option) -> bool { + use super::time::Timespec; + use crate::ptr::null; + use crate::sync::atomic::Ordering::Relaxed; + + // Calculate the timeout as an absolute timespec. + // + // Overflows are rounded up to an infinite timeout (None). + let timespec = timeout + .and_then(|d| Timespec::now(libc::CLOCK_MONOTONIC).checked_add_duration(&d)) + .and_then(|t| t.to_timespec()); + + loop { + // No need to wait if the value already changed. + if futex.load(Relaxed) != expected { + return true; + } + + let r = unsafe { + cfg_if::cfg_if! { + if #[cfg(target_os = "freebsd")] { + // FreeBSD doesn't have futex(), but it has + // _umtx_op(UMTX_OP_WAIT_UINT_PRIVATE), which is nearly + // identical. It supports absolute timeouts through a flag + // in the _umtx_time struct. + let umtx_timeout = timespec.map(|t| libc::_umtx_time { + _timeout: t, + _flags: libc::UMTX_ABSTIME, + _clockid: libc::CLOCK_MONOTONIC as u32, + }); + let umtx_timeout_ptr = umtx_timeout.as_ref().map_or(null(), |t| t as *const _); + let umtx_timeout_size = umtx_timeout.as_ref().map_or(0, |t| crate::mem::size_of_val(t)); + libc::_umtx_op( + futex as *const AtomicU32 as *mut _, + libc::UMTX_OP_WAIT_UINT_PRIVATE, + expected as libc::c_ulong, + crate::ptr::invalid_mut(umtx_timeout_size), + umtx_timeout_ptr as *mut _, + ) + } else if #[cfg(any(target_os = "linux", target_os = "android"))] { + // Use FUTEX_WAIT_BITSET rather than FUTEX_WAIT to be able to give an + // absolute time rather than a relative time. + libc::syscall( + libc::SYS_futex, + futex as *const AtomicU32, + libc::FUTEX_WAIT_BITSET | libc::FUTEX_PRIVATE_FLAG, + expected, + timespec.as_ref().map_or(null(), |t| t as *const libc::timespec), + null::(), // This argument is unused for FUTEX_WAIT_BITSET. + !0u32, // A full bitmask, to make it behave like a regular FUTEX_WAIT. + ) + } else { + compile_error!("unknown target_os"); + } + } + }; + + match (r < 0).then(super::os::errno) { + Some(libc::ETIMEDOUT) => return false, + Some(libc::EINTR) => continue, + _ => return true, + } + } +} + +/// Wake up one thread that's blocked on futex_wait on this futex. +/// +/// Returns true if this actually woke up such a thread, +/// or false if no thread was waiting on this futex. +/// +/// On some platforms, this always returns false. +#[cfg(any(target_os = "linux", target_os = "android"))] +pub fn futex_wake(futex: &AtomicU32) -> bool { + let ptr = futex as *const AtomicU32; + let op = libc::FUTEX_WAKE | libc::FUTEX_PRIVATE_FLAG; + unsafe { libc::syscall(libc::SYS_futex, ptr, op, 1) > 0 } +} + +/// Wake up all threads that are waiting on futex_wait on this futex. +#[cfg(any(target_os = "linux", target_os = "android"))] +pub fn futex_wake_all(futex: &AtomicU32) { + let ptr = futex as *const AtomicU32; + let op = libc::FUTEX_WAKE | libc::FUTEX_PRIVATE_FLAG; + unsafe { + libc::syscall(libc::SYS_futex, ptr, op, i32::MAX); + } +} + +// FreeBSD doesn't tell us how many threads are woken up, so this always returns false. +#[cfg(target_os = "freebsd")] +pub fn futex_wake(futex: &AtomicU32) -> bool { + use crate::ptr::null_mut; + unsafe { + libc::_umtx_op( + futex as *const AtomicU32 as *mut _, + libc::UMTX_OP_WAKE_PRIVATE, + 1, + null_mut(), + null_mut(), + ) + }; + false +} + +#[cfg(target_os = "freebsd")] +pub fn futex_wake_all(futex: &AtomicU32) { + use crate::ptr::null_mut; + unsafe { + libc::_umtx_op( + futex as *const AtomicU32 as *mut _, + libc::UMTX_OP_WAKE_PRIVATE, + i32::MAX as libc::c_ulong, + null_mut(), + null_mut(), + ) + }; +} + +#[cfg(target_os = "openbsd")] +pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option) -> bool { + use super::time::Timespec; + use crate::ptr::{null, null_mut}; + + // Overflows are rounded up to an infinite timeout (None). + let timespec = timeout + .and_then(|d| Timespec::zero().checked_add_duration(&d)) + .and_then(|t| t.to_timespec()); + + let r = unsafe { + libc::futex( + futex as *const AtomicU32 as *mut u32, + libc::FUTEX_WAIT, + expected as i32, + timespec.as_ref().map_or(null(), |t| t as *const libc::timespec), + null_mut(), + ) + }; + + r == 0 || super::os::errno() != libc::ETIMEDOUT +} + +#[cfg(target_os = "openbsd")] +pub fn futex_wake(futex: &AtomicU32) -> bool { + use crate::ptr::{null, null_mut}; + unsafe { + libc::futex(futex as *const AtomicU32 as *mut u32, libc::FUTEX_WAKE, 1, null(), null_mut()) + > 0 + } +} + +#[cfg(target_os = "openbsd")] +pub fn futex_wake_all(futex: &AtomicU32) { + use crate::ptr::{null, null_mut}; + unsafe { + libc::futex( + futex as *const AtomicU32 as *mut u32, + libc::FUTEX_WAKE, + i32::MAX, + null(), + null_mut(), + ); + } +} + +#[cfg(target_os = "dragonfly")] +pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option) -> bool { + // A timeout of 0 means infinite. + // We round smaller timeouts up to 1 millisecond. + // Overflows are rounded up to an infinite timeout. + let timeout_ms = + timeout.and_then(|d| Some(i32::try_from(d.as_millis()).ok()?.max(1))).unwrap_or(0); + + let r = unsafe { + libc::umtx_sleep(futex as *const AtomicU32 as *const i32, expected as i32, timeout_ms) + }; + + r == 0 || super::os::errno() != libc::ETIMEDOUT +} + +// DragonflyBSD doesn't tell us how many threads are woken up, so this always returns false. +#[cfg(target_os = "dragonfly")] +pub fn futex_wake(futex: &AtomicU32) -> bool { + unsafe { libc::umtx_wakeup(futex as *const AtomicU32 as *const i32, 1) }; + false +} + +#[cfg(target_os = "dragonfly")] +pub fn futex_wake_all(futex: &AtomicU32) { + unsafe { libc::umtx_wakeup(futex as *const AtomicU32 as *const i32, i32::MAX) }; +} + +#[cfg(target_os = "emscripten")] +extern "C" { + fn emscripten_futex_wake(addr: *const AtomicU32, count: libc::c_int) -> libc::c_int; + fn emscripten_futex_wait( + addr: *const AtomicU32, + val: libc::c_uint, + max_wait_ms: libc::c_double, + ) -> libc::c_int; +} + +#[cfg(target_os = "emscripten")] +pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option) -> bool { + unsafe { + emscripten_futex_wait( + futex, + expected, + timeout.map_or(f64::INFINITY, |d| d.as_secs_f64() * 1000.0), + ) != -libc::ETIMEDOUT + } +} + +#[cfg(target_os = "emscripten")] +pub fn futex_wake(futex: &AtomicU32) -> bool { + unsafe { emscripten_futex_wake(futex, 1) > 0 } +} + +#[cfg(target_os = "emscripten")] +pub fn futex_wake_all(futex: &AtomicU32) { + unsafe { emscripten_futex_wake(futex, i32::MAX) }; +} + +#[cfg(target_os = "fuchsia")] +pub mod zircon { + pub type zx_futex_t = crate::sync::atomic::AtomicU32; + pub type zx_handle_t = u32; + pub type zx_status_t = i32; + pub type zx_time_t = i64; + + pub const ZX_HANDLE_INVALID: zx_handle_t = 0; + + pub const ZX_TIME_INFINITE: zx_time_t = zx_time_t::MAX; + + pub const ZX_OK: zx_status_t = 0; + pub const ZX_ERR_INVALID_ARGS: zx_status_t = -10; + pub const ZX_ERR_BAD_HANDLE: zx_status_t = -11; + pub const ZX_ERR_WRONG_TYPE: zx_status_t = -12; + pub const ZX_ERR_BAD_STATE: zx_status_t = -20; + pub const ZX_ERR_TIMED_OUT: zx_status_t = -21; + + extern "C" { + pub fn zx_clock_get_monotonic() -> zx_time_t; + pub fn zx_futex_wait( + value_ptr: *const zx_futex_t, + current_value: zx_futex_t, + new_futex_owner: zx_handle_t, + deadline: zx_time_t, + ) -> zx_status_t; + pub fn zx_futex_wake(value_ptr: *const zx_futex_t, wake_count: u32) -> zx_status_t; + pub fn zx_futex_wake_single_owner(value_ptr: *const zx_futex_t) -> zx_status_t; + pub fn zx_thread_self() -> zx_handle_t; + } +} + +#[cfg(target_os = "fuchsia")] +pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option) -> bool { + use crate::convert::TryFrom; + + // Sleep forever if the timeout is longer than fits in a i64. + let deadline = timeout + .and_then(|d| { + i64::try_from(d.as_nanos()) + .ok()? + .checked_add(unsafe { zircon::zx_clock_get_monotonic() }) + }) + .unwrap_or(zircon::ZX_TIME_INFINITE); + + unsafe { + zircon::zx_futex_wait(futex, AtomicU32::new(expected), zircon::ZX_HANDLE_INVALID, deadline) + != zircon::ZX_ERR_TIMED_OUT + } +} + +// Fuchsia doesn't tell us how many threads are woken up, so this always returns false. +#[cfg(target_os = "fuchsia")] +pub fn futex_wake(futex: &AtomicU32) -> bool { + unsafe { zircon::zx_futex_wake(futex, 1) }; + false +} + +#[cfg(target_os = "fuchsia")] +pub fn futex_wake_all(futex: &AtomicU32) { + unsafe { zircon::zx_futex_wake(futex, u32::MAX) }; +} diff --git a/crux-mir/lib/std/src/sys/unix/io.rs b/crux-mir/lib/std/src/sys/unix/io.rs index deb5ee76b..29c340dd3 100644 --- a/crux-mir/lib/std/src/sys/unix/io.rs +++ b/crux-mir/lib/std/src/sys/unix/io.rs @@ -1,4 +1,5 @@ use crate::marker::PhantomData; +use crate::os::fd::{AsFd, AsRawFd}; use crate::slice; use libc::{c_void, iovec}; @@ -74,3 +75,8 @@ impl<'a> IoSliceMut<'a> { unsafe { slice::from_raw_parts_mut(self.vec.iov_base as *mut u8, self.vec.iov_len) } } } + +pub fn is_terminal(fd: &impl AsFd) -> bool { + let fd = fd.as_fd(); + unsafe { libc::isatty(fd.as_raw_fd()) != 0 } +} diff --git a/crux-mir/lib/std/src/sys/unix/kernel_copy.rs b/crux-mir/lib/std/src/sys/unix/kernel_copy.rs new file mode 100644 index 000000000..73b9bef7e --- /dev/null +++ b/crux-mir/lib/std/src/sys/unix/kernel_copy.rs @@ -0,0 +1,690 @@ +//! This module contains specializations that can offload `io::copy()` operations on file descriptor +//! containing types (`File`, `TcpStream`, etc.) to more efficient syscalls than `read(2)` and `write(2)`. +//! +//! Specialization is only applied to wholly std-owned types so that user code can't observe +//! that the `Read` and `Write` traits are not used. +//! +//! Since a copy operation involves a reader and writer side where each can consist of different types +//! and also involve generic wrappers (e.g. `Take`, `BufReader`) it is not practical to specialize +//! a single method on all possible combinations. +//! +//! Instead readers and writers are handled separately by the `CopyRead` and `CopyWrite` specialization +//! traits and then specialized on by the `Copier::copy` method. +//! +//! `Copier` uses the specialization traits to unpack the underlying file descriptors and +//! additional prerequisites and constraints imposed by the wrapper types. +//! +//! Once it has obtained all necessary pieces and brought any wrapper types into a state where they +//! can be safely bypassed it will attempt to use the `copy_file_range(2)`, +//! `sendfile(2)` or `splice(2)` syscalls to move data directly between file descriptors. +//! Since those syscalls have requirements that cannot be fully checked in advance and +//! gathering additional information about file descriptors would require additional syscalls +//! anyway it simply attempts to use them one after another (guided by inaccurate hints) to +//! figure out which one works and falls back to the generic read-write copy loop if none of them +//! does. +//! Once a working syscall is found for a pair of file descriptors it will be called in a loop +//! until the copy operation is completed. +//! +//! Advantages of using these syscalls: +//! +//! * fewer context switches since reads and writes are coalesced into a single syscall +//! and more bytes are transferred per syscall. This translates to higher throughput +//! and fewer CPU cycles, at least for sufficiently large transfers to amortize the initial probing. +//! * `copy_file_range` creates reflink copies on CoW filesystems, thus moving less data and +//! consuming less disk space +//! * `sendfile` and `splice` can perform zero-copy IO under some circumstances while +//! a naive copy loop would move every byte through the CPU. +//! +//! Drawbacks: +//! +//! * copy operations smaller than the default buffer size can under some circumstances, especially +//! on older kernels, incur more syscalls than the naive approach would. As mentioned above +//! the syscall selection is guided by hints to minimize this possibility but they are not perfect. +//! * optimizations only apply to std types. If a user adds a custom wrapper type, e.g. to report +//! progress, they can hit a performance cliff. +//! * complexity + +use crate::cmp::min; +use crate::fs::{File, Metadata}; +use crate::io::copy::generic_copy; +use crate::io::{ + BufRead, BufReader, BufWriter, Error, Read, Result, StderrLock, StdinLock, StdoutLock, Take, + Write, +}; +use crate::mem::ManuallyDrop; +use crate::net::TcpStream; +use crate::os::unix::fs::FileTypeExt; +use crate::os::unix::io::{AsRawFd, FromRawFd, RawFd}; +use crate::os::unix::net::UnixStream; +use crate::process::{ChildStderr, ChildStdin, ChildStdout}; +use crate::ptr; +use crate::sync::atomic::{AtomicBool, AtomicU8, Ordering}; +use crate::sys::cvt; +use crate::sys::weak::syscall; +#[cfg(not(all(target_os = "linux", target_env = "gnu")))] +use libc::sendfile as sendfile64; +#[cfg(all(target_os = "linux", target_env = "gnu"))] +use libc::sendfile64; +use libc::{EBADF, EINVAL, ENOSYS, EOPNOTSUPP, EOVERFLOW, EPERM, EXDEV}; + +#[cfg(test)] +mod tests; + +pub(crate) fn copy_spec( + read: &mut R, + write: &mut W, +) -> Result { + let copier = Copier { read, write }; + SpecCopy::copy(copier) +} + +/// This type represents either the inferred `FileType` of a `RawFd` based on the source +/// type from which it was extracted or the actual metadata +/// +/// The methods on this type only provide hints, due to `AsRawFd` and `FromRawFd` the inferred +/// type may be wrong. +enum FdMeta { + /// We obtained the FD from a type that can contain any type of `FileType` and queried the metadata + /// because it is cheaper than probing all possible syscalls (reader side) + Metadata(Metadata), + Socket, + Pipe, + /// We don't have any metadata, e.g. because the original type was `File` which can represent + /// any `FileType` and we did not query the metadata either since it did not seem beneficial + /// (writer side) + NoneObtained, +} + +impl FdMeta { + fn maybe_fifo(&self) -> bool { + match self { + FdMeta::Metadata(meta) => meta.file_type().is_fifo(), + FdMeta::Socket => false, + FdMeta::Pipe => true, + FdMeta::NoneObtained => true, + } + } + + fn potential_sendfile_source(&self) -> bool { + match self { + // procfs erroneously shows 0 length on non-empty readable files. + // and if a file is truly empty then a `read` syscall will determine that and skip the write syscall + // thus there would be benefit from attempting sendfile + FdMeta::Metadata(meta) + if meta.file_type().is_file() && meta.len() > 0 + || meta.file_type().is_block_device() => + { + true + } + _ => false, + } + } + + fn copy_file_range_candidate(&self) -> bool { + match self { + // copy_file_range will fail on empty procfs files. `read` can determine whether EOF has been reached + // without extra cost and skip the write, thus there is no benefit in attempting copy_file_range + FdMeta::Metadata(meta) if meta.is_file() && meta.len() > 0 => true, + FdMeta::NoneObtained => true, + _ => false, + } + } +} + +struct CopyParams(FdMeta, Option); + +struct Copier<'a, 'b, R: Read + ?Sized, W: Write + ?Sized> { + read: &'a mut R, + write: &'b mut W, +} + +trait SpecCopy { + fn copy(self) -> Result; +} + +impl SpecCopy for Copier<'_, '_, R, W> { + default fn copy(self) -> Result { + generic_copy(self.read, self.write) + } +} + +impl SpecCopy for Copier<'_, '_, R, W> { + fn copy(self) -> Result { + let (reader, writer) = (self.read, self.write); + let r_cfg = reader.properties(); + let w_cfg = writer.properties(); + + // before direct operations on file descriptors ensure that all source and sink buffers are empty + let mut flush = || -> crate::io::Result { + let bytes = reader.drain_to(writer, u64::MAX)?; + // BufWriter buffered bytes have already been accounted for in earlier write() calls + writer.flush()?; + Ok(bytes) + }; + + let mut written = 0u64; + + if let (CopyParams(input_meta, Some(readfd)), CopyParams(output_meta, Some(writefd))) = + (r_cfg, w_cfg) + { + written += flush()?; + let max_write = reader.min_limit(); + + if input_meta.copy_file_range_candidate() && output_meta.copy_file_range_candidate() { + let result = copy_regular_files(readfd, writefd, max_write); + result.update_take(reader); + + match result { + CopyResult::Ended(bytes_copied) => return Ok(bytes_copied + written), + CopyResult::Error(e, _) => return Err(e), + CopyResult::Fallback(bytes) => written += bytes, + } + } + + // on modern kernels sendfile can copy from any mmapable type (some but not all regular files and block devices) + // to any writable file descriptor. On older kernels the writer side can only be a socket. + // So we just try and fallback if needed. + // If current file offsets + write sizes overflow it may also fail, we do not try to fix that and instead + // fall back to the generic copy loop. + if input_meta.potential_sendfile_source() { + let result = sendfile_splice(SpliceMode::Sendfile, readfd, writefd, max_write); + result.update_take(reader); + + match result { + CopyResult::Ended(bytes_copied) => return Ok(bytes_copied + written), + CopyResult::Error(e, _) => return Err(e), + CopyResult::Fallback(bytes) => written += bytes, + } + } + + if input_meta.maybe_fifo() || output_meta.maybe_fifo() { + let result = sendfile_splice(SpliceMode::Splice, readfd, writefd, max_write); + result.update_take(reader); + + match result { + CopyResult::Ended(bytes_copied) => return Ok(bytes_copied + written), + CopyResult::Error(e, _) => return Err(e), + CopyResult::Fallback(0) => { /* use the fallback below */ } + CopyResult::Fallback(_) => { + unreachable!("splice should not return > 0 bytes on the fallback path") + } + } + } + } + + // fallback if none of the more specialized syscalls wants to work with these file descriptors + match generic_copy(reader, writer) { + Ok(bytes) => Ok(bytes + written), + err => err, + } + } +} + +#[rustc_specialization_trait] +trait CopyRead: Read { + /// Implementations that contain buffers (i.e. `BufReader`) must transfer data from their internal + /// buffers into `writer` until either the buffers are emptied or `limit` bytes have been + /// transferred, whichever occurs sooner. + /// If nested buffers are present the outer buffers must be drained first. + /// + /// This is necessary to directly bypass the wrapper types while preserving the data order + /// when operating directly on the underlying file descriptors. + fn drain_to(&mut self, _writer: &mut W, _limit: u64) -> Result { + Ok(0) + } + + /// Updates `Take` wrappers to remove the number of bytes copied. + fn taken(&mut self, _bytes: u64) {} + + /// The minimum of the limit of all `Take<_>` wrappers, `u64::MAX` otherwise. + /// This method does not account for data `BufReader` buffers and would underreport + /// the limit of a `Take>>` type. Thus its result is only valid + /// after draining the buffers via `drain_to`. + fn min_limit(&self) -> u64 { + u64::MAX + } + + /// Extracts the file descriptor and hints/metadata, delegating through wrappers if necessary. + fn properties(&self) -> CopyParams; +} + +#[rustc_specialization_trait] +trait CopyWrite: Write { + /// Extracts the file descriptor and hints/metadata, delegating through wrappers if necessary. + fn properties(&self) -> CopyParams; +} + +impl CopyRead for &mut T +where + T: CopyRead, +{ + fn drain_to(&mut self, writer: &mut W, limit: u64) -> Result { + (**self).drain_to(writer, limit) + } + + fn taken(&mut self, bytes: u64) { + (**self).taken(bytes); + } + + fn min_limit(&self) -> u64 { + (**self).min_limit() + } + + fn properties(&self) -> CopyParams { + (**self).properties() + } +} + +impl CopyWrite for &mut T +where + T: CopyWrite, +{ + fn properties(&self) -> CopyParams { + (**self).properties() + } +} + +impl CopyRead for File { + fn properties(&self) -> CopyParams { + CopyParams(fd_to_meta(self), Some(self.as_raw_fd())) + } +} + +impl CopyRead for &File { + fn properties(&self) -> CopyParams { + CopyParams(fd_to_meta(*self), Some(self.as_raw_fd())) + } +} + +impl CopyWrite for File { + fn properties(&self) -> CopyParams { + CopyParams(FdMeta::NoneObtained, Some(self.as_raw_fd())) + } +} + +impl CopyWrite for &File { + fn properties(&self) -> CopyParams { + CopyParams(FdMeta::NoneObtained, Some(self.as_raw_fd())) + } +} + +impl CopyRead for TcpStream { + fn properties(&self) -> CopyParams { + // avoid the stat syscall since we can be fairly sure it's a socket + CopyParams(FdMeta::Socket, Some(self.as_raw_fd())) + } +} + +impl CopyRead for &TcpStream { + fn properties(&self) -> CopyParams { + // avoid the stat syscall since we can be fairly sure it's a socket + CopyParams(FdMeta::Socket, Some(self.as_raw_fd())) + } +} + +impl CopyWrite for TcpStream { + fn properties(&self) -> CopyParams { + // avoid the stat syscall since we can be fairly sure it's a socket + CopyParams(FdMeta::Socket, Some(self.as_raw_fd())) + } +} + +impl CopyWrite for &TcpStream { + fn properties(&self) -> CopyParams { + // avoid the stat syscall since we can be fairly sure it's a socket + CopyParams(FdMeta::Socket, Some(self.as_raw_fd())) + } +} + +impl CopyRead for UnixStream { + fn properties(&self) -> CopyParams { + // avoid the stat syscall since we can be fairly sure it's a socket + CopyParams(FdMeta::Socket, Some(self.as_raw_fd())) + } +} + +impl CopyRead for &UnixStream { + fn properties(&self) -> CopyParams { + // avoid the stat syscall since we can be fairly sure it's a socket + CopyParams(FdMeta::Socket, Some(self.as_raw_fd())) + } +} + +impl CopyWrite for UnixStream { + fn properties(&self) -> CopyParams { + // avoid the stat syscall since we can be fairly sure it's a socket + CopyParams(FdMeta::Socket, Some(self.as_raw_fd())) + } +} + +impl CopyWrite for &UnixStream { + fn properties(&self) -> CopyParams { + // avoid the stat syscall since we can be fairly sure it's a socket + CopyParams(FdMeta::Socket, Some(self.as_raw_fd())) + } +} + +impl CopyWrite for ChildStdin { + fn properties(&self) -> CopyParams { + CopyParams(FdMeta::Pipe, Some(self.as_raw_fd())) + } +} + +impl CopyRead for ChildStdout { + fn properties(&self) -> CopyParams { + CopyParams(FdMeta::Pipe, Some(self.as_raw_fd())) + } +} + +impl CopyRead for ChildStderr { + fn properties(&self) -> CopyParams { + CopyParams(FdMeta::Pipe, Some(self.as_raw_fd())) + } +} + +impl CopyRead for StdinLock<'_> { + fn drain_to(&mut self, writer: &mut W, outer_limit: u64) -> Result { + let buf_reader = self.as_mut_buf(); + let buf = buf_reader.buffer(); + let buf = &buf[0..min(buf.len(), outer_limit.try_into().unwrap_or(usize::MAX))]; + let bytes_drained = buf.len(); + writer.write_all(buf)?; + buf_reader.consume(bytes_drained); + + Ok(bytes_drained as u64) + } + + fn properties(&self) -> CopyParams { + CopyParams(fd_to_meta(self), Some(self.as_raw_fd())) + } +} + +impl CopyWrite for StdoutLock<'_> { + fn properties(&self) -> CopyParams { + CopyParams(FdMeta::NoneObtained, Some(self.as_raw_fd())) + } +} + +impl CopyWrite for StderrLock<'_> { + fn properties(&self) -> CopyParams { + CopyParams(FdMeta::NoneObtained, Some(self.as_raw_fd())) + } +} + +impl CopyRead for Take { + fn drain_to(&mut self, writer: &mut W, outer_limit: u64) -> Result { + let local_limit = self.limit(); + let combined_limit = min(outer_limit, local_limit); + let bytes_drained = self.get_mut().drain_to(writer, combined_limit)?; + // update limit since read() was bypassed + self.set_limit(local_limit - bytes_drained); + + Ok(bytes_drained) + } + + fn taken(&mut self, bytes: u64) { + self.set_limit(self.limit() - bytes); + self.get_mut().taken(bytes); + } + + fn min_limit(&self) -> u64 { + min(Take::limit(self), self.get_ref().min_limit()) + } + + fn properties(&self) -> CopyParams { + self.get_ref().properties() + } +} + +impl CopyRead for BufReader { + fn drain_to(&mut self, writer: &mut W, outer_limit: u64) -> Result { + let buf = self.buffer(); + let buf = &buf[0..min(buf.len(), outer_limit.try_into().unwrap_or(usize::MAX))]; + let bytes = buf.len(); + writer.write_all(buf)?; + self.consume(bytes); + + let remaining = outer_limit - bytes as u64; + + // in case of nested bufreaders we also need to drain the ones closer to the source + let inner_bytes = self.get_mut().drain_to(writer, remaining)?; + + Ok(bytes as u64 + inner_bytes) + } + + fn taken(&mut self, bytes: u64) { + self.get_mut().taken(bytes); + } + + fn min_limit(&self) -> u64 { + self.get_ref().min_limit() + } + + fn properties(&self) -> CopyParams { + self.get_ref().properties() + } +} + +impl CopyWrite for BufWriter { + fn properties(&self) -> CopyParams { + self.get_ref().properties() + } +} + +fn fd_to_meta(fd: &T) -> FdMeta { + let fd = fd.as_raw_fd(); + let file: ManuallyDrop = ManuallyDrop::new(unsafe { File::from_raw_fd(fd) }); + match file.metadata() { + Ok(meta) => FdMeta::Metadata(meta), + Err(_) => FdMeta::NoneObtained, + } +} + +pub(super) enum CopyResult { + Ended(u64), + Error(Error, u64), + Fallback(u64), +} + +impl CopyResult { + fn update_take(&self, reader: &mut impl CopyRead) { + match *self { + CopyResult::Fallback(bytes) + | CopyResult::Ended(bytes) + | CopyResult::Error(_, bytes) => reader.taken(bytes), + } + } +} + +/// Invalid file descriptor. +/// +/// Valid file descriptors are guaranteed to be positive numbers (see `open()` manpage) +/// while negative values are used to indicate errors. +/// Thus -1 will never be overlap with a valid open file. +const INVALID_FD: RawFd = -1; + +/// Linux-specific implementation that will attempt to use copy_file_range for copy offloading. +/// As the name says, it only works on regular files. +/// +/// Callers must handle fallback to a generic copy loop. +/// `Fallback` may indicate non-zero number of bytes already written +/// if one of the files' cursor +`max_len` would exceed u64::MAX (`EOVERFLOW`). +pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) -> CopyResult { + use crate::cmp; + + const NOT_PROBED: u8 = 0; + const UNAVAILABLE: u8 = 1; + const AVAILABLE: u8 = 2; + + // Kernel prior to 4.5 don't have copy_file_range + // We store the availability in a global to avoid unnecessary syscalls + static HAS_COPY_FILE_RANGE: AtomicU8 = AtomicU8::new(NOT_PROBED); + + syscall! { + fn copy_file_range( + fd_in: libc::c_int, + off_in: *mut libc::loff_t, + fd_out: libc::c_int, + off_out: *mut libc::loff_t, + len: libc::size_t, + flags: libc::c_uint + ) -> libc::ssize_t + } + + match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) { + NOT_PROBED => { + // EPERM can indicate seccomp filters or an immutable file. + // To distinguish these cases we probe with invalid file descriptors which should result in EBADF if the syscall is supported + // and some other error (ENOSYS or EPERM) if it's not available + let result = unsafe { + cvt(copy_file_range(INVALID_FD, ptr::null_mut(), INVALID_FD, ptr::null_mut(), 1, 0)) + }; + + if matches!(result.map_err(|e| e.raw_os_error()), Err(Some(EBADF))) { + HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed); + } else { + HAS_COPY_FILE_RANGE.store(UNAVAILABLE, Ordering::Relaxed); + return CopyResult::Fallback(0); + } + } + UNAVAILABLE => return CopyResult::Fallback(0), + _ => {} + }; + + let mut written = 0u64; + while written < max_len { + let bytes_to_copy = cmp::min(max_len - written, usize::MAX as u64); + // cap to 1GB chunks in case u64::MAX is passed as max_len and the file has a non-zero seek position + // this allows us to copy large chunks without hitting EOVERFLOW, + // unless someone sets a file offset close to u64::MAX - 1GB, in which case a fallback would be required + let bytes_to_copy = cmp::min(bytes_to_copy as usize, 0x4000_0000usize); + let copy_result = unsafe { + // We actually don't have to adjust the offsets, + // because copy_file_range adjusts the file offset automatically + cvt(copy_file_range(reader, ptr::null_mut(), writer, ptr::null_mut(), bytes_to_copy, 0)) + }; + + match copy_result { + Ok(0) if written == 0 => { + // fallback to work around several kernel bugs where copy_file_range will fail to + // copy any bytes and return 0 instead of an error if + // - reading virtual files from the proc filesystem which appear to have 0 size + // but are not empty. noted in coreutils to affect kernels at least up to 5.6.19. + // - copying from an overlay filesystem in docker. reported to occur on fedora 32. + return CopyResult::Fallback(0); + } + Ok(0) => return CopyResult::Ended(written), // reached EOF + Ok(ret) => written += ret as u64, + Err(err) => { + return match err.raw_os_error() { + // when file offset + max_length > u64::MAX + Some(EOVERFLOW) => CopyResult::Fallback(written), + Some(ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF) if written == 0 => { + // Try fallback io::copy if either: + // - Kernel version is < 4.5 (ENOSYS¹) + // - Files are mounted on different fs (EXDEV) + // - copy_file_range is broken in various ways on RHEL/CentOS 7 (EOPNOTSUPP) + // - copy_file_range file is immutable or syscall is blocked by seccomp¹ (EPERM) + // - copy_file_range cannot be used with pipes or device nodes (EINVAL) + // - the writer fd was opened with O_APPEND (EBADF²) + // and no bytes were written successfully yet. (All these errnos should + // not be returned if something was already written, but they happen in + // the wild, see #91152.) + // + // ¹ these cases should be detected by the initial probe but we handle them here + // anyway in case syscall interception changes during runtime + // ² actually invalid file descriptors would cause this too, but in that case + // the fallback code path is expected to encounter the same error again + CopyResult::Fallback(0) + } + _ => CopyResult::Error(err, written), + }; + } + } + } + CopyResult::Ended(written) +} + +#[derive(PartialEq)] +enum SpliceMode { + Sendfile, + Splice, +} + +/// performs splice or sendfile between file descriptors +/// Does _not_ fall back to a generic copy loop. +fn sendfile_splice(mode: SpliceMode, reader: RawFd, writer: RawFd, len: u64) -> CopyResult { + static HAS_SENDFILE: AtomicBool = AtomicBool::new(true); + static HAS_SPLICE: AtomicBool = AtomicBool::new(true); + + // Android builds use feature level 14, but the libc wrapper for splice is + // gated on feature level 21+, so we have to invoke the syscall directly. + #[cfg(target_os = "android")] + syscall! { + fn splice( + srcfd: libc::c_int, + src_offset: *const i64, + dstfd: libc::c_int, + dst_offset: *const i64, + len: libc::size_t, + flags: libc::c_int + ) -> libc::ssize_t + } + + #[cfg(target_os = "linux")] + use libc::splice; + + match mode { + SpliceMode::Sendfile if !HAS_SENDFILE.load(Ordering::Relaxed) => { + return CopyResult::Fallback(0); + } + SpliceMode::Splice if !HAS_SPLICE.load(Ordering::Relaxed) => { + return CopyResult::Fallback(0); + } + _ => (), + } + + let mut written = 0u64; + while written < len { + // according to its manpage that's the maximum size sendfile() will copy per invocation + let chunk_size = crate::cmp::min(len - written, 0x7ffff000_u64) as usize; + + let result = match mode { + SpliceMode::Sendfile => { + cvt(unsafe { sendfile64(writer, reader, ptr::null_mut(), chunk_size) }) + } + SpliceMode::Splice => cvt(unsafe { + splice(reader, ptr::null_mut(), writer, ptr::null_mut(), chunk_size, 0) + }), + }; + + match result { + Ok(0) => break, // EOF + Ok(ret) => written += ret as u64, + Err(err) => { + return match err.raw_os_error() { + Some(ENOSYS | EPERM) => { + // syscall not supported (ENOSYS) + // syscall is disallowed, e.g. by seccomp (EPERM) + match mode { + SpliceMode::Sendfile => HAS_SENDFILE.store(false, Ordering::Relaxed), + SpliceMode::Splice => HAS_SPLICE.store(false, Ordering::Relaxed), + } + assert_eq!(written, 0); + CopyResult::Fallback(0) + } + Some(EINVAL) => { + // splice/sendfile do not support this particular file descriptor (EINVAL) + assert_eq!(written, 0); + CopyResult::Fallback(0) + } + Some(os_err) if mode == SpliceMode::Sendfile && os_err == EOVERFLOW => { + CopyResult::Fallback(written) + } + _ => CopyResult::Error(err, written), + }; + } + } + } + CopyResult::Ended(written) +} diff --git a/crux-mir/lib/std/src/sys/unix/kernel_copy/tests.rs b/crux-mir/lib/std/src/sys/unix/kernel_copy/tests.rs new file mode 100644 index 000000000..3fe849e23 --- /dev/null +++ b/crux-mir/lib/std/src/sys/unix/kernel_copy/tests.rs @@ -0,0 +1,270 @@ +use crate::fs::OpenOptions; +use crate::io; +use crate::io::Result; +use crate::io::SeekFrom; +use crate::io::{BufRead, Read, Seek, Write}; +use crate::os::unix::io::AsRawFd; +use crate::sys_common::io::test::tmpdir; + +#[test] +fn copy_specialization() -> Result<()> { + use crate::io::{BufReader, BufWriter}; + + let tmp_path = tmpdir(); + let source_path = tmp_path.join("copy-spec.source"); + let sink_path = tmp_path.join("copy-spec.sink"); + + let result: Result<()> = try { + let mut source = crate::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(&source_path)?; + source.write_all(b"abcdefghiklmnopqr")?; + source.seek(SeekFrom::Start(8))?; + let mut source = BufReader::with_capacity(8, source.take(5)); + source.fill_buf()?; + assert_eq!(source.buffer(), b"iklmn"); + source.get_mut().set_limit(6); + source.get_mut().get_mut().seek(SeekFrom::Start(1))?; // "bcdefg" + let mut source = source.take(10); // "iklmnbcdef" + + let mut sink = crate::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(&sink_path)?; + sink.write_all(b"000000")?; + let mut sink = BufWriter::with_capacity(5, sink); + sink.write_all(b"wxyz")?; + assert_eq!(sink.buffer(), b"wxyz"); + + let copied = crate::io::copy(&mut source, &mut sink)?; + assert_eq!(copied, 10, "copy obeyed limit imposed by Take"); + assert_eq!(sink.buffer().len(), 0, "sink buffer was flushed"); + assert_eq!(source.limit(), 0, "outer Take was exhausted"); + assert_eq!(source.get_ref().buffer().len(), 0, "source buffer should be drained"); + assert_eq!( + source.get_ref().get_ref().limit(), + 1, + "inner Take allowed reading beyond end of file, some bytes should be left" + ); + + let mut sink = sink.into_inner()?; + sink.seek(SeekFrom::Start(0))?; + let mut copied = Vec::new(); + sink.read_to_end(&mut copied)?; + assert_eq!(&copied, b"000000wxyziklmnbcdef"); + }; + + let rm1 = crate::fs::remove_file(source_path); + let rm2 = crate::fs::remove_file(sink_path); + + result.and(rm1).and(rm2) +} + +#[test] +fn copies_append_mode_sink() -> Result<()> { + let tmp_path = tmpdir(); + let source_path = tmp_path.join("copies_append_mode.source"); + let sink_path = tmp_path.join("copies_append_mode.sink"); + let mut source = + OpenOptions::new().create(true).truncate(true).write(true).read(true).open(&source_path)?; + write!(source, "not empty")?; + source.seek(SeekFrom::Start(0))?; + let mut sink = OpenOptions::new().create(true).append(true).open(&sink_path)?; + + let copied = crate::io::copy(&mut source, &mut sink)?; + + assert_eq!(copied, 9); + + Ok(()) +} + +#[bench] +fn bench_file_to_file_copy(b: &mut test::Bencher) { + const BYTES: usize = 128 * 1024; + let temp_path = tmpdir(); + let src_path = temp_path.join("file-copy-bench-src"); + let mut src = crate::fs::OpenOptions::new() + .create(true) + .truncate(true) + .read(true) + .write(true) + .open(src_path) + .unwrap(); + src.write(&vec![0u8; BYTES]).unwrap(); + + let sink_path = temp_path.join("file-copy-bench-sink"); + let mut sink = crate::fs::OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(sink_path) + .unwrap(); + + b.bytes = BYTES as u64; + b.iter(|| { + src.seek(SeekFrom::Start(0)).unwrap(); + sink.seek(SeekFrom::Start(0)).unwrap(); + assert_eq!(BYTES as u64, io::copy(&mut src, &mut sink).unwrap()); + }); +} + +#[bench] +fn bench_file_to_socket_copy(b: &mut test::Bencher) { + const BYTES: usize = 128 * 1024; + let temp_path = tmpdir(); + let src_path = temp_path.join("pipe-copy-bench-src"); + let mut src = OpenOptions::new() + .create(true) + .truncate(true) + .read(true) + .write(true) + .open(src_path) + .unwrap(); + src.write(&vec![0u8; BYTES]).unwrap(); + + let sink_drainer = crate::net::TcpListener::bind("localhost:0").unwrap(); + let mut sink = crate::net::TcpStream::connect(sink_drainer.local_addr().unwrap()).unwrap(); + let mut sink_drainer = sink_drainer.accept().unwrap().0; + + crate::thread::spawn(move || { + let mut sink_buf = vec![0u8; 1024 * 1024]; + loop { + sink_drainer.read(&mut sink_buf[..]).unwrap(); + } + }); + + b.bytes = BYTES as u64; + b.iter(|| { + src.seek(SeekFrom::Start(0)).unwrap(); + assert_eq!(BYTES as u64, io::copy(&mut src, &mut sink).unwrap()); + }); +} + +#[bench] +fn bench_file_to_uds_copy(b: &mut test::Bencher) { + const BYTES: usize = 128 * 1024; + let temp_path = tmpdir(); + let src_path = temp_path.join("uds-copy-bench-src"); + let mut src = OpenOptions::new() + .create(true) + .truncate(true) + .read(true) + .write(true) + .open(src_path) + .unwrap(); + src.write(&vec![0u8; BYTES]).unwrap(); + + let (mut sink, mut sink_drainer) = crate::os::unix::net::UnixStream::pair().unwrap(); + + crate::thread::spawn(move || { + let mut sink_buf = vec![0u8; 1024 * 1024]; + loop { + sink_drainer.read(&mut sink_buf[..]).unwrap(); + } + }); + + b.bytes = BYTES as u64; + b.iter(|| { + src.seek(SeekFrom::Start(0)).unwrap(); + assert_eq!(BYTES as u64, io::copy(&mut src, &mut sink).unwrap()); + }); +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +#[bench] +fn bench_socket_pipe_socket_copy(b: &mut test::Bencher) { + use super::CopyResult; + use crate::io::ErrorKind; + use crate::process::{ChildStdin, ChildStdout}; + use crate::sys_common::FromInner; + + let (read_end, write_end) = crate::sys::pipe::anon_pipe().unwrap(); + + let mut read_end = ChildStdout::from_inner(read_end); + let write_end = ChildStdin::from_inner(write_end); + + let acceptor = crate::net::TcpListener::bind("localhost:0").unwrap(); + let mut remote_end = crate::net::TcpStream::connect(acceptor.local_addr().unwrap()).unwrap(); + + let local_end = crate::sync::Arc::new(acceptor.accept().unwrap().0); + + // the data flow in this benchmark: + // + // socket(tx) local_source + // remote_end (write) +--------> (splice to) + // write_end + // + + // | + // | pipe + // v + // read_end + // remote_end (read) <---------+ (splice to) * + // socket(rx) local_end + // + // * benchmark loop using io::copy + + crate::thread::spawn(move || { + let mut sink_buf = vec![0u8; 1024 * 1024]; + remote_end.set_nonblocking(true).unwrap(); + loop { + match remote_end.write(&mut sink_buf[..]) { + Err(err) if err.kind() == ErrorKind::WouldBlock => {} + Ok(_) => {} + err => { + err.expect("write failed"); + } + }; + match remote_end.read(&mut sink_buf[..]) { + Err(err) if err.kind() == ErrorKind::WouldBlock => {} + Ok(_) => {} + err => { + err.expect("read failed"); + } + }; + } + }); + + // check that splice works, otherwise the benchmark would hang + let probe = super::sendfile_splice( + super::SpliceMode::Splice, + local_end.as_raw_fd(), + write_end.as_raw_fd(), + 1, + ); + + match probe { + CopyResult::Ended(1) => { + // splice works + } + _ => { + eprintln!("splice failed, skipping benchmark"); + return; + } + } + + let local_source = local_end.clone(); + crate::thread::spawn(move || { + loop { + super::sendfile_splice( + super::SpliceMode::Splice, + local_source.as_raw_fd(), + write_end.as_raw_fd(), + u64::MAX, + ); + } + }); + + const BYTES: usize = 128 * 1024; + b.bytes = BYTES as u64; + b.iter(|| { + assert_eq!( + BYTES as u64, + io::copy(&mut (&mut read_end).take(BYTES as u64), &mut &*local_end).unwrap() + ); + }); +} diff --git a/crux-mir/lib/std/src/sys/unix/l4re.rs b/crux-mir/lib/std/src/sys/unix/l4re.rs index c6e4f5693..996758893 100644 --- a/crux-mir/lib/std/src/sys/unix/l4re.rs +++ b/crux-mir/lib/std/src/sys/unix/l4re.rs @@ -1,15 +1,18 @@ macro_rules! unimpl { () => { - return Err(io::Error::new(io::ErrorKind::Other, "No networking available on L4Re.")); + return Err(io::const_io_error!( + io::ErrorKind::Unsupported, + "No networking available on L4Re.", + )); }; } pub mod net { #![allow(warnings)] - use crate::convert::TryFrom; use crate::fmt; use crate::io::{self, IoSlice, IoSliceMut}; use crate::net::{Ipv4Addr, Ipv6Addr, Shutdown, SocketAddr}; + use crate::os::unix::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, RawFd}; use crate::sys::fd::FileDesc; use crate::sys_common::{AsInner, FromInner, IntoInner}; use crate::time::Duration; @@ -55,6 +58,10 @@ pub mod net { unimpl!(); } + pub fn is_read_vectored(&self) -> bool { + false + } + pub fn peek(&self, _: &mut [u8]) -> io::Result { unimpl!(); } @@ -75,6 +82,10 @@ pub mod net { unimpl!(); } + pub fn is_write_vectored(&self) -> bool { + false + } + pub fn set_timeout(&self, _: Option, _: libc::c_int) -> io::Result<()> { unimpl!(); } @@ -87,6 +98,14 @@ pub mod net { unimpl!(); } + pub fn set_linger(&self, _: Option) -> io::Result<()> { + unimpl!(); + } + + pub fn linger(&self) -> io::Result> { + unimpl!(); + } + pub fn set_nodelay(&self, _: bool) -> io::Result<()> { unimpl!(); } @@ -102,23 +121,52 @@ pub mod net { pub fn take_error(&self) -> io::Result> { unimpl!(); } + + // This is used by sys_common code to abstract over Windows and Unix. + pub fn as_raw(&self) -> RawFd { + self.as_raw_fd() + } + } + + impl AsInner for Socket { + fn as_inner(&self) -> &FileDesc { + &self.0 + } + } + + impl FromInner for Socket { + fn from_inner(file_desc: FileDesc) -> Socket { + Socket(file_desc) + } } - impl AsInner for Socket { - fn as_inner(&self) -> &libc::c_int { - self.0.as_inner() + impl IntoInner for Socket { + fn into_inner(self) -> FileDesc { + self.0 } } - impl FromInner for Socket { - fn from_inner(fd: libc::c_int) -> Socket { - Socket(FileDesc::new(fd)) + impl AsFd for Socket { + fn as_fd(&self) -> BorrowedFd<'_> { + self.0.as_fd() } } - impl IntoInner for Socket { - fn into_inner(self) -> libc::c_int { - self.0.into_raw() + impl AsRawFd for Socket { + fn as_raw_fd(&self) -> RawFd { + self.0.as_raw_fd() + } + } + + impl IntoRawFd for Socket { + fn into_raw_fd(self) -> RawFd { + self.0.into_raw_fd() + } + } + + impl FromRawFd for Socket { + unsafe fn from_raw_fd(raw_fd: RawFd) -> Self { + Self(FromRawFd::from_raw_fd(raw_fd)) } } @@ -171,6 +219,10 @@ pub mod net { unimpl!(); } + pub fn is_read_vectored(&self) -> bool { + false + } + pub fn write(&self, _: &[u8]) -> io::Result { unimpl!(); } @@ -179,6 +231,10 @@ pub mod net { unimpl!(); } + pub fn is_write_vectored(&self) -> bool { + false + } + pub fn peer_addr(&self) -> io::Result { unimpl!(); } @@ -195,6 +251,14 @@ pub mod net { unimpl!(); } + pub fn set_linger(&self, _: Option) -> io::Result<()> { + unimpl!(); + } + + pub fn linger(&self) -> io::Result> { + unimpl!(); + } + pub fn set_nodelay(&self, _: bool) -> io::Result<()> { unimpl!(); } @@ -462,7 +526,7 @@ pub mod net { impl LookupHost { pub fn port(&self) -> u16 { - unimpl!(); + 0 // unimplemented } } diff --git a/crux-mir/lib/std/src/sys/unix/locks/fuchsia_mutex.rs b/crux-mir/lib/std/src/sys/unix/locks/fuchsia_mutex.rs new file mode 100644 index 000000000..5d89e5a13 --- /dev/null +++ b/crux-mir/lib/std/src/sys/unix/locks/fuchsia_mutex.rs @@ -0,0 +1,164 @@ +//! A priority inheriting mutex for Fuchsia. +//! +//! This is a port of the [mutex in Fuchsia's libsync]. Contrary to the original, +//! it does not abort the process when reentrant locking is detected, but deadlocks. +//! +//! Priority inheritance is achieved by storing the owning thread's handle in an +//! atomic variable. Fuchsia's futex operations support setting an owner thread +//! for a futex, which can boost that thread's priority while the futex is waited +//! upon. +//! +//! libsync is licenced under the following BSD-style licence: +//! +//! Copyright 2016 The Fuchsia Authors. +//! +//! Redistribution and use in source and binary forms, with or without +//! modification, are permitted provided that the following conditions are +//! met: +//! +//! * Redistributions of source code must retain the above copyright +//! notice, this list of conditions and the following disclaimer. +//! * Redistributions in binary form must reproduce the above +//! copyright notice, this list of conditions and the following +//! disclaimer in the documentation and/or other materials provided +//! with the distribution. +//! +//! THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +//! "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +//! LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +//! A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +//! OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +//! SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +//! LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +//! DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +//! THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +//! (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +//! OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +//! +//! [mutex in Fuchsia's libsync]: https://cs.opensource.google/fuchsia/fuchsia/+/main:zircon/system/ulib/sync/mutex.c + +use crate::sync::atomic::{ + AtomicU32, + Ordering::{Acquire, Relaxed, Release}, +}; +use crate::sys::futex::zircon::{ + zx_futex_wait, zx_futex_wake_single_owner, zx_handle_t, zx_thread_self, ZX_ERR_BAD_HANDLE, + ZX_ERR_BAD_STATE, ZX_ERR_INVALID_ARGS, ZX_ERR_TIMED_OUT, ZX_ERR_WRONG_TYPE, ZX_OK, + ZX_TIME_INFINITE, +}; + +// The lowest two bits of a `zx_handle_t` are always set, so the lowest bit is used to mark the +// mutex as contested by clearing it. +const CONTESTED_BIT: u32 = 1; +// This can never be a valid `zx_handle_t`. +const UNLOCKED: u32 = 0; + +pub struct Mutex { + futex: AtomicU32, +} + +#[inline] +fn to_state(owner: zx_handle_t) -> u32 { + owner +} + +#[inline] +fn to_owner(state: u32) -> zx_handle_t { + state | CONTESTED_BIT +} + +#[inline] +fn is_contested(state: u32) -> bool { + state & CONTESTED_BIT == 0 +} + +#[inline] +fn mark_contested(state: u32) -> u32 { + state & !CONTESTED_BIT +} + +impl Mutex { + #[inline] + pub const fn new() -> Mutex { + Mutex { futex: AtomicU32::new(UNLOCKED) } + } + + #[inline] + pub fn try_lock(&self) -> bool { + let thread_self = unsafe { zx_thread_self() }; + self.futex.compare_exchange(UNLOCKED, to_state(thread_self), Acquire, Relaxed).is_ok() + } + + #[inline] + pub fn lock(&self) { + let thread_self = unsafe { zx_thread_self() }; + if let Err(state) = + self.futex.compare_exchange(UNLOCKED, to_state(thread_self), Acquire, Relaxed) + { + unsafe { + self.lock_contested(state, thread_self); + } + } + } + + /// # Safety + /// `thread_self` must be the handle for the current thread. + #[cold] + unsafe fn lock_contested(&self, mut state: u32, thread_self: zx_handle_t) { + let owned_state = mark_contested(to_state(thread_self)); + loop { + // Mark the mutex as contested if it is not already. + let contested = mark_contested(state); + if is_contested(state) + || self.futex.compare_exchange(state, contested, Relaxed, Relaxed).is_ok() + { + // The mutex has been marked as contested, wait for the state to change. + unsafe { + match zx_futex_wait( + &self.futex, + AtomicU32::new(contested), + to_owner(state), + ZX_TIME_INFINITE, + ) { + ZX_OK | ZX_ERR_BAD_STATE | ZX_ERR_TIMED_OUT => (), + // Note that if a thread handle is reused after its associated thread + // exits without unlocking the mutex, an arbitrary thread's priority + // could be boosted by the wait, but there is currently no way to + // prevent that. + ZX_ERR_INVALID_ARGS | ZX_ERR_BAD_HANDLE | ZX_ERR_WRONG_TYPE => { + panic!( + "either the current thread is trying to lock a mutex it has + already locked, or the previous owner did not unlock the mutex + before exiting" + ) + } + error => panic!("unexpected error in zx_futex_wait: {error}"), + } + } + } + + // The state has changed or a wakeup occurred, try to lock the mutex. + match self.futex.compare_exchange(UNLOCKED, owned_state, Acquire, Relaxed) { + Ok(_) => return, + Err(updated) => state = updated, + } + } + } + + #[inline] + pub unsafe fn unlock(&self) { + if is_contested(self.futex.swap(UNLOCKED, Release)) { + // The woken thread will mark the mutex as contested again, + // and return here, waking until there are no waiters left, + // in which case this is a noop. + self.wake(); + } + } + + #[cold] + fn wake(&self) { + unsafe { + zx_futex_wake_single_owner(&self.futex); + } + } +} diff --git a/crux-mir/lib/std/src/sys/unix/locks/futex_condvar.rs b/crux-mir/lib/std/src/sys/unix/locks/futex_condvar.rs new file mode 100644 index 000000000..4bd65dd25 --- /dev/null +++ b/crux-mir/lib/std/src/sys/unix/locks/futex_condvar.rs @@ -0,0 +1,56 @@ +use super::Mutex; +use crate::sync::atomic::{AtomicU32, Ordering::Relaxed}; +use crate::sys::futex::{futex_wait, futex_wake, futex_wake_all}; +use crate::time::Duration; + +pub struct Condvar { + // The value of this atomic is simply incremented on every notification. + // This is used by `.wait()` to not miss any notifications after + // unlocking the mutex and before waiting for notifications. + futex: AtomicU32, +} + +impl Condvar { + #[inline] + pub const fn new() -> Self { + Self { futex: AtomicU32::new(0) } + } + + // All the memory orderings here are `Relaxed`, + // because synchronization is done by unlocking and locking the mutex. + + pub fn notify_one(&self) { + self.futex.fetch_add(1, Relaxed); + futex_wake(&self.futex); + } + + pub fn notify_all(&self) { + self.futex.fetch_add(1, Relaxed); + futex_wake_all(&self.futex); + } + + pub unsafe fn wait(&self, mutex: &Mutex) { + self.wait_optional_timeout(mutex, None); + } + + pub unsafe fn wait_timeout(&self, mutex: &Mutex, timeout: Duration) -> bool { + self.wait_optional_timeout(mutex, Some(timeout)) + } + + unsafe fn wait_optional_timeout(&self, mutex: &Mutex, timeout: Option) -> bool { + // Examine the notification counter _before_ we unlock the mutex. + let futex_value = self.futex.load(Relaxed); + + // Unlock the mutex before going to sleep. + mutex.unlock(); + + // Wait, but only if there hasn't been any + // notification since we unlocked the mutex. + let r = futex_wait(&self.futex, futex_value, timeout); + + // Lock the mutex again. + mutex.lock(); + + r + } +} diff --git a/crux-mir/lib/std/src/sys/unix/locks/futex_mutex.rs b/crux-mir/lib/std/src/sys/unix/locks/futex_mutex.rs new file mode 100644 index 000000000..c01229586 --- /dev/null +++ b/crux-mir/lib/std/src/sys/unix/locks/futex_mutex.rs @@ -0,0 +1,96 @@ +use crate::sync::atomic::{ + AtomicU32, + Ordering::{Acquire, Relaxed, Release}, +}; +use crate::sys::futex::{futex_wait, futex_wake}; + +pub struct Mutex { + /// 0: unlocked + /// 1: locked, no other threads waiting + /// 2: locked, and other threads waiting (contended) + futex: AtomicU32, +} + +impl Mutex { + #[inline] + pub const fn new() -> Self { + Self { futex: AtomicU32::new(0) } + } + + #[inline] + pub fn try_lock(&self) -> bool { + self.futex.compare_exchange(0, 1, Acquire, Relaxed).is_ok() + } + + #[inline] + pub fn lock(&self) { + if self.futex.compare_exchange(0, 1, Acquire, Relaxed).is_err() { + self.lock_contended(); + } + } + + #[cold] + fn lock_contended(&self) { + // Spin first to speed things up if the lock is released quickly. + let mut state = self.spin(); + + // If it's unlocked now, attempt to take the lock + // without marking it as contended. + if state == 0 { + match self.futex.compare_exchange(0, 1, Acquire, Relaxed) { + Ok(_) => return, // Locked! + Err(s) => state = s, + } + } + + loop { + // Put the lock in contended state. + // We avoid an unnecessary write if it as already set to 2, + // to be friendlier for the caches. + if state != 2 && self.futex.swap(2, Acquire) == 0 { + // We changed it from 0 to 2, so we just successfully locked it. + return; + } + + // Wait for the futex to change state, assuming it is still 2. + futex_wait(&self.futex, 2, None); + + // Spin again after waking up. + state = self.spin(); + } + } + + fn spin(&self) -> u32 { + let mut spin = 100; + loop { + // We only use `load` (and not `swap` or `compare_exchange`) + // while spinning, to be easier on the caches. + let state = self.futex.load(Relaxed); + + // We stop spinning when the mutex is unlocked (0), + // but also when it's contended (2). + if state != 1 || spin == 0 { + return state; + } + + crate::hint::spin_loop(); + spin -= 1; + } + } + + #[inline] + pub unsafe fn unlock(&self) { + if self.futex.swap(0, Release) == 2 { + // We only wake up one thread. When that thread locks the mutex, it + // will mark the mutex as contended (2) (see lock_contended above), + // which makes sure that any other waiting threads will also be + // woken up eventually. + self.wake(); + } + } + + #[cold] + fn wake(&self) { + futex_wake(&self.futex); + } +} diff --git a/crux-mir/lib/std/src/sys/unix/locks/futex_rwlock.rs b/crux-mir/lib/std/src/sys/unix/locks/futex_rwlock.rs new file mode 100644 index 000000000..aa0de9002 --- /dev/null +++ b/crux-mir/lib/std/src/sys/unix/locks/futex_rwlock.rs @@ -0,0 +1,320 @@ +use crate::sync::atomic::{ + AtomicU32, + Ordering::{Acquire, Relaxed, Release}, +}; +use crate::sys::futex::{futex_wait, futex_wake, futex_wake_all}; + +pub struct RwLock { + // The state consists of a 30-bit reader counter, a 'readers waiting' flag, and a 'writers waiting' flag. + // Bits 0..30: + // 0: Unlocked + // 1..=0x3FFF_FFFE: Locked by N readers + // 0x3FFF_FFFF: Write locked + // Bit 30: Readers are waiting on this futex. + // Bit 31: Writers are waiting on the writer_notify futex. + state: AtomicU32, + // The 'condition variable' to notify writers through. + // Incremented on every signal. + writer_notify: AtomicU32, +} + +const READ_LOCKED: u32 = 1; +const MASK: u32 = (1 << 30) - 1; +const WRITE_LOCKED: u32 = MASK; +const MAX_READERS: u32 = MASK - 1; +const READERS_WAITING: u32 = 1 << 30; +const WRITERS_WAITING: u32 = 1 << 31; + +#[inline] +fn is_unlocked(state: u32) -> bool { + state & MASK == 0 +} + +#[inline] +fn is_write_locked(state: u32) -> bool { + state & MASK == WRITE_LOCKED +} + +#[inline] +fn has_readers_waiting(state: u32) -> bool { + state & READERS_WAITING != 0 +} + +#[inline] +fn has_writers_waiting(state: u32) -> bool { + state & WRITERS_WAITING != 0 +} + +#[inline] +fn is_read_lockable(state: u32) -> bool { + // This also returns false if the counter could overflow if we tried to read lock it. + // + // We don't allow read-locking if there's readers waiting, even if the lock is unlocked + // and there's no writers waiting. The only situation when this happens is after unlocking, + // at which point the unlocking thread might be waking up writers, which have priority over readers. + // The unlocking thread will clear the readers waiting bit and wake up readers, if necessary. + state & MASK < MAX_READERS && !has_readers_waiting(state) && !has_writers_waiting(state) +} + +#[inline] +fn has_reached_max_readers(state: u32) -> bool { + state & MASK == MAX_READERS +} + +impl RwLock { + #[inline] + pub const fn new() -> Self { + Self { state: AtomicU32::new(0), writer_notify: AtomicU32::new(0) } + } + + #[inline] + pub fn try_read(&self) -> bool { + self.state + .fetch_update(Acquire, Relaxed, |s| is_read_lockable(s).then(|| s + READ_LOCKED)) + .is_ok() + } + + #[inline] + pub fn read(&self) { + let state = self.state.load(Relaxed); + if !is_read_lockable(state) + || self + .state + .compare_exchange_weak(state, state + READ_LOCKED, Acquire, Relaxed) + .is_err() + { + self.read_contended(); + } + } + + #[inline] + pub unsafe fn read_unlock(&self) { + let state = self.state.fetch_sub(READ_LOCKED, Release) - READ_LOCKED; + + // It's impossible for a reader to be waiting on a read-locked RwLock, + // except if there is also a writer waiting. + debug_assert!(!has_readers_waiting(state) || has_writers_waiting(state)); + + // Wake up a writer if we were the last reader and there's a writer waiting. + if is_unlocked(state) && has_writers_waiting(state) { + self.wake_writer_or_readers(state); + } + } + + #[cold] + fn read_contended(&self) { + let mut state = self.spin_read(); + + loop { + // If we can lock it, lock it. + if is_read_lockable(state) { + match self.state.compare_exchange_weak(state, state + READ_LOCKED, Acquire, Relaxed) + { + Ok(_) => return, // Locked! + Err(s) => { + state = s; + continue; + } + } + } + + // Check for overflow. + if has_reached_max_readers(state) { + panic!("too many active read locks on RwLock"); + } + + // Make sure the readers waiting bit is set before we go to sleep. + if !has_readers_waiting(state) { + if let Err(s) = + self.state.compare_exchange(state, state | READERS_WAITING, Relaxed, Relaxed) + { + state = s; + continue; + } + } + + // Wait for the state to change. + futex_wait(&self.state, state | READERS_WAITING, None); + + // Spin again after waking up. + state = self.spin_read(); + } + } + + #[inline] + pub fn try_write(&self) -> bool { + self.state + .fetch_update(Acquire, Relaxed, |s| is_unlocked(s).then(|| s + WRITE_LOCKED)) + .is_ok() + } + + #[inline] + pub fn write(&self) { + if self.state.compare_exchange_weak(0, WRITE_LOCKED, Acquire, Relaxed).is_err() { + self.write_contended(); + } + } + + #[inline] + pub unsafe fn write_unlock(&self) { + let state = self.state.fetch_sub(WRITE_LOCKED, Release) - WRITE_LOCKED; + + debug_assert!(is_unlocked(state)); + + if has_writers_waiting(state) || has_readers_waiting(state) { + self.wake_writer_or_readers(state); + } + } + + #[cold] + fn write_contended(&self) { + let mut state = self.spin_write(); + + let mut other_writers_waiting = 0; + + loop { + // If it's unlocked, we try to lock it. + if is_unlocked(state) { + match self.state.compare_exchange_weak( + state, + state | WRITE_LOCKED | other_writers_waiting, + Acquire, + Relaxed, + ) { + Ok(_) => return, // Locked! + Err(s) => { + state = s; + continue; + } + } + } + + // Set the waiting bit indicating that we're waiting on it. + if !has_writers_waiting(state) { + if let Err(s) = + self.state.compare_exchange(state, state | WRITERS_WAITING, Relaxed, Relaxed) + { + state = s; + continue; + } + } + + // Other writers might be waiting now too, so we should make sure + // we keep that bit on once we manage lock it. + other_writers_waiting = WRITERS_WAITING; + + // Examine the notification counter before we check if `state` has changed, + // to make sure we don't miss any notifications. + let seq = self.writer_notify.load(Acquire); + + // Don't go to sleep if the lock has become available, + // or if the writers waiting bit is no longer set. + state = self.state.load(Relaxed); + if is_unlocked(state) || !has_writers_waiting(state) { + continue; + } + + // Wait for the state to change. + futex_wait(&self.writer_notify, seq, None); + + // Spin again after waking up. + state = self.spin_write(); + } + } + + /// Wake up waiting threads after unlocking. + /// + /// If both are waiting, this will wake up only one writer, but will fall + /// back to waking up readers if there was no writer to wake up. + #[cold] + fn wake_writer_or_readers(&self, mut state: u32) { + assert!(is_unlocked(state)); + + // The readers waiting bit might be turned on at any point now, + // since readers will block when there's anything waiting. + // Writers will just lock the lock though, regardless of the waiting bits, + // so we don't have to worry about the writer waiting bit. + // + // If the lock gets locked in the meantime, we don't have to do + // anything, because then the thread that locked the lock will take + // care of waking up waiters when it unlocks. + + // If only writers are waiting, wake one of them up. + if state == WRITERS_WAITING { + match self.state.compare_exchange(state, 0, Relaxed, Relaxed) { + Ok(_) => { + self.wake_writer(); + return; + } + Err(s) => { + // Maybe some readers are now waiting too. So, continue to the next `if`. + state = s; + } + } + } + + // If both writers and readers are waiting, leave the readers waiting + // and only wake up one writer. + if state == READERS_WAITING + WRITERS_WAITING { + if self.state.compare_exchange(state, READERS_WAITING, Relaxed, Relaxed).is_err() { + // The lock got locked. Not our problem anymore. + return; + } + if self.wake_writer() { + return; + } + // No writers were actually blocked on futex_wait, so we continue + // to wake up readers instead, since we can't be sure if we notified a writer. + state = READERS_WAITING; + } + + // If readers are waiting, wake them all up. + if state == READERS_WAITING { + if self.state.compare_exchange(state, 0, Relaxed, Relaxed).is_ok() { + futex_wake_all(&self.state); + } + } + } + + /// This wakes one writer and returns true if we woke up a writer that was + /// blocked on futex_wait. + /// + /// If this returns false, it might still be the case that we notified a + /// writer that was about to go to sleep. + fn wake_writer(&self) -> bool { + self.writer_notify.fetch_add(1, Release); + futex_wake(&self.writer_notify) + // Note that FreeBSD and DragonFlyBSD don't tell us whether they woke + // up any threads or not, and always return `false` here. That still + // results in correct behaviour: it just means readers get woken up as + // well in case both readers and writers were waiting. + } + + /// Spin for a while, but stop directly at the given condition. + #[inline] + fn spin_until(&self, f: impl Fn(u32) -> bool) -> u32 { + let mut spin = 100; // Chosen by fair dice roll. + loop { + let state = self.state.load(Relaxed); + if f(state) || spin == 0 { + return state; + } + crate::hint::spin_loop(); + spin -= 1; + } + } + + #[inline] + fn spin_write(&self) -> u32 { + // Stop spinning when it's unlocked or when there's waiting writers, to keep things somewhat fair. + self.spin_until(|state| is_unlocked(state) || has_writers_waiting(state)) + } + + #[inline] + fn spin_read(&self) -> u32 { + // Stop spinning when it's unlocked or read locked, or when there's waiting threads. + self.spin_until(|state| { + !is_write_locked(state) || has_readers_waiting(state) || has_writers_waiting(state) + }) + } +} diff --git a/crux-mir/lib/std/src/sys/unix/locks/mod.rs b/crux-mir/lib/std/src/sys/unix/locks/mod.rs new file mode 100644 index 000000000..b2e0e49ad --- /dev/null +++ b/crux-mir/lib/std/src/sys/unix/locks/mod.rs @@ -0,0 +1,31 @@ +cfg_if::cfg_if! { + if #[cfg(any( + target_os = "linux", + target_os = "android", + all(target_os = "emscripten", target_feature = "atomics"), + target_os = "freebsd", + target_os = "openbsd", + target_os = "dragonfly", + ))] { + mod futex_mutex; + mod futex_rwlock; + mod futex_condvar; + pub(crate) use futex_mutex::Mutex; + pub(crate) use futex_rwlock::RwLock; + pub(crate) use futex_condvar::Condvar; + } else if #[cfg(target_os = "fuchsia")] { + mod fuchsia_mutex; + mod futex_rwlock; + mod futex_condvar; + pub(crate) use fuchsia_mutex::Mutex; + pub(crate) use futex_rwlock::RwLock; + pub(crate) use futex_condvar::Condvar; + } else { + mod pthread_mutex; + mod pthread_rwlock; + mod pthread_condvar; + pub(crate) use pthread_mutex::Mutex; + pub(crate) use pthread_rwlock::RwLock; + pub(crate) use pthread_condvar::Condvar; + } +} diff --git a/crux-mir/lib/std/src/sys/unix/locks/pthread_condvar.rs b/crux-mir/lib/std/src/sys/unix/locks/pthread_condvar.rs new file mode 100644 index 000000000..6be1abc2b --- /dev/null +++ b/crux-mir/lib/std/src/sys/unix/locks/pthread_condvar.rs @@ -0,0 +1,192 @@ +use crate::cell::UnsafeCell; +use crate::ptr; +use crate::sync::atomic::{AtomicPtr, Ordering::Relaxed}; +use crate::sys::locks::{pthread_mutex, Mutex}; +use crate::sys::time::TIMESPEC_MAX; +use crate::sys_common::lazy_box::{LazyBox, LazyInit}; +use crate::time::Duration; + +struct AllocatedCondvar(UnsafeCell); + +pub struct Condvar { + inner: LazyBox, + mutex: AtomicPtr, +} + +#[inline] +fn raw(c: &Condvar) -> *mut libc::pthread_cond_t { + c.inner.0.get() +} + +unsafe impl Send for AllocatedCondvar {} +unsafe impl Sync for AllocatedCondvar {} + +impl LazyInit for AllocatedCondvar { + fn init() -> Box { + let condvar = Box::new(AllocatedCondvar(UnsafeCell::new(libc::PTHREAD_COND_INITIALIZER))); + + cfg_if::cfg_if! { + if #[cfg(any( + target_os = "macos", + target_os = "ios", + target_os = "watchos", + target_os = "l4re", + target_os = "android", + target_os = "redox" + ))] { + // `pthread_condattr_setclock` is unfortunately not supported on these platforms. + } else if #[cfg(any(target_os = "espidf", target_os = "horizon"))] { + // NOTE: ESP-IDF's PTHREAD_COND_INITIALIZER support is not released yet + // So on that platform, init() should always be called + // Moreover, that platform does not have pthread_condattr_setclock support, + // hence that initialization should be skipped as well + // + // Similar story for the 3DS (horizon). + let r = unsafe { libc::pthread_cond_init(condvar.0.get(), crate::ptr::null()) }; + assert_eq!(r, 0); + } else { + use crate::mem::MaybeUninit; + let mut attr = MaybeUninit::::uninit(); + let r = unsafe { libc::pthread_condattr_init(attr.as_mut_ptr()) }; + assert_eq!(r, 0); + let r = unsafe { libc::pthread_condattr_setclock(attr.as_mut_ptr(), libc::CLOCK_MONOTONIC) }; + assert_eq!(r, 0); + let r = unsafe { libc::pthread_cond_init(condvar.0.get(), attr.as_ptr()) }; + assert_eq!(r, 0); + let r = unsafe { libc::pthread_condattr_destroy(attr.as_mut_ptr()) }; + assert_eq!(r, 0); + } + } + + condvar + } +} + +impl Drop for AllocatedCondvar { + #[inline] + fn drop(&mut self) { + let r = unsafe { libc::pthread_cond_destroy(self.0.get()) }; + if cfg!(target_os = "dragonfly") { + // On DragonFly pthread_cond_destroy() returns EINVAL if called on + // a condvar that was just initialized with + // libc::PTHREAD_COND_INITIALIZER. Once it is used or + // pthread_cond_init() is called, this behaviour no longer occurs. + debug_assert!(r == 0 || r == libc::EINVAL); + } else { + debug_assert_eq!(r, 0); + } + } +} + +impl Condvar { + pub const fn new() -> Condvar { + Condvar { inner: LazyBox::new(), mutex: AtomicPtr::new(ptr::null_mut()) } + } + + #[inline] + fn verify(&self, mutex: *mut libc::pthread_mutex_t) { + // Relaxed is okay here because we never read through `self.addr`, and only use it to + // compare addresses. + match self.mutex.compare_exchange(ptr::null_mut(), mutex, Relaxed, Relaxed) { + Ok(_) => {} // Stored the address + Err(n) if n == mutex => {} // Lost a race to store the same address + _ => panic!("attempted to use a condition variable with two mutexes"), + } + } + + #[inline] + pub fn notify_one(&self) { + let r = unsafe { libc::pthread_cond_signal(raw(self)) }; + debug_assert_eq!(r, 0); + } + + #[inline] + pub fn notify_all(&self) { + let r = unsafe { libc::pthread_cond_broadcast(raw(self)) }; + debug_assert_eq!(r, 0); + } + + #[inline] + pub unsafe fn wait(&self, mutex: &Mutex) { + let mutex = pthread_mutex::raw(mutex); + self.verify(mutex); + let r = libc::pthread_cond_wait(raw(self), mutex); + debug_assert_eq!(r, 0); + } + + // This implementation is used on systems that support pthread_condattr_setclock + // where we configure condition variable to use monotonic clock (instead of + // default system clock). This approach avoids all problems that result + // from changes made to the system time. + #[cfg(not(any( + target_os = "macos", + target_os = "ios", + target_os = "watchos", + target_os = "android", + target_os = "espidf", + target_os = "horizon" + )))] + pub unsafe fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool { + use crate::sys::time::Timespec; + + let mutex = pthread_mutex::raw(mutex); + self.verify(mutex); + + let timeout = Timespec::now(libc::CLOCK_MONOTONIC) + .checked_add_duration(&dur) + .and_then(|t| t.to_timespec()) + .unwrap_or(TIMESPEC_MAX); + let r = libc::pthread_cond_timedwait(raw(self), mutex, &timeout); + assert!(r == libc::ETIMEDOUT || r == 0); + r == 0 + } + + // This implementation is modeled after libcxx's condition_variable + // https://github.com/llvm-mirror/libcxx/blob/release_35/src/condition_variable.cpp#L46 + // https://github.com/llvm-mirror/libcxx/blob/release_35/include/__mutex_base#L367 + #[cfg(any( + target_os = "macos", + target_os = "ios", + target_os = "watchos", + target_os = "android", + target_os = "espidf", + target_os = "horizon" + ))] + pub unsafe fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool { + use crate::sys::time::SystemTime; + use crate::time::Instant; + + let mutex = pthread_mutex::raw(mutex); + self.verify(mutex); + + // OSX implementation of `pthread_cond_timedwait` is buggy + // with super long durations. When duration is greater than + // 0x100_0000_0000_0000 seconds, `pthread_cond_timedwait` + // in macOS Sierra returns error 316. + // + // This program demonstrates the issue: + // https://gist.github.com/stepancheg/198db4623a20aad2ad7cddb8fda4a63c + // + // To work around this issue, and possible bugs of other OSes, timeout + // is clamped to 1000 years, which is allowable per the API of `wait_timeout` + // because of spurious wakeups. + let dur = Duration::min(dur, Duration::from_secs(1000 * 365 * 86400)); + + // pthread_cond_timedwait uses system time, but we want to report timeout + // based on stable time. + let now = Instant::now(); + + let timeout = SystemTime::now() + .t + .checked_add_duration(&dur) + .and_then(|t| t.to_timespec()) + .unwrap_or(TIMESPEC_MAX); + + let r = libc::pthread_cond_timedwait(raw(self), mutex, &timeout); + debug_assert!(r == libc::ETIMEDOUT || r == 0); + + // ETIMEDOUT is not a totally reliable method of determining timeout due + // to clock shifts, so do the check ourselves + now.elapsed() < dur + } +} diff --git a/crux-mir/lib/std/src/sys/unix/locks/pthread_mutex.rs b/crux-mir/lib/std/src/sys/unix/locks/pthread_mutex.rs new file mode 100644 index 000000000..8a78bc1fd --- /dev/null +++ b/crux-mir/lib/std/src/sys/unix/locks/pthread_mutex.rs @@ -0,0 +1,131 @@ +use crate::cell::UnsafeCell; +use crate::mem::{forget, MaybeUninit}; +use crate::sys::cvt_nz; +use crate::sys_common::lazy_box::{LazyBox, LazyInit}; + +struct AllocatedMutex(UnsafeCell); + +pub struct Mutex { + inner: LazyBox, +} + +#[inline] +pub unsafe fn raw(m: &Mutex) -> *mut libc::pthread_mutex_t { + m.inner.0.get() +} + +unsafe impl Send for AllocatedMutex {} +unsafe impl Sync for AllocatedMutex {} + +impl LazyInit for AllocatedMutex { + fn init() -> Box { + let mutex = Box::new(AllocatedMutex(UnsafeCell::new(libc::PTHREAD_MUTEX_INITIALIZER))); + + // Issue #33770 + // + // A pthread mutex initialized with PTHREAD_MUTEX_INITIALIZER will have + // a type of PTHREAD_MUTEX_DEFAULT, which has undefined behavior if you + // try to re-lock it from the same thread when you already hold a lock + // (https://pubs.opengroup.org/onlinepubs/9699919799/functions/pthread_mutex_init.html). + // This is the case even if PTHREAD_MUTEX_DEFAULT == PTHREAD_MUTEX_NORMAL + // (https://github.com/rust-lang/rust/issues/33770#issuecomment-220847521) -- in that + // case, `pthread_mutexattr_settype(PTHREAD_MUTEX_DEFAULT)` will of course be the same + // as setting it to `PTHREAD_MUTEX_NORMAL`, but not setting any mode will result in + // a Mutex where re-locking is UB. + // + // In practice, glibc takes advantage of this undefined behavior to + // implement hardware lock elision, which uses hardware transactional + // memory to avoid acquiring the lock. While a transaction is in + // progress, the lock appears to be unlocked. This isn't a problem for + // other threads since the transactional memory will abort if a conflict + // is detected, however no abort is generated when re-locking from the + // same thread. + // + // Since locking the same mutex twice will result in two aliasing &mut + // references, we instead create the mutex with type + // PTHREAD_MUTEX_NORMAL which is guaranteed to deadlock if we try to + // re-lock it from the same thread, thus avoiding undefined behavior. + unsafe { + let mut attr = MaybeUninit::::uninit(); + cvt_nz(libc::pthread_mutexattr_init(attr.as_mut_ptr())).unwrap(); + let attr = PthreadMutexAttr(&mut attr); + cvt_nz(libc::pthread_mutexattr_settype( + attr.0.as_mut_ptr(), + libc::PTHREAD_MUTEX_NORMAL, + )) + .unwrap(); + cvt_nz(libc::pthread_mutex_init(mutex.0.get(), attr.0.as_ptr())).unwrap(); + } + + mutex + } + + fn destroy(mutex: Box) { + // We're not allowed to pthread_mutex_destroy a locked mutex, + // so check first if it's unlocked. + if unsafe { libc::pthread_mutex_trylock(mutex.0.get()) == 0 } { + unsafe { libc::pthread_mutex_unlock(mutex.0.get()) }; + drop(mutex); + } else { + // The mutex is locked. This happens if a MutexGuard is leaked. + // In this case, we just leak the Mutex too. + forget(mutex); + } + } + + fn cancel_init(_: Box) { + // In this case, we can just drop it without any checks, + // since it cannot have been locked yet. + } +} + +impl Drop for AllocatedMutex { + #[inline] + fn drop(&mut self) { + let r = unsafe { libc::pthread_mutex_destroy(self.0.get()) }; + if cfg!(target_os = "dragonfly") { + // On DragonFly pthread_mutex_destroy() returns EINVAL if called on a + // mutex that was just initialized with libc::PTHREAD_MUTEX_INITIALIZER. + // Once it is used (locked/unlocked) or pthread_mutex_init() is called, + // this behaviour no longer occurs. + debug_assert!(r == 0 || r == libc::EINVAL); + } else { + debug_assert_eq!(r, 0); + } + } +} + +impl Mutex { + #[inline] + pub const fn new() -> Mutex { + Mutex { inner: LazyBox::new() } + } + + #[inline] + pub unsafe fn lock(&self) { + let r = libc::pthread_mutex_lock(raw(self)); + debug_assert_eq!(r, 0); + } + + #[inline] + pub unsafe fn unlock(&self) { + let r = libc::pthread_mutex_unlock(raw(self)); + debug_assert_eq!(r, 0); + } + + #[inline] + pub unsafe fn try_lock(&self) -> bool { + libc::pthread_mutex_trylock(raw(self)) == 0 + } +} + +pub(super) struct PthreadMutexAttr<'a>(pub &'a mut MaybeUninit); + +impl Drop for PthreadMutexAttr<'_> { + fn drop(&mut self) { + unsafe { + let result = libc::pthread_mutexattr_destroy(self.0.as_mut_ptr()); + debug_assert_eq!(result, 0); + } + } +} diff --git a/crux-mir/lib/std/src/sys/unix/locks/pthread_rwlock.rs b/crux-mir/lib/std/src/sys/unix/locks/pthread_rwlock.rs new file mode 100644 index 000000000..04662be9d --- /dev/null +++ b/crux-mir/lib/std/src/sys/unix/locks/pthread_rwlock.rs @@ -0,0 +1,195 @@ +use crate::cell::UnsafeCell; +use crate::mem::forget; +use crate::sync::atomic::{AtomicUsize, Ordering}; +use crate::sys_common::lazy_box::{LazyBox, LazyInit}; + +struct AllocatedRwLock { + inner: UnsafeCell, + write_locked: UnsafeCell, // guarded by the `inner` RwLock + num_readers: AtomicUsize, +} + +unsafe impl Send for AllocatedRwLock {} +unsafe impl Sync for AllocatedRwLock {} + +pub struct RwLock { + inner: LazyBox, +} + +impl LazyInit for AllocatedRwLock { + fn init() -> Box { + Box::new(AllocatedRwLock { + inner: UnsafeCell::new(libc::PTHREAD_RWLOCK_INITIALIZER), + write_locked: UnsafeCell::new(false), + num_readers: AtomicUsize::new(0), + }) + } + + fn destroy(mut rwlock: Box) { + // We're not allowed to pthread_rwlock_destroy a locked rwlock, + // so check first if it's unlocked. + if *rwlock.write_locked.get_mut() || *rwlock.num_readers.get_mut() != 0 { + // The rwlock is locked. This happens if a RwLock{Read,Write}Guard is leaked. + // In this case, we just leak the RwLock too. + forget(rwlock); + } + } + + fn cancel_init(_: Box) { + // In this case, we can just drop it without any checks, + // since it cannot have been locked yet. + } +} + +impl AllocatedRwLock { + #[inline] + unsafe fn raw_unlock(&self) { + let r = libc::pthread_rwlock_unlock(self.inner.get()); + debug_assert_eq!(r, 0); + } +} + +impl Drop for AllocatedRwLock { + fn drop(&mut self) { + let r = unsafe { libc::pthread_rwlock_destroy(self.inner.get()) }; + // On DragonFly pthread_rwlock_destroy() returns EINVAL if called on a + // rwlock that was just initialized with + // libc::PTHREAD_RWLOCK_INITIALIZER. Once it is used (locked/unlocked) + // or pthread_rwlock_init() is called, this behaviour no longer occurs. + if cfg!(target_os = "dragonfly") { + debug_assert!(r == 0 || r == libc::EINVAL); + } else { + debug_assert_eq!(r, 0); + } + } +} + +impl RwLock { + #[inline] + pub const fn new() -> RwLock { + RwLock { inner: LazyBox::new() } + } + + #[inline] + pub fn read(&self) { + let lock = &*self.inner; + let r = unsafe { libc::pthread_rwlock_rdlock(lock.inner.get()) }; + + // According to POSIX, when a thread tries to acquire this read lock + // while it already holds the write lock + // (or vice versa, or tries to acquire the write lock twice), + // "the call shall either deadlock or return [EDEADLK]" + // (https://pubs.opengroup.org/onlinepubs/9699919799/functions/pthread_rwlock_wrlock.html, + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/pthread_rwlock_rdlock.html). + // So, in principle, all we have to do here is check `r == 0` to be sure we properly + // got the lock. + // + // However, (at least) glibc before version 2.25 does not conform to this spec, + // and can return `r == 0` even when this thread already holds the write lock. + // We thus check for this situation ourselves and panic when detecting that a thread + // got the write lock more than once, or got a read and a write lock. + if r == libc::EAGAIN { + panic!("rwlock maximum reader count exceeded"); + } else if r == libc::EDEADLK || (r == 0 && unsafe { *lock.write_locked.get() }) { + // Above, we make sure to only access `write_locked` when `r == 0` to avoid + // data races. + if r == 0 { + // `pthread_rwlock_rdlock` succeeded when it should not have. + unsafe { + lock.raw_unlock(); + } + } + panic!("rwlock read lock would result in deadlock"); + } else { + // POSIX does not make guarantees about all the errors that may be returned. + // See issue #94705 for more details. + assert_eq!(r, 0, "unexpected error during rwlock read lock: {:?}", r); + lock.num_readers.fetch_add(1, Ordering::Relaxed); + } + } + + #[inline] + pub fn try_read(&self) -> bool { + let lock = &*self.inner; + let r = unsafe { libc::pthread_rwlock_tryrdlock(lock.inner.get()) }; + if r == 0 { + if unsafe { *lock.write_locked.get() } { + // `pthread_rwlock_tryrdlock` succeeded when it should not have. + unsafe { + lock.raw_unlock(); + } + false + } else { + lock.num_readers.fetch_add(1, Ordering::Relaxed); + true + } + } else { + false + } + } + + #[inline] + pub fn write(&self) { + let lock = &*self.inner; + let r = unsafe { libc::pthread_rwlock_wrlock(lock.inner.get()) }; + // See comments above for why we check for EDEADLK and write_locked. For the same reason, + // we also need to check that there are no readers (tracked in `num_readers`). + if r == libc::EDEADLK + || (r == 0 && unsafe { *lock.write_locked.get() }) + || lock.num_readers.load(Ordering::Relaxed) != 0 + { + // Above, we make sure to only access `write_locked` when `r == 0` to avoid + // data races. + if r == 0 { + // `pthread_rwlock_wrlock` succeeded when it should not have. + unsafe { + lock.raw_unlock(); + } + } + panic!("rwlock write lock would result in deadlock"); + } else { + // According to POSIX, for a properly initialized rwlock this can only + // return EDEADLK or 0. We rely on that. + debug_assert_eq!(r, 0); + } + + unsafe { + *lock.write_locked.get() = true; + } + } + + #[inline] + pub unsafe fn try_write(&self) -> bool { + let lock = &*self.inner; + let r = libc::pthread_rwlock_trywrlock(lock.inner.get()); + if r == 0 { + if *lock.write_locked.get() || lock.num_readers.load(Ordering::Relaxed) != 0 { + // `pthread_rwlock_trywrlock` succeeded when it should not have. + lock.raw_unlock(); + false + } else { + *lock.write_locked.get() = true; + true + } + } else { + false + } + } + + #[inline] + pub unsafe fn read_unlock(&self) { + let lock = &*self.inner; + debug_assert!(!*lock.write_locked.get()); + lock.num_readers.fetch_sub(1, Ordering::Relaxed); + lock.raw_unlock(); + } + + #[inline] + pub unsafe fn write_unlock(&self) { + let lock = &*self.inner; + debug_assert_eq!(lock.num_readers.load(Ordering::Relaxed), 0); + debug_assert!(*lock.write_locked.get()); + *lock.write_locked.get() = false; + lock.raw_unlock(); + } +} diff --git a/crux-mir/lib/std/src/sys/unix/memchr.rs b/crux-mir/lib/std/src/sys/unix/memchr.rs index a9273ea67..73ba604ec 100644 --- a/crux-mir/lib/std/src/sys/unix/memchr.rs +++ b/crux-mir/lib/std/src/sys/unix/memchr.rs @@ -9,7 +9,7 @@ pub fn memchr(needle: u8, haystack: &[u8]) -> Option { haystack.len(), ) }; - if p.is_null() { None } else { Some(p as usize - (haystack.as_ptr() as usize)) } + if p.is_null() { None } else { Some(p.addr() - haystack.as_ptr().addr()) } } pub fn memrchr(needle: u8, haystack: &[u8]) -> Option { @@ -26,7 +26,9 @@ pub fn memrchr(needle: u8, haystack: &[u8]) -> Option { haystack.len(), ) }; - if p.is_null() { None } else { Some(p as usize - (haystack.as_ptr() as usize)) } + // FIXME: this should *likely* use `offset_from`, but more + // investigation is needed (including running tests in miri). + if p.is_null() { None } else { Some(p.addr() - haystack.as_ptr().addr()) } } #[cfg(not(target_os = "linux"))] diff --git a/crux-mir/lib/std/src/sys/unix/mod.rs b/crux-mir/lib/std/src/sys/unix/mod.rs index fbcb006ec..30a96be14 100644 --- a/crux-mir/lib/std/src/sys/unix/mod.rs +++ b/crux-mir/lib/std/src/sys/unix/mod.rs @@ -1,95 +1,223 @@ #![allow(missing_docs, nonstandard_style)] +use crate::ffi::CStr; use crate::io::ErrorKind; -#[cfg(any(doc, target_os = "linux"))] -pub use crate::os::linux as platform; - -#[cfg(all(not(doc), target_os = "android"))] -pub use crate::os::android as platform; -#[cfg(all(not(doc), target_os = "dragonfly"))] -pub use crate::os::dragonfly as platform; -#[cfg(all(not(doc), target_os = "emscripten"))] -pub use crate::os::emscripten as platform; -#[cfg(all(not(doc), target_os = "freebsd"))] -pub use crate::os::freebsd as platform; -#[cfg(all(not(doc), target_os = "fuchsia"))] -pub use crate::os::fuchsia as platform; -#[cfg(all(not(doc), target_os = "haiku"))] -pub use crate::os::haiku as platform; -#[cfg(all(not(doc), target_os = "ios"))] -pub use crate::os::ios as platform; -#[cfg(all(not(doc), target_os = "l4re"))] -pub use crate::os::linux as platform; -#[cfg(all(not(doc), target_os = "macos"))] -pub use crate::os::macos as platform; -#[cfg(all(not(doc), target_os = "netbsd"))] -pub use crate::os::netbsd as platform; -#[cfg(all(not(doc), target_os = "openbsd"))] -pub use crate::os::openbsd as platform; -#[cfg(all(not(doc), target_os = "redox"))] -pub use crate::os::redox as platform; -#[cfg(all(not(doc), target_os = "solaris"))] -pub use crate::os::solaris as platform; - pub use self::rand::hashmap_random_keys; -pub use libc::strlen; +#[cfg(not(target_os = "espidf"))] #[macro_use] pub mod weak; pub mod alloc; pub mod android; pub mod args; +#[path = "../unix/cmath.rs"] pub mod cmath; -pub mod condvar; pub mod env; -pub mod ext; -pub mod fast_thread_local; pub mod fd; pub mod fs; +pub mod futex; pub mod io; +#[cfg(any(target_os = "linux", target_os = "android"))] +pub mod kernel_copy; #[cfg(target_os = "l4re")] mod l4re; +pub mod locks; pub mod memchr; -pub mod mutex; #[cfg(not(target_os = "l4re"))] pub mod net; #[cfg(target_os = "l4re")] pub use self::l4re::net; pub mod os; +pub mod os_str; pub mod path; pub mod pipe; pub mod process; pub mod rand; -pub mod rwlock; pub mod stack_overflow; pub mod stdio; pub mod thread; -pub mod thread_local; +pub mod thread_local_dtor; +pub mod thread_local_key; +pub mod thread_parking; pub mod time; -pub use crate::sys_common::os_str_bytes as os_str; +#[cfg(target_os = "espidf")] +pub fn init(argc: isize, argv: *const *const u8, _sigpipe: u8) {} + +#[cfg(not(target_os = "espidf"))] +// SAFETY: must be called only once during runtime initialization. +// NOTE: this is not guaranteed to run, for example when Rust code is called externally. +// See `fn init()` in `library/std/src/rt.rs` for docs on `sigpipe`. +pub unsafe fn init(argc: isize, argv: *const *const u8, sigpipe: u8) { + // The standard streams might be closed on application startup. To prevent + // std::io::{stdin, stdout,stderr} objects from using other unrelated file + // resources opened later, we reopen standards streams when they are closed. + sanitize_standard_fds(); -#[cfg(not(test))] -pub fn init() { // By default, some platforms will send a *signal* when an EPIPE error // would otherwise be delivered. This runtime doesn't install a SIGPIPE // handler, causing it to kill the program, which isn't exactly what we // want! // // Hence, we set SIGPIPE to ignore when the program starts up in order - // to prevent this problem. - unsafe { - reset_sigpipe(); + // to prevent this problem. Add `#[unix_sigpipe = "..."]` above `fn main()` to + // alter this behavior. + reset_sigpipe(sigpipe); + + stack_overflow::init(); + args::init(argc, argv); + + // Normally, `thread::spawn` will call `Thread::set_name` but since this thread + // already exists, we have to call it ourselves. We only do this on macos + // because some unix-like operating systems such as Linux share process-id and + // thread-id for the main thread and so renaming the main thread will rename the + // process and we only want to enable this on platforms we've tested. + if cfg!(target_os = "macos") { + thread::Thread::set_name(&CStr::from_bytes_with_nul_unchecked(b"main\0")); } - #[cfg(not(any(target_os = "emscripten", target_os = "fuchsia")))] - unsafe fn reset_sigpipe() { - assert!(signal(libc::SIGPIPE, libc::SIG_IGN) != libc::SIG_ERR); + unsafe fn sanitize_standard_fds() { + // fast path with a single syscall for systems with poll() + #[cfg(not(any( + miri, + target_os = "emscripten", + target_os = "fuchsia", + target_os = "vxworks", + // The poll on Darwin doesn't set POLLNVAL for closed fds. + target_os = "macos", + target_os = "ios", + target_os = "watchos", + target_os = "redox", + target_os = "l4re", + target_os = "horizon", + )))] + 'poll: { + use crate::sys::os::errno; + #[cfg(not(all(target_os = "linux", target_env = "gnu")))] + use libc::open as open64; + #[cfg(all(target_os = "linux", target_env = "gnu"))] + use libc::open64; + let pfds: &mut [_] = &mut [ + libc::pollfd { fd: 0, events: 0, revents: 0 }, + libc::pollfd { fd: 1, events: 0, revents: 0 }, + libc::pollfd { fd: 2, events: 0, revents: 0 }, + ]; + + while libc::poll(pfds.as_mut_ptr(), 3, 0) == -1 { + match errno() { + libc::EINTR => continue, + libc::EINVAL | libc::EAGAIN | libc::ENOMEM => { + // RLIMIT_NOFILE or temporary allocation failures + // may be preventing use of poll(), fall back to fcntl + break 'poll; + } + _ => libc::abort(), + } + } + for pfd in pfds { + if pfd.revents & libc::POLLNVAL == 0 { + continue; + } + if open64("/dev/null\0".as_ptr().cast(), libc::O_RDWR, 0) == -1 { + // If the stream is closed but we failed to reopen it, abort the + // process. Otherwise we wouldn't preserve the safety of + // operations on the corresponding Rust object Stdin, Stdout, or + // Stderr. + libc::abort(); + } + } + return; + } + + // fallback in case poll isn't available or limited by RLIMIT_NOFILE + #[cfg(not(any( + // The standard fds are always available in Miri. + miri, + target_os = "emscripten", + target_os = "fuchsia", + target_os = "vxworks", + target_os = "l4re", + target_os = "horizon", + )))] + { + use crate::sys::os::errno; + #[cfg(not(all(target_os = "linux", target_env = "gnu")))] + use libc::open as open64; + #[cfg(all(target_os = "linux", target_env = "gnu"))] + use libc::open64; + for fd in 0..3 { + if libc::fcntl(fd, libc::F_GETFD) == -1 && errno() == libc::EBADF { + if open64("/dev/null\0".as_ptr().cast(), libc::O_RDWR, 0) == -1 { + // If the stream is closed but we failed to reopen it, abort the + // process. Otherwise we wouldn't preserve the safety of + // operations on the corresponding Rust object Stdin, Stdout, or + // Stderr. + libc::abort(); + } + } + } + } } - #[cfg(any(target_os = "emscripten", target_os = "fuchsia"))] - unsafe fn reset_sigpipe() {} + + unsafe fn reset_sigpipe(#[allow(unused_variables)] sigpipe: u8) { + #[cfg(not(any(target_os = "emscripten", target_os = "fuchsia", target_os = "horizon")))] + { + // We don't want to add this as a public type to std, nor do we + // want to `include!` a file from the compiler (which would break + // Miri and xargo for example), so we choose to duplicate these + // constants from `compiler/rustc_session/src/config/sigpipe.rs`. + // See the other file for docs. NOTE: Make sure to keep them in + // sync! + mod sigpipe { + pub const DEFAULT: u8 = 0; + pub const INHERIT: u8 = 1; + pub const SIG_IGN: u8 = 2; + pub const SIG_DFL: u8 = 3; + } + + let (sigpipe_attr_specified, handler) = match sigpipe { + sigpipe::DEFAULT => (false, Some(libc::SIG_IGN)), + sigpipe::INHERIT => (true, None), + sigpipe::SIG_IGN => (true, Some(libc::SIG_IGN)), + sigpipe::SIG_DFL => (true, Some(libc::SIG_DFL)), + _ => unreachable!(), + }; + if sigpipe_attr_specified { + UNIX_SIGPIPE_ATTR_SPECIFIED.store(true, crate::sync::atomic::Ordering::Relaxed); + } + if let Some(handler) = handler { + rtassert!(signal(libc::SIGPIPE, handler) != libc::SIG_ERR); + } + } + } +} + +// This is set (up to once) in reset_sigpipe. +#[cfg(not(any( + target_os = "espidf", + target_os = "emscripten", + target_os = "fuchsia", + target_os = "horizon" +)))] +static UNIX_SIGPIPE_ATTR_SPECIFIED: crate::sync::atomic::AtomicBool = + crate::sync::atomic::AtomicBool::new(false); + +#[cfg(not(any( + target_os = "espidf", + target_os = "emscripten", + target_os = "fuchsia", + target_os = "horizon" +)))] +pub(crate) fn unix_sigpipe_attr_specified() -> bool { + UNIX_SIGPIPE_ATTR_SPECIFIED.load(crate::sync::atomic::Ordering::Relaxed) +} + +// SAFETY: must be called only once during runtime cleanup. +// NOTE: this is not guaranteed to run, for example when the program aborts. +pub unsafe fn cleanup() { + stack_overflow::cleanup(); } #[cfg(target_os = "android")] @@ -98,27 +226,51 @@ pub use crate::sys::android::signal; pub use libc::signal; pub fn decode_error_kind(errno: i32) -> ErrorKind { + use ErrorKind::*; match errno as libc::c_int { - libc::ECONNREFUSED => ErrorKind::ConnectionRefused, - libc::ECONNRESET => ErrorKind::ConnectionReset, - libc::EPERM | libc::EACCES => ErrorKind::PermissionDenied, - libc::EPIPE => ErrorKind::BrokenPipe, - libc::ENOTCONN => ErrorKind::NotConnected, - libc::ECONNABORTED => ErrorKind::ConnectionAborted, - libc::EADDRNOTAVAIL => ErrorKind::AddrNotAvailable, - libc::EADDRINUSE => ErrorKind::AddrInUse, - libc::ENOENT => ErrorKind::NotFound, - libc::EINTR => ErrorKind::Interrupted, - libc::EINVAL => ErrorKind::InvalidInput, - libc::ETIMEDOUT => ErrorKind::TimedOut, - libc::EEXIST => ErrorKind::AlreadyExists, + libc::E2BIG => ArgumentListTooLong, + libc::EADDRINUSE => AddrInUse, + libc::EADDRNOTAVAIL => AddrNotAvailable, + libc::EBUSY => ResourceBusy, + libc::ECONNABORTED => ConnectionAborted, + libc::ECONNREFUSED => ConnectionRefused, + libc::ECONNRESET => ConnectionReset, + libc::EDEADLK => Deadlock, + libc::EDQUOT => FilesystemQuotaExceeded, + libc::EEXIST => AlreadyExists, + libc::EFBIG => FileTooLarge, + libc::EHOSTUNREACH => HostUnreachable, + libc::EINTR => Interrupted, + libc::EINVAL => InvalidInput, + libc::EISDIR => IsADirectory, + libc::ELOOP => FilesystemLoop, + libc::ENOENT => NotFound, + libc::ENOMEM => OutOfMemory, + libc::ENOSPC => StorageFull, + libc::ENOSYS => Unsupported, + libc::EMLINK => TooManyLinks, + libc::ENAMETOOLONG => InvalidFilename, + libc::ENETDOWN => NetworkDown, + libc::ENETUNREACH => NetworkUnreachable, + libc::ENOTCONN => NotConnected, + libc::ENOTDIR => NotADirectory, + libc::ENOTEMPTY => DirectoryNotEmpty, + libc::EPIPE => BrokenPipe, + libc::EROFS => ReadOnlyFilesystem, + libc::ESPIPE => NotSeekable, + libc::ESTALE => StaleNetworkFileHandle, + libc::ETIMEDOUT => TimedOut, + libc::ETXTBSY => ExecutableFileBusy, + libc::EXDEV => CrossesDevices, + + libc::EACCES | libc::EPERM => PermissionDenied, // These two constants can have the same value on some systems, // but different values on others, so we can't use a match // clause - x if x == libc::EAGAIN || x == libc::EWOULDBLOCK => ErrorKind::WouldBlock, + x if x == libc::EAGAIN || x == libc::EWOULDBLOCK => WouldBlock, - _ => ErrorKind::Other, + _ => Uncategorized, } } @@ -154,13 +306,111 @@ where } } -// On Unix-like platforms, libc::abort will unregister signal handlers -// including the SIGABRT handler, preventing the abort from being blocked, and -// fclose streams, with the side effect of flushing them so libc buffered -// output will be printed. Additionally the shell will generally print a more -// understandable error message like "Abort trap" rather than "Illegal -// instruction" that intrinsics::abort would cause, as intrinsics::abort is -// implemented as an illegal instruction. -pub unsafe fn abort_internal() -> ! { - libc::abort() +#[allow(dead_code)] // Not used on all platforms. +pub fn cvt_nz(error: libc::c_int) -> crate::io::Result<()> { + if error == 0 { Ok(()) } else { Err(crate::io::Error::from_raw_os_error(error)) } +} + +// libc::abort() will run the SIGABRT handler. That's fine because anyone who +// installs a SIGABRT handler already has to expect it to run in Very Bad +// situations (eg, malloc crashing). +// +// Current glibc's abort() function unblocks SIGABRT, raises SIGABRT, clears the +// SIGABRT handler and raises it again, and then starts to get creative. +// +// See the public documentation for `intrinsics::abort()` and `process::abort()` +// for further discussion. +// +// There is confusion about whether libc::abort() flushes stdio streams. +// libc::abort() is required by ISO C 99 (7.14.1.1p5) to be async-signal-safe, +// so flushing streams is at least extremely hard, if not entirely impossible. +// +// However, some versions of POSIX (eg IEEE Std 1003.1-2001) required abort to +// do so. In 1003.1-2004 this was fixed. +// +// glibc's implementation did the flush, unsafely, before glibc commit +// 91e7cf982d01 `abort: Do not flush stdio streams [BZ #15436]' by Florian +// Weimer. According to glibc's NEWS: +// +// The abort function terminates the process immediately, without flushing +// stdio streams. Previous glibc versions used to flush streams, resulting +// in deadlocks and further data corruption. This change also affects +// process aborts as the result of assertion failures. +// +// This is an accurate description of the problem. The only solution for +// program with nontrivial use of C stdio is a fixed libc - one which does not +// try to flush in abort - since even libc-internal errors, and assertion +// failures generated from C, will go via abort(). +// +// On systems with old, buggy, libcs, the impact can be severe for a +// multithreaded C program. It is much less severe for Rust, because Rust +// stdlib doesn't use libc stdio buffering. In a typical Rust program, which +// does not use C stdio, even a buggy libc::abort() is, in fact, safe. +pub fn abort_internal() -> ! { + unsafe { libc::abort() } +} + +cfg_if::cfg_if! { + if #[cfg(target_os = "android")] { + #[link(name = "dl", kind = "static", modifiers = "-bundle", + cfg(target_feature = "crt-static"))] + #[link(name = "dl", cfg(not(target_feature = "crt-static")))] + #[link(name = "log", cfg(not(target_feature = "crt-static")))] + extern "C" {} + } else if #[cfg(target_os = "freebsd")] { + #[link(name = "execinfo")] + #[link(name = "pthread")] + extern "C" {} + } else if #[cfg(target_os = "netbsd")] { + #[link(name = "pthread")] + #[link(name = "rt")] + extern "C" {} + } else if #[cfg(any(target_os = "dragonfly", target_os = "openbsd"))] { + #[link(name = "pthread")] + extern "C" {} + } else if #[cfg(target_os = "solaris")] { + #[link(name = "socket")] + #[link(name = "posix4")] + #[link(name = "pthread")] + #[link(name = "resolv")] + extern "C" {} + } else if #[cfg(target_os = "illumos")] { + #[link(name = "socket")] + #[link(name = "posix4")] + #[link(name = "pthread")] + #[link(name = "resolv")] + #[link(name = "nsl")] + // Use libumem for the (malloc-compatible) allocator + #[link(name = "umem")] + extern "C" {} + } else if #[cfg(target_os = "macos")] { + #[link(name = "System")] + extern "C" {} + } else if #[cfg(any(target_os = "ios", target_os = "watchos"))] { + #[link(name = "System")] + #[link(name = "objc")] + #[link(name = "Security", kind = "framework")] + #[link(name = "Foundation", kind = "framework")] + extern "C" {} + } else if #[cfg(target_os = "fuchsia")] { + #[link(name = "zircon")] + #[link(name = "fdio")] + extern "C" {} + } else if #[cfg(all(target_os = "linux", target_env = "uclibc"))] { + #[link(name = "dl")] + extern "C" {} + } +} + +#[cfg(any(target_os = "espidf", target_os = "horizon"))] +mod unsupported { + use crate::io; + + pub fn unsupported() -> io::Result { + Err(unsupported_err()) + } + + pub fn unsupported_err() -> io::Error { + io::const_io_error!(io::ErrorKind::Unsupported, "operation not supported on this platform",) + } } diff --git a/crux-mir/lib/std/src/sys/unix/mutex.rs b/crux-mir/lib/std/src/sys/unix/mutex.rs deleted file mode 100644 index 103d87e3d..000000000 --- a/crux-mir/lib/std/src/sys/unix/mutex.rs +++ /dev/null @@ -1,131 +0,0 @@ -use crate::cell::UnsafeCell; -use crate::mem::MaybeUninit; - -pub struct Mutex { - inner: UnsafeCell, -} - -#[inline] -pub unsafe fn raw(m: &Mutex) -> *mut libc::pthread_mutex_t { - m.inner.get() -} - -unsafe impl Send for Mutex {} -unsafe impl Sync for Mutex {} - -#[allow(dead_code)] // sys isn't exported yet -impl Mutex { - pub const fn new() -> Mutex { - // Might be moved to a different address, so it is better to avoid - // initialization of potentially opaque OS data before it landed. - // Be very careful using this newly constructed `Mutex`, reentrant - // locking is undefined behavior until `init` is called! - Mutex { inner: UnsafeCell::new(libc::PTHREAD_MUTEX_INITIALIZER) } - } - #[inline] - pub unsafe fn init(&mut self) { - // Issue #33770 - // - // A pthread mutex initialized with PTHREAD_MUTEX_INITIALIZER will have - // a type of PTHREAD_MUTEX_DEFAULT, which has undefined behavior if you - // try to re-lock it from the same thread when you already hold a lock. - // - // In practice, glibc takes advantage of this undefined behavior to - // implement hardware lock elision, which uses hardware transactional - // memory to avoid acquiring the lock. While a transaction is in - // progress, the lock appears to be unlocked. This isn't a problem for - // other threads since the transactional memory will abort if a conflict - // is detected, however no abort is generated if re-locking from the - // same thread. - // - // Since locking the same mutex twice will result in two aliasing &mut - // references, we instead create the mutex with type - // PTHREAD_MUTEX_NORMAL which is guaranteed to deadlock if we try to - // re-lock it from the same thread, thus avoiding undefined behavior. - let mut attr = MaybeUninit::::uninit(); - let r = libc::pthread_mutexattr_init(attr.as_mut_ptr()); - debug_assert_eq!(r, 0); - let r = libc::pthread_mutexattr_settype(attr.as_mut_ptr(), libc::PTHREAD_MUTEX_NORMAL); - debug_assert_eq!(r, 0); - let r = libc::pthread_mutex_init(self.inner.get(), attr.as_ptr()); - debug_assert_eq!(r, 0); - let r = libc::pthread_mutexattr_destroy(attr.as_mut_ptr()); - debug_assert_eq!(r, 0); - } - #[inline] - pub unsafe fn lock(&self) { - let r = libc::pthread_mutex_lock(self.inner.get()); - debug_assert_eq!(r, 0); - } - #[inline] - pub unsafe fn unlock(&self) { - let r = libc::pthread_mutex_unlock(self.inner.get()); - debug_assert_eq!(r, 0); - } - #[inline] - pub unsafe fn try_lock(&self) -> bool { - libc::pthread_mutex_trylock(self.inner.get()) == 0 - } - #[inline] - #[cfg(not(target_os = "dragonfly"))] - pub unsafe fn destroy(&self) { - let r = libc::pthread_mutex_destroy(self.inner.get()); - debug_assert_eq!(r, 0); - } - #[inline] - #[cfg(target_os = "dragonfly")] - pub unsafe fn destroy(&self) { - let r = libc::pthread_mutex_destroy(self.inner.get()); - // On DragonFly pthread_mutex_destroy() returns EINVAL if called on a - // mutex that was just initialized with libc::PTHREAD_MUTEX_INITIALIZER. - // Once it is used (locked/unlocked) or pthread_mutex_init() is called, - // this behaviour no longer occurs. - debug_assert!(r == 0 || r == libc::EINVAL); - } -} - -pub struct ReentrantMutex { - inner: UnsafeCell, -} - -unsafe impl Send for ReentrantMutex {} -unsafe impl Sync for ReentrantMutex {} - -impl ReentrantMutex { - pub const unsafe fn uninitialized() -> ReentrantMutex { - ReentrantMutex { inner: UnsafeCell::new(libc::PTHREAD_MUTEX_INITIALIZER) } - } - - pub unsafe fn init(&self) { - let mut attr = MaybeUninit::::uninit(); - let result = libc::pthread_mutexattr_init(attr.as_mut_ptr()); - debug_assert_eq!(result, 0); - let result = - libc::pthread_mutexattr_settype(attr.as_mut_ptr(), libc::PTHREAD_MUTEX_RECURSIVE); - debug_assert_eq!(result, 0); - let result = libc::pthread_mutex_init(self.inner.get(), attr.as_ptr()); - debug_assert_eq!(result, 0); - let result = libc::pthread_mutexattr_destroy(attr.as_mut_ptr()); - debug_assert_eq!(result, 0); - } - - pub unsafe fn lock(&self) { - let result = libc::pthread_mutex_lock(self.inner.get()); - debug_assert_eq!(result, 0); - } - - #[inline] - pub unsafe fn try_lock(&self) -> bool { - libc::pthread_mutex_trylock(self.inner.get()) == 0 - } - - pub unsafe fn unlock(&self) { - let result = libc::pthread_mutex_unlock(self.inner.get()); - debug_assert_eq!(result, 0); - } - - pub unsafe fn destroy(&self) { - let result = libc::pthread_mutex_destroy(self.inner.get()); - debug_assert_eq!(result, 0); - } -} diff --git a/crux-mir/lib/std/src/sys/unix/net.rs b/crux-mir/lib/std/src/sys/unix/net.rs index b37675e0a..c86f80972 100644 --- a/crux-mir/lib/std/src/sys/unix/net.rs +++ b/crux-mir/lib/std/src/sys/unix/net.rs @@ -3,13 +3,22 @@ use crate::ffi::CStr; use crate::io::{self, IoSlice, IoSliceMut}; use crate::mem; use crate::net::{Shutdown, SocketAddr}; +use crate::os::unix::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, RawFd}; use crate::str; use crate::sys::fd::FileDesc; use crate::sys_common::net::{getsockopt, setsockopt, sockaddr_to_addr}; use crate::sys_common::{AsInner, FromInner, IntoInner}; use crate::time::{Duration, Instant}; -use libc::{c_int, c_void, size_t, sockaddr, socklen_t, EAI_SYSTEM, MSG_PEEK}; +use libc::{c_int, c_void, size_t, sockaddr, socklen_t, MSG_PEEK}; + +cfg_if::cfg_if! { + if #[cfg(target_vendor = "apple")] { + use libc::SO_LINGER_SEC as SO_LINGER; + } else { + use libc::SO_LINGER; + } +} pub use crate::sys::{cvt, cvt_r}; @@ -30,16 +39,22 @@ pub fn cvt_gai(err: c_int) -> io::Result<()> { // We may need to trigger a glibc workaround. See on_resolver_failure() for details. on_resolver_failure(); - if err == EAI_SYSTEM { + #[cfg(not(target_os = "espidf"))] + if err == libc::EAI_SYSTEM { return Err(io::Error::last_os_error()); } + #[cfg(not(target_os = "espidf"))] let detail = unsafe { str::from_utf8(CStr::from_ptr(libc::gai_strerror(err)).to_bytes()).unwrap().to_owned() }; + + #[cfg(target_os = "espidf")] + let detail = ""; + Err(io::Error::new( - io::ErrorKind::Other, - &format!("failed to lookup address information: {}", detail)[..], + io::ErrorKind::Uncategorized, + &format!("failed to lookup address information: {detail}")[..], )) } @@ -54,64 +69,78 @@ impl Socket { pub fn new_raw(fam: c_int, ty: c_int) -> io::Result { unsafe { - // On linux we first attempt to pass the SOCK_CLOEXEC flag to - // atomically create the socket and set it as CLOEXEC. Support for - // this option, however, was added in 2.6.27, and we still support - // 2.6.18 as a kernel, so if the returned error is EINVAL we - // fallthrough to the fallback. - #[cfg(target_os = "linux")] - { - match cvt(libc::socket(fam, ty | libc::SOCK_CLOEXEC, 0)) { - Ok(fd) => return Ok(Socket(FileDesc::new(fd))), - Err(ref e) if e.raw_os_error() == Some(libc::EINVAL) => {} - Err(e) => return Err(e), - } - } - - let fd = cvt(libc::socket(fam, ty, 0))?; - let fd = FileDesc::new(fd); - fd.set_cloexec()?; - let socket = Socket(fd); + cfg_if::cfg_if! { + if #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "illumos", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + ))] { + // On platforms that support it we pass the SOCK_CLOEXEC + // flag to atomically create the socket and set it as + // CLOEXEC. On Linux this was added in 2.6.27. + let fd = cvt(libc::socket(fam, ty | libc::SOCK_CLOEXEC, 0))?; + Ok(Socket(FileDesc::from_raw_fd(fd))) + } else { + let fd = cvt(libc::socket(fam, ty, 0))?; + let fd = FileDesc::from_raw_fd(fd); + fd.set_cloexec()?; + let socket = Socket(fd); - // macOS and iOS use `SO_NOSIGPIPE` as a `setsockopt` - // flag to disable `SIGPIPE` emission on socket. - #[cfg(target_vendor = "apple")] - setsockopt(&socket, libc::SOL_SOCKET, libc::SO_NOSIGPIPE, 1)?; + // macOS and iOS use `SO_NOSIGPIPE` as a `setsockopt` + // flag to disable `SIGPIPE` emission on socket. + #[cfg(target_vendor = "apple")] + setsockopt(&socket, libc::SOL_SOCKET, libc::SO_NOSIGPIPE, 1)?; - Ok(socket) + Ok(socket) + } + } } } + #[cfg(not(target_os = "vxworks"))] pub fn new_pair(fam: c_int, ty: c_int) -> io::Result<(Socket, Socket)> { unsafe { let mut fds = [0, 0]; - // Like above, see if we can set cloexec atomically - #[cfg(target_os = "linux")] - { - match cvt(libc::socketpair(fam, ty | libc::SOCK_CLOEXEC, 0, fds.as_mut_ptr())) { - Ok(_) => { - return Ok((Socket(FileDesc::new(fds[0])), Socket(FileDesc::new(fds[1])))); - } - Err(ref e) if e.raw_os_error() == Some(libc::EINVAL) => {} - Err(e) => return Err(e), + cfg_if::cfg_if! { + if #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "illumos", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + ))] { + // Like above, set cloexec atomically + cvt(libc::socketpair(fam, ty | libc::SOCK_CLOEXEC, 0, fds.as_mut_ptr()))?; + Ok((Socket(FileDesc::from_raw_fd(fds[0])), Socket(FileDesc::from_raw_fd(fds[1])))) + } else { + cvt(libc::socketpair(fam, ty, 0, fds.as_mut_ptr()))?; + let a = FileDesc::from_raw_fd(fds[0]); + let b = FileDesc::from_raw_fd(fds[1]); + a.set_cloexec()?; + b.set_cloexec()?; + Ok((Socket(a), Socket(b))) } } - - cvt(libc::socketpair(fam, ty, 0, fds.as_mut_ptr()))?; - let a = FileDesc::new(fds[0]); - let b = FileDesc::new(fds[1]); - a.set_cloexec()?; - b.set_cloexec()?; - Ok((Socket(a), Socket(b))) } } + #[cfg(target_os = "vxworks")] + pub fn new_pair(_fam: c_int, _ty: c_int) -> io::Result<(Socket, Socket)> { + unimplemented!() + } + pub fn connect_timeout(&self, addr: &SocketAddr, timeout: Duration) -> io::Result<()> { self.set_nonblocking(true)?; let r = unsafe { - let (addrp, len) = addr.into_inner(); - cvt(libc::connect(self.0.raw(), addrp, len)) + let (addr, len) = addr.into_inner(); + cvt(libc::connect(self.as_raw_fd(), addr.as_ptr(), len)) }; self.set_nonblocking(false)?; @@ -122,10 +151,10 @@ impl Socket { Err(e) => return Err(e), } - let mut pollfd = libc::pollfd { fd: self.0.raw(), events: libc::POLLOUT, revents: 0 }; + let mut pollfd = libc::pollfd { fd: self.as_raw_fd(), events: libc::POLLOUT, revents: 0 }; if timeout.as_secs() == 0 && timeout.subsec_nanos() == 0 { - return Err(io::Error::new( + return Err(io::const_io_error!( io::ErrorKind::InvalidInput, "cannot set a 0 duration timeout", )); @@ -136,7 +165,7 @@ impl Socket { loop { let elapsed = start.elapsed(); if elapsed >= timeout { - return Err(io::Error::new(io::ErrorKind::TimedOut, "connection timed out")); + return Err(io::const_io_error!(io::ErrorKind::TimedOut, "connection timed out")); } let timeout = timeout - elapsed; @@ -148,7 +177,7 @@ impl Socket { timeout = 1; } - let timeout = cmp::min(timeout, c_int::max_value() as u64) as c_int; + let timeout = cmp::min(timeout, c_int::MAX as u64) as c_int; match unsafe { libc::poll(&mut pollfd, 1, timeout) } { -1 => { @@ -163,7 +192,10 @@ impl Socket { // for POLLHUP rather than read readiness if pollfd.revents & libc::POLLHUP != 0 { let e = self.take_error()?.unwrap_or_else(|| { - io::Error::new(io::ErrorKind::Other, "no error set after POLLHUP") + io::const_io_error!( + io::ErrorKind::Uncategorized, + "no error set after POLLHUP", + ) }); return Err(e); } @@ -177,30 +209,31 @@ impl Socket { pub fn accept(&self, storage: *mut sockaddr, len: *mut socklen_t) -> io::Result { // Unfortunately the only known way right now to accept a socket and // atomically set the CLOEXEC flag is to use the `accept4` syscall on - // Linux. This was added in 2.6.28, however, and because we support - // 2.6.18 we must detect this support dynamically. - #[cfg(target_os = "linux")] - { - syscall! { - fn accept4( - fd: c_int, - addr: *mut sockaddr, - addr_len: *mut socklen_t, - flags: c_int - ) -> c_int - } - let res = cvt_r(|| unsafe { accept4(self.0.raw(), storage, len, libc::SOCK_CLOEXEC) }); - match res { - Ok(fd) => return Ok(Socket(FileDesc::new(fd))), - Err(ref e) if e.raw_os_error() == Some(libc::ENOSYS) => {} - Err(e) => return Err(e), + // platforms that support it. On Linux, this was added in 2.6.28, + // glibc 2.10 and musl 0.9.5. + cfg_if::cfg_if! { + if #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "illumos", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + ))] { + unsafe { + let fd = cvt_r(|| libc::accept4(self.as_raw_fd(), storage, len, libc::SOCK_CLOEXEC))?; + Ok(Socket(FileDesc::from_raw_fd(fd))) + } + } else { + unsafe { + let fd = cvt_r(|| libc::accept(self.as_raw_fd(), storage, len))?; + let fd = FileDesc::from_raw_fd(fd); + fd.set_cloexec()?; + Ok(Socket(fd)) + } } } - - let fd = cvt_r(|| unsafe { libc::accept(self.0.raw(), storage, len) })?; - let fd = FileDesc::new(fd); - fd.set_cloexec()?; - Ok(Socket(fd)) } pub fn duplicate(&self) -> io::Result { @@ -209,7 +242,7 @@ impl Socket { fn recv_with_flags(&self, buf: &mut [u8], flags: c_int) -> io::Result { let ret = cvt(unsafe { - libc::recv(self.0.raw(), buf.as_mut_ptr() as *mut c_void, buf.len(), flags) + libc::recv(self.as_raw_fd(), buf.as_mut_ptr() as *mut c_void, buf.len(), flags) })?; Ok(ret as usize) } @@ -226,6 +259,11 @@ impl Socket { self.0.read_vectored(bufs) } + #[inline] + pub fn is_read_vectored(&self) -> bool { + self.0.is_read_vectored() + } + fn recv_from_with_flags( &self, buf: &mut [u8], @@ -236,7 +274,7 @@ impl Socket { let n = cvt(unsafe { libc::recvfrom( - self.0.raw(), + self.as_raw_fd(), buf.as_mut_ptr() as *mut c_void, buf.len(), flags, @@ -251,6 +289,12 @@ impl Socket { self.recv_from_with_flags(buf, 0) } + #[cfg(any(target_os = "android", target_os = "linux"))] + pub fn recv_msg(&self, msg: &mut libc::msghdr) -> io::Result { + let n = cvt(unsafe { libc::recvmsg(self.as_raw_fd(), msg, libc::MSG_CMSG_CLOEXEC) })?; + Ok(n as usize) + } + pub fn peek_from(&self, buf: &mut [u8]) -> io::Result<(usize, SocketAddr)> { self.recv_from_with_flags(buf, MSG_PEEK) } @@ -263,18 +307,29 @@ impl Socket { self.0.write_vectored(bufs) } + #[inline] + pub fn is_write_vectored(&self) -> bool { + self.0.is_write_vectored() + } + + #[cfg(any(target_os = "android", target_os = "linux"))] + pub fn send_msg(&self, msg: &mut libc::msghdr) -> io::Result { + let n = cvt(unsafe { libc::sendmsg(self.as_raw_fd(), msg, 0) })?; + Ok(n as usize) + } + pub fn set_timeout(&self, dur: Option, kind: libc::c_int) -> io::Result<()> { let timeout = match dur { Some(dur) => { if dur.as_secs() == 0 && dur.subsec_nanos() == 0 { - return Err(io::Error::new( + return Err(io::const_io_error!( io::ErrorKind::InvalidInput, "cannot set a 0 duration timeout", )); } - let secs = if dur.as_secs() > libc::time_t::max_value() as u64 { - libc::time_t::max_value() + let secs = if dur.as_secs() > libc::time_t::MAX as u64 { + libc::time_t::MAX } else { dur.as_secs() as libc::time_t }; @@ -309,10 +364,25 @@ impl Socket { Shutdown::Read => libc::SHUT_RD, Shutdown::Both => libc::SHUT_RDWR, }; - cvt(unsafe { libc::shutdown(self.0.raw(), how) })?; + cvt(unsafe { libc::shutdown(self.as_raw_fd(), how) })?; Ok(()) } + pub fn set_linger(&self, linger: Option) -> io::Result<()> { + let linger = libc::linger { + l_onoff: linger.is_some() as libc::c_int, + l_linger: linger.unwrap_or_default().as_secs() as libc::c_int, + }; + + setsockopt(self, libc::SOL_SOCKET, SO_LINGER, linger) + } + + pub fn linger(&self) -> io::Result> { + let val: libc::linger = getsockopt(self, libc::SOL_SOCKET, SO_LINGER)?; + + Ok((val.l_onoff != 0).then(|| Duration::from_secs(val.l_linger as u64))) + } + pub fn set_nodelay(&self, nodelay: bool) -> io::Result<()> { setsockopt(self, libc::IPPROTO_TCP, libc::TCP_NODELAY, nodelay as c_int) } @@ -322,32 +392,113 @@ impl Socket { Ok(raw != 0) } + #[cfg(any(target_os = "android", target_os = "linux",))] + pub fn set_quickack(&self, quickack: bool) -> io::Result<()> { + setsockopt(self, libc::IPPROTO_TCP, libc::TCP_QUICKACK, quickack as c_int) + } + + #[cfg(any(target_os = "android", target_os = "linux",))] + pub fn quickack(&self) -> io::Result { + let raw: c_int = getsockopt(self, libc::IPPROTO_TCP, libc::TCP_QUICKACK)?; + Ok(raw != 0) + } + + #[cfg(any(target_os = "android", target_os = "linux",))] + pub fn set_passcred(&self, passcred: bool) -> io::Result<()> { + setsockopt(self, libc::SOL_SOCKET, libc::SO_PASSCRED, passcred as libc::c_int) + } + + #[cfg(any(target_os = "android", target_os = "linux",))] + pub fn passcred(&self) -> io::Result { + let passcred: libc::c_int = getsockopt(self, libc::SOL_SOCKET, libc::SO_PASSCRED)?; + Ok(passcred != 0) + } + + #[cfg(target_os = "netbsd")] + pub fn set_passcred(&self, passcred: bool) -> io::Result<()> { + setsockopt(self, 0 as libc::c_int, libc::LOCAL_CREDS, passcred as libc::c_int) + } + + #[cfg(target_os = "netbsd")] + pub fn passcred(&self) -> io::Result { + let passcred: libc::c_int = getsockopt(self, 0 as libc::c_int, libc::LOCAL_CREDS)?; + Ok(passcred != 0) + } + + #[cfg(not(any(target_os = "solaris", target_os = "illumos")))] pub fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()> { let mut nonblocking = nonblocking as libc::c_int; - cvt(unsafe { libc::ioctl(*self.as_inner(), libc::FIONBIO, &mut nonblocking) }).map(drop) + cvt(unsafe { libc::ioctl(self.as_raw_fd(), libc::FIONBIO, &mut nonblocking) }).map(drop) + } + + #[cfg(any(target_os = "solaris", target_os = "illumos"))] + pub fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()> { + // FIONBIO is inadequate for sockets on illumos/Solaris, so use the + // fcntl(F_[GS]ETFL)-based method provided by FileDesc instead. + self.0.set_nonblocking(nonblocking) + } + + #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "openbsd"))] + pub fn set_mark(&self, mark: u32) -> io::Result<()> { + #[cfg(target_os = "linux")] + let option = libc::SO_MARK; + #[cfg(target_os = "freebsd")] + let option = libc::SO_USER_COOKIE; + #[cfg(target_os = "openbsd")] + let option = libc::SO_RTABLE; + setsockopt(self, libc::SOL_SOCKET, option, mark as libc::c_int) } pub fn take_error(&self) -> io::Result> { let raw: c_int = getsockopt(self, libc::SOL_SOCKET, libc::SO_ERROR)?; if raw == 0 { Ok(None) } else { Ok(Some(io::Error::from_raw_os_error(raw as i32))) } } + + // This is used by sys_common code to abstract over Windows and Unix. + pub fn as_raw(&self) -> RawFd { + self.as_raw_fd() + } +} + +impl AsInner for Socket { + fn as_inner(&self) -> &FileDesc { + &self.0 + } +} + +impl IntoInner for Socket { + fn into_inner(self) -> FileDesc { + self.0 + } +} + +impl FromInner for Socket { + fn from_inner(file_desc: FileDesc) -> Self { + Self(file_desc) + } +} + +impl AsFd for Socket { + fn as_fd(&self) -> BorrowedFd<'_> { + self.0.as_fd() + } } -impl AsInner for Socket { - fn as_inner(&self) -> &c_int { - self.0.as_inner() +impl AsRawFd for Socket { + fn as_raw_fd(&self) -> RawFd { + self.0.as_raw_fd() } } -impl FromInner for Socket { - fn from_inner(fd: c_int) -> Socket { - Socket(FileDesc::new(fd)) +impl IntoRawFd for Socket { + fn into_raw_fd(self) -> RawFd { + self.0.into_raw_fd() } } -impl IntoInner for Socket { - fn into_inner(self) -> c_int { - self.0.into_raw() +impl FromRawFd for Socket { + unsafe fn from_raw_fd(raw_fd: RawFd) -> Self { + Self(FromRawFd::from_raw_fd(raw_fd)) } } @@ -361,13 +512,13 @@ impl IntoInner for Socket { // A workaround for this bug is to call the res_init libc function, to clear // the cached configs. Unfortunately, while we believe glibc's implementation // of res_init is thread-safe, we know that other implementations are not -// (https://github.com/rust-lang/rust/issues/43592). Code here in libstd could +// (https://github.com/rust-lang/rust/issues/43592). Code here in std could // try to synchronize its res_init calls with a Mutex, but that wouldn't // protect programs that call into libc in other ways. So instead of calling // res_init unconditionally, we call it only when we detect we're linking // against glibc version < 2.26. (That is, when we both know its needed and // believe it's thread-safe). -#[cfg(target_env = "gnu")] +#[cfg(all(target_os = "linux", target_env = "gnu"))] fn on_resolver_failure() { use crate::sys; @@ -379,5 +530,5 @@ fn on_resolver_failure() { } } -#[cfg(not(target_env = "gnu"))] +#[cfg(not(all(target_os = "linux", target_env = "gnu")))] fn on_resolver_failure() {} diff --git a/crux-mir/lib/std/src/sys/unix/os.rs b/crux-mir/lib/std/src/sys/unix/os.rs index 91f7d1524..2f2663db6 100644 --- a/crux-mir/lib/std/src/sys/unix/os.rs +++ b/crux-mir/lib/std/src/sys/unix/os.rs @@ -2,25 +2,32 @@ #![allow(unused_imports)] // lots of cfg code here +#[cfg(test)] +mod tests; + use crate::os::unix::prelude::*; +use crate::convert::TryFrom; use crate::error::Error as StdError; use crate::ffi::{CStr, CString, OsStr, OsString}; use crate::fmt; use crate::io; use crate::iter; -use crate::marker::PhantomData; use crate::mem; -use crate::memchr; use crate::path::{self, PathBuf}; use crate::ptr; use crate::slice; use crate::str; +use crate::sync::{PoisonError, RwLock}; +use crate::sys::common::small_c_string::{run_path_with_cstr, run_with_cstr}; use crate::sys::cvt; use crate::sys::fd; -use crate::sys_common::mutex::{Mutex, MutexGuard}; +use crate::sys::memchr; use crate::vec; +#[cfg(all(target_env = "gnu", not(target_os = "vxworks")))] +use crate::sys::weak::weak; + use libc::{c_char, c_int, c_void}; const TMPBUF_SZ: usize = 128; @@ -34,7 +41,7 @@ cfg_if::cfg_if! { } extern "C" { - #[cfg(not(target_os = "dragonfly"))] + #[cfg(not(any(target_os = "dragonfly", target_os = "vxworks")))] #[cfg_attr( any( target_os = "linux", @@ -54,9 +61,9 @@ extern "C" { ), link_name = "__errno" )] - #[cfg_attr(target_os = "solaris", link_name = "___errno")] + #[cfg_attr(any(target_os = "solaris", target_os = "illumos"), link_name = "___errno")] #[cfg_attr( - any(target_os = "macos", target_os = "ios", target_os = "freebsd"), + any(target_os = "macos", target_os = "ios", target_os = "freebsd", target_os = "watchos"), link_name = "__error" )] #[cfg_attr(target_os = "haiku", link_name = "_errnop")] @@ -64,17 +71,23 @@ extern "C" { } /// Returns the platform-specific value of errno -#[cfg(not(target_os = "dragonfly"))] +#[cfg(not(any(target_os = "dragonfly", target_os = "vxworks")))] pub fn errno() -> i32 { unsafe { (*errno_location()) as i32 } } /// Sets the platform-specific value of errno -#[cfg(all(not(target_os = "linux"), not(target_os = "dragonfly")))] // needed for readdir and syscall! +#[cfg(all(not(target_os = "dragonfly"), not(target_os = "vxworks")))] // needed for readdir and syscall! +#[allow(dead_code)] // but not all target cfgs actually end up using it pub fn set_errno(e: i32) { unsafe { *errno_location() = e as c_int } } +#[cfg(target_os = "vxworks")] +pub fn errno() -> i32 { + unsafe { libc::errnoGet() } +} + #[cfg(target_os = "dragonfly")] pub fn errno() -> i32 { extern "C" { @@ -86,6 +99,7 @@ pub fn errno() -> i32 { } #[cfg(target_os = "dragonfly")] +#[allow(dead_code)] pub fn set_errno(e: i32) { extern "C" { #[thread_local] @@ -113,10 +127,18 @@ pub fn error_string(errno: i32) -> String { } let p = p as *const _; - str::from_utf8(CStr::from_ptr(p).to_bytes()).unwrap().to_owned() + // We can't always expect a UTF-8 environment. When we don't get that luxury, + // it's better to give a low-quality error message than none at all. + String::from_utf8_lossy(CStr::from_ptr(p).to_bytes()).into() } } +#[cfg(target_os = "espidf")] +pub fn getcwd() -> io::Result { + Ok(PathBuf::from("/")) +} + +#[cfg(not(target_os = "espidf"))] pub fn getcwd() -> io::Result { let mut buf = Vec::with_capacity(512); loop { @@ -143,15 +165,15 @@ pub fn getcwd() -> io::Result { } } +#[cfg(target_os = "espidf")] pub fn chdir(p: &path::Path) -> io::Result<()> { - let p: &OsStr = p.as_ref(); - let p = CString::new(p.as_bytes())?; - unsafe { - match libc::chdir(p.as_ptr()) == (0 as c_int) { - true => Ok(()), - false => Err(io::Error::last_os_error()), - } - } + super::unsupported::unsupported() +} + +#[cfg(not(target_os = "espidf"))] +pub fn chdir(p: &path::Path) -> io::Result<()> { + let result = run_path_with_cstr(p, |p| unsafe { Ok(libc::chdir(p.as_ptr())) })?; + if result == 0 { Ok(()) } else { Err(io::Error::last_os_error()) } } pub struct SplitPaths<'a> { @@ -208,7 +230,7 @@ where impl fmt::Display for JoinPathsError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "path segment contains separator `{}`", PATH_SEPARATOR) + write!(f, "path segment contains separator `{}`", char::from(PATH_SEPARATOR)) } } @@ -272,8 +294,8 @@ pub fn current_exe() -> io::Result { 0, ))?; if path_len <= 1 { - return Err(io::Error::new( - io::ErrorKind::Other, + return Err(io::const_io_error!( + io::ErrorKind::Uncategorized, "KERN_PROC_PATHNAME sysctl returned zero-length string", )); } @@ -295,8 +317,8 @@ pub fn current_exe() -> io::Result { if curproc_exe.is_file() { return crate::fs::read_link(curproc_exe); } - Err(io::Error::new( - io::ErrorKind::Other, + Err(io::const_io_error!( + io::ErrorKind::Uncategorized, "/proc/curproc/exe doesn't point to regular file.", )) } @@ -314,7 +336,10 @@ pub fn current_exe() -> io::Result { cvt(libc::sysctl(mib, 4, argv.as_mut_ptr() as *mut _, &mut argv_len, ptr::null_mut(), 0))?; argv.set_len(argv_len as usize); if argv[0].is_null() { - return Err(io::Error::new(io::ErrorKind::Other, "no current exe available")); + return Err(io::const_io_error!( + io::ErrorKind::Uncategorized, + "no current exe available", + )); } let argv0 = CStr::from_ptr(argv[0]).to_bytes(); if argv0[0] == b'.' || argv0.iter().any(|b| *b == b'/') { @@ -328,27 +353,24 @@ pub fn current_exe() -> io::Result { #[cfg(any(target_os = "linux", target_os = "android", target_os = "emscripten"))] pub fn current_exe() -> io::Result { match crate::fs::read_link("/proc/self/exe") { - Err(ref e) if e.kind() == io::ErrorKind::NotFound => Err(io::Error::new( - io::ErrorKind::Other, + Err(ref e) if e.kind() == io::ErrorKind::NotFound => Err(io::const_io_error!( + io::ErrorKind::Uncategorized, "no /proc/self/exe available. Is /proc mounted?", )), other => other, } } -#[cfg(any(target_os = "macos", target_os = "ios"))] +#[cfg(any(target_os = "macos", target_os = "ios", target_os = "watchos"))] pub fn current_exe() -> io::Result { - extern "C" { - fn _NSGetExecutablePath(buf: *mut libc::c_char, bufsize: *mut u32) -> libc::c_int; - } unsafe { let mut sz: u32 = 0; - _NSGetExecutablePath(ptr::null_mut(), &mut sz); + libc::_NSGetExecutablePath(ptr::null_mut(), &mut sz); if sz == 0 { return Err(io::Error::last_os_error()); } let mut v: Vec = Vec::with_capacity(sz as usize); - let err = _NSGetExecutablePath(v.as_mut_ptr() as *mut i8, &mut sz); + let err = libc::_NSGetExecutablePath(v.as_mut_ptr() as *mut i8, &mut sz); if err != 0 { return Err(io::Error::last_os_error()); } @@ -357,68 +379,44 @@ pub fn current_exe() -> io::Result { } } -#[cfg(any(target_os = "solaris"))] +#[cfg(any(target_os = "solaris", target_os = "illumos"))] pub fn current_exe() -> io::Result { - extern "C" { - fn getexecname() -> *const c_char; - } - unsafe { - let path = getexecname(); - if path.is_null() { - Err(io::Error::last_os_error()) - } else { - let filename = CStr::from_ptr(path).to_bytes(); - let path = PathBuf::from(::from_bytes(filename)); + if let Ok(path) = crate::fs::read_link("/proc/self/path/a.out") { + Ok(path) + } else { + unsafe { + let path = libc::getexecname(); + if path.is_null() { + Err(io::Error::last_os_error()) + } else { + let filename = CStr::from_ptr(path).to_bytes(); + let path = PathBuf::from(::from_bytes(filename)); - // Prepend a current working directory to the path if - // it doesn't contain an absolute pathname. - if filename[0] == b'/' { Ok(path) } else { getcwd().map(|cwd| cwd.join(path)) } + // Prepend a current working directory to the path if + // it doesn't contain an absolute pathname. + if filename[0] == b'/' { Ok(path) } else { getcwd().map(|cwd| cwd.join(path)) } + } } } } #[cfg(target_os = "haiku")] pub fn current_exe() -> io::Result { - // Use Haiku's image info functions - #[repr(C)] - struct image_info { - id: i32, - type_: i32, - sequence: i32, - init_order: i32, - init_routine: *mut libc::c_void, // function pointer - term_routine: *mut libc::c_void, // function pointer - device: libc::dev_t, - node: libc::ino_t, - name: [libc::c_char; 1024], // MAXPATHLEN - text: *mut libc::c_void, - data: *mut libc::c_void, - text_size: i32, - data_size: i32, - api_version: i32, - abi: i32, - } - unsafe { - extern "C" { - fn _get_next_image_info( - team_id: i32, - cookie: *mut i32, - info: *mut image_info, - size: i32, - ) -> i32; - } - - let mut info: image_info = mem::zeroed(); + let mut info: mem::MaybeUninit = mem::MaybeUninit::uninit(); let mut cookie: i32 = 0; // the executable can be found at team id 0 - let result = - _get_next_image_info(0, &mut cookie, &mut info, mem::size_of::() as i32); + let result = libc::_get_next_image_info( + 0, + &mut cookie, + info.as_mut_ptr(), + mem::size_of::(), + ); if result != 0 { use crate::io::ErrorKind; - Err(io::Error::new(ErrorKind::Other, "Error getting executable path")) + Err(io::const_io_error!(ErrorKind::Uncategorized, "Error getting executable path")) } else { - let name = CStr::from_ptr(info.name.as_ptr()).to_bytes(); + let name = CStr::from_ptr((*info.as_ptr()).name.as_ptr()).to_bytes(); Ok(PathBuf::from(OsStr::from_bytes(name))) } } @@ -429,17 +427,57 @@ pub fn current_exe() -> io::Result { crate::fs::read_to_string("sys:exe").map(PathBuf::from) } -#[cfg(any(target_os = "fuchsia", target_os = "l4re"))] +#[cfg(target_os = "l4re")] +pub fn current_exe() -> io::Result { + use crate::io::ErrorKind; + Err(io::const_io_error!(ErrorKind::Unsupported, "Not yet implemented!")) +} + +#[cfg(target_os = "vxworks")] +pub fn current_exe() -> io::Result { + #[cfg(test)] + use realstd::env; + + #[cfg(not(test))] + use crate::env; + + let exe_path = env::args().next().unwrap(); + let path = path::Path::new(&exe_path); + path.canonicalize() +} + +#[cfg(any(target_os = "espidf", target_os = "horizon"))] +pub fn current_exe() -> io::Result { + super::unsupported::unsupported() +} + +#[cfg(target_os = "fuchsia")] pub fn current_exe() -> io::Result { use crate::io::ErrorKind; - Err(io::Error::new(ErrorKind::Other, "Not yet implemented!")) + + #[cfg(test)] + use realstd::env; + + #[cfg(not(test))] + use crate::env; + + let exe_path = env::args().next().ok_or(io::const_io_error!( + ErrorKind::Uncategorized, + "an executable path was not found because no arguments were provided through argv" + ))?; + let path = PathBuf::from(exe_path); + + // Prepend the current working directory to the path if it's not absolute. + if !path.is_absolute() { getcwd().map(|cwd| cwd.join(path)) } else { Ok(path) } } pub struct Env { iter: vec::IntoIter<(OsString, OsString)>, - _dont_send_or_sync_me: PhantomData<*mut ()>, } +impl !Send for Env {} +impl !Sync for Env {} + impl Iterator for Env { type Item = (OsString, OsString); fn next(&mut self) -> Option<(OsString, OsString)> { @@ -452,10 +490,7 @@ impl Iterator for Env { #[cfg(target_os = "macos")] pub unsafe fn environ() -> *mut *const *const c_char { - extern "C" { - fn _NSGetEnviron() -> *mut *const *const c_char; - } - _NSGetEnviron() + libc::_NSGetEnviron() as *mut *const *const c_char } #[cfg(not(target_os = "macos"))] @@ -463,21 +498,20 @@ pub unsafe fn environ() -> *mut *const *const c_char { extern "C" { static mut environ: *const *const c_char; } - &mut environ + ptr::addr_of_mut!(environ) } -pub unsafe fn env_lock() -> MutexGuard<'static> { - // We never call `ENV_LOCK.init()`, so it is UB to attempt to - // acquire this mutex reentrantly! - static ENV_LOCK: Mutex = Mutex::new(); - ENV_LOCK.lock() +static ENV_LOCK: RwLock<()> = RwLock::new(()); + +pub fn env_read_lock() -> impl Drop { + ENV_LOCK.read().unwrap_or_else(PoisonError::into_inner) } /// Returns a vector of (variable, value) byte-vector pairs for all the /// environment variables of the current process. pub fn env() -> Env { unsafe { - let _guard = env_lock(); + let _guard = env_read_lock(); let mut environ = *environ(); let mut result = Vec::new(); if !environ.is_null() { @@ -488,7 +522,7 @@ pub fn env() -> Env { environ = environ.add(1); } } - return Env { iter: result.into_iter(), _dont_send_or_sync_me: PhantomData }; + return Env { iter: result.into_iter() }; } fn parse(input: &[u8]) -> Option<(OsString, OsString)> { @@ -509,41 +543,38 @@ pub fn env() -> Env { } } -pub fn getenv(k: &OsStr) -> io::Result> { +pub fn getenv(k: &OsStr) -> Option { // environment variables with a nul byte can't be set, so their value is // always None as well - let k = CString::new(k.as_bytes())?; - unsafe { - let _guard = env_lock(); - let s = libc::getenv(k.as_ptr()) as *const libc::c_char; - let ret = if s.is_null() { - None - } else { - Some(OsStringExt::from_vec(CStr::from_ptr(s).to_bytes().to_vec())) - }; - Ok(ret) + let s = run_with_cstr(k.as_bytes(), |k| { + let _guard = env_read_lock(); + Ok(unsafe { libc::getenv(k.as_ptr()) } as *const libc::c_char) + }) + .ok()?; + if s.is_null() { + None + } else { + Some(OsStringExt::from_vec(unsafe { CStr::from_ptr(s) }.to_bytes().to_vec())) } } pub fn setenv(k: &OsStr, v: &OsStr) -> io::Result<()> { - let k = CString::new(k.as_bytes())?; - let v = CString::new(v.as_bytes())?; - - unsafe { - let _guard = env_lock(); - cvt(libc::setenv(k.as_ptr(), v.as_ptr(), 1)).map(drop) - } + run_with_cstr(k.as_bytes(), |k| { + run_with_cstr(v.as_bytes(), |v| { + let _guard = ENV_LOCK.write(); + cvt(unsafe { libc::setenv(k.as_ptr(), v.as_ptr(), 1) }).map(drop) + }) + }) } pub fn unsetenv(n: &OsStr) -> io::Result<()> { - let nbuf = CString::new(n.as_bytes())?; - - unsafe { - let _guard = env_lock(); - cvt(libc::unsetenv(nbuf.as_ptr())).map(drop) - } + run_with_cstr(n.as_bytes(), |nbuf| { + let _guard = ENV_LOCK.write(); + cvt(unsafe { libc::unsetenv(nbuf.as_ptr()) }).map(drop) + }) } +#[cfg(not(target_os = "espidf"))] pub fn page_size() -> usize { unsafe { libc::sysconf(libc::_SC_PAGESIZE) as usize } } @@ -564,8 +595,12 @@ pub fn home_dir() -> Option { #[cfg(any( target_os = "android", target_os = "ios", + target_os = "watchos", target_os = "emscripten", - target_os = "redox" + target_os = "redox", + target_os = "vxworks", + target_os = "espidf", + target_os = "horizon" ))] unsafe fn fallback() -> Option { None @@ -573,8 +608,12 @@ pub fn home_dir() -> Option { #[cfg(not(any( target_os = "android", target_os = "ios", + target_os = "watchos", target_os = "emscripten", - target_os = "redox" + target_os = "redox", + target_os = "vxworks", + target_os = "espidf", + target_os = "horizon" )))] unsafe fn fallback() -> Option { let amt = match libc::sysconf(libc::_SC_GETPW_R_SIZE_MAX) { @@ -613,22 +652,14 @@ pub fn getppid() -> u32 { unsafe { libc::getppid() as u32 } } -#[cfg(target_env = "gnu")] +#[cfg(all(target_os = "linux", target_env = "gnu"))] pub fn glibc_version() -> Option<(usize, usize)> { - if let Some(Ok(version_str)) = glibc_version_cstr().map(CStr::to_str) { - parse_glibc_version(version_str) - } else { - None - } -} - -#[cfg(target_env = "gnu")] -fn glibc_version_cstr() -> Option<&'static CStr> { - weak! { - fn gnu_get_libc_version() -> *const libc::c_char + extern "C" { + fn gnu_get_libc_version() -> *const libc::c_char; } - if let Some(f) = gnu_get_libc_version.get() { - unsafe { Some(CStr::from_ptr(f())) } + let version_cstr = unsafe { CStr::from_ptr(gnu_get_libc_version()) }; + if let Ok(version_str) = version_cstr.to_str() { + parse_glibc_version(version_str) } else { None } @@ -636,7 +667,7 @@ fn glibc_version_cstr() -> Option<&'static CStr> { // Returns Some((major, minor)) if the string is a valid "x.y" version, // ignoring any extra dot-separated parts. Otherwise return None. -#[cfg(target_env = "gnu")] +#[cfg(all(target_os = "linux", target_env = "gnu"))] fn parse_glibc_version(version: &str) -> Option<(usize, usize)> { let mut parsed_ints = version.split('.').map(str::parse::).fuse(); match (parsed_ints.next(), parsed_ints.next()) { @@ -644,30 +675,3 @@ fn parse_glibc_version(version: &str) -> Option<(usize, usize)> { _ => None, } } - -#[cfg(all(test, target_env = "gnu"))] -mod test { - use super::*; - - #[test] - fn test_glibc_version() { - // This mostly just tests that the weak linkage doesn't panic wildly... - glibc_version(); - } - - #[test] - fn test_parse_glibc_version() { - let cases = [ - ("0.0", Some((0, 0))), - ("01.+2", Some((1, 2))), - ("3.4.5.six", Some((3, 4))), - ("1", None), - ("1.-2", None), - ("1.foo", None), - ("foo.1", None), - ]; - for &(version_str, parsed) in cases.iter() { - assert_eq!(parsed, parse_glibc_version(version_str)); - } - } -} diff --git a/crux-mir/lib/std/src/sys/unix/os/tests.rs b/crux-mir/lib/std/src/sys/unix/os/tests.rs new file mode 100644 index 000000000..efc29955b --- /dev/null +++ b/crux-mir/lib/std/src/sys/unix/os/tests.rs @@ -0,0 +1,23 @@ +#[test] +#[cfg(all(target_os = "linux", target_env = "gnu"))] +fn test_glibc_version() { + // This mostly just tests that the weak linkage doesn't panic wildly... + super::glibc_version(); +} + +#[test] +#[cfg(all(target_os = "linux", target_env = "gnu"))] +fn test_parse_glibc_version() { + let cases = [ + ("0.0", Some((0, 0))), + ("01.+2", Some((1, 2))), + ("3.4.5.six", Some((3, 4))), + ("1", None), + ("1.-2", None), + ("1.foo", None), + ("foo.1", None), + ]; + for &(version_str, parsed) in cases.iter() { + assert_eq!(parsed, super::parse_glibc_version(version_str)); + } +} diff --git a/crux-mir/lib/std/src/sys_common/os_str_bytes.rs b/crux-mir/lib/std/src/sys/unix/os_str.rs similarity index 56% rename from crux-mir/lib/std/src/sys_common/os_str_bytes.rs rename to crux-mir/lib/std/src/sys/unix/os_str.rs index e965ea79a..017e2af29 100644 --- a/crux-mir/lib/std/src/sys_common/os_str_bytes.rs +++ b/crux-mir/lib/std/src/sys/unix/os_str.rs @@ -2,41 +2,59 @@ //! systems: just a `Vec`/`[u8]`. use crate::borrow::Cow; -use crate::ffi::{OsStr, OsString}; +use crate::collections::TryReserveError; use crate::fmt; +use crate::fmt::Write; use crate::mem; use crate::rc::Rc; use crate::str; use crate::sync::Arc; -use crate::sys_common::bytestring::debug_fmt_bytestring; -use crate::sys_common::{AsInner, FromInner, IntoInner}; +use crate::sys_common::{AsInner, IntoInner}; -use core::str::lossy::Utf8Lossy; +use core::str::Utf8Chunks; -#[derive(Clone, Hash)] -pub(crate) struct Buf { +#[cfg(test)] +#[path = "../unix/os_str/tests.rs"] +mod tests; + +#[derive(Hash)] +#[repr(transparent)] +pub struct Buf { pub inner: Vec, } -// FIXME: -// `Buf::as_slice` current implementation relies -// on `Slice` being layout-compatible with `[u8]`. -// When attribute privacy is implemented, `Slice` should be annotated as `#[repr(transparent)]`. -// Anyway, `Slice` representation and layout are considered implementation detail, are -// not documented and must not be relied upon. -pub(crate) struct Slice { +#[repr(transparent)] +pub struct Slice { pub inner: [u8], } impl fmt::Debug for Slice { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - debug_fmt_bytestring(&self.inner, formatter) + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&Utf8Chunks::new(&self.inner).debug(), f) } } impl fmt::Display for Slice { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&Utf8Lossy::from_bytes(&self.inner), formatter) + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // If we're the empty string then our iterator won't actually yield + // anything, so perform the formatting manually + if self.inner.is_empty() { + return "".fmt(f); + } + + for chunk in Utf8Chunks::new(&self.inner) { + let valid = chunk.valid(); + // If we successfully decoded the whole chunk as a valid string then + // we can return a direct formatting of the string which will also + // respect various formatting flags if possible. + if chunk.invalid().is_empty() { + return valid.fmt(f); + } + + f.write_str(valid)?; + f.write_char(char::REPLACEMENT_CHARACTER)?; + } + Ok(()) } } @@ -52,6 +70,18 @@ impl fmt::Display for Buf { } } +impl Clone for Buf { + #[inline] + fn clone(&self) -> Self { + Buf { inner: self.inner.clone() } + } + + #[inline] + fn clone_from(&mut self, source: &Self) { + self.inner.clone_from(&source.inner) + } +} + impl IntoInner> for Buf { fn into_inner(self) -> Vec { self.inner @@ -89,11 +119,21 @@ impl Buf { self.inner.reserve(additional) } + #[inline] + pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.inner.try_reserve(additional) + } + #[inline] pub fn reserve_exact(&mut self, additional: usize) { self.inner.reserve_exact(additional) } + #[inline] + pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.inner.try_reserve_exact(additional) + } + #[inline] pub fn shrink_to_fit(&mut self) { self.inner.shrink_to_fit() @@ -106,9 +146,20 @@ impl Buf { #[inline] pub fn as_slice(&self) -> &Slice { + // SAFETY: Slice just wraps [u8], + // and &*self.inner is &[u8], therefore + // transmuting &[u8] to &Slice is safe. unsafe { mem::transmute(&*self.inner) } } + #[inline] + pub fn as_mut_slice(&mut self) -> &mut Slice { + // SAFETY: Slice just wraps [u8], + // and &mut *self.inner is &mut [u8], therefore + // transmuting &mut [u8] to &mut Slice is safe. + unsafe { mem::transmute(&mut *self.inner) } + } + pub fn into_string(self) -> Result { String::from_utf8(self.inner).map_err(|p| Buf { inner: p.into_bytes() }) } @@ -162,6 +213,10 @@ impl Slice { Buf { inner: self.inner.to_vec() } } + pub fn clone_into(&self, buf: &mut Buf) { + self.inner.clone_into(&mut buf.inner) + } + #[inline] pub fn into_box(&self) -> Box { let boxed: Box<[u8]> = self.inner.into(); @@ -184,70 +239,34 @@ impl Slice { let rc: Rc<[u8]> = Rc::from(&self.inner); unsafe { Rc::from_raw(Rc::into_raw(rc) as *const Slice) } } -} -/// Platform-specific extensions to [`OsString`]. -/// -/// [`OsString`]: ../../../../std/ffi/struct.OsString.html -#[stable(feature = "rust1", since = "1.0.0")] -pub trait OsStringExt { - /// Creates an [`OsString`] from a byte vector. - /// - /// See the module documentation for an example. - /// - /// [`OsString`]: ../../../ffi/struct.OsString.html - #[stable(feature = "rust1", since = "1.0.0")] - fn from_vec(vec: Vec) -> Self; - - /// Yields the underlying byte vector of this [`OsString`]. - /// - /// See the module documentation for an example. - /// - /// [`OsString`]: ../../../ffi/struct.OsString.html - #[stable(feature = "rust1", since = "1.0.0")] - fn into_vec(self) -> Vec; -} + #[inline] + pub fn make_ascii_lowercase(&mut self) { + self.inner.make_ascii_lowercase() + } -#[stable(feature = "rust1", since = "1.0.0")] -impl OsStringExt for OsString { - fn from_vec(vec: Vec) -> OsString { - FromInner::from_inner(Buf { inner: vec }) + #[inline] + pub fn make_ascii_uppercase(&mut self) { + self.inner.make_ascii_uppercase() } - fn into_vec(self) -> Vec { - self.into_inner().inner + + #[inline] + pub fn to_ascii_lowercase(&self) -> Buf { + Buf { inner: self.inner.to_ascii_lowercase() } } -} -/// Platform-specific extensions to [`OsStr`]. -/// -/// [`OsStr`]: ../../../../std/ffi/struct.OsStr.html -#[stable(feature = "rust1", since = "1.0.0")] -pub trait OsStrExt { - #[stable(feature = "rust1", since = "1.0.0")] - /// Creates an [`OsStr`] from a byte slice. - /// - /// See the module documentation for an example. - /// - /// [`OsStr`]: ../../../ffi/struct.OsStr.html - fn from_bytes(slice: &[u8]) -> &Self; - - /// Gets the underlying byte view of the [`OsStr`] slice. - /// - /// See the module documentation for an example. - /// - /// [`OsStr`]: ../../../ffi/struct.OsStr.html - #[stable(feature = "rust1", since = "1.0.0")] - fn as_bytes(&self) -> &[u8]; -} + #[inline] + pub fn to_ascii_uppercase(&self) -> Buf { + Buf { inner: self.inner.to_ascii_uppercase() } + } -#[stable(feature = "rust1", since = "1.0.0")] -impl OsStrExt for OsStr { #[inline] - fn from_bytes(slice: &[u8]) -> &OsStr { - unsafe { mem::transmute(slice) } + pub fn is_ascii(&self) -> bool { + self.inner.is_ascii() } + #[inline] - fn as_bytes(&self) -> &[u8] { - &self.as_inner().inner + pub fn eq_ignore_ascii_case(&self, other: &Self) -> bool { + self.inner.eq_ignore_ascii_case(&other.inner) } } diff --git a/crux-mir/lib/std/src/sys/unix/os_str/tests.rs b/crux-mir/lib/std/src/sys/unix/os_str/tests.rs new file mode 100644 index 000000000..22ba0c923 --- /dev/null +++ b/crux-mir/lib/std/src/sys/unix/os_str/tests.rs @@ -0,0 +1,18 @@ +use super::*; + +#[test] +fn slice_debug_output() { + let input = Slice::from_u8_slice(b"\xF0hello,\tworld"); + let expected = r#""\xF0hello,\tworld""#; + let output = format!("{input:?}"); + + assert_eq!(output, expected); +} + +#[test] +fn display() { + assert_eq!( + "Hello\u{FFFD}\u{FFFD} There\u{FFFD} Goodbye", + Slice::from_u8_slice(b"Hello\xC0\x80 There\xE6\x83 Goodbye").to_string(), + ); +} diff --git a/crux-mir/lib/std/src/sys/unix/path.rs b/crux-mir/lib/std/src/sys/unix/path.rs index 840a7ae04..a98a69e2d 100644 --- a/crux-mir/lib/std/src/sys/unix/path.rs +++ b/crux-mir/lib/std/src/sys/unix/path.rs @@ -1,5 +1,7 @@ +use crate::env; use crate::ffi::OsStr; -use crate::path::Prefix; +use crate::io; +use crate::path::{Path, PathBuf, Prefix}; #[inline] pub fn is_sep_byte(b: u8) -> bool { @@ -11,9 +13,51 @@ pub fn is_verbatim_sep(b: u8) -> bool { b == b'/' } +#[inline] pub fn parse_prefix(_: &OsStr) -> Option> { None } pub const MAIN_SEP_STR: &str = "/"; pub const MAIN_SEP: char = '/'; + +/// Make a POSIX path absolute without changing its semantics. +pub(crate) fn absolute(path: &Path) -> io::Result { + // This is mostly a wrapper around collecting `Path::components`, with + // exceptions made where this conflicts with the POSIX specification. + // See 4.13 Pathname Resolution, IEEE Std 1003.1-2017 + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 + + // Get the components, skipping the redundant leading "." component if it exists. + let mut components = path.strip_prefix(".").unwrap_or(path).components(); + let path_os = path.as_os_str().bytes(); + + let mut normalized = if path.is_absolute() { + // "If a pathname begins with two successive characters, the + // first component following the leading characters may be + // interpreted in an implementation-defined manner, although more than + // two leading characters shall be treated as a single + // character." + if path_os.starts_with(b"//") && !path_os.starts_with(b"///") { + components.next(); + PathBuf::from("//") + } else { + PathBuf::new() + } + } else { + env::current_dir()? + }; + normalized.extend(components); + + // "Interfaces using pathname resolution may specify additional constraints + // when a pathname that does not name an existing directory contains at + // least one non- character and contains one or more trailing + // characters". + // A trailing is also meaningful if "a symbolic link is + // encountered during pathname resolution". + if path_os.ends_with(b"/") { + normalized.push(""); + } + + Ok(normalized) +} diff --git a/crux-mir/lib/std/src/sys/unix/pipe.rs b/crux-mir/lib/std/src/sys/unix/pipe.rs index 2a861c878..a744d0ab6 100644 --- a/crux-mir/lib/std/src/sys/unix/pipe.rs +++ b/crux-mir/lib/std/src/sys/unix/pipe.rs @@ -1,10 +1,9 @@ use crate::io::{self, IoSlice, IoSliceMut}; use crate::mem; -use crate::sync::atomic::{AtomicBool, Ordering}; +use crate::os::unix::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, RawFd}; use crate::sys::fd::FileDesc; use crate::sys::{cvt, cvt_r}; - -use libc::c_int; +use crate::sys_common::IntoInner; //////////////////////////////////////////////////////////////////////////////// // Anonymous pipes @@ -13,46 +12,36 @@ use libc::c_int; pub struct AnonPipe(FileDesc); pub fn anon_pipe() -> io::Result<(AnonPipe, AnonPipe)> { - syscall! { fn pipe2(fds: *mut c_int, flags: c_int) -> c_int } - static INVALID: AtomicBool = AtomicBool::new(false); - let mut fds = [0; 2]; - // Unfortunately the only known way right now to create atomically set the - // CLOEXEC flag is to use the `pipe2` syscall on Linux. This was added in - // 2.6.27, however, and because we support 2.6.18 we must detect this - // support dynamically. - if cfg!(any( - target_os = "dragonfly", - target_os = "freebsd", - target_os = "linux", - target_os = "netbsd", - target_os = "openbsd", - target_os = "redox" - )) && !INVALID.load(Ordering::SeqCst) - { - // Note that despite calling a glibc function here we may still - // get ENOSYS. Glibc has `pipe2` since 2.9 and doesn't try to - // emulate on older kernels, so if you happen to be running on - // an older kernel you may see `pipe2` as a symbol but still not - // see the syscall. - match cvt(unsafe { pipe2(fds.as_mut_ptr(), libc::O_CLOEXEC) }) { - Ok(_) => { - return Ok((AnonPipe(FileDesc::new(fds[0])), AnonPipe(FileDesc::new(fds[1])))); + // The only known way right now to create atomically set the CLOEXEC flag is + // to use the `pipe2` syscall. This was added to Linux in 2.6.27, glibc 2.9 + // and musl 0.9.3, and some other targets also have it. + cfg_if::cfg_if! { + if #[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_os = "redox" + ))] { + unsafe { + cvt(libc::pipe2(fds.as_mut_ptr(), libc::O_CLOEXEC))?; + Ok((AnonPipe(FileDesc::from_raw_fd(fds[0])), AnonPipe(FileDesc::from_raw_fd(fds[1])))) } - Err(ref e) if e.raw_os_error() == Some(libc::ENOSYS) => { - INVALID.store(true, Ordering::SeqCst); + } else { + unsafe { + cvt(libc::pipe(fds.as_mut_ptr()))?; + + let fd0 = FileDesc::from_raw_fd(fds[0]); + let fd1 = FileDesc::from_raw_fd(fds[1]); + fd0.set_cloexec()?; + fd1.set_cloexec()?; + Ok((AnonPipe(fd0), AnonPipe(fd1))) } - Err(e) => return Err(e), } } - cvt(unsafe { libc::pipe(fds.as_mut_ptr()) })?; - - let fd0 = FileDesc::new(fds[0]); - let fd1 = FileDesc::new(fds[1]); - fd0.set_cloexec()?; - fd1.set_cloexec()?; - Ok((AnonPipe(fd0), AnonPipe(fd1))) } impl AnonPipe { @@ -64,6 +53,15 @@ impl AnonPipe { self.0.read_vectored(bufs) } + #[inline] + pub fn is_read_vectored(&self) -> bool { + self.0.is_read_vectored() + } + + pub fn read_to_end(&self, buf: &mut Vec) -> io::Result { + self.0.read_to_end(buf) + } + pub fn write(&self, buf: &[u8]) -> io::Result { self.0.write(buf) } @@ -72,10 +70,14 @@ impl AnonPipe { self.0.write_vectored(bufs) } - pub fn fd(&self) -> &FileDesc { - &self.0 + #[inline] + pub fn is_write_vectored(&self) -> bool { + self.0.is_write_vectored() } - pub fn into_fd(self) -> FileDesc { +} + +impl IntoInner for AnonPipe { + fn into_inner(self) -> FileDesc { self.0 } } @@ -83,15 +85,15 @@ impl AnonPipe { pub fn read2(p1: AnonPipe, v1: &mut Vec, p2: AnonPipe, v2: &mut Vec) -> io::Result<()> { // Set both pipes into nonblocking mode as we're gonna be reading from both // in the `select` loop below, and we wouldn't want one to block the other! - let p1 = p1.into_fd(); - let p2 = p2.into_fd(); + let p1 = p1.into_inner(); + let p2 = p2.into_inner(); p1.set_nonblocking(true)?; p2.set_nonblocking(true)?; let mut fds: [libc::pollfd; 2] = unsafe { mem::zeroed() }; - fds[0].fd = p1.raw(); + fds[0].fd = p1.as_raw_fd(); fds[0].events = libc::POLLIN; - fds[1].fd = p2.raw(); + fds[1].fd = p2.as_raw_fd(); fds[1].events = libc::POLLIN; loop { // wait for either pipe to become readable using `poll` @@ -127,3 +129,27 @@ pub fn read2(p1: AnonPipe, v1: &mut Vec, p2: AnonPipe, v2: &mut Vec) -> } } } + +impl AsRawFd for AnonPipe { + fn as_raw_fd(&self) -> RawFd { + self.0.as_raw_fd() + } +} + +impl AsFd for AnonPipe { + fn as_fd(&self) -> BorrowedFd<'_> { + self.0.as_fd() + } +} + +impl IntoRawFd for AnonPipe { + fn into_raw_fd(self) -> RawFd { + self.0.into_raw_fd() + } +} + +impl FromRawFd for AnonPipe { + unsafe fn from_raw_fd(raw_fd: RawFd) -> Self { + Self(FromRawFd::from_raw_fd(raw_fd)) + } +} diff --git a/crux-mir/lib/std/src/sys/unix/process/mod.rs b/crux-mir/lib/std/src/sys/unix/process/mod.rs index 553e980f0..3701510f3 100644 --- a/crux-mir/lib/std/src/sys/unix/process/mod.rs +++ b/crux-mir/lib/std/src/sys/unix/process/mod.rs @@ -1,13 +1,24 @@ -pub use self::process_common::{Command, ExitCode, Stdio, StdioPipes}; -pub use self::process_inner::{ExitStatus, Process}; +pub use self::process_common::{Command, CommandArgs, ExitCode, Stdio, StdioPipes}; +pub use self::process_inner::{ExitStatus, ExitStatusError, Process}; pub use crate::ffi::OsString as EnvKey; +pub use crate::sys_common::process::CommandEnvs; +#[cfg_attr(any(target_os = "espidf", target_os = "horizon"), allow(unused))] mod process_common; -#[cfg(not(target_os = "fuchsia"))] -#[path = "process_unix.rs"] -mod process_inner; -#[cfg(target_os = "fuchsia")] -#[path = "process_fuchsia.rs"] -mod process_inner; -#[cfg(target_os = "fuchsia")] -mod zircon; + +cfg_if::cfg_if! { + if #[cfg(target_os = "fuchsia")] { + #[path = "process_fuchsia.rs"] + mod process_inner; + mod zircon; + } else if #[cfg(target_os = "vxworks")] { + #[path = "process_vxworks.rs"] + mod process_inner; + } else if #[cfg(any(target_os = "espidf", target_os = "horizon"))] { + #[path = "process_unsupported.rs"] + mod process_inner; + } else { + #[path = "process_unix.rs"] + mod process_inner; + } +} diff --git a/crux-mir/lib/std/src/sys/unix/process/process_common.rs b/crux-mir/lib/std/src/sys/unix/process/process_common.rs index 859da691a..afd03d79c 100644 --- a/crux-mir/lib/std/src/sys/unix/process/process_common.rs +++ b/crux-mir/lib/std/src/sys/unix/process/process_common.rs @@ -1,25 +1,32 @@ +#[cfg(all(test, not(target_os = "emscripten")))] +mod tests; + use crate::os::unix::prelude::*; use crate::collections::BTreeMap; use crate::ffi::{CStr, CString, OsStr, OsString}; use crate::fmt; use crate::io; +use crate::path::Path; use crate::ptr; use crate::sys::fd::FileDesc; use crate::sys::fs::File; use crate::sys::pipe::{self, AnonPipe}; -use crate::sys_common::process::CommandEnv; +use crate::sys_common::process::{CommandEnv, CommandEnvs}; +use crate::sys_common::IntoInner; #[cfg(not(target_os = "fuchsia"))] use crate::sys::fs::OpenOptions; -use libc::{c_char, c_int, gid_t, uid_t, EXIT_FAILURE, EXIT_SUCCESS}; +use libc::{c_char, c_int, gid_t, pid_t, uid_t, EXIT_FAILURE, EXIT_SUCCESS}; cfg_if::cfg_if! { if #[cfg(target_os = "fuchsia")] { // fuchsia doesn't have /dev/null } else if #[cfg(target_os = "redox")] { const DEV_NULL: &str = "null:\0"; + } else if #[cfg(target_os = "vxworks")] { + const DEV_NULL: &str = "/null\0"; } else { const DEV_NULL: &str = "/dev/null\0"; } @@ -28,20 +35,43 @@ cfg_if::cfg_if! { // Android with api less than 21 define sig* functions inline, so it is not // available for dynamic link. Implementing sigemptyset and sigaddset allow us // to support older Android version (independent of libc version). -// The following implementations are based on https://git.io/vSkNf +// The following implementations are based on +// https://github.com/aosp-mirror/platform_bionic/blob/ad8dcd6023294b646e5a8288c0ed431b0845da49/libc/include/android/legacy_signal_inlines.h cfg_if::cfg_if! { if #[cfg(target_os = "android")] { + #[allow(dead_code)] pub unsafe fn sigemptyset(set: *mut libc::sigset_t) -> libc::c_int { set.write_bytes(0u8, 1); return 0; } + #[allow(dead_code)] pub unsafe fn sigaddset(set: *mut libc::sigset_t, signum: libc::c_int) -> libc::c_int { - use crate::{slice, mem}; + use crate::{ + mem::{align_of, size_of}, + slice, + }; + use libc::{c_ulong, sigset_t}; + + // The implementations from bionic (android libc) type pun `sigset_t` as an + // array of `c_ulong`. This works, but lets add a smoke check to make sure + // that doesn't change. + const _: () = assert!( + align_of::() == align_of::() + && (size_of::() % size_of::()) == 0 + ); - let raw = slice::from_raw_parts_mut(set as *mut u8, mem::size_of::()); let bit = (signum - 1) as usize; - raw[bit / 8] |= 1 << (bit % 8); + if set.is_null() || bit >= (8 * size_of::()) { + crate::sys::unix::os::set_errno(libc::EINVAL); + return -1; + } + let raw = slice::from_raw_parts_mut( + set as *mut c_ulong, + size_of::() / size_of::(), + ); + const LONG_BIT: usize = size_of::() * 8; + raw[bit / LONG_BIT] |= 1 << (bit % LONG_BIT); return 0; } } else { @@ -54,43 +84,38 @@ cfg_if::cfg_if! { //////////////////////////////////////////////////////////////////////////////// pub struct Command { - // Currently we try hard to ensure that the call to `.exec()` doesn't - // actually allocate any memory. While many platforms try to ensure that - // memory allocation works after a fork in a multithreaded process, it's - // been observed to be buggy and somewhat unreliable, so we do our best to - // just not do it at all! - // - // Along those lines, the `argv` and `envp` raw pointers here are exactly - // what's gonna get passed to `execvp`. The `argv` array starts with the - // `program` and ends with a NULL, and the `envp` pointer, if present, is - // also null-terminated. - // - // Right now we don't support removing arguments, so there's no much fancy - // support there, but we support adding and removing environment variables, - // so a side table is used to track where in the `envp` array each key is - // located. Whenever we add a key we update it in place if it's already - // present, and whenever we remove a key we update the locations of all - // other keys. program: CString, args: Vec, + /// Exactly what will be passed to `execvp`. + /// + /// First element is a pointer to `program`, followed by pointers to + /// `args`, followed by a `null`. Be careful when modifying `program` or + /// `args` to properly update this as well. argv: Argv, env: CommandEnv, + program_kind: ProgramKind, cwd: Option, uid: Option, gid: Option, saw_nul: bool, closures: Vec io::Result<()> + Send + Sync>>, + groups: Option>, stdin: Option, stdout: Option, stderr: Option, + #[cfg(target_os = "linux")] + create_pidfd: bool, + pgroup: Option, } -// Create a new type for argv, so that we can make it `Send` +// Create a new type for argv, so that we can make it `Send` and `Sync` struct Argv(Vec<*const c_char>); -// It is safe to make Argv Send, because it contains pointers to memory owned by `Command.args` +// It is safe to make `Argv` `Send` and `Sync`, because it contains +// pointers to memory owned by `Command.args` unsafe impl Send for Argv {} +unsafe impl Sync for Argv {} // passed back to std::process with the pipes connected to the child, if any // were requested @@ -119,6 +144,7 @@ pub enum ChildStdio { Null, } +#[derive(Debug)] pub enum Stdio { Inherit, Null, @@ -126,23 +152,76 @@ pub enum Stdio { Fd(FileDesc), } +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum ProgramKind { + /// A program that would be looked up on the PATH (e.g. `ls`) + PathLookup, + /// A relative path (e.g. `my-dir/foo`, `../foo`, `./foo`) + Relative, + /// An absolute path. + Absolute, +} + +impl ProgramKind { + fn new(program: &OsStr) -> Self { + if program.bytes().starts_with(b"/") { + Self::Absolute + } else if program.bytes().contains(&b'/') { + // If the program has more than one component in it, it is a relative path. + Self::Relative + } else { + Self::PathLookup + } + } +} + impl Command { + #[cfg(not(target_os = "linux"))] pub fn new(program: &OsStr) -> Command { let mut saw_nul = false; + let program_kind = ProgramKind::new(program.as_ref()); let program = os2c(program, &mut saw_nul); Command { argv: Argv(vec![program.as_ptr(), ptr::null()]), args: vec![program.clone()], program, + program_kind, env: Default::default(), cwd: None, uid: None, gid: None, saw_nul, closures: Vec::new(), + groups: None, stdin: None, stdout: None, stderr: None, + pgroup: None, + } + } + + #[cfg(target_os = "linux")] + pub fn new(program: &OsStr) -> Command { + let mut saw_nul = false; + let program_kind = ProgramKind::new(program.as_ref()); + let program = os2c(program, &mut saw_nul); + Command { + argv: Argv(vec![program.as_ptr(), ptr::null()]), + args: vec![program.clone()], + program, + program_kind, + env: Default::default(), + cwd: None, + uid: None, + gid: None, + saw_nul, + closures: Vec::new(), + groups: None, + stdin: None, + stdout: None, + stderr: None, + create_pidfd: false, + pgroup: None, } } @@ -155,7 +234,7 @@ impl Command { } pub fn arg(&mut self, arg: &OsStr) { - // Overwrite the trailing NULL pointer in `argv` and then add a new null + // Overwrite the trailing null pointer in `argv` and then add a new null // pointer. let arg = os2c(arg, &mut self.saw_nul); self.argv.0[self.args.len()] = arg.as_ptr(); @@ -175,15 +254,61 @@ impl Command { pub fn gid(&mut self, id: gid_t) { self.gid = Some(id); } + pub fn groups(&mut self, groups: &[gid_t]) { + self.groups = Some(Box::from(groups)); + } + pub fn pgroup(&mut self, pgroup: pid_t) { + self.pgroup = Some(pgroup); + } + + #[cfg(target_os = "linux")] + pub fn create_pidfd(&mut self, val: bool) { + self.create_pidfd = val; + } + + #[cfg(not(target_os = "linux"))] + #[allow(dead_code)] + pub fn get_create_pidfd(&self) -> bool { + false + } + + #[cfg(target_os = "linux")] + pub fn get_create_pidfd(&self) -> bool { + self.create_pidfd + } pub fn saw_nul(&self) -> bool { self.saw_nul } + + pub fn get_program(&self) -> &OsStr { + OsStr::from_bytes(self.program.as_bytes()) + } + + #[allow(dead_code)] + pub fn get_program_kind(&self) -> ProgramKind { + self.program_kind + } + + pub fn get_args(&self) -> CommandArgs<'_> { + let mut iter = self.args.iter(); + iter.next(); + CommandArgs { iter } + } + + pub fn get_envs(&self) -> CommandEnvs<'_> { + self.env.iter() + } + + pub fn get_current_dir(&self) -> Option<&Path> { + self.cwd.as_ref().map(|cs| Path::new(OsStr::from_bytes(cs.as_bytes()))) + } + pub fn get_argv(&self) -> &Vec<*const c_char> { &self.argv.0 } - pub fn get_program(&self) -> &CStr { + pub fn get_program_cstr(&self) -> &CStr { &*self.program } @@ -199,6 +324,14 @@ impl Command { pub fn get_gid(&self) -> Option { self.gid } + #[allow(dead_code)] + pub fn get_groups(&self) -> Option<&[gid_t]> { + self.groups.as_deref() + } + #[allow(dead_code)] + pub fn get_pgroup(&self) -> Option { + self.pgroup + } pub fn get_closures(&mut self) -> &mut Vec io::Result<()> + Send + Sync>> { &mut self.closures @@ -228,11 +361,17 @@ impl Command { let maybe_env = self.env.capture_if_changed(); maybe_env.map(|env| construct_envp(env, &mut self.saw_nul)) } + #[allow(dead_code)] pub fn env_saw_path(&self) -> bool { self.env.have_changed_path() } + #[allow(dead_code)] + pub fn program_is_path(&self) -> bool { + self.program.to_bytes().contains(&b'/') + } + pub fn setup_io( &self, default: Stdio, @@ -317,17 +456,17 @@ impl Stdio { // stderr. No matter which we dup first, the second will get // overwritten prematurely. Stdio::Fd(ref fd) => { - if fd.raw() >= 0 && fd.raw() <= libc::STDERR_FILENO { + if fd.as_raw_fd() >= 0 && fd.as_raw_fd() <= libc::STDERR_FILENO { Ok((ChildStdio::Owned(fd.duplicate()?), None)) } else { - Ok((ChildStdio::Explicit(fd.raw()), None)) + Ok((ChildStdio::Explicit(fd.as_raw_fd()), None)) } } Stdio::MakePipe => { let (reader, writer) = pipe::anon_pipe()?; let (ours, theirs) = if readable { (writer, reader) } else { (reader, writer) }; - Ok((ChildStdio::Owned(theirs.into_fd()), Some(ours))) + Ok((ChildStdio::Owned(theirs.into_inner()), Some(ours))) } #[cfg(not(target_os = "fuchsia"))] @@ -337,7 +476,7 @@ impl Stdio { opts.write(!readable); let path = unsafe { CStr::from_ptr(DEV_NULL.as_ptr() as *const _) }; let fd = File::open_c(&path, &opts)?; - Ok((ChildStdio::Owned(fd.into_fd()), None)) + Ok((ChildStdio::Owned(fd.into_inner()), None)) } #[cfg(target_os = "fuchsia")] @@ -348,13 +487,13 @@ impl Stdio { impl From for Stdio { fn from(pipe: AnonPipe) -> Stdio { - Stdio::Fd(pipe.into_fd()) + Stdio::Fd(pipe.into_inner()) } } impl From for Stdio { fn from(file: File) -> Stdio { - Stdio::Fd(file.into_fd()) + Stdio::Fd(file.into_inner()) } } @@ -363,7 +502,7 @@ impl ChildStdio { match *self { ChildStdio::Inherit => None, ChildStdio::Explicit(fd) => Some(fd), - ChildStdio::Owned(ref fd) => Some(fd.raw()), + ChildStdio::Owned(ref fd) => Some(fd.as_raw_fd()), #[cfg(target_os = "fuchsia")] ChildStdio::Null => None, @@ -372,22 +511,80 @@ impl ChildStdio { } impl fmt::Debug for Command { + // show all attributes but `self.closures` which does not implement `Debug` + // and `self.argv` which is not useful for debugging fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.program != self.args[0] { - write!(f, "[{:?}] ", self.program)?; - } - write!(f, "{:?}", self.args[0])?; + if f.alternate() { + let mut debug_command = f.debug_struct("Command"); + debug_command.field("program", &self.program).field("args", &self.args); + if !self.env.is_unchanged() { + debug_command.field("env", &self.env); + } + + if self.cwd.is_some() { + debug_command.field("cwd", &self.cwd); + } + if self.uid.is_some() { + debug_command.field("uid", &self.uid); + } + if self.gid.is_some() { + debug_command.field("gid", &self.gid); + } + + if self.groups.is_some() { + debug_command.field("groups", &self.groups); + } - for arg in &self.args[1..] { - write!(f, " {:?}", arg)?; + if self.stdin.is_some() { + debug_command.field("stdin", &self.stdin); + } + if self.stdout.is_some() { + debug_command.field("stdout", &self.stdout); + } + if self.stderr.is_some() { + debug_command.field("stderr", &self.stderr); + } + if self.pgroup.is_some() { + debug_command.field("pgroup", &self.pgroup); + } + + #[cfg(target_os = "linux")] + { + debug_command.field("create_pidfd", &self.create_pidfd); + } + + debug_command.finish() + } else { + if let Some(ref cwd) = self.cwd { + write!(f, "cd {cwd:?} && ")?; + } + for (key, value_opt) in self.get_envs() { + if let Some(value) = value_opt { + write!(f, "{}={value:?} ", key.to_string_lossy())?; + } + } + if self.program != self.args[0] { + write!(f, "[{:?}] ", self.program)?; + } + write!(f, "{:?}", self.args[0])?; + + for arg in &self.args[1..] { + write!(f, " {:?}", arg)?; + } + Ok(()) } - Ok(()) } } -#[derive(PartialEq, Eq, Clone, Copy, Debug)] +#[derive(PartialEq, Eq, Clone, Copy)] pub struct ExitCode(u8); +impl fmt::Debug for ExitCode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("unix_exit_status").field(&self.0).finish() + } +} + impl ExitCode { pub const SUCCESS: ExitCode = ExitCode(EXIT_SUCCESS as _); pub const FAILURE: ExitCode = ExitCode(EXIT_FAILURE as _); @@ -398,69 +595,37 @@ impl ExitCode { } } -#[cfg(all(test, not(target_os = "emscripten")))] -mod tests { - use super::*; - - use crate::ffi::OsStr; - use crate::mem; - use crate::ptr; - use crate::sys::cvt; - - macro_rules! t { - ($e:expr) => { - match $e { - Ok(t) => t, - Err(e) => panic!("received error for `{}`: {}", stringify!($e), e), - } - }; +impl From for ExitCode { + fn from(code: u8) -> Self { + Self(code) } +} - // See #14232 for more information, but it appears that signal delivery to a - // newly spawned process may just be raced in the macOS, so to prevent this - // test from being flaky we ignore it on macOS. - #[test] - #[cfg_attr(target_os = "macos", ignore)] - // When run under our current QEMU emulation test suite this test fails, - // although the reason isn't very clear as to why. For now this test is - // ignored there. - #[cfg_attr(target_arch = "arm", ignore)] - #[cfg_attr(target_arch = "aarch64", ignore)] - fn test_process_mask() { - unsafe { - // Test to make sure that a signal mask does not get inherited. - let mut cmd = Command::new(OsStr::new("cat")); - - let mut set = mem::MaybeUninit::::uninit(); - let mut old_set = mem::MaybeUninit::::uninit(); - t!(cvt(sigemptyset(set.as_mut_ptr()))); - t!(cvt(sigaddset(set.as_mut_ptr(), libc::SIGINT))); - t!(cvt(libc::pthread_sigmask(libc::SIG_SETMASK, set.as_ptr(), old_set.as_mut_ptr()))); - - cmd.stdin(Stdio::MakePipe); - cmd.stdout(Stdio::MakePipe); - - let (mut cat, mut pipes) = t!(cmd.spawn(Stdio::Null, true)); - let stdin_write = pipes.stdin.take().unwrap(); - let stdout_read = pipes.stdout.take().unwrap(); - - t!(cvt(libc::pthread_sigmask(libc::SIG_SETMASK, old_set.as_ptr(), ptr::null_mut()))); - - t!(cvt(libc::kill(cat.id() as libc::pid_t, libc::SIGINT))); - // We need to wait until SIGINT is definitely delivered. The - // easiest way is to write something to cat, and try to read it - // back: if SIGINT is unmasked, it'll get delivered when cat is - // next scheduled. - let _ = stdin_write.write(b"Hello"); - drop(stdin_write); - - // Either EOF or failure (EPIPE) is okay. - let mut buf = [0; 5]; - if let Ok(ret) = stdout_read.read(&mut buf) { - assert_eq!(ret, 0); - } +pub struct CommandArgs<'a> { + iter: crate::slice::Iter<'a, CString>, +} - t!(cat.wait()); - } +impl<'a> Iterator for CommandArgs<'a> { + type Item = &'a OsStr; + fn next(&mut self) -> Option<&'a OsStr> { + self.iter.next().map(|cs| OsStr::from_bytes(cs.as_bytes())) + } + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } +} + +impl<'a> ExactSizeIterator for CommandArgs<'a> { + fn len(&self) -> usize { + self.iter.len() + } + fn is_empty(&self) -> bool { + self.iter.is_empty() + } +} + +impl<'a> fmt::Debug for CommandArgs<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.iter.clone()).finish() } } diff --git a/crux-mir/lib/std/src/sys/unix/process/process_common/tests.rs b/crux-mir/lib/std/src/sys/unix/process/process_common/tests.rs new file mode 100644 index 000000000..03631e4e3 --- /dev/null +++ b/crux-mir/lib/std/src/sys/unix/process/process_common/tests.rs @@ -0,0 +1,161 @@ +use super::*; + +use crate::ffi::OsStr; +use crate::mem; +use crate::ptr; +use crate::sys::{cvt, cvt_nz}; + +macro_rules! t { + ($e:expr) => { + match $e { + Ok(t) => t, + Err(e) => panic!("received error for `{}`: {}", stringify!($e), e), + } + }; +} + +#[test] +#[cfg_attr( + any( + // See #14232 for more information, but it appears that signal delivery to a + // newly spawned process may just be raced in the macOS, so to prevent this + // test from being flaky we ignore it on macOS. + target_os = "macos", + // When run under our current QEMU emulation test suite this test fails, + // although the reason isn't very clear as to why. For now this test is + // ignored there. + target_arch = "arm", + target_arch = "aarch64", + target_arch = "riscv64", + ), + ignore +)] +fn test_process_mask() { + // Test to make sure that a signal mask *does* get inherited. + fn test_inner(mut cmd: Command) { + unsafe { + let mut set = mem::MaybeUninit::::uninit(); + let mut old_set = mem::MaybeUninit::::uninit(); + t!(cvt(sigemptyset(set.as_mut_ptr()))); + t!(cvt(sigaddset(set.as_mut_ptr(), libc::SIGINT))); + t!(cvt_nz(libc::pthread_sigmask( + libc::SIG_SETMASK, + set.as_ptr(), + old_set.as_mut_ptr() + ))); + + cmd.stdin(Stdio::MakePipe); + cmd.stdout(Stdio::MakePipe); + + let (mut cat, mut pipes) = t!(cmd.spawn(Stdio::Null, true)); + let stdin_write = pipes.stdin.take().unwrap(); + let stdout_read = pipes.stdout.take().unwrap(); + + t!(cvt_nz(libc::pthread_sigmask(libc::SIG_SETMASK, old_set.as_ptr(), ptr::null_mut()))); + + t!(cvt(libc::kill(cat.id() as libc::pid_t, libc::SIGINT))); + // We need to wait until SIGINT is definitely delivered. The + // easiest way is to write something to cat, and try to read it + // back: if SIGINT is unmasked, it'll get delivered when cat is + // next scheduled. + let _ = stdin_write.write(b"Hello"); + drop(stdin_write); + + // Exactly 5 bytes should be read. + let mut buf = [0; 5]; + let ret = t!(stdout_read.read(&mut buf)); + assert_eq!(ret, 5); + assert_eq!(&buf, b"Hello"); + + t!(cat.wait()); + } + } + + // A plain `Command::new` uses the posix_spawn path on many platforms. + let cmd = Command::new(OsStr::new("cat")); + test_inner(cmd); + + // Specifying `pre_exec` forces the fork/exec path. + let mut cmd = Command::new(OsStr::new("cat")); + unsafe { cmd.pre_exec(Box::new(|| Ok(()))) }; + test_inner(cmd); +} + +#[test] +#[cfg_attr( + any( + // See test_process_mask + target_os = "macos", + target_arch = "arm", + target_arch = "aarch64", + target_arch = "riscv64", + ), + ignore +)] +fn test_process_group_posix_spawn() { + unsafe { + // Spawn a cat subprocess that's just going to hang since there is no I/O. + let mut cmd = Command::new(OsStr::new("cat")); + cmd.pgroup(0); + cmd.stdin(Stdio::MakePipe); + cmd.stdout(Stdio::MakePipe); + let (mut cat, _pipes) = t!(cmd.spawn(Stdio::Null, true)); + + // Check that we can kill its process group, which means there *is* one. + t!(cvt(libc::kill(-(cat.id() as libc::pid_t), libc::SIGINT))); + + t!(cat.wait()); + } +} + +#[test] +#[cfg_attr( + any( + // See test_process_mask + target_os = "macos", + target_arch = "arm", + target_arch = "aarch64", + target_arch = "riscv64", + ), + ignore +)] +fn test_process_group_no_posix_spawn() { + unsafe { + // Same as above, create hang-y cat. This time, force using the non-posix_spawnp path. + let mut cmd = Command::new(OsStr::new("cat")); + cmd.pgroup(0); + cmd.pre_exec(Box::new(|| Ok(()))); // pre_exec forces fork + exec + cmd.stdin(Stdio::MakePipe); + cmd.stdout(Stdio::MakePipe); + let (mut cat, _pipes) = t!(cmd.spawn(Stdio::Null, true)); + + // Check that we can kill its process group, which means there *is* one. + t!(cvt(libc::kill(-(cat.id() as libc::pid_t), libc::SIGINT))); + + t!(cat.wait()); + } +} + +#[test] +fn test_program_kind() { + let vectors = &[ + ("foo", ProgramKind::PathLookup), + ("foo.out", ProgramKind::PathLookup), + ("./foo", ProgramKind::Relative), + ("../foo", ProgramKind::Relative), + ("dir/foo", ProgramKind::Relative), + // Note that paths on Unix can't contain / in them, so this is actually the directory "fo\\" + // followed by the file "o". + ("fo\\/o", ProgramKind::Relative), + ("/foo", ProgramKind::Absolute), + ("/dir/../foo", ProgramKind::Absolute), + ]; + + for (program, expected_kind) in vectors { + assert_eq!( + ProgramKind::new(program.as_ref()), + *expected_kind, + "actual != expected program kind for input {program}", + ); + } +} diff --git a/crux-mir/lib/std/src/sys/unix/process/process_fuchsia.rs b/crux-mir/lib/std/src/sys/unix/process/process_fuchsia.rs index f0bd1cdfe..d4c7e58b3 100644 --- a/crux-mir/lib/std/src/sys/unix/process/process_fuchsia.rs +++ b/crux-mir/lib/std/src/sys/unix/process/process_fuchsia.rs @@ -1,7 +1,7 @@ -use crate::convert::TryInto; use crate::fmt; use crate::io; use crate::mem; +use crate::num::{NonZeroI32, NonZeroI64}; use crate::ptr; use crate::sys::process::process_common::*; @@ -22,7 +22,7 @@ impl Command { let envp = self.capture_env(); if self.saw_nul() { - return Err(io::Error::new( + return Err(io::const_io_error!( io::ErrorKind::InvalidInput, "nul byte found in provided data", )); @@ -35,9 +35,17 @@ impl Command { Ok((Process { handle: Handle::new(process_handle) }, ours)) } + pub fn output(&mut self) -> io::Result<(ExitStatus, Vec, Vec)> { + let (proc, pipes) = self.spawn(Stdio::MakePipe, false)?; + crate::sys_common::process::wait_with_output(proc, pipes) + } + pub fn exec(&mut self, default: Stdio) -> io::Error { if self.saw_nul() { - return io::Error::new(io::ErrorKind::InvalidInput, "nul byte found in provided data"); + return io::const_io_error!( + io::ErrorKind::InvalidInput, + "nul byte found in provided data", + ); } match self.setup_io(default, true) { @@ -118,8 +126,9 @@ impl Command { FDIO_SPAWN_CLONE_JOB | FDIO_SPAWN_CLONE_LDSVC | FDIO_SPAWN_CLONE_NAMESPACE - | FDIO_SPAWN_CLONE_ENVIRON, // this is ignored when envp is non-null - self.get_program().as_ptr(), + | FDIO_SPAWN_CLONE_ENVIRON // this is ignored when envp is non-null + | FDIO_SPAWN_CLONE_UTC_CLOCK, + self.get_program_cstr().as_ptr(), self.get_argv().as_ptr(), envp, actions.len() as size_t, @@ -181,7 +190,7 @@ impl Process { ))?; } if actual != 1 { - return Err(io::Error::new( + return Err(io::const_io_error!( io::ErrorKind::InvalidData, "Failed to get exit status of process", )); @@ -206,7 +215,7 @@ impl Process { return Ok(None); } _ => { - panic!("Failed to wait on process handle: {}", status); + panic!("Failed to wait on process handle: {status}"); } } zx_cvt(zx_object_get_info( @@ -219,7 +228,7 @@ impl Process { ))?; } if actual != 1 { - return Err(io::Error::new( + return Err(io::const_io_error!( io::ErrorKind::InvalidData, "Failed to get exit status of process", )); @@ -232,8 +241,11 @@ impl Process { pub struct ExitStatus(i64); impl ExitStatus { - pub fn success(&self) -> bool { - self.code() == Some(0) + pub fn exit_ok(&self) -> Result<(), ExitStatusError> { + match NonZeroI64::try_from(self.0) { + /* was nonzero */ Ok(failure) => Err(ExitStatusError(failure)), + /* was zero, couldn't convert */ Err(_) => Ok(()), + } } pub fn code(&self) -> Option { @@ -244,8 +256,53 @@ impl ExitStatus { pub fn signal(&self) -> Option { None } + + // FIXME: The actually-Unix implementation in process_unix.rs uses WSTOPSIG, WCOREDUMP et al. + // I infer from the implementation of `success`, `code` and `signal` above that these are not + // available on Fuchsia. + // + // It does not appear that Fuchsia is Unix-like enough to implement ExitStatus (or indeed many + // other things from std::os::unix) properly. This veneer is always going to be a bodge. So + // while I don't know if these implementations are actually correct, I think they will do for + // now at least. + pub fn core_dumped(&self) -> bool { + false + } + pub fn stopped_signal(&self) -> Option { + None + } + pub fn continued(&self) -> bool { + false + } + + pub fn into_raw(&self) -> c_int { + // We don't know what someone who calls into_raw() will do with this value, but it should + // have the conventional Unix representation. Despite the fact that this is not + // standardised in SuS or POSIX, all Unix systems encode the signal and exit status the + // same way. (Ie the WIFEXITED, WEXITSTATUS etc. macros have identical behaviour on every + // Unix.) + // + // The caller of `std::os::unix::into_raw` is probably wanting a Unix exit status, and may + // do their own shifting and masking, or even pass the status to another computer running a + // different Unix variant. + // + // The other view would be to say that the caller on Fuchsia ought to know that `into_raw` + // will give a raw Fuchsia status (whatever that is - I don't know, personally). That is + // not possible here because we must return a c_int because that's what Unix (including + // SuS and POSIX) say a wait status is, but Fuchsia apparently uses a u64, so it won't + // necessarily fit. + // + // It seems to me that the right answer would be to provide std::os::fuchsia with its + // own ExitStatusExt, rather that trying to provide a not very convincing imitation of + // Unix. Ie, std::os::unix::process:ExitStatusExt ought not to exist on Fuchsia. But + // fixing this up that is beyond the scope of my efforts now. + let exit_status_as_if_unix: u8 = self.0.try_into().expect("Fuchsia process return code bigger than 8 bits, but std::os::unix::ExitStatusExt::into_raw() was called to try to convert the value into a traditional Unix-style wait status, which cannot represent values greater than 255."); + let wait_status_as_if_unix = (exit_status_as_if_unix as c_int) << 8; + wait_status_as_if_unix + } } +/// Converts a raw `c_int` to a type-safe `ExitStatus` by wrapping it without copying. impl From for ExitStatus { fn from(a: c_int) -> ExitStatus { ExitStatus(a as i64) @@ -257,3 +314,19 @@ impl fmt::Display for ExitStatus { write!(f, "exit code: {}", self.0) } } + +#[derive(PartialEq, Eq, Clone, Copy, Debug)] +pub struct ExitStatusError(NonZeroI64); + +impl Into for ExitStatusError { + fn into(self) -> ExitStatus { + ExitStatus(self.0.into()) + } +} + +impl ExitStatusError { + pub fn code(self) -> Option { + // fixme: affected by the same bug as ExitStatus::code() + ExitStatus(self.0.into()).code().map(|st| st.try_into().unwrap()) + } +} diff --git a/crux-mir/lib/std/src/sys/unix/process/process_unix.rs b/crux-mir/lib/std/src/sys/unix/process/process_unix.rs index 07d0fbf61..3bc17b775 100644 --- a/crux-mir/lib/std/src/sys/unix/process/process_unix.rs +++ b/crux-mir/lib/std/src/sys/unix/process/process_unix.rs @@ -1,11 +1,34 @@ use crate::fmt; use crate::io::{self, Error, ErrorKind}; -use crate::ptr; +use crate::mem; +use crate::num::NonZeroI32; use crate::sys; use crate::sys::cvt; use crate::sys::process::process_common::*; +use core::ffi::NonZero_c_int; -use libc::{c_int, gid_t, pid_t, uid_t}; +#[cfg(target_os = "linux")] +use crate::os::linux::process::PidFd; + +#[cfg(target_os = "linux")] +use crate::sys::weak::raw_syscall; + +#[cfg(any( + target_os = "macos", + target_os = "freebsd", + all(target_os = "linux", target_env = "gnu"), + all(target_os = "linux", target_env = "musl"), +))] +use crate::sys::weak::weak; + +#[cfg(target_os = "vxworks")] +use libc::RTP_ID as pid_t; + +#[cfg(not(target_os = "vxworks"))] +use libc::{c_int, pid_t}; + +#[cfg(not(any(target_os = "vxworks", target_os = "l4re")))] +use libc::{gid_t, uid_t}; //////////////////////////////////////////////////////////////////////////////// // Command @@ -17,12 +40,15 @@ impl Command { default: Stdio, needs_stdin: bool, ) -> io::Result<(Process, StdioPipes)> { - const CLOEXEC_MSG_FOOTER: &[u8] = b"NOEX"; + const CLOEXEC_MSG_FOOTER: [u8; 4] = *b"NOEX"; let envp = self.capture_env(); if self.saw_nul() { - return Err(io::Error::new(ErrorKind::InvalidInput, "nul byte found in provided data")); + return Err(io::const_io_error!( + ErrorKind::InvalidInput, + "nul byte found in provided data", + )); } let (ours, theirs) = self.setup_io(default, needs_stdin)?; @@ -40,40 +66,42 @@ impl Command { // // Note that as soon as we're done with the fork there's no need to hold // a lock any more because the parent won't do anything and the child is - // in its own process. - let result = unsafe { - let _env_lock = sys::os::env_lock(); - cvt(libc::fork())? - }; + // in its own process. Thus the parent drops the lock guard immediately. + // The child calls `mem::forget` to leak the lock, which is crucial because + // releasing a lock is not async-signal-safe. + let env_lock = sys::os::env_read_lock(); + let (pid, pidfd) = unsafe { self.do_fork()? }; - let pid = unsafe { - match result { - 0 => { - drop(input); - let Err(err) = self.do_exec(theirs, envp.as_ref()); - let errno = err.raw_os_error().unwrap_or(libc::EINVAL) as u32; - let bytes = [ - (errno >> 24) as u8, - (errno >> 16) as u8, - (errno >> 8) as u8, - (errno >> 0) as u8, - CLOEXEC_MSG_FOOTER[0], - CLOEXEC_MSG_FOOTER[1], - CLOEXEC_MSG_FOOTER[2], - CLOEXEC_MSG_FOOTER[3], - ]; - // pipe I/O up to PIPE_BUF bytes should be atomic, and then - // we want to be sure we *don't* run at_exit destructors as - // we're being torn down regardless - assert!(output.write(&bytes).is_ok()); - libc::_exit(1) - } - n => n, - } - }; + if pid == 0 { + crate::panic::always_abort(); + mem::forget(env_lock); // avoid non-async-signal-safe unlocking + drop(input); + let Err(err) = unsafe { self.do_exec(theirs, envp.as_ref()) }; + let errno = err.raw_os_error().unwrap_or(libc::EINVAL) as u32; + let errno = errno.to_be_bytes(); + let bytes = [ + errno[0], + errno[1], + errno[2], + errno[3], + CLOEXEC_MSG_FOOTER[0], + CLOEXEC_MSG_FOOTER[1], + CLOEXEC_MSG_FOOTER[2], + CLOEXEC_MSG_FOOTER[3], + ]; + // pipe I/O up to PIPE_BUF bytes should be atomic, and then + // we want to be sure we *don't* run at_exit destructors as + // we're being torn down regardless + rtassert!(output.write(&bytes).is_ok()); + unsafe { libc::_exit(1) } + } - let mut p = Process { pid: pid, status: None }; + drop(env_lock); drop(output); + + // Safety: We obtained the pidfd from calling `clone3` with + // `CLONE_PIDFD` so it's valid an otherwise unowned. + let mut p = unsafe { Process::new(pid, pidfd) }; let mut bytes = [0; 8]; // loop to handle EINTR @@ -81,19 +109,20 @@ impl Command { match input.read(&mut bytes) { Ok(0) => return Ok((p, ours)), Ok(8) => { - assert!( - combine(CLOEXEC_MSG_FOOTER) == combine(&bytes[4..8]), + let (errno, footer) = bytes.split_at(4); + assert_eq!( + CLOEXEC_MSG_FOOTER, footer, "Validation on the CLOEXEC pipe failed: {:?}", bytes ); - let errno = combine(&bytes[0..4]); + let errno = i32::from_be_bytes(errno.try_into().unwrap()); assert!(p.wait().is_ok(), "wait() should either return Ok or panic"); return Err(Error::from_raw_os_error(errno)); } Err(ref e) if e.kind() == ErrorKind::Interrupted => {} Err(e) => { assert!(p.wait().is_ok(), "wait() should either return Ok or panic"); - panic!("the CLOEXEC pipe failed: {:?}", e) + panic!("the CLOEXEC pipe failed: {e:?}") } Ok(..) => { // pipe I/O up to PIPE_BUF bytes should be atomic @@ -102,22 +131,105 @@ impl Command { } } } + } + + pub fn output(&mut self) -> io::Result<(ExitStatus, Vec, Vec)> { + let (proc, pipes) = self.spawn(Stdio::MakePipe, false)?; + crate::sys_common::process::wait_with_output(proc, pipes) + } - fn combine(arr: &[u8]) -> i32 { - let a = arr[0] as u32; - let b = arr[1] as u32; - let c = arr[2] as u32; - let d = arr[3] as u32; + // Attempts to fork the process. If successful, returns Ok((0, -1)) + // in the child, and Ok((child_pid, -1)) in the parent. + #[cfg(not(target_os = "linux"))] + unsafe fn do_fork(&mut self) -> Result<(pid_t, pid_t), io::Error> { + cvt(libc::fork()).map(|res| (res, -1)) + } + + // Attempts to fork the process. If successful, returns Ok((0, -1)) + // in the child, and Ok((child_pid, child_pidfd)) in the parent. + #[cfg(target_os = "linux")] + unsafe fn do_fork(&mut self) -> Result<(pid_t, pid_t), io::Error> { + use crate::sync::atomic::{AtomicBool, Ordering}; + + static HAS_CLONE3: AtomicBool = AtomicBool::new(true); + const CLONE_PIDFD: u64 = 0x00001000; + + #[repr(C)] + struct clone_args { + flags: u64, + pidfd: u64, + child_tid: u64, + parent_tid: u64, + exit_signal: u64, + stack: u64, + stack_size: u64, + tls: u64, + set_tid: u64, + set_tid_size: u64, + cgroup: u64, + } - ((a << 24) | (b << 16) | (c << 8) | (d << 0)) as i32 + raw_syscall! { + fn clone3(cl_args: *mut clone_args, len: libc::size_t) -> libc::c_long } + + // Bypassing libc for `clone3` can make further libc calls unsafe, + // so we use it sparingly for now. See #89522 for details. + // Some tools (e.g. sandboxing tools) may also expect `fork` + // rather than `clone3`. + let want_clone3_pidfd = self.get_create_pidfd(); + + // If we fail to create a pidfd for any reason, this will + // stay as -1, which indicates an error. + let mut pidfd: pid_t = -1; + + // Attempt to use the `clone3` syscall, which supports more arguments + // (in particular, the ability to create a pidfd). If this fails, + // we will fall through this block to a call to `fork()` + if want_clone3_pidfd && HAS_CLONE3.load(Ordering::Relaxed) { + let mut args = clone_args { + flags: CLONE_PIDFD, + pidfd: &mut pidfd as *mut pid_t as u64, + child_tid: 0, + parent_tid: 0, + exit_signal: libc::SIGCHLD as u64, + stack: 0, + stack_size: 0, + tls: 0, + set_tid: 0, + set_tid_size: 0, + cgroup: 0, + }; + + let args_ptr = &mut args as *mut clone_args; + let args_size = crate::mem::size_of::(); + + let res = cvt(clone3(args_ptr, args_size)); + match res { + Ok(n) => return Ok((n as pid_t, pidfd)), + Err(e) => match e.raw_os_error() { + // Multiple threads can race to execute this store, + // but that's fine - that just means that multiple threads + // will have tried and failed to execute the same syscall, + // with no other side effects. + Some(libc::ENOSYS) => HAS_CLONE3.store(false, Ordering::Relaxed), + // Fallback to fork if `EPERM` is returned. (e.g. blocked by seccomp) + Some(libc::EPERM) => {} + _ => return Err(e), + }, + } + } + + // Generally, we just call `fork`. If we get here after wanting `clone3`, + // then the syscall does not exist or we do not have permission to call it. + cvt(libc::fork()).map(|res| (res, pidfd)) } pub fn exec(&mut self, default: Stdio) -> io::Error { let envp = self.capture_env(); if self.saw_nul() { - return io::Error::new(ErrorKind::InvalidInput, "nul byte found in provided data"); + return io::const_io_error!(ErrorKind::InvalidInput, "nul byte found in provided data",); } match self.setup_io(default, true) { @@ -126,7 +238,7 @@ impl Command { // Similar to when forking, we want to ensure that access to // the environment is synchronized, so make sure to grab the // environment lock before we try to exec. - let _lock = sys::os::env_lock(); + let _lock = sys::os::env_read_lock(); let Err(e) = self.do_exec(theirs, envp.as_ref()); e @@ -185,20 +297,26 @@ impl Command { #[cfg(not(target_os = "l4re"))] { + if let Some(_g) = self.get_groups() { + //FIXME: Redox kernel does not support setgroups yet + #[cfg(not(target_os = "redox"))] + cvt(libc::setgroups(_g.len().try_into().unwrap(), _g.as_ptr()))?; + } if let Some(u) = self.get_gid() { cvt(libc::setgid(u as gid_t))?; } if let Some(u) = self.get_uid() { // When dropping privileges from root, the `setgroups` call - // will remove any extraneous groups. If we don't call this, - // then even though our uid has dropped, we may still have - // groups that enable us to do super-user things. This will - // fail if we aren't root, so don't bother checking the - // return value, this is just done as an optimistic - // privilege dropping function. + // will remove any extraneous groups. We only drop groups + // if the current uid is 0 and we weren't given an explicit + // set of groups. If we don't call this, then even though our + // uid has dropped, we may still have groups that enable us to + // do super-user things. //FIXME: Redox kernel does not support setgroups yet #[cfg(not(target_os = "redox"))] - let _ = libc::setgroups(0, ptr::null()); + if libc::getuid() == 0 && self.get_groups().is_none() { + cvt(libc::setgroups(0, crate::ptr::null()))?; + } cvt(libc::setuid(u as uid_t))?; } } @@ -206,23 +324,34 @@ impl Command { cvt(libc::chdir(cwd.as_ptr()))?; } + if let Some(pgroup) = self.get_pgroup() { + cvt(libc::setpgid(0, pgroup))?; + } + // emscripten has no signal support. #[cfg(not(target_os = "emscripten"))] { - use crate::mem::MaybeUninit; - // Reset signal handling so the child process starts in a - // standardized state. libstd ignores SIGPIPE, and signal-handling - // libraries often set a mask. Child processes inherit ignored - // signals and the signal mask from their parent, but most - // UNIX programs do not reset these things on their own, so we - // need to clean things up now to avoid confusing the program - // we're about to run. - let mut set = MaybeUninit::::uninit(); - cvt(sigemptyset(set.as_mut_ptr()))?; - cvt(libc::pthread_sigmask(libc::SIG_SETMASK, set.as_ptr(), ptr::null_mut()))?; - let ret = sys::signal(libc::SIGPIPE, libc::SIG_DFL); - if ret == libc::SIG_ERR { - return Err(io::Error::last_os_error()); + // Inherit the signal mask from the parent rather than resetting it (i.e. do not call + // pthread_sigmask). + + // If #[unix_sigpipe] is specified, don't reset SIGPIPE to SIG_DFL. + // If #[unix_sigpipe] is not specified, reset SIGPIPE to SIG_DFL for backward compatibility. + // + // #[unix_sigpipe] is an opportunity to change the default here. + if !crate::sys::unix_sigpipe_attr_specified() { + #[cfg(target_os = "android")] // see issue #88585 + { + let mut action: libc::sigaction = mem::zeroed(); + action.sa_sigaction = libc::SIG_DFL; + cvt(libc::sigaction(libc::SIGPIPE, &action, crate::ptr::null_mut()))?; + } + #[cfg(not(target_os = "android"))] + { + let ret = sys::signal(libc::SIGPIPE, libc::SIG_DFL); + if ret == libc::SIG_ERR { + return Err(io::Error::last_os_error()); + } + } } } @@ -251,14 +380,15 @@ impl Command { *sys::os::environ() = envp.as_ptr(); } - libc::execvp(self.get_program().as_ptr(), self.get_argv().as_ptr()); + libc::execvp(self.get_program_cstr().as_ptr(), self.get_argv().as_ptr()); Err(io::Error::last_os_error()) } #[cfg(not(any( target_os = "macos", target_os = "freebsd", - all(target_os = "linux", target_env = "gnu") + all(target_os = "linux", target_env = "gnu"), + all(target_os = "linux", target_env = "musl"), )))] fn posix_spawn( &mut self, @@ -273,7 +403,8 @@ impl Command { #[cfg(any( target_os = "macos", target_os = "freebsd", - all(target_os = "linux", target_env = "gnu") + all(target_os = "linux", target_env = "gnu"), + all(target_os = "linux", target_env = "musl"), ))] fn posix_spawn( &mut self, @@ -281,12 +412,14 @@ impl Command { envp: Option<&CStringArray>, ) -> io::Result> { use crate::mem::MaybeUninit; - use crate::sys; + use crate::sys::{self, cvt_nz, unix_sigpipe_attr_specified}; if self.get_gid().is_some() || self.get_uid().is_some() - || self.env_saw_path() + || (self.env_saw_path() && !self.program_is_path()) || !self.get_closures().is_empty() + || self.get_groups().is_some() + || self.get_create_pidfd() { return Ok(None); } @@ -303,10 +436,10 @@ impl Command { } } - // Solaris and glibc 2.29+ can set a new working directory, and maybe - // others will gain this non-POSIX function too. We'll check for this - // weak symbol as soon as it's needed, so we can return early otherwise - // to do a manual chdir before exec. + // Solaris, glibc 2.29+, and musl 1.24+ can set a new working directory, + // and maybe others will gain this non-POSIX function too. We'll check + // for this weak symbol as soon as it's needed, so we can return early + // otherwise to do a manual chdir before exec. weak! { fn posix_spawn_file_actions_addchdir_np( *mut libc::posix_spawn_file_actions_t, @@ -314,18 +447,33 @@ impl Command { ) -> libc::c_int } let addchdir = match self.get_cwd() { - Some(cwd) => match posix_spawn_file_actions_addchdir_np.get() { - Some(f) => Some((f, cwd)), - None => return Ok(None), - }, + Some(cwd) => { + if cfg!(target_os = "macos") { + // There is a bug in macOS where a relative executable + // path like "../myprogram" will cause `posix_spawn` to + // successfully launch the program, but erroneously return + // ENOENT when used with posix_spawn_file_actions_addchdir_np + // which was introduced in macOS 10.15. + if self.get_program_kind() == ProgramKind::Relative { + return Ok(None); + } + } + match posix_spawn_file_actions_addchdir_np.get() { + Some(f) => Some((f, cwd)), + None => return Ok(None), + } + } None => None, }; - let mut p = Process { pid: 0, status: None }; + let pgroup = self.get_pgroup(); - struct PosixSpawnFileActions(MaybeUninit); + // Safety: -1 indicates we don't have a pidfd. + let mut p = unsafe { Process::new(0, -1) }; - impl Drop for PosixSpawnFileActions { + struct PosixSpawnFileActions<'a>(&'a mut MaybeUninit); + + impl Drop for PosixSpawnFileActions<'_> { fn drop(&mut self) { unsafe { libc::posix_spawn_file_actions_destroy(self.0.as_mut_ptr()); @@ -333,9 +481,9 @@ impl Command { } } - struct PosixSpawnattr(MaybeUninit); + struct PosixSpawnattr<'a>(&'a mut MaybeUninit); - impl Drop for PosixSpawnattr { + impl Drop for PosixSpawnattr<'_> { fn drop(&mut self) { unsafe { libc::posix_spawnattr_destroy(self.0.as_mut_ptr()); @@ -344,58 +492,78 @@ impl Command { } unsafe { - let mut file_actions = PosixSpawnFileActions(MaybeUninit::uninit()); - let mut attrs = PosixSpawnattr(MaybeUninit::uninit()); + let mut attrs = MaybeUninit::uninit(); + cvt_nz(libc::posix_spawnattr_init(attrs.as_mut_ptr()))?; + let attrs = PosixSpawnattr(&mut attrs); + + let mut flags = 0; - libc::posix_spawnattr_init(attrs.0.as_mut_ptr()); - libc::posix_spawn_file_actions_init(file_actions.0.as_mut_ptr()); + let mut file_actions = MaybeUninit::uninit(); + cvt_nz(libc::posix_spawn_file_actions_init(file_actions.as_mut_ptr()))?; + let file_actions = PosixSpawnFileActions(&mut file_actions); if let Some(fd) = stdio.stdin.fd() { - cvt(libc::posix_spawn_file_actions_adddup2( + cvt_nz(libc::posix_spawn_file_actions_adddup2( file_actions.0.as_mut_ptr(), fd, libc::STDIN_FILENO, ))?; } if let Some(fd) = stdio.stdout.fd() { - cvt(libc::posix_spawn_file_actions_adddup2( + cvt_nz(libc::posix_spawn_file_actions_adddup2( file_actions.0.as_mut_ptr(), fd, libc::STDOUT_FILENO, ))?; } if let Some(fd) = stdio.stderr.fd() { - cvt(libc::posix_spawn_file_actions_adddup2( + cvt_nz(libc::posix_spawn_file_actions_adddup2( file_actions.0.as_mut_ptr(), fd, libc::STDERR_FILENO, ))?; } if let Some((f, cwd)) = addchdir { - cvt(f(file_actions.0.as_mut_ptr(), cwd.as_ptr()))?; + cvt_nz(f(file_actions.0.as_mut_ptr(), cwd.as_ptr()))?; } - let mut set = MaybeUninit::::uninit(); - cvt(sigemptyset(set.as_mut_ptr()))?; - cvt(libc::posix_spawnattr_setsigmask(attrs.0.as_mut_ptr(), set.as_ptr()))?; - cvt(sigaddset(set.as_mut_ptr(), libc::SIGPIPE))?; - cvt(libc::posix_spawnattr_setsigdefault(attrs.0.as_mut_ptr(), set.as_ptr()))?; + if let Some(pgroup) = pgroup { + flags |= libc::POSIX_SPAWN_SETPGROUP; + cvt_nz(libc::posix_spawnattr_setpgroup(attrs.0.as_mut_ptr(), pgroup))?; + } - let flags = libc::POSIX_SPAWN_SETSIGDEF | libc::POSIX_SPAWN_SETSIGMASK; - cvt(libc::posix_spawnattr_setflags(attrs.0.as_mut_ptr(), flags as _))?; + // Inherit the signal mask from this process rather than resetting it (i.e. do not call + // posix_spawnattr_setsigmask). + + // If #[unix_sigpipe] is specified, don't reset SIGPIPE to SIG_DFL. + // If #[unix_sigpipe] is not specified, reset SIGPIPE to SIG_DFL for backward compatibility. + // + // #[unix_sigpipe] is an opportunity to change the default here. + if !unix_sigpipe_attr_specified() { + let mut default_set = MaybeUninit::::uninit(); + cvt(sigemptyset(default_set.as_mut_ptr()))?; + cvt(sigaddset(default_set.as_mut_ptr(), libc::SIGPIPE))?; + cvt_nz(libc::posix_spawnattr_setsigdefault( + attrs.0.as_mut_ptr(), + default_set.as_ptr(), + ))?; + flags |= libc::POSIX_SPAWN_SETSIGDEF; + } + + cvt_nz(libc::posix_spawnattr_setflags(attrs.0.as_mut_ptr(), flags as _))?; // Make sure we synchronize access to the global `environ` resource - let _env_lock = sys::os::env_lock(); + let _env_lock = sys::os::env_read_lock(); let envp = envp.map(|c| c.as_ptr()).unwrap_or_else(|| *sys::os::environ() as *const _); - let ret = libc::posix_spawnp( + cvt_nz(libc::posix_spawnp( &mut p.pid, - self.get_program().as_ptr(), + self.get_program_cstr().as_ptr(), file_actions.0.as_ptr(), attrs.0.as_ptr(), self.get_argv().as_ptr() as *const _, envp as *const _, - ); - if ret == 0 { Ok(Some(p)) } else { Err(io::Error::from_raw_os_error(ret)) } + ))?; + Ok(Some(p)) } } } @@ -408,9 +576,29 @@ impl Command { pub struct Process { pid: pid_t, status: Option, + // On Linux, stores the pidfd created for this child. + // This is None if the user did not request pidfd creation, + // or if the pidfd could not be created for some reason + // (e.g. the `clone3` syscall was not available). + #[cfg(target_os = "linux")] + pidfd: Option, } impl Process { + #[cfg(target_os = "linux")] + unsafe fn new(pid: pid_t, pidfd: pid_t) -> Self { + use crate::os::unix::io::FromRawFd; + use crate::sys_common::FromInner; + // Safety: If `pidfd` is nonnegative, we assume it's valid and otherwise unowned. + let pidfd = (pidfd >= 0).then(|| PidFd::from_inner(sys::fd::FileDesc::from_raw_fd(pidfd))); + Process { pid, status: None, pidfd } + } + + #[cfg(not(target_os = "linux"))] + unsafe fn new(pid: pid_t, _pidfd: pid_t) -> Self { + Process { pid, status: None } + } + pub fn id(&self) -> u32 { self.pid as u32 } @@ -420,7 +608,7 @@ impl Process { // and used for another process, and we probably shouldn't be killing // random processes, so just return an error. if self.status.is_some() { - Err(Error::new( + Err(io::const_io_error!( ErrorKind::InvalidInput, "invalid argument: can't kill an exited process", )) @@ -456,44 +644,209 @@ impl Process { } /// Unix exit statuses -#[derive(PartialEq, Eq, Clone, Copy, Debug)] +// +// This is not actually an "exit status" in Unix terminology. Rather, it is a "wait status". +// See the discussion in comments and doc comments for `std::process::ExitStatus`. +#[derive(PartialEq, Eq, Clone, Copy)] pub struct ExitStatus(c_int); +impl fmt::Debug for ExitStatus { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("unix_wait_status").field(&self.0).finish() + } +} + impl ExitStatus { pub fn new(status: c_int) -> ExitStatus { ExitStatus(status) } fn exited(&self) -> bool { - unsafe { libc::WIFEXITED(self.0) } + libc::WIFEXITED(self.0) } - pub fn success(&self) -> bool { - self.code() == Some(0) + pub fn exit_ok(&self) -> Result<(), ExitStatusError> { + // This assumes that WIFEXITED(status) && WEXITSTATUS==0 corresponds to status==0. This is + // true on all actual versions of Unix, is widely assumed, and is specified in SuS + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/wait.html. If it is not + // true for a platform pretending to be Unix, the tests (our doctests, and also + // procsss_unix/tests.rs) will spot it. `ExitStatusError::code` assumes this too. + match NonZero_c_int::try_from(self.0) { + /* was nonzero */ Ok(failure) => Err(ExitStatusError(failure)), + /* was zero, couldn't convert */ Err(_) => Ok(()), + } } pub fn code(&self) -> Option { - if self.exited() { Some(unsafe { libc::WEXITSTATUS(self.0) }) } else { None } + self.exited().then(|| libc::WEXITSTATUS(self.0)) } pub fn signal(&self) -> Option { - if !self.exited() { Some(unsafe { libc::WTERMSIG(self.0) }) } else { None } + libc::WIFSIGNALED(self.0).then(|| libc::WTERMSIG(self.0)) + } + + pub fn core_dumped(&self) -> bool { + libc::WIFSIGNALED(self.0) && libc::WCOREDUMP(self.0) + } + + pub fn stopped_signal(&self) -> Option { + libc::WIFSTOPPED(self.0).then(|| libc::WSTOPSIG(self.0)) + } + + pub fn continued(&self) -> bool { + libc::WIFCONTINUED(self.0) + } + + pub fn into_raw(&self) -> c_int { + self.0 } } +/// Converts a raw `c_int` to a type-safe `ExitStatus` by wrapping it without copying. impl From for ExitStatus { fn from(a: c_int) -> ExitStatus { ExitStatus(a) } } +/// Convert a signal number to a readable, searchable name. +/// +/// This string should be displayed right after the signal number. +/// If a signal is unrecognized, it returns the empty string, so that +/// you just get the number like "0". If it is recognized, you'll get +/// something like "9 (SIGKILL)". +fn signal_string(signal: i32) -> &'static str { + match signal { + libc::SIGHUP => " (SIGHUP)", + libc::SIGINT => " (SIGINT)", + libc::SIGQUIT => " (SIGQUIT)", + libc::SIGILL => " (SIGILL)", + libc::SIGTRAP => " (SIGTRAP)", + libc::SIGABRT => " (SIGABRT)", + libc::SIGBUS => " (SIGBUS)", + libc::SIGFPE => " (SIGFPE)", + libc::SIGKILL => " (SIGKILL)", + libc::SIGUSR1 => " (SIGUSR1)", + libc::SIGSEGV => " (SIGSEGV)", + libc::SIGUSR2 => " (SIGUSR2)", + libc::SIGPIPE => " (SIGPIPE)", + libc::SIGALRM => " (SIGALRM)", + libc::SIGTERM => " (SIGTERM)", + libc::SIGCHLD => " (SIGCHLD)", + libc::SIGCONT => " (SIGCONT)", + libc::SIGSTOP => " (SIGSTOP)", + libc::SIGTSTP => " (SIGTSTP)", + libc::SIGTTIN => " (SIGTTIN)", + libc::SIGTTOU => " (SIGTTOU)", + libc::SIGURG => " (SIGURG)", + libc::SIGXCPU => " (SIGXCPU)", + libc::SIGXFSZ => " (SIGXFSZ)", + libc::SIGVTALRM => " (SIGVTALRM)", + libc::SIGPROF => " (SIGPROF)", + libc::SIGWINCH => " (SIGWINCH)", + #[cfg(not(target_os = "haiku"))] + libc::SIGIO => " (SIGIO)", + #[cfg(target_os = "haiku")] + libc::SIGPOLL => " (SIGPOLL)", + libc::SIGSYS => " (SIGSYS)", + // For information on Linux signals, run `man 7 signal` + #[cfg(all( + target_os = "linux", + any( + target_arch = "x86_64", + target_arch = "x86", + target_arch = "arm", + target_arch = "aarch64" + ) + ))] + libc::SIGSTKFLT => " (SIGSTKFLT)", + #[cfg(target_os = "linux")] + libc::SIGPWR => " (SIGPWR)", + #[cfg(any( + target_os = "macos", + target_os = "ios", + target_os = "tvos", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_os = "dragonfly" + ))] + libc::SIGEMT => " (SIGEMT)", + #[cfg(any( + target_os = "macos", + target_os = "ios", + target_os = "tvos", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_os = "dragonfly" + ))] + libc::SIGINFO => " (SIGINFO)", + _ => "", + } +} + impl fmt::Display for ExitStatus { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if let Some(code) = self.code() { - write!(f, "exit code: {}", code) + write!(f, "exit status: {code}") + } else if let Some(signal) = self.signal() { + let signal_string = signal_string(signal); + if self.core_dumped() { + write!(f, "signal: {signal}{signal_string} (core dumped)") + } else { + write!(f, "signal: {signal}{signal_string}") + } + } else if let Some(signal) = self.stopped_signal() { + let signal_string = signal_string(signal); + write!(f, "stopped (not terminated) by signal: {signal}{signal_string}") + } else if self.continued() { + write!(f, "continued (WIFCONTINUED)") } else { - let signal = self.signal().unwrap(); - write!(f, "signal: {}", signal) + write!(f, "unrecognised wait status: {} {:#x}", self.0, self.0) } } } + +#[derive(PartialEq, Eq, Clone, Copy)] +pub struct ExitStatusError(NonZero_c_int); + +impl Into for ExitStatusError { + fn into(self) -> ExitStatus { + ExitStatus(self.0.into()) + } +} + +impl fmt::Debug for ExitStatusError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("unix_wait_status").field(&self.0).finish() + } +} + +impl ExitStatusError { + pub fn code(self) -> Option { + ExitStatus(self.0.into()).code().map(|st| st.try_into().unwrap()) + } +} + +#[cfg(target_os = "linux")] +#[unstable(feature = "linux_pidfd", issue = "82971")] +impl crate::os::linux::process::ChildExt for crate::process::Child { + fn pidfd(&self) -> io::Result<&PidFd> { + self.handle + .pidfd + .as_ref() + .ok_or_else(|| Error::new(ErrorKind::Uncategorized, "No pidfd was created.")) + } + + fn take_pidfd(&mut self) -> io::Result { + self.handle + .pidfd + .take() + .ok_or_else(|| Error::new(ErrorKind::Uncategorized, "No pidfd was created.")) + } +} + +#[cfg(test)] +#[path = "process_unix/tests.rs"] +mod tests; diff --git a/crux-mir/lib/std/src/sys/unix/process/process_unix/tests.rs b/crux-mir/lib/std/src/sys/unix/process/process_unix/tests.rs new file mode 100644 index 000000000..e5e1f956b --- /dev/null +++ b/crux-mir/lib/std/src/sys/unix/process/process_unix/tests.rs @@ -0,0 +1,62 @@ +use crate::os::unix::process::{CommandExt, ExitStatusExt}; +use crate::panic::catch_unwind; +use crate::process::Command; + +// Many of the other aspects of this situation, including heap alloc concurrency +// safety etc., are tested in tests/ui/process/process-panic-after-fork.rs + +#[test] +fn exitstatus_display_tests() { + // In practice this is the same on every Unix. + // If some weird platform turns out to be different, and this test fails, use #[cfg]. + use crate::os::unix::process::ExitStatusExt; + use crate::process::ExitStatus; + + let t = |v, s| assert_eq!(s, format!("{}", ::from_raw(v))); + + t(0x0000f, "signal: 15 (SIGTERM)"); + t(0x0008b, "signal: 11 (SIGSEGV) (core dumped)"); + t(0x00000, "exit status: 0"); + t(0x0ff00, "exit status: 255"); + + // On MacOS, 0x0137f is WIFCONTINUED, not WIFSTOPPED. Probably *BSD is similar. + // https://github.com/rust-lang/rust/pull/82749#issuecomment-790525956 + // The purpose of this test is to test our string formatting, not our understanding of the wait + // status magic numbers. So restrict these to Linux. + if cfg!(target_os = "linux") { + t(0x0137f, "stopped (not terminated) by signal: 19 (SIGSTOP)"); + t(0x0ffff, "continued (WIFCONTINUED)"); + } + + // Testing "unrecognised wait status" is hard because the wait.h macros typically + // assume that the value came from wait and isn't mad. With the glibc I have here + // this works: + if cfg!(all(target_os = "linux", target_env = "gnu")) { + t(0x000ff, "unrecognised wait status: 255 0xff"); + } +} + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn test_command_fork_no_unwind() { + let got = catch_unwind(|| { + let mut c = Command::new("echo"); + c.arg("hi"); + unsafe { + c.pre_exec(|| panic!("{}", "crash now!")); + } + let st = c.status().expect("failed to get command status"); + dbg!(st); + st + }); + dbg!(&got); + let status = got.expect("panic unexpectedly propagated"); + dbg!(status); + let signal = status.signal().expect("expected child process to die of signal"); + assert!( + signal == libc::SIGABRT + || signal == libc::SIGILL + || signal == libc::SIGTRAP + || signal == libc::SIGSEGV + ); +} diff --git a/crux-mir/lib/std/src/sys/unix/process/process_unsupported.rs b/crux-mir/lib/std/src/sys/unix/process/process_unsupported.rs new file mode 100644 index 000000000..f28ca58d0 --- /dev/null +++ b/crux-mir/lib/std/src/sys/unix/process/process_unsupported.rs @@ -0,0 +1,122 @@ +use crate::fmt; +use crate::io; +use crate::num::NonZeroI32; +use crate::sys::process::process_common::*; +use crate::sys::unix::unsupported::*; +use core::ffi::NonZero_c_int; + +use libc::{c_int, pid_t}; + +//////////////////////////////////////////////////////////////////////////////// +// Command +//////////////////////////////////////////////////////////////////////////////// + +impl Command { + pub fn spawn( + &mut self, + _default: Stdio, + _needs_stdin: bool, + ) -> io::Result<(Process, StdioPipes)> { + unsupported() + } + + pub fn output(&mut self) -> io::Result<(ExitStatus, Vec, Vec)> { + unsupported() + } + + pub fn exec(&mut self, _default: Stdio) -> io::Error { + unsupported_err() + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Processes +//////////////////////////////////////////////////////////////////////////////// + +pub struct Process { + _handle: pid_t, +} + +impl Process { + pub fn id(&self) -> u32 { + 0 + } + + pub fn kill(&mut self) -> io::Result<()> { + unsupported() + } + + pub fn wait(&mut self) -> io::Result { + unsupported() + } + + pub fn try_wait(&mut self) -> io::Result> { + unsupported() + } +} + +#[derive(PartialEq, Eq, Clone, Copy, Debug)] +pub struct ExitStatus(c_int); + +impl ExitStatus { + #[cfg_attr(target_os = "horizon", allow(unused))] + pub fn success(&self) -> bool { + self.code() == Some(0) + } + + pub fn exit_ok(&self) -> Result<(), ExitStatusError> { + Err(ExitStatusError(1.try_into().unwrap())) + } + + pub fn code(&self) -> Option { + None + } + + pub fn signal(&self) -> Option { + None + } + + pub fn core_dumped(&self) -> bool { + false + } + + pub fn stopped_signal(&self) -> Option { + None + } + + pub fn continued(&self) -> bool { + false + } + + pub fn into_raw(&self) -> c_int { + 0 + } +} + +/// Converts a raw `c_int` to a type-safe `ExitStatus` by wrapping it without copying. +impl From for ExitStatus { + fn from(a: c_int) -> ExitStatus { + ExitStatus(a as i32) + } +} + +impl fmt::Display for ExitStatus { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "exit code: {}", self.0) + } +} + +#[derive(PartialEq, Eq, Clone, Copy, Debug)] +pub struct ExitStatusError(NonZero_c_int); + +impl Into for ExitStatusError { + fn into(self) -> ExitStatus { + ExitStatus(self.0.into()) + } +} + +impl ExitStatusError { + pub fn code(self) -> Option { + ExitStatus(self.0.into()).code().map(|st| st.try_into().unwrap()) + } +} diff --git a/crux-mir/lib/std/src/sys/vxworks/process/process_vxworks.rs b/crux-mir/lib/std/src/sys/unix/process/process_vxworks.rs similarity index 63% rename from crux-mir/lib/std/src/sys/vxworks/process/process_vxworks.rs rename to crux-mir/lib/std/src/sys/unix/process/process_vxworks.rs index f7e84ae3d..569a4b149 100644 --- a/crux-mir/lib/std/src/sys/vxworks/process/process_vxworks.rs +++ b/crux-mir/lib/std/src/sys/unix/process/process_vxworks.rs @@ -1,8 +1,11 @@ +use crate::fmt; use crate::io::{self, Error, ErrorKind}; +use crate::num::NonZeroI32; use crate::sys; use crate::sys::cvt; use crate::sys::process::process_common::*; use crate::sys_common::thread; +use core::ffi::NonZero_c_int; use libc::RTP_ID; use libc::{self, c_char, c_int}; @@ -17,11 +20,13 @@ impl Command { needs_stdin: bool, ) -> io::Result<(Process, StdioPipes)> { use crate::sys::cvt_r; - const CLOEXEC_MSG_FOOTER: &'static [u8] = b"NOEX"; let envp = self.capture_env(); if self.saw_nul() { - return Err(io::Error::new(ErrorKind::InvalidInput, "nul byte found in provided data")); + return Err(io::const_io_error!( + ErrorKind::InvalidInput, + "nul byte found in provided data", + )); } let (ours, theirs) = self.setup_io(default, needs_stdin)?; let mut p = Process { pid: 0, status: None }; @@ -57,6 +62,9 @@ impl Command { t!(cvt(libc::chdir(cwd.as_ptr()))); } + // pre_exec closures are ignored on VxWorks + let _ = self.get_closures(); + let c_envp = envp .as_ref() .map(|c| c.as_ptr()) @@ -64,10 +72,10 @@ impl Command { let stack_size = thread::min_stack(); // ensure that access to the environment is synchronized - let _lock = sys::os::env_lock(); + let _lock = sys::os::env_read_lock(); let ret = libc::rtpSpawn( - self.get_program().as_ptr(), + self.get_program_cstr().as_ptr(), self.get_argv().as_ptr() as *mut *const c_char, // argv c_envp as *mut *const c_char, 100 as c_int, // initial priority @@ -100,6 +108,11 @@ impl Command { } } + pub fn output(&mut self) -> io::Result<(ExitStatus, Vec, Vec)> { + let (proc, pipes) = self.spawn(Stdio::MakePipe, false)?; + crate::sys_common::process::wait_with_output(proc, pipes) + } + pub fn exec(&mut self, default: Stdio) -> io::Error { let ret = Command::spawn(self, default, false); match ret { @@ -133,7 +146,7 @@ impl Process { // and used for another process, and we probably shouldn't be killing // random processes, so just return an error. if self.status.is_some() { - Err(Error::new( + Err(io::const_io_error!( ErrorKind::InvalidInput, "invalid argument: can't kill an exited process", )) @@ -167,3 +180,88 @@ impl Process { } } } + +/// Unix exit statuses +#[derive(PartialEq, Eq, Clone, Copy, Debug)] +pub struct ExitStatus(c_int); + +impl ExitStatus { + pub fn new(status: c_int) -> ExitStatus { + ExitStatus(status) + } + + fn exited(&self) -> bool { + libc::WIFEXITED(self.0) + } + + pub fn exit_ok(&self) -> Result<(), ExitStatusError> { + // This assumes that WIFEXITED(status) && WEXITSTATUS==0 corresponds to status==0. This is + // true on all actual versions of Unix, is widely assumed, and is specified in SuS + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/wait.html. If it is not + // true for a platform pretending to be Unix, the tests (our doctests, and also + // procsss_unix/tests.rs) will spot it. `ExitStatusError::code` assumes this too. + match NonZero_c_int::try_from(self.0) { + Ok(failure) => Err(ExitStatusError(failure)), + Err(_) => Ok(()), + } + } + + pub fn code(&self) -> Option { + if self.exited() { Some(libc::WEXITSTATUS(self.0)) } else { None } + } + + pub fn signal(&self) -> Option { + if !self.exited() { Some(libc::WTERMSIG(self.0)) } else { None } + } + + pub fn core_dumped(&self) -> bool { + // This method is not yet properly implemented on VxWorks + false + } + + pub fn stopped_signal(&self) -> Option { + if libc::WIFSTOPPED(self.0) { Some(libc::WSTOPSIG(self.0)) } else { None } + } + + pub fn continued(&self) -> bool { + // This method is not yet properly implemented on VxWorks + false + } + + pub fn into_raw(&self) -> c_int { + self.0 + } +} + +/// Converts a raw `c_int` to a type-safe `ExitStatus` by wrapping it without copying. +impl From for ExitStatus { + fn from(a: c_int) -> ExitStatus { + ExitStatus(a) + } +} + +impl fmt::Display for ExitStatus { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if let Some(code) = self.code() { + write!(f, "exit code: {code}") + } else { + let signal = self.signal().unwrap(); + write!(f, "signal: {signal}") + } + } +} + +#[derive(PartialEq, Eq, Clone, Copy, Debug)] +pub struct ExitStatusError(NonZero_c_int); + +impl Into for ExitStatusError { + fn into(self) -> ExitStatus { + ExitStatus(self.0.into()) + } +} + +impl ExitStatusError { + pub fn code(self) -> Option { + ExitStatus(self.0.into()).code().map(|st| st.try_into().unwrap()) + } +} diff --git a/crux-mir/lib/std/src/sys/unix/process/zircon.rs b/crux-mir/lib/std/src/sys/unix/process/zircon.rs index 750b8f076..2e596486f 100644 --- a/crux-mir/lib/std/src/sys/unix/process/zircon.rs +++ b/crux-mir/lib/std/src/sys/unix/process/zircon.rs @@ -1,7 +1,5 @@ #![allow(non_camel_case_types, unused)] -use crate::convert::TryInto; -use crate::i64; use crate::io; use crate::mem::MaybeUninit; use crate::os::raw::c_char; @@ -26,9 +24,12 @@ pub const ZX_TASK_TERMINATED: zx_signals_t = ZX_OBJECT_SIGNAL_3; pub const ZX_RIGHT_SAME_RIGHTS: zx_rights_t = 1 << 31; +// The upper four bits gives the minor version. pub type zx_object_info_topic_t = u32; -pub const ZX_INFO_PROCESS: zx_object_info_topic_t = 3; +pub const ZX_INFO_PROCESS: zx_object_info_topic_t = 3 | (1 << 28); + +pub type zx_info_process_flags_t = u32; pub fn zx_cvt(t: T) -> io::Result where @@ -69,9 +70,9 @@ impl Drop for Handle { #[repr(C)] pub struct zx_info_process_t { pub return_code: i64, - pub started: bool, - pub exited: bool, - pub debugger_attached: bool, + pub start_time: zx_time_t, + pub flags: zx_info_process_flags_t, + pub reserved1: u32, } extern "C" { @@ -138,6 +139,7 @@ pub const FDIO_SPAWN_CLONE_LDSVC: u32 = 0x0002; pub const FDIO_SPAWN_CLONE_NAMESPACE: u32 = 0x0004; pub const FDIO_SPAWN_CLONE_STDIO: u32 = 0x0008; pub const FDIO_SPAWN_CLONE_ENVIRON: u32 = 0x0010; +pub const FDIO_SPAWN_CLONE_UTC_CLOCK: u32 = 0x0020; pub const FDIO_SPAWN_CLONE_ALL: u32 = 0xFFFF; // fdio_spawn_etc actions diff --git a/crux-mir/lib/std/src/sys/unix/rand.rs b/crux-mir/lib/std/src/sys/unix/rand.rs index 9ce5f3d01..a6fe07873 100644 --- a/crux-mir/lib/std/src/sys/unix/rand.rs +++ b/crux-mir/lib/std/src/sys/unix/rand.rs @@ -1,41 +1,86 @@ -use crate::mem; -use crate::slice; - pub fn hashmap_random_keys() -> (u64, u64) { - let mut v = (0, 0); - unsafe { - let view = slice::from_raw_parts_mut(&mut v as *mut _ as *mut u8, mem::size_of_val(&v)); - imp::fill_bytes(view); - } - v + const KEY_LEN: usize = core::mem::size_of::(); + + let mut v = [0u8; KEY_LEN * 2]; + imp::fill_bytes(&mut v); + + let key1 = v[0..KEY_LEN].try_into().unwrap(); + let key2 = v[KEY_LEN..].try_into().unwrap(); + + (u64::from_ne_bytes(key1), u64::from_ne_bytes(key2)) } #[cfg(all( unix, + not(target_os = "macos"), not(target_os = "ios"), + not(target_os = "watchos"), not(target_os = "openbsd"), not(target_os = "freebsd"), not(target_os = "netbsd"), not(target_os = "fuchsia"), - not(target_os = "redox") + not(target_os = "redox"), + not(target_os = "vxworks") ))] mod imp { use crate::fs::File; use crate::io::Read; #[cfg(any(target_os = "linux", target_os = "android"))] - fn getrandom(buf: &mut [u8]) -> libc::c_long { - unsafe { - libc::syscall(libc::SYS_getrandom, buf.as_mut_ptr(), buf.len(), libc::GRND_NONBLOCK) + use crate::sys::weak::syscall; + + #[cfg(any(target_os = "linux", target_os = "android"))] + fn getrandom(buf: &mut [u8]) -> libc::ssize_t { + use crate::sync::atomic::{AtomicBool, Ordering}; + use crate::sys::os::errno; + + // A weak symbol allows interposition, e.g. for perf measurements that want to + // disable randomness for consistency. Otherwise, we'll try a raw syscall. + // (`getrandom` was added in glibc 2.25, musl 1.1.20, android API level 28) + syscall! { + fn getrandom( + buffer: *mut libc::c_void, + length: libc::size_t, + flags: libc::c_uint + ) -> libc::ssize_t + } + + // This provides the best quality random numbers available at the given moment + // without ever blocking, and is preferable to falling back to /dev/urandom. + static GRND_INSECURE_AVAILABLE: AtomicBool = AtomicBool::new(true); + if GRND_INSECURE_AVAILABLE.load(Ordering::Relaxed) { + let ret = unsafe { getrandom(buf.as_mut_ptr().cast(), buf.len(), libc::GRND_INSECURE) }; + if ret == -1 && errno() as libc::c_int == libc::EINVAL { + GRND_INSECURE_AVAILABLE.store(false, Ordering::Relaxed); + } else { + return ret; + } } + + unsafe { getrandom(buf.as_mut_ptr().cast(), buf.len(), libc::GRND_NONBLOCK) } } - #[cfg(not(any(target_os = "linux", target_os = "android")))] + #[cfg(any(target_os = "espidf", target_os = "horizon"))] + fn getrandom(buf: &mut [u8]) -> libc::ssize_t { + unsafe { libc::getrandom(buf.as_mut_ptr().cast(), buf.len(), 0) } + } + + #[cfg(not(any( + target_os = "linux", + target_os = "android", + target_os = "espidf", + target_os = "horizon" + )))] fn getrandom_fill_bytes(_buf: &mut [u8]) -> bool { false } - #[cfg(any(target_os = "linux", target_os = "android"))] + #[cfg(any( + target_os = "linux", + target_os = "android", + target_os = "espidf", + target_os = "horizon" + ))] fn getrandom_fill_bytes(v: &mut [u8]) -> bool { use crate::sync::atomic::{AtomicBool, Ordering}; use crate::sys::os::errno; @@ -63,7 +108,7 @@ mod imp { } else if err == libc::EAGAIN { return false; } else { - panic!("unexpected getrandom error: {}", err); + panic!("unexpected getrandom error: {err}"); } } else { read += result as usize; @@ -92,6 +137,43 @@ mod imp { } } +#[cfg(target_os = "macos")] +mod imp { + use crate::fs::File; + use crate::io::Read; + use crate::sys::os::errno; + use crate::sys::weak::weak; + use libc::{c_int, c_void, size_t}; + + fn getentropy_fill_bytes(v: &mut [u8]) -> bool { + weak!(fn getentropy(*mut c_void, size_t) -> c_int); + + getentropy + .get() + .map(|f| { + // getentropy(2) permits a maximum buffer size of 256 bytes + for s in v.chunks_mut(256) { + let ret = unsafe { f(s.as_mut_ptr() as *mut c_void, s.len()) }; + if ret == -1 { + panic!("unexpected getentropy error: {}", errno()); + } + } + true + }) + .unwrap_or(false) + } + + pub fn fill_bytes(v: &mut [u8]) { + if getentropy_fill_bytes(v) { + return; + } + + // for older macos which doesn't support getentropy + let mut file = File::open("/dev/urandom").expect("failed to open /dev/urandom"); + file.read_exact(v).expect("failed to read /dev/urandom") + } +} + #[cfg(target_os = "openbsd")] mod imp { use crate::sys::os::errno; @@ -114,7 +196,7 @@ mod imp { // once per thread in `hashmap_random_keys`. Therefore `SecRandomCopyBytes` is // only used on iOS where direct access to `/dev/urandom` is blocked by the // sandbox. -#[cfg(target_os = "ios")] +#[cfg(any(target_os = "ios", target_os = "watchos"))] mod imp { use crate::io; use crate::ptr; @@ -191,3 +273,29 @@ mod imp { file.read_exact(v).expect("failed to read rand:") } } + +#[cfg(target_os = "vxworks")] +mod imp { + use crate::io; + use core::sync::atomic::{AtomicBool, Ordering::Relaxed}; + + pub fn fill_bytes(v: &mut [u8]) { + static RNG_INIT: AtomicBool = AtomicBool::new(false); + while !RNG_INIT.load(Relaxed) { + let ret = unsafe { libc::randSecure() }; + if ret < 0 { + panic!("couldn't generate random bytes: {}", io::Error::last_os_error()); + } else if ret > 0 { + RNG_INIT.store(true, Relaxed); + break; + } + unsafe { libc::usleep(10) }; + } + let ret = unsafe { + libc::randABytes(v.as_mut_ptr() as *mut libc::c_uchar, v.len() as libc::c_int) + }; + if ret < 0 { + panic!("couldn't generate random bytes: {}", io::Error::last_os_error()); + } + } +} diff --git a/crux-mir/lib/std/src/sys/unix/rwlock.rs b/crux-mir/lib/std/src/sys/unix/rwlock.rs deleted file mode 100644 index 079dea671..000000000 --- a/crux-mir/lib/std/src/sys/unix/rwlock.rs +++ /dev/null @@ -1,133 +0,0 @@ -use crate::cell::UnsafeCell; -use crate::sync::atomic::{AtomicUsize, Ordering}; - -pub struct RWLock { - inner: UnsafeCell, - write_locked: UnsafeCell, // guarded by the `inner` RwLock - num_readers: AtomicUsize, -} - -unsafe impl Send for RWLock {} -unsafe impl Sync for RWLock {} - -impl RWLock { - pub const fn new() -> RWLock { - RWLock { - inner: UnsafeCell::new(libc::PTHREAD_RWLOCK_INITIALIZER), - write_locked: UnsafeCell::new(false), - num_readers: AtomicUsize::new(0), - } - } - #[inline] - pub unsafe fn read(&self) { - let r = libc::pthread_rwlock_rdlock(self.inner.get()); - - // According to the pthread_rwlock_rdlock spec, this function **may** - // fail with EDEADLK if a deadlock is detected. On the other hand - // pthread mutexes will *never* return EDEADLK if they are initialized - // as the "fast" kind (which ours always are). As a result, a deadlock - // situation may actually return from the call to pthread_rwlock_rdlock - // instead of blocking forever (as mutexes and Windows rwlocks do). Note - // that not all unix implementations, however, will return EDEADLK for - // their rwlocks. - // - // We roughly maintain the deadlocking behavior by panicking to ensure - // that this lock acquisition does not succeed. - // - // We also check whether this lock is already write locked. This - // is only possible if it was write locked by the current thread and - // the implementation allows recursive locking. The POSIX standard - // doesn't require recursively locking a rwlock to deadlock, but we can't - // allow that because it could lead to aliasing issues. - if r == libc::EAGAIN { - panic!("rwlock maximum reader count exceeded"); - } else if r == libc::EDEADLK || (r == 0 && *self.write_locked.get()) { - if r == 0 { - self.raw_unlock(); - } - panic!("rwlock read lock would result in deadlock"); - } else { - assert_eq!(r, 0); - self.num_readers.fetch_add(1, Ordering::Relaxed); - } - } - #[inline] - pub unsafe fn try_read(&self) -> bool { - let r = libc::pthread_rwlock_tryrdlock(self.inner.get()); - if r == 0 { - if *self.write_locked.get() { - self.raw_unlock(); - false - } else { - self.num_readers.fetch_add(1, Ordering::Relaxed); - true - } - } else { - false - } - } - #[inline] - pub unsafe fn write(&self) { - let r = libc::pthread_rwlock_wrlock(self.inner.get()); - // See comments above for why we check for EDEADLK and write_locked. We - // also need to check that num_readers is 0. - if r == libc::EDEADLK - || *self.write_locked.get() - || self.num_readers.load(Ordering::Relaxed) != 0 - { - if r == 0 { - self.raw_unlock(); - } - panic!("rwlock write lock would result in deadlock"); - } else { - debug_assert_eq!(r, 0); - } - *self.write_locked.get() = true; - } - #[inline] - pub unsafe fn try_write(&self) -> bool { - let r = libc::pthread_rwlock_trywrlock(self.inner.get()); - if r == 0 { - if *self.write_locked.get() || self.num_readers.load(Ordering::Relaxed) != 0 { - self.raw_unlock(); - false - } else { - *self.write_locked.get() = true; - true - } - } else { - false - } - } - #[inline] - unsafe fn raw_unlock(&self) { - let r = libc::pthread_rwlock_unlock(self.inner.get()); - debug_assert_eq!(r, 0); - } - #[inline] - pub unsafe fn read_unlock(&self) { - debug_assert!(!*self.write_locked.get()); - self.num_readers.fetch_sub(1, Ordering::Relaxed); - self.raw_unlock(); - } - #[inline] - pub unsafe fn write_unlock(&self) { - debug_assert_eq!(self.num_readers.load(Ordering::Relaxed), 0); - debug_assert!(*self.write_locked.get()); - *self.write_locked.get() = false; - self.raw_unlock(); - } - #[inline] - pub unsafe fn destroy(&self) { - let r = libc::pthread_rwlock_destroy(self.inner.get()); - // On DragonFly pthread_rwlock_destroy() returns EINVAL if called on a - // rwlock that was just initialized with - // libc::PTHREAD_RWLOCK_INITIALIZER. Once it is used (locked/unlocked) - // or pthread_rwlock_init() is called, this behaviour no longer occurs. - if cfg!(target_os = "dragonfly") { - debug_assert!(r == 0 || r == libc::EINVAL); - } else { - debug_assert_eq!(r, 0); - } - } -} diff --git a/crux-mir/lib/std/src/sys/unix/stack_overflow.rs b/crux-mir/lib/std/src/sys/unix/stack_overflow.rs index 2626ca37c..b59d4ba26 100644 --- a/crux-mir/lib/std/src/sys/unix/stack_overflow.rs +++ b/crux-mir/lib/std/src/sys/unix/stack_overflow.rs @@ -6,7 +6,7 @@ pub use self::imp::cleanup; pub use self::imp::init; pub struct Handler { - _data: *mut libc::c_void, + data: *mut libc::c_void, } impl Handler { @@ -15,14 +15,14 @@ impl Handler { } fn null() -> Handler { - Handler { _data: crate::ptr::null_mut() } + Handler { data: crate::ptr::null_mut() } } } impl Drop for Handler { fn drop(&mut self) { unsafe { - drop_handler(self); + drop_handler(self.data); } } } @@ -33,39 +33,30 @@ impl Drop for Handler { target_os = "dragonfly", target_os = "freebsd", target_os = "solaris", - all(target_os = "netbsd", not(target_vendor = "rumprun")), + target_os = "illumos", + target_os = "netbsd", target_os = "openbsd" ))] mod imp { use super::Handler; + use crate::io; use crate::mem; use crate::ptr; + use crate::thread; use libc::MAP_FAILED; - use libc::{mmap, munmap}; + #[cfg(not(all(target_os = "linux", target_env = "gnu")))] + use libc::{mmap as mmap64, munmap}; + #[cfg(all(target_os = "linux", target_env = "gnu"))] + use libc::{mmap64, munmap}; use libc::{sigaction, sighandler_t, SA_ONSTACK, SA_SIGINFO, SIGBUS, SIG_DFL}; use libc::{sigaltstack, SIGSTKSZ, SS_DISABLE}; use libc::{MAP_ANON, MAP_PRIVATE, PROT_NONE, PROT_READ, PROT_WRITE, SIGSEGV}; + use crate::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use crate::sys::unix::os::page_size; use crate::sys_common::thread_info; - #[cfg(any(target_os = "linux", target_os = "android"))] - unsafe fn siginfo_si_addr(info: *mut libc::siginfo_t) -> usize { - #[repr(C)] - struct siginfo_t { - a: [libc::c_int; 3], // si_signo, si_errno, si_code - si_addr: *mut libc::c_void, - } - - (*(info as *const siginfo_t)).si_addr as usize - } - - #[cfg(not(any(target_os = "linux", target_os = "android")))] - unsafe fn siginfo_si_addr(info: *mut libc::siginfo_t) -> usize { - (*info).si_addr as usize - } - // Signal handler for the SIGSEGV and SIGBUS handlers. We've got guard pages // (unmapped pages) at the end of every thread's stack, so if a thread ends // up running into the guard page it'll trigger this handler. We want to @@ -92,15 +83,16 @@ mod imp { info: *mut libc::siginfo_t, _data: *mut libc::c_void, ) { - use crate::sys_common::util::report_overflow; - let guard = thread_info::stack_guard().unwrap_or(0..0); - let addr = siginfo_si_addr(info); + let addr = (*info).si_addr() as usize; // If the faulting address is within the guard page, then we print a // message saying so and abort. if guard.start <= addr && addr < guard.end { - report_overflow(); + rtprintpanic!( + "\nthread '{}' has overflowed its stack\n", + thread::current().name().unwrap_or("") + ); rtabort!("stack overflow"); } else { // Unregister ourselves by reverting back to the default behavior. @@ -112,8 +104,8 @@ mod imp { } } - static mut MAIN_ALTSTACK: *mut libc::c_void = ptr::null_mut(); - static mut NEED_ALTSTACK: bool = false; + static MAIN_ALTSTACK: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + static NEED_ALTSTACK: AtomicBool = AtomicBool::new(false); pub unsafe fn init() { let mut action: sigaction = mem::zeroed(); @@ -124,57 +116,45 @@ mod imp { action.sa_flags = SA_SIGINFO | SA_ONSTACK; action.sa_sigaction = signal_handler as sighandler_t; sigaction(signal, &action, ptr::null_mut()); - NEED_ALTSTACK = true; + NEED_ALTSTACK.store(true, Ordering::Relaxed); } } let handler = make_handler(); - MAIN_ALTSTACK = handler._data; + MAIN_ALTSTACK.store(handler.data, Ordering::Relaxed); mem::forget(handler); } pub unsafe fn cleanup() { - Handler { _data: MAIN_ALTSTACK }; + drop_handler(MAIN_ALTSTACK.load(Ordering::Relaxed)); } unsafe fn get_stackp() -> *mut libc::c_void { - let stackp = mmap( - ptr::null_mut(), - SIGSTKSZ + page_size(), - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, - -1, - 0, - ); + // OpenBSD requires this flag for stack mapping + // otherwise the said mapping will fail as a no-op on most systems + // and has a different meaning on FreeBSD + #[cfg(any(target_os = "openbsd", target_os = "netbsd", target_os = "linux",))] + let flags = MAP_PRIVATE | MAP_ANON | libc::MAP_STACK; + #[cfg(not(any(target_os = "openbsd", target_os = "netbsd", target_os = "linux",)))] + let flags = MAP_PRIVATE | MAP_ANON; + let stackp = + mmap64(ptr::null_mut(), SIGSTKSZ + page_size(), PROT_READ | PROT_WRITE, flags, -1, 0); if stackp == MAP_FAILED { - panic!("failed to allocate an alternative stack"); + panic!("failed to allocate an alternative stack: {}", io::Error::last_os_error()); } let guard_result = libc::mprotect(stackp, page_size(), PROT_NONE); if guard_result != 0 { - panic!("failed to set up alternative stack guard page"); + panic!("failed to set up alternative stack guard page: {}", io::Error::last_os_error()); } stackp.add(page_size()) } - #[cfg(any( - target_os = "linux", - target_os = "macos", - target_os = "freebsd", - target_os = "netbsd", - target_os = "openbsd", - target_os = "solaris" - ))] unsafe fn get_stack() -> libc::stack_t { libc::stack_t { ss_sp: get_stackp(), ss_flags: 0, ss_size: SIGSTKSZ } } - #[cfg(target_os = "dragonfly")] - unsafe fn get_stack() -> libc::stack_t { - libc::stack_t { ss_sp: get_stackp() as *mut i8, ss_flags: 0, ss_size: SIGSTKSZ } - } - pub unsafe fn make_handler() -> Handler { - if !NEED_ALTSTACK { + if !NEED_ALTSTACK.load(Ordering::Relaxed) { return Handler::null(); } let mut stack = mem::zeroed(); @@ -183,14 +163,14 @@ mod imp { if stack.ss_flags & SS_DISABLE != 0 { stack = get_stack(); sigaltstack(&stack, ptr::null_mut()); - Handler { _data: stack.ss_sp as *mut libc::c_void } + Handler { data: stack.ss_sp as *mut libc::c_void } } else { Handler::null() } } - pub unsafe fn drop_handler(handler: &mut Handler) { - if !handler._data.is_null() { + pub unsafe fn drop_handler(data: *mut libc::c_void) { + if !data.is_null() { let stack = libc::stack_t { ss_sp: ptr::null_mut(), ss_flags: SS_DISABLE, @@ -203,7 +183,7 @@ mod imp { sigaltstack(&stack, ptr::null_mut()); // We know from `get_stackp` that the alternate stack we installed is part of a mapping // that started one page earlier, so walk back a page and unmap from there. - munmap(handler._data.sub(page_size()), SIGSTKSZ + page_size()); + munmap(data.sub(page_size()), SIGSTKSZ + page_size()); } } } @@ -214,8 +194,9 @@ mod imp { target_os = "dragonfly", target_os = "freebsd", target_os = "solaris", - all(target_os = "netbsd", not(target_vendor = "rumprun")), - target_os = "openbsd" + target_os = "illumos", + target_os = "netbsd", + target_os = "openbsd", )))] mod imp { pub unsafe fn init() {} @@ -226,5 +207,5 @@ mod imp { super::Handler::null() } - pub unsafe fn drop_handler(_handler: &mut super::Handler) {} + pub unsafe fn drop_handler(_data: *mut libc::c_void) {} } diff --git a/crux-mir/lib/std/src/sys/unix/stdio.rs b/crux-mir/lib/std/src/sys/unix/stdio.rs index b9c569638..b3626c564 100644 --- a/crux-mir/lib/std/src/sys/unix/stdio.rs +++ b/crux-mir/lib/std/src/sys/unix/stdio.rs @@ -1,5 +1,6 @@ use crate::io::{self, IoSlice, IoSliceMut}; use crate::mem::ManuallyDrop; +use crate::os::unix::io::FromRawFd; use crate::sys::fd::FileDesc; pub struct Stdin(()); @@ -7,34 +8,46 @@ pub struct Stdout(()); pub struct Stderr(()); impl Stdin { - pub fn new() -> io::Result { - Ok(Stdin(())) + pub const fn new() -> Stdin { + Stdin(()) } } impl io::Read for Stdin { fn read(&mut self, buf: &mut [u8]) -> io::Result { - ManuallyDrop::new(FileDesc::new(libc::STDIN_FILENO)).read(buf) + unsafe { ManuallyDrop::new(FileDesc::from_raw_fd(libc::STDIN_FILENO)).read(buf) } } fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { - ManuallyDrop::new(FileDesc::new(libc::STDIN_FILENO)).read_vectored(bufs) + unsafe { ManuallyDrop::new(FileDesc::from_raw_fd(libc::STDIN_FILENO)).read_vectored(bufs) } + } + + #[inline] + fn is_read_vectored(&self) -> bool { + true } } impl Stdout { - pub fn new() -> io::Result { - Ok(Stdout(())) + pub const fn new() -> Stdout { + Stdout(()) } } impl io::Write for Stdout { fn write(&mut self, buf: &[u8]) -> io::Result { - ManuallyDrop::new(FileDesc::new(libc::STDOUT_FILENO)).write(buf) + unsafe { ManuallyDrop::new(FileDesc::from_raw_fd(libc::STDOUT_FILENO)).write(buf) } } fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { - ManuallyDrop::new(FileDesc::new(libc::STDOUT_FILENO)).write_vectored(bufs) + unsafe { + ManuallyDrop::new(FileDesc::from_raw_fd(libc::STDOUT_FILENO)).write_vectored(bufs) + } + } + + #[inline] + fn is_write_vectored(&self) -> bool { + true } fn flush(&mut self) -> io::Result<()> { @@ -43,18 +56,25 @@ impl io::Write for Stdout { } impl Stderr { - pub fn new() -> io::Result { - Ok(Stderr(())) + pub const fn new() -> Stderr { + Stderr(()) } } impl io::Write for Stderr { fn write(&mut self, buf: &[u8]) -> io::Result { - ManuallyDrop::new(FileDesc::new(libc::STDERR_FILENO)).write(buf) + unsafe { ManuallyDrop::new(FileDesc::from_raw_fd(libc::STDERR_FILENO)).write(buf) } } fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { - ManuallyDrop::new(FileDesc::new(libc::STDERR_FILENO)).write_vectored(bufs) + unsafe { + ManuallyDrop::new(FileDesc::from_raw_fd(libc::STDERR_FILENO)).write_vectored(bufs) + } + } + + #[inline] + fn is_write_vectored(&self) -> bool { + true } fn flush(&mut self) -> io::Result<()> { @@ -69,5 +89,5 @@ pub fn is_ebadf(err: &io::Error) -> bool { pub const STDIN_BUF_SIZE: usize = crate::sys_common::io::DEFAULT_BUF_SIZE; pub fn panic_output() -> Option { - Stderr::new().ok() + Some(Stderr::new()) } diff --git a/crux-mir/lib/std/src/sys/unix/thread.rs b/crux-mir/lib/std/src/sys/unix/thread.rs index 674d4c711..2a1830d06 100644 --- a/crux-mir/lib/std/src/sys/unix/thread.rs +++ b/crux-mir/lib/std/src/sys/unix/thread.rs @@ -2,16 +2,40 @@ use crate::cmp; use crate::ffi::CStr; use crate::io; use crate::mem; +use crate::num::NonZeroUsize; use crate::ptr; -use crate::sys::os; +use crate::sys::{os, stack_overflow}; use crate::time::Duration; -use crate::sys_common::thread::*; - -#[cfg(not(target_os = "l4re"))] +#[cfg(all(target_os = "linux", target_env = "gnu"))] +use crate::sys::weak::dlsym; +#[cfg(any(target_os = "solaris", target_os = "illumos"))] +use crate::sys::weak::weak; +#[cfg(not(any(target_os = "l4re", target_os = "vxworks", target_os = "espidf")))] pub const DEFAULT_MIN_STACK_SIZE: usize = 2 * 1024 * 1024; #[cfg(target_os = "l4re")] pub const DEFAULT_MIN_STACK_SIZE: usize = 1024 * 1024; +#[cfg(target_os = "vxworks")] +pub const DEFAULT_MIN_STACK_SIZE: usize = 256 * 1024; +#[cfg(target_os = "espidf")] +pub const DEFAULT_MIN_STACK_SIZE: usize = 0; // 0 indicates that the stack size configured in the ESP-IDF menuconfig system should be used + +#[cfg(target_os = "fuchsia")] +mod zircon { + type zx_handle_t = u32; + type zx_status_t = i32; + pub const ZX_PROP_NAME: u32 = 3; + + extern "C" { + pub fn zx_object_set_property( + handle: zx_handle_t, + property: u32, + value: *const libc::c_void, + value_size: libc::size_t, + ) -> zx_status_t; + pub fn zx_thread_self() -> zx_handle_t; + } +} pub struct Thread { id: libc::pthread_t, @@ -22,62 +46,66 @@ pub struct Thread { unsafe impl Send for Thread {} unsafe impl Sync for Thread {} -// The pthread_attr_setstacksize symbol doesn't exist in the emscripten libc, -// so we have to not link to it to satisfy emcc's ERROR_ON_UNDEFINED_SYMBOLS. -#[cfg(not(target_os = "emscripten"))] -unsafe fn pthread_attr_setstacksize( - attr: *mut libc::pthread_attr_t, - stack_size: libc::size_t, -) -> libc::c_int { - libc::pthread_attr_setstacksize(attr, stack_size) -} - -#[cfg(target_os = "emscripten")] -unsafe fn pthread_attr_setstacksize( - _attr: *mut libc::pthread_attr_t, - _stack_size: libc::size_t, -) -> libc::c_int { - panic!() -} - impl Thread { // unsafe: see thread::Builder::spawn_unchecked for safety requirements pub unsafe fn new(stack: usize, p: Box) -> io::Result { - let p = box p; + let p = Box::into_raw(box p); let mut native: libc::pthread_t = mem::zeroed(); let mut attr: libc::pthread_attr_t = mem::zeroed(); assert_eq!(libc::pthread_attr_init(&mut attr), 0); - let stack_size = cmp::max(stack, min_stack_size(&attr)); - - match pthread_attr_setstacksize(&mut attr, stack_size) { - 0 => {} - n => { - assert_eq!(n, libc::EINVAL); - // EINVAL means |stack_size| is either too small or not a - // multiple of the system page size. Because it's definitely - // >= PTHREAD_STACK_MIN, it must be an alignment issue. - // Round up to the nearest page and try again. - let page_size = os::page_size(); - let stack_size = - (stack_size + page_size - 1) & (-(page_size as isize - 1) as usize - 1); - assert_eq!(libc::pthread_attr_setstacksize(&mut attr, stack_size), 0); - } - }; + #[cfg(target_os = "espidf")] + if stack > 0 { + // Only set the stack if a non-zero value is passed + // 0 is used as an indication that the default stack size configured in the ESP-IDF menuconfig system should be used + assert_eq!( + libc::pthread_attr_setstacksize(&mut attr, cmp::max(stack, min_stack_size(&attr))), + 0 + ); + } + + #[cfg(not(target_os = "espidf"))] + { + let stack_size = cmp::max(stack, min_stack_size(&attr)); + + match libc::pthread_attr_setstacksize(&mut attr, stack_size) { + 0 => {} + n => { + assert_eq!(n, libc::EINVAL); + // EINVAL means |stack_size| is either too small or not a + // multiple of the system page size. Because it's definitely + // >= PTHREAD_STACK_MIN, it must be an alignment issue. + // Round up to the nearest page and try again. + let page_size = os::page_size(); + let stack_size = + (stack_size + page_size - 1) & (-(page_size as isize - 1) as usize - 1); + assert_eq!(libc::pthread_attr_setstacksize(&mut attr, stack_size), 0); + } + }; + } - let ret = libc::pthread_create(&mut native, &attr, thread_start, &*p as *const _ as *mut _); + let ret = libc::pthread_create(&mut native, &attr, thread_start, p as *mut _); + // Note: if the thread creation fails and this assert fails, then p will + // be leaked. However, an alternative design could cause double-free + // which is clearly worse. assert_eq!(libc::pthread_attr_destroy(&mut attr), 0); return if ret != 0 { + // The thread failed to start and as a result p was not consumed. Therefore, it is + // safe to reconstruct the box so that it gets deallocated. + drop(Box::from_raw(p)); Err(io::Error::from_raw_os_error(ret)) } else { - mem::forget(p); // ownership passed to pthread_create Ok(Thread { id: native }) }; extern "C" fn thread_start(main: *mut libc::c_void) -> *mut libc::c_void { unsafe { - start_thread(main as *mut u8); + // Next, set up our stack overflow handler which may get triggered if we run + // out of stack. + let _handler = stack_overflow::Handler::new(); + // Finally, let's run some code. + Box::from_raw(main as *mut Box)(); } ptr::null_mut() } @@ -88,13 +116,30 @@ impl Thread { debug_assert_eq!(ret, 0); } - #[cfg(any(target_os = "linux", target_os = "android"))] + #[cfg(target_os = "android")] pub fn set_name(name: &CStr) { const PR_SET_NAME: libc::c_int = 15; - // pthread wrapper only appeared in glibc 2.12, so we use syscall - // directly. unsafe { - libc::prctl(PR_SET_NAME, name.as_ptr() as libc::c_ulong, 0, 0, 0); + libc::prctl( + PR_SET_NAME, + name.as_ptr(), + 0 as libc::c_ulong, + 0 as libc::c_ulong, + 0 as libc::c_ulong, + ); + } + } + + #[cfg(target_os = "linux")] + pub fn set_name(name: &CStr) { + const TASK_COMM_LEN: usize = 16; + + unsafe { + // Available since glibc 2.12, musl 1.1.16, and uClibc 1.0.20. + let name = truncate_cstr::<{ TASK_COMM_LEN }>(name); + let res = libc::pthread_setname_np(libc::pthread_self(), name.as_ptr()); + // We have no good way of propagating errors here, but in debug-builds let's check that this actually worked. + debug_assert_eq!(res, 0); } } @@ -105,27 +150,30 @@ impl Thread { } } - #[cfg(any(target_os = "macos", target_os = "ios"))] + #[cfg(any(target_os = "macos", target_os = "ios", target_os = "watchos"))] pub fn set_name(name: &CStr) { unsafe { - libc::pthread_setname_np(name.as_ptr()); + let name = truncate_cstr::<{ libc::MAXTHREADNAMESIZE }>(name); + let res = libc::pthread_setname_np(name.as_ptr()); + // We have no good way of propagating errors here, but in debug-builds let's check that this actually worked. + debug_assert_eq!(res, 0); } } #[cfg(target_os = "netbsd")] pub fn set_name(name: &CStr) { - use crate::ffi::CString; - let cname = CString::new(&b"%s"[..]).unwrap(); unsafe { - libc::pthread_setname_np( + let cname = CStr::from_bytes_with_nul_unchecked(b"%s\0".as_slice()); + let res = libc::pthread_setname_np( libc::pthread_self(), cname.as_ptr(), name.as_ptr() as *mut libc::c_void, ); + debug_assert_eq!(res, 0); } } - #[cfg(target_os = "solaris")] + #[cfg(any(target_os = "solaris", target_os = "illumos"))] pub fn set_name(name: &CStr) { weak! { fn pthread_setname_np( @@ -134,27 +182,44 @@ impl Thread { } if let Some(f) = pthread_setname_np.get() { - unsafe { - f(libc::pthread_self(), name.as_ptr()); - } + let res = unsafe { f(libc::pthread_self(), name.as_ptr()) }; + debug_assert_eq!(res, 0); + } + } + + #[cfg(target_os = "fuchsia")] + pub fn set_name(name: &CStr) { + use self::zircon::*; + unsafe { + zx_object_set_property( + zx_thread_self(), + ZX_PROP_NAME, + name.as_ptr() as *const libc::c_void, + name.to_bytes().len(), + ); + } + } + + #[cfg(target_os = "haiku")] + pub fn set_name(name: &CStr) { + unsafe { + let thread_self = libc::find_thread(ptr::null_mut()); + libc::rename_thread(thread_self, name.as_ptr()); } } #[cfg(any( target_env = "newlib", - target_os = "haiku", target_os = "l4re", target_os = "emscripten", - target_os = "redox" + target_os = "redox", + target_os = "vxworks" ))] pub fn set_name(_name: &CStr) { - // Newlib, Illumos, Haiku, and Emscripten have no way to set a thread name. - } - #[cfg(target_os = "fuchsia")] - pub fn set_name(_name: &CStr) { - // FIXME: determine whether Fuchsia has a way to set a thread name. + // Newlib, Emscripten, and VxWorks have no way to set a thread name. } + #[cfg(not(target_os = "espidf"))] pub fn sleep(dur: Duration) { let mut secs = dur.as_secs(); let mut nsecs = dur.subsec_nanos() as _; @@ -164,11 +229,12 @@ impl Thread { unsafe { while secs > 0 || nsecs > 0 { let mut ts = libc::timespec { - tv_sec: cmp::min(libc::time_t::max_value() as u64, secs) as libc::time_t, + tv_sec: cmp::min(libc::time_t::MAX as u64, secs) as libc::time_t, tv_nsec: nsecs, }; secs -= ts.tv_sec as u64; - if libc::nanosleep(&ts, &mut ts) == -1 { + let ts_ptr = &mut ts as *mut _; + if libc::nanosleep(ts_ptr, ts_ptr) == -1 { assert_eq!(os::errno(), libc::EINTR); secs += ts.tv_sec as u64; nsecs = ts.tv_nsec; @@ -179,6 +245,19 @@ impl Thread { } } + #[cfg(target_os = "espidf")] + pub fn sleep(dur: Duration) { + let mut micros = dur.as_micros(); + unsafe { + while micros > 0 { + let st = if micros > u32::MAX as u128 { u32::MAX } else { micros as u32 }; + libc::usleep(st); + + micros -= st as u128; + } + } + } + pub fn join(self) { unsafe { let ret = libc::pthread_join(self.id, ptr::null_mut()); @@ -205,11 +284,350 @@ impl Drop for Thread { } } +#[cfg(any(target_os = "linux", target_os = "macos", target_os = "ios", target_os = "watchos"))] +fn truncate_cstr(cstr: &CStr) -> [libc::c_char; MAX_WITH_NUL] { + let mut result = [0; MAX_WITH_NUL]; + for (src, dst) in cstr.to_bytes().iter().zip(&mut result[..MAX_WITH_NUL - 1]) { + *dst = *src as libc::c_char; + } + result +} + +pub fn available_parallelism() -> io::Result { + cfg_if::cfg_if! { + if #[cfg(any( + target_os = "android", + target_os = "emscripten", + target_os = "fuchsia", + target_os = "ios", + target_os = "linux", + target_os = "macos", + target_os = "solaris", + target_os = "illumos", + ))] { + #[cfg(any(target_os = "android", target_os = "linux"))] + { + let quota = cgroups::quota().max(1); + let mut set: libc::cpu_set_t = unsafe { mem::zeroed() }; + unsafe { + if libc::sched_getaffinity(0, mem::size_of::(), &mut set) == 0 { + let count = libc::CPU_COUNT(&set) as usize; + let count = count.min(quota); + // SAFETY: affinity mask can't be empty and the quota gets clamped to a minimum of 1 + return Ok(NonZeroUsize::new_unchecked(count)); + } + } + } + match unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) } { + -1 => Err(io::Error::last_os_error()), + 0 => Err(io::const_io_error!(io::ErrorKind::NotFound, "The number of hardware threads is not known for the target platform")), + cpus => Ok(unsafe { NonZeroUsize::new_unchecked(cpus as usize) }), + } + } else if #[cfg(any(target_os = "freebsd", target_os = "dragonfly", target_os = "netbsd"))] { + use crate::ptr; + + let mut cpus: libc::c_uint = 0; + let mut cpus_size = crate::mem::size_of_val(&cpus); + + unsafe { + cpus = libc::sysconf(libc::_SC_NPROCESSORS_ONLN) as libc::c_uint; + } + + // Fallback approach in case of errors or no hardware threads. + if cpus < 1 { + let mut mib = [libc::CTL_HW, libc::HW_NCPU, 0, 0]; + let res = unsafe { + libc::sysctl( + mib.as_mut_ptr(), + 2, + &mut cpus as *mut _ as *mut _, + &mut cpus_size as *mut _ as *mut _, + ptr::null_mut(), + 0, + ) + }; + + // Handle errors if any. + if res == -1 { + return Err(io::Error::last_os_error()); + } else if cpus == 0 { + return Err(io::const_io_error!(io::ErrorKind::NotFound, "The number of hardware threads is not known for the target platform")); + } + } + Ok(unsafe { NonZeroUsize::new_unchecked(cpus as usize) }) + } else if #[cfg(target_os = "openbsd")] { + use crate::ptr; + + let mut cpus: libc::c_uint = 0; + let mut cpus_size = crate::mem::size_of_val(&cpus); + let mut mib = [libc::CTL_HW, libc::HW_NCPU, 0, 0]; + + let res = unsafe { + libc::sysctl( + mib.as_mut_ptr(), + 2, + &mut cpus as *mut _ as *mut _, + &mut cpus_size as *mut _ as *mut _, + ptr::null_mut(), + 0, + ) + }; + + // Handle errors if any. + if res == -1 { + return Err(io::Error::last_os_error()); + } else if cpus == 0 { + return Err(io::const_io_error!(io::ErrorKind::NotFound, "The number of hardware threads is not known for the target platform")); + } + + Ok(unsafe { NonZeroUsize::new_unchecked(cpus as usize) }) + } else if #[cfg(target_os = "haiku")] { + // system_info cpu_count field gets the static data set at boot time with `smp_set_num_cpus` + // `get_system_info` calls then `smp_get_num_cpus` + unsafe { + let mut sinfo: libc::system_info = crate::mem::zeroed(); + let res = libc::get_system_info(&mut sinfo); + + if res != libc::B_OK { + return Err(io::const_io_error!(io::ErrorKind::NotFound, "The number of hardware threads is not known for the target platform")); + } + + Ok(NonZeroUsize::new_unchecked(sinfo.cpu_count as usize)) + } + } else { + // FIXME: implement on vxWorks, Redox, l4re + Err(io::const_io_error!(io::ErrorKind::Unsupported, "Getting the number of hardware threads is not supported on the target platform")) + } + } +} + +#[cfg(any(target_os = "android", target_os = "linux"))] +mod cgroups { + //! Currently not covered + //! * cgroup v2 in non-standard mountpoints + //! * paths containing control characters or spaces, since those would be escaped in procfs + //! output and we don't unescape + use crate::borrow::Cow; + use crate::ffi::OsString; + use crate::fs::{try_exists, File}; + use crate::io::Read; + use crate::io::{BufRead, BufReader}; + use crate::os::unix::ffi::OsStringExt; + use crate::path::Path; + use crate::path::PathBuf; + use crate::str::from_utf8; + + #[derive(PartialEq)] + enum Cgroup { + V1, + V2, + } + + /// Returns cgroup CPU quota in core-equivalents, rounded down or usize::MAX if the quota cannot + /// be determined or is not set. + pub(super) fn quota() -> usize { + let mut quota = usize::MAX; + if cfg!(miri) { + // Attempting to open a file fails under default flags due to isolation. + // And Miri does not have parallelism anyway. + return quota; + } + + let _: Option<()> = try { + let mut buf = Vec::with_capacity(128); + // find our place in the cgroup hierarchy + File::open("/proc/self/cgroup").ok()?.read_to_end(&mut buf).ok()?; + let (cgroup_path, version) = + buf.split(|&c| c == b'\n').fold(None, |previous, line| { + let mut fields = line.splitn(3, |&c| c == b':'); + // 2nd field is a list of controllers for v1 or empty for v2 + let version = match fields.nth(1) { + Some(b"") => Cgroup::V2, + Some(controllers) + if from_utf8(controllers) + .is_ok_and(|c| c.split(',').any(|c| c == "cpu")) => + { + Cgroup::V1 + } + _ => return previous, + }; + + // already-found v1 trumps v2 since it explicitly specifies its controllers + if previous.is_some() && version == Cgroup::V2 { + return previous; + } + + let path = fields.last()?; + // skip leading slash + Some((path[1..].to_owned(), version)) + })?; + let cgroup_path = PathBuf::from(OsString::from_vec(cgroup_path)); + + quota = match version { + Cgroup::V1 => quota_v1(cgroup_path), + Cgroup::V2 => quota_v2(cgroup_path), + }; + }; + + quota + } + + fn quota_v2(group_path: PathBuf) -> usize { + let mut quota = usize::MAX; + + let mut path = PathBuf::with_capacity(128); + let mut read_buf = String::with_capacity(20); + + // standard mount location defined in file-hierarchy(7) manpage + let cgroup_mount = "/sys/fs/cgroup"; + + path.push(cgroup_mount); + path.push(&group_path); + + path.push("cgroup.controllers"); + + // skip if we're not looking at cgroup2 + if matches!(try_exists(&path), Err(_) | Ok(false)) { + return usize::MAX; + }; + + path.pop(); + + let _: Option<()> = try { + while path.starts_with(cgroup_mount) { + path.push("cpu.max"); + + read_buf.clear(); + + if File::open(&path).and_then(|mut f| f.read_to_string(&mut read_buf)).is_ok() { + let raw_quota = read_buf.lines().next()?; + let mut raw_quota = raw_quota.split(' '); + let limit = raw_quota.next()?; + let period = raw_quota.next()?; + match (limit.parse::(), period.parse::()) { + (Ok(limit), Ok(period)) if period > 0 => { + quota = quota.min(limit / period); + } + _ => {} + } + } + + path.pop(); // pop filename + path.pop(); // pop dir + } + }; + + quota + } + + fn quota_v1(group_path: PathBuf) -> usize { + let mut quota = usize::MAX; + let mut path = PathBuf::with_capacity(128); + let mut read_buf = String::with_capacity(20); + + // Hardcode commonly used locations mentioned in the cgroups(7) manpage + // if that doesn't work scan mountinfo and adjust `group_path` for bind-mounts + let mounts: &[fn(&Path) -> Option<(_, &Path)>] = &[ + |p| Some((Cow::Borrowed("/sys/fs/cgroup/cpu"), p)), + |p| Some((Cow::Borrowed("/sys/fs/cgroup/cpu,cpuacct"), p)), + // this can be expensive on systems with tons of mountpoints + // but we only get to this point when /proc/self/cgroups explicitly indicated + // this process belongs to a cpu-controller cgroup v1 and the defaults didn't work + find_mountpoint, + ]; + + for mount in mounts { + let Some((mount, group_path)) = mount(&group_path) else { continue }; + + path.clear(); + path.push(mount.as_ref()); + path.push(&group_path); + + // skip if we guessed the mount incorrectly + if matches!(try_exists(&path), Err(_) | Ok(false)) { + continue; + } + + while path.starts_with(mount.as_ref()) { + let mut parse_file = |name| { + path.push(name); + read_buf.clear(); + + let f = File::open(&path); + path.pop(); // restore buffer before any early returns + f.ok()?.read_to_string(&mut read_buf).ok()?; + let parsed = read_buf.trim().parse::().ok()?; + + Some(parsed) + }; + + let limit = parse_file("cpu.cfs_quota_us"); + let period = parse_file("cpu.cfs_period_us"); + + match (limit, period) { + (Some(limit), Some(period)) if period > 0 => quota = quota.min(limit / period), + _ => {} + } + + path.pop(); + } + + // we passed the try_exists above so we should have traversed the correct hierarchy + // when reaching this line + break; + } + + quota + } + + /// Scan mountinfo for cgroup v1 mountpoint with a cpu controller + /// + /// If the cgroupfs is a bind mount then `group_path` is adjusted to skip + /// over the already-included prefix + fn find_mountpoint(group_path: &Path) -> Option<(Cow<'static, str>, &Path)> { + let mut reader = BufReader::new(File::open("/proc/self/mountinfo").ok()?); + let mut line = String::with_capacity(256); + loop { + line.clear(); + if reader.read_line(&mut line).ok()? == 0 { + break; + } + + let line = line.trim(); + let mut items = line.split(' '); + + let sub_path = items.nth(3)?; + let mount_point = items.next()?; + let mount_opts = items.next_back()?; + let filesystem_type = items.nth_back(1)?; + + if filesystem_type != "cgroup" || !mount_opts.split(',').any(|opt| opt == "cpu") { + // not a cgroup / not a cpu-controller + continue; + } + + let sub_path = Path::new(sub_path).strip_prefix("/").ok()?; + + if !group_path.starts_with(sub_path) { + // this is a bind-mount and the bound subdirectory + // does not contain the cgroup this process belongs to + continue; + } + + let trimmed_group_path = group_path.strip_prefix(sub_path).ok()?; + + return Some((Cow::Owned(mount_point.to_owned()), trimmed_group_path)); + } + + None + } +} + #[cfg(all( - not(all(target_os = "linux", not(target_env = "musl"))), + not(target_os = "linux"), not(target_os = "freebsd"), not(target_os = "macos"), - not(all(target_os = "netbsd", not(target_vendor = "rumprun"))), + not(target_os = "netbsd"), not(target_os = "openbsd"), not(target_os = "solaris") ))] @@ -226,23 +644,28 @@ pub mod guard { } #[cfg(any( - all(target_os = "linux", not(target_env = "musl")), + target_os = "linux", target_os = "freebsd", target_os = "macos", - all(target_os = "netbsd", not(target_vendor = "rumprun")), + target_os = "netbsd", target_os = "openbsd", target_os = "solaris" ))] #[cfg_attr(test, allow(dead_code))] pub mod guard { - use libc::{mmap, mprotect}; + #[cfg(not(all(target_os = "linux", target_env = "gnu")))] + use libc::{mmap as mmap64, mprotect}; + #[cfg(all(target_os = "linux", target_env = "gnu"))] + use libc::{mmap64, mprotect}; use libc::{MAP_ANON, MAP_FAILED, MAP_FIXED, MAP_PRIVATE, PROT_NONE, PROT_READ, PROT_WRITE}; + use crate::io; use crate::ops::Range; + use crate::sync::atomic::{AtomicUsize, Ordering}; use crate::sys::os; // This is initialized in init() and only read from after - static mut PAGE_SIZE: usize = 0; + static PAGE_SIZE: AtomicUsize = AtomicUsize::new(0); pub type Guard = Range; @@ -256,9 +679,8 @@ pub mod guard { #[cfg(target_os = "macos")] unsafe fn get_stack_start() -> Option<*mut libc::c_void> { let th = libc::pthread_self(); - let stackaddr = - libc::pthread_get_stackaddr_np(th) as usize - libc::pthread_get_stacksize_np(th); - Some(stackaddr as *mut libc::c_void) + let stackptr = libc::pthread_get_stackaddr_np(th); + Some(stackptr.map_addr(|addr| addr - libc::pthread_get_stacksize_np(th))) } #[cfg(target_os = "openbsd")] @@ -266,14 +688,15 @@ pub mod guard { let mut current_stack: libc::stack_t = crate::mem::zeroed(); assert_eq!(libc::pthread_stackseg_np(libc::pthread_self(), &mut current_stack), 0); + let stack_ptr = current_stack.ss_sp; let stackaddr = if libc::pthread_main_np() == 1 { // main thread - current_stack.ss_sp as usize - current_stack.ss_size + PAGE_SIZE + stack_ptr.addr() - current_stack.ss_size + PAGE_SIZE.load(Ordering::Relaxed) } else { // new thread - current_stack.ss_sp as usize - current_stack.ss_size + stack_ptr.addr() - current_stack.ss_size }; - Some(stackaddr as *mut libc::c_void) + Some(stack_ptr.with_addr(stackaddr)) } #[cfg(any( @@ -286,6 +709,7 @@ pub mod guard { unsafe fn get_stack_start() -> Option<*mut libc::c_void> { let mut ret = None; let mut attr: libc::pthread_attr_t = crate::mem::zeroed(); + #[cfg(target_os = "freebsd")] assert_eq!(libc::pthread_attr_init(&mut attr), 0); #[cfg(target_os = "freebsd")] let e = libc::pthread_attr_get_np(libc::pthread_self(), &mut attr); @@ -297,14 +721,18 @@ pub mod guard { assert_eq!(libc::pthread_attr_getstack(&attr, &mut stackaddr, &mut stacksize), 0); ret = Some(stackaddr); } - assert_eq!(libc::pthread_attr_destroy(&mut attr), 0); + if e == 0 || cfg!(target_os = "freebsd") { + assert_eq!(libc::pthread_attr_destroy(&mut attr), 0); + } ret } // Precondition: PAGE_SIZE is initialized. unsafe fn get_stack_start_aligned() -> Option<*mut libc::c_void> { - assert!(PAGE_SIZE != 0); - let stackaddr = get_stack_start()?; + let page_size = PAGE_SIZE.load(Ordering::Relaxed); + assert!(page_size != 0); + let stackptr = get_stack_start()?; + let stackaddr = stackptr.addr(); // Ensure stackaddr is page aligned! A parent process might // have reset RLIMIT_STACK to be non-page aligned. The @@ -312,32 +740,62 @@ pub mod guard { // stackaddr < stackaddr + stacksize, so if stackaddr is not // page-aligned, calculate the fix such that stackaddr < // new_page_aligned_stackaddr < stackaddr + stacksize - let remainder = (stackaddr as usize) % PAGE_SIZE; + let remainder = stackaddr % page_size; Some(if remainder == 0 { - stackaddr + stackptr } else { - ((stackaddr as usize) + PAGE_SIZE - remainder) as *mut libc::c_void + stackptr.with_addr(stackaddr + page_size - remainder) }) } pub unsafe fn init() -> Option { - PAGE_SIZE = os::page_size(); - - let stackaddr = get_stack_start_aligned()?; + let page_size = os::page_size(); + PAGE_SIZE.store(page_size, Ordering::Relaxed); - if cfg!(target_os = "linux") { + if cfg!(all(target_os = "linux", not(target_env = "musl"))) { // Linux doesn't allocate the whole stack right away, and // the kernel has its own stack-guard mechanism to fault - // when growing too close to an existing mapping. If we map + // when growing too close to an existing mapping. If we map // our own guard, then the kernel starts enforcing a rather // large gap above that, rendering much of the possible - // stack space useless. See #43052. + // stack space useless. See #43052. // // Instead, we'll just note where we expect rlimit to start // faulting, so our handler can report "stack overflow", and // trust that the kernel's own stack guard will work. - let stackaddr = stackaddr as usize; - Some(stackaddr - PAGE_SIZE..stackaddr) + let stackptr = get_stack_start_aligned()?; + let stackaddr = stackptr.addr(); + Some(stackaddr - page_size..stackaddr) + } else if cfg!(all(target_os = "linux", target_env = "musl")) { + // For the main thread, the musl's pthread_attr_getstack + // returns the current stack size, rather than maximum size + // it can eventually grow to. It cannot be used to determine + // the position of kernel's stack guard. + None + } else if cfg!(target_os = "freebsd") { + // FreeBSD's stack autogrows, and optionally includes a guard page + // at the bottom. If we try to remap the bottom of the stack + // ourselves, FreeBSD's guard page moves upwards. So we'll just use + // the builtin guard page. + let stackptr = get_stack_start_aligned()?; + let guardaddr = stackptr.addr(); + // Technically the number of guard pages is tunable and controlled + // by the security.bsd.stack_guard_page sysctl, but there are + // few reasons to change it from the default. The default value has + // been 1 ever since FreeBSD 11.1 and 10.4. + const GUARD_PAGES: usize = 1; + let guard = guardaddr..guardaddr + GUARD_PAGES * page_size; + Some(guard) + } else if cfg!(target_os = "openbsd") { + // OpenBSD stack already includes a guard page, and stack is + // immutable. + // + // We'll just note where we expect rlimit to start + // faulting, so our handler can report "stack overflow", and + // trust that the kernel's own stack guard will work. + let stackptr = get_stack_start_aligned()?; + let stackaddr = stackptr.addr(); + Some(stackaddr - page_size..stackaddr) } else { // Reallocate the last page of the stack. // This ensures SIGBUS will be raised on @@ -347,34 +805,35 @@ pub mod guard { // than the initial mmap() used, so we mmap() here with // read/write permissions and only then mprotect() it to // no permissions at all. See issue #50313. - let result = mmap( - stackaddr, - PAGE_SIZE, + let stackptr = get_stack_start_aligned()?; + let result = mmap64( + stackptr, + page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0, ); - if result != stackaddr || result == MAP_FAILED { - panic!("failed to allocate a guard page"); + if result != stackptr || result == MAP_FAILED { + panic!("failed to allocate a guard page: {}", io::Error::last_os_error()); } - let result = mprotect(stackaddr, PAGE_SIZE, PROT_NONE); + let result = mprotect(stackptr, page_size, PROT_NONE); if result != 0 { - panic!("failed to protect the guard page"); + panic!("failed to protect the guard page: {}", io::Error::last_os_error()); } - let guardaddr = stackaddr as usize; - let offset = if cfg!(target_os = "freebsd") { 2 } else { 1 }; + let guardaddr = stackptr.addr(); - Some(guardaddr..guardaddr + offset * PAGE_SIZE) + Some(guardaddr..guardaddr + page_size) } } #[cfg(any(target_os = "macos", target_os = "openbsd", target_os = "solaris"))] pub unsafe fn current() -> Option { - let stackaddr = get_stack_start()? as usize; - Some(stackaddr - PAGE_SIZE..stackaddr) + let stackptr = get_stack_start()?; + let stackaddr = stackptr.addr(); + Some(stackaddr - PAGE_SIZE.load(Ordering::Relaxed)..stackaddr) } #[cfg(any( @@ -387,6 +846,7 @@ pub mod guard { pub unsafe fn current() -> Option { let mut ret = None; let mut attr: libc::pthread_attr_t = crate::mem::zeroed(); + #[cfg(target_os = "freebsd")] assert_eq!(libc::pthread_attr_init(&mut attr), 0); #[cfg(target_os = "freebsd")] let e = libc::pthread_attr_get_np(libc::pthread_self(), &mut attr); @@ -396,24 +856,30 @@ pub mod guard { let mut guardsize = 0; assert_eq!(libc::pthread_attr_getguardsize(&attr, &mut guardsize), 0); if guardsize == 0 { - panic!("there is no guard page"); + if cfg!(all(target_os = "linux", target_env = "musl")) { + // musl versions before 1.1.19 always reported guard + // size obtained from pthread_attr_get_np as zero. + // Use page size as a fallback. + guardsize = PAGE_SIZE.load(Ordering::Relaxed); + } else { + panic!("there is no guard page"); + } } - let mut stackaddr = crate::ptr::null_mut(); + let mut stackptr = crate::ptr::null_mut::(); let mut size = 0; - assert_eq!(libc::pthread_attr_getstack(&attr, &mut stackaddr, &mut size), 0); - - let stackaddr = stackaddr as usize; - ret = if cfg!(target_os = "freebsd") { - // FIXME does freebsd really fault *below* the guard addr? - let guardaddr = stackaddr - guardsize; - Some(guardaddr - PAGE_SIZE..guardaddr) - } else if cfg!(target_os = "netbsd") { + assert_eq!(libc::pthread_attr_getstack(&attr, &mut stackptr, &mut size), 0); + + let stackaddr = stackptr.addr(); + ret = if cfg!(any(target_os = "freebsd", target_os = "netbsd")) { + Some(stackaddr - guardsize..stackaddr) + } else if cfg!(all(target_os = "linux", target_env = "musl")) { Some(stackaddr - guardsize..stackaddr) - } else if cfg!(all(target_os = "linux", target_env = "gnu")) { + } else if cfg!(all(target_os = "linux", any(target_env = "gnu", target_env = "uclibc"))) + { // glibc used to include the guard area within the stack, as noted in the BUGS - // section of `man pthread_attr_getguardsize`. This has been corrected starting + // section of `man pthread_attr_getguardsize`. This has been corrected starting // with glibc 2.27, and in some distro backports, so the guard is now placed at the - // end (below) the stack. There's no easy way for us to know which we have at + // end (below) the stack. There's no easy way for us to know which we have at // runtime, so we'll just match any fault in the range right above or below the // stack base to call that fault a stack overflow. Some(stackaddr - guardsize..stackaddr + guardsize) @@ -421,7 +887,9 @@ pub mod guard { Some(stackaddr..stackaddr + guardsize) }; } - assert_eq!(libc::pthread_attr_destroy(&mut attr), 0); + if e == 0 || cfg!(target_os = "freebsd") { + assert_eq!(libc::pthread_attr_destroy(&mut attr), 0); + } ret } } @@ -431,10 +899,12 @@ pub mod guard { // We need that information to avoid blowing up when a small stack // is created in an application with big thread-local storage requirements. // See #6233 for rationale and details. -#[cfg(target_os = "linux")] -#[allow(deprecated)] +#[cfg(all(target_os = "linux", target_env = "gnu"))] fn min_stack_size(attr: *const libc::pthread_attr_t) -> usize { - weak!(fn __pthread_get_minstack(*const libc::pthread_attr_t) -> libc::size_t); + // We use dlsym to avoid an ELF version dependency on GLIBC_PRIVATE. (#23628) + // We shouldn't really be using such an internal symbol, but there's currently + // no other way to account for the TLS size. + dlsym!(fn __pthread_get_minstack(*const libc::pthread_attr_t) -> libc::size_t); match __pthread_get_minstack.get() { None => libc::PTHREAD_STACK_MIN, @@ -442,9 +912,8 @@ fn min_stack_size(attr: *const libc::pthread_attr_t) -> usize { } } -// No point in looking up __pthread_get_minstack() on non-glibc -// platforms. -#[cfg(all(not(target_os = "linux"), not(target_os = "netbsd")))] +// No point in looking up __pthread_get_minstack() on non-glibc platforms. +#[cfg(all(not(all(target_os = "linux", target_env = "gnu")), not(target_os = "netbsd")))] fn min_stack_size(_: *const libc::pthread_attr_t) -> usize { libc::PTHREAD_STACK_MIN } diff --git a/crux-mir/lib/std/src/sys/unix/fast_thread_local.rs b/crux-mir/lib/std/src/sys/unix/thread_local_dtor.rs similarity index 84% rename from crux-mir/lib/std/src/sys/unix/fast_thread_local.rs rename to crux-mir/lib/std/src/sys/unix/thread_local_dtor.rs index 8730b4de8..d7fd2130f 100644 --- a/crux-mir/lib/std/src/sys/unix/fast_thread_local.rs +++ b/crux-mir/lib/std/src/sys/unix/thread_local_dtor.rs @@ -1,6 +1,9 @@ #![cfg(target_thread_local)] #![unstable(feature = "thread_local_internals", issue = "none")] +//! Provides thread-local destructors without an associated "key", which +//! can be more efficient. + // Since what appears to be glibc 2.18 this symbol has been shipped which // GCC and clang both use to invoke destructors in thread_local globals, so // let's do the same! @@ -14,9 +17,10 @@ target_os = "redox", target_os = "emscripten" ))] +#[cfg_attr(target_family = "wasm", allow(unused))] // might remain unused depending on target details (e.g. wasm32-unknown-emscripten) pub unsafe fn register_dtor(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { use crate::mem; - use crate::sys_common::thread_local::register_dtor_fallback; + use crate::sys_common::thread_local_dtor::register_dtor_fallback; extern "C" { #[linkage = "extern_weak"] @@ -89,3 +93,9 @@ pub unsafe fn register_dtor(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { } } } + +#[cfg(any(target_os = "vxworks", target_os = "horizon"))] +pub unsafe fn register_dtor(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { + use crate::sys_common::thread_local_dtor::register_dtor_fallback; + register_dtor_fallback(t, dtor); +} diff --git a/crux-mir/lib/std/src/sys/unix/thread_local.rs b/crux-mir/lib/std/src/sys/unix/thread_local_key.rs similarity index 90% rename from crux-mir/lib/std/src/sys/unix/thread_local.rs rename to crux-mir/lib/std/src/sys/unix/thread_local_key.rs index 2c5b94b1e..2b2d079ee 100644 --- a/crux-mir/lib/std/src/sys/unix/thread_local.rs +++ b/crux-mir/lib/std/src/sys/unix/thread_local_key.rs @@ -27,8 +27,3 @@ pub unsafe fn destroy(key: Key) { let r = libc::pthread_key_delete(key); debug_assert_eq!(r, 0); } - -#[inline] -pub fn requires_synchronized_create() -> bool { - false -} diff --git a/crux-mir/lib/std/src/sys/unix/thread_parking/darwin.rs b/crux-mir/lib/std/src/sys/unix/thread_parking/darwin.rs new file mode 100644 index 000000000..b709fada3 --- /dev/null +++ b/crux-mir/lib/std/src/sys/unix/thread_parking/darwin.rs @@ -0,0 +1,131 @@ +//! Thread parking for Darwin-based systems. +//! +//! Darwin actually has futex syscalls (`__ulock_wait`/`__ulock_wake`), but they +//! cannot be used in `std` because they are non-public (their use will lead to +//! rejection from the App Store) and because they are only available starting +//! with macOS version 10.12, even though the minimum target version is 10.7. +//! +//! Therefore, we need to look for other synchronization primitives. Luckily, Darwin +//! supports semaphores, which allow us to implement the behaviour we need with +//! only one primitive (as opposed to a mutex-condvar pair). We use the semaphore +//! provided by libdispatch, as the underlying Mach semaphore is only dubiously +//! public. + +use crate::pin::Pin; +use crate::sync::atomic::{ + AtomicI8, + Ordering::{Acquire, Release}, +}; +use crate::time::Duration; + +type dispatch_semaphore_t = *mut crate::ffi::c_void; +type dispatch_time_t = u64; + +const DISPATCH_TIME_NOW: dispatch_time_t = 0; +const DISPATCH_TIME_FOREVER: dispatch_time_t = !0; + +// Contained in libSystem.dylib, which is linked by default. +extern "C" { + fn dispatch_time(when: dispatch_time_t, delta: i64) -> dispatch_time_t; + fn dispatch_semaphore_create(val: isize) -> dispatch_semaphore_t; + fn dispatch_semaphore_wait(dsema: dispatch_semaphore_t, timeout: dispatch_time_t) -> isize; + fn dispatch_semaphore_signal(dsema: dispatch_semaphore_t) -> isize; + fn dispatch_release(object: *mut crate::ffi::c_void); +} + +const EMPTY: i8 = 0; +const NOTIFIED: i8 = 1; +const PARKED: i8 = -1; + +pub struct Parker { + semaphore: dispatch_semaphore_t, + state: AtomicI8, +} + +unsafe impl Sync for Parker {} +unsafe impl Send for Parker {} + +impl Parker { + pub unsafe fn new_in_place(parker: *mut Parker) { + let semaphore = dispatch_semaphore_create(0); + assert!( + !semaphore.is_null(), + "failed to create dispatch semaphore for thread synchronization" + ); + parker.write(Parker { semaphore, state: AtomicI8::new(EMPTY) }) + } + + // Does not need `Pin`, but other implementation do. + pub unsafe fn park(self: Pin<&Self>) { + // The semaphore counter must be zero at this point, because unparking + // threads will not actually increase it until we signalled that we + // are waiting. + + // Change NOTIFIED to EMPTY and EMPTY to PARKED. + if self.state.fetch_sub(1, Acquire) == NOTIFIED { + return; + } + + // Another thread may increase the semaphore counter from this point on. + // If it is faster than us, we will decrement it again immediately below. + // If we are faster, we wait. + + // Ensure that the semaphore counter has actually been decremented, even + // if the call timed out for some reason. + while dispatch_semaphore_wait(self.semaphore, DISPATCH_TIME_FOREVER) != 0 {} + + // At this point, the semaphore counter is zero again. + + // We were definitely woken up, so we don't need to check the state. + // Still, we need to reset the state using a swap to observe the state + // change with acquire ordering. + self.state.swap(EMPTY, Acquire); + } + + // Does not need `Pin`, but other implementation do. + pub unsafe fn park_timeout(self: Pin<&Self>, dur: Duration) { + if self.state.fetch_sub(1, Acquire) == NOTIFIED { + return; + } + + let nanos = dur.as_nanos().try_into().unwrap_or(i64::MAX); + let timeout = dispatch_time(DISPATCH_TIME_NOW, nanos); + + let timeout = dispatch_semaphore_wait(self.semaphore, timeout) != 0; + + let state = self.state.swap(EMPTY, Acquire); + if state == NOTIFIED && timeout { + // If the state was NOTIFIED but semaphore_wait returned without + // decrementing the count because of a timeout, it means another + // thread is about to call semaphore_signal. We must wait for that + // to happen to ensure the semaphore count is reset. + while dispatch_semaphore_wait(self.semaphore, DISPATCH_TIME_FOREVER) != 0 {} + } else { + // Either a timeout occurred and we reset the state before any thread + // tried to wake us up, or we were woken up and reset the state, + // making sure to observe the state change with acquire ordering. + // Either way, the semaphore counter is now zero again. + } + } + + // Does not need `Pin`, but other implementation do. + pub fn unpark(self: Pin<&Self>) { + let state = self.state.swap(NOTIFIED, Release); + if state == PARKED { + unsafe { + dispatch_semaphore_signal(self.semaphore); + } + } + } +} + +impl Drop for Parker { + fn drop(&mut self) { + // SAFETY: + // We always ensure that the semaphore count is reset, so this will + // never cause an exception. + unsafe { + dispatch_release(self.semaphore); + } + } +} diff --git a/crux-mir/lib/std/src/sys/unix/thread_parking/mod.rs b/crux-mir/lib/std/src/sys/unix/thread_parking/mod.rs new file mode 100644 index 000000000..185333c07 --- /dev/null +++ b/crux-mir/lib/std/src/sys/unix/thread_parking/mod.rs @@ -0,0 +1,32 @@ +//! Thread parking on systems without futex support. + +#![cfg(not(any( + target_os = "linux", + target_os = "android", + all(target_os = "emscripten", target_feature = "atomics"), + target_os = "freebsd", + target_os = "openbsd", + target_os = "dragonfly", + target_os = "fuchsia", +)))] + +cfg_if::cfg_if! { + if #[cfg(all( + any( + target_os = "macos", + target_os = "ios", + target_os = "watchos", + target_os = "tvos", + ), + not(miri), + ))] { + mod darwin; + pub use darwin::Parker; + } else if #[cfg(target_os = "netbsd")] { + mod netbsd; + pub use netbsd::{current, park, park_timeout, unpark, ThreadId}; + } else { + mod pthread; + pub use pthread::Parker; + } +} diff --git a/crux-mir/lib/std/src/sys/unix/thread_parking/netbsd.rs b/crux-mir/lib/std/src/sys/unix/thread_parking/netbsd.rs new file mode 100644 index 000000000..3be081221 --- /dev/null +++ b/crux-mir/lib/std/src/sys/unix/thread_parking/netbsd.rs @@ -0,0 +1,52 @@ +use crate::ffi::{c_int, c_void}; +use crate::ptr; +use crate::time::Duration; +use libc::{_lwp_self, clockid_t, lwpid_t, time_t, timespec, CLOCK_MONOTONIC}; + +extern "C" { + fn ___lwp_park60( + clock_id: clockid_t, + flags: c_int, + ts: *mut timespec, + unpark: lwpid_t, + hint: *const c_void, + unparkhint: *const c_void, + ) -> c_int; + fn _lwp_unpark(lwp: lwpid_t, hint: *const c_void) -> c_int; +} + +pub type ThreadId = lwpid_t; + +#[inline] +pub fn current() -> ThreadId { + unsafe { _lwp_self() } +} + +#[inline] +pub fn park(hint: usize) { + unsafe { + ___lwp_park60(0, 0, ptr::null_mut(), 0, ptr::invalid(hint), ptr::null()); + } +} + +pub fn park_timeout(dur: Duration, hint: usize) { + let mut timeout = timespec { + // Saturate so that the operation will definitely time out + // (even if it is after the heat death of the universe). + tv_sec: dur.as_secs().try_into().ok().unwrap_or(time_t::MAX), + tv_nsec: dur.subsec_nanos().into(), + }; + + // Timeout needs to be mutable since it is modified on NetBSD 9.0 and + // above. + unsafe { + ___lwp_park60(CLOCK_MONOTONIC, 0, &mut timeout, 0, ptr::invalid(hint), ptr::null()); + } +} + +#[inline] +pub fn unpark(tid: ThreadId, hint: usize) { + unsafe { + _lwp_unpark(tid, ptr::invalid(hint)); + } +} diff --git a/crux-mir/lib/std/src/sys/unix/thread_parking/pthread.rs b/crux-mir/lib/std/src/sys/unix/thread_parking/pthread.rs new file mode 100644 index 000000000..082d25e68 --- /dev/null +++ b/crux-mir/lib/std/src/sys/unix/thread_parking/pthread.rs @@ -0,0 +1,271 @@ +//! Thread parking without `futex` using the `pthread` synchronization primitives. + +use crate::cell::UnsafeCell; +use crate::marker::PhantomPinned; +use crate::pin::Pin; +use crate::ptr::addr_of_mut; +use crate::sync::atomic::AtomicUsize; +use crate::sync::atomic::Ordering::SeqCst; +use crate::sys::time::TIMESPEC_MAX; +use crate::time::Duration; + +const EMPTY: usize = 0; +const PARKED: usize = 1; +const NOTIFIED: usize = 2; + +unsafe fn lock(lock: *mut libc::pthread_mutex_t) { + let r = libc::pthread_mutex_lock(lock); + debug_assert_eq!(r, 0); +} + +unsafe fn unlock(lock: *mut libc::pthread_mutex_t) { + let r = libc::pthread_mutex_unlock(lock); + debug_assert_eq!(r, 0); +} + +unsafe fn notify_one(cond: *mut libc::pthread_cond_t) { + let r = libc::pthread_cond_signal(cond); + debug_assert_eq!(r, 0); +} + +unsafe fn wait(cond: *mut libc::pthread_cond_t, lock: *mut libc::pthread_mutex_t) { + let r = libc::pthread_cond_wait(cond, lock); + debug_assert_eq!(r, 0); +} + +unsafe fn wait_timeout( + cond: *mut libc::pthread_cond_t, + lock: *mut libc::pthread_mutex_t, + dur: Duration, +) { + // Use the system clock on systems that do not support pthread_condattr_setclock. + // This unfortunately results in problems when the system time changes. + #[cfg(any( + target_os = "macos", + target_os = "ios", + target_os = "watchos", + target_os = "espidf", + target_os = "horizon", + ))] + let (now, dur) = { + use crate::cmp::min; + use crate::sys::time::SystemTime; + + // OSX implementation of `pthread_cond_timedwait` is buggy + // with super long durations. When duration is greater than + // 0x100_0000_0000_0000 seconds, `pthread_cond_timedwait` + // in macOS Sierra return error 316. + // + // This program demonstrates the issue: + // https://gist.github.com/stepancheg/198db4623a20aad2ad7cddb8fda4a63c + // + // To work around this issue, and possible bugs of other OSes, timeout + // is clamped to 1000 years, which is allowable per the API of `park_timeout` + // because of spurious wakeups. + let dur = min(dur, Duration::from_secs(1000 * 365 * 86400)); + let now = SystemTime::now().t; + (now, dur) + }; + // Use the monotonic clock on other systems. + #[cfg(not(any( + target_os = "macos", + target_os = "ios", + target_os = "watchos", + target_os = "espidf", + target_os = "horizon", + )))] + let (now, dur) = { + use crate::sys::time::Timespec; + + (Timespec::now(libc::CLOCK_MONOTONIC), dur) + }; + + let timeout = + now.checked_add_duration(&dur).and_then(|t| t.to_timespec()).unwrap_or(TIMESPEC_MAX); + let r = libc::pthread_cond_timedwait(cond, lock, &timeout); + debug_assert!(r == libc::ETIMEDOUT || r == 0); +} + +pub struct Parker { + state: AtomicUsize, + lock: UnsafeCell, + cvar: UnsafeCell, + // The `pthread` primitives require a stable address, so make this struct `!Unpin`. + _pinned: PhantomPinned, +} + +impl Parker { + /// Construct the UNIX parker in-place. + /// + /// # Safety + /// The constructed parker must never be moved. + pub unsafe fn new_in_place(parker: *mut Parker) { + // Use the default mutex implementation to allow for simpler initialization. + // This could lead to undefined behaviour when deadlocking. This is avoided + // by not deadlocking. Note in particular the unlocking operation before any + // panic, as code after the panic could try to park again. + addr_of_mut!((*parker).state).write(AtomicUsize::new(EMPTY)); + addr_of_mut!((*parker).lock).write(UnsafeCell::new(libc::PTHREAD_MUTEX_INITIALIZER)); + + cfg_if::cfg_if! { + if #[cfg(any( + target_os = "macos", + target_os = "ios", + target_os = "watchos", + target_os = "l4re", + target_os = "android", + target_os = "redox" + ))] { + addr_of_mut!((*parker).cvar).write(UnsafeCell::new(libc::PTHREAD_COND_INITIALIZER)); + } else if #[cfg(any(target_os = "espidf", target_os = "horizon"))] { + let r = libc::pthread_cond_init(addr_of_mut!((*parker).cvar).cast(), crate::ptr::null()); + assert_eq!(r, 0); + } else { + use crate::mem::MaybeUninit; + let mut attr = MaybeUninit::::uninit(); + let r = libc::pthread_condattr_init(attr.as_mut_ptr()); + assert_eq!(r, 0); + let r = libc::pthread_condattr_setclock(attr.as_mut_ptr(), libc::CLOCK_MONOTONIC); + assert_eq!(r, 0); + let r = libc::pthread_cond_init(addr_of_mut!((*parker).cvar).cast(), attr.as_ptr()); + assert_eq!(r, 0); + let r = libc::pthread_condattr_destroy(attr.as_mut_ptr()); + assert_eq!(r, 0); + } + } + } + + // This implementation doesn't require `unsafe`, but other implementations + // may assume this is only called by the thread that owns the Parker. + pub unsafe fn park(self: Pin<&Self>) { + // If we were previously notified then we consume this notification and + // return quickly. + if self.state.compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst).is_ok() { + return; + } + + // Otherwise we need to coordinate going to sleep + lock(self.lock.get()); + match self.state.compare_exchange(EMPTY, PARKED, SeqCst, SeqCst) { + Ok(_) => {} + Err(NOTIFIED) => { + // We must read here, even though we know it will be `NOTIFIED`. + // This is because `unpark` may have been called again since we read + // `NOTIFIED` in the `compare_exchange` above. We must perform an + // acquire operation that synchronizes with that `unpark` to observe + // any writes it made before the call to unpark. To do that we must + // read from the write it made to `state`. + let old = self.state.swap(EMPTY, SeqCst); + + unlock(self.lock.get()); + + assert_eq!(old, NOTIFIED, "park state changed unexpectedly"); + return; + } // should consume this notification, so prohibit spurious wakeups in next park. + Err(_) => { + unlock(self.lock.get()); + + panic!("inconsistent park state") + } + } + + loop { + wait(self.cvar.get(), self.lock.get()); + + match self.state.compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) { + Ok(_) => break, // got a notification + Err(_) => {} // spurious wakeup, go back to sleep + } + } + + unlock(self.lock.get()); + } + + // This implementation doesn't require `unsafe`, but other implementations + // may assume this is only called by the thread that owns the Parker. Use + // `Pin` to guarantee a stable address for the mutex and condition variable. + pub unsafe fn park_timeout(self: Pin<&Self>, dur: Duration) { + // Like `park` above we have a fast path for an already-notified thread, and + // afterwards we start coordinating for a sleep. + // return quickly. + if self.state.compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst).is_ok() { + return; + } + + lock(self.lock.get()); + match self.state.compare_exchange(EMPTY, PARKED, SeqCst, SeqCst) { + Ok(_) => {} + Err(NOTIFIED) => { + // We must read again here, see `park`. + let old = self.state.swap(EMPTY, SeqCst); + unlock(self.lock.get()); + + assert_eq!(old, NOTIFIED, "park state changed unexpectedly"); + return; + } // should consume this notification, so prohibit spurious wakeups in next park. + Err(_) => { + unlock(self.lock.get()); + panic!("inconsistent park_timeout state") + } + } + + // Wait with a timeout, and if we spuriously wake up or otherwise wake up + // from a notification we just want to unconditionally set the state back to + // empty, either consuming a notification or un-flagging ourselves as + // parked. + wait_timeout(self.cvar.get(), self.lock.get(), dur); + + match self.state.swap(EMPTY, SeqCst) { + NOTIFIED => unlock(self.lock.get()), // got a notification, hurray! + PARKED => unlock(self.lock.get()), // no notification, alas + n => { + unlock(self.lock.get()); + panic!("inconsistent park_timeout state: {n}") + } + } + } + + pub fn unpark(self: Pin<&Self>) { + // To ensure the unparked thread will observe any writes we made + // before this call, we must perform a release operation that `park` + // can synchronize with. To do that we must write `NOTIFIED` even if + // `state` is already `NOTIFIED`. That is why this must be a swap + // rather than a compare-and-swap that returns if it reads `NOTIFIED` + // on failure. + match self.state.swap(NOTIFIED, SeqCst) { + EMPTY => return, // no one was waiting + NOTIFIED => return, // already unparked + PARKED => {} // gotta go wake someone up + _ => panic!("inconsistent state in unpark"), + } + + // There is a period between when the parked thread sets `state` to + // `PARKED` (or last checked `state` in the case of a spurious wake + // up) and when it actually waits on `cvar`. If we were to notify + // during this period it would be ignored and then when the parked + // thread went to sleep it would never wake up. Fortunately, it has + // `lock` locked at this stage so we can acquire `lock` to wait until + // it is ready to receive the notification. + // + // Releasing `lock` before the call to `notify_one` means that when the + // parked thread wakes it doesn't get woken only to have to wait for us + // to release `lock`. + unsafe { + lock(self.lock.get()); + unlock(self.lock.get()); + notify_one(self.cvar.get()); + } + } +} + +impl Drop for Parker { + fn drop(&mut self) { + unsafe { + libc::pthread_cond_destroy(self.cvar.get_mut()); + libc::pthread_mutex_destroy(self.lock.get_mut()); + } + } +} + +unsafe impl Sync for Parker {} +unsafe impl Send for Parker {} diff --git a/crux-mir/lib/std/src/sys/unix/time.rs b/crux-mir/lib/std/src/sys/unix/time.rs index 6707f790c..2daad981b 100644 --- a/crux-mir/lib/std/src/sys/unix/time.rs +++ b/crux-mir/lib/std/src/sys/unix/time.rs @@ -1,36 +1,101 @@ -use crate::cmp::Ordering; +use crate::fmt; use crate::time::Duration; -use core::hash::{Hash, Hasher}; - -pub use self::inner::{Instant, SystemTime, UNIX_EPOCH}; -use crate::convert::TryInto; +pub use self::inner::Instant; const NSEC_PER_SEC: u64 = 1_000_000_000; +pub const UNIX_EPOCH: SystemTime = SystemTime { t: Timespec::zero() }; +#[allow(dead_code)] // Used for pthread condvar timeouts +pub const TIMESPEC_MAX: libc::timespec = + libc::timespec { tv_sec: ::MAX, tv_nsec: 1_000_000_000 - 1 }; + +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[repr(transparent)] +#[rustc_layout_scalar_valid_range_start(0)] +#[rustc_layout_scalar_valid_range_end(999_999_999)] +struct Nanoseconds(u32); + +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct SystemTime { + pub(in crate::sys::unix) t: Timespec, +} + +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(in crate::sys::unix) struct Timespec { + tv_sec: i64, + tv_nsec: Nanoseconds, +} + +impl SystemTime { + #[cfg_attr(target_os = "horizon", allow(unused))] + pub fn new(tv_sec: i64, tv_nsec: i64) -> SystemTime { + SystemTime { t: Timespec::new(tv_sec, tv_nsec) } + } -#[derive(Copy, Clone)] -struct Timespec { - t: libc::timespec, + pub fn sub_time(&self, other: &SystemTime) -> Result { + self.t.sub_timespec(&other.t) + } + + pub fn checked_add_duration(&self, other: &Duration) -> Option { + Some(SystemTime { t: self.t.checked_add_duration(other)? }) + } + + pub fn checked_sub_duration(&self, other: &Duration) -> Option { + Some(SystemTime { t: self.t.checked_sub_duration(other)? }) + } +} + +impl From for SystemTime { + fn from(t: libc::timespec) -> SystemTime { + SystemTime { t: Timespec::from(t) } + } +} + +impl fmt::Debug for SystemTime { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SystemTime") + .field("tv_sec", &self.t.tv_sec) + .field("tv_nsec", &self.t.tv_nsec.0) + .finish() + } } impl Timespec { - const fn zero() -> Timespec { - Timespec { t: libc::timespec { tv_sec: 0, tv_nsec: 0 } } + pub const fn zero() -> Timespec { + Timespec::new(0, 0) } - fn sub_timespec(&self, other: &Timespec) -> Result { + const fn new(tv_sec: i64, tv_nsec: i64) -> Timespec { + assert!(tv_nsec >= 0 && tv_nsec < NSEC_PER_SEC as i64); + // SAFETY: The assert above checks tv_nsec is within the valid range + Timespec { tv_sec, tv_nsec: unsafe { Nanoseconds(tv_nsec as u32) } } + } + + pub fn sub_timespec(&self, other: &Timespec) -> Result { if self >= other { - Ok(if self.t.tv_nsec >= other.t.tv_nsec { - Duration::new( - (self.t.tv_sec - other.t.tv_sec) as u64, - (self.t.tv_nsec - other.t.tv_nsec) as u32, - ) + // NOTE(eddyb) two aspects of this `if`-`else` are required for LLVM + // to optimize it into a branchless form (see also #75545): + // + // 1. `self.tv_sec - other.tv_sec` shows up as a common expression + // in both branches, i.e. the `else` must have its `- 1` + // subtraction after the common one, not interleaved with it + // (it used to be `self.tv_sec - 1 - other.tv_sec`) + // + // 2. the `Duration::new` call (or any other additional complexity) + // is outside of the `if`-`else`, not duplicated in both branches + // + // Ideally this code could be rearranged such that it more + // directly expresses the lower-cost behavior we want from it. + let (secs, nsec) = if self.tv_nsec.0 >= other.tv_nsec.0 { + ((self.tv_sec - other.tv_sec) as u64, self.tv_nsec.0 - other.tv_nsec.0) } else { - Duration::new( - (self.t.tv_sec - 1 - other.t.tv_sec) as u64, - self.t.tv_nsec as u32 + (NSEC_PER_SEC as u32) - other.t.tv_nsec as u32, + ( + (self.tv_sec - other.tv_sec - 1) as u64, + self.tv_nsec.0 + (NSEC_PER_SEC as u32) - other.tv_nsec.0, ) - }) + }; + + Ok(Duration::new(secs, nsec)) } else { match other.sub_timespec(self) { Ok(d) => Err(d), @@ -39,93 +104,72 @@ impl Timespec { } } - fn checked_add_duration(&self, other: &Duration) -> Option { + pub fn checked_add_duration(&self, other: &Duration) -> Option { let mut secs = other .as_secs() - .try_into() // <- target type would be `libc::time_t` + .try_into() // <- target type would be `i64` .ok() - .and_then(|secs| self.t.tv_sec.checked_add(secs))?; + .and_then(|secs| self.tv_sec.checked_add(secs))?; // Nano calculations can't overflow because nanos are <1B which fit // in a u32. - let mut nsec = other.subsec_nanos() + self.t.tv_nsec as u32; + let mut nsec = other.subsec_nanos() + self.tv_nsec.0; if nsec >= NSEC_PER_SEC as u32 { nsec -= NSEC_PER_SEC as u32; secs = secs.checked_add(1)?; } - Some(Timespec { t: libc::timespec { tv_sec: secs, tv_nsec: nsec as _ } }) + Some(Timespec::new(secs, nsec as i64)) } - fn checked_sub_duration(&self, other: &Duration) -> Option { + pub fn checked_sub_duration(&self, other: &Duration) -> Option { let mut secs = other .as_secs() - .try_into() // <- target type would be `libc::time_t` + .try_into() // <- target type would be `i64` .ok() - .and_then(|secs| self.t.tv_sec.checked_sub(secs))?; + .and_then(|secs| self.tv_sec.checked_sub(secs))?; // Similar to above, nanos can't overflow. - let mut nsec = self.t.tv_nsec as i32 - other.subsec_nanos() as i32; + let mut nsec = self.tv_nsec.0 as i32 - other.subsec_nanos() as i32; if nsec < 0 { nsec += NSEC_PER_SEC as i32; secs = secs.checked_sub(1)?; } - Some(Timespec { t: libc::timespec { tv_sec: secs, tv_nsec: nsec as _ } }) + Some(Timespec::new(secs, nsec as i64)) } -} -impl PartialEq for Timespec { - fn eq(&self, other: &Timespec) -> bool { - self.t.tv_sec == other.t.tv_sec && self.t.tv_nsec == other.t.tv_nsec + #[allow(dead_code)] + pub fn to_timespec(&self) -> Option { + Some(libc::timespec { + tv_sec: self.tv_sec.try_into().ok()?, + tv_nsec: self.tv_nsec.0.try_into().ok()?, + }) } } -impl Eq for Timespec {} - -impl PartialOrd for Timespec { - fn partial_cmp(&self, other: &Timespec) -> Option { - Some(self.cmp(other)) +impl From for Timespec { + fn from(t: libc::timespec) -> Timespec { + Timespec::new(t.tv_sec as i64, t.tv_nsec as i64) } } -impl Ord for Timespec { - fn cmp(&self, other: &Timespec) -> Ordering { - let me = (self.t.tv_sec, self.t.tv_nsec); - let other = (other.t.tv_sec, other.t.tv_nsec); - me.cmp(&other) - } -} - -impl Hash for Timespec { - fn hash(&self, state: &mut H) { - self.t.tv_sec.hash(state); - self.t.tv_nsec.hash(state); - } -} - -#[cfg(any(target_os = "macos", target_os = "ios"))] +#[cfg(any( + all(target_os = "macos", any(not(target_arch = "aarch64"))), + target_os = "ios", + target_os = "watchos" +))] mod inner { - use crate::fmt; - use crate::mem; - use crate::sync::atomic::{AtomicUsize, Ordering::SeqCst}; + use crate::sync::atomic::{AtomicU64, Ordering}; use crate::sys::cvt; use crate::sys_common::mul_div_u64; use crate::time::Duration; - use super::Timespec; - use super::NSEC_PER_SEC; + use super::{SystemTime, Timespec, NSEC_PER_SEC}; #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] pub struct Instant { t: u64, } - #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] - pub struct SystemTime { - t: Timespec, - } - - pub const UNIX_EPOCH: SystemTime = SystemTime { t: Timespec::zero() }; - #[repr(C)] #[derive(Copy, Clone)] struct mach_timebase_info { @@ -143,14 +187,6 @@ mod inner { Instant { t: unsafe { mach_absolute_time() } } } - pub const fn zero() -> Instant { - Instant { t: 0 } - } - - pub fn actually_monotonic() -> bool { - true - } - pub fn checked_sub_instant(&self, other: &Instant) -> Option { let diff = self.t.checked_sub(other.t)?; let info = info(); @@ -175,41 +211,17 @@ mod inner { cvt(unsafe { libc::gettimeofday(&mut s, ptr::null_mut()) }).unwrap(); return SystemTime::from(s); } + } - pub fn sub_time(&self, other: &SystemTime) -> Result { - self.t.sub_timespec(&other.t) - } - - pub fn checked_add_duration(&self, other: &Duration) -> Option { - Some(SystemTime { t: self.t.checked_add_duration(other)? }) - } - - pub fn checked_sub_duration(&self, other: &Duration) -> Option { - Some(SystemTime { t: self.t.checked_sub_duration(other)? }) + impl From for Timespec { + fn from(t: libc::timeval) -> Timespec { + Timespec::new(t.tv_sec as i64, 1000 * t.tv_usec as i64) } } impl From for SystemTime { fn from(t: libc::timeval) -> SystemTime { - SystemTime::from(libc::timespec { - tv_sec: t.tv_sec, - tv_nsec: (t.tv_usec * 1000) as libc::c_long, - }) - } - } - - impl From for SystemTime { - fn from(t: libc::timespec) -> SystemTime { - SystemTime { t: Timespec { t } } - } - } - - impl fmt::Debug for SystemTime { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SystemTime") - .field("tv_sec", &self.t.t.tv_sec) - .field("tv_nsec", &self.t.t.tv_nsec) - .finish() + SystemTime { t: Timespec::from(t) } } } @@ -221,67 +233,70 @@ mod inner { } fn info() -> mach_timebase_info { - static mut INFO: mach_timebase_info = mach_timebase_info { numer: 0, denom: 0 }; - static STATE: AtomicUsize = AtomicUsize::new(0); - - unsafe { - // If a previous thread has filled in this global state, use that. - if STATE.load(SeqCst) == 2 { - return INFO; - } + // INFO_BITS conceptually is an `Option`. We can do + // this in 64 bits because we know 0 is never a valid value for the + // `denom` field. + // + // Encoding this as a single `AtomicU64` allows us to use `Relaxed` + // operations, as we are only interested in the effects on a single + // memory location. + static INFO_BITS: AtomicU64 = AtomicU64::new(0); + + // If a previous thread has initialized `INFO_BITS`, use it. + let info_bits = INFO_BITS.load(Ordering::Relaxed); + if info_bits != 0 { + return info_from_bits(info_bits); + } - // ... otherwise learn for ourselves ... - let mut info = mem::zeroed(); - extern "C" { - fn mach_timebase_info(info: mach_timebase_info_t) -> kern_return_t; - } + // ... otherwise learn for ourselves ... + extern "C" { + fn mach_timebase_info(info: mach_timebase_info_t) -> kern_return_t; + } + let mut info = info_from_bits(0); + unsafe { mach_timebase_info(&mut info); - - // ... and attempt to be the one thread that stores it globally for - // all other threads - if STATE.compare_exchange(0, 1, SeqCst, SeqCst).is_ok() { - INFO = info; - STATE.store(2, SeqCst); - } - return info; } + INFO_BITS.store(info_to_bits(info), Ordering::Relaxed); + info + } + + #[inline] + fn info_to_bits(info: mach_timebase_info) -> u64 { + ((info.denom as u64) << 32) | (info.numer as u64) + } + + #[inline] + fn info_from_bits(bits: u64) -> mach_timebase_info { + mach_timebase_info { numer: bits as u32, denom: (bits >> 32) as u32 } } } -#[cfg(not(any(target_os = "macos", target_os = "ios")))] +#[cfg(not(any( + all(target_os = "macos", any(not(target_arch = "aarch64"))), + target_os = "ios", + target_os = "watchos" +)))] mod inner { use crate::fmt; + use crate::mem::MaybeUninit; use crate::sys::cvt; use crate::time::Duration; - use super::Timespec; + use super::{SystemTime, Timespec}; #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct Instant { t: Timespec, } - #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] - pub struct SystemTime { - t: Timespec, - } - - pub const UNIX_EPOCH: SystemTime = SystemTime { t: Timespec::zero() }; - impl Instant { pub fn now() -> Instant { - Instant { t: now(libc::CLOCK_MONOTONIC) } - } - - pub const fn zero() -> Instant { - Instant { t: Timespec::zero() } - } - - pub fn actually_monotonic() -> bool { - (cfg!(target_os = "linux") && cfg!(target_arch = "x86_64")) - || (cfg!(target_os = "linux") && cfg!(target_arch = "x86")) - || cfg!(target_os = "fuchsia") + #[cfg(target_os = "macos")] + const clock_id: libc::clockid_t = libc::CLOCK_UPTIME_RAW; + #[cfg(not(target_os = "macos"))] + const clock_id: libc::clockid_t = libc::CLOCK_MONOTONIC; + Instant { t: Timespec::now(clock_id) } } pub fn checked_sub_instant(&self, other: &Instant) -> Option { @@ -300,53 +315,50 @@ mod inner { impl fmt::Debug for Instant { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Instant") - .field("tv_sec", &self.t.t.tv_sec) - .field("tv_nsec", &self.t.t.tv_nsec) + .field("tv_sec", &self.t.tv_sec) + .field("tv_nsec", &self.t.tv_nsec.0) .finish() } } impl SystemTime { pub fn now() -> SystemTime { - SystemTime { t: now(libc::CLOCK_REALTIME) } - } - - pub fn sub_time(&self, other: &SystemTime) -> Result { - self.t.sub_timespec(&other.t) - } - - pub fn checked_add_duration(&self, other: &Duration) -> Option { - Some(SystemTime { t: self.t.checked_add_duration(other)? }) - } - - pub fn checked_sub_duration(&self, other: &Duration) -> Option { - Some(SystemTime { t: self.t.checked_sub_duration(other)? }) + SystemTime { t: Timespec::now(libc::CLOCK_REALTIME) } } } - impl From for SystemTime { - fn from(t: libc::timespec) -> SystemTime { - SystemTime { t: Timespec { t } } - } - } + impl Timespec { + pub fn now(clock: libc::clockid_t) -> Timespec { + // Try to use 64-bit time in preparation for Y2038. + #[cfg(all(target_os = "linux", target_env = "gnu", target_pointer_width = "32"))] + { + use crate::sys::weak::weak; + + // __clock_gettime64 was added to 32-bit arches in glibc 2.34, + // and it handles both vDSO calls and ENOSYS fallbacks itself. + weak!(fn __clock_gettime64(libc::clockid_t, *mut __timespec64) -> libc::c_int); + + #[repr(C)] + struct __timespec64 { + tv_sec: i64, + #[cfg(target_endian = "big")] + _padding: i32, + tv_nsec: i32, + #[cfg(target_endian = "little")] + _padding: i32, + } + + if let Some(clock_gettime64) = __clock_gettime64.get() { + let mut t = MaybeUninit::uninit(); + cvt(unsafe { clock_gettime64(clock, t.as_mut_ptr()) }).unwrap(); + let t = unsafe { t.assume_init() }; + return Timespec::new(t.tv_sec, t.tv_nsec as i64); + } + } - impl fmt::Debug for SystemTime { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SystemTime") - .field("tv_sec", &self.t.t.tv_sec) - .field("tv_nsec", &self.t.t.tv_nsec) - .finish() + let mut t = MaybeUninit::uninit(); + cvt(unsafe { libc::clock_gettime(clock, t.as_mut_ptr()) }).unwrap(); + Timespec::from(unsafe { t.assume_init() }) } } - - #[cfg(not(target_os = "dragonfly"))] - pub type clock_t = libc::c_int; - #[cfg(target_os = "dragonfly")] - pub type clock_t = libc::c_ulong; - - fn now(clock: clock_t) -> Timespec { - let mut t = Timespec { t: libc::timespec { tv_sec: 0, tv_nsec: 0 } }; - cvt(unsafe { libc::clock_gettime(clock, &mut t.t) }).unwrap(); - t - } } diff --git a/crux-mir/lib/std/src/sys/unix/weak.rs b/crux-mir/lib/std/src/sys/unix/weak.rs index 08cbe5961..62ffee70b 100644 --- a/crux-mir/lib/std/src/sys/unix/weak.rs +++ b/crux-mir/lib/std/src/sys/unix/weak.rs @@ -1,12 +1,11 @@ //! Support for "weak linkage" to symbols on Unix //! -//! Some I/O operations we do in libstd require newer versions of OSes but we -//! need to maintain binary compatibility with older releases for now. In order -//! to use the new functionality when available we use this module for -//! detection. +//! Some I/O operations we do in std require newer versions of OSes but we need +//! to maintain binary compatibility with older releases for now. In order to +//! use the new functionality when available we use this module for detection. //! //! One option to use here is weak linkage, but that is unfortunately only -//! really workable on Linux. Hence, use dlsym to get the symbol value at +//! really workable with ELF. Otherwise, use dlsym to get the symbol value at //! runtime. This is also done for compatibility with older versions of glibc, //! and to avoid creating dependencies on GLIBC_PRIVATE symbols. It assumes that //! we've been dynamically linked to the library the symbol comes from, but that @@ -14,73 +13,173 @@ //! //! A long time ago this used weak linkage for the __pthread_get_minstack //! symbol, but that caused Debian to detect an unnecessarily strict versioned -//! dependency on libc6 (#23628). +//! dependency on libc6 (#23628) because it is GLIBC_PRIVATE. We now use `dlsym` +//! for a runtime lookup of that symbol to avoid the ELF versioned dependency. + +// There are a variety of `#[cfg]`s controlling which targets are involved in +// each instance of `weak!` and `syscall!`. Rather than trying to unify all of +// that, we'll just allow that some unix targets don't use this module at all. +#![allow(dead_code, unused_macros)] use crate::ffi::CStr; -use crate::marker; +use crate::marker::PhantomData; use crate::mem; -use crate::sync::atomic::{AtomicUsize, Ordering}; +use crate::ptr; +use crate::sync::atomic::{self, AtomicPtr, Ordering}; -macro_rules! weak { +// We can use true weak linkage on ELF targets. +#[cfg(not(any(target_os = "macos", target_os = "ios")))] +pub(crate) macro weak { (fn $name:ident($($t:ty),*) -> $ret:ty) => ( - static $name: crate::sys::weak::Weak $ret> = - crate::sys::weak::Weak::new(concat!(stringify!($name), '\0')); + let ref $name: ExternWeak $ret> = { + extern "C" { + #[linkage = "extern_weak"] + static $name: Option $ret>; + } + #[allow(unused_unsafe)] + ExternWeak::new(unsafe { $name }) + }; ) } -pub struct Weak { +// On non-ELF targets, use the dlsym approximation of weak linkage. +#[cfg(any(target_os = "macos", target_os = "ios"))] +pub(crate) use self::dlsym as weak; + +pub(crate) struct ExternWeak { + weak_ptr: Option, +} + +impl ExternWeak { + #[inline] + pub(crate) fn new(weak_ptr: Option) -> Self { + ExternWeak { weak_ptr } + } + + #[inline] + pub(crate) fn get(&self) -> Option { + self.weak_ptr + } +} + +pub(crate) macro dlsym { + (fn $name:ident($($t:ty),*) -> $ret:ty) => ( + dlsym!(fn $name($($t),*) -> $ret, stringify!($name)); + ), + (fn $name:ident($($t:ty),*) -> $ret:ty, $sym:expr) => ( + static DLSYM: DlsymWeak $ret> = + DlsymWeak::new(concat!($sym, '\0')); + let $name = &DLSYM; + ) +} +pub(crate) struct DlsymWeak { name: &'static str, - addr: AtomicUsize, - _marker: marker::PhantomData, + func: AtomicPtr, + _marker: PhantomData, } -impl Weak { - pub const fn new(name: &'static str) -> Weak { - Weak { name, addr: AtomicUsize::new(1), _marker: marker::PhantomData } +impl DlsymWeak { + pub(crate) const fn new(name: &'static str) -> Self { + DlsymWeak { name, func: AtomicPtr::new(ptr::invalid_mut(1)), _marker: PhantomData } } - pub fn get(&self) -> Option { - assert_eq!(mem::size_of::(), mem::size_of::()); + #[inline] + pub(crate) fn get(&self) -> Option { unsafe { - if self.addr.load(Ordering::SeqCst) == 1 { - self.addr.store(fetch(self.name), Ordering::SeqCst); - } - match self.addr.load(Ordering::SeqCst) { - 0 => None, - addr => Some(mem::transmute_copy::(&addr)), + // Relaxed is fine here because we fence before reading through the + // pointer (see the comment below). + match self.func.load(Ordering::Relaxed) { + func if func.addr() == 1 => self.initialize(), + func if func.is_null() => None, + func => { + let func = mem::transmute_copy::<*mut libc::c_void, F>(&func); + // The caller is presumably going to read through this value + // (by calling the function we've dlsymed). This means we'd + // need to have loaded it with at least C11's consume + // ordering in order to be guaranteed that the data we read + // from the pointer isn't from before the pointer was + // stored. Rust has no equivalent to memory_order_consume, + // so we use an acquire fence (sorry, ARM). + // + // Now, in practice this likely isn't needed even on CPUs + // where relaxed and consume mean different things. The + // symbols we're loading are probably present (or not) at + // init, and even if they aren't the runtime dynamic loader + // is extremely likely have sufficient barriers internally + // (possibly implicitly, for example the ones provided by + // invoking `mprotect`). + // + // That said, none of that's *guaranteed*, and so we fence. + atomic::fence(Ordering::Acquire); + Some(func) + } } } } + + // Cold because it should only happen during first-time initialization. + #[cold] + unsafe fn initialize(&self) -> Option { + assert_eq!(mem::size_of::(), mem::size_of::<*mut libc::c_void>()); + + let val = fetch(self.name); + // This synchronizes with the acquire fence in `get`. + self.func.store(val, Ordering::Release); + + if val.is_null() { None } else { Some(mem::transmute_copy::<*mut libc::c_void, F>(&val)) } + } } -unsafe fn fetch(name: &str) -> usize { +unsafe fn fetch(name: &str) -> *mut libc::c_void { let name = match CStr::from_bytes_with_nul(name.as_bytes()) { Ok(cstr) => cstr, - Err(..) => return 0, + Err(..) => return ptr::null_mut(), }; - libc::dlsym(libc::RTLD_DEFAULT, name.as_ptr()) as usize + libc::dlsym(libc::RTLD_DEFAULT, name.as_ptr()) } -#[cfg(not(target_os = "linux"))] -macro_rules! syscall { +#[cfg(not(any(target_os = "linux", target_os = "android")))] +pub(crate) macro syscall { (fn $name:ident($($arg_name:ident: $t:ty),*) -> $ret:ty) => ( unsafe fn $name($($arg_name: $t),*) -> $ret { - use super::os; - weak! { fn $name($($t),*) -> $ret } if let Some(fun) = $name.get() { fun($($arg_name),*) } else { - os::set_errno(libc::ENOSYS); + super::os::set_errno(libc::ENOSYS); -1 } } ) } -#[cfg(target_os = "linux")] -macro_rules! syscall { +#[cfg(any(target_os = "linux", target_os = "android"))] +pub(crate) macro syscall { + (fn $name:ident($($arg_name:ident: $t:ty),*) -> $ret:ty) => ( + unsafe fn $name($($arg_name:$t),*) -> $ret { + weak! { fn $name($($t),*) -> $ret } + + // Use a weak symbol from libc when possible, allowing `LD_PRELOAD` + // interposition, but if it's not found just use a raw syscall. + if let Some(fun) = $name.get() { + fun($($arg_name),*) + } else { + // This looks like a hack, but concat_idents only accepts idents + // (not paths). + use libc::*; + + syscall( + concat_idents!(SYS_, $name), + $($arg_name),* + ) as $ret + } + } + ) +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +pub(crate) macro raw_syscall { (fn $name:ident($($arg_name:ident: $t:ty),*) -> $ret:ty) => ( unsafe fn $name($($arg_name:$t),*) -> $ret { // This looks like a hack, but concat_idents only accepts idents @@ -89,7 +188,7 @@ macro_rules! syscall { syscall( concat_idents!(SYS_, $name), - $($arg_name as c_long),* + $($arg_name),* ) as $ret } ) diff --git a/crux-mir/lib/std/src/sys/unsupported/alloc.rs b/crux-mir/lib/std/src/sys/unsupported/alloc.rs new file mode 100644 index 000000000..d715ae454 --- /dev/null +++ b/crux-mir/lib/std/src/sys/unsupported/alloc.rs @@ -0,0 +1,23 @@ +use crate::alloc::{GlobalAlloc, Layout, System}; +use crate::ptr::null_mut; + +#[stable(feature = "alloc_system_type", since = "1.28.0")] +unsafe impl GlobalAlloc for System { + #[inline] + unsafe fn alloc(&self, _layout: Layout) -> *mut u8 { + null_mut() + } + + #[inline] + unsafe fn alloc_zeroed(&self, _layout: Layout) -> *mut u8 { + null_mut() + } + + #[inline] + unsafe fn dealloc(&self, _ptr: *mut u8, _layout: Layout) {} + + #[inline] + unsafe fn realloc(&self, _ptr: *mut u8, _layout: Layout, _new_size: usize) -> *mut u8 { + null_mut() + } +} diff --git a/crux-mir/lib/std/src/sys/cloudabi/shims/args.rs b/crux-mir/lib/std/src/sys/unsupported/args.rs similarity index 72% rename from crux-mir/lib/std/src/sys/cloudabi/shims/args.rs rename to crux-mir/lib/std/src/sys/unsupported/args.rs index f5cf71caf..a2d75a619 100644 --- a/crux-mir/lib/std/src/sys/cloudabi/shims/args.rs +++ b/crux-mir/lib/std/src/sys/unsupported/args.rs @@ -1,10 +1,15 @@ use crate::ffi::OsString; +use crate::fmt; -pub struct Args(()); +pub struct Args {} -impl Args { - pub fn inner_debug(&self) -> &[OsString] { - &[] +pub fn args() -> Args { + Args {} +} + +impl fmt::Debug for Args { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().finish() } } @@ -29,7 +34,3 @@ impl DoubleEndedIterator for Args { None } } - -pub fn args() -> Args { - Args(()) -} diff --git a/crux-mir/lib/std/src/sys/unsupported/common.rs b/crux-mir/lib/std/src/sys/unsupported/common.rs new file mode 100644 index 000000000..5cd9e57de --- /dev/null +++ b/crux-mir/lib/std/src/sys/unsupported/common.rs @@ -0,0 +1,36 @@ +use crate::io as std_io; + +pub mod memchr { + pub use core::slice::memchr::{memchr, memrchr}; +} + +// SAFETY: must be called only once during runtime initialization. +// NOTE: this is not guaranteed to run, for example when Rust code is called externally. +pub unsafe fn init(_argc: isize, _argv: *const *const u8, _sigpipe: u8) {} + +// SAFETY: must be called only once during runtime cleanup. +// NOTE: this is not guaranteed to run, for example when the program aborts. +pub unsafe fn cleanup() {} + +pub fn unsupported() -> std_io::Result { + Err(unsupported_err()) +} + +pub fn unsupported_err() -> std_io::Error { + std_io::const_io_error!( + std_io::ErrorKind::Unsupported, + "operation not supported on this platform", + ) +} + +pub fn decode_error_kind(_code: i32) -> crate::io::ErrorKind { + crate::io::ErrorKind::Uncategorized +} + +pub fn abort_internal() -> ! { + core::intrinsics::abort(); +} + +pub fn hashmap_random_keys() -> (u64, u64) { + (1, 2) +} diff --git a/crux-mir/lib/std/src/sys/unsupported/env.rs b/crux-mir/lib/std/src/sys/unsupported/env.rs new file mode 100644 index 000000000..d2efec506 --- /dev/null +++ b/crux-mir/lib/std/src/sys/unsupported/env.rs @@ -0,0 +1,9 @@ +pub mod os { + pub const FAMILY: &str = ""; + pub const OS: &str = ""; + pub const DLL_PREFIX: &str = ""; + pub const DLL_SUFFIX: &str = ""; + pub const DLL_EXTENSION: &str = ""; + pub const EXE_SUFFIX: &str = ""; + pub const EXE_EXTENSION: &str = ""; +} diff --git a/crux-mir/lib/std/src/sys/wasm/fs.rs b/crux-mir/lib/std/src/sys/unsupported/fs.rs similarity index 75% rename from crux-mir/lib/std/src/sys/wasm/fs.rs rename to crux-mir/lib/std/src/sys/unsupported/fs.rs index e6160d145..6ac1b5d2b 100644 --- a/crux-mir/lib/std/src/sys/wasm/fs.rs +++ b/crux-mir/lib/std/src/sys/unsupported/fs.rs @@ -1,80 +1,83 @@ use crate::ffi::OsString; use crate::fmt; use crate::hash::{Hash, Hasher}; -use crate::io::{self, IoSlice, IoSliceMut, SeekFrom}; +use crate::io::{self, BorrowedCursor, IoSlice, IoSliceMut, SeekFrom}; use crate::path::{Path, PathBuf}; use crate::sys::time::SystemTime; -use crate::sys::{unsupported, Void}; +use crate::sys::unsupported; -pub struct File(Void); +pub struct File(!); -pub struct FileAttr(Void); +pub struct FileAttr(!); -pub struct ReadDir(Void); +pub struct ReadDir(!); -pub struct DirEntry(Void); +pub struct DirEntry(!); #[derive(Clone, Debug)] pub struct OpenOptions {} -pub struct FilePermissions(Void); +#[derive(Copy, Clone, Debug, Default)] +pub struct FileTimes {} -pub struct FileType(Void); +pub struct FilePermissions(!); + +pub struct FileType(!); #[derive(Debug)] pub struct DirBuilder {} impl FileAttr { pub fn size(&self) -> u64 { - match self.0 {} + self.0 } pub fn perm(&self) -> FilePermissions { - match self.0 {} + self.0 } pub fn file_type(&self) -> FileType { - match self.0 {} + self.0 } pub fn modified(&self) -> io::Result { - match self.0 {} + self.0 } pub fn accessed(&self) -> io::Result { - match self.0 {} + self.0 } pub fn created(&self) -> io::Result { - match self.0 {} + self.0 } } impl Clone for FileAttr { fn clone(&self) -> FileAttr { - match self.0 {} + self.0 } } impl FilePermissions { pub fn readonly(&self) -> bool { - match self.0 {} + self.0 } pub fn set_readonly(&mut self, _readonly: bool) { - match self.0 {} + self.0 } } impl Clone for FilePermissions { fn clone(&self) -> FilePermissions { - match self.0 {} + self.0 } } impl PartialEq for FilePermissions { fn eq(&self, _other: &FilePermissions) -> bool { - match self.0 {} + self.0 } } @@ -82,27 +85,32 @@ impl Eq for FilePermissions {} impl fmt::Debug for FilePermissions { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} + self.0 } } +impl FileTimes { + pub fn set_accessed(&mut self, _t: SystemTime) {} + pub fn set_modified(&mut self, _t: SystemTime) {} +} + impl FileType { pub fn is_dir(&self) -> bool { - match self.0 {} + self.0 } pub fn is_file(&self) -> bool { - match self.0 {} + self.0 } pub fn is_symlink(&self) -> bool { - match self.0 {} + self.0 } } impl Clone for FileType { fn clone(&self) -> FileType { - match self.0 {} + self.0 } } @@ -110,7 +118,7 @@ impl Copy for FileType {} impl PartialEq for FileType { fn eq(&self, _other: &FileType) -> bool { - match self.0 {} + self.0 } } @@ -118,19 +126,19 @@ impl Eq for FileType {} impl Hash for FileType { fn hash(&self, _h: &mut H) { - match self.0 {} + self.0 } } impl fmt::Debug for FileType { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} + self.0 } } impl fmt::Debug for ReadDir { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} + self.0 } } @@ -138,25 +146,25 @@ impl Iterator for ReadDir { type Item = io::Result; fn next(&mut self) -> Option> { - match self.0 {} + self.0 } } impl DirEntry { pub fn path(&self) -> PathBuf { - match self.0 {} + self.0 } pub fn file_name(&self) -> OsString { - match self.0 {} + self.0 } pub fn metadata(&self) -> io::Result { - match self.0 {} + self.0 } pub fn file_type(&self) -> io::Result { - match self.0 {} + self.0 } } @@ -179,55 +187,67 @@ impl File { } pub fn file_attr(&self) -> io::Result { - match self.0 {} + self.0 } pub fn fsync(&self) -> io::Result<()> { - match self.0 {} + self.0 } pub fn datasync(&self) -> io::Result<()> { - match self.0 {} + self.0 } pub fn truncate(&self, _size: u64) -> io::Result<()> { - match self.0 {} + self.0 } pub fn read(&self, _buf: &mut [u8]) -> io::Result { - match self.0 {} + self.0 } pub fn read_vectored(&self, _bufs: &mut [IoSliceMut<'_>]) -> io::Result { - match self.0 {} + self.0 + } + + pub fn is_read_vectored(&self) -> bool { + self.0 + } + + pub fn read_buf(&self, _cursor: BorrowedCursor<'_>) -> io::Result<()> { + self.0 } pub fn write(&self, _buf: &[u8]) -> io::Result { - match self.0 {} + self.0 } pub fn write_vectored(&self, _bufs: &[IoSlice<'_>]) -> io::Result { - match self.0 {} + self.0 + } + + pub fn is_write_vectored(&self) -> bool { + self.0 } pub fn flush(&self) -> io::Result<()> { - match self.0 {} + self.0 } pub fn seek(&self, _pos: SeekFrom) -> io::Result { - match self.0 {} + self.0 } pub fn duplicate(&self) -> io::Result { - match self.0 {} + self.0 } pub fn set_permissions(&self, _perm: FilePermissions) -> io::Result<()> { - match self.0 {} + self.0 } - pub fn diverge(&self) -> ! { - match self.0 {} + pub fn set_times(&self, _times: FileTimes) -> io::Result<()> { + self.0 } } @@ -243,7 +263,7 @@ impl DirBuilder { impl fmt::Debug for File { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} + self.0 } } @@ -271,11 +291,15 @@ pub fn remove_dir_all(_path: &Path) -> io::Result<()> { unsupported() } +pub fn try_exists(_path: &Path) -> io::Result { + unsupported() +} + pub fn readlink(_p: &Path) -> io::Result { unsupported() } -pub fn symlink(_src: &Path, _dst: &Path) -> io::Result<()> { +pub fn symlink(_original: &Path, _link: &Path) -> io::Result<()> { unsupported() } diff --git a/crux-mir/lib/std/src/sys/hermit/io.rs b/crux-mir/lib/std/src/sys/unsupported/io.rs similarity index 94% rename from crux-mir/lib/std/src/sys/hermit/io.rs rename to crux-mir/lib/std/src/sys/unsupported/io.rs index d5f475b43..82610ffab 100644 --- a/crux-mir/lib/std/src/sys/hermit/io.rs +++ b/crux-mir/lib/std/src/sys/unsupported/io.rs @@ -45,3 +45,7 @@ impl<'a> IoSliceMut<'a> { self.0 } } + +pub fn is_terminal(_: &T) -> bool { + false +} diff --git a/crux-mir/lib/std/src/sys/wasm/condvar.rs b/crux-mir/lib/std/src/sys/unsupported/locks/condvar.rs similarity index 51% rename from crux-mir/lib/std/src/sys/wasm/condvar.rs rename to crux-mir/lib/std/src/sys/unsupported/locks/condvar.rs index 9fd781c72..3f0943b50 100644 --- a/crux-mir/lib/std/src/sys/wasm/condvar.rs +++ b/crux-mir/lib/std/src/sys/unsupported/locks/condvar.rs @@ -1,30 +1,26 @@ -use crate::sys::mutex::Mutex; +use crate::sys::locks::Mutex; use crate::time::Duration; pub struct Condvar {} impl Condvar { + #[inline] + #[rustc_const_stable(feature = "const_locks", since = "1.63.0")] pub const fn new() -> Condvar { Condvar {} } #[inline] - pub unsafe fn init(&mut self) {} - - #[inline] - pub unsafe fn notify_one(&self) {} + pub fn notify_one(&self) {} #[inline] - pub unsafe fn notify_all(&self) {} + pub fn notify_all(&self) {} pub unsafe fn wait(&self, _mutex: &Mutex) { - panic!("can't block with web assembly") + panic!("condvar wait not supported") } pub unsafe fn wait_timeout(&self, _mutex: &Mutex, _dur: Duration) -> bool { - panic!("can't block with web assembly"); + panic!("condvar wait not supported"); } - - #[inline] - pub unsafe fn destroy(&self) {} } diff --git a/crux-mir/lib/std/src/sys/unsupported/locks/mod.rs b/crux-mir/lib/std/src/sys/unsupported/locks/mod.rs new file mode 100644 index 000000000..0e0f9eccb --- /dev/null +++ b/crux-mir/lib/std/src/sys/unsupported/locks/mod.rs @@ -0,0 +1,6 @@ +mod condvar; +mod mutex; +mod rwlock; +pub use condvar::Condvar; +pub use mutex::Mutex; +pub use rwlock::RwLock; diff --git a/crux-mir/lib/std/src/sys/unsupported/locks/mutex.rs b/crux-mir/lib/std/src/sys/unsupported/locks/mutex.rs new file mode 100644 index 000000000..4a13c55fb --- /dev/null +++ b/crux-mir/lib/std/src/sys/unsupported/locks/mutex.rs @@ -0,0 +1,32 @@ +use crate::cell::Cell; + +pub struct Mutex { + // This platform has no threads, so we can use a Cell here. + locked: Cell, +} + +unsafe impl Send for Mutex {} +unsafe impl Sync for Mutex {} // no threads on this platform + +impl Mutex { + #[inline] + #[rustc_const_stable(feature = "const_locks", since = "1.63.0")] + pub const fn new() -> Mutex { + Mutex { locked: Cell::new(false) } + } + + #[inline] + pub fn lock(&self) { + assert_eq!(self.locked.replace(true), false, "cannot recursively acquire mutex"); + } + + #[inline] + pub unsafe fn unlock(&self) { + self.locked.set(false); + } + + #[inline] + pub fn try_lock(&self) -> bool { + self.locked.replace(true) == false + } +} diff --git a/crux-mir/lib/std/src/sys/unsupported/locks/rwlock.rs b/crux-mir/lib/std/src/sys/unsupported/locks/rwlock.rs new file mode 100644 index 000000000..789ef9b29 --- /dev/null +++ b/crux-mir/lib/std/src/sys/unsupported/locks/rwlock.rs @@ -0,0 +1,65 @@ +use crate::cell::Cell; + +pub struct RwLock { + // This platform has no threads, so we can use a Cell here. + mode: Cell, +} + +unsafe impl Send for RwLock {} +unsafe impl Sync for RwLock {} // no threads on this platform + +impl RwLock { + #[inline] + #[rustc_const_stable(feature = "const_locks", since = "1.63.0")] + pub const fn new() -> RwLock { + RwLock { mode: Cell::new(0) } + } + + #[inline] + pub fn read(&self) { + let m = self.mode.get(); + if m >= 0 { + self.mode.set(m + 1); + } else { + rtabort!("rwlock locked for writing"); + } + } + + #[inline] + pub fn try_read(&self) -> bool { + let m = self.mode.get(); + if m >= 0 { + self.mode.set(m + 1); + true + } else { + false + } + } + + #[inline] + pub fn write(&self) { + if self.mode.replace(-1) != 0 { + rtabort!("rwlock locked for reading") + } + } + + #[inline] + pub fn try_write(&self) -> bool { + if self.mode.get() == 0 { + self.mode.set(-1); + true + } else { + false + } + } + + #[inline] + pub unsafe fn read_unlock(&self) { + self.mode.set(self.mode.get() - 1); + } + + #[inline] + pub unsafe fn write_unlock(&self) { + assert_eq!(self.mode.replace(0), -1); + } +} diff --git a/crux-mir/lib/std/src/sys/unsupported/mod.rs b/crux-mir/lib/std/src/sys/unsupported/mod.rs new file mode 100644 index 000000000..15b22c620 --- /dev/null +++ b/crux-mir/lib/std/src/sys/unsupported/mod.rs @@ -0,0 +1,28 @@ +#![deny(unsafe_op_in_unsafe_fn)] + +pub mod alloc; +pub mod args; +#[path = "../unix/cmath.rs"] +pub mod cmath; +pub mod env; +pub mod fs; +pub mod io; +pub mod locks; +pub mod net; +pub mod once; +pub mod os; +#[path = "../unix/os_str.rs"] +pub mod os_str; +#[path = "../unix/path.rs"] +pub mod path; +pub mod pipe; +pub mod process; +pub mod stdio; +pub mod thread; +#[cfg(target_thread_local)] +pub mod thread_local_dtor; +pub mod thread_local_key; +pub mod time; + +mod common; +pub use common::*; diff --git a/crux-mir/lib/std/src/sys/wasm/net.rs b/crux-mir/lib/std/src/sys/unsupported/net.rs similarity index 78% rename from crux-mir/lib/std/src/sys/wasm/net.rs rename to crux-mir/lib/std/src/sys/unsupported/net.rs index b7c3108f1..a5204a084 100644 --- a/crux-mir/lib/std/src/sys/wasm/net.rs +++ b/crux-mir/lib/std/src/sys/unsupported/net.rs @@ -1,11 +1,10 @@ -use crate::convert::TryFrom; use crate::fmt; use crate::io::{self, IoSlice, IoSliceMut}; use crate::net::{Ipv4Addr, Ipv6Addr, Shutdown, SocketAddr}; -use crate::sys::{unsupported, Void}; +use crate::sys::unsupported; use crate::time::Duration; -pub struct TcpStream(Void); +pub struct TcpStream(!); impl TcpStream { pub fn connect(_: io::Result<&SocketAddr>) -> io::Result { @@ -17,89 +16,105 @@ impl TcpStream { } pub fn set_read_timeout(&self, _: Option) -> io::Result<()> { - match self.0 {} + self.0 } pub fn set_write_timeout(&self, _: Option) -> io::Result<()> { - match self.0 {} + self.0 } pub fn read_timeout(&self) -> io::Result> { - match self.0 {} + self.0 } pub fn write_timeout(&self) -> io::Result> { - match self.0 {} + self.0 } pub fn peek(&self, _: &mut [u8]) -> io::Result { - match self.0 {} + self.0 } pub fn read(&self, _: &mut [u8]) -> io::Result { - match self.0 {} + self.0 } pub fn read_vectored(&self, _: &mut [IoSliceMut<'_>]) -> io::Result { - match self.0 {} + self.0 + } + + pub fn is_read_vectored(&self) -> bool { + self.0 } pub fn write(&self, _: &[u8]) -> io::Result { - match self.0 {} + self.0 } pub fn write_vectored(&self, _: &[IoSlice<'_>]) -> io::Result { - match self.0 {} + self.0 + } + + pub fn is_write_vectored(&self) -> bool { + self.0 } pub fn peer_addr(&self) -> io::Result { - match self.0 {} + self.0 } pub fn socket_addr(&self) -> io::Result { - match self.0 {} + self.0 } pub fn shutdown(&self, _: Shutdown) -> io::Result<()> { - match self.0 {} + self.0 } pub fn duplicate(&self) -> io::Result { - match self.0 {} + self.0 + } + + pub fn set_linger(&self, _: Option) -> io::Result<()> { + self.0 + } + + pub fn linger(&self) -> io::Result> { + self.0 } pub fn set_nodelay(&self, _: bool) -> io::Result<()> { - match self.0 {} + self.0 } pub fn nodelay(&self) -> io::Result { - match self.0 {} + self.0 } pub fn set_ttl(&self, _: u32) -> io::Result<()> { - match self.0 {} + self.0 } pub fn ttl(&self) -> io::Result { - match self.0 {} + self.0 } pub fn take_error(&self) -> io::Result> { - match self.0 {} + self.0 } pub fn set_nonblocking(&self, _: bool) -> io::Result<()> { - match self.0 {} + self.0 } } impl fmt::Debug for TcpStream { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} + self.0 } } -pub struct TcpListener(Void); +pub struct TcpListener(!); impl TcpListener { pub fn bind(_: io::Result<&SocketAddr>) -> io::Result { @@ -107,49 +122,49 @@ impl TcpListener { } pub fn socket_addr(&self) -> io::Result { - match self.0 {} + self.0 } pub fn accept(&self) -> io::Result<(TcpStream, SocketAddr)> { - match self.0 {} + self.0 } pub fn duplicate(&self) -> io::Result { - match self.0 {} + self.0 } pub fn set_ttl(&self, _: u32) -> io::Result<()> { - match self.0 {} + self.0 } pub fn ttl(&self) -> io::Result { - match self.0 {} + self.0 } pub fn set_only_v6(&self, _: bool) -> io::Result<()> { - match self.0 {} + self.0 } pub fn only_v6(&self) -> io::Result { - match self.0 {} + self.0 } pub fn take_error(&self) -> io::Result> { - match self.0 {} + self.0 } pub fn set_nonblocking(&self, _: bool) -> io::Result<()> { - match self.0 {} + self.0 } } impl fmt::Debug for TcpListener { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} + self.0 } } -pub struct UdpSocket(Void); +pub struct UdpSocket(!); impl UdpSocket { pub fn bind(_: io::Result<&SocketAddr>) -> io::Result { @@ -157,144 +172,144 @@ impl UdpSocket { } pub fn peer_addr(&self) -> io::Result { - match self.0 {} + self.0 } pub fn socket_addr(&self) -> io::Result { - match self.0 {} + self.0 } pub fn recv_from(&self, _: &mut [u8]) -> io::Result<(usize, SocketAddr)> { - match self.0 {} + self.0 } pub fn peek_from(&self, _: &mut [u8]) -> io::Result<(usize, SocketAddr)> { - match self.0 {} + self.0 } pub fn send_to(&self, _: &[u8], _: &SocketAddr) -> io::Result { - match self.0 {} + self.0 } pub fn duplicate(&self) -> io::Result { - match self.0 {} + self.0 } pub fn set_read_timeout(&self, _: Option) -> io::Result<()> { - match self.0 {} + self.0 } pub fn set_write_timeout(&self, _: Option) -> io::Result<()> { - match self.0 {} + self.0 } pub fn read_timeout(&self) -> io::Result> { - match self.0 {} + self.0 } pub fn write_timeout(&self) -> io::Result> { - match self.0 {} + self.0 } pub fn set_broadcast(&self, _: bool) -> io::Result<()> { - match self.0 {} + self.0 } pub fn broadcast(&self) -> io::Result { - match self.0 {} + self.0 } pub fn set_multicast_loop_v4(&self, _: bool) -> io::Result<()> { - match self.0 {} + self.0 } pub fn multicast_loop_v4(&self) -> io::Result { - match self.0 {} + self.0 } pub fn set_multicast_ttl_v4(&self, _: u32) -> io::Result<()> { - match self.0 {} + self.0 } pub fn multicast_ttl_v4(&self) -> io::Result { - match self.0 {} + self.0 } pub fn set_multicast_loop_v6(&self, _: bool) -> io::Result<()> { - match self.0 {} + self.0 } pub fn multicast_loop_v6(&self) -> io::Result { - match self.0 {} + self.0 } pub fn join_multicast_v4(&self, _: &Ipv4Addr, _: &Ipv4Addr) -> io::Result<()> { - match self.0 {} + self.0 } pub fn join_multicast_v6(&self, _: &Ipv6Addr, _: u32) -> io::Result<()> { - match self.0 {} + self.0 } pub fn leave_multicast_v4(&self, _: &Ipv4Addr, _: &Ipv4Addr) -> io::Result<()> { - match self.0 {} + self.0 } pub fn leave_multicast_v6(&self, _: &Ipv6Addr, _: u32) -> io::Result<()> { - match self.0 {} + self.0 } pub fn set_ttl(&self, _: u32) -> io::Result<()> { - match self.0 {} + self.0 } pub fn ttl(&self) -> io::Result { - match self.0 {} + self.0 } pub fn take_error(&self) -> io::Result> { - match self.0 {} + self.0 } pub fn set_nonblocking(&self, _: bool) -> io::Result<()> { - match self.0 {} + self.0 } pub fn recv(&self, _: &mut [u8]) -> io::Result { - match self.0 {} + self.0 } pub fn peek(&self, _: &mut [u8]) -> io::Result { - match self.0 {} + self.0 } pub fn send(&self, _: &[u8]) -> io::Result { - match self.0 {} + self.0 } pub fn connect(&self, _: io::Result<&SocketAddr>) -> io::Result<()> { - match self.0 {} + self.0 } } impl fmt::Debug for UdpSocket { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} + self.0 } } -pub struct LookupHost(Void); +pub struct LookupHost(!); impl LookupHost { pub fn port(&self) -> u16 { - match self.0 {} + self.0 } } impl Iterator for LookupHost { type Item = SocketAddr; fn next(&mut self) -> Option { - match self.0 {} + self.0 } } @@ -348,6 +363,4 @@ pub mod netc { #[derive(Copy, Clone)] pub struct sockaddr {} - - pub type socklen_t = usize; } diff --git a/crux-mir/lib/std/src/sys/unsupported/once.rs b/crux-mir/lib/std/src/sys/unsupported/once.rs new file mode 100644 index 000000000..b4bb4975f --- /dev/null +++ b/crux-mir/lib/std/src/sys/unsupported/once.rs @@ -0,0 +1,89 @@ +use crate::cell::Cell; +use crate::sync as public; + +pub struct Once { + state: Cell, +} + +pub struct OnceState { + poisoned: bool, + set_state_to: Cell, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum State { + Incomplete, + Poisoned, + Running, + Complete, +} + +struct CompletionGuard<'a> { + state: &'a Cell, + set_state_on_drop_to: State, +} + +impl<'a> Drop for CompletionGuard<'a> { + fn drop(&mut self) { + self.state.set(self.set_state_on_drop_to); + } +} + +// Safety: threads are not supported on this platform. +unsafe impl Sync for Once {} + +impl Once { + #[inline] + #[rustc_const_stable(feature = "const_once_new", since = "1.32.0")] + pub const fn new() -> Once { + Once { state: Cell::new(State::Incomplete) } + } + + #[inline] + pub fn is_completed(&self) -> bool { + self.state.get() == State::Complete + } + + #[cold] + #[track_caller] + pub fn call(&self, ignore_poisoning: bool, f: &mut impl FnMut(&public::OnceState)) { + let state = self.state.get(); + match state { + State::Poisoned if !ignore_poisoning => { + // Panic to propagate the poison. + panic!("Once instance has previously been poisoned"); + } + State::Incomplete | State::Poisoned => { + self.state.set(State::Running); + // `guard` will set the new state on drop. + let mut guard = + CompletionGuard { state: &self.state, set_state_on_drop_to: State::Poisoned }; + // Run the function, letting it know if we're poisoned or not. + let f_state = public::OnceState { + inner: OnceState { + poisoned: state == State::Poisoned, + set_state_to: Cell::new(State::Complete), + }, + }; + f(&f_state); + guard.set_state_on_drop_to = f_state.inner.set_state_to.get(); + } + State::Running => { + panic!("one-time initialization may not be performed recursively"); + } + State::Complete => {} + } + } +} + +impl OnceState { + #[inline] + pub fn is_poisoned(&self) -> bool { + self.poisoned + } + + #[inline] + pub fn poison(&self) { + self.set_state_to.set(State::Poisoned) + } +} diff --git a/crux-mir/lib/std/src/sys/wasm/os.rs b/crux-mir/lib/std/src/sys/unsupported/os.rs similarity index 68% rename from crux-mir/lib/std/src/sys/wasm/os.rs rename to crux-mir/lib/std/src/sys/unsupported/os.rs index 91afdc8a5..e150ae143 100644 --- a/crux-mir/lib/std/src/sys/wasm/os.rs +++ b/crux-mir/lib/std/src/sys/unsupported/os.rs @@ -1,10 +1,10 @@ +use super::unsupported; use crate::error::Error as StdError; use crate::ffi::{OsStr, OsString}; use crate::fmt; use crate::io; +use crate::marker::PhantomData; use crate::path::{self, PathBuf}; -use crate::str; -use crate::sys::{unsupported, Void}; pub fn errno() -> i32 { 0 @@ -22,7 +22,7 @@ pub fn chdir(_: &path::Path) -> io::Result<()> { unsupported() } -pub struct SplitPaths<'a>(&'a Void); +pub struct SplitPaths<'a>(!, PhantomData<&'a ()>); pub fn split_paths(_unparsed: &OsStr) -> SplitPaths<'_> { panic!("unsupported") @@ -31,7 +31,7 @@ pub fn split_paths(_unparsed: &OsStr) -> SplitPaths<'_> { impl<'a> Iterator for SplitPaths<'a> { type Item = PathBuf; fn next(&mut self) -> Option { - match *self.0 {} + self.0 } } @@ -48,14 +48,14 @@ where impl fmt::Display for JoinPathsError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - "not supported on wasm yet".fmt(f) + "not supported on this platform yet".fmt(f) } } impl StdError for JoinPathsError { #[allow(deprecated)] fn description(&self) -> &str { - "not supported on wasm yet" + "not supported on this platform yet" } } @@ -63,33 +63,33 @@ pub fn current_exe() -> io::Result { unsupported() } -pub struct Env(Void); +pub struct Env(!); impl Iterator for Env { type Item = (OsString, OsString); fn next(&mut self) -> Option<(OsString, OsString)> { - match self.0 {} + self.0 } } pub fn env() -> Env { - panic!("not supported on web assembly") + panic!("not supported on this platform") } -pub fn getenv(_: &OsStr) -> io::Result> { - Ok(None) +pub fn getenv(_: &OsStr) -> Option { + None } pub fn setenv(_: &OsStr, _: &OsStr) -> io::Result<()> { - Err(io::Error::new(io::ErrorKind::Other, "cannot set env vars on wasm32-unknown-unknown")) + Err(io::const_io_error!(io::ErrorKind::Unsupported, "cannot set env vars on this platform")) } pub fn unsetenv(_: &OsStr) -> io::Result<()> { - Err(io::Error::new(io::ErrorKind::Other, "cannot unset env vars on wasm32-unknown-unknown")) + Err(io::const_io_error!(io::ErrorKind::Unsupported, "cannot unset env vars on this platform")) } pub fn temp_dir() -> PathBuf { - panic!("no filesystem on wasm") + panic!("no filesystem on this platform") } pub fn home_dir() -> Option { @@ -97,11 +97,9 @@ pub fn home_dir() -> Option { } pub fn exit(_code: i32) -> ! { - unsafe { - crate::arch::wasm32::unreachable(); - } + crate::intrinsics::abort() } pub fn getpid() -> u32 { - panic!("no pids on wasm") + panic!("no pids on this platform") } diff --git a/crux-mir/lib/std/src/sys/sgx/pipe.rs b/crux-mir/lib/std/src/sys/unsupported/pipe.rs similarity index 62% rename from crux-mir/lib/std/src/sys/sgx/pipe.rs rename to crux-mir/lib/std/src/sys/unsupported/pipe.rs index fb14dc591..0bba673b4 100644 --- a/crux-mir/lib/std/src/sys/sgx/pipe.rs +++ b/crux-mir/lib/std/src/sys/unsupported/pipe.rs @@ -1,27 +1,38 @@ use crate::io::{self, IoSlice, IoSliceMut}; -use crate::sys::Void; -pub struct AnonPipe(Void); +pub struct AnonPipe(!); impl AnonPipe { pub fn read(&self, _buf: &mut [u8]) -> io::Result { - match self.0 {} + self.0 } pub fn read_vectored(&self, _bufs: &mut [IoSliceMut<'_>]) -> io::Result { - match self.0 {} + self.0 + } + + pub fn is_read_vectored(&self) -> bool { + self.0 + } + + pub fn read_to_end(&self, _buf: &mut Vec) -> io::Result { + self.0 } pub fn write(&self, _buf: &[u8]) -> io::Result { - match self.0 {} + self.0 } pub fn write_vectored(&self, _bufs: &[IoSlice<'_>]) -> io::Result { - match self.0 {} + self.0 + } + + pub fn is_write_vectored(&self) -> bool { + self.0 } pub fn diverge(&self) -> ! { - match self.0 {} + self.0 } } diff --git a/crux-mir/lib/std/src/sys/wasi/process.rs b/crux-mir/lib/std/src/sys/unsupported/process.rs similarity index 59% rename from crux-mir/lib/std/src/sys/wasi/process.rs rename to crux-mir/lib/std/src/sys/unsupported/process.rs index 7156c9ab9..a494f2d6b 100644 --- a/crux-mir/lib/std/src/sys/wasi/process.rs +++ b/crux-mir/lib/std/src/sys/unsupported/process.rs @@ -1,10 +1,13 @@ use crate::ffi::OsStr; use crate::fmt; use crate::io; +use crate::marker::PhantomData; +use crate::num::NonZeroI32; +use crate::path::Path; use crate::sys::fs::File; use crate::sys::pipe::AnonPipe; -use crate::sys::{unsupported, Void}; -use crate::sys_common::process::CommandEnv; +use crate::sys::unsupported; +use crate::sys_common::process::{CommandEnv, CommandEnvs}; pub use crate::ffi::OsString as EnvKey; @@ -49,6 +52,22 @@ impl Command { pub fn stderr(&mut self, _stderr: Stdio) {} + pub fn get_program(&self) -> &OsStr { + panic!("unsupported") + } + + pub fn get_args(&self) -> CommandArgs<'_> { + CommandArgs { _p: PhantomData } + } + + pub fn get_envs(&self) -> CommandEnvs<'_> { + self.env.iter() + } + + pub fn get_current_dir(&self) -> Option<&Path> { + None + } + pub fn spawn( &mut self, _default: Stdio, @@ -56,6 +75,10 @@ impl Command { ) -> io::Result<(Process, StdioPipes)> { unsupported() } + + pub fn output(&mut self) -> io::Result<(ExitStatus, Vec, Vec)> { + unsupported() + } } impl From for Stdio { @@ -76,21 +99,21 @@ impl fmt::Debug for Command { } } -pub struct ExitStatus(Void); +pub struct ExitStatus(!); impl ExitStatus { - pub fn success(&self) -> bool { - match self.0 {} + pub fn exit_ok(&self) -> Result<(), ExitStatusError> { + self.0 } pub fn code(&self) -> Option { - match self.0 {} + self.0 } } impl Clone for ExitStatus { fn clone(&self) -> ExitStatus { - match self.0 {} + self.0 } } @@ -98,7 +121,7 @@ impl Copy for ExitStatus {} impl PartialEq for ExitStatus { fn eq(&self, _other: &ExitStatus) -> bool { - match self.0 {} + self.0 } } @@ -106,13 +129,28 @@ impl Eq for ExitStatus {} impl fmt::Debug for ExitStatus { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} + self.0 } } impl fmt::Display for ExitStatus { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} + self.0 + } +} + +#[derive(PartialEq, Eq, Clone, Copy, Debug)] +pub struct ExitStatusError(ExitStatus); + +impl Into for ExitStatusError { + fn into(self) -> ExitStatus { + self.0.0 + } +} + +impl ExitStatusError { + pub fn code(self) -> Option { + self.0.0 } } @@ -128,22 +166,53 @@ impl ExitCode { } } -pub struct Process(Void); +impl From for ExitCode { + fn from(code: u8) -> Self { + match code { + 0 => Self::SUCCESS, + 1..=255 => Self::FAILURE, + } + } +} + +pub struct Process(!); impl Process { pub fn id(&self) -> u32 { - match self.0 {} + self.0 } pub fn kill(&mut self) -> io::Result<()> { - match self.0 {} + self.0 } pub fn wait(&mut self) -> io::Result { - match self.0 {} + self.0 } pub fn try_wait(&mut self) -> io::Result> { - match self.0 {} + self.0 + } +} + +pub struct CommandArgs<'a> { + _p: PhantomData<&'a ()>, +} + +impl<'a> Iterator for CommandArgs<'a> { + type Item = &'a OsStr; + fn next(&mut self) -> Option<&'a OsStr> { + None + } + fn size_hint(&self) -> (usize, Option) { + (0, Some(0)) + } +} + +impl<'a> ExactSizeIterator for CommandArgs<'a> {} + +impl<'a> fmt::Debug for CommandArgs<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().finish() } } diff --git a/crux-mir/lib/std/src/sys/wasm/stdio.rs b/crux-mir/lib/std/src/sys/unsupported/stdio.rs similarity index 81% rename from crux-mir/lib/std/src/sys/wasm/stdio.rs rename to crux-mir/lib/std/src/sys/unsupported/stdio.rs index 5a4e4505e..b5e3f5be9 100644 --- a/crux-mir/lib/std/src/sys/wasm/stdio.rs +++ b/crux-mir/lib/std/src/sys/unsupported/stdio.rs @@ -5,8 +5,8 @@ pub struct Stdout; pub struct Stderr; impl Stdin { - pub fn new() -> io::Result { - Ok(Stdin) + pub const fn new() -> Stdin { + Stdin } } @@ -17,8 +17,8 @@ impl io::Read for Stdin { } impl Stdout { - pub fn new() -> io::Result { - Ok(Stdout) + pub const fn new() -> Stdout { + Stdout } } @@ -33,8 +33,8 @@ impl io::Write for Stdout { } impl Stderr { - pub fn new() -> io::Result { - Ok(Stderr) + pub const fn new() -> Stderr { + Stderr } } diff --git a/crux-mir/lib/std/src/sys/unsupported/thread.rs b/crux-mir/lib/std/src/sys/unsupported/thread.rs new file mode 100644 index 000000000..a8db251de --- /dev/null +++ b/crux-mir/lib/std/src/sys/unsupported/thread.rs @@ -0,0 +1,46 @@ +use super::unsupported; +use crate::ffi::CStr; +use crate::io; +use crate::num::NonZeroUsize; +use crate::time::Duration; + +pub struct Thread(!); + +pub const DEFAULT_MIN_STACK_SIZE: usize = 4096; + +impl Thread { + // unsafe: see thread::Builder::spawn_unchecked for safety requirements + pub unsafe fn new(_stack: usize, _p: Box) -> io::Result { + unsupported() + } + + pub fn yield_now() { + // do nothing + } + + pub fn set_name(_name: &CStr) { + // nope + } + + pub fn sleep(_dur: Duration) { + panic!("can't sleep"); + } + + pub fn join(self) { + self.0 + } +} + +pub fn available_parallelism() -> io::Result { + unsupported() +} + +pub mod guard { + pub type Guard = !; + pub unsafe fn current() -> Option { + None + } + pub unsafe fn init() -> Option { + None + } +} diff --git a/crux-mir/lib/std/src/sys/wasm/fast_thread_local.rs b/crux-mir/lib/std/src/sys/unsupported/thread_local_dtor.rs similarity index 86% rename from crux-mir/lib/std/src/sys/wasm/fast_thread_local.rs rename to crux-mir/lib/std/src/sys/unsupported/thread_local_dtor.rs index 85d660983..84660ea58 100644 --- a/crux-mir/lib/std/src/sys/wasm/fast_thread_local.rs +++ b/crux-mir/lib/std/src/sys/unsupported/thread_local_dtor.rs @@ -1,5 +1,6 @@ #![unstable(feature = "thread_local_internals", issue = "none")] +#[cfg_attr(target_family = "wasm", allow(unused))] // unused on wasm32-unknown-unknown pub unsafe fn register_dtor(_t: *mut u8, _dtor: unsafe extern "C" fn(*mut u8)) { // FIXME: right now there is no concept of "thread exit", but this is likely // going to show up at some point in the form of an exported symbol that the diff --git a/crux-mir/lib/std/src/sys/unsupported/thread_local_key.rs b/crux-mir/lib/std/src/sys/unsupported/thread_local_key.rs new file mode 100644 index 000000000..b6e5e4cd2 --- /dev/null +++ b/crux-mir/lib/std/src/sys/unsupported/thread_local_key.rs @@ -0,0 +1,21 @@ +pub type Key = usize; + +#[inline] +pub unsafe fn create(_dtor: Option) -> Key { + panic!("should not be used on this target"); +} + +#[inline] +pub unsafe fn set(_key: Key, _value: *mut u8) { + panic!("should not be used on this target"); +} + +#[inline] +pub unsafe fn get(_key: Key) -> *mut u8 { + panic!("should not be used on this target"); +} + +#[inline] +pub unsafe fn destroy(_key: Key) { + panic!("should not be used on this target"); +} diff --git a/crux-mir/lib/std/src/sys/wasm/time.rs b/crux-mir/lib/std/src/sys/unsupported/time.rs similarity index 81% rename from crux-mir/lib/std/src/sys/wasm/time.rs rename to crux-mir/lib/std/src/sys/unsupported/time.rs index d9edc7fdc..6d67b538a 100644 --- a/crux-mir/lib/std/src/sys/wasm/time.rs +++ b/crux-mir/lib/std/src/sys/unsupported/time.rs @@ -10,15 +10,7 @@ pub const UNIX_EPOCH: SystemTime = SystemTime(Duration::from_secs(0)); impl Instant { pub fn now() -> Instant { - panic!("time not implemented on wasm32-unknown-unknown") - } - - pub const fn zero() -> Instant { - Instant(Duration::from_secs(0)) - } - - pub fn actually_monotonic() -> bool { - false + panic!("time not implemented on this platform") } pub fn checked_sub_instant(&self, other: &Instant) -> Option { @@ -36,7 +28,7 @@ impl Instant { impl SystemTime { pub fn now() -> SystemTime { - panic!("time not implemented on wasm32-unknown-unknown") + panic!("time not implemented on this platform") } pub fn sub_time(&self, other: &SystemTime) -> Result { diff --git a/crux-mir/lib/std/src/sys/vxworks/alloc.rs b/crux-mir/lib/std/src/sys/vxworks/alloc.rs deleted file mode 100644 index 97a191d72..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/alloc.rs +++ /dev/null @@ -1,49 +0,0 @@ -use crate::alloc::{GlobalAlloc, Layout, System}; -use crate::ptr; -use crate::sys_common::alloc::{realloc_fallback, MIN_ALIGN}; - -#[stable(feature = "alloc_system_type", since = "1.28.0")] -unsafe impl GlobalAlloc for System { - #[inline] - unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - if layout.align() <= MIN_ALIGN && layout.align() <= layout.size() { - libc::malloc(layout.size()) as *mut u8 - } else { - aligned_malloc(&layout) - } - } - - #[inline] - unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { - if layout.align() <= MIN_ALIGN && layout.align() <= layout.size() { - libc::calloc(layout.size(), 1) as *mut u8 - } else { - let ptr = self.alloc(layout.clone()); - if !ptr.is_null() { - ptr::write_bytes(ptr, 0, layout.size()); - } - ptr - } - } - - #[inline] - unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { - libc::free(ptr as *mut libc::c_void) - } - - #[inline] - unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - if layout.align() <= MIN_ALIGN && layout.align() <= new_size { - libc::realloc(ptr as *mut libc::c_void, new_size) as *mut u8 - } else { - realloc_fallback(self, ptr, layout, new_size) - } - } -} - -#[inline] -unsafe fn aligned_malloc(layout: &Layout) -> *mut u8 { - let mut out = ptr::null_mut(); - let ret = libc::posix_memalign(&mut out, layout.align(), layout.size()); - if ret != 0 { ptr::null_mut() } else { out as *mut u8 } -} diff --git a/crux-mir/lib/std/src/sys/vxworks/args.rs b/crux-mir/lib/std/src/sys/vxworks/args.rs deleted file mode 100644 index efd615f40..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/args.rs +++ /dev/null @@ -1,96 +0,0 @@ -#![allow(dead_code)] // runtime init functions not used during testing -use crate::ffi::OsString; -use crate::marker::PhantomData; -use crate::vec; - -/// One-time global initialization. -pub unsafe fn init(argc: isize, argv: *const *const u8) { - imp::init(argc, argv) -} - -/// One-time global cleanup. -pub unsafe fn cleanup() { - imp::cleanup() -} - -/// Returns the command line arguments -pub fn args() -> Args { - imp::args() -} - -pub struct Args { - iter: vec::IntoIter, - _dont_send_or_sync_me: PhantomData<*mut ()>, -} - -impl Args { - pub fn inner_debug(&self) -> &[OsString] { - self.iter.as_slice() - } -} - -impl Iterator for Args { - type Item = OsString; - fn next(&mut self) -> Option { - self.iter.next() - } - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() - } -} - -impl ExactSizeIterator for Args { - fn len(&self) -> usize { - self.iter.len() - } -} - -impl DoubleEndedIterator for Args { - fn next_back(&mut self) -> Option { - self.iter.next_back() - } -} - -mod imp { - use super::Args; - use crate::ffi::{CStr, OsString}; - use crate::marker::PhantomData; - use crate::ptr; - use libc; - - use crate::sys_common::mutex::Mutex; - - static mut ARGC: isize = 0; - static mut ARGV: *const *const u8 = ptr::null(); - static LOCK: Mutex = Mutex::new(); - - pub unsafe fn init(argc: isize, argv: *const *const u8) { - let _guard = LOCK.lock(); - ARGC = argc; - ARGV = argv; - } - - pub unsafe fn cleanup() { - let _guard = LOCK.lock(); - ARGC = 0; - ARGV = ptr::null(); - } - - pub fn args() -> Args { - Args { iter: clone().into_iter(), _dont_send_or_sync_me: PhantomData } - } - - fn clone() -> Vec { - unsafe { - let _guard = LOCK.lock(); - let ret = (0..ARGC) - .map(|i| { - let cstr = CStr::from_ptr(*ARGV.offset(i) as *const libc::c_char); - use crate::sys::vxworks::ext::ffi::OsStringExt; - OsStringExt::from_vec(cstr.to_bytes().to_vec()) - }) - .collect(); - return ret; - } - } -} diff --git a/crux-mir/lib/std/src/sys/vxworks/cmath.rs b/crux-mir/lib/std/src/sys/vxworks/cmath.rs deleted file mode 100644 index f327b69fc..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/cmath.rs +++ /dev/null @@ -1,32 +0,0 @@ -#![cfg(not(test))] - -use libc::{c_double, c_float}; - -extern "C" { - pub fn acos(n: c_double) -> c_double; - pub fn acosf(n: c_float) -> c_float; - pub fn asin(n: c_double) -> c_double; - pub fn asinf(n: c_float) -> c_float; - pub fn atan(n: c_double) -> c_double; - pub fn atan2(a: c_double, b: c_double) -> c_double; - pub fn atan2f(a: c_float, b: c_float) -> c_float; - pub fn atanf(n: c_float) -> c_float; - pub fn cbrt(n: c_double) -> c_double; - pub fn cbrtf(n: c_float) -> c_float; - pub fn cosh(n: c_double) -> c_double; - pub fn coshf(n: c_float) -> c_float; - pub fn expm1(n: c_double) -> c_double; - pub fn expm1f(n: c_float) -> c_float; - pub fn fdim(a: c_double, b: c_double) -> c_double; - pub fn fdimf(a: c_float, b: c_float) -> c_float; - pub fn hypot(x: c_double, y: c_double) -> c_double; - pub fn hypotf(x: c_float, y: c_float) -> c_float; - pub fn log1p(n: c_double) -> c_double; - pub fn log1pf(n: c_float) -> c_float; - pub fn sinh(n: c_double) -> c_double; - pub fn sinhf(n: c_float) -> c_float; - pub fn tan(n: c_double) -> c_double; - pub fn tanf(n: c_float) -> c_float; - pub fn tanh(n: c_double) -> c_double; - pub fn tanhf(n: c_float) -> c_float; -} diff --git a/crux-mir/lib/std/src/sys/vxworks/condvar.rs b/crux-mir/lib/std/src/sys/vxworks/condvar.rs deleted file mode 100644 index f2a1d6815..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/condvar.rs +++ /dev/null @@ -1,93 +0,0 @@ -use crate::cell::UnsafeCell; -use crate::sys::mutex::{self, Mutex}; -use crate::time::Duration; - -pub struct Condvar { - inner: UnsafeCell, -} - -unsafe impl Send for Condvar {} -unsafe impl Sync for Condvar {} - -const TIMESPEC_MAX: libc::timespec = - libc::timespec { tv_sec: ::max_value(), tv_nsec: 1_000_000_000 - 1 }; - -fn saturating_cast_to_time_t(value: u64) -> libc::time_t { - if value > ::max_value() as u64 { - ::max_value() - } else { - value as libc::time_t - } -} - -impl Condvar { - pub const fn new() -> Condvar { - // Might be moved and address is changing it is better to avoid - // initialization of potentially opaque OS data before it landed - Condvar { inner: UnsafeCell::new(libc::PTHREAD_COND_INITIALIZER) } - } - - pub unsafe fn init(&mut self) { - use crate::mem::MaybeUninit; - let mut attr = MaybeUninit::::uninit(); - let r = libc::pthread_condattr_init(attr.as_mut_ptr()); - assert_eq!(r, 0); - let r = libc::pthread_condattr_setclock(attr.as_mut_ptr(), libc::CLOCK_MONOTONIC); - assert_eq!(r, 0); - let r = libc::pthread_cond_init(self.inner.get(), attr.as_ptr()); - assert_eq!(r, 0); - let r = libc::pthread_condattr_destroy(attr.as_mut_ptr()); - assert_eq!(r, 0); - } - - #[inline] - pub unsafe fn notify_one(&self) { - let r = libc::pthread_cond_signal(self.inner.get()); - debug_assert_eq!(r, 0); - } - - #[inline] - pub unsafe fn notify_all(&self) { - let r = libc::pthread_cond_broadcast(self.inner.get()); - debug_assert_eq!(r, 0); - } - - #[inline] - pub unsafe fn wait(&self, mutex: &Mutex) { - let r = libc::pthread_cond_wait(self.inner.get(), mutex::raw(mutex)); - debug_assert_eq!(r, 0); - } - - // This implementation is used on systems that support pthread_condattr_setclock - // where we configure condition variable to use monotonic clock (instead of - // default system clock). This approach avoids all problems that result - // from changes made to the system time. - pub unsafe fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool { - use crate::mem; - - let mut now: libc::timespec = mem::zeroed(); - let r = libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut now); - assert_eq!(r, 0); - - // Nanosecond calculations can't overflow because both values are below 1e9. - let nsec = dur.subsec_nanos() + now.tv_nsec as u32; - - let sec = saturating_cast_to_time_t(dur.as_secs()) - .checked_add((nsec / 1_000_000_000) as libc::time_t) - .and_then(|s| s.checked_add(now.tv_sec)); - let nsec = nsec % 1_000_000_000; - - let timeout = - sec.map(|s| libc::timespec { tv_sec: s, tv_nsec: nsec as _ }).unwrap_or(TIMESPEC_MAX); - - let r = libc::pthread_cond_timedwait(self.inner.get(), mutex::raw(mutex), &timeout); - assert!(r == libc::ETIMEDOUT || r == 0); - r == 0 - } - - #[inline] - pub unsafe fn destroy(&self) { - let r = libc::pthread_cond_destroy(self.inner.get()); - debug_assert_eq!(r, 0); - } -} diff --git a/crux-mir/lib/std/src/sys/vxworks/ext/fs.rs b/crux-mir/lib/std/src/sys/vxworks/ext/fs.rs deleted file mode 100644 index 9864a855d..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/ext/fs.rs +++ /dev/null @@ -1,841 +0,0 @@ -#![stable(feature = "rust1", since = "1.0.0")] - -use crate::fs::{self, Permissions}; -use crate::io; -use crate::path::Path; -use crate::sys; -use crate::sys::platform::fs::MetadataExt as UnixMetadataExt; -use crate::sys_common::{AsInner, AsInnerMut, FromInner}; -use libc; - -/// Unix-specific extensions to [`File`]. -/// -/// [`File`]: ../../../../std/fs/struct.File.html -#[stable(feature = "file_offset", since = "1.15.0")] -pub trait FileExt { - /// Reads a number of bytes starting from a given offset. - /// - /// Returns the number of bytes read. - /// - /// The offset is relative to the start of the file and thus independent - /// from the current cursor. - /// - /// The current file cursor is not affected by this function. - /// - /// Note that similar to [`File::read`], it is not an error to return with a - /// short read. - /// - /// [`File::read`]: ../../../../std/fs/struct.File.html#method.read - /// - /// # Examples - /// - /// ```no_run - /// use std::io; - /// use std::fs::File; - /// use std::os::unix::prelude::FileExt; - /// - /// fn main() -> io::Result<()> { - /// let mut buf = [0u8; 8]; - /// let file = File::open("foo.txt")?; - /// - /// // We now read 8 bytes from the offset 10. - /// let num_bytes_read = file.read_at(&mut buf, 10)?; - /// println!("read {} bytes: {:?}", num_bytes_read, buf); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "file_offset", since = "1.15.0")] - fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result; - - /// Reads the exact number of byte required to fill `buf` from the given offset. - /// - /// The offset is relative to the start of the file and thus independent - /// from the current cursor. - /// - /// The current file cursor is not affected by this function. - /// - /// Similar to [`Read::read_exact`] but uses [`read_at`] instead of `read`. - /// - /// [`Read::read_exact`]: ../../../../std/io/trait.Read.html#method.read_exact - /// [`read_at`]: #tymethod.read_at - /// - /// # Errors - /// - /// If this function encounters an error of the kind - /// [`ErrorKind::Interrupted`] then the error is ignored and the operation - /// will continue. - /// - /// If this function encounters an "end of file" before completely filling - /// the buffer, it returns an error of the kind [`ErrorKind::UnexpectedEof`]. - /// The contents of `buf` are unspecified in this case. - /// - /// If any other read error is encountered then this function immediately - /// returns. The contents of `buf` are unspecified in this case. - /// - /// If this function returns an error, it is unspecified how many bytes it - /// has read, but it will never read more than would be necessary to - /// completely fill the buffer. - /// - /// [`ErrorKind::Interrupted`]: ../../../../std/io/enum.ErrorKind.html#variant.Interrupted - /// [`ErrorKind::UnexpectedEof`]: ../../../../std/io/enum.ErrorKind.html#variant.UnexpectedEof - /// - /// # Examples - /// - /// ```no_run - /// #![feature(rw_exact_all_at)] - /// use std::io; - /// use std::fs::File; - /// use std::os::unix::prelude::FileExt; - /// - /// fn main() -> io::Result<()> { - /// let mut buf = [0u8; 8]; - /// let file = File::open("foo.txt")?; - /// - /// // We now read exactly 8 bytes from the offset 10. - /// file.read_exact_at(&mut buf, 10)?; - /// println!("read {} bytes: {:?}", buf.len(), buf); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "rw_exact_all_at", since = "1.33.0")] - fn read_exact_at(&self, mut buf: &mut [u8], mut offset: u64) -> io::Result<()> { - while !buf.is_empty() { - match self.read_at(buf, offset) { - Ok(0) => break, - Ok(n) => { - let tmp = buf; - buf = &mut tmp[n..]; - offset += n as u64; - } - Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} - Err(e) => return Err(e), - } - } - if !buf.is_empty() { - Err(io::Error::new(io::ErrorKind::UnexpectedEof, "failed to fill whole buffer")) - } else { - Ok(()) - } - } - - /// Writes a number of bytes starting from a given offset. - /// - /// Returns the number of bytes written. - /// - /// The offset is relative to the start of the file and thus independent - /// from the current cursor. - /// - /// The current file cursor is not affected by this function. - /// - /// When writing beyond the end of the file, the file is appropriately - /// extended and the intermediate bytes are initialized with the value 0. - /// - /// Note that similar to [`File::write`], it is not an error to return a - /// short write. - /// - /// [`File::write`]: ../../../../std/fs/struct.File.html#write.v - /// - /// # Examples - /// - /// ```no_run - /// use std::fs::File; - /// use std::io; - /// use std::os::unix::prelude::FileExt; - /// - /// fn main() -> io::Result<()> { - /// let file = File::open("foo.txt")?; - /// - /// // We now write at the offset 10. - /// file.write_at(b"sushi", 10)?; - /// Ok(()) - /// } - /// ``` - #[stable(feature = "file_offset", since = "1.15.0")] - fn write_at(&self, buf: &[u8], offset: u64) -> io::Result; - - /// Attempts to write an entire buffer starting from a given offset. - /// - /// The offset is relative to the start of the file and thus independent - /// from the current cursor. - /// - /// The current file cursor is not affected by this function. - /// - /// This method will continuously call [`write_at`] until there is no more data - /// to be written or an error of non-[`ErrorKind::Interrupted`] kind is - /// returned. This method will not return until the entire buffer has been - /// successfully written or such an error occurs. The first error that is - /// not of [`ErrorKind::Interrupted`] kind generated from this method will be - /// returned. - /// - /// # Errors - /// - /// This function will return the first error of - /// non-[`ErrorKind::Interrupted`] kind that [`write_at`] returns. - /// - /// [`ErrorKind::Interrupted`]: ../../../../std/io/enum.ErrorKind.html#variant.Interrupted - /// [`write_at`]: #tymethod.write_at - /// - /// # Examples - /// - /// ```no_run - /// #![feature(rw_exact_all_at)] - /// use std::fs::File; - /// use std::io; - /// use std::os::unix::prelude::FileExt; - /// - /// fn main() -> io::Result<()> { - /// let file = File::open("foo.txt")?; - /// - /// // We now write at the offset 10. - /// file.write_all_at(b"sushi", 10)?; - /// Ok(()) - /// } - /// ``` - #[stable(feature = "rw_exact_all_at", since = "1.33.0")] - fn write_all_at(&self, mut buf: &[u8], mut offset: u64) -> io::Result<()> { - while !buf.is_empty() { - match self.write_at(buf, offset) { - Ok(0) => { - return Err(io::Error::new( - io::ErrorKind::WriteZero, - "failed to write whole buffer", - )); - } - Ok(n) => { - buf = &buf[n..]; - offset += n as u64 - } - Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} - Err(e) => return Err(e), - } - } - Ok(()) - } -} - -#[stable(feature = "file_offset", since = "1.15.0")] -impl FileExt for fs::File { - fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result { - self.as_inner().read_at(buf, offset) - } - fn write_at(&self, buf: &[u8], offset: u64) -> io::Result { - self.as_inner().write_at(buf, offset) - } -} - -/// Unix-specific extensions to [`fs::Permissions`]. -/// -/// [`fs::Permissions`]: ../../../../std/fs/struct.Permissions.html -#[stable(feature = "fs_ext", since = "1.1.0")] -pub trait PermissionsExt { - /// Returns the underlying raw `st_mode` bits that contain the standard - /// Unix permissions for this file. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs::File; - /// use std::os::unix::fs::PermissionsExt; - /// - /// fn main() -> std::io::Result<()> { - /// let f = File::create("foo.txt")?; - /// let metadata = f.metadata()?; - /// let permissions = metadata.permissions(); - /// - /// println!("permissions: {}", permissions.mode()); - /// Ok(()) } - /// ``` - #[stable(feature = "fs_ext", since = "1.1.0")] - fn mode(&self) -> u32; - - /// Sets the underlying raw bits for this set of permissions. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs::File; - /// use std::os::unix::fs::PermissionsExt; - /// - /// fn main() -> std::io::Result<()> { - /// let f = File::create("foo.txt")?; - /// let metadata = f.metadata()?; - /// let mut permissions = metadata.permissions(); - /// - /// permissions.set_mode(0o644); // Read/write for owner and read for others. - /// assert_eq!(permissions.mode(), 0o644); - /// Ok(()) } - /// ``` - #[stable(feature = "fs_ext", since = "1.1.0")] - fn set_mode(&mut self, mode: u32); - - /// Creates a new instance of `Permissions` from the given set of Unix - /// permission bits. - /// - /// # Examples - /// - /// ``` - /// use std::fs::Permissions; - /// use std::os::unix::fs::PermissionsExt; - /// - /// // Read/write for owner and read for others. - /// let permissions = Permissions::from_mode(0o644); - /// assert_eq!(permissions.mode(), 0o644); - /// ``` - #[stable(feature = "fs_ext", since = "1.1.0")] - fn from_mode(mode: u32) -> Self; -} - -#[stable(feature = "fs_ext", since = "1.1.0")] -impl PermissionsExt for Permissions { - fn mode(&self) -> u32 { - self.as_inner().mode() - } - - fn set_mode(&mut self, mode: u32) { - *self = Permissions::from_inner(FromInner::from_inner(mode)); - } - - fn from_mode(mode: u32) -> Permissions { - Permissions::from_inner(FromInner::from_inner(mode)) - } -} - -/// Unix-specific extensions to [`fs::OpenOptions`]. -/// -/// [`fs::OpenOptions`]: ../../../../std/fs/struct.OpenOptions.html -#[stable(feature = "fs_ext", since = "1.1.0")] -pub trait OpenOptionsExt { - /// Sets the mode bits that a new file will be created with. - /// - /// If a new file is created as part of an `OpenOptions::open` call then this - /// specified `mode` will be used as the permission bits for the new file. - /// If no `mode` is set, the default of `0o666` will be used. - /// The operating system masks out bits with the system's `umask`, to produce - /// the final permissions. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs::OpenOptions; - /// use std::os::unix::fs::OpenOptionsExt; - /// - /// # fn main() { - /// let mut options = OpenOptions::new(); - /// options.mode(0o644); // Give read/write for owner and read for others. - /// let file = options.open("foo.txt"); - /// # } - /// ``` - #[stable(feature = "fs_ext", since = "1.1.0")] - fn mode(&mut self, mode: u32) -> &mut Self; - - /// Pass custom flags to the `flags` argument of `open`. - /// - /// The bits that define the access mode are masked out with `O_ACCMODE`, to - /// ensure they do not interfere with the access mode set by Rusts options. - /// - /// Custom flags can only set flags, not remove flags set by Rusts options. - /// This options overwrites any previously set custom flags. - /// - /// # Examples - /// - /// ```no_run - /// # #![feature(libc)] - /// extern crate libc; - /// use std::fs::OpenOptions; - /// use std::os::unix::fs::OpenOptionsExt; - /// - /// # fn main() { - /// let mut options = OpenOptions::new(); - /// options.write(true); - /// if cfg!(unix) { - /// options.custom_flags(libc::O_NOFOLLOW); - /// } - /// let file = options.open("foo.txt"); - /// # } - /// ``` - #[stable(feature = "open_options_ext", since = "1.10.0")] - fn custom_flags(&mut self, flags: i32) -> &mut Self; -} - -/*#[stable(feature = "fs_ext", since = "1.1.0")] -impl OpenOptionsExt for OpenOptions { - fn mode(&mut self, mode: u32) -> &mut OpenOptions { - self.as_inner_mut().mode(mode); self - } - - fn custom_flags(&mut self, flags: i32) -> &mut OpenOptions { - self.as_inner_mut().custom_flags(flags); self - } -} -*/ - -/// Unix-specific extensions to [`fs::Metadata`]. -/// -/// [`fs::Metadata`]: ../../../../std/fs/struct.Metadata.html -#[stable(feature = "metadata_ext", since = "1.1.0")] -pub trait MetadataExt { - /// Returns the ID of the device containing the file. - /// - /// # Examples - /// - /// ```no_run - /// use std::io; - /// use std::fs; - /// use std::os::unix::fs::MetadataExt; - /// - /// fn main() -> io::Result<()> { - /// let meta = fs::metadata("some_file")?; - /// let dev_id = meta.dev(); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "metadata_ext", since = "1.1.0")] - fn dev(&self) -> u64; - /// Returns the inode number. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs; - /// use std::os::unix::fs::MetadataExt; - /// use std::io; - /// - /// fn main() -> io::Result<()> { - /// let meta = fs::metadata("some_file")?; - /// let inode = meta.ino(); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "metadata_ext", since = "1.1.0")] - fn ino(&self) -> u64; - /// Returns the rights applied to this file. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs; - /// use std::os::unix::fs::MetadataExt; - /// use std::io; - /// - /// fn main() -> io::Result<()> { - /// let meta = fs::metadata("some_file")?; - /// let mode = meta.mode(); - /// let user_has_write_access = mode & 0o200; - /// let user_has_read_write_access = mode & 0o600; - /// let group_has_read_access = mode & 0o040; - /// let others_have_exec_access = mode & 0o001; - /// Ok(()) - /// } - /// ``` - #[stable(feature = "metadata_ext", since = "1.1.0")] - fn mode(&self) -> u32; - /// Returns the number of hard links pointing to this file. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs; - /// use std::os::unix::fs::MetadataExt; - /// use std::io; - /// - /// fn main() -> io::Result<()> { - /// let meta = fs::metadata("some_file")?; - /// let nb_hard_links = meta.nlink(); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "metadata_ext", since = "1.1.0")] - fn nlink(&self) -> u64; - /// Returns the user ID of the owner of this file. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs; - /// use std::os::unix::fs::MetadataExt; - /// use std::io; - /// - /// fn main() -> io::Result<()> { - /// let meta = fs::metadata("some_file")?; - /// let user_id = meta.uid(); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "metadata_ext", since = "1.1.0")] - fn uid(&self) -> u32; - /// Returns the group ID of the owner of this file. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs; - /// use std::os::unix::fs::MetadataExt; - /// use std::io; - /// - /// fn main() -> io::Result<()> { - /// let meta = fs::metadata("some_file")?; - /// let group_id = meta.gid(); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "metadata_ext", since = "1.1.0")] - fn gid(&self) -> u32; - /// Returns the device ID of this file (if it is a special one). - /// - /// # Examples - /// - /// ```no_run - /// use std::fs; - /// use std::os::unix::fs::MetadataExt; - /// use std::io; - /// - /// fn main() -> io::Result<()> { - /// let meta = fs::metadata("some_file")?; - /// let device_id = meta.rdev(); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "metadata_ext", since = "1.1.0")] - fn rdev(&self) -> u64; - /// Returns the total size of this file in bytes. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs; - /// use std::os::unix::fs::MetadataExt; - /// use std::io; - /// - /// fn main() -> io::Result<()> { - /// let meta = fs::metadata("some_file")?; - /// let file_size = meta.size(); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "metadata_ext", since = "1.1.0")] - fn size(&self) -> u64; - /// Returns the time of the last access to the file. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs; - /// use std::os::unix::fs::MetadataExt; - /// use std::io; - /// - /// fn main() -> io::Result<()> { - /// let meta = fs::metadata("some_file")?; - /// let last_access_time = meta.atime(); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "metadata_ext", since = "1.1.0")] - fn atime(&self) -> i64; - /// Returns the time of the last access to the file in nanoseconds. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs; - /// use std::os::unix::fs::MetadataExt; - /// use std::io; - /// - /// fn main() -> io::Result<()> { - /// let meta = fs::metadata("some_file")?; - /// let nano_last_access_time = meta.atime_nsec(); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "metadata_ext", since = "1.1.0")] - fn mtime(&self) -> i64; - /// Returns the time of the last modification of the file in nanoseconds. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs; - /// use std::os::unix::fs::MetadataExt; - /// use std::io; - /// - /// fn main() -> io::Result<()> { - /// let meta = fs::metadata("some_file")?; - /// let nano_last_modification_time = meta.mtime_nsec(); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "metadata_ext", since = "1.1.0")] - fn ctime(&self) -> i64; - /// Returns the time of the last status change of the file in nanoseconds. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs; - /// use std::os::unix::fs::MetadataExt; - /// use std::io; - /// - /// fn main() -> io::Result<()> { - /// let meta = fs::metadata("some_file")?; - /// let nano_last_status_change_time = meta.ctime_nsec(); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "metadata_ext", since = "1.1.0")] - fn blksize(&self) -> u64; - /// Returns the number of blocks allocated to the file, in 512-byte units. - /// - /// Please note that this may be smaller than `st_size / 512` when the file has holes. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs; - /// use std::os::unix::fs::MetadataExt; - /// use std::io; - /// - /// fn main() -> io::Result<()> { - /// let meta = fs::metadata("some_file")?; - /// let blocks = meta.blocks(); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "metadata_ext", since = "1.1.0")] - fn blocks(&self) -> u64; - #[stable(feature = "metadata_ext", since = "1.1.0")] - fn attrib(&self) -> u8; -} - -#[stable(feature = "metadata_ext", since = "1.1.0")] -impl MetadataExt for fs::Metadata { - fn dev(&self) -> u64 { - self.st_dev() - } - fn ino(&self) -> u64 { - self.st_ino() - } - fn mode(&self) -> u32 { - self.st_mode() - } - fn nlink(&self) -> u64 { - self.st_nlink() - } - fn uid(&self) -> u32 { - self.st_uid() - } - fn gid(&self) -> u32 { - self.st_gid() - } - fn rdev(&self) -> u64 { - self.st_rdev() - } - fn size(&self) -> u64 { - self.st_size() - } - fn atime(&self) -> i64 { - self.st_atime() - } - fn mtime(&self) -> i64 { - self.st_mtime() - } - fn ctime(&self) -> i64 { - self.st_ctime() - } - fn blksize(&self) -> u64 { - self.st_blksize() - } - fn blocks(&self) -> u64 { - self.st_blocks() - } - fn attrib(&self) -> u8 { - self.st_attrib() - } -} - -/// Unix-specific extensions for [`FileType`]. -/// -/// Adds support for special Unix file types such as block/character devices, -/// pipes, and sockets. -/// -/// [`FileType`]: ../../../../std/fs/struct.FileType.html -#[stable(feature = "file_type_ext", since = "1.5.0")] -pub trait FileTypeExt { - /// Returns whether this file type is a block device. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs; - /// use std::os::unix::fs::FileTypeExt; - /// use std::io; - /// - /// fn main() -> io::Result<()> { - /// let meta = fs::metadata("block_device_file")?; - /// let file_type = meta.file_type(); - /// assert!(file_type.is_block_device()); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "file_type_ext", since = "1.5.0")] - fn is_block_device(&self) -> bool; - /// Returns whether this file type is a char device. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs; - /// use std::os::unix::fs::FileTypeExt; - /// use std::io; - /// - /// fn main() -> io::Result<()> { - /// let meta = fs::metadata("char_device_file")?; - /// let file_type = meta.file_type(); - /// assert!(file_type.is_char_device()); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "file_type_ext", since = "1.5.0")] - fn is_char_device(&self) -> bool; - /// Returns whether this file type is a fifo. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs; - /// use std::os::unix::fs::FileTypeExt; - /// use std::io; - /// - /// fn main() -> io::Result<()> { - /// let meta = fs::metadata("fifo_file")?; - /// let file_type = meta.file_type(); - /// assert!(file_type.is_fifo()); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "file_type_ext", since = "1.5.0")] - fn is_fifo(&self) -> bool; - /// Returns whether this file type is a socket. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs; - /// use std::os::unix::fs::FileTypeExt; - /// use std::io; - /// - /// fn main() -> io::Result<()> { - /// let meta = fs::metadata("unix.socket")?; - /// let file_type = meta.file_type(); - /// assert!(file_type.is_socket()); - /// Ok(()) - /// } - /// ``` - #[stable(feature = "file_type_ext", since = "1.5.0")] - fn is_socket(&self) -> bool; -} - -#[stable(feature = "file_type_ext", since = "1.5.0")] -impl FileTypeExt for fs::FileType { - fn is_block_device(&self) -> bool { - self.as_inner().is(libc::S_IFBLK) - } - fn is_char_device(&self) -> bool { - self.as_inner().is(libc::S_IFCHR) - } - fn is_fifo(&self) -> bool { - self.as_inner().is(libc::S_IFIFO) - } - fn is_socket(&self) -> bool { - self.as_inner().is(libc::S_IFSOCK) - } -} - -/// Unix-specific extension methods for [`fs::DirEntry`]. -/// -/// [`fs::DirEntry`]: ../../../../std/fs/struct.DirEntry.html -#[stable(feature = "dir_entry_ext", since = "1.1.0")] -pub trait DirEntryExt { - /// Returns the underlying `d_ino` field in the contained `dirent` - /// structure. - /// - /// # Examples - /// - /// ``` - /// use std::fs; - /// use std::os::unix::fs::DirEntryExt; - /// - /// if let Ok(entries) = fs::read_dir(".") { - /// for entry in entries { - /// if let Ok(entry) = entry { - /// // Here, `entry` is a `DirEntry`. - /// println!("{:?}: {}", entry.file_name(), entry.ino()); - /// } - /// } - /// } - /// ``` - #[stable(feature = "dir_entry_ext", since = "1.1.0")] - fn ino(&self) -> u64; -} - -#[stable(feature = "dir_entry_ext", since = "1.1.0")] -impl DirEntryExt for fs::DirEntry { - fn ino(&self) -> u64 { - self.as_inner().ino() - } -} - -/// Creates a new symbolic link on the filesystem. -/// -/// The `dst` path will be a symbolic link pointing to the `src` path. -/// -/// # Note -/// -/// On Windows, you must specify whether a symbolic link points to a file -/// or directory. Use `os::windows::fs::symlink_file` to create a -/// symbolic link to a file, or `os::windows::fs::symlink_dir` to create a -/// symbolic link to a directory. Additionally, the process must have -/// `SeCreateSymbolicLinkPrivilege` in order to be able to create a -/// symbolic link. -/// -/// # Examples -/// -/// ```no_run -/// use std::os::unix::fs; -/// -/// fn main() -> std::io::Result<()> { -/// fs::symlink("a.txt", "b.txt")?; -/// Ok(()) -/// } -/// ``` -#[stable(feature = "symlink", since = "1.1.0")] -pub fn symlink, Q: AsRef>(src: P, dst: Q) -> io::Result<()> { - sys::fs::symlink(src.as_ref(), dst.as_ref()) -} - -/// Unix-specific extensions to [`fs::DirBuilder`]. -/// -/// [`fs::DirBuilder`]: ../../../../std/fs/struct.DirBuilder.html -#[stable(feature = "dir_builder", since = "1.6.0")] -pub trait DirBuilderExt { - /// Sets the mode to create new directories with. This option defaults to - /// 0o777. - /// - /// # Examples - /// - /// ```no_run - /// use std::fs::DirBuilder; - /// use std::os::unix::fs::DirBuilderExt; - /// - /// let mut builder = DirBuilder::new(); - /// builder.mode(0o755); - /// ``` - #[stable(feature = "dir_builder", since = "1.6.0")] - fn mode(&mut self, mode: u32) -> &mut Self; -} - -#[stable(feature = "dir_builder", since = "1.6.0")] -impl DirBuilderExt for fs::DirBuilder { - fn mode(&mut self, mode: u32) -> &mut fs::DirBuilder { - self.as_inner_mut().set_mode(mode); - self - } -} diff --git a/crux-mir/lib/std/src/sys/vxworks/ext/io.rs b/crux-mir/lib/std/src/sys/vxworks/ext/io.rs deleted file mode 100644 index 25c6e26d9..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/ext/io.rs +++ /dev/null @@ -1,189 +0,0 @@ -//! Unix-specific extensions to general I/O primitives - -#![stable(feature = "rust1", since = "1.0.0")] - -use crate::fs; -use crate::io; -use crate::net; -use crate::os::raw; -use crate::sys; -use crate::sys_common::{self, AsInner, FromInner, IntoInner}; - -/// Raw file descriptors. -#[stable(feature = "rust1", since = "1.0.0")] -pub type RawFd = raw::c_int; - -/// A trait to extract the raw unix file descriptor from an underlying -/// object. -/// -/// This is only available on unix platforms and must be imported in order -/// to call the method. Windows platforms have a corresponding `AsRawHandle` -/// and `AsRawSocket` set of traits. -#[stable(feature = "rust1", since = "1.0.0")] -pub trait AsRawFd { - /// Extracts the raw file descriptor. - /// - /// This method does **not** pass ownership of the raw file descriptor - /// to the caller. The descriptor is only guaranteed to be valid while - /// the original object has not yet been destroyed. - #[stable(feature = "rust1", since = "1.0.0")] - fn as_raw_fd(&self) -> RawFd; -} - -/// A trait to express the ability to construct an object from a raw file -/// descriptor. -#[stable(feature = "from_raw_os", since = "1.1.0")] -pub trait FromRawFd { - /// Constructs a new instance of `Self` from the given raw file - /// descriptor. - /// - /// This function **consumes ownership** of the specified file - /// descriptor. The returned object will take responsibility for closing - /// it when the object goes out of scope. - /// - /// This function is also unsafe as the primitives currently returned - /// have the contract that they are the sole owner of the file - /// descriptor they are wrapping. Usage of this function could - /// accidentally allow violating this contract which can cause memory - /// unsafety in code that relies on it being true. - #[stable(feature = "from_raw_os", since = "1.1.0")] - unsafe fn from_raw_fd(fd: RawFd) -> Self; -} - -/// A trait to express the ability to consume an object and acquire ownership of -/// its raw file descriptor. -#[stable(feature = "into_raw_os", since = "1.4.0")] -pub trait IntoRawFd { - /// Consumes this object, returning the raw underlying file descriptor. - /// - /// This function **transfers ownership** of the underlying file descriptor - /// to the caller. Callers are then the unique owners of the file descriptor - /// and must close the descriptor once it's no longer needed. - #[stable(feature = "into_raw_os", since = "1.4.0")] - fn into_raw_fd(self) -> RawFd; -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl AsRawFd for fs::File { - fn as_raw_fd(&self) -> RawFd { - self.as_inner().fd().raw() - } -} -#[stable(feature = "from_raw_os", since = "1.1.0")] -impl FromRawFd for fs::File { - unsafe fn from_raw_fd(fd: RawFd) -> fs::File { - fs::File::from_inner(sys::fs::File::from_inner(fd)) - } -} -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawFd for fs::File { - fn into_raw_fd(self) -> RawFd { - self.into_inner().into_fd().into_raw() - } -} - -#[stable(feature = "asraw_stdio", since = "1.21.0")] -impl AsRawFd for io::Stdin { - fn as_raw_fd(&self) -> RawFd { - libc::STDIN_FILENO - } -} - -#[stable(feature = "asraw_stdio", since = "1.21.0")] -impl AsRawFd for io::Stdout { - fn as_raw_fd(&self) -> RawFd { - libc::STDOUT_FILENO - } -} - -#[stable(feature = "asraw_stdio", since = "1.21.0")] -impl AsRawFd for io::Stderr { - fn as_raw_fd(&self) -> RawFd { - libc::STDERR_FILENO - } -} - -#[stable(feature = "asraw_stdio_locks", since = "1.35.0")] -impl<'a> AsRawFd for io::StdinLock<'a> { - fn as_raw_fd(&self) -> RawFd { - libc::STDIN_FILENO - } -} - -#[stable(feature = "asraw_stdio_locks", since = "1.35.0")] -impl<'a> AsRawFd for io::StdoutLock<'a> { - fn as_raw_fd(&self) -> RawFd { - libc::STDOUT_FILENO - } -} - -#[stable(feature = "asraw_stdio_locks", since = "1.35.0")] -impl<'a> AsRawFd for io::StderrLock<'a> { - fn as_raw_fd(&self) -> RawFd { - libc::STDERR_FILENO - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl AsRawFd for net::TcpStream { - fn as_raw_fd(&self) -> RawFd { - *self.as_inner().socket().as_inner() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl AsRawFd for net::TcpListener { - fn as_raw_fd(&self) -> RawFd { - *self.as_inner().socket().as_inner() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl AsRawFd for net::UdpSocket { - fn as_raw_fd(&self) -> RawFd { - *self.as_inner().socket().as_inner() - } -} - -#[stable(feature = "from_raw_os", since = "1.1.0")] -impl FromRawFd for net::TcpStream { - unsafe fn from_raw_fd(fd: RawFd) -> net::TcpStream { - let socket = sys::net::Socket::from_inner(fd); - net::TcpStream::from_inner(sys_common::net::TcpStream::from_inner(socket)) - } -} - -#[stable(feature = "from_raw_os", since = "1.1.0")] -impl FromRawFd for net::TcpListener { - unsafe fn from_raw_fd(fd: RawFd) -> net::TcpListener { - let socket = sys::net::Socket::from_inner(fd); - net::TcpListener::from_inner(sys_common::net::TcpListener::from_inner(socket)) - } -} - -#[stable(feature = "from_raw_os", since = "1.1.0")] -impl FromRawFd for net::UdpSocket { - unsafe fn from_raw_fd(fd: RawFd) -> net::UdpSocket { - let socket = sys::net::Socket::from_inner(fd); - net::UdpSocket::from_inner(sys_common::net::UdpSocket::from_inner(socket)) - } -} - -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawFd for net::TcpStream { - fn into_raw_fd(self) -> RawFd { - self.into_inner().into_socket().into_inner() - } -} -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawFd for net::TcpListener { - fn into_raw_fd(self) -> RawFd { - self.into_inner().into_socket().into_inner() - } -} -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawFd for net::UdpSocket { - fn into_raw_fd(self) -> RawFd { - self.into_inner().into_socket().into_inner() - } -} diff --git a/crux-mir/lib/std/src/sys/vxworks/ext/process.rs b/crux-mir/lib/std/src/sys/vxworks/ext/process.rs deleted file mode 100644 index 31e691dd1..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/ext/process.rs +++ /dev/null @@ -1,234 +0,0 @@ -//! Unix-specific extensions to primitives in the `std::process` module. - -#![stable(feature = "rust1", since = "1.0.0")] - -use crate::ffi::OsStr; -use crate::io; -use crate::process; -use crate::sys; -use crate::sys::vxworks::ext::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd}; -use crate::sys_common::{AsInner, AsInnerMut, FromInner, IntoInner}; - -/// Unix-specific extensions to the [`process::Command`] builder. -/// -/// [`process::Command`]: ../../../../std/process/struct.Command.html -#[stable(feature = "rust1", since = "1.0.0")] -pub trait CommandExt { - /// Sets the child process's user ID. This translates to a - /// `setuid` call in the child process. Failure in the `setuid` - /// call will cause the spawn to fail. - #[stable(feature = "rust1", since = "1.0.0")] - fn uid(&mut self, id: u16) -> &mut process::Command; - - /// Similar to `uid`, but sets the group ID of the child process. This has - /// the same semantics as the `uid` field. - #[stable(feature = "rust1", since = "1.0.0")] - fn gid(&mut self, id: u16) -> &mut process::Command; - - /// Schedules a closure to be run just before the `exec` function is - /// invoked. - /// - /// The closure is allowed to return an I/O error whose OS error code will - /// be communicated back to the parent and returned as an error from when - /// the spawn was requested. - /// - /// Multiple closures can be registered and they will be called in order of - /// their registration. If a closure returns `Err` then no further closures - /// will be called and the spawn operation will immediately return with a - /// failure. - /// - /// # Notes and Safety - /// - /// This closure will be run in the context of the child process after a - /// `fork`. This primarily means that any modifications made to memory on - /// behalf of this closure will **not** be visible to the parent process. - /// This is often a very constrained environment where normal operations - /// like `malloc` or acquiring a mutex are not guaranteed to work (due to - /// other threads perhaps still running when the `fork` was run). - /// - /// This also means that all resources such as file descriptors and - /// memory-mapped regions got duplicated. It is your responsibility to make - /// sure that the closure does not violate library invariants by making - /// invalid use of these duplicates. - /// - /// When this closure is run, aspects such as the stdio file descriptors and - /// working directory have successfully been changed, so output to these - /// locations may not appear where intended. - #[stable(feature = "process_pre_exec", since = "1.34.0")] - unsafe fn pre_exec(&mut self, f: F) -> &mut process::Command - where - F: FnMut() -> io::Result<()> + Send + Sync + 'static; - - /// Schedules a closure to be run just before the `exec` function is - /// invoked. - /// - /// This method is stable and usable, but it should be unsafe. To fix - /// that, it got deprecated in favor of the unsafe [`pre_exec`]. - /// - /// [`pre_exec`]: #tymethod.pre_exec - #[stable(feature = "process_exec", since = "1.15.0")] - #[rustc_deprecated(since = "1.37.0", reason = "should be unsafe, use `pre_exec` instead")] - fn before_exec(&mut self, f: F) -> &mut process::Command - where - F: FnMut() -> io::Result<()> + Send + Sync + 'static, - { - unsafe { self.pre_exec(f) } - } - - /// Performs all the required setup by this `Command`, followed by calling - /// the `execvp` syscall. - /// - /// On success this function will not return, and otherwise it will return - /// an error indicating why the exec (or another part of the setup of the - /// `Command`) failed. - /// - /// `exec` not returning has the same implications as calling - /// [`process::exit`] – no destructors on the current stack or any other - /// thread’s stack will be run. Therefore, it is recommended to only call - /// `exec` at a point where it is fine to not run any destructors. Note, - /// that the `execvp` syscall independently guarantees that all memory is - /// freed and all file descriptors with the `CLOEXEC` option (set by default - /// on all file descriptors opened by the standard library) are closed. - /// - /// This function, unlike `spawn`, will **not** `fork` the process to create - /// a new child. Like spawn, however, the default behavior for the stdio - /// descriptors will be to inherited from the current process. - /// - /// [`process::exit`]: ../../../process/fn.exit.html - /// - /// # Notes - /// - /// The process may be in a "broken state" if this function returns in - /// error. For example the working directory, environment variables, signal - /// handling settings, various user/group information, or aspects of stdio - /// file descriptors may have changed. If a "transactional spawn" is - /// required to gracefully handle errors it is recommended to use the - /// cross-platform `spawn` instead. - #[stable(feature = "process_exec2", since = "1.9.0")] - fn exec(&mut self) -> io::Error; - - /// Set executable argument - /// - /// Set the first process argument, `argv[0]`, to something other than the - /// default executable path. - #[unstable(feature = "process_set_argv0", issue = "66510")] - fn arg0(&mut self, arg: S) -> &mut process::Command - where - S: AsRef; -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl CommandExt for process::Command { - fn uid(&mut self, id: u16) -> &mut process::Command { - self.as_inner_mut().uid(id); - self - } - - fn gid(&mut self, id: u16) -> &mut process::Command { - self.as_inner_mut().gid(id); - self - } - - unsafe fn pre_exec(&mut self, f: F) -> &mut process::Command - where - F: FnMut() -> io::Result<()> + Send + Sync + 'static, - { - self.as_inner_mut().pre_exec(Box::new(f)); - self - } - - fn exec(&mut self) -> io::Error { - self.as_inner_mut().exec(sys::process::Stdio::Inherit) - } - - fn arg0(&mut self, arg: S) -> &mut process::Command - where - S: AsRef, - { - self.as_inner_mut().set_arg_0(arg.as_ref()); - self - } -} - -/// Unix-specific extensions to [`process::ExitStatus`]. -/// -/// [`process::ExitStatus`]: ../../../../std/process/struct.ExitStatus.html -#[stable(feature = "rust1", since = "1.0.0")] -pub trait ExitStatusExt { - /// Creates a new `ExitStatus` from the raw underlying `i32` return value of - /// a process. - #[stable(feature = "exit_status_from", since = "1.12.0")] - fn from_raw(raw: i32) -> Self; - - /// If the process was terminated by a signal, returns that signal. - #[stable(feature = "rust1", since = "1.0.0")] - fn signal(&self) -> Option; -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ExitStatusExt for process::ExitStatus { - fn from_raw(raw: i32) -> Self { - process::ExitStatus::from_inner(From::from(raw)) - } - - fn signal(&self) -> Option { - self.as_inner().signal() - } -} - -#[stable(feature = "process_extensions", since = "1.2.0")] -impl FromRawFd for process::Stdio { - unsafe fn from_raw_fd(fd: RawFd) -> process::Stdio { - let fd = sys::fd::FileDesc::new(fd); - let io = sys::process::Stdio::Fd(fd); - process::Stdio::from_inner(io) - } -} - -#[stable(feature = "process_extensions", since = "1.2.0")] -impl AsRawFd for process::ChildStdin { - fn as_raw_fd(&self) -> RawFd { - self.as_inner().fd().raw() - } -} - -#[stable(feature = "process_extensions", since = "1.2.0")] -impl AsRawFd for process::ChildStdout { - fn as_raw_fd(&self) -> RawFd { - self.as_inner().fd().raw() - } -} - -#[stable(feature = "process_extensions", since = "1.2.0")] -impl AsRawFd for process::ChildStderr { - fn as_raw_fd(&self) -> RawFd { - self.as_inner().fd().raw() - } -} - -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawFd for process::ChildStdin { - fn into_raw_fd(self) -> RawFd { - self.into_inner().into_fd().into_raw() - } -} - -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawFd for process::ChildStdout { - fn into_raw_fd(self) -> RawFd { - self.into_inner().into_fd().into_raw() - } -} - -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawFd for process::ChildStderr { - fn into_raw_fd(self) -> RawFd { - self.into_inner().into_fd().into_raw() - } -} - -/// Returns the OS-assigned process identifier associated with this process's parent. -#[stable(feature = "unix_ppid", since = "1.27.0")] -pub fn parent_id() -> u32 { - crate::sys::os::getppid() -} diff --git a/crux-mir/lib/std/src/sys/vxworks/ext/raw.rs b/crux-mir/lib/std/src/sys/vxworks/ext/raw.rs deleted file mode 100644 index 1f134f4e2..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/ext/raw.rs +++ /dev/null @@ -1,5 +0,0 @@ -#![stable(feature = "raw_ext", since = "1.1.0")] - -#[doc(inline)] -#[stable(feature = "pthread_t", since = "1.8.0")] -pub use crate::sys::platform::raw::pthread_t; diff --git a/crux-mir/lib/std/src/sys/vxworks/fast_thread_local.rs b/crux-mir/lib/std/src/sys/vxworks/fast_thread_local.rs deleted file mode 100644 index 098668cf5..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/fast_thread_local.rs +++ /dev/null @@ -1,11 +0,0 @@ -#![cfg(target_thread_local)] -#![unstable(feature = "thread_local_internals", issue = "none")] - -pub unsafe fn register_dtor(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { - use crate::sys_common::thread_local::register_dtor_fallback; - register_dtor_fallback(t, dtor); -} - -pub fn requires_move_before_drop() -> bool { - false -} diff --git a/crux-mir/lib/std/src/sys/vxworks/fd.rs b/crux-mir/lib/std/src/sys/vxworks/fd.rs deleted file mode 100644 index 65c67dabc..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/fd.rs +++ /dev/null @@ -1,193 +0,0 @@ -#![unstable(reason = "not public", issue = "none", feature = "fd")] - -use crate::cmp; -use crate::io::{self, Initializer, IoSlice, IoSliceMut, Read}; -use crate::mem; -use crate::sys::cvt; -use crate::sys_common::AsInner; - -use libc::{self, c_int, c_void, ssize_t}; - -#[derive(Debug)] -pub struct FileDesc { - fd: c_int, -} - -fn max_len() -> usize { - // The maximum read limit on most posix-like systems is `SSIZE_MAX`, - // with the man page quoting that if the count of bytes to read is - // greater than `SSIZE_MAX` the result is "unspecified". - ::max_value() as usize -} - -impl FileDesc { - pub fn new(fd: c_int) -> FileDesc { - FileDesc { fd: fd } - } - - pub fn raw(&self) -> c_int { - self.fd - } - - /// Extracts the actual filedescriptor without closing it. - pub fn into_raw(self) -> c_int { - let fd = self.fd; - mem::forget(self); - fd - } - - pub fn read(&self, buf: &mut [u8]) -> io::Result { - let ret = cvt(unsafe { - libc::read(self.fd, buf.as_mut_ptr() as *mut c_void, cmp::min(buf.len(), max_len())) - })?; - Ok(ret as usize) - } - - pub fn read_vectored(&self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { - let ret = cvt(unsafe { - libc::readv( - self.fd, - bufs.as_ptr() as *const libc::iovec, - cmp::min(bufs.len(), c_int::max_value() as usize) as c_int, - ) - })?; - Ok(ret as usize) - } - - pub fn read_to_end(&self, buf: &mut Vec) -> io::Result { - let mut me = self; - (&mut me).read_to_end(buf) - } - - pub fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result { - unsafe fn cvt_pread( - fd: c_int, - buf: *mut c_void, - count: usize, - offset: i64, - ) -> io::Result { - use libc::pread; - cvt(pread(fd, buf, count, offset)) - } - - unsafe { - cvt_pread( - self.fd, - buf.as_mut_ptr() as *mut c_void, - cmp::min(buf.len(), max_len()), - offset as i64, - ) - .map(|n| n as usize) - } - } - - pub fn write(&self, buf: &[u8]) -> io::Result { - let ret = cvt(unsafe { - libc::write(self.fd, buf.as_ptr() as *const c_void, cmp::min(buf.len(), max_len())) - })?; - Ok(ret as usize) - } - - pub fn write_vectored(&self, bufs: &[IoSlice<'_>]) -> io::Result { - let ret = cvt(unsafe { - libc::writev( - self.fd, - bufs.as_ptr() as *const libc::iovec, - cmp::min(bufs.len(), c_int::max_value() as usize) as c_int, - ) - })?; - Ok(ret as usize) - } - - pub fn write_at(&self, buf: &[u8], offset: u64) -> io::Result { - unsafe fn cvt_pwrite( - fd: c_int, - buf: *const c_void, - count: usize, - offset: i64, - ) -> io::Result { - use libc::pwrite; - cvt(pwrite(fd, buf, count, offset)) - } - - unsafe { - cvt_pwrite( - self.fd, - buf.as_ptr() as *const c_void, - cmp::min(buf.len(), max_len()), - offset as i64, - ) - .map(|n| n as usize) - } - } - - pub fn get_cloexec(&self) -> io::Result { - unsafe { Ok((cvt(libc::fcntl(self.fd, libc::F_GETFD))? & libc::FD_CLOEXEC) != 0) } - } - - pub fn set_cloexec(&self) -> io::Result<()> { - unsafe { - let previous = cvt(libc::fcntl(self.fd, libc::F_GETFD))?; - let new = previous | libc::FD_CLOEXEC; - if new != previous { - cvt(libc::fcntl(self.fd, libc::F_SETFD, new))?; - } - Ok(()) - } - } - - pub fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()> { - unsafe { - let v = nonblocking as c_int; - cvt(libc::ioctl(self.fd, libc::FIONBIO, &v))?; - Ok(()) - } - } - - // refer to pxPipeDrv library documentation. - // VxWorks uses fcntl to set O_NONBLOCK to the pipes - pub fn set_nonblocking_pipe(&self, nonblocking: bool) -> io::Result<()> { - unsafe { - let mut flags = cvt(libc::fcntl(self.fd, libc::F_GETFL, 0))?; - flags = if nonblocking { flags | libc::O_NONBLOCK } else { flags & !libc::O_NONBLOCK }; - cvt(libc::fcntl(self.fd, libc::F_SETFL, flags))?; - Ok(()) - } - } - - pub fn duplicate(&self) -> io::Result { - let fd = self.raw(); - match cvt(unsafe { libc::fcntl(fd, libc::F_DUPFD_CLOEXEC, 0) }) { - Ok(newfd) => Ok(FileDesc::new(newfd)), - Err(e) => return Err(e), - } - } -} - -impl<'a> Read for &'a FileDesc { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - (**self).read(buf) - } - - #[inline] - unsafe fn initializer(&self) -> Initializer { - Initializer::nop() - } -} - -impl AsInner for FileDesc { - fn as_inner(&self) -> &c_int { - &self.fd - } -} - -impl Drop for FileDesc { - fn drop(&mut self) { - // Note that errors are ignored when closing a file descriptor. The - // reason for this is that if an error occurs we don't actually know if - // the file descriptor was closed or not, and if we retried (for - // something like EINTR), we might close another valid file descriptor - // (opened after we closed ours. - let _ = unsafe { libc::close(self.fd) }; - } -} diff --git a/crux-mir/lib/std/src/sys/vxworks/fs.rs b/crux-mir/lib/std/src/sys/vxworks/fs.rs deleted file mode 100644 index 68f2c1331..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/fs.rs +++ /dev/null @@ -1,615 +0,0 @@ -// copies from linuxx -use crate::ffi::{CStr, CString, OsStr, OsString}; -use crate::fmt; -use crate::io::{self, Error, ErrorKind, IoSlice, IoSliceMut, SeekFrom}; -use crate::mem; -use crate::path::{Path, PathBuf}; -use crate::ptr; -use crate::sync::Arc; -use crate::sys::fd::FileDesc; -use crate::sys::time::SystemTime; -use crate::sys::vxworks::ext::ffi::OsStrExt; -use crate::sys::vxworks::ext::ffi::OsStringExt; -use crate::sys::{cvt, cvt_r}; -use crate::sys_common::{AsInner, FromInner}; -use libc::{self, c_int, mode_t, off_t, stat64}; -use libc::{dirent, ftruncate, lseek, open, readdir_r as readdir64_r}; -pub struct File(FileDesc); - -#[derive(Clone)] -pub struct FileAttr { - stat: stat64, -} - -// all DirEntry's will have a reference to this struct -struct InnerReadDir { - dirp: Dir, - root: PathBuf, -} - -#[derive(Clone)] -pub struct ReadDir { - inner: Arc, - end_of_stream: bool, -} - -struct Dir(*mut libc::DIR); - -unsafe impl Send for Dir {} -unsafe impl Sync for Dir {} - -pub struct DirEntry { - entry: dirent, - dir: ReadDir, -} - -#[derive(Clone, Debug)] -pub struct OpenOptions { - // generic - read: bool, - write: bool, - append: bool, - truncate: bool, - create: bool, - create_new: bool, - // system-specific - custom_flags: i32, - mode: mode_t, -} - -#[derive(Clone, PartialEq, Eq, Debug)] -pub struct FilePermissions { - mode: mode_t, -} - -#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] -pub struct FileType { - mode: mode_t, -} - -#[derive(Debug)] -pub struct DirBuilder { - mode: mode_t, -} - -impl FileAttr { - pub fn size(&self) -> u64 { - self.stat.st_size as u64 - } - pub fn perm(&self) -> FilePermissions { - FilePermissions { mode: (self.stat.st_mode as mode_t) } - } - - pub fn file_type(&self) -> FileType { - FileType { mode: self.stat.st_mode as mode_t } - } - - pub fn modified(&self) -> io::Result { - Ok(SystemTime::from(libc::timespec { - tv_sec: self.stat.st_mtime as libc::time_t, - tv_nsec: 0, // hack 2.0; - })) - } - - pub fn accessed(&self) -> io::Result { - Ok(SystemTime::from(libc::timespec { - tv_sec: self.stat.st_atime as libc::time_t, - tv_nsec: 0, // hack - a proper fix would be better - })) - } - - pub fn created(&self) -> io::Result { - Err(io::Error::new( - io::ErrorKind::Other, - "creation time is not available on this platform currently", - )) - } -} - -impl AsInner for FileAttr { - fn as_inner(&self) -> &stat64 { - &self.stat - } -} - -impl FilePermissions { - pub fn readonly(&self) -> bool { - // check if any class (owner, group, others) has write permission - self.mode & 0o222 == 0 - } - - pub fn set_readonly(&mut self, readonly: bool) { - if readonly { - // remove write permission for all classes; equivalent to `chmod a-w ` - self.mode &= !0o222; - } else { - // add write permission for all classes; equivalent to `chmod a+w ` - self.mode |= 0o222; - } - } - pub fn mode(&self) -> u32 { - self.mode as u32 - } -} - -impl FileType { - pub fn is_dir(&self) -> bool { - self.is(libc::S_IFDIR) - } - pub fn is_file(&self) -> bool { - self.is(libc::S_IFREG) - } - pub fn is_symlink(&self) -> bool { - self.is(libc::S_IFLNK) - } - - pub fn is(&self, mode: mode_t) -> bool { - self.mode & libc::S_IFMT == mode - } -} - -impl FromInner for FilePermissions { - fn from_inner(mode: u32) -> FilePermissions { - FilePermissions { mode: mode as mode_t } - } -} - -impl fmt::Debug for ReadDir { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // This will only be called from std::fs::ReadDir, which will add a "ReadDir()" frame. - // Thus the result will be e g 'ReadDir("/home")' - fmt::Debug::fmt(&*self.inner.root, f) - } -} - -impl Iterator for ReadDir { - type Item = io::Result; - fn next(&mut self) -> Option> { - if self.end_of_stream { - return None; - } - - unsafe { - let mut ret = DirEntry { entry: mem::zeroed(), dir: self.clone() }; - let mut entry_ptr = ptr::null_mut(); - loop { - if readdir64_r(self.inner.dirp.0, &mut ret.entry, &mut entry_ptr) != 0 { - if entry_ptr.is_null() { - // We encountered an error (which will be returned in this iteration), but - // we also reached the end of the directory stream. The `end_of_stream` - // flag is enabled to make sure that we return `None` in the next iteration - // (instead of looping forever) - self.end_of_stream = true; - } - return Some(Err(Error::last_os_error())); - } - if entry_ptr.is_null() { - return None; - } - if ret.name_bytes() != b"." && ret.name_bytes() != b".." { - return Some(Ok(ret)); - } - } - } - } -} - -impl Drop for Dir { - fn drop(&mut self) { - let r = unsafe { libc::closedir(self.0) }; - debug_assert_eq!(r, 0); - } -} - -impl DirEntry { - pub fn path(&self) -> PathBuf { - use crate::sys::vxworks::ext::ffi::OsStrExt; - self.dir.inner.root.join(OsStr::from_bytes(self.name_bytes())) - } - - pub fn file_name(&self) -> OsString { - OsStr::from_bytes(self.name_bytes()).to_os_string() - } - - pub fn metadata(&self) -> io::Result { - lstat(&self.path()) - } - - pub fn file_type(&self) -> io::Result { - lstat(&self.path()).map(|m| m.file_type()) - } - - pub fn ino(&self) -> u64 { - self.entry.d_ino as u64 - } - - fn name_bytes(&self) -> &[u8] { - unsafe { - //&*self.name - CStr::from_ptr(self.entry.d_name.as_ptr()).to_bytes() - } - } -} - -impl OpenOptions { - pub fn new() -> OpenOptions { - OpenOptions { - // generic - read: false, - write: false, - append: false, - truncate: false, - create: false, - create_new: false, - // system-specific - custom_flags: 0, - mode: 0o666, - } - } - - pub fn read(&mut self, read: bool) { - self.read = read; - } - pub fn write(&mut self, write: bool) { - self.write = write; - } - pub fn append(&mut self, append: bool) { - self.append = append; - } - pub fn truncate(&mut self, truncate: bool) { - self.truncate = truncate; - } - pub fn create(&mut self, create: bool) { - self.create = create; - } - pub fn create_new(&mut self, create_new: bool) { - self.create_new = create_new; - } - pub fn mode(&mut self, mode: u32) { - self.mode = mode as mode_t; - } - - fn get_access_mode(&self) -> io::Result { - match (self.read, self.write, self.append) { - (true, false, false) => Ok(libc::O_RDONLY), - (false, true, false) => Ok(libc::O_WRONLY), - (true, true, false) => Ok(libc::O_RDWR), - (false, _, true) => Ok(libc::O_WRONLY | libc::O_APPEND), - (true, _, true) => Ok(libc::O_RDWR | libc::O_APPEND), - (false, false, false) => Err(Error::from_raw_os_error(libc::EINVAL)), - } - } - - fn get_creation_mode(&self) -> io::Result { - match (self.write, self.append) { - (true, false) => {} - (false, false) => { - if self.truncate || self.create || self.create_new { - return Err(Error::from_raw_os_error(libc::EINVAL)); - } - } - (_, true) => { - if self.truncate && !self.create_new { - return Err(Error::from_raw_os_error(libc::EINVAL)); - } - } - } - - Ok(match (self.create, self.truncate, self.create_new) { - (false, false, false) => 0, - (true, false, false) => libc::O_CREAT, - (false, true, false) => libc::O_TRUNC, - (true, true, false) => libc::O_CREAT | libc::O_TRUNC, - (_, _, true) => libc::O_CREAT | libc::O_EXCL, - }) - } -} - -impl File { - pub fn open(path: &Path, opts: &OpenOptions) -> io::Result { - let path = cstr(path)?; - File::open_c(&path, opts) - } - - pub fn open_c(path: &CStr, opts: &OpenOptions) -> io::Result { - let flags = libc::O_CLOEXEC - | opts.get_access_mode()? - | opts.get_creation_mode()? - | (opts.custom_flags as c_int & !libc::O_ACCMODE); - let fd = cvt_r(|| unsafe { open(path.as_ptr(), flags, opts.mode as c_int) })?; - Ok(File(FileDesc::new(fd))) - } - - pub fn file_attr(&self) -> io::Result { - let mut stat: stat64 = unsafe { mem::zeroed() }; - cvt(unsafe { ::libc::fstat(self.0.raw(), &mut stat) })?; - Ok(FileAttr { stat: stat }) - } - - pub fn fsync(&self) -> io::Result<()> { - cvt_r(|| unsafe { libc::fsync(self.0.raw()) })?; - Ok(()) - } - - pub fn datasync(&self) -> io::Result<()> { - cvt_r(|| unsafe { os_datasync(self.0.raw()) })?; - return Ok(()); - unsafe fn os_datasync(fd: c_int) -> c_int { - libc::fsync(fd) - } //not supported - } - - pub fn truncate(&self, size: u64) -> io::Result<()> { - return cvt_r(|| unsafe { ftruncate(self.0.raw(), size as off_t) }).map(drop); - } - - pub fn read(&self, buf: &mut [u8]) -> io::Result { - self.0.read(buf) - } - - pub fn read_vectored(&self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { - self.0.read_vectored(bufs) - } - - pub fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result { - self.0.read_at(buf, offset) - } - - pub fn write(&self, buf: &[u8]) -> io::Result { - self.0.write(buf) - } - - pub fn write_vectored(&self, bufs: &[IoSlice<'_>]) -> io::Result { - self.0.write_vectored(bufs) - } - - pub fn write_at(&self, buf: &[u8], offset: u64) -> io::Result { - self.0.write_at(buf, offset) - } - - pub fn flush(&self) -> io::Result<()> { - Ok(()) - } - - pub fn seek(&self, pos: SeekFrom) -> io::Result { - let (whence, pos) = match pos { - // Casting to `i64` is fine, too large values will end up as - // negative which will cause an error in `"lseek64"`. - SeekFrom::Start(off) => (libc::SEEK_SET, off as i64), - SeekFrom::End(off) => (libc::SEEK_END, off), - SeekFrom::Current(off) => (libc::SEEK_CUR, off), - }; - let n = cvt(unsafe { lseek(self.0.raw(), pos, whence) })?; - Ok(n as u64) - } - - pub fn duplicate(&self) -> io::Result { - self.0.duplicate().map(File) - } - - pub fn fd(&self) -> &FileDesc { - &self.0 - } - - pub fn into_fd(self) -> FileDesc { - self.0 - } - - pub fn set_permissions(&self, perm: FilePermissions) -> io::Result<()> { - cvt_r(|| unsafe { libc::fchmod(self.0.raw(), perm.mode) })?; - Ok(()) - } - - pub fn diverge(&self) -> ! { - panic!() - } -} - -impl DirBuilder { - pub fn new() -> DirBuilder { - DirBuilder { mode: 0o777 } - } - - pub fn mkdir(&self, p: &Path) -> io::Result<()> { - let p = cstr(p)?; - cvt(unsafe { libc::mkdir(p.as_ptr(), self.mode) })?; - Ok(()) - } - - pub fn set_mode(&mut self, mode: u32) { - self.mode = mode as mode_t; - } -} - -fn cstr(path: &Path) -> io::Result { - use crate::sys::vxworks::ext::ffi::OsStrExt; - Ok(CString::new(path.as_os_str().as_bytes())?) -} - -impl FromInner for File { - fn from_inner(fd: c_int) -> File { - File(FileDesc::new(fd)) - } -} - -impl fmt::Debug for File { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fn get_path(fd: c_int) -> Option { - let mut buf = vec![0; libc::PATH_MAX as usize]; - let n = unsafe { libc::ioctl(fd, libc::FIOGETNAME, buf.as_ptr()) }; - if n == -1 { - return None; - } - let l = buf.iter().position(|&c| c == 0).unwrap(); - buf.truncate(l as usize); - Some(PathBuf::from(OsString::from_vec(buf))) - } - fn get_mode(fd: c_int) -> Option<(bool, bool)> { - let mode = unsafe { libc::fcntl(fd, libc::F_GETFL) }; - if mode == -1 { - return None; - } - match mode & libc::O_ACCMODE { - libc::O_RDONLY => Some((true, false)), - libc::O_RDWR => Some((true, true)), - libc::O_WRONLY => Some((false, true)), - _ => None, - } - } - - let fd = self.0.raw(); - let mut b = f.debug_struct("File"); - b.field("fd", &fd); - if let Some(path) = get_path(fd) { - b.field("path", &path); - } - if let Some((read, write)) = get_mode(fd) { - b.field("read", &read).field("write", &write); - } - b.finish() - } -} - -pub fn readdir(p: &Path) -> io::Result { - let root = p.to_path_buf(); - let p = cstr(p)?; - unsafe { - let ptr = libc::opendir(p.as_ptr()); - if ptr.is_null() { - Err(Error::last_os_error()) - } else { - let inner = InnerReadDir { dirp: Dir(ptr), root }; - Ok(ReadDir { inner: Arc::new(inner), end_of_stream: false }) - } - } -} - -pub fn unlink(p: &Path) -> io::Result<()> { - let p = cstr(p)?; - cvt(unsafe { libc::unlink(p.as_ptr()) })?; - Ok(()) -} - -pub fn rename(old: &Path, new: &Path) -> io::Result<()> { - let old = cstr(old)?; - let new = cstr(new)?; - cvt(unsafe { libc::rename(old.as_ptr(), new.as_ptr()) })?; - Ok(()) -} - -pub fn set_perm(p: &Path, perm: FilePermissions) -> io::Result<()> { - let p = cstr(p)?; - cvt_r(|| unsafe { libc::chmod(p.as_ptr(), perm.mode) })?; - Ok(()) -} - -pub fn rmdir(p: &Path) -> io::Result<()> { - let p = cstr(p)?; - cvt(unsafe { libc::rmdir(p.as_ptr()) })?; - Ok(()) -} - -pub fn remove_dir_all(path: &Path) -> io::Result<()> { - let filetype = lstat(path)?.file_type(); - if filetype.is_symlink() { unlink(path) } else { remove_dir_all_recursive(path) } -} - -fn remove_dir_all_recursive(path: &Path) -> io::Result<()> { - for child in readdir(path)? { - let child = child?; - if child.file_type()?.is_dir() { - remove_dir_all_recursive(&child.path())?; - } else { - unlink(&child.path())?; - } - } - rmdir(path) -} - -pub fn readlink(p: &Path) -> io::Result { - let c_path = cstr(p)?; - let p = c_path.as_ptr(); - - let mut buf = Vec::with_capacity(256); - - loop { - let buf_read = - cvt(unsafe { libc::readlink(p, buf.as_mut_ptr() as *mut _, buf.capacity()) })? as usize; - - unsafe { - buf.set_len(buf_read); - } - - if buf_read != buf.capacity() { - buf.shrink_to_fit(); - - return Ok(PathBuf::from(OsString::from_vec(buf))); - } - - // Trigger the internal buffer resizing logic of `Vec` by requiring - // more space than the current capacity. The length is guaranteed to be - // the same as the capacity due to the if statement above. - buf.reserve(1); - } -} - -pub fn symlink(src: &Path, dst: &Path) -> io::Result<()> { - let src = cstr(src)?; - let dst = cstr(dst)?; - cvt(unsafe { libc::symlink(src.as_ptr(), dst.as_ptr()) })?; - Ok(()) -} - -pub fn link(src: &Path, dst: &Path) -> io::Result<()> { - let src = cstr(src)?; - let dst = cstr(dst)?; - cvt(unsafe { libc::link(src.as_ptr(), dst.as_ptr()) })?; - Ok(()) -} - -pub fn stat(p: &Path) -> io::Result { - let p = cstr(p)?; - let mut stat: stat64 = unsafe { mem::zeroed() }; - cvt(unsafe { libc::stat(p.as_ptr(), &mut stat as *mut _ as *mut _) })?; - Ok(FileAttr { stat }) -} - -pub fn lstat(p: &Path) -> io::Result { - let p = cstr(p)?; - let mut stat: stat64 = unsafe { mem::zeroed() }; - cvt(unsafe { ::libc::lstat(p.as_ptr(), &mut stat as *mut _ as *mut _) })?; - Ok(FileAttr { stat }) -} - -pub fn canonicalize(p: &Path) -> io::Result { - use crate::sys::vxworks::ext::ffi::OsStrExt; - let path = CString::new(p.as_os_str().as_bytes())?; - let buf; - unsafe { - let r = libc::realpath(path.as_ptr(), ptr::null_mut()); - if r.is_null() { - return Err(io::Error::last_os_error()); - } - buf = CStr::from_ptr(r).to_bytes().to_vec(); - libc::free(r as *mut _); - } - Ok(PathBuf::from(OsString::from_vec(buf))) -} - -pub fn copy(from: &Path, to: &Path) -> io::Result { - use crate::fs::File; - if !from.is_file() { - return Err(Error::new( - ErrorKind::InvalidInput, - "the source path is not an existing regular file", - )); - } - - let mut reader = File::open(from)?; - let mut writer = File::create(to)?; - let perm = reader.metadata()?.permissions(); - - let ret = io::copy(&mut reader, &mut writer)?; - writer.set_permissions(perm)?; - Ok(ret) -} diff --git a/crux-mir/lib/std/src/sys/vxworks/mod.rs b/crux-mir/lib/std/src/sys/vxworks/mod.rs deleted file mode 100644 index e23191c94..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/mod.rs +++ /dev/null @@ -1,113 +0,0 @@ -#![allow(dead_code)] -#![allow(missing_docs, nonstandard_style)] - -use crate::io::ErrorKind; - -pub use self::rand::hashmap_random_keys; -pub use crate::os::vxworks as platform; -pub use libc::strlen; - -pub mod alloc; -pub mod args; -pub mod cmath; -pub mod condvar; -pub mod env; -pub mod ext; -pub mod fast_thread_local; -pub mod fd; -pub mod fs; -pub mod io; -pub mod memchr; -pub mod mutex; -pub mod net; -pub mod os; -pub mod path; -pub mod pipe; -pub mod process; -pub mod rand; -pub mod rwlock; -pub mod stack_overflow; -pub mod stdio; -pub mod thread; -pub mod thread_local; -pub mod time; - -pub use crate::sys_common::os_str_bytes as os_str; - -#[cfg(not(test))] -pub fn init() { - // ignore SIGPIPE - unsafe { - assert!(signal(libc::SIGPIPE, libc::SIG_IGN) != libc::SIG_ERR); - } -} - -pub use libc::signal; - -pub fn decode_error_kind(errno: i32) -> ErrorKind { - match errno as libc::c_int { - libc::ECONNREFUSED => ErrorKind::ConnectionRefused, - libc::ECONNRESET => ErrorKind::ConnectionReset, - libc::EPERM | libc::EACCES => ErrorKind::PermissionDenied, - libc::EPIPE => ErrorKind::BrokenPipe, - libc::ENOTCONN => ErrorKind::NotConnected, - libc::ECONNABORTED => ErrorKind::ConnectionAborted, - libc::EADDRNOTAVAIL => ErrorKind::AddrNotAvailable, - libc::EADDRINUSE => ErrorKind::AddrInUse, - libc::ENOENT => ErrorKind::NotFound, - libc::EINTR => ErrorKind::Interrupted, - libc::EINVAL => ErrorKind::InvalidInput, - libc::ETIMEDOUT => ErrorKind::TimedOut, - libc::EEXIST => ErrorKind::AlreadyExists, - - // These two constants can have the same value on some systems, - // but different values on others, so we can't use a match - // clause - x if x == libc::EAGAIN || x == libc::EWOULDBLOCK => ErrorKind::WouldBlock, - - _ => ErrorKind::Other, - } -} - -#[doc(hidden)] -pub trait IsMinusOne { - fn is_minus_one(&self) -> bool; -} - -macro_rules! impl_is_minus_one { - ($($t:ident)*) => ($(impl IsMinusOne for $t { - fn is_minus_one(&self) -> bool { - *self == -1 - } - })*) -} - -impl_is_minus_one! { i8 i16 i32 i64 isize } - -pub fn cvt(t: T) -> crate::io::Result { - if t.is_minus_one() { Err(crate::io::Error::last_os_error()) } else { Ok(t) } -} - -pub fn cvt_r(mut f: F) -> crate::io::Result -where - T: IsMinusOne, - F: FnMut() -> T, -{ - loop { - match cvt(f()) { - Err(ref e) if e.kind() == ErrorKind::Interrupted => {} - other => return other, - } - } -} - -// On Unix-like platforms, libc::abort will unregister signal handlers -// including the SIGABRT handler, preventing the abort from being blocked, and -// fclose streams, with the side effect of flushing them so libc buffered -// output will be printed. Additionally the shell will generally print a more -// understandable error message like "Abort trap" rather than "Illegal -// instruction" that intrinsics::abort would cause, as intrinsics::abort is -// implemented as an illegal instruction. -pub unsafe fn abort_internal() -> ! { - libc::abort() -} diff --git a/crux-mir/lib/std/src/sys/vxworks/mutex.rs b/crux-mir/lib/std/src/sys/vxworks/mutex.rs deleted file mode 100644 index 103d87e3d..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/mutex.rs +++ /dev/null @@ -1,131 +0,0 @@ -use crate::cell::UnsafeCell; -use crate::mem::MaybeUninit; - -pub struct Mutex { - inner: UnsafeCell, -} - -#[inline] -pub unsafe fn raw(m: &Mutex) -> *mut libc::pthread_mutex_t { - m.inner.get() -} - -unsafe impl Send for Mutex {} -unsafe impl Sync for Mutex {} - -#[allow(dead_code)] // sys isn't exported yet -impl Mutex { - pub const fn new() -> Mutex { - // Might be moved to a different address, so it is better to avoid - // initialization of potentially opaque OS data before it landed. - // Be very careful using this newly constructed `Mutex`, reentrant - // locking is undefined behavior until `init` is called! - Mutex { inner: UnsafeCell::new(libc::PTHREAD_MUTEX_INITIALIZER) } - } - #[inline] - pub unsafe fn init(&mut self) { - // Issue #33770 - // - // A pthread mutex initialized with PTHREAD_MUTEX_INITIALIZER will have - // a type of PTHREAD_MUTEX_DEFAULT, which has undefined behavior if you - // try to re-lock it from the same thread when you already hold a lock. - // - // In practice, glibc takes advantage of this undefined behavior to - // implement hardware lock elision, which uses hardware transactional - // memory to avoid acquiring the lock. While a transaction is in - // progress, the lock appears to be unlocked. This isn't a problem for - // other threads since the transactional memory will abort if a conflict - // is detected, however no abort is generated if re-locking from the - // same thread. - // - // Since locking the same mutex twice will result in two aliasing &mut - // references, we instead create the mutex with type - // PTHREAD_MUTEX_NORMAL which is guaranteed to deadlock if we try to - // re-lock it from the same thread, thus avoiding undefined behavior. - let mut attr = MaybeUninit::::uninit(); - let r = libc::pthread_mutexattr_init(attr.as_mut_ptr()); - debug_assert_eq!(r, 0); - let r = libc::pthread_mutexattr_settype(attr.as_mut_ptr(), libc::PTHREAD_MUTEX_NORMAL); - debug_assert_eq!(r, 0); - let r = libc::pthread_mutex_init(self.inner.get(), attr.as_ptr()); - debug_assert_eq!(r, 0); - let r = libc::pthread_mutexattr_destroy(attr.as_mut_ptr()); - debug_assert_eq!(r, 0); - } - #[inline] - pub unsafe fn lock(&self) { - let r = libc::pthread_mutex_lock(self.inner.get()); - debug_assert_eq!(r, 0); - } - #[inline] - pub unsafe fn unlock(&self) { - let r = libc::pthread_mutex_unlock(self.inner.get()); - debug_assert_eq!(r, 0); - } - #[inline] - pub unsafe fn try_lock(&self) -> bool { - libc::pthread_mutex_trylock(self.inner.get()) == 0 - } - #[inline] - #[cfg(not(target_os = "dragonfly"))] - pub unsafe fn destroy(&self) { - let r = libc::pthread_mutex_destroy(self.inner.get()); - debug_assert_eq!(r, 0); - } - #[inline] - #[cfg(target_os = "dragonfly")] - pub unsafe fn destroy(&self) { - let r = libc::pthread_mutex_destroy(self.inner.get()); - // On DragonFly pthread_mutex_destroy() returns EINVAL if called on a - // mutex that was just initialized with libc::PTHREAD_MUTEX_INITIALIZER. - // Once it is used (locked/unlocked) or pthread_mutex_init() is called, - // this behaviour no longer occurs. - debug_assert!(r == 0 || r == libc::EINVAL); - } -} - -pub struct ReentrantMutex { - inner: UnsafeCell, -} - -unsafe impl Send for ReentrantMutex {} -unsafe impl Sync for ReentrantMutex {} - -impl ReentrantMutex { - pub const unsafe fn uninitialized() -> ReentrantMutex { - ReentrantMutex { inner: UnsafeCell::new(libc::PTHREAD_MUTEX_INITIALIZER) } - } - - pub unsafe fn init(&self) { - let mut attr = MaybeUninit::::uninit(); - let result = libc::pthread_mutexattr_init(attr.as_mut_ptr()); - debug_assert_eq!(result, 0); - let result = - libc::pthread_mutexattr_settype(attr.as_mut_ptr(), libc::PTHREAD_MUTEX_RECURSIVE); - debug_assert_eq!(result, 0); - let result = libc::pthread_mutex_init(self.inner.get(), attr.as_ptr()); - debug_assert_eq!(result, 0); - let result = libc::pthread_mutexattr_destroy(attr.as_mut_ptr()); - debug_assert_eq!(result, 0); - } - - pub unsafe fn lock(&self) { - let result = libc::pthread_mutex_lock(self.inner.get()); - debug_assert_eq!(result, 0); - } - - #[inline] - pub unsafe fn try_lock(&self) -> bool { - libc::pthread_mutex_trylock(self.inner.get()) == 0 - } - - pub unsafe fn unlock(&self) { - let result = libc::pthread_mutex_unlock(self.inner.get()); - debug_assert_eq!(result, 0); - } - - pub unsafe fn destroy(&self) { - let result = libc::pthread_mutex_destroy(self.inner.get()); - debug_assert_eq!(result, 0); - } -} diff --git a/crux-mir/lib/std/src/sys/vxworks/net.rs b/crux-mir/lib/std/src/sys/vxworks/net.rs deleted file mode 100644 index 7d4e5624f..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/net.rs +++ /dev/null @@ -1,349 +0,0 @@ -use crate::cmp; -use crate::ffi::CStr; -use crate::io; -use crate::io::{IoSlice, IoSliceMut}; -use crate::mem; -use crate::net::{Shutdown, SocketAddr}; -use crate::str; -use crate::sys::fd::FileDesc; -use crate::sys_common::net::{getsockopt, setsockopt, sockaddr_to_addr}; -use crate::sys_common::{AsInner, FromInner, IntoInner}; -use crate::time::{Duration, Instant}; -use libc::{self, c_int, c_void, size_t, sockaddr, socklen_t, EAI_SYSTEM, MSG_PEEK}; - -pub use crate::sys::{cvt, cvt_r}; - -#[allow(unused_extern_crates)] -pub extern crate libc as netc; - -pub type wrlen_t = size_t; - -pub struct Socket(FileDesc); - -pub fn init() {} - -pub fn cvt_gai(err: c_int) -> io::Result<()> { - if err == 0 { - return Ok(()); - } - - // We may need to trigger a glibc workaround. See on_resolver_failure() for details. - on_resolver_failure(); - - if err == EAI_SYSTEM { - return Err(io::Error::last_os_error()); - } - - let detail = unsafe { - str::from_utf8(CStr::from_ptr(libc::gai_strerror(err)).to_bytes()).unwrap().to_owned() - }; - Err(io::Error::new( - io::ErrorKind::Other, - &format!("failed to lookup address information: {}", detail)[..], - )) -} - -impl Socket { - pub fn new(addr: &SocketAddr, ty: c_int) -> io::Result { - let fam = match *addr { - SocketAddr::V4(..) => libc::AF_INET, - SocketAddr::V6(..) => libc::AF_INET6, - }; - Socket::new_raw(fam, ty) - } - - pub fn new_raw(fam: c_int, ty: c_int) -> io::Result { - unsafe { - let fd = cvt(libc::socket(fam, ty, 0))?; - let fd = FileDesc::new(fd); - fd.set_cloexec()?; - let socket = Socket(fd); - Ok(socket) - } - } - - pub fn new_pair(_fam: c_int, _ty: c_int) -> io::Result<(Socket, Socket)> { - unimplemented!(); - } - - pub fn connect_timeout(&self, addr: &SocketAddr, timeout: Duration) -> io::Result<()> { - self.set_nonblocking(true)?; - let r = unsafe { - let (addrp, len) = addr.into_inner(); - cvt(libc::connect(self.0.raw(), addrp, len)) - }; - self.set_nonblocking(false)?; - - match r { - Ok(_) => return Ok(()), - // there's no ErrorKind for EINPROGRESS :( - Err(ref e) if e.raw_os_error() == Some(libc::EINPROGRESS) => {} - Err(e) => return Err(e), - } - - let mut pollfd = libc::pollfd { fd: self.0.raw(), events: libc::POLLOUT, revents: 0 }; - - if timeout.as_secs() == 0 && timeout.subsec_nanos() == 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "cannot set a 0 duration timeout", - )); - } - - let start = Instant::now(); - - loop { - let elapsed = start.elapsed(); - if elapsed >= timeout { - return Err(io::Error::new(io::ErrorKind::TimedOut, "connection timed out")); - } - - let timeout = timeout - elapsed; - let mut timeout = timeout - .as_secs() - .saturating_mul(1_000) - .saturating_add(timeout.subsec_nanos() as u64 / 1_000_000); - if timeout == 0 { - timeout = 1; - } - - let timeout = cmp::min(timeout, c_int::max_value() as u64) as c_int; - - match unsafe { libc::poll(&mut pollfd, 1, timeout) } { - -1 => { - let err = io::Error::last_os_error(); - if err.kind() != io::ErrorKind::Interrupted { - return Err(err); - } - } - 0 => {} - _ => { - // linux returns POLLOUT|POLLERR|POLLHUP for refused connections (!), so look - // for POLLHUP rather than read readiness - if pollfd.revents & libc::POLLHUP != 0 { - let e = self.take_error()?.unwrap_or_else(|| { - io::Error::new(io::ErrorKind::Other, "no error set after POLLHUP") - }); - return Err(e); - } - - return Ok(()); - } - } - } - } - - pub fn accept(&self, storage: *mut sockaddr, len: *mut socklen_t) -> io::Result { - let fd = cvt_r(|| unsafe { libc::accept(self.0.raw(), storage, len) })?; - let fd = FileDesc::new(fd); - fd.set_cloexec()?; - Ok(Socket(fd)) - } - - pub fn duplicate(&self) -> io::Result { - self.0.duplicate().map(Socket) - } - - fn recv_with_flags(&self, buf: &mut [u8], flags: c_int) -> io::Result { - let ret = cvt(unsafe { - libc::recv(self.0.raw(), buf.as_mut_ptr() as *mut c_void, buf.len(), flags) - })?; - Ok(ret as usize) - } - - pub fn read(&self, buf: &mut [u8]) -> io::Result { - self.recv_with_flags(buf, 0) - } - - pub fn peek(&self, buf: &mut [u8]) -> io::Result { - self.recv_with_flags(buf, MSG_PEEK) - } - - pub fn read_vectored(&self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { - self.0.read_vectored(bufs) - } - - fn recv_from_with_flags( - &self, - buf: &mut [u8], - flags: c_int, - ) -> io::Result<(usize, SocketAddr)> { - let mut storage: libc::sockaddr_storage = unsafe { mem::zeroed() }; - let mut addrlen = mem::size_of_val(&storage) as libc::socklen_t; - - let n = cvt(unsafe { - libc::recvfrom( - self.0.raw(), - buf.as_mut_ptr() as *mut c_void, - buf.len(), - flags, - &mut storage as *mut _ as *mut _, - &mut addrlen, - ) - })?; - Ok((n as usize, sockaddr_to_addr(&storage, addrlen as usize)?)) - } - - pub fn recv_from(&self, buf: &mut [u8]) -> io::Result<(usize, SocketAddr)> { - self.recv_from_with_flags(buf, 0) - } - - pub fn peek_from(&self, buf: &mut [u8]) -> io::Result<(usize, SocketAddr)> { - self.recv_from_with_flags(buf, MSG_PEEK) - } - - pub fn write(&self, buf: &[u8]) -> io::Result { - self.0.write(buf) - } - - pub fn write_vectored(&self, bufs: &[IoSlice<'_>]) -> io::Result { - self.0.write_vectored(bufs) - } - - pub fn set_timeout(&self, dur: Option, kind: libc::c_int) -> io::Result<()> { - let timeout = match dur { - Some(dur) => { - if dur.as_secs() == 0 && dur.subsec_nanos() == 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "cannot set a 0 duration timeout", - )); - } - - let secs = if dur.as_secs() > libc::time_t::max_value() as u64 { - libc::time_t::max_value() - } else { - dur.as_secs() as libc::time_t - }; - let mut timeout = libc::timeval { - tv_sec: secs, - tv_usec: (dur.subsec_nanos() / 1000) as libc::suseconds_t, - }; - if timeout.tv_sec == 0 && timeout.tv_usec == 0 { - timeout.tv_usec = 1; - } - timeout - } - None => libc::timeval { tv_sec: 0, tv_usec: 0 }, - }; - setsockopt(self, libc::SOL_SOCKET, kind, timeout) - } - - pub fn timeout(&self, kind: libc::c_int) -> io::Result> { - let raw: libc::timeval = getsockopt(self, libc::SOL_SOCKET, kind)?; - if raw.tv_sec == 0 && raw.tv_usec == 0 { - Ok(None) - } else { - let sec = raw.tv_sec as u64; - let nsec = (raw.tv_usec as u32) * 1000; - Ok(Some(Duration::new(sec, nsec))) - } - } - - pub fn shutdown(&self, how: Shutdown) -> io::Result<()> { - let how = match how { - Shutdown::Write => libc::SHUT_WR, - Shutdown::Read => libc::SHUT_RD, - Shutdown::Both => libc::SHUT_RDWR, - }; - cvt(unsafe { libc::shutdown(self.0.raw(), how) })?; - Ok(()) - } - - pub fn set_nodelay(&self, nodelay: bool) -> io::Result<()> { - setsockopt(self, libc::IPPROTO_TCP, libc::TCP_NODELAY, nodelay as c_int) - } - - pub fn nodelay(&self) -> io::Result { - let raw: c_int = getsockopt(self, libc::IPPROTO_TCP, libc::TCP_NODELAY)?; - Ok(raw != 0) - } - - pub fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()> { - let mut nonblocking = nonblocking as libc::c_int; - cvt(unsafe { libc::ioctl(*self.as_inner(), libc::FIONBIO, &mut nonblocking) }).map(drop) - } - - pub fn take_error(&self) -> io::Result> { - let raw: c_int = getsockopt(self, libc::SOL_SOCKET, libc::SO_ERROR)?; - if raw == 0 { Ok(None) } else { Ok(Some(io::Error::from_raw_os_error(raw as i32))) } - } -} - -impl AsInner for Socket { - fn as_inner(&self) -> &c_int { - self.0.as_inner() - } -} - -impl FromInner for Socket { - fn from_inner(fd: c_int) -> Socket { - Socket(FileDesc::new(fd)) - } -} - -impl IntoInner for Socket { - fn into_inner(self) -> c_int { - self.0.into_raw() - } -} - -// In versions of glibc prior to 2.26, there's a bug where the DNS resolver -// will cache the contents of /etc/resolv.conf, so changes to that file on disk -// can be ignored by a long-running program. That can break DNS lookups on e.g. -// laptops where the network comes and goes. See -// https://sourceware.org/bugzilla/show_bug.cgi?id=984. Note however that some -// distros including Debian have patched glibc to fix this for a long time. -// -// A workaround for this bug is to call the res_init libc function, to clear -// the cached configs. Unfortunately, while we believe glibc's implementation -// of res_init is thread-safe, we know that other implementations are not -// (https://github.com/rust-lang/rust/issues/43592). Code here in libstd could -// try to synchronize its res_init calls with a Mutex, but that wouldn't -// protect programs that call into libc in other ways. So instead of calling -// res_init unconditionally, we call it only when we detect we're linking -// against glibc version < 2.26. (That is, when we both know its needed and -// believe it's thread-safe). -#[cfg(target_env = "gnu")] -fn on_resolver_failure() { - /* - use crate::sys; - - // If the version fails to parse, we treat it the same as "not glibc". - if let Some(version) = sys::os::glibc_version() { - if version < (2, 26) { - unsafe { libc::res_init() }; - } - } - */ -} - -#[cfg(not(target_env = "gnu"))] -fn on_resolver_failure() {} - -#[cfg(all(test, taget_env = "gnu"))] -mod test { - use super::*; - - #[test] - fn test_res_init() { - // This mostly just tests that the weak linkage doesn't panic wildly... - res_init_if_glibc_before_2_26().unwrap(); - } - - #[test] - fn test_parse_glibc_version() { - let cases = [ - ("0.0", Some((0, 0))), - ("01.+2", Some((1, 2))), - ("3.4.5.six", Some((3, 4))), - ("1", None), - ("1.-2", None), - ("1.foo", None), - ("foo.1", None), - ]; - for &(version_str, parsed) in cases.iter() { - assert_eq!(parsed, parse_glibc_version(version_str)); - } - } -} diff --git a/crux-mir/lib/std/src/sys/vxworks/os.rs b/crux-mir/lib/std/src/sys/vxworks/os.rs deleted file mode 100644 index 1fadf7161..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/os.rs +++ /dev/null @@ -1,316 +0,0 @@ -use crate::error::Error as StdError; -use crate::ffi::{CStr, CString, OsStr, OsString}; -use crate::fmt; -use crate::io; -use crate::iter; -use crate::marker::PhantomData; -use crate::mem; -use crate::memchr; -use crate::path::{self, Path, PathBuf}; -use crate::slice; -use crate::str; -use crate::sys::cvt; -use crate::sys_common::mutex::{Mutex, MutexGuard}; -use libc::{self, c_char /*,c_void */, c_int}; -/*use sys::fd; this one is probably important */ -use crate::vec; - -const TMPBUF_SZ: usize = 128; - -// This is a terrible fix -use crate::sys::os_str::Buf; -use crate::sys_common::{AsInner, FromInner, IntoInner}; - -pub trait OsStringExt { - fn from_vec(vec: Vec) -> Self; - fn into_vec(self) -> Vec; -} - -impl OsStringExt for OsString { - fn from_vec(vec: Vec) -> OsString { - FromInner::from_inner(Buf { inner: vec }) - } - fn into_vec(self) -> Vec { - self.into_inner().inner - } -} - -pub trait OsStrExt { - fn from_bytes(slice: &[u8]) -> &Self; - fn as_bytes(&self) -> &[u8]; -} - -impl OsStrExt for OsStr { - fn from_bytes(slice: &[u8]) -> &OsStr { - unsafe { mem::transmute(slice) } - } - fn as_bytes(&self) -> &[u8] { - &self.as_inner().inner - } -} - -pub fn errno() -> i32 { - unsafe { libc::errnoGet() } -} - -pub fn set_errno(e: i32) { - unsafe { - libc::errnoSet(e as c_int); - } -} - -/// Gets a detailed string description for the given error number. -pub fn error_string(errno: i32) -> String { - let mut buf = [0 as c_char; TMPBUF_SZ]; - extern "C" { - fn strerror_r(errnum: c_int, buf: *mut c_char, buflen: libc::size_t) -> c_int; - } - - let p = buf.as_mut_ptr(); - unsafe { - if strerror_r(errno as c_int, p, buf.len()) < 0 { - panic!("strerror_r failure"); - } - let p = p as *const _; - str::from_utf8(CStr::from_ptr(p).to_bytes()).unwrap().to_owned() - } -} - -pub fn getcwd() -> io::Result { - let mut buf = Vec::with_capacity(512); - loop { - unsafe { - let ptr = buf.as_mut_ptr() as *mut libc::c_char; - if !libc::getcwd(ptr, buf.capacity() as libc::size_t).is_null() { - let len = CStr::from_ptr(buf.as_ptr() as *const libc::c_char).to_bytes().len(); - buf.set_len(len); - buf.shrink_to_fit(); - return Ok(PathBuf::from(OsString::from_vec(buf))); - } else { - let error = io::Error::last_os_error(); - if error.raw_os_error() != Some(libc::ERANGE) { - return Err(error); - } - } - // Trigger the internal buffer resizing logic of `Vec` by requiring - // more space than the current capacity. - let cap = buf.capacity(); - buf.set_len(cap); - buf.reserve(1); - } - } -} - -pub fn chdir(p: &path::Path) -> io::Result<()> { - let p: &OsStr = p.as_ref(); - let p = CString::new(p.as_bytes())?; - unsafe { - match libc::chdir(p.as_ptr()) == (0 as c_int) { - true => Ok(()), - false => Err(io::Error::last_os_error()), - } - } -} - -pub struct SplitPaths<'a> { - iter: iter::Map bool>, fn(&'a [u8]) -> PathBuf>, -} - -pub fn split_paths(unparsed: &OsStr) -> SplitPaths<'_> { - fn bytes_to_path(b: &[u8]) -> PathBuf { - PathBuf::from(::from_bytes(b)) - } - fn is_colon(b: &u8) -> bool { - *b == b':' - } - let unparsed = unparsed.as_bytes(); - SplitPaths { - iter: unparsed - .split(is_colon as fn(&u8) -> bool) - .map(bytes_to_path as fn(&[u8]) -> PathBuf), - } -} - -impl<'a> Iterator for SplitPaths<'a> { - type Item = PathBuf; - fn next(&mut self) -> Option { - self.iter.next() - } - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() - } -} - -#[derive(Debug)] -pub struct JoinPathsError; - -pub fn join_paths(paths: I) -> Result -where - I: Iterator, - T: AsRef, -{ - let mut joined = Vec::new(); - let sep = b':'; - - for (i, path) in paths.enumerate() { - let path = path.as_ref().as_bytes(); - if i > 0 { - joined.push(sep) - } - if path.contains(&sep) { - return Err(JoinPathsError); - } - joined.extend_from_slice(path); - } - Ok(OsStringExt::from_vec(joined)) -} - -impl fmt::Display for JoinPathsError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - "path segment contains separator `:`".fmt(f) - } -} - -impl StdError for JoinPathsError { - #[allow(deprecated)] - fn description(&self) -> &str { - "failed to join paths" - } -} - -pub fn current_exe() -> io::Result { - #[cfg(test)] - use realstd::env; - - #[cfg(not(test))] - use crate::env; - - let exe_path = env::args().next().unwrap(); - let path = Path::new(&exe_path); - path.canonicalize() -} - -pub struct Env { - iter: vec::IntoIter<(OsString, OsString)>, - _dont_send_or_sync_me: PhantomData<*mut ()>, -} - -impl Iterator for Env { - type Item = (OsString, OsString); - fn next(&mut self) -> Option<(OsString, OsString)> { - self.iter.next() - } - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() - } -} - -pub unsafe fn environ() -> *mut *const *const c_char { - extern "C" { - static mut environ: *const *const c_char; - } - &mut environ -} - -pub unsafe fn env_lock() -> MutexGuard<'static> { - // We never call `ENV_LOCK.init()`, so it is UB to attempt to - // acquire this mutex reentrantly! - static ENV_LOCK: Mutex = Mutex::new(); - ENV_LOCK.lock() -} - -/// Returns a vector of (variable, value) byte-vector pairs for all the -/// environment variables of the current process. -pub fn env() -> Env { - unsafe { - let _guard = env_lock(); - let mut environ = *environ(); - if environ.is_null() { - panic!("os::env() failure getting env string from OS: {}", io::Error::last_os_error()); - } - let mut result = Vec::new(); - while !(*environ).is_null() { - if let Some(key_value) = parse(CStr::from_ptr(*environ).to_bytes()) { - result.push(key_value); - } - environ = environ.add(1); - } - return Env { iter: result.into_iter(), _dont_send_or_sync_me: PhantomData }; - } - - fn parse(input: &[u8]) -> Option<(OsString, OsString)> { - // Strategy (copied from glibc): Variable name and value are separated - // by an ASCII equals sign '='. Since a variable name must not be - // empty, allow variable names starting with an equals sign. Skip all - // malformed lines. - if input.is_empty() { - return None; - } - let pos = memchr::memchr(b'=', &input[1..]).map(|p| p + 1); - pos.map(|p| { - ( - OsStringExt::from_vec(input[..p].to_vec()), - OsStringExt::from_vec(input[p + 1..].to_vec()), - ) - }) - } -} - -pub fn getenv(k: &OsStr) -> io::Result> { - // environment variables with a nul byte can't be set, so their value is - // always None as well - let k = CString::new(k.as_bytes())?; - unsafe { - let _guard = env_lock(); - let s = libc::getenv(k.as_ptr()) as *const libc::c_char; - let ret = if s.is_null() { - None - } else { - Some(OsStringExt::from_vec(CStr::from_ptr(s).to_bytes().to_vec())) - }; - Ok(ret) - } -} - -pub fn setenv(k: &OsStr, v: &OsStr) -> io::Result<()> { - let k = CString::new(k.as_bytes())?; - let v = CString::new(v.as_bytes())?; - - unsafe { - let _guard = env_lock(); - cvt(libc::setenv(k.as_ptr(), v.as_ptr(), 1)).map(drop) - } -} - -pub fn unsetenv(n: &OsStr) -> io::Result<()> { - let nbuf = CString::new(n.as_bytes())?; - - unsafe { - let _guard = env_lock(); - cvt(libc::unsetenv(nbuf.as_ptr())).map(drop) - } -} - -pub fn page_size() -> usize { - unsafe { libc::sysconf(libc::_SC_PAGESIZE) as usize } -} - -pub fn temp_dir() -> PathBuf { - crate::env::var_os("TMPDIR").map(PathBuf::from).unwrap_or_else(|| PathBuf::from("/tmp")) -} - -pub fn home_dir() -> Option { - crate::env::var_os("HOME").or_else(|| None).map(PathBuf::from) -} - -pub fn exit(code: i32) -> ! { - unsafe { libc::exit(code as c_int) } -} - -pub fn getpid() -> u32 { - unsafe { libc::getpid() as u32 } -} - -pub fn getppid() -> u32 { - unsafe { libc::getppid() as u32 } -} diff --git a/crux-mir/lib/std/src/sys/vxworks/path.rs b/crux-mir/lib/std/src/sys/vxworks/path.rs deleted file mode 100644 index 840a7ae04..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/path.rs +++ /dev/null @@ -1,19 +0,0 @@ -use crate::ffi::OsStr; -use crate::path::Prefix; - -#[inline] -pub fn is_sep_byte(b: u8) -> bool { - b == b'/' -} - -#[inline] -pub fn is_verbatim_sep(b: u8) -> bool { - b == b'/' -} - -pub fn parse_prefix(_: &OsStr) -> Option> { - None -} - -pub const MAIN_SEP_STR: &str = "/"; -pub const MAIN_SEP: char = '/'; diff --git a/crux-mir/lib/std/src/sys/vxworks/pipe.rs b/crux-mir/lib/std/src/sys/vxworks/pipe.rs deleted file mode 100644 index 0990cb8e8..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/pipe.rs +++ /dev/null @@ -1,96 +0,0 @@ -use crate::io::{self, IoSlice, IoSliceMut}; -use crate::mem; -use crate::sync::atomic::AtomicBool; -use crate::sys::fd::FileDesc; -use crate::sys::{cvt, cvt_r}; -use libc::{self /*, c_int apparently not used? */}; - -pub struct AnonPipe(FileDesc); - -pub fn anon_pipe() -> io::Result<(AnonPipe, AnonPipe)> { - static INVALID: AtomicBool = AtomicBool::new(false); - - let mut fds = [0; 2]; - cvt(unsafe { libc::pipe(fds.as_mut_ptr()) })?; - - let fd0 = FileDesc::new(fds[0]); - let fd1 = FileDesc::new(fds[1]); - fd0.set_cloexec()?; - fd1.set_cloexec()?; - Ok((AnonPipe(fd0), AnonPipe(fd1))) -} - -impl AnonPipe { - pub fn read(&self, buf: &mut [u8]) -> io::Result { - self.0.read(buf) - } - pub fn read_vectored(&self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { - self.0.read_vectored(bufs) - } - - pub fn write(&self, buf: &[u8]) -> io::Result { - self.0.write(buf) - } - - pub fn write_vectored(&self, bufs: &[IoSlice<'_>]) -> io::Result { - self.0.write_vectored(bufs) - } - - pub fn fd(&self) -> &FileDesc { - &self.0 - } - pub fn into_fd(self) -> FileDesc { - self.0 - } - pub fn diverge(&self) -> ! { - panic!() - } -} - -pub fn read2(p1: AnonPipe, v1: &mut Vec, p2: AnonPipe, v2: &mut Vec) -> io::Result<()> { - // Set both pipes into nonblocking mode as we're gonna be reading from both - // in the `select` loop below, and we wouldn't want one to block the other! - let p1 = p1.into_fd(); - let p2 = p2.into_fd(); - p1.set_nonblocking_pipe(true)?; - p2.set_nonblocking_pipe(true)?; - - let mut fds: [libc::pollfd; 2] = unsafe { mem::zeroed() }; - fds[0].fd = p1.raw(); - fds[0].events = libc::POLLIN; - fds[1].fd = p2.raw(); - fds[1].events = libc::POLLIN; - loop { - // wait for either pipe to become readable using `poll` - cvt_r(|| unsafe { libc::poll(fds.as_mut_ptr(), 2, -1) })?; - - if fds[0].revents != 0 && read(&p1, v1)? { - p2.set_nonblocking_pipe(false)?; - return p2.read_to_end(v2).map(drop); - } - if fds[1].revents != 0 && read(&p2, v2)? { - p1.set_nonblocking_pipe(false)?; - return p1.read_to_end(v1).map(drop); - } - } - - // Read as much as we can from each pipe, ignoring EWOULDBLOCK or - // EAGAIN. If we hit EOF, then this will happen because the underlying - // reader will return Ok(0), in which case we'll see `Ok` ourselves. In - // this case we flip the other fd back into blocking mode and read - // whatever's leftover on that file descriptor. - fn read(fd: &FileDesc, dst: &mut Vec) -> Result { - match fd.read_to_end(dst) { - Ok(_) => Ok(true), - Err(e) => { - if e.raw_os_error() == Some(libc::EWOULDBLOCK) - || e.raw_os_error() == Some(libc::EAGAIN) - { - Ok(false) - } else { - Err(e) - } - } - } - } -} diff --git a/crux-mir/lib/std/src/sys/vxworks/process/mod.rs b/crux-mir/lib/std/src/sys/vxworks/process/mod.rs deleted file mode 100644 index c59782ff4..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/process/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub use self::process_common::{Command, ExitCode, ExitStatus, Stdio, StdioPipes}; -pub use self::process_inner::Process; -pub use crate::ffi::OsString as EnvKey; - -mod process_common; -#[path = "process_vxworks.rs"] -mod process_inner; diff --git a/crux-mir/lib/std/src/sys/vxworks/process/process_common.rs b/crux-mir/lib/std/src/sys/vxworks/process/process_common.rs deleted file mode 100644 index 6d5506bec..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/process/process_common.rs +++ /dev/null @@ -1,405 +0,0 @@ -use crate::os::unix::prelude::*; - -use crate::collections::BTreeMap; -use crate::ffi::{CStr, CString, OsStr, OsString}; -use crate::fmt; -use crate::io; -use crate::ptr; -use crate::sys::fd::FileDesc; -use crate::sys::fs::{File, OpenOptions}; -use crate::sys::pipe::{self, AnonPipe}; -use crate::sys_common::process::CommandEnv; - -use libc::{c_char, c_int, gid_t, uid_t, EXIT_FAILURE, EXIT_SUCCESS}; - -//////////////////////////////////////////////////////////////////////////////// -// Command -//////////////////////////////////////////////////////////////////////////////// - -pub struct Command { - // Currently we try hard to ensure that the call to `.exec()` doesn't - // actually allocate any memory. While many platforms try to ensure that - // memory allocation works after a fork in a multithreaded process, it's - // been observed to be buggy and somewhat unreliable, so we do our best to - // just not do it at all! - // - // Along those lines, the `argv` and `envp` raw pointers here are exactly - // what's gonna get passed to `execvp`. The `argv` array starts with the - // `program` and ends with a NULL, and the `envp` pointer, if present, is - // also null-terminated. - // - // Right now we don't support removing arguments, so there's no much fancy - // support there, but we support adding and removing environment variables, - // so a side table is used to track where in the `envp` array each key is - // located. Whenever we add a key we update it in place if it's already - // present, and whenever we remove a key we update the locations of all - // other keys. - program: CString, - args: Vec, - argv: Argv, - env: CommandEnv, - - cwd: Option, - uid: Option, - gid: Option, - saw_nul: bool, - closures: Vec io::Result<()> + Send + Sync>>, - stdin: Option, - stdout: Option, - stderr: Option, -} - -// Create a new type for argv, so that we can make it `Send` -struct Argv(Vec<*const c_char>); - -// It is safe to make Argv Send, because it contains pointers to memory owned by `Command.args` -unsafe impl Send for Argv {} - -// passed back to std::process with the pipes connected to the child, if any -// were requested -pub struct StdioPipes { - pub stdin: Option, - pub stdout: Option, - pub stderr: Option, -} - -// passed to do_exec() with configuration of what the child stdio should look -// like -pub struct ChildPipes { - pub stdin: ChildStdio, - pub stdout: ChildStdio, - pub stderr: ChildStdio, -} - -pub enum ChildStdio { - Inherit, - Explicit(c_int), - Owned(FileDesc), -} - -pub enum Stdio { - Inherit, - Null, - MakePipe, - Fd(FileDesc), -} - -impl Command { - pub fn new(program: &OsStr) -> Command { - let mut saw_nul = false; - let program = os2c(program, &mut saw_nul); - Command { - argv: Argv(vec![program.as_ptr(), ptr::null()]), - args: vec![program.clone()], - program, - env: Default::default(), - cwd: None, - uid: None, - gid: None, - saw_nul, - closures: Vec::new(), - stdin: None, - stdout: None, - stderr: None, - } - } - - pub fn set_arg_0(&mut self, arg: &OsStr) { - // Set a new arg0 - let arg = os2c(arg, &mut self.saw_nul); - debug_assert!(self.argv.0.len() > 1); - self.argv.0[0] = arg.as_ptr(); - self.args[0] = arg; - } - - pub fn arg(&mut self, arg: &OsStr) { - // Overwrite the trailing NULL pointer in `argv` and then add a new null - // pointer. - let arg = os2c(arg, &mut self.saw_nul); - self.argv.0[self.args.len()] = arg.as_ptr(); - self.argv.0.push(ptr::null()); - - // Also make sure we keep track of the owned value to schedule a - // destructor for this memory. - self.args.push(arg); - } - - pub fn cwd(&mut self, dir: &OsStr) { - self.cwd = Some(os2c(dir, &mut self.saw_nul)); - } - pub fn uid(&mut self, id: uid_t) { - self.uid = Some(id); - } - pub fn gid(&mut self, id: gid_t) { - self.gid = Some(id); - } - - pub fn saw_nul(&self) -> bool { - self.saw_nul - } - pub fn get_argv(&self) -> &Vec<*const c_char> { - &self.argv.0 - } - - pub fn get_program(&self) -> &CStr { - &*self.program - } - - #[allow(dead_code)] - pub fn get_cwd(&self) -> &Option { - &self.cwd - } - #[allow(dead_code)] - pub fn get_uid(&self) -> Option { - self.uid - } - #[allow(dead_code)] - pub fn get_gid(&self) -> Option { - self.gid - } - - pub fn get_closures(&mut self) -> &mut Vec io::Result<()> + Send + Sync>> { - &mut self.closures - } - - pub unsafe fn pre_exec(&mut self, _f: Box io::Result<()> + Send + Sync>) { - // Fork() is not supported in vxWorks so no way to run the closure in the new procecss. - unimplemented!(); - } - - pub fn stdin(&mut self, stdin: Stdio) { - self.stdin = Some(stdin); - } - - pub fn stdout(&mut self, stdout: Stdio) { - self.stdout = Some(stdout); - } - - pub fn stderr(&mut self, stderr: Stdio) { - self.stderr = Some(stderr); - } - - pub fn env_mut(&mut self) -> &mut CommandEnv { - &mut self.env - } - - pub fn capture_env(&mut self) -> Option { - let maybe_env = self.env.capture_if_changed(); - maybe_env.map(|env| construct_envp(env, &mut self.saw_nul)) - } - #[allow(dead_code)] - pub fn env_saw_path(&self) -> bool { - self.env.have_changed_path() - } - - pub fn setup_io( - &self, - default: Stdio, - needs_stdin: bool, - ) -> io::Result<(StdioPipes, ChildPipes)> { - let null = Stdio::Null; - let default_stdin = if needs_stdin { &default } else { &null }; - let stdin = self.stdin.as_ref().unwrap_or(default_stdin); - let stdout = self.stdout.as_ref().unwrap_or(&default); - let stderr = self.stderr.as_ref().unwrap_or(&default); - let (their_stdin, our_stdin) = stdin.to_child_stdio(true)?; - let (their_stdout, our_stdout) = stdout.to_child_stdio(false)?; - let (their_stderr, our_stderr) = stderr.to_child_stdio(false)?; - let ours = StdioPipes { stdin: our_stdin, stdout: our_stdout, stderr: our_stderr }; - let theirs = ChildPipes { stdin: their_stdin, stdout: their_stdout, stderr: their_stderr }; - Ok((ours, theirs)) - } -} - -fn os2c(s: &OsStr, saw_nul: &mut bool) -> CString { - CString::new(s.as_bytes()).unwrap_or_else(|_e| { - *saw_nul = true; - CString::new("").unwrap() - }) -} - -// Helper type to manage ownership of the strings within a C-style array. -pub struct CStringArray { - items: Vec, - ptrs: Vec<*const c_char>, -} - -impl CStringArray { - pub fn with_capacity(capacity: usize) -> Self { - let mut result = CStringArray { - items: Vec::with_capacity(capacity), - ptrs: Vec::with_capacity(capacity + 1), - }; - result.ptrs.push(ptr::null()); - result - } - pub fn push(&mut self, item: CString) { - let l = self.ptrs.len(); - self.ptrs[l - 1] = item.as_ptr(); - self.ptrs.push(ptr::null()); - self.items.push(item); - } - pub fn as_ptr(&self) -> *const *const c_char { - self.ptrs.as_ptr() - } -} - -fn construct_envp(env: BTreeMap, saw_nul: &mut bool) -> CStringArray { - let mut result = CStringArray::with_capacity(env.len()); - for (k, v) in env { - let mut k: OsString = k.into(); - - // Reserve additional space for '=' and null terminator - k.reserve_exact(v.len() + 2); - k.push("="); - k.push(&v); - - // Add the new entry into the array - if let Ok(item) = CString::new(k.into_vec()) { - result.push(item); - } else { - *saw_nul = true; - } - } - - result -} - -impl Stdio { - pub fn to_child_stdio(&self, readable: bool) -> io::Result<(ChildStdio, Option)> { - match *self { - Stdio::Inherit => Ok((ChildStdio::Inherit, None)), - - // Make sure that the source descriptors are not an stdio - // descriptor, otherwise the order which we set the child's - // descriptors may blow away a descriptor which we are hoping to - // save. For example, suppose we want the child's stderr to be the - // parent's stdout, and the child's stdout to be the parent's - // stderr. No matter which we dup first, the second will get - // overwritten prematurely. - Stdio::Fd(ref fd) => { - if fd.raw() >= 0 && fd.raw() <= libc::STDERR_FILENO { - Ok((ChildStdio::Owned(fd.duplicate()?), None)) - } else { - Ok((ChildStdio::Explicit(fd.raw()), None)) - } - } - - Stdio::MakePipe => { - let (reader, writer) = pipe::anon_pipe()?; - let (ours, theirs) = if readable { (writer, reader) } else { (reader, writer) }; - Ok((ChildStdio::Owned(theirs.into_fd()), Some(ours))) - } - - Stdio::Null => { - let mut opts = OpenOptions::new(); - opts.read(readable); - opts.write(!readable); - let path = unsafe { CStr::from_ptr("/null\0".as_ptr() as *const _) }; - let fd = File::open_c(&path, &opts)?; - Ok((ChildStdio::Owned(fd.into_fd()), None)) - } - } - } -} - -impl From for Stdio { - fn from(pipe: AnonPipe) -> Stdio { - Stdio::Fd(pipe.into_fd()) - } -} - -impl From for Stdio { - fn from(file: File) -> Stdio { - Stdio::Fd(file.into_fd()) - } -} - -impl ChildStdio { - pub fn fd(&self) -> Option { - match *self { - ChildStdio::Inherit => None, - ChildStdio::Explicit(fd) => Some(fd), - ChildStdio::Owned(ref fd) => Some(fd.raw()), - } - } -} - -impl fmt::Debug for Command { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.program != self.args[0] { - write!(f, "[{:?}] ", self.program)?; - } - write!(f, "{:?}", self.args[0])?; - - for arg in &self.args[1..] { - write!(f, " {:?}", arg)?; - } - Ok(()) - } -} - -/// Unix exit statuses -#[derive(PartialEq, Eq, Clone, Copy, Debug)] -pub struct ExitStatus(c_int); - -impl ExitStatus { - pub fn new(status: c_int) -> ExitStatus { - ExitStatus(status) - } - - fn exited(&self) -> bool { - /*unsafe*/ - { libc::WIFEXITED(self.0) } - } - - pub fn success(&self) -> bool { - self.code() == Some(0) - } - - pub fn code(&self) -> Option { - if self.exited() { - Some(/*unsafe*/ { libc::WEXITSTATUS(self.0) }) - } else { - None - } - } - - pub fn signal(&self) -> Option { - if !self.exited() { - Some(/*unsafe*/ { libc::WTERMSIG(self.0) }) - } else { - None - } - } -} - -impl From for ExitStatus { - fn from(a: c_int) -> ExitStatus { - ExitStatus(a) - } -} - -impl fmt::Display for ExitStatus { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if let Some(code) = self.code() { - write!(f, "exit code: {}", code) - } else { - let signal = self.signal().unwrap(); - write!(f, "signal: {}", signal) - } - } -} - -#[derive(PartialEq, Eq, Clone, Copy, Debug)] -pub struct ExitCode(u8); - -impl ExitCode { - pub const SUCCESS: ExitCode = ExitCode(EXIT_SUCCESS as _); - pub const FAILURE: ExitCode = ExitCode(EXIT_FAILURE as _); - - #[inline] - pub fn as_i32(&self) -> i32 { - self.0 as i32 - } -} diff --git a/crux-mir/lib/std/src/sys/vxworks/rand.rs b/crux-mir/lib/std/src/sys/vxworks/rand.rs deleted file mode 100644 index 87ebd2c95..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/rand.rs +++ /dev/null @@ -1,37 +0,0 @@ -use crate::mem; -use crate::slice; - -pub fn hashmap_random_keys() -> (u64, u64) { - let mut v = (0, 0); - unsafe { - let view = slice::from_raw_parts_mut(&mut v as *mut _ as *mut u8, mem::size_of_val(&v)); - imp::fill_bytes(view); - } - return v; -} - -mod imp { - use crate::io; - use core::sync::atomic::{AtomicBool, Ordering::Relaxed}; - use libc; - - pub fn fill_bytes(v: &mut [u8]) { - static RNG_INIT: AtomicBool = AtomicBool::new(false); - while !RNG_INIT.load(Relaxed) { - let ret = unsafe { libc::randSecure() }; - if ret < 0 { - panic!("couldn't generate random bytes: {}", io::Error::last_os_error()); - } else if ret > 0 { - RNG_INIT.store(true, Relaxed); - break; - } - unsafe { libc::usleep(10) }; - } - let ret = unsafe { - libc::randABytes(v.as_mut_ptr() as *mut libc::c_uchar, v.len() as libc::c_int) - }; - if ret < 0 { - panic!("couldn't generate random bytes: {}", io::Error::last_os_error()); - } - } -} diff --git a/crux-mir/lib/std/src/sys/vxworks/rwlock.rs b/crux-mir/lib/std/src/sys/vxworks/rwlock.rs deleted file mode 100644 index fd2e1a6e7..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/rwlock.rs +++ /dev/null @@ -1,115 +0,0 @@ -use crate::cell::UnsafeCell; -use crate::sync::atomic::{AtomicUsize, Ordering}; -use libc; - -pub struct RWLock { - inner: UnsafeCell, - write_locked: UnsafeCell, - num_readers: AtomicUsize, -} - -unsafe impl Send for RWLock {} -unsafe impl Sync for RWLock {} - -impl RWLock { - pub const fn new() -> RWLock { - RWLock { - inner: UnsafeCell::new(libc::PTHREAD_RWLOCK_INITIALIZER), - write_locked: UnsafeCell::new(false), - num_readers: AtomicUsize::new(0), - } - } - - #[inline] - pub unsafe fn read(&self) { - let r = libc::pthread_rwlock_rdlock(self.inner.get()); - if r == libc::EAGAIN { - panic!("rwlock maximum reader count exceeded"); - } else if r == libc::EDEADLK || (r == 0 && *self.write_locked.get()) { - if r == 0 { - self.raw_unlock(); - } - panic!("rwlock read lock would result in deadlock"); - } else { - debug_assert_eq!(r, 0); - self.num_readers.fetch_add(1, Ordering::Relaxed); - } - } - - #[inline] - pub unsafe fn try_read(&self) -> bool { - let r = libc::pthread_rwlock_tryrdlock(self.inner.get()); - if r == 0 { - if *self.write_locked.get() { - self.raw_unlock(); - false - } else { - self.num_readers.fetch_add(1, Ordering::Relaxed); - true - } - } else { - false - } - } - - #[inline] - pub unsafe fn write(&self) { - let r = libc::pthread_rwlock_wrlock(self.inner.get()); - // See comments above for why we check for EDEADLK and write_locked. We - // also need to check that num_readers is 0. - if r == libc::EDEADLK - || *self.write_locked.get() - || self.num_readers.load(Ordering::Relaxed) != 0 - { - if r == 0 { - self.raw_unlock(); - } - panic!("rwlock write lock would result in deadlock"); - } else { - debug_assert_eq!(r, 0); - } - *self.write_locked.get() = true; - } - - #[inline] - pub unsafe fn try_write(&self) -> bool { - let r = libc::pthread_rwlock_trywrlock(self.inner.get()); - if r == 0 { - if *self.write_locked.get() || self.num_readers.load(Ordering::Relaxed) != 0 { - self.raw_unlock(); - false - } else { - *self.write_locked.get() = true; - true - } - } else { - false - } - } - - #[inline] - unsafe fn raw_unlock(&self) { - let r = libc::pthread_rwlock_unlock(self.inner.get()); - debug_assert_eq!(r, 0); - } - - #[inline] - pub unsafe fn read_unlock(&self) { - debug_assert!(!*self.write_locked.get()); - self.num_readers.fetch_sub(1, Ordering::Relaxed); - self.raw_unlock(); - } - - #[inline] - pub unsafe fn write_unlock(&self) { - debug_assert_eq!(self.num_readers.load(Ordering::Relaxed), 0); - debug_assert!(*self.write_locked.get()); - *self.write_locked.get() = false; - self.raw_unlock(); - } - #[inline] - pub unsafe fn destroy(&self) { - let r = libc::pthread_rwlock_destroy(self.inner.get()); - debug_assert_eq!(r, 0); - } -} diff --git a/crux-mir/lib/std/src/sys/vxworks/stack_overflow.rs b/crux-mir/lib/std/src/sys/vxworks/stack_overflow.rs deleted file mode 100644 index 7b58c8319..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/stack_overflow.rs +++ /dev/null @@ -1,38 +0,0 @@ -#![cfg_attr(test, allow(dead_code))] - -use self::imp::{drop_handler, make_handler}; - -pub use self::imp::cleanup; -pub use self::imp::init; - -pub struct Handler { - _data: *mut libc::c_void, -} - -impl Handler { - pub unsafe fn new() -> Handler { - make_handler() - } -} - -impl Drop for Handler { - fn drop(&mut self) { - unsafe { - drop_handler(self); - } - } -} - -mod imp { - use crate::ptr; - - pub unsafe fn init() {} - - pub unsafe fn cleanup() {} - - pub unsafe fn make_handler() -> super::Handler { - super::Handler { _data: ptr::null_mut() } - } - - pub unsafe fn drop_handler(_handler: &mut super::Handler) {} -} diff --git a/crux-mir/lib/std/src/sys/vxworks/stdio.rs b/crux-mir/lib/std/src/sys/vxworks/stdio.rs deleted file mode 100644 index 622444cca..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/stdio.rs +++ /dev/null @@ -1,69 +0,0 @@ -use crate::io; -use crate::sys::fd::FileDesc; - -pub struct Stdin(()); -pub struct Stdout(()); -pub struct Stderr(()); - -impl Stdin { - pub fn new() -> io::Result { - Ok(Stdin(())) - } -} - -impl io::Read for Stdin { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - let fd = FileDesc::new(libc::STDIN_FILENO); - let ret = fd.read(buf); - fd.into_raw(); // do not close this FD - ret - } -} - -impl Stdout { - pub fn new() -> io::Result { - Ok(Stdout(())) - } -} - -impl io::Write for Stdout { - fn write(&mut self, buf: &[u8]) -> io::Result { - let fd = FileDesc::new(libc::STDOUT_FILENO); - let ret = fd.write(buf); - fd.into_raw(); // do not close this FD - ret - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -impl Stderr { - pub fn new() -> io::Result { - Ok(Stderr(())) - } -} - -impl io::Write for Stderr { - fn write(&mut self, buf: &[u8]) -> io::Result { - let fd = FileDesc::new(libc::STDERR_FILENO); - let ret = fd.write(buf); - fd.into_raw(); // do not close this FD - ret - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -pub fn is_ebadf(err: &io::Error) -> bool { - err.raw_os_error() == Some(libc::EBADF as i32) -} - -pub const STDIN_BUF_SIZE: usize = crate::sys_common::io::DEFAULT_BUF_SIZE; - -pub fn panic_output() -> Option { - Stderr::new().ok() -} diff --git a/crux-mir/lib/std/src/sys/vxworks/thread.rs b/crux-mir/lib/std/src/sys/vxworks/thread.rs deleted file mode 100644 index e0d104b5f..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/thread.rs +++ /dev/null @@ -1,148 +0,0 @@ -use crate::cmp; -use crate::ffi::CStr; -use crate::io; -use crate::mem; -use crate::ptr; -use crate::sys::os; -use crate::time::Duration; - -use crate::sys_common::thread::*; - -pub const DEFAULT_MIN_STACK_SIZE: usize = 0x40000; // 256K - -pub struct Thread { - id: libc::pthread_t, -} - -// Some platforms may have pthread_t as a pointer in which case we still want -// a thread to be Send/Sync -unsafe impl Send for Thread {} -unsafe impl Sync for Thread {} - -// The pthread_attr_setstacksize symbol doesn't exist in the emscripten libc, -// so we have to not link to it to satisfy emcc's ERROR_ON_UNDEFINED_SYMBOLS. -unsafe fn pthread_attr_setstacksize( - attr: *mut libc::pthread_attr_t, - stack_size: libc::size_t, -) -> libc::c_int { - libc::pthread_attr_setstacksize(attr, stack_size) -} - -impl Thread { - // unsafe: see thread::Builder::spawn_unchecked for safety requirements - pub unsafe fn new(stack: usize, p: Box) -> io::Result { - let p = box p; - let mut native: libc::pthread_t = mem::zeroed(); - let mut attr: libc::pthread_attr_t = mem::zeroed(); - assert_eq!(libc::pthread_attr_init(&mut attr), 0); - - let stack_size = cmp::max(stack, min_stack_size(&attr)); - - match pthread_attr_setstacksize(&mut attr, stack_size) { - 0 => {} - n => { - assert_eq!(n, libc::EINVAL); - // EINVAL means |stack_size| is either too small or not a - // multiple of the system page size. Because it's definitely - // >= PTHREAD_STACK_MIN, it must be an alignment issue. - // Round up to the nearest page and try again. - let page_size = os::page_size(); - let stack_size = - (stack_size + page_size - 1) & (-(page_size as isize - 1) as usize - 1); - assert_eq!(libc::pthread_attr_setstacksize(&mut attr, stack_size), 0); - } - }; - - let ret = libc::pthread_create(&mut native, &attr, thread_start, &*p as *const _ as *mut _); - assert_eq!(libc::pthread_attr_destroy(&mut attr), 0); - - return if ret != 0 { - Err(io::Error::from_raw_os_error(ret)) - } else { - mem::forget(p); // ownership passed to pthread_create - Ok(Thread { id: native }) - }; - - extern "C" fn thread_start(main: *mut libc::c_void) -> *mut libc::c_void { - unsafe { - start_thread(main as *mut u8); - } - ptr::null_mut() - } - } - - pub fn yield_now() { - let ret = unsafe { libc::sched_yield() }; - debug_assert_eq!(ret, 0); - } - - pub fn set_name(_name: &CStr) { - // VxWorks does not provide a way to set the task name except at creation time - } - - pub fn sleep(dur: Duration) { - let mut secs = dur.as_secs(); - let mut nsecs = dur.subsec_nanos() as _; - - // If we're awoken with a signal then the return value will be -1 and - // nanosleep will fill in `ts` with the remaining time. - unsafe { - while secs > 0 || nsecs > 0 { - let mut ts = libc::timespec { - tv_sec: cmp::min(libc::time_t::max_value() as u64, secs) as libc::time_t, - tv_nsec: nsecs, - }; - secs -= ts.tv_sec as u64; - if libc::nanosleep(&ts, &mut ts) == -1 { - assert_eq!(os::errno(), libc::EINTR); - secs += ts.tv_sec as u64; - nsecs = ts.tv_nsec; - } else { - nsecs = 0; - } - } - } - } - - pub fn join(self) { - unsafe { - let ret = libc::pthread_join(self.id, ptr::null_mut()); - mem::forget(self); - assert!(ret == 0, "failed to join thread: {}", io::Error::from_raw_os_error(ret)); - } - } - - pub fn id(&self) -> libc::pthread_t { - self.id - } - - pub fn into_id(self) -> libc::pthread_t { - let id = self.id; - mem::forget(self); - id - } -} - -impl Drop for Thread { - fn drop(&mut self) { - let ret = unsafe { libc::pthread_detach(self.id) }; - debug_assert_eq!(ret, 0); - } -} - -#[cfg_attr(test, allow(dead_code))] -pub mod guard { - use crate::ops::Range; - pub type Guard = Range; - pub unsafe fn current() -> Option { - None - } - pub unsafe fn init() -> Option { - None - } - pub unsafe fn deinit() {} -} - -fn min_stack_size(_: *const libc::pthread_attr_t) -> usize { - libc::PTHREAD_STACK_MIN -} diff --git a/crux-mir/lib/std/src/sys/vxworks/thread_local.rs b/crux-mir/lib/std/src/sys/vxworks/thread_local.rs deleted file mode 100644 index 2c5b94b1e..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/thread_local.rs +++ /dev/null @@ -1,34 +0,0 @@ -#![allow(dead_code)] // not used on all platforms - -use crate::mem; - -pub type Key = libc::pthread_key_t; - -#[inline] -pub unsafe fn create(dtor: Option) -> Key { - let mut key = 0; - assert_eq!(libc::pthread_key_create(&mut key, mem::transmute(dtor)), 0); - key -} - -#[inline] -pub unsafe fn set(key: Key, value: *mut u8) { - let r = libc::pthread_setspecific(key, value as *mut _); - debug_assert_eq!(r, 0); -} - -#[inline] -pub unsafe fn get(key: Key) -> *mut u8 { - libc::pthread_getspecific(key) as *mut u8 -} - -#[inline] -pub unsafe fn destroy(key: Key) { - let r = libc::pthread_key_delete(key); - debug_assert_eq!(r, 0); -} - -#[inline] -pub fn requires_synchronized_create() -> bool { - false -} diff --git a/crux-mir/lib/std/src/sys/vxworks/time.rs b/crux-mir/lib/std/src/sys/vxworks/time.rs deleted file mode 100644 index 8ebbf8921..000000000 --- a/crux-mir/lib/std/src/sys/vxworks/time.rs +++ /dev/null @@ -1,199 +0,0 @@ -use crate::cmp::Ordering; -use crate::time::Duration; -use ::core::hash::{Hash, Hasher}; -use libc; - -pub use self::inner::{Instant, SystemTime, UNIX_EPOCH}; -use crate::convert::TryInto; - -const NSEC_PER_SEC: u64 = 1_000_000_000; - -#[derive(Copy, Clone)] -struct Timespec { - t: libc::timespec, -} - -impl Timespec { - const fn zero() -> Timespec { - Timespec { t: libc::timespec { tv_sec: 0, tv_nsec: 0 } } - } - fn sub_timespec(&self, other: &Timespec) -> Result { - if self >= other { - Ok(if self.t.tv_nsec >= other.t.tv_nsec { - Duration::new( - (self.t.tv_sec - other.t.tv_sec) as u64, - (self.t.tv_nsec - other.t.tv_nsec) as u32, - ) - } else { - Duration::new( - (self.t.tv_sec - 1 - other.t.tv_sec) as u64, - self.t.tv_nsec as u32 + (NSEC_PER_SEC as u32) - other.t.tv_nsec as u32, - ) - }) - } else { - match other.sub_timespec(self) { - Ok(d) => Err(d), - Err(d) => Ok(d), - } - } - } - - fn checked_add_duration(&self, other: &Duration) -> Option { - let mut secs = other - .as_secs() - .try_into() // <- target type would be `libc::time_t` - .ok() - .and_then(|secs| self.t.tv_sec.checked_add(secs))?; - - // Nano calculations can't overflow because nanos are <1B which fit - // in a u32. - let mut nsec = other.subsec_nanos() + self.t.tv_nsec as u32; - if nsec >= NSEC_PER_SEC as u32 { - nsec -= NSEC_PER_SEC as u32; - secs = secs.checked_add(1)?; - } - Some(Timespec { t: libc::timespec { tv_sec: secs, tv_nsec: nsec as _ } }) - } - - fn checked_sub_duration(&self, other: &Duration) -> Option { - let mut secs = other - .as_secs() - .try_into() // <- target type would be `libc::time_t` - .ok() - .and_then(|secs| self.t.tv_sec.checked_sub(secs))?; - - // Similar to above, nanos can't overflow. - let mut nsec = self.t.tv_nsec as i32 - other.subsec_nanos() as i32; - if nsec < 0 { - nsec += NSEC_PER_SEC as i32; - secs = secs.checked_sub(1)?; - } - Some(Timespec { t: libc::timespec { tv_sec: secs, tv_nsec: nsec as _ } }) - } -} - -impl PartialEq for Timespec { - fn eq(&self, other: &Timespec) -> bool { - self.t.tv_sec == other.t.tv_sec && self.t.tv_nsec == other.t.tv_nsec - } -} - -impl Eq for Timespec {} - -impl PartialOrd for Timespec { - fn partial_cmp(&self, other: &Timespec) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for Timespec { - fn cmp(&self, other: &Timespec) -> Ordering { - let me = (self.t.tv_sec, self.t.tv_nsec); - let other = (other.t.tv_sec, other.t.tv_nsec); - me.cmp(&other) - } -} - -impl Hash for Timespec { - fn hash(&self, state: &mut H) { - self.t.tv_sec.hash(state); - self.t.tv_nsec.hash(state); - } -} -mod inner { - use crate::fmt; - use crate::sys::cvt; - use crate::time::Duration; - use libc; - - use super::Timespec; - - #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] - pub struct Instant { - t: Timespec, - } - - #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] - pub struct SystemTime { - t: Timespec, - } - - pub const UNIX_EPOCH: SystemTime = - SystemTime { t: Timespec { t: libc::timespec { tv_sec: 0, tv_nsec: 0 } } }; - - impl Instant { - pub fn now() -> Instant { - Instant { t: now(libc::CLOCK_MONOTONIC) } - } - - pub const fn zero() -> Instant { - Instant { t: Timespec::zero() } - } - - pub fn actually_monotonic() -> bool { - true - } - - pub fn checked_sub_instant(&self, other: &Instant) -> Option { - self.t.sub_timespec(&other.t).ok() - } - - pub fn checked_add_duration(&self, other: &Duration) -> Option { - Some(Instant { t: self.t.checked_add_duration(other)? }) - } - - pub fn checked_sub_duration(&self, other: &Duration) -> Option { - Some(Instant { t: self.t.checked_sub_duration(other)? }) - } - } - - impl fmt::Debug for Instant { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Instant") - .field("tv_sec", &self.t.t.tv_sec) - .field("tv_nsec", &self.t.t.tv_nsec) - .finish() - } - } - - impl SystemTime { - pub fn now() -> SystemTime { - SystemTime { t: now(libc::CLOCK_REALTIME) } - } - - pub fn sub_time(&self, other: &SystemTime) -> Result { - self.t.sub_timespec(&other.t) - } - - pub fn checked_add_duration(&self, other: &Duration) -> Option { - Some(SystemTime { t: self.t.checked_add_duration(other)? }) - } - - pub fn checked_sub_duration(&self, other: &Duration) -> Option { - Some(SystemTime { t: self.t.checked_sub_duration(other)? }) - } - } - - impl From for SystemTime { - fn from(t: libc::timespec) -> SystemTime { - SystemTime { t: Timespec { t: t } } - } - } - - impl fmt::Debug for SystemTime { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SystemTime") - .field("tv_sec", &self.t.t.tv_sec) - .field("tv_nsec", &self.t.t.tv_nsec) - .finish() - } - } - - pub type clock_t = libc::c_int; - - fn now(clock: clock_t) -> Timespec { - let mut t = Timespec { t: libc::timespec { tv_sec: 0, tv_nsec: 0 } }; - cvt(unsafe { libc::clock_gettime(clock, &mut t.t) }).unwrap(); - t - } -} diff --git a/crux-mir/lib/std/src/sys/wasi/alloc.rs b/crux-mir/lib/std/src/sys/wasi/alloc.rs deleted file mode 100644 index e9760d050..000000000 --- a/crux-mir/lib/std/src/sys/wasi/alloc.rs +++ /dev/null @@ -1,43 +0,0 @@ -use crate::alloc::{GlobalAlloc, Layout, System}; -use crate::ptr; -use crate::sys_common::alloc::{realloc_fallback, MIN_ALIGN}; -use libc; - -#[stable(feature = "alloc_system_type", since = "1.28.0")] -unsafe impl GlobalAlloc for System { - #[inline] - unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - if layout.align() <= MIN_ALIGN && layout.align() <= layout.size() { - libc::malloc(layout.size()) as *mut u8 - } else { - libc::aligned_alloc(layout.size(), layout.align()) as *mut u8 - } - } - - #[inline] - unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { - if layout.align() <= MIN_ALIGN && layout.align() <= layout.size() { - libc::calloc(layout.size(), 1) as *mut u8 - } else { - let ptr = self.alloc(layout.clone()); - if !ptr.is_null() { - ptr::write_bytes(ptr, 0, layout.size()); - } - ptr - } - } - - #[inline] - unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { - libc::free(ptr as *mut libc::c_void) - } - - #[inline] - unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - if layout.align() <= MIN_ALIGN && layout.align() <= new_size { - libc::realloc(ptr as *mut libc::c_void, new_size) as *mut u8 - } else { - realloc_fallback(self, ptr, layout, new_size) - } - } -} diff --git a/crux-mir/lib/std/src/sys/wasi/args.rs b/crux-mir/lib/std/src/sys/wasi/args.rs index 02aa68d6f..c42c310e3 100644 --- a/crux-mir/lib/std/src/sys/wasi/args.rs +++ b/crux-mir/lib/std/src/sys/wasi/args.rs @@ -1,23 +1,20 @@ +#![deny(unsafe_op_in_unsafe_fn)] + use crate::ffi::{CStr, OsStr, OsString}; -use crate::marker::PhantomData; +use crate::fmt; use crate::os::wasi::ffi::OsStrExt; use crate::vec; -pub unsafe fn init(_argc: isize, _argv: *const *const u8) {} - -pub unsafe fn cleanup() {} - pub struct Args { iter: vec::IntoIter, - _dont_send_or_sync_me: PhantomData<*mut ()>, } +impl !Send for Args {} +impl !Sync for Args {} + /// Returns the command line arguments pub fn args() -> Args { - Args { - iter: maybe_args().unwrap_or(Vec::new()).into_iter(), - _dont_send_or_sync_me: PhantomData, - } + Args { iter: maybe_args().unwrap_or(Vec::new()).into_iter() } } fn maybe_args() -> Option> { @@ -36,9 +33,9 @@ fn maybe_args() -> Option> { } } -impl Args { - pub fn inner_debug(&self) -> &[OsString] { - self.iter.as_slice() +impl fmt::Debug for Args { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.iter.as_slice().fmt(f) } } diff --git a/crux-mir/lib/std/src/sys/wasi/ext/ffi.rs b/crux-mir/lib/std/src/sys/wasi/ext/ffi.rs deleted file mode 100644 index f71f316d1..000000000 --- a/crux-mir/lib/std/src/sys/wasi/ext/ffi.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! WASI-specific extension to the primitives in the `std::ffi` module - -#![stable(feature = "rust1", since = "1.0.0")] - -#[stable(feature = "rust1", since = "1.0.0")] -pub use crate::sys_common::os_str_bytes::*; diff --git a/crux-mir/lib/std/src/sys/wasi/ext/io.rs b/crux-mir/lib/std/src/sys/wasi/ext/io.rs deleted file mode 100644 index e849400d6..000000000 --- a/crux-mir/lib/std/src/sys/wasi/ext/io.rs +++ /dev/null @@ -1,142 +0,0 @@ -//! WASI-specific extensions to general I/O primitives - -#![unstable(feature = "wasi_ext", issue = "none")] - -use crate::fs; -use crate::io; -use crate::net; -use crate::sys; -use crate::sys_common::{AsInner, FromInner, IntoInner}; - -/// Raw file descriptors. -pub type RawFd = u32; - -/// A trait to extract the raw WASI file descriptor from an underlying -/// object. -pub trait AsRawFd { - /// Extracts the raw file descriptor. - /// - /// This method does **not** pass ownership of the raw file descriptor - /// to the caller. The descriptor is only guaranteed to be valid while - /// the original object has not yet been destroyed. - fn as_raw_fd(&self) -> RawFd; -} - -/// A trait to express the ability to construct an object from a raw file -/// descriptor. -pub trait FromRawFd { - /// Constructs a new instance of `Self` from the given raw file - /// descriptor. - /// - /// This function **consumes ownership** of the specified file - /// descriptor. The returned object will take responsibility for closing - /// it when the object goes out of scope. - /// - /// This function is also unsafe as the primitives currently returned - /// have the contract that they are the sole owner of the file - /// descriptor they are wrapping. Usage of this function could - /// accidentally allow violating this contract which can cause memory - /// unsafety in code that relies on it being true. - unsafe fn from_raw_fd(fd: RawFd) -> Self; -} - -/// A trait to express the ability to consume an object and acquire ownership of -/// its raw file descriptor. -pub trait IntoRawFd { - /// Consumes this object, returning the raw underlying file descriptor. - /// - /// This function **transfers ownership** of the underlying file descriptor - /// to the caller. Callers are then the unique owners of the file descriptor - /// and must close the descriptor once it's no longer needed. - fn into_raw_fd(self) -> RawFd; -} - -impl AsRawFd for net::TcpStream { - fn as_raw_fd(&self) -> RawFd { - self.as_inner().fd().as_raw() - } -} - -impl FromRawFd for net::TcpStream { - unsafe fn from_raw_fd(fd: RawFd) -> net::TcpStream { - net::TcpStream::from_inner(sys::net::TcpStream::from_inner(fd)) - } -} - -impl IntoRawFd for net::TcpStream { - fn into_raw_fd(self) -> RawFd { - self.into_inner().into_fd().into_raw() - } -} - -impl AsRawFd for net::TcpListener { - fn as_raw_fd(&self) -> RawFd { - self.as_inner().fd().as_raw() - } -} - -impl FromRawFd for net::TcpListener { - unsafe fn from_raw_fd(fd: RawFd) -> net::TcpListener { - net::TcpListener::from_inner(sys::net::TcpListener::from_inner(fd)) - } -} - -impl IntoRawFd for net::TcpListener { - fn into_raw_fd(self) -> RawFd { - self.into_inner().into_fd().into_raw() - } -} - -impl AsRawFd for net::UdpSocket { - fn as_raw_fd(&self) -> RawFd { - self.as_inner().fd().as_raw() - } -} - -impl FromRawFd for net::UdpSocket { - unsafe fn from_raw_fd(fd: RawFd) -> net::UdpSocket { - net::UdpSocket::from_inner(sys::net::UdpSocket::from_inner(fd)) - } -} - -impl IntoRawFd for net::UdpSocket { - fn into_raw_fd(self) -> RawFd { - self.into_inner().into_fd().into_raw() - } -} - -impl AsRawFd for fs::File { - fn as_raw_fd(&self) -> RawFd { - self.as_inner().fd().as_raw() - } -} - -impl FromRawFd for fs::File { - unsafe fn from_raw_fd(fd: RawFd) -> fs::File { - fs::File::from_inner(sys::fs::File::from_inner(fd)) - } -} - -impl IntoRawFd for fs::File { - fn into_raw_fd(self) -> RawFd { - self.into_inner().into_fd().into_raw() - } -} - -impl AsRawFd for io::Stdin { - fn as_raw_fd(&self) -> RawFd { - sys::stdio::Stdin.as_raw_fd() - } -} - -impl AsRawFd for io::Stdout { - fn as_raw_fd(&self) -> RawFd { - sys::stdio::Stdout.as_raw_fd() - } -} - -impl AsRawFd for io::Stderr { - fn as_raw_fd(&self) -> RawFd { - sys::stdio::Stderr.as_raw_fd() - } -} diff --git a/crux-mir/lib/std/src/sys/wasi/ext/mod.rs b/crux-mir/lib/std/src/sys/wasi/ext/mod.rs deleted file mode 100644 index 5f8b1cbfa..000000000 --- a/crux-mir/lib/std/src/sys/wasi/ext/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -pub mod ffi; -pub mod fs; -pub mod io; - -/// A prelude for conveniently writing platform-specific code. -/// -/// Includes all extension traits, and some important type definitions. -#[stable(feature = "rust1", since = "1.0.0")] -pub mod prelude { - #[doc(no_inline)] - #[stable(feature = "rust1", since = "1.0.0")] - pub use crate::sys::ext::ffi::{OsStrExt, OsStringExt}; - #[doc(no_inline)] - #[stable(feature = "rust1", since = "1.0.0")] - pub use crate::sys::ext::fs::FileTypeExt; - #[doc(no_inline)] - #[stable(feature = "rust1", since = "1.0.0")] - pub use crate::sys::ext::fs::{DirEntryExt, FileExt, MetadataExt, OpenOptionsExt}; - #[doc(no_inline)] - #[stable(feature = "rust1", since = "1.0.0")] - pub use crate::sys::ext::io::{AsRawFd, FromRawFd, IntoRawFd}; -} diff --git a/crux-mir/lib/std/src/sys/wasi/fd.rs b/crux-mir/lib/std/src/sys/wasi/fd.rs index 8458ded5d..0b9c8e61d 100644 --- a/crux-mir/lib/std/src/sys/wasi/fd.rs +++ b/crux-mir/lib/std/src/sys/wasi/fd.rs @@ -1,13 +1,16 @@ +#![deny(unsafe_op_in_unsafe_fn)] #![allow(dead_code)] use super::err2io; use crate::io::{self, IoSlice, IoSliceMut, SeekFrom}; use crate::mem; use crate::net::Shutdown; +use crate::os::wasi::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; +use crate::sys_common::{AsInner, AsInnerMut, FromInner, IntoInner}; #[derive(Debug)] pub struct WasiFd { - fd: wasi::Fd, + fd: OwnedFd, } fn iovec<'a>(a: &'a mut [IoSliceMut<'_>]) -> &'a [wasi::Iovec] { @@ -25,38 +28,26 @@ fn ciovec<'a>(a: &'a [IoSlice<'_>]) -> &'a [wasi::Ciovec] { } impl WasiFd { - pub unsafe fn from_raw(fd: wasi::Fd) -> WasiFd { - WasiFd { fd } - } - - pub fn into_raw(self) -> wasi::Fd { - let ret = self.fd; - mem::forget(self); - ret - } - - pub fn as_raw(&self) -> wasi::Fd { - self.fd - } - pub fn datasync(&self) -> io::Result<()> { - unsafe { wasi::fd_datasync(self.fd).map_err(err2io) } + unsafe { wasi::fd_datasync(self.as_raw_fd() as wasi::Fd).map_err(err2io) } } pub fn pread(&self, bufs: &mut [IoSliceMut<'_>], offset: u64) -> io::Result { - unsafe { wasi::fd_pread(self.fd, iovec(bufs), offset).map_err(err2io) } + unsafe { wasi::fd_pread(self.as_raw_fd() as wasi::Fd, iovec(bufs), offset).map_err(err2io) } } pub fn pwrite(&self, bufs: &[IoSlice<'_>], offset: u64) -> io::Result { - unsafe { wasi::fd_pwrite(self.fd, ciovec(bufs), offset).map_err(err2io) } + unsafe { + wasi::fd_pwrite(self.as_raw_fd() as wasi::Fd, ciovec(bufs), offset).map_err(err2io) + } } pub fn read(&self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { - unsafe { wasi::fd_read(self.fd, iovec(bufs)).map_err(err2io) } + unsafe { wasi::fd_read(self.as_raw_fd() as wasi::Fd, iovec(bufs)).map_err(err2io) } } pub fn write(&self, bufs: &[IoSlice<'_>]) -> io::Result { - unsafe { wasi::fd_write(self.fd, ciovec(bufs)).map_err(err2io) } + unsafe { wasi::fd_write(self.as_raw_fd() as wasi::Fd, ciovec(bufs)).map_err(err2io) } } pub fn seek(&self, pos: SeekFrom) -> io::Result { @@ -65,37 +56,42 @@ impl WasiFd { SeekFrom::End(pos) => (wasi::WHENCE_END, pos), SeekFrom::Current(pos) => (wasi::WHENCE_CUR, pos), }; - unsafe { wasi::fd_seek(self.fd, offset, whence).map_err(err2io) } + unsafe { wasi::fd_seek(self.as_raw_fd() as wasi::Fd, offset, whence).map_err(err2io) } } pub fn tell(&self) -> io::Result { - unsafe { wasi::fd_tell(self.fd).map_err(err2io) } + unsafe { wasi::fd_tell(self.as_raw_fd() as wasi::Fd).map_err(err2io) } } // FIXME: __wasi_fd_fdstat_get pub fn set_flags(&self, flags: wasi::Fdflags) -> io::Result<()> { - unsafe { wasi::fd_fdstat_set_flags(self.fd, flags).map_err(err2io) } + unsafe { wasi::fd_fdstat_set_flags(self.as_raw_fd() as wasi::Fd, flags).map_err(err2io) } } pub fn set_rights(&self, base: wasi::Rights, inheriting: wasi::Rights) -> io::Result<()> { - unsafe { wasi::fd_fdstat_set_rights(self.fd, base, inheriting).map_err(err2io) } + unsafe { + wasi::fd_fdstat_set_rights(self.as_raw_fd() as wasi::Fd, base, inheriting) + .map_err(err2io) + } } pub fn sync(&self) -> io::Result<()> { - unsafe { wasi::fd_sync(self.fd).map_err(err2io) } + unsafe { wasi::fd_sync(self.as_raw_fd() as wasi::Fd).map_err(err2io) } } pub fn advise(&self, offset: u64, len: u64, advice: wasi::Advice) -> io::Result<()> { - unsafe { wasi::fd_advise(self.fd, offset, len, advice).map_err(err2io) } + unsafe { + wasi::fd_advise(self.as_raw_fd() as wasi::Fd, offset, len, advice).map_err(err2io) + } } pub fn allocate(&self, offset: u64, len: u64) -> io::Result<()> { - unsafe { wasi::fd_allocate(self.fd, offset, len).map_err(err2io) } + unsafe { wasi::fd_allocate(self.as_raw_fd() as wasi::Fd, offset, len).map_err(err2io) } } pub fn create_directory(&self, path: &str) -> io::Result<()> { - unsafe { wasi::path_create_directory(self.fd, path).map_err(err2io) } + unsafe { wasi::path_create_directory(self.as_raw_fd() as wasi::Fd, path).map_err(err2io) } } pub fn link( @@ -106,7 +102,14 @@ impl WasiFd { new_path: &str, ) -> io::Result<()> { unsafe { - wasi::path_link(self.fd, old_flags, old_path, new_fd.fd, new_path).map_err(err2io) + wasi::path_link( + self.as_raw_fd() as wasi::Fd, + old_flags, + old_path, + new_fd.as_raw_fd() as wasi::Fd, + new_path, + ) + .map_err(err2io) } } @@ -121,7 +124,7 @@ impl WasiFd { ) -> io::Result { unsafe { wasi::path_open( - self.fd, + self.as_raw_fd() as wasi::Fd, dirflags, path, oflags, @@ -129,25 +132,39 @@ impl WasiFd { fs_rights_inheriting, fs_flags, ) - .map(|fd| WasiFd::from_raw(fd)) + .map(|fd| WasiFd::from_raw_fd(fd as RawFd)) .map_err(err2io) } } pub fn readdir(&self, buf: &mut [u8], cookie: wasi::Dircookie) -> io::Result { - unsafe { wasi::fd_readdir(self.fd, buf.as_mut_ptr(), buf.len(), cookie).map_err(err2io) } + unsafe { + wasi::fd_readdir(self.as_raw_fd() as wasi::Fd, buf.as_mut_ptr(), buf.len(), cookie) + .map_err(err2io) + } } pub fn readlink(&self, path: &str, buf: &mut [u8]) -> io::Result { - unsafe { wasi::path_readlink(self.fd, path, buf.as_mut_ptr(), buf.len()).map_err(err2io) } + unsafe { + wasi::path_readlink(self.as_raw_fd() as wasi::Fd, path, buf.as_mut_ptr(), buf.len()) + .map_err(err2io) + } } pub fn rename(&self, old_path: &str, new_fd: &WasiFd, new_path: &str) -> io::Result<()> { - unsafe { wasi::path_rename(self.fd, old_path, new_fd.fd, new_path).map_err(err2io) } + unsafe { + wasi::path_rename( + self.as_raw_fd() as wasi::Fd, + old_path, + new_fd.as_raw_fd() as wasi::Fd, + new_path, + ) + .map_err(err2io) + } } pub fn filestat_get(&self) -> io::Result { - unsafe { wasi::fd_filestat_get(self.fd).map_err(err2io) } + unsafe { wasi::fd_filestat_get(self.as_raw_fd() as wasi::Fd).map_err(err2io) } } pub fn filestat_set_times( @@ -156,11 +173,14 @@ impl WasiFd { mtim: wasi::Timestamp, fstflags: wasi::Fstflags, ) -> io::Result<()> { - unsafe { wasi::fd_filestat_set_times(self.fd, atim, mtim, fstflags).map_err(err2io) } + unsafe { + wasi::fd_filestat_set_times(self.as_raw_fd() as wasi::Fd, atim, mtim, fstflags) + .map_err(err2io) + } } pub fn filestat_set_size(&self, size: u64) -> io::Result<()> { - unsafe { wasi::fd_filestat_set_size(self.fd, size).map_err(err2io) } + unsafe { wasi::fd_filestat_set_size(self.as_raw_fd() as wasi::Fd, size).map_err(err2io) } } pub fn path_filestat_get( @@ -168,7 +188,9 @@ impl WasiFd { flags: wasi::Lookupflags, path: &str, ) -> io::Result { - unsafe { wasi::path_filestat_get(self.fd, flags, path).map_err(err2io) } + unsafe { + wasi::path_filestat_get(self.as_raw_fd() as wasi::Fd, flags, path).map_err(err2io) + } } pub fn path_filestat_set_times( @@ -180,21 +202,34 @@ impl WasiFd { fstflags: wasi::Fstflags, ) -> io::Result<()> { unsafe { - wasi::path_filestat_set_times(self.fd, flags, path, atim, mtim, fstflags) - .map_err(err2io) + wasi::path_filestat_set_times( + self.as_raw_fd() as wasi::Fd, + flags, + path, + atim, + mtim, + fstflags, + ) + .map_err(err2io) } } pub fn symlink(&self, old_path: &str, new_path: &str) -> io::Result<()> { - unsafe { wasi::path_symlink(old_path, self.fd, new_path).map_err(err2io) } + unsafe { + wasi::path_symlink(old_path, self.as_raw_fd() as wasi::Fd, new_path).map_err(err2io) + } } pub fn unlink_file(&self, path: &str) -> io::Result<()> { - unsafe { wasi::path_unlink_file(self.fd, path).map_err(err2io) } + unsafe { wasi::path_unlink_file(self.as_raw_fd() as wasi::Fd, path).map_err(err2io) } } pub fn remove_directory(&self, path: &str) -> io::Result<()> { - unsafe { wasi::path_remove_directory(self.fd, path).map_err(err2io) } + unsafe { wasi::path_remove_directory(self.as_raw_fd() as wasi::Fd, path).map_err(err2io) } + } + + pub fn sock_accept(&self, flags: wasi::Fdflags) -> io::Result { + unsafe { wasi::sock_accept(self.as_raw_fd() as wasi::Fd, flags).map_err(err2io) } } pub fn sock_recv( @@ -202,11 +237,15 @@ impl WasiFd { ri_data: &mut [IoSliceMut<'_>], ri_flags: wasi::Riflags, ) -> io::Result<(usize, wasi::Roflags)> { - unsafe { wasi::sock_recv(self.fd, iovec(ri_data), ri_flags).map_err(err2io) } + unsafe { + wasi::sock_recv(self.as_raw_fd() as wasi::Fd, iovec(ri_data), ri_flags).map_err(err2io) + } } pub fn sock_send(&self, si_data: &[IoSlice<'_>], si_flags: wasi::Siflags) -> io::Result { - unsafe { wasi::sock_send(self.fd, ciovec(si_data), si_flags).map_err(err2io) } + unsafe { + wasi::sock_send(self.as_raw_fd() as wasi::Fd, ciovec(si_data), si_flags).map_err(err2io) + } } pub fn sock_shutdown(&self, how: Shutdown) -> io::Result<()> { @@ -215,14 +254,54 @@ impl WasiFd { Shutdown::Write => wasi::SDFLAGS_WR, Shutdown::Both => wasi::SDFLAGS_WR | wasi::SDFLAGS_RD, }; - unsafe { wasi::sock_shutdown(self.fd, how).map_err(err2io) } + unsafe { wasi::sock_shutdown(self.as_raw_fd() as wasi::Fd, how).map_err(err2io) } + } +} + +impl AsInner for WasiFd { + fn as_inner(&self) -> &OwnedFd { + &self.fd + } +} + +impl AsInnerMut for WasiFd { + fn as_inner_mut(&mut self) -> &mut OwnedFd { + &mut self.fd + } +} + +impl IntoInner for WasiFd { + fn into_inner(self) -> OwnedFd { + self.fd + } +} + +impl FromInner for WasiFd { + fn from_inner(owned_fd: OwnedFd) -> Self { + Self { fd: owned_fd } + } +} + +impl AsFd for WasiFd { + fn as_fd(&self) -> BorrowedFd<'_> { + self.fd.as_fd() + } +} + +impl AsRawFd for WasiFd { + fn as_raw_fd(&self) -> RawFd { + self.fd.as_raw_fd() + } +} + +impl IntoRawFd for WasiFd { + fn into_raw_fd(self) -> RawFd { + self.fd.into_raw_fd() } } -impl Drop for WasiFd { - fn drop(&mut self) { - // FIXME: can we handle the return code here even though we can't on - // unix? - let _ = unsafe { wasi::fd_close(self.fd) }; +impl FromRawFd for WasiFd { + unsafe fn from_raw_fd(raw_fd: RawFd) -> Self { + unsafe { Self { fd: FromRawFd::from_raw_fd(raw_fd) } } } } diff --git a/crux-mir/lib/std/src/sys/wasi/fs.rs b/crux-mir/lib/std/src/sys/wasi/fs.rs index a11f61fdd..d4866bbc3 100644 --- a/crux-mir/lib/std/src/sys/wasi/fs.rs +++ b/crux-mir/lib/std/src/sys/wasi/fs.rs @@ -1,18 +1,23 @@ -use crate::ffi::{CStr, CString, OsStr, OsString}; +#![deny(unsafe_op_in_unsafe_fn)] + +use super::fd::WasiFd; +use crate::ffi::{CStr, OsStr, OsString}; use crate::fmt; -use crate::io::{self, IoSlice, IoSliceMut, SeekFrom}; +use crate::io::{self, BorrowedCursor, IoSlice, IoSliceMut, SeekFrom}; use crate::iter; use crate::mem::{self, ManuallyDrop}; +use crate::os::raw::c_int; use crate::os::wasi::ffi::{OsStrExt, OsStringExt}; +use crate::os::wasi::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, RawFd}; use crate::path::{Path, PathBuf}; use crate::ptr; use crate::sync::Arc; -use crate::sys::fd::WasiFd; +use crate::sys::common::small_c_string::run_path_with_cstr; use crate::sys::time::SystemTime; use crate::sys::unsupported; -use crate::sys_common::FromInner; +use crate::sys_common::{AsInner, FromInner, IntoInner}; -pub use crate::sys_common::fs::remove_dir_all; +pub use crate::sys_common::fs::try_exists; pub struct File { fd: WasiFd, @@ -46,6 +51,7 @@ pub struct DirEntry { pub struct OpenOptions { read: bool, write: bool, + append: bool, dirflags: wasi::Lookupflags, fdflags: wasi::Fdflags, oflags: wasi::Oflags, @@ -58,6 +64,12 @@ pub struct FilePermissions { readonly: bool, } +#[derive(Copy, Clone, Debug, Default)] +pub struct FileTimes { + accessed: Option, + modified: Option, +} + #[derive(PartialEq, Eq, Hash, Debug, Copy, Clone)] pub struct FileType { bits: wasi::Filetype, @@ -107,6 +119,16 @@ impl FilePermissions { } } +impl FileTimes { + pub fn set_accessed(&mut self, t: SystemTime) { + self.accessed = Some(t); + } + + pub fn set_modified(&mut self, t: SystemTime) { + self.modified = Some(t); + } +} + impl FileType { pub fn is_dir(&self) -> bool { self.bits == wasi::FILETYPE_DIRECTORY @@ -125,9 +147,21 @@ impl FileType { } } +impl ReadDir { + fn new(dir: File, root: PathBuf) -> ReadDir { + ReadDir { + cookie: Some(0), + buf: vec![0; 128], + offset: 0, + cap: 0, + inner: Arc::new(ReadDirInner { dir, root }), + } + } +} + impl fmt::Debug for ReadDir { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("ReadDir").finish() + f.debug_struct("ReadDir").finish_non_exhaustive() } } @@ -270,8 +304,9 @@ impl OpenOptions { } } - pub fn append(&mut self, set: bool) { - self.fdflag(wasi::FDFLAGS_APPEND, set); + pub fn append(&mut self, append: bool) { + self.append = append; + self.fdflag(wasi::FDFLAGS_APPEND, append); } pub fn dsync(&mut self, set: bool) { @@ -321,7 +356,7 @@ impl OpenOptions { base |= wasi::RIGHTS_FD_READ; base |= wasi::RIGHTS_FD_READDIR; } - if self.write { + if self.write || self.append { base |= wasi::RIGHTS_FD_WRITE; base |= wasi::RIGHTS_FD_DATASYNC; base |= wasi::RIGHTS_FD_ALLOCATE; @@ -331,6 +366,7 @@ impl OpenOptions { // FIXME: some of these should probably be read-only or write-only... base |= wasi::RIGHTS_FD_ADVISE; base |= wasi::RIGHTS_FD_FDSTAT_SET_FLAGS; + base |= wasi::RIGHTS_FD_FILESTAT_GET; base |= wasi::RIGHTS_FD_FILESTAT_SET_TIMES; base |= wasi::RIGHTS_FD_SEEK; base |= wasi::RIGHTS_FD_SYNC; @@ -399,6 +435,15 @@ impl File { self.fd.read(bufs) } + #[inline] + pub fn is_read_vectored(&self) -> bool { + true + } + + pub fn read_buf(&self, cursor: BorrowedCursor<'_>) -> io::Result<()> { + crate::io::default_read_buf(|buf| self.read(buf), cursor) + } + pub fn write(&self, buf: &[u8]) -> io::Result { self.write_vectored(&[IoSlice::new(buf)]) } @@ -407,6 +452,11 @@ impl File { self.fd.write(bufs) } + #[inline] + pub fn is_write_vectored(&self) -> bool { + true + } + pub fn flush(&self) -> io::Result<()> { Ok(()) } @@ -426,22 +476,66 @@ impl File { unsupported() } - pub fn fd(&self) -> &WasiFd { + pub fn set_times(&self, times: FileTimes) -> io::Result<()> { + let to_timestamp = |time: Option| { + match time { + Some(time) if let Some(ts) = time.to_wasi_timestamp() => Ok(ts), + Some(_) => Err(io::const_io_error!(io::ErrorKind::InvalidInput, "timestamp is too large to set as a file time")), + None => Ok(0), + } + }; + self.fd.filestat_set_times( + to_timestamp(times.accessed)?, + to_timestamp(times.modified)?, + times.accessed.map_or(0, |_| wasi::FSTFLAGS_ATIM) + | times.modified.map_or(0, |_| wasi::FSTFLAGS_MTIM), + ) + } + + pub fn read_link(&self, file: &Path) -> io::Result { + read_link(&self.fd, file) + } +} + +impl AsInner for File { + fn as_inner(&self) -> &WasiFd { &self.fd } +} - pub fn into_fd(self) -> WasiFd { +impl IntoInner for File { + fn into_inner(self) -> WasiFd { self.fd } +} - pub fn read_link(&self, file: &Path) -> io::Result { - read_link(&self.fd, file) +impl FromInner for File { + fn from_inner(fd: WasiFd) -> File { + File { fd } + } +} + +impl AsFd for File { + fn as_fd(&self) -> BorrowedFd<'_> { + self.fd.as_fd() } } -impl FromInner for File { - fn from_inner(fd: u32) -> File { - unsafe { File { fd: WasiFd::from_raw(fd) } } +impl AsRawFd for File { + fn as_raw_fd(&self) -> RawFd { + self.fd.as_raw_fd() + } +} + +impl IntoRawFd for File { + fn into_raw_fd(self) -> RawFd { + self.fd.into_raw_fd() + } +} + +impl FromRawFd for File { + unsafe fn from_raw_fd(raw_fd: RawFd) -> Self { + unsafe { Self { fd: FromRawFd::from_raw_fd(raw_fd) } } } } @@ -458,7 +552,7 @@ impl DirBuilder { impl fmt::Debug for File { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("File").field("fd", &self.fd.as_raw()).finish() + f.debug_struct("File").field("fd", &self.as_raw_fd()).finish() } } @@ -467,13 +561,7 @@ pub fn readdir(p: &Path) -> io::Result { opts.directory(true); opts.read(true); let dir = File::open(p, &opts)?; - Ok(ReadDir { - cookie: Some(0), - buf: vec![0; 128], - offset: 0, - cap: 0, - inner: Arc::new(ReadDirInner { dir, root: p.to_path_buf() }), - }) + Ok(ReadDir::new(dir, p.to_path_buf())) } pub fn unlink(p: &Path) -> io::Result<()> { @@ -534,20 +622,16 @@ fn read_link(fd: &WasiFd, file: &Path) -> io::Result { } } -pub fn symlink(src: &Path, dst: &Path) -> io::Result<()> { - let (dst, dst_file) = open_parent(dst)?; - dst.symlink(osstr2str(src.as_ref())?, osstr2str(dst_file.as_ref())?) +pub fn symlink(original: &Path, link: &Path) -> io::Result<()> { + let (link, link_file) = open_parent(link)?; + link.symlink(osstr2str(original.as_ref())?, osstr2str(link_file.as_ref())?) } -pub fn link(src: &Path, dst: &Path) -> io::Result<()> { - let (src, src_file) = open_parent(src)?; - let (dst, dst_file) = open_parent(dst)?; - src.link( - wasi::LOOKUPFLAGS_SYMLINK_FOLLOW, - osstr2str(src_file.as_ref())?, - &dst, - osstr2str(dst_file.as_ref())?, - ) +pub fn link(original: &Path, link: &Path) -> io::Result<()> { + let (original, original_file) = open_parent(original)?; + let (link, link_file) = open_parent(link)?; + // Pass 0 as the flags argument, meaning don't follow symlinks. + original.link(0, osstr2str(original_file.as_ref())?, &link, osstr2str(link_file.as_ref())?) } pub fn stat(p: &Path) -> io::Result { @@ -587,14 +671,14 @@ fn open_at(fd: &WasiFd, path: &Path, opts: &OpenOptions) -> io::Result { /// /// WASI has no fundamental capability to do this. All syscalls and operations /// are relative to already-open file descriptors. The C library, however, -/// manages a map of preopened file descriptors to their path, and then the C +/// manages a map of pre-opened file descriptors to their path, and then the C /// library provides an API to look at this. In other words, when you want to /// open a path `p`, you have to find a previously opened file descriptor in a /// global table and then see if `p` is relative to that file descriptor. /// /// This function, if successful, will return two items: /// -/// * The first is a `ManuallyDrop`. This represents a preopened file +/// * The first is a `ManuallyDrop`. This represents a pre-opened file /// descriptor which we don't have ownership of, but we can use. You shouldn't /// actually drop the `fd`. /// @@ -609,42 +693,59 @@ fn open_at(fd: &WasiFd, path: &Path, opts: &OpenOptions) -> io::Result { /// appropriate rights for performing `rights` actions. /// /// Note that this can fail if `p` doesn't look like it can be opened relative -/// to any preopened file descriptor. +/// to any pre-opened file descriptor. fn open_parent(p: &Path) -> io::Result<(ManuallyDrop, PathBuf)> { - let p = CString::new(p.as_os_str().as_bytes())?; - unsafe { - let mut ret = ptr::null(); - let fd = __wasilibc_find_relpath(p.as_ptr(), &mut ret); - if fd == -1 { - let msg = format!( - "failed to find a preopened file descriptor \ - through which {:?} could be opened", - p - ); - return Err(io::Error::new(io::ErrorKind::Other, msg)); - } - let path = Path::new(OsStr::from_bytes(CStr::from_ptr(ret).to_bytes())); - - // FIXME: right now `path` is a pointer into `p`, the `CString` above. - // When we return `p` is deallocated and we can't use it, so we need to - // currently separately allocate `path`. If this becomes an issue though - // we should probably turn this into a closure-taking interface or take - // `&CString` and then pass off `&Path` tied to the same lifetime. - let path = path.to_path_buf(); + run_path_with_cstr(p, |p| { + let mut buf = Vec::::with_capacity(512); + loop { + unsafe { + let mut relative_path = buf.as_ptr().cast(); + let mut abs_prefix = ptr::null(); + let fd = __wasilibc_find_relpath( + p.as_ptr(), + &mut abs_prefix, + &mut relative_path, + buf.capacity(), + ); + if fd == -1 { + if io::Error::last_os_error().raw_os_error() == Some(libc::ENOMEM) { + // Trigger the internal buffer resizing logic of `Vec` by requiring + // more space than the current capacity. + let cap = buf.capacity(); + buf.set_len(cap); + buf.reserve(1); + continue; + } + let msg = format!( + "failed to find a pre-opened file descriptor \ + through which {:?} could be opened", + p + ); + return Err(io::Error::new(io::ErrorKind::Uncategorized, msg)); + } + let relative = CStr::from_ptr(relative_path).to_bytes().to_vec(); - return Ok((ManuallyDrop::new(WasiFd::from_raw(fd as u32)), path)); - } + return Ok(( + ManuallyDrop::new(WasiFd::from_raw_fd(fd as c_int)), + PathBuf::from(OsString::from_vec(relative)), + )); + } + } - extern "C" { - pub fn __wasilibc_find_relpath( - path: *const libc::c_char, - relative_path: *mut *const libc::c_char, - ) -> libc::c_int; - } + extern "C" { + pub fn __wasilibc_find_relpath( + path: *const libc::c_char, + abs_prefix: *mut *const libc::c_char, + relative_path: *mut *const libc::c_char, + relative_path_len: libc::size_t, + ) -> libc::c_int; + } + }) } pub fn osstr2str(f: &OsStr) -> io::Result<&str> { - f.to_str().ok_or_else(|| io::Error::new(io::ErrorKind::Other, "input must be utf-8")) + f.to_str() + .ok_or_else(|| io::const_io_error!(io::ErrorKind::Uncategorized, "input must be utf-8")) } pub fn copy(from: &Path, to: &Path) -> io::Result { @@ -655,3 +756,52 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { io::copy(&mut reader, &mut writer) } + +pub fn remove_dir_all(path: &Path) -> io::Result<()> { + let (parent, path) = open_parent(path)?; + remove_dir_all_recursive(&parent, &path) +} + +fn remove_dir_all_recursive(parent: &WasiFd, path: &Path) -> io::Result<()> { + // Open up a file descriptor for the directory itself. Note that we don't + // follow symlinks here and we specifically open directories. + // + // At the root invocation of this function this will correctly handle + // symlinks passed to the top-level `remove_dir_all`. At the recursive + // level this will double-check that after the `readdir` call deduced this + // was a directory it's still a directory by the time we open it up. + // + // If the opened file was actually a symlink then the symlink is deleted, + // not the directory recursively. + let mut opts = OpenOptions::new(); + opts.lookup_flags(0); + opts.directory(true); + opts.read(true); + let fd = open_at(parent, path, &opts)?; + if fd.file_attr()?.file_type().is_symlink() { + return parent.unlink_file(osstr2str(path.as_ref())?); + } + + // this "root" is only used by `DirEntry::path` which we don't use below so + // it's ok for this to be a bogus value + let dummy_root = PathBuf::new(); + + // Iterate over all the entries in this directory, and travel recursively if + // necessary + for entry in ReadDir::new(fd, dummy_root) { + let entry = entry?; + let path = crate::str::from_utf8(&entry.name).map_err(|_| { + io::const_io_error!(io::ErrorKind::Uncategorized, "invalid utf-8 file name found") + })?; + + if entry.file_type()?.is_dir() { + remove_dir_all_recursive(&entry.inner.dir.fd, path.as_ref())?; + } else { + entry.inner.dir.fd.unlink_file(path)?; + } + } + + // Once all this directory's contents are deleted it should be safe to + // delete the directory tiself. + parent.remove_directory(osstr2str(path.as_ref())?) +} diff --git a/crux-mir/lib/std/src/sys/wasi/io.rs b/crux-mir/lib/std/src/sys/wasi/io.rs index 0ad2e1528..2cd45df88 100644 --- a/crux-mir/lib/std/src/sys/wasi/io.rs +++ b/crux-mir/lib/std/src/sys/wasi/io.rs @@ -1,4 +1,7 @@ +#![deny(unsafe_op_in_unsafe_fn)] + use crate::marker::PhantomData; +use crate::os::fd::{AsFd, AsRawFd}; use crate::slice; #[derive(Copy, Clone)] @@ -69,3 +72,8 @@ impl<'a> IoSliceMut<'a> { unsafe { slice::from_raw_parts_mut(self.vec.buf as *mut u8, self.vec.buf_len) } } } + +pub fn is_terminal(fd: &impl AsFd) -> bool { + let fd = fd.as_fd(); + unsafe { libc::isatty(fd.as_raw_fd()) != 0 } +} diff --git a/crux-mir/lib/std/src/sys/wasi/mod.rs b/crux-mir/lib/std/src/sys/wasi/mod.rs index 241d499ca..1dc3f2b20 100644 --- a/crux-mir/lib/std/src/sys/wasi/mod.rs +++ b/crux-mir/lib/std/src/sys/wasi/mod.rs @@ -16,92 +16,78 @@ use crate::io as std_io; use crate::mem; -use crate::os::raw::c_char; +#[path = "../unix/alloc.rs"] pub mod alloc; pub mod args; -#[path = "../wasm/cmath.rs"] +#[path = "../unix/cmath.rs"] pub mod cmath; -#[path = "../wasm/condvar.rs"] -pub mod condvar; pub mod env; pub mod fd; pub mod fs; +#[allow(unused)] +#[path = "../wasm/atomics/futex.rs"] +pub mod futex; pub mod io; -#[path = "../wasm/memchr.rs"] -pub mod memchr; -#[path = "../wasm/mutex.rs"] -pub mod mutex; +#[path = "../unsupported/locks/mod.rs"] +pub mod locks; pub mod net; +#[path = "../unsupported/once.rs"] +pub mod once; pub mod os; -pub use crate::sys_common::os_str_bytes as os_str; -pub mod ext; -#[path = "../wasm/fast_thread_local.rs"] -pub mod fast_thread_local; +#[path = "../unix/os_str.rs"] +pub mod os_str; +#[path = "../unix/path.rs"] pub mod path; +#[path = "../unsupported/pipe.rs"] pub mod pipe; +#[path = "../unsupported/process.rs"] pub mod process; -#[path = "../wasm/rwlock.rs"] -pub mod rwlock; -#[path = "../wasm/stack_overflow.rs"] -pub mod stack_overflow; pub mod stdio; pub mod thread; -#[path = "../wasm/thread_local.rs"] -pub mod thread_local; +#[path = "../unsupported/thread_local_dtor.rs"] +pub mod thread_local_dtor; +#[path = "../unsupported/thread_local_key.rs"] +pub mod thread_local_key; pub mod time; -#[cfg(not(test))] -pub fn init() {} - -pub fn unsupported() -> std_io::Result { - Err(unsupported_err()) -} - -pub fn unsupported_err() -> std_io::Error { - std_io::Error::new(std_io::ErrorKind::Other, "operation not supported on wasm yet") -} +#[path = "../unsupported/common.rs"] +#[deny(unsafe_op_in_unsafe_fn)] +#[allow(unused)] +mod common; +pub use common::*; pub fn decode_error_kind(errno: i32) -> std_io::ErrorKind { use std_io::ErrorKind::*; - if errno > u16::max_value() as i32 || errno < 0 { - return Other; + if errno > u16::MAX as i32 || errno < 0 { + return Uncategorized; } - match errno as u16 { - wasi::ERRNO_CONNREFUSED => ConnectionRefused, - wasi::ERRNO_CONNRESET => ConnectionReset, - wasi::ERRNO_PERM | wasi::ERRNO_ACCES => PermissionDenied, - wasi::ERRNO_PIPE => BrokenPipe, - wasi::ERRNO_NOTCONN => NotConnected, - wasi::ERRNO_CONNABORTED => ConnectionAborted, - wasi::ERRNO_ADDRNOTAVAIL => AddrNotAvailable, - wasi::ERRNO_ADDRINUSE => AddrInUse, - wasi::ERRNO_NOENT => NotFound, - wasi::ERRNO_INTR => Interrupted, - wasi::ERRNO_INVAL => InvalidInput, - wasi::ERRNO_TIMEDOUT => TimedOut, - wasi::ERRNO_EXIST => AlreadyExists, - wasi::ERRNO_AGAIN => WouldBlock, - _ => Other, - } -} - -// This enum is used as the storage for a bunch of types which can't actually -// exist. -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] -pub enum Void {} -pub unsafe fn strlen(mut s: *const c_char) -> usize { - let mut n = 0; - while *s != 0 { - n += 1; - s = s.offset(1); + match errno { + e if e == wasi::ERRNO_CONNREFUSED.raw().into() => ConnectionRefused, + e if e == wasi::ERRNO_CONNRESET.raw().into() => ConnectionReset, + e if e == wasi::ERRNO_PERM.raw().into() || e == wasi::ERRNO_ACCES.raw().into() => { + PermissionDenied + } + e if e == wasi::ERRNO_PIPE.raw().into() => BrokenPipe, + e if e == wasi::ERRNO_NOTCONN.raw().into() => NotConnected, + e if e == wasi::ERRNO_CONNABORTED.raw().into() => ConnectionAborted, + e if e == wasi::ERRNO_ADDRNOTAVAIL.raw().into() => AddrNotAvailable, + e if e == wasi::ERRNO_ADDRINUSE.raw().into() => AddrInUse, + e if e == wasi::ERRNO_NOENT.raw().into() => NotFound, + e if e == wasi::ERRNO_INTR.raw().into() => Interrupted, + e if e == wasi::ERRNO_INVAL.raw().into() => InvalidInput, + e if e == wasi::ERRNO_TIMEDOUT.raw().into() => TimedOut, + e if e == wasi::ERRNO_EXIST.raw().into() => AlreadyExists, + e if e == wasi::ERRNO_AGAIN.raw().into() => WouldBlock, + e if e == wasi::ERRNO_NOSYS.raw().into() => Unsupported, + e if e == wasi::ERRNO_NOMEM.raw().into() => OutOfMemory, + _ => Uncategorized, } - return n; } -pub unsafe fn abort_internal() -> ! { - libc::abort() +pub fn abort_internal() -> ! { + unsafe { libc::abort() } } pub fn hashmap_random_keys() -> (u64, u64) { @@ -114,6 +100,6 @@ pub fn hashmap_random_keys() -> (u64, u64) { return ret; } -fn err2io(err: wasi::Error) -> std_io::Error { - std_io::Error::from_raw_os_error(err.raw_error().into()) +fn err2io(err: wasi::Errno) -> std_io::Error { + std_io::Error::from_raw_os_error(err.raw().into()) } diff --git a/crux-mir/lib/std/src/sys/wasi/net.rs b/crux-mir/lib/std/src/sys/wasi/net.rs index 8a69028ff..cf4ebba1a 100644 --- a/crux-mir/lib/std/src/sys/wasi/net.rs +++ b/crux-mir/lib/std/src/sys/wasi/net.rs @@ -1,14 +1,61 @@ -use crate::convert::TryFrom; +#![deny(unsafe_op_in_unsafe_fn)] + +use super::err2io; +use super::fd::WasiFd; use crate::fmt; use crate::io::{self, IoSlice, IoSliceMut}; use crate::net::{Ipv4Addr, Ipv6Addr, Shutdown, SocketAddr}; -use crate::sys::fd::WasiFd; -use crate::sys::{unsupported, Void}; -use crate::sys_common::FromInner; +use crate::os::wasi::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, RawFd}; +use crate::sys::unsupported; +use crate::sys_common::{AsInner, FromInner, IntoInner}; use crate::time::Duration; +pub struct Socket(WasiFd); + pub struct TcpStream { - fd: WasiFd, + inner: Socket, +} + +impl AsInner for Socket { + fn as_inner(&self) -> &WasiFd { + &self.0 + } +} + +impl IntoInner for Socket { + fn into_inner(self) -> WasiFd { + self.0 + } +} + +impl FromInner for Socket { + fn from_inner(inner: WasiFd) -> Socket { + Socket(inner) + } +} + +impl AsFd for Socket { + fn as_fd(&self) -> BorrowedFd<'_> { + self.0.as_fd() + } +} + +impl AsRawFd for Socket { + fn as_raw_fd(&self) -> RawFd { + self.0.as_raw_fd() + } +} + +impl IntoRawFd for Socket { + fn into_raw_fd(self) -> RawFd { + self.0.into_raw_fd() + } +} + +impl FromRawFd for Socket { + unsafe fn from_raw_fd(raw_fd: RawFd) -> Self { + unsafe { Self(FromRawFd::from_raw_fd(raw_fd)) } + } } impl TcpStream { @@ -40,20 +87,28 @@ impl TcpStream { unsupported() } - pub fn read(&self, _: &mut [u8]) -> io::Result { - unsupported() + pub fn read(&self, buf: &mut [u8]) -> io::Result { + self.read_vectored(&mut [IoSliceMut::new(buf)]) } - pub fn read_vectored(&self, _: &mut [IoSliceMut<'_>]) -> io::Result { - unsupported() + pub fn read_vectored(&self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { + self.socket().as_inner().read(bufs) } - pub fn write(&self, _: &[u8]) -> io::Result { - unsupported() + pub fn is_read_vectored(&self) -> bool { + true } - pub fn write_vectored(&self, _: &[IoSlice<'_>]) -> io::Result { - unsupported() + pub fn write(&self, buf: &[u8]) -> io::Result { + self.write_vectored(&[IoSlice::new(buf)]) + } + + pub fn write_vectored(&self, bufs: &[IoSlice<'_>]) -> io::Result { + self.socket().as_inner().write(bufs) + } + + pub fn is_write_vectored(&self) -> bool { + true } pub fn peer_addr(&self) -> io::Result { @@ -64,14 +119,28 @@ impl TcpStream { unsupported() } - pub fn shutdown(&self, _: Shutdown) -> io::Result<()> { - unsupported() + pub fn shutdown(&self, how: Shutdown) -> io::Result<()> { + let wasi_how = match how { + Shutdown::Read => wasi::SDFLAGS_RD, + Shutdown::Write => wasi::SDFLAGS_WR, + Shutdown::Both => wasi::SDFLAGS_RD | wasi::SDFLAGS_WR, + }; + + unsafe { wasi::sock_shutdown(self.socket().as_raw_fd() as _, wasi_how).map_err(err2io) } } pub fn duplicate(&self) -> io::Result { unsupported() } + pub fn set_linger(&self, _: Option) -> io::Result<()> { + unsupported() + } + + pub fn linger(&self) -> io::Result> { + unsupported() + } + pub fn set_nodelay(&self, _: bool) -> io::Result<()> { unsupported() } @@ -92,33 +161,48 @@ impl TcpStream { unsupported() } - pub fn set_nonblocking(&self, _: bool) -> io::Result<()> { - unsupported() + pub fn set_nonblocking(&self, state: bool) -> io::Result<()> { + let fdstat = unsafe { + wasi::fd_fdstat_get(self.socket().as_inner().as_raw_fd() as wasi::Fd).map_err(err2io)? + }; + + let mut flags = fdstat.fs_flags; + + if state { + flags |= wasi::FDFLAGS_NONBLOCK; + } else { + flags &= !wasi::FDFLAGS_NONBLOCK; + } + + unsafe { + wasi::fd_fdstat_set_flags(self.socket().as_inner().as_raw_fd() as wasi::Fd, flags) + .map_err(err2io) + } } - pub fn fd(&self) -> &WasiFd { - &self.fd + pub fn socket(&self) -> &Socket { + &self.inner } - pub fn into_fd(self) -> WasiFd { - self.fd + pub fn into_socket(self) -> Socket { + self.inner } } -impl FromInner for TcpStream { - fn from_inner(fd: u32) -> TcpStream { - unsafe { TcpStream { fd: WasiFd::from_raw(fd) } } +impl FromInner for TcpStream { + fn from_inner(socket: Socket) -> TcpStream { + TcpStream { inner: socket } } } impl fmt::Debug for TcpStream { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("TcpStream").field("fd", &self.fd.as_raw()).finish() + f.debug_struct("TcpStream").field("fd", &self.inner.as_raw_fd()).finish() } } pub struct TcpListener { - fd: WasiFd, + inner: Socket, } impl TcpListener { @@ -131,7 +215,16 @@ impl TcpListener { } pub fn accept(&self) -> io::Result<(TcpStream, SocketAddr)> { - unsupported() + let fd = unsafe { + wasi::sock_accept(self.as_inner().as_inner().as_raw_fd() as _, 0).map_err(err2io)? + }; + + Ok(( + TcpStream::from_inner(unsafe { Socket::from_raw_fd(fd as _) }), + // WASI has no concept of SocketAddr yet + // return an unspecified IPv4Addr + SocketAddr::new(Ipv4Addr::UNSPECIFIED.into(), 0), + )) } pub fn duplicate(&self) -> io::Result { @@ -158,33 +251,60 @@ impl TcpListener { unsupported() } - pub fn set_nonblocking(&self, _: bool) -> io::Result<()> { - unsupported() + pub fn set_nonblocking(&self, state: bool) -> io::Result<()> { + let fdstat = unsafe { + wasi::fd_fdstat_get(self.socket().as_inner().as_raw_fd() as wasi::Fd).map_err(err2io)? + }; + + let mut flags = fdstat.fs_flags; + + if state { + flags |= wasi::FDFLAGS_NONBLOCK; + } else { + flags &= !wasi::FDFLAGS_NONBLOCK; + } + + unsafe { + wasi::fd_fdstat_set_flags(self.socket().as_inner().as_raw_fd() as wasi::Fd, flags) + .map_err(err2io) + } + } + + pub fn socket(&self) -> &Socket { + &self.inner + } + + pub fn into_socket(self) -> Socket { + self.inner } +} - pub fn fd(&self) -> &WasiFd { - &self.fd +impl AsInner for TcpListener { + fn as_inner(&self) -> &Socket { + &self.inner } +} - pub fn into_fd(self) -> WasiFd { - self.fd +impl IntoInner for TcpListener { + fn into_inner(self) -> Socket { + self.inner } } -impl FromInner for TcpListener { - fn from_inner(fd: u32) -> TcpListener { - unsafe { TcpListener { fd: WasiFd::from_raw(fd) } } +impl FromInner for TcpListener { + fn from_inner(inner: Socket) -> TcpListener { + TcpListener { inner } } } impl fmt::Debug for TcpListener { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("TcpListener").field("fd", &self.fd.as_raw()).finish() + f.debug_struct("TcpListener").field("fd", &self.inner.as_raw_fd()).finish() } } pub struct UdpSocket { - fd: WasiFd, + inner: Socket, } impl UdpSocket { @@ -312,39 +432,51 @@ impl UdpSocket { unsupported() } - pub fn fd(&self) -> &WasiFd { - &self.fd + pub fn socket(&self) -> &Socket { + &self.inner + } + + pub fn into_socket(self) -> Socket { + self.inner + } +} + +impl AsInner for UdpSocket { + fn as_inner(&self) -> &Socket { + &self.inner } +} - pub fn into_fd(self) -> WasiFd { - self.fd +impl IntoInner for UdpSocket { + fn into_inner(self) -> Socket { + self.inner } } -impl FromInner for UdpSocket { - fn from_inner(fd: u32) -> UdpSocket { - unsafe { UdpSocket { fd: WasiFd::from_raw(fd) } } +impl FromInner for UdpSocket { + fn from_inner(inner: Socket) -> UdpSocket { + UdpSocket { inner } } } impl fmt::Debug for UdpSocket { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("UdpSocket").field("fd", &self.fd.as_raw()).finish() + f.debug_struct("UdpSocket").field("fd", &self.inner.as_raw_fd()).finish() } } -pub struct LookupHost(Void); +pub struct LookupHost(!); impl LookupHost { pub fn port(&self) -> u16 { - match self.0 {} + self.0 } } impl Iterator for LookupHost { type Item = SocketAddr; fn next(&mut self) -> Option { - match self.0 {} + self.0 } } @@ -398,6 +530,4 @@ pub mod netc { #[derive(Copy, Clone)] pub struct sockaddr {} - - pub type socklen_t = usize; } diff --git a/crux-mir/lib/std/src/sys/wasi/os.rs b/crux-mir/lib/std/src/sys/wasi/os.rs index 8052c0aa8..f5513e999 100644 --- a/crux-mir/lib/std/src/sys/wasi/os.rs +++ b/crux-mir/lib/std/src/sys/wasi/os.rs @@ -1,20 +1,49 @@ -use crate::any::Any; +#![deny(unsafe_op_in_unsafe_fn)] + use crate::error::Error as StdError; -use crate::ffi::{CStr, CString, OsStr, OsString}; +use crate::ffi::{CStr, OsStr, OsString}; use crate::fmt; use crate::io; use crate::marker::PhantomData; +use crate::ops::Drop; use crate::os::wasi::prelude::*; use crate::path::{self, PathBuf}; use crate::str; +use crate::sys::common::small_c_string::{run_path_with_cstr, run_with_cstr}; use crate::sys::memchr; -use crate::sys::{unsupported, Void}; +use crate::sys::unsupported; use crate::vec; -#[cfg(not(target_feature = "atomics"))] -pub unsafe fn env_lock() -> impl Any { - // No need for a lock if we're single-threaded, but this function will need - // to get implemented for multi-threaded scenarios +// Add a few symbols not in upstream `libc` just yet. +mod libc { + pub use libc::*; + + extern "C" { + pub fn getcwd(buf: *mut c_char, size: size_t) -> *mut c_char; + pub fn chdir(dir: *const c_char) -> c_int; + } +} + +cfg_if::cfg_if! { + if #[cfg(target_feature = "atomics")] { + // Access to the environment must be protected by a lock in multi-threaded scenarios. + use crate::sync::{PoisonError, RwLock}; + static ENV_LOCK: RwLock<()> = RwLock::new(()); + pub fn env_read_lock() -> impl Drop { + ENV_LOCK.read().unwrap_or_else(PoisonError::into_inner) + } + pub fn env_write_lock() -> impl Drop { + ENV_LOCK.write().unwrap_or_else(PoisonError::into_inner) + } + } else { + // No need for a lock if we are single-threaded. + pub fn env_read_lock() -> impl Drop { + Box::new(()) + } + pub fn env_write_lock() -> impl Drop { + Box::new(()) + } + } } pub fn errno() -> i32 { @@ -39,14 +68,40 @@ pub fn error_string(errno: i32) -> String { } pub fn getcwd() -> io::Result { - unsupported() + let mut buf = Vec::with_capacity(512); + loop { + unsafe { + let ptr = buf.as_mut_ptr() as *mut libc::c_char; + if !libc::getcwd(ptr, buf.capacity()).is_null() { + let len = CStr::from_ptr(buf.as_ptr() as *const libc::c_char).to_bytes().len(); + buf.set_len(len); + buf.shrink_to_fit(); + return Ok(PathBuf::from(OsString::from_vec(buf))); + } else { + let error = io::Error::last_os_error(); + if error.raw_os_error() != Some(libc::ERANGE) { + return Err(error); + } + } + + // Trigger the internal buffer resizing logic of `Vec` by requiring + // more space than the current capacity. + let cap = buf.capacity(); + buf.set_len(cap); + buf.reserve(1); + } + } } -pub fn chdir(_: &path::Path) -> io::Result<()> { - unsupported() +pub fn chdir(p: &path::Path) -> io::Result<()> { + let result = run_path_with_cstr(p, |p| unsafe { Ok(libc::chdir(p.as_ptr())) })?; + match result == (0 as libc::c_int) { + true => Ok(()), + false => Err(io::Error::last_os_error()), + } } -pub struct SplitPaths<'a>(&'a Void); +pub struct SplitPaths<'a>(!, PhantomData<&'a ()>); pub fn split_paths(_unparsed: &OsStr) -> SplitPaths<'_> { panic!("unsupported") @@ -55,7 +110,7 @@ pub fn split_paths(_unparsed: &OsStr) -> SplitPaths<'_> { impl<'a> Iterator for SplitPaths<'a> { type Item = PathBuf; fn next(&mut self) -> Option { - match *self.0 {} + self.0 } } @@ -88,9 +143,11 @@ pub fn current_exe() -> io::Result { } pub struct Env { iter: vec::IntoIter<(OsString, OsString)>, - _dont_send_or_sync_me: PhantomData<*mut ()>, } +impl !Send for Env {} +impl !Sync for Env {} + impl Iterator for Env { type Item = (OsString, OsString); fn next(&mut self) -> Option<(OsString, OsString)> { @@ -103,7 +160,7 @@ impl Iterator for Env { pub fn env() -> Env { unsafe { - let _guard = env_lock(); + let _guard = env_read_lock(); let mut environ = libc::environ; let mut result = Vec::new(); if !environ.is_null() { @@ -114,7 +171,7 @@ pub fn env() -> Env { environ = environ.add(1); } } - return Env { iter: result.into_iter(), _dont_send_or_sync_me: PhantomData }; + return Env { iter: result.into_iter() }; } // See src/libstd/sys/unix/os.rs, same as that @@ -132,37 +189,33 @@ pub fn env() -> Env { } } -pub fn getenv(k: &OsStr) -> io::Result> { - let k = CString::new(k.as_bytes())?; - unsafe { - let _guard = env_lock(); - let s = libc::getenv(k.as_ptr()) as *const libc::c_char; - let ret = if s.is_null() { - None - } else { - Some(OsStringExt::from_vec(CStr::from_ptr(s).to_bytes().to_vec())) - }; - Ok(ret) +pub fn getenv(k: &OsStr) -> Option { + let s = run_with_cstr(k.as_bytes(), |k| unsafe { + let _guard = env_read_lock(); + Ok(libc::getenv(k.as_ptr()) as *const libc::c_char) + }) + .ok()?; + if s.is_null() { + None + } else { + Some(OsStringExt::from_vec(unsafe { CStr::from_ptr(s) }.to_bytes().to_vec())) } } pub fn setenv(k: &OsStr, v: &OsStr) -> io::Result<()> { - let k = CString::new(k.as_bytes())?; - let v = CString::new(v.as_bytes())?; - - unsafe { - let _guard = env_lock(); - cvt(libc::setenv(k.as_ptr(), v.as_ptr(), 1)).map(drop) - } + run_with_cstr(k.as_bytes(), |k| { + run_with_cstr(v.as_bytes(), |v| unsafe { + let _guard = env_write_lock(); + cvt(libc::setenv(k.as_ptr(), v.as_ptr(), 1)).map(drop) + }) + }) } pub fn unsetenv(n: &OsStr) -> io::Result<()> { - let nbuf = CString::new(n.as_bytes())?; - - unsafe { - let _guard = env_lock(); + run_with_cstr(n.as_bytes(), |nbuf| unsafe { + let _guard = env_write_lock(); cvt(libc::unsetenv(nbuf.as_ptr())).map(drop) - } + }) } pub fn temp_dir() -> PathBuf { diff --git a/crux-mir/lib/std/src/sys/wasi/path.rs b/crux-mir/lib/std/src/sys/wasi/path.rs deleted file mode 100644 index 840a7ae04..000000000 --- a/crux-mir/lib/std/src/sys/wasi/path.rs +++ /dev/null @@ -1,19 +0,0 @@ -use crate::ffi::OsStr; -use crate::path::Prefix; - -#[inline] -pub fn is_sep_byte(b: u8) -> bool { - b == b'/' -} - -#[inline] -pub fn is_verbatim_sep(b: u8) -> bool { - b == b'/' -} - -pub fn parse_prefix(_: &OsStr) -> Option> { - None -} - -pub const MAIN_SEP_STR: &str = "/"; -pub const MAIN_SEP: char = '/'; diff --git a/crux-mir/lib/std/src/sys/wasi/pipe.rs b/crux-mir/lib/std/src/sys/wasi/pipe.rs deleted file mode 100644 index fb14dc591..000000000 --- a/crux-mir/lib/std/src/sys/wasi/pipe.rs +++ /dev/null @@ -1,30 +0,0 @@ -use crate::io::{self, IoSlice, IoSliceMut}; -use crate::sys::Void; - -pub struct AnonPipe(Void); - -impl AnonPipe { - pub fn read(&self, _buf: &mut [u8]) -> io::Result { - match self.0 {} - } - - pub fn read_vectored(&self, _bufs: &mut [IoSliceMut<'_>]) -> io::Result { - match self.0 {} - } - - pub fn write(&self, _buf: &[u8]) -> io::Result { - match self.0 {} - } - - pub fn write_vectored(&self, _bufs: &[IoSlice<'_>]) -> io::Result { - match self.0 {} - } - - pub fn diverge(&self) -> ! { - match self.0 {} - } -} - -pub fn read2(p1: AnonPipe, _v1: &mut Vec, _p2: AnonPipe, _v2: &mut Vec) -> io::Result<()> { - match p1.0 {} -} diff --git a/crux-mir/lib/std/src/sys/wasi/stdio.rs b/crux-mir/lib/std/src/sys/wasi/stdio.rs index 1d53884f2..4cc0e4ed5 100644 --- a/crux-mir/lib/std/src/sys/wasi/stdio.rs +++ b/crux-mir/lib/std/src/sys/wasi/stdio.rs @@ -1,89 +1,112 @@ +#![deny(unsafe_op_in_unsafe_fn)] + +use super::fd::WasiFd; use crate::io::{self, IoSlice, IoSliceMut}; use crate::mem::ManuallyDrop; -use crate::sys::fd::WasiFd; +use crate::os::raw; +use crate::os::wasi::io::{AsRawFd, FromRawFd}; pub struct Stdin; pub struct Stdout; pub struct Stderr; impl Stdin { - pub fn new() -> io::Result { - Ok(Stdin) - } - - pub fn read(&self, data: &mut [u8]) -> io::Result { - self.read_vectored(&mut [IoSliceMut::new(data)]) - } - - pub fn read_vectored(&self, data: &mut [IoSliceMut<'_>]) -> io::Result { - ManuallyDrop::new(unsafe { WasiFd::from_raw(self.as_raw_fd()) }).read(data) + pub const fn new() -> Stdin { + Stdin } +} - pub fn as_raw_fd(&self) -> u32 { +impl AsRawFd for Stdin { + #[inline] + fn as_raw_fd(&self) -> raw::c_int { 0 } } -impl Stdout { - pub fn new() -> io::Result { - Ok(Stdout) +impl io::Read for Stdin { + fn read(&mut self, data: &mut [u8]) -> io::Result { + self.read_vectored(&mut [IoSliceMut::new(data)]) } - pub fn write(&self, data: &[u8]) -> io::Result { - self.write_vectored(&[IoSlice::new(data)]) + fn read_vectored(&mut self, data: &mut [IoSliceMut<'_>]) -> io::Result { + ManuallyDrop::new(unsafe { WasiFd::from_raw_fd(self.as_raw_fd()) }).read(data) } - pub fn write_vectored(&self, data: &[IoSlice<'_>]) -> io::Result { - ManuallyDrop::new(unsafe { WasiFd::from_raw(self.as_raw_fd()) }).write(data) + #[inline] + fn is_read_vectored(&self) -> bool { + true } +} - pub fn flush(&self) -> io::Result<()> { - Ok(()) +impl Stdout { + pub const fn new() -> Stdout { + Stdout } +} - pub fn as_raw_fd(&self) -> u32 { +impl AsRawFd for Stdout { + #[inline] + fn as_raw_fd(&self) -> raw::c_int { 1 } } -impl Stderr { - pub fn new() -> io::Result { - Ok(Stderr) - } - - pub fn write(&self, data: &[u8]) -> io::Result { +impl io::Write for Stdout { + fn write(&mut self, data: &[u8]) -> io::Result { self.write_vectored(&[IoSlice::new(data)]) } - pub fn write_vectored(&self, data: &[IoSlice<'_>]) -> io::Result { - ManuallyDrop::new(unsafe { WasiFd::from_raw(self.as_raw_fd()) }).write(data) + fn write_vectored(&mut self, data: &[IoSlice<'_>]) -> io::Result { + ManuallyDrop::new(unsafe { WasiFd::from_raw_fd(self.as_raw_fd()) }).write(data) } - pub fn flush(&self) -> io::Result<()> { + #[inline] + fn is_write_vectored(&self) -> bool { + true + } + fn flush(&mut self) -> io::Result<()> { Ok(()) } +} - pub fn as_raw_fd(&self) -> u32 { +impl Stderr { + pub const fn new() -> Stderr { + Stderr + } +} + +impl AsRawFd for Stderr { + #[inline] + fn as_raw_fd(&self) -> raw::c_int { 2 } } impl io::Write for Stderr { fn write(&mut self, data: &[u8]) -> io::Result { - (&*self).write(data) + self.write_vectored(&[IoSlice::new(data)]) + } + + fn write_vectored(&mut self, data: &[IoSlice<'_>]) -> io::Result { + ManuallyDrop::new(unsafe { WasiFd::from_raw_fd(self.as_raw_fd()) }).write(data) + } + + #[inline] + fn is_write_vectored(&self) -> bool { + true } fn flush(&mut self) -> io::Result<()> { - (&*self).flush() + Ok(()) } } pub const STDIN_BUF_SIZE: usize = crate::sys_common::io::DEFAULT_BUF_SIZE; pub fn is_ebadf(err: &io::Error) -> bool { - err.raw_os_error() == Some(wasi::ERRNO_BADF.into()) + err.raw_os_error() == Some(wasi::ERRNO_BADF.raw().into()) } pub fn panic_output() -> Option { - Stderr::new().ok() + Some(Stderr::new()) } diff --git a/crux-mir/lib/std/src/sys/wasi/thread.rs b/crux-mir/lib/std/src/sys/wasi/thread.rs index 0986759b8..e7a6ab4be 100644 --- a/crux-mir/lib/std/src/sys/wasi/thread.rs +++ b/crux-mir/lib/std/src/sys/wasi/thread.rs @@ -1,10 +1,13 @@ +#![deny(unsafe_op_in_unsafe_fn)] + use crate::ffi::CStr; use crate::io; use crate::mem; -use crate::sys::{unsupported, Void}; +use crate::num::NonZeroUsize; +use crate::sys::unsupported; use crate::time::Duration; -pub struct Thread(Void); +pub struct Thread(!); pub const DEFAULT_MIN_STACK_SIZE: usize = 4096; @@ -25,7 +28,7 @@ impl Thread { pub fn sleep(dur: Duration) { let nanos = dur.as_nanos(); - assert!(nanos <= u64::max_value() as u128); + assert!(nanos <= u64::MAX as u128); const USERDATA: wasi::Userdata = 0x0123_45678; @@ -38,8 +41,7 @@ impl Thread { let in_ = wasi::Subscription { userdata: USERDATA, - r#type: wasi::EVENTTYPE_CLOCK, - u: wasi::SubscriptionU { clock }, + u: wasi::SubscriptionU { tag: 0, u: wasi::SubscriptionUU { clock } }, }; unsafe { let mut event: wasi::Event = mem::zeroed(); @@ -48,7 +50,10 @@ impl Thread { ( Ok(1), wasi::Event { - userdata: USERDATA, error: 0, r#type: wasi::EVENTTYPE_CLOCK, .. + userdata: USERDATA, + error: wasi::ERRNO_SUCCESS, + type_: wasi::EVENTTYPE_CLOCK, + .. }, ) => {} _ => panic!("thread::sleep(): unexpected result of poll_oneoff"), @@ -57,10 +62,14 @@ impl Thread { } pub fn join(self) { - match self.0 {} + self.0 } } +pub fn available_parallelism() -> io::Result { + unsupported() +} + pub mod guard { pub type Guard = !; pub unsafe fn current() -> Option { diff --git a/crux-mir/lib/std/src/sys/wasi/time.rs b/crux-mir/lib/std/src/sys/wasi/time.rs index 80ec317b5..016b06efb 100644 --- a/crux-mir/lib/std/src/sys/wasi/time.rs +++ b/crux-mir/lib/std/src/sys/wasi/time.rs @@ -1,3 +1,5 @@ +#![deny(unsafe_op_in_unsafe_fn)] + use crate::time::Duration; #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] @@ -8,7 +10,7 @@ pub struct SystemTime(Duration); pub const UNIX_EPOCH: SystemTime = SystemTime(Duration::from_secs(0)); -fn current_time(clock: u32) -> Duration { +fn current_time(clock: wasi::Clockid) -> Duration { let ts = unsafe { wasi::clock_time_get( clock, 1, // precision... seems ignored though? @@ -23,14 +25,6 @@ impl Instant { Instant(current_time(wasi::CLOCKID_MONOTONIC)) } - pub const fn zero() -> Instant { - Instant(Duration::from_secs(0)) - } - - pub fn actually_monotonic() -> bool { - true - } - pub fn checked_sub_instant(&self, other: &Instant) -> Option { self.0.checked_sub(other.0) } @@ -53,6 +47,10 @@ impl SystemTime { SystemTime(Duration::from_nanos(ts)) } + pub fn to_wasi_timestamp(&self) -> Option { + self.0.as_nanos().try_into().ok() + } + pub fn sub_time(&self, other: &SystemTime) -> Result { self.0.checked_sub(other.0).ok_or_else(|| other.0 - self.0) } diff --git a/crux-mir/lib/std/src/sys/wasm/alloc.rs b/crux-mir/lib/std/src/sys/wasm/alloc.rs index 32b8b5bda..6dceb1689 100644 --- a/crux-mir/lib/std/src/sys/wasm/alloc.rs +++ b/crux-mir/lib/std/src/sys/wasm/alloc.rs @@ -1,8 +1,8 @@ -//! This is an implementation of a global allocator on the wasm32 platform when +//! This is an implementation of a global allocator on wasm targets when //! emscripten is not in use. In that situation there's no actual runtime for us //! to lean on for allocation, so instead we provide our own! //! -//! The wasm32 instruction set has two instructions for getting the current +//! The wasm instruction set has two instructions for getting the current //! amount of memory and growing the amount of memory. These instructions are the //! foundation on which we're able to build an allocator, so we do so! Note that //! the instructions are also pretty "global" and this is the "global" allocator @@ -18,32 +18,40 @@ use crate::alloc::{GlobalAlloc, Layout, System}; -static mut DLMALLOC: dlmalloc::Dlmalloc = dlmalloc::DLMALLOC_INIT; +static mut DLMALLOC: dlmalloc::Dlmalloc = dlmalloc::Dlmalloc::new(); #[stable(feature = "alloc_system_type", since = "1.28.0")] unsafe impl GlobalAlloc for System { #[inline] unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + // SAFETY: DLMALLOC access is guaranteed to be safe because the lock gives us unique and non-reentrant access. + // Calling malloc() is safe because preconditions on this function match the trait method preconditions. let _lock = lock::lock(); - DLMALLOC.malloc(layout.size(), layout.align()) + unsafe { DLMALLOC.malloc(layout.size(), layout.align()) } } #[inline] unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + // SAFETY: DLMALLOC access is guaranteed to be safe because the lock gives us unique and non-reentrant access. + // Calling calloc() is safe because preconditions on this function match the trait method preconditions. let _lock = lock::lock(); - DLMALLOC.calloc(layout.size(), layout.align()) + unsafe { DLMALLOC.calloc(layout.size(), layout.align()) } } #[inline] unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + // SAFETY: DLMALLOC access is guaranteed to be safe because the lock gives us unique and non-reentrant access. + // Calling free() is safe because preconditions on this function match the trait method preconditions. let _lock = lock::lock(); - DLMALLOC.free(ptr, layout.size(), layout.align()) + unsafe { DLMALLOC.free(ptr, layout.size(), layout.align()) } } #[inline] unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + // SAFETY: DLMALLOC access is guaranteed to be safe because the lock gives us unique and non-reentrant access. + // Calling realloc() is safe because preconditions on this function match the trait method preconditions. let _lock = lock::lock(); - DLMALLOC.realloc(ptr, layout.size(), layout.align(), new_size) + unsafe { DLMALLOC.realloc(ptr, layout.size(), layout.align(), new_size) } } } diff --git a/crux-mir/lib/std/src/sys/wasm/args.rs b/crux-mir/lib/std/src/sys/wasm/args.rs deleted file mode 100644 index 3b6557ae3..000000000 --- a/crux-mir/lib/std/src/sys/wasm/args.rs +++ /dev/null @@ -1,46 +0,0 @@ -use crate::ffi::OsString; -use crate::marker::PhantomData; -use crate::vec; - -pub unsafe fn init(_argc: isize, _argv: *const *const u8) { - // On wasm these should always be null, so there's nothing for us to do here -} - -pub unsafe fn cleanup() {} - -pub fn args() -> Args { - Args { iter: Vec::new().into_iter(), _dont_send_or_sync_me: PhantomData } -} - -pub struct Args { - iter: vec::IntoIter, - _dont_send_or_sync_me: PhantomData<*mut ()>, -} - -impl Args { - pub fn inner_debug(&self) -> &[OsString] { - self.iter.as_slice() - } -} - -impl Iterator for Args { - type Item = OsString; - fn next(&mut self) -> Option { - self.iter.next() - } - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() - } -} - -impl ExactSizeIterator for Args { - fn len(&self) -> usize { - self.iter.len() - } -} - -impl DoubleEndedIterator for Args { - fn next_back(&mut self) -> Option { - self.iter.next_back() - } -} diff --git a/crux-mir/lib/std/src/sys/wasm/atomics/futex.rs b/crux-mir/lib/std/src/sys/wasm/atomics/futex.rs new file mode 100644 index 000000000..f4fbe9f48 --- /dev/null +++ b/crux-mir/lib/std/src/sys/wasm/atomics/futex.rs @@ -0,0 +1,34 @@ +use crate::arch::wasm32; +use crate::sync::atomic::AtomicU32; +use crate::time::Duration; + +/// Wait for a futex_wake operation to wake us. +/// +/// Returns directly if the futex doesn't hold the expected value. +/// +/// Returns false on timeout, and true in all other cases. +pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option) -> bool { + let timeout = timeout.and_then(|t| t.as_nanos().try_into().ok()).unwrap_or(-1); + unsafe { + wasm32::memory_atomic_wait32( + futex as *const AtomicU32 as *mut i32, + expected as i32, + timeout, + ) < 2 + } +} + +/// Wake up one thread that's blocked on futex_wait on this futex. +/// +/// Returns true if this actually woke up such a thread, +/// or false if no thread was waiting on this futex. +pub fn futex_wake(futex: &AtomicU32) -> bool { + unsafe { wasm32::memory_atomic_notify(futex as *const AtomicU32 as *mut i32, 1) > 0 } +} + +/// Wake up all threads that are waiting on futex_wait on this futex. +pub fn futex_wake_all(futex: &AtomicU32) { + unsafe { + wasm32::memory_atomic_notify(futex as *const AtomicU32 as *mut i32, i32::MAX as u32); + } +} diff --git a/crux-mir/lib/std/src/sys/wasm/atomics/thread.rs b/crux-mir/lib/std/src/sys/wasm/atomics/thread.rs new file mode 100644 index 000000000..714b70492 --- /dev/null +++ b/crux-mir/lib/std/src/sys/wasm/atomics/thread.rs @@ -0,0 +1,55 @@ +use crate::ffi::CStr; +use crate::io; +use crate::num::NonZeroUsize; +use crate::sys::unsupported; +use crate::time::Duration; + +pub struct Thread(!); + +pub const DEFAULT_MIN_STACK_SIZE: usize = 4096; + +impl Thread { + // unsafe: see thread::Builder::spawn_unchecked for safety requirements + pub unsafe fn new(_stack: usize, _p: Box) -> io::Result { + unsupported() + } + + pub fn yield_now() {} + + pub fn set_name(_name: &CStr) {} + + pub fn sleep(dur: Duration) { + use crate::arch::wasm32; + use crate::cmp; + + // Use an atomic wait to block the current thread artificially with a + // timeout listed. Note that we should never be notified (return value + // of 0) or our comparison should never fail (return value of 1) so we + // should always only resume execution through a timeout (return value + // 2). + let mut nanos = dur.as_nanos(); + while nanos > 0 { + let amt = cmp::min(i64::MAX as u128, nanos); + let mut x = 0; + let val = unsafe { wasm32::memory_atomic_wait32(&mut x, 0, amt as i64) }; + debug_assert_eq!(val, 2); + nanos -= amt; + } + } + + pub fn join(self) {} +} + +pub fn available_parallelism() -> io::Result { + unsupported() +} + +pub mod guard { + pub type Guard = !; + pub unsafe fn current() -> Option { + None + } + pub unsafe fn init() -> Option { + None + } +} diff --git a/crux-mir/lib/std/src/sys/wasm/cmath.rs b/crux-mir/lib/std/src/sys/wasm/cmath.rs deleted file mode 100644 index 304cf906b..000000000 --- a/crux-mir/lib/std/src/sys/wasm/cmath.rs +++ /dev/null @@ -1,29 +0,0 @@ -// These symbols are all defined in `compiler-builtins` -extern "C" { - pub fn acos(n: f64) -> f64; - pub fn acosf(n: f32) -> f32; - pub fn asin(n: f64) -> f64; - pub fn asinf(n: f32) -> f32; - pub fn atan(n: f64) -> f64; - pub fn atan2(a: f64, b: f64) -> f64; - pub fn atan2f(a: f32, b: f32) -> f32; - pub fn atanf(n: f32) -> f32; - pub fn cbrt(n: f64) -> f64; - pub fn cbrtf(n: f32) -> f32; - pub fn cosh(n: f64) -> f64; - pub fn coshf(n: f32) -> f32; - pub fn expm1(n: f64) -> f64; - pub fn expm1f(n: f32) -> f32; - pub fn fdim(a: f64, b: f64) -> f64; - pub fn fdimf(a: f32, b: f32) -> f32; - pub fn hypot(x: f64, y: f64) -> f64; - pub fn hypotf(x: f32, y: f32) -> f32; - pub fn log1p(n: f64) -> f64; - pub fn log1pf(n: f32) -> f32; - pub fn sinh(n: f64) -> f64; - pub fn sinhf(n: f32) -> f32; - pub fn tan(n: f64) -> f64; - pub fn tanf(n: f32) -> f32; - pub fn tanh(n: f64) -> f64; - pub fn tanhf(n: f32) -> f32; -} diff --git a/crux-mir/lib/std/src/sys/wasm/condvar_atomics.rs b/crux-mir/lib/std/src/sys/wasm/condvar_atomics.rs deleted file mode 100644 index a4021c0ee..000000000 --- a/crux-mir/lib/std/src/sys/wasm/condvar_atomics.rs +++ /dev/null @@ -1,94 +0,0 @@ -use crate::arch::wasm32; -use crate::cmp; -use crate::mem; -use crate::sync::atomic::{AtomicUsize, Ordering::SeqCst}; -use crate::sys::mutex::Mutex; -use crate::time::Duration; - -pub struct Condvar { - cnt: AtomicUsize, -} - -// Condition variables are implemented with a simple counter internally that is -// likely to cause spurious wakeups. Blocking on a condition variable will first -// read the value of the internal counter, unlock the given mutex, and then -// block if and only if the counter's value is still the same. Notifying a -// condition variable will modify the counter (add one for now) and then wake up -// a thread waiting on the address of the counter. -// -// A thread waiting on the condition variable will as a result avoid going to -// sleep if it's notified after the lock is unlocked but before it fully goes to -// sleep. A sleeping thread is guaranteed to be woken up at some point as it can -// only be woken up with a call to `wake`. -// -// Note that it's possible for 2 or more threads to be woken up by a call to -// `notify_one` with this implementation. That can happen where the modification -// of `cnt` causes any threads in the middle of `wait` to avoid going to sleep, -// and the subsequent `wake` may wake up a thread that's actually blocking. We -// consider this a spurious wakeup, though, which all users of condition -// variables must already be prepared to handle. As a result, this source of -// spurious wakeups is currently though to be ok, although it may be problematic -// later on if it causes too many spurious wakeups. - -impl Condvar { - pub const fn new() -> Condvar { - Condvar { cnt: AtomicUsize::new(0) } - } - - #[inline] - pub unsafe fn init(&mut self) { - // nothing to do - } - - pub unsafe fn notify_one(&self) { - self.cnt.fetch_add(1, SeqCst); - wasm32::atomic_notify(self.ptr(), 1); - } - - #[inline] - pub unsafe fn notify_all(&self) { - self.cnt.fetch_add(1, SeqCst); - wasm32::atomic_notify(self.ptr(), u32::max_value()); // -1 == "wake everyone" - } - - pub unsafe fn wait(&self, mutex: &Mutex) { - // "atomically block and unlock" implemented by loading our current - // counter's value, unlocking the mutex, and blocking if the counter - // still has the same value. - // - // Notifications happen by incrementing the counter and then waking a - // thread. Incrementing the counter after we unlock the mutex will - // prevent us from sleeping and otherwise the call to `wake` will - // wake us up once we're asleep. - let ticket = self.cnt.load(SeqCst) as i32; - mutex.unlock(); - let val = wasm32::i32_atomic_wait(self.ptr(), ticket, -1); - // 0 == woken, 1 == not equal to `ticket`, 2 == timeout (shouldn't happen) - debug_assert!(val == 0 || val == 1); - mutex.lock(); - } - - pub unsafe fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool { - let ticket = self.cnt.load(SeqCst) as i32; - mutex.unlock(); - let nanos = dur.as_nanos(); - let nanos = cmp::min(i64::max_value() as u128, nanos); - - // If the return value is 2 then a timeout happened, so we return - // `false` as we weren't actually notified. - let ret = wasm32::i32_atomic_wait(self.ptr(), ticket, nanos as i64) != 2; - mutex.lock(); - return ret; - } - - #[inline] - pub unsafe fn destroy(&self) { - // nothing to do - } - - #[inline] - fn ptr(&self) -> *mut i32 { - assert_eq!(mem::size_of::(), mem::size_of::()); - self.cnt.as_mut_ptr() as *mut i32 - } -} diff --git a/crux-mir/lib/std/src/sys/wasm/io.rs b/crux-mir/lib/std/src/sys/wasm/io.rs deleted file mode 100644 index d5f475b43..000000000 --- a/crux-mir/lib/std/src/sys/wasm/io.rs +++ /dev/null @@ -1,47 +0,0 @@ -use crate::mem; - -#[derive(Copy, Clone)] -pub struct IoSlice<'a>(&'a [u8]); - -impl<'a> IoSlice<'a> { - #[inline] - pub fn new(buf: &'a [u8]) -> IoSlice<'a> { - IoSlice(buf) - } - - #[inline] - pub fn advance(&mut self, n: usize) { - self.0 = &self.0[n..] - } - - #[inline] - pub fn as_slice(&self) -> &[u8] { - self.0 - } -} - -pub struct IoSliceMut<'a>(&'a mut [u8]); - -impl<'a> IoSliceMut<'a> { - #[inline] - pub fn new(buf: &'a mut [u8]) -> IoSliceMut<'a> { - IoSliceMut(buf) - } - - #[inline] - pub fn advance(&mut self, n: usize) { - let slice = mem::replace(&mut self.0, &mut []); - let (_, remaining) = slice.split_at_mut(n); - self.0 = remaining; - } - - #[inline] - pub fn as_slice(&self) -> &[u8] { - self.0 - } - - #[inline] - pub fn as_mut_slice(&mut self) -> &mut [u8] { - self.0 - } -} diff --git a/crux-mir/lib/std/src/sys/wasm/memchr.rs b/crux-mir/lib/std/src/sys/wasm/memchr.rs deleted file mode 100644 index 996748219..000000000 --- a/crux-mir/lib/std/src/sys/wasm/memchr.rs +++ /dev/null @@ -1 +0,0 @@ -pub use core::slice::memchr::{memchr, memrchr}; diff --git a/crux-mir/lib/std/src/sys/wasm/mod.rs b/crux-mir/lib/std/src/sys/wasm/mod.rs index c115f7564..77ebe3c4a 100644 --- a/crux-mir/lib/std/src/sys/wasm/mod.rs +++ b/crux-mir/lib/std/src/sys/wasm/mod.rs @@ -14,82 +14,66 @@ //! compiling for wasm. That way it's a compile time error for something that's //! guaranteed to be a runtime error! -use crate::os::raw::c_char; +#![deny(unsafe_op_in_unsafe_fn)] pub mod alloc; +#[path = "../unsupported/args.rs"] pub mod args; +#[path = "../unix/cmath.rs"] pub mod cmath; pub mod env; -pub mod fast_thread_local; +#[path = "../unsupported/fs.rs"] pub mod fs; +#[path = "../unsupported/io.rs"] pub mod io; -pub mod memchr; +#[path = "../unsupported/net.rs"] pub mod net; +#[path = "../unsupported/os.rs"] pub mod os; +#[path = "../unix/os_str.rs"] +pub mod os_str; +#[path = "../unix/path.rs"] pub mod path; +#[path = "../unsupported/pipe.rs"] pub mod pipe; +#[path = "../unsupported/process.rs"] pub mod process; -pub mod stack_overflow; +#[path = "../unsupported/stdio.rs"] pub mod stdio; -pub mod thread; -pub mod thread_local; +#[path = "../unsupported/thread_local_dtor.rs"] +pub mod thread_local_dtor; +#[path = "../unsupported/thread_local_key.rs"] +pub mod thread_local_key; +#[path = "../unsupported/time.rs"] pub mod time; -pub use crate::sys_common::os_str_bytes as os_str; - cfg_if::cfg_if! { if #[cfg(target_feature = "atomics")] { - #[path = "condvar_atomics.rs"] - pub mod condvar; - #[path = "mutex_atomics.rs"] - pub mod mutex; - #[path = "rwlock_atomics.rs"] - pub mod rwlock; + #[path = "../unix/locks"] + pub mod locks { + #![allow(unsafe_op_in_unsafe_fn)] + mod futex_condvar; + mod futex_mutex; + mod futex_rwlock; + pub(crate) use futex_condvar::Condvar; + pub(crate) use futex_mutex::Mutex; + pub(crate) use futex_rwlock::RwLock; + } + #[path = "atomics/futex.rs"] + pub mod futex; + #[path = "atomics/thread.rs"] + pub mod thread; } else { - pub mod condvar; - pub mod mutex; - pub mod rwlock; + #[path = "../unsupported/locks/mod.rs"] + pub mod locks; + #[path = "../unsupported/once.rs"] + pub mod once; + #[path = "../unsupported/thread.rs"] + pub mod thread; } } -#[cfg(not(test))] -pub fn init() {} - -pub fn unsupported() -> crate::io::Result { - Err(unsupported_err()) -} - -pub fn unsupported_err() -> crate::io::Error { - crate::io::Error::new(crate::io::ErrorKind::Other, "operation not supported on wasm yet") -} - -pub fn decode_error_kind(_code: i32) -> crate::io::ErrorKind { - crate::io::ErrorKind::Other -} - -// This enum is used as the storage for a bunch of types which can't actually -// exist. -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] -pub enum Void {} - -pub unsafe fn strlen(mut s: *const c_char) -> usize { - let mut n = 0; - while *s != 0 { - n += 1; - s = s.offset(1); - } - return n; -} - -pub unsafe fn abort_internal() -> ! { - crate::arch::wasm32::unreachable() -} - -// We don't have randomness yet, but I totally used a random number generator to -// generate these numbers. -// -// More seriously though this is just for DOS protection in hash maps. It's ok -// if we don't do that on wasm just yet. -pub fn hashmap_random_keys() -> (u64, u64) { - (1, 2) -} +#[path = "../unsupported/common.rs"] +#[deny(unsafe_op_in_unsafe_fn)] +mod common; +pub use common::*; diff --git a/crux-mir/lib/std/src/sys/wasm/mutex.rs b/crux-mir/lib/std/src/sys/wasm/mutex.rs deleted file mode 100644 index 7aaf1b3a3..000000000 --- a/crux-mir/lib/std/src/sys/wasm/mutex.rs +++ /dev/null @@ -1,66 +0,0 @@ -use crate::cell::UnsafeCell; - -pub struct Mutex { - locked: UnsafeCell, -} - -unsafe impl Send for Mutex {} -unsafe impl Sync for Mutex {} // no threads on wasm - -impl Mutex { - pub const fn new() -> Mutex { - Mutex { locked: UnsafeCell::new(false) } - } - - #[inline] - pub unsafe fn init(&mut self) {} - - #[inline] - pub unsafe fn lock(&self) { - let locked = self.locked.get(); - assert!(!*locked, "cannot recursively acquire mutex"); - *locked = true; - } - - #[inline] - pub unsafe fn unlock(&self) { - *self.locked.get() = false; - } - - #[inline] - pub unsafe fn try_lock(&self) -> bool { - let locked = self.locked.get(); - if *locked { - false - } else { - *locked = true; - true - } - } - - #[inline] - pub unsafe fn destroy(&self) {} -} - -// All empty stubs because wasm has no threads yet, so lock acquisition always -// succeeds. -pub struct ReentrantMutex {} - -impl ReentrantMutex { - pub const unsafe fn uninitialized() -> ReentrantMutex { - ReentrantMutex {} - } - - pub unsafe fn init(&self) {} - - pub unsafe fn lock(&self) {} - - #[inline] - pub unsafe fn try_lock(&self) -> bool { - true - } - - pub unsafe fn unlock(&self) {} - - pub unsafe fn destroy(&self) {} -} diff --git a/crux-mir/lib/std/src/sys/wasm/mutex_atomics.rs b/crux-mir/lib/std/src/sys/wasm/mutex_atomics.rs deleted file mode 100644 index 268a53bb5..000000000 --- a/crux-mir/lib/std/src/sys/wasm/mutex_atomics.rs +++ /dev/null @@ -1,147 +0,0 @@ -use crate::arch::wasm32; -use crate::cell::UnsafeCell; -use crate::mem; -use crate::sync::atomic::{AtomicU32, AtomicUsize, Ordering::SeqCst}; -use crate::sys::thread; - -pub struct Mutex { - locked: AtomicUsize, -} - -// Mutexes have a pretty simple implementation where they contain an `i32` -// internally that is 0 when unlocked and 1 when the mutex is locked. -// Acquisition has a fast path where it attempts to cmpxchg the 0 to a 1, and -// if it fails it then waits for a notification. Releasing a lock is then done -// by swapping in 0 and then notifying any waiters, if present. - -impl Mutex { - pub const fn new() -> Mutex { - Mutex { locked: AtomicUsize::new(0) } - } - - #[inline] - pub unsafe fn init(&mut self) { - // nothing to do - } - - pub unsafe fn lock(&self) { - while !self.try_lock() { - let val = wasm32::i32_atomic_wait( - self.ptr(), - 1, // we expect our mutex is locked - -1, // wait infinitely - ); - // we should have either woke up (0) or got a not-equal due to a - // race (1). We should never time out (2) - debug_assert!(val == 0 || val == 1); - } - } - - pub unsafe fn unlock(&self) { - let prev = self.locked.swap(0, SeqCst); - debug_assert_eq!(prev, 1); - wasm32::atomic_notify(self.ptr(), 1); // wake up one waiter, if any - } - - #[inline] - pub unsafe fn try_lock(&self) -> bool { - self.locked.compare_exchange(0, 1, SeqCst, SeqCst).is_ok() - } - - #[inline] - pub unsafe fn destroy(&self) { - // nothing to do - } - - #[inline] - fn ptr(&self) -> *mut i32 { - assert_eq!(mem::size_of::(), mem::size_of::()); - self.locked.as_mut_ptr() as *mut i32 - } -} - -pub struct ReentrantMutex { - owner: AtomicU32, - recursions: UnsafeCell, -} - -unsafe impl Send for ReentrantMutex {} -unsafe impl Sync for ReentrantMutex {} - -// Reentrant mutexes are similarly implemented to mutexs above except that -// instead of "1" meaning unlocked we use the id of a thread to represent -// whether it has locked a mutex. That way we have an atomic counter which -// always holds the id of the thread that currently holds the lock (or 0 if the -// lock is unlocked). -// -// Once a thread acquires a lock recursively, which it detects by looking at -// the value that's already there, it will update a local `recursions` counter -// in a nonatomic fashion (as we hold the lock). The lock is then fully -// released when this recursion counter reaches 0. - -impl ReentrantMutex { - pub const unsafe fn uninitialized() -> ReentrantMutex { - ReentrantMutex { owner: AtomicU32::new(0), recursions: UnsafeCell::new(0) } - } - - pub unsafe fn init(&self) { - // nothing to do... - } - - pub unsafe fn lock(&self) { - let me = thread::my_id(); - while let Err(owner) = self._try_lock(me) { - let val = wasm32::i32_atomic_wait(self.ptr(), owner as i32, -1); - debug_assert!(val == 0 || val == 1); - } - } - - #[inline] - pub unsafe fn try_lock(&self) -> bool { - self._try_lock(thread::my_id()).is_ok() - } - - #[inline] - unsafe fn _try_lock(&self, id: u32) -> Result<(), u32> { - let id = id.checked_add(1).unwrap(); // make sure `id` isn't 0 - match self.owner.compare_exchange(0, id, SeqCst, SeqCst) { - // we transitioned from unlocked to locked - Ok(_) => { - debug_assert_eq!(*self.recursions.get(), 0); - Ok(()) - } - - // we currently own this lock, so let's update our count and return - // true. - Err(n) if n == id => { - *self.recursions.get() += 1; - Ok(()) - } - - // Someone else owns the lock, let our caller take care of it - Err(other) => Err(other), - } - } - - pub unsafe fn unlock(&self) { - // If we didn't ever recursively lock the lock then we fully unlock the - // mutex and wake up a waiter, if any. Otherwise we decrement our - // recursive counter and let some one else take care of the zero. - match *self.recursions.get() { - 0 => { - self.owner.swap(0, SeqCst); - wasm32::atomic_notify(self.ptr() as *mut i32, 1); // wake up one waiter, if any - } - ref mut n => *n -= 1, - } - } - - pub unsafe fn destroy(&self) { - // nothing to do... - } - - #[inline] - fn ptr(&self) -> *mut i32 { - self.owner.as_mut_ptr() as *mut i32 - } -} diff --git a/crux-mir/lib/std/src/sys/wasm/path.rs b/crux-mir/lib/std/src/sys/wasm/path.rs deleted file mode 100644 index 840a7ae04..000000000 --- a/crux-mir/lib/std/src/sys/wasm/path.rs +++ /dev/null @@ -1,19 +0,0 @@ -use crate::ffi::OsStr; -use crate::path::Prefix; - -#[inline] -pub fn is_sep_byte(b: u8) -> bool { - b == b'/' -} - -#[inline] -pub fn is_verbatim_sep(b: u8) -> bool { - b == b'/' -} - -pub fn parse_prefix(_: &OsStr) -> Option> { - None -} - -pub const MAIN_SEP_STR: &str = "/"; -pub const MAIN_SEP: char = '/'; diff --git a/crux-mir/lib/std/src/sys/wasm/pipe.rs b/crux-mir/lib/std/src/sys/wasm/pipe.rs deleted file mode 100644 index fb14dc591..000000000 --- a/crux-mir/lib/std/src/sys/wasm/pipe.rs +++ /dev/null @@ -1,30 +0,0 @@ -use crate::io::{self, IoSlice, IoSliceMut}; -use crate::sys::Void; - -pub struct AnonPipe(Void); - -impl AnonPipe { - pub fn read(&self, _buf: &mut [u8]) -> io::Result { - match self.0 {} - } - - pub fn read_vectored(&self, _bufs: &mut [IoSliceMut<'_>]) -> io::Result { - match self.0 {} - } - - pub fn write(&self, _buf: &[u8]) -> io::Result { - match self.0 {} - } - - pub fn write_vectored(&self, _bufs: &[IoSlice<'_>]) -> io::Result { - match self.0 {} - } - - pub fn diverge(&self) -> ! { - match self.0 {} - } -} - -pub fn read2(p1: AnonPipe, _v1: &mut Vec, _p2: AnonPipe, _v2: &mut Vec) -> io::Result<()> { - match p1.0 {} -} diff --git a/crux-mir/lib/std/src/sys/wasm/process.rs b/crux-mir/lib/std/src/sys/wasm/process.rs deleted file mode 100644 index 4702e5c54..000000000 --- a/crux-mir/lib/std/src/sys/wasm/process.rs +++ /dev/null @@ -1,149 +0,0 @@ -use crate::ffi::OsStr; -use crate::fmt; -use crate::io; -use crate::sys::fs::File; -use crate::sys::pipe::AnonPipe; -use crate::sys::{unsupported, Void}; -use crate::sys_common::process::CommandEnv; - -pub use crate::ffi::OsString as EnvKey; - -//////////////////////////////////////////////////////////////////////////////// -// Command -//////////////////////////////////////////////////////////////////////////////// - -pub struct Command { - env: CommandEnv, -} - -// passed back to std::process with the pipes connected to the child, if any -// were requested -pub struct StdioPipes { - pub stdin: Option, - pub stdout: Option, - pub stderr: Option, -} - -pub enum Stdio { - Inherit, - Null, - MakePipe, -} - -impl Command { - pub fn new(_program: &OsStr) -> Command { - Command { env: Default::default() } - } - - pub fn arg(&mut self, _arg: &OsStr) {} - - pub fn env_mut(&mut self) -> &mut CommandEnv { - &mut self.env - } - - pub fn cwd(&mut self, _dir: &OsStr) {} - - pub fn stdin(&mut self, _stdin: Stdio) {} - - pub fn stdout(&mut self, _stdout: Stdio) {} - - pub fn stderr(&mut self, _stderr: Stdio) {} - - pub fn spawn( - &mut self, - _default: Stdio, - _needs_stdin: bool, - ) -> io::Result<(Process, StdioPipes)> { - unsupported() - } -} - -impl From for Stdio { - fn from(pipe: AnonPipe) -> Stdio { - pipe.diverge() - } -} - -impl From for Stdio { - fn from(file: File) -> Stdio { - file.diverge() - } -} - -impl fmt::Debug for Command { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - Ok(()) - } -} - -pub struct ExitStatus(Void); - -impl ExitStatus { - pub fn success(&self) -> bool { - match self.0 {} - } - - pub fn code(&self) -> Option { - match self.0 {} - } -} - -impl Clone for ExitStatus { - fn clone(&self) -> ExitStatus { - match self.0 {} - } -} - -impl Copy for ExitStatus {} - -impl PartialEq for ExitStatus { - fn eq(&self, _other: &ExitStatus) -> bool { - match self.0 {} - } -} - -impl Eq for ExitStatus {} - -impl fmt::Debug for ExitStatus { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} - } -} - -impl fmt::Display for ExitStatus { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.0 {} - } -} - -#[derive(PartialEq, Eq, Clone, Copy, Debug)] -pub struct ExitCode(bool); - -impl ExitCode { - pub const SUCCESS: ExitCode = ExitCode(false); - pub const FAILURE: ExitCode = ExitCode(true); - - pub fn as_i32(&self) -> i32 { - self.0 as i32 - } -} - -pub struct Process(Void); - -impl Process { - pub fn id(&self) -> u32 { - match self.0 {} - } - - pub fn kill(&mut self) -> io::Result<()> { - match self.0 {} - } - - pub fn wait(&mut self) -> io::Result { - match self.0 {} - } - - pub fn try_wait(&mut self) -> io::Result> { - match self.0 {} - } -} diff --git a/crux-mir/lib/std/src/sys/wasm/rwlock.rs b/crux-mir/lib/std/src/sys/wasm/rwlock.rs deleted file mode 100644 index a59944482..000000000 --- a/crux-mir/lib/std/src/sys/wasm/rwlock.rs +++ /dev/null @@ -1,69 +0,0 @@ -use crate::cell::UnsafeCell; - -pub struct RWLock { - mode: UnsafeCell, -} - -unsafe impl Send for RWLock {} -unsafe impl Sync for RWLock {} // no threads on wasm - -impl RWLock { - pub const fn new() -> RWLock { - RWLock { mode: UnsafeCell::new(0) } - } - - #[inline] - pub unsafe fn read(&self) { - let mode = self.mode.get(); - if *mode >= 0 { - *mode += 1; - } else { - rtabort!("rwlock locked for writing"); - } - } - - #[inline] - pub unsafe fn try_read(&self) -> bool { - let mode = self.mode.get(); - if *mode >= 0 { - *mode += 1; - true - } else { - false - } - } - - #[inline] - pub unsafe fn write(&self) { - let mode = self.mode.get(); - if *mode == 0 { - *mode = -1; - } else { - rtabort!("rwlock locked for reading") - } - } - - #[inline] - pub unsafe fn try_write(&self) -> bool { - let mode = self.mode.get(); - if *mode == 0 { - *mode = -1; - true - } else { - false - } - } - - #[inline] - pub unsafe fn read_unlock(&self) { - *self.mode.get() -= 1; - } - - #[inline] - pub unsafe fn write_unlock(&self) { - *self.mode.get() += 1; - } - - #[inline] - pub unsafe fn destroy(&self) {} -} diff --git a/crux-mir/lib/std/src/sys/wasm/rwlock_atomics.rs b/crux-mir/lib/std/src/sys/wasm/rwlock_atomics.rs deleted file mode 100644 index 06442e925..000000000 --- a/crux-mir/lib/std/src/sys/wasm/rwlock_atomics.rs +++ /dev/null @@ -1,144 +0,0 @@ -use crate::cell::UnsafeCell; -use crate::sys::condvar::Condvar; -use crate::sys::mutex::Mutex; - -pub struct RWLock { - lock: Mutex, - cond: Condvar, - state: UnsafeCell, -} - -enum State { - Unlocked, - Reading(usize), - Writing, -} - -unsafe impl Send for RWLock {} -unsafe impl Sync for RWLock {} - -// This rwlock implementation is a relatively simple implementation which has a -// condition variable for readers/writers as well as a mutex protecting the -// internal state of the lock. A current downside of the implementation is that -// unlocking the lock will notify *all* waiters rather than just readers or just -// writers. This can cause lots of "thundering stampede" problems. While -// hopefully correct this implementation is very likely to want to be changed in -// the future. - -impl RWLock { - pub const fn new() -> RWLock { - RWLock { lock: Mutex::new(), cond: Condvar::new(), state: UnsafeCell::new(State::Unlocked) } - } - - #[inline] - pub unsafe fn read(&self) { - self.lock.lock(); - while !(*self.state.get()).inc_readers() { - self.cond.wait(&self.lock); - } - self.lock.unlock(); - } - - #[inline] - pub unsafe fn try_read(&self) -> bool { - self.lock.lock(); - let ok = (*self.state.get()).inc_readers(); - self.lock.unlock(); - return ok; - } - - #[inline] - pub unsafe fn write(&self) { - self.lock.lock(); - while !(*self.state.get()).inc_writers() { - self.cond.wait(&self.lock); - } - self.lock.unlock(); - } - - #[inline] - pub unsafe fn try_write(&self) -> bool { - self.lock.lock(); - let ok = (*self.state.get()).inc_writers(); - self.lock.unlock(); - return ok; - } - - #[inline] - pub unsafe fn read_unlock(&self) { - self.lock.lock(); - let notify = (*self.state.get()).dec_readers(); - self.lock.unlock(); - if notify { - // FIXME: should only wake up one of these some of the time - self.cond.notify_all(); - } - } - - #[inline] - pub unsafe fn write_unlock(&self) { - self.lock.lock(); - (*self.state.get()).dec_writers(); - self.lock.unlock(); - // FIXME: should only wake up one of these some of the time - self.cond.notify_all(); - } - - #[inline] - pub unsafe fn destroy(&self) { - self.lock.destroy(); - self.cond.destroy(); - } -} - -impl State { - fn inc_readers(&mut self) -> bool { - match *self { - State::Unlocked => { - *self = State::Reading(1); - true - } - State::Reading(ref mut cnt) => { - *cnt += 1; - true - } - State::Writing => false, - } - } - - fn inc_writers(&mut self) -> bool { - match *self { - State::Unlocked => { - *self = State::Writing; - true - } - State::Reading(_) | State::Writing => false, - } - } - - fn dec_readers(&mut self) -> bool { - let zero = match *self { - State::Reading(ref mut cnt) => { - *cnt -= 1; - *cnt == 0 - } - State::Unlocked | State::Writing => invalid(), - }; - if zero { - *self = State::Unlocked; - } - zero - } - - fn dec_writers(&mut self) { - match *self { - State::Writing => {} - State::Unlocked | State::Reading(_) => invalid(), - } - *self = State::Unlocked; - } -} - -fn invalid() -> ! { - panic!("inconsistent rwlock"); -} diff --git a/crux-mir/lib/std/src/sys/wasm/stack_overflow.rs b/crux-mir/lib/std/src/sys/wasm/stack_overflow.rs deleted file mode 100644 index cbf62b6e5..000000000 --- a/crux-mir/lib/std/src/sys/wasm/stack_overflow.rs +++ /dev/null @@ -1,11 +0,0 @@ -pub struct Handler; - -impl Handler { - pub unsafe fn new() -> Handler { - Handler - } -} - -pub unsafe fn init() {} - -pub unsafe fn cleanup() {} diff --git a/crux-mir/lib/std/src/sys/wasm/thread.rs b/crux-mir/lib/std/src/sys/wasm/thread.rs deleted file mode 100644 index 0e0e78a82..000000000 --- a/crux-mir/lib/std/src/sys/wasm/thread.rs +++ /dev/null @@ -1,98 +0,0 @@ -use crate::ffi::CStr; -use crate::io; -use crate::sys::{unsupported, Void}; -use crate::time::Duration; - -pub struct Thread(Void); - -pub const DEFAULT_MIN_STACK_SIZE: usize = 4096; - -impl Thread { - // unsafe: see thread::Builder::spawn_unchecked for safety requirements - pub unsafe fn new(_stack: usize, _p: Box) -> io::Result { - unsupported() - } - - pub fn yield_now() { - // do nothing - } - - pub fn set_name(_name: &CStr) { - // nope - } - - #[cfg(not(target_feature = "atomics"))] - pub fn sleep(_dur: Duration) { - panic!("can't sleep"); - } - - #[cfg(target_feature = "atomics")] - pub fn sleep(dur: Duration) { - use crate::arch::wasm32; - use crate::cmp; - - // Use an atomic wait to block the current thread artificially with a - // timeout listed. Note that we should never be notified (return value - // of 0) or our comparison should never fail (return value of 1) so we - // should always only resume execution through a timeout (return value - // 2). - let mut nanos = dur.as_nanos(); - while nanos > 0 { - let amt = cmp::min(i64::max_value() as u128, nanos); - let mut x = 0; - let val = unsafe { wasm32::i32_atomic_wait(&mut x, 0, amt as i64) }; - debug_assert_eq!(val, 2); - nanos -= amt; - } - } - - pub fn join(self) { - match self.0 {} - } -} - -pub mod guard { - pub type Guard = !; - pub unsafe fn current() -> Option { - None - } - pub unsafe fn init() -> Option { - None - } -} - -// This is only used by atomics primitives when the `atomics` feature is -// enabled. In that mode we currently just use our own thread-local to store our -// current thread's ID, and then we lazily initialize it to something allocated -// from a global counter. -#[cfg(target_feature = "atomics")] -pub fn my_id() -> u32 { - use crate::sync::atomic::{AtomicU32, Ordering::SeqCst}; - - static NEXT_ID: AtomicU32 = AtomicU32::new(0); - - #[thread_local] - static mut MY_ID: u32 = 0; - - unsafe { - // If our thread ID isn't set yet then we need to allocate one. Do so - // with with a simple "atomically add to a global counter" strategy. - // This strategy doesn't handled what happens when the counter - // overflows, however, so just abort everything once the counter - // overflows and eventually we could have some sort of recycling scheme - // (or maybe this is all totally irrelevant by that point!). In any case - // though we're using a CAS loop instead of a `fetch_add` to ensure that - // the global counter never overflows. - if MY_ID == 0 { - let mut cur = NEXT_ID.load(SeqCst); - MY_ID = loop { - let next = cur.checked_add(1).unwrap_or_else(|| crate::arch::wasm32::unreachable()); - match NEXT_ID.compare_exchange(cur, next, SeqCst, SeqCst) { - Ok(_) => break next, - Err(i) => cur = i, - } - }; - } - MY_ID - } -} diff --git a/crux-mir/lib/std/src/sys/wasm/thread_local.rs b/crux-mir/lib/std/src/sys/wasm/thread_local.rs deleted file mode 100644 index f8be9863e..000000000 --- a/crux-mir/lib/std/src/sys/wasm/thread_local.rs +++ /dev/null @@ -1,26 +0,0 @@ -pub type Key = usize; - -#[inline] -pub unsafe fn create(_dtor: Option) -> Key { - panic!("should not be used on the wasm target"); -} - -#[inline] -pub unsafe fn set(_key: Key, _value: *mut u8) { - panic!("should not be used on the wasm target"); -} - -#[inline] -pub unsafe fn get(_key: Key) -> *mut u8 { - panic!("should not be used on the wasm target"); -} - -#[inline] -pub unsafe fn destroy(_key: Key) { - panic!("should not be used on the wasm target"); -} - -#[inline] -pub fn requires_synchronized_create() -> bool { - panic!("should not be used on the wasm target"); -} diff --git a/crux-mir/lib/std/src/sys/windows/alloc.rs b/crux-mir/lib/std/src/sys/windows/alloc.rs index 99b4d6c72..d53ea1600 100644 --- a/crux-mir/lib/std/src/sys/windows/alloc.rs +++ b/crux-mir/lib/std/src/sys/windows/alloc.rs @@ -1,61 +1,247 @@ +#![deny(unsafe_op_in_unsafe_fn)] + use crate::alloc::{GlobalAlloc, Layout, System}; +use crate::ffi::c_void; +use crate::ptr; +use crate::sync::atomic::{AtomicPtr, Ordering}; use crate::sys::c; -use crate::sys_common::alloc::{realloc_fallback, MIN_ALIGN}; +use crate::sys::common::alloc::{realloc_fallback, MIN_ALIGN}; -#[repr(C)] -struct Header(*mut u8); +#[cfg(test)] +mod tests; + +// Heap memory management on Windows is done by using the system Heap API (heapapi.h) +// See https://docs.microsoft.com/windows/win32/api/heapapi/ + +// Flag to indicate that the memory returned by `HeapAlloc` should be zeroed. +const HEAP_ZERO_MEMORY: c::DWORD = 0x00000008; -unsafe fn get_header<'a>(ptr: *mut u8) -> &'a mut Header { - &mut *(ptr as *mut Header).offset(-1) +#[link(name = "kernel32")] +extern "system" { + // Get a handle to the default heap of the current process, or null if the operation fails. + // + // SAFETY: Successful calls to this function within the same process are assumed to + // always return the same handle, which remains valid for the entire lifetime of the process. + // + // See https://docs.microsoft.com/windows/win32/api/heapapi/nf-heapapi-getprocessheap + fn GetProcessHeap() -> c::HANDLE; + + // Allocate a block of `dwBytes` bytes of memory from a given heap `hHeap`. + // The allocated memory may be uninitialized, or zeroed if `dwFlags` is + // set to `HEAP_ZERO_MEMORY`. + // + // Returns a pointer to the newly-allocated memory or null if the operation fails. + // The returned pointer will be aligned to at least `MIN_ALIGN`. + // + // SAFETY: + // - `hHeap` must be a non-null handle returned by `GetProcessHeap`. + // - `dwFlags` must be set to either zero or `HEAP_ZERO_MEMORY`. + // + // Note that `dwBytes` is allowed to be zero, contrary to some other allocators. + // + // See https://docs.microsoft.com/windows/win32/api/heapapi/nf-heapapi-heapalloc + fn HeapAlloc(hHeap: c::HANDLE, dwFlags: c::DWORD, dwBytes: c::SIZE_T) -> c::LPVOID; + + // Reallocate a block of memory behind a given pointer `lpMem` from a given heap `hHeap`, + // to a block of at least `dwBytes` bytes, either shrinking the block in place, + // or allocating at a new location, copying memory, and freeing the original location. + // + // Returns a pointer to the reallocated memory or null if the operation fails. + // The returned pointer will be aligned to at least `MIN_ALIGN`. + // If the operation fails the given block will never have been freed. + // + // SAFETY: + // - `hHeap` must be a non-null handle returned by `GetProcessHeap`. + // - `dwFlags` must be set to zero. + // - `lpMem` must be a non-null pointer to an allocated block returned by `HeapAlloc` or + // `HeapReAlloc`, that has not already been freed. + // If the block was successfully reallocated at a new location, pointers pointing to + // the freed memory, such as `lpMem`, must not be dereferenced ever again. + // + // Note that `dwBytes` is allowed to be zero, contrary to some other allocators. + // + // See https://docs.microsoft.com/windows/win32/api/heapapi/nf-heapapi-heaprealloc + fn HeapReAlloc( + hHeap: c::HANDLE, + dwFlags: c::DWORD, + lpMem: c::LPVOID, + dwBytes: c::SIZE_T, + ) -> c::LPVOID; + + // Free a block of memory behind a given pointer `lpMem` from a given heap `hHeap`. + // Returns a nonzero value if the operation is successful, and zero if the operation fails. + // + // SAFETY: + // - `hHeap` must be a non-null handle returned by `GetProcessHeap`. + // - `dwFlags` must be set to zero. + // - `lpMem` must be a pointer to an allocated block returned by `HeapAlloc` or `HeapReAlloc`, + // that has not already been freed. + // If the block was successfully freed, pointers pointing to the freed memory, such as `lpMem`, + // must not be dereferenced ever again. + // + // Note that `lpMem` is allowed to be null, which will not cause the operation to fail. + // + // See https://docs.microsoft.com/windows/win32/api/heapapi/nf-heapapi-heapfree + fn HeapFree(hHeap: c::HANDLE, dwFlags: c::DWORD, lpMem: c::LPVOID) -> c::BOOL; } -unsafe fn align_ptr(ptr: *mut u8, align: usize) -> *mut u8 { - let aligned = ptr.add(align - (ptr as usize & (align - 1))); - *get_header(aligned) = Header(ptr); - aligned +// Cached handle to the default heap of the current process. +// Either a non-null handle returned by `GetProcessHeap`, or null when not yet initialized or `GetProcessHeap` failed. +static HEAP: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + +// Get a handle to the default heap of the current process, or null if the operation fails. +// If this operation is successful, `HEAP` will be successfully initialized and contain +// a non-null handle returned by `GetProcessHeap`. +#[inline] +fn init_or_get_process_heap() -> c::HANDLE { + let heap = HEAP.load(Ordering::Relaxed); + if heap.is_null() { + // `HEAP` has not yet been successfully initialized + let heap = unsafe { GetProcessHeap() }; + if !heap.is_null() { + // SAFETY: No locking is needed because within the same process, + // successful calls to `GetProcessHeap` will always return the same value, even on different threads. + HEAP.store(heap, Ordering::Release); + + // SAFETY: `HEAP` contains a non-null handle returned by `GetProcessHeap` + heap + } else { + // Could not get the current process heap. + ptr::null_mut() + } + } else { + // SAFETY: `HEAP` contains a non-null handle returned by `GetProcessHeap` + heap + } } +// Get a non-null handle to the default heap of the current process. +// SAFETY: `HEAP` must have been successfully initialized. #[inline] -unsafe fn allocate_with_flags(layout: Layout, flags: c::DWORD) -> *mut u8 { - if layout.align() <= MIN_ALIGN { - return c::HeapAlloc(c::GetProcessHeap(), flags, layout.size()) as *mut u8; +unsafe fn get_process_heap() -> c::HANDLE { + HEAP.load(Ordering::Acquire) +} + +// Header containing a pointer to the start of an allocated block. +// SAFETY: Size and alignment must be <= `MIN_ALIGN`. +#[repr(C)] +struct Header(*mut u8); + +// Allocate a block of optionally zeroed memory for a given `layout`. +// SAFETY: Returns a pointer satisfying the guarantees of `System` about allocated pointers, +// or null if the operation fails. If this returns non-null `HEAP` will have been successfully +// initialized. +#[inline] +unsafe fn allocate(layout: Layout, zeroed: bool) -> *mut u8 { + let heap = init_or_get_process_heap(); + if heap.is_null() { + // Allocation has failed, could not get the current process heap. + return ptr::null_mut(); } - let size = layout.size() + layout.align(); - let ptr = c::HeapAlloc(c::GetProcessHeap(), flags, size); - if ptr.is_null() { ptr as *mut u8 } else { align_ptr(ptr as *mut u8, layout.align()) } + // Allocated memory will be either zeroed or uninitialized. + let flags = if zeroed { HEAP_ZERO_MEMORY } else { 0 }; + + if layout.align() <= MIN_ALIGN { + // SAFETY: `heap` is a non-null handle returned by `GetProcessHeap`. + // The returned pointer points to the start of an allocated block. + unsafe { HeapAlloc(heap, flags, layout.size()) as *mut u8 } + } else { + // Allocate extra padding in order to be able to satisfy the alignment. + let total = layout.align() + layout.size(); + + // SAFETY: `heap` is a non-null handle returned by `GetProcessHeap`. + let ptr = unsafe { HeapAlloc(heap, flags, total) as *mut u8 }; + if ptr.is_null() { + // Allocation has failed. + return ptr::null_mut(); + } + + // Create a correctly aligned pointer offset from the start of the allocated block, + // and write a header before it. + + let offset = layout.align() - (ptr.addr() & (layout.align() - 1)); + // SAFETY: `MIN_ALIGN` <= `offset` <= `layout.align()` and the size of the allocated + // block is `layout.align() + layout.size()`. `aligned` will thus be a correctly aligned + // pointer inside the allocated block with at least `layout.size()` bytes after it and at + // least `MIN_ALIGN` bytes of padding before it. + let aligned = unsafe { ptr.add(offset) }; + // SAFETY: Because the size and alignment of a header is <= `MIN_ALIGN` and `aligned` + // is aligned to at least `MIN_ALIGN` and has at least `MIN_ALIGN` bytes of padding before + // it, it is safe to write a header directly before it. + unsafe { ptr::write((aligned as *mut Header).sub(1), Header(ptr)) }; + + // SAFETY: The returned pointer does not point to the to the start of an allocated block, + // but there is a header readable directly before it containing the location of the start + // of the block. + aligned + } } +// All pointers returned by this allocator have, in addition to the guarantees of `GlobalAlloc`, the +// following properties: +// +// If the pointer was allocated or reallocated with a `layout` specifying an alignment <= `MIN_ALIGN` +// the pointer will be aligned to at least `MIN_ALIGN` and point to the start of the allocated block. +// +// If the pointer was allocated or reallocated with a `layout` specifying an alignment > `MIN_ALIGN` +// the pointer will be aligned to the specified alignment and not point to the start of the allocated block. +// Instead there will be a header readable directly before the returned pointer, containing the actual +// location of the start of the block. #[stable(feature = "alloc_system_type", since = "1.28.0")] unsafe impl GlobalAlloc for System { #[inline] unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - allocate_with_flags(layout, 0) + // SAFETY: Pointers returned by `allocate` satisfy the guarantees of `System` + let zeroed = false; + unsafe { allocate(layout, zeroed) } } #[inline] unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { - allocate_with_flags(layout, c::HEAP_ZERO_MEMORY) + // SAFETY: Pointers returned by `allocate` satisfy the guarantees of `System` + let zeroed = true; + unsafe { allocate(layout, zeroed) } } #[inline] unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - if layout.align() <= MIN_ALIGN { - let err = c::HeapFree(c::GetProcessHeap(), 0, ptr as c::LPVOID); - debug_assert!(err != 0, "Failed to free heap memory: {}", c::GetLastError()); - } else { - let header = get_header(ptr); - let err = c::HeapFree(c::GetProcessHeap(), 0, header.0 as c::LPVOID); - debug_assert!(err != 0, "Failed to free heap memory: {}", c::GetLastError()); - } + let block = { + if layout.align() <= MIN_ALIGN { + ptr + } else { + // The location of the start of the block is stored in the padding before `ptr`. + + // SAFETY: Because of the contract of `System`, `ptr` is guaranteed to be non-null + // and have a header readable directly before it. + unsafe { ptr::read((ptr as *mut Header).sub(1)).0 } + } + }; + + // SAFETY: because `ptr` has been successfully allocated with this allocator, + // `HEAP` must have been successfully initialized. + let heap = unsafe { get_process_heap() }; + + // SAFETY: `heap` is a non-null handle returned by `GetProcessHeap`, + // `block` is a pointer to the start of an allocated block. + unsafe { HeapFree(heap, 0, block as c::LPVOID) }; } #[inline] unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { if layout.align() <= MIN_ALIGN { - c::HeapReAlloc(c::GetProcessHeap(), 0, ptr as c::LPVOID, new_size) as *mut u8 + // SAFETY: because `ptr` has been successfully allocated with this allocator, + // `HEAP` must have been successfully initialized. + let heap = unsafe { get_process_heap() }; + + // SAFETY: `heap` is a non-null handle returned by `GetProcessHeap`, + // `ptr` is a pointer to the start of an allocated block. + // The returned pointer points to the start of an allocated block. + unsafe { HeapReAlloc(heap, 0, ptr as c::LPVOID, new_size) as *mut u8 } } else { - realloc_fallback(self, ptr, layout, new_size) + // SAFETY: `realloc_fallback` is implemented using `dealloc` and `alloc`, which will + // correctly handle `ptr` and return a pointer satisfying the guarantees of `System` + unsafe { realloc_fallback(self, ptr, layout, new_size) } } } } diff --git a/crux-mir/lib/std/src/sys/windows/alloc/tests.rs b/crux-mir/lib/std/src/sys/windows/alloc/tests.rs new file mode 100644 index 000000000..674a3e1d9 --- /dev/null +++ b/crux-mir/lib/std/src/sys/windows/alloc/tests.rs @@ -0,0 +1,9 @@ +use super::{Header, MIN_ALIGN}; +use crate::mem; + +#[test] +fn alloc_header() { + // Header must fit in the padding before an aligned pointer + assert!(mem::size_of::

() <= MIN_ALIGN); + assert!(mem::align_of::
() <= MIN_ALIGN); +} diff --git a/crux-mir/lib/std/src/sys/windows/args.rs b/crux-mir/lib/std/src/sys/windows/args.rs index 5fbea2a29..6741ae46d 100644 --- a/crux-mir/lib/std/src/sys/windows/args.rs +++ b/crux-mir/lib/std/src/sys/windows/args.rs @@ -1,24 +1,42 @@ -#![allow(dead_code)] // runtime init functions not used during testing +//! The Windows command line is just a string +//! +//! +//! This module implements the parsing necessary to turn that string into a list of arguments. + +#[cfg(test)] +mod tests; use crate::ffi::OsString; use crate::fmt; +use crate::io; +use crate::num::NonZeroU16; use crate::os::windows::prelude::*; use crate::path::PathBuf; -use crate::slice; use crate::sys::c; +use crate::sys::process::ensure_no_nuls; use crate::sys::windows::os::current_exe; +use crate::sys_common::wstr::WStrUnits; use crate::vec; -use core::iter; +use crate::iter; -pub unsafe fn init(_argc: isize, _argv: *const *const u8) {} - -pub unsafe fn cleanup() {} +/// This is the const equivalent to `NonZeroU16::new(n).unwrap()` +/// +/// FIXME: This can be removed once `Option::unwrap` is stably const. +/// See the `const_option` feature (#67441). +const fn non_zero_u16(n: u16) -> NonZeroU16 { + match NonZeroU16::new(n) { + Some(n) => n, + None => panic!("called `unwrap` on a `None` value"), + } +} pub fn args() -> Args { + // SAFETY: `GetCommandLineW` returns a pointer to a null terminated UTF-16 + // string so it's safe for `WStrUnits` to use. unsafe { let lp_cmd_line = c::GetCommandLineW(); - let parsed_args_list = parse_lp_cmd_line(lp_cmd_line as *const u16, || { + let parsed_args_list = parse_lp_cmd_line(WStrUnits::new(lp_cmd_line), || { current_exe().map(PathBuf::into_os_string).unwrap_or_else(|_| OsString::new()) }); @@ -29,129 +47,120 @@ pub fn args() -> Args { /// Implements the Windows command-line argument parsing algorithm. /// /// Microsoft's documentation for the Windows CLI argument format can be found at -/// . +/// +/// +/// A more in-depth explanation is here: +/// +/// +/// Windows includes a function to do command line parsing in shell32.dll. +/// However, this is not used for two reasons: /// -/// Windows includes a function to do this in shell32.dll, -/// but linking with that DLL causes the process to be registered as a GUI application. +/// 1. Linking with that DLL causes the process to be registered as a GUI application. /// GUI applications add a bunch of overhead, even if no windows are drawn. See /// . /// -/// This function was tested for equivalence to the shell32.dll implementation in -/// Windows 10 Pro v1803, using an exhaustive test suite available at -/// or -/// . -unsafe fn parse_lp_cmd_line OsString>( - lp_cmd_line: *const u16, +/// 2. It does not follow the modern C/C++ argv rules outlined in the first two links above. +/// +/// This function was tested for equivalence to the C/C++ parsing rules using an +/// extensive test suite available at +/// . +fn parse_lp_cmd_line<'a, F: Fn() -> OsString>( + lp_cmd_line: Option>, exe_name: F, ) -> Vec { - const BACKSLASH: u16 = '\\' as u16; - const QUOTE: u16 = '"' as u16; - const TAB: u16 = '\t' as u16; - const SPACE: u16 = ' ' as u16; + const BACKSLASH: NonZeroU16 = non_zero_u16(b'\\' as u16); + const QUOTE: NonZeroU16 = non_zero_u16(b'"' as u16); + const TAB: NonZeroU16 = non_zero_u16(b'\t' as u16); + const SPACE: NonZeroU16 = non_zero_u16(b' ' as u16); + let mut ret_val = Vec::new(); - if lp_cmd_line.is_null() || *lp_cmd_line == 0 { + // If the cmd line pointer is null or it points to an empty string then + // return the name of the executable as argv[0]. + if lp_cmd_line.as_ref().and_then(|cmd| cmd.peek()).is_none() { ret_val.push(exe_name()); return ret_val; } - let mut cmd_line = { - let mut end = 0; - while *lp_cmd_line.offset(end) != 0 { - end += 1; - } - slice::from_raw_parts(lp_cmd_line, end as usize) - }; + let mut code_units = lp_cmd_line.unwrap(); + // The executable name at the beginning is special. - cmd_line = match cmd_line[0] { - // The executable name ends at the next quote mark, - // no matter what. - QUOTE => { - let args = { - let mut cut = cmd_line[1..].splitn(2, |&c| c == QUOTE); - if let Some(exe) = cut.next() { - ret_val.push(OsString::from_wide(exe)); - } - cut.next() - }; - if let Some(args) = args { - args - } else { - return ret_val; - } - } - // Implement quirk: when they say whitespace here, - // they include the entire ASCII control plane: - // "However, if lpCmdLine starts with any amount of whitespace, CommandLineToArgvW - // will consider the first argument to be an empty string. Excess whitespace at the - // end of lpCmdLine is ignored." - 0..=SPACE => { - ret_val.push(OsString::new()); - &cmd_line[1..] - } - // The executable name ends at the next whitespace, - // no matter what. - _ => { - let args = { - let mut cut = cmd_line.splitn(2, |&c| c > 0 && c <= SPACE); - if let Some(exe) = cut.next() { - ret_val.push(OsString::from_wide(exe)); - } - cut.next() - }; - if let Some(args) = args { - args - } else { - return ret_val; - } + let mut in_quotes = false; + let mut cur = Vec::new(); + for w in &mut code_units { + match w { + // A quote mark always toggles `in_quotes` no matter what because + // there are no escape characters when parsing the executable name. + QUOTE => in_quotes = !in_quotes, + // If not `in_quotes` then whitespace ends argv[0]. + SPACE | TAB if !in_quotes => break, + // In all other cases the code unit is taken literally. + _ => cur.push(w.get()), } - }; + } + // Skip whitespace. + code_units.advance_while(|w| w == SPACE || w == TAB); + ret_val.push(OsString::from_wide(&cur)); + + // Parse the arguments according to these rules: + // * All code units are taken literally except space, tab, quote and backslash. + // * When not `in_quotes`, space and tab separate arguments. Consecutive spaces and tabs are + // treated as a single separator. + // * A space or tab `in_quotes` is taken literally. + // * A quote toggles `in_quotes` mode unless it's escaped. An escaped quote is taken literally. + // * A quote can be escaped if preceded by an odd number of backslashes. + // * If any number of backslashes is immediately followed by a quote then the number of + // backslashes is halved (rounding down). + // * Backslashes not followed by a quote are all taken literally. + // * If `in_quotes` then a quote can also be escaped using another quote + // (i.e. two consecutive quotes become one literal quote). let mut cur = Vec::new(); let mut in_quotes = false; - let mut was_in_quotes = false; - let mut backslash_count: usize = 0; - for &c in cmd_line { - match c { - // backslash - BACKSLASH => { - backslash_count += 1; - was_in_quotes = false; + while let Some(w) = code_units.next() { + match w { + // If not `in_quotes`, a space or tab ends the argument. + SPACE | TAB if !in_quotes => { + ret_val.push(OsString::from_wide(&cur[..])); + cur.truncate(0); + + // Skip whitespace. + code_units.advance_while(|w| w == SPACE || w == TAB); } - QUOTE if backslash_count % 2 == 0 => { - cur.extend(iter::repeat(b'\\' as u16).take(backslash_count / 2)); - backslash_count = 0; - if was_in_quotes { - cur.push('"' as u16); - was_in_quotes = false; + // Backslashes can escape quotes or backslashes but only if consecutive backslashes are followed by a quote. + BACKSLASH => { + let backslash_count = code_units.advance_while(|w| w == BACKSLASH) + 1; + if code_units.peek() == Some(QUOTE) { + cur.extend(iter::repeat(BACKSLASH.get()).take(backslash_count / 2)); + // The quote is escaped if there are an odd number of backslashes. + if backslash_count % 2 == 1 { + code_units.next(); + cur.push(QUOTE.get()); + } } else { - was_in_quotes = in_quotes; - in_quotes = !in_quotes; + // If there is no quote on the end then there is no escaping. + cur.extend(iter::repeat(BACKSLASH.get()).take(backslash_count)); } } - QUOTE if backslash_count % 2 != 0 => { - cur.extend(iter::repeat(b'\\' as u16).take(backslash_count / 2)); - backslash_count = 0; - was_in_quotes = false; - cur.push(b'"' as u16); - } - SPACE | TAB if !in_quotes => { - cur.extend(iter::repeat(b'\\' as u16).take(backslash_count)); - if !cur.is_empty() || was_in_quotes { - ret_val.push(OsString::from_wide(&cur[..])); - cur.truncate(0); + // If `in_quotes` and not backslash escaped (see above) then a quote either + // unsets `in_quote` or is escaped by another quote. + QUOTE if in_quotes => match code_units.peek() { + // Two consecutive quotes when `in_quotes` produces one literal quote. + Some(QUOTE) => { + cur.push(QUOTE.get()); + code_units.next(); } - backslash_count = 0; - was_in_quotes = false; - } - _ => { - cur.extend(iter::repeat(b'\\' as u16).take(backslash_count)); - backslash_count = 0; - was_in_quotes = false; - cur.push(c); - } + // Otherwise set `in_quotes`. + Some(_) => in_quotes = false, + // The end of the command line. + // Push `cur` even if empty, which we do by breaking while `in_quotes` is still set. + None => break, + }, + // If not `in_quotes` and not BACKSLASH escaped (see above) then a quote sets `in_quote`. + QUOTE => in_quotes = true, + // Everything else is always taken literally. + _ => cur.push(w.get()), } } - cur.extend(iter::repeat(b'\\' as u16).take(backslash_count)); - // include empty quoted strings at the end of the arguments list - if !cur.is_empty() || was_in_quotes || in_quotes { + // Push the final argument, if any. + if !cur.is_empty() || in_quotes { ret_val.push(OsString::from_wide(&cur[..])); } ret_val @@ -161,19 +170,9 @@ pub struct Args { parsed_args_list: vec::IntoIter, } -pub struct ArgsInnerDebug<'a> { - args: &'a Args, -} - -impl<'a> fmt::Debug for ArgsInnerDebug<'a> { +impl fmt::Debug for Args { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.args.parsed_args_list.as_slice().fmt(f) - } -} - -impl Args { - pub fn inner_debug(&self) -> ArgsInnerDebug<'_> { - ArgsInnerDebug { args: self } + self.parsed_args_list.as_slice().fmt(f) } } @@ -199,68 +198,159 @@ impl ExactSizeIterator for Args { } } -#[cfg(test)] -mod tests { - use crate::ffi::OsString; - use crate::sys::windows::args::*; +#[derive(Debug)] +pub(crate) enum Arg { + /// Add quotes (if needed) + Regular(OsString), + /// Append raw string without quoting + Raw(OsString), +} + +enum Quote { + // Every arg is quoted + Always, + // Whitespace and empty args are quoted + Auto, + // Arg appended without any changes (#29494) + Never, +} + +pub(crate) fn append_arg(cmd: &mut Vec, arg: &Arg, force_quotes: bool) -> io::Result<()> { + let (arg, quote) = match arg { + Arg::Regular(arg) => (arg, if force_quotes { Quote::Always } else { Quote::Auto }), + Arg::Raw(arg) => (arg, Quote::Never), + }; + + // If an argument has 0 characters then we need to quote it to ensure + // that it actually gets passed through on the command line or otherwise + // it will be dropped entirely when parsed on the other end. + ensure_no_nuls(arg)?; + let arg_bytes = arg.bytes(); + let (quote, escape) = match quote { + Quote::Always => (true, true), + Quote::Auto => { + (arg_bytes.iter().any(|c| *c == b' ' || *c == b'\t') || arg_bytes.is_empty(), true) + } + Quote::Never => (false, false), + }; + if quote { + cmd.push('"' as u16); + } - fn chk(string: &str, parts: &[&str]) { - let mut wide: Vec = OsString::from(string).encode_wide().collect(); - wide.push(0); - let parsed = unsafe { - parse_lp_cmd_line(wide.as_ptr() as *const u16, || OsString::from("TEST.EXE")) - }; - let expected: Vec = parts.iter().map(|k| OsString::from(k)).collect(); - assert_eq!(parsed.as_slice(), expected.as_slice()); + let mut backslashes: usize = 0; + for x in arg.encode_wide() { + if escape { + if x == '\\' as u16 { + backslashes += 1; + } else { + if x == '"' as u16 { + // Add n+1 backslashes to total 2n+1 before internal '"'. + cmd.extend((0..=backslashes).map(|_| '\\' as u16)); + } + backslashes = 0; + } + } + cmd.push(x); } - #[test] - fn empty() { - chk("", &["TEST.EXE"]); - chk("\0", &["TEST.EXE"]); + if quote { + // Add n backslashes to total 2n before ending '"'. + cmd.extend((0..backslashes).map(|_| '\\' as u16)); + cmd.push('"' as u16); } + Ok(()) +} - #[test] - fn single_words() { - chk("EXE one_word", &["EXE", "one_word"]); - chk("EXE a", &["EXE", "a"]); - chk("EXE 😅", &["EXE", "😅"]); - chk("EXE 😅🤦", &["EXE", "😅🤦"]); +pub(crate) fn make_bat_command_line( + script: &[u16], + args: &[Arg], + force_quotes: bool, +) -> io::Result> { + // Set the start of the command line to `cmd.exe /c "` + // It is necessary to surround the command in an extra pair of quotes, + // hence the trailing quote here. It will be closed after all arguments + // have been added. + let mut cmd: Vec = "cmd.exe /c \"".encode_utf16().collect(); + + // Push the script name surrounded by its quote pair. + cmd.push(b'"' as u16); + // Windows file names cannot contain a `"` character or end with `\\`. + // If the script name does then return an error. + if script.contains(&(b'"' as u16)) || script.last() == Some(&(b'\\' as u16)) { + return Err(io::const_io_error!( + io::ErrorKind::InvalidInput, + "Windows file names may not contain `\"` or end with `\\`" + )); } + cmd.extend_from_slice(script.strip_suffix(&[0]).unwrap_or(script)); + cmd.push(b'"' as u16); - #[test] - fn official_examples() { - chk(r#"EXE "abc" d e"#, &["EXE", "abc", "d", "e"]); - chk(r#"EXE a\\\b d"e f"g h"#, &["EXE", r#"a\\\b"#, "de fg", "h"]); - chk(r#"EXE a\\\"b c d"#, &["EXE", r#"a\"b"#, "c", "d"]); - chk(r#"EXE a\\\\"b c" d e"#, &["EXE", r#"a\\b c"#, "d", "e"]); + // Append the arguments. + // FIXME: This needs tests to ensure that the arguments are properly + // reconstructed by the batch script by default. + for arg in args { + cmd.push(' ' as u16); + append_arg(&mut cmd, arg, force_quotes)?; } - #[test] - fn whitespace_behavior() { - chk(r#" test"#, &["", "test"]); - chk(r#" test"#, &["", "test"]); - chk(r#" test test2"#, &["", "test", "test2"]); - chk(r#" test test2"#, &["", "test", "test2"]); - chk(r#"test test2 "#, &["test", "test2"]); - chk(r#"test test2 "#, &["test", "test2"]); - chk(r#"test "#, &["test"]); + // Close the quote we left opened earlier. + cmd.push(b'"' as u16); + + Ok(cmd) +} + +/// Takes a path and tries to return a non-verbatim path. +/// +/// This is necessary because cmd.exe does not support verbatim paths. +pub(crate) fn to_user_path(mut path: Vec) -> io::Result> { + use crate::ptr; + use crate::sys::windows::fill_utf16_buf; + + // UTF-16 encoded code points, used in parsing and building UTF-16 paths. + // All of these are in the ASCII range so they can be cast directly to `u16`. + const SEP: u16 = b'\\' as _; + const QUERY: u16 = b'?' as _; + const COLON: u16 = b':' as _; + const U: u16 = b'U' as _; + const N: u16 = b'N' as _; + const C: u16 = b'C' as _; + + // Early return if the path is too long to remove the verbatim prefix. + const LEGACY_MAX_PATH: usize = 260; + if path.len() > LEGACY_MAX_PATH { + return Ok(path); } - #[test] - fn genius_quotes() { - chk(r#"EXE "" """#, &["EXE", "", ""]); - chk(r#"EXE "" """"#, &["EXE", "", "\""]); - chk( - r#"EXE "this is """all""" in the same argument""#, - &["EXE", "this is \"all\" in the same argument"], - ); - chk(r#"EXE "a"""#, &["EXE", "a\""]); - chk(r#"EXE "a"" a"#, &["EXE", "a\"", "a"]); - // quotes cannot be escaped in command names - chk(r#""EXE" check"#, &["EXE", "check"]); - chk(r#""EXE check""#, &["EXE check"]); - chk(r#""EXE """for""" check"#, &["EXE ", r#"for""#, "check"]); - chk(r#""EXE \"for\" check"#, &[r#"EXE \"#, r#"for""#, "check"]); + match &path[..] { + // `\\?\C:\...` => `C:\...` + [SEP, SEP, QUERY, SEP, _, COLON, SEP, ..] => unsafe { + let lpfilename = path[4..].as_ptr(); + fill_utf16_buf( + |buffer, size| c::GetFullPathNameW(lpfilename, size, buffer, ptr::null_mut()), + |full_path: &[u16]| { + if full_path == &path[4..path.len() - 1] { full_path.into() } else { path } + }, + ) + }, + // `\\?\UNC\...` => `\\...` + [SEP, SEP, QUERY, SEP, U, N, C, SEP, ..] => unsafe { + // Change the `C` in `UNC\` to `\` so we can get a slice that starts with `\\`. + path[6] = b'\\' as u16; + let lpfilename = path[6..].as_ptr(); + fill_utf16_buf( + |buffer, size| c::GetFullPathNameW(lpfilename, size, buffer, ptr::null_mut()), + |full_path: &[u16]| { + if full_path == &path[6..path.len() - 1] { + full_path.into() + } else { + // Restore the 'C' in "UNC". + path[6] = b'C' as u16; + path + } + }, + ) + }, + // For everything else, leave the path unchanged. + _ => Ok(path), } } diff --git a/crux-mir/lib/std/src/sys/windows/args/tests.rs b/crux-mir/lib/std/src/sys/windows/args/tests.rs new file mode 100644 index 000000000..82c32d08c --- /dev/null +++ b/crux-mir/lib/std/src/sys/windows/args/tests.rs @@ -0,0 +1,91 @@ +use crate::ffi::OsString; +use crate::sys::windows::args::*; + +fn chk(string: &str, parts: &[&str]) { + let mut wide: Vec = OsString::from(string).encode_wide().collect(); + wide.push(0); + let parsed = + unsafe { parse_lp_cmd_line(WStrUnits::new(wide.as_ptr()), || OsString::from("TEST.EXE")) }; + let expected: Vec = parts.iter().map(|k| OsString::from(k)).collect(); + assert_eq!(parsed.as_slice(), expected.as_slice(), "{:?}", string); +} + +#[test] +fn empty() { + chk("", &["TEST.EXE"]); + chk("\0", &["TEST.EXE"]); +} + +#[test] +fn single_words() { + chk("EXE one_word", &["EXE", "one_word"]); + chk("EXE a", &["EXE", "a"]); + chk("EXE 😅", &["EXE", "😅"]); + chk("EXE 😅🤦", &["EXE", "😅🤦"]); +} + +#[test] +fn official_examples() { + chk(r#"EXE "abc" d e"#, &["EXE", "abc", "d", "e"]); + chk(r#"EXE a\\\b d"e f"g h"#, &["EXE", r"a\\\b", "de fg", "h"]); + chk(r#"EXE a\\\"b c d"#, &["EXE", r#"a\"b"#, "c", "d"]); + chk(r#"EXE a\\\\"b c" d e"#, &["EXE", r"a\\b c", "d", "e"]); +} + +#[test] +fn whitespace_behavior() { + chk(" test", &["", "test"]); + chk(" test", &["", "test"]); + chk(" test test2", &["", "test", "test2"]); + chk(" test test2", &["", "test", "test2"]); + chk("test test2 ", &["test", "test2"]); + chk("test test2 ", &["test", "test2"]); + chk("test ", &["test"]); +} + +#[test] +fn genius_quotes() { + chk(r#"EXE "" """#, &["EXE", "", ""]); + chk(r#"EXE "" """"#, &["EXE", "", r#"""#]); + chk( + r#"EXE "this is """all""" in the same argument""#, + &["EXE", r#"this is "all" in the same argument"#], + ); + chk(r#"EXE "a"""#, &["EXE", r#"a""#]); + chk(r#"EXE "a"" a"#, &["EXE", r#"a" a"#]); + // quotes cannot be escaped in command names + chk(r#""EXE" check"#, &["EXE", "check"]); + chk(r#""EXE check""#, &["EXE check"]); + chk(r#""EXE """for""" check"#, &["EXE for check"]); + chk(r#""EXE \"for\" check"#, &[r"EXE \for\ check"]); + chk(r#""EXE \" for \" check"#, &[r"EXE \", "for", r#"""#, "check"]); + chk(r#"E"X"E test"#, &["EXE", "test"]); + chk(r#"EX""E test"#, &["EXE", "test"]); +} + +// from https://daviddeley.com/autohotkey/parameters/parameters.htm#WINCRULESEX +#[test] +fn post_2008() { + chk("EXE CallMeIshmael", &["EXE", "CallMeIshmael"]); + chk(r#"EXE "Call Me Ishmael""#, &["EXE", "Call Me Ishmael"]); + chk(r#"EXE Cal"l Me I"shmael"#, &["EXE", "Call Me Ishmael"]); + chk(r#"EXE CallMe\"Ishmael"#, &["EXE", r#"CallMe"Ishmael"#]); + chk(r#"EXE "CallMe\"Ishmael""#, &["EXE", r#"CallMe"Ishmael"#]); + chk(r#"EXE "Call Me Ishmael\\""#, &["EXE", r"Call Me Ishmael\"]); + chk(r#"EXE "CallMe\\\"Ishmael""#, &["EXE", r#"CallMe\"Ishmael"#]); + chk(r#"EXE a\\\b"#, &["EXE", r"a\\\b"]); + chk(r#"EXE "a\\\b""#, &["EXE", r"a\\\b"]); + chk(r#"EXE "\"Call Me Ishmael\"""#, &["EXE", r#""Call Me Ishmael""#]); + chk(r#"EXE "C:\TEST A\\""#, &["EXE", r"C:\TEST A\"]); + chk(r#"EXE "\"C:\TEST A\\\"""#, &["EXE", r#""C:\TEST A\""#]); + chk(r#"EXE "a b c" d e"#, &["EXE", "a b c", "d", "e"]); + chk(r#"EXE "ab\"c" "\\" d"#, &["EXE", r#"ab"c"#, r"\", "d"]); + chk(r#"EXE a\\\b d"e f"g h"#, &["EXE", r"a\\\b", "de fg", "h"]); + chk(r#"EXE a\\\"b c d"#, &["EXE", r#"a\"b"#, "c", "d"]); + chk(r#"EXE a\\\\"b c" d e"#, &["EXE", r"a\\b c", "d", "e"]); + // Double Double Quotes + chk(r#"EXE "a b c"""#, &["EXE", r#"a b c""#]); + chk(r#"EXE """CallMeIshmael""" b c"#, &["EXE", r#""CallMeIshmael""#, "b", "c"]); + chk(r#"EXE """Call Me Ishmael""""#, &["EXE", r#""Call Me Ishmael""#]); + chk(r#"EXE """"Call Me Ishmael"" b c"#, &["EXE", r#""Call"#, "Me", "Ishmael", "b", "c"]); +} diff --git a/crux-mir/lib/std/src/sys/windows/c.rs b/crux-mir/lib/std/src/sys/windows/c.rs index 7f93ef879..81461de4f 100644 --- a/crux-mir/lib/std/src/sys/windows/c.rs +++ b/crux-mir/lib/std/src/sys/windows/c.rs @@ -4,15 +4,25 @@ #![cfg_attr(test, allow(dead_code))] #![unstable(issue = "none", feature = "windows_c")] +use crate::ffi::CStr; +use crate::mem; use crate::os::raw::{c_char, c_int, c_long, c_longlong, c_uint, c_ulong, c_ushort}; +use crate::os::windows::io::{BorrowedHandle, HandleOrInvalid, HandleOrNull}; use crate::ptr; +use core::ffi::NonZero_c_ulong; use libc::{c_void, size_t, wchar_t}; +#[path = "c/errors.rs"] // c.rs is included from two places so we need to specify this +mod errors; +pub use errors::*; + pub use self::EXCEPTION_DISPOSITION::*; pub use self::FILE_INFO_BY_HANDLE_CLASS::*; +pub type DWORD_PTR = ULONG_PTR; pub type DWORD = c_ulong; +pub type NonZeroDWORD = NonZero_c_ulong; pub type HANDLE = LPVOID; pub type HINSTANCE = HANDLE; pub type HMODULE = HINSTANCE; @@ -29,8 +39,11 @@ pub type USHORT = c_ushort; pub type SIZE_T = usize; pub type WORD = u16; pub type CHAR = c_char; +pub type CCHAR = c_char; pub type ULONG_PTR = usize; pub type ULONG = c_ulong; +pub type NTSTATUS = LONG; +pub type ACCESS_MASK = DWORD; pub type LPBOOL = *mut BOOL; pub type LPBYTE = *mut BYTE; @@ -43,20 +56,23 @@ pub type LPPROCESS_INFORMATION = *mut PROCESS_INFORMATION; pub type LPSECURITY_ATTRIBUTES = *mut SECURITY_ATTRIBUTES; pub type LPSTARTUPINFO = *mut STARTUPINFO; pub type LPVOID = *mut c_void; +pub type LPCVOID = *const c_void; pub type LPWCH = *mut WCHAR; pub type LPWIN32_FIND_DATAW = *mut WIN32_FIND_DATAW; pub type LPWSADATA = *mut WSADATA; pub type LPWSAPROTOCOL_INFO = *mut WSAPROTOCOL_INFO; -pub type LPSTR = *mut CHAR; pub type LPWSTR = *mut WCHAR; pub type LPFILETIME = *mut FILETIME; +pub type LPSYSTEM_INFO = *mut SYSTEM_INFO; pub type LPWSABUF = *mut WSABUF; pub type LPWSAOVERLAPPED = *mut c_void; pub type LPWSAOVERLAPPED_COMPLETION_ROUTINE = *mut c_void; +pub type BCRYPT_ALG_HANDLE = LPVOID; pub type PCONDITION_VARIABLE = *mut CONDITION_VARIABLE; pub type PLARGE_INTEGER = *mut c_longlong; pub type PSRWLOCK = *mut SRWLOCK; +pub type LPINIT_ONCE = *mut INIT_ONCE; pub type SOCKET = crate::os::windows::raw::SOCKET; pub type socklen_t = c_int; @@ -65,24 +81,35 @@ pub type ADDRESS_FAMILY = USHORT; pub const TRUE: BOOL = 1; pub const FALSE: BOOL = 0; +pub const CSTR_LESS_THAN: c_int = 1; +pub const CSTR_EQUAL: c_int = 2; +pub const CSTR_GREATER_THAN: c_int = 3; + pub const FILE_ATTRIBUTE_READONLY: DWORD = 0x1; pub const FILE_ATTRIBUTE_DIRECTORY: DWORD = 0x10; pub const FILE_ATTRIBUTE_REPARSE_POINT: DWORD = 0x400; +pub const INVALID_FILE_ATTRIBUTES: DWORD = DWORD::MAX; pub const FILE_SHARE_DELETE: DWORD = 0x4; pub const FILE_SHARE_READ: DWORD = 0x1; pub const FILE_SHARE_WRITE: DWORD = 0x2; +pub const FILE_OPEN: ULONG = 0x00000001; +pub const FILE_OPEN_REPARSE_POINT: ULONG = 0x200000; +pub const OBJ_DONT_REPARSE: ULONG = 0x1000; + pub const CREATE_ALWAYS: DWORD = 2; pub const CREATE_NEW: DWORD = 1; pub const OPEN_ALWAYS: DWORD = 4; pub const OPEN_EXISTING: DWORD = 3; pub const TRUNCATE_EXISTING: DWORD = 5; +pub const FILE_LIST_DIRECTORY: DWORD = 0x1; pub const FILE_WRITE_DATA: DWORD = 0x00000002; pub const FILE_APPEND_DATA: DWORD = 0x00000004; pub const FILE_WRITE_EA: DWORD = 0x00000010; pub const FILE_WRITE_ATTRIBUTES: DWORD = 0x00000100; +pub const DELETE: DWORD = 0x10000; pub const READ_CONTROL: DWORD = 0x00020000; pub const SYNCHRONIZE: DWORD = 0x00100000; pub const GENERIC_READ: DWORD = 0x80000000; @@ -101,6 +128,10 @@ pub const SECURITY_SQOS_PRESENT: DWORD = 0x00100000; pub const FIONBIO: c_ulong = 0x8004667e; +pub const MAX_PATH: usize = 260; + +pub const FILE_TYPE_PIPE: u32 = 3; + #[repr(C)] #[derive(Copy)] pub struct WIN32_FIND_DATAW { @@ -129,19 +160,6 @@ pub const WSASYS_STATUS_LEN: usize = 128; pub const WSAPROTOCOL_LEN: DWORD = 255; pub const INVALID_SOCKET: SOCKET = !0; -pub const WSAEACCES: c_int = 10013; -pub const WSAEINVAL: c_int = 10022; -pub const WSAEWOULDBLOCK: c_int = 10035; -pub const WSAEPROTOTYPE: c_int = 10041; -pub const WSAEADDRINUSE: c_int = 10048; -pub const WSAEADDRNOTAVAIL: c_int = 10049; -pub const WSAECONNABORTED: c_int = 10053; -pub const WSAECONNRESET: c_int = 10054; -pub const WSAENOTCONN: c_int = 10057; -pub const WSAESHUTDOWN: c_int = 10058; -pub const WSAETIMEDOUT: c_int = 10060; -pub const WSAECONNREFUSED: c_int = 10061; - pub const MAX_PROTOCOL_CHAIN: DWORD = 7; pub const MAXIMUM_REPARSE_DATA_BUFFER_SIZE: usize = 16 * 1024; @@ -161,27 +179,9 @@ pub const STD_ERROR_HANDLE: DWORD = -12i32 as DWORD; pub const PROGRESS_CONTINUE: DWORD = 0; -pub const ERROR_FILE_NOT_FOUND: DWORD = 2; -pub const ERROR_PATH_NOT_FOUND: DWORD = 3; -pub const ERROR_ACCESS_DENIED: DWORD = 5; -pub const ERROR_INVALID_HANDLE: DWORD = 6; -pub const ERROR_NO_MORE_FILES: DWORD = 18; -pub const ERROR_HANDLE_EOF: DWORD = 38; -pub const ERROR_FILE_EXISTS: DWORD = 80; -pub const ERROR_INVALID_PARAMETER: DWORD = 87; -pub const ERROR_BROKEN_PIPE: DWORD = 109; -pub const ERROR_CALL_NOT_IMPLEMENTED: DWORD = 120; -pub const ERROR_INSUFFICIENT_BUFFER: DWORD = 122; -pub const ERROR_ALREADY_EXISTS: DWORD = 183; -pub const ERROR_NO_DATA: DWORD = 232; -pub const ERROR_ENVVAR_NOT_FOUND: DWORD = 203; -pub const ERROR_OPERATION_ABORTED: DWORD = 995; -pub const ERROR_IO_PENDING: DWORD = 997; -pub const ERROR_TIMEOUT: DWORD = 0x5B4; - pub const E_NOTIMPL: HRESULT = 0x80004001u32 as HRESULT; -pub const INVALID_HANDLE_VALUE: HANDLE = !0 as HANDLE; +pub const INVALID_HANDLE_VALUE: HANDLE = ptr::invalid_mut(!0); pub const FACILITY_NT_BIT: DWORD = 0x1000_0000; @@ -200,6 +200,9 @@ pub const DUPLICATE_SAME_ACCESS: DWORD = 0x00000002; pub const CONDITION_VARIABLE_INIT: CONDITION_VARIABLE = CONDITION_VARIABLE { ptr: ptr::null_mut() }; pub const SRWLOCK_INIT: SRWLOCK = SRWLOCK { ptr: ptr::null_mut() }; +pub const INIT_ONCE_STATIC_INIT: INIT_ONCE = INIT_ONCE { ptr: ptr::null_mut() }; + +pub const INIT_ONCE_INIT_FAILED: DWORD = 0x00000004; pub const DETACHED_PROCESS: DWORD = 0x00000008; pub const CREATE_NEW_PROCESS_GROUP: DWORD = 0x00000200; @@ -213,10 +216,11 @@ pub const SD_RECEIVE: c_int = 0; pub const SD_SEND: c_int = 1; pub const SOCK_DGRAM: c_int = 2; pub const SOCK_STREAM: c_int = 1; +pub const SOCKET_ERROR: c_int = -1; pub const SOL_SOCKET: c_int = 0xffff; +pub const SO_LINGER: c_int = 0x0080; pub const SO_RCVTIMEO: c_int = 0x1006; pub const SO_SNDTIMEO: c_int = 0x1005; -pub const SO_REUSEADDR: c_int = 0x0004; pub const IPPROTO_IP: c_int = 0; pub const IPPROTO_TCP: c_int = 6; pub const IPPROTO_IPV6: c_int = 41; @@ -234,6 +238,13 @@ pub const IPV6_ADD_MEMBERSHIP: c_int = 12; pub const IPV6_DROP_MEMBERSHIP: c_int = 13; pub const MSG_PEEK: c_int = 0x2; +#[repr(C)] +#[derive(Copy, Clone)] +pub struct linger { + pub l_onoff: c_ushort, + pub l_linger: c_ushort, +} + #[repr(C)] pub struct ip_mreq { pub imr_multiaddr: in_addr, @@ -270,7 +281,97 @@ pub const FD_SETSIZE: usize = 64; pub const STACK_SIZE_PARAM_IS_A_RESERVATION: DWORD = 0x00010000; -pub const HEAP_ZERO_MEMORY: DWORD = 0x00000008; +pub const STATUS_SUCCESS: NTSTATUS = 0x00000000; +pub const STATUS_DELETE_PENDING: NTSTATUS = 0xc0000056_u32 as _; +pub const STATUS_INVALID_PARAMETER: NTSTATUS = 0xc000000d_u32 as _; + +pub const STATUS_PENDING: NTSTATUS = 0x103 as _; +pub const STATUS_END_OF_FILE: NTSTATUS = 0xC0000011_u32 as _; +pub const STATUS_NOT_IMPLEMENTED: NTSTATUS = 0xC0000002_u32 as _; + +// Equivalent to the `NT_SUCCESS` C preprocessor macro. +// See: https://docs.microsoft.com/en-us/windows-hardware/drivers/kernel/using-ntstatus-values +pub fn nt_success(status: NTSTATUS) -> bool { + status >= 0 +} + +// "RNG\0" +pub const BCRYPT_RNG_ALGORITHM: &[u16] = &[b'R' as u16, b'N' as u16, b'G' as u16, 0]; +pub const BCRYPT_USE_SYSTEM_PREFERRED_RNG: DWORD = 0x00000002; + +#[repr(C)] +pub struct UNICODE_STRING { + pub Length: u16, + pub MaximumLength: u16, + pub Buffer: *mut u16, +} +impl UNICODE_STRING { + pub fn from_ref(slice: &[u16]) -> Self { + let len = slice.len() * mem::size_of::(); + Self { Length: len as _, MaximumLength: len as _, Buffer: slice.as_ptr() as _ } + } +} +#[repr(C)] +pub struct OBJECT_ATTRIBUTES { + pub Length: ULONG, + pub RootDirectory: HANDLE, + pub ObjectName: *const UNICODE_STRING, + pub Attributes: ULONG, + pub SecurityDescriptor: *mut c_void, + pub SecurityQualityOfService: *mut c_void, +} +impl Default for OBJECT_ATTRIBUTES { + fn default() -> Self { + Self { + Length: mem::size_of::() as _, + RootDirectory: ptr::null_mut(), + ObjectName: ptr::null_mut(), + Attributes: 0, + SecurityDescriptor: ptr::null_mut(), + SecurityQualityOfService: ptr::null_mut(), + } + } +} +#[repr(C)] +union IO_STATUS_BLOCK_union { + Status: NTSTATUS, + Pointer: *mut c_void, +} +impl Default for IO_STATUS_BLOCK_union { + fn default() -> Self { + let mut this = Self { Pointer: ptr::null_mut() }; + this.Status = STATUS_PENDING; + this + } +} +#[repr(C)] +#[derive(Default)] +pub struct IO_STATUS_BLOCK { + u: IO_STATUS_BLOCK_union, + pub Information: usize, +} +impl IO_STATUS_BLOCK { + pub fn status(&self) -> NTSTATUS { + // SAFETY: If `self.u.Status` was set then this is obviously safe. + // If `self.u.Pointer` was set then this is the equivalent to converting + // the pointer to an integer, which is also safe. + // Currently the only safe way to construct `IO_STATUS_BLOCK` outside of + // this module is to call the `default` method, which sets the `Status`. + unsafe { self.u.Status } + } +} + +pub type LPOVERLAPPED_COMPLETION_ROUTINE = unsafe extern "system" fn( + dwErrorCode: DWORD, + dwNumberOfBytesTransferred: DWORD, + lpOverlapped: *mut OVERLAPPED, +); + +type IO_APC_ROUTINE = unsafe extern "system" fn( + ApcContext: *mut c_void, + IoStatusBlock: *mut IO_STATUS_BLOCK, + Reserved: ULONG, +); #[repr(C)] #[cfg(not(target_pointer_width = "64"))] @@ -361,9 +462,49 @@ pub enum FILE_INFO_BY_HANDLE_CLASS { FileIdInfo = 18, // 0x12 FileIdExtdDirectoryInfo = 19, // 0x13 FileIdExtdDirectoryRestartInfo = 20, // 0x14 + FileDispositionInfoEx = 21, // 0x15, Windows 10 version 1607 MaximumFileInfoByHandlesClass, } +#[repr(C)] +pub struct FILE_ATTRIBUTE_TAG_INFO { + pub FileAttributes: DWORD, + pub ReparseTag: DWORD, +} + +#[repr(C)] +pub struct FILE_DISPOSITION_INFO { + pub DeleteFile: BOOLEAN, +} + +pub const FILE_DISPOSITION_DELETE: DWORD = 0x1; +pub const FILE_DISPOSITION_POSIX_SEMANTICS: DWORD = 0x2; +pub const FILE_DISPOSITION_IGNORE_READONLY_ATTRIBUTE: DWORD = 0x10; + +#[repr(C)] +pub struct FILE_DISPOSITION_INFO_EX { + pub Flags: DWORD, +} + +#[repr(C)] +#[derive(Default)] +pub struct FILE_ID_BOTH_DIR_INFO { + pub NextEntryOffset: DWORD, + pub FileIndex: DWORD, + pub CreationTime: LARGE_INTEGER, + pub LastAccessTime: LARGE_INTEGER, + pub LastWriteTime: LARGE_INTEGER, + pub ChangeTime: LARGE_INTEGER, + pub EndOfFile: LARGE_INTEGER, + pub AllocationSize: LARGE_INTEGER, + pub FileAttributes: DWORD, + pub FileNameLength: DWORD, + pub EaSize: DWORD, + pub ShortNameLength: CCHAR, + pub ShortName: [WCHAR; 12], + pub FileId: LARGE_INTEGER, + pub FileName: [WCHAR; 1], +} #[repr(C)] pub struct FILE_BASIC_INFO { pub CreationTime: LARGE_INTEGER, @@ -378,6 +519,8 @@ pub struct FILE_END_OF_FILE_INFO { pub EndOfFile: LARGE_INTEGER, } +/// NB: Use carefully! In general using this as a reference is likely to get the +/// provenance wrong for the `rest` field! #[repr(C)] pub struct REPARSE_DATA_BUFFER { pub ReparseTag: c_uint, @@ -386,6 +529,8 @@ pub struct REPARSE_DATA_BUFFER { pub rest: (), } +/// NB: Use carefully! In general using this as a reference is likely to get the +/// provenance wrong for the `PathBuffer` field! #[repr(C)] pub struct SYMBOLIC_LINK_REPARSE_BUFFER { pub SubstituteNameOffset: c_ushort, @@ -396,6 +541,14 @@ pub struct SYMBOLIC_LINK_REPARSE_BUFFER { pub PathBuffer: WCHAR, } +/// NB: Use carefully! In general using this as a reference is likely to get the +/// provenance wrong for the `PathBuffer` field! +#[repr(C)] +pub struct FILE_NAME_INFO { + pub FileNameLength: DWORD, + pub FileName: [WCHAR; 1], +} + #[repr(C)] pub struct MOUNT_POINT_REPARSE_BUFFER { pub SubstituteNameOffset: c_ushort, @@ -428,13 +581,8 @@ pub struct SRWLOCK { pub ptr: LPVOID, } #[repr(C)] -pub struct CRITICAL_SECTION { - CriticalSectionDebug: LPVOID, - LockCount: LONG, - RecursionCount: LONG, - OwningThread: HANDLE, - LockSemaphore: HANDLE, - SpinCount: ULONG_PTR, +pub struct INIT_ONCE { + pub ptr: LPVOID, } #[repr(C)] @@ -506,12 +654,27 @@ pub struct SOCKADDR { } #[repr(C)] -#[derive(Copy, Clone)] +#[derive(Copy, Clone, Debug, Default)] pub struct FILETIME { pub dwLowDateTime: DWORD, pub dwHighDateTime: DWORD, } +#[repr(C)] +pub struct SYSTEM_INFO { + pub wProcessorArchitecture: WORD, + pub wReserved: WORD, + pub dwPageSize: DWORD, + pub lpMinimumApplicationAddress: LPVOID, + pub lpMaximumApplicationAddress: LPVOID, + pub dwActiveProcessorMask: DWORD_PTR, + pub dwNumberOfProcessors: DWORD, + pub dwProcessorType: DWORD, + pub dwAllocationGranularity: DWORD, + pub wProcessorLevel: WORD, + pub wProcessorRevision: WORD, +} + #[repr(C)] pub struct OVERLAPPED { pub Internal: *mut c_ulong, @@ -611,7 +774,17 @@ pub struct timeval { pub tv_usec: c_long, } -// Functions forbidden when targeting UWP +#[repr(C)] +#[derive(Copy, Clone)] +pub struct CONSOLE_READCONSOLE_CONTROL { + pub nLength: ULONG, + pub nInitialChars: ULONG, + pub dwCtrlWakeupMask: ULONG, + pub dwControlKeyState: ULONG, +} +pub type PCONSOLE_READCONSOLE_CONTROL = *mut CONSOLE_READCONSOLE_CONTROL; + +// Desktop specific functions & types cfg_if::cfg_if! { if #[cfg(not(target_vendor = "uwp"))] { pub const EXCEPTION_CONTINUE_SEARCH: LONG = 0; @@ -625,7 +798,7 @@ if #[cfg(not(target_vendor = "uwp"))] { pub ExceptionRecord: *mut EXCEPTION_RECORD, pub ExceptionAddress: LPVOID, pub NumberParameters: DWORD, - pub ExceptionInformation: [LPVOID; EXCEPTION_MAXIMUM_PARAMETERS] + pub ExceptionInformation: [LPVOID; EXCEPTION_MAXIMUM_PARAMETERS], } pub enum CONTEXT {} @@ -636,19 +809,8 @@ if #[cfg(not(target_vendor = "uwp"))] { pub ContextRecord: *mut CONTEXT, } - pub type PVECTORED_EXCEPTION_HANDLER = extern "system" - fn(ExceptionInfo: *mut EXCEPTION_POINTERS) -> LONG; - - #[repr(C)] - #[derive(Copy, Clone)] - pub struct CONSOLE_READCONSOLE_CONTROL { - pub nLength: ULONG, - pub nInitialChars: ULONG, - pub dwCtrlWakeupMask: ULONG, - pub dwControlKeyState: ULONG, - } - - pub type PCONSOLE_READCONSOLE_CONTROL = *mut CONSOLE_READCONSOLE_CONTROL; + pub type PVECTORED_EXCEPTION_HANDLER = + extern "system" fn(ExceptionInfo: *mut EXCEPTION_POINTERS) -> LONG; #[repr(C)] pub struct BY_HANDLE_FILE_INFORMATION { @@ -665,50 +827,50 @@ if #[cfg(not(target_vendor = "uwp"))] { } pub type LPBY_HANDLE_FILE_INFORMATION = *mut BY_HANDLE_FILE_INFORMATION; - pub type LPCVOID = *const c_void; pub const HANDLE_FLAG_INHERIT: DWORD = 0x00000001; pub const TOKEN_READ: DWORD = 0x20008; + #[link(name = "advapi32")] + extern "system" { + // Allowed but unused by UWP + pub fn OpenProcessToken( + ProcessHandle: HANDLE, + DesiredAccess: DWORD, + TokenHandle: *mut HANDLE, + ) -> BOOL; + } + + #[link(name = "userenv")] extern "system" { - #[link_name = "SystemFunction036"] - pub fn RtlGenRandom(RandomBuffer: *mut u8, RandomBufferLength: ULONG) -> BOOLEAN; - - pub fn ReadConsoleW(hConsoleInput: HANDLE, - lpBuffer: LPVOID, - nNumberOfCharsToRead: DWORD, - lpNumberOfCharsRead: LPDWORD, - pInputControl: PCONSOLE_READCONSOLE_CONTROL) -> BOOL; - - pub fn WriteConsoleW(hConsoleOutput: HANDLE, - lpBuffer: LPCVOID, - nNumberOfCharsToWrite: DWORD, - lpNumberOfCharsWritten: LPDWORD, - lpReserved: LPVOID) -> BOOL; - - pub fn GetConsoleMode(hConsoleHandle: HANDLE, - lpMode: LPDWORD) -> BOOL; // Allowed but unused by UWP - pub fn OpenProcessToken(ProcessHandle: HANDLE, - DesiredAccess: DWORD, - TokenHandle: *mut HANDLE) -> BOOL; - pub fn GetUserProfileDirectoryW(hToken: HANDLE, - lpProfileDir: LPWSTR, - lpcchSize: *mut DWORD) -> BOOL; - pub fn GetFileInformationByHandle(hFile: HANDLE, - lpFileInformation: LPBY_HANDLE_FILE_INFORMATION) - -> BOOL; - pub fn SetHandleInformation(hObject: HANDLE, - dwMask: DWORD, - dwFlags: DWORD) -> BOOL; - pub fn AddVectoredExceptionHandler(FirstHandler: ULONG, - VectoredHandler: PVECTORED_EXCEPTION_HANDLER) - -> LPVOID; - pub fn CreateHardLinkW(lpSymlinkFileName: LPCWSTR, - lpTargetFileName: LPCWSTR, - lpSecurityAttributes: LPSECURITY_ATTRIBUTES) - -> BOOL; + pub fn GetUserProfileDirectoryW( + hToken: HANDLE, + lpProfileDir: LPWSTR, + lpcchSize: *mut DWORD, + ) -> BOOL; + } + + #[link(name = "kernel32")] + extern "system" { + // Allowed but unused by UWP + pub fn GetFileInformationByHandle( + hFile: HANDLE, + lpFileInformation: LPBY_HANDLE_FILE_INFORMATION, + ) -> BOOL; + pub fn SetHandleInformation(hObject: HANDLE, dwMask: DWORD, dwFlags: DWORD) -> BOOL; + pub fn AddVectoredExceptionHandler( + FirstHandler: ULONG, + VectoredHandler: PVECTORED_EXCEPTION_HANDLER, + ) -> LPVOID; + pub fn CreateHardLinkW( + lpSymlinkFileName: LPCWSTR, + lpTargetFileName: LPCWSTR, + lpSecurityAttributes: LPSECURITY_ATTRIBUTES, + ) -> BOOL; + pub fn SetThreadStackGuarantee(_size: *mut c_ulong) -> BOOL; + pub fn GetWindowsDirectoryW(lpBuffer: LPWSTR, uSize: UINT) -> UINT; } } } @@ -716,8 +878,6 @@ if #[cfg(not(target_vendor = "uwp"))] { // UWP specific functions & types cfg_if::cfg_if! { if #[cfg(target_vendor = "uwp")] { - pub const BCRYPT_USE_SYSTEM_PREFERRED_RNG: DWORD = 0x00000002; - #[repr(C)] pub struct FILE_STANDARD_INFO { pub AllocationSize: LARGE_INTEGER, @@ -726,66 +886,42 @@ if #[cfg(target_vendor = "uwp")] { pub DeletePending: BOOLEAN, pub Directory: BOOLEAN, } - - extern "system" { - pub fn GetFileInformationByHandleEx(hFile: HANDLE, - fileInfoClass: FILE_INFO_BY_HANDLE_CLASS, - lpFileInformation: LPVOID, - dwBufferSize: DWORD) -> BOOL; - pub fn BCryptGenRandom(hAlgorithm: LPVOID, pBuffer: *mut u8, - cbBuffer: ULONG, dwFlags: ULONG) -> LONG; - } } } // Shared between Desktop & UWP + +#[link(name = "kernel32")] extern "system" { - pub fn WSAStartup(wVersionRequested: WORD, lpWSAData: LPWSADATA) -> c_int; - pub fn WSACleanup() -> c_int; - pub fn WSAGetLastError() -> c_int; - pub fn WSADuplicateSocketW( - s: SOCKET, - dwProcessId: DWORD, - lpProtocolInfo: LPWSAPROTOCOL_INFO, - ) -> c_int; - pub fn WSASend( - s: SOCKET, - lpBuffers: LPWSABUF, - dwBufferCount: DWORD, - lpNumberOfBytesSent: LPDWORD, - dwFlags: DWORD, - lpOverlapped: LPWSAOVERLAPPED, - lpCompletionRoutine: LPWSAOVERLAPPED_COMPLETION_ROUTINE, - ) -> c_int; - pub fn WSARecv( - s: SOCKET, - lpBuffers: LPWSABUF, - dwBufferCount: DWORD, - lpNumberOfBytesRecvd: LPDWORD, - lpFlags: LPDWORD, - lpOverlapped: LPWSAOVERLAPPED, - lpCompletionRoutine: LPWSAOVERLAPPED_COMPLETION_ROUTINE, - ) -> c_int; pub fn GetCurrentProcessId() -> DWORD; - pub fn WSASocketW( - af: c_int, - kind: c_int, - protocol: c_int, - lpProtocolInfo: LPWSAPROTOCOL_INFO, - g: GROUP, - dwFlags: DWORD, - ) -> SOCKET; - pub fn ioctlsocket(s: SOCKET, cmd: c_long, argp: *mut c_ulong) -> c_int; - pub fn InitializeCriticalSection(CriticalSection: *mut CRITICAL_SECTION); - pub fn EnterCriticalSection(CriticalSection: *mut CRITICAL_SECTION); - pub fn TryEnterCriticalSection(CriticalSection: *mut CRITICAL_SECTION) -> BOOLEAN; - pub fn LeaveCriticalSection(CriticalSection: *mut CRITICAL_SECTION); - pub fn DeleteCriticalSection(CriticalSection: *mut CRITICAL_SECTION); + pub fn ReadConsoleW( + hConsoleInput: HANDLE, + lpBuffer: LPVOID, + nNumberOfCharsToRead: DWORD, + lpNumberOfCharsRead: LPDWORD, + pInputControl: PCONSOLE_READCONSOLE_CONTROL, + ) -> BOOL; + pub fn WriteConsoleW( + hConsoleOutput: HANDLE, + lpBuffer: LPCVOID, + nNumberOfCharsToWrite: DWORD, + lpNumberOfCharsWritten: LPDWORD, + lpReserved: LPVOID, + ) -> BOOL; + pub fn GetConsoleMode(hConsoleHandle: HANDLE, lpMode: LPDWORD) -> BOOL; + + pub fn GetSystemDirectoryW(lpBuffer: LPWSTR, uSize: UINT) -> UINT; pub fn RemoveDirectoryW(lpPathName: LPCWSTR) -> BOOL; pub fn SetFileAttributesW(lpFileName: LPCWSTR, dwFileAttributes: DWORD) -> BOOL; + pub fn SetFileTime( + hFile: BorrowedHandle<'_>, + lpCreationTime: Option<&FILETIME>, + lpLastAccessTime: Option<&FILETIME>, + lpLastWriteTime: Option<&FILETIME>, + ) -> BOOL; pub fn SetLastError(dwErrCode: DWORD); - pub fn GetCommandLineW() -> *mut LPCWSTR; + pub fn GetCommandLineW() -> LPWSTR; pub fn GetTempPathW(nBufferLength: DWORD, lpBuffer: LPCWSTR) -> DWORD; pub fn GetCurrentProcess() -> HANDLE; pub fn GetCurrentThread() -> HANDLE; @@ -808,10 +944,11 @@ extern "system" { lpParameter: LPVOID, dwCreationFlags: DWORD, lpThreadId: LPDWORD, - ) -> HANDLE; + ) -> HandleOrNull; pub fn WaitForSingleObject(hHandle: HANDLE, dwMilliseconds: DWORD) -> DWORD; pub fn SwitchToThread() -> BOOL; pub fn Sleep(dwMilliseconds: DWORD); + pub fn SleepEx(dwMilliseconds: DWORD, bAlertable: BOOL) -> DWORD; pub fn GetProcessId(handle: HANDLE) -> DWORD; pub fn CopyFileExW( lpExistingFileName: LPCWSTR, @@ -833,6 +970,7 @@ extern "system" { pub fn TlsAlloc() -> DWORD; pub fn TlsGetValue(dwTlsIndex: DWORD) -> LPVOID; pub fn TlsSetValue(dwTlsIndex: DWORD, lpTlsvalue: LPVOID) -> BOOL; + pub fn TlsFree(dwTlsIndex: DWORD) -> BOOL; pub fn GetLastError() -> DWORD; pub fn QueryPerformanceFrequency(lpFrequency: *mut LARGE_INTEGER) -> BOOL; pub fn QueryPerformanceCounter(lpPerformanceCount: *mut LARGE_INTEGER) -> BOOL; @@ -862,38 +1000,6 @@ extern "system" { pub fn DeleteFileW(lpPathName: LPCWSTR) -> BOOL; pub fn GetCurrentDirectoryW(nBufferLength: DWORD, lpBuffer: LPWSTR) -> DWORD; pub fn SetCurrentDirectoryW(lpPathName: LPCWSTR) -> BOOL; - pub fn WideCharToMultiByte( - CodePage: UINT, - dwFlags: DWORD, - lpWideCharStr: LPCWSTR, - cchWideChar: c_int, - lpMultiByteStr: LPSTR, - cbMultiByte: c_int, - lpDefaultChar: LPCSTR, - lpUsedDefaultChar: LPBOOL, - ) -> c_int; - - pub fn closesocket(socket: SOCKET) -> c_int; - pub fn recv(socket: SOCKET, buf: *mut c_void, len: c_int, flags: c_int) -> c_int; - pub fn send(socket: SOCKET, buf: *const c_void, len: c_int, flags: c_int) -> c_int; - pub fn recvfrom( - socket: SOCKET, - buf: *mut c_void, - len: c_int, - flags: c_int, - addr: *mut SOCKADDR, - addrlen: *mut c_int, - ) -> c_int; - pub fn sendto( - socket: SOCKET, - buf: *const c_void, - len: c_int, - flags: c_int, - addr: *const SOCKADDR, - addrlen: c_int, - ) -> c_int; - pub fn shutdown(socket: SOCKET, how: c_int) -> c_int; - pub fn accept(socket: SOCKET, address: *mut SOCKADDR, address_len: *mut c_int) -> SOCKET; pub fn DuplicateHandle( hSourceProcessHandle: HANDLE, hSourceHandle: HANDLE, @@ -904,18 +1010,25 @@ extern "system" { dwOptions: DWORD, ) -> BOOL; pub fn ReadFile( - hFile: HANDLE, + hFile: BorrowedHandle<'_>, lpBuffer: LPVOID, nNumberOfBytesToRead: DWORD, lpNumberOfBytesRead: LPDWORD, lpOverlapped: LPOVERLAPPED, ) -> BOOL; - pub fn WriteFile( - hFile: HANDLE, + pub fn ReadFileEx( + hFile: BorrowedHandle<'_>, + lpBuffer: LPVOID, + nNumberOfBytesToRead: DWORD, + lpOverlapped: LPOVERLAPPED, + lpCompletionRoutine: LPOVERLAPPED_COMPLETION_ROUTINE, + ) -> BOOL; + pub fn WriteFileEx( + hFile: BorrowedHandle<'_>, lpBuffer: LPVOID, nNumberOfBytesToWrite: DWORD, - lpNumberOfBytesWritten: LPDWORD, lpOverlapped: LPOVERLAPPED, + lpCompletionRoutine: LPOVERLAPPED_COMPLETION_ROUTINE, ) -> BOOL; pub fn CloseHandle(hObject: HANDLE) -> BOOL; pub fn MoveFileExW(lpExistingFileName: LPCWSTR, lpNewFileName: LPCWSTR, dwFlags: DWORD) @@ -935,42 +1048,18 @@ extern "system" { dwCreationDisposition: DWORD, dwFlagsAndAttributes: DWORD, hTemplateFile: HANDLE, - ) -> HANDLE; + ) -> HandleOrInvalid; pub fn FindFirstFileW(fileName: LPCWSTR, findFileData: LPWIN32_FIND_DATAW) -> HANDLE; pub fn FindNextFileW(findFile: HANDLE, findFileData: LPWIN32_FIND_DATAW) -> BOOL; pub fn FindClose(findFile: HANDLE) -> BOOL; - pub fn getsockopt( - s: SOCKET, - level: c_int, - optname: c_int, - optval: *mut c_char, - optlen: *mut c_int, - ) -> c_int; - pub fn setsockopt( - s: SOCKET, - level: c_int, - optname: c_int, - optval: *const c_void, - optlen: c_int, - ) -> c_int; - pub fn getsockname(socket: SOCKET, address: *mut SOCKADDR, address_len: *mut c_int) -> c_int; - pub fn getpeername(socket: SOCKET, address: *mut SOCKADDR, address_len: *mut c_int) -> c_int; - pub fn bind(socket: SOCKET, address: *const SOCKADDR, address_len: socklen_t) -> c_int; - pub fn listen(socket: SOCKET, backlog: c_int) -> c_int; - pub fn connect(socket: SOCKET, address: *const SOCKADDR, len: c_int) -> c_int; - pub fn getaddrinfo( - node: *const c_char, - service: *const c_char, - hints: *const ADDRINFOA, - res: *mut *mut ADDRINFOA, - ) -> c_int; - pub fn freeaddrinfo(res: *mut ADDRINFOA); pub fn GetProcAddress(handle: HMODULE, name: LPCSTR) -> *mut c_void; + pub fn GetModuleHandleA(lpModuleName: LPCSTR) -> HMODULE; pub fn GetModuleHandleW(lpModuleName: LPCWSTR) -> HMODULE; pub fn GetSystemTimeAsFileTime(lpSystemTimeAsFileTime: LPFILETIME); + pub fn GetSystemInfo(lpSystemInfo: LPSYSTEM_INFO); pub fn CreateEventW( lpEventAttributes: LPSECURITY_ATTRIBUTES, @@ -1001,6 +1090,155 @@ extern "system" { lpNumberOfBytesTransferred: LPDWORD, bWait: BOOL, ) -> BOOL; + pub fn CreateSymbolicLinkW( + lpSymlinkFileName: LPCWSTR, + lpTargetFileName: LPCWSTR, + dwFlags: DWORD, + ) -> BOOLEAN; + pub fn GetFinalPathNameByHandleW( + hFile: HANDLE, + lpszFilePath: LPCWSTR, + cchFilePath: DWORD, + dwFlags: DWORD, + ) -> DWORD; + pub fn GetFileInformationByHandleEx( + hFile: HANDLE, + fileInfoClass: FILE_INFO_BY_HANDLE_CLASS, + lpFileInformation: LPVOID, + dwBufferSize: DWORD, + ) -> BOOL; + pub fn SetFileInformationByHandle( + hFile: HANDLE, + FileInformationClass: FILE_INFO_BY_HANDLE_CLASS, + lpFileInformation: LPVOID, + dwBufferSize: DWORD, + ) -> BOOL; + pub fn GetFileType(hfile: HANDLE) -> DWORD; + pub fn SleepConditionVariableSRW( + ConditionVariable: PCONDITION_VARIABLE, + SRWLock: PSRWLOCK, + dwMilliseconds: DWORD, + Flags: ULONG, + ) -> BOOL; + + pub fn WakeConditionVariable(ConditionVariable: PCONDITION_VARIABLE); + pub fn WakeAllConditionVariable(ConditionVariable: PCONDITION_VARIABLE); + + pub fn AcquireSRWLockExclusive(SRWLock: PSRWLOCK); + pub fn AcquireSRWLockShared(SRWLock: PSRWLOCK); + pub fn ReleaseSRWLockExclusive(SRWLock: PSRWLOCK); + pub fn ReleaseSRWLockShared(SRWLock: PSRWLOCK); + pub fn TryAcquireSRWLockExclusive(SRWLock: PSRWLOCK) -> BOOLEAN; + pub fn TryAcquireSRWLockShared(SRWLock: PSRWLOCK) -> BOOLEAN; + + pub fn InitOnceBeginInitialize( + lpInitOnce: LPINIT_ONCE, + dwFlags: DWORD, + fPending: LPBOOL, + lpContext: *mut LPVOID, + ) -> BOOL; + pub fn InitOnceComplete(lpInitOnce: LPINIT_ONCE, dwFlags: DWORD, lpContext: LPVOID) -> BOOL; + + pub fn CompareStringOrdinal( + lpString1: LPCWSTR, + cchCount1: c_int, + lpString2: LPCWSTR, + cchCount2: c_int, + bIgnoreCase: BOOL, + ) -> c_int; + pub fn GetFullPathNameW( + lpFileName: LPCWSTR, + nBufferLength: DWORD, + lpBuffer: LPWSTR, + lpFilePart: *mut LPWSTR, + ) -> DWORD; + pub fn GetFileAttributesW(lpFileName: LPCWSTR) -> DWORD; +} + +#[link(name = "ws2_32")] +extern "system" { + pub fn WSAStartup(wVersionRequested: WORD, lpWSAData: LPWSADATA) -> c_int; + pub fn WSACleanup() -> c_int; + pub fn WSAGetLastError() -> c_int; + pub fn WSADuplicateSocketW( + s: SOCKET, + dwProcessId: DWORD, + lpProtocolInfo: LPWSAPROTOCOL_INFO, + ) -> c_int; + pub fn WSASend( + s: SOCKET, + lpBuffers: LPWSABUF, + dwBufferCount: DWORD, + lpNumberOfBytesSent: LPDWORD, + dwFlags: DWORD, + lpOverlapped: LPWSAOVERLAPPED, + lpCompletionRoutine: LPWSAOVERLAPPED_COMPLETION_ROUTINE, + ) -> c_int; + pub fn WSARecv( + s: SOCKET, + lpBuffers: LPWSABUF, + dwBufferCount: DWORD, + lpNumberOfBytesRecvd: LPDWORD, + lpFlags: LPDWORD, + lpOverlapped: LPWSAOVERLAPPED, + lpCompletionRoutine: LPWSAOVERLAPPED_COMPLETION_ROUTINE, + ) -> c_int; + pub fn WSASocketW( + af: c_int, + kind: c_int, + protocol: c_int, + lpProtocolInfo: LPWSAPROTOCOL_INFO, + g: GROUP, + dwFlags: DWORD, + ) -> SOCKET; + pub fn ioctlsocket(s: SOCKET, cmd: c_long, argp: *mut c_ulong) -> c_int; + pub fn closesocket(socket: SOCKET) -> c_int; + pub fn recv(socket: SOCKET, buf: *mut c_void, len: c_int, flags: c_int) -> c_int; + pub fn send(socket: SOCKET, buf: *const c_void, len: c_int, flags: c_int) -> c_int; + pub fn recvfrom( + socket: SOCKET, + buf: *mut c_void, + len: c_int, + flags: c_int, + addr: *mut SOCKADDR, + addrlen: *mut c_int, + ) -> c_int; + pub fn sendto( + socket: SOCKET, + buf: *const c_void, + len: c_int, + flags: c_int, + addr: *const SOCKADDR, + addrlen: c_int, + ) -> c_int; + pub fn shutdown(socket: SOCKET, how: c_int) -> c_int; + pub fn accept(socket: SOCKET, address: *mut SOCKADDR, address_len: *mut c_int) -> SOCKET; + pub fn getsockopt( + s: SOCKET, + level: c_int, + optname: c_int, + optval: *mut c_char, + optlen: *mut c_int, + ) -> c_int; + pub fn setsockopt( + s: SOCKET, + level: c_int, + optname: c_int, + optval: *const c_void, + optlen: c_int, + ) -> c_int; + pub fn getsockname(socket: SOCKET, address: *mut SOCKADDR, address_len: *mut c_int) -> c_int; + pub fn getpeername(socket: SOCKET, address: *mut SOCKADDR, address_len: *mut c_int) -> c_int; + pub fn bind(socket: SOCKET, address: *const SOCKADDR, address_len: socklen_t) -> c_int; + pub fn listen(socket: SOCKET, backlog: c_int) -> c_int; + pub fn connect(socket: SOCKET, address: *const SOCKADDR, len: c_int) -> c_int; + pub fn getaddrinfo( + node: *const c_char, + service: *const c_char, + hints: *const ADDRINFOA, + res: *mut *mut ADDRINFOA, + ) -> c_int; + pub fn freeaddrinfo(res: *mut ADDRINFOA); pub fn select( nfds: c_int, readfds: *mut fd_set, @@ -1008,77 +1246,135 @@ extern "system" { exceptfds: *mut fd_set, timeout: *const timeval, ) -> c_int; +} - pub fn GetProcessHeap() -> HANDLE; - pub fn HeapAlloc(hHeap: HANDLE, dwFlags: DWORD, dwBytes: SIZE_T) -> LPVOID; - pub fn HeapReAlloc(hHeap: HANDLE, dwFlags: DWORD, lpMem: LPVOID, dwBytes: SIZE_T) -> LPVOID; - pub fn HeapFree(hHeap: HANDLE, dwFlags: DWORD, lpMem: LPVOID) -> BOOL; +#[link(name = "bcrypt")] +extern "system" { + // >= Vista / Server 2008 + // https://docs.microsoft.com/en-us/windows/win32/api/bcrypt/nf-bcrypt-bcryptgenrandom + pub fn BCryptGenRandom( + hAlgorithm: BCRYPT_ALG_HANDLE, + pBuffer: *mut u8, + cbBuffer: ULONG, + dwFlags: ULONG, + ) -> NTSTATUS; + pub fn BCryptOpenAlgorithmProvider( + phalgorithm: *mut BCRYPT_ALG_HANDLE, + pszAlgId: LPCWSTR, + pszimplementation: LPCWSTR, + dwflags: ULONG, + ) -> NTSTATUS; + pub fn BCryptCloseAlgorithmProvider(hAlgorithm: BCRYPT_ALG_HANDLE, dwFlags: ULONG) -> NTSTATUS; } // Functions that aren't available on every version of Windows that we support, // but we still use them and just provide some form of a fallback implementation. -compat_fn! { - kernel32: +compat_fn_with_fallback! { + pub static KERNEL32: &CStr = ansi_str!("kernel32"); - pub fn CreateSymbolicLinkW(_lpSymlinkFileName: LPCWSTR, - _lpTargetFileName: LPCWSTR, - _dwFlags: DWORD) -> BOOLEAN { - SetLastError(ERROR_CALL_NOT_IMPLEMENTED as DWORD); 0 - } - pub fn GetFinalPathNameByHandleW(_hFile: HANDLE, - _lpszFilePath: LPCWSTR, - _cchFilePath: DWORD, - _dwFlags: DWORD) -> DWORD { - SetLastError(ERROR_CALL_NOT_IMPLEMENTED as DWORD); 0 - } - #[cfg(not(target_vendor = "uwp"))] - pub fn SetThreadStackGuarantee(_size: *mut c_ulong) -> BOOL { - SetLastError(ERROR_CALL_NOT_IMPLEMENTED as DWORD); 0 - } + // >= Win10 1607 + // https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-setthreaddescription pub fn SetThreadDescription(hThread: HANDLE, lpThreadDescription: LPCWSTR) -> HRESULT { SetLastError(ERROR_CALL_NOT_IMPLEMENTED as DWORD); E_NOTIMPL } - pub fn SetFileInformationByHandle(_hFile: HANDLE, - _FileInformationClass: FILE_INFO_BY_HANDLE_CLASS, - _lpFileInformation: LPVOID, - _dwBufferSize: DWORD) -> BOOL { - SetLastError(ERROR_CALL_NOT_IMPLEMENTED as DWORD); 0 - } + + // >= Win8 / Server 2012 + // https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-getsystemtimepreciseasfiletime pub fn GetSystemTimePreciseAsFileTime(lpSystemTimeAsFileTime: LPFILETIME) -> () { GetSystemTimeAsFileTime(lpSystemTimeAsFileTime) } - pub fn SleepConditionVariableSRW(ConditionVariable: PCONDITION_VARIABLE, - SRWLock: PSRWLOCK, - dwMilliseconds: DWORD, - Flags: ULONG) -> BOOL { - panic!("condition variables not available") - } - pub fn WakeConditionVariable(ConditionVariable: PCONDITION_VARIABLE) - -> () { - panic!("condition variables not available") + + // >= Win11 / Server 2022 + // https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettemppath2a + pub fn GetTempPath2W(nBufferLength: DWORD, lpBuffer: LPCWSTR) -> DWORD { + GetTempPathW(nBufferLength, lpBuffer) } - pub fn WakeAllConditionVariable(ConditionVariable: PCONDITION_VARIABLE) - -> () { - panic!("condition variables not available") +} + +compat_fn_optional! { + crate::sys::compat::load_synch_functions(); + pub fn WaitOnAddress( + Address: LPVOID, + CompareAddress: LPVOID, + AddressSize: SIZE_T, + dwMilliseconds: DWORD + ); + pub fn WakeByAddressSingle(Address: LPVOID); +} + +compat_fn_with_fallback! { + pub static NTDLL: &CStr = ansi_str!("ntdll"); + + pub fn NtCreateFile( + FileHandle: *mut HANDLE, + DesiredAccess: ACCESS_MASK, + ObjectAttributes: *const OBJECT_ATTRIBUTES, + IoStatusBlock: *mut IO_STATUS_BLOCK, + AllocationSize: *mut i64, + FileAttributes: ULONG, + ShareAccess: ULONG, + CreateDisposition: ULONG, + CreateOptions: ULONG, + EaBuffer: *mut c_void, + EaLength: ULONG + ) -> NTSTATUS { + STATUS_NOT_IMPLEMENTED } - pub fn AcquireSRWLockExclusive(SRWLock: PSRWLOCK) -> () { - panic!("rwlocks not available") + pub fn NtReadFile( + FileHandle: BorrowedHandle<'_>, + Event: HANDLE, + ApcRoutine: Option, + ApcContext: *mut c_void, + IoStatusBlock: &mut IO_STATUS_BLOCK, + Buffer: *mut crate::mem::MaybeUninit, + Length: ULONG, + ByteOffset: Option<&LARGE_INTEGER>, + Key: Option<&ULONG> + ) -> NTSTATUS { + STATUS_NOT_IMPLEMENTED } - pub fn AcquireSRWLockShared(SRWLock: PSRWLOCK) -> () { - panic!("rwlocks not available") + pub fn NtWriteFile( + FileHandle: BorrowedHandle<'_>, + Event: HANDLE, + ApcRoutine: Option, + ApcContext: *mut c_void, + IoStatusBlock: &mut IO_STATUS_BLOCK, + Buffer: *const u8, + Length: ULONG, + ByteOffset: Option<&LARGE_INTEGER>, + Key: Option<&ULONG> + ) -> NTSTATUS { + STATUS_NOT_IMPLEMENTED } - pub fn ReleaseSRWLockExclusive(SRWLock: PSRWLOCK) -> () { - panic!("rwlocks not available") + pub fn RtlNtStatusToDosError( + Status: NTSTATUS + ) -> ULONG { + Status as ULONG } - pub fn ReleaseSRWLockShared(SRWLock: PSRWLOCK) -> () { - panic!("rwlocks not available") + pub fn NtCreateKeyedEvent( + KeyedEventHandle: LPHANDLE, + DesiredAccess: ACCESS_MASK, + ObjectAttributes: LPVOID, + Flags: ULONG + ) -> NTSTATUS { + panic!("keyed events not available") } - pub fn TryAcquireSRWLockExclusive(SRWLock: PSRWLOCK) -> BOOLEAN { - panic!("rwlocks not available") + pub fn NtReleaseKeyedEvent( + EventHandle: HANDLE, + Key: LPVOID, + Alertable: BOOLEAN, + Timeout: PLARGE_INTEGER + ) -> NTSTATUS { + panic!("keyed events not available") } - pub fn TryAcquireSRWLockShared(SRWLock: PSRWLOCK) -> BOOLEAN { - panic!("rwlocks not available") + pub fn NtWaitForKeyedEvent( + EventHandle: HANDLE, + Key: LPVOID, + Alertable: BOOLEAN, + Timeout: PLARGE_INTEGER + ) -> NTSTATUS { + panic!("keyed events not available") } } diff --git a/crux-mir/lib/std/src/sys/windows/c/errors.rs b/crux-mir/lib/std/src/sys/windows/c/errors.rs new file mode 100644 index 000000000..23dcc119d --- /dev/null +++ b/crux-mir/lib/std/src/sys/windows/c/errors.rs @@ -0,0 +1,1883 @@ +// List of Windows system error codes with descriptions: +// https://docs.microsoft.com/en-us/windows/win32/debug/system-error-codes#system-error-codes + +#![allow(dead_code)] + +use super::{c_int, DWORD}; + +pub const ERROR_DIRECTORY_NOT_SUPPORTED: DWORD = 336; +pub const ERROR_DRIVER_CANCEL_TIMEOUT: DWORD = 594; +pub const ERROR_DISK_QUOTA_EXCEEDED: DWORD = 1295; +pub const ERROR_RESOURCE_CALL_TIMED_OUT: DWORD = 5910; +pub const FRS_ERR_SYSVOL_POPULATE_TIMEOUT: DWORD = 8014; +pub const DNS_ERROR_RECORD_TIMED_OUT: DWORD = 9705; + +// The followiung list was obtained from +// `/usr/x86_64-w64-mingw32/include/winerror.h` +// in the Debian package +// mingw-w64_6.0.0-3_all.deb +// +// The header of that file says: +// * This file has no copyright assigned and is placed in the Public Domain. +// * This file is part of the mingw-w64 runtime package. +// * No warranty is given; refer to the file DISCLAIMER.PD within this package. +// +// The text here is the result of the following rune: +// grep -P '#define ERROR' /usr/x86_64-w64-mingw32/include/winerror.h >>library/std/src/sys/windows/c/errors.rs +// grep -P '#define WSA' /usr/x86_64-w64-mingw32/include/winerror.h >>library/std/src/sys/windows/c/errors.rs +// and then using some manually-invented but rather obvious editor search-and-replace +// invocations, plus some straightforward manual fixups, to turn it into Rust syntax +// and remove all the duplicates from the manual table above. + +pub const ERROR_SUCCESS: DWORD = 0; +pub const ERROR_INVALID_FUNCTION: DWORD = 1; +pub const ERROR_FILE_NOT_FOUND: DWORD = 2; +pub const ERROR_PATH_NOT_FOUND: DWORD = 3; +pub const ERROR_TOO_MANY_OPEN_FILES: DWORD = 4; +pub const ERROR_ACCESS_DENIED: DWORD = 5; +pub const ERROR_INVALID_HANDLE: DWORD = 6; +pub const ERROR_ARENA_TRASHED: DWORD = 7; +pub const ERROR_NOT_ENOUGH_MEMORY: DWORD = 8; +pub const ERROR_INVALID_BLOCK: DWORD = 9; +pub const ERROR_BAD_ENVIRONMENT: DWORD = 10; +pub const ERROR_BAD_FORMAT: DWORD = 11; +pub const ERROR_INVALID_ACCESS: DWORD = 12; +pub const ERROR_INVALID_DATA: DWORD = 13; +pub const ERROR_OUTOFMEMORY: DWORD = 14; +pub const ERROR_INVALID_DRIVE: DWORD = 15; +pub const ERROR_CURRENT_DIRECTORY: DWORD = 16; +pub const ERROR_NOT_SAME_DEVICE: DWORD = 17; +pub const ERROR_NO_MORE_FILES: DWORD = 18; +pub const ERROR_WRITE_PROTECT: DWORD = 19; +pub const ERROR_BAD_UNIT: DWORD = 20; +pub const ERROR_NOT_READY: DWORD = 21; +pub const ERROR_BAD_COMMAND: DWORD = 22; +pub const ERROR_CRC: DWORD = 23; +pub const ERROR_BAD_LENGTH: DWORD = 24; +pub const ERROR_SEEK: DWORD = 25; +pub const ERROR_NOT_DOS_DISK: DWORD = 26; +pub const ERROR_SECTOR_NOT_FOUND: DWORD = 27; +pub const ERROR_OUT_OF_PAPER: DWORD = 28; +pub const ERROR_WRITE_FAULT: DWORD = 29; +pub const ERROR_READ_FAULT: DWORD = 30; +pub const ERROR_GEN_FAILURE: DWORD = 31; +pub const ERROR_SHARING_VIOLATION: DWORD = 32; +pub const ERROR_LOCK_VIOLATION: DWORD = 33; +pub const ERROR_WRONG_DISK: DWORD = 34; +pub const ERROR_SHARING_BUFFER_EXCEEDED: DWORD = 36; +pub const ERROR_HANDLE_EOF: DWORD = 38; +pub const ERROR_HANDLE_DISK_FULL: DWORD = 39; +pub const ERROR_NOT_SUPPORTED: DWORD = 50; +pub const ERROR_REM_NOT_LIST: DWORD = 51; +pub const ERROR_DUP_NAME: DWORD = 52; +pub const ERROR_BAD_NETPATH: DWORD = 53; +pub const ERROR_NETWORK_BUSY: DWORD = 54; +pub const ERROR_DEV_NOT_EXIST: DWORD = 55; +pub const ERROR_TOO_MANY_CMDS: DWORD = 56; +pub const ERROR_ADAP_HDW_ERR: DWORD = 57; +pub const ERROR_BAD_NET_RESP: DWORD = 58; +pub const ERROR_UNEXP_NET_ERR: DWORD = 59; +pub const ERROR_BAD_REM_ADAP: DWORD = 60; +pub const ERROR_PRINTQ_FULL: DWORD = 61; +pub const ERROR_NO_SPOOL_SPACE: DWORD = 62; +pub const ERROR_PRINT_CANCELLED: DWORD = 63; +pub const ERROR_NETNAME_DELETED: DWORD = 64; +pub const ERROR_NETWORK_ACCESS_DENIED: DWORD = 65; +pub const ERROR_BAD_DEV_TYPE: DWORD = 66; +pub const ERROR_BAD_NET_NAME: DWORD = 67; +pub const ERROR_TOO_MANY_NAMES: DWORD = 68; +pub const ERROR_TOO_MANY_SESS: DWORD = 69; +pub const ERROR_SHARING_PAUSED: DWORD = 70; +pub const ERROR_REQ_NOT_ACCEP: DWORD = 71; +pub const ERROR_REDIR_PAUSED: DWORD = 72; +pub const ERROR_FILE_EXISTS: DWORD = 80; +pub const ERROR_CANNOT_MAKE: DWORD = 82; +pub const ERROR_FAIL_I24: DWORD = 83; +pub const ERROR_OUT_OF_STRUCTURES: DWORD = 84; +pub const ERROR_ALREADY_ASSIGNED: DWORD = 85; +pub const ERROR_INVALID_PASSWORD: DWORD = 86; +pub const ERROR_INVALID_PARAMETER: DWORD = 87; +pub const ERROR_NET_WRITE_FAULT: DWORD = 88; +pub const ERROR_NO_PROC_SLOTS: DWORD = 89; +pub const ERROR_TOO_MANY_SEMAPHORES: DWORD = 100; +pub const ERROR_EXCL_SEM_ALREADY_OWNED: DWORD = 101; +pub const ERROR_SEM_IS_SET: DWORD = 102; +pub const ERROR_TOO_MANY_SEM_REQUESTS: DWORD = 103; +pub const ERROR_INVALID_AT_INTERRUPT_TIME: DWORD = 104; +pub const ERROR_SEM_OWNER_DIED: DWORD = 105; +pub const ERROR_SEM_USER_LIMIT: DWORD = 106; +pub const ERROR_DISK_CHANGE: DWORD = 107; +pub const ERROR_DRIVE_LOCKED: DWORD = 108; +pub const ERROR_BROKEN_PIPE: DWORD = 109; +pub const ERROR_OPEN_FAILED: DWORD = 110; +pub const ERROR_BUFFER_OVERFLOW: DWORD = 111; +pub const ERROR_DISK_FULL: DWORD = 112; +pub const ERROR_NO_MORE_SEARCH_HANDLES: DWORD = 113; +pub const ERROR_INVALID_TARGET_HANDLE: DWORD = 114; +pub const ERROR_INVALID_CATEGORY: DWORD = 117; +pub const ERROR_INVALID_VERIFY_SWITCH: DWORD = 118; +pub const ERROR_BAD_DRIVER_LEVEL: DWORD = 119; +pub const ERROR_CALL_NOT_IMPLEMENTED: DWORD = 120; +pub const ERROR_SEM_TIMEOUT: DWORD = 121; +pub const ERROR_INSUFFICIENT_BUFFER: DWORD = 122; +pub const ERROR_INVALID_NAME: DWORD = 123; +pub const ERROR_INVALID_LEVEL: DWORD = 124; +pub const ERROR_NO_VOLUME_LABEL: DWORD = 125; +pub const ERROR_MOD_NOT_FOUND: DWORD = 126; +pub const ERROR_PROC_NOT_FOUND: DWORD = 127; +pub const ERROR_WAIT_NO_CHILDREN: DWORD = 128; +pub const ERROR_CHILD_NOT_COMPLETE: DWORD = 129; +pub const ERROR_DIRECT_ACCESS_HANDLE: DWORD = 130; +pub const ERROR_NEGATIVE_SEEK: DWORD = 131; +pub const ERROR_SEEK_ON_DEVICE: DWORD = 132; +pub const ERROR_IS_JOIN_TARGET: DWORD = 133; +pub const ERROR_IS_JOINED: DWORD = 134; +pub const ERROR_IS_SUBSTED: DWORD = 135; +pub const ERROR_NOT_JOINED: DWORD = 136; +pub const ERROR_NOT_SUBSTED: DWORD = 137; +pub const ERROR_JOIN_TO_JOIN: DWORD = 138; +pub const ERROR_SUBST_TO_SUBST: DWORD = 139; +pub const ERROR_JOIN_TO_SUBST: DWORD = 140; +pub const ERROR_SUBST_TO_JOIN: DWORD = 141; +pub const ERROR_BUSY_DRIVE: DWORD = 142; +pub const ERROR_SAME_DRIVE: DWORD = 143; +pub const ERROR_DIR_NOT_ROOT: DWORD = 144; +pub const ERROR_DIR_NOT_EMPTY: DWORD = 145; +pub const ERROR_IS_SUBST_PATH: DWORD = 146; +pub const ERROR_IS_JOIN_PATH: DWORD = 147; +pub const ERROR_PATH_BUSY: DWORD = 148; +pub const ERROR_IS_SUBST_TARGET: DWORD = 149; +pub const ERROR_SYSTEM_TRACE: DWORD = 150; +pub const ERROR_INVALID_EVENT_COUNT: DWORD = 151; +pub const ERROR_TOO_MANY_MUXWAITERS: DWORD = 152; +pub const ERROR_INVALID_LIST_FORMAT: DWORD = 153; +pub const ERROR_LABEL_TOO_LONG: DWORD = 154; +pub const ERROR_TOO_MANY_TCBS: DWORD = 155; +pub const ERROR_SIGNAL_REFUSED: DWORD = 156; +pub const ERROR_DISCARDED: DWORD = 157; +pub const ERROR_NOT_LOCKED: DWORD = 158; +pub const ERROR_BAD_THREADID_ADDR: DWORD = 159; +pub const ERROR_BAD_ARGUMENTS: DWORD = 160; +pub const ERROR_BAD_PATHNAME: DWORD = 161; +pub const ERROR_SIGNAL_PENDING: DWORD = 162; +pub const ERROR_MAX_THRDS_REACHED: DWORD = 164; +pub const ERROR_LOCK_FAILED: DWORD = 167; +pub const ERROR_BUSY: DWORD = 170; +pub const ERROR_CANCEL_VIOLATION: DWORD = 173; +pub const ERROR_ATOMIC_LOCKS_NOT_SUPPORTED: DWORD = 174; +pub const ERROR_INVALID_SEGMENT_NUMBER: DWORD = 180; +pub const ERROR_INVALID_ORDINAL: DWORD = 182; +pub const ERROR_ALREADY_EXISTS: DWORD = 183; +pub const ERROR_INVALID_FLAG_NUMBER: DWORD = 186; +pub const ERROR_SEM_NOT_FOUND: DWORD = 187; +pub const ERROR_INVALID_STARTING_CODESEG: DWORD = 188; +pub const ERROR_INVALID_STACKSEG: DWORD = 189; +pub const ERROR_INVALID_MODULETYPE: DWORD = 190; +pub const ERROR_INVALID_EXE_SIGNATURE: DWORD = 191; +pub const ERROR_EXE_MARKED_INVALID: DWORD = 192; +pub const ERROR_BAD_EXE_FORMAT: DWORD = 193; +pub const ERROR_ITERATED_DATA_EXCEEDS_64k: DWORD = 194; +pub const ERROR_INVALID_MINALLOCSIZE: DWORD = 195; +pub const ERROR_DYNLINK_FROM_INVALID_RING: DWORD = 196; +pub const ERROR_IOPL_NOT_ENABLED: DWORD = 197; +pub const ERROR_INVALID_SEGDPL: DWORD = 198; +pub const ERROR_AUTODATASEG_EXCEEDS_64k: DWORD = 199; +pub const ERROR_RING2SEG_MUST_BE_MOVABLE: DWORD = 200; +pub const ERROR_RELOC_CHAIN_XEEDS_SEGLIM: DWORD = 201; +pub const ERROR_INFLOOP_IN_RELOC_CHAIN: DWORD = 202; +pub const ERROR_ENVVAR_NOT_FOUND: DWORD = 203; +pub const ERROR_NO_SIGNAL_SENT: DWORD = 205; +pub const ERROR_FILENAME_EXCED_RANGE: DWORD = 206; +pub const ERROR_RING2_STACK_IN_USE: DWORD = 207; +pub const ERROR_META_EXPANSION_TOO_LONG: DWORD = 208; +pub const ERROR_INVALID_SIGNAL_NUMBER: DWORD = 209; +pub const ERROR_THREAD_1_INACTIVE: DWORD = 210; +pub const ERROR_LOCKED: DWORD = 212; +pub const ERROR_TOO_MANY_MODULES: DWORD = 214; +pub const ERROR_NESTING_NOT_ALLOWED: DWORD = 215; +pub const ERROR_EXE_MACHINE_TYPE_MISMATCH: DWORD = 216; +pub const ERROR_EXE_CANNOT_MODIFY_SIGNED_BINARY: DWORD = 217; +pub const ERROR_EXE_CANNOT_MODIFY_STRONG_SIGNED_BINARY: DWORD = 218; +pub const ERROR_FILE_CHECKED_OUT: DWORD = 220; +pub const ERROR_CHECKOUT_REQUIRED: DWORD = 221; +pub const ERROR_BAD_FILE_TYPE: DWORD = 222; +pub const ERROR_FILE_TOO_LARGE: DWORD = 223; +pub const ERROR_FORMS_AUTH_REQUIRED: DWORD = 224; +pub const ERROR_PIPE_LOCAL: DWORD = 229; +pub const ERROR_BAD_PIPE: DWORD = 230; +pub const ERROR_PIPE_BUSY: DWORD = 231; +pub const ERROR_NO_DATA: DWORD = 232; +pub const ERROR_PIPE_NOT_CONNECTED: DWORD = 233; +pub const ERROR_MORE_DATA: DWORD = 234; +pub const ERROR_VC_DISCONNECTED: DWORD = 240; +pub const ERROR_INVALID_EA_NAME: DWORD = 254; +pub const ERROR_EA_LIST_INCONSISTENT: DWORD = 255; +pub const ERROR_NO_MORE_ITEMS: DWORD = 259; +pub const ERROR_CANNOT_COPY: DWORD = 266; +pub const ERROR_DIRECTORY: DWORD = 267; +pub const ERROR_EAS_DIDNT_FIT: DWORD = 275; +pub const ERROR_EA_FILE_CORRUPT: DWORD = 276; +pub const ERROR_EA_TABLE_FULL: DWORD = 277; +pub const ERROR_INVALID_EA_HANDLE: DWORD = 278; +pub const ERROR_EAS_NOT_SUPPORTED: DWORD = 282; +pub const ERROR_NOT_OWNER: DWORD = 288; +pub const ERROR_TOO_MANY_POSTS: DWORD = 298; +pub const ERROR_PARTIAL_COPY: DWORD = 299; +pub const ERROR_OPLOCK_NOT_GRANTED: DWORD = 300; +pub const ERROR_INVALID_OPLOCK_PROTOCOL: DWORD = 301; +pub const ERROR_DISK_TOO_FRAGMENTED: DWORD = 302; +pub const ERROR_DELETE_PENDING: DWORD = 303; +pub const ERROR_INVALID_TOKEN: DWORD = 315; +pub const ERROR_MR_MID_NOT_FOUND: DWORD = 317; +pub const ERROR_SCOPE_NOT_FOUND: DWORD = 318; +pub const ERROR_INVALID_ADDRESS: DWORD = 487; +pub const ERROR_ARITHMETIC_OVERFLOW: DWORD = 534; +pub const ERROR_PIPE_CONNECTED: DWORD = 535; +pub const ERROR_PIPE_LISTENING: DWORD = 536; +pub const ERROR_WAKE_SYSTEM: DWORD = 730; +pub const ERROR_WAIT_1: DWORD = 731; +pub const ERROR_WAIT_2: DWORD = 732; +pub const ERROR_WAIT_3: DWORD = 733; +pub const ERROR_WAIT_63: DWORD = 734; +pub const ERROR_ABANDONED_WAIT_0: DWORD = 735; +pub const ERROR_ABANDONED_WAIT_63: DWORD = 736; +pub const ERROR_USER_APC: DWORD = 737; +pub const ERROR_KERNEL_APC: DWORD = 738; +pub const ERROR_ALERTED: DWORD = 739; +pub const ERROR_EA_ACCESS_DENIED: DWORD = 994; +pub const ERROR_OPERATION_ABORTED: DWORD = 995; +pub const ERROR_IO_INCOMPLETE: DWORD = 996; +pub const ERROR_IO_PENDING: DWORD = 997; +pub const ERROR_NOACCESS: DWORD = 998; +pub const ERROR_SWAPERROR: DWORD = 999; +pub const ERROR_STACK_OVERFLOW: DWORD = 1001; +pub const ERROR_INVALID_MESSAGE: DWORD = 1002; +pub const ERROR_CAN_NOT_COMPLETE: DWORD = 1003; +pub const ERROR_INVALID_FLAGS: DWORD = 1004; +pub const ERROR_UNRECOGNIZED_VOLUME: DWORD = 1005; +pub const ERROR_FILE_INVALID: DWORD = 1006; +pub const ERROR_FULLSCREEN_MODE: DWORD = 1007; +pub const ERROR_NO_TOKEN: DWORD = 1008; +pub const ERROR_BADDB: DWORD = 1009; +pub const ERROR_BADKEY: DWORD = 1010; +pub const ERROR_CANTOPEN: DWORD = 1011; +pub const ERROR_CANTREAD: DWORD = 1012; +pub const ERROR_CANTWRITE: DWORD = 1013; +pub const ERROR_REGISTRY_RECOVERED: DWORD = 1014; +pub const ERROR_REGISTRY_CORRUPT: DWORD = 1015; +pub const ERROR_REGISTRY_IO_FAILED: DWORD = 1016; +pub const ERROR_NOT_REGISTRY_FILE: DWORD = 1017; +pub const ERROR_KEY_DELETED: DWORD = 1018; +pub const ERROR_NO_LOG_SPACE: DWORD = 1019; +pub const ERROR_KEY_HAS_CHILDREN: DWORD = 1020; +pub const ERROR_CHILD_MUST_BE_VOLATILE: DWORD = 1021; +pub const ERROR_NOTIFY_ENUM_DIR: DWORD = 1022; +pub const ERROR_DEPENDENT_SERVICES_RUNNING: DWORD = 1051; +pub const ERROR_INVALID_SERVICE_CONTROL: DWORD = 1052; +pub const ERROR_SERVICE_REQUEST_TIMEOUT: DWORD = 1053; +pub const ERROR_SERVICE_NO_THREAD: DWORD = 1054; +pub const ERROR_SERVICE_DATABASE_LOCKED: DWORD = 1055; +pub const ERROR_SERVICE_ALREADY_RUNNING: DWORD = 1056; +pub const ERROR_INVALID_SERVICE_ACCOUNT: DWORD = 1057; +pub const ERROR_SERVICE_DISABLED: DWORD = 1058; +pub const ERROR_CIRCULAR_DEPENDENCY: DWORD = 1059; +pub const ERROR_SERVICE_DOES_NOT_EXIST: DWORD = 1060; +pub const ERROR_SERVICE_CANNOT_ACCEPT_CTRL: DWORD = 1061; +pub const ERROR_SERVICE_NOT_ACTIVE: DWORD = 1062; +pub const ERROR_FAILED_SERVICE_CONTROLLER_CONNECT: DWORD = 1063; +pub const ERROR_EXCEPTION_IN_SERVICE: DWORD = 1064; +pub const ERROR_DATABASE_DOES_NOT_EXIST: DWORD = 1065; +pub const ERROR_SERVICE_SPECIFIC_ERROR: DWORD = 1066; +pub const ERROR_PROCESS_ABORTED: DWORD = 1067; +pub const ERROR_SERVICE_DEPENDENCY_FAIL: DWORD = 1068; +pub const ERROR_SERVICE_LOGON_FAILED: DWORD = 1069; +pub const ERROR_SERVICE_START_HANG: DWORD = 1070; +pub const ERROR_INVALID_SERVICE_LOCK: DWORD = 1071; +pub const ERROR_SERVICE_MARKED_FOR_DELETE: DWORD = 1072; +pub const ERROR_SERVICE_EXISTS: DWORD = 1073; +pub const ERROR_ALREADY_RUNNING_LKG: DWORD = 1074; +pub const ERROR_SERVICE_DEPENDENCY_DELETED: DWORD = 1075; +pub const ERROR_BOOT_ALREADY_ACCEPTED: DWORD = 1076; +pub const ERROR_SERVICE_NEVER_STARTED: DWORD = 1077; +pub const ERROR_DUPLICATE_SERVICE_NAME: DWORD = 1078; +pub const ERROR_DIFFERENT_SERVICE_ACCOUNT: DWORD = 1079; +pub const ERROR_CANNOT_DETECT_DRIVER_FAILURE: DWORD = 1080; +pub const ERROR_CANNOT_DETECT_PROCESS_ABORT: DWORD = 1081; +pub const ERROR_NO_RECOVERY_PROGRAM: DWORD = 1082; +pub const ERROR_SERVICE_NOT_IN_EXE: DWORD = 1083; +pub const ERROR_NOT_SAFEBOOT_SERVICE: DWORD = 1084; +pub const ERROR_END_OF_MEDIA: DWORD = 1100; +pub const ERROR_FILEMARK_DETECTED: DWORD = 1101; +pub const ERROR_BEGINNING_OF_MEDIA: DWORD = 1102; +pub const ERROR_SETMARK_DETECTED: DWORD = 1103; +pub const ERROR_NO_DATA_DETECTED: DWORD = 1104; +pub const ERROR_PARTITION_FAILURE: DWORD = 1105; +pub const ERROR_INVALID_BLOCK_LENGTH: DWORD = 1106; +pub const ERROR_DEVICE_NOT_PARTITIONED: DWORD = 1107; +pub const ERROR_UNABLE_TO_LOCK_MEDIA: DWORD = 1108; +pub const ERROR_UNABLE_TO_UNLOAD_MEDIA: DWORD = 1109; +pub const ERROR_MEDIA_CHANGED: DWORD = 1110; +pub const ERROR_BUS_RESET: DWORD = 1111; +pub const ERROR_NO_MEDIA_IN_DRIVE: DWORD = 1112; +pub const ERROR_NO_UNICODE_TRANSLATION: DWORD = 1113; +pub const ERROR_DLL_INIT_FAILED: DWORD = 1114; +pub const ERROR_SHUTDOWN_IN_PROGRESS: DWORD = 1115; +pub const ERROR_NO_SHUTDOWN_IN_PROGRESS: DWORD = 1116; +pub const ERROR_IO_DEVICE: DWORD = 1117; +pub const ERROR_SERIAL_NO_DEVICE: DWORD = 1118; +pub const ERROR_IRQ_BUSY: DWORD = 1119; +pub const ERROR_MORE_WRITES: DWORD = 1120; +pub const ERROR_COUNTER_TIMEOUT: DWORD = 1121; +pub const ERROR_FLOPPY_ID_MARK_NOT_FOUND: DWORD = 1122; +pub const ERROR_FLOPPY_WRONG_CYLINDER: DWORD = 1123; +pub const ERROR_FLOPPY_UNKNOWN_ERROR: DWORD = 1124; +pub const ERROR_FLOPPY_BAD_REGISTERS: DWORD = 1125; +pub const ERROR_DISK_RECALIBRATE_FAILED: DWORD = 1126; +pub const ERROR_DISK_OPERATION_FAILED: DWORD = 1127; +pub const ERROR_DISK_RESET_FAILED: DWORD = 1128; +pub const ERROR_EOM_OVERFLOW: DWORD = 1129; +pub const ERROR_NOT_ENOUGH_SERVER_MEMORY: DWORD = 1130; +pub const ERROR_POSSIBLE_DEADLOCK: DWORD = 1131; +pub const ERROR_MAPPED_ALIGNMENT: DWORD = 1132; +pub const ERROR_SET_POWER_STATE_VETOED: DWORD = 1140; +pub const ERROR_SET_POWER_STATE_FAILED: DWORD = 1141; +pub const ERROR_TOO_MANY_LINKS: DWORD = 1142; +pub const ERROR_OLD_WIN_VERSION: DWORD = 1150; +pub const ERROR_APP_WRONG_OS: DWORD = 1151; +pub const ERROR_SINGLE_INSTANCE_APP: DWORD = 1152; +pub const ERROR_RMODE_APP: DWORD = 1153; +pub const ERROR_INVALID_DLL: DWORD = 1154; +pub const ERROR_NO_ASSOCIATION: DWORD = 1155; +pub const ERROR_DDE_FAIL: DWORD = 1156; +pub const ERROR_DLL_NOT_FOUND: DWORD = 1157; +pub const ERROR_NO_MORE_USER_HANDLES: DWORD = 1158; +pub const ERROR_MESSAGE_SYNC_ONLY: DWORD = 1159; +pub const ERROR_SOURCE_ELEMENT_EMPTY: DWORD = 1160; +pub const ERROR_DESTINATION_ELEMENT_FULL: DWORD = 1161; +pub const ERROR_ILLEGAL_ELEMENT_ADDRESS: DWORD = 1162; +pub const ERROR_MAGAZINE_NOT_PRESENT: DWORD = 1163; +pub const ERROR_DEVICE_REINITIALIZATION_NEEDED: DWORD = 1164; +pub const ERROR_DEVICE_REQUIRES_CLEANING: DWORD = 1165; +pub const ERROR_DEVICE_DOOR_OPEN: DWORD = 1166; +pub const ERROR_DEVICE_NOT_CONNECTED: DWORD = 1167; +pub const ERROR_NOT_FOUND: DWORD = 1168; +pub const ERROR_NO_MATCH: DWORD = 1169; +pub const ERROR_SET_NOT_FOUND: DWORD = 1170; +pub const ERROR_POINT_NOT_FOUND: DWORD = 1171; +pub const ERROR_NO_TRACKING_SERVICE: DWORD = 1172; +pub const ERROR_NO_VOLUME_ID: DWORD = 1173; +pub const ERROR_UNABLE_TO_REMOVE_REPLACED: DWORD = 1175; +pub const ERROR_UNABLE_TO_MOVE_REPLACEMENT: DWORD = 1176; +pub const ERROR_UNABLE_TO_MOVE_REPLACEMENT_2: DWORD = 1177; +pub const ERROR_JOURNAL_DELETE_IN_PROGRESS: DWORD = 1178; +pub const ERROR_JOURNAL_NOT_ACTIVE: DWORD = 1179; +pub const ERROR_POTENTIAL_FILE_FOUND: DWORD = 1180; +pub const ERROR_JOURNAL_ENTRY_DELETED: DWORD = 1181; +pub const ERROR_BAD_DEVICE: DWORD = 1200; +pub const ERROR_CONNECTION_UNAVAIL: DWORD = 1201; +pub const ERROR_DEVICE_ALREADY_REMEMBERED: DWORD = 1202; +pub const ERROR_NO_NET_OR_BAD_PATH: DWORD = 1203; +pub const ERROR_BAD_PROVIDER: DWORD = 1204; +pub const ERROR_CANNOT_OPEN_PROFILE: DWORD = 1205; +pub const ERROR_BAD_PROFILE: DWORD = 1206; +pub const ERROR_NOT_CONTAINER: DWORD = 1207; +pub const ERROR_EXTENDED_ERROR: DWORD = 1208; +pub const ERROR_INVALID_GROUPNAME: DWORD = 1209; +pub const ERROR_INVALID_COMPUTERNAME: DWORD = 1210; +pub const ERROR_INVALID_EVENTNAME: DWORD = 1211; +pub const ERROR_INVALID_DOMAINNAME: DWORD = 1212; +pub const ERROR_INVALID_SERVICENAME: DWORD = 1213; +pub const ERROR_INVALID_NETNAME: DWORD = 1214; +pub const ERROR_INVALID_SHARENAME: DWORD = 1215; +pub const ERROR_INVALID_PASSWORDNAME: DWORD = 1216; +pub const ERROR_INVALID_MESSAGENAME: DWORD = 1217; +pub const ERROR_INVALID_MESSAGEDEST: DWORD = 1218; +pub const ERROR_SESSION_CREDENTIAL_CONFLICT: DWORD = 1219; +pub const ERROR_REMOTE_SESSION_LIMIT_EXCEEDED: DWORD = 1220; +pub const ERROR_DUP_DOMAINNAME: DWORD = 1221; +pub const ERROR_NO_NETWORK: DWORD = 1222; +pub const ERROR_CANCELLED: DWORD = 1223; +pub const ERROR_USER_MAPPED_FILE: DWORD = 1224; +pub const ERROR_CONNECTION_REFUSED: DWORD = 1225; +pub const ERROR_GRACEFUL_DISCONNECT: DWORD = 1226; +pub const ERROR_ADDRESS_ALREADY_ASSOCIATED: DWORD = 1227; +pub const ERROR_ADDRESS_NOT_ASSOCIATED: DWORD = 1228; +pub const ERROR_CONNECTION_INVALID: DWORD = 1229; +pub const ERROR_CONNECTION_ACTIVE: DWORD = 1230; +pub const ERROR_NETWORK_UNREACHABLE: DWORD = 1231; +pub const ERROR_HOST_UNREACHABLE: DWORD = 1232; +pub const ERROR_PROTOCOL_UNREACHABLE: DWORD = 1233; +pub const ERROR_PORT_UNREACHABLE: DWORD = 1234; +pub const ERROR_REQUEST_ABORTED: DWORD = 1235; +pub const ERROR_CONNECTION_ABORTED: DWORD = 1236; +pub const ERROR_RETRY: DWORD = 1237; +pub const ERROR_CONNECTION_COUNT_LIMIT: DWORD = 1238; +pub const ERROR_LOGIN_TIME_RESTRICTION: DWORD = 1239; +pub const ERROR_LOGIN_WKSTA_RESTRICTION: DWORD = 1240; +pub const ERROR_INCORRECT_ADDRESS: DWORD = 1241; +pub const ERROR_ALREADY_REGISTERED: DWORD = 1242; +pub const ERROR_SERVICE_NOT_FOUND: DWORD = 1243; +pub const ERROR_NOT_AUTHENTICATED: DWORD = 1244; +pub const ERROR_NOT_LOGGED_ON: DWORD = 1245; +pub const ERROR_CONTINUE: DWORD = 1246; +pub const ERROR_ALREADY_INITIALIZED: DWORD = 1247; +pub const ERROR_NO_MORE_DEVICES: DWORD = 1248; +pub const ERROR_NO_SUCH_SITE: DWORD = 1249; +pub const ERROR_DOMAIN_CONTROLLER_EXISTS: DWORD = 1250; +pub const ERROR_ONLY_IF_CONNECTED: DWORD = 1251; +pub const ERROR_OVERRIDE_NOCHANGES: DWORD = 1252; +pub const ERROR_BAD_USER_PROFILE: DWORD = 1253; +pub const ERROR_NOT_SUPPORTED_ON_SBS: DWORD = 1254; +pub const ERROR_SERVER_SHUTDOWN_IN_PROGRESS: DWORD = 1255; +pub const ERROR_HOST_DOWN: DWORD = 1256; +pub const ERROR_NON_ACCOUNT_SID: DWORD = 1257; +pub const ERROR_NON_DOMAIN_SID: DWORD = 1258; +pub const ERROR_APPHELP_BLOCK: DWORD = 1259; +pub const ERROR_ACCESS_DISABLED_BY_POLICY: DWORD = 1260; +pub const ERROR_REG_NAT_CONSUMPTION: DWORD = 1261; +pub const ERROR_CSCSHARE_OFFLINE: DWORD = 1262; +pub const ERROR_PKINIT_FAILURE: DWORD = 1263; +pub const ERROR_SMARTCARD_SUBSYSTEM_FAILURE: DWORD = 1264; +pub const ERROR_DOWNGRADE_DETECTED: DWORD = 1265; +pub const ERROR_MACHINE_LOCKED: DWORD = 1271; +pub const ERROR_CALLBACK_SUPPLIED_INVALID_DATA: DWORD = 1273; +pub const ERROR_SYNC_FOREGROUND_REFRESH_REQUIRED: DWORD = 1274; +pub const ERROR_DRIVER_BLOCKED: DWORD = 1275; +pub const ERROR_INVALID_IMPORT_OF_NON_DLL: DWORD = 1276; +pub const ERROR_ACCESS_DISABLED_WEBBLADE: DWORD = 1277; +pub const ERROR_ACCESS_DISABLED_WEBBLADE_TAMPER: DWORD = 1278; +pub const ERROR_RECOVERY_FAILURE: DWORD = 1279; +pub const ERROR_ALREADY_FIBER: DWORD = 1280; +pub const ERROR_ALREADY_THREAD: DWORD = 1281; +pub const ERROR_STACK_BUFFER_OVERRUN: DWORD = 1282; +pub const ERROR_PARAMETER_QUOTA_EXCEEDED: DWORD = 1283; +pub const ERROR_DEBUGGER_INACTIVE: DWORD = 1284; +pub const ERROR_DELAY_LOAD_FAILED: DWORD = 1285; +pub const ERROR_VDM_DISALLOWED: DWORD = 1286; +pub const ERROR_UNIDENTIFIED_ERROR: DWORD = 1287; +pub const ERROR_NOT_ALL_ASSIGNED: DWORD = 1300; +pub const ERROR_SOME_NOT_MAPPED: DWORD = 1301; +pub const ERROR_NO_QUOTAS_FOR_ACCOUNT: DWORD = 1302; +pub const ERROR_LOCAL_USER_SESSION_KEY: DWORD = 1303; +pub const ERROR_NULL_LM_PASSWORD: DWORD = 1304; +pub const ERROR_UNKNOWN_REVISION: DWORD = 1305; +pub const ERROR_REVISION_MISMATCH: DWORD = 1306; +pub const ERROR_INVALID_OWNER: DWORD = 1307; +pub const ERROR_INVALID_PRIMARY_GROUP: DWORD = 1308; +pub const ERROR_NO_IMPERSONATION_TOKEN: DWORD = 1309; +pub const ERROR_CANT_DISABLE_MANDATORY: DWORD = 1310; +pub const ERROR_NO_LOGON_SERVERS: DWORD = 1311; +pub const ERROR_NO_SUCH_LOGON_SESSION: DWORD = 1312; +pub const ERROR_NO_SUCH_PRIVILEGE: DWORD = 1313; +pub const ERROR_PRIVILEGE_NOT_HELD: DWORD = 1314; +pub const ERROR_INVALID_ACCOUNT_NAME: DWORD = 1315; +pub const ERROR_USER_EXISTS: DWORD = 1316; +pub const ERROR_NO_SUCH_USER: DWORD = 1317; +pub const ERROR_GROUP_EXISTS: DWORD = 1318; +pub const ERROR_NO_SUCH_GROUP: DWORD = 1319; +pub const ERROR_MEMBER_IN_GROUP: DWORD = 1320; +pub const ERROR_MEMBER_NOT_IN_GROUP: DWORD = 1321; +pub const ERROR_LAST_ADMIN: DWORD = 1322; +pub const ERROR_WRONG_PASSWORD: DWORD = 1323; +pub const ERROR_ILL_FORMED_PASSWORD: DWORD = 1324; +pub const ERROR_PASSWORD_RESTRICTION: DWORD = 1325; +pub const ERROR_LOGON_FAILURE: DWORD = 1326; +pub const ERROR_ACCOUNT_RESTRICTION: DWORD = 1327; +pub const ERROR_INVALID_LOGON_HOURS: DWORD = 1328; +pub const ERROR_INVALID_WORKSTATION: DWORD = 1329; +pub const ERROR_PASSWORD_EXPIRED: DWORD = 1330; +pub const ERROR_ACCOUNT_DISABLED: DWORD = 1331; +pub const ERROR_NONE_MAPPED: DWORD = 1332; +pub const ERROR_TOO_MANY_LUIDS_REQUESTED: DWORD = 1333; +pub const ERROR_LUIDS_EXHAUSTED: DWORD = 1334; +pub const ERROR_INVALID_SUB_AUTHORITY: DWORD = 1335; +pub const ERROR_INVALID_ACL: DWORD = 1336; +pub const ERROR_INVALID_SID: DWORD = 1337; +pub const ERROR_INVALID_SECURITY_DESCR: DWORD = 1338; +pub const ERROR_BAD_INHERITANCE_ACL: DWORD = 1340; +pub const ERROR_SERVER_DISABLED: DWORD = 1341; +pub const ERROR_SERVER_NOT_DISABLED: DWORD = 1342; +pub const ERROR_INVALID_ID_AUTHORITY: DWORD = 1343; +pub const ERROR_ALLOTTED_SPACE_EXCEEDED: DWORD = 1344; +pub const ERROR_INVALID_GROUP_ATTRIBUTES: DWORD = 1345; +pub const ERROR_BAD_IMPERSONATION_LEVEL: DWORD = 1346; +pub const ERROR_CANT_OPEN_ANONYMOUS: DWORD = 1347; +pub const ERROR_BAD_VALIDATION_CLASS: DWORD = 1348; +pub const ERROR_BAD_TOKEN_TYPE: DWORD = 1349; +pub const ERROR_NO_SECURITY_ON_OBJECT: DWORD = 1350; +pub const ERROR_CANT_ACCESS_DOMAIN_INFO: DWORD = 1351; +pub const ERROR_INVALID_SERVER_STATE: DWORD = 1352; +pub const ERROR_INVALID_DOMAIN_STATE: DWORD = 1353; +pub const ERROR_INVALID_DOMAIN_ROLE: DWORD = 1354; +pub const ERROR_NO_SUCH_DOMAIN: DWORD = 1355; +pub const ERROR_DOMAIN_EXISTS: DWORD = 1356; +pub const ERROR_DOMAIN_LIMIT_EXCEEDED: DWORD = 1357; +pub const ERROR_INTERNAL_DB_CORRUPTION: DWORD = 1358; +pub const ERROR_INTERNAL_ERROR: DWORD = 1359; +pub const ERROR_GENERIC_NOT_MAPPED: DWORD = 1360; +pub const ERROR_BAD_DESCRIPTOR_FORMAT: DWORD = 1361; +pub const ERROR_NOT_LOGON_PROCESS: DWORD = 1362; +pub const ERROR_LOGON_SESSION_EXISTS: DWORD = 1363; +pub const ERROR_NO_SUCH_PACKAGE: DWORD = 1364; +pub const ERROR_BAD_LOGON_SESSION_STATE: DWORD = 1365; +pub const ERROR_LOGON_SESSION_COLLISION: DWORD = 1366; +pub const ERROR_INVALID_LOGON_TYPE: DWORD = 1367; +pub const ERROR_CANNOT_IMPERSONATE: DWORD = 1368; +pub const ERROR_RXACT_INVALID_STATE: DWORD = 1369; +pub const ERROR_RXACT_COMMIT_FAILURE: DWORD = 1370; +pub const ERROR_SPECIAL_ACCOUNT: DWORD = 1371; +pub const ERROR_SPECIAL_GROUP: DWORD = 1372; +pub const ERROR_SPECIAL_USER: DWORD = 1373; +pub const ERROR_MEMBERS_PRIMARY_GROUP: DWORD = 1374; +pub const ERROR_TOKEN_ALREADY_IN_USE: DWORD = 1375; +pub const ERROR_NO_SUCH_ALIAS: DWORD = 1376; +pub const ERROR_MEMBER_NOT_IN_ALIAS: DWORD = 1377; +pub const ERROR_MEMBER_IN_ALIAS: DWORD = 1378; +pub const ERROR_ALIAS_EXISTS: DWORD = 1379; +pub const ERROR_LOGON_NOT_GRANTED: DWORD = 1380; +pub const ERROR_TOO_MANY_SECRETS: DWORD = 1381; +pub const ERROR_SECRET_TOO_LONG: DWORD = 1382; +pub const ERROR_INTERNAL_DB_ERROR: DWORD = 1383; +pub const ERROR_TOO_MANY_CONTEXT_IDS: DWORD = 1384; +pub const ERROR_LOGON_TYPE_NOT_GRANTED: DWORD = 1385; +pub const ERROR_NT_CROSS_ENCRYPTION_REQUIRED: DWORD = 1386; +pub const ERROR_NO_SUCH_MEMBER: DWORD = 1387; +pub const ERROR_INVALID_MEMBER: DWORD = 1388; +pub const ERROR_TOO_MANY_SIDS: DWORD = 1389; +pub const ERROR_LM_CROSS_ENCRYPTION_REQUIRED: DWORD = 1390; +pub const ERROR_NO_INHERITANCE: DWORD = 1391; +pub const ERROR_FILE_CORRUPT: DWORD = 1392; +pub const ERROR_DISK_CORRUPT: DWORD = 1393; +pub const ERROR_NO_USER_SESSION_KEY: DWORD = 1394; +pub const ERROR_LICENSE_QUOTA_EXCEEDED: DWORD = 1395; +pub const ERROR_WRONG_TARGET_NAME: DWORD = 1396; +pub const ERROR_MUTUAL_AUTH_FAILED: DWORD = 1397; +pub const ERROR_TIME_SKEW: DWORD = 1398; +pub const ERROR_CURRENT_DOMAIN_NOT_ALLOWED: DWORD = 1399; +pub const ERROR_INVALID_WINDOW_HANDLE: DWORD = 1400; +pub const ERROR_INVALID_MENU_HANDLE: DWORD = 1401; +pub const ERROR_INVALID_CURSOR_HANDLE: DWORD = 1402; +pub const ERROR_INVALID_ACCEL_HANDLE: DWORD = 1403; +pub const ERROR_INVALID_HOOK_HANDLE: DWORD = 1404; +pub const ERROR_INVALID_DWP_HANDLE: DWORD = 1405; +pub const ERROR_TLW_WITH_WSCHILD: DWORD = 1406; +pub const ERROR_CANNOT_FIND_WND_CLASS: DWORD = 1407; +pub const ERROR_WINDOW_OF_OTHER_THREAD: DWORD = 1408; +pub const ERROR_HOTKEY_ALREADY_REGISTERED: DWORD = 1409; +pub const ERROR_CLASS_ALREADY_EXISTS: DWORD = 1410; +pub const ERROR_CLASS_DOES_NOT_EXIST: DWORD = 1411; +pub const ERROR_CLASS_HAS_WINDOWS: DWORD = 1412; +pub const ERROR_INVALID_INDEX: DWORD = 1413; +pub const ERROR_INVALID_ICON_HANDLE: DWORD = 1414; +pub const ERROR_PRIVATE_DIALOG_INDEX: DWORD = 1415; +pub const ERROR_LISTBOX_ID_NOT_FOUND: DWORD = 1416; +pub const ERROR_NO_WILDCARD_CHARACTERS: DWORD = 1417; +pub const ERROR_CLIPBOARD_NOT_OPEN: DWORD = 1418; +pub const ERROR_HOTKEY_NOT_REGISTERED: DWORD = 1419; +pub const ERROR_WINDOW_NOT_DIALOG: DWORD = 1420; +pub const ERROR_CONTROL_ID_NOT_FOUND: DWORD = 1421; +pub const ERROR_INVALID_COMBOBOX_MESSAGE: DWORD = 1422; +pub const ERROR_WINDOW_NOT_COMBOBOX: DWORD = 1423; +pub const ERROR_INVALID_EDIT_HEIGHT: DWORD = 1424; +pub const ERROR_DC_NOT_FOUND: DWORD = 1425; +pub const ERROR_INVALID_HOOK_FILTER: DWORD = 1426; +pub const ERROR_INVALID_FILTER_PROC: DWORD = 1427; +pub const ERROR_HOOK_NEEDS_HMOD: DWORD = 1428; +pub const ERROR_GLOBAL_ONLY_HOOK: DWORD = 1429; +pub const ERROR_JOURNAL_HOOK_SET: DWORD = 1430; +pub const ERROR_HOOK_NOT_INSTALLED: DWORD = 1431; +pub const ERROR_INVALID_LB_MESSAGE: DWORD = 1432; +pub const ERROR_SETCOUNT_ON_BAD_LB: DWORD = 1433; +pub const ERROR_LB_WITHOUT_TABSTOPS: DWORD = 1434; +pub const ERROR_DESTROY_OBJECT_OF_OTHER_THREAD: DWORD = 1435; +pub const ERROR_CHILD_WINDOW_MENU: DWORD = 1436; +pub const ERROR_NO_SYSTEM_MENU: DWORD = 1437; +pub const ERROR_INVALID_MSGBOX_STYLE: DWORD = 1438; +pub const ERROR_INVALID_SPI_VALUE: DWORD = 1439; +pub const ERROR_SCREEN_ALREADY_LOCKED: DWORD = 1440; +pub const ERROR_HWNDS_HAVE_DIFF_PARENT: DWORD = 1441; +pub const ERROR_NOT_CHILD_WINDOW: DWORD = 1442; +pub const ERROR_INVALID_GW_COMMAND: DWORD = 1443; +pub const ERROR_INVALID_THREAD_ID: DWORD = 1444; +pub const ERROR_NON_MDICHILD_WINDOW: DWORD = 1445; +pub const ERROR_POPUP_ALREADY_ACTIVE: DWORD = 1446; +pub const ERROR_NO_SCROLLBARS: DWORD = 1447; +pub const ERROR_INVALID_SCROLLBAR_RANGE: DWORD = 1448; +pub const ERROR_INVALID_SHOWWIN_COMMAND: DWORD = 1449; +pub const ERROR_NO_SYSTEM_RESOURCES: DWORD = 1450; +pub const ERROR_NONPAGED_SYSTEM_RESOURCES: DWORD = 1451; +pub const ERROR_PAGED_SYSTEM_RESOURCES: DWORD = 1452; +pub const ERROR_WORKING_SET_QUOTA: DWORD = 1453; +pub const ERROR_PAGEFILE_QUOTA: DWORD = 1454; +pub const ERROR_COMMITMENT_LIMIT: DWORD = 1455; +pub const ERROR_MENU_ITEM_NOT_FOUND: DWORD = 1456; +pub const ERROR_INVALID_KEYBOARD_HANDLE: DWORD = 1457; +pub const ERROR_HOOK_TYPE_NOT_ALLOWED: DWORD = 1458; +pub const ERROR_REQUIRES_INTERACTIVE_WINDOWSTATION: DWORD = 1459; +pub const ERROR_TIMEOUT: DWORD = 1460; +pub const ERROR_INVALID_MONITOR_HANDLE: DWORD = 1461; +pub const ERROR_INCORRECT_SIZE: DWORD = 1462; +pub const ERROR_SYMLINK_CLASS_DISABLED: DWORD = 1463; +pub const ERROR_SYMLINK_NOT_SUPPORTED: DWORD = 1464; +pub const ERROR_XML_PARSE_ERROR: DWORD = 1465; +pub const ERROR_XMLDSIG_ERROR: DWORD = 1466; +pub const ERROR_RESTART_APPLICATION: DWORD = 1467; +pub const ERROR_WRONG_COMPARTMENT: DWORD = 1468; +pub const ERROR_AUTHIP_FAILURE: DWORD = 1469; +pub const ERROR_NO_NVRAM_RESOURCES: DWORD = 1470; +pub const ERROR_NOT_GUI_PROCESS: DWORD = 1471; +pub const ERROR_EVENTLOG_FILE_CORRUPT: DWORD = 1500; +pub const ERROR_EVENTLOG_CANT_START: DWORD = 1501; +pub const ERROR_LOG_FILE_FULL: DWORD = 1502; +pub const ERROR_EVENTLOG_FILE_CHANGED: DWORD = 1503; +pub const ERROR_INSTALL_SERVICE_FAILURE: DWORD = 1601; +pub const ERROR_INSTALL_USEREXIT: DWORD = 1602; +pub const ERROR_INSTALL_FAILURE: DWORD = 1603; +pub const ERROR_INSTALL_SUSPEND: DWORD = 1604; +pub const ERROR_UNKNOWN_PRODUCT: DWORD = 1605; +pub const ERROR_UNKNOWN_FEATURE: DWORD = 1606; +pub const ERROR_UNKNOWN_COMPONENT: DWORD = 1607; +pub const ERROR_UNKNOWN_PROPERTY: DWORD = 1608; +pub const ERROR_INVALID_HANDLE_STATE: DWORD = 1609; +pub const ERROR_BAD_CONFIGURATION: DWORD = 1610; +pub const ERROR_INDEX_ABSENT: DWORD = 1611; +pub const ERROR_INSTALL_SOURCE_ABSENT: DWORD = 1612; +pub const ERROR_INSTALL_PACKAGE_VERSION: DWORD = 1613; +pub const ERROR_PRODUCT_UNINSTALLED: DWORD = 1614; +pub const ERROR_BAD_QUERY_SYNTAX: DWORD = 1615; +pub const ERROR_INVALID_FIELD: DWORD = 1616; +pub const ERROR_DEVICE_REMOVED: DWORD = 1617; +pub const ERROR_INSTALL_ALREADY_RUNNING: DWORD = 1618; +pub const ERROR_INSTALL_PACKAGE_OPEN_FAILED: DWORD = 1619; +pub const ERROR_INSTALL_PACKAGE_INVALID: DWORD = 1620; +pub const ERROR_INSTALL_UI_FAILURE: DWORD = 1621; +pub const ERROR_INSTALL_LOG_FAILURE: DWORD = 1622; +pub const ERROR_INSTALL_LANGUAGE_UNSUPPORTED: DWORD = 1623; +pub const ERROR_INSTALL_TRANSFORM_FAILURE: DWORD = 1624; +pub const ERROR_INSTALL_PACKAGE_REJECTED: DWORD = 1625; +pub const ERROR_FUNCTION_NOT_CALLED: DWORD = 1626; +pub const ERROR_FUNCTION_FAILED: DWORD = 1627; +pub const ERROR_INVALID_TABLE: DWORD = 1628; +pub const ERROR_DATATYPE_MISMATCH: DWORD = 1629; +pub const ERROR_UNSUPPORTED_TYPE: DWORD = 1630; +pub const ERROR_CREATE_FAILED: DWORD = 1631; +pub const ERROR_INSTALL_TEMP_UNWRITABLE: DWORD = 1632; +pub const ERROR_INSTALL_PLATFORM_UNSUPPORTED: DWORD = 1633; +pub const ERROR_INSTALL_NOTUSED: DWORD = 1634; +pub const ERROR_PATCH_PACKAGE_OPEN_FAILED: DWORD = 1635; +pub const ERROR_PATCH_PACKAGE_INVALID: DWORD = 1636; +pub const ERROR_PATCH_PACKAGE_UNSUPPORTED: DWORD = 1637; +pub const ERROR_PRODUCT_VERSION: DWORD = 1638; +pub const ERROR_INVALID_COMMAND_LINE: DWORD = 1639; +pub const ERROR_INSTALL_REMOTE_DISALLOWED: DWORD = 1640; +pub const ERROR_SUCCESS_REBOOT_INITIATED: DWORD = 1641; +pub const ERROR_PATCH_TARGET_NOT_FOUND: DWORD = 1642; +pub const ERROR_PATCH_PACKAGE_REJECTED: DWORD = 1643; +pub const ERROR_INSTALL_TRANSFORM_REJECTED: DWORD = 1644; +pub const ERROR_INSTALL_REMOTE_PROHIBITED: DWORD = 1645; +pub const ERROR_INVALID_USER_BUFFER: DWORD = 1784; +pub const ERROR_UNRECOGNIZED_MEDIA: DWORD = 1785; +pub const ERROR_NO_TRUST_LSA_SECRET: DWORD = 1786; +pub const ERROR_NO_TRUST_SAM_ACCOUNT: DWORD = 1787; +pub const ERROR_TRUSTED_DOMAIN_FAILURE: DWORD = 1788; +pub const ERROR_TRUSTED_RELATIONSHIP_FAILURE: DWORD = 1789; +pub const ERROR_TRUST_FAILURE: DWORD = 1790; +pub const ERROR_NETLOGON_NOT_STARTED: DWORD = 1792; +pub const ERROR_ACCOUNT_EXPIRED: DWORD = 1793; +pub const ERROR_REDIRECTOR_HAS_OPEN_HANDLES: DWORD = 1794; +pub const ERROR_PRINTER_DRIVER_ALREADY_INSTALLED: DWORD = 1795; +pub const ERROR_UNKNOWN_PORT: DWORD = 1796; +pub const ERROR_UNKNOWN_PRINTER_DRIVER: DWORD = 1797; +pub const ERROR_UNKNOWN_PRINTPROCESSOR: DWORD = 1798; +pub const ERROR_INVALID_SEPARATOR_FILE: DWORD = 1799; +pub const ERROR_INVALID_PRIORITY: DWORD = 1800; +pub const ERROR_INVALID_PRINTER_NAME: DWORD = 1801; +pub const ERROR_PRINTER_ALREADY_EXISTS: DWORD = 1802; +pub const ERROR_INVALID_PRINTER_COMMAND: DWORD = 1803; +pub const ERROR_INVALID_DATATYPE: DWORD = 1804; +pub const ERROR_INVALID_ENVIRONMENT: DWORD = 1805; +pub const ERROR_NOLOGON_INTERDOMAIN_TRUST_ACCOUNT: DWORD = 1807; +pub const ERROR_NOLOGON_WORKSTATION_TRUST_ACCOUNT: DWORD = 1808; +pub const ERROR_NOLOGON_SERVER_TRUST_ACCOUNT: DWORD = 1809; +pub const ERROR_DOMAIN_TRUST_INCONSISTENT: DWORD = 1810; +pub const ERROR_SERVER_HAS_OPEN_HANDLES: DWORD = 1811; +pub const ERROR_RESOURCE_DATA_NOT_FOUND: DWORD = 1812; +pub const ERROR_RESOURCE_TYPE_NOT_FOUND: DWORD = 1813; +pub const ERROR_RESOURCE_NAME_NOT_FOUND: DWORD = 1814; +pub const ERROR_RESOURCE_LANG_NOT_FOUND: DWORD = 1815; +pub const ERROR_NOT_ENOUGH_QUOTA: DWORD = 1816; +pub const ERROR_INVALID_TIME: DWORD = 1901; +pub const ERROR_INVALID_FORM_NAME: DWORD = 1902; +pub const ERROR_INVALID_FORM_SIZE: DWORD = 1903; +pub const ERROR_ALREADY_WAITING: DWORD = 1904; +pub const ERROR_PRINTER_DELETED: DWORD = 1905; +pub const ERROR_INVALID_PRINTER_STATE: DWORD = 1906; +pub const ERROR_PASSWORD_MUST_CHANGE: DWORD = 1907; +pub const ERROR_DOMAIN_CONTROLLER_NOT_FOUND: DWORD = 1908; +pub const ERROR_ACCOUNT_LOCKED_OUT: DWORD = 1909; +pub const ERROR_NO_SITENAME: DWORD = 1919; +pub const ERROR_CANT_ACCESS_FILE: DWORD = 1920; +pub const ERROR_CANT_RESOLVE_FILENAME: DWORD = 1921; +pub const ERROR_KM_DRIVER_BLOCKED: DWORD = 1930; +pub const ERROR_CONTEXT_EXPIRED: DWORD = 1931; +pub const ERROR_PER_USER_TRUST_QUOTA_EXCEEDED: DWORD = 1932; +pub const ERROR_ALL_USER_TRUST_QUOTA_EXCEEDED: DWORD = 1933; +pub const ERROR_USER_DELETE_TRUST_QUOTA_EXCEEDED: DWORD = 1934; +pub const ERROR_AUTHENTICATION_FIREWALL_FAILED: DWORD = 1935; +pub const ERROR_REMOTE_PRINT_CONNECTIONS_BLOCKED: DWORD = 1936; +pub const ERROR_INVALID_PIXEL_FORMAT: DWORD = 2000; +pub const ERROR_BAD_DRIVER: DWORD = 2001; +pub const ERROR_INVALID_WINDOW_STYLE: DWORD = 2002; +pub const ERROR_METAFILE_NOT_SUPPORTED: DWORD = 2003; +pub const ERROR_TRANSFORM_NOT_SUPPORTED: DWORD = 2004; +pub const ERROR_CLIPPING_NOT_SUPPORTED: DWORD = 2005; +pub const ERROR_INVALID_CMM: DWORD = 2010; +pub const ERROR_INVALID_PROFILE: DWORD = 2011; +pub const ERROR_TAG_NOT_FOUND: DWORD = 2012; +pub const ERROR_TAG_NOT_PRESENT: DWORD = 2013; +pub const ERROR_DUPLICATE_TAG: DWORD = 2014; +pub const ERROR_PROFILE_NOT_ASSOCIATED_WITH_DEVICE: DWORD = 2015; +pub const ERROR_PROFILE_NOT_FOUND: DWORD = 2016; +pub const ERROR_INVALID_COLORSPACE: DWORD = 2017; +pub const ERROR_ICM_NOT_ENABLED: DWORD = 2018; +pub const ERROR_DELETING_ICM_XFORM: DWORD = 2019; +pub const ERROR_INVALID_TRANSFORM: DWORD = 2020; +pub const ERROR_COLORSPACE_MISMATCH: DWORD = 2021; +pub const ERROR_INVALID_COLORINDEX: DWORD = 2022; +pub const ERROR_CONNECTED_OTHER_PASSWORD: DWORD = 2108; +pub const ERROR_CONNECTED_OTHER_PASSWORD_DEFAULT: DWORD = 2109; +pub const ERROR_BAD_USERNAME: DWORD = 2202; +pub const ERROR_NOT_CONNECTED: DWORD = 2250; +pub const ERROR_OPEN_FILES: DWORD = 2401; +pub const ERROR_ACTIVE_CONNECTIONS: DWORD = 2402; +pub const ERROR_DEVICE_IN_USE: DWORD = 2404; +pub const ERROR_UNKNOWN_PRINT_MONITOR: DWORD = 3000; +pub const ERROR_PRINTER_DRIVER_IN_USE: DWORD = 3001; +pub const ERROR_SPOOL_FILE_NOT_FOUND: DWORD = 3002; +pub const ERROR_SPL_NO_STARTDOC: DWORD = 3003; +pub const ERROR_SPL_NO_ADDJOB: DWORD = 3004; +pub const ERROR_PRINT_PROCESSOR_ALREADY_INSTALLED: DWORD = 3005; +pub const ERROR_PRINT_MONITOR_ALREADY_INSTALLED: DWORD = 3006; +pub const ERROR_INVALID_PRINT_MONITOR: DWORD = 3007; +pub const ERROR_PRINT_MONITOR_IN_USE: DWORD = 3008; +pub const ERROR_PRINTER_HAS_JOBS_QUEUED: DWORD = 3009; +pub const ERROR_SUCCESS_REBOOT_REQUIRED: DWORD = 3010; +pub const ERROR_SUCCESS_RESTART_REQUIRED: DWORD = 3011; +pub const ERROR_PRINTER_NOT_FOUND: DWORD = 3012; +pub const ERROR_PRINTER_DRIVER_WARNED: DWORD = 3013; +pub const ERROR_PRINTER_DRIVER_BLOCKED: DWORD = 3014; +pub const ERROR_WINS_INTERNAL: DWORD = 4000; +pub const ERROR_CAN_NOT_DEL_LOCAL_WINS: DWORD = 4001; +pub const ERROR_STATIC_INIT: DWORD = 4002; +pub const ERROR_INC_BACKUP: DWORD = 4003; +pub const ERROR_FULL_BACKUP: DWORD = 4004; +pub const ERROR_REC_NON_EXISTENT: DWORD = 4005; +pub const ERROR_RPL_NOT_ALLOWED: DWORD = 4006; +pub const ERROR_DHCP_ADDRESS_CONFLICT: DWORD = 4100; +pub const ERROR_WMI_GUID_NOT_FOUND: DWORD = 4200; +pub const ERROR_WMI_INSTANCE_NOT_FOUND: DWORD = 4201; +pub const ERROR_WMI_ITEMID_NOT_FOUND: DWORD = 4202; +pub const ERROR_WMI_TRY_AGAIN: DWORD = 4203; +pub const ERROR_WMI_DP_NOT_FOUND: DWORD = 4204; +pub const ERROR_WMI_UNRESOLVED_INSTANCE_REF: DWORD = 4205; +pub const ERROR_WMI_ALREADY_ENABLED: DWORD = 4206; +pub const ERROR_WMI_GUID_DISCONNECTED: DWORD = 4207; +pub const ERROR_WMI_SERVER_UNAVAILABLE: DWORD = 4208; +pub const ERROR_WMI_DP_FAILED: DWORD = 4209; +pub const ERROR_WMI_INVALID_MOF: DWORD = 4210; +pub const ERROR_WMI_INVALID_REGINFO: DWORD = 4211; +pub const ERROR_WMI_ALREADY_DISABLED: DWORD = 4212; +pub const ERROR_WMI_READ_ONLY: DWORD = 4213; +pub const ERROR_WMI_SET_FAILURE: DWORD = 4214; +pub const ERROR_INVALID_MEDIA: DWORD = 4300; +pub const ERROR_INVALID_LIBRARY: DWORD = 4301; +pub const ERROR_INVALID_MEDIA_POOL: DWORD = 4302; +pub const ERROR_DRIVE_MEDIA_MISMATCH: DWORD = 4303; +pub const ERROR_MEDIA_OFFLINE: DWORD = 4304; +pub const ERROR_LIBRARY_OFFLINE: DWORD = 4305; +pub const ERROR_EMPTY: DWORD = 4306; +pub const ERROR_NOT_EMPTY: DWORD = 4307; +pub const ERROR_MEDIA_UNAVAILABLE: DWORD = 4308; +pub const ERROR_RESOURCE_DISABLED: DWORD = 4309; +pub const ERROR_INVALID_CLEANER: DWORD = 4310; +pub const ERROR_UNABLE_TO_CLEAN: DWORD = 4311; +pub const ERROR_OBJECT_NOT_FOUND: DWORD = 4312; +pub const ERROR_DATABASE_FAILURE: DWORD = 4313; +pub const ERROR_DATABASE_FULL: DWORD = 4314; +pub const ERROR_MEDIA_INCOMPATIBLE: DWORD = 4315; +pub const ERROR_RESOURCE_NOT_PRESENT: DWORD = 4316; +pub const ERROR_INVALID_OPERATION: DWORD = 4317; +pub const ERROR_MEDIA_NOT_AVAILABLE: DWORD = 4318; +pub const ERROR_DEVICE_NOT_AVAILABLE: DWORD = 4319; +pub const ERROR_REQUEST_REFUSED: DWORD = 4320; +pub const ERROR_INVALID_DRIVE_OBJECT: DWORD = 4321; +pub const ERROR_LIBRARY_FULL: DWORD = 4322; +pub const ERROR_MEDIUM_NOT_ACCESSIBLE: DWORD = 4323; +pub const ERROR_UNABLE_TO_LOAD_MEDIUM: DWORD = 4324; +pub const ERROR_UNABLE_TO_INVENTORY_DRIVE: DWORD = 4325; +pub const ERROR_UNABLE_TO_INVENTORY_SLOT: DWORD = 4326; +pub const ERROR_UNABLE_TO_INVENTORY_TRANSPORT: DWORD = 4327; +pub const ERROR_TRANSPORT_FULL: DWORD = 4328; +pub const ERROR_CONTROLLING_IEPORT: DWORD = 4329; +pub const ERROR_UNABLE_TO_EJECT_MOUNTED_MEDIA: DWORD = 4330; +pub const ERROR_CLEANER_SLOT_SET: DWORD = 4331; +pub const ERROR_CLEANER_SLOT_NOT_SET: DWORD = 4332; +pub const ERROR_CLEANER_CARTRIDGE_SPENT: DWORD = 4333; +pub const ERROR_UNEXPECTED_OMID: DWORD = 4334; +pub const ERROR_CANT_DELETE_LAST_ITEM: DWORD = 4335; +pub const ERROR_MESSAGE_EXCEEDS_MAX_SIZE: DWORD = 4336; +pub const ERROR_VOLUME_CONTAINS_SYS_FILES: DWORD = 4337; +pub const ERROR_INDIGENOUS_TYPE: DWORD = 4338; +pub const ERROR_NO_SUPPORTING_DRIVES: DWORD = 4339; +pub const ERROR_CLEANER_CARTRIDGE_INSTALLED: DWORD = 4340; +pub const ERROR_IEPORT_FULL: DWORD = 4341; +pub const ERROR_FILE_OFFLINE: DWORD = 4350; +pub const ERROR_REMOTE_STORAGE_NOT_ACTIVE: DWORD = 4351; +pub const ERROR_REMOTE_STORAGE_MEDIA_ERROR: DWORD = 4352; +pub const ERROR_NOT_A_REPARSE_POINT: DWORD = 4390; +pub const ERROR_REPARSE_ATTRIBUTE_CONFLICT: DWORD = 4391; +pub const ERROR_INVALID_REPARSE_DATA: DWORD = 4392; +pub const ERROR_REPARSE_TAG_INVALID: DWORD = 4393; +pub const ERROR_REPARSE_TAG_MISMATCH: DWORD = 4394; +pub const ERROR_VOLUME_NOT_SIS_ENABLED: DWORD = 4500; +pub const ERROR_DEPENDENT_RESOURCE_EXISTS: DWORD = 5001; +pub const ERROR_DEPENDENCY_NOT_FOUND: DWORD = 5002; +pub const ERROR_DEPENDENCY_ALREADY_EXISTS: DWORD = 5003; +pub const ERROR_RESOURCE_NOT_ONLINE: DWORD = 5004; +pub const ERROR_HOST_NODE_NOT_AVAILABLE: DWORD = 5005; +pub const ERROR_RESOURCE_NOT_AVAILABLE: DWORD = 5006; +pub const ERROR_RESOURCE_NOT_FOUND: DWORD = 5007; +pub const ERROR_SHUTDOWN_CLUSTER: DWORD = 5008; +pub const ERROR_CANT_EVICT_ACTIVE_NODE: DWORD = 5009; +pub const ERROR_OBJECT_ALREADY_EXISTS: DWORD = 5010; +pub const ERROR_OBJECT_IN_LIST: DWORD = 5011; +pub const ERROR_GROUP_NOT_AVAILABLE: DWORD = 5012; +pub const ERROR_GROUP_NOT_FOUND: DWORD = 5013; +pub const ERROR_GROUP_NOT_ONLINE: DWORD = 5014; +pub const ERROR_HOST_NODE_NOT_RESOURCE_OWNER: DWORD = 5015; +pub const ERROR_HOST_NODE_NOT_GROUP_OWNER: DWORD = 5016; +pub const ERROR_RESMON_CREATE_FAILED: DWORD = 5017; +pub const ERROR_RESMON_ONLINE_FAILED: DWORD = 5018; +pub const ERROR_RESOURCE_ONLINE: DWORD = 5019; +pub const ERROR_QUORUM_RESOURCE: DWORD = 5020; +pub const ERROR_NOT_QUORUM_CAPABLE: DWORD = 5021; +pub const ERROR_CLUSTER_SHUTTING_DOWN: DWORD = 5022; +pub const ERROR_INVALID_STATE: DWORD = 5023; +pub const ERROR_RESOURCE_PROPERTIES_STORED: DWORD = 5024; +pub const ERROR_NOT_QUORUM_CLASS: DWORD = 5025; +pub const ERROR_CORE_RESOURCE: DWORD = 5026; +pub const ERROR_QUORUM_RESOURCE_ONLINE_FAILED: DWORD = 5027; +pub const ERROR_QUORUMLOG_OPEN_FAILED: DWORD = 5028; +pub const ERROR_CLUSTERLOG_CORRUPT: DWORD = 5029; +pub const ERROR_CLUSTERLOG_RECORD_EXCEEDS_MAXSIZE: DWORD = 5030; +pub const ERROR_CLUSTERLOG_EXCEEDS_MAXSIZE: DWORD = 5031; +pub const ERROR_CLUSTERLOG_CHKPOINT_NOT_FOUND: DWORD = 5032; +pub const ERROR_CLUSTERLOG_NOT_ENOUGH_SPACE: DWORD = 5033; +pub const ERROR_QUORUM_OWNER_ALIVE: DWORD = 5034; +pub const ERROR_NETWORK_NOT_AVAILABLE: DWORD = 5035; +pub const ERROR_NODE_NOT_AVAILABLE: DWORD = 5036; +pub const ERROR_ALL_NODES_NOT_AVAILABLE: DWORD = 5037; +pub const ERROR_RESOURCE_FAILED: DWORD = 5038; +pub const ERROR_CLUSTER_INVALID_NODE: DWORD = 5039; +pub const ERROR_CLUSTER_NODE_EXISTS: DWORD = 5040; +pub const ERROR_CLUSTER_JOIN_IN_PROGRESS: DWORD = 5041; +pub const ERROR_CLUSTER_NODE_NOT_FOUND: DWORD = 5042; +pub const ERROR_CLUSTER_LOCAL_NODE_NOT_FOUND: DWORD = 5043; +pub const ERROR_CLUSTER_NETWORK_EXISTS: DWORD = 5044; +pub const ERROR_CLUSTER_NETWORK_NOT_FOUND: DWORD = 5045; +pub const ERROR_CLUSTER_NETINTERFACE_EXISTS: DWORD = 5046; +pub const ERROR_CLUSTER_NETINTERFACE_NOT_FOUND: DWORD = 5047; +pub const ERROR_CLUSTER_INVALID_REQUEST: DWORD = 5048; +pub const ERROR_CLUSTER_INVALID_NETWORK_PROVIDER: DWORD = 5049; +pub const ERROR_CLUSTER_NODE_DOWN: DWORD = 5050; +pub const ERROR_CLUSTER_NODE_UNREACHABLE: DWORD = 5051; +pub const ERROR_CLUSTER_NODE_NOT_MEMBER: DWORD = 5052; +pub const ERROR_CLUSTER_JOIN_NOT_IN_PROGRESS: DWORD = 5053; +pub const ERROR_CLUSTER_INVALID_NETWORK: DWORD = 5054; +pub const ERROR_CLUSTER_NODE_UP: DWORD = 5056; +pub const ERROR_CLUSTER_IPADDR_IN_USE: DWORD = 5057; +pub const ERROR_CLUSTER_NODE_NOT_PAUSED: DWORD = 5058; +pub const ERROR_CLUSTER_NO_SECURITY_CONTEXT: DWORD = 5059; +pub const ERROR_CLUSTER_NETWORK_NOT_INTERNAL: DWORD = 5060; +pub const ERROR_CLUSTER_NODE_ALREADY_UP: DWORD = 5061; +pub const ERROR_CLUSTER_NODE_ALREADY_DOWN: DWORD = 5062; +pub const ERROR_CLUSTER_NETWORK_ALREADY_ONLINE: DWORD = 5063; +pub const ERROR_CLUSTER_NETWORK_ALREADY_OFFLINE: DWORD = 5064; +pub const ERROR_CLUSTER_NODE_ALREADY_MEMBER: DWORD = 5065; +pub const ERROR_CLUSTER_LAST_INTERNAL_NETWORK: DWORD = 5066; +pub const ERROR_CLUSTER_NETWORK_HAS_DEPENDENTS: DWORD = 5067; +pub const ERROR_INVALID_OPERATION_ON_QUORUM: DWORD = 5068; +pub const ERROR_DEPENDENCY_NOT_ALLOWED: DWORD = 5069; +pub const ERROR_CLUSTER_NODE_PAUSED: DWORD = 5070; +pub const ERROR_NODE_CANT_HOST_RESOURCE: DWORD = 5071; +pub const ERROR_CLUSTER_NODE_NOT_READY: DWORD = 5072; +pub const ERROR_CLUSTER_NODE_SHUTTING_DOWN: DWORD = 5073; +pub const ERROR_CLUSTER_JOIN_ABORTED: DWORD = 5074; +pub const ERROR_CLUSTER_INCOMPATIBLE_VERSIONS: DWORD = 5075; +pub const ERROR_CLUSTER_MAXNUM_OF_RESOURCES_EXCEEDED: DWORD = 5076; +pub const ERROR_CLUSTER_SYSTEM_CONFIG_CHANGED: DWORD = 5077; +pub const ERROR_CLUSTER_RESOURCE_TYPE_NOT_FOUND: DWORD = 5078; +pub const ERROR_CLUSTER_RESTYPE_NOT_SUPPORTED: DWORD = 5079; +pub const ERROR_CLUSTER_RESNAME_NOT_FOUND: DWORD = 5080; +pub const ERROR_CLUSTER_NO_RPC_PACKAGES_REGISTERED: DWORD = 5081; +pub const ERROR_CLUSTER_OWNER_NOT_IN_PREFLIST: DWORD = 5082; +pub const ERROR_CLUSTER_DATABASE_SEQMISMATCH: DWORD = 5083; +pub const ERROR_RESMON_INVALID_STATE: DWORD = 5084; +pub const ERROR_CLUSTER_GUM_NOT_LOCKER: DWORD = 5085; +pub const ERROR_QUORUM_DISK_NOT_FOUND: DWORD = 5086; +pub const ERROR_DATABASE_BACKUP_CORRUPT: DWORD = 5087; +pub const ERROR_CLUSTER_NODE_ALREADY_HAS_DFS_ROOT: DWORD = 5088; +pub const ERROR_RESOURCE_PROPERTY_UNCHANGEABLE: DWORD = 5089; +pub const ERROR_CLUSTER_MEMBERSHIP_INVALID_STATE: DWORD = 5890; +pub const ERROR_CLUSTER_QUORUMLOG_NOT_FOUND: DWORD = 5891; +pub const ERROR_CLUSTER_MEMBERSHIP_HALT: DWORD = 5892; +pub const ERROR_CLUSTER_INSTANCE_ID_MISMATCH: DWORD = 5893; +pub const ERROR_CLUSTER_NETWORK_NOT_FOUND_FOR_IP: DWORD = 5894; +pub const ERROR_CLUSTER_PROPERTY_DATA_TYPE_MISMATCH: DWORD = 5895; +pub const ERROR_CLUSTER_EVICT_WITHOUT_CLEANUP: DWORD = 5896; +pub const ERROR_CLUSTER_PARAMETER_MISMATCH: DWORD = 5897; +pub const ERROR_NODE_CANNOT_BE_CLUSTERED: DWORD = 5898; +pub const ERROR_CLUSTER_WRONG_OS_VERSION: DWORD = 5899; +pub const ERROR_CLUSTER_CANT_CREATE_DUP_CLUSTER_NAME: DWORD = 5900; +pub const ERROR_CLUSCFG_ALREADY_COMMITTED: DWORD = 5901; +pub const ERROR_CLUSCFG_ROLLBACK_FAILED: DWORD = 5902; +pub const ERROR_CLUSCFG_SYSTEM_DISK_DRIVE_LETTER_CONFLICT: DWORD = 5903; +pub const ERROR_CLUSTER_OLD_VERSION: DWORD = 5904; +pub const ERROR_CLUSTER_MISMATCHED_COMPUTER_ACCT_NAME: DWORD = 5905; +pub const ERROR_ENCRYPTION_FAILED: DWORD = 6000; +pub const ERROR_DECRYPTION_FAILED: DWORD = 6001; +pub const ERROR_FILE_ENCRYPTED: DWORD = 6002; +pub const ERROR_NO_RECOVERY_POLICY: DWORD = 6003; +pub const ERROR_NO_EFS: DWORD = 6004; +pub const ERROR_WRONG_EFS: DWORD = 6005; +pub const ERROR_NO_USER_KEYS: DWORD = 6006; +pub const ERROR_FILE_NOT_ENCRYPTED: DWORD = 6007; +pub const ERROR_NOT_EXPORT_FORMAT: DWORD = 6008; +pub const ERROR_FILE_READ_ONLY: DWORD = 6009; +pub const ERROR_DIR_EFS_DISALLOWED: DWORD = 6010; +pub const ERROR_EFS_SERVER_NOT_TRUSTED: DWORD = 6011; +pub const ERROR_BAD_RECOVERY_POLICY: DWORD = 6012; +pub const ERROR_EFS_ALG_BLOB_TOO_BIG: DWORD = 6013; +pub const ERROR_VOLUME_NOT_SUPPORT_EFS: DWORD = 6014; +pub const ERROR_EFS_DISABLED: DWORD = 6015; +pub const ERROR_EFS_VERSION_NOT_SUPPORT: DWORD = 6016; +pub const ERROR_NO_BROWSER_SERVERS_FOUND: DWORD = 6118; +pub const ERROR_CTX_WINSTATION_NAME_INVALID: DWORD = 7001; +pub const ERROR_CTX_INVALID_PD: DWORD = 7002; +pub const ERROR_CTX_PD_NOT_FOUND: DWORD = 7003; +pub const ERROR_CTX_WD_NOT_FOUND: DWORD = 7004; +pub const ERROR_CTX_CANNOT_MAKE_EVENTLOG_ENTRY: DWORD = 7005; +pub const ERROR_CTX_SERVICE_NAME_COLLISION: DWORD = 7006; +pub const ERROR_CTX_CLOSE_PENDING: DWORD = 7007; +pub const ERROR_CTX_NO_OUTBUF: DWORD = 7008; +pub const ERROR_CTX_MODEM_INF_NOT_FOUND: DWORD = 7009; +pub const ERROR_CTX_INVALID_MODEMNAME: DWORD = 7010; +pub const ERROR_CTX_MODEM_RESPONSE_ERROR: DWORD = 7011; +pub const ERROR_CTX_MODEM_RESPONSE_TIMEOUT: DWORD = 7012; +pub const ERROR_CTX_MODEM_RESPONSE_NO_CARRIER: DWORD = 7013; +pub const ERROR_CTX_MODEM_RESPONSE_NO_DIALTONE: DWORD = 7014; +pub const ERROR_CTX_MODEM_RESPONSE_BUSY: DWORD = 7015; +pub const ERROR_CTX_MODEM_RESPONSE_VOICE: DWORD = 7016; +pub const ERROR_CTX_TD_ERROR: DWORD = 7017; +pub const ERROR_CTX_WINSTATION_NOT_FOUND: DWORD = 7022; +pub const ERROR_CTX_WINSTATION_ALREADY_EXISTS: DWORD = 7023; +pub const ERROR_CTX_WINSTATION_BUSY: DWORD = 7024; +pub const ERROR_CTX_BAD_VIDEO_MODE: DWORD = 7025; +pub const ERROR_CTX_GRAPHICS_INVALID: DWORD = 7035; +pub const ERROR_CTX_LOGON_DISABLED: DWORD = 7037; +pub const ERROR_CTX_NOT_CONSOLE: DWORD = 7038; +pub const ERROR_CTX_CLIENT_QUERY_TIMEOUT: DWORD = 7040; +pub const ERROR_CTX_CONSOLE_DISCONNECT: DWORD = 7041; +pub const ERROR_CTX_CONSOLE_CONNECT: DWORD = 7042; +pub const ERROR_CTX_SHADOW_DENIED: DWORD = 7044; +pub const ERROR_CTX_WINSTATION_ACCESS_DENIED: DWORD = 7045; +pub const ERROR_CTX_INVALID_WD: DWORD = 7049; +pub const ERROR_CTX_SHADOW_INVALID: DWORD = 7050; +pub const ERROR_CTX_SHADOW_DISABLED: DWORD = 7051; +pub const ERROR_CTX_CLIENT_LICENSE_IN_USE: DWORD = 7052; +pub const ERROR_CTX_CLIENT_LICENSE_NOT_SET: DWORD = 7053; +pub const ERROR_CTX_LICENSE_NOT_AVAILABLE: DWORD = 7054; +pub const ERROR_CTX_LICENSE_CLIENT_INVALID: DWORD = 7055; +pub const ERROR_CTX_LICENSE_EXPIRED: DWORD = 7056; +pub const ERROR_CTX_SHADOW_NOT_RUNNING: DWORD = 7057; +pub const ERROR_CTX_SHADOW_ENDED_BY_MODE_CHANGE: DWORD = 7058; +pub const ERROR_ACTIVATION_COUNT_EXCEEDED: DWORD = 7059; +pub const ERROR_DS_NOT_INSTALLED: DWORD = 8200; +pub const ERROR_DS_MEMBERSHIP_EVALUATED_LOCALLY: DWORD = 8201; +pub const ERROR_DS_NO_ATTRIBUTE_OR_VALUE: DWORD = 8202; +pub const ERROR_DS_INVALID_ATTRIBUTE_SYNTAX: DWORD = 8203; +pub const ERROR_DS_ATTRIBUTE_TYPE_UNDEFINED: DWORD = 8204; +pub const ERROR_DS_ATTRIBUTE_OR_VALUE_EXISTS: DWORD = 8205; +pub const ERROR_DS_BUSY: DWORD = 8206; +pub const ERROR_DS_UNAVAILABLE: DWORD = 8207; +pub const ERROR_DS_NO_RIDS_ALLOCATED: DWORD = 8208; +pub const ERROR_DS_NO_MORE_RIDS: DWORD = 8209; +pub const ERROR_DS_INCORRECT_ROLE_OWNER: DWORD = 8210; +pub const ERROR_DS_RIDMGR_INIT_ERROR: DWORD = 8211; +pub const ERROR_DS_OBJ_CLASS_VIOLATION: DWORD = 8212; +pub const ERROR_DS_CANT_ON_NON_LEAF: DWORD = 8213; +pub const ERROR_DS_CANT_ON_RDN: DWORD = 8214; +pub const ERROR_DS_CANT_MOD_OBJ_CLASS: DWORD = 8215; +pub const ERROR_DS_CROSS_DOM_MOVE_ERROR: DWORD = 8216; +pub const ERROR_DS_GC_NOT_AVAILABLE: DWORD = 8217; +pub const ERROR_SHARED_POLICY: DWORD = 8218; +pub const ERROR_POLICY_OBJECT_NOT_FOUND: DWORD = 8219; +pub const ERROR_POLICY_ONLY_IN_DS: DWORD = 8220; +pub const ERROR_PROMOTION_ACTIVE: DWORD = 8221; +pub const ERROR_NO_PROMOTION_ACTIVE: DWORD = 8222; +pub const ERROR_DS_OPERATIONS_ERROR: DWORD = 8224; +pub const ERROR_DS_PROTOCOL_ERROR: DWORD = 8225; +pub const ERROR_DS_TIMELIMIT_EXCEEDED: DWORD = 8226; +pub const ERROR_DS_SIZELIMIT_EXCEEDED: DWORD = 8227; +pub const ERROR_DS_ADMIN_LIMIT_EXCEEDED: DWORD = 8228; +pub const ERROR_DS_COMPARE_FALSE: DWORD = 8229; +pub const ERROR_DS_COMPARE_TRUE: DWORD = 8230; +pub const ERROR_DS_AUTH_METHOD_NOT_SUPPORTED: DWORD = 8231; +pub const ERROR_DS_STRONG_AUTH_REQUIRED: DWORD = 8232; +pub const ERROR_DS_INAPPROPRIATE_AUTH: DWORD = 8233; +pub const ERROR_DS_AUTH_UNKNOWN: DWORD = 8234; +pub const ERROR_DS_REFERRAL: DWORD = 8235; +pub const ERROR_DS_UNAVAILABLE_CRIT_EXTENSION: DWORD = 8236; +pub const ERROR_DS_CONFIDENTIALITY_REQUIRED: DWORD = 8237; +pub const ERROR_DS_INAPPROPRIATE_MATCHING: DWORD = 8238; +pub const ERROR_DS_CONSTRAINT_VIOLATION: DWORD = 8239; +pub const ERROR_DS_NO_SUCH_OBJECT: DWORD = 8240; +pub const ERROR_DS_ALIAS_PROBLEM: DWORD = 8241; +pub const ERROR_DS_INVALID_DN_SYNTAX: DWORD = 8242; +pub const ERROR_DS_IS_LEAF: DWORD = 8243; +pub const ERROR_DS_ALIAS_DEREF_PROBLEM: DWORD = 8244; +pub const ERROR_DS_UNWILLING_TO_PERFORM: DWORD = 8245; +pub const ERROR_DS_LOOP_DETECT: DWORD = 8246; +pub const ERROR_DS_NAMING_VIOLATION: DWORD = 8247; +pub const ERROR_DS_OBJECT_RESULTS_TOO_LARGE: DWORD = 8248; +pub const ERROR_DS_AFFECTS_MULTIPLE_DSAS: DWORD = 8249; +pub const ERROR_DS_SERVER_DOWN: DWORD = 8250; +pub const ERROR_DS_LOCAL_ERROR: DWORD = 8251; +pub const ERROR_DS_ENCODING_ERROR: DWORD = 8252; +pub const ERROR_DS_DECODING_ERROR: DWORD = 8253; +pub const ERROR_DS_FILTER_UNKNOWN: DWORD = 8254; +pub const ERROR_DS_PARAM_ERROR: DWORD = 8255; +pub const ERROR_DS_NOT_SUPPORTED: DWORD = 8256; +pub const ERROR_DS_NO_RESULTS_RETURNED: DWORD = 8257; +pub const ERROR_DS_CONTROL_NOT_FOUND: DWORD = 8258; +pub const ERROR_DS_CLIENT_LOOP: DWORD = 8259; +pub const ERROR_DS_REFERRAL_LIMIT_EXCEEDED: DWORD = 8260; +pub const ERROR_DS_SORT_CONTROL_MISSING: DWORD = 8261; +pub const ERROR_DS_OFFSET_RANGE_ERROR: DWORD = 8262; +pub const ERROR_DS_ROOT_MUST_BE_NC: DWORD = 8301; +pub const ERROR_DS_ADD_REPLICA_INHIBITED: DWORD = 8302; +pub const ERROR_DS_ATT_NOT_DEF_IN_SCHEMA: DWORD = 8303; +pub const ERROR_DS_MAX_OBJ_SIZE_EXCEEDED: DWORD = 8304; +pub const ERROR_DS_OBJ_STRING_NAME_EXISTS: DWORD = 8305; +pub const ERROR_DS_NO_RDN_DEFINED_IN_SCHEMA: DWORD = 8306; +pub const ERROR_DS_RDN_DOESNT_MATCH_SCHEMA: DWORD = 8307; +pub const ERROR_DS_NO_REQUESTED_ATTS_FOUND: DWORD = 8308; +pub const ERROR_DS_USER_BUFFER_TO_SMALL: DWORD = 8309; +pub const ERROR_DS_ATT_IS_NOT_ON_OBJ: DWORD = 8310; +pub const ERROR_DS_ILLEGAL_MOD_OPERATION: DWORD = 8311; +pub const ERROR_DS_OBJ_TOO_LARGE: DWORD = 8312; +pub const ERROR_DS_BAD_INSTANCE_TYPE: DWORD = 8313; +pub const ERROR_DS_MASTERDSA_REQUIRED: DWORD = 8314; +pub const ERROR_DS_OBJECT_CLASS_REQUIRED: DWORD = 8315; +pub const ERROR_DS_MISSING_REQUIRED_ATT: DWORD = 8316; +pub const ERROR_DS_ATT_NOT_DEF_FOR_CLASS: DWORD = 8317; +pub const ERROR_DS_ATT_ALREADY_EXISTS: DWORD = 8318; +pub const ERROR_DS_CANT_ADD_ATT_VALUES: DWORD = 8320; +pub const ERROR_DS_SINGLE_VALUE_CONSTRAINT: DWORD = 8321; +pub const ERROR_DS_RANGE_CONSTRAINT: DWORD = 8322; +pub const ERROR_DS_ATT_VAL_ALREADY_EXISTS: DWORD = 8323; +pub const ERROR_DS_CANT_REM_MISSING_ATT: DWORD = 8324; +pub const ERROR_DS_CANT_REM_MISSING_ATT_VAL: DWORD = 8325; +pub const ERROR_DS_ROOT_CANT_BE_SUBREF: DWORD = 8326; +pub const ERROR_DS_NO_CHAINING: DWORD = 8327; +pub const ERROR_DS_NO_CHAINED_EVAL: DWORD = 8328; +pub const ERROR_DS_NO_PARENT_OBJECT: DWORD = 8329; +pub const ERROR_DS_PARENT_IS_AN_ALIAS: DWORD = 8330; +pub const ERROR_DS_CANT_MIX_MASTER_AND_REPS: DWORD = 8331; +pub const ERROR_DS_CHILDREN_EXIST: DWORD = 8332; +pub const ERROR_DS_OBJ_NOT_FOUND: DWORD = 8333; +pub const ERROR_DS_ALIASED_OBJ_MISSING: DWORD = 8334; +pub const ERROR_DS_BAD_NAME_SYNTAX: DWORD = 8335; +pub const ERROR_DS_ALIAS_POINTS_TO_ALIAS: DWORD = 8336; +pub const ERROR_DS_CANT_DEREF_ALIAS: DWORD = 8337; +pub const ERROR_DS_OUT_OF_SCOPE: DWORD = 8338; +pub const ERROR_DS_OBJECT_BEING_REMOVED: DWORD = 8339; +pub const ERROR_DS_CANT_DELETE_DSA_OBJ: DWORD = 8340; +pub const ERROR_DS_GENERIC_ERROR: DWORD = 8341; +pub const ERROR_DS_DSA_MUST_BE_INT_MASTER: DWORD = 8342; +pub const ERROR_DS_CLASS_NOT_DSA: DWORD = 8343; +pub const ERROR_DS_INSUFF_ACCESS_RIGHTS: DWORD = 8344; +pub const ERROR_DS_ILLEGAL_SUPERIOR: DWORD = 8345; +pub const ERROR_DS_ATTRIBUTE_OWNED_BY_SAM: DWORD = 8346; +pub const ERROR_DS_NAME_TOO_MANY_PARTS: DWORD = 8347; +pub const ERROR_DS_NAME_TOO_LONG: DWORD = 8348; +pub const ERROR_DS_NAME_VALUE_TOO_LONG: DWORD = 8349; +pub const ERROR_DS_NAME_UNPARSEABLE: DWORD = 8350; +pub const ERROR_DS_NAME_TYPE_UNKNOWN: DWORD = 8351; +pub const ERROR_DS_NOT_AN_OBJECT: DWORD = 8352; +pub const ERROR_DS_SEC_DESC_TOO_SHORT: DWORD = 8353; +pub const ERROR_DS_SEC_DESC_INVALID: DWORD = 8354; +pub const ERROR_DS_NO_DELETED_NAME: DWORD = 8355; +pub const ERROR_DS_SUBREF_MUST_HAVE_PARENT: DWORD = 8356; +pub const ERROR_DS_NCNAME_MUST_BE_NC: DWORD = 8357; +pub const ERROR_DS_CANT_ADD_SYSTEM_ONLY: DWORD = 8358; +pub const ERROR_DS_CLASS_MUST_BE_CONCRETE: DWORD = 8359; +pub const ERROR_DS_INVALID_DMD: DWORD = 8360; +pub const ERROR_DS_OBJ_GUID_EXISTS: DWORD = 8361; +pub const ERROR_DS_NOT_ON_BACKLINK: DWORD = 8362; +pub const ERROR_DS_NO_CROSSREF_FOR_NC: DWORD = 8363; +pub const ERROR_DS_SHUTTING_DOWN: DWORD = 8364; +pub const ERROR_DS_UNKNOWN_OPERATION: DWORD = 8365; +pub const ERROR_DS_INVALID_ROLE_OWNER: DWORD = 8366; +pub const ERROR_DS_COULDNT_CONTACT_FSMO: DWORD = 8367; +pub const ERROR_DS_CROSS_NC_DN_RENAME: DWORD = 8368; +pub const ERROR_DS_CANT_MOD_SYSTEM_ONLY: DWORD = 8369; +pub const ERROR_DS_REPLICATOR_ONLY: DWORD = 8370; +pub const ERROR_DS_OBJ_CLASS_NOT_DEFINED: DWORD = 8371; +pub const ERROR_DS_OBJ_CLASS_NOT_SUBCLASS: DWORD = 8372; +pub const ERROR_DS_NAME_REFERENCE_INVALID: DWORD = 8373; +pub const ERROR_DS_CROSS_REF_EXISTS: DWORD = 8374; +pub const ERROR_DS_CANT_DEL_MASTER_CROSSREF: DWORD = 8375; +pub const ERROR_DS_SUBTREE_NOTIFY_NOT_NC_HEAD: DWORD = 8376; +pub const ERROR_DS_NOTIFY_FILTER_TOO_COMPLEX: DWORD = 8377; +pub const ERROR_DS_DUP_RDN: DWORD = 8378; +pub const ERROR_DS_DUP_OID: DWORD = 8379; +pub const ERROR_DS_DUP_MAPI_ID: DWORD = 8380; +pub const ERROR_DS_DUP_SCHEMA_ID_GUID: DWORD = 8381; +pub const ERROR_DS_DUP_LDAP_DISPLAY_NAME: DWORD = 8382; +pub const ERROR_DS_SEMANTIC_ATT_TEST: DWORD = 8383; +pub const ERROR_DS_SYNTAX_MISMATCH: DWORD = 8384; +pub const ERROR_DS_EXISTS_IN_MUST_HAVE: DWORD = 8385; +pub const ERROR_DS_EXISTS_IN_MAY_HAVE: DWORD = 8386; +pub const ERROR_DS_NONEXISTENT_MAY_HAVE: DWORD = 8387; +pub const ERROR_DS_NONEXISTENT_MUST_HAVE: DWORD = 8388; +pub const ERROR_DS_AUX_CLS_TEST_FAIL: DWORD = 8389; +pub const ERROR_DS_NONEXISTENT_POSS_SUP: DWORD = 8390; +pub const ERROR_DS_SUB_CLS_TEST_FAIL: DWORD = 8391; +pub const ERROR_DS_BAD_RDN_ATT_ID_SYNTAX: DWORD = 8392; +pub const ERROR_DS_EXISTS_IN_AUX_CLS: DWORD = 8393; +pub const ERROR_DS_EXISTS_IN_SUB_CLS: DWORD = 8394; +pub const ERROR_DS_EXISTS_IN_POSS_SUP: DWORD = 8395; +pub const ERROR_DS_RECALCSCHEMA_FAILED: DWORD = 8396; +pub const ERROR_DS_TREE_DELETE_NOT_FINISHED: DWORD = 8397; +pub const ERROR_DS_CANT_DELETE: DWORD = 8398; +pub const ERROR_DS_ATT_SCHEMA_REQ_ID: DWORD = 8399; +pub const ERROR_DS_BAD_ATT_SCHEMA_SYNTAX: DWORD = 8400; +pub const ERROR_DS_CANT_CACHE_ATT: DWORD = 8401; +pub const ERROR_DS_CANT_CACHE_CLASS: DWORD = 8402; +pub const ERROR_DS_CANT_REMOVE_ATT_CACHE: DWORD = 8403; +pub const ERROR_DS_CANT_REMOVE_CLASS_CACHE: DWORD = 8404; +pub const ERROR_DS_CANT_RETRIEVE_DN: DWORD = 8405; +pub const ERROR_DS_MISSING_SUPREF: DWORD = 8406; +pub const ERROR_DS_CANT_RETRIEVE_INSTANCE: DWORD = 8407; +pub const ERROR_DS_CODE_INCONSISTENCY: DWORD = 8408; +pub const ERROR_DS_DATABASE_ERROR: DWORD = 8409; +pub const ERROR_DS_GOVERNSID_MISSING: DWORD = 8410; +pub const ERROR_DS_MISSING_EXPECTED_ATT: DWORD = 8411; +pub const ERROR_DS_NCNAME_MISSING_CR_REF: DWORD = 8412; +pub const ERROR_DS_SECURITY_CHECKING_ERROR: DWORD = 8413; +pub const ERROR_DS_SCHEMA_NOT_LOADED: DWORD = 8414; +pub const ERROR_DS_SCHEMA_ALLOC_FAILED: DWORD = 8415; +pub const ERROR_DS_ATT_SCHEMA_REQ_SYNTAX: DWORD = 8416; +pub const ERROR_DS_GCVERIFY_ERROR: DWORD = 8417; +pub const ERROR_DS_DRA_SCHEMA_MISMATCH: DWORD = 8418; +pub const ERROR_DS_CANT_FIND_DSA_OBJ: DWORD = 8419; +pub const ERROR_DS_CANT_FIND_EXPECTED_NC: DWORD = 8420; +pub const ERROR_DS_CANT_FIND_NC_IN_CACHE: DWORD = 8421; +pub const ERROR_DS_CANT_RETRIEVE_CHILD: DWORD = 8422; +pub const ERROR_DS_SECURITY_ILLEGAL_MODIFY: DWORD = 8423; +pub const ERROR_DS_CANT_REPLACE_HIDDEN_REC: DWORD = 8424; +pub const ERROR_DS_BAD_HIERARCHY_FILE: DWORD = 8425; +pub const ERROR_DS_BUILD_HIERARCHY_TABLE_FAILED: DWORD = 8426; +pub const ERROR_DS_CONFIG_PARAM_MISSING: DWORD = 8427; +pub const ERROR_DS_COUNTING_AB_INDICES_FAILED: DWORD = 8428; +pub const ERROR_DS_HIERARCHY_TABLE_MALLOC_FAILED: DWORD = 8429; +pub const ERROR_DS_INTERNAL_FAILURE: DWORD = 8430; +pub const ERROR_DS_UNKNOWN_ERROR: DWORD = 8431; +pub const ERROR_DS_ROOT_REQUIRES_CLASS_TOP: DWORD = 8432; +pub const ERROR_DS_REFUSING_FSMO_ROLES: DWORD = 8433; +pub const ERROR_DS_MISSING_FSMO_SETTINGS: DWORD = 8434; +pub const ERROR_DS_UNABLE_TO_SURRENDER_ROLES: DWORD = 8435; +pub const ERROR_DS_DRA_GENERIC: DWORD = 8436; +pub const ERROR_DS_DRA_INVALID_PARAMETER: DWORD = 8437; +pub const ERROR_DS_DRA_BUSY: DWORD = 8438; +pub const ERROR_DS_DRA_BAD_DN: DWORD = 8439; +pub const ERROR_DS_DRA_BAD_NC: DWORD = 8440; +pub const ERROR_DS_DRA_DN_EXISTS: DWORD = 8441; +pub const ERROR_DS_DRA_INTERNAL_ERROR: DWORD = 8442; +pub const ERROR_DS_DRA_INCONSISTENT_DIT: DWORD = 8443; +pub const ERROR_DS_DRA_CONNECTION_FAILED: DWORD = 8444; +pub const ERROR_DS_DRA_BAD_INSTANCE_TYPE: DWORD = 8445; +pub const ERROR_DS_DRA_OUT_OF_MEM: DWORD = 8446; +pub const ERROR_DS_DRA_MAIL_PROBLEM: DWORD = 8447; +pub const ERROR_DS_DRA_REF_ALREADY_EXISTS: DWORD = 8448; +pub const ERROR_DS_DRA_REF_NOT_FOUND: DWORD = 8449; +pub const ERROR_DS_DRA_OBJ_IS_REP_SOURCE: DWORD = 8450; +pub const ERROR_DS_DRA_DB_ERROR: DWORD = 8451; +pub const ERROR_DS_DRA_NO_REPLICA: DWORD = 8452; +pub const ERROR_DS_DRA_ACCESS_DENIED: DWORD = 8453; +pub const ERROR_DS_DRA_NOT_SUPPORTED: DWORD = 8454; +pub const ERROR_DS_DRA_RPC_CANCELLED: DWORD = 8455; +pub const ERROR_DS_DRA_SOURCE_DISABLED: DWORD = 8456; +pub const ERROR_DS_DRA_SINK_DISABLED: DWORD = 8457; +pub const ERROR_DS_DRA_NAME_COLLISION: DWORD = 8458; +pub const ERROR_DS_DRA_SOURCE_REINSTALLED: DWORD = 8459; +pub const ERROR_DS_DRA_MISSING_PARENT: DWORD = 8460; +pub const ERROR_DS_DRA_PREEMPTED: DWORD = 8461; +pub const ERROR_DS_DRA_ABANDON_SYNC: DWORD = 8462; +pub const ERROR_DS_DRA_SHUTDOWN: DWORD = 8463; +pub const ERROR_DS_DRA_INCOMPATIBLE_PARTIAL_SET: DWORD = 8464; +pub const ERROR_DS_DRA_SOURCE_IS_PARTIAL_REPLICA: DWORD = 8465; +pub const ERROR_DS_DRA_EXTN_CONNECTION_FAILED: DWORD = 8466; +pub const ERROR_DS_INSTALL_SCHEMA_MISMATCH: DWORD = 8467; +pub const ERROR_DS_DUP_LINK_ID: DWORD = 8468; +pub const ERROR_DS_NAME_ERROR_RESOLVING: DWORD = 8469; +pub const ERROR_DS_NAME_ERROR_NOT_FOUND: DWORD = 8470; +pub const ERROR_DS_NAME_ERROR_NOT_UNIQUE: DWORD = 8471; +pub const ERROR_DS_NAME_ERROR_NO_MAPPING: DWORD = 8472; +pub const ERROR_DS_NAME_ERROR_DOMAIN_ONLY: DWORD = 8473; +pub const ERROR_DS_NAME_ERROR_NO_SYNTACTICAL_MAPPING: DWORD = 8474; +pub const ERROR_DS_CONSTRUCTED_ATT_MOD: DWORD = 8475; +pub const ERROR_DS_WRONG_OM_OBJ_CLASS: DWORD = 8476; +pub const ERROR_DS_DRA_REPL_PENDING: DWORD = 8477; +pub const ERROR_DS_DS_REQUIRED: DWORD = 8478; +pub const ERROR_DS_INVALID_LDAP_DISPLAY_NAME: DWORD = 8479; +pub const ERROR_DS_NON_BASE_SEARCH: DWORD = 8480; +pub const ERROR_DS_CANT_RETRIEVE_ATTS: DWORD = 8481; +pub const ERROR_DS_BACKLINK_WITHOUT_LINK: DWORD = 8482; +pub const ERROR_DS_EPOCH_MISMATCH: DWORD = 8483; +pub const ERROR_DS_SRC_NAME_MISMATCH: DWORD = 8484; +pub const ERROR_DS_SRC_AND_DST_NC_IDENTICAL: DWORD = 8485; +pub const ERROR_DS_DST_NC_MISMATCH: DWORD = 8486; +pub const ERROR_DS_NOT_AUTHORITIVE_FOR_DST_NC: DWORD = 8487; +pub const ERROR_DS_SRC_GUID_MISMATCH: DWORD = 8488; +pub const ERROR_DS_CANT_MOVE_DELETED_OBJECT: DWORD = 8489; +pub const ERROR_DS_PDC_OPERATION_IN_PROGRESS: DWORD = 8490; +pub const ERROR_DS_CROSS_DOMAIN_CLEANUP_REQD: DWORD = 8491; +pub const ERROR_DS_ILLEGAL_XDOM_MOVE_OPERATION: DWORD = 8492; +pub const ERROR_DS_CANT_WITH_ACCT_GROUP_MEMBERSHPS: DWORD = 8493; +pub const ERROR_DS_NC_MUST_HAVE_NC_PARENT: DWORD = 8494; +pub const ERROR_DS_CR_IMPOSSIBLE_TO_VALIDATE: DWORD = 8495; +pub const ERROR_DS_DST_DOMAIN_NOT_NATIVE: DWORD = 8496; +pub const ERROR_DS_MISSING_INFRASTRUCTURE_CONTAINER: DWORD = 8497; +pub const ERROR_DS_CANT_MOVE_ACCOUNT_GROUP: DWORD = 8498; +pub const ERROR_DS_CANT_MOVE_RESOURCE_GROUP: DWORD = 8499; +pub const ERROR_DS_INVALID_SEARCH_FLAG: DWORD = 8500; +pub const ERROR_DS_NO_TREE_DELETE_ABOVE_NC: DWORD = 8501; +pub const ERROR_DS_COULDNT_LOCK_TREE_FOR_DELETE: DWORD = 8502; +pub const ERROR_DS_COULDNT_IDENTIFY_OBJECTS_FOR_TREE_DELETE: DWORD = 8503; +pub const ERROR_DS_SAM_INIT_FAILURE: DWORD = 8504; +pub const ERROR_DS_SENSITIVE_GROUP_VIOLATION: DWORD = 8505; +pub const ERROR_DS_CANT_MOD_PRIMARYGROUPID: DWORD = 8506; +pub const ERROR_DS_ILLEGAL_BASE_SCHEMA_MOD: DWORD = 8507; +pub const ERROR_DS_NONSAFE_SCHEMA_CHANGE: DWORD = 8508; +pub const ERROR_DS_SCHEMA_UPDATE_DISALLOWED: DWORD = 8509; +pub const ERROR_DS_CANT_CREATE_UNDER_SCHEMA: DWORD = 8510; +pub const ERROR_DS_INSTALL_NO_SRC_SCH_VERSION: DWORD = 8511; +pub const ERROR_DS_INSTALL_NO_SCH_VERSION_IN_INIFILE: DWORD = 8512; +pub const ERROR_DS_INVALID_GROUP_TYPE: DWORD = 8513; +pub const ERROR_DS_NO_NEST_GLOBALGROUP_IN_MIXEDDOMAIN: DWORD = 8514; +pub const ERROR_DS_NO_NEST_LOCALGROUP_IN_MIXEDDOMAIN: DWORD = 8515; +pub const ERROR_DS_GLOBAL_CANT_HAVE_LOCAL_MEMBER: DWORD = 8516; +pub const ERROR_DS_GLOBAL_CANT_HAVE_UNIVERSAL_MEMBER: DWORD = 8517; +pub const ERROR_DS_UNIVERSAL_CANT_HAVE_LOCAL_MEMBER: DWORD = 8518; +pub const ERROR_DS_GLOBAL_CANT_HAVE_CROSSDOMAIN_MEMBER: DWORD = 8519; +pub const ERROR_DS_LOCAL_CANT_HAVE_CROSSDOMAIN_LOCAL_MEMBER: DWORD = 8520; +pub const ERROR_DS_HAVE_PRIMARY_MEMBERS: DWORD = 8521; +pub const ERROR_DS_STRING_SD_CONVERSION_FAILED: DWORD = 8522; +pub const ERROR_DS_NAMING_MASTER_GC: DWORD = 8523; +pub const ERROR_DS_DNS_LOOKUP_FAILURE: DWORD = 8524; +pub const ERROR_DS_COULDNT_UPDATE_SPNS: DWORD = 8525; +pub const ERROR_DS_CANT_RETRIEVE_SD: DWORD = 8526; +pub const ERROR_DS_KEY_NOT_UNIQUE: DWORD = 8527; +pub const ERROR_DS_WRONG_LINKED_ATT_SYNTAX: DWORD = 8528; +pub const ERROR_DS_SAM_NEED_BOOTKEY_PASSWORD: DWORD = 8529; +pub const ERROR_DS_SAM_NEED_BOOTKEY_FLOPPY: DWORD = 8530; +pub const ERROR_DS_CANT_START: DWORD = 8531; +pub const ERROR_DS_INIT_FAILURE: DWORD = 8532; +pub const ERROR_DS_NO_PKT_PRIVACY_ON_CONNECTION: DWORD = 8533; +pub const ERROR_DS_SOURCE_DOMAIN_IN_FOREST: DWORD = 8534; +pub const ERROR_DS_DESTINATION_DOMAIN_NOT_IN_FOREST: DWORD = 8535; +pub const ERROR_DS_DESTINATION_AUDITING_NOT_ENABLED: DWORD = 8536; +pub const ERROR_DS_CANT_FIND_DC_FOR_SRC_DOMAIN: DWORD = 8537; +pub const ERROR_DS_SRC_OBJ_NOT_GROUP_OR_USER: DWORD = 8538; +pub const ERROR_DS_SRC_SID_EXISTS_IN_FOREST: DWORD = 8539; +pub const ERROR_DS_SRC_AND_DST_OBJECT_CLASS_MISMATCH: DWORD = 8540; +pub const ERROR_SAM_INIT_FAILURE: DWORD = 8541; +pub const ERROR_DS_DRA_SCHEMA_INFO_SHIP: DWORD = 8542; +pub const ERROR_DS_DRA_SCHEMA_CONFLICT: DWORD = 8543; +pub const ERROR_DS_DRA_EARLIER_SCHEMA_CONFLICT: DWORD = 8544; +pub const ERROR_DS_DRA_OBJ_NC_MISMATCH: DWORD = 8545; +pub const ERROR_DS_NC_STILL_HAS_DSAS: DWORD = 8546; +pub const ERROR_DS_GC_REQUIRED: DWORD = 8547; +pub const ERROR_DS_LOCAL_MEMBER_OF_LOCAL_ONLY: DWORD = 8548; +pub const ERROR_DS_NO_FPO_IN_UNIVERSAL_GROUPS: DWORD = 8549; +pub const ERROR_DS_CANT_ADD_TO_GC: DWORD = 8550; +pub const ERROR_DS_NO_CHECKPOINT_WITH_PDC: DWORD = 8551; +pub const ERROR_DS_SOURCE_AUDITING_NOT_ENABLED: DWORD = 8552; +pub const ERROR_DS_CANT_CREATE_IN_NONDOMAIN_NC: DWORD = 8553; +pub const ERROR_DS_INVALID_NAME_FOR_SPN: DWORD = 8554; +pub const ERROR_DS_FILTER_USES_CONTRUCTED_ATTRS: DWORD = 8555; +pub const ERROR_DS_UNICODEPWD_NOT_IN_QUOTES: DWORD = 8556; +pub const ERROR_DS_MACHINE_ACCOUNT_QUOTA_EXCEEDED: DWORD = 8557; +pub const ERROR_DS_MUST_BE_RUN_ON_DST_DC: DWORD = 8558; +pub const ERROR_DS_SRC_DC_MUST_BE_SP4_OR_GREATER: DWORD = 8559; +pub const ERROR_DS_CANT_TREE_DELETE_CRITICAL_OBJ: DWORD = 8560; +pub const ERROR_DS_INIT_FAILURE_CONSOLE: DWORD = 8561; +pub const ERROR_DS_SAM_INIT_FAILURE_CONSOLE: DWORD = 8562; +pub const ERROR_DS_FOREST_VERSION_TOO_HIGH: DWORD = 8563; +pub const ERROR_DS_DOMAIN_VERSION_TOO_HIGH: DWORD = 8564; +pub const ERROR_DS_FOREST_VERSION_TOO_LOW: DWORD = 8565; +pub const ERROR_DS_DOMAIN_VERSION_TOO_LOW: DWORD = 8566; +pub const ERROR_DS_INCOMPATIBLE_VERSION: DWORD = 8567; +pub const ERROR_DS_LOW_DSA_VERSION: DWORD = 8568; +pub const ERROR_DS_NO_BEHAVIOR_VERSION_IN_MIXEDDOMAIN: DWORD = 8569; +pub const ERROR_DS_NOT_SUPPORTED_SORT_ORDER: DWORD = 8570; +pub const ERROR_DS_NAME_NOT_UNIQUE: DWORD = 8571; +pub const ERROR_DS_MACHINE_ACCOUNT_CREATED_PRENT4: DWORD = 8572; +pub const ERROR_DS_OUT_OF_VERSION_STORE: DWORD = 8573; +pub const ERROR_DS_INCOMPATIBLE_CONTROLS_USED: DWORD = 8574; +pub const ERROR_DS_NO_REF_DOMAIN: DWORD = 8575; +pub const ERROR_DS_RESERVED_LINK_ID: DWORD = 8576; +pub const ERROR_DS_LINK_ID_NOT_AVAILABLE: DWORD = 8577; +pub const ERROR_DS_AG_CANT_HAVE_UNIVERSAL_MEMBER: DWORD = 8578; +pub const ERROR_DS_MODIFYDN_DISALLOWED_BY_INSTANCE_TYPE: DWORD = 8579; +pub const ERROR_DS_NO_OBJECT_MOVE_IN_SCHEMA_NC: DWORD = 8580; +pub const ERROR_DS_MODIFYDN_DISALLOWED_BY_FLAG: DWORD = 8581; +pub const ERROR_DS_MODIFYDN_WRONG_GRANDPARENT: DWORD = 8582; +pub const ERROR_DS_NAME_ERROR_TRUST_REFERRAL: DWORD = 8583; +pub const ERROR_NOT_SUPPORTED_ON_STANDARD_SERVER: DWORD = 8584; +pub const ERROR_DS_CANT_ACCESS_REMOTE_PART_OF_AD: DWORD = 8585; +pub const ERROR_DS_CR_IMPOSSIBLE_TO_VALIDATE_V2: DWORD = 8586; +pub const ERROR_DS_THREAD_LIMIT_EXCEEDED: DWORD = 8587; +pub const ERROR_DS_NOT_CLOSEST: DWORD = 8588; +pub const ERROR_DS_CANT_DERIVE_SPN_WITHOUT_SERVER_REF: DWORD = 8589; +pub const ERROR_DS_SINGLE_USER_MODE_FAILED: DWORD = 8590; +pub const ERROR_DS_NTDSCRIPT_SYNTAX_ERROR: DWORD = 8591; +pub const ERROR_DS_NTDSCRIPT_PROCESS_ERROR: DWORD = 8592; +pub const ERROR_DS_DIFFERENT_REPL_EPOCHS: DWORD = 8593; +pub const ERROR_DS_DRS_EXTENSIONS_CHANGED: DWORD = 8594; +pub const ERROR_DS_REPLICA_SET_CHANGE_NOT_ALLOWED_ON_DISABLED_CR: DWORD = 8595; +pub const ERROR_DS_NO_MSDS_INTID: DWORD = 8596; +pub const ERROR_DS_DUP_MSDS_INTID: DWORD = 8597; +pub const ERROR_DS_EXISTS_IN_RDNATTID: DWORD = 8598; +pub const ERROR_DS_AUTHORIZATION_FAILED: DWORD = 8599; +pub const ERROR_DS_INVALID_SCRIPT: DWORD = 8600; +pub const ERROR_DS_REMOTE_CROSSREF_OP_FAILED: DWORD = 8601; +pub const ERROR_DS_CROSS_REF_BUSY: DWORD = 8602; +pub const ERROR_DS_CANT_DERIVE_SPN_FOR_DELETED_DOMAIN: DWORD = 8603; +pub const ERROR_DS_CANT_DEMOTE_WITH_WRITEABLE_NC: DWORD = 8604; +pub const ERROR_DS_DUPLICATE_ID_FOUND: DWORD = 8605; +pub const ERROR_DS_INSUFFICIENT_ATTR_TO_CREATE_OBJECT: DWORD = 8606; +pub const ERROR_DS_GROUP_CONVERSION_ERROR: DWORD = 8607; +pub const ERROR_DS_CANT_MOVE_APP_BASIC_GROUP: DWORD = 8608; +pub const ERROR_DS_CANT_MOVE_APP_QUERY_GROUP: DWORD = 8609; +pub const ERROR_DS_ROLE_NOT_VERIFIED: DWORD = 8610; +pub const ERROR_DS_WKO_CONTAINER_CANNOT_BE_SPECIAL: DWORD = 8611; +pub const ERROR_DS_DOMAIN_RENAME_IN_PROGRESS: DWORD = 8612; +pub const ERROR_DS_EXISTING_AD_CHILD_NC: DWORD = 8613; +pub const ERROR_DS_REPL_LIFETIME_EXCEEDED: DWORD = 8614; +pub const ERROR_DS_DISALLOWED_IN_SYSTEM_CONTAINER: DWORD = 8615; +pub const ERROR_DS_LDAP_SEND_QUEUE_FULL: DWORD = 8616; +pub const ERROR_DS_DRA_OUT_SCHEDULE_WINDOW: DWORD = 8617; +pub const ERROR_SXS_SECTION_NOT_FOUND: DWORD = 14000; +pub const ERROR_SXS_CANT_GEN_ACTCTX: DWORD = 14001; +pub const ERROR_SXS_INVALID_ACTCTXDATA_FORMAT: DWORD = 14002; +pub const ERROR_SXS_ASSEMBLY_NOT_FOUND: DWORD = 14003; +pub const ERROR_SXS_MANIFEST_FORMAT_ERROR: DWORD = 14004; +pub const ERROR_SXS_MANIFEST_PARSE_ERROR: DWORD = 14005; +pub const ERROR_SXS_ACTIVATION_CONTEXT_DISABLED: DWORD = 14006; +pub const ERROR_SXS_KEY_NOT_FOUND: DWORD = 14007; +pub const ERROR_SXS_VERSION_CONFLICT: DWORD = 14008; +pub const ERROR_SXS_WRONG_SECTION_TYPE: DWORD = 14009; +pub const ERROR_SXS_THREAD_QUERIES_DISABLED: DWORD = 14010; +pub const ERROR_SXS_PROCESS_DEFAULT_ALREADY_SET: DWORD = 14011; +pub const ERROR_SXS_UNKNOWN_ENCODING_GROUP: DWORD = 14012; +pub const ERROR_SXS_UNKNOWN_ENCODING: DWORD = 14013; +pub const ERROR_SXS_INVALID_XML_NAMESPACE_URI: DWORD = 14014; +pub const ERROR_SXS_ROOT_MANIFEST_DEPENDENCY_NOT_INSTALLED: DWORD = 14015; +pub const ERROR_SXS_LEAF_MANIFEST_DEPENDENCY_NOT_INSTALLED: DWORD = 14016; +pub const ERROR_SXS_INVALID_ASSEMBLY_IDENTITY_ATTRIBUTE: DWORD = 14017; +pub const ERROR_SXS_MANIFEST_MISSING_REQUIRED_DEFAULT_NAMESPACE: DWORD = 14018; +pub const ERROR_SXS_MANIFEST_INVALID_REQUIRED_DEFAULT_NAMESPACE: DWORD = 14019; +pub const ERROR_SXS_PRIVATE_MANIFEST_CROSS_PATH_WITH_REPARSE_POINT: DWORD = 14020; +pub const ERROR_SXS_DUPLICATE_DLL_NAME: DWORD = 14021; +pub const ERROR_SXS_DUPLICATE_WINDOWCLASS_NAME: DWORD = 14022; +pub const ERROR_SXS_DUPLICATE_CLSID: DWORD = 14023; +pub const ERROR_SXS_DUPLICATE_IID: DWORD = 14024; +pub const ERROR_SXS_DUPLICATE_TLBID: DWORD = 14025; +pub const ERROR_SXS_DUPLICATE_PROGID: DWORD = 14026; +pub const ERROR_SXS_DUPLICATE_ASSEMBLY_NAME: DWORD = 14027; +pub const ERROR_SXS_FILE_HASH_MISMATCH: DWORD = 14028; +pub const ERROR_SXS_POLICY_PARSE_ERROR: DWORD = 14029; +pub const ERROR_SXS_XML_E_MISSINGQUOTE: DWORD = 14030; +pub const ERROR_SXS_XML_E_COMMENTSYNTAX: DWORD = 14031; +pub const ERROR_SXS_XML_E_BADSTARTNAMECHAR: DWORD = 14032; +pub const ERROR_SXS_XML_E_BADNAMECHAR: DWORD = 14033; +pub const ERROR_SXS_XML_E_BADCHARINSTRING: DWORD = 14034; +pub const ERROR_SXS_XML_E_XMLDECLSYNTAX: DWORD = 14035; +pub const ERROR_SXS_XML_E_BADCHARDATA: DWORD = 14036; +pub const ERROR_SXS_XML_E_MISSINGWHITESPACE: DWORD = 14037; +pub const ERROR_SXS_XML_E_EXPECTINGTAGEND: DWORD = 14038; +pub const ERROR_SXS_XML_E_MISSINGSEMICOLON: DWORD = 14039; +pub const ERROR_SXS_XML_E_UNBALANCEDPAREN: DWORD = 14040; +pub const ERROR_SXS_XML_E_INTERNALERROR: DWORD = 14041; +pub const ERROR_SXS_XML_E_UNEXPECTED_WHITESPACE: DWORD = 14042; +pub const ERROR_SXS_XML_E_INCOMPLETE_ENCODING: DWORD = 14043; +pub const ERROR_SXS_XML_E_MISSING_PAREN: DWORD = 14044; +pub const ERROR_SXS_XML_E_EXPECTINGCLOSEQUOTE: DWORD = 14045; +pub const ERROR_SXS_XML_E_MULTIPLE_COLONS: DWORD = 14046; +pub const ERROR_SXS_XML_E_INVALID_DECIMAL: DWORD = 14047; +pub const ERROR_SXS_XML_E_INVALID_HEXIDECIMAL: DWORD = 14048; +pub const ERROR_SXS_XML_E_INVALID_UNICODE: DWORD = 14049; +pub const ERROR_SXS_XML_E_WHITESPACEORQUESTIONMARK: DWORD = 14050; +pub const ERROR_SXS_XML_E_UNEXPECTEDENDTAG: DWORD = 14051; +pub const ERROR_SXS_XML_E_UNCLOSEDTAG: DWORD = 14052; +pub const ERROR_SXS_XML_E_DUPLICATEATTRIBUTE: DWORD = 14053; +pub const ERROR_SXS_XML_E_MULTIPLEROOTS: DWORD = 14054; +pub const ERROR_SXS_XML_E_INVALIDATROOTLEVEL: DWORD = 14055; +pub const ERROR_SXS_XML_E_BADXMLDECL: DWORD = 14056; +pub const ERROR_SXS_XML_E_MISSINGROOT: DWORD = 14057; +pub const ERROR_SXS_XML_E_UNEXPECTEDEOF: DWORD = 14058; +pub const ERROR_SXS_XML_E_BADPEREFINSUBSET: DWORD = 14059; +pub const ERROR_SXS_XML_E_UNCLOSEDSTARTTAG: DWORD = 14060; +pub const ERROR_SXS_XML_E_UNCLOSEDENDTAG: DWORD = 14061; +pub const ERROR_SXS_XML_E_UNCLOSEDSTRING: DWORD = 14062; +pub const ERROR_SXS_XML_E_UNCLOSEDCOMMENT: DWORD = 14063; +pub const ERROR_SXS_XML_E_UNCLOSEDDECL: DWORD = 14064; +pub const ERROR_SXS_XML_E_UNCLOSEDCDATA: DWORD = 14065; +pub const ERROR_SXS_XML_E_RESERVEDNAMESPACE: DWORD = 14066; +pub const ERROR_SXS_XML_E_INVALIDENCODING: DWORD = 14067; +pub const ERROR_SXS_XML_E_INVALIDSWITCH: DWORD = 14068; +pub const ERROR_SXS_XML_E_BADXMLCASE: DWORD = 14069; +pub const ERROR_SXS_XML_E_INVALID_STANDALONE: DWORD = 14070; +pub const ERROR_SXS_XML_E_UNEXPECTED_STANDALONE: DWORD = 14071; +pub const ERROR_SXS_XML_E_INVALID_VERSION: DWORD = 14072; +pub const ERROR_SXS_XML_E_MISSINGEQUALS: DWORD = 14073; +pub const ERROR_SXS_PROTECTION_RECOVERY_FAILED: DWORD = 14074; +pub const ERROR_SXS_PROTECTION_PUBLIC_KEY_TOO_SHORT: DWORD = 14075; +pub const ERROR_SXS_PROTECTION_CATALOG_NOT_VALID: DWORD = 14076; +pub const ERROR_SXS_UNTRANSLATABLE_HRESULT: DWORD = 14077; +pub const ERROR_SXS_PROTECTION_CATALOG_FILE_MISSING: DWORD = 14078; +pub const ERROR_SXS_MISSING_ASSEMBLY_IDENTITY_ATTRIBUTE: DWORD = 14079; +pub const ERROR_SXS_INVALID_ASSEMBLY_IDENTITY_ATTRIBUTE_NAME: DWORD = 14080; +pub const ERROR_SXS_ASSEMBLY_MISSING: DWORD = 14081; +pub const ERROR_SXS_CORRUPT_ACTIVATION_STACK: DWORD = 14082; +pub const ERROR_SXS_CORRUPTION: DWORD = 14083; +pub const ERROR_SXS_EARLY_DEACTIVATION: DWORD = 14084; +pub const ERROR_SXS_INVALID_DEACTIVATION: DWORD = 14085; +pub const ERROR_SXS_MULTIPLE_DEACTIVATION: DWORD = 14086; +pub const ERROR_SXS_PROCESS_TERMINATION_REQUESTED: DWORD = 14087; +pub const ERROR_SXS_RELEASE_ACTIVATION_CONTEXT: DWORD = 14088; +pub const ERROR_SXS_SYSTEM_DEFAULT_ACTIVATION_CONTEXT_EMPTY: DWORD = 14089; +pub const ERROR_SXS_INVALID_IDENTITY_ATTRIBUTE_VALUE: DWORD = 14090; +pub const ERROR_SXS_INVALID_IDENTITY_ATTRIBUTE_NAME: DWORD = 14091; +pub const ERROR_SXS_IDENTITY_DUPLICATE_ATTRIBUTE: DWORD = 14092; +pub const ERROR_SXS_IDENTITY_PARSE_ERROR: DWORD = 14093; +pub const ERROR_MALFORMED_SUBSTITUTION_STRING: DWORD = 14094; +pub const ERROR_SXS_INCORRECT_PUBLIC_KEY_TOKEN: DWORD = 14095; +pub const ERROR_UNMAPPED_SUBSTITUTION_STRING: DWORD = 14096; +pub const ERROR_SXS_ASSEMBLY_NOT_LOCKED: DWORD = 14097; +pub const ERROR_SXS_COMPONENT_STORE_CORRUPT: DWORD = 14098; +pub const ERROR_ADVANCED_INSTALLER_FAILED: DWORD = 14099; +pub const ERROR_XML_ENCODING_MISMATCH: DWORD = 14100; +pub const ERROR_SXS_MANIFEST_IDENTITY_SAME_BUT_CONTENTS_DIFFERENT: DWORD = 14101; +pub const ERROR_SXS_IDENTITIES_DIFFERENT: DWORD = 14102; +pub const ERROR_SXS_ASSEMBLY_IS_NOT_A_DEPLOYMENT: DWORD = 14103; +pub const ERROR_SXS_FILE_NOT_PART_OF_ASSEMBLY: DWORD = 14104; +pub const ERROR_SXS_MANIFEST_TOO_BIG: DWORD = 14105; +pub const ERROR_SXS_SETTING_NOT_REGISTERED: DWORD = 14106; +pub const ERROR_SXS_TRANSACTION_CLOSURE_INCOMPLETE: DWORD = 14107; +pub const ERROR_SMI_PRIMITIVE_INSTALLER_FAILED: DWORD = 14108; +pub const ERROR_GENERIC_COMMAND_FAILED: DWORD = 14109; +pub const ERROR_SXS_FILE_HASH_MISSING: DWORD = 14110; +pub const ERROR_IPSEC_QM_POLICY_EXISTS: DWORD = 13000; +pub const ERROR_IPSEC_QM_POLICY_NOT_FOUND: DWORD = 13001; +pub const ERROR_IPSEC_QM_POLICY_IN_USE: DWORD = 13002; +pub const ERROR_IPSEC_MM_POLICY_EXISTS: DWORD = 13003; +pub const ERROR_IPSEC_MM_POLICY_NOT_FOUND: DWORD = 13004; +pub const ERROR_IPSEC_MM_POLICY_IN_USE: DWORD = 13005; +pub const ERROR_IPSEC_MM_FILTER_EXISTS: DWORD = 13006; +pub const ERROR_IPSEC_MM_FILTER_NOT_FOUND: DWORD = 13007; +pub const ERROR_IPSEC_TRANSPORT_FILTER_EXISTS: DWORD = 13008; +pub const ERROR_IPSEC_TRANSPORT_FILTER_NOT_FOUND: DWORD = 13009; +pub const ERROR_IPSEC_MM_AUTH_EXISTS: DWORD = 13010; +pub const ERROR_IPSEC_MM_AUTH_NOT_FOUND: DWORD = 13011; +pub const ERROR_IPSEC_MM_AUTH_IN_USE: DWORD = 13012; +pub const ERROR_IPSEC_DEFAULT_MM_POLICY_NOT_FOUND: DWORD = 13013; +pub const ERROR_IPSEC_DEFAULT_MM_AUTH_NOT_FOUND: DWORD = 13014; +pub const ERROR_IPSEC_DEFAULT_QM_POLICY_NOT_FOUND: DWORD = 13015; +pub const ERROR_IPSEC_TUNNEL_FILTER_EXISTS: DWORD = 13016; +pub const ERROR_IPSEC_TUNNEL_FILTER_NOT_FOUND: DWORD = 13017; +pub const ERROR_IPSEC_MM_FILTER_PENDING_DELETION: DWORD = 13018; +pub const ERROR_IPSEC_TRANSPORT_FILTER_PENDING_DELETION: DWORD = 13019; +pub const ERROR_IPSEC_TUNNEL_FILTER_PENDING_DELETION: DWORD = 13020; +pub const ERROR_IPSEC_MM_POLICY_PENDING_DELETION: DWORD = 13021; +pub const ERROR_IPSEC_MM_AUTH_PENDING_DELETION: DWORD = 13022; +pub const ERROR_IPSEC_QM_POLICY_PENDING_DELETION: DWORD = 13023; +pub const ERROR_IPSEC_IKE_NEG_STATUS_BEGIN: DWORD = 13800; +pub const ERROR_IPSEC_IKE_AUTH_FAIL: DWORD = 13801; +pub const ERROR_IPSEC_IKE_ATTRIB_FAIL: DWORD = 13802; +pub const ERROR_IPSEC_IKE_NEGOTIATION_PENDING: DWORD = 13803; +pub const ERROR_IPSEC_IKE_GENERAL_PROCESSING_ERROR: DWORD = 13804; +pub const ERROR_IPSEC_IKE_TIMED_OUT: DWORD = 13805; +pub const ERROR_IPSEC_IKE_NO_CERT: DWORD = 13806; +pub const ERROR_IPSEC_IKE_SA_DELETED: DWORD = 13807; +pub const ERROR_IPSEC_IKE_SA_REAPED: DWORD = 13808; +pub const ERROR_IPSEC_IKE_MM_ACQUIRE_DROP: DWORD = 13809; +pub const ERROR_IPSEC_IKE_QM_ACQUIRE_DROP: DWORD = 13810; +pub const ERROR_IPSEC_IKE_QUEUE_DROP_MM: DWORD = 13811; +pub const ERROR_IPSEC_IKE_QUEUE_DROP_NO_MM: DWORD = 13812; +pub const ERROR_IPSEC_IKE_DROP_NO_RESPONSE: DWORD = 13813; +pub const ERROR_IPSEC_IKE_MM_DELAY_DROP: DWORD = 13814; +pub const ERROR_IPSEC_IKE_QM_DELAY_DROP: DWORD = 13815; +pub const ERROR_IPSEC_IKE_ERROR: DWORD = 13816; +pub const ERROR_IPSEC_IKE_CRL_FAILED: DWORD = 13817; +pub const ERROR_IPSEC_IKE_INVALID_KEY_USAGE: DWORD = 13818; +pub const ERROR_IPSEC_IKE_INVALID_CERT_TYPE: DWORD = 13819; +pub const ERROR_IPSEC_IKE_NO_PRIVATE_KEY: DWORD = 13820; +pub const ERROR_IPSEC_IKE_DH_FAIL: DWORD = 13822; +pub const ERROR_IPSEC_IKE_INVALID_HEADER: DWORD = 13824; +pub const ERROR_IPSEC_IKE_NO_POLICY: DWORD = 13825; +pub const ERROR_IPSEC_IKE_INVALID_SIGNATURE: DWORD = 13826; +pub const ERROR_IPSEC_IKE_KERBEROS_ERROR: DWORD = 13827; +pub const ERROR_IPSEC_IKE_NO_PUBLIC_KEY: DWORD = 13828; +pub const ERROR_IPSEC_IKE_PROCESS_ERR: DWORD = 13829; +pub const ERROR_IPSEC_IKE_PROCESS_ERR_SA: DWORD = 13830; +pub const ERROR_IPSEC_IKE_PROCESS_ERR_PROP: DWORD = 13831; +pub const ERROR_IPSEC_IKE_PROCESS_ERR_TRANS: DWORD = 13832; +pub const ERROR_IPSEC_IKE_PROCESS_ERR_KE: DWORD = 13833; +pub const ERROR_IPSEC_IKE_PROCESS_ERR_ID: DWORD = 13834; +pub const ERROR_IPSEC_IKE_PROCESS_ERR_CERT: DWORD = 13835; +pub const ERROR_IPSEC_IKE_PROCESS_ERR_CERT_REQ: DWORD = 13836; +pub const ERROR_IPSEC_IKE_PROCESS_ERR_HASH: DWORD = 13837; +pub const ERROR_IPSEC_IKE_PROCESS_ERR_SIG: DWORD = 13838; +pub const ERROR_IPSEC_IKE_PROCESS_ERR_NONCE: DWORD = 13839; +pub const ERROR_IPSEC_IKE_PROCESS_ERR_NOTIFY: DWORD = 13840; +pub const ERROR_IPSEC_IKE_PROCESS_ERR_DELETE: DWORD = 13841; +pub const ERROR_IPSEC_IKE_PROCESS_ERR_VENDOR: DWORD = 13842; +pub const ERROR_IPSEC_IKE_INVALID_PAYLOAD: DWORD = 13843; +pub const ERROR_IPSEC_IKE_LOAD_SOFT_SA: DWORD = 13844; +pub const ERROR_IPSEC_IKE_SOFT_SA_TORN_DOWN: DWORD = 13845; +pub const ERROR_IPSEC_IKE_INVALID_COOKIE: DWORD = 13846; +pub const ERROR_IPSEC_IKE_NO_PEER_CERT: DWORD = 13847; +pub const ERROR_IPSEC_IKE_PEER_CRL_FAILED: DWORD = 13848; +pub const ERROR_IPSEC_IKE_POLICY_CHANGE: DWORD = 13849; +pub const ERROR_IPSEC_IKE_NO_MM_POLICY: DWORD = 13850; +pub const ERROR_IPSEC_IKE_NOTCBPRIV: DWORD = 13851; +pub const ERROR_IPSEC_IKE_SECLOADFAIL: DWORD = 13852; +pub const ERROR_IPSEC_IKE_FAILSSPINIT: DWORD = 13853; +pub const ERROR_IPSEC_IKE_FAILQUERYSSP: DWORD = 13854; +pub const ERROR_IPSEC_IKE_SRVACQFAIL: DWORD = 13855; +pub const ERROR_IPSEC_IKE_SRVQUERYCRED: DWORD = 13856; +pub const ERROR_IPSEC_IKE_GETSPIFAIL: DWORD = 13857; +pub const ERROR_IPSEC_IKE_INVALID_FILTER: DWORD = 13858; +pub const ERROR_IPSEC_IKE_OUT_OF_MEMORY: DWORD = 13859; +pub const ERROR_IPSEC_IKE_ADD_UPDATE_KEY_FAILED: DWORD = 13860; +pub const ERROR_IPSEC_IKE_INVALID_POLICY: DWORD = 13861; +pub const ERROR_IPSEC_IKE_UNKNOWN_DOI: DWORD = 13862; +pub const ERROR_IPSEC_IKE_INVALID_SITUATION: DWORD = 13863; +pub const ERROR_IPSEC_IKE_DH_FAILURE: DWORD = 13864; +pub const ERROR_IPSEC_IKE_INVALID_GROUP: DWORD = 13865; +pub const ERROR_IPSEC_IKE_ENCRYPT: DWORD = 13866; +pub const ERROR_IPSEC_IKE_DECRYPT: DWORD = 13867; +pub const ERROR_IPSEC_IKE_POLICY_MATCH: DWORD = 13868; +pub const ERROR_IPSEC_IKE_UNSUPPORTED_ID: DWORD = 13869; +pub const ERROR_IPSEC_IKE_INVALID_HASH: DWORD = 13870; +pub const ERROR_IPSEC_IKE_INVALID_HASH_ALG: DWORD = 13871; +pub const ERROR_IPSEC_IKE_INVALID_HASH_SIZE: DWORD = 13872; +pub const ERROR_IPSEC_IKE_INVALID_ENCRYPT_ALG: DWORD = 13873; +pub const ERROR_IPSEC_IKE_INVALID_AUTH_ALG: DWORD = 13874; +pub const ERROR_IPSEC_IKE_INVALID_SIG: DWORD = 13875; +pub const ERROR_IPSEC_IKE_LOAD_FAILED: DWORD = 13876; +pub const ERROR_IPSEC_IKE_RPC_DELETE: DWORD = 13877; +pub const ERROR_IPSEC_IKE_BENIGN_REINIT: DWORD = 13878; +pub const ERROR_IPSEC_IKE_INVALID_RESPONDER_LIFETIME_NOTIFY: DWORD = 13879; +pub const ERROR_IPSEC_IKE_INVALID_CERT_KEYLEN: DWORD = 13881; +pub const ERROR_IPSEC_IKE_MM_LIMIT: DWORD = 13882; +pub const ERROR_IPSEC_IKE_NEGOTIATION_DISABLED: DWORD = 13883; +/*pub const ERROR_IPSEC_IKE_NEG_STATUS_END: DWORD = 13884)*/ +pub const ERROR_IPSEC_IKE_QM_LIMIT: DWORD = 13884; +pub const ERROR_IPSEC_IKE_MM_EXPIRED: DWORD = 13885; +pub const ERROR_IPSEC_IKE_PEER_MM_ASSUMED_INVALID: DWORD = 13886; +pub const ERROR_IPSEC_IKE_CERT_CHAIN_POLICY_MISMATCH: DWORD = 13887; +pub const ERROR_IPSEC_IKE_UNEXPECTED_MESSAGE_ID: DWORD = 13888; +pub const ERROR_IPSEC_IKE_INVALID_AUTH_PAYLOAD: DWORD = 13889; +pub const ERROR_IPSEC_IKE_DOS_COOKIE_SENT: DWORD = 13890; +pub const ERROR_IPSEC_IKE_SHUTTING_DOWN: DWORD = 13891; +pub const ERROR_IPSEC_IKE_CGA_AUTH_FAILED: DWORD = 13892; +pub const ERROR_IPSEC_IKE_PROCESS_ERR_NATOA: DWORD = 13893; +pub const ERROR_IPSEC_IKE_INVALID_MM_FOR_QM: DWORD = 13894; +pub const ERROR_IPSEC_IKE_QM_EXPIRED: DWORD = 13895; +pub const ERROR_IPSEC_IKE_TOO_MANY_FILTERS: DWORD = 13896; +pub const ERROR_IPSEC_IKE_NEG_STATUS_END: DWORD = 13897; +pub const ERROR_IPSEC_IKE_KILL_DUMMY_NAP_TUNNEL: DWORD = 13898; +pub const ERROR_IPSEC_IKE_INNER_IP_ASSIGNMENT_FAILURE: DWORD = 13899; +pub const ERROR_IPSEC_IKE_REQUIRE_CP_PAYLOAD_MISSING: DWORD = 13900; +pub const ERROR_IPSEC_KEY_MODULE_IMPERSONATION_NEGOTIATION_PENDING: DWORD = 13901; +pub const ERROR_IPSEC_IKE_COEXISTENCE_SUPPRESS: DWORD = 13902; +pub const ERROR_IPSEC_IKE_RATELIMIT_DROP: DWORD = 13903; +pub const ERROR_IPSEC_IKE_PEER_DOESNT_SUPPORT_MOBIKE: DWORD = 13904; +pub const ERROR_IPSEC_IKE_AUTHORIZATION_FAILURE: DWORD = 13905; +pub const ERROR_IPSEC_IKE_STRONG_CRED_AUTHORIZATION_FAILURE: DWORD = 13906; +pub const ERROR_IPSEC_IKE_AUTHORIZATION_FAILURE_WITH_OPTIONAL_RETRY: DWORD = 13907; +pub const ERROR_IPSEC_IKE_STRONG_CRED_AUTHORIZATION_AND_CERTMAP_FAILURE: DWORD = 13908; +pub const ERROR_IPSEC_IKE_NEG_STATUS_EXTENDED_END: DWORD = 13909; +pub const ERROR_IPSEC_BAD_SPI: DWORD = 13910; +pub const ERROR_IPSEC_SA_LIFETIME_EXPIRED: DWORD = 13911; +pub const ERROR_IPSEC_WRONG_SA: DWORD = 13912; +pub const ERROR_IPSEC_REPLAY_CHECK_FAILED: DWORD = 13913; +pub const ERROR_IPSEC_INVALID_PACKET: DWORD = 13914; +pub const ERROR_IPSEC_INTEGRITY_CHECK_FAILED: DWORD = 13915; +pub const ERROR_IPSEC_CLEAR_TEXT_DROP: DWORD = 13916; +pub const ERROR_IPSEC_AUTH_FIREWALL_DROP: DWORD = 13917; +pub const ERROR_IPSEC_THROTTLE_DROP: DWORD = 13918; +pub const ERROR_IPSEC_DOSP_BLOCK: DWORD = 13925; +pub const ERROR_IPSEC_DOSP_RECEIVED_MULTICAST: DWORD = 13926; +pub const ERROR_IPSEC_DOSP_INVALID_PACKET: DWORD = 13927; +pub const ERROR_IPSEC_DOSP_STATE_LOOKUP_FAILED: DWORD = 13928; +pub const ERROR_IPSEC_DOSP_MAX_ENTRIES: DWORD = 13929; +pub const ERROR_IPSEC_DOSP_KEYMOD_NOT_ALLOWED: DWORD = 13930; +pub const ERROR_IPSEC_DOSP_NOT_INSTALLED: DWORD = 13931; +pub const ERROR_IPSEC_DOSP_MAX_PER_IP_RATELIMIT_QUEUES: DWORD = 13932; +pub const ERROR_EVT_INVALID_CHANNEL_PATH: DWORD = 15000; +pub const ERROR_EVT_INVALID_QUERY: DWORD = 15001; +pub const ERROR_EVT_PUBLISHER_METADATA_NOT_FOUND: DWORD = 15002; +pub const ERROR_EVT_EVENT_TEMPLATE_NOT_FOUND: DWORD = 15003; +pub const ERROR_EVT_INVALID_PUBLISHER_NAME: DWORD = 15004; +pub const ERROR_EVT_INVALID_EVENT_DATA: DWORD = 15005; +pub const ERROR_EVT_CHANNEL_NOT_FOUND: DWORD = 15007; +pub const ERROR_EVT_MALFORMED_XML_TEXT: DWORD = 15008; +pub const ERROR_EVT_SUBSCRIPTION_TO_DIRECT_CHANNEL: DWORD = 15009; +pub const ERROR_EVT_CONFIGURATION_ERROR: DWORD = 15010; +pub const ERROR_EVT_QUERY_RESULT_STALE: DWORD = 15011; +pub const ERROR_EVT_QUERY_RESULT_INVALID_POSITION: DWORD = 15012; +pub const ERROR_EVT_NON_VALIDATING_MSXML: DWORD = 15013; +pub const ERROR_EVT_FILTER_ALREADYSCOPED: DWORD = 15014; +pub const ERROR_EVT_FILTER_NOTELTSET: DWORD = 15015; +pub const ERROR_EVT_FILTER_INVARG: DWORD = 15016; +pub const ERROR_EVT_FILTER_INVTEST: DWORD = 15017; +pub const ERROR_EVT_FILTER_INVTYPE: DWORD = 15018; +pub const ERROR_EVT_FILTER_PARSEERR: DWORD = 15019; +pub const ERROR_EVT_FILTER_UNSUPPORTEDOP: DWORD = 15020; +pub const ERROR_EVT_FILTER_UNEXPECTEDTOKEN: DWORD = 15021; +pub const ERROR_EVT_INVALID_OPERATION_OVER_ENABLED_DIRECT_CHANNEL: DWORD = 15022; +pub const ERROR_EVT_INVALID_CHANNEL_PROPERTY_VALUE: DWORD = 15023; +pub const ERROR_EVT_INVALID_PUBLISHER_PROPERTY_VALUE: DWORD = 15024; +pub const ERROR_EVT_CHANNEL_CANNOT_ACTIVATE: DWORD = 15025; +pub const ERROR_EVT_FILTER_TOO_COMPLEX: DWORD = 15026; +pub const ERROR_EVT_MESSAGE_NOT_FOUND: DWORD = 15027; +pub const ERROR_EVT_MESSAGE_ID_NOT_FOUND: DWORD = 15028; +pub const ERROR_EVT_UNRESOLVED_VALUE_INSERT: DWORD = 15029; +pub const ERROR_EVT_UNRESOLVED_PARAMETER_INSERT: DWORD = 15030; +pub const ERROR_EVT_MAX_INSERTS_REACHED: DWORD = 15031; +pub const ERROR_EVT_EVENT_DEFINITION_NOT_FOUND: DWORD = 15032; +pub const ERROR_EVT_MESSAGE_LOCALE_NOT_FOUND: DWORD = 15033; +pub const ERROR_EVT_VERSION_TOO_OLD: DWORD = 15034; +pub const ERROR_EVT_VERSION_TOO_NEW: DWORD = 15035; +pub const ERROR_EVT_CANNOT_OPEN_CHANNEL_OF_QUERY: DWORD = 15036; +pub const ERROR_EVT_PUBLISHER_DISABLED: DWORD = 15037; +pub const ERROR_EVT_FILTER_OUT_OF_RANGE: DWORD = 15038; +pub const ERROR_EC_SUBSCRIPTION_CANNOT_ACTIVATE: DWORD = 15080; +pub const ERROR_EC_LOG_DISABLED: DWORD = 15081; +pub const ERROR_EC_CIRCULAR_FORWARDING: DWORD = 15082; +pub const ERROR_EC_CREDSTORE_FULL: DWORD = 15083; +pub const ERROR_EC_CRED_NOT_FOUND: DWORD = 15084; +pub const ERROR_EC_NO_ACTIVE_CHANNEL: DWORD = 15085; +pub const ERROR_MUI_FILE_NOT_FOUND: DWORD = 15100; +pub const ERROR_MUI_INVALID_FILE: DWORD = 15101; +pub const ERROR_MUI_INVALID_RC_CONFIG: DWORD = 15102; +pub const ERROR_MUI_INVALID_LOCALE_NAME: DWORD = 15103; +pub const ERROR_MUI_INVALID_ULTIMATEFALLBACK_NAME: DWORD = 15104; +pub const ERROR_MUI_FILE_NOT_LOADED: DWORD = 15105; +pub const ERROR_RESOURCE_ENUM_USER_STOP: DWORD = 15106; +pub const ERROR_MUI_INTLSETTINGS_UILANG_NOT_INSTALLED: DWORD = 15107; +pub const ERROR_MUI_INTLSETTINGS_INVALID_LOCALE_NAME: DWORD = 15108; +pub const ERROR_MRM_RUNTIME_NO_DEFAULT_OR_NEUTRAL_RESOURCE: DWORD = 15110; +pub const ERROR_MRM_INVALID_PRICONFIG: DWORD = 15111; +pub const ERROR_MRM_INVALID_FILE_TYPE: DWORD = 15112; +pub const ERROR_MRM_UNKNOWN_QUALIFIER: DWORD = 15113; +pub const ERROR_MRM_INVALID_QUALIFIER_VALUE: DWORD = 15114; +pub const ERROR_MRM_NO_CANDIDATE: DWORD = 15115; +pub const ERROR_MRM_NO_MATCH_OR_DEFAULT_CANDIDATE: DWORD = 15116; +pub const ERROR_MRM_RESOURCE_TYPE_MISMATCH: DWORD = 15117; +pub const ERROR_MRM_DUPLICATE_MAP_NAME: DWORD = 15118; +pub const ERROR_MRM_DUPLICATE_ENTRY: DWORD = 15119; +pub const ERROR_MRM_INVALID_RESOURCE_IDENTIFIER: DWORD = 15120; +pub const ERROR_MRM_FILEPATH_TOO_LONG: DWORD = 15121; +pub const ERROR_MRM_UNSUPPORTED_DIRECTORY_TYPE: DWORD = 15122; +pub const ERROR_MRM_INVALID_PRI_FILE: DWORD = 15126; +pub const ERROR_MRM_NAMED_RESOURCE_NOT_FOUND: DWORD = 15127; +pub const ERROR_MRM_MAP_NOT_FOUND: DWORD = 15135; +pub const ERROR_MRM_UNSUPPORTED_PROFILE_TYPE: DWORD = 15136; +pub const ERROR_MRM_INVALID_QUALIFIER_OPERATOR: DWORD = 15137; +pub const ERROR_MRM_INDETERMINATE_QUALIFIER_VALUE: DWORD = 15138; +pub const ERROR_MRM_AUTOMERGE_ENABLED: DWORD = 15139; +pub const ERROR_MRM_TOO_MANY_RESOURCES: DWORD = 15140; +pub const ERROR_MCA_INVALID_CAPABILITIES_STRING: DWORD = 15200; +pub const ERROR_MCA_INVALID_VCP_VERSION: DWORD = 15201; +pub const ERROR_MCA_MONITOR_VIOLATES_MCCS_SPECIFICATION: DWORD = 15202; +pub const ERROR_MCA_MCCS_VERSION_MISMATCH: DWORD = 15203; +pub const ERROR_MCA_UNSUPPORTED_MCCS_VERSION: DWORD = 15204; +pub const ERROR_MCA_INTERNAL_ERROR: DWORD = 15205; +pub const ERROR_MCA_INVALID_TECHNOLOGY_TYPE_RETURNED: DWORD = 15206; +pub const ERROR_MCA_UNSUPPORTED_COLOR_TEMPERATURE: DWORD = 15207; +pub const ERROR_AMBIGUOUS_SYSTEM_DEVICE: DWORD = 15250; +pub const ERROR_SYSTEM_DEVICE_NOT_FOUND: DWORD = 15299; +pub const ERROR_HASH_NOT_SUPPORTED: DWORD = 15300; +pub const ERROR_HASH_NOT_PRESENT: DWORD = 15301; +pub const ERROR_SECONDARY_IC_PROVIDER_NOT_REGISTERED: DWORD = 15321; +pub const ERROR_GPIO_CLIENT_INFORMATION_INVALID: DWORD = 15322; +pub const ERROR_GPIO_VERSION_NOT_SUPPORTED: DWORD = 15323; +pub const ERROR_GPIO_INVALID_REGISTRATION_PACKET: DWORD = 15324; +pub const ERROR_GPIO_OPERATION_DENIED: DWORD = 15325; +pub const ERROR_GPIO_INCOMPATIBLE_CONNECT_MODE: DWORD = 15326; +pub const ERROR_GPIO_INTERRUPT_ALREADY_UNMASKED: DWORD = 15327; +pub const ERROR_CANNOT_SWITCH_RUNLEVEL: DWORD = 15400; +pub const ERROR_INVALID_RUNLEVEL_SETTING: DWORD = 15401; +pub const ERROR_RUNLEVEL_SWITCH_TIMEOUT: DWORD = 15402; +pub const ERROR_RUNLEVEL_SWITCH_AGENT_TIMEOUT: DWORD = 15403; +pub const ERROR_RUNLEVEL_SWITCH_IN_PROGRESS: DWORD = 15404; +pub const ERROR_SERVICES_FAILED_AUTOSTART: DWORD = 15405; +pub const ERROR_COM_TASK_STOP_PENDING: DWORD = 15501; +pub const ERROR_INSTALL_OPEN_PACKAGE_FAILED: DWORD = 15600; +pub const ERROR_INSTALL_PACKAGE_NOT_FOUND: DWORD = 15601; +pub const ERROR_INSTALL_INVALID_PACKAGE: DWORD = 15602; +pub const ERROR_INSTALL_RESOLVE_DEPENDENCY_FAILED: DWORD = 15603; +pub const ERROR_INSTALL_OUT_OF_DISK_SPACE: DWORD = 15604; +pub const ERROR_INSTALL_NETWORK_FAILURE: DWORD = 15605; +pub const ERROR_INSTALL_REGISTRATION_FAILURE: DWORD = 15606; +pub const ERROR_INSTALL_DEREGISTRATION_FAILURE: DWORD = 15607; +pub const ERROR_INSTALL_CANCEL: DWORD = 15608; +pub const ERROR_INSTALL_FAILED: DWORD = 15609; +pub const ERROR_REMOVE_FAILED: DWORD = 15610; +pub const ERROR_PACKAGE_ALREADY_EXISTS: DWORD = 15611; +pub const ERROR_NEEDS_REMEDIATION: DWORD = 15612; +pub const ERROR_INSTALL_PREREQUISITE_FAILED: DWORD = 15613; +pub const ERROR_PACKAGE_REPOSITORY_CORRUPTED: DWORD = 15614; +pub const ERROR_INSTALL_POLICY_FAILURE: DWORD = 15615; +pub const ERROR_PACKAGE_UPDATING: DWORD = 15616; +pub const ERROR_DEPLOYMENT_BLOCKED_BY_POLICY: DWORD = 15617; +pub const ERROR_PACKAGES_IN_USE: DWORD = 15618; +pub const ERROR_RECOVERY_FILE_CORRUPT: DWORD = 15619; +pub const ERROR_INVALID_STAGED_SIGNATURE: DWORD = 15620; +pub const ERROR_DELETING_EXISTING_APPLICATIONDATA_STORE_FAILED: DWORD = 15621; +pub const ERROR_INSTALL_PACKAGE_DOWNGRADE: DWORD = 15622; +pub const ERROR_SYSTEM_NEEDS_REMEDIATION: DWORD = 15623; +pub const ERROR_APPX_INTEGRITY_FAILURE_CLR_NGEN: DWORD = 15624; +pub const ERROR_RESILIENCY_FILE_CORRUPT: DWORD = 15625; +pub const ERROR_INSTALL_FIREWALL_SERVICE_NOT_RUNNING: DWORD = 15626; +pub const ERROR_STATE_LOAD_STORE_FAILED: DWORD = 15800; +pub const ERROR_STATE_GET_VERSION_FAILED: DWORD = 15801; +pub const ERROR_STATE_SET_VERSION_FAILED: DWORD = 15802; +pub const ERROR_STATE_STRUCTURED_RESET_FAILED: DWORD = 15803; +pub const ERROR_STATE_OPEN_CONTAINER_FAILED: DWORD = 15804; +pub const ERROR_STATE_CREATE_CONTAINER_FAILED: DWORD = 15805; +pub const ERROR_STATE_DELETE_CONTAINER_FAILED: DWORD = 15806; +pub const ERROR_STATE_READ_SETTING_FAILED: DWORD = 15807; +pub const ERROR_STATE_WRITE_SETTING_FAILED: DWORD = 15808; +pub const ERROR_STATE_DELETE_SETTING_FAILED: DWORD = 15809; +pub const ERROR_STATE_QUERY_SETTING_FAILED: DWORD = 15810; +pub const ERROR_STATE_READ_COMPOSITE_SETTING_FAILED: DWORD = 15811; +pub const ERROR_STATE_WRITE_COMPOSITE_SETTING_FAILED: DWORD = 15812; +pub const ERROR_STATE_ENUMERATE_CONTAINER_FAILED: DWORD = 15813; +pub const ERROR_STATE_ENUMERATE_SETTINGS_FAILED: DWORD = 15814; +pub const ERROR_STATE_COMPOSITE_SETTING_VALUE_SIZE_LIMIT_EXCEEDED: DWORD = 15815; +pub const ERROR_STATE_SETTING_VALUE_SIZE_LIMIT_EXCEEDED: DWORD = 15816; +pub const ERROR_STATE_SETTING_NAME_SIZE_LIMIT_EXCEEDED: DWORD = 15817; +pub const ERROR_STATE_CONTAINER_NAME_SIZE_LIMIT_EXCEEDED: DWORD = 15818; +pub const ERROR_API_UNAVAILABLE: DWORD = 15841; +pub const ERROR_AUDITING_DISABLED: DWORD = 0xC0090001; +pub const ERROR_ALL_SIDS_FILTERED: DWORD = 0xC0090002; + +pub const WSABASEERR: c_int = 10000; +pub const WSAEINTR: c_int = WSABASEERR + 4; +pub const WSAEBADF: c_int = WSABASEERR + 9; +pub const WSAEACCES: c_int = WSABASEERR + 13; +pub const WSAEFAULT: c_int = WSABASEERR + 14; +pub const WSAEINVAL: c_int = WSABASEERR + 22; +pub const WSAEMFILE: c_int = WSABASEERR + 24; +pub const WSAEWOULDBLOCK: c_int = WSABASEERR + 35; +pub const WSAEINPROGRESS: c_int = WSABASEERR + 36; +pub const WSAEALREADY: c_int = WSABASEERR + 37; +pub const WSAENOTSOCK: c_int = WSABASEERR + 38; +pub const WSAEDESTADDRREQ: c_int = WSABASEERR + 39; +pub const WSAEMSGSIZE: c_int = WSABASEERR + 40; +pub const WSAEPROTOTYPE: c_int = WSABASEERR + 41; +pub const WSAENOPROTOOPT: c_int = WSABASEERR + 42; +pub const WSAEPROTONOSUPPORT: c_int = WSABASEERR + 43; +pub const WSAESOCKTNOSUPPORT: c_int = WSABASEERR + 44; +pub const WSAEOPNOTSUPP: c_int = WSABASEERR + 45; +pub const WSAEPFNOSUPPORT: c_int = WSABASEERR + 46; +pub const WSAEAFNOSUPPORT: c_int = WSABASEERR + 47; +pub const WSAEADDRINUSE: c_int = WSABASEERR + 48; +pub const WSAEADDRNOTAVAIL: c_int = WSABASEERR + 49; +pub const WSAENETDOWN: c_int = WSABASEERR + 50; +pub const WSAENETUNREACH: c_int = WSABASEERR + 51; +pub const WSAENETRESET: c_int = WSABASEERR + 52; +pub const WSAECONNABORTED: c_int = WSABASEERR + 53; +pub const WSAECONNRESET: c_int = WSABASEERR + 54; +pub const WSAENOBUFS: c_int = WSABASEERR + 55; +pub const WSAEISCONN: c_int = WSABASEERR + 56; +pub const WSAENOTCONN: c_int = WSABASEERR + 57; +pub const WSAESHUTDOWN: c_int = WSABASEERR + 58; +pub const WSAETOOMANYREFS: c_int = WSABASEERR + 59; +pub const WSAETIMEDOUT: c_int = WSABASEERR + 60; +pub const WSAECONNREFUSED: c_int = WSABASEERR + 61; +pub const WSAELOOP: c_int = WSABASEERR + 62; +pub const WSAENAMETOOLONG: c_int = WSABASEERR + 63; +pub const WSAEHOSTDOWN: c_int = WSABASEERR + 64; +pub const WSAEHOSTUNREACH: c_int = WSABASEERR + 65; +pub const WSAENOTEMPTY: c_int = WSABASEERR + 66; +pub const WSAEPROCLIM: c_int = WSABASEERR + 67; +pub const WSAEUSERS: c_int = WSABASEERR + 68; +pub const WSAEDQUOT: c_int = WSABASEERR + 69; +pub const WSAESTALE: c_int = WSABASEERR + 70; +pub const WSAEREMOTE: c_int = WSABASEERR + 71; +pub const WSASYSNOTREADY: c_int = WSABASEERR + 91; +pub const WSAVERNOTSUPPORTED: c_int = WSABASEERR + 92; +pub const WSANOTINITIALISED: c_int = WSABASEERR + 93; +pub const WSAEDISCON: c_int = WSABASEERR + 101; +pub const WSAENOMORE: c_int = WSABASEERR + 102; +pub const WSAECANCELLED: c_int = WSABASEERR + 103; +pub const WSAEINVALIDPROCTABLE: c_int = WSABASEERR + 104; +pub const WSAEINVALIDPROVIDER: c_int = WSABASEERR + 105; +pub const WSAEPROVIDERFAILEDINIT: c_int = WSABASEERR + 106; +pub const WSASYSCALLFAILURE: c_int = WSABASEERR + 107; +pub const WSASERVICE_NOT_FOUND: c_int = WSABASEERR + 108; +pub const WSATYPE_NOT_FOUND: c_int = WSABASEERR + 109; +pub const WSA_E_NO_MORE: c_int = WSABASEERR + 110; +pub const WSA_E_CANCELLED: c_int = WSABASEERR + 111; +pub const WSAEREFUSED: c_int = WSABASEERR + 112; +pub const WSAHOST_NOT_FOUND: c_int = WSABASEERR + 1001; +pub const WSATRY_AGAIN: c_int = WSABASEERR + 1002; +pub const WSANO_RECOVERY: c_int = WSABASEERR + 1003; +pub const WSANO_DATA: c_int = WSABASEERR + 1004; +pub const WSA_QOS_RECEIVERS: c_int = WSABASEERR + 1005; +pub const WSA_QOS_SENDERS: c_int = WSABASEERR + 1006; +pub const WSA_QOS_NO_SENDERS: c_int = WSABASEERR + 1007; +pub const WSA_QOS_NO_RECEIVERS: c_int = WSABASEERR + 1008; +pub const WSA_QOS_REQUEST_CONFIRMED: c_int = WSABASEERR + 1009; +pub const WSA_QOS_ADMISSION_FAILURE: c_int = WSABASEERR + 1010; +pub const WSA_QOS_POLICY_FAILURE: c_int = WSABASEERR + 1011; +pub const WSA_QOS_BAD_STYLE: c_int = WSABASEERR + 1012; +pub const WSA_QOS_BAD_OBJECT: c_int = WSABASEERR + 1013; +pub const WSA_QOS_TRAFFIC_CTRL_ERROR: c_int = WSABASEERR + 1014; +pub const WSA_QOS_GENERIC_ERROR: c_int = WSABASEERR + 1015; +pub const WSA_QOS_ESERVICETYPE: c_int = WSABASEERR + 1016; +pub const WSA_QOS_EFLOWSPEC: c_int = WSABASEERR + 1017; +pub const WSA_QOS_EPROVSPECBUF: c_int = WSABASEERR + 1018; +pub const WSA_QOS_EFILTERSTYLE: c_int = WSABASEERR + 1019; +pub const WSA_QOS_EFILTERTYPE: c_int = WSABASEERR + 1020; +pub const WSA_QOS_EFILTERCOUNT: c_int = WSABASEERR + 1021; +pub const WSA_QOS_EOBJLENGTH: c_int = WSABASEERR + 1022; +pub const WSA_QOS_EFLOWCOUNT: c_int = WSABASEERR + 1023; +pub const WSA_QOS_EUNKNOWNPSOBJ: c_int = WSABASEERR + 1024; +pub const WSA_QOS_EUNKOWNPSOBJ: c_int = WSA_QOS_EUNKNOWNPSOBJ; +pub const WSA_QOS_EPOLICYOBJ: c_int = WSABASEERR + 1025; +pub const WSA_QOS_EFLOWDESC: c_int = WSABASEERR + 1026; +pub const WSA_QOS_EPSFLOWSPEC: c_int = WSABASEERR + 1027; +pub const WSA_QOS_EPSFILTERSPEC: c_int = WSABASEERR + 1028; +pub const WSA_QOS_ESDMODEOBJ: c_int = WSABASEERR + 1029; +pub const WSA_QOS_ESHAPERATEOBJ: c_int = WSABASEERR + 1030; +pub const WSA_QOS_RESERVED_PETYPE: c_int = WSABASEERR + 1031; diff --git a/crux-mir/lib/std/src/sys/windows/cmath.rs b/crux-mir/lib/std/src/sys/windows/cmath.rs index 1a5421fac..43ab8c7ee 100644 --- a/crux-mir/lib/std/src/sys/windows/cmath.rs +++ b/crux-mir/lib/std/src/sys/windows/cmath.rs @@ -44,7 +44,7 @@ mod shims { } // On 32-bit x86 MSVC these functions aren't defined, so we just define shims -// which promote everything fo f64, perform the calculation, and then demote +// which promote everything to f64, perform the calculation, and then demote // back to f32. While not precisely correct should be "correct enough" for now. #[cfg(all(target_env = "msvc", target_arch = "x86"))] mod shims { diff --git a/crux-mir/lib/std/src/sys/windows/compat.rs b/crux-mir/lib/std/src/sys/windows/compat.rs index d6d433f9d..7dff81ecb 100644 --- a/crux-mir/lib/std/src/sys/windows/compat.rs +++ b/crux-mir/lib/std/src/sys/windows/compat.rs @@ -1,72 +1,243 @@ -//! A "compatibility layer" for spanning XP and Windows 7 +//! A "compatibility layer" for supporting older versions of Windows //! -//! The standard library currently binds many functions that are not available -//! on Windows XP, but we would also like to support building executables that -//! run on XP. To do this we specify all non-XP APIs as having a fallback -//! implementation to do something reasonable. +//! The standard library uses some Windows API functions that are not present +//! on older versions of Windows. (Note that the oldest version of Windows +//! that Rust supports is Windows 7 (client) and Windows Server 2008 (server).) +//! This module implements a form of delayed DLL import binding, using +//! `GetModuleHandle` and `GetProcAddress` to look up DLL entry points at +//! runtime. //! -//! This dynamic runtime detection of whether a function is available is -//! implemented with `GetModuleHandle` and `GetProcAddress` paired with a -//! static-per-function which caches the result of the first check. In this -//! manner we pay a semi-large one-time cost up front for detecting whether a -//! function is available but afterwards it's just a load and a jump. - -use crate::ffi::CString; -use crate::sync::atomic::{AtomicUsize, Ordering}; +//! This is implemented simply by storing a function pointer in an atomic. +//! Loading and calling this function will have little or no overhead +//! compared with calling any other dynamically imported function. +//! +//! The stored function pointer starts out as an importer function which will +//! swap itself with the real function when it's called for the first time. If +//! the real function can't be imported then a fallback function is used in its +//! place. While this is low cost for the happy path (where the function is +//! already loaded) it does mean there's some overhead the first time the +//! function is called. In the worst case, multiple threads may all end up +//! importing the same function unnecessarily. + +use crate::ffi::{c_void, CStr}; +use crate::ptr::NonNull; +use crate::sync::atomic::Ordering; use crate::sys::c; -pub fn lookup(module: &str, symbol: &str) -> Option { - let mut module: Vec = module.encode_utf16().collect(); - module.push(0); - let symbol = CString::new(symbol).unwrap(); - unsafe { - let handle = c::GetModuleHandleW(module.as_ptr()); - match c::GetProcAddress(handle, symbol.as_ptr()) as usize { - 0 => None, - n => Some(n), +// This uses a static initializer to preload some imported functions. +// The CRT (C runtime) executes static initializers before `main` +// is called (for binaries) and before `DllMain` is called (for DLLs). +// +// It works by contributing a global symbol to the `.CRT$XCT` section. +// The linker builds a table of all static initializer functions. +// The CRT startup code then iterates that table, calling each +// initializer function. +// +// NOTE: User code should instead use .CRT$XCU to reliably run after std's initializer. +// If you're reading this and would like a guarantee here, please +// file an issue for discussion; currently we don't guarantee any functionality +// before main. +// See https://docs.microsoft.com/en-us/cpp/c-runtime-library/crt-initialization?view=msvc-170 +#[used] +#[link_section = ".CRT$XCT"] +static INIT_TABLE_ENTRY: unsafe extern "C" fn() = init; + +/// Preload some imported functions. +/// +/// Note that any functions included here will be unconditionally loaded in +/// the final binary, regardless of whether or not they're actually used. +/// +/// Therefore, this should be limited to `compat_fn_optional` functions which +/// must be preloaded or any functions where lazier loading demonstrates a +/// negative performance impact in practical situations. +/// +/// Currently we only preload `WaitOnAddress` and `WakeByAddressSingle`. +unsafe extern "C" fn init() { + // In an exe this code is executed before main() so is single threaded. + // In a DLL the system's loader lock will be held thereby synchronizing + // access. So the same best practices apply here as they do to running in DllMain: + // https://docs.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-best-practices + // + // DO NOT do anything interesting or complicated in this function! DO NOT call + // any Rust functions or CRT functions if those functions touch any global state, + // because this function runs during global initialization. For example, DO NOT + // do any dynamic allocation, don't call LoadLibrary, etc. + + // Attempt to preload the synch functions. + load_synch_functions(); +} + +/// Helper macro for creating CStrs from literals and symbol names. +macro_rules! ansi_str { + (sym $ident:ident) => {{ + #[allow(unused_unsafe)] + crate::sys::compat::const_cstr_from_bytes(concat!(stringify!($ident), "\0").as_bytes()) + }}; + ($lit:literal) => {{ crate::sys::compat::const_cstr_from_bytes(concat!($lit, "\0").as_bytes()) }}; +} + +/// Creates a C string wrapper from a byte slice, in a constant context. +/// +/// This is a utility function used by the [`ansi_str`] macro. +/// +/// # Panics +/// +/// Panics if the slice is not null terminated or contains nulls, except as the last item +pub(crate) const fn const_cstr_from_bytes(bytes: &'static [u8]) -> &'static CStr { + if !matches!(bytes.last(), Some(&0)) { + panic!("A CStr must be null terminated"); + } + let mut i = 0; + // At this point `len()` is at least 1. + while i < bytes.len() - 1 { + if bytes[i] == 0 { + panic!("A CStr must not have interior nulls") } + i += 1; } + // SAFETY: The safety is ensured by the above checks. + unsafe { crate::ffi::CStr::from_bytes_with_nul_unchecked(bytes) } } -pub fn store_func(ptr: &AtomicUsize, module: &str, symbol: &str, fallback: usize) -> usize { - let value = lookup(module, symbol).unwrap_or(fallback); - ptr.store(value, Ordering::SeqCst); - value +/// Represents a loaded module. +/// +/// Note that the modules std depends on must not be unloaded. +/// Therefore a `Module` is always valid for the lifetime of std. +#[derive(Copy, Clone)] +pub(in crate::sys) struct Module(NonNull); +impl Module { + /// Try to get a handle to a loaded module. + /// + /// # SAFETY + /// + /// This should only be use for modules that exist for the lifetime of std + /// (e.g. kernel32 and ntdll). + pub unsafe fn new(name: &CStr) -> Option { + // SAFETY: A CStr is always null terminated. + let module = c::GetModuleHandleA(name.as_ptr()); + NonNull::new(module).map(Self) + } + + // Try to get the address of a function. + pub fn proc_address(self, name: &CStr) -> Option> { + // SAFETY: + // `self.0` will always be a valid module. + // A CStr is always null terminated. + let proc = unsafe { c::GetProcAddress(self.0.as_ptr(), name.as_ptr()) }; + NonNull::new(proc) + } } -macro_rules! compat_fn { - ($module:ident: $( +/// Load a function or use a fallback implementation if that fails. +macro_rules! compat_fn_with_fallback { + (pub static $module:ident: &CStr = $name:expr; $( $(#[$meta:meta])* - pub fn $symbol:ident($($argname:ident: $argtype:ty),*) - -> $rettype:ty { - $($body:expr);* - } - )*) => ($( - #[allow(unused_variables)] + $vis:vis fn $symbol:ident($($argname:ident: $argtype:ty),*) -> $rettype:ty $fallback_body:block + )*) => ( + pub static $module: &CStr = $name; + $( $(#[$meta])* - pub unsafe fn $symbol($($argname: $argtype),*) -> $rettype { - use crate::sync::atomic::{AtomicUsize, Ordering}; + pub mod $symbol { + #[allow(unused_imports)] + use super::*; use crate::mem; + use crate::ffi::CStr; + use crate::sync::atomic::{AtomicPtr, Ordering}; + use crate::sys::compat::Module; + type F = unsafe extern "system" fn($($argtype),*) -> $rettype; - static PTR: AtomicUsize = AtomicUsize::new(0); + /// `PTR` contains a function pointer to one of three functions. + /// It starts with the `load` function. + /// When that is called it attempts to load the requested symbol. + /// If it succeeds, `PTR` is set to the address of that symbol. + /// If it fails, then `PTR` is set to `fallback`. + static PTR: AtomicPtr = AtomicPtr::new(load as *mut _); + + unsafe extern "system" fn load($($argname: $argtype),*) -> $rettype { + let func = load_from_module(Module::new($module)); + func($($argname),*) + } - fn load() -> usize { - crate::sys::compat::store_func(&PTR, - stringify!($module), - stringify!($symbol), - fallback as usize) + fn load_from_module(module: Option) -> F { + unsafe { + static SYMBOL_NAME: &CStr = ansi_str!(sym $symbol); + if let Some(f) = module.and_then(|m| m.proc_address(SYMBOL_NAME)) { + PTR.store(f.as_ptr(), Ordering::Relaxed); + mem::transmute(f) + } else { + PTR.store(fallback as *mut _, Ordering::Relaxed); + fallback + } + } } - unsafe extern "system" fn fallback($($argname: $argtype),*) - -> $rettype { - $($body);* + + #[allow(unused_variables)] + unsafe extern "system" fn fallback($($argname: $argtype),*) -> $rettype { + $fallback_body } - let addr = match PTR.load(Ordering::SeqCst) { - 0 => load(), - n => n, - }; - mem::transmute::(addr)($($argname),*) + #[inline(always)] + pub unsafe fn call($($argname: $argtype),*) -> $rettype { + let func: F = mem::transmute(PTR.load(Ordering::Relaxed)); + func($($argname),*) + } } + $(#[$meta])* + $vis use $symbol::call as $symbol; )*) } + +/// Optionally loaded functions. +/// +/// Actual loading of the function defers to $load_functions. +macro_rules! compat_fn_optional { + ($load_functions:expr; + $( + $(#[$meta:meta])* + $vis:vis fn $symbol:ident($($argname:ident: $argtype:ty),*) $(-> $rettype:ty)?; + )+) => ( + $( + pub mod $symbol { + use super::*; + use crate::ffi::c_void; + use crate::mem; + use crate::ptr::{self, NonNull}; + use crate::sync::atomic::{AtomicPtr, Ordering}; + + pub(in crate::sys) static PTR: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + + type F = unsafe extern "system" fn($($argtype),*) $(-> $rettype)?; + + #[inline(always)] + pub fn option() -> Option { + // Miri does not understand the way we do preloading + // therefore load the function here instead. + #[cfg(miri)] $load_functions; + NonNull::new(PTR.load(Ordering::Relaxed)).map(|f| unsafe { mem::transmute(f) }) + } + } + )+ + ) +} + +/// Load all needed functions from "api-ms-win-core-synch-l1-2-0". +pub(super) fn load_synch_functions() { + fn try_load() -> Option<()> { + const MODULE_NAME: &CStr = ansi_str!("api-ms-win-core-synch-l1-2-0"); + const WAIT_ON_ADDRESS: &CStr = ansi_str!("WaitOnAddress"); + const WAKE_BY_ADDRESS_SINGLE: &CStr = ansi_str!("WakeByAddressSingle"); + + // Try loading the library and all the required functions. + // If any step fails, then they all fail. + let library = unsafe { Module::new(MODULE_NAME) }?; + let wait_on_address = library.proc_address(WAIT_ON_ADDRESS)?; + let wake_by_address_single = library.proc_address(WAKE_BY_ADDRESS_SINGLE)?; + + c::WaitOnAddress::PTR.store(wait_on_address.as_ptr(), Ordering::Relaxed); + c::WakeByAddressSingle::PTR.store(wake_by_address_single.as_ptr(), Ordering::Relaxed); + Some(()) + } + + try_load(); +} diff --git a/crux-mir/lib/std/src/sys/windows/ext/io.rs b/crux-mir/lib/std/src/sys/windows/ext/io.rs deleted file mode 100644 index 4573ee589..000000000 --- a/crux-mir/lib/std/src/sys/windows/ext/io.rs +++ /dev/null @@ -1,220 +0,0 @@ -#![stable(feature = "rust1", since = "1.0.0")] - -use crate::fs; -use crate::io; -use crate::net; -use crate::os::windows::raw; -use crate::sys; -use crate::sys::c; -use crate::sys_common::{self, AsInner, FromInner, IntoInner}; - -/// Raw HANDLEs. -#[stable(feature = "rust1", since = "1.0.0")] -pub type RawHandle = raw::HANDLE; - -/// Raw SOCKETs. -#[stable(feature = "rust1", since = "1.0.0")] -pub type RawSocket = raw::SOCKET; - -/// Extracts raw handles. -#[stable(feature = "rust1", since = "1.0.0")] -pub trait AsRawHandle { - /// Extracts the raw handle, without taking any ownership. - #[stable(feature = "rust1", since = "1.0.0")] - fn as_raw_handle(&self) -> RawHandle; -} - -/// Construct I/O objects from raw handles. -#[stable(feature = "from_raw_os", since = "1.1.0")] -pub trait FromRawHandle { - /// Constructs a new I/O object from the specified raw handle. - /// - /// This function will **consume ownership** of the handle given, - /// passing responsibility for closing the handle to the returned - /// object. - /// - /// This function is also unsafe as the primitives currently returned - /// have the contract that they are the sole owner of the file - /// descriptor they are wrapping. Usage of this function could - /// accidentally allow violating this contract which can cause memory - /// unsafety in code that relies on it being true. - #[stable(feature = "from_raw_os", since = "1.1.0")] - unsafe fn from_raw_handle(handle: RawHandle) -> Self; -} - -/// A trait to express the ability to consume an object and acquire ownership of -/// its raw `HANDLE`. -#[stable(feature = "into_raw_os", since = "1.4.0")] -pub trait IntoRawHandle { - /// Consumes this object, returning the raw underlying handle. - /// - /// This function **transfers ownership** of the underlying handle to the - /// caller. Callers are then the unique owners of the handle and must close - /// it once it's no longer needed. - #[stable(feature = "into_raw_os", since = "1.4.0")] - fn into_raw_handle(self) -> RawHandle; -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl AsRawHandle for fs::File { - fn as_raw_handle(&self) -> RawHandle { - self.as_inner().handle().raw() as RawHandle - } -} - -#[stable(feature = "asraw_stdio", since = "1.21.0")] -impl AsRawHandle for io::Stdin { - fn as_raw_handle(&self) -> RawHandle { - unsafe { c::GetStdHandle(c::STD_INPUT_HANDLE) as RawHandle } - } -} - -#[stable(feature = "asraw_stdio", since = "1.21.0")] -impl AsRawHandle for io::Stdout { - fn as_raw_handle(&self) -> RawHandle { - unsafe { c::GetStdHandle(c::STD_OUTPUT_HANDLE) as RawHandle } - } -} - -#[stable(feature = "asraw_stdio", since = "1.21.0")] -impl AsRawHandle for io::Stderr { - fn as_raw_handle(&self) -> RawHandle { - unsafe { c::GetStdHandle(c::STD_ERROR_HANDLE) as RawHandle } - } -} - -#[stable(feature = "asraw_stdio_locks", since = "1.35.0")] -impl<'a> AsRawHandle for io::StdinLock<'a> { - fn as_raw_handle(&self) -> RawHandle { - unsafe { c::GetStdHandle(c::STD_INPUT_HANDLE) as RawHandle } - } -} - -#[stable(feature = "asraw_stdio_locks", since = "1.35.0")] -impl<'a> AsRawHandle for io::StdoutLock<'a> { - fn as_raw_handle(&self) -> RawHandle { - unsafe { c::GetStdHandle(c::STD_OUTPUT_HANDLE) as RawHandle } - } -} - -#[stable(feature = "asraw_stdio_locks", since = "1.35.0")] -impl<'a> AsRawHandle for io::StderrLock<'a> { - fn as_raw_handle(&self) -> RawHandle { - unsafe { c::GetStdHandle(c::STD_ERROR_HANDLE) as RawHandle } - } -} - -#[stable(feature = "from_raw_os", since = "1.1.0")] -impl FromRawHandle for fs::File { - unsafe fn from_raw_handle(handle: RawHandle) -> fs::File { - let handle = handle as c::HANDLE; - fs::File::from_inner(sys::fs::File::from_inner(handle)) - } -} - -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawHandle for fs::File { - fn into_raw_handle(self) -> RawHandle { - self.into_inner().into_handle().into_raw() as *mut _ - } -} - -/// Extracts raw sockets. -#[stable(feature = "rust1", since = "1.0.0")] -pub trait AsRawSocket { - /// Extracts the underlying raw socket from this object. - #[stable(feature = "rust1", since = "1.0.0")] - fn as_raw_socket(&self) -> RawSocket; -} - -/// Creates I/O objects from raw sockets. -#[stable(feature = "from_raw_os", since = "1.1.0")] -pub trait FromRawSocket { - /// Creates a new I/O object from the given raw socket. - /// - /// This function will **consume ownership** of the socket provided and - /// it will be closed when the returned object goes out of scope. - /// - /// This function is also unsafe as the primitives currently returned - /// have the contract that they are the sole owner of the file - /// descriptor they are wrapping. Usage of this function could - /// accidentally allow violating this contract which can cause memory - /// unsafety in code that relies on it being true. - #[stable(feature = "from_raw_os", since = "1.1.0")] - unsafe fn from_raw_socket(sock: RawSocket) -> Self; -} - -/// A trait to express the ability to consume an object and acquire ownership of -/// its raw `SOCKET`. -#[stable(feature = "into_raw_os", since = "1.4.0")] -pub trait IntoRawSocket { - /// Consumes this object, returning the raw underlying socket. - /// - /// This function **transfers ownership** of the underlying socket to the - /// caller. Callers are then the unique owners of the socket and must close - /// it once it's no longer needed. - #[stable(feature = "into_raw_os", since = "1.4.0")] - fn into_raw_socket(self) -> RawSocket; -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl AsRawSocket for net::TcpStream { - fn as_raw_socket(&self) -> RawSocket { - *self.as_inner().socket().as_inner() - } -} -#[stable(feature = "rust1", since = "1.0.0")] -impl AsRawSocket for net::TcpListener { - fn as_raw_socket(&self) -> RawSocket { - *self.as_inner().socket().as_inner() - } -} -#[stable(feature = "rust1", since = "1.0.0")] -impl AsRawSocket for net::UdpSocket { - fn as_raw_socket(&self) -> RawSocket { - *self.as_inner().socket().as_inner() - } -} - -#[stable(feature = "from_raw_os", since = "1.1.0")] -impl FromRawSocket for net::TcpStream { - unsafe fn from_raw_socket(sock: RawSocket) -> net::TcpStream { - let sock = sys::net::Socket::from_inner(sock); - net::TcpStream::from_inner(sys_common::net::TcpStream::from_inner(sock)) - } -} -#[stable(feature = "from_raw_os", since = "1.1.0")] -impl FromRawSocket for net::TcpListener { - unsafe fn from_raw_socket(sock: RawSocket) -> net::TcpListener { - let sock = sys::net::Socket::from_inner(sock); - net::TcpListener::from_inner(sys_common::net::TcpListener::from_inner(sock)) - } -} -#[stable(feature = "from_raw_os", since = "1.1.0")] -impl FromRawSocket for net::UdpSocket { - unsafe fn from_raw_socket(sock: RawSocket) -> net::UdpSocket { - let sock = sys::net::Socket::from_inner(sock); - net::UdpSocket::from_inner(sys_common::net::UdpSocket::from_inner(sock)) - } -} - -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawSocket for net::TcpStream { - fn into_raw_socket(self) -> RawSocket { - self.into_inner().into_socket().into_inner() - } -} - -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawSocket for net::TcpListener { - fn into_raw_socket(self) -> RawSocket { - self.into_inner().into_socket().into_inner() - } -} - -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawSocket for net::UdpSocket { - fn into_raw_socket(self) -> RawSocket { - self.into_inner().into_socket().into_inner() - } -} diff --git a/crux-mir/lib/std/src/sys/windows/ext/process.rs b/crux-mir/lib/std/src/sys/windows/ext/process.rs deleted file mode 100644 index 8c34a9faf..000000000 --- a/crux-mir/lib/std/src/sys/windows/ext/process.rs +++ /dev/null @@ -1,113 +0,0 @@ -//! Extensions to `std::process` for Windows. - -#![stable(feature = "process_extensions", since = "1.2.0")] - -use crate::os::windows::io::{AsRawHandle, FromRawHandle, IntoRawHandle, RawHandle}; -use crate::process; -use crate::sys; -use crate::sys_common::{AsInner, AsInnerMut, FromInner, IntoInner}; - -#[stable(feature = "process_extensions", since = "1.2.0")] -impl FromRawHandle for process::Stdio { - unsafe fn from_raw_handle(handle: RawHandle) -> process::Stdio { - let handle = sys::handle::Handle::new(handle as *mut _); - let io = sys::process::Stdio::Handle(handle); - process::Stdio::from_inner(io) - } -} - -#[stable(feature = "process_extensions", since = "1.2.0")] -impl AsRawHandle for process::Child { - fn as_raw_handle(&self) -> RawHandle { - self.as_inner().handle().raw() as *mut _ - } -} - -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawHandle for process::Child { - fn into_raw_handle(self) -> RawHandle { - self.into_inner().into_handle().into_raw() as *mut _ - } -} - -#[stable(feature = "process_extensions", since = "1.2.0")] -impl AsRawHandle for process::ChildStdin { - fn as_raw_handle(&self) -> RawHandle { - self.as_inner().handle().raw() as *mut _ - } -} - -#[stable(feature = "process_extensions", since = "1.2.0")] -impl AsRawHandle for process::ChildStdout { - fn as_raw_handle(&self) -> RawHandle { - self.as_inner().handle().raw() as *mut _ - } -} - -#[stable(feature = "process_extensions", since = "1.2.0")] -impl AsRawHandle for process::ChildStderr { - fn as_raw_handle(&self) -> RawHandle { - self.as_inner().handle().raw() as *mut _ - } -} - -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawHandle for process::ChildStdin { - fn into_raw_handle(self) -> RawHandle { - self.into_inner().into_handle().into_raw() as *mut _ - } -} - -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawHandle for process::ChildStdout { - fn into_raw_handle(self) -> RawHandle { - self.into_inner().into_handle().into_raw() as *mut _ - } -} - -#[stable(feature = "into_raw_os", since = "1.4.0")] -impl IntoRawHandle for process::ChildStderr { - fn into_raw_handle(self) -> RawHandle { - self.into_inner().into_handle().into_raw() as *mut _ - } -} - -/// Windows-specific extensions to [`process::ExitStatus`]. -/// -/// [`process::ExitStatus`]: ../../../../std/process/struct.ExitStatus.html -#[stable(feature = "exit_status_from", since = "1.12.0")] -pub trait ExitStatusExt { - /// Creates a new `ExitStatus` from the raw underlying `u32` return value of - /// a process. - #[stable(feature = "exit_status_from", since = "1.12.0")] - fn from_raw(raw: u32) -> Self; -} - -#[stable(feature = "exit_status_from", since = "1.12.0")] -impl ExitStatusExt for process::ExitStatus { - fn from_raw(raw: u32) -> Self { - process::ExitStatus::from_inner(From::from(raw)) - } -} - -/// Windows-specific extensions to the [`process::Command`] builder. -/// -/// [`process::Command`]: ../../../../std/process/struct.Command.html -#[stable(feature = "windows_process_extensions", since = "1.16.0")] -pub trait CommandExt { - /// Sets the [process creation flags][1] to be passed to `CreateProcess`. - /// - /// These will always be ORed with `CREATE_UNICODE_ENVIRONMENT`. - /// - /// [1]: https://docs.microsoft.com/en-us/windows/win32/procthread/process-creation-flags - #[stable(feature = "windows_process_extensions", since = "1.16.0")] - fn creation_flags(&mut self, flags: u32) -> &mut process::Command; -} - -#[stable(feature = "windows_process_extensions", since = "1.16.0")] -impl CommandExt for process::Command { - fn creation_flags(&mut self, flags: u32) -> &mut process::Command { - self.as_inner_mut().creation_flags(flags); - self - } -} diff --git a/crux-mir/lib/std/src/sys/windows/fast_thread_local.rs b/crux-mir/lib/std/src/sys/windows/fast_thread_local.rs deleted file mode 100644 index 191fa07f3..000000000 --- a/crux-mir/lib/std/src/sys/windows/fast_thread_local.rs +++ /dev/null @@ -1,4 +0,0 @@ -#![unstable(feature = "thread_local_internals", issue = "none")] -#![cfg(target_thread_local)] - -pub use crate::sys_common::thread_local::register_dtor_fallback as register_dtor; diff --git a/crux-mir/lib/std/src/sys/windows/fs.rs b/crux-mir/lib/std/src/sys/windows/fs.rs index 427f4b684..378098038 100644 --- a/crux-mir/lib/std/src/sys/windows/fs.rs +++ b/crux-mir/lib/std/src/sys/windows/fs.rs @@ -1,18 +1,22 @@ use crate::os::windows::prelude::*; +use crate::borrow::Cow; use crate::ffi::OsString; use crate::fmt; -use crate::io::{self, Error, IoSlice, IoSliceMut, SeekFrom}; -use crate::mem; +use crate::io::{self, BorrowedCursor, Error, IoSlice, IoSliceMut, SeekFrom}; +use crate::mem::{self, MaybeUninit}; +use crate::os::windows::io::{AsHandle, BorrowedHandle}; use crate::path::{Path, PathBuf}; use crate::ptr; use crate::slice; use crate::sync::Arc; use crate::sys::handle::Handle; use crate::sys::time::SystemTime; -use crate::sys::{c, cvt}; -use crate::sys_common::FromInner; +use crate::sys::{c, cvt, Align8}; +use crate::sys_common::{AsInner, FromInner, IntoInner}; +use crate::thread; +use super::path::maybe_verbatim; use super::to_u16s; pub struct File { @@ -54,6 +58,9 @@ pub struct DirEntry { data: c::WIN32_FIND_DATAW, } +unsafe impl Send for OpenOptions {} +unsafe impl Sync for OpenOptions {} + #[derive(Clone, Debug)] pub struct OpenOptions { // generic @@ -69,7 +76,7 @@ pub struct OpenOptions { attributes: c::DWORD, share_mode: c::DWORD, security_qos_flags: c::DWORD, - security_attributes: usize, // FIXME: should be a reference + security_attributes: c::LPSECURITY_ATTRIBUTES, } #[derive(Clone, PartialEq, Eq, Debug)] @@ -77,6 +84,12 @@ pub struct FilePermissions { attrs: c::DWORD, } +#[derive(Copy, Clone, Debug, Default)] +pub struct FileTimes { + accessed: Option, + modified: Option, +} + #[derive(Debug)] pub struct DirBuilder; @@ -149,22 +162,7 @@ impl DirEntry { } pub fn metadata(&self) -> io::Result { - Ok(FileAttr { - attributes: self.data.dwFileAttributes, - creation_time: self.data.ftCreationTime, - last_access_time: self.data.ftLastAccessTime, - last_write_time: self.data.ftLastWriteTime, - file_size: ((self.data.nFileSizeHigh as u64) << 32) | (self.data.nFileSizeLow as u64), - reparse_tag: if self.data.dwFileAttributes & c::FILE_ATTRIBUTE_REPARSE_POINT != 0 { - // reserved unless this is a reparse point - self.data.dwReserved0 - } else { - 0 - }, - volume_serial_number: None, - number_of_links: None, - file_index: None, - }) + Ok(self.data.into()) } } @@ -184,7 +182,7 @@ impl OpenOptions { share_mode: c::FILE_SHARE_READ | c::FILE_SHARE_WRITE | c::FILE_SHARE_DELETE, attributes: 0, security_qos_flags: 0, - security_attributes: 0, + security_attributes: ptr::null_mut(), } } @@ -225,7 +223,7 @@ impl OpenOptions { self.security_qos_flags = flags | c::SECURITY_SQOS_PRESENT; } pub fn security_attributes(&mut self, attrs: c::LPSECURITY_ATTRIBUTES) { - self.security_attributes = attrs as usize; + self.security_attributes = attrs; } fn get_access_mode(&self) -> io::Result { @@ -280,27 +278,27 @@ impl OpenOptions { impl File { pub fn open(path: &Path, opts: &OpenOptions) -> io::Result { - let path = to_u16s(path)?; + let path = maybe_verbatim(path)?; let handle = unsafe { c::CreateFileW( path.as_ptr(), opts.get_access_mode()?, opts.share_mode, - opts.security_attributes as *mut _, + opts.security_attributes, opts.get_creation_mode()?, opts.get_flags_and_attributes(), ptr::null_mut(), ) }; - if handle == c::INVALID_HANDLE_VALUE { - Err(Error::last_os_error()) + if let Ok(handle) = handle.try_into() { + Ok(File { handle: Handle::from_inner(handle) }) } else { - Ok(File { handle: Handle::new(handle) }) + Err(Error::last_os_error()) } } pub fn fsync(&self) -> io::Result<()> { - cvt(unsafe { c::FlushFileBuffers(self.handle.raw()) })?; + cvt(unsafe { c::FlushFileBuffers(self.handle.as_raw_handle()) })?; Ok(()) } @@ -313,7 +311,7 @@ impl File { let size = mem::size_of_val(&info); cvt(unsafe { c::SetFileInformationByHandle( - self.handle.raw(), + self.handle.as_raw_handle(), c::FileEndOfFileInfo, &mut info as *mut _ as *mut _, size as c::DWORD, @@ -326,12 +324,18 @@ impl File { pub fn file_attr(&self) -> io::Result { unsafe { let mut info: c::BY_HANDLE_FILE_INFORMATION = mem::zeroed(); - cvt(c::GetFileInformationByHandle(self.handle.raw(), &mut info))?; + cvt(c::GetFileInformationByHandle(self.handle.as_raw_handle(), &mut info))?; let mut reparse_tag = 0; if info.dwFileAttributes & c::FILE_ATTRIBUTE_REPARSE_POINT != 0 { - let mut b = [0; c::MAXIMUM_REPARSE_DATA_BUFFER_SIZE]; - if let Ok((_, buf)) = self.reparse_point(&mut b) { - reparse_tag = buf.ReparseTag; + let mut attr_tag: c::FILE_ATTRIBUTE_TAG_INFO = mem::zeroed(); + cvt(c::GetFileInformationByHandleEx( + self.handle.as_raw_handle(), + c::FileAttributeTagInfo, + ptr::addr_of_mut!(attr_tag).cast(), + mem::size_of::().try_into().unwrap(), + ))?; + if attr_tag.FileAttributes & c::FILE_ATTRIBUTE_REPARSE_POINT != 0 { + reparse_tag = attr_tag.ReparseTag; } } Ok(FileAttr { @@ -356,7 +360,7 @@ impl File { let mut info: c::FILE_BASIC_INFO = mem::zeroed(); let size = mem::size_of_val(&info); cvt(c::GetFileInformationByHandleEx( - self.handle.raw(), + self.handle.as_raw_handle(), c::FileBasicInfo, &mut info as *mut _ as *mut libc::c_void, size as c::DWORD, @@ -384,7 +388,7 @@ impl File { let mut info: c::FILE_STANDARD_INFO = mem::zeroed(); let size = mem::size_of_val(&info); cvt(c::GetFileInformationByHandleEx( - self.handle.raw(), + self.handle.as_raw_handle(), c::FileStandardInfo, &mut info as *mut _ as *mut libc::c_void, size as c::DWORD, @@ -392,9 +396,15 @@ impl File { attr.file_size = info.AllocationSize as u64; attr.number_of_links = Some(info.NumberOfLinks); if attr.file_type().is_reparse_point() { - let mut b = [0; c::MAXIMUM_REPARSE_DATA_BUFFER_SIZE]; - if let Ok((_, buf)) = self.reparse_point(&mut b) { - attr.reparse_tag = buf.ReparseTag; + let mut attr_tag: c::FILE_ATTRIBUTE_TAG_INFO = mem::zeroed(); + cvt(c::GetFileInformationByHandleEx( + self.handle.as_raw_handle(), + c::FileAttributeTagInfo, + ptr::addr_of_mut!(attr_tag).cast(), + mem::size_of::().try_into().unwrap(), + ))?; + if attr_tag.FileAttributes & c::FILE_ATTRIBUTE_REPARSE_POINT != 0 { + attr.reparse_tag = attr_tag.ReparseTag; } } Ok(attr) @@ -409,10 +419,19 @@ impl File { self.handle.read_vectored(bufs) } + #[inline] + pub fn is_read_vectored(&self) -> bool { + self.handle.is_read_vectored() + } + pub fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result { self.handle.read_at(buf, offset) } + pub fn read_buf(&self, cursor: BorrowedCursor<'_>) -> io::Result<()> { + self.handle.read_buf(cursor) + } + pub fn write(&self, buf: &[u8]) -> io::Result { self.handle.write(buf) } @@ -421,6 +440,11 @@ impl File { self.handle.write_vectored(bufs) } + #[inline] + pub fn is_write_vectored(&self) -> bool { + self.handle.is_write_vectored() + } + pub fn write_at(&self, buf: &[u8], offset: u64) -> io::Result { self.handle.write_at(buf, offset) } @@ -439,54 +463,54 @@ impl File { }; let pos = pos as c::LARGE_INTEGER; let mut newpos = 0; - cvt(unsafe { c::SetFilePointerEx(self.handle.raw(), pos, &mut newpos, whence) })?; + cvt(unsafe { c::SetFilePointerEx(self.handle.as_raw_handle(), pos, &mut newpos, whence) })?; Ok(newpos as u64) } pub fn duplicate(&self) -> io::Result { - Ok(File { handle: self.handle.duplicate(0, false, c::DUPLICATE_SAME_ACCESS)? }) - } - - pub fn handle(&self) -> &Handle { - &self.handle + Ok(Self { handle: self.handle.try_clone()? }) } - pub fn into_handle(self) -> Handle { - self.handle - } - - fn reparse_point<'a>( + // NB: returned pointer is derived from `space`, and has provenance to + // match. A raw pointer is returned rather than a reference in order to + // avoid narrowing provenance to the actual `REPARSE_DATA_BUFFER`. + fn reparse_point( &self, - space: &'a mut [u8; c::MAXIMUM_REPARSE_DATA_BUFFER_SIZE], - ) -> io::Result<(c::DWORD, &'a c::REPARSE_DATA_BUFFER)> { + space: &mut Align8<[MaybeUninit]>, + ) -> io::Result<(c::DWORD, *const c::REPARSE_DATA_BUFFER)> { unsafe { let mut bytes = 0; cvt({ + // Grab this in advance to avoid it invalidating the pointer + // we get from `space.0.as_mut_ptr()`. + let len = space.0.len(); c::DeviceIoControl( - self.handle.raw(), + self.handle.as_raw_handle(), c::FSCTL_GET_REPARSE_POINT, ptr::null_mut(), 0, - space.as_mut_ptr() as *mut _, - space.len() as c::DWORD, + space.0.as_mut_ptr().cast(), + len as c::DWORD, &mut bytes, ptr::null_mut(), ) })?; - Ok((bytes, &*(space.as_ptr() as *const c::REPARSE_DATA_BUFFER))) + const _: () = assert!(core::mem::align_of::() <= 8); + Ok((bytes, space.0.as_ptr().cast::())) } } fn readlink(&self) -> io::Result { - let mut space = [0u8; c::MAXIMUM_REPARSE_DATA_BUFFER_SIZE]; + let mut space = Align8([MaybeUninit::::uninit(); c::MAXIMUM_REPARSE_DATA_BUFFER_SIZE]); let (_bytes, buf) = self.reparse_point(&mut space)?; unsafe { - let (path_buffer, subst_off, subst_len, relative) = match buf.ReparseTag { + let (path_buffer, subst_off, subst_len, relative) = match (*buf).ReparseTag { c::IO_REPARSE_TAG_SYMLINK => { let info: *const c::SYMBOLIC_LINK_REPARSE_BUFFER = - &buf.rest as *const _ as *const _; + ptr::addr_of!((*buf).rest).cast(); + assert!(info.is_aligned()); ( - &(*info).PathBuffer as *const _ as *const u16, + ptr::addr_of!((*info).PathBuffer).cast::(), (*info).SubstituteNameOffset / 2, (*info).SubstituteNameLength / 2, (*info).Flags & c::SYMLINK_FLAG_RELATIVE != 0, @@ -494,22 +518,23 @@ impl File { } c::IO_REPARSE_TAG_MOUNT_POINT => { let info: *const c::MOUNT_POINT_REPARSE_BUFFER = - &buf.rest as *const _ as *const _; + ptr::addr_of!((*buf).rest).cast(); + assert!(info.is_aligned()); ( - &(*info).PathBuffer as *const _ as *const u16, + ptr::addr_of!((*info).PathBuffer).cast::(), (*info).SubstituteNameOffset / 2, (*info).SubstituteNameLength / 2, false, ) } _ => { - return Err(io::Error::new( - io::ErrorKind::Other, + return Err(io::const_io_error!( + io::ErrorKind::Uncategorized, "Unsupported reparse point type", )); } }; - let subst_ptr = path_buffer.offset(subst_off as isize); + let subst_ptr = path_buffer.add(subst_off.into()); let mut subst = slice::from_raw_parts(subst_ptr, subst_len as usize); // Absolute paths start with an NT internal namespace prefix `\??\` // We should not let it leak through. @@ -531,7 +556,7 @@ impl File { let size = mem::size_of_val(&info); cvt(unsafe { c::SetFileInformationByHandle( - self.handle.raw(), + self.handle.as_raw_handle(), c::FileBasicInfo, &mut info as *mut _ as *mut _, size as c::DWORD, @@ -539,11 +564,312 @@ impl File { })?; Ok(()) } + + pub fn set_times(&self, times: FileTimes) -> io::Result<()> { + let is_zero = |t: c::FILETIME| t.dwLowDateTime == 0 && t.dwHighDateTime == 0; + if times.accessed.map_or(false, is_zero) || times.modified.map_or(false, is_zero) { + return Err(io::const_io_error!( + io::ErrorKind::InvalidInput, + "Cannot set file timestamp to 0", + )); + } + let is_max = + |t: c::FILETIME| t.dwLowDateTime == c::DWORD::MAX && t.dwHighDateTime == c::DWORD::MAX; + if times.accessed.map_or(false, is_max) || times.modified.map_or(false, is_max) { + return Err(io::const_io_error!( + io::ErrorKind::InvalidInput, + "Cannot set file timestamp to 0xFFFF_FFFF_FFFF_FFFF", + )); + } + cvt(unsafe { + c::SetFileTime(self.as_handle(), None, times.accessed.as_ref(), times.modified.as_ref()) + })?; + Ok(()) + } + + /// Get only basic file information such as attributes and file times. + fn basic_info(&self) -> io::Result { + unsafe { + let mut info: c::FILE_BASIC_INFO = mem::zeroed(); + let size = mem::size_of_val(&info); + cvt(c::GetFileInformationByHandleEx( + self.handle.as_raw_handle(), + c::FileBasicInfo, + &mut info as *mut _ as *mut libc::c_void, + size as c::DWORD, + ))?; + Ok(info) + } + } + /// Delete using POSIX semantics. + /// + /// Files will be deleted as soon as the handle is closed. This is supported + /// for Windows 10 1607 (aka RS1) and later. However some filesystem + /// drivers will not support it even then, e.g. FAT32. + /// + /// If the operation is not supported for this filesystem or OS version + /// then errors will be `ERROR_NOT_SUPPORTED` or `ERROR_INVALID_PARAMETER`. + fn posix_delete(&self) -> io::Result<()> { + let mut info = c::FILE_DISPOSITION_INFO_EX { + Flags: c::FILE_DISPOSITION_DELETE + | c::FILE_DISPOSITION_POSIX_SEMANTICS + | c::FILE_DISPOSITION_IGNORE_READONLY_ATTRIBUTE, + }; + let size = mem::size_of_val(&info); + cvt(unsafe { + c::SetFileInformationByHandle( + self.handle.as_raw_handle(), + c::FileDispositionInfoEx, + &mut info as *mut _ as *mut _, + size as c::DWORD, + ) + })?; + Ok(()) + } + + /// Delete a file using win32 semantics. The file won't actually be deleted + /// until all file handles are closed. However, marking a file for deletion + /// will prevent anyone from opening a new handle to the file. + fn win32_delete(&self) -> io::Result<()> { + let mut info = c::FILE_DISPOSITION_INFO { DeleteFile: c::TRUE as _ }; + let size = mem::size_of_val(&info); + cvt(unsafe { + c::SetFileInformationByHandle( + self.handle.as_raw_handle(), + c::FileDispositionInfo, + &mut info as *mut _ as *mut _, + size as c::DWORD, + ) + })?; + Ok(()) + } + + /// Fill the given buffer with as many directory entries as will fit. + /// This will remember its position and continue from the last call unless + /// `restart` is set to `true`. + /// + /// The returned bool indicates if there are more entries or not. + /// It is an error if `self` is not a directory. + /// + /// # Symlinks and other reparse points + /// + /// On Windows a file is either a directory or a non-directory. + /// A symlink directory is simply an empty directory with some "reparse" metadata attached. + /// So if you open a link (not its target) and iterate the directory, + /// you will always iterate an empty directory regardless of the target. + fn fill_dir_buff(&self, buffer: &mut DirBuff, restart: bool) -> io::Result { + let class = + if restart { c::FileIdBothDirectoryRestartInfo } else { c::FileIdBothDirectoryInfo }; + + unsafe { + let result = cvt(c::GetFileInformationByHandleEx( + self.handle.as_raw_handle(), + class, + buffer.as_mut_ptr().cast(), + buffer.capacity() as _, + )); + match result { + Ok(_) => Ok(true), + Err(e) if e.raw_os_error() == Some(c::ERROR_NO_MORE_FILES as _) => Ok(false), + Err(e) => Err(e), + } + } + } +} + +/// A buffer for holding directory entries. +struct DirBuff { + buffer: Box; Self::BUFFER_SIZE]>>, +} +impl DirBuff { + const BUFFER_SIZE: usize = 1024; + fn new() -> Self { + Self { + // Safety: `Align8<[MaybeUninit; N]>` does not need + // initialization. + buffer: unsafe { Box::new_uninit().assume_init() }, + } + } + fn capacity(&self) -> usize { + self.buffer.0.len() + } + fn as_mut_ptr(&mut self) -> *mut u8 { + self.buffer.0.as_mut_ptr().cast() + } + /// Returns a `DirBuffIter`. + fn iter(&self) -> DirBuffIter<'_> { + DirBuffIter::new(self) + } +} +impl AsRef<[MaybeUninit]> for DirBuff { + fn as_ref(&self) -> &[MaybeUninit] { + &self.buffer.0 + } +} + +/// An iterator over entries stored in a `DirBuff`. +/// +/// Currently only returns file names (UTF-16 encoded). +struct DirBuffIter<'a> { + buffer: Option<&'a [MaybeUninit]>, + cursor: usize, +} +impl<'a> DirBuffIter<'a> { + fn new(buffer: &'a DirBuff) -> Self { + Self { buffer: Some(buffer.as_ref()), cursor: 0 } + } +} +impl<'a> Iterator for DirBuffIter<'a> { + type Item = (Cow<'a, [u16]>, bool); + fn next(&mut self) -> Option { + use crate::mem::size_of; + let buffer = &self.buffer?[self.cursor..]; + + // Get the name and next entry from the buffer. + // SAFETY: + // - The buffer contains a `FILE_ID_BOTH_DIR_INFO` struct but the last + // field (the file name) is unsized. So an offset has to be used to + // get the file name slice. + // - The OS has guaranteed initialization of the fields of + // `FILE_ID_BOTH_DIR_INFO` and the trailing filename (for at least + // `FileNameLength` bytes) + let (name, is_directory, next_entry) = unsafe { + let info = buffer.as_ptr().cast::(); + // While this is guaranteed to be aligned in documentation for + // https://docs.microsoft.com/en-us/windows/win32/api/winbase/ns-winbase-file_id_both_dir_info + // it does not seem that reality is so kind, and assuming this + // caused crashes in some cases (https://github.com/rust-lang/rust/issues/104530) + // presumably, this can be blamed on buggy filesystem drivers, but who knows. + let next_entry = ptr::addr_of!((*info).NextEntryOffset).read_unaligned() as usize; + let length = ptr::addr_of!((*info).FileNameLength).read_unaligned() as usize; + let attrs = ptr::addr_of!((*info).FileAttributes).read_unaligned(); + let name = from_maybe_unaligned( + ptr::addr_of!((*info).FileName).cast::(), + length / size_of::(), + ); + let is_directory = (attrs & c::FILE_ATTRIBUTE_DIRECTORY) != 0; + + (name, is_directory, next_entry) + }; + + if next_entry == 0 { + self.buffer = None + } else { + self.cursor += next_entry + } + + // Skip `.` and `..` pseudo entries. + const DOT: u16 = b'.' as u16; + match &name[..] { + [DOT] | [DOT, DOT] => self.next(), + _ => Some((name, is_directory)), + } + } +} + +unsafe fn from_maybe_unaligned<'a>(p: *const u16, len: usize) -> Cow<'a, [u16]> { + if p.is_aligned() { + Cow::Borrowed(crate::slice::from_raw_parts(p, len)) + } else { + Cow::Owned((0..len).map(|i| p.add(i).read_unaligned()).collect()) + } +} + +/// Open a link relative to the parent directory, ensure no symlinks are followed. +fn open_link_no_reparse(parent: &File, name: &[u16], access: u32) -> io::Result { + // This is implemented using the lower level `NtCreateFile` function as + // unfortunately opening a file relative to a parent is not supported by + // win32 functions. It is however a fundamental feature of the NT kernel. + // + // See https://docs.microsoft.com/en-us/windows/win32/api/winternl/nf-winternl-ntcreatefile + unsafe { + let mut handle = ptr::null_mut(); + let mut io_status = c::IO_STATUS_BLOCK::default(); + let name_str = c::UNICODE_STRING::from_ref(name); + use crate::sync::atomic::{AtomicU32, Ordering}; + // The `OBJ_DONT_REPARSE` attribute ensures that we haven't been + // tricked into following a symlink. However, it may not be available in + // earlier versions of Windows. + static ATTRIBUTES: AtomicU32 = AtomicU32::new(c::OBJ_DONT_REPARSE); + let object = c::OBJECT_ATTRIBUTES { + ObjectName: &name_str, + RootDirectory: parent.as_raw_handle(), + Attributes: ATTRIBUTES.load(Ordering::Relaxed), + ..c::OBJECT_ATTRIBUTES::default() + }; + let status = c::NtCreateFile( + &mut handle, + access, + &object, + &mut io_status, + crate::ptr::null_mut(), + 0, + c::FILE_SHARE_DELETE | c::FILE_SHARE_READ | c::FILE_SHARE_WRITE, + c::FILE_OPEN, + // If `name` is a symlink then open the link rather than the target. + c::FILE_OPEN_REPARSE_POINT, + crate::ptr::null_mut(), + 0, + ); + // Convert an NTSTATUS to the more familiar Win32 error codes (aka "DosError") + if c::nt_success(status) { + Ok(File::from_raw_handle(handle)) + } else if status == c::STATUS_DELETE_PENDING { + // We make a special exception for `STATUS_DELETE_PENDING` because + // otherwise this will be mapped to `ERROR_ACCESS_DENIED` which is + // very unhelpful. + Err(io::Error::from_raw_os_error(c::ERROR_DELETE_PENDING as _)) + } else if status == c::STATUS_INVALID_PARAMETER + && ATTRIBUTES.load(Ordering::Relaxed) == c::OBJ_DONT_REPARSE + { + // Try without `OBJ_DONT_REPARSE`. See above. + ATTRIBUTES.store(0, Ordering::Relaxed); + open_link_no_reparse(parent, name, access) + } else { + Err(io::Error::from_raw_os_error(c::RtlNtStatusToDosError(status) as _)) + } + } +} + +impl AsInner for File { + fn as_inner(&self) -> &Handle { + &self.handle + } +} + +impl IntoInner for File { + fn into_inner(self) -> Handle { + self.handle + } +} + +impl FromInner for File { + fn from_inner(handle: Handle) -> File { + File { handle } + } } -impl FromInner for File { - fn from_inner(handle: c::HANDLE) -> File { - File { handle: Handle::new(handle) } +impl AsHandle for File { + fn as_handle(&self) -> BorrowedHandle<'_> { + self.as_inner().as_handle() + } +} + +impl AsRawHandle for File { + fn as_raw_handle(&self) -> RawHandle { + self.as_inner().as_raw_handle() + } +} + +impl IntoRawHandle for File { + fn into_raw_handle(self) -> RawHandle { + self.into_inner().into_raw_handle() + } +} + +impl FromRawHandle for File { + unsafe fn from_raw_handle(raw_handle: RawHandle) -> Self { + Self { handle: FromInner::from_inner(FromRawHandle::from_raw_handle(raw_handle)) } } } @@ -551,7 +877,7 @@ impl fmt::Debug for File { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // FIXME(#24570): add more info here (e.g., mode) let mut b = f.debug_struct("File"); - b.field("handle", &self.handle.raw()); + b.field("handle", &self.handle.as_raw_handle()); if let Ok(path) = get_path(&self) { b.field("path", &path); } @@ -612,6 +938,26 @@ impl FileAttr { self.file_index } } +impl From for FileAttr { + fn from(wfd: c::WIN32_FIND_DATAW) -> Self { + FileAttr { + attributes: wfd.dwFileAttributes, + creation_time: wfd.ftCreationTime, + last_access_time: wfd.ftLastAccessTime, + last_write_time: wfd.ftLastWriteTime, + file_size: ((wfd.nFileSizeHigh as u64) << 32) | (wfd.nFileSizeLow as u64), + reparse_tag: if wfd.dwFileAttributes & c::FILE_ATTRIBUTE_REPARSE_POINT != 0 { + // reserved unless this is a reparse point + wfd.dwReserved0 + } else { + 0 + }, + volume_serial_number: None, + number_of_links: None, + file_index: None, + } + } +} fn to_u64(ft: &c::FILETIME) -> u64 { (ft.dwLowDateTime as u64) | ((ft.dwHighDateTime as u64) << 32) @@ -631,9 +977,19 @@ impl FilePermissions { } } +impl FileTimes { + pub fn set_accessed(&mut self, t: SystemTime) { + self.accessed = Some(t.into_inner()); + } + + pub fn set_modified(&mut self, t: SystemTime) { + self.modified = Some(t.into_inner()); + } +} + impl FileType { fn new(attrs: c::DWORD, reparse_tag: c::DWORD) -> FileType { - FileType { attributes: attrs, reparse_tag: reparse_tag } + FileType { attributes: attrs, reparse_tag } } pub fn is_dir(&self) -> bool { !self.is_symlink() && self.is_directory() @@ -667,7 +1023,7 @@ impl DirBuilder { } pub fn mkdir(&self, p: &Path) -> io::Result<()> { - let p = to_u16s(p)?; + let p = maybe_verbatim(p)?; cvt(unsafe { c::CreateDirectoryW(p.as_ptr(), ptr::null_mut()) })?; Ok(()) } @@ -676,7 +1032,7 @@ impl DirBuilder { pub fn readdir(p: &Path) -> io::Result { let root = p.to_path_buf(); let star = p.join("*"); - let path = to_u16s(&star)?; + let path = maybe_verbatim(&star)?; unsafe { let mut wfd = mem::zeroed(); @@ -694,48 +1050,129 @@ pub fn readdir(p: &Path) -> io::Result { } pub fn unlink(p: &Path) -> io::Result<()> { - let p_u16s = to_u16s(p)?; + let p_u16s = maybe_verbatim(p)?; cvt(unsafe { c::DeleteFileW(p_u16s.as_ptr()) })?; Ok(()) } pub fn rename(old: &Path, new: &Path) -> io::Result<()> { - let old = to_u16s(old)?; - let new = to_u16s(new)?; + let old = maybe_verbatim(old)?; + let new = maybe_verbatim(new)?; cvt(unsafe { c::MoveFileExW(old.as_ptr(), new.as_ptr(), c::MOVEFILE_REPLACE_EXISTING) })?; Ok(()) } pub fn rmdir(p: &Path) -> io::Result<()> { - let p = to_u16s(p)?; + let p = maybe_verbatim(p)?; cvt(unsafe { c::RemoveDirectoryW(p.as_ptr()) })?; Ok(()) } +/// Open a file or directory without following symlinks. +fn open_link(path: &Path, access_mode: u32) -> io::Result { + let mut opts = OpenOptions::new(); + opts.access_mode(access_mode); + // `FILE_FLAG_BACKUP_SEMANTICS` allows opening directories. + // `FILE_FLAG_OPEN_REPARSE_POINT` opens a link instead of its target. + opts.custom_flags(c::FILE_FLAG_BACKUP_SEMANTICS | c::FILE_FLAG_OPEN_REPARSE_POINT); + File::open(path, &opts) +} + pub fn remove_dir_all(path: &Path) -> io::Result<()> { - let filetype = lstat(path)?.file_type(); - if filetype.is_symlink() { - // On Windows symlinks to files and directories are removed differently. - // rmdir only deletes dir symlinks and junctions, not file symlinks. - rmdir(path) - } else { - remove_dir_all_recursive(path) + let file = open_link(path, c::DELETE | c::FILE_LIST_DIRECTORY)?; + + // Test if the file is not a directory or a symlink to a directory. + if (file.basic_info()?.FileAttributes & c::FILE_ATTRIBUTE_DIRECTORY) == 0 { + return Err(io::Error::from_raw_os_error(c::ERROR_DIRECTORY as _)); + } + + match remove_dir_all_iterative(&file, File::posix_delete) { + Err(e) => { + if let Some(code) = e.raw_os_error() { + match code as u32 { + // If POSIX delete is not supported for this filesystem then fallback to win32 delete. + c::ERROR_NOT_SUPPORTED + | c::ERROR_INVALID_FUNCTION + | c::ERROR_INVALID_PARAMETER => { + remove_dir_all_iterative(&file, File::win32_delete) + } + _ => Err(e), + } + } else { + Err(e) + } + } + ok => ok, } } -fn remove_dir_all_recursive(path: &Path) -> io::Result<()> { - for child in readdir(path)? { - let child = child?; - let child_type = child.file_type()?; - if child_type.is_dir() { - remove_dir_all_recursive(&child.path())?; - } else if child_type.is_symlink_dir() { - rmdir(&child.path())?; - } else { - unlink(&child.path())?; +fn remove_dir_all_iterative(f: &File, delete: fn(&File) -> io::Result<()>) -> io::Result<()> { + // When deleting files we may loop this many times when certain error conditions occur. + // This allows remove_dir_all to succeed when the error is temporary. + const MAX_RETRIES: u32 = 10; + + let mut buffer = DirBuff::new(); + let mut dirlist = vec![f.duplicate()?]; + + // FIXME: This is a hack so we can push to the dirlist vec after borrowing from it. + fn copy_handle(f: &File) -> mem::ManuallyDrop { + unsafe { mem::ManuallyDrop::new(File::from_raw_handle(f.as_raw_handle())) } + } + + let mut restart = true; + while let Some(dir) = dirlist.last() { + let dir = copy_handle(dir); + + // Fill the buffer and iterate the entries. + let more_data = dir.fill_dir_buff(&mut buffer, restart)?; + restart = false; + for (name, is_directory) in buffer.iter() { + if is_directory { + let child_dir = open_link_no_reparse( + &dir, + &name, + c::SYNCHRONIZE | c::DELETE | c::FILE_LIST_DIRECTORY, + )?; + dirlist.push(child_dir); + } else { + for i in 1..=MAX_RETRIES { + let result = open_link_no_reparse(&dir, &name, c::SYNCHRONIZE | c::DELETE); + match result { + Ok(f) => delete(&f)?, + // Already deleted, so skip. + Err(e) if e.kind() == io::ErrorKind::NotFound => break, + // Retry a few times if the file is locked or a delete is already in progress. + Err(e) + if i < MAX_RETRIES + && (e.raw_os_error() == Some(c::ERROR_DELETE_PENDING as _) + || e.raw_os_error() + == Some(c::ERROR_SHARING_VIOLATION as _)) => {} + // Otherwise return the error. + Err(e) => return Err(e), + } + thread::yield_now(); + } + } + } + // If there were no more files then delete the directory. + if !more_data { + if let Some(dir) = dirlist.pop() { + // Retry deleting a few times in case we need to wait for a file to be deleted. + for i in 1..=MAX_RETRIES { + let result = delete(&dir); + if let Err(e) = result { + if i == MAX_RETRIES || e.kind() != io::ErrorKind::DirectoryNotEmpty { + return Err(e); + } + thread::yield_now(); + } else { + break; + } + } + } } } - rmdir(path) + Ok(()) } pub fn readlink(path: &Path) -> io::Result { @@ -749,13 +1186,13 @@ pub fn readlink(path: &Path) -> io::Result { file.readlink() } -pub fn symlink(src: &Path, dst: &Path) -> io::Result<()> { - symlink_inner(src, dst, false) +pub fn symlink(original: &Path, link: &Path) -> io::Result<()> { + symlink_inner(original, link, false) } -pub fn symlink_inner(src: &Path, dst: &Path, dir: bool) -> io::Result<()> { - let src = to_u16s(src)?; - let dst = to_u16s(dst)?; +pub fn symlink_inner(original: &Path, link: &Path, dir: bool) -> io::Result<()> { + let original = to_u16s(original)?; + let link = maybe_verbatim(link)?; let flags = if dir { c::SYMBOLIC_LINK_FLAG_DIRECTORY } else { 0 }; // Formerly, symlink creation required the SeCreateSymbolicLink privilege. For the Windows 10 // Creators Update, Microsoft loosened this to allow unprivileged symlink creation if the @@ -763,8 +1200,8 @@ pub fn symlink_inner(src: &Path, dst: &Path, dir: bool) -> io::Result<()> { // added to dwFlags to opt into this behaviour. let result = cvt(unsafe { c::CreateSymbolicLinkW( - dst.as_ptr(), - src.as_ptr(), + link.as_ptr(), + original.as_ptr(), flags | c::SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE, ) as c::BOOL }); @@ -772,7 +1209,9 @@ pub fn symlink_inner(src: &Path, dst: &Path, dir: bool) -> io::Result<()> { if err.raw_os_error() == Some(c::ERROR_INVALID_PARAMETER as i32) { // Older Windows objects to SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE, // so if we encounter ERROR_INVALID_PARAMETER, retry without that flag. - cvt(unsafe { c::CreateSymbolicLinkW(dst.as_ptr(), src.as_ptr(), flags) as c::BOOL })?; + cvt(unsafe { + c::CreateSymbolicLinkW(link.as_ptr(), original.as_ptr(), flags) as c::BOOL + })?; } else { return Err(err); } @@ -781,39 +1220,93 @@ pub fn symlink_inner(src: &Path, dst: &Path, dir: bool) -> io::Result<()> { } #[cfg(not(target_vendor = "uwp"))] -pub fn link(src: &Path, dst: &Path) -> io::Result<()> { - let src = to_u16s(src)?; - let dst = to_u16s(dst)?; - cvt(unsafe { c::CreateHardLinkW(dst.as_ptr(), src.as_ptr(), ptr::null_mut()) })?; +pub fn link(original: &Path, link: &Path) -> io::Result<()> { + let original = maybe_verbatim(original)?; + let link = maybe_verbatim(link)?; + cvt(unsafe { c::CreateHardLinkW(link.as_ptr(), original.as_ptr(), ptr::null_mut()) })?; Ok(()) } #[cfg(target_vendor = "uwp")] -pub fn link(_src: &Path, _dst: &Path) -> io::Result<()> { - return Err(io::Error::new(io::ErrorKind::Other, "hard link are not supported on UWP")); +pub fn link(_original: &Path, _link: &Path) -> io::Result<()> { + return Err(io::const_io_error!( + io::ErrorKind::Unsupported, + "hard link are not supported on UWP", + )); } pub fn stat(path: &Path) -> io::Result { - let mut opts = OpenOptions::new(); - // No read or write permissions are necessary - opts.access_mode(0); - // This flag is so we can open directories too - opts.custom_flags(c::FILE_FLAG_BACKUP_SEMANTICS); - let file = File::open(path, &opts)?; - file.file_attr() + metadata(path, ReparsePoint::Follow) } pub fn lstat(path: &Path) -> io::Result { + metadata(path, ReparsePoint::Open) +} + +#[repr(u32)] +#[derive(Clone, Copy, PartialEq, Eq)] +enum ReparsePoint { + Follow = 0, + Open = c::FILE_FLAG_OPEN_REPARSE_POINT, +} +impl ReparsePoint { + fn as_flag(self) -> u32 { + self as u32 + } +} + +fn metadata(path: &Path, reparse: ReparsePoint) -> io::Result { let mut opts = OpenOptions::new(); // No read or write permissions are necessary opts.access_mode(0); - opts.custom_flags(c::FILE_FLAG_BACKUP_SEMANTICS | c::FILE_FLAG_OPEN_REPARSE_POINT); - let file = File::open(path, &opts)?; - file.file_attr() + opts.custom_flags(c::FILE_FLAG_BACKUP_SEMANTICS | reparse.as_flag()); + + // Attempt to open the file normally. + // If that fails with `ERROR_SHARING_VIOLATION` then retry using `FindFirstFileW`. + // If the fallback fails for any reason we return the original error. + match File::open(path, &opts) { + Ok(file) => file.file_attr(), + Err(e) if e.raw_os_error() == Some(c::ERROR_SHARING_VIOLATION as _) => { + // `ERROR_SHARING_VIOLATION` will almost never be returned. + // Usually if a file is locked you can still read some metadata. + // However, there are special system files, such as + // `C:\hiberfil.sys`, that are locked in a way that denies even that. + unsafe { + let path = maybe_verbatim(path)?; + + // `FindFirstFileW` accepts wildcard file names. + // Fortunately wildcards are not valid file names and + // `ERROR_SHARING_VIOLATION` means the file exists (but is locked) + // therefore it's safe to assume the file name given does not + // include wildcards. + let mut wfd = mem::zeroed(); + let handle = c::FindFirstFileW(path.as_ptr(), &mut wfd); + + if handle == c::INVALID_HANDLE_VALUE { + // This can fail if the user does not have read access to the + // directory. + Err(e) + } else { + // We no longer need the find handle. + c::FindClose(handle); + + // `FindFirstFileW` reads the cached file information from the + // directory. The downside is that this metadata may be outdated. + let attrs = FileAttr::from(wfd); + if reparse == ReparsePoint::Follow && attrs.file_type().is_symlink() { + Err(e) + } else { + Ok(attrs) + } + } + } + } + Err(e) => Err(e), + } } pub fn set_perm(p: &Path, perm: FilePermissions) -> io::Result<()> { - let p = to_u16s(p)?; + let p = maybe_verbatim(p)?; unsafe { cvt(c::SetFileAttributesW(p.as_ptr(), perm.attrs))?; Ok(()) @@ -823,7 +1316,7 @@ pub fn set_perm(p: &Path, perm: FilePermissions) -> io::Result<()> { fn get_path(f: &File) -> io::Result { super::fill_utf16_buf( |buf, sz| unsafe { - c::GetFinalPathNameByHandleW(f.handle.raw(), buf, sz, c::VOLUME_NAME_DOS) + c::GetFinalPathNameByHandleW(f.handle.as_raw_handle(), buf, sz, c::VOLUME_NAME_DOS) }, |buf| PathBuf::from(OsString::from_wide(buf)), ) @@ -856,8 +1349,8 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { } c::PROGRESS_CONTINUE } - let pfrom = to_u16s(from)?; - let pto = to_u16s(to)?; + let pfrom = maybe_verbatim(from)?; + let pto = maybe_verbatim(to)?; let mut size = 0i64; cvt(unsafe { c::CopyFileExW( @@ -873,8 +1366,11 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { } #[allow(dead_code)] -pub fn symlink_junction, Q: AsRef>(src: P, dst: Q) -> io::Result<()> { - symlink_junction_inner(src.as_ref(), dst.as_ref()) +pub fn symlink_junction, Q: AsRef>( + original: P, + junction: Q, +) -> io::Result<()> { + symlink_junction_inner(original.as_ref(), junction.as_ref()) } // Creating a directory junction on windows involves dealing with reparse @@ -883,7 +1379,7 @@ pub fn symlink_junction, Q: AsRef>(src: P, dst: Q) -> io::R // // http://www.flexhex.com/docs/articles/hard-links.phtml #[allow(dead_code)] -fn symlink_junction_inner(target: &Path, junction: &Path) -> io::Result<()> { +fn symlink_junction_inner(original: &Path, junction: &Path) -> io::Result<()> { let d = DirBuilder::new(); d.mkdir(&junction)?; @@ -891,21 +1387,22 @@ fn symlink_junction_inner(target: &Path, junction: &Path) -> io::Result<()> { opts.write(true); opts.custom_flags(c::FILE_FLAG_OPEN_REPARSE_POINT | c::FILE_FLAG_BACKUP_SEMANTICS); let f = File::open(junction, &opts)?; - let h = f.handle().raw(); + let h = f.as_inner().as_raw_handle(); unsafe { - let mut data = [0u8; c::MAXIMUM_REPARSE_DATA_BUFFER_SIZE]; - let db = data.as_mut_ptr() as *mut c::REPARSE_MOUNTPOINT_DATA_BUFFER; - let buf = &mut (*db).ReparseTarget as *mut c::WCHAR; + let mut data = Align8([MaybeUninit::::uninit(); c::MAXIMUM_REPARSE_DATA_BUFFER_SIZE]); + let data_ptr = data.0.as_mut_ptr(); + let db = data_ptr.cast::(); + let buf = ptr::addr_of_mut!((*db).ReparseTarget).cast::(); let mut i = 0; // FIXME: this conversion is very hacky let v = br"\??\"; let v = v.iter().map(|x| *x as u16); - for c in v.chain(target.as_os_str().encode_wide()) { - *buf.offset(i) = c; + for c in v.chain(original.as_os_str().encode_wide()) { + *buf.add(i) = c; i += 1; } - *buf.offset(i) = 0; + *buf.add(i) = 0; i += 1; (*db).ReparseTag = c::IO_REPARSE_TAG_MOUNT_POINT; (*db).ReparseTargetMaximumLength = (i * 2) as c::WORD; @@ -916,7 +1413,7 @@ fn symlink_junction_inner(target: &Path, junction: &Path) -> io::Result<()> { cvt(c::DeviceIoControl( h as *mut _, c::FSCTL_SET_REPARSE_POINT, - data.as_ptr() as *mut _, + data_ptr.cast(), (*db).ReparseDataLength + 8, ptr::null_mut(), 0, @@ -926,3 +1423,31 @@ fn symlink_junction_inner(target: &Path, junction: &Path) -> io::Result<()> { .map(drop) } } + +// Try to see if a file exists but, unlike `exists`, report I/O errors. +pub fn try_exists(path: &Path) -> io::Result { + // Open the file to ensure any symlinks are followed to their target. + let mut opts = OpenOptions::new(); + // No read, write, etc access rights are needed. + opts.access_mode(0); + // Backup semantics enables opening directories as well as files. + opts.custom_flags(c::FILE_FLAG_BACKUP_SEMANTICS); + match File::open(path, &opts) { + Err(e) => match e.kind() { + // The file definitely does not exist + io::ErrorKind::NotFound => Ok(false), + + // `ERROR_SHARING_VIOLATION` means that the file has been locked by + // another process. This is often temporary so we simply report it + // as the file existing. + _ if e.raw_os_error() == Some(c::ERROR_SHARING_VIOLATION as i32) => Ok(true), + + // Other errors such as `ERROR_ACCESS_DENIED` may indicate that the + // file exists. However, these types of errors are usually more + // permanent so we report them here. + _ => Err(e), + }, + // The file was opened successfully therefore it must exist, + Ok(_) => Ok(true), + } +} diff --git a/crux-mir/lib/std/src/sys/windows/handle.rs b/crux-mir/lib/std/src/sys/windows/handle.rs index f2ad057b6..ae33d48c6 100644 --- a/crux-mir/lib/std/src/sys/windows/handle.rs +++ b/crux-mir/lib/std/src/sys/windows/handle.rs @@ -1,82 +1,86 @@ #![unstable(issue = "none", feature = "windows_handle")] +#[cfg(test)] +mod tests; + use crate::cmp; -use crate::io::{self, ErrorKind, IoSlice, IoSliceMut, Read}; +use crate::io::{self, BorrowedCursor, ErrorKind, IoSlice, IoSliceMut, Read}; use crate::mem; -use crate::ops::Deref; +use crate::os::windows::io::{ + AsHandle, AsRawHandle, BorrowedHandle, FromRawHandle, IntoRawHandle, OwnedHandle, RawHandle, +}; use crate::ptr; use crate::sys::c; use crate::sys::cvt; +use crate::sys_common::{AsInner, FromInner, IntoInner}; /// An owned container for `HANDLE` object, closing them on Drop. /// /// All methods are inherited through a `Deref` impl to `RawHandle` -pub struct Handle(RawHandle); - -/// A wrapper type for `HANDLE` objects to give them proper Send/Sync inference -/// as well as Rust-y methods. -/// -/// This does **not** drop the handle when it goes out of scope, use `Handle` -/// instead for that. -#[derive(Copy, Clone)] -pub struct RawHandle(c::HANDLE); - -unsafe impl Send for RawHandle {} -unsafe impl Sync for RawHandle {} +pub struct Handle(OwnedHandle); impl Handle { - pub fn new(handle: c::HANDLE) -> Handle { - Handle(RawHandle::new(handle)) - } - pub fn new_event(manual: bool, init: bool) -> io::Result { unsafe { let event = c::CreateEventW(ptr::null_mut(), manual as c::BOOL, init as c::BOOL, ptr::null()); - if event.is_null() { Err(io::Error::last_os_error()) } else { Ok(Handle::new(event)) } + if event.is_null() { + Err(io::Error::last_os_error()) + } else { + Ok(Handle::from_raw_handle(event)) + } } } +} - pub fn into_raw(self) -> c::HANDLE { - let ret = self.raw(); - mem::forget(self); - ret +impl AsInner for Handle { + fn as_inner(&self) -> &OwnedHandle { + &self.0 } } -impl Deref for Handle { - type Target = RawHandle; - fn deref(&self) -> &RawHandle { - &self.0 +impl IntoInner for Handle { + fn into_inner(self) -> OwnedHandle { + self.0 } } -impl Drop for Handle { - fn drop(&mut self) { - unsafe { - let _ = c::CloseHandle(self.raw()); - } +impl FromInner for Handle { + fn from_inner(file_desc: OwnedHandle) -> Self { + Self(file_desc) } } -impl RawHandle { - pub fn new(handle: c::HANDLE) -> RawHandle { - RawHandle(handle) +impl AsHandle for Handle { + fn as_handle(&self) -> BorrowedHandle<'_> { + self.0.as_handle() } +} - pub fn raw(&self) -> c::HANDLE { - self.0 +impl AsRawHandle for Handle { + fn as_raw_handle(&self) -> RawHandle { + self.0.as_raw_handle() } +} + +impl IntoRawHandle for Handle { + fn into_raw_handle(self) -> RawHandle { + self.0.into_raw_handle() + } +} +impl FromRawHandle for Handle { + unsafe fn from_raw_handle(raw_handle: RawHandle) -> Self { + Self(FromRawHandle::from_raw_handle(raw_handle)) + } +} + +impl Handle { pub fn read(&self, buf: &mut [u8]) -> io::Result { - let mut read = 0; - let len = cmp::min(buf.len(), ::max_value() as usize) as c::DWORD; - let res = cvt(unsafe { - c::ReadFile(self.0, buf.as_mut_ptr() as c::LPVOID, len, &mut read, ptr::null_mut()) - }); + let res = unsafe { self.synchronous_read(buf.as_mut_ptr().cast(), buf.len(), None) }; match res { - Ok(_) => Ok(read as usize), + Ok(read) => Ok(read as usize), // The special treatment of BrokenPipe is to deal with Windows // pipe semantics, which yields this error when *reading* from @@ -92,31 +96,59 @@ impl RawHandle { crate::io::default_read_vectored(|buf| self.read(buf), bufs) } + #[inline] + pub fn is_read_vectored(&self) -> bool { + false + } + pub fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result { - let mut read = 0; - let len = cmp::min(buf.len(), ::max_value() as usize) as c::DWORD; - let res = unsafe { - let mut overlapped: c::OVERLAPPED = mem::zeroed(); - overlapped.Offset = offset as u32; - overlapped.OffsetHigh = (offset >> 32) as u32; - cvt(c::ReadFile(self.0, buf.as_mut_ptr() as c::LPVOID, len, &mut read, &mut overlapped)) - }; + let res = + unsafe { self.synchronous_read(buf.as_mut_ptr().cast(), buf.len(), Some(offset)) }; + match res { - Ok(_) => Ok(read as usize), + Ok(read) => Ok(read as usize), Err(ref e) if e.raw_os_error() == Some(c::ERROR_HANDLE_EOF as i32) => Ok(0), Err(e) => Err(e), } } + pub fn read_buf(&self, mut cursor: BorrowedCursor<'_>) -> io::Result<()> { + let res = + unsafe { self.synchronous_read(cursor.as_mut().as_mut_ptr(), cursor.capacity(), None) }; + + match res { + Ok(read) => { + // Safety: `read` bytes were written to the initialized portion of the buffer + unsafe { + cursor.advance(read as usize); + } + Ok(()) + } + + // The special treatment of BrokenPipe is to deal with Windows + // pipe semantics, which yields this error when *reading* from + // a pipe after the other end has closed; we interpret that as + // EOF on the pipe. + Err(ref e) if e.kind() == ErrorKind::BrokenPipe => Ok(()), + + Err(e) => Err(e), + } + } + pub unsafe fn read_overlapped( &self, buf: &mut [u8], overlapped: *mut c::OVERLAPPED, ) -> io::Result> { - let len = cmp::min(buf.len(), ::max_value() as usize) as c::DWORD; + let len = cmp::min(buf.len(), ::MAX as usize) as c::DWORD; let mut amt = 0; - let res = - cvt({ c::ReadFile(self.0, buf.as_ptr() as c::LPVOID, len, &mut amt, overlapped) }); + let res = cvt(c::ReadFile( + self.as_handle(), + buf.as_ptr() as c::LPVOID, + len, + &mut amt, + overlapped, + )); match res { Ok(_) => Ok(Some(amt as usize)), Err(e) => { @@ -139,7 +171,8 @@ impl RawHandle { unsafe { let mut bytes = 0; let wait = if wait { c::TRUE } else { c::FALSE }; - let res = cvt({ c::GetOverlappedResult(self.raw(), overlapped, &mut bytes, wait) }); + let res = + cvt(c::GetOverlappedResult(self.as_raw_handle(), overlapped, &mut bytes, wait)); match res { Ok(_) => Ok(bytes as usize), Err(e) => { @@ -156,38 +189,28 @@ impl RawHandle { } pub fn cancel_io(&self) -> io::Result<()> { - unsafe { cvt(c::CancelIo(self.raw())).map(drop) } + unsafe { cvt(c::CancelIo(self.as_raw_handle())).map(drop) } } pub fn write(&self, buf: &[u8]) -> io::Result { - let mut amt = 0; - let len = cmp::min(buf.len(), ::max_value() as usize) as c::DWORD; - cvt(unsafe { - c::WriteFile(self.0, buf.as_ptr() as c::LPVOID, len, &mut amt, ptr::null_mut()) - })?; - Ok(amt as usize) + self.synchronous_write(&buf, None) } pub fn write_vectored(&self, bufs: &[IoSlice<'_>]) -> io::Result { crate::io::default_write_vectored(|buf| self.write(buf), bufs) } + #[inline] + pub fn is_write_vectored(&self) -> bool { + false + } + pub fn write_at(&self, buf: &[u8], offset: u64) -> io::Result { - let mut written = 0; - let len = cmp::min(buf.len(), ::max_value() as usize) as c::DWORD; - unsafe { - let mut overlapped: c::OVERLAPPED = mem::zeroed(); - overlapped.Offset = offset as u32; - overlapped.OffsetHigh = (offset >> 32) as u32; - cvt(c::WriteFile( - self.0, - buf.as_ptr() as c::LPVOID, - len, - &mut written, - &mut overlapped, - ))?; - } - Ok(written as usize) + self.synchronous_write(&buf, Some(offset)) + } + + pub fn try_clone(&self) -> io::Result { + Ok(Self(self.0.try_clone()?)) } pub fn duplicate( @@ -195,25 +218,111 @@ impl RawHandle { access: c::DWORD, inherit: bool, options: c::DWORD, - ) -> io::Result { - let mut ret = 0 as c::HANDLE; - cvt(unsafe { - let cur_proc = c::GetCurrentProcess(); - c::DuplicateHandle( - cur_proc, - self.0, - cur_proc, - &mut ret, - access, - inherit as c::BOOL, - options, + ) -> io::Result { + Ok(Self(self.0.as_handle().duplicate(access, inherit, options)?)) + } + + /// Performs a synchronous read. + /// + /// If the handle is opened for asynchronous I/O then this abort the process. + /// See #81357. + /// + /// If `offset` is `None` then the current file position is used. + unsafe fn synchronous_read( + &self, + buf: *mut mem::MaybeUninit, + len: usize, + offset: Option, + ) -> io::Result { + let mut io_status = c::IO_STATUS_BLOCK::default(); + + // The length is clamped at u32::MAX. + let len = cmp::min(len, c::DWORD::MAX as usize) as c::DWORD; + let status = c::NtReadFile( + self.as_handle(), + ptr::null_mut(), + None, + ptr::null_mut(), + &mut io_status, + buf, + len, + offset.map(|n| n as _).as_ref(), + None, + ); + + let status = if status == c::STATUS_PENDING { + c::WaitForSingleObject(self.as_raw_handle(), c::INFINITE); + io_status.status() + } else { + status + }; + match status { + // If the operation has not completed then abort the process. + // Doing otherwise means that the buffer and stack may be written to + // after this function returns. + c::STATUS_PENDING => rtabort!("I/O error: operation failed to complete synchronously"), + + // Return `Ok(0)` when there's nothing more to read. + c::STATUS_END_OF_FILE => Ok(0), + + // Success! + status if c::nt_success(status) => Ok(io_status.Information), + + status => { + let error = c::RtlNtStatusToDosError(status); + Err(io::Error::from_raw_os_error(error as _)) + } + } + } + + /// Performs a synchronous write. + /// + /// If the handle is opened for asynchronous I/O then this abort the process. + /// See #81357. + /// + /// If `offset` is `None` then the current file position is used. + fn synchronous_write(&self, buf: &[u8], offset: Option) -> io::Result { + let mut io_status = c::IO_STATUS_BLOCK::default(); + + // The length is clamped at u32::MAX. + let len = cmp::min(buf.len(), c::DWORD::MAX as usize) as c::DWORD; + let status = unsafe { + c::NtWriteFile( + self.as_handle(), + ptr::null_mut(), + None, + ptr::null_mut(), + &mut io_status, + buf.as_ptr(), + len, + offset.map(|n| n as _).as_ref(), + None, ) - })?; - Ok(Handle::new(ret)) + }; + let status = if status == c::STATUS_PENDING { + unsafe { c::WaitForSingleObject(self.as_raw_handle(), c::INFINITE) }; + io_status.status() + } else { + status + }; + match status { + // If the operation has not completed then abort the process. + // Doing otherwise means that the buffer may be read and the stack + // written to after this function returns. + c::STATUS_PENDING => rtabort!("I/O error: operation failed to complete synchronously"), + + // Success! + status if c::nt_success(status) => Ok(io_status.Information), + + status => { + let error = unsafe { c::RtlNtStatusToDosError(status) }; + Err(io::Error::from_raw_os_error(error as _)) + } + } } } -impl<'a> Read for &'a RawHandle { +impl<'a> Read for &'a Handle { fn read(&mut self, buf: &mut [u8]) -> io::Result { (**self).read(buf) } diff --git a/crux-mir/lib/std/src/sys/windows/handle/tests.rs b/crux-mir/lib/std/src/sys/windows/handle/tests.rs new file mode 100644 index 000000000..d836dae4c --- /dev/null +++ b/crux-mir/lib/std/src/sys/windows/handle/tests.rs @@ -0,0 +1,22 @@ +use crate::sys::pipe::{anon_pipe, Pipes}; +use crate::{thread, time}; + +/// Test the synchronous fallback for overlapped I/O. +#[test] +fn overlapped_handle_fallback() { + // Create some pipes. `ours` will be asynchronous. + let Pipes { ours, theirs } = anon_pipe(true, false).unwrap(); + + let async_readable = ours.into_handle(); + let sync_writeable = theirs.into_handle(); + + thread::scope(|_| { + thread::sleep(time::Duration::from_millis(100)); + sync_writeable.write(b"hello world!").unwrap(); + }); + + // The pipe buffer starts empty so reading won't complete synchronously unless + // our fallback path works. + let mut buffer = [0u8; 1024]; + async_readable.read(&mut buffer).unwrap(); +} diff --git a/crux-mir/lib/std/src/sys/windows/io.rs b/crux-mir/lib/std/src/sys/windows/io.rs index 5525d2832..2cc34c986 100644 --- a/crux-mir/lib/std/src/sys/windows/io.rs +++ b/crux-mir/lib/std/src/sys/windows/io.rs @@ -1,6 +1,10 @@ use crate::marker::PhantomData; +use crate::mem::size_of; +use crate::os::windows::io::{AsHandle, AsRawHandle, BorrowedHandle}; use crate::slice; -use crate::sys::c; +use crate::sys::{c, Align8}; +use core; +use libc; #[derive(Copy, Clone)] #[repr(transparent)] @@ -12,7 +16,7 @@ pub struct IoSlice<'a> { impl<'a> IoSlice<'a> { #[inline] pub fn new(buf: &'a [u8]) -> IoSlice<'a> { - assert!(buf.len() <= c::ULONG::max_value() as usize); + assert!(buf.len() <= c::ULONG::MAX as usize); IoSlice { vec: c::WSABUF { len: buf.len() as c::ULONG, @@ -49,7 +53,7 @@ pub struct IoSliceMut<'a> { impl<'a> IoSliceMut<'a> { #[inline] pub fn new(buf: &'a mut [u8]) -> IoSliceMut<'a> { - assert!(buf.len() <= c::ULONG::max_value() as usize); + assert!(buf.len() <= c::ULONG::MAX as usize); IoSliceMut { vec: c::WSABUF { len: buf.len() as c::ULONG, buf: buf.as_mut_ptr() as *mut c::CHAR }, _p: PhantomData, @@ -78,3 +82,73 @@ impl<'a> IoSliceMut<'a> { unsafe { slice::from_raw_parts_mut(self.vec.buf as *mut u8, self.vec.len as usize) } } } + +pub fn is_terminal(h: &impl AsHandle) -> bool { + unsafe { handle_is_console(h.as_handle()) } +} + +unsafe fn handle_is_console(handle: BorrowedHandle<'_>) -> bool { + let handle = handle.as_raw_handle(); + + // A null handle means the process has no console. + if handle.is_null() { + return false; + } + + let mut out = 0; + if c::GetConsoleMode(handle, &mut out) != 0 { + // False positives aren't possible. If we got a console then we definitely have a console. + return true; + } + + // At this point, we *could* have a false negative. We can determine that this is a true + // negative if we can detect the presence of a console on any of the standard I/O streams. If + // another stream has a console, then we know we're in a Windows console and can therefore + // trust the negative. + for std_handle in [c::STD_INPUT_HANDLE, c::STD_OUTPUT_HANDLE, c::STD_ERROR_HANDLE] { + let std_handle = c::GetStdHandle(std_handle); + if !std_handle.is_null() + && std_handle != handle + && c::GetConsoleMode(std_handle, &mut out) != 0 + { + return false; + } + } + + // Otherwise, we fall back to an msys hack to see if we can detect the presence of a pty. + msys_tty_on(handle) +} + +unsafe fn msys_tty_on(handle: c::HANDLE) -> bool { + // Early return if the handle is not a pipe. + if c::GetFileType(handle) != c::FILE_TYPE_PIPE { + return false; + } + + const SIZE: usize = size_of::() + c::MAX_PATH * size_of::(); + let mut name_info_bytes = Align8([0u8; SIZE]); + let res = c::GetFileInformationByHandleEx( + handle, + c::FileNameInfo, + name_info_bytes.0.as_mut_ptr() as *mut libc::c_void, + SIZE as u32, + ); + if res == 0 { + return false; + } + let name_info: &c::FILE_NAME_INFO = &*(name_info_bytes.0.as_ptr() as *const c::FILE_NAME_INFO); + let name_len = name_info.FileNameLength as usize / 2; + // Offset to get the `FileName` field. + let name_ptr = name_info_bytes.0.as_ptr().offset(size_of::() as isize).cast::(); + let s = core::slice::from_raw_parts(name_ptr, name_len); + let name = String::from_utf16_lossy(s); + // Get the file name only. + let name = name.rsplit('\\').next().unwrap_or(&name); + // This checks whether 'pty' exists in the file name, which indicates that + // a pseudo-terminal is attached. To mitigate against false positives + // (e.g., an actual file name that contains 'pty'), we also require that + // the file name begins with either the strings 'msys-' or 'cygwin-'.) + let is_msys = name.starts_with("msys-") || name.starts_with("cygwin-"); + let is_pty = name.contains("-pty"); + is_msys && is_pty +} diff --git a/crux-mir/lib/std/src/sys/windows/condvar.rs b/crux-mir/lib/std/src/sys/windows/locks/condvar.rs similarity index 73% rename from crux-mir/lib/std/src/sys/windows/condvar.rs rename to crux-mir/lib/std/src/sys/windows/locks/condvar.rs index 8f7f6854c..66fafa2c0 100644 --- a/crux-mir/lib/std/src/sys/windows/condvar.rs +++ b/crux-mir/lib/std/src/sys/windows/locks/condvar.rs @@ -1,6 +1,6 @@ use crate::cell::UnsafeCell; use crate::sys::c; -use crate::sys::mutex::{self, Mutex}; +use crate::sys::locks::{mutex, Mutex}; use crate::sys::os; use crate::time::Duration; @@ -12,13 +12,11 @@ unsafe impl Send for Condvar {} unsafe impl Sync for Condvar {} impl Condvar { + #[inline] pub const fn new() -> Condvar { Condvar { inner: UnsafeCell::new(c::CONDITION_VARIABLE_INIT) } } - #[inline] - pub unsafe fn init(&mut self) {} - #[inline] pub unsafe fn wait(&self, mutex: &Mutex) { let r = c::SleepConditionVariableSRW(self.inner.get(), mutex::raw(mutex), c::INFINITE, 0); @@ -29,7 +27,7 @@ impl Condvar { let r = c::SleepConditionVariableSRW( self.inner.get(), mutex::raw(mutex), - super::dur2timeout(dur), + crate::sys::windows::dur2timeout(dur), 0, ); if r == 0 { @@ -41,16 +39,12 @@ impl Condvar { } #[inline] - pub unsafe fn notify_one(&self) { - c::WakeConditionVariable(self.inner.get()) + pub fn notify_one(&self) { + unsafe { c::WakeConditionVariable(self.inner.get()) } } #[inline] - pub unsafe fn notify_all(&self) { - c::WakeAllConditionVariable(self.inner.get()) - } - - pub unsafe fn destroy(&self) { - // ... + pub fn notify_all(&self) { + unsafe { c::WakeAllConditionVariable(self.inner.get()) } } } diff --git a/crux-mir/lib/std/src/sys/windows/locks/mod.rs b/crux-mir/lib/std/src/sys/windows/locks/mod.rs new file mode 100644 index 000000000..0e0f9eccb --- /dev/null +++ b/crux-mir/lib/std/src/sys/windows/locks/mod.rs @@ -0,0 +1,6 @@ +mod condvar; +mod mutex; +mod rwlock; +pub use condvar::Condvar; +pub use mutex::Mutex; +pub use rwlock::RwLock; diff --git a/crux-mir/lib/std/src/sys/windows/locks/mutex.rs b/crux-mir/lib/std/src/sys/windows/locks/mutex.rs new file mode 100644 index 000000000..ef2f84082 --- /dev/null +++ b/crux-mir/lib/std/src/sys/windows/locks/mutex.rs @@ -0,0 +1,54 @@ +//! System Mutexes +//! +//! The Windows implementation of mutexes is a little odd and it might not be +//! immediately obvious what's going on. The primary oddness is that SRWLock is +//! used instead of CriticalSection, and this is done because: +//! +//! 1. SRWLock is several times faster than CriticalSection according to +//! benchmarks performed on both Windows 8 and Windows 7. +//! +//! 2. CriticalSection allows recursive locking while SRWLock deadlocks. The +//! Unix implementation deadlocks so consistency is preferred. See #19962 for +//! more details. +//! +//! 3. While CriticalSection is fair and SRWLock is not, the current Rust policy +//! is that there are no guarantees of fairness. + +use crate::cell::UnsafeCell; +use crate::sys::c; + +pub struct Mutex { + srwlock: UnsafeCell, +} + +unsafe impl Send for Mutex {} +unsafe impl Sync for Mutex {} + +#[inline] +pub unsafe fn raw(m: &Mutex) -> c::PSRWLOCK { + m.srwlock.get() +} + +impl Mutex { + #[inline] + pub const fn new() -> Mutex { + Mutex { srwlock: UnsafeCell::new(c::SRWLOCK_INIT) } + } + + #[inline] + pub fn lock(&self) { + unsafe { + c::AcquireSRWLockExclusive(raw(self)); + } + } + + #[inline] + pub fn try_lock(&self) -> bool { + unsafe { c::TryAcquireSRWLockExclusive(raw(self)) != 0 } + } + + #[inline] + pub unsafe fn unlock(&self) { + c::ReleaseSRWLockExclusive(raw(self)); + } +} diff --git a/crux-mir/lib/std/src/sys/windows/locks/rwlock.rs b/crux-mir/lib/std/src/sys/windows/locks/rwlock.rs new file mode 100644 index 000000000..e69415baa --- /dev/null +++ b/crux-mir/lib/std/src/sys/windows/locks/rwlock.rs @@ -0,0 +1,40 @@ +use crate::cell::UnsafeCell; +use crate::sys::c; + +pub struct RwLock { + inner: UnsafeCell, +} + +unsafe impl Send for RwLock {} +unsafe impl Sync for RwLock {} + +impl RwLock { + #[inline] + pub const fn new() -> RwLock { + RwLock { inner: UnsafeCell::new(c::SRWLOCK_INIT) } + } + #[inline] + pub fn read(&self) { + unsafe { c::AcquireSRWLockShared(self.inner.get()) } + } + #[inline] + pub fn try_read(&self) -> bool { + unsafe { c::TryAcquireSRWLockShared(self.inner.get()) != 0 } + } + #[inline] + pub fn write(&self) { + unsafe { c::AcquireSRWLockExclusive(self.inner.get()) } + } + #[inline] + pub fn try_write(&self) -> bool { + unsafe { c::TryAcquireSRWLockExclusive(self.inner.get()) != 0 } + } + #[inline] + pub unsafe fn read_unlock(&self) { + c::ReleaseSRWLockShared(self.inner.get()) + } + #[inline] + pub unsafe fn write_unlock(&self) { + c::ReleaseSRWLockExclusive(self.inner.get()) + } +} diff --git a/crux-mir/lib/std/src/sys/windows/mod.rs b/crux-mir/lib/std/src/sys/windows/mod.rs index b004cd190..77359abe4 100644 --- a/crux-mir/lib/std/src/sys/windows/mod.rs +++ b/crux-mir/lib/std/src/sys/windows/mod.rs @@ -1,14 +1,13 @@ #![allow(missing_docs, nonstandard_style)] -use crate::ffi::{OsStr, OsString}; +use crate::ffi::{CStr, OsStr, OsString}; use crate::io::ErrorKind; +use crate::mem::MaybeUninit; use crate::os::windows::ffi::{OsStrExt, OsStringExt}; use crate::path::PathBuf; -use crate::ptr; use crate::time::Duration; pub use self::rand::hashmap_random_keys; -pub use libc::strlen; #[macro_use] pub mod compat; @@ -17,15 +16,12 @@ pub mod alloc; pub mod args; pub mod c; pub mod cmath; -pub mod condvar; pub mod env; -pub mod ext; -pub mod fast_thread_local; pub mod fs; pub mod handle; pub mod io; +pub mod locks; pub mod memchr; -pub mod mutex; pub mod net; pub mod os; pub mod os_str; @@ -33,59 +29,145 @@ pub mod path; pub mod pipe; pub mod process; pub mod rand; -pub mod rwlock; +pub mod stdio; pub mod thread; -pub mod thread_local; +pub mod thread_local_dtor; +pub mod thread_local_key; +pub mod thread_parking; pub mod time; cfg_if::cfg_if! { if #[cfg(not(target_vendor = "uwp"))] { - pub mod stdio; pub mod stack_overflow; } else { - pub mod stdio_uwp; pub mod stack_overflow_uwp; - pub use self::stdio_uwp as stdio; pub use self::stack_overflow_uwp as stack_overflow; } } -#[cfg(not(test))] -pub fn init() {} +// SAFETY: must be called only once during runtime initialization. +// NOTE: this is not guaranteed to run, for example when Rust code is called externally. +pub unsafe fn init(_argc: isize, _argv: *const *const u8, _sigpipe: u8) { + stack_overflow::init(); + + // Normally, `thread::spawn` will call `Thread::set_name` but since this thread already + // exists, we have to call it ourselves. + thread::Thread::set_name(&CStr::from_bytes_with_nul_unchecked(b"main\0")); +} + +// SAFETY: must be called only once during runtime cleanup. +// NOTE: this is not guaranteed to run, for example when the program aborts. +pub unsafe fn cleanup() { + net::cleanup(); +} pub fn decode_error_kind(errno: i32) -> ErrorKind { + use ErrorKind::*; + match errno as c::DWORD { - c::ERROR_ACCESS_DENIED => return ErrorKind::PermissionDenied, - c::ERROR_ALREADY_EXISTS => return ErrorKind::AlreadyExists, - c::ERROR_FILE_EXISTS => return ErrorKind::AlreadyExists, - c::ERROR_BROKEN_PIPE => return ErrorKind::BrokenPipe, - c::ERROR_FILE_NOT_FOUND => return ErrorKind::NotFound, - c::ERROR_PATH_NOT_FOUND => return ErrorKind::NotFound, - c::ERROR_NO_DATA => return ErrorKind::BrokenPipe, - c::ERROR_OPERATION_ABORTED => return ErrorKind::TimedOut, + c::ERROR_ACCESS_DENIED => return PermissionDenied, + c::ERROR_ALREADY_EXISTS => return AlreadyExists, + c::ERROR_FILE_EXISTS => return AlreadyExists, + c::ERROR_BROKEN_PIPE => return BrokenPipe, + c::ERROR_FILE_NOT_FOUND => return NotFound, + c::ERROR_PATH_NOT_FOUND => return NotFound, + c::ERROR_NO_DATA => return BrokenPipe, + c::ERROR_INVALID_NAME => return InvalidFilename, + c::ERROR_INVALID_PARAMETER => return InvalidInput, + c::ERROR_NOT_ENOUGH_MEMORY | c::ERROR_OUTOFMEMORY => return OutOfMemory, + c::ERROR_SEM_TIMEOUT + | c::WAIT_TIMEOUT + | c::ERROR_DRIVER_CANCEL_TIMEOUT + | c::ERROR_OPERATION_ABORTED + | c::ERROR_SERVICE_REQUEST_TIMEOUT + | c::ERROR_COUNTER_TIMEOUT + | c::ERROR_TIMEOUT + | c::ERROR_RESOURCE_CALL_TIMED_OUT + | c::ERROR_CTX_MODEM_RESPONSE_TIMEOUT + | c::ERROR_CTX_CLIENT_QUERY_TIMEOUT + | c::FRS_ERR_SYSVOL_POPULATE_TIMEOUT + | c::ERROR_DS_TIMELIMIT_EXCEEDED + | c::DNS_ERROR_RECORD_TIMED_OUT + | c::ERROR_IPSEC_IKE_TIMED_OUT + | c::ERROR_RUNLEVEL_SWITCH_TIMEOUT + | c::ERROR_RUNLEVEL_SWITCH_AGENT_TIMEOUT => return TimedOut, + c::ERROR_CALL_NOT_IMPLEMENTED => return Unsupported, + c::ERROR_HOST_UNREACHABLE => return HostUnreachable, + c::ERROR_NETWORK_UNREACHABLE => return NetworkUnreachable, + c::ERROR_DIRECTORY => return NotADirectory, + c::ERROR_DIRECTORY_NOT_SUPPORTED => return IsADirectory, + c::ERROR_DIR_NOT_EMPTY => return DirectoryNotEmpty, + c::ERROR_WRITE_PROTECT => return ReadOnlyFilesystem, + c::ERROR_DISK_FULL | c::ERROR_HANDLE_DISK_FULL => return StorageFull, + c::ERROR_SEEK_ON_DEVICE => return NotSeekable, + c::ERROR_DISK_QUOTA_EXCEEDED => return FilesystemQuotaExceeded, + c::ERROR_FILE_TOO_LARGE => return FileTooLarge, + c::ERROR_BUSY => return ResourceBusy, + c::ERROR_POSSIBLE_DEADLOCK => return Deadlock, + c::ERROR_NOT_SAME_DEVICE => return CrossesDevices, + c::ERROR_TOO_MANY_LINKS => return TooManyLinks, + c::ERROR_FILENAME_EXCED_RANGE => return InvalidFilename, _ => {} } match errno { - c::WSAEACCES => ErrorKind::PermissionDenied, - c::WSAEADDRINUSE => ErrorKind::AddrInUse, - c::WSAEADDRNOTAVAIL => ErrorKind::AddrNotAvailable, - c::WSAECONNABORTED => ErrorKind::ConnectionAborted, - c::WSAECONNREFUSED => ErrorKind::ConnectionRefused, - c::WSAECONNRESET => ErrorKind::ConnectionReset, - c::WSAEINVAL => ErrorKind::InvalidInput, - c::WSAENOTCONN => ErrorKind::NotConnected, - c::WSAEWOULDBLOCK => ErrorKind::WouldBlock, - c::WSAETIMEDOUT => ErrorKind::TimedOut, + c::WSAEACCES => PermissionDenied, + c::WSAEADDRINUSE => AddrInUse, + c::WSAEADDRNOTAVAIL => AddrNotAvailable, + c::WSAECONNABORTED => ConnectionAborted, + c::WSAECONNREFUSED => ConnectionRefused, + c::WSAECONNRESET => ConnectionReset, + c::WSAEINVAL => InvalidInput, + c::WSAENOTCONN => NotConnected, + c::WSAEWOULDBLOCK => WouldBlock, + c::WSAETIMEDOUT => TimedOut, + c::WSAEHOSTUNREACH => HostUnreachable, + c::WSAENETDOWN => NetworkDown, + c::WSAENETUNREACH => NetworkUnreachable, + + _ => Uncategorized, + } +} + +pub fn unrolled_find_u16s(needle: u16, haystack: &[u16]) -> Option { + let ptr = haystack.as_ptr(); + let mut start = &haystack[..]; - _ => ErrorKind::Other, + // For performance reasons unfold the loop eight times. + while start.len() >= 8 { + macro_rules! if_return { + ($($n:literal,)+) => { + $( + if start[$n] == needle { + return Some(((&start[$n] as *const u16).addr() - ptr.addr()) / 2); + } + )+ + } + } + + if_return!(0, 1, 2, 3, 4, 5, 6, 7,); + + start = &start[8..]; } + + for c in start { + if *c == needle { + return Some(((c as *const u16).addr() - ptr.addr()) / 2); + } + } + None } pub fn to_u16s>(s: S) -> crate::io::Result> { fn inner(s: &OsStr) -> crate::io::Result> { - let mut maybe_result: Vec = s.encode_wide().collect(); - if maybe_result.iter().any(|&u| u == 0) { - return Err(crate::io::Error::new( + // Most paths are ASCII, so reserve capacity for as much as there are bytes + // in the OsStr plus one for the null-terminating character. We are not + // wasting bytes here as paths created by this function are primarily used + // in an ephemeral fashion. + let mut maybe_result = Vec::with_capacity(s.len() + 1); + maybe_result.extend(s.encode_wide()); + + if unrolled_find_u16s(0, &maybe_result).is_some() { + return Err(crate::io::const_io_error!( ErrorKind::InvalidInput, "strings passed to WinAPI cannot contain NULs", )); @@ -117,8 +199,12 @@ where { // Start off with a stack buf but then spill over to the heap if we end up // needing more space. - let mut stack_buf = [0u16; 512]; - let mut heap_buf = Vec::new(); + // + // This initial size also works around `GetFullPathNameW` returning + // incorrect size hints for some short paths: + // https://github.com/dylni/normpath/issues/5 + let mut stack_buf: [MaybeUninit; 512] = MaybeUninit::uninit_array(); + let mut heap_buf: Vec> = Vec::new(); unsafe { let mut n = stack_buf.len(); loop { @@ -127,6 +213,11 @@ where } else { let extra = n - heap_buf.len(); heap_buf.reserve(extra); + // We used `reserve` and not `reserve_exact`, so in theory we + // may have gotten more than requested. If so, we'd like to use + // it... so long as we won't cause overflow. + n = heap_buf.capacity().min(c::DWORD::MAX as usize); + // Safety: MaybeUninit does not need initialization heap_buf.set_len(n); &mut heap_buf[..] }; @@ -141,17 +232,25 @@ where // error" is still 0 then we interpret it as a 0 length buffer and // not an actual error. c::SetLastError(0); - let k = match f1(buf.as_mut_ptr(), n as c::DWORD) { + let k = match f1(buf.as_mut_ptr().cast::(), n as c::DWORD) { 0 if c::GetLastError() == 0 => 0, 0 => return Err(crate::io::Error::last_os_error()), n => n, } as usize; if k == n && c::GetLastError() == c::ERROR_INSUFFICIENT_BUFFER { - n *= 2; - } else if k >= n { + n = n.saturating_mul(2).min(c::DWORD::MAX as usize); + } else if k > n { n = k; + } else if k == n { + // It is impossible to reach this point. + // On success, k is the returned string length excluding the null. + // On failure, k is the required buffer length including the null. + // Therefore k never equals n. + unreachable!(); } else { - return Ok(f2(&buf[..k])); + // Safety: First `k` values are initialized. + let slice: &[u16] = MaybeUninit::slice_assume_init_ref(&buf[..k]); + return Ok(f2(slice)); } } } @@ -161,60 +260,8 @@ fn os2path(s: &[u16]) -> PathBuf { PathBuf::from(OsString::from_wide(s)) } -#[allow(dead_code)] // Only used in backtrace::gnu::get_executable_filename() -fn wide_char_to_multi_byte( - code_page: u32, - flags: u32, - s: &[u16], - no_default_char: bool, -) -> crate::io::Result> { - unsafe { - let mut size = c::WideCharToMultiByte( - code_page, - flags, - s.as_ptr(), - s.len() as i32, - ptr::null_mut(), - 0, - ptr::null(), - ptr::null_mut(), - ); - if size == 0 { - return Err(crate::io::Error::last_os_error()); - } - - let mut buf = Vec::with_capacity(size as usize); - buf.set_len(size as usize); - - let mut used_default_char = c::FALSE; - size = c::WideCharToMultiByte( - code_page, - flags, - s.as_ptr(), - s.len() as i32, - buf.as_mut_ptr(), - buf.len() as i32, - ptr::null(), - if no_default_char { &mut used_default_char } else { ptr::null_mut() }, - ); - if size == 0 { - return Err(crate::io::Error::last_os_error()); - } - if no_default_char && used_default_char == c::TRUE { - return Err(crate::io::Error::new( - crate::io::ErrorKind::InvalidData, - "string cannot be converted to requested code page", - )); - } - - buf.set_len(size as usize); - - Ok(buf) - } -} - pub fn truncate_utf16_at_nul(v: &[u16]) -> &[u16] { - match v.iter().position(|c| *c == 0) { + match unrolled_find_u16s(0, v) { // don't include the 0 Some(i) => &v[..i], None => v, @@ -251,24 +298,40 @@ pub fn dur2timeout(dur: Duration) -> c::DWORD { .checked_mul(1000) .and_then(|ms| ms.checked_add((dur.subsec_nanos() as u64) / 1_000_000)) .and_then(|ms| ms.checked_add(if dur.subsec_nanos() % 1_000_000 > 0 { 1 } else { 0 })) - .map(|ms| if ms > ::max_value() as u64 { c::INFINITE } else { ms as c::DWORD }) + .map(|ms| if ms > ::MAX as u64 { c::INFINITE } else { ms as c::DWORD }) .unwrap_or(c::INFINITE) } -// On Windows, use the processor-specific __fastfail mechanism. In Windows 8 -// and later, this will terminate the process immediately without running any -// in-process exception handlers. In earlier versions of Windows, this -// sequence of instructions will be treated as an access violation, -// terminating the process but without necessarily bypassing all exception -// handlers. -// -// https://docs.microsoft.com/en-us/cpp/intrinsics/fastfail +/// Use `__fastfail` to abort the process +/// +/// This is the same implementation as in libpanic_abort's `__rust_start_panic`. See +/// that function for more information on `__fastfail` #[allow(unreachable_code)] -pub unsafe fn abort_internal() -> ! { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - asm!("int $$0x29" :: "{ecx}"(7) ::: volatile); // 7 is FAST_FAIL_FATAL_APP_EXIT - crate::intrinsics::unreachable(); +pub fn abort_internal() -> ! { + #[allow(unused)] + const FAST_FAIL_FATAL_APP_EXIT: usize = 7; + #[cfg(not(miri))] // inline assembly does not work in Miri + unsafe { + cfg_if::cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + core::arch::asm!("int $$0x29", in("ecx") FAST_FAIL_FATAL_APP_EXIT); + crate::intrinsics::unreachable(); + } else if #[cfg(all(target_arch = "arm", target_feature = "thumb-mode"))] { + core::arch::asm!(".inst 0xDEFB", in("r0") FAST_FAIL_FATAL_APP_EXIT); + crate::intrinsics::unreachable(); + } else if #[cfg(target_arch = "aarch64")] { + core::arch::asm!("brk 0xF003", in("x0") FAST_FAIL_FATAL_APP_EXIT); + crate::intrinsics::unreachable(); + } + } } crate::intrinsics::abort(); } + +/// Align the inner value to 8 bytes. +/// +/// This is enough for almost all of the buffers we're likely to work with in +/// the Windows APIs we use. +#[repr(C, align(8))] +#[derive(Copy, Clone)] +pub(crate) struct Align8(pub T); diff --git a/crux-mir/lib/std/src/sys/windows/mutex.rs b/crux-mir/lib/std/src/sys/windows/mutex.rs deleted file mode 100644 index 63dfc6409..000000000 --- a/crux-mir/lib/std/src/sys/windows/mutex.rs +++ /dev/null @@ -1,184 +0,0 @@ -//! System Mutexes -//! -//! The Windows implementation of mutexes is a little odd and it may not be -//! immediately obvious what's going on. The primary oddness is that SRWLock is -//! used instead of CriticalSection, and this is done because: -//! -//! 1. SRWLock is several times faster than CriticalSection according to -//! benchmarks performed on both Windows 8 and Windows 7. -//! -//! 2. CriticalSection allows recursive locking while SRWLock deadlocks. The -//! Unix implementation deadlocks so consistency is preferred. See #19962 for -//! more details. -//! -//! 3. While CriticalSection is fair and SRWLock is not, the current Rust policy -//! is that there are no guarantees of fairness. -//! -//! The downside of this approach, however, is that SRWLock is not available on -//! Windows XP, so we continue to have a fallback implementation where -//! CriticalSection is used and we keep track of who's holding the mutex to -//! detect recursive locks. - -use crate::cell::UnsafeCell; -use crate::mem::{self, MaybeUninit}; -use crate::sync::atomic::{AtomicUsize, Ordering}; -use crate::sys::c; -use crate::sys::compat; - -pub struct Mutex { - lock: AtomicUsize, - held: UnsafeCell, -} - -unsafe impl Send for Mutex {} -unsafe impl Sync for Mutex {} - -#[derive(Clone, Copy)] -enum Kind { - SRWLock = 1, - CriticalSection = 2, -} - -#[inline] -pub unsafe fn raw(m: &Mutex) -> c::PSRWLOCK { - debug_assert!(mem::size_of::() <= mem::size_of_val(&m.lock)); - &m.lock as *const _ as *mut _ -} - -impl Mutex { - pub const fn new() -> Mutex { - Mutex { - // This works because SRWLOCK_INIT is 0 (wrapped in a struct), so we are also properly - // initializing an SRWLOCK here. - lock: AtomicUsize::new(0), - held: UnsafeCell::new(false), - } - } - #[inline] - pub unsafe fn init(&mut self) {} - pub unsafe fn lock(&self) { - match kind() { - Kind::SRWLock => c::AcquireSRWLockExclusive(raw(self)), - Kind::CriticalSection => { - let re = self.remutex(); - (*re).lock(); - if !self.flag_locked() { - (*re).unlock(); - panic!("cannot recursively lock a mutex"); - } - } - } - } - pub unsafe fn try_lock(&self) -> bool { - match kind() { - Kind::SRWLock => c::TryAcquireSRWLockExclusive(raw(self)) != 0, - Kind::CriticalSection => { - let re = self.remutex(); - if !(*re).try_lock() { - false - } else if self.flag_locked() { - true - } else { - (*re).unlock(); - false - } - } - } - } - pub unsafe fn unlock(&self) { - *self.held.get() = false; - match kind() { - Kind::SRWLock => c::ReleaseSRWLockExclusive(raw(self)), - Kind::CriticalSection => (*self.remutex()).unlock(), - } - } - pub unsafe fn destroy(&self) { - match kind() { - Kind::SRWLock => {} - Kind::CriticalSection => match self.lock.load(Ordering::SeqCst) { - 0 => {} - n => { - Box::from_raw(n as *mut ReentrantMutex).destroy(); - } - }, - } - } - - unsafe fn remutex(&self) -> *mut ReentrantMutex { - match self.lock.load(Ordering::SeqCst) { - 0 => {} - n => return n as *mut _, - } - let re = box ReentrantMutex::uninitialized(); - re.init(); - let re = Box::into_raw(re); - match self.lock.compare_and_swap(0, re as usize, Ordering::SeqCst) { - 0 => re, - n => { - Box::from_raw(re).destroy(); - n as *mut _ - } - } - } - - unsafe fn flag_locked(&self) -> bool { - if *self.held.get() { - false - } else { - *self.held.get() = true; - true - } - } -} - -fn kind() -> Kind { - static KIND: AtomicUsize = AtomicUsize::new(0); - - let val = KIND.load(Ordering::SeqCst); - if val == Kind::SRWLock as usize { - return Kind::SRWLock; - } else if val == Kind::CriticalSection as usize { - return Kind::CriticalSection; - } - - let ret = match compat::lookup("kernel32", "AcquireSRWLockExclusive") { - None => Kind::CriticalSection, - Some(..) => Kind::SRWLock, - }; - KIND.store(ret as usize, Ordering::SeqCst); - ret -} - -pub struct ReentrantMutex { - inner: UnsafeCell>, -} - -unsafe impl Send for ReentrantMutex {} -unsafe impl Sync for ReentrantMutex {} - -impl ReentrantMutex { - pub const fn uninitialized() -> ReentrantMutex { - ReentrantMutex { inner: UnsafeCell::new(MaybeUninit::uninit()) } - } - - pub unsafe fn init(&self) { - c::InitializeCriticalSection((&mut *self.inner.get()).as_mut_ptr()); - } - - pub unsafe fn lock(&self) { - c::EnterCriticalSection((&mut *self.inner.get()).as_mut_ptr()); - } - - #[inline] - pub unsafe fn try_lock(&self) -> bool { - c::TryEnterCriticalSection((&mut *self.inner.get()).as_mut_ptr()) != 0 - } - - pub unsafe fn unlock(&self) { - c::LeaveCriticalSection((&mut *self.inner.get()).as_mut_ptr()); - } - - pub unsafe fn destroy(&self) { - c::DeleteCriticalSection((&mut *self.inner.get()).as_mut_ptr()); - } -} diff --git a/crux-mir/lib/std/src/sys/windows/net.rs b/crux-mir/lib/std/src/sys/windows/net.rs index d8d4fdfce..e0701a498 100644 --- a/crux-mir/lib/std/src/sys/windows/net.rs +++ b/crux-mir/lib/std/src/sys/windows/net.rs @@ -4,15 +4,18 @@ use crate::cmp; use crate::io::{self, IoSlice, IoSliceMut, Read}; use crate::mem; use crate::net::{Shutdown, SocketAddr}; +use crate::os::windows::io::{ + AsRawSocket, AsSocket, BorrowedSocket, FromRawSocket, IntoRawSocket, OwnedSocket, RawSocket, +}; use crate::ptr; -use crate::sync::Once; +use crate::sync::OnceLock; use crate::sys; use crate::sys::c; use crate::sys_common::net; -use crate::sys_common::{self, AsInner, FromInner, IntoInner}; +use crate::sys_common::{AsInner, FromInner, IntoInner}; use crate::time::Duration; -use libc::{c_int, c_long, c_ulong, c_void}; +use libc::{c_int, c_long, c_ulong, c_ushort}; pub type wrlen_t = i32; @@ -24,14 +27,14 @@ pub mod netc { pub use crate::sys::c::*; } -pub struct Socket(c::SOCKET); +pub struct Socket(OwnedSocket); + +static WSA_CLEANUP: OnceLock i32> = OnceLock::new(); /// Checks whether the Windows socket interface has been started already, and /// if not, starts it. pub fn init() { - static START: Once = Once::new(); - - START.call_once(|| unsafe { + let _ = WSA_CLEANUP.get_or_init(|| unsafe { let mut data: c::WSADATA = mem::zeroed(); let ret = c::WSAStartup( 0x202, // version 2.2 @@ -39,12 +42,22 @@ pub fn init() { ); assert_eq!(ret, 0); - let _ = sys_common::at_exit(|| { - c::WSACleanup(); - }); + // Only register `WSACleanup` if `WSAStartup` is actually ever called. + // Workaround to prevent linking to `WS2_32.dll` when no network functionality is used. + // See issue #85441. + c::WSACleanup }); } +pub fn cleanup() { + // only perform cleanup if network functionality was actually initialized + if let Some(cleanup) = WSA_CLEANUP.get() { + unsafe { + cleanup(); + } + } +} + /// Returns the last error from the Windows socket interface. fn last_error() -> io::Error { io::Error::from_raw_os_error(unsafe { c::WSAGetLastError() }) @@ -88,153 +101,137 @@ where impl Socket { pub fn new(addr: &SocketAddr, ty: c_int) -> io::Result { - let fam = match *addr { + let family = match *addr { SocketAddr::V4(..) => c::AF_INET, SocketAddr::V6(..) => c::AF_INET6, }; let socket = unsafe { - match c::WSASocketW( - fam, + c::WSASocketW( + family, ty, 0, ptr::null_mut(), 0, c::WSA_FLAG_OVERLAPPED | c::WSA_FLAG_NO_HANDLE_INHERIT, - ) { - c::INVALID_SOCKET => match c::WSAGetLastError() { - c::WSAEPROTOTYPE | c::WSAEINVAL => { - match c::WSASocketW(fam, ty, 0, ptr::null_mut(), 0, c::WSA_FLAG_OVERLAPPED) - { - c::INVALID_SOCKET => Err(last_error()), - n => { - let s = Socket(n); - s.set_no_inherit()?; - Ok(s) - } - } - } - n => Err(io::Error::from_raw_os_error(n)), - }, - n => Ok(Socket(n)), + ) + }; + + if socket != c::INVALID_SOCKET { + unsafe { Ok(Self::from_raw_socket(socket)) } + } else { + let error = unsafe { c::WSAGetLastError() }; + + if error != c::WSAEPROTOTYPE && error != c::WSAEINVAL { + return Err(io::Error::from_raw_os_error(error)); + } + + let socket = + unsafe { c::WSASocketW(family, ty, 0, ptr::null_mut(), 0, c::WSA_FLAG_OVERLAPPED) }; + + if socket == c::INVALID_SOCKET { + return Err(last_error()); } - }?; - Ok(socket) + + unsafe { + let socket = Self::from_raw_socket(socket); + socket.0.set_no_inherit()?; + Ok(socket) + } + } } pub fn connect_timeout(&self, addr: &SocketAddr, timeout: Duration) -> io::Result<()> { self.set_nonblocking(true)?; - let r = unsafe { - let (addrp, len) = addr.into_inner(); - cvt(c::connect(self.0, addrp, len)) + let result = { + let (addr, len) = addr.into_inner(); + let result = unsafe { c::connect(self.as_raw_socket(), addr.as_ptr(), len) }; + cvt(result).map(drop) }; self.set_nonblocking(false)?; - match r { - Ok(_) => return Ok(()), - Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {} - Err(e) => return Err(e), - } - - if timeout.as_secs() == 0 && timeout.subsec_nanos() == 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "cannot set a 0 duration timeout", - )); - } - - let mut timeout = c::timeval { - tv_sec: timeout.as_secs() as c_long, - tv_usec: (timeout.subsec_nanos() / 1000) as c_long, - }; - if timeout.tv_sec == 0 && timeout.tv_usec == 0 { - timeout.tv_usec = 1; - } + match result { + Err(ref error) if error.kind() == io::ErrorKind::WouldBlock => { + if timeout.as_secs() == 0 && timeout.subsec_nanos() == 0 { + return Err(io::const_io_error!( + io::ErrorKind::InvalidInput, + "cannot set a 0 duration timeout", + )); + } - let fds = unsafe { - let mut fds = mem::zeroed::(); - fds.fd_count = 1; - fds.fd_array[0] = self.0; - fds - }; + let mut timeout = c::timeval { + tv_sec: timeout.as_secs() as c_long, + tv_usec: (timeout.subsec_nanos() / 1000) as c_long, + }; - let mut writefds = fds; - let mut errorfds = fds; + if timeout.tv_sec == 0 && timeout.tv_usec == 0 { + timeout.tv_usec = 1; + } - let n = - unsafe { cvt(c::select(1, ptr::null_mut(), &mut writefds, &mut errorfds, &timeout))? }; + let fds = { + let mut fds = unsafe { mem::zeroed::() }; + fds.fd_count = 1; + fds.fd_array[0] = self.as_raw_socket(); + fds + }; + + let mut writefds = fds; + let mut errorfds = fds; + + let count = { + let result = unsafe { + c::select(1, ptr::null_mut(), &mut writefds, &mut errorfds, &timeout) + }; + cvt(result)? + }; + + match count { + 0 => Err(io::const_io_error!(io::ErrorKind::TimedOut, "connection timed out")), + _ => { + if writefds.fd_count != 1 { + if let Some(e) = self.take_error()? { + return Err(e); + } + } - match n { - 0 => Err(io::Error::new(io::ErrorKind::TimedOut, "connection timed out")), - _ => { - if writefds.fd_count != 1 { - if let Some(e) = self.take_error()? { - return Err(e); + Ok(()) } } - Ok(()) } + _ => result, } } pub fn accept(&self, storage: *mut c::SOCKADDR, len: *mut c_int) -> io::Result { - let socket = unsafe { - match c::accept(self.0, storage, len) { - c::INVALID_SOCKET => Err(last_error()), - n => Ok(Socket(n)), - } - }?; - Ok(socket) + let socket = unsafe { c::accept(self.as_raw_socket(), storage, len) }; + + match socket { + c::INVALID_SOCKET => Err(last_error()), + _ => unsafe { Ok(Self::from_raw_socket(socket)) }, + } } pub fn duplicate(&self) -> io::Result { - let socket = unsafe { - let mut info: c::WSAPROTOCOL_INFO = mem::zeroed(); - cvt(c::WSADuplicateSocketW(self.0, c::GetCurrentProcessId(), &mut info))?; - - match c::WSASocketW( - info.iAddressFamily, - info.iSocketType, - info.iProtocol, - &mut info, - 0, - c::WSA_FLAG_OVERLAPPED | c::WSA_FLAG_NO_HANDLE_INHERIT, - ) { - c::INVALID_SOCKET => match c::WSAGetLastError() { - c::WSAEPROTOTYPE | c::WSAEINVAL => { - match c::WSASocketW( - info.iAddressFamily, - info.iSocketType, - info.iProtocol, - &mut info, - 0, - c::WSA_FLAG_OVERLAPPED, - ) { - c::INVALID_SOCKET => Err(last_error()), - n => { - let s = Socket(n); - s.set_no_inherit()?; - Ok(s) - } - } - } - n => Err(io::Error::from_raw_os_error(n)), - }, - n => Ok(Socket(n)), - } - }?; - Ok(socket) + Ok(Self(self.0.try_clone()?)) } fn recv_with_flags(&self, buf: &mut [u8], flags: c_int) -> io::Result { // On unix when a socket is shut down all further reads return 0, so we // do the same on windows to map a shut down socket to returning EOF. - let len = cmp::min(buf.len(), i32::max_value() as usize) as i32; - unsafe { - match c::recv(self.0, buf.as_mut_ptr() as *mut c_void, len, flags) { - -1 if c::WSAGetLastError() == c::WSAESHUTDOWN => Ok(0), - -1 => Err(last_error()), - n => Ok(n as usize), + let length = cmp::min(buf.len(), i32::MAX as usize) as i32; + let result = + unsafe { c::recv(self.as_raw_socket(), buf.as_mut_ptr() as *mut _, length, flags) }; + + match result { + c::SOCKET_ERROR => { + let error = unsafe { c::WSAGetLastError() }; + + if error == c::WSAESHUTDOWN { + Ok(0) + } else { + Err(io::Error::from_raw_os_error(error)) + } } + _ => Ok(result as usize), } } @@ -245,27 +242,40 @@ impl Socket { pub fn read_vectored(&self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { // On unix when a socket is shut down all further reads return 0, so we // do the same on windows to map a shut down socket to returning EOF. - let len = cmp::min(bufs.len(), c::DWORD::max_value() as usize) as c::DWORD; + let length = cmp::min(bufs.len(), c::DWORD::MAX as usize) as c::DWORD; let mut nread = 0; let mut flags = 0; - unsafe { - let ret = c::WSARecv( - self.0, + let result = unsafe { + c::WSARecv( + self.as_raw_socket(), bufs.as_mut_ptr() as *mut c::WSABUF, - len, + length, &mut nread, &mut flags, ptr::null_mut(), ptr::null_mut(), - ); - match ret { - 0 => Ok(nread as usize), - _ if c::WSAGetLastError() == c::WSAESHUTDOWN => Ok(0), - _ => Err(last_error()), + ) + }; + + match result { + 0 => Ok(nread as usize), + _ => { + let error = unsafe { c::WSAGetLastError() }; + + if error == c::WSAESHUTDOWN { + Ok(0) + } else { + Err(io::Error::from_raw_os_error(error)) + } } } } + #[inline] + pub fn is_read_vectored(&self) -> bool { + true + } + pub fn peek(&self, buf: &mut [u8]) -> io::Result { self.recv_with_flags(buf, c::MSG_PEEK) } @@ -275,27 +285,34 @@ impl Socket { buf: &mut [u8], flags: c_int, ) -> io::Result<(usize, SocketAddr)> { - let mut storage: c::SOCKADDR_STORAGE_LH = unsafe { mem::zeroed() }; + let mut storage = unsafe { mem::zeroed::() }; let mut addrlen = mem::size_of_val(&storage) as c::socklen_t; - let len = cmp::min(buf.len(), ::max_value() as usize) as wrlen_t; + let length = cmp::min(buf.len(), ::MAX as usize) as wrlen_t; // On unix when a socket is shut down all further reads return 0, so we // do the same on windows to map a shut down socket to returning EOF. - unsafe { - match c::recvfrom( - self.0, - buf.as_mut_ptr() as *mut c_void, - len, + let result = unsafe { + c::recvfrom( + self.as_raw_socket(), + buf.as_mut_ptr() as *mut _, + length, flags, &mut storage as *mut _ as *mut _, &mut addrlen, - ) { - -1 if c::WSAGetLastError() == c::WSAESHUTDOWN => { + ) + }; + + match result { + c::SOCKET_ERROR => { + let error = unsafe { c::WSAGetLastError() }; + + if error == c::WSAESHUTDOWN { Ok((0, net::sockaddr_to_addr(&storage, addrlen as usize)?)) + } else { + Err(io::Error::from_raw_os_error(error)) } - -1 => Err(last_error()), - n => Ok((n as usize, net::sockaddr_to_addr(&storage, addrlen as usize)?)), } + _ => Ok((result as usize, net::sockaddr_to_addr(&storage, addrlen as usize)?)), } } @@ -308,20 +325,25 @@ impl Socket { } pub fn write_vectored(&self, bufs: &[IoSlice<'_>]) -> io::Result { - let len = cmp::min(bufs.len(), c::DWORD::max_value() as usize) as c::DWORD; + let length = cmp::min(bufs.len(), c::DWORD::MAX as usize) as c::DWORD; let mut nwritten = 0; - unsafe { - cvt(c::WSASend( - self.0, - bufs.as_ptr() as *const c::WSABUF as *mut c::WSABUF, - len, + let result = unsafe { + c::WSASend( + self.as_raw_socket(), + bufs.as_ptr() as *const c::WSABUF as *mut _, + length, &mut nwritten, 0, ptr::null_mut(), ptr::null_mut(), - ))?; - } - Ok(nwritten as usize) + ) + }; + cvt(result).map(|_| nwritten as usize) + } + + #[inline] + pub fn is_write_vectored(&self) -> bool { + true } pub fn set_timeout(&self, dur: Option, kind: c_int) -> io::Result<()> { @@ -329,7 +351,7 @@ impl Socket { Some(dur) => { let timeout = sys::dur2timeout(dur); if timeout == 0 { - return Err(io::Error::new( + return Err(io::const_io_error!( io::ErrorKind::InvalidInput, "cannot set a 0 duration timeout", )); @@ -352,39 +374,44 @@ impl Socket { } } - #[cfg(not(target_vendor = "uwp"))] - fn set_no_inherit(&self) -> io::Result<()> { - sys::cvt(unsafe { c::SetHandleInformation(self.0 as c::HANDLE, c::HANDLE_FLAG_INHERIT, 0) }) - .map(drop) - } - - #[cfg(target_vendor = "uwp")] - fn set_no_inherit(&self) -> io::Result<()> { - Err(io::Error::new(io::ErrorKind::Other, "Unavailable on UWP")) - } - pub fn shutdown(&self, how: Shutdown) -> io::Result<()> { let how = match how { Shutdown::Write => c::SD_SEND, Shutdown::Read => c::SD_RECEIVE, Shutdown::Both => c::SD_BOTH, }; - cvt(unsafe { c::shutdown(self.0, how) })?; - Ok(()) + let result = unsafe { c::shutdown(self.as_raw_socket(), how) }; + cvt(result).map(drop) } pub fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()> { let mut nonblocking = nonblocking as c_ulong; - let r = unsafe { c::ioctlsocket(self.0, c::FIONBIO as c_int, &mut nonblocking) }; - if r == 0 { Ok(()) } else { Err(io::Error::last_os_error()) } + let result = + unsafe { c::ioctlsocket(self.as_raw_socket(), c::FIONBIO as c_int, &mut nonblocking) }; + cvt(result).map(drop) + } + + pub fn set_linger(&self, linger: Option) -> io::Result<()> { + let linger = c::linger { + l_onoff: linger.is_some() as c_ushort, + l_linger: linger.unwrap_or_default().as_secs() as c_ushort, + }; + + net::setsockopt(self, c::SOL_SOCKET, c::SO_LINGER, linger) + } + + pub fn linger(&self) -> io::Result> { + let val: c::linger = net::getsockopt(self, c::SOL_SOCKET, c::SO_LINGER)?; + + Ok((val.l_onoff != 0).then(|| Duration::from_secs(val.l_linger as u64))) } pub fn set_nodelay(&self, nodelay: bool) -> io::Result<()> { - net::setsockopt(self, c::IPPROTO_TCP, c::TCP_NODELAY, nodelay as c::BYTE) + net::setsockopt(self, c::IPPROTO_TCP, c::TCP_NODELAY, nodelay as c::BOOL) } pub fn nodelay(&self) -> io::Result { - let raw: c::BYTE = net::getsockopt(self, c::IPPROTO_TCP, c::TCP_NODELAY)?; + let raw: c::BOOL = net::getsockopt(self, c::IPPROTO_TCP, c::TCP_NODELAY)?; Ok(raw != 0) } @@ -392,6 +419,11 @@ impl Socket { let raw: c_int = net::getsockopt(self, c::SOL_SOCKET, c::SO_ERROR)?; if raw == 0 { Ok(None) } else { Ok(Some(io::Error::from_raw_os_error(raw as i32))) } } + + // This is used by sys_common code to abstract over Windows and Unix. + pub fn as_raw(&self) -> RawSocket { + self.as_inner().as_raw_socket() + } } #[unstable(reason = "not public", issue = "none", feature = "fd_read")] @@ -401,28 +433,44 @@ impl<'a> Read for &'a Socket { } } -impl Drop for Socket { - fn drop(&mut self) { - let _ = unsafe { c::closesocket(self.0) }; +impl AsInner for Socket { + fn as_inner(&self) -> &OwnedSocket { + &self.0 } } -impl AsInner for Socket { - fn as_inner(&self) -> &c::SOCKET { - &self.0 +impl FromInner for Socket { + fn from_inner(sock: OwnedSocket) -> Socket { + Socket(sock) } } -impl FromInner for Socket { - fn from_inner(sock: c::SOCKET) -> Socket { - Socket(sock) +impl IntoInner for Socket { + fn into_inner(self) -> OwnedSocket { + self.0 + } +} + +impl AsSocket for Socket { + fn as_socket(&self) -> BorrowedSocket<'_> { + self.0.as_socket() + } +} + +impl AsRawSocket for Socket { + fn as_raw_socket(&self) -> RawSocket { + self.0.as_raw_socket() + } +} + +impl IntoRawSocket for Socket { + fn into_raw_socket(self) -> RawSocket { + self.0.into_raw_socket() } } -impl IntoInner for Socket { - fn into_inner(self) -> c::SOCKET { - let ret = self.0; - mem::forget(self); - ret +impl FromRawSocket for Socket { + unsafe fn from_raw_socket(raw_socket: RawSocket) -> Self { + Self(FromRawSocket::from_raw_socket(raw_socket)) } } diff --git a/crux-mir/lib/std/src/sys/windows/os.rs b/crux-mir/lib/std/src/sys/windows/os.rs index cc4ae4059..d7adeb266 100644 --- a/crux-mir/lib/std/src/sys/windows/os.rs +++ b/crux-mir/lib/std/src/sys/windows/os.rs @@ -2,6 +2,9 @@ #![allow(nonstandard_style)] +#[cfg(test)] +mod tests; + use crate::os::windows::prelude::*; use crate::error::Error as StdError; @@ -61,7 +64,7 @@ pub fn error_string(mut errnum: i32) -> String { if res == 0 { // Sometimes FormatMessageW can fail e.g., system doesn't like langId, let fm_err = errno(); - return format!("OS Error {} (FormatMessageW() returned error {})", errnum, fm_err); + return format!("OS Error {errnum} (FormatMessageW() returned error {fm_err})"); } match String::from_utf16(&buf[..res]) { @@ -94,13 +97,13 @@ impl Iterator for Env { if *self.cur == 0 { return None; } - let p = &*self.cur as *const u16; + let p = self.cur as *const u16; let mut len = 0; - while *p.offset(len) != 0 { + while *p.add(len) != 0 { len += 1; } - let s = slice::from_raw_parts(p, len as usize); - self.cur = self.cur.offset(len + 1); + let s = slice::from_raw_parts(p, len); + self.cur = self.cur.add(len + 1); // Windows allows environment variables to start with an equals // symbol (in any other position, this is the separator between @@ -131,7 +134,7 @@ impl Drop for Env { pub fn env() -> Env { unsafe { let ch = c::GetEnvironmentStringsW(); - if ch as usize == 0 { + if ch.is_null() { panic!("failure getting env string from OS: {}", io::Error::last_os_error()); } Env { base: ch, cur: ch } @@ -154,7 +157,7 @@ impl<'a> Iterator for SplitPaths<'a> { // Double quotes are used as a way of introducing literal semicolons // (since c:\some;dir is a valid Windows path). Double quotes are not // themselves permitted in path names, so there is no way to escape a - // double quote. Quoted regions can appear in arbitrary locations, so + // double quote. Quoted regions can appear in arbitrary locations, so // // c:\foo;c:\som"e;di"r;c:\bar // @@ -250,22 +253,13 @@ pub fn chdir(p: &path::Path) -> io::Result<()> { cvt(unsafe { c::SetCurrentDirectoryW(p.as_ptr()) }).map(drop) } -pub fn getenv(k: &OsStr) -> io::Result> { - let k = to_u16s(k)?; - let res = super::fill_utf16_buf( +pub fn getenv(k: &OsStr) -> Option { + let k = to_u16s(k).ok()?; + super::fill_utf16_buf( |buf, sz| unsafe { c::GetEnvironmentVariableW(k.as_ptr(), buf, sz) }, |buf| OsStringExt::from_wide(buf), - ); - match res { - Ok(value) => Ok(Some(value)), - Err(e) => { - if e.raw_os_error() == Some(c::ERROR_ENVVAR_NOT_FOUND as i32) { - Ok(None) - } else { - Err(e) - } - } - } + ) + .ok() } pub fn setenv(k: &OsStr, v: &OsStr) -> io::Result<()> { @@ -281,12 +275,16 @@ pub fn unsetenv(n: &OsStr) -> io::Result<()> { } pub fn temp_dir() -> PathBuf { - super::fill_utf16_buf(|buf, sz| unsafe { c::GetTempPathW(sz, buf) }, super::os2path).unwrap() + super::fill_utf16_buf(|buf, sz| unsafe { c::GetTempPath2W(sz, buf) }, super::os2path).unwrap() } #[cfg(not(target_vendor = "uwp"))] fn home_dir_crt() -> Option { unsafe { + // The magic constant -4 can be used as the token passed to GetUserProfileDirectoryW below + // instead of us having to go through these multiple steps to get a token. However this is + // not implemented on Windows 7, only Windows 8 and up. When we drop support for Windows 7 + // we can simplify this code. See #90144 for details. use crate::sys::handle::Handle; let me = c::GetCurrentProcess(); @@ -294,7 +292,7 @@ fn home_dir_crt() -> Option { if c::OpenProcessToken(me, c::TOKEN_READ, &mut token) == 0 { return None; } - let _handle = Handle::new(token); + let _handle = Handle::from_raw_handle(token); super::fill_utf16_buf( |buf, mut sz| { match c::GetUserProfileDirectoryW(token, buf, &mut sz) { @@ -328,20 +326,3 @@ pub fn exit(code: i32) -> ! { pub fn getpid() -> u32 { unsafe { c::GetCurrentProcessId() as u32 } } - -#[cfg(test)] -mod tests { - use crate::io::Error; - use crate::sys::c; - - // tests `error_string` above - #[test] - fn ntstatus_error() { - const STATUS_UNSUCCESSFUL: u32 = 0xc000_0001; - assert!( - !Error::from_raw_os_error((STATUS_UNSUCCESSFUL | c::FACILITY_NT_BIT) as _) - .to_string() - .contains("FormatMessageW() returned error") - ); - } -} diff --git a/crux-mir/lib/std/src/sys/windows/os/tests.rs b/crux-mir/lib/std/src/sys/windows/os/tests.rs new file mode 100644 index 000000000..458d6e11c --- /dev/null +++ b/crux-mir/lib/std/src/sys/windows/os/tests.rs @@ -0,0 +1,13 @@ +use crate::io::Error; +use crate::sys::c; + +// tests `error_string` above +#[test] +fn ntstatus_error() { + const STATUS_UNSUCCESSFUL: u32 = 0xc000_0001; + assert!( + !Error::from_raw_os_error((STATUS_UNSUCCESSFUL | c::FACILITY_NT_BIT) as _) + .to_string() + .contains("FormatMessageW() returned error") + ); +} diff --git a/crux-mir/lib/std/src/sys/windows/os_str.rs b/crux-mir/lib/std/src/sys/windows/os_str.rs index ef260f9c5..4bdd8c505 100644 --- a/crux-mir/lib/std/src/sys/windows/os_str.rs +++ b/crux-mir/lib/std/src/sys/windows/os_str.rs @@ -1,6 +1,7 @@ /// The underlying OsString/OsStr implementation on Windows is a /// wrapper around the "WTF-8" encoding; see the `wtf8` module for more. use crate::borrow::Cow; +use crate::collections::TryReserveError; use crate::fmt; use crate::mem; use crate::rc::Rc; @@ -43,6 +44,7 @@ impl fmt::Display for Buf { } } +#[repr(transparent)] pub struct Slice { pub inner: Wtf8, } @@ -77,9 +79,21 @@ impl Buf { } pub fn as_slice(&self) -> &Slice { + // SAFETY: Slice is just a wrapper for Wtf8, + // and self.inner.as_slice() returns &Wtf8. + // Therefore, transmuting &Wtf8 to &Slice is safe. unsafe { mem::transmute(self.inner.as_slice()) } } + pub fn as_mut_slice(&mut self) -> &mut Slice { + // SAFETY: Slice is just a wrapper for Wtf8, + // and self.inner.as_mut_slice() returns &mut Wtf8. + // Therefore, transmuting &mut Wtf8 to &mut Slice is safe. + // Additionally, care should be taken to ensure the slice + // is always valid Wtf8. + unsafe { mem::transmute(self.inner.as_mut_slice()) } + } + pub fn into_string(self) -> Result { self.inner.into_string().map_err(|buf| Buf { inner: buf }) } @@ -92,10 +106,18 @@ impl Buf { self.inner.reserve(additional) } + pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.inner.try_reserve(additional) + } + pub fn reserve_exact(&mut self, additional: usize) { self.inner.reserve_exact(additional) } + pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.inner.try_reserve_exact(additional) + } + pub fn shrink_to_fit(&mut self) { self.inner.shrink_to_fit() } @@ -142,9 +164,11 @@ impl Slice { } pub fn to_owned(&self) -> Buf { - let mut buf = Wtf8Buf::with_capacity(self.inner.len()); - buf.push_wtf8(&self.inner); - Buf { inner: buf } + Buf { inner: self.inner.to_owned() } + } + + pub fn clone_into(&self, buf: &mut Buf) { + self.inner.clone_into(&mut buf.inner) } #[inline] @@ -167,4 +191,34 @@ impl Slice { let rc = self.inner.into_rc(); unsafe { Rc::from_raw(Rc::into_raw(rc) as *const Slice) } } + + #[inline] + pub fn make_ascii_lowercase(&mut self) { + self.inner.make_ascii_lowercase() + } + + #[inline] + pub fn make_ascii_uppercase(&mut self) { + self.inner.make_ascii_uppercase() + } + + #[inline] + pub fn to_ascii_lowercase(&self) -> Buf { + Buf { inner: self.inner.to_ascii_lowercase() } + } + + #[inline] + pub fn to_ascii_uppercase(&self) -> Buf { + Buf { inner: self.inner.to_ascii_uppercase() } + } + + #[inline] + pub fn is_ascii(&self) -> bool { + self.inner.is_ascii() + } + + #[inline] + pub fn eq_ignore_ascii_case(&self, other: &Self) -> bool { + self.inner.eq_ignore_ascii_case(&other.inner) + } } diff --git a/crux-mir/lib/std/src/sys/windows/path.rs b/crux-mir/lib/std/src/sys/windows/path.rs index 524f21f88..beeca1917 100644 --- a/crux-mir/lib/std/src/sys/windows/path.rs +++ b/crux-mir/lib/std/src/sys/windows/path.rs @@ -1,12 +1,24 @@ -use crate::ffi::OsStr; +use super::{c, fill_utf16_buf, to_u16s}; +use crate::ffi::{OsStr, OsString}; +use crate::io; use crate::mem; -use crate::path::Prefix; +use crate::path::{Path, PathBuf, Prefix}; +use crate::ptr; -fn os_str_as_u8_slice(s: &OsStr) -> &[u8] { - unsafe { mem::transmute(s) } -} -unsafe fn u8_slice_as_os_str(s: &[u8]) -> &OsStr { - mem::transmute(s) +#[cfg(test)] +mod tests; + +pub const MAIN_SEP_STR: &str = "\\"; +pub const MAIN_SEP: char = '\\'; + +/// # Safety +/// +/// `bytes` must be a valid wtf8 encoded slice +#[inline] +unsafe fn bytes_as_os_str(bytes: &[u8]) -> &OsStr { + // &OsStr is layout compatible with &Slice, which is compatible with &Wtf8, + // which is compatible with &[u8]. + mem::transmute(bytes) } #[inline] @@ -19,76 +31,303 @@ pub fn is_verbatim_sep(b: u8) -> bool { b == b'\\' } +/// Returns true if `path` looks like a lone filename. +pub(crate) fn is_file_name(path: &OsStr) -> bool { + !path.bytes().iter().copied().any(is_sep_byte) +} +pub(crate) fn has_trailing_slash(path: &OsStr) -> bool { + let is_verbatim = path.bytes().starts_with(br"\\?\"); + let is_separator = if is_verbatim { is_verbatim_sep } else { is_sep_byte }; + if let Some(&c) = path.bytes().last() { is_separator(c) } else { false } +} + +/// Appends a suffix to a path. +/// +/// Can be used to append an extension without removing an existing extension. +pub(crate) fn append_suffix(path: PathBuf, suffix: &OsStr) -> PathBuf { + let mut path = OsString::from(path); + path.push(suffix); + path.into() +} + +struct PrefixParser<'a, const LEN: usize> { + path: &'a OsStr, + prefix: [u8; LEN], +} + +impl<'a, const LEN: usize> PrefixParser<'a, LEN> { + #[inline] + fn get_prefix(path: &OsStr) -> [u8; LEN] { + let mut prefix = [0; LEN]; + // SAFETY: Only ASCII characters are modified. + for (i, &ch) in path.bytes().iter().take(LEN).enumerate() { + prefix[i] = if ch == b'/' { b'\\' } else { ch }; + } + prefix + } + + fn new(path: &'a OsStr) -> Self { + Self { path, prefix: Self::get_prefix(path) } + } + + fn as_slice(&self) -> PrefixParserSlice<'a, '_> { + PrefixParserSlice { + path: self.path, + prefix: &self.prefix[..LEN.min(self.path.len())], + index: 0, + } + } +} + +struct PrefixParserSlice<'a, 'b> { + path: &'a OsStr, + prefix: &'b [u8], + index: usize, +} + +impl<'a> PrefixParserSlice<'a, '_> { + fn strip_prefix(&self, prefix: &str) -> Option { + self.prefix[self.index..] + .starts_with(prefix.as_bytes()) + .then(|| Self { index: self.index + prefix.len(), ..*self }) + } + + fn prefix_bytes(&self) -> &'a [u8] { + &self.path.bytes()[..self.index] + } + + fn finish(self) -> &'a OsStr { + // SAFETY: The unsafety here stems from converting between &OsStr and + // &[u8] and back. This is safe to do because (1) we only look at ASCII + // contents of the encoding and (2) new &OsStr values are produced only + // from ASCII-bounded slices of existing &OsStr values. + unsafe { bytes_as_os_str(&self.path.bytes()[self.index..]) } + } +} + pub fn parse_prefix(path: &OsStr) -> Option> { - use crate::path::Prefix::*; - unsafe { - // The unsafety here stems from converting between &OsStr and &[u8] - // and back. This is safe to do because (1) we only look at ASCII - // contents of the encoding and (2) new &OsStr values are produced - // only from ASCII-bounded slices of existing &OsStr values. - let mut path = os_str_as_u8_slice(path); - - if path.starts_with(br"\\") { - // \\ - path = &path[2..]; - if path.starts_with(br"?\") { - // \\?\ - path = &path[2..]; - if path.starts_with(br"UNC\") { - // \\?\UNC\server\share - path = &path[4..]; - let (server, share) = match parse_two_comps(path, is_verbatim_sep) { - Some((server, share)) => { - (u8_slice_as_os_str(server), u8_slice_as_os_str(share)) - } - None => (u8_slice_as_os_str(path), u8_slice_as_os_str(&[])), - }; - return Some(VerbatimUNC(server, share)); + use Prefix::{DeviceNS, Disk, Verbatim, VerbatimDisk, VerbatimUNC, UNC}; + + let parser = PrefixParser::<8>::new(path); + let parser = parser.as_slice(); + if let Some(parser) = parser.strip_prefix(r"\\") { + // \\ + + // The meaning of verbatim paths can change when they use a different + // separator. + if let Some(parser) = parser.strip_prefix(r"?\") && !parser.prefix_bytes().iter().any(|&x| x == b'/') { + // \\?\ + if let Some(parser) = parser.strip_prefix(r"UNC\") { + // \\?\UNC\server\share + + let path = parser.finish(); + let (server, path) = parse_next_component(path, true); + let (share, _) = parse_next_component(path, true); + + Some(VerbatimUNC(server, share)) + } else { + let path = parser.finish(); + + // in verbatim paths only recognize an exact drive prefix + if let Some(drive) = parse_drive_exact(path) { + // \\?\C: + Some(VerbatimDisk(drive)) } else { - // \\?\path - let idx = path.iter().position(|&b| b == b'\\'); - if idx == Some(2) && path[1] == b':' { - let c = path[0]; - if c.is_ascii() && (c as char).is_alphabetic() { - // \\?\C:\ path - return Some(VerbatimDisk(c.to_ascii_uppercase())); - } - } - let slice = &path[..idx.unwrap_or(path.len())]; - return Some(Verbatim(u8_slice_as_os_str(slice))); + // \\?\prefix + let (prefix, _) = parse_next_component(path, true); + Some(Verbatim(prefix)) } - } else if path.starts_with(b".\\") { - // \\.\path - path = &path[2..]; - let pos = path.iter().position(|&b| b == b'\\'); - let slice = &path[..pos.unwrap_or(path.len())]; - return Some(DeviceNS(u8_slice_as_os_str(slice))); } - match parse_two_comps(path, is_sep_byte) { - Some((server, share)) if !server.is_empty() && !share.is_empty() => { - // \\server\share - return Some(UNC(u8_slice_as_os_str(server), u8_slice_as_os_str(share))); - } - _ => (), - } - } else if path.get(1) == Some(&b':') { - // C: - let c = path[0]; - if c.is_ascii() && (c as char).is_alphabetic() { - return Some(Disk(c.to_ascii_uppercase())); + } else if let Some(parser) = parser.strip_prefix(r".\") { + // \\.\COM42 + let path = parser.finish(); + let (prefix, _) = parse_next_component(path, false); + Some(DeviceNS(prefix)) + } else { + let path = parser.finish(); + let (server, path) = parse_next_component(path, false); + let (share, _) = parse_next_component(path, false); + + if !server.is_empty() && !share.is_empty() { + // \\server\share + Some(UNC(server, share)) + } else { + // no valid prefix beginning with "\\" recognized + None } } - return None; + } else if let Some(drive) = parse_drive(path) { + // C: + Some(Disk(drive)) + } else { + // no prefix + None } +} - fn parse_two_comps(mut path: &[u8], f: fn(u8) -> bool) -> Option<(&[u8], &[u8])> { - let first = &path[..path.iter().position(|x| f(*x))?]; - path = &path[(first.len() + 1)..]; - let idx = path.iter().position(|x| f(*x)); - let second = &path[..idx.unwrap_or(path.len())]; - Some((first, second)) +// Parses a drive prefix, e.g. "C:" and "C:\whatever" +fn parse_drive(path: &OsStr) -> Option { + // In most DOS systems, it is not possible to have more than 26 drive letters. + // See . + fn is_valid_drive_letter(drive: &u8) -> bool { + drive.is_ascii_alphabetic() + } + + match path.bytes() { + [drive, b':', ..] if is_valid_drive_letter(drive) => Some(drive.to_ascii_uppercase()), + _ => None, } } -pub const MAIN_SEP_STR: &str = "\\"; -pub const MAIN_SEP: char = '\\'; +// Parses a drive prefix exactly, e.g. "C:" +fn parse_drive_exact(path: &OsStr) -> Option { + // only parse two bytes: the drive letter and the drive separator + if path.bytes().get(2).map(|&x| is_sep_byte(x)).unwrap_or(true) { + parse_drive(path) + } else { + None + } +} + +// Parse the next path component. +// +// Returns the next component and the rest of the path excluding the component and separator. +// Does not recognize `/` as a separator character if `verbatim` is true. +fn parse_next_component(path: &OsStr, verbatim: bool) -> (&OsStr, &OsStr) { + let separator = if verbatim { is_verbatim_sep } else { is_sep_byte }; + + match path.bytes().iter().position(|&x| separator(x)) { + Some(separator_start) => { + let separator_end = separator_start + 1; + + let component = &path.bytes()[..separator_start]; + + // Panic safe + // The max `separator_end` is `bytes.len()` and `bytes[bytes.len()..]` is a valid index. + let path = &path.bytes()[separator_end..]; + + // SAFETY: `path` is a valid wtf8 encoded slice and each of the separators ('/', '\') + // is encoded in a single byte, therefore `bytes[separator_start]` and + // `bytes[separator_end]` must be code point boundaries and thus + // `bytes[..separator_start]` and `bytes[separator_end..]` are valid wtf8 slices. + unsafe { (bytes_as_os_str(component), bytes_as_os_str(path)) } + } + None => (path, OsStr::new("")), + } +} + +/// Returns a UTF-16 encoded path capable of bypassing the legacy `MAX_PATH` limits. +/// +/// This path may or may not have a verbatim prefix. +pub(crate) fn maybe_verbatim(path: &Path) -> io::Result> { + // Normally the MAX_PATH is 260 UTF-16 code units (including the NULL). + // However, for APIs such as CreateDirectory[1], the limit is 248. + // + // [1]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createdirectorya#parameters + const LEGACY_MAX_PATH: usize = 248; + // UTF-16 encoded code points, used in parsing and building UTF-16 paths. + // All of these are in the ASCII range so they can be cast directly to `u16`. + const SEP: u16 = b'\\' as _; + const ALT_SEP: u16 = b'/' as _; + const QUERY: u16 = b'?' as _; + const COLON: u16 = b':' as _; + const DOT: u16 = b'.' as _; + const U: u16 = b'U' as _; + const N: u16 = b'N' as _; + const C: u16 = b'C' as _; + + // \\?\ + const VERBATIM_PREFIX: &[u16] = &[SEP, SEP, QUERY, SEP]; + // \??\ + const NT_PREFIX: &[u16] = &[SEP, QUERY, QUERY, SEP]; + // \\?\UNC\ + const UNC_PREFIX: &[u16] = &[SEP, SEP, QUERY, SEP, U, N, C, SEP]; + + let mut path = to_u16s(path)?; + if path.starts_with(VERBATIM_PREFIX) || path.starts_with(NT_PREFIX) || path == &[0] { + // Early return for paths that are already verbatim or empty. + return Ok(path); + } else if path.len() < LEGACY_MAX_PATH { + // Early return if an absolute path is less < 260 UTF-16 code units. + // This is an optimization to avoid calling `GetFullPathNameW` unnecessarily. + match path.as_slice() { + // Starts with `D:`, `D:\`, `D:/`, etc. + // Does not match if the path starts with a `\` or `/`. + [drive, COLON, 0] | [drive, COLON, SEP | ALT_SEP, ..] + if *drive != SEP && *drive != ALT_SEP => + { + return Ok(path); + } + // Starts with `\\`, `//`, etc + [SEP | ALT_SEP, SEP | ALT_SEP, ..] => return Ok(path), + _ => {} + } + } + + // Firstly, get the absolute path using `GetFullPathNameW`. + // https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getfullpathnamew + let lpfilename = path.as_ptr(); + fill_utf16_buf( + // SAFETY: `fill_utf16_buf` ensures the `buffer` and `size` are valid. + // `lpfilename` is a pointer to a null terminated string that is not + // invalidated until after `GetFullPathNameW` returns successfully. + |buffer, size| unsafe { c::GetFullPathNameW(lpfilename, size, buffer, ptr::null_mut()) }, + |mut absolute| { + path.clear(); + + // Secondly, add the verbatim prefix. This is easier here because we know the + // path is now absolute and fully normalized (e.g. `/` has been changed to `\`). + let prefix = match absolute { + // C:\ => \\?\C:\ + [_, COLON, SEP, ..] => VERBATIM_PREFIX, + // \\.\ => \\?\ + [SEP, SEP, DOT, SEP, ..] => { + absolute = &absolute[4..]; + VERBATIM_PREFIX + } + // Leave \\?\ and \??\ as-is. + [SEP, SEP, QUERY, SEP, ..] | [SEP, QUERY, QUERY, SEP, ..] => &[], + // \\ => \\?\UNC\ + [SEP, SEP, ..] => { + absolute = &absolute[2..]; + UNC_PREFIX + } + // Anything else we leave alone. + _ => &[], + }; + + path.reserve_exact(prefix.len() + absolute.len() + 1); + path.extend_from_slice(prefix); + path.extend_from_slice(absolute); + path.push(0); + }, + )?; + Ok(path) +} + +/// Make a Windows path absolute. +pub(crate) fn absolute(path: &Path) -> io::Result { + let path = path.as_os_str(); + let prefix = parse_prefix(path); + // Verbatim paths should not be modified. + if prefix.map(|x| x.is_verbatim()).unwrap_or(false) { + // NULs in verbatim paths are rejected for consistency. + if path.bytes().contains(&0) { + return Err(io::const_io_error!( + io::ErrorKind::InvalidInput, + "strings passed to WinAPI cannot contain NULs", + )); + } + return Ok(path.to_owned().into()); + } + + let path = to_u16s(path)?; + let lpfilename = path.as_ptr(); + fill_utf16_buf( + // SAFETY: `fill_utf16_buf` ensures the `buffer` and `size` are valid. + // `lpfilename` is a pointer to a null terminated string that is not + // invalidated until after `GetFullPathNameW` returns successfully. + |buffer, size| unsafe { c::GetFullPathNameW(lpfilename, size, buffer, ptr::null_mut()) }, + super::os2path, + ) +} diff --git a/crux-mir/lib/std/src/sys/windows/path/tests.rs b/crux-mir/lib/std/src/sys/windows/path/tests.rs new file mode 100644 index 000000000..623c62361 --- /dev/null +++ b/crux-mir/lib/std/src/sys/windows/path/tests.rs @@ -0,0 +1,137 @@ +use super::*; + +#[test] +fn test_parse_next_component() { + assert_eq!( + parse_next_component(OsStr::new(r"server\share"), true), + (OsStr::new(r"server"), OsStr::new(r"share")) + ); + + assert_eq!( + parse_next_component(OsStr::new(r"server/share"), true), + (OsStr::new(r"server/share"), OsStr::new(r"")) + ); + + assert_eq!( + parse_next_component(OsStr::new(r"server/share"), false), + (OsStr::new(r"server"), OsStr::new(r"share")) + ); + + assert_eq!( + parse_next_component(OsStr::new(r"server\"), false), + (OsStr::new(r"server"), OsStr::new(r"")) + ); + + assert_eq!( + parse_next_component(OsStr::new(r"\server\"), false), + (OsStr::new(r""), OsStr::new(r"server\")) + ); + + assert_eq!( + parse_next_component(OsStr::new(r"servershare"), false), + (OsStr::new(r"servershare"), OsStr::new("")) + ); +} + +#[test] +fn verbatim() { + use crate::path::Path; + fn check(path: &str, expected: &str) { + let verbatim = maybe_verbatim(Path::new(path)).unwrap(); + let verbatim = String::from_utf16_lossy(verbatim.strip_suffix(&[0]).unwrap()); + assert_eq!(&verbatim, expected, "{}", path); + } + + // Ensure long paths are correctly prefixed. + check( + r"C:\Program Files\Rust\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\foo.txt", + r"\\?\C:\Program Files\Rust\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\foo.txt", + ); + check( + r"\\server\share\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\foo.txt", + r"\\?\UNC\server\share\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\foo.txt", + ); + check( + r"\\.\PIPE\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\foo.txt", + r"\\?\PIPE\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\foo.txt", + ); + // `\\?\` prefixed paths are left unchanged... + check( + r"\\?\verbatim.\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\foo.txt", + r"\\?\verbatim.\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\foo.txt", + ); + // But `//?/` is not a verbatim prefix so it will be normalized. + check( + r"//?/E:/verbatim.\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\foo.txt", + r"\\?\E:\verbatim\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\foo.txt", + ); + + // For performance, short absolute paths are left unchanged. + check(r"C:\Program Files\Rust", r"C:\Program Files\Rust"); + check(r"\\server\share", r"\\server\share"); + check(r"\\.\COM1", r"\\.\COM1"); + + // Check that paths of length 247 are converted to verbatim. + // This is necessary for `CreateDirectory`. + check( + r"C:\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + r"\\?\C:\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ); + + // Make sure opening a drive will work. + check("Z:", "Z:"); + + // A path that contains null is not a valid path. + assert!(maybe_verbatim(Path::new("\0")).is_err()); +} + +fn parse_prefix(path: &str) -> Option> { + super::parse_prefix(OsStr::new(path)) +} + +#[test] +fn test_parse_prefix_verbatim() { + let prefix = Some(Prefix::VerbatimDisk(b'C')); + assert_eq!(prefix, parse_prefix(r"\\?\C:/windows/system32/notepad.exe")); + assert_eq!(prefix, parse_prefix(r"\\?\C:\windows\system32\notepad.exe")); +} + +#[test] +fn test_parse_prefix_verbatim_device() { + let prefix = Some(Prefix::UNC(OsStr::new("?"), OsStr::new("C:"))); + assert_eq!(prefix, parse_prefix(r"//?/C:/windows/system32/notepad.exe")); + assert_eq!(prefix, parse_prefix(r"//?/C:\windows\system32\notepad.exe")); + assert_eq!(prefix, parse_prefix(r"/\?\C:\windows\system32\notepad.exe")); + assert_eq!(prefix, parse_prefix(r"\\?/C:\windows\system32\notepad.exe")); +} + +// See #93586 for more information. +#[test] +fn test_windows_prefix_components() { + use crate::path::Path; + + let path = Path::new("C:"); + let mut components = path.components(); + let drive = components.next().expect("drive is expected here"); + assert_eq!(drive.as_os_str(), OsStr::new("C:")); + assert_eq!(components.as_path(), Path::new("")); +} + +/// See #101358. +/// +/// Note that the exact behaviour here may change in the future. +/// In which case this test will need to adjusted. +#[test] +fn broken_unc_path() { + use crate::path::Component; + + let mut components = Path::new(r"\\foo\\bar\\").components(); + assert_eq!(components.next(), Some(Component::RootDir)); + assert_eq!(components.next(), Some(Component::Normal("foo".as_ref()))); + assert_eq!(components.next(), Some(Component::Normal("bar".as_ref()))); + + let mut components = Path::new("//foo//bar//").components(); + assert_eq!(components.next(), Some(Component::RootDir)); + assert_eq!(components.next(), Some(Component::Normal("foo".as_ref()))); + assert_eq!(components.next(), Some(Component::Normal("bar".as_ref()))); +} diff --git a/crux-mir/lib/std/src/sys/windows/pipe.rs b/crux-mir/lib/std/src/sys/windows/pipe.rs index 992e634de..7b25edaa5 100644 --- a/crux-mir/lib/std/src/sys/windows/pipe.rs +++ b/crux-mir/lib/std/src/sys/windows/pipe.rs @@ -1,7 +1,7 @@ use crate::os::windows::prelude::*; use crate::ffi::OsStr; -use crate::io::{self, IoSlice, IoSliceMut}; +use crate::io::{self, IoSlice, IoSliceMut, Read}; use crate::mem; use crate::path::Path; use crate::ptr; @@ -12,6 +12,7 @@ use crate::sys::c; use crate::sys::fs::{File, OpenOptions}; use crate::sys::handle::Handle; use crate::sys::hashmap_random_keys; +use crate::sys_common::IntoInner; //////////////////////////////////////////////////////////////////////////////// // Anonymous pipes @@ -21,6 +22,12 @@ pub struct AnonPipe { inner: Handle, } +impl IntoInner for AnonPipe { + fn into_inner(self) -> Handle { + self.inner + } +} + pub struct Pipes { pub ours: AnonPipe, pub theirs: AnonPipe, @@ -46,6 +53,9 @@ pub struct Pipes { /// with `OVERLAPPED` instances, but also works out ok if it's only ever used /// once at a time (which we do indeed guarantee). pub fn anon_pipe(ours_readable: bool, their_handle_inheritable: bool) -> io::Result { + // A 64kb pipe capacity is the same as a typical Linux default. + const PIPE_BUFFER_CAPACITY: u32 = 64 * 1024; + // Note that we specifically do *not* use `CreatePipe` here because // unfortunately the anonymous pipes returned do not support overlapped // operations. Instead, we create a "hopefully unique" name and create a @@ -84,8 +94,8 @@ pub fn anon_pipe(ours_readable: bool, their_handle_inheritable: bool) -> io::Res | c::PIPE_WAIT | reject_remote_clients_flag, 1, - 4096, - 4096, + PIPE_BUFFER_CAPACITY, + PIPE_BUFFER_CAPACITY, 0, ptr::null_mut(), ); @@ -123,7 +133,7 @@ pub fn anon_pipe(ours_readable: bool, their_handle_inheritable: bool) -> io::Res } return Err(err); } - ours = Handle::new(handle); + ours = Handle::from_raw_handle(handle); break; } @@ -146,15 +156,55 @@ pub fn anon_pipe(ours_readable: bool, their_handle_inheritable: bool) -> io::Res }; opts.security_attributes(&mut sa); let theirs = File::open(Path::new(&name), &opts)?; - let theirs = AnonPipe { inner: theirs.into_handle() }; + let theirs = AnonPipe { inner: theirs.into_inner() }; Ok(Pipes { ours: AnonPipe { inner: ours }, - theirs: AnonPipe { inner: theirs.into_handle() }, + theirs: AnonPipe { inner: theirs.into_inner() }, }) } } +/// Takes an asynchronous source pipe and returns a synchronous pipe suitable +/// for sending to a child process. +/// +/// This is achieved by creating a new set of pipes and spawning a thread that +/// relays messages between the source and the synchronous pipe. +pub fn spawn_pipe_relay( + source: &AnonPipe, + ours_readable: bool, + their_handle_inheritable: bool, +) -> io::Result { + // We need this handle to live for the lifetime of the thread spawned below. + let source = source.duplicate()?; + + // create a new pair of anon pipes. + let Pipes { theirs, ours } = anon_pipe(ours_readable, their_handle_inheritable)?; + + // Spawn a thread that passes messages from one pipe to the other. + // Any errors will simply cause the thread to exit. + let (reader, writer) = if ours_readable { (ours, source) } else { (source, ours) }; + crate::thread::spawn(move || { + let mut buf = [0_u8; 4096]; + 'reader: while let Ok(len) = reader.read(&mut buf) { + if len == 0 { + break; + } + let mut start = 0; + while let Ok(written) = writer.write(&buf[start..len]) { + start += written; + if start == len { + continue 'reader; + } + } + break; + } + }); + + // Return the pipe that should be sent to the child process. + Ok(theirs) +} + fn random_number() -> usize { static N: AtomicUsize = AtomicUsize::new(0); loop { @@ -166,6 +216,15 @@ fn random_number() -> usize { } } +// Abstracts over `ReadFileEx` and `WriteFileEx` +type AlertableIoFn = unsafe extern "system" fn( + BorrowedHandle<'_>, + c::LPVOID, + c::DWORD, + c::LPOVERLAPPED, + c::LPOVERLAPPED_COMPLETION_ROUTINE, +) -> c::BOOL; + impl AnonPipe { pub fn handle(&self) -> &Handle { &self.inner @@ -173,22 +232,148 @@ impl AnonPipe { pub fn into_handle(self) -> Handle { self.inner } + fn duplicate(&self) -> io::Result { + self.inner.duplicate(0, false, c::DUPLICATE_SAME_ACCESS).map(|inner| AnonPipe { inner }) + } pub fn read(&self, buf: &mut [u8]) -> io::Result { - self.inner.read(buf) + let result = unsafe { + let len = crate::cmp::min(buf.len(), c::DWORD::MAX as usize) as c::DWORD; + self.alertable_io_internal(c::ReadFileEx, buf.as_mut_ptr() as _, len) + }; + + match result { + // The special treatment of BrokenPipe is to deal with Windows + // pipe semantics, which yields this error when *reading* from + // a pipe after the other end has closed; we interpret that as + // EOF on the pipe. + Err(ref e) if e.kind() == io::ErrorKind::BrokenPipe => Ok(0), + _ => result, + } } pub fn read_vectored(&self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { self.inner.read_vectored(bufs) } + #[inline] + pub fn is_read_vectored(&self) -> bool { + self.inner.is_read_vectored() + } + + pub fn read_to_end(&self, buf: &mut Vec) -> io::Result { + self.handle().read_to_end(buf) + } + pub fn write(&self, buf: &[u8]) -> io::Result { - self.inner.write(buf) + unsafe { + let len = crate::cmp::min(buf.len(), c::DWORD::MAX as usize) as c::DWORD; + self.alertable_io_internal(c::WriteFileEx, buf.as_ptr() as _, len) + } } pub fn write_vectored(&self, bufs: &[IoSlice<'_>]) -> io::Result { self.inner.write_vectored(bufs) } + + #[inline] + pub fn is_write_vectored(&self) -> bool { + self.inner.is_write_vectored() + } + + /// Synchronizes asynchronous reads or writes using our anonymous pipe. + /// + /// This is a wrapper around [`ReadFileEx`] or [`WriteFileEx`] that uses + /// [Asynchronous Procedure Call] (APC) to synchronize reads or writes. + /// + /// Note: This should not be used for handles we don't create. + /// + /// # Safety + /// + /// `buf` must be a pointer to a buffer that's valid for reads or writes + /// up to `len` bytes. The `AlertableIoFn` must be either `ReadFileEx` or `WriteFileEx` + /// + /// [`ReadFileEx`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-readfileex + /// [`WriteFileEx`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-writefileex + /// [Asynchronous Procedure Call]: https://docs.microsoft.com/en-us/windows/win32/sync/asynchronous-procedure-calls + unsafe fn alertable_io_internal( + &self, + io: AlertableIoFn, + buf: c::LPVOID, + len: c::DWORD, + ) -> io::Result { + // Use "alertable I/O" to synchronize the pipe I/O. + // This has four steps. + // + // STEP 1: Start the asynchronous I/O operation. + // This simply calls either `ReadFileEx` or `WriteFileEx`, + // giving it a pointer to the buffer and callback function. + // + // STEP 2: Enter an alertable state. + // The callback set in step 1 will not be called until the thread + // enters an "alertable" state. This can be done using `SleepEx`. + // + // STEP 3: The callback + // Once the I/O is complete and the thread is in an alertable state, + // the callback will be run on the same thread as the call to + // `ReadFileEx` or `WriteFileEx` done in step 1. + // In the callback we simply set the result of the async operation. + // + // STEP 4: Return the result. + // At this point we'll have a result from the callback function + // and can simply return it. Note that we must not return earlier, + // while the I/O is still in progress. + + // The result that will be set from the asynchronous callback. + let mut async_result: Option = None; + struct AsyncResult { + error: u32, + transferred: u32, + } + + // STEP 3: The callback. + unsafe extern "system" fn callback( + dwErrorCode: u32, + dwNumberOfBytesTransferred: u32, + lpOverlapped: *mut c::OVERLAPPED, + ) { + // Set `async_result` using a pointer smuggled through `hEvent`. + let result = + AsyncResult { error: dwErrorCode, transferred: dwNumberOfBytesTransferred }; + *(*lpOverlapped).hEvent.cast::>() = Some(result); + } + + // STEP 1: Start the I/O operation. + let mut overlapped: c::OVERLAPPED = crate::mem::zeroed(); + // `hEvent` is unused by `ReadFileEx` and `WriteFileEx`. + // Therefore the documentation suggests using it to smuggle a pointer to the callback. + overlapped.hEvent = &mut async_result as *mut _ as *mut _; + + // Asynchronous read of the pipe. + // If successful, `callback` will be called once it completes. + let result = io(self.inner.as_handle(), buf, len, &mut overlapped, callback); + if result == c::FALSE { + // We can return here because the call failed. + // After this we must not return until the I/O completes. + return Err(io::Error::last_os_error()); + } + + // Wait indefinitely for the result. + let result = loop { + // STEP 2: Enter an alertable state. + // The second parameter of `SleepEx` is used to make this sleep alertable. + c::SleepEx(c::INFINITE, c::TRUE); + if let Some(result) = async_result { + break result; + } + }; + // STEP 4: Return the result. + // `async_result` is always `Some` at this point + match result.error { + c::ERROR_SUCCESS => Ok(result.transferred as usize), + error => Err(io::Error::from_raw_os_error(error as _)), + } + } } pub fn read2(p1: AnonPipe, v1: &mut Vec, p2: AnonPipe, v2: &mut Vec) -> io::Result<()> { @@ -197,7 +382,7 @@ pub fn read2(p1: AnonPipe, v1: &mut Vec, p2: AnonPipe, v2: &mut Vec) -> let mut p1 = AsyncPipe::new(p1, v1)?; let mut p2 = AsyncPipe::new(p2, v2)?; - let objs = [p1.event.raw(), p2.event.raw()]; + let objs = [p1.event.as_raw_handle(), p2.event.as_raw_handle()]; // In a loop we wait for either pipe's scheduled read operation to complete. // If the operation completes with 0 bytes, that means EOF was reached, in @@ -252,7 +437,7 @@ impl<'a> AsyncPipe<'a> { // I/O operation is successfully scheduled (what we want). let event = Handle::new_event(true, true)?; let mut overlapped: Box = unsafe { Box::new(mem::zeroed()) }; - overlapped.hEvent = event.raw(); + overlapped.hEvent = event.as_raw_handle(); Ok(AsyncPipe { pipe, overlapped, event, dst, state: State::NotReading }) } diff --git a/crux-mir/lib/std/src/sys/windows/process.rs b/crux-mir/lib/std/src/sys/windows/process.rs index a62a63739..10bc949e1 100644 --- a/crux-mir/lib/std/src/sys/windows/process.rs +++ b/crux-mir/lib/std/src/sys/windows/process.rs @@ -1,26 +1,33 @@ #![unstable(feature = "process_internals", issue = "none")] -use crate::borrow::Borrow; +#[cfg(test)] +mod tests; + +use crate::cmp; use crate::collections::BTreeMap; use crate::env; -use crate::env::split_paths; +use crate::env::consts::{EXE_EXTENSION, EXE_SUFFIX}; use crate::ffi::{OsStr, OsString}; use crate::fmt; -use crate::fs; use crate::io::{self, Error, ErrorKind}; use crate::mem; -use crate::os::windows::ffi::OsStrExt; -use crate::path::Path; +use crate::num::NonZeroI32; +use crate::os::windows::ffi::{OsStrExt, OsStringExt}; +use crate::os::windows::io::{AsHandle, AsRawHandle, BorrowedHandle, FromRawHandle, IntoRawHandle}; +use crate::path::{Path, PathBuf}; use crate::ptr; +use crate::sync::Mutex; +use crate::sys::args::{self, Arg}; use crate::sys::c; +use crate::sys::c::NonZeroDWORD; use crate::sys::cvt; use crate::sys::fs::{File, OpenOptions}; use crate::sys::handle::Handle; -use crate::sys::mutex::Mutex; +use crate::sys::path; use crate::sys::pipe::{self, AnonPipe}; use crate::sys::stdio; -use crate::sys_common::process::CommandEnv; -use crate::sys_common::{AsInner, FromInner, IntoInner}; +use crate::sys_common::process::{CommandEnv, CommandEnvs}; +use crate::sys_common::IntoInner; use libc::{c_void, EXIT_FAILURE, EXIT_SUCCESS}; @@ -28,39 +35,121 @@ use libc::{c_void, EXIT_FAILURE, EXIT_SUCCESS}; // Command //////////////////////////////////////////////////////////////////////////////// -#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] +#[derive(Clone, Debug, Eq)] #[doc(hidden)] -pub struct EnvKey(OsString); +pub struct EnvKey { + os_string: OsString, + // This stores a UTF-16 encoded string to workaround the mismatch between + // Rust's OsString (WTF-8) and the Windows API string type (UTF-16). + // Normally converting on every API call is acceptable but here + // `c::CompareStringOrdinal` will be called for every use of `==`. + utf16: Vec, +} + +impl EnvKey { + fn new>(key: T) -> Self { + EnvKey::from(key.into()) + } +} + +// Comparing Windows environment variable keys[1] are behaviourally the +// composition of two operations[2]: +// +// 1. Case-fold both strings. This is done using a language-independent +// uppercase mapping that's unique to Windows (albeit based on data from an +// older Unicode spec). It only operates on individual UTF-16 code units so +// surrogates are left unchanged. This uppercase mapping can potentially change +// between Windows versions. +// +// 2. Perform an ordinal comparison of the strings. A comparison using ordinal +// is just a comparison based on the numerical value of each UTF-16 code unit[3]. +// +// Because the case-folding mapping is unique to Windows and not guaranteed to +// be stable, we ask the OS to compare the strings for us. This is done by +// calling `CompareStringOrdinal`[4] with `bIgnoreCase` set to `TRUE`. +// +// [1] https://docs.microsoft.com/en-us/dotnet/standard/base-types/best-practices-strings#choosing-a-stringcomparison-member-for-your-method-call +// [2] https://docs.microsoft.com/en-us/dotnet/standard/base-types/best-practices-strings#stringtoupper-and-stringtolower +// [3] https://docs.microsoft.com/en-us/dotnet/api/system.stringcomparison?view=net-5.0#System_StringComparison_Ordinal +// [4] https://docs.microsoft.com/en-us/windows/win32/api/stringapiset/nf-stringapiset-comparestringordinal +impl Ord for EnvKey { + fn cmp(&self, other: &Self) -> cmp::Ordering { + unsafe { + let result = c::CompareStringOrdinal( + self.utf16.as_ptr(), + self.utf16.len() as _, + other.utf16.as_ptr(), + other.utf16.len() as _, + c::TRUE, + ); + match result { + c::CSTR_LESS_THAN => cmp::Ordering::Less, + c::CSTR_EQUAL => cmp::Ordering::Equal, + c::CSTR_GREATER_THAN => cmp::Ordering::Greater, + // `CompareStringOrdinal` should never fail so long as the parameters are correct. + _ => panic!("comparing environment keys failed: {}", Error::last_os_error()), + } + } + } +} +impl PartialOrd for EnvKey { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} +impl PartialEq for EnvKey { + fn eq(&self, other: &Self) -> bool { + if self.utf16.len() != other.utf16.len() { + false + } else { + self.cmp(other) == cmp::Ordering::Equal + } + } +} +impl PartialOrd for EnvKey { + fn partial_cmp(&self, other: &str) -> Option { + Some(self.cmp(&EnvKey::new(other))) + } +} +impl PartialEq for EnvKey { + fn eq(&self, other: &str) -> bool { + if self.os_string.len() != other.len() { + false + } else { + self.cmp(&EnvKey::new(other)) == cmp::Ordering::Equal + } + } +} +// Environment variable keys should preserve their original case even though +// they are compared using a caseless string mapping. impl From for EnvKey { fn from(k: OsString) -> Self { - let mut buf = k.into_inner().into_inner(); - buf.make_ascii_uppercase(); - EnvKey(FromInner::from_inner(FromInner::from_inner(buf))) + EnvKey { utf16: k.encode_wide().collect(), os_string: k } } } impl From for OsString { fn from(k: EnvKey) -> Self { - k.0 + k.os_string } } -impl Borrow for EnvKey { - fn borrow(&self) -> &OsStr { - &self.0 +impl From<&OsStr> for EnvKey { + fn from(k: &OsStr) -> Self { + Self::from(k.to_os_string()) } } impl AsRef for EnvKey { fn as_ref(&self) -> &OsStr { - &self.0 + &self.os_string } } -fn ensure_no_nuls>(str: T) -> io::Result { +pub(crate) fn ensure_no_nuls>(str: T) -> io::Result { if str.as_ref().encode_wide().any(|b| b == 0) { - Err(io::Error::new(ErrorKind::InvalidInput, "nul byte found in provided data")) + Err(io::const_io_error!(ErrorKind::InvalidInput, "nul byte found in provided data")) } else { Ok(str) } @@ -68,7 +157,7 @@ fn ensure_no_nuls>(str: T) -> io::Result { pub struct Command { program: OsString, - args: Vec, + args: Vec, env: CommandEnv, cwd: Option, flags: u32, @@ -76,12 +165,14 @@ pub struct Command { stdin: Option, stdout: Option, stderr: Option, + force_quotes_enabled: bool, } pub enum Stdio { Inherit, Null, MakePipe, + Pipe(AnonPipe), Handle(Handle), } @@ -91,10 +182,6 @@ pub struct StdioPipes { pub stderr: Option, } -struct DropGuard<'a> { - lock: &'a Mutex, -} - impl Command { pub fn new(program: &OsStr) -> Command { Command { @@ -107,11 +194,12 @@ impl Command { stdin: None, stdout: None, stderr: None, + force_quotes_enabled: false, } } pub fn arg(&mut self, arg: &OsStr) { - self.args.push(arg.to_os_string()) + self.args.push(Arg::Regular(arg.to_os_string())) } pub fn env_mut(&mut self) -> &mut CommandEnv { &mut self.env @@ -132,37 +220,62 @@ impl Command { self.flags = flags; } + pub fn force_quotes(&mut self, enabled: bool) { + self.force_quotes_enabled = enabled; + } + + pub fn raw_arg(&mut self, command_str_to_append: &OsStr) { + self.args.push(Arg::Raw(command_str_to_append.to_os_string())) + } + + pub fn get_program(&self) -> &OsStr { + &self.program + } + + pub fn get_args(&self) -> CommandArgs<'_> { + let iter = self.args.iter(); + CommandArgs { iter } + } + + pub fn get_envs(&self) -> CommandEnvs<'_> { + self.env.iter() + } + + pub fn get_current_dir(&self) -> Option<&Path> { + self.cwd.as_ref().map(|cwd| Path::new(cwd)) + } + pub fn spawn( &mut self, default: Stdio, needs_stdin: bool, ) -> io::Result<(Process, StdioPipes)> { let maybe_env = self.env.capture_if_changed(); - // To have the spawning semantics of unix/windows stay the same, we need - // to read the *child's* PATH if one is provided. See #15149 for more - // details. - let program = maybe_env.as_ref().and_then(|env| { - if let Some(v) = env.get(OsStr::new("PATH")) { - // Split the value and test each path to see if the - // program exists. - for path in split_paths(&v) { - let path = path - .join(self.program.to_str().unwrap()) - .with_extension(env::consts::EXE_EXTENSION); - if fs::metadata(&path).is_ok() { - return Some(path.into_os_string()); - } - } - } - None - }); - - let mut si = zeroed_startupinfo(); - si.cb = mem::size_of::() as c::DWORD; - si.dwFlags = c::STARTF_USESTDHANDLES; - let program = program.as_ref().unwrap_or(&self.program); - let mut cmd_str = make_command_line(program, &self.args)?; + let child_paths = if let Some(env) = maybe_env.as_ref() { + env.get(&EnvKey::new("PATH")).map(|s| s.as_os_str()) + } else { + None + }; + let program = resolve_exe(&self.program, || env::var_os("PATH"), child_paths)?; + // Case insensitive "ends_with" of UTF-16 encoded ".bat" or ".cmd" + let is_batch_file = matches!( + program.len().checked_sub(5).and_then(|i| program.get(i..)), + Some([46, 98 | 66, 97 | 65, 116 | 84, 0] | [46, 99 | 67, 109 | 77, 100 | 68, 0]) + ); + let (program, mut cmd_str) = if is_batch_file { + ( + command_prompt()?, + args::make_bat_command_line( + &args::to_user_path(program)?, + &self.args, + self.force_quotes_enabled, + )?, + ) + } else { + let cmd_str = make_command_line(&self.program, &self.args, self.force_quotes_enabled)?; + (program, cmd_str) + }; cmd_str.push(0); // add null terminator // stolen from the libuv code. @@ -183,9 +296,10 @@ impl Command { // the remaining portion of this spawn in a mutex. // // For more information, msdn also has an article about this race: - // http://support.microsoft.com/kb/315939 - static CREATE_PROCESS_LOCK: Mutex = Mutex::new(); - let _guard = DropGuard::new(&CREATE_PROCESS_LOCK); + // https://support.microsoft.com/kb/315939 + static CREATE_PROCESS_LOCK: Mutex<()> = Mutex::new(()); + + let _guard = CREATE_PROCESS_LOCK.lock(); let mut pipes = StdioPipes { stdin: None, stdout: None, stderr: None }; let null = Stdio::Null; @@ -196,13 +310,25 @@ impl Command { let stdin = stdin.to_handle(c::STD_INPUT_HANDLE, &mut pipes.stdin)?; let stdout = stdout.to_handle(c::STD_OUTPUT_HANDLE, &mut pipes.stdout)?; let stderr = stderr.to_handle(c::STD_ERROR_HANDLE, &mut pipes.stderr)?; - si.hStdInput = stdin.raw(); - si.hStdOutput = stdout.raw(); - si.hStdError = stderr.raw(); + + let mut si = zeroed_startupinfo(); + si.cb = mem::size_of::() as c::DWORD; + + // If at least one of stdin, stdout or stderr are set (i.e. are non null) + // then set the `hStd` fields in `STARTUPINFO`. + // Otherwise skip this and allow the OS to apply its default behaviour. + // This provides more consistent behaviour between Win7 and Win8+. + let is_set = |stdio: &Handle| !stdio.as_raw_handle().is_null(); + if is_set(&stderr) || is_set(&stdout) || is_set(&stdin) { + si.dwFlags |= c::STARTF_USESTDHANDLES; + si.hStdInput = stdin.as_raw_handle(); + si.hStdOutput = stdout.as_raw_handle(); + si.hStdError = stderr.as_raw_handle(); + } unsafe { cvt(c::CreateProcessW( - ptr::null(), + program.as_ptr(), cmd_str.as_mut_ptr(), ptr::null_mut(), ptr::null_mut(), @@ -215,38 +341,184 @@ impl Command { )) }?; - // We close the thread handle because we don't care about keeping - // the thread id valid, and we aren't keeping the thread handle - // around to be able to close it later. - drop(Handle::new(pi.hThread)); + unsafe { + Ok(( + Process { + handle: Handle::from_raw_handle(pi.hProcess), + main_thread_handle: Handle::from_raw_handle(pi.hThread), + }, + pipes, + )) + } + } - Ok((Process { handle: Handle::new(pi.hProcess) }, pipes)) + pub fn output(&mut self) -> io::Result<(ExitStatus, Vec, Vec)> { + let (proc, pipes) = self.spawn(Stdio::MakePipe, false)?; + crate::sys_common::process::wait_with_output(proc, pipes) } } impl fmt::Debug for Command { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self.program)?; + self.program.fmt(f)?; for arg in &self.args { - write!(f, " {:?}", arg)?; + f.write_str(" ")?; + match arg { + Arg::Regular(s) => s.fmt(f), + Arg::Raw(s) => f.write_str(&s.to_string_lossy()), + }?; } Ok(()) } } -impl<'a> DropGuard<'a> { - fn new(lock: &'a Mutex) -> DropGuard<'a> { - unsafe { - lock.lock(); - DropGuard { lock } +// Resolve `exe_path` to the executable name. +// +// * If the path is simply a file name then use the paths given by `search_paths` to find the executable. +// * Otherwise use the `exe_path` as given. +// +// This function may also append `.exe` to the name. The rationale for doing so is as follows: +// +// It is a very strong convention that Windows executables have the `exe` extension. +// In Rust, it is common to omit this extension. +// Therefore this functions first assumes `.exe` was intended. +// It falls back to the plain file name if a full path is given and the extension is omitted +// or if only a file name is given and it already contains an extension. +fn resolve_exe<'a>( + exe_path: &'a OsStr, + parent_paths: impl FnOnce() -> Option, + child_paths: Option<&OsStr>, +) -> io::Result> { + // Early return if there is no filename. + if exe_path.is_empty() || path::has_trailing_slash(exe_path) { + return Err(io::const_io_error!( + io::ErrorKind::InvalidInput, + "program path has no file name", + )); + } + // Test if the file name has the `exe` extension. + // This does a case-insensitive `ends_with`. + let has_exe_suffix = if exe_path.len() >= EXE_SUFFIX.len() { + exe_path.bytes()[exe_path.len() - EXE_SUFFIX.len()..] + .eq_ignore_ascii_case(EXE_SUFFIX.as_bytes()) + } else { + false + }; + + // If `exe_path` is an absolute path or a sub-path then don't search `PATH` for it. + if !path::is_file_name(exe_path) { + if has_exe_suffix { + // The application name is a path to a `.exe` file. + // Let `CreateProcessW` figure out if it exists or not. + return path::maybe_verbatim(Path::new(exe_path)); + } + let mut path = PathBuf::from(exe_path); + + // Append `.exe` if not already there. + path = path::append_suffix(path, EXE_SUFFIX.as_ref()); + if let Some(path) = program_exists(&path) { + return Ok(path); + } else { + // It's ok to use `set_extension` here because the intent is to + // remove the extension that was just added. + path.set_extension(""); + return path::maybe_verbatim(&path); + } + } else { + ensure_no_nuls(exe_path)?; + // From the `CreateProcessW` docs: + // > If the file name does not contain an extension, .exe is appended. + // Note that this rule only applies when searching paths. + let has_extension = exe_path.bytes().contains(&b'.'); + + // Search the directories given by `search_paths`. + let result = search_paths(parent_paths, child_paths, |mut path| { + path.push(&exe_path); + if !has_extension { + path.set_extension(EXE_EXTENSION); + } + program_exists(&path) + }); + if let Some(path) = result { + return Ok(path); } } + // If we get here then the executable cannot be found. + Err(io::const_io_error!(io::ErrorKind::NotFound, "program not found")) } -impl<'a> Drop for DropGuard<'a> { - fn drop(&mut self) { - unsafe { - self.lock.unlock(); +// Calls `f` for every path that should be used to find an executable. +// Returns once `f` returns the path to an executable or all paths have been searched. +fn search_paths( + parent_paths: Paths, + child_paths: Option<&OsStr>, + mut exists: Exists, +) -> Option> +where + Paths: FnOnce() -> Option, + Exists: FnMut(PathBuf) -> Option>, +{ + // 1. Child paths + // This is for consistency with Rust's historic behaviour. + if let Some(paths) = child_paths { + for path in env::split_paths(paths).filter(|p| !p.as_os_str().is_empty()) { + if let Some(path) = exists(path) { + return Some(path); + } + } + } + + // 2. Application path + if let Ok(mut app_path) = env::current_exe() { + app_path.pop(); + if let Some(path) = exists(app_path) { + return Some(path); + } + } + + // 3 & 4. System paths + // SAFETY: This uses `fill_utf16_buf` to safely call the OS functions. + unsafe { + if let Ok(Some(path)) = super::fill_utf16_buf( + |buf, size| c::GetSystemDirectoryW(buf, size), + |buf| exists(PathBuf::from(OsString::from_wide(buf))), + ) { + return Some(path); + } + #[cfg(not(target_vendor = "uwp"))] + { + if let Ok(Some(path)) = super::fill_utf16_buf( + |buf, size| c::GetWindowsDirectoryW(buf, size), + |buf| exists(PathBuf::from(OsString::from_wide(buf))), + ) { + return Some(path); + } + } + } + + // 5. Parent paths + if let Some(parent_paths) = parent_paths() { + for path in env::split_paths(&parent_paths).filter(|p| !p.as_os_str().is_empty()) { + if let Some(path) = exists(path) { + return Some(path); + } + } + } + None +} + +/// Check if a file exists without following symlinks. +fn program_exists(path: &Path) -> Option> { + unsafe { + let path = path::maybe_verbatim(path).ok()?; + // Getting attributes using `GetFileAttributesW` does not follow symlinks + // and it will almost always be successful if the link exists. + // There are some exceptions for special system files (e.g. the pagefile) + // but these are not executable. + if c::GetFileAttributesW(path.as_ptr()) == c::INVALID_FILE_ATTRIBUTES { + None + } else { + Some(path) } } } @@ -254,17 +526,15 @@ impl<'a> Drop for DropGuard<'a> { impl Stdio { fn to_handle(&self, stdio_id: c::DWORD, pipe: &mut Option) -> io::Result { match *self { - // If no stdio handle is available, then inherit means that it - // should still be unavailable so propagate the - // INVALID_HANDLE_VALUE. Stdio::Inherit => match stdio::get_handle(stdio_id) { - Ok(io) => { - let io = Handle::new(io); + Ok(io) => unsafe { + let io = Handle::from_raw_handle(io); let ret = io.duplicate(0, true, c::DUPLICATE_SAME_ACCESS); - io.into_raw(); + io.into_raw_handle(); ret - } - Err(..) => Ok(Handle::new(c::INVALID_HANDLE_VALUE)), + }, + // If no stdio handle is available, then propagate the null value. + Err(..) => unsafe { Ok(Handle::from_raw_handle(ptr::null_mut())) }, }, Stdio::MakePipe => { @@ -274,6 +544,11 @@ impl Stdio { Ok(pipes.theirs.into_handle()) } + Stdio::Pipe(ref source) => { + let ours_readable = stdio_id != c::STD_INPUT_HANDLE; + pipe::spawn_pipe_relay(source, ours_readable, true).map(AnonPipe::into_handle) + } + Stdio::Handle(ref handle) => handle.duplicate(0, true, c::DUPLICATE_SAME_ACCESS), // Open up a reference to NUL with appropriate read/write @@ -290,7 +565,7 @@ impl Stdio { opts.read(stdio_id == c::STD_INPUT_HANDLE); opts.write(stdio_id != c::STD_INPUT_HANDLE); opts.security_attributes(&mut sa); - File::open(Path::new("NUL"), &opts).map(|file| file.into_handle()) + File::open(Path::new("NUL"), &opts).map(|file| file.into_inner()) } } } @@ -298,13 +573,13 @@ impl Stdio { impl From for Stdio { fn from(pipe: AnonPipe) -> Stdio { - Stdio::Handle(pipe.into_handle()) + Stdio::Pipe(pipe) } } impl From for Stdio { fn from(file: File) -> Stdio { - Stdio::Handle(file.into_handle()) + Stdio::Handle(file.into_inner()) } } @@ -319,33 +594,38 @@ impl From for Stdio { /// for the process to terminate. pub struct Process { handle: Handle, + main_thread_handle: Handle, } impl Process { pub fn kill(&mut self) -> io::Result<()> { - cvt(unsafe { c::TerminateProcess(self.handle.raw(), 1) })?; + cvt(unsafe { c::TerminateProcess(self.handle.as_raw_handle(), 1) })?; Ok(()) } pub fn id(&self) -> u32 { - unsafe { c::GetProcessId(self.handle.raw()) as u32 } + unsafe { c::GetProcessId(self.handle.as_raw_handle()) as u32 } + } + + pub fn main_thread_handle(&self) -> BorrowedHandle<'_> { + self.main_thread_handle.as_handle() } pub fn wait(&mut self) -> io::Result { unsafe { - let res = c::WaitForSingleObject(self.handle.raw(), c::INFINITE); + let res = c::WaitForSingleObject(self.handle.as_raw_handle(), c::INFINITE); if res != c::WAIT_OBJECT_0 { return Err(Error::last_os_error()); } let mut status = 0; - cvt(c::GetExitCodeProcess(self.handle.raw(), &mut status))?; + cvt(c::GetExitCodeProcess(self.handle.as_raw_handle(), &mut status))?; Ok(ExitStatus(status)) } } pub fn try_wait(&mut self) -> io::Result> { unsafe { - match c::WaitForSingleObject(self.handle.raw(), 0) { + match c::WaitForSingleObject(self.handle.as_raw_handle(), 0) { c::WAIT_OBJECT_0 => {} c::WAIT_TIMEOUT => { return Ok(None); @@ -353,7 +633,7 @@ impl Process { _ => return Err(io::Error::last_os_error()), } let mut status = 0; - cvt(c::GetExitCodeProcess(self.handle.raw(), &mut status))?; + cvt(c::GetExitCodeProcess(self.handle.as_raw_handle(), &mut status))?; Ok(Some(ExitStatus(status))) } } @@ -371,14 +651,18 @@ impl Process { pub struct ExitStatus(c::DWORD); impl ExitStatus { - pub fn success(&self) -> bool { - self.0 == 0 + pub fn exit_ok(&self) -> Result<(), ExitStatusError> { + match NonZeroDWORD::try_from(self.0) { + /* was nonzero */ Ok(failure) => Err(ExitStatusError(failure)), + /* was zero, couldn't convert */ Err(_) => Ok(()), + } } pub fn code(&self) -> Option { Some(self.0 as i32) } } +/// Converts a raw `c::DWORD` to a type-safe `ExitStatus` by wrapping it without copying. impl From for ExitStatus { fn from(u: c::DWORD) -> ExitStatus { ExitStatus(u) @@ -400,6 +684,21 @@ impl fmt::Display for ExitStatus { } } +#[derive(PartialEq, Eq, Clone, Copy, Debug)] +pub struct ExitStatusError(c::NonZeroDWORD); + +impl Into for ExitStatusError { + fn into(self) -> ExitStatus { + ExitStatus(self.0.into()) + } +} + +impl ExitStatusError { + pub fn code(self) -> Option { + Some((u32::from(self.0) as i32).try_into().unwrap()) + } +} + #[derive(PartialEq, Eq, Clone, Copy, Debug)] pub struct ExitCode(c::DWORD); @@ -413,6 +712,18 @@ impl ExitCode { } } +impl From for ExitCode { + fn from(code: u8) -> Self { + ExitCode(c::DWORD::from(code)) + } +} + +impl From for ExitCode { + fn from(code: u32) -> Self { + ExitCode(c::DWORD::from(code)) + } +} + fn zeroed_startupinfo() -> c::STARTUPINFO { c::STARTUPINFO { cb: 0, @@ -430,9 +741,9 @@ fn zeroed_startupinfo() -> c::STARTUPINFO { wShowWindow: 0, cbReserved2: 0, lpReserved2: ptr::null_mut(), - hStdInput: c::INVALID_HANDLE_VALUE, - hStdOutput: c::INVALID_HANDLE_VALUE, - hStdError: c::INVALID_HANDLE_VALUE, + hStdInput: ptr::null_mut(), + hStdOutput: ptr::null_mut(), + hStdError: ptr::null_mut(), } } @@ -447,53 +758,34 @@ fn zeroed_process_information() -> c::PROCESS_INFORMATION { // Produces a wide string *without terminating null*; returns an error if // `prog` or any of the `args` contain a nul. -fn make_command_line(prog: &OsStr, args: &[OsString]) -> io::Result> { +fn make_command_line(argv0: &OsStr, args: &[Arg], force_quotes: bool) -> io::Result> { // Encode the command and arguments in a command line string such // that the spawned process may recover them using CommandLineToArgvW. let mut cmd: Vec = Vec::new(); - // Always quote the program name so CreateProcess doesn't interpret args as - // part of the name if the binary wasn't found first time. - append_arg(&mut cmd, prog, true)?; - for arg in args { - cmd.push(' ' as u16); - append_arg(&mut cmd, arg, false)?; - } - return Ok(cmd); - - fn append_arg(cmd: &mut Vec, arg: &OsStr, force_quotes: bool) -> io::Result<()> { - // If an argument has 0 characters then we need to quote it to ensure - // that it actually gets passed through on the command line or otherwise - // it will be dropped entirely when parsed on the other end. - ensure_no_nuls(arg)?; - let arg_bytes = &arg.as_inner().inner.as_inner(); - let quote = force_quotes - || arg_bytes.iter().any(|c| *c == b' ' || *c == b'\t') - || arg_bytes.is_empty(); - if quote { - cmd.push('"' as u16); - } - let mut backslashes: usize = 0; - for x in arg.encode_wide() { - if x == '\\' as u16 { - backslashes += 1; - } else { - if x == '"' as u16 { - // Add n+1 backslashes to total 2n+1 before internal '"'. - cmd.extend((0..=backslashes).map(|_| '\\' as u16)); - } - backslashes = 0; - } - cmd.push(x); - } + // Always quote the program name so CreateProcess to avoid ambiguity when + // the child process parses its arguments. + // Note that quotes aren't escaped here because they can't be used in arg0. + // But that's ok because file paths can't contain quotes. + cmd.push(b'"' as u16); + cmd.extend(argv0.encode_wide()); + cmd.push(b'"' as u16); - if quote { - // Add n backslashes to total 2n before ending '"'. - cmd.extend((0..backslashes).map(|_| '\\' as u16)); - cmd.push('"' as u16); - } - Ok(()) + for arg in args { + cmd.push(' ' as u16); + args::append_arg(&mut cmd, arg, force_quotes)?; } + Ok(cmd) +} + +// Get `cmd.exe` for use with bat scripts, encoded as a UTF-16 string. +fn command_prompt() -> io::Result> { + let mut system: Vec = super::fill_utf16_buf( + |buf, size| unsafe { c::GetSystemDirectoryW(buf, size) }, + |buf| buf.into(), + )?; + system.extend("\\cmd.exe".encode_utf16().chain([0])); + Ok(system) } fn make_envp(maybe_env: Option>) -> io::Result<(*mut c_void, Vec)> { @@ -503,8 +795,15 @@ fn make_envp(maybe_env: Option>) -> io::Result<(*mut if let Some(env) = maybe_env { let mut blk = Vec::new(); + // If there are no environment variables to set then signal this by + // pushing a null. + if env.is_empty() { + blk.push(0); + } + for (k, v) in env { - blk.extend(ensure_no_nuls(k.0)?.encode_wide()); + ensure_no_nuls(k.os_string)?; + blk.extend(k.utf16); blk.push('=' as u16); blk.extend(ensure_no_nuls(v)?.encode_wide()); blk.push(0); @@ -527,40 +826,33 @@ fn make_dirp(d: Option<&OsString>) -> io::Result<(*const u16, Vec)> { } } -#[cfg(test)] -mod tests { - use super::make_command_line; - use crate::ffi::{OsStr, OsString}; - - #[test] - fn test_make_command_line() { - fn test_wrapper(prog: &str, args: &[&str]) -> String { - let command_line = &make_command_line( - OsStr::new(prog), - &args.iter().map(|a| OsString::from(a)).collect::>(), - ) - .unwrap(); - String::from_utf16(command_line).unwrap() - } +pub struct CommandArgs<'a> { + iter: crate::slice::Iter<'a, Arg>, +} - assert_eq!(test_wrapper("prog", &["aaa", "bbb", "ccc"]), "\"prog\" aaa bbb ccc"); +impl<'a> Iterator for CommandArgs<'a> { + type Item = &'a OsStr; + fn next(&mut self) -> Option<&'a OsStr> { + self.iter.next().map(|arg| match arg { + Arg::Regular(s) | Arg::Raw(s) => s.as_ref(), + }) + } + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } +} - assert_eq!( - test_wrapper("C:\\Program Files\\blah\\blah.exe", &["aaa"]), - "\"C:\\Program Files\\blah\\blah.exe\" aaa" - ); - assert_eq!( - test_wrapper("C:\\Program Files\\test", &["aa\"bb"]), - "\"C:\\Program Files\\test\" aa\\\"bb" - ); - assert_eq!(test_wrapper("echo", &["a b c"]), "\"echo\" \"a b c\""); - assert_eq!( - test_wrapper("echo", &["\" \\\" \\", "\\"]), - "\"echo\" \"\\\" \\\\\\\" \\\\\" \\" - ); - assert_eq!( - test_wrapper("\u{03c0}\u{042f}\u{97f3}\u{00e6}\u{221e}", &[]), - "\"\u{03c0}\u{042f}\u{97f3}\u{00e6}\u{221e}\"" - ); +impl<'a> ExactSizeIterator for CommandArgs<'a> { + fn len(&self) -> usize { + self.iter.len() + } + fn is_empty(&self) -> bool { + self.iter.is_empty() + } +} + +impl<'a> fmt::Debug for CommandArgs<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.iter.clone()).finish() } } diff --git a/crux-mir/lib/std/src/sys/windows/process/tests.rs b/crux-mir/lib/std/src/sys/windows/process/tests.rs new file mode 100644 index 000000000..3fc0c7524 --- /dev/null +++ b/crux-mir/lib/std/src/sys/windows/process/tests.rs @@ -0,0 +1,222 @@ +use super::make_command_line; +use super::Arg; +use crate::env; +use crate::ffi::{OsStr, OsString}; +use crate::process::Command; + +#[test] +fn test_raw_args() { + let command_line = &make_command_line( + OsStr::new("quoted exe"), + &[ + Arg::Regular(OsString::from("quote me")), + Arg::Raw(OsString::from("quote me *not*")), + Arg::Raw(OsString::from("\t\\")), + Arg::Raw(OsString::from("internal \\\"backslash-\"quote")), + Arg::Regular(OsString::from("optional-quotes")), + ], + false, + ) + .unwrap(); + assert_eq!( + String::from_utf16(command_line).unwrap(), + "\"quoted exe\" \"quote me\" quote me *not* \t\\ internal \\\"backslash-\"quote optional-quotes" + ); +} + +#[test] +fn test_thread_handle() { + use crate::os::windows::io::BorrowedHandle; + use crate::os::windows::process::{ChildExt, CommandExt}; + const CREATE_SUSPENDED: u32 = 0x00000004; + + let p = Command::new("cmd").args(&["/C", "exit 0"]).creation_flags(CREATE_SUSPENDED).spawn(); + assert!(p.is_ok()); + let mut p = p.unwrap(); + + extern "system" { + fn ResumeThread(_: BorrowedHandle<'_>) -> u32; + } + unsafe { + ResumeThread(p.main_thread_handle()); + } + + crate::thread::sleep(crate::time::Duration::from_millis(100)); + + let res = p.try_wait(); + assert!(res.is_ok()); + assert!(res.unwrap().is_some()); + assert!(p.try_wait().unwrap().unwrap().success()); +} + +#[test] +fn test_make_command_line() { + fn test_wrapper(prog: &str, args: &[&str], force_quotes: bool) -> String { + let command_line = &make_command_line( + OsStr::new(prog), + &args.iter().map(|a| Arg::Regular(OsString::from(a))).collect::>(), + force_quotes, + ) + .unwrap(); + String::from_utf16(command_line).unwrap() + } + + assert_eq!(test_wrapper("prog", &["aaa", "bbb", "ccc"], false), "\"prog\" aaa bbb ccc"); + + assert_eq!(test_wrapper("prog", &[r"C:\"], false), r#""prog" C:\"#); + assert_eq!(test_wrapper("prog", &[r"2slashes\\"], false), r#""prog" 2slashes\\"#); + assert_eq!(test_wrapper("prog", &[r" C:\"], false), r#""prog" " C:\\""#); + assert_eq!(test_wrapper("prog", &[r" 2slashes\\"], false), r#""prog" " 2slashes\\\\""#); + + assert_eq!( + test_wrapper("C:\\Program Files\\blah\\blah.exe", &["aaa"], false), + "\"C:\\Program Files\\blah\\blah.exe\" aaa" + ); + assert_eq!( + test_wrapper("C:\\Program Files\\blah\\blah.exe", &["aaa", "v*"], false), + "\"C:\\Program Files\\blah\\blah.exe\" aaa v*" + ); + assert_eq!( + test_wrapper("C:\\Program Files\\blah\\blah.exe", &["aaa", "v*"], true), + "\"C:\\Program Files\\blah\\blah.exe\" \"aaa\" \"v*\"" + ); + assert_eq!( + test_wrapper("C:\\Program Files\\test", &["aa\"bb"], false), + "\"C:\\Program Files\\test\" aa\\\"bb" + ); + assert_eq!(test_wrapper("echo", &["a b c"], false), "\"echo\" \"a b c\""); + assert_eq!( + test_wrapper("echo", &["\" \\\" \\", "\\"], false), + "\"echo\" \"\\\" \\\\\\\" \\\\\" \\" + ); + assert_eq!( + test_wrapper("\u{03c0}\u{042f}\u{97f3}\u{00e6}\u{221e}", &[], false), + "\"\u{03c0}\u{042f}\u{97f3}\u{00e6}\u{221e}\"" + ); +} + +// On Windows, environment args are case preserving but comparisons are case-insensitive. +// See: #85242 +#[test] +fn windows_env_unicode_case() { + let test_cases = [ + ("ä", "Ä"), + ("ß", "SS"), + ("Ä", "Ö"), + ("Ä", "Ö"), + ("I", "İ"), + ("I", "i"), + ("I", "ı"), + ("i", "I"), + ("i", "İ"), + ("i", "ı"), + ("İ", "I"), + ("İ", "i"), + ("İ", "ı"), + ("ı", "I"), + ("ı", "i"), + ("ı", "İ"), + ("ä", "Ä"), + ("ß", "SS"), + ("Ä", "Ö"), + ("Ä", "Ö"), + ("I", "İ"), + ("I", "i"), + ("I", "ı"), + ("i", "I"), + ("i", "İ"), + ("i", "ı"), + ("İ", "I"), + ("İ", "i"), + ("İ", "ı"), + ("ı", "I"), + ("ı", "i"), + ("ı", "İ"), + ]; + // Test that `cmd.env` matches `env::set_var` when setting two strings that + // may (or may not) be case-folded when compared. + for (a, b) in test_cases.iter() { + let mut cmd = Command::new("cmd"); + cmd.env(a, "1"); + cmd.env(b, "2"); + env::set_var(a, "1"); + env::set_var(b, "2"); + + for (key, value) in cmd.get_envs() { + assert_eq!( + env::var(key).ok(), + value.map(|s| s.to_string_lossy().into_owned()), + "command environment mismatch: {a} {b}", + ); + } + } +} + +// UWP applications run in a restricted environment which means this test may not work. +#[cfg(not(target_vendor = "uwp"))] +#[test] +fn windows_exe_resolver() { + use super::resolve_exe; + use crate::io; + use crate::sys::fs::symlink; + use crate::sys_common::io::test::tmpdir; + + let env_paths = || env::var_os("PATH"); + + // Test a full path, with and without the `exe` extension. + let mut current_exe = env::current_exe().unwrap(); + assert!(resolve_exe(current_exe.as_ref(), env_paths, None).is_ok()); + current_exe.set_extension(""); + assert!(resolve_exe(current_exe.as_ref(), env_paths, None).is_ok()); + + // Test lone file names. + assert!(resolve_exe(OsStr::new("cmd"), env_paths, None).is_ok()); + assert!(resolve_exe(OsStr::new("cmd.exe"), env_paths, None).is_ok()); + assert!(resolve_exe(OsStr::new("cmd.EXE"), env_paths, None).is_ok()); + assert!(resolve_exe(OsStr::new("fc"), env_paths, None).is_ok()); + + // Invalid file names should return InvalidInput. + assert_eq!( + resolve_exe(OsStr::new(""), env_paths, None).unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + assert_eq!( + resolve_exe(OsStr::new("\0"), env_paths, None).unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + // Trailing slash, therefore there's no file name component. + assert_eq!( + resolve_exe(OsStr::new(r"C:\Path\to\"), env_paths, None).unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + + /* + Some of the following tests may need to be changed if you are deliberately + changing the behaviour of `resolve_exe`. + */ + + let empty_paths = || None; + + // The resolver looks in system directories even when `PATH` is empty. + assert!(resolve_exe(OsStr::new("cmd.exe"), empty_paths, None).is_ok()); + + // The application's directory is also searched. + let current_exe = env::current_exe().unwrap(); + assert!(resolve_exe(current_exe.file_name().unwrap().as_ref(), empty_paths, None).is_ok()); + + // Create a temporary path and add a broken symlink. + let temp = tmpdir(); + let mut exe_path = temp.path().to_owned(); + exe_path.push("exists.exe"); + + // A broken symlink should still be resolved. + // Skip this check if not in CI and creating symlinks isn't possible. + let is_ci = env::var("CI").is_ok(); + let result = symlink("".as_ref(), &exe_path); + if is_ci || result.is_ok() { + result.unwrap(); + assert!( + resolve_exe(OsStr::new("exists.exe"), empty_paths, Some(temp.path().as_ref())).is_ok() + ); + } +} diff --git a/crux-mir/lib/std/src/sys/windows/rand.rs b/crux-mir/lib/std/src/sys/windows/rand.rs index 87ea416bf..b5a49489d 100644 --- a/crux-mir/lib/std/src/sys/windows/rand.rs +++ b/crux-mir/lib/std/src/sys/windows/rand.rs @@ -1,33 +1,106 @@ -use crate::io; +//! # Random key generation +//! +//! This module wraps the RNG provided by the OS. There are a few different +//! ways to interface with the OS RNG so it's worth exploring each of the options. +//! Note that at the time of writing these all go through the (undocumented) +//! `bcryptPrimitives.dll` but they use different route to get there. +//! +//! Originally we were using [`RtlGenRandom`], however that function is +//! deprecated and warns it "may be altered or unavailable in subsequent versions". +//! +//! So we switched to [`BCryptGenRandom`] with the `BCRYPT_USE_SYSTEM_PREFERRED_RNG` +//! flag to query and find the system configured RNG. However, this change caused a small +//! but significant number of users to experience panics caused by a failure of +//! this function. See [#94098]. +//! +//! The current version falls back to using `BCryptOpenAlgorithmProvider` if +//! `BCRYPT_USE_SYSTEM_PREFERRED_RNG` fails for any reason. +//! +//! [#94098]: https://github.com/rust-lang/rust/issues/94098 +//! [`RtlGenRandom`]: https://docs.microsoft.com/en-us/windows/win32/api/ntsecapi/nf-ntsecapi-rtlgenrandom +//! [`BCryptGenRandom`]: https://docs.microsoft.com/en-us/windows/win32/api/bcrypt/nf-bcrypt-bcryptgenrandom use crate::mem; +use crate::ptr; use crate::sys::c; -#[cfg(not(target_vendor = "uwp"))] +/// Generates high quality secure random keys for use by [`HashMap`]. +/// +/// This is used to seed the default [`RandomState`]. +/// +/// [`HashMap`]: crate::collections::HashMap +/// [`RandomState`]: crate::collections::hash_map::RandomState pub fn hashmap_random_keys() -> (u64, u64) { - let mut v = (0, 0); - let ret = - unsafe { c::RtlGenRandom(&mut v as *mut _ as *mut u8, mem::size_of_val(&v) as c::ULONG) }; - if ret == 0 { - panic!("couldn't generate random bytes: {}", io::Error::last_os_error()); + Rng::SYSTEM.gen_random_keys().unwrap_or_else(fallback_rng) +} + +struct Rng { + algorithm: c::BCRYPT_ALG_HANDLE, + flags: u32, +} +impl Rng { + const SYSTEM: Self = unsafe { Self::new(ptr::null_mut(), c::BCRYPT_USE_SYSTEM_PREFERRED_RNG) }; + + /// Create the RNG from an existing algorithm handle. + /// + /// # Safety + /// + /// The handle must either be null or a valid algorithm handle. + const unsafe fn new(algorithm: c::BCRYPT_ALG_HANDLE, flags: u32) -> Self { + Self { algorithm, flags } + } + + /// Open a handle to the RNG algorithm. + fn open() -> Result { + use crate::sync::atomic::AtomicPtr; + use crate::sync::atomic::Ordering::{Acquire, Release}; + + // An atomic is used so we don't need to reopen the handle every time. + static HANDLE: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + + let mut handle = HANDLE.load(Acquire); + if handle.is_null() { + let status = unsafe { + c::BCryptOpenAlgorithmProvider( + &mut handle, + c::BCRYPT_RNG_ALGORITHM.as_ptr(), + ptr::null(), + 0, + ) + }; + if c::nt_success(status) { + // If another thread opens a handle first then use that handle instead. + let result = HANDLE.compare_exchange(ptr::null_mut(), handle, Release, Acquire); + if let Err(previous_handle) = result { + // Close our handle and return the previous one. + unsafe { c::BCryptCloseAlgorithmProvider(handle, 0) }; + handle = previous_handle; + } + Ok(unsafe { Self::new(handle, 0) }) + } else { + Err(status) + } + } else { + Ok(unsafe { Self::new(handle, 0) }) + } + } + + fn gen_random_keys(self) -> Result<(u64, u64), c::NTSTATUS> { + let mut v = (0, 0); + let status = unsafe { + let size = mem::size_of_val(&v).try_into().unwrap(); + c::BCryptGenRandom(self.algorithm, ptr::addr_of_mut!(v).cast(), size, self.flags) + }; + if c::nt_success(status) { Ok(v) } else { Err(status) } } - v } -#[cfg(target_vendor = "uwp")] -pub fn hashmap_random_keys() -> (u64, u64) { - use crate::ptr; - - let mut v = (0, 0); - let ret = unsafe { - c::BCryptGenRandom( - ptr::null_mut(), - &mut v as *mut _ as *mut u8, - mem::size_of_val(&v) as c::ULONG, - c::BCRYPT_USE_SYSTEM_PREFERRED_RNG, - ) - }; - if ret != 0 { - panic!("couldn't generate random bytes: {}", io::Error::last_os_error()); +/// Generate random numbers using the fallback RNG function +#[inline(never)] +fn fallback_rng(rng_status: c::NTSTATUS) -> (u64, u64) { + match Rng::open().and_then(|rng| rng.gen_random_keys()) { + Ok(keys) => keys, + Err(status) => { + panic!("RNG broken: {rng_status:#x}, fallback RNG broken: {status:#x}") + } } - return v; } diff --git a/crux-mir/lib/std/src/sys/windows/rwlock.rs b/crux-mir/lib/std/src/sys/windows/rwlock.rs deleted file mode 100644 index a76932635..000000000 --- a/crux-mir/lib/std/src/sys/windows/rwlock.rs +++ /dev/null @@ -1,44 +0,0 @@ -use crate::cell::UnsafeCell; -use crate::sys::c; - -pub struct RWLock { - inner: UnsafeCell, -} - -unsafe impl Send for RWLock {} -unsafe impl Sync for RWLock {} - -impl RWLock { - pub const fn new() -> RWLock { - RWLock { inner: UnsafeCell::new(c::SRWLOCK_INIT) } - } - #[inline] - pub unsafe fn read(&self) { - c::AcquireSRWLockShared(self.inner.get()) - } - #[inline] - pub unsafe fn try_read(&self) -> bool { - c::TryAcquireSRWLockShared(self.inner.get()) != 0 - } - #[inline] - pub unsafe fn write(&self) { - c::AcquireSRWLockExclusive(self.inner.get()) - } - #[inline] - pub unsafe fn try_write(&self) -> bool { - c::TryAcquireSRWLockExclusive(self.inner.get()) != 0 - } - #[inline] - pub unsafe fn read_unlock(&self) { - c::ReleaseSRWLockShared(self.inner.get()) - } - #[inline] - pub unsafe fn write_unlock(&self) { - c::ReleaseSRWLockExclusive(self.inner.get()) - } - - #[inline] - pub unsafe fn destroy(&self) { - // ... - } -} diff --git a/crux-mir/lib/std/src/sys/windows/stack_overflow.rs b/crux-mir/lib/std/src/sys/windows/stack_overflow.rs index 187ad4e66..18a2a36ad 100644 --- a/crux-mir/lib/std/src/sys/windows/stack_overflow.rs +++ b/crux-mir/lib/std/src/sys/windows/stack_overflow.rs @@ -1,7 +1,7 @@ #![cfg_attr(test, allow(dead_code))] use crate::sys::c; -use crate::sys_common::util::report_overflow; +use crate::thread; pub struct Handler; @@ -9,10 +9,10 @@ impl Handler { pub unsafe fn new() -> Handler { // This API isn't available on XP, so don't panic in that case and just // pray it works out ok. - if c::SetThreadStackGuarantee(&mut 0x5000) == 0 { - if c::GetLastError() as u32 != c::ERROR_CALL_NOT_IMPLEMENTED as u32 { - panic!("failed to reserve stack space for exception handling"); - } + if c::SetThreadStackGuarantee(&mut 0x5000) == 0 + && c::GetLastError() as u32 != c::ERROR_CALL_NOT_IMPLEMENTED as u32 + { + panic!("failed to reserve stack space for exception handling"); } Handler } @@ -24,7 +24,10 @@ extern "system" fn vectored_handler(ExceptionInfo: *mut c::EXCEPTION_POINTERS) - let code = rec.ExceptionCode; if code == c::EXCEPTION_STACK_OVERFLOW { - report_overflow(); + rtprintpanic!( + "\nthread '{}' has overflowed its stack\n", + thread::current().name().unwrap_or("") + ); } c::EXCEPTION_CONTINUE_SEARCH } @@ -37,5 +40,3 @@ pub unsafe fn init() { // Set the thread stack guarantee for the main thread. let _h = Handler::new(); } - -pub unsafe fn cleanup() {} diff --git a/crux-mir/lib/std/src/sys/windows/stack_overflow_uwp.rs b/crux-mir/lib/std/src/sys/windows/stack_overflow_uwp.rs index e7236cf35..afdf7f566 100644 --- a/crux-mir/lib/std/src/sys/windows/stack_overflow_uwp.rs +++ b/crux-mir/lib/std/src/sys/windows/stack_overflow_uwp.rs @@ -9,5 +9,3 @@ impl Handler { } pub unsafe fn init() {} - -pub unsafe fn cleanup() {} diff --git a/crux-mir/lib/std/src/sys/windows/stdio.rs b/crux-mir/lib/std/src/sys/windows/stdio.rs index c84896296..70c9b14a0 100644 --- a/crux-mir/lib/std/src/sys/windows/stdio.rs +++ b/crux-mir/lib/std/src/sys/windows/stdio.rs @@ -3,19 +3,53 @@ use crate::char::decode_utf16; use crate::cmp; use crate::io; +use crate::mem::MaybeUninit; +use crate::os::windows::io::{FromRawHandle, IntoRawHandle}; use crate::ptr; use crate::str; use crate::sys::c; use crate::sys::cvt; use crate::sys::handle::Handle; +use core::str::utf8_char_width; // Don't cache handles but get them fresh for every read/write. This allows us to track changes to // the value over time (such as if a process calls `SetStdHandle` while it's running). See #40490. pub struct Stdin { surrogate: u16, + incomplete_utf8: IncompleteUtf8, +} + +pub struct Stdout { + incomplete_utf8: IncompleteUtf8, +} + +pub struct Stderr { + incomplete_utf8: IncompleteUtf8, +} + +struct IncompleteUtf8 { + bytes: [u8; 4], + len: u8, +} + +impl IncompleteUtf8 { + // Implemented for use in Stdin::read. + fn read(&mut self, buf: &mut [u8]) -> usize { + // Write to buffer until the buffer is full or we run out of bytes. + let to_write = cmp::min(buf.len(), self.len as usize); + buf[..to_write].copy_from_slice(&self.bytes[..to_write]); + + // Rotate the remaining bytes if not enough remaining space in buffer. + if usize::from(self.len) > buf.len() { + self.bytes.copy_within(to_write.., 0); + self.len -= to_write as u8; + } else { + self.len = 0; + } + + to_write + } } -pub struct Stdout; -pub struct Stderr; // Apparently Windows doesn't handle large reads on stdin or writes to stdout/stderr well (see // #13304 for details). @@ -50,38 +84,100 @@ fn is_console(handle: c::HANDLE) -> bool { unsafe { c::GetConsoleMode(handle, &mut mode) != 0 } } -fn write(handle_id: c::DWORD, data: &[u8]) -> io::Result { +fn write( + handle_id: c::DWORD, + data: &[u8], + incomplete_utf8: &mut IncompleteUtf8, +) -> io::Result { + if data.is_empty() { + return Ok(0); + } + let handle = get_handle(handle_id)?; if !is_console(handle) { - let handle = Handle::new(handle); - let ret = handle.write(data); - handle.into_raw(); // Don't close the handle - return ret; + unsafe { + let handle = Handle::from_raw_handle(handle); + let ret = handle.write(data); + handle.into_raw_handle(); // Don't close the handle + return ret; + } } - // As the console is meant for presenting text, we assume bytes of `data` come from a string - // and are encoded as UTF-8, which needs to be encoded as UTF-16. + if incomplete_utf8.len > 0 { + assert!( + incomplete_utf8.len < 4, + "Unexpected number of bytes for incomplete UTF-8 codepoint." + ); + if data[0] >> 6 != 0b10 { + // not a continuation byte - reject + incomplete_utf8.len = 0; + return Err(io::const_io_error!( + io::ErrorKind::InvalidData, + "Windows stdio in console mode does not support writing non-UTF-8 byte sequences", + )); + } + incomplete_utf8.bytes[incomplete_utf8.len as usize] = data[0]; + incomplete_utf8.len += 1; + let char_width = utf8_char_width(incomplete_utf8.bytes[0]); + if (incomplete_utf8.len as usize) < char_width { + // more bytes needed + return Ok(1); + } + let s = str::from_utf8(&incomplete_utf8.bytes[0..incomplete_utf8.len as usize]); + incomplete_utf8.len = 0; + match s { + Ok(s) => { + assert_eq!(char_width, s.len()); + let written = write_valid_utf8_to_console(handle, s)?; + assert_eq!(written, s.len()); // guaranteed by write_valid_utf8_to_console() for single codepoint writes + return Ok(1); + } + Err(_) => { + return Err(io::const_io_error!( + io::ErrorKind::InvalidData, + "Windows stdio in console mode does not support writing non-UTF-8 byte sequences", + )); + } + } + } + + // As the console is meant for presenting text, we assume bytes of `data` are encoded as UTF-8, + // which needs to be encoded as UTF-16. // // If the data is not valid UTF-8 we write out as many bytes as are valid. - // Only when there are no valid bytes (which will happen on the next call), return an error. + // If the first byte is invalid it is either first byte of a multi-byte sequence but the + // provided byte slice is too short or it is the first byte of an invalid multi-byte sequence. let len = cmp::min(data.len(), MAX_BUFFER_SIZE / 2); let utf8 = match str::from_utf8(&data[..len]) { Ok(s) => s, Err(ref e) if e.valid_up_to() == 0 => { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Windows stdio in console mode does not support writing non-UTF-8 byte sequences", - )); + let first_byte_char_width = utf8_char_width(data[0]); + if first_byte_char_width > 1 && data.len() < first_byte_char_width { + incomplete_utf8.bytes[0] = data[0]; + incomplete_utf8.len = 1; + return Ok(1); + } else { + return Err(io::const_io_error!( + io::ErrorKind::InvalidData, + "Windows stdio in console mode does not support writing non-UTF-8 byte sequences", + )); + } } Err(e) => str::from_utf8(&data[..e.valid_up_to()]).unwrap(), }; - let mut utf16 = [0u16; MAX_BUFFER_SIZE / 2]; + + write_valid_utf8_to_console(handle, utf8) +} + +fn write_valid_utf8_to_console(handle: c::HANDLE, utf8: &str) -> io::Result { + let mut utf16 = [MaybeUninit::::uninit(); MAX_BUFFER_SIZE / 2]; let mut len_utf16 = 0; for (chr, dest) in utf8.encode_utf16().zip(utf16.iter_mut()) { - *dest = chr; + *dest = MaybeUninit::new(chr); len_utf16 += 1; } - let utf16 = &utf16[..len_utf16]; + // Safety: We've initialized `len_utf16` values. + let utf16: &[u16] = unsafe { MaybeUninit::slice_assume_init_ref(&utf16[..len_utf16]) }; let mut written = write_u16s(handle, &utf16)?; @@ -131,8 +227,8 @@ fn write_u16s(handle: c::HANDLE, data: &[u16]) -> io::Result { } impl Stdin { - pub fn new() -> io::Result { - Ok(Stdin { surrogate: 0 }) + pub const fn new() -> Stdin { + Stdin { surrogate: 0, incomplete_utf8: IncompleteUtf8::new() } } } @@ -140,46 +236,69 @@ impl io::Read for Stdin { fn read(&mut self, buf: &mut [u8]) -> io::Result { let handle = get_handle(c::STD_INPUT_HANDLE)?; if !is_console(handle) { - let handle = Handle::new(handle); - let ret = handle.read(buf); - handle.into_raw(); // Don't close the handle - return ret; + unsafe { + let handle = Handle::from_raw_handle(handle); + let ret = handle.read(buf); + handle.into_raw_handle(); // Don't close the handle + return ret; + } } - if buf.len() == 0 { - return Ok(0); - } else if buf.len() < 4 { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Windows stdin in console mode does not support a buffer too small to \ - guarantee holding one arbitrary UTF-8 character (4 bytes)", - )); - } + // If there are bytes in the incomplete utf-8, start with those. + // (No-op if there is nothing in the buffer.) + let mut bytes_copied = self.incomplete_utf8.read(buf); + + if bytes_copied == buf.len() { + return Ok(bytes_copied); + } else if buf.len() - bytes_copied < 4 { + // Not enough space to get a UTF-8 byte. We will use the incomplete UTF8. + let mut utf16_buf = [MaybeUninit::new(0); 1]; + // Read one u16 character. + let read = read_u16s_fixup_surrogates(handle, &mut utf16_buf, 1, &mut self.surrogate)?; + // Read bytes, using the (now-empty) self.incomplete_utf8 as extra space. + let read_bytes = utf16_to_utf8( + unsafe { MaybeUninit::slice_assume_init_ref(&utf16_buf[..read]) }, + &mut self.incomplete_utf8.bytes, + )?; - let mut utf16_buf = [0u16; MAX_BUFFER_SIZE / 2]; - // In the worst case, an UTF-8 string can take 3 bytes for every `u16` of an UTF-16. So - // we can read at most a third of `buf.len()` chars and uphold the guarantee no data gets - // lost. - let amount = cmp::min(buf.len() / 3, utf16_buf.len()); - let read = read_u16s_fixup_surrogates(handle, &mut utf16_buf, amount, &mut self.surrogate)?; + // Read in the bytes from incomplete_utf8 until the buffer is full. + self.incomplete_utf8.len = read_bytes as u8; + // No-op if no bytes. + bytes_copied += self.incomplete_utf8.read(&mut buf[bytes_copied..]); + Ok(bytes_copied) + } else { + let mut utf16_buf = [MaybeUninit::::uninit(); MAX_BUFFER_SIZE / 2]; - utf16_to_utf8(&utf16_buf[..read], buf) + // In the worst case, a UTF-8 string can take 3 bytes for every `u16` of a UTF-16. So + // we can read at most a third of `buf.len()` chars and uphold the guarantee no data gets + // lost. + let amount = cmp::min(buf.len() / 3, utf16_buf.len()); + let read = + read_u16s_fixup_surrogates(handle, &mut utf16_buf, amount, &mut self.surrogate)?; + // Safety `read_u16s_fixup_surrogates` returns the number of items + // initialized. + let utf16s = unsafe { MaybeUninit::slice_assume_init_ref(&utf16_buf[..read]) }; + match utf16_to_utf8(utf16s, buf) { + Ok(value) => return Ok(bytes_copied + value), + Err(e) => return Err(e), + } + } } } // We assume that if the last `u16` is an unpaired surrogate they got sliced apart by our // buffer size, and keep it around for the next read hoping to put them together. -// This is a best effort, and may not work if we are not the only reader on Stdin. +// This is a best effort, and might not work if we are not the only reader on Stdin. fn read_u16s_fixup_surrogates( handle: c::HANDLE, - buf: &mut [u16], + buf: &mut [MaybeUninit], mut amount: usize, surrogate: &mut u16, ) -> io::Result { // Insert possibly remaining unpaired surrogate from last read. let mut start = 0; if *surrogate != 0 { - buf[0] = *surrogate; + buf[0] = MaybeUninit::new(*surrogate); *surrogate = 0; start = 1; if amount == 1 { @@ -192,7 +311,10 @@ fn read_u16s_fixup_surrogates( let mut amount = read_u16s(handle, &mut buf[start..amount])? + start; if amount > 0 { - let last_char = buf[amount - 1]; + // Safety: The returned `amount` is the number of values initialized, + // and it is not 0, so we know that `buf[amount - 1]` have been + // initialized. + let last_char = unsafe { buf[amount - 1].assume_init() }; if last_char >= 0xD800 && last_char <= 0xDBFF { // high surrogate *surrogate = last_char; @@ -202,7 +324,8 @@ fn read_u16s_fixup_surrogates( Ok(amount) } -fn read_u16s(handle: c::HANDLE, buf: &mut [u16]) -> io::Result { +// Returns `Ok(n)` if it initialized `n` values in `buf`. +fn read_u16s(handle: c::HANDLE, buf: &mut [MaybeUninit]) -> io::Result { // Configure the `pInputControl` parameter to not only return on `\r\n` but also Ctrl-Z, the // traditional DOS method to indicate end of character stream / user input (SUB). // See #38274 and https://stackoverflow.com/questions/43836040/win-api-readconsole. @@ -216,17 +339,28 @@ fn read_u16s(handle: c::HANDLE, buf: &mut [u16]) -> io::Result { }; let mut amount = 0; - cvt(unsafe { - c::ReadConsoleW( - handle, - buf.as_mut_ptr() as c::LPVOID, - buf.len() as u32, - &mut amount, - &mut input_control as c::PCONSOLE_READCONSOLE_CONTROL, - ) - })?; + loop { + cvt(unsafe { + c::SetLastError(0); + c::ReadConsoleW( + handle, + buf.as_mut_ptr() as c::LPVOID, + buf.len() as u32, + &mut amount, + &mut input_control as c::PCONSOLE_READCONSOLE_CONTROL, + ) + })?; - if amount > 0 && buf[amount as usize - 1] == CTRL_Z { + // ReadConsoleW returns success with ERROR_OPERATION_ABORTED for Ctrl-C or Ctrl-Break. + // Explicitly check for that case here and try again. + if amount == 0 && unsafe { c::GetLastError() } == c::ERROR_OPERATION_ABORTED { + continue; + } + break; + } + // Safety: if `amount > 0`, then that many bytes were written, so + // `buf[amount as usize - 1]` has been initialized. + if amount > 0 && unsafe { buf[amount as usize - 1].assume_init() } == CTRL_Z { amount -= 1; } Ok(amount as usize) @@ -243,7 +377,7 @@ fn utf16_to_utf8(utf16: &[u16], utf8: &mut [u8]) -> io::Result { } Err(_) => { // We can't really do any better than forget all data and return an error. - return Err(io::Error::new( + return Err(io::const_io_error!( io::ErrorKind::InvalidData, "Windows stdin in console mode does not support non-UTF-16 input; \ encountered unpaired surrogate", @@ -254,15 +388,21 @@ fn utf16_to_utf8(utf16: &[u16], utf8: &mut [u8]) -> io::Result { Ok(written) } +impl IncompleteUtf8 { + pub const fn new() -> IncompleteUtf8 { + IncompleteUtf8 { bytes: [0; 4], len: 0 } + } +} + impl Stdout { - pub fn new() -> io::Result { - Ok(Stdout) + pub const fn new() -> Stdout { + Stdout { incomplete_utf8: IncompleteUtf8::new() } } } impl io::Write for Stdout { fn write(&mut self, buf: &[u8]) -> io::Result { - write(c::STD_OUTPUT_HANDLE, buf) + write(c::STD_OUTPUT_HANDLE, buf, &mut self.incomplete_utf8) } fn flush(&mut self) -> io::Result<()> { @@ -271,14 +411,14 @@ impl io::Write for Stdout { } impl Stderr { - pub fn new() -> io::Result { - Ok(Stderr) + pub const fn new() -> Stderr { + Stderr { incomplete_utf8: IncompleteUtf8::new() } } } impl io::Write for Stderr { fn write(&mut self, buf: &[u8]) -> io::Result { - write(c::STD_ERROR_HANDLE, buf) + write(c::STD_ERROR_HANDLE, buf, &mut self.incomplete_utf8) } fn flush(&mut self) -> io::Result<()> { @@ -291,5 +431,5 @@ pub fn is_ebadf(err: &io::Error) -> bool { } pub fn panic_output() -> Option { - Stderr::new().ok() + Some(Stderr::new()) } diff --git a/crux-mir/lib/std/src/sys/windows/stdio_uwp.rs b/crux-mir/lib/std/src/sys/windows/stdio_uwp.rs deleted file mode 100644 index 5bdabf6d4..000000000 --- a/crux-mir/lib/std/src/sys/windows/stdio_uwp.rs +++ /dev/null @@ -1,84 +0,0 @@ -#![unstable(issue = "none", feature = "windows_stdio")] - -use crate::io; -use crate::mem::ManuallyDrop; -use crate::sys::c; -use crate::sys::handle::Handle; - -pub struct Stdin {} -pub struct Stdout; -pub struct Stderr; - -const MAX_BUFFER_SIZE: usize = 8192; -pub const STDIN_BUF_SIZE: usize = MAX_BUFFER_SIZE / 2 * 3; - -pub fn get_handle(handle_id: c::DWORD) -> io::Result { - let handle = unsafe { c::GetStdHandle(handle_id) }; - if handle == c::INVALID_HANDLE_VALUE { - Err(io::Error::last_os_error()) - } else if handle.is_null() { - Err(io::Error::from_raw_os_error(c::ERROR_INVALID_HANDLE as i32)) - } else { - Ok(handle) - } -} - -fn write(handle_id: c::DWORD, data: &[u8]) -> io::Result { - let handle = get_handle(handle_id)?; - let handle = Handle::new(handle); - ManuallyDrop::new(handle).write(data) -} - -impl Stdin { - pub fn new() -> io::Result { - Ok(Stdin {}) - } -} - -impl io::Read for Stdin { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - let handle = get_handle(c::STD_INPUT_HANDLE)?; - let handle = Handle::new(handle); - ManuallyDrop::new(handle).read(buf) - } -} - -impl Stdout { - pub fn new() -> io::Result { - Ok(Stdout) - } -} - -impl io::Write for Stdout { - fn write(&mut self, buf: &[u8]) -> io::Result { - write(c::STD_OUTPUT_HANDLE, buf) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -impl Stderr { - pub fn new() -> io::Result { - Ok(Stderr) - } -} - -impl io::Write for Stderr { - fn write(&mut self, buf: &[u8]) -> io::Result { - write(c::STD_ERROR_HANDLE, buf) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -pub fn is_ebadf(err: &io::Error) -> bool { - err.raw_os_error() == Some(c::ERROR_INVALID_HANDLE as i32) -} - -pub fn panic_output() -> Option { - Stderr::new().ok() -} diff --git a/crux-mir/lib/std/src/sys/windows/thread.rs b/crux-mir/lib/std/src/sys/windows/thread.rs index c828243a5..1cb576c95 100644 --- a/crux-mir/lib/std/src/sys/windows/thread.rs +++ b/crux-mir/lib/std/src/sys/windows/thread.rs @@ -1,10 +1,12 @@ use crate::ffi::CStr; use crate::io; -use crate::mem; +use crate::num::NonZeroUsize; +use crate::os::windows::io::AsRawHandle; use crate::ptr; use crate::sys::c; use crate::sys::handle::Handle; -use crate::sys_common::thread::*; +use crate::sys::stack_overflow; +use crate::sys_common::FromInner; use crate::time::Duration; use libc::c_void; @@ -20,35 +22,38 @@ pub struct Thread { impl Thread { // unsafe: see thread::Builder::spawn_unchecked for safety requirements pub unsafe fn new(stack: usize, p: Box) -> io::Result { - let p = box p; + let p = Box::into_raw(box p); // FIXME On UNIX, we guard against stack sizes that are too small but // that's because pthreads enforces that stacks are at least - // PTHREAD_STACK_MIN bytes big. Windows has no such lower limit, it's + // PTHREAD_STACK_MIN bytes big. Windows has no such lower limit, it's // just that below a certain threshold you can't do anything useful. // That threshold is application and architecture-specific, however. - // Round up to the next 64 kB because that's what the NT kernel does, - // might as well make it explicit. - let stack_size = (stack + 0xfffe) & (!0xfffe); let ret = c::CreateThread( ptr::null_mut(), - stack_size, + stack, thread_start, - &*p as *const _ as *mut _, + p as *mut _, c::STACK_SIZE_PARAM_IS_A_RESERVATION, ptr::null_mut(), ); - return if ret as usize == 0 { - Err(io::Error::last_os_error()) + return if let Ok(handle) = ret.try_into() { + Ok(Thread { handle: Handle::from_inner(handle) }) } else { - mem::forget(p); // ownership passed to CreateThread - Ok(Thread { handle: Handle::new(ret) }) + // The thread failed to start and as a result p was not consumed. Therefore, it is + // safe to reconstruct the box so that it gets deallocated. + drop(Box::from_raw(p)); + Err(io::Error::last_os_error()) }; extern "system" fn thread_start(main: *mut c_void) -> c::DWORD { unsafe { - start_thread(main as *mut u8); + // Next, set up our stack overflow handler which may get triggered if we run + // out of stack. + let _handler = stack_overflow::Handler::new(); + // Finally, let's run some code. + Box::from_raw(main as *mut Box)(); } 0 } @@ -65,7 +70,7 @@ impl Thread { } pub fn join(self) { - let rc = unsafe { c::WaitForSingleObject(self.handle.raw(), c::INFINITE) }; + let rc = unsafe { c::WaitForSingleObject(self.handle.as_raw_handle(), c::INFINITE) }; if rc == c::WAIT_FAILED { panic!("failed to join on thread: {}", io::Error::last_os_error()); } @@ -93,6 +98,21 @@ impl Thread { } } +pub fn available_parallelism() -> io::Result { + let res = unsafe { + let mut sysinfo: c::SYSTEM_INFO = crate::mem::zeroed(); + c::GetSystemInfo(&mut sysinfo); + sysinfo.dwNumberOfProcessors as usize + }; + match res { + 0 => Err(io::const_io_error!( + io::ErrorKind::NotFound, + "The number of hardware threads is not known for the target platform", + )), + cpus => Ok(unsafe { NonZeroUsize::new_unchecked(cpus) }), + } +} + #[cfg_attr(test, allow(dead_code))] pub mod guard { pub type Guard = !; diff --git a/crux-mir/lib/std/src/sys/windows/thread_local_dtor.rs b/crux-mir/lib/std/src/sys/windows/thread_local_dtor.rs new file mode 100644 index 000000000..9707a95df --- /dev/null +++ b/crux-mir/lib/std/src/sys/windows/thread_local_dtor.rs @@ -0,0 +1,32 @@ +//! Implements thread-local destructors that are not associated with any +//! particular data. + +#![unstable(feature = "thread_local_internals", issue = "none")] +#![cfg(target_thread_local)] + +// Using a per-thread list avoids the problems in synchronizing global state. +#[thread_local] +static mut DESTRUCTORS: Vec<(*mut u8, unsafe extern "C" fn(*mut u8))> = Vec::new(); + +// Ensure this can never be inlined because otherwise this may break in dylibs. +// See #44391. +#[inline(never)] +pub unsafe fn register_dtor(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { + DESTRUCTORS.push((t, dtor)); +} + +#[inline(never)] // See comment above +/// Runs destructors. This should not be called until thread exit. +pub unsafe fn run_keyless_dtors() { + // Drop all the destructors. + // + // Note: While this is potentially an infinite loop, it *should* be + // the case that this loop always terminates because we provide the + // guarantee that a TLS key cannot be set after it is flagged for + // destruction. + while let Some((ptr, dtor)) = DESTRUCTORS.pop() { + (dtor)(ptr); + } + // We're done so free the memory. + DESTRUCTORS = Vec::new(); +} diff --git a/crux-mir/lib/std/src/sys/windows/thread_local.rs b/crux-mir/lib/std/src/sys/windows/thread_local_key.rs similarity index 59% rename from crux-mir/lib/std/src/sys/windows/thread_local.rs rename to crux-mir/lib/std/src/sys/windows/thread_local_key.rs index e0bb102b3..17628b757 100644 --- a/crux-mir/lib/std/src/sys/windows/thread_local.rs +++ b/crux-mir/lib/std/src/sys/windows/thread_local_key.rs @@ -1,11 +1,16 @@ -use crate::mem; +use crate::cell::UnsafeCell; use crate::ptr; -use crate::sync::atomic::AtomicPtr; -use crate::sync::atomic::Ordering::SeqCst; +use crate::sync::atomic::{ + AtomicPtr, AtomicU32, + Ordering::{AcqRel, Acquire, Relaxed, Release}, +}; use crate::sys::c; -pub type Key = c::DWORD; -pub type Dtor = unsafe extern "C" fn(*mut u8); +#[cfg(test)] +mod tests; + +type Key = c::DWORD; +type Dtor = unsafe extern "C" fn(*mut u8); // Turns out, like pretty much everything, Windows is pretty close the // functionality that Unix provides, but slightly different! In the case of @@ -22,60 +27,109 @@ pub type Dtor = unsafe extern "C" fn(*mut u8); // To accomplish this feat, we perform a number of threads, all contained // within this module: // -// * All TLS destructors are tracked by *us*, not the windows runtime. This +// * All TLS destructors are tracked by *us*, not the Windows runtime. This // means that we have a global list of destructors for each TLS key that // we know about. // * When a thread exits, we run over the entire list and run dtors for all // non-null keys. This attempts to match Unix semantics in this regard. // -// This ends up having the overhead of using a global list, having some -// locks here and there, and in general just adding some more code bloat. We -// attempt to optimize runtime by forgetting keys that don't have -// destructors, but this only gets us so far. -// // For more details and nitty-gritty, see the code sections below! // -// [1]: http://www.codeproject.com/Articles/8113/Thread-Local-Storage-The-C-Way -// [2]: https://github.com/ChromiumWebApps/chromium/blob/master/base -// /threading/thread_local_storage_win.cc#L42 +// [1]: https://www.codeproject.com/Articles/8113/Thread-Local-Storage-The-C-Way +// [2]: https://github.com/ChromiumWebApps/chromium/blob/master/base/threading/thread_local_storage_win.cc#L42 -// ------------------------------------------------------------------------- -// Native bindings -// -// This section is just raw bindings to the native functions that Windows -// provides, There's a few extra calls to deal with destructors. +pub struct StaticKey { + /// The key value shifted up by one. Since TLS_OUT_OF_INDEXES == DWORD::MAX + /// is not a valid key value, this allows us to use zero as sentinel value + /// without risking overflow. + key: AtomicU32, + dtor: Option, + next: AtomicPtr, + /// Currently, destructors cannot be unregistered, so we cannot use racy + /// initialization for keys. Instead, we need synchronize initialization. + /// Use the Windows-provided `Once` since it does not require TLS. + once: UnsafeCell, +} -#[inline] -pub unsafe fn create(dtor: Option) -> Key { - let key = c::TlsAlloc(); - assert!(key != c::TLS_OUT_OF_INDEXES); - if let Some(f) = dtor { - register_dtor(key, f); +impl StaticKey { + #[inline] + pub const fn new(dtor: Option) -> StaticKey { + StaticKey { + key: AtomicU32::new(0), + dtor, + next: AtomicPtr::new(ptr::null_mut()), + once: UnsafeCell::new(c::INIT_ONCE_STATIC_INIT), + } } - key -} -#[inline] -pub unsafe fn set(key: Key, value: *mut u8) { - let r = c::TlsSetValue(key, value as c::LPVOID); - debug_assert!(r != 0); -} + #[inline] + pub unsafe fn set(&'static self, val: *mut u8) { + let r = c::TlsSetValue(self.key(), val.cast()); + debug_assert_eq!(r, c::TRUE); + } -#[inline] -pub unsafe fn get(key: Key) -> *mut u8 { - c::TlsGetValue(key) as *mut u8 -} + #[inline] + pub unsafe fn get(&'static self) -> *mut u8 { + c::TlsGetValue(self.key()).cast() + } -#[inline] -pub unsafe fn destroy(_key: Key) { - rtabort!("can't destroy tls keys on windows") -} + #[inline] + unsafe fn key(&'static self) -> Key { + match self.key.load(Acquire) { + 0 => self.init(), + key => key - 1, + } + } + + #[cold] + unsafe fn init(&'static self) -> Key { + if self.dtor.is_some() { + let mut pending = c::FALSE; + let r = c::InitOnceBeginInitialize(self.once.get(), 0, &mut pending, ptr::null_mut()); + assert_eq!(r, c::TRUE); -#[inline] -pub fn requires_synchronized_create() -> bool { - true + if pending == c::FALSE { + // Some other thread initialized the key, load it. + self.key.load(Relaxed) - 1 + } else { + let key = c::TlsAlloc(); + if key == c::TLS_OUT_OF_INDEXES { + // Wakeup the waiting threads before panicking to avoid deadlock. + c::InitOnceComplete(self.once.get(), c::INIT_ONCE_INIT_FAILED, ptr::null_mut()); + panic!("out of TLS indexes"); + } + + self.key.store(key + 1, Release); + register_dtor(self); + + let r = c::InitOnceComplete(self.once.get(), 0, ptr::null_mut()); + debug_assert_eq!(r, c::TRUE); + + key + } + } else { + // If there is no destructor to clean up, we can use racy initialization. + + let key = c::TlsAlloc(); + assert_ne!(key, c::TLS_OUT_OF_INDEXES, "out of TLS indexes"); + + match self.key.compare_exchange(0, key + 1, AcqRel, Acquire) { + Ok(_) => key, + Err(new) => { + // Some other thread completed initialization first, so destroy + // our key and use theirs. + let r = c::TlsFree(key); + debug_assert_eq!(r, c::TRUE); + new - 1 + } + } + } + } } +unsafe impl Send for StaticKey {} +unsafe impl Sync for StaticKey {} + // ------------------------------------------------------------------------- // Dtor registration // @@ -96,29 +150,21 @@ pub fn requires_synchronized_create() -> bool { // Typically processes have a statically known set of TLS keys which is pretty // small, and we'd want to keep this memory alive for the whole process anyway // really. -// -// Perhaps one day we can fold the `Box` here into a static allocation, -// expanding the `StaticKey` structure to contain not only a slot for the TLS -// key but also a slot for the destructor queue on windows. An optimization for -// another day! - -static DTORS: AtomicPtr = AtomicPtr::new(ptr::null_mut()); - -struct Node { - dtor: Dtor, - key: Key, - next: *mut Node, -} -unsafe fn register_dtor(key: Key, dtor: Dtor) { - let mut node = Box::new(Node { key, dtor, next: ptr::null_mut() }); +static DTORS: AtomicPtr = AtomicPtr::new(ptr::null_mut()); - let mut head = DTORS.load(SeqCst); +/// Should only be called once per key, otherwise loops or breaks may occur in +/// the linked list. +unsafe fn register_dtor(key: &'static StaticKey) { + let this = <*const StaticKey>::cast_mut(key); + // Use acquire ordering to pass along the changes done by the previously + // registered keys when we store the new head with release ordering. + let mut head = DTORS.load(Acquire); loop { - node.next = head; - match DTORS.compare_exchange(head, &mut *node, SeqCst, SeqCst) { - Ok(_) => return mem::forget(node), - Err(cur) => head = cur, + key.next.store(head, Relaxed); + match DTORS.compare_exchange_weak(head, this, Release, Acquire) { + Ok(_) => break, + Err(new) => head = new, } } } @@ -196,6 +242,8 @@ pub static p_thread_callback: unsafe extern "system" fn(c::LPVOID, c::DWORD, c:: unsafe extern "system" fn on_tls_callback(h: c::LPVOID, dwReason: c::DWORD, pv: c::LPVOID) { if dwReason == c::DLL_THREAD_DETACH || dwReason == c::DLL_PROCESS_DETACH { run_dtors(); + #[cfg(target_thread_local)] + super::thread_local_dtor::run_keyless_dtors(); } // See comments above for what this is doing. Note that we don't need this @@ -212,25 +260,29 @@ unsafe extern "system" fn on_tls_callback(h: c::LPVOID, dwReason: c::DWORD, pv: unsafe fn reference_tls_used() {} } -#[allow(dead_code)] // actually called above +#[allow(dead_code)] // actually called below unsafe fn run_dtors() { - let mut any_run = true; for _ in 0..5 { - if !any_run { - break; - } - any_run = false; - let mut cur = DTORS.load(SeqCst); + let mut any_run = false; + + // Use acquire ordering to observe key initialization. + let mut cur = DTORS.load(Acquire); while !cur.is_null() { - let ptr = c::TlsGetValue((*cur).key); + let key = (*cur).key.load(Relaxed) - 1; + let dtor = (*cur).dtor.unwrap(); + let ptr = c::TlsGetValue(key); if !ptr.is_null() { - c::TlsSetValue((*cur).key, ptr::null_mut()); - ((*cur).dtor)(ptr as *mut _); + c::TlsSetValue(key, ptr::null_mut()); + dtor(ptr as *mut _); any_run = true; } - cur = (*cur).next; + cur = (*cur).next.load(Relaxed); + } + + if !any_run { + break; } } } diff --git a/crux-mir/lib/std/src/sys/windows/thread_local_key/tests.rs b/crux-mir/lib/std/src/sys/windows/thread_local_key/tests.rs new file mode 100644 index 000000000..c95f383fb --- /dev/null +++ b/crux-mir/lib/std/src/sys/windows/thread_local_key/tests.rs @@ -0,0 +1,53 @@ +use super::StaticKey; +use crate::ptr; + +#[test] +fn smoke() { + static K1: StaticKey = StaticKey::new(None); + static K2: StaticKey = StaticKey::new(None); + + unsafe { + assert!(K1.get().is_null()); + assert!(K2.get().is_null()); + K1.set(ptr::invalid_mut(1)); + K2.set(ptr::invalid_mut(2)); + assert_eq!(K1.get() as usize, 1); + assert_eq!(K2.get() as usize, 2); + } +} + +#[test] +fn destructors() { + use crate::mem::ManuallyDrop; + use crate::sync::Arc; + use crate::thread; + + unsafe extern "C" fn destruct(ptr: *mut u8) { + drop(Arc::from_raw(ptr as *const ())); + } + + static KEY: StaticKey = StaticKey::new(Some(destruct)); + + let shared1 = Arc::new(()); + let shared2 = Arc::clone(&shared1); + + unsafe { + assert!(KEY.get().is_null()); + KEY.set(Arc::into_raw(shared1) as *mut u8); + } + + thread::spawn(move || unsafe { + assert!(KEY.get().is_null()); + KEY.set(Arc::into_raw(shared2) as *mut u8); + }) + .join() + .unwrap(); + + // Leak the Arc, let the TLS destructor clean it up. + let shared1 = unsafe { ManuallyDrop::new(Arc::from_raw(KEY.get() as *const ())) }; + assert_eq!( + Arc::strong_count(&shared1), + 1, + "destructor should have dropped the other reference on thread exit" + ); +} diff --git a/crux-mir/lib/std/src/sys/windows/thread_parking.rs b/crux-mir/lib/std/src/sys/windows/thread_parking.rs new file mode 100644 index 000000000..5d43676ad --- /dev/null +++ b/crux-mir/lib/std/src/sys/windows/thread_parking.rs @@ -0,0 +1,253 @@ +// Thread parker implementation for Windows. +// +// This uses WaitOnAddress and WakeByAddressSingle if available (Windows 8+). +// This modern API is exactly the same as the futex syscalls the Linux thread +// parker uses. When These APIs are available, the implementation of this +// thread parker matches the Linux thread parker exactly. +// +// However, when the modern API is not available, this implementation falls +// back to NT Keyed Events, which are similar, but have some important +// differences. These are available since Windows XP. +// +// WaitOnAddress first checks the state of the thread parker to make sure it no +// WakeByAddressSingle calls can be missed between updating the parker state +// and calling the function. +// +// NtWaitForKeyedEvent does not have this option, and unconditionally blocks +// without checking the parker state first. Instead, NtReleaseKeyedEvent +// (unlike WakeByAddressSingle) *blocks* until it woke up a thread waiting for +// it by NtWaitForKeyedEvent. This way, we can be sure no events are missed, +// but we need to be careful not to block unpark() if park_timeout() was woken +// up by a timeout instead of unpark(). +// +// Unlike WaitOnAddress, NtWaitForKeyedEvent/NtReleaseKeyedEvent operate on a +// HANDLE (created with NtCreateKeyedEvent). This means that we can be sure +// a successfully awoken park() was awoken by unpark() and not a +// NtReleaseKeyedEvent call from some other code, as these events are not only +// matched by the key (address of the parker (state)), but also by this HANDLE. +// We lazily allocate this handle the first time it is needed. +// +// The fast path (calling park() after unpark() was already called) and the +// possible states are the same for both implementations. This is used here to +// make sure the fast path does not even check which API to use, but can return +// right away, independent of the used API. Only the slow paths (which will +// actually block/wake a thread) check which API is available and have +// different implementations. +// +// Unfortunately, NT Keyed Events are an undocumented Windows API. However: +// - This API is relatively simple with obvious behaviour, and there are +// several (unofficial) articles documenting the details. [1] +// - `parking_lot` has been using this API for years (on Windows versions +// before Windows 8). [2] Many big projects extensively use parking_lot, +// such as servo and the Rust compiler itself. +// - It is the underlying API used by Windows SRW locks and Windows critical +// sections. [3] [4] +// - The source code of the implementations of Wine, ReactOs, and Windows XP +// are available and match the expected behaviour. +// - The main risk with an undocumented API is that it might change in the +// future. But since we only use it for older versions of Windows, that's not +// a problem. +// - Even if these functions do not block or wake as we expect (which is +// unlikely, see all previous points), this implementation would still be +// memory safe. The NT Keyed Events API is only used to sleep/block in the +// right place. +// +// [1]: http://www.locklessinc.com/articles/keyed_events/ +// [2]: https://github.com/Amanieu/parking_lot/commit/43abbc964e +// [3]: https://docs.microsoft.com/en-us/archive/msdn-magazine/2012/november/windows-with-c-the-evolution-of-synchronization-in-windows-and-c +// [4]: Windows Internals, Part 1, ISBN 9780735671300 + +use crate::pin::Pin; +use crate::ptr; +use crate::sync::atomic::{ + AtomicI8, AtomicPtr, + Ordering::{Acquire, Relaxed, Release}, +}; +use crate::sys::{c, dur2timeout}; +use crate::time::Duration; + +pub struct Parker { + state: AtomicI8, +} + +const PARKED: i8 = -1; +const EMPTY: i8 = 0; +const NOTIFIED: i8 = 1; + +// Notes about memory ordering: +// +// Memory ordering is only relevant for the relative ordering of operations +// between different variables. Even Ordering::Relaxed guarantees a +// monotonic/consistent order when looking at just a single atomic variable. +// +// So, since this parker is just a single atomic variable, we only need to look +// at the ordering guarantees we need to provide to the 'outside world'. +// +// The only memory ordering guarantee that parking and unparking provide, is +// that things which happened before unpark() are visible on the thread +// returning from park() afterwards. Otherwise, it was effectively unparked +// before unpark() was called while still consuming the 'token'. +// +// In other words, unpark() needs to synchronize with the part of park() that +// consumes the token and returns. +// +// This is done with a release-acquire synchronization, by using +// Ordering::Release when writing NOTIFIED (the 'token') in unpark(), and using +// Ordering::Acquire when reading this state in park() after waking up. +impl Parker { + /// Construct the Windows parker. The UNIX parker implementation + /// requires this to happen in-place. + pub unsafe fn new_in_place(parker: *mut Parker) { + parker.write(Self { state: AtomicI8::new(EMPTY) }); + } + + // Assumes this is only called by the thread that owns the Parker, + // which means that `self.state != PARKED`. This implementation doesn't require `Pin`, + // but other implementations do. + pub unsafe fn park(self: Pin<&Self>) { + // Change NOTIFIED=>EMPTY or EMPTY=>PARKED, and directly return in the + // first case. + if self.state.fetch_sub(1, Acquire) == NOTIFIED { + return; + } + + if let Some(wait_on_address) = c::WaitOnAddress::option() { + loop { + // Wait for something to happen, assuming it's still set to PARKED. + wait_on_address(self.ptr(), &PARKED as *const _ as c::LPVOID, 1, c::INFINITE); + // Change NOTIFIED=>EMPTY but leave PARKED alone. + if self.state.compare_exchange(NOTIFIED, EMPTY, Acquire, Acquire).is_ok() { + // Actually woken up by unpark(). + return; + } else { + // Spurious wake up. We loop to try again. + } + } + } else { + // Wait for unpark() to produce this event. + c::NtWaitForKeyedEvent(keyed_event_handle(), self.ptr(), 0, ptr::null_mut()); + // Set the state back to EMPTY (from either PARKED or NOTIFIED). + // Note that we don't just write EMPTY, but use swap() to also + // include an acquire-ordered read to synchronize with unpark()'s + // release-ordered write. + self.state.swap(EMPTY, Acquire); + } + } + + // Assumes this is only called by the thread that owns the Parker, + // which means that `self.state != PARKED`. This implementation doesn't require `Pin`, + // but other implementations do. + pub unsafe fn park_timeout(self: Pin<&Self>, timeout: Duration) { + // Change NOTIFIED=>EMPTY or EMPTY=>PARKED, and directly return in the + // first case. + if self.state.fetch_sub(1, Acquire) == NOTIFIED { + return; + } + + if let Some(wait_on_address) = c::WaitOnAddress::option() { + // Wait for something to happen, assuming it's still set to PARKED. + wait_on_address(self.ptr(), &PARKED as *const _ as c::LPVOID, 1, dur2timeout(timeout)); + // Set the state back to EMPTY (from either PARKED or NOTIFIED). + // Note that we don't just write EMPTY, but use swap() to also + // include an acquire-ordered read to synchronize with unpark()'s + // release-ordered write. + if self.state.swap(EMPTY, Acquire) == NOTIFIED { + // Actually woken up by unpark(). + } else { + // Timeout or spurious wake up. + // We return either way, because we can't easily tell if it was the + // timeout or not. + } + } else { + // Need to wait for unpark() using NtWaitForKeyedEvent. + let handle = keyed_event_handle(); + + // NtWaitForKeyedEvent uses a unit of 100ns, and uses negative + // values to indicate a relative time on the monotonic clock. + // This is documented here for the underlying KeWaitForSingleObject function: + // https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/nf-wdm-kewaitforsingleobject + let mut timeout = match i64::try_from((timeout.as_nanos() + 99) / 100) { + Ok(t) => -t, + Err(_) => i64::MIN, + }; + + // Wait for unpark() to produce this event. + let unparked = + c::NtWaitForKeyedEvent(handle, self.ptr(), 0, &mut timeout) == c::STATUS_SUCCESS; + + // Set the state back to EMPTY (from either PARKED or NOTIFIED). + let prev_state = self.state.swap(EMPTY, Acquire); + + if !unparked && prev_state == NOTIFIED { + // We were awoken by a timeout, not by unpark(), but the state + // was set to NOTIFIED, which means we *just* missed an + // unpark(), which is now blocked on us to wait for it. + // Wait for it to consume the event and unblock that thread. + c::NtWaitForKeyedEvent(handle, self.ptr(), 0, ptr::null_mut()); + } + } + } + + // This implementation doesn't require `Pin`, but other implementations do. + pub fn unpark(self: Pin<&Self>) { + // Change PARKED=>NOTIFIED, EMPTY=>NOTIFIED, or NOTIFIED=>NOTIFIED, and + // wake the thread in the first case. + // + // Note that even NOTIFIED=>NOTIFIED results in a write. This is on + // purpose, to make sure every unpark() has a release-acquire ordering + // with park(). + if self.state.swap(NOTIFIED, Release) == PARKED { + unsafe { + if let Some(wake_by_address_single) = c::WakeByAddressSingle::option() { + wake_by_address_single(self.ptr()); + } else { + // If we run NtReleaseKeyedEvent before the waiting thread runs + // NtWaitForKeyedEvent, this (shortly) blocks until we can wake it up. + // If the waiting thread wakes up before we run NtReleaseKeyedEvent + // (e.g. due to a timeout), this blocks until we do wake up a thread. + // To prevent this thread from blocking indefinitely in that case, + // park_impl() will, after seeing the state set to NOTIFIED after + // waking up, call NtWaitForKeyedEvent again to unblock us. + c::NtReleaseKeyedEvent(keyed_event_handle(), self.ptr(), 0, ptr::null_mut()); + } + } + } + } + + fn ptr(&self) -> c::LPVOID { + &self.state as *const _ as c::LPVOID + } +} + +fn keyed_event_handle() -> c::HANDLE { + const INVALID: c::HANDLE = ptr::invalid_mut(!0); + static HANDLE: AtomicPtr = AtomicPtr::new(INVALID); + match HANDLE.load(Relaxed) { + INVALID => { + let mut handle = c::INVALID_HANDLE_VALUE; + unsafe { + match c::NtCreateKeyedEvent( + &mut handle, + c::GENERIC_READ | c::GENERIC_WRITE, + ptr::null_mut(), + 0, + ) { + c::STATUS_SUCCESS => {} + r => panic!("Unable to create keyed event handle: error {r}"), + } + } + match HANDLE.compare_exchange(INVALID, handle, Relaxed, Relaxed) { + Ok(_) => handle, + Err(h) => { + // Lost the race to another thread initializing HANDLE before we did. + // Closing our handle and using theirs instead. + unsafe { + c::CloseHandle(handle); + } + h + } + } + } + handle => handle, + } +} diff --git a/crux-mir/lib/std/src/sys/windows/time.rs b/crux-mir/lib/std/src/sys/windows/time.rs index 900260169..b8209a854 100644 --- a/crux-mir/lib/std/src/sys/windows/time.rs +++ b/crux-mir/lib/std/src/sys/windows/time.rs @@ -1,8 +1,8 @@ use crate::cmp::Ordering; -use crate::convert::TryInto; use crate::fmt; use crate::mem; use crate::sys::c; +use crate::sys_common::IntoInner; use crate::time::Duration; use core::hash::{Hash, Hasher}; @@ -41,14 +41,6 @@ impl Instant { perf_counter::PerformanceCounterInstant::now().into() } - pub fn actually_monotonic() -> bool { - false - } - - pub const fn zero() -> Instant { - Instant { t: Duration::from_secs(0) } - } - pub fn checked_sub_instant(&self, other: &Instant) -> Option { // On windows there's a threshold below which we consider two timestamps // equivalent due to measurement error. For more details + doc link, @@ -145,6 +137,12 @@ impl From for SystemTime { } } +impl IntoInner for SystemTime { + fn into_inner(self) -> c::FILETIME { + self.t + } +} + impl Hash for SystemTime { fn hash(&self, state: &mut H) { self.intervals().hash(state) @@ -165,7 +163,7 @@ fn intervals2dur(intervals: u64) -> Duration { mod perf_counter { use super::NANOS_PER_SEC; - use crate::sync::atomic::{AtomicUsize, Ordering::SeqCst}; + use crate::sync::atomic::{AtomicU64, Ordering}; use crate::sys::c; use crate::sys::cvt; use crate::sys_common::mul_div_u64; @@ -197,27 +195,25 @@ mod perf_counter { } fn frequency() -> c::LARGE_INTEGER { - static mut FREQUENCY: c::LARGE_INTEGER = 0; - static STATE: AtomicUsize = AtomicUsize::new(0); - + // Either the cached result of `QueryPerformanceFrequency` or `0` for + // uninitialized. Storing this as a single `AtomicU64` allows us to use + // `Relaxed` operations, as we are only interested in the effects on a + // single memory location. + static FREQUENCY: AtomicU64 = AtomicU64::new(0); + + let cached = FREQUENCY.load(Ordering::Relaxed); + // If a previous thread has filled in this global state, use that. + if cached != 0 { + return cached as c::LARGE_INTEGER; + } + // ... otherwise learn for ourselves ... + let mut frequency = 0; unsafe { - // If a previous thread has filled in this global state, use that. - if STATE.load(SeqCst) == 2 { - return FREQUENCY; - } - - // ... otherwise learn for ourselves ... - let mut frequency = 0; cvt(c::QueryPerformanceFrequency(&mut frequency)).unwrap(); - - // ... and attempt to be the one thread that stores it globally for - // all other threads - if STATE.compare_exchange(0, 1, SeqCst, SeqCst).is_ok() { - FREQUENCY = frequency; - STATE.store(2, SeqCst); - } - frequency } + + FREQUENCY.store(frequency as u64, Ordering::Relaxed); + frequency } fn query() -> c::LARGE_INTEGER { diff --git a/crux-mir/lib/std/src/sys_common/at_exit_imp.rs b/crux-mir/lib/std/src/sys_common/at_exit_imp.rs deleted file mode 100644 index 6b799db85..000000000 --- a/crux-mir/lib/std/src/sys_common/at_exit_imp.rs +++ /dev/null @@ -1,75 +0,0 @@ -//! Implementation of running at_exit routines -//! -//! Documentation can be found on the `rt::at_exit` function. - -use crate::mem; -use crate::ptr; -use crate::sys_common::mutex::Mutex; - -type Queue = Vec>; - -// NB these are specifically not types from `std::sync` as they currently rely -// on poisoning and this module needs to operate at a lower level than requiring -// the thread infrastructure to be in place (useful on the borders of -// initialization/destruction). -// We never call `LOCK.init()`, so it is UB to attempt to -// acquire this mutex reentrantly! -static LOCK: Mutex = Mutex::new(); -static mut QUEUE: *mut Queue = ptr::null_mut(); - -const DONE: *mut Queue = 1_usize as *mut _; - -// The maximum number of times the cleanup routines will be run. While running -// the at_exit closures new ones may be registered, and this count is the number -// of times the new closures will be allowed to register successfully. After -// this number of iterations all new registrations will return `false`. -const ITERS: usize = 10; - -unsafe fn init() -> bool { - if QUEUE.is_null() { - let state: Box = box Vec::new(); - QUEUE = Box::into_raw(state); - } else if QUEUE == DONE { - // can't re-init after a cleanup - return false; - } - - true -} - -pub fn cleanup() { - for i in 1..=ITERS { - unsafe { - let queue = { - let _guard = LOCK.lock(); - mem::replace(&mut QUEUE, if i == ITERS { DONE } else { ptr::null_mut() }) - }; - - // make sure we're not recursively cleaning up - assert!(queue != DONE); - - // If we never called init, not need to cleanup! - if !queue.is_null() { - let queue: Box = Box::from_raw(queue); - for to_run in *queue { - // We are not holding any lock, so reentrancy is fine. - to_run(); - } - } - } - } -} - -pub fn push(f: Box) -> bool { - unsafe { - let _guard = LOCK.lock(); - if init() { - // We are just moving `f` around, not calling it. - // There is no possibility of reentrancy here. - (*QUEUE).push(f); - true - } else { - false - } - } -} diff --git a/crux-mir/lib/std/src/sys_common/backtrace.rs b/crux-mir/lib/std/src/sys_common/backtrace.rs index e9b1e86d7..f1d804ef4 100644 --- a/crux-mir/lib/std/src/sys_common/backtrace.rs +++ b/crux-mir/lib/std/src/sys_common/backtrace.rs @@ -1,3 +1,4 @@ +use crate::backtrace_rs::{self, BacktraceFmt, BytesOrWideString, PrintFmt}; use crate::borrow::Cow; /// Common code for printing the backtrace in the same way across the different /// supported platforms. @@ -6,36 +7,20 @@ use crate::fmt; use crate::io; use crate::io::prelude::*; use crate::path::{self, Path, PathBuf}; -use crate::sync::atomic::{self, Ordering}; -use crate::sys::mutex::Mutex; - -use backtrace_rs::{BacktraceFmt, BytesOrWideString, PrintFmt}; +use crate::sync::{Mutex, PoisonError}; /// Max number of frames to print. const MAX_NB_FRAMES: usize = 100; pub fn lock() -> impl Drop { - struct Guard; - static LOCK: Mutex = Mutex::new(); - - impl Drop for Guard { - fn drop(&mut self) { - unsafe { - LOCK.unlock(); - } - } - } - - unsafe { - LOCK.lock(); - Guard - } + static LOCK: Mutex<()> = Mutex::new(()); + LOCK.lock().unwrap_or_else(PoisonError::into_inner) } /// Prints the current backtrace. pub fn print(w: &mut dyn Write, format: PrintFmt) -> io::Result<()> { // There are issues currently linking libbacktrace into tests, and in - // general during libstd's own unit tests we're not testing this path. In + // general during std's own unit tests we're not testing this path. In // test mode immediately return here to optimize away any references to the // libbacktrace symbols if cfg!(test) { @@ -75,6 +60,8 @@ unsafe fn _print_fmt(fmt: &mut fmt::Formatter<'_>, print_fmt: PrintFmt) -> fmt:: bt_fmt.add_context()?; let mut idx = 0; let mut res = Ok(()); + // Start immediately if we're not using a short backtrace. + let mut start = print_fmt != PrintFmt::Short; backtrace_rs::trace_unsynchronized(|frame| { if print_fmt == PrintFmt::Short && idx > MAX_NB_FRAMES { return false; @@ -86,19 +73,25 @@ unsafe fn _print_fmt(fmt: &mut fmt::Formatter<'_>, print_fmt: PrintFmt) -> fmt:: hit = true; if print_fmt == PrintFmt::Short { if let Some(sym) = symbol.name().and_then(|s| s.as_str()) { - if sym.contains("__rust_begin_short_backtrace") { + if start && sym.contains("__rust_begin_short_backtrace") { stop = true; return; } + if sym.contains("__rust_end_short_backtrace") { + start = true; + return; + } } } - res = bt_fmt.frame().symbol(frame, symbol); + if start { + res = bt_fmt.frame().symbol(frame, symbol); + } }); if stop { return false; } - if !hit { + if !hit && start { res = bt_fmt.frame().print_raw(frame.ip(), None, None, None); } @@ -118,61 +111,35 @@ unsafe fn _print_fmt(fmt: &mut fmt::Formatter<'_>, print_fmt: PrintFmt) -> fmt:: } /// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`. Note that -/// this is only inline(never) when backtraces in libstd are enabled, otherwise +/// this is only inline(never) when backtraces in std are enabled, otherwise /// it's fine to optimize away. #[cfg_attr(feature = "backtrace", inline(never))] pub fn __rust_begin_short_backtrace(f: F) -> T where F: FnOnce() -> T, - F: Send, - T: Send, { - f() -} + let result = f(); -pub enum RustBacktrace { - Print(PrintFmt), - Disabled, - RuntimeDisabled, -} + // prevent this frame from being tail-call optimised away + crate::hint::black_box(()); -// For now logging is turned off by default, and this function checks to see -// whether the magical environment variable is present to see if it's turned on. -pub fn rust_backtrace_env() -> RustBacktrace { - // If the `backtrace` feature of this crate isn't enabled quickly return - // `None` so this can be constant propagated all over the place to turn - // optimize away callers. - if !cfg!(feature = "backtrace") { - return RustBacktrace::Disabled; - } + result +} - // Setting environment variables for Fuchsia components isn't a standard - // or easily supported workflow. For now, always display backtraces. - if cfg!(target_os = "fuchsia") { - return RustBacktrace::Print(PrintFmt::Full); - } +/// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`. Note that +/// this is only inline(never) when backtraces in std are enabled, otherwise +/// it's fine to optimize away. +#[cfg_attr(feature = "backtrace", inline(never))] +pub fn __rust_end_short_backtrace(f: F) -> T +where + F: FnOnce() -> T, +{ + let result = f(); - static ENABLED: atomic::AtomicIsize = atomic::AtomicIsize::new(0); - match ENABLED.load(Ordering::SeqCst) { - 0 => {} - 1 => return RustBacktrace::RuntimeDisabled, - 2 => return RustBacktrace::Print(PrintFmt::Short), - _ => return RustBacktrace::Print(PrintFmt::Full), - } + // prevent this frame from being tail-call optimised away + crate::hint::black_box(()); - let (format, cache) = env::var_os("RUST_BACKTRACE") - .map(|x| { - if &x == "0" { - (RustBacktrace::RuntimeDisabled, 1) - } else if &x == "full" { - (RustBacktrace::Print(PrintFmt::Full), 3) - } else { - (RustBacktrace::Print(PrintFmt::Short), 2) - } - }) - .unwrap_or((RustBacktrace::RuntimeDisabled, 1)); - ENABLED.store(cache, Ordering::SeqCst); - format + result } /// Prints the filename of the backtrace frame. @@ -206,7 +173,7 @@ pub fn output_filename( if let Some(cwd) = cwd { if let Ok(stripped) = file.strip_prefix(&cwd) { if let Some(s) = stripped.to_str() { - return write!(fmt, ".{}{}", path::MAIN_SEPARATOR, s); + return write!(fmt, ".{}{s}", path::MAIN_SEPARATOR); } } } diff --git a/crux-mir/lib/std/src/sys_common/bytestring.rs b/crux-mir/lib/std/src/sys_common/bytestring.rs deleted file mode 100644 index dccc3bc4a..000000000 --- a/crux-mir/lib/std/src/sys_common/bytestring.rs +++ /dev/null @@ -1,46 +0,0 @@ -#![allow(dead_code)] - -use crate::fmt::{Formatter, Result, Write}; -use core::str::lossy::{Utf8Lossy, Utf8LossyChunk}; - -pub fn debug_fmt_bytestring(slice: &[u8], f: &mut Formatter<'_>) -> Result { - // Writes out a valid unicode string with the correct escape sequences - fn write_str_escaped(f: &mut Formatter<'_>, s: &str) -> Result { - for c in s.chars().flat_map(|c| c.escape_debug()) { - f.write_char(c)? - } - Ok(()) - } - - f.write_str("\"")?; - for Utf8LossyChunk { valid, broken } in Utf8Lossy::from_bytes(slice).chunks() { - write_str_escaped(f, valid)?; - for b in broken { - write!(f, "\\x{:02X}", b)?; - } - } - f.write_str("\"") -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::fmt::{Debug, Formatter, Result}; - - #[test] - fn smoke() { - struct Helper<'a>(&'a [u8]); - - impl Debug for Helper<'_> { - fn fmt(&self, f: &mut Formatter<'_>) -> Result { - debug_fmt_bytestring(self.0, f) - } - } - - let input = b"\xF0hello,\tworld"; - let expected = r#""\xF0hello,\tworld""#; - let output = format!("{:?}", Helper(input)); - - assert!(output == expected); - } -} diff --git a/crux-mir/lib/std/src/sys_common/condvar.rs b/crux-mir/lib/std/src/sys_common/condvar.rs deleted file mode 100644 index 554185270..000000000 --- a/crux-mir/lib/std/src/sys_common/condvar.rs +++ /dev/null @@ -1,72 +0,0 @@ -use crate::sys::crux::condvar as imp; -use crate::sys_common::mutex::{self, Mutex}; -use crate::time::Duration; - -/// An OS-based condition variable. -/// -/// This structure is the lowest layer possible on top of the OS-provided -/// condition variables. It is consequently entirely unsafe to use. It is -/// recommended to use the safer types at the top level of this crate instead of -/// this type. -pub struct Condvar(imp::Condvar); - -impl Condvar { - /// Creates a new condition variable for use. - /// - /// Behavior is undefined if the condition variable is moved after it is - /// first used with any of the functions below. - pub const fn new() -> Condvar { - Condvar(imp::Condvar::new()) - } - - /// Prepares the condition variable for use. - /// - /// This should be called once the condition variable is at a stable memory - /// address. - #[inline] - pub unsafe fn init(&mut self) { - self.0.init() - } - - /// Signals one waiter on this condition variable to wake up. - #[inline] - pub unsafe fn notify_one(&self) { - self.0.notify_one() - } - - /// Awakens all current waiters on this condition variable. - #[inline] - pub unsafe fn notify_all(&self) { - self.0.notify_all() - } - - /// Waits for a signal on the specified mutex. - /// - /// Behavior is undefined if the mutex is not locked by the current thread. - /// Behavior is also undefined if more than one mutex is used concurrently - /// on this condition variable. - #[inline] - pub unsafe fn wait(&self, mutex: &Mutex) { - self.0.wait(mutex::raw(mutex)) - } - - /// Waits for a signal on the specified mutex with a timeout duration - /// specified by `dur` (a relative time into the future). - /// - /// Behavior is undefined if the mutex is not locked by the current thread. - /// Behavior is also undefined if more than one mutex is used concurrently - /// on this condition variable. - #[inline] - pub unsafe fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool { - self.0.wait_timeout(mutex::raw(mutex), dur) - } - - /// Deallocates all resources associated with this condition variable. - /// - /// Behavior is undefined if there are current or will be future users of - /// this condition variable. - #[inline] - pub unsafe fn destroy(&self) { - self.0.destroy() - } -} diff --git a/crux-mir/lib/std/src/sys_common/fs.rs b/crux-mir/lib/std/src/sys_common/fs.rs index e30e8018a..617ac52e5 100644 --- a/crux-mir/lib/std/src/sys_common/fs.rs +++ b/crux-mir/lib/std/src/sys_common/fs.rs @@ -4,20 +4,24 @@ use crate::fs; use crate::io::{self, Error, ErrorKind}; use crate::path::Path; +pub(crate) const NOT_FILE_ERROR: Error = io::const_io_error!( + ErrorKind::InvalidInput, + "the source path is neither a regular file nor a symlink to a regular file", +); + pub fn copy(from: &Path, to: &Path) -> io::Result { - if !from.is_file() { - return Err(Error::new( - ErrorKind::InvalidInput, - "the source path is not an existing regular file", - )); + let mut reader = fs::File::open(from)?; + let metadata = reader.metadata()?; + + if !metadata.is_file() { + return Err(NOT_FILE_ERROR); } - let mut reader = fs::File::open(from)?; let mut writer = fs::File::create(to)?; - let perm = reader.metadata()?.permissions(); + let perm = metadata.permissions(); let ret = io::copy(&mut reader, &mut writer)?; - fs::set_permissions(to, perm)?; + writer.set_permissions(perm)?; Ok(ret) } @@ -37,3 +41,11 @@ fn remove_dir_all_recursive(path: &Path) -> io::Result<()> { } fs::remove_dir(path) } + +pub fn try_exists(path: &Path) -> io::Result { + match fs::metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} diff --git a/crux-mir/lib/std/src/sys_common/io.rs b/crux-mir/lib/std/src/sys_common/io.rs index 7c1d98a5a..4a42ff3c6 100644 --- a/crux-mir/lib/std/src/sys_common/io.rs +++ b/crux-mir/lib/std/src/sys_common/io.rs @@ -1,4 +1,6 @@ -pub const DEFAULT_BUF_SIZE: usize = 8 * 1024; +// Bare metal platforms usually have very small amounts of RAM +// (in the order of hundreds of KB) +pub const DEFAULT_BUF_SIZE: usize = if cfg!(target_os = "espidf") { 512 } else { 8 * 1024 }; #[cfg(test)] #[allow(dead_code)] // not used on emscripten @@ -6,6 +8,7 @@ pub mod test { use crate::env; use crate::fs; use crate::path::{Path, PathBuf}; + use crate::thread; use rand::RngCore; pub struct TempDir(PathBuf); @@ -27,13 +30,19 @@ pub mod test { // Gee, seeing how we're testing the fs module I sure hope that we // at least implement this correctly! let TempDir(ref p) = *self; - fs::remove_dir_all(p).unwrap(); + let result = fs::remove_dir_all(p); + // Avoid panicking while panicking as this causes the process to + // immediately abort, without displaying test results. + if !thread::panicking() { + result.unwrap(); + } } } + #[track_caller] // for `test_rng` pub fn tmpdir() -> TempDir { let p = env::temp_dir(); - let mut r = rand::thread_rng(); + let mut r = crate::test_helpers::test_rng(); let ret = p.join(&format!("rust-{}", r.next_u32())); fs::create_dir(&ret).unwrap(); TempDir(ret) diff --git a/crux-mir/lib/std/src/sys_common/lazy_box.rs b/crux-mir/lib/std/src/sys_common/lazy_box.rs new file mode 100644 index 000000000..63c3316bd --- /dev/null +++ b/crux-mir/lib/std/src/sys_common/lazy_box.rs @@ -0,0 +1,90 @@ +#![allow(dead_code)] // Only used on some platforms. + +// This is used to wrap pthread {Mutex, Condvar, RwLock} in. + +use crate::marker::PhantomData; +use crate::ops::{Deref, DerefMut}; +use crate::ptr::null_mut; +use crate::sync::atomic::{ + AtomicPtr, + Ordering::{AcqRel, Acquire}, +}; + +pub(crate) struct LazyBox { + ptr: AtomicPtr, + _phantom: PhantomData, +} + +pub(crate) trait LazyInit { + /// This is called before the box is allocated, to provide the value to + /// move into the new box. + /// + /// It might be called more than once per LazyBox, as multiple threads + /// might race to initialize it concurrently, each constructing and initializing + /// their own box. All but one of them will be passed to `cancel_init` right after. + fn init() -> Box; + + /// Any surplus boxes from `init()` that lost the initialization race + /// are passed to this function for disposal. + /// + /// The default implementation calls destroy(). + fn cancel_init(x: Box) { + Self::destroy(x); + } + + /// This is called to destroy a used box. + /// + /// The default implementation just drops it. + fn destroy(_: Box) {} +} + +impl LazyBox { + #[inline] + pub const fn new() -> Self { + Self { ptr: AtomicPtr::new(null_mut()), _phantom: PhantomData } + } + + #[inline] + fn get_pointer(&self) -> *mut T { + let ptr = self.ptr.load(Acquire); + if ptr.is_null() { self.initialize() } else { ptr } + } + + #[cold] + fn initialize(&self) -> *mut T { + let new_ptr = Box::into_raw(T::init()); + match self.ptr.compare_exchange(null_mut(), new_ptr, AcqRel, Acquire) { + Ok(_) => new_ptr, + Err(ptr) => { + // Lost the race to another thread. + // Drop the box we created, and use the one from the other thread instead. + T::cancel_init(unsafe { Box::from_raw(new_ptr) }); + ptr + } + } + } +} + +impl Deref for LazyBox { + type Target = T; + #[inline] + fn deref(&self) -> &T { + unsafe { &*self.get_pointer() } + } +} + +impl DerefMut for LazyBox { + #[inline] + fn deref_mut(&mut self) -> &mut T { + unsafe { &mut *self.get_pointer() } + } +} + +impl Drop for LazyBox { + fn drop(&mut self) { + let ptr = *self.ptr.get_mut(); + if !ptr.is_null() { + T::destroy(unsafe { Box::from_raw(ptr) }); + } + } +} diff --git a/crux-mir/lib/std/src/sys_common/memchr.rs b/crux-mir/lib/std/src/sys_common/memchr.rs new file mode 100644 index 000000000..b219e8789 --- /dev/null +++ b/crux-mir/lib/std/src/sys_common/memchr.rs @@ -0,0 +1,51 @@ +// Original implementation taken from rust-memchr. +// Copyright 2015 Andrew Gallant, bluss and Nicolas Koch + +use crate::sys::memchr as sys; + +#[cfg(test)] +mod tests; + +/// A safe interface to `memchr`. +/// +/// Returns the index corresponding to the first occurrence of `needle` in +/// `haystack`, or `None` if one is not found. +/// +/// memchr reduces to super-optimized machine code at around an order of +/// magnitude faster than `haystack.iter().position(|&b| b == needle)`. +/// (See benchmarks.) +/// +/// # Examples +/// +/// This shows how to find the first position of a byte in a byte string. +/// +/// ```ignore (cannot-doctest-private-modules) +/// use memchr::memchr; +/// +/// let haystack = b"the quick brown fox"; +/// assert_eq!(memchr(b'k', haystack), Some(8)); +/// ``` +#[inline] +pub fn memchr(needle: u8, haystack: &[u8]) -> Option { + sys::memchr(needle, haystack) +} + +/// A safe interface to `memrchr`. +/// +/// Returns the index corresponding to the last occurrence of `needle` in +/// `haystack`, or `None` if one is not found. +/// +/// # Examples +/// +/// This shows how to find the last position of a byte in a byte string. +/// +/// ```ignore (cannot-doctest-private-modules) +/// use memchr::memrchr; +/// +/// let haystack = b"the quick brown fox"; +/// assert_eq!(memrchr(b'o', haystack), Some(17)); +/// ``` +#[inline] +pub fn memrchr(needle: u8, haystack: &[u8]) -> Option { + sys::memrchr(needle, haystack) +} diff --git a/crux-mir/lib/std/src/sys_common/memchr/tests.rs b/crux-mir/lib/std/src/sys_common/memchr/tests.rs new file mode 100644 index 000000000..557d749c7 --- /dev/null +++ b/crux-mir/lib/std/src/sys_common/memchr/tests.rs @@ -0,0 +1,86 @@ +// Original implementation taken from rust-memchr. +// Copyright 2015 Andrew Gallant, bluss and Nicolas Koch + +// test the implementations for the current platform +use super::{memchr, memrchr}; + +#[test] +fn matches_one() { + assert_eq!(Some(0), memchr(b'a', b"a")); +} + +#[test] +fn matches_begin() { + assert_eq!(Some(0), memchr(b'a', b"aaaa")); +} + +#[test] +fn matches_end() { + assert_eq!(Some(4), memchr(b'z', b"aaaaz")); +} + +#[test] +fn matches_nul() { + assert_eq!(Some(4), memchr(b'\x00', b"aaaa\x00")); +} + +#[test] +fn matches_past_nul() { + assert_eq!(Some(5), memchr(b'z', b"aaaa\x00z")); +} + +#[test] +fn no_match_empty() { + assert_eq!(None, memchr(b'a', b"")); +} + +#[test] +fn no_match() { + assert_eq!(None, memchr(b'a', b"xyz")); +} + +#[test] +fn matches_one_reversed() { + assert_eq!(Some(0), memrchr(b'a', b"a")); +} + +#[test] +fn matches_begin_reversed() { + assert_eq!(Some(3), memrchr(b'a', b"aaaa")); +} + +#[test] +fn matches_end_reversed() { + assert_eq!(Some(0), memrchr(b'z', b"zaaaa")); +} + +#[test] +fn matches_nul_reversed() { + assert_eq!(Some(4), memrchr(b'\x00', b"aaaa\x00")); +} + +#[test] +fn matches_past_nul_reversed() { + assert_eq!(Some(0), memrchr(b'z', b"z\x00aaaa")); +} + +#[test] +fn no_match_empty_reversed() { + assert_eq!(None, memrchr(b'a', b"")); +} + +#[test] +fn no_match_reversed() { + assert_eq!(None, memrchr(b'a', b"xyz")); +} + +#[test] +fn each_alignment() { + let mut data = [1u8; 64]; + let needle = 2; + let pos = 40; + data[pos] = needle; + for start in 0..16 { + assert_eq!(Some(pos - start), memchr(needle, &data[start..])); + } +} diff --git a/crux-mir/lib/std/src/sys_common/mod.rs b/crux-mir/lib/std/src/sys_common/mod.rs index e03e0fc83..6b24b0e9a 100644 --- a/crux-mir/lib/std/src/sys_common/mod.rs +++ b/crux-mir/lib/std/src/sys_common/mod.rs @@ -8,72 +8,45 @@ //! rest of `std` is complex, with dependencies going in all //! directions: `std` depending on `sys_common`, `sys_common` //! depending on `sys`, and `sys` depending on `sys_common` and `std`. -//! Ideally `sys_common` would be split into two and the dependencies -//! between them all would form a dag, facilitating the extraction of -//! `std::sys` from the standard library. +//! This is because `sys_common` not only contains platform-independent code, +//! but also code that is shared between the different platforms in `sys`. +//! Ideally all that shared code should be moved to `sys::common`, +//! and the dependencies between `std`, `sys_common` and `sys` all would form a dag. +//! Progress on this is tracked in #84187. #![allow(missing_docs)] #![allow(missing_debug_implementations)] -use crate::sync::Once; -use crate::sys; +#[cfg(test)] +mod tests; -macro_rules! rtabort { - ($($t:tt)*) => (crate::sys_common::util::abort(format_args!($($t)*))) -} - -macro_rules! rtassert { - ($e:expr) => { - if !$e { - rtabort!(concat!("assertion failed: ", stringify!($e))); - } - }; -} - -#[allow(unused_macros)] // not used on all platforms -macro_rules! rtunwrap { - ($ok:ident, $e:expr) => { - match $e { - $ok(v) => v, - ref err => { - let err = err.as_ref().map(drop); // map Ok/Some which might not be Debug - rtabort!(concat!("unwrap failed: ", stringify!($e), " = {:?}"), err) - } - } - }; -} - -pub mod alloc; -pub mod at_exit_imp; pub mod backtrace; -pub mod bytestring; -pub mod condvar; pub mod fs; pub mod io; -pub mod mutex; -#[cfg(any(doc, // see `mod os`, docs are generated for multiple platforms - unix, - target_os = "redox", - target_os = "cloudabi", - target_os = "hermit", - target_arch = "wasm32", - all(target_vendor = "fortanix", target_env = "sgx")))] -pub mod os_str_bytes; -pub mod poison; +pub mod lazy_box; +pub mod memchr; +pub mod once; pub mod process; -pub mod remutex; -pub mod rwlock; pub mod thread; pub mod thread_info; -pub mod thread_local; -pub mod util; +pub mod thread_local_dtor; +pub mod thread_parking; +pub mod wstr; pub mod wtf8; cfg_if::cfg_if! { - if #[cfg(any(target_os = "cloudabi", - target_os = "l4re", + if #[cfg(target_os = "windows")] { + pub use crate::sys::thread_local_key; + } else { + pub mod thread_local_key; + } +} + +cfg_if::cfg_if! { + if #[cfg(any(target_os = "l4re", target_os = "hermit", - all(target_arch = "wasm32", not(target_os = "emscripten")), + feature = "restricted-std", + all(target_family = "wasm", not(target_os = "emscripten")), all(target_vendor = "fortanix", target_env = "sgx")))] { pub use crate::sys::net; } else { @@ -107,30 +80,6 @@ pub trait FromInner { fn from_inner(inner: Inner) -> Self; } -/// Enqueues a procedure to run when the main thread exits. -/// -/// Currently these closures are only run once the main *Rust* thread exits. -/// Once the `at_exit` handlers begin running, more may be enqueued, but not -/// infinitely so. Eventually a handler registration will be forced to fail. -/// -/// Returns `Ok` if the handler was successfully registered, meaning that the -/// closure will be run once the main thread exits. Returns `Err` to indicate -/// that the closure could not be registered, meaning that it is not scheduled -/// to be run. -pub fn at_exit(f: F) -> Result<(), ()> { - if at_exit_imp::push(Box::new(f)) { Ok(()) } else { Err(()) } -} - -/// One-time runtime cleanup. -pub fn cleanup() { - static CLEANUP: Once = Once::new(); - CLEANUP.call_once(|| unsafe { - sys::args::cleanup(); - sys::stack_overflow::cleanup(); - at_exit_imp::cleanup(); - }); -} - // Computes (value*numer)/denom without overflow, as long as both // (numer*denom) and the overall result fit into i64 (which is the case // for our time conversions). @@ -143,8 +92,3 @@ pub fn mul_div_u64(value: u64, numer: u64, denom: u64) -> u64 { // r < denom, so (denom*numer) is the upper bound of (r*numer) q * numer + r * numer / denom } - -#[test] -fn test_muldiv() { - assert_eq!(mul_div_u64(1_000_000_000_001, 1_000_000_000, 1_000_000), 1_000_000_000_001_000); -} diff --git a/crux-mir/lib/std/src/sys_common/mutex.rs b/crux-mir/lib/std/src/sys_common/mutex.rs deleted file mode 100644 index ce74ebee1..000000000 --- a/crux-mir/lib/std/src/sys_common/mutex.rs +++ /dev/null @@ -1,100 +0,0 @@ -use crate::sys::crux::mutex as imp; - -/// An OS-based mutual exclusion lock. -/// -/// This is the thinnest cross-platform wrapper around OS mutexes. All usage of -/// this mutex is unsafe and it is recommended to instead use the safe wrapper -/// at the top level of the crate instead of this type. -pub struct Mutex(imp::Mutex); - -unsafe impl Sync for Mutex {} - -impl Mutex { - /// Creates a new mutex for use. - /// - /// Behavior is undefined if the mutex is moved after it is - /// first used with any of the functions below. - /// Also, until `init` is called, behavior is undefined if this - /// mutex is ever used reentrantly, i.e., `raw_lock` or `try_lock` - /// are called by the thread currently holding the lock. - pub const fn new() -> Mutex { - Mutex(imp::Mutex::new()) - } - - /// Prepare the mutex for use. - /// - /// This should be called once the mutex is at a stable memory address. - /// If called, this must be the very first thing that happens to the mutex. - /// Calling it in parallel with or after any operation (including another - /// `init()`) is undefined behavior. - #[inline] - pub unsafe fn init(&mut self) { - self.0.init() - } - - /// Locks the mutex blocking the current thread until it is available. - /// - /// Behavior is undefined if the mutex has been moved between this and any - /// previous function call. - #[inline] - pub unsafe fn raw_lock(&self) { - self.0.lock() - } - - /// Calls raw_lock() and then returns an RAII guard to guarantee the mutex - /// will be unlocked. - #[inline] - pub unsafe fn lock(&self) -> MutexGuard<'_> { - self.raw_lock(); - MutexGuard(&self.0) - } - - /// Attempts to lock the mutex without blocking, returning whether it was - /// successfully acquired or not. - /// - /// Behavior is undefined if the mutex has been moved between this and any - /// previous function call. - #[inline] - pub unsafe fn try_lock(&self) -> bool { - self.0.try_lock() - } - - /// Unlocks the mutex. - /// - /// Behavior is undefined if the current thread does not actually hold the - /// mutex. - /// - /// Consider switching from the pair of raw_lock() and raw_unlock() to - /// lock() whenever possible. - #[inline] - pub unsafe fn raw_unlock(&self) { - self.0.unlock() - } - - /// Deallocates all resources associated with this mutex. - /// - /// Behavior is undefined if there are current or will be future users of - /// this mutex. - #[inline] - pub unsafe fn destroy(&self) { - self.0.destroy() - } -} - -// not meant to be exported to the outside world, just the containing module -pub fn raw(mutex: &Mutex) -> &imp::Mutex { - &mutex.0 -} - -#[must_use] -/// A simple RAII utility for the above Mutex without the poisoning semantics. -pub struct MutexGuard<'a>(&'a imp::Mutex); - -impl Drop for MutexGuard<'_> { - #[inline] - fn drop(&mut self) { - unsafe { - self.0.unlock(); - } - } -} diff --git a/crux-mir/lib/std/src/sys_common/net.rs b/crux-mir/lib/std/src/sys_common/net.rs index 135e8308a..fad4a6333 100644 --- a/crux-mir/lib/std/src/sys_common/net.rs +++ b/crux-mir/lib/std/src/sys_common/net.rs @@ -1,11 +1,14 @@ +#[cfg(test)] +mod tests; + use crate::cmp; use crate::convert::{TryFrom, TryInto}; -use crate::ffi::CString; use crate::fmt; -use crate::io::{self, Error, ErrorKind, IoSlice, IoSliceMut}; +use crate::io::{self, ErrorKind, IoSlice, IoSliceMut}; use crate::mem; use crate::net::{Ipv4Addr, Ipv6Addr, Shutdown, SocketAddr}; use crate::ptr; +use crate::sys::common::small_c_string::run_with_cstr; use crate::sys::net::netc as c; use crate::sys::net::{cvt, cvt_gai, cvt_r, init, wrlen_t, Socket}; use crate::sys_common::{AsInner, FromInner, IntoInner}; @@ -16,8 +19,8 @@ use libc::{c_int, c_void}; cfg_if::cfg_if! { if #[cfg(any( target_os = "dragonfly", target_os = "freebsd", - target_os = "ios", target_os = "macos", - target_os = "openbsd", target_os = "netbsd", + target_os = "ios", target_os = "macos", target_os = "watchos", + target_os = "openbsd", target_os = "netbsd", target_os = "illumos", target_os = "solaris", target_os = "haiku", target_os = "l4re"))] { use crate::sys::net::netc::IPV6_JOIN_GROUP as IPV6_ADD_MEMBERSHIP; use crate::sys::net::netc::IPV6_LEAVE_GROUP as IPV6_DROP_MEMBERSHIP; @@ -43,7 +46,7 @@ cfg_if::cfg_if! { if #[cfg(any( target_os = "dragonfly", target_os = "freebsd", target_os = "openbsd", target_os = "netbsd", - target_os = "solaris"))] { + target_os = "solaris", target_os = "illumos"))] { use libc::c_uchar; type IpV4MultiCastType = c_uchar; } else { @@ -55,27 +58,36 @@ cfg_if::cfg_if! { // sockaddr and misc bindings //////////////////////////////////////////////////////////////////////////////// -pub fn setsockopt(sock: &Socket, opt: c_int, val: c_int, payload: T) -> io::Result<()> { +pub fn setsockopt( + sock: &Socket, + level: c_int, + option_name: c_int, + option_value: T, +) -> io::Result<()> { unsafe { - let payload = &payload as *const T as *const c_void; cvt(c::setsockopt( - *sock.as_inner(), - opt, - val, - payload, + sock.as_raw(), + level, + option_name, + &option_value as *const T as *const _, mem::size_of::() as c::socklen_t, ))?; Ok(()) } } -pub fn getsockopt(sock: &Socket, opt: c_int, val: c_int) -> io::Result { +pub fn getsockopt(sock: &Socket, level: c_int, option_name: c_int) -> io::Result { unsafe { - let mut slot: T = mem::zeroed(); - let mut len = mem::size_of::() as c::socklen_t; - cvt(c::getsockopt(*sock.as_inner(), opt, val, &mut slot as *mut _ as *mut _, &mut len))?; - assert_eq!(len as usize, mem::size_of::()); - Ok(slot) + let mut option_value: T = mem::zeroed(); + let mut option_len = mem::size_of::() as c::socklen_t; + cvt(c::getsockopt( + sock.as_raw(), + level, + option_name, + &mut option_value as *mut T as *mut _, + &mut option_len, + ))?; + Ok(option_value) } } @@ -105,7 +117,7 @@ pub fn sockaddr_to_addr(storage: &c::sockaddr_storage, len: usize) -> io::Result *(storage as *const _ as *const c::sockaddr_in6) }))) } - _ => Err(Error::new(ErrorKind::InvalidInput, "invalid argument")), + _ => Err(io::const_io_error!(ErrorKind::InvalidInput, "invalid argument")), } } @@ -168,17 +180,14 @@ impl TryFrom<&str> for LookupHost { ($e:expr, $msg:expr) => { match $e { Some(r) => r, - None => return Err(io::Error::new(io::ErrorKind::InvalidInput, $msg)), + None => return Err(io::const_io_error!(io::ErrorKind::InvalidInput, $msg)), } }; } // split the string by ':' and convert the second part to u16 - let mut parts_iter = s.rsplitn(2, ':'); - let port_str = try_opt!(parts_iter.next(), "invalid socket address"); - let host = try_opt!(parts_iter.next(), "invalid socket address"); + let (host, port_str) = try_opt!(s.rsplit_once(':'), "invalid socket address"); let port: u16 = try_opt!(port_str.parse().ok(), "invalid port value"); - (host, port).try_into() } } @@ -189,14 +198,15 @@ impl<'a> TryFrom<(&'a str, u16)> for LookupHost { fn try_from((host, port): (&'a str, u16)) -> io::Result { init(); - let c_host = CString::new(host)?; - let mut hints: c::addrinfo = unsafe { mem::zeroed() }; - hints.ai_socktype = c::SOCK_STREAM; - let mut res = ptr::null_mut(); - unsafe { - cvt_gai(c::getaddrinfo(c_host.as_ptr(), ptr::null(), &hints, &mut res)) - .map(|_| LookupHost { original: res, cur: res, port }) - } + run_with_cstr(host.as_bytes(), |c_host| { + let mut hints: c::addrinfo = unsafe { mem::zeroed() }; + hints.ai_socktype = c::SOCK_STREAM; + let mut res = ptr::null_mut(); + unsafe { + cvt_gai(c::getaddrinfo(c_host.as_ptr(), ptr::null(), &hints, &mut res)) + .map(|_| LookupHost { original: res, cur: res, port }) + } + }) } } @@ -216,8 +226,8 @@ impl TcpStream { let sock = Socket::new(addr, c::SOCK_STREAM)?; - let (addrp, len) = addr.into_inner(); - cvt_r(|| unsafe { c::connect(*sock.as_inner(), addrp, len) })?; + let (addr, len) = addr.into_inner(); + cvt_r(|| unsafe { c::connect(sock.as_raw(), addr.as_ptr(), len) })?; Ok(TcpStream { inner: sock }) } @@ -265,10 +275,15 @@ impl TcpStream { self.inner.read_vectored(bufs) } + #[inline] + pub fn is_read_vectored(&self) -> bool { + self.inner.is_read_vectored() + } + pub fn write(&self, buf: &[u8]) -> io::Result { - let len = cmp::min(buf.len(), ::max_value() as usize) as wrlen_t; + let len = cmp::min(buf.len(), ::MAX as usize) as wrlen_t; let ret = cvt(unsafe { - c::send(*self.inner.as_inner(), buf.as_ptr() as *const c_void, len, MSG_NOSIGNAL) + c::send(self.inner.as_raw(), buf.as_ptr() as *const c_void, len, MSG_NOSIGNAL) })?; Ok(ret as usize) } @@ -277,12 +292,17 @@ impl TcpStream { self.inner.write_vectored(bufs) } + #[inline] + pub fn is_write_vectored(&self) -> bool { + self.inner.is_write_vectored() + } + pub fn peer_addr(&self) -> io::Result { - sockname(|buf, len| unsafe { c::getpeername(*self.inner.as_inner(), buf, len) }) + sockname(|buf, len| unsafe { c::getpeername(self.inner.as_raw(), buf, len) }) } pub fn socket_addr(&self) -> io::Result { - sockname(|buf, len| unsafe { c::getsockname(*self.inner.as_inner(), buf, len) }) + sockname(|buf, len| unsafe { c::getsockname(self.inner.as_raw(), buf, len) }) } pub fn shutdown(&self, how: Shutdown) -> io::Result<()> { @@ -293,6 +313,14 @@ impl TcpStream { self.inner.duplicate().map(|s| TcpStream { inner: s }) } + pub fn set_linger(&self, linger: Option) -> io::Result<()> { + self.inner.set_linger(linger) + } + + pub fn linger(&self) -> io::Result> { + self.inner.linger() + } + pub fn set_nodelay(&self, nodelay: bool) -> io::Result<()> { self.inner.set_nodelay(nodelay) } @@ -319,6 +347,12 @@ impl TcpStream { } } +impl AsInner for TcpStream { + fn as_inner(&self) -> &Socket { + &self.inner + } +} + impl FromInner for TcpStream { fn from_inner(socket: Socket) -> TcpStream { TcpStream { inner: socket } @@ -338,7 +372,7 @@ impl fmt::Debug for TcpStream { } let name = if cfg!(windows) { "socket" } else { "fd" }; - res.field(name, &self.inner.as_inner()).finish() + res.field(name, &self.inner.as_raw()).finish() } } @@ -358,19 +392,34 @@ impl TcpListener { let sock = Socket::new(addr, c::SOCK_STREAM)?; - // On platforms with Berkeley-derived sockets, this allows - // to quickly rebind a socket, without needing to wait for - // the OS to clean up the previous one. - if !cfg!(windows) { - setsockopt(&sock, c::SOL_SOCKET, c::SO_REUSEADDR, 1 as c_int)?; - } + // On platforms with Berkeley-derived sockets, this allows to quickly + // rebind a socket, without needing to wait for the OS to clean up the + // previous one. + // + // On Windows, this allows rebinding sockets which are actively in use, + // which allows “socket hijacking”, so we explicitly don't set it here. + // https://docs.microsoft.com/en-us/windows/win32/winsock/using-so-reuseaddr-and-so-exclusiveaddruse + #[cfg(not(windows))] + setsockopt(&sock, c::SOL_SOCKET, c::SO_REUSEADDR, 1 as c_int)?; // Bind our new socket - let (addrp, len) = addr.into_inner(); - cvt(unsafe { c::bind(*sock.as_inner(), addrp, len as _) })?; + let (addr, len) = addr.into_inner(); + cvt(unsafe { c::bind(sock.as_raw(), addr.as_ptr(), len as _) })?; + + cfg_if::cfg_if! { + if #[cfg(target_os = "horizon")] { + // The 3DS doesn't support a big connection backlog. Sometimes + // it allows up to about 37, but other times it doesn't even + // accept 32. There may be a global limitation causing this. + let backlog = 20; + } else { + // The default for all other platforms + let backlog = 128; + } + } // Start listening - cvt(unsafe { c::listen(*sock.as_inner(), 128) })?; + cvt(unsafe { c::listen(sock.as_raw(), backlog) })?; Ok(TcpListener { inner: sock }) } @@ -383,7 +432,7 @@ impl TcpListener { } pub fn socket_addr(&self) -> io::Result { - sockname(|buf, len| unsafe { c::getsockname(*self.inner.as_inner(), buf, len) }) + sockname(|buf, len| unsafe { c::getsockname(self.inner.as_raw(), buf, len) }) } pub fn accept(&self) -> io::Result<(TcpStream, SocketAddr)> { @@ -440,7 +489,7 @@ impl fmt::Debug for TcpListener { } let name = if cfg!(windows) { "socket" } else { "fd" }; - res.field(name, &self.inner.as_inner()).finish() + res.field(name, &self.inner.as_raw()).finish() } } @@ -459,8 +508,8 @@ impl UdpSocket { init(); let sock = Socket::new(addr, c::SOCK_DGRAM)?; - let (addrp, len) = addr.into_inner(); - cvt(unsafe { c::bind(*sock.as_inner(), addrp, len as _) })?; + let (addr, len) = addr.into_inner(); + cvt(unsafe { c::bind(sock.as_raw(), addr.as_ptr(), len as _) })?; Ok(UdpSocket { inner: sock }) } @@ -473,11 +522,11 @@ impl UdpSocket { } pub fn peer_addr(&self) -> io::Result { - sockname(|buf, len| unsafe { c::getpeername(*self.inner.as_inner(), buf, len) }) + sockname(|buf, len| unsafe { c::getpeername(self.inner.as_raw(), buf, len) }) } pub fn socket_addr(&self) -> io::Result { - sockname(|buf, len| unsafe { c::getsockname(*self.inner.as_inner(), buf, len) }) + sockname(|buf, len| unsafe { c::getsockname(self.inner.as_raw(), buf, len) }) } pub fn recv_from(&self, buf: &mut [u8]) -> io::Result<(usize, SocketAddr)> { @@ -489,15 +538,15 @@ impl UdpSocket { } pub fn send_to(&self, buf: &[u8], dst: &SocketAddr) -> io::Result { - let len = cmp::min(buf.len(), ::max_value() as usize) as wrlen_t; - let (dstp, dstlen) = dst.into_inner(); + let len = cmp::min(buf.len(), ::MAX as usize) as wrlen_t; + let (dst, dstlen) = dst.into_inner(); let ret = cvt(unsafe { c::sendto( - *self.inner.as_inner(), + self.inner.as_raw(), buf.as_ptr() as *const c_void, len, MSG_NOSIGNAL, - dstp, + dst.as_ptr(), dstlen, ) })?; @@ -572,15 +621,15 @@ impl UdpSocket { pub fn join_multicast_v4(&self, multiaddr: &Ipv4Addr, interface: &Ipv4Addr) -> io::Result<()> { let mreq = c::ip_mreq { - imr_multiaddr: *multiaddr.as_inner(), - imr_interface: *interface.as_inner(), + imr_multiaddr: multiaddr.into_inner(), + imr_interface: interface.into_inner(), }; setsockopt(&self.inner, c::IPPROTO_IP, c::IP_ADD_MEMBERSHIP, mreq) } pub fn join_multicast_v6(&self, multiaddr: &Ipv6Addr, interface: u32) -> io::Result<()> { let mreq = c::ipv6_mreq { - ipv6mr_multiaddr: *multiaddr.as_inner(), + ipv6mr_multiaddr: multiaddr.into_inner(), ipv6mr_interface: to_ipv6mr_interface(interface), }; setsockopt(&self.inner, c::IPPROTO_IPV6, IPV6_ADD_MEMBERSHIP, mreq) @@ -588,15 +637,15 @@ impl UdpSocket { pub fn leave_multicast_v4(&self, multiaddr: &Ipv4Addr, interface: &Ipv4Addr) -> io::Result<()> { let mreq = c::ip_mreq { - imr_multiaddr: *multiaddr.as_inner(), - imr_interface: *interface.as_inner(), + imr_multiaddr: multiaddr.into_inner(), + imr_interface: interface.into_inner(), }; setsockopt(&self.inner, c::IPPROTO_IP, c::IP_DROP_MEMBERSHIP, mreq) } pub fn leave_multicast_v6(&self, multiaddr: &Ipv6Addr, interface: u32) -> io::Result<()> { let mreq = c::ipv6_mreq { - ipv6mr_multiaddr: *multiaddr.as_inner(), + ipv6mr_multiaddr: multiaddr.into_inner(), ipv6mr_interface: to_ipv6mr_interface(interface), }; setsockopt(&self.inner, c::IPPROTO_IPV6, IPV6_DROP_MEMBERSHIP, mreq) @@ -628,16 +677,16 @@ impl UdpSocket { } pub fn send(&self, buf: &[u8]) -> io::Result { - let len = cmp::min(buf.len(), ::max_value() as usize) as wrlen_t; + let len = cmp::min(buf.len(), ::MAX as usize) as wrlen_t; let ret = cvt(unsafe { - c::send(*self.inner.as_inner(), buf.as_ptr() as *const c_void, len, MSG_NOSIGNAL) + c::send(self.inner.as_raw(), buf.as_ptr() as *const c_void, len, MSG_NOSIGNAL) })?; Ok(ret as usize) } pub fn connect(&self, addr: io::Result<&SocketAddr>) -> io::Result<()> { - let (addrp, len) = addr?.into_inner(); - cvt_r(|| unsafe { c::connect(*self.inner.as_inner(), addrp, len) }).map(drop) + let (addr, len) = addr?.into_inner(); + cvt_r(|| unsafe { c::connect(self.inner.as_raw(), addr.as_ptr(), len) }).map(drop) } } @@ -656,29 +705,41 @@ impl fmt::Debug for UdpSocket { } let name = if cfg!(windows) { "socket" } else { "fd" }; - res.field(name, &self.inner.as_inner()).finish() + res.field(name, &self.inner.as_raw()).finish() } } -#[cfg(test)] -mod tests { - use super::*; - use crate::collections::HashMap; - - #[test] - fn no_lookup_host_duplicates() { - let mut addrs = HashMap::new(); - let lh = match LookupHost::try_from(("localhost", 0)) { - Ok(lh) => lh, - Err(e) => panic!("couldn't resolve `localhost': {}", e), - }; - for sa in lh { - *addrs.entry(sa).or_insert(0) += 1; +//////////////////////////////////////////////////////////////////////////////// +// Converting SocketAddr to libc representation +//////////////////////////////////////////////////////////////////////////////// + +/// A type with the same memory layout as `c::sockaddr`. Used in converting Rust level +/// SocketAddr* types into their system representation. The benefit of this specific +/// type over using `c::sockaddr_storage` is that this type is exactly as large as it +/// needs to be and not a lot larger. And it can be initialized more cleanly from Rust. +#[repr(C)] +pub(crate) union SocketAddrCRepr { + v4: c::sockaddr_in, + v6: c::sockaddr_in6, +} + +impl SocketAddrCRepr { + pub fn as_ptr(&self) -> *const c::sockaddr { + self as *const _ as *const c::sockaddr + } +} + +impl<'a> IntoInner<(SocketAddrCRepr, c::socklen_t)> for &'a SocketAddr { + fn into_inner(self) -> (SocketAddrCRepr, c::socklen_t) { + match *self { + SocketAddr::V4(ref a) => { + let sockaddr = SocketAddrCRepr { v4: a.into_inner() }; + (sockaddr, mem::size_of::() as c::socklen_t) + } + SocketAddr::V6(ref a) => { + let sockaddr = SocketAddrCRepr { v6: a.into_inner() }; + (sockaddr, mem::size_of::() as c::socklen_t) + } } - assert_eq!( - addrs.iter().filter(|&(_, &v)| v > 1).collect::>(), - vec![], - "There should be no duplicate localhost entries" - ); } } diff --git a/crux-mir/lib/std/src/sys_common/net/tests.rs b/crux-mir/lib/std/src/sys_common/net/tests.rs new file mode 100644 index 000000000..ac75d9ebf --- /dev/null +++ b/crux-mir/lib/std/src/sys_common/net/tests.rs @@ -0,0 +1,19 @@ +use super::*; +use crate::collections::HashMap; + +#[test] +fn no_lookup_host_duplicates() { + let mut addrs = HashMap::new(); + let lh = match LookupHost::try_from(("localhost", 0)) { + Ok(lh) => lh, + Err(e) => panic!("couldn't resolve `localhost': {e}"), + }; + for sa in lh { + *addrs.entry(sa).or_insert(0) += 1; + } + assert_eq!( + addrs.iter().filter(|&(_, &v)| v > 1).collect::>(), + vec![], + "There should be no duplicate localhost entries" + ); +} diff --git a/crux-mir/lib/std/src/sys_common/once/futex.rs b/crux-mir/lib/std/src/sys_common/once/futex.rs new file mode 100644 index 000000000..5c7e6c013 --- /dev/null +++ b/crux-mir/lib/std/src/sys_common/once/futex.rs @@ -0,0 +1,134 @@ +use crate::cell::Cell; +use crate::sync as public; +use crate::sync::atomic::{ + AtomicU32, + Ordering::{Acquire, Relaxed, Release}, +}; +use crate::sys::futex::{futex_wait, futex_wake_all}; + +// On some platforms, the OS is very nice and handles the waiter queue for us. +// This means we only need one atomic value with 5 states: + +/// No initialization has run yet, and no thread is currently using the Once. +const INCOMPLETE: u32 = 0; +/// Some thread has previously attempted to initialize the Once, but it panicked, +/// so the Once is now poisoned. There are no other threads currently accessing +/// this Once. +const POISONED: u32 = 1; +/// Some thread is currently attempting to run initialization. It may succeed, +/// so all future threads need to wait for it to finish. +const RUNNING: u32 = 2; +/// Some thread is currently attempting to run initialization and there are threads +/// waiting for it to finish. +const QUEUED: u32 = 3; +/// Initialization has completed and all future calls should finish immediately. +const COMPLETE: u32 = 4; + +// Threads wait by setting the state to QUEUED and calling `futex_wait` on the state +// variable. When the running thread finishes, it will wake all waiting threads using +// `futex_wake_all`. + +pub struct OnceState { + poisoned: bool, + set_state_to: Cell, +} + +impl OnceState { + #[inline] + pub fn is_poisoned(&self) -> bool { + self.poisoned + } + + #[inline] + pub fn poison(&self) { + self.set_state_to.set(POISONED); + } +} + +struct CompletionGuard<'a> { + state: &'a AtomicU32, + set_state_on_drop_to: u32, +} + +impl<'a> Drop for CompletionGuard<'a> { + fn drop(&mut self) { + // Use release ordering to propagate changes to all threads checking + // up on the Once. `futex_wake_all` does its own synchronization, hence + // we do not need `AcqRel`. + if self.state.swap(self.set_state_on_drop_to, Release) == QUEUED { + futex_wake_all(&self.state); + } + } +} + +pub struct Once { + state: AtomicU32, +} + +impl Once { + #[inline] + pub const fn new() -> Once { + Once { state: AtomicU32::new(INCOMPLETE) } + } + + #[inline] + pub fn is_completed(&self) -> bool { + // Use acquire ordering to make all initialization changes visible to the + // current thread. + self.state.load(Acquire) == COMPLETE + } + + // This uses FnMut to match the API of the generic implementation. As this + // implementation is quite light-weight, it is generic over the closure and + // so avoids the cost of dynamic dispatch. + #[cold] + #[track_caller] + pub fn call(&self, ignore_poisoning: bool, f: &mut impl FnMut(&public::OnceState)) { + let mut state = self.state.load(Acquire); + loop { + match state { + POISONED if !ignore_poisoning => { + // Panic to propagate the poison. + panic!("Once instance has previously been poisoned"); + } + INCOMPLETE | POISONED => { + // Try to register the current thread as the one running. + if let Err(new) = + self.state.compare_exchange_weak(state, RUNNING, Acquire, Acquire) + { + state = new; + continue; + } + // `waiter_queue` will manage other waiting threads, and + // wake them up on drop. + let mut waiter_queue = + CompletionGuard { state: &self.state, set_state_on_drop_to: POISONED }; + // Run the function, letting it know if we're poisoned or not. + let f_state = public::OnceState { + inner: OnceState { + poisoned: state == POISONED, + set_state_to: Cell::new(COMPLETE), + }, + }; + f(&f_state); + waiter_queue.set_state_on_drop_to = f_state.inner.set_state_to.get(); + return; + } + RUNNING | QUEUED => { + // Set the state to QUEUED if it is not already. + if state == RUNNING + && let Err(new) = self.state.compare_exchange_weak(RUNNING, QUEUED, Relaxed, Acquire) + { + state = new; + continue; + } + + futex_wait(&self.state, QUEUED, None); + state = self.state.load(Acquire); + } + COMPLETE => return, + _ => unreachable!("state is never set to invalid values"), + } + } + } +} diff --git a/crux-mir/lib/std/src/sys_common/once/mod.rs b/crux-mir/lib/std/src/sys_common/once/mod.rs new file mode 100644 index 000000000..359697d83 --- /dev/null +++ b/crux-mir/lib/std/src/sys_common/once/mod.rs @@ -0,0 +1,34 @@ +// A "once" is a relatively simple primitive, and it's also typically provided +// by the OS as well (see `pthread_once` or `InitOnceExecuteOnce`). The OS +// primitives, however, tend to have surprising restrictions, such as the Unix +// one doesn't allow an argument to be passed to the function. +// +// As a result, we end up implementing it ourselves in the standard library. +// This also gives us the opportunity to optimize the implementation a bit which +// should help the fast path on call sites. + +cfg_if::cfg_if! { + if #[cfg(any( + target_os = "linux", + target_os = "android", + all(target_arch = "wasm32", target_feature = "atomics"), + target_os = "freebsd", + target_os = "openbsd", + target_os = "dragonfly", + target_os = "fuchsia", + target_os = "hermit", + ))] { + mod futex; + pub use futex::{Once, OnceState}; + } else if #[cfg(any( + windows, + target_family = "unix", + all(target_vendor = "fortanix", target_env = "sgx"), + target_os = "solid_asp3", + ))] { + mod queue; + pub use queue::{Once, OnceState}; + } else { + pub use crate::sys::once::{Once, OnceState}; + } +} diff --git a/crux-mir/lib/std/src/sys_common/once/queue.rs b/crux-mir/lib/std/src/sys_common/once/queue.rs new file mode 100644 index 000000000..d953a6745 --- /dev/null +++ b/crux-mir/lib/std/src/sys_common/once/queue.rs @@ -0,0 +1,283 @@ +// Each `Once` has one word of atomic state, and this state is CAS'd on to +// determine what to do. There are four possible state of a `Once`: +// +// * Incomplete - no initialization has run yet, and no thread is currently +// using the Once. +// * Poisoned - some thread has previously attempted to initialize the Once, but +// it panicked, so the Once is now poisoned. There are no other +// threads currently accessing this Once. +// * Running - some thread is currently attempting to run initialization. It may +// succeed, so all future threads need to wait for it to finish. +// Note that this state is accompanied with a payload, described +// below. +// * Complete - initialization has completed and all future calls should finish +// immediately. +// +// With 4 states we need 2 bits to encode this, and we use the remaining bits +// in the word we have allocated as a queue of threads waiting for the thread +// responsible for entering the RUNNING state. This queue is just a linked list +// of Waiter nodes which is monotonically increasing in size. Each node is +// allocated on the stack, and whenever the running closure finishes it will +// consume the entire queue and notify all waiters they should try again. +// +// You'll find a few more details in the implementation, but that's the gist of +// it! +// +// Atomic orderings: +// When running `Once` we deal with multiple atomics: +// `Once.state_and_queue` and an unknown number of `Waiter.signaled`. +// * `state_and_queue` is used (1) as a state flag, (2) for synchronizing the +// result of the `Once`, and (3) for synchronizing `Waiter` nodes. +// - At the end of the `call` function we have to make sure the result +// of the `Once` is acquired. So every load which can be the only one to +// load COMPLETED must have at least acquire ordering, which means all +// three of them. +// - `WaiterQueue::drop` is the only place that may store COMPLETED, and +// must do so with release ordering to make the result available. +// - `wait` inserts `Waiter` nodes as a pointer in `state_and_queue`, and +// needs to make the nodes available with release ordering. The load in +// its `compare_exchange` can be relaxed because it only has to compare +// the atomic, not to read other data. +// - `WaiterQueue::drop` must see the `Waiter` nodes, so it must load +// `state_and_queue` with acquire ordering. +// - There is just one store where `state_and_queue` is used only as a +// state flag, without having to synchronize data: switching the state +// from INCOMPLETE to RUNNING in `call`. This store can be Relaxed, +// but the read has to be Acquire because of the requirements mentioned +// above. +// * `Waiter.signaled` is both used as a flag, and to protect a field with +// interior mutability in `Waiter`. `Waiter.thread` is changed in +// `WaiterQueue::drop` which then sets `signaled` with release ordering. +// After `wait` loads `signaled` with acquire ordering and sees it is true, +// it needs to see the changes to drop the `Waiter` struct correctly. +// * There is one place where the two atomics `Once.state_and_queue` and +// `Waiter.signaled` come together, and might be reordered by the compiler or +// processor. Because both use acquire ordering such a reordering is not +// allowed, so no need for `SeqCst`. + +use crate::cell::Cell; +use crate::fmt; +use crate::ptr; +use crate::sync as public; +use crate::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; +use crate::thread::{self, Thread}; + +type Masked = (); + +pub struct Once { + state_and_queue: AtomicPtr, +} + +pub struct OnceState { + poisoned: bool, + set_state_on_drop_to: Cell<*mut Masked>, +} + +// Four states that a Once can be in, encoded into the lower bits of +// `state_and_queue` in the Once structure. +const INCOMPLETE: usize = 0x0; +const POISONED: usize = 0x1; +const RUNNING: usize = 0x2; +const COMPLETE: usize = 0x3; + +// Mask to learn about the state. All other bits are the queue of waiters if +// this is in the RUNNING state. +const STATE_MASK: usize = 0x3; + +// Representation of a node in the linked list of waiters, used while in the +// RUNNING state. +// Note: `Waiter` can't hold a mutable pointer to the next thread, because then +// `wait` would both hand out a mutable reference to its `Waiter` node, and keep +// a shared reference to check `signaled`. Instead we hold shared references and +// use interior mutability. +#[repr(align(4))] // Ensure the two lower bits are free to use as state bits. +struct Waiter { + thread: Cell>, + signaled: AtomicBool, + next: *const Waiter, +} + +// Head of a linked list of waiters. +// Every node is a struct on the stack of a waiting thread. +// Will wake up the waiters when it gets dropped, i.e. also on panic. +struct WaiterQueue<'a> { + state_and_queue: &'a AtomicPtr, + set_state_on_drop_to: *mut Masked, +} + +impl Once { + #[inline] + #[rustc_const_stable(feature = "const_once_new", since = "1.32.0")] + pub const fn new() -> Once { + Once { state_and_queue: AtomicPtr::new(ptr::invalid_mut(INCOMPLETE)) } + } + + #[inline] + pub fn is_completed(&self) -> bool { + // An `Acquire` load is enough because that makes all the initialization + // operations visible to us, and, this being a fast path, weaker + // ordering helps with performance. This `Acquire` synchronizes with + // `Release` operations on the slow path. + self.state_and_queue.load(Ordering::Acquire).addr() == COMPLETE + } + + // This is a non-generic function to reduce the monomorphization cost of + // using `call_once` (this isn't exactly a trivial or small implementation). + // + // Additionally, this is tagged with `#[cold]` as it should indeed be cold + // and it helps let LLVM know that calls to this function should be off the + // fast path. Essentially, this should help generate more straight line code + // in LLVM. + // + // Finally, this takes an `FnMut` instead of a `FnOnce` because there's + // currently no way to take an `FnOnce` and call it via virtual dispatch + // without some allocation overhead. + #[cold] + #[track_caller] + pub fn call(&self, ignore_poisoning: bool, init: &mut dyn FnMut(&public::OnceState)) { + let mut state_and_queue = self.state_and_queue.load(Ordering::Acquire); + loop { + match state_and_queue.addr() { + COMPLETE => break, + POISONED if !ignore_poisoning => { + // Panic to propagate the poison. + panic!("Once instance has previously been poisoned"); + } + POISONED | INCOMPLETE => { + // Try to register this thread as the one RUNNING. + let exchange_result = self.state_and_queue.compare_exchange( + state_and_queue, + ptr::invalid_mut(RUNNING), + Ordering::Acquire, + Ordering::Acquire, + ); + if let Err(old) = exchange_result { + state_and_queue = old; + continue; + } + // `waiter_queue` will manage other waiting threads, and + // wake them up on drop. + let mut waiter_queue = WaiterQueue { + state_and_queue: &self.state_and_queue, + set_state_on_drop_to: ptr::invalid_mut(POISONED), + }; + // Run the initialization function, letting it know if we're + // poisoned or not. + let init_state = public::OnceState { + inner: OnceState { + poisoned: state_and_queue.addr() == POISONED, + set_state_on_drop_to: Cell::new(ptr::invalid_mut(COMPLETE)), + }, + }; + init(&init_state); + waiter_queue.set_state_on_drop_to = init_state.inner.set_state_on_drop_to.get(); + break; + } + _ => { + // All other values must be RUNNING with possibly a + // pointer to the waiter queue in the more significant bits. + assert!(state_and_queue.addr() & STATE_MASK == RUNNING); + wait(&self.state_and_queue, state_and_queue); + state_and_queue = self.state_and_queue.load(Ordering::Acquire); + } + } + } + } +} + +fn wait(state_and_queue: &AtomicPtr, mut current_state: *mut Masked) { + // Note: the following code was carefully written to avoid creating a + // mutable reference to `node` that gets aliased. + loop { + // Don't queue this thread if the status is no longer running, + // otherwise we will not be woken up. + if current_state.addr() & STATE_MASK != RUNNING { + return; + } + + // Create the node for our current thread. + let node = Waiter { + thread: Cell::new(Some(thread::current())), + signaled: AtomicBool::new(false), + next: current_state.with_addr(current_state.addr() & !STATE_MASK) as *const Waiter, + }; + let me = &node as *const Waiter as *const Masked as *mut Masked; + + // Try to slide in the node at the head of the linked list, making sure + // that another thread didn't just replace the head of the linked list. + let exchange_result = state_and_queue.compare_exchange( + current_state, + me.with_addr(me.addr() | RUNNING), + Ordering::Release, + Ordering::Relaxed, + ); + if let Err(old) = exchange_result { + current_state = old; + continue; + } + + // We have enqueued ourselves, now lets wait. + // It is important not to return before being signaled, otherwise we + // would drop our `Waiter` node and leave a hole in the linked list + // (and a dangling reference). Guard against spurious wakeups by + // reparking ourselves until we are signaled. + while !node.signaled.load(Ordering::Acquire) { + // If the managing thread happens to signal and unpark us before we + // can park ourselves, the result could be this thread never gets + // unparked. Luckily `park` comes with the guarantee that if it got + // an `unpark` just before on an unparked thread it does not park. + thread::park(); + } + break; + } +} + +#[stable(feature = "std_debug", since = "1.16.0")] +impl fmt::Debug for Once { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Once").finish_non_exhaustive() + } +} + +impl Drop for WaiterQueue<'_> { + fn drop(&mut self) { + // Swap out our state with however we finished. + let state_and_queue = + self.state_and_queue.swap(self.set_state_on_drop_to, Ordering::AcqRel); + + // We should only ever see an old state which was RUNNING. + assert_eq!(state_and_queue.addr() & STATE_MASK, RUNNING); + + // Walk the entire linked list of waiters and wake them up (in lifo + // order, last to register is first to wake up). + unsafe { + // Right after setting `node.signaled = true` the other thread may + // free `node` if there happens to be has a spurious wakeup. + // So we have to take out the `thread` field and copy the pointer to + // `next` first. + let mut queue = + state_and_queue.with_addr(state_and_queue.addr() & !STATE_MASK) as *const Waiter; + while !queue.is_null() { + let next = (*queue).next; + let thread = (*queue).thread.take().unwrap(); + (*queue).signaled.store(true, Ordering::Release); + // ^- FIXME (maybe): This is another case of issue #55005 + // `store()` has a potentially dangling ref to `signaled`. + queue = next; + thread.unpark(); + } + } + } +} + +impl OnceState { + #[inline] + pub fn is_poisoned(&self) -> bool { + self.poisoned + } + + #[inline] + pub fn poison(&self) { + self.set_state_on_drop_to.set(ptr::invalid_mut(POISONED)); + } +} diff --git a/crux-mir/lib/std/src/sys_common/process.rs b/crux-mir/lib/std/src/sys_common/process.rs index f3a296209..18883048d 100644 --- a/crux-mir/lib/std/src/sys_common/process.rs +++ b/crux-mir/lib/std/src/sys_common/process.rs @@ -4,10 +4,13 @@ use crate::collections::BTreeMap; use crate::env; use crate::ffi::{OsStr, OsString}; -use crate::sys::process::EnvKey; +use crate::fmt; +use crate::io; +use crate::sys::pipe::read2; +use crate::sys::process::{EnvKey, ExitStatus, Process, StdioPipes}; // Stores a set of changes to an environment -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct CommandEnv { clear: bool, saw_path: bool, @@ -20,6 +23,14 @@ impl Default for CommandEnv { } } +impl fmt::Debug for CommandEnv { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut debug_command_env = f.debug_struct("CommandEnv"); + debug_command_env.field("clear", &self.clear).field("vars", &self.vars); + debug_command_env.finish() + } +} + impl CommandEnv { // Capture the current environment with these changes applied pub fn capture(&self) -> BTreeMap { @@ -39,22 +50,6 @@ impl CommandEnv { result } - // Apply these changes directly to the current environment - pub fn apply(&self) { - if self.clear { - for (k, _) in env::vars_os() { - env::remove_var(k); - } - } - for (key, maybe_val) in self.vars.iter() { - if let Some(ref val) = maybe_val { - env::set_var(key, val); - } else { - env::remove_var(key); - } - } - } - pub fn is_unchanged(&self) -> bool { !self.clear && self.vars.is_empty() } @@ -65,16 +60,18 @@ impl CommandEnv { // The following functions build up changes pub fn set(&mut self, key: &OsStr, value: &OsStr) { + let key = EnvKey::from(key); self.maybe_saw_path(&key); - self.vars.insert(key.to_owned().into(), Some(value.to_owned())); + self.vars.insert(key, Some(value.to_owned())); } pub fn remove(&mut self, key: &OsStr) { + let key = EnvKey::from(key); self.maybe_saw_path(&key); if self.clear { - self.vars.remove(key); + self.vars.remove(&key); } else { - self.vars.insert(key.to_owned().into(), None); + self.vars.insert(key, None); } } @@ -87,9 +84,74 @@ impl CommandEnv { self.saw_path || self.clear } - fn maybe_saw_path(&mut self, key: &OsStr) { + fn maybe_saw_path(&mut self, key: &EnvKey) { if !self.saw_path && key == "PATH" { self.saw_path = true; } } + + pub fn iter(&self) -> CommandEnvs<'_> { + let iter = self.vars.iter(); + CommandEnvs { iter } + } +} + +/// An iterator over the command environment variables. +/// +/// This struct is created by +/// [`Command::get_envs`][crate::process::Command::get_envs]. See its +/// documentation for more. +#[must_use = "iterators are lazy and do nothing unless consumed"] +#[stable(feature = "command_access", since = "1.57.0")] +#[derive(Debug)] +pub struct CommandEnvs<'a> { + iter: crate::collections::btree_map::Iter<'a, EnvKey, Option>, +} + +#[stable(feature = "command_access", since = "1.57.0")] +impl<'a> Iterator for CommandEnvs<'a> { + type Item = (&'a OsStr, Option<&'a OsStr>); + fn next(&mut self) -> Option { + self.iter.next().map(|(key, value)| (key.as_ref(), value.as_deref())) + } + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } +} + +#[stable(feature = "command_access", since = "1.57.0")] +impl<'a> ExactSizeIterator for CommandEnvs<'a> { + fn len(&self) -> usize { + self.iter.len() + } + fn is_empty(&self) -> bool { + self.iter.is_empty() + } +} + +pub fn wait_with_output( + mut process: Process, + mut pipes: StdioPipes, +) -> io::Result<(ExitStatus, Vec, Vec)> { + drop(pipes.stdin.take()); + + let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); + match (pipes.stdout.take(), pipes.stderr.take()) { + (None, None) => {} + (Some(out), None) => { + let res = out.read_to_end(&mut stdout); + res.unwrap(); + } + (None, Some(err)) => { + let res = err.read_to_end(&mut stderr); + res.unwrap(); + } + (Some(out), Some(err)) => { + let res = read2(out, &mut stdout, err, &mut stderr); + res.unwrap(); + } + } + + let status = process.wait()?; + Ok((status, stdout, stderr)) } diff --git a/crux-mir/lib/std/src/sys_common/remutex.rs b/crux-mir/lib/std/src/sys_common/remutex.rs deleted file mode 100644 index 61cfce430..000000000 --- a/crux-mir/lib/std/src/sys_common/remutex.rs +++ /dev/null @@ -1,224 +0,0 @@ -use crate::fmt; -use crate::marker; -use crate::ops::Deref; -use crate::panic::{RefUnwindSafe, UnwindSafe}; -use crate::sys::crux::mutex as sys; - -/// A re-entrant mutual exclusion -/// -/// This mutex will block *other* threads waiting for the lock to become -/// available. The thread which has already locked the mutex can lock it -/// multiple times without blocking, preventing a common source of deadlocks. -pub struct ReentrantMutex { - inner: sys::ReentrantMutex, - data: T, -} - -unsafe impl Send for ReentrantMutex {} -unsafe impl Sync for ReentrantMutex {} - -impl UnwindSafe for ReentrantMutex {} -impl RefUnwindSafe for ReentrantMutex {} - -/// An RAII implementation of a "scoped lock" of a mutex. When this structure is -/// dropped (falls out of scope), the lock will be unlocked. -/// -/// The data protected by the mutex can be accessed through this guard via its -/// Deref implementation. -/// -/// # Mutability -/// -/// Unlike `MutexGuard`, `ReentrantMutexGuard` does not implement `DerefMut`, -/// because implementation of the trait would violate Rust’s reference aliasing -/// rules. Use interior mutability (usually `RefCell`) in order to mutate the -/// guarded data. -#[must_use = "if unused the ReentrantMutex will immediately unlock"] -pub struct ReentrantMutexGuard<'a, T: 'a> { - // funny underscores due to how Deref currently works (it disregards field - // privacy). - __lock: &'a ReentrantMutex, -} - -impl !marker::Send for ReentrantMutexGuard<'_, T> {} - -impl ReentrantMutex { - /// Creates a new reentrant mutex in an unlocked state. - /// - /// # Unsafety - /// - /// This function is unsafe because it is required that `init` is called - /// once this mutex is in its final resting place, and only then are the - /// lock/unlock methods safe. - pub const unsafe fn new(t: T) -> ReentrantMutex { - ReentrantMutex { inner: sys::ReentrantMutex::uninitialized(), data: t } - } - - /// Initializes this mutex so it's ready for use. - /// - /// # Unsafety - /// - /// Unsafe to call more than once, and must be called after this will no - /// longer move in memory. - pub unsafe fn init(&self) { - self.inner.init(); - } - - /// Acquires a mutex, blocking the current thread until it is able to do so. - /// - /// This function will block the caller until it is available to acquire the mutex. - /// Upon returning, the thread is the only thread with the mutex held. When the thread - /// calling this method already holds the lock, the call shall succeed without - /// blocking. - /// - /// # Errors - /// - /// If another user of this mutex panicked while holding the mutex, then - /// this call will return failure if the mutex would otherwise be - /// acquired. - pub fn lock(&self) -> ReentrantMutexGuard<'_, T> { - unsafe { self.inner.lock() } - ReentrantMutexGuard::new(&self) - } - - /// Attempts to acquire this lock. - /// - /// If the lock could not be acquired at this time, then `Err` is returned. - /// Otherwise, an RAII guard is returned. - /// - /// This function does not block. - /// - /// # Errors - /// - /// If another user of this mutex panicked while holding the mutex, then - /// this call will return failure if the mutex would otherwise be - /// acquired. - pub fn try_lock(&self) -> Option> { - if unsafe { self.inner.try_lock() } { Some(ReentrantMutexGuard::new(&self)) } else { None } - } -} - -impl Drop for ReentrantMutex { - fn drop(&mut self) { - // This is actually safe b/c we know that there is no further usage of - // this mutex (it's up to the user to arrange for a mutex to get - // dropped, that's not our job) - unsafe { self.inner.destroy() } - } -} - -impl fmt::Debug for ReentrantMutex { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.try_lock() { - Some(guard) => f.debug_struct("ReentrantMutex").field("data", &*guard).finish(), - None => { - struct LockedPlaceholder; - impl fmt::Debug for LockedPlaceholder { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("") - } - } - - f.debug_struct("ReentrantMutex").field("data", &LockedPlaceholder).finish() - } - } - } -} - -impl<'mutex, T> ReentrantMutexGuard<'mutex, T> { - fn new(lock: &'mutex ReentrantMutex) -> ReentrantMutexGuard<'mutex, T> { - ReentrantMutexGuard { __lock: lock } - } -} - -impl Deref for ReentrantMutexGuard<'_, T> { - type Target = T; - - fn deref(&self) -> &T { - &self.__lock.data - } -} - -impl Drop for ReentrantMutexGuard<'_, T> { - #[inline] - fn drop(&mut self) { - unsafe { - self.__lock.inner.unlock(); - } - } -} - -#[cfg(all(test, not(target_os = "emscripten")))] -mod tests { - use crate::cell::RefCell; - use crate::sync::Arc; - use crate::sys_common::remutex::{ReentrantMutex, ReentrantMutexGuard}; - use crate::thread; - - #[test] - fn smoke() { - let m = unsafe { - let m = ReentrantMutex::new(()); - m.init(); - m - }; - { - let a = m.lock(); - { - let b = m.lock(); - { - let c = m.lock(); - assert_eq!(*c, ()); - } - assert_eq!(*b, ()); - } - assert_eq!(*a, ()); - } - } - - #[test] - fn is_mutex() { - let m = unsafe { - let m = Arc::new(ReentrantMutex::new(RefCell::new(0))); - m.init(); - m - }; - let m2 = m.clone(); - let lock = m.lock(); - let child = thread::spawn(move || { - let lock = m2.lock(); - assert_eq!(*lock.borrow(), 4950); - }); - for i in 0..100 { - let lock = m.lock(); - *lock.borrow_mut() += i; - } - drop(lock); - child.join().unwrap(); - } - - #[test] - fn trylock_works() { - let m = unsafe { - let m = Arc::new(ReentrantMutex::new(())); - m.init(); - m - }; - let m2 = m.clone(); - let _lock = m.try_lock(); - let _lock2 = m.try_lock(); - thread::spawn(move || { - let lock = m2.try_lock(); - assert!(lock.is_none()); - }) - .join() - .unwrap(); - let _lock3 = m.try_lock(); - } - - pub struct Answer<'a>(pub ReentrantMutexGuard<'a, RefCell>); - impl Drop for Answer<'_> { - fn drop(&mut self) { - *self.0.borrow_mut() = 42; - } - } -} diff --git a/crux-mir/lib/std/src/sys_common/rwlock.rs b/crux-mir/lib/std/src/sys_common/rwlock.rs deleted file mode 100644 index 87ee7b13c..000000000 --- a/crux-mir/lib/std/src/sys_common/rwlock.rs +++ /dev/null @@ -1,88 +0,0 @@ -use crate::sys::crux::rwlock as imp; - -/// An OS-based reader-writer lock. -/// -/// This structure is entirely unsafe and serves as the lowest layer of a -/// cross-platform binding of system rwlocks. It is recommended to use the -/// safer types at the top level of this crate instead of this type. -pub struct RWLock(imp::RWLock); - -impl RWLock { - /// Creates a new reader-writer lock for use. - /// - /// Behavior is undefined if the reader-writer lock is moved after it is - /// first used with any of the functions below. - pub const fn new() -> RWLock { - RWLock(imp::RWLock::new()) - } - - /// Acquires shared access to the underlying lock, blocking the current - /// thread to do so. - /// - /// Behavior is undefined if the rwlock has been moved between this and any - /// previous method call. - #[inline] - pub unsafe fn read(&self) { - self.0.read() - } - - /// Attempts to acquire shared access to this lock, returning whether it - /// succeeded or not. - /// - /// This function does not block the current thread. - /// - /// Behavior is undefined if the rwlock has been moved between this and any - /// previous method call. - #[inline] - pub unsafe fn try_read(&self) -> bool { - self.0.try_read() - } - - /// Acquires write access to the underlying lock, blocking the current thread - /// to do so. - /// - /// Behavior is undefined if the rwlock has been moved between this and any - /// previous method call. - #[inline] - pub unsafe fn write(&self) { - self.0.write() - } - - /// Attempts to acquire exclusive access to this lock, returning whether it - /// succeeded or not. - /// - /// This function does not block the current thread. - /// - /// Behavior is undefined if the rwlock has been moved between this and any - /// previous method call. - #[inline] - pub unsafe fn try_write(&self) -> bool { - self.0.try_write() - } - - /// Unlocks previously acquired shared access to this lock. - /// - /// Behavior is undefined if the current thread does not have shared access. - #[inline] - pub unsafe fn read_unlock(&self) { - self.0.read_unlock() - } - - /// Unlocks previously acquired exclusive access to this lock. - /// - /// Behavior is undefined if the current thread does not currently have - /// exclusive access. - #[inline] - pub unsafe fn write_unlock(&self) { - self.0.write_unlock() - } - - /// Destroys OS-related resources with this RWLock. - /// - /// Behavior is undefined if there are any currently active users of this - /// lock. - #[inline] - pub unsafe fn destroy(&self) { - self.0.destroy() - } -} diff --git a/crux-mir/lib/std/src/sys_common/tests.rs b/crux-mir/lib/std/src/sys_common/tests.rs new file mode 100644 index 000000000..1b6446db5 --- /dev/null +++ b/crux-mir/lib/std/src/sys_common/tests.rs @@ -0,0 +1,6 @@ +use super::mul_div_u64; + +#[test] +fn test_muldiv() { + assert_eq!(mul_div_u64(1_000_000_000_001, 1_000_000_000, 1_000_000), 1_000_000_000_001_000); +} diff --git a/crux-mir/lib/std/src/sys_common/thread.rs b/crux-mir/lib/std/src/sys_common/thread.rs index 6ab0d5cbe..76466b2b3 100644 --- a/crux-mir/lib/std/src/sys_common/thread.rs +++ b/crux-mir/lib/std/src/sys_common/thread.rs @@ -1,21 +1,10 @@ use crate::env; use crate::sync::atomic::{self, Ordering}; -use crate::sys::stack_overflow; use crate::sys::thread as imp; -#[allow(dead_code)] -pub unsafe fn start_thread(main: *mut u8) { - // Next, set up our stack overflow handler which may get triggered if we run - // out of stack. - let _handler = stack_overflow::Handler::new(); - - // Finally, let's run some code. - Box::from_raw(main as *mut Box)() -} - pub fn min_stack() -> usize { static MIN: atomic::AtomicUsize = atomic::AtomicUsize::new(0); - match MIN.load(Ordering::SeqCst) { + match MIN.load(Ordering::Relaxed) { 0 => {} n => return n - 1, } @@ -24,6 +13,6 @@ pub fn min_stack() -> usize { // 0 is our sentinel value, so ensure that we'll never see 0 after // initialization has run - MIN.store(amt + 1, Ordering::SeqCst); + MIN.store(amt + 1, Ordering::Relaxed); amt } diff --git a/crux-mir/lib/std/src/sys_common/thread_info.rs b/crux-mir/lib/std/src/sys_common/thread_info.rs index f09d16c33..38c9e5000 100644 --- a/crux-mir/lib/std/src/sys_common/thread_info.rs +++ b/crux-mir/lib/std/src/sys_common/thread_info.rs @@ -1,4 +1,5 @@ #![allow(dead_code)] // stack_guard isn't used right now on all platforms +#![allow(unused_unsafe)] // thread_local with `const {}` triggers this liny use crate::cell::RefCell; use crate::sys::thread::guard::Guard; @@ -9,7 +10,7 @@ struct ThreadInfo { thread: Thread, } -thread_local! { static THREAD_INFO: RefCell> = RefCell::new(None) } +thread_local! { static THREAD_INFO: RefCell> = const { RefCell::new(None) } } impl ThreadInfo { fn with(f: F) -> Option @@ -17,12 +18,13 @@ impl ThreadInfo { F: FnOnce(&mut ThreadInfo) -> R, { THREAD_INFO - .try_with(move |c| { - if c.borrow().is_none() { - *c.borrow_mut() = - Some(ThreadInfo { stack_guard: None, thread: Thread::new(None) }) - } - f(c.borrow_mut().as_mut().unwrap()) + .try_with(move |thread_info| { + let mut thread_info = thread_info.borrow_mut(); + let thread_info = thread_info.get_or_insert_with(|| ThreadInfo { + stack_guard: None, + thread: Thread::new(None), + }); + f(thread_info) }) .ok() } @@ -37,10 +39,9 @@ pub fn stack_guard() -> Option { } pub fn set(stack_guard: Option, thread: Thread) { - THREAD_INFO.with(|c| assert!(c.borrow().is_none())); - THREAD_INFO.with(move |c| *c.borrow_mut() = Some(ThreadInfo { stack_guard, thread })); -} - -pub fn reset_guard(stack_guard: Option) { - THREAD_INFO.with(move |c| c.borrow_mut().as_mut().unwrap().stack_guard = stack_guard); + THREAD_INFO.with(move |thread_info| { + let mut thread_info = thread_info.borrow_mut(); + rtassert!(thread_info.is_none()); + *thread_info = Some(ThreadInfo { stack_guard, thread }); + }); } diff --git a/crux-mir/lib/std/src/sys_common/thread_local_dtor.rs b/crux-mir/lib/std/src/sys_common/thread_local_dtor.rs new file mode 100644 index 000000000..1d13a7171 --- /dev/null +++ b/crux-mir/lib/std/src/sys_common/thread_local_dtor.rs @@ -0,0 +1,49 @@ +//! Thread-local destructor +//! +//! Besides thread-local "keys" (pointer-sized non-addressable thread-local store +//! with an associated destructor), many platforms also provide thread-local +//! destructors that are not associated with any particular data. These are +//! often more efficient. +//! +//! This module provides a fallback implementation for that interface, based +//! on the less efficient thread-local "keys". Each platform provides +//! a `thread_local_dtor` module which will either re-export the fallback, +//! or implement something more efficient. + +#![unstable(feature = "thread_local_internals", issue = "none")] +#![allow(dead_code)] + +use crate::ptr; +use crate::sys_common::thread_local_key::StaticKey; + +pub unsafe fn register_dtor_fallback(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { + // The fallback implementation uses a vanilla OS-based TLS key to track + // the list of destructors that need to be run for this thread. The key + // then has its own destructor which runs all the other destructors. + // + // The destructor for DTORS is a little special in that it has a `while` + // loop to continuously drain the list of registered destructors. It + // *should* be the case that this loop always terminates because we + // provide the guarantee that a TLS key cannot be set after it is + // flagged for destruction. + + static DTORS: StaticKey = StaticKey::new(Some(run_dtors)); + type List = Vec<(*mut u8, unsafe extern "C" fn(*mut u8))>; + if DTORS.get().is_null() { + let v: Box = box Vec::new(); + DTORS.set(Box::into_raw(v) as *mut u8); + } + let list: &mut List = &mut *(DTORS.get() as *mut List); + list.push((t, dtor)); + + unsafe extern "C" fn run_dtors(mut ptr: *mut u8) { + while !ptr.is_null() { + let list: Box = Box::from_raw(ptr as *mut List); + for (ptr, dtor) in list.into_iter() { + dtor(ptr); + } + ptr = DTORS.get(); + DTORS.set(ptr::null_mut()); + } + } +} diff --git a/crux-mir/lib/std/src/sys_common/thread_local.rs b/crux-mir/lib/std/src/sys_common/thread_local_key.rs similarity index 60% rename from crux-mir/lib/std/src/sys_common/thread_local.rs rename to crux-mir/lib/std/src/sys_common/thread_local_key.rs index 756b8d044..2672a2a75 100644 --- a/crux-mir/lib/std/src/sys_common/thread_local.rs +++ b/crux-mir/lib/std/src/sys_common/thread_local_key.rs @@ -4,7 +4,7 @@ //! using the native OS-provided facilities (think `TlsAlloc` or //! `pthread_setspecific`). The interface of this differs from the other types //! of thread-local-storage provided in this crate in that OS-based TLS can only -//! get/set pointers, +//! get/set pointer-sized data, possibly with an associated destructor. //! //! This module also provides two flavors of TLS. One is intended for static //! initialization, and does not contain a `Drop` implementation to deallocate @@ -14,7 +14,7 @@ //! # Usage //! //! This module should likely not be used directly unless other primitives are -//! being built on. types such as `thread_local::spawn::Key` are likely much +//! being built on. Types such as `thread_local::spawn::Key` are likely much //! more useful in practice than this OS-based version which likely requires //! unsafe code to interoperate with. //! @@ -46,12 +46,13 @@ #![allow(non_camel_case_types)] #![unstable(feature = "thread_local_internals", issue = "none")] -#![allow(dead_code)] // sys isn't exported yet +#![allow(dead_code)] + +#[cfg(test)] +mod tests; -use crate::ptr; use crate::sync::atomic::{self, AtomicUsize, Ordering}; -use crate::sys::thread_local as imp; -use crate::sys_common::mutex::Mutex; +use crate::sys::thread_local_key as imp; /// A type for TLS keys that are statically allocated. /// @@ -67,8 +68,10 @@ use crate::sys_common::mutex::Mutex; /// ```ignore (cannot-doctest-private-modules) /// use tls::os::{StaticKey, INIT}; /// +/// // Use a regular global static to store the key. /// static KEY: StaticKey = INIT; /// +/// // The state provided via `get` and `set` is thread-local. /// unsafe { /// assert!(KEY.get().is_null()); /// KEY.set(1 as *mut u8); @@ -114,9 +117,14 @@ pub struct Key { /// This value specifies no destructor by default. pub const INIT: StaticKey = StaticKey::new(None); +// Define a sentinel value that is unlikely to be returned +// as a TLS key (but it may be returned). +const KEY_SENTVAL: usize = 0; + impl StaticKey { + #[rustc_const_unstable(feature = "thread_local_internals", issue = "none")] pub const fn new(dtor: Option) -> StaticKey { - StaticKey { key: atomic::AtomicUsize::new(0), dtor } + StaticKey { key: atomic::AtomicUsize::new(KEY_SENTVAL), dtor } } /// Gets the value associated with this TLS key @@ -140,54 +148,40 @@ impl StaticKey { #[inline] unsafe fn key(&self) -> imp::Key { match self.key.load(Ordering::Relaxed) { - 0 => self.lazy_init() as imp::Key, + KEY_SENTVAL => self.lazy_init() as imp::Key, n => n as imp::Key, } } unsafe fn lazy_init(&self) -> usize { - // Currently the Windows implementation of TLS is pretty hairy, and - // it greatly simplifies creation if we just synchronize everything. - // - // Additionally a 0-index of a tls key hasn't been seen on windows, so - // we just simplify the whole branch. - if imp::requires_synchronized_create() { - // We never call `INIT_LOCK.init()`, so it is UB to attempt to - // acquire this mutex reentrantly! - static INIT_LOCK: Mutex = Mutex::new(); - let _guard = INIT_LOCK.lock(); - let mut key = self.key.load(Ordering::SeqCst); - if key == 0 { - key = imp::create(self.dtor) as usize; - self.key.store(key, Ordering::SeqCst); - } - rtassert!(key != 0); - return key; - } - - // POSIX allows the key created here to be 0, but the compare_and_swap - // below relies on using 0 as a sentinel value to check who won the + // POSIX allows the key created here to be KEY_SENTVAL, but the compare_exchange + // below relies on using KEY_SENTVAL as a sentinel value to check who won the // race to set the shared TLS key. As far as I know, there is no // guaranteed value that cannot be returned as a posix_key_create key, // so there is no value we can initialize the inner key with to // prove that it has not yet been set. As such, we'll continue using a - // value of 0, but with some gyrations to make sure we have a non-0 + // value of KEY_SENTVAL, but with some gyrations to make sure we have a non-KEY_SENTVAL // value returned from the creation routine. // FIXME: this is clearly a hack, and should be cleaned up. let key1 = imp::create(self.dtor); - let key = if key1 != 0 { + let key = if key1 as usize != KEY_SENTVAL { key1 } else { let key2 = imp::create(self.dtor); imp::destroy(key1); key2 }; - rtassert!(key != 0); - match self.key.compare_and_swap(0, key as usize, Ordering::SeqCst) { + rtassert!(key as usize != KEY_SENTVAL); + match self.key.compare_exchange( + KEY_SENTVAL, + key as usize, + Ordering::SeqCst, + Ordering::SeqCst, + ) { // The CAS succeeded, so we've created the actual key - 0 => key as usize, + Ok(_) => key as usize, // If someone beat us to the punch, use their key instead - n => { + Err(n) => { imp::destroy(key); n } @@ -227,78 +221,6 @@ impl Key { impl Drop for Key { fn drop(&mut self) { - // Right now Windows doesn't support TLS key destruction, but this also - // isn't used anywhere other than tests, so just leak the TLS key. - // unsafe { imp::destroy(self.key) } - } -} - -pub unsafe fn register_dtor_fallback(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { - // The fallback implementation uses a vanilla OS-based TLS key to track - // the list of destructors that need to be run for this thread. The key - // then has its own destructor which runs all the other destructors. - // - // The destructor for DTORS is a little special in that it has a `while` - // loop to continuously drain the list of registered destructors. It - // *should* be the case that this loop always terminates because we - // provide the guarantee that a TLS key cannot be set after it is - // flagged for destruction. - - static DTORS: StaticKey = StaticKey::new(Some(run_dtors)); - type List = Vec<(*mut u8, unsafe extern "C" fn(*mut u8))>; - if DTORS.get().is_null() { - let v: Box = box Vec::new(); - DTORS.set(Box::into_raw(v) as *mut u8); - } - let list: &mut List = &mut *(DTORS.get() as *mut List); - list.push((t, dtor)); - - unsafe extern "C" fn run_dtors(mut ptr: *mut u8) { - while !ptr.is_null() { - let list: Box = Box::from_raw(ptr as *mut List); - for (ptr, dtor) in list.into_iter() { - dtor(ptr); - } - ptr = DTORS.get(); - DTORS.set(ptr::null_mut()); - } - } -} - -#[cfg(test)] -mod tests { - use super::{Key, StaticKey}; - - fn assert_sync() {} - fn assert_send() {} - - #[test] - fn smoke() { - assert_sync::(); - assert_send::(); - - let k1 = Key::new(None); - let k2 = Key::new(None); - assert!(k1.get().is_null()); - assert!(k2.get().is_null()); - k1.set(1 as *mut _); - k2.set(2 as *mut _); - assert_eq!(k1.get() as usize, 1); - assert_eq!(k2.get() as usize, 2); - } - - #[test] - fn statik() { - static K1: StaticKey = StaticKey::new(None); - static K2: StaticKey = StaticKey::new(None); - - unsafe { - assert!(K1.get().is_null()); - assert!(K2.get().is_null()); - K1.set(1 as *mut _); - K2.set(2 as *mut _); - assert_eq!(K1.get() as usize, 1); - assert_eq!(K2.get() as usize, 2); - } + unsafe { imp::destroy(self.key) } } } diff --git a/crux-mir/lib/std/src/sys_common/thread_local_key/tests.rs b/crux-mir/lib/std/src/sys_common/thread_local_key/tests.rs new file mode 100644 index 000000000..6f32b858f --- /dev/null +++ b/crux-mir/lib/std/src/sys_common/thread_local_key/tests.rs @@ -0,0 +1,35 @@ +use super::{Key, StaticKey}; +use core::ptr; + +fn assert_sync() {} +fn assert_send() {} + +#[test] +fn smoke() { + assert_sync::(); + assert_send::(); + + let k1 = Key::new(None); + let k2 = Key::new(None); + assert!(k1.get().is_null()); + assert!(k2.get().is_null()); + k1.set(ptr::invalid_mut(1)); + k2.set(ptr::invalid_mut(2)); + assert_eq!(k1.get() as usize, 1); + assert_eq!(k2.get() as usize, 2); +} + +#[test] +fn statik() { + static K1: StaticKey = StaticKey::new(None); + static K2: StaticKey = StaticKey::new(None); + + unsafe { + assert!(K1.get().is_null()); + assert!(K2.get().is_null()); + K1.set(ptr::invalid_mut(1)); + K2.set(ptr::invalid_mut(2)); + assert_eq!(K1.get() as usize, 1); + assert_eq!(K2.get() as usize, 2); + } +} diff --git a/crux-mir/lib/std/src/sys_common/thread_parking/futex.rs b/crux-mir/lib/std/src/sys_common/thread_parking/futex.rs new file mode 100644 index 000000000..588e7b278 --- /dev/null +++ b/crux-mir/lib/std/src/sys_common/thread_parking/futex.rs @@ -0,0 +1,97 @@ +use crate::pin::Pin; +use crate::sync::atomic::AtomicU32; +use crate::sync::atomic::Ordering::{Acquire, Release}; +use crate::sys::futex::{futex_wait, futex_wake}; +use crate::time::Duration; + +const PARKED: u32 = u32::MAX; +const EMPTY: u32 = 0; +const NOTIFIED: u32 = 1; + +pub struct Parker { + state: AtomicU32, +} + +// Notes about memory ordering: +// +// Memory ordering is only relevant for the relative ordering of operations +// between different variables. Even Ordering::Relaxed guarantees a +// monotonic/consistent order when looking at just a single atomic variable. +// +// So, since this parker is just a single atomic variable, we only need to look +// at the ordering guarantees we need to provide to the 'outside world'. +// +// The only memory ordering guarantee that parking and unparking provide, is +// that things which happened before unpark() are visible on the thread +// returning from park() afterwards. Otherwise, it was effectively unparked +// before unpark() was called while still consuming the 'token'. +// +// In other words, unpark() needs to synchronize with the part of park() that +// consumes the token and returns. +// +// This is done with a release-acquire synchronization, by using +// Ordering::Release when writing NOTIFIED (the 'token') in unpark(), and using +// Ordering::Acquire when checking for this state in park(). +impl Parker { + /// Construct the futex parker. The UNIX parker implementation + /// requires this to happen in-place. + pub unsafe fn new_in_place(parker: *mut Parker) { + parker.write(Self { state: AtomicU32::new(EMPTY) }); + } + + // Assumes this is only called by the thread that owns the Parker, + // which means that `self.state != PARKED`. + pub unsafe fn park(self: Pin<&Self>) { + // Change NOTIFIED=>EMPTY or EMPTY=>PARKED, and directly return in the + // first case. + if self.state.fetch_sub(1, Acquire) == NOTIFIED { + return; + } + loop { + // Wait for something to happen, assuming it's still set to PARKED. + futex_wait(&self.state, PARKED, None); + // Change NOTIFIED=>EMPTY and return in that case. + if self.state.compare_exchange(NOTIFIED, EMPTY, Acquire, Acquire).is_ok() { + return; + } else { + // Spurious wake up. We loop to try again. + } + } + } + + // Assumes this is only called by the thread that owns the Parker, + // which means that `self.state != PARKED`. This implementation doesn't + // require `Pin`, but other implementations do. + pub unsafe fn park_timeout(self: Pin<&Self>, timeout: Duration) { + // Change NOTIFIED=>EMPTY or EMPTY=>PARKED, and directly return in the + // first case. + if self.state.fetch_sub(1, Acquire) == NOTIFIED { + return; + } + // Wait for something to happen, assuming it's still set to PARKED. + futex_wait(&self.state, PARKED, Some(timeout)); + // This is not just a store, because we need to establish a + // release-acquire ordering with unpark(). + if self.state.swap(EMPTY, Acquire) == NOTIFIED { + // Woke up because of unpark(). + } else { + // Timeout or spurious wake up. + // We return either way, because we can't easily tell if it was the + // timeout or not. + } + } + + // This implementation doesn't require `Pin`, but other implementations do. + #[inline] + pub fn unpark(self: Pin<&Self>) { + // Change PARKED=>NOTIFIED, EMPTY=>NOTIFIED, or NOTIFIED=>NOTIFIED, and + // wake the thread in the first case. + // + // Note that even NOTIFIED=>NOTIFIED results in a write. This is on + // purpose, to make sure every unpark() has a release-acquire ordering + // with park(). + if self.state.swap(NOTIFIED, Release) == PARKED { + futex_wake(&self.state); + } + } +} diff --git a/crux-mir/lib/std/src/sys_common/thread_parking/generic.rs b/crux-mir/lib/std/src/sys_common/thread_parking/generic.rs new file mode 100644 index 000000000..3209bffe3 --- /dev/null +++ b/crux-mir/lib/std/src/sys_common/thread_parking/generic.rs @@ -0,0 +1,125 @@ +//! Parker implementation based on a Mutex and Condvar. + +use crate::pin::Pin; +use crate::sync::atomic::AtomicUsize; +use crate::sync::atomic::Ordering::SeqCst; +use crate::sync::{Condvar, Mutex}; +use crate::time::Duration; + +const EMPTY: usize = 0; +const PARKED: usize = 1; +const NOTIFIED: usize = 2; + +pub struct Parker { + state: AtomicUsize, + lock: Mutex<()>, + cvar: Condvar, +} + +impl Parker { + /// Construct the generic parker. The UNIX parker implementation + /// requires this to happen in-place. + pub unsafe fn new_in_place(parker: *mut Parker) { + parker.write(Parker { + state: AtomicUsize::new(EMPTY), + lock: Mutex::new(()), + cvar: Condvar::new(), + }); + } + + // This implementation doesn't require `unsafe` and `Pin`, but other implementations do. + pub unsafe fn park(self: Pin<&Self>) { + // If we were previously notified then we consume this notification and + // return quickly. + if self.state.compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst).is_ok() { + return; + } + + // Otherwise we need to coordinate going to sleep + let mut m = self.lock.lock().unwrap(); + match self.state.compare_exchange(EMPTY, PARKED, SeqCst, SeqCst) { + Ok(_) => {} + Err(NOTIFIED) => { + // We must read here, even though we know it will be `NOTIFIED`. + // This is because `unpark` may have been called again since we read + // `NOTIFIED` in the `compare_exchange` above. We must perform an + // acquire operation that synchronizes with that `unpark` to observe + // any writes it made before the call to unpark. To do that we must + // read from the write it made to `state`. + let old = self.state.swap(EMPTY, SeqCst); + assert_eq!(old, NOTIFIED, "park state changed unexpectedly"); + return; + } // should consume this notification, so prohibit spurious wakeups in next park. + Err(_) => panic!("inconsistent park state"), + } + loop { + m = self.cvar.wait(m).unwrap(); + match self.state.compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) { + Ok(_) => return, // got a notification + Err(_) => {} // spurious wakeup, go back to sleep + } + } + } + + // This implementation doesn't require `unsafe` and `Pin`, but other implementations do. + pub unsafe fn park_timeout(self: Pin<&Self>, dur: Duration) { + // Like `park` above we have a fast path for an already-notified thread, and + // afterwards we start coordinating for a sleep. + // return quickly. + if self.state.compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst).is_ok() { + return; + } + let m = self.lock.lock().unwrap(); + match self.state.compare_exchange(EMPTY, PARKED, SeqCst, SeqCst) { + Ok(_) => {} + Err(NOTIFIED) => { + // We must read again here, see `park`. + let old = self.state.swap(EMPTY, SeqCst); + assert_eq!(old, NOTIFIED, "park state changed unexpectedly"); + return; + } // should consume this notification, so prohibit spurious wakeups in next park. + Err(_) => panic!("inconsistent park_timeout state"), + } + + // Wait with a timeout, and if we spuriously wake up or otherwise wake up + // from a notification we just want to unconditionally set the state back to + // empty, either consuming a notification or un-flagging ourselves as + // parked. + let (_m, _result) = self.cvar.wait_timeout(m, dur).unwrap(); + match self.state.swap(EMPTY, SeqCst) { + NOTIFIED => {} // got a notification, hurray! + PARKED => {} // no notification, alas + n => panic!("inconsistent park_timeout state: {n}"), + } + } + + // This implementation doesn't require `Pin`, but other implementations do. + pub fn unpark(self: Pin<&Self>) { + // To ensure the unparked thread will observe any writes we made + // before this call, we must perform a release operation that `park` + // can synchronize with. To do that we must write `NOTIFIED` even if + // `state` is already `NOTIFIED`. That is why this must be a swap + // rather than a compare-and-swap that returns if it reads `NOTIFIED` + // on failure. + match self.state.swap(NOTIFIED, SeqCst) { + EMPTY => return, // no one was waiting + NOTIFIED => return, // already unparked + PARKED => {} // gotta go wake someone up + _ => panic!("inconsistent state in unpark"), + } + + // There is a period between when the parked thread sets `state` to + // `PARKED` (or last checked `state` in the case of a spurious wake + // up) and when it actually waits on `cvar`. If we were to notify + // during this period it would be ignored and then when the parked + // thread went to sleep it would never wake up. Fortunately, it has + // `lock` locked at this stage so we can acquire `lock` to wait until + // it is ready to receive the notification. + // + // Releasing `lock` before the call to `notify_one` means that when the + // parked thread wakes it doesn't get woken only to have to wait for us + // to release `lock`. + drop(self.lock.lock().unwrap()); + self.cvar.notify_one() + } +} diff --git a/crux-mir/lib/std/src/sys_common/thread_parking/id.rs b/crux-mir/lib/std/src/sys_common/thread_parking/id.rs new file mode 100644 index 000000000..e98169597 --- /dev/null +++ b/crux-mir/lib/std/src/sys_common/thread_parking/id.rs @@ -0,0 +1,108 @@ +//! Thread parking using thread ids. +//! +//! Some platforms (notably NetBSD) have thread parking primitives whose semantics +//! match those offered by `thread::park`, with the difference that the thread to +//! be unparked is referenced by a platform-specific thread id. Since the thread +//! parker is constructed before that id is known, an atomic state variable is used +//! to manage the park state and propagate the thread id. This also avoids platform +//! calls in the case where `unpark` is called before `park`. + +use crate::cell::UnsafeCell; +use crate::pin::Pin; +use crate::sync::atomic::{ + fence, AtomicI8, + Ordering::{Acquire, Relaxed, Release}, +}; +use crate::sys::thread_parking::{current, park, park_timeout, unpark, ThreadId}; +use crate::time::Duration; + +pub struct Parker { + state: AtomicI8, + tid: UnsafeCell>, +} + +const PARKED: i8 = -1; +const EMPTY: i8 = 0; +const NOTIFIED: i8 = 1; + +impl Parker { + pub fn new() -> Parker { + Parker { state: AtomicI8::new(EMPTY), tid: UnsafeCell::new(None) } + } + + /// Create a new thread parker. UNIX requires this to happen in-place. + pub unsafe fn new_in_place(parker: *mut Parker) { + parker.write(Parker::new()) + } + + /// # Safety + /// * must always be called from the same thread + /// * must be called before the state is set to PARKED + unsafe fn init_tid(&self) { + // The field is only ever written to from this thread, so we don't need + // synchronization to read it here. + if self.tid.get().read().is_none() { + // Because this point is only reached once, before the state is set + // to PARKED for the first time, the non-atomic write here can not + // conflict with reads by other threads. + self.tid.get().write(Some(current())); + // Ensure that the write can be observed by all threads reading the + // state. Synchronizes with the acquire barrier in `unpark`. + fence(Release); + } + } + + pub unsafe fn park(self: Pin<&Self>) { + self.init_tid(); + + // Changes NOTIFIED to EMPTY and EMPTY to PARKED. + let mut state = self.state.fetch_sub(1, Acquire).wrapping_sub(1); + if state == PARKED { + // Loop to guard against spurious wakeups. + while state == PARKED { + park(self.state.as_mut_ptr().addr()); + state = self.state.load(Acquire); + } + + // Since the state change has already been observed with acquire + // ordering, the state can be reset with a relaxed store instead + // of a swap. + self.state.store(EMPTY, Relaxed); + } + } + + pub unsafe fn park_timeout(self: Pin<&Self>, dur: Duration) { + self.init_tid(); + + let state = self.state.fetch_sub(1, Acquire).wrapping_sub(1); + if state == PARKED { + park_timeout(dur, self.state.as_mut_ptr().addr()); + // Swap to ensure that we observe all state changes with acquire + // ordering, even if the state has been changed after the timeout + // occured. + self.state.swap(EMPTY, Acquire); + } + } + + pub fn unpark(self: Pin<&Self>) { + let state = self.state.swap(NOTIFIED, Release); + if state == PARKED { + // Synchronize with the release fence in `init_tid` to observe the + // write to `tid`. + fence(Acquire); + // # Safety + // The thread id is initialized before the state is set to `PARKED` + // for the first time and is not written to from that point on + // (negating the need for an atomic read). + let tid = unsafe { self.tid.get().read().unwrap_unchecked() }; + // It is possible that the waiting thread woke up because of a timeout + // and terminated before this call is made. This call then returns an + // error or wakes up an unrelated thread. The platform API and + // environment does allow this, however. + unpark(tid, self.state.as_mut_ptr().addr()); + } + } +} + +unsafe impl Send for Parker {} +unsafe impl Sync for Parker {} diff --git a/crux-mir/lib/std/src/sys_common/thread_parking/mod.rs b/crux-mir/lib/std/src/sys_common/thread_parking/mod.rs new file mode 100644 index 000000000..0ead6633c --- /dev/null +++ b/crux-mir/lib/std/src/sys_common/thread_parking/mod.rs @@ -0,0 +1,29 @@ +cfg_if::cfg_if! { + if #[cfg(any( + target_os = "linux", + target_os = "android", + all(target_arch = "wasm32", target_feature = "atomics"), + target_os = "freebsd", + target_os = "openbsd", + target_os = "dragonfly", + target_os = "fuchsia", + target_os = "hermit", + ))] { + mod futex; + pub use futex::Parker; + } else if #[cfg(any( + target_os = "netbsd", + all(target_vendor = "fortanix", target_env = "sgx"), + ))] { + mod id; + pub use id::Parker; + } else if #[cfg(target_os = "solid_asp3")] { + mod wait_flag; + pub use wait_flag::Parker; + } else if #[cfg(any(windows, target_family = "unix"))] { + pub use crate::sys::thread_parking::Parker; + } else { + mod generic; + pub use generic::Parker; + } +} diff --git a/crux-mir/lib/std/src/sys_common/thread_parking/wait_flag.rs b/crux-mir/lib/std/src/sys_common/thread_parking/wait_flag.rs new file mode 100644 index 000000000..d0f8899a9 --- /dev/null +++ b/crux-mir/lib/std/src/sys_common/thread_parking/wait_flag.rs @@ -0,0 +1,102 @@ +//! A wait-flag-based thread parker. +//! +//! Some operating systems provide low-level parking primitives like wait counts, +//! event flags or semaphores which are not susceptible to race conditions (meaning +//! the wakeup can occur before the wait operation). To implement the `std` thread +//! parker on top of these primitives, we only have to ensure that parking is fast +//! when the thread token is available, the atomic ordering guarantees are maintained +//! and spurious wakeups are minimized. +//! +//! To achieve this, this parker uses an atomic variable with three states: `EMPTY`, +//! `PARKED` and `NOTIFIED`: +//! * `EMPTY` means the token has not been made available, but the thread is not +//! currently waiting on it. +//! * `PARKED` means the token is not available and the thread is parked. +//! * `NOTIFIED` means the token is available. +//! +//! `park` and `park_timeout` change the state from `EMPTY` to `PARKED` and from +//! `NOTIFIED` to `EMPTY`. If the state was `NOTIFIED`, the thread was unparked and +//! execution can continue without calling into the OS. If the state was `EMPTY`, +//! the token is not available and the thread waits on the primitive (here called +//! "wait flag"). +//! +//! `unpark` changes the state to `NOTIFIED`. If the state was `PARKED`, the thread +//! is or will be sleeping on the wait flag, so we raise it. + +use crate::pin::Pin; +use crate::sync::atomic::AtomicI8; +use crate::sync::atomic::Ordering::{Acquire, Relaxed, Release}; +use crate::sys::wait_flag::WaitFlag; +use crate::time::Duration; + +const EMPTY: i8 = 0; +const PARKED: i8 = -1; +const NOTIFIED: i8 = 1; + +pub struct Parker { + state: AtomicI8, + wait_flag: WaitFlag, +} + +impl Parker { + /// Construct a parker for the current thread. The UNIX parker + /// implementation requires this to happen in-place. + pub unsafe fn new_in_place(parker: *mut Parker) { + parker.write(Parker { state: AtomicI8::new(EMPTY), wait_flag: WaitFlag::new() }) + } + + // This implementation doesn't require `unsafe` and `Pin`, but other implementations do. + pub unsafe fn park(self: Pin<&Self>) { + match self.state.fetch_sub(1, Acquire) { + // NOTIFIED => EMPTY + NOTIFIED => return, + // EMPTY => PARKED + EMPTY => (), + _ => panic!("inconsistent park state"), + } + + // Avoid waking up from spurious wakeups (these are quite likely, see below). + loop { + self.wait_flag.wait(); + + match self.state.compare_exchange(NOTIFIED, EMPTY, Acquire, Relaxed) { + Ok(_) => return, + Err(PARKED) => (), + Err(_) => panic!("inconsistent park state"), + } + } + } + + // This implementation doesn't require `unsafe` and `Pin`, but other implementations do. + pub unsafe fn park_timeout(self: Pin<&Self>, dur: Duration) { + match self.state.fetch_sub(1, Acquire) { + NOTIFIED => return, + EMPTY => (), + _ => panic!("inconsistent park state"), + } + + self.wait_flag.wait_timeout(dur); + + // Either a wakeup or a timeout occurred. Wakeups may be spurious, as there can be + // a race condition when `unpark` is performed between receiving the timeout and + // resetting the state, resulting in the eventflag being set unnecessarily. `park` + // is protected against this by looping until the token is actually given, but + // here we cannot easily tell. + + // Use `swap` to provide acquire ordering. + match self.state.swap(EMPTY, Acquire) { + NOTIFIED => (), + PARKED => (), + _ => panic!("inconsistent park state"), + } + } + + // This implementation doesn't require `Pin`, but other implementations do. + pub fn unpark(self: Pin<&Self>) { + let state = self.state.swap(NOTIFIED, Release); + + if state == PARKED { + self.wait_flag.raise(); + } + } +} diff --git a/crux-mir/lib/std/src/sys_common/util.rs b/crux-mir/lib/std/src/sys_common/util.rs deleted file mode 100644 index 00f7db4c0..000000000 --- a/crux-mir/lib/std/src/sys_common/util.rs +++ /dev/null @@ -1,30 +0,0 @@ -use crate::fmt; -use crate::io::prelude::*; -use crate::sys::stdio::panic_output; -use crate::thread; - -pub fn dumb_print(args: fmt::Arguments<'_>) { - if let Some(mut out) = panic_output() { - let _ = out.write_fmt(args); - } -} - -// Other platforms should use the appropriate platform-specific mechanism for -// aborting the process. If no platform-specific mechanism is available, -// crate::intrinsics::abort() may be used instead. The above implementations cover -// all targets currently supported by libstd. - -pub fn abort(args: fmt::Arguments<'_>) -> ! { - dumb_print(format_args!("fatal runtime error: {}\n", args)); - unsafe { - crate::sys::abort_internal(); - } -} - -#[allow(dead_code)] // stack overflow detection not enabled on all platforms -pub unsafe fn report_overflow() { - dumb_print(format_args!( - "\nthread '{}' has overflowed its stack\n", - thread::current().name().unwrap_or("") - )); -} diff --git a/crux-mir/lib/std/src/sys_common/wstr.rs b/crux-mir/lib/std/src/sys_common/wstr.rs new file mode 100644 index 000000000..b230fd1a8 --- /dev/null +++ b/crux-mir/lib/std/src/sys_common/wstr.rs @@ -0,0 +1,59 @@ +//! This module contains constructs to work with 16-bit characters (UCS-2 or UTF-16) +#![allow(dead_code)] + +use crate::marker::PhantomData; +use crate::num::NonZeroU16; +use crate::ptr::NonNull; + +/// A safe iterator over a LPWSTR +/// (aka a pointer to a series of UTF-16 code units terminated by a NULL). +pub struct WStrUnits<'a> { + // The pointer must never be null... + lpwstr: NonNull, + // ...and the memory it points to must be valid for this lifetime. + lifetime: PhantomData<&'a [u16]>, +} + +impl WStrUnits<'_> { + /// Create the iterator. Returns `None` if `lpwstr` is null. + /// + /// SAFETY: `lpwstr` must point to a null-terminated wide string that lives + /// at least as long as the lifetime of this struct. + pub unsafe fn new(lpwstr: *const u16) -> Option { + Some(Self { lpwstr: NonNull::new(lpwstr as _)?, lifetime: PhantomData }) + } + + pub fn peek(&self) -> Option { + // SAFETY: It's always safe to read the current item because we don't + // ever move out of the array's bounds. + unsafe { NonZeroU16::new(*self.lpwstr.as_ptr()) } + } + + /// Advance the iterator while `predicate` returns true. + /// Returns the number of items it advanced by. + pub fn advance_while bool>(&mut self, mut predicate: P) -> usize { + let mut counter = 0; + while let Some(w) = self.peek() { + if !predicate(w) { + break; + } + counter += 1; + self.next(); + } + counter + } +} + +impl Iterator for WStrUnits<'_> { + // This can never return zero as that marks the end of the string. + type Item = NonZeroU16; + fn next(&mut self) -> Option { + // SAFETY: If NULL is reached we immediately return. + // Therefore it's safe to advance the pointer after that. + unsafe { + let next = self.peek()?; + self.lpwstr = NonNull::new_unchecked(self.lpwstr.as_ptr().add(1)); + Some(next) + } + } +} diff --git a/crux-mir/lib/std/src/sys_common/wtf8.rs b/crux-mir/lib/std/src/sys_common/wtf8.rs index 498950e68..dd53767d4 100644 --- a/crux-mir/lib/std/src/sys_common/wtf8.rs +++ b/crux-mir/lib/std/src/sys_common/wtf8.rs @@ -15,13 +15,17 @@ // unix (it's mostly used on windows), so don't worry about dead code here. #![allow(dead_code)] +#[cfg(test)] +mod tests; + use core::str::next_code_point; use crate::borrow::Cow; use crate::char; +use crate::collections::TryReserveError; use crate::fmt; use crate::hash::{Hash, Hasher}; -use crate::iter::FromIterator; +use crate::iter::FusedIterator; use crate::mem; use crate::ops; use crate::rc::Rc; @@ -85,6 +89,24 @@ impl CodePoint { self.value } + /// Returns the numeric value of the code point if it is a leading surrogate. + #[inline] + pub fn to_lead_surrogate(&self) -> Option { + match self.value { + lead @ 0xD800..=0xDBFF => Some(lead as u16), + _ => None, + } + } + + /// Returns the numeric value of the code point if it is a trailing surrogate. + #[inline] + pub fn to_trail_surrogate(&self) -> Option { + match self.value { + trail @ 0xDC00..=0xDFFF => Some(trail as u16), + _ => None, + } + } + /// Optionally returns a Unicode scalar value for the code point. /// /// Returns `None` if the code point is a surrogate (from U+D800 to U+DFFF). @@ -113,6 +135,14 @@ impl CodePoint { #[derive(Eq, PartialEq, Ord, PartialOrd, Clone)] pub struct Wtf8Buf { bytes: Vec, + + /// Do we know that `bytes` holds a valid UTF-8 encoding? We can easily + /// know this if we're constructed from a `String` or `&str`. + /// + /// It is possible for `bytes` to have valid UTF-8 without this being + /// set, such as when we're concatenating `&Wtf8`'s and surrogates become + /// paired, as we don't bother to rescan the entire string. + is_known_utf8: bool, } impl ops::Deref for Wtf8Buf { @@ -143,13 +173,13 @@ impl Wtf8Buf { /// Creates a new, empty WTF-8 string. #[inline] pub fn new() -> Wtf8Buf { - Wtf8Buf { bytes: Vec::new() } + Wtf8Buf { bytes: Vec::new(), is_known_utf8: true } } /// Creates a new, empty WTF-8 string with pre-allocated capacity for `capacity` bytes. #[inline] pub fn with_capacity(capacity: usize) -> Wtf8Buf { - Wtf8Buf { bytes: Vec::with_capacity(capacity) } + Wtf8Buf { bytes: Vec::with_capacity(capacity), is_known_utf8: true } } /// Creates a WTF-8 string from a UTF-8 `String`. @@ -159,7 +189,7 @@ impl Wtf8Buf { /// Since WTF-8 is a superset of UTF-8, this always succeeds. #[inline] pub fn from_string(string: String) -> Wtf8Buf { - Wtf8Buf { bytes: string.into_bytes() } + Wtf8Buf { bytes: string.into_bytes(), is_known_utf8: true } } /// Creates a WTF-8 string from a UTF-8 `&str` slice. @@ -169,11 +199,12 @@ impl Wtf8Buf { /// Since WTF-8 is a superset of UTF-8, this always succeeds. #[inline] pub fn from_str(str: &str) -> Wtf8Buf { - Wtf8Buf { bytes: <[_]>::to_vec(str.as_bytes()) } + Wtf8Buf { bytes: <[_]>::to_vec(str.as_bytes()), is_known_utf8: true } } pub fn clear(&mut self) { - self.bytes.clear() + self.bytes.clear(); + self.is_known_utf8 = true; } /// Creates a WTF-8 string from a potentially ill-formed UTF-16 slice of 16-bit code units. @@ -189,9 +220,11 @@ impl Wtf8Buf { let surrogate = surrogate.unpaired_surrogate(); // Surrogates are known to be in the code point range. let code_point = unsafe { CodePoint::from_u32_unchecked(surrogate as u32) }; + // The string will now contain an unpaired surrogate. + string.is_known_utf8 = false; // Skip the WTF-8 concatenation check, // surrogate pairs are already decoded by decode_utf16 - string.push_code_point_unchecked(code_point) + string.push_code_point_unchecked(code_point); } } } @@ -199,11 +232,10 @@ impl Wtf8Buf { } /// Copied from String::push - /// This does **not** include the WTF-8 concatenation check. + /// This does **not** include the WTF-8 concatenation check or `is_known_utf8` check. fn push_code_point_unchecked(&mut self, code_point: CodePoint) { - let c = unsafe { char::from_u32_unchecked(code_point.value) }; let mut bytes = [0; 4]; - let bytes = c.encode_utf8(&mut bytes).as_bytes(); + let bytes = char::encode_utf8_raw(code_point.value, &mut bytes); self.bytes.extend_from_slice(bytes) } @@ -214,6 +246,9 @@ impl Wtf8Buf { #[inline] pub fn as_mut_slice(&mut self) -> &mut Wtf8 { + // Safety: `Wtf8` doesn't expose any way to mutate the bytes that would + // cause them to change from well-formed UTF-8 to ill-formed UTF-8, + // which would break the assumptions of the `is_known_utf8` field. unsafe { Wtf8::from_mut_bytes_unchecked(&mut self.bytes) } } @@ -229,11 +264,48 @@ impl Wtf8Buf { self.bytes.reserve(additional) } + /// Tries to reserve capacity for at least `additional` more length units + /// in the given `Wtf8Buf`. The `Wtf8Buf` may reserve more space to avoid + /// frequent reallocations. After calling `try_reserve`, capacity will be + /// greater than or equal to `self.len() + additional`. Does nothing if + /// capacity is already sufficient. This method preserves the contents even + /// if an error occurs. + /// + /// # Errors + /// + /// If the capacity overflows, or the allocator reports a failure, then an error + /// is returned. + #[inline] + pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.bytes.try_reserve(additional) + } + #[inline] pub fn reserve_exact(&mut self, additional: usize) { self.bytes.reserve_exact(additional) } + /// Tries to reserve the minimum capacity for exactly `additional` + /// length units in the given `Wtf8Buf`. After calling + /// `try_reserve_exact`, capacity will be greater than or equal to + /// `self.len() + additional` if it returns `Ok(())`. + /// Does nothing if the capacity is already sufficient. + /// + /// Note that the allocator may give the `Wtf8Buf` more space than it + /// requests. Therefore, capacity can not be relied upon to be precisely + /// minimal. Prefer [`try_reserve`] if future insertions are expected. + /// + /// [`try_reserve`]: Wtf8Buf::try_reserve + /// + /// # Errors + /// + /// If the capacity overflows, or the allocator reports a failure, then an error + /// is returned. + #[inline] + pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.bytes.try_reserve_exact(additional) + } + #[inline] pub fn shrink_to_fit(&mut self) { self.bytes.shrink_to_fit() @@ -274,7 +346,15 @@ impl Wtf8Buf { self.push_char(decode_surrogate_pair(lead, trail)); self.bytes.extend_from_slice(other_without_trail_surrogate); } - _ => self.bytes.extend_from_slice(&other.bytes), + _ => { + // If we'll be pushing a string containing a surrogate, we may + // no longer have UTF-8. + if other.next_surrogate(0).is_some() { + self.is_known_utf8 = false; + } + + self.bytes.extend_from_slice(&other.bytes); + } } } @@ -291,13 +371,19 @@ impl Wtf8Buf { /// like concatenating ill-formed UTF-16 strings effectively would. #[inline] pub fn push(&mut self, code_point: CodePoint) { - if let trail @ 0xDC00..=0xDFFF = code_point.to_u32() { + if let Some(trail) = code_point.to_trail_surrogate() { if let Some(lead) = (&*self).final_lead_surrogate() { let len_without_lead_surrogate = self.len() - 3; self.bytes.truncate(len_without_lead_surrogate); - self.push_char(decode_surrogate_pair(lead, trail as u16)); + self.push_char(decode_surrogate_pair(lead, trail)); return; } + + // We're pushing a trailing surrogate. + self.is_known_utf8 = false; + } else if code_point.to_lead_surrogate().is_some() { + // We're pushing a leading surrogate. + self.is_known_utf8 = false; } // No newly paired surrogates at the boundary. @@ -324,9 +410,10 @@ impl Wtf8Buf { /// (that is, if the string contains surrogates), /// the original WTF-8 string is returned instead. pub fn into_string(self) -> Result { - match self.next_surrogate(0) { - None => Ok(unsafe { String::from_utf8_unchecked(self.bytes) }), - Some(_) => Err(self), + if self.is_known_utf8 || self.next_surrogate(0).is_none() { + Ok(unsafe { String::from_utf8_unchecked(self.bytes) }) + } else { + Err(self) } } @@ -336,6 +423,11 @@ impl Wtf8Buf { /// /// Surrogates are replaced with `"\u{FFFD}"` (the replacement character “�”) pub fn into_string_lossy(mut self) -> String { + // Fast path: If we already have UTF-8, we can return it immediately. + if self.is_known_utf8 { + return unsafe { String::from_utf8_unchecked(self.bytes) }; + } + let mut pos = 0; loop { match self.next_surrogate(pos) { @@ -358,7 +450,7 @@ impl Wtf8Buf { /// Converts a `Box` into a `Wtf8Buf`. pub fn from_box(boxed: Box) -> Wtf8Buf { let bytes: Box<[u8]> = unsafe { mem::transmute(boxed) }; - Wtf8Buf { bytes: bytes.into_vec() } + Wtf8Buf { bytes: bytes.into_vec(), is_known_utf8: false } } } @@ -386,6 +478,17 @@ impl Extend for Wtf8Buf { self.bytes.reserve(low); iterator.for_each(move |code_point| self.push(code_point)); } + + #[inline] + fn extend_one(&mut self, code_point: CodePoint) { + self.push(code_point); + } + + #[inline] + fn extend_reserve(&mut self, additional: usize) { + // Lower bound of one byte per code point (ASCII only) + self.bytes.reserve(additional); + } } /// A borrowed slice of well-formed WTF-8 data. @@ -525,6 +628,11 @@ impl Wtf8 { } } + /// Creates an owned `Wtf8Buf` from a borrowed `Wtf8`. + pub fn to_owned(&self) -> Wtf8Buf { + Wtf8Buf { bytes: self.bytes.to_vec(), is_known_utf8: false } + } + /// Lossily converts the string to UTF-8. /// Returns a UTF-8 `&str` slice if the contents are well-formed in UTF-8. /// @@ -613,6 +721,11 @@ impl Wtf8 { } } + pub fn clone_into(&self, buf: &mut Wtf8Buf) { + buf.is_known_utf8 = false; + self.bytes.clone_into(&mut buf.bytes); + } + /// Boxes this `Wtf8`. #[inline] pub fn into_box(&self) -> Box { @@ -637,9 +750,39 @@ impl Wtf8 { let rc: Rc<[u8]> = Rc::from(&self.bytes); unsafe { Rc::from_raw(Rc::into_raw(rc) as *const Wtf8) } } + + #[inline] + pub fn make_ascii_lowercase(&mut self) { + self.bytes.make_ascii_lowercase() + } + + #[inline] + pub fn make_ascii_uppercase(&mut self) { + self.bytes.make_ascii_uppercase() + } + + #[inline] + pub fn to_ascii_lowercase(&self) -> Wtf8Buf { + Wtf8Buf { bytes: self.bytes.to_ascii_lowercase(), is_known_utf8: false } + } + + #[inline] + pub fn to_ascii_uppercase(&self) -> Wtf8Buf { + Wtf8Buf { bytes: self.bytes.to_ascii_uppercase(), is_known_utf8: false } + } + + #[inline] + pub fn is_ascii(&self) -> bool { + self.bytes.is_ascii() + } + + #[inline] + pub fn eq_ignore_ascii_case(&self, other: &Self) -> bool { + self.bytes.eq_ignore_ascii_case(&other.bytes) + } } -/// Returns a slice of the given string for the byte range [`begin`..`end`). +/// Returns a slice of the given string for the byte range \[`begin`..`end`). /// /// # Panics /// @@ -738,7 +881,7 @@ pub fn is_code_point_boundary(slice: &Wtf8, index: usize) -> bool { /// Copied from core::str::raw::slice_unchecked #[inline] pub unsafe fn slice_unchecked(s: &Wtf8, begin: usize, end: usize) -> &Wtf8 { - // memory layout of an &[u8] and &Wtf8 are the same + // memory layout of a &[u8] and &Wtf8 are the same Wtf8::from_bytes_unchecked(slice::from_raw_parts(s.bytes.as_ptr().add(begin), end - begin)) } @@ -746,7 +889,7 @@ pub unsafe fn slice_unchecked(s: &Wtf8, begin: usize, end: usize) -> &Wtf8 { #[inline(never)] pub fn slice_error_fail(s: &Wtf8, begin: usize, end: usize) -> ! { assert!(begin <= end); - panic!("index {} and/or {} in `{:?}` do not lie on character boundary", begin, end, s); + panic!("index {begin} and/or {end} in `{s:?}` do not lie on character boundary"); } /// Iterator for the code points of a WTF-8 string. @@ -762,7 +905,8 @@ impl<'a> Iterator for Wtf8CodePoints<'a> { #[inline] fn next(&mut self) -> Option { - next_code_point(&mut self.bytes).map(|c| CodePoint { value: c }) + // SAFETY: `self.bytes` has been created from a WTF-8 string + unsafe { next_code_point(&mut self.bytes).map(|c| CodePoint { value: c }) } } #[inline] @@ -795,8 +939,7 @@ impl<'a> Iterator for EncodeWide<'a> { let mut buf = [0; 2]; self.code_points.next().map(|code_point| { - let c = unsafe { char::from_u32_unchecked(code_point.value) }; - let n = c.encode_utf16(&mut buf).len(); + let n = char::encode_utf16_raw(code_point.value, &mut buf).len(); if n == 2 { self.extra = buf[1]; } @@ -807,13 +950,17 @@ impl<'a> Iterator for EncodeWide<'a> { #[inline] fn size_hint(&self) -> (usize, Option) { let (low, high) = self.code_points.size_hint(); + let ext = (self.extra != 0) as usize; // every code point gets either one u16 or two u16, // so this iterator is between 1 or 2 times as // long as the underlying iterator. - (low, high.and_then(|n| n.checked_mul(2))) + (low + ext, high.and_then(|n| n.checked_mul(2)).and_then(|n| n.checked_add(ext))) } } +#[stable(feature = "encode_wide_fused_iterator", since = "1.62.0")] +impl FusedIterator for EncodeWide<'_> {} + impl Hash for CodePoint { #[inline] fn hash(&self, state: &mut H) { @@ -836,413 +983,3 @@ impl Hash for Wtf8 { 0xfeu8.hash(state) } } - -impl Wtf8 { - pub fn make_ascii_uppercase(&mut self) { - self.bytes.make_ascii_uppercase() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::borrow::Cow; - - #[test] - fn code_point_from_u32() { - assert!(CodePoint::from_u32(0).is_some()); - assert!(CodePoint::from_u32(0xD800).is_some()); - assert!(CodePoint::from_u32(0x10FFFF).is_some()); - assert!(CodePoint::from_u32(0x110000).is_none()); - } - - #[test] - fn code_point_to_u32() { - fn c(value: u32) -> CodePoint { - CodePoint::from_u32(value).unwrap() - } - assert_eq!(c(0).to_u32(), 0); - assert_eq!(c(0xD800).to_u32(), 0xD800); - assert_eq!(c(0x10FFFF).to_u32(), 0x10FFFF); - } - - #[test] - fn code_point_from_char() { - assert_eq!(CodePoint::from_char('a').to_u32(), 0x61); - assert_eq!(CodePoint::from_char('💩').to_u32(), 0x1F4A9); - } - - #[test] - fn code_point_to_string() { - assert_eq!(format!("{:?}", CodePoint::from_char('a')), "U+0061"); - assert_eq!(format!("{:?}", CodePoint::from_char('💩')), "U+1F4A9"); - } - - #[test] - fn code_point_to_char() { - fn c(value: u32) -> CodePoint { - CodePoint::from_u32(value).unwrap() - } - assert_eq!(c(0x61).to_char(), Some('a')); - assert_eq!(c(0x1F4A9).to_char(), Some('💩')); - assert_eq!(c(0xD800).to_char(), None); - } - - #[test] - fn code_point_to_char_lossy() { - fn c(value: u32) -> CodePoint { - CodePoint::from_u32(value).unwrap() - } - assert_eq!(c(0x61).to_char_lossy(), 'a'); - assert_eq!(c(0x1F4A9).to_char_lossy(), '💩'); - assert_eq!(c(0xD800).to_char_lossy(), '\u{FFFD}'); - } - - #[test] - fn wtf8buf_new() { - assert_eq!(Wtf8Buf::new().bytes, b""); - } - - #[test] - fn wtf8buf_from_str() { - assert_eq!(Wtf8Buf::from_str("").bytes, b""); - assert_eq!(Wtf8Buf::from_str("aé 💩").bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); - } - - #[test] - fn wtf8buf_from_string() { - assert_eq!(Wtf8Buf::from_string(String::from("")).bytes, b""); - assert_eq!( - Wtf8Buf::from_string(String::from("aé 💩")).bytes, - b"a\xC3\xA9 \xF0\x9F\x92\xA9" - ); - } - - #[test] - fn wtf8buf_from_wide() { - assert_eq!(Wtf8Buf::from_wide(&[]).bytes, b""); - assert_eq!( - Wtf8Buf::from_wide(&[0x61, 0xE9, 0x20, 0xD83D, 0xD83D, 0xDCA9]).bytes, - b"a\xC3\xA9 \xED\xA0\xBD\xF0\x9F\x92\xA9" - ); - } - - #[test] - fn wtf8buf_push_str() { - let mut string = Wtf8Buf::new(); - assert_eq!(string.bytes, b""); - string.push_str("aé 💩"); - assert_eq!(string.bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); - } - - #[test] - fn wtf8buf_push_char() { - let mut string = Wtf8Buf::from_str("aé "); - assert_eq!(string.bytes, b"a\xC3\xA9 "); - string.push_char('💩'); - assert_eq!(string.bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); - } - - #[test] - fn wtf8buf_push() { - let mut string = Wtf8Buf::from_str("aé "); - assert_eq!(string.bytes, b"a\xC3\xA9 "); - string.push(CodePoint::from_char('💩')); - assert_eq!(string.bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); - - fn c(value: u32) -> CodePoint { - CodePoint::from_u32(value).unwrap() - } - - let mut string = Wtf8Buf::new(); - string.push(c(0xD83D)); // lead - string.push(c(0xDCA9)); // trail - assert_eq!(string.bytes, b"\xF0\x9F\x92\xA9"); // Magic! - - let mut string = Wtf8Buf::new(); - string.push(c(0xD83D)); // lead - string.push(c(0x20)); // not surrogate - string.push(c(0xDCA9)); // trail - assert_eq!(string.bytes, b"\xED\xA0\xBD \xED\xB2\xA9"); - - let mut string = Wtf8Buf::new(); - string.push(c(0xD800)); // lead - string.push(c(0xDBFF)); // lead - assert_eq!(string.bytes, b"\xED\xA0\x80\xED\xAF\xBF"); - - let mut string = Wtf8Buf::new(); - string.push(c(0xD800)); // lead - string.push(c(0xE000)); // not surrogate - assert_eq!(string.bytes, b"\xED\xA0\x80\xEE\x80\x80"); - - let mut string = Wtf8Buf::new(); - string.push(c(0xD7FF)); // not surrogate - string.push(c(0xDC00)); // trail - assert_eq!(string.bytes, b"\xED\x9F\xBF\xED\xB0\x80"); - - let mut string = Wtf8Buf::new(); - string.push(c(0x61)); // not surrogate, < 3 bytes - string.push(c(0xDC00)); // trail - assert_eq!(string.bytes, b"\x61\xED\xB0\x80"); - - let mut string = Wtf8Buf::new(); - string.push(c(0xDC00)); // trail - assert_eq!(string.bytes, b"\xED\xB0\x80"); - } - - #[test] - fn wtf8buf_push_wtf8() { - let mut string = Wtf8Buf::from_str("aé"); - assert_eq!(string.bytes, b"a\xC3\xA9"); - string.push_wtf8(Wtf8::from_str(" 💩")); - assert_eq!(string.bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); - - fn w(v: &[u8]) -> &Wtf8 { - unsafe { Wtf8::from_bytes_unchecked(v) } - } - - let mut string = Wtf8Buf::new(); - string.push_wtf8(w(b"\xED\xA0\xBD")); // lead - string.push_wtf8(w(b"\xED\xB2\xA9")); // trail - assert_eq!(string.bytes, b"\xF0\x9F\x92\xA9"); // Magic! - - let mut string = Wtf8Buf::new(); - string.push_wtf8(w(b"\xED\xA0\xBD")); // lead - string.push_wtf8(w(b" ")); // not surrogate - string.push_wtf8(w(b"\xED\xB2\xA9")); // trail - assert_eq!(string.bytes, b"\xED\xA0\xBD \xED\xB2\xA9"); - - let mut string = Wtf8Buf::new(); - string.push_wtf8(w(b"\xED\xA0\x80")); // lead - string.push_wtf8(w(b"\xED\xAF\xBF")); // lead - assert_eq!(string.bytes, b"\xED\xA0\x80\xED\xAF\xBF"); - - let mut string = Wtf8Buf::new(); - string.push_wtf8(w(b"\xED\xA0\x80")); // lead - string.push_wtf8(w(b"\xEE\x80\x80")); // not surrogate - assert_eq!(string.bytes, b"\xED\xA0\x80\xEE\x80\x80"); - - let mut string = Wtf8Buf::new(); - string.push_wtf8(w(b"\xED\x9F\xBF")); // not surrogate - string.push_wtf8(w(b"\xED\xB0\x80")); // trail - assert_eq!(string.bytes, b"\xED\x9F\xBF\xED\xB0\x80"); - - let mut string = Wtf8Buf::new(); - string.push_wtf8(w(b"a")); // not surrogate, < 3 bytes - string.push_wtf8(w(b"\xED\xB0\x80")); // trail - assert_eq!(string.bytes, b"\x61\xED\xB0\x80"); - - let mut string = Wtf8Buf::new(); - string.push_wtf8(w(b"\xED\xB0\x80")); // trail - assert_eq!(string.bytes, b"\xED\xB0\x80"); - } - - #[test] - fn wtf8buf_truncate() { - let mut string = Wtf8Buf::from_str("aé"); - string.truncate(1); - assert_eq!(string.bytes, b"a"); - } - - #[test] - #[should_panic] - fn wtf8buf_truncate_fail_code_point_boundary() { - let mut string = Wtf8Buf::from_str("aé"); - string.truncate(2); - } - - #[test] - #[should_panic] - fn wtf8buf_truncate_fail_longer() { - let mut string = Wtf8Buf::from_str("aé"); - string.truncate(4); - } - - #[test] - fn wtf8buf_into_string() { - let mut string = Wtf8Buf::from_str("aé 💩"); - assert_eq!(string.clone().into_string(), Ok(String::from("aé 💩"))); - string.push(CodePoint::from_u32(0xD800).unwrap()); - assert_eq!(string.clone().into_string(), Err(string)); - } - - #[test] - fn wtf8buf_into_string_lossy() { - let mut string = Wtf8Buf::from_str("aé 💩"); - assert_eq!(string.clone().into_string_lossy(), String::from("aé 💩")); - string.push(CodePoint::from_u32(0xD800).unwrap()); - assert_eq!(string.clone().into_string_lossy(), String::from("aé 💩�")); - } - - #[test] - fn wtf8buf_from_iterator() { - fn f(values: &[u32]) -> Wtf8Buf { - values.iter().map(|&c| CodePoint::from_u32(c).unwrap()).collect::() - } - assert_eq!(f(&[0x61, 0xE9, 0x20, 0x1F4A9]).bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); - - assert_eq!(f(&[0xD83D, 0xDCA9]).bytes, b"\xF0\x9F\x92\xA9"); // Magic! - assert_eq!(f(&[0xD83D, 0x20, 0xDCA9]).bytes, b"\xED\xA0\xBD \xED\xB2\xA9"); - assert_eq!(f(&[0xD800, 0xDBFF]).bytes, b"\xED\xA0\x80\xED\xAF\xBF"); - assert_eq!(f(&[0xD800, 0xE000]).bytes, b"\xED\xA0\x80\xEE\x80\x80"); - assert_eq!(f(&[0xD7FF, 0xDC00]).bytes, b"\xED\x9F\xBF\xED\xB0\x80"); - assert_eq!(f(&[0x61, 0xDC00]).bytes, b"\x61\xED\xB0\x80"); - assert_eq!(f(&[0xDC00]).bytes, b"\xED\xB0\x80"); - } - - #[test] - fn wtf8buf_extend() { - fn e(initial: &[u32], extended: &[u32]) -> Wtf8Buf { - fn c(value: &u32) -> CodePoint { - CodePoint::from_u32(*value).unwrap() - } - let mut string = initial.iter().map(c).collect::(); - string.extend(extended.iter().map(c)); - string - } - - assert_eq!(e(&[0x61, 0xE9], &[0x20, 0x1F4A9]).bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); - - assert_eq!(e(&[0xD83D], &[0xDCA9]).bytes, b"\xF0\x9F\x92\xA9"); // Magic! - assert_eq!(e(&[0xD83D, 0x20], &[0xDCA9]).bytes, b"\xED\xA0\xBD \xED\xB2\xA9"); - assert_eq!(e(&[0xD800], &[0xDBFF]).bytes, b"\xED\xA0\x80\xED\xAF\xBF"); - assert_eq!(e(&[0xD800], &[0xE000]).bytes, b"\xED\xA0\x80\xEE\x80\x80"); - assert_eq!(e(&[0xD7FF], &[0xDC00]).bytes, b"\xED\x9F\xBF\xED\xB0\x80"); - assert_eq!(e(&[0x61], &[0xDC00]).bytes, b"\x61\xED\xB0\x80"); - assert_eq!(e(&[], &[0xDC00]).bytes, b"\xED\xB0\x80"); - } - - #[test] - fn wtf8buf_show() { - let mut string = Wtf8Buf::from_str("a\té \u{7f}💩\r"); - string.push(CodePoint::from_u32(0xD800).unwrap()); - assert_eq!(format!("{:?}", string), "\"a\\té \\u{7f}\u{1f4a9}\\r\\u{d800}\""); - } - - #[test] - fn wtf8buf_as_slice() { - assert_eq!(Wtf8Buf::from_str("aé").as_slice(), Wtf8::from_str("aé")); - } - - #[test] - fn wtf8buf_show_str() { - let text = "a\té 💩\r"; - let string = Wtf8Buf::from_str(text); - assert_eq!(format!("{:?}", text), format!("{:?}", string)); - } - - #[test] - fn wtf8_from_str() { - assert_eq!(&Wtf8::from_str("").bytes, b""); - assert_eq!(&Wtf8::from_str("aé 💩").bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); - } - - #[test] - fn wtf8_len() { - assert_eq!(Wtf8::from_str("").len(), 0); - assert_eq!(Wtf8::from_str("aé 💩").len(), 8); - } - - #[test] - fn wtf8_slice() { - assert_eq!(&Wtf8::from_str("aé 💩")[1..4].bytes, b"\xC3\xA9 "); - } - - #[test] - #[should_panic] - fn wtf8_slice_not_code_point_boundary() { - &Wtf8::from_str("aé 💩")[2..4]; - } - - #[test] - fn wtf8_slice_from() { - assert_eq!(&Wtf8::from_str("aé 💩")[1..].bytes, b"\xC3\xA9 \xF0\x9F\x92\xA9"); - } - - #[test] - #[should_panic] - fn wtf8_slice_from_not_code_point_boundary() { - &Wtf8::from_str("aé 💩")[2..]; - } - - #[test] - fn wtf8_slice_to() { - assert_eq!(&Wtf8::from_str("aé 💩")[..4].bytes, b"a\xC3\xA9 "); - } - - #[test] - #[should_panic] - fn wtf8_slice_to_not_code_point_boundary() { - &Wtf8::from_str("aé 💩")[5..]; - } - - #[test] - fn wtf8_ascii_byte_at() { - let slice = Wtf8::from_str("aé 💩"); - assert_eq!(slice.ascii_byte_at(0), b'a'); - assert_eq!(slice.ascii_byte_at(1), b'\xFF'); - assert_eq!(slice.ascii_byte_at(2), b'\xFF'); - assert_eq!(slice.ascii_byte_at(3), b' '); - assert_eq!(slice.ascii_byte_at(4), b'\xFF'); - } - - #[test] - fn wtf8_code_points() { - fn c(value: u32) -> CodePoint { - CodePoint::from_u32(value).unwrap() - } - fn cp(string: &Wtf8Buf) -> Vec> { - string.code_points().map(|c| c.to_char()).collect::>() - } - let mut string = Wtf8Buf::from_str("é "); - assert_eq!(cp(&string), [Some('é'), Some(' ')]); - string.push(c(0xD83D)); - assert_eq!(cp(&string), [Some('é'), Some(' '), None]); - string.push(c(0xDCA9)); - assert_eq!(cp(&string), [Some('é'), Some(' '), Some('💩')]); - } - - #[test] - fn wtf8_as_str() { - assert_eq!(Wtf8::from_str("").as_str(), Some("")); - assert_eq!(Wtf8::from_str("aé 💩").as_str(), Some("aé 💩")); - let mut string = Wtf8Buf::new(); - string.push(CodePoint::from_u32(0xD800).unwrap()); - assert_eq!(string.as_str(), None); - } - - #[test] - fn wtf8_to_string_lossy() { - assert_eq!(Wtf8::from_str("").to_string_lossy(), Cow::Borrowed("")); - assert_eq!(Wtf8::from_str("aé 💩").to_string_lossy(), Cow::Borrowed("aé 💩")); - let mut string = Wtf8Buf::from_str("aé 💩"); - string.push(CodePoint::from_u32(0xD800).unwrap()); - let expected: Cow<'_, str> = Cow::Owned(String::from("aé 💩�")); - assert_eq!(string.to_string_lossy(), expected); - } - - #[test] - fn wtf8_display() { - fn d(b: &[u8]) -> String { - (&unsafe { Wtf8::from_bytes_unchecked(b) }).to_string() - } - - assert_eq!("", d("".as_bytes())); - assert_eq!("aé 💩", d("aé 💩".as_bytes())); - - let mut string = Wtf8Buf::from_str("aé 💩"); - string.push(CodePoint::from_u32(0xD800).unwrap()); - assert_eq!("aé 💩�", d(string.as_inner())); - } - - #[test] - fn wtf8_encode_wide() { - let mut string = Wtf8Buf::from_str("aé "); - string.push(CodePoint::from_u32(0xD83D).unwrap()); - string.push_char('💩'); - assert_eq!( - string.encode_wide().collect::>(), - vec![0x61, 0xE9, 0x20, 0xD83D, 0xD83D, 0xDCA9] - ); - } -} diff --git a/crux-mir/lib/std/src/sys_common/wtf8/tests.rs b/crux-mir/lib/std/src/sys_common/wtf8/tests.rs new file mode 100644 index 000000000..1a302d646 --- /dev/null +++ b/crux-mir/lib/std/src/sys_common/wtf8/tests.rs @@ -0,0 +1,666 @@ +use super::*; +use crate::borrow::Cow; + +#[test] +fn code_point_from_u32() { + assert!(CodePoint::from_u32(0).is_some()); + assert!(CodePoint::from_u32(0xD800).is_some()); + assert!(CodePoint::from_u32(0x10FFFF).is_some()); + assert!(CodePoint::from_u32(0x110000).is_none()); +} + +#[test] +fn code_point_to_u32() { + fn c(value: u32) -> CodePoint { + CodePoint::from_u32(value).unwrap() + } + assert_eq!(c(0).to_u32(), 0); + assert_eq!(c(0xD800).to_u32(), 0xD800); + assert_eq!(c(0x10FFFF).to_u32(), 0x10FFFF); +} + +#[test] +fn code_point_to_lead_surrogate() { + fn c(value: u32) -> CodePoint { + CodePoint::from_u32(value).unwrap() + } + assert_eq!(c(0).to_lead_surrogate(), None); + assert_eq!(c(0xE9).to_lead_surrogate(), None); + assert_eq!(c(0xD800).to_lead_surrogate(), Some(0xD800)); + assert_eq!(c(0xDBFF).to_lead_surrogate(), Some(0xDBFF)); + assert_eq!(c(0xDC00).to_lead_surrogate(), None); + assert_eq!(c(0xDFFF).to_lead_surrogate(), None); + assert_eq!(c(0x1F4A9).to_lead_surrogate(), None); + assert_eq!(c(0x10FFFF).to_lead_surrogate(), None); +} + +#[test] +fn code_point_to_trail_surrogate() { + fn c(value: u32) -> CodePoint { + CodePoint::from_u32(value).unwrap() + } + assert_eq!(c(0).to_trail_surrogate(), None); + assert_eq!(c(0xE9).to_trail_surrogate(), None); + assert_eq!(c(0xD800).to_trail_surrogate(), None); + assert_eq!(c(0xDBFF).to_trail_surrogate(), None); + assert_eq!(c(0xDC00).to_trail_surrogate(), Some(0xDC00)); + assert_eq!(c(0xDFFF).to_trail_surrogate(), Some(0xDFFF)); + assert_eq!(c(0x1F4A9).to_trail_surrogate(), None); + assert_eq!(c(0x10FFFF).to_trail_surrogate(), None); +} + +#[test] +fn code_point_from_char() { + assert_eq!(CodePoint::from_char('a').to_u32(), 0x61); + assert_eq!(CodePoint::from_char('💩').to_u32(), 0x1F4A9); +} + +#[test] +fn code_point_to_string() { + assert_eq!(format!("{:?}", CodePoint::from_char('a')), "U+0061"); + assert_eq!(format!("{:?}", CodePoint::from_char('💩')), "U+1F4A9"); +} + +#[test] +fn code_point_to_char() { + fn c(value: u32) -> CodePoint { + CodePoint::from_u32(value).unwrap() + } + assert_eq!(c(0x61).to_char(), Some('a')); + assert_eq!(c(0x1F4A9).to_char(), Some('💩')); + assert_eq!(c(0xD800).to_char(), None); +} + +#[test] +fn code_point_to_char_lossy() { + fn c(value: u32) -> CodePoint { + CodePoint::from_u32(value).unwrap() + } + assert_eq!(c(0x61).to_char_lossy(), 'a'); + assert_eq!(c(0x1F4A9).to_char_lossy(), '💩'); + assert_eq!(c(0xD800).to_char_lossy(), '\u{FFFD}'); +} + +#[test] +fn wtf8buf_new() { + assert_eq!(Wtf8Buf::new().bytes, b""); +} + +#[test] +fn wtf8buf_from_str() { + assert_eq!(Wtf8Buf::from_str("").bytes, b""); + assert_eq!(Wtf8Buf::from_str("aé 💩").bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); +} + +#[test] +fn wtf8buf_from_string() { + assert_eq!(Wtf8Buf::from_string(String::from("")).bytes, b""); + assert_eq!(Wtf8Buf::from_string(String::from("aé 💩")).bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); +} + +#[test] +fn wtf8buf_from_wide() { + let buf = Wtf8Buf::from_wide(&[]); + assert_eq!(buf.bytes, b""); + assert!(buf.is_known_utf8); + + let buf = Wtf8Buf::from_wide(&[0x61, 0xE9, 0x20, 0xD83D, 0xDCA9]); + assert_eq!(buf.bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); + assert!(buf.is_known_utf8); + + let buf = Wtf8Buf::from_wide(&[0x61, 0xE9, 0x20, 0xD83D, 0xD83D, 0xDCA9]); + assert_eq!(buf.bytes, b"a\xC3\xA9 \xED\xA0\xBD\xF0\x9F\x92\xA9"); + assert!(!buf.is_known_utf8); + + let buf = Wtf8Buf::from_wide(&[0xD800]); + assert_eq!(buf.bytes, b"\xED\xA0\x80"); + assert!(!buf.is_known_utf8); + + let buf = Wtf8Buf::from_wide(&[0xDBFF]); + assert_eq!(buf.bytes, b"\xED\xAF\xBF"); + assert!(!buf.is_known_utf8); + + let buf = Wtf8Buf::from_wide(&[0xDC00]); + assert_eq!(buf.bytes, b"\xED\xB0\x80"); + assert!(!buf.is_known_utf8); + + let buf = Wtf8Buf::from_wide(&[0xDFFF]); + assert_eq!(buf.bytes, b"\xED\xBF\xBF"); + assert!(!buf.is_known_utf8); +} + +#[test] +fn wtf8buf_push_str() { + let mut string = Wtf8Buf::new(); + assert_eq!(string.bytes, b""); + assert!(string.is_known_utf8); + + string.push_str("aé 💩"); + assert_eq!(string.bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); + assert!(string.is_known_utf8); +} + +#[test] +fn wtf8buf_push_char() { + let mut string = Wtf8Buf::from_str("aé "); + assert_eq!(string.bytes, b"a\xC3\xA9 "); + assert!(string.is_known_utf8); + + string.push_char('💩'); + assert_eq!(string.bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); + assert!(string.is_known_utf8); +} + +#[test] +fn wtf8buf_push() { + let mut string = Wtf8Buf::from_str("aé "); + assert_eq!(string.bytes, b"a\xC3\xA9 "); + assert!(string.is_known_utf8); + + string.push(CodePoint::from_char('💩')); + assert_eq!(string.bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); + assert!(string.is_known_utf8); + + fn c(value: u32) -> CodePoint { + CodePoint::from_u32(value).unwrap() + } + + let mut string = Wtf8Buf::new(); + string.push(c(0xD83D)); // lead + assert!(!string.is_known_utf8); + string.push(c(0xDCA9)); // trail + assert_eq!(string.bytes, b"\xF0\x9F\x92\xA9"); // Magic! + + let mut string = Wtf8Buf::new(); + string.push(c(0xD83D)); // lead + assert!(!string.is_known_utf8); + string.push(c(0x20)); // not surrogate + string.push(c(0xDCA9)); // trail + assert_eq!(string.bytes, b"\xED\xA0\xBD \xED\xB2\xA9"); + + let mut string = Wtf8Buf::new(); + string.push(c(0xD800)); // lead + assert!(!string.is_known_utf8); + string.push(c(0xDBFF)); // lead + assert_eq!(string.bytes, b"\xED\xA0\x80\xED\xAF\xBF"); + + let mut string = Wtf8Buf::new(); + string.push(c(0xD800)); // lead + assert!(!string.is_known_utf8); + string.push(c(0xE000)); // not surrogate + assert_eq!(string.bytes, b"\xED\xA0\x80\xEE\x80\x80"); + + let mut string = Wtf8Buf::new(); + string.push(c(0xD7FF)); // not surrogate + assert!(string.is_known_utf8); + string.push(c(0xDC00)); // trail + assert!(!string.is_known_utf8); + assert_eq!(string.bytes, b"\xED\x9F\xBF\xED\xB0\x80"); + + let mut string = Wtf8Buf::new(); + string.push(c(0x61)); // not surrogate, < 3 bytes + assert!(string.is_known_utf8); + string.push(c(0xDC00)); // trail + assert!(!string.is_known_utf8); + assert_eq!(string.bytes, b"\x61\xED\xB0\x80"); + + let mut string = Wtf8Buf::new(); + string.push(c(0xDC00)); // trail + assert!(!string.is_known_utf8); + assert_eq!(string.bytes, b"\xED\xB0\x80"); +} + +#[test] +fn wtf8buf_push_wtf8() { + let mut string = Wtf8Buf::from_str("aé"); + assert_eq!(string.bytes, b"a\xC3\xA9"); + string.push_wtf8(Wtf8::from_str(" 💩")); + assert_eq!(string.bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); + assert!(string.is_known_utf8); + + fn w(v: &[u8]) -> &Wtf8 { + unsafe { Wtf8::from_bytes_unchecked(v) } + } + + let mut string = Wtf8Buf::new(); + string.push_wtf8(w(b"\xED\xA0\xBD")); // lead + string.push_wtf8(w(b"\xED\xB2\xA9")); // trail + assert_eq!(string.bytes, b"\xF0\x9F\x92\xA9"); // Magic! + + let mut string = Wtf8Buf::new(); + string.push_wtf8(w(b"\xED\xA0\xBD")); // lead + string.push_wtf8(w(b" ")); // not surrogate + string.push_wtf8(w(b"\xED\xB2\xA9")); // trail + assert_eq!(string.bytes, b"\xED\xA0\xBD \xED\xB2\xA9"); + assert!(!string.is_known_utf8); + + let mut string = Wtf8Buf::new(); + string.push_wtf8(w(b"\xED\xA0\x80")); // lead + string.push_wtf8(w(b"\xED\xAF\xBF")); // lead + assert_eq!(string.bytes, b"\xED\xA0\x80\xED\xAF\xBF"); + assert!(!string.is_known_utf8); + + let mut string = Wtf8Buf::new(); + string.push_wtf8(w(b"\xED\xA0\x80")); // lead + string.push_wtf8(w(b"\xEE\x80\x80")); // not surrogate + assert_eq!(string.bytes, b"\xED\xA0\x80\xEE\x80\x80"); + assert!(!string.is_known_utf8); + + let mut string = Wtf8Buf::new(); + string.push_wtf8(w(b"\xED\x9F\xBF")); // not surrogate + string.push_wtf8(w(b"\xED\xB0\x80")); // trail + assert_eq!(string.bytes, b"\xED\x9F\xBF\xED\xB0\x80"); + assert!(!string.is_known_utf8); + + let mut string = Wtf8Buf::new(); + string.push_wtf8(w(b"a")); // not surrogate, < 3 bytes + string.push_wtf8(w(b"\xED\xB0\x80")); // trail + assert_eq!(string.bytes, b"\x61\xED\xB0\x80"); + assert!(!string.is_known_utf8); + + let mut string = Wtf8Buf::new(); + string.push_wtf8(w(b"\xED\xB0\x80")); // trail + assert_eq!(string.bytes, b"\xED\xB0\x80"); + assert!(!string.is_known_utf8); +} + +#[test] +fn wtf8buf_truncate() { + let mut string = Wtf8Buf::from_str("aé"); + assert!(string.is_known_utf8); + + string.truncate(3); + assert_eq!(string.bytes, b"a\xC3\xA9"); + assert!(string.is_known_utf8); + + string.truncate(1); + assert_eq!(string.bytes, b"a"); + assert!(string.is_known_utf8); + + string.truncate(0); + assert_eq!(string.bytes, b""); + assert!(string.is_known_utf8); +} + +#[test] +fn wtf8buf_truncate_around_non_bmp() { + let mut string = Wtf8Buf::from_str("💩"); + assert!(string.is_known_utf8); + + string.truncate(4); + assert_eq!(string.bytes, b"\xF0\x9F\x92\xA9"); + assert!(string.is_known_utf8); + + string.truncate(0); + assert_eq!(string.bytes, b""); + assert!(string.is_known_utf8); +} + +#[test] +#[should_panic] +fn wtf8buf_truncate_fail_code_point_boundary() { + let mut string = Wtf8Buf::from_str("aé"); + string.truncate(2); +} + +#[test] +#[should_panic] +fn wtf8buf_truncate_fail_longer() { + let mut string = Wtf8Buf::from_str("aé"); + string.truncate(4); +} + +#[test] +#[should_panic] +fn wtf8buf_truncate_splitting_non_bmp3() { + let mut string = Wtf8Buf::from_str("💩"); + assert!(string.is_known_utf8); + string.truncate(3); +} + +#[test] +#[should_panic] +fn wtf8buf_truncate_splitting_non_bmp2() { + let mut string = Wtf8Buf::from_str("💩"); + assert!(string.is_known_utf8); + string.truncate(2); +} + +#[test] +#[should_panic] +fn wtf8buf_truncate_splitting_non_bmp1() { + let mut string = Wtf8Buf::from_str("💩"); + assert!(string.is_known_utf8); + string.truncate(1); +} + +#[test] +fn wtf8buf_into_string() { + let mut string = Wtf8Buf::from_str("aé 💩"); + assert!(string.is_known_utf8); + assert_eq!(string.clone().into_string(), Ok(String::from("aé 💩"))); + string.push(CodePoint::from_u32(0xD800).unwrap()); + assert!(!string.is_known_utf8); + assert_eq!(string.clone().into_string(), Err(string)); +} + +#[test] +fn wtf8buf_into_string_lossy() { + let mut string = Wtf8Buf::from_str("aé 💩"); + assert_eq!(string.clone().into_string_lossy(), String::from("aé 💩")); + string.push(CodePoint::from_u32(0xD800).unwrap()); + assert_eq!(string.clone().into_string_lossy(), String::from("aé 💩�")); +} + +#[test] +fn wtf8buf_from_iterator() { + fn f(values: &[u32]) -> Wtf8Buf { + values.iter().map(|&c| CodePoint::from_u32(c).unwrap()).collect::() + } + assert_eq!( + f(&[0x61, 0xE9, 0x20, 0x1F4A9]), + Wtf8Buf { bytes: b"a\xC3\xA9 \xF0\x9F\x92\xA9".to_vec(), is_known_utf8: true } + ); + + assert_eq!(f(&[0xD83D, 0xDCA9]).bytes, b"\xF0\x9F\x92\xA9"); // Magic! + assert_eq!( + f(&[0xD83D, 0x20, 0xDCA9]), + Wtf8Buf { bytes: b"\xED\xA0\xBD \xED\xB2\xA9".to_vec(), is_known_utf8: false } + ); + assert_eq!( + f(&[0xD800, 0xDBFF]), + Wtf8Buf { bytes: b"\xED\xA0\x80\xED\xAF\xBF".to_vec(), is_known_utf8: false } + ); + assert_eq!( + f(&[0xD800, 0xE000]), + Wtf8Buf { bytes: b"\xED\xA0\x80\xEE\x80\x80".to_vec(), is_known_utf8: false } + ); + assert_eq!( + f(&[0xD7FF, 0xDC00]), + Wtf8Buf { bytes: b"\xED\x9F\xBF\xED\xB0\x80".to_vec(), is_known_utf8: false } + ); + assert_eq!( + f(&[0x61, 0xDC00]), + Wtf8Buf { bytes: b"\x61\xED\xB0\x80".to_vec(), is_known_utf8: false } + ); + assert_eq!(f(&[0xDC00]), Wtf8Buf { bytes: b"\xED\xB0\x80".to_vec(), is_known_utf8: false }); +} + +#[test] +fn wtf8buf_extend() { + fn e(initial: &[u32], extended: &[u32]) -> Wtf8Buf { + fn c(value: &u32) -> CodePoint { + CodePoint::from_u32(*value).unwrap() + } + let mut string = initial.iter().map(c).collect::(); + string.extend(extended.iter().map(c)); + string + } + + assert_eq!( + e(&[0x61, 0xE9], &[0x20, 0x1F4A9]), + Wtf8Buf { bytes: b"a\xC3\xA9 \xF0\x9F\x92\xA9".to_vec(), is_known_utf8: true } + ); + + assert_eq!(e(&[0xD83D], &[0xDCA9]).bytes, b"\xF0\x9F\x92\xA9"); // Magic! + assert_eq!( + e(&[0xD83D, 0x20], &[0xDCA9]), + Wtf8Buf { bytes: b"\xED\xA0\xBD \xED\xB2\xA9".to_vec(), is_known_utf8: false } + ); + assert_eq!( + e(&[0xD800], &[0xDBFF]), + Wtf8Buf { bytes: b"\xED\xA0\x80\xED\xAF\xBF".to_vec(), is_known_utf8: false } + ); + assert_eq!( + e(&[0xD800], &[0xE000]), + Wtf8Buf { bytes: b"\xED\xA0\x80\xEE\x80\x80".to_vec(), is_known_utf8: false } + ); + assert_eq!( + e(&[0xD7FF], &[0xDC00]), + Wtf8Buf { bytes: b"\xED\x9F\xBF\xED\xB0\x80".to_vec(), is_known_utf8: false } + ); + assert_eq!( + e(&[0x61], &[0xDC00]), + Wtf8Buf { bytes: b"\x61\xED\xB0\x80".to_vec(), is_known_utf8: false } + ); + assert_eq!( + e(&[], &[0xDC00]), + Wtf8Buf { bytes: b"\xED\xB0\x80".to_vec(), is_known_utf8: false } + ); +} + +#[test] +fn wtf8buf_show() { + let mut string = Wtf8Buf::from_str("a\té \u{7f}💩\r"); + string.push(CodePoint::from_u32(0xD800).unwrap()); + assert_eq!(format!("{string:?}"), "\"a\\té \\u{7f}\u{1f4a9}\\r\\u{d800}\""); +} + +#[test] +fn wtf8buf_as_slice() { + assert_eq!(Wtf8Buf::from_str("aé").as_slice(), Wtf8::from_str("aé")); +} + +#[test] +fn wtf8buf_show_str() { + let text = "a\té 💩\r"; + let string = Wtf8Buf::from_str(text); + assert_eq!(format!("{text:?}"), format!("{string:?}")); +} + +#[test] +fn wtf8_from_str() { + assert_eq!(&Wtf8::from_str("").bytes, b""); + assert_eq!(&Wtf8::from_str("aé 💩").bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); +} + +#[test] +fn wtf8_len() { + assert_eq!(Wtf8::from_str("").len(), 0); + assert_eq!(Wtf8::from_str("aé 💩").len(), 8); +} + +#[test] +fn wtf8_slice() { + assert_eq!(&Wtf8::from_str("aé 💩")[1..4].bytes, b"\xC3\xA9 "); +} + +#[test] +#[should_panic] +fn wtf8_slice_not_code_point_boundary() { + let _ = &Wtf8::from_str("aé 💩")[2..4]; +} + +#[test] +fn wtf8_slice_from() { + assert_eq!(&Wtf8::from_str("aé 💩")[1..].bytes, b"\xC3\xA9 \xF0\x9F\x92\xA9"); +} + +#[test] +#[should_panic] +fn wtf8_slice_from_not_code_point_boundary() { + let _ = &Wtf8::from_str("aé 💩")[2..]; +} + +#[test] +fn wtf8_slice_to() { + assert_eq!(&Wtf8::from_str("aé 💩")[..4].bytes, b"a\xC3\xA9 "); +} + +#[test] +#[should_panic] +fn wtf8_slice_to_not_code_point_boundary() { + let _ = &Wtf8::from_str("aé 💩")[5..]; +} + +#[test] +fn wtf8_ascii_byte_at() { + let slice = Wtf8::from_str("aé 💩"); + assert_eq!(slice.ascii_byte_at(0), b'a'); + assert_eq!(slice.ascii_byte_at(1), b'\xFF'); + assert_eq!(slice.ascii_byte_at(2), b'\xFF'); + assert_eq!(slice.ascii_byte_at(3), b' '); + assert_eq!(slice.ascii_byte_at(4), b'\xFF'); +} + +#[test] +fn wtf8_code_points() { + fn c(value: u32) -> CodePoint { + CodePoint::from_u32(value).unwrap() + } + fn cp(string: &Wtf8Buf) -> Vec> { + string.code_points().map(|c| c.to_char()).collect::>() + } + let mut string = Wtf8Buf::from_str("é "); + assert_eq!(cp(&string), [Some('é'), Some(' ')]); + string.push(c(0xD83D)); + assert_eq!(cp(&string), [Some('é'), Some(' '), None]); + string.push(c(0xDCA9)); + assert_eq!(cp(&string), [Some('é'), Some(' '), Some('💩')]); +} + +#[test] +fn wtf8_as_str() { + assert_eq!(Wtf8::from_str("").as_str(), Some("")); + assert_eq!(Wtf8::from_str("aé 💩").as_str(), Some("aé 💩")); + let mut string = Wtf8Buf::new(); + string.push(CodePoint::from_u32(0xD800).unwrap()); + assert_eq!(string.as_str(), None); +} + +#[test] +fn wtf8_to_string_lossy() { + assert_eq!(Wtf8::from_str("").to_string_lossy(), Cow::Borrowed("")); + assert_eq!(Wtf8::from_str("aé 💩").to_string_lossy(), Cow::Borrowed("aé 💩")); + let mut string = Wtf8Buf::from_str("aé 💩"); + string.push(CodePoint::from_u32(0xD800).unwrap()); + let expected: Cow<'_, str> = Cow::Owned(String::from("aé 💩�")); + assert_eq!(string.to_string_lossy(), expected); +} + +#[test] +fn wtf8_display() { + fn d(b: &[u8]) -> String { + (&unsafe { Wtf8::from_bytes_unchecked(b) }).to_string() + } + + assert_eq!("", d("".as_bytes())); + assert_eq!("aé 💩", d("aé 💩".as_bytes())); + + let mut string = Wtf8Buf::from_str("aé 💩"); + string.push(CodePoint::from_u32(0xD800).unwrap()); + assert_eq!("aé 💩�", d(string.as_inner())); +} + +#[test] +fn wtf8_encode_wide() { + let mut string = Wtf8Buf::from_str("aé "); + string.push(CodePoint::from_u32(0xD83D).unwrap()); + string.push_char('💩'); + assert_eq!( + string.encode_wide().collect::>(), + vec![0x61, 0xE9, 0x20, 0xD83D, 0xD83D, 0xDCA9] + ); +} + +#[test] +fn wtf8_encode_wide_size_hint() { + let string = Wtf8Buf::from_str("\u{12345}"); + let mut iter = string.encode_wide(); + assert_eq!((1, Some(8)), iter.size_hint()); + iter.next().unwrap(); + assert_eq!((1, Some(1)), iter.size_hint()); + iter.next().unwrap(); + assert_eq!((0, Some(0)), iter.size_hint()); + assert!(iter.next().is_none()); +} + +#[test] +fn wtf8_clone_into() { + let mut string = Wtf8Buf::new(); + Wtf8::from_str("green").clone_into(&mut string); + assert_eq!(string.bytes, b"green"); + + let mut string = Wtf8Buf::from_str("green"); + Wtf8::from_str("").clone_into(&mut string); + assert_eq!(string.bytes, b""); + + let mut string = Wtf8Buf::from_str("red"); + Wtf8::from_str("green").clone_into(&mut string); + assert_eq!(string.bytes, b"green"); + + let mut string = Wtf8Buf::from_str("green"); + Wtf8::from_str("red").clone_into(&mut string); + assert_eq!(string.bytes, b"red"); + + let mut string = Wtf8Buf::from_str("green"); + assert!(string.is_known_utf8); + unsafe { Wtf8::from_bytes_unchecked(b"\xED\xA0\x80").clone_into(&mut string) }; + assert_eq!(string.bytes, b"\xED\xA0\x80"); + assert!(!string.is_known_utf8); +} + +#[test] +fn wtf8_to_ascii_lowercase() { + let lowercase = Wtf8::from_str("").to_ascii_lowercase(); + assert_eq!(lowercase.bytes, b""); + + let lowercase = Wtf8::from_str("GrEeN gRaPeS! 🍇").to_ascii_lowercase(); + assert_eq!(lowercase.bytes, b"green grapes! \xf0\x9f\x8d\x87"); + + let lowercase = unsafe { Wtf8::from_bytes_unchecked(b"\xED\xA0\x80").to_ascii_lowercase() }; + assert_eq!(lowercase.bytes, b"\xED\xA0\x80"); + assert!(!lowercase.is_known_utf8); +} + +#[test] +fn wtf8_to_ascii_uppercase() { + let uppercase = Wtf8::from_str("").to_ascii_uppercase(); + assert_eq!(uppercase.bytes, b""); + + let uppercase = Wtf8::from_str("GrEeN gRaPeS! 🍇").to_ascii_uppercase(); + assert_eq!(uppercase.bytes, b"GREEN GRAPES! \xf0\x9f\x8d\x87"); + + let uppercase = unsafe { Wtf8::from_bytes_unchecked(b"\xED\xA0\x80").to_ascii_uppercase() }; + assert_eq!(uppercase.bytes, b"\xED\xA0\x80"); + assert!(!uppercase.is_known_utf8); +} + +#[test] +fn wtf8_make_ascii_lowercase() { + let mut lowercase = Wtf8Buf::from_str(""); + lowercase.make_ascii_lowercase(); + assert_eq!(lowercase.bytes, b""); + + let mut lowercase = Wtf8Buf::from_str("GrEeN gRaPeS! 🍇"); + lowercase.make_ascii_lowercase(); + assert_eq!(lowercase.bytes, b"green grapes! \xf0\x9f\x8d\x87"); + + let mut lowercase = unsafe { Wtf8::from_bytes_unchecked(b"\xED\xA0\x80").to_owned() }; + lowercase.make_ascii_lowercase(); + assert_eq!(lowercase.bytes, b"\xED\xA0\x80"); + assert!(!lowercase.is_known_utf8); +} + +#[test] +fn wtf8_make_ascii_uppercase() { + let mut uppercase = Wtf8Buf::from_str(""); + uppercase.make_ascii_uppercase(); + assert_eq!(uppercase.bytes, b""); + + let mut uppercase = Wtf8Buf::from_str("GrEeN gRaPeS! 🍇"); + uppercase.make_ascii_uppercase(); + assert_eq!(uppercase.bytes, b"GREEN GRAPES! \xf0\x9f\x8d\x87"); + + let mut uppercase = unsafe { Wtf8::from_bytes_unchecked(b"\xED\xA0\x80").to_owned() }; + uppercase.make_ascii_uppercase(); + assert_eq!(uppercase.bytes, b"\xED\xA0\x80"); + assert!(!uppercase.is_known_utf8); +} + +#[test] +fn wtf8_to_owned() { + let string = unsafe { Wtf8::from_bytes_unchecked(b"\xED\xA0\x80").to_owned() }; + assert_eq!(string.bytes, b"\xED\xA0\x80"); + assert!(!string.is_known_utf8); +} diff --git a/crux-mir/lib/std/src/thread/local.rs b/crux-mir/lib/std/src/thread/local.rs index 1dd942e25..b30bb7b77 100644 --- a/crux-mir/lib/std/src/thread/local.rs +++ b/crux-mir/lib/std/src/thread/local.rs @@ -2,6 +2,13 @@ #![unstable(feature = "thread_local_internals", issue = "none")] +#[cfg(all(test, not(target_os = "emscripten")))] +mod tests; + +#[cfg(test)] +mod dynamic_tests; + +use crate::cell::{Cell, RefCell}; use crate::error::Error; use crate::fmt; @@ -14,6 +21,8 @@ use crate::fmt; /// The [`with`] method yields a reference to the contained value which cannot be /// sent across threads or escape the given closure. /// +/// [`thread_local!`]: crate::thread_local +/// /// # Initialization and Destruction /// /// Initialization is dynamically performed on the first call to [`with`] @@ -70,10 +79,23 @@ use crate::fmt; /// destroyed, but not all platforms have this guard. Those platforms that do /// not guard typically have a synthetic limit after which point no more /// destructors are run. +/// 3. When the process exits on Windows systems, TLS destructors may only be +/// run on the thread that causes the process to exit. This is because the +/// other threads may be forcibly terminated. /// -/// [`with`]: ../../std/thread/struct.LocalKey.html#method.with -/// [`thread_local!`]: ../../std/macro.thread_local.html -/// [`Drop`]: ../../std/ops/trait.Drop.html +/// ## Synchronization in thread-local destructors +/// +/// On Windows, synchronization operations (such as [`JoinHandle::join`]) in +/// thread local destructors are prone to deadlocks and so should be avoided. +/// This is because the [loader lock] is held while a destructor is run. The +/// lock is acquired whenever a thread starts or exits or when a DLL is loaded +/// or unloaded. Therefore these events are blocked for as long as a thread +/// local destructor is running. +/// +/// [loader lock]: https://docs.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-best-practices +/// [`JoinHandle::join`]: crate::thread::JoinHandle::join +/// [`with`]: LocalKey::with +#[cfg_attr(not(test), rustc_diagnostic_item = "LocalKey")] #[stable(feature = "rust1", since = "1.0.0")] pub struct LocalKey { // This outer `LocalKey` type is what's going to be stored in statics, @@ -90,13 +112,13 @@ pub struct LocalKey { // trivially devirtualizable by LLVM because the value of `inner` never // changes and the constant should be readonly within a crate. This mainly // only runs into problems when TLS statics are exported across crates. - inner: unsafe fn() -> Option<&'static T>, + inner: unsafe fn(Option<&mut Option>) -> Option<&'static T>, } #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for LocalKey { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("LocalKey { .. }") + f.debug_struct("LocalKey").finish_non_exhaustive() } } @@ -118,17 +140,27 @@ impl fmt::Debug for LocalKey { /// # fn main() {} /// ``` /// -/// See [LocalKey documentation][`std::thread::LocalKey`] for more +/// See [`LocalKey` documentation][`std::thread::LocalKey`] for more /// information. /// -/// [`std::thread::LocalKey`]: ../std/thread/struct.LocalKey.html +/// [`std::thread::LocalKey`]: crate::thread::LocalKey #[macro_export] #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "thread_local_macro")] #[allow_internal_unstable(thread_local_internals)] macro_rules! thread_local { // empty (base case for the recursion) () => {}; + ($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty = const { $init:expr }; $($rest:tt)*) => ( + $crate::__thread_local_inner!($(#[$attr])* $vis $name, $t, const $init); + $crate::thread_local!($($rest)*); + ); + + ($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty = const { $init:expr }) => ( + $crate::__thread_local_inner!($(#[$attr])* $vis $name, $t, const $init); + ); + // process multiple declarations ($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty = $init:expr; $($rest:tt)*) => ( $crate::__thread_local_inner!($(#[$attr])* $vis $name, $t, $init); @@ -147,32 +179,181 @@ macro_rules! thread_local { #[allow_internal_unstable(thread_local_internals, cfg_target_thread_local, thread_local)] #[allow_internal_unsafe] macro_rules! __thread_local_inner { + // used to generate the `LocalKey` value for const-initialized thread locals + (@key $t:ty, const $init:expr) => {{ + #[cfg_attr(not(windows), inline)] // see comments below + #[deny(unsafe_op_in_unsafe_fn)] + unsafe fn __getit( + _init: $crate::option::Option<&mut $crate::option::Option<$t>>, + ) -> $crate::option::Option<&'static $t> { + const INIT_EXPR: $t = $init; + + // wasm without atomics maps directly to `static mut`, and dtors + // aren't implemented because thread dtors aren't really a thing + // on wasm right now + // + // FIXME(#84224) this should come after the `target_thread_local` + // block. + #[cfg(all(target_family = "wasm", not(target_feature = "atomics")))] + { + static mut VAL: $t = INIT_EXPR; + unsafe { $crate::option::Option::Some(&VAL) } + } + + // If the platform has support for `#[thread_local]`, use it. + #[cfg(all( + target_thread_local, + not(all(target_family = "wasm", not(target_feature = "atomics"))), + ))] + { + #[thread_local] + static mut VAL: $t = INIT_EXPR; + + // If a dtor isn't needed we can do something "very raw" and + // just get going. + if !$crate::mem::needs_drop::<$t>() { + unsafe { + return $crate::option::Option::Some(&VAL) + } + } + + // 0 == dtor not registered + // 1 == dtor registered, dtor not run + // 2 == dtor registered and is running or has run + #[thread_local] + static mut STATE: $crate::primitive::u8 = 0; + + unsafe extern "C" fn destroy(ptr: *mut $crate::primitive::u8) { + let ptr = ptr as *mut $t; + + unsafe { + $crate::debug_assert_eq!(STATE, 1); + STATE = 2; + $crate::ptr::drop_in_place(ptr); + } + } + + unsafe { + match STATE { + // 0 == we haven't registered a destructor, so do + // so now. + 0 => { + $crate::thread::__FastLocalKeyInner::<$t>::register_dtor( + $crate::ptr::addr_of_mut!(VAL) as *mut $crate::primitive::u8, + destroy, + ); + STATE = 1; + $crate::option::Option::Some(&VAL) + } + // 1 == the destructor is registered and the value + // is valid, so return the pointer. + 1 => $crate::option::Option::Some(&VAL), + // otherwise the destructor has already run, so we + // can't give access. + _ => $crate::option::Option::None, + } + } + } + + // On platforms without `#[thread_local]` we fall back to the + // same implementation as below for os thread locals. + #[cfg(all( + not(target_thread_local), + not(all(target_family = "wasm", not(target_feature = "atomics"))), + ))] + { + #[inline] + const fn __init() -> $t { INIT_EXPR } + static __KEY: $crate::thread::__OsLocalKeyInner<$t> = + $crate::thread::__OsLocalKeyInner::new(); + #[allow(unused_unsafe)] + unsafe { + __KEY.get(move || { + if let $crate::option::Option::Some(init) = _init { + if let $crate::option::Option::Some(value) = init.take() { + return value; + } else if $crate::cfg!(debug_assertions) { + $crate::unreachable!("missing initial value"); + } + } + __init() + }) + } + } + } + + unsafe { + $crate::thread::LocalKey::new(__getit) + } + }}; + + // used to generate the `LocalKey` value for `thread_local!` (@key $t:ty, $init:expr) => { { #[inline] fn __init() -> $t { $init } - unsafe fn __getit() -> $crate::option::Option<&'static $t> { - #[cfg(all(target_arch = "wasm32", not(target_feature = "atomics")))] + // When reading this function you might ask "why is this inlined + // everywhere other than Windows?", and that's a very reasonable + // question to ask. The short story is that it segfaults rustc if + // this function is inlined. The longer story is that Windows looks + // to not support `extern` references to thread locals across DLL + // boundaries. This appears to at least not be supported in the ABI + // that LLVM implements. + // + // Because of this we never inline on Windows, but we do inline on + // other platforms (where external references to thread locals + // across DLLs are supported). A better fix for this would be to + // inline this function on Windows, but only for "statically linked" + // components. For example if two separately compiled rlibs end up + // getting linked into a DLL then it's fine to inline this function + // across that boundary. It's only not fine to inline this function + // across a DLL boundary. Unfortunately rustc doesn't currently + // have this sort of logic available in an attribute, and it's not + // clear that rustc is even equipped to answer this (it's more of a + // Cargo question kinda). This means that, unfortunately, Windows + // gets the pessimistic path for now where it's never inlined. + // + // The issue of "should enable on Windows sometimes" is #84933 + #[cfg_attr(not(windows), inline)] + unsafe fn __getit( + init: $crate::option::Option<&mut $crate::option::Option<$t>>, + ) -> $crate::option::Option<&'static $t> { + #[cfg(all(target_family = "wasm", not(target_feature = "atomics")))] static __KEY: $crate::thread::__StaticLocalKeyInner<$t> = $crate::thread::__StaticLocalKeyInner::new(); #[thread_local] #[cfg(all( target_thread_local, - not(all(target_arch = "wasm32", not(target_feature = "atomics"))), + not(all(target_family = "wasm", not(target_feature = "atomics"))), ))] static __KEY: $crate::thread::__FastLocalKeyInner<$t> = $crate::thread::__FastLocalKeyInner::new(); #[cfg(all( not(target_thread_local), - not(all(target_arch = "wasm32", not(target_feature = "atomics"))), + not(all(target_family = "wasm", not(target_feature = "atomics"))), ))] static __KEY: $crate::thread::__OsLocalKeyInner<$t> = $crate::thread::__OsLocalKeyInner::new(); - __KEY.get(__init) + // FIXME: remove the #[allow(...)] marker when macros don't + // raise warning for missing/extraneous unsafe blocks anymore. + // See https://github.com/rust-lang/rust/issues/74838. + #[allow(unused_unsafe)] + unsafe { + __KEY.get(move || { + if let $crate::option::Option::Some(init) = init { + if let $crate::option::Option::Some(value) = init.take() { + return value; + } else if $crate::cfg!(debug_assertions) { + $crate::unreachable!("missing default value"); + } + } + __init() + }) + } } unsafe { @@ -180,18 +361,17 @@ macro_rules! __thread_local_inner { } } }; - ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $init:expr) => { + ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => { $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> = - $crate::__thread_local_inner!(@key $t, $init); + $crate::__thread_local_inner!(@key $t, $($init)*); } } /// An error returned by [`LocalKey::try_with`](struct.LocalKey.html#method.try_with). #[stable(feature = "thread_local_try_with", since = "1.26.0")] +#[non_exhaustive] #[derive(Clone, Copy, Eq, PartialEq)] -pub struct AccessError { - _private: (), -} +pub struct AccessError; #[stable(feature = "thread_local_try_with", since = "1.26.0")] impl fmt::Debug for AccessError { @@ -217,7 +397,10 @@ impl LocalKey { reason = "recently added to create a key", issue = "none" )] - pub const unsafe fn new(inner: unsafe fn() -> Option<&'static T>) -> LocalKey { + #[rustc_const_unstable(feature = "thread_local_internals", issue = "none")] + pub const unsafe fn new( + inner: unsafe fn(Option<&mut Option>) -> Option<&'static T>, + ) -> LocalKey { LocalKey { inner } } @@ -246,22 +429,355 @@ impl LocalKey { /// /// This will lazily initialize the value if this thread has not referenced /// this key yet. If the key has been destroyed (which may happen if this is called - /// in a destructor), this function will return an [`AccessError`](struct.AccessError.html). + /// in a destructor), this function will return an [`AccessError`]. /// /// # Panics /// /// This function will still `panic!()` if the key is uninitialized and the /// key's initializer panics. #[stable(feature = "thread_local_try_with", since = "1.26.0")] + #[inline] pub fn try_with(&'static self, f: F) -> Result where F: FnOnce(&T) -> R, { unsafe { - let thread_local = (self.inner)().ok_or(AccessError { _private: () })?; + let thread_local = (self.inner)(None).ok_or(AccessError)?; Ok(f(thread_local)) } } + + /// Acquires a reference to the value in this TLS key, initializing it with + /// `init` if it wasn't already initialized on this thread. + /// + /// If `init` was used to initialize the thread local variable, `None` is + /// passed as the first argument to `f`. If it was already initialized, + /// `Some(init)` is passed to `f`. + /// + /// # Panics + /// + /// This function will panic if the key currently has its destructor + /// running, and it **may** panic if the destructor has previously been run + /// for this thread. + fn initialize_with(&'static self, init: T, f: F) -> R + where + F: FnOnce(Option, &T) -> R, + { + unsafe { + let mut init = Some(init); + let reference = (self.inner)(Some(&mut init)).expect( + "cannot access a Thread Local Storage value \ + during or after destruction", + ); + f(init, reference) + } + } +} + +impl LocalKey> { + /// Sets or initializes the contained value. + /// + /// Unlike the other methods, this will *not* run the lazy initializer of + /// the thread local. Instead, it will be directly initialized with the + /// given value if it wasn't initialized yet. + /// + /// # Panics + /// + /// Panics if the key currently has its destructor running, + /// and it **may** panic if the destructor has previously been run for this thread. + /// + /// # Examples + /// + /// ``` + /// #![feature(local_key_cell_methods)] + /// use std::cell::Cell; + /// + /// thread_local! { + /// static X: Cell = panic!("!"); + /// } + /// + /// // Calling X.get() here would result in a panic. + /// + /// X.set(123); // But X.set() is fine, as it skips the initializer above. + /// + /// assert_eq!(X.get(), 123); + /// ``` + #[unstable(feature = "local_key_cell_methods", issue = "92122")] + pub fn set(&'static self, value: T) { + self.initialize_with(Cell::new(value), |value, cell| { + if let Some(value) = value { + // The cell was already initialized, so `value` wasn't used to + // initialize it. So we overwrite the current value with the + // new one instead. + cell.set(value.into_inner()); + } + }); + } + + /// Returns a copy of the contained value. + /// + /// This will lazily initialize the value if this thread has not referenced + /// this key yet. + /// + /// # Panics + /// + /// Panics if the key currently has its destructor running, + /// and it **may** panic if the destructor has previously been run for this thread. + /// + /// # Examples + /// + /// ``` + /// #![feature(local_key_cell_methods)] + /// use std::cell::Cell; + /// + /// thread_local! { + /// static X: Cell = Cell::new(1); + /// } + /// + /// assert_eq!(X.get(), 1); + /// ``` + #[unstable(feature = "local_key_cell_methods", issue = "92122")] + pub fn get(&'static self) -> T + where + T: Copy, + { + self.with(|cell| cell.get()) + } + + /// Takes the contained value, leaving `Default::default()` in its place. + /// + /// This will lazily initialize the value if this thread has not referenced + /// this key yet. + /// + /// # Panics + /// + /// Panics if the key currently has its destructor running, + /// and it **may** panic if the destructor has previously been run for this thread. + /// + /// # Examples + /// + /// ``` + /// #![feature(local_key_cell_methods)] + /// use std::cell::Cell; + /// + /// thread_local! { + /// static X: Cell> = Cell::new(Some(1)); + /// } + /// + /// assert_eq!(X.take(), Some(1)); + /// assert_eq!(X.take(), None); + /// ``` + #[unstable(feature = "local_key_cell_methods", issue = "92122")] + pub fn take(&'static self) -> T + where + T: Default, + { + self.with(|cell| cell.take()) + } + + /// Replaces the contained value, returning the old value. + /// + /// This will lazily initialize the value if this thread has not referenced + /// this key yet. + /// + /// # Panics + /// + /// Panics if the key currently has its destructor running, + /// and it **may** panic if the destructor has previously been run for this thread. + /// + /// # Examples + /// + /// ``` + /// #![feature(local_key_cell_methods)] + /// use std::cell::Cell; + /// + /// thread_local! { + /// static X: Cell = Cell::new(1); + /// } + /// + /// assert_eq!(X.replace(2), 1); + /// assert_eq!(X.replace(3), 2); + /// ``` + #[unstable(feature = "local_key_cell_methods", issue = "92122")] + pub fn replace(&'static self, value: T) -> T { + self.with(|cell| cell.replace(value)) + } +} + +impl LocalKey> { + /// Acquires a reference to the contained value. + /// + /// This will lazily initialize the value if this thread has not referenced + /// this key yet. + /// + /// # Panics + /// + /// Panics if the value is currently mutably borrowed. + /// + /// Panics if the key currently has its destructor running, + /// and it **may** panic if the destructor has previously been run for this thread. + /// + /// # Example + /// + /// ``` + /// #![feature(local_key_cell_methods)] + /// use std::cell::RefCell; + /// + /// thread_local! { + /// static X: RefCell> = RefCell::new(Vec::new()); + /// } + /// + /// X.with_borrow(|v| assert!(v.is_empty())); + /// ``` + #[unstable(feature = "local_key_cell_methods", issue = "92122")] + pub fn with_borrow(&'static self, f: F) -> R + where + F: FnOnce(&T) -> R, + { + self.with(|cell| f(&cell.borrow())) + } + + /// Acquires a mutable reference to the contained value. + /// + /// This will lazily initialize the value if this thread has not referenced + /// this key yet. + /// + /// # Panics + /// + /// Panics if the value is currently borrowed. + /// + /// Panics if the key currently has its destructor running, + /// and it **may** panic if the destructor has previously been run for this thread. + /// + /// # Example + /// + /// ``` + /// #![feature(local_key_cell_methods)] + /// use std::cell::RefCell; + /// + /// thread_local! { + /// static X: RefCell> = RefCell::new(Vec::new()); + /// } + /// + /// X.with_borrow_mut(|v| v.push(1)); + /// + /// X.with_borrow(|v| assert_eq!(*v, vec![1])); + /// ``` + #[unstable(feature = "local_key_cell_methods", issue = "92122")] + pub fn with_borrow_mut(&'static self, f: F) -> R + where + F: FnOnce(&mut T) -> R, + { + self.with(|cell| f(&mut cell.borrow_mut())) + } + + /// Sets or initializes the contained value. + /// + /// Unlike the other methods, this will *not* run the lazy initializer of + /// the thread local. Instead, it will be directly initialized with the + /// given value if it wasn't initialized yet. + /// + /// # Panics + /// + /// Panics if the value is currently borrowed. + /// + /// Panics if the key currently has its destructor running, + /// and it **may** panic if the destructor has previously been run for this thread. + /// + /// # Examples + /// + /// ``` + /// #![feature(local_key_cell_methods)] + /// use std::cell::RefCell; + /// + /// thread_local! { + /// static X: RefCell> = panic!("!"); + /// } + /// + /// // Calling X.with() here would result in a panic. + /// + /// X.set(vec![1, 2, 3]); // But X.set() is fine, as it skips the initializer above. + /// + /// X.with_borrow(|v| assert_eq!(*v, vec![1, 2, 3])); + /// ``` + #[unstable(feature = "local_key_cell_methods", issue = "92122")] + pub fn set(&'static self, value: T) { + self.initialize_with(RefCell::new(value), |value, cell| { + if let Some(value) = value { + // The cell was already initialized, so `value` wasn't used to + // initialize it. So we overwrite the current value with the + // new one instead. + *cell.borrow_mut() = value.into_inner(); + } + }); + } + + /// Takes the contained value, leaving `Default::default()` in its place. + /// + /// This will lazily initialize the value if this thread has not referenced + /// this key yet. + /// + /// # Panics + /// + /// Panics if the value is currently borrowed. + /// + /// Panics if the key currently has its destructor running, + /// and it **may** panic if the destructor has previously been run for this thread. + /// + /// # Examples + /// + /// ``` + /// #![feature(local_key_cell_methods)] + /// use std::cell::RefCell; + /// + /// thread_local! { + /// static X: RefCell> = RefCell::new(Vec::new()); + /// } + /// + /// X.with_borrow_mut(|v| v.push(1)); + /// + /// let a = X.take(); + /// + /// assert_eq!(a, vec![1]); + /// + /// X.with_borrow(|v| assert!(v.is_empty())); + /// ``` + #[unstable(feature = "local_key_cell_methods", issue = "92122")] + pub fn take(&'static self) -> T + where + T: Default, + { + self.with(|cell| cell.take()) + } + + /// Replaces the contained value, returning the old value. + /// + /// # Panics + /// + /// Panics if the value is currently borrowed. + /// + /// Panics if the key currently has its destructor running, + /// and it **may** panic if the destructor has previously been run for this thread. + /// + /// # Examples + /// + /// ``` + /// #![feature(local_key_cell_methods)] + /// use std::cell::RefCell; + /// + /// thread_local! { + /// static X: RefCell> = RefCell::new(Vec::new()); + /// } + /// + /// let prev = X.replace(vec![1, 2, 3]); + /// assert!(prev.is_empty()); + /// + /// X.with_borrow(|v| assert_eq!(*v, vec![1, 2, 3])); + /// ``` + #[unstable(feature = "local_key_cell_methods", issue = "92122")] + pub fn replace(&'static self, value: T) -> T { + self.with(|cell| cell.replace(value)) + } } mod lazy { @@ -279,15 +795,23 @@ mod lazy { } pub unsafe fn get(&self) -> Option<&'static T> { - (*self.inner.get()).as_ref() + // SAFETY: The caller must ensure no reference is ever handed out to + // the inner cell nor mutable reference to the Option inside said + // cell. This make it safe to hand a reference, though the lifetime + // of 'static is itself unsafe, making the get method unsafe. + unsafe { (*self.inner.get()).as_ref() } } + /// The caller must ensure that no reference is active: this method + /// needs unique access. pub unsafe fn initialize T>(&self, init: F) -> &'static T { // Execute the initialization up front, *then* move it into our slot, // just in case initialization fails. let value = init(); let ptr = self.inner.get(); + // SAFETY: + // // note that this can in theory just be `*ptr = Some(value)`, but due to // the compiler will currently codegen that pattern with something like: // @@ -300,30 +824,44 @@ mod lazy { // value (an aliasing violation). To avoid setting the "I'm running a // destructor" flag we just use `mem::replace` which should sequence the // operations a little differently and make this safe to call. - mem::replace(&mut *ptr, Some(value)); - - // After storing `Some` we want to get a reference to the contents of - // what we just stored. While we could use `unwrap` here and it should - // always work it empirically doesn't seem to always get optimized away, - // which means that using something like `try_with` can pull in - // panicking code and cause a large size bloat. - match *ptr { - Some(ref x) => x, - None => hint::unreachable_unchecked(), + // + // The precondition also ensures that we are the only one accessing + // `self` at the moment so replacing is fine. + unsafe { + let _ = mem::replace(&mut *ptr, Some(value)); + } + + // SAFETY: With the call to `mem::replace` it is guaranteed there is + // a `Some` behind `ptr`, not a `None` so `unreachable_unchecked` + // will never be reached. + unsafe { + // After storing `Some` we want to get a reference to the contents of + // what we just stored. While we could use `unwrap` here and it should + // always work it empirically doesn't seem to always get optimized away, + // which means that using something like `try_with` can pull in + // panicking code and cause a large size bloat. + match *ptr { + Some(ref x) => x, + None => hint::unreachable_unchecked(), + } } } + /// The other methods hand out references while taking &self. + /// As such, callers of this method must ensure no `&` and `&mut` are + /// available and used at the same time. #[allow(unused)] pub unsafe fn take(&mut self) -> Option { - (*self.inner.get()).take() + // SAFETY: See doc comment for this method. + unsafe { (*self.inner.get()).take() } } } } -/// On some platforms like wasm32 there's no threads, so no need to generate +/// On some targets like wasm there's no threads, so no need to generate /// thread locals and we can instead just use plain statics! #[doc(hidden)] -#[cfg(all(target_arch = "wasm32", not(target_feature = "atomics")))] +#[cfg(all(target_family = "wasm", not(target_feature = "atomics")))] pub mod statik { use super::lazy::LazyKeyInner; use crate::fmt; @@ -336,7 +874,7 @@ pub mod statik { impl fmt::Debug for Key { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("Key { .. }") + f.debug_struct("Key").finish_non_exhaustive() } } @@ -345,24 +883,30 @@ pub mod statik { Key { inner: LazyKeyInner::new() } } - pub unsafe fn get(&self, init: fn() -> T) -> Option<&'static T> { - let value = match self.inner.get() { - Some(ref value) => value, - None => self.inner.initialize(init), + pub unsafe fn get(&self, init: impl FnOnce() -> T) -> Option<&'static T> { + // SAFETY: The caller must ensure no reference is ever handed out to + // the inner cell nor mutable reference to the Option inside said + // cell. This make it safe to hand a reference, though the lifetime + // of 'static is itself unsafe, making the get method unsafe. + let value = unsafe { + match self.inner.get() { + Some(ref value) => value, + None => self.inner.initialize(init), + } }; + Some(value) } } } #[doc(hidden)] -#[cfg(target_thread_local)] +#[cfg(all(target_thread_local, not(all(target_family = "wasm", not(target_feature = "atomics"))),))] pub mod fast { use super::lazy::LazyKeyInner; use crate::cell::Cell; - use crate::fmt; - use crate::mem; - use crate::sys::fast_thread_local::register_dtor; + use crate::sys::thread_local_dtor::register_dtor; + use crate::{fmt, mem, panic}; #[derive(Copy, Clone)] enum DtorState { @@ -394,7 +938,7 @@ pub mod fast { impl fmt::Debug for Key { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("Key { .. }") + f.debug_struct("Key").finish_non_exhaustive() } } @@ -403,10 +947,28 @@ pub mod fast { Key { inner: LazyKeyInner::new(), dtor_state: Cell::new(DtorState::Unregistered) } } + // note that this is just a publicly-callable function only for the + // const-initialized form of thread locals, basically a way to call the + // free `register_dtor` function defined elsewhere in std. + pub unsafe fn register_dtor(a: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { + unsafe { + register_dtor(a, dtor); + } + } + pub unsafe fn get T>(&self, init: F) -> Option<&'static T> { - match self.inner.get() { - Some(val) => Some(val), - None => self.try_initialize(init), + // SAFETY: See the definitions of `LazyKeyInner::get` and + // `try_initialize` for more information. + // + // The caller must ensure no mutable references are ever active to + // the inner cell or the inner T when this is called. + // The `try_initialize` is dependant on the passed `init` function + // for this. + unsafe { + match self.inner.get() { + Some(val) => Some(val), + None => self.try_initialize(init), + } } } @@ -415,13 +977,14 @@ pub mod fast { // thread_local's, or it is being recursively initialized. // // Macos: Inlining this function can cause two `tlv_get_addr` calls to - // be performed for every call to `Key::get`. The #[cold] hint makes - // that less likely. + // be performed for every call to `Key::get`. // LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722 - #[cold] + #[inline(never)] unsafe fn try_initialize T>(&self, init: F) -> Option<&'static T> { - if !mem::needs_drop::() || self.try_register_dtor() { - Some(self.inner.initialize(init)) + // SAFETY: See comment above (this function doc). + if !mem::needs_drop::() || unsafe { self.try_register_dtor() } { + // SAFETY: See comment above (this function doc). + Some(unsafe { self.inner.initialize(init) }) } else { None } @@ -433,8 +996,12 @@ pub mod fast { unsafe fn try_register_dtor(&self) -> bool { match self.dtor_state.get() { DtorState::Unregistered => { - // dtor registration happens before initialization. - register_dtor(self as *const _ as *mut u8, destroy_value::); + // SAFETY: dtor registration happens before initialization. + // Passing `self` as a pointer while using `destroy_value` + // is safe because the function will build a pointer to a + // Key, which is the type of self and so find the correct + // size. + unsafe { register_dtor(self as *const _ as *mut u8, destroy_value::) }; self.dtor_state.set(DtorState::Registered); true } @@ -450,25 +1017,42 @@ pub mod fast { unsafe extern "C" fn destroy_value(ptr: *mut u8) { let ptr = ptr as *mut Key; + // SAFETY: + // + // The pointer `ptr` has been built just above and comes from + // `try_register_dtor` where it is originally a Key coming from `self`, + // making it non-NUL and of the correct type. + // // Right before we run the user destructor be sure to set the // `Option` to `None`, and `dtor_state` to `RunningOrHasRun`. This // causes future calls to `get` to run `try_initialize_drop` again, // which will now fail, and return `None`. - let value = (*ptr).inner.take(); - (*ptr).dtor_state.set(DtorState::RunningOrHasRun); - drop(value); + // + // Wrap the call in a catch to ensure unwinding is caught in the event + // a panic takes place in a destructor. + if let Err(_) = panic::catch_unwind(panic::AssertUnwindSafe(|| unsafe { + let value = (*ptr).inner.take(); + (*ptr).dtor_state.set(DtorState::RunningOrHasRun); + drop(value); + })) { + rtabort!("thread local panicked on drop"); + } } } #[doc(hidden)] +#[cfg(all( + not(target_thread_local), + not(all(target_family = "wasm", not(target_feature = "atomics"))), +))] pub mod os { use super::lazy::LazyKeyInner; use crate::cell::Cell; - use crate::fmt; - use crate::marker; - use crate::ptr; - use crate::sys_common::thread_local::StaticKey as OsStaticKey; + use crate::sys_common::thread_local_key::StaticKey as OsStaticKey; + use crate::{fmt, marker, panic, ptr}; + /// Use a regular global static to store this key; the state provided will then be + /// thread-local. pub struct Key { // OS-TLS key that we'll use to key off. os: OsStaticKey, @@ -477,7 +1061,7 @@ pub mod os { impl fmt::Debug for Key { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("Key { .. }") + f.debug_struct("Key").finish_non_exhaustive() } } @@ -489,26 +1073,36 @@ pub mod os { } impl Key { + #[rustc_const_unstable(feature = "thread_local_internals", issue = "none")] pub const fn new() -> Key { Key { os: OsStaticKey::new(Some(destroy_value::)), marker: marker::PhantomData } } - pub unsafe fn get(&'static self, init: fn() -> T) -> Option<&'static T> { - let ptr = self.os.get() as *mut Value; - if ptr as usize > 1 { - if let Some(ref value) = (*ptr).inner.get() { + /// It is a requirement for the caller to ensure that no mutable + /// reference is active when this method is called. + pub unsafe fn get(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> { + // SAFETY: See the documentation for this method. + let ptr = unsafe { self.os.get() as *mut Value }; + if ptr.addr() > 1 { + // SAFETY: the check ensured the pointer is safe (its destructor + // is not running) + it is coming from a trusted source (self). + if let Some(ref value) = unsafe { (*ptr).inner.get() } { return Some(value); } } - self.try_initialize(init) + // SAFETY: At this point we are sure we have no value and so + // initializing (or trying to) is safe. + unsafe { self.try_initialize(init) } } // `try_initialize` is only called once per os thread local variable, // except in corner cases where thread_local dtors reference other // thread_local's, or it is being recursively initialized. - unsafe fn try_initialize(&'static self, init: fn() -> T) -> Option<&'static T> { - let ptr = self.os.get() as *mut Value; - if ptr as usize == 1 { + unsafe fn try_initialize(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> { + // SAFETY: No mutable references are ever handed out meaning getting + // the value is ok. + let ptr = unsafe { self.os.get() as *mut Value }; + if ptr.addr() == 1 { // destructor is running return None; } @@ -518,229 +1112,44 @@ pub mod os { // local copy, so do that now. let ptr: Box> = box Value { inner: LazyKeyInner::new(), key: self }; let ptr = Box::into_raw(ptr); - self.os.set(ptr as *mut u8); + // SAFETY: At this point we are sure there is no value inside + // ptr so setting it will not affect anyone else. + unsafe { + self.os.set(ptr as *mut u8); + } ptr } else { // recursive initialization ptr }; - Some((*ptr).inner.initialize(init)) + // SAFETY: ptr has been ensured as non-NUL just above an so can be + // dereferenced safely. + unsafe { Some((*ptr).inner.initialize(init)) } } } unsafe extern "C" fn destroy_value(ptr: *mut u8) { - // The OS TLS ensures that this key contains a NULL value when this + // SAFETY: + // + // The OS TLS ensures that this key contains a null value when this // destructor starts to run. We set it back to a sentinel value of 1 to // ensure that any future calls to `get` for this thread will return // `None`. // // Note that to prevent an infinite loop we reset it back to null right // before we return from the destructor ourselves. - let ptr = Box::from_raw(ptr as *mut Value); - let key = ptr.key; - key.os.set(1 as *mut u8); - drop(ptr); - key.os.set(ptr::null_mut()); - } -} - -#[cfg(all(test, not(target_os = "emscripten")))] -mod tests { - use crate::cell::{Cell, UnsafeCell}; - use crate::sync::mpsc::{channel, Sender}; - use crate::thread; - - struct Foo(Sender<()>); - - impl Drop for Foo { - fn drop(&mut self) { - let Foo(ref s) = *self; - s.send(()).unwrap(); - } - } - - #[test] - fn smoke_no_dtor() { - thread_local!(static FOO: Cell = Cell::new(1)); - - FOO.with(|f| { - assert_eq!(f.get(), 1); - f.set(2); - }); - let (tx, rx) = channel(); - let _t = thread::spawn(move || { - FOO.with(|f| { - assert_eq!(f.get(), 1); - }); - tx.send(()).unwrap(); - }); - rx.recv().unwrap(); - - FOO.with(|f| { - assert_eq!(f.get(), 2); - }); - } - - #[test] - fn states() { - struct Foo; - impl Drop for Foo { - fn drop(&mut self) { - assert!(FOO.try_with(|_| ()).is_err()); - } - } - thread_local!(static FOO: Foo = Foo); - - thread::spawn(|| { - assert!(FOO.try_with(|_| ()).is_ok()); - }) - .join() - .ok() - .expect("thread panicked"); - } - - #[test] - fn smoke_dtor() { - thread_local!(static FOO: UnsafeCell> = UnsafeCell::new(None)); - - let (tx, rx) = channel(); - let _t = thread::spawn(move || unsafe { - let mut tx = Some(tx); - FOO.with(|f| { - *f.get() = Some(Foo(tx.take().unwrap())); - }); - }); - rx.recv().unwrap(); - } - - #[test] - fn circular() { - struct S1; - struct S2; - thread_local!(static K1: UnsafeCell> = UnsafeCell::new(None)); - thread_local!(static K2: UnsafeCell> = UnsafeCell::new(None)); - static mut HITS: u32 = 0; - - impl Drop for S1 { - fn drop(&mut self) { - unsafe { - HITS += 1; - if K2.try_with(|_| ()).is_err() { - assert_eq!(HITS, 3); - } else { - if HITS == 1 { - K2.with(|s| *s.get() = Some(S2)); - } else { - assert_eq!(HITS, 3); - } - } - } - } - } - impl Drop for S2 { - fn drop(&mut self) { - unsafe { - HITS += 1; - assert!(K1.try_with(|_| ()).is_ok()); - assert_eq!(HITS, 2); - K1.with(|s| *s.get() = Some(S1)); - } - } - } - - thread::spawn(move || { - drop(S1); - }) - .join() - .ok() - .expect("thread panicked"); - } - - #[test] - fn self_referential() { - struct S1; - thread_local!(static K1: UnsafeCell> = UnsafeCell::new(None)); - - impl Drop for S1 { - fn drop(&mut self) { - assert!(K1.try_with(|_| ()).is_err()); - } - } - - thread::spawn(move || unsafe { - K1.with(|s| *s.get() = Some(S1)); - }) - .join() - .ok() - .expect("thread panicked"); - } - - // Note that this test will deadlock if TLS destructors aren't run (this - // requires the destructor to be run to pass the test). - #[test] - fn dtors_in_dtors_in_dtors() { - struct S1(Sender<()>); - thread_local!(static K1: UnsafeCell> = UnsafeCell::new(None)); - thread_local!(static K2: UnsafeCell> = UnsafeCell::new(None)); - - impl Drop for S1 { - fn drop(&mut self) { - let S1(ref tx) = *self; - unsafe { - let _ = K2.try_with(|s| *s.get() = Some(Foo(tx.clone()))); - } - } - } - - let (tx, rx) = channel(); - let _t = thread::spawn(move || unsafe { - let mut tx = Some(tx); - K1.with(|s| *s.get() = Some(S1(tx.take().unwrap()))); - }); - rx.recv().unwrap(); - } -} - -#[cfg(test)] -mod dynamic_tests { - use crate::cell::RefCell; - use crate::collections::HashMap; - - #[test] - fn smoke() { - fn square(i: i32) -> i32 { - i * i - } - thread_local!(static FOO: i32 = square(3)); - - FOO.with(|f| { - assert_eq!(*f, 9); - }); - } - - #[test] - fn hashmap() { - fn map() -> RefCell> { - let mut m = HashMap::new(); - m.insert(1, 2); - RefCell::new(m) + // + // Wrap the call in a catch to ensure unwinding is caught in the event + // a panic takes place in a destructor. + if let Err(_) = panic::catch_unwind(|| unsafe { + let ptr = Box::from_raw(ptr as *mut Value); + let key = ptr.key; + key.os.set(ptr::invalid_mut(1)); + drop(ptr); + key.os.set(ptr::null_mut()); + }) { + rtabort!("thread local panicked on drop"); } - thread_local!(static FOO: RefCell> = map()); - - FOO.with(|map| { - assert_eq!(map.borrow()[&1], 2); - }); - } - - #[test] - fn refcell_vec() { - thread_local!(static FOO: RefCell> = RefCell::new(vec![1, 2, 3])); - - FOO.with(|vec| { - assert_eq!(vec.borrow().len(), 3); - vec.borrow_mut().push(4); - assert_eq!(vec.borrow()[3], 4); - }); } } diff --git a/crux-mir/lib/std/src/thread/local/dynamic_tests.rs b/crux-mir/lib/std/src/thread/local/dynamic_tests.rs new file mode 100644 index 000000000..dd1800416 --- /dev/null +++ b/crux-mir/lib/std/src/thread/local/dynamic_tests.rs @@ -0,0 +1,40 @@ +use crate::cell::RefCell; +use crate::collections::HashMap; +use crate::thread_local; + +#[test] +fn smoke() { + fn square(i: i32) -> i32 { + i * i + } + thread_local!(static FOO: i32 = square(3)); + + FOO.with(|f| { + assert_eq!(*f, 9); + }); +} + +#[test] +fn hashmap() { + fn map() -> RefCell> { + let mut m = HashMap::new(); + m.insert(1, 2); + RefCell::new(m) + } + thread_local!(static FOO: RefCell> = map()); + + FOO.with(|map| { + assert_eq!(map.borrow()[&1], 2); + }); +} + +#[test] +fn refcell_vec() { + thread_local!(static FOO: RefCell> = RefCell::new(vec![1, 2, 3])); + + FOO.with(|vec| { + assert_eq!(vec.borrow().len(), 3); + vec.borrow_mut().push(4); + assert_eq!(vec.borrow()[3], 4); + }); +} diff --git a/crux-mir/lib/std/src/thread/local/tests.rs b/crux-mir/lib/std/src/thread/local/tests.rs new file mode 100644 index 000000000..964c7fc5b --- /dev/null +++ b/crux-mir/lib/std/src/thread/local/tests.rs @@ -0,0 +1,339 @@ +use crate::cell::{Cell, UnsafeCell}; +use crate::sync::atomic::{AtomicU8, Ordering}; +use crate::sync::{Arc, Condvar, Mutex}; +use crate::thread::{self, LocalKey}; +use crate::thread_local; + +#[derive(Clone, Default)] +struct Signal(Arc<(Mutex, Condvar)>); + +impl Signal { + fn notify(&self) { + let (set, cvar) = &*self.0; + *set.lock().unwrap() = true; + cvar.notify_one(); + } + + fn wait(&self) { + let (set, cvar) = &*self.0; + let mut set = set.lock().unwrap(); + while !*set { + set = cvar.wait(set).unwrap(); + } + } +} + +struct NotifyOnDrop(Signal); + +impl Drop for NotifyOnDrop { + fn drop(&mut self) { + let NotifyOnDrop(ref f) = *self; + f.notify(); + } +} + +#[test] +fn smoke_no_dtor() { + thread_local!(static FOO: Cell = Cell::new(1)); + run(&FOO); + thread_local!(static FOO2: Cell = const { Cell::new(1) }); + run(&FOO2); + + fn run(key: &'static LocalKey>) { + key.with(|f| { + assert_eq!(f.get(), 1); + f.set(2); + }); + let t = thread::spawn(move || { + key.with(|f| { + assert_eq!(f.get(), 1); + }); + }); + t.join().unwrap(); + + key.with(|f| { + assert_eq!(f.get(), 2); + }); + } +} + +#[test] +fn states() { + struct Foo(&'static LocalKey); + impl Drop for Foo { + fn drop(&mut self) { + assert!(self.0.try_with(|_| ()).is_err()); + } + } + + thread_local!(static FOO: Foo = Foo(&FOO)); + run(&FOO); + thread_local!(static FOO2: Foo = const { Foo(&FOO2) }); + run(&FOO2); + + fn run(foo: &'static LocalKey) { + thread::spawn(move || { + assert!(foo.try_with(|_| ()).is_ok()); + }) + .join() + .unwrap(); + } +} + +#[test] +fn smoke_dtor() { + thread_local!(static FOO: UnsafeCell> = UnsafeCell::new(None)); + run(&FOO); + thread_local!(static FOO2: UnsafeCell> = const { UnsafeCell::new(None) }); + run(&FOO2); + + fn run(key: &'static LocalKey>>) { + let signal = Signal::default(); + let signal2 = signal.clone(); + let t = thread::spawn(move || unsafe { + let mut signal = Some(signal2); + key.with(|f| { + *f.get() = Some(NotifyOnDrop(signal.take().unwrap())); + }); + }); + signal.wait(); + t.join().unwrap(); + } +} + +#[test] +fn circular() { + struct S1(&'static LocalKey>>, &'static LocalKey>>); + struct S2(&'static LocalKey>>, &'static LocalKey>>); + thread_local!(static K1: UnsafeCell> = UnsafeCell::new(None)); + thread_local!(static K2: UnsafeCell> = UnsafeCell::new(None)); + thread_local!(static K3: UnsafeCell> = const { UnsafeCell::new(None) }); + thread_local!(static K4: UnsafeCell> = const { UnsafeCell::new(None) }); + static mut HITS: usize = 0; + + impl Drop for S1 { + fn drop(&mut self) { + unsafe { + HITS += 1; + if self.1.try_with(|_| ()).is_err() { + assert_eq!(HITS, 3); + } else { + if HITS == 1 { + self.1.with(|s| *s.get() = Some(S2(self.0, self.1))); + } else { + assert_eq!(HITS, 3); + } + } + } + } + } + impl Drop for S2 { + fn drop(&mut self) { + unsafe { + HITS += 1; + assert!(self.0.try_with(|_| ()).is_ok()); + assert_eq!(HITS, 2); + self.0.with(|s| *s.get() = Some(S1(self.0, self.1))); + } + } + } + + thread::spawn(move || { + drop(S1(&K1, &K2)); + }) + .join() + .unwrap(); + + unsafe { + HITS = 0; + } + + thread::spawn(move || { + drop(S1(&K3, &K4)); + }) + .join() + .unwrap(); +} + +#[test] +fn self_referential() { + struct S1(&'static LocalKey>>); + + thread_local!(static K1: UnsafeCell> = UnsafeCell::new(None)); + thread_local!(static K2: UnsafeCell> = const { UnsafeCell::new(None) }); + + impl Drop for S1 { + fn drop(&mut self) { + assert!(self.0.try_with(|_| ()).is_err()); + } + } + + thread::spawn(move || unsafe { + K1.with(|s| *s.get() = Some(S1(&K1))); + }) + .join() + .unwrap(); + + thread::spawn(move || unsafe { + K2.with(|s| *s.get() = Some(S1(&K2))); + }) + .join() + .unwrap(); +} + +// Note that this test will deadlock if TLS destructors aren't run (this +// requires the destructor to be run to pass the test). +#[test] +fn dtors_in_dtors_in_dtors() { + struct S1(Signal); + thread_local!(static K1: UnsafeCell> = UnsafeCell::new(None)); + thread_local!(static K2: UnsafeCell> = UnsafeCell::new(None)); + + impl Drop for S1 { + fn drop(&mut self) { + let S1(ref signal) = *self; + unsafe { + let _ = K2.try_with(|s| *s.get() = Some(NotifyOnDrop(signal.clone()))); + } + } + } + + let signal = Signal::default(); + let signal2 = signal.clone(); + let _t = thread::spawn(move || unsafe { + let mut signal = Some(signal2); + K1.with(|s| *s.get() = Some(S1(signal.take().unwrap()))); + }); + signal.wait(); +} + +#[test] +fn dtors_in_dtors_in_dtors_const_init() { + struct S1(Signal); + thread_local!(static K1: UnsafeCell> = const { UnsafeCell::new(None) }); + thread_local!(static K2: UnsafeCell> = const { UnsafeCell::new(None) }); + + impl Drop for S1 { + fn drop(&mut self) { + let S1(ref signal) = *self; + unsafe { + let _ = K2.try_with(|s| *s.get() = Some(NotifyOnDrop(signal.clone()))); + } + } + } + + let signal = Signal::default(); + let signal2 = signal.clone(); + let _t = thread::spawn(move || unsafe { + let mut signal = Some(signal2); + K1.with(|s| *s.get() = Some(S1(signal.take().unwrap()))); + }); + signal.wait(); +} + +// This test tests that TLS destructors have run before the thread joins. The +// test has no false positives (meaning: if the test fails, there's actually +// an ordering problem). It may have false negatives, where the test passes but +// join is not guaranteed to be after the TLS destructors. However, false +// negatives should be exceedingly rare due to judicious use of +// thread::yield_now and running the test several times. +#[test] +fn join_orders_after_tls_destructors() { + // We emulate a synchronous MPSC rendezvous channel using only atomics and + // thread::yield_now. We can't use std::mpsc as the implementation itself + // may rely on thread locals. + // + // The basic state machine for an SPSC rendezvous channel is: + // FRESH -> THREAD1_WAITING -> MAIN_THREAD_RENDEZVOUS + // where the first transition is done by the “receiving” thread and the 2nd + // transition is done by the “sending” thread. + // + // We add an additional state `THREAD2_LAUNCHED` between `FRESH` and + // `THREAD1_WAITING` to block until all threads are actually running. + // + // A thread that joins on the “receiving” thread completion should never + // observe the channel in the `THREAD1_WAITING` state. If this does occur, + // we switch to the “poison” state `THREAD2_JOINED` and panic all around. + // (This is equivalent to “sending” from an alternate producer thread.) + const FRESH: u8 = 0; + const THREAD2_LAUNCHED: u8 = 1; + const THREAD1_WAITING: u8 = 2; + const MAIN_THREAD_RENDEZVOUS: u8 = 3; + const THREAD2_JOINED: u8 = 4; + static SYNC_STATE: AtomicU8 = AtomicU8::new(FRESH); + + for _ in 0..10 { + SYNC_STATE.store(FRESH, Ordering::SeqCst); + + let jh = thread::Builder::new() + .name("thread1".into()) + .spawn(move || { + struct TlDrop; + + impl Drop for TlDrop { + fn drop(&mut self) { + let mut sync_state = SYNC_STATE.swap(THREAD1_WAITING, Ordering::SeqCst); + loop { + match sync_state { + THREAD2_LAUNCHED | THREAD1_WAITING => thread::yield_now(), + MAIN_THREAD_RENDEZVOUS => break, + THREAD2_JOINED => panic!( + "Thread 1 still running after thread 2 joined on thread 1" + ), + v => unreachable!("sync state: {}", v), + } + sync_state = SYNC_STATE.load(Ordering::SeqCst); + } + } + } + + thread_local! { + static TL_DROP: TlDrop = TlDrop; + } + + TL_DROP.with(|_| {}); + + loop { + match SYNC_STATE.load(Ordering::SeqCst) { + FRESH => thread::yield_now(), + THREAD2_LAUNCHED => break, + v => unreachable!("sync state: {}", v), + } + } + }) + .unwrap(); + + let jh2 = thread::Builder::new() + .name("thread2".into()) + .spawn(move || { + assert_eq!(SYNC_STATE.swap(THREAD2_LAUNCHED, Ordering::SeqCst), FRESH); + jh.join().unwrap(); + match SYNC_STATE.swap(THREAD2_JOINED, Ordering::SeqCst) { + MAIN_THREAD_RENDEZVOUS => return, + THREAD2_LAUNCHED | THREAD1_WAITING => { + panic!("Thread 2 running after thread 1 join before main thread rendezvous") + } + v => unreachable!("sync state: {:?}", v), + } + }) + .unwrap(); + + loop { + match SYNC_STATE.compare_exchange( + THREAD1_WAITING, + MAIN_THREAD_RENDEZVOUS, + Ordering::SeqCst, + Ordering::SeqCst, + ) { + Ok(_) => break, + Err(FRESH) => thread::yield_now(), + Err(THREAD2_LAUNCHED) => thread::yield_now(), + Err(THREAD2_JOINED) => { + panic!("Main thread rendezvous after thread 2 joined thread 1") + } + v => unreachable!("sync state: {:?}", v), + } + } + jh2.join().unwrap(); + } +} diff --git a/crux-mir/lib/std/src/thread/mod.rs b/crux-mir/lib/std/src/thread/mod.rs index 7a410c158..692ff0cbc 100644 --- a/crux-mir/lib/std/src/thread/mod.rs +++ b/crux-mir/lib/std/src/thread/mod.rs @@ -28,7 +28,7 @@ //! When the main thread of a Rust program terminates, the entire program shuts //! down, even if other threads are still running. However, this module provides //! convenient facilities for automatically waiting for the termination of a -//! child thread (i.e., join). +//! thread (i.e., join). //! //! ## Spawning a thread //! @@ -42,38 +42,43 @@ //! }); //! ``` //! -//! In this example, the spawned thread is "detached" from the current -//! thread. This means that it can outlive its parent (the thread that spawned -//! it), unless this parent is the main thread. +//! In this example, the spawned thread is "detached," which means that there is +//! no way for the program to learn when the spawned thread completes or otherwise +//! terminates. //! -//! The parent thread can also wait on the completion of the child -//! thread; a call to [`spawn`] produces a [`JoinHandle`], which provides -//! a `join` method for waiting: +//! To learn when a thread completes, it is necessary to capture the [`JoinHandle`] +//! object that is returned by the call to [`spawn`], which provides +//! a `join` method that allows the caller to wait for the completion of the +//! spawned thread: //! //! ```rust //! use std::thread; //! -//! let child = thread::spawn(move || { +//! let thread_join_handle = thread::spawn(move || { //! // some work here //! }); //! // some work here -//! let res = child.join(); +//! let res = thread_join_handle.join(); //! ``` //! //! The [`join`] method returns a [`thread::Result`] containing [`Ok`] of the final -//! value produced by the child thread, or [`Err`] of the value given to -//! a call to [`panic!`] if the child panicked. +//! value produced by the spawned thread, or [`Err`] of the value given to +//! a call to [`panic!`] if the thread panicked. +//! +//! Note that there is no parent/child relationship between a thread that spawns a +//! new thread and the thread being spawned. In particular, the spawned thread may or +//! may not outlive the spawning thread, unless the spawning thread is the main thread. //! //! ## Configuring threads //! //! A new thread can be configured before it is spawned via the [`Builder`] type, -//! which currently allows you to set the name and stack size for the child thread: +//! which currently allows you to set the name and stack size for the thread: //! //! ```rust //! # #![allow(unused_must_use)] //! use std::thread; //! -//! thread::Builder::new().name("child1".to_string()).spawn(move || { +//! thread::Builder::new().name("thread1".to_string()).spawn(move || { //! println!("Hello, world!"); //! }); //! ``` @@ -111,7 +116,7 @@ //! Threads are able to have associated names for identification purposes. By default, spawned //! threads are unnamed. To specify a name for a thread, build the thread with [`Builder`] and pass //! the desired thread name to [`Builder::name`]. To retrieve the thread name from within the -//! thread, use [`Thread::name`]. A couple examples of where the name of a thread gets used: +//! thread, use [`Thread::name`]. A couple of examples where the name of a thread gets used: //! //! * If a panic occurs in a named thread, the thread name will be printed in the panic message. //! * The thread name is provided to the OS where applicable (e.g., `pthread_setname_np` in @@ -119,9 +124,8 @@ //! //! ## Stack size //! -//! The default stack size for spawned threads is 2 MiB, though this particular stack size is -//! subject to change in the future. There are two ways to manually specify the stack size for -//! spawned threads: +//! The default stack size is platform-dependent and subject to change. Currently it is 2MB on all +//! Tier-1 platforms. There are two ways to manually specify the stack size for spawned threads: //! //! * Build the thread with [`Builder`] and pass the desired stack size to [`Builder::stack_size`]. //! * Set the `RUST_MIN_STACK` environment variable to an integer representing the desired stack @@ -129,54 +133,55 @@ //! //! Note that the stack size of the main thread is *not* determined by Rust. //! -//! [channels]: ../../std/sync/mpsc/index.html -//! [`Arc`]: ../../std/sync/struct.Arc.html -//! [`spawn`]: ../../std/thread/fn.spawn.html -//! [`JoinHandle`]: ../../std/thread/struct.JoinHandle.html -//! [`JoinHandle::thread`]: ../../std/thread/struct.JoinHandle.html#method.thread -//! [`join`]: ../../std/thread/struct.JoinHandle.html#method.join -//! [`Result`]: ../../std/result/enum.Result.html -//! [`Ok`]: ../../std/result/enum.Result.html#variant.Ok -//! [`Err`]: ../../std/result/enum.Result.html#variant.Err -//! [`panic!`]: ../../std/macro.panic.html -//! [`Builder`]: ../../std/thread/struct.Builder.html -//! [`Builder::stack_size`]: ../../std/thread/struct.Builder.html#method.stack_size -//! [`Builder::name`]: ../../std/thread/struct.Builder.html#method.name -//! [`thread::current`]: ../../std/thread/fn.current.html -//! [`thread::Result`]: ../../std/thread/type.Result.html -//! [`Thread`]: ../../std/thread/struct.Thread.html -//! [`park`]: ../../std/thread/fn.park.html -//! [`unpark`]: ../../std/thread/struct.Thread.html#method.unpark -//! [`Thread::name`]: ../../std/thread/struct.Thread.html#method.name -//! [`thread::park_timeout`]: ../../std/thread/fn.park_timeout.html -//! [`Cell`]: ../cell/struct.Cell.html -//! [`RefCell`]: ../cell/struct.RefCell.html -//! [`thread_local!`]: ../macro.thread_local.html -//! [`with`]: struct.LocalKey.html#method.with +//! [channels]: crate::sync::mpsc +//! [`join`]: JoinHandle::join +//! [`Result`]: crate::result::Result +//! [`Ok`]: crate::result::Result::Ok +//! [`Err`]: crate::result::Result::Err +//! [`thread::current`]: current +//! [`thread::Result`]: Result +//! [`unpark`]: Thread::unpark +//! [`thread::park_timeout`]: park_timeout +//! [`Cell`]: crate::cell::Cell +//! [`RefCell`]: crate::cell::RefCell +//! [`with`]: LocalKey::with +//! [`thread_local!`]: crate::thread_local #![stable(feature = "rust1", since = "1.0.0")] +#![deny(unsafe_op_in_unsafe_fn)] +// Under `test`, `__FastLocalKeyInner` seems unused. +#![cfg_attr(test, allow(dead_code))] + +#[cfg(all(test, not(target_os = "emscripten")))] +mod tests; use crate::any::Any; use crate::cell::UnsafeCell; use crate::ffi::{CStr, CString}; use crate::fmt; use crate::io; -use crate::mem; +use crate::marker::PhantomData; +use crate::mem::{self, forget}; use crate::num::NonZeroU64; +use crate::num::NonZeroUsize; use crate::panic; use crate::panicking; +use crate::pin::Pin; +use crate::ptr::addr_of_mut; use crate::str; -use crate::sync::atomic::AtomicUsize; -use crate::sync::atomic::Ordering::SeqCst; -use crate::sync::{Arc, Condvar, Mutex}; +use crate::sync::Arc; use crate::sys::thread as imp; -use crate::sys_common::mutex; use crate::sys_common::thread; use crate::sys_common::thread_info; +use crate::sys_common::thread_parking::Parker; use crate::sys_common::{AsInner, IntoInner}; use crate::time::Duration; -use crate::io::{Error,ErrorKind}; +#[stable(feature = "scoped_threads", since = "1.63.0")] +mod scoped; + +#[stable(feature = "scoped_threads", since = "1.63.0")] +pub use scoped::{scope, Scope, ScopedJoinHandle}; //////////////////////////////////////////////////////////////////////////////// // Thread-local storage @@ -188,29 +193,53 @@ mod local; #[stable(feature = "rust1", since = "1.0.0")] pub use self::local::{AccessError, LocalKey}; -// The types used by the thread_local! macro to access TLS keys. Note that there -// are two types, the "OS" type and the "fast" type. The OS thread local key -// type is accessed via platform-specific API calls and is slow, while the fast +// Provide the type used by the thread_local! macro to access TLS keys. This +// needs to be kept in sync with the macro itself (in `local.rs`). +// There are three types: "static", "fast", "OS". The "OS" thread local key +// type is accessed via platform-specific API calls and is slow, while the "fast" // key type is accessed via code generated via LLVM, where TLS keys are set up -// by the elf linker. Note that the OS TLS type is always available: on macOS -// the standard library is compiled with support for older platform versions -// where fast TLS was not available; end-user code is compiled with fast TLS -// where available, but both are needed. +// by the elf linker. "static" is for single-threaded platforms where a global +// static is sufficient. #[unstable(feature = "libstd_thread_internals", issue = "none")] -#[cfg(target_thread_local)] +#[cfg(not(test))] +#[cfg(all( + target_thread_local, + not(all(target_family = "wasm", not(target_feature = "atomics"))), +))] #[doc(hidden)] pub use self::local::fast::Key as __FastLocalKeyInner; +// when building for tests, use real std's type +#[unstable(feature = "libstd_thread_internals", issue = "none")] +#[cfg(test)] +#[cfg(all( + target_thread_local, + not(all(target_family = "wasm", not(target_feature = "atomics"))), +))] +pub use realstd::thread::__FastLocalKeyInner; + #[unstable(feature = "libstd_thread_internals", issue = "none")] +#[cfg(not(test))] +#[cfg(all( + not(target_thread_local), + not(all(target_family = "wasm", not(target_feature = "atomics"))), +))] #[doc(hidden)] pub use self::local::os::Key as __OsLocalKeyInner; +// when building for tests, use real std's type #[unstable(feature = "libstd_thread_internals", issue = "none")] -#[cfg(all(target_arch = "wasm32", not(target_feature = "atomics")))] +#[cfg(test)] +#[cfg(all( + not(target_thread_local), + not(all(target_family = "wasm", not(target_feature = "atomics"))), +))] +pub use realstd::thread::__OsLocalKeyInner; + +#[unstable(feature = "libstd_thread_internals", issue = "none")] +#[cfg(all(target_family = "wasm", not(target_feature = "atomics")))] #[doc(hidden)] pub use self::local::statik::Key as __StaticLocalKeyInner; -use core::crucible::concurrency; - //////////////////////////////////////////////////////////////////////////////// // Builder //////////////////////////////////////////////////////////////////////////////// @@ -249,14 +278,15 @@ use core::crucible::concurrency; /// handler.join().unwrap(); /// ``` /// -/// [`thread::spawn`]: ../../std/thread/fn.spawn.html -/// [`stack_size`]: ../../std/thread/struct.Builder.html#method.stack_size -/// [`name`]: ../../std/thread/struct.Builder.html#method.name -/// [`spawn`]: ../../std/thread/struct.Builder.html#method.spawn -/// [`io::Result`]: ../../std/io/type.Result.html -/// [`unwrap`]: ../../std/result/enum.Result.html#method.unwrap +/// [`stack_size`]: Builder::stack_size +/// [`name`]: Builder::name +/// [`spawn`]: Builder::spawn +/// [`thread::spawn`]: spawn +/// [`io::Result`]: crate::io::Result +/// [`unwrap`]: crate::result::Result::unwrap /// [naming-threads]: ./index.html#naming-threads /// [stack-size]: ./index.html#stack-size +#[must_use = "must eventually spawn the thread"] #[stable(feature = "rust1", since = "1.0.0")] #[derive(Debug)] pub struct Builder { @@ -349,7 +379,7 @@ impl Builder { /// The spawned thread may outlive the caller (unless the caller thread /// is the main thread; the whole process is terminated when the main /// thread finishes). The join handle can be used to block on - /// termination of the child thread, including recovering its panics. + /// termination of the spawned thread, including recovering its panics. /// /// For a more complete documentation see [`thread::spawn`][`spawn`]. /// @@ -359,9 +389,7 @@ impl Builder { /// [`io::Result`] to capture any failure to create the thread at /// the OS level. /// - /// [`spawn`]: ../../std/thread/fn.spawn.html - /// [`io::Result`]: ../../std/io/type.Result.html - /// [`JoinHandle`]: ../../std/thread/struct.JoinHandle.html + /// [`io::Result`]: crate::io::Result /// /// # Panics /// @@ -387,8 +415,7 @@ impl Builder { F: Send + 'static, T: Send + 'static, { - panic!("Not implemented!") - // unsafe { self.spawn_unchecked(f) } + unsafe { self.spawn_unchecked(f) } } /// Spawns a new thread without any lifetime restrictions by taking ownership @@ -397,7 +424,7 @@ impl Builder { /// The spawned thread may outlive the caller (unless the caller thread /// is the main thread; the whole process is terminated when the main /// thread finishes). The join handle can be used to block on - /// termination of the child thread, including recovering its panics. + /// termination of the spawned thread, including recovering its panics. /// /// This method is identical to [`thread::Builder::spawn`][`Builder::spawn`], /// except for the relaxed lifetime bounds, which render it unsafe. @@ -415,9 +442,9 @@ impl Builder { /// /// # Safety /// - /// The caller has to ensure that no references in the supplied thread closure - /// or its return type can outlive the spawned thread's lifetime. This can be - /// guaranteed in two ways: + /// The caller has to ensure that the spawned thread does not outlive any + /// references in the supplied thread closure and its return type. + /// This can be guaranteed in two ways: /// /// - ensure that [`join`][`JoinHandle::join`] is called before any referenced /// data is dropped @@ -448,11 +475,7 @@ impl Builder { /// handler.join().unwrap(); /// ``` /// - /// [`spawn`]: ../../std/thread/fn.spawn.html - /// [`Builder::spawn`]: ../../std/thread/struct.Builder.html#method.spawn - /// [`io::Result`]: ../../std/io/type.Result.html - /// [`JoinHandle`]: ../../std/thread/struct.JoinHandle.html - /// [`JoinHandle::join`]: ../../std/thread/struct.JoinHandle.html#method.join + /// [`io::Result`]: crate::io::Result #[unstable(feature = "thread_spawn_unchecked", issue = "55132")] pub unsafe fn spawn_unchecked<'a, F, T>(self, f: F) -> io::Result> where @@ -460,50 +483,122 @@ impl Builder { F: Send + 'a, T: Send + 'a, { - panic!("spawn_unchecked: Not implemented!") - // let Builder { name, stack_size } = self; - - // let stack_size = stack_size.unwrap_or_else(thread::min_stack); - - // let my_thread = Thread::new(name); - // let their_thread = my_thread.clone(); - - // let my_packet: Arc>>> = Arc::new(UnsafeCell::new(None)); - // let their_packet = my_packet.clone(); - - // let main = move || { - // if let Some(name) = their_thread.cname() { - // imp::Thread::set_name(name); - // } - - // thread_info::set(imp::guard::current(), their_thread); - // let try_result = panic::catch_unwind(panic::AssertUnwindSafe(|| { - // crate::sys_common::backtrace::__rust_begin_short_backtrace(f) - // })); - // *their_packet.get() = Some(try_result); - // }; - - // Ok(JoinHandle(JoinInner { - // // `imp::Thread::new` takes a closure with a `'static` lifetime, since it's passed - // // through FFI or otherwise used with low-level threading primitives that have no - // // notion of or way to enforce lifetimes. - // // - // // As mentioned in the `Safety` section of this function's documentation, the caller of - // // this function needs to guarantee that the passed-in lifetime is sufficiently long - // // for the lifetime of the thread. - // // - // // Similarly, the `sys` implementation must guarantee that no references to the closure - // // exist after the thread has terminated, which is signaled by `Thread::join` - // // returning. - // native: Some(imp::Thread::new( - // stack_size, - // mem::transmute::, Box>(Box::new( - // main, - // )), - // )?), - // thread: my_thread, - // packet: Packet(my_packet), - // })) + Ok(JoinHandle(unsafe { self.spawn_unchecked_(f, None) }?)) + } + + unsafe fn spawn_unchecked_<'a, 'scope, F, T>( + self, + f: F, + scope_data: Option>, + ) -> io::Result> + where + F: FnOnce() -> T, + F: Send + 'a, + T: Send + 'a, + 'scope: 'a, + { + let Builder { name, stack_size } = self; + + let stack_size = stack_size.unwrap_or_else(thread::min_stack); + + let my_thread = Thread::new(name.map(|name| { + CString::new(name).expect("thread name may not contain interior null bytes") + })); + let their_thread = my_thread.clone(); + + let my_packet: Arc> = Arc::new(Packet { + scope: scope_data, + result: UnsafeCell::new(None), + _marker: PhantomData, + }); + let their_packet = my_packet.clone(); + + let output_capture = crate::io::set_output_capture(None); + crate::io::set_output_capture(output_capture.clone()); + + // Pass `f` in `MaybeUninit` because actually that closure might *run longer than the lifetime of `F`*. + // See for more details. + // To prevent leaks we use a wrapper that drops its contents. + #[repr(transparent)] + struct MaybeDangling(mem::MaybeUninit); + impl MaybeDangling { + fn new(x: T) -> Self { + MaybeDangling(mem::MaybeUninit::new(x)) + } + fn into_inner(self) -> T { + // SAFETY: we are always initiailized. + let ret = unsafe { self.0.assume_init_read() }; + // Make sure we don't drop. + mem::forget(self); + ret + } + } + impl Drop for MaybeDangling { + fn drop(&mut self) { + // SAFETY: we are always initiailized. + unsafe { self.0.assume_init_drop() }; + } + } + + let f = MaybeDangling::new(f); + let main = move || { + if let Some(name) = their_thread.cname() { + imp::Thread::set_name(name); + } + + crate::io::set_output_capture(output_capture); + + // SAFETY: we constructed `f` initialized. + let f = f.into_inner(); + // SAFETY: the stack guard passed is the one for the current thread. + // This means the current thread's stack and the new thread's stack + // are properly set and protected from each other. + thread_info::set(unsafe { imp::guard::current() }, their_thread); + let try_result = panic::catch_unwind(panic::AssertUnwindSafe(|| { + crate::sys_common::backtrace::__rust_begin_short_backtrace(f) + })); + // SAFETY: `their_packet` as been built just above and moved by the + // closure (it is an Arc<...>) and `my_packet` will be stored in the + // same `JoinInner` as this closure meaning the mutation will be + // safe (not modify it and affect a value far away). + unsafe { *their_packet.result.get() = Some(try_result) }; + // Here `their_packet` gets dropped, and if this is the last `Arc` for that packet that + // will call `decrement_num_running_threads` and therefore signal that this thread is + // done. + drop(their_packet); + // Here, the lifetime `'a` and even `'scope` can end. `main` keeps running for a bit + // after that before returning itself. + }; + + if let Some(scope_data) = &my_packet.scope { + scope_data.increment_num_running_threads(); + } + + Ok(JoinInner { + // SAFETY: + // + // `imp::Thread::new` takes a closure with a `'static` lifetime, since it's passed + // through FFI or otherwise used with low-level threading primitives that have no + // notion of or way to enforce lifetimes. + // + // As mentioned in the `Safety` section of this function's documentation, the caller of + // this function needs to guarantee that the passed-in lifetime is sufficiently long + // for the lifetime of the thread. + // + // Similarly, the `sys` implementation must guarantee that no references to the closure + // exist after the thread has terminated, which is signaled by `Thread::join` + // returning. + native: unsafe { + imp::Thread::new( + stack_size, + mem::transmute::, Box>( + Box::new(main), + ), + )? + }, + thread: my_thread, + packet: my_packet, + }) } } @@ -513,15 +608,16 @@ impl Builder { /// Spawns a new thread, returning a [`JoinHandle`] for it. /// -/// The join handle will implicitly *detach* the child thread upon being -/// dropped. In this case, the child thread may outlive the parent (unless -/// the parent thread is the main thread; the whole process is terminated when -/// the main thread finishes). Additionally, the join handle provides a [`join`] -/// method that can be used to join the child thread. If the child thread -/// panics, [`join`] will return an [`Err`] containing the argument given to -/// [`panic`]. +/// The join handle provides a [`join`] method that can be used to join the spawned +/// thread. If the spawned thread panics, [`join`] will return an [`Err`] containing +/// the argument given to [`panic!`]. +/// +/// If the join handle is dropped, the spawned thread will implicitly be *detached*. +/// In this case, the spawned thread may no longer be joined. +/// (It is the responsibility of the program to either eventually join threads it +/// creates or detach them; otherwise, a resource leak will result.) /// -/// This will create a thread using default parameters of [`Builder`], if you +/// This call will create a thread using default parameters of [`Builder`], if you /// want to specify the stack size or the name of the thread, use this API /// instead. /// @@ -530,8 +626,8 @@ impl Builder { /// /// - The `'static` constraint means that the closure and its return value /// must have a lifetime of the whole program execution. The reason for this -/// is that threads can `detach` and outlive the lifetime they have been -/// created in. +/// is that threads can outlive the lifetime they have been created in. +/// /// Indeed if the thread, and by extension its return value, can outlive their /// caller, we need to make sure that they will be valid afterwards, and since /// we *can't* know when it will return we need to have them valid as long as @@ -583,7 +679,7 @@ impl Builder { /// /// let receiver = thread::spawn(move || { /// let value = rx.recv().expect("Unable to receive from channel"); -/// println!("{}", value); +/// println!("{value}"); /// }); /// /// sender.join().expect("The sender thread has panicked"); @@ -603,19 +699,12 @@ impl Builder { /// }); /// /// let result = computation.join().unwrap(); -/// println!("{}", result); +/// println!("{result}"); /// ``` /// -/// [`channels`]: ../../std/sync/mpsc/index.html -/// [`JoinHandle`]: ../../std/thread/struct.JoinHandle.html -/// [`join`]: ../../std/thread/struct.JoinHandle.html#method.join -/// [`Err`]: ../../std/result/enum.Result.html#variant.Err -/// [`panic`]: ../../std/macro.panic.html -/// [`Builder::spawn`]: ../../std/thread/struct.Builder.html#method.spawn -/// [`Builder`]: ../../std/thread/struct.Builder.html -/// [`Send`]: ../../std/marker/trait.Send.html -/// [`Sync`]: ../../std/marker/trait.Sync.html - +/// [`channels`]: crate::sync::mpsc +/// [`join`]: JoinHandle::join +/// [`Err`]: crate::result::Result::Err #[stable(feature = "rust1", since = "1.0.0")] pub fn spawn(f: F) -> JoinHandle where @@ -623,23 +712,7 @@ where F: Send + 'static, T: Send + 'static, { - let id = concurrency::spawn(f); - let t = - Thread { - inner: Arc::new(Inner { - name: None, - id: ThreadId(NonZeroU64::new(id.into()).unwrap()), - state: AtomicUsize::new(EMPTY), - lock: Mutex::new(()), - cvar: Condvar::new(), - }), - }; - JoinHandle( - JoinInner { - thread: t, - phantom: core::marker::PhantomData, - } - ) + Builder::new().spawn(f).expect("failed to spawn thread") } /// Gets a handle to the thread that invokes it. @@ -661,33 +734,34 @@ where /// /// handler.join().unwrap(); /// ``` +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn current() -> Thread { thread_info::current_thread().expect( - "use of std::thread::current() is not \ - possible after the thread's local \ - data has been destroyed", + "use of std::thread::current() is not possible \ + after the thread's local data has been destroyed", ) } /// Cooperatively gives up a timeslice to the OS scheduler. /// -/// This is used when the programmer knows that the thread will have nothing -/// to do for some time, and thus avoid wasting computing time. -/// -/// For example when polling on a resource, it is common to check that it is -/// available, and if not to yield in order to avoid busy waiting. +/// This calls the underlying OS scheduler's yield primitive, signaling +/// that the calling thread is willing to give up its remaining timeslice +/// so that the OS may schedule other threads on the CPU. /// -/// Thus the pattern of `yield`ing after a failed poll is rather common when -/// implementing low-level shared resources or synchronization primitives. +/// A drawback of yielding in a loop is that if the OS does not have any +/// other ready threads to run on the current CPU, the thread will effectively +/// busy-wait, which wastes CPU time and energy. /// -/// However programmers will usually prefer to use [`channel`]s, [`Condvar`]s, -/// [`Mutex`]es or [`join`] for their synchronization routines, as they avoid -/// thinking about thread scheduling. +/// Therefore, when waiting for events of interest, a programmer's first +/// choice should be to use synchronization devices such as [`channel`]s, +/// [`Condvar`]s, [`Mutex`]es or [`join`] since these primitives are +/// implemented in a blocking manner, giving up the CPU until the event +/// of interest has occurred which avoids repeated yielding. /// -/// Note that [`channel`]s for example are implemented using this primitive. -/// Indeed when you call `send` or `recv`, which are blocking, they will yield -/// if the channel is not available. +/// `yield_now` should thus be used only rarely, mostly in situations where +/// repeated polling is required because there is no other suitable way to +/// learn when an event of interest has occurred. /// /// # Examples /// @@ -697,11 +771,10 @@ pub fn current() -> Thread { /// thread::yield_now(); /// ``` /// -/// [`channel`]: ../../std/sync/mpsc/index.html -/// [`spawn`]: ../../std/thread/fn.spawn.html -/// [`join`]: ../../std/thread/struct.JoinHandle.html#method.join -/// [`Mutex`]: ../../std/sync/struct.Mutex.html -/// [`Condvar`]: ../../std/sync/struct.Condvar.html +/// [`channel`]: crate::sync::mpsc +/// [`join`]: JoinHandle::join +/// [`Condvar`]: crate::sync::Condvar +/// [`Mutex`]: crate::sync::Mutex #[stable(feature = "rust1", since = "1.0.0")] pub fn yield_now() { imp::Thread::yield_now() @@ -748,18 +821,23 @@ pub fn yield_now() { /// } /// ``` /// -/// [Mutex]: ../../std/sync/struct.Mutex.html +/// [Mutex]: crate::sync::Mutex #[inline] +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn panicking() -> bool { panicking::panicking() } +/// Use [`sleep`]. +/// /// Puts the current thread to sleep for at least the specified amount of time. /// /// The thread may sleep longer than the duration specified due to scheduling /// specifics or platform-dependent functionality. It will never sleep less. /// +/// This function is blocking, and should not be used in `async` functions. +/// /// # Platform-specific behavior /// /// On Unix platforms, the underlying syscall may be interrupted by a @@ -776,7 +854,7 @@ pub fn panicking() -> bool { /// thread::sleep_ms(2000); /// ``` #[stable(feature = "rust1", since = "1.0.0")] -#[rustc_deprecated(since = "1.6.0", reason = "replaced by `std::thread::sleep`")] +#[deprecated(since = "1.6.0", note = "replaced by `std::thread::sleep`")] pub fn sleep_ms(ms: u32) { sleep(Duration::from_millis(ms as u64)) } @@ -786,6 +864,8 @@ pub fn sleep_ms(ms: u32) { /// The thread may sleep longer than the duration specified due to scheduling /// specifics or platform-dependent functionality. It will never sleep less. /// +/// This function is blocking, and should not be used in `async` functions. +/// /// # Platform-specific behavior /// /// On Unix platforms, the underlying syscall may be interrupted by a @@ -795,6 +875,15 @@ pub fn sleep_ms(ms: u32) { /// Platforms which do not support nanosecond precision for sleeping will /// have `dur` rounded up to the nearest granularity of time they can sleep for. /// +/// Currently, specifying a zero duration on Unix platforms returns immediately +/// without invoking the underlying [`nanosleep`] syscall, whereas on Windows +/// platforms the underlying [`Sleep`] syscall is always invoked. +/// If the intention is to yield the current time-slice you may want to use +/// [`yield_now`] instead. +/// +/// [`nanosleep`]: https://linux.die.net/man/2/nanosleep +/// [`Sleep`]: https://docs.microsoft.com/en-us/windows/win32/api/synchapi/nf-synchapi-sleep +/// /// # Examples /// /// ```no_run @@ -812,15 +901,22 @@ pub fn sleep(dur: Duration) { imp::Thread::sleep(dur) } -// constants for park/unpark -const EMPTY: usize = 0; -const PARKED: usize = 1; -const NOTIFIED: usize = 2; +/// Used to ensure that `park` and `park_timeout` do not unwind, as that can +/// cause undefined behaviour if not handled correctly (see #102398 for context). +struct PanicGuard; + +impl Drop for PanicGuard { + fn drop(&mut self) { + rtabort!("an irrecoverable error occurred while synchronizing threads") + } +} /// Blocks unless or until the current thread's token is made available. /// /// A call to `park` does not guarantee that the thread will remain parked -/// forever, and callers should be prepared for this possibility. +/// forever, and callers should be prepared for this possibility. However, +/// it is guaranteed that this function will not panic (it may abort the +/// process if the implementation encounters some rare errors). /// /// # park and unpark /// @@ -901,50 +997,17 @@ const NOTIFIED: usize = 2; /// parked_thread.join().unwrap(); /// ``` /// -/// [`Thread`]: ../../std/thread/struct.Thread.html -/// [`park`]: ../../std/thread/fn.park.html -/// [`unpark`]: ../../std/thread/struct.Thread.html#method.unpark -/// [`thread::park_timeout`]: ../../std/thread/fn.park_timeout.html -// -// The implementation currently uses the trivial strategy of a Mutex+Condvar -// with wakeup flag, which does not actually allow spurious wakeups. In the -// future, this will be implemented in a more efficient way, perhaps along the lines of -// http://cr.openjdk.java.net/~stefank/6989984.1/raw_files/new/src/os/linux/vm/os_linux.cpp -// or futuxes, and in either case may allow spurious wakeups. +/// [`unpark`]: Thread::unpark +/// [`thread::park_timeout`]: park_timeout #[stable(feature = "rust1", since = "1.0.0")] pub fn park() { - let thread = current(); - - // If we were previously notified then we consume this notification and - // return quickly. - if thread.inner.state.compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst).is_ok() { - return; - } - - // Otherwise we need to coordinate going to sleep - let mut m = thread.inner.lock.lock().unwrap(); - match thread.inner.state.compare_exchange(EMPTY, PARKED, SeqCst, SeqCst) { - Ok(_) => {} - Err(NOTIFIED) => { - // We must read here, even though we know it will be `NOTIFIED`. - // This is because `unpark` may have been called again since we read - // `NOTIFIED` in the `compare_exchange` above. We must perform an - // acquire operation that synchronizes with that `unpark` to observe - // any writes it made before the call to unpark. To do that we must - // read from the write it made to `state`. - let old = thread.inner.state.swap(EMPTY, SeqCst); - assert_eq!(old, NOTIFIED, "park state changed unexpectedly"); - return; - } // should consume this notification, so prohibit spurious wakeups in next park. - Err(_) => panic!("inconsistent park state"), - } - loop { - m = thread.inner.cvar.wait(m).unwrap(); - match thread.inner.state.compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) { - Ok(_) => return, // got a notification - Err(_) => {} // spurious wakeup, go back to sleep - } + let guard = PanicGuard; + // SAFETY: park_timeout is called on the parker owned by this thread. + unsafe { + current().inner.as_ref().parker().park(); } + // No panic occurred, do not abort. + forget(guard); } /// Use [`park_timeout`]. @@ -955,15 +1018,12 @@ pub fn park() { /// The semantics of this function are equivalent to [`park`] except /// that the thread will be blocked for roughly no longer than `dur`. This /// method should not be used for precise timing due to anomalies such as -/// preemption or platform differences that may not cause the maximum +/// preemption or platform differences that might not cause the maximum /// amount of time waited to be precisely `ms` long. /// /// See the [park documentation][`park`] for more detail. -/// -/// [`park_timeout`]: fn.park_timeout.html -/// [`park`]: ../../std/thread/fn.park.html #[stable(feature = "rust1", since = "1.0.0")] -#[rustc_deprecated(since = "1.6.0", reason = "replaced by `std::thread::park_timeout`")] +#[deprecated(since = "1.6.0", note = "replaced by `std::thread::park_timeout`")] pub fn park_timeout_ms(ms: u32) { park_timeout(Duration::from_millis(ms as u64)) } @@ -974,7 +1034,7 @@ pub fn park_timeout_ms(ms: u32) { /// The semantics of this function are equivalent to [`park`][park] except /// that the thread will be blocked for roughly no longer than `dur`. This /// method should not be used for precise timing due to anomalies such as -/// preemption or platform differences that may not cause the maximum +/// preemption or platform differences that might not cause the maximum /// amount of time waited to be precisely `dur` long. /// /// See the [park documentation][park] for more details. @@ -1002,44 +1062,19 @@ pub fn park_timeout_ms(ms: u32) { /// if elapsed >= timeout { /// break; /// } -/// println!("restarting park_timeout after {:?}", elapsed); +/// println!("restarting park_timeout after {elapsed:?}"); /// timeout_remaining = timeout - elapsed; /// } /// ``` -/// -/// [park]: fn.park.html #[stable(feature = "park_timeout", since = "1.4.0")] pub fn park_timeout(dur: Duration) { - let thread = current(); - - // Like `park` above we have a fast path for an already-notified thread, and - // afterwards we start coordinating for a sleep. - // return quickly. - if thread.inner.state.compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst).is_ok() { - return; - } - let m = thread.inner.lock.lock().unwrap(); - match thread.inner.state.compare_exchange(EMPTY, PARKED, SeqCst, SeqCst) { - Ok(_) => {} - Err(NOTIFIED) => { - // We must read again here, see `park`. - let old = thread.inner.state.swap(EMPTY, SeqCst); - assert_eq!(old, NOTIFIED, "park state changed unexpectedly"); - return; - } // should consume this notification, so prohibit spurious wakeups in next park. - Err(_) => panic!("inconsistent park_timeout state"), - } - - // Wait with a timeout, and if we spuriously wake up or otherwise wake up - // from a notification we just want to unconditionally set the state back to - // empty, either consuming a notification or un-flagging ourselves as - // parked. - let (_m, _result) = thread.inner.cvar.wait_timeout(m, dur).unwrap(); - match thread.inner.state.swap(EMPTY, SeqCst) { - NOTIFIED => {} // got a notification, hurray! - PARKED => {} // no notification, alas - n => panic!("inconsistent park_timeout state: {}", n), + let guard = PanicGuard; + // SAFETY: park_timeout is called on the parker owned by this thread. + unsafe { + current().inner.as_ref().parker().park_timeout(dur); } + // No panic occurred, do not abort. + forget(guard); } //////////////////////////////////////////////////////////////////////////////// @@ -1048,10 +1083,13 @@ pub fn park_timeout(dur: Duration) { /// A unique identifier for a running thread. /// -/// A `ThreadId` is an opaque object that has a unique value for each thread -/// that creates one. `ThreadId`s are not guaranteed to correspond to a thread's -/// system-designated identifier. A `ThreadId` can be retrieved from the [`id`] -/// method on a [`Thread`]. +/// A `ThreadId` is an opaque object that uniquely identifies each thread +/// created during the lifetime of a process. `ThreadId`s are guaranteed not to +/// be reused, even when a thread terminates. `ThreadId`s are under the control +/// of Rust's standard library and there may not be any relationship between +/// `ThreadId` and the underlying platform's notion of a thread identifier -- +/// the two concepts cannot, therefore, be used interchangeably. A `ThreadId` +/// can be retrieved from the [`id`] method on a [`Thread`]. /// /// # Examples /// @@ -1066,8 +1104,7 @@ pub fn park_timeout(dur: Duration) { /// assert!(thread::current().id() != other_thread_id); /// ``` /// -/// [`id`]: ../../std/thread/struct.Thread.html#method.id -/// [`Thread`]: ../../std/thread/struct.Thread.html +/// [`id`]: Thread::id #[stable(feature = "thread_id", since = "1.19.0")] #[derive(Eq, PartialEq, Clone, Copy, Hash, Debug)] pub struct ThreadId(NonZeroU64); @@ -1075,24 +1112,45 @@ pub struct ThreadId(NonZeroU64); impl ThreadId { // Generate a new unique thread ID. fn new() -> ThreadId { - // We never call `GUARD.init()`, so it is UB to attempt to - // acquire this mutex reentrantly! - static GUARD: mutex::Mutex = mutex::Mutex::new(); - static mut COUNTER: u64 = 1; - - unsafe { - let _guard = GUARD.lock(); - - // If we somehow use up all our bits, panic so that we're not - // covering up subtle bugs of IDs being reused. - if COUNTER == crate::u64::MAX { - panic!("failed to generate unique thread ID: bitspace exhausted"); - } + #[cold] + fn exhausted() -> ! { + panic!("failed to generate unique thread ID: bitspace exhausted") + } + + cfg_if::cfg_if! { + if #[cfg(target_has_atomic = "64")] { + use crate::sync::atomic::{AtomicU64, Ordering::Relaxed}; + + static COUNTER: AtomicU64 = AtomicU64::new(0); - let id = COUNTER; - COUNTER += 1; + let mut last = COUNTER.load(Relaxed); + loop { + let Some(id) = last.checked_add(1) else { + exhausted(); + }; - ThreadId(NonZeroU64::new(id).unwrap()) + match COUNTER.compare_exchange_weak(last, id, Relaxed, Relaxed) { + Ok(_) => return ThreadId(NonZeroU64::new(id).unwrap()), + Err(id) => last = id, + } + } + } else { + use crate::sync::{Mutex, PoisonError}; + + static COUNTER: Mutex = Mutex::new(0); + + let mut counter = COUNTER.lock().unwrap_or_else(PoisonError::into_inner); + let Some(id) = counter.checked_add(1) else { + // in case the panic handler ends up calling `ThreadId::new()`, + // avoid reentrant lock acquire. + drop(counter); + exhausted(); + }; + + *counter = id; + drop(counter); + ThreadId(NonZeroU64::new(id).unwrap()) + } } } @@ -1104,9 +1162,10 @@ impl ThreadId { /// value is entirely opaque -- only equality testing is stable. Note that /// it is not guaranteed which values new threads will return, and this may /// change across Rust versions. + #[must_use] #[unstable(feature = "thread_id_value", issue = "67939")] - pub fn as_u64(&self) -> u64 { - self.0.get() + pub fn as_u64(&self) -> NonZeroU64 { + self.0 } } @@ -1118,11 +1177,13 @@ impl ThreadId { struct Inner { name: Option, // Guaranteed to be UTF-8 id: ThreadId, + parker: Parker, +} - // state for thread park/unpark - state: AtomicUsize, - lock: Mutex<()>, - cvar: Condvar, +impl Inner { + fn parker(self: Pin<&Self>) -> Pin<&Parker> { + unsafe { Pin::map_unchecked(self, |inner| &inner.parker) } + } } #[derive(Clone)] @@ -1144,31 +1205,30 @@ struct Inner { /// should instead use a function like `spawn` to create new threads, see the /// docs of [`Builder`] and [`spawn`] for more details. /// -/// [`Builder`]: ../../std/thread/struct.Builder.html -/// [`JoinHandle::thread`]: ../../std/thread/struct.JoinHandle.html#method.thread -/// [`JoinHandle`]: ../../std/thread/struct.JoinHandle.html -/// [`thread::current`]: ../../std/thread/fn.current.html -/// [`spawn`]: ../../std/thread/fn.spawn.html - +/// [`thread::current`]: current pub struct Thread { - inner: Arc, + inner: Pin>, } impl Thread { // Used only internally to construct a thread object without spawning // Panics if the name contains nuls. - pub(crate) fn new(name: Option) -> Thread { - let cname = - name.map(|n| CString::new(n).expect("thread name may not contain interior null bytes")); - Thread { - inner: Arc::new(Inner { - name: cname, - id: ThreadId::new(), - state: AtomicUsize::new(EMPTY), - lock: Mutex::new(()), - cvar: Condvar::new(), - }), - } + pub(crate) fn new(name: Option) -> Thread { + // We have to use `unsafe` here to construct the `Parker` in-place, + // which is required for the UNIX implementation. + // + // SAFETY: We pin the Arc immediately after creation, so its address never + // changes. + let inner = unsafe { + let mut arc = Arc::::new_uninit(); + let ptr = Arc::get_mut_unchecked(&mut arc).as_mut_ptr(); + addr_of_mut!((*ptr).name).write(name); + addr_of_mut!((*ptr).id).write(ThreadId::new()); + Parker::new_in_place(addr_of_mut!((*ptr).parker)); + Pin::new_unchecked(arc.assume_init()) + }; + + Thread { inner } } /// Atomically makes the handle's token available if it is not already. @@ -1201,36 +1261,10 @@ impl Thread { /// /// parked_thread.join().unwrap(); /// ``` - /// - /// [park]: fn.park.html #[stable(feature = "rust1", since = "1.0.0")] + #[inline] pub fn unpark(&self) { - // To ensure the unparked thread will observe any writes we made - // before this call, we must perform a release operation that `park` - // can synchronize with. To do that we must write `NOTIFIED` even if - // `state` is already `NOTIFIED`. That is why this must be a swap - // rather than a compare-and-swap that returns if it reads `NOTIFIED` - // on failure. - match self.inner.state.swap(NOTIFIED, SeqCst) { - EMPTY => return, // no one was waiting - NOTIFIED => return, // already unparked - PARKED => {} // gotta go wake someone up - _ => panic!("inconsistent state in unpark"), - } - - // There is a period between when the parked thread sets `state` to - // `PARKED` (or last checked `state` in the case of a spurious wake - // up) and when it actually waits on `cvar`. If we were to notify - // during this period it would be ignored and then when the parked - // thread went to sleep it would never wake up. Fortunately, it has - // `lock` locked at this stage so we can acquire `lock` to wait until - // it is ready to receive the notification. - // - // Releasing `lock` before the call to `notify_one` means that when the - // parked thread wakes it doesn't get woken only to have to wait for us - // to release `lock`. - drop(self.inner.lock.lock().unwrap()); - self.inner.cvar.notify_one() + self.inner.as_ref().parker().unpark(); } /// Gets the thread's unique identifier. @@ -1248,6 +1282,7 @@ impl Thread { /// assert!(thread::current().id() != other_thread_id); /// ``` #[stable(feature = "thread_id", since = "1.19.0")] + #[must_use] pub fn id(&self) -> ThreadId { self.inner.id } @@ -1290,19 +1325,23 @@ impl Thread { /// /// [naming-threads]: ./index.html#naming-threads #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] pub fn name(&self) -> Option<&str> { self.cname().map(|s| unsafe { str::from_utf8_unchecked(s.to_bytes()) }) } fn cname(&self) -> Option<&CStr> { - self.inner.name.as_ref().map(|s| &**s) + self.inner.name.as_deref() } } #[stable(feature = "rust1", since = "1.0.0")] impl fmt::Debug for Thread { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Thread").field("id", &self.id()).field("name", &self.name()).finish() + f.debug_struct("Thread") + .field("id", &self.id()) + .field("name", &self.name()) + .finish_non_exhaustive() } } @@ -1321,73 +1360,109 @@ impl fmt::Debug for Thread { /// the [`Error`](crate::error::Error) trait. /// /// Thus, a sensible way to handle a thread panic is to either: -/// 1. `unwrap` the `Result`, propagating the panic +/// +/// 1. propagate the panic with [`std::panic::resume_unwind`] /// 2. or in case the thread is intended to be a subsystem boundary /// that is supposed to isolate system-level failures, -/// match on the `Err` variant and handle the panic in an appropriate way. +/// match on the `Err` variant and handle the panic in an appropriate way /// /// A thread that completes without panicking is considered to exit successfully. /// /// # Examples /// +/// Matching on the result of a joined thread: +/// /// ```no_run -/// use std::thread; -/// use std::fs; +/// use std::{fs, thread, panic}; /// /// fn copy_in_thread() -> thread::Result<()> { -/// thread::spawn(move || { fs::copy("foo.txt", "bar.txt").unwrap(); }).join() +/// thread::spawn(|| { +/// fs::copy("foo.txt", "bar.txt").unwrap(); +/// }).join() /// } /// /// fn main() { /// match copy_in_thread() { -/// Ok(_) => println!("this is fine"), -/// Err(_) => println!("thread panicked"), +/// Ok(_) => println!("copy succeeded"), +/// Err(e) => panic::resume_unwind(e), /// } /// } /// ``` /// -/// [`Result`]: ../../std/result/enum.Result.html +/// [`Result`]: crate::result::Result +/// [`std::panic::resume_unwind`]: crate::panic::resume_unwind #[stable(feature = "rust1", since = "1.0.0")] pub type Result = crate::result::Result>; -// This packet is used to communicate the return value between the child thread -// and the parent thread. Memory is shared through the `Arc` within and there's -// no need for a mutex here because synchronization happens with `join()` (the -// parent thread never reads this packet until the child has exited). +// This packet is used to communicate the return value between the spawned +// thread and the rest of the program. It is shared through an `Arc` and +// there's no need for a mutex here because synchronization happens with `join()` +// (the caller will never read this packet until the thread has exited). // -// This packet itself is then stored into a `JoinInner` which in turns is placed -// in `JoinHandle` and `JoinGuard`. Due to the usage of `UnsafeCell` we need to -// manually worry about impls like Send and Sync. The type `T` should -// already always be Send (otherwise the thread could not have been created) and -// this type is inherently Sync because no methods take &self. Regardless, -// however, we add inheriting impls for Send/Sync to this type to ensure it's -// Send/Sync and that future modifications will still appropriately classify it. -struct Packet(Arc>>>); - -unsafe impl Send for Packet {} -unsafe impl Sync for Packet {} +// An Arc to the packet is stored into a `JoinInner` which in turns is placed +// in `JoinHandle`. +struct Packet<'scope, T> { + scope: Option>, + result: UnsafeCell>>, + _marker: PhantomData>, +} + +// Due to the usage of `UnsafeCell` we need to manually implement Sync. +// The type `T` should already always be Send (otherwise the thread could not +// have been created) and the Packet is Sync because all access to the +// `UnsafeCell` synchronized (by the `join()` boundary), and `ScopeData` is Sync. +unsafe impl<'scope, T: Sync> Sync for Packet<'scope, T> {} + +impl<'scope, T> Drop for Packet<'scope, T> { + fn drop(&mut self) { + // If this packet was for a thread that ran in a scope, the thread + // panicked, and nobody consumed the panic payload, we make sure + // the scope function will panic. + let unhandled_panic = matches!(self.result.get_mut(), Some(Err(_))); + // Drop the result without causing unwinding. + // This is only relevant for threads that aren't join()ed, as + // join() will take the `result` and set it to None, such that + // there is nothing left to drop here. + // If this panics, we should handle that, because we're outside the + // outermost `catch_unwind` of our thread. + // We just abort in that case, since there's nothing else we can do. + // (And even if we tried to handle it somehow, we'd also need to handle + // the case where the panic payload we get out of it also panics on + // drop, and so on. See issue #86027.) + if let Err(_) = panic::catch_unwind(panic::AssertUnwindSafe(|| { + *self.result.get_mut() = None; + })) { + rtabort!("thread result panicked on drop"); + } + // Book-keeping so the scope knows when it's done. + if let Some(scope) = &self.scope { + // Now that there will be no more user code running on this thread + // that can use 'scope, mark the thread as 'finished'. + // It's important we only do this after the `result` has been dropped, + // since dropping it might still use things it borrowed from 'scope. + scope.decrement_num_running_threads(unhandled_panic); + } + } +} /// Inner representation for JoinHandle -struct JoinInner { - // native: Option, +struct JoinInner<'scope, T> { + native: imp::Thread, thread: Thread, - // packet: Packet, - phantom: core::marker::PhantomData, + packet: Arc>, } - -impl JoinInner { - fn join(&mut self) -> Result { - let ThreadId(i) = self.thread.id(); - let result = concurrency::join(i.get()); - Ok(result) +impl<'scope, T> JoinInner<'scope, T> { + fn join(mut self) -> Result { + self.native.join(); + Arc::get_mut(&mut self.packet).unwrap().result.get_mut().take().unwrap() } } /// An owned permission to join on a thread (block on its termination). /// /// A `JoinHandle` *detaches* the associated thread when it is dropped, which -/// means that there is no longer any handle to thread and no way to `join` +/// means that there is no longer any handle to the thread and no way to `join` /// on it. /// /// Due to platform restrictions, it is not possible to [`Clone`] this @@ -1420,7 +1495,7 @@ impl JoinInner { /// }).unwrap(); /// ``` /// -/// Child being detached and outliving its parent: +/// A thread being detached and outliving the thread that spawned it: /// /// ```no_run /// use std::thread; @@ -1444,11 +1519,10 @@ impl JoinInner { /// thread::sleep(Duration::from_millis(1000)); /// ``` /// -/// [`Clone`]: ../../std/clone/trait.Clone.html -/// [`thread::spawn`]: fn.spawn.html -/// [`thread::Builder::spawn`]: struct.Builder.html#method.spawn +/// [`thread::Builder::spawn`]: Builder::spawn +/// [`thread::spawn`]: spawn #[stable(feature = "rust1", since = "1.0.0")] -pub struct JoinHandle(JoinInner); +pub struct JoinHandle(JoinInner<'static, T>); #[stable(feature = "joinhandle_impl_send_sync", since = "1.29.0")] unsafe impl Send for JoinHandle {} @@ -1473,23 +1547,26 @@ impl JoinHandle { /// println!("thread id: {:?}", thread.id()); /// ``` #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] pub fn thread(&self) -> &Thread { &self.0.thread } /// Waits for the associated thread to finish. /// + /// This function will return immediately if the associated thread has already finished. + /// /// In terms of [atomic memory orderings], the completion of the associated /// thread synchronizes with this function returning. In other words, all - /// operations performed by that thread are ordered before all + /// operations performed by that thread [happen + /// before](https://doc.rust-lang.org/nomicon/atomics.html#data-accesses) all /// operations that happen after `join` returns. /// - /// If the child thread panics, [`Err`] is returned with the parameter given - /// to [`panic`]. + /// If the associated thread panics, [`Err`] is returned with the parameter given + /// to [`panic!`]. /// - /// [`Err`]: ../../std/result/enum.Result.html#variant.Err - /// [`panic`]: ../../std/macro.panic.html - /// [atomic memory orderings]: ../../std/sync/atomic/index.html + /// [`Err`]: crate::result::Result::Err + /// [atomic memory orderings]: crate::sync::atomic /// /// # Panics /// @@ -1509,28 +1586,42 @@ impl JoinHandle { /// join_handle.join().expect("Couldn't join on the associated thread"); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - pub fn join(mut self) -> Result { + pub fn join(self) -> Result { self.0.join() } + + /// Checks if the associated thread has finished running its main function. + /// + /// `is_finished` supports implementing a non-blocking join operation, by checking + /// `is_finished`, and calling `join` if it returns `true`. This function does not block. To + /// block while waiting on the thread to finish, use [`join`][Self::join]. + /// + /// This might return `true` for a brief moment after the thread's main + /// function has returned, but before the thread itself has stopped running. + /// However, once this returns `true`, [`join`][Self::join] can be expected + /// to return quickly, without blocking for any significant amount of time. + #[stable(feature = "thread_is_running", since = "1.61.0")] + pub fn is_finished(&self) -> bool { + Arc::strong_count(&self.0.packet) == 1 + } } impl AsInner for JoinHandle { fn as_inner(&self) -> &imp::Thread { - panic!("ASDF") - // self.0.native.as_ref().unwrap() + &self.0.native } } impl IntoInner for JoinHandle { fn into_inner(self) -> imp::Thread { - panic!("ASDF") + self.0.native } } #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for JoinHandle { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("JoinHandle { .. }") + f.debug_struct("JoinHandle").finish_non_exhaustive() } } @@ -1540,276 +1631,91 @@ fn _assert_sync_and_send() { _assert_both::(); } -//////////////////////////////////////////////////////////////////////////////// -// Tests -//////////////////////////////////////////////////////////////////////////////// - -#[cfg(all(test, not(target_os = "emscripten")))] -mod tests { - use super::Builder; - use crate::any::Any; - use crate::mem; - use crate::result; - use crate::sync::mpsc::{channel, Sender}; - use crate::thread::{self, ThreadId}; - use crate::time::Duration; - use crate::u32; - - // !!! These tests are dangerous. If something is buggy, they will hang, !!! - // !!! instead of exiting cleanly. This might wedge the buildbots. !!! - - #[test] - fn test_unnamed_thread() { - thread::spawn(move || { - assert!(thread::current().name().is_none()); - }) - .join() - .ok() - .expect("thread panicked"); - } - - #[test] - fn test_named_thread() { - Builder::new() - .name("ada lovelace".to_string()) - .spawn(move || { - assert!(thread::current().name().unwrap() == "ada lovelace".to_string()); - }) - .unwrap() - .join() - .unwrap(); - } - - #[test] - #[should_panic] - fn test_invalid_named_thread() { - let _ = Builder::new().name("ada l\0velace".to_string()).spawn(|| {}); - } - - #[test] - fn test_run_basic() { - let (tx, rx) = channel(); - thread::spawn(move || { - tx.send(()).unwrap(); - }); - rx.recv().unwrap(); - } - - #[test] - fn test_join_panic() { - match thread::spawn(move || panic!()).join() { - result::Result::Err(_) => (), - result::Result::Ok(()) => panic!(), - } - } - - #[test] - fn test_spawn_sched() { - let (tx, rx) = channel(); - - fn f(i: i32, tx: Sender<()>) { - let tx = tx.clone(); - thread::spawn(move || { - if i == 0 { - tx.send(()).unwrap(); - } else { - f(i - 1, tx); - } - }); - } - f(10, tx); - rx.recv().unwrap(); - } - - #[test] - fn test_spawn_sched_childs_on_default_sched() { - let (tx, rx) = channel(); - - thread::spawn(move || { - thread::spawn(move || { - tx.send(()).unwrap(); - }); - }); - - rx.recv().unwrap(); - } - - fn avoid_copying_the_body(spawnfn: F) - where - F: FnOnce(Box), - { - let (tx, rx) = channel(); - - let x: Box<_> = box 1; - let x_in_parent = (&*x) as *const i32 as usize; - - spawnfn(Box::new(move || { - let x_in_child = (&*x) as *const i32 as usize; - tx.send(x_in_child).unwrap(); - })); - - let x_in_child = rx.recv().unwrap(); - assert_eq!(x_in_parent, x_in_child); - } - - #[test] - fn test_avoid_copying_the_body_spawn() { - avoid_copying_the_body(|v| { - thread::spawn(move || v()); - }); - } - - #[test] - fn test_avoid_copying_the_body_thread_spawn() { - avoid_copying_the_body(|f| { - thread::spawn(move || { - f(); - }); - }) - } - - #[test] - fn test_avoid_copying_the_body_join() { - avoid_copying_the_body(|f| { - let _ = thread::spawn(move || f()).join(); - }) - } - - #[test] - fn test_child_doesnt_ref_parent() { - // If the child refcounts the parent thread, this will stack overflow when - // climbing the thread tree to dereference each ancestor. (See #1789) - // (well, it would if the constant were 8000+ - I lowered it to be more - // valgrind-friendly. try this at home, instead..!) - const GENERATIONS: u32 = 16; - fn child_no(x: u32) -> Box { - return Box::new(move || { - if x < GENERATIONS { - thread::spawn(move || child_no(x + 1)()); - } - }); - } - thread::spawn(|| child_no(0)()); - } - - #[test] - fn test_simple_newsched_spawn() { - thread::spawn(move || {}); - } - - #[test] - fn test_try_panic_message_static_str() { - match thread::spawn(move || { - panic!("static string"); - }) - .join() - { - Err(e) => { - type T = &'static str; - assert!(e.is::()); - assert_eq!(*e.downcast::().unwrap(), "static string"); - } - Ok(()) => panic!(), - } - } - - #[test] - fn test_try_panic_message_owned_str() { - match thread::spawn(move || { - panic!("owned string".to_string()); - }) - .join() - { - Err(e) => { - type T = String; - assert!(e.is::()); - assert_eq!(*e.downcast::().unwrap(), "owned string".to_string()); - } - Ok(()) => panic!(), - } - } - - #[test] - fn test_try_panic_message_any() { - match thread::spawn(move || { - panic!(box 413u16 as Box); - }) - .join() - { - Err(e) => { - type T = Box; - assert!(e.is::()); - let any = e.downcast::().unwrap(); - assert!(any.is::()); - assert_eq!(*any.downcast::().unwrap(), 413); - } - Ok(()) => panic!(), - } - } - - #[test] - fn test_try_panic_message_unit_struct() { - struct Juju; - - match thread::spawn(move || panic!(Juju)).join() { - Err(ref e) if e.is::() => {} - Err(_) | Ok(()) => panic!(), - } - } - - #[test] - fn test_park_timeout_unpark_before() { - for _ in 0..10 { - thread::current().unpark(); - thread::park_timeout(Duration::from_millis(u32::MAX as u64)); - } - } - - #[test] - #[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - fn test_park_timeout_unpark_not_called() { - for _ in 0..10 { - thread::park_timeout(Duration::from_millis(10)); - } - } - - #[test] - #[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - fn test_park_timeout_unpark_called_other_thread() { - for _ in 0..10 { - let th = thread::current(); - - let _guard = thread::spawn(move || { - super::sleep(Duration::from_millis(50)); - th.unpark(); - }); - - thread::park_timeout(Duration::from_millis(u32::MAX as u64)); - } - } - - #[test] - #[cfg_attr(target_env = "sgx", ignore)] // FIXME: https://github.com/fortanix/rust-sgx/issues/31 - fn sleep_ms_smoke() { - thread::sleep(Duration::from_millis(2)); - } - - #[test] - fn test_size_of_option_thread_id() { - assert_eq!(mem::size_of::>(), mem::size_of::()); - } - - #[test] - fn test_thread_id_equal() { - assert!(thread::current().id() == thread::current().id()); - } - - #[test] - fn test_thread_id_not_equal() { - let spawned_id = thread::spawn(|| thread::current().id()).join().unwrap(); - assert!(thread::current().id() != spawned_id); - } - - // NOTE: the corresponding test for stderr is in ui/thread-stderr, due - // to the test harness apparently interfering with stderr configuration. +/// Returns an estimate of the default amount of parallelism a program should use. +/// +/// Parallelism is a resource. A given machine provides a certain capacity for +/// parallelism, i.e., a bound on the number of computations it can perform +/// simultaneously. This number often corresponds to the amount of CPUs a +/// computer has, but it may diverge in various cases. +/// +/// Host environments such as VMs or container orchestrators may want to +/// restrict the amount of parallelism made available to programs in them. This +/// is often done to limit the potential impact of (unintentionally) +/// resource-intensive programs on other programs running on the same machine. +/// +/// # Limitations +/// +/// The purpose of this API is to provide an easy and portable way to query +/// the default amount of parallelism the program should use. Among other things it +/// does not expose information on NUMA regions, does not account for +/// differences in (co)processor capabilities or current system load, +/// and will not modify the program's global state in order to more accurately +/// query the amount of available parallelism. +/// +/// Where both fixed steady-state and burst limits are available the steady-state +/// capacity will be used to ensure more predictable latencies. +/// +/// Resource limits can be changed during the runtime of a program, therefore the value is +/// not cached and instead recomputed every time this function is called. It should not be +/// called from hot code. +/// +/// The value returned by this function should be considered a simplified +/// approximation of the actual amount of parallelism available at any given +/// time. To get a more detailed or precise overview of the amount of +/// parallelism available to the program, you may wish to use +/// platform-specific APIs as well. The following platform limitations currently +/// apply to `available_parallelism`: +/// +/// On Windows: +/// - It may undercount the amount of parallelism available on systems with more +/// than 64 logical CPUs. However, programs typically need specific support to +/// take advantage of more than 64 logical CPUs, and in the absence of such +/// support, the number returned by this function accurately reflects the +/// number of logical CPUs the program can use by default. +/// - It may overcount the amount of parallelism available on systems limited by +/// process-wide affinity masks, or job object limitations. +/// +/// On Linux: +/// - It may overcount the amount of parallelism available when limited by a +/// process-wide affinity mask or cgroup quotas and `sched_getaffinity()` or cgroup fs can't be +/// queried, e.g. due to sandboxing. +/// - It may undercount the amount of parallelism if the current thread's affinity mask +/// does not reflect the process' cpuset, e.g. due to pinned threads. +/// - If the process is in a cgroup v1 cpu controller, this may need to +/// scan mountpoints to find the corresponding cgroup v1 controller, +/// which may take time on systems with large numbers of mountpoints. +/// (This does not apply to cgroup v2, or to processes not in a +/// cgroup.) +/// +/// On all targets: +/// - It may overcount the amount of parallelism available when running in a VM +/// with CPU usage limits (e.g. an overcommitted host). +/// +/// # Errors +/// +/// This function will, but is not limited to, return errors in the following +/// cases: +/// +/// - If the amount of parallelism is not known for the target platform. +/// - If the program lacks permission to query the amount of parallelism made +/// available to it. +/// +/// # Examples +/// +/// ``` +/// # #![allow(dead_code)] +/// use std::{io, thread}; +/// +/// fn main() -> io::Result<()> { +/// let count = thread::available_parallelism()?.get(); +/// assert!(count >= 1_usize); +/// Ok(()) +/// } +/// ``` +#[doc(alias = "available_concurrency")] // Alias for a previous name we gave this API on unstable. +#[doc(alias = "hardware_concurrency")] // Alias for C++ `std::thread::hardware_concurrency`. +#[doc(alias = "num_cpus")] // Alias for a popular ecosystem crate which provides similar functionality. +#[stable(feature = "available_parallelism", since = "1.59.0")] +pub fn available_parallelism() -> io::Result { + imp::available_parallelism() } diff --git a/crux-mir/lib/std/src/thread/scoped.rs b/crux-mir/lib/std/src/thread/scoped.rs new file mode 100644 index 000000000..ada69aa82 --- /dev/null +++ b/crux-mir/lib/std/src/thread/scoped.rs @@ -0,0 +1,343 @@ +use super::{current, park, Builder, JoinInner, Result, Thread}; +use crate::fmt; +use crate::io; +use crate::marker::PhantomData; +use crate::panic::{catch_unwind, resume_unwind, AssertUnwindSafe}; +use crate::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use crate::sync::Arc; + +/// A scope to spawn scoped threads in. +/// +/// See [`scope`] for details. +#[stable(feature = "scoped_threads", since = "1.63.0")] +pub struct Scope<'scope, 'env: 'scope> { + data: Arc, + /// Invariance over 'scope, to make sure 'scope cannot shrink, + /// which is necessary for soundness. + /// + /// Without invariance, this would compile fine but be unsound: + /// + /// ```compile_fail,E0373 + /// std::thread::scope(|s| { + /// s.spawn(|| { + /// let a = String::from("abcd"); + /// s.spawn(|| println!("{a:?}")); // might run after `a` is dropped + /// }); + /// }); + /// ``` + scope: PhantomData<&'scope mut &'scope ()>, + env: PhantomData<&'env mut &'env ()>, +} + +/// An owned permission to join on a scoped thread (block on its termination). +/// +/// See [`Scope::spawn`] for details. +#[stable(feature = "scoped_threads", since = "1.63.0")] +pub struct ScopedJoinHandle<'scope, T>(JoinInner<'scope, T>); + +pub(super) struct ScopeData { + num_running_threads: AtomicUsize, + a_thread_panicked: AtomicBool, + main_thread: Thread, +} + +impl ScopeData { + pub(super) fn increment_num_running_threads(&self) { + // We check for 'overflow' with usize::MAX / 2, to make sure there's no + // chance it overflows to 0, which would result in unsoundness. + if self.num_running_threads.fetch_add(1, Ordering::Relaxed) > usize::MAX / 2 { + // This can only reasonably happen by mem::forget()'ing a lot of ScopedJoinHandles. + self.decrement_num_running_threads(false); + panic!("too many running threads in thread scope"); + } + } + pub(super) fn decrement_num_running_threads(&self, panic: bool) { + if panic { + self.a_thread_panicked.store(true, Ordering::Relaxed); + } + if self.num_running_threads.fetch_sub(1, Ordering::Release) == 1 { + self.main_thread.unpark(); + } + } +} + +/// Create a scope for spawning scoped threads. +/// +/// The function passed to `scope` will be provided a [`Scope`] object, +/// through which scoped threads can be [spawned][`Scope::spawn`]. +/// +/// Unlike non-scoped threads, scoped threads can borrow non-`'static` data, +/// as the scope guarantees all threads will be joined at the end of the scope. +/// +/// All threads spawned within the scope that haven't been manually joined +/// will be automatically joined before this function returns. +/// +/// # Panics +/// +/// If any of the automatically joined threads panicked, this function will panic. +/// +/// If you want to handle panics from spawned threads, +/// [`join`][ScopedJoinHandle::join] them before the end of the scope. +/// +/// # Example +/// +/// ``` +/// use std::thread; +/// +/// let mut a = vec![1, 2, 3]; +/// let mut x = 0; +/// +/// thread::scope(|s| { +/// s.spawn(|| { +/// println!("hello from the first scoped thread"); +/// // We can borrow `a` here. +/// dbg!(&a); +/// }); +/// s.spawn(|| { +/// println!("hello from the second scoped thread"); +/// // We can even mutably borrow `x` here, +/// // because no other threads are using it. +/// x += a[0] + a[2]; +/// }); +/// println!("hello from the main thread"); +/// }); +/// +/// // After the scope, we can modify and access our variables again: +/// a.push(4); +/// assert_eq!(x, a.len()); +/// ``` +/// +/// # Lifetimes +/// +/// Scoped threads involve two lifetimes: `'scope` and `'env`. +/// +/// The `'scope` lifetime represents the lifetime of the scope itself. +/// That is: the time during which new scoped threads may be spawned, +/// and also the time during which they might still be running. +/// Once this lifetime ends, all scoped threads are joined. +/// This lifetime starts within the `scope` function, before `f` (the argument to `scope`) starts. +/// It ends after `f` returns and all scoped threads have been joined, but before `scope` returns. +/// +/// The `'env` lifetime represents the lifetime of whatever is borrowed by the scoped threads. +/// This lifetime must outlast the call to `scope`, and thus cannot be smaller than `'scope`. +/// It can be as small as the call to `scope`, meaning that anything that outlives this call, +/// such as local variables defined right before the scope, can be borrowed by the scoped threads. +/// +/// The `'env: 'scope` bound is part of the definition of the `Scope` type. +#[track_caller] +#[stable(feature = "scoped_threads", since = "1.63.0")] +pub fn scope<'env, F, T>(f: F) -> T +where + F: for<'scope> FnOnce(&'scope Scope<'scope, 'env>) -> T, +{ + // We put the `ScopeData` into an `Arc` so that other threads can finish their + // `decrement_num_running_threads` even after this function returns. + let scope = Scope { + data: Arc::new(ScopeData { + num_running_threads: AtomicUsize::new(0), + main_thread: current(), + a_thread_panicked: AtomicBool::new(false), + }), + env: PhantomData, + scope: PhantomData, + }; + + // Run `f`, but catch panics so we can make sure to wait for all the threads to join. + let result = catch_unwind(AssertUnwindSafe(|| f(&scope))); + + // Wait until all the threads are finished. + while scope.data.num_running_threads.load(Ordering::Acquire) != 0 { + park(); + } + + // Throw any panic from `f`, or the return value of `f` if no thread panicked. + match result { + Err(e) => resume_unwind(e), + Ok(_) if scope.data.a_thread_panicked.load(Ordering::Relaxed) => { + panic!("a scoped thread panicked") + } + Ok(result) => result, + } +} + +impl<'scope, 'env> Scope<'scope, 'env> { + /// Spawns a new thread within a scope, returning a [`ScopedJoinHandle`] for it. + /// + /// Unlike non-scoped threads, threads spawned with this function may + /// borrow non-`'static` data from the outside the scope. See [`scope`] for + /// details. + /// + /// The join handle provides a [`join`] method that can be used to join the spawned + /// thread. If the spawned thread panics, [`join`] will return an [`Err`] containing + /// the panic payload. + /// + /// If the join handle is dropped, the spawned thread will implicitly joined at the + /// end of the scope. In that case, if the spawned thread panics, [`scope`] will + /// panic after all threads are joined. + /// + /// This call will create a thread using default parameters of [`Builder`]. + /// If you want to specify the stack size or the name of the thread, use + /// [`Builder::spawn_scoped`] instead. + /// + /// # Panics + /// + /// Panics if the OS fails to create a thread; use [`Builder::spawn_scoped`] + /// to recover from such errors. + /// + /// [`join`]: ScopedJoinHandle::join + #[stable(feature = "scoped_threads", since = "1.63.0")] + pub fn spawn(&'scope self, f: F) -> ScopedJoinHandle<'scope, T> + where + F: FnOnce() -> T + Send + 'scope, + T: Send + 'scope, + { + Builder::new().spawn_scoped(self, f).expect("failed to spawn thread") + } +} + +impl Builder { + /// Spawns a new scoped thread using the settings set through this `Builder`. + /// + /// Unlike [`Scope::spawn`], this method yields an [`io::Result`] to + /// capture any failure to create the thread at the OS level. + /// + /// [`io::Result`]: crate::io::Result + /// + /// # Panics + /// + /// Panics if a thread name was set and it contained null bytes. + /// + /// # Example + /// + /// ``` + /// use std::thread; + /// + /// let mut a = vec![1, 2, 3]; + /// let mut x = 0; + /// + /// thread::scope(|s| { + /// thread::Builder::new() + /// .name("first".to_string()) + /// .spawn_scoped(s, || + /// { + /// println!("hello from the {:?} scoped thread", thread::current().name()); + /// // We can borrow `a` here. + /// dbg!(&a); + /// }) + /// .unwrap(); + /// thread::Builder::new() + /// .name("second".to_string()) + /// .spawn_scoped(s, || + /// { + /// println!("hello from the {:?} scoped thread", thread::current().name()); + /// // We can even mutably borrow `x` here, + /// // because no other threads are using it. + /// x += a[0] + a[2]; + /// }) + /// .unwrap(); + /// println!("hello from the main thread"); + /// }); + /// + /// // After the scope, we can modify and access our variables again: + /// a.push(4); + /// assert_eq!(x, a.len()); + /// ``` + #[stable(feature = "scoped_threads", since = "1.63.0")] + pub fn spawn_scoped<'scope, 'env, F, T>( + self, + scope: &'scope Scope<'scope, 'env>, + f: F, + ) -> io::Result> + where + F: FnOnce() -> T + Send + 'scope, + T: Send + 'scope, + { + Ok(ScopedJoinHandle(unsafe { self.spawn_unchecked_(f, Some(scope.data.clone())) }?)) + } +} + +impl<'scope, T> ScopedJoinHandle<'scope, T> { + /// Extracts a handle to the underlying thread. + /// + /// # Examples + /// + /// ``` + /// use std::thread; + /// + /// thread::scope(|s| { + /// let t = s.spawn(|| { + /// println!("hello"); + /// }); + /// println!("thread id: {:?}", t.thread().id()); + /// }); + /// ``` + #[must_use] + #[stable(feature = "scoped_threads", since = "1.63.0")] + pub fn thread(&self) -> &Thread { + &self.0.thread + } + + /// Waits for the associated thread to finish. + /// + /// This function will return immediately if the associated thread has already finished. + /// + /// In terms of [atomic memory orderings], the completion of the associated + /// thread synchronizes with this function returning. + /// In other words, all operations performed by that thread + /// [happen before](https://doc.rust-lang.org/nomicon/atomics.html#data-accesses) + /// all operations that happen after `join` returns. + /// + /// If the associated thread panics, [`Err`] is returned with the panic payload. + /// + /// [atomic memory orderings]: crate::sync::atomic + /// + /// # Examples + /// + /// ``` + /// use std::thread; + /// + /// thread::scope(|s| { + /// let t = s.spawn(|| { + /// panic!("oh no"); + /// }); + /// assert!(t.join().is_err()); + /// }); + /// ``` + #[stable(feature = "scoped_threads", since = "1.63.0")] + pub fn join(self) -> Result { + self.0.join() + } + + /// Checks if the associated thread has finished running its main function. + /// + /// `is_finished` supports implementing a non-blocking join operation, by checking + /// `is_finished`, and calling `join` if it returns `false`. This function does not block. To + /// block while waiting on the thread to finish, use [`join`][Self::join]. + /// + /// This might return `true` for a brief moment after the thread's main + /// function has returned, but before the thread itself has stopped running. + /// However, once this returns `true`, [`join`][Self::join] can be expected + /// to return quickly, without blocking for any significant amount of time. + #[stable(feature = "scoped_threads", since = "1.63.0")] + pub fn is_finished(&self) -> bool { + Arc::strong_count(&self.0.packet) == 1 + } +} + +#[stable(feature = "scoped_threads", since = "1.63.0")] +impl fmt::Debug for Scope<'_, '_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Scope") + .field("num_running_threads", &self.data.num_running_threads.load(Ordering::Relaxed)) + .field("a_thread_panicked", &self.data.a_thread_panicked.load(Ordering::Relaxed)) + .field("main_thread", &self.data.main_thread) + .finish_non_exhaustive() + } +} + +#[stable(feature = "scoped_threads", since = "1.63.0")] +impl<'scope, T> fmt::Debug for ScopedJoinHandle<'scope, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ScopedJoinHandle").finish_non_exhaustive() + } +} diff --git a/crux-mir/lib/std/src/thread/tests.rs b/crux-mir/lib/std/src/thread/tests.rs new file mode 100644 index 000000000..6c9ce6fa0 --- /dev/null +++ b/crux-mir/lib/std/src/thread/tests.rs @@ -0,0 +1,403 @@ +use super::Builder; +use crate::any::Any; +use crate::mem; +use crate::panic::panic_any; +use crate::result; +use crate::sync::{ + atomic::{AtomicBool, Ordering}, + mpsc::{channel, Sender}, + Arc, Barrier, +}; +use crate::thread::{self, Scope, ThreadId}; +use crate::time::Duration; +use crate::time::Instant; + +// !!! These tests are dangerous. If something is buggy, they will hang, !!! +// !!! instead of exiting cleanly. This might wedge the buildbots. !!! + +#[test] +fn test_unnamed_thread() { + thread::spawn(move || { + assert!(thread::current().name().is_none()); + }) + .join() + .ok() + .expect("thread panicked"); +} + +#[test] +fn test_named_thread() { + Builder::new() + .name("ada lovelace".to_string()) + .spawn(move || { + assert!(thread::current().name().unwrap() == "ada lovelace".to_string()); + }) + .unwrap() + .join() + .unwrap(); +} + +#[cfg(any( + // Note: musl didn't add pthread_getname_np until 1.2.3 + all(target_os = "linux", target_env = "gnu"), + target_os = "macos", + target_os = "ios", + target_os = "watchos" +))] +#[test] +fn test_named_thread_truncation() { + use crate::ffi::CStr; + + let long_name = crate::iter::once("test_named_thread_truncation") + .chain(crate::iter::repeat(" yada").take(100)) + .collect::(); + + let result = Builder::new().name(long_name.clone()).spawn(move || { + // Rust remembers the full thread name itself. + assert_eq!(thread::current().name(), Some(long_name.as_str())); + + // But the system is limited -- make sure we successfully set a truncation. + let mut buf = vec![0u8; long_name.len() + 1]; + unsafe { + libc::pthread_getname_np(libc::pthread_self(), buf.as_mut_ptr().cast(), buf.len()); + } + let cstr = CStr::from_bytes_until_nul(&buf).unwrap(); + assert!(cstr.to_bytes().len() > 0); + assert!(long_name.as_bytes().starts_with(cstr.to_bytes())); + }); + result.unwrap().join().unwrap(); +} + +#[test] +#[should_panic] +fn test_invalid_named_thread() { + let _ = Builder::new().name("ada l\0velace".to_string()).spawn(|| {}); +} + +#[test] +fn test_run_basic() { + let (tx, rx) = channel(); + thread::spawn(move || { + tx.send(()).unwrap(); + }); + rx.recv().unwrap(); +} + +#[test] +fn test_is_finished() { + let b = Arc::new(Barrier::new(2)); + let t = thread::spawn({ + let b = b.clone(); + move || { + b.wait(); + 1234 + } + }); + + // Thread is definitely running here, since it's still waiting for the barrier. + assert_eq!(t.is_finished(), false); + + // Unblock the barrier. + b.wait(); + + // Now check that t.is_finished() becomes true within a reasonable time. + let start = Instant::now(); + while !t.is_finished() { + assert!(start.elapsed() < Duration::from_secs(2)); + thread::sleep(Duration::from_millis(15)); + } + + // Joining the thread should not block for a significant time now. + let join_time = Instant::now(); + assert_eq!(t.join().unwrap(), 1234); + assert!(join_time.elapsed() < Duration::from_secs(2)); +} + +#[test] +fn test_join_panic() { + match thread::spawn(move || panic!()).join() { + result::Result::Err(_) => (), + result::Result::Ok(()) => panic!(), + } +} + +#[test] +fn test_spawn_sched() { + let (tx, rx) = channel(); + + fn f(i: i32, tx: Sender<()>) { + let tx = tx.clone(); + thread::spawn(move || { + if i == 0 { + tx.send(()).unwrap(); + } else { + f(i - 1, tx); + } + }); + } + f(10, tx); + rx.recv().unwrap(); +} + +#[test] +fn test_spawn_sched_childs_on_default_sched() { + let (tx, rx) = channel(); + + thread::spawn(move || { + thread::spawn(move || { + tx.send(()).unwrap(); + }); + }); + + rx.recv().unwrap(); +} + +fn avoid_copying_the_body(spawnfn: F) +where + F: FnOnce(Box), +{ + let (tx, rx) = channel(); + + let x: Box<_> = Box::new(1); + let x_in_parent = (&*x) as *const i32 as usize; + + spawnfn(Box::new(move || { + let x_in_child = (&*x) as *const i32 as usize; + tx.send(x_in_child).unwrap(); + })); + + let x_in_child = rx.recv().unwrap(); + assert_eq!(x_in_parent, x_in_child); +} + +#[test] +fn test_avoid_copying_the_body_spawn() { + avoid_copying_the_body(|v| { + thread::spawn(move || v()); + }); +} + +#[test] +fn test_avoid_copying_the_body_thread_spawn() { + avoid_copying_the_body(|f| { + thread::spawn(move || { + f(); + }); + }) +} + +#[test] +fn test_avoid_copying_the_body_join() { + avoid_copying_the_body(|f| { + let _ = thread::spawn(move || f()).join(); + }) +} + +#[test] +fn test_child_doesnt_ref_parent() { + // If the child refcounts the parent thread, this will stack overflow when + // climbing the thread tree to dereference each ancestor. (See #1789) + // (well, it would if the constant were 8000+ - I lowered it to be more + // valgrind-friendly. try this at home, instead..!) + const GENERATIONS: u32 = 16; + fn child_no(x: u32) -> Box { + return Box::new(move || { + if x < GENERATIONS { + thread::spawn(move || child_no(x + 1)()); + } + }); + } + thread::spawn(|| child_no(0)()); +} + +#[test] +fn test_simple_newsched_spawn() { + thread::spawn(move || {}); +} + +#[test] +fn test_try_panic_message_string_literal() { + match thread::spawn(move || { + panic!("static string"); + }) + .join() + { + Err(e) => { + type T = &'static str; + assert!(e.is::()); + assert_eq!(*e.downcast::().unwrap(), "static string"); + } + Ok(()) => panic!(), + } +} + +#[test] +fn test_try_panic_any_message_owned_str() { + match thread::spawn(move || { + panic_any("owned string".to_string()); + }) + .join() + { + Err(e) => { + type T = String; + assert!(e.is::()); + assert_eq!(*e.downcast::().unwrap(), "owned string".to_string()); + } + Ok(()) => panic!(), + } +} + +#[test] +fn test_try_panic_any_message_any() { + match thread::spawn(move || { + panic_any(Box::new(413u16) as Box); + }) + .join() + { + Err(e) => { + type T = Box; + assert!(e.is::()); + let any = e.downcast::().unwrap(); + assert!(any.is::()); + assert_eq!(*any.downcast::().unwrap(), 413); + } + Ok(()) => panic!(), + } +} + +#[test] +fn test_try_panic_any_message_unit_struct() { + struct Juju; + + match thread::spawn(move || panic_any(Juju)).join() { + Err(ref e) if e.is::() => {} + Err(_) | Ok(()) => panic!(), + } +} + +#[test] +fn test_park_unpark_before() { + for _ in 0..10 { + thread::current().unpark(); + thread::park(); + } +} + +#[test] +fn test_park_unpark_called_other_thread() { + for _ in 0..10 { + let th = thread::current(); + + let _guard = thread::spawn(move || { + super::sleep(Duration::from_millis(50)); + th.unpark(); + }); + + thread::park(); + } +} + +#[test] +fn test_park_timeout_unpark_before() { + for _ in 0..10 { + thread::current().unpark(); + thread::park_timeout(Duration::from_millis(u32::MAX as u64)); + } +} + +#[test] +fn test_park_timeout_unpark_not_called() { + for _ in 0..10 { + thread::park_timeout(Duration::from_millis(10)); + } +} + +#[test] +fn test_park_timeout_unpark_called_other_thread() { + for _ in 0..10 { + let th = thread::current(); + + let _guard = thread::spawn(move || { + super::sleep(Duration::from_millis(50)); + th.unpark(); + }); + + thread::park_timeout(Duration::from_millis(u32::MAX as u64)); + } +} + +#[test] +fn sleep_ms_smoke() { + thread::sleep(Duration::from_millis(2)); +} + +#[test] +fn test_size_of_option_thread_id() { + assert_eq!(mem::size_of::>(), mem::size_of::()); +} + +#[test] +fn test_thread_id_equal() { + assert!(thread::current().id() == thread::current().id()); +} + +#[test] +fn test_thread_id_not_equal() { + let spawned_id = thread::spawn(|| thread::current().id()).join().unwrap(); + assert!(thread::current().id() != spawned_id); +} + +#[test] +fn test_scoped_threads_drop_result_before_join() { + let actually_finished = &AtomicBool::new(false); + struct X<'scope, 'env>(&'scope Scope<'scope, 'env>, &'env AtomicBool); + impl Drop for X<'_, '_> { + fn drop(&mut self) { + thread::sleep(Duration::from_millis(20)); + let actually_finished = self.1; + self.0.spawn(move || { + thread::sleep(Duration::from_millis(20)); + actually_finished.store(true, Ordering::Relaxed); + }); + } + } + thread::scope(|s| { + s.spawn(move || { + thread::sleep(Duration::from_millis(20)); + X(s, actually_finished) + }); + }); + assert!(actually_finished.load(Ordering::Relaxed)); +} + +#[test] +fn test_scoped_threads_nll() { + // this is mostly a *compilation test* for this exact function: + fn foo(x: &u8) { + thread::scope(|s| { + s.spawn(|| drop(x)); + }); + } + // let's also run it for good measure + let x = 42_u8; + foo(&x); +} + +// Regression test for https://github.com/rust-lang/rust/issues/98498. +#[test] +#[cfg(miri)] // relies on Miri's data race detector +fn scope_join_race() { + for _ in 0..100 { + let a_bool = AtomicBool::new(false); + + thread::scope(|s| { + for _ in 0..5 { + s.spawn(|| a_bool.load(Ordering::Relaxed)); + } + + for _ in 0..5 { + s.spawn(|| a_bool.load(Ordering::Relaxed)); + } + }); + } +} diff --git a/crux-mir/lib/std/src/time.rs b/crux-mir/lib/std/src/time.rs index 65fd87c22..ecd06ebf7 100644 --- a/crux-mir/lib/std/src/time.rs +++ b/crux-mir/lib/std/src/time.rs @@ -1,38 +1,60 @@ //! Temporal quantification. //! -//! Example: +//! # Examples: //! +//! There are multiple ways to create a new [`Duration`]: +//! +//! ``` +//! # use std::time::Duration; +//! let five_seconds = Duration::from_secs(5); +//! assert_eq!(five_seconds, Duration::from_millis(5_000)); +//! assert_eq!(five_seconds, Duration::from_micros(5_000_000)); +//! assert_eq!(five_seconds, Duration::from_nanos(5_000_000_000)); +//! +//! let ten_seconds = Duration::from_secs(10); +//! let seven_nanos = Duration::from_nanos(7); +//! let total = ten_seconds + seven_nanos; +//! assert_eq!(total, Duration::new(10, 7)); //! ``` -//! use std::time::Duration; //! -//! let five_seconds = Duration::new(5, 0); -//! // both declarations are equivalent -//! assert_eq!(Duration::new(5, 0), Duration::from_secs(5)); +//! Using [`Instant`] to calculate how long a function took to run: +//! +//! ```ignore (incomplete) +//! let now = Instant::now(); +//! +//! // Calling a slow function, it may take a while +//! slow_function(); +//! +//! let elapsed_time = now.elapsed(); +//! println!("Running slow_function() took {} seconds.", elapsed_time.as_secs()); //! ``` #![stable(feature = "time", since = "1.3.0")] -use crate::cmp; +#[cfg(test)] +mod tests; + use crate::error::Error; use crate::fmt; use crate::ops::{Add, AddAssign, Sub, SubAssign}; -use crate::sys; -use crate::sys::crux::time; -use crate::sys_common::mutex::Mutex; -use crate::sys_common::FromInner; +use crate::sys::time; +use crate::sys_common::{FromInner, IntoInner}; #[stable(feature = "time", since = "1.3.0")] pub use core::time::Duration; +#[stable(feature = "duration_checked_float", since = "1.66.0")] +pub use core::time::TryFromFloatSecsError; + /// A measurement of a monotonically nondecreasing clock. -/// Opaque and useful only with `Duration`. +/// Opaque and useful only with [`Duration`]. /// -/// Instants are always guaranteed to be no less than any previously measured -/// instant when created, and are often useful for tasks such as measuring +/// Instants are always guaranteed, barring [platform bugs], to be no less than any previously +/// measured instant when created, and are often useful for tasks such as measuring /// benchmarks or timing how long an operation takes. /// -/// Note, however, that instants are not guaranteed to be **steady**. In other -/// words, each tick of the underlying clock may not be the same length (e.g. +/// Note, however, that instants are **not** guaranteed to be **steady**. In other +/// words, each tick of the underlying clock might not be the same length (e.g. /// some seconds may be longer than others). An instant may jump forwards or /// experience time dilation (slow down or speed up), but it will never go /// backwards. @@ -61,29 +83,72 @@ pub use core::time::Duration; /// } /// ``` /// +/// [platform bugs]: Instant#monotonicity +/// +/// # OS-specific behaviors +/// +/// An `Instant` is a wrapper around system-specific types and it may behave +/// differently depending on the underlying operating system. For example, +/// the following snippet is fine on Linux but panics on macOS: +/// +/// ```no_run +/// use std::time::{Instant, Duration}; +/// +/// let now = Instant::now(); +/// let max_seconds = u64::MAX / 1_000_000_000; +/// let duration = Duration::new(max_seconds, 0); +/// println!("{:?}", now + duration); +/// ``` +/// /// # Underlying System calls -/// Currently, the following system calls are being used to get the current time using `now()`: +/// +/// The following system calls are [currently] being used by `now()` to find out +/// the current time: /// /// | Platform | System call | -/// |:---------:|:--------------------------------------------------------------------:| -/// | CloudABI | [clock_time_get (Monotonic Clock)] | +/// |-----------|----------------------------------------------------------------------| /// | SGX | [`insecure_time` usercall]. More information on [timekeeping in SGX] | /// | UNIX | [clock_gettime (Monotonic Clock)] | /// | Darwin | [mach_absolute_time] | /// | VXWorks | [clock_gettime (Monotonic Clock)] | +/// | SOLID | `get_tim` | /// | WASI | [__wasi_clock_time_get (Monotonic Clock)] | /// | Windows | [QueryPerformanceCounter] | /// +/// [currently]: crate::io#platform-specific-behavior /// [QueryPerformanceCounter]: https://docs.microsoft.com/en-us/windows/win32/api/profileapi/nf-profileapi-queryperformancecounter /// [`insecure_time` usercall]: https://edp.fortanix.com/docs/api/fortanix_sgx_abi/struct.Usercalls.html#method.insecure_time /// [timekeeping in SGX]: https://edp.fortanix.com/docs/concepts/rust-std/#codestdtimecode /// [__wasi_clock_time_get (Monotonic Clock)]: https://github.com/WebAssembly/WASI/blob/master/phases/snapshot/docs.md#clock_time_get /// [clock_gettime (Monotonic Clock)]: https://linux.die.net/man/3/clock_gettime /// [mach_absolute_time]: https://developer.apple.com/library/archive/documentation/Darwin/Conceptual/KernelProgramming/services/services.html -/// [clock_time_get (Monotonic Clock)]: https://nuxi.nl/cloudabi/#clock_time_get /// /// **Disclaimer:** These system calls might change over time. /// +/// > Note: mathematical operations like [`add`] may panic if the underlying +/// > structure cannot represent the new point in time. +/// +/// [`add`]: Instant::add +/// +/// ## Monotonicity +/// +/// On all platforms `Instant` will try to use an OS API that guarantees monotonic behavior +/// if available, which is the case for all [tier 1] platforms. +/// In practice such guarantees are – under rare circumstances – broken by hardware, virtualization +/// or operating system bugs. To work around these bugs and platforms not offering monotonic clocks +/// [`duration_since`], [`elapsed`] and [`sub`] saturate to zero. In older Rust versions this +/// lead to a panic instead. [`checked_duration_since`] can be used to detect and handle situations +/// where monotonicity is violated, or `Instant`s are subtracted in the wrong order. +/// +/// This workaround obscures programming errors where earlier and later instants are accidentally +/// swapped. For this reason future rust versions may reintroduce panics. +/// +/// [tier 1]: https://doc.rust-lang.org/rustc/platform-support.html +/// [`duration_since`]: Instant::duration_since +/// [`elapsed`]: Instant::elapsed +/// [`sub`]: Instant::sub +/// [`checked_duration_since`]: Instant::checked_duration_since +/// #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] #[stable(feature = "time2", since = "1.8.0")] pub struct Instant(time::Instant); @@ -111,11 +176,6 @@ pub struct Instant(time::Instant); /// The size of a `SystemTime` struct may vary depending on the target operating /// system. /// -/// [`Instant`]: ../../std/time/struct.Instant.html -/// [`Result`]: ../../std/result/enum.Result.html -/// [`Duration`]: ../../std/time/struct.Duration.html -/// [`UNIX_EPOCH`]: ../../std/time/constant.UNIX_EPOCH.html -/// /// Example: /// /// ```no_run @@ -134,35 +194,46 @@ pub struct Instant(time::Instant); /// } /// Err(e) => { /// // an error occurred! -/// println!("Error: {:?}", e); +/// println!("Error: {e:?}"); /// } /// } /// } /// ``` /// -/// # Underlying System calls -/// Currently, the following system calls are being used to get the current time using `now()`: +/// # Platform-specific behavior +/// +/// The precision of `SystemTime` can depend on the underlying OS-specific time format. +/// For example, on Windows the time is represented in 100 nanosecond intervals whereas Linux +/// can represent nanosecond intervals. +/// +/// The following system calls are [currently] being used by `now()` to find out +/// the current time: /// /// | Platform | System call | -/// |:---------:|:--------------------------------------------------------------------:| -/// | CloudABI | [clock_time_get (Realtime Clock)] | +/// |-----------|----------------------------------------------------------------------| /// | SGX | [`insecure_time` usercall]. More information on [timekeeping in SGX] | /// | UNIX | [clock_gettime (Realtime Clock)] | -/// | DARWIN | [gettimeofday] | +/// | Darwin | [gettimeofday] | /// | VXWorks | [clock_gettime (Realtime Clock)] | +/// | SOLID | `SOLID_RTC_ReadTime` | /// | WASI | [__wasi_clock_time_get (Realtime Clock)] | -/// | Windows | [GetSystemTimeAsFileTime] | +/// | Windows | [GetSystemTimePreciseAsFileTime] / [GetSystemTimeAsFileTime] | /// -/// [clock_time_get (Realtime Clock)]: https://nuxi.nl/cloudabi/#clock_time_get +/// [currently]: crate::io#platform-specific-behavior /// [`insecure_time` usercall]: https://edp.fortanix.com/docs/api/fortanix_sgx_abi/struct.Usercalls.html#method.insecure_time /// [timekeeping in SGX]: https://edp.fortanix.com/docs/concepts/rust-std/#codestdtimecode -/// [gettimeofday]: http://man7.org/linux/man-pages/man2/gettimeofday.2.html +/// [gettimeofday]: https://man7.org/linux/man-pages/man2/gettimeofday.2.html /// [clock_gettime (Realtime Clock)]: https://linux.die.net/man/3/clock_gettime /// [__wasi_clock_time_get (Realtime Clock)]: https://github.com/WebAssembly/WASI/blob/master/phases/snapshot/docs.md#clock_time_get +/// [GetSystemTimePreciseAsFileTime]: https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-getsystemtimepreciseasfiletime /// [GetSystemTimeAsFileTime]: https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-getsystemtimeasfiletime /// /// **Disclaimer:** These system calls might change over time. /// +/// > Note: mathematical operations like [`add`] may panic if the underlying +/// > structure cannot represent the new point in time. +/// +/// [`add`]: SystemTime::add #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] #[stable(feature = "time2", since = "1.8.0")] pub struct SystemTime(time::SystemTime); @@ -199,54 +270,22 @@ impl Instant { /// /// let now = Instant::now(); /// ``` + #[must_use] #[stable(feature = "time2", since = "1.8.0")] pub fn now() -> Instant { - let os_now = time::Instant::now(); - - // And here we come upon a sad state of affairs. The whole point of - // `Instant` is that it's monotonically increasing. We've found in the - // wild, however, that it's not actually monotonically increasing for - // one reason or another. These appear to be OS and hardware level bugs, - // and there's not really a whole lot we can do about them. Here's a - // taste of what we've found: - // - // * #48514 - OpenBSD, x86_64 - // * #49281 - linux arm64 and s390x - // * #51648 - windows, x86 - // * #56560 - windows, x86_64, AWS - // * #56612 - windows, x86, vm (?) - // * #56940 - linux, arm64 - // * https://bugzilla.mozilla.org/show_bug.cgi?id=1487778 - a similar - // Firefox bug - // - // It seems that this just happens a lot in the wild. - // We're seeing panics across various platforms where consecutive calls - // to `Instant::now`, such as via the `elapsed` function, are panicking - // as they're going backwards. Placed here is a last-ditch effort to try - // to fix things up. We keep a global "latest now" instance which is - // returned instead of what the OS says if the OS goes backwards. - // - // To hopefully mitigate the impact of this, a few platforms are - // whitelisted as "these at least haven't gone backwards yet". - if time::Instant::actually_monotonic() { - return Instant(os_now); - } - - static LOCK: Mutex = Mutex::new(); - static mut LAST_NOW: time::Instant = time::Instant::zero(); - unsafe { - let _lock = LOCK.lock(); - let now = cmp::max(LAST_NOW, os_now); - LAST_NOW = now; - Instant(now) - } + Instant(time::Instant::now()) } - /// Returns the amount of time elapsed from another instant to this one. + /// Returns the amount of time elapsed from another instant to this one, + /// or zero duration if that instant is later than this one. /// /// # Panics /// - /// This function will panic if `earlier` is later than `self`. + /// Previous rust versions panicked when `earlier` was later than `self`. Currently this + /// method saturates. Future versions may reintroduce the panic in some circumstances. + /// See [Monotonicity]. + /// + /// [Monotonicity]: Instant#monotonicity /// /// # Examples /// @@ -258,15 +297,22 @@ impl Instant { /// sleep(Duration::new(1, 0)); /// let new_now = Instant::now(); /// println!("{:?}", new_now.duration_since(now)); + /// println!("{:?}", now.duration_since(new_now)); // 0ns /// ``` + #[must_use] #[stable(feature = "time2", since = "1.8.0")] pub fn duration_since(&self, earlier: Instant) -> Duration { - self.0.checked_sub_instant(&earlier.0).expect("supplied instant is later than self") + self.checked_duration_since(earlier).unwrap_or_default() } /// Returns the amount of time elapsed from another instant to this one, /// or None if that instant is later than this one. /// + /// Due to [monotonicity bugs], even under correct logical ordering of the passed `Instant`s, + /// this method can return `None`. + /// + /// [monotonicity bugs]: Instant#monotonicity + /// /// # Examples /// /// ```no_run @@ -279,6 +325,7 @@ impl Instant { /// println!("{:?}", new_now.checked_duration_since(now)); /// println!("{:?}", now.checked_duration_since(new_now)); // None /// ``` + #[must_use] #[stable(feature = "checked_duration_since", since = "1.39.0")] pub fn checked_duration_since(&self, earlier: Instant) -> Option { self.0.checked_sub_instant(&earlier.0) @@ -299,18 +346,21 @@ impl Instant { /// println!("{:?}", new_now.saturating_duration_since(now)); /// println!("{:?}", now.saturating_duration_since(new_now)); // 0ns /// ``` + #[must_use] #[stable(feature = "checked_duration_since", since = "1.39.0")] pub fn saturating_duration_since(&self, earlier: Instant) -> Duration { - self.checked_duration_since(earlier).unwrap_or(Duration::new(0, 0)) + self.checked_duration_since(earlier).unwrap_or_default() } /// Returns the amount of time elapsed since this instant was created. /// /// # Panics /// - /// This function may panic if the current time is earlier than this - /// instant, which is something that can happen if an `Instant` is - /// produced synthetically. + /// Previous rust versions panicked when the current time was earlier than self. Currently this + /// method returns a Duration of zero in that case. Future versions may reintroduce the panic. + /// See [Monotonicity]. + /// + /// [Monotonicity]: Instant#monotonicity /// /// # Examples /// @@ -323,6 +373,7 @@ impl Instant { /// sleep(three_secs); /// assert!(instant.elapsed() >= three_secs); /// ``` + #[must_use] #[stable(feature = "time2", since = "1.8.0")] pub fn elapsed(&self) -> Duration { Instant::now() - *self @@ -352,9 +403,7 @@ impl Add for Instant { /// # Panics /// /// This function may panic if the resulting point in time cannot be represented by the - /// underlying data structure. See [`checked_add`] for a version without panic. - /// - /// [`checked_add`]: ../../std/time/struct.Instant.html#method.checked_add + /// underlying data structure. See [`Instant::checked_add`] for a version without panic. fn add(self, other: Duration) -> Instant { self.checked_add(other).expect("overflow when adding duration to instant") } @@ -387,6 +436,16 @@ impl SubAssign for Instant { impl Sub for Instant { type Output = Duration; + /// Returns the amount of time elapsed from another instant to this one, + /// or zero duration if that instant is later than this one. + /// + /// # Panics + /// + /// Previous rust versions panicked when `other` was later than `self`. Currently this + /// method saturates. Future versions may reintroduce the panic in some circumstances. + /// See [Monotonicity]. + /// + /// [Monotonicity]: Instant#monotonicity fn sub(self, other: Instant) -> Duration { self.duration_since(other) } @@ -431,6 +490,7 @@ impl SystemTime { /// /// let sys_time = SystemTime::now(); /// ``` + #[must_use] #[stable(feature = "time2", since = "1.8.0")] pub fn now() -> SystemTime { SystemTime(time::SystemTime::now()) @@ -443,26 +503,22 @@ impl SystemTime { /// as the system clock being adjusted either forwards or backwards). /// [`Instant`] can be used to measure elapsed time without this risk of failure. /// - /// If successful, [`Ok`]`(`[`Duration`]`)` is returned where the duration represents + /// If successful, [Ok]\([Duration]) is returned where the duration represents /// the amount of time elapsed from the specified measurement to this one. /// /// Returns an [`Err`] if `earlier` is later than `self`, and the error /// contains how far from `self` the time is. /// - /// [`Ok`]: ../../std/result/enum.Result.html#variant.Ok - /// [`Duration`]: ../../std/time/struct.Duration.html - /// [`Err`]: ../../std/result/enum.Result.html#variant.Err - /// [`Instant`]: ../../std/time/struct.Instant.html - /// /// # Examples /// - /// ``` + /// ```no_run /// use std::time::SystemTime; /// /// let sys_time = SystemTime::now(); - /// let difference = sys_time.duration_since(sys_time) - /// .expect("Clock may have gone backwards"); - /// println!("{:?}", difference); + /// let new_sys_time = SystemTime::now(); + /// let difference = new_sys_time.duration_since(sys_time) + /// .expect("Clock may have gone backwards"); + /// println!("{difference:?}"); /// ``` #[stable(feature = "time2", since = "1.8.0")] pub fn duration_since(&self, earlier: SystemTime) -> Result { @@ -474,7 +530,7 @@ impl SystemTime { /// /// This function may fail as the underlying system clock is susceptible to /// drift and updates (e.g., the system clock could go backwards), so this - /// function may not always succeed. If successful, [`Ok`]`(`[`Duration`]`)` is + /// function might not always succeed. If successful, [Ok]\([Duration]) is /// returned where the duration represents the amount of time elapsed from /// this time measurement to the current time. /// @@ -483,11 +539,6 @@ impl SystemTime { /// Returns an [`Err`] if `self` is later than the current system time, and /// the error contains how far from the current system time `self` is. /// - /// [`Ok`]: ../../std/result/enum.Result.html#variant.Ok - /// [`Duration`]: ../../std/time/struct.Duration.html - /// [`Err`]: ../../std/result/enum.Result.html#variant.Err - /// [`Instant`]: ../../std/time/struct.Instant.html - /// /// # Examples /// /// ```no_run @@ -528,9 +579,7 @@ impl Add for SystemTime { /// # Panics /// /// This function may panic if the resulting point in time cannot be represented by the - /// underlying data structure. See [`checked_add`] for a version without panic. - /// - /// [`checked_add`]: ../../std/time/struct.SystemTime.html#method.checked_add + /// underlying data structure. See [`SystemTime::checked_add`] for a version without panic. fn add(self, dur: Duration) -> SystemTime { self.checked_add(dur).expect("overflow when adding duration to instant") } @@ -575,8 +624,6 @@ impl fmt::Debug for SystemTime { /// measurement lies, and using `UNIX_EPOCH + duration` can be used to create a /// [`SystemTime`] instance to represent another fixed point in time. /// -/// [`SystemTime`]: ../../std/time/struct.SystemTime.html -/// /// # Examples /// /// ```no_run @@ -594,13 +641,9 @@ impl SystemTimeError { /// Returns the positive duration which represents how far forward the /// second system time was from the first. /// - /// A `SystemTimeError` is returned from the [`duration_since`] and [`elapsed`] - /// methods of [`SystemTime`] whenever the second system time represents a point later - /// in time than the `self` of the method call. - /// - /// [`duration_since`]: ../../std/time/struct.SystemTime.html#method.duration_since - /// [`elapsed`]: ../../std/time/struct.SystemTime.html#method.elapsed - /// [`SystemTime`]: ../../std/time/struct.SystemTime.html + /// A `SystemTimeError` is returned from the [`SystemTime::duration_since`] + /// and [`SystemTime::elapsed`] methods whenever the second system time + /// represents a point later in time than the `self` of the method call. /// /// # Examples /// @@ -616,6 +659,7 @@ impl SystemTimeError { /// Err(e) => println!("SystemTimeError difference: {:?}", e.duration()), /// } /// ``` + #[must_use] #[stable(feature = "time2", since = "1.8.0")] pub fn duration(&self) -> Duration { self.0 @@ -643,177 +687,8 @@ impl FromInner for SystemTime { } } -impl FromInner for SystemTime { - fn from_inner(time: sys::time::SystemTime) -> SystemTime { - SystemTime(time::SystemTime::from_inner(time)) - } -} - -#[cfg(test)] -mod tests { - use super::{Duration, Instant, SystemTime, UNIX_EPOCH}; - - macro_rules! assert_almost_eq { - ($a:expr, $b:expr) => {{ - let (a, b) = ($a, $b); - if a != b { - let (a, b) = if a > b { (a, b) } else { (b, a) }; - assert!(a - Duration::new(0, 1000) <= b, "{:?} is not almost equal to {:?}", a, b); - } - }}; - } - - #[test] - fn instant_monotonic() { - let a = Instant::now(); - let b = Instant::now(); - assert!(b >= a); - } - - #[test] - fn instant_elapsed() { - let a = Instant::now(); - a.elapsed(); - } - - #[test] - fn instant_math() { - let a = Instant::now(); - let b = Instant::now(); - println!("a: {:?}", a); - println!("b: {:?}", b); - let dur = b.duration_since(a); - println!("dur: {:?}", dur); - assert_almost_eq!(b - dur, a); - assert_almost_eq!(a + dur, b); - - let second = Duration::new(1, 0); - assert_almost_eq!(a - second + second, a); - assert_almost_eq!(a.checked_sub(second).unwrap().checked_add(second).unwrap(), a); - - // checked_add_duration will not panic on overflow - let mut maybe_t = Some(Instant::now()); - let max_duration = Duration::from_secs(u64::max_value()); - // in case `Instant` can store `>= now + max_duration`. - for _ in 0..2 { - maybe_t = maybe_t.and_then(|t| t.checked_add(max_duration)); - } - assert_eq!(maybe_t, None); - - // checked_add_duration calculates the right time and will work for another year - let year = Duration::from_secs(60 * 60 * 24 * 365); - assert_eq!(a + year, a.checked_add(year).unwrap()); - } - - #[test] - fn instant_math_is_associative() { - let now = Instant::now(); - let offset = Duration::from_millis(5); - // Changing the order of instant math shouldn't change the results, - // especially when the expression reduces to X + identity. - assert_eq!((now + offset) - now, (now - now) + offset); - } - - #[test] - #[should_panic] - fn instant_duration_since_panic() { - let a = Instant::now(); - (a - Duration::new(1, 0)).duration_since(a); - } - - #[test] - fn instant_checked_duration_since_nopanic() { - let now = Instant::now(); - let earlier = now - Duration::new(1, 0); - let later = now + Duration::new(1, 0); - assert_eq!(earlier.checked_duration_since(now), None); - assert_eq!(later.checked_duration_since(now), Some(Duration::new(1, 0))); - assert_eq!(now.checked_duration_since(now), Some(Duration::new(0, 0))); - } - - #[test] - fn instant_saturating_duration_since_nopanic() { - let a = Instant::now(); - let ret = (a - Duration::new(1, 0)).saturating_duration_since(a); - assert_eq!(ret, Duration::new(0, 0)); - } - - #[test] - fn system_time_math() { - let a = SystemTime::now(); - let b = SystemTime::now(); - match b.duration_since(a) { - Ok(dur) if dur == Duration::new(0, 0) => { - assert_almost_eq!(a, b); - } - Ok(dur) => { - assert!(b > a); - assert_almost_eq!(b - dur, a); - assert_almost_eq!(a + dur, b); - } - Err(dur) => { - let dur = dur.duration(); - assert!(a > b); - assert_almost_eq!(b + dur, a); - assert_almost_eq!(a - dur, b); - } - } - - let second = Duration::new(1, 0); - assert_almost_eq!(a.duration_since(a - second).unwrap(), second); - assert_almost_eq!(a.duration_since(a + second).unwrap_err().duration(), second); - - assert_almost_eq!(a - second + second, a); - assert_almost_eq!(a.checked_sub(second).unwrap().checked_add(second).unwrap(), a); - - let one_second_from_epoch = UNIX_EPOCH + Duration::new(1, 0); - let one_second_from_epoch2 = - UNIX_EPOCH + Duration::new(0, 500_000_000) + Duration::new(0, 500_000_000); - assert_eq!(one_second_from_epoch, one_second_from_epoch2); - - // checked_add_duration will not panic on overflow - let mut maybe_t = Some(SystemTime::UNIX_EPOCH); - let max_duration = Duration::from_secs(u64::max_value()); - // in case `SystemTime` can store `>= UNIX_EPOCH + max_duration`. - for _ in 0..2 { - maybe_t = maybe_t.and_then(|t| t.checked_add(max_duration)); - } - assert_eq!(maybe_t, None); - - // checked_add_duration calculates the right time and will work for another year - let year = Duration::from_secs(60 * 60 * 24 * 365); - assert_eq!(a + year, a.checked_add(year).unwrap()); - } - - #[test] - fn system_time_elapsed() { - let a = SystemTime::now(); - drop(a.elapsed()); - } - - #[test] - fn since_epoch() { - let ts = SystemTime::now(); - let a = ts.duration_since(UNIX_EPOCH + Duration::new(1, 0)).unwrap(); - let b = ts.duration_since(UNIX_EPOCH).unwrap(); - assert!(b > a); - assert_eq!(b - a, Duration::new(1, 0)); - - let thirty_years = Duration::new(1, 0) * 60 * 60 * 24 * 365 * 30; - - // Right now for CI this test is run in an emulator, and apparently the - // aarch64 emulator's sense of time is that we're still living in the - // 70s. - // - // Otherwise let's assume that we're all running computers later than - // 2000. - if !cfg!(target_arch = "aarch64") { - assert!(a > thirty_years); - } - - // let's assume that we're all running computers earlier than 2090. - // Should give us ~70 years to fix this! - let hundred_twenty_years = thirty_years * 4; - assert!(a < hundred_twenty_years); +impl IntoInner for SystemTime { + fn into_inner(self) -> time::SystemTime { + self.0 } } diff --git a/crux-mir/lib/std/src/time/tests.rs b/crux-mir/lib/std/src/time/tests.rs new file mode 100644 index 000000000..2e64ae59a --- /dev/null +++ b/crux-mir/lib/std/src/time/tests.rs @@ -0,0 +1,245 @@ +use super::{Duration, Instant, SystemTime, UNIX_EPOCH}; +#[cfg(not(target_arch = "wasm32"))] +use test::{black_box, Bencher}; + +macro_rules! assert_almost_eq { + ($a:expr, $b:expr) => {{ + let (a, b) = ($a, $b); + if a != b { + let (a, b) = if a > b { (a, b) } else { (b, a) }; + assert!(a - Duration::from_micros(1) <= b, "{:?} is not almost equal to {:?}", a, b); + } + }}; +} + +#[test] +fn instant_monotonic() { + let a = Instant::now(); + loop { + let b = Instant::now(); + assert!(b >= a); + if b > a { + break; + } + } +} + +#[test] +#[cfg(not(target_arch = "wasm32"))] +fn instant_monotonic_concurrent() -> crate::thread::Result<()> { + let threads: Vec<_> = (0..8) + .map(|_| { + crate::thread::spawn(|| { + let mut old = Instant::now(); + let count = if cfg!(miri) { 1_000 } else { 5_000_000 }; + for _ in 0..count { + let new = Instant::now(); + assert!(new >= old); + old = new; + } + }) + }) + .collect(); + for t in threads { + t.join()?; + } + Ok(()) +} + +#[test] +fn instant_elapsed() { + let a = Instant::now(); + let _ = a.elapsed(); +} + +#[test] +fn instant_math() { + let a = Instant::now(); + let b = Instant::now(); + println!("a: {a:?}"); + println!("b: {b:?}"); + let dur = b.duration_since(a); + println!("dur: {dur:?}"); + assert_almost_eq!(b - dur, a); + assert_almost_eq!(a + dur, b); + + let second = Duration::SECOND; + assert_almost_eq!(a - second + second, a); + assert_almost_eq!(a.checked_sub(second).unwrap().checked_add(second).unwrap(), a); + + // checked_add_duration will not panic on overflow + let mut maybe_t = Some(Instant::now()); + let max_duration = Duration::from_secs(u64::MAX); + // in case `Instant` can store `>= now + max_duration`. + for _ in 0..2 { + maybe_t = maybe_t.and_then(|t| t.checked_add(max_duration)); + } + assert_eq!(maybe_t, None); + + // checked_add_duration calculates the right time and will work for another year + let year = Duration::from_secs(60 * 60 * 24 * 365); + assert_eq!(a + year, a.checked_add(year).unwrap()); +} + +#[test] +fn instant_math_is_associative() { + let now = Instant::now(); + let offset = Duration::from_millis(5); + // Changing the order of instant math shouldn't change the results, + // especially when the expression reduces to X + identity. + assert_eq!((now + offset) - now, (now - now) + offset); + + // On any platform, `Instant` should have the same resolution as `Duration` (e.g. 1 nanosecond) + // or better. Otherwise, math will be non-associative (see #91417). + let now = Instant::now(); + let provided_offset = Duration::from_nanos(1); + let later = now + provided_offset; + let measured_offset = later - now; + assert_eq!(measured_offset, provided_offset); +} + +#[test] +fn instant_duration_since_saturates() { + let a = Instant::now(); + assert_eq!((a - Duration::SECOND).duration_since(a), Duration::ZERO); +} + +#[test] +fn instant_checked_duration_since_nopanic() { + let now = Instant::now(); + let earlier = now - Duration::SECOND; + let later = now + Duration::SECOND; + assert_eq!(earlier.checked_duration_since(now), None); + assert_eq!(later.checked_duration_since(now), Some(Duration::SECOND)); + assert_eq!(now.checked_duration_since(now), Some(Duration::ZERO)); +} + +#[test] +fn instant_saturating_duration_since_nopanic() { + let a = Instant::now(); + #[allow(deprecated, deprecated_in_future)] + let ret = (a - Duration::SECOND).saturating_duration_since(a); + assert_eq!(ret, Duration::ZERO); +} + +#[test] +fn system_time_math() { + let a = SystemTime::now(); + let b = SystemTime::now(); + match b.duration_since(a) { + Ok(Duration::ZERO) => { + assert_almost_eq!(a, b); + } + Ok(dur) => { + assert!(b > a); + assert_almost_eq!(b - dur, a); + assert_almost_eq!(a + dur, b); + } + Err(dur) => { + let dur = dur.duration(); + assert!(a > b); + assert_almost_eq!(b + dur, a); + assert_almost_eq!(a - dur, b); + } + } + + let second = Duration::SECOND; + assert_almost_eq!(a.duration_since(a - second).unwrap(), second); + assert_almost_eq!(a.duration_since(a + second).unwrap_err().duration(), second); + + assert_almost_eq!(a - second + second, a); + assert_almost_eq!(a.checked_sub(second).unwrap().checked_add(second).unwrap(), a); + + let one_second_from_epoch = UNIX_EPOCH + Duration::SECOND; + let one_second_from_epoch2 = + UNIX_EPOCH + Duration::from_millis(500) + Duration::from_millis(500); + assert_eq!(one_second_from_epoch, one_second_from_epoch2); + + // checked_add_duration will not panic on overflow + let mut maybe_t = Some(SystemTime::UNIX_EPOCH); + let max_duration = Duration::from_secs(u64::MAX); + // in case `SystemTime` can store `>= UNIX_EPOCH + max_duration`. + for _ in 0..2 { + maybe_t = maybe_t.and_then(|t| t.checked_add(max_duration)); + } + assert_eq!(maybe_t, None); + + // checked_add_duration calculates the right time and will work for another year + let year = Duration::from_secs(60 * 60 * 24 * 365); + assert_eq!(a + year, a.checked_add(year).unwrap()); +} + +#[test] +fn system_time_elapsed() { + let a = SystemTime::now(); + drop(a.elapsed()); +} + +#[test] +fn since_epoch() { + let ts = SystemTime::now(); + let a = ts.duration_since(UNIX_EPOCH + Duration::SECOND).unwrap(); + let b = ts.duration_since(UNIX_EPOCH).unwrap(); + assert!(b > a); + assert_eq!(b - a, Duration::SECOND); + + let thirty_years = Duration::SECOND * 60 * 60 * 24 * 365 * 30; + + // Right now for CI this test is run in an emulator, and apparently the + // aarch64 emulator's sense of time is that we're still living in the + // 70s. This is also true for riscv (also qemu) + // + // Otherwise let's assume that we're all running computers later than + // 2000. + if !cfg!(target_arch = "aarch64") && !cfg!(target_arch = "riscv64") { + assert!(a > thirty_years); + } + + // let's assume that we're all running computers earlier than 2090. + // Should give us ~70 years to fix this! + let hundred_twenty_years = thirty_years * 4; + assert!(a < hundred_twenty_years); +} + +macro_rules! bench_instant_threaded { + ($bench_name:ident, $thread_count:expr) => { + #[bench] + #[cfg(not(target_arch = "wasm32"))] + fn $bench_name(b: &mut Bencher) -> crate::thread::Result<()> { + use crate::sync::atomic::{AtomicBool, Ordering}; + use crate::sync::Arc; + + let running = Arc::new(AtomicBool::new(true)); + + let threads: Vec<_> = (0..$thread_count) + .map(|_| { + let flag = Arc::clone(&running); + crate::thread::spawn(move || { + while flag.load(Ordering::Relaxed) { + black_box(Instant::now()); + } + }) + }) + .collect(); + + b.iter(|| { + let a = Instant::now(); + let b = Instant::now(); + assert!(b >= a); + }); + + running.store(false, Ordering::Relaxed); + + for t in threads { + t.join()?; + } + Ok(()) + } + }; +} + +bench_instant_threaded!(instant_contention_01_threads, 0); +bench_instant_threaded!(instant_contention_02_threads, 1); +bench_instant_threaded!(instant_contention_04_threads, 3); +bench_instant_threaded!(instant_contention_08_threads, 7); +bench_instant_threaded!(instant_contention_16_threads, 15); diff --git a/crux-mir/lib/std/tests/env.rs b/crux-mir/lib/std/tests/env.rs index c94fc4117..aae2c49d8 100644 --- a/crux-mir/lib/std/tests/env.rs +++ b/crux-mir/lib/std/tests/env.rs @@ -1,12 +1,24 @@ use std::env::*; use std::ffi::{OsStr, OsString}; -use rand::distributions::Alphanumeric; -use rand::{thread_rng, Rng}; +use rand::distributions::{Alphanumeric, DistString}; +/// Copied from `std::test_helpers::test_rng`, since these tests rely on the +/// seed not being the same for every RNG invocation too. +#[track_caller] +pub(crate) fn test_rng() -> rand_xorshift::XorShiftRng { + use core::hash::{BuildHasher, Hash, Hasher}; + let mut hasher = std::collections::hash_map::RandomState::new().build_hasher(); + core::panic::Location::caller().hash(&mut hasher); + let hc64 = hasher.finish(); + let seed_vec = hc64.to_le_bytes().into_iter().chain(0u8..8).collect::>(); + let seed: [u8; 16] = seed_vec.as_slice().try_into().unwrap(); + rand::SeedableRng::from_seed(seed) +} + +#[track_caller] fn make_rand_name() -> OsString { - let rng = thread_rng(); - let n = format!("TEST{}", rng.sample_iter(&Alphanumeric).take(10).collect::()); + let n = format!("TEST{}", Alphanumeric.sample_string(&mut test_rng(), 10)); let n = OsString::from(n); assert!(var_os(&n).is_none()); n @@ -76,3 +88,65 @@ fn test_env_set_var() { assert!(vars_os().any(|(k, v)| { &*k == &*n && &*v == "VALUE" })); } + +#[test] +#[cfg_attr(not(any(unix, windows)), ignore, allow(unused))] +#[allow(deprecated)] +fn env_home_dir() { + use std::path::PathBuf; + + fn var_to_os_string(var: Result) -> Option { + match var { + Ok(var) => Some(OsString::from(var)), + Err(VarError::NotUnicode(var)) => Some(var), + _ => None, + } + } + + cfg_if::cfg_if! { + if #[cfg(unix)] { + let oldhome = var_to_os_string(var("HOME")); + + set_var("HOME", "/home/MountainView"); + assert_eq!(home_dir(), Some(PathBuf::from("/home/MountainView"))); + + remove_var("HOME"); + if cfg!(target_os = "android") { + assert!(home_dir().is_none()); + } else { + // When HOME is not set, some platforms return `None`, + // but others return `Some` with a default. + // Just check that it is not "/home/MountainView". + assert_ne!(home_dir(), Some(PathBuf::from("/home/MountainView"))); + } + + if let Some(oldhome) = oldhome { set_var("HOME", oldhome); } + } else if #[cfg(windows)] { + let oldhome = var_to_os_string(var("HOME")); + let olduserprofile = var_to_os_string(var("USERPROFILE")); + + remove_var("HOME"); + remove_var("USERPROFILE"); + + assert!(home_dir().is_some()); + + set_var("HOME", "/home/MountainView"); + assert_eq!(home_dir(), Some(PathBuf::from("/home/MountainView"))); + + remove_var("HOME"); + + set_var("USERPROFILE", "/home/MountainView"); + assert_eq!(home_dir(), Some(PathBuf::from("/home/MountainView"))); + + set_var("HOME", "/home/MountainView"); + set_var("USERPROFILE", "/home/PaloAlto"); + assert_eq!(home_dir(), Some(PathBuf::from("/home/MountainView"))); + + remove_var("HOME"); + remove_var("USERPROFILE"); + + if let Some(oldhome) = oldhome { set_var("HOME", oldhome); } + if let Some(olduserprofile) = olduserprofile { set_var("USERPROFILE", olduserprofile); } + } + } +} diff --git a/crux-mir/lib/std/tests/run-time-detect.rs b/crux-mir/lib/std/tests/run-time-detect.rs index 2e6d1bc8e..1a2c12556 100644 --- a/crux-mir/lib/std/tests/run-time-detect.rs +++ b/crux-mir/lib/std/tests/run-time-detect.rs @@ -1,12 +1,10 @@ -//! These tests just check that the macros are available in libstd. +//! These tests just check that the macros are available in std. #![cfg_attr( any( all(target_arch = "arm", any(target_os = "linux", target_os = "android")), - all(target_arch = "aarch64", any(target_os = "linux", target_os = "android")), all(target_arch = "powerpc", target_os = "linux"), all(target_arch = "powerpc64", target_os = "linux"), - any(target_arch = "x86", target_arch = "x86_64"), ), feature(stdsimd) )] @@ -14,81 +12,151 @@ #[test] #[cfg(all(target_arch = "arm", any(target_os = "linux", target_os = "android")))] fn arm_linux() { + use std::arch::is_arm_feature_detected; + // tidy-alphabetical-start + println!("aes: {}", is_arm_feature_detected!("aes")); + println!("crc: {}", is_arm_feature_detected!("crc")); + println!("crypto: {}", is_arm_feature_detected!("crypto")); println!("neon: {}", is_arm_feature_detected!("neon")); println!("pmull: {}", is_arm_feature_detected!("pmull")); + println!("sha2: {}", is_arm_feature_detected!("sha2")); + // tidy-alphabetical-end } #[test] #[cfg(all(target_arch = "aarch64", any(target_os = "linux", target_os = "android")))] fn aarch64_linux() { - println!("fp: {}", is_aarch64_feature_detected!("fp")); - println!("fp16: {}", is_aarch64_feature_detected!("fp16")); - println!("neon: {}", is_aarch64_feature_detected!("neon")); + use std::arch::is_aarch64_feature_detected; + // tidy-alphabetical-start + println!("aes: {}", is_aarch64_feature_detected!("aes")); println!("asimd: {}", is_aarch64_feature_detected!("asimd")); - println!("sve: {}", is_aarch64_feature_detected!("sve")); + println!("bf16: {}", is_aarch64_feature_detected!("bf16")); + println!("bti: {}", is_aarch64_feature_detected!("bti")); println!("crc: {}", is_aarch64_feature_detected!("crc")); - println!("crypto: {}", is_aarch64_feature_detected!("crypto")); + println!("dit: {}", is_aarch64_feature_detected!("dit")); + println!("dotprod: {}", is_aarch64_feature_detected!("dotprod")); + println!("dpb2: {}", is_aarch64_feature_detected!("dpb2")); + println!("dpb: {}", is_aarch64_feature_detected!("dpb")); + println!("f32mm: {}", is_aarch64_feature_detected!("f32mm")); + println!("f64mm: {}", is_aarch64_feature_detected!("f64mm")); + println!("fcma: {}", is_aarch64_feature_detected!("fcma")); + println!("fhm: {}", is_aarch64_feature_detected!("fhm")); + println!("flagm: {}", is_aarch64_feature_detected!("flagm")); + println!("fp16: {}", is_aarch64_feature_detected!("fp16")); + println!("frintts: {}", is_aarch64_feature_detected!("frintts")); + println!("i8mm: {}", is_aarch64_feature_detected!("i8mm")); + println!("jsconv: {}", is_aarch64_feature_detected!("jsconv")); + println!("lse2: {}", is_aarch64_feature_detected!("lse2")); println!("lse: {}", is_aarch64_feature_detected!("lse")); - println!("rdm: {}", is_aarch64_feature_detected!("rdm")); + println!("mte: {}", is_aarch64_feature_detected!("mte")); + println!("neon: {}", is_aarch64_feature_detected!("neon")); + println!("paca: {}", is_aarch64_feature_detected!("paca")); + println!("pacg: {}", is_aarch64_feature_detected!("pacg")); + println!("pmull: {}", is_aarch64_feature_detected!("pmull")); + println!("rand: {}", is_aarch64_feature_detected!("rand")); + println!("rcpc2: {}", is_aarch64_feature_detected!("rcpc2")); println!("rcpc: {}", is_aarch64_feature_detected!("rcpc")); - println!("dotprod: {}", is_aarch64_feature_detected!("dotprod")); + println!("rdm: {}", is_aarch64_feature_detected!("rdm")); + println!("sb: {}", is_aarch64_feature_detected!("sb")); + println!("sha2: {}", is_aarch64_feature_detected!("sha2")); + println!("sha3: {}", is_aarch64_feature_detected!("sha3")); + println!("sm4: {}", is_aarch64_feature_detected!("sm4")); + println!("ssbs: {}", is_aarch64_feature_detected!("ssbs")); + println!("sve2-aes: {}", is_aarch64_feature_detected!("sve2-aes")); + println!("sve2-bitperm: {}", is_aarch64_feature_detected!("sve2-bitperm")); + println!("sve2-sha3: {}", is_aarch64_feature_detected!("sve2-sha3")); + println!("sve2-sm4: {}", is_aarch64_feature_detected!("sve2-sm4")); + println!("sve2: {}", is_aarch64_feature_detected!("sve2")); + println!("sve: {}", is_aarch64_feature_detected!("sve")); + println!("tme: {}", is_aarch64_feature_detected!("tme")); + // tidy-alphabetical-end } #[test] #[cfg(all(target_arch = "powerpc", target_os = "linux"))] fn powerpc_linux() { + use std::arch::is_powerpc_feature_detected; + // tidy-alphabetical-start println!("altivec: {}", is_powerpc_feature_detected!("altivec")); - println!("vsx: {}", is_powerpc_feature_detected!("vsx")); println!("power8: {}", is_powerpc_feature_detected!("power8")); + println!("vsx: {}", is_powerpc_feature_detected!("vsx")); + // tidy-alphabetical-end } #[test] #[cfg(all(target_arch = "powerpc64", target_os = "linux"))] fn powerpc64_linux() { + use std::arch::is_powerpc64_feature_detected; + // tidy-alphabetical-start println!("altivec: {}", is_powerpc64_feature_detected!("altivec")); - println!("vsx: {}", is_powerpc64_feature_detected!("vsx")); println!("power8: {}", is_powerpc64_feature_detected!("power8")); + println!("vsx: {}", is_powerpc64_feature_detected!("vsx")); + // tidy-alphabetical-end } #[test] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] fn x86_all() { + use std::arch::is_x86_feature_detected; + + // the below is the set of features we can test at runtime, but don't actually + // use to gate anything and are thus not part of the X86_ALLOWED_FEATURES list + + println!("abm: {:?}", is_x86_feature_detected!("abm")); // this is a synonym for lzcnt but we test it anyways + println!("mmx: {:?}", is_x86_feature_detected!("mmx")); + println!("tsc: {:?}", is_x86_feature_detected!("tsc")); + + // the below is in alphabetical order and matches + // the order of X86_ALLOWED_FEATURES in rustc_codegen_ssa's target_features.rs + + // tidy-alphabetical-start + println!("adx: {:?}", is_x86_feature_detected!("adx")); println!("aes: {:?}", is_x86_feature_detected!("aes")); - println!("pcmulqdq: {:?}", is_x86_feature_detected!("pclmulqdq")); + println!("avx2: {:?}", is_x86_feature_detected!("avx2")); + println!("avx512bf16: {:?}", is_x86_feature_detected!("avx512bf16")); + println!("avx512bitalg: {:?}", is_x86_feature_detected!("avx512bitalg")); + println!("avx512bw: {:?}", is_x86_feature_detected!("avx512bw")); + println!("avx512cd: {:?}", is_x86_feature_detected!("avx512cd")); + println!("avx512dq: {:?}", is_x86_feature_detected!("avx512dq")); + println!("avx512er: {:?}", is_x86_feature_detected!("avx512er")); + println!("avx512f: {:?}", is_x86_feature_detected!("avx512f")); + println!("avx512gfni: {:?}", is_x86_feature_detected!("avx512gfni")); + println!("avx512ifma: {:?}", is_x86_feature_detected!("avx512ifma")); + println!("avx512pf: {:?}", is_x86_feature_detected!("avx512pf")); + println!("avx512vaes: {:?}", is_x86_feature_detected!("avx512vaes")); + println!("avx512vbmi2: {:?}", is_x86_feature_detected!("avx512vbmi2")); + println!("avx512vbmi: {:?}", is_x86_feature_detected!("avx512vbmi")); + println!("avx512vl: {:?}", is_x86_feature_detected!("avx512vl")); + println!("avx512vnni: {:?}", is_x86_feature_detected!("avx512vnni")); + println!("avx512vp2intersect: {:?}", is_x86_feature_detected!("avx512vp2intersect")); + println!("avx512vpclmulqdq: {:?}", is_x86_feature_detected!("avx512vpclmulqdq")); + println!("avx512vpopcntdq: {:?}", is_x86_feature_detected!("avx512vpopcntdq")); + println!("avx: {:?}", is_x86_feature_detected!("avx")); + println!("bmi1: {:?}", is_x86_feature_detected!("bmi1")); + println!("bmi2: {:?}", is_x86_feature_detected!("bmi2")); + println!("cmpxchg16b: {:?}", is_x86_feature_detected!("cmpxchg16b")); + println!("f16c: {:?}", is_x86_feature_detected!("f16c")); + println!("fma: {:?}", is_x86_feature_detected!("fma")); + println!("fxsr: {:?}", is_x86_feature_detected!("fxsr")); + println!("lzcnt: {:?}", is_x86_feature_detected!("lzcnt")); + //println!("movbe: {:?}", is_x86_feature_detected!("movbe")); // movbe is unsupported as a target feature + println!("pclmulqdq: {:?}", is_x86_feature_detected!("pclmulqdq")); + println!("popcnt: {:?}", is_x86_feature_detected!("popcnt")); println!("rdrand: {:?}", is_x86_feature_detected!("rdrand")); println!("rdseed: {:?}", is_x86_feature_detected!("rdseed")); - println!("tsc: {:?}", is_x86_feature_detected!("tsc")); - println!("mmx: {:?}", is_x86_feature_detected!("mmx")); - println!("sse: {:?}", is_x86_feature_detected!("sse")); + println!("rtm: {:?}", is_x86_feature_detected!("rtm")); + println!("sha: {:?}", is_x86_feature_detected!("sha")); println!("sse2: {:?}", is_x86_feature_detected!("sse2")); println!("sse3: {:?}", is_x86_feature_detected!("sse3")); - println!("ssse3: {:?}", is_x86_feature_detected!("ssse3")); println!("sse4.1: {:?}", is_x86_feature_detected!("sse4.1")); println!("sse4.2: {:?}", is_x86_feature_detected!("sse4.2")); println!("sse4a: {:?}", is_x86_feature_detected!("sse4a")); - println!("sha: {:?}", is_x86_feature_detected!("sha")); - println!("avx: {:?}", is_x86_feature_detected!("avx")); - println!("avx2: {:?}", is_x86_feature_detected!("avx2")); - println!("avx512f {:?}", is_x86_feature_detected!("avx512f")); - println!("avx512cd {:?}", is_x86_feature_detected!("avx512cd")); - println!("avx512er {:?}", is_x86_feature_detected!("avx512er")); - println!("avx512pf {:?}", is_x86_feature_detected!("avx512pf")); - println!("avx512bw {:?}", is_x86_feature_detected!("avx512bw")); - println!("avx512dq {:?}", is_x86_feature_detected!("avx512dq")); - println!("avx512vl {:?}", is_x86_feature_detected!("avx512vl")); - println!("avx512_ifma {:?}", is_x86_feature_detected!("avx512ifma")); - println!("avx512_vbmi {:?}", is_x86_feature_detected!("avx512vbmi")); - println!("avx512_vpopcntdq {:?}", is_x86_feature_detected!("avx512vpopcntdq")); - println!("fma: {:?}", is_x86_feature_detected!("fma")); - println!("bmi1: {:?}", is_x86_feature_detected!("bmi1")); - println!("bmi2: {:?}", is_x86_feature_detected!("bmi2")); - println!("abm: {:?}", is_x86_feature_detected!("abm")); - println!("lzcnt: {:?}", is_x86_feature_detected!("lzcnt")); + println!("sse: {:?}", is_x86_feature_detected!("sse")); + println!("ssse3: {:?}", is_x86_feature_detected!("ssse3")); println!("tbm: {:?}", is_x86_feature_detected!("tbm")); - println!("popcnt: {:?}", is_x86_feature_detected!("popcnt")); - println!("fxsr: {:?}", is_x86_feature_detected!("fxsr")); println!("xsave: {:?}", is_x86_feature_detected!("xsave")); + println!("xsavec: {:?}", is_x86_feature_detected!("xsavec")); println!("xsaveopt: {:?}", is_x86_feature_detected!("xsaveopt")); println!("xsaves: {:?}", is_x86_feature_detected!("xsaves")); - println!("xsavec: {:?}", is_x86_feature_detected!("xsavec")); + // tidy-alphabetical-end } diff --git a/crux-mir/lib/std/tests/thread.rs b/crux-mir/lib/std/tests/thread.rs new file mode 100644 index 000000000..754b264c6 --- /dev/null +++ b/crux-mir/lib/std/tests/thread.rs @@ -0,0 +1,16 @@ +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::Duration; + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn sleep() { + let finished = Arc::new(Mutex::new(false)); + let t_finished = finished.clone(); + thread::spawn(move || { + thread::sleep(Duration::new(u64::MAX, 0)); + *t_finished.lock().unwrap() = true; + }); + thread::sleep(Duration::from_millis(100)); + assert_eq!(*finished.lock().unwrap(), false); +} diff --git a/crux-mir/lib/std_detect/Cargo.toml b/crux-mir/lib/std_detect/Cargo.toml new file mode 100644 index 000000000..3a482564e --- /dev/null +++ b/crux-mir/lib/std_detect/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "std_detect" +version = "0.1.5" +authors = [ + "Alex Crichton ", + "Andrew Gallant ", + "Gonzalo Brito Gadeschi ", +] +description = "`std::detect` - Rust's standard library run-time CPU feature detection." +homepage = "https://github.com/rust-lang/stdarch" +repository = "https://github.com/rust-lang/stdarch" +readme = "README.md" +keywords = ["std", "run-time", "feature", "detection"] +categories = ["hardware-support"] +license = "MIT OR Apache-2.0" +edition = "2018" + +[badges] +is-it-maintained-issue-resolution = { repository = "rust-lang/stdarch" } +is-it-maintained-open-issues = { repository = "rust-lang/stdarch" } +maintenance = { status = "experimental" } + +[dependencies] +libc = { version = "0.2", optional = true, default-features = false } +cfg-if = "1.0.0" + +# When built as part of libstd +core = { version = "1.0.0", optional = true, package = "rustc-std-workspace-core" } +compiler_builtins = { version = "0.1.2", optional = true } +alloc = { version = "1.0.0", optional = true, package = "rustc-std-workspace-alloc" } + +[dev-dependencies] +auxv = "0.3.3" +cupid = "0.6.0" + +[features] +default = [ "std_detect_dlsym_getauxval", "std_detect_file_io" ] +std_detect_file_io = [ "libc" ] +std_detect_dlsym_getauxval = [ "libc" ] +std_detect_env_override = [ "libc" ] +rustc-dep-of-std = [ + "core", + "compiler_builtins", + "alloc", +] diff --git a/crux-mir/lib/bigint/LICENSE-APACHE b/crux-mir/lib/std_detect/LICENSE-APACHE similarity index 99% rename from crux-mir/lib/bigint/LICENSE-APACHE rename to crux-mir/lib/std_detect/LICENSE-APACHE index f8e5e5ea0..16fe87b06 100644 --- a/crux-mir/lib/bigint/LICENSE-APACHE +++ b/crux-mir/lib/std_detect/LICENSE-APACHE @@ -198,4 +198,4 @@ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. \ No newline at end of file +limitations under the License. diff --git a/crux-mir/lib/std_detect/LICENSE-MIT b/crux-mir/lib/std_detect/LICENSE-MIT new file mode 100644 index 000000000..52d82415d --- /dev/null +++ b/crux-mir/lib/std_detect/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2017 The Rust Project Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/crux-mir/lib/std_detect/README.md b/crux-mir/lib/std_detect/README.md new file mode 100644 index 000000000..bea7d941a --- /dev/null +++ b/crux-mir/lib/std_detect/README.md @@ -0,0 +1,73 @@ +`std::detect` - Rust's standard library run-time CPU feature detection +======= + +The private `std::detect` module implements run-time feature detection in Rust's +standard library. This allows detecting whether the CPU the binary runs on +supports certain features, like SIMD instructions. + +# Usage + +`std::detect` APIs are available as part of `libstd`. Prefer using it via the +standard library than through this crate. Unstable features of `std::detect` are +available on nightly Rust behind the `feature(stdsimd)` feature-gate. + +If you need run-time feature detection in `#[no_std]` environments, Rust `core` +library cannot help you. By design, Rust `core` is platform independent, but +performing run-time feature detection requires a certain level of cooperation +from the platform. + +You can then manually include `std_detect` as a dependency to get similar +run-time feature detection support than the one offered by Rust's standard +library. We intend to make `std_detect` more flexible and configurable in this +regard to better serve the needs of `#[no_std]` targets. + +# Features + +* `std_detect_dlsym_getauxval` (enabled by default, requires `libc`): Enable to +use `libc::dlsym` to query whether [`getauxval`] is linked into the binary. When +this is not the case, this feature allows other fallback methods to perform +run-time feature detection. When this feature is disabled, `std_detect` assumes +that [`getauxval`] is linked to the binary. If that is not the case the behavior +is undefined. + +* `std_detect_file_io` (enabled by default, requires `std`): Enable to perform run-time feature +detection using file APIs (e.g. `/proc/cpuinfo`, etc.) if other more performant +methods fail. This feature requires `libstd` as a dependency, preventing the +crate from working on applications in which `std` is not available. + +[`getauxval`]: http://man7.org/linux/man-pages/man3/getauxval.3.html + +# Platform support + +* All `x86`/`x86_64` targets are supported on all platforms by querying the + `cpuid` instruction directly for the features supported by the hardware and + the operating system. `std_detect` assumes that the binary is an user-space + application. If you need raw support for querying `cpuid`, consider using the + [`cupid`](https://crates.io/crates/cupid) crate. + +* Linux: + * `arm{32, 64}`, `mips{32,64}{,el}`, `powerpc{32,64}{,le}`: `std_detect` + supports these on Linux by querying ELF auxiliary vectors (using `getauxval` + when available), and if that fails, by querying `/proc/cpuinfo`. + * `arm64`: partial support for doing run-time feature detection by directly + querying `mrs` is implemented for Linux >= 4.11, but not enabled by default. + +* FreeBSD: + * `arm64`: run-time feature detection is implemented by directly querying `mrs`. + +# License + +This project is licensed under either of + + * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or + http://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([LICENSE-MIT](LICENSE-MIT) or + http://opensource.org/licenses/MIT) + +at your option. + +# Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in `std_detect` by you, as defined in the Apache-2.0 license, +shall be dual licensed as above, without any additional terms or conditions. diff --git a/crux-mir/lib/std_detect/src/detect/arch/aarch64.rs b/crux-mir/lib/std_detect/src/detect/arch/aarch64.rs new file mode 100644 index 000000000..5f46c7696 --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/arch/aarch64.rs @@ -0,0 +1,153 @@ +//! Aarch64 run-time features. + +features! { + @TARGET: aarch64; + @CFG: target_arch = "aarch64"; + @MACRO_NAME: is_aarch64_feature_detected; + @MACRO_ATTRS: + /// This macro tests, at runtime, whether an `aarch64` feature is enabled on aarch64 platforms. + /// Currently most features are only supported on linux-based platforms. + /// + /// This macro takes one argument which is a string literal of the feature being tested for. + /// The feature names are mostly taken from their FEAT_* definitions in the [ARM Architecture + /// Reference Manual][docs]. + /// + /// ## Supported arguments + /// + /// * `"asimd"` or "neon" - FEAT_AdvSIMD + /// * `"pmull"` - FEAT_PMULL + /// * `"fp"` - FEAT_FP + /// * `"fp16"` - FEAT_FP16 + /// * `"sve"` - FEAT_SVE + /// * `"crc"` - FEAT_CRC + /// * `"lse"` - FEAT_LSE + /// * `"lse2"` - FEAT_LSE2 + /// * `"rdm"` - FEAT_RDM + /// * `"rcpc"` - FEAT_LRCPC + /// * `"rcpc2"` - FEAT_LRCPC2 + /// * `"dotprod"` - FEAT_DotProd + /// * `"tme"` - FEAT_TME + /// * `"fhm"` - FEAT_FHM + /// * `"dit"` - FEAT_DIT + /// * `"flagm"` - FEAT_FLAGM + /// * `"ssbs"` - FEAT_SSBS + /// * `"sb"` - FEAT_SB + /// * `"paca"` - FEAT_PAuth (address authentication) + /// * `"pacg"` - FEAT_Pauth (generic authentication) + /// * `"dpb"` - FEAT_DPB + /// * `"dpb2"` - FEAT_DPB2 + /// * `"sve2"` - FEAT_SVE2 + /// * `"sve2-aes"` - FEAT_SVE2_AES + /// * `"sve2-sm4"` - FEAT_SVE2_SM4 + /// * `"sve2-sha3"` - FEAT_SVE2_SHA3 + /// * `"sve2-bitperm"` - FEAT_SVE2_BitPerm + /// * `"frintts"` - FEAT_FRINTTS + /// * `"i8mm"` - FEAT_I8MM + /// * `"f32mm"` - FEAT_F32MM + /// * `"f64mm"` - FEAT_F64MM + /// * `"bf16"` - FEAT_BF16 + /// * `"rand"` - FEAT_RNG + /// * `"bti"` - FEAT_BTI + /// * `"mte"` - FEAT_MTE + /// * `"jsconv"` - FEAT_JSCVT + /// * `"fcma"` - FEAT_FCMA + /// * `"aes"` - FEAT_AES + /// * `"sha2"` - FEAT_SHA1 & FEAT_SHA256 + /// * `"sha3"` - FEAT_SHA512 & FEAT_SHA3 + /// * `"sm4"` - FEAT_SM3 & FEAT_SM4 + /// + /// [docs]: https://developer.arm.com/documentation/ddi0487/latest + #[stable(feature = "simd_aarch64", since = "1.60.0")] + @BIND_FEATURE_NAME: "asimd"; "neon"; + @NO_RUNTIME_DETECTION: "ras"; + @NO_RUNTIME_DETECTION: "v8.1a"; + @NO_RUNTIME_DETECTION: "v8.2a"; + @NO_RUNTIME_DETECTION: "v8.3a"; + @NO_RUNTIME_DETECTION: "v8.4a"; + @NO_RUNTIME_DETECTION: "v8.5a"; + @NO_RUNTIME_DETECTION: "v8.6a"; + @NO_RUNTIME_DETECTION: "v8.7a"; + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] asimd: "neon"; + /// FEAT_AdvSIMD (Advanced SIMD/NEON) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] pmull: "pmull"; + /// FEAT_PMULL (Polynomial Multiply) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] fp: "fp"; + implied by target_features: ["neon"]; + /// FEAT_FP (Floating point support) - Implied by `neon` target_feature + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] fp16: "fp16"; + /// FEAT_FP16 (Half-float support) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] sve: "sve"; + /// FEAT_SVE (Scalable Vector Extension) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] crc: "crc"; + /// FEAT_CRC32 (Cyclic Redundancy Check) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] lse: "lse"; + /// FEAT_LSE (Large System Extension - atomics) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] lse2: "lse2"; + /// FEAT_LSE2 (unaligned and register-pair atomics) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] rdm: "rdm"; + /// FEAT_RDM (Rounding Doubling Multiply - ASIMDRDM) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] rcpc: "rcpc"; + /// FEAT_LRCPC (Release consistent Processor consistent) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] rcpc2: "rcpc2"; + /// FEAT_LRCPC2 (RCPC with immediate offsets) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] dotprod: "dotprod"; + /// FEAT_DotProd (Vector Dot-Product - ASIMDDP) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] tme: "tme"; + /// FEAT_TME (Transactional Memory Extensions) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] fhm: "fhm"; + /// FEAT_FHM (fp16 multiplication instructions) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] dit: "dit"; + /// FEAT_DIT (Data Independent Timing instructions) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] flagm: "flagm"; + /// FEAT_FLAGM (flag manipulation instructions) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] ssbs: "ssbs"; + /// FEAT_SSBS (speculative store bypass safe) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] sb: "sb"; + /// FEAT_SB (speculation barrier) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] paca: "paca"; + /// FEAT_PAuth (address authentication) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] pacg: "pacg"; + /// FEAT_PAuth (generic authentication) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] dpb: "dpb"; + /// FEAT_DPB (aka dcpop - data cache clean to point of persistence) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] dpb2: "dpb2"; + /// FEAT_DPB2 (aka dcpodp - data cache clean to point of deep persistence) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] sve2: "sve2"; + /// FEAT_SVE2 (Scalable Vector Extension 2) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] sve2_aes: "sve2-aes"; + /// FEAT_SVE_AES (SVE2 AES crypto) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] sve2_sm4: "sve2-sm4"; + /// FEAT_SVE_SM4 (SVE2 SM4 crypto) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] sve2_sha3: "sve2-sha3"; + /// FEAT_SVE_SHA3 (SVE2 SHA3 crypto) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] sve2_bitperm: "sve2-bitperm"; + /// FEAT_SVE_BitPerm (SVE2 bit permutation instructions) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] frintts: "frintts"; + /// FEAT_FRINTTS (float to integer rounding instructions) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] i8mm: "i8mm"; + /// FEAT_I8MM (integer matrix multiplication, plus ASIMD support) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] f32mm: "f32mm"; + /// FEAT_F32MM (single-precision matrix multiplication) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] f64mm: "f64mm"; + /// FEAT_F64MM (double-precision matrix multiplication) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] bf16: "bf16"; + /// FEAT_BF16 (BFloat16 type, plus MM instructions, plus ASIMD support) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] rand: "rand"; + /// FEAT_RNG (Random Number Generator) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] bti: "bti"; + /// FEAT_BTI (Branch Target Identification) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] mte: "mte"; + /// FEAT_MTE (Memory Tagging Extension) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] jsconv: "jsconv"; + /// FEAT_JSCVT (JavaScript float conversion instructions) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] fcma: "fcma"; + /// FEAT_FCMA (float complex number operations) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] aes: "aes"; + /// FEAT_AES (AES instructions) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] sha2: "sha2"; + /// FEAT_SHA1 & FEAT_SHA256 (SHA1 & SHA2-256 instructions) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] sha3: "sha3"; + /// FEAT_SHA512 & FEAT_SHA3 (SHA2-512 & SHA3 instructions) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] sm4: "sm4"; + /// FEAT_SM3 & FEAT_SM4 (SM3 & SM4 instructions) +} diff --git a/crux-mir/lib/std_detect/src/detect/arch/arm.rs b/crux-mir/lib/std_detect/src/detect/arch/arm.rs new file mode 100644 index 000000000..897dc314c --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/arch/arm.rs @@ -0,0 +1,28 @@ +//! Run-time feature detection on ARM Aarch32. + +features! { + @TARGET: arm; + @CFG: target_arch = "arm"; + @MACRO_NAME: is_arm_feature_detected; + @MACRO_ATTRS: + /// Checks if `arm` feature is enabled. + #[unstable(feature = "stdsimd", issue = "27731")] + @NO_RUNTIME_DETECTION: "v7"; + @NO_RUNTIME_DETECTION: "vfp2"; + @NO_RUNTIME_DETECTION: "vfp3"; + @NO_RUNTIME_DETECTION: "vfp4"; + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] neon: "neon"; + /// ARM Advanced SIMD (NEON) - Aarch32 + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] pmull: "pmull"; + /// Polynomial Multiply + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] crc: "crc"; + /// CRC32 (Cyclic Redundancy Check) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] crypto: "crypto"; + /// Crypto: AES + PMULL + SHA1 + SHA256. Prefer using the individual features where possible. + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] aes: "aes"; + /// FEAT_AES (AES instructions) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] sha2: "sha2"; + /// FEAT_SHA1 & FEAT_SHA256 (SHA1 & SHA2-256 instructions) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] i8mm: "i8mm"; + /// FEAT_I8MM +} diff --git a/crux-mir/lib/std_detect/src/detect/arch/mips.rs b/crux-mir/lib/std_detect/src/detect/arch/mips.rs new file mode 100644 index 000000000..ae27d0093 --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/arch/mips.rs @@ -0,0 +1,12 @@ +//! Run-time feature detection on MIPS. + +features! { + @TARGET: mips; + @CFG: target_arch = "mips"; + @MACRO_NAME: is_mips_feature_detected; + @MACRO_ATTRS: + /// Checks if `mips` feature is enabled. + #[unstable(feature = "stdsimd", issue = "27731")] + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] msa: "msa"; + /// MIPS SIMD Architecture (MSA) +} diff --git a/crux-mir/lib/std_detect/src/detect/arch/mips64.rs b/crux-mir/lib/std_detect/src/detect/arch/mips64.rs new file mode 100644 index 000000000..7182ec2da --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/arch/mips64.rs @@ -0,0 +1,12 @@ +//! Run-time feature detection on MIPS64. + +features! { + @TARGET: mips64; + @CFG: target_arch = "mips64"; + @MACRO_NAME: is_mips64_feature_detected; + @MACRO_ATTRS: + /// Checks if `mips64` feature is enabled. + #[unstable(feature = "stdsimd", issue = "27731")] + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] msa: "msa"; + /// MIPS SIMD Architecture (MSA) +} diff --git a/crux-mir/lib/std_detect/src/detect/arch/mod.rs b/crux-mir/lib/std_detect/src/detect/arch/mod.rs new file mode 100644 index 000000000..81a1f23e8 --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/arch/mod.rs @@ -0,0 +1,56 @@ +#![allow(dead_code)] + +use cfg_if::cfg_if; + +// Export the macros for all supported architectures. +#[macro_use] +mod x86; +#[macro_use] +mod arm; +#[macro_use] +mod aarch64; +#[macro_use] +mod riscv; +#[macro_use] +mod powerpc; +#[macro_use] +mod powerpc64; +#[macro_use] +mod mips; +#[macro_use] +mod mips64; + +cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + pub use x86::*; + } else if #[cfg(target_arch = "arm")] { + pub use arm::*; + } else if #[cfg(target_arch = "aarch64")] { + pub use aarch64::*; + } else if #[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] { + pub use riscv::*; + } else if #[cfg(target_arch = "powerpc")] { + pub use powerpc::*; + } else if #[cfg(target_arch = "powerpc64")] { + pub use powerpc64::*; + } else if #[cfg(target_arch = "mips")] { + pub use mips::*; + } else if #[cfg(target_arch = "mips64")] { + pub use mips64::*; + } else { + // Unimplemented architecture: + #[doc(hidden)] + pub(crate) enum Feature { + Null + } + #[doc(hidden)] + pub mod __is_feature_detected {} + + impl Feature { + #[doc(hidden)] + pub(crate) fn from_str(_s: &str) -> Result { Err(()) } + #[doc(hidden)] + pub(crate) fn to_str(self) -> &'static str { "" } + } + } +} diff --git a/crux-mir/lib/std_detect/src/detect/arch/powerpc.rs b/crux-mir/lib/std_detect/src/detect/arch/powerpc.rs new file mode 100644 index 000000000..d135cd95d --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/arch/powerpc.rs @@ -0,0 +1,16 @@ +//! Run-time feature detection on PowerPC. + +features! { + @TARGET: powerpc; + @CFG: target_arch = "powerpc"; + @MACRO_NAME: is_powerpc_feature_detected; + @MACRO_ATTRS: + /// Checks if `powerpc` feature is enabled. + #[unstable(feature = "stdsimd", issue = "27731")] + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] altivec: "altivec"; + /// Altivec + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] vsx: "vsx"; + /// VSX + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] power8: "power8"; + /// Power8 +} diff --git a/crux-mir/lib/std_detect/src/detect/arch/powerpc64.rs b/crux-mir/lib/std_detect/src/detect/arch/powerpc64.rs new file mode 100644 index 000000000..773afd6ce --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/arch/powerpc64.rs @@ -0,0 +1,16 @@ +//! Run-time feature detection on PowerPC64. + +features! { + @TARGET: powerpc64; + @CFG: target_arch = "powerpc64"; + @MACRO_NAME: is_powerpc64_feature_detected; + @MACRO_ATTRS: + /// Checks if `powerpc` feature is enabled. + #[unstable(feature = "stdsimd", issue = "27731")] + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] altivec: "altivec"; + /// Altivec + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] vsx: "vsx"; + /// VSX + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] power8: "power8"; + /// Power8 +} diff --git a/crux-mir/lib/std_detect/src/detect/arch/riscv.rs b/crux-mir/lib/std_detect/src/detect/arch/riscv.rs new file mode 100644 index 000000000..5ea36e7c1 --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/arch/riscv.rs @@ -0,0 +1,206 @@ +//! Run-time feature detection on RISC-V. + +features! { + @TARGET: riscv; + @CFG: any(target_arch = "riscv32", target_arch = "riscv64"); + @MACRO_NAME: is_riscv_feature_detected; + @MACRO_ATTRS: + /// A macro to test at *runtime* whether instruction sets are available on + /// RISC-V platforms. + /// + /// RISC-V standard defined the base sets and the extension sets. + /// The base sets are RV32I, RV64I, RV32E or RV128I. Any RISC-V platform + /// must support one base set and/or multiple extension sets. + /// + /// Any RISC-V standard instruction sets can be in state of either ratified, + /// frozen or draft. The version and status of current standard instruction + /// sets can be checked out from preface section of the [ISA manual]. + /// + /// Platform may define and support their own custom instruction sets with + /// ISA prefix X. These sets are highly platform specific and should be + /// detected with their own platform support crates. + /// + /// # Unprivileged Specification + /// + /// The supported ratified RISC-V instruction sets are as follows: + /// + /// * RV32I: `"rv32i"` + /// * Zifencei: `"zifencei"` + /// * Zihintpause: `"zihintpause"` + /// * RV64I: `"rv64i"` + /// * M: `"m"` + /// * A: `"a"` + /// * Zicsr: `"zicsr"` + /// * Zicntr: `"zicntr"` + /// * Zihpm: `"zihpm"` + /// * F: `"f"` + /// * D: `"d"` + /// * Q: `"q"` + /// * C: `"c"` + /// + /// There's also bases and extensions marked as standard instruction set, + /// but they are in frozen or draft state. These instruction sets are also + /// reserved by this macro and can be detected in the future platforms. + /// + /// Frozen RISC-V instruction sets: + /// + /// * Zfinx: `"zfinx"` + /// * Zdinx: `"zdinx"` + /// * Zhinx: `"zhinx"` + /// * Zhinxmin: `"zhinxmin"` + /// * Ztso: `"ztso"` + /// + /// Draft RISC-V instruction sets: + /// + /// * RV32E: `"rv32e"` + /// * RV128I: `"rv128i"` + /// * Zfh: `"zfh"` + /// * Zfhmin: `"zfhmin"` + /// * B: `"b"` + /// * J: `"j"` + /// * P: `"p"` + /// * V: `"v"` + /// * Zam: `"zam"` + /// + /// Defined by Privileged Specification: + /// + /// * Supervisor: `"s"` + /// * Svnapot: `"svnapot"` + /// * Svpbmt: `"svpbmt"` + /// * Svinval: `"svinval"` + /// * Hypervisor: `"h"` + /// + /// # RISC-V Bit-Manipulation ISA-extensions + /// + /// This document defined the following extensions: + /// + /// * Zba: `"zba"` + /// * Zbb: `"zbb"` + /// * Zbc: `"zbc"` + /// * Zbs: `"zbs"` + /// + /// # RISC-V Cryptography Extensions + /// + /// These extensions are defined in Volume I, Scalar & Entropy Source + /// Instructions: + /// + /// * Zbkb: `"zbkb"` + /// * Zbkc: `"zbkc"` + /// * Zbkx: `"zbkx"` + /// * Zknd: `"zknd"` + /// * Zkne: `"zkne"` + /// * Zknh: `"zknh"` + /// * Zksed: `"zksed"` + /// * Zksh: `"zksh"` + /// * Zkr: `"zkr"` + /// * Zkn: `"zkn"` + /// * Zks: `"zks"` + /// * Zk: `"zk"` + /// * Zkt: `"zkt"` + /// + /// [ISA manual]: https://github.com/riscv/riscv-isa-manual/ + #[unstable(feature = "stdsimd", issue = "27731")] + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] rv32i: "rv32i"; + /// RV32I Base Integer Instruction Set + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zifencei: "zifencei"; + /// "Zifencei" Instruction-Fetch Fence + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zihintpause: "zihintpause"; + /// "Zihintpause" Pause Hint + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] rv64i: "rv64i"; + /// RV64I Base Integer Instruction Set + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] m: "m"; + /// "M" Standard Extension for Integer Multiplication and Division + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] a: "a"; + /// "A" Standard Extension for Atomic Instructions + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zicsr: "zicsr"; + /// "Zicsr", Control and Status Register (CSR) Instructions + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zicntr: "zicntr"; + /// "Zicntr", Standard Extension for Base Counters and Timers + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zihpm: "zihpm"; + /// "Zihpm", Standard Extension for Hardware Performance Counters + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] f: "f"; + /// "F" Standard Extension for Single-Precision Floating-Point + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] d: "d"; + /// "D" Standard Extension for Double-Precision Floating-Point + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] q: "q"; + /// "Q" Standard Extension for Quad-Precision Floating-Point + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] c: "c"; + /// "C" Standard Extension for Compressed Instructions + + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zfinx: "zfinx"; + /// "Zfinx" Standard Extension for Single-Precision Floating-Point in Integer Registers + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zdinx: "zdinx"; + /// "Zdinx" Standard Extension for Double-Precision Floating-Point in Integer Registers + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zhinx: "zhinx"; + /// "Zhinx" Standard Extension for Half-Precision Floating-Point in Integer Registers + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zhinxmin: "zhinxmin"; + /// "Zhinxmin" Standard Extension for Minimal Half-Precision Floating-Point in Integer Registers + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] ztso: "ztso"; + /// "Ztso" Standard Extension for Total Store Ordering + + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] rv32e: "rv32e"; + /// RV32E Base Integer Instruction Set + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] rv128i: "rv128i"; + /// RV128I Base Integer Instruction Set + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zfh: "zfh"; + /// "Zfh" Standard Extension for 16-Bit Half-Precision Floating-Point + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zfhmin: "zfhmin"; + /// "Zfhmin" Standard Extension for Minimal Half-Precision Floating-Point Support + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] b: "b"; + /// "B" Standard Extension for Bit Manipulation + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] j: "j"; + /// "J" Standard Extension for Dynamically Translated Languages + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] p: "p"; + /// "P" Standard Extension for Packed-SIMD Instructions + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] v: "v"; + /// "V" Standard Extension for Vector Operations + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zam: "zam"; + /// "Zam" Standard Extension for Misaligned Atomics + + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] s: "s"; + /// Supervisor-Level ISA + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] svnapot: "svnapot"; + /// "Svnapot" Standard Extension for NAPOT Translation Contiguity + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] svpbmt: "svpbmt"; + /// "Svpbmt" Standard Extension for Page-Based Memory Types + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] svinval: "svinval"; + /// "Svinval" Standard Extension for Fine-Grained Address-Translation Cache Invalidation + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] h: "h"; + /// Hypervisor Extension + + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zba: "zba"; + /// "Zba" Standard Extension for Address Generation Instructions + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zbb: "zbb"; + /// "Zbb" Standard Extension for Basic Bit-Manipulation + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zbc: "zbc"; + /// "Zbc" Standard Extension for Carry-less Multiplication + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zbs: "zbs"; + /// "Zbs" Standard Extension for Single-Bit instructions + + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zbkb: "zbkb"; + /// "Zbkb" Standard Extension for Bitmanip instructions for Cryptography + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zbkc: "zbkc"; + /// "Zbkc" Standard Extension for Carry-less multiply instructions + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zbkx: "zbkx"; + /// "Zbkx" Standard Extension for Crossbar permutation instructions + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zknd: "zknd"; + /// "Zknd" Standard Extension for NIST Suite: AES Decryption + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zkne: "zkne"; + /// "Zkne" Standard Extension for NIST Suite: AES Encryption + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zknh: "zknh"; + /// "Zknh" Standard Extension for NIST Suite: Hash Function Instructions + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zksed: "zksed"; + /// "Zksed" Standard Extension for ShangMi Suite: SM4 Block Cipher Instructions + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zksh: "zksh"; + /// "Zksh" Standard Extension for ShangMi Suite: SM3 Hash Function Instructions + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zkr: "zkr"; + /// "Zkr" Standard Extension for Entropy Source Extension + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zkn: "zkn"; + /// "Zkn" Standard Extension for NIST Algorithm Suite + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zks: "zks"; + /// "Zks" Standard Extension for ShangMi Algorithm Suite + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zk: "zk"; + /// "Zk" Standard Extension for Standard scalar cryptography extension + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zkt: "zkt"; + /// "Zkt" Standard Extension for Data Independent Execution Latency +} diff --git a/crux-mir/lib/std_detect/src/detect/arch/x86.rs b/crux-mir/lib/std_detect/src/detect/arch/x86.rs new file mode 100644 index 000000000..893e1a887 --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/arch/x86.rs @@ -0,0 +1,197 @@ +//! This module implements minimal run-time feature detection for x86. +//! +//! The features are detected using the `detect_features` function below. +//! This function uses the CPUID instruction to read the feature flags from the +//! CPU and encodes them in a `usize` where each bit position represents +//! whether a feature is available (bit is set) or unavailable (bit is cleared). +//! +//! The enum `Feature` is used to map bit positions to feature names, and the +//! the `__crate::detect::check_for!` macro is used to map string literals (e.g., +//! "avx") to these bit positions (e.g., `Feature::avx`). +//! +//! The run-time feature detection is performed by the +//! `__crate::detect::check_for(Feature) -> bool` function. On its first call, +//! this functions queries the CPU for the available features and stores them +//! in a global `AtomicUsize` variable. The query is performed by just checking +//! whether the feature bit in this global variable is set or cleared. + +features! { + @TARGET: x86; + @CFG: any(target_arch = "x86", target_arch = "x86_64"); + @MACRO_NAME: is_x86_feature_detected; + @MACRO_ATTRS: + /// A macro to test at *runtime* whether a CPU feature is available on + /// x86/x86-64 platforms. + /// + /// This macro is provided in the standard library and will detect at runtime + /// whether the specified CPU feature is detected. This does **not** resolve at + /// compile time unless the specified feature is already enabled for the entire + /// crate. Runtime detection currently relies mostly on the `cpuid` instruction. + /// + /// This macro only takes one argument which is a string literal of the feature + /// being tested for. The feature names supported are the lowercase versions of + /// the ones defined by Intel in [their documentation][docs]. + /// + /// ## Supported arguments + /// + /// This macro supports the same names that `#[target_feature]` supports. Unlike + /// `#[target_feature]`, however, this macro does not support names separated + /// with a comma. Instead testing for multiple features must be done through + /// separate macro invocations for now. + /// + /// Supported arguments are: + /// + /// * `"aes"` + /// * `"pclmulqdq"` + /// * `"rdrand"` + /// * `"rdseed"` + /// * `"tsc"` + /// * `"mmx"` + /// * `"sse"` + /// * `"sse2"` + /// * `"sse3"` + /// * `"ssse3"` + /// * `"sse4.1"` + /// * `"sse4.2"` + /// * `"sse4a"` + /// * `"sha"` + /// * `"avx"` + /// * `"avx2"` + /// * `"avx512f"` + /// * `"avx512cd"` + /// * `"avx512er"` + /// * `"avx512pf"` + /// * `"avx512bw"` + /// * `"avx512dq"` + /// * `"avx512vl"` + /// * `"avx512ifma"` + /// * `"avx512vbmi"` + /// * `"avx512vpopcntdq"` + /// * `"avx512vbmi2"` + /// * `"avx512gfni"` + /// * `"avx512vaes"` + /// * `"avx512vpclmulqdq"` + /// * `"avx512vnni"` + /// * `"avx512bitalg"` + /// * `"avx512bf16"` + /// * `"avx512vp2intersect"` + /// * `"f16c"` + /// * `"fma"` + /// * `"bmi1"` + /// * `"bmi2"` + /// * `"abm"` + /// * `"lzcnt"` + /// * `"tbm"` + /// * `"popcnt"` + /// * `"fxsr"` + /// * `"xsave"` + /// * `"xsaveopt"` + /// * `"xsaves"` + /// * `"xsavec"` + /// * `"cmpxchg16b"` + /// * `"adx"` + /// * `"rtm"` + /// + /// [docs]: https://software.intel.com/sites/landingpage/IntrinsicsGuide + #[stable(feature = "simd_x86", since = "1.27.0")] + @BIND_FEATURE_NAME: "abm"; "lzcnt"; // abm is a synonym for lzcnt + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] aes: "aes"; + /// AES (Advanced Encryption Standard New Instructions AES-NI) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] pclmulqdq: "pclmulqdq"; + /// CLMUL (Carry-less Multiplication) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] rdrand: "rdrand"; + /// RDRAND + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] rdseed: "rdseed"; + /// RDSEED + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] tsc: "tsc"; + /// TSC (Time Stamp Counter) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] mmx: "mmx"; + /// MMX (MultiMedia eXtensions) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] sse: "sse"; + /// SSE (Streaming SIMD Extensions) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] sse2: "sse2"; + /// SSE2 (Streaming SIMD Extensions 2) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] sse3: "sse3"; + /// SSE3 (Streaming SIMD Extensions 3) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] ssse3: "ssse3"; + /// SSSE3 (Supplemental Streaming SIMD Extensions 3) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] sse4_1: "sse4.1"; + /// SSE4.1 (Streaming SIMD Extensions 4.1) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] sse4_2: "sse4.2"; + /// SSE4.2 (Streaming SIMD Extensions 4.2) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] sse4a: "sse4a"; + /// SSE4a (Streaming SIMD Extensions 4a) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] sha: "sha"; + /// SHA + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx: "avx"; + /// AVX (Advanced Vector Extensions) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx2: "avx2"; + /// AVX2 (Advanced Vector Extensions 2) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512f: "avx512f" ; + /// AVX-512 F (Foundation) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512cd: "avx512cd" ; + /// AVX-512 CD (Conflict Detection Instructions) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512er: "avx512er"; + /// AVX-512 ER (Expo nential and Reciprocal Instructions) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512pf: "avx512pf"; + /// AVX-512 PF (Prefetch Instructions) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512bw: "avx512bw"; + /// AVX-512 BW (Byte and Word Instructions) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512dq: "avx512dq"; + /// AVX-512 DQ (Doubleword and Quadword) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512vl: "avx512vl"; + /// AVX-512 VL (Vector Length Extensions) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512ifma: "avx512ifma"; + /// AVX-512 IFMA (Integer Fused Multiply Add) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512vbmi: "avx512vbmi"; + /// AVX-512 VBMI (Vector Byte Manipulation Instructions) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512vpopcntdq: "avx512vpopcntdq"; + /// AVX-512 VPOPCNTDQ (Vector Population Count Doubleword and + /// Quadword) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512vbmi2: "avx512vbmi2"; + /// AVX-512 VBMI2 (Additional byte, word, dword and qword capabilities) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512gfni: "avx512gfni"; + /// AVX-512 GFNI (Galois Field New Instruction) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512vaes: "avx512vaes"; + /// AVX-512 VAES (Vector AES instruction) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512vpclmulqdq: "avx512vpclmulqdq"; + /// AVX-512 VPCLMULQDQ (Vector PCLMULQDQ instructions) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512vnni: "avx512vnni"; + /// AVX-512 VNNI (Vector Neural Network Instructions) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512bitalg: "avx512bitalg"; + /// AVX-512 BITALG (Support for VPOPCNT\[B,W\] and VPSHUFBITQMB) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512bf16: "avx512bf16"; + /// AVX-512 BF16 (BFLOAT16 instructions) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512vp2intersect: "avx512vp2intersect"; + /// AVX-512 P2INTERSECT + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] f16c: "f16c"; + /// F16C (Conversions between IEEE-754 `binary16` and `binary32` formats) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] fma: "fma"; + /// FMA (Fused Multiply Add) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] bmi1: "bmi1" ; + /// BMI1 (Bit Manipulation Instructions 1) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] bmi2: "bmi2" ; + /// BMI2 (Bit Manipulation Instructions 2) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] lzcnt: "lzcnt"; + /// ABM (Advanced Bit Manipulation) / LZCNT (Leading Zero Count) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] tbm: "tbm"; + /// TBM (Trailing Bit Manipulation) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] popcnt: "popcnt"; + /// POPCNT (Population Count) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] fxsr: "fxsr"; + /// FXSR (Floating-point context fast save and restore) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] xsave: "xsave"; + /// XSAVE (Save Processor Extended States) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] xsaveopt: "xsaveopt"; + /// XSAVEOPT (Save Processor Extended States Optimized) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] xsaves: "xsaves"; + /// XSAVES (Save Processor Extended States Supervisor) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] xsavec: "xsavec"; + /// XSAVEC (Save Processor Extended States Compacted) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] cmpxchg16b: "cmpxchg16b"; + /// CMPXCH16B (16-byte compare-and-swap instruction) + @FEATURE: #[stable(feature = "simd_x86_adx", since = "1.33.0")] adx: "adx"; + /// ADX, Intel ADX (Multi-Precision Add-Carry Instruction Extensions) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] rtm: "rtm"; + /// RTM, Intel (Restricted Transactional Memory) +} diff --git a/crux-mir/lib/std_detect/src/detect/bit.rs b/crux-mir/lib/std_detect/src/detect/bit.rs new file mode 100644 index 000000000..6f06c5523 --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/bit.rs @@ -0,0 +1,9 @@ +//! Bit manipulation utilities. + +/// Tests the `bit` of `x`. +#[allow(dead_code)] +#[inline] +pub(crate) fn test(x: usize, bit: u32) -> bool { + debug_assert!(bit < usize::BITS, "bit index out-of-bounds"); + x & (1 << bit) != 0 +} diff --git a/crux-mir/lib/std_detect/src/detect/cache.rs b/crux-mir/lib/std_detect/src/detect/cache.rs new file mode 100644 index 000000000..d01a5ea24 --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/cache.rs @@ -0,0 +1,194 @@ +//! Caches run-time feature detection so that it only needs to be computed +//! once. + +#![allow(dead_code)] // not used on all platforms + +use core::sync::atomic::Ordering; + +use core::sync::atomic::AtomicUsize; + +/// Sets the `bit` of `x`. +#[inline] +const fn set_bit(x: u64, bit: u32) -> u64 { + x | 1 << bit +} + +/// Tests the `bit` of `x`. +#[inline] +const fn test_bit(x: u64, bit: u32) -> bool { + x & (1 << bit) != 0 +} + +/// Unset the `bit of `x`. +#[inline] +const fn unset_bit(x: u64, bit: u32) -> u64 { + x & !(1 << bit) +} + +/// Maximum number of features that can be cached. +const CACHE_CAPACITY: u32 = 62; + +/// This type is used to initialize the cache +#[derive(Copy, Clone)] +pub(crate) struct Initializer(u64); + +#[allow(clippy::use_self)] +impl Default for Initializer { + fn default() -> Self { + Initializer(0) + } +} + +// NOTE: the `debug_assert!` would catch that we do not add more Features than +// the one fitting our cache. +impl Initializer { + /// Tests the `bit` of the cache. + #[inline] + pub(crate) fn test(self, bit: u32) -> bool { + debug_assert!( + bit < CACHE_CAPACITY, + "too many features, time to increase the cache size!" + ); + test_bit(self.0, bit) + } + + /// Sets the `bit` of the cache. + #[inline] + pub(crate) fn set(&mut self, bit: u32) { + debug_assert!( + bit < CACHE_CAPACITY, + "too many features, time to increase the cache size!" + ); + let v = self.0; + self.0 = set_bit(v, bit); + } + + /// Unsets the `bit` of the cache. + #[inline] + pub(crate) fn unset(&mut self, bit: u32) { + debug_assert!( + bit < CACHE_CAPACITY, + "too many features, time to increase the cache size!" + ); + let v = self.0; + self.0 = unset_bit(v, bit); + } +} + +/// This global variable is a cache of the features supported by the CPU. +// Note: on x64, we only use the first slot +static CACHE: [Cache; 2] = [Cache::uninitialized(), Cache::uninitialized()]; + +/// Feature cache with capacity for `size_of::() * 8 - 1` features. +/// +/// Note: 0 is used to represent an uninitialized cache, and (at least) the most +/// significant bit is set on any cache which has been initialized. +/// +/// Note: we use `Relaxed` atomic operations, because we are only interested in +/// the effects of operations on a single memory location. That is, we only need +/// "modification order", and not the full-blown "happens before". +struct Cache(AtomicUsize); + +impl Cache { + const CAPACITY: u32 = (core::mem::size_of::() * 8 - 1) as u32; + const MASK: usize = (1 << Cache::CAPACITY) - 1; + const INITIALIZED_BIT: usize = 1usize << Cache::CAPACITY; + + /// Creates an uninitialized cache. + #[allow(clippy::declare_interior_mutable_const)] + const fn uninitialized() -> Self { + Cache(AtomicUsize::new(0)) + } + + /// Is the `bit` in the cache set? Returns `None` if the cache has not been initialized. + #[inline] + pub(crate) fn test(&self, bit: u32) -> Option { + let cached = self.0.load(Ordering::Relaxed); + if cached == 0 { + None + } else { + Some(test_bit(cached as u64, bit)) + } + } + + /// Initializes the cache. + #[inline] + fn initialize(&self, value: usize) -> usize { + debug_assert_eq!((value & !Cache::MASK), 0); + self.0 + .store(value | Cache::INITIALIZED_BIT, Ordering::Relaxed); + value + } +} + +cfg_if::cfg_if! { + if #[cfg(feature = "std_detect_env_override")] { + #[inline] + fn initialize(mut value: Initializer) -> Initializer { + let env = unsafe { + libc::getenv(b"RUST_STD_DETECT_UNSTABLE\0".as_ptr() as *const libc::c_char) + }; + if !env.is_null() { + let len = unsafe { libc::strlen(env) }; + let env = unsafe { core::slice::from_raw_parts(env as *const u8, len) }; + if let Ok(disable) = core::str::from_utf8(env) { + for v in disable.split(" ") { + let _ = super::Feature::from_str(v).map(|v| value.unset(v as u32)); + } + } + } + do_initialize(value); + value + } + } else { + #[inline] + fn initialize(value: Initializer) -> Initializer { + do_initialize(value); + value + } + } +} + +#[inline] +fn do_initialize(value: Initializer) { + CACHE[0].initialize((value.0) as usize & Cache::MASK); + CACHE[1].initialize((value.0 >> Cache::CAPACITY) as usize & Cache::MASK); +} + +// We only have to detect features once, and it's fairly costly, so hint to LLVM +// that it should assume that cache hits are more common than misses (which is +// the point of caching). It's possibly unfortunate that this function needs to +// reach across modules like this to call `os::detect_features`, but it produces +// the best code out of several attempted variants. +// +// The `Initializer` that the cache was initialized with is returned, so that +// the caller can call `test()` on it without having to load the value from the +// cache again. +#[cold] +fn detect_and_initialize() -> Initializer { + initialize(super::os::detect_features()) +} + +/// Tests the `bit` of the storage. If the storage has not been initialized, +/// initializes it with the result of `os::detect_features()`. +/// +/// On its first invocation, it detects the CPU features and caches them in the +/// `CACHE` global variable as an `AtomicU64`. +/// +/// It uses the `Feature` variant to index into this variable as a bitset. If +/// the bit is set, the feature is enabled, and otherwise it is disabled. +/// +/// If the feature `std_detect_env_override` is enabled looks for the env +/// variable `RUST_STD_DETECT_UNSTABLE` and uses its its content to disable +/// Features that would had been otherwise detected. +#[inline] +pub(crate) fn test(bit: u32) -> bool { + let (relative_bit, idx) = if bit < Cache::CAPACITY { + (bit, 0) + } else { + (bit - Cache::CAPACITY, 1) + }; + CACHE[idx] + .test(relative_bit) + .unwrap_or_else(|| detect_and_initialize().test(bit)) +} diff --git a/crux-mir/lib/std_detect/src/detect/macros.rs b/crux-mir/lib/std_detect/src/detect/macros.rs new file mode 100644 index 000000000..a467f9db6 --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/macros.rs @@ -0,0 +1,166 @@ +#[macro_export] +#[allow_internal_unstable(stdsimd)] +macro_rules! detect_feature { + ($feature:tt, $feature_lit:tt) => { + $crate::detect_feature!($feature, $feature_lit : $feature_lit) + }; + ($feature:tt, $feature_lit:tt : $($target_feature_lit:tt),*) => { + $(cfg!(target_feature = $target_feature_lit) ||)* + $crate::detect::__is_feature_detected::$feature() + }; +} + +#[allow(unused)] +macro_rules! features { + ( + @TARGET: $target:ident; + @CFG: $cfg:meta; + @MACRO_NAME: $macro_name:ident; + @MACRO_ATTRS: $(#[$macro_attrs:meta])* + $(@BIND_FEATURE_NAME: $bind_feature:tt; $feature_impl:tt; )* + $(@NO_RUNTIME_DETECTION: $nort_feature:tt; )* + $(@FEATURE: #[$stability_attr:meta] $feature:ident: $feature_lit:tt; + $(implied by target_features: [$($target_feature_lit:tt),*];)? + $(#[$feature_comment:meta])*)* + ) => { + #[macro_export] + $(#[$macro_attrs])* + #[allow_internal_unstable(stdsimd_internal, stdsimd)] + #[cfg($cfg)] + #[doc(cfg($cfg))] + macro_rules! $macro_name { + $( + ($feature_lit) => { + $crate::detect_feature!($feature, $feature_lit $(: $($target_feature_lit),*)?) + }; + )* + $( + ($bind_feature) => { $crate::$macro_name!($feature_impl) }; + )* + $( + ($nort_feature) => { + compile_error!( + concat!( + stringify!($nort_feature), + " feature cannot be detected at run-time" + ) + ) + }; + )* + ($t:tt,) => { + $crate::$macro_name!($t); + }; + ($t:tt) => { + compile_error!( + concat!( + concat!("unknown ", stringify!($target)), + concat!(" target feature: ", $t) + ) + ) + }; + } + + $(#[$macro_attrs])* + #[macro_export] + #[cfg(not($cfg))] + #[doc(cfg($cfg))] + macro_rules! $macro_name { + $( + ($feature_lit) => { + compile_error!( + concat!( + r#"This macro cannot be used on the current target. + You can prevent it from being used in other architectures by + guarding it behind a cfg("#, + stringify!($cfg), + ")." + ) + ) + }; + )* + $( + ($bind_feature) => { $crate::$macro_name!($feature_impl) }; + )* + $( + ($nort_feature) => { + compile_error!( + concat!( + stringify!($nort_feature), + " feature cannot be detected at run-time" + ) + ) + }; + )* + ($t:tt,) => { + $crate::$macro_name!($t); + }; + ($t:tt) => { + compile_error!( + concat!( + concat!("unknown ", stringify!($target)), + concat!(" target feature: ", $t) + ) + ) + }; + } + + /// Each variant denotes a position in a bitset for a particular feature. + /// + /// PLEASE: do not use this, it is an implementation detail subject + /// to change. + #[doc(hidden)] + #[allow(non_camel_case_types)] + #[derive(Copy, Clone)] + #[repr(u8)] + #[unstable(feature = "stdsimd_internal", issue = "none")] + #[cfg($cfg)] + pub(crate) enum Feature { + $( + $(#[$feature_comment])* + $feature, + )* + + // Do not add variants after last: + _last + } + + #[cfg($cfg)] + impl Feature { + pub(crate) fn to_str(self) -> &'static str { + match self { + $(Feature::$feature => $feature_lit,)* + Feature::_last => unreachable!(), + } + } + #[cfg(feature = "std_detect_env_override")] + pub(crate) fn from_str(s: &str) -> Result { + match s { + $($feature_lit => Ok(Feature::$feature),)* + _ => Err(()) + } + } + } + + /// Each function performs run-time feature detection for a single + /// feature. This allow us to use stability attributes on a per feature + /// basis. + /// + /// PLEASE: do not use this, it is an implementation detail subject + /// to change. + #[doc(hidden)] + #[cfg($cfg)] + pub mod __is_feature_detected { + $( + + /// PLEASE: do not use this, it is an implementation detail + /// subject to change. + #[inline] + #[doc(hidden)] + #[$stability_attr] + pub fn $feature() -> bool { + $crate::detect::check_for($crate::detect::Feature::$feature) + } + )* + } + }; +} diff --git a/crux-mir/lib/std_detect/src/detect/mod.rs b/crux-mir/lib/std_detect/src/detect/mod.rs new file mode 100644 index 000000000..2bca84ca1 --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/mod.rs @@ -0,0 +1,104 @@ +//! This module implements run-time feature detection. +//! +//! The `is_{arch}_feature_detected!("feature-name")` macros take the name of a +//! feature as a string-literal, and return a boolean indicating whether the +//! feature is enabled at run-time or not. +//! +//! These macros do two things: +//! * map the string-literal into an integer stored as a `Feature` enum, +//! * call a `os::check_for(x: Feature)` function that returns `true` if the +//! feature is enabled. +//! +//! The `Feature` enums are also implemented in the `arch/{target_arch}.rs` +//! modules. +//! +//! The `check_for` functions are, in general, Operating System dependent. Most +//! architectures do not allow user-space programs to query the feature bits +//! due to security concerns (x86 is the big exception). These functions are +//! implemented in the `os/{target_os}.rs` modules. + +use cfg_if::cfg_if; + +#[macro_use] +mod macros; + +mod arch; + +// This module needs to be public because the `is_{arch}_feature_detected!` +// macros expand calls to items within it in user crates. +#[doc(hidden)] +pub use self::arch::__is_feature_detected; + +pub(crate) use self::arch::Feature; + +mod bit; +mod cache; + +cfg_if! { + if #[cfg(miri)] { + // When running under miri all target-features that are not enabled at + // compile-time are reported as disabled at run-time. + // + // For features for which `cfg(target_feature)` returns true, + // this run-time detection logic is never called. + #[path = "os/other.rs"] + mod os; + } else if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + // On x86/x86_64 no OS specific functionality is required. + #[path = "os/x86.rs"] + mod os; + } else if #[cfg(all(target_os = "linux", feature = "libc"))] { + #[path = "os/linux/mod.rs"] + mod os; + } else if #[cfg(all(target_os = "freebsd", feature = "libc"))] { + #[cfg(target_arch = "aarch64")] + #[path = "os/aarch64.rs"] + mod aarch64; + #[path = "os/freebsd/mod.rs"] + mod os; + } else if #[cfg(all(target_os = "windows", target_arch = "aarch64"))] { + #[path = "os/windows/aarch64.rs"] + mod os; + } else { + #[path = "os/other.rs"] + mod os; + } +} + +/// Performs run-time feature detection. +#[inline] +#[allow(dead_code)] +fn check_for(x: Feature) -> bool { + cache::test(x as u32) +} + +/// Returns an `Iterator` where +/// `Item.0` is the feature name, and `Item.1` is a `bool` which +/// is `true` if the feature is supported by the host and `false` otherwise. +#[unstable(feature = "stdsimd", issue = "27731")] +pub fn features() -> impl Iterator { + cfg_if! { + if #[cfg(any( + target_arch = "x86", + target_arch = "x86_64", + target_arch = "arm", + target_arch = "aarch64", + target_arch = "riscv32", + target_arch = "riscv64", + target_arch = "powerpc", + target_arch = "powerpc64", + target_arch = "mips", + target_arch = "mips64", + ))] { + (0_u8..Feature::_last as u8).map(|discriminant: u8| { + #[allow(bindings_with_variant_name)] // RISC-V has Feature::f + let f: Feature = unsafe { core::mem::transmute(discriminant) }; + let name: &'static str = f.to_str(); + let enabled: bool = check_for(f); + (name, enabled) + }) + } else { + None.into_iter() + } + } +} diff --git a/crux-mir/lib/std_detect/src/detect/os/aarch64.rs b/crux-mir/lib/std_detect/src/detect/os/aarch64.rs new file mode 100644 index 000000000..e0e62ee33 --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/os/aarch64.rs @@ -0,0 +1,104 @@ +//! Run-time feature detection for Aarch64 on any OS that emulates the mrs instruction. +//! +//! On FreeBSD >= 12.0, Linux >= 4.11 and other operating systems, it is possible to use +//! privileged system registers from userspace to check CPU feature support. +//! +//! AArch64 system registers ID_AA64ISAR0_EL1, ID_AA64PFR0_EL1, ID_AA64ISAR1_EL1 +//! have bits dedicated to features like AdvSIMD, CRC32, AES, atomics (LSE), etc. +//! Each part of the register indicates the level of support for a certain feature, e.g. +//! when ID_AA64ISAR0_EL1\[7:4\] is >= 1, AES is supported; when it's >= 2, PMULL is supported. +//! +//! For proper support of [SoCs where different cores have different capabilities](https://medium.com/@jadr2ddude/a-big-little-problem-a-tale-of-big-little-gone-wrong-e7778ce744bb), +//! the OS has to always report only the features supported by all cores, like [FreeBSD does](https://reviews.freebsd.org/D17137#393947). +//! +//! References: +//! +//! - [Zircon implementation](https://fuchsia.googlesource.com/zircon/+/master/kernel/arch/arm64/feature.cpp) +//! - [Linux documentation](https://www.kernel.org/doc/Documentation/arm64/cpu-feature-registers.txt) + +use crate::detect::{cache, Feature}; +use core::arch::asm; + +/// Try to read the features from the system registers. +/// +/// This will cause SIGILL if the current OS is not trapping the mrs instruction. +pub(crate) fn detect_features() -> cache::Initializer { + let mut value = cache::Initializer::default(); + + { + let mut enable_feature = |f, enable| { + if enable { + value.set(f as u32); + } + }; + + // ID_AA64ISAR0_EL1 - Instruction Set Attribute Register 0 + let aa64isar0: u64; + unsafe { + asm!( + "mrs {}, ID_AA64ISAR0_EL1", + out(reg) aa64isar0, + options(pure, nomem, preserves_flags, nostack) + ); + } + + enable_feature(Feature::pmull, bits_shift(aa64isar0, 7, 4) >= 2); + enable_feature(Feature::tme, bits_shift(aa64isar0, 27, 24) == 1); + enable_feature(Feature::lse, bits_shift(aa64isar0, 23, 20) >= 1); + enable_feature(Feature::crc, bits_shift(aa64isar0, 19, 16) >= 1); + + // ID_AA64PFR0_EL1 - Processor Feature Register 0 + let aa64pfr0: u64; + unsafe { + asm!( + "mrs {}, ID_AA64PFR0_EL1", + out(reg) aa64pfr0, + options(pure, nomem, preserves_flags, nostack) + ); + } + + let fp = bits_shift(aa64pfr0, 19, 16) < 0xF; + let fphp = bits_shift(aa64pfr0, 19, 16) >= 1; + let asimd = bits_shift(aa64pfr0, 23, 20) < 0xF; + let asimdhp = bits_shift(aa64pfr0, 23, 20) >= 1; + enable_feature(Feature::fp, fp); + enable_feature(Feature::fp16, fphp); + // SIMD support requires float support - if half-floats are + // supported, it also requires half-float support: + enable_feature(Feature::asimd, fp && asimd && (!fphp | asimdhp)); + // SIMD extensions require SIMD support: + enable_feature(Feature::aes, asimd && bits_shift(aa64isar0, 7, 4) >= 1); + let sha1 = bits_shift(aa64isar0, 11, 8) >= 1; + let sha2 = bits_shift(aa64isar0, 15, 12) >= 1; + enable_feature(Feature::sha2, asimd && sha1 && sha2); + enable_feature(Feature::rdm, asimd && bits_shift(aa64isar0, 31, 28) >= 1); + enable_feature( + Feature::dotprod, + asimd && bits_shift(aa64isar0, 47, 44) >= 1, + ); + enable_feature(Feature::sve, asimd && bits_shift(aa64pfr0, 35, 32) >= 1); + + // ID_AA64ISAR1_EL1 - Instruction Set Attribute Register 1 + let aa64isar1: u64; + unsafe { + asm!( + "mrs {}, ID_AA64ISAR1_EL1", + out(reg) aa64isar1, + options(pure, nomem, preserves_flags, nostack) + ); + } + + // Check for either APA or API field + enable_feature(Feature::paca, bits_shift(aa64isar1, 11, 4) >= 1); + enable_feature(Feature::rcpc, bits_shift(aa64isar1, 23, 20) >= 1); + // Check for either GPA or GPI field + enable_feature(Feature::pacg, bits_shift(aa64isar1, 31, 24) >= 1); + } + + value +} + +#[inline] +fn bits_shift(x: u64, high: usize, low: usize) -> u64 { + (x >> low) & ((1 << (high - low + 1)) - 1) +} diff --git a/crux-mir/lib/std_detect/src/detect/os/freebsd/aarch64.rs b/crux-mir/lib/std_detect/src/detect/os/freebsd/aarch64.rs new file mode 100644 index 000000000..7d972b373 --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/os/freebsd/aarch64.rs @@ -0,0 +1,21 @@ +//! Run-time feature detection for Aarch64 on FreeBSD. + +pub(crate) use super::super::aarch64::detect_features; + +#[cfg(test)] +mod tests { + #[test] + fn dump() { + println!("asimd: {:?}", is_aarch64_feature_detected!("asimd")); + println!("pmull: {:?}", is_aarch64_feature_detected!("pmull")); + println!("fp: {:?}", is_aarch64_feature_detected!("fp")); + println!("fp16: {:?}", is_aarch64_feature_detected!("fp16")); + println!("sve: {:?}", is_aarch64_feature_detected!("sve")); + println!("crc: {:?}", is_aarch64_feature_detected!("crc")); + println!("lse: {:?}", is_aarch64_feature_detected!("lse")); + println!("rdm: {:?}", is_aarch64_feature_detected!("rdm")); + println!("rcpc: {:?}", is_aarch64_feature_detected!("rcpc")); + println!("dotprod: {:?}", is_aarch64_feature_detected!("dotprod")); + println!("tme: {:?}", is_aarch64_feature_detected!("tme")); + } +} diff --git a/crux-mir/lib/std_detect/src/detect/os/freebsd/arm.rs b/crux-mir/lib/std_detect/src/detect/os/freebsd/arm.rs new file mode 100644 index 000000000..4c9d763b4 --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/os/freebsd/arm.rs @@ -0,0 +1,21 @@ +//! Run-time feature detection for ARM on FreeBSD + +use super::auxvec; +use crate::detect::{cache, Feature}; + +/// Try to read the features from the auxiliary vector +pub(crate) fn detect_features() -> cache::Initializer { + let mut value = cache::Initializer::default(); + let enable_feature = |value: &mut cache::Initializer, f, enable| { + if enable { + value.set(f as u32); + } + }; + + if let Ok(auxv) = auxvec::auxv() { + enable_feature(&mut value, Feature::neon, auxv.hwcap & 0x00001000 != 0); + enable_feature(&mut value, Feature::pmull, auxv.hwcap2 & 0x00000002 != 0); + return value; + } + value +} diff --git a/crux-mir/lib/std_detect/src/detect/os/freebsd/auxvec.rs b/crux-mir/lib/std_detect/src/detect/os/freebsd/auxvec.rs new file mode 100644 index 000000000..29fcc8cb0 --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/os/freebsd/auxvec.rs @@ -0,0 +1,102 @@ +//! Parses ELF auxiliary vectors. +#![cfg_attr( + any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc64", + target_arch = "riscv64" + ), + allow(dead_code) +)] + +/// Key to access the CPU Hardware capabilities bitfield. +pub(crate) const AT_HWCAP: usize = 25; +/// Key to access the CPU Hardware capabilities 2 bitfield. +pub(crate) const AT_HWCAP2: usize = 26; + +/// Cache HWCAP bitfields of the ELF Auxiliary Vector. +/// +/// If an entry cannot be read all the bits in the bitfield are set to zero. +/// This should be interpreted as all the features being disabled. +#[derive(Debug, Copy, Clone)] +pub(crate) struct AuxVec { + pub hwcap: usize, + pub hwcap2: usize, +} + +/// ELF Auxiliary Vector +/// +/// The auxiliary vector is a memory region in a running ELF program's stack +/// composed of (key: usize, value: usize) pairs. +/// +/// The keys used in the aux vector are platform dependent. For FreeBSD, they are +/// defined in [sys/elf_common.h][elf_common_h]. The hardware capabilities of a given +/// CPU can be queried with the `AT_HWCAP` and `AT_HWCAP2` keys. +/// +/// Note that run-time feature detection is not invoked for features that can +/// be detected at compile-time. +/// +/// [elf_common.h]: https://svnweb.freebsd.org/base/release/12.0.0/sys/sys/elf_common.h?revision=341707 +pub(crate) fn auxv() -> Result { + if let Ok(hwcap) = archauxv(AT_HWCAP) { + if let Ok(hwcap2) = archauxv(AT_HWCAP2) { + if hwcap != 0 && hwcap2 != 0 { + return Ok(AuxVec { hwcap, hwcap2 }); + } + } + } + Err(()) +} + +/// Tries to read the `key` from the auxiliary vector. +fn archauxv(key: usize) -> Result { + use core::mem; + + #[derive(Copy, Clone)] + #[repr(C)] + pub struct Elf_Auxinfo { + pub a_type: usize, + pub a_un: unnamed, + } + #[derive(Copy, Clone)] + #[repr(C)] + pub union unnamed { + pub a_val: libc::c_long, + pub a_ptr: *mut libc::c_void, + pub a_fcn: Option ()>, + } + + let mut auxv: [Elf_Auxinfo; 27] = [Elf_Auxinfo { + a_type: 0, + a_un: unnamed { a_val: 0 }, + }; 27]; + + let mut len: libc::c_uint = mem::size_of_val(&auxv) as libc::c_uint; + + unsafe { + let mut mib = [ + libc::CTL_KERN, + libc::KERN_PROC, + libc::KERN_PROC_AUXV, + libc::getpid(), + ]; + + let ret = libc::sysctl( + mib.as_mut_ptr(), + mib.len() as u32, + &mut auxv as *mut _ as *mut _, + &mut len as *mut _ as *mut _, + 0 as *mut libc::c_void, + 0, + ); + + if ret != -1 { + for i in 0..auxv.len() { + if auxv[i].a_type == key { + return Ok(auxv[i].a_un.a_val as usize); + } + } + } + } + return Ok(0); +} diff --git a/crux-mir/lib/std_detect/src/detect/os/freebsd/mod.rs b/crux-mir/lib/std_detect/src/detect/os/freebsd/mod.rs new file mode 100644 index 000000000..ade7fb626 --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/os/freebsd/mod.rs @@ -0,0 +1,22 @@ +//! Run-time feature detection on FreeBSD + +mod auxvec; + +cfg_if::cfg_if! { + if #[cfg(target_arch = "aarch64")] { + mod aarch64; + pub(crate) use self::aarch64::detect_features; + } else if #[cfg(target_arch = "arm")] { + mod arm; + pub(crate) use self::arm::detect_features; + } else if #[cfg(target_arch = "powerpc64")] { + mod powerpc; + pub(crate) use self::powerpc::detect_features; + } else { + use crate::detect::cache; + /// Performs run-time feature detection. + pub(crate) fn detect_features() -> cache::Initializer { + cache::Initializer::default() + } + } +} diff --git a/crux-mir/lib/std_detect/src/detect/os/freebsd/powerpc.rs b/crux-mir/lib/std_detect/src/detect/os/freebsd/powerpc.rs new file mode 100644 index 000000000..6bfab631a --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/os/freebsd/powerpc.rs @@ -0,0 +1,21 @@ +//! Run-time feature detection for PowerPC on FreeBSD. + +use super::auxvec; +use crate::detect::{cache, Feature}; + +pub(crate) fn detect_features() -> cache::Initializer { + let mut value = cache::Initializer::default(); + let enable_feature = |value: &mut cache::Initializer, f, enable| { + if enable { + value.set(f as u32); + } + }; + + if let Ok(auxv) = auxvec::auxv() { + enable_feature(&mut value, Feature::altivec, auxv.hwcap & 0x10000000 != 0); + enable_feature(&mut value, Feature::vsx, auxv.hwcap & 0x00000080 != 0); + enable_feature(&mut value, Feature::power8, auxv.hwcap2 & 0x80000000 != 0); + return value; + } + value +} diff --git a/crux-mir/lib/std_detect/src/detect/os/linux/aarch64.rs b/crux-mir/lib/std_detect/src/detect/os/linux/aarch64.rs new file mode 100644 index 000000000..6c79ba86d --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/os/linux/aarch64.rs @@ -0,0 +1,377 @@ +//! Run-time feature detection for Aarch64 on Linux. + +use super::auxvec; +use crate::detect::{bit, cache, Feature}; + +/// Try to read the features from the auxiliary vector, and if that fails, try +/// to read them from /proc/cpuinfo. +pub(crate) fn detect_features() -> cache::Initializer { + if let Ok(auxv) = auxvec::auxv() { + let hwcap: AtHwcap = auxv.into(); + return hwcap.cache(); + } + #[cfg(feature = "std_detect_file_io")] + if let Ok(c) = super::cpuinfo::CpuInfo::new() { + let hwcap: AtHwcap = c.into(); + return hwcap.cache(); + } + cache::Initializer::default() +} + +/// These values are part of the platform-specific [asm/hwcap.h][hwcap] . +/// +/// The names match those used for cpuinfo. +/// +/// [hwcap]: https://github.com/torvalds/linux/blob/master/arch/arm64/include/uapi/asm/hwcap.h +#[derive(Debug, Default, PartialEq)] +struct AtHwcap { + // AT_HWCAP + fp: bool, + asimd: bool, + // evtstrm: No LLVM support. + aes: bool, + pmull: bool, + sha1: bool, + sha2: bool, + crc32: bool, + atomics: bool, + fphp: bool, + asimdhp: bool, + // cpuid: No LLVM support. + asimdrdm: bool, + jscvt: bool, + fcma: bool, + lrcpc: bool, + dcpop: bool, + sha3: bool, + sm3: bool, + sm4: bool, + asimddp: bool, + sha512: bool, + sve: bool, + fhm: bool, + dit: bool, + uscat: bool, + ilrcpc: bool, + flagm: bool, + ssbs: bool, + sb: bool, + paca: bool, + pacg: bool, + + // AT_HWCAP2 + dcpodp: bool, + sve2: bool, + sveaes: bool, + // svepmull: No LLVM support. + svebitperm: bool, + svesha3: bool, + svesm4: bool, + // flagm2: No LLVM support. + frint: bool, + // svei8mm: See i8mm feature. + svef32mm: bool, + svef64mm: bool, + // svebf16: See bf16 feature. + i8mm: bool, + bf16: bool, + // dgh: No LLVM support. + rng: bool, + bti: bool, + mte: bool, +} + +impl From for AtHwcap { + /// Reads AtHwcap from the auxiliary vector. + fn from(auxv: auxvec::AuxVec) -> Self { + AtHwcap { + fp: bit::test(auxv.hwcap, 0), + asimd: bit::test(auxv.hwcap, 1), + // evtstrm: bit::test(auxv.hwcap, 2), + aes: bit::test(auxv.hwcap, 3), + pmull: bit::test(auxv.hwcap, 4), + sha1: bit::test(auxv.hwcap, 5), + sha2: bit::test(auxv.hwcap, 6), + crc32: bit::test(auxv.hwcap, 7), + atomics: bit::test(auxv.hwcap, 8), + fphp: bit::test(auxv.hwcap, 9), + asimdhp: bit::test(auxv.hwcap, 10), + // cpuid: bit::test(auxv.hwcap, 11), + asimdrdm: bit::test(auxv.hwcap, 12), + jscvt: bit::test(auxv.hwcap, 13), + fcma: bit::test(auxv.hwcap, 14), + lrcpc: bit::test(auxv.hwcap, 15), + dcpop: bit::test(auxv.hwcap, 16), + sha3: bit::test(auxv.hwcap, 17), + sm3: bit::test(auxv.hwcap, 18), + sm4: bit::test(auxv.hwcap, 19), + asimddp: bit::test(auxv.hwcap, 20), + sha512: bit::test(auxv.hwcap, 21), + sve: bit::test(auxv.hwcap, 22), + fhm: bit::test(auxv.hwcap, 23), + dit: bit::test(auxv.hwcap, 24), + uscat: bit::test(auxv.hwcap, 25), + ilrcpc: bit::test(auxv.hwcap, 26), + flagm: bit::test(auxv.hwcap, 27), + ssbs: bit::test(auxv.hwcap, 28), + sb: bit::test(auxv.hwcap, 29), + paca: bit::test(auxv.hwcap, 30), + pacg: bit::test(auxv.hwcap, 31), + dcpodp: bit::test(auxv.hwcap2, 0), + sve2: bit::test(auxv.hwcap2, 1), + sveaes: bit::test(auxv.hwcap2, 2), + // svepmull: bit::test(auxv.hwcap2, 3), + svebitperm: bit::test(auxv.hwcap2, 4), + svesha3: bit::test(auxv.hwcap2, 5), + svesm4: bit::test(auxv.hwcap2, 6), + // flagm2: bit::test(auxv.hwcap2, 7), + frint: bit::test(auxv.hwcap2, 8), + // svei8mm: bit::test(auxv.hwcap2, 9), + svef32mm: bit::test(auxv.hwcap2, 10), + svef64mm: bit::test(auxv.hwcap2, 11), + // svebf16: bit::test(auxv.hwcap2, 12), + i8mm: bit::test(auxv.hwcap2, 13), + bf16: bit::test(auxv.hwcap2, 14), + // dgh: bit::test(auxv.hwcap2, 15), + rng: bit::test(auxv.hwcap2, 16), + bti: bit::test(auxv.hwcap2, 17), + mte: bit::test(auxv.hwcap2, 18), + } + } +} + +#[cfg(feature = "std_detect_file_io")] +impl From for AtHwcap { + /// Reads AtHwcap from /proc/cpuinfo . + fn from(c: super::cpuinfo::CpuInfo) -> Self { + let f = &c.field("Features"); + AtHwcap { + // 64-bit names. FIXME: In 32-bit compatibility mode /proc/cpuinfo will + // map some of the 64-bit names to some 32-bit feature names. This does not + // cover that yet. + fp: f.has("fp"), + asimd: f.has("asimd"), + // evtstrm: f.has("evtstrm"), + aes: f.has("aes"), + pmull: f.has("pmull"), + sha1: f.has("sha1"), + sha2: f.has("sha2"), + crc32: f.has("crc32"), + atomics: f.has("atomics"), + fphp: f.has("fphp"), + asimdhp: f.has("asimdhp"), + // cpuid: f.has("cpuid"), + asimdrdm: f.has("asimdrdm"), + jscvt: f.has("jscvt"), + fcma: f.has("fcma"), + lrcpc: f.has("lrcpc"), + dcpop: f.has("dcpop"), + sha3: f.has("sha3"), + sm3: f.has("sm3"), + sm4: f.has("sm4"), + asimddp: f.has("asimddp"), + sha512: f.has("sha512"), + sve: f.has("sve"), + fhm: f.has("asimdfhm"), + dit: f.has("dit"), + uscat: f.has("uscat"), + ilrcpc: f.has("ilrcpc"), + flagm: f.has("flagm"), + ssbs: f.has("ssbs"), + sb: f.has("sb"), + paca: f.has("paca"), + pacg: f.has("pacg"), + dcpodp: f.has("dcpodp"), + sve2: f.has("sve2"), + sveaes: f.has("sveaes"), + // svepmull: f.has("svepmull"), + svebitperm: f.has("svebitperm"), + svesha3: f.has("svesha3"), + svesm4: f.has("svesm4"), + // flagm2: f.has("flagm2"), + frint: f.has("frint"), + // svei8mm: f.has("svei8mm"), + svef32mm: f.has("svef32mm"), + svef64mm: f.has("svef64mm"), + // svebf16: f.has("svebf16"), + i8mm: f.has("i8mm"), + bf16: f.has("bf16"), + // dgh: f.has("dgh"), + rng: f.has("rng"), + bti: f.has("bti"), + mte: f.has("mte"), + } + } +} + +impl AtHwcap { + /// Initializes the cache from the feature -bits. + /// + /// The feature dependencies here come directly from LLVM's feature definintions: + /// https://github.com/llvm/llvm-project/blob/main/llvm/lib/Target/AArch64/AArch64.td + fn cache(self) -> cache::Initializer { + let mut value = cache::Initializer::default(); + { + let mut enable_feature = |f, enable| { + if enable { + value.set(f as u32); + } + }; + + enable_feature(Feature::fp, self.fp); + // Half-float support requires float support + enable_feature(Feature::fp16, self.fp && self.fphp); + // FHM (fp16fml in LLVM) requires half float support + enable_feature(Feature::fhm, self.fphp && self.fhm); + enable_feature(Feature::pmull, self.pmull); + enable_feature(Feature::crc, self.crc32); + enable_feature(Feature::lse, self.atomics); + enable_feature(Feature::lse2, self.uscat); + enable_feature(Feature::rcpc, self.lrcpc); + // RCPC2 (rcpc-immo in LLVM) requires RCPC support + enable_feature(Feature::rcpc2, self.ilrcpc && self.lrcpc); + enable_feature(Feature::dit, self.dit); + enable_feature(Feature::flagm, self.flagm); + enable_feature(Feature::ssbs, self.ssbs); + enable_feature(Feature::sb, self.sb); + enable_feature(Feature::paca, self.paca); + enable_feature(Feature::pacg, self.pacg); + enable_feature(Feature::dpb, self.dcpop); + enable_feature(Feature::dpb2, self.dcpodp); + enable_feature(Feature::rand, self.rng); + enable_feature(Feature::bti, self.bti); + enable_feature(Feature::mte, self.mte); + // jsconv requires float support + enable_feature(Feature::jsconv, self.jscvt && self.fp); + enable_feature(Feature::rdm, self.asimdrdm); + enable_feature(Feature::dotprod, self.asimddp); + enable_feature(Feature::frintts, self.frint); + + // FEAT_I8MM & FEAT_BF16 also include optional SVE components which linux exposes + // separately. We ignore that distinction here. + enable_feature(Feature::i8mm, self.i8mm); + enable_feature(Feature::bf16, self.bf16); + + // ASIMD support requires float support - if half-floats are + // supported, it also requires half-float support: + let asimd = self.fp && self.asimd && (!self.fphp | self.asimdhp); + enable_feature(Feature::asimd, asimd); + // ASIMD extensions require ASIMD support: + enable_feature(Feature::fcma, self.fcma && asimd); + enable_feature(Feature::sve, self.sve && asimd); + + // SVE extensions require SVE & ASIMD + enable_feature(Feature::f32mm, self.svef32mm && self.sve && asimd); + enable_feature(Feature::f64mm, self.svef64mm && self.sve && asimd); + + // Cryptographic extensions require ASIMD + enable_feature(Feature::aes, self.aes && asimd); + enable_feature(Feature::sha2, self.sha1 && self.sha2 && asimd); + // SHA512/SHA3 require SHA1 & SHA256 + enable_feature( + Feature::sha3, + self.sha512 && self.sha3 && self.sha1 && self.sha2 && asimd, + ); + enable_feature(Feature::sm4, self.sm3 && self.sm4 && asimd); + + // SVE2 requires SVE + let sve2 = self.sve2 && self.sve && asimd; + enable_feature(Feature::sve2, sve2); + // SVE2 extensions require SVE2 and crypto features + enable_feature(Feature::sve2_aes, self.sveaes && sve2 && self.aes); + enable_feature( + Feature::sve2_sm4, + self.svesm4 && sve2 && self.sm3 && self.sm4, + ); + enable_feature( + Feature::sve2_sha3, + self.svesha3 && sve2 && self.sha512 && self.sha3 && self.sha1 && self.sha2, + ); + enable_feature(Feature::sve2_bitperm, self.svebitperm && self.sve2); + } + value + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(feature = "std_detect_file_io")] + mod auxv_from_file { + use super::auxvec::auxv_from_file; + use super::*; + // The baseline hwcaps used in the (artificial) auxv test files. + fn baseline_hwcaps() -> AtHwcap { + AtHwcap { + fp: true, + asimd: true, + aes: true, + pmull: true, + sha1: true, + sha2: true, + crc32: true, + atomics: true, + fphp: true, + asimdhp: true, + asimdrdm: true, + lrcpc: true, + dcpop: true, + asimddp: true, + ssbs: true, + ..AtHwcap::default() + } + } + + #[test] + fn linux_empty_hwcap2_aarch64() { + let file = concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/detect/test_data/linux-empty-hwcap2-aarch64.auxv" + ); + println!("file: {}", file); + let v = auxv_from_file(file).unwrap(); + println!("HWCAP : 0x{:0x}", v.hwcap); + println!("HWCAP2: 0x{:0x}", v.hwcap2); + assert_eq!(AtHwcap::from(v), baseline_hwcaps()); + } + #[test] + fn linux_no_hwcap2_aarch64() { + let file = concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/detect/test_data/linux-no-hwcap2-aarch64.auxv" + ); + println!("file: {}", file); + let v = auxv_from_file(file).unwrap(); + println!("HWCAP : 0x{:0x}", v.hwcap); + println!("HWCAP2: 0x{:0x}", v.hwcap2); + assert_eq!(AtHwcap::from(v), baseline_hwcaps()); + } + #[test] + fn linux_hwcap2_aarch64() { + let file = concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/detect/test_data/linux-hwcap2-aarch64.auxv" + ); + println!("file: {}", file); + let v = auxv_from_file(file).unwrap(); + println!("HWCAP : 0x{:0x}", v.hwcap); + println!("HWCAP2: 0x{:0x}", v.hwcap2); + assert_eq!( + AtHwcap::from(v), + AtHwcap { + // Some other HWCAP bits. + paca: true, + pacg: true, + // HWCAP2-only bits. + dcpodp: true, + frint: true, + rng: true, + bti: true, + mte: true, + ..baseline_hwcaps() + } + ); + } + } +} diff --git a/crux-mir/lib/std_detect/src/detect/os/linux/arm.rs b/crux-mir/lib/std_detect/src/detect/os/linux/arm.rs new file mode 100644 index 000000000..7383e487f --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/os/linux/arm.rs @@ -0,0 +1,79 @@ +//! Run-time feature detection for ARM on Linux. + +use super::auxvec; +use crate::detect::{bit, cache, Feature}; + +/// Try to read the features from the auxiliary vector, and if that fails, try +/// to read them from /proc/cpuinfo. +pub(crate) fn detect_features() -> cache::Initializer { + let mut value = cache::Initializer::default(); + let enable_feature = |value: &mut cache::Initializer, f, enable| { + if enable { + value.set(f as u32); + } + }; + + // The values are part of the platform-specific [asm/hwcap.h][hwcap] + // + // [hwcap]: https://github.com/torvalds/linux/blob/master/arch/arm/include/uapi/asm/hwcap.h + if let Ok(auxv) = auxvec::auxv() { + enable_feature(&mut value, Feature::neon, bit::test(auxv.hwcap, 12)); + enable_feature(&mut value, Feature::pmull, bit::test(auxv.hwcap2, 1)); + enable_feature(&mut value, Feature::crc, bit::test(auxv.hwcap2, 4)); + enable_feature( + &mut value, + Feature::crypto, + bit::test(auxv.hwcap2, 0) + && bit::test(auxv.hwcap2, 1) + && bit::test(auxv.hwcap2, 2) + && bit::test(auxv.hwcap2, 3), + ); + enable_feature(&mut value, Feature::aes, bit::test(auxv.hwcap2, 0)); + // SHA2 requires SHA1 & SHA2 features + enable_feature( + &mut value, + Feature::sha2, + bit::test(auxv.hwcap2, 2) && bit::test(auxv.hwcap2, 3), + ); + return value; + } + + #[cfg(feature = "std_detect_file_io")] + if let Ok(c) = super::cpuinfo::CpuInfo::new() { + enable_feature( + &mut value, + Feature::neon, + c.field("Features").has("neon") && !has_broken_neon(&c), + ); + enable_feature(&mut value, Feature::pmull, c.field("Features").has("pmull")); + enable_feature(&mut value, Feature::crc, c.field("Features").has("crc32")); + enable_feature( + &mut value, + Feature::crypto, + c.field("Features").has("aes") + && c.field("Features").has("pmull") + && c.field("Features").has("sha1") + && c.field("Features").has("sha2"), + ); + enable_feature(&mut value, Feature::aes, c.field("Features").has("aes")); + enable_feature( + &mut value, + Feature::sha2, + c.field("Features").has("sha1") && c.field("Features").has("sha2"), + ); + return value; + } + value +} + +/// Is the CPU known to have a broken NEON unit? +/// +/// See https://crbug.com/341598. +#[cfg(feature = "std_detect_file_io")] +fn has_broken_neon(cpuinfo: &super::cpuinfo::CpuInfo) -> bool { + cpuinfo.field("CPU implementer") == "0x51" + && cpuinfo.field("CPU architecture") == "7" + && cpuinfo.field("CPU variant") == "0x1" + && cpuinfo.field("CPU part") == "0x04d" + && cpuinfo.field("CPU revision") == "0" +} diff --git a/crux-mir/lib/std_detect/src/detect/os/linux/auxvec.rs b/crux-mir/lib/std_detect/src/detect/os/linux/auxvec.rs new file mode 100644 index 000000000..c903903bd --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/os/linux/auxvec.rs @@ -0,0 +1,389 @@ +//! Parses ELF auxiliary vectors. +#![allow(dead_code)] + +pub(crate) const AT_NULL: usize = 0; + +/// Key to access the CPU Hardware capabilities bitfield. +pub(crate) const AT_HWCAP: usize = 16; +/// Key to access the CPU Hardware capabilities 2 bitfield. +#[cfg(any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc", + target_arch = "powerpc64" +))] +pub(crate) const AT_HWCAP2: usize = 26; + +/// Cache HWCAP bitfields of the ELF Auxiliary Vector. +/// +/// If an entry cannot be read all the bits in the bitfield are set to zero. +/// This should be interpreted as all the features being disabled. +#[derive(Debug, Copy, Clone)] +pub(crate) struct AuxVec { + pub hwcap: usize, + #[cfg(any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc", + target_arch = "powerpc64" + ))] + pub hwcap2: usize, +} + +/// ELF Auxiliary Vector +/// +/// The auxiliary vector is a memory region in a running ELF program's stack +/// composed of (key: usize, value: usize) pairs. +/// +/// The keys used in the aux vector are platform dependent. For Linux, they are +/// defined in [linux/auxvec.h][auxvec_h]. The hardware capabilities of a given +/// CPU can be queried with the `AT_HWCAP` and `AT_HWCAP2` keys. +/// +/// There is no perfect way of reading the auxiliary vector. +/// +/// - If the `std_detect_dlsym_getauxval` cargo feature is enabled, this will use +/// `getauxval` if its linked to the binary, and otherwise proceed to a fallback implementation. +/// When `std_detect_dlsym_getauxval` is disabled, this will assume that `getauxval` is +/// linked to the binary - if that is not the case the behavior is undefined. +/// - Otherwise, if the `std_detect_file_io` cargo feature is enabled, it will +/// try to read `/proc/self/auxv`. +/// - If that fails, this function returns an error. +/// +/// Note that run-time feature detection is not invoked for features that can +/// be detected at compile-time. Also note that if this function returns an +/// error, cpuinfo still can (and will) be used to try to perform run-time +/// feature detecton on some platforms. +/// +/// For more information about when `getauxval` is available check the great +/// [`auxv` crate documentation][auxv_docs]. +/// +/// [auxvec_h]: https://github.com/torvalds/linux/blob/master/include/uapi/linux/auxvec.h +/// [auxv_docs]: https://docs.rs/auxv/0.3.3/auxv/ +pub(crate) fn auxv() -> Result { + #[cfg(feature = "std_detect_dlsym_getauxval")] + { + // Try to call a dynamically-linked getauxval function. + if let Ok(hwcap) = getauxval(AT_HWCAP) { + // Targets with only AT_HWCAP: + #[cfg(any( + target_arch = "riscv32", + target_arch = "riscv64", + target_arch = "mips", + target_arch = "mips64" + ))] + { + // Zero could indicate that no features were detected, but it's also used to + // indicate an error. In either case, try the fallback. + if hwcap != 0 { + return Ok(AuxVec { hwcap }); + } + } + + // Targets with AT_HWCAP and AT_HWCAP2: + #[cfg(any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc", + target_arch = "powerpc64" + ))] + { + if let Ok(hwcap2) = getauxval(AT_HWCAP2) { + // Zero could indicate that no features were detected, but it's also used to + // indicate an error. In particular, on many platforms AT_HWCAP2 will be + // legitimately zero, since it contains the most recent feature flags. Use the + // fallback only if no features were detected at all. + if hwcap != 0 || hwcap2 != 0 { + return Ok(AuxVec { hwcap, hwcap2 }); + } + } + } + drop(hwcap); + } + } + + #[cfg(not(feature = "std_detect_dlsym_getauxval"))] + { + // Targets with only AT_HWCAP: + #[cfg(any( + target_arch = "riscv32", + target_arch = "riscv64", + target_arch = "mips", + target_arch = "mips64" + ))] + { + let hwcap = unsafe { libc::getauxval(AT_HWCAP as libc::c_ulong) as usize }; + // Zero could indicate that no features were detected, but it's also used to indicate + // an error. In either case, try the fallback. + if hwcap != 0 { + return Ok(AuxVec { hwcap }); + } + } + + // Targets with AT_HWCAP and AT_HWCAP2: + #[cfg(any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc", + target_arch = "powerpc64" + ))] + { + let hwcap = unsafe { libc::getauxval(AT_HWCAP as libc::c_ulong) as usize }; + let hwcap2 = unsafe { libc::getauxval(AT_HWCAP2 as libc::c_ulong) as usize }; + // Zero could indicate that no features were detected, but it's also used to indicate + // an error. In particular, on many platforms AT_HWCAP2 will be legitimately zero, + // since it contains the most recent feature flags. Use the fallback only if no + // features were detected at all. + if hwcap != 0 || hwcap2 != 0 { + return Ok(AuxVec { hwcap, hwcap2 }); + } + } + } + + #[cfg(feature = "std_detect_file_io")] + { + // If calling getauxval fails, try to read the auxiliary vector from + // its file: + auxv_from_file("/proc/self/auxv") + } + #[cfg(not(feature = "std_detect_file_io"))] + { + Err(()) + } +} + +/// Tries to read the `key` from the auxiliary vector by calling the +/// dynamically-linked `getauxval` function. If the function is not linked, +/// this function return `Err`. +#[cfg(feature = "std_detect_dlsym_getauxval")] +fn getauxval(key: usize) -> Result { + use libc; + pub type F = unsafe extern "C" fn(usize) -> usize; + unsafe { + let ptr = libc::dlsym(libc::RTLD_DEFAULT, "getauxval\0".as_ptr() as *const _); + if ptr.is_null() { + return Err(()); + } + + let ffi_getauxval: F = core::mem::transmute(ptr); + Ok(ffi_getauxval(key)) + } +} + +/// Tries to read the auxiliary vector from the `file`. If this fails, this +/// function returns `Err`. +#[cfg(feature = "std_detect_file_io")] +pub(super) fn auxv_from_file(file: &str) -> Result { + let file = super::read_file(file)?; + + // See . + // + // The auxiliary vector contains at most 32 (key,value) fields: from + // `AT_EXECFN = 31` to `AT_NULL = 0`. That is, a buffer of + // 2*32 `usize` elements is enough to read the whole vector. + let mut buf = [0_usize; 64]; + let len = core::mem::size_of_val(&buf).max(file.len()); + unsafe { + core::ptr::copy_nonoverlapping(file.as_ptr(), buf.as_mut_ptr() as *mut u8, len); + } + + auxv_from_buf(&buf) +} + +/// Tries to interpret the `buffer` as an auxiliary vector. If that fails, this +/// function returns `Err`. +#[cfg(feature = "std_detect_file_io")] +fn auxv_from_buf(buf: &[usize; 64]) -> Result { + // Targets with only AT_HWCAP: + #[cfg(any( + target_arch = "riscv32", + target_arch = "riscv64", + target_arch = "mips", + target_arch = "mips64", + ))] + { + for el in buf.chunks(2) { + match el[0] { + AT_NULL => break, + AT_HWCAP => return Ok(AuxVec { hwcap: el[1] }), + _ => (), + } + } + } + // Targets with AT_HWCAP and AT_HWCAP2: + #[cfg(any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc", + target_arch = "powerpc64" + ))] + { + let mut hwcap = None; + // For some platforms, AT_HWCAP2 was added recently, so let it default to zero. + let mut hwcap2 = 0; + for el in buf.chunks(2) { + match el[0] { + AT_NULL => break, + AT_HWCAP => hwcap = Some(el[1]), + AT_HWCAP2 => hwcap2 = el[1], + _ => (), + } + } + + if let Some(hwcap) = hwcap { + return Ok(AuxVec { hwcap, hwcap2 }); + } + } + drop(buf); + Err(()) +} + +#[cfg(test)] +mod tests { + extern crate auxv as auxv_crate; + use super::*; + + // Reads the Auxiliary Vector key from /proc/self/auxv + // using the auxv crate. + #[cfg(feature = "std_detect_file_io")] + fn auxv_crate_getprocfs(key: usize) -> Option { + use self::auxv_crate::procfs::search_procfs_auxv; + use self::auxv_crate::AuxvType; + let k = key as AuxvType; + match search_procfs_auxv(&[k]) { + Ok(v) => Some(v[&k] as usize), + Err(_) => None, + } + } + + // Reads the Auxiliary Vector key from getauxval() + // using the auxv crate. + #[cfg(not(any(target_arch = "mips", target_arch = "mips64")))] + fn auxv_crate_getauxval(key: usize) -> Option { + use self::auxv_crate::getauxval::Getauxval; + use self::auxv_crate::AuxvType; + let q = auxv_crate::getauxval::NativeGetauxval {}; + match q.getauxval(key as AuxvType) { + Ok(v) => Some(v as usize), + Err(_) => None, + } + } + + // FIXME: on mips/mips64 getauxval returns 0, and /proc/self/auxv + // does not always contain the AT_HWCAP key under qemu. + #[cfg(any( + target_arch = "arm", + target_arch = "powerpc", + target_arch = "powerpc64" + ))] + #[test] + fn auxv_crate() { + let v = auxv(); + if let Some(hwcap) = auxv_crate_getauxval(AT_HWCAP) { + let rt_hwcap = v.expect("failed to find hwcap key").hwcap; + assert_eq!(rt_hwcap, hwcap); + } + + // Targets with AT_HWCAP and AT_HWCAP2: + #[cfg(any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc", + target_arch = "powerpc64" + ))] + { + if let Some(hwcap2) = auxv_crate_getauxval(AT_HWCAP2) { + let rt_hwcap2 = v.expect("failed to find hwcap2 key").hwcap2; + assert_eq!(rt_hwcap2, hwcap2); + } + } + } + + #[test] + fn auxv_dump() { + if let Ok(auxvec) = auxv() { + println!("{:?}", auxvec); + } else { + println!("both getauxval() and reading /proc/self/auxv failed!"); + } + } + + #[cfg(feature = "std_detect_file_io")] + cfg_if::cfg_if! { + if #[cfg(target_arch = "arm")] { + #[test] + fn linux_rpi3() { + let file = concat!(env!("CARGO_MANIFEST_DIR"), "/src/detect/test_data/linux-rpi3.auxv"); + println!("file: {}", file); + let v = auxv_from_file(file).unwrap(); + assert_eq!(v.hwcap, 4174038); + assert_eq!(v.hwcap2, 16); + } + + #[test] + fn linux_macos_vb() { + let file = concat!(env!("CARGO_MANIFEST_DIR"), "/src/detect/test_data/macos-virtualbox-linux-x86-4850HQ.auxv"); + println!("file: {}", file); + // The file contains HWCAP but not HWCAP2. In that case, we treat HWCAP2 as zero. + let v = auxv_from_file(file).unwrap(); + assert_eq!(v.hwcap, 126614527); + assert_eq!(v.hwcap2, 0); + } + } else if #[cfg(target_arch = "aarch64")] { + #[test] + fn linux_artificial_aarch64() { + let file = concat!(env!("CARGO_MANIFEST_DIR"), "/src/detect/test_data/linux-artificial-aarch64.auxv"); + println!("file: {}", file); + let v = auxv_from_file(file).unwrap(); + assert_eq!(v.hwcap, 0x0123456789abcdef); + assert_eq!(v.hwcap2, 0x02468ace13579bdf); + } + #[test] + fn linux_no_hwcap2_aarch64() { + let file = concat!(env!("CARGO_MANIFEST_DIR"), "/src/detect/test_data/linux-no-hwcap2-aarch64.auxv"); + println!("file: {}", file); + let v = auxv_from_file(file).unwrap(); + // An absent HWCAP2 is treated as zero, and does not prevent acceptance of HWCAP. + assert_ne!(v.hwcap, 0); + assert_eq!(v.hwcap2, 0); + } + } + } + + #[test] + #[cfg(feature = "std_detect_file_io")] + fn auxv_dump_procfs() { + if let Ok(auxvec) = auxv_from_file("/proc/self/auxv") { + println!("{:?}", auxvec); + } else { + println!("reading /proc/self/auxv failed!"); + } + } + + #[cfg(any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc", + target_arch = "powerpc64" + ))] + #[test] + #[cfg(feature = "std_detect_file_io")] + fn auxv_crate_procfs() { + let v = auxv(); + if let Some(hwcap) = auxv_crate_getprocfs(AT_HWCAP) { + assert_eq!(v.unwrap().hwcap, hwcap); + } + + // Targets with AT_HWCAP and AT_HWCAP2: + #[cfg(any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc", + target_arch = "powerpc64" + ))] + { + if let Some(hwcap2) = auxv_crate_getprocfs(AT_HWCAP2) { + assert_eq!(v.unwrap().hwcap2, hwcap2); + } + } + } +} diff --git a/crux-mir/lib/std_detect/src/detect/os/linux/cpuinfo.rs b/crux-mir/lib/std_detect/src/detect/os/linux/cpuinfo.rs new file mode 100644 index 000000000..48a5c9728 --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/os/linux/cpuinfo.rs @@ -0,0 +1,331 @@ +//! Parses /proc/cpuinfo +#![cfg_attr(not(target_arch = "arm"), allow(dead_code))] + +use alloc::string::String; + +/// cpuinfo +pub(crate) struct CpuInfo { + raw: String, +} + +impl CpuInfo { + /// Reads /proc/cpuinfo into CpuInfo. + pub(crate) fn new() -> Result { + let raw = super::read_file("/proc/cpuinfo")?; + Ok(Self { + raw: String::from_utf8(raw).map_err(|_| ())?, + }) + } + /// Returns the value of the cpuinfo `field`. + pub(crate) fn field(&self, field: &str) -> CpuInfoField<'_> { + for l in self.raw.lines() { + if l.trim().starts_with(field) { + return CpuInfoField::new(l.split(": ").nth(1)); + } + } + CpuInfoField(None) + } + + /// Returns the `raw` contents of `/proc/cpuinfo` + #[cfg(test)] + fn raw(&self) -> &String { + &self.raw + } + + #[cfg(test)] + fn from_str(other: &str) -> Result { + Ok(Self { + raw: String::from(other), + }) + } +} + +/// Field of cpuinfo +#[derive(Debug)] +pub(crate) struct CpuInfoField<'a>(Option<&'a str>); + +impl<'a> PartialEq<&'a str> for CpuInfoField<'a> { + fn eq(&self, other: &&'a str) -> bool { + match self.0 { + None => other.is_empty(), + Some(f) => f == other.trim(), + } + } +} + +impl<'a> CpuInfoField<'a> { + pub(crate) fn new<'b>(v: Option<&'b str>) -> CpuInfoField<'b> { + match v { + None => CpuInfoField::<'b>(None), + Some(f) => CpuInfoField::<'b>(Some(f.trim())), + } + } + /// Does the field exist? + #[cfg(test)] + pub(crate) fn exists(&self) -> bool { + self.0.is_some() + } + /// Does the field contain `other`? + pub(crate) fn has(&self, other: &str) -> bool { + match self.0 { + None => other.is_empty(), + Some(f) => { + let other = other.trim(); + for v in f.split(' ') { + if v == other { + return true; + } + } + false + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn raw_dump() { + let cpuinfo = CpuInfo::new().unwrap(); + if cpuinfo.field("vendor_id") == "GenuineIntel" { + assert!(cpuinfo.field("flags").exists()); + assert!(!cpuinfo.field("vendor33_id").exists()); + assert!(cpuinfo.field("flags").has("sse")); + assert!(!cpuinfo.field("flags").has("avx314")); + } + println!("{}", cpuinfo.raw()); + } + + const CORE_DUO_T6500: &str = r"processor : 0 +vendor_id : GenuineIntel +cpu family : 6 +model : 23 +model name : Intel(R) Core(TM)2 Duo CPU T6500 @ 2.10GHz +stepping : 10 +microcode : 0xa0b +cpu MHz : 1600.000 +cache size : 2048 KB +physical id : 0 +siblings : 2 +core id : 0 +cpu cores : 2 +apicid : 0 +initial apicid : 0 +fdiv_bug : no +hlt_bug : no +f00f_bug : no +coma_bug : no +fpu : yes +fpu_exception : yes +cpuid level : 13 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe nx lm constant_tsc arch_perfmon pebs bts aperfmperf pni dtes64 monitor ds_cpl est tm2 ssse3 cx16 xtpr pdcm sse4_1 xsave lahf_lm dtherm +bogomips : 4190.43 +clflush size : 64 +cache_alignment : 64 +address sizes : 36 bits physical, 48 bits virtual +power management: +"; + + #[test] + fn core_duo_t6500() { + let cpuinfo = CpuInfo::from_str(CORE_DUO_T6500).unwrap(); + assert_eq!(cpuinfo.field("vendor_id"), "GenuineIntel"); + assert_eq!(cpuinfo.field("cpu family"), "6"); + assert_eq!(cpuinfo.field("model"), "23"); + assert_eq!( + cpuinfo.field("model name"), + "Intel(R) Core(TM)2 Duo CPU T6500 @ 2.10GHz" + ); + assert_eq!( + cpuinfo.field("flags"), + "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe nx lm constant_tsc arch_perfmon pebs bts aperfmperf pni dtes64 monitor ds_cpl est tm2 ssse3 cx16 xtpr pdcm sse4_1 xsave lahf_lm dtherm" + ); + assert!(cpuinfo.field("flags").has("fpu")); + assert!(cpuinfo.field("flags").has("dtherm")); + assert!(cpuinfo.field("flags").has("sse2")); + assert!(!cpuinfo.field("flags").has("avx")); + } + + const ARM_CORTEX_A53: &str = r"Processor : AArch64 Processor rev 3 (aarch64) + processor : 0 + processor : 1 + processor : 2 + processor : 3 + processor : 4 + processor : 5 + processor : 6 + processor : 7 + Features : fp asimd evtstrm aes pmull sha1 sha2 crc32 + CPU implementer : 0x41 + CPU architecture: AArch64 + CPU variant : 0x0 + CPU part : 0xd03 + CPU revision : 3 + + Hardware : HiKey Development Board + "; + + #[test] + fn arm_cortex_a53() { + let cpuinfo = CpuInfo::from_str(ARM_CORTEX_A53).unwrap(); + assert_eq!( + cpuinfo.field("Processor"), + "AArch64 Processor rev 3 (aarch64)" + ); + assert_eq!( + cpuinfo.field("Features"), + "fp asimd evtstrm aes pmull sha1 sha2 crc32" + ); + assert!(cpuinfo.field("Features").has("pmull")); + assert!(!cpuinfo.field("Features").has("neon")); + assert!(cpuinfo.field("Features").has("asimd")); + } + + const ARM_CORTEX_A57: &str = r"Processor : Cortex A57 Processor rev 1 (aarch64) +processor : 0 +processor : 1 +processor : 2 +processor : 3 +Features : fp asimd aes pmull sha1 sha2 crc32 wp half thumb fastmult vfp edsp neon vfpv3 tlsi vfpv4 idiva idivt +CPU implementer : 0x41 +CPU architecture: 8 +CPU variant : 0x1 +CPU part : 0xd07 +CPU revision : 1"; + + #[test] + fn arm_cortex_a57() { + let cpuinfo = CpuInfo::from_str(ARM_CORTEX_A57).unwrap(); + assert_eq!( + cpuinfo.field("Processor"), + "Cortex A57 Processor rev 1 (aarch64)" + ); + assert_eq!( + cpuinfo.field("Features"), + "fp asimd aes pmull sha1 sha2 crc32 wp half thumb fastmult vfp edsp neon vfpv3 tlsi vfpv4 idiva idivt" + ); + assert!(cpuinfo.field("Features").has("pmull")); + assert!(cpuinfo.field("Features").has("neon")); + assert!(cpuinfo.field("Features").has("asimd")); + } + + const RISCV_RV64GC: &str = r"processor : 0 +hart : 3 +isa : rv64imafdc +mmu : sv39 +uarch : sifive,u74-mc + +processor : 1 +hart : 1 +isa : rv64imafdc +mmu : sv39 +uarch : sifive,u74-mc + +processor : 2 +hart : 2 +isa : rv64imafdc +mmu : sv39 +uarch : sifive,u74-mc + +processor : 3 +hart : 4 +isa : rv64imafdc +mmu : sv39 +uarch : sifive,u74-mc"; + + #[test] + fn riscv_rv64gc() { + let cpuinfo = CpuInfo::from_str(RISCV_RV64GC).unwrap(); + assert_eq!(cpuinfo.field("isa"), "rv64imafdc"); + assert_eq!(cpuinfo.field("mmu"), "sv39"); + assert_eq!(cpuinfo.field("uarch"), "sifive,u74-mc"); + } + + const POWER8E_POWERKVM: &str = r"processor : 0 +cpu : POWER8E (raw), altivec supported +clock : 3425.000000MHz +revision : 2.1 (pvr 004b 0201) + +processor : 1 +cpu : POWER8E (raw), altivec supported +clock : 3425.000000MHz +revision : 2.1 (pvr 004b 0201) + +processor : 2 +cpu : POWER8E (raw), altivec supported +clock : 3425.000000MHz +revision : 2.1 (pvr 004b 0201) + +processor : 3 +cpu : POWER8E (raw), altivec supported +clock : 3425.000000MHz +revision : 2.1 (pvr 004b 0201) + +timebase : 512000000 +platform : pSeries +model : IBM pSeries (emulated by qemu) +machine : CHRP IBM pSeries (emulated by qemu)"; + + #[test] + fn power8_powerkvm() { + let cpuinfo = CpuInfo::from_str(POWER8E_POWERKVM).unwrap(); + assert_eq!(cpuinfo.field("cpu"), "POWER8E (raw), altivec supported"); + + assert!(cpuinfo.field("cpu").has("altivec")); + } + + const POWER5P: &str = r"processor : 0 +cpu : POWER5+ (gs) +clock : 1900.098000MHz +revision : 2.1 (pvr 003b 0201) + +processor : 1 +cpu : POWER5+ (gs) +clock : 1900.098000MHz +revision : 2.1 (pvr 003b 0201) + +processor : 2 +cpu : POWER5+ (gs) +clock : 1900.098000MHz +revision : 2.1 (pvr 003b 0201) + +processor : 3 +cpu : POWER5+ (gs) +clock : 1900.098000MHz +revision : 2.1 (pvr 003b 0201) + +processor : 4 +cpu : POWER5+ (gs) +clock : 1900.098000MHz +revision : 2.1 (pvr 003b 0201) + +processor : 5 +cpu : POWER5+ (gs) +clock : 1900.098000MHz +revision : 2.1 (pvr 003b 0201) + +processor : 6 +cpu : POWER5+ (gs) +clock : 1900.098000MHz +revision : 2.1 (pvr 003b 0201) + +processor : 7 +cpu : POWER5+ (gs) +clock : 1900.098000MHz +revision : 2.1 (pvr 003b 0201) + +timebase : 237331000 +platform : pSeries +machine : CHRP IBM,9133-55A"; + + #[test] + fn power5p() { + let cpuinfo = CpuInfo::from_str(POWER5P).unwrap(); + assert_eq!(cpuinfo.field("cpu"), "POWER5+ (gs)"); + + assert!(!cpuinfo.field("cpu").has("altivec")); + } +} diff --git a/crux-mir/lib/std_detect/src/detect/os/linux/mips.rs b/crux-mir/lib/std_detect/src/detect/os/linux/mips.rs new file mode 100644 index 000000000..9c030f41a --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/os/linux/mips.rs @@ -0,0 +1,25 @@ +//! Run-time feature detection for MIPS on Linux. + +use super::auxvec; +use crate::detect::{bit, cache, Feature}; + +/// Try to read the features from the auxiliary vector, and if that fails, try +/// to read them from `/proc/cpuinfo`. +pub(crate) fn detect_features() -> cache::Initializer { + let mut value = cache::Initializer::default(); + let enable_feature = |value: &mut cache::Initializer, f, enable| { + if enable { + value.set(f as u32); + } + }; + + // The values are part of the platform-specific [asm/hwcap.h][hwcap] + // + // [hwcap]: https://github.com/torvalds/linux/blob/master/arch/arm64/include/uapi/asm/hwcap.h + if let Ok(auxv) = auxvec::auxv() { + enable_feature(&mut value, Feature::msa, bit::test(auxv.hwcap, 1)); + return value; + } + // TODO: fall back via `cpuinfo`. + value +} diff --git a/crux-mir/lib/std_detect/src/detect/os/linux/mod.rs b/crux-mir/lib/std_detect/src/detect/os/linux/mod.rs new file mode 100644 index 000000000..a49a72783 --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/os/linux/mod.rs @@ -0,0 +1,64 @@ +//! Run-time feature detection on Linux +//! +#[cfg(feature = "std_detect_file_io")] +use alloc::vec::Vec; + +mod auxvec; + +#[cfg(feature = "std_detect_file_io")] +mod cpuinfo; + +#[cfg(feature = "std_detect_file_io")] +fn read_file(path: &str) -> Result, ()> { + let mut path = Vec::from(path.as_bytes()); + path.push(0); + + unsafe { + let file = libc::open(path.as_ptr() as *const libc::c_char, libc::O_RDONLY); + if file == -1 { + return Err(()); + } + + let mut data = Vec::new(); + loop { + data.reserve(4096); + let spare = data.spare_capacity_mut(); + match libc::read(file, spare.as_mut_ptr() as *mut _, spare.len()) { + -1 => { + libc::close(file); + return Err(()); + } + 0 => break, + n => data.set_len(data.len() + n as usize), + } + } + + libc::close(file); + Ok(data) + } +} + +cfg_if::cfg_if! { + if #[cfg(target_arch = "aarch64")] { + mod aarch64; + pub(crate) use self::aarch64::detect_features; + } else if #[cfg(target_arch = "arm")] { + mod arm; + pub(crate) use self::arm::detect_features; + } else if #[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] { + mod riscv; + pub(crate) use self::riscv::detect_features; + } else if #[cfg(any(target_arch = "mips", target_arch = "mips64"))] { + mod mips; + pub(crate) use self::mips::detect_features; + } else if #[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] { + mod powerpc; + pub(crate) use self::powerpc::detect_features; + } else { + use crate::detect::cache; + /// Performs run-time feature detection. + pub(crate) fn detect_features() -> cache::Initializer { + cache::Initializer::default() + } + } +} diff --git a/crux-mir/lib/std_detect/src/detect/os/linux/powerpc.rs b/crux-mir/lib/std_detect/src/detect/os/linux/powerpc.rs new file mode 100644 index 000000000..c3308e815 --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/os/linux/powerpc.rs @@ -0,0 +1,36 @@ +//! Run-time feature detection for PowerPC on Linux. + +use super::auxvec; +use crate::detect::{cache, Feature}; + +/// Try to read the features from the auxiliary vector, and if that fails, try +/// to read them from /proc/cpuinfo. +pub(crate) fn detect_features() -> cache::Initializer { + let mut value = cache::Initializer::default(); + let enable_feature = |value: &mut cache::Initializer, f, enable| { + if enable { + value.set(f as u32); + } + }; + + // The values are part of the platform-specific [asm/cputable.h][cputable] + // + // [cputable]: https://github.com/torvalds/linux/blob/master/arch/powerpc/include/uapi/asm/cputable.h + if let Ok(auxv) = auxvec::auxv() { + // note: the PowerPC values are the mask to do the test (instead of the + // index of the bit to test like in ARM and Aarch64) + enable_feature(&mut value, Feature::altivec, auxv.hwcap & 0x10000000 != 0); + enable_feature(&mut value, Feature::vsx, auxv.hwcap & 0x00000080 != 0); + enable_feature(&mut value, Feature::power8, auxv.hwcap2 & 0x80000000 != 0); + return value; + } + + // PowerPC's /proc/cpuinfo lacks a proper Feature field, + // but `altivec` support is indicated in the `cpu` field. + #[cfg(feature = "std_detect_file_io")] + if let Ok(c) = super::cpuinfo::CpuInfo::new() { + enable_feature(&mut value, Feature::altivec, c.field("cpu").has("altivec")); + return value; + } + value +} diff --git a/crux-mir/lib/std_detect/src/detect/os/linux/riscv.rs b/crux-mir/lib/std_detect/src/detect/os/linux/riscv.rs new file mode 100644 index 000000000..1ec06959a --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/os/linux/riscv.rs @@ -0,0 +1,73 @@ +//! Run-time feature detection for RISC-V on Linux. + +use super::auxvec; +use crate::detect::{bit, cache, Feature}; + +/// Read list of supported features from the auxiliary vector. +pub(crate) fn detect_features() -> cache::Initializer { + let mut value = cache::Initializer::default(); + let enable_feature = |value: &mut cache::Initializer, feature, enable| { + if enable { + value.set(feature as u32); + } + }; + let enable_features = |value: &mut cache::Initializer, feature_slice: &[Feature], enable| { + if enable { + for feature in feature_slice { + value.set(*feature as u32); + } + } + }; + + // The values are part of the platform-specific [asm/hwcap.h][hwcap] + // + // [hwcap]: https://github.com/torvalds/linux/blob/master/arch/riscv/include/asm/hwcap.h + let auxv = auxvec::auxv().expect("read auxvec"); // should not fail on RISC-V platform + enable_feature( + &mut value, + Feature::a, + bit::test(auxv.hwcap, (b'a' - b'a').into()), + ); + enable_feature( + &mut value, + Feature::c, + bit::test(auxv.hwcap, (b'c' - b'a').into()), + ); + enable_features( + &mut value, + &[Feature::d, Feature::f, Feature::zicsr], + bit::test(auxv.hwcap, (b'd' - b'a').into()), + ); + enable_features( + &mut value, + &[Feature::f, Feature::zicsr], + bit::test(auxv.hwcap, (b'f' - b'a').into()), + ); + let has_i = bit::test(auxv.hwcap, (b'i' - b'a').into()); + // If future RV128I is supported, implement with `enable_feature` here + #[cfg(target_pointer_width = "64")] + enable_feature(&mut value, Feature::rv64i, has_i); + #[cfg(target_pointer_width = "32")] + enable_feature(&mut value, Feature::rv32i, has_i); + #[cfg(target_pointer_width = "32")] + enable_feature( + &mut value, + Feature::rv32e, + bit::test(auxv.hwcap, (b'e' - b'a').into()), + ); + enable_feature( + &mut value, + Feature::h, + bit::test(auxv.hwcap, (b'h' - b'a').into()), + ); + enable_feature( + &mut value, + Feature::m, + bit::test(auxv.hwcap, (b'm' - b'a').into()), + ); + // FIXME: Auxvec does not show supervisor feature support, but this mode may be useful + // to detect when Rust is used to write Linux kernel modules. + // These should be more than Auxvec way to detect supervisor features. + + value +} diff --git a/crux-mir/lib/std_detect/src/detect/os/other.rs b/crux-mir/lib/std_detect/src/detect/os/other.rs new file mode 100644 index 000000000..091fafc4e --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/os/other.rs @@ -0,0 +1,8 @@ +//! Other operating systems + +use crate::detect::cache; + +#[allow(dead_code)] +pub(crate) fn detect_features() -> cache::Initializer { + cache::Initializer::default() +} diff --git a/crux-mir/lib/std_detect/src/detect/os/windows/aarch64.rs b/crux-mir/lib/std_detect/src/detect/os/windows/aarch64.rs new file mode 100644 index 000000000..051ad6d1b --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/os/windows/aarch64.rs @@ -0,0 +1,59 @@ +//! Run-time feature detection for Aarch64 on Windows. + +use crate::detect::{cache, Feature}; + +/// Try to read the features using IsProcessorFeaturePresent. +pub(crate) fn detect_features() -> cache::Initializer { + type DWORD = u32; + type BOOL = i32; + + const FALSE: BOOL = 0; + // The following Microsoft documents isn't updated for aarch64. + // https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-isprocessorfeaturepresent + // These are defined in winnt.h of Windows SDK + const PF_ARM_NEON_INSTRUCTIONS_AVAILABLE: u32 = 19; + const PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE: u32 = 30; + const PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE: u32 = 31; + + extern "system" { + pub fn IsProcessorFeaturePresent(ProcessorFeature: DWORD) -> BOOL; + } + + let mut value = cache::Initializer::default(); + { + let mut enable_feature = |f, enable| { + if enable { + value.set(f as u32); + } + }; + + // Some features such Feature::fp may be supported on current CPU, + // but no way to detect it by OS API. + // Also, we require unsafe block for the extern "system" calls. + unsafe { + enable_feature( + Feature::asimd, + IsProcessorFeaturePresent(PF_ARM_NEON_INSTRUCTIONS_AVAILABLE) != FALSE, + ); + enable_feature( + Feature::crc, + IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE) != FALSE, + ); + // PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE means aes, sha1, sha2 and + // pmull support + enable_feature( + Feature::aes, + IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) != FALSE, + ); + enable_feature( + Feature::pmull, + IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) != FALSE, + ); + enable_feature( + Feature::sha2, + IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) != FALSE, + ); + } + } + value +} diff --git a/crux-mir/lib/std_detect/src/detect/os/x86.rs b/crux-mir/lib/std_detect/src/detect/os/x86.rs new file mode 100644 index 000000000..ea5f595ec --- /dev/null +++ b/crux-mir/lib/std_detect/src/detect/os/x86.rs @@ -0,0 +1,273 @@ +//! x86 run-time feature detection is OS independent. + +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +use core::mem; + +use crate::detect::{bit, cache, Feature}; + +/// Run-time feature detection on x86 works by using the CPUID instruction. +/// +/// The [CPUID Wikipedia page][wiki_cpuid] contains +/// all the information about which flags to set to query which values, and in +/// which registers these are reported. +/// +/// The definitive references are: +/// - [Intel 64 and IA-32 Architectures Software Developer's Manual Volume 2: +/// Instruction Set Reference, A-Z][intel64_ref]. +/// - [AMD64 Architecture Programmer's Manual, Volume 3: General-Purpose and +/// System Instructions][amd64_ref]. +/// +/// [wiki_cpuid]: https://en.wikipedia.org/wiki/CPUID +/// [intel64_ref]: http://www.intel.de/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf +/// [amd64_ref]: http://support.amd.com/TechDocs/24594.pdf +#[allow(clippy::similar_names)] +pub(crate) fn detect_features() -> cache::Initializer { + let mut value = cache::Initializer::default(); + + // If the x86 CPU does not support the CPUID instruction then it is too + // old to support any of the currently-detectable features. + if !has_cpuid() { + return value; + } + + // Calling `__cpuid`/`__cpuid_count` from here on is safe because the CPU + // has `cpuid` support. + + // 0. EAX = 0: Basic Information: + // - EAX returns the "Highest Function Parameter", that is, the maximum + // leaf value for subsequent calls of `cpuinfo` in range [0, + // 0x8000_0000]. - The vendor ID is stored in 12 u8 ascii chars, + // returned in EBX, EDX, and ECX (in that order): + let (max_basic_leaf, vendor_id) = unsafe { + let CpuidResult { + eax: max_basic_leaf, + ebx, + ecx, + edx, + } = __cpuid(0); + let vendor_id: [[u8; 4]; 3] = [ + mem::transmute(ebx), + mem::transmute(edx), + mem::transmute(ecx), + ]; + let vendor_id: [u8; 12] = mem::transmute(vendor_id); + (max_basic_leaf, vendor_id) + }; + + if max_basic_leaf < 1 { + // Earlier Intel 486, CPUID not implemented + return value; + } + + // EAX = 1, ECX = 0: Queries "Processor Info and Feature Bits"; + // Contains information about most x86 features. + let CpuidResult { + ecx: proc_info_ecx, + edx: proc_info_edx, + .. + } = unsafe { __cpuid(0x0000_0001_u32) }; + + // EAX = 7, ECX = 0: Queries "Extended Features"; + // Contains information about bmi,bmi2, and avx2 support. + let (extended_features_ebx, extended_features_ecx) = if max_basic_leaf >= 7 { + let CpuidResult { ebx, ecx, .. } = unsafe { __cpuid(0x0000_0007_u32) }; + (ebx, ecx) + } else { + (0, 0) // CPUID does not support "Extended Features" + }; + + // EAX = 0x8000_0000, ECX = 0: Get Highest Extended Function Supported + // - EAX returns the max leaf value for extended information, that is, + // `cpuid` calls in range [0x8000_0000; u32::MAX]: + let CpuidResult { + eax: extended_max_basic_leaf, + .. + } = unsafe { __cpuid(0x8000_0000_u32) }; + + // EAX = 0x8000_0001, ECX=0: Queries "Extended Processor Info and Feature + // Bits" + let extended_proc_info_ecx = if extended_max_basic_leaf >= 1 { + let CpuidResult { ecx, .. } = unsafe { __cpuid(0x8000_0001_u32) }; + ecx + } else { + 0 + }; + + { + // borrows value till the end of this scope: + let mut enable = |r, rb, f| { + if bit::test(r as usize, rb) { + value.set(f as u32); + } + }; + + enable(proc_info_ecx, 0, Feature::sse3); + enable(proc_info_ecx, 1, Feature::pclmulqdq); + enable(proc_info_ecx, 9, Feature::ssse3); + enable(proc_info_ecx, 13, Feature::cmpxchg16b); + enable(proc_info_ecx, 19, Feature::sse4_1); + enable(proc_info_ecx, 20, Feature::sse4_2); + enable(proc_info_ecx, 23, Feature::popcnt); + enable(proc_info_ecx, 25, Feature::aes); + enable(proc_info_ecx, 29, Feature::f16c); + enable(proc_info_ecx, 30, Feature::rdrand); + enable(extended_features_ebx, 18, Feature::rdseed); + enable(extended_features_ebx, 19, Feature::adx); + enable(extended_features_ebx, 11, Feature::rtm); + enable(proc_info_edx, 4, Feature::tsc); + enable(proc_info_edx, 23, Feature::mmx); + enable(proc_info_edx, 24, Feature::fxsr); + enable(proc_info_edx, 25, Feature::sse); + enable(proc_info_edx, 26, Feature::sse2); + enable(extended_features_ebx, 29, Feature::sha); + + enable(extended_features_ebx, 3, Feature::bmi1); + enable(extended_features_ebx, 8, Feature::bmi2); + + // `XSAVE` and `AVX` support: + let cpu_xsave = bit::test(proc_info_ecx as usize, 26); + if cpu_xsave { + // 0. Here the CPU supports `XSAVE`. + + // 1. Detect `OSXSAVE`, that is, whether the OS is AVX enabled and + // supports saving the state of the AVX/AVX2 vector registers on + // context-switches, see: + // + // - [intel: is avx enabled?][is_avx_enabled], + // - [mozilla: sse.cpp][mozilla_sse_cpp]. + // + // [is_avx_enabled]: https://software.intel.com/en-us/blogs/2011/04/14/is-avx-enabled + // [mozilla_sse_cpp]: https://hg.mozilla.org/mozilla-central/file/64bab5cbb9b6/mozglue/build/SSE.cpp#l190 + let cpu_osxsave = bit::test(proc_info_ecx as usize, 27); + + if cpu_osxsave { + // 2. The OS must have signaled the CPU that it supports saving and + // restoring the: + // + // * SSE -> `XCR0.SSE[1]` + // * AVX -> `XCR0.AVX[2]` + // * AVX-512 -> `XCR0.AVX-512[7:5]`. + // + // by setting the corresponding bits of `XCR0` to `1`. + // + // This is safe because the CPU supports `xsave` + // and the OS has set `osxsave`. + let xcr0 = unsafe { _xgetbv(0) }; + // Test `XCR0.SSE[1]` and `XCR0.AVX[2]` with the mask `0b110 == 6`: + let os_avx_support = xcr0 & 6 == 6; + // Test `XCR0.AVX-512[7:5]` with the mask `0b1110_0000 == 224`: + let os_avx512_support = xcr0 & 224 == 224; + + // Only if the OS and the CPU support saving/restoring the AVX + // registers we enable `xsave` support: + if os_avx_support { + // See "13.3 ENABLING THE XSAVE FEATURE SET AND XSAVE-ENABLED + // FEATURES" in the "Intel® 64 and IA-32 Architectures Software + // Developer’s Manual, Volume 1: Basic Architecture": + // + // "Software enables the XSAVE feature set by setting + // CR4.OSXSAVE[bit 18] to 1 (e.g., with the MOV to CR4 + // instruction). If this bit is 0, execution of any of XGETBV, + // XRSTOR, XRSTORS, XSAVE, XSAVEC, XSAVEOPT, XSAVES, and XSETBV + // causes an invalid-opcode exception (#UD)" + // + enable(proc_info_ecx, 26, Feature::xsave); + + // For `xsaveopt`, `xsavec`, and `xsaves` we need to query: + // Processor Extended State Enumeration Sub-leaf (EAX = 0DH, + // ECX = 1): + if max_basic_leaf >= 0xd { + let CpuidResult { + eax: proc_extended_state1_eax, + .. + } = unsafe { __cpuid_count(0xd_u32, 1) }; + enable(proc_extended_state1_eax, 0, Feature::xsaveopt); + enable(proc_extended_state1_eax, 1, Feature::xsavec); + enable(proc_extended_state1_eax, 3, Feature::xsaves); + } + + // FMA (uses 256-bit wide registers): + enable(proc_info_ecx, 12, Feature::fma); + + // And AVX/AVX2: + enable(proc_info_ecx, 28, Feature::avx); + enable(extended_features_ebx, 5, Feature::avx2); + + // For AVX-512 the OS also needs to support saving/restoring + // the extended state, only then we enable AVX-512 support: + if os_avx512_support { + enable(extended_features_ebx, 16, Feature::avx512f); + enable(extended_features_ebx, 17, Feature::avx512dq); + enable(extended_features_ebx, 21, Feature::avx512ifma); + enable(extended_features_ebx, 26, Feature::avx512pf); + enable(extended_features_ebx, 27, Feature::avx512er); + enable(extended_features_ebx, 28, Feature::avx512cd); + enable(extended_features_ebx, 30, Feature::avx512bw); + enable(extended_features_ebx, 31, Feature::avx512vl); + enable(extended_features_ecx, 1, Feature::avx512vbmi); + enable(extended_features_ecx, 5, Feature::avx512bf16); + enable(extended_features_ecx, 6, Feature::avx512vbmi2); + enable(extended_features_ecx, 8, Feature::avx512gfni); + enable(extended_features_ecx, 8, Feature::avx512vp2intersect); + enable(extended_features_ecx, 9, Feature::avx512vaes); + enable(extended_features_ecx, 10, Feature::avx512vpclmulqdq); + enable(extended_features_ecx, 11, Feature::avx512vnni); + enable(extended_features_ecx, 12, Feature::avx512bitalg); + enable(extended_features_ecx, 14, Feature::avx512vpopcntdq); + } + } + } + } + + // This detects ABM on AMD CPUs and LZCNT on Intel CPUs. + // On intel CPUs with popcnt, lzcnt implements the + // "missing part" of ABM, so we map both to the same + // internal feature. + // + // The `is_x86_feature_detected!("lzcnt")` macro then + // internally maps to Feature::abm. + enable(extended_proc_info_ecx, 5, Feature::lzcnt); + + // As Hygon Dhyana originates from AMD technology and shares most of the architecture with + // AMD's family 17h, but with different CPU Vendor ID("HygonGenuine")/Family series + // number(Family 18h). + // + // For CPUID feature bits, Hygon Dhyana(family 18h) share the same definition with AMD + // family 17h. + // + // Related AMD CPUID specification is https://www.amd.com/system/files/TechDocs/25481.pdf. + // Related Hygon kernel patch can be found on + // http://lkml.kernel.org/r/5ce86123a7b9dad925ac583d88d2f921040e859b.1538583282.git.puwen@hygon.cn + if vendor_id == *b"AuthenticAMD" || vendor_id == *b"HygonGenuine" { + // These features are available on AMD arch CPUs: + enable(extended_proc_info_ecx, 6, Feature::sse4a); + enable(extended_proc_info_ecx, 21, Feature::tbm); + } + } + + // Unfortunately, some Skylake chips erroneously report support for BMI1 and + // BMI2 without actual support. These chips don't support AVX, and it seems + // that all Intel chips with non-erroneous support BMI do (I didn't check + // other vendors), so we can disable these flags for chips that don't also + // report support for AVX. + // + // It's possible this will pessimize future chips that do support BMI and + // not AVX, but this seems minor compared to a hard crash you get when + // executing an unsupported instruction (to put it another way, it's safe + // for us to under-report CPU features, but not to over-report them). Still, + // to limit any impact this may have in the future, we only do this for + // Intel chips, as it's a bug only present in their chips. + // + // This bug is documented as `SKL052` in the errata section of this document: + // http://www.intel.com/content/dam/www/public/us/en/documents/specification-updates/desktop-6th-gen-core-family-spec-update.pdf + if vendor_id == *b"GenuineIntel" && !value.test(Feature::avx as u32) { + value.unset(Feature::bmi1 as u32); + value.unset(Feature::bmi2 as u32); + } + + value +} diff --git a/crux-mir/lib/std_detect/src/detect/test_data/linux-artificial-aarch64.auxv b/crux-mir/lib/std_detect/src/detect/test_data/linux-artificial-aarch64.auxv new file mode 100644 index 0000000000000000000000000000000000000000..ec826afcf38179904c8b4312842dc474687d254c GIT binary patch literal 336 zcmY#nKn2Dyt`vkJ0Oh|wySg*oRhf|uCJ&Vmgz}-v8JJ=6P?`me&x*!pN8@v#@j21> r+)zHuJ{~kaFB+c@jV})6!zf7@15Mri+2O+Hy4;xL(d7Bj_~={!cy$wI|)_SNs gSJrR?cW@66@F@G<1kdmSC)u&PV9(#hMY+P{19AETwEzGB literal 0 HcmV?d00001 diff --git a/crux-mir/lib/std_detect/src/detect/test_data/linux-hwcap2-aarch64.auxv b/crux-mir/lib/std_detect/src/detect/test_data/linux-hwcap2-aarch64.auxv new file mode 100644 index 0000000000000000000000000000000000000000..1d87264b221901c140c379792e7e14c7e74cbf5c GIT binary patch literal 336 zcmZ9`OA3H6420nn-}vmR=tew)2lh(dNRdt!{@r9gCeR%Af2VWHVUeFJ-e0U`QxRpY kTj9NC30H6fw{Qpdvi~>0BRs)L_OW`vUI_WJ9sLZGFHf=regFUf literal 0 HcmV?d00001 diff --git a/crux-mir/lib/std_detect/src/detect/test_data/linux-no-hwcap2-aarch64.auxv b/crux-mir/lib/std_detect/src/detect/test_data/linux-no-hwcap2-aarch64.auxv new file mode 100644 index 0000000000000000000000000000000000000000..35f01cc767c507ed0d28a1d349fa84ba33dc7899 GIT binary patch literal 320 zcmZ9_OA3H66hqO1fBbe-?4-B`7wl3MG#Pk1Ne+3?BmCcK=C5q>eZ{x7-Zey7=b`eh ctlYO+n*aa+ literal 0 HcmV?d00001 diff --git a/crux-mir/lib/std_detect/src/detect/test_data/linux-rpi3.auxv b/crux-mir/lib/std_detect/src/detect/test_data/linux-rpi3.auxv new file mode 100644 index 0000000000000000000000000000000000000000..0538e661f63ad0676ced42f9ac4cd57787555c20 GIT binary patch literal 160 zcmY#n0D=qcbpk-@+6H?DHV_}k5d=~xK+FupCJc-WEI?WTh*^P{6NuS?m|?-&G7b<8 z1R$|WX+{QaApHWUi3dvaLTNrIEe@nXKoW>&`?b|c0cnuk@<9CAzpaiRNUsKp11T^7 E07AJE2LJ#7 literal 0 HcmV?d00001 diff --git a/crux-mir/lib/std_detect/src/detect/test_data/macos-virtualbox-linux-x86-4850HQ.auxv b/crux-mir/lib/std_detect/src/detect/test_data/macos-virtualbox-linux-x86-4850HQ.auxv new file mode 100644 index 0000000000000000000000000000000000000000..75abc02d17813074bf947fa0725e8509e21109ee GIT binary patch literal 160 zcmY#jU|{%DQ@&jhNHY|aZx;Yk|9^M0vw`?Pjv$aq0b*t#HfdntU;)w!K+FonoIuPD z#0(S4w{w7KAOMMF%wyr;2J;vgc%U>dl;(rd;!s)=NN0P!*e?&HKgYk=&kv-lfqbYm F0{~-J6Wag) literal 0 HcmV?d00001 diff --git a/crux-mir/lib/std_detect/src/lib.rs b/crux-mir/lib/std_detect/src/lib.rs new file mode 100644 index 000000000..c0e0de0dd --- /dev/null +++ b/crux-mir/lib/std_detect/src/lib.rs @@ -0,0 +1,34 @@ +//! Run-time feature detection for the Rust standard library. +//! +//! To detect whether a feature is enabled in the system running the binary +//! use one of the appropriate macro for the target: +//! +//! * `x86` and `x86_64`: [`is_x86_feature_detected`] +//! * `arm`: [`is_arm_feature_detected`] +//! * `aarch64`: [`is_aarch64_feature_detected`] +//! * `riscv`: [`is_riscv_feature_detected`] +//! * `mips`: [`is_mips_feature_detected`] +//! * `mips64`: [`is_mips64_feature_detected`] +//! * `powerpc`: [`is_powerpc_feature_detected`] +//! * `powerpc64`: [`is_powerpc64_feature_detected`] + +#![unstable(feature = "stdsimd", issue = "27731")] +#![feature(staged_api, stdsimd, doc_cfg, allow_internal_unstable)] +#![deny(rust_2018_idioms)] +#![allow(clippy::shadow_reuse)] +#![deny(clippy::missing_inline_in_public_items)] +#![cfg_attr(test, allow(unused_imports))] +#![no_std] + +#[cfg(test)] +#[macro_use] +extern crate std; + +// rust-lang/rust#83888: removing `extern crate` gives an error that `vec_spare> +#[cfg_attr(feature = "std_detect_file_io", allow(unused_extern_crates))] +#[cfg(feature = "std_detect_file_io")] +extern crate alloc; + +#[doc(hidden)] +#[unstable(feature = "stdsimd", issue = "27731")] +pub mod detect; diff --git a/crux-mir/lib/std_detect/tests/cpu-detection.rs b/crux-mir/lib/std_detect/tests/cpu-detection.rs new file mode 100644 index 000000000..ca8bf28f4 --- /dev/null +++ b/crux-mir/lib/std_detect/tests/cpu-detection.rs @@ -0,0 +1,164 @@ +#![feature(stdsimd)] +#![allow(clippy::unwrap_used, clippy::use_debug, clippy::print_stdout)] +#![cfg(any( + target_arch = "arm", + target_arch = "aarch64", + target_arch = "x86", + target_arch = "x86_64", + target_arch = "powerpc", + target_arch = "powerpc64" +))] + +#[macro_use] +extern crate std_detect; + +#[test] +fn all() { + for (f, e) in std_detect::detect::features() { + println!("{}: {}", f, e); + } +} + +#[test] +#[cfg(all(target_arch = "arm", any(target_os = "linux", target_os = "android")))] +fn arm_linux() { + println!("neon: {}", is_arm_feature_detected!("neon")); + println!("pmull: {}", is_arm_feature_detected!("pmull")); + println!("crc: {}", is_arm_feature_detected!("crc")); + println!("crypto: {}", is_arm_feature_detected!("crypto")); + println!("aes: {}", is_arm_feature_detected!("aes")); + println!("sha2: {}", is_arm_feature_detected!("sha2")); +} + +#[test] +#[cfg(all( + target_arch = "aarch64", + any(target_os = "linux", target_os = "android") +))] +fn aarch64_linux() { + println!("asimd: {}", is_aarch64_feature_detected!("asimd")); + println!("neon: {}", is_aarch64_feature_detected!("neon")); + println!("pmull: {}", is_aarch64_feature_detected!("pmull")); + println!("fp: {}", is_aarch64_feature_detected!("fp")); + println!("fp16: {}", is_aarch64_feature_detected!("fp16")); + println!("sve: {}", is_aarch64_feature_detected!("sve")); + println!("crc: {}", is_aarch64_feature_detected!("crc")); + println!("lse: {}", is_aarch64_feature_detected!("lse")); + println!("lse2: {}", is_aarch64_feature_detected!("lse2")); + println!("rdm: {}", is_aarch64_feature_detected!("rdm")); + println!("rcpc: {}", is_aarch64_feature_detected!("rcpc")); + println!("rcpc2: {}", is_aarch64_feature_detected!("rcpc2")); + println!("dotprod: {}", is_aarch64_feature_detected!("dotprod")); + println!("tme: {}", is_aarch64_feature_detected!("tme")); + println!("fhm: {}", is_aarch64_feature_detected!("fhm")); + println!("dit: {}", is_aarch64_feature_detected!("dit")); + println!("flagm: {}", is_aarch64_feature_detected!("flagm")); + println!("ssbs: {}", is_aarch64_feature_detected!("ssbs")); + println!("sb: {}", is_aarch64_feature_detected!("sb")); + println!("paca: {}", is_aarch64_feature_detected!("paca")); + println!("pacg: {}", is_aarch64_feature_detected!("pacg")); + println!("dpb: {}", is_aarch64_feature_detected!("dpb")); + println!("dpb2: {}", is_aarch64_feature_detected!("dpb2")); + println!("sve2: {}", is_aarch64_feature_detected!("sve2")); + println!("sve2-aes: {}", is_aarch64_feature_detected!("sve2-aes")); + println!("sve2-sm4: {}", is_aarch64_feature_detected!("sve2-sm4")); + println!("sve2-sha3: {}", is_aarch64_feature_detected!("sve2-sha3")); + println!( + "sve2-bitperm: {}", + is_aarch64_feature_detected!("sve2-bitperm") + ); + println!("frintts: {}", is_aarch64_feature_detected!("frintts")); + println!("i8mm: {}", is_aarch64_feature_detected!("i8mm")); + println!("f32mm: {}", is_aarch64_feature_detected!("f32mm")); + println!("f64mm: {}", is_aarch64_feature_detected!("f64mm")); + println!("bf16: {}", is_aarch64_feature_detected!("bf16")); + println!("rand: {}", is_aarch64_feature_detected!("rand")); + println!("bti: {}", is_aarch64_feature_detected!("bti")); + println!("mte: {}", is_aarch64_feature_detected!("mte")); + println!("jsconv: {}", is_aarch64_feature_detected!("jsconv")); + println!("fcma: {}", is_aarch64_feature_detected!("fcma")); + println!("aes: {}", is_aarch64_feature_detected!("aes")); + println!("sha2: {}", is_aarch64_feature_detected!("sha2")); + println!("sha3: {}", is_aarch64_feature_detected!("sha3")); + println!("sm4: {}", is_aarch64_feature_detected!("sm4")); +} + +#[test] +#[cfg(all(target_arch = "powerpc", target_os = "linux"))] +fn powerpc_linux() { + println!("altivec: {}", is_powerpc_feature_detected!("altivec")); + println!("vsx: {}", is_powerpc_feature_detected!("vsx")); + println!("power8: {}", is_powerpc_feature_detected!("power8")); +} + +#[test] +#[cfg(all(target_arch = "powerpc64", target_os = "linux"))] +fn powerpc64_linux() { + println!("altivec: {}", is_powerpc64_feature_detected!("altivec")); + println!("vsx: {}", is_powerpc64_feature_detected!("vsx")); + println!("power8: {}", is_powerpc64_feature_detected!("power8")); +} + +#[test] +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +fn x86_all() { + println!("aes: {:?}", is_x86_feature_detected!("aes")); + println!("pcmulqdq: {:?}", is_x86_feature_detected!("pclmulqdq")); + println!("rdrand: {:?}", is_x86_feature_detected!("rdrand")); + println!("rdseed: {:?}", is_x86_feature_detected!("rdseed")); + println!("tsc: {:?}", is_x86_feature_detected!("tsc")); + println!("mmx: {:?}", is_x86_feature_detected!("mmx")); + println!("sse: {:?}", is_x86_feature_detected!("sse")); + println!("sse2: {:?}", is_x86_feature_detected!("sse2")); + println!("sse3: {:?}", is_x86_feature_detected!("sse3")); + println!("ssse3: {:?}", is_x86_feature_detected!("ssse3")); + println!("sse4.1: {:?}", is_x86_feature_detected!("sse4.1")); + println!("sse4.2: {:?}", is_x86_feature_detected!("sse4.2")); + println!("sse4a: {:?}", is_x86_feature_detected!("sse4a")); + println!("sha: {:?}", is_x86_feature_detected!("sha")); + println!("avx: {:?}", is_x86_feature_detected!("avx")); + println!("avx2: {:?}", is_x86_feature_detected!("avx2")); + println!("avx512f: {:?}", is_x86_feature_detected!("avx512f")); + println!("avx512cd: {:?}", is_x86_feature_detected!("avx512cd")); + println!("avx512er: {:?}", is_x86_feature_detected!("avx512er")); + println!("avx512pf: {:?}", is_x86_feature_detected!("avx512pf")); + println!("avx512bw: {:?}", is_x86_feature_detected!("avx512bw")); + println!("avx512dq: {:?}", is_x86_feature_detected!("avx512dq")); + println!("avx512vl: {:?}", is_x86_feature_detected!("avx512vl")); + println!("avx512ifma: {:?}", is_x86_feature_detected!("avx512ifma")); + println!("avx512vbmi: {:?}", is_x86_feature_detected!("avx512vbmi")); + println!( + "avx512vpopcntdq: {:?}", + is_x86_feature_detected!("avx512vpopcntdq") + ); + println!("avx512vbmi2 {:?}", is_x86_feature_detected!("avx512vbmi2")); + println!("avx512gfni {:?}", is_x86_feature_detected!("avx512gfni")); + println!("avx512vaes {:?}", is_x86_feature_detected!("avx512vaes")); + println!( + "avx512vpclmulqdq {:?}", + is_x86_feature_detected!("avx512vpclmulqdq") + ); + println!("avx512vnni {:?}", is_x86_feature_detected!("avx512vnni")); + println!( + "avx512bitalg {:?}", + is_x86_feature_detected!("avx512bitalg") + ); + println!("avx512bf16 {:?}", is_x86_feature_detected!("avx512bf16")); + println!( + "avx512vp2intersect {:?}", + is_x86_feature_detected!("avx512vp2intersect") + ); + println!("f16c: {:?}", is_x86_feature_detected!("f16c")); + println!("fma: {:?}", is_x86_feature_detected!("fma")); + println!("bmi1: {:?}", is_x86_feature_detected!("bmi1")); + println!("bmi2: {:?}", is_x86_feature_detected!("bmi2")); + println!("abm: {:?}", is_x86_feature_detected!("abm")); + println!("lzcnt: {:?}", is_x86_feature_detected!("lzcnt")); + println!("tbm: {:?}", is_x86_feature_detected!("tbm")); + println!("popcnt: {:?}", is_x86_feature_detected!("popcnt")); + println!("fxsr: {:?}", is_x86_feature_detected!("fxsr")); + println!("xsave: {:?}", is_x86_feature_detected!("xsave")); + println!("xsaveopt: {:?}", is_x86_feature_detected!("xsaveopt")); + println!("xsaves: {:?}", is_x86_feature_detected!("xsaves")); + println!("xsavec: {:?}", is_x86_feature_detected!("xsavec")); +} diff --git a/crux-mir/lib/std_detect/tests/macro_trailing_commas.rs b/crux-mir/lib/std_detect/tests/macro_trailing_commas.rs new file mode 100644 index 000000000..cd597af73 --- /dev/null +++ b/crux-mir/lib/std_detect/tests/macro_trailing_commas.rs @@ -0,0 +1,51 @@ +#![feature(stdsimd)] +#![allow(clippy::unwrap_used, clippy::use_debug, clippy::print_stdout)] + +#[cfg(any( + target_arch = "arm", + target_arch = "aarch64", + target_arch = "x86", + target_arch = "x86_64", + target_arch = "powerpc", + target_arch = "powerpc64" +))] +#[macro_use] +extern crate std_detect; + +#[test] +#[cfg(all(target_arch = "arm", any(target_os = "linux", target_os = "android")))] +fn arm_linux() { + let _ = is_arm_feature_detected!("neon"); + let _ = is_arm_feature_detected!("neon",); +} + +#[test] +#[cfg(all( + target_arch = "aarch64", + any(target_os = "linux", target_os = "android") +))] +fn aarch64_linux() { + let _ = is_aarch64_feature_detected!("fp"); + let _ = is_aarch64_feature_detected!("fp",); +} + +#[test] +#[cfg(all(target_arch = "powerpc", target_os = "linux"))] +fn powerpc_linux() { + let _ = is_powerpc_feature_detected!("altivec"); + let _ = is_powerpc_feature_detected!("altivec",); +} + +#[test] +#[cfg(all(target_arch = "powerpc64", target_os = "linux"))] +fn powerpc64_linux() { + let _ = is_powerpc64_feature_detected!("altivec"); + let _ = is_powerpc64_feature_detected!("altivec",); +} + +#[test] +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +fn x86_all() { + let _ = is_x86_feature_detected!("sse"); + let _ = is_x86_feature_detected!("sse",); +} diff --git a/crux-mir/lib/std_detect/tests/x86-specific.rs b/crux-mir/lib/std_detect/tests/x86-specific.rs new file mode 100644 index 000000000..59e9a62fd --- /dev/null +++ b/crux-mir/lib/std_detect/tests/x86-specific.rs @@ -0,0 +1,158 @@ +#![feature(stdsimd)] +#![cfg(any(target_arch = "x86", target_arch = "x86_64"))] + +extern crate cupid; +#[macro_use] +extern crate std_detect; + +#[test] +fn dump() { + println!("aes: {:?}", is_x86_feature_detected!("aes")); + println!("pclmulqdq: {:?}", is_x86_feature_detected!("pclmulqdq")); + println!("rdrand: {:?}", is_x86_feature_detected!("rdrand")); + println!("rdseed: {:?}", is_x86_feature_detected!("rdseed")); + println!("tsc: {:?}", is_x86_feature_detected!("tsc")); + println!("sse: {:?}", is_x86_feature_detected!("sse")); + println!("sse2: {:?}", is_x86_feature_detected!("sse2")); + println!("sse3: {:?}", is_x86_feature_detected!("sse3")); + println!("ssse3: {:?}", is_x86_feature_detected!("ssse3")); + println!("sse4.1: {:?}", is_x86_feature_detected!("sse4.1")); + println!("sse4.2: {:?}", is_x86_feature_detected!("sse4.2")); + println!("sse4a: {:?}", is_x86_feature_detected!("sse4a")); + println!("sha: {:?}", is_x86_feature_detected!("sha")); + println!("avx: {:?}", is_x86_feature_detected!("avx")); + println!("avx2: {:?}", is_x86_feature_detected!("avx2")); + println!("avx512f {:?}", is_x86_feature_detected!("avx512f")); + println!("avx512cd {:?}", is_x86_feature_detected!("avx512cd")); + println!("avx512er {:?}", is_x86_feature_detected!("avx512er")); + println!("avx512pf {:?}", is_x86_feature_detected!("avx512pf")); + println!("avx512bw {:?}", is_x86_feature_detected!("avx512bw")); + println!("avx512dq {:?}", is_x86_feature_detected!("avx512dq")); + println!("avx512vl {:?}", is_x86_feature_detected!("avx512vl")); + println!("avx512_ifma {:?}", is_x86_feature_detected!("avx512ifma")); + println!("avx512vbmi {:?}", is_x86_feature_detected!("avx512vbmi")); + println!( + "avx512_vpopcntdq {:?}", + is_x86_feature_detected!("avx512vpopcntdq") + ); + println!("avx512vbmi2 {:?}", is_x86_feature_detected!("avx512vbmi2")); + println!("avx512gfni {:?}", is_x86_feature_detected!("avx512gfni")); + println!("avx512vaes {:?}", is_x86_feature_detected!("avx512vaes")); + println!( + "avx512vpclmulqdq {:?}", + is_x86_feature_detected!("avx512vpclmulqdq") + ); + println!("avx512vnni {:?}", is_x86_feature_detected!("avx512vnni")); + println!( + "avx512bitalg {:?}", + is_x86_feature_detected!("avx512bitalg") + ); + println!("avx512bf16 {:?}", is_x86_feature_detected!("avx512bf16")); + println!( + "avx512vp2intersect {:?}", + is_x86_feature_detected!("avx512vp2intersect") + ); + println!("fma: {:?}", is_x86_feature_detected!("fma")); + println!("abm: {:?}", is_x86_feature_detected!("abm")); + println!("bmi: {:?}", is_x86_feature_detected!("bmi1")); + println!("bmi2: {:?}", is_x86_feature_detected!("bmi2")); + println!("tbm: {:?}", is_x86_feature_detected!("tbm")); + println!("popcnt: {:?}", is_x86_feature_detected!("popcnt")); + println!("lzcnt: {:?}", is_x86_feature_detected!("lzcnt")); + println!("fxsr: {:?}", is_x86_feature_detected!("fxsr")); + println!("xsave: {:?}", is_x86_feature_detected!("xsave")); + println!("xsaveopt: {:?}", is_x86_feature_detected!("xsaveopt")); + println!("xsaves: {:?}", is_x86_feature_detected!("xsaves")); + println!("xsavec: {:?}", is_x86_feature_detected!("xsavec")); + println!("cmpxchg16b: {:?}", is_x86_feature_detected!("cmpxchg16b")); + println!("adx: {:?}", is_x86_feature_detected!("adx")); + println!("rtm: {:?}", is_x86_feature_detected!("rtm")); +} + +#[cfg(feature = "std_detect_env_override")] +#[test] +fn env_override_no_avx() { + if let Ok(disable) = std::env::var("RUST_STD_DETECT_UNSTABLE") { + let information = cupid::master().unwrap(); + for d in disable.split(" ") { + match d { + "avx" => { + if information.avx() { + assert_ne!(is_x86_feature_detected!("avx"), information.avx()) + } + } + "avx2" => { + if information.avx2() { + assert_ne!(is_x86_feature_detected!("avx2"), information.avx2()) + } + } + _ => {} + } + } + } +} + +#[test] +fn compare_with_cupid() { + let information = cupid::master().unwrap(); + assert_eq!(is_x86_feature_detected!("aes"), information.aesni()); + assert_eq!( + is_x86_feature_detected!("pclmulqdq"), + information.pclmulqdq() + ); + assert_eq!(is_x86_feature_detected!("rdrand"), information.rdrand()); + assert_eq!(is_x86_feature_detected!("rdseed"), information.rdseed()); + assert_eq!(is_x86_feature_detected!("tsc"), information.tsc()); + assert_eq!(is_x86_feature_detected!("sse"), information.sse()); + assert_eq!(is_x86_feature_detected!("sse2"), information.sse2()); + assert_eq!(is_x86_feature_detected!("sse3"), information.sse3()); + assert_eq!(is_x86_feature_detected!("ssse3"), information.ssse3()); + assert_eq!(is_x86_feature_detected!("sse4.1"), information.sse4_1()); + assert_eq!(is_x86_feature_detected!("sse4.2"), information.sse4_2()); + assert_eq!(is_x86_feature_detected!("sse4a"), information.sse4a()); + assert_eq!(is_x86_feature_detected!("sha"), information.sha()); + assert_eq!(is_x86_feature_detected!("avx"), information.avx()); + assert_eq!(is_x86_feature_detected!("avx2"), information.avx2()); + assert_eq!(is_x86_feature_detected!("avx512f"), information.avx512f()); + assert_eq!(is_x86_feature_detected!("avx512cd"), information.avx512cd()); + assert_eq!(is_x86_feature_detected!("avx512er"), information.avx512er()); + assert_eq!(is_x86_feature_detected!("avx512pf"), information.avx512pf()); + assert_eq!(is_x86_feature_detected!("avx512bw"), information.avx512bw()); + assert_eq!(is_x86_feature_detected!("avx512dq"), information.avx512dq()); + assert_eq!(is_x86_feature_detected!("avx512vl"), information.avx512vl()); + assert_eq!( + is_x86_feature_detected!("avx512ifma"), + information.avx512_ifma() + ); + assert_eq!( + is_x86_feature_detected!("avx512vbmi"), + information.avx512_vbmi() + ); + assert_eq!( + is_x86_feature_detected!("avx512vpopcntdq"), + information.avx512_vpopcntdq() + ); + assert_eq!(is_x86_feature_detected!("fma"), information.fma()); + assert_eq!(is_x86_feature_detected!("bmi1"), information.bmi1()); + assert_eq!(is_x86_feature_detected!("bmi2"), information.bmi2()); + assert_eq!(is_x86_feature_detected!("popcnt"), information.popcnt()); + assert_eq!(is_x86_feature_detected!("abm"), information.lzcnt()); + assert_eq!(is_x86_feature_detected!("tbm"), information.tbm()); + assert_eq!(is_x86_feature_detected!("lzcnt"), information.lzcnt()); + assert_eq!(is_x86_feature_detected!("xsave"), information.xsave()); + assert_eq!(is_x86_feature_detected!("xsaveopt"), information.xsaveopt()); + assert_eq!( + is_x86_feature_detected!("xsavec"), + information.xsavec_and_xrstor() + ); + assert_eq!( + is_x86_feature_detected!("xsaves"), + information.xsaves_xrstors_and_ia32_xss() + ); + assert_eq!( + is_x86_feature_detected!("cmpxchg16b"), + information.cmpxchg16b(), + ); + assert_eq!(is_x86_feature_detected!("adx"), information.adx(),); + assert_eq!(is_x86_feature_detected!("rtm"), information.rtm(),); +} diff --git a/crux-mir/lib/stdarch/.cirrus.yml b/crux-mir/lib/stdarch/.cirrus.yml index b5c1196c2..7b75ba39d 100644 --- a/crux-mir/lib/stdarch/.cirrus.yml +++ b/crux-mir/lib/stdarch/.cirrus.yml @@ -1,7 +1,11 @@ +env: + # Temporary fix for https://github.com/rust-lang/rustup/issues/2774. + RUSTUP_IO_THREADS: "1" + task: name: x86_64-unknown-freebsd freebsd_instance: - image: freebsd-12-0-release-amd64 + image: freebsd-12-2-release-amd64 setup_script: - pkg install -y curl - curl https://sh.rustup.rs -sSf --output rustup.sh diff --git a/crux-mir/lib/stdarch/.github/workflows/main.yml b/crux-mir/lib/stdarch/.github/workflows/main.yml index c5ef15004..fd8713ff8 100644 --- a/crux-mir/lib/stdarch/.github/workflows/main.yml +++ b/crux-mir/lib/stdarch/.github/workflows/main.yml @@ -72,12 +72,13 @@ jobs: - arm-unknown-linux-gnueabihf - armv7-unknown-linux-gnueabihf - aarch64-unknown-linux-gnu + - riscv64gc-unknown-linux-gnu - powerpc64le-unknown-linux-gnu - mips-unknown-linux-gnu - mips64-unknown-linux-gnuabi64 - mips64el-unknown-linux-gnuabi64 - s390x-unknown-linux-gnu - - wasm32-unknown-unknown + - wasm32-wasi - i586-unknown-linux-gnu - x86_64-linux-android - arm-linux-androideabi @@ -90,16 +91,18 @@ jobs: - thumbv7em-none-eabihf # macOS targets - - x86_64-apple-darwin + #- x86_64-apple-darwin + - aarch64-apple-darwin # FIXME: gh-actions build environment doesn't have linker support # - i686-apple-darwin # Windows targets - x86_64-pc-windows-msvc + - i686-pc-windows-msvc # FIXME: Disassembly not implemented for the # following targets: # - x86_64-pc-windows-gnu: # - i686-pc-windows-gnu: - # - i686-pc-windows-msvc: + # - aarch64-pc-windows-msvc: include: - target: i686-unknown-linux-gnu @@ -114,7 +117,6 @@ jobs: os: ubuntu-latest - target: armv7-unknown-linux-gnueabihf os: ubuntu-latest - rustflags: -C target-feature=+neon - target: mips-unknown-linux-gnu os: ubuntu-latest norun: true @@ -129,14 +131,19 @@ jobs: disable_assert_instr: true - target: s390x-unknown-linux-gnu os: ubuntu-latest - - target: wasm32-unknown-unknown + - target: wasm32-wasi os: ubuntu-latest + - target: aarch64-apple-darwin + os: macos-latest + norun: true - target: aarch64-unknown-linux-gnu os: ubuntu-latest - target: x86_64-apple-darwin - os: macos-latest + os: macos-11 - target: x86_64-pc-windows-msvc os: windows-latest + - target: i686-pc-windows-msvc + os: windows-latest - target: i586-unknown-linux-gnu os: ubuntu-latest - target: x86_64-linux-android @@ -161,9 +168,13 @@ jobs: os: ubuntu-latest - target: thumbv7em-none-eabihf os: ubuntu-latest + - target: riscv64gc-unknown-linux-gnu + os: ubuntu-latest steps: - uses: actions/checkout@master + with: + submodules: recursive - name: Install Rust (rustup) run: | rustup update nightly --no-self-update @@ -172,7 +183,7 @@ jobs: - name: Install Rust (macos) run: | curl https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly - echo "##[add-path]$HOME/.cargo/bin" + echo "$HOME/.cargo/bin" >> $GITHUB_PATH rustup update nightly --no-self-update rustup default nightly if: matrix.os == 'macos-latest' @@ -180,18 +191,23 @@ jobs: rustup default nightly rustup target add ${{ matrix.target }} if: "!endsWith(matrix.target, 'emulated')" + - name: Setup (aarch64-apple-darwin) + run: | + echo "SDKROOT=$(xcrun -sdk macosx11.0 --show-sdk-path)" >> $GITHUB_ENV + echo "MACOS_DEPLOYMENT_TARGET=$(xcrun -sdk macosx11.0 --show-sdk-platform-version)" >> $GITHUB_ENV + if: matrix.target == 'aarch64-apple-darwin' - run: cargo generate-lockfile # Configure some env vars based on matrix configuration - - run: echo "##[set-env name=NORUN]1" + - run: echo "NORUN=1" >> $GITHUB_ENV if: matrix.norun != '' || startsWith(matrix.target, 'thumb') || matrix.target == 'nvptx64-nvidia-cuda' - - run: echo "##[set-env name=STDARCH_TEST_EVERYTHING]1" + - run: echo "STDARCH_TEST_EVERYTHING=1" >> $GITHUB_ENV if: matrix.test_everything != '' - - run: echo "##[set-env name=RUSTFLAGS]${{ matrix.rustflags }}" + - run: echo "RUSTFLAGS=${{ matrix.rustflags }}" >> $GITHUB_ENV if: matrix.rustflags != '' - - run: echo "##[set-env name=STDARCH_DISABLE_ASSERT_INSTR]1" + - run: echo "STDARCH_DISABLE_ASSERT_INSTR=1" >> $GITHUB_ENV if: matrix.disable_assert_instr != '' - - run: echo "##[set-env name=NOSTD]1" + - run: echo "NOSTD=1" >> $GITHUB_ENV if: startsWith(matrix.target, 'thumb') || matrix.target == 'nvptx64-nvidia-cuda' # Windows & OSX go straight to `run.sh` ... diff --git a/crux-mir/lib/stdarch/CONTRIBUTING.md b/crux-mir/lib/stdarch/CONTRIBUTING.md index 85b7bb474..4212abcd7 100644 --- a/crux-mir/lib/stdarch/CONTRIBUTING.md +++ b/crux-mir/lib/stdarch/CONTRIBUTING.md @@ -6,17 +6,14 @@ probably want to check out the repository and make sure that tests pass for you: ``` $ git clone https://github.com/rust-lang/stdarch $ cd stdarch -$ cargo +nightly test +$ TARGET="" ci/run.sh ``` -To run codegen tests, run in release mode: +Where `` is the target triple as used by `rustup`, e.g. `x86_64-unknown-linux-gnu` (without any preceding `nightly-` or similar). +Also remember that this repository requires the nightly channel of Rust! +The above tests do in fact require nightly rust to be the default on your system, to set that use `rustup default nightly` (and `rustup default stable` to revert). -``` -$ cargo +nightly test --release -p coresimd -``` - -Remember that this repository requires the nightly channel of Rust! If any of -the above steps don't work, [please let us know][new]! +If any of the above steps don't work, [please let us know][new]! Next up you can [find an issue][issues] to help out on, we've selected a few with the [`help wanted`][help] and [`impl-period`][impl] tags which could @@ -71,6 +68,21 @@ of the [Rust Book] describes the `rustdoc` syntax quite well. As always, feel fr to [join us on gitter][gitter] and ask us if you hit any snags, and thank you for helping to improve the documentation of `stdarch`! +# Alternative Testing Instructions + +It is generally recommended that you use `ci/run.sh` to run the tests. +However this might not work for you, e.g. if you are on Windows. + +In that case you can fall back to running `cargo +nightly test` and `cargo +nightly test --release -p core_arch` for testing the code generation. +Note that these require the nightly toolchain to be installed and for `rustc` to know about your target triple and its CPU. +In particular you need to set the `TARGET` environment variable as you would for `ci/run.sh`. +In addition you need to set `RUSTCFLAGS` (need the `C`) to indicate target features, e.g. `RUSTCFLAGS="-C -target-features=+avx2"`. +You can also set `-C -target-cpu=native` if you're "just" developing against your current CPU. + +Be warned that when you use these alternative instructions, [things may go less smoothly than they would with `ci/run.sh`][ci-run-good], e.g. instruction generation tests may fail because the disassembler named them differently, e.g. it may generate `vaesenc` instead of `aesenc` instructions despite them behaving the same. +Also these instructions execute less tests than would normally be done, so don't be surprised that when you eventually pull-request some errors may show up for tests not covered here. + + [new]: https://github.com/rust-lang/stdarch/issues/new [issues]: https://github.com/rust-lang/stdarch/issues [help]: https://github.com/rust-lang/stdarch/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 @@ -78,3 +90,4 @@ to improve the documentation of `stdarch`! [vendor]: https://github.com/rust-lang/stdarch/issues/40 [Documentation as tests]: https://doc.rust-lang.org/book/first-edition/documentation.html#documentation-as-tests [Rust Book]: https://doc.rust-lang.org/book/first-edition +[ci-run-good]: https://github.com/rust-lang/stdarch/issues/931#issuecomment-711412126 diff --git a/crux-mir/lib/stdarch/Cargo.toml b/crux-mir/lib/stdarch/Cargo.toml index 7b4c5ead8..6efd6b189 100644 --- a/crux-mir/lib/stdarch/Cargo.toml +++ b/crux-mir/lib/stdarch/Cargo.toml @@ -3,6 +3,8 @@ members = [ "crates/stdarch-verify", "crates/core_arch", "crates/std_detect", + "crates/stdarch-gen", + "crates/intrinsic-test", "examples/" ] exclude = [ diff --git a/crux-mir/lib/stdarch/README.md b/crux-mir/lib/stdarch/README.md index 490ca6fe7..70ec256e6 100644 --- a/crux-mir/lib/stdarch/README.md +++ b/crux-mir/lib/stdarch/README.md @@ -8,33 +8,11 @@ stdarch - Rust's standard library SIMD components This repository contains two main crates: -* [![core_arch_crate_badge]][core_arch_crate_link] - [![core_arch_docs_badge]][core_arch_docs_link] - [`core_arch`](crates/core_arch/README.md) implements `core::arch` - Rust's +* [`core_arch`](crates/core_arch/README.md) implements `core::arch` - Rust's core library architecture-specific intrinsics, and -* [![std_detect_crate_badge]][std_detect_crate_link] - [![std_detect_docs_badge]][std_detect_docs_link] - [`std_detect`](crates/std_detect/README.md) implements `std::detect` - Rust's +* [`std_detect`](crates/std_detect/README.md) implements `std::detect` - Rust's standard library run-time CPU feature detection. The `std::simd` component now lives in the -[`packed_simd`](https://github.com/rust-lang-nursery/packed_simd) crate. - -# How to do a release - -To do a release of the `core_arch` and `std_detect` crates, - -* bump up the version appropriately, -* comment out the `dev-dependencies` in their `Cargo.toml` files (due to - https://github.com/rust-lang/cargo/issues/4242), -* publish the crates. - -[core_arch_crate_badge]: https://img.shields.io/crates/v/core_arch.svg -[core_arch_crate_link]: https://crates.io/crates/core_arch -[core_arch_docs_badge]: https://docs.rs/core_arch/badge.svg -[core_arch_docs_link]: https://docs.rs/core_arch/ -[std_detect_crate_badge]: https://img.shields.io/crates/v/std_detect.svg -[std_detect_crate_link]: https://crates.io/crates/std_detect -[std_detect_docs_badge]: https://docs.rs/std_detect/badge.svg -[std_detect_docs_link]: https://docs.rs/std_detect/ +[`packed_simd_2`](https://github.com/rust-lang/packed_simd) crate. diff --git a/crux-mir/lib/stdarch/ci/android-install-sdk.sh b/crux-mir/lib/stdarch/ci/android-install-sdk.sh index 1beeb312a..3383dcb7f 100644 --- a/crux-mir/lib/stdarch/ci/android-install-sdk.sh +++ b/crux-mir/lib/stdarch/ci/android-install-sdk.sh @@ -19,8 +19,8 @@ set -ex # which apparently magically accepts the licenses. mkdir sdk -curl --retry 5 https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip -O -unzip -d sdk sdk-tools-linux-3859397.zip +curl --retry 5 https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip -O +unzip -d sdk sdk-tools-linux-4333796.zip case "$1" in arm | armv7) diff --git a/crux-mir/lib/stdarch/ci/docker/aarch64-linux-android/Dockerfile b/crux-mir/lib/stdarch/ci/docker/aarch64-linux-android/Dockerfile index 27bde89c5..6cf9b5061 100644 --- a/crux-mir/lib/stdarch/ci/docker/aarch64-linux-android/Dockerfile +++ b/crux-mir/lib/stdarch/ci/docker/aarch64-linux-android/Dockerfile @@ -1,17 +1,16 @@ -FROM ubuntu:16.04 +FROM ubuntu:22.04 -RUN dpkg --add-architecture i386 && \ - apt-get update && \ +RUN apt-get update && \ apt-get install -y --no-install-recommends \ file \ make \ curl \ ca-certificates \ - python \ + python-is-python3 \ unzip \ expect \ - openjdk-9-jre \ - libstdc++6:i386 \ + openjdk-8-jre \ + libstdc++6-i386-cross \ libpulse0 \ gcc \ libc6-dev diff --git a/crux-mir/lib/stdarch/ci/docker/aarch64-unknown-linux-gnu/Dockerfile b/crux-mir/lib/stdarch/ci/docker/aarch64-unknown-linux-gnu/Dockerfile index 41ff4729a..2f99999da 100644 --- a/crux-mir/lib/stdarch/ci/docker/aarch64-unknown-linux-gnu/Dockerfile +++ b/crux-mir/lib/stdarch/ci/docker/aarch64-unknown-linux-gnu/Dockerfile @@ -1,13 +1,17 @@ -FROM ubuntu:18.04 +FROM ubuntu:22.04 RUN apt-get update && apt-get install -y --no-install-recommends \ gcc \ + g++ \ ca-certificates \ libc6-dev \ gcc-aarch64-linux-gnu \ + g++-aarch64-linux-gnu \ libc6-dev-arm64-cross \ qemu-user \ make \ - file + file \ + clang-13 \ + lld ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \ CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER="qemu-aarch64 -L /usr/aarch64-linux-gnu" \ diff --git a/crux-mir/lib/stdarch/ci/docker/arm-linux-androideabi/Dockerfile b/crux-mir/lib/stdarch/ci/docker/arm-linux-androideabi/Dockerfile index 995a9e30e..fb1a0cecf 100644 --- a/crux-mir/lib/stdarch/ci/docker/arm-linux-androideabi/Dockerfile +++ b/crux-mir/lib/stdarch/ci/docker/arm-linux-androideabi/Dockerfile @@ -1,17 +1,16 @@ -FROM ubuntu:16.04 +FROM ubuntu:22.04 -RUN dpkg --add-architecture i386 && \ - apt-get update && \ +RUN apt-get update && \ apt-get install -y --no-install-recommends \ file \ make \ curl \ ca-certificates \ - python \ + python-is-python3 \ unzip \ expect \ - openjdk-9-jre \ - libstdc++6:i386 \ + openjdk-8-jre \ + libstdc++6-i386-cross \ libpulse0 \ gcc \ libc6-dev diff --git a/crux-mir/lib/stdarch/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile b/crux-mir/lib/stdarch/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile index 253906293..b4cd0a68a 100644 --- a/crux-mir/lib/stdarch/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile +++ b/crux-mir/lib/stdarch/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile @@ -1,13 +1,17 @@ -FROM ubuntu:18.04 +FROM ubuntu:22.04 RUN apt-get update && apt-get install -y --no-install-recommends \ gcc \ + g++ \ ca-certificates \ libc6-dev \ gcc-arm-linux-gnueabihf \ + g++-arm-linux-gnueabihf \ libc6-dev-armhf-cross \ qemu-user \ make \ - file + file \ + clang-13 \ + lld ENV CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER=arm-linux-gnueabihf-gcc \ CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_RUNNER="qemu-arm -L /usr/arm-linux-gnueabihf" \ OBJDUMP=arm-linux-gnueabihf-objdump diff --git a/crux-mir/lib/stdarch/ci/docker/i586-unknown-linux-gnu/Dockerfile b/crux-mir/lib/stdarch/ci/docker/i586-unknown-linux-gnu/Dockerfile index 01093698f..0e4d1c6eb 100644 --- a/crux-mir/lib/stdarch/ci/docker/i586-unknown-linux-gnu/Dockerfile +++ b/crux-mir/lib/stdarch/ci/docker/i586-unknown-linux-gnu/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 RUN apt-get update && apt-get install -y --no-install-recommends \ gcc-multilib \ libc6-dev \ diff --git a/crux-mir/lib/stdarch/ci/docker/i686-unknown-linux-gnu/Dockerfile b/crux-mir/lib/stdarch/ci/docker/i686-unknown-linux-gnu/Dockerfile index 01093698f..0e4d1c6eb 100644 --- a/crux-mir/lib/stdarch/ci/docker/i686-unknown-linux-gnu/Dockerfile +++ b/crux-mir/lib/stdarch/ci/docker/i686-unknown-linux-gnu/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 RUN apt-get update && apt-get install -y --no-install-recommends \ gcc-multilib \ libc6-dev \ diff --git a/crux-mir/lib/stdarch/ci/docker/mipsel-unknown-linux-musl/Dockerfile b/crux-mir/lib/stdarch/ci/docker/mipsel-unknown-linux-musl/Dockerfile index 7488662ef..5d19d7c93 100644 --- a/crux-mir/lib/stdarch/ci/docker/mipsel-unknown-linux-musl/Dockerfile +++ b/crux-mir/lib/stdarch/ci/docker/mipsel-unknown-linux-musl/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.10 +FROM ubuntu:18.04 RUN apt-get update && \ apt-get install -y --no-install-recommends \ diff --git a/crux-mir/lib/stdarch/ci/docker/riscv64gc-unknown-linux-gnu/Dockerfile b/crux-mir/lib/stdarch/ci/docker/riscv64gc-unknown-linux-gnu/Dockerfile new file mode 100644 index 000000000..b9b3c682e --- /dev/null +++ b/crux-mir/lib/stdarch/ci/docker/riscv64gc-unknown-linux-gnu/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:22.04 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc libc6-dev qemu-user ca-certificates \ + gcc-riscv64-linux-gnu libc6-dev-riscv64-cross \ + qemu-user + +ENV CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER=riscv64-linux-gnu-gcc \ + CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_RUNNER="qemu-riscv64 -L /usr/riscv64-linux-gnu" \ + OBJDUMP=riscv64-linux-gnu-objdump diff --git a/crux-mir/lib/stdarch/ci/docker/wasm32-unknown-unknown/Dockerfile b/crux-mir/lib/stdarch/ci/docker/wasm32-unknown-unknown/Dockerfile deleted file mode 100644 index ff8eb2844..000000000 --- a/crux-mir/lib/stdarch/ci/docker/wasm32-unknown-unknown/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM ubuntu:18.04 - -RUN apt-get update -y && apt-get install -y --no-install-recommends \ - ca-certificates \ - clang \ - cmake \ - curl \ - git \ - libc6-dev \ - make \ - python \ - xz-utils - -# Install `wasm2wat` -RUN git clone --recursive https://github.com/WebAssembly/wabt -RUN make -C wabt -j$(nproc) -ENV PATH=$PATH:/wabt/bin - -# Install `node` -RUN curl https://nodejs.org/dist/v12.0.0/node-v12.0.0-linux-x64.tar.xz | tar xJf - -ENV PATH=$PATH:/node-v12.0.0-linux-x64/bin - -COPY docker/wasm32-unknown-unknown/wasm-entrypoint.sh /wasm-entrypoint.sh -ENTRYPOINT ["/wasm-entrypoint.sh"] diff --git a/crux-mir/lib/stdarch/ci/docker/wasm32-unknown-unknown/wasm-entrypoint.sh b/crux-mir/lib/stdarch/ci/docker/wasm32-unknown-unknown/wasm-entrypoint.sh deleted file mode 100755 index d2e4d42bc..000000000 --- a/crux-mir/lib/stdarch/ci/docker/wasm32-unknown-unknown/wasm-entrypoint.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -set -e - -# Download an appropriate version of wasm-bindgen based off of what's being used -# in the lock file. Ideally we'd use `wasm-pack` at some point for this! -version=$(grep -A 1 'name = "wasm-bindgen"' Cargo.lock | grep version) -version=$(echo $version | awk '{print $3}' | sed 's/"//g') -curl -L https://github.com/rustwasm/wasm-bindgen/releases/download/$version/wasm-bindgen-$version-x86_64-unknown-linux-musl.tar.gz \ - | tar xzf - -C target -export PATH=$PATH:`pwd`/target/wasm-bindgen-$version-x86_64-unknown-linux-musl -export CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER=wasm-bindgen-test-runner -export NODE_ARGS=--experimental-wasm-simd - -exec "$@" diff --git a/crux-mir/lib/stdarch/ci/docker/wasm32-wasi/Dockerfile b/crux-mir/lib/stdarch/ci/docker/wasm32-wasi/Dockerfile new file mode 100644 index 000000000..3e250f8b5 --- /dev/null +++ b/crux-mir/lib/stdarch/ci/docker/wasm32-wasi/Dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:20.04 + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update -y && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + xz-utils \ + clang + +RUN curl -L https://github.com/bytecodealliance/wasmtime/releases/download/v0.29.0/wasmtime-v0.29.0-x86_64-linux.tar.xz | tar xJf - +ENV PATH=$PATH:/wasmtime-v0.29.0-x86_64-linux + +ENV CARGO_TARGET_WASM32_WASI_RUNNER="wasmtime \ + --enable-simd \ + --enable-threads \ + --mapdir .::/checkout/target/wasm32-wasi/release/deps \ + --" diff --git a/crux-mir/lib/stdarch/ci/docker/x86_64-linux-android/Dockerfile b/crux-mir/lib/stdarch/ci/docker/x86_64-linux-android/Dockerfile index d52dd45b1..82119be74 100644 --- a/crux-mir/lib/stdarch/ci/docker/x86_64-linux-android/Dockerfile +++ b/crux-mir/lib/stdarch/ci/docker/x86_64-linux-android/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 +FROM ubuntu:22.04 RUN apt-get update && \ apt-get install -y --no-install-recommends \ @@ -6,7 +6,7 @@ RUN apt-get update && \ curl \ gcc \ libc-dev \ - python \ + python-is-python3 \ unzip \ file \ make @@ -17,7 +17,7 @@ COPY android-install-ndk.sh /android/ RUN sh /android/android-install-ndk.sh $ANDROID_ARCH # We do not run x86_64-linux-android tests on an android emulator. -# See ci/android-sysimage.sh for informations about how tests are run. +# See ci/android-sysimage.sh for information about how tests are run. COPY android-sysimage.sh /android/ RUN bash /android/android-sysimage.sh x86_64 x86_64-24_r07.zip diff --git a/crux-mir/lib/stdarch/ci/docker/x86_64-unknown-linux-gnu-emulated/Dockerfile b/crux-mir/lib/stdarch/ci/docker/x86_64-unknown-linux-gnu-emulated/Dockerfile index 40dbebdcc..b7fc93052 100644 --- a/crux-mir/lib/stdarch/ci/docker/x86_64-unknown-linux-gnu-emulated/Dockerfile +++ b/crux-mir/lib/stdarch/ci/docker/x86_64-unknown-linux-gnu-emulated/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 RUN apt-get update && apt-get install -y --no-install-recommends \ gcc \ libc6-dev \ @@ -10,4 +10,5 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN wget https://github.com/gnzlbg/intel_sde/raw/master/sde-external-8.35.0-2019-03-11-lin.tar.bz2 RUN tar -xjf sde-external-8.35.0-2019-03-11-lin.tar.bz2 +ENV SKIP_TESTS="avx512bf16" ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="/sde-external-8.35.0-2019-03-11-lin/sde64 -rtm_mode full --" diff --git a/crux-mir/lib/stdarch/ci/docker/x86_64-unknown-linux-gnu/Dockerfile b/crux-mir/lib/stdarch/ci/docker/x86_64-unknown-linux-gnu/Dockerfile index db64f7f91..dc4c4e598 100644 --- a/crux-mir/lib/stdarch/ci/docker/x86_64-unknown-linux-gnu/Dockerfile +++ b/crux-mir/lib/stdarch/ci/docker/x86_64-unknown-linux-gnu/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 RUN apt-get update && apt-get install -y --no-install-recommends \ gcc \ libc6-dev \ diff --git a/crux-mir/lib/stdarch/ci/dox.sh b/crux-mir/lib/stdarch/ci/dox.sh index e70a32b2d..3e507b456 100755 --- a/crux-mir/lib/stdarch/ci/dox.sh +++ b/crux-mir/lib/stdarch/ci/dox.sh @@ -22,9 +22,6 @@ dox() { rm -rf "target/doc/${arch}" mkdir "target/doc/${arch}" - export RUSTFLAGS="--cfg core_arch_docs" - export RUSTDOCFLAGS="--cfg core_arch_docs" - cargo build --verbose --target "${target}" --manifest-path crates/core_arch/Cargo.toml cargo build --verbose --target "${target}" --manifest-path crates/std_detect/Cargo.toml @@ -32,16 +29,14 @@ dox() { -o "target/doc/${arch}" crates/core_arch/src/lib.rs \ --edition=2018 \ --crate-name core_arch \ - --library-path "target/${target}/debug/deps" \ - --cfg core_arch_docs + --library-path "target/${target}/debug/deps" rustdoc --verbose --target "${target}" \ -o "target/doc/${arch}" crates/std_detect/src/lib.rs \ --edition=2018 \ --crate-name std_detect \ --library-path "target/${target}/debug/deps" \ --extern cfg_if="$(ls target/"${target}"/debug/deps/libcfg_if-*.rlib)" \ - --extern libc="$(ls target/"${target}"/debug/deps/liblibc-*.rlib)" \ - --cfg core_arch_docs + --extern libc="$(ls target/"${target}"/debug/deps/liblibc-*.rlib)" } dox i686 i686-unknown-linux-gnu diff --git a/crux-mir/lib/stdarch/ci/run-docker.sh b/crux-mir/lib/stdarch/ci/run-docker.sh index ec9393adf..32209d96c 100755 --- a/crux-mir/lib/stdarch/ci/run-docker.sh +++ b/crux-mir/lib/stdarch/ci/run-docker.sh @@ -9,7 +9,7 @@ run() { target=$(echo "${1}" | sed 's/-emulated//') echo "Building docker container for TARGET=${1}" docker build -t stdarch -f "ci/docker/${1}/Dockerfile" ci/ - mkdir -p target + mkdir -p target c_programs rust_programs echo "Running docker" # shellcheck disable=SC2016 docker run \ @@ -25,10 +25,12 @@ run() { --env NORUN \ --env RUSTFLAGS \ --env STDARCH_TEST_NORUN \ - --volume "$(dirname "$(dirname "$(command -v cargo)")")":/cargo \ + --volume "${HOME}/.cargo":/cargo \ --volume "$(rustc --print sysroot)":/rust:ro \ --volume "$(pwd)":/checkout:ro \ --volume "$(pwd)"/target:/checkout/target \ + --volume "$(pwd)"/c_programs:/checkout/c_programs \ + --volume "$(pwd)"/rust_programs:/checkout/rust_programs \ --init \ --workdir /checkout \ --privileged \ diff --git a/crux-mir/lib/stdarch/ci/run.sh b/crux-mir/lib/stdarch/ci/run.sh index 682a38636..54145a0e7 100755 --- a/crux-mir/lib/stdarch/ci/run.sh +++ b/crux-mir/lib/stdarch/ci/run.sh @@ -10,9 +10,18 @@ set -ex #export RUST_TEST_NOCAPTURE=1 #export RUST_TEST_THREADS=1 -RUSTFLAGS="$RUSTFLAGS -D warnings " +export RUSTFLAGS="${RUSTFLAGS} -D warnings -Z merge-functions=disabled " + +export STDARCH_DISABLE_DEDUP_GUARD=1 case ${TARGET} in + # On Windows the linker performs identical COMDAT folding (ICF) by default + # in release mode which removes identical COMDAT sections. This interferes + # with our instruction assertions just like LLVM's MergeFunctions pass so + # we disable it. + *-pc-windows-msvc) + export RUSTFLAGS="${RUSTFLAGS} -Clink-args=/OPT:NOICF" + ;; # On 32-bit use a static relocation model which avoids some extra # instructions when dealing with static data, notably allowing some # instruction assertion checks to pass below the 20 instruction limit. If @@ -28,6 +37,18 @@ case ${TARGET} in mips-* | mipsel-*) export RUSTFLAGS="${RUSTFLAGS} -C llvm-args=-fast-isel=false" ;; + # Some of our test dependencies use the deprecated `gcc` crates which is + # missing a fix from https://github.com/alexcrichton/cc-rs/pull/627. Apply + # the workaround manually here. + armv7-*eabihf | thumbv7-*eabihf) + export RUSTFLAGS="${RUSTFLAGS} -Ctarget-feature=+neon" + export TARGET_CFLAGS="-mfpu=vfpv3-d16" + ;; + # Some of our test dependencies use the deprecated `gcc` crates which + # doesn't detect RISC-V compilers automatically, so do it manually here. + riscv64*) + export TARGET_CC="riscv64-linux-gnu-gcc" + ;; esac echo "RUSTFLAGS=${RUSTFLAGS}" @@ -44,12 +65,27 @@ cargo_test() { fi cmd="$cmd ${subcmd} --target=$TARGET $1" cmd="$cmd -- $2" + + # wasm targets can't catch panics so if a test failures make sure the test + # harness isn't trying to capture output, otherwise we won't get any useful + # output. + case ${TARGET} in + wasm32*) + cmd="$cmd --nocapture" + ;; + esac + + if [ "$SKIP_TESTS" != "" ]; then + cmd="$cmd --skip "$SKIP_TESTS + fi $cmd } CORE_ARCH="--manifest-path=crates/core_arch/Cargo.toml" STD_DETECT="--manifest-path=crates/std_detect/Cargo.toml" STDARCH_EXAMPLES="--manifest-path=examples/Cargo.toml" +INTRINSIC_TEST="--manifest-path=crates/intrinsic-test/Cargo.toml" + cargo_test "${CORE_ARCH} --release" if [ "$NOSTD" != "1" ]; then @@ -72,21 +108,6 @@ case ${TARGET} in export RUSTFLAGS="${RUSTFLAGS} -C target-feature=+avx" cargo_test "--release" ;; - wasm32-unknown-unknown*) - # Attempt to actually run some SIMD tests in node.js. Unfortunately - # though node.js (transitively through v8) doesn't have support for the - # full SIMD spec yet, only some functions. As a result only pass in - # some target features and a special `--cfg` - # FIXME: broken - #export RUSTFLAGS="${RUSTFLAGS} -C target-feature=+simd128 --cfg only_node_compatible_functions" - #cargo_test "--release" - - # After that passes make sure that all intrinsics compile, passing in - # the extra feature to compile in non-node-compatible SIMD. - # FIXME: broken - #export RUSTFLAGS="${RUSTFLAGS} -C target-feature=+simd128,+unimplemented-simd128" - #cargo_test "--release --no-run" - ;; # FIXME: don't build anymore #mips-*gnu* | mipsel-*gnu*) # export RUSTFLAGS="${RUSTFLAGS} -C target-feature=+msa,+fp64,+mips32r5" @@ -111,7 +132,15 @@ case ${TARGET} in esac -if [ "$NORUN" != "1" ] && [ "$NOSTD" != 1 ] && [ "$TARGET" != "wasm32-unknown-unknown" ]; then +if [ "${TARGET}" = "aarch64-unknown-linux-gnu" ]; then + export CPPFLAGS="-fuse-ld=lld -I/usr/aarch64-linux-gnu/include/ -I/usr/aarch64-linux-gnu/include/c++/9/aarch64-linux-gnu/" + RUST_LOG=warn cargo run ${INTRINSIC_TEST} --release --bin intrinsic-test -- crates/intrinsic-test/acle/tools/intrinsic_db/advsimd.csv --runner "${CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER}" --cppcompiler "clang++-13" --skip crates/intrinsic-test/missing_aarch64.txt +elif [ "${TARGET}" = "armv7-unknown-linux-gnueabihf" ]; then + export CPPFLAGS="-fuse-ld=lld -I/usr/arm-linux-gnueabihf/include/ -I/usr/arm-linux-gnueabihf/include/c++/9/arm-linux-gnueabihf/" + RUST_LOG=warn cargo run ${INTRINSIC_TEST} --release --bin intrinsic-test -- crates/intrinsic-test/acle/tools/intrinsic_db/advsimd.csv --runner "${CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_RUNNER}" --cppcompiler "clang++-13" --skip crates/intrinsic-test/missing_arm.txt --a32 +fi + +if [ "$NORUN" != "1" ] && [ "$NOSTD" != 1 ]; then # Test examples ( cd examples diff --git a/crux-mir/lib/stdarch/crates/assert-instr-macro/Cargo.toml b/crux-mir/lib/stdarch/crates/assert-instr-macro/Cargo.toml index eeefca254..3d9b32067 100644 --- a/crux-mir/lib/stdarch/crates/assert-instr-macro/Cargo.toml +++ b/crux-mir/lib/stdarch/crates/assert-instr-macro/Cargo.toml @@ -2,6 +2,7 @@ name = "assert-instr-macro" version = "0.1.0" authors = ["Alex Crichton "] +edition = "2018" [lib] proc-macro = true diff --git a/crux-mir/lib/stdarch/crates/assert-instr-macro/build.rs b/crux-mir/lib/stdarch/crates/assert-instr-macro/build.rs index 45a868441..01ae79660 100644 --- a/crux-mir/lib/stdarch/crates/assert-instr-macro/build.rs +++ b/crux-mir/lib/stdarch/crates/assert-instr-macro/build.rs @@ -6,7 +6,7 @@ fn main() { .ok() .and_then(|s| s.parse().ok()) .unwrap_or(0); - let profile = env::var("PROFILE").unwrap_or(String::new()); + let profile = env::var("PROFILE").unwrap_or_default(); if profile == "release" || opt_level >= 2 { println!("cargo:rustc-cfg=optimized"); } diff --git a/crux-mir/lib/stdarch/crates/assert-instr-macro/src/lib.rs b/crux-mir/lib/stdarch/crates/assert-instr-macro/src/lib.rs index 8198aebc2..9fa411df3 100644 --- a/crux-mir/lib/stdarch/crates/assert-instr-macro/src/lib.rs +++ b/crux-mir/lib/stdarch/crates/assert-instr-macro/src/lib.rs @@ -7,12 +7,10 @@ //! The procedural macro here is relatively simple, it simply appends a //! `#[test]` function to the original token stream which asserts that the //! function itself contains the relevant instruction. +#![deny(rust_2018_idioms)] -extern crate proc_macro; -extern crate proc_macro2; #[macro_use] extern crate quote; -extern crate syn; use proc_macro2::TokenStream; use quote::ToTokens; @@ -43,6 +41,10 @@ pub fn assert_instr( // testing for. let disable_assert_instr = std::env::var("STDARCH_DISABLE_ASSERT_INSTR").is_ok(); + // Disable dedup guard. Only works if the LLVM MergeFunctions pass is disabled, e.g. + // with `-Z merge-functions=disabled` in RUSTFLAGS. + let disable_dedup_guard = std::env::var("STDARCH_DISABLE_DEDUP_GUARD").is_ok(); + // If instruction tests are disabled avoid emitting this shim at all, just // return the original item without our attribute. if !cfg!(optimized) || disable_assert_instr { @@ -60,8 +62,13 @@ pub fn assert_instr( &format!("stdarch_test_shim_{}_{}", name, instr_str), name.span(), ); + let shim_name_ptr = syn::Ident::new( + &format!("stdarch_test_shim_{}_{}_ptr", name, instr_str).to_ascii_uppercase(), + name.span(), + ); let mut inputs = Vec::new(); let mut input_vals = Vec::new(); + let mut const_vals = Vec::new(); let ret = &func.sig.output; for arg in func.sig.inputs.iter() { let capture = match *arg { @@ -82,6 +89,20 @@ pub fn assert_instr( input_vals.push(quote! { #ident }); } } + for arg in func.sig.generics.params.iter() { + let c = match *arg { + syn::GenericParam::Const(ref c) => c, + ref v => panic!( + "only const generics are allowed: `{:?}`", + v.clone().into_token_stream() + ), + }; + if let Some(&(_, ref tokens)) = invoc.args.iter().find(|a| c.ident == a.0) { + const_vals.push(quote! { #tokens }); + } else { + panic!("const generics must have a value for tests"); + } + } let attrs = func .attrs @@ -101,56 +122,62 @@ pub fn assert_instr( // Use an ABI on Windows that passes SIMD values in registers, like what // happens on Unix (I think?) by default. let abi = if cfg!(windows) { - syn::LitStr::new("vectorcall", proc_macro2::Span::call_site()) + let target = std::env::var("TARGET").unwrap(); + if target.contains("x86_64") { + syn::LitStr::new("sysv64", proc_macro2::Span::call_site()) + } else { + syn::LitStr::new("vectorcall", proc_macro2::Span::call_site()) + } } else { syn::LitStr::new("C", proc_macro2::Span::call_site()) }; let shim_name_str = format!("{}{}", shim_name, assert_name); - let to_test = quote! { - #attrs - #[no_mangle] - #[inline(never)] - pub unsafe extern #abi fn #shim_name(#(#inputs),*) #ret { - // The compiler in optimized mode by default runs a pass called - // "mergefunc" where it'll merge functions that look identical. - // Turns out some intrinsics produce identical code and they're - // folded together, meaning that one just jumps to another. This - // messes up our inspection of the disassembly of this function and - // we're not a huge fan of that. - // - // To thwart this pass and prevent functions from being merged we - // generate some code that's hopefully very tight in terms of - // codegen but is otherwise unique to prevent code from being - // folded. - ::stdarch_test::_DONT_DEDUP.store( - std::mem::transmute(#shim_name_str.as_bytes().as_ptr()), - std::sync::atomic::Ordering::Relaxed, - ); - #name(#(#input_vals),*) + let to_test = if disable_dedup_guard { + quote! { + #attrs + #[no_mangle] + #[inline(never)] + pub unsafe extern #abi fn #shim_name(#(#inputs),*) #ret { + #name::<#(#const_vals),*>(#(#input_vals),*) + } + } + } else { + quote! { + + const #shim_name_ptr : *const u8 = #shim_name_str.as_ptr(); + + #attrs + #[no_mangle] + #[inline(never)] + pub unsafe extern #abi fn #shim_name(#(#inputs),*) #ret { + // The compiler in optimized mode by default runs a pass called + // "mergefunc" where it'll merge functions that look identical. + // Turns out some intrinsics produce identical code and they're + // folded together, meaning that one just jumps to another. This + // messes up our inspection of the disassembly of this function and + // we're not a huge fan of that. + // + // To thwart this pass and prevent functions from being merged we + // generate some code that's hopefully very tight in terms of + // codegen but is otherwise unique to prevent code from being + // folded. + ::stdarch_test::_DONT_DEDUP = #shim_name_ptr; + #name::<#(#const_vals),*>(#(#input_vals),*) + } } }; let tokens: TokenStream = quote! { - #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] - #[cfg_attr(not(target_arch = "wasm32"), test)] + #[test] #[allow(non_snake_case)] fn #assert_name() { #to_test - // Make sure that the shim is not removed by leaking it to unknown - // code: - unsafe { asm!("" : : "r"(#shim_name as usize) : "memory" : "volatile") }; - ::stdarch_test::assert(#shim_name as usize, stringify!(#shim_name), #instr); } }; - // why? necessary now to get tests to work? - let tokens: TokenStream = tokens - .to_string() - .parse() - .expect("cannot parse tokenstream"); let tokens: TokenStream = quote! { #item @@ -165,7 +192,7 @@ struct Invoc { } impl syn::parse::Parse for Invoc { - fn parse(input: syn::parse::ParseStream) -> syn::Result { + fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { use syn::{ext::IdentExt, Token}; let mut instr = String::new(); @@ -178,7 +205,7 @@ impl syn::parse::Parse for Invoc { continue; } if input.parse::().is_ok() { - instr.push_str("."); + instr.push('.'); continue; } if let Ok(s) = input.parse::() { diff --git a/crux-mir/lib/stdarch/crates/core_arch/Cargo.toml b/crux-mir/lib/stdarch/crates/core_arch/Cargo.toml index 72d89b016..e2b332af2 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/Cargo.toml +++ b/crux-mir/lib/stdarch/crates/core_arch/Cargo.toml @@ -7,14 +7,12 @@ authors = [ "Gonzalo Brito Gadeschi ", ] description = "`core::arch` - Rust's core library architecture-specific intrinsics." -documentation = "https://docs.rs/core_arch" homepage = "https://github.com/rust-lang/stdarch" repository = "https://github.com/rust-lang/stdarch" readme = "README.md" keywords = ["core", "simd", "arch", "intrinsics"] categories = ["hardware-support", "no-std"] -license = "MIT/Apache-2.0" -build = "build.rs" +license = "MIT OR Apache-2.0" edition = "2018" [badges] @@ -25,9 +23,3 @@ maintenance = { status = "experimental" } [dev-dependencies] stdarch-test = { version = "0.*", path = "../stdarch-test" } std_detect = { version = "0.*", path = "../std_detect" } - -[target.wasm32-unknown-unknown.dev-dependencies] -wasm-bindgen-test = "0.2.47" - -[package.metadata.docs.rs] -rustdoc-args = [ "--cfg", "dox" ] diff --git a/crux-mir/lib/stdarch/crates/core_arch/MISSING.md b/crux-mir/lib/stdarch/crates/core_arch/MISSING.md new file mode 100644 index 000000000..c948f3f8c --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/MISSING.md @@ -0,0 +1,116 @@ +## The following neon instructions are currently not implemented in stdarch + +### Not implemented on arm: + +`vcadd_rot270_f32` + +`vcadd_rot90_f32` + +`vcaddq_rot270_f32` + +`vcaddq_rot90_f32` + +`vdot_s32` + +`vdot_u32` + +`vdotq_s32` + +`vdotq_u32` + +`vdot_lane_s32` + +`vdot_lane_u32` + +`vdotq_lane_s32` + +`vdotq_lane_u32` + +`vcmla_f32` + +`vcmla_lane_f32` + +`vcmla_laneq_f32` + +`vcmla_rot180_f32` + +`vcmla_rot180_lane_f32` + +`vcmla_rot180_laneq_f32` + +`vcmla_rot270_f32` + +`vcmla_rot270_lane_f32` + +`vcmla_rot270_laneq_f32` + +`vcmla_rot90_f32` + +`vcmla_rot90_lane_f32` + +`vcmla_rot90_laneq_f32` + +`vcmlaq_f32` + +`vcmlaq_lane_f32` + +`vcmlaq_laneq_f32` + +`vcmlaq_rot180_f32` + +`vcmlaq_rot180_lane_f32` + +`vcmlaq_rot180_laneq_f32` + +`vcmlaq_rot270_f32` + +`vcmlaq_rot270_lane_f32` + +`vcmlaq_rot270_laneq_f32` + +`vcmlaq_rot90_f32` + +`vcmlaq_rot90_lane_f32` + +`vcmlaq_rot90_laneq_f32` + +### Not implemented in LLVM: + +`vrnd32x_f64` + +`vrnd32xq_f64` + +`vrnd32z_f64` + +`vrnd32zq_f64` + +`vrnd64x_f64` + +`vrnd64xq_f64` + +`vrnd64z_f64` + +`vrnd64zq_f64` + +### LLVM Select errors may occur: + +`vsudot_lane_s32` + +`vsudot_laneq_s32` + +`vsudotq_lane_s32` + +`vsudotq_laneq_s32` + +`vusdot_lane_s32` + +`vusdot_laneq_s32` + +`vusdot_s32` + +`vusdotq_lane_s32` + +`vusdotq_laneq_s32` + +`vusdotq_s32v` + diff --git a/crux-mir/lib/stdarch/crates/core_arch/README.md b/crux-mir/lib/stdarch/crates/core_arch/README.md index 1f53db4c4..a5f490bcf 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/README.md +++ b/crux-mir/lib/stdarch/crates/core_arch/README.md @@ -1,9 +1,6 @@ `core::arch` - Rust's core library architecture-specific intrinsics ======= -[![core_arch_crate_badge]][core_arch_crate_link] [![core_arch_docs_badge]][core_arch_docs_link] - - The `core::arch` module implements architecture-dependent intrinsics (e.g. SIMD). # Usage @@ -61,8 +58,3 @@ See LICENSE-APACHE, and LICENSE-MIT for details. Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in `core_arch` by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. - -[core_arch_crate_badge]: https://img.shields.io/crates/v/core_arch.svg -[core_arch_crate_link]: https://crates.io/crates/core_arch -[core_arch_docs_badge]: https://docs.rs/core_arch/badge.svg -[core_arch_docs_link]: https://docs.rs/core_arch/ diff --git a/crux-mir/lib/stdarch/crates/core_arch/avx512bw.md b/crux-mir/lib/stdarch/crates/core_arch/avx512bw.md new file mode 100644 index 000000000..20c8c2f14 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/avx512bw.md @@ -0,0 +1,764 @@ +["AVX512BW"]

+ + * [x] [`_mm512_loadu_epi16`] + * [x] [`_mm512_mask_loadu_epi16`] //need i1 + * [x] [`_mm512_maskz_loadu_epi16`] //need i1 + * [x] [`_mm_loadu_epi16`] + * [x] [`_mm_mask_loadu_epi16`] //need i1 + * [x] [`_mm_maskz_loadu_epi16`] //need i1 + * [x] [`_mm256_loadu_epi16`] + * [x] [`_mm256_mask_loadu_epi16`] //need i1 + * [x] [`_mm256_maskz_loadu_epi16`] //need i1 + * [x] [`_mm512_loadu_epi8`] + * [x] [`_mm512_mask_loadu_epi8`] //need i1 + * [x] [`_mm512_maskz_loadu_epi8`] //need i1 + * [x] [`_mm_loadu_epi8`] + * [x] [`_mm_mask_loadu_epi8`] //need i1 + * [x] [`_mm_maskz_loadu_epi8`] //need i1 + * [x] [`_mm256_loadu_epi8`] + * [x] [`_mm256_mask_loadu_epi8`] //need i1 + * [x] [`_mm256_maskz_loadu_epi8`] //need i1 + * [x] [`_mm512_mask_storeu_epi16`] + * [x] [`_mm512_storeu_epi16`] + * [x] [`_mm_mask_storeu_epi16`] //need i1 + * [x] [`_mm_storeu_epi16`] + * [x] [`_mm256_mask_storeu_epi16`] //need i1 + * [x] [`_mm256_storeu_epi16`] + * [x] [`_mm512_mask_storeu_epi8`] //need i1 + * [x] [`_mm512_storeu_epi8`] + * [x] [`_mm_mask_storeu_epi8`] //need i1 + * [x] [`_mm_storeu_epi8`] + * [x] [`_mm256_mask_storeu_epi8`] //need i1 + * [x] [`_mm256_storeu_epi8`] + * [x] [`_mm512_abs_epi16`] + * [x] [`_mm512_mask_abs_epi16`] + * [x] [`_mm512_maskz_abs_epi16`] + * [x] [`_mm_mask_abs_epi16`] + * [x] [`_mm_maskz_abs_epi16`] + * [x] [`_mm256_mask_abs_epi16`] + * [x] [`_mm256_maskz_abs_epi16`] + * [x] [`_mm512_abs_epi8`] + * [x] [`_mm512_mask_abs_epi8`] + * [x] [`_mm512_maskz_abs_epi8`] + * [x] [`_mm_mask_abs_epi8`] + * [x] [`_mm_maskz_abs_epi8`] + * [x] [`_mm256_mask_abs_epi8`] + * [x] [`_mm256_maskz_abs_epi8`] + * [x] [`_mm512_add_epi16`] + * [x] [`_mm512_mask_add_epi16`] + * [x] [`_mm512_maskz_add_epi16`] + * [x] [`_mm_mask_add_epi16`] + * [x] [`_mm_maskz_add_epi16`] + * [x] [`_mm256_mask_add_epi16`] + * [x] [`_mm256_maskz_add_epi16`] + * [x] [`_mm512_add_epi8`] + * [x] [`_mm512_mask_add_epi8`] + * [x] [`_mm512_maskz_add_epi8`] + * [x] [`_mm_mask_add_epi8`] + * [x] [`_mm_maskz_add_epi8`] + * [x] [`_mm256_mask_add_epi8`] + * [x] [`_mm256_maskz_add_epi8`] + * [x] [`_mm512_adds_epi16`] + * [x] [`_mm512_mask_adds_epi16`] + * [x] [`_mm512_maskz_adds_epi16`] + * [x] [`_mm_mask_adds_epi16`] + * [x] [`_mm_maskz_adds_epi16`] + * [x] [`_mm256_mask_adds_epi16`] + * [x] [`_mm256_maskz_adds_epi16`] + * [x] [`_mm512_adds_epi8`] + * [x] [`_mm512_mask_adds_epi8`] + * [x] [`_mm512_maskz_adds_epi8`] + * [x] [`_mm_mask_adds_epi8`] + * [x] [`_mm_maskz_adds_epi8`] + * [x] [`_mm256_mask_adds_epi8`] + * [x] [`_mm256_maskz_adds_epi8`] + * [x] [`_mm512_adds_epu16`] + * [x] [`_mm512_mask_adds_epu16`] + * [x] [`_mm512_maskz_adds_epu16`] + * [x] [`_mm_mask_adds_epu16`] + * [x] [`_mm_maskz_adds_epu16`] + * [x] [`_mm256_mask_adds_epu16`] + * [x] [`_mm256_maskz_adds_epu16`] + * [x] [`_mm512_adds_epu8`] + * [x] [`_mm512_mask_adds_epu8`] + * [x] [`_mm512_maskz_adds_epu8`] + * [x] [`_mm_mask_adds_epu8`] + * [x] [`_mm_maskz_adds_epu8`] + * [x] [`_mm256_mask_adds_epu8`] + * [x] [`_mm256_maskz_adds_epu8`] + * [x] [`_mm512_alignr_epi8`] + * [x] [`_mm512_mask_alignr_epi8`] + * [x] [`_mm512_maskz_alignr_epi8`] + * [x] [`_mm_mask_alignr_epi8`] + * [x] [`_mm_maskz_alignr_epi8`] + * [x] [`_mm256_mask_alignr_epi8`] + * [x] [`_mm256_maskz_alignr_epi8`] + * [x] [`_mm512_avg_epu16`] + * [x] [`_mm512_mask_avg_epu16`] + * [x] [`_mm512_maskz_avg_epu16`] + * [x] [`_mm_mask_avg_epu16`] + * [x] [`_mm_maskz_avg_epu16`] + * [x] [`_mm256_mask_avg_epu16`] + * [x] [`_mm256_maskz_avg_epu16`] + * [x] [`_mm512_avg_epu8`] + * [x] [`_mm512_mask_avg_epu8`] + * [x] [`_mm512_maskz_avg_epu8`] + * [x] [`_mm_mask_avg_epu8`] + * [x] [`_mm_maskz_avg_epu8`] + * [x] [`_mm256_mask_avg_epu8`] + * [x] [`_mm256_maskz_avg_epu8`] + * [x] [`_mm512_mask_blend_epi16`] + * [x] [`_mm_mask_blend_epi16`] + * [x] [`_mm256_mask_blend_epi16`] + * [x] [`_mm512_mask_blend_epi8`] + * [x] [`_mm512_broadcastb_epi8`] + * [x] [`_mm_mask_blend_epi8`] + * [x] [`_mm256_mask_blend_epi8`] + * [x] [`_mm512_mask_broadcastb_epi8`] + * [x] [`_mm512_maskz_broadcastb_epi8`] + * [x] [`_mm_mask_broadcastb_epi8`] + * [x] [`_mm_maskz_broadcastb_epi8`] + * [x] [`_mm256_mask_broadcastb_epi8`] + * [x] [`_mm256_maskz_broadcastb_epi8`] + * [x] [`_mm512_broadcastw_epi16`] + * [x] [`_mm512_mask_broadcastw_epi16`] + * [x] [`_mm512_maskz_broadcastw_epi16`] + * [x] [`_mm_mask_broadcastw_epi16`] + * [x] [`_mm_maskz_broadcastw_epi16`] + * [x] [`_mm256_mask_broadcastw_epi16`] + * [x] [`_mm256_maskz_broadcastw_epi16`] + * [x] [`_mm512_bslli_epi128`] + * [x] [`_mm512_bsrli_epi128`] + * [x] [`_mm512_cmp_epi16_mask`] + * [x] [`_mm512_mask_cmp_epi16_mask`] + * [x] [`_mm_cmp_epi16_mask`] + * [x] [`_mm_mask_cmp_epi16_mask`] + * [x] [`_mm256_cmp_epi16_mask`] + * [x] [`_mm256_mask_cmp_epi16_mask`] + * [x] [`_mm512_cmp_epi8_mask`] + * [x] [`_mm512_mask_cmp_epi8_mask`] + * [x] [`_mm_cmp_epi8_mask`] + * [x] [`_mm_mask_cmp_epi8_mask`] + * [x] [`_mm256_cmp_epi8_mask`] + * [x] [`_mm256_mask_cmp_epi8_mask`] + * [x] [`_mm512_cmp_epu16_mask`] + * [x] [`_mm512_mask_cmp_epu16_mask`] + * [x] [`_mm_cmp_epu16_mask`] + * [x] [`_mm_mask_cmp_epu16_mask`] + * [x] [`_mm256_cmp_epu16_mask`] + * [x] [`_mm256_mask_cmp_epu16_mask`] + * [x] [`_mm512_cmp_epu8_mask`] + * [x] [`_mm512_mask_cmp_epu8_mask`] + * [x] [`_mm_cmp_epu8_mask`] + * [x] [`_mm_mask_cmp_epu8_mask`] + * [x] [`_mm256_cmp_epu8_mask`] + * [x] [`_mm256_mask_cmp_epu8_mask`] + * [x] [`_mm512_cmpeq_epi16_mask`] + * [x] [`_mm512_mask_cmpeq_epi16_mask`] + * [x] [`_mm_cmpeq_epi16_mask`] + * [x] [`_mm_mask_cmpeq_epi16_mask`] + * [x] [`_mm256_cmpeq_epi16_mask`] + * [x] [`_mm256_mask_cmpeq_epi16_mask`] + * [x] [`_mm512_cmpeq_epi8_mask`] + * [x] [`_mm512_mask_cmpeq_epi8_mask`] + * [x] [`_mm_cmpeq_epi8_mask`] + * [x] [`_mm_mask_cmpeq_epi8_mask`] + * [x] [`_mm256_cmpeq_epi8_mask`] + * [x] [`_mm256_mask_cmpeq_epi8_mask`] + * [x] [`_mm512_cmpeq_epu16_mask`] + * [x] [`_mm512_mask_cmpeq_epu16_mask`] + * [x] [`_mm_cmpeq_epu16_mask`] + * [x] [`_mm_mask_cmpeq_epu16_mask`] + * [x] [`_mm256_cmpeq_epu16_mask`] + * [x] [`_mm256_mask_cmpeq_epu16_mask`] + * [x] [`_mm512_cmpeq_epu8_mask`] + * [x] [`_mm512_mask_cmpeq_epu8_mask`] + * [x] [`_mm_cmpeq_epu8_mask`] + * [x] [`_mm_mask_cmpeq_epu8_mask`] + * [x] [`_mm256_cmpeq_epu8_mask`] + * [x] [`_mm256_mask_cmpeq_epu8_mask`] + * [x] [`_mm512_cmpge_epi16_mask`] + * [x] [`_mm512_mask_cmpge_epi16_mask`] + * [x] [`_mm_cmpge_epi16_mask`] + * [x] [`_mm_mask_cmpge_epi16_mask`] + * [x] [`_mm256_cmpge_epi16_mask`] + * [x] [`_mm256_mask_cmpge_epi16_mask`] + * [x] [`_mm512_cmpge_epi8_mask`] + * [x] [`_mm512_mask_cmpge_epi8_mask`] + * [x] [`_mm_cmpge_epi8_mask`] + * [x] [`_mm_mask_cmpge_epi8_mask`] + * [x] [`_mm256_cmpge_epi8_mask`] + * [x] [`_mm256_mask_cmpge_epi8_mask`] + * [x] [`_mm512_cmpge_epu16_mask`] + * [x] [`_mm512_mask_cmpge_epu16_mask`] + * [x] [`_mm_cmpge_epu16_mask`] + * [x] [`_mm_mask_cmpge_epu16_mask`] + * [x] [`_mm256_cmpge_epu16_mask`] + * [x] [`_mm256_mask_cmpge_epu16_mask`] + * [x] [`_mm512_cmpge_epu8_mask`] + * [x] [`_mm512_mask_cmpge_epu8_mask`] + * [x] [`_mm_cmpge_epu8_mask`] + * [x] [`_mm_mask_cmpge_epu8_mask`] + * [x] [`_mm256_cmpge_epu8_mask`] + * [x] [`_mm256_mask_cmpge_epu8_mask`] + * [x] [`_mm512_cmpgt_epi16_mask`] + * [x] [`_mm512_mask_cmpgt_epi16_mask`] + * [x] [`_mm512_cmpgt_epi8_mask`] + * [x] [`_mm512_mask_cmpgt_epi8_mask`] + * [x] [`_mm_cmpgt_epi8_mask`] + * [x] [`_mm_mask_cmpgt_epi8_mask`] + * [x] [`_mm256_cmpgt_epi8_mask`] + * [x] [`_mm256_mask_cmpgt_epi8_mask`] + * [x] [`_mm512_cmpgt_epu16_mask`] + * [x] [`_mm512_mask_cmpgt_epu16_mask`] + * [x] [`_mm_cmpgt_epu16_mask`] + * [x] [`_mm_mask_cmpgt_epu16_mask`] + * [x] [`_mm256_cmpgt_epu16_mask`] + * [x] [`_mm256_mask_cmpgt_epu16_mask`] + * [x] [`_mm512_cmpgt_epu8_mask`] + * [x] [`_mm512_mask_cmpgt_epu8_mask`] + * [x] [`_mm_cmpgt_epu8_mask`] + * [x] [`_mm_mask_cmpgt_epu8_mask`] + * [x] [`_mm256_cmpgt_epu8_mask`] + * [x] [`_mm256_mask_cmpgt_epu8_mask`] + * [x] [`_mm512_cmple_epi16_mask`] + * [x] [`_mm512_mask_cmple_epi16_mask`] + * [x] [`_mm_cmpgt_epi16_mask`] + * [x] [`_mm_mask_cmpgt_epi16_mask`] + * [x] [`_mm256_cmpgt_epi16_mask`] + * [x] [`_mm256_mask_cmpgt_epi16_mask`] + * [x] [`_mm512_cmple_epi8_mask`] + * [x] [`_mm512_mask_cmple_epi8_mask`] + * [x] [`_mm_cmple_epi8_mask`] + * [x] [`_mm_mask_cmple_epi8_mask`] + * [x] [`_mm256_cmple_epi8_mask`] + * [x] [`_mm256_mask_cmple_epi8_mask`] + * [x] [`_mm512_cmple_epu16_mask`] + * [x] [`_mm512_mask_cmple_epu16_mask`] + * [x] [`_mm_cmple_epu16_mask`] + * [x] [`_mm_mask_cmple_epu16_mask`] + * [x] [`_mm256_cmple_epu16_mask`] + * [x] [`_mm256_mask_cmple_epu16_mask`] + * [x] [`_mm512_cmple_epu8_mask`] + * [x] [`_mm512_mask_cmple_epu8_mask`] + * [x] [`_mm_cmple_epu8_mask`] + * [x] [`_mm_mask_cmple_epu8_mask`] + * [x] [`_mm256_cmple_epu8_mask`] + * [x] [`_mm256_mask_cmple_epu8_mask`] + * [x] [`_mm512_cmplt_epi16_mask`] + * [x] [`_mm512_mask_cmplt_epi16_mask`] + * [x] [`_mm_cmple_epi16_mask`] + * [x] [`_mm_mask_cmple_epi16_mask`] + * [x] [`_mm256_cmple_epi16_mask`] + * [x] [`_mm256_mask_cmple_epi16_mask`] + * [x] [`_mm512_cmplt_epi8_mask`] + * [x] [`_mm512_mask_cmplt_epi8_mask`] + * [x] [`_mm_cmplt_epi8_mask`] + * [x] [`_mm_mask_cmplt_epi8_mask`] + * [x] [`_mm256_cmplt_epi8_mask`] + * [x] [`_mm256_mask_cmplt_epi8_mask`] + * [x] [`_mm512_cmplt_epu16_mask`] + * [x] [`_mm512_mask_cmplt_epu16_mask`] + * [x] [`_mm_cmplt_epu16_mask`] + * [x] [`_mm_mask_cmplt_epu16_mask`] + * [x] [`_mm256_cmplt_epu16_mask`] + * [x] [`_mm256_mask_cmplt_epu16_mask`] + * [x] [`_mm512_cmplt_epu8_mask`] + * [x] [`_mm512_mask_cmplt_epu8_mask`] + * [x] [`_mm_cmplt_epu8_mask`] + * [x] [`_mm_mask_cmplt_epu8_mask`] + * [x] [`_mm256_cmplt_epu8_mask`] + * [x] [`_mm256_mask_cmplt_epu8_mask`] + * [x] [`_mm512_cmpneq_epi16_mask`] + * [x] [`_mm512_mask_cmpneq_epi16_mask`] + * [x] [`_mm_cmpneq_epi16_mask`] + * [x] [`_mm_mask_cmpneq_epi16_mask`] + * [x] [`_mm256_cmpneq_epi16_mask`] + * [x] [`_mm256_mask_cmpneq_epi16_mask`] + * [x] [`_mm512_cmpneq_epi8_mask`] + * [x] [`_mm512_mask_cmpneq_epi8_mask`] + * [x] [`_mm_cmpneq_epi8_mask`] + * [x] [`_mm_mask_cmpneq_epi8_mask`] + * [x] [`_mm256_cmpneq_epi8_mask`] + * [x] [`_mm256_mask_cmpneq_epi8_mask`] + * [x] [`_mm512_cmpneq_epu16_mask`] + * [x] [`_mm512_mask_cmpneq_epu16_mask`] + * [x] [`_mm_cmpneq_epu16_mask`] + * [x] [`_mm_mask_cmpneq_epu16_mask`] + * [x] [`_mm256_cmpneq_epu16_mask`] + * [x] [`_mm256_mask_cmpneq_epu16_mask`] + * [x] [`_mm512_cmpneq_epu8_mask`] + * [x] [`_mm512_mask_cmpneq_epu8_mask`] + * [x] [`_mm_cmpneq_epu8_mask`] + * [x] [`_mm_mask_cmpneq_epu8_mask`] + * [x] [`_mm256_cmpneq_epu8_mask`] + * [x] [`_mm256_mask_cmpneq_epu8_mask`] + * [x] [`_mm512_cvtepi16_epi8`] + * [x] [`_mm512_mask_cvtepi16_epi8`] + * [x] [`_mm512_maskz_cvtepi16_epi8`] + * [x] [`_mm512_mask_cvtepi16_storeu_epi8`] + * [x] [`_mm_mask_cvtepi16_storeu_epi8`] + * [x] [`_mm256_mask_cvtepi16_storeu_epi8`] + * [x] [`_mm_cvtepi16_epi8`] + * [x] [`_mm_mask_cvtepi16_epi8`] + * [x] [`_mm_maskz_cvtepi16_epi8`] + * [x] [`_mm256_cvtepi16_epi8`] + * [x] [`_mm256_mask_cvtepi16_epi8`] + * [x] [`_mm256_maskz_cvtepi16_epi8`] + * [x] [`_mm512_cvtepi8_epi16`] + * [x] [`_mm512_mask_cvtepi8_epi16`] + * [x] [`_mm512_maskz_cvtepi8_epi16`] + * [x] [`_mm_mask_cvtepi8_epi16`] + * [x] [`_mm_maskz_cvtepi8_epi16`] + * [x] [`_mm256_mask_cvtepi8_epi16`] + * [x] [`_mm256_maskz_cvtepi8_epi16`] + * [x] [`_mm512_cvtsepi16_epi8`] + * [x] [`_mm512_mask_cvtsepi16_epi8`] + * [x] [`_mm512_maskz_cvtsepi16_epi8`] + * [x] [`_mm_cvtsepi16_epi8`] + * [x] [`_mm_mask_cvtsepi16_epi8`] + * [x] [`_mm_maskz_cvtsepi16_epi8`] + * [x] [`_mm256_cvtsepi16_epi8`] + * [x] [`_mm256_mask_cvtsepi16_epi8`] + * [x] [`_mm256_maskz_cvtsepi16_epi8`] + * [x] [`_mm512_mask_cvtsepi16_storeu_epi8`] + * [x] [`_mm_mask_cvtsepi16_storeu_epi8`] + * [x] [`_mm256_mask_cvtsepi16_storeu_epi8`] + * [x] [`_mm512_cvtepu8_epi16`] + * [x] [`_mm512_mask_cvtepu8_epi16`] + * [x] [`_mm512_maskz_cvtepu8_epi16`] + * [x] [`_mm_mask_cvtepu8_epi16`] + * [x] [`_mm_maskz_cvtepu8_epi16`] + * [x] [`_mm256_mask_cvtepu8_epi16`] + * [x] [`_mm256_maskz_cvtepu8_epi16`] + * [_] [`_cvtmask32_u32`] + * [_] [`_cvtmask64_u64`] + * [_] [`_cvtu32_mask32`] + * [_] [`_cvtu64_mask64`] + * [x] [`_mm512_cvtusepi16_epi8`] + * [x] [`_mm512_mask_cvtusepi16_epi8`] + * [x] [`_mm512_maskz_cvtusepi16_epi8`] + * [x] [`_mm_cvtusepi16_epi8`] + * [x] [`_mm_mask_cvtusepi16_epi8`] + * [x] [`_mm_maskz_cvtusepi16_epi8`] + * [x] [`_mm256_cvtusepi16_epi8`] + * [x] [`_mm256_mask_cvtusepi16_epi8`] + * [x] [`_mm256_maskz_cvtusepi16_epi8`] + * [x] [`_mm512_mask_cvtusepi16_storeu_epi8`] + * [x] [`_mm_mask_cvtusepi16_storeu_epi8`] + * [x] [`_mm256_mask_cvtusepi16_storeu_epi8`] + * [x] [`_mm512_dbsad_epu8`] + * [x] [`_mm512_mask_dbsad_epu8`] + * [x] [`_mm512_maskz_dbsad_epu8`] + * [x] [`_mm_dbsad_epu8`] + * [x] [`_mm_mask_dbsad_epu8`] + * [x] [`_mm_maskz_dbsad_epu8`] + * [x] [`_mm256_dbsad_epu8`] + * [x] [`_mm256_mask_dbsad_epu8`] + * [x] [`_mm256_maskz_dbsad_epu8`] + * [x] [`_kadd_mask32`] + * [x] [`_kadd_mask64`] + * [x] [`_kand_mask32`] + * [x] [`_kand_mask64`] + * [x] [`_kandn_mask32`] + * [x] [`_kandn_mask64`] + * [x] [`_knot_mask32`] + * [x] [`_knot_mask64`] + * [x] [`_kor_mask32`] + * [x] [`_kor_mask64`] + * [_] [`_kortest_mask32_u8`] + * [_] [`_kortest_mask64_u8`] + * [_] [`_kortestc_mask32_u8`] + * [_] [`_kortestc_mask64_u8`] + * [_] [`_kortestz_mask32_u8`] + * [_] [`_kortestz_mask64_u8`] + * [_] [`_kshiftli_mask32`] + * [_] [`_kshiftli_mask64`] + * [_] [`_kshiftri_mask32`] + * [_] [`_kshiftri_mask64`] + * [_] [`_ktest_mask32_u8`] + * [_] [`_ktest_mask64_u8`] + * [_] [`_ktestc_mask32_u8`] + * [_] [`_ktestc_mask64_u8`] + * [_] [`_ktestz_mask32_u8`] + * [_] [`_ktestz_mask64_u8`] + * [_] [`_mm512_kunpackd`] + * [_] [`_mm512_kunpackw`] + * [x] [`_kxnor_mask32`] + * [x] [`_kxnor_mask64`] + * [x] [`_kxor_mask32`] + * [x] [`_kxor_mask64`] + * [x] [`_load_mask32`] + * [x] [`_load_mask64`] + * [x] [`_mm512_madd_epi16`] + * [x] [`_mm512_mask_madd_epi16`] + * [x] [`_mm512_maskz_madd_epi16`] + * [x] [`_mm_mask_madd_epi16`] + * [x] [`_mm_maskz_madd_epi16`] + * [x] [`_mm256_mask_madd_epi16`] + * [x] [`_mm256_maskz_madd_epi16`] + * [x] [`_mm512_maddubs_epi16`] + * [x] [`_mm512_mask_maddubs_epi16`] + * [x] [`_mm512_maskz_maddubs_epi16`] + * [x] [`_mm_mask_maddubs_epi16`] + * [x] [`_mm_maskz_maddubs_epi16`] + * [x] [`_mm256_mask_maddubs_epi16`] + * [x] [`_mm256_maskz_maddubs_epi16`] + * [x] [`_mm512_mask_max_epi16`] + * [x] [`_mm512_maskz_max_epi16`] + * [x] [`_mm512_max_epi16`] + * [x] [`_mm_mask_max_epi16`] + * [x] [`_mm_maskz_max_epi16`] + * [x] [`_mm256_mask_max_epi16`] + * [x] [`_mm256_maskz_max_epi16`] + * [x] [`_mm512_mask_max_epi8`] + * [x] [`_mm512_maskz_max_epi8`] + * [x] [`_mm512_max_epi8`] + * [x] [`_mm_mask_max_epi8`] + * [x] [`_mm_maskz_max_epi8`] + * [x] [`_mm256_mask_max_epi8`] + * [x] [`_mm256_maskz_max_epi8`] + * [x] [`_mm512_mask_max_epu16`] + * [x] [`_mm512_maskz_max_epu16`] + * [x] [`_mm512_max_epu16`] + * [x] [`_mm_mask_max_epu16`] + * [x] [`_mm_maskz_max_epu16`] + * [x] [`_mm256_mask_max_epu16`] + * [x] [`_mm256_maskz_max_epu16`] + * [x] [`_mm512_mask_max_epu8`] + * [x] [`_mm512_maskz_max_epu8`] + * [x] [`_mm512_max_epu8`] + * [x] [`_mm_mask_max_epu8`] + * [x] [`_mm_maskz_max_epu8`] + * [x] [`_mm256_mask_max_epu8`] + * [x] [`_mm256_maskz_max_epu8`] + * [x] [`_mm512_mask_min_epi16`] + * [x] [`_mm512_maskz_min_epi16`] + * [x] [`_mm512_min_epi16`] + * [x] [`_mm_mask_min_epi16`] + * [x] [`_mm_maskz_min_epi16`] + * [x] [`_mm256_mask_min_epi16`] + * [x] [`_mm256_maskz_min_epi16`] + * [x] [`_mm512_mask_min_epi8`] + * [x] [`_mm512_maskz_min_epi8`] + * [x] [`_mm512_min_epi8`] + * [x] [`_mm_mask_min_epi8`] + * [x] [`_mm_maskz_min_epi8`] + * [x] [`_mm256_mask_min_epi8`] + * [x] [`_mm256_maskz_min_epi8`] + * [x] [`_mm512_mask_min_epu16`] + * [x] [`_mm512_maskz_min_epu16`] + * [x] [`_mm512_min_epu16`] + * [x] [`_mm_mask_min_epu16`] + * [x] [`_mm_maskz_min_epu16`] + * [x] [`_mm256_mask_min_epu16`] + * [x] [`_mm256_maskz_min_epu16`] + * [x] [`_mm512_mask_min_epu8`] + * [x] [`_mm512_maskz_min_epu8`] + * [x] [`_mm512_min_epu8`] + * [x] [`_mm_mask_min_epu8`] + * [x] [`_mm_maskz_min_epu8`] + * [x] [`_mm256_mask_min_epu8`] + * [x] [`_mm256_maskz_min_epu8`] + * [x] [`_mm512_mask_mov_epi16`] + * [x] [`_mm512_maskz_mov_epi16`] + * [x] [`_mm_mask_mov_epi16`] + * [x] [`_mm_maskz_mov_epi16`] + * [x] [`_mm256_mask_mov_epi16`] + * [x] [`_mm256_maskz_mov_epi16`] + * [x] [`_mm512_mask_mov_epi8`] + * [x] [`_mm512_maskz_mov_epi8`] + * [x] [`_mm_mask_mov_epi8`] + * [x] [`_mm_maskz_mov_epi8`] + * [x] [`_mm256_mask_mov_epi8`] + * [x] [`_mm256_maskz_mov_epi8`] + * [x] [`_mm512_movepi16_mask`] + * [x] [`_mm_movepi16_mask`] + * [x] [`_mm256_movepi16_mask`] + * [x] [`_mm512_movepi8_mask`] + * [x] [`_mm_movepi8_mask`] + * [x] [`_mm256_movepi8_mask`] + * [x] [`_mm512_movm_epi16`] + * [x] [`_mm_movm_epi16`] + * [x] [`_mm256_movm_epi16`] + * [x] [`_mm512_movm_epi8`] + * [x] [`_mm_movm_epi8`] + * [x] [`_mm256_movm_epi8`] + * [x] [`_mm512_mask_mulhi_epi16`] + * [x] [`_mm512_maskz_mulhi_epi16`] + * [x] [`_mm512_mulhi_epi16`] + * [x] [`_mm512_mask_mulhi_epu16`] + * [x] [`_mm512_maskz_mulhi_epu16`] + * [x] [`_mm_mask_mulhi_epi16`] + * [x] [`_mm_maskz_mulhi_epi16`] + * [x] [`_mm256_mask_mulhi_epi16`] + * [x] [`_mm256_maskz_mulhi_epi16`] + * [x] [`_mm512_mulhi_epu16`] + * [x] [`_mm_mask_mulhi_epu16`] + * [x] [`_mm_maskz_mulhi_epu16`] + * [x] [`_mm256_mask_mulhi_epu16`] + * [x] [`_mm256_maskz_mulhi_epu16`] + * [x] [`_mm512_mask_mulhrs_epi16`] + * [x] [`_mm512_maskz_mulhrs_epi16`] + * [x] [`_mm512_mulhrs_epi16`] + * [x] [`_mm_mask_mulhrs_epi16`] + * [x] [`_mm_maskz_mulhrs_epi16`] + * [x] [`_mm256_mask_mulhrs_epi16`] + * [x] [`_mm256_maskz_mulhrs_epi16`] + * [x] [`_mm512_mask_mullo_epi16`] + * [x] [`_mm512_maskz_mullo_epi16`] + * [x] [`_mm512_mullo_epi16`] + * [x] [`_mm_mask_mullo_epi16`] + * [x] [`_mm_maskz_mullo_epi16`] + * [x] [`_mm256_mask_mullo_epi16`] + * [x] [`_mm256_maskz_mullo_epi16`] + * [x] [`_mm512_mask_packs_epi16`] + * [x] [`_mm512_maskz_packs_epi16`] + * [x] [`_mm512_packs_epi16`] + * [x] [`_mm_mask_packs_epi16`] + * [x] [`_mm_maskz_packs_epi16`] + * [x] [`_mm256_mask_packs_epi16`] + * [x] [`_mm256_maskz_packs_epi16`] + * [x] [`_mm512_mask_packs_epi32`] + * [x] [`_mm512_maskz_packs_epi32`] + * [x] [`_mm512_packs_epi32`] + * [x] [`_mm_mask_packs_epi32`] + * [x] [`_mm_maskz_packs_epi32`] + * [x] [`_mm256_mask_packs_epi32`] + * [x] [`_mm256_maskz_packs_epi32`] + * [x] [`_mm512_mask_packus_epi16`] + * [x] [`_mm512_maskz_packus_epi16`] + * [x] [`_mm512_packus_epi16`] + * [x] [`_mm_mask_packus_epi16`] + * [x] [`_mm_maskz_packus_epi16`] + * [x] [`_mm256_mask_packus_epi16`] + * [x] [`_mm256_maskz_packus_epi16`] + * [x] [`_mm512_mask_packus_epi32`] + * [x] [`_mm512_maskz_packus_epi32`] + * [x] [`_mm512_packus_epi32`] + * [x] [`_mm_mask_packus_epi32`] + * [x] [`_mm_maskz_packus_epi32`] + * [x] [`_mm256_mask_packus_epi32`] + * [x] [`_mm256_maskz_packus_epi32`] + * [x] [`_mm512_mask_permutex2var_epi16`] + * [x] [`_mm512_mask2_permutex2var_epi16`] + * [x] [`_mm512_maskz_permutex2var_epi16`] + * [x] [`_mm512_permutex2var_epi16`] + * [x] [`_mm_mask_permutex2var_epi16`] + * [x] [`_mm_mask2_permutex2var_epi16`] + * [x] [`_mm_maskz_permutex2var_epi16`] + * [x] [`_mm_permutex2var_epi16`] + * [x] [`_mm256_mask_permutex2var_epi16`] + * [x] [`_mm256_mask2_permutex2var_epi16`] + * [x] [`_mm256_maskz_permutex2var_epi16`] + * [x] [`_mm256_permutex2var_epi16`] + * [x] [`_mm512_mask_permutexvar_epi16`] + * [x] [`_mm512_maskz_permutexvar_epi16`] + * [x] [`_mm512_permutexvar_epi16`] + * [x] [`_mm_mask_permutexvar_epi16`] + * [x] [`_mm_maskz_permutexvar_epi16`] + * [x] [`_mm_permutexvar_epi16`] + * [x] [`_mm256_mask_permutexvar_epi16`] + * [x] [`_mm256_maskz_permutexvar_epi16`] + * [x] [`_mm256_permutexvar_epi16`] + * [x] [`_mm512_sad_epu8`] + * [x] [`_mm512_mask_set1_epi16`] + * [x] [`_mm512_maskz_set1_epi16`] + * [x] [`_mm_mask_set1_epi16`] + * [x] [`_mm_maskz_set1_epi16`] + * [x] [`_mm256_mask_set1_epi16`] + * [x] [`_mm256_maskz_set1_epi16`] + * [x] [`_mm512_mask_set1_epi8`] + * [x] [`_mm512_maskz_set1_epi8`] + * [x] [`_mm_mask_set1_epi8`] + * [x] [`_mm_maskz_set1_epi8`] + * [x] [`_mm256_mask_set1_epi8`] + * [x] [`_mm256_maskz_set1_epi8`] + * [x] [`_mm512_mask_shuffle_epi8`] + * [x] [`_mm512_maskz_shuffle_epi8`] + * [x] [`_mm512_shuffle_epi8`] + * [x] [`_mm_mask_shuffle_epi8`] + * [x] [`_mm_maskz_shuffle_epi8`] + * [x] [`_mm256_mask_shuffle_epi8`] + * [x] [`_mm256_maskz_shuffle_epi8`] + * [x] [`_mm512_mask_shufflehi_epi16`] + * [x] [`_mm512_maskz_shufflehi_epi16`] + * [x] [`_mm512_shufflehi_epi16`] + * [x] [`_mm_mask_shufflehi_epi16`] + * [x] [`_mm_maskz_shufflehi_epi16`] + * [x] [`_mm256_mask_shufflehi_epi16`] + * [x] [`_mm256_maskz_shufflehi_epi16`] + * [x] [`_mm512_mask_shufflelo_epi16`] + * [x] [`_mm512_maskz_shufflelo_epi16`] + * [x] [`_mm512_shufflelo_epi16`] + * [x] [`_mm_mask_shufflelo_epi16`] + * [x] [`_mm_maskz_shufflelo_epi16`] + * [x] [`_mm256_mask_shufflelo_epi16`] + * [x] [`_mm256_maskz_shufflelo_epi16`] + * [x] [`_mm512_mask_sll_epi16`] + * [x] [`_mm512_maskz_sll_epi16`] + * [x] [`_mm512_sll_epi16`] + * [x] [`_mm_mask_sll_epi16`] + * [x] [`_mm_maskz_sll_epi16`] + * [x] [`_mm256_mask_sll_epi16`] + * [x] [`_mm256_maskz_sll_epi16`] + * [x] [`_mm512_mask_slli_epi16`] + * [x] [`_mm512_maskz_slli_epi16`] + * [x] [`_mm512_slli_epi16`] + * [x] [`_mm_mask_slli_epi16 + * [x] [`_mm_maskz_slli_epi16 + * [x] [`_mm256_mask_slli_epi16 + * [x] [`_mm256_maskz_slli_epi16 + * [x] [`_mm512_mask_sllv_epi16`] + * [x] [`_mm512_maskz_sllv_epi16`] + * [x] [`_mm512_sllv_epi16`] + * [x] [`_mm_mask_sllv_epi16`] + * [x] [`_mm_maskz_sllv_epi16`] + * [x] [`_mm_sllv_epi16`] + * [x] [`_mm256_mask_sllv_epi16`] + * [x] [`_mm256_maskz_sllv_epi16`] + * [x] [`_mm256_sllv_epi16`] + * [x] [`_mm512_mask_sra_epi16`] + * [x] [`_mm512_maskz_sra_epi16`] + * [x] [`_mm512_sra_epi16`] + * [x] [`_mm_mask_sra_epi16`] + * [x] [`_mm_maskz_sra_epi16`] + * [x] [`_mm256_mask_sra_epi16`] + * [x] [`_mm256_maskz_sra_epi16`] + * [x] [`_mm512_mask_srai_epi16`] + * [x] [`_mm512_maskz_srai_epi16`] + * [x] [`_mm512_srai_epi16`] + * [x] [`_mm_mask_srai_epi16`] + * [x] [`_mm_maskz_srai_epi16`] + * [x] [`_mm256_mask_srai_epi16`] + * [x] [`_mm256_maskz_srai_epi16`] + * [x] [`_mm512_mask_srav_epi16`] + * [x] [`_mm512_maskz_srav_epi16`] + * [x] [`_mm512_srav_epi16`] + * [x] [`_mm_mask_srav_epi16`] + * [x] [`_mm_maskz_srav_epi16`] + * [x] [`_mm_srav_epi16`] + * [x] [`_mm256_mask_srav_epi16`] + * [x] [`_mm256_maskz_srav_epi16`] + * [x] [`_mm256_srav_epi16`] + * [x] [`_mm512_mask_srl_epi16`] + * [x] [`_mm512_maskz_srl_epi16`] + * [x] [`_mm512_srl_epi16`] + * [x] [`_mm_mask_srl_epi16`] + * [x] [`_mm_maskz_srl_epi16`] + * [x] [`_mm256_mask_srl_epi16`] + * [x] [`_mm256_maskz_srl_epi16`] + * [x] [`_mm512_mask_srli_epi16`] + * [x] [`_mm512_maskz_srli_epi16`] + * [x] [`_mm512_srli_epi16`] + * [x] [`_mm_mask_srli_epi16`] + * [x] [`_mm_maskz_srli_epi16`] + * [x] [`_mm256_mask_srli_epi16`] + * [x] [`_mm256_maskz_srli_epi16`] + * [x] [`_mm512_mask_srlv_epi16`] + * [x] [`_mm512_maskz_srlv_epi16`] + * [x] [`_mm512_srlv_epi16`] + * [x] [`_mm_mask_srlv_epi16`] + * [x] [`_mm_maskz_srlv_epi16`] + * [x] [`_mm_srlv_epi16`] + * [x] [`_mm256_mask_srlv_epi16`] + * [x] [`_mm256_maskz_srlv_epi16`] + * [x] [`_mm256_srlv_epi16`] + * [x] [`_store_mask32`] + * [x] [`_store_mask64`] + * [x] [`_mm512_mask_sub_epi16`] + * [x] [`_mm512_maskz_sub_epi16`] + * [x] [`_mm512_sub_epi16`] + * [x] [`_mm_mask_sub_epi16`] + * [x] [`_mm_maskz_sub_epi16`] + * [x] [`_mm256_mask_sub_epi16`] + * [x] [`_mm256_maskz_sub_epi16`] + * [x] [`_mm512_mask_sub_epi8`] + * [x] [`_mm512_maskz_sub_epi8`] + * [x] [`_mm_mask_sub_epi8`] + * [x] [`_mm_maskz_sub_epi8`] + * [x] [`_mm256_mask_sub_epi8`] + * [x] [`_mm256_maskz_sub_epi8`] + * [x] [`_mm512_sub_epi8`] + * [x] [`_mm512_mask_subs_epi16`] + * [x] [`_mm512_maskz_subs_epi16`] + * [x] [`_mm512_subs_epi16`] + * [x] [`_mm_mask_subs_epi16`] + * [x] [`_mm_maskz_subs_epi16`] + * [x] [`_mm256_mask_subs_epi16`] + * [x] [`_mm256_maskz_subs_epi16`] + * [x] [`_mm512_mask_subs_epi8`] + * [x] [`_mm512_maskz_subs_epi8`] + * [x] [`_mm512_subs_epi8`] + * [x] [`_mm_mask_subs_epi8`] + * [x] [`_mm_maskz_subs_epi8`] + * [x] [`_mm256_mask_subs_epi8`] + * [x] [`_mm256_maskz_subs_epi8`] + * [x] [`_mm512_mask_subs_epu16`] + * [x] [`_mm512_maskz_subs_epu16`] + * [x] [`_mm512_subs_epu16`] + * [x] [`_mm_mask_subs_epu16`] + * [x] [`_mm_maskz_subs_epu16`] + * [x] [`_mm256_mask_subs_epu16`] + * [x] [`_mm256_maskz_subs_epu16`] + * [x] [`_mm512_mask_subs_epu8`] + * [x] [`_mm512_maskz_subs_epu8`] + * [x] [`_mm512_subs_epu8`] + * [x] [`_mm_mask_subs_epu8`] + * [x] [`_mm_maskz_subs_epu8`] + * [x] [`_mm256_mask_subs_epu8`] + * [x] [`_mm256_maskz_subs_epu8`] + * [x] [`_mm512_mask_test_epi16_mask`] + * [x] [`_mm512_test_epi16_mask`] + * [x] [`_mm_mask_test_epi16_mask`] + * [x] [`_mm_test_epi16_mask`] + * [x] [`_mm256_mask_test_epi16_mask`] + * [x] [`_mm256_test_epi16_mask`] + * [x] [`_mm512_mask_test_epi8_mask`] + * [x] [`_mm512_test_epi8_mask`] + * [x] [`_mm_mask_test_epi8_mask`] + * [x] [`_mm_test_epi8_mask`] + * [x] [`_mm256_mask_test_epi8_mask`] + * [x] [`_mm256_test_epi8_mask`] + * [x] [`_mm512_mask_testn_epi16_mask`] + * [x] [`_mm512_testn_epi16_mask`] + * [x] [`_mm_mask_testn_epi16_mask`] + * [x] [`_mm_testn_epi16_mask`] + * [x] [`_mm256_mask_testn_epi16_mask`] + * [x] [`_mm256_testn_epi16_mask`] + * [x] [`_mm512_mask_testn_epi8_mask`] + * [x] [`_mm512_testn_epi8_mask`] + * [x] [`_mm_mask_testn_epi8_mask`] + * [x] [`_mm_testn_epi8_mask`] + * [x] [`_mm256_mask_testn_epi8_mask`] + * [x] [`_mm256_testn_epi8_mask`] + * [x] [`_mm512_mask_unpackhi_epi16`] + * [x] [`_mm512_maskz_unpackhi_epi16`] + * [x] [`_mm512_unpackhi_epi16`] + * [x] [`_mm_mask_unpackhi_epi16`] + * [x] [`_mm_maskz_unpackhi_epi16`] + * [x] [`_mm256_mask_unpackhi_epi16`] + * [x] [`_mm256_maskz_unpackhi_epi16`] + * [x] [`_mm512_mask_unpackhi_epi8`] + * [x] [`_mm512_maskz_unpackhi_epi8`] + * [x] [`_mm512_unpackhi_epi8`] + * [x] [`_mm_mask_unpackhi_epi8`] + * [x] [`_mm_maskz_unpackhi_epi8`] + * [x] [`_mm256_mask_unpackhi_epi8`] + * [x] [`_mm256_maskz_unpackhi_epi8`] + * [x] [`_mm512_mask_unpacklo_epi16`] + * [x] [`_mm512_maskz_unpacklo_epi16`] + * [x] [`_mm512_unpacklo_epi16`] + * [x] [`_mm_mask_unpacklo_epi16`] + * [x] [`_mm_maskz_unpacklo_epi16`] + * [x] [`_mm256_mask_unpacklo_epi16`] + * [x] [`_mm256_maskz_unpacklo_epi16`] + * [x] [`_mm512_mask_unpacklo_epi8`] + * [x] [`_mm512_maskz_unpacklo_epi8`] + * [x] [`_mm512_unpacklo_epi8`] + * [x] [`_mm_mask_unpacklo_epi8`] + * [x] [`_mm_maskz_unpacklo_epi8`] + * [x] [`_mm256_mask_unpacklo_epi8`] + * [x] [`_mm256_maskz_unpacklo_epi8`] + +

diff --git a/crux-mir/lib/stdarch/crates/core_arch/avx512f.md b/crux-mir/lib/stdarch/crates/core_arch/avx512f.md new file mode 100644 index 000000000..6cb6e6564 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/avx512f.md @@ -0,0 +1,2633 @@ +["AVX512F"]

+ + * [x] [`_mm512_abs_epi32`] + * [x] [`_mm512_mask_abs_epi32`] + * [x] [`_mm512_maskz_abs_epi32`] + * [x] [`_mm_mask_abs_epi32`] + * [x] [`_mm_maskz_abs_epi32`] + * [x] [`_mm256_mask_abs_epi32`] + * [x] [`_mm256_maskz_abs_epi32`] + * [x] [`_mm512_abs_epi64`] + * [x] [`_mm512_mask_abs_epi64`] + * [x] [`_mm512_maskz_abs_epi64`] + * [x] [`_mm_abs_epi64`] + * [x] [`_mm_mask_abs_epi64`] + * [x] [`_mm_maskz_abs_epi64`] + * [x] [`_mm256_abs_epi64`] + * [x] [`_mm256_mask_abs_epi64`] + * [x] [`_mm256_maskz_abs_epi64`] + * [x] [`_mm512_abs_pd`] + * [x] [`_mm512_mask_abs_pd`] + * [x] [`_mm512_abs_ps`] + * [x] [`_mm512_mask_abs_ps`] + * [x] [`_mm512_add_epi32`] + * [x] [`_mm512_mask_add_epi32`] + * [x] [`_mm512_maskz_add_epi32`] + * [x] [`_mm_mask_add_epi32`] + * [x] [`_mm_maskz_add_epi32`] + * [x] [`_mm256_mask_add_epi32`] + * [x] [`_mm256_maskz_add_epi32`] + * [x] [`_mm512_add_epi64`] + * [x] [`_mm512_mask_add_epi64`] + * [x] [`_mm512_maskz_add_epi64`] + * [x] [`_mm_mask_add_epi64`] + * [x] [`_mm_maskz_add_epi64`] + * [x] [`_mm256_mask_add_epi64`] + * [x] [`_mm256_maskz_add_epi64`] + * [x] [`_mm512_add_ps`] + * [x] [`_mm512_mask_add_ps`] + * [x] [`_mm512_maskz_add_ps`] + * [x] [`_mm_mask_add_ps`] + * [x] [`_mm_maskz_add_ps`] + * [x] [`_mm256_mask_add_ps`] + * [x] [`_mm256_maskz_add_ps`] + * [x] [`_mm512_add_pd`] + * [x] [`_mm512_mask_add_pd`] + * [x] [`_mm512_maskz_add_pd`] + * [x] [`_mm_mask_add_pd`] + * [x] [`_mm_maskz_add_pd`] + * [x] [`_mm256_mask_add_pd`] + * [x] [`_mm256_maskz_add_pd`] + * [x] [`_mm512_add_round_ps`] + * [x] [`_mm512_mask_add_round_ps`] + * [x] [`_mm512_maskz_add_round_ps`] + * [x] [`_mm512_add_round_pd`] + * [x] [`_mm512_mask_add_round_pd`] + * [x] [`_mm512_maskz_add_round_pd`] + * [x] [`_mm512_sub_epi32`] + * [x] [`_mm512_mask_sub_epi32`] + * [x] [`_mm512_maskz_sub_epi32`] + * [x] [`_mm_mask_sub_epi32`] + * [x] [`_mm_maskz_sub_epi32`] + * [x] [`_mm256_mask_sub_epi32`] + * [x] [`_mm256_maskz_sub_epi32`] + * [x] [`_mm512_sub_epi64`] + * [x] [`_mm512_mask_sub_epi64`] + * [x] [`_mm512_maskz_sub_epi64`] + * [x] [`_mm_mask_sub_epi64`] + * [x] [`_mm_maskz_sub_epi64`] + * [x] [`_mm256_mask_sub_epi64`] + * [x] [`_mm256_maskz_sub_epi64`] + * [x] [`_mm512_sub_ps`] + * [x] [`_mm512_mask_sub_ps`] + * [x] [`_mm512_maskz_sub_ps`] + * [x] [`_mm_mask_sub_ps`] + * [x] [`_mm_maskz_sub_ps`] + * [x] [`_mm256_mask_sub_ps`] + * [x] [`_mm256_maskz_sub_ps`] + * [x] [`_mm512_sub_pd`] + * [x] [`_mm512_mask_sub_pd`] + * [x] [`_mm512_maskz_sub_pd`] + * [x] [`_mm_mask_sub_pd`] + * [x] [`_mm_maskz_sub_pd`] + * [x] [`_mm256_mask_sub_pd`] + * [x] [`_mm256_maskz_sub_pd`] + * [x] [`_mm512_sub_round_ps`] + * [x] [`_mm512_mask_sub_round_ps`] + * [x] [`_mm512_maskz_sub_round_ps`] + * [x] [`_mm512_sub_round_pd`] + * [x] [`_mm512_mask_sub_round_pd`] + * [x] [`_mm512_maskz_sub_round_pd`] + * [x] [`_mm512_mul_epi32`] + * [x] [`_mm512_mask_mul_epi32`] + * [x] [`_mm512_maskz_mul_epi32`] + * [x] [`_mm_mask_mul_epi32`] + * [x] [`_mm_maskz_mul_epi32`] + * [x] [`_mm256_mask_mul_epi32`] + * [x] [`_mm256_maskz_mul_epi32`] + * [x] [`_mm512_mul_epu32`] + * [x] [`_mm512_mask_mul_epu32`] + * [x] [`_mm512_maskz_mul_epu32`] + * [x] [`_mm_mask_mul_epu32`] + * [x] [`_mm_maskz_mul_epu32`] + * [x] [`_mm256_mask_mul_epu32`] + * [x] [`_mm256_maskz_mul_epu32`] + * [x] [`_mm512_mul_ps`] + * [x] [`_mm512_mask_mul_ps`] + * [x] [`_mm512_maskz_mul_ps`] + * [x] [`_mm_mask_mul_ps`] + * [x] [`_mm_maskz_mul_ps`] + * [x] [`_mm256_mask_mul_ps`] + * [x] [`_mm256_maskz_mul_ps`] + * [x] [`_mm512_mul_pd`] + * [x] [`_mm512_mask_mul_pd`] + * [x] [`_mm512_maskz_mul_pd`] + * [x] [`_mm_mask_mul_pd`] + * [x] [`_mm_maskz_mul_pd`] + * [x] [`_mm256_mask_mul_pd`] + * [x] [`_mm256_maskz_mul_pd`] + * [x] [`_mm512_mul_round_ps`] + * [x] [`_mm512_mask_mul_round_ps`] + * [x] [`_mm512_maskz_mul_round_ps`] + * [x] [`_mm512_mul_round_pd`] + * [x] [`_mm512_mask_mul_round_pd`] + * [x] [`_mm512_maskz_mul_round_pd`] + * [x] [`_mm512_mullo_epi32`] + * [x] [`_mm512_mask_mullo_epi32`] + * [x] [`_mm512_maskz_mullo_epi32`] + * [x] [`_mm_mask_mullo_epi32`] + * [x] [`_mm_maskz_mullo_epi32`] + * [x] [`_mm256_mask_mullo_epi32`] + * [x] [`_mm256_maskz_mullo_epi32`] + * [x] [`_mm512_mullox_epi64`] + * [x] [`_mm512_mask_mullox_epi64`] + * [x] [`_mm512_div_ps`] + * [x] [`_mm512_mask_div_ps`] + * [x] [`_mm512_maskz_div_ps`] + * [x] [`_mm_mask_div_ps`] + * [x] [`_mm_maskz_div_ps`] + * [x] [`_mm256_mask_div_ps`] + * [x] [`_mm256_maskz_div_ps`] + * [x] [`_mm512_div_pd`] + * [x] [`_mm512_mask_div_pd`] + * [x] [`_mm512_maskz_div_pd`] + * [x] [`_mm_mask_div_pd`] + * [x] [`_mm_maskz_div_pd`] + * [x] [`_mm256_mask_div_pd`] + * [x] [`_mm256_maskz_div_pd`] + * [x] [`_mm512_div_round_ps`] + * [x] [`_mm512_mask_div_round_ps`] + * [x] [`_mm512_maskz_div_round_ps`] + * [x] [`_mm512_div_round_pd`] + * [x] [`_mm512_mask_div_round_pd`] + * [x] [`_mm512_maskz_div_round_pd`] + * [x] [`_mm512_max_epi32`] + * [x] [`_mm512_mask_max_epi32`] + * [x] [`_mm512_maskz_max_epi32`] + * [x] [`_mm_mask_max_epi32`] + * [x] [`_mm_maskz_max_epi32`] + * [x] [`_mm256_mask_max_epi32`] + * [x] [`_mm256_maskz_max_epi32`] + * [x] [`_mm512_max_epu32`] + * [x] [`_mm512_mask_max_epu32`] + * [x] [`_mm512_maskz_max_epu32`] + * [x] [`_mm_mask_max_epu32`] + * [x] [`_mm_maskz_max_epu32`] + * [x] [`_mm256_mask_max_epu32`] + * [x] [`_mm256_maskz_max_epu32`] + * [x] [`_mm512_max_epi64`] + * [x] [`_mm512_mask_max_epi64`] + * [x] [`_mm512_maskz_max_epi64`] + * [x] [`_mm_mask_max_epi64`] + * [x] [`_mm_maskz_max_epi64`] + * [x] [`_mm_max_epi64`] + * [x] [`_mm256_mask_max_epi64`] + * [x] [`_mm256_maskz_max_epi64`] + * [x] [`_mm256_max_epi64`] + * [x] [`_mm512_max_epu64`] + * [x] [`_mm512_mask_max_epu64`] + * [x] [`_mm512_maskz_max_epu64`] + * [x] [`_mm_mask_max_epu64`] + * [x] [`_mm_maskz_max_epu64`] + * [x] [`_mm_max_epu64`] + * [x] [`_mm256_mask_max_epu64`] + * [x] [`_mm256_maskz_max_epu64`] + * [x] [`_mm256_max_epu64`] + * [x] [`_mm512_max_ps`] + * [x] [`_mm512_mask_max_ps`] + * [x] [`_mm512_maskz_max_ps`] + * [x] [`_mm_mask_max_ps`] + * [x] [`_mm_maskz_max_ps`] + * [x] [`_mm256_mask_max_ps`] + * [x] [`_mm256_maskz_max_ps`] + * [x] [`_mm512_max_pd`] + * [x] [`_mm512_mask_max_pd`] + * [x] [`_mm512_maskz_max_pd`] + * [x] [`_mm_mask_max_pd`] + * [x] [`_mm_maskz_max_pd`] + * [x] [`_mm256_mask_max_pd`] + * [x] [`_mm256_maskz_max_pd`] + * [x] [`_mm512_max_round_ps`] + * [x] [`_mm512_mask_max_round_ps`] + * [x] [`_mm512_maskz_max_round_ps`] + * [x] [`_mm512_max_round_pd`] + * [x] [`_mm512_mask_max_round_pd`] + * [x] [`_mm512_maskz_max_round_pd`] + * [x] [`_mm512_min_epi32`] + * [x] [`_mm512_mask_min_epi32`] + * [x] [`_mm512_maskz_min_epi32`] + * [x] [`_mm_mask_min_epi32`] + * [x] [`_mm_maskz_min_epi32`] + * [x] [`_mm256_mask_min_epi32`] + * [x] [`_mm256_maskz_min_epi32`] + * [x] [`_mm512_min_epi64`] + * [x] [`_mm512_mask_min_epi64`] + * [x] [`_mm512_maskz_min_epi64`] + * [x] [`_mm_mask_min_epi64`] + * [x] [`_mm_maskz_min_epi64`] + * [x] [`_mm_min_epi64`] + * [x] [`_mm256_mask_min_epi64`] + * [x] [`_mm256_maskz_min_epi64`] + * [x] [`_mm256_min_epi64`] + * [x] [`_mm512_min_epu32`] + * [x] [`_mm512_mask_min_epu32`] + * [x] [`_mm512_maskz_min_epu32`] + * [x] [`_mm_mask_min_epu32`] + * [x] [`_mm_maskz_min_epu32`] + * [x] [`_mm256_mask_min_epu32`] + * [x] [`_mm256_maskz_min_epu32`] + * [x] [`_mm512_min_epu64`] + * [x] [`_mm512_mask_min_epu64`] + * [x] [`_mm512_maskz_min_epu64`] + * [x] [`_mm_mask_min_epu64`] + * [x] [`_mm_maskz_min_epu64`] + * [x] [`_mm_min_epu64`] + * [x] [`_mm256_mask_min_epu64`] + * [x] [`_mm256_maskz_min_epu64`] + * [x] [`_mm256_min_epu64`] + * [x] [`_mm512_min_ps`] + * [x] [`_mm512_mask_min_ps`] + * [x] [`_mm512_maskz_min_ps`] + * [x] [`_mm_mask_min_ps`] + * [x] [`_mm_maskz_min_ps`] + * [x] [`_mm256_mask_min_ps`] + * [x] [`_mm256_maskz_min_ps`] + * [x] [`_mm512_min_pd`] + * [x] [`_mm512_mask_min_pd`] + * [x] [`_mm512_maskz_min_pd`] + * [x] [`_mm_mask_min_pd`] + * [x] [`_mm_maskz_min_pd`] + * [x] [`_mm256_mask_min_pd`] + * [x] [`_mm256_maskz_min_pd`] + * [x] [`_mm512_min_round_ps`] + * [x] [`_mm512_mask_min_round_ps`] + * [x] [`_mm512_maskz_min_round_ps`] + * [x] [`_mm512_min_round_pd`] + * [x] [`_mm512_mask_min_round_pd`] + * [x] [`_mm512_maskz_min_round_pd`] + * [x] [`_mm512_sqrt_ps`] + * [x] [`_mm512_mask_sqrt_ps`] + * [x] [`_mm512_maskz_sqrt_ps`] + * [x] [`_mm_mask_sqrt_ps`] + * [x] [`_mm_maskz_sqrt_ps`] + * [x] [`_mm256_mask_sqrt_ps`] + * [x] [`_mm256_maskz_sqrt_ps`] + * [x] [`_mm512_sqrt_pd`] + * [x] [`_mm512_mask_sqrt_pd`] + * [x] [`_mm512_maskz_sqrt_pd`] + * [x] [`_mm_mask_sqrt_pd`] + * [x] [`_mm_maskz_sqrt_pd`] + * [x] [`_mm256_mask_sqrt_pd`] + * [x] [`_mm256_maskz_sqrt_pd`] + * [x] [`_mm512_sqrt_round_ps`] + * [x] [`_mm512_mask_sqrt_round_ps`] + * [x] [`_mm512_maskz_sqrt_round_ps`] + * [x] [`_mm512_sqrt_round_pd`] + * [x] [`_mm512_mask_sqrt_round_pd`] + * [x] [`_mm512_maskz_sqrt_round_pd`] + * [x] [`_mm512_rsqrt14_ps`] + * [x] [`_mm512_mask_rsqrt14_ps`] + * [x] [`_mm512_maskz_rsqrt14_ps`] + * [x] [`_mm_mask_rsqrt14_ps`] + * [x] [`_mm_maskz_rsqrt14_ps`] + * [x] [`_mm256_mask_rsqrt14_ps`] + * [x] [`_mm256_maskz_rsqrt14_ps`] + * [x] [`_mm512_rsqrt14_pd`] + * [x] [`_mm512_mask_rsqrt14_pd`] + * [x] [`_mm512_maskz_rsqrt14_pd`] + * [x] [`_mm_mask_rsqrt14_pd`] + * [x] [`_mm_maskz_rsqrt14_pd`] + * [x] [`_mm256_mask_rsqrt14_pd`] + * [x] [`_mm256_maskz_rsqrt14_pd`] + * [x] [`_mm512_rcp14_ps`] + * [x] [`_mm512_mask_rcp14_ps`] + * [x] [`_mm512_maskz_rcp14_ps`] + * [x] [`_mm_mask_rcp14_ps`] + * [x] [`_mm_maskz_rcp14_ps`] + * [x] [`_mm_rcp14_ps`] + * [x] [`_mm256_mask_rcp14_ps`] + * [x] [`_mm256_maskz_rcp14_ps`] + * [x] [`_mm256_rcp14_ps`] + * [x] [`_mm512_rcp14_pd`] + * [x] [`_mm512_mask_rcp14_pd`] + * [x] [`_mm512_maskz_rcp14_pd`] + * [x] [`_mm_mask_rcp14_pd`] + * [x] [`_mm_maskz_rcp14_pd`] + * [x] [`_mm_rcp14_pd`] + * [x] [`_mm256_mask_rcp14_pd`] + * [x] [`_mm256_maskz_rcp14_pd`] + * [x] [`_mm256_rcp14_pd`] + * [x] [`_mm512_getexp_ps`] + * [x] [`_mm512_mask_getexp_ps`] + * [x] [`_mm512_maskz_getexp_ps`] + * [x] [`_mm_getexp_ps`] + * [x] [`_mm_mask_getexp_ps`] + * [x] [`_mm_maskz_getexp_ps`] + * [x] [`_mm256_getexp_ps`] + * [x] [`_mm256_mask_getexp_ps`] + * [x] [`_mm256_maskz_getexp_ps`] + * [x] [`_mm512_getexp_pd`] + * [x] [`_mm512_mask_getexp_pd`] + * [x] [`_mm512_maskz_getexp_pd`] + * [x] [`_mm_getexp_pd`] + * [x] [`_mm_mask_getexp_pd`] + * [x] [`_mm_maskz_getexp_pd`] + * [x] [`_mm256_getexp_pd`] + * [x] [`_mm256_mask_getexp_pd`] + * [x] [`_mm256_maskz_getexp_pd`] + * [x] [`_mm512_getexp_round_ps`] + * [x] [`_mm512_mask_getexp_round_ps`] + * [x] [`_mm512_maskz_getexp_round_ps`] + * [x] [`_mm512_getexp_round_pd`] + * [x] [`_mm512_mask_getexp_round_pd`] + * [x] [`_mm512_maskz_getexp_round_pd`] + * [x] [`_mm512_getmant_ps`] + * [x] [`_mm512_mask_getmant_ps`] + * [x] [`_mm512_maskz_getmant_ps`] + * [x] [`_mm_getmant_ps`] + * [x] [`_mm_mask_getmant_ps`] + * [x] [`_mm_maskz_getmant_ps`] + * [x] [`_mm256_getmant_ps`] + * [x] [`_mm256_mask_getmant_ps`] + * [x] [`_mm256_maskz_getmant_ps`] + * [x] [`_mm512_getmant_pd`] + * [x] [`_mm512_mask_getmant_pd`] + * [x] [`_mm512_maskz_getmant_pd`] + * [x] [`_mm_getmant_pd`] + * [x] [`_mm_mask_getmant_pd`] + * [x] [`_mm_maskz_getmant_pd`] + * [x] [`_mm256_getmant_pd`] + * [x] [`_mm256_mask_getmant_pd`] + * [x] [`_mm256_maskz_getmant_pd`] + * [x] [`_mm512_getmant_round_ps`] + * [x] [`_mm512_mask_getmant_round_ps`] + * [x] [`_mm512_maskz_getmant_round_ps`] + * [x] [`_mm512_getmant_round_pd`] + * [x] [`_mm512_mask_getmant_round_pd`] + * [x] [`_mm512_maskz_getmant_round_pd`] + * [x] [`_mm512_roundscale_ps`] + * [x] [`_mm512_mask_roundscale_ps`] + * [x] [`_mm512_maskz_roundscale_ps`] + * [x] [`_mm_mask_roundscale_ps`] + * [x] [`_mm_maskz_roundscale_ps`] + * [x] [`_mm_roundscale_ps`] + * [x] [`_mm256_mask_roundscale_ps`] + * [x] [`_mm256_maskz_roundscale_ps`] + * [x] [`_mm256_roundscale_ps`] + * [x] [`_mm512_roundscale_pd`] + * [x] [`_mm512_mask_roundscale_pd`] + * [x] [`_mm512_maskz_roundscale_pd`] + * [x] [`_mm_mask_roundscale_pd`] + * [x] [`_mm_maskz_roundscale_pd`] + * [x] [`_mm_roundscale_pd`] + * [x] [`_mm256_mask_roundscale_pd`] + * [x] [`_mm256_maskz_roundscale_pd`] + * [x] [`_mm256_roundscale_pd`] + * [x] [`_mm512_roundscale_round_ps`] + * [x] [`_mm512_mask_roundscale_round_ps`] + * [x] [`_mm512_maskz_roundscale_round_ps`] + * [x] [`_mm512_roundscale_round_pd`] + * [x] [`_mm512_mask_roundscale_round_pd`] + * [x] [`_mm512_maskz_roundscale_round_pd`] + * [x] [`_mm512_scalef_ps`] + * [x] [`_mm512_mask_scalef_ps`] + * [x] [`_mm512_maskz_scalef_ps`] + * [x] [`_mm_mask_scalef_ps`] + * [x] [`_mm_maskz_scalef_ps`] + * [x] [`_mm_scalef_ps`] + * [x] [`_mm256_mask_scalef_ps`] + * [x] [`_mm256_maskz_scalef_ps`] + * [x] [`_mm256_scalef_ps`] + * [x] [`_mm512_scalef_pd`] + * [x] [`_mm512_mask_scalef_pd`] + * [x] [`_mm512_maskz_scalef_pd`] + * [x] [`_mm_mask_scalef_pd`] + * [x] [`_mm_maskz_scalef_pd`] + * [x] [`_mm_scalef_pd`] + * [x] [`_mm256_mask_scalef_pd`] + * [x] [`_mm256_maskz_scalef_pd`] + * [x] [`_mm256_scalef_pd`] + * [x] [`_mm512_scalef_round_ps`] + * [x] [`_mm512_mask_scalef_round_ps`] + * [x] [`_mm512_maskz_scalef_round_ps`] + * [x] [`_mm512_scalef_round_pd`] + * [x] [`_mm512_mask_scalef_round_pd`] + * [x] [`_mm512_maskz_scalef_round_pd`] + * [x] [`_mm512_fixupimm_ps`] + * [x] [`_mm512_mask_fixupimm_ps`] + * [x] [`_mm512_maskz_fixupimm_ps`] + * [x] [`_mm_fixupimm_ps`] + * [x] [`_mm_mask_fixupimm_ps`] + * [x] [`_mm_maskz_fixupimm_ps`] + * [x] [`_mm256_fixupimm_ps`] + * [x] [`_mm256_mask_fixupimm_ps`] + * [x] [`_mm256_maskz_fixupimm_ps`] + * [x] [`_mm512_fixupimm_pd`] + * [x] [`_mm512_mask_fixupimm_pd`] + * [x] [`_mm512_maskz_fixupimm_pd`] + * [x] [`_mm_fixupimm_pd`] + * [x] [`_mm_mask_fixupimm_pd`] + * [x] [`_mm_maskz_fixupimm_pd`] + * [x] [`_mm256_fixupimm_pd`] + * [x] [`_mm256_mask_fixupimm_pd`] + * [x] [`_mm256_maskz_fixupimm_pd`] + * [x] [`_mm512_fixupimm_round_ps`] + * [x] [`_mm512_mask_fixupimm_round_ps`] + * [x] [`_mm512_maskz_fixupimm_round_ps`] + * [x] [`_mm512_fixupimm_round_pd`] + * [x] [`_mm512_mask_fixupimm_round_pd`] + * [x] [`_mm512_maskz_fixupimm_round_pd`] + * [x] [`_mm512_fmadd_ps`] + * [x] [`_mm512_mask_fmadd_ps`] + * [x] [`_mm512_maskz_fmadd_ps`] + * [x] [`_mm512_mask3_fmadd_ps`] + * [x] [`_mm_mask_fmadd_ps`] + * [x] [`_mm_mask3_fmadd_ps`] + * [x] [`_mm_maskz_fmadd_ps`] + * [x] [`_mm256_mask_fmadd_ps`] + * [x] [`_mm256_mask3_fmadd_ps`] + * [x] [`_mm256_maskz_fmadd_ps`] + * [x] [`_mm512_fmadd_pd`] + * [x] [`_mm512_mask_fmadd_pd`] + * [x] [`_mm512_maskz_fmadd_pd`] + * [x] [`_mm512_mask3_fmadd_pd`] + * [x] [`_mm_mask_fmadd_pd`] + * [x] [`_mm_mask3_fmadd_pd`] + * [x] [`_mm_maskz_fmadd_pd`] + * [x] [`_mm256_mask_fmadd_pd`] + * [x] [`_mm256_mask3_fmadd_pd`] + * [x] [`_mm256_maskz_fmadd_pd`] + * [x] [`_mm512_fmadd_round_ps`] + * [x] [`_mm512_mask_fmadd_round_ps`] + * [x] [`_mm512_maskz_fmadd_round_ps`] + * [x] [`_mm512_mask3_fmadd_round_ps`] + * [x] [`_mm512_fmadd_round_pd`] + * [x] [`_mm512_mask_fmadd_round_pd`] + * [x] [`_mm512_maskz_fmadd_round_pd`] + * [x] [`_mm512_mask3_fmadd_round_pd`] + * [x] [`_mm512_fmsub_ps`] + * [x] [`_mm512_mask_fmsub_ps`] + * [x] [`_mm512_maskz_fmsub_ps`] + * [x] [`_mm512_mask3_fmsub_ps`] + * [x] [`_mm_mask_fmsub_ps`] + * [x] [`_mm_mask3_fmsub_ps`] + * [x] [`_mm_maskz_fmsub_ps`] + * [x] [`_mm256_mask_fmsub_ps`] + * [x] [`_mm256_mask3_fmsub_ps`] + * [x] [`_mm256_maskz_fmsub_ps`] + * [x] [`_mm512_fmsub_pd`] + * [x] [`_mm512_mask_fmsub_pd`] + * [x] [`_mm512_maskz_fmsub_pd`] + * [x] [`_mm512_mask3_fmsub_pd`] + * [x] [`_mm_mask_fmsub_pd`] + * [x] [`_mm_mask3_fmsub_pd`] + * [x] [`_mm_maskz_fmsub_pd`] + * [x] [`_mm256_mask_fmsub_pd`] + * [x] [`_mm256_mask3_fmsub_pd`] + * [x] [`_mm256_maskz_fmsub_pd`] + * [x] [`_mm512_fmsub_round_ps`] + * [x] [`_mm512_mask_fmsub_round_ps`] + * [x] [`_mm512_maskz_fmsub_round_ps`] + * [x] [`_mm512_mask3_fmsub_round_ps`] + * [x] [`_mm512_fmsub_round_pd`] + * [x] [`_mm512_mask_fmsub_round_pd`] + * [x] [`_mm512_maskz_fmsub_round_pd`] + * [x] [`_mm512_mask3_fmsub_round_pd`] + * [x] [`_mm512_fmaddsub_ps`] + * [x] [`_mm512_mask_fmaddsub_ps`] + * [x] [`_mm512_maskz_fmaddsub_ps`] + * [x] [`_mm512_mask3_fmaddsub_ps`] + * [x] [`_mm_mask_fmaddsub_ps`] + * [x] [`_mm_mask3_fmaddsub_ps`] + * [x] [`_mm_maskz_fmaddsub_ps`] + * [x] [`_mm256_mask_fmaddsub_ps`] + * [x] [`_mm256_mask3_fmaddsub_ps`] + * [x] [`_mm256_maskz_fmaddsub_ps`] + * [x] [`_mm512_fmaddsub_pd`] + * [x] [`_mm512_mask_fmaddsub_pd`] + * [x] [`_mm512_maskz_fmaddsub_pd`] + * [x] [`_mm512_mask3_fmaddsub_pd`] + * [x] [`_mm_mask_fmaddsub_pd`] + * [x] [`_mm_mask3_fmaddsub_pd`] + * [x] [`_mm_maskz_fmaddsub_pd`] + * [x] [`_mm256_mask_fmaddsub_pd`] + * [x] [`_mm256_mask3_fmaddsub_pd`] + * [x] [`_mm256_maskz_fmaddsub_pd`] + * [x] [`_mm512_fmaddsub_round_ps`] + * [x] [`_mm512_mask_fmaddsub_round_ps`] + * [x] [`_mm512_maskz_fmaddsub_round_ps`] + * [x] [`_mm512_mask3_fmaddsub_round_ps`] + * [x] [`_mm512_fmaddsub_round_pd`] + * [x] [`_mm512_mask_fmaddsub_round_pd`] + * [x] [`_mm512_maskz_fmaddsub_round_pd`] + * [x] [`_mm512_mask3_fmaddsub_round_pd`] + * [x] [`_mm512_fmsubadd_ps`] + * [x] [`_mm512_mask_fmsubadd_ps`] + * [x] [`_mm512_maskz_fmsubadd_ps`] + * [x] [`_mm512_mask3_fmsubadd_ps`] + * [x] [`_mm_mask_fmsubadd_ps`] + * [x] [`_mm_mask3_fmsubadd_ps`] + * [x] [`_mm_maskz_fmsubadd_ps`] + * [x] [`_mm256_mask_fmsubadd_ps`] + * [x] [`_mm256_mask3_fmsubadd_ps`] + * [x] [`_mm256_maskz_fmsubadd_ps`] + * [x] [`_mm512_fmsubadd_pd`] + * [x] [`_mm512_mask_fmsubadd_pd`] + * [x] [`_mm512_maskz_fmsubadd_pd`] + * [x] [`_mm512_mask3_fmsubadd_pd`] + * [x] [`_mm_mask_fmsubadd_pd`] + * [x] [`_mm_mask3_fmsubadd_pd`] + * [x] [`_mm_maskz_fmsubadd_pd`] + * [x] [`_mm256_mask_fmsubadd_pd`] + * [x] [`_mm256_mask3_fmsubadd_pd`] + * [x] [`_mm256_maskz_fmsubadd_pd`] + * [x] [`_mm512_fmsubadd_round_ps`] + * [x] [`_mm512_mask_fmsubadd_round_ps`] + * [x] [`_mm512_maskz_fmsubadd_round_ps`] + * [x] [`_mm512_mask3_fmsubadd_round_ps`] + * [x] [`_mm512_fmsubadd_round_pd`] + * [x] [`_mm512_mask_fmsubadd_round_pd`] + * [x] [`_mm512_maskz_fmsubadd_round_pd`] + * [x] [`_mm512_mask3_fmsubadd_round_pd`] + * [x] [`_mm512_fnmadd_ps`] + * [x] [`_mm512_mask_fnmadd_ps`] + * [x] [`_mm512_maskz_fnmadd_ps`] + * [x] [`_mm512_mask3_fnmadd_ps`] + * [x] [`_mm_mask_fnmadd_ps`] + * [x] [`_mm_mask3_fnmadd_ps`] + * [x] [`_mm_maskz_fnmadd_ps`] + * [x] [`_mm256_mask_fnmadd_ps`] + * [x] [`_mm256_mask3_fnmadd_ps`] + * [x] [`_mm256_maskz_fnmadd_ps`] + * [x] [`_mm512_fnmadd_pd`] + * [x] [`_mm512_mask_fnmadd_pd`] + * [x] [`_mm512_maskz_fnmadd_pd`] + * [x] [`_mm512_mask3_fnmadd_pd`] + * [x] [`_mm_mask_fnmadd_pd`] + * [x] [`_mm_mask3_fnmadd_pd`] + * [x] [`_mm_maskz_fnmadd_pd`] + * [x] [`_mm256_mask_fnmadd_pd`] + * [x] [`_mm256_mask3_fnmadd_pd`] + * [x] [`_mm256_maskz_fnmadd_pd`] + * [x] [`_mm512_fnmadd_round_ps`] + * [x] [`_mm512_mask_fnmadd_round_ps`] + * [x] [`_mm512_maskz_fnmadd_round_ps`] + * [x] [`_mm512_mask3_fnmadd_round_ps`] + * [x] [`_mm512_fnmadd_round_pd`] + * [x] [`_mm512_mask_fnmadd_round_pd`] + * [x] [`_mm512_maskz_fnmadd_round_pd`] + * [x] [`_mm512_mask3_fnmadd_round_pd`] + * [x] [`_mm512_fnmsub_ps`] + * [x] [`_mm512_mask_fnmsub_ps`] + * [x] [`_mm512_maskz_fnmsub_ps`] + * [x] [`_mm512_mask3_fnmsub_ps`] + * [x] [`_mm_mask_fnmsub_ps`] + * [x] [`_mm_mask3_fnmsub_ps`] + * [x] [`_mm_maskz_fnmsub_ps`] + * [x] [`_mm256_mask_fnmsub_ps`] + * [x] [`_mm256_mask3_fnmsub_ps`] + * [x] [`_mm256_maskz_fnmsub_ps`] + * [x] [`_mm512_fnmsub_pd`] + * [x] [`_mm512_mask_fnmsub_pd`] + * [x] [`_mm512_maskz_fnmsub_pd`] + * [x] [`_mm512_mask3_fnmsub_pd`] + * [x] [`_mm_mask_fnmsub_pd`] + * [x] [`_mm_mask3_fnmsub_pd`] + * [x] [`_mm_maskz_fnmsub_pd`] + * [x] [`_mm256_mask_fnmsub_pd`] + * [x] [`_mm256_mask3_fnmsub_pd`] + * [x] [`_mm256_maskz_fnmsub_pd`] + * [x] [`_mm512_fnmsub_round_ps`] + * [x] [`_mm512_mask_fnmsub_round_ps`] + * [x] [`_mm512_maskz_fnmsub_round_ps`] + * [x] [`_mm512_mask3_fnmsub_round_ps`] + * [x] [`_mm512_fnmsub_round_pd`] + * [x] [`_mm512_mask_fnmsub_round_pd`] + * [x] [`_mm512_maskz_fnmsub_round_pd`] + * [x] [`_mm512_mask3_fnmsub_round_pd`] + * [x] [`_mm512_cmp_epi32_mask`] + * [x] [`_mm512_mask_cmp_epi32_mask`] + * [x] [`_mm_cmp_epi32_mask`] + * [x] [`_mm_mask_cmp_epi32_mask`] + * [x] [`_mm256_cmp_epi32_mask`] + * [x] [`_mm256_mask_cmp_epi32_mask`] + * [x] [`_mm512_cmp_epi64_mask`] + * [x] [`_mm512_mask_cmp_epi64_mask`] + * [x] [`_mm_cmp_epi64_mask`] + * [x] [`_mm_mask_cmp_epi64_mask`] + * [x] [`_mm256_cmp_epi64_mask`] + * [x] [`_mm256_mask_cmp_epi64_mask`] + * [x] [`_mm512_cmp_epu32_mask`] + * [x] [`_mm512_mask_cmp_epu32_mask`] + * [x] [`_mm_cmp_epu32_mask`] + * [x] [`_mm_mask_cmp_epu32_mask`] + * [x] [`_mm256_cmp_epu32_mask`] + * [x] [`_mm256_mask_cmp_epu32_mask`] + * [x] [`_mm512_cmp_epu64_mask`] + * [x] [`_mm512_mask_cmp_epu64_mask`] + * [x] [`_mm_cmp_epu64_mask`] + * [x] [`_mm_mask_cmp_epu64_mask`] + * [x] [`_mm256_cmp_epu64_mask`] + * [x] [`_mm256_mask_cmp_epu64_mask`] + * [x] [`_mm512_cmp_ps_mask`] + * [x] [`_mm512_mask_cmp_ps_mask`] + * [x] [`_mm_cmp_ps_mask`] + * [x] [`_mm_mask_cmp_ps_mask`] + * [x] [`_mm256_cmp_ps_mask`] + * [x] [`_mm256_mask_cmp_ps_mask`] + * [x] [`_mm512_cmp_round_ps_mask`] + * [x] [`_mm512_mask_cmp_round_ps_mask`] + * [x] [`_mm512_cmp_pd_mask`] + * [x] [`_mm512_mask_cmp_pd_mask`] + * [x] [`_mm_cmp_pd_mask`] + * [x] [`_mm_mask_cmp_pd_mask`] + * [x] [`_mm256_cmp_pd_mask`] + * [x] [`_mm256_mask_cmp_pd_mask`] + * [x] [`_mm512_cmp_round_pd_mask`] + * [x] [`_mm512_mask_cmp_round_pd_mask`] + * [x] [`_mm512_cmpeq_epi32_mask`] + * [x] [`_mm512_mask_cmpeq_epi32_mask`] + * [x] [`_mm_cmpeq_epi32_mask`] + * [x] [`_mm_mask_cmpeq_epi32_mask`] + * [x] [`_mm256_cmpeq_epi32_mask`] + * [x] [`_mm256_mask_cmpeq_epi32_mask`] + * [x] [`_mm512_cmpeq_epi64_mask`] + * [x] [`_mm512_mask_cmpeq_epi64_mask`] + * [x] [`_mm_cmpeq_epi64_mask`] + * [x] [`_mm_mask_cmpeq_epi64_mask`] + * [x] [`_mm256_cmpeq_epi64_mask`] + * [x] [`_mm256_mask_cmpeq_epi64_mask`] + * [x] [`_mm512_cmpeq_epu32_mask`] + * [x] [`_mm512_mask_cmpeq_epu32_mask`] + * [x] [`_mm_cmpeq_epu32_mask`] + * [x] [`_mm_mask_cmpeq_epu32_mask`] + * [x] [`_mm256_cmpeq_epu32_mask`] + * [x] [`_mm256_mask_cmpeq_epu32_mask`] + * [x] [`_mm512_cmpeq_epu64_mask`] + * [x] [`_mm512_mask_cmpeq_epu64_mask`] + * [x] [`_mm_cmpeq_epu64_mask`] + * [x] [`_mm_mask_cmpeq_epu64_mask`] + * [x] [`_mm256_cmpeq_epu64_mask`] + * [x] [`_mm256_mask_cmpeq_epu64_mask`] + * [x] [`_mm512_cmpneq_epi32_mask`] + * [x] [`_mm512_mask_cmpneq_epi32_mask`] + * [x] [`_mm_cmpneq_epi32_mask`] + * [x] [`_mm_mask_cmpneq_epi32_mask`] + * [x] [`_mm256_cmpneq_epi32_mask`] + * [x] [`_mm256_mask_cmpneq_epi32_mask`] + * [x] [`_mm512_cmpneq_epi64_mask`] + * [x] [`_mm512_mask_cmpneq_epi64_mask`] + * [x] [`_mm_cmpneq_epi64_mask`] + * [x] [`_mm_mask_cmpneq_epi64_mask`] + * [x] [`_mm256_cmpneq_epi64_mask`] + * [x] [`_mm256_mask_cmpneq_epi64_mask`] + * [x] [`_mm512_cmpneq_epu32_mask`] + * [x] [`_mm512_mask_cmpneq_epu32_mask`] + * [x] [`_mm_cmpneq_epu32_mask`] + * [x] [`_mm_mask_cmpneq_epu32_mask`] + * [x] [`_mm256_cmpneq_epu32_mask`] + * [x] [`_mm256_mask_cmpneq_epu32_mask`] + * [x] [`_mm512_cmpneq_epu64_mask`] + * [x] [`_mm512_mask_cmpneq_epu64_mask`] + * [x] [`_mm_cmpneq_epu64_mask`] + * [x] [`_mm_mask_cmpneq_epu64_mask`] + * [x] [`_mm256_cmpneq_epu64_mask`] + * [x] [`_mm256_mask_cmpneq_epu64_mask`] + * [x] [`_mm512_cmpge_epi32_mask`] + * [x] [`_mm512_mask_cmpge_epi32_mask`] + * [x] [`_mm_cmpge_epi32_mask`] + * [x] [`_mm_mask_cmpge_epi32_mask`] + * [x] [`_mm256_cmpge_epi32_mask`] + * [x] [`_mm256_mask_cmpge_epi32_mask`] + * [x] [`_mm512_cmpge_epi64_mask`] + * [x] [`_mm512_mask_cmpge_epi64_mask`] + * [x] [`_mm_cmpge_epi64_mask`] + * [x] [`_mm_mask_cmpge_epi64_mask`] + * [x] [`_mm256_cmpge_epi64_mask`] + * [x] [`_mm256_mask_cmpge_epi64_mask`] + * [x] [`_mm512_cmpge_epu32_mask`] + * [x] [`_mm512_mask_cmpge_epu32_mask`] + * [x] [`_mm_cmpge_epu32_mask`] + * [x] [`_mm_mask_cmpge_epu32_mask`] + * [x] [`_mm256_cmpge_epu32_mask`] + * [x] [`_mm256_mask_cmpge_epu32_mask`] + * [x] [`_mm512_cmpge_epu64_mask`] + * [x] [`_mm512_mask_cmpge_epu64_mask`] + * [x] [`_mm_cmpge_epu64_mask`] + * [x] [`_mm_mask_cmpge_epu64_mask`] + * [x] [`_mm256_cmpge_epu64_mask`] + * [x] [`_mm256_mask_cmpge_epu64_mask`] + * [x] [`_mm512_cmpgt_epi32_mask`] + * [x] [`_mm512_mask_cmpgt_epi32_mask`] + * [x] [`_mm_cmpgt_epi32_mask`] + * [x] [`_mm_mask_cmpgt_epi32_mask`] + * [x] [`_mm256_cmpgt_epi32_mask`] + * [x] [`_mm256_mask_cmpgt_epi32_mask`] + * [x] [`_mm512_cmpgt_epi64_mask`] + * [x] [`_mm512_mask_cmpgt_epi64_mask`] + * [x] [`_mm_cmpgt_epi64_mask`] + * [x] [`_mm_mask_cmpgt_epi64_mask`] + * [x] [`_mm256_cmpgt_epi64_mask`] + * [x] [`_mm256_mask_cmpgt_epi64_mask`] + * [x] [`_mm512_cmpgt_epu32_mask`] + * [x] [`_mm512_mask_cmpgt_epu32_mask`] + * [x] [`_mm_cmpgt_epu32_mask`] + * [x] [`_mm_mask_cmpgt_epu32_mask`] + * [x] [`_mm256_cmpgt_epu32_mask`] + * [x] [`_mm256_mask_cmpgt_epu32_mask`] + * [x] [`_mm512_cmpgt_epu64_mask`] + * [x] [`_mm512_mask_cmpgt_epu64_mask`] + * [x] [`_mm_cmpgt_epu64_mask`] + * [x] [`_mm_mask_cmpgt_epu64_mask`] + * [x] [`_mm256_cmpgt_epu64_mask`] + * [x] [`_mm256_mask_cmpgt_epu64_mask`] + * [x] [`_mm512_cmple_epi32_mask`] + * [x] [`_mm512_mask_cmple_epi32_mask`] + * [x] [`_mm_cmple_epi32_mask`] + * [x] [`_mm_mask_cmple_epi32_mask`] + * [x] [`_mm256_cmple_epi32_mask`] + * [x] [`_mm256_mask_cmple_epi32_mask`] + * [x] [`_mm512_cmple_epi64_mask`] + * [x] [`_mm512_mask_cmple_epi64_mask`] + * [x] [`_mm_cmple_epi64_mask`] + * [x] [`_mm_mask_cmple_epi64_mask`] + * [x] [`_mm256_cmple_epi64_mask`] + * [x] [`_mm256_mask_cmple_epi64_mask`] + * [x] [`_mm512_cmple_epu32_mask`] + * [x] [`_mm512_mask_cmple_epu32_mask`] + * [x] [`_mm_cmple_epu32_mask`] + * [x] [`_mm_mask_cmple_epu32_mask`] + * [x] [`_mm256_cmple_epu32_mask`] + * [x] [`_mm256_mask_cmple_epu32_mask`] + * [x] [`_mm512_cmple_epu64_mask`] + * [x] [`_mm512_mask_cmple_epu64_mask`] + * [x] [`_mm_cmple_epu64_mask`] + * [x] [`_mm_mask_cmple_epu64_mask`] + * [x] [`_mm256_cmple_epu64_mask`] + * [x] [`_mm256_mask_cmple_epu64_mask`] + * [x] [`_mm512_cmplt_epi32_mask`] + * [x] [`_mm512_mask_cmplt_epi32_mask`] + * [x] [`_mm_cmplt_epi32_mask`] + * [x] [`_mm_mask_cmplt_epi32_mask`] + * [x] [`_mm256_cmplt_epi32_mask`] + * [x] [`_mm256_mask_cmplt_epi32_mask`] + * [x] [`_mm512_cmplt_epi64_mask`] + * [x] [`_mm512_mask_cmplt_epi64_mask`] + * [x] [`_mm_cmplt_epi64_mask`] + * [x] [`_mm_mask_cmplt_epi64_mask`] + * [x] [`_mm256_cmplt_epi64_mask`] + * [x] [`_mm256_mask_cmplt_epi64_mask`] + * [x] [`_mm512_cmplt_epu32_mask`] + * [x] [`_mm512_mask_cmplt_epu32_mask`] + * [x] [`_mm_cmplt_epu32_mask`] + * [x] [`_mm_mask_cmplt_epu32_mask`] + * [x] [`_mm256_cmplt_epu32_mask`] + * [x] [`_mm256_mask_cmplt_epu32_mask`] + * [x] [`_mm512_cmplt_epu64_mask`] + * [x] [`_mm512_mask_cmplt_epu64_mask`] + * [x] [`_mm_cmplt_epu64_mask`] + * [x] [`_mm_mask_cmplt_epu64_mask`] + * [x] [`_mm256_cmplt_epu64_mask`] + * [x] [`_mm256_mask_cmplt_epu64_mask`] + * [x] [`_mm512_cmpeq_ps_mask`] + * [x] [`_mm512_mask_cmpeq_ps_mask`] + * [x] [`_mm512_cmpeq_pd_mask`] + * [x] [`_mm512_mask_cmpeq_pd_mask`] + * [x] [`_mm512_cmpneq_ps_mask`] + * [x] [`_mm512_mask_cmpneq_ps_mask`] + * [x] [`_mm512_cmpneq_pd_mask`] + * [x] [`_mm512_mask_cmpneq_pd_mask`] + * [x] [`_mm512_cmple_ps_mask`] + * [x] [`_mm512_mask_cmple_ps_mask`] + * [x] [`_mm512_cmple_pd_mask`] + * [x] [`_mm512_mask_cmple_pd_mask`] + * [x] [`_mm512_cmplt_ps_mask`] + * [x] [`_mm512_mask_cmplt_ps_mask`] + * [x] [`_mm512_cmplt_pd_mask`] + * [x] [`_mm512_mask_cmplt_pd_mask`] + * [x] [`_mm512_cmpnle_ps_mask`] + * [x] [`_mm512_mask_cmpnle_ps_mask`] + * [x] [`_mm512_cmpnle_pd_mask`] + * [x] [`_mm512_mask_cmpnle_pd_mask`] + * [x] [`_mm512_cmpnlt_ps_mask`] + * [x] [`_mm512_mask_cmpnlt_ps_mask`] + * [x] [`_mm512_cmpnlt_pd_mask`] + * [x] [`_mm512_mask_cmpnlt_pd_mask`] + * [x] [`_mm512_cmpord_ps_mask`] + * [x] [`_mm512_mask_cmpord_ps_mask`] + * [x] [`_mm512_cmpord_pd_mask`] + * [x] [`_mm512_mask_cmpord_pd_mask`] + * [x] [`_mm512_cmpunord_ps_mask`] + * [x] [`_mm512_mask_cmpunord_ps_mask`] + * [x] [`_mm512_cmpunord_pd_mask`] + * [x] [`_mm512_mask_cmpunord_pd_mask`] + * [x] [`_mm512_reduce_add_epi32`] + * [x] [`_mm512_mask_reduce_add_epi32`] + * [x] [`_mm512_reduce_add_epi64`] + * [x] [`_mm512_mask_reduce_add_epi64`] + * [x] [`_mm512_reduce_add_ps`] + * [x] [`_mm512_mask_reduce_add_ps`] + * [x] [`_mm512_reduce_add_pd`] + * [x] [`_mm512_mask_reduce_add_pd`] + * [x] [`_mm512_reduce_and_epi32`] + * [x] [`_mm512_mask_reduce_and_epi32`] + * [x] [`_mm512_reduce_and_epi64`] + * [x] [`_mm512_mask_reduce_and_epi64`] + * [x] [`_mm512_reduce_max_epi32`] + * [x] [`_mm512_mask_reduce_max_epi32`] + * [x] [`_mm512_reduce_max_epi64`] + * [x] [`_mm512_mask_reduce_max_epi64`] + * [x] [`_mm512_reduce_max_epu32`] + * [x] [`_mm512_mask_reduce_max_epu32`] + * [x] [`_mm512_reduce_max_epu64`] + * [x] [`_mm512_mask_reduce_max_epu64`] + * [x] [`_mm512_reduce_max_ps`] + * [x] [`_mm512_mask_reduce_max_ps`] + * [x] [`_mm512_reduce_max_pd`] + * [x] [`_mm512_mask_reduce_max_pd`] + * [x] [`_mm512_reduce_min_epi32`] + * [x] [`_mm512_mask_reduce_min_epi32`] + * [x] [`_mm512_reduce_min_epi64`] + * [x] [`_mm512_mask_reduce_min_epi64`] + * [x] [`_mm512_reduce_min_epu32`] + * [x] [`_mm512_mask_reduce_min_epu32`] + * [x] [`_mm512_reduce_min_epu64`] + * [x] [`_mm512_mask_reduce_min_epu64`] + * [x] [`_mm512_reduce_min_ps`] + * [x] [`_mm512_mask_reduce_min_ps`] + * [x] [`_mm512_reduce_min_pd`] + * [x] [`_mm512_mask_reduce_min_pd`] + * [x] [`_mm512_reduce_mul_epi32`] + * [x] [`_mm512_mask_reduce_mul_epi32`] + * [x] [`_mm512_reduce_mul_epi64`] + * [x] [`_mm512_mask_reduce_mul_epi64`] + * [x] [`_mm512_reduce_mul_ps`] + * [x] [`_mm512_mask_reduce_mul_ps`] + * [x] [`_mm512_reduce_mul_pd`] + * [x] [`_mm512_mask_reduce_mul_pd`] + * [x] [`_mm512_reduce_or_epi32`] + * [x] [`_mm512_mask_reduce_or_epi32`] + * [x] [`_mm512_reduce_or_epi64`] + * [x] [`_mm512_mask_reduce_or_epi64`] + * [x] [`_mm512_rol_epi32`] + * [x] [`_mm512_mask_rol_epi32`] + * [x] [`_mm512_maskz_rol_epi32`] + * [x] [`_mm_mask_rol_epi32`] + * [x] [`_mm_maskz_rol_epi32`] + * [x] [`_mm_rol_epi32`] + * [x] [`_mm256_mask_rol_epi32`] + * [x] [`_mm256_maskz_rol_epi32`] + * [x] [`_mm256_rol_epi32`] + * [x] [`_mm512_rol_epi64`] + * [x] [`_mm512_mask_rol_epi64`] + * [x] [`_mm512_maskz_rol_epi64`] + * [x] [`_mm_mask_rol_epi64`] + * [x] [`_mm_maskz_rol_epi64`] + * [x] [`_mm_rol_epi64`] + * [x] [`_mm256_mask_rol_epi64`] + * [x] [`_mm256_maskz_rol_epi64`] + * [x] [`_mm256_rol_epi64`] + * [x] [`_mm512_rolv_epi32`] + * [x] [`_mm512_mask_rolv_epi32`] + * [x] [`_mm512_maskz_rolv_epi32`] + * [x] [`_mm_mask_rolv_epi32`] + * [x] [`_mm_maskz_rolv_epi32`] + * [x] [`_mm_rolv_epi32`] + * [x] [`_mm256_mask_rolv_epi32`] + * [x] [`_mm256_maskz_rolv_epi32`] + * [x] [`_mm256_rolv_epi32`] + * [x] [`_mm512_rolv_epi64`] + * [x] [`_mm512_mask_rolv_epi64`] + * [x] [`_mm512_maskz_rolv_epi64`] + * [x] [`_mm_mask_rolv_epi64`] + * [x] [`_mm_maskz_rolv_epi64`] + * [x] [`_mm_rolv_epi64`] + * [x] [`_mm256_mask_rolv_epi64`] + * [x] [`_mm256_maskz_rolv_epi64`] + * [x] [`_mm256_rolv_epi64`] + * [x] [`_mm512_ror_epi32`] + * [x] [`_mm512_mask_ror_epi32`] + * [x] [`_mm512_maskz_ror_epi32`] + * [x] [`_mm_mask_ror_epi32`] + * [x] [`_mm_maskz_ror_epi32`] + * [x] [`_mm_ror_epi32`] + * [x] [`_mm256_mask_ror_epi32`] + * [x] [`_mm256_maskz_ror_epi32`] + * [x] [`_mm256_ror_epi32`] + * [x] [`_mm512_ror_epi64`] + * [x] [`_mm512_mask_ror_epi64`] + * [x] [`_mm512_maskz_ror_epi64`] + * [x] [`_mm_mask_ror_epi64`] + * [x] [`_mm_maskz_ror_epi64`] + * [x] [`_mm_ror_epi64`] + * [x] [`_mm256_mask_ror_epi64`] + * [x] [`_mm256_maskz_ror_epi64`] + * [x] [`_mm256_ror_epi64`] + * [x] [`_mm512_rorv_epi32`] + * [x] [`_mm512_mask_rorv_epi32`] + * [x] [`_mm512_maskz_rorv_epi32`] + * [x] [`_mm_mask_rorv_epi32`] + * [x] [`_mm_maskz_rorv_epi32`] + * [x] [`_mm_rorv_epi32`] + * [x] [`_mm256_mask_rorv_epi32`] + * [x] [`_mm256_maskz_rorv_epi32`] + * [x] [`_mm256_rorv_epi32`] + * [x] [`_mm512_rorv_epi64`] + * [x] [`_mm512_mask_rorv_epi64`] + * [x] [`_mm512_maskz_rorv_epi64`] + * [x] [`_mm_mask_rorv_epi64`] + * [x] [`_mm_maskz_rorv_epi64`] + * [x] [`_mm_rorv_epi64`] + * [x] [`_mm256_mask_rorv_epi64`] + * [x] [`_mm256_maskz_rorv_epi64`] + * [x] [`_mm256_rorv_epi64`] + * [x] [`_mm512_sll_epi32`] + * [x] [`_mm512_mask_sll_epi32`] + * [x] [`_mm512_maskz_sll_epi32`] + * [x] [`_mm_mask_sll_epi32`] + * [x] [`_mm_maskz_sll_epi32`] + * [x] [`_mm256_mask_sll_epi32`] + * [x] [`_mm256_maskz_sll_epi32`] + * [x] [`_mm512_sll_epi64`] + * [x] [`_mm512_mask_sll_epi64`] + * [x] [`_mm512_maskz_sll_epi64`] + * [x] [`_mm_mask_sll_epi64`] + * [x] [`_mm_maskz_sll_epi64`] + * [x] [`_mm256_mask_sll_epi64`] + * [x] [`_mm256_maskz_sll_epi64`] + * [x] [`_mm512_slli_epi32`] + * [x] [`_mm512_mask_slli_epi32`] + * [x] [`_mm512_maskz_slli_epi32`] + * [x] [`_mm_mask_slli_epi32`] + * [x] [`_mm_maskz_slli_epi32`] + * [x] [`_mm256_mask_slli_epi32`] + * [x] [`_mm256_maskz_slli_epi32`] + * [x] [`_mm512_slli_epi64`] + * [x] [`_mm512_mask_slli_epi64`] + * [x] [`_mm512_maskz_slli_epi64`] + * [x] [`_mm_mask_slli_epi64`] + * [x] [`_mm_maskz_slli_epi64`] + * [x] [`_mm256_mask_slli_epi64`] + * [x] [`_mm256_maskz_slli_epi64`] + * [x] [`_mm512_sllv_epi32`] + * [x] [`_mm512_mask_sllv_epi32`] + * [x] [`_mm512_maskz_sllv_epi32`] + * [x] [`_mm_mask_sllv_epi32`] + * [x] [`_mm_maskz_sllv_epi32`] + * [x] [`_mm256_mask_sllv_epi32`] + * [x] [`_mm256_maskz_sllv_epi32`] + * [x] [`_mm512_sllv_epi64`] + * [x] [`_mm512_mask_sllv_epi64`] + * [x] [`_mm512_maskz_sllv_epi64`] + * [x] [`_mm_mask_sllv_epi64`] + * [x] [`_mm_maskz_sllv_epi64`] + * [x] [`_mm256_mask_sllv_epi64`] + * [x] [`_mm256_maskz_sllv_epi64`] + * [x] [`_mm512_sra_epi32`] + * [x] [`_mm512_mask_sra_epi32`] + * [x] [`_mm512_maskz_sra_epi32`] + * [x] [`_mm_mask_sra_epi32`] + * [x] [`_mm_maskz_sra_epi32`] + * [x] [`_mm256_mask_sra_epi32`] + * [x] [`_mm256_maskz_sra_epi32`] + * [x] [`_mm512_sra_epi64`] + * [x] [`_mm512_mask_sra_epi64`] + * [x] [`_mm512_maskz_sra_epi64`] + * [x] [`_mm_mask_sra_epi64`] + * [x] [`_mm_maskz_sra_epi64`] + * [x] [`_mm_sra_epi64`] + * [x] [`_mm256_mask_sra_epi64`] + * [x] [`_mm256_maskz_sra_epi64`] + * [x] [`_mm256_sra_epi64`] + * [x] [`_mm512_srai_epi32`] + * [x] [`_mm512_mask_srai_epi32`] + * [x] [`_mm512_maskz_srai_epi32`] + * [x] [`_mm_mask_srai_epi32`] + * [x] [`_mm_maskz_srai_epi32`] + * [x] [`_mm256_mask_srai_epi32`] + * [x] [`_mm256_maskz_srai_epi32`] + * [x] [`_mm512_srai_epi64`] + * [x] [`_mm512_mask_srai_epi64`] + * [x] [`_mm512_maskz_srai_epi64`] + * [x] [`_mm_mask_srai_epi64`] + * [x] [`_mm_maskz_srai_epi64`] + * [x] [`_mm_srai_epi64`] + * [x] [`_mm256_mask_srai_epi64`] + * [x] [`_mm256_maskz_srai_epi64`] + * [x] [`_mm256_srai_epi64`] + * [x] [`_mm512_srav_epi32`] + * [x] [`_mm512_mask_srav_epi32`] + * [x] [`_mm512_maskz_srav_epi32`] + * [x] [`_mm_mask_srav_epi32`] + * [x] [`_mm_maskz_srav_epi32`] + * [x] [`_mm256_mask_srav_epi32`] + * [x] [`_mm256_maskz_srav_epi32`] + * [x] [`_mm512_srav_epi64`] + * [x] [`_mm512_mask_srav_epi64`] + * [x] [`_mm512_maskz_srav_epi64`] + * [x] [`_mm_mask_srav_epi64`] + * [x] [`_mm_maskz_srav_epi64`] + * [x] [`_mm_srav_epi64`] + * [x] [`_mm256_mask_srav_epi64`] + * [x] [`_mm256_maskz_srav_epi64`] + * [x] [`_mm256_srav_epi64`] + * [x] [`_mm512_srl_epi32`] + * [x] [`_mm512_mask_srl_epi32`] + * [x] [`_mm512_maskz_srl_epi32`] + * [x] [`_mm_mask_srl_epi32`] + * [x] [`_mm_maskz_srl_epi32`] + * [x] [`_mm256_mask_srl_epi32`] + * [x] [`_mm256_maskz_srl_epi32`] + * [x] [`_mm512_srl_epi64`] + * [x] [`_mm512_mask_srl_epi64`] + * [x] [`_mm512_maskz_srl_epi64`] + * [x] [`_mm_mask_srl_epi64`] + * [x] [`_mm_maskz_srl_epi64`] + * [x] [`_mm256_mask_srl_epi64`] + * [x] [`_mm256_maskz_srl_epi64`] + * [x] [`_mm512_srli_epi32`] + * [x] [`_mm512_mask_srli_epi32`] + * [x] [`_mm512_maskz_srli_epi32`] + * [x] [`_mm_mask_srli_epi32`] + * [x] [`_mm_maskz_srli_epi32`] + * [x] [`_mm256_mask_srli_epi32`] + * [x] [`_mm256_maskz_srli_epi32`] + * [x] [`_mm512_srli_epi64`] + * [x] [`_mm512_mask_srli_epi64`] + * [x] [`_mm512_maskz_srli_epi64`] + * [x] [`_mm_mask_srli_epi64`] + * [x] [`_mm_maskz_srli_epi64`] + * [x] [`_mm256_mask_srli_epi64`] + * [x] [`_mm256_maskz_srli_epi64`] + * [x] [`_mm512_srlv_epi32`] + * [x] [`_mm512_mask_srlv_epi32`] + * [x] [`_mm512_maskz_srlv_epi32`] + * [x] [`_mm_mask_srlv_epi32`] + * [x] [`_mm_maskz_srlv_epi32`] + * [x] [`_mm256_mask_srlv_epi32`] + * [x] [`_mm256_maskz_srlv_epi32`] + * [x] [`_mm512_srlv_epi64`] + * [x] [`_mm512_mask_srlv_epi64`] + * [x] [`_mm512_maskz_srlv_epi64`] + * [x] [`_mm_mask_srlv_epi64`] + * [x] [`_mm_maskz_srlv_epi64`] + * [x] [`_mm256_mask_srlv_epi64`] + * [x] [`_mm256_maskz_srlv_epi64`] + * [x] [`_mm512_mask_mov_epi32`] + * [x] [`_mm512_maskz_mov_epi32`] + * [x] [`_mm_mask_mov_epi32`] + * [x] [`_mm_maskz_mov_epi32`] + * [x] [`_mm256_mask_mov_epi32`] + * [x] [`_mm256_maskz_mov_epi32`] + * [x] [`_mm512_mask_mov_epi64`] + * [x] [`_mm512_maskz_mov_epi64`] + * [x] [`_mm_mask_mov_epi64`] + * [x] [`_mm_maskz_mov_epi64`] + * [x] [`_mm256_mask_mov_epi64`] + * [x] [`_mm256_maskz_mov_epi64`] + * [x] [`_mm512_mask_mov_ps`] + * [x] [`_mm512_maskz_mov_ps`] + * [x] [`_mm_mask_mov_ps`] + * [x] [`_mm_maskz_mov_ps`] + * [x] [`_mm256_mask_mov_ps`] + * [x] [`_mm256_maskz_mov_ps`] + * [x] [`_mm512_mask_mov_pd`] + * [x] [`_mm512_maskz_mov_pd`] + * [x] [`_mm_mask_mov_pd`] + * [x] [`_mm_maskz_mov_pd`] + * [x] [`_mm256_mask_mov_pd`] + * [x] [`_mm256_maskz_mov_pd`] + * [x] [`_mm512_movehdup_ps`] + * [x] [`_mm512_mask_movehdup_ps`] + * [x] [`_mm512_maskz_movehdup_ps`] + * [x] [`_mm_mask_movehdup_ps`] + * [x] [`_mm_maskz_movehdup_ps`] + * [x] [`_mm256_mask_movehdup_ps`] + * [x] [`_mm256_maskz_movehdup_ps`] + * [x] [`_mm512_moveldup_ps`] + * [x] [`_mm512_mask_moveldup_ps`] + * [x] [`_mm512_maskz_moveldup_ps`] + * [x] [`_mm_mask_moveldup_ps`] + * [x] [`_mm_maskz_moveldup_ps`] + * [x] [`_mm256_mask_moveldup_ps`] + * [x] [`_mm256_maskz_moveldup_ps`] + * [x] [`_mm512_movedup_pd`] + * [x] [`_mm512_mask_movedup_pd`] + * [x] [`_mm512_maskz_movedup_pd`] + * [x] [`_mm_mask_movedup_pd`] + * [x] [`_mm_maskz_movedup_pd`] + * [x] [`_mm256_mask_movedup_pd`] + * [x] [`_mm256_maskz_movedup_pd`] + * [x] [`_mm512_or_epi32`] + * [x] [`_mm512_mask_or_epi32`] + * [x] [`_mm512_maskz_or_epi32`] + * [x] [`_mm_mask_or_epi32`] + * [x] [`_mm_maskz_or_epi32`] + * [x] [`_mm_or_epi32`] + * [x] [`_mm256_mask_or_epi32`] + * [x] [`_mm256_maskz_or_epi32`] + * [x] [`_mm256_or_epi32`] + * [x] [`_mm512_or_epi64`] + * [x] [`_mm512_mask_or_epi64`] + * [x] [`_mm512_maskz_or_epi64`] + * [x] [`_mm_mask_or_epi64`] + * [x] [`_mm_maskz_or_epi64`] + * [x] [`_mm_or_epi64`] + * [x] [`_mm256_mask_or_epi64`] + * [x] [`_mm256_maskz_or_epi64`] + * [x] [`_mm256_or_epi64`] + * [x] [`_mm512_or_si512`] + * [x] [`_mm512_and_epi32`] + * [x] [`_mm512_mask_and_epi32`] + * [x] [`_mm512_maskz_and_epi32`] + * [x] [`_mm_mask_and_epi32`] + * [x] [`_mm_maskz_and_epi32`] + * [x] [`_mm256_mask_and_epi32`] + * [x] [`_mm256_maskz_and_epi32`] + * [x] [`_mm512_and_epi64`] + * [x] [`_mm512_mask_and_epi64`] + * [x] [`_mm512_maskz_and_epi64`] + * [x] [`_mm_mask_and_epi64`] + * [x] [`_mm_maskz_and_epi64`] + * [x] [`_mm256_mask_and_epi64`] + * [x] [`_mm256_maskz_and_epi64`] + * [x] [`_mm512_and_si512`] + * [x] [`_mm512_xor_epi32`] + * [x] [`_mm512_mask_xor_epi32`] + * [x] [`_mm512_maskz_xor_epi32`] + * [x] [`_mm_mask_xor_epi32`] + * [x] [`_mm_maskz_xor_epi32`] + * [x] [`_mm_xor_epi32`] + * [x] [`_mm256_mask_xor_epi32`] + * [x] [`_mm256_maskz_xor_epi32`] + * [x] [`_mm256_xor_epi32`] + * [x] [`_mm512_xor_epi64`] + * [x] [`_mm512_mask_xor_epi64`] + * [x] [`_mm512_maskz_xor_epi64`] + * [x] [`_mm_mask_xor_epi64`] + * [x] [`_mm_maskz_xor_epi64`] + * [x] [`_mm_xor_epi64`] + * [x] [`_mm256_mask_xor_epi64`] + * [x] [`_mm256_maskz_xor_epi64`] + * [x] [`_mm256_xor_epi64`] + * [x] [`_mm512_xor_si512`] + * [x] [`_mm512_andnot_epi32`] + * [x] [`_mm512_mask_andnot_epi32`] + * [x] [`_mm512_maskz_andnot_epi32`] + * [x] [`_mm_mask_andnot_epi32`] + * [x] [`_mm_maskz_andnot_epi32`] + * [x] [`_mm256_mask_andnot_epi32`] + * [x] [`_mm256_maskz_andnot_epi32`] + * [x] [`_mm512_andnot_epi64`] + * [x] [`_mm512_mask_andnot_epi64`] + * [x] [`_mm512_maskz_andnot_epi64`] + * [x] [`_mm_mask_andnot_epi64`] + * [x] [`_mm_maskz_andnot_epi64`] + * [x] [`_mm256_mask_andnot_epi64`] + * [x] [`_mm256_maskz_andnot_epi64`] + * [x] [`_mm512_andnot_si512`] + * [x] [`_mm512_unpackhi_epi32`] + * [x] [`_mm512_mask_unpackhi_epi32`] + * [x] [`_mm512_maskz_unpackhi_epi32`] + * [x] [`_mm_mask_unpackhi_epi32`] + * [x] [`_mm_maskz_unpackhi_epi32`] + * [x] [`_mm256_mask_unpackhi_epi32`] + * [x] [`_mm256_maskz_unpackhi_epi32`] + * [x] [`_mm512_unpackhi_epi64`] + * [x] [`_mm512_mask_unpackhi_epi64`] + * [x] [`_mm512_maskz_unpackhi_epi64`] + * [x] [`_mm_mask_unpackhi_epi64`] + * [x] [`_mm_maskz_unpackhi_epi64`] + * [x] [`_mm256_mask_unpackhi_epi64`] + * [x] [`_mm256_maskz_unpackhi_epi64`] + * [x] [`_mm512_unpackhi_ps`] + * [x] [`_mm512_mask_unpackhi_ps`] + * [x] [`_mm512_maskz_unpackhi_ps`] + * [x] [`_mm_mask_unpackhi_ps`] + * [x] [`_mm_maskz_unpackhi_ps`] + * [x] [`_mm256_mask_unpackhi_ps`] + * [x] [`_mm256_maskz_unpackhi_ps`] + * [x] [`_mm512_unpackhi_pd`] + * [x] [`_mm512_mask_unpackhi_pd`] + * [x] [`_mm512_maskz_unpackhi_pd`] + * [x] [`_mm_mask_unpackhi_pd`] + * [x] [`_mm_maskz_unpackhi_pd`] + * [x] [`_mm256_mask_unpackhi_pd`] + * [x] [`_mm256_maskz_unpackhi_pd`] + * [x] [`_mm512_unpacklo_epi32`] + * [x] [`_mm512_mask_unpacklo_epi32`] + * [x] [`_mm512_maskz_unpacklo_epi32`] + * [x] [`_mm_mask_unpacklo_epi32`] + * [x] [`_mm_maskz_unpacklo_epi32`] + * [x] [`_mm256_mask_unpacklo_epi32`] + * [x] [`_mm256_maskz_unpacklo_epi32`] + * [x] [`_mm512_unpacklo_epi64`] + * [x] [`_mm512_mask_unpacklo_epi64`] + * [x] [`_mm512_maskz_unpacklo_epi64`] + * [x] [`_mm_mask_unpacklo_epi64`] + * [x] [`_mm_maskz_unpacklo_epi64`] + * [x] [`_mm256_mask_unpacklo_epi64`] + * [x] [`_mm256_maskz_unpacklo_epi64`] + * [x] [`_mm512_unpacklo_ps`] + * [x] [`_mm512_mask_unpacklo_ps`] + * [x] [`_mm512_maskz_unpacklo_ps`] + * [x] [`_mm_mask_unpacklo_ps`] + * [x] [`_mm_maskz_unpacklo_ps`] + * [x] [`_mm256_mask_unpacklo_ps`] + * [x] [`_mm256_maskz_unpacklo_ps`] + * [x] [`_mm512_unpacklo_pd`] + * [x] [`_mm512_mask_unpacklo_pd`] + * [x] [`_mm512_maskz_unpacklo_pd`] + * [x] [`_mm_mask_unpacklo_pd`] + * [x] [`_mm_maskz_unpacklo_pd`] + * [x] [`_mm256_mask_unpacklo_pd`] + * [x] [`_mm256_maskz_unpacklo_pd`] + * [x] [`_mm512_mask_blend_epi32`] + * [x] [`_mm_mask_blend_epi32`] + * [x] [`_mm256_mask_blend_epi32`] + * [x] [`_mm512_mask_blend_epi64`] + * [x] [`_mm_mask_blend_epi64`] + * [x] [`_mm256_mask_blend_epi64`] + * [x] [`_mm512_mask_blend_ps`] + * [x] [`_mm_mask_blend_ps`] + * [x] [`_mm256_mask_blend_ps`] + * [x] [`_mm512_mask_blend_pd`] + * [x] [`_mm_mask_blend_pd`] + * [x] [`_mm256_mask_blend_pd`] + * [x] [`_mm512_broadcast_f32x4`] + * [x] [`_mm512_mask_broadcast_f32x4`] + * [x] [`_mm512_maskz_broadcast_f32x4`] + * [x] [`_mm256_broadcast_f32x4`] + * [x] [`_mm256_mask_broadcast_f32x4`] + * [x] [`_mm256_maskz_broadcast_f32x4`] + * [x] [`_mm512_broadcast_f64x4`] + * [x] [`_mm512_mask_broadcast_f64x4`] + * [x] [`_mm512_maskz_broadcast_f64x4`] + * [x] [`_mm512_broadcast_i32x4`] + * [x] [`_mm512_mask_broadcast_i32x4`] + * [x] [`_mm512_maskz_broadcast_i32x4`] + * [x] [`_mm256_broadcast_i32x4`] + * [x] [`_mm256_mask_broadcast_i32x4`] + * [x] [`_mm256_maskz_broadcast_i32x4`] + * [x] [`_mm512_broadcast_i64x4`] + * [x] [`_mm512_mask_broadcast_i64x4`] + * [x] [`_mm512_maskz_broadcast_i64x4`] + * [x] [`_mm512_broadcastd_epi32`] + * [x] [`_mm512_mask_broadcastd_epi32`] + * [x] [`_mm512_maskz_broadcastd_epi32`] + * [x] [`_mm_mask_broadcastd_epi32`] + * [x] [`_mm_maskz_broadcastd_epi32`] + * [x] [`_mm256_mask_broadcastd_epi32`] + * [x] [`_mm256_maskz_broadcastd_epi32`] + * [x] [`_mm512_broadcastq_epi64`] + * [x] [`_mm512_mask_broadcastq_epi64`] + * [x] [`_mm512_maskz_broadcastq_epi64`] + * [x] [`_mm_mask_broadcastq_epi64`] + * [x] [`_mm_maskz_broadcastq_epi64`] + * [x] [`_mm256_mask_broadcastq_epi64`] + * [x] [`_mm256_maskz_broadcastq_epi64`] + * [x] [`_mm512_broadcastss_ps`] + * [x] [`_mm512_mask_broadcastss_ps`] + * [x] [`_mm512_maskz_broadcastss_ps`] + * [x] [`_mm_mask_broadcastss_ps`] + * [x] [`_mm_maskz_broadcastss_ps`] + * [x] [`_mm256_mask_broadcastss_ps`] + * [x] [`_mm256_maskz_broadcastss_ps`] + * [x] [`_mm512_broadcastsd_pd`] + * [x] [`_mm512_mask_broadcastsd_pd`] + * [x] [`_mm512_maskz_broadcastsd_pd`] + * [x] [`_mm256_mask_broadcastsd_pd`] + * [x] [`_mm256_maskz_broadcastsd_pd`] + * [x] [`_mm512_shuffle_epi32`] + * [x] [`_mm512_mask_shuffle_epi32`] + * [x] [`_mm512_maskz_shuffle_epi32`] + * [x] [`_mm_mask_shuffle_epi32`] + * [x] [`_mm_maskz_shuffle_epi32`] + * [x] [`_mm256_mask_shuffle_epi32`] + * [x] [`_mm256_maskz_shuffle_epi32`] + * [x] [`_mm512_shuffle_ps`] + * [x] [`_mm512_mask_shuffle_ps`] + * [x] [`_mm512_maskz_shuffle_ps`] + * [x] [`_mm_mask_shuffle_ps`] + * [x] [`_mm_maskz_shuffle_ps`] + * [x] [`_mm256_mask_shuffle_ps`] + * [x] [`_mm256_maskz_shuffle_ps`] + * [x] [`_mm512_shuffle_pd`] + * [x] [`_mm512_mask_shuffle_pd`] + * [x] [`_mm512_maskz_shuffle_pd`] + * [x] [`_mm_mask_shuffle_pd`] + * [x] [`_mm_maskz_shuffle_pd`] + * [x] [`_mm256_mask_shuffle_pd`] + * [x] [`_mm256_maskz_shuffle_pd`] + * [x] [`_mm512_shuffle_i32x4`] + * [x] [`_mm512_mask_shuffle_i32x4`] + * [x] [`_mm512_maskz_shuffle_i32x4`] + * [x] [`_mm256_mask_shuffle_i32x4`] + * [x] [`_mm256_maskz_shuffle_i32x4`] + * [x] [`_mm256_shuffle_i32x4`] + * [x] [`_mm512_shuffle_i64x2`] + * [x] [`_mm512_mask_shuffle_i64x2`] + * [x] [`_mm512_maskz_shuffle_i64x2`] + * [x] [`_mm256_mask_shuffle_i64x2`] + * [x] [`_mm256_maskz_shuffle_i64x2`] + * [x] [`_mm256_shuffle_i64x2`] + * [x] [`_mm512_shuffle_f32x4`] + * [x] [`_mm512_mask_shuffle_f32x4`] + * [x] [`_mm512_maskz_shuffle_f32x4`] + * [x] [`_mm256_mask_shuffle_f32x4`] + * [x] [`_mm256_maskz_shuffle_f32x4`] + * [x] [`_mm256_shuffle_f32x4`] + * [x] [`_mm512_shuffle_f64x2`] + * [x] [`_mm512_mask_shuffle_f64x2`] + * [x] [`_mm512_maskz_shuffle_f64x2`] + * [x] [`_mm256_mask_shuffle_f64x2`] + * [x] [`_mm256_maskz_shuffle_f64x2`] + * [x] [`_mm256_shuffle_f64x2`] + * [x] [`_mm512_alignr_epi32`] + * [x] [`_mm512_mask_alignr_epi32`] + * [x] [`_mm512_maskz_alignr_epi32`] + * [x] [`_mm_alignr_epi32`] + * [x] [`_mm_mask_alignr_epi32`] + * [x] [`_mm_maskz_alignr_epi32`] + * [x] [`_mm256_alignr_epi32`] + * [x] [`_mm256_mask_alignr_epi32`] + * [x] [`_mm256_maskz_alignr_epi32`] + * [x] [`_mm512_alignr_epi64`] + * [x] [`_mm512_mask_alignr_epi64`] + * [x] [`_mm512_maskz_alignr_epi64`] + * [x] [`_mm_alignr_epi64`] + * [x] [`_mm_mask_alignr_epi64`] + * [x] [`_mm_maskz_alignr_epi64`] + * [x] [`_mm256_alignr_epi64`] + * [x] [`_mm256_mask_alignr_epi64`] + * [x] [`_mm256_maskz_alignr_epi64`] + * [x] [`_mm512_permute_ps`] + * [x] [`_mm512_mask_permute_ps`] + * [x] [`_mm512_maskz_permute_ps`] + * [x] [`_mm_mask_permute_ps`] + * [x] [`_mm_maskz_permute_ps`] + * [x] [`_mm256_mask_permute_ps`] + * [x] [`_mm256_maskz_permute_ps`] + * [x] [`_mm512_permute_pd`] + * [x] [`_mm512_mask_permute_pd`] + * [x] [`_mm512_maskz_permute_pd`] + * [x] [`_mm_mask_permute_pd`] + * [x] [`_mm_maskz_permute_pd`] + * [x] [`_mm256_mask_permute_pd`] + * [x] [`_mm256_maskz_permute_pd`] + * [x] [`_mm512_permutevar_epi32`] + * [x] [`_mm512_mask_permutevar_epi32`] + * [x] [`_mm512_permutevar_ps`] + * [x] [`_mm512_mask_permutevar_ps`] + * [x] [`_mm512_maskz_permutevar_ps`] + * [x] [`_mm_mask_permutevar_ps`] + * [x] [`_mm_maskz_permutevar_ps`] + * [x] [`_mm256_mask_permutevar_ps`] + * [x] [`_mm256_maskz_permutevar_ps`] + * [x] [`_mm512_permutevar_pd`] + * [x] [`_mm512_mask_permutevar_pd`] + * [x] [`_mm512_maskz_permutevar_pd`] + * [x] [`_mm_mask_permutevar_pd`] + * [x] [`_mm_maskz_permutevar_pd`] + * [x] [`_mm256_mask_permutevar_pd`] + * [x] [`_mm256_maskz_permutevar_pd`] + * [x] [`_mm512_permutex2var_epi32`] + * [x] [`_mm512_mask_permutex2var_epi32`] + * [x] [`_mm512_maskz_permutex2var_epi32`] + * [x] [`_mm512_mask2_permutex2var_epi32`] + * [x] [`_mm_mask_permutex2var_epi32`] + * [x] [`_mm_mask2_permutex2var_epi32`] + * [x] [`_mm_maskz_permutex2var_epi32`] + * [x] [`_mm_permutex2var_epi32`] + * [x] [`_mm256_mask_permutex2var_epi32`] + * [x] [`_mm256_mask2_permutex2var_epi32`] + * [x] [`_mm256_maskz_permutex2var_epi32`] + * [x] [`_mm256_permutex2var_epi32`] + * [x] [`_mm512_permutex2var_epi64`] + * [x] [`_mm512_mask_permutex2var_epi64`] + * [x] [`_mm512_maskz_permutex2var_epi64`] + * [x] [`_mm512_mask2_permutex2var_epi64`] + * [x] [`_mm_mask_permutex2var_epi64`] + * [x] [`_mm_mask2_permutex2var_epi64`] + * [x] [`_mm_maskz_permutex2var_epi64`] + * [x] [`_mm_permutex2var_epi64`] + * [x] [`_mm256_mask_permutex2var_epi64`] + * [x] [`_mm256_mask2_permutex2var_epi64`] + * [x] [`_mm256_maskz_permutex2var_epi64`] + * [x] [`_mm256_permutex2var_epi64`] + * [x] [`_mm512_permutex2var_ps`] + * [x] [`_mm512_mask_permutex2var_ps`] + * [x] [`_mm512_maskz_permutex2var_ps`] + * [x] [`_mm512_mask2_permutex2var_ps`] + * [x] [`_mm_mask_permutex2var_ps`] + * [x] [`_mm_mask2_permutex2var_ps`] + * [x] [`_mm_maskz_permutex2var_ps`] + * [x] [`_mm_permutex2var_ps`] + * [x] [`_mm256_mask_permutex2var_ps`] + * [x] [`_mm256_mask2_permutex2var_ps`] + * [x] [`_mm256_maskz_permutex2var_ps`] + * [x] [`_mm256_permutex2var_ps`] + * [x] [`_mm512_permutex2var_pd`] + * [x] [`_mm512_mask_permutex2var_pd`] + * [x] [`_mm512_maskz_permutex2var_pd`] + * [x] [`_mm512_mask2_permutex2var_pd`] + * [x] [`_mm_mask_permutex2var_pd`] + * [x] [`_mm_mask2_permutex2var_pd`] + * [x] [`_mm_maskz_permutex2var_pd`] + * [x] [`_mm_permutex2var_pd`] + * [x] [`_mm256_mask_permutex2var_pd`] + * [x] [`_mm256_mask2_permutex2var_pd`] + * [x] [`_mm256_maskz_permutex2var_pd`] + * [x] [`_mm256_permutex2var_pd`] + * [x] [`_mm512_permutex_epi64`] + * [x] [`_mm512_mask_permutex_epi64`] + * [x] [`_mm512_maskz_permutex_epi64`] + * [x] [`_mm256_mask_permutex_epi64`] + * [x] [`_mm256_maskz_permutex_epi64`] + * [x] [`_mm256_permutex_epi64`] + * [x] [`_mm512_permutex_pd`] + * [x] [`_mm512_mask_permutex_pd`] + * [x] [`_mm512_maskz_permutex_pd`] + * [x] [`_mm256_mask_permutex_pd`] + * [x] [`_mm256_maskz_permutex_pd`] + * [x] [`_mm256_permutex_pd`] + * [x] [`_mm512_permutexvar_epi32`] + * [x] [`_mm512_mask_permutexvar_epi32`] + * [x] [`_mm512_maskz_permutexvar_epi32`] + * [x] [`_mm256_mask_permutexvar_epi32`] + * [x] [`_mm256_maskz_permutexvar_epi32`] + * [x] [`_mm256_permutexvar_epi32`] + * [x] [`_mm512_permutexvar_epi64`] + * [x] [`_mm512_mask_permutexvar_epi64`] + * [x] [`_mm512_maskz_permutexvar_epi64`] + * [x] [`_mm256_mask_permutexvar_epi64`] + * [x] [`_mm256_maskz_permutexvar_epi64`] + * [x] [`_mm256_permutexvar_epi64`] + * [x] [`_mm512_permutexvar_ps`] + * [x] [`_mm512_mask_permutexvar_ps`] + * [x] [`_mm512_maskz_permutexvar_ps`] + * [x] [`_mm256_mask_permutexvar_ps`] + * [x] [`_mm256_maskz_permutexvar_ps`] + * [x] [`_mm256_permutexvar_ps`] + * [x] [`_mm512_permutexvar_pd`] + * [x] [`_mm512_mask_permutexvar_pd`] + * [x] [`_mm512_maskz_permutexvar_pd`] + * [x] [`_mm256_mask_permutexvar_pd`] + * [x] [`_mm256_maskz_permutexvar_pd`] + * [x] [`_mm256_permutexvar_pd`] + * [x] [`_mm512_i32gather_epi32`] + * [x] [`_mm512_mask_i32gather_epi32`] + * [_] [`_mm_mmask_i32gather_epi32`] //need i1 + * [_] [`_mm256_mmask_i32gather_epi32`] //need i1 + * [x] [`_mm512_i32gather_epi64`] + * [x] [`_mm512_mask_i32gather_epi64`] + * [_] [`_mm_mmask_i32gather_epi64`] //need i1 + * [_] [`_mm256_mmask_i32gather_epi64`] //need i1 + * [x] [`_mm512_i32gather_ps`] + * [x] [`_mm512_mask_i32gather_ps`] + * [_] [`_mm_mmask_i32gather_ps`] //need i1 + * [_] [`_mm256_mmask_i32gather_ps`] //need i1 + * [x] [`_mm512_i32gather_pd`] + * [x] [`_mm512_mask_i32gather_pd`] + * [_] [`_mm_mmask_i32gather_pd`] //need i1 + * [_] [`_mm256_mmask_i32gather_pd`] //need i1 + * [x] [`_mm512_i64gather_epi32`] + * [x] [`_mm512_mask_i64gather_epi32`] + * [_] [`_mm_mmask_i64gather_epi32`] //need i1 + * [_] [`_mm256_mmask_i64gather_epi32`] //need i1 + * [x] [`_mm512_i64gather_epi64`] + * [x] [`_mm512_mask_i64gather_epi64`] + * [_] [`_mm_mmask_i64gather_epi64`] //need i1 + * [_] [`_mm256_mmask_i64gather_epi64`] //need i1 + * [x] [`_mm512_i64gather_ps`] + * [x] [`_mm512_mask_i64gather_ps`] + * [_] [`_mm_mmask_i64gather_ps`] //need i1 + * [_] [`_mm256_mmask_i64gather_ps`] //need i1 + * [x] [`_mm512_i64gather_pd`] + * [x] [`_mm512_mask_i64gather_pd`] + * [_] [`_mm_mmask_i64gather_pd`] //need i1 + * [_] [`_mm256_mmask_i64gather_pd`] //need i1 + * [ ] [`_mm512_i32extgather_epi32`] //not in llvm + * [ ] [`_mm512_mask_i32extgather_epi32`] //not in llvm + * [ ] [`_mm512_i32extgather_ps`] // not in llvm + * [ ] [`_mm512_mask_i32extgather_ps`] //not in llvm + * [ ] [`_mm512_i32loextgather_epi64`] //not in llvm + * [ ] [`_mm512_mask_i32loextgather_epi64`] //not in llvm + * [ ] [`_mm512_i32loextgather_pd`] //not in llvm + * [ ] [`_mm512_mask_i32loextgather_pd`] //not in llvm + * [ ] [`_mm512_i32logather_epi64`] //not in llvm + * [ ] [`_mm512_mask_i32logather_epi64`] //not in llvm + * [ ] [`_mm512_i32logather_pd`] //not in llvm + * [ ] [`_mm512_mask_i32logather_pd`] //not in llvm + * [x] [`_mm512_i32scatter_epi32`] + * [x] [`_mm512_mask_i32scatter_epi32`] + * [_] [`_mm_i32scatter_epi32`] //need i1 + * [_] [`_mm_mask_i32scatter_epi32`] // need i1 + * [_] [`_mm256_i32scatter_epi32`] //need i1 + * [_] [`_mm256_mask_i32scatter_epi32`] //need i1 + * [x] [`_mm512_i32scatter_epi64`] + * [x] [`_mm512_mask_i32scatter_epi64`] + * [_] [`_mm_i32scatter_epi64`]//need i1 + * [_] [`_mm_mask_i32scatter_epi64`] //need i1 + * [_] [`_mm256_i32scatter_epi64`] //need i1 + * [_] [`_mm256_mask_i32scatter_epi64`] //need i1 + * [x] [`_mm512_i32scatter_ps`] + * [x] [`_mm512_mask_i32scatter_ps`] + * [_] [`_mm_i32scatter_ps`] //need i1 + * [_] [`_mm_mask_i32scatter_ps`] //need i1 + * [_] [`_mm256_i32scatter_ps`] //need i1 + * [_] [`_mm256_mask_i32scatter_ps`] //need i1 + * [x] [`_mm512_i32scatter_pd`] + * [x] [`_mm512_mask_i32scatter_pd`] + * [_] [`_mm_i32scatter_pd`] //need i1 + * [_] [`_mm_mask_i32scatter_pd`] //need i1 + * [_] [`_mm256_i32scatter_pd`] //need i1 + * [_] [`_mm256_mask_i32scatter_pd`] //need i1 + * [x] [`_mm512_i64scatter_epi32`] + * [x] [`_mm512_mask_i64scatter_epi32`] + * [_] [`_mm_i64scatter_epi32`] //need i1 + * [_] [`_mm_mask_i64scatter_epi32`] //need i1 + * [_] [`_mm256_i64scatter_epi32`] //need i1 + * [_] [`_mm256_mask_i64scatter_epi32`] //need i1 + * [x] [`_mm512_mask_i64scatter_epi64`] + * [x] [`_mm512_i64scatter_epi64`] + * [_] [`_mm_i64scatter_epi64`] //need i1 + * [_] [`_mm_mask_i64scatter_epi64`] //need i1 + * [_] [`_mm256_i64scatter_epi64`] //need i1 + * [_] [`_mm256_mask_i64scatter_epi64`] //need i1 + * [x] [`_mm512_i64scatter_ps`] + * [x] [`_mm512_mask_i64scatter_ps`] + * [_] [`_mm_i64scatter_ps`] //need i1 + * [_] [`_mm_mask_i64scatter_ps`] //need i1 + * [_] [`_mm256_i64scatter_ps`] //need i1 + * [_] [`_mm256_mask_i64scatter_ps`] //need i1 + * [x] [`_mm512_i64scatter_pd`] + * [x] [`_mm512_mask_i64scatter_pd`] + * [_] [`_mm_i64scatter_pd`] //need i1 + * [_] [`_mm_mask_i64scatter_pd`] //need i1 + * [_] [`_mm256_i64scatter_pd`] //need i1 + * [_] [`_mm256_mask_i64scatter_pd`] //need i1 + * [ ] [`_mm512_i32extscatter_epi32`] //not in llvm + * [ ] [`_mm512_mask_i32extscatter_epi32`] //not in llvm + * [ ] [`_mm512_i32extscatter_ps`] //not in llvm + * [ ] [`_mm512_mask_i32extscatter_ps`] //not in llvm + * [ ] [`_mm512_i32loextscatter_epi64`] //not in llvm + * [ ] [`_mm512_mask_i32loextscatter_epi64`] //not in llvm + * [ ] [`_mm512_i32loextscatter_pd`] //not in llvm + * [ ] [`_mm512_mask_i32loextscatter_pd`] //not in llvm + * [ ] [`_mm512_i32loscatter_epi64`] //not in llvm + * [ ] [`_mm512_mask_i32loscatter_epi64`] //not in llvm + * [ ] [`_mm512_i32loscatter_pd`] //not in llvm + * [ ] [`_mm512_mask_i32loscatter_pd`] //not in llvm + * [x] [`_mm512_inserti32x4`] + * [x] [`_mm512_mask_inserti32x4`] + * [x] [`_mm512_maskz_inserti32x4`] + * [x] [`_mm256_inserti32x4`] + * [x] [`_mm256_mask_inserti32x4`] + * [x] [`_mm256_maskz_inserti32x4`] + * [x] [`_mm512_inserti64x4`] + * [x] [`_mm512_mask_inserti64x4`] + * [x] [`_mm512_maskz_inserti64x4`] + * [x] [`_mm512_insertf32x4`] + * [x] [`_mm512_mask_insertf32x4`] + * [x] [`_mm512_maskz_insertf32x4`] + * [x] [`_mm256_insertf32x4`] + * [x] [`_mm256_mask_insertf32x4`] + * [x] [`_mm256_maskz_insertf32x4`] + * [x] [`_mm512_insertf64x4`] + * [x] [`_mm512_mask_insertf64x4`] + * [x] [`_mm512_maskz_insertf64x4`] + * [x] [`_mm512_extracti32x4_epi32`] + * [x] [`_mm512_mask_extracti32x4_epi32`] + * [x] [`_mm512_maskz_extracti32x4_epi32`] + * [x] [`_mm256_extracti32x4_epi32`] + * [x] [`_mm256_mask_extracti32x4_epi32`] + * [x] [`_mm256_maskz_extracti32x4_epi32`] + * [x] [`_mm512_extracti64x4_epi64`] + * [x] [`_mm512_mask_extracti64x4_epi64`] + * [x] [`_mm512_maskz_extracti64x4_epi64`] + * [x] [`_mm512_extractf32x4_ps`] + * [x] [`_mm512_mask_extractf32x4_ps`] + * [x] [`_mm512_maskz_extractf32x4_ps`] + * [x] [`_mm256_extractf32x4_ps`] + * [x] [`_mm256_mask_extractf32x4_ps`] + * [x] [`_mm256_maskz_extractf32x4_ps`] + * [x] [`_mm512_extractf64x4_pd`] + * [x] [`_mm512_mask_extractf64x4_pd`] + * [x] [`_mm512_maskz_extractf64x4_pd`] + * [x] [`_mm512_maskz_compress_epi32`] + * [x] [`_mm512_mask_compress_epi32`] + * [x] [`_mm_mask_compress_epi32`] + * [x] [`_mm_maskz_compress_epi32`] + * [x] [`_mm256_mask_compress_epi32`] + * [x] [`_mm256_maskz_compress_epi32`] + * [x] [`_mm512_mask_compress_epi64`] + * [x] [`_mm512_maskz_compress_epi64`] + * [x] [`_mm_mask_compress_epi64`] + * [x] [`_mm_maskz_compress_epi64`] + * [x] [`_mm256_mask_compress_epi64`] + * [x] [`_mm256_maskz_compress_epi64`] + * [x] [`_mm512_mask_compress_ps`] + * [x] [`_mm512_maskz_compress_ps`] + * [x] [`_mm_mask_compress_ps`] + * [x] [`_mm_maskz_compress_ps`] + * [x] [`_mm256_mask_compress_ps`] + * [x] [`_mm256_maskz_compress_ps`] + * [x] [`_mm512_mask_compress_pd`] + * [x] [`_mm512_maskz_compress_pd`] + * [x] [`_mm_mask_compress_pd`] + * [x] [`_mm_maskz_compress_pd`] + * [x] [`_mm256_mask_compress_pd`] + * [x] [`_mm256_maskz_compress_pd`] + * [x] [`_mm512_mask_compressstoreu_epi32`] //need i1 + * [x] [`_mm_mask_compressstoreu_epi32`] //need i1 + * [x] [`_mm256_mask_compressstoreu_epi32`] //need i1 + * [x] [`_mm512_mask_compressstoreu_epi64`] //need i1 + * [x] [`_mm_mask_compressstoreu_epi64`] //need i1 + * [x] [`_mm256_mask_compressstoreu_epi64`] //need i1 + * [x] [`_mm512_mask_compressstoreu_ps`] //need i1 + * [x] [`_mm_mask_compressstoreu_ps`] //need i1 + * [x] [`_mm256_mask_compressstoreu_ps`] //need i1 + * [x] [`_mm512_mask_compressstoreu_pd`] //need i1 + * [x] [`_mm_mask_compressstoreu_pd`] //need i1 + * [x] [`_mm256_mask_compressstoreu_pd`] //need i1 + * [x] [`_mm512_mask_expand_epi32`] + * [x] [`_mm512_maskz_expand_epi32`] + * [x] [`_mm_mask_expand_epi32`] + * [x] [`_mm_maskz_expand_epi32`] + * [x] [`_mm256_mask_expand_epi32`] + * [x] [`_mm256_maskz_expand_epi32`] + * [x] [`_mm512_mask_expand_epi64`] + * [x] [`_mm512_maskz_expand_epi64`] + * [x] [`_mm_mask_expand_epi64`] + * [x] [`_mm_maskz_expand_epi64`] + * [x] [`_mm256_mask_expand_epi64`] + * [x] [`_mm256_maskz_expand_epi64`] + * [x] [`_mm512_mask_expand_ps`] + * [x] [`_mm512_maskz_expand_ps`] + * [x] [`_mm_mask_expand_ps`] + * [x] [`_mm_maskz_expand_ps`] + * [x] [`_mm256_mask_expand_ps`] + * [x] [`_mm256_maskz_expand_ps`] + * [x] [`_mm512_mask_expand_pd`] + * [x] [`_mm512_maskz_expand_pd`] + * [x] [`_mm_mask_expand_pd`] + * [x] [`_mm_maskz_expand_pd`] + * [x] [`_mm256_mask_expand_pd`] + * [x] [`_mm256_maskz_expand_pd`] + * [x] [`_mm512_mask_expandloadu_epi32`] //need i1 + * [x] [`_mm512_maskz_expandloadu_epi32`] //need i1 + * [x] [`_mm_mask_expandloadu_epi32`] //need i1 + * [x] [`_mm_maskz_expandloadu_epi32`] //need i1 + * [x] [`_mm256_mask_expandloadu_epi32`] //need i1 + * [x] [`_mm256_maskz_expandloadu_epi32`] //need i1 + * [x] [`_mm512_mask_expandloadu_epi64`] //need i1 + * [x] [`_mm512_maskz_expandloadu_epi64`] //need i1 + * [x] [`_mm_mask_expandloadu_epi64`] //need i1 + * [x] [`_mm_maskz_expandloadu_epi64`] //need i1 + * [x] [`_mm256_mask_expandloadu_epi64`] //need i1 + * [x] [`_mm256_maskz_expandloadu_epi64`] //need i1 + * [x] [`_mm512_mask_expandloadu_ps`] //need i1 + * [x] [`_mm512_maskz_expandloadu_ps`] //need i1 + * [x] [`_mm_mask_expandloadu_ps`] //need i1 + * [x] [`_mm_maskz_expandloadu_ps`] //need i1 + * [x] [`_mm256_mask_expandloadu_ps`] //need i1 + * [x] [`_mm256_maskz_expandloadu_ps`] //need i1 + * [x] [`_mm512_mask_expandloadu_pd`] //need i1 + * [x] [`_mm512_maskz_expandloadu_pd`] //need i1 + * [x] [`_mm_mask_expandloadu_pd`] //need i1 + * [x] [`_mm_maskz_expandloadu_pd`] //need i1 + * [x] [`_mm256_mask_expandloadu_pd`] //need i1 + * [x] [`_mm256_maskz_expandloadu_pd`] //need i1 + * [x] [`_mm512_zextpd128_pd512`] + * [x] [`_mm512_zextpd256_pd512`] + * [x] [`_mm512_zextps128_ps512`] + * [x] [`_mm512_zextps256_ps512`] + * [x] [`_mm512_zextsi128_si512`] + * [x] [`_mm512_zextsi256_si512`] + * [x] [`_mm512_undefined_epi32`] + * [x] [`_mm512_undefined_pd`] + * [x] [`_mm512_undefined_ps`] + * [x] [`_mm512_undefined`] + * [ ] [`_mm512_svml_round_pd`] //not in llvm + * [x] [`_mm512_ternarylogic_epi32`] + * [x] [`_mm512_mask_ternarylogic_epi32`] + * [x] [`_mm512_maskz_ternarylogic_epi32`] + * [x] [`_mm_mask_ternarylogic_epi32`] + * [x] [`_mm_maskz_ternarylogic_epi32`] + * [x] [`_mm_ternarylogic_epi32`] + * [x] [`_mm256_mask_ternarylogic_epi32`] + * [x] [`_mm256_maskz_ternarylogic_epi32`] + * [x] [`_mm256_ternarylogic_epi32`] + * [x] [`_mm512_ternarylogic_epi64`] + * [x] [`_mm512_mask_ternarylogic_epi64`] + * [x] [`_mm512_maskz_ternarylogic_epi64`] + * [x] [`_mm_mask_ternarylogic_epi64`] + * [x] [`_mm_maskz_ternarylogic_epi64`] + * [x] [`_mm_ternarylogic_epi64`] + * [x] [`_mm256_mask_ternarylogic_epi64`] + * [x] [`_mm256_maskz_ternarylogic_epi64`] + * [x] [`_mm256_ternarylogic_epi64`] + * [x] [`_mm512_test_epi32_mask`] + * [x] [`_mm512_mask_test_epi32_mask`] + * [x] [`_mm_mask_test_epi32_mask`] + * [x] [`_mm_test_epi32_mask`] + * [x] [`_mm256_mask_test_epi32_mask`] + * [x] [`_mm256_test_epi32_mask`] + * [x] [`_mm512_test_epi64_mask`] + * [x] [`_mm512_mask_test_epi64_mask`] + * [x] [`_mm_mask_test_epi64_mask`] + * [x] [`_mm_test_epi64_mask`] + * [x] [`_mm256_mask_test_epi64_mask`] + * [x] [`_mm256_test_epi64_mask`] + * [x] [`_mm512_testn_epi32_mask`] + * [x] [`_mm512_mask_testn_epi32_mask`] + * [x] [`_mm_mask_testn_epi32_mask`] + * [x] [`_mm_testn_epi32_mask`] + * [x] [`_mm256_mask_testn_epi32_mask`] + * [x] [`_mm256_testn_epi32_mask`] + * [x] [`_mm512_testn_epi64_mask`] + * [x] [`_mm512_mask_testn_epi64_mask`] + * [x] [`_mm_mask_testn_epi64_mask`] + * [x] [`_mm_testn_epi64_mask`] + * [x] [`_mm256_mask_testn_epi64_mask`] + * [x] [`_mm256_testn_epi64_mask`] + * [x] [`_mm512_set1_epi8`] + * [x] [`_mm512_set1_epi16`] + * [x] [`_mm512_set1_epi32`] + * [x] [`_mm512_mask_set1_epi32`] + * [x] [`_mm512_maskz_set1_epi32`] + * [x] [`_mm_mask_set1_epi32`] + * [x] [`_mm_maskz_set1_epi32`] + * [x] [`_mm256_mask_set1_epi32`] + * [x] [`_mm256_maskz_set1_epi32`] + * [x] [`_mm512_set1_epi64`] + * [x] [`_mm512_mask_set1_epi64`] + * [x] [`_mm512_maskz_set1_epi64`] + * [x] [`_mm_mask_set1_epi64`] + * [x] [`_mm_maskz_set1_epi64`] + * [x] [`_mm256_mask_set1_epi64`] + * [x] [`_mm256_maskz_set1_epi64`] + * [x] [`_mm512_set1_ps`] + * [x] [`_mm512_set1_pd`] + * [x] [`_mm512_set4_epi32`] + * [x] [`_mm512_set4_epi64`] + * [x] [`_mm512_set4_pd`] + * [x] [`_mm512_set4_ps`] + * [x] [`_mm512_set_epi16`] + * [x] [`_mm512_set_epi32`] + * [x] [`_mm512_set_epi64`] + * [x] [`_mm512_set_epi8`] + * [x] [`_mm512_set_pd`] + * [x] [`_mm512_set_ps`] + * [x] [`_mm512_setr4_epi32`] + * [x] [`_mm512_setr4_epi64`] + * [x] [`_mm512_setr4_pd`] + * [x] [`_mm512_setr4_ps`] + * [x] [`_mm512_setr_epi32`] + * [x] [`_mm512_setr_epi64`] + * [x] [`_mm512_setr_pd`] + * [x] [`_mm512_setr_ps`] + * [x] [`_mm512_setzero_epi32`] + * [x] [`_mm512_setzero_pd`] + * [x] [`_mm512_setzero_ps`] + * [x] [`_mm512_setzero_si512`] + * [x] [`_mm512_setzero`] + * [x] [`_mm512_load_epi32`] + * [x] [`_mm512_mask_load_epi32`] //need i1 + * [x] [`_mm512_maskz_load_epi32`] //need i1 + * [x] [`_mm_load_epi32`] + * [x] [`_mm_mask_load_epi32`] //need i1 + * [x] [`_mm_maskz_load_epi32`] //need i1 + * [x] [`_mm256_load_epi32`] + * [x] [`_mm256_mask_load_epi32`] //need i1 + * [x] [`_mm256_maskz_load_epi32`] //need i1 + * [x] [`_mm512_load_epi64`] + * [x] [`_mm512_mask_load_epi64`] //need i1 + * [x] [`_mm512_maskz_load_epi64`] //need i1 + * [x] [`_mm_load_epi64`] //need i1 + * [x] [`_mm_mask_load_epi64`] //need i1 + * [x] [`_mm_maskz_load_epi64`] //need i1 + * [x] [`_mm256_load_epi64`] //need i1 + * [x] [`_mm256_mask_load_epi64`] //need i1 + * [x] [`_mm256_maskz_load_epi64`] //need i1 + * [x] [`_mm512_load_ps`] + * [x] [`_mm512_mask_load_ps`] //need i1 + * [x] [`_mm512_maskz_load_ps`] //need i1 + * [x] [`_mm_maskz_load_ps`] //need i + * [x] [`_mm_mask_load_ps`] //need i1 + * [x] [`_mm_maskz_load_ps`] //need i1 + * [x] [`_mm256_mask_load_ps`] //need i1 + * [x] [`_mm256_maskz_load_ps`] //need i1 + * [x] [`_mm512_load_pd`] + * [x] [`_mm512_mask_load_pd`] //need i1 + * [x] [`_mm512_maskz_load_pd`] //need i1 + * [x] [`_mm_mask_load_pd`] //need i1 + * [x] [`_mm_maskz_load_pd`] //need i1 + * [x] [`_mm256_mask_load_pd`] //need i1 + * [x] [`_mm256_maskz_load_pd`] //need i1 + * [x] [`_mm512_load_si512`] + * [x] [`_mm512_loadu_epi32`] + * [x] [`_mm512_mask_loadu_epi32`] //need i1 + * [x] [`_mm_loadu_epi32`] + * [x] [`_mm_mask_loadu_epi32`] //need i1 + * [x] [`_mm_maskz_loadu_epi32`] //need i1 + * [x] [`_mm512_maskz_loadu_epi32`] //need i1 + * [x] [`_mm256_loadu_epi32`] + * [x] [`_mm256_mask_loadu_epi32`] //need i1 + * [x] [`_mm256_maskz_loadu_epi32`] //need i1 + * [x] [`_mm512_loadu_epi64`] + * [x] [`_mm512_mask_loadu_epi64`] //need i1 + * [x] [`_mm512_maskz_loadu_epi64`] //need i1 + * [x] [`_mm_loadu_epi64`] + * [x] [`_mm_mask_loadu_epi64`] //need i1 + * [x] [`_mm_maskz_loadu_epi64`] //need i1 + * [x] [`_mm256_loadu_epi64`] + * [x] [`_mm256_mask_loadu_epi64`] //need i1 + * [x] [`_mm256_maskz_loadu_epi64`] //need i1 + * [x] [`_mm512_loadu_ps`] + * [x] [`_mm512_mask_loadu_ps`] //need i1 + * [x] [`_mm512_maskz_loadu_ps`] //need i1 + * [x] [`_mm_mask_loadu_ps`] //need i1 + * [x] [`_mm_maskz_loadu_ps`] //need i1 + * [x] [`_mm256_mask_loadu_ps`] //need i1 + * [x] [`_mm256_maskz_loadu_ps`] //need i1 + * [x] [`_mm512_loadu_pd`] + * [x] [`_mm512_mask_loadu_pd`] //need i1 + * [x] [`_mm512_maskz_loadu_pd`] //need i1 + * [x] [`_mm_mask_loadu_pd`] //need i1 + * [x] [`_mm_maskz_loadu_pd`] //need i1 + * [x] [`_mm256_mask_loadu_pd`] //need i1 + * [x] [`_mm256_maskz_loadu_pd`] //need i1 + * [x] [`_mm512_loadu_si512`] + * [x] [`_mm512_store_epi32`] + * [x] [`_mm512_mask_store_epi32`] //need i1 + * [x] [`_mm_mask_store_epi32`] //need i1 + * [x] [`_mm_store_epi32`] + * [x] [`_mm256_mask_store_epi32`] //need i1 + * [x] [`_mm256_store_epi32`] + * [x] [`_mm512_store_epi64`] + * [x] [`_mm512_mask_store_epi64`] //need i1 + * [x] [`_mm_mask_store_epi64`] //need i1 + * [x] [`_mm_store_epi64`] + * [x] [`_mm256_mask_store_epi64`] //need i1 + * [x] [`_mm256_store_epi64`] + * [x] [`_mm512_store_ps`] + * [x] [`_mm512_mask_store_ps`] //need i1 + * [x] [`_mm_mask_store_ps`] //need i1 + * [x] [`_mm256_mask_store_ps`] //need i1 + * [x] [`_mm512_store_pd`] + * [x] [`_mm512_mask_store_pd`] //need i1 + * [x] [`_mm_mask_store_pd`] //need i1 + * [x] [`_mm256_mask_store_pd`] //need i1 + * [x] [`_mm512_store_si512`] + * [x] [`_mm512_storeu_epi32`] + * [x] [`_mm512_mask_storeu_epi32`] //need i1 + * [x] [`_mm_mask_storeu_epi32`] //need i1 + * [x] [`_mm_storeu_epi32`] + * [x] [`_mm256_mask_storeu_epi32`] //need i1 + * [x] [`_mm256_storeu_epi32`] + * [x] [`_mm512_storeu_epi64`] + * [x] [`_mm512_mask_storeu_epi64`] //need i1 + * [x] [`_mm_mask_storeu_epi64`] //need i1 + * [x] [`_mm_storeu_epi64`] + * [x] [`_mm256_mask_storeu_epi64`] //need i1 + * [x] [`_mm256_storeu_epi64`] + * [x] [`_mm512_storeu_ps`] + * [x] [`_mm512_mask_storeu_ps`] //need i1 + * [x] [`_mm_mask_storeu_ps`] //need i1 + * [x] [`_mm256_mask_storeu_ps`] //need i1 + * [x] [`_mm512_storeu_pd`] + * [x] [`_mm512_mask_storeu_pd`] //need i1 + * [x] [`_mm_mask_storeu_pd`] //need i1 + * [x] [`_mm256_mask_storeu_pd`] //need i1 + * [x] [`_mm512_storeu_si512`] + * [ ] [`_mm512_stream_load_si512`] //stream_load_si256, ... not implment yet + * [x] [`_mm512_stream_pd`] + * [x] [`_mm512_stream_ps`] + * [x] [`_mm512_stream_si512`] + * [x] [`_mm512_castpd128_pd512`] + * [x] [`_mm512_castpd256_pd512`] + * [x] [`_mm512_castpd512_pd128`] + * [x] [`_mm512_castpd512_pd256`] + * [x] [`_mm512_castpd_ps`] + * [x] [`_mm512_castpd_si512`] + * [x] [`_mm512_castps128_ps512`] + * [x] [`_mm512_castps256_ps512`] + * [x] [`_mm512_castps512_ps128`] + * [x] [`_mm512_castps512_ps256`] + * [x] [`_mm512_castps_pd`] + * [x] [`_mm512_castps_si512`] + * [x] [`_mm512_castsi128_si512`] + * [x] [`_mm512_castsi256_si512`] + * [x] [`_mm512_castsi512_pd`] + * [x] [`_mm512_castsi512_ps`] + * [x] [`_mm512_castsi512_si128`] + * [x] [`_mm512_castsi512_si256`] + * [x] [`_mm512_cvt_roundps_ph`] + * [x] [`_mm512_mask_cvt_roundps_ph`] + * [x] [`_mm512_maskz_cvt_roundps_ph`] + * [x] [`_mm_mask_cvt_roundps_ph`] + * [x] [`_mm_maskz_cvt_roundps_ph`] + * [x] [`_mm256_mask_cvt_roundps_ph`] + * [x] [`_mm256_maskz_cvt_roundps_ph`] + * [x] [`_mm512_cvtepi16_epi32`] + * [x] [`_mm512_mask_cvtepi16_epi32`] + * [x] [`_mm512_maskz_cvtepi16_epi32`] + * [x] [`_mm_mask_cvtepi16_epi32`] + * [x] [`_mm_maskz_cvtepi16_epi32`] + * [x] [`_mm256_mask_cvtepi16_epi32`] + * [x] [`_mm256_maskz_cvtepi16_epi32`] + * [x] [`_mm512_cvtepi16_epi64`] + * [x] [`_mm512_mask_cvtepi16_epi64`] + * [x] [`_mm512_maskz_cvtepi16_epi64`] + * [x] [`_mm_mask_cvtepi16_epi64`] + * [x] [`_mm_maskz_cvtepi16_epi64`] + * [x] [`_mm256_mask_cvtepi16_epi64`] + * [x] [`_mm256_maskz_cvtepi16_epi64`] + * [x] [`_mm512_cvtepi32_epi16`] + * [x] [`_mm512_mask_cvtepi32_epi16`] + * [x] [`_mm512_maskz_cvtepi32_epi16`] + * [x] [`_mm512_mask_cvtepi32_storeu_epi16`] + * [x] [`_mm_mask_cvtepi32_storeu_epi16`] + * [x] [`_mm256_mask_cvtepi32_storeu_epi16`] + * [x] [`_mm_cvtepi32_epi16`] + * [x] [`_mm_mask_cvtepi32_epi16`] + * [x] [`_mm_maskz_cvtepi32_epi16`] + * [x] [`_mm256_cvtepi32_epi16`] + * [x] [`_mm256_mask_cvtepi32_epi16`] + * [x] [`_mm256_maskz_cvtepi32_epi16`] + * [x] [`_mm512_cvtepi32_epi64`] + * [x] [`_mm512_mask_cvtepi32_epi64`] + * [x] [`_mm512_maskz_cvtepi32_epi64`] + * [x] [`_mm_mask_cvtepi32_epi64`] + * [x] [`_mm_maskz_cvtepi32_epi64`] + * [x] [`_mm256_mask_cvtepi32_epi64`] + * [x] [`_mm256_maskz_cvtepi32_epi64`] + * [x] [`_mm512_cvtepi32_epi8`] + * [x] [`_mm512_mask_cvtepi32_epi8`] + * [x] [`_mm512_maskz_cvtepi32_epi8`] + * [x] [`_mm512_mask_cvtepi32_storeu_epi8`] + * [x] [`_mm_mask_cvtepi32_storeu_epi8`] + * [x] [`_mm256_mask_cvtepi32_storeu_epi8`] + * [x] [`_mm_cvtepi32_epi8`] + * [x] [`_mm_mask_cvtepi32_epi8`] + * [x] [`_mm_maskz_cvtepi32_epi8`] + * [x] [`_mm256_cvtepi32_epi8`] + * [x] [`_mm256_mask_cvtepi32_epi8`] + * [x] [`_mm256_maskz_cvtepi32_epi8`] + * [x] [`_mm512_cvtepi32_ps`] + * [x] [`_mm512_mask_cvtepi32_ps`] + * [x] [`_mm512_maskz_cvtepi32_ps`] + * [x] [`_mm_mask_cvtepi32_ps`] + * [x] [`_mm_maskz_cvtepi32_ps`] + * [x] [`_mm256_mask_cvtepi32_ps`] + * [x] [`_mm256_maskz_cvtepi32_ps`] + * [x] [`_mm512_cvtepi32_pd`] + * [x] [`_mm512_mask_cvtepi32_pd`] + * [x] [`_mm512_maskz_cvtepi32_pd`] + * [x] [`_mm_mask_cvtepi32_pd`] + * [x] [`_mm_maskz_cvtepi32_pd`] + * [x] [`_mm256_mask_cvtepi32_pd`] + * [x] [`_mm256_maskz_cvtepi32_pd`] + * [x] [`_mm512_cvtepi32lo_pd`] + * [x] [`_mm512_mask_cvtepi32lo_pd`] + * [x] [`_mm512_cvtepi64_epi16`] + * [x] [`_mm512_mask_cvtepi64_epi16`] + * [x] [`_mm512_maskz_cvtepi64_epi16`] + * [x] [`_mm_cvtepi64_epi16`] + * [x] [`_mm_mask_cvtepi64_epi16`] + * [x] [`_mm_maskz_cvtepi64_epi16`] + * [x] [`_mm256_cvtepi64_epi16`] + * [x] [`_mm256_mask_cvtepi64_epi16`] + * [x] [`_mm256_maskz_cvtepi64_epi16`] + * [x] [`_mm512_mask_cvtepi64_storeu_epi16`] + * [x] [`_mm_mask_cvtepi64_storeu_epi16`] + * [x] [`_mm256_mask_cvtepi64_storeu_epi16`] + * [x] [`_mm512_cvtepi64_epi8`] + * [x] [`_mm512_mask_cvtepi64_epi8`] + * [x] [`_mm512_maskz_cvtepi64_epi8`] + * [x] [`_mm_cvtepi64_epi8`] + * [x] [`_mm_mask_cvtepi64_epi8`] + * [x] [`_mm_maskz_cvtepi64_epi8`] + * [x] [`_mm256_cvtepi64_epi8`] + * [x] [`_mm256_mask_cvtepi64_epi8`] + * [x] [`_mm256_maskz_cvtepi64_epi8`] + * [x] [`_mm512_mask_cvtepi64_storeu_epi8`] + * [x] [`_mm_mask_cvtepi64_storeu_epi8`] + * [x] [`_mm256_mask_cvtepi64_storeu_epi8`] + * [x] [`_mm512_cvtepi64_epi32`] + * [x] [`_mm512_mask_cvtepi64_epi32`] + * [x] [`_mm512_maskz_cvtepi64_epi32`] + * [x] [`_mm_cvtepi64_epi32`] + * [x] [`_mm_mask_cvtepi64_epi32`] + * [x] [`_mm_maskz_cvtepi64_epi32`] + * [x] [`_mm256_cvtepi64_epi32`] + * [x] [`_mm256_mask_cvtepi64_epi32`] + * [x] [`_mm256_maskz_cvtepi64_epi32`] + * [x] [`_mm512_mask_cvtepi64_storeu_epi32`] + * [x] [`_mm_mask_cvtepi64_storeu_epi32`] + * [x] [`_mm256_mask_cvtepi64_storeu_epi32`] + * [x] [`_mm512_cvtepi8_epi32`] + * [x] [`_mm512_mask_cvtepi8_epi32`] + * [x] [`_mm512_maskz_cvtepi8_epi32`] + * [x] [`_mm_mask_cvtepi8_epi32`] + * [x] [`_mm_maskz_cvtepi8_epi32`] + * [x] [`_mm256_mask_cvtepi8_epi32`] + * [x] [`_mm256_maskz_cvtepi8_epi32`] + * [x] [`_mm512_cvtepi8_epi64`] + * [x] [`_mm512_mask_cvtepi8_epi64`] + * [x] [`_mm512_maskz_cvtepi8_epi64`] + * [x] [`_mm_mask_cvtepi8_epi64`] + * [x] [`_mm_maskz_cvtepi8_epi64`] + * [x] [`_mm256_mask_cvtepi8_epi64`] + * [x] [`_mm256_maskz_cvtepi8_epi64`] + * [x] [`_mm512_cvtepu16_epi32`] + * [x] [`_mm512_mask_cvtepu16_epi32`] + * [x] [`_mm512_maskz_cvtepu16_epi32`] + * [x] [`_mm_mask_cvtepu16_epi32`] + * [x] [`_mm_maskz_cvtepu16_epi32`] + * [x] [`_mm256_mask_cvtepu16_epi32`] + * [x] [`_mm256_maskz_cvtepu16_epi32`] + * [x] [`_mm512_cvtepu16_epi64`] + * [x] [`_mm512_mask_cvtepu16_epi64`] + * [x] [`_mm512_maskz_cvtepu16_epi64`] + * [x] [`_mm_mask_cvtepu16_epi64`] + * [x] [`_mm_maskz_cvtepu16_epi64`] + * [x] [`_mm256_mask_cvtepu16_epi64`] + * [x] [`_mm256_maskz_cvtepu16_epi64`] + * [x] [`_mm512_cvtepu32_epi64`] + * [x] [`_mm512_mask_cvtepu32_epi64`] + * [x] [`_mm512_maskz_cvtepu32_epi64`] + * [x] [`_mm_mask_cvtepu32_epi64`] + * [x] [`_mm_maskz_cvtepu32_epi64`] + * [x] [`_mm256_mask_cvtepu32_epi64`] + * [x] [`_mm256_maskz_cvtepu32_epi64`] + * [x] [`_mm512_cvtepu32_ps`] + * [x] [`_mm512_mask_cvtepu32_ps`] + * [x] [`_mm512_maskz_cvtepu32_ps`] + * [x] [`_mm512_cvtepu32_pd`] + * [x] [`_mm512_mask_cvtepu32_pd`] + * [x] [`_mm512_maskz_cvtepu32_pd`] + * [x] [`_mm_cvtepu32_pd`] + * [x] [`_mm_mask_cvtepu32_pd`] + * [x] [`_mm_maskz_cvtepu32_pd`] + * [x] [`_mm256_cvtepu32_pd`] + * [x] [`_mm256_mask_cvtepu32_pd`] + * [x] [`_mm256_maskz_cvtepu32_pd`] + * [x] [`_mm512_cvtepu32lo_pd`] + * [x] [`_mm512_mask_cvtepu32lo_pd`] + * [x] [`_mm512_cvtepu8_epi32`] + * [x] [`_mm512_mask_cvtepu8_epi32`] + * [x] [`_mm512_maskz_cvtepu8_epi32`] + * [x] [`_mm_mask_cvtepu8_epi32`] + * [x] [`_mm_maskz_cvtepu8_epi32`] + * [x] [`_mm256_mask_cvtepu8_epi32`] + * [x] [`_mm256_maskz_cvtepu8_epi32`] + * [x] [`_mm512_cvtepu8_epi64`] + * [x] [`_mm512_mask_cvtepu8_epi64`] + * [x] [`_mm512_maskz_cvtepu8_epi64`] + * [x] [`_mm_mask_cvtepu8_epi64`] + * [x] [`_mm_maskz_cvtepu8_epi64`] + * [x] [`_mm256_mask_cvtepu8_epi64`] + * [x] [`_mm256_maskz_cvtepu8_epi64`] + * [x] [`_mm512_cvtpd_epi32`] + * [x] [`_mm512_mask_cvtpd_epi32`] + * [x] [`_mm512_maskz_cvtpd_epi32`] + * [x] [`_mm_mask_cvtpd_epi32`] + * [x] [`_mm_maskz_cvtpd_epi32`] + * [x] [`_mm256_mask_cvtpd_epi32`] + * [x] [`_mm256_maskz_cvtpd_epi32`] + * [x] [`_mm512_cvtpd_epu32`] + * [x] [`_mm512_mask_cvtpd_epu32`] + * [x] [`_mm512_maskz_cvtpd_epu32`] + * [x] [`_mm_cvtpd_epu32`] + * [x] [`_mm_mask_cvtpd_epu32`] + * [x] [`_mm_maskz_cvtpd_epu32`] + * [x] [`_mm256_cvtpd_epu32`] + * [x] [`_mm256_mask_cvtpd_epu32`] + * [x] [`_mm256_maskz_cvtpd_epu32`] + * [x] [`_mm512_cvtpd_ps`] + * [x] [`_mm512_mask_cvtpd_ps`] + * [x] [`_mm512_maskz_cvtpd_ps`] + * [x] [`_mm_mask_cvtpd_ps`] + * [x] [`_mm_maskz_cvtpd_ps`] + * [x] [`_mm256_mask_cvtpd_ps`] + * [x] [`_mm256_maskz_cvtpd_ps`] + * [x] [`_mm512_cvtpd_pslo`] + * [x] [`_mm512_mask_cvtpd_pslo`] + * [x] [`_mm512_cvtph_ps`] + * [x] [`_mm512_mask_cvtph_ps`] + * [x] [`_mm512_maskz_cvtph_ps`] + * [x] [`_mm_mask_cvtph_ps`] + * [x] [`_mm_maskz_cvtph_ps`] + * [x] [`_mm256_mask_cvtph_ps`] + * [x] [`_mm256_maskz_cvtph_ps`] + * [x] [`_mm512_cvtps_epi32`] + * [x] [`_mm512_mask_cvtps_epi32`] + * [x] [`_mm512_maskz_cvtps_epi32`] + * [x] [`_mm_mask_cvtps_epi32`] + * [x] [`_mm_maskz_cvtps_epi32`] + * [x] [`_mm256_mask_cvtps_epi32`] + * [x] [`_mm256_maskz_cvtps_epi32`] + * [x] [`_mm512_cvtps_epu32`] + * [x] [`_mm512_mask_cvtps_epu32`] + * [x] [`_mm512_maskz_cvtps_epu32`] + * [x] [`_mm_cvtps_epu32`] + * [x] [`_mm_mask_cvtps_epu32`] + * [x] [`_mm_maskz_cvtps_epu32`] + * [x] [`_mm256_cvtps_epu32`] + * [x] [`_mm256_mask_cvtps_epu32`] + * [x] [`_mm256_maskz_cvtps_epu32`] + * [x] [`_mm512_cvtps_pd`] + * [x] [`_mm512_mask_cvtps_pd`] + * [x] [`_mm512_maskz_cvtps_pd`] + * [x] [`_mm512_cvtps_ph`] + * [x] [`_mm512_mask_cvtps_ph`] + * [x] [`_mm512_maskz_cvtps_ph`] + * [x] [`_mm_mask_cvtps_ph`] + * [x] [`_mm_maskz_cvtps_ph`] + * [x] [`_mm256_mask_cvtps_ph`] + * [x] [`_mm256_maskz_cvtps_ph`] + * [x] [`_mm512_cvtpslo_pd`] + * [x] [`_mm512_mask_cvtpslo_pd`] + * [x] [`_mm512_cvtsepi32_epi16`] + * [x] [`_mm512_mask_cvtsepi32_epi16`] + * [x] [`_mm512_maskz_cvtsepi32_epi16`] + * [x] [`_mm_cvtsepi32_epi16`] + * [x] [`_mm_mask_cvtsepi32_epi16`] + * [x] [`_mm_maskz_cvtsepi32_epi16`] + * [x] [`_mm256_cvtsepi32_epi16`] + * [x] [`_mm256_mask_cvtsepi32_epi16`] + * [x] [`_mm256_maskz_cvtsepi32_epi16`] + * [x] [`_mm512_cvtsepi32_epi8`] + * [x] [`_mm512_mask_cvtsepi32_epi8`] + * [x] [`_mm512_maskz_cvtsepi32_epi8`] + * [x] [`_mm_cvtsepi32_epi8`] + * [x] [`_mm_mask_cvtsepi32_epi8`] + * [x] [`_mm_maskz_cvtsepi32_epi8`] + * [x] [`_mm256_cvtsepi32_epi8`] + * [x] [`_mm256_mask_cvtsepi32_epi8`] + * [x] [`_mm256_maskz_cvtsepi32_epi8`] + * [x] [`_mm512_mask_cvtsepi32_storeu_epi16`] + * [x] [`_mm_mask_cvtsepi32_storeu_epi16`] + * [x] [`_mm256_mask_cvtsepi32_storeu_epi16`] + * [x] [`_mm512_mask_cvtsepi32_storeu_epi8`] + * [x] [`_mm_mask_cvtsepi32_storeu_epi8`] + * [x] [`_mm256_mask_cvtsepi32_storeu_epi8`] + * [x] [`_mm512_cvtsepi64_epi16`] + * [x] [`_mm512_mask_cvtsepi64_epi16`] + * [x] [`_mm512_maskz_cvtsepi64_epi16`] + * [x] [`_mm_cvtsepi64_epi16`] + * [x] [`_mm_mask_cvtsepi64_epi16`] + * [x] [`_mm_maskz_cvtsepi64_epi16`] + * [x] [`_mm256_cvtsepi64_epi16`] + * [x] [`_mm256_mask_cvtsepi64_epi16`] + * [x] [`_mm256_maskz_cvtsepi64_epi16`] + * [x] [`_mm512_cvtsepi64_epi32`] + * [x] [`_mm512_mask_cvtsepi64_epi32`] + * [x] [`_mm512_maskz_cvtsepi64_epi32`] + * [x] [`_mm_cvtsepi64_epi32`] + * [x] [`_mm_mask_cvtsepi64_epi32`] + * [x] [`_mm_maskz_cvtsepi64_epi32`] + * [x] [`_mm256_cvtsepi64_epi32`] + * [x] [`_mm256_mask_cvtsepi64_epi32`] + * [x] [`_mm256_maskz_cvtsepi64_epi32`] + * [x] [`_mm512_cvtsepi64_epi8`] + * [x] [`_mm512_mask_cvtsepi64_epi8`] + * [x] [`_mm512_maskz_cvtsepi64_epi8`] + * [x] [`_mm_cvtsepi64_epi8`] + * [x] [`_mm_mask_cvtsepi64_epi8`] + * [x] [`_mm_maskz_cvtsepi64_epi8`] + * [x] [`_mm256_cvtsepi64_epi8`] + * [x] [`_mm256_mask_cvtsepi64_epi8`] + * [x] [`_mm256_maskz_cvtsepi64_epi8`] + * [x] [`_mm512_mask_cvtsepi64_storeu_epi16`] + * [x] [`_mm_mask_cvtsepi64_storeu_epi16`] + * [x] [`_mm256_mask_cvtsepi64_storeu_epi16`] + * [x] [`_mm512_mask_cvtsepi64_storeu_epi32`] + * [x] [`_mm_mask_cvtsepi64_storeu_epi32`] + * [x] [`_mm256_mask_cvtsepi64_storeu_epi32`] + * [x] [`_mm512_mask_cvtsepi64_storeu_epi8`] + * [x] [`_mm_mask_cvtsepi64_storeu_epi8`] + * [x] [`_mm256_mask_cvtsepi64_storeu_epi8`] + * [x] [`_mm512_cvtusepi32_epi16`] + * [x] [`_mm512_mask_cvtusepi32_epi16`] + * [x] [`_mm512_maskz_cvtusepi32_epi16`] + * [x] [`_mm_cvtusepi32_epi16`] + * [x] [`_mm_mask_cvtusepi32_epi16`] + * [x] [`_mm_maskz_cvtusepi32_epi16`] + * [x] [`_mm256_cvtusepi32_epi16`] + * [x] [`_mm256_mask_cvtusepi32_epi16`] + * [x] [`_mm256_maskz_cvtusepi32_epi16`] + * [x] [`_mm512_cvtusepi32_epi8`] + * [x] [`_mm512_mask_cvtusepi32_epi8`] + * [x] [`_mm512_maskz_cvtusepi32_epi8`] + * [x] [`_mm_cvtusepi32_epi8`] + * [x] [`_mm_mask_cvtusepi32_epi8`] + * [x] [`_mm_maskz_cvtusepi32_epi8`] + * [x] [`_mm256_cvtusepi32_epi8`] + * [x] [`_mm256_mask_cvtusepi32_epi8`] + * [x] [`_mm256_maskz_cvtusepi32_epi8`] + * [x] [`_mm512_mask_cvtusepi32_storeu_epi16`] + * [x] [`_mm_mask_cvtusepi32_storeu_epi16`] + * [x] [`_mm256_mask_cvtusepi32_storeu_epi16`] + * [x] [`_mm512_mask_cvtusepi32_storeu_epi8`] + * [x] [`_mm_mask_cvtusepi32_storeu_epi8`] + * [x] [`_mm256_mask_cvtusepi32_storeu_epi8`] + * [x] [`_mm512_cvtusepi64_epi16`] + * [x] [`_mm512_mask_cvtusepi64_epi16`] + * [x] [`_mm512_maskz_cvtusepi64_epi16`] + * [x] [`_mm_cvtusepi64_epi16`] + * [x] [`_mm_mask_cvtusepi64_epi16`] + * [x] [`_mm_maskz_cvtusepi64_epi16`] + * [x] [`_mm256_cvtusepi64_epi16`] + * [x] [`_mm256_mask_cvtusepi64_epi16`] + * [x] [`_mm256_maskz_cvtusepi64_epi16`] + * [x] [`_mm512_cvtusepi64_epi32`] + * [x] [`_mm512_mask_cvtusepi64_epi32`] + * [x] [`_mm512_maskz_cvtusepi64_epi32`] + * [x] [`_mm_cvtusepi64_epi32`] + * [x] [`_mm_mask_cvtusepi64_epi32`] + * [x] [`_mm_maskz_cvtusepi64_epi32`] + * [x] [`_mm256_cvtusepi64_epi32`] + * [x] [`_mm256_mask_cvtusepi64_epi32`] + * [x] [`_mm256_maskz_cvtusepi64_epi32`] + * [x] [`_mm512_cvtusepi64_epi8`] + * [x] [`_mm512_mask_cvtusepi64_epi8`] + * [x] [`_mm512_maskz_cvtusepi64_epi8`] + * [x] [`_mm_cvtusepi64_epi8`] + * [x] [`_mm_mask_cvtusepi64_epi8`] + * [x] [`_mm_maskz_cvtusepi64_epi8`] + * [x] [`_mm256_cvtusepi64_epi8`] + * [x] [`_mm256_mask_cvtusepi64_epi8`] + * [x] [`_mm256_maskz_cvtusepi64_epi8`] + * [x] [`_mm512_mask_cvtusepi64_storeu_epi16`] + * [x] [`_mm_mask_cvtusepi64_storeu_epi16`] + * [x] [`_mm256_mask_cvtusepi64_storeu_epi16`] + * [x] [`_mm512_mask_cvtusepi64_storeu_epi32`] + * [x] [`_mm_mask_cvtusepi64_storeu_epi32`] + * [x] [`_mm256_mask_cvtusepi64_storeu_epi32`] + * [x] [`_mm512_mask_cvtusepi64_storeu_epi8`] + * [x] [`_mm_mask_cvtusepi64_storeu_epi8`] + * [x] [`_mm256_mask_cvtusepi64_storeu_epi8`] + * [x] [`_mm512_cvtsi512_si32`] + * [x] [`_mm512_cvttpd_epi32`] + * [x] [`_mm512_mask_cvttpd_epi32`] + * [x] [`_mm512_maskz_cvttpd_epi32`] + * [x] [`_mm_mask_cvttpd_epi32`] + * [x] [`_mm_maskz_cvttpd_epi32`] + * [x] [`_mm256_mask_cvttpd_epi32`] + * [x] [`_mm256_maskz_cvttpd_epi32`] + * [x] [`_mm512_cvttpd_epu32`] + * [x] [`_mm512_mask_cvttpd_epu32`] + * [x] [`_mm512_maskz_cvttpd_epu32`] + * [x] [`_mm_cvttpd_epu32`] + * [x] [`_mm_mask_cvttpd_epu32`] + * [x] [`_mm_maskz_cvttpd_epu32`] + * [x] [`_mm256_cvttpd_epu32`] + * [x] [`_mm256_mask_cvttpd_epu32`] + * [x] [`_mm256_maskz_cvttpd_epu32`] + * [x] [`_mm512_cvttps_epi32`] + * [x] [`_mm512_mask_cvttps_epi32`] + * [x] [`_mm512_maskz_cvttps_epi32`] + * [x] [`_mm_mask_cvttps_epi32`] + * [x] [`_mm_maskz_cvttps_epi32`] + * [x] [`_mm256_mask_cvttps_epi32`] + * [x] [`_mm256_maskz_cvttps_epi32`] + * [x] [`_mm512_cvttps_epu32`] + * [x] [`_mm512_mask_cvttps_epu32`] + * [x] [`_mm512_maskz_cvttps_epu32`] + * [x] [`_mm_cvttps_epu32`] + * [x] [`_mm_mask_cvttps_epu32`] + * [x] [`_mm_maskz_cvttps_epu32`] + * [x] [`_mm256_cvttps_epu32`] + * [x] [`_mm256_mask_cvttps_epu32`] + * [x] [`_mm256_maskz_cvttps_epu32`] + * [x] [`_mm512_cvt_roundepi32_ps`] + * [x] [`_mm512_mask_cvt_roundepi32_ps`] + * [x] [`_mm512_maskz_cvt_roundepi32_ps`] + * [x] [`_mm512_cvt_roundepu32_ps`] + * [x] [`_mm512_mask_cvt_roundepu32_ps`] + * [x] [`_mm512_maskz_cvt_roundepu32_ps`] + * [x] [`_mm512_cvt_roundpd_epi32`] + * [x] [`_mm512_mask_cvt_roundpd_epi32`] + * [x] [`_mm512_maskz_cvt_roundpd_epi32`] + * [x] [`_mm512_cvt_roundpd_epu32`] + * [x] [`_mm512_mask_cvt_roundpd_epu32`] + * [x] [`_mm512_maskz_cvt_roundpd_epu32`] + * [x] [`_mm512_cvt_roundpd_ps`] + * [x] [`_mm512_mask_cvt_roundpd_ps`] + * [x] [`_mm512_maskz_cvt_roundpd_ps`] + * [x] [`_mm512_cvt_roundph_ps`] + * [x] [`_mm512_mask_cvt_roundph_ps`] + * [x] [`_mm512_maskz_cvt_roundph_ps`] + * [x] [`_mm512_cvt_roundps_epi32`] + * [x] [`_mm512_mask_cvt_roundps_epi32`] + * [x] [`_mm512_maskz_cvt_roundps_epi32`] + * [x] [`_mm512_cvt_roundps_epu32`] + * [x] [`_mm512_mask_cvt_roundps_epu32`] + * [x] [`_mm512_maskz_cvt_roundps_epu32`] + * [x] [`_mm512_cvt_roundps_pd`] + * [x] [`_mm512_mask_cvt_roundps_pd`] + * [x] [`_mm512_maskz_cvt_roundps_pd`] + * [x] [`_mm512_cvtt_roundpd_epi32`] + * [x] [`_mm512_mask_cvtt_roundpd_epi32`] + * [x] [`_mm512_maskz_cvtt_roundpd_epi32`] + * [x] [`_mm512_cvtt_roundpd_epu32`] + * [x] [`_mm512_mask_cvtt_roundpd_epu32`] + * [x] [`_mm512_maskz_cvtt_roundpd_epu32`] + * [x] [`_mm512_cvtt_roundps_epi32`] + * [x] [`_mm512_mask_cvtt_roundps_epi32`] + * [x] [`_mm512_maskz_cvtt_roundps_epi32`] + * [x] [`_mm512_cvtt_roundps_epu32`] + * [x] [`_mm512_mask_cvtt_roundps_epu32`] + * [x] [`_mm512_maskz_cvtt_roundps_epu32`] + * [x] [`_mm_add_round_sd`] + * [x] [`_mm_add_round_ss`] + * [x] [`_mm_cmp_round_sd_mask`] + * [x] [`_mm_cmp_round_ss_mask`] + * [x] [`_mm_cmp_sd_mask`] + * [x] [`_mm_cmp_ss_mask`] + * [x] [`_mm_comi_round_sd`] + * [x] [`_mm_comi_round_ss`] + * [x] [`_mm_cvt_roundi32_ss`] + * [x] [`_mm_cvt_roundi64_sd`] + * [x] [`_mm_cvt_roundi64_ss`] + * [x] [`_mm_cvt_roundsd_i32`] + * [x] [`_mm_cvt_roundsd_i64`] + * [x] [`_mm_cvt_roundsd_si32`] + * [x] [`_mm_cvt_roundsd_si64`] + * [x] [`_mm_cvt_roundsd_ss`] + * [x] [`_mm_cvt_roundsd_u32`] + * [x] [`_mm_cvt_roundsd_u64`] + * [x] [`_mm_cvt_roundsi32_ss`] + * [x] [`_mm_cvt_roundsi64_sd`] + * [x] [`_mm_cvt_roundsi64_ss`] + * [x] [`_mm_cvt_roundss_i32`] + * [x] [`_mm_cvt_roundss_i64`] + * [x] [`_mm_cvt_roundss_sd`] + * [x] [`_mm_cvt_roundss_si32`] + * [x] [`_mm_cvt_roundss_si64`] + * [x] [`_mm_cvt_roundss_u32`] + * [x] [`_mm_cvt_roundss_u64`] + * [x] [`_mm_cvt_roundu32_ss`] + * [x] [`_mm_cvt_roundu64_sd`] + * [x] [`_mm_cvt_roundu64_ss`] + * [x] [`_mm_cvti32_sd`] + * [x] [`_mm_cvti32_ss`] + * [x] [`_mm_cvti64_sd`] + * [x] [`_mm_cvti64_ss`] + * [x] [`_mm_cvtsd_i32`] + * [x] [`_mm_cvtsd_i64`] + * [x] [`_mm_cvtsd_u32`] + * [x] [`_mm_cvtsd_u64`] + * [x] [`_mm_cvtss_i32`] + * [x] [`_mm_cvtss_i64`] + * [x] [`_mm_cvtss_u32`] + * [x] [`_mm_cvtss_u64`] + * [x] [`_mm_cvtt_roundsd_i32`] + * [x] [`_mm_cvtt_roundsd_i64`] + * [x] [`_mm_cvtt_roundsd_si32`] + * [x] [`_mm_cvtt_roundsd_si64`] + * [x] [`_mm_cvtt_roundsd_u32`] + * [x] [`_mm_cvtt_roundsd_u64`] + * [x] [`_mm_cvtt_roundss_i32`] + * [x] [`_mm_cvtt_roundss_i64`] + * [x] [`_mm_cvtt_roundss_si32`] + * [x] [`_mm_cvtt_roundss_si64`] + * [x] [`_mm_cvtt_roundss_u32`] + * [x] [`_mm_cvtt_roundss_u64`] + * [x] [`_mm_cvttsd_i32`] + * [x] [`_mm_cvttsd_i64`] + * [x] [`_mm_cvttsd_u32`] + * [x] [`_mm_cvttsd_u64`] + * [x] [`_mm_cvttss_i32`] + * [x] [`_mm_cvttss_i64`] + * [x] [`_mm_cvttss_u32`] + * [x] [`_mm_cvttss_u64`] + * [x] [`_mm_cvtu32_sd`] + * [x] [`_mm_cvtu32_ss`] + * [x] [`_mm_cvtu64_sd`] + * [x] [`_mm_cvtu64_ss`] + * [x] [`_mm_div_round_sd`] + * [x] [`_mm_div_round_ss`] + * [x] [`_mm_fixupimm_round_sd`] + * [x] [`_mm_fixupimm_round_ss`] + * [x] [`_mm_fixupimm_sd`] + * [x] [`_mm_fixupimm_ss`] + * [x] [`_mm_fmadd_round_sd`] + * [x] [`_mm_fmadd_round_ss`] + * [x] [`_mm_fmsub_round_sd`] + * [x] [`_mm_fmsub_round_ss`] + * [x] [`_mm_fnmadd_round_sd`] + * [x] [`_mm_fnmadd_round_ss`] + * [x] [`_mm_fnmsub_round_sd`] + * [x] [`_mm_fnmsub_round_ss`] + * [x] [`_mm_getexp_round_sd`] + * [x] [`_mm_getexp_round_ss`] + * [x] [`_mm_getexp_sd`] + * [x] [`_mm_getexp_ss`] + * [x] [`_mm_getmant_round_sd`] + * [x] [`_mm_getmant_round_ss`] + * [x] [`_mm_getmant_sd`] + * [x] [`_mm_getmant_ss`] + * [x] [`_mm_mask3_fmadd_round_sd`] + * [x] [`_mm_mask3_fmadd_round_ss`] + * [x] [`_mm_mask3_fmadd_sd`] + * [x] [`_mm_mask3_fmadd_ss`] + * [x] [`_mm_mask3_fmsub_round_sd`] + * [x] [`_mm_mask3_fmsub_round_ss`] + * [x] [`_mm_mask3_fmsub_sd`] + * [x] [`_mm_mask3_fmsub_ss`] + * [x] [`_mm_mask3_fnmadd_round_sd`] + * [x] [`_mm_mask3_fnmadd_round_ss`] + * [x] [`_mm_mask3_fnmadd_sd`] + * [x] [`_mm_mask3_fnmadd_ss`] + * [x] [`_mm_mask3_fnmsub_round_sd`] + * [x] [`_mm_mask3_fnmsub_round_ss`] + * [x] [`_mm_mask3_fnmsub_sd`] + * [x] [`_mm_mask3_fnmsub_ss`] + * [x] [`_mm_mask_add_round_sd`] + * [x] [`_mm_mask_add_round_ss`] + * [x] [`_mm_mask_add_sd`] + * [x] [`_mm_mask_add_ss`] + * [x] [`_mm_mask_cmp_round_sd_mask`] + * [x] [`_mm_mask_cmp_round_ss_mask`] + * [x] [`_mm_mask_cmp_sd_mask`] + * [x] [`_mm_mask_cmp_ss_mask`] + * [x] [`_mm_mask_cvt_roundsd_ss`] + * [x] [`_mm_mask_cvt_roundss_sd`] + * [x] [`_mm_mask_cvtsd_ss`] + * [x] [`_mm_mask_cvtss_sd`] + * [x] [`_mm_mask_div_round_sd`] + * [x] [`_mm_mask_div_round_ss`] + * [x] [`_mm_mask_div_sd`] + * [x] [`_mm_mask_div_ss`] + * [x] [`_mm_mask_fixupimm_round_sd`] + * [x] [`_mm_mask_fixupimm_round_ss`] + * [x] [`_mm_mask_fixupimm_sd`] + * [x] [`_mm_mask_fixupimm_ss`] + * [x] [`_mm_mask_fmadd_round_sd`] + * [x] [`_mm_mask_fmadd_round_ss`] + * [x] [`_mm_mask_fmadd_sd`] + * [x] [`_mm_mask_fmadd_ss`] + * [x] [`_mm_mask_fmsub_round_sd`] + * [x] [`_mm_mask_fmsub_round_ss`] + * [x] [`_mm_mask_fmsub_sd`] + * [x] [`_mm_mask_fmsub_ss`] + * [x] [`_mm_mask_fnmadd_round_sd`] + * [x] [`_mm_mask_fnmadd_round_ss`] + * [x] [`_mm_mask_fnmadd_sd`] + * [x] [`_mm_mask_fnmadd_ss`] + * [x] [`_mm_mask_fnmsub_round_sd`] + * [x] [`_mm_mask_fnmsub_round_ss`] + * [x] [`_mm_mask_fnmsub_sd`] + * [x] [`_mm_mask_fnmsub_ss`] + * [x] [`_mm_mask_getexp_round_sd`] + * [x] [`_mm_mask_getexp_round_ss`] + * [x] [`_mm_mask_getexp_sd`] + * [x] [`_mm_mask_getexp_ss`] + * [x] [`_mm_mask_getmant_round_sd`] + * [x] [`_mm_mask_getmant_round_ss`] + * [x] [`_mm_mask_getmant_sd`] + * [x] [`_mm_mask_getmant_ss`] + * [ ] [`_mm_mask_load_sd`] //need i1 + * [ ] [`_mm_mask_load_ss`] //need i1 + * [x] [`_mm_mask_max_round_sd`] + * [x] [`_mm_mask_max_round_ss`] + * [x] [`_mm_mask_max_sd`] + * [x] [`_mm_mask_max_ss`] + * [x] [`_mm_mask_min_round_sd`] + * [x] [`_mm_mask_min_round_ss`] + * [x] [`_mm_mask_min_sd`] + * [x] [`_mm_mask_min_ss`] + * [x] [`_mm_mask_move_sd`] + * [x] [`_mm_mask_move_ss`] + * [x] [`_mm_mask_mul_round_sd`] + * [x] [`_mm_mask_mul_round_ss`] + * [x] [`_mm_mask_mul_sd`] + * [x] [`_mm_mask_mul_ss`] + * [x] [`_mm_mask_rcp14_sd`] + * [x] [`_mm_mask_rcp14_ss`] + * [x] [`_mm_mask_roundscale_round_sd`] + * [x] [`_mm_mask_roundscale_round_ss`] + * [x] [`_mm_mask_roundscale_sd`] + * [x] [`_mm_mask_roundscale_ss`] + * [x] [`_mm_mask_rsqrt14_sd`] + * [x] [`_mm_mask_rsqrt14_ss`] + * [x] [`_mm_mask_scalef_round_sd`] + * [x] [`_mm_mask_scalef_round_ss`] + * [x] [`_mm_mask_scalef_sd`] + * [x] [`_mm_mask_scalef_ss`] + * [x] [`_mm_mask_sqrt_round_sd`] + * [x] [`_mm_mask_sqrt_round_ss`] + * [x] [`_mm_mask_sqrt_sd`] + * [x] [`_mm_mask_sqrt_ss`] + * [ ] [`_mm_mask_store_sd`] //need i1 + * [ ] [`_mm_mask_store_ss`] //need i1 + * [x] [`_mm_mask_sub_round_sd`] + * [x] [`_mm_mask_sub_round_ss`] + * [x] [`_mm_mask_sub_sd`] + * [x] [`_mm_mask_sub_ss`] + * [x] [`_mm_maskz_add_round_sd`] + * [x] [`_mm_maskz_add_round_ss`] + * [x] [`_mm_maskz_add_sd`] + * [x] [`_mm_maskz_add_ss`] + * [x] [`_mm_maskz_cvt_roundsd_ss`] + * [x] [`_mm_maskz_cvt_roundss_sd`] + * [x] [`_mm_maskz_cvtsd_ss`] + * [x] [`_mm_maskz_cvtss_sd`] + * [x] [`_mm_maskz_div_round_sd`] + * [x] [`_mm_maskz_div_round_ss`] + * [x] [`_mm_maskz_div_sd`] + * [x] [`_mm_maskz_div_ss`] + * [x] [`_mm_maskz_fixupimm_round_sd`] + * [x] [`_mm_maskz_fixupimm_round_ss`] + * [x] [`_mm_maskz_fixupimm_sd`] + * [x] [`_mm_maskz_fixupimm_ss`] + * [x] [`_mm_maskz_fmadd_round_sd`] + * [x] [`_mm_maskz_fmadd_round_ss`] + * [x] [`_mm_maskz_fmadd_sd`] + * [x] [`_mm_maskz_fmadd_ss`] + * [x] [`_mm_maskz_fmsub_round_sd`] + * [x] [`_mm_maskz_fmsub_round_ss`] + * [x] [`_mm_maskz_fmsub_sd`] + * [x] [`_mm_maskz_fmsub_ss`] + * [x] [`_mm_maskz_fnmadd_round_sd`] + * [x] [`_mm_maskz_fnmadd_round_ss`] + * [x] [`_mm_maskz_fnmadd_sd`] + * [x] [`_mm_maskz_fnmadd_ss`] + * [x] [`_mm_maskz_fnmsub_round_sd`] + * [x] [`_mm_maskz_fnmsub_round_ss`] + * [x] [`_mm_maskz_fnmsub_sd`] + * [x] [`_mm_maskz_fnmsub_ss`] + * [x] [`_mm_maskz_getexp_round_sd`] + * [x] [`_mm_maskz_getexp_round_ss`] + * [x] [`_mm_maskz_getexp_sd`] + * [x] [`_mm_maskz_getexp_ss`] + * [x] [`_mm_maskz_getmant_round_sd`] + * [x] [`_mm_maskz_getmant_round_ss`] + * [x] [`_mm_maskz_getmant_sd`] + * [x] [`_mm_maskz_getmant_ss`] + * [ ] [`_mm_maskz_load_sd`] //need i1 + * [ ] [`_mm_maskz_load_ss`] //need i1 + * [x] [`_mm_maskz_max_round_sd`] + * [x] [`_mm_maskz_max_round_ss`] + * [x] [`_mm_maskz_max_sd`] + * [x] [`_mm_maskz_max_ss`] + * [x] [`_mm_maskz_min_round_sd`] + * [x] [`_mm_maskz_min_round_ss`] + * [x] [`_mm_maskz_min_sd`] + * [x] [`_mm_maskz_min_ss`] + * [x] [`_mm_maskz_move_sd`] + * [x] [`_mm_maskz_move_ss`] + * [x] [`_mm_maskz_mul_round_sd`] + * [x] [`_mm_maskz_mul_round_ss`] + * [x] [`_mm_maskz_mul_sd`] + * [x] [`_mm_maskz_mul_ss`] + * [x] [`_mm_maskz_rcp14_sd`] + * [x] [`_mm_maskz_rcp14_ss`] + * [x] [`_mm_maskz_roundscale_round_sd`] + * [x] [`_mm_maskz_roundscale_round_ss`] + * [x] [`_mm_maskz_roundscale_sd`] + * [x] [`_mm_maskz_roundscale_ss`] + * [x] [`_mm_maskz_rsqrt14_sd`] + * [x] [`_mm_maskz_rsqrt14_ss`] + * [x] [`_mm_maskz_scalef_round_sd`] + * [x] [`_mm_maskz_scalef_round_ss`] + * [x] [`_mm_maskz_scalef_sd`] + * [x] [`_mm_maskz_scalef_ss`] + * [x] [`_mm_maskz_sqrt_round_sd`] + * [x] [`_mm_maskz_sqrt_round_ss`] + * [x] [`_mm_maskz_sqrt_sd`] + * [x] [`_mm_maskz_sqrt_ss`] + * [x] [`_mm_maskz_sub_round_sd`] + * [x] [`_mm_maskz_sub_round_ss`] + * [x] [`_mm_maskz_sub_sd`] + * [x] [`_mm_maskz_sub_ss`] + * [x] [`_mm_max_round_sd`] + * [x] [`_mm_max_round_ss`] + * [x] [`_mm_min_round_sd`] + * [x] [`_mm_min_round_ss`] + * [x] [`_mm_mul_round_sd`] + * [x] [`_mm_mul_round_ss`] + * [x] [`_mm_rcp14_sd`] + * [x] [`_mm_rcp14_ss`] + * [x] [`_mm_roundscale_round_sd`] + * [x] [`_mm_roundscale_round_ss`] + * [x] [`_mm_roundscale_sd`] + * [x] [`_mm_roundscale_ss`] + * [x] [`_mm_rsqrt14_sd`] + * [x] [`_mm_rsqrt14_ss`] + * [x] [`_mm_scalef_round_sd`] + * [x] [`_mm_scalef_round_ss`] + * [x] [`_mm_scalef_sd`] + * [x] [`_mm_scalef_ss`] + * [x] [`_mm_sqrt_round_sd`] + * [x] [`_mm_sqrt_round_ss`] + * [x] [`_mm_sub_round_sd`] + * [x] [`_mm_sub_round_ss`] + * [x] [`_mm512_int2mask`] + * [x] [`_mm512_kand`] + * [x] [`_mm512_kandn`] + * [x] [`_mm512_kmov`] + * [x] [`_mm512_knot`] + * [x] [`_mm512_kor`] + * [x] [`_mm512_kortestc`] + * [ ] [`_mm512_kortestz`] //not sure + * [x] [`_mm512_kunpackb`] + * [x] [`_mm512_kxnor`] + * [x] [`_mm512_kxor`] + * [x] [`_mm512_mask2int`] +

diff --git a/crux-mir/lib/stdarch/crates/core_arch/build.rs b/crux-mir/lib/stdarch/crates/core_arch/build.rs deleted file mode 100644 index 4d65e9ddc..000000000 --- a/crux-mir/lib/stdarch/crates/core_arch/build.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - println!("cargo:rustc-cfg=core_arch_docs"); -} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/armclang.rs b/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/armclang.rs new file mode 100644 index 000000000..7ad6ae50c --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/armclang.rs @@ -0,0 +1,23 @@ +//! ARM compiler specific intrinsics +//! +//! # References +//! +//! - [ARM Compiler v 6.10 - armclang Reference Guide][arm_comp_ref] +//! +//! [arm_comp_ref]: https://developer.arm.com/docs/100067/0610 + +#[cfg(test)] +use stdarch_test::assert_instr; + +/// Inserts a breakpoint instruction. +/// +/// `VAL` is a compile-time constant integer in range `[0, 65535]`. +/// +/// The breakpoint instruction inserted is `BRK` on A64. +#[cfg_attr(test, assert_instr(brk, VAL = 0))] +#[inline(always)] +#[rustc_legacy_const_generics(0)] +pub unsafe fn __breakpoint() { + static_assert_imm16!(VAL); + crate::arch::asm!("brk {}", const VAL); +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/crc.rs b/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/crc.rs index 32dddab81..ac3f8d815 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/crc.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/crc.rs @@ -1,19 +1,7 @@ -extern "C" { - #[link_name = "llvm.aarch64.crc32b"] - fn crc32b_(crc: u32, data: u32) -> u32; - #[link_name = "llvm.aarch64.crc32h"] - fn crc32h_(crc: u32, data: u32) -> u32; - #[link_name = "llvm.aarch64.crc32w"] - fn crc32w_(crc: u32, data: u32) -> u32; +extern "unadjusted" { #[link_name = "llvm.aarch64.crc32x"] fn crc32x_(crc: u32, data: u64) -> u32; - #[link_name = "llvm.aarch64.crc32cb"] - fn crc32cb_(crc: u32, data: u32) -> u32; - #[link_name = "llvm.aarch64.crc32ch"] - fn crc32ch_(crc: u32, data: u32) -> u32; - #[link_name = "llvm.aarch64.crc32cw"] - fn crc32cw_(crc: u32, data: u32) -> u32; #[link_name = "llvm.aarch64.crc32cx"] fn crc32cx_(crc: u32, data: u64) -> u32; } @@ -21,31 +9,9 @@ extern "C" { #[cfg(test)] use stdarch_test::assert_instr; -/// CRC32 single round checksum for bytes (8 bits). -#[inline] -#[target_feature(enable = "crc")] -#[cfg_attr(test, assert_instr(crc32b))] -pub unsafe fn __crc32b(crc: u32, data: u8) -> u32 { - crc32b_(crc, data as u32) -} - -/// CRC32 single round checksum for half words (16 bits). -#[inline] -#[target_feature(enable = "crc")] -#[cfg_attr(test, assert_instr(crc32h))] -pub unsafe fn __crc32h(crc: u32, data: u16) -> u32 { - crc32h_(crc, data as u32) -} - -/// CRC32 single round checksum for words (32 bits). -#[inline] -#[target_feature(enable = "crc")] -#[cfg_attr(test, assert_instr(crc32w))] -pub unsafe fn __crc32w(crc: u32, data: u32) -> u32 { - crc32w_(crc, data) -} - /// CRC32 single round checksum for quad words (64 bits). +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/__crc32d) #[inline] #[target_feature(enable = "crc")] #[cfg_attr(test, assert_instr(crc32x))] @@ -53,31 +19,9 @@ pub unsafe fn __crc32d(crc: u32, data: u64) -> u32 { crc32x_(crc, data) } -/// CRC32-C single round checksum for bytes (8 bits). -#[inline] -#[target_feature(enable = "crc")] -#[cfg_attr(test, assert_instr(crc32cb))] -pub unsafe fn __crc32cb(crc: u32, data: u8) -> u32 { - crc32cb_(crc, data as u32) -} - -/// CRC32-C single round checksum for half words (16 bits). -#[inline] -#[target_feature(enable = "crc")] -#[cfg_attr(test, assert_instr(crc32ch))] -pub unsafe fn __crc32ch(crc: u32, data: u16) -> u32 { - crc32ch_(crc, data as u32) -} - -/// CRC32-C single round checksum for words (32 bits). -#[inline] -#[target_feature(enable = "crc")] -#[cfg_attr(test, assert_instr(crc32cw))] -pub unsafe fn __crc32cw(crc: u32, data: u32) -> u32 { - crc32cw_(crc, data) -} - /// CRC32-C single round checksum for quad words (64 bits). +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/__crc32cd) #[inline] #[target_feature(enable = "crc")] #[cfg_attr(test, assert_instr(crc32cx))] @@ -91,48 +35,12 @@ mod tests { use std::mem; use stdarch_test::simd_test; - #[simd_test(enable = "crc")] - unsafe fn test_crc32b() { - assert_eq!(__crc32b(0, 0), 0); - assert_eq!(__crc32b(0, 255), 755167117); - } - - #[simd_test(enable = "crc")] - unsafe fn test_crc32h() { - assert_eq!(__crc32h(0, 0), 0); - assert_eq!(__crc32h(0, 16384), 1994146192); - } - - #[simd_test(enable = "crc")] - unsafe fn test_crc32w() { - assert_eq!(__crc32w(0, 0), 0); - assert_eq!(__crc32w(0, 4294967295), 3736805603); - } - #[simd_test(enable = "crc")] unsafe fn test_crc32d() { assert_eq!(__crc32d(0, 0), 0); assert_eq!(__crc32d(0, 18446744073709551615), 1147535477); } - #[simd_test(enable = "crc")] - unsafe fn test_crc32cb() { - assert_eq!(__crc32cb(0, 0), 0); - assert_eq!(__crc32cb(0, 255), 2910671697); - } - - #[simd_test(enable = "crc")] - unsafe fn test_crc32ch() { - assert_eq!(__crc32ch(0, 0), 0); - assert_eq!(__crc32ch(0, 16384), 1098587580); - } - - #[simd_test(enable = "crc")] - unsafe fn test_crc32cw() { - assert_eq!(__crc32cw(0, 0), 0); - assert_eq!(__crc32cw(0, 4294967295), 3080238136); - } - #[simd_test(enable = "crc")] unsafe fn test_crc32cd() { assert_eq!(__crc32cd(0, 0), 0); diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/mod.rs b/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/mod.rs index e33dc7eaf..0411fc106 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/mod.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/mod.rs @@ -12,13 +12,20 @@ pub use self::v8::*; mod neon; pub use self::neon::*; -mod crypto; -pub use self::crypto::*; +mod tme; +pub use self::tme::*; mod crc; pub use self::crc::*; -pub use super::acle::*; +mod prefetch; +pub use self::prefetch::*; + +pub use super::arm_shared::*; + +mod armclang; + +pub use self::armclang::*; #[cfg(test)] use stdarch_test::assert_instr; @@ -29,3 +36,6 @@ use stdarch_test::assert_instr; pub unsafe fn brk() -> ! { crate::intrinsics::abort() } + +#[cfg(test)] +pub(crate) mod test_support; diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/neon.rs b/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/neon.rs deleted file mode 100644 index 2ddd97273..000000000 --- a/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/neon.rs +++ /dev/null @@ -1,1988 +0,0 @@ -//! ARMv8 ASIMD intrinsics - -#![allow(non_camel_case_types)] - -// FIXME: replace neon with asimd - -use crate::{ - core_arch::{arm::*, simd_llvm::*}, - mem::{transmute, zeroed}, -}; -#[cfg(test)] -use stdarch_test::assert_instr; - -types! { - /// ARM-specific 64-bit wide vector of one packed `f64`. - pub struct float64x1_t(f64); // FIXME: check this! - /// ARM-specific 128-bit wide vector of two packed `f64`. - pub struct float64x2_t(f64, f64); - /// ARM-specific 64-bit wide vector of one packed `p64`. - pub struct poly64x1_t(i64); // FIXME: check this! - /// ARM-specific 64-bit wide vector of two packed `p64`. - pub struct poly64x2_t(i64, i64); // FIXME: check this! -} - -/// ARM-specific type containing two `int8x16_t` vectors. -#[derive(Copy, Clone)] -pub struct int8x16x2_t(pub int8x16_t, pub int8x16_t); -/// ARM-specific type containing three `int8x16_t` vectors. -#[derive(Copy, Clone)] -pub struct int8x16x3_t(pub int8x16_t, pub int8x16_t, pub int8x16_t); -/// ARM-specific type containing four `int8x16_t` vectors. -#[derive(Copy, Clone)] -pub struct int8x16x4_t(pub int8x16_t, pub int8x16_t, pub int8x16_t, pub int8x16_t); - -/// ARM-specific type containing two `uint8x16_t` vectors. -#[derive(Copy, Clone)] -pub struct uint8x16x2_t(pub uint8x16_t, pub uint8x16_t); -/// ARM-specific type containing three `uint8x16_t` vectors. -#[derive(Copy, Clone)] -pub struct uint8x16x3_t(pub uint8x16_t, pub uint8x16_t, pub uint8x16_t); -/// ARM-specific type containing four `uint8x16_t` vectors. -#[derive(Copy, Clone)] -pub struct uint8x16x4_t( - pub uint8x16_t, - pub uint8x16_t, - pub uint8x16_t, - pub uint8x16_t, -); - -/// ARM-specific type containing two `poly8x16_t` vectors. -#[derive(Copy, Clone)] -pub struct poly8x16x2_t(pub poly8x16_t, pub poly8x16_t); -/// ARM-specific type containing three `poly8x16_t` vectors. -#[derive(Copy, Clone)] -pub struct poly8x16x3_t(pub poly8x16_t, pub poly8x16_t, pub poly8x16_t); -/// ARM-specific type containing four `poly8x16_t` vectors. -#[derive(Copy, Clone)] -pub struct poly8x16x4_t( - pub poly8x16_t, - pub poly8x16_t, - pub poly8x16_t, - pub poly8x16_t, -); - -#[allow(improper_ctypes)] -extern "C" { - #[link_name = "llvm.aarch64.neon.smaxv.i8.v8i8"] - fn vmaxv_s8_(a: int8x8_t) -> i8; - #[link_name = "llvm.aarch64.neon.smaxv.i8.6i8"] - fn vmaxvq_s8_(a: int8x16_t) -> i8; - #[link_name = "llvm.aarch64.neon.smaxv.i16.v4i16"] - fn vmaxv_s16_(a: int16x4_t) -> i16; - #[link_name = "llvm.aarch64.neon.smaxv.i16.v8i16"] - fn vmaxvq_s16_(a: int16x8_t) -> i16; - #[link_name = "llvm.aarch64.neon.smaxv.i32.v2i32"] - fn vmaxv_s32_(a: int32x2_t) -> i32; - #[link_name = "llvm.aarch64.neon.smaxv.i32.v4i32"] - fn vmaxvq_s32_(a: int32x4_t) -> i32; - - #[link_name = "llvm.aarch64.neon.umaxv.i8.v8i8"] - fn vmaxv_u8_(a: uint8x8_t) -> u8; - #[link_name = "llvm.aarch64.neon.umaxv.i8.6i8"] - fn vmaxvq_u8_(a: uint8x16_t) -> u8; - #[link_name = "llvm.aarch64.neon.umaxv.i16.v4i16"] - fn vmaxv_u16_(a: uint16x4_t) -> u16; - #[link_name = "llvm.aarch64.neon.umaxv.i16.v8i16"] - fn vmaxvq_u16_(a: uint16x8_t) -> u16; - #[link_name = "llvm.aarch64.neon.umaxv.i32.v2i32"] - fn vmaxv_u32_(a: uint32x2_t) -> u32; - #[link_name = "llvm.aarch64.neon.umaxv.i32.v4i32"] - fn vmaxvq_u32_(a: uint32x4_t) -> u32; - - #[link_name = "llvm.aarch64.neon.fmaxv.f32.v2f32"] - fn vmaxv_f32_(a: float32x2_t) -> f32; - #[link_name = "llvm.aarch64.neon.fmaxv.f32.v4f32"] - fn vmaxvq_f32_(a: float32x4_t) -> f32; - #[link_name = "llvm.aarch64.neon.fmaxv.f64.v2f64"] - fn vmaxvq_f64_(a: float64x2_t) -> f64; - - #[link_name = "llvm.aarch64.neon.sminv.i8.v8i8"] - fn vminv_s8_(a: int8x8_t) -> i8; - #[link_name = "llvm.aarch64.neon.sminv.i8.6i8"] - fn vminvq_s8_(a: int8x16_t) -> i8; - #[link_name = "llvm.aarch64.neon.sminv.i16.v4i16"] - fn vminv_s16_(a: int16x4_t) -> i16; - #[link_name = "llvm.aarch64.neon.sminv.i16.v8i16"] - fn vminvq_s16_(a: int16x8_t) -> i16; - #[link_name = "llvm.aarch64.neon.sminv.i32.v2i32"] - fn vminv_s32_(a: int32x2_t) -> i32; - #[link_name = "llvm.aarch64.neon.sminv.i32.v4i32"] - fn vminvq_s32_(a: int32x4_t) -> i32; - - #[link_name = "llvm.aarch64.neon.uminv.i8.v8i8"] - fn vminv_u8_(a: uint8x8_t) -> u8; - #[link_name = "llvm.aarch64.neon.uminv.i8.6i8"] - fn vminvq_u8_(a: uint8x16_t) -> u8; - #[link_name = "llvm.aarch64.neon.uminv.i16.v4i16"] - fn vminv_u16_(a: uint16x4_t) -> u16; - #[link_name = "llvm.aarch64.neon.uminv.i16.v8i16"] - fn vminvq_u16_(a: uint16x8_t) -> u16; - #[link_name = "llvm.aarch64.neon.uminv.i32.v2i32"] - fn vminv_u32_(a: uint32x2_t) -> u32; - #[link_name = "llvm.aarch64.neon.uminv.i32.v4i32"] - fn vminvq_u32_(a: uint32x4_t) -> u32; - - #[link_name = "llvm.aarch64.neon.fminv.f32.v2f32"] - fn vminv_f32_(a: float32x2_t) -> f32; - #[link_name = "llvm.aarch64.neon.fminv.f32.v4f32"] - fn vminvq_f32_(a: float32x4_t) -> f32; - #[link_name = "llvm.aarch64.neon.fminv.f64.v2f64"] - fn vminvq_f64_(a: float64x2_t) -> f64; - - #[link_name = "llvm.aarch64.neon.sminp.v16i8"] - fn vpminq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; - #[link_name = "llvm.aarch64.neon.sminp.v8i16"] - fn vpminq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; - #[link_name = "llvm.aarch64.neon.sminp.v4i32"] - fn vpminq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; - #[link_name = "llvm.aarch64.neon.uminp.v16i8"] - fn vpminq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; - #[link_name = "llvm.aarch64.neon.uminp.v8i16"] - fn vpminq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; - #[link_name = "llvm.aarch64.neon.uminp.v4i32"] - fn vpminq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; - #[link_name = "llvm.aarch64.neon.fminp.4f32"] - fn vpminq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; - #[link_name = "llvm.aarch64.neon.fminp.v2f64"] - fn vpminq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; - - #[link_name = "llvm.aarch64.neon.smaxp.v16i8"] - fn vpmaxq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; - #[link_name = "llvm.aarch64.neon.smaxp.v8i16"] - fn vpmaxq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; - #[link_name = "llvm.aarch64.neon.smaxp.v4i32"] - fn vpmaxq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; - #[link_name = "llvm.aarch64.neon.umaxp.v16i8"] - fn vpmaxq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; - #[link_name = "llvm.aarch64.neon.umaxp.v8i16"] - fn vpmaxq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; - #[link_name = "llvm.aarch64.neon.umaxp.v4i32"] - fn vpmaxq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; - #[link_name = "llvm.aarch64.neon.fmaxp.4f32"] - fn vpmaxq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; - #[link_name = "llvm.aarch64.neon.fmaxp.v2f64"] - fn vpmaxq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; - - #[link_name = "llvm.aarch64.neon.tbl1.v8i8"] - fn vqtbl1(a: int8x16_t, b: uint8x8_t) -> int8x8_t; - #[link_name = "llvm.aarch64.neon.tbl1.v16i8"] - fn vqtbl1q(a: int8x16_t, b: uint8x16_t) -> int8x16_t; - - #[link_name = "llvm.aarch64.neon.tbx1.v8i8"] - fn vqtbx1(a: int8x8_t, b: int8x16_t, c: uint8x8_t) -> int8x8_t; - #[link_name = "llvm.aarch64.neon.tbx1.v16i8"] - fn vqtbx1q(a: int8x16_t, b: int8x16_t, c: uint8x16_t) -> int8x16_t; - - #[link_name = "llvm.aarch64.neon.tbl2.v8i8"] - fn vqtbl2(a0: int8x16_t, a1: int8x16_t, b: uint8x8_t) -> int8x8_t; - #[link_name = "llvm.aarch64.neon.tbl2.v16i8"] - fn vqtbl2q(a0: int8x16_t, a1: int8x16_t, b: uint8x16_t) -> int8x16_t; - - #[link_name = "llvm.aarch64.neon.tbx2.v8i8"] - fn vqtbx2(a: int8x8_t, b0: int8x16_t, b1: int8x16_t, c: uint8x8_t) -> int8x8_t; - #[link_name = "llvm.aarch64.neon.tbx2.v16i8"] - fn vqtbx2q(a: int8x16_t, b0: int8x16_t, b1: int8x16_t, c: uint8x16_t) -> int8x16_t; - - #[link_name = "llvm.aarch64.neon.tbl3.v8i8"] - fn vqtbl3(a0: int8x16_t, a1: int8x16_t, a2: int8x16_t, b: uint8x8_t) -> int8x8_t; - #[link_name = "llvm.aarch64.neon.tbl3.v16i8"] - fn vqtbl3q(a0: int8x16_t, a1: int8x16_t, a2: int8x16_t, b: uint8x16_t) -> int8x16_t; - - #[link_name = "llvm.aarch64.neon.tbx3.v8i8"] - fn vqtbx3(a: int8x8_t, b0: int8x16_t, b1: int8x16_t, b2: int8x16_t, c: uint8x8_t) -> int8x8_t; - #[link_name = "llvm.aarch64.neon.tbx3.v16i8"] - fn vqtbx3q( - a: int8x16_t, - b0: int8x16_t, - b1: int8x16_t, - b2: int8x16_t, - c: uint8x16_t, - ) -> int8x16_t; - - #[link_name = "llvm.aarch64.neon.tbl4.v8i8"] - fn vqtbl4(a0: int8x16_t, a1: int8x16_t, a2: int8x16_t, a3: int8x16_t, b: uint8x8_t) - -> int8x8_t; - #[link_name = "llvm.aarch64.neon.tbl4.v16i8"] - fn vqtbl4q( - a0: int8x16_t, - a1: int8x16_t, - a2: int8x16_t, - a3: int8x16_t, - b: uint8x16_t, - ) -> int8x16_t; - - #[link_name = "llvm.aarch64.neon.tbx4.v8i8"] - fn vqtbx4( - a: int8x8_t, - b0: int8x16_t, - b1: int8x16_t, - b2: int8x16_t, - b3: int8x16_t, - c: uint8x8_t, - ) -> int8x8_t; - #[link_name = "llvm.aarch64.neon.tbx4.v16i8"] - fn vqtbx4q( - a: int8x16_t, - b0: int8x16_t, - b1: int8x16_t, - b2: int8x16_t, - b3: int8x16_t, - c: uint8x16_t, - ) -> int8x16_t; -} - -/// Vector add. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fadd))] -pub unsafe fn vadd_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { - simd_add(a, b) -} - -/// Vector add. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fadd))] -pub unsafe fn vaddq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { - simd_add(a, b) -} - -/// Vector add. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(add))] -pub unsafe fn vaddd_s64(a: i64, b: i64) -> i64 { - let a: int64x1_t = transmute(a); - let b: int64x1_t = transmute(b); - simd_extract(simd_add(a, b), 0) -} - -/// Vector add. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(add))] -pub unsafe fn vaddd_u64(a: u64, b: u64) -> u64 { - let a: uint64x1_t = transmute(a); - let b: uint64x1_t = transmute(b); - simd_extract(simd_add(a, b), 0) -} - -/// Horizontal vector max. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(smaxv))] -pub unsafe fn vmaxv_s8(a: int8x8_t) -> i8 { - vmaxv_s8_(a) -} - -/// Horizontal vector max. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(smaxv))] -pub unsafe fn vmaxvq_s8(a: int8x16_t) -> i8 { - vmaxvq_s8_(a) -} - -/// Horizontal vector max. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(smaxv))] -pub unsafe fn vmaxv_s16(a: int16x4_t) -> i16 { - vmaxv_s16_(a) -} - -/// Horizontal vector max. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(smaxv))] -pub unsafe fn vmaxvq_s16(a: int16x8_t) -> i16 { - vmaxvq_s16_(a) -} - -/// Horizontal vector max. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(smaxp))] -pub unsafe fn vmaxv_s32(a: int32x2_t) -> i32 { - vmaxv_s32_(a) -} - -/// Horizontal vector max. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(smaxv))] -pub unsafe fn vmaxvq_s32(a: int32x4_t) -> i32 { - vmaxvq_s32_(a) -} - -/// Horizontal vector max. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(umaxv))] -pub unsafe fn vmaxv_u8(a: uint8x8_t) -> u8 { - vmaxv_u8_(a) -} - -/// Horizontal vector max. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(umaxv))] -pub unsafe fn vmaxvq_u8(a: uint8x16_t) -> u8 { - vmaxvq_u8_(a) -} - -/// Horizontal vector max. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(umaxv))] -pub unsafe fn vmaxv_u16(a: uint16x4_t) -> u16 { - vmaxv_u16_(a) -} - -/// Horizontal vector max. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(umaxv))] -pub unsafe fn vmaxvq_u16(a: uint16x8_t) -> u16 { - vmaxvq_u16_(a) -} - -/// Horizontal vector max. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(umaxp))] -pub unsafe fn vmaxv_u32(a: uint32x2_t) -> u32 { - vmaxv_u32_(a) -} - -/// Horizontal vector max. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(umaxv))] -pub unsafe fn vmaxvq_u32(a: uint32x4_t) -> u32 { - vmaxvq_u32_(a) -} - -/// Horizontal vector max. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fmaxp))] -pub unsafe fn vmaxv_f32(a: float32x2_t) -> f32 { - vmaxv_f32_(a) -} - -/// Horizontal vector max. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fmaxv))] -pub unsafe fn vmaxvq_f32(a: float32x4_t) -> f32 { - vmaxvq_f32_(a) -} - -/// Horizontal vector max. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fmaxp))] -pub unsafe fn vmaxvq_f64(a: float64x2_t) -> f64 { - vmaxvq_f64_(a) -} - -/// Horizontal vector min. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sminv))] -pub unsafe fn vminv_s8(a: int8x8_t) -> i8 { - vminv_s8_(a) -} - -/// Horizontal vector min. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sminv))] -pub unsafe fn vminvq_s8(a: int8x16_t) -> i8 { - vminvq_s8_(a) -} - -/// Horizontal vector min. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sminv))] -pub unsafe fn vminv_s16(a: int16x4_t) -> i16 { - vminv_s16_(a) -} - -/// Horizontal vector min. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sminv))] -pub unsafe fn vminvq_s16(a: int16x8_t) -> i16 { - vminvq_s16_(a) -} - -/// Horizontal vector min. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sminp))] -pub unsafe fn vminv_s32(a: int32x2_t) -> i32 { - vminv_s32_(a) -} - -/// Horizontal vector min. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sminv))] -pub unsafe fn vminvq_s32(a: int32x4_t) -> i32 { - vminvq_s32_(a) -} - -/// Horizontal vector min. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(uminv))] -pub unsafe fn vminv_u8(a: uint8x8_t) -> u8 { - vminv_u8_(a) -} - -/// Horizontal vector min. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(uminv))] -pub unsafe fn vminvq_u8(a: uint8x16_t) -> u8 { - vminvq_u8_(a) -} - -/// Horizontal vector min. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(uminv))] -pub unsafe fn vminv_u16(a: uint16x4_t) -> u16 { - vminv_u16_(a) -} - -/// Horizontal vector min. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(uminv))] -pub unsafe fn vminvq_u16(a: uint16x8_t) -> u16 { - vminvq_u16_(a) -} - -/// Horizontal vector min. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(uminp))] -pub unsafe fn vminv_u32(a: uint32x2_t) -> u32 { - vminv_u32_(a) -} - -/// Horizontal vector min. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(uminv))] -pub unsafe fn vminvq_u32(a: uint32x4_t) -> u32 { - vminvq_u32_(a) -} - -/// Horizontal vector min. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fminp))] -pub unsafe fn vminv_f32(a: float32x2_t) -> f32 { - vminv_f32_(a) -} - -/// Horizontal vector min. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fminv))] -pub unsafe fn vminvq_f32(a: float32x4_t) -> f32 { - vminvq_f32_(a) -} - -/// Horizontal vector min. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fminp))] -pub unsafe fn vminvq_f64(a: float64x2_t) -> f64 { - vminvq_f64_(a) -} - -/// Folding minimum of adjacent pairs -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sminp))] -pub unsafe fn vpminq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { - vpminq_s8_(a, b) -} - -/// Folding minimum of adjacent pairs -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sminp))] -pub unsafe fn vpminq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { - vpminq_s16_(a, b) -} - -/// Folding minimum of adjacent pairs -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sminp))] -pub unsafe fn vpminq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { - vpminq_s32_(a, b) -} - -/// Folding minimum of adjacent pairs -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(uminp))] -pub unsafe fn vpminq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { - vpminq_u8_(a, b) -} - -/// Folding minimum of adjacent pairs -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(uminp))] -pub unsafe fn vpminq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { - vpminq_u16_(a, b) -} - -/// Folding minimum of adjacent pairs -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(uminp))] -pub unsafe fn vpminq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { - vpminq_u32_(a, b) -} - -/// Folding minimum of adjacent pairs -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fminp))] -pub unsafe fn vpminq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { - vpminq_f32_(a, b) -} - -/// Folding minimum of adjacent pairs -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fminp))] -pub unsafe fn vpminq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { - vpminq_f64_(a, b) -} - -/// Folding maximum of adjacent pairs -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(smaxp))] -pub unsafe fn vpmaxq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { - vpmaxq_s8_(a, b) -} - -/// Folding maximum of adjacent pairs -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(smaxp))] -pub unsafe fn vpmaxq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { - vpmaxq_s16_(a, b) -} - -/// Folding maximum of adjacent pairs -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(smaxp))] -pub unsafe fn vpmaxq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { - vpmaxq_s32_(a, b) -} - -/// Folding maximum of adjacent pairs -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(umaxp))] -pub unsafe fn vpmaxq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { - vpmaxq_u8_(a, b) -} - -/// Folding maximum of adjacent pairs -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(umaxp))] -pub unsafe fn vpmaxq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { - vpmaxq_u16_(a, b) -} - -/// Folding maximum of adjacent pairs -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(umaxp))] -pub unsafe fn vpmaxq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { - vpmaxq_u32_(a, b) -} - -/// Folding maximum of adjacent pairs -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fmaxp))] -pub unsafe fn vpmaxq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { - vpmaxq_f32_(a, b) -} - -/// Folding maximum of adjacent pairs -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fmaxp))] -pub unsafe fn vpmaxq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { - vpmaxq_f64_(a, b) -} - -/// Vector combine -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(mov))] -pub unsafe fn vcombine_s8(low: int8x8_t, high: int8x8_t) -> int8x16_t { - simd_shuffle16( - low, - high, - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - ) -} - -/// Vector combine -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(mov))] -pub unsafe fn vcombine_s16(low: int16x4_t, high: int16x4_t) -> int16x8_t { - simd_shuffle8(low, high, [0, 1, 2, 3, 4, 5, 6, 7]) -} - -/// Vector combine -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(mov))] -pub unsafe fn vcombine_s32(low: int32x2_t, high: int32x2_t) -> int32x4_t { - simd_shuffle4(low, high, [0, 1, 2, 3]) -} - -/// Vector combine -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(mov))] -pub unsafe fn vcombine_s64(low: int64x1_t, high: int64x1_t) -> int64x2_t { - simd_shuffle2(low, high, [0, 1]) -} - -/// Vector combine -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(mov))] -pub unsafe fn vcombine_u8(low: uint8x8_t, high: uint8x8_t) -> uint8x16_t { - simd_shuffle16( - low, - high, - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - ) -} - -/// Vector combine -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(mov))] -pub unsafe fn vcombine_u16(low: uint16x4_t, high: uint16x4_t) -> uint16x8_t { - simd_shuffle8(low, high, [0, 1, 2, 3, 4, 5, 6, 7]) -} - -/// Vector combine -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(mov))] -pub unsafe fn vcombine_u32(low: uint32x2_t, high: uint32x2_t) -> uint32x4_t { - simd_shuffle4(low, high, [0, 1, 2, 3]) -} - -/// Vector combine -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(mov))] -pub unsafe fn vcombine_u64(low: uint64x1_t, high: uint64x1_t) -> uint64x2_t { - simd_shuffle2(low, high, [0, 1]) -} - -/// Vector combine -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(mov))] -pub unsafe fn vcombine_p64(low: poly64x1_t, high: poly64x1_t) -> poly64x2_t { - simd_shuffle2(low, high, [0, 1]) -} - -/* FIXME: 16-bit float -/// Vector combine -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(mov))] -pub unsafe fn vcombine_f16 ( low: float16x4_t, high: float16x4_t) -> float16x8_t { - simd_shuffle8(low, high, [0, 1, 2, 3, 4, 5, 6, 7]) -} -*/ - -/// Vector combine -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(mov))] -pub unsafe fn vcombine_f32(low: float32x2_t, high: float32x2_t) -> float32x4_t { - simd_shuffle4(low, high, [0, 1, 2, 3]) -} - -/// Vector combine -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(mov))] -pub unsafe fn vcombine_p8(low: poly8x8_t, high: poly8x8_t) -> poly8x16_t { - simd_shuffle16( - low, - high, - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - ) -} - -/// Vector combine -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(mov))] -pub unsafe fn vcombine_p16(low: poly16x4_t, high: poly16x4_t) -> poly16x8_t { - simd_shuffle8(low, high, [0, 1, 2, 3, 4, 5, 6, 7]) -} - -/// Vector combine -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(mov))] -pub unsafe fn vcombine_f64(low: float64x1_t, high: float64x1_t) -> float64x2_t { - simd_shuffle2(low, high, [0, 1]) -} - -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vtbl1_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { - vqtbl1_s8(vcombine_s8(a, zeroed()), transmute(b)) -} - -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vtbl1_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - vqtbl1_u8(vcombine_u8(a, zeroed()), b) -} - -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vtbl1_p8(a: poly8x8_t, b: uint8x8_t) -> poly8x8_t { - vqtbl1_p8(vcombine_p8(a, zeroed()), b) -} - -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vtbl2_s8(a: int8x8x2_t, b: int8x8_t) -> int8x8_t { - vqtbl1_s8(vcombine_s8(a.0, a.1), transmute(b)) -} - -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vtbl2_u8(a: uint8x8x2_t, b: uint8x8_t) -> uint8x8_t { - vqtbl1_u8(vcombine_u8(a.0, a.1), b) -} - -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vtbl2_p8(a: poly8x8x2_t, b: uint8x8_t) -> poly8x8_t { - vqtbl1_p8(vcombine_p8(a.0, a.1), b) -} - -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vtbl3_s8(a: int8x8x3_t, b: int8x8_t) -> int8x8_t { - vqtbl2_s8( - int8x16x2_t(vcombine_s8(a.0, a.1), vcombine_s8(a.2, zeroed())), - transmute(b), - ) -} - -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vtbl3_u8(a: uint8x8x3_t, b: uint8x8_t) -> uint8x8_t { - vqtbl2_u8( - uint8x16x2_t(vcombine_u8(a.0, a.1), vcombine_u8(a.2, zeroed())), - b, - ) -} - -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vtbl3_p8(a: poly8x8x3_t, b: uint8x8_t) -> poly8x8_t { - vqtbl2_p8( - poly8x16x2_t(vcombine_p8(a.0, a.1), vcombine_p8(a.2, zeroed())), - b, - ) -} - -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vtbl4_s8(a: int8x8x4_t, b: int8x8_t) -> int8x8_t { - vqtbl2_s8( - int8x16x2_t(vcombine_s8(a.0, a.1), vcombine_s8(a.2, a.3)), - transmute(b), - ) -} - -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vtbl4_u8(a: uint8x8x4_t, b: uint8x8_t) -> uint8x8_t { - vqtbl2_u8( - uint8x16x2_t(vcombine_u8(a.0, a.1), vcombine_u8(a.2, a.3)), - b, - ) -} - -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vtbl4_p8(a: poly8x8x4_t, b: uint8x8_t) -> poly8x8_t { - vqtbl2_p8( - poly8x16x2_t(vcombine_p8(a.0, a.1), vcombine_p8(a.2, a.3)), - b, - ) -} - -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vtbx1_s8(a: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t { - use crate::core_arch::simd::i8x8; - let r = vqtbx1_s8(a, vcombine_s8(b, zeroed()), transmute(c)); - let m: int8x8_t = simd_lt(c, transmute(i8x8::splat(8))); - simd_select(m, r, a) -} - -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vtbx1_u8(a: uint8x8_t, b: uint8x8_t, c: uint8x8_t) -> uint8x8_t { - use crate::core_arch::simd::u8x8; - let r = vqtbx1_u8(a, vcombine_u8(b, zeroed()), c); - let m: int8x8_t = simd_lt(c, transmute(u8x8::splat(8))); - simd_select(m, r, a) -} - -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vtbx1_p8(a: poly8x8_t, b: poly8x8_t, c: uint8x8_t) -> poly8x8_t { - use crate::core_arch::simd::u8x8; - let r = vqtbx1_p8(a, vcombine_p8(b, zeroed()), c); - let m: int8x8_t = simd_lt(c, transmute(u8x8::splat(8))); - simd_select(m, r, a) -} - -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vtbx2_s8(a: int8x8_t, b: int8x8x2_t, c: int8x8_t) -> int8x8_t { - vqtbx1_s8(a, vcombine_s8(b.0, b.1), transmute(c)) -} - -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vtbx2_u8(a: uint8x8_t, b: uint8x8x2_t, c: uint8x8_t) -> uint8x8_t { - vqtbx1_u8(a, vcombine_u8(b.0, b.1), c) -} - -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vtbx2_p8(a: poly8x8_t, b: poly8x8x2_t, c: uint8x8_t) -> poly8x8_t { - vqtbx1_p8(a, vcombine_p8(b.0, b.1), c) -} - -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vtbx3_s8(a: int8x8_t, b: int8x8x3_t, c: int8x8_t) -> int8x8_t { - use crate::core_arch::simd::i8x8; - let r = vqtbx2_s8( - a, - int8x16x2_t(vcombine_s8(b.0, b.1), vcombine_s8(b.2, zeroed())), - transmute(c), - ); - let m: int8x8_t = simd_lt(c, transmute(i8x8::splat(24))); - simd_select(m, r, a) -} - -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vtbx3_u8(a: uint8x8_t, b: uint8x8x3_t, c: uint8x8_t) -> uint8x8_t { - use crate::core_arch::simd::u8x8; - let r = vqtbx2_u8( - a, - uint8x16x2_t(vcombine_u8(b.0, b.1), vcombine_u8(b.2, zeroed())), - c, - ); - let m: int8x8_t = simd_lt(c, transmute(u8x8::splat(24))); - simd_select(m, r, a) -} - -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vtbx3_p8(a: poly8x8_t, b: poly8x8x3_t, c: uint8x8_t) -> poly8x8_t { - use crate::core_arch::simd::u8x8; - let r = vqtbx2_p8( - a, - poly8x16x2_t(vcombine_p8(b.0, b.1), vcombine_p8(b.2, zeroed())), - c, - ); - let m: int8x8_t = simd_lt(c, transmute(u8x8::splat(24))); - simd_select(m, r, a) -} - -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vtbx4_s8(a: int8x8_t, b: int8x8x4_t, c: int8x8_t) -> int8x8_t { - vqtbx2_s8( - a, - int8x16x2_t(vcombine_s8(b.0, b.1), vcombine_s8(b.2, b.3)), - transmute(c), - ) -} - -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vtbx4_u8(a: uint8x8_t, b: uint8x8x4_t, c: uint8x8_t) -> uint8x8_t { - vqtbx2_u8( - a, - uint8x16x2_t(vcombine_u8(b.0, b.1), vcombine_u8(b.2, b.3)), - c, - ) -} - -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vtbx4_p8(a: poly8x8_t, b: poly8x8x4_t, c: uint8x8_t) -> poly8x8_t { - vqtbx2_p8( - a, - poly8x16x2_t(vcombine_p8(b.0, b.1), vcombine_p8(b.2, b.3)), - c, - ) -} - -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl1_s8(t: int8x16_t, idx: uint8x8_t) -> int8x8_t { - vqtbl1(t, idx) -} -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl1q_s8(t: int8x16_t, idx: uint8x16_t) -> int8x16_t { - vqtbl1q(t, idx) -} -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl1_u8(t: uint8x16_t, idx: uint8x8_t) -> uint8x8_t { - transmute(vqtbl1(transmute(t), transmute(idx))) -} -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl1q_u8(t: uint8x16_t, idx: uint8x16_t) -> uint8x16_t { - transmute(vqtbl1q(transmute(t), transmute(idx))) -} -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl1_p8(t: poly8x16_t, idx: uint8x8_t) -> poly8x8_t { - transmute(vqtbl1(transmute(t), transmute(idx))) -} -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl1q_p8(t: poly8x16_t, idx: uint8x16_t) -> poly8x16_t { - transmute(vqtbl1q(transmute(t), transmute(idx))) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx1_s8(a: int8x8_t, t: int8x16_t, idx: uint8x8_t) -> int8x8_t { - vqtbx1(a, t, idx) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx1q_s8(a: int8x16_t, t: int8x16_t, idx: uint8x16_t) -> int8x16_t { - vqtbx1q(a, t, idx) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx1_u8(a: uint8x8_t, t: uint8x16_t, idx: uint8x8_t) -> uint8x8_t { - transmute(vqtbx1(transmute(a), transmute(t), transmute(idx))) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx1q_u8(a: uint8x16_t, t: uint8x16_t, idx: uint8x16_t) -> uint8x16_t { - transmute(vqtbx1q(transmute(a), transmute(t), transmute(idx))) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx1_p8(a: poly8x8_t, t: poly8x16_t, idx: uint8x8_t) -> poly8x8_t { - transmute(vqtbx1(transmute(a), transmute(t), transmute(idx))) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx1q_p8(a: poly8x16_t, t: poly8x16_t, idx: uint8x16_t) -> poly8x16_t { - transmute(vqtbx1q(transmute(a), transmute(t), transmute(idx))) -} - -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl2_s8(t: int8x16x2_t, idx: uint8x8_t) -> int8x8_t { - vqtbl2(t.0, t.1, idx) -} -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl2q_s8(t: int8x16x2_t, idx: uint8x16_t) -> int8x16_t { - vqtbl2q(t.0, t.1, idx) -} -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl2_u8(t: uint8x16x2_t, idx: uint8x8_t) -> uint8x8_t { - transmute(vqtbl2(transmute(t.0), transmute(t.1), transmute(idx))) -} -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl2q_u8(t: uint8x16x2_t, idx: uint8x16_t) -> uint8x16_t { - transmute(vqtbl2q(transmute(t.0), transmute(t.1), transmute(idx))) -} -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl2_p8(t: poly8x16x2_t, idx: uint8x8_t) -> poly8x8_t { - transmute(vqtbl2(transmute(t.0), transmute(t.1), transmute(idx))) -} -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl2q_p8(t: poly8x16x2_t, idx: uint8x16_t) -> poly8x16_t { - transmute(vqtbl2q(transmute(t.0), transmute(t.1), transmute(idx))) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx2_s8(a: int8x8_t, t: int8x16x2_t, idx: uint8x8_t) -> int8x8_t { - vqtbx2(a, t.0, t.1, idx) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx2q_s8(a: int8x16_t, t: int8x16x2_t, idx: uint8x16_t) -> int8x16_t { - vqtbx2q(a, t.0, t.1, idx) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx2_u8(a: uint8x8_t, t: uint8x16x2_t, idx: uint8x8_t) -> uint8x8_t { - transmute(vqtbx2( - transmute(a), - transmute(t.0), - transmute(t.1), - transmute(idx), - )) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx2q_u8(a: uint8x16_t, t: uint8x16x2_t, idx: uint8x16_t) -> uint8x16_t { - transmute(vqtbx2q( - transmute(a), - transmute(t.0), - transmute(t.1), - transmute(idx), - )) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx2_p8(a: poly8x8_t, t: poly8x16x2_t, idx: uint8x8_t) -> poly8x8_t { - transmute(vqtbx2( - transmute(a), - transmute(t.0), - transmute(t.1), - transmute(idx), - )) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx2q_p8(a: poly8x16_t, t: poly8x16x2_t, idx: uint8x16_t) -> poly8x16_t { - transmute(vqtbx2q( - transmute(a), - transmute(t.0), - transmute(t.1), - transmute(idx), - )) -} - -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl3_s8(t: int8x16x3_t, idx: uint8x8_t) -> int8x8_t { - vqtbl3(t.0, t.1, t.2, idx) -} -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl3q_s8(t: int8x16x3_t, idx: uint8x16_t) -> int8x16_t { - vqtbl3q(t.0, t.1, t.2, idx) -} -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl3_u8(t: uint8x16x3_t, idx: uint8x8_t) -> uint8x8_t { - transmute(vqtbl3( - transmute(t.0), - transmute(t.1), - transmute(t.2), - transmute(idx), - )) -} -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl3q_u8(t: uint8x16x3_t, idx: uint8x16_t) -> uint8x16_t { - transmute(vqtbl3q( - transmute(t.0), - transmute(t.1), - transmute(t.2), - transmute(idx), - )) -} -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl3_p8(t: poly8x16x3_t, idx: uint8x8_t) -> poly8x8_t { - transmute(vqtbl3( - transmute(t.0), - transmute(t.1), - transmute(t.2), - transmute(idx), - )) -} -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl3q_p8(t: poly8x16x3_t, idx: uint8x16_t) -> poly8x16_t { - transmute(vqtbl3q( - transmute(t.0), - transmute(t.1), - transmute(t.2), - transmute(idx), - )) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx3_s8(a: int8x8_t, t: int8x16x3_t, idx: uint8x8_t) -> int8x8_t { - vqtbx3(a, t.0, t.1, t.2, idx) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx3q_s8(a: int8x16_t, t: int8x16x3_t, idx: uint8x16_t) -> int8x16_t { - vqtbx3q(a, t.0, t.1, t.2, idx) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx3_u8(a: uint8x8_t, t: uint8x16x3_t, idx: uint8x8_t) -> uint8x8_t { - transmute(vqtbx3( - transmute(a), - transmute(t.0), - transmute(t.1), - transmute(t.2), - transmute(idx), - )) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx3q_u8(a: uint8x16_t, t: uint8x16x3_t, idx: uint8x16_t) -> uint8x16_t { - transmute(vqtbx3q( - transmute(a), - transmute(t.0), - transmute(t.1), - transmute(t.2), - transmute(idx), - )) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx3_p8(a: poly8x8_t, t: poly8x16x3_t, idx: uint8x8_t) -> poly8x8_t { - transmute(vqtbx3( - transmute(a), - transmute(t.0), - transmute(t.1), - transmute(t.2), - transmute(idx), - )) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx3q_p8(a: poly8x16_t, t: poly8x16x3_t, idx: uint8x16_t) -> poly8x16_t { - transmute(vqtbx3q( - transmute(a), - transmute(t.0), - transmute(t.1), - transmute(t.2), - transmute(idx), - )) -} - -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl4_s8(t: int8x16x4_t, idx: uint8x8_t) -> int8x8_t { - vqtbl4(t.0, t.1, t.2, t.3, idx) -} -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl4q_s8(t: int8x16x4_t, idx: uint8x16_t) -> int8x16_t { - vqtbl4q(t.0, t.1, t.2, t.3, idx) -} -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl4_u8(t: uint8x16x4_t, idx: uint8x8_t) -> uint8x8_t { - transmute(vqtbl4( - transmute(t.0), - transmute(t.1), - transmute(t.2), - transmute(t.3), - transmute(idx), - )) -} -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl4q_u8(t: uint8x16x4_t, idx: uint8x16_t) -> uint8x16_t { - transmute(vqtbl4q( - transmute(t.0), - transmute(t.1), - transmute(t.2), - transmute(t.3), - transmute(idx), - )) -} -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl4_p8(t: poly8x16x4_t, idx: uint8x8_t) -> poly8x8_t { - transmute(vqtbl4( - transmute(t.0), - transmute(t.1), - transmute(t.2), - transmute(t.3), - transmute(idx), - )) -} -/// Table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbl))] -pub unsafe fn vqtbl4q_p8(t: poly8x16x4_t, idx: uint8x16_t) -> poly8x16_t { - transmute(vqtbl4q( - transmute(t.0), - transmute(t.1), - transmute(t.2), - transmute(t.3), - transmute(idx), - )) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx4_s8(a: int8x8_t, t: int8x16x4_t, idx: uint8x8_t) -> int8x8_t { - vqtbx4(a, t.0, t.1, t.2, t.3, idx) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx4q_s8(a: int8x16_t, t: int8x16x4_t, idx: uint8x16_t) -> int8x16_t { - vqtbx4q(a, t.0, t.1, t.2, t.3, idx) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx4_u8(a: uint8x8_t, t: uint8x16x4_t, idx: uint8x8_t) -> uint8x8_t { - transmute(vqtbx4( - transmute(a), - transmute(t.0), - transmute(t.1), - transmute(t.2), - transmute(t.3), - transmute(idx), - )) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx4q_u8(a: uint8x16_t, t: uint8x16x4_t, idx: uint8x16_t) -> uint8x16_t { - transmute(vqtbx4q( - transmute(a), - transmute(t.0), - transmute(t.1), - transmute(t.2), - transmute(t.3), - transmute(idx), - )) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx4_p8(a: poly8x8_t, t: poly8x16x4_t, idx: uint8x8_t) -> poly8x8_t { - transmute(vqtbx4( - transmute(a), - transmute(t.0), - transmute(t.1), - transmute(t.2), - transmute(t.3), - transmute(idx), - )) -} -/// Extended table look-up -#[inline] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(tbx))] -pub unsafe fn vqtbx4q_p8(a: poly8x16_t, t: poly8x16x4_t, idx: uint8x16_t) -> poly8x16_t { - transmute(vqtbx4q( - transmute(a), - transmute(t.0), - transmute(t.1), - transmute(t.2), - transmute(t.3), - transmute(idx), - )) -} - -#[cfg(test)] -mod tests { - use crate::core_arch::{aarch64::*, simd::*}; - use std::mem::transmute; - use stdarch_test::simd_test; - - #[simd_test(enable = "neon")] - unsafe fn test_vadd_f64() { - let a = 1.; - let b = 8.; - let e = 9.; - let r: f64 = transmute(vadd_f64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vaddq_f64() { - let a = f64x2::new(1., 2.); - let b = f64x2::new(8., 7.); - let e = f64x2::new(9., 9.); - let r: f64x2 = transmute(vaddq_f64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vaddd_s64() { - let a = 1_i64; - let b = 8_i64; - let e = 9_i64; - let r: i64 = transmute(vaddd_s64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vaddd_u64() { - let a = 1_u64; - let b = 8_u64; - let e = 9_u64; - let r: u64 = transmute(vaddd_u64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxv_s8() { - let r = vmaxv_s8(transmute(i8x8::new(1, 2, 3, 4, -8, 6, 7, 5))); - assert_eq!(r, 7_i8); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxvq_s8() { - #[rustfmt::skip] - let r = vmaxvq_s8(transmute(i8x16::new( - 1, 2, 3, 4, - -16, 6, 7, 5, - 8, 1, 1, 1, - 1, 1, 1, 1, - ))); - assert_eq!(r, 8_i8); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxv_s16() { - let r = vmaxv_s16(transmute(i16x4::new(1, 2, -4, 3))); - assert_eq!(r, 3_i16); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxvq_s16() { - let r = vmaxvq_s16(transmute(i16x8::new(1, 2, 7, 4, -16, 6, 7, 5))); - assert_eq!(r, 7_i16); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxv_s32() { - let r = vmaxv_s32(transmute(i32x2::new(1, -4))); - assert_eq!(r, 1_i32); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxvq_s32() { - let r = vmaxvq_s32(transmute(i32x4::new(1, 2, -32, 4))); - assert_eq!(r, 4_i32); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxv_u8() { - let r = vmaxv_u8(transmute(u8x8::new(1, 2, 3, 4, 8, 6, 7, 5))); - assert_eq!(r, 8_u8); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxvq_u8() { - #[rustfmt::skip] - let r = vmaxvq_u8(transmute(u8x16::new( - 1, 2, 3, 4, - 16, 6, 7, 5, - 8, 1, 1, 1, - 1, 1, 1, 1, - ))); - assert_eq!(r, 16_u8); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxv_u16() { - let r = vmaxv_u16(transmute(u16x4::new(1, 2, 4, 3))); - assert_eq!(r, 4_u16); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxvq_u16() { - let r = vmaxvq_u16(transmute(u16x8::new(1, 2, 7, 4, 16, 6, 7, 5))); - assert_eq!(r, 16_u16); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxv_u32() { - let r = vmaxv_u32(transmute(u32x2::new(1, 4))); - assert_eq!(r, 4_u32); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxvq_u32() { - let r = vmaxvq_u32(transmute(u32x4::new(1, 2, 32, 4))); - assert_eq!(r, 32_u32); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxv_f32() { - let r = vmaxv_f32(transmute(f32x2::new(1., 4.))); - assert_eq!(r, 4_f32); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxvq_f32() { - let r = vmaxvq_f32(transmute(f32x4::new(1., 2., 32., 4.))); - assert_eq!(r, 32_f32); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxvq_f64() { - let r = vmaxvq_f64(transmute(f64x2::new(1., 4.))); - assert_eq!(r, 4_f64); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminv_s8() { - let r = vminv_s8(transmute(i8x8::new(1, 2, 3, 4, -8, 6, 7, 5))); - assert_eq!(r, -8_i8); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminvq_s8() { - #[rustfmt::skip] - let r = vminvq_s8(transmute(i8x16::new( - 1, 2, 3, 4, - -16, 6, 7, 5, - 8, 1, 1, 1, - 1, 1, 1, 1, - ))); - assert_eq!(r, -16_i8); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminv_s16() { - let r = vminv_s16(transmute(i16x4::new(1, 2, -4, 3))); - assert_eq!(r, -4_i16); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminvq_s16() { - let r = vminvq_s16(transmute(i16x8::new(1, 2, 7, 4, -16, 6, 7, 5))); - assert_eq!(r, -16_i16); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminv_s32() { - let r = vminv_s32(transmute(i32x2::new(1, -4))); - assert_eq!(r, -4_i32); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminvq_s32() { - let r = vminvq_s32(transmute(i32x4::new(1, 2, -32, 4))); - assert_eq!(r, -32_i32); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminv_u8() { - let r = vminv_u8(transmute(u8x8::new(1, 2, 3, 4, 8, 6, 7, 5))); - assert_eq!(r, 1_u8); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminvq_u8() { - #[rustfmt::skip] - let r = vminvq_u8(transmute(u8x16::new( - 1, 2, 3, 4, - 16, 6, 7, 5, - 8, 1, 1, 1, - 1, 1, 1, 1, - ))); - assert_eq!(r, 1_u8); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminv_u16() { - let r = vminv_u16(transmute(u16x4::new(1, 2, 4, 3))); - assert_eq!(r, 1_u16); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminvq_u16() { - let r = vminvq_u16(transmute(u16x8::new(1, 2, 7, 4, 16, 6, 7, 5))); - assert_eq!(r, 1_u16); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminv_u32() { - let r = vminv_u32(transmute(u32x2::new(1, 4))); - assert_eq!(r, 1_u32); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminvq_u32() { - let r = vminvq_u32(transmute(u32x4::new(1, 2, 32, 4))); - assert_eq!(r, 1_u32); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminv_f32() { - let r = vminv_f32(transmute(f32x2::new(1., 4.))); - assert_eq!(r, 1_f32); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminvq_f32() { - let r = vminvq_f32(transmute(f32x4::new(1., 2., 32., 4.))); - assert_eq!(r, 1_f32); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminvq_f64() { - let r = vminvq_f64(transmute(f64x2::new(1., 4.))); - assert_eq!(r, 1_f64); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpminq_s8() { - #[cfg_attr(rustfmt, skip)] - let a = i8x16::new(1, -2, 3, -4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); - #[cfg_attr(rustfmt, skip)] - let b = i8x16::new(0, 3, 2, 5, 4, 7, 6, 9, 0, 3, 2, 5, 4, 7, 6, 9); - #[cfg_attr(rustfmt, skip)] - let e = i8x16::new(-2, -4, 5, 7, 1, 3, 5, 7, 0, 2, 4, 6, 0, 2, 4, 6); - let r: i8x16 = transmute(vpminq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpminq_s16() { - let a = i16x8::new(1, -2, 3, 4, 5, 6, 7, 8); - let b = i16x8::new(0, 3, 2, 5, 4, 7, 6, 9); - let e = i16x8::new(-2, 3, 5, 7, 0, 2, 4, 6); - let r: i16x8 = transmute(vpminq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpminq_s32() { - let a = i32x4::new(1, -2, 3, 4); - let b = i32x4::new(0, 3, 2, 5); - let e = i32x4::new(-2, 3, 0, 2); - let r: i32x4 = transmute(vpminq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpminq_u8() { - #[cfg_attr(rustfmt, skip)] - let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); - #[cfg_attr(rustfmt, skip)] - let b = u8x16::new(0, 3, 2, 5, 4, 7, 6, 9, 0, 3, 2, 5, 4, 7, 6, 9); - #[cfg_attr(rustfmt, skip)] - let e = u8x16::new(1, 3, 5, 7, 1, 3, 5, 7, 0, 2, 4, 6, 0, 2, 4, 6); - let r: u8x16 = transmute(vpminq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpminq_u16() { - let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b = u16x8::new(0, 3, 2, 5, 4, 7, 6, 9); - let e = u16x8::new(1, 3, 5, 7, 0, 2, 4, 6); - let r: u16x8 = transmute(vpminq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpminq_u32() { - let a = u32x4::new(1, 2, 3, 4); - let b = u32x4::new(0, 3, 2, 5); - let e = u32x4::new(1, 3, 0, 2); - let r: u32x4 = transmute(vpminq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmin_f32() { - let a = f32x4::new(1., -2., 3., 4.); - let b = f32x4::new(0., 3., 2., 5.); - let e = f32x4::new(-2., 3., 0., 2.); - let r: f32x4 = transmute(vpminq_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmin_f64() { - let a = f64x2::new(1., -2.); - let b = f64x2::new(0., 3.); - let e = f64x2::new(-2., 0.); - let r: f64x2 = transmute(vpminq_f64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmaxq_s8() { - #[cfg_attr(rustfmt, skip)] - let a = i8x16::new(1, -2, 3, -4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); - #[cfg_attr(rustfmt, skip)] - let b = i8x16::new(0, 3, 2, 5, 4, 7, 6, 9, 0, 3, 2, 5, 4, 7, 6, 9); - #[cfg_attr(rustfmt, skip)] - let e = i8x16::new(1, 3, 6, 8, 2, 4, 6, 8, 3, 5, 7, 9, 3, 5, 7, 9); - let r: i8x16 = transmute(vpmaxq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmaxq_s16() { - let a = i16x8::new(1, -2, 3, 4, 5, 6, 7, 8); - let b = i16x8::new(0, 3, 2, 5, 4, 7, 6, 9); - let e = i16x8::new(1, 4, 6, 8, 3, 5, 7, 9); - let r: i16x8 = transmute(vpmaxq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmaxq_s32() { - let a = i32x4::new(1, -2, 3, 4); - let b = i32x4::new(0, 3, 2, 5); - let e = i32x4::new(1, 4, 3, 5); - let r: i32x4 = transmute(vpmaxq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmaxq_u8() { - #[cfg_attr(rustfmt, skip)] - let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); - #[cfg_attr(rustfmt, skip)] - let b = u8x16::new(0, 3, 2, 5, 4, 7, 6, 9, 0, 3, 2, 5, 4, 7, 6, 9); - #[cfg_attr(rustfmt, skip)] - let e = u8x16::new(2, 4, 6, 8, 2, 4, 6, 8, 3, 5, 7, 9, 3, 5, 7, 9); - let r: u8x16 = transmute(vpmaxq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmaxq_u16() { - let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b = u16x8::new(0, 3, 2, 5, 4, 7, 6, 9); - let e = u16x8::new(2, 4, 6, 8, 3, 5, 7, 9); - let r: u16x8 = transmute(vpmaxq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmaxq_u32() { - let a = u32x4::new(1, 2, 3, 4); - let b = u32x4::new(0, 3, 2, 5); - let e = u32x4::new(2, 4, 3, 5); - let r: u32x4 = transmute(vpmaxq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmax_f32() { - let a = f32x4::new(1., -2., 3., 4.); - let b = f32x4::new(0., 3., 2., 5.); - let e = f32x4::new(1., 4., 3., 5.); - let r: f32x4 = transmute(vpmaxq_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmax_f64() { - let a = f64x2::new(1., -2.); - let b = f64x2::new(0., 3.); - let e = f64x2::new(1., 3.); - let r: f64x2 = transmute(vpmaxq_f64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - macro_rules! test_vcombine { - ($test_id:ident => $fn_id:ident ([$($a:expr),*], [$($b:expr),*])) => { - #[allow(unused_assignments)] - #[simd_test(enable = "neon")] - unsafe fn $test_id() { - let a = [$($a),*]; - let b = [$($b),*]; - let e = [$($a),* $(, $b)*]; - let c = $fn_id(transmute(a), transmute(b)); - let mut d = e; - d = transmute(c); - assert_eq!(d, e); - } - } - } - - test_vcombine!(test_vcombine_s8 => vcombine_s8([3_i8, -4, 5, -6, 7, 8, 9, 10], [13_i8, -14, 15, -16, 17, 18, 19, 110])); - test_vcombine!(test_vcombine_u8 => vcombine_u8([3_u8, 4, 5, 6, 7, 8, 9, 10], [13_u8, 14, 15, 16, 17, 18, 19, 110])); - test_vcombine!(test_vcombine_p8 => vcombine_p8([3_u8, 4, 5, 6, 7, 8, 9, 10], [13_u8, 14, 15, 16, 17, 18, 19, 110])); - - test_vcombine!(test_vcombine_s16 => vcombine_s16([3_i16, -4, 5, -6], [13_i16, -14, 15, -16])); - test_vcombine!(test_vcombine_u16 => vcombine_u16([3_u16, 4, 5, 6], [13_u16, 14, 15, 16])); - test_vcombine!(test_vcombine_p16 => vcombine_p16([3_u16, 4, 5, 6], [13_u16, 14, 15, 16])); - // FIXME: 16-bit floats - // test_vcombine!(test_vcombine_f16 => vcombine_f16([3_f16, 4., 5., 6.], - // [13_f16, 14., 15., 16.])); - - test_vcombine!(test_vcombine_s32 => vcombine_s32([3_i32, -4], [13_i32, -14])); - test_vcombine!(test_vcombine_u32 => vcombine_u32([3_u32, 4], [13_u32, 14])); - // note: poly32x4 does not exist, and neither does vcombine_p32 - test_vcombine!(test_vcombine_f32 => vcombine_f32([3_f32, -4.], [13_f32, -14.])); - - test_vcombine!(test_vcombine_s64 => vcombine_s64([-3_i64], [13_i64])); - test_vcombine!(test_vcombine_u64 => vcombine_u64([3_u64], [13_u64])); - test_vcombine!(test_vcombine_p64 => vcombine_p64([3_u64], [13_u64])); - test_vcombine!(test_vcombine_f64 => vcombine_f64([-3_f64], [13_f64])); -} - -#[cfg(test)] -#[cfg(target_endian = "little")] -#[path = "../arm/table_lookup_tests.rs"] -mod table_lookup_tests; diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/neon/generated.rs b/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/neon/generated.rs new file mode 100644 index 000000000..ac05a0c23 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/neon/generated.rs @@ -0,0 +1,28284 @@ +// This code is automatically generated. DO NOT MODIFY. +// +// Instead, modify `crates/stdarch-gen/neon.spec` and run the following command to re-generate this file: +// +// ``` +// OUT_DIR=`pwd`/crates/core_arch cargo run -p stdarch-gen -- crates/stdarch-gen/neon.spec +// ``` +use super::*; +#[cfg(test)] +use stdarch_test::assert_instr; + +/// Three-way exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor3q_s8) +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(eor3))] +pub unsafe fn veor3q_s8(a: int8x16_t, b: int8x16_t, c: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.eor3s.v16i8")] + fn veor3q_s8_(a: int8x16_t, b: int8x16_t, c: int8x16_t) -> int8x16_t; + } + veor3q_s8_(a, b, c) +} + +/// Three-way exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor3q_s16) +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(eor3))] +pub unsafe fn veor3q_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.eor3s.v8i16")] + fn veor3q_s16_(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t; + } + veor3q_s16_(a, b, c) +} + +/// Three-way exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor3q_s32) +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(eor3))] +pub unsafe fn veor3q_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.eor3s.v4i32")] + fn veor3q_s32_(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t; + } + veor3q_s32_(a, b, c) +} + +/// Three-way exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor3q_s64) +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(eor3))] +pub unsafe fn veor3q_s64(a: int64x2_t, b: int64x2_t, c: int64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.eor3s.v2i64")] + fn veor3q_s64_(a: int64x2_t, b: int64x2_t, c: int64x2_t) -> int64x2_t; + } + veor3q_s64_(a, b, c) +} + +/// Three-way exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor3q_u8) +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(eor3))] +pub unsafe fn veor3q_u8(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.eor3u.v16i8")] + fn veor3q_u8_(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t; + } + veor3q_u8_(a, b, c) +} + +/// Three-way exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor3q_u16) +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(eor3))] +pub unsafe fn veor3q_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.eor3u.v8i16")] + fn veor3q_u16_(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t; + } + veor3q_u16_(a, b, c) +} + +/// Three-way exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor3q_u32) +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(eor3))] +pub unsafe fn veor3q_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.eor3u.v4i32")] + fn veor3q_u32_(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t; + } + veor3q_u32_(a, b, c) +} + +/// Three-way exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor3q_u64) +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(eor3))] +pub unsafe fn veor3q_u64(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.eor3u.v2i64")] + fn veor3q_u64_(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t; + } + veor3q_u64_(a, b, c) +} + +/// Absolute difference between the arguments of Floating +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fabd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vabd_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fabd.v1f64")] + fn vabd_f64_(a: float64x1_t, b: float64x1_t) -> float64x1_t; + } + vabd_f64_(a, b) +} + +/// Absolute difference between the arguments of Floating +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fabd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vabdq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fabd.v2f64")] + fn vabdq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + } + vabdq_f64_(a, b) +} + +/// Floating-point absolute difference +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabds_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fabd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vabds_f32(a: f32, b: f32) -> f32 { + simd_extract(vabd_f32(vdup_n_f32(a), vdup_n_f32(b)), 0) +} + +/// Floating-point absolute difference +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fabd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vabdd_f64(a: f64, b: f64) -> f64 { + simd_extract(vabd_f64(vdup_n_f64(a), vdup_n_f64(b)), 0) +} + +/// Unsigned Absolute difference Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_high_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uabdl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vabdl_high_u8(a: uint8x16_t, b: uint8x16_t) -> uint16x8_t { + let c: uint8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + let d: uint8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + simd_cast(vabd_u8(c, d)) +} + +/// Unsigned Absolute difference Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_high_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uabdl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vabdl_high_u16(a: uint16x8_t, b: uint16x8_t) -> uint32x4_t { + let c: uint16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let d: uint16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + simd_cast(vabd_u16(c, d)) +} + +/// Unsigned Absolute difference Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_high_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uabdl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vabdl_high_u32(a: uint32x4_t, b: uint32x4_t) -> uint64x2_t { + let c: uint32x2_t = simd_shuffle2!(a, a, [2, 3]); + let d: uint32x2_t = simd_shuffle2!(b, b, [2, 3]); + simd_cast(vabd_u32(c, d)) +} + +/// Signed Absolute difference Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_high_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sabdl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vabdl_high_s8(a: int8x16_t, b: int8x16_t) -> int16x8_t { + let c: int8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + let d: int8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let e: uint8x8_t = simd_cast(vabd_s8(c, d)); + simd_cast(e) +} + +/// Signed Absolute difference Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_high_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sabdl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vabdl_high_s16(a: int16x8_t, b: int16x8_t) -> int32x4_t { + let c: int16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let d: int16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let e: uint16x4_t = simd_cast(vabd_s16(c, d)); + simd_cast(e) +} + +/// Signed Absolute difference Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_high_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sabdl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vabdl_high_s32(a: int32x4_t, b: int32x4_t) -> int64x2_t { + let c: int32x2_t = simd_shuffle2!(a, a, [2, 3]); + let d: int32x2_t = simd_shuffle2!(b, b, [2, 3]); + let e: uint32x2_t = simd_cast(vabd_s32(c, d)); + simd_cast(e) +} + +/// Compare bitwise Equal (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceq_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceq_s64(a: int64x1_t, b: int64x1_t) -> uint64x1_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqq_s64(a: int64x2_t, b: int64x2_t) -> uint64x2_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceq_p64(a: poly64x1_t, b: poly64x1_t) -> uint64x1_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqq_p64(a: poly64x2_t, b: poly64x2_t) -> uint64x2_t { + simd_eq(a, b) +} + +/// Floating-point compare equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceq_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t { + simd_eq(a, b) +} + +/// Floating-point compare equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t { + simd_eq(a, b) +} + +/// Compare bitwise equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqd_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqd_s64(a: i64, b: i64) -> u64 { + transmute(vceq_s64(transmute(a), transmute(b))) +} + +/// Compare bitwise equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqd_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqd_u64(a: u64, b: u64) -> u64 { + transmute(vceq_u64(transmute(a), transmute(b))) +} + +/// Floating-point compare equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqs_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqs_f32(a: f32, b: f32) -> u32 { + simd_extract(vceq_f32(vdup_n_f32(a), vdup_n_f32(b)), 0) +} + +/// Floating-point compare equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqd_f64(a: f64, b: f64) -> u64 { + simd_extract(vceq_f64(vdup_n_f64(a), vdup_n_f64(b)), 0) +} + +/// Signed compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqz_s8(a: int8x8_t) -> uint8x8_t { + let b: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqzq_s8(a: int8x16_t) -> uint8x16_t { + let b: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqz_s16(a: int16x4_t) -> uint16x4_t { + let b: i16x4 = i16x4::new(0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqzq_s16(a: int16x8_t) -> uint16x8_t { + let b: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqz_s32(a: int32x2_t) -> uint32x2_t { + let b: i32x2 = i32x2::new(0, 0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqzq_s32(a: int32x4_t) -> uint32x4_t { + let b: i32x4 = i32x4::new(0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqz_s64(a: int64x1_t) -> uint64x1_t { + let b: i64x1 = i64x1::new(0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqzq_s64(a: int64x2_t) -> uint64x2_t { + let b: i64x2 = i64x2::new(0, 0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqz_p8(a: poly8x8_t) -> uint8x8_t { + let b: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqzq_p8(a: poly8x16_t) -> uint8x16_t { + let b: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqz_p64(a: poly64x1_t) -> uint64x1_t { + let b: i64x1 = i64x1::new(0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqzq_p64(a: poly64x2_t) -> uint64x2_t { + let b: i64x2 = i64x2::new(0, 0); + simd_eq(a, transmute(b)) +} + +/// Unsigned compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqz_u8(a: uint8x8_t) -> uint8x8_t { + let b: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Unsigned compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqzq_u8(a: uint8x16_t) -> uint8x16_t { + let b: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Unsigned compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqz_u16(a: uint16x4_t) -> uint16x4_t { + let b: u16x4 = u16x4::new(0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Unsigned compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqzq_u16(a: uint16x8_t) -> uint16x8_t { + let b: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Unsigned compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqz_u32(a: uint32x2_t) -> uint32x2_t { + let b: u32x2 = u32x2::new(0, 0); + simd_eq(a, transmute(b)) +} + +/// Unsigned compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqzq_u32(a: uint32x4_t) -> uint32x4_t { + let b: u32x4 = u32x4::new(0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Unsigned compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqz_u64(a: uint64x1_t) -> uint64x1_t { + let b: u64x1 = u64x1::new(0); + simd_eq(a, transmute(b)) +} + +/// Unsigned compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqzq_u64(a: uint64x2_t) -> uint64x2_t { + let b: u64x2 = u64x2::new(0, 0); + simd_eq(a, transmute(b)) +} + +/// Floating-point compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqz_f32(a: float32x2_t) -> uint32x2_t { + let b: f32x2 = f32x2::new(0.0, 0.0); + simd_eq(a, transmute(b)) +} + +/// Floating-point compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqzq_f32(a: float32x4_t) -> uint32x4_t { + let b: f32x4 = f32x4::new(0.0, 0.0, 0.0, 0.0); + simd_eq(a, transmute(b)) +} + +/// Floating-point compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqz_f64(a: float64x1_t) -> uint64x1_t { + let b: f64 = 0.0; + simd_eq(a, transmute(b)) +} + +/// Floating-point compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmeq))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqzq_f64(a: float64x2_t) -> uint64x2_t { + let b: f64x2 = f64x2::new(0.0, 0.0); + simd_eq(a, transmute(b)) +} + +/// Compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzd_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqzd_s64(a: i64) -> u64 { + transmute(vceqz_s64(transmute(a))) +} + +/// Compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzd_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqzd_u64(a: u64) -> u64 { + transmute(vceqz_u64(transmute(a))) +} + +/// Floating-point compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzs_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqzs_f32(a: f32) -> u32 { + simd_extract(vceqz_f32(vdup_n_f32(a)), 0) +} + +/// Floating-point compare bitwise equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vceqzd_f64(a: f64) -> u64 { + simd_extract(vceqz_f64(vdup_n_f64(a)), 0) +} + +/// Signed compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmtst))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtst_s64(a: int64x1_t, b: int64x1_t) -> uint64x1_t { + let c: int64x1_t = simd_and(a, b); + let d: i64x1 = i64x1::new(0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmtst))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtstq_s64(a: int64x2_t, b: int64x2_t) -> uint64x2_t { + let c: int64x2_t = simd_and(a, b); + let d: i64x2 = i64x2::new(0, 0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmtst))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtst_p64(a: poly64x1_t, b: poly64x1_t) -> uint64x1_t { + let c: poly64x1_t = simd_and(a, b); + let d: i64x1 = i64x1::new(0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmtst))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtstq_p64(a: poly64x2_t, b: poly64x2_t) -> uint64x2_t { + let c: poly64x2_t = simd_and(a, b); + let d: i64x2 = i64x2::new(0, 0); + simd_ne(c, transmute(d)) +} + +/// Unsigned compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmtst))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtst_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + let c: uint64x1_t = simd_and(a, b); + let d: u64x1 = u64x1::new(0); + simd_ne(c, transmute(d)) +} + +/// Unsigned compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmtst))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtstq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + let c: uint64x2_t = simd_and(a, b); + let d: u64x2 = u64x2::new(0, 0); + simd_ne(c, transmute(d)) +} + +/// Compare bitwise test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstd_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tst))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtstd_s64(a: i64, b: i64) -> u64 { + transmute(vtst_s64(transmute(a), transmute(b))) +} + +/// Compare bitwise test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstd_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tst))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtstd_u64(a: u64, b: u64) -> u64 { + transmute(vtst_u64(transmute(a), transmute(b))) +} + +/// Signed saturating accumulate of unsigned value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuqadds_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(suqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuqadds_s32(a: i32, b: u32) -> i32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.suqadd.i32")] + fn vuqadds_s32_(a: i32, b: u32) -> i32; + } + vuqadds_s32_(a, b) +} + +/// Signed saturating accumulate of unsigned value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuqaddd_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(suqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuqaddd_s64(a: i64, b: u64) -> i64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.suqadd.i64")] + fn vuqaddd_s64_(a: i64, b: u64) -> i64; + } + vuqaddd_s64_(a, b) +} + +/// Signed saturating accumulate of unsigned value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuqaddb_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(suqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuqaddb_s8(a: i8, b: u8) -> i8 { + simd_extract(vuqadd_s8(vdup_n_s8(a), vdup_n_u8(b)), 0) +} + +/// Signed saturating accumulate of unsigned value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuqaddh_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(suqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuqaddh_s16(a: i16, b: u16) -> i16 { + simd_extract(vuqadd_s16(vdup_n_s16(a), vdup_n_u16(b)), 0) +} + +/// Floating-point absolute value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabs_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fabs))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vabs_f64(a: float64x1_t) -> float64x1_t { + simd_fabs(a) +} + +/// Floating-point absolute value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabsq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fabs))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vabsq_f64(a: float64x2_t) -> float64x2_t { + simd_fabs(a) +} + +/// Compare signed greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgt_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgt_s64(a: int64x1_t, b: int64x1_t) -> uint64x1_t { + simd_gt(a, b) +} + +/// Compare signed greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgtq_s64(a: int64x2_t, b: int64x2_t) -> uint64x2_t { + simd_gt(a, b) +} + +/// Compare unsigned highe +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgt_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmhi))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgt_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + simd_gt(a, b) +} + +/// Compare unsigned highe +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmhi))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgtq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_gt(a, b) +} + +/// Floating-point compare greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgt_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgt_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t { + simd_gt(a, b) +} + +/// Floating-point compare greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgtq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t { + simd_gt(a, b) +} + +/// Compare greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtd_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgtd_s64(a: i64, b: i64) -> u64 { + transmute(vcgt_s64(transmute(a), transmute(b))) +} + +/// Compare greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtd_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgtd_u64(a: u64, b: u64) -> u64 { + transmute(vcgt_u64(transmute(a), transmute(b))) +} + +/// Floating-point compare greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgts_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgts_f32(a: f32, b: f32) -> u32 { + simd_extract(vcgt_f32(vdup_n_f32(a), vdup_n_f32(b)), 0) +} + +/// Floating-point compare greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgtd_f64(a: f64, b: f64) -> u64 { + simd_extract(vcgt_f64(vdup_n_f64(a), vdup_n_f64(b)), 0) +} + +/// Compare signed less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclt_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vclt_s64(a: int64x1_t, b: int64x1_t) -> uint64x1_t { + simd_lt(a, b) +} + +/// Compare signed less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcltq_s64(a: int64x2_t, b: int64x2_t) -> uint64x2_t { + simd_lt(a, b) +} + +/// Compare unsigned less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclt_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmhi))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vclt_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + simd_lt(a, b) +} + +/// Compare unsigned less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmhi))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcltq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_lt(a, b) +} + +/// Floating-point compare less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclt_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vclt_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t { + simd_lt(a, b) +} + +/// Floating-point compare less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcltq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t { + simd_lt(a, b) +} + +/// Compare less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltd_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcltd_s64(a: i64, b: i64) -> u64 { + transmute(vclt_s64(transmute(a), transmute(b))) +} + +/// Compare less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltd_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcltd_u64(a: u64, b: u64) -> u64 { + transmute(vclt_u64(transmute(a), transmute(b))) +} + +/// Floating-point compare less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclts_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vclts_f32(a: f32, b: f32) -> u32 { + simd_extract(vclt_f32(vdup_n_f32(a), vdup_n_f32(b)), 0) +} + +/// Floating-point compare less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcltd_f64(a: f64, b: f64) -> u64 { + simd_extract(vclt_f64(vdup_n_f64(a), vdup_n_f64(b)), 0) +} + +/// Compare signed less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcle_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcle_s64(a: int64x1_t, b: int64x1_t) -> uint64x1_t { + simd_le(a, b) +} + +/// Compare signed less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcleq_s64(a: int64x2_t, b: int64x2_t) -> uint64x2_t { + simd_le(a, b) +} + +/// Compare greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcged_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcged_s64(a: i64, b: i64) -> u64 { + transmute(vcge_s64(transmute(a), transmute(b))) +} + +/// Compare greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcged_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcged_u64(a: u64, b: u64) -> u64 { + transmute(vcge_u64(transmute(a), transmute(b))) +} + +/// Floating-point compare greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcges_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcges_f32(a: f32, b: f32) -> u32 { + simd_extract(vcge_f32(vdup_n_f32(a), vdup_n_f32(b)), 0) +} + +/// Floating-point compare greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcged_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcged_f64(a: f64, b: f64) -> u64 { + simd_extract(vcge_f64(vdup_n_f64(a), vdup_n_f64(b)), 0) +} + +/// Compare unsigned less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcle_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmhs))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcle_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + simd_le(a, b) +} + +/// Compare unsigned less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmhs))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcleq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_le(a, b) +} + +/// Floating-point compare less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcle_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcle_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t { + simd_le(a, b) +} + +/// Floating-point compare less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcleq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t { + simd_le(a, b) +} + +/// Compare less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcled_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcled_s64(a: i64, b: i64) -> u64 { + transmute(vcle_s64(transmute(a), transmute(b))) +} + +/// Compare less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcled_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcled_u64(a: u64, b: u64) -> u64 { + transmute(vcle_u64(transmute(a), transmute(b))) +} + +/// Floating-point compare less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcles_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcles_f32(a: f32, b: f32) -> u32 { + simd_extract(vcle_f32(vdup_n_f32(a), vdup_n_f32(b)), 0) +} + +/// Floating-point compare less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcled_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcled_f64(a: f64, b: f64) -> u64 { + simd_extract(vcle_f64(vdup_n_f64(a), vdup_n_f64(b)), 0) +} + +/// Compare signed greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcge_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcge_s64(a: int64x1_t, b: int64x1_t) -> uint64x1_t { + simd_ge(a, b) +} + +/// Compare signed greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgeq_s64(a: int64x2_t, b: int64x2_t) -> uint64x2_t { + simd_ge(a, b) +} + +/// Compare unsigned greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcge_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmhs))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcge_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + simd_ge(a, b) +} + +/// Compare unsigned greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmhs))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgeq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_ge(a, b) +} + +/// Floating-point compare greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcge_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcge_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t { + simd_ge(a, b) +} + +/// Floating-point compare greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgeq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t { + simd_ge(a, b) +} + +/// Compare signed greater than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgez_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgez_s8(a: int8x8_t) -> uint8x8_t { + let b: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_ge(a, transmute(b)) +} + +/// Compare signed greater than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgezq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgezq_s8(a: int8x16_t) -> uint8x16_t { + let b: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + simd_ge(a, transmute(b)) +} + +/// Compare signed greater than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgez_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgez_s16(a: int16x4_t) -> uint16x4_t { + let b: i16x4 = i16x4::new(0, 0, 0, 0); + simd_ge(a, transmute(b)) +} + +/// Compare signed greater than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgezq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgezq_s16(a: int16x8_t) -> uint16x8_t { + let b: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_ge(a, transmute(b)) +} + +/// Compare signed greater than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgez_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgez_s32(a: int32x2_t) -> uint32x2_t { + let b: i32x2 = i32x2::new(0, 0); + simd_ge(a, transmute(b)) +} + +/// Compare signed greater than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgezq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgezq_s32(a: int32x4_t) -> uint32x4_t { + let b: i32x4 = i32x4::new(0, 0, 0, 0); + simd_ge(a, transmute(b)) +} + +/// Compare signed greater than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgez_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgez_s64(a: int64x1_t) -> uint64x1_t { + let b: i64x1 = i64x1::new(0); + simd_ge(a, transmute(b)) +} + +/// Compare signed greater than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgezq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgezq_s64(a: int64x2_t) -> uint64x2_t { + let b: i64x2 = i64x2::new(0, 0); + simd_ge(a, transmute(b)) +} + +/// Floating-point compare greater than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgez_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgez_f32(a: float32x2_t) -> uint32x2_t { + let b: f32x2 = f32x2::new(0.0, 0.0); + simd_ge(a, transmute(b)) +} + +/// Floating-point compare greater than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgezq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgezq_f32(a: float32x4_t) -> uint32x4_t { + let b: f32x4 = f32x4::new(0.0, 0.0, 0.0, 0.0); + simd_ge(a, transmute(b)) +} + +/// Floating-point compare greater than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgez_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgez_f64(a: float64x1_t) -> uint64x1_t { + let b: f64 = 0.0; + simd_ge(a, transmute(b)) +} + +/// Floating-point compare greater than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgezq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgezq_f64(a: float64x2_t) -> uint64x2_t { + let b: f64x2 = f64x2::new(0.0, 0.0); + simd_ge(a, transmute(b)) +} + +/// Compare signed greater than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgezd_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(eor))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgezd_s64(a: i64) -> u64 { + transmute(vcgez_s64(transmute(a))) +} + +/// Floating-point compare greater than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgezs_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgezs_f32(a: f32) -> u32 { + simd_extract(vcgez_f32(vdup_n_f32(a)), 0) +} + +/// Floating-point compare greater than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgezd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgezd_f64(a: f64) -> u64 { + simd_extract(vcgez_f64(vdup_n_f64(a)), 0) +} + +/// Compare signed greater than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtz_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgtz_s8(a: int8x8_t) -> uint8x8_t { + let b: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_gt(a, transmute(b)) +} + +/// Compare signed greater than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtzq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgtzq_s8(a: int8x16_t) -> uint8x16_t { + let b: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + simd_gt(a, transmute(b)) +} + +/// Compare signed greater than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtz_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgtz_s16(a: int16x4_t) -> uint16x4_t { + let b: i16x4 = i16x4::new(0, 0, 0, 0); + simd_gt(a, transmute(b)) +} + +/// Compare signed greater than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtzq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgtzq_s16(a: int16x8_t) -> uint16x8_t { + let b: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_gt(a, transmute(b)) +} + +/// Compare signed greater than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtz_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgtz_s32(a: int32x2_t) -> uint32x2_t { + let b: i32x2 = i32x2::new(0, 0); + simd_gt(a, transmute(b)) +} + +/// Compare signed greater than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtzq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgtzq_s32(a: int32x4_t) -> uint32x4_t { + let b: i32x4 = i32x4::new(0, 0, 0, 0); + simd_gt(a, transmute(b)) +} + +/// Compare signed greater than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtz_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgtz_s64(a: int64x1_t) -> uint64x1_t { + let b: i64x1 = i64x1::new(0); + simd_gt(a, transmute(b)) +} + +/// Compare signed greater than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtzq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgtzq_s64(a: int64x2_t) -> uint64x2_t { + let b: i64x2 = i64x2::new(0, 0); + simd_gt(a, transmute(b)) +} + +/// Floating-point compare greater than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtz_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgtz_f32(a: float32x2_t) -> uint32x2_t { + let b: f32x2 = f32x2::new(0.0, 0.0); + simd_gt(a, transmute(b)) +} + +/// Floating-point compare greater than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtzq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgtzq_f32(a: float32x4_t) -> uint32x4_t { + let b: f32x4 = f32x4::new(0.0, 0.0, 0.0, 0.0); + simd_gt(a, transmute(b)) +} + +/// Floating-point compare greater than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtz_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgtz_f64(a: float64x1_t) -> uint64x1_t { + let b: f64 = 0.0; + simd_gt(a, transmute(b)) +} + +/// Floating-point compare greater than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtzq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgtzq_f64(a: float64x2_t) -> uint64x2_t { + let b: f64x2 = f64x2::new(0.0, 0.0); + simd_gt(a, transmute(b)) +} + +/// Compare signed greater than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtzd_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgtzd_s64(a: i64) -> u64 { + transmute(vcgtz_s64(transmute(a))) +} + +/// Floating-point compare greater than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtzs_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgtzs_f32(a: f32) -> u32 { + simd_extract(vcgtz_f32(vdup_n_f32(a)), 0) +} + +/// Floating-point compare greater than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtzd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcgtzd_f64(a: f64) -> u64 { + simd_extract(vcgtz_f64(vdup_n_f64(a)), 0) +} + +/// Compare signed less than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclez_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vclez_s8(a: int8x8_t) -> uint8x8_t { + let b: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_le(a, transmute(b)) +} + +/// Compare signed less than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclezq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vclezq_s8(a: int8x16_t) -> uint8x16_t { + let b: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + simd_le(a, transmute(b)) +} + +/// Compare signed less than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclez_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vclez_s16(a: int16x4_t) -> uint16x4_t { + let b: i16x4 = i16x4::new(0, 0, 0, 0); + simd_le(a, transmute(b)) +} + +/// Compare signed less than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclezq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vclezq_s16(a: int16x8_t) -> uint16x8_t { + let b: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_le(a, transmute(b)) +} + +/// Compare signed less than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclez_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vclez_s32(a: int32x2_t) -> uint32x2_t { + let b: i32x2 = i32x2::new(0, 0); + simd_le(a, transmute(b)) +} + +/// Compare signed less than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclezq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vclezq_s32(a: int32x4_t) -> uint32x4_t { + let b: i32x4 = i32x4::new(0, 0, 0, 0); + simd_le(a, transmute(b)) +} + +/// Compare signed less than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclez_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vclez_s64(a: int64x1_t) -> uint64x1_t { + let b: i64x1 = i64x1::new(0); + simd_le(a, transmute(b)) +} + +/// Compare signed less than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclezq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vclezq_s64(a: int64x2_t) -> uint64x2_t { + let b: i64x2 = i64x2::new(0, 0); + simd_le(a, transmute(b)) +} + +/// Floating-point compare less than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclez_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmle))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vclez_f32(a: float32x2_t) -> uint32x2_t { + let b: f32x2 = f32x2::new(0.0, 0.0); + simd_le(a, transmute(b)) +} + +/// Floating-point compare less than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclezq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmle))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vclezq_f32(a: float32x4_t) -> uint32x4_t { + let b: f32x4 = f32x4::new(0.0, 0.0, 0.0, 0.0); + simd_le(a, transmute(b)) +} + +/// Floating-point compare less than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclez_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmle))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vclez_f64(a: float64x1_t) -> uint64x1_t { + let b: f64 = 0.0; + simd_le(a, transmute(b)) +} + +/// Floating-point compare less than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclezq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmle))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vclezq_f64(a: float64x2_t) -> uint64x2_t { + let b: f64x2 = f64x2::new(0.0, 0.0); + simd_le(a, transmute(b)) +} + +/// Compare less than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclezd_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vclezd_s64(a: i64) -> u64 { + transmute(vclez_s64(transmute(a))) +} + +/// Floating-point compare less than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclezs_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vclezs_f32(a: f32) -> u32 { + simd_extract(vclez_f32(vdup_n_f32(a)), 0) +} + +/// Floating-point compare less than or equal to zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclezd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vclezd_f64(a: f64) -> u64 { + simd_extract(vclez_f64(vdup_n_f64(a)), 0) +} + +/// Compare signed less than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltz_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmlt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcltz_s8(a: int8x8_t) -> uint8x8_t { + let b: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_lt(a, transmute(b)) +} + +/// Compare signed less than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltzq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmlt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcltzq_s8(a: int8x16_t) -> uint8x16_t { + let b: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + simd_lt(a, transmute(b)) +} + +/// Compare signed less than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltz_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmlt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcltz_s16(a: int16x4_t) -> uint16x4_t { + let b: i16x4 = i16x4::new(0, 0, 0, 0); + simd_lt(a, transmute(b)) +} + +/// Compare signed less than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltzq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmlt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcltzq_s16(a: int16x8_t) -> uint16x8_t { + let b: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_lt(a, transmute(b)) +} + +/// Compare signed less than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltz_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmlt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcltz_s32(a: int32x2_t) -> uint32x2_t { + let b: i32x2 = i32x2::new(0, 0); + simd_lt(a, transmute(b)) +} + +/// Compare signed less than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltzq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmlt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcltzq_s32(a: int32x4_t) -> uint32x4_t { + let b: i32x4 = i32x4::new(0, 0, 0, 0); + simd_lt(a, transmute(b)) +} + +/// Compare signed less than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltz_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmlt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcltz_s64(a: int64x1_t) -> uint64x1_t { + let b: i64x1 = i64x1::new(0); + simd_lt(a, transmute(b)) +} + +/// Compare signed less than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltzq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmlt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcltzq_s64(a: int64x2_t) -> uint64x2_t { + let b: i64x2 = i64x2::new(0, 0); + simd_lt(a, transmute(b)) +} + +/// Floating-point compare less than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltz_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmlt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcltz_f32(a: float32x2_t) -> uint32x2_t { + let b: f32x2 = f32x2::new(0.0, 0.0); + simd_lt(a, transmute(b)) +} + +/// Floating-point compare less than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltzq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmlt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcltzq_f32(a: float32x4_t) -> uint32x4_t { + let b: f32x4 = f32x4::new(0.0, 0.0, 0.0, 0.0); + simd_lt(a, transmute(b)) +} + +/// Floating-point compare less than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltz_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmlt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcltz_f64(a: float64x1_t) -> uint64x1_t { + let b: f64 = 0.0; + simd_lt(a, transmute(b)) +} + +/// Floating-point compare less than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltzq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmlt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcltzq_f64(a: float64x2_t) -> uint64x2_t { + let b: f64x2 = f64x2::new(0.0, 0.0); + simd_lt(a, transmute(b)) +} + +/// Compare less than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltzd_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(asr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcltzd_s64(a: i64) -> u64 { + transmute(vcltz_s64(transmute(a))) +} + +/// Floating-point compare less than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltzs_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcltzs_f32(a: f32) -> u32 { + simd_extract(vcltz_f32(vdup_n_f32(a)), 0) +} + +/// Floating-point compare less than zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltzd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcltzd_f64(a: f64) -> u64 { + simd_extract(vcltz_f64(vdup_n_f64(a)), 0) +} + +/// Floating-point absolute compare greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcagt_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcagt_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.facgt.v1i64.v1f64")] + fn vcagt_f64_(a: float64x1_t, b: float64x1_t) -> uint64x1_t; + } + vcagt_f64_(a, b) +} + +/// Floating-point absolute compare greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcagtq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcagtq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.facgt.v2i64.v2f64")] + fn vcagtq_f64_(a: float64x2_t, b: float64x2_t) -> uint64x2_t; + } + vcagtq_f64_(a, b) +} + +/// Floating-point absolute compare greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcagts_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcagts_f32(a: f32, b: f32) -> u32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.facgt.i32.f32")] + fn vcagts_f32_(a: f32, b: f32) -> u32; + } + vcagts_f32_(a, b) +} + +/// Floating-point absolute compare greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcagtd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcagtd_f64(a: f64, b: f64) -> u64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.facgt.i64.f64")] + fn vcagtd_f64_(a: f64, b: f64) -> u64; + } + vcagtd_f64_(a, b) +} + +/// Floating-point absolute compare greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcage_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcage_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.facge.v1i64.v1f64")] + fn vcage_f64_(a: float64x1_t, b: float64x1_t) -> uint64x1_t; + } + vcage_f64_(a, b) +} + +/// Floating-point absolute compare greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcageq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcageq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.facge.v2i64.v2f64")] + fn vcageq_f64_(a: float64x2_t, b: float64x2_t) -> uint64x2_t; + } + vcageq_f64_(a, b) +} + +/// Floating-point absolute compare greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcages_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcages_f32(a: f32, b: f32) -> u32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.facge.i32.f32")] + fn vcages_f32_(a: f32, b: f32) -> u32; + } + vcages_f32_(a, b) +} + +/// Floating-point absolute compare greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaged_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcaged_f64(a: f64, b: f64) -> u64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.facge.i64.f64")] + fn vcaged_f64_(a: f64, b: f64) -> u64; + } + vcaged_f64_(a, b) +} + +/// Floating-point absolute compare less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcalt_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcalt_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t { + vcagt_f64(b, a) +} + +/// Floating-point absolute compare less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaltq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcaltq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t { + vcagtq_f64(b, a) +} + +/// Floating-point absolute compare less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcalts_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcalts_f32(a: f32, b: f32) -> u32 { + vcagts_f32(b, a) +} + +/// Floating-point absolute compare less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaltd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facgt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcaltd_f64(a: f64, b: f64) -> u64 { + vcagtd_f64(b, a) +} + +/// Floating-point absolute compare less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcale_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcale_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t { + vcage_f64(b, a) +} + +/// Floating-point absolute compare less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaleq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcaleq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t { + vcageq_f64(b, a) +} + +/// Floating-point absolute compare less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcales_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcales_f32(a: f32, b: f32) -> u32 { + vcages_f32(b, a) +} + +/// Floating-point absolute compare less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaled_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facge))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcaled_f64(a: f64, b: f64) -> u64 { + vcaged_f64(b, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_lane_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_lane_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + static_assert_imm3!(LANE1); + static_assert_imm3!(LANE2); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_laneq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + static_assert_imm4!(LANE1); + static_assert_imm4!(LANE2); + match LANE1 & 0b1111 { + 0 => simd_shuffle16!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle16!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 2 => simd_shuffle16!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 3 => simd_shuffle16!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 4 => simd_shuffle16!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 5 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 6 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 7 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), + 8 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), + 9 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), + 10 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), + 11 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), + 12 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), + 13 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), + 14 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), + 15 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_lane_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + static_assert_imm2!(LANE1); + static_assert_imm2!(LANE2); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [4 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 4 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 4 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_laneq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + static_assert_imm3!(LANE1); + static_assert_imm3!(LANE2); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_lane_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + static_assert_imm1!(LANE1); + static_assert_imm1!(LANE2); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_laneq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + static_assert_imm2!(LANE1); + static_assert_imm2!(LANE2); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [4 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 4 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 4 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_laneq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + static_assert_imm1!(LANE1); + static_assert_imm1!(LANE2); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_lane_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_lane_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + static_assert_imm3!(LANE1); + static_assert_imm3!(LANE2); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_laneq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + static_assert_imm4!(LANE1); + static_assert_imm4!(LANE2); + match LANE1 & 0b1111 { + 0 => simd_shuffle16!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle16!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 2 => simd_shuffle16!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 3 => simd_shuffle16!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 4 => simd_shuffle16!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 5 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 6 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 7 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), + 8 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), + 9 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), + 10 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), + 11 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), + 12 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), + 13 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), + 14 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), + 15 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_lane_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + static_assert_imm2!(LANE1); + static_assert_imm2!(LANE2); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [4 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 4 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 4 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_laneq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + static_assert_imm3!(LANE1); + static_assert_imm3!(LANE2); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_lane_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + static_assert_imm1!(LANE1); + static_assert_imm1!(LANE2); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_laneq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + static_assert_imm2!(LANE1); + static_assert_imm2!(LANE2); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [4 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 4 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 4 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_laneq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + static_assert_imm1!(LANE1); + static_assert_imm1!(LANE2); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_lane_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_lane_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + static_assert_imm3!(LANE1); + static_assert_imm3!(LANE2); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_laneq_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + static_assert_imm4!(LANE1); + static_assert_imm4!(LANE2); + match LANE1 & 0b1111 { + 0 => simd_shuffle16!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle16!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 2 => simd_shuffle16!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 3 => simd_shuffle16!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 4 => simd_shuffle16!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 5 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 6 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 7 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), + 8 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), + 9 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), + 10 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), + 11 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), + 12 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), + 13 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), + 14 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), + 15 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_lane_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_lane_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + static_assert_imm2!(LANE1); + static_assert_imm2!(LANE2); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [4 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 4 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 4 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_laneq_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + static_assert_imm3!(LANE1); + static_assert_imm3!(LANE2); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_laneq_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t { + static_assert_imm1!(LANE1); + static_assert_imm1!(LANE2); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_lane_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + static_assert_imm1!(LANE1); + static_assert_imm1!(LANE2); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_laneq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + static_assert_imm2!(LANE1); + static_assert_imm2!(LANE2); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [4 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 4 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 4 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_laneq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + static_assert_imm1!(LANE1); + static_assert_imm1!(LANE2); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_laneq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_laneq_s8(a: int8x8_t, b: int8x16_t) -> int8x8_t { + static_assert_imm3!(LANE1); + static_assert_imm4!(LANE2); + let a: int8x16_t = simd_shuffle16!(a, a, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_laneq_s16(a: int16x4_t, b: int16x8_t) -> int16x4_t { + static_assert_imm2!(LANE1); + static_assert_imm3!(LANE2); + let a: int16x8_t = simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [8 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 8 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 8 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_laneq_s32(a: int32x2_t, b: int32x4_t) -> int32x2_t { + static_assert_imm1!(LANE1); + static_assert_imm2!(LANE2); + let a: int32x4_t = simd_shuffle4!(a, a, [0, 1, 2, 3]); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [4 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_laneq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_laneq_u8(a: uint8x8_t, b: uint8x16_t) -> uint8x8_t { + static_assert_imm3!(LANE1); + static_assert_imm4!(LANE2); + let a: uint8x16_t = simd_shuffle16!(a, a, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_laneq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_laneq_u16(a: uint16x4_t, b: uint16x8_t) -> uint16x4_t { + static_assert_imm2!(LANE1); + static_assert_imm3!(LANE2); + let a: uint16x8_t = simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [8 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 8 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 8 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_laneq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_laneq_u32(a: uint32x2_t, b: uint32x4_t) -> uint32x2_t { + static_assert_imm1!(LANE1); + static_assert_imm2!(LANE2); + let a: uint32x4_t = simd_shuffle4!(a, a, [0, 1, 2, 3]); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [4 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_laneq_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_laneq_p8(a: poly8x8_t, b: poly8x16_t) -> poly8x8_t { + static_assert_imm3!(LANE1); + static_assert_imm4!(LANE2); + let a: poly8x16_t = simd_shuffle16!(a, a, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_laneq_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_laneq_p16(a: poly16x4_t, b: poly16x8_t) -> poly16x4_t { + static_assert_imm2!(LANE1); + static_assert_imm3!(LANE2); + let a: poly16x8_t = simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [8 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 8 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 8 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_laneq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_laneq_f32(a: float32x2_t, b: float32x4_t) -> float32x2_t { + static_assert_imm1!(LANE1); + static_assert_imm2!(LANE2); + let a: float32x4_t = simd_shuffle4!(a, a, [0, 1, 2, 3]); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [4 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_lane_s8(a: int8x16_t, b: int8x8_t) -> int8x16_t { + static_assert_imm4!(LANE1); + static_assert_imm3!(LANE2); + let b: int8x16_t = simd_shuffle16!(b, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + match LANE1 & 0b1111 { + 0 => simd_shuffle16!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle16!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 2 => simd_shuffle16!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 3 => simd_shuffle16!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 4 => simd_shuffle16!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 5 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 6 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 7 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), + 8 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), + 9 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), + 10 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), + 11 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), + 12 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), + 13 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), + 14 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), + 15 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_lane_s16(a: int16x8_t, b: int16x4_t) -> int16x8_t { + static_assert_imm3!(LANE1); + static_assert_imm2!(LANE2); + let b: int16x8_t = simd_shuffle8!(b, b, [0, 1, 2, 3, 4, 5, 6, 7]); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_lane_s32(a: int32x4_t, b: int32x2_t) -> int32x4_t { + static_assert_imm2!(LANE1); + static_assert_imm1!(LANE2); + let b: int32x4_t = simd_shuffle4!(b, b, [0, 1, 2, 3]); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [4 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 4 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 4 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_lane_u8(a: uint8x16_t, b: uint8x8_t) -> uint8x16_t { + static_assert_imm4!(LANE1); + static_assert_imm3!(LANE2); + let b: uint8x16_t = simd_shuffle16!(b, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + match LANE1 & 0b1111 { + 0 => simd_shuffle16!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle16!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 2 => simd_shuffle16!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 3 => simd_shuffle16!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 4 => simd_shuffle16!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 5 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 6 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 7 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), + 8 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), + 9 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), + 10 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), + 11 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), + 12 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), + 13 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), + 14 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), + 15 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_lane_u16(a: uint16x8_t, b: uint16x4_t) -> uint16x8_t { + static_assert_imm3!(LANE1); + static_assert_imm2!(LANE2); + let b: uint16x8_t = simd_shuffle8!(b, b, [0, 1, 2, 3, 4, 5, 6, 7]); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_lane_u32(a: uint32x4_t, b: uint32x2_t) -> uint32x4_t { + static_assert_imm2!(LANE1); + static_assert_imm1!(LANE2); + let b: uint32x4_t = simd_shuffle4!(b, b, [0, 1, 2, 3]); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [4 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 4 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 4 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_lane_p8(a: poly8x16_t, b: poly8x8_t) -> poly8x16_t { + static_assert_imm4!(LANE1); + static_assert_imm3!(LANE2); + let b: poly8x16_t = simd_shuffle16!(b, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + match LANE1 & 0b1111 { + 0 => simd_shuffle16!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle16!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 2 => simd_shuffle16!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 3 => simd_shuffle16!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 4 => simd_shuffle16!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 5 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 6 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 7 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), + 8 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), + 9 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), + 10 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), + 11 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), + 12 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), + 13 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), + 14 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), + 15 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_lane_p16(a: poly16x8_t, b: poly16x4_t) -> poly16x8_t { + static_assert_imm3!(LANE1); + static_assert_imm2!(LANE2); + let b: poly16x8_t = simd_shuffle8!(b, b, [0, 1, 2, 3, 4, 5, 6, 7]); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 1, LANE2 = 0))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_lane_s64(a: int64x2_t, b: int64x1_t) -> int64x2_t { + static_assert_imm1!(LANE1); + static_assert!(LANE2 : i32 where LANE2 == 0); + let b: int64x2_t = simd_shuffle2!(b, b, [0, 1]); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 1, LANE2 = 0))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_lane_u64(a: uint64x2_t, b: uint64x1_t) -> uint64x2_t { + static_assert_imm1!(LANE1); + static_assert!(LANE2 : i32 where LANE2 == 0); + let b: uint64x2_t = simd_shuffle2!(b, b, [0, 1]); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 1, LANE2 = 0))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_lane_p64(a: poly64x2_t, b: poly64x1_t) -> poly64x2_t { + static_assert_imm1!(LANE1); + static_assert!(LANE2 : i32 where LANE2 == 0); + let b: poly64x2_t = simd_shuffle2!(b, b, [0, 1]); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 1, LANE2 = 0))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_lane_f32(a: float32x4_t, b: float32x2_t) -> float32x4_t { + static_assert_imm2!(LANE1); + static_assert_imm1!(LANE2); + let b: float32x4_t = simd_shuffle4!(b, b, [0, 1, 2, 3]); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [4 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 4 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 4 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 1, LANE2 = 0))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopyq_lane_f64(a: float64x2_t, b: float64x1_t) -> float64x2_t { + static_assert_imm1!(LANE1); + static_assert!(LANE2 : i32 where LANE2 == 0); + let b: float64x2_t = simd_shuffle2!(b, b, [0, 1]); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcreate_f64(a: u64) -> float64x1_t { + transmute(a) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_f64_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(scvtf))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvt_f64_s64(a: int64x1_t) -> float64x1_t { + simd_cast(a) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_f64_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(scvtf))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtq_f64_s64(a: int64x2_t) -> float64x2_t { + simd_cast(a) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_f64_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ucvtf))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvt_f64_u64(a: uint64x1_t) -> float64x1_t { + simd_cast(a) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_f64_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ucvtf))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtq_f64_u64(a: uint64x2_t) -> float64x2_t { + simd_cast(a) +} + +/// Floating-point convert to higher precision long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_f64_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvt_f64_f32(a: float32x2_t) -> float64x2_t { + simd_cast(a) +} + +/// Floating-point convert to higher precision long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_high_f64_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvt_high_f64_f32(a: float32x4_t) -> float64x2_t { + let b: float32x2_t = simd_shuffle2!(a, a, [2, 3]); + simd_cast(b) +} + +/// Floating-point convert to lower precision narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_f32_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtn))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvt_f32_f64(a: float64x2_t) -> float32x2_t { + simd_cast(a) +} + +/// Floating-point convert to lower precision narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_high_f32_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtn))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvt_high_f32_f64(a: float32x2_t, b: float64x2_t) -> float32x4_t { + simd_shuffle4!(a, simd_cast(b), [0, 1, 2, 3]) +} + +/// Floating-point convert to lower precision narrow, rounding to odd +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtx_f32_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtxn))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtx_f32_f64(a: float64x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtxn.v2f32.v2f64")] + fn vcvtx_f32_f64_(a: float64x2_t) -> float32x2_t; + } + vcvtx_f32_f64_(a) +} + +/// Floating-point convert to lower precision narrow, rounding to odd +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtxd_f32_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtxn))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtxd_f32_f64(a: f64) -> f32 { + simd_extract(vcvtx_f32_f64(vdupq_n_f64(a)), 0) +} + +/// Floating-point convert to lower precision narrow, rounding to odd +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtx_high_f32_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtxn))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtx_high_f32_f64(a: float32x2_t, b: float64x2_t) -> float32x4_t { + simd_shuffle4!(a, vcvtx_f32_f64(b), [0, 1, 2, 3]) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_f64_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(scvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvt_n_f64_s64(a: int64x1_t) -> float64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxs2fp.v1f64.v1i64")] + fn vcvt_n_f64_s64_(a: int64x1_t, n: i32) -> float64x1_t; + } + vcvt_n_f64_s64_(a, N) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_f64_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(scvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtq_n_f64_s64(a: int64x2_t) -> float64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxs2fp.v2f64.v2i64")] + fn vcvtq_n_f64_s64_(a: int64x2_t, n: i32) -> float64x2_t; + } + vcvtq_n_f64_s64_(a, N) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvts_n_f32_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(scvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvts_n_f32_s32(a: i32) -> f32 { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxs2fp.f32.i32")] + fn vcvts_n_f32_s32_(a: i32, n: i32) -> f32; + } + vcvts_n_f32_s32_(a, N) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtd_n_f64_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(scvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtd_n_f64_s64(a: i64) -> f64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxs2fp.f64.i64")] + fn vcvtd_n_f64_s64_(a: i64, n: i32) -> f64; + } + vcvtd_n_f64_s64_(a, N) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_f64_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ucvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvt_n_f64_u64(a: uint64x1_t) -> float64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxu2fp.v1f64.v1i64")] + fn vcvt_n_f64_u64_(a: uint64x1_t, n: i32) -> float64x1_t; + } + vcvt_n_f64_u64_(a, N) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_f64_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ucvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtq_n_f64_u64(a: uint64x2_t) -> float64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxu2fp.v2f64.v2i64")] + fn vcvtq_n_f64_u64_(a: uint64x2_t, n: i32) -> float64x2_t; + } + vcvtq_n_f64_u64_(a, N) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvts_n_f32_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ucvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvts_n_f32_u32(a: u32) -> f32 { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxu2fp.f32.i32")] + fn vcvts_n_f32_u32_(a: u32, n: i32) -> f32; + } + vcvts_n_f32_u32_(a, N) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtd_n_f64_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ucvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtd_n_f64_u64(a: u64) -> f64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxu2fp.f64.i64")] + fn vcvtd_n_f64_u64_(a: u64, n: i32) -> f64; + } + vcvtd_n_f64_u64_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_s64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzs, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvt_n_s64_f64(a: float64x1_t) -> int64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxs.v1i64.v1f64")] + fn vcvt_n_s64_f64_(a: float64x1_t, n: i32) -> int64x1_t; + } + vcvt_n_s64_f64_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_s64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzs, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtq_n_s64_f64(a: float64x2_t) -> int64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxs.v2i64.v2f64")] + fn vcvtq_n_s64_f64_(a: float64x2_t, n: i32) -> int64x2_t; + } + vcvtq_n_s64_f64_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvts_n_s32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzs, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvts_n_s32_f32(a: f32) -> i32 { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxs.i32.f32")] + fn vcvts_n_s32_f32_(a: f32, n: i32) -> i32; + } + vcvts_n_s32_f32_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtd_n_s64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzs, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtd_n_s64_f64(a: f64) -> i64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxs.i64.f64")] + fn vcvtd_n_s64_f64_(a: f64, n: i32) -> i64; + } + vcvtd_n_s64_f64_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_u64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzu, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvt_n_u64_f64(a: float64x1_t) -> uint64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxu.v1i64.v1f64")] + fn vcvt_n_u64_f64_(a: float64x1_t, n: i32) -> uint64x1_t; + } + vcvt_n_u64_f64_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_u64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzu, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtq_n_u64_f64(a: float64x2_t) -> uint64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxu.v2i64.v2f64")] + fn vcvtq_n_u64_f64_(a: float64x2_t, n: i32) -> uint64x2_t; + } + vcvtq_n_u64_f64_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvts_n_u32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzu, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvts_n_u32_f32(a: f32) -> u32 { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxu.i32.f32")] + fn vcvts_n_u32_f32_(a: f32, n: i32) -> u32; + } + vcvts_n_u32_f32_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtd_n_u64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzu, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtd_n_u64_f64(a: f64) -> u64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxu.i64.f64")] + fn vcvtd_n_u64_f64_(a: f64, n: i32) -> u64; + } + vcvtd_n_u64_f64_(a, N) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvts_f32_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(scvtf))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvts_f32_s32(a: i32) -> f32 { + a as f32 +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtd_f64_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(scvtf))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtd_f64_s64(a: i64) -> f64 { + a as f64 +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvts_f32_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ucvtf))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvts_f32_u32(a: u32) -> f32 { + a as f32 +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtd_f64_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ucvtf))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtd_f64_u64(a: u64) -> f64 { + a as f64 +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvts_s32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzs))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvts_s32_f32(a: f32) -> i32 { + a as i32 +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtd_s64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzs))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtd_s64_f64(a: f64) -> i64 { + a as i64 +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvts_u32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvts_u32_f32(a: f32) -> u32 { + a as u32 +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtd_u64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtd_u64_f64(a: f64) -> u64 { + a as u64 +} + +/// Floating-point convert to signed fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_s64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzs))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvt_s64_f64(a: float64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fptosi.sat.v1i64.v1f64")] + fn vcvt_s64_f64_(a: float64x1_t) -> int64x1_t; + } + vcvt_s64_f64_(a) +} + +/// Floating-point convert to signed fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_s64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzs))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtq_s64_f64(a: float64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fptosi.sat.v2i64.v2f64")] + fn vcvtq_s64_f64_(a: float64x2_t) -> int64x2_t; + } + vcvtq_s64_f64_(a) +} + +/// Floating-point convert to unsigned fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_u64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvt_u64_f64(a: float64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fptoui.sat.v1i64.v1f64")] + fn vcvt_u64_f64_(a: float64x1_t) -> uint64x1_t; + } + vcvt_u64_f64_(a) +} + +/// Floating-point convert to unsigned fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_u64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtq_u64_f64(a: float64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fptoui.sat.v2i64.v2f64")] + fn vcvtq_u64_f64_(a: float64x2_t) -> uint64x2_t; + } + vcvtq_u64_f64_(a) +} + +/// Floating-point convert to signed integer, rounding to nearest with ties to away +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvta_s32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtas))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvta_s32_f32(a: float32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtas.v2i32.v2f32")] + fn vcvta_s32_f32_(a: float32x2_t) -> int32x2_t; + } + vcvta_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding to nearest with ties to away +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtaq_s32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtas))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtaq_s32_f32(a: float32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtas.v4i32.v4f32")] + fn vcvtaq_s32_f32_(a: float32x4_t) -> int32x4_t; + } + vcvtaq_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding to nearest with ties to away +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvta_s64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtas))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvta_s64_f64(a: float64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtas.v1i64.v1f64")] + fn vcvta_s64_f64_(a: float64x1_t) -> int64x1_t; + } + vcvta_s64_f64_(a) +} + +/// Floating-point convert to signed integer, rounding to nearest with ties to away +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtaq_s64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtas))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtaq_s64_f64(a: float64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtas.v2i64.v2f64")] + fn vcvtaq_s64_f64_(a: float64x2_t) -> int64x2_t; + } + vcvtaq_s64_f64_(a) +} + +/// Floating-point convert to integer, rounding to nearest with ties to away +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtas_s32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtas))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtas_s32_f32(a: f32) -> i32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtas.i32.f32")] + fn vcvtas_s32_f32_(a: f32) -> i32; + } + vcvtas_s32_f32_(a) +} + +/// Floating-point convert to integer, rounding to nearest with ties to away +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtad_s64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtas))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtad_s64_f64(a: f64) -> i64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtas.i64.f64")] + fn vcvtad_s64_f64_(a: f64) -> i64; + } + vcvtad_s64_f64_(a) +} + +/// Floating-point convert to integer, rounding to nearest with ties to away +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtas_u32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtau))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtas_u32_f32(a: f32) -> u32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtau.i32.f32")] + fn vcvtas_u32_f32_(a: f32) -> u32; + } + vcvtas_u32_f32_(a) +} + +/// Floating-point convert to integer, rounding to nearest with ties to away +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtad_u64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtau))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtad_u64_f64(a: f64) -> u64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtau.i64.f64")] + fn vcvtad_u64_f64_(a: f64) -> u64; + } + vcvtad_u64_f64_(a) +} + +/// Floating-point convert to signed integer, rounding to nearest with ties to even +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtn_s32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtns))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtn_s32_f32(a: float32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtns.v2i32.v2f32")] + fn vcvtn_s32_f32_(a: float32x2_t) -> int32x2_t; + } + vcvtn_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding to nearest with ties to even +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtnq_s32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtns))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtnq_s32_f32(a: float32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtns.v4i32.v4f32")] + fn vcvtnq_s32_f32_(a: float32x4_t) -> int32x4_t; + } + vcvtnq_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding to nearest with ties to even +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtn_s64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtns))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtn_s64_f64(a: float64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtns.v1i64.v1f64")] + fn vcvtn_s64_f64_(a: float64x1_t) -> int64x1_t; + } + vcvtn_s64_f64_(a) +} + +/// Floating-point convert to signed integer, rounding to nearest with ties to even +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtnq_s64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtns))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtnq_s64_f64(a: float64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtns.v2i64.v2f64")] + fn vcvtnq_s64_f64_(a: float64x2_t) -> int64x2_t; + } + vcvtnq_s64_f64_(a) +} + +/// Floating-point convert to signed integer, rounding to nearest with ties to even +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtns_s32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtns))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtns_s32_f32(a: f32) -> i32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtns.i32.f32")] + fn vcvtns_s32_f32_(a: f32) -> i32; + } + vcvtns_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding to nearest with ties to even +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtnd_s64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtns))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtnd_s64_f64(a: f64) -> i64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtns.i64.f64")] + fn vcvtnd_s64_f64_(a: f64) -> i64; + } + vcvtnd_s64_f64_(a) +} + +/// Floating-point convert to signed integer, rounding toward minus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtm_s32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtms))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtm_s32_f32(a: float32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtms.v2i32.v2f32")] + fn vcvtm_s32_f32_(a: float32x2_t) -> int32x2_t; + } + vcvtm_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding toward minus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtmq_s32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtms))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtmq_s32_f32(a: float32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtms.v4i32.v4f32")] + fn vcvtmq_s32_f32_(a: float32x4_t) -> int32x4_t; + } + vcvtmq_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding toward minus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtm_s64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtms))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtm_s64_f64(a: float64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtms.v1i64.v1f64")] + fn vcvtm_s64_f64_(a: float64x1_t) -> int64x1_t; + } + vcvtm_s64_f64_(a) +} + +/// Floating-point convert to signed integer, rounding toward minus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtmq_s64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtms))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtmq_s64_f64(a: float64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtms.v2i64.v2f64")] + fn vcvtmq_s64_f64_(a: float64x2_t) -> int64x2_t; + } + vcvtmq_s64_f64_(a) +} + +/// Floating-point convert to signed integer, rounding toward minus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtms_s32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtms))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtms_s32_f32(a: f32) -> i32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtms.i32.f32")] + fn vcvtms_s32_f32_(a: f32) -> i32; + } + vcvtms_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding toward minus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtmd_s64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtms))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtmd_s64_f64(a: f64) -> i64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtms.i64.f64")] + fn vcvtmd_s64_f64_(a: f64) -> i64; + } + vcvtmd_s64_f64_(a) +} + +/// Floating-point convert to signed integer, rounding toward plus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtp_s32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtps))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtp_s32_f32(a: float32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtps.v2i32.v2f32")] + fn vcvtp_s32_f32_(a: float32x2_t) -> int32x2_t; + } + vcvtp_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding toward plus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtpq_s32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtps))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtpq_s32_f32(a: float32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtps.v4i32.v4f32")] + fn vcvtpq_s32_f32_(a: float32x4_t) -> int32x4_t; + } + vcvtpq_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding toward plus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtp_s64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtps))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtp_s64_f64(a: float64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtps.v1i64.v1f64")] + fn vcvtp_s64_f64_(a: float64x1_t) -> int64x1_t; + } + vcvtp_s64_f64_(a) +} + +/// Floating-point convert to signed integer, rounding toward plus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtpq_s64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtps))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtpq_s64_f64(a: float64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtps.v2i64.v2f64")] + fn vcvtpq_s64_f64_(a: float64x2_t) -> int64x2_t; + } + vcvtpq_s64_f64_(a) +} + +/// Floating-point convert to signed integer, rounding toward plus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtps_s32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtps))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtps_s32_f32(a: f32) -> i32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtps.i32.f32")] + fn vcvtps_s32_f32_(a: f32) -> i32; + } + vcvtps_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding toward plus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtpd_s64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtps))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtpd_s64_f64(a: f64) -> i64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtps.i64.f64")] + fn vcvtpd_s64_f64_(a: f64) -> i64; + } + vcvtpd_s64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to away +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvta_u32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtau))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvta_u32_f32(a: float32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtau.v2i32.v2f32")] + fn vcvta_u32_f32_(a: float32x2_t) -> uint32x2_t; + } + vcvta_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to away +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtaq_u32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtau))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtaq_u32_f32(a: float32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtau.v4i32.v4f32")] + fn vcvtaq_u32_f32_(a: float32x4_t) -> uint32x4_t; + } + vcvtaq_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to away +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvta_u64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtau))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvta_u64_f64(a: float64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtau.v1i64.v1f64")] + fn vcvta_u64_f64_(a: float64x1_t) -> uint64x1_t; + } + vcvta_u64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to away +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtaq_u64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtau))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtaq_u64_f64(a: float64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtau.v2i64.v2f64")] + fn vcvtaq_u64_f64_(a: float64x2_t) -> uint64x2_t; + } + vcvtaq_u64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to even +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtn_u32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtnu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtn_u32_f32(a: float32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtnu.v2i32.v2f32")] + fn vcvtn_u32_f32_(a: float32x2_t) -> uint32x2_t; + } + vcvtn_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to even +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtnq_u32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtnu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtnq_u32_f32(a: float32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtnu.v4i32.v4f32")] + fn vcvtnq_u32_f32_(a: float32x4_t) -> uint32x4_t; + } + vcvtnq_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to even +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtn_u64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtnu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtn_u64_f64(a: float64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtnu.v1i64.v1f64")] + fn vcvtn_u64_f64_(a: float64x1_t) -> uint64x1_t; + } + vcvtn_u64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to even +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtnq_u64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtnu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtnq_u64_f64(a: float64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtnu.v2i64.v2f64")] + fn vcvtnq_u64_f64_(a: float64x2_t) -> uint64x2_t; + } + vcvtnq_u64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to even +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtns_u32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtnu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtns_u32_f32(a: f32) -> u32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtnu.i32.f32")] + fn vcvtns_u32_f32_(a: f32) -> u32; + } + vcvtns_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to even +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtnd_u64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtnu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtnd_u64_f64(a: f64) -> u64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtnu.i64.f64")] + fn vcvtnd_u64_f64_(a: f64) -> u64; + } + vcvtnd_u64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward minus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtm_u32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtmu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtm_u32_f32(a: float32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtmu.v2i32.v2f32")] + fn vcvtm_u32_f32_(a: float32x2_t) -> uint32x2_t; + } + vcvtm_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward minus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtmq_u32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtmu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtmq_u32_f32(a: float32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtmu.v4i32.v4f32")] + fn vcvtmq_u32_f32_(a: float32x4_t) -> uint32x4_t; + } + vcvtmq_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward minus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtm_u64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtmu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtm_u64_f64(a: float64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtmu.v1i64.v1f64")] + fn vcvtm_u64_f64_(a: float64x1_t) -> uint64x1_t; + } + vcvtm_u64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward minus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtmq_u64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtmu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtmq_u64_f64(a: float64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtmu.v2i64.v2f64")] + fn vcvtmq_u64_f64_(a: float64x2_t) -> uint64x2_t; + } + vcvtmq_u64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward minus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtms_u32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtmu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtms_u32_f32(a: f32) -> u32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtmu.i32.f32")] + fn vcvtms_u32_f32_(a: f32) -> u32; + } + vcvtms_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward minus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtmd_u64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtmu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtmd_u64_f64(a: f64) -> u64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtmu.i64.f64")] + fn vcvtmd_u64_f64_(a: f64) -> u64; + } + vcvtmd_u64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward plus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtp_u32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtpu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtp_u32_f32(a: float32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtpu.v2i32.v2f32")] + fn vcvtp_u32_f32_(a: float32x2_t) -> uint32x2_t; + } + vcvtp_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward plus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtpq_u32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtpu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtpq_u32_f32(a: float32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtpu.v4i32.v4f32")] + fn vcvtpq_u32_f32_(a: float32x4_t) -> uint32x4_t; + } + vcvtpq_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward plus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtp_u64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtpu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtp_u64_f64(a: float64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtpu.v1i64.v1f64")] + fn vcvtp_u64_f64_(a: float64x1_t) -> uint64x1_t; + } + vcvtp_u64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward plus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtpq_u64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtpu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtpq_u64_f64(a: float64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtpu.v2i64.v2f64")] + fn vcvtpq_u64_f64_(a: float64x2_t) -> uint64x2_t; + } + vcvtpq_u64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward plus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtps_u32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtpu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtps_u32_f32(a: f32) -> u32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtpu.i32.f32")] + fn vcvtps_u32_f32_(a: f32) -> u32; + } + vcvtps_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward plus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtpd_u64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtpu))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtpd_u64_f64(a: f64) -> u64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtpu.i64.f64")] + fn vcvtpd_u64_f64_(a: f64) -> u64; + } + vcvtpd_u64_f64_(a) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(dup, N = 1))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdupq_laneq_p64(a: poly64x2_t) -> poly64x2_t { + static_assert_imm1!(N); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(dup, N = 0))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdupq_lane_p64(a: poly64x1_t) -> poly64x2_t { + static_assert!(N : i32 where N == 0); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(dup, N = 1))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdupq_laneq_f64(a: float64x2_t) -> float64x2_t { + static_assert_imm1!(N); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(dup, N = 0))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdupq_lane_f64(a: float64x1_t) -> float64x2_t { + static_assert!(N : i32 where N == 0); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 0))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdup_lane_p64(a: poly64x1_t) -> poly64x1_t { + static_assert!(N : i32 where N == 0); + a +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 0))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdup_lane_f64(a: float64x1_t) -> float64x1_t { + static_assert!(N : i32 where N == 0); + a +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 1))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdup_laneq_p64(a: poly64x2_t) -> poly64x1_t { + static_assert_imm1!(N); + transmute::(simd_extract(a, N as u32)) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 1))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdup_laneq_f64(a: float64x2_t) -> float64x1_t { + static_assert_imm1!(N); + transmute::(simd_extract(a, N as u32)) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupb_lane_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 4))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdupb_lane_s8(a: int8x8_t) -> i8 { + static_assert_imm3!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupb_laneq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 8))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdupb_laneq_s8(a: int8x16_t) -> i8 { + static_assert_imm4!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vduph_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vduph_lane_s16(a: int16x4_t) -> i16 { + static_assert_imm2!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vduph_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 4))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vduph_laneq_s16(a: int16x8_t) -> i16 { + static_assert_imm3!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdups_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 1))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdups_lane_s32(a: int32x2_t) -> i32 { + static_assert_imm1!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdups_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdups_laneq_s32(a: int32x4_t) -> i32 { + static_assert_imm2!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupd_lane_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 0))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdupd_lane_s64(a: int64x1_t) -> i64 { + static_assert!(N : i32 where N == 0); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupd_laneq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 1))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdupd_laneq_s64(a: int64x2_t) -> i64 { + static_assert_imm1!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupb_lane_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 4))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdupb_lane_u8(a: uint8x8_t) -> u8 { + static_assert_imm3!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupb_laneq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 8))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdupb_laneq_u8(a: uint8x16_t) -> u8 { + static_assert_imm4!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vduph_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vduph_lane_u16(a: uint16x4_t) -> u16 { + static_assert_imm2!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vduph_laneq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 4))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vduph_laneq_u16(a: uint16x8_t) -> u16 { + static_assert_imm3!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdups_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 1))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdups_lane_u32(a: uint32x2_t) -> u32 { + static_assert_imm1!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdups_laneq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdups_laneq_u32(a: uint32x4_t) -> u32 { + static_assert_imm2!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupd_lane_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 0))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdupd_lane_u64(a: uint64x1_t) -> u64 { + static_assert!(N : i32 where N == 0); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupd_laneq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 1))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdupd_laneq_u64(a: uint64x2_t) -> u64 { + static_assert_imm1!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupb_lane_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 4))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdupb_lane_p8(a: poly8x8_t) -> p8 { + static_assert_imm3!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupb_laneq_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 8))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdupb_laneq_p8(a: poly8x16_t) -> p8 { + static_assert_imm4!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vduph_lane_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vduph_lane_p16(a: poly16x4_t) -> p16 { + static_assert_imm2!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vduph_laneq_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 4))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vduph_laneq_p16(a: poly16x8_t) -> p16 { + static_assert_imm3!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdups_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 1))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdups_lane_f32(a: float32x2_t) -> f32 { + static_assert_imm1!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdups_laneq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdups_laneq_f32(a: float32x4_t) -> f32 { + static_assert_imm2!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupd_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 0))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdupd_lane_f64(a: float64x1_t) -> f64 { + static_assert!(N : i32 where N == 0); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupd_laneq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 1))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdupd_laneq_f64(a: float64x2_t) -> f64 { + static_assert_imm1!(N); + simd_extract(a, N as u32) +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ext, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vextq_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t { + static_assert_imm1!(N); + match N & 0b1 { + 0 => simd_shuffle2!(a, b, [0, 1]), + 1 => simd_shuffle2!(a, b, [1, 2]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ext, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vextq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + static_assert_imm1!(N); + match N & 0b1 { + 0 => simd_shuffle2!(a, b, [0, 1]), + 1 => simd_shuffle2!(a, b, [1, 2]), + _ => unreachable_unchecked(), + } +} + +/// Floating-point multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmla_f64(a: float64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t { + simd_add(a, simd_mul(b, c)) +} + +/// Floating-point multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlaq_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t { + simd_add(a, simd_mul(b, c)) +} + +/// Signed multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlal2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlal_high_s8(a: int16x8_t, b: int8x16_t, c: int8x16_t) -> int16x8_t { + let b: int8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let c: int8x8_t = simd_shuffle8!(c, c, [8, 9, 10, 11, 12, 13, 14, 15]); + vmlal_s8(a, b, c) +} + +/// Signed multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlal2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlal_high_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t { + let b: int16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let c: int16x4_t = simd_shuffle4!(c, c, [4, 5, 6, 7]); + vmlal_s16(a, b, c) +} + +/// Signed multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlal2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlal_high_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t { + let b: int32x2_t = simd_shuffle2!(b, b, [2, 3]); + let c: int32x2_t = simd_shuffle2!(c, c, [2, 3]); + vmlal_s32(a, b, c) +} + +/// Unsigned multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlal2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlal_high_u8(a: uint16x8_t, b: uint8x16_t, c: uint8x16_t) -> uint16x8_t { + let b: uint8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let c: uint8x8_t = simd_shuffle8!(c, c, [8, 9, 10, 11, 12, 13, 14, 15]); + vmlal_u8(a, b, c) +} + +/// Unsigned multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlal2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlal_high_u16(a: uint32x4_t, b: uint16x8_t, c: uint16x8_t) -> uint32x4_t { + let b: uint16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let c: uint16x4_t = simd_shuffle4!(c, c, [4, 5, 6, 7]); + vmlal_u16(a, b, c) +} + +/// Unsigned multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlal2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlal_high_u32(a: uint64x2_t, b: uint32x4_t, c: uint32x4_t) -> uint64x2_t { + let b: uint32x2_t = simd_shuffle2!(b, b, [2, 3]); + let c: uint32x2_t = simd_shuffle2!(c, c, [2, 3]); + vmlal_u32(a, b, c) +} + +/// Multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlal2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlal_high_n_s16(a: int32x4_t, b: int16x8_t, c: i16) -> int32x4_t { + vmlal_high_s16(a, b, vdupq_n_s16(c)) +} + +/// Multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlal2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlal_high_n_s32(a: int64x2_t, b: int32x4_t, c: i32) -> int64x2_t { + vmlal_high_s32(a, b, vdupq_n_s32(c)) +} + +/// Multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlal2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlal_high_n_u16(a: uint32x4_t, b: uint16x8_t, c: u16) -> uint32x4_t { + vmlal_high_u16(a, b, vdupq_n_u16(c)) +} + +/// Multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlal2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlal_high_n_u32(a: uint64x2_t, b: uint32x4_t, c: u32) -> uint64x2_t { + vmlal_high_u32(a, b, vdupq_n_u32(c)) +} + +/// Multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlal2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlal_high_lane_s16(a: int32x4_t, b: int16x8_t, c: int16x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + vmlal_high_s16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlal2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlal_high_laneq_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t { + static_assert_imm3!(LANE); + vmlal_high_s16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlal2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlal_high_lane_s32(a: int64x2_t, b: int32x4_t, c: int32x2_t) -> int64x2_t { + static_assert_imm1!(LANE); + vmlal_high_s32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlal2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlal_high_laneq_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t { + static_assert_imm2!(LANE); + vmlal_high_s32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlal2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlal_high_lane_u16(a: uint32x4_t, b: uint16x8_t, c: uint16x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + vmlal_high_u16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_laneq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlal2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlal_high_laneq_u16(a: uint32x4_t, b: uint16x8_t, c: uint16x8_t) -> uint32x4_t { + static_assert_imm3!(LANE); + vmlal_high_u16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlal2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlal_high_lane_u32(a: uint64x2_t, b: uint32x4_t, c: uint32x2_t) -> uint64x2_t { + static_assert_imm1!(LANE); + vmlal_high_u32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_laneq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlal2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlal_high_laneq_u32(a: uint64x2_t, b: uint32x4_t, c: uint32x4_t) -> uint64x2_t { + static_assert_imm2!(LANE); + vmlal_high_u32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Floating-point multiply-subtract from accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmls_f64(a: float64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Floating-point multiply-subtract from accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlsq_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Signed multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlsl2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlsl_high_s8(a: int16x8_t, b: int8x16_t, c: int8x16_t) -> int16x8_t { + let b: int8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let c: int8x8_t = simd_shuffle8!(c, c, [8, 9, 10, 11, 12, 13, 14, 15]); + vmlsl_s8(a, b, c) +} + +/// Signed multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlsl2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlsl_high_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t { + let b: int16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let c: int16x4_t = simd_shuffle4!(c, c, [4, 5, 6, 7]); + vmlsl_s16(a, b, c) +} + +/// Signed multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlsl2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlsl_high_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t { + let b: int32x2_t = simd_shuffle2!(b, b, [2, 3]); + let c: int32x2_t = simd_shuffle2!(c, c, [2, 3]); + vmlsl_s32(a, b, c) +} + +/// Unsigned multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlsl2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlsl_high_u8(a: uint16x8_t, b: uint8x16_t, c: uint8x16_t) -> uint16x8_t { + let b: uint8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let c: uint8x8_t = simd_shuffle8!(c, c, [8, 9, 10, 11, 12, 13, 14, 15]); + vmlsl_u8(a, b, c) +} + +/// Unsigned multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlsl2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlsl_high_u16(a: uint32x4_t, b: uint16x8_t, c: uint16x8_t) -> uint32x4_t { + let b: uint16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let c: uint16x4_t = simd_shuffle4!(c, c, [4, 5, 6, 7]); + vmlsl_u16(a, b, c) +} + +/// Unsigned multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlsl2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlsl_high_u32(a: uint64x2_t, b: uint32x4_t, c: uint32x4_t) -> uint64x2_t { + let b: uint32x2_t = simd_shuffle2!(b, b, [2, 3]); + let c: uint32x2_t = simd_shuffle2!(c, c, [2, 3]); + vmlsl_u32(a, b, c) +} + +/// Multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlsl2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlsl_high_n_s16(a: int32x4_t, b: int16x8_t, c: i16) -> int32x4_t { + vmlsl_high_s16(a, b, vdupq_n_s16(c)) +} + +/// Multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlsl2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlsl_high_n_s32(a: int64x2_t, b: int32x4_t, c: i32) -> int64x2_t { + vmlsl_high_s32(a, b, vdupq_n_s32(c)) +} + +/// Multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlsl2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlsl_high_n_u16(a: uint32x4_t, b: uint16x8_t, c: u16) -> uint32x4_t { + vmlsl_high_u16(a, b, vdupq_n_u16(c)) +} + +/// Multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlsl2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlsl_high_n_u32(a: uint64x2_t, b: uint32x4_t, c: u32) -> uint64x2_t { + vmlsl_high_u32(a, b, vdupq_n_u32(c)) +} + +/// Multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlsl2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlsl_high_lane_s16(a: int32x4_t, b: int16x8_t, c: int16x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + vmlsl_high_s16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlsl2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlsl_high_laneq_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t { + static_assert_imm3!(LANE); + vmlsl_high_s16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlsl2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlsl_high_lane_s32(a: int64x2_t, b: int32x4_t, c: int32x2_t) -> int64x2_t { + static_assert_imm1!(LANE); + vmlsl_high_s32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlsl2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlsl_high_laneq_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t { + static_assert_imm2!(LANE); + vmlsl_high_s32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlsl2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlsl_high_lane_u16(a: uint32x4_t, b: uint16x8_t, c: uint16x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + vmlsl_high_u16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_laneq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlsl2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlsl_high_laneq_u16(a: uint32x4_t, b: uint16x8_t, c: uint16x8_t) -> uint32x4_t { + static_assert_imm3!(LANE); + vmlsl_high_u16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlsl2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlsl_high_lane_u32(a: uint64x2_t, b: uint32x4_t, c: uint32x2_t) -> uint64x2_t { + static_assert_imm1!(LANE); + vmlsl_high_u32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_laneq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlsl2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmlsl_high_laneq_u32(a: uint64x2_t, b: uint32x4_t, c: uint32x4_t) -> uint64x2_t { + static_assert_imm2!(LANE); + vmlsl_high_u32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovn_high_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(xtn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmovn_high_s16(a: int8x8_t, b: int16x8_t) -> int8x16_t { + let c: int8x8_t = simd_cast(b); + simd_shuffle16!(a, c, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovn_high_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(xtn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmovn_high_s32(a: int16x4_t, b: int32x4_t) -> int16x8_t { + let c: int16x4_t = simd_cast(b); + simd_shuffle8!(a, c, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovn_high_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(xtn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmovn_high_s64(a: int32x2_t, b: int64x2_t) -> int32x4_t { + let c: int32x2_t = simd_cast(b); + simd_shuffle4!(a, c, [0, 1, 2, 3]) +} + +/// Extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovn_high_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(xtn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmovn_high_u16(a: uint8x8_t, b: uint16x8_t) -> uint8x16_t { + let c: uint8x8_t = simd_cast(b); + simd_shuffle16!(a, c, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovn_high_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(xtn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmovn_high_u32(a: uint16x4_t, b: uint32x4_t) -> uint16x8_t { + let c: uint16x4_t = simd_cast(b); + simd_shuffle8!(a, c, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovn_high_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(xtn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmovn_high_u64(a: uint32x2_t, b: uint64x2_t) -> uint32x4_t { + let c: uint32x2_t = simd_cast(b); + simd_shuffle4!(a, c, [0, 1, 2, 3]) +} + +/// Negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vneg_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(neg))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vneg_s64(a: int64x1_t) -> int64x1_t { + simd_neg(a) +} + +/// Negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vnegq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(neg))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vnegq_s64(a: int64x2_t) -> int64x2_t { + simd_neg(a) +} + +/// Negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vnegd_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(neg))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vnegd_s64(a: i64) -> i64 { + a.wrapping_neg() +} + +/// Negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vneg_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fneg))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vneg_f64(a: float64x1_t) -> float64x1_t { + simd_neg(a) +} + +/// Negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vnegq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fneg))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vnegq_f64(a: float64x2_t) -> float64x2_t { + simd_neg(a) +} + +/// Signed saturating negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqneg_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqneg))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqneg_s64(a: int64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqneg.v1i64")] + fn vqneg_s64_(a: int64x1_t) -> int64x1_t; + } + vqneg_s64_(a) +} + +/// Signed saturating negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqnegq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqneg))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqnegq_s64(a: int64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqneg.v2i64")] + fn vqnegq_s64_(a: int64x2_t) -> int64x2_t; + } + vqnegq_s64_(a) +} + +/// Signed saturating negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqnegb_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqneg))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqnegb_s8(a: i8) -> i8 { + simd_extract(vqneg_s8(vdup_n_s8(a)), 0) +} + +/// Signed saturating negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqnegh_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqneg))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqnegh_s16(a: i16) -> i16 { + simd_extract(vqneg_s16(vdup_n_s16(a)), 0) +} + +/// Signed saturating negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqnegs_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqneg))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqnegs_s32(a: i32) -> i32 { + simd_extract(vqneg_s32(vdup_n_s32(a)), 0) +} + +/// Signed saturating negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqnegd_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqneg))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqnegd_s64(a: i64) -> i64 { + simd_extract(vqneg_s64(vdup_n_s64(a)), 0) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubb_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqsub))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqsubb_s8(a: i8, b: i8) -> i8 { + let a: int8x8_t = vdup_n_s8(a); + let b: int8x8_t = vdup_n_s8(b); + simd_extract(vqsub_s8(a, b), 0) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubh_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqsub))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqsubh_s16(a: i16, b: i16) -> i16 { + let a: int16x4_t = vdup_n_s16(a); + let b: int16x4_t = vdup_n_s16(b); + simd_extract(vqsub_s16(a, b), 0) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubb_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqsub))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqsubb_u8(a: u8, b: u8) -> u8 { + let a: uint8x8_t = vdup_n_u8(a); + let b: uint8x8_t = vdup_n_u8(b); + simd_extract(vqsub_u8(a, b), 0) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubh_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqsub))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqsubh_u16(a: u16, b: u16) -> u16 { + let a: uint16x4_t = vdup_n_u16(a); + let b: uint16x4_t = vdup_n_u16(b); + simd_extract(vqsub_u16(a, b), 0) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubs_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqsub))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqsubs_u32(a: u32, b: u32) -> u32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.i32")] + fn vqsubs_u32_(a: u32, b: u32) -> u32; + } + vqsubs_u32_(a, b) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubd_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqsub))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqsubd_u64(a: u64, b: u64) -> u64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.i64")] + fn vqsubd_u64_(a: u64, b: u64) -> u64; + } + vqsubd_u64_(a, b) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubs_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqsub))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqsubs_s32(a: i32, b: i32) -> i32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.i32")] + fn vqsubs_s32_(a: i32, b: i32) -> i32; + } + vqsubs_s32_(a, b) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubd_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqsub))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqsubd_s64(a: i64, b: i64) -> i64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.i64")] + fn vqsubd_s64_(a: i64, b: i64) -> i64; + } + vqsubd_s64_(a, b) +} + +/// Reverse bit order +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrbit_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rbit))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrbit_s8(a: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.rbit.v8i8")] + fn vrbit_s8_(a: int8x8_t) -> int8x8_t; + } + vrbit_s8_(a) +} + +/// Reverse bit order +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrbitq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rbit))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrbitq_s8(a: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.rbit.v16i8")] + fn vrbitq_s8_(a: int8x16_t) -> int8x16_t; + } + vrbitq_s8_(a) +} + +/// Reverse bit order +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrbit_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rbit))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrbit_u8(a: uint8x8_t) -> uint8x8_t { + transmute(vrbit_s8(transmute(a))) +} + +/// Reverse bit order +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrbitq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rbit))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrbitq_u8(a: uint8x16_t) -> uint8x16_t { + transmute(vrbitq_s8(transmute(a))) +} + +/// Reverse bit order +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrbit_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rbit))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrbit_p8(a: poly8x8_t) -> poly8x8_t { + transmute(vrbit_s8(transmute(a))) +} + +/// Reverse bit order +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrbitq_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rbit))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrbitq_p8(a: poly8x16_t) -> poly8x16_t { + transmute(vrbitq_s8(transmute(a))) +} + +/// Floating-point round to integral exact, using current rounding mode +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndx_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndx_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.rint.v2f32")] + fn vrndx_f32_(a: float32x2_t) -> float32x2_t; + } + vrndx_f32_(a) +} + +/// Floating-point round to integral exact, using current rounding mode +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndxq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndxq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.rint.v4f32")] + fn vrndxq_f32_(a: float32x4_t) -> float32x4_t; + } + vrndxq_f32_(a) +} + +/// Floating-point round to integral exact, using current rounding mode +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndx_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndx_f64(a: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.rint.v1f64")] + fn vrndx_f64_(a: float64x1_t) -> float64x1_t; + } + vrndx_f64_(a) +} + +/// Floating-point round to integral exact, using current rounding mode +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndxq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndxq_f64(a: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.rint.v2f64")] + fn vrndxq_f64_(a: float64x2_t) -> float64x2_t; + } + vrndxq_f64_(a) +} + +/// Floating-point round to integral, to nearest with ties to away +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnda_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frinta))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrnda_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.round.v2f32")] + fn vrnda_f32_(a: float32x2_t) -> float32x2_t; + } + vrnda_f32_(a) +} + +/// Floating-point round to integral, to nearest with ties to away +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndaq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frinta))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndaq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.round.v4f32")] + fn vrndaq_f32_(a: float32x4_t) -> float32x4_t; + } + vrndaq_f32_(a) +} + +/// Floating-point round to integral, to nearest with ties to away +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnda_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frinta))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrnda_f64(a: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.round.v1f64")] + fn vrnda_f64_(a: float64x1_t) -> float64x1_t; + } + vrnda_f64_(a) +} + +/// Floating-point round to integral, to nearest with ties to away +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndaq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frinta))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndaq_f64(a: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.round.v2f64")] + fn vrndaq_f64_(a: float64x2_t) -> float64x2_t; + } + vrndaq_f64_(a) +} + +/// Floating-point round to integral, to nearest with ties to even +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndn_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintn))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndn_f64(a: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frintn.v1f64")] + fn vrndn_f64_(a: float64x1_t) -> float64x1_t; + } + vrndn_f64_(a) +} + +/// Floating-point round to integral, to nearest with ties to even +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndnq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintn))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndnq_f64(a: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frintn.v2f64")] + fn vrndnq_f64_(a: float64x2_t) -> float64x2_t; + } + vrndnq_f64_(a) +} + +/// Floating-point round to integral, to nearest with ties to even +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndns_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintn))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndns_f32(a: f32) -> f32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.roundeven.f32")] + fn vrndns_f32_(a: f32) -> f32; + } + vrndns_f32_(a) +} + +/// Floating-point round to integral, toward minus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndm_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintm))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndm_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.floor.v2f32")] + fn vrndm_f32_(a: float32x2_t) -> float32x2_t; + } + vrndm_f32_(a) +} + +/// Floating-point round to integral, toward minus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndmq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintm))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndmq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.floor.v4f32")] + fn vrndmq_f32_(a: float32x4_t) -> float32x4_t; + } + vrndmq_f32_(a) +} + +/// Floating-point round to integral, toward minus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndm_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintm))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndm_f64(a: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.floor.v1f64")] + fn vrndm_f64_(a: float64x1_t) -> float64x1_t; + } + vrndm_f64_(a) +} + +/// Floating-point round to integral, toward minus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndmq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintm))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndmq_f64(a: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.floor.v2f64")] + fn vrndmq_f64_(a: float64x2_t) -> float64x2_t; + } + vrndmq_f64_(a) +} + +/// Floating-point round to integral, toward plus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndp_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndp_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ceil.v2f32")] + fn vrndp_f32_(a: float32x2_t) -> float32x2_t; + } + vrndp_f32_(a) +} + +/// Floating-point round to integral, toward plus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndpq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndpq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ceil.v4f32")] + fn vrndpq_f32_(a: float32x4_t) -> float32x4_t; + } + vrndpq_f32_(a) +} + +/// Floating-point round to integral, toward plus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndp_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndp_f64(a: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ceil.v1f64")] + fn vrndp_f64_(a: float64x1_t) -> float64x1_t; + } + vrndp_f64_(a) +} + +/// Floating-point round to integral, toward plus infinity +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndpq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndpq_f64(a: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ceil.v2f64")] + fn vrndpq_f64_(a: float64x2_t) -> float64x2_t; + } + vrndpq_f64_(a) +} + +/// Floating-point round to integral, toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintz))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrnd_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.trunc.v2f32")] + fn vrnd_f32_(a: float32x2_t) -> float32x2_t; + } + vrnd_f32_(a) +} + +/// Floating-point round to integral, toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintz))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.trunc.v4f32")] + fn vrndq_f32_(a: float32x4_t) -> float32x4_t; + } + vrndq_f32_(a) +} + +/// Floating-point round to integral, toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintz))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrnd_f64(a: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.trunc.v1f64")] + fn vrnd_f64_(a: float64x1_t) -> float64x1_t; + } + vrnd_f64_(a) +} + +/// Floating-point round to integral, toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintz))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndq_f64(a: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.trunc.v2f64")] + fn vrndq_f64_(a: float64x2_t) -> float64x2_t; + } + vrndq_f64_(a) +} + +/// Floating-point round to integral, using current rounding mode +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndi_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frinti))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndi_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.nearbyint.v2f32")] + fn vrndi_f32_(a: float32x2_t) -> float32x2_t; + } + vrndi_f32_(a) +} + +/// Floating-point round to integral, using current rounding mode +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndiq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frinti))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndiq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.nearbyint.v4f32")] + fn vrndiq_f32_(a: float32x4_t) -> float32x4_t; + } + vrndiq_f32_(a) +} + +/// Floating-point round to integral, using current rounding mode +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndi_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frinti))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndi_f64(a: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.nearbyint.v1f64")] + fn vrndi_f64_(a: float64x1_t) -> float64x1_t; + } + vrndi_f64_(a) +} + +/// Floating-point round to integral, using current rounding mode +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndiq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frinti))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrndiq_f64(a: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.nearbyint.v2f64")] + fn vrndiq_f64_(a: float64x2_t) -> float64x2_t; + } + vrndiq_f64_(a) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddb_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqaddb_s8(a: i8, b: i8) -> i8 { + let a: int8x8_t = vdup_n_s8(a); + let b: int8x8_t = vdup_n_s8(b); + simd_extract(vqadd_s8(a, b), 0) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddh_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqaddh_s16(a: i16, b: i16) -> i16 { + let a: int16x4_t = vdup_n_s16(a); + let b: int16x4_t = vdup_n_s16(b); + simd_extract(vqadd_s16(a, b), 0) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddb_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqaddb_u8(a: u8, b: u8) -> u8 { + let a: uint8x8_t = vdup_n_u8(a); + let b: uint8x8_t = vdup_n_u8(b); + simd_extract(vqadd_u8(a, b), 0) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddh_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqaddh_u16(a: u16, b: u16) -> u16 { + let a: uint16x4_t = vdup_n_u16(a); + let b: uint16x4_t = vdup_n_u16(b); + simd_extract(vqadd_u16(a, b), 0) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqadds_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqadds_u32(a: u32, b: u32) -> u32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.i32")] + fn vqadds_u32_(a: u32, b: u32) -> u32; + } + vqadds_u32_(a, b) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddd_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqaddd_u64(a: u64, b: u64) -> u64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.i64")] + fn vqaddd_u64_(a: u64, b: u64) -> u64; + } + vqaddd_u64_(a, b) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqadds_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqadds_s32(a: i32, b: i32) -> i32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.i32")] + fn vqadds_s32_(a: i32, b: i32) -> i32; + } + vqadds_s32_(a, b) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddd_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqaddd_s64(a: i64, b: i64) -> i64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.i64")] + fn vqaddd_s64_(a: i64, b: i64) -> i64; + } + vqaddd_s64_(a, b) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_f64_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1_f64_x2(a: *const f64) -> float64x1x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x2.v1f64.p0f64")] + fn vld1_f64_x2_(a: *const f64) -> float64x1x2_t; + } + vld1_f64_x2_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_f64_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1q_f64_x2(a: *const f64) -> float64x2x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x2.v2f64.p0f64")] + fn vld1q_f64_x2_(a: *const f64) -> float64x2x2_t; + } + vld1q_f64_x2_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_f64_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1_f64_x3(a: *const f64) -> float64x1x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x3.v1f64.p0f64")] + fn vld1_f64_x3_(a: *const f64) -> float64x1x3_t; + } + vld1_f64_x3_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_f64_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1q_f64_x3(a: *const f64) -> float64x2x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x3.v2f64.p0f64")] + fn vld1q_f64_x3_(a: *const f64) -> float64x2x3_t; + } + vld1q_f64_x3_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_f64_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1_f64_x4(a: *const f64) -> float64x1x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x4.v1f64.p0f64")] + fn vld1_f64_x4_(a: *const f64) -> float64x1x4_t; + } + vld1_f64_x4_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_f64_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1q_f64_x4(a: *const f64) -> float64x2x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x4.v2f64.p0f64")] + fn vld1q_f64_x4_(a: *const f64) -> float64x2x4_t; + } + vld1q_f64_x4_(a) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_s64(a: *const i64) -> int64x2x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2.v2i64.p0v2i64")] + fn vld2q_s64_(ptr: *const int64x2_t) -> int64x2x2_t; + } + vld2q_s64_(a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_u64(a: *const u64) -> uint64x2x2_t { + transmute(vld2q_s64(transmute(a))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(ld2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_p64(a: *const p64) -> poly64x2x2_t { + transmute(vld2q_s64(transmute(a))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2_f64(a: *const f64) -> float64x1x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2.v1f64.p0v1f64")] + fn vld2_f64_(ptr: *const float64x1_t) -> float64x1x2_t; + } + vld2_f64_(a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_f64(a: *const f64) -> float64x2x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2.v2f64.p0v2f64")] + fn vld2q_f64_(ptr: *const float64x2_t) -> float64x2x2_t; + } + vld2q_f64_(a as _) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_dup_s64(a: *const i64) -> int64x2x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2r.v2i64.p0i64")] + fn vld2q_dup_s64_(ptr: *const i64) -> int64x2x2_t; + } + vld2q_dup_s64_(a as _) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_dup_u64(a: *const u64) -> uint64x2x2_t { + transmute(vld2q_dup_s64(transmute(a))) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(ld2r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_dup_p64(a: *const p64) -> poly64x2x2_t { + transmute(vld2q_dup_s64(transmute(a))) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2_dup_f64(a: *const f64) -> float64x1x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2r.v1f64.p0f64")] + fn vld2_dup_f64_(ptr: *const f64) -> float64x1x2_t; + } + vld2_dup_f64_(a as _) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_dup_f64(a: *const f64) -> float64x2x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2r.v2f64.p0f64")] + fn vld2q_dup_f64_(ptr: *const f64) -> float64x2x2_t; + } + vld2q_dup_f64_(a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_lane_s8(a: *const i8, b: int8x16x2_t) -> int8x16x2_t { + static_assert_imm4!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2lane.v16i8.p0i8")] + fn vld2q_lane_s8_(a: int8x16_t, b: int8x16_t, n: i64, ptr: *const i8) -> int8x16x2_t; + } + vld2q_lane_s8_(b.0, b.1, LANE as i64, a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2_lane_s64(a: *const i64, b: int64x1x2_t) -> int64x1x2_t { + static_assert!(LANE : i32 where LANE == 0); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2lane.v1i64.p0i8")] + fn vld2_lane_s64_(a: int64x1_t, b: int64x1_t, n: i64, ptr: *const i8) -> int64x1x2_t; + } + vld2_lane_s64_(b.0, b.1, LANE as i64, a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_lane_s64(a: *const i64, b: int64x2x2_t) -> int64x2x2_t { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2lane.v2i64.p0i8")] + fn vld2q_lane_s64_(a: int64x2_t, b: int64x2_t, n: i64, ptr: *const i8) -> int64x2x2_t; + } + vld2q_lane_s64_(b.0, b.1, LANE as i64, a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2_lane_p64(a: *const p64, b: poly64x1x2_t) -> poly64x1x2_t { + static_assert!(LANE : i32 where LANE == 0); + transmute(vld2_lane_s64::(transmute(a), transmute(b))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_lane_p64(a: *const p64, b: poly64x2x2_t) -> poly64x2x2_t { + static_assert_imm1!(LANE); + transmute(vld2q_lane_s64::(transmute(a), transmute(b))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_lane_u8(a: *const u8, b: uint8x16x2_t) -> uint8x16x2_t { + static_assert_imm4!(LANE); + transmute(vld2q_lane_s8::(transmute(a), transmute(b))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2_lane_u64(a: *const u64, b: uint64x1x2_t) -> uint64x1x2_t { + static_assert!(LANE : i32 where LANE == 0); + transmute(vld2_lane_s64::(transmute(a), transmute(b))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_lane_u64(a: *const u64, b: uint64x2x2_t) -> uint64x2x2_t { + static_assert_imm1!(LANE); + transmute(vld2q_lane_s64::(transmute(a), transmute(b))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_lane_p8(a: *const p8, b: poly8x16x2_t) -> poly8x16x2_t { + static_assert_imm4!(LANE); + transmute(vld2q_lane_s8::(transmute(a), transmute(b))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2_lane_f64(a: *const f64, b: float64x1x2_t) -> float64x1x2_t { + static_assert!(LANE : i32 where LANE == 0); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2lane.v1f64.p0i8")] + fn vld2_lane_f64_(a: float64x1_t, b: float64x1_t, n: i64, ptr: *const i8) -> float64x1x2_t; + } + vld2_lane_f64_(b.0, b.1, LANE as i64, a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_lane_f64(a: *const f64, b: float64x2x2_t) -> float64x2x2_t { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2lane.v2f64.p0i8")] + fn vld2q_lane_f64_(a: float64x2_t, b: float64x2_t, n: i64, ptr: *const i8) -> float64x2x2_t; + } + vld2q_lane_f64_(b.0, b.1, LANE as i64, a as _) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_s64(a: *const i64) -> int64x2x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3.v2i64.p0v2i64")] + fn vld3q_s64_(ptr: *const int64x2_t) -> int64x2x3_t; + } + vld3q_s64_(a as _) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_u64(a: *const u64) -> uint64x2x3_t { + transmute(vld3q_s64(transmute(a))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(ld3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_p64(a: *const p64) -> poly64x2x3_t { + transmute(vld3q_s64(transmute(a))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3_f64(a: *const f64) -> float64x1x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3.v1f64.p0v1f64")] + fn vld3_f64_(ptr: *const float64x1_t) -> float64x1x3_t; + } + vld3_f64_(a as _) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_f64(a: *const f64) -> float64x2x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3.v2f64.p0v2f64")] + fn vld3q_f64_(ptr: *const float64x2_t) -> float64x2x3_t; + } + vld3q_f64_(a as _) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_dup_s64(a: *const i64) -> int64x2x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3r.v2i64.p0i64")] + fn vld3q_dup_s64_(ptr: *const i64) -> int64x2x3_t; + } + vld3q_dup_s64_(a as _) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_dup_u64(a: *const u64) -> uint64x2x3_t { + transmute(vld3q_dup_s64(transmute(a))) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(ld3r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_dup_p64(a: *const p64) -> poly64x2x3_t { + transmute(vld3q_dup_s64(transmute(a))) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3_dup_f64(a: *const f64) -> float64x1x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3r.v1f64.p0f64")] + fn vld3_dup_f64_(ptr: *const f64) -> float64x1x3_t; + } + vld3_dup_f64_(a as _) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_dup_f64(a: *const f64) -> float64x2x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3r.v2f64.p0f64")] + fn vld3q_dup_f64_(ptr: *const f64) -> float64x2x3_t; + } + vld3q_dup_f64_(a as _) +} + +/// Load multiple 3-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_lane_s8(a: *const i8, b: int8x16x3_t) -> int8x16x3_t { + static_assert_imm4!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3lane.v16i8.p0i8")] + fn vld3q_lane_s8_(a: int8x16_t, b: int8x16_t, c: int8x16_t, n: i64, ptr: *const i8) -> int8x16x3_t; + } + vld3q_lane_s8_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Load multiple 3-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3_lane_s64(a: *const i64, b: int64x1x3_t) -> int64x1x3_t { + static_assert!(LANE : i32 where LANE == 0); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3lane.v1i64.p0i8")] + fn vld3_lane_s64_(a: int64x1_t, b: int64x1_t, c: int64x1_t, n: i64, ptr: *const i8) -> int64x1x3_t; + } + vld3_lane_s64_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Load multiple 3-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_lane_s64(a: *const i64, b: int64x2x3_t) -> int64x2x3_t { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3lane.v2i64.p0i8")] + fn vld3q_lane_s64_(a: int64x2_t, b: int64x2_t, c: int64x2_t, n: i64, ptr: *const i8) -> int64x2x3_t; + } + vld3q_lane_s64_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3_lane_p64(a: *const p64, b: poly64x1x3_t) -> poly64x1x3_t { + static_assert!(LANE : i32 where LANE == 0); + transmute(vld3_lane_s64::(transmute(a), transmute(b))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_lane_p64(a: *const p64, b: poly64x2x3_t) -> poly64x2x3_t { + static_assert_imm1!(LANE); + transmute(vld3q_lane_s64::(transmute(a), transmute(b))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_lane_p8(a: *const p8, b: poly8x16x3_t) -> poly8x16x3_t { + static_assert_imm4!(LANE); + transmute(vld3q_lane_s8::(transmute(a), transmute(b))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_lane_u8(a: *const u8, b: uint8x16x3_t) -> uint8x16x3_t { + static_assert_imm4!(LANE); + transmute(vld3q_lane_s8::(transmute(a), transmute(b))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3_lane_u64(a: *const u64, b: uint64x1x3_t) -> uint64x1x3_t { + static_assert!(LANE : i32 where LANE == 0); + transmute(vld3_lane_s64::(transmute(a), transmute(b))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_lane_u64(a: *const u64, b: uint64x2x3_t) -> uint64x2x3_t { + static_assert_imm1!(LANE); + transmute(vld3q_lane_s64::(transmute(a), transmute(b))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3_lane_f64(a: *const f64, b: float64x1x3_t) -> float64x1x3_t { + static_assert!(LANE : i32 where LANE == 0); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3lane.v1f64.p0i8")] + fn vld3_lane_f64_(a: float64x1_t, b: float64x1_t, c: float64x1_t, n: i64, ptr: *const i8) -> float64x1x3_t; + } + vld3_lane_f64_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_lane_f64(a: *const f64, b: float64x2x3_t) -> float64x2x3_t { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3lane.v2f64.p0i8")] + fn vld3q_lane_f64_(a: float64x2_t, b: float64x2_t, c: float64x2_t, n: i64, ptr: *const i8) -> float64x2x3_t; + } + vld3q_lane_f64_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_s64(a: *const i64) -> int64x2x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4.v2i64.p0v2i64")] + fn vld4q_s64_(ptr: *const int64x2_t) -> int64x2x4_t; + } + vld4q_s64_(a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_u64(a: *const u64) -> uint64x2x4_t { + transmute(vld4q_s64(transmute(a))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(ld4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_p64(a: *const p64) -> poly64x2x4_t { + transmute(vld4q_s64(transmute(a))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4_f64(a: *const f64) -> float64x1x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4.v1f64.p0v1f64")] + fn vld4_f64_(ptr: *const float64x1_t) -> float64x1x4_t; + } + vld4_f64_(a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_f64(a: *const f64) -> float64x2x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4.v2f64.p0v2f64")] + fn vld4q_f64_(ptr: *const float64x2_t) -> float64x2x4_t; + } + vld4q_f64_(a as _) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_dup_s64(a: *const i64) -> int64x2x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4r.v2i64.p0i64")] + fn vld4q_dup_s64_(ptr: *const i64) -> int64x2x4_t; + } + vld4q_dup_s64_(a as _) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_dup_u64(a: *const u64) -> uint64x2x4_t { + transmute(vld4q_dup_s64(transmute(a))) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(ld4r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_dup_p64(a: *const p64) -> poly64x2x4_t { + transmute(vld4q_dup_s64(transmute(a))) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4_dup_f64(a: *const f64) -> float64x1x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4r.v1f64.p0f64")] + fn vld4_dup_f64_(ptr: *const f64) -> float64x1x4_t; + } + vld4_dup_f64_(a as _) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_dup_f64(a: *const f64) -> float64x2x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4r.v2f64.p0f64")] + fn vld4q_dup_f64_(ptr: *const f64) -> float64x2x4_t; + } + vld4q_dup_f64_(a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_lane_s8(a: *const i8, b: int8x16x4_t) -> int8x16x4_t { + static_assert_imm4!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4lane.v16i8.p0i8")] + fn vld4q_lane_s8_(a: int8x16_t, b: int8x16_t, c: int8x16_t, d: int8x16_t, n: i64, ptr: *const i8) -> int8x16x4_t; + } + vld4q_lane_s8_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4_lane_s64(a: *const i64, b: int64x1x4_t) -> int64x1x4_t { + static_assert!(LANE : i32 where LANE == 0); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4lane.v1i64.p0i8")] + fn vld4_lane_s64_(a: int64x1_t, b: int64x1_t, c: int64x1_t, d: int64x1_t, n: i64, ptr: *const i8) -> int64x1x4_t; + } + vld4_lane_s64_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_lane_s64(a: *const i64, b: int64x2x4_t) -> int64x2x4_t { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4lane.v2i64.p0i8")] + fn vld4q_lane_s64_(a: int64x2_t, b: int64x2_t, c: int64x2_t, d: int64x2_t, n: i64, ptr: *const i8) -> int64x2x4_t; + } + vld4q_lane_s64_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4_lane_p64(a: *const p64, b: poly64x1x4_t) -> poly64x1x4_t { + static_assert!(LANE : i32 where LANE == 0); + transmute(vld4_lane_s64::(transmute(a), transmute(b))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_lane_p64(a: *const p64, b: poly64x2x4_t) -> poly64x2x4_t { + static_assert_imm1!(LANE); + transmute(vld4q_lane_s64::(transmute(a), transmute(b))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_lane_p8(a: *const p8, b: poly8x16x4_t) -> poly8x16x4_t { + static_assert_imm4!(LANE); + transmute(vld4q_lane_s8::(transmute(a), transmute(b))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_lane_u8(a: *const u8, b: uint8x16x4_t) -> uint8x16x4_t { + static_assert_imm4!(LANE); + transmute(vld4q_lane_s8::(transmute(a), transmute(b))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4_lane_u64(a: *const u64, b: uint64x1x4_t) -> uint64x1x4_t { + static_assert!(LANE : i32 where LANE == 0); + transmute(vld4_lane_s64::(transmute(a), transmute(b))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_lane_u64(a: *const u64, b: uint64x2x4_t) -> uint64x2x4_t { + static_assert_imm1!(LANE); + transmute(vld4q_lane_s64::(transmute(a), transmute(b))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4_lane_f64(a: *const f64, b: float64x1x4_t) -> float64x1x4_t { + static_assert!(LANE : i32 where LANE == 0); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4lane.v1f64.p0i8")] + fn vld4_lane_f64_(a: float64x1_t, b: float64x1_t, c: float64x1_t, d: float64x1_t, n: i64, ptr: *const i8) -> float64x1x4_t; + } + vld4_lane_f64_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_lane_f64(a: *const f64, b: float64x2x4_t) -> float64x2x4_t { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4lane.v2f64.p0i8")] + fn vld4q_lane_f64_(a: float64x2_t, b: float64x2_t, c: float64x2_t, d: float64x2_t, n: i64, ptr: *const i8) -> float64x2x4_t; + } + vld4q_lane_f64_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_lane_f64(a: *mut f64, b: float64x1_t) { + static_assert!(LANE : i32 where LANE == 0); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_lane_f64(a: *mut f64, b: float64x2_t) { + static_assert_imm1!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f64_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_f64_x2(a: *mut f64, b: float64x1x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x2.v1f64.p0f64")] + fn vst1_f64_x2_(a: float64x1_t, b: float64x1_t, ptr: *mut f64); + } + vst1_f64_x2_(b.0, b.1, a) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f64_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_f64_x2(a: *mut f64, b: float64x2x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x2.v2f64.p0f64")] + fn vst1q_f64_x2_(a: float64x2_t, b: float64x2_t, ptr: *mut f64); + } + vst1q_f64_x2_(b.0, b.1, a) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f64_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_f64_x3(a: *mut f64, b: float64x1x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x3.v1f64.p0f64")] + fn vst1_f64_x3_(a: float64x1_t, b: float64x1_t, c: float64x1_t, ptr: *mut f64); + } + vst1_f64_x3_(b.0, b.1, b.2, a) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f64_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_f64_x3(a: *mut f64, b: float64x2x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x3.v2f64.p0f64")] + fn vst1q_f64_x3_(a: float64x2_t, b: float64x2_t, c: float64x2_t, ptr: *mut f64); + } + vst1q_f64_x3_(b.0, b.1, b.2, a) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f64_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_f64_x4(a: *mut f64, b: float64x1x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x4.v1f64.p0f64")] + fn vst1_f64_x4_(a: float64x1_t, b: float64x1_t, c: float64x1_t, d: float64x1_t, ptr: *mut f64); + } + vst1_f64_x4_(b.0, b.1, b.2, b.3, a) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f64_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_f64_x4(a: *mut f64, b: float64x2x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x4.v2f64.p0f64")] + fn vst1q_f64_x4_(a: float64x2_t, b: float64x2_t, c: float64x2_t, d: float64x2_t, ptr: *mut f64); + } + vst1q_f64_x4_(b.0, b.1, b.2, b.3, a) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2q_s64(a: *mut i64, b: int64x2x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2.v2i64.p0i8")] + fn vst2q_s64_(a: int64x2_t, b: int64x2_t, ptr: *mut i8); + } + vst2q_s64_(b.0, b.1, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2q_u64(a: *mut u64, b: uint64x2x2_t) { + transmute(vst2q_s64(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(st2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2q_p64(a: *mut p64, b: poly64x2x2_t) { + transmute(vst2q_s64(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2_f64(a: *mut f64, b: float64x1x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2.v1f64.p0i8")] + fn vst2_f64_(a: float64x1_t, b: float64x1_t, ptr: *mut i8); + } + vst2_f64_(b.0, b.1, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2q_f64(a: *mut f64, b: float64x2x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2.v2f64.p0i8")] + fn vst2q_f64_(a: float64x2_t, b: float64x2_t, ptr: *mut i8); + } + vst2q_f64_(b.0, b.1, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2q_lane_s8(a: *mut i8, b: int8x16x2_t) { + static_assert_imm4!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2lane.v16i8.p0i8")] + fn vst2q_lane_s8_(a: int8x16_t, b: int8x16_t, n: i64, ptr: *mut i8); + } + vst2q_lane_s8_(b.0, b.1, LANE as i64, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2_lane_s64(a: *mut i64, b: int64x1x2_t) { + static_assert!(LANE : i32 where LANE == 0); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2lane.v1i64.p0i8")] + fn vst2_lane_s64_(a: int64x1_t, b: int64x1_t, n: i64, ptr: *mut i8); + } + vst2_lane_s64_(b.0, b.1, LANE as i64, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2q_lane_s64(a: *mut i64, b: int64x2x2_t) { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2lane.v2i64.p0i8")] + fn vst2q_lane_s64_(a: int64x2_t, b: int64x2_t, n: i64, ptr: *mut i8); + } + vst2q_lane_s64_(b.0, b.1, LANE as i64, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2q_lane_u8(a: *mut u8, b: uint8x16x2_t) { + static_assert_imm4!(LANE); + transmute(vst2q_lane_s8::(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2_lane_u64(a: *mut u64, b: uint64x1x2_t) { + static_assert!(LANE : i32 where LANE == 0); + transmute(vst2_lane_s64::(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2q_lane_u64(a: *mut u64, b: uint64x2x2_t) { + static_assert_imm1!(LANE); + transmute(vst2q_lane_s64::(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2q_lane_p8(a: *mut p8, b: poly8x16x2_t) { + static_assert_imm4!(LANE); + transmute(vst2q_lane_s8::(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2_lane_p64(a: *mut p64, b: poly64x1x2_t) { + static_assert!(LANE : i32 where LANE == 0); + transmute(vst2_lane_s64::(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2q_lane_p64(a: *mut p64, b: poly64x2x2_t) { + static_assert_imm1!(LANE); + transmute(vst2q_lane_s64::(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2_lane_f64(a: *mut f64, b: float64x1x2_t) { + static_assert!(LANE : i32 where LANE == 0); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2lane.v1f64.p0i8")] + fn vst2_lane_f64_(a: float64x1_t, b: float64x1_t, n: i64, ptr: *mut i8); + } + vst2_lane_f64_(b.0, b.1, LANE as i64, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2q_lane_f64(a: *mut f64, b: float64x2x2_t) { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2lane.v2f64.p0i8")] + fn vst2q_lane_f64_(a: float64x2_t, b: float64x2_t, n: i64, ptr: *mut i8); + } + vst2q_lane_f64_(b.0, b.1, LANE as i64, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3q_s64(a: *mut i64, b: int64x2x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3.v2i64.p0i8")] + fn vst3q_s64_(a: int64x2_t, b: int64x2_t, c: int64x2_t, ptr: *mut i8); + } + vst3q_s64_(b.0, b.1, b.2, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3q_u64(a: *mut u64, b: uint64x2x3_t) { + transmute(vst3q_s64(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(st3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3q_p64(a: *mut p64, b: poly64x2x3_t) { + transmute(vst3q_s64(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3_f64(a: *mut f64, b: float64x1x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3.v1f64.p0i8")] + fn vst3_f64_(a: float64x1_t, b: float64x1_t, c: float64x1_t, ptr: *mut i8); + } + vst3_f64_(b.0, b.1, b.2, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3q_f64(a: *mut f64, b: float64x2x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3.v2f64.p0i8")] + fn vst3q_f64_(a: float64x2_t, b: float64x2_t, c: float64x2_t, ptr: *mut i8); + } + vst3q_f64_(b.0, b.1, b.2, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3q_lane_s8(a: *mut i8, b: int8x16x3_t) { + static_assert_imm4!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3lane.v16i8.p0i8")] + fn vst3q_lane_s8_(a: int8x16_t, b: int8x16_t, c: int8x16_t, n: i64, ptr: *mut i8); + } + vst3q_lane_s8_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3_lane_s64(a: *mut i64, b: int64x1x3_t) { + static_assert!(LANE : i32 where LANE == 0); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3lane.v1i64.p0i8")] + fn vst3_lane_s64_(a: int64x1_t, b: int64x1_t, c: int64x1_t, n: i64, ptr: *mut i8); + } + vst3_lane_s64_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3q_lane_s64(a: *mut i64, b: int64x2x3_t) { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3lane.v2i64.p0i8")] + fn vst3q_lane_s64_(a: int64x2_t, b: int64x2_t, c: int64x2_t, n: i64, ptr: *mut i8); + } + vst3q_lane_s64_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3q_lane_u8(a: *mut u8, b: uint8x16x3_t) { + static_assert_imm4!(LANE); + transmute(vst3q_lane_s8::(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3_lane_u64(a: *mut u64, b: uint64x1x3_t) { + static_assert!(LANE : i32 where LANE == 0); + transmute(vst3_lane_s64::(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3q_lane_u64(a: *mut u64, b: uint64x2x3_t) { + static_assert_imm1!(LANE); + transmute(vst3q_lane_s64::(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3q_lane_p8(a: *mut p8, b: poly8x16x3_t) { + static_assert_imm4!(LANE); + transmute(vst3q_lane_s8::(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3_lane_p64(a: *mut p64, b: poly64x1x3_t) { + static_assert!(LANE : i32 where LANE == 0); + transmute(vst3_lane_s64::(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3q_lane_p64(a: *mut p64, b: poly64x2x3_t) { + static_assert_imm1!(LANE); + transmute(vst3q_lane_s64::(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3_lane_f64(a: *mut f64, b: float64x1x3_t) { + static_assert!(LANE : i32 where LANE == 0); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3lane.v1f64.p0i8")] + fn vst3_lane_f64_(a: float64x1_t, b: float64x1_t, c: float64x1_t, n: i64, ptr: *mut i8); + } + vst3_lane_f64_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3q_lane_f64(a: *mut f64, b: float64x2x3_t) { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3lane.v2f64.p0i8")] + fn vst3q_lane_f64_(a: float64x2_t, b: float64x2_t, c: float64x2_t, n: i64, ptr: *mut i8); + } + vst3q_lane_f64_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4q_s64(a: *mut i64, b: int64x2x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4.v2i64.p0i8")] + fn vst4q_s64_(a: int64x2_t, b: int64x2_t, c: int64x2_t, d: int64x2_t, ptr: *mut i8); + } + vst4q_s64_(b.0, b.1, b.2, b.3, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4q_u64(a: *mut u64, b: uint64x2x4_t) { + transmute(vst4q_s64(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(st4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4q_p64(a: *mut p64, b: poly64x2x4_t) { + transmute(vst4q_s64(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4_f64(a: *mut f64, b: float64x1x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4.v1f64.p0i8")] + fn vst4_f64_(a: float64x1_t, b: float64x1_t, c: float64x1_t, d: float64x1_t, ptr: *mut i8); + } + vst4_f64_(b.0, b.1, b.2, b.3, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4q_f64(a: *mut f64, b: float64x2x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4.v2f64.p0i8")] + fn vst4q_f64_(a: float64x2_t, b: float64x2_t, c: float64x2_t, d: float64x2_t, ptr: *mut i8); + } + vst4q_f64_(b.0, b.1, b.2, b.3, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4q_lane_s8(a: *mut i8, b: int8x16x4_t) { + static_assert_imm4!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4lane.v16i8.p0i8")] + fn vst4q_lane_s8_(a: int8x16_t, b: int8x16_t, c: int8x16_t, d: int8x16_t, n: i64, ptr: *mut i8); + } + vst4q_lane_s8_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4_lane_s64(a: *mut i64, b: int64x1x4_t) { + static_assert!(LANE : i32 where LANE == 0); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4lane.v1i64.p0i8")] + fn vst4_lane_s64_(a: int64x1_t, b: int64x1_t, c: int64x1_t, d: int64x1_t, n: i64, ptr: *mut i8); + } + vst4_lane_s64_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4q_lane_s64(a: *mut i64, b: int64x2x4_t) { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4lane.v2i64.p0i8")] + fn vst4q_lane_s64_(a: int64x2_t, b: int64x2_t, c: int64x2_t, d: int64x2_t, n: i64, ptr: *mut i8); + } + vst4q_lane_s64_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4q_lane_u8(a: *mut u8, b: uint8x16x4_t) { + static_assert_imm4!(LANE); + transmute(vst4q_lane_s8::(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4_lane_u64(a: *mut u64, b: uint64x1x4_t) { + static_assert!(LANE : i32 where LANE == 0); + transmute(vst4_lane_s64::(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4q_lane_u64(a: *mut u64, b: uint64x2x4_t) { + static_assert_imm1!(LANE); + transmute(vst4q_lane_s64::(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4q_lane_p8(a: *mut p8, b: poly8x16x4_t) { + static_assert_imm4!(LANE); + transmute(vst4q_lane_s8::(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4_lane_p64(a: *mut p64, b: poly64x1x4_t) { + static_assert!(LANE : i32 where LANE == 0); + transmute(vst4_lane_s64::(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4q_lane_p64(a: *mut p64, b: poly64x2x4_t) { + static_assert_imm1!(LANE); + transmute(vst4q_lane_s64::(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4_lane_f64(a: *mut f64, b: float64x1x4_t) { + static_assert!(LANE : i32 where LANE == 0); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4lane.v1f64.p0i8")] + fn vst4_lane_f64_(a: float64x1_t, b: float64x1_t, c: float64x1_t, d: float64x1_t, n: i64, ptr: *mut i8); + } + vst4_lane_f64_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4q_lane_f64(a: *mut f64, b: float64x2x4_t) { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4lane.v2f64.p0i8")] + fn vst4q_lane_f64_(a: float64x2_t, b: float64x2_t, c: float64x2_t, d: float64x2_t, n: i64, ptr: *mut i8); + } + vst4q_lane_f64_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmul_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + simd_mul(a, b) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + simd_mul(a, b) +} + +/// Vector multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_n_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmul_n_f64(a: float64x1_t, b: f64) -> float64x1_t { + simd_mul(a, vdup_n_f64(b)) +} + +/// Vector multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_n_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulq_n_f64(a: float64x2_t, b: f64) -> float64x2_t { + simd_mul(a, vdupq_n_f64(b)) +} + +/// Floating-point multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmul_lane_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + static_assert!(LANE : i32 where LANE == 0); + simd_mul(a, transmute::(simd_extract(b, LANE as u32))) +} + +/// Floating-point multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_laneq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmul_laneq_f64(a: float64x1_t, b: float64x2_t) -> float64x1_t { + static_assert_imm1!(LANE); + simd_mul(a, transmute::(simd_extract(b, LANE as u32))) +} + +/// Floating-point multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulq_lane_f64(a: float64x2_t, b: float64x1_t) -> float64x2_t { + static_assert!(LANE : i32 where LANE == 0); + simd_mul(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Floating-point multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_laneq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulq_laneq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + static_assert_imm1!(LANE); + simd_mul(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Floating-point multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmuls_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmuls_lane_f32(a: f32, b: float32x2_t) -> f32 { + static_assert_imm1!(LANE); + let b: f32 = simd_extract(b, LANE as u32); + a * b +} + +/// Floating-point multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmuls_laneq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmuls_laneq_f32(a: f32, b: float32x4_t) -> f32 { + static_assert_imm2!(LANE); + let b: f32 = simd_extract(b, LANE as u32); + a * b +} + +/// Floating-point multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmuld_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmuld_lane_f64(a: f64, b: float64x1_t) -> f64 { + static_assert!(LANE : i32 where LANE == 0); + let b: f64 = simd_extract(b, LANE as u32); + a * b +} + +/// Floating-point multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmuld_laneq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmuld_laneq_f64(a: f64, b: float64x2_t) -> f64 { + static_assert_imm1!(LANE); + let b: f64 = simd_extract(b, LANE as u32); + a * b +} + +/// Signed multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smull2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmull_high_s8(a: int8x16_t, b: int8x16_t) -> int16x8_t { + let a: int8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + let b: int8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + vmull_s8(a, b) +} + +/// Signed multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smull2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmull_high_s16(a: int16x8_t, b: int16x8_t) -> int32x4_t { + let a: int16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let b: int16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + vmull_s16(a, b) +} + +/// Signed multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smull2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmull_high_s32(a: int32x4_t, b: int32x4_t) -> int64x2_t { + let a: int32x2_t = simd_shuffle2!(a, a, [2, 3]); + let b: int32x2_t = simd_shuffle2!(b, b, [2, 3]); + vmull_s32(a, b) +} + +/// Unsigned multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umull2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmull_high_u8(a: uint8x16_t, b: uint8x16_t) -> uint16x8_t { + let a: uint8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + let b: uint8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + vmull_u8(a, b) +} + +/// Unsigned multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umull2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmull_high_u16(a: uint16x8_t, b: uint16x8_t) -> uint32x4_t { + let a: uint16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let b: uint16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + vmull_u16(a, b) +} + +/// Unsigned multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umull2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmull_high_u32(a: uint32x4_t, b: uint32x4_t) -> uint64x2_t { + let a: uint32x2_t = simd_shuffle2!(a, a, [2, 3]); + let b: uint32x2_t = simd_shuffle2!(b, b, [2, 3]); + vmull_u32(a, b) +} + +/// Polynomial multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(pmull))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmull_p64(a: p64, b: p64) -> p128 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.pmull64")] + fn vmull_p64_(a: p64, b: p64) -> int8x16_t; + } + transmute(vmull_p64_(a, b)) +} + +/// Polynomial multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(pmull))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmull_high_p8(a: poly8x16_t, b: poly8x16_t) -> poly16x8_t { + let a: poly8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + let b: poly8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + vmull_p8(a, b) +} + +/// Polynomial multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(pmull))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmull_high_p64(a: poly64x2_t, b: poly64x2_t) -> p128 { + vmull_p64(simd_extract(a, 1), simd_extract(b, 1)) +} + +/// Multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smull2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmull_high_n_s16(a: int16x8_t, b: i16) -> int32x4_t { + vmull_high_s16(a, vdupq_n_s16(b)) +} + +/// Multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smull2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmull_high_n_s32(a: int32x4_t, b: i32) -> int64x2_t { + vmull_high_s32(a, vdupq_n_s32(b)) +} + +/// Multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umull2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmull_high_n_u16(a: uint16x8_t, b: u16) -> uint32x4_t { + vmull_high_u16(a, vdupq_n_u16(b)) +} + +/// Multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umull2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmull_high_n_u32(a: uint32x4_t, b: u32) -> uint64x2_t { + vmull_high_u32(a, vdupq_n_u32(b)) +} + +/// Multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smull2, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmull_high_lane_s16(a: int16x8_t, b: int16x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + vmull_high_s16(a, simd_shuffle8!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smull2, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmull_high_laneq_s16(a: int16x8_t, b: int16x8_t) -> int32x4_t { + static_assert_imm3!(LANE); + vmull_high_s16(a, simd_shuffle8!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smull2, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmull_high_lane_s32(a: int32x4_t, b: int32x2_t) -> int64x2_t { + static_assert_imm1!(LANE); + vmull_high_s32(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smull2, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmull_high_laneq_s32(a: int32x4_t, b: int32x4_t) -> int64x2_t { + static_assert_imm2!(LANE); + vmull_high_s32(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umull2, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmull_high_lane_u16(a: uint16x8_t, b: uint16x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + vmull_high_u16(a, simd_shuffle8!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_laneq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umull2, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmull_high_laneq_u16(a: uint16x8_t, b: uint16x8_t) -> uint32x4_t { + static_assert_imm3!(LANE); + vmull_high_u16(a, simd_shuffle8!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umull2, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmull_high_lane_u32(a: uint32x4_t, b: uint32x2_t) -> uint64x2_t { + static_assert_imm1!(LANE); + vmull_high_u32(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_laneq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umull2, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmull_high_laneq_u32(a: uint32x4_t, b: uint32x4_t) -> uint64x2_t { + static_assert_imm2!(LANE); + vmull_high_u32(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Floating-point multiply extended +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulx_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulx_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmulx.v2f32")] + fn vmulx_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; + } + vmulx_f32_(a, b) +} + +/// Floating-point multiply extended +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulxq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmulx.v4f32")] + fn vmulxq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + } + vmulxq_f32_(a, b) +} + +/// Floating-point multiply extended +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulx_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulx_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmulx.v1f64")] + fn vmulx_f64_(a: float64x1_t, b: float64x1_t) -> float64x1_t; + } + vmulx_f64_(a, b) +} + +/// Floating-point multiply extended +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulxq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmulx.v2f64")] + fn vmulxq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + } + vmulxq_f64_(a, b) +} + +/// Floating-point multiply extended +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulx_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulx_lane_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + static_assert!(LANE : i32 where LANE == 0); + vmulx_f64(a, transmute::(simd_extract(b, LANE as u32))) +} + +/// Floating-point multiply extended +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulx_laneq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulx_laneq_f64(a: float64x1_t, b: float64x2_t) -> float64x1_t { + static_assert_imm1!(LANE); + vmulx_f64(a, transmute::(simd_extract(b, LANE as u32))) +} + +/// Floating-point multiply extended +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulx_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulx_lane_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + static_assert_imm1!(LANE); + vmulx_f32(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Floating-point multiply extended +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulx_laneq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulx_laneq_f32(a: float32x2_t, b: float32x4_t) -> float32x2_t { + static_assert_imm2!(LANE); + vmulx_f32(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Floating-point multiply extended +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxq_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulxq_lane_f32(a: float32x4_t, b: float32x2_t) -> float32x4_t { + static_assert_imm1!(LANE); + vmulxq_f32(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Floating-point multiply extended +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxq_laneq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulxq_laneq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + static_assert_imm2!(LANE); + vmulxq_f32(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Floating-point multiply extended +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxq_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulxq_lane_f64(a: float64x2_t, b: float64x1_t) -> float64x2_t { + static_assert!(LANE : i32 where LANE == 0); + vmulxq_f64(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Floating-point multiply extended +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxq_laneq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulxq_laneq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + static_assert_imm1!(LANE); + vmulxq_f64(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Floating-point multiply extended +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxs_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulxs_f32(a: f32, b: f32) -> f32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmulx.f32")] + fn vmulxs_f32_(a: f32, b: f32) -> f32; + } + vmulxs_f32_(a, b) +} + +/// Floating-point multiply extended +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulxd_f64(a: f64, b: f64) -> f64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmulx.f64")] + fn vmulxd_f64_(a: f64, b: f64) -> f64; + } + vmulxd_f64_(a, b) +} + +/// Floating-point multiply extended +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxs_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulxs_lane_f32(a: f32, b: float32x2_t) -> f32 { + static_assert_imm1!(LANE); + vmulxs_f32(a, simd_extract(b, LANE as u32)) +} + +/// Floating-point multiply extended +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxs_laneq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulxs_laneq_f32(a: f32, b: float32x4_t) -> f32 { + static_assert_imm2!(LANE); + vmulxs_f32(a, simd_extract(b, LANE as u32)) +} + +/// Floating-point multiply extended +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxd_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulxd_lane_f64(a: f64, b: float64x1_t) -> f64 { + static_assert!(LANE : i32 where LANE == 0); + vmulxd_f64(a, simd_extract(b, LANE as u32)) +} + +/// Floating-point multiply extended +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxd_laneq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmulxd_laneq_f64(a: f64, b: float64x2_t) -> f64 { + static_assert_imm1!(LANE); + vmulxd_f64(a, simd_extract(b, LANE as u32)) +} + +/// Floating-point fused Multiply-Add to accumulator(vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfma_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfma_f64(a: float64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fma.v1f64")] + fn vfma_f64_(a: float64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t; + } + vfma_f64_(b, c, a) +} + +/// Floating-point fused Multiply-Add to accumulator(vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmaq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfmaq_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fma.v2f64")] + fn vfmaq_f64_(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t; + } + vfmaq_f64_(b, c, a) +} + +/// Floating-point fused Multiply-Add to accumulator(vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfma_n_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfma_n_f64(a: float64x1_t, b: float64x1_t, c: f64) -> float64x1_t { + vfma_f64(a, b, vdup_n_f64(c)) +} + +/// Floating-point fused Multiply-Add to accumulator(vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmaq_n_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfmaq_n_f64(a: float64x2_t, b: float64x2_t, c: f64) -> float64x2_t { + vfmaq_f64(a, b, vdupq_n_f64(c)) +} + +/// Floating-point fused multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfma_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfma_lane_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + static_assert_imm1!(LANE); + vfma_f32(a, b, vdup_n_f32(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfma_laneq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfma_laneq_f32(a: float32x2_t, b: float32x2_t, c: float32x4_t) -> float32x2_t { + static_assert_imm2!(LANE); + vfma_f32(a, b, vdup_n_f32(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmaq_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfmaq_lane_f32(a: float32x4_t, b: float32x4_t, c: float32x2_t) -> float32x4_t { + static_assert_imm1!(LANE); + vfmaq_f32(a, b, vdupq_n_f32(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmaq_laneq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfmaq_laneq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + static_assert_imm2!(LANE); + vfmaq_f32(a, b, vdupq_n_f32(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfma_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmadd, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfma_lane_f64(a: float64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t { + static_assert!(LANE : i32 where LANE == 0); + vfma_f64(a, b, vdup_n_f64(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfma_laneq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfma_laneq_f64(a: float64x1_t, b: float64x1_t, c: float64x2_t) -> float64x1_t { + static_assert_imm1!(LANE); + vfma_f64(a, b, vdup_n_f64(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmaq_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfmaq_lane_f64(a: float64x2_t, b: float64x2_t, c: float64x1_t) -> float64x2_t { + static_assert!(LANE : i32 where LANE == 0); + vfmaq_f64(a, b, vdupq_n_f64(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmaq_laneq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfmaq_laneq_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t { + static_assert_imm1!(LANE); + vfmaq_f64(a, b, vdupq_n_f64(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmas_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfmas_lane_f32(a: f32, b: f32, c: float32x2_t) -> f32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fma.f32")] + fn vfmas_lane_f32_(a: f32, b: f32, c: f32) -> f32; + } + static_assert_imm1!(LANE); + let c: f32 = simd_extract(c, LANE as u32); + vfmas_lane_f32_(b, c, a) +} + +/// Floating-point fused multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmas_laneq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfmas_laneq_f32(a: f32, b: f32, c: float32x4_t) -> f32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fma.f32")] + fn vfmas_laneq_f32_(a: f32, b: f32, c: f32) -> f32; + } + static_assert_imm2!(LANE); + let c: f32 = simd_extract(c, LANE as u32); + vfmas_laneq_f32_(b, c, a) +} + +/// Floating-point fused multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmad_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmadd, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfmad_lane_f64(a: f64, b: f64, c: float64x1_t) -> f64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fma.f64")] + fn vfmad_lane_f64_(a: f64, b: f64, c: f64) -> f64; + } + static_assert!(LANE : i32 where LANE == 0); + let c: f64 = simd_extract(c, LANE as u32); + vfmad_lane_f64_(b, c, a) +} + +/// Floating-point fused multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmad_laneq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfmad_laneq_f64(a: f64, b: f64, c: float64x2_t) -> f64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fma.f64")] + fn vfmad_laneq_f64_(a: f64, b: f64, c: f64) -> f64; + } + static_assert_imm1!(LANE); + let c: f64 = simd_extract(c, LANE as u32); + vfmad_laneq_f64_(b, c, a) +} + +/// Floating-point fused multiply-subtract from accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfms_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmsub))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfms_f64(a: float64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t { + let b: float64x1_t = simd_neg(b); + vfma_f64(a, b, c) +} + +/// Floating-point fused multiply-subtract from accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfmsq_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t { + let b: float64x2_t = simd_neg(b); + vfmaq_f64(a, b, c) +} + +/// Floating-point fused Multiply-subtract to accumulator(vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfms_n_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmsub))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfms_n_f64(a: float64x1_t, b: float64x1_t, c: f64) -> float64x1_t { + vfms_f64(a, b, vdup_n_f64(c)) +} + +/// Floating-point fused Multiply-subtract to accumulator(vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsq_n_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfmsq_n_f64(a: float64x2_t, b: float64x2_t, c: f64) -> float64x2_t { + vfmsq_f64(a, b, vdupq_n_f64(c)) +} + +/// Floating-point fused multiply-subtract to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfms_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfms_lane_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + static_assert_imm1!(LANE); + vfms_f32(a, b, vdup_n_f32(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-subtract to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfms_laneq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfms_laneq_f32(a: float32x2_t, b: float32x2_t, c: float32x4_t) -> float32x2_t { + static_assert_imm2!(LANE); + vfms_f32(a, b, vdup_n_f32(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-subtract to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsq_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfmsq_lane_f32(a: float32x4_t, b: float32x4_t, c: float32x2_t) -> float32x4_t { + static_assert_imm1!(LANE); + vfmsq_f32(a, b, vdupq_n_f32(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-subtract to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsq_laneq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfmsq_laneq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + static_assert_imm2!(LANE); + vfmsq_f32(a, b, vdupq_n_f32(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-subtract to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfms_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmsub, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfms_lane_f64(a: float64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t { + static_assert!(LANE : i32 where LANE == 0); + vfms_f64(a, b, vdup_n_f64(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-subtract to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfms_laneq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfms_laneq_f64(a: float64x1_t, b: float64x1_t, c: float64x2_t) -> float64x1_t { + static_assert_imm1!(LANE); + vfms_f64(a, b, vdup_n_f64(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-subtract to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsq_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfmsq_lane_f64(a: float64x2_t, b: float64x2_t, c: float64x1_t) -> float64x2_t { + static_assert!(LANE : i32 where LANE == 0); + vfmsq_f64(a, b, vdupq_n_f64(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-subtract to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsq_laneq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfmsq_laneq_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t { + static_assert_imm1!(LANE); + vfmsq_f64(a, b, vdupq_n_f64(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-subtract to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmss_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfmss_lane_f32(a: f32, b: f32, c: float32x2_t) -> f32 { + vfmas_lane_f32::(a, -b, c) +} + +/// Floating-point fused multiply-subtract to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmss_laneq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfmss_laneq_f32(a: f32, b: f32, c: float32x4_t) -> f32 { + vfmas_laneq_f32::(a, -b, c) +} + +/// Floating-point fused multiply-subtract to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsd_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmsub, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfmsd_lane_f64(a: f64, b: f64, c: float64x1_t) -> f64 { + vfmad_lane_f64::(a, -b, c) +} + +/// Floating-point fused multiply-subtract to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsd_laneq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vfmsd_laneq_f64(a: f64, b: f64, c: float64x2_t) -> f64 { + vfmad_laneq_f64::(a, -b, c) +} + +/// Divide +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdiv_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fdiv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdiv_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + simd_div(a, b) +} + +/// Divide +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdivq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fdiv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdivq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + simd_div(a, b) +} + +/// Divide +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdiv_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fdiv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdiv_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + simd_div(a, b) +} + +/// Divide +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdivq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fdiv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdivq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + simd_div(a, b) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsub_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fsub))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsub_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + simd_sub(a, b) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fsub))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsubq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + simd_sub(a, b) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubd_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsubd_s64(a: i64, b: i64) -> i64 { + a.wrapping_sub(b) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubd_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsubd_u64(a: u64, b: u64) -> u64 { + a.wrapping_sub(b) +} + +/// Add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddd_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddd_s64(a: i64, b: i64) -> i64 { + a.wrapping_add(b) +} + +/// Add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddd_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddd_u64(a: u64, b: u64) -> u64 { + a.wrapping_add(b) +} + +/// Floating-point add across vector +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddv_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(faddp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddv_f32(a: float32x2_t) -> f32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.faddv.f32.v2f32")] + fn vaddv_f32_(a: float32x2_t) -> f32; + } + vaddv_f32_(a) +} + +/// Floating-point add across vector +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddvq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(faddp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddvq_f32(a: float32x4_t) -> f32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.faddv.f32.v4f32")] + fn vaddvq_f32_(a: float32x4_t) -> f32; + } + vaddvq_f32_(a) +} + +/// Floating-point add across vector +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddvq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(faddp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddvq_f64(a: float64x2_t) -> f64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.faddv.f64.v2f64")] + fn vaddvq_f64_(a: float64x2_t) -> f64; + } + vaddvq_f64_(a) +} + +/// Signed Add Long across Vector +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddlv_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(saddlv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddlv_s16(a: int16x4_t) -> i32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.saddlv.i32.v4i16")] + fn vaddlv_s16_(a: int16x4_t) -> i32; + } + vaddlv_s16_(a) +} + +/// Signed Add Long across Vector +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddlvq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(saddlv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddlvq_s16(a: int16x8_t) -> i32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.saddlv.i32.v8i16")] + fn vaddlvq_s16_(a: int16x8_t) -> i32; + } + vaddlvq_s16_(a) +} + +/// Signed Add Long across Vector +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddlv_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(saddlp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddlv_s32(a: int32x2_t) -> i64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.saddlv.i64.v2i32")] + fn vaddlv_s32_(a: int32x2_t) -> i64; + } + vaddlv_s32_(a) +} + +/// Signed Add Long across Vector +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddlvq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(saddlv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddlvq_s32(a: int32x4_t) -> i64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.saddlv.i64.v4i32")] + fn vaddlvq_s32_(a: int32x4_t) -> i64; + } + vaddlvq_s32_(a) +} + +/// Unsigned Add Long across Vector +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddlv_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uaddlv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddlv_u16(a: uint16x4_t) -> u32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uaddlv.i32.v4i16")] + fn vaddlv_u16_(a: uint16x4_t) -> u32; + } + vaddlv_u16_(a) +} + +/// Unsigned Add Long across Vector +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddlvq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uaddlv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddlvq_u16(a: uint16x8_t) -> u32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uaddlv.i32.v8i16")] + fn vaddlvq_u16_(a: uint16x8_t) -> u32; + } + vaddlvq_u16_(a) +} + +/// Unsigned Add Long across Vector +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddlv_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uaddlp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddlv_u32(a: uint32x2_t) -> u64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uaddlv.i64.v2i32")] + fn vaddlv_u32_(a: uint32x2_t) -> u64; + } + vaddlv_u32_(a) +} + +/// Unsigned Add Long across Vector +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddlvq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uaddlv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddlvq_u32(a: uint32x4_t) -> u64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uaddlv.i64.v4i32")] + fn vaddlvq_u32_(a: uint32x4_t) -> u64; + } + vaddlvq_u32_(a) +} + +/// Signed Subtract Wide +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_high_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ssubw))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsubw_high_s8(a: int16x8_t, b: int8x16_t) -> int16x8_t { + let c: int8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + simd_sub(a, simd_cast(c)) +} + +/// Signed Subtract Wide +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_high_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ssubw))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsubw_high_s16(a: int32x4_t, b: int16x8_t) -> int32x4_t { + let c: int16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + simd_sub(a, simd_cast(c)) +} + +/// Signed Subtract Wide +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_high_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ssubw))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsubw_high_s32(a: int64x2_t, b: int32x4_t) -> int64x2_t { + let c: int32x2_t = simd_shuffle2!(b, b, [2, 3]); + simd_sub(a, simd_cast(c)) +} + +/// Unsigned Subtract Wide +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_high_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usubw))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsubw_high_u8(a: uint16x8_t, b: uint8x16_t) -> uint16x8_t { + let c: uint8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + simd_sub(a, simd_cast(c)) +} + +/// Unsigned Subtract Wide +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_high_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usubw))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsubw_high_u16(a: uint32x4_t, b: uint16x8_t) -> uint32x4_t { + let c: uint16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + simd_sub(a, simd_cast(c)) +} + +/// Unsigned Subtract Wide +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_high_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usubw))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsubw_high_u32(a: uint64x2_t, b: uint32x4_t) -> uint64x2_t { + let c: uint32x2_t = simd_shuffle2!(b, b, [2, 3]); + simd_sub(a, simd_cast(c)) +} + +/// Signed Subtract Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_high_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ssubl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsubl_high_s8(a: int8x16_t, b: int8x16_t) -> int16x8_t { + let c: int8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + let d: int16x8_t = simd_cast(c); + let e: int8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let f: int16x8_t = simd_cast(e); + simd_sub(d, f) +} + +/// Signed Subtract Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_high_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ssubl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsubl_high_s16(a: int16x8_t, b: int16x8_t) -> int32x4_t { + let c: int16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let d: int32x4_t = simd_cast(c); + let e: int16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let f: int32x4_t = simd_cast(e); + simd_sub(d, f) +} + +/// Signed Subtract Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_high_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ssubl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsubl_high_s32(a: int32x4_t, b: int32x4_t) -> int64x2_t { + let c: int32x2_t = simd_shuffle2!(a, a, [2, 3]); + let d: int64x2_t = simd_cast(c); + let e: int32x2_t = simd_shuffle2!(b, b, [2, 3]); + let f: int64x2_t = simd_cast(e); + simd_sub(d, f) +} + +/// Unsigned Subtract Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_high_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usubl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsubl_high_u8(a: uint8x16_t, b: uint8x16_t) -> uint16x8_t { + let c: uint8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + let d: uint16x8_t = simd_cast(c); + let e: uint8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let f: uint16x8_t = simd_cast(e); + simd_sub(d, f) +} + +/// Unsigned Subtract Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_high_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usubl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsubl_high_u16(a: uint16x8_t, b: uint16x8_t) -> uint32x4_t { + let c: uint16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let d: uint32x4_t = simd_cast(c); + let e: uint16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let f: uint32x4_t = simd_cast(e); + simd_sub(d, f) +} + +/// Unsigned Subtract Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_high_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usubl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsubl_high_u32(a: uint32x4_t, b: uint32x4_t) -> uint64x2_t { + let c: uint32x2_t = simd_shuffle2!(a, a, [2, 3]); + let d: uint64x2_t = simd_cast(c); + let e: uint32x2_t = simd_shuffle2!(b, b, [2, 3]); + let f: uint64x2_t = simd_cast(e); + simd_sub(d, f) +} + +/// Bit clear and exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbcaxq_s8) +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(bcax))] +pub unsafe fn vbcaxq_s8(a: int8x16_t, b: int8x16_t, c: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.bcaxs.v16i8")] + fn vbcaxq_s8_(a: int8x16_t, b: int8x16_t, c: int8x16_t) -> int8x16_t; + } + vbcaxq_s8_(a, b, c) +} + +/// Bit clear and exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbcaxq_s16) +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(bcax))] +pub unsafe fn vbcaxq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.bcaxs.v8i16")] + fn vbcaxq_s16_(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t; + } + vbcaxq_s16_(a, b, c) +} + +/// Bit clear and exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbcaxq_s32) +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(bcax))] +pub unsafe fn vbcaxq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.bcaxs.v4i32")] + fn vbcaxq_s32_(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t; + } + vbcaxq_s32_(a, b, c) +} + +/// Bit clear and exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbcaxq_s64) +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(bcax))] +pub unsafe fn vbcaxq_s64(a: int64x2_t, b: int64x2_t, c: int64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.bcaxs.v2i64")] + fn vbcaxq_s64_(a: int64x2_t, b: int64x2_t, c: int64x2_t) -> int64x2_t; + } + vbcaxq_s64_(a, b, c) +} + +/// Bit clear and exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbcaxq_u8) +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(bcax))] +pub unsafe fn vbcaxq_u8(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.bcaxu.v16i8")] + fn vbcaxq_u8_(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t; + } + vbcaxq_u8_(a, b, c) +} + +/// Bit clear and exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbcaxq_u16) +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(bcax))] +pub unsafe fn vbcaxq_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.bcaxu.v8i16")] + fn vbcaxq_u16_(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t; + } + vbcaxq_u16_(a, b, c) +} + +/// Bit clear and exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbcaxq_u32) +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(bcax))] +pub unsafe fn vbcaxq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.bcaxu.v4i32")] + fn vbcaxq_u32_(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t; + } + vbcaxq_u32_(a, b, c) +} + +/// Bit clear and exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbcaxq_u64) +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(bcax))] +pub unsafe fn vbcaxq_u64(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.bcaxu.v2i64")] + fn vbcaxq_u64_(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t; + } + vbcaxq_u64_(a, b, c) +} + +/// Floating-point complex add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcadd_rot270_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcadd))] +pub unsafe fn vcadd_rot270_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcadd.rot270.v2f32")] + fn vcadd_rot270_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; + } + vcadd_rot270_f32_(a, b) +} + +/// Floating-point complex add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaddq_rot270_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcadd))] +pub unsafe fn vcaddq_rot270_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcadd.rot270.v4f32")] + fn vcaddq_rot270_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + } + vcaddq_rot270_f32_(a, b) +} + +/// Floating-point complex add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaddq_rot270_f64) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcadd))] +pub unsafe fn vcaddq_rot270_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcadd.rot270.v2f64")] + fn vcaddq_rot270_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + } + vcaddq_rot270_f64_(a, b) +} + +/// Floating-point complex add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcadd_rot90_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcadd))] +pub unsafe fn vcadd_rot90_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcadd.rot90.v2f32")] + fn vcadd_rot90_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; + } + vcadd_rot90_f32_(a, b) +} + +/// Floating-point complex add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaddq_rot90_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcadd))] +pub unsafe fn vcaddq_rot90_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcadd.rot90.v4f32")] + fn vcaddq_rot90_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + } + vcaddq_rot90_f32_(a, b) +} + +/// Floating-point complex add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaddq_rot90_f64) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcadd))] +pub unsafe fn vcaddq_rot90_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcadd.rot90.v2f64")] + fn vcaddq_rot90_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + } + vcaddq_rot90_f64_(a, b) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla))] +pub unsafe fn vcmla_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcmla.rot0.v2f32")] + fn vcmla_f32_(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t; + } + vcmla_f32_(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla))] +pub unsafe fn vcmlaq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcmla.rot0.v4f32")] + fn vcmlaq_f32_(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t; + } + vcmlaq_f32_(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_f64) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla))] +pub unsafe fn vcmlaq_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcmla.rot0.v2f64")] + fn vcmlaq_f64_(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t; + } + vcmlaq_f64_(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot90_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla))] +pub unsafe fn vcmla_rot90_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcmla.rot90.v2f32")] + fn vcmla_rot90_f32_(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t; + } + vcmla_rot90_f32_(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot90_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla))] +pub unsafe fn vcmlaq_rot90_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcmla.rot90.v4f32")] + fn vcmlaq_rot90_f32_(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t; + } + vcmlaq_rot90_f32_(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot90_f64) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla))] +pub unsafe fn vcmlaq_rot90_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcmla.rot90.v2f64")] + fn vcmlaq_rot90_f64_(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t; + } + vcmlaq_rot90_f64_(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot180_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla))] +pub unsafe fn vcmla_rot180_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcmla.rot180.v2f32")] + fn vcmla_rot180_f32_(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t; + } + vcmla_rot180_f32_(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot180_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla))] +pub unsafe fn vcmlaq_rot180_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcmla.rot180.v4f32")] + fn vcmlaq_rot180_f32_(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t; + } + vcmlaq_rot180_f32_(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot180_f64) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla))] +pub unsafe fn vcmlaq_rot180_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcmla.rot180.v2f64")] + fn vcmlaq_rot180_f64_(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t; + } + vcmlaq_rot180_f64_(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot270_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla))] +pub unsafe fn vcmla_rot270_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcmla.rot270.v2f32")] + fn vcmla_rot270_f32_(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t; + } + vcmla_rot270_f32_(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot270_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla))] +pub unsafe fn vcmlaq_rot270_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcmla.rot270.v4f32")] + fn vcmlaq_rot270_f32_(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t; + } + vcmlaq_rot270_f32_(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot270_f64) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla))] +pub unsafe fn vcmlaq_rot270_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcmla.rot270.v2f64")] + fn vcmlaq_rot270_f64_(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t; + } + vcmlaq_rot270_f64_(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_lane_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vcmla_lane_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + static_assert!(LANE : i32 where LANE == 0); + let c: float32x2_t = simd_shuffle2!(c, c, [2 * LANE as u32, 2 * LANE as u32 + 1]); + vcmla_f32(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_laneq_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vcmla_laneq_f32(a: float32x2_t, b: float32x2_t, c: float32x4_t) -> float32x2_t { + static_assert_imm1!(LANE); + let c: float32x2_t = simd_shuffle2!(c, c, [2 * LANE as u32, 2 * LANE as u32 + 1]); + vcmla_f32(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_lane_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vcmlaq_lane_f32(a: float32x4_t, b: float32x4_t, c: float32x2_t) -> float32x4_t { + static_assert!(LANE : i32 where LANE == 0); + let c: float32x4_t = simd_shuffle4!(c, c, [2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]); + vcmlaq_f32(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_laneq_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vcmlaq_laneq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + static_assert_imm1!(LANE); + let c: float32x4_t = simd_shuffle4!(c, c, [2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]); + vcmlaq_f32(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot90_lane_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vcmla_rot90_lane_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + static_assert!(LANE : i32 where LANE == 0); + let c: float32x2_t = simd_shuffle2!(c, c, [2 * LANE as u32, 2 * LANE as u32 + 1]); + vcmla_rot90_f32(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot90_laneq_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vcmla_rot90_laneq_f32(a: float32x2_t, b: float32x2_t, c: float32x4_t) -> float32x2_t { + static_assert_imm1!(LANE); + let c: float32x2_t = simd_shuffle2!(c, c, [2 * LANE as u32, 2 * LANE as u32 + 1]); + vcmla_rot90_f32(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot90_lane_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vcmlaq_rot90_lane_f32(a: float32x4_t, b: float32x4_t, c: float32x2_t) -> float32x4_t { + static_assert!(LANE : i32 where LANE == 0); + let c: float32x4_t = simd_shuffle4!(c, c, [2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]); + vcmlaq_rot90_f32(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot90_laneq_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vcmlaq_rot90_laneq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + static_assert_imm1!(LANE); + let c: float32x4_t = simd_shuffle4!(c, c, [2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]); + vcmlaq_rot90_f32(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot180_lane_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vcmla_rot180_lane_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + static_assert!(LANE : i32 where LANE == 0); + let c: float32x2_t = simd_shuffle2!(c, c, [2 * LANE as u32, 2 * LANE as u32 + 1]); + vcmla_rot180_f32(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot180_laneq_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vcmla_rot180_laneq_f32(a: float32x2_t, b: float32x2_t, c: float32x4_t) -> float32x2_t { + static_assert_imm1!(LANE); + let c: float32x2_t = simd_shuffle2!(c, c, [2 * LANE as u32, 2 * LANE as u32 + 1]); + vcmla_rot180_f32(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot180_lane_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vcmlaq_rot180_lane_f32(a: float32x4_t, b: float32x4_t, c: float32x2_t) -> float32x4_t { + static_assert!(LANE : i32 where LANE == 0); + let c: float32x4_t = simd_shuffle4!(c, c, [2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]); + vcmlaq_rot180_f32(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot180_laneq_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vcmlaq_rot180_laneq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + static_assert_imm1!(LANE); + let c: float32x4_t = simd_shuffle4!(c, c, [2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]); + vcmlaq_rot180_f32(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot270_lane_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vcmla_rot270_lane_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + static_assert!(LANE : i32 where LANE == 0); + let c: float32x2_t = simd_shuffle2!(c, c, [2 * LANE as u32, 2 * LANE as u32 + 1]); + vcmla_rot270_f32(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot270_laneq_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vcmla_rot270_laneq_f32(a: float32x2_t, b: float32x2_t, c: float32x4_t) -> float32x2_t { + static_assert_imm1!(LANE); + let c: float32x2_t = simd_shuffle2!(c, c, [2 * LANE as u32, 2 * LANE as u32 + 1]); + vcmla_rot270_f32(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot270_lane_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vcmlaq_rot270_lane_f32(a: float32x4_t, b: float32x4_t, c: float32x2_t) -> float32x4_t { + static_assert!(LANE : i32 where LANE == 0); + let c: float32x4_t = simd_shuffle4!(c, c, [2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]); + vcmlaq_rot270_f32(a, b, c) +} + +/// Floating-point complex multiply accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot270_laneq_f32) +#[inline] +#[target_feature(enable = "neon,fcma")] +#[cfg_attr(test, assert_instr(fcmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vcmlaq_rot270_laneq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + static_assert_imm1!(LANE); + let c: float32x4_t = simd_shuffle4!(c, c, [2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]); + vcmlaq_rot270_f32(a, b, c) +} + +/// Dot product arithmetic +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdot_s32) +#[inline] +#[target_feature(enable = "neon,dotprod")] +#[cfg_attr(test, assert_instr(sdot))] +pub unsafe fn vdot_s32(a: int32x2_t, b: int8x8_t, c: int8x8_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sdot.v2i32.v8i8")] + fn vdot_s32_(a: int32x2_t, b: int8x8_t, c: int8x8_t) -> int32x2_t; + } + vdot_s32_(a, b, c) +} + +/// Dot product arithmetic +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdotq_s32) +#[inline] +#[target_feature(enable = "neon,dotprod")] +#[cfg_attr(test, assert_instr(sdot))] +pub unsafe fn vdotq_s32(a: int32x4_t, b: int8x16_t, c: int8x16_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sdot.v4i32.v16i8")] + fn vdotq_s32_(a: int32x4_t, b: int8x16_t, c: int8x16_t) -> int32x4_t; + } + vdotq_s32_(a, b, c) +} + +/// Dot product arithmetic +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdot_u32) +#[inline] +#[target_feature(enable = "neon,dotprod")] +#[cfg_attr(test, assert_instr(udot))] +pub unsafe fn vdot_u32(a: uint32x2_t, b: uint8x8_t, c: uint8x8_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.udot.v2i32.v8i8")] + fn vdot_u32_(a: uint32x2_t, b: uint8x8_t, c: uint8x8_t) -> uint32x2_t; + } + vdot_u32_(a, b, c) +} + +/// Dot product arithmetic +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdotq_u32) +#[inline] +#[target_feature(enable = "neon,dotprod")] +#[cfg_attr(test, assert_instr(udot))] +pub unsafe fn vdotq_u32(a: uint32x4_t, b: uint8x16_t, c: uint8x16_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.udot.v4i32.v16i8")] + fn vdotq_u32_(a: uint32x4_t, b: uint8x16_t, c: uint8x16_t) -> uint32x4_t; + } + vdotq_u32_(a, b, c) +} + +/// Dot product arithmetic +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdot_lane_s32) +#[inline] +#[target_feature(enable = "neon,dotprod")] +#[cfg_attr(test, assert_instr(sdot, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vdot_lane_s32(a: int32x2_t, b: int8x8_t, c: int8x8_t) -> int32x2_t { + static_assert_imm1!(LANE); + let c: int8x8_t = simd_shuffle8!(c, c, [4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3, 4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3]); + vdot_s32(a, b, c) +} + +/// Dot product arithmetic +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdot_laneq_s32) +#[inline] +#[target_feature(enable = "neon,dotprod")] +#[cfg_attr(test, assert_instr(sdot, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vdot_laneq_s32(a: int32x2_t, b: int8x8_t, c: int8x16_t) -> int32x2_t { + static_assert_imm2!(LANE); + let c: int8x8_t = simd_shuffle8!(c, c, [4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3, 4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3]); + vdot_s32(a, b, c) +} + +/// Dot product arithmetic +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdotq_lane_s32) +#[inline] +#[target_feature(enable = "neon,dotprod")] +#[cfg_attr(test, assert_instr(sdot, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vdotq_lane_s32(a: int32x4_t, b: int8x16_t, c: int8x8_t) -> int32x4_t { + static_assert_imm1!(LANE); + let c: int8x16_t = simd_shuffle16!(c, c, [4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3, 4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3, 4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3, 4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3]); + vdotq_s32(a, b, c) +} + +/// Dot product arithmetic +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdotq_laneq_s32) +#[inline] +#[target_feature(enable = "neon,dotprod")] +#[cfg_attr(test, assert_instr(sdot, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vdotq_laneq_s32(a: int32x4_t, b: int8x16_t, c: int8x16_t) -> int32x4_t { + static_assert_imm2!(LANE); + let c: int8x16_t = simd_shuffle16!(c, c, [4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3, 4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3, 4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3, 4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3]); + vdotq_s32(a, b, c) +} + +/// Dot product arithmetic +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdot_lane_u32) +#[inline] +#[target_feature(enable = "neon,dotprod")] +#[cfg_attr(test, assert_instr(udot, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vdot_lane_u32(a: uint32x2_t, b: uint8x8_t, c: uint8x8_t) -> uint32x2_t { + static_assert_imm1!(LANE); + let c: uint8x8_t = simd_shuffle8!(c, c, [4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3, 4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3]); + vdot_u32(a, b, c) +} + +/// Dot product arithmetic +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdot_laneq_u32) +#[inline] +#[target_feature(enable = "neon,dotprod")] +#[cfg_attr(test, assert_instr(udot, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vdot_laneq_u32(a: uint32x2_t, b: uint8x8_t, c: uint8x16_t) -> uint32x2_t { + static_assert_imm2!(LANE); + let c: uint8x8_t = simd_shuffle8!(c, c, [4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3, 4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3]); + vdot_u32(a, b, c) +} + +/// Dot product arithmetic +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdotq_lane_u32) +#[inline] +#[target_feature(enable = "neon,dotprod")] +#[cfg_attr(test, assert_instr(udot, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vdotq_lane_u32(a: uint32x4_t, b: uint8x16_t, c: uint8x8_t) -> uint32x4_t { + static_assert_imm1!(LANE); + let c: uint8x16_t = simd_shuffle16!(c, c, [4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3, 4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3, 4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3, 4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3]); + vdotq_u32(a, b, c) +} + +/// Dot product arithmetic +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdotq_laneq_u32) +#[inline] +#[target_feature(enable = "neon,dotprod")] +#[cfg_attr(test, assert_instr(udot, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vdotq_laneq_u32(a: uint32x4_t, b: uint8x16_t, c: uint8x16_t) -> uint32x4_t { + static_assert_imm2!(LANE); + let c: uint8x16_t = simd_shuffle16!(c, c, [4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3, 4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3, 4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3, 4 * LANE as u32, 4 * LANE as u32 + 1, 4 * LANE as u32 + 2, 4 * LANE as u32 + 3]); + vdotq_u32(a, b, c) +} + +/// Maximum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmax_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmax))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmax_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmax.v1f64")] + fn vmax_f64_(a: float64x1_t, b: float64x1_t) -> float64x1_t; + } + vmax_f64_(a, b) +} + +/// Maximum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmax))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmaxq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmax.v2f64")] + fn vmaxq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + } + vmaxq_f64_(a, b) +} + +/// Floating-point Maximum Number (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxnm_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxnm))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmaxnm_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxnm.v1f64")] + fn vmaxnm_f64_(a: float64x1_t, b: float64x1_t) -> float64x1_t; + } + vmaxnm_f64_(a, b) +} + +/// Floating-point Maximum Number (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxnmq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxnm))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmaxnmq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxnm.v2f64")] + fn vmaxnmq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + } + vmaxnmq_f64_(a, b) +} + +/// Floating-point maximum number across vector +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxnmv_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxnmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmaxnmv_f32(a: float32x2_t) -> f32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxnmv.f32.v2f32")] + fn vmaxnmv_f32_(a: float32x2_t) -> f32; + } + vmaxnmv_f32_(a) +} + +/// Floating-point maximum number across vector +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxnmvq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxnmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmaxnmvq_f64(a: float64x2_t) -> f64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxnmv.f64.v2f64")] + fn vmaxnmvq_f64_(a: float64x2_t) -> f64; + } + vmaxnmvq_f64_(a) +} + +/// Floating-point maximum number across vector +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxnmvq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxnmv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmaxnmvq_f32(a: float32x4_t) -> f32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxnmv.f32.v4f32")] + fn vmaxnmvq_f32_(a: float32x4_t) -> f32; + } + vmaxnmvq_f32_(a) +} + +/// Floating-point Maximum Number Pairwise (vector). +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxnm_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxnmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpmaxnm_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxnmp.v2f32")] + fn vpmaxnm_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; + } + vpmaxnm_f32_(a, b) +} + +/// Floating-point Maximum Number Pairwise (vector). +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxnmq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxnmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpmaxnmq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxnmp.v2f64")] + fn vpmaxnmq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + } + vpmaxnmq_f64_(a, b) +} + +/// Floating-point Maximum Number Pairwise (vector). +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxnmq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxnmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpmaxnmq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxnmp.v4f32")] + fn vpmaxnmq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + } + vpmaxnmq_f32_(a, b) +} + +/// Floating-point maximum number pairwise +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxnms_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxnmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpmaxnms_f32(a: float32x2_t) -> f32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxnmv.f32.v2f32")] + fn vpmaxnms_f32_(a: float32x2_t) -> f32; + } + vpmaxnms_f32_(a) +} + +/// Floating-point maximum number pairwise +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxnmqd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxnmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpmaxnmqd_f64(a: float64x2_t) -> f64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxnmv.f64.v2f64")] + fn vpmaxnmqd_f64_(a: float64x2_t) -> f64; + } + vpmaxnmqd_f64_(a) +} + +/// Floating-point maximum pairwise +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxs_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpmaxs_f32(a: float32x2_t) -> f32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxv.f32.v2f32")] + fn vpmaxs_f32_(a: float32x2_t) -> f32; + } + vpmaxs_f32_(a) +} + +/// Floating-point maximum pairwise +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxqd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpmaxqd_f64(a: float64x2_t) -> f64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxv.f64.v2f64")] + fn vpmaxqd_f64_(a: float64x2_t) -> f64; + } + vpmaxqd_f64_(a) +} + +/// Minimum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmin_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmin))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmin_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmin.v1f64")] + fn vmin_f64_(a: float64x1_t, b: float64x1_t) -> float64x1_t; + } + vmin_f64_(a, b) +} + +/// Minimum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmin))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vminq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmin.v2f64")] + fn vminq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + } + vminq_f64_(a, b) +} + +/// Floating-point Minimum Number (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminnm_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminnm))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vminnm_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminnm.v1f64")] + fn vminnm_f64_(a: float64x1_t, b: float64x1_t) -> float64x1_t; + } + vminnm_f64_(a, b) +} + +/// Floating-point Minimum Number (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminnmq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminnm))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vminnmq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminnm.v2f64")] + fn vminnmq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + } + vminnmq_f64_(a, b) +} + +/// Floating-point minimum number across vector +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminnmv_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminnmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vminnmv_f32(a: float32x2_t) -> f32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminnmv.f32.v2f32")] + fn vminnmv_f32_(a: float32x2_t) -> f32; + } + vminnmv_f32_(a) +} + +/// Floating-point minimum number across vector +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminnmvq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminnmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vminnmvq_f64(a: float64x2_t) -> f64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminnmv.f64.v2f64")] + fn vminnmvq_f64_(a: float64x2_t) -> f64; + } + vminnmvq_f64_(a) +} + +/// Floating-point minimum number across vector +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminnmvq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminnmv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vminnmvq_f32(a: float32x4_t) -> f32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminnmv.f32.v4f32")] + fn vminnmvq_f32_(a: float32x4_t) -> f32; + } + vminnmvq_f32_(a) +} + +/// Vector move +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovl_high_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sxtl2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmovl_high_s8(a: int8x16_t) -> int16x8_t { + let a: int8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + vmovl_s8(a) +} + +/// Vector move +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovl_high_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sxtl2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmovl_high_s16(a: int16x8_t) -> int32x4_t { + let a: int16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + vmovl_s16(a) +} + +/// Vector move +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovl_high_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sxtl2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmovl_high_s32(a: int32x4_t) -> int64x2_t { + let a: int32x2_t = simd_shuffle2!(a, a, [2, 3]); + vmovl_s32(a) +} + +/// Vector move +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovl_high_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uxtl2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmovl_high_u8(a: uint8x16_t) -> uint16x8_t { + let a: uint8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + vmovl_u8(a) +} + +/// Vector move +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovl_high_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uxtl2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmovl_high_u16(a: uint16x8_t) -> uint32x4_t { + let a: uint16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + vmovl_u16(a) +} + +/// Vector move +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovl_high_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uxtl2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmovl_high_u32(a: uint32x4_t) -> uint64x2_t { + let a: uint32x2_t = simd_shuffle2!(a, a, [2, 3]); + vmovl_u32(a) +} + +/// Floating-point add pairwise +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(faddp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpaddq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.faddp.v4f32")] + fn vpaddq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + } + vpaddq_f32_(a, b) +} + +/// Floating-point add pairwise +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(faddp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpaddq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.faddp.v2f64")] + fn vpaddq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + } + vpaddq_f64_(a, b) +} + +/// Floating-point add pairwise +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadds_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpadds_f32(a: float32x2_t) -> f32 { + let a1: f32 = simd_extract(a, 0); + let a2: f32 = simd_extract(a, 1); + a1 + a2 +} + +/// Floating-point add pairwise +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpaddd_f64(a: float64x2_t) -> f64 { + let a1: f64 = simd_extract(a, 0); + let a2: f64 = simd_extract(a, 1); + a1 + a2 +} + +/// Floating-point Minimum Number Pairwise (vector). +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminnm_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminnmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpminnm_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminnmp.v2f32")] + fn vpminnm_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; + } + vpminnm_f32_(a, b) +} + +/// Floating-point Minimum Number Pairwise (vector). +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminnmq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminnmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpminnmq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminnmp.v2f64")] + fn vpminnmq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + } + vpminnmq_f64_(a, b) +} + +/// Floating-point Minimum Number Pairwise (vector). +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminnmq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminnmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpminnmq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminnmp.v4f32")] + fn vpminnmq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + } + vpminnmq_f32_(a, b) +} + +/// Floating-point minimum number pairwise +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminnms_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminnmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpminnms_f32(a: float32x2_t) -> f32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminnmv.f32.v2f32")] + fn vpminnms_f32_(a: float32x2_t) -> f32; + } + vpminnms_f32_(a) +} + +/// Floating-point minimum number pairwise +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminnmqd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminnmp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpminnmqd_f64(a: float64x2_t) -> f64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminnmv.f64.v2f64")] + fn vpminnmqd_f64_(a: float64x2_t) -> f64; + } + vpminnmqd_f64_(a) +} + +/// Floating-point minimum pairwise +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmins_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpmins_f32(a: float32x2_t) -> f32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminv.f32.v2f32")] + fn vpmins_f32_(a: float32x2_t) -> f32; + } + vpmins_f32_(a) +} + +/// Floating-point minimum pairwise +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminqd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpminqd_f64(a: float64x2_t) -> f64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminv.f64.v2f64")] + fn vpminqd_f64_(a: float64x2_t) -> f64; + } + vpminqd_f64_(a) +} + +/// Signed saturating doubling multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmullh_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmullh_s16(a: i16, b: i16) -> i32 { + let a: int16x4_t = vdup_n_s16(a); + let b: int16x4_t = vdup_n_s16(b); + simd_extract(vqdmull_s16(a, b), 0) +} + +/// Signed saturating doubling multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulls_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmulls_s32(a: i32, b: i32) -> i64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqdmulls.scalar")] + fn vqdmulls_s32_(a: i32, b: i32) -> i64; + } + vqdmulls_s32_(a, b) +} + +/// Signed saturating doubling multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_high_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmull_high_s16(a: int16x8_t, b: int16x8_t) -> int32x4_t { + let a: int16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let b: int16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + vqdmull_s16(a, b) +} + +/// Signed saturating doubling multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_high_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmull_high_s32(a: int32x4_t, b: int32x4_t) -> int64x2_t { + let a: int32x2_t = simd_shuffle2!(a, a, [2, 3]); + let b: int32x2_t = simd_shuffle2!(b, b, [2, 3]); + vqdmull_s32(a, b) +} + +/// Signed saturating doubling multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_high_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmull_high_n_s16(a: int16x8_t, b: i16) -> int32x4_t { + let a: int16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let b: int16x4_t = vdup_n_s16(b); + vqdmull_s16(a, b) +} + +/// Signed saturating doubling multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_high_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmull_high_n_s32(a: int32x4_t, b: i32) -> int64x2_t { + let a: int32x2_t = simd_shuffle2!(a, a, [2, 3]); + let b: int32x2_t = vdup_n_s32(b); + vqdmull_s32(a, b) +} + +/// Vector saturating doubling long multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull, N = 4))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmull_laneq_s16(a: int16x4_t, b: int16x8_t) -> int32x4_t { + static_assert_imm3!(N); + let b: int16x4_t = simd_shuffle4!(b, b, [N as u32, N as u32, N as u32, N as u32]); + vqdmull_s16(a, b) +} + +/// Vector saturating doubling long multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmull_laneq_s32(a: int32x2_t, b: int32x4_t) -> int64x2_t { + static_assert_imm2!(N); + let b: int32x2_t = simd_shuffle2!(b, b, [N as u32, N as u32]); + vqdmull_s32(a, b) +} + +/// Signed saturating doubling multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmullh_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmullh_lane_s16(a: i16, b: int16x4_t) -> i32 { + static_assert_imm2!(N); + let b: i16 = simd_extract(b, N as u32); + vqdmullh_s16(a, b) +} + +/// Signed saturating doubling multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmullh_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull, N = 4))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmullh_laneq_s16(a: i16, b: int16x8_t) -> i32 { + static_assert_imm3!(N); + let b: i16 = simd_extract(b, N as u32); + vqdmullh_s16(a, b) +} + +/// Signed saturating doubling multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulls_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmulls_lane_s32(a: i32, b: int32x2_t) -> i64 { + static_assert_imm1!(N); + let b: i32 = simd_extract(b, N as u32); + vqdmulls_s32(a, b) +} + +/// Signed saturating doubling multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulls_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmulls_laneq_s32(a: i32, b: int32x4_t) -> i64 { + static_assert_imm2!(N); + let b: i32 = simd_extract(b, N as u32); + vqdmulls_s32(a, b) +} + +/// Signed saturating doubling multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_high_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmull_high_lane_s16(a: int16x8_t, b: int16x4_t) -> int32x4_t { + static_assert_imm2!(N); + let a: int16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let b: int16x4_t = simd_shuffle4!(b, b, [N as u32, N as u32, N as u32, N as u32]); + vqdmull_s16(a, b) +} + +/// Signed saturating doubling multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_high_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull2, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmull_high_lane_s32(a: int32x4_t, b: int32x2_t) -> int64x2_t { + static_assert_imm1!(N); + let a: int32x2_t = simd_shuffle2!(a, a, [2, 3]); + let b: int32x2_t = simd_shuffle2!(b, b, [N as u32, N as u32]); + vqdmull_s32(a, b) +} + +/// Signed saturating doubling multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_high_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull2, N = 4))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmull_high_laneq_s16(a: int16x8_t, b: int16x8_t) -> int32x4_t { + static_assert_imm3!(N); + let a: int16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let b: int16x4_t = simd_shuffle4!(b, b, [N as u32, N as u32, N as u32, N as u32]); + vqdmull_s16(a, b) +} + +/// Signed saturating doubling multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_high_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmull_high_laneq_s32(a: int32x4_t, b: int32x4_t) -> int64x2_t { + static_assert_imm2!(N); + let a: int32x2_t = simd_shuffle2!(a, a, [2, 3]); + let b: int32x2_t = simd_shuffle2!(b, b, [N as u32, N as u32]); + vqdmull_s32(a, b) +} + +/// Signed saturating doubling multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_high_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlal_high_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t { + vqaddq_s32(a, vqdmull_high_s16(b, c)) +} + +/// Signed saturating doubling multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_high_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlal_high_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t { + vqaddq_s64(a, vqdmull_high_s32(b, c)) +} + +/// Signed saturating doubling multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_high_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlal_high_n_s16(a: int32x4_t, b: int16x8_t, c: i16) -> int32x4_t { + vqaddq_s32(a, vqdmull_high_n_s16(b, c)) +} + +/// Signed saturating doubling multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_high_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlal_high_n_s32(a: int64x2_t, b: int32x4_t, c: i32) -> int64x2_t { + vqaddq_s64(a, vqdmull_high_n_s32(b, c)) +} + +/// Vector widening saturating doubling multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal, N = 2))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlal_laneq_s16(a: int32x4_t, b: int16x4_t, c: int16x8_t) -> int32x4_t { + static_assert_imm3!(N); + vqaddq_s32(a, vqdmull_laneq_s16::(b, c)) +} + +/// Vector widening saturating doubling multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal, N = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlal_laneq_s32(a: int64x2_t, b: int32x2_t, c: int32x4_t) -> int64x2_t { + static_assert_imm2!(N); + vqaddq_s64(a, vqdmull_laneq_s32::(b, c)) +} + +/// Signed saturating doubling multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_high_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal2, N = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlal_high_lane_s16(a: int32x4_t, b: int16x8_t, c: int16x4_t) -> int32x4_t { + static_assert_imm2!(N); + vqaddq_s32(a, vqdmull_high_lane_s16::(b, c)) +} + +/// Signed saturating doubling multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_high_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal2, N = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlal_high_laneq_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t { + static_assert_imm3!(N); + vqaddq_s32(a, vqdmull_high_laneq_s16::(b, c)) +} + +/// Signed saturating doubling multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_high_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal2, N = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlal_high_lane_s32(a: int64x2_t, b: int32x4_t, c: int32x2_t) -> int64x2_t { + static_assert_imm1!(N); + vqaddq_s64(a, vqdmull_high_lane_s32::(b, c)) +} + +/// Signed saturating doubling multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_high_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal2, N = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlal_high_laneq_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t { + static_assert_imm2!(N); + vqaddq_s64(a, vqdmull_high_laneq_s32::(b, c)) +} + +/// Signed saturating doubling multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlalh_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlalh_s16(a: i32, b: i16, c: i16) -> i32 { + let x: int32x4_t = vqdmull_s16(vdup_n_s16(b), vdup_n_s16(c)); + vqadds_s32(a, simd_extract(x, 0)) +} + +/// Signed saturating doubling multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlals_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlals_s32(a: i64, b: i32, c: i32) -> i64 { + let x: int64x2_t = vqdmull_s32(vdup_n_s32(b), vdup_n_s32(c)); + vqaddd_s64(a, simd_extract(x, 0)) +} + +/// Signed saturating doubling multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlalh_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlalh_lane_s16(a: i32, b: i16, c: int16x4_t) -> i32 { + static_assert_imm2!(LANE); + vqdmlalh_s16(a, b, simd_extract(c, LANE as u32)) +} + +/// Signed saturating doubling multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlalh_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlalh_laneq_s16(a: i32, b: i16, c: int16x8_t) -> i32 { + static_assert_imm3!(LANE); + vqdmlalh_s16(a, b, simd_extract(c, LANE as u32)) +} + +/// Signed saturating doubling multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlals_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlals_lane_s32(a: i64, b: i32, c: int32x2_t) -> i64 { + static_assert_imm1!(LANE); + vqdmlals_s32(a, b, simd_extract(c, LANE as u32)) +} + +/// Signed saturating doubling multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlals_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlals_laneq_s32(a: i64, b: i32, c: int32x4_t) -> i64 { + static_assert_imm2!(LANE); + vqdmlals_s32(a, b, simd_extract(c, LANE as u32)) +} + +/// Signed saturating doubling multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_high_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlsl_high_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t { + vqsubq_s32(a, vqdmull_high_s16(b, c)) +} + +/// Signed saturating doubling multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_high_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlsl_high_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t { + vqsubq_s64(a, vqdmull_high_s32(b, c)) +} + +/// Signed saturating doubling multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_high_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlsl_high_n_s16(a: int32x4_t, b: int16x8_t, c: i16) -> int32x4_t { + vqsubq_s32(a, vqdmull_high_n_s16(b, c)) +} + +/// Signed saturating doubling multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_high_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlsl_high_n_s32(a: int64x2_t, b: int32x4_t, c: i32) -> int64x2_t { + vqsubq_s64(a, vqdmull_high_n_s32(b, c)) +} + +/// Vector widening saturating doubling multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl, N = 2))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlsl_laneq_s16(a: int32x4_t, b: int16x4_t, c: int16x8_t) -> int32x4_t { + static_assert_imm3!(N); + vqsubq_s32(a, vqdmull_laneq_s16::(b, c)) +} + +/// Vector widening saturating doubling multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl, N = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlsl_laneq_s32(a: int64x2_t, b: int32x2_t, c: int32x4_t) -> int64x2_t { + static_assert_imm2!(N); + vqsubq_s64(a, vqdmull_laneq_s32::(b, c)) +} + +/// Signed saturating doubling multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_high_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl2, N = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlsl_high_lane_s16(a: int32x4_t, b: int16x8_t, c: int16x4_t) -> int32x4_t { + static_assert_imm2!(N); + vqsubq_s32(a, vqdmull_high_lane_s16::(b, c)) +} + +/// Signed saturating doubling multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_high_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl2, N = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlsl_high_laneq_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t { + static_assert_imm3!(N); + vqsubq_s32(a, vqdmull_high_laneq_s16::(b, c)) +} + +/// Signed saturating doubling multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_high_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl2, N = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlsl_high_lane_s32(a: int64x2_t, b: int32x4_t, c: int32x2_t) -> int64x2_t { + static_assert_imm1!(N); + vqsubq_s64(a, vqdmull_high_lane_s32::(b, c)) +} + +/// Signed saturating doubling multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_high_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl2, N = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlsl_high_laneq_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t { + static_assert_imm2!(N); + vqsubq_s64(a, vqdmull_high_laneq_s32::(b, c)) +} + +/// Signed saturating doubling multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlslh_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlslh_s16(a: i32, b: i16, c: i16) -> i32 { + let x: int32x4_t = vqdmull_s16(vdup_n_s16(b), vdup_n_s16(c)); + vqsubs_s32(a, simd_extract(x, 0)) +} + +/// Signed saturating doubling multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsls_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlsls_s32(a: i64, b: i32, c: i32) -> i64 { + let x: int64x2_t = vqdmull_s32(vdup_n_s32(b), vdup_n_s32(c)); + vqsubd_s64(a, simd_extract(x, 0)) +} + +/// Signed saturating doubling multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlslh_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlslh_lane_s16(a: i32, b: i16, c: int16x4_t) -> i32 { + static_assert_imm2!(LANE); + vqdmlslh_s16(a, b, simd_extract(c, LANE as u32)) +} + +/// Signed saturating doubling multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlslh_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlslh_laneq_s16(a: i32, b: i16, c: int16x8_t) -> i32 { + static_assert_imm3!(LANE); + vqdmlslh_s16(a, b, simd_extract(c, LANE as u32)) +} + +/// Signed saturating doubling multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsls_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlsls_lane_s32(a: i64, b: i32, c: int32x2_t) -> i64 { + static_assert_imm1!(LANE); + vqdmlsls_s32(a, b, simd_extract(c, LANE as u32)) +} + +/// Signed saturating doubling multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsls_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull, LANE = 0))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmlsls_laneq_s32(a: i64, b: i32, c: int32x4_t) -> i64 { + static_assert_imm2!(LANE); + vqdmlsls_s32(a, b, simd_extract(c, LANE as u32)) +} + +/// Signed saturating doubling multiply returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhh_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmulh))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmulhh_s16(a: i16, b: i16) -> i16 { + let a: int16x4_t = vdup_n_s16(a); + let b: int16x4_t = vdup_n_s16(b); + simd_extract(vqdmulh_s16(a, b), 0) +} + +/// Signed saturating doubling multiply returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhs_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmulh))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmulhs_s32(a: i32, b: i32) -> i32 { + let a: int32x2_t = vdup_n_s32(a); + let b: int32x2_t = vdup_n_s32(b); + simd_extract(vqdmulh_s32(a, b), 0) +} + +/// Signed saturating doubling multiply returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhh_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmulh, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmulhh_lane_s16(a: i16, b: int16x4_t) -> i16 { + static_assert_imm2!(N); + let b: i16 = simd_extract(b, N as u32); + vqdmulhh_s16(a, b) +} + +/// Signed saturating doubling multiply returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhh_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmulh, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmulhh_laneq_s16(a: i16, b: int16x8_t) -> i16 { + static_assert_imm3!(N); + let b: i16 = simd_extract(b, N as u32); + vqdmulhh_s16(a, b) +} + +/// Signed saturating doubling multiply returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhs_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmulh, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmulhs_lane_s32(a: i32, b: int32x2_t) -> i32 { + static_assert_imm1!(N); + let b: i32 = simd_extract(b, N as u32); + vqdmulhs_s32(a, b) +} + +/// Signed saturating doubling multiply returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhs_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmulh, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmulhs_laneq_s32(a: i32, b: int32x4_t) -> i32 { + static_assert_imm2!(N); + let b: i32 = simd_extract(b, N as u32); + vqdmulhs_s32(a, b) +} + +/// Vector saturating doubling multiply high by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulh_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmulh, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmulh_lane_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + static_assert_imm2!(LANE); + vqdmulh_s16(a, vdup_n_s16(simd_extract(b, LANE as u32))) +} + +/// Vector saturating doubling multiply high by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhq_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmulh, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmulhq_lane_s16(a: int16x8_t, b: int16x4_t) -> int16x8_t { + static_assert_imm2!(LANE); + vqdmulhq_s16(a, vdupq_n_s16(simd_extract(b, LANE as u32))) +} + +/// Vector saturating doubling multiply high by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulh_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmulh, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmulh_lane_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + static_assert_imm1!(LANE); + vqdmulh_s32(a, vdup_n_s32(simd_extract(b, LANE as u32))) +} + +/// Vector saturating doubling multiply high by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhq_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmulh, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqdmulhq_lane_s32(a: int32x4_t, b: int32x2_t) -> int32x4_t { + static_assert_imm1!(LANE); + vqdmulhq_s32(a, vdupq_n_s32(simd_extract(b, LANE as u32))) +} + +/// Saturating extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovnh_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtn))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqmovnh_s16(a: i16) -> i8 { + simd_extract(vqmovn_s16(vdupq_n_s16(a)), 0) +} + +/// Saturating extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovns_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtn))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqmovns_s32(a: i32) -> i16 { + simd_extract(vqmovn_s32(vdupq_n_s32(a)), 0) +} + +/// Saturating extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovnh_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqxtn))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqmovnh_u16(a: u16) -> u8 { + simd_extract(vqmovn_u16(vdupq_n_u16(a)), 0) +} + +/// Saturating extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovns_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqxtn))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqmovns_u32(a: u32) -> u16 { + simd_extract(vqmovn_u32(vdupq_n_u32(a)), 0) +} + +/// Saturating extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovnd_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtn))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqmovnd_s64(a: i64) -> i32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.scalar.sqxtn.i32.i64")] + fn vqmovnd_s64_(a: i64) -> i32; + } + vqmovnd_s64_(a) +} + +/// Saturating extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovnd_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqxtn))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqmovnd_u64(a: u64) -> u32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.scalar.uqxtn.i32.i64")] + fn vqmovnd_u64_(a: u64) -> u32; + } + vqmovnd_u64_(a) +} + +/// Signed saturating extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_high_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqmovn_high_s16(a: int8x8_t, b: int16x8_t) -> int8x16_t { + simd_shuffle16!(a, vqmovn_s16(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Signed saturating extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_high_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqmovn_high_s32(a: int16x4_t, b: int32x4_t) -> int16x8_t { + simd_shuffle8!(a, vqmovn_s32(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Signed saturating extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_high_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqmovn_high_s64(a: int32x2_t, b: int64x2_t) -> int32x4_t { + simd_shuffle4!(a, vqmovn_s64(b), [0, 1, 2, 3]) +} + +/// Signed saturating extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_high_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqxtn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqmovn_high_u16(a: uint8x8_t, b: uint16x8_t) -> uint8x16_t { + simd_shuffle16!(a, vqmovn_u16(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Signed saturating extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_high_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqxtn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqmovn_high_u32(a: uint16x4_t, b: uint32x4_t) -> uint16x8_t { + simd_shuffle8!(a, vqmovn_u32(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Signed saturating extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_high_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqxtn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqmovn_high_u64(a: uint32x2_t, b: uint64x2_t) -> uint32x4_t { + simd_shuffle4!(a, vqmovn_u64(b), [0, 1, 2, 3]) +} + +/// Signed saturating extract unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovunh_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtun))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqmovunh_s16(a: i16) -> u8 { + simd_extract(vqmovun_s16(vdupq_n_s16(a)), 0) +} + +/// Signed saturating extract unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovuns_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtun))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqmovuns_s32(a: i32) -> u16 { + simd_extract(vqmovun_s32(vdupq_n_s32(a)), 0) +} + +/// Signed saturating extract unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovund_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtun))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqmovund_s64(a: i64) -> u32 { + simd_extract(vqmovun_s64(vdupq_n_s64(a)), 0) +} + +/// Signed saturating extract unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovun_high_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtun2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqmovun_high_s16(a: uint8x8_t, b: int16x8_t) -> uint8x16_t { + simd_shuffle16!(a, vqmovun_s16(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Signed saturating extract unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovun_high_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtun2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqmovun_high_s32(a: uint16x4_t, b: int32x4_t) -> uint16x8_t { + simd_shuffle8!(a, vqmovun_s32(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Signed saturating extract unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovun_high_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtun2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqmovun_high_s64(a: uint32x2_t, b: int64x2_t) -> uint32x4_t { + simd_shuffle4!(a, vqmovun_s64(b), [0, 1, 2, 3]) +} + +/// Signed saturating rounding doubling multiply returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhh_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrdmulhh_s16(a: i16, b: i16) -> i16 { + simd_extract(vqrdmulh_s16(vdup_n_s16(a), vdup_n_s16(b)), 0) +} + +/// Signed saturating rounding doubling multiply returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhs_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrdmulhs_s32(a: i32, b: i32) -> i32 { + simd_extract(vqrdmulh_s32(vdup_n_s32(a), vdup_n_s32(b)), 0) +} + +/// Signed saturating rounding doubling multiply returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhh_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrdmulhh_lane_s16(a: i16, b: int16x4_t) -> i16 { + static_assert_imm2!(LANE); + vqrdmulhh_s16(a, simd_extract(b, LANE as u32)) +} + +/// Signed saturating rounding doubling multiply returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhh_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrdmulhh_laneq_s16(a: i16, b: int16x8_t) -> i16 { + static_assert_imm3!(LANE); + vqrdmulhh_s16(a, simd_extract(b, LANE as u32)) +} + +/// Signed saturating rounding doubling multiply returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhs_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrdmulhs_lane_s32(a: i32, b: int32x2_t) -> i32 { + static_assert_imm1!(LANE); + vqrdmulhs_s32(a, simd_extract(b, LANE as u32)) +} + +/// Signed saturating rounding doubling multiply returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhs_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrdmulhs_laneq_s32(a: i32, b: int32x4_t) -> i32 { + static_assert_imm2!(LANE); + vqrdmulhs_s32(a, simd_extract(b, LANE as u32)) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlah_s16) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlah))] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlah_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrdmlah.v4i16")] + fn vqrdmlah_s16_(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t; + } + vqrdmlah_s16_(a, b, c) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahq_s16) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlah))] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlahq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrdmlah.v8i16")] + fn vqrdmlahq_s16_(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t; + } + vqrdmlahq_s16_(a, b, c) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlah_s32) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlah))] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlah_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrdmlah.v2i32")] + fn vqrdmlah_s32_(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t; + } + vqrdmlah_s32_(a, b, c) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahq_s32) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlah))] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlahq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrdmlah.v4i32")] + fn vqrdmlahq_s32_(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t; + } + vqrdmlahq_s32_(a, b, c) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahh_s16) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlah))] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlahh_s16(a: i16, b: i16, c: i16) -> i16 { + let a: int16x4_t = vdup_n_s16(a); + let b: int16x4_t = vdup_n_s16(b); + let c: int16x4_t = vdup_n_s16(c); + simd_extract(vqrdmlah_s16(a, b, c), 0) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahs_s32) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlah))] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlahs_s32(a: i32, b: i32, c: i32) -> i32 { + let a: int32x2_t = vdup_n_s32(a); + let b: int32x2_t = vdup_n_s32(b); + let c: int32x2_t = vdup_n_s32(c); + simd_extract(vqrdmlah_s32(a, b, c), 0) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlah_lane_s16) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlah_lane_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t { + static_assert_imm2!(LANE); + let c: int16x4_t = simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmlah_s16(a, b, c) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlah_laneq_s16) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlah_laneq_s16(a: int16x4_t, b: int16x4_t, c: int16x8_t) -> int16x4_t { + static_assert_imm3!(LANE); + let c: int16x4_t = simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmlah_s16(a, b, c) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahq_lane_s16) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlahq_lane_s16(a: int16x8_t, b: int16x8_t, c: int16x4_t) -> int16x8_t { + static_assert_imm2!(LANE); + let c: int16x8_t = simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmlahq_s16(a, b, c) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahq_laneq_s16) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlahq_laneq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + static_assert_imm3!(LANE); + let c: int16x8_t = simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmlahq_s16(a, b, c) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlah_lane_s32) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlah_lane_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t { + static_assert_imm1!(LANE); + let c: int32x2_t = simd_shuffle2!(c, c, [LANE as u32, LANE as u32]); + vqrdmlah_s32(a, b, c) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlah_laneq_s32) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlah_laneq_s32(a: int32x2_t, b: int32x2_t, c: int32x4_t) -> int32x2_t { + static_assert_imm2!(LANE); + let c: int32x2_t = simd_shuffle2!(c, c, [LANE as u32, LANE as u32]); + vqrdmlah_s32(a, b, c) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahq_lane_s32) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlahq_lane_s32(a: int32x4_t, b: int32x4_t, c: int32x2_t) -> int32x4_t { + static_assert_imm1!(LANE); + let c: int32x4_t = simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmlahq_s32(a, b, c) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahq_laneq_s32) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlahq_laneq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + let c: int32x4_t = simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmlahq_s32(a, b, c) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahh_lane_s16) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlahh_lane_s16(a: i16, b: i16, c: int16x4_t) -> i16 { + static_assert_imm2!(LANE); + vqrdmlahh_s16(a, b, simd_extract(c, LANE as u32)) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahh_laneq_s16) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlahh_laneq_s16(a: i16, b: i16, c: int16x8_t) -> i16 { + static_assert_imm3!(LANE); + vqrdmlahh_s16(a, b, simd_extract(c, LANE as u32)) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahs_lane_s32) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlahs_lane_s32(a: i32, b: i32, c: int32x2_t) -> i32 { + static_assert_imm1!(LANE); + vqrdmlahs_s32(a, b, simd_extract(c, LANE as u32)) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahs_laneq_s32) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlahs_laneq_s32(a: i32, b: i32, c: int32x4_t) -> i32 { + static_assert_imm2!(LANE); + vqrdmlahs_s32(a, b, simd_extract(c, LANE as u32)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlsh_s16) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlsh))] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlsh_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrdmlsh.v4i16")] + fn vqrdmlsh_s16_(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t; + } + vqrdmlsh_s16_(a, b, c) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshq_s16) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlsh))] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlshq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrdmlsh.v8i16")] + fn vqrdmlshq_s16_(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t; + } + vqrdmlshq_s16_(a, b, c) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlsh_s32) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlsh))] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlsh_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrdmlsh.v2i32")] + fn vqrdmlsh_s32_(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t; + } + vqrdmlsh_s32_(a, b, c) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshq_s32) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlsh))] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlshq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrdmlsh.v4i32")] + fn vqrdmlshq_s32_(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t; + } + vqrdmlshq_s32_(a, b, c) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshh_s16) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlsh))] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlshh_s16(a: i16, b: i16, c: i16) -> i16 { + let a: int16x4_t = vdup_n_s16(a); + let b: int16x4_t = vdup_n_s16(b); + let c: int16x4_t = vdup_n_s16(c); + simd_extract(vqrdmlsh_s16(a, b, c), 0) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshs_s32) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlsh))] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlshs_s32(a: i32, b: i32, c: i32) -> i32 { + let a: int32x2_t = vdup_n_s32(a); + let b: int32x2_t = vdup_n_s32(b); + let c: int32x2_t = vdup_n_s32(c); + simd_extract(vqrdmlsh_s32(a, b, c), 0) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlsh_lane_s16) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlsh_lane_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t { + static_assert_imm2!(LANE); + let c: int16x4_t = simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmlsh_s16(a, b, c) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlsh_laneq_s16) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlsh_laneq_s16(a: int16x4_t, b: int16x4_t, c: int16x8_t) -> int16x4_t { + static_assert_imm3!(LANE); + let c: int16x4_t = simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmlsh_s16(a, b, c) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshq_lane_s16) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlshq_lane_s16(a: int16x8_t, b: int16x8_t, c: int16x4_t) -> int16x8_t { + static_assert_imm2!(LANE); + let c: int16x8_t = simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmlshq_s16(a, b, c) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshq_laneq_s16) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlshq_laneq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + static_assert_imm3!(LANE); + let c: int16x8_t = simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmlshq_s16(a, b, c) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlsh_lane_s32) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlsh_lane_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t { + static_assert_imm1!(LANE); + let c: int32x2_t = simd_shuffle2!(c, c, [LANE as u32, LANE as u32]); + vqrdmlsh_s32(a, b, c) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlsh_laneq_s32) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlsh_laneq_s32(a: int32x2_t, b: int32x2_t, c: int32x4_t) -> int32x2_t { + static_assert_imm2!(LANE); + let c: int32x2_t = simd_shuffle2!(c, c, [LANE as u32, LANE as u32]); + vqrdmlsh_s32(a, b, c) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshq_lane_s32) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlshq_lane_s32(a: int32x4_t, b: int32x4_t, c: int32x2_t) -> int32x4_t { + static_assert_imm1!(LANE); + let c: int32x4_t = simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmlshq_s32(a, b, c) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshq_laneq_s32) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlshq_laneq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + let c: int32x4_t = simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmlshq_s32(a, b, c) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshh_lane_s16) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlshh_lane_s16(a: i16, b: i16, c: int16x4_t) -> i16 { + static_assert_imm2!(LANE); + vqrdmlshh_s16(a, b, simd_extract(c, LANE as u32)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshh_laneq_s16) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlshh_laneq_s16(a: i16, b: i16, c: int16x8_t) -> i16 { + static_assert_imm3!(LANE); + vqrdmlshh_s16(a, b, simd_extract(c, LANE as u32)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshs_lane_s32) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlshs_lane_s32(a: i32, b: i32, c: int32x2_t) -> i32 { + static_assert_imm1!(LANE); + vqrdmlshs_s32(a, b, simd_extract(c, LANE as u32)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshs_laneq_s32) +#[inline] +#[target_feature(enable = "rdm")] +#[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[stable(feature = "rdm_intrinsics", since = "1.62.0")] +pub unsafe fn vqrdmlshs_laneq_s32(a: i32, b: i32, c: int32x4_t) -> i32 { + static_assert_imm2!(LANE); + vqrdmlshs_s32(a, b, simd_extract(c, LANE as u32)) +} + +/// Signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshls_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshls_s32(a: i32, b: i32) -> i32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshl.i32")] + fn vqrshls_s32_(a: i32, b: i32) -> i32; + } + vqrshls_s32_(a, b) +} + +/// Signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshld_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshld_s64(a: i64, b: i64) -> i64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshl.i64")] + fn vqrshld_s64_(a: i64, b: i64) -> i64; + } + vqrshld_s64_(a, b) +} + +/// Signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlb_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshlb_s8(a: i8, b: i8) -> i8 { + let a: int8x8_t = vdup_n_s8(a); + let b: int8x8_t = vdup_n_s8(b); + simd_extract(vqrshl_s8(a, b), 0) +} + +/// Signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlh_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshlh_s16(a: i16, b: i16) -> i16 { + let a: int16x4_t = vdup_n_s16(a); + let b: int16x4_t = vdup_n_s16(b); + simd_extract(vqrshl_s16(a, b), 0) +} + +/// Unsigned signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshls_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshls_u32(a: u32, b: i32) -> u32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshl.i32")] + fn vqrshls_u32_(a: u32, b: i32) -> u32; + } + vqrshls_u32_(a, b) +} + +/// Unsigned signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshld_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshld_u64(a: u64, b: i64) -> u64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshl.i64")] + fn vqrshld_u64_(a: u64, b: i64) -> u64; + } + vqrshld_u64_(a, b) +} + +/// Unsigned signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlb_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshlb_u8(a: u8, b: i8) -> u8 { + let a: uint8x8_t = vdup_n_u8(a); + let b: int8x8_t = vdup_n_s8(b); + simd_extract(vqrshl_u8(a, b), 0) +} + +/// Unsigned signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlh_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshlh_u16(a: u16, b: i16) -> u16 { + let a: uint16x4_t = vdup_n_u16(a); + let b: int16x4_t = vdup_n_s16(b); + simd_extract(vqrshl_u16(a, b), 0) +} + +/// Signed saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrnh_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrnh_n_s16(a: i16) -> i8 { + static_assert!(N : i32 where N >= 1 && N <= 8); + let a: int16x8_t = vdupq_n_s16(a); + simd_extract(vqrshrn_n_s16::(a), 0) +} + +/// Signed saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrns_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrns_n_s32(a: i32) -> i16 { + static_assert!(N : i32 where N >= 1 && N <= 16); + let a: int32x4_t = vdupq_n_s32(a); + simd_extract(vqrshrn_n_s32::(a), 0) +} + +/// Signed saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrnd_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrnd_n_s64(a: i64) -> i32 { + static_assert!(N : i32 where N >= 1 && N <= 32); + let a: int64x2_t = vdupq_n_s64(a); + simd_extract(vqrshrn_n_s64::(a), 0) +} + +/// Signed saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_high_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrn_high_n_s16(a: int8x8_t, b: int16x8_t) -> int8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shuffle16!(a, vqrshrn_n_s16::(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Signed saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_high_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrn_high_n_s32(a: int16x4_t, b: int32x4_t) -> int16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shuffle8!(a, vqrshrn_n_s32::(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Signed saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_high_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrn_high_n_s64(a: int32x2_t, b: int64x2_t) -> int32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shuffle4!(a, vqrshrn_n_s64::(b), [0, 1, 2, 3]) +} + +/// Unsigned saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrnh_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrnh_n_u16(a: u16) -> u8 { + static_assert!(N : i32 where N >= 1 && N <= 8); + let a: uint16x8_t = vdupq_n_u16(a); + simd_extract(vqrshrn_n_u16::(a), 0) +} + +/// Unsigned saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrns_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrns_n_u32(a: u32) -> u16 { + static_assert!(N : i32 where N >= 1 && N <= 16); + let a: uint32x4_t = vdupq_n_u32(a); + simd_extract(vqrshrn_n_u32::(a), 0) +} + +/// Unsigned saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrnd_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrnd_n_u64(a: u64) -> u32 { + static_assert!(N : i32 where N >= 1 && N <= 32); + let a: uint64x2_t = vdupq_n_u64(a); + simd_extract(vqrshrn_n_u64::(a), 0) +} + +/// Unsigned saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_high_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrn_high_n_u16(a: uint8x8_t, b: uint16x8_t) -> uint8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shuffle16!(a, vqrshrn_n_u16::(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Unsigned saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_high_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrn_high_n_u32(a: uint16x4_t, b: uint32x4_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shuffle8!(a, vqrshrn_n_u32::(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Unsigned saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_high_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrn_high_n_u64(a: uint32x2_t, b: uint64x2_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shuffle4!(a, vqrshrn_n_u64::(b), [0, 1, 2, 3]) +} + +/// Signed saturating rounded shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrunh_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrunh_n_s16(a: i16) -> u8 { + static_assert!(N : i32 where N >= 1 && N <= 8); + let a: int16x8_t = vdupq_n_s16(a); + simd_extract(vqrshrun_n_s16::(a), 0) +} + +/// Signed saturating rounded shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshruns_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshruns_n_s32(a: i32) -> u16 { + static_assert!(N : i32 where N >= 1 && N <= 16); + let a: int32x4_t = vdupq_n_s32(a); + simd_extract(vqrshrun_n_s32::(a), 0) +} + +/// Signed saturating rounded shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrund_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrund_n_s64(a: i64) -> u32 { + static_assert!(N : i32 where N >= 1 && N <= 32); + let a: int64x2_t = vdupq_n_s64(a); + simd_extract(vqrshrun_n_s64::(a), 0) +} + +/// Signed saturating rounded shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrun_high_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrun2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrun_high_n_s16(a: uint8x8_t, b: int16x8_t) -> uint8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shuffle16!(a, vqrshrun_n_s16::(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Signed saturating rounded shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrun_high_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrun2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrun_high_n_s32(a: uint16x4_t, b: int32x4_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shuffle8!(a, vqrshrun_n_s32::(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Signed saturating rounded shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrun_high_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrun2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrun_high_n_s64(a: uint32x2_t, b: int64x2_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shuffle4!(a, vqrshrun_n_s64::(b), [0, 1, 2, 3]) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshld_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshld_s64(a: i64, b: i64) -> i64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshl.i64")] + fn vqshld_s64_(a: i64, b: i64) -> i64; + } + vqshld_s64_(a, b) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlb_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshlb_s8(a: i8, b: i8) -> i8 { + let c: int8x8_t = vqshl_s8(vdup_n_s8(a), vdup_n_s8(b)); + simd_extract(c, 0) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlh_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshlh_s16(a: i16, b: i16) -> i16 { + let c: int16x4_t = vqshl_s16(vdup_n_s16(a), vdup_n_s16(b)); + simd_extract(c, 0) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshls_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshls_s32(a: i32, b: i32) -> i32 { + let c: int32x2_t = vqshl_s32(vdup_n_s32(a), vdup_n_s32(b)); + simd_extract(c, 0) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshld_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshld_u64(a: u64, b: i64) -> u64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshl.i64")] + fn vqshld_u64_(a: u64, b: i64) -> u64; + } + vqshld_u64_(a, b) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlb_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshlb_u8(a: u8, b: i8) -> u8 { + let c: uint8x8_t = vqshl_u8(vdup_n_u8(a), vdup_n_s8(b)); + simd_extract(c, 0) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlh_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshlh_u16(a: u16, b: i16) -> u16 { + let c: uint16x4_t = vqshl_u16(vdup_n_u16(a), vdup_n_s16(b)); + simd_extract(c, 0) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshls_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshls_u32(a: u32, b: i32) -> u32 { + let c: uint32x2_t = vqshl_u32(vdup_n_u32(a), vdup_n_s32(b)); + simd_extract(c, 0) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlb_n_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshlb_n_s8(a: i8) -> i8 { + static_assert_imm3!(N); + simd_extract(vqshl_n_s8::(vdup_n_s8(a)), 0) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlh_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshlh_n_s16(a: i16) -> i16 { + static_assert_imm4!(N); + simd_extract(vqshl_n_s16::(vdup_n_s16(a)), 0) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshls_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshls_n_s32(a: i32) -> i32 { + static_assert_imm5!(N); + simd_extract(vqshl_n_s32::(vdup_n_s32(a)), 0) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshld_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshld_n_s64(a: i64) -> i64 { + static_assert_imm6!(N); + simd_extract(vqshl_n_s64::(vdup_n_s64(a)), 0) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlb_n_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshlb_n_u8(a: u8) -> u8 { + static_assert_imm3!(N); + simd_extract(vqshl_n_u8::(vdup_n_u8(a)), 0) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlh_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshlh_n_u16(a: u16) -> u16 { + static_assert_imm4!(N); + simd_extract(vqshl_n_u16::(vdup_n_u16(a)), 0) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshls_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshls_n_u32(a: u32) -> u32 { + static_assert_imm5!(N); + simd_extract(vqshl_n_u32::(vdup_n_u32(a)), 0) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshld_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshld_n_u64(a: u64) -> u64 { + static_assert_imm6!(N); + simd_extract(vqshl_n_u64::(vdup_n_u64(a)), 0) +} + +/// Signed saturating shift left unsigned +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlub_n_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshlu, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshlub_n_s8(a: i8) -> u8 { + static_assert_imm3!(N); + simd_extract(vqshlu_n_s8::(vdup_n_s8(a)), 0) +} + +/// Signed saturating shift left unsigned +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshluh_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshlu, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshluh_n_s16(a: i16) -> u16 { + static_assert_imm4!(N); + simd_extract(vqshlu_n_s16::(vdup_n_s16(a)), 0) +} + +/// Signed saturating shift left unsigned +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlus_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshlu, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshlus_n_s32(a: i32) -> u32 { + static_assert_imm5!(N); + simd_extract(vqshlu_n_s32::(vdup_n_s32(a)), 0) +} + +/// Signed saturating shift left unsigned +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlud_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshlu, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshlud_n_s64(a: i64) -> u64 { + static_assert_imm6!(N); + simd_extract(vqshlu_n_s64::(vdup_n_s64(a)), 0) +} + +/// Signed saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrnd_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrnd_n_s64(a: i64) -> i32 { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshrn.i32")] + fn vqshrnd_n_s64_(a: i64, n: i32) -> i32; + } + vqshrnd_n_s64_(a, N) +} + +/// Signed saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrnh_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrnh_n_s16(a: i16) -> i8 { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_extract(vqshrn_n_s16::(vdupq_n_s16(a)), 0) +} + +/// Signed saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrns_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrns_n_s32(a: i32) -> i16 { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_extract(vqshrn_n_s32::(vdupq_n_s32(a)), 0) +} + +/// Signed saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_high_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrn_high_n_s16(a: int8x8_t, b: int16x8_t) -> int8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shuffle16!(a, vqshrn_n_s16::(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Signed saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_high_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrn_high_n_s32(a: int16x4_t, b: int32x4_t) -> int16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shuffle8!(a, vqshrn_n_s32::(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Signed saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_high_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrn_high_n_s64(a: int32x2_t, b: int64x2_t) -> int32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shuffle4!(a, vqshrn_n_s64::(b), [0, 1, 2, 3]) +} + +/// Unsigned saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrnd_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrnd_n_u64(a: u64) -> u32 { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshrn.i32")] + fn vqshrnd_n_u64_(a: u64, n: i32) -> u32; + } + vqshrnd_n_u64_(a, N) +} + +/// Unsigned saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrnh_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrnh_n_u16(a: u16) -> u8 { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_extract(vqshrn_n_u16::(vdupq_n_u16(a)), 0) +} + +/// Unsigned saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrns_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrns_n_u32(a: u32) -> u16 { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_extract(vqshrn_n_u32::(vdupq_n_u32(a)), 0) +} + +/// Unsigned saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_high_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrn_high_n_u16(a: uint8x8_t, b: uint16x8_t) -> uint8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shuffle16!(a, vqshrn_n_u16::(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Unsigned saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_high_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrn_high_n_u32(a: uint16x4_t, b: uint32x4_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shuffle8!(a, vqshrn_n_u32::(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Unsigned saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_high_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrn_high_n_u64(a: uint32x2_t, b: uint64x2_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shuffle4!(a, vqshrn_n_u64::(b), [0, 1, 2, 3]) +} + +/// Signed saturating shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrunh_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrunh_n_s16(a: i16) -> u8 { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_extract(vqshrun_n_s16::(vdupq_n_s16(a)), 0) +} + +/// Signed saturating shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshruns_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshruns_n_s32(a: i32) -> u16 { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_extract(vqshrun_n_s32::(vdupq_n_s32(a)), 0) +} + +/// Signed saturating shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrund_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrund_n_s64(a: i64) -> u32 { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_extract(vqshrun_n_s64::(vdupq_n_s64(a)), 0) +} + +/// Signed saturating shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrun_high_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrun2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrun_high_n_s16(a: uint8x8_t, b: int16x8_t) -> uint8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shuffle16!(a, vqshrun_n_s16::(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Signed saturating shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrun_high_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrun2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrun_high_n_s32(a: uint16x4_t, b: int32x4_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shuffle8!(a, vqshrun_n_s32::(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Signed saturating shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrun_high_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrun2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrun_high_n_s64(a: uint32x2_t, b: int64x2_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shuffle4!(a, vqshrun_n_s64::(b), [0, 1, 2, 3]) +} + +/// Unsigned saturating accumulate of signed value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqaddb_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsqaddb_u8(a: u8, b: i8) -> u8 { + simd_extract(vsqadd_u8(vdup_n_u8(a), vdup_n_s8(b)), 0) +} + +/// Unsigned saturating accumulate of signed value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqaddh_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsqaddh_u16(a: u16, b: i16) -> u16 { + simd_extract(vsqadd_u16(vdup_n_u16(a), vdup_n_s16(b)), 0) +} + +/// Unsigned saturating accumulate of signed value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqadds_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsqadds_u32(a: u32, b: i32) -> u32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.usqadd.i32")] + fn vsqadds_u32_(a: u32, b: i32) -> u32; + } + vsqadds_u32_(a, b) +} + +/// Unsigned saturating accumulate of signed value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqaddd_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsqaddd_u64(a: u64, b: i64) -> u64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.usqadd.i64")] + fn vsqaddd_u64_(a: u64, b: i64) -> u64; + } + vsqaddd_u64_(a, b) +} + +/// Calculates the square root of each lane. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqrt_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fsqrt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsqrt_f32(a: float32x2_t) -> float32x2_t { + simd_fsqrt(a) +} + +/// Calculates the square root of each lane. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqrtq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fsqrt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsqrtq_f32(a: float32x4_t) -> float32x4_t { + simd_fsqrt(a) +} + +/// Calculates the square root of each lane. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqrt_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fsqrt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsqrt_f64(a: float64x1_t) -> float64x1_t { + simd_fsqrt(a) +} + +/// Calculates the square root of each lane. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqrtq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fsqrt))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsqrtq_f64(a: float64x2_t) -> float64x2_t { + simd_fsqrt(a) +} + +/// Reciprocal square-root estimate. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrte_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frsqrte))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrsqrte_f64(a: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frsqrte.v1f64")] + fn vrsqrte_f64_(a: float64x1_t) -> float64x1_t; + } + vrsqrte_f64_(a) +} + +/// Reciprocal square-root estimate. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrteq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frsqrte))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrsqrteq_f64(a: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frsqrte.v2f64")] + fn vrsqrteq_f64_(a: float64x2_t) -> float64x2_t; + } + vrsqrteq_f64_(a) +} + +/// Reciprocal square-root estimate. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrtes_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frsqrte))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrsqrtes_f32(a: f32) -> f32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frsqrte.f32")] + fn vrsqrtes_f32_(a: f32) -> f32; + } + vrsqrtes_f32_(a) +} + +/// Reciprocal square-root estimate. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrted_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frsqrte))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrsqrted_f64(a: f64) -> f64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frsqrte.f64")] + fn vrsqrted_f64_(a: f64) -> f64; + } + vrsqrted_f64_(a) +} + +/// Floating-point reciprocal square root step +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrts_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frsqrts))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrsqrts_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frsqrts.v1f64")] + fn vrsqrts_f64_(a: float64x1_t, b: float64x1_t) -> float64x1_t; + } + vrsqrts_f64_(a, b) +} + +/// Floating-point reciprocal square root step +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrtsq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frsqrts))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrsqrtsq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frsqrts.v2f64")] + fn vrsqrtsq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + } + vrsqrtsq_f64_(a, b) +} + +/// Floating-point reciprocal square root step +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrtss_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frsqrts))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrsqrtss_f32(a: f32, b: f32) -> f32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frsqrts.f32")] + fn vrsqrtss_f32_(a: f32, b: f32) -> f32; + } + vrsqrtss_f32_(a, b) +} + +/// Floating-point reciprocal square root step +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrtsd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frsqrts))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrsqrtsd_f64(a: f64, b: f64) -> f64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frsqrts.f64")] + fn vrsqrtsd_f64_(a: f64, b: f64) -> f64; + } + vrsqrtsd_f64_(a, b) +} + +/// Reciprocal estimate. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpe_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frecpe))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrecpe_f64(a: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frecpe.v1f64")] + fn vrecpe_f64_(a: float64x1_t) -> float64x1_t; + } + vrecpe_f64_(a) +} + +/// Reciprocal estimate. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpeq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frecpe))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrecpeq_f64(a: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frecpe.v2f64")] + fn vrecpeq_f64_(a: float64x2_t) -> float64x2_t; + } + vrecpeq_f64_(a) +} + +/// Reciprocal estimate. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpes_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frecpe))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrecpes_f32(a: f32) -> f32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frecpe.f32")] + fn vrecpes_f32_(a: f32) -> f32; + } + vrecpes_f32_(a) +} + +/// Reciprocal estimate. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecped_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frecpe))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrecped_f64(a: f64) -> f64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frecpe.f64")] + fn vrecped_f64_(a: f64) -> f64; + } + vrecped_f64_(a) +} + +/// Floating-point reciprocal step +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecps_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frecps))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrecps_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frecps.v1f64")] + fn vrecps_f64_(a: float64x1_t, b: float64x1_t) -> float64x1_t; + } + vrecps_f64_(a, b) +} + +/// Floating-point reciprocal step +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpsq_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frecps))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrecpsq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frecps.v2f64")] + fn vrecpsq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + } + vrecpsq_f64_(a, b) +} + +/// Floating-point reciprocal step +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpss_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frecps))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrecpss_f32(a: f32, b: f32) -> f32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frecps.f32")] + fn vrecpss_f32_(a: f32, b: f32) -> f32; + } + vrecpss_f32_(a, b) +} + +/// Floating-point reciprocal step +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpsd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frecps))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrecpsd_f64(a: f64, b: f64) -> f64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frecps.f64")] + fn vrecpsd_f64_(a: f64, b: f64) -> f64; + } + vrecpsd_f64_(a, b) +} + +/// Floating-point reciprocal exponent +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpxs_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frecpx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrecpxs_f32(a: f32) -> f32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frecpx.f32")] + fn vrecpxs_f32_(a: f32) -> f32; + } + vrecpxs_f32_(a) +} + +/// Floating-point reciprocal exponent +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpxd_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frecpx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrecpxd_f64(a: f64) -> f64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frecpx.f64")] + fn vrecpxd_f64_(a: f64) -> f64; + } + vrecpxd_f64_(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_s64_p64(a: poly64x1_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_u64_p64(a: poly64x1_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_p64_s64(a: int64x1_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_p64_u64(a: uint64x1_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_s64_p64(a: poly64x2_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_u64_p64(a: poly64x2_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_p64_s64(a: int64x2_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_p64_u64(a: uint64x2_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_s8_f64(a: float64x1_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_s16_f64(a: float64x1_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_s32_f64(a: float64x1_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_s64_f64(a: float64x1_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_s8_f64(a: float64x2_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_s16_f64(a: float64x2_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_s32_f64(a: float64x2_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_s64_f64(a: float64x2_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_u8_f64(a: float64x1_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_u16_f64(a: float64x1_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_u32_f64(a: float64x1_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_u64_f64(a: float64x1_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_u8_f64(a: float64x2_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_u16_f64(a: float64x2_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_u32_f64(a: float64x2_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_u64_f64(a: float64x2_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_p8_f64(a: float64x1_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_p16_f64(a: float64x1_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_p64_f32(a: float32x2_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_p64_f64(a: float64x1_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_p8_f64(a: float64x2_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_p16_f64(a: float64x2_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_p64_f32(a: float32x4_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_p64_f64(a: float64x2_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_p128_f64(a: float64x2_t) -> p128 { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_f64_s8(a: int8x8_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_f64_s16(a: int16x4_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_f64_s32(a: int32x2_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_f64_s64(a: int64x1_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_f64_s8(a: int8x16_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_f64_s16(a: int16x8_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_f64_s32(a: int32x4_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_f64_s64(a: int64x2_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_f64_p8(a: poly8x8_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_f64_u16(a: uint16x4_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_f64_u32(a: uint32x2_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_f64_u64(a: uint64x1_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_f64_p8(a: poly8x16_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_f64_u16(a: uint16x8_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_f64_u32(a: uint32x4_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_f64_u64(a: uint64x2_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_f64_u8(a: uint8x8_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_f64_p16(a: poly16x4_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_f64_p64(a: poly64x1_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_f32_p64(a: poly64x1_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_f64_u8(a: uint8x16_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_f64_p16(a: poly16x8_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_f64_p64(a: poly64x2_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_f32_p64(a: poly64x2_t) -> float32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_p128) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_f64_p128(a: p128) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_f64_f32(a: float32x2_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpret_f32_f64(a: float64x1_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_f64_f32(a: float32x4_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vreinterpretq_f32_f64(a: float64x2_t) -> float32x4_t { + transmute(a) +} + +/// Signed rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshld_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(srshl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrshld_s64(a: i64, b: i64) -> i64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srshl.i64")] + fn vrshld_s64_(a: i64, b: i64) -> i64; + } + vrshld_s64_(a, b) +} + +/// Unsigned rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshld_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(urshl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrshld_u64(a: u64, b: i64) -> u64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urshl.i64")] + fn vrshld_u64_(a: u64, b: i64) -> u64; + } + vrshld_u64_(a, b) +} + +/// Signed rounding shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrd_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(srshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrshrd_n_s64(a: i64) -> i64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + vrshld_s64(a, -N as i64) +} + +/// Unsigned rounding shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrd_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(urshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrshrd_n_u64(a: u64) -> u64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + vrshld_u64(a, -N as i64) +} + +/// Rounding shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_high_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrshrn_high_n_s16(a: int8x8_t, b: int16x8_t) -> int8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shuffle16!(a, vrshrn_n_s16::(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Rounding shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_high_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrshrn_high_n_s32(a: int16x4_t, b: int32x4_t) -> int16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shuffle8!(a, vrshrn_n_s32::(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Rounding shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_high_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrshrn_high_n_s64(a: int32x2_t, b: int64x2_t) -> int32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shuffle4!(a, vrshrn_n_s64::(b), [0, 1, 2, 3]) +} + +/// Rounding shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_high_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrshrn_high_n_u16(a: uint8x8_t, b: uint16x8_t) -> uint8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shuffle16!(a, vrshrn_n_u16::(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Rounding shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_high_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrshrn_high_n_u32(a: uint16x4_t, b: uint32x4_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shuffle8!(a, vrshrn_n_u32::(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Rounding shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_high_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrshrn_high_n_u64(a: uint32x2_t, b: uint64x2_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shuffle4!(a, vrshrn_n_u64::(b), [0, 1, 2, 3]) +} + +/// Signed rounding shift right and accumulate. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsrad_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(srsra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrsrad_n_s64(a: i64, b: i64) -> i64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + let b: i64 = vrshrd_n_s64::(b); + a.wrapping_add(b) +} + +/// Ungisned rounding shift right and accumulate. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsrad_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ursra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrsrad_n_u64(a: u64, b: u64) -> u64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + let b: u64 = vrshrd_n_u64::(b); + a.wrapping_add(b) +} + +/// Rounding subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_high_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rsubhn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrsubhn_high_s16(a: int8x8_t, b: int16x8_t, c: int16x8_t) -> int8x16_t { + let x: int8x8_t = vrsubhn_s16(b, c); + simd_shuffle16!(a, x, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Rounding subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_high_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rsubhn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrsubhn_high_s32(a: int16x4_t, b: int32x4_t, c: int32x4_t) -> int16x8_t { + let x: int16x4_t = vrsubhn_s32(b, c); + simd_shuffle8!(a, x, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Rounding subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_high_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rsubhn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrsubhn_high_s64(a: int32x2_t, b: int64x2_t, c: int64x2_t) -> int32x4_t { + let x: int32x2_t = vrsubhn_s64(b, c); + simd_shuffle4!(a, x, [0, 1, 2, 3]) +} + +/// Rounding subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_high_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rsubhn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrsubhn_high_u16(a: uint8x8_t, b: uint16x8_t, c: uint16x8_t) -> uint8x16_t { + let x: uint8x8_t = vrsubhn_u16(b, c); + simd_shuffle16!(a, x, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Rounding subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_high_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rsubhn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrsubhn_high_u32(a: uint16x4_t, b: uint32x4_t, c: uint32x4_t) -> uint16x8_t { + let x: uint16x4_t = vrsubhn_u32(b, c); + simd_shuffle8!(a, x, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Rounding subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_high_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rsubhn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrsubhn_high_u64(a: uint32x2_t, b: uint64x2_t, c: uint64x2_t) -> uint32x4_t { + let x: uint32x2_t = vrsubhn_u64(b, c); + simd_shuffle4!(a, x, [0, 1, 2, 3]) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vset_lane_f64(a: f64, b: float64x1_t) -> float64x1_t { + static_assert!(LANE : i32 where LANE == 0); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsetq_lane_f64(a: f64, b: float64x2_t) -> float64x2_t { + static_assert_imm1!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Signed Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshld_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sshl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vshld_s64(a: i64, b: i64) -> i64 { + transmute(vshl_s64(transmute(a), transmute(b))) +} + +/// Unsigned Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshld_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ushl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vshld_u64(a: u64, b: i64) -> u64 { + transmute(vshl_u64(transmute(a), transmute(b))) +} + +/// Signed shift left long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_high_n_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sshll2, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vshll_high_n_s8(a: int8x16_t) -> int16x8_t { + static_assert!(N : i32 where N >= 0 && N <= 8); + let b: int8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + vshll_n_s8::(b) +} + +/// Signed shift left long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_high_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sshll2, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vshll_high_n_s16(a: int16x8_t) -> int32x4_t { + static_assert!(N : i32 where N >= 0 && N <= 16); + let b: int16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + vshll_n_s16::(b) +} + +/// Signed shift left long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_high_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sshll2, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vshll_high_n_s32(a: int32x4_t) -> int64x2_t { + static_assert!(N : i32 where N >= 0 && N <= 32); + let b: int32x2_t = simd_shuffle2!(a, a, [2, 3]); + vshll_n_s32::(b) +} + +/// Signed shift left long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_high_n_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ushll2, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vshll_high_n_u8(a: uint8x16_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 0 && N <= 8); + let b: uint8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + vshll_n_u8::(b) +} + +/// Signed shift left long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_high_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ushll2, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vshll_high_n_u16(a: uint16x8_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 0 && N <= 16); + let b: uint16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + vshll_n_u16::(b) +} + +/// Signed shift left long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_high_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ushll2, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vshll_high_n_u32(a: uint32x4_t) -> uint64x2_t { + static_assert!(N : i32 where N >= 0 && N <= 32); + let b: uint32x2_t = simd_shuffle2!(a, a, [2, 3]); + vshll_n_u32::(b) +} + +/// Shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_high_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(shrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vshrn_high_n_s16(a: int8x8_t, b: int16x8_t) -> int8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shuffle16!(a, vshrn_n_s16::(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_high_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(shrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vshrn_high_n_s32(a: int16x4_t, b: int32x4_t) -> int16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shuffle8!(a, vshrn_n_s32::(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_high_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(shrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vshrn_high_n_s64(a: int32x2_t, b: int64x2_t) -> int32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shuffle4!(a, vshrn_n_s64::(b), [0, 1, 2, 3]) +} + +/// Shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_high_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(shrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vshrn_high_n_u16(a: uint8x8_t, b: uint16x8_t) -> uint8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shuffle16!(a, vshrn_n_u16::(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_high_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(shrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vshrn_high_n_u32(a: uint16x4_t, b: uint32x4_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shuffle8!(a, vshrn_n_u32::(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_high_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(shrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vshrn_high_n_u64(a: uint32x2_t, b: uint64x2_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shuffle4!(a, vshrn_n_u64::(b), [0, 1, 2, 3]) +} + +/// SM3PARTW1 +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsm3partw1q_u32) +#[inline] +#[target_feature(enable = "neon,sm4")] +#[cfg_attr(test, assert_instr(sm3partw1))] +pub unsafe fn vsm3partw1q_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sm3partw1")] + fn vsm3partw1q_u32_(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t; + } + vsm3partw1q_u32_(a, b, c) +} + +/// SM3PARTW2 +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsm3partw2q_u32) +#[inline] +#[target_feature(enable = "neon,sm4")] +#[cfg_attr(test, assert_instr(sm3partw2))] +pub unsafe fn vsm3partw2q_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sm3partw2")] + fn vsm3partw2q_u32_(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t; + } + vsm3partw2q_u32_(a, b, c) +} + +/// SM3SS1 +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsm3ss1q_u32) +#[inline] +#[target_feature(enable = "neon,sm4")] +#[cfg_attr(test, assert_instr(sm3ss1))] +pub unsafe fn vsm3ss1q_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sm3ss1")] + fn vsm3ss1q_u32_(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t; + } + vsm3ss1q_u32_(a, b, c) +} + +/// SM4 key +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsm4ekeyq_u32) +#[inline] +#[target_feature(enable = "neon,sm4")] +#[cfg_attr(test, assert_instr(sm4ekey))] +pub unsafe fn vsm4ekeyq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sm4ekey")] + fn vsm4ekeyq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; + } + vsm4ekeyq_u32_(a, b) +} + +/// SM4 encode +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsm4eq_u32) +#[inline] +#[target_feature(enable = "neon,sm4")] +#[cfg_attr(test, assert_instr(sm4e))] +pub unsafe fn vsm4eq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sm4e")] + fn vsm4eq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; + } + vsm4eq_u32_(a, b) +} + +/// Rotate and exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrax1q_u64) +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(rax1))] +pub unsafe fn vrax1q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.rax1")] + fn vrax1q_u64_(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t; + } + vrax1q_u64_(a, b) +} + +/// SHA512 hash update part 1 +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha512hq_u64) +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(sha512h))] +pub unsafe fn vsha512hq_u64(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sha512h")] + fn vsha512hq_u64_(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t; + } + vsha512hq_u64_(a, b, c) +} + +/// SHA512 hash update part 2 +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha512h2q_u64) +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(sha512h2))] +pub unsafe fn vsha512h2q_u64(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sha512h2")] + fn vsha512h2q_u64_(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t; + } + vsha512h2q_u64_(a, b, c) +} + +/// SHA512 schedule update 0 +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha512su0q_u64) +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(sha512su0))] +pub unsafe fn vsha512su0q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sha512su0")] + fn vsha512su0q_u64_(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t; + } + vsha512su0q_u64_(a, b) +} + +/// SHA512 schedule update 1 +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha512su1q_u64) +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(sha512su1))] +pub unsafe fn vsha512su1q_u64(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sha512su1")] + fn vsha512su1q_u64_(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t; + } + vsha512su1q_u64_(a, b, c) +} + +/// Floating-point round to 32-bit integer, using current rounding mode +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd32x_f32) +#[inline] +#[target_feature(enable = "neon,frintts")] +#[cfg_attr(test, assert_instr(frint32x))] +pub unsafe fn vrnd32x_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frint32x.v2f32")] + fn vrnd32x_f32_(a: float32x2_t) -> float32x2_t; + } + vrnd32x_f32_(a) +} + +/// Floating-point round to 32-bit integer, using current rounding mode +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd32xq_f32) +#[inline] +#[target_feature(enable = "neon,frintts")] +#[cfg_attr(test, assert_instr(frint32x))] +pub unsafe fn vrnd32xq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frint32x.v4f32")] + fn vrnd32xq_f32_(a: float32x4_t) -> float32x4_t; + } + vrnd32xq_f32_(a) +} + +/// Floating-point round to 32-bit integer toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd32z_f32) +#[inline] +#[target_feature(enable = "neon,frintts")] +#[cfg_attr(test, assert_instr(frint32z))] +pub unsafe fn vrnd32z_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frint32z.v2f32")] + fn vrnd32z_f32_(a: float32x2_t) -> float32x2_t; + } + vrnd32z_f32_(a) +} + +/// Floating-point round to 32-bit integer toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd32zq_f32) +#[inline] +#[target_feature(enable = "neon,frintts")] +#[cfg_attr(test, assert_instr(frint32z))] +pub unsafe fn vrnd32zq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frint32z.v4f32")] + fn vrnd32zq_f32_(a: float32x4_t) -> float32x4_t; + } + vrnd32zq_f32_(a) +} + +/// Floating-point round to 64-bit integer, using current rounding mode +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd64x_f32) +#[inline] +#[target_feature(enable = "neon,frintts")] +#[cfg_attr(test, assert_instr(frint64x))] +pub unsafe fn vrnd64x_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frint64x.v2f32")] + fn vrnd64x_f32_(a: float32x2_t) -> float32x2_t; + } + vrnd64x_f32_(a) +} + +/// Floating-point round to 64-bit integer, using current rounding mode +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd64xq_f32) +#[inline] +#[target_feature(enable = "neon,frintts")] +#[cfg_attr(test, assert_instr(frint64x))] +pub unsafe fn vrnd64xq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frint64x.v4f32")] + fn vrnd64xq_f32_(a: float32x4_t) -> float32x4_t; + } + vrnd64xq_f32_(a) +} + +/// Floating-point round to 64-bit integer toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd64z_f32) +#[inline] +#[target_feature(enable = "neon,frintts")] +#[cfg_attr(test, assert_instr(frint64z))] +pub unsafe fn vrnd64z_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frint64z.v2f32")] + fn vrnd64z_f32_(a: float32x2_t) -> float32x2_t; + } + vrnd64z_f32_(a) +} + +/// Floating-point round to 64-bit integer toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd64zq_f32) +#[inline] +#[target_feature(enable = "neon,frintts")] +#[cfg_attr(test, assert_instr(frint64z))] +pub unsafe fn vrnd64zq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frint64z.v4f32")] + fn vrnd64zq_f32_(a: float32x4_t) -> float32x4_t; + } + vrnd64zq_f32_(a) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_shuffle8!(a, b, [0, 8, 2, 10, 4, 12, 6, 14]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1q_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_shuffle16!(a, b, [0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_shuffle4!(a, b, [0, 4, 2, 6]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1q_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_shuffle8!(a, b, [0, 8, 2, 10, 4, 12, 6, 14]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1q_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_shuffle4!(a, b, [0, 4, 2, 6]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_shuffle8!(a, b, [0, 8, 2, 10, 4, 12, 6, 14]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1q_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_shuffle16!(a, b, [0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_shuffle4!(a, b, [0, 4, 2, 6]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1q_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_shuffle8!(a, b, [0, 8, 2, 10, 4, 12, 6, 14]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1q_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_shuffle4!(a, b, [0, 4, 2, 6]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + simd_shuffle8!(a, b, [0, 8, 2, 10, 4, 12, 6, 14]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1q_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + simd_shuffle16!(a, b, [0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + simd_shuffle4!(a, b, [0, 4, 2, 6]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1q_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + simd_shuffle8!(a, b, [0, 8, 2, 10, 4, 12, 6, 14]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1q_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1q_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1q_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + simd_shuffle4!(a, b, [0, 4, 2, 6]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn1q_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_shuffle8!(a, b, [1, 9, 3, 11, 5, 13, 7, 15]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2q_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_shuffle16!(a, b, [1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_shuffle4!(a, b, [1, 5, 3, 7]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2q_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_shuffle8!(a, b, [1, 9, 3, 11, 5, 13, 7, 15]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2q_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_shuffle4!(a, b, [1, 5, 3, 7]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_shuffle8!(a, b, [1, 9, 3, 11, 5, 13, 7, 15]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2q_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_shuffle16!(a, b, [1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_shuffle4!(a, b, [1, 5, 3, 7]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2q_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_shuffle8!(a, b, [1, 9, 3, 11, 5, 13, 7, 15]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2q_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_shuffle4!(a, b, [1, 5, 3, 7]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + simd_shuffle8!(a, b, [1, 9, 3, 11, 5, 13, 7, 15]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2q_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + simd_shuffle16!(a, b, [1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + simd_shuffle4!(a, b, [1, 5, 3, 7]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2q_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + simd_shuffle8!(a, b, [1, 9, 3, 11, 5, 13, 7, 15]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2q_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2q_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2q_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + simd_shuffle4!(a, b, [1, 5, 3, 7]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Transpose vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtrn2q_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_shuffle8!(a, b, [0, 8, 1, 9, 2, 10, 3, 11]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1q_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_shuffle16!(a, b, [0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_shuffle4!(a, b, [0, 4, 1, 5]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1q_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_shuffle8!(a, b, [0, 8, 1, 9, 2, 10, 3, 11]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1q_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_shuffle4!(a, b, [0, 4, 1, 5]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1q_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_shuffle8!(a, b, [0, 8, 1, 9, 2, 10, 3, 11]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1q_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_shuffle16!(a, b, [0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_shuffle4!(a, b, [0, 4, 1, 5]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1q_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_shuffle8!(a, b, [0, 8, 1, 9, 2, 10, 3, 11]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1q_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_shuffle4!(a, b, [0, 4, 1, 5]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + simd_shuffle8!(a, b, [0, 8, 1, 9, 2, 10, 3, 11]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1q_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + simd_shuffle16!(a, b, [0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + simd_shuffle4!(a, b, [0, 4, 1, 5]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1q_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + simd_shuffle8!(a, b, [0, 8, 1, 9, 2, 10, 3, 11]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1q_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1q_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + simd_shuffle4!(a, b, [0, 4, 1, 5]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip1q_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_shuffle8!(a, b, [4, 12, 5, 13, 6, 14, 7, 15]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2q_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_shuffle16!(a, b, [8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_shuffle4!(a, b, [2, 6, 3, 7]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2q_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_shuffle8!(a, b, [4, 12, 5, 13, 6, 14, 7, 15]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2q_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_shuffle4!(a, b, [2, 6, 3, 7]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2q_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_shuffle8!(a, b, [4, 12, 5, 13, 6, 14, 7, 15]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2q_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_shuffle16!(a, b, [8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_shuffle4!(a, b, [2, 6, 3, 7]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2q_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_shuffle8!(a, b, [4, 12, 5, 13, 6, 14, 7, 15]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2q_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_shuffle4!(a, b, [2, 6, 3, 7]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + simd_shuffle8!(a, b, [4, 12, 5, 13, 6, 14, 7, 15]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2q_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + simd_shuffle16!(a, b, [8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + simd_shuffle4!(a, b, [2, 6, 3, 7]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2q_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + simd_shuffle8!(a, b, [4, 12, 5, 13, 6, 14, 7, 15]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2q_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2q_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + simd_shuffle4!(a, b, [2, 6, 3, 7]) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vzip2q_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_shuffle8!(a, b, [0, 2, 4, 6, 8, 10, 12, 14]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1q_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_shuffle16!(a, b, [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_shuffle4!(a, b, [0, 2, 4, 6]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1q_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_shuffle8!(a, b, [0, 2, 4, 6, 8, 10, 12, 14]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1q_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_shuffle4!(a, b, [0, 2, 4, 6]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_shuffle8!(a, b, [0, 2, 4, 6, 8, 10, 12, 14]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1q_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_shuffle16!(a, b, [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_shuffle4!(a, b, [0, 2, 4, 6]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1q_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_shuffle8!(a, b, [0, 2, 4, 6, 8, 10, 12, 14]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1q_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_shuffle4!(a, b, [0, 2, 4, 6]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + simd_shuffle8!(a, b, [0, 2, 4, 6, 8, 10, 12, 14]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1q_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + simd_shuffle16!(a, b, [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + simd_shuffle4!(a, b, [0, 2, 4, 6]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1q_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + simd_shuffle8!(a, b, [0, 2, 4, 6, 8, 10, 12, 14]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1q_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1q_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1q_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + simd_shuffle4!(a, b, [0, 2, 4, 6]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp1q_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_shuffle8!(a, b, [1, 3, 5, 7, 9, 11, 13, 15]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2q_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_shuffle16!(a, b, [1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_shuffle4!(a, b, [1, 3, 5, 7]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2q_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_shuffle8!(a, b, [1, 3, 5, 7, 9, 11, 13, 15]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2q_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_shuffle4!(a, b, [1, 3, 5, 7]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_shuffle8!(a, b, [1, 3, 5, 7, 9, 11, 13, 15]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2q_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_shuffle16!(a, b, [1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_shuffle4!(a, b, [1, 3, 5, 7]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2q_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_shuffle8!(a, b, [1, 3, 5, 7, 9, 11, 13, 15]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2q_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_shuffle4!(a, b, [1, 3, 5, 7]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + simd_shuffle8!(a, b, [1, 3, 5, 7, 9, 11, 13, 15]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2q_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + simd_shuffle16!(a, b, [1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + simd_shuffle4!(a, b, [1, 3, 5, 7]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2q_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + simd_shuffle8!(a, b, [1, 3, 5, 7, 9, 11, 13, 15]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2q_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2q_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2q_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + simd_shuffle4!(a, b, [1, 3, 5, 7]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_f64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuzp2q_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Unsigned Absolute difference and Accumulate Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_high_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uabal))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vabal_high_u8(a: uint16x8_t, b: uint8x16_t, c: uint8x16_t) -> uint16x8_t { + let d: uint8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let e: uint8x8_t = simd_shuffle8!(c, c, [8, 9, 10, 11, 12, 13, 14, 15]); + let f: uint8x8_t = vabd_u8(d, e); + simd_add(a, simd_cast(f)) +} + +/// Unsigned Absolute difference and Accumulate Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_high_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uabal))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vabal_high_u16(a: uint32x4_t, b: uint16x8_t, c: uint16x8_t) -> uint32x4_t { + let d: uint16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let e: uint16x4_t = simd_shuffle4!(c, c, [4, 5, 6, 7]); + let f: uint16x4_t = vabd_u16(d, e); + simd_add(a, simd_cast(f)) +} + +/// Unsigned Absolute difference and Accumulate Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_high_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uabal))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vabal_high_u32(a: uint64x2_t, b: uint32x4_t, c: uint32x4_t) -> uint64x2_t { + let d: uint32x2_t = simd_shuffle2!(b, b, [2, 3]); + let e: uint32x2_t = simd_shuffle2!(c, c, [2, 3]); + let f: uint32x2_t = vabd_u32(d, e); + simd_add(a, simd_cast(f)) +} + +/// Signed Absolute difference and Accumulate Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_high_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sabal))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vabal_high_s8(a: int16x8_t, b: int8x16_t, c: int8x16_t) -> int16x8_t { + let d: int8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let e: int8x8_t = simd_shuffle8!(c, c, [8, 9, 10, 11, 12, 13, 14, 15]); + let f: int8x8_t = vabd_s8(d, e); + let f: uint8x8_t = simd_cast(f); + simd_add(a, simd_cast(f)) +} + +/// Signed Absolute difference and Accumulate Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_high_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sabal))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vabal_high_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t { + let d: int16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let e: int16x4_t = simd_shuffle4!(c, c, [4, 5, 6, 7]); + let f: int16x4_t = vabd_s16(d, e); + let f: uint16x4_t = simd_cast(f); + simd_add(a, simd_cast(f)) +} + +/// Signed Absolute difference and Accumulate Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_high_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sabal))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vabal_high_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t { + let d: int32x2_t = simd_shuffle2!(b, b, [2, 3]); + let e: int32x2_t = simd_shuffle2!(c, c, [2, 3]); + let f: int32x2_t = vabd_s32(d, e); + let f: uint32x2_t = simd_cast(f); + simd_add(a, simd_cast(f)) +} + +/// Singned saturating Absolute value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabs_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqabs))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqabs_s64(a: int64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqabs.v1i64")] + fn vqabs_s64_(a: int64x1_t) -> int64x1_t; + } + vqabs_s64_(a) +} + +/// Singned saturating Absolute value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabsq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqabs))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqabsq_s64(a: int64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqabs.v2i64")] + fn vqabsq_s64_(a: int64x2_t) -> int64x2_t; + } + vqabsq_s64_(a) +} + +/// Signed saturating absolute value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabsb_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqabs))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqabsb_s8(a: i8) -> i8 { + simd_extract(vqabs_s8(vdup_n_s8(a)), 0) +} + +/// Signed saturating absolute value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabsh_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqabs))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqabsh_s16(a: i16) -> i16 { + simd_extract(vqabs_s16(vdup_n_s16(a)), 0) +} + +/// Signed saturating absolute value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabss_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqabs))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqabss_s32(a: i32) -> i32 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqabs.i32")] + fn vqabss_s32_(a: i32) -> i32; + } + vqabss_s32_(a) +} + +/// Signed saturating absolute value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabsd_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqabs))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqabsd_s64(a: i64) -> i64 { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqabs.i64")] + fn vqabsd_s64_(a: i64) -> i64; + } + vqabsd_s64_(a) +} + +/// Shift left and insert +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vslid_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vslid_n_s64(a: i64, b: i64) -> i64 { + static_assert!(N : i32 where N >= 0 && N <= 63); + transmute(vsli_n_s64::(transmute(a), transmute(b))) +} + +/// Shift left and insert +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vslid_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vslid_n_u64(a: u64, b: u64) -> u64 { + static_assert!(N : i32 where N >= 0 && N <= 63); + transmute(vsli_n_u64::(transmute(a), transmute(b))) +} + +/// Shift right and insert +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsrid_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsrid_n_s64(a: i64, b: i64) -> i64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + transmute(vsri_n_s64::(transmute(a), transmute(b))) +} + +/// Shift right and insert +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsrid_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsrid_n_u64(a: u64, b: u64) -> u64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + transmute(vsri_n_u64::(transmute(a), transmute(b))) +} + +#[cfg(test)] +mod test { + use super::*; + use crate::core_arch::simd::*; + use std::mem::transmute; + use stdarch_test::simd_test; + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_veor3q_s8() { + let a: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); + let b: i8x16 = i8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let c: i8x16 = i8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); + let r: i8x16 = transmute(veor3q_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_veor3q_s16() { + let a: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i16x8 = i16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let c: i16x8 = i16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: i16x8 = transmute(veor3q_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_veor3q_s32() { + let a: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); + let b: i32x4 = i32x4::new(0x00, 0x00, 0x00, 0x00); + let c: i32x4 = i32x4::new(0x00, 0x00, 0x00, 0x00); + let e: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); + let r: i32x4 = transmute(veor3q_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_veor3q_s64() { + let a: i64x2 = i64x2::new(0x00, 0x01); + let b: i64x2 = i64x2::new(0x00, 0x00); + let c: i64x2 = i64x2::new(0x00, 0x00); + let e: i64x2 = i64x2::new(0x00, 0x01); + let r: i64x2 = transmute(veor3q_s64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_veor3q_u8() { + let a: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); + let b: u8x16 = u8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let c: u8x16 = u8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); + let r: u8x16 = transmute(veor3q_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_veor3q_u16() { + let a: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u16x8 = u16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let c: u16x8 = u16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: u16x8 = transmute(veor3q_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_veor3q_u32() { + let a: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); + let b: u32x4 = u32x4::new(0x00, 0x00, 0x00, 0x00); + let c: u32x4 = u32x4::new(0x00, 0x00, 0x00, 0x00); + let e: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); + let r: u32x4 = transmute(veor3q_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_veor3q_u64() { + let a: u64x2 = u64x2::new(0x00, 0x01); + let b: u64x2 = u64x2::new(0x00, 0x00); + let c: u64x2 = u64x2::new(0x00, 0x00); + let e: u64x2 = u64x2::new(0x00, 0x01); + let r: u64x2 = transmute(veor3q_u64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabd_f64() { + let a: f64 = 1.0; + let b: f64 = 9.0; + let e: f64 = 8.0; + let r: f64 = transmute(vabd_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdq_f64() { + let a: f64x2 = f64x2::new(1.0, 2.0); + let b: f64x2 = f64x2::new(9.0, 3.0); + let e: f64x2 = f64x2::new(8.0, 1.0); + let r: f64x2 = transmute(vabdq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabds_f32() { + let a: f32 = 1.0; + let b: f32 = 9.0; + let e: f32 = 8.0; + let r: f32 = transmute(vabds_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdd_f64() { + let a: f64 = 1.0; + let b: f64 = 9.0; + let e: f64 = 8.0; + let r: f64 = transmute(vabdd_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_high_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10); + let e: u16x8 = u16x8::new(1, 0, 1, 2, 3, 4, 5, 6); + let r: u16x8 = transmute(vabdl_high_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_high_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 8, 9, 11, 12); + let b: u16x8 = u16x8::new(10, 10, 10, 10, 10, 10, 10, 10); + let e: u32x4 = u32x4::new(2, 1, 1, 2); + let r: u32x4 = transmute(vabdl_high_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_high_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(10, 10, 10, 10); + let e: u64x2 = u64x2::new(7, 6); + let r: u64x2 = transmute(vabdl_high_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_high_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10); + let e: i16x8 = i16x8::new(1, 0, 1, 2, 3, 4, 5, 6); + let r: i16x8 = transmute(vabdl_high_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_high_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 9, 10, 11, 12); + let b: i16x8 = i16x8::new(10, 10, 10, 10, 10, 10, 10, 10); + let e: i32x4 = i32x4::new(1, 0, 1, 2); + let r: i32x4 = transmute(vabdl_high_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_high_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(10, 10, 10, 10); + let e: i64x2 = i64x2::new(7, 6); + let r: i64x2 = transmute(vabdl_high_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_u64() { + let a: u64x1 = u64x1::new(0); + let b: u64x1 = u64x1::new(0); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vceq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u64x1 = u64x1::new(0); + let b: u64x1 = u64x1::new(0); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vceq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_u64() { + let a: u64x2 = u64x2::new(0, 0x01); + let b: u64x2 = u64x2::new(0, 0x01); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vceqq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u64x2 = u64x2::new(0, 0); + let b: u64x2 = u64x2::new(0, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); + let r: u64x2 = transmute(vceqq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_s64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let b: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vceq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i64x1 = i64x1::new(-9223372036854775808); + let b: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vceq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_s64() { + let a: i64x2 = i64x2::new(-9223372036854775808, 0x01); + let b: i64x2 = i64x2::new(-9223372036854775808, 0x01); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vceqq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i64x2 = i64x2::new(-9223372036854775808, -9223372036854775808); + let b: i64x2 = i64x2::new(-9223372036854775808, 0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); + let r: u64x2 = transmute(vceqq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_p64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let b: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vceq_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i64x1 = i64x1::new(-9223372036854775808); + let b: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vceq_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_p64() { + let a: i64x2 = i64x2::new(-9223372036854775808, 0x01); + let b: i64x2 = i64x2::new(-9223372036854775808, 0x01); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vceqq_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i64x2 = i64x2::new(-9223372036854775808, -9223372036854775808); + let b: i64x2 = i64x2::new(-9223372036854775808, 0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); + let r: u64x2 = transmute(vceqq_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_f64() { + let a: f64 = 1.2; + let b: f64 = 1.2; + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vceq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_f64() { + let a: f64x2 = f64x2::new(1.2, 3.4); + let b: f64x2 = f64x2::new(1.2, 3.4); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vceqq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqd_s64() { + let a: i64 = 1; + let b: i64 = 2; + let e: u64 = 0; + let r: u64 = transmute(vceqd_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqd_u64() { + let a: u64 = 1; + let b: u64 = 2; + let e: u64 = 0; + let r: u64 = transmute(vceqd_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqs_f32() { + let a: f32 = 1.; + let b: f32 = 2.; + let e: u32 = 0; + let r: u32 = transmute(vceqs_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqd_f64() { + let a: f64 = 1.; + let b: f64 = 2.; + let e: u64 = 0; + let r: u64 = transmute(vceqd_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_s8() { + let a: i8x8 = i8x8::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let e: u8x8 = u8x8::new(0, 0xFF, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vceqz_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_s8() { + let a: i8x16 = i8x16::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x7F); + let e: u8x16 = u8x16::new(0, 0xFF, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vceqzq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_s16() { + let a: i16x4 = i16x4::new(-32768, 0x00, 0x01, 0x02); + let e: u16x4 = u16x4::new(0, 0xFF_FF, 0, 0); + let r: u16x4 = transmute(vceqz_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_s16() { + let a: i16x8 = i16x8::new(-32768, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let e: u16x8 = u16x8::new(0, 0xFF_FF, 0, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vceqzq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_s32() { + let a: i32x2 = i32x2::new(-2147483648, 0x00); + let e: u32x2 = u32x2::new(0, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vceqz_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_s32() { + let a: i32x4 = i32x4::new(-2147483648, 0x00, 0x01, 0x02); + let e: u32x4 = u32x4::new(0, 0xFF_FF_FF_FF, 0, 0); + let r: u32x4 = transmute(vceqzq_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_s64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vceqz_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_s64() { + let a: i64x2 = i64x2::new(-9223372036854775808, 0x00); + let e: u64x2 = u64x2::new(0, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vceqzq_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_p8() { + let a: i8x8 = i8x8::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let e: u8x8 = u8x8::new(0, 0xFF, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vceqz_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_p8() { + let a: i8x16 = i8x16::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x7F); + let e: u8x16 = u8x16::new(0, 0xFF, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vceqzq_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_p64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vceqz_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_p64() { + let a: i64x2 = i64x2::new(-9223372036854775808, 0x00); + let e: u64x2 = u64x2::new(0, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vceqzq_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_u8() { + let a: u8x8 = u8x8::new(0, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vceqz_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_u8() { + let a: u8x16 = u8x16::new(0, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0xFF); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vceqzq_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_u16() { + let a: u16x4 = u16x4::new(0, 0x00, 0x01, 0x02); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0, 0); + let r: u16x4 = transmute(vceqz_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_u16() { + let a: u16x8 = u16x8::new(0, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vceqzq_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_u32() { + let a: u32x2 = u32x2::new(0, 0x00); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vceqz_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_u32() { + let a: u32x4 = u32x4::new(0, 0x00, 0x01, 0x02); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0, 0); + let r: u32x4 = transmute(vceqzq_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_u64() { + let a: u64x1 = u64x1::new(0); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vceqz_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_u64() { + let a: u64x2 = u64x2::new(0, 0x00); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vceqzq_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_f32() { + let a: f32x2 = f32x2::new(0.0, 1.2); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); + let r: u32x2 = transmute(vceqz_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_f32() { + let a: f32x4 = f32x4::new(0.0, 1.2, 3.4, 5.6); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0, 0, 0); + let r: u32x4 = transmute(vceqzq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_f64() { + let a: f64 = 0.0; + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vceqz_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_f64() { + let a: f64x2 = f64x2::new(0.0, 1.2); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); + let r: u64x2 = transmute(vceqzq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzd_s64() { + let a: i64 = 1; + let e: u64 = 0; + let r: u64 = transmute(vceqzd_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzd_u64() { + let a: u64 = 1; + let e: u64 = 0; + let r: u64 = transmute(vceqzd_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzs_f32() { + let a: f32 = 1.; + let e: u32 = 0; + let r: u32 = transmute(vceqzs_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzd_f64() { + let a: f64 = 1.; + let e: u64 = 0; + let r: u64 = transmute(vceqzd_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtst_s64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let b: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vtst_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstq_s64() { + let a: i64x2 = i64x2::new(-9223372036854775808, 0x00); + let b: i64x2 = i64x2::new(-9223372036854775808, 0x00); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); + let r: u64x2 = transmute(vtstq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtst_p64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let b: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vtst_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstq_p64() { + let a: i64x2 = i64x2::new(-9223372036854775808, 0x00); + let b: i64x2 = i64x2::new(-9223372036854775808, 0x00); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); + let r: u64x2 = transmute(vtstq_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtst_u64() { + let a: u64x1 = u64x1::new(0); + let b: u64x1 = u64x1::new(0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vtst_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstq_u64() { + let a: u64x2 = u64x2::new(0, 0x00); + let b: u64x2 = u64x2::new(0, 0x00); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vtstq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstd_s64() { + let a: i64 = 0; + let b: i64 = 0; + let e: u64 = 0; + let r: u64 = transmute(vtstd_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstd_u64() { + let a: u64 = 0; + let b: u64 = 0; + let e: u64 = 0; + let r: u64 = transmute(vtstd_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuqadds_s32() { + let a: i32 = 1; + let b: u32 = 1; + let e: i32 = 2; + let r: i32 = transmute(vuqadds_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuqaddd_s64() { + let a: i64 = 1; + let b: u64 = 1; + let e: i64 = 2; + let r: i64 = transmute(vuqaddd_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuqaddb_s8() { + let a: i8 = 1; + let b: u8 = 2; + let e: i8 = 3; + let r: i8 = transmute(vuqaddb_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuqaddh_s16() { + let a: i16 = 1; + let b: u16 = 2; + let e: i16 = 3; + let r: i16 = transmute(vuqaddh_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabs_f64() { + let a: f64 = -0.1; + let e: f64 = 0.1; + let r: f64 = transmute(vabs_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabsq_f64() { + let a: f64x2 = f64x2::new(-0.1, -2.2); + let e: f64x2 = f64x2::new(0.1, 2.2); + let r: f64x2 = transmute(vabsq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_s64() { + let a: i64x1 = i64x1::new(1); + let b: i64x1 = i64x1::new(0); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcgt_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_s64() { + let a: i64x2 = i64x2::new(1, 2); + let b: i64x2 = i64x2::new(0, 1); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcgtq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_u64() { + let a: u64x1 = u64x1::new(1); + let b: u64x1 = u64x1::new(0); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcgt_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_u64() { + let a: u64x2 = u64x2::new(1, 2); + let b: u64x2 = u64x2::new(0, 1); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcgtq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_f64() { + let a: f64 = 1.2; + let b: f64 = 0.1; + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcgt_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_f64() { + let a: f64x2 = f64x2::new(1.2, 2.3); + let b: f64x2 = f64x2::new(0.1, 1.2); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcgtq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtd_s64() { + let a: i64 = 1; + let b: i64 = 2; + let e: u64 = 0; + let r: u64 = transmute(vcgtd_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtd_u64() { + let a: u64 = 1; + let b: u64 = 2; + let e: u64 = 0; + let r: u64 = transmute(vcgtd_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgts_f32() { + let a: f32 = 1.; + let b: f32 = 2.; + let e: u32 = 0; + let r: u32 = transmute(vcgts_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtd_f64() { + let a: f64 = 1.; + let b: f64 = 2.; + let e: u64 = 0; + let r: u64 = transmute(vcgtd_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_s64() { + let a: i64x1 = i64x1::new(0); + let b: i64x1 = i64x1::new(1); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vclt_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_s64() { + let a: i64x2 = i64x2::new(0, 1); + let b: i64x2 = i64x2::new(1, 2); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcltq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_u64() { + let a: u64x1 = u64x1::new(0); + let b: u64x1 = u64x1::new(1); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vclt_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_u64() { + let a: u64x2 = u64x2::new(0, 1); + let b: u64x2 = u64x2::new(1, 2); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcltq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_f64() { + let a: f64 = 0.1; + let b: f64 = 1.2; + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vclt_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_f64() { + let a: f64x2 = f64x2::new(0.1, 1.2); + let b: f64x2 = f64x2::new(1.2, 2.3); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcltq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltd_s64() { + let a: i64 = 2; + let b: i64 = 1; + let e: u64 = 0; + let r: u64 = transmute(vcltd_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltd_u64() { + let a: u64 = 2; + let b: u64 = 1; + let e: u64 = 0; + let r: u64 = transmute(vcltd_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclts_f32() { + let a: f32 = 2.; + let b: f32 = 1.; + let e: u32 = 0; + let r: u32 = transmute(vclts_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltd_f64() { + let a: f64 = 2.; + let b: f64 = 1.; + let e: u64 = 0; + let r: u64 = transmute(vcltd_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_s64() { + let a: i64x1 = i64x1::new(0); + let b: i64x1 = i64x1::new(1); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcle_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_s64() { + let a: i64x2 = i64x2::new(0, 1); + let b: i64x2 = i64x2::new(1, 2); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcleq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcged_s64() { + let a: i64 = 1; + let b: i64 = 2; + let e: u64 = 0; + let r: u64 = transmute(vcged_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcged_u64() { + let a: u64 = 1; + let b: u64 = 2; + let e: u64 = 0; + let r: u64 = transmute(vcged_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcges_f32() { + let a: f32 = 1.; + let b: f32 = 2.; + let e: u32 = 0; + let r: u32 = transmute(vcges_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcged_f64() { + let a: f64 = 1.; + let b: f64 = 2.; + let e: u64 = 0; + let r: u64 = transmute(vcged_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_u64() { + let a: u64x1 = u64x1::new(0); + let b: u64x1 = u64x1::new(1); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcle_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_u64() { + let a: u64x2 = u64x2::new(0, 1); + let b: u64x2 = u64x2::new(1, 2); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcleq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_f64() { + let a: f64 = 0.1; + let b: f64 = 1.2; + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcle_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_f64() { + let a: f64x2 = f64x2::new(0.1, 1.2); + let b: f64x2 = f64x2::new(1.2, 2.3); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcleq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcled_s64() { + let a: i64 = 2; + let b: i64 = 1; + let e: u64 = 0; + let r: u64 = transmute(vcled_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcled_u64() { + let a: u64 = 2; + let b: u64 = 1; + let e: u64 = 0; + let r: u64 = transmute(vcled_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcles_f32() { + let a: f32 = 2.; + let b: f32 = 1.; + let e: u32 = 0; + let r: u32 = transmute(vcles_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcled_f64() { + let a: f64 = 2.; + let b: f64 = 1.; + let e: u64 = 0; + let r: u64 = transmute(vcled_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_s64() { + let a: i64x1 = i64x1::new(1); + let b: i64x1 = i64x1::new(0); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcge_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_s64() { + let a: i64x2 = i64x2::new(1, 2); + let b: i64x2 = i64x2::new(0, 1); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcgeq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_u64() { + let a: u64x1 = u64x1::new(1); + let b: u64x1 = u64x1::new(0); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcge_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_u64() { + let a: u64x2 = u64x2::new(1, 2); + let b: u64x2 = u64x2::new(0, 1); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcgeq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_f64() { + let a: f64 = 1.2; + let b: f64 = 0.1; + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcge_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_f64() { + let a: f64x2 = f64x2::new(1.2, 2.3); + let b: f64x2 = f64x2::new(0.1, 1.2); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcgeq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgez_s8() { + let a: i8x8 = i8x8::new(-128, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05); + let e: u8x8 = u8x8::new(0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vcgez_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgezq_s8() { + let a: i8x16 = i8x16::new(-128, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x7F); + let e: u8x16 = u8x16::new(0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vcgezq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgez_s16() { + let a: i16x4 = i16x4::new(-32768, -1, 0x00, 0x01); + let e: u16x4 = u16x4::new(0, 0, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vcgez_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgezq_s16() { + let a: i16x8 = i16x8::new(-32768, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05); + let e: u16x8 = u16x8::new(0, 0, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vcgezq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgez_s32() { + let a: i32x2 = i32x2::new(-2147483648, -1); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vcgez_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgezq_s32() { + let a: i32x4 = i32x4::new(-2147483648, -1, 0x00, 0x01); + let e: u32x4 = u32x4::new(0, 0, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcgezq_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgez_s64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vcgez_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgezq_s64() { + let a: i64x2 = i64x2::new(-9223372036854775808, -1); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vcgezq_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgez_f32() { + let a: f32x2 = f32x2::new(-1.2, 0.0); + let e: u32x2 = u32x2::new(0, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcgez_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgezq_f32() { + let a: f32x4 = f32x4::new(-1.2, 0.0, 1.2, 2.3); + let e: u32x4 = u32x4::new(0, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcgezq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgez_f64() { + let a: f64 = -1.2; + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vcgez_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgezq_f64() { + let a: f64x2 = f64x2::new(-1.2, 0.0); + let e: u64x2 = u64x2::new(0, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcgezq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgezd_s64() { + let a: i64 = -1; + let e: u64 = 0; + let r: u64 = transmute(vcgezd_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgezs_f32() { + let a: f32 = -1.; + let e: u32 = 0; + let r: u32 = transmute(vcgezs_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgezd_f64() { + let a: f64 = -1.; + let e: u64 = 0; + let r: u64 = transmute(vcgezd_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtz_s8() { + let a: i8x8 = i8x8::new(-128, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05); + let e: u8x8 = u8x8::new(0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vcgtz_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtzq_s8() { + let a: i8x16 = i8x16::new(-128, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x7F); + let e: u8x16 = u8x16::new(0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vcgtzq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtz_s16() { + let a: i16x4 = i16x4::new(-32768, -1, 0x00, 0x01); + let e: u16x4 = u16x4::new(0, 0, 0, 0xFF_FF); + let r: u16x4 = transmute(vcgtz_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtzq_s16() { + let a: i16x8 = i16x8::new(-32768, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05); + let e: u16x8 = u16x8::new(0, 0, 0, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vcgtzq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtz_s32() { + let a: i32x2 = i32x2::new(-2147483648, -1); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vcgtz_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtzq_s32() { + let a: i32x4 = i32x4::new(-2147483648, -1, 0x00, 0x01); + let e: u32x4 = u32x4::new(0, 0, 0, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcgtzq_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtz_s64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vcgtz_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtzq_s64() { + let a: i64x2 = i64x2::new(-9223372036854775808, -1); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vcgtzq_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtz_f32() { + let a: f32x2 = f32x2::new(-1.2, 0.0); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vcgtz_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtzq_f32() { + let a: f32x4 = f32x4::new(-1.2, 0.0, 1.2, 2.3); + let e: u32x4 = u32x4::new(0, 0, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcgtzq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtz_f64() { + let a: f64 = -1.2; + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vcgtz_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtzq_f64() { + let a: f64x2 = f64x2::new(-1.2, 0.0); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vcgtzq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtzd_s64() { + let a: i64 = -1; + let e: u64 = 0; + let r: u64 = transmute(vcgtzd_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtzs_f32() { + let a: f32 = -1.; + let e: u32 = 0; + let r: u32 = transmute(vcgtzs_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtzd_f64() { + let a: f64 = -1.; + let e: u64 = 0; + let r: u64 = transmute(vcgtzd_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclez_s8() { + let a: i8x8 = i8x8::new(-128, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vclez_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclezq_s8() { + let a: i8x16 = i8x16::new(-128, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x7F); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vclezq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclez_s16() { + let a: i16x4 = i16x4::new(-32768, -1, 0x00, 0x01); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0); + let r: u16x4 = transmute(vclez_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclezq_s16() { + let a: i16x8 = i16x8::new(-32768, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vclezq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclez_s32() { + let a: i32x2 = i32x2::new(-2147483648, -1); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vclez_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclezq_s32() { + let a: i32x4 = i32x4::new(-2147483648, -1, 0x00, 0x01); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0); + let r: u32x4 = transmute(vclezq_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclez_s64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vclez_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclezq_s64() { + let a: i64x2 = i64x2::new(-9223372036854775808, -1); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vclezq_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclez_f32() { + let a: f32x2 = f32x2::new(-1.2, 0.0); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vclez_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclezq_f32() { + let a: f32x4 = f32x4::new(-1.2, 0.0, 1.2, 2.3); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0, 0); + let r: u32x4 = transmute(vclezq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclez_f64() { + let a: f64 = -1.2; + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vclez_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclezq_f64() { + let a: f64x2 = f64x2::new(-1.2, 0.0); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vclezq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclezd_s64() { + let a: i64 = 2; + let e: u64 = 0; + let r: u64 = transmute(vclezd_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclezs_f32() { + let a: f32 = 2.; + let e: u32 = 0; + let r: u32 = transmute(vclezs_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclezd_f64() { + let a: f64 = 2.; + let e: u64 = 0; + let r: u64 = transmute(vclezd_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltz_s8() { + let a: i8x8 = i8x8::new(-128, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vcltz_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltzq_s8() { + let a: i8x16 = i8x16::new(-128, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x7F); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vcltzq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltz_s16() { + let a: i16x4 = i16x4::new(-32768, -1, 0x00, 0x01); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0, 0); + let r: u16x4 = transmute(vcltz_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltzq_s16() { + let a: i16x8 = i16x8::new(-32768, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vcltzq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltz_s32() { + let a: i32x2 = i32x2::new(-2147483648, -1); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcltz_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltzq_s32() { + let a: i32x4 = i32x4::new(-2147483648, -1, 0x00, 0x01); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0, 0); + let r: u32x4 = transmute(vcltzq_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltz_s64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcltz_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltzq_s64() { + let a: i64x2 = i64x2::new(-9223372036854775808, -1); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcltzq_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltz_f32() { + let a: f32x2 = f32x2::new(-1.2, 0.0); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); + let r: u32x2 = transmute(vcltz_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltzq_f32() { + let a: f32x4 = f32x4::new(-1.2, 0.0, 1.2, 2.3); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0, 0, 0); + let r: u32x4 = transmute(vcltzq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltz_f64() { + let a: f64 = -1.2; + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcltz_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltzq_f64() { + let a: f64x2 = f64x2::new(-1.2, 0.0); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); + let r: u64x2 = transmute(vcltzq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltzd_s64() { + let a: i64 = 2; + let e: u64 = 0; + let r: u64 = transmute(vcltzd_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltzs_f32() { + let a: f32 = 2.; + let e: u32 = 0; + let r: u32 = transmute(vcltzs_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltzd_f64() { + let a: f64 = 2.; + let e: u64 = 0; + let r: u64 = transmute(vcltzd_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcagt_f64() { + let a: f64 = -1.2; + let b: f64 = -1.1; + let e: u64x1 = u64x1::new(!0); + let r: u64x1 = transmute(vcagt_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcagtq_f64() { + let a: f64x2 = f64x2::new(-1.2, 0.0); + let b: f64x2 = f64x2::new(-1.1, 0.0); + let e: u64x2 = u64x2::new(!0, 0); + let r: u64x2 = transmute(vcagtq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcagts_f32() { + let a: f32 = -1.2; + let b: f32 = -1.1; + let e: u32 = !0; + let r: u32 = transmute(vcagts_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcagtd_f64() { + let a: f64 = -1.2; + let b: f64 = -1.1; + let e: u64 = !0; + let r: u64 = transmute(vcagtd_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcage_f64() { + let a: f64 = -1.2; + let b: f64 = -1.1; + let e: u64x1 = u64x1::new(!0); + let r: u64x1 = transmute(vcage_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcageq_f64() { + let a: f64x2 = f64x2::new(-1.2, 0.0); + let b: f64x2 = f64x2::new(-1.1, 0.0); + let e: u64x2 = u64x2::new(!0, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcageq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcages_f32() { + let a: f32 = -1.2; + let b: f32 = -1.1; + let e: u32 = !0; + let r: u32 = transmute(vcages_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcaged_f64() { + let a: f64 = -1.2; + let b: f64 = -1.1; + let e: u64 = !0; + let r: u64 = transmute(vcaged_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcalt_f64() { + let a: f64 = -1.2; + let b: f64 = -1.1; + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vcalt_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcaltq_f64() { + let a: f64x2 = f64x2::new(-1.2, 0.0); + let b: f64x2 = f64x2::new(-1.1, 0.0); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vcaltq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcalts_f32() { + let a: f32 = -1.2; + let b: f32 = -1.1; + let e: u32 = 0; + let r: u32 = transmute(vcalts_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcaltd_f64() { + let a: f64 = -1.2; + let b: f64 = -1.1; + let e: u64 = 0; + let r: u64 = transmute(vcaltd_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcale_f64() { + let a: f64 = -1.2; + let b: f64 = -1.1; + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vcale_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcaleq_f64() { + let a: f64x2 = f64x2::new(-1.2, 0.0); + let b: f64x2 = f64x2::new(-1.1, 0.0); + let e: u64x2 = u64x2::new(0, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcaleq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcales_f32() { + let a: f32 = -1.2; + let b: f32 = -1.1; + let e: u32 = 0; + let r: u32 = transmute(vcales_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcaled_f64() { + let a: f64 = -1.2; + let b: f64 = -1.1; + let e: u64 = 0; + let r: u64 = transmute(vcaled_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(0, 0x7F, 0, 0, 0, 0, 0, 0); + let e: i8x8 = i8x8::new(0x7F, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vcopy_lane_s8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(0, 0x7F, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: i8x16 = i8x16::new(0x7F, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: i8x16 = transmute(vcopyq_laneq_s8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(0, 0x7F_FF, 0, 0); + let e: i16x4 = i16x4::new(0x7F_FF, 2, 3, 4); + let r: i16x4 = transmute(vcopy_lane_s16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(0, 0x7F_FF, 0, 0, 0, 0, 0, 0); + let e: i16x8 = i16x8::new(0x7F_FF, 2, 3, 4, 5, 6, 7, 8); + let r: i16x8 = transmute(vcopyq_laneq_s16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(0, 0x7F_FF_FF_FF); + let e: i32x2 = i32x2::new(0x7F_FF_FF_FF, 2); + let r: i32x2 = transmute(vcopy_lane_s32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(0, 0x7F_FF_FF_FF, 0, 0); + let e: i32x4 = i32x4::new(0x7F_FF_FF_FF, 2, 3, 4); + let r: i32x4 = transmute(vcopyq_laneq_s32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_s64() { + let a: i64x2 = i64x2::new(1, 2); + let b: i64x2 = i64x2::new(0, 0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, 2); + let r: i64x2 = transmute(vcopyq_laneq_s64::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x8 = u8x8::new(0, 0xFF, 0, 0, 0, 0, 0, 0); + let e: u8x8 = u8x8::new(0xFF, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vcopy_lane_u8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(0, 0xFF, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: u8x16 = u8x16::new(0xFF, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: u8x16 = transmute(vcopyq_laneq_u8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(0, 0xFF_FF, 0, 0); + let e: u16x4 = u16x4::new(0xFF_FF, 2, 3, 4); + let r: u16x4 = transmute(vcopy_lane_u16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x8 = u16x8::new(0, 0xFF_FF, 0, 0, 0, 0, 0, 0); + let e: u16x8 = u16x8::new(0xFF_FF, 2, 3, 4, 5, 6, 7, 8); + let r: u16x8 = transmute(vcopyq_laneq_u16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(0, 0xFF_FF_FF_FF); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 2); + let r: u32x2 = transmute(vcopy_lane_u32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(0, 0xFF_FF_FF_FF, 0, 0); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 2, 3, 4); + let r: u32x4 = transmute(vcopyq_laneq_u32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_u64() { + let a: u64x2 = u64x2::new(1, 2); + let b: u64x2 = u64x2::new(0, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 2); + let r: u64x2 = transmute(vcopyq_laneq_u64::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_p8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(0, 0x7F, 0, 0, 0, 0, 0, 0); + let e: i8x8 = i8x8::new(0x7F, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vcopy_lane_p8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_p8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(0, 0x7F, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: i8x16 = i8x16::new(0x7F, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: i8x16 = transmute(vcopyq_laneq_p8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_p16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(0, 0x7F_FF, 0, 0); + let e: i16x4 = i16x4::new(0x7F_FF, 2, 3, 4); + let r: i16x4 = transmute(vcopy_lane_p16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_p16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(0, 0x7F_FF, 0, 0, 0, 0, 0, 0); + let e: i16x8 = i16x8::new(0x7F_FF, 2, 3, 4, 5, 6, 7, 8); + let r: i16x8 = transmute(vcopyq_laneq_p16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_p64() { + let a: i64x2 = i64x2::new(1, 2); + let b: i64x2 = i64x2::new(0, 0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, 2); + let r: i64x2 = transmute(vcopyq_laneq_p64::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let b: f32x2 = f32x2::new(0., 0.5); + let e: f32x2 = f32x2::new(0.5, 2.); + let r: f32x2 = transmute(vcopy_lane_f32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_f32() { + let a: f32x4 = f32x4::new(1., 2., 3., 4.); + let b: f32x4 = f32x4::new(0., 0.5, 0., 0.); + let e: f32x4 = f32x4::new(0.5, 2., 3., 4.); + let r: f32x4 = transmute(vcopyq_laneq_f32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let b: f64x2 = f64x2::new(0., 0.5); + let e: f64x2 = f64x2::new(0.5, 2.); + let r: f64x2 = transmute(vcopyq_laneq_f64::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x16 = i8x16::new(0, 0x7F, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: i8x8 = i8x8::new(0x7F, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vcopy_laneq_s8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x8 = i16x8::new(0, 0x7F_FF, 0, 0, 0, 0, 0, 0); + let e: i16x4 = i16x4::new(0x7F_FF, 2, 3, 4); + let r: i16x4 = transmute(vcopy_laneq_s16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x4 = i32x4::new(0, 0x7F_FF_FF_FF, 0, 0); + let e: i32x2 = i32x2::new(0x7F_FF_FF_FF, 2); + let r: i32x2 = transmute(vcopy_laneq_s32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x16 = u8x16::new(0, 0xFF, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: u8x8 = u8x8::new(0xFF, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vcopy_laneq_u8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x8 = u16x8::new(0, 0xFF_FF, 0, 0, 0, 0, 0, 0); + let e: u16x4 = u16x4::new(0xFF_FF, 2, 3, 4); + let r: u16x4 = transmute(vcopy_laneq_u16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x4 = u32x4::new(0, 0xFF_FF_FF_FF, 0, 0); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 2); + let r: u32x2 = transmute(vcopy_laneq_u32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_p8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x16 = i8x16::new(0, 0x7F, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: i8x8 = i8x8::new(0x7F, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vcopy_laneq_p8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_p16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x8 = i16x8::new(0, 0x7F_FF, 0, 0, 0, 0, 0, 0); + let e: i16x4 = i16x4::new(0x7F_FF, 2, 3, 4); + let r: i16x4 = transmute(vcopy_laneq_p16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let b: f32x4 = f32x4::new(0., 0.5, 0., 0.); + let e: f32x2 = f32x2::new(0.5, 2.); + let r: f32x2 = transmute(vcopy_laneq_f32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x8 = i8x8::new(0, 0x7F, 0, 0, 0, 0, 0, 0); + let e: i8x16 = i8x16::new(0x7F, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: i8x16 = transmute(vcopyq_lane_s8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x4 = i16x4::new(0, 0x7F_FF, 0, 0); + let e: i16x8 = i16x8::new(0x7F_FF, 2, 3, 4, 5, 6, 7, 8); + let r: i16x8 = transmute(vcopyq_lane_s16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x2 = i32x2::new(0, 0x7F_FF_FF_FF); + let e: i32x4 = i32x4::new(0x7F_FF_FF_FF, 2, 3, 4); + let r: i32x4 = transmute(vcopyq_lane_s32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x8 = u8x8::new(0, 0xFF, 0, 0, 0, 0, 0, 0); + let e: u8x16 = u8x16::new(0xFF, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: u8x16 = transmute(vcopyq_lane_u8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x4 = u16x4::new(0, 0xFF_FF, 0, 0); + let e: u16x8 = u16x8::new(0xFF_FF, 2, 3, 4, 5, 6, 7, 8); + let r: u16x8 = transmute(vcopyq_lane_u16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x2 = u32x2::new(0, 0xFF_FF_FF_FF); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 2, 3, 4); + let r: u32x4 = transmute(vcopyq_lane_u32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_p8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x8 = i8x8::new(0, 0x7F, 0, 0, 0, 0, 0, 0); + let e: i8x16 = i8x16::new(0x7F, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: i8x16 = transmute(vcopyq_lane_p8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_p16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x4 = i16x4::new(0, 0x7F_FF, 0, 0); + let e: i16x8 = i16x8::new(0x7F_FF, 2, 3, 4, 5, 6, 7, 8); + let r: i16x8 = transmute(vcopyq_lane_p16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_s64() { + let a: i64x2 = i64x2::new(1, 2); + let b: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: i64x2 = i64x2::new(1, 0x7F_FF_FF_FF_FF_FF_FF_FF); + let r: i64x2 = transmute(vcopyq_lane_s64::<1, 0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_u64() { + let a: u64x2 = u64x2::new(1, 2); + let b: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let e: u64x2 = u64x2::new(1, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcopyq_lane_u64::<1, 0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_p64() { + let a: i64x2 = i64x2::new(1, 2); + let b: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: i64x2 = i64x2::new(1, 0x7F_FF_FF_FF_FF_FF_FF_FF); + let r: i64x2 = transmute(vcopyq_lane_p64::<1, 0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_f32() { + let a: f32x4 = f32x4::new(1., 2., 3., 4.); + let b: f32x2 = f32x2::new(0.5, 0.); + let e: f32x4 = f32x4::new(1., 0.5, 3., 4.); + let r: f32x4 = transmute(vcopyq_lane_f32::<1, 0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let b: f64 = 0.5; + let e: f64x2 = f64x2::new(1., 0.5); + let r: f64x2 = transmute(vcopyq_lane_f64::<1, 0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_f64() { + let a: u64 = 0; + let e: f64 = 0.; + let r: f64 = transmute(vcreate_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_f64_s64() { + let a: i64x1 = i64x1::new(1); + let e: f64 = 1.; + let r: f64 = transmute(vcvt_f64_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_f64_s64() { + let a: i64x2 = i64x2::new(1, 2); + let e: f64x2 = f64x2::new(1., 2.); + let r: f64x2 = transmute(vcvtq_f64_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_f64_u64() { + let a: u64x1 = u64x1::new(1); + let e: f64 = 1.; + let r: f64 = transmute(vcvt_f64_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_f64_u64() { + let a: u64x2 = u64x2::new(1, 2); + let e: f64x2 = f64x2::new(1., 2.); + let r: f64x2 = transmute(vcvtq_f64_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_f64_f32() { + let a: f32x2 = f32x2::new(-1.2, 1.2); + let e: f64x2 = f64x2::new(-1.2f32 as f64, 1.2f32 as f64); + let r: f64x2 = transmute(vcvt_f64_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_high_f64_f32() { + let a: f32x4 = f32x4::new(-1.2, 1.2, 2.3, 3.4); + let e: f64x2 = f64x2::new(2.3f32 as f64, 3.4f32 as f64); + let r: f64x2 = transmute(vcvt_high_f64_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_f32_f64() { + let a: f64x2 = f64x2::new(-1.2, 1.2); + let e: f32x2 = f32x2::new(-1.2f64 as f32, 1.2f64 as f32); + let r: f32x2 = transmute(vcvt_f32_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_high_f32_f64() { + let a: f32x2 = f32x2::new(-1.2, 1.2); + let b: f64x2 = f64x2::new(-2.3, 3.4); + let e: f32x4 = f32x4::new(-1.2, 1.2, -2.3f64 as f32, 3.4f64 as f32); + let r: f32x4 = transmute(vcvt_high_f32_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtx_f32_f64() { + let a: f64x2 = f64x2::new(-1.0, 2.0); + let e: f32x2 = f32x2::new(-1.0, 2.0); + let r: f32x2 = transmute(vcvtx_f32_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtxd_f32_f64() { + let a: f64 = -1.0; + let e: f32 = -1.0; + let r: f32 = transmute(vcvtxd_f32_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtx_high_f32_f64() { + let a: f32x2 = f32x2::new(-1.0, 2.0); + let b: f64x2 = f64x2::new(-3.0, 4.0); + let e: f32x4 = f32x4::new(-1.0, 2.0, -3.0, 4.0); + let r: f32x4 = transmute(vcvtx_high_f32_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_n_f64_s64() { + let a: i64x1 = i64x1::new(1); + let e: f64 = 0.25; + let r: f64 = transmute(vcvt_n_f64_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_n_f64_s64() { + let a: i64x2 = i64x2::new(1, 2); + let e: f64x2 = f64x2::new(0.25, 0.5); + let r: f64x2 = transmute(vcvtq_n_f64_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvts_n_f32_s32() { + let a: i32 = 1; + let e: f32 = 0.25; + let r: f32 = transmute(vcvts_n_f32_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtd_n_f64_s64() { + let a: i64 = 1; + let e: f64 = 0.25; + let r: f64 = transmute(vcvtd_n_f64_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_n_f64_u64() { + let a: u64x1 = u64x1::new(1); + let e: f64 = 0.25; + let r: f64 = transmute(vcvt_n_f64_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_n_f64_u64() { + let a: u64x2 = u64x2::new(1, 2); + let e: f64x2 = f64x2::new(0.25, 0.5); + let r: f64x2 = transmute(vcvtq_n_f64_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvts_n_f32_u32() { + let a: u32 = 1; + let e: f32 = 0.25; + let r: f32 = transmute(vcvts_n_f32_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtd_n_f64_u64() { + let a: u64 = 1; + let e: f64 = 0.25; + let r: f64 = transmute(vcvtd_n_f64_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_n_s64_f64() { + let a: f64 = 0.25; + let e: i64x1 = i64x1::new(1); + let r: i64x1 = transmute(vcvt_n_s64_f64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_n_s64_f64() { + let a: f64x2 = f64x2::new(0.25, 0.5); + let e: i64x2 = i64x2::new(1, 2); + let r: i64x2 = transmute(vcvtq_n_s64_f64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvts_n_s32_f32() { + let a: f32 = 0.25; + let e: i32 = 1; + let r: i32 = transmute(vcvts_n_s32_f32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtd_n_s64_f64() { + let a: f64 = 0.25; + let e: i64 = 1; + let r: i64 = transmute(vcvtd_n_s64_f64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_n_u64_f64() { + let a: f64 = 0.25; + let e: u64x1 = u64x1::new(1); + let r: u64x1 = transmute(vcvt_n_u64_f64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_n_u64_f64() { + let a: f64x2 = f64x2::new(0.25, 0.5); + let e: u64x2 = u64x2::new(1, 2); + let r: u64x2 = transmute(vcvtq_n_u64_f64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvts_n_u32_f32() { + let a: f32 = 0.25; + let e: u32 = 1; + let r: u32 = transmute(vcvts_n_u32_f32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtd_n_u64_f64() { + let a: f64 = 0.25; + let e: u64 = 1; + let r: u64 = transmute(vcvtd_n_u64_f64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvts_f32_s32() { + let a: i32 = 1; + let e: f32 = 1.; + let r: f32 = transmute(vcvts_f32_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtd_f64_s64() { + let a: i64 = 1; + let e: f64 = 1.; + let r: f64 = transmute(vcvtd_f64_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvts_f32_u32() { + let a: u32 = 1; + let e: f32 = 1.; + let r: f32 = transmute(vcvts_f32_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtd_f64_u64() { + let a: u64 = 1; + let e: f64 = 1.; + let r: f64 = transmute(vcvtd_f64_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvts_s32_f32() { + let a: f32 = 1.; + let e: i32 = 1; + let r: i32 = transmute(vcvts_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtd_s64_f64() { + let a: f64 = 1.; + let e: i64 = 1; + let r: i64 = transmute(vcvtd_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvts_u32_f32() { + let a: f32 = 1.; + let e: u32 = 1; + let r: u32 = transmute(vcvts_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtd_u64_f64() { + let a: f64 = 1.; + let e: u64 = 1; + let r: u64 = transmute(vcvtd_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_s64_f64() { + let a: f64 = -1.1; + let e: i64x1 = i64x1::new(-1); + let r: i64x1 = transmute(vcvt_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_s64_f64() { + let a: f64x2 = f64x2::new(-1.1, 2.1); + let e: i64x2 = i64x2::new(-1, 2); + let r: i64x2 = transmute(vcvtq_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_u64_f64() { + let a: f64 = 1.1; + let e: u64x1 = u64x1::new(1); + let r: u64x1 = transmute(vcvt_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_u64_f64() { + let a: f64x2 = f64x2::new(1.1, 2.1); + let e: u64x2 = u64x2::new(1, 2); + let r: u64x2 = transmute(vcvtq_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvta_s32_f32() { + let a: f32x2 = f32x2::new(-1.1, 2.1); + let e: i32x2 = i32x2::new(-1, 2); + let r: i32x2 = transmute(vcvta_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtaq_s32_f32() { + let a: f32x4 = f32x4::new(-1.1, 2.1, -2.9, 3.9); + let e: i32x4 = i32x4::new(-1, 2, -3, 4); + let r: i32x4 = transmute(vcvtaq_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvta_s64_f64() { + let a: f64 = -1.1; + let e: i64x1 = i64x1::new(-1); + let r: i64x1 = transmute(vcvta_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtaq_s64_f64() { + let a: f64x2 = f64x2::new(-1.1, 2.1); + let e: i64x2 = i64x2::new(-1, 2); + let r: i64x2 = transmute(vcvtaq_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtas_s32_f32() { + let a: f32 = 2.9; + let e: i32 = 3; + let r: i32 = transmute(vcvtas_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtad_s64_f64() { + let a: f64 = 2.9; + let e: i64 = 3; + let r: i64 = transmute(vcvtad_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtas_u32_f32() { + let a: f32 = 2.9; + let e: u32 = 3; + let r: u32 = transmute(vcvtas_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtad_u64_f64() { + let a: f64 = 2.9; + let e: u64 = 3; + let r: u64 = transmute(vcvtad_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtn_s32_f32() { + let a: f32x2 = f32x2::new(-1.5, 2.1); + let e: i32x2 = i32x2::new(-2, 2); + let r: i32x2 = transmute(vcvtn_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtnq_s32_f32() { + let a: f32x4 = f32x4::new(-1.5, 2.1, -2.9, 3.9); + let e: i32x4 = i32x4::new(-2, 2, -3, 4); + let r: i32x4 = transmute(vcvtnq_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtn_s64_f64() { + let a: f64 = -1.5; + let e: i64x1 = i64x1::new(-2); + let r: i64x1 = transmute(vcvtn_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtnq_s64_f64() { + let a: f64x2 = f64x2::new(-1.5, 2.1); + let e: i64x2 = i64x2::new(-2, 2); + let r: i64x2 = transmute(vcvtnq_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtns_s32_f32() { + let a: f32 = -1.5; + let e: i32 = -2; + let r: i32 = transmute(vcvtns_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtnd_s64_f64() { + let a: f64 = -1.5; + let e: i64 = -2; + let r: i64 = transmute(vcvtnd_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtm_s32_f32() { + let a: f32x2 = f32x2::new(-1.1, 2.1); + let e: i32x2 = i32x2::new(-2, 2); + let r: i32x2 = transmute(vcvtm_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtmq_s32_f32() { + let a: f32x4 = f32x4::new(-1.1, 2.1, -2.9, 3.9); + let e: i32x4 = i32x4::new(-2, 2, -3, 3); + let r: i32x4 = transmute(vcvtmq_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtm_s64_f64() { + let a: f64 = -1.1; + let e: i64x1 = i64x1::new(-2); + let r: i64x1 = transmute(vcvtm_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtmq_s64_f64() { + let a: f64x2 = f64x2::new(-1.1, 2.1); + let e: i64x2 = i64x2::new(-2, 2); + let r: i64x2 = transmute(vcvtmq_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtms_s32_f32() { + let a: f32 = -1.1; + let e: i32 = -2; + let r: i32 = transmute(vcvtms_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtmd_s64_f64() { + let a: f64 = -1.1; + let e: i64 = -2; + let r: i64 = transmute(vcvtmd_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtp_s32_f32() { + let a: f32x2 = f32x2::new(-1.1, 2.1); + let e: i32x2 = i32x2::new(-1, 3); + let r: i32x2 = transmute(vcvtp_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtpq_s32_f32() { + let a: f32x4 = f32x4::new(-1.1, 2.1, -2.9, 3.9); + let e: i32x4 = i32x4::new(-1, 3, -2, 4); + let r: i32x4 = transmute(vcvtpq_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtp_s64_f64() { + let a: f64 = -1.1; + let e: i64x1 = i64x1::new(-1); + let r: i64x1 = transmute(vcvtp_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtpq_s64_f64() { + let a: f64x2 = f64x2::new(-1.1, 2.1); + let e: i64x2 = i64x2::new(-1, 3); + let r: i64x2 = transmute(vcvtpq_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtps_s32_f32() { + let a: f32 = -1.1; + let e: i32 = -1; + let r: i32 = transmute(vcvtps_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtpd_s64_f64() { + let a: f64 = -1.1; + let e: i64 = -1; + let r: i64 = transmute(vcvtpd_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvta_u32_f32() { + let a: f32x2 = f32x2::new(1.1, 2.1); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vcvta_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtaq_u32_f32() { + let a: f32x4 = f32x4::new(1.1, 2.1, 2.9, 3.9); + let e: u32x4 = u32x4::new(1, 2, 3, 4); + let r: u32x4 = transmute(vcvtaq_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvta_u64_f64() { + let a: f64 = 1.1; + let e: u64x1 = u64x1::new(1); + let r: u64x1 = transmute(vcvta_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtaq_u64_f64() { + let a: f64x2 = f64x2::new(1.1, 2.1); + let e: u64x2 = u64x2::new(1, 2); + let r: u64x2 = transmute(vcvtaq_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtn_u32_f32() { + let a: f32x2 = f32x2::new(1.5, 2.1); + let e: u32x2 = u32x2::new(2, 2); + let r: u32x2 = transmute(vcvtn_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtnq_u32_f32() { + let a: f32x4 = f32x4::new(1.5, 2.1, 2.9, 3.9); + let e: u32x4 = u32x4::new(2, 2, 3, 4); + let r: u32x4 = transmute(vcvtnq_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtn_u64_f64() { + let a: f64 = 1.5; + let e: u64x1 = u64x1::new(2); + let r: u64x1 = transmute(vcvtn_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtnq_u64_f64() { + let a: f64x2 = f64x2::new(1.5, 2.1); + let e: u64x2 = u64x2::new(2, 2); + let r: u64x2 = transmute(vcvtnq_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtns_u32_f32() { + let a: f32 = 1.5; + let e: u32 = 2; + let r: u32 = transmute(vcvtns_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtnd_u64_f64() { + let a: f64 = 1.5; + let e: u64 = 2; + let r: u64 = transmute(vcvtnd_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtm_u32_f32() { + let a: f32x2 = f32x2::new(1.1, 2.1); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vcvtm_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtmq_u32_f32() { + let a: f32x4 = f32x4::new(1.1, 2.1, 2.9, 3.9); + let e: u32x4 = u32x4::new(1, 2, 2, 3); + let r: u32x4 = transmute(vcvtmq_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtm_u64_f64() { + let a: f64 = 1.1; + let e: u64x1 = u64x1::new(1); + let r: u64x1 = transmute(vcvtm_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtmq_u64_f64() { + let a: f64x2 = f64x2::new(1.1, 2.1); + let e: u64x2 = u64x2::new(1, 2); + let r: u64x2 = transmute(vcvtmq_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtms_u32_f32() { + let a: f32 = 1.1; + let e: u32 = 1; + let r: u32 = transmute(vcvtms_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtmd_u64_f64() { + let a: f64 = 1.1; + let e: u64 = 1; + let r: u64 = transmute(vcvtmd_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtp_u32_f32() { + let a: f32x2 = f32x2::new(1.1, 2.1); + let e: u32x2 = u32x2::new(2, 3); + let r: u32x2 = transmute(vcvtp_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtpq_u32_f32() { + let a: f32x4 = f32x4::new(1.1, 2.1, 2.9, 3.9); + let e: u32x4 = u32x4::new(2, 3, 3, 4); + let r: u32x4 = transmute(vcvtpq_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtp_u64_f64() { + let a: f64 = 1.1; + let e: u64x1 = u64x1::new(2); + let r: u64x1 = transmute(vcvtp_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtpq_u64_f64() { + let a: f64x2 = f64x2::new(1.1, 2.1); + let e: u64x2 = u64x2::new(2, 3); + let r: u64x2 = transmute(vcvtpq_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtps_u32_f32() { + let a: f32 = 1.1; + let e: u32 = 2; + let r: u32 = transmute(vcvtps_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtpd_u64_f64() { + let a: f64 = 1.1; + let e: u64 = 2; + let r: u64 = transmute(vcvtpd_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_p64() { + let a: i64x2 = i64x2::new(1, 1); + let e: i64x2 = i64x2::new(1, 1); + let r: i64x2 = transmute(vdupq_laneq_p64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_p64() { + let a: i64x1 = i64x1::new(1); + let e: i64x2 = i64x2::new(1, 1); + let r: i64x2 = transmute(vdupq_lane_p64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_f64() { + let a: f64x2 = f64x2::new(1., 1.); + let e: f64x2 = f64x2::new(1., 1.); + let r: f64x2 = transmute(vdupq_laneq_f64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_f64() { + let a: f64 = 1.; + let e: f64x2 = f64x2::new(1., 1.); + let r: f64x2 = transmute(vdupq_lane_f64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_p64() { + let a: i64x1 = i64x1::new(0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vdup_lane_p64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_f64() { + let a: f64 = 0.; + let e: f64 = 0.; + let r: f64 = transmute(vdup_lane_f64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i64x1 = i64x1::new(1); + let r: i64x1 = transmute(vdup_laneq_p64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_f64() { + let a: f64x2 = f64x2::new(0., 1.); + let e: f64 = 1.; + let r: f64 = transmute(vdup_laneq_f64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupb_lane_s8() { + let a: i8x8 = i8x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: i8 = 1; + let r: i8 = transmute(vdupb_lane_s8::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupb_laneq_s8() { + let a: i8x16 = i8x16::new(1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16); + let e: i8 = 1; + let r: i8 = transmute(vdupb_laneq_s8::<8>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vduph_lane_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 4); + let e: i16 = 1; + let r: i16 = transmute(vduph_lane_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vduph_laneq_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: i16 = 1; + let r: i16 = transmute(vduph_laneq_s16::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdups_lane_s32() { + let a: i32x2 = i32x2::new(1, 1); + let e: i32 = 1; + let r: i32 = transmute(vdups_lane_s32::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdups_laneq_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 4); + let e: i32 = 1; + let r: i32 = transmute(vdups_laneq_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupd_lane_s64() { + let a: i64x1 = i64x1::new(1); + let e: i64 = 1; + let r: i64 = transmute(vdupd_lane_s64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupd_laneq_s64() { + let a: i64x2 = i64x2::new(1, 1); + let e: i64 = 1; + let r: i64 = transmute(vdupd_laneq_s64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupb_lane_u8() { + let a: u8x8 = u8x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: u8 = 1; + let r: u8 = transmute(vdupb_lane_u8::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupb_laneq_u8() { + let a: u8x16 = u8x16::new(1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16); + let e: u8 = 1; + let r: u8 = transmute(vdupb_laneq_u8::<8>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vduph_lane_u16() { + let a: u16x4 = u16x4::new(1, 1, 1, 4); + let e: u16 = 1; + let r: u16 = transmute(vduph_lane_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vduph_laneq_u16() { + let a: u16x8 = u16x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: u16 = 1; + let r: u16 = transmute(vduph_laneq_u16::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdups_lane_u32() { + let a: u32x2 = u32x2::new(1, 1); + let e: u32 = 1; + let r: u32 = transmute(vdups_lane_u32::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdups_laneq_u32() { + let a: u32x4 = u32x4::new(1, 1, 1, 4); + let e: u32 = 1; + let r: u32 = transmute(vdups_laneq_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupd_lane_u64() { + let a: u64x1 = u64x1::new(1); + let e: u64 = 1; + let r: u64 = transmute(vdupd_lane_u64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupd_laneq_u64() { + let a: u64x2 = u64x2::new(1, 1); + let e: u64 = 1; + let r: u64 = transmute(vdupd_laneq_u64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupb_lane_p8() { + let a: i8x8 = i8x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: p8 = 1; + let r: p8 = transmute(vdupb_lane_p8::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupb_laneq_p8() { + let a: i8x16 = i8x16::new(1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16); + let e: p8 = 1; + let r: p8 = transmute(vdupb_laneq_p8::<8>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vduph_lane_p16() { + let a: i16x4 = i16x4::new(1, 1, 1, 4); + let e: p16 = 1; + let r: p16 = transmute(vduph_lane_p16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vduph_laneq_p16() { + let a: i16x8 = i16x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: p16 = 1; + let r: p16 = transmute(vduph_laneq_p16::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdups_lane_f32() { + let a: f32x2 = f32x2::new(1., 1.); + let e: f32 = 1.; + let r: f32 = transmute(vdups_lane_f32::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdups_laneq_f32() { + let a: f32x4 = f32x4::new(1., 1., 1., 4.); + let e: f32 = 1.; + let r: f32 = transmute(vdups_laneq_f32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupd_lane_f64() { + let a: f64 = 1.; + let e: f64 = 1.; + let r: f64 = transmute(vdupd_lane_f64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupd_laneq_f64() { + let a: f64x2 = f64x2::new(1., 1.); + let e: f64 = 1.; + let r: f64 = transmute(vdupd_laneq_f64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_p64() { + let a: i64x2 = i64x2::new(1, 1); + let b: i64x2 = i64x2::new(2, 2); + let e: i64x2 = i64x2::new(1, 2); + let r: i64x2 = transmute(vextq_p64::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_f64() { + let a: f64x2 = f64x2::new(1., 1.); + let b: f64x2 = f64x2::new(2., 2.); + let e: f64x2 = f64x2::new(1., 2.); + let r: f64x2 = transmute(vextq_f64::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_f64() { + let a: f64 = 0.; + let b: f64 = 2.; + let c: f64 = 3.; + let e: f64 = 6.; + let r: f64 = transmute(vmla_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_f64() { + let a: f64x2 = f64x2::new(0., 1.); + let b: f64x2 = f64x2::new(2., 2.); + let c: f64x2 = f64x2::new(3., 3.); + let e: f64x2 = f64x2::new(6., 7.); + let r: f64x2 = transmute(vmlaq_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_s8() { + let a: i16x8 = i16x8::new(8, 7, 6, 5, 4, 3, 2, 1); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let c: i8x16 = i8x16::new(3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7); + let e: i16x8 = i16x8::new(8, 9, 10, 11, 12, 13, 14, 15); + let r: i16x8 = transmute(vmlal_high_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_s16() { + let a: i32x4 = i32x4::new(8, 7, 6, 5); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i16x8 = i16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let e: i32x4 = i32x4::new(8, 9, 10, 11); + let r: i32x4 = transmute(vmlal_high_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_s32() { + let a: i64x2 = i64x2::new(8, 7); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let c: i32x4 = i32x4::new(3, 3, 0, 1); + let e: i64x2 = i64x2::new(8, 9); + let r: i64x2 = transmute(vmlal_high_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_u8() { + let a: u16x8 = u16x8::new(8, 7, 6, 5, 4, 3, 2, 1); + let b: u8x16 = u8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let c: u8x16 = u8x16::new(3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(8, 9, 10, 11, 12, 13, 14, 15); + let r: u16x8 = transmute(vmlal_high_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_u16() { + let a: u32x4 = u32x4::new(8, 7, 6, 5); + let b: u16x8 = u16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u16x8 = u16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let e: u32x4 = u32x4::new(8, 9, 10, 11); + let r: u32x4 = transmute(vmlal_high_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_u32() { + let a: u64x2 = u64x2::new(8, 7); + let b: u32x4 = u32x4::new(2, 2, 2, 2); + let c: u32x4 = u32x4::new(3, 3, 0, 1); + let e: u64x2 = u64x2::new(8, 9); + let r: u64x2 = transmute(vmlal_high_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_n_s16() { + let a: i32x4 = i32x4::new(8, 7, 6, 5); + let b: i16x8 = i16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: i16 = 2; + let e: i32x4 = i32x4::new(8, 9, 10, 11); + let r: i32x4 = transmute(vmlal_high_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_n_s32() { + let a: i64x2 = i64x2::new(8, 7); + let b: i32x4 = i32x4::new(3, 3, 0, 1); + let c: i32 = 2; + let e: i64x2 = i64x2::new(8, 9); + let r: i64x2 = transmute(vmlal_high_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_n_u16() { + let a: u32x4 = u32x4::new(8, 7, 6, 5); + let b: u16x8 = u16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: u16 = 2; + let e: u32x4 = u32x4::new(8, 9, 10, 11); + let r: u32x4 = transmute(vmlal_high_n_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_n_u32() { + let a: u64x2 = u64x2::new(8, 7); + let b: u32x4 = u32x4::new(3, 3, 0, 1); + let c: u32 = 2; + let e: u64x2 = u64x2::new(8, 9); + let r: u64x2 = transmute(vmlal_high_n_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_lane_s16() { + let a: i32x4 = i32x4::new(8, 7, 6, 5); + let b: i16x8 = i16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i32x4 = i32x4::new(8, 9, 10, 11); + let r: i32x4 = transmute(vmlal_high_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_laneq_s16() { + let a: i32x4 = i32x4::new(8, 7, 6, 5); + let b: i16x8 = i16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i32x4 = i32x4::new(8, 9, 10, 11); + let r: i32x4 = transmute(vmlal_high_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_lane_s32() { + let a: i64x2 = i64x2::new(8, 7); + let b: i32x4 = i32x4::new(3, 3, 0, 1); + let c: i32x2 = i32x2::new(0, 2); + let e: i64x2 = i64x2::new(8, 9); + let r: i64x2 = transmute(vmlal_high_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_laneq_s32() { + let a: i64x2 = i64x2::new(8, 7); + let b: i32x4 = i32x4::new(3, 3, 0, 1); + let c: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i64x2 = i64x2::new(8, 9); + let r: i64x2 = transmute(vmlal_high_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_lane_u16() { + let a: u32x4 = u32x4::new(8, 7, 6, 5); + let b: u16x8 = u16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: u16x4 = u16x4::new(0, 2, 0, 0); + let e: u32x4 = u32x4::new(8, 9, 10, 11); + let r: u32x4 = transmute(vmlal_high_lane_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_laneq_u16() { + let a: u32x4 = u32x4::new(8, 7, 6, 5); + let b: u16x8 = u16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: u16x8 = u16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: u32x4 = u32x4::new(8, 9, 10, 11); + let r: u32x4 = transmute(vmlal_high_laneq_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_lane_u32() { + let a: u64x2 = u64x2::new(8, 7); + let b: u32x4 = u32x4::new(3, 3, 0, 1); + let c: u32x2 = u32x2::new(0, 2); + let e: u64x2 = u64x2::new(8, 9); + let r: u64x2 = transmute(vmlal_high_lane_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_laneq_u32() { + let a: u64x2 = u64x2::new(8, 7); + let b: u32x4 = u32x4::new(3, 3, 0, 1); + let c: u32x4 = u32x4::new(0, 2, 0, 0); + let e: u64x2 = u64x2::new(8, 9); + let r: u64x2 = transmute(vmlal_high_laneq_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_f64() { + let a: f64 = 6.; + let b: f64 = 2.; + let c: f64 = 3.; + let e: f64 = 0.; + let r: f64 = transmute(vmls_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_f64() { + let a: f64x2 = f64x2::new(6., 7.); + let b: f64x2 = f64x2::new(2., 2.); + let c: f64x2 = f64x2::new(3., 3.); + let e: f64x2 = f64x2::new(0., 1.); + let r: f64x2 = transmute(vmlsq_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_s8() { + let a: i16x8 = i16x8::new(14, 15, 16, 17, 18, 19, 20, 21); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let c: i8x16 = i8x16::new(3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7); + let e: i16x8 = i16x8::new(14, 13, 12, 11, 10, 9, 8, 7); + let r: i16x8 = transmute(vmlsl_high_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_s16() { + let a: i32x4 = i32x4::new(14, 15, 16, 17); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i16x8 = i16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let e: i32x4 = i32x4::new(14, 13, 12, 11); + let r: i32x4 = transmute(vmlsl_high_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_s32() { + let a: i64x2 = i64x2::new(14, 15); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let c: i32x4 = i32x4::new(3, 3, 0, 1); + let e: i64x2 = i64x2::new(14, 13); + let r: i64x2 = transmute(vmlsl_high_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_u8() { + let a: u16x8 = u16x8::new(14, 15, 16, 17, 18, 19, 20, 21); + let b: u8x16 = u8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let c: u8x16 = u8x16::new(3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(14, 13, 12, 11, 10, 9, 8, 7); + let r: u16x8 = transmute(vmlsl_high_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_u16() { + let a: u32x4 = u32x4::new(14, 15, 16, 17); + let b: u16x8 = u16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u16x8 = u16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let e: u32x4 = u32x4::new(14, 13, 12, 11); + let r: u32x4 = transmute(vmlsl_high_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_u32() { + let a: u64x2 = u64x2::new(14, 15); + let b: u32x4 = u32x4::new(2, 2, 2, 2); + let c: u32x4 = u32x4::new(3, 3, 0, 1); + let e: u64x2 = u64x2::new(14, 13); + let r: u64x2 = transmute(vmlsl_high_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_n_s16() { + let a: i32x4 = i32x4::new(14, 15, 16, 17); + let b: i16x8 = i16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: i16 = 2; + let e: i32x4 = i32x4::new(14, 13, 12, 11); + let r: i32x4 = transmute(vmlsl_high_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_n_s32() { + let a: i64x2 = i64x2::new(14, 15); + let b: i32x4 = i32x4::new(3, 3, 0, 1); + let c: i32 = 2; + let e: i64x2 = i64x2::new(14, 13); + let r: i64x2 = transmute(vmlsl_high_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_n_u16() { + let a: u32x4 = u32x4::new(14, 15, 16, 17); + let b: u16x8 = u16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: u16 = 2; + let e: u32x4 = u32x4::new(14, 13, 12, 11); + let r: u32x4 = transmute(vmlsl_high_n_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_n_u32() { + let a: u64x2 = u64x2::new(14, 15); + let b: u32x4 = u32x4::new(3, 3, 0, 1); + let c: u32 = 2; + let e: u64x2 = u64x2::new(14, 13); + let r: u64x2 = transmute(vmlsl_high_n_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_lane_s16() { + let a: i32x4 = i32x4::new(14, 15, 16, 17); + let b: i16x8 = i16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i32x4 = i32x4::new(14, 13, 12, 11); + let r: i32x4 = transmute(vmlsl_high_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_laneq_s16() { + let a: i32x4 = i32x4::new(14, 15, 16, 17); + let b: i16x8 = i16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i32x4 = i32x4::new(14, 13, 12, 11); + let r: i32x4 = transmute(vmlsl_high_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_lane_s32() { + let a: i64x2 = i64x2::new(14, 15); + let b: i32x4 = i32x4::new(3, 3, 0, 1); + let c: i32x2 = i32x2::new(0, 2); + let e: i64x2 = i64x2::new(14, 13); + let r: i64x2 = transmute(vmlsl_high_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_laneq_s32() { + let a: i64x2 = i64x2::new(14, 15); + let b: i32x4 = i32x4::new(3, 3, 0, 1); + let c: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i64x2 = i64x2::new(14, 13); + let r: i64x2 = transmute(vmlsl_high_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_lane_u16() { + let a: u32x4 = u32x4::new(14, 15, 16, 17); + let b: u16x8 = u16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: u16x4 = u16x4::new(0, 2, 0, 0); + let e: u32x4 = u32x4::new(14, 13, 12, 11); + let r: u32x4 = transmute(vmlsl_high_lane_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_laneq_u16() { + let a: u32x4 = u32x4::new(14, 15, 16, 17); + let b: u16x8 = u16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: u16x8 = u16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: u32x4 = u32x4::new(14, 13, 12, 11); + let r: u32x4 = transmute(vmlsl_high_laneq_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_lane_u32() { + let a: u64x2 = u64x2::new(14, 15); + let b: u32x4 = u32x4::new(3, 3, 0, 1); + let c: u32x2 = u32x2::new(0, 2); + let e: u64x2 = u64x2::new(14, 13); + let r: u64x2 = transmute(vmlsl_high_lane_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_laneq_u32() { + let a: u64x2 = u64x2::new(14, 15); + let b: u32x4 = u32x4::new(3, 3, 0, 1); + let c: u32x4 = u32x4::new(0, 2, 0, 0); + let e: u64x2 = u64x2::new(14, 13); + let r: u64x2 = transmute(vmlsl_high_laneq_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovn_high_s16() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 2, 3, 4, 5); + let b: i16x8 = i16x8::new(2, 3, 4, 5, 12, 13, 14, 15); + let e: i8x16 = i8x16::new(0, 1, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 12, 13, 14, 15); + let r: i8x16 = transmute(vmovn_high_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovn_high_s32() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(2, 3, 4, 5); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 2, 3, 4, 5); + let r: i16x8 = transmute(vmovn_high_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovn_high_s64() { + let a: i32x2 = i32x2::new(0, 1); + let b: i64x2 = i64x2::new(2, 3); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vmovn_high_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovn_high_u16() { + let a: u8x8 = u8x8::new(0, 1, 2, 3, 2, 3, 4, 5); + let b: u16x8 = u16x8::new(2, 3, 4, 5, 12, 13, 14, 15); + let e: u8x16 = u8x16::new(0, 1, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 12, 13, 14, 15); + let r: u8x16 = transmute(vmovn_high_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovn_high_u32() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let b: u32x4 = u32x4::new(2, 3, 4, 5); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 2, 3, 4, 5); + let r: u16x8 = transmute(vmovn_high_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovn_high_u64() { + let a: u32x2 = u32x2::new(0, 1); + let b: u64x2 = u64x2::new(2, 3); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vmovn_high_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vneg_s64() { + let a: i64x1 = i64x1::new(0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vneg_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vnegq_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i64x2 = i64x2::new(0, -1); + let r: i64x2 = transmute(vnegq_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vnegd_s64() { + let a: i64 = 1; + let e: i64 = -1; + let r: i64 = transmute(vnegd_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vneg_f64() { + let a: f64 = 0.; + let e: f64 = 0.; + let r: f64 = transmute(vneg_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vnegq_f64() { + let a: f64x2 = f64x2::new(0., 1.); + let e: f64x2 = f64x2::new(0., -1.); + let r: f64x2 = transmute(vnegq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqneg_s64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let e: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); + let r: i64x1 = transmute(vqneg_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqnegq_s64() { + let a: i64x2 = i64x2::new(-9223372036854775808, 0); + let e: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, 0); + let r: i64x2 = transmute(vqnegq_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqnegb_s8() { + let a: i8 = 1; + let e: i8 = -1; + let r: i8 = transmute(vqnegb_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqnegh_s16() { + let a: i16 = 1; + let e: i16 = -1; + let r: i16 = transmute(vqnegh_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqnegs_s32() { + let a: i32 = 1; + let e: i32 = -1; + let r: i32 = transmute(vqnegs_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqnegd_s64() { + let a: i64 = 1; + let e: i64 = -1; + let r: i64 = transmute(vqnegd_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubb_s8() { + let a: i8 = 42; + let b: i8 = 1; + let e: i8 = 41; + let r: i8 = transmute(vqsubb_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubh_s16() { + let a: i16 = 42; + let b: i16 = 1; + let e: i16 = 41; + let r: i16 = transmute(vqsubh_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubb_u8() { + let a: u8 = 42; + let b: u8 = 1; + let e: u8 = 41; + let r: u8 = transmute(vqsubb_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubh_u16() { + let a: u16 = 42; + let b: u16 = 1; + let e: u16 = 41; + let r: u16 = transmute(vqsubh_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubs_u32() { + let a: u32 = 42; + let b: u32 = 1; + let e: u32 = 41; + let r: u32 = transmute(vqsubs_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubd_u64() { + let a: u64 = 42; + let b: u64 = 1; + let e: u64 = 41; + let r: u64 = transmute(vqsubd_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubs_s32() { + let a: i32 = 42; + let b: i32 = 1; + let e: i32 = 41; + let r: i32 = transmute(vqsubs_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubd_s64() { + let a: i64 = 42; + let b: i64 = 1; + let e: i64 = 41; + let r: i64 = transmute(vqsubd_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrbit_s8() { + let a: i8x8 = i8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let e: i8x8 = i8x8::new(0, 64, 32, 96, 16, 80, 48, 112); + let r: i8x8 = transmute(vrbit_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrbitq_s8() { + let a: i8x16 = i8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let e: i8x16 = i8x16::new(0, 64, 32, 96, 16, 80, 48, 112, 8, 72, 40, 104, 24, 88, 56, 120); + let r: i8x16 = transmute(vrbitq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrbit_u8() { + let a: u8x8 = u8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let e: u8x8 = u8x8::new(0, 64, 32, 96, 16, 80, 48, 112); + let r: u8x8 = transmute(vrbit_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrbitq_u8() { + let a: u8x16 = u8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let e: u8x16 = u8x16::new(0, 64, 32, 96, 16, 80, 48, 112, 8, 72, 40, 104, 24, 88, 56, 120); + let r: u8x16 = transmute(vrbitq_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrbit_p8() { + let a: i8x8 = i8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let e: i8x8 = i8x8::new(0, 64, 32, 96, 16, 80, 48, 112); + let r: i8x8 = transmute(vrbit_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrbitq_p8() { + let a: i8x16 = i8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let e: i8x16 = i8x16::new(0, 64, 32, 96, 16, 80, 48, 112, 8, 72, 40, 104, 24, 88, 56, 120); + let r: i8x16 = transmute(vrbitq_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndx_f32() { + let a: f32x2 = f32x2::new(-1.5, 0.5); + let e: f32x2 = f32x2::new(-2.0, 0.0); + let r: f32x2 = transmute(vrndx_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndxq_f32() { + let a: f32x4 = f32x4::new(-1.5, 0.5, 1.5, 2.5); + let e: f32x4 = f32x4::new(-2.0, 0.0, 2.0, 2.0); + let r: f32x4 = transmute(vrndxq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndx_f64() { + let a: f64 = -1.5; + let e: f64 = -2.0; + let r: f64 = transmute(vrndx_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndxq_f64() { + let a: f64x2 = f64x2::new(-1.5, 0.5); + let e: f64x2 = f64x2::new(-2.0, 0.0); + let r: f64x2 = transmute(vrndxq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrnda_f32() { + let a: f32x2 = f32x2::new(-1.5, 0.5); + let e: f32x2 = f32x2::new(-2.0, 1.0); + let r: f32x2 = transmute(vrnda_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndaq_f32() { + let a: f32x4 = f32x4::new(-1.5, 0.5, 1.5, 2.5); + let e: f32x4 = f32x4::new(-2.0, 1.0, 2.0, 3.0); + let r: f32x4 = transmute(vrndaq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrnda_f64() { + let a: f64 = -1.5; + let e: f64 = -2.0; + let r: f64 = transmute(vrnda_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndaq_f64() { + let a: f64x2 = f64x2::new(-1.5, 0.5); + let e: f64x2 = f64x2::new(-2.0, 1.0); + let r: f64x2 = transmute(vrndaq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndn_f64() { + let a: f64 = -1.5; + let e: f64 = -2.0; + let r: f64 = transmute(vrndn_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndnq_f64() { + let a: f64x2 = f64x2::new(-1.5, 0.5); + let e: f64x2 = f64x2::new(-2.0, 0.0); + let r: f64x2 = transmute(vrndnq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndns_f32() { + let a: f32 = -1.5; + let e: f32 = -2.0; + let r: f32 = transmute(vrndns_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndm_f32() { + let a: f32x2 = f32x2::new(-1.5, 0.5); + let e: f32x2 = f32x2::new(-2.0, 0.0); + let r: f32x2 = transmute(vrndm_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndmq_f32() { + let a: f32x4 = f32x4::new(-1.5, 0.5, 1.5, 2.5); + let e: f32x4 = f32x4::new(-2.0, 0.0, 1.0, 2.0); + let r: f32x4 = transmute(vrndmq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndm_f64() { + let a: f64 = -1.5; + let e: f64 = -2.0; + let r: f64 = transmute(vrndm_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndmq_f64() { + let a: f64x2 = f64x2::new(-1.5, 0.5); + let e: f64x2 = f64x2::new(-2.0, 0.0); + let r: f64x2 = transmute(vrndmq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndp_f32() { + let a: f32x2 = f32x2::new(-1.5, 0.5); + let e: f32x2 = f32x2::new(-1.0, 1.0); + let r: f32x2 = transmute(vrndp_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndpq_f32() { + let a: f32x4 = f32x4::new(-1.5, 0.5, 1.5, 2.5); + let e: f32x4 = f32x4::new(-1.0, 1.0, 2.0, 3.0); + let r: f32x4 = transmute(vrndpq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndp_f64() { + let a: f64 = -1.5; + let e: f64 = -1.0; + let r: f64 = transmute(vrndp_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndpq_f64() { + let a: f64x2 = f64x2::new(-1.5, 0.5); + let e: f64x2 = f64x2::new(-1.0, 1.0); + let r: f64x2 = transmute(vrndpq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrnd_f32() { + let a: f32x2 = f32x2::new(-1.5, 0.5); + let e: f32x2 = f32x2::new(-1.0, 0.0); + let r: f32x2 = transmute(vrnd_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndq_f32() { + let a: f32x4 = f32x4::new(-1.5, 0.5, 1.5, 2.5); + let e: f32x4 = f32x4::new(-1.0, 0.0, 1.0, 2.0); + let r: f32x4 = transmute(vrndq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrnd_f64() { + let a: f64 = -1.5; + let e: f64 = -1.0; + let r: f64 = transmute(vrnd_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndq_f64() { + let a: f64x2 = f64x2::new(-1.5, 0.5); + let e: f64x2 = f64x2::new(-1.0, 0.0); + let r: f64x2 = transmute(vrndq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndi_f32() { + let a: f32x2 = f32x2::new(-1.5, 0.5); + let e: f32x2 = f32x2::new(-2.0, 0.0); + let r: f32x2 = transmute(vrndi_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndiq_f32() { + let a: f32x4 = f32x4::new(-1.5, 0.5, 1.5, 2.5); + let e: f32x4 = f32x4::new(-2.0, 0.0, 2.0, 2.0); + let r: f32x4 = transmute(vrndiq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndi_f64() { + let a: f64 = -1.5; + let e: f64 = -2.0; + let r: f64 = transmute(vrndi_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndiq_f64() { + let a: f64x2 = f64x2::new(-1.5, 0.5); + let e: f64x2 = f64x2::new(-2.0, 0.0); + let r: f64x2 = transmute(vrndiq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddb_s8() { + let a: i8 = 42; + let b: i8 = 1; + let e: i8 = 43; + let r: i8 = transmute(vqaddb_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddh_s16() { + let a: i16 = 42; + let b: i16 = 1; + let e: i16 = 43; + let r: i16 = transmute(vqaddh_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddb_u8() { + let a: u8 = 42; + let b: u8 = 1; + let e: u8 = 43; + let r: u8 = transmute(vqaddb_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddh_u16() { + let a: u16 = 42; + let b: u16 = 1; + let e: u16 = 43; + let r: u16 = transmute(vqaddh_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadds_u32() { + let a: u32 = 42; + let b: u32 = 1; + let e: u32 = 43; + let r: u32 = transmute(vqadds_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddd_u64() { + let a: u64 = 42; + let b: u64 = 1; + let e: u64 = 43; + let r: u64 = transmute(vqaddd_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadds_s32() { + let a: i32 = 42; + let b: i32 = 1; + let e: i32 = 43; + let r: i32 = transmute(vqadds_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddd_s64() { + let a: i64 = 42; + let b: i64 = 1; + let e: i64 = 43; + let r: i64 = transmute(vqaddd_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_f64_x2() { + let a: [f64; 3] = [0., 1., 2.]; + let e: [f64; 2] = [1., 2.]; + let r: [f64; 2] = transmute(vld1_f64_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_f64_x2() { + let a: [f64; 5] = [0., 1., 2., 3., 4.]; + let e: [f64x2; 2] = [f64x2::new(1., 2.), f64x2::new(3., 4.)]; + let r: [f64x2; 2] = transmute(vld1q_f64_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_f64_x3() { + let a: [f64; 4] = [0., 1., 2., 3.]; + let e: [f64; 3] = [1., 2., 3.]; + let r: [f64; 3] = transmute(vld1_f64_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_f64_x3() { + let a: [f64; 7] = [0., 1., 2., 3., 4., 5., 6.]; + let e: [f64x2; 3] = [f64x2::new(1., 2.), f64x2::new(3., 4.), f64x2::new(5., 6.)]; + let r: [f64x2; 3] = transmute(vld1q_f64_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_f64_x4() { + let a: [f64; 5] = [0., 1., 2., 3., 4.]; + let e: [f64; 4] = [1., 2., 3., 4.]; + let r: [f64; 4] = transmute(vld1_f64_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_f64_x4() { + let a: [f64; 9] = [0., 1., 2., 3., 4., 5., 6., 7., 8.]; + let e: [f64x2; 4] = [f64x2::new(1., 2.), f64x2::new(3., 4.), f64x2::new(5., 6.), f64x2::new(7., 8.)]; + let r: [f64x2; 4] = transmute(vld1q_f64_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_s64() { + let a: [i64; 5] = [0, 1, 2, 2, 3]; + let e: [i64x2; 2] = [i64x2::new(1, 2), i64x2::new(2, 3)]; + let r: [i64x2; 2] = transmute(vld2q_s64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_u64() { + let a: [u64; 5] = [0, 1, 2, 2, 3]; + let e: [u64x2; 2] = [u64x2::new(1, 2), u64x2::new(2, 3)]; + let r: [u64x2; 2] = transmute(vld2q_u64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_p64() { + let a: [u64; 5] = [0, 1, 2, 2, 3]; + let e: [i64x2; 2] = [i64x2::new(1, 2), i64x2::new(2, 3)]; + let r: [i64x2; 2] = transmute(vld2q_p64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_f64() { + let a: [f64; 3] = [0., 1., 2.]; + let e: [f64; 2] = [1., 2.]; + let r: [f64; 2] = transmute(vld2_f64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_f64() { + let a: [f64; 5] = [0., 1., 2., 2., 3.]; + let e: [f64x2; 2] = [f64x2::new(1., 2.), f64x2::new(2., 3.)]; + let r: [f64x2; 2] = transmute(vld2q_f64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_dup_s64() { + let a: [i64; 5] = [0, 1, 1, 2, 3]; + let e: [i64x2; 2] = [i64x2::new(1, 1), i64x2::new(1, 1)]; + let r: [i64x2; 2] = transmute(vld2q_dup_s64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_dup_u64() { + let a: [u64; 5] = [0, 1, 1, 2, 3]; + let e: [u64x2; 2] = [u64x2::new(1, 1), u64x2::new(1, 1)]; + let r: [u64x2; 2] = transmute(vld2q_dup_u64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_dup_p64() { + let a: [u64; 5] = [0, 1, 1, 2, 3]; + let e: [i64x2; 2] = [i64x2::new(1, 1), i64x2::new(1, 1)]; + let r: [i64x2; 2] = transmute(vld2q_dup_p64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_dup_f64() { + let a: [f64; 3] = [0., 1., 1.]; + let e: [f64; 2] = [1., 1.]; + let r: [f64; 2] = transmute(vld2_dup_f64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_dup_f64() { + let a: [f64; 5] = [0., 1., 1., 2., 3.]; + let e: [f64x2; 2] = [f64x2::new(1., 1.), f64x2::new(1., 1.)]; + let r: [f64x2; 2] = transmute(vld2q_dup_f64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_lane_s8() { + let a: [i8; 33] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i8x16; 2] = [i8x16::new(0, 2, 2, 14, 2, 16, 17, 18, 2, 20, 21, 22, 23, 24, 25, 26), i8x16::new(11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26)]; + let e: [i8x16; 2] = [i8x16::new(1, 2, 2, 14, 2, 16, 17, 18, 2, 20, 21, 22, 23, 24, 25, 26), i8x16::new(2, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26)]; + let r: [i8x16; 2] = transmute(vld2q_lane_s8::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_lane_s64() { + let a: [i64; 3] = [0, 1, 2]; + let b: [i64x1; 2] = [i64x1::new(0), i64x1::new(2)]; + let e: [i64x1; 2] = [i64x1::new(1), i64x1::new(2)]; + let r: [i64x1; 2] = transmute(vld2_lane_s64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_lane_s64() { + let a: [i64; 5] = [0, 1, 2, 3, 4]; + let b: [i64x2; 2] = [i64x2::new(0, 2), i64x2::new(2, 14)]; + let e: [i64x2; 2] = [i64x2::new(1, 2), i64x2::new(2, 14)]; + let r: [i64x2; 2] = transmute(vld2q_lane_s64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_lane_p64() { + let a: [u64; 3] = [0, 1, 2]; + let b: [i64x1; 2] = [i64x1::new(0), i64x1::new(2)]; + let e: [i64x1; 2] = [i64x1::new(1), i64x1::new(2)]; + let r: [i64x1; 2] = transmute(vld2_lane_p64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_lane_p64() { + let a: [u64; 5] = [0, 1, 2, 3, 4]; + let b: [i64x2; 2] = [i64x2::new(0, 2), i64x2::new(2, 14)]; + let e: [i64x2; 2] = [i64x2::new(1, 2), i64x2::new(2, 14)]; + let r: [i64x2; 2] = transmute(vld2q_lane_p64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_lane_u8() { + let a: [u8; 33] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [u8x16; 2] = [u8x16::new(0, 2, 2, 14, 2, 16, 17, 18, 2, 20, 21, 22, 23, 24, 25, 26), u8x16::new(11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26)]; + let e: [u8x16; 2] = [u8x16::new(1, 2, 2, 14, 2, 16, 17, 18, 2, 20, 21, 22, 23, 24, 25, 26), u8x16::new(2, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26)]; + let r: [u8x16; 2] = transmute(vld2q_lane_u8::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_lane_u64() { + let a: [u64; 3] = [0, 1, 2]; + let b: [u64x1; 2] = [u64x1::new(0), u64x1::new(2)]; + let e: [u64x1; 2] = [u64x1::new(1), u64x1::new(2)]; + let r: [u64x1; 2] = transmute(vld2_lane_u64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_lane_u64() { + let a: [u64; 5] = [0, 1, 2, 3, 4]; + let b: [u64x2; 2] = [u64x2::new(0, 2), u64x2::new(2, 14)]; + let e: [u64x2; 2] = [u64x2::new(1, 2), u64x2::new(2, 14)]; + let r: [u64x2; 2] = transmute(vld2q_lane_u64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_lane_p8() { + let a: [u8; 33] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i8x16; 2] = [i8x16::new(0, 2, 2, 14, 2, 16, 17, 18, 2, 20, 21, 22, 23, 24, 25, 26), i8x16::new(11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26)]; + let e: [i8x16; 2] = [i8x16::new(1, 2, 2, 14, 2, 16, 17, 18, 2, 20, 21, 22, 23, 24, 25, 26), i8x16::new(2, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26)]; + let r: [i8x16; 2] = transmute(vld2q_lane_p8::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_lane_f64() { + let a: [f64; 3] = [0., 1., 2.]; + let b: [f64; 2] = [0., 2.]; + let e: [f64; 2] = [1., 2.]; + let r: [f64; 2] = transmute(vld2_lane_f64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_lane_f64() { + let a: [f64; 5] = [0., 1., 2., 3., 4.]; + let b: [f64x2; 2] = [f64x2::new(0., 2.), f64x2::new(2., 14.)]; + let e: [f64x2; 2] = [f64x2::new(1., 2.), f64x2::new(2., 14.)]; + let r: [f64x2; 2] = transmute(vld2q_lane_f64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_s64() { + let a: [i64; 7] = [0, 1, 2, 2, 2, 4, 4]; + let e: [i64x2; 3] = [i64x2::new(1, 2), i64x2::new(2, 4), i64x2::new(2, 4)]; + let r: [i64x2; 3] = transmute(vld3q_s64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_u64() { + let a: [u64; 7] = [0, 1, 2, 2, 2, 4, 4]; + let e: [u64x2; 3] = [u64x2::new(1, 2), u64x2::new(2, 4), u64x2::new(2, 4)]; + let r: [u64x2; 3] = transmute(vld3q_u64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_p64() { + let a: [u64; 7] = [0, 1, 2, 2, 2, 4, 4]; + let e: [i64x2; 3] = [i64x2::new(1, 2), i64x2::new(2, 4), i64x2::new(2, 4)]; + let r: [i64x2; 3] = transmute(vld3q_p64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_f64() { + let a: [f64; 4] = [0., 1., 2., 2.]; + let e: [f64; 3] = [1., 2., 2.]; + let r: [f64; 3] = transmute(vld3_f64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_f64() { + let a: [f64; 7] = [0., 1., 2., 2., 2., 4., 4.]; + let e: [f64x2; 3] = [f64x2::new(1., 2.), f64x2::new(2., 4.), f64x2::new(2., 4.)]; + let r: [f64x2; 3] = transmute(vld3q_f64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_dup_s64() { + let a: [i64; 7] = [0, 1, 1, 1, 3, 1, 4]; + let e: [i64x2; 3] = [i64x2::new(1, 1), i64x2::new(1, 1), i64x2::new(1, 1)]; + let r: [i64x2; 3] = transmute(vld3q_dup_s64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_dup_u64() { + let a: [u64; 7] = [0, 1, 1, 1, 3, 1, 4]; + let e: [u64x2; 3] = [u64x2::new(1, 1), u64x2::new(1, 1), u64x2::new(1, 1)]; + let r: [u64x2; 3] = transmute(vld3q_dup_u64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_dup_p64() { + let a: [u64; 7] = [0, 1, 1, 1, 3, 1, 4]; + let e: [i64x2; 3] = [i64x2::new(1, 1), i64x2::new(1, 1), i64x2::new(1, 1)]; + let r: [i64x2; 3] = transmute(vld3q_dup_p64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_dup_f64() { + let a: [f64; 4] = [0., 1., 1., 1.]; + let e: [f64; 3] = [1., 1., 1.]; + let r: [f64; 3] = transmute(vld3_dup_f64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_dup_f64() { + let a: [f64; 7] = [0., 1., 1., 1., 3., 1., 4.]; + let e: [f64x2; 3] = [f64x2::new(1., 1.), f64x2::new(1., 1.), f64x2::new(1., 1.)]; + let r: [f64x2; 3] = transmute(vld3q_dup_f64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_lane_s8() { + let a: [i8; 49] = [0, 1, 2, 2, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i8x16; 3] = [i8x16::new(0, 2, 2, 14, 2, 16, 17, 18, 2, 20, 21, 22, 23, 24, 25, 26), i8x16::new(11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26), i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8)]; + let e: [i8x16; 3] = [i8x16::new(1, 2, 2, 14, 2, 16, 17, 18, 2, 20, 21, 22, 23, 24, 25, 26), i8x16::new(2, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26), i8x16::new(2, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8)]; + let r: [i8x16; 3] = transmute(vld3q_lane_s8::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_lane_s64() { + let a: [i64; 4] = [0, 1, 2, 2]; + let b: [i64x1; 3] = [i64x1::new(0), i64x1::new(2), i64x1::new(2)]; + let e: [i64x1; 3] = [i64x1::new(1), i64x1::new(2), i64x1::new(2)]; + let r: [i64x1; 3] = transmute(vld3_lane_s64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_lane_s64() { + let a: [i64; 7] = [0, 1, 2, 2, 4, 5, 6]; + let b: [i64x2; 3] = [i64x2::new(0, 2), i64x2::new(2, 14), i64x2::new(2, 16)]; + let e: [i64x2; 3] = [i64x2::new(1, 2), i64x2::new(2, 14), i64x2::new(2, 16)]; + let r: [i64x2; 3] = transmute(vld3q_lane_s64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_lane_p64() { + let a: [u64; 4] = [0, 1, 2, 2]; + let b: [i64x1; 3] = [i64x1::new(0), i64x1::new(2), i64x1::new(2)]; + let e: [i64x1; 3] = [i64x1::new(1), i64x1::new(2), i64x1::new(2)]; + let r: [i64x1; 3] = transmute(vld3_lane_p64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_lane_p64() { + let a: [u64; 7] = [0, 1, 2, 2, 4, 5, 6]; + let b: [i64x2; 3] = [i64x2::new(0, 2), i64x2::new(2, 14), i64x2::new(2, 16)]; + let e: [i64x2; 3] = [i64x2::new(1, 2), i64x2::new(2, 14), i64x2::new(2, 16)]; + let r: [i64x2; 3] = transmute(vld3q_lane_p64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_lane_p8() { + let a: [u8; 49] = [0, 1, 2, 2, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i8x16; 3] = [i8x16::new(0, 2, 2, 14, 2, 16, 17, 18, 2, 20, 21, 22, 23, 24, 25, 26), i8x16::new(11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26), i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8)]; + let e: [i8x16; 3] = [i8x16::new(1, 2, 2, 14, 2, 16, 17, 18, 2, 20, 21, 22, 23, 24, 25, 26), i8x16::new(2, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26), i8x16::new(2, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8)]; + let r: [i8x16; 3] = transmute(vld3q_lane_p8::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_lane_u8() { + let a: [u8; 49] = [0, 1, 2, 2, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [u8x16; 3] = [u8x16::new(0, 2, 2, 14, 2, 16, 17, 18, 2, 20, 21, 22, 23, 24, 25, 26), u8x16::new(11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26), u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8)]; + let e: [u8x16; 3] = [u8x16::new(1, 2, 2, 14, 2, 16, 17, 18, 2, 20, 21, 22, 23, 24, 25, 26), u8x16::new(2, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26), u8x16::new(2, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8)]; + let r: [u8x16; 3] = transmute(vld3q_lane_u8::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_lane_u64() { + let a: [u64; 4] = [0, 1, 2, 2]; + let b: [u64x1; 3] = [u64x1::new(0), u64x1::new(2), u64x1::new(2)]; + let e: [u64x1; 3] = [u64x1::new(1), u64x1::new(2), u64x1::new(2)]; + let r: [u64x1; 3] = transmute(vld3_lane_u64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_lane_u64() { + let a: [u64; 7] = [0, 1, 2, 2, 4, 5, 6]; + let b: [u64x2; 3] = [u64x2::new(0, 2), u64x2::new(2, 14), u64x2::new(2, 16)]; + let e: [u64x2; 3] = [u64x2::new(1, 2), u64x2::new(2, 14), u64x2::new(2, 16)]; + let r: [u64x2; 3] = transmute(vld3q_lane_u64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_lane_f64() { + let a: [f64; 4] = [0., 1., 2., 2.]; + let b: [f64; 3] = [0., 2., 2.]; + let e: [f64; 3] = [1., 2., 2.]; + let r: [f64; 3] = transmute(vld3_lane_f64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_lane_f64() { + let a: [f64; 7] = [0., 1., 2., 2., 4., 5., 6.]; + let b: [f64x2; 3] = [f64x2::new(0., 2.), f64x2::new(2., 14.), f64x2::new(9., 16.)]; + let e: [f64x2; 3] = [f64x2::new(1., 2.), f64x2::new(2., 14.), f64x2::new(2., 16.)]; + let r: [f64x2; 3] = transmute(vld3q_lane_f64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_s64() { + let a: [i64; 9] = [0, 1, 2, 2, 6, 2, 6, 6, 8]; + let e: [i64x2; 4] = [i64x2::new(1, 2), i64x2::new(2, 6), i64x2::new(2, 6), i64x2::new(6, 8)]; + let r: [i64x2; 4] = transmute(vld4q_s64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_u64() { + let a: [u64; 9] = [0, 1, 2, 2, 6, 2, 6, 6, 8]; + let e: [u64x2; 4] = [u64x2::new(1, 2), u64x2::new(2, 6), u64x2::new(2, 6), u64x2::new(6, 8)]; + let r: [u64x2; 4] = transmute(vld4q_u64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_p64() { + let a: [u64; 9] = [0, 1, 2, 2, 6, 2, 6, 6, 8]; + let e: [i64x2; 4] = [i64x2::new(1, 2), i64x2::new(2, 6), i64x2::new(2, 6), i64x2::new(6, 8)]; + let r: [i64x2; 4] = transmute(vld4q_p64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_f64() { + let a: [f64; 5] = [0., 1., 2., 2., 6.]; + let e: [f64; 4] = [1., 2., 2., 6.]; + let r: [f64; 4] = transmute(vld4_f64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_f64() { + let a: [f64; 9] = [0., 1., 2., 2., 6., 2., 6., 6., 8.]; + let e: [f64x2; 4] = [f64x2::new(1., 2.), f64x2::new(2., 6.), f64x2::new(2., 6.), f64x2::new(6., 8.)]; + let r: [f64x2; 4] = transmute(vld4q_f64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_dup_s64() { + let a: [i64; 9] = [0, 1, 1, 1, 1, 2, 4, 3, 5]; + let e: [i64x2; 4] = [i64x2::new(1, 1), i64x2::new(1, 1), i64x2::new(1, 1), i64x2::new(1, 1)]; + let r: [i64x2; 4] = transmute(vld4q_dup_s64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_dup_u64() { + let a: [u64; 9] = [0, 1, 1, 1, 1, 2, 4, 3, 5]; + let e: [u64x2; 4] = [u64x2::new(1, 1), u64x2::new(1, 1), u64x2::new(1, 1), u64x2::new(1, 1)]; + let r: [u64x2; 4] = transmute(vld4q_dup_u64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_dup_p64() { + let a: [u64; 9] = [0, 1, 1, 1, 1, 2, 4, 3, 5]; + let e: [i64x2; 4] = [i64x2::new(1, 1), i64x2::new(1, 1), i64x2::new(1, 1), i64x2::new(1, 1)]; + let r: [i64x2; 4] = transmute(vld4q_dup_p64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_dup_f64() { + let a: [f64; 5] = [0., 1., 1., 1., 1.]; + let e: [f64; 4] = [1., 1., 1., 1.]; + let r: [f64; 4] = transmute(vld4_dup_f64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_dup_f64() { + let a: [f64; 9] = [0., 1., 1., 1., 1., 6., 4., 3., 5.]; + let e: [f64x2; 4] = [f64x2::new(1., 1.), f64x2::new(1., 1.), f64x2::new(1., 1.), f64x2::new(1., 1.)]; + let r: [f64x2; 4] = transmute(vld4q_dup_f64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_lane_s8() { + let a: [i8; 65] = [0, 1, 2, 2, 2, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16]; + let b: [i8x16; 4] = [i8x16::new(0, 2, 2, 2, 2, 16, 2, 18, 2, 20, 21, 22, 2, 24, 25, 26), i8x16::new(11, 12, 13, 14, 15, 16, 2, 18, 2, 20, 21, 22, 23, 24, 25, 26), i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8), i8x16::new(1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16)]; + let e: [i8x16; 4] = [i8x16::new(1, 2, 2, 2, 2, 16, 2, 18, 2, 20, 21, 22, 2, 24, 25, 26), i8x16::new(2, 12, 13, 14, 15, 16, 2, 18, 2, 20, 21, 22, 23, 24, 25, 26), i8x16::new(2, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8), i8x16::new(2, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16)]; + let r: [i8x16; 4] = transmute(vld4q_lane_s8::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_lane_s64() { + let a: [i64; 5] = [0, 1, 2, 2, 2]; + let b: [i64x1; 4] = [i64x1::new(0), i64x1::new(2), i64x1::new(2), i64x1::new(2)]; + let e: [i64x1; 4] = [i64x1::new(1), i64x1::new(2), i64x1::new(2), i64x1::new(2)]; + let r: [i64x1; 4] = transmute(vld4_lane_s64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_lane_s64() { + let a: [i64; 9] = [0, 1, 2, 2, 2, 5, 6, 7, 8]; + let b: [i64x2; 4] = [i64x2::new(0, 2), i64x2::new(2, 2), i64x2::new(2, 16), i64x2::new(2, 18)]; + let e: [i64x2; 4] = [i64x2::new(1, 2), i64x2::new(2, 2), i64x2::new(2, 16), i64x2::new(2, 18)]; + let r: [i64x2; 4] = transmute(vld4q_lane_s64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_lane_p64() { + let a: [u64; 5] = [0, 1, 2, 2, 2]; + let b: [i64x1; 4] = [i64x1::new(0), i64x1::new(2), i64x1::new(2), i64x1::new(2)]; + let e: [i64x1; 4] = [i64x1::new(1), i64x1::new(2), i64x1::new(2), i64x1::new(2)]; + let r: [i64x1; 4] = transmute(vld4_lane_p64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_lane_p64() { + let a: [u64; 9] = [0, 1, 2, 2, 2, 5, 6, 7, 8]; + let b: [i64x2; 4] = [i64x2::new(0, 2), i64x2::new(2, 2), i64x2::new(2, 16), i64x2::new(2, 18)]; + let e: [i64x2; 4] = [i64x2::new(1, 2), i64x2::new(2, 2), i64x2::new(2, 16), i64x2::new(2, 18)]; + let r: [i64x2; 4] = transmute(vld4q_lane_p64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_lane_p8() { + let a: [u8; 65] = [0, 1, 2, 2, 2, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16]; + let b: [i8x16; 4] = [i8x16::new(0, 2, 2, 2, 2, 16, 2, 18, 2, 20, 21, 22, 2, 24, 25, 26), i8x16::new(11, 12, 13, 14, 15, 16, 2, 18, 2, 20, 21, 22, 23, 24, 25, 26), i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8), i8x16::new(1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16)]; + let e: [i8x16; 4] = [i8x16::new(1, 2, 2, 2, 2, 16, 2, 18, 2, 20, 21, 22, 2, 24, 25, 26), i8x16::new(2, 12, 13, 14, 15, 16, 2, 18, 2, 20, 21, 22, 23, 24, 25, 26), i8x16::new(2, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8), i8x16::new(2, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16)]; + let r: [i8x16; 4] = transmute(vld4q_lane_p8::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_lane_u8() { + let a: [u8; 65] = [0, 1, 2, 2, 2, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16]; + let b: [u8x16; 4] = [u8x16::new(0, 2, 2, 2, 2, 16, 2, 18, 2, 20, 21, 22, 2, 24, 25, 26), u8x16::new(11, 12, 13, 14, 15, 16, 2, 18, 2, 20, 21, 22, 23, 24, 25, 26), u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8), u8x16::new(1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16)]; + let e: [u8x16; 4] = [u8x16::new(1, 2, 2, 2, 2, 16, 2, 18, 2, 20, 21, 22, 2, 24, 25, 26), u8x16::new(2, 12, 13, 14, 15, 16, 2, 18, 2, 20, 21, 22, 23, 24, 25, 26), u8x16::new(2, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8), u8x16::new(2, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16)]; + let r: [u8x16; 4] = transmute(vld4q_lane_u8::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_lane_u64() { + let a: [u64; 5] = [0, 1, 2, 2, 2]; + let b: [u64x1; 4] = [u64x1::new(0), u64x1::new(2), u64x1::new(2), u64x1::new(2)]; + let e: [u64x1; 4] = [u64x1::new(1), u64x1::new(2), u64x1::new(2), u64x1::new(2)]; + let r: [u64x1; 4] = transmute(vld4_lane_u64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_lane_u64() { + let a: [u64; 9] = [0, 1, 2, 2, 2, 5, 6, 7, 8]; + let b: [u64x2; 4] = [u64x2::new(0, 2), u64x2::new(2, 2), u64x2::new(2, 16), u64x2::new(2, 18)]; + let e: [u64x2; 4] = [u64x2::new(1, 2), u64x2::new(2, 2), u64x2::new(2, 16), u64x2::new(2, 18)]; + let r: [u64x2; 4] = transmute(vld4q_lane_u64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_lane_f64() { + let a: [f64; 5] = [0., 1., 2., 2., 2.]; + let b: [f64; 4] = [0., 2., 2., 2.]; + let e: [f64; 4] = [1., 2., 2., 2.]; + let r: [f64; 4] = transmute(vld4_lane_f64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_lane_f64() { + let a: [f64; 9] = [0., 1., 2., 2., 2., 5., 6., 7., 8.]; + let b: [f64x2; 4] = [f64x2::new(0., 2.), f64x2::new(2., 2.), f64x2::new(2., 16.), f64x2::new(2., 18.)]; + let e: [f64x2; 4] = [f64x2::new(1., 2.), f64x2::new(2., 2.), f64x2::new(2., 16.), f64x2::new(2., 18.)]; + let r: [f64x2; 4] = transmute(vld4q_lane_f64::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_lane_f64() { + let a: [f64; 2] = [0., 1.]; + let e: [f64; 1] = [1.]; + let mut r: [f64; 1] = [0f64; 1]; + vst1_lane_f64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_lane_f64() { + let a: [f64; 3] = [0., 1., 2.]; + let e: [f64; 2] = [1., 0.]; + let mut r: [f64; 2] = [0f64; 2]; + vst1q_lane_f64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_f64_x2() { + let a: [f64; 3] = [0., 1., 2.]; + let e: [f64; 2] = [1., 2.]; + let mut r: [f64; 2] = [0f64; 2]; + vst1_f64_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_f64_x2() { + let a: [f64; 5] = [0., 1., 2., 3., 4.]; + let e: [f64; 4] = [1., 2., 3., 4.]; + let mut r: [f64; 4] = [0f64; 4]; + vst1q_f64_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_f64_x3() { + let a: [f64; 4] = [0., 1., 2., 3.]; + let e: [f64; 3] = [1., 2., 3.]; + let mut r: [f64; 3] = [0f64; 3]; + vst1_f64_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_f64_x3() { + let a: [f64; 7] = [0., 1., 2., 3., 4., 5., 6.]; + let e: [f64; 6] = [1., 2., 3., 4., 5., 6.]; + let mut r: [f64; 6] = [0f64; 6]; + vst1q_f64_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_f64_x4() { + let a: [f64; 5] = [0., 1., 2., 3., 4.]; + let e: [f64; 4] = [1., 2., 3., 4.]; + let mut r: [f64; 4] = [0f64; 4]; + vst1_f64_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_f64_x4() { + let a: [f64; 9] = [0., 1., 2., 3., 4., 5., 6., 7., 8.]; + let e: [f64; 8] = [1., 2., 3., 4., 5., 6., 7., 8.]; + let mut r: [f64; 8] = [0f64; 8]; + vst1q_f64_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_s64() { + let a: [i64; 5] = [0, 1, 2, 2, 3]; + let e: [i64; 4] = [1, 2, 2, 3]; + let mut r: [i64; 4] = [0i64; 4]; + vst2q_s64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_u64() { + let a: [u64; 5] = [0, 1, 2, 2, 3]; + let e: [u64; 4] = [1, 2, 2, 3]; + let mut r: [u64; 4] = [0u64; 4]; + vst2q_u64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_p64() { + let a: [u64; 5] = [0, 1, 2, 2, 3]; + let e: [u64; 4] = [1, 2, 2, 3]; + let mut r: [u64; 4] = [0u64; 4]; + vst2q_p64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_f64() { + let a: [f64; 3] = [0., 1., 2.]; + let e: [f64; 2] = [1., 2.]; + let mut r: [f64; 2] = [0f64; 2]; + vst2_f64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_f64() { + let a: [f64; 5] = [0., 1., 2., 2., 3.]; + let e: [f64; 4] = [1., 2., 2., 3.]; + let mut r: [f64; 4] = [0f64; 4]; + vst2q_f64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_lane_s8() { + let a: [i8; 33] = [0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]; + let e: [i8; 32] = [1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [i8; 32] = [0i8; 32]; + vst2q_lane_s8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_lane_s64() { + let a: [i64; 3] = [0, 1, 2]; + let e: [i64; 2] = [1, 2]; + let mut r: [i64; 2] = [0i64; 2]; + vst2_lane_s64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_lane_s64() { + let a: [i64; 5] = [0, 1, 2, 2, 3]; + let e: [i64; 4] = [1, 2, 0, 0]; + let mut r: [i64; 4] = [0i64; 4]; + vst2q_lane_s64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_lane_u8() { + let a: [u8; 33] = [0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]; + let e: [u8; 32] = [1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u8; 32] = [0u8; 32]; + vst2q_lane_u8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_lane_u64() { + let a: [u64; 3] = [0, 1, 2]; + let e: [u64; 2] = [1, 2]; + let mut r: [u64; 2] = [0u64; 2]; + vst2_lane_u64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_lane_u64() { + let a: [u64; 5] = [0, 1, 2, 2, 3]; + let e: [u64; 4] = [1, 2, 0, 0]; + let mut r: [u64; 4] = [0u64; 4]; + vst2q_lane_u64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_lane_p8() { + let a: [u8; 33] = [0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]; + let e: [u8; 32] = [1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u8; 32] = [0u8; 32]; + vst2q_lane_p8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_lane_p64() { + let a: [u64; 3] = [0, 1, 2]; + let e: [u64; 2] = [1, 2]; + let mut r: [u64; 2] = [0u64; 2]; + vst2_lane_p64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_lane_p64() { + let a: [u64; 5] = [0, 1, 2, 2, 3]; + let e: [u64; 4] = [1, 2, 0, 0]; + let mut r: [u64; 4] = [0u64; 4]; + vst2q_lane_p64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_lane_f64() { + let a: [f64; 3] = [0., 1., 2.]; + let e: [f64; 2] = [1., 2.]; + let mut r: [f64; 2] = [0f64; 2]; + vst2_lane_f64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_lane_f64() { + let a: [f64; 5] = [0., 1., 2., 2., 3.]; + let e: [f64; 4] = [1., 2., 0., 0.]; + let mut r: [f64; 4] = [0f64; 4]; + vst2q_lane_f64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_s64() { + let a: [i64; 7] = [0, 1, 2, 2, 4, 2, 4]; + let e: [i64; 6] = [1, 2, 2, 2, 4, 4]; + let mut r: [i64; 6] = [0i64; 6]; + vst3q_s64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_u64() { + let a: [u64; 7] = [0, 1, 2, 2, 4, 2, 4]; + let e: [u64; 6] = [1, 2, 2, 2, 4, 4]; + let mut r: [u64; 6] = [0u64; 6]; + vst3q_u64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_p64() { + let a: [u64; 7] = [0, 1, 2, 2, 4, 2, 4]; + let e: [u64; 6] = [1, 2, 2, 2, 4, 4]; + let mut r: [u64; 6] = [0u64; 6]; + vst3q_p64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_f64() { + let a: [f64; 4] = [0., 1., 2., 2.]; + let e: [f64; 3] = [1., 2., 2.]; + let mut r: [f64; 3] = [0f64; 3]; + vst3_f64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_f64() { + let a: [f64; 7] = [0., 1., 2., 2., 4., 2., 4.]; + let e: [f64; 6] = [1., 2., 2., 2., 4., 4.]; + let mut r: [f64; 6] = [0f64; 6]; + vst3q_f64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_lane_s8() { + let a: [i8; 49] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16, 25, 26, 27, 28, 29, 30, 31, 32, 2, 4, 7, 8, 13, 14, 15, 16, 41, 42, 43, 44, 45, 46, 47, 48]; + let e: [i8; 48] = [1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [i8; 48] = [0i8; 48]; + vst3q_lane_s8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_lane_s64() { + let a: [i64; 4] = [0, 1, 2, 2]; + let e: [i64; 3] = [1, 2, 2]; + let mut r: [i64; 3] = [0i64; 3]; + vst3_lane_s64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_lane_s64() { + let a: [i64; 7] = [0, 1, 2, 2, 4, 2, 4]; + let e: [i64; 6] = [1, 2, 2, 0, 0, 0]; + let mut r: [i64; 6] = [0i64; 6]; + vst3q_lane_s64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_lane_u8() { + let a: [u8; 49] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16, 25, 26, 27, 28, 29, 30, 31, 32, 2, 4, 7, 8, 13, 14, 15, 16, 41, 42, 43, 44, 45, 46, 47, 48]; + let e: [u8; 48] = [1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u8; 48] = [0u8; 48]; + vst3q_lane_u8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_lane_u64() { + let a: [u64; 4] = [0, 1, 2, 2]; + let e: [u64; 3] = [1, 2, 2]; + let mut r: [u64; 3] = [0u64; 3]; + vst3_lane_u64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_lane_u64() { + let a: [u64; 7] = [0, 1, 2, 2, 4, 2, 4]; + let e: [u64; 6] = [1, 2, 2, 0, 0, 0]; + let mut r: [u64; 6] = [0u64; 6]; + vst3q_lane_u64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_lane_p8() { + let a: [u8; 49] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16, 25, 26, 27, 28, 29, 30, 31, 32, 2, 4, 7, 8, 13, 14, 15, 16, 41, 42, 43, 44, 45, 46, 47, 48]; + let e: [u8; 48] = [1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u8; 48] = [0u8; 48]; + vst3q_lane_p8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_lane_p64() { + let a: [u64; 4] = [0, 1, 2, 2]; + let e: [u64; 3] = [1, 2, 2]; + let mut r: [u64; 3] = [0u64; 3]; + vst3_lane_p64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_lane_p64() { + let a: [u64; 7] = [0, 1, 2, 2, 4, 2, 4]; + let e: [u64; 6] = [1, 2, 2, 0, 0, 0]; + let mut r: [u64; 6] = [0u64; 6]; + vst3q_lane_p64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_lane_f64() { + let a: [f64; 4] = [0., 1., 2., 2.]; + let e: [f64; 3] = [1., 2., 2.]; + let mut r: [f64; 3] = [0f64; 3]; + vst3_lane_f64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_lane_f64() { + let a: [f64; 7] = [0., 1., 2., 2., 3., 2., 3.]; + let e: [f64; 6] = [1., 2., 2., 0., 0., 0.]; + let mut r: [f64; 6] = [0f64; 6]; + vst3q_lane_f64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_s64() { + let a: [i64; 9] = [0, 1, 2, 2, 6, 2, 6, 6, 8]; + let e: [i64; 8] = [1, 2, 2, 6, 2, 6, 6, 8]; + let mut r: [i64; 8] = [0i64; 8]; + vst4q_s64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_u64() { + let a: [u64; 9] = [0, 1, 2, 2, 6, 2, 6, 6, 8]; + let e: [u64; 8] = [1, 2, 2, 6, 2, 6, 6, 8]; + let mut r: [u64; 8] = [0u64; 8]; + vst4q_u64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_p64() { + let a: [u64; 9] = [0, 1, 2, 2, 6, 2, 6, 6, 8]; + let e: [u64; 8] = [1, 2, 2, 6, 2, 6, 6, 8]; + let mut r: [u64; 8] = [0u64; 8]; + vst4q_p64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_f64() { + let a: [f64; 5] = [0., 1., 2., 2., 6.]; + let e: [f64; 4] = [1., 2., 2., 6.]; + let mut r: [f64; 4] = [0f64; 4]; + vst4_f64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_f64() { + let a: [f64; 9] = [0., 1., 2., 2., 6., 2., 6., 6., 8.]; + let e: [f64; 8] = [1., 2., 2., 6., 2., 6., 6., 8.]; + let mut r: [f64; 8] = [0f64; 8]; + vst4q_f64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_lane_s8() { + let a: [i8; 65] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64]; + let e: [i8; 64] = [1, 2, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [i8; 64] = [0i8; 64]; + vst4q_lane_s8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_lane_s64() { + let a: [i64; 5] = [0, 1, 2, 2, 6]; + let e: [i64; 4] = [1, 2, 2, 6]; + let mut r: [i64; 4] = [0i64; 4]; + vst4_lane_s64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_lane_s64() { + let a: [i64; 9] = [0, 1, 2, 2, 6, 2, 6, 6, 8]; + let e: [i64; 8] = [1, 2, 2, 6, 0, 0, 0, 0]; + let mut r: [i64; 8] = [0i64; 8]; + vst4q_lane_s64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_lane_u8() { + let a: [u8; 65] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64]; + let e: [u8; 64] = [1, 2, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u8; 64] = [0u8; 64]; + vst4q_lane_u8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_lane_u64() { + let a: [u64; 5] = [0, 1, 2, 2, 6]; + let e: [u64; 4] = [1, 2, 2, 6]; + let mut r: [u64; 4] = [0u64; 4]; + vst4_lane_u64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_lane_u64() { + let a: [u64; 9] = [0, 1, 2, 2, 6, 2, 6, 6, 8]; + let e: [u64; 8] = [1, 2, 2, 6, 0, 0, 0, 0]; + let mut r: [u64; 8] = [0u64; 8]; + vst4q_lane_u64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_lane_p8() { + let a: [u8; 65] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64]; + let e: [u8; 64] = [1, 2, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u8; 64] = [0u8; 64]; + vst4q_lane_p8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_lane_p64() { + let a: [u64; 5] = [0, 1, 2, 2, 6]; + let e: [u64; 4] = [1, 2, 2, 6]; + let mut r: [u64; 4] = [0u64; 4]; + vst4_lane_p64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_lane_p64() { + let a: [u64; 9] = [0, 1, 2, 2, 6, 2, 6, 6, 8]; + let e: [u64; 8] = [1, 2, 2, 6, 0, 0, 0, 0]; + let mut r: [u64; 8] = [0u64; 8]; + vst4q_lane_p64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_lane_f64() { + let a: [f64; 5] = [0., 1., 2., 2., 6.]; + let e: [f64; 4] = [1., 2., 2., 6.]; + let mut r: [f64; 4] = [0f64; 4]; + vst4_lane_f64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_lane_f64() { + let a: [f64; 9] = [0., 1., 2., 2., 6., 2., 6., 6., 8.]; + let e: [f64; 8] = [1., 2., 2., 6., 0., 0., 0., 0.]; + let mut r: [f64; 8] = [0f64; 8]; + vst4q_lane_f64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_f64() { + let a: f64 = 1.0; + let b: f64 = 2.0; + let e: f64 = 2.0; + let r: f64 = transmute(vmul_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_f64() { + let a: f64x2 = f64x2::new(1.0, 2.0); + let b: f64x2 = f64x2::new(2.0, 3.0); + let e: f64x2 = f64x2::new(2.0, 6.0); + let r: f64x2 = transmute(vmulq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_n_f64() { + let a: f64 = 1.; + let b: f64 = 2.; + let e: f64 = 2.; + let r: f64 = transmute(vmul_n_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_n_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let b: f64 = 2.; + let e: f64x2 = f64x2::new(2., 4.); + let r: f64x2 = transmute(vmulq_n_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_lane_f64() { + let a: f64 = 1.; + let b: f64 = 2.; + let e: f64 = 2.; + let r: f64 = transmute(vmul_lane_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_laneq_f64() { + let a: f64 = 1.; + let b: f64x2 = f64x2::new(2., 0.); + let e: f64 = 2.; + let r: f64 = transmute(vmul_laneq_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_lane_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let b: f64 = 2.; + let e: f64x2 = f64x2::new(2., 4.); + let r: f64x2 = transmute(vmulq_lane_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_laneq_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let b: f64x2 = f64x2::new(2., 0.); + let e: f64x2 = f64x2::new(2., 4.); + let r: f64x2 = transmute(vmulq_laneq_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmuls_lane_f32() { + let a: f32 = 1.; + let b: f32x2 = f32x2::new(2., 0.); + let e: f32 = 2.; + let r: f32 = transmute(vmuls_lane_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmuls_laneq_f32() { + let a: f32 = 1.; + let b: f32x4 = f32x4::new(2., 0., 0., 0.); + let e: f32 = 2.; + let r: f32 = transmute(vmuls_laneq_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmuld_lane_f64() { + let a: f64 = 1.; + let b: f64 = 2.; + let e: f64 = 2.; + let r: f64 = transmute(vmuld_lane_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmuld_laneq_f64() { + let a: f64 = 1.; + let b: f64x2 = f64x2::new(2., 0.); + let e: f64 = 2.; + let r: f64 = transmute(vmuld_laneq_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_s8() { + let a: i8x16 = i8x16::new(1, 2, 9, 10, 9, 10, 11, 12, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); + let e: i16x8 = i16x8::new(9, 20, 11, 24, 13, 28, 15, 32); + let r: i16x8 = transmute(vmull_high_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_s16() { + let a: i16x8 = i16x8::new(1, 2, 9, 10, 9, 10, 11, 12); + let b: i16x8 = i16x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: i32x4 = i32x4::new(9, 20, 11, 24); + let r: i32x4 = transmute(vmull_high_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_s32() { + let a: i32x4 = i32x4::new(1, 2, 9, 10); + let b: i32x4 = i32x4::new(1, 2, 1, 2); + let e: i64x2 = i64x2::new(9, 20); + let r: i64x2 = transmute(vmull_high_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_u8() { + let a: u8x16 = u8x16::new(1, 2, 9, 10, 9, 10, 11, 12, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); + let e: u16x8 = u16x8::new(9, 20, 11, 24, 13, 28, 15, 32); + let r: u16x8 = transmute(vmull_high_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_u16() { + let a: u16x8 = u16x8::new(1, 2, 9, 10, 9, 10, 11, 12); + let b: u16x8 = u16x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: u32x4 = u32x4::new(9, 20, 11, 24); + let r: u32x4 = transmute(vmull_high_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_u32() { + let a: u32x4 = u32x4::new(1, 2, 9, 10); + let b: u32x4 = u32x4::new(1, 2, 1, 2); + let e: u64x2 = u64x2::new(9, 20); + let r: u64x2 = transmute(vmull_high_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_p64() { + let a: p64 = 15; + let b: p64 = 3; + let e: p128 = 17; + let r: p128 = transmute(vmull_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_p8() { + let a: i8x16 = i8x16::new(1, 2, 9, 10, 9, 10, 11, 12, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3); + let e: i16x8 = i16x8::new(9, 30, 11, 20, 13, 18, 15, 48); + let r: i16x8 = transmute(vmull_high_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_p64() { + let a: i64x2 = i64x2::new(1, 15); + let b: i64x2 = i64x2::new(1, 3); + let e: p128 = 17; + let r: p128 = transmute(vmull_high_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_n_s16() { + let a: i16x8 = i16x8::new(1, 2, 9, 10, 9, 10, 11, 12); + let b: i16 = 2; + let e: i32x4 = i32x4::new(18, 20, 22, 24); + let r: i32x4 = transmute(vmull_high_n_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_n_s32() { + let a: i32x4 = i32x4::new(1, 2, 9, 10); + let b: i32 = 2; + let e: i64x2 = i64x2::new(18, 20); + let r: i64x2 = transmute(vmull_high_n_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_n_u16() { + let a: u16x8 = u16x8::new(1, 2, 9, 10, 9, 10, 11, 12); + let b: u16 = 2; + let e: u32x4 = u32x4::new(18, 20, 22, 24); + let r: u32x4 = transmute(vmull_high_n_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_n_u32() { + let a: u32x4 = u32x4::new(1, 2, 9, 10); + let b: u32 = 2; + let e: u64x2 = u64x2::new(18, 20); + let r: u64x2 = transmute(vmull_high_n_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_lane_s16() { + let a: i16x8 = i16x8::new(1, 2, 9, 10, 9, 10, 11, 12); + let b: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i32x4 = i32x4::new(18, 20, 22, 24); + let r: i32x4 = transmute(vmull_high_lane_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_laneq_s16() { + let a: i16x8 = i16x8::new(1, 2, 9, 10, 9, 10, 11, 12); + let b: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i32x4 = i32x4::new(18, 20, 22, 24); + let r: i32x4 = transmute(vmull_high_laneq_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_lane_s32() { + let a: i32x4 = i32x4::new(1, 2, 9, 10); + let b: i32x2 = i32x2::new(0, 2); + let e: i64x2 = i64x2::new(18, 20); + let r: i64x2 = transmute(vmull_high_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_laneq_s32() { + let a: i32x4 = i32x4::new(1, 2, 9, 10); + let b: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i64x2 = i64x2::new(18, 20); + let r: i64x2 = transmute(vmull_high_laneq_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_lane_u16() { + let a: u16x8 = u16x8::new(1, 2, 9, 10, 9, 10, 11, 12); + let b: u16x4 = u16x4::new(0, 2, 0, 0); + let e: u32x4 = u32x4::new(18, 20, 22, 24); + let r: u32x4 = transmute(vmull_high_lane_u16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_laneq_u16() { + let a: u16x8 = u16x8::new(1, 2, 9, 10, 9, 10, 11, 12); + let b: u16x8 = u16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: u32x4 = u32x4::new(18, 20, 22, 24); + let r: u32x4 = transmute(vmull_high_laneq_u16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_lane_u32() { + let a: u32x4 = u32x4::new(1, 2, 9, 10); + let b: u32x2 = u32x2::new(0, 2); + let e: u64x2 = u64x2::new(18, 20); + let r: u64x2 = transmute(vmull_high_lane_u32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_laneq_u32() { + let a: u32x4 = u32x4::new(1, 2, 9, 10); + let b: u32x4 = u32x4::new(0, 2, 0, 0); + let e: u64x2 = u64x2::new(18, 20); + let r: u64x2 = transmute(vmull_high_laneq_u32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulx_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let b: f32x2 = f32x2::new(2., 2.); + let e: f32x2 = f32x2::new(2., 4.); + let r: f32x2 = transmute(vmulx_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxq_f32() { + let a: f32x4 = f32x4::new(1., 2., 3., 4.); + let b: f32x4 = f32x4::new(2., 2., 2., 2.); + let e: f32x4 = f32x4::new(2., 4., 6., 8.); + let r: f32x4 = transmute(vmulxq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulx_f64() { + let a: f64 = 1.; + let b: f64 = 2.; + let e: f64 = 2.; + let r: f64 = transmute(vmulx_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxq_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let b: f64x2 = f64x2::new(2., 2.); + let e: f64x2 = f64x2::new(2., 4.); + let r: f64x2 = transmute(vmulxq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulx_lane_f64() { + let a: f64 = 1.; + let b: f64 = 2.; + let e: f64 = 2.; + let r: f64 = transmute(vmulx_lane_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulx_laneq_f64() { + let a: f64 = 1.; + let b: f64x2 = f64x2::new(2., 0.); + let e: f64 = 2.; + let r: f64 = transmute(vmulx_laneq_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulx_lane_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let b: f32x2 = f32x2::new(2., 0.); + let e: f32x2 = f32x2::new(2., 4.); + let r: f32x2 = transmute(vmulx_lane_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulx_laneq_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let b: f32x4 = f32x4::new(2., 0., 0., 0.); + let e: f32x2 = f32x2::new(2., 4.); + let r: f32x2 = transmute(vmulx_laneq_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxq_lane_f32() { + let a: f32x4 = f32x4::new(1., 2., 3., 4.); + let b: f32x2 = f32x2::new(2., 0.); + let e: f32x4 = f32x4::new(2., 4., 6., 8.); + let r: f32x4 = transmute(vmulxq_lane_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxq_laneq_f32() { + let a: f32x4 = f32x4::new(1., 2., 3., 4.); + let b: f32x4 = f32x4::new(2., 0., 0., 0.); + let e: f32x4 = f32x4::new(2., 4., 6., 8.); + let r: f32x4 = transmute(vmulxq_laneq_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxq_lane_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let b: f64 = 2.; + let e: f64x2 = f64x2::new(2., 4.); + let r: f64x2 = transmute(vmulxq_lane_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxq_laneq_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let b: f64x2 = f64x2::new(2., 0.); + let e: f64x2 = f64x2::new(2., 4.); + let r: f64x2 = transmute(vmulxq_laneq_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxs_f32() { + let a: f32 = 2.; + let b: f32 = 3.; + let e: f32 = 6.; + let r: f32 = transmute(vmulxs_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxd_f64() { + let a: f64 = 2.; + let b: f64 = 3.; + let e: f64 = 6.; + let r: f64 = transmute(vmulxd_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxs_lane_f32() { + let a: f32 = 2.; + let b: f32x2 = f32x2::new(3., 0.); + let e: f32 = 6.; + let r: f32 = transmute(vmulxs_lane_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxs_laneq_f32() { + let a: f32 = 2.; + let b: f32x4 = f32x4::new(3., 0., 0., 0.); + let e: f32 = 6.; + let r: f32 = transmute(vmulxs_laneq_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxd_lane_f64() { + let a: f64 = 2.; + let b: f64 = 3.; + let e: f64 = 6.; + let r: f64 = transmute(vmulxd_lane_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxd_laneq_f64() { + let a: f64 = 2.; + let b: f64x2 = f64x2::new(3., 0.); + let e: f64 = 6.; + let r: f64 = transmute(vmulxd_laneq_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfma_f64() { + let a: f64 = 8.0; + let b: f64 = 6.0; + let c: f64 = 2.0; + let e: f64 = 20.0; + let r: f64 = transmute(vfma_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmaq_f64() { + let a: f64x2 = f64x2::new(8.0, 18.0); + let b: f64x2 = f64x2::new(6.0, 4.0); + let c: f64x2 = f64x2::new(2.0, 3.0); + let e: f64x2 = f64x2::new(20.0, 30.0); + let r: f64x2 = transmute(vfmaq_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfma_n_f64() { + let a: f64 = 2.0; + let b: f64 = 6.0; + let c: f64 = 8.0; + let e: f64 = 50.0; + let r: f64 = transmute(vfma_n_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmaq_n_f64() { + let a: f64x2 = f64x2::new(2.0, 3.0); + let b: f64x2 = f64x2::new(6.0, 4.0); + let c: f64 = 8.0; + let e: f64x2 = f64x2::new(50.0, 35.0); + let r: f64x2 = transmute(vfmaq_n_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfma_lane_f32() { + let a: f32x2 = f32x2::new(2., 3.); + let b: f32x2 = f32x2::new(6., 4.); + let c: f32x2 = f32x2::new(2., 0.); + let e: f32x2 = f32x2::new(14., 11.); + let r: f32x2 = transmute(vfma_lane_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfma_laneq_f32() { + let a: f32x2 = f32x2::new(2., 3.); + let b: f32x2 = f32x2::new(6., 4.); + let c: f32x4 = f32x4::new(2., 0., 0., 0.); + let e: f32x2 = f32x2::new(14., 11.); + let r: f32x2 = transmute(vfma_laneq_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmaq_lane_f32() { + let a: f32x4 = f32x4::new(2., 3., 4., 5.); + let b: f32x4 = f32x4::new(6., 4., 7., 8.); + let c: f32x2 = f32x2::new(2., 0.); + let e: f32x4 = f32x4::new(14., 11., 18., 21.); + let r: f32x4 = transmute(vfmaq_lane_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmaq_laneq_f32() { + let a: f32x4 = f32x4::new(2., 3., 4., 5.); + let b: f32x4 = f32x4::new(6., 4., 7., 8.); + let c: f32x4 = f32x4::new(2., 0., 0., 0.); + let e: f32x4 = f32x4::new(14., 11., 18., 21.); + let r: f32x4 = transmute(vfmaq_laneq_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfma_lane_f64() { + let a: f64 = 2.; + let b: f64 = 6.; + let c: f64 = 2.; + let e: f64 = 14.; + let r: f64 = transmute(vfma_lane_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfma_laneq_f64() { + let a: f64 = 2.; + let b: f64 = 6.; + let c: f64x2 = f64x2::new(2., 0.); + let e: f64 = 14.; + let r: f64 = transmute(vfma_laneq_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmaq_lane_f64() { + let a: f64x2 = f64x2::new(2., 3.); + let b: f64x2 = f64x2::new(6., 4.); + let c: f64 = 2.; + let e: f64x2 = f64x2::new(14., 11.); + let r: f64x2 = transmute(vfmaq_lane_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmaq_laneq_f64() { + let a: f64x2 = f64x2::new(2., 3.); + let b: f64x2 = f64x2::new(6., 4.); + let c: f64x2 = f64x2::new(2., 0.); + let e: f64x2 = f64x2::new(14., 11.); + let r: f64x2 = transmute(vfmaq_laneq_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmas_lane_f32() { + let a: f32 = 2.; + let b: f32 = 6.; + let c: f32x2 = f32x2::new(3., 0.); + let e: f32 = 20.; + let r: f32 = transmute(vfmas_lane_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmas_laneq_f32() { + let a: f32 = 2.; + let b: f32 = 6.; + let c: f32x4 = f32x4::new(3., 0., 0., 0.); + let e: f32 = 20.; + let r: f32 = transmute(vfmas_laneq_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmad_lane_f64() { + let a: f64 = 2.; + let b: f64 = 6.; + let c: f64 = 3.; + let e: f64 = 20.; + let r: f64 = transmute(vfmad_lane_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmad_laneq_f64() { + let a: f64 = 2.; + let b: f64 = 6.; + let c: f64x2 = f64x2::new(3., 0.); + let e: f64 = 20.; + let r: f64 = transmute(vfmad_laneq_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfms_f64() { + let a: f64 = 20.0; + let b: f64 = 6.0; + let c: f64 = 2.0; + let e: f64 = 8.0; + let r: f64 = transmute(vfms_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmsq_f64() { + let a: f64x2 = f64x2::new(20.0, 30.0); + let b: f64x2 = f64x2::new(6.0, 4.0); + let c: f64x2 = f64x2::new(2.0, 3.0); + let e: f64x2 = f64x2::new(8.0, 18.0); + let r: f64x2 = transmute(vfmsq_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfms_n_f64() { + let a: f64 = 50.0; + let b: f64 = 6.0; + let c: f64 = 8.0; + let e: f64 = 2.0; + let r: f64 = transmute(vfms_n_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmsq_n_f64() { + let a: f64x2 = f64x2::new(50.0, 35.0); + let b: f64x2 = f64x2::new(6.0, 4.0); + let c: f64 = 8.0; + let e: f64x2 = f64x2::new(2.0, 3.0); + let r: f64x2 = transmute(vfmsq_n_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfms_lane_f32() { + let a: f32x2 = f32x2::new(14., 11.); + let b: f32x2 = f32x2::new(6., 4.); + let c: f32x2 = f32x2::new(2., 0.); + let e: f32x2 = f32x2::new(2., 3.); + let r: f32x2 = transmute(vfms_lane_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfms_laneq_f32() { + let a: f32x2 = f32x2::new(14., 11.); + let b: f32x2 = f32x2::new(6., 4.); + let c: f32x4 = f32x4::new(2., 0., 0., 0.); + let e: f32x2 = f32x2::new(2., 3.); + let r: f32x2 = transmute(vfms_laneq_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmsq_lane_f32() { + let a: f32x4 = f32x4::new(14., 11., 18., 21.); + let b: f32x4 = f32x4::new(6., 4., 7., 8.); + let c: f32x2 = f32x2::new(2., 0.); + let e: f32x4 = f32x4::new(2., 3., 4., 5.); + let r: f32x4 = transmute(vfmsq_lane_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmsq_laneq_f32() { + let a: f32x4 = f32x4::new(14., 11., 18., 21.); + let b: f32x4 = f32x4::new(6., 4., 7., 8.); + let c: f32x4 = f32x4::new(2., 0., 0., 0.); + let e: f32x4 = f32x4::new(2., 3., 4., 5.); + let r: f32x4 = transmute(vfmsq_laneq_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfms_lane_f64() { + let a: f64 = 14.; + let b: f64 = 6.; + let c: f64 = 2.; + let e: f64 = 2.; + let r: f64 = transmute(vfms_lane_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfms_laneq_f64() { + let a: f64 = 14.; + let b: f64 = 6.; + let c: f64x2 = f64x2::new(2., 0.); + let e: f64 = 2.; + let r: f64 = transmute(vfms_laneq_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmsq_lane_f64() { + let a: f64x2 = f64x2::new(14., 11.); + let b: f64x2 = f64x2::new(6., 4.); + let c: f64 = 2.; + let e: f64x2 = f64x2::new(2., 3.); + let r: f64x2 = transmute(vfmsq_lane_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmsq_laneq_f64() { + let a: f64x2 = f64x2::new(14., 11.); + let b: f64x2 = f64x2::new(6., 4.); + let c: f64x2 = f64x2::new(2., 0.); + let e: f64x2 = f64x2::new(2., 3.); + let r: f64x2 = transmute(vfmsq_laneq_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmss_lane_f32() { + let a: f32 = 14.; + let b: f32 = 6.; + let c: f32x2 = f32x2::new(2., 0.); + let e: f32 = 2.; + let r: f32 = transmute(vfmss_lane_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmss_laneq_f32() { + let a: f32 = 14.; + let b: f32 = 6.; + let c: f32x4 = f32x4::new(2., 0., 0., 0.); + let e: f32 = 2.; + let r: f32 = transmute(vfmss_laneq_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmsd_lane_f64() { + let a: f64 = 14.; + let b: f64 = 6.; + let c: f64 = 2.; + let e: f64 = 2.; + let r: f64 = transmute(vfmsd_lane_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmsd_laneq_f64() { + let a: f64 = 14.; + let b: f64 = 6.; + let c: f64x2 = f64x2::new(2., 0.); + let e: f64 = 2.; + let r: f64 = transmute(vfmsd_laneq_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdiv_f32() { + let a: f32x2 = f32x2::new(2.0, 6.0); + let b: f32x2 = f32x2::new(1.0, 2.0); + let e: f32x2 = f32x2::new(2.0, 3.0); + let r: f32x2 = transmute(vdiv_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdivq_f32() { + let a: f32x4 = f32x4::new(2.0, 6.0, 4.0, 10.0); + let b: f32x4 = f32x4::new(1.0, 2.0, 1.0, 2.0); + let e: f32x4 = f32x4::new(2.0, 3.0, 4.0, 5.0); + let r: f32x4 = transmute(vdivq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdiv_f64() { + let a: f64 = 2.0; + let b: f64 = 1.0; + let e: f64 = 2.0; + let r: f64 = transmute(vdiv_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdivq_f64() { + let a: f64x2 = f64x2::new(2.0, 6.0); + let b: f64x2 = f64x2::new(1.0, 2.0); + let e: f64x2 = f64x2::new(2.0, 3.0); + let r: f64x2 = transmute(vdivq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_f64() { + let a: f64 = 1.0; + let b: f64 = 1.0; + let e: f64 = 0.0; + let r: f64 = transmute(vsub_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_f64() { + let a: f64x2 = f64x2::new(1.0, 4.0); + let b: f64x2 = f64x2::new(1.0, 2.0); + let e: f64x2 = f64x2::new(0.0, 2.0); + let r: f64x2 = transmute(vsubq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubd_s64() { + let a: i64 = 3; + let b: i64 = 2; + let e: i64 = 1; + let r: i64 = transmute(vsubd_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubd_u64() { + let a: u64 = 3; + let b: u64 = 2; + let e: u64 = 1; + let r: u64 = transmute(vsubd_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddd_s64() { + let a: i64 = 1; + let b: i64 = 2; + let e: i64 = 3; + let r: i64 = transmute(vaddd_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddd_u64() { + let a: u64 = 1; + let b: u64 = 2; + let e: u64 = 3; + let r: u64 = transmute(vaddd_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddv_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let e: f32 = 3.; + let r: f32 = transmute(vaddv_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddvq_f32() { + let a: f32x4 = f32x4::new(1., 2., 0., 0.); + let e: f32 = 3.; + let r: f32 = transmute(vaddvq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddvq_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let e: f64 = 3.; + let r: f64 = transmute(vaddvq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddlv_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let e: i32 = 10; + let r: i32 = transmute(vaddlv_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddlvq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i32 = 36; + let r: i32 = transmute(vaddlvq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddlv_s32() { + let a: i32x2 = i32x2::new(1, 2); + let e: i64 = 3; + let r: i64 = transmute(vaddlv_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddlvq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let e: i64 = 10; + let r: i64 = transmute(vaddlvq_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddlv_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let e: u32 = 10; + let r: u32 = transmute(vaddlv_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddlvq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u32 = 36; + let r: u32 = transmute(vaddlvq_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddlv_u32() { + let a: u32x2 = u32x2::new(1, 2); + let e: u64 = 3; + let r: u64 = transmute(vaddlv_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddlvq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u64 = 10; + let r: u64 = transmute(vaddlvq_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_high_s8() { + let a: i16x8 = i16x8::new(8, 9, 10, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i16x8 = transmute(vsubw_high_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_high_s16() { + let a: i32x4 = i32x4::new(8, 9, 10, 11); + let b: i16x8 = i16x8::new(0, 1, 2, 3, 8, 9, 10, 11); + let e: i32x4 = i32x4::new(0, 0, 0, 0); + let r: i32x4 = transmute(vsubw_high_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_high_s32() { + let a: i64x2 = i64x2::new(8, 9); + let b: i32x4 = i32x4::new(6, 7, 8, 9); + let e: i64x2 = i64x2::new(0, 0); + let r: i64x2 = transmute(vsubw_high_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_high_u8() { + let a: u16x8 = u16x8::new(8, 9, 10, 11, 12, 13, 14, 15); + let b: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vsubw_high_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_high_u16() { + let a: u32x4 = u32x4::new(8, 9, 10, 11); + let b: u16x8 = u16x8::new(0, 1, 2, 3, 8, 9, 10, 11); + let e: u32x4 = u32x4::new(0, 0, 0, 0); + let r: u32x4 = transmute(vsubw_high_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_high_u32() { + let a: u64x2 = u64x2::new(8, 9); + let b: u32x4 = u32x4::new(6, 7, 8, 9); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vsubw_high_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_high_s8() { + let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: i8x16 = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2); + let e: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: i16x8 = transmute(vsubl_high_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_high_s16() { + let a: i16x8 = i16x8::new(8, 9, 10, 11, 12, 13, 14, 15); + let b: i16x8 = i16x8::new(6, 6, 6, 6, 8, 8, 8, 8); + let e: i32x4 = i32x4::new(4, 5, 6, 7); + let r: i32x4 = transmute(vsubl_high_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_high_s32() { + let a: i32x4 = i32x4::new(12, 13, 14, 15); + let b: i32x4 = i32x4::new(6, 6, 8, 8); + let e: i64x2 = i64x2::new(6, 7); + let r: i64x2 = transmute(vsubl_high_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_high_u8() { + let a: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: u8x16 = u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2); + let e: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: u16x8 = transmute(vsubl_high_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_high_u16() { + let a: u16x8 = u16x8::new(8, 9, 10, 11, 12, 13, 14, 15); + let b: u16x8 = u16x8::new(6, 6, 6, 6, 8, 8, 8, 8); + let e: u32x4 = u32x4::new(4, 5, 6, 7); + let r: u32x4 = transmute(vsubl_high_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_high_u32() { + let a: u32x4 = u32x4::new(12, 13, 14, 15); + let b: u32x4 = u32x4::new(6, 6, 8, 8); + let e: u64x2 = u64x2::new(6, 7); + let r: u64x2 = transmute(vsubl_high_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_vbcaxq_s8() { + let a: i8x16 = i8x16::new(1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0); + let b: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let c: i8x16 = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let e: i8x16 = i8x16::new(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + let r: i8x16 = transmute(vbcaxq_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_vbcaxq_s16() { + let a: i16x8 = i16x8::new(1, 0, 1, 0, 1, 0, 1, 0); + let b: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let c: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let e: i16x8 = i16x8::new(1, 0, 3, 2, 5, 4, 7, 6); + let r: i16x8 = transmute(vbcaxq_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_vbcaxq_s32() { + let a: i32x4 = i32x4::new(1, 0, 1, 0); + let b: i32x4 = i32x4::new(0, 1, 2, 3); + let c: i32x4 = i32x4::new(1, 1, 1, 1); + let e: i32x4 = i32x4::new(1, 0, 3, 2); + let r: i32x4 = transmute(vbcaxq_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_vbcaxq_s64() { + let a: i64x2 = i64x2::new(1, 0); + let b: i64x2 = i64x2::new(0, 1); + let c: i64x2 = i64x2::new(1, 1); + let e: i64x2 = i64x2::new(1, 0); + let r: i64x2 = transmute(vbcaxq_s64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_vbcaxq_u8() { + let a: u8x16 = u8x16::new(1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0); + let b: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let c: u8x16 = u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let e: u8x16 = u8x16::new(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + let r: u8x16 = transmute(vbcaxq_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_vbcaxq_u16() { + let a: u16x8 = u16x8::new(1, 0, 1, 0, 1, 0, 1, 0); + let b: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let c: u16x8 = u16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let e: u16x8 = u16x8::new(1, 0, 3, 2, 5, 4, 7, 6); + let r: u16x8 = transmute(vbcaxq_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_vbcaxq_u32() { + let a: u32x4 = u32x4::new(1, 0, 1, 0); + let b: u32x4 = u32x4::new(0, 1, 2, 3); + let c: u32x4 = u32x4::new(1, 1, 1, 1); + let e: u32x4 = u32x4::new(1, 0, 3, 2); + let r: u32x4 = transmute(vbcaxq_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_vbcaxq_u64() { + let a: u64x2 = u64x2::new(1, 0); + let b: u64x2 = u64x2::new(0, 1); + let c: u64x2 = u64x2::new(1, 1); + let e: u64x2 = u64x2::new(1, 0); + let r: u64x2 = transmute(vbcaxq_u64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcadd_rot270_f32() { + let a: f32x2 = f32x2::new(1., -1.); + let b: f32x2 = f32x2::new(-1., 1.); + let e: f32x2 = f32x2::new(2., 0.); + let r: f32x2 = transmute(vcadd_rot270_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcaddq_rot270_f32() { + let a: f32x4 = f32x4::new(1., -1., 1., -1.); + let b: f32x4 = f32x4::new(-1., 1., -1., 1.); + let e: f32x4 = f32x4::new(2., 0., 2., 0.); + let r: f32x4 = transmute(vcaddq_rot270_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcaddq_rot270_f64() { + let a: f64x2 = f64x2::new(1., -1.); + let b: f64x2 = f64x2::new(-1., 1.); + let e: f64x2 = f64x2::new(2., 0.); + let r: f64x2 = transmute(vcaddq_rot270_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcadd_rot90_f32() { + let a: f32x2 = f32x2::new(1., -1.); + let b: f32x2 = f32x2::new(-1., 1.); + let e: f32x2 = f32x2::new(0., -2.); + let r: f32x2 = transmute(vcadd_rot90_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcaddq_rot90_f32() { + let a: f32x4 = f32x4::new(1., -1., 1., -1.); + let b: f32x4 = f32x4::new(-1., 1., -1., 1.); + let e: f32x4 = f32x4::new(0., -2., 0., -2.); + let r: f32x4 = transmute(vcaddq_rot90_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcaddq_rot90_f64() { + let a: f64x2 = f64x2::new(1., -1.); + let b: f64x2 = f64x2::new(-1., 1.); + let e: f64x2 = f64x2::new(0., -2.); + let r: f64x2 = transmute(vcaddq_rot90_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmla_f32() { + let a: f32x2 = f32x2::new(1., -1.); + let b: f32x2 = f32x2::new(-1., 1.); + let c: f32x2 = f32x2::new(1., 1.); + let e: f32x2 = f32x2::new(0., -2.); + let r: f32x2 = transmute(vcmla_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmlaq_f32() { + let a: f32x4 = f32x4::new(1., -1., 1., -1.); + let b: f32x4 = f32x4::new(-1., 1., -1., 1.); + let c: f32x4 = f32x4::new(1., 1., -1., -1.); + let e: f32x4 = f32x4::new(0., -2., 2., 0.); + let r: f32x4 = transmute(vcmlaq_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmlaq_f64() { + let a: f64x2 = f64x2::new(1., -1.); + let b: f64x2 = f64x2::new(-1., 1.); + let c: f64x2 = f64x2::new(1., 1.); + let e: f64x2 = f64x2::new(0., -2.); + let r: f64x2 = transmute(vcmlaq_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmla_rot90_f32() { + let a: f32x2 = f32x2::new(1., 1.); + let b: f32x2 = f32x2::new(1., -1.); + let c: f32x2 = f32x2::new(1., 1.); + let e: f32x2 = f32x2::new(2., 0.); + let r: f32x2 = transmute(vcmla_rot90_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmlaq_rot90_f32() { + let a: f32x4 = f32x4::new(1., 1., 1., 1.); + let b: f32x4 = f32x4::new(1., -1., 1., -1.); + let c: f32x4 = f32x4::new(1., 1., 1., 1.); + let e: f32x4 = f32x4::new(2., 0., 2., 0.); + let r: f32x4 = transmute(vcmlaq_rot90_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmlaq_rot90_f64() { + let a: f64x2 = f64x2::new(1., 1.); + let b: f64x2 = f64x2::new(1., -1.); + let c: f64x2 = f64x2::new(1., 1.); + let e: f64x2 = f64x2::new(2., 0.); + let r: f64x2 = transmute(vcmlaq_rot90_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmla_rot180_f32() { + let a: f32x2 = f32x2::new(1., 1.); + let b: f32x2 = f32x2::new(1., -1.); + let c: f32x2 = f32x2::new(1., 1.); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vcmla_rot180_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmlaq_rot180_f32() { + let a: f32x4 = f32x4::new(1., 1., 1., 1.); + let b: f32x4 = f32x4::new(1., -1., 1., -1.); + let c: f32x4 = f32x4::new(1., 1., 1., 1.); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vcmlaq_rot180_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmlaq_rot180_f64() { + let a: f64x2 = f64x2::new(1., 1.); + let b: f64x2 = f64x2::new(1., -1.); + let c: f64x2 = f64x2::new(1., 1.); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vcmlaq_rot180_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmla_rot270_f32() { + let a: f32x2 = f32x2::new(1., 1.); + let b: f32x2 = f32x2::new(1., -1.); + let c: f32x2 = f32x2::new(1., 1.); + let e: f32x2 = f32x2::new(0., 2.); + let r: f32x2 = transmute(vcmla_rot270_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmlaq_rot270_f32() { + let a: f32x4 = f32x4::new(1., 1., 1., 1.); + let b: f32x4 = f32x4::new(1., -1., 1., -1.); + let c: f32x4 = f32x4::new(1., 1., 1., 1.); + let e: f32x4 = f32x4::new(0., 2., 0., 2.); + let r: f32x4 = transmute(vcmlaq_rot270_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmlaq_rot270_f64() { + let a: f64x2 = f64x2::new(1., 1.); + let b: f64x2 = f64x2::new(1., -1.); + let c: f64x2 = f64x2::new(1., 1.); + let e: f64x2 = f64x2::new(0., 2.); + let r: f64x2 = transmute(vcmlaq_rot270_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmla_lane_f32() { + let a: f32x2 = f32x2::new(1., -1.); + let b: f32x2 = f32x2::new(-1., 1.); + let c: f32x2 = f32x2::new(1., 1.); + let e: f32x2 = f32x2::new(0., -2.); + let r: f32x2 = transmute(vcmla_lane_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmla_laneq_f32() { + let a: f32x2 = f32x2::new(1., -1.); + let b: f32x2 = f32x2::new(-1., 1.); + let c: f32x4 = f32x4::new(1., 1., -1., -1.); + let e: f32x2 = f32x2::new(0., -2.); + let r: f32x2 = transmute(vcmla_laneq_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmlaq_lane_f32() { + let a: f32x4 = f32x4::new(1., -1., 1., -1.); + let b: f32x4 = f32x4::new(-1., 1., -1., 1.); + let c: f32x2 = f32x2::new(1., 1.); + let e: f32x4 = f32x4::new(0., -2., 0., -2.); + let r: f32x4 = transmute(vcmlaq_lane_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmlaq_laneq_f32() { + let a: f32x4 = f32x4::new(1., -1., 1., -1.); + let b: f32x4 = f32x4::new(-1., 1., -1., 1.); + let c: f32x4 = f32x4::new(1., 1., -1., -1.); + let e: f32x4 = f32x4::new(0., -2., 0., -2.); + let r: f32x4 = transmute(vcmlaq_laneq_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmla_rot90_lane_f32() { + let a: f32x2 = f32x2::new(1., -1.); + let b: f32x2 = f32x2::new(-1., 1.); + let c: f32x2 = f32x2::new(1., 1.); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vcmla_rot90_lane_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmla_rot90_laneq_f32() { + let a: f32x2 = f32x2::new(1., -1.); + let b: f32x2 = f32x2::new(-1., 1.); + let c: f32x4 = f32x4::new(1., 1., -1., -1.); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vcmla_rot90_laneq_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmlaq_rot90_lane_f32() { + let a: f32x4 = f32x4::new(1., -1., 1., -1.); + let b: f32x4 = f32x4::new(-1., 1., -1., 1.); + let c: f32x2 = f32x2::new(1., 1.); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vcmlaq_rot90_lane_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmlaq_rot90_laneq_f32() { + let a: f32x4 = f32x4::new(1., -1., 1., -1.); + let b: f32x4 = f32x4::new(-1., 1., -1., 1.); + let c: f32x4 = f32x4::new(1., 1., -1., -1.); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vcmlaq_rot90_laneq_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmla_rot180_lane_f32() { + let a: f32x2 = f32x2::new(1., -1.); + let b: f32x2 = f32x2::new(-1., 1.); + let c: f32x2 = f32x2::new(1., 1.); + let e: f32x2 = f32x2::new(2., 0.); + let r: f32x2 = transmute(vcmla_rot180_lane_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmla_rot180_laneq_f32() { + let a: f32x2 = f32x2::new(1., -1.); + let b: f32x2 = f32x2::new(-1., 1.); + let c: f32x4 = f32x4::new(1., 1., -1., -1.); + let e: f32x2 = f32x2::new(2., 0.); + let r: f32x2 = transmute(vcmla_rot180_laneq_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmlaq_rot180_lane_f32() { + let a: f32x4 = f32x4::new(1., -1., 1., -1.); + let b: f32x4 = f32x4::new(-1., 1., -1., 1.); + let c: f32x2 = f32x2::new(1., 1.); + let e: f32x4 = f32x4::new(2., 0., 2., 0.); + let r: f32x4 = transmute(vcmlaq_rot180_lane_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmlaq_rot180_laneq_f32() { + let a: f32x4 = f32x4::new(1., -1., 1., -1.); + let b: f32x4 = f32x4::new(-1., 1., -1., 1.); + let c: f32x4 = f32x4::new(1., 1., -1., -1.); + let e: f32x4 = f32x4::new(2., 0., 2., 0.); + let r: f32x4 = transmute(vcmlaq_rot180_laneq_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmla_rot270_lane_f32() { + let a: f32x2 = f32x2::new(1., -1.); + let b: f32x2 = f32x2::new(-1., 1.); + let c: f32x2 = f32x2::new(1., 1.); + let e: f32x2 = f32x2::new(2., -2.); + let r: f32x2 = transmute(vcmla_rot270_lane_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmla_rot270_laneq_f32() { + let a: f32x2 = f32x2::new(1., -1.); + let b: f32x2 = f32x2::new(-1., 1.); + let c: f32x4 = f32x4::new(1., 1., -1., -1.); + let e: f32x2 = f32x2::new(2., -2.); + let r: f32x2 = transmute(vcmla_rot270_laneq_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmlaq_rot270_lane_f32() { + let a: f32x4 = f32x4::new(1., -1., 1., -1.); + let b: f32x4 = f32x4::new(-1., 1., -1., 1.); + let c: f32x2 = f32x2::new(1., 1.); + let e: f32x4 = f32x4::new(2., -2., 2., -2.); + let r: f32x4 = transmute(vcmlaq_rot270_lane_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcmlaq_rot270_laneq_f32() { + let a: f32x4 = f32x4::new(1., -1., 1., -1.); + let b: f32x4 = f32x4::new(-1., 1., -1., 1.); + let c: f32x4 = f32x4::new(1., 1., -1., -1.); + let e: f32x4 = f32x4::new(2., -2., 2., -2.); + let r: f32x4 = transmute(vcmlaq_rot270_laneq_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdot_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let c: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i32x2 = i32x2::new(31, 176); + let r: i32x2 = transmute(vdot_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdotq_s32() { + let a: i32x4 = i32x4::new(1, 2, 1, 2); + let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); + let c: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); + let e: i32x4 = i32x4::new(31, 176, 31, 176); + let r: i32x4 = transmute(vdotq_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdot_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let c: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u32x2 = u32x2::new(31, 176); + let r: u32x2 = transmute(vdot_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdotq_u32() { + let a: u32x4 = u32x4::new(1, 2, 1, 2); + let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); + let c: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); + let e: u32x4 = u32x4::new(31, 176, 31, 176); + let r: u32x4 = transmute(vdotq_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdot_lane_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let c: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i32x2 = i32x2::new(31, 72); + let r: i32x2 = transmute(vdot_lane_s32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdot_laneq_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let c: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); + let e: i32x2 = i32x2::new(31, 72); + let r: i32x2 = transmute(vdot_laneq_s32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdotq_lane_s32() { + let a: i32x4 = i32x4::new(1, 2, 1, 2); + let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); + let c: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i32x4 = i32x4::new(31, 72, 31, 72); + let r: i32x4 = transmute(vdotq_lane_s32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdotq_laneq_s32() { + let a: i32x4 = i32x4::new(1, 2, 1, 2); + let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); + let c: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); + let e: i32x4 = i32x4::new(31, 72, 31, 72); + let r: i32x4 = transmute(vdotq_laneq_s32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdot_lane_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let c: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u32x2 = u32x2::new(31, 72); + let r: u32x2 = transmute(vdot_lane_u32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdot_laneq_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let c: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); + let e: u32x2 = u32x2::new(31, 72); + let r: u32x2 = transmute(vdot_laneq_u32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdotq_lane_u32() { + let a: u32x4 = u32x4::new(1, 2, 1, 2); + let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); + let c: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u32x4 = u32x4::new(31, 72, 31, 72); + let r: u32x4 = transmute(vdotq_lane_u32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdotq_laneq_u32() { + let a: u32x4 = u32x4::new(1, 2, 1, 2); + let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); + let c: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); + let e: u32x4 = u32x4::new(31, 72, 31, 72); + let r: u32x4 = transmute(vdotq_laneq_u32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmax_f64() { + let a: f64 = 1.0; + let b: f64 = 0.0; + let e: f64 = 1.0; + let r: f64 = transmute(vmax_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxq_f64() { + let a: f64x2 = f64x2::new(1.0, -2.0); + let b: f64x2 = f64x2::new(0.0, 3.0); + let e: f64x2 = f64x2::new(1.0, 3.0); + let r: f64x2 = transmute(vmaxq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxnm_f64() { + let a: f64 = 1.0; + let b: f64 = 8.0; + let e: f64 = 8.0; + let r: f64 = transmute(vmaxnm_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxnmq_f64() { + let a: f64x2 = f64x2::new(1.0, 2.0); + let b: f64x2 = f64x2::new(8.0, 16.0); + let e: f64x2 = f64x2::new(8.0, 16.0); + let r: f64x2 = transmute(vmaxnmq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxnmv_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let e: f32 = 2.; + let r: f32 = transmute(vmaxnmv_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxnmvq_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let e: f64 = 2.; + let r: f64 = transmute(vmaxnmvq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxnmvq_f32() { + let a: f32x4 = f32x4::new(1., 2., 0., 1.); + let e: f32 = 2.; + let r: f32 = transmute(vmaxnmvq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmaxnm_f32() { + let a: f32x2 = f32x2::new(1.0, 2.0); + let b: f32x2 = f32x2::new(6.0, -3.0); + let e: f32x2 = f32x2::new(2.0, 6.0); + let r: f32x2 = transmute(vpmaxnm_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmaxnmq_f64() { + let a: f64x2 = f64x2::new(1.0, 2.0); + let b: f64x2 = f64x2::new(6.0, -3.0); + let e: f64x2 = f64x2::new(2.0, 6.0); + let r: f64x2 = transmute(vpmaxnmq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmaxnmq_f32() { + let a: f32x4 = f32x4::new(1.0, 2.0, 3.0, -4.0); + let b: f32x4 = f32x4::new(8.0, 16.0, -1.0, 6.0); + let e: f32x4 = f32x4::new(2.0, 3.0, 16.0, 6.0); + let r: f32x4 = transmute(vpmaxnmq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmaxnms_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let e: f32 = 2.; + let r: f32 = transmute(vpmaxnms_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmaxnmqd_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let e: f64 = 2.; + let r: f64 = transmute(vpmaxnmqd_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmaxs_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let e: f32 = 2.; + let r: f32 = transmute(vpmaxs_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmaxqd_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let e: f64 = 2.; + let r: f64 = transmute(vpmaxqd_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmin_f64() { + let a: f64 = 1.0; + let b: f64 = 0.0; + let e: f64 = 0.0; + let r: f64 = transmute(vmin_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminq_f64() { + let a: f64x2 = f64x2::new(1.0, -2.0); + let b: f64x2 = f64x2::new(0.0, 3.0); + let e: f64x2 = f64x2::new(0.0, -2.0); + let r: f64x2 = transmute(vminq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminnm_f64() { + let a: f64 = 1.0; + let b: f64 = 8.0; + let e: f64 = 1.0; + let r: f64 = transmute(vminnm_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminnmq_f64() { + let a: f64x2 = f64x2::new(1.0, 2.0); + let b: f64x2 = f64x2::new(8.0, 16.0); + let e: f64x2 = f64x2::new(1.0, 2.0); + let r: f64x2 = transmute(vminnmq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminnmv_f32() { + let a: f32x2 = f32x2::new(1., 0.); + let e: f32 = 0.; + let r: f32 = transmute(vminnmv_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminnmvq_f64() { + let a: f64x2 = f64x2::new(1., 0.); + let e: f64 = 0.; + let r: f64 = transmute(vminnmvq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminnmvq_f32() { + let a: f32x4 = f32x4::new(1., 0., 2., 3.); + let e: f32 = 0.; + let r: f32 = transmute(vminnmvq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovl_high_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 3, 4, 5, 6, 3, 4, 5, 6, 7, 8, 9, 10); + let e: i16x8 = i16x8::new(3, 4, 5, 6, 7, 8, 9, 10); + let r: i16x8 = transmute(vmovl_high_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovl_high_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 3, 4, 5, 6); + let e: i32x4 = i32x4::new(3, 4, 5, 6); + let r: i32x4 = transmute(vmovl_high_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovl_high_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let e: i64x2 = i64x2::new(3, 4); + let r: i64x2 = transmute(vmovl_high_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovl_high_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 3, 4, 5, 6, 3, 4, 5, 6, 7, 8, 9, 10); + let e: u16x8 = u16x8::new(3, 4, 5, 6, 7, 8, 9, 10); + let r: u16x8 = transmute(vmovl_high_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovl_high_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 3, 4, 5, 6); + let e: u32x4 = u32x4::new(3, 4, 5, 6); + let r: u32x4 = transmute(vmovl_high_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovl_high_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u64x2 = u64x2::new(3, 4); + let r: u64x2 = transmute(vmovl_high_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpaddq_f32() { + let a: f32x4 = f32x4::new(1., 2., 3., 4.); + let b: f32x4 = f32x4::new(3., 4., 5., 6.); + let e: f32x4 = f32x4::new(3., 7., 7., 11.); + let r: f32x4 = transmute(vpaddq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpaddq_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let b: f64x2 = f64x2::new(3., 4.); + let e: f64x2 = f64x2::new(3., 7.); + let r: f64x2 = transmute(vpaddq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpadds_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let e: f32 = 3.; + let r: f32 = transmute(vpadds_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpaddd_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let e: f64 = 3.; + let r: f64 = transmute(vpaddd_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpminnm_f32() { + let a: f32x2 = f32x2::new(1.0, 2.0); + let b: f32x2 = f32x2::new(6.0, -3.0); + let e: f32x2 = f32x2::new(1.0, -3.0); + let r: f32x2 = transmute(vpminnm_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpminnmq_f64() { + let a: f64x2 = f64x2::new(1.0, 2.0); + let b: f64x2 = f64x2::new(6.0, -3.0); + let e: f64x2 = f64x2::new(1.0, -3.0); + let r: f64x2 = transmute(vpminnmq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpminnmq_f32() { + let a: f32x4 = f32x4::new(1.0, 2.0, 3.0, -4.0); + let b: f32x4 = f32x4::new(8.0, 16.0, -1.0, 6.0); + let e: f32x4 = f32x4::new(1.0, -4.0, 8.0, -1.0); + let r: f32x4 = transmute(vpminnmq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpminnms_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let e: f32 = 1.; + let r: f32 = transmute(vpminnms_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpminnmqd_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let e: f64 = 1.; + let r: f64 = transmute(vpminnmqd_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmins_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let e: f32 = 1.; + let r: f32 = transmute(vpmins_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpminqd_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let e: f64 = 1.; + let r: f64 = transmute(vpminqd_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmullh_s16() { + let a: i16 = 2; + let b: i16 = 3; + let e: i32 = 12; + let r: i32 = transmute(vqdmullh_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulls_s32() { + let a: i32 = 2; + let b: i32 = 3; + let e: i64 = 12; + let r: i64 = transmute(vqdmulls_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_high_s16() { + let a: i16x8 = i16x8::new(0, 1, 4, 5, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(1, 2, 5, 6, 5, 6, 7, 8); + let e: i32x4 = i32x4::new(40, 60, 84, 112); + let r: i32x4 = transmute(vqdmull_high_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_high_s32() { + let a: i32x4 = i32x4::new(0, 1, 4, 5); + let b: i32x4 = i32x4::new(1, 2, 5, 6); + let e: i64x2 = i64x2::new(40, 60); + let r: i64x2 = transmute(vqdmull_high_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_high_n_s16() { + let a: i16x8 = i16x8::new(0, 2, 8, 10, 8, 10, 12, 14); + let b: i16 = 2; + let e: i32x4 = i32x4::new(32, 40, 48, 56); + let r: i32x4 = transmute(vqdmull_high_n_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_high_n_s32() { + let a: i32x4 = i32x4::new(0, 2, 8, 10); + let b: i32 = 2; + let e: i64x2 = i64x2::new(32, 40); + let r: i64x2 = transmute(vqdmull_high_n_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_laneq_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x8 = i16x8::new(0, 2, 2, 0, 2, 0, 0, 0); + let e: i32x4 = i32x4::new(4, 8, 12, 16); + let r: i32x4 = transmute(vqdmull_laneq_s16::<4>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_laneq_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x4 = i32x4::new(0, 2, 2, 0); + let e: i64x2 = i64x2::new(4, 8); + let r: i64x2 = transmute(vqdmull_laneq_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmullh_lane_s16() { + let a: i16 = 2; + let b: i16x4 = i16x4::new(0, 2, 2, 0); + let e: i32 = 8; + let r: i32 = transmute(vqdmullh_lane_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmullh_laneq_s16() { + let a: i16 = 2; + let b: i16x8 = i16x8::new(0, 2, 2, 0, 2, 0, 0, 0); + let e: i32 = 8; + let r: i32 = transmute(vqdmullh_laneq_s16::<4>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulls_lane_s32() { + let a: i32 = 2; + let b: i32x2 = i32x2::new(0, 2); + let e: i64 = 8; + let r: i64 = transmute(vqdmulls_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulls_laneq_s32() { + let a: i32 = 2; + let b: i32x4 = i32x4::new(0, 2, 2, 0); + let e: i64 = 8; + let r: i64 = transmute(vqdmulls_laneq_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_high_lane_s16() { + let a: i16x8 = i16x8::new(0, 1, 4, 5, 4, 5, 6, 7); + let b: i16x4 = i16x4::new(0, 2, 2, 0); + let e: i32x4 = i32x4::new(16, 20, 24, 28); + let r: i32x4 = transmute(vqdmull_high_lane_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_high_lane_s32() { + let a: i32x4 = i32x4::new(0, 1, 4, 5); + let b: i32x2 = i32x2::new(0, 2); + let e: i64x2 = i64x2::new(16, 20); + let r: i64x2 = transmute(vqdmull_high_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_high_laneq_s16() { + let a: i16x8 = i16x8::new(0, 1, 4, 5, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(0, 2, 2, 0, 2, 0, 0, 0); + let e: i32x4 = i32x4::new(16, 20, 24, 28); + let r: i32x4 = transmute(vqdmull_high_laneq_s16::<4>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_high_laneq_s32() { + let a: i32x4 = i32x4::new(0, 1, 4, 5); + let b: i32x4 = i32x4::new(0, 2, 2, 0); + let e: i64x2 = i64x2::new(16, 20); + let r: i64x2 = transmute(vqdmull_high_laneq_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_high_s16() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i16x8 = i16x8::new(0, 1, 4, 5, 4, 5, 6, 7); + let c: i16x8 = i16x8::new(1, 2, 5, 6, 5, 6, 7, 8); + let e: i32x4 = i32x4::new(41, 62, 87, 116); + let r: i32x4 = transmute(vqdmlal_high_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_high_s32() { + let a: i64x2 = i64x2::new(1, 2); + let b: i32x4 = i32x4::new(0, 1, 4, 5); + let c: i32x4 = i32x4::new(1, 2, 5, 6); + let e: i64x2 = i64x2::new(41, 62); + let r: i64x2 = transmute(vqdmlal_high_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_high_n_s16() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i16x8 = i16x8::new(0, 2, 8, 10, 8, 10, 12, 14); + let c: i16 = 2; + let e: i32x4 = i32x4::new(33, 42, 51, 60); + let r: i32x4 = transmute(vqdmlal_high_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_high_n_s32() { + let a: i64x2 = i64x2::new(1, 2); + let b: i32x4 = i32x4::new(0, 2, 8, 10); + let c: i32 = 2; + let e: i64x2 = i64x2::new(33, 42); + let r: i64x2 = transmute(vqdmlal_high_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_laneq_s16() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let c: i16x8 = i16x8::new(0, 2, 2, 0, 2, 0, 0, 0); + let e: i32x4 = i32x4::new(5, 10, 15, 20); + let r: i32x4 = transmute(vqdmlal_laneq_s16::<2>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_laneq_s32() { + let a: i64x2 = i64x2::new(1, 2); + let b: i32x2 = i32x2::new(1, 2); + let c: i32x4 = i32x4::new(0, 2, 2, 0); + let e: i64x2 = i64x2::new(5, 10); + let r: i64x2 = transmute(vqdmlal_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_high_lane_s16() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i16x8 = i16x8::new(0, 1, 4, 5, 4, 5, 6, 7); + let c: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i32x4 = i32x4::new(17, 22, 27, 32); + let r: i32x4 = transmute(vqdmlal_high_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_high_laneq_s16() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i16x8 = i16x8::new(0, 1, 4, 5, 4, 5, 6, 7); + let c: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i32x4 = i32x4::new(17, 22, 27, 32); + let r: i32x4 = transmute(vqdmlal_high_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_high_lane_s32() { + let a: i64x2 = i64x2::new(1, 2); + let b: i32x4 = i32x4::new(0, 1, 4, 5); + let c: i32x2 = i32x2::new(0, 2); + let e: i64x2 = i64x2::new(17, 22); + let r: i64x2 = transmute(vqdmlal_high_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_high_laneq_s32() { + let a: i64x2 = i64x2::new(1, 2); + let b: i32x4 = i32x4::new(0, 1, 4, 5); + let c: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i64x2 = i64x2::new(17, 22); + let r: i64x2 = transmute(vqdmlal_high_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlalh_s16() { + let a: i32 = 1; + let b: i16 = 1; + let c: i16 = 2; + let e: i32 = 5; + let r: i32 = transmute(vqdmlalh_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlals_s32() { + let a: i64 = 1; + let b: i32 = 1; + let c: i32 = 2; + let e: i64 = 5; + let r: i64 = transmute(vqdmlals_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlalh_lane_s16() { + let a: i32 = 1; + let b: i16 = 1; + let c: i16x4 = i16x4::new(2, 1, 1, 1); + let e: i32 = 5; + let r: i32 = transmute(vqdmlalh_lane_s16::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlalh_laneq_s16() { + let a: i32 = 1; + let b: i16 = 1; + let c: i16x8 = i16x8::new(2, 1, 1, 1, 1, 1, 1, 1); + let e: i32 = 5; + let r: i32 = transmute(vqdmlalh_laneq_s16::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlals_lane_s32() { + let a: i64 = 1; + let b: i32 = 1; + let c: i32x2 = i32x2::new(2, 1); + let e: i64 = 5; + let r: i64 = transmute(vqdmlals_lane_s32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlals_laneq_s32() { + let a: i64 = 1; + let b: i32 = 1; + let c: i32x4 = i32x4::new(2, 1, 1, 1); + let e: i64 = 5; + let r: i64 = transmute(vqdmlals_laneq_s32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_high_s16() { + let a: i32x4 = i32x4::new(39, 58, 81, 108); + let b: i16x8 = i16x8::new(0, 1, 4, 5, 4, 5, 6, 7); + let c: i16x8 = i16x8::new(1, 2, 5, 6, 5, 6, 7, 8); + let e: i32x4 = i32x4::new(-1, -2, -3, -4); + let r: i32x4 = transmute(vqdmlsl_high_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_high_s32() { + let a: i64x2 = i64x2::new(39, 58); + let b: i32x4 = i32x4::new(0, 1, 4, 5); + let c: i32x4 = i32x4::new(1, 2, 5, 6); + let e: i64x2 = i64x2::new(-1, -2); + let r: i64x2 = transmute(vqdmlsl_high_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_high_n_s16() { + let a: i32x4 = i32x4::new(31, 38, 45, 52); + let b: i16x8 = i16x8::new(0, 2, 8, 10, 8, 10, 12, 14); + let c: i16 = 2; + let e: i32x4 = i32x4::new(-1, -2, -3, -4); + let r: i32x4 = transmute(vqdmlsl_high_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_high_n_s32() { + let a: i64x2 = i64x2::new(31, 38); + let b: i32x4 = i32x4::new(0, 2, 8, 10); + let c: i32 = 2; + let e: i64x2 = i64x2::new(-1, -2); + let r: i64x2 = transmute(vqdmlsl_high_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_laneq_s16() { + let a: i32x4 = i32x4::new(3, 6, 9, 12); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let c: i16x8 = i16x8::new(0, 2, 2, 0, 2, 0, 0, 0); + let e: i32x4 = i32x4::new(-1, -2, -3, -4); + let r: i32x4 = transmute(vqdmlsl_laneq_s16::<2>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_laneq_s32() { + let a: i64x2 = i64x2::new(3, 6); + let b: i32x2 = i32x2::new(1, 2); + let c: i32x4 = i32x4::new(0, 2, 2, 0); + let e: i64x2 = i64x2::new(-1, -2); + let r: i64x2 = transmute(vqdmlsl_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_high_lane_s16() { + let a: i32x4 = i32x4::new(15, 18, 21, 24); + let b: i16x8 = i16x8::new(0, 1, 4, 5, 4, 5, 6, 7); + let c: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i32x4 = i32x4::new(-1, -2, -3, -4); + let r: i32x4 = transmute(vqdmlsl_high_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_high_laneq_s16() { + let a: i32x4 = i32x4::new(15, 18, 21, 24); + let b: i16x8 = i16x8::new(0, 1, 4, 5, 4, 5, 6, 7); + let c: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i32x4 = i32x4::new(-1, -2, -3, -4); + let r: i32x4 = transmute(vqdmlsl_high_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_high_lane_s32() { + let a: i64x2 = i64x2::new(15, 18); + let b: i32x4 = i32x4::new(0, 1, 4, 5); + let c: i32x2 = i32x2::new(0, 2); + let e: i64x2 = i64x2::new(-1, -2); + let r: i64x2 = transmute(vqdmlsl_high_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_high_laneq_s32() { + let a: i64x2 = i64x2::new(15, 18); + let b: i32x4 = i32x4::new(0, 1, 4, 5); + let c: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i64x2 = i64x2::new(-1, -2); + let r: i64x2 = transmute(vqdmlsl_high_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlslh_s16() { + let a: i32 = 10; + let b: i16 = 1; + let c: i16 = 2; + let e: i32 = 6; + let r: i32 = transmute(vqdmlslh_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsls_s32() { + let a: i64 = 10; + let b: i32 = 1; + let c: i32 = 2; + let e: i64 = 6; + let r: i64 = transmute(vqdmlsls_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlslh_lane_s16() { + let a: i32 = 10; + let b: i16 = 1; + let c: i16x4 = i16x4::new(2, 1, 1, 1); + let e: i32 = 6; + let r: i32 = transmute(vqdmlslh_lane_s16::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlslh_laneq_s16() { + let a: i32 = 10; + let b: i16 = 1; + let c: i16x8 = i16x8::new(2, 1, 1, 1, 1, 1, 1, 1); + let e: i32 = 6; + let r: i32 = transmute(vqdmlslh_laneq_s16::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsls_lane_s32() { + let a: i64 = 10; + let b: i32 = 1; + let c: i32x2 = i32x2::new(2, 1); + let e: i64 = 6; + let r: i64 = transmute(vqdmlsls_lane_s32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsls_laneq_s32() { + let a: i64 = 10; + let b: i32 = 1; + let c: i32x4 = i32x4::new(2, 1, 1, 1); + let e: i64 = 6; + let r: i64 = transmute(vqdmlsls_laneq_s32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhh_s16() { + let a: i16 = 1; + let b: i16 = 2; + let e: i16 = 0; + let r: i16 = transmute(vqdmulhh_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhs_s32() { + let a: i32 = 1; + let b: i32 = 2; + let e: i32 = 0; + let r: i32 = transmute(vqdmulhs_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhh_lane_s16() { + let a: i16 = 2; + let b: i16x4 = i16x4::new(0, 0, 0x7F_FF, 0); + let e: i16 = 1; + let r: i16 = transmute(vqdmulhh_lane_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhh_laneq_s16() { + let a: i16 = 2; + let b: i16x8 = i16x8::new(0, 0, 0x7F_FF, 0, 0, 0, 0, 0); + let e: i16 = 1; + let r: i16 = transmute(vqdmulhh_laneq_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhs_lane_s32() { + let a: i32 = 2; + let b: i32x2 = i32x2::new(0, 0x7F_FF_FF_FF); + let e: i32 = 1; + let r: i32 = transmute(vqdmulhs_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhs_laneq_s32() { + let a: i32 = 2; + let b: i32x4 = i32x4::new(0, 0x7F_FF_FF_FF, 0, 0); + let e: i32 = 1; + let r: i32 = transmute(vqdmulhs_laneq_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulh_lane_s16() { + let a: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16x4 = i16x4::new(2, 1, 1, 1); + let e: i16x4 = i16x4::new(1, 1, 1, 1); + let r: i16x4 = transmute(vqdmulh_lane_s16::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhq_lane_s16() { + let a: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16x4 = i16x4::new(2, 1, 1, 1); + let e: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i16x8 = transmute(vqdmulhq_lane_s16::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulh_lane_s32() { + let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32x2 = i32x2::new(2, 1); + let e: i32x2 = i32x2::new(1, 1); + let r: i32x2 = transmute(vqdmulh_lane_s32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhq_lane_s32() { + let a: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32x2 = i32x2::new(2, 1); + let e: i32x4 = i32x4::new(1, 1, 1, 1); + let r: i32x4 = transmute(vqdmulhq_lane_s32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovnh_s16() { + let a: i16 = 1; + let e: i8 = 1; + let r: i8 = transmute(vqmovnh_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovns_s32() { + let a: i32 = 1; + let e: i16 = 1; + let r: i16 = transmute(vqmovns_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovnh_u16() { + let a: u16 = 1; + let e: u8 = 1; + let r: u8 = transmute(vqmovnh_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovns_u32() { + let a: u32 = 1; + let e: u16 = 1; + let r: u16 = transmute(vqmovns_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovnd_s64() { + let a: i64 = 1; + let e: i32 = 1; + let r: i32 = transmute(vqmovnd_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovnd_u64() { + let a: u64 = 1; + let e: u32 = 1; + let r: u32 = transmute(vqmovnd_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_high_s16() { + let a: i8x8 = i8x8::new(0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F); + let b: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let e: i8x16 = i8x16::new(0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F); + let r: i8x16 = transmute(vqmovn_high_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_high_s32() { + let a: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let e: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let r: i16x8 = transmute(vqmovn_high_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_high_s64() { + let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, 0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let r: i32x4 = transmute(vqmovn_high_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_high_u16() { + let a: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let b: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vqmovn_high_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_high_u32() { + let a: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let b: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vqmovn_high_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_high_u64() { + let a: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let b: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vqmovn_high_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovunh_s16() { + let a: i16 = 1; + let e: u8 = 1; + let r: u8 = transmute(vqmovunh_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovuns_s32() { + let a: i32 = 1; + let e: u16 = 1; + let r: u16 = transmute(vqmovuns_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovund_s64() { + let a: i64 = 1; + let e: u32 = 1; + let r: u32 = transmute(vqmovund_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovun_high_s16() { + let a: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let b: i16x8 = i16x8::new(-1, -1, -1, -1, -1, -1, -1, -1); + let e: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vqmovun_high_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovun_high_s32() { + let a: u16x4 = u16x4::new(0, 0, 0, 0); + let b: i32x4 = i32x4::new(-1, -1, -1, -1); + let e: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vqmovun_high_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovun_high_s64() { + let a: u32x2 = u32x2::new(0, 0); + let b: i64x2 = i64x2::new(-1, -1); + let e: u32x4 = u32x4::new(0, 0, 0, 0); + let r: u32x4 = transmute(vqmovun_high_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhh_s16() { + let a: i16 = 1; + let b: i16 = 2; + let e: i16 = 0; + let r: i16 = transmute(vqrdmulhh_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhs_s32() { + let a: i32 = 1; + let b: i32 = 2; + let e: i32 = 0; + let r: i32 = transmute(vqrdmulhs_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhh_lane_s16() { + let a: i16 = 1; + let b: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16 = 0; + let r: i16 = transmute(vqrdmulhh_lane_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhh_laneq_s16() { + let a: i16 = 1; + let b: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16 = 0; + let r: i16 = transmute(vqrdmulhh_laneq_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhs_lane_s32() { + let a: i32 = 1; + let b: i32x2 = i32x2::new(0, 2); + let e: i32 = 0; + let r: i32 = transmute(vqrdmulhs_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhs_laneq_s32() { + let a: i32 = 1; + let b: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32 = 0; + let r: i32 = transmute(vqrdmulhs_laneq_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlah_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i16x4 = i16x4::new(3, 3, 3, 3); + let r: i16x4 = transmute(vqrdmlah_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahq_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i16x8 = i16x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let r: i16x8 = transmute(vqrdmlahq_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlah_s32() { + let a: i32x2 = i32x2::new(1, 1); + let b: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x2 = i32x2::new(2, 2); + let e: i32x2 = i32x2::new(3, 3); + let r: i32x2 = transmute(vqrdmlah_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahq_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 1); + let b: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x4 = i32x4::new(2, 2, 2, 2); + let e: i32x4 = i32x4::new(3, 3, 3, 3); + let r: i32x4 = transmute(vqrdmlahq_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahh_s16() { + let a: i16 = 1; + let b: i16 = 1; + let c: i16 = 2; + let e: i16 = 1; + let r: i16 = transmute(vqrdmlahh_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahs_s32() { + let a: i32 = 1; + let b: i32 = 1; + let c: i32 = 2; + let e: i32 = 1; + let r: i32 = transmute(vqrdmlahs_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlah_lane_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16x4 = i16x4::new(3, 3, 3, 3); + let r: i16x4 = transmute(vqrdmlah_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlah_laneq_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16x4 = i16x4::new(3, 3, 3, 3); + let r: i16x4 = transmute(vqrdmlah_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahq_lane_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16x8 = i16x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let r: i16x8 = transmute(vqrdmlahq_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahq_laneq_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16x8 = i16x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let r: i16x8 = transmute(vqrdmlahq_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlah_lane_s32() { + let a: i32x2 = i32x2::new(1, 1); + let b: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x2 = i32x2::new(0, 2); + let e: i32x2 = i32x2::new(3, 3); + let r: i32x2 = transmute(vqrdmlah_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlah_laneq_s32() { + let a: i32x2 = i32x2::new(1, 1); + let b: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32x2 = i32x2::new(3, 3); + let r: i32x2 = transmute(vqrdmlah_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahq_lane_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 1); + let b: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x2 = i32x2::new(0, 2); + let e: i32x4 = i32x4::new(3, 3, 3, 3); + let r: i32x4 = transmute(vqrdmlahq_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahq_laneq_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 1); + let b: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32x4 = i32x4::new(3, 3, 3, 3); + let r: i32x4 = transmute(vqrdmlahq_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahh_lane_s16() { + let a: i16 = 1; + let b: i16 = 1; + let c: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16 = 1; + let r: i16 = transmute(vqrdmlahh_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahh_laneq_s16() { + let a: i16 = 1; + let b: i16 = 1; + let c: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16 = 1; + let r: i16 = transmute(vqrdmlahh_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahs_lane_s32() { + let a: i32 = 1; + let b: i32 = 1; + let c: i32x2 = i32x2::new(0, 2); + let e: i32 = 1; + let r: i32 = transmute(vqrdmlahs_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahs_laneq_s32() { + let a: i32 = 1; + let b: i32 = 1; + let c: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32 = 1; + let r: i32 = transmute(vqrdmlahs_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlsh_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i16x4 = i16x4::new(-1, -1, -1, -1); + let r: i16x4 = transmute(vqrdmlsh_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshq_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i16x8 = i16x8::new(-1, -1, -1, -1, -1, -1, -1, -1); + let r: i16x8 = transmute(vqrdmlshq_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlsh_s32() { + let a: i32x2 = i32x2::new(1, 1); + let b: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x2 = i32x2::new(2, 2); + let e: i32x2 = i32x2::new(-1, -1); + let r: i32x2 = transmute(vqrdmlsh_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshq_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 1); + let b: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x4 = i32x4::new(2, 2, 2, 2); + let e: i32x4 = i32x4::new(-1, -1, -1, -1); + let r: i32x4 = transmute(vqrdmlshq_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshh_s16() { + let a: i16 = 1; + let b: i16 = 1; + let c: i16 = 2; + let e: i16 = 1; + let r: i16 = transmute(vqrdmlshh_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshs_s32() { + let a: i32 = 1; + let b: i32 = 1; + let c: i32 = 2; + let e: i32 = 1; + let r: i32 = transmute(vqrdmlshs_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlsh_lane_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16x4 = i16x4::new(-1, -1, -1, -1); + let r: i16x4 = transmute(vqrdmlsh_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlsh_laneq_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16x4 = i16x4::new(-1, -1, -1, -1); + let r: i16x4 = transmute(vqrdmlsh_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshq_lane_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16x8 = i16x8::new(-1, -1, -1, -1, -1, -1, -1, -1); + let r: i16x8 = transmute(vqrdmlshq_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshq_laneq_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16x8 = i16x8::new(-1, -1, -1, -1, -1, -1, -1, -1); + let r: i16x8 = transmute(vqrdmlshq_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlsh_lane_s32() { + let a: i32x2 = i32x2::new(1, 1); + let b: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x2 = i32x2::new(0, 2); + let e: i32x2 = i32x2::new(-1, -1); + let r: i32x2 = transmute(vqrdmlsh_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlsh_laneq_s32() { + let a: i32x2 = i32x2::new(1, 1); + let b: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32x2 = i32x2::new(-1, -1); + let r: i32x2 = transmute(vqrdmlsh_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshq_lane_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 1); + let b: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x2 = i32x2::new(0, 2); + let e: i32x4 = i32x4::new(-1, -1, -1, -1); + let r: i32x4 = transmute(vqrdmlshq_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshq_laneq_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 1); + let b: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32x4 = i32x4::new(-1, -1, -1, -1); + let r: i32x4 = transmute(vqrdmlshq_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshh_lane_s16() { + let a: i16 = 1; + let b: i16 = 1; + let c: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16 = 1; + let r: i16 = transmute(vqrdmlshh_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshh_laneq_s16() { + let a: i16 = 1; + let b: i16 = 1; + let c: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16 = 1; + let r: i16 = transmute(vqrdmlshh_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshs_lane_s32() { + let a: i32 = 1; + let b: i32 = 1; + let c: i32x2 = i32x2::new(0, 2); + let e: i32 = 1; + let r: i32 = transmute(vqrdmlshs_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshs_laneq_s32() { + let a: i32 = 1; + let b: i32 = 1; + let c: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32 = 1; + let r: i32 = transmute(vqrdmlshs_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshls_s32() { + let a: i32 = 2; + let b: i32 = 2; + let e: i32 = 8; + let r: i32 = transmute(vqrshls_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshld_s64() { + let a: i64 = 2; + let b: i64 = 2; + let e: i64 = 8; + let r: i64 = transmute(vqrshld_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlb_s8() { + let a: i8 = 1; + let b: i8 = 2; + let e: i8 = 4; + let r: i8 = transmute(vqrshlb_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlh_s16() { + let a: i16 = 1; + let b: i16 = 2; + let e: i16 = 4; + let r: i16 = transmute(vqrshlh_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshls_u32() { + let a: u32 = 2; + let b: i32 = 2; + let e: u32 = 8; + let r: u32 = transmute(vqrshls_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshld_u64() { + let a: u64 = 2; + let b: i64 = 2; + let e: u64 = 8; + let r: u64 = transmute(vqrshld_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlb_u8() { + let a: u8 = 1; + let b: i8 = 2; + let e: u8 = 4; + let r: u8 = transmute(vqrshlb_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlh_u16() { + let a: u16 = 1; + let b: i16 = 2; + let e: u16 = 4; + let r: u16 = transmute(vqrshlh_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrnh_n_s16() { + let a: i16 = 4; + let e: i8 = 1; + let r: i8 = transmute(vqrshrnh_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrns_n_s32() { + let a: i32 = 4; + let e: i16 = 1; + let r: i16 = transmute(vqrshrns_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrnd_n_s64() { + let a: i64 = 4; + let e: i32 = 1; + let r: i32 = transmute(vqrshrnd_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_high_n_s16() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 2, 3, 6, 7); + let b: i16x8 = i16x8::new(8, 12, 24, 28, 48, 52, 56, 60); + let e: i8x16 = i8x16::new(0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 12, 13, 14, 15); + let r: i8x16 = transmute(vqrshrn_high_n_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_high_n_s32() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(8, 12, 24, 28); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 2, 3, 6, 7); + let r: i16x8 = transmute(vqrshrn_high_n_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_high_n_s64() { + let a: i32x2 = i32x2::new(0, 1); + let b: i64x2 = i64x2::new(8, 12); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vqrshrn_high_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrnh_n_u16() { + let a: u16 = 4; + let e: u8 = 1; + let r: u8 = transmute(vqrshrnh_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrns_n_u32() { + let a: u32 = 4; + let e: u16 = 1; + let r: u16 = transmute(vqrshrns_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrnd_n_u64() { + let a: u64 = 4; + let e: u32 = 1; + let r: u32 = transmute(vqrshrnd_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_high_n_u16() { + let a: u8x8 = u8x8::new(0, 1, 2, 3, 2, 3, 6, 7); + let b: u16x8 = u16x8::new(8, 12, 24, 28, 48, 52, 56, 60); + let e: u8x16 = u8x16::new(0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 12, 13, 14, 15); + let r: u8x16 = transmute(vqrshrn_high_n_u16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_high_n_u32() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let b: u32x4 = u32x4::new(8, 12, 24, 28); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 2, 3, 6, 7); + let r: u16x8 = transmute(vqrshrn_high_n_u32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_high_n_u64() { + let a: u32x2 = u32x2::new(0, 1); + let b: u64x2 = u64x2::new(8, 12); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vqrshrn_high_n_u64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrunh_n_s16() { + let a: i16 = 4; + let e: u8 = 1; + let r: u8 = transmute(vqrshrunh_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshruns_n_s32() { + let a: i32 = 4; + let e: u16 = 1; + let r: u16 = transmute(vqrshruns_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrund_n_s64() { + let a: i64 = 4; + let e: u32 = 1; + let r: u32 = transmute(vqrshrund_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrun_high_n_s16() { + let a: u8x8 = u8x8::new(0, 1, 2, 3, 2, 3, 6, 7); + let b: i16x8 = i16x8::new(8, 12, 24, 28, 48, 52, 56, 60); + let e: u8x16 = u8x16::new(0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 12, 13, 14, 15); + let r: u8x16 = transmute(vqrshrun_high_n_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrun_high_n_s32() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(8, 12, 24, 28); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 2, 3, 6, 7); + let r: u16x8 = transmute(vqrshrun_high_n_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrun_high_n_s64() { + let a: u32x2 = u32x2::new(0, 1); + let b: i64x2 = i64x2::new(8, 12); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vqrshrun_high_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshld_s64() { + let a: i64 = 0; + let b: i64 = 2; + let e: i64 = 0; + let r: i64 = transmute(vqshld_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlb_s8() { + let a: i8 = 1; + let b: i8 = 2; + let e: i8 = 4; + let r: i8 = transmute(vqshlb_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlh_s16() { + let a: i16 = 1; + let b: i16 = 2; + let e: i16 = 4; + let r: i16 = transmute(vqshlh_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshls_s32() { + let a: i32 = 1; + let b: i32 = 2; + let e: i32 = 4; + let r: i32 = transmute(vqshls_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshld_u64() { + let a: u64 = 0; + let b: i64 = 2; + let e: u64 = 0; + let r: u64 = transmute(vqshld_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlb_u8() { + let a: u8 = 1; + let b: i8 = 2; + let e: u8 = 4; + let r: u8 = transmute(vqshlb_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlh_u16() { + let a: u16 = 1; + let b: i16 = 2; + let e: u16 = 4; + let r: u16 = transmute(vqshlh_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshls_u32() { + let a: u32 = 1; + let b: i32 = 2; + let e: u32 = 4; + let r: u32 = transmute(vqshls_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlb_n_s8() { + let a: i8 = 1; + let e: i8 = 4; + let r: i8 = transmute(vqshlb_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlh_n_s16() { + let a: i16 = 1; + let e: i16 = 4; + let r: i16 = transmute(vqshlh_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshls_n_s32() { + let a: i32 = 1; + let e: i32 = 4; + let r: i32 = transmute(vqshls_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshld_n_s64() { + let a: i64 = 1; + let e: i64 = 4; + let r: i64 = transmute(vqshld_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlb_n_u8() { + let a: u8 = 1; + let e: u8 = 4; + let r: u8 = transmute(vqshlb_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlh_n_u16() { + let a: u16 = 1; + let e: u16 = 4; + let r: u16 = transmute(vqshlh_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshls_n_u32() { + let a: u32 = 1; + let e: u32 = 4; + let r: u32 = transmute(vqshls_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshld_n_u64() { + let a: u64 = 1; + let e: u64 = 4; + let r: u64 = transmute(vqshld_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlub_n_s8() { + let a: i8 = 1; + let e: u8 = 4; + let r: u8 = transmute(vqshlub_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshluh_n_s16() { + let a: i16 = 1; + let e: u16 = 4; + let r: u16 = transmute(vqshluh_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlus_n_s32() { + let a: i32 = 1; + let e: u32 = 4; + let r: u32 = transmute(vqshlus_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlud_n_s64() { + let a: i64 = 1; + let e: u64 = 4; + let r: u64 = transmute(vqshlud_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrnd_n_s64() { + let a: i64 = 0; + let e: i32 = 0; + let r: i32 = transmute(vqshrnd_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrnh_n_s16() { + let a: i16 = 4; + let e: i8 = 1; + let r: i8 = transmute(vqshrnh_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrns_n_s32() { + let a: i32 = 4; + let e: i16 = 1; + let r: i16 = transmute(vqshrns_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_high_n_s16() { + let a: i8x8 = i8x8::new(0, 1, 8, 9, 8, 9, 10, 11); + let b: i16x8 = i16x8::new(32, 36, 40, 44, 48, 52, 56, 60); + let e: i8x16 = i8x16::new(0, 1, 8, 9, 8, 9, 10, 11, 8, 9, 10, 11, 12, 13, 14, 15); + let r: i8x16 = transmute(vqshrn_high_n_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_high_n_s32() { + let a: i16x4 = i16x4::new(0, 1, 8, 9); + let b: i32x4 = i32x4::new(32, 36, 40, 44); + let e: i16x8 = i16x8::new(0, 1, 8, 9, 8, 9, 10, 11); + let r: i16x8 = transmute(vqshrn_high_n_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_high_n_s64() { + let a: i32x2 = i32x2::new(0, 1); + let b: i64x2 = i64x2::new(32, 36); + let e: i32x4 = i32x4::new(0, 1, 8, 9); + let r: i32x4 = transmute(vqshrn_high_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrnd_n_u64() { + let a: u64 = 0; + let e: u32 = 0; + let r: u32 = transmute(vqshrnd_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrnh_n_u16() { + let a: u16 = 4; + let e: u8 = 1; + let r: u8 = transmute(vqshrnh_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrns_n_u32() { + let a: u32 = 4; + let e: u16 = 1; + let r: u16 = transmute(vqshrns_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_high_n_u16() { + let a: u8x8 = u8x8::new(0, 1, 8, 9, 8, 9, 10, 11); + let b: u16x8 = u16x8::new(32, 36, 40, 44, 48, 52, 56, 60); + let e: u8x16 = u8x16::new(0, 1, 8, 9, 8, 9, 10, 11, 8, 9, 10, 11, 12, 13, 14, 15); + let r: u8x16 = transmute(vqshrn_high_n_u16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_high_n_u32() { + let a: u16x4 = u16x4::new(0, 1, 8, 9); + let b: u32x4 = u32x4::new(32, 36, 40, 44); + let e: u16x8 = u16x8::new(0, 1, 8, 9, 8, 9, 10, 11); + let r: u16x8 = transmute(vqshrn_high_n_u32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_high_n_u64() { + let a: u32x2 = u32x2::new(0, 1); + let b: u64x2 = u64x2::new(32, 36); + let e: u32x4 = u32x4::new(0, 1, 8, 9); + let r: u32x4 = transmute(vqshrn_high_n_u64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrunh_n_s16() { + let a: i16 = 4; + let e: u8 = 1; + let r: u8 = transmute(vqshrunh_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshruns_n_s32() { + let a: i32 = 4; + let e: u16 = 1; + let r: u16 = transmute(vqshruns_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrund_n_s64() { + let a: i64 = 4; + let e: u32 = 1; + let r: u32 = transmute(vqshrund_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrun_high_n_s16() { + let a: u8x8 = u8x8::new(0, 1, 8, 9, 8, 9, 10, 11); + let b: i16x8 = i16x8::new(32, 36, 40, 44, 48, 52, 56, 60); + let e: u8x16 = u8x16::new(0, 1, 8, 9, 8, 9, 10, 11, 8, 9, 10, 11, 12, 13, 14, 15); + let r: u8x16 = transmute(vqshrun_high_n_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrun_high_n_s32() { + let a: u16x4 = u16x4::new(0, 1, 8, 9); + let b: i32x4 = i32x4::new(32, 36, 40, 44); + let e: u16x8 = u16x8::new(0, 1, 8, 9, 8, 9, 10, 11); + let r: u16x8 = transmute(vqshrun_high_n_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrun_high_n_s64() { + let a: u32x2 = u32x2::new(0, 1); + let b: i64x2 = i64x2::new(32, 36); + let e: u32x4 = u32x4::new(0, 1, 8, 9); + let r: u32x4 = transmute(vqshrun_high_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsqaddb_u8() { + let a: u8 = 2; + let b: i8 = 2; + let e: u8 = 4; + let r: u8 = transmute(vsqaddb_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsqaddh_u16() { + let a: u16 = 2; + let b: i16 = 2; + let e: u16 = 4; + let r: u16 = transmute(vsqaddh_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsqadds_u32() { + let a: u32 = 2; + let b: i32 = 2; + let e: u32 = 4; + let r: u32 = transmute(vsqadds_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsqaddd_u64() { + let a: u64 = 2; + let b: i64 = 2; + let e: u64 = 4; + let r: u64 = transmute(vsqaddd_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsqrt_f32() { + let a: f32x2 = f32x2::new(4.0, 9.0); + let e: f32x2 = f32x2::new(2.0, 3.0); + let r: f32x2 = transmute(vsqrt_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsqrtq_f32() { + let a: f32x4 = f32x4::new(4.0, 9.0, 16.0, 25.0); + let e: f32x4 = f32x4::new(2.0, 3.0, 4.0, 5.0); + let r: f32x4 = transmute(vsqrtq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsqrt_f64() { + let a: f64 = 4.0; + let e: f64 = 2.0; + let r: f64 = transmute(vsqrt_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsqrtq_f64() { + let a: f64x2 = f64x2::new(4.0, 9.0); + let e: f64x2 = f64x2::new(2.0, 3.0); + let r: f64x2 = transmute(vsqrtq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsqrte_f64() { + let a: f64 = 1.0; + let e: f64 = 0.998046875; + let r: f64 = transmute(vrsqrte_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsqrteq_f64() { + let a: f64x2 = f64x2::new(1.0, 2.0); + let e: f64x2 = f64x2::new(0.998046875, 0.705078125); + let r: f64x2 = transmute(vrsqrteq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsqrtes_f32() { + let a: f32 = 1.0; + let e: f32 = 0.998046875; + let r: f32 = transmute(vrsqrtes_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsqrted_f64() { + let a: f64 = 1.0; + let e: f64 = 0.998046875; + let r: f64 = transmute(vrsqrted_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsqrts_f64() { + let a: f64 = 1.0; + let b: f64 = 1.0; + let e: f64 = 1.; + let r: f64 = transmute(vrsqrts_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsqrtsq_f64() { + let a: f64x2 = f64x2::new(1.0, 2.0); + let b: f64x2 = f64x2::new(1.0, 2.0); + let e: f64x2 = f64x2::new(1., -0.5); + let r: f64x2 = transmute(vrsqrtsq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsqrtss_f32() { + let a: f32 = 1.0; + let b: f32 = 1.0; + let e: f32 = 1.; + let r: f32 = transmute(vrsqrtss_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsqrtsd_f64() { + let a: f64 = 1.0; + let b: f64 = 1.0; + let e: f64 = 1.; + let r: f64 = transmute(vrsqrtsd_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrecpe_f64() { + let a: f64 = 4.0; + let e: f64 = 0.24951171875; + let r: f64 = transmute(vrecpe_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrecpeq_f64() { + let a: f64x2 = f64x2::new(4.0, 3.0); + let e: f64x2 = f64x2::new(0.24951171875, 0.3330078125); + let r: f64x2 = transmute(vrecpeq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrecpes_f32() { + let a: f32 = 4.0; + let e: f32 = 0.24951171875; + let r: f32 = transmute(vrecpes_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrecped_f64() { + let a: f64 = 4.0; + let e: f64 = 0.24951171875; + let r: f64 = transmute(vrecped_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrecps_f64() { + let a: f64 = 4.0; + let b: f64 = 4.0; + let e: f64 = -14.; + let r: f64 = transmute(vrecps_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrecpsq_f64() { + let a: f64x2 = f64x2::new(4.0, 3.0); + let b: f64x2 = f64x2::new(4.0, 3.0); + let e: f64x2 = f64x2::new(-14., -7.); + let r: f64x2 = transmute(vrecpsq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrecpss_f32() { + let a: f32 = 4.0; + let b: f32 = 4.0; + let e: f32 = -14.; + let r: f32 = transmute(vrecpss_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrecpsd_f64() { + let a: f64 = 4.0; + let b: f64 = 4.0; + let e: f64 = -14.; + let r: f64 = transmute(vrecpsd_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrecpxs_f32() { + let a: f32 = 4.0; + let e: f32 = 0.5; + let r: f32 = transmute(vrecpxs_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrecpxd_f64() { + let a: f64 = 4.0; + let e: f64 = 0.5; + let r: f64 = transmute(vrecpxd_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_p64() { + let a: i64x1 = i64x1::new(0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_p64() { + let a: i64x1 = i64x1::new(0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_s64() { + let a: i64x1 = i64x1::new(0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_u64() { + let a: u64x1 = u64x1::new(0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_s64_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vreinterpretq_u64_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_p64_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_p64_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_f64() { + let a: f64 = 0.; + let e: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_s8_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_f64() { + let a: f64 = 0.; + let e: i16x4 = i16x4::new(0, 0, 0, 0); + let r: i16x4 = transmute(vreinterpret_s16_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_f64() { + let a: f64 = 0.; + let e: i32x2 = i32x2::new(0, 0); + let r: i32x2 = transmute(vreinterpret_s32_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_f64() { + let a: f64 = 0.; + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_s8_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_s16_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: i32x4 = i32x4::new(0, 0, 0, 0); + let r: i32x4 = transmute(vreinterpretq_s32_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: i64x2 = i64x2::new(0, 0); + let r: i64x2 = transmute(vreinterpretq_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_f64() { + let a: f64 = 0.; + let e: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vreinterpret_u8_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_f64() { + let a: f64 = 0.; + let e: u16x4 = u16x4::new(0, 0, 0, 0); + let r: u16x4 = transmute(vreinterpret_u16_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_f64() { + let a: f64 = 0.; + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vreinterpret_u32_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_f64() { + let a: f64 = 0.; + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vreinterpretq_u8_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vreinterpretq_u16_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: u32x4 = u32x4::new(0, 0, 0, 0); + let r: u32x4 = transmute(vreinterpretq_u32_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vreinterpretq_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_f64() { + let a: f64 = 0.; + let e: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_p8_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_f64() { + let a: f64 = 0.; + let e: i16x4 = i16x4::new(0, 0, 0, 0); + let r: i16x4 = transmute(vreinterpret_p16_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_f64() { + let a: f64 = 0.; + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_p8_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_p16_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: i64x2 = i64x2::new(0, 0); + let r: i64x2 = transmute(vreinterpretq_p64_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: i64x2 = i64x2::new(0, 0); + let r: i64x2 = transmute(vreinterpretq_p64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p128_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: p128 = 0; + let r: p128 = transmute(vreinterpretq_p128_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_s8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_s16() { + let a: i16x4 = i16x4::new(0, 0, 0, 0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_s32() { + let a: i32x2 = i32x2::new(0, 0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_s64() { + let a: i64x1 = i64x1::new(0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_s8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_s16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_s32() { + let a: i32x4 = i32x4::new(0, 0, 0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_s64() { + let a: i64x2 = i64x2::new(0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_p8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_u16() { + let a: u16x4 = u16x4::new(0, 0, 0, 0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_u32() { + let a: u32x2 = u32x2::new(0, 0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_u64() { + let a: u64x1 = u64x1::new(0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_p8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_u16() { + let a: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_u32() { + let a: u32x4 = u32x4::new(0, 0, 0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_u64() { + let a: u64x2 = u64x2::new(0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_u8() { + let a: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_p16() { + let a: i16x4 = i16x4::new(0, 0, 0, 0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_p64() { + let a: i64x1 = i64x1::new(0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_p64() { + let a: i64x1 = i64x1::new(0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_u8() { + let a: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_p16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_p64() { + let a: i64x2 = i64x2::new(0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_p64() { + let a: i64x2 = i64x2::new(0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_p128() { + let a: p128 = 0; + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_p128(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_f64() { + let a: f64 = 0.; + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshld_s64() { + let a: i64 = 1; + let b: i64 = 2; + let e: i64 = 4; + let r: i64 = transmute(vrshld_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshld_u64() { + let a: u64 = 1; + let b: i64 = 2; + let e: u64 = 4; + let r: u64 = transmute(vrshld_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrd_n_s64() { + let a: i64 = 4; + let e: i64 = 1; + let r: i64 = transmute(vrshrd_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrd_n_u64() { + let a: u64 = 4; + let e: u64 = 1; + let r: u64 = transmute(vrshrd_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_high_n_s16() { + let a: i8x8 = i8x8::new(0, 1, 8, 9, 8, 9, 10, 11); + let b: i16x8 = i16x8::new(32, 36, 40, 44, 48, 52, 56, 60); + let e: i8x16 = i8x16::new(0, 1, 8, 9, 8, 9, 10, 11, 8, 9, 10, 11, 12, 13, 14, 15); + let r: i8x16 = transmute(vrshrn_high_n_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_high_n_s32() { + let a: i16x4 = i16x4::new(0, 1, 8, 9); + let b: i32x4 = i32x4::new(32, 36, 40, 44); + let e: i16x8 = i16x8::new(0, 1, 8, 9, 8, 9, 10, 11); + let r: i16x8 = transmute(vrshrn_high_n_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_high_n_s64() { + let a: i32x2 = i32x2::new(0, 1); + let b: i64x2 = i64x2::new(32, 36); + let e: i32x4 = i32x4::new(0, 1, 8, 9); + let r: i32x4 = transmute(vrshrn_high_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_high_n_u16() { + let a: u8x8 = u8x8::new(0, 1, 8, 9, 8, 9, 10, 11); + let b: u16x8 = u16x8::new(32, 36, 40, 44, 48, 52, 56, 60); + let e: u8x16 = u8x16::new(0, 1, 8, 9, 8, 9, 10, 11, 8, 9, 10, 11, 12, 13, 14, 15); + let r: u8x16 = transmute(vrshrn_high_n_u16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_high_n_u32() { + let a: u16x4 = u16x4::new(0, 1, 8, 9); + let b: u32x4 = u32x4::new(32, 36, 40, 44); + let e: u16x8 = u16x8::new(0, 1, 8, 9, 8, 9, 10, 11); + let r: u16x8 = transmute(vrshrn_high_n_u32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_high_n_u64() { + let a: u32x2 = u32x2::new(0, 1); + let b: u64x2 = u64x2::new(32, 36); + let e: u32x4 = u32x4::new(0, 1, 8, 9); + let r: u32x4 = transmute(vrshrn_high_n_u64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsrad_n_s64() { + let a: i64 = 1; + let b: i64 = 4; + let e: i64 = 2; + let r: i64 = transmute(vrsrad_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsrad_n_u64() { + let a: u64 = 1; + let b: u64 = 4; + let e: u64 = 2; + let r: u64 = transmute(vrsrad_n_u64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsubhn_high_s16() { + let a: i8x8 = i8x8::new(1, 2, 0, 0, 0, 0, 0, 0); + let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let c: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i8x16 = i8x16::new(1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vrsubhn_high_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsubhn_high_s32() { + let a: i16x4 = i16x4::new(1, 2, 0, 0); + let b: i32x4 = i32x4::new(1, 2, 3, 4); + let c: i32x4 = i32x4::new(1, 2, 3, 4); + let e: i16x8 = i16x8::new(1, 2, 0, 0, 0, 0, 0, 0); + let r: i16x8 = transmute(vrsubhn_high_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsubhn_high_s64() { + let a: i32x2 = i32x2::new(1, 2); + let b: i64x2 = i64x2::new(1, 2); + let c: i64x2 = i64x2::new(1, 2); + let e: i32x4 = i32x4::new(1, 2, 0, 0); + let r: i32x4 = transmute(vrsubhn_high_s64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsubhn_high_u16() { + let a: u8x8 = u8x8::new(1, 2, 0, 0, 0, 0, 0, 0); + let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let c: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x16 = u8x16::new(1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vrsubhn_high_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsubhn_high_u32() { + let a: u16x4 = u16x4::new(1, 2, 0, 0); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let c: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u16x8 = u16x8::new(1, 2, 0, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vrsubhn_high_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsubhn_high_u64() { + let a: u32x2 = u32x2::new(1, 2); + let b: u64x2 = u64x2::new(1, 2); + let c: u64x2 = u64x2::new(1, 2); + let e: u32x4 = u32x4::new(1, 2, 0, 0); + let r: u32x4 = transmute(vrsubhn_high_u64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_f64() { + let a: f64 = 1.; + let b: f64 = 0.; + let e: f64 = 1.; + let r: f64 = transmute(vset_lane_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_f64() { + let a: f64 = 1.; + let b: f64x2 = f64x2::new(0., 2.); + let e: f64x2 = f64x2::new(1., 2.); + let r: f64x2 = transmute(vsetq_lane_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshld_s64() { + let a: i64 = 1; + let b: i64 = 2; + let e: i64 = 4; + let r: i64 = transmute(vshld_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshld_u64() { + let a: u64 = 1; + let b: i64 = 2; + let e: u64 = 4; + let r: u64 = transmute(vshld_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_high_n_s8() { + let a: i8x16 = i8x16::new(0, 0, 1, 2, 1, 2, 3, 4, 1, 2, 3, 4, 5, 6, 7, 8); + let e: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: i16x8 = transmute(vshll_high_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_high_n_s16() { + let a: i16x8 = i16x8::new(0, 0, 1, 2, 1, 2, 3, 4); + let e: i32x4 = i32x4::new(4, 8, 12, 16); + let r: i32x4 = transmute(vshll_high_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_high_n_s32() { + let a: i32x4 = i32x4::new(0, 0, 1, 2); + let e: i64x2 = i64x2::new(4, 8); + let r: i64x2 = transmute(vshll_high_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_high_n_u8() { + let a: u8x16 = u8x16::new(0, 0, 1, 2, 1, 2, 3, 4, 1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: u16x8 = transmute(vshll_high_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_high_n_u16() { + let a: u16x8 = u16x8::new(0, 0, 1, 2, 1, 2, 3, 4); + let e: u32x4 = u32x4::new(4, 8, 12, 16); + let r: u32x4 = transmute(vshll_high_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_high_n_u32() { + let a: u32x4 = u32x4::new(0, 0, 1, 2); + let e: u64x2 = u64x2::new(4, 8); + let r: u64x2 = transmute(vshll_high_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_high_n_s16() { + let a: i8x8 = i8x8::new(1, 2, 5, 6, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(20, 24, 28, 32, 52, 56, 60, 64); + let e: i8x16 = i8x16::new(1, 2, 5, 6, 5, 6, 7, 8, 5, 6, 7, 8, 13, 14, 15, 16); + let r: i8x16 = transmute(vshrn_high_n_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_high_n_s32() { + let a: i16x4 = i16x4::new(1, 2, 5, 6); + let b: i32x4 = i32x4::new(20, 24, 28, 32); + let e: i16x8 = i16x8::new(1, 2, 5, 6, 5, 6, 7, 8); + let r: i16x8 = transmute(vshrn_high_n_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_high_n_s64() { + let a: i32x2 = i32x2::new(1, 2); + let b: i64x2 = i64x2::new(20, 24); + let e: i32x4 = i32x4::new(1, 2, 5, 6); + let r: i32x4 = transmute(vshrn_high_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_high_n_u16() { + let a: u8x8 = u8x8::new(1, 2, 5, 6, 5, 6, 7, 8); + let b: u16x8 = u16x8::new(20, 24, 28, 32, 52, 56, 60, 64); + let e: u8x16 = u8x16::new(1, 2, 5, 6, 5, 6, 7, 8, 5, 6, 7, 8, 13, 14, 15, 16); + let r: u8x16 = transmute(vshrn_high_n_u16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_high_n_u32() { + let a: u16x4 = u16x4::new(1, 2, 5, 6); + let b: u32x4 = u32x4::new(20, 24, 28, 32); + let e: u16x8 = u16x8::new(1, 2, 5, 6, 5, 6, 7, 8); + let r: u16x8 = transmute(vshrn_high_n_u32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_high_n_u64() { + let a: u32x2 = u32x2::new(1, 2); + let b: u64x2 = u64x2::new(20, 24); + let e: u32x4 = u32x4::new(1, 2, 5, 6); + let r: u32x4 = transmute(vshrn_high_n_u64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sm4")] + unsafe fn test_vsm3partw1q_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let c: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(2147549312, 3221323968, 131329, 2684362752); + let r: u32x4 = transmute(vsm3partw1q_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sm4")] + unsafe fn test_vsm3partw2q_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let c: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(128, 256, 384, 1077977696); + let r: u32x4 = transmute(vsm3partw2q_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sm4")] + unsafe fn test_vsm3ss1q_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let c: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(0, 0, 0, 2098176); + let r: u32x4 = transmute(vsm3ss1q_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sm4")] + unsafe fn test_vsm4ekeyq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(1784948604, 136020997, 2940231695, 3789947679); + let r: u32x4 = transmute(vsm4ekeyq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sm4")] + unsafe fn test_vsm4eq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(1093874472, 3616769504, 3878330411, 2765298765); + let r: u32x4 = transmute(vsm4eq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_vrax1q_u64() { + let a: u64x2 = u64x2::new(1, 2); + let b: u64x2 = u64x2::new(3, 4); + let e: u64x2 = u64x2::new(7, 10); + let r: u64x2 = transmute(vrax1q_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_vsha512hq_u64() { + let a: u64x2 = u64x2::new(1, 2); + let b: u64x2 = u64x2::new(3, 4); + let c: u64x2 = u64x2::new(5, 6); + let e: u64x2 = u64x2::new(11189044327219203, 7177611956453380); + let r: u64x2 = transmute(vsha512hq_u64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_vsha512h2q_u64() { + let a: u64x2 = u64x2::new(1, 2); + let b: u64x2 = u64x2::new(3, 4); + let c: u64x2 = u64x2::new(5, 6); + let e: u64x2 = u64x2::new(5770237651009406214, 349133864969); + let r: u64x2 = transmute(vsha512h2q_u64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_vsha512su0q_u64() { + let a: u64x2 = u64x2::new(1, 2); + let b: u64x2 = u64x2::new(3, 4); + let e: u64x2 = u64x2::new(144115188075855874, 9439544818968559619); + let r: u64x2 = transmute(vsha512su0q_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_vsha512su1q_u64() { + let a: u64x2 = u64x2::new(1, 2); + let b: u64x2 = u64x2::new(3, 4); + let c: u64x2 = u64x2::new(5, 6); + let e: u64x2 = u64x2::new(105553116266526, 140737488355368); + let r: u64x2 = transmute(vsha512su1q_u64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,frintts")] + unsafe fn test_vrnd32x_f32() { + let a: f32x2 = f32x2::new(1.1, 1.9); + let e: f32x2 = f32x2::new(1.0, 2.0); + let r: f32x2 = transmute(vrnd32x_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,frintts")] + unsafe fn test_vrnd32xq_f32() { + let a: f32x4 = f32x4::new(1.1, 1.9, -1.7, -2.3); + let e: f32x4 = f32x4::new(1.0, 2.0, -2.0, -2.0); + let r: f32x4 = transmute(vrnd32xq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,frintts")] + unsafe fn test_vrnd32z_f32() { + let a: f32x2 = f32x2::new(1.1, 1.9); + let e: f32x2 = f32x2::new(1.0, 1.0); + let r: f32x2 = transmute(vrnd32z_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,frintts")] + unsafe fn test_vrnd32zq_f32() { + let a: f32x4 = f32x4::new(1.1, 1.9, -1.7, -2.3); + let e: f32x4 = f32x4::new(1.0, 1.0, -1.0, -2.0); + let r: f32x4 = transmute(vrnd32zq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,frintts")] + unsafe fn test_vrnd64x_f32() { + let a: f32x2 = f32x2::new(1.1, 1.9); + let e: f32x2 = f32x2::new(1.0, 2.0); + let r: f32x2 = transmute(vrnd64x_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,frintts")] + unsafe fn test_vrnd64xq_f32() { + let a: f32x4 = f32x4::new(1.1, 1.9, -1.7, -2.3); + let e: f32x4 = f32x4::new(1.0, 2.0, -2.0, -2.0); + let r: f32x4 = transmute(vrnd64xq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,frintts")] + unsafe fn test_vrnd64z_f32() { + let a: f32x2 = f32x2::new(1.1, 1.9); + let e: f32x2 = f32x2::new(1.0, 1.0); + let r: f32x2 = transmute(vrnd64z_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,frintts")] + unsafe fn test_vrnd64zq_f32() { + let a: f32x4 = f32x4::new(1.1, 1.9, -1.7, -2.3); + let e: f32x4 = f32x4::new(1.0, 1.0, -1.0, -2.0); + let r: f32x4 = transmute(vrnd64zq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1_s8() { + let a: i8x8 = i8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i8x8 = i8x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i8x8 = i8x8::new(0, 1, 4, 5, 8, 9, 12, 13); + let r: i8x8 = transmute(vtrn1_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_s8() { + let a: i8x16 = i8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let b: i8x16 = i8x16::new(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + let e: i8x16 = i8x16::new(0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29); + let r: i8x16 = transmute(vtrn1q_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1_s16() { + let a: i16x4 = i16x4::new(0, 2, 4, 6); + let b: i16x4 = i16x4::new(1, 3, 5, 7); + let e: i16x4 = i16x4::new(0, 1, 4, 5); + let r: i16x4 = transmute(vtrn1_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_s16() { + let a: i16x8 = i16x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i16x8 = i16x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i16x8 = i16x8::new(0, 1, 4, 5, 8, 9, 12, 13); + let r: i16x8 = transmute(vtrn1q_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_s32() { + let a: i32x4 = i32x4::new(0, 2, 4, 6); + let b: i32x4 = i32x4::new(1, 3, 5, 7); + let e: i32x4 = i32x4::new(0, 1, 4, 5); + let r: i32x4 = transmute(vtrn1q_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1_u8() { + let a: u8x8 = u8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: u8x8 = u8x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: u8x8 = u8x8::new(0, 1, 4, 5, 8, 9, 12, 13); + let r: u8x8 = transmute(vtrn1_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_u8() { + let a: u8x16 = u8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let b: u8x16 = u8x16::new(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + let e: u8x16 = u8x16::new(0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29); + let r: u8x16 = transmute(vtrn1q_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1_u16() { + let a: u16x4 = u16x4::new(0, 2, 4, 6); + let b: u16x4 = u16x4::new(1, 3, 5, 7); + let e: u16x4 = u16x4::new(0, 1, 4, 5); + let r: u16x4 = transmute(vtrn1_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_u16() { + let a: u16x8 = u16x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: u16x8 = u16x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: u16x8 = u16x8::new(0, 1, 4, 5, 8, 9, 12, 13); + let r: u16x8 = transmute(vtrn1q_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_u32() { + let a: u32x4 = u32x4::new(0, 2, 4, 6); + let b: u32x4 = u32x4::new(1, 3, 5, 7); + let e: u32x4 = u32x4::new(0, 1, 4, 5); + let r: u32x4 = transmute(vtrn1q_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1_p8() { + let a: i8x8 = i8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i8x8 = i8x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i8x8 = i8x8::new(0, 1, 4, 5, 8, 9, 12, 13); + let r: i8x8 = transmute(vtrn1_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_p8() { + let a: i8x16 = i8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let b: i8x16 = i8x16::new(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + let e: i8x16 = i8x16::new(0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29); + let r: i8x16 = transmute(vtrn1q_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1_p16() { + let a: i16x4 = i16x4::new(0, 2, 4, 6); + let b: i16x4 = i16x4::new(1, 3, 5, 7); + let e: i16x4 = i16x4::new(0, 1, 4, 5); + let r: i16x4 = transmute(vtrn1_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_p16() { + let a: i16x8 = i16x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i16x8 = i16x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i16x8 = i16x8::new(0, 1, 4, 5, 8, 9, 12, 13); + let r: i16x8 = transmute(vtrn1q_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1_s32() { + let a: i32x2 = i32x2::new(0, 2); + let b: i32x2 = i32x2::new(1, 3); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vtrn1_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_s64() { + let a: i64x2 = i64x2::new(0, 2); + let b: i64x2 = i64x2::new(1, 3); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vtrn1q_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1_u32() { + let a: u32x2 = u32x2::new(0, 2); + let b: u32x2 = u32x2::new(1, 3); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vtrn1_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_u64() { + let a: u64x2 = u64x2::new(0, 2); + let b: u64x2 = u64x2::new(1, 3); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vtrn1q_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_p64() { + let a: i64x2 = i64x2::new(0, 2); + let b: i64x2 = i64x2::new(1, 3); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vtrn1q_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_f32() { + let a: f32x4 = f32x4::new(0., 2., 4., 6.); + let b: f32x4 = f32x4::new(1., 3., 5., 7.); + let e: f32x4 = f32x4::new(0., 1., 4., 5.); + let r: f32x4 = transmute(vtrn1q_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1_f32() { + let a: f32x2 = f32x2::new(0., 2.); + let b: f32x2 = f32x2::new(1., 3.); + let e: f32x2 = f32x2::new(0., 1.); + let r: f32x2 = transmute(vtrn1_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_f64() { + let a: f64x2 = f64x2::new(0., 2.); + let b: f64x2 = f64x2::new(1., 3.); + let e: f64x2 = f64x2::new(0., 1.); + let r: f64x2 = transmute(vtrn1q_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2_s8() { + let a: i8x8 = i8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i8x8 = i8x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i8x8 = i8x8::new(2, 3, 6, 7, 10, 11, 14, 15); + let r: i8x8 = transmute(vtrn2_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_s8() { + let a: i8x16 = i8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let b: i8x16 = i8x16::new(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + let e: i8x16 = i8x16::new(2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31); + let r: i8x16 = transmute(vtrn2q_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2_s16() { + let a: i16x4 = i16x4::new(0, 2, 4, 6); + let b: i16x4 = i16x4::new(1, 3, 5, 7); + let e: i16x4 = i16x4::new(2, 3, 6, 7); + let r: i16x4 = transmute(vtrn2_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_s16() { + let a: i16x8 = i16x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i16x8 = i16x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i16x8 = i16x8::new(2, 3, 6, 7, 10, 11, 14, 15); + let r: i16x8 = transmute(vtrn2q_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_s32() { + let a: i32x4 = i32x4::new(0, 2, 4, 6); + let b: i32x4 = i32x4::new(1, 3, 5, 7); + let e: i32x4 = i32x4::new(2, 3, 6, 7); + let r: i32x4 = transmute(vtrn2q_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2_u8() { + let a: u8x8 = u8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: u8x8 = u8x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: u8x8 = u8x8::new(2, 3, 6, 7, 10, 11, 14, 15); + let r: u8x8 = transmute(vtrn2_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_u8() { + let a: u8x16 = u8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let b: u8x16 = u8x16::new(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + let e: u8x16 = u8x16::new(2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31); + let r: u8x16 = transmute(vtrn2q_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2_u16() { + let a: u16x4 = u16x4::new(0, 2, 4, 6); + let b: u16x4 = u16x4::new(1, 3, 5, 7); + let e: u16x4 = u16x4::new(2, 3, 6, 7); + let r: u16x4 = transmute(vtrn2_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_u16() { + let a: u16x8 = u16x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: u16x8 = u16x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: u16x8 = u16x8::new(2, 3, 6, 7, 10, 11, 14, 15); + let r: u16x8 = transmute(vtrn2q_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_u32() { + let a: u32x4 = u32x4::new(0, 2, 4, 6); + let b: u32x4 = u32x4::new(1, 3, 5, 7); + let e: u32x4 = u32x4::new(2, 3, 6, 7); + let r: u32x4 = transmute(vtrn2q_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2_p8() { + let a: i8x8 = i8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i8x8 = i8x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i8x8 = i8x8::new(2, 3, 6, 7, 10, 11, 14, 15); + let r: i8x8 = transmute(vtrn2_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_p8() { + let a: i8x16 = i8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let b: i8x16 = i8x16::new(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + let e: i8x16 = i8x16::new(2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31); + let r: i8x16 = transmute(vtrn2q_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2_p16() { + let a: i16x4 = i16x4::new(0, 2, 4, 6); + let b: i16x4 = i16x4::new(1, 3, 5, 7); + let e: i16x4 = i16x4::new(2, 3, 6, 7); + let r: i16x4 = transmute(vtrn2_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_p16() { + let a: i16x8 = i16x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i16x8 = i16x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i16x8 = i16x8::new(2, 3, 6, 7, 10, 11, 14, 15); + let r: i16x8 = transmute(vtrn2q_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2_s32() { + let a: i32x2 = i32x2::new(0, 2); + let b: i32x2 = i32x2::new(1, 3); + let e: i32x2 = i32x2::new(2, 3); + let r: i32x2 = transmute(vtrn2_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_s64() { + let a: i64x2 = i64x2::new(0, 2); + let b: i64x2 = i64x2::new(1, 3); + let e: i64x2 = i64x2::new(2, 3); + let r: i64x2 = transmute(vtrn2q_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2_u32() { + let a: u32x2 = u32x2::new(0, 2); + let b: u32x2 = u32x2::new(1, 3); + let e: u32x2 = u32x2::new(2, 3); + let r: u32x2 = transmute(vtrn2_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_u64() { + let a: u64x2 = u64x2::new(0, 2); + let b: u64x2 = u64x2::new(1, 3); + let e: u64x2 = u64x2::new(2, 3); + let r: u64x2 = transmute(vtrn2q_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_p64() { + let a: i64x2 = i64x2::new(0, 2); + let b: i64x2 = i64x2::new(1, 3); + let e: i64x2 = i64x2::new(2, 3); + let r: i64x2 = transmute(vtrn2q_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_f32() { + let a: f32x4 = f32x4::new(0., 2., 4., 6.); + let b: f32x4 = f32x4::new(1., 3., 5., 7.); + let e: f32x4 = f32x4::new(2., 3., 6., 7.); + let r: f32x4 = transmute(vtrn2q_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2_f32() { + let a: f32x2 = f32x2::new(0., 2.); + let b: f32x2 = f32x2::new(1., 3.); + let e: f32x2 = f32x2::new(2., 3.); + let r: f32x2 = transmute(vtrn2_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_f64() { + let a: f64x2 = f64x2::new(0., 2.); + let b: f64x2 = f64x2::new(1., 3.); + let e: f64x2 = f64x2::new(2., 3.); + let r: f64x2 = transmute(vtrn2q_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1_s8() { + let a: i8x8 = i8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i8x8 = i8x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i8x8 = transmute(vzip1_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_s8() { + let a: i8x16 = i8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let b: i8x16 = i8x16::new(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + let e: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: i8x16 = transmute(vzip1q_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1_s16() { + let a: i16x4 = i16x4::new(0, 2, 4, 6); + let b: i16x4 = i16x4::new(1, 3, 5, 7); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vzip1_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_s16() { + let a: i16x8 = i16x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i16x8 = i16x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vzip1q_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1_s32() { + let a: i32x2 = i32x2::new(0, 2); + let b: i32x2 = i32x2::new(1, 3); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vzip1_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_s32() { + let a: i32x4 = i32x4::new(0, 2, 4, 6); + let b: i32x4 = i32x4::new(1, 3, 5, 7); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vzip1q_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_s64() { + let a: i64x2 = i64x2::new(0, 2); + let b: i64x2 = i64x2::new(1, 3); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vzip1q_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1_u8() { + let a: u8x8 = u8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: u8x8 = u8x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u8x8 = transmute(vzip1_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_u8() { + let a: u8x16 = u8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let b: u8x16 = u8x16::new(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + let e: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: u8x16 = transmute(vzip1q_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1_u16() { + let a: u16x4 = u16x4::new(0, 2, 4, 6); + let b: u16x4 = u16x4::new(1, 3, 5, 7); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vzip1_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_u16() { + let a: u16x8 = u16x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: u16x8 = u16x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vzip1q_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1_u32() { + let a: u32x2 = u32x2::new(0, 2); + let b: u32x2 = u32x2::new(1, 3); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vzip1_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_u32() { + let a: u32x4 = u32x4::new(0, 2, 4, 6); + let b: u32x4 = u32x4::new(1, 3, 5, 7); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vzip1q_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_u64() { + let a: u64x2 = u64x2::new(0, 2); + let b: u64x2 = u64x2::new(1, 3); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vzip1q_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1_p8() { + let a: i8x8 = i8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i8x8 = i8x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i8x8 = transmute(vzip1_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_p8() { + let a: i8x16 = i8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let b: i8x16 = i8x16::new(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + let e: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: i8x16 = transmute(vzip1q_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1_p16() { + let a: i16x4 = i16x4::new(0, 2, 4, 6); + let b: i16x4 = i16x4::new(1, 3, 5, 7); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vzip1_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_p16() { + let a: i16x8 = i16x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i16x8 = i16x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vzip1q_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_p64() { + let a: i64x2 = i64x2::new(0, 2); + let b: i64x2 = i64x2::new(1, 3); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vzip1q_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1_f32() { + let a: f32x2 = f32x2::new(0., 2.); + let b: f32x2 = f32x2::new(1., 3.); + let e: f32x2 = f32x2::new(0., 1.); + let r: f32x2 = transmute(vzip1_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_f32() { + let a: f32x4 = f32x4::new(0., 2., 4., 6.); + let b: f32x4 = f32x4::new(1., 3., 5., 7.); + let e: f32x4 = f32x4::new(0., 1., 2., 3.); + let r: f32x4 = transmute(vzip1q_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_f64() { + let a: f64x2 = f64x2::new(0., 2.); + let b: f64x2 = f64x2::new(1., 3.); + let e: f64x2 = f64x2::new(0., 1.); + let r: f64x2 = transmute(vzip1q_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2_s8() { + let a: i8x8 = i8x8::new(0, 16, 16, 18, 16, 18, 20, 22); + let b: i8x8 = i8x8::new(1, 17, 17, 19, 17, 19, 21, 23); + let e: i8x8 = i8x8::new(16, 17, 18, 19, 20, 21, 22, 23); + let r: i8x8 = transmute(vzip2_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_s8() { + let a: i8x16 = i8x16::new(0, 16, 16, 18, 16, 18, 20, 22, 16, 18, 20, 22, 24, 26, 28, 30); + let b: i8x16 = i8x16::new(1, 17, 17, 19, 17, 19, 21, 23, 17, 19, 21, 23, 25, 27, 29, 31); + let e: i8x16 = i8x16::new(16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let r: i8x16 = transmute(vzip2q_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2_s16() { + let a: i16x4 = i16x4::new(0, 16, 16, 18); + let b: i16x4 = i16x4::new(1, 17, 17, 19); + let e: i16x4 = i16x4::new(16, 17, 18, 19); + let r: i16x4 = transmute(vzip2_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_s16() { + let a: i16x8 = i16x8::new(0, 16, 16, 18, 16, 18, 20, 22); + let b: i16x8 = i16x8::new(1, 17, 17, 19, 17, 19, 21, 23); + let e: i16x8 = i16x8::new(16, 17, 18, 19, 20, 21, 22, 23); + let r: i16x8 = transmute(vzip2q_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2_s32() { + let a: i32x2 = i32x2::new(0, 16); + let b: i32x2 = i32x2::new(1, 17); + let e: i32x2 = i32x2::new(16, 17); + let r: i32x2 = transmute(vzip2_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_s32() { + let a: i32x4 = i32x4::new(0, 16, 16, 18); + let b: i32x4 = i32x4::new(1, 17, 17, 19); + let e: i32x4 = i32x4::new(16, 17, 18, 19); + let r: i32x4 = transmute(vzip2q_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_s64() { + let a: i64x2 = i64x2::new(0, 16); + let b: i64x2 = i64x2::new(1, 17); + let e: i64x2 = i64x2::new(16, 17); + let r: i64x2 = transmute(vzip2q_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2_u8() { + let a: u8x8 = u8x8::new(0, 16, 16, 18, 16, 18, 20, 22); + let b: u8x8 = u8x8::new(1, 17, 17, 19, 17, 19, 21, 23); + let e: u8x8 = u8x8::new(16, 17, 18, 19, 20, 21, 22, 23); + let r: u8x8 = transmute(vzip2_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_u8() { + let a: u8x16 = u8x16::new(0, 16, 16, 18, 16, 18, 20, 22, 16, 18, 20, 22, 24, 26, 28, 30); + let b: u8x16 = u8x16::new(1, 17, 17, 19, 17, 19, 21, 23, 17, 19, 21, 23, 25, 27, 29, 31); + let e: u8x16 = u8x16::new(16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let r: u8x16 = transmute(vzip2q_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2_u16() { + let a: u16x4 = u16x4::new(0, 16, 16, 18); + let b: u16x4 = u16x4::new(1, 17, 17, 19); + let e: u16x4 = u16x4::new(16, 17, 18, 19); + let r: u16x4 = transmute(vzip2_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_u16() { + let a: u16x8 = u16x8::new(0, 16, 16, 18, 16, 18, 20, 22); + let b: u16x8 = u16x8::new(1, 17, 17, 19, 17, 19, 21, 23); + let e: u16x8 = u16x8::new(16, 17, 18, 19, 20, 21, 22, 23); + let r: u16x8 = transmute(vzip2q_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2_u32() { + let a: u32x2 = u32x2::new(0, 16); + let b: u32x2 = u32x2::new(1, 17); + let e: u32x2 = u32x2::new(16, 17); + let r: u32x2 = transmute(vzip2_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_u32() { + let a: u32x4 = u32x4::new(0, 16, 16, 18); + let b: u32x4 = u32x4::new(1, 17, 17, 19); + let e: u32x4 = u32x4::new(16, 17, 18, 19); + let r: u32x4 = transmute(vzip2q_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_u64() { + let a: u64x2 = u64x2::new(0, 16); + let b: u64x2 = u64x2::new(1, 17); + let e: u64x2 = u64x2::new(16, 17); + let r: u64x2 = transmute(vzip2q_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2_p8() { + let a: i8x8 = i8x8::new(0, 16, 16, 18, 16, 18, 20, 22); + let b: i8x8 = i8x8::new(1, 17, 17, 19, 17, 19, 21, 23); + let e: i8x8 = i8x8::new(16, 17, 18, 19, 20, 21, 22, 23); + let r: i8x8 = transmute(vzip2_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_p8() { + let a: i8x16 = i8x16::new(0, 16, 16, 18, 16, 18, 20, 22, 16, 18, 20, 22, 24, 26, 28, 30); + let b: i8x16 = i8x16::new(1, 17, 17, 19, 17, 19, 21, 23, 17, 19, 21, 23, 25, 27, 29, 31); + let e: i8x16 = i8x16::new(16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let r: i8x16 = transmute(vzip2q_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2_p16() { + let a: i16x4 = i16x4::new(0, 16, 16, 18); + let b: i16x4 = i16x4::new(1, 17, 17, 19); + let e: i16x4 = i16x4::new(16, 17, 18, 19); + let r: i16x4 = transmute(vzip2_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_p16() { + let a: i16x8 = i16x8::new(0, 16, 16, 18, 16, 18, 20, 22); + let b: i16x8 = i16x8::new(1, 17, 17, 19, 17, 19, 21, 23); + let e: i16x8 = i16x8::new(16, 17, 18, 19, 20, 21, 22, 23); + let r: i16x8 = transmute(vzip2q_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_p64() { + let a: i64x2 = i64x2::new(0, 16); + let b: i64x2 = i64x2::new(1, 17); + let e: i64x2 = i64x2::new(16, 17); + let r: i64x2 = transmute(vzip2q_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2_f32() { + let a: f32x2 = f32x2::new(0., 8.); + let b: f32x2 = f32x2::new(1., 9.); + let e: f32x2 = f32x2::new(8., 9.); + let r: f32x2 = transmute(vzip2_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_f32() { + let a: f32x4 = f32x4::new(0., 8., 8., 10.); + let b: f32x4 = f32x4::new(1., 9., 9., 11.); + let e: f32x4 = f32x4::new(8., 9., 10., 11.); + let r: f32x4 = transmute(vzip2q_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_f64() { + let a: f64x2 = f64x2::new(0., 8.); + let b: f64x2 = f64x2::new(1., 9.); + let e: f64x2 = f64x2::new(8., 9.); + let r: f64x2 = transmute(vzip2q_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1_s8() { + let a: i8x8 = i8x8::new(1, 0, 2, 0, 2, 0, 3, 0); + let b: i8x8 = i8x8::new(2, 0, 3, 0, 7, 0, 8, 0); + let e: i8x8 = i8x8::new(1, 2, 2, 3, 2, 3, 7, 8); + let r: i8x8 = transmute(vuzp1_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1q_s8() { + let a: i8x16 = i8x16::new(1, 0, 2, 0, 2, 0, 3, 0, 2, 0, 3, 0, 7, 0, 8, 0); + let b: i8x16 = i8x16::new(2, 0, 3, 0, 7, 0, 8, 0, 13, 0, 14, 0, 15, 0, 16, 0); + let e: i8x16 = i8x16::new(1, 2, 2, 3, 2, 3, 7, 8, 2, 3, 7, 8, 13, 14, 15, 16); + let r: i8x16 = transmute(vuzp1q_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1_s16() { + let a: i16x4 = i16x4::new(1, 0, 2, 0); + let b: i16x4 = i16x4::new(2, 0, 3, 0); + let e: i16x4 = i16x4::new(1, 2, 2, 3); + let r: i16x4 = transmute(vuzp1_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1q_s16() { + let a: i16x8 = i16x8::new(1, 0, 2, 0, 2, 0, 3, 0); + let b: i16x8 = i16x8::new(2, 0, 3, 0, 7, 0, 8, 0); + let e: i16x8 = i16x8::new(1, 2, 2, 3, 2, 3, 7, 8); + let r: i16x8 = transmute(vuzp1q_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1q_s32() { + let a: i32x4 = i32x4::new(1, 0, 2, 0); + let b: i32x4 = i32x4::new(2, 0, 3, 0); + let e: i32x4 = i32x4::new(1, 2, 2, 3); + let r: i32x4 = transmute(vuzp1q_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1_u8() { + let a: u8x8 = u8x8::new(1, 0, 2, 0, 2, 0, 3, 0); + let b: u8x8 = u8x8::new(2, 0, 3, 0, 7, 0, 8, 0); + let e: u8x8 = u8x8::new(1, 2, 2, 3, 2, 3, 7, 8); + let r: u8x8 = transmute(vuzp1_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1q_u8() { + let a: u8x16 = u8x16::new(1, 0, 2, 0, 2, 0, 3, 0, 2, 0, 3, 0, 7, 0, 8, 0); + let b: u8x16 = u8x16::new(2, 0, 3, 0, 7, 0, 8, 0, 13, 0, 14, 0, 15, 0, 16, 0); + let e: u8x16 = u8x16::new(1, 2, 2, 3, 2, 3, 7, 8, 2, 3, 7, 8, 13, 14, 15, 16); + let r: u8x16 = transmute(vuzp1q_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1_u16() { + let a: u16x4 = u16x4::new(1, 0, 2, 0); + let b: u16x4 = u16x4::new(2, 0, 3, 0); + let e: u16x4 = u16x4::new(1, 2, 2, 3); + let r: u16x4 = transmute(vuzp1_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1q_u16() { + let a: u16x8 = u16x8::new(1, 0, 2, 0, 2, 0, 3, 0); + let b: u16x8 = u16x8::new(2, 0, 3, 0, 7, 0, 8, 0); + let e: u16x8 = u16x8::new(1, 2, 2, 3, 2, 3, 7, 8); + let r: u16x8 = transmute(vuzp1q_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1q_u32() { + let a: u32x4 = u32x4::new(1, 0, 2, 0); + let b: u32x4 = u32x4::new(2, 0, 3, 0); + let e: u32x4 = u32x4::new(1, 2, 2, 3); + let r: u32x4 = transmute(vuzp1q_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1_p8() { + let a: i8x8 = i8x8::new(1, 0, 2, 0, 2, 0, 3, 0); + let b: i8x8 = i8x8::new(2, 0, 3, 0, 7, 0, 8, 0); + let e: i8x8 = i8x8::new(1, 2, 2, 3, 2, 3, 7, 8); + let r: i8x8 = transmute(vuzp1_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1q_p8() { + let a: i8x16 = i8x16::new(1, 0, 2, 0, 2, 0, 3, 0, 2, 0, 3, 0, 7, 0, 8, 0); + let b: i8x16 = i8x16::new(2, 0, 3, 0, 7, 0, 8, 0, 13, 0, 14, 0, 15, 0, 16, 0); + let e: i8x16 = i8x16::new(1, 2, 2, 3, 2, 3, 7, 8, 2, 3, 7, 8, 13, 14, 15, 16); + let r: i8x16 = transmute(vuzp1q_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1_p16() { + let a: i16x4 = i16x4::new(1, 0, 2, 0); + let b: i16x4 = i16x4::new(2, 0, 3, 0); + let e: i16x4 = i16x4::new(1, 2, 2, 3); + let r: i16x4 = transmute(vuzp1_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1q_p16() { + let a: i16x8 = i16x8::new(1, 0, 2, 0, 2, 0, 3, 0); + let b: i16x8 = i16x8::new(2, 0, 3, 0, 7, 0, 8, 0); + let e: i16x8 = i16x8::new(1, 2, 2, 3, 2, 3, 7, 8); + let r: i16x8 = transmute(vuzp1q_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1_s32() { + let a: i32x2 = i32x2::new(1, 0); + let b: i32x2 = i32x2::new(2, 0); + let e: i32x2 = i32x2::new(1, 2); + let r: i32x2 = transmute(vuzp1_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1q_s64() { + let a: i64x2 = i64x2::new(1, 0); + let b: i64x2 = i64x2::new(2, 0); + let e: i64x2 = i64x2::new(1, 2); + let r: i64x2 = transmute(vuzp1q_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1_u32() { + let a: u32x2 = u32x2::new(1, 0); + let b: u32x2 = u32x2::new(2, 0); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vuzp1_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1q_u64() { + let a: u64x2 = u64x2::new(1, 0); + let b: u64x2 = u64x2::new(2, 0); + let e: u64x2 = u64x2::new(1, 2); + let r: u64x2 = transmute(vuzp1q_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1q_p64() { + let a: i64x2 = i64x2::new(1, 0); + let b: i64x2 = i64x2::new(2, 0); + let e: i64x2 = i64x2::new(1, 2); + let r: i64x2 = transmute(vuzp1q_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1q_f32() { + let a: f32x4 = f32x4::new(0., 8., 1., 9.); + let b: f32x4 = f32x4::new(1., 10., 3., 11.); + let e: f32x4 = f32x4::new(0., 1., 1., 3.); + let r: f32x4 = transmute(vuzp1q_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1_f32() { + let a: f32x2 = f32x2::new(0., 8.); + let b: f32x2 = f32x2::new(1., 10.); + let e: f32x2 = f32x2::new(0., 1.); + let r: f32x2 = transmute(vuzp1_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1q_f64() { + let a: f64x2 = f64x2::new(0., 8.); + let b: f64x2 = f64x2::new(1., 10.); + let e: f64x2 = f64x2::new(0., 1.); + let r: f64x2 = transmute(vuzp1q_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2_s8() { + let a: i8x8 = i8x8::new(0, 17, 0, 18, 0, 18, 0, 19); + let b: i8x8 = i8x8::new(0, 18, 0, 19, 0, 23, 0, 24); + let e: i8x8 = i8x8::new(17, 18, 18, 19, 18, 19, 23, 24); + let r: i8x8 = transmute(vuzp2_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2q_s8() { + let a: i8x16 = i8x16::new(0, 17, 0, 18, 0, 18, 0, 19, 0, 18, 0, 19, 0, 23, 0, 24); + let b: i8x16 = i8x16::new(0, 18, 0, 19, 0, 23, 0, 24, 0, 29, 0, 30, 0, 31, 0, 32); + let e: i8x16 = i8x16::new(17, 18, 18, 19, 18, 19, 23, 24, 18, 19, 23, 24, 29, 30, 31, 32); + let r: i8x16 = transmute(vuzp2q_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2_s16() { + let a: i16x4 = i16x4::new(0, 17, 0, 18); + let b: i16x4 = i16x4::new(0, 18, 0, 19); + let e: i16x4 = i16x4::new(17, 18, 18, 19); + let r: i16x4 = transmute(vuzp2_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2q_s16() { + let a: i16x8 = i16x8::new(0, 17, 0, 18, 0, 18, 0, 19); + let b: i16x8 = i16x8::new(0, 18, 0, 19, 0, 23, 0, 24); + let e: i16x8 = i16x8::new(17, 18, 18, 19, 18, 19, 23, 24); + let r: i16x8 = transmute(vuzp2q_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2q_s32() { + let a: i32x4 = i32x4::new(0, 17, 0, 18); + let b: i32x4 = i32x4::new(0, 18, 0, 19); + let e: i32x4 = i32x4::new(17, 18, 18, 19); + let r: i32x4 = transmute(vuzp2q_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2_u8() { + let a: u8x8 = u8x8::new(0, 17, 0, 18, 0, 18, 0, 19); + let b: u8x8 = u8x8::new(0, 18, 0, 19, 0, 23, 0, 24); + let e: u8x8 = u8x8::new(17, 18, 18, 19, 18, 19, 23, 24); + let r: u8x8 = transmute(vuzp2_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2q_u8() { + let a: u8x16 = u8x16::new(0, 17, 0, 18, 0, 18, 0, 19, 0, 18, 0, 19, 0, 23, 0, 24); + let b: u8x16 = u8x16::new(0, 18, 0, 19, 0, 23, 0, 24, 0, 29, 0, 30, 0, 31, 0, 32); + let e: u8x16 = u8x16::new(17, 18, 18, 19, 18, 19, 23, 24, 18, 19, 23, 24, 29, 30, 31, 32); + let r: u8x16 = transmute(vuzp2q_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2_u16() { + let a: u16x4 = u16x4::new(0, 17, 0, 18); + let b: u16x4 = u16x4::new(0, 18, 0, 19); + let e: u16x4 = u16x4::new(17, 18, 18, 19); + let r: u16x4 = transmute(vuzp2_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2q_u16() { + let a: u16x8 = u16x8::new(0, 17, 0, 18, 0, 18, 0, 19); + let b: u16x8 = u16x8::new(0, 18, 0, 19, 0, 23, 0, 24); + let e: u16x8 = u16x8::new(17, 18, 18, 19, 18, 19, 23, 24); + let r: u16x8 = transmute(vuzp2q_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2q_u32() { + let a: u32x4 = u32x4::new(0, 17, 0, 18); + let b: u32x4 = u32x4::new(0, 18, 0, 19); + let e: u32x4 = u32x4::new(17, 18, 18, 19); + let r: u32x4 = transmute(vuzp2q_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2_p8() { + let a: i8x8 = i8x8::new(0, 17, 0, 18, 0, 18, 0, 19); + let b: i8x8 = i8x8::new(0, 18, 0, 19, 0, 23, 0, 24); + let e: i8x8 = i8x8::new(17, 18, 18, 19, 18, 19, 23, 24); + let r: i8x8 = transmute(vuzp2_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2q_p8() { + let a: i8x16 = i8x16::new(0, 17, 0, 18, 0, 18, 0, 19, 0, 18, 0, 19, 0, 23, 0, 24); + let b: i8x16 = i8x16::new(0, 18, 0, 19, 0, 23, 0, 24, 0, 29, 0, 30, 0, 31, 0, 32); + let e: i8x16 = i8x16::new(17, 18, 18, 19, 18, 19, 23, 24, 18, 19, 23, 24, 29, 30, 31, 32); + let r: i8x16 = transmute(vuzp2q_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2_p16() { + let a: i16x4 = i16x4::new(0, 17, 0, 18); + let b: i16x4 = i16x4::new(0, 18, 0, 19); + let e: i16x4 = i16x4::new(17, 18, 18, 19); + let r: i16x4 = transmute(vuzp2_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2q_p16() { + let a: i16x8 = i16x8::new(0, 17, 0, 18, 0, 18, 0, 19); + let b: i16x8 = i16x8::new(0, 18, 0, 19, 0, 23, 0, 24); + let e: i16x8 = i16x8::new(17, 18, 18, 19, 18, 19, 23, 24); + let r: i16x8 = transmute(vuzp2q_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2_s32() { + let a: i32x2 = i32x2::new(0, 17); + let b: i32x2 = i32x2::new(0, 18); + let e: i32x2 = i32x2::new(17, 18); + let r: i32x2 = transmute(vuzp2_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2q_s64() { + let a: i64x2 = i64x2::new(0, 17); + let b: i64x2 = i64x2::new(0, 18); + let e: i64x2 = i64x2::new(17, 18); + let r: i64x2 = transmute(vuzp2q_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2_u32() { + let a: u32x2 = u32x2::new(0, 17); + let b: u32x2 = u32x2::new(0, 18); + let e: u32x2 = u32x2::new(17, 18); + let r: u32x2 = transmute(vuzp2_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2q_u64() { + let a: u64x2 = u64x2::new(0, 17); + let b: u64x2 = u64x2::new(0, 18); + let e: u64x2 = u64x2::new(17, 18); + let r: u64x2 = transmute(vuzp2q_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2q_p64() { + let a: i64x2 = i64x2::new(0, 17); + let b: i64x2 = i64x2::new(0, 18); + let e: i64x2 = i64x2::new(17, 18); + let r: i64x2 = transmute(vuzp2q_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2q_f32() { + let a: f32x4 = f32x4::new(0., 8., 1., 9.); + let b: f32x4 = f32x4::new(2., 9., 3., 11.); + let e: f32x4 = f32x4::new(8., 9., 9., 11.); + let r: f32x4 = transmute(vuzp2q_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2_f32() { + let a: f32x2 = f32x2::new(0., 8.); + let b: f32x2 = f32x2::new(2., 9.); + let e: f32x2 = f32x2::new(8., 9.); + let r: f32x2 = transmute(vuzp2_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2q_f64() { + let a: f64x2 = f64x2::new(0., 8.); + let b: f64x2 = f64x2::new(2., 9.); + let e: f64x2 = f64x2::new(8., 9.); + let r: f64x2 = transmute(vuzp2q_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_high_u8() { + let a: u16x8 = u16x8::new(9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let c: u8x16 = u8x16::new(10, 10, 10, 10, 10, 10, 10, 10, 20, 0, 2, 4, 6, 8, 10, 12); + let e: u16x8 = u16x8::new(20, 20, 20, 20, 20, 20, 20, 20); + let r: u16x8 = transmute(vabal_high_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_high_u16() { + let a: u32x4 = u32x4::new(9, 10, 11, 12); + let b: u16x8 = u16x8::new(1, 2, 3, 4, 9, 10, 11, 12); + let c: u16x8 = u16x8::new(10, 10, 10, 10, 20, 0, 2, 4); + let e: u32x4 = u32x4::new(20, 20, 20, 20); + let r: u32x4 = transmute(vabal_high_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_high_u32() { + let a: u64x2 = u64x2::new(15, 16); + let b: u32x4 = u32x4::new(1, 2, 15, 16); + let c: u32x4 = u32x4::new(10, 10, 10, 12); + let e: u64x2 = u64x2::new(20, 20); + let r: u64x2 = transmute(vabal_high_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_high_s8() { + let a: i16x8 = i16x8::new(9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let c: i8x16 = i8x16::new(10, 10, 10, 10, 10, 10, 10, 10, 20, 0, 2, 4, 6, 8, 10, 12); + let e: i16x8 = i16x8::new(20, 20, 20, 20, 20, 20, 20, 20); + let r: i16x8 = transmute(vabal_high_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_high_s16() { + let a: i32x4 = i32x4::new(9, 10, 11, 12); + let b: i16x8 = i16x8::new(1, 2, 3, 4, 9, 10, 11, 12); + let c: i16x8 = i16x8::new(10, 10, 10, 10, 20, 0, 2, 4); + let e: i32x4 = i32x4::new(20, 20, 20, 20); + let r: i32x4 = transmute(vabal_high_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_high_s32() { + let a: i64x2 = i64x2::new(15, 16); + let b: i32x4 = i32x4::new(1, 2, 15, 16); + let c: i32x4 = i32x4::new(10, 10, 10, 12); + let e: i64x2 = i64x2::new(20, 20); + let r: i64x2 = transmute(vabal_high_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqabs_s64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let e: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); + let r: i64x1 = transmute(vqabs_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqabsq_s64() { + let a: i64x2 = i64x2::new(-9223372036854775808, -7); + let e: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, 7); + let r: i64x2 = transmute(vqabsq_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqabsb_s8() { + let a: i8 = -7; + let e: i8 = 7; + let r: i8 = transmute(vqabsb_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqabsh_s16() { + let a: i16 = -7; + let e: i16 = 7; + let r: i16 = transmute(vqabsh_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqabss_s32() { + let a: i32 = -7; + let e: i32 = 7; + let r: i32 = transmute(vqabss_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqabsd_s64() { + let a: i64 = -7; + let e: i64 = 7; + let r: i64 = transmute(vqabsd_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vslid_n_s64() { + let a: i64 = 333; + let b: i64 = 2042; + let e: i64 = 8169; + let r: i64 = transmute(vslid_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vslid_n_u64() { + let a: u64 = 333; + let b: u64 = 2042; + let e: u64 = 8169; + let r: u64 = transmute(vslid_n_u64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsrid_n_s64() { + let a: i64 = 333; + let b: i64 = 2042; + let e: i64 = 510; + let r: i64 = transmute(vsrid_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsrid_n_u64() { + let a: u64 = 333; + let b: u64 = 2042; + let e: u64 = 510; + let r: u64 = transmute(vsrid_n_u64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/neon/mod.rs b/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/neon/mod.rs new file mode 100644 index 000000000..9d9946b4f --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/neon/mod.rs @@ -0,0 +1,5482 @@ +//! ARMv8 ASIMD intrinsics + +#![allow(non_camel_case_types)] + +#[rustfmt::skip] +mod generated; +#[rustfmt::skip] +pub use self::generated::*; + +// FIXME: replace neon with asimd + +use crate::{ + core_arch::{arm_shared::*, simd::*, simd_llvm::*}, + hint::unreachable_unchecked, + mem::{transmute, zeroed}, + ptr::{read_unaligned, write_unaligned}, +}; +#[cfg(test)] +use stdarch_test::assert_instr; + +types! { + /// ARM-specific 64-bit wide vector of one packed `f64`. + #[stable(feature = "neon_intrinsics", since = "1.59.0")] + pub struct float64x1_t(f64); // FIXME: check this! + /// ARM-specific 128-bit wide vector of two packed `f64`. + #[stable(feature = "neon_intrinsics", since = "1.59.0")] + pub struct float64x2_t(f64, f64); +} + +/// ARM-specific type containing two `float64x1_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub struct float64x1x2_t(pub float64x1_t, pub float64x1_t); +/// ARM-specific type containing three `float64x1_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub struct float64x1x3_t(pub float64x1_t, pub float64x1_t, pub float64x1_t); +/// ARM-specific type containing four `float64x1_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub struct float64x1x4_t( + pub float64x1_t, + pub float64x1_t, + pub float64x1_t, + pub float64x1_t, +); + +/// ARM-specific type containing two `float64x2_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub struct float64x2x2_t(pub float64x2_t, pub float64x2_t); +/// ARM-specific type containing three `float64x2_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub struct float64x2x3_t(pub float64x2_t, pub float64x2_t, pub float64x2_t); +/// ARM-specific type containing four `float64x2_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub struct float64x2x4_t( + pub float64x2_t, + pub float64x2_t, + pub float64x2_t, + pub float64x2_t, +); + +#[allow(improper_ctypes)] +extern "unadjusted" { + // absolute value + #[link_name = "llvm.aarch64.neon.abs.i64"] + fn vabsd_s64_(a: i64) -> i64; + #[link_name = "llvm.aarch64.neon.abs.v1i64"] + fn vabs_s64_(a: int64x1_t) -> int64x1_t; + #[link_name = "llvm.aarch64.neon.abs.v2i64"] + fn vabsq_s64_(a: int64x2_t) -> int64x2_t; + + #[link_name = "llvm.aarch64.neon.suqadd.v8i8"] + fn vuqadd_s8_(a: int8x8_t, b: uint8x8_t) -> int8x8_t; + #[link_name = "llvm.aarch64.neon.suqadd.v16i8"] + fn vuqaddq_s8_(a: int8x16_t, b: uint8x16_t) -> int8x16_t; + #[link_name = "llvm.aarch64.neon.suqadd.v4i16"] + fn vuqadd_s16_(a: int16x4_t, b: uint16x4_t) -> int16x4_t; + #[link_name = "llvm.aarch64.neon.suqadd.v8i16"] + fn vuqaddq_s16_(a: int16x8_t, b: uint16x8_t) -> int16x8_t; + #[link_name = "llvm.aarch64.neon.suqadd.v2i32"] + fn vuqadd_s32_(a: int32x2_t, b: uint32x2_t) -> int32x2_t; + #[link_name = "llvm.aarch64.neon.suqadd.v4i32"] + fn vuqaddq_s32_(a: int32x4_t, b: uint32x4_t) -> int32x4_t; + #[link_name = "llvm.aarch64.neon.suqadd.v1i64"] + fn vuqadd_s64_(a: int64x1_t, b: uint64x1_t) -> int64x1_t; + #[link_name = "llvm.aarch64.neon.suqadd.v2i64"] + fn vuqaddq_s64_(a: int64x2_t, b: uint64x2_t) -> int64x2_t; + + #[link_name = "llvm.aarch64.neon.usqadd.v8i8"] + fn vsqadd_u8_(a: uint8x8_t, b: int8x8_t) -> uint8x8_t; + #[link_name = "llvm.aarch64.neon.usqadd.v16i8"] + fn vsqaddq_u8_(a: uint8x16_t, b: int8x16_t) -> uint8x16_t; + #[link_name = "llvm.aarch64.neon.usqadd.v4i16"] + fn vsqadd_u16_(a: uint16x4_t, b: int16x4_t) -> uint16x4_t; + #[link_name = "llvm.aarch64.neon.usqadd.v8i16"] + fn vsqaddq_u16_(a: uint16x8_t, b: int16x8_t) -> uint16x8_t; + #[link_name = "llvm.aarch64.neon.usqadd.v2i32"] + fn vsqadd_u32_(a: uint32x2_t, b: int32x2_t) -> uint32x2_t; + #[link_name = "llvm.aarch64.neon.usqadd.v4i32"] + fn vsqaddq_u32_(a: uint32x4_t, b: int32x4_t) -> uint32x4_t; + #[link_name = "llvm.aarch64.neon.usqadd.v1i64"] + fn vsqadd_u64_(a: uint64x1_t, b: int64x1_t) -> uint64x1_t; + #[link_name = "llvm.aarch64.neon.usqadd.v2i64"] + fn vsqaddq_u64_(a: uint64x2_t, b: int64x2_t) -> uint64x2_t; + + #[link_name = "llvm.aarch64.neon.addp.v8i16"] + fn vpaddq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + #[link_name = "llvm.aarch64.neon.addp.v4i32"] + fn vpaddq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + #[link_name = "llvm.aarch64.neon.addp.v2i64"] + fn vpaddq_s64_(a: int64x2_t, b: int64x2_t) -> int64x2_t; + #[link_name = "llvm.aarch64.neon.addp.v16i8"] + fn vpaddq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + + #[link_name = "llvm.aarch64.neon.saddv.i32.v4i16"] + fn vaddv_s16_(a: int16x4_t) -> i16; + #[link_name = "llvm.aarch64.neon.saddv.i32.v2i32"] + fn vaddv_s32_(a: int32x2_t) -> i32; + #[link_name = "llvm.aarch64.neon.saddv.i32.v8i8"] + fn vaddv_s8_(a: int8x8_t) -> i8; + #[link_name = "llvm.aarch64.neon.uaddv.i32.v4i16"] + fn vaddv_u16_(a: uint16x4_t) -> u16; + #[link_name = "llvm.aarch64.neon.uaddv.i32.v2i32"] + fn vaddv_u32_(a: uint32x2_t) -> u32; + #[link_name = "llvm.aarch64.neon.uaddv.i32.v8i8"] + fn vaddv_u8_(a: uint8x8_t) -> u8; + #[link_name = "llvm.aarch64.neon.saddv.i32.v8i16"] + fn vaddvq_s16_(a: int16x8_t) -> i16; + #[link_name = "llvm.aarch64.neon.saddv.i32.v4i32"] + fn vaddvq_s32_(a: int32x4_t) -> i32; + #[link_name = "llvm.aarch64.neon.saddv.i32.v16i8"] + fn vaddvq_s8_(a: int8x16_t) -> i8; + #[link_name = "llvm.aarch64.neon.uaddv.i32.v8i16"] + fn vaddvq_u16_(a: uint16x8_t) -> u16; + #[link_name = "llvm.aarch64.neon.uaddv.i32.v4i32"] + fn vaddvq_u32_(a: uint32x4_t) -> u32; + #[link_name = "llvm.aarch64.neon.uaddv.i32.v16i8"] + fn vaddvq_u8_(a: uint8x16_t) -> u8; + #[link_name = "llvm.aarch64.neon.saddv.i64.v2i64"] + fn vaddvq_s64_(a: int64x2_t) -> i64; + #[link_name = "llvm.aarch64.neon.uaddv.i64.v2i64"] + fn vaddvq_u64_(a: uint64x2_t) -> u64; + + #[link_name = "llvm.aarch64.neon.saddlv.i32.v8i8"] + fn vaddlv_s8_(a: int8x8_t) -> i32; + #[link_name = "llvm.aarch64.neon.uaddlv.i32.v8i8"] + fn vaddlv_u8_(a: uint8x8_t) -> u32; + #[link_name = "llvm.aarch64.neon.saddlv.i32.v16i8"] + fn vaddlvq_s8_(a: int8x16_t) -> i32; + #[link_name = "llvm.aarch64.neon.uaddlv.i32.v16i8"] + fn vaddlvq_u8_(a: uint8x16_t) -> u32; + + #[link_name = "llvm.aarch64.neon.smaxv.i8.v8i8"] + fn vmaxv_s8_(a: int8x8_t) -> i8; + #[link_name = "llvm.aarch64.neon.smaxv.i8.6i8"] + fn vmaxvq_s8_(a: int8x16_t) -> i8; + #[link_name = "llvm.aarch64.neon.smaxv.i16.v4i16"] + fn vmaxv_s16_(a: int16x4_t) -> i16; + #[link_name = "llvm.aarch64.neon.smaxv.i16.v8i16"] + fn vmaxvq_s16_(a: int16x8_t) -> i16; + #[link_name = "llvm.aarch64.neon.smaxv.i32.v2i32"] + fn vmaxv_s32_(a: int32x2_t) -> i32; + #[link_name = "llvm.aarch64.neon.smaxv.i32.v4i32"] + fn vmaxvq_s32_(a: int32x4_t) -> i32; + + #[link_name = "llvm.aarch64.neon.umaxv.i8.v8i8"] + fn vmaxv_u8_(a: uint8x8_t) -> u8; + #[link_name = "llvm.aarch64.neon.umaxv.i8.6i8"] + fn vmaxvq_u8_(a: uint8x16_t) -> u8; + #[link_name = "llvm.aarch64.neon.umaxv.i16.v4i16"] + fn vmaxv_u16_(a: uint16x4_t) -> u16; + #[link_name = "llvm.aarch64.neon.umaxv.i16.v8i16"] + fn vmaxvq_u16_(a: uint16x8_t) -> u16; + #[link_name = "llvm.aarch64.neon.umaxv.i32.v2i32"] + fn vmaxv_u32_(a: uint32x2_t) -> u32; + #[link_name = "llvm.aarch64.neon.umaxv.i32.v4i32"] + fn vmaxvq_u32_(a: uint32x4_t) -> u32; + + #[link_name = "llvm.aarch64.neon.fmaxv.f32.v2f32"] + fn vmaxv_f32_(a: float32x2_t) -> f32; + #[link_name = "llvm.aarch64.neon.fmaxv.f32.v4f32"] + fn vmaxvq_f32_(a: float32x4_t) -> f32; + #[link_name = "llvm.aarch64.neon.fmaxv.f64.v2f64"] + fn vmaxvq_f64_(a: float64x2_t) -> f64; + + #[link_name = "llvm.aarch64.neon.sminv.i8.v8i8"] + fn vminv_s8_(a: int8x8_t) -> i8; + #[link_name = "llvm.aarch64.neon.sminv.i8.6i8"] + fn vminvq_s8_(a: int8x16_t) -> i8; + #[link_name = "llvm.aarch64.neon.sminv.i16.v4i16"] + fn vminv_s16_(a: int16x4_t) -> i16; + #[link_name = "llvm.aarch64.neon.sminv.i16.v8i16"] + fn vminvq_s16_(a: int16x8_t) -> i16; + #[link_name = "llvm.aarch64.neon.sminv.i32.v2i32"] + fn vminv_s32_(a: int32x2_t) -> i32; + #[link_name = "llvm.aarch64.neon.sminv.i32.v4i32"] + fn vminvq_s32_(a: int32x4_t) -> i32; + + #[link_name = "llvm.aarch64.neon.uminv.i8.v8i8"] + fn vminv_u8_(a: uint8x8_t) -> u8; + #[link_name = "llvm.aarch64.neon.uminv.i8.6i8"] + fn vminvq_u8_(a: uint8x16_t) -> u8; + #[link_name = "llvm.aarch64.neon.uminv.i16.v4i16"] + fn vminv_u16_(a: uint16x4_t) -> u16; + #[link_name = "llvm.aarch64.neon.uminv.i16.v8i16"] + fn vminvq_u16_(a: uint16x8_t) -> u16; + #[link_name = "llvm.aarch64.neon.uminv.i32.v2i32"] + fn vminv_u32_(a: uint32x2_t) -> u32; + #[link_name = "llvm.aarch64.neon.uminv.i32.v4i32"] + fn vminvq_u32_(a: uint32x4_t) -> u32; + + #[link_name = "llvm.aarch64.neon.fminv.f32.v2f32"] + fn vminv_f32_(a: float32x2_t) -> f32; + #[link_name = "llvm.aarch64.neon.fminv.f32.v4f32"] + fn vminvq_f32_(a: float32x4_t) -> f32; + #[link_name = "llvm.aarch64.neon.fminv.f64.v2f64"] + fn vminvq_f64_(a: float64x2_t) -> f64; + + #[link_name = "llvm.aarch64.neon.sminp.v16i8"] + fn vpminq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + #[link_name = "llvm.aarch64.neon.sminp.v8i16"] + fn vpminq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + #[link_name = "llvm.aarch64.neon.sminp.v4i32"] + fn vpminq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + #[link_name = "llvm.aarch64.neon.uminp.v16i8"] + fn vpminq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; + #[link_name = "llvm.aarch64.neon.uminp.v8i16"] + fn vpminq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; + #[link_name = "llvm.aarch64.neon.uminp.v4i32"] + fn vpminq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; + #[link_name = "llvm.aarch64.neon.fminp.4f32"] + fn vpminq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + #[link_name = "llvm.aarch64.neon.fminp.v2f64"] + fn vpminq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + + #[link_name = "llvm.aarch64.neon.smaxp.v16i8"] + fn vpmaxq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + #[link_name = "llvm.aarch64.neon.smaxp.v8i16"] + fn vpmaxq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + #[link_name = "llvm.aarch64.neon.smaxp.v4i32"] + fn vpmaxq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + #[link_name = "llvm.aarch64.neon.umaxp.v16i8"] + fn vpmaxq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; + #[link_name = "llvm.aarch64.neon.umaxp.v8i16"] + fn vpmaxq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; + #[link_name = "llvm.aarch64.neon.umaxp.v4i32"] + fn vpmaxq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; + #[link_name = "llvm.aarch64.neon.fmaxp.4f32"] + fn vpmaxq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + #[link_name = "llvm.aarch64.neon.fmaxp.v2f64"] + fn vpmaxq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + + #[link_name = "llvm.aarch64.neon.tbl1.v8i8"] + fn vqtbl1(a: int8x16_t, b: uint8x8_t) -> int8x8_t; + #[link_name = "llvm.aarch64.neon.tbl1.v16i8"] + fn vqtbl1q(a: int8x16_t, b: uint8x16_t) -> int8x16_t; + + #[link_name = "llvm.aarch64.neon.tbx1.v8i8"] + fn vqtbx1(a: int8x8_t, b: int8x16_t, c: uint8x8_t) -> int8x8_t; + #[link_name = "llvm.aarch64.neon.tbx1.v16i8"] + fn vqtbx1q(a: int8x16_t, b: int8x16_t, c: uint8x16_t) -> int8x16_t; + + #[link_name = "llvm.aarch64.neon.tbl2.v8i8"] + fn vqtbl2(a0: int8x16_t, a1: int8x16_t, b: uint8x8_t) -> int8x8_t; + #[link_name = "llvm.aarch64.neon.tbl2.v16i8"] + fn vqtbl2q(a0: int8x16_t, a1: int8x16_t, b: uint8x16_t) -> int8x16_t; + + #[link_name = "llvm.aarch64.neon.tbx2.v8i8"] + fn vqtbx2(a: int8x8_t, b0: int8x16_t, b1: int8x16_t, c: uint8x8_t) -> int8x8_t; + #[link_name = "llvm.aarch64.neon.tbx2.v16i8"] + fn vqtbx2q(a: int8x16_t, b0: int8x16_t, b1: int8x16_t, c: uint8x16_t) -> int8x16_t; + + #[link_name = "llvm.aarch64.neon.tbl3.v8i8"] + fn vqtbl3(a0: int8x16_t, a1: int8x16_t, a2: int8x16_t, b: uint8x8_t) -> int8x8_t; + #[link_name = "llvm.aarch64.neon.tbl3.v16i8"] + fn vqtbl3q(a0: int8x16_t, a1: int8x16_t, a2: int8x16_t, b: uint8x16_t) -> int8x16_t; + + #[link_name = "llvm.aarch64.neon.tbx3.v8i8"] + fn vqtbx3(a: int8x8_t, b0: int8x16_t, b1: int8x16_t, b2: int8x16_t, c: uint8x8_t) -> int8x8_t; + #[link_name = "llvm.aarch64.neon.tbx3.v16i8"] + fn vqtbx3q( + a: int8x16_t, + b0: int8x16_t, + b1: int8x16_t, + b2: int8x16_t, + c: uint8x16_t, + ) -> int8x16_t; + + #[link_name = "llvm.aarch64.neon.tbl4.v8i8"] + fn vqtbl4(a0: int8x16_t, a1: int8x16_t, a2: int8x16_t, a3: int8x16_t, b: uint8x8_t) + -> int8x8_t; + #[link_name = "llvm.aarch64.neon.tbl4.v16i8"] + fn vqtbl4q( + a0: int8x16_t, + a1: int8x16_t, + a2: int8x16_t, + a3: int8x16_t, + b: uint8x16_t, + ) -> int8x16_t; + + #[link_name = "llvm.aarch64.neon.tbx4.v8i8"] + fn vqtbx4( + a: int8x8_t, + b0: int8x16_t, + b1: int8x16_t, + b2: int8x16_t, + b3: int8x16_t, + c: uint8x8_t, + ) -> int8x8_t; + + #[link_name = "llvm.aarch64.neon.tbx4.v16i8"] + fn vqtbx4q( + a: int8x16_t, + b0: int8x16_t, + b1: int8x16_t, + b2: int8x16_t, + b3: int8x16_t, + c: uint8x16_t, + ) -> int8x16_t; + + #[link_name = "llvm.aarch64.neon.vsli.v8i8"] + fn vsli_n_s8_(a: int8x8_t, b: int8x8_t, n: i32) -> int8x8_t; + #[link_name = "llvm.aarch64.neon.vsli.v16i8"] + fn vsliq_n_s8_(a: int8x16_t, b: int8x16_t, n: i32) -> int8x16_t; + #[link_name = "llvm.aarch64.neon.vsli.v4i16"] + fn vsli_n_s16_(a: int16x4_t, b: int16x4_t, n: i32) -> int16x4_t; + #[link_name = "llvm.aarch64.neon.vsli.v8i16"] + fn vsliq_n_s16_(a: int16x8_t, b: int16x8_t, n: i32) -> int16x8_t; + #[link_name = "llvm.aarch64.neon.vsli.v2i32"] + fn vsli_n_s32_(a: int32x2_t, b: int32x2_t, n: i32) -> int32x2_t; + #[link_name = "llvm.aarch64.neon.vsli.v4i32"] + fn vsliq_n_s32_(a: int32x4_t, b: int32x4_t, n: i32) -> int32x4_t; + #[link_name = "llvm.aarch64.neon.vsli.v1i64"] + fn vsli_n_s64_(a: int64x1_t, b: int64x1_t, n: i32) -> int64x1_t; + #[link_name = "llvm.aarch64.neon.vsli.v2i64"] + fn vsliq_n_s64_(a: int64x2_t, b: int64x2_t, n: i32) -> int64x2_t; + + #[link_name = "llvm.aarch64.neon.vsri.v8i8"] + fn vsri_n_s8_(a: int8x8_t, b: int8x8_t, n: i32) -> int8x8_t; + #[link_name = "llvm.aarch64.neon.vsri.v16i8"] + fn vsriq_n_s8_(a: int8x16_t, b: int8x16_t, n: i32) -> int8x16_t; + #[link_name = "llvm.aarch64.neon.vsri.v4i16"] + fn vsri_n_s16_(a: int16x4_t, b: int16x4_t, n: i32) -> int16x4_t; + #[link_name = "llvm.aarch64.neon.vsri.v8i16"] + fn vsriq_n_s16_(a: int16x8_t, b: int16x8_t, n: i32) -> int16x8_t; + #[link_name = "llvm.aarch64.neon.vsri.v2i32"] + fn vsri_n_s32_(a: int32x2_t, b: int32x2_t, n: i32) -> int32x2_t; + #[link_name = "llvm.aarch64.neon.vsri.v4i32"] + fn vsriq_n_s32_(a: int32x4_t, b: int32x4_t, n: i32) -> int32x4_t; + #[link_name = "llvm.aarch64.neon.vsri.v1i64"] + fn vsri_n_s64_(a: int64x1_t, b: int64x1_t, n: i32) -> int64x1_t; + #[link_name = "llvm.aarch64.neon.vsri.v2i64"] + fn vsriq_n_s64_(a: int64x2_t, b: int64x2_t, n: i32) -> int64x2_t; +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N1 = 0, N2 = 0))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_lane_s64( + _a: int64x1_t, + b: int64x1_t, +) -> int64x1_t { + static_assert!(N1 : i32 where N1 == 0); + static_assert!(N2 : i32 where N2 == 0); + b +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N1 = 0, N2 = 0))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_lane_u64( + _a: uint64x1_t, + b: uint64x1_t, +) -> uint64x1_t { + static_assert!(N1 : i32 where N1 == 0); + static_assert!(N2 : i32 where N2 == 0); + b +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N1 = 0, N2 = 0))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_lane_p64( + _a: poly64x1_t, + b: poly64x1_t, +) -> poly64x1_t { + static_assert!(N1 : i32 where N1 == 0); + static_assert!(N2 : i32 where N2 == 0); + b +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N1 = 0, N2 = 0))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_lane_f64( + _a: float64x1_t, + b: float64x1_t, +) -> float64x1_t { + static_assert!(N1 : i32 where N1 == 0); + static_assert!(N2 : i32 where N2 == 0); + b +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_laneq_s64( + _a: int64x1_t, + b: int64x2_t, +) -> int64x1_t { + static_assert!(LANE1 : i32 where LANE1 == 0); + static_assert_imm1!(LANE2); + transmute::(simd_extract(b, LANE2 as u32)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_laneq_u64( + _a: uint64x1_t, + b: uint64x2_t, +) -> uint64x1_t { + static_assert!(LANE1 : i32 where LANE1 == 0); + static_assert_imm1!(LANE2); + transmute::(simd_extract(b, LANE2 as u32)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_laneq_p64( + _a: poly64x1_t, + b: poly64x2_t, +) -> poly64x1_t { + static_assert!(LANE1 : i32 where LANE1 == 0); + static_assert_imm1!(LANE2); + transmute::(simd_extract(b, LANE2 as u32)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcopy_laneq_f64( + _a: float64x1_t, + b: float64x2_t, +) -> float64x1_t { + static_assert!(LANE1 : i32 where LANE1 == 0); + static_assert_imm1!(LANE2); + transmute::(simd_extract(b, LANE2 as u32)) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1_s8(ptr: *const i8) -> int8x8_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1q_s8(ptr: *const i8) -> int8x16_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1_s16(ptr: *const i16) -> int16x4_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1q_s16(ptr: *const i16) -> int16x8_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1_s32(ptr: *const i32) -> int32x2_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1q_s32(ptr: *const i32) -> int32x4_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1_s64(ptr: *const i64) -> int64x1_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1q_s64(ptr: *const i64) -> int64x2_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1_u8(ptr: *const u8) -> uint8x8_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1q_u8(ptr: *const u8) -> uint8x16_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1_u16(ptr: *const u16) -> uint16x4_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1q_u16(ptr: *const u16) -> uint16x8_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1_u32(ptr: *const u32) -> uint32x2_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1q_u32(ptr: *const u32) -> uint32x4_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1_u64(ptr: *const u64) -> uint64x1_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1q_u64(ptr: *const u64) -> uint64x2_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1_p8(ptr: *const p8) -> poly8x8_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1q_p8(ptr: *const p8) -> poly8x16_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1_p16(ptr: *const p16) -> poly16x4_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1q_p16(ptr: *const p16) -> poly16x8_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1_p64(ptr: *const p64) -> poly64x1_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1q_p64(ptr: *const p64) -> poly64x2_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1_f32(ptr: *const f32) -> float32x2_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1q_f32(ptr: *const f32) -> float32x4_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1_f64(ptr: *const f64) -> float64x1_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1q_f64(ptr: *const f64) -> float64x2_t { + read_unaligned(ptr.cast()) +} + +/// Load multiple single-element structures to one, two, three, or four registers +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1_dup_f64(ptr: *const f64) -> float64x1_t { + vld1_f64(ptr) +} + +/// Load multiple single-element structures to one, two, three, or four registers +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld1r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1q_dup_f64(ptr: *const f64) -> float64x2_t { + let x = vld1q_lane_f64::<0>(ptr, transmute(f64x2::splat(0.))); + simd_shuffle2!(x, x, [0, 0]) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(ldr, LANE = 0))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1_lane_f64(ptr: *const f64, src: float64x1_t) -> float64x1_t { + static_assert!(LANE : i32 where LANE == 0); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(ld1, LANE = 1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld1q_lane_f64(ptr: *const f64, src: float64x2_t) -> float64x2_t { + static_assert_imm1!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_s8(ptr: *mut i8, a: int8x8_t) { + write_unaligned(ptr.cast(), a); +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_s8(ptr: *mut i8, a: int8x16_t) { + write_unaligned(ptr.cast(), a); +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_s16(ptr: *mut i16, a: int16x4_t) { + write_unaligned(ptr.cast(), a); +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_s16(ptr: *mut i16, a: int16x8_t) { + write_unaligned(ptr.cast(), a); +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_s32(ptr: *mut i32, a: int32x2_t) { + write_unaligned(ptr.cast(), a); +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_s32(ptr: *mut i32, a: int32x4_t) { + write_unaligned(ptr.cast(), a); +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_s64(ptr: *mut i64, a: int64x1_t) { + write_unaligned(ptr.cast(), a); +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_s64(ptr: *mut i64, a: int64x2_t) { + write_unaligned(ptr.cast(), a); +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_u8(ptr: *mut u8, a: uint8x8_t) { + write_unaligned(ptr.cast(), a); +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_u8(ptr: *mut u8, a: uint8x16_t) { + write_unaligned(ptr.cast(), a); +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_u16(ptr: *mut u16, a: uint16x4_t) { + write_unaligned(ptr.cast(), a); +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_u16(ptr: *mut u16, a: uint16x8_t) { + write_unaligned(ptr.cast(), a); +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_u32(ptr: *mut u32, a: uint32x2_t) { + write_unaligned(ptr.cast(), a); +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_u32(ptr: *mut u32, a: uint32x4_t) { + write_unaligned(ptr.cast(), a); +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_u64(ptr: *mut u64, a: uint64x1_t) { + write_unaligned(ptr.cast(), a); +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_u64(ptr: *mut u64, a: uint64x2_t) { + write_unaligned(ptr.cast(), a); +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_p8(ptr: *mut p8, a: poly8x8_t) { + write_unaligned(ptr.cast(), a); +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_p8(ptr: *mut p8, a: poly8x16_t) { + write_unaligned(ptr.cast(), a); +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_p16(ptr: *mut p16, a: poly16x4_t) { + write_unaligned(ptr.cast(), a); +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_p16(ptr: *mut p16, a: poly16x8_t) { + write_unaligned(ptr.cast(), a); +} + +// Store multiple single-element structures from one, two, three, or four registers. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_p64(ptr: *mut p64, a: poly64x1_t) { + write_unaligned(ptr.cast(), a); +} + +// Store multiple single-element structures from one, two, three, or four registers. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_p64(ptr: *mut p64, a: poly64x2_t) { + write_unaligned(ptr.cast(), a); +} + +// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_f32(ptr: *mut f32, a: float32x2_t) { + write_unaligned(ptr.cast(), a); +} + +// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_f32(ptr: *mut f32, a: float32x4_t) { + write_unaligned(ptr.cast(), a); +} + +// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_f64(ptr: *mut f64, a: float64x1_t) { + write_unaligned(ptr.cast(), a); +} + +// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_f64(ptr: *mut f64, a: float64x2_t) { + write_unaligned(ptr.cast(), a); +} + +/// Absolute Value (wrapping). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(abs))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vabsd_s64(a: i64) -> i64 { + vabsd_s64_(a) +} +/// Absolute Value (wrapping). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(abs))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vabs_s64(a: int64x1_t) -> int64x1_t { + vabs_s64_(a) +} +/// Absolute Value (wrapping). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(abs))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vabsq_s64(a: int64x2_t) -> int64x2_t { + vabsq_s64_(a) +} + +/// Bitwise Select instructions. This instruction sets each bit in the destination SIMD&FP register +/// to the corresponding bit from the first source SIMD&FP register when the original +/// destination bit was 1, otherwise from the second source SIMD&FP register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(bsl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vbsl_f64(a: uint64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t { + let not = int64x1_t(-1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} +/// Bitwise Select. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(bsl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vbsl_p64(a: poly64x1_t, b: poly64x1_t, c: poly64x1_t) -> poly64x1_t { + let not = int64x1_t(-1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} +/// Bitwise Select. (128-bit) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(bsl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vbslq_f64(a: uint64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t { + let not = int64x2_t(-1, -1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} +/// Bitwise Select. (128-bit) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(bsl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vbslq_p64(a: poly64x2_t, b: poly64x2_t, c: poly64x2_t) -> poly64x2_t { + let not = int64x2_t(-1, -1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Signed saturating Accumulate of Unsigned value. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(suqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuqadd_s8(a: int8x8_t, b: uint8x8_t) -> int8x8_t { + vuqadd_s8_(a, b) +} +/// Signed saturating Accumulate of Unsigned value. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(suqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuqaddq_s8(a: int8x16_t, b: uint8x16_t) -> int8x16_t { + vuqaddq_s8_(a, b) +} +/// Signed saturating Accumulate of Unsigned value. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(suqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuqadd_s16(a: int16x4_t, b: uint16x4_t) -> int16x4_t { + vuqadd_s16_(a, b) +} +/// Signed saturating Accumulate of Unsigned value. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(suqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuqaddq_s16(a: int16x8_t, b: uint16x8_t) -> int16x8_t { + vuqaddq_s16_(a, b) +} +/// Signed saturating Accumulate of Unsigned value. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(suqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuqadd_s32(a: int32x2_t, b: uint32x2_t) -> int32x2_t { + vuqadd_s32_(a, b) +} +/// Signed saturating Accumulate of Unsigned value. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(suqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuqaddq_s32(a: int32x4_t, b: uint32x4_t) -> int32x4_t { + vuqaddq_s32_(a, b) +} +/// Signed saturating Accumulate of Unsigned value. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(suqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuqadd_s64(a: int64x1_t, b: uint64x1_t) -> int64x1_t { + vuqadd_s64_(a, b) +} +/// Signed saturating Accumulate of Unsigned value. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(suqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vuqaddq_s64(a: int64x2_t, b: uint64x2_t) -> int64x2_t { + vuqaddq_s64_(a, b) +} + +/// Unsigned saturating Accumulate of Signed value. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsqadd_u8(a: uint8x8_t, b: int8x8_t) -> uint8x8_t { + vsqadd_u8_(a, b) +} +/// Unsigned saturating Accumulate of Signed value. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsqaddq_u8(a: uint8x16_t, b: int8x16_t) -> uint8x16_t { + vsqaddq_u8_(a, b) +} +/// Unsigned saturating Accumulate of Signed value. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsqadd_u16(a: uint16x4_t, b: int16x4_t) -> uint16x4_t { + vsqadd_u16_(a, b) +} +/// Unsigned saturating Accumulate of Signed value. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsqaddq_u16(a: uint16x8_t, b: int16x8_t) -> uint16x8_t { + vsqaddq_u16_(a, b) +} +/// Unsigned saturating Accumulate of Signed value. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsqadd_u32(a: uint32x2_t, b: int32x2_t) -> uint32x2_t { + vsqadd_u32_(a, b) +} +/// Unsigned saturating Accumulate of Signed value. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsqaddq_u32(a: uint32x4_t, b: int32x4_t) -> uint32x4_t { + vsqaddq_u32_(a, b) +} +/// Unsigned saturating Accumulate of Signed value. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsqadd_u64(a: uint64x1_t, b: int64x1_t) -> uint64x1_t { + vsqadd_u64_(a, b) +} +/// Unsigned saturating Accumulate of Signed value. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usqadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsqaddq_u64(a: uint64x2_t, b: int64x2_t) -> uint64x2_t { + vsqaddq_u64_(a, b) +} + +/// Add pairwise +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpaddq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + vpaddq_s16_(a, b) +} +/// Add pairwise +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpaddq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + transmute(vpaddq_s16_(transmute(a), transmute(b))) +} +/// Add pairwise +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpaddq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + vpaddq_s32_(a, b) +} +/// Add pairwise +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpaddq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + transmute(vpaddq_s32_(transmute(a), transmute(b))) +} +/// Add pairwise +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpaddq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + vpaddq_s64_(a, b) +} +/// Add pairwise +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpaddq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + transmute(vpaddq_s64_(transmute(a), transmute(b))) +} +/// Add pairwise +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpaddq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + vpaddq_s8_(a, b) +} +/// Add pairwise +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpaddq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + transmute(vpaddq_s8_(transmute(a), transmute(b))) +} +/// Add pairwise +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpaddd_s64(a: int64x2_t) -> i64 { + transmute(vaddvq_u64_(transmute(a))) +} +/// Add pairwise +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpaddd_u64(a: uint64x2_t) -> u64 { + transmute(vaddvq_u64_(transmute(a))) +} + +/// Add across vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddv_s16(a: int16x4_t) -> i16 { + vaddv_s16_(a) +} + +/// Add across vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddv_s32(a: int32x2_t) -> i32 { + vaddv_s32_(a) +} + +/// Add across vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddv_s8(a: int8x8_t) -> i8 { + vaddv_s8_(a) +} + +/// Add across vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddv_u16(a: uint16x4_t) -> u16 { + vaddv_u16_(a) +} + +/// Add across vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddv_u32(a: uint32x2_t) -> u32 { + vaddv_u32_(a) +} + +/// Add across vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddv_u8(a: uint8x8_t) -> u8 { + vaddv_u8_(a) +} + +/// Add across vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddvq_s16(a: int16x8_t) -> i16 { + vaddvq_s16_(a) +} + +/// Add across vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddvq_s32(a: int32x4_t) -> i32 { + vaddvq_s32_(a) +} + +/// Add across vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddvq_s8(a: int8x16_t) -> i8 { + vaddvq_s8_(a) +} + +/// Add across vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddvq_u16(a: uint16x8_t) -> u16 { + vaddvq_u16_(a) +} + +/// Add across vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddvq_u32(a: uint32x4_t) -> u32 { + vaddvq_u32_(a) +} + +/// Add across vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddvq_u8(a: uint8x16_t) -> u8 { + vaddvq_u8_(a) +} + +/// Add across vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddvq_s64(a: int64x2_t) -> i64 { + vaddvq_s64_(a) +} + +/// Add across vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(addp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddvq_u64(a: uint64x2_t) -> u64 { + vaddvq_u64_(a) +} + +/// Signed Add Long across Vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(saddlv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddlv_s8(a: int8x8_t) -> i16 { + vaddlv_s8_(a) as i16 +} + +/// Signed Add Long across Vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(saddlv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddlvq_s8(a: int8x16_t) -> i16 { + vaddlvq_s8_(a) as i16 +} + +/// Unsigned Add Long across Vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uaddlv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddlv_u8(a: uint8x8_t) -> u16 { + vaddlv_u8_(a) as u16 +} + +/// Unsigned Add Long across Vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uaddlv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddlvq_u8(a: uint8x16_t) -> u16 { + vaddlvq_u8_(a) as u16 +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vadd_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + simd_add(a, b) +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fadd))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + simd_add(a, b) +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(add))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vadd_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + simd_add(a, b) +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(add))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vadd_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + simd_add(a, b) +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(add))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddd_s64(a: i64, b: i64) -> i64 { + a.wrapping_add(b) +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(add))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vaddd_u64(a: u64, b: u64) -> u64 { + a.wrapping_add(b) +} + +/// Horizontal vector max. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smaxv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmaxv_s8(a: int8x8_t) -> i8 { + vmaxv_s8_(a) +} + +/// Horizontal vector max. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smaxv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmaxvq_s8(a: int8x16_t) -> i8 { + vmaxvq_s8_(a) +} + +/// Horizontal vector max. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smaxv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmaxv_s16(a: int16x4_t) -> i16 { + vmaxv_s16_(a) +} + +/// Horizontal vector max. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smaxv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmaxvq_s16(a: int16x8_t) -> i16 { + vmaxvq_s16_(a) +} + +/// Horizontal vector max. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smaxp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmaxv_s32(a: int32x2_t) -> i32 { + vmaxv_s32_(a) +} + +/// Horizontal vector max. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smaxv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmaxvq_s32(a: int32x4_t) -> i32 { + vmaxvq_s32_(a) +} + +/// Horizontal vector max. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umaxv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmaxv_u8(a: uint8x8_t) -> u8 { + vmaxv_u8_(a) +} + +/// Horizontal vector max. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umaxv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmaxvq_u8(a: uint8x16_t) -> u8 { + vmaxvq_u8_(a) +} + +/// Horizontal vector max. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umaxv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmaxv_u16(a: uint16x4_t) -> u16 { + vmaxv_u16_(a) +} + +/// Horizontal vector max. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umaxv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmaxvq_u16(a: uint16x8_t) -> u16 { + vmaxvq_u16_(a) +} + +/// Horizontal vector max. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umaxp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmaxv_u32(a: uint32x2_t) -> u32 { + vmaxv_u32_(a) +} + +/// Horizontal vector max. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umaxv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmaxvq_u32(a: uint32x4_t) -> u32 { + vmaxvq_u32_(a) +} + +/// Horizontal vector max. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmaxv_f32(a: float32x2_t) -> f32 { + vmaxv_f32_(a) +} + +/// Horizontal vector max. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmaxvq_f32(a: float32x4_t) -> f32 { + vmaxvq_f32_(a) +} + +/// Horizontal vector max. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmaxvq_f64(a: float64x2_t) -> f64 { + vmaxvq_f64_(a) +} + +/// Horizontal vector min. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sminv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vminv_s8(a: int8x8_t) -> i8 { + vminv_s8_(a) +} + +/// Horizontal vector min. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sminv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vminvq_s8(a: int8x16_t) -> i8 { + vminvq_s8_(a) +} + +/// Horizontal vector min. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sminv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vminv_s16(a: int16x4_t) -> i16 { + vminv_s16_(a) +} + +/// Horizontal vector min. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sminv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vminvq_s16(a: int16x8_t) -> i16 { + vminvq_s16_(a) +} + +/// Horizontal vector min. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sminp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vminv_s32(a: int32x2_t) -> i32 { + vminv_s32_(a) +} + +/// Horizontal vector min. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sminv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vminvq_s32(a: int32x4_t) -> i32 { + vminvq_s32_(a) +} + +/// Horizontal vector min. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uminv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vminv_u8(a: uint8x8_t) -> u8 { + vminv_u8_(a) +} + +/// Horizontal vector min. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uminv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vminvq_u8(a: uint8x16_t) -> u8 { + vminvq_u8_(a) +} + +/// Horizontal vector min. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uminv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vminv_u16(a: uint16x4_t) -> u16 { + vminv_u16_(a) +} + +/// Horizontal vector min. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uminv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vminvq_u16(a: uint16x8_t) -> u16 { + vminvq_u16_(a) +} + +/// Horizontal vector min. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uminp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vminv_u32(a: uint32x2_t) -> u32 { + vminv_u32_(a) +} + +/// Horizontal vector min. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uminv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vminvq_u32(a: uint32x4_t) -> u32 { + vminvq_u32_(a) +} + +/// Horizontal vector min. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vminv_f32(a: float32x2_t) -> f32 { + vminv_f32_(a) +} + +/// Horizontal vector min. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminv))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vminvq_f32(a: float32x4_t) -> f32 { + vminvq_f32_(a) +} + +/// Horizontal vector min. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vminvq_f64(a: float64x2_t) -> f64 { + vminvq_f64_(a) +} + +/// Folding minimum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sminp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpminq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + vpminq_s8_(a, b) +} + +/// Folding minimum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sminp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpminq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + vpminq_s16_(a, b) +} + +/// Folding minimum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sminp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpminq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + vpminq_s32_(a, b) +} + +/// Folding minimum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uminp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpminq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + vpminq_u8_(a, b) +} + +/// Folding minimum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uminp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpminq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + vpminq_u16_(a, b) +} + +/// Folding minimum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uminp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpminq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + vpminq_u32_(a, b) +} + +/// Folding minimum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpminq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + vpminq_f32_(a, b) +} + +/// Folding minimum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpminq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + vpminq_f64_(a, b) +} + +/// Folding maximum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smaxp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpmaxq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + vpmaxq_s8_(a, b) +} + +/// Folding maximum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smaxp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpmaxq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + vpmaxq_s16_(a, b) +} + +/// Folding maximum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smaxp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpmaxq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + vpmaxq_s32_(a, b) +} + +/// Folding maximum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umaxp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpmaxq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + vpmaxq_u8_(a, b) +} + +/// Folding maximum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umaxp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpmaxq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + vpmaxq_u16_(a, b) +} + +/// Folding maximum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umaxp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpmaxq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + vpmaxq_u32_(a, b) +} + +/// Folding maximum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpmaxq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + vpmaxq_f32_(a, b) +} + +/// Folding maximum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxp))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vpmaxq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + vpmaxq_f64_(a, b) +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vext_p64(a: poly64x1_t, _b: poly64x1_t) -> poly64x1_t { + static_assert!(N : i32 where N == 0); + a +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vext_f64(a: float64x1_t, _b: float64x1_t) -> float64x1_t { + static_assert!(N : i32 where N == 0); + a +} +/// Vector combine +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcombine_s8(low: int8x8_t, high: int8x8_t) -> int8x16_t { + simd_shuffle16!( + low, + high, + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + ) +} + +/// Vector combine +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcombine_s16(low: int16x4_t, high: int16x4_t) -> int16x8_t { + simd_shuffle8!(low, high, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Vector combine +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcombine_s32(low: int32x2_t, high: int32x2_t) -> int32x4_t { + simd_shuffle4!(low, high, [0, 1, 2, 3]) +} + +/// Vector combine +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcombine_s64(low: int64x1_t, high: int64x1_t) -> int64x2_t { + simd_shuffle2!(low, high, [0, 1]) +} + +/// Vector combine +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcombine_u8(low: uint8x8_t, high: uint8x8_t) -> uint8x16_t { + simd_shuffle16!( + low, + high, + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + ) +} + +/// Vector combine +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcombine_u16(low: uint16x4_t, high: uint16x4_t) -> uint16x8_t { + simd_shuffle8!(low, high, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Vector combine +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcombine_u32(low: uint32x2_t, high: uint32x2_t) -> uint32x4_t { + simd_shuffle4!(low, high, [0, 1, 2, 3]) +} + +/// Vector combine +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcombine_u64(low: uint64x1_t, high: uint64x1_t) -> uint64x2_t { + simd_shuffle2!(low, high, [0, 1]) +} + +/// Vector combine +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcombine_p64(low: poly64x1_t, high: poly64x1_t) -> poly64x2_t { + simd_shuffle2!(low, high, [0, 1]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmov))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdup_n_p64(value: p64) -> poly64x1_t { + transmute(u64x1::new(value)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdup_n_f64(value: f64) -> float64x1_t { + float64x1_t(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(dup))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdupq_n_p64(value: p64) -> poly64x2_t { + transmute(u64x2::new(value, value)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(dup))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vdupq_n_f64(value: f64) -> float64x2_t { + float64x2_t(value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmov))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmov_n_p64(value: p64) -> poly64x1_t { + vdup_n_p64(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmov_n_f64(value: f64) -> float64x1_t { + vdup_n_f64(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(dup))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmovq_n_p64(value: p64) -> poly64x2_t { + vdupq_n_p64(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(dup))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vmovq_n_f64(value: f64) -> float64x2_t { + vdupq_n_f64(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vget_high_f64(a: float64x2_t) -> float64x1_t { + float64x1_t(simd_extract(a, 1)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ext))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vget_high_p64(a: poly64x2_t) -> poly64x1_t { + transmute(u64x1::new(simd_extract(a, 1))) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vget_low_f64(a: float64x2_t) -> float64x1_t { + float64x1_t(simd_extract(a, 0)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vget_low_p64(a: poly64x2_t) -> poly64x1_t { + transmute(u64x1::new(simd_extract(a, 0))) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, IMM5 = 0))] +pub unsafe fn vget_lane_f64(v: float64x1_t) -> f64 { + static_assert!(IMM5 : i32 where IMM5 == 0); + simd_extract(v, IMM5 as u32) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, IMM5 = 0))] +pub unsafe fn vgetq_lane_f64(v: float64x2_t) -> f64 { + static_assert_imm1!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/* FIXME: 16-bit float +/// Vector combine +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov))] +pub unsafe fn vcombine_f16 ( low: float16x4_t, high: float16x4_t) -> float16x8_t { + simd_shuffle8!(low, high, [0, 1, 2, 3, 4, 5, 6, 7]) +} +*/ + +/// Vector combine +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcombine_f32(low: float32x2_t, high: float32x2_t) -> float32x4_t { + simd_shuffle4!(low, high, [0, 1, 2, 3]) +} + +/// Vector combine +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcombine_p8(low: poly8x8_t, high: poly8x8_t) -> poly8x16_t { + simd_shuffle16!( + low, + high, + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + ) +} + +/// Vector combine +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcombine_p16(low: poly16x4_t, high: poly16x4_t) -> poly16x8_t { + simd_shuffle8!(low, high, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Vector combine +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcombine_f64(low: float64x1_t, high: float64x1_t) -> float64x2_t { + simd_shuffle2!(low, high, [0, 1]) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbl1_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + vqtbl1_s8(vcombine_s8(a, zeroed()), transmute(b)) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbl1_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + vqtbl1_u8(vcombine_u8(a, zeroed()), b) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbl1_p8(a: poly8x8_t, b: uint8x8_t) -> poly8x8_t { + vqtbl1_p8(vcombine_p8(a, zeroed()), b) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbl2_s8(a: int8x8x2_t, b: int8x8_t) -> int8x8_t { + vqtbl1_s8(vcombine_s8(a.0, a.1), transmute(b)) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbl2_u8(a: uint8x8x2_t, b: uint8x8_t) -> uint8x8_t { + vqtbl1_u8(vcombine_u8(a.0, a.1), b) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbl2_p8(a: poly8x8x2_t, b: uint8x8_t) -> poly8x8_t { + vqtbl1_p8(vcombine_p8(a.0, a.1), b) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbl3_s8(a: int8x8x3_t, b: int8x8_t) -> int8x8_t { + vqtbl2_s8( + int8x16x2_t(vcombine_s8(a.0, a.1), vcombine_s8(a.2, zeroed())), + transmute(b), + ) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbl3_u8(a: uint8x8x3_t, b: uint8x8_t) -> uint8x8_t { + vqtbl2_u8( + uint8x16x2_t(vcombine_u8(a.0, a.1), vcombine_u8(a.2, zeroed())), + b, + ) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbl3_p8(a: poly8x8x3_t, b: uint8x8_t) -> poly8x8_t { + vqtbl2_p8( + poly8x16x2_t(vcombine_p8(a.0, a.1), vcombine_p8(a.2, zeroed())), + b, + ) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbl4_s8(a: int8x8x4_t, b: int8x8_t) -> int8x8_t { + vqtbl2_s8( + int8x16x2_t(vcombine_s8(a.0, a.1), vcombine_s8(a.2, a.3)), + transmute(b), + ) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbl4_u8(a: uint8x8x4_t, b: uint8x8_t) -> uint8x8_t { + vqtbl2_u8( + uint8x16x2_t(vcombine_u8(a.0, a.1), vcombine_u8(a.2, a.3)), + b, + ) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbl4_p8(a: poly8x8x4_t, b: uint8x8_t) -> poly8x8_t { + vqtbl2_p8( + poly8x16x2_t(vcombine_p8(a.0, a.1), vcombine_p8(a.2, a.3)), + b, + ) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbx1_s8(a: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t { + let r = vqtbx1_s8(a, vcombine_s8(b, zeroed()), transmute(c)); + let m: int8x8_t = simd_lt(c, transmute(i8x8::splat(8))); + simd_select(m, r, a) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbx1_u8(a: uint8x8_t, b: uint8x8_t, c: uint8x8_t) -> uint8x8_t { + let r = vqtbx1_u8(a, vcombine_u8(b, zeroed()), c); + let m: int8x8_t = simd_lt(c, transmute(u8x8::splat(8))); + simd_select(m, r, a) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbx1_p8(a: poly8x8_t, b: poly8x8_t, c: uint8x8_t) -> poly8x8_t { + let r = vqtbx1_p8(a, vcombine_p8(b, zeroed()), c); + let m: int8x8_t = simd_lt(c, transmute(u8x8::splat(8))); + simd_select(m, r, a) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbx2_s8(a: int8x8_t, b: int8x8x2_t, c: int8x8_t) -> int8x8_t { + vqtbx1_s8(a, vcombine_s8(b.0, b.1), transmute(c)) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbx2_u8(a: uint8x8_t, b: uint8x8x2_t, c: uint8x8_t) -> uint8x8_t { + vqtbx1_u8(a, vcombine_u8(b.0, b.1), c) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbx2_p8(a: poly8x8_t, b: poly8x8x2_t, c: uint8x8_t) -> poly8x8_t { + vqtbx1_p8(a, vcombine_p8(b.0, b.1), c) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbx3_s8(a: int8x8_t, b: int8x8x3_t, c: int8x8_t) -> int8x8_t { + let r = vqtbx2_s8( + a, + int8x16x2_t(vcombine_s8(b.0, b.1), vcombine_s8(b.2, zeroed())), + transmute(c), + ); + let m: int8x8_t = simd_lt(c, transmute(i8x8::splat(24))); + simd_select(m, r, a) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbx3_u8(a: uint8x8_t, b: uint8x8x3_t, c: uint8x8_t) -> uint8x8_t { + let r = vqtbx2_u8( + a, + uint8x16x2_t(vcombine_u8(b.0, b.1), vcombine_u8(b.2, zeroed())), + c, + ); + let m: int8x8_t = simd_lt(c, transmute(u8x8::splat(24))); + simd_select(m, r, a) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbx3_p8(a: poly8x8_t, b: poly8x8x3_t, c: uint8x8_t) -> poly8x8_t { + let r = vqtbx2_p8( + a, + poly8x16x2_t(vcombine_p8(b.0, b.1), vcombine_p8(b.2, zeroed())), + c, + ); + let m: int8x8_t = simd_lt(c, transmute(u8x8::splat(24))); + simd_select(m, r, a) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbx4_s8(a: int8x8_t, b: int8x8x4_t, c: int8x8_t) -> int8x8_t { + vqtbx2_s8( + a, + int8x16x2_t(vcombine_s8(b.0, b.1), vcombine_s8(b.2, b.3)), + transmute(c), + ) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbx4_u8(a: uint8x8_t, b: uint8x8x4_t, c: uint8x8_t) -> uint8x8_t { + vqtbx2_u8( + a, + uint8x16x2_t(vcombine_u8(b.0, b.1), vcombine_u8(b.2, b.3)), + c, + ) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vtbx4_p8(a: poly8x8_t, b: poly8x8x4_t, c: uint8x8_t) -> poly8x8_t { + vqtbx2_p8( + a, + poly8x16x2_t(vcombine_p8(b.0, b.1), vcombine_p8(b.2, b.3)), + c, + ) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl1_s8(t: int8x16_t, idx: uint8x8_t) -> int8x8_t { + vqtbl1(t, idx) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl1q_s8(t: int8x16_t, idx: uint8x16_t) -> int8x16_t { + vqtbl1q(t, idx) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl1_u8(t: uint8x16_t, idx: uint8x8_t) -> uint8x8_t { + transmute(vqtbl1(transmute(t), transmute(idx))) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl1q_u8(t: uint8x16_t, idx: uint8x16_t) -> uint8x16_t { + transmute(vqtbl1q(transmute(t), transmute(idx))) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl1_p8(t: poly8x16_t, idx: uint8x8_t) -> poly8x8_t { + transmute(vqtbl1(transmute(t), transmute(idx))) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl1q_p8(t: poly8x16_t, idx: uint8x16_t) -> poly8x16_t { + transmute(vqtbl1q(transmute(t), transmute(idx))) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx1_s8(a: int8x8_t, t: int8x16_t, idx: uint8x8_t) -> int8x8_t { + vqtbx1(a, t, idx) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx1q_s8(a: int8x16_t, t: int8x16_t, idx: uint8x16_t) -> int8x16_t { + vqtbx1q(a, t, idx) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx1_u8(a: uint8x8_t, t: uint8x16_t, idx: uint8x8_t) -> uint8x8_t { + transmute(vqtbx1(transmute(a), transmute(t), transmute(idx))) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx1q_u8(a: uint8x16_t, t: uint8x16_t, idx: uint8x16_t) -> uint8x16_t { + transmute(vqtbx1q(transmute(a), transmute(t), transmute(idx))) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx1_p8(a: poly8x8_t, t: poly8x16_t, idx: uint8x8_t) -> poly8x8_t { + transmute(vqtbx1(transmute(a), transmute(t), transmute(idx))) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx1q_p8(a: poly8x16_t, t: poly8x16_t, idx: uint8x16_t) -> poly8x16_t { + transmute(vqtbx1q(transmute(a), transmute(t), transmute(idx))) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl2_s8(t: int8x16x2_t, idx: uint8x8_t) -> int8x8_t { + vqtbl2(t.0, t.1, idx) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl2q_s8(t: int8x16x2_t, idx: uint8x16_t) -> int8x16_t { + vqtbl2q(t.0, t.1, idx) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl2_u8(t: uint8x16x2_t, idx: uint8x8_t) -> uint8x8_t { + transmute(vqtbl2(transmute(t.0), transmute(t.1), transmute(idx))) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl2q_u8(t: uint8x16x2_t, idx: uint8x16_t) -> uint8x16_t { + transmute(vqtbl2q(transmute(t.0), transmute(t.1), transmute(idx))) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl2_p8(t: poly8x16x2_t, idx: uint8x8_t) -> poly8x8_t { + transmute(vqtbl2(transmute(t.0), transmute(t.1), transmute(idx))) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl2q_p8(t: poly8x16x2_t, idx: uint8x16_t) -> poly8x16_t { + transmute(vqtbl2q(transmute(t.0), transmute(t.1), transmute(idx))) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx2_s8(a: int8x8_t, t: int8x16x2_t, idx: uint8x8_t) -> int8x8_t { + vqtbx2(a, t.0, t.1, idx) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx2q_s8(a: int8x16_t, t: int8x16x2_t, idx: uint8x16_t) -> int8x16_t { + vqtbx2q(a, t.0, t.1, idx) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx2_u8(a: uint8x8_t, t: uint8x16x2_t, idx: uint8x8_t) -> uint8x8_t { + transmute(vqtbx2( + transmute(a), + transmute(t.0), + transmute(t.1), + transmute(idx), + )) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx2q_u8(a: uint8x16_t, t: uint8x16x2_t, idx: uint8x16_t) -> uint8x16_t { + transmute(vqtbx2q( + transmute(a), + transmute(t.0), + transmute(t.1), + transmute(idx), + )) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx2_p8(a: poly8x8_t, t: poly8x16x2_t, idx: uint8x8_t) -> poly8x8_t { + transmute(vqtbx2( + transmute(a), + transmute(t.0), + transmute(t.1), + transmute(idx), + )) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx2q_p8(a: poly8x16_t, t: poly8x16x2_t, idx: uint8x16_t) -> poly8x16_t { + transmute(vqtbx2q( + transmute(a), + transmute(t.0), + transmute(t.1), + transmute(idx), + )) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl3_s8(t: int8x16x3_t, idx: uint8x8_t) -> int8x8_t { + vqtbl3(t.0, t.1, t.2, idx) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl3q_s8(t: int8x16x3_t, idx: uint8x16_t) -> int8x16_t { + vqtbl3q(t.0, t.1, t.2, idx) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl3_u8(t: uint8x16x3_t, idx: uint8x8_t) -> uint8x8_t { + transmute(vqtbl3( + transmute(t.0), + transmute(t.1), + transmute(t.2), + transmute(idx), + )) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl3q_u8(t: uint8x16x3_t, idx: uint8x16_t) -> uint8x16_t { + transmute(vqtbl3q( + transmute(t.0), + transmute(t.1), + transmute(t.2), + transmute(idx), + )) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl3_p8(t: poly8x16x3_t, idx: uint8x8_t) -> poly8x8_t { + transmute(vqtbl3( + transmute(t.0), + transmute(t.1), + transmute(t.2), + transmute(idx), + )) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl3q_p8(t: poly8x16x3_t, idx: uint8x16_t) -> poly8x16_t { + transmute(vqtbl3q( + transmute(t.0), + transmute(t.1), + transmute(t.2), + transmute(idx), + )) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx3_s8(a: int8x8_t, t: int8x16x3_t, idx: uint8x8_t) -> int8x8_t { + vqtbx3(a, t.0, t.1, t.2, idx) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx3q_s8(a: int8x16_t, t: int8x16x3_t, idx: uint8x16_t) -> int8x16_t { + vqtbx3q(a, t.0, t.1, t.2, idx) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx3_u8(a: uint8x8_t, t: uint8x16x3_t, idx: uint8x8_t) -> uint8x8_t { + transmute(vqtbx3( + transmute(a), + transmute(t.0), + transmute(t.1), + transmute(t.2), + transmute(idx), + )) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx3q_u8(a: uint8x16_t, t: uint8x16x3_t, idx: uint8x16_t) -> uint8x16_t { + transmute(vqtbx3q( + transmute(a), + transmute(t.0), + transmute(t.1), + transmute(t.2), + transmute(idx), + )) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx3_p8(a: poly8x8_t, t: poly8x16x3_t, idx: uint8x8_t) -> poly8x8_t { + transmute(vqtbx3( + transmute(a), + transmute(t.0), + transmute(t.1), + transmute(t.2), + transmute(idx), + )) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx3q_p8(a: poly8x16_t, t: poly8x16x3_t, idx: uint8x16_t) -> poly8x16_t { + transmute(vqtbx3q( + transmute(a), + transmute(t.0), + transmute(t.1), + transmute(t.2), + transmute(idx), + )) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl4_s8(t: int8x16x4_t, idx: uint8x8_t) -> int8x8_t { + vqtbl4(t.0, t.1, t.2, t.3, idx) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl4q_s8(t: int8x16x4_t, idx: uint8x16_t) -> int8x16_t { + vqtbl4q(t.0, t.1, t.2, t.3, idx) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl4_u8(t: uint8x16x4_t, idx: uint8x8_t) -> uint8x8_t { + transmute(vqtbl4( + transmute(t.0), + transmute(t.1), + transmute(t.2), + transmute(t.3), + transmute(idx), + )) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl4q_u8(t: uint8x16x4_t, idx: uint8x16_t) -> uint8x16_t { + transmute(vqtbl4q( + transmute(t.0), + transmute(t.1), + transmute(t.2), + transmute(t.3), + transmute(idx), + )) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl4_p8(t: poly8x16x4_t, idx: uint8x8_t) -> poly8x8_t { + transmute(vqtbl4( + transmute(t.0), + transmute(t.1), + transmute(t.2), + transmute(t.3), + transmute(idx), + )) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbl))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbl4q_p8(t: poly8x16x4_t, idx: uint8x16_t) -> poly8x16_t { + transmute(vqtbl4q( + transmute(t.0), + transmute(t.1), + transmute(t.2), + transmute(t.3), + transmute(idx), + )) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx4_s8(a: int8x8_t, t: int8x16x4_t, idx: uint8x8_t) -> int8x8_t { + vqtbx4(a, t.0, t.1, t.2, t.3, idx) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx4q_s8(a: int8x16_t, t: int8x16x4_t, idx: uint8x16_t) -> int8x16_t { + vqtbx4q(a, t.0, t.1, t.2, t.3, idx) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx4_u8(a: uint8x8_t, t: uint8x16x4_t, idx: uint8x8_t) -> uint8x8_t { + transmute(vqtbx4( + transmute(a), + transmute(t.0), + transmute(t.1), + transmute(t.2), + transmute(t.3), + transmute(idx), + )) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx4q_u8(a: uint8x16_t, t: uint8x16x4_t, idx: uint8x16_t) -> uint8x16_t { + transmute(vqtbx4q( + transmute(a), + transmute(t.0), + transmute(t.1), + transmute(t.2), + transmute(t.3), + transmute(idx), + )) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx4_p8(a: poly8x8_t, t: poly8x16x4_t, idx: uint8x8_t) -> poly8x8_t { + transmute(vqtbx4( + transmute(a), + transmute(t.0), + transmute(t.1), + transmute(t.2), + transmute(t.3), + transmute(idx), + )) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(tbx))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqtbx4q_p8(a: poly8x16_t, t: poly8x16x4_t, idx: uint8x16_t) -> poly8x16_t { + transmute(vqtbx4q( + transmute(a), + transmute(t.0), + transmute(t.1), + transmute(t.2), + transmute(t.3), + transmute(idx), + )) +} + +/// Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vshld_n_s64(a: i64) -> i64 { + static_assert_imm6!(N); + a << N +} + +/// Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vshld_n_u64(a: u64) -> u64 { + static_assert_imm6!(N); + a << N +} + +/// Signed shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vshrd_n_s64(a: i64) -> i64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + let n: i32 = if N == 64 { 63 } else { N }; + a >> n +} + +/// Unsigned shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vshrd_n_u64(a: u64) -> u64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + let n: i32 = if N == 64 { + return 0; + } else { + N + }; + a >> n +} + +/// Signed shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsrad_n_s64(a: i64, b: i64) -> i64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + a.wrapping_add(vshrd_n_s64::(b)) +} + +/// Unsigned shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsrad_n_u64(a: u64, b: u64) -> u64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + a.wrapping_add(vshrd_n_u64::(b)) +} + +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsli_n_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + static_assert_imm3!(N); + vsli_n_s8_(a, b, N) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsliq_n_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + static_assert_imm3!(N); + vsliq_n_s8_(a, b, N) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsli_n_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + static_assert_imm4!(N); + vsli_n_s16_(a, b, N) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsliq_n_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + static_assert_imm4!(N); + vsliq_n_s16_(a, b, N) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsli_n_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + static_assert!(N: i32 where N >= 0 && N <= 31); + vsli_n_s32_(a, b, N) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsliq_n_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + static_assert!(N: i32 where N >= 0 && N <= 31); + vsliq_n_s32_(a, b, N) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsli_n_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + static_assert!(N: i32 where N >= 0 && N <= 63); + vsli_n_s64_(a, b, N) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsliq_n_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + static_assert!(N: i32 where N >= 0 && N <= 63); + vsliq_n_s64_(a, b, N) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsli_n_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + static_assert_imm3!(N); + transmute(vsli_n_s8_(transmute(a), transmute(b), N)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsliq_n_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + static_assert_imm3!(N); + transmute(vsliq_n_s8_(transmute(a), transmute(b), N)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsli_n_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + static_assert_imm4!(N); + transmute(vsli_n_s16_(transmute(a), transmute(b), N)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsliq_n_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + static_assert_imm4!(N); + transmute(vsliq_n_s16_(transmute(a), transmute(b), N)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsli_n_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + static_assert!(N: i32 where N >= 0 && N <= 31); + transmute(vsli_n_s32_(transmute(a), transmute(b), N)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsliq_n_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + static_assert!(N: i32 where N >= 0 && N <= 31); + transmute(vsliq_n_s32_(transmute(a), transmute(b), N)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsli_n_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + static_assert!(N: i32 where N >= 0 && N <= 63); + transmute(vsli_n_s64_(transmute(a), transmute(b), N)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsliq_n_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + static_assert!(N: i32 where N >= 0 && N <= 63); + transmute(vsliq_n_s64_(transmute(a), transmute(b), N)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsli_n_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + static_assert_imm3!(N); + transmute(vsli_n_s8_(transmute(a), transmute(b), N)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsliq_n_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + static_assert_imm3!(N); + transmute(vsliq_n_s8_(transmute(a), transmute(b), N)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsli_n_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + static_assert_imm4!(N); + transmute(vsli_n_s16_(transmute(a), transmute(b), N)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsliq_n_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + static_assert_imm4!(N); + transmute(vsliq_n_s16_(transmute(a), transmute(b), N)) +} + +/// Shift Left and Insert (immediate) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsli_n_p64(a: poly64x1_t, b: poly64x1_t) -> poly64x1_t { + static_assert!(N: i32 where N >= 0 && N <= 63); + transmute(vsli_n_s64_(transmute(a), transmute(b), N)) +} + +/// Shift Left and Insert (immediate) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsliq_n_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t { + static_assert!(N: i32 where N >= 0 && N <= 63); + transmute(vsliq_n_s64_(transmute(a), transmute(b), N)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsri_n_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + static_assert!(N: i32 where N >= 1 && N <= 8); + vsri_n_s8_(a, b, N) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsriq_n_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + static_assert!(N: i32 where N >= 1 && N <= 8); + vsriq_n_s8_(a, b, N) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsri_n_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + static_assert!(N: i32 where N >= 1 && N <= 16); + vsri_n_s16_(a, b, N) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsriq_n_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + static_assert!(N: i32 where N >= 1 && N <= 16); + vsriq_n_s16_(a, b, N) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsri_n_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + static_assert!(N: i32 where N >= 1 && N <= 32); + vsri_n_s32_(a, b, N) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsriq_n_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + static_assert!(N: i32 where N >= 1 && N <= 32); + vsriq_n_s32_(a, b, N) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsri_n_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + static_assert!(N: i32 where N >= 1 && N <= 64); + vsri_n_s64_(a, b, N) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsriq_n_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + static_assert!(N: i32 where N >= 1 && N <= 64); + vsriq_n_s64_(a, b, N) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsri_n_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + static_assert!(N: i32 where N >= 1 && N <= 8); + transmute(vsri_n_s8_(transmute(a), transmute(b), N)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsriq_n_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + static_assert!(N: i32 where N >= 1 && N <= 8); + transmute(vsriq_n_s8_(transmute(a), transmute(b), N)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsri_n_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + static_assert!(N: i32 where N >= 1 && N <= 16); + transmute(vsri_n_s16_(transmute(a), transmute(b), N)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsriq_n_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + static_assert!(N: i32 where N >= 1 && N <= 16); + transmute(vsriq_n_s16_(transmute(a), transmute(b), N)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsri_n_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + static_assert!(N: i32 where N >= 1 && N <= 32); + transmute(vsri_n_s32_(transmute(a), transmute(b), N)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsriq_n_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + static_assert!(N: i32 where N >= 1 && N <= 32); + transmute(vsriq_n_s32_(transmute(a), transmute(b), N)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsri_n_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + static_assert!(N: i32 where N >= 1 && N <= 64); + transmute(vsri_n_s64_(transmute(a), transmute(b), N)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsriq_n_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + static_assert!(N: i32 where N >= 1 && N <= 64); + transmute(vsriq_n_s64_(transmute(a), transmute(b), N)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsri_n_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + static_assert!(N: i32 where N >= 1 && N <= 8); + transmute(vsri_n_s8_(transmute(a), transmute(b), N)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsriq_n_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + static_assert!(N: i32 where N >= 1 && N <= 8); + transmute(vsriq_n_s8_(transmute(a), transmute(b), N)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsri_n_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + static_assert!(N: i32 where N >= 1 && N <= 16); + transmute(vsri_n_s16_(transmute(a), transmute(b), N)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsriq_n_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + static_assert!(N: i32 where N >= 1 && N <= 16); + transmute(vsriq_n_s16_(transmute(a), transmute(b), N)) +} + +/// Shift Right and Insert (immediate) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsri_n_p64(a: poly64x1_t, b: poly64x1_t) -> poly64x1_t { + static_assert!(N: i32 where N >= 1 && N <= 64); + transmute(vsri_n_s64_(transmute(a), transmute(b), N)) +} + +/// Shift Right and Insert (immediate) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vsriq_n_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t { + static_assert!(N: i32 where N >= 1 && N <= 64); + transmute(vsriq_n_s64_(transmute(a), transmute(b), N)) +} + +/// SM3TT1A +#[inline] +#[target_feature(enable = "neon,sm4")] +#[cfg_attr(test, assert_instr(sm3tt1a, IMM2 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vsm3tt1aq_u32( + a: uint32x4_t, + b: uint32x4_t, + c: uint32x4_t, +) -> uint32x4_t { + static_assert_imm2!(IMM2); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sm3tt1a")] + fn vsm3tt1aq_u32_(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t, imm2: i64) -> uint32x4_t; + } + vsm3tt1aq_u32_(a, b, c, IMM2 as i64) +} + +/// SM3TT1B +#[inline] +#[target_feature(enable = "neon,sm4")] +#[cfg_attr(test, assert_instr(sm3tt1b, IMM2 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vsm3tt1bq_u32( + a: uint32x4_t, + b: uint32x4_t, + c: uint32x4_t, +) -> uint32x4_t { + static_assert_imm2!(IMM2); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sm3tt1b")] + fn vsm3tt1bq_u32_(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t, imm2: i64) -> uint32x4_t; + } + vsm3tt1bq_u32_(a, b, c, IMM2 as i64) +} + +/// SM3TT2A +#[inline] +#[target_feature(enable = "neon,sm4")] +#[cfg_attr(test, assert_instr(sm3tt2a, IMM2 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vsm3tt2aq_u32( + a: uint32x4_t, + b: uint32x4_t, + c: uint32x4_t, +) -> uint32x4_t { + static_assert_imm2!(IMM2); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sm3tt2a")] + fn vsm3tt2aq_u32_(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t, imm2: i64) -> uint32x4_t; + } + vsm3tt2aq_u32_(a, b, c, IMM2 as i64) +} + +/// SM3TT2B +#[inline] +#[target_feature(enable = "neon,sm4")] +#[cfg_attr(test, assert_instr(sm3tt2b, IMM2 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vsm3tt2bq_u32( + a: uint32x4_t, + b: uint32x4_t, + c: uint32x4_t, +) -> uint32x4_t { + static_assert_imm2!(IMM2); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sm3tt2b")] + fn vsm3tt2bq_u32_(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t, imm2: i64) -> uint32x4_t; + } + vsm3tt2bq_u32_(a, b, c, IMM2 as i64) +} + +/// Exclusive OR and rotate +#[inline] +#[target_feature(enable = "neon,sha3")] +#[cfg_attr(test, assert_instr(xar, IMM6 = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vxarq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + static_assert_imm6!(IMM6); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.xar")] + fn vxarq_u64_(a: uint64x2_t, b: uint64x2_t, n: i64) -> uint64x2_t; + } + vxarq_u64_(a, b, IMM6 as i64) +} + +#[cfg(test)] +mod tests { + use crate::core_arch::aarch64::test_support::*; + use crate::core_arch::arm_shared::test_support::*; + use crate::core_arch::{aarch64::neon::*, aarch64::*, simd::*}; + use std::mem::transmute; + use stdarch_test::simd_test; + + #[simd_test(enable = "neon")] + unsafe fn test_vuqadd_s8() { + let a = i8x8::new(i8::MIN, -3, -2, -1, 0, 1, 2, i8::MAX); + let b = u8x8::new(u8::MAX, 1, 2, 3, 4, 5, 6, 7); + let e = i8x8::new(i8::MAX, -2, 0, 2, 4, 6, 8, i8::MAX); + let r: i8x8 = transmute(vuqadd_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vuqaddq_s8() { + let a = i8x16::new( + i8::MIN, + -7, + -6, + -5, + -4, + -3, + -2, + -1, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + i8::MAX, + ); + let b = u8x16::new(u8::MAX, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e = i8x16::new( + i8::MAX, + -6, + -4, + -2, + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 18, + 20, + i8::MAX, + ); + let r: i8x16 = transmute(vuqaddq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vuqadd_s16() { + let a = i16x4::new(i16::MIN, -1, 0, i16::MAX); + let b = u16x4::new(u16::MAX, 1, 2, 3); + let e = i16x4::new(i16::MAX, 0, 2, i16::MAX); + let r: i16x4 = transmute(vuqadd_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vuqaddq_s16() { + let a = i16x8::new(i16::MIN, -3, -2, -1, 0, 1, 2, i16::MAX); + let b = u16x8::new(u16::MAX, 1, 2, 3, 4, 5, 6, 7); + let e = i16x8::new(i16::MAX, -2, 0, 2, 4, 6, 8, i16::MAX); + let r: i16x8 = transmute(vuqaddq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vuqadd_s32() { + let a = i32x2::new(i32::MIN, i32::MAX); + let b = u32x2::new(u32::MAX, 1); + let e = i32x2::new(i32::MAX, i32::MAX); + let r: i32x2 = transmute(vuqadd_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vuqaddq_s32() { + let a = i32x4::new(i32::MIN, -1, 0, i32::MAX); + let b = u32x4::new(u32::MAX, 1, 2, 3); + let e = i32x4::new(i32::MAX, 0, 2, i32::MAX); + let r: i32x4 = transmute(vuqaddq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vuqadd_s64() { + let a = i64x1::new(i64::MIN); + let b = u64x1::new(u64::MAX); + let e = i64x1::new(i64::MAX); + let r: i64x1 = transmute(vuqadd_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vuqaddq_s64() { + let a = i64x2::new(i64::MIN, i64::MAX); + let b = u64x2::new(u64::MAX, 1); + let e = i64x2::new(i64::MAX, i64::MAX); + let r: i64x2 = transmute(vuqaddq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsqadd_u8() { + let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, u8::MAX); + let b = i8x8::new(i8::MIN, -3, -2, -1, 0, 1, 2, 3); + let e = u8x8::new(0, 0, 0, 2, 4, 6, 8, u8::MAX); + let r: u8x8 = transmute(vsqadd_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vsqaddq_u8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, u8::MAX); + let b = i8x16::new(i8::MIN, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7); + let e = u8x16::new(0, 0, 0, 0, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, u8::MAX); + let r: u8x16 = transmute(vsqaddq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vsqadd_u16() { + let a = u16x4::new(0, 1, 2, u16::MAX); + let b = i16x4::new(i16::MIN, -1, 0, 1); + let e = u16x4::new(0, 0, 2, u16::MAX); + let r: u16x4 = transmute(vsqadd_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vsqaddq_u16() { + let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, u16::MAX); + let b = i16x8::new(i16::MIN, -3, -2, -1, 0, 1, 2, 3); + let e = u16x8::new(0, 0, 0, 2, 4, 6, 8, u16::MAX); + let r: u16x8 = transmute(vsqaddq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vsqadd_u32() { + let a = u32x2::new(0, u32::MAX); + let b = i32x2::new(i32::MIN, 1); + let e = u32x2::new(0, u32::MAX); + let r: u32x2 = transmute(vsqadd_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vsqaddq_u32() { + let a = u32x4::new(0, 1, 2, u32::MAX); + let b = i32x4::new(i32::MIN, -1, 0, 1); + let e = u32x4::new(0, 0, 2, u32::MAX); + let r: u32x4 = transmute(vsqaddq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vsqadd_u64() { + let a = u64x1::new(0); + let b = i64x1::new(i64::MIN); + let e = u64x1::new(0); + let r: u64x1 = transmute(vsqadd_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vsqaddq_u64() { + let a = u64x2::new(0, u64::MAX); + let b = i64x2::new(i64::MIN, 1); + let e = u64x2::new(0, u64::MAX); + let r: u64x2 = transmute(vsqaddq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpaddq_s16() { + let a = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b = i16x8::new(0, -1, -2, -3, -4, -5, -6, -7); + let r: i16x8 = transmute(vpaddq_s16(transmute(a), transmute(b))); + let e = i16x8::new(3, 7, 11, 15, -1, -5, -9, -13); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vpaddq_s32() { + let a = i32x4::new(1, 2, 3, 4); + let b = i32x4::new(0, -1, -2, -3); + let r: i32x4 = transmute(vpaddq_s32(transmute(a), transmute(b))); + let e = i32x4::new(3, 7, -1, -5); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vpaddq_s64() { + let a = i64x2::new(1, 2); + let b = i64x2::new(0, -1); + let r: i64x2 = transmute(vpaddq_s64(transmute(a), transmute(b))); + let e = i64x2::new(3, -1); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vpaddq_s8() { + let a = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b = i8x16::new( + 0, -1, -2, -3, -4, -5, -6, -7, -8, -8, -10, -11, -12, -13, -14, -15, + ); + let r: i8x16 = transmute(vpaddq_s8(transmute(a), transmute(b))); + let e = i8x16::new( + 3, 7, 11, 15, 19, 23, 27, 31, -1, -5, -9, -13, -16, -21, -25, -29, + ); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vpaddq_u16() { + let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b = u16x8::new(17, 18, 19, 20, 20, 21, 22, 23); + let r: u16x8 = transmute(vpaddq_u16(transmute(a), transmute(b))); + let e = u16x8::new(1, 5, 9, 13, 35, 39, 41, 45); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vpaddq_u32() { + let a = u32x4::new(0, 1, 2, 3); + let b = u32x4::new(17, 18, 19, 20); + let r: u32x4 = transmute(vpaddq_u32(transmute(a), transmute(b))); + let e = u32x4::new(1, 5, 35, 39); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vpaddq_u64() { + let a = u64x2::new(0, 1); + let b = u64x2::new(17, 18); + let r: u64x2 = transmute(vpaddq_u64(transmute(a), transmute(b))); + let e = u64x2::new(1, 35); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vpaddq_u8() { + let a = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = i8x16::new( + 17, 18, 19, 20, 20, 21, 22, 23, 24, 25, 26, 27, 29, 29, 30, 31, + ); + let r = i8x16(1, 5, 9, 13, 17, 21, 25, 29, 35, 39, 41, 45, 49, 53, 58, 61); + let e: i8x16 = transmute(vpaddq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vpaddd_s64() { + let a = i64x2::new(2, -3); + let r: i64 = transmute(vpaddd_s64(transmute(a))); + let e = -1_i64; + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vpaddd_u64() { + let a = i64x2::new(2, 3); + let r: u64 = transmute(vpaddd_u64(transmute(a))); + let e = 5_u64; + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vadd_f64() { + let a = 1.; + let b = 8.; + let e = 9.; + let r: f64 = transmute(vadd_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddq_f64() { + let a = f64x2::new(1., 2.); + let b = f64x2::new(8., 7.); + let e = f64x2::new(9., 9.); + let r: f64x2 = transmute(vaddq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vadd_s64() { + let a = 1_i64; + let b = 8_i64; + let e = 9_i64; + let r: i64 = transmute(vadd_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vadd_u64() { + let a = 1_u64; + let b = 8_u64; + let e = 9_u64; + let r: u64 = transmute(vadd_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddd_s64() { + let a = 1_i64; + let b = 8_i64; + let e = 9_i64; + let r: i64 = transmute(vaddd_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddd_u64() { + let a = 1_u64; + let b = 8_u64; + let e = 9_u64; + let r: u64 = transmute(vaddd_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxv_s8() { + let r = vmaxv_s8(transmute(i8x8::new(1, 2, 3, 4, -8, 6, 7, 5))); + assert_eq!(r, 7_i8); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxvq_s8() { + #[rustfmt::skip] + let r = vmaxvq_s8(transmute(i8x16::new( + 1, 2, 3, 4, + -16, 6, 7, 5, + 8, 1, 1, 1, + 1, 1, 1, 1, + ))); + assert_eq!(r, 8_i8); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxv_s16() { + let r = vmaxv_s16(transmute(i16x4::new(1, 2, -4, 3))); + assert_eq!(r, 3_i16); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxvq_s16() { + let r = vmaxvq_s16(transmute(i16x8::new(1, 2, 7, 4, -16, 6, 7, 5))); + assert_eq!(r, 7_i16); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxv_s32() { + let r = vmaxv_s32(transmute(i32x2::new(1, -4))); + assert_eq!(r, 1_i32); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxvq_s32() { + let r = vmaxvq_s32(transmute(i32x4::new(1, 2, -32, 4))); + assert_eq!(r, 4_i32); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxv_u8() { + let r = vmaxv_u8(transmute(u8x8::new(1, 2, 3, 4, 8, 6, 7, 5))); + assert_eq!(r, 8_u8); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxvq_u8() { + #[rustfmt::skip] + let r = vmaxvq_u8(transmute(u8x16::new( + 1, 2, 3, 4, + 16, 6, 7, 5, + 8, 1, 1, 1, + 1, 1, 1, 1, + ))); + assert_eq!(r, 16_u8); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxv_u16() { + let r = vmaxv_u16(transmute(u16x4::new(1, 2, 4, 3))); + assert_eq!(r, 4_u16); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxvq_u16() { + let r = vmaxvq_u16(transmute(u16x8::new(1, 2, 7, 4, 16, 6, 7, 5))); + assert_eq!(r, 16_u16); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxv_u32() { + let r = vmaxv_u32(transmute(u32x2::new(1, 4))); + assert_eq!(r, 4_u32); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxvq_u32() { + let r = vmaxvq_u32(transmute(u32x4::new(1, 2, 32, 4))); + assert_eq!(r, 32_u32); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxv_f32() { + let r = vmaxv_f32(transmute(f32x2::new(1., 4.))); + assert_eq!(r, 4_f32); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxvq_f32() { + let r = vmaxvq_f32(transmute(f32x4::new(1., 2., 32., 4.))); + assert_eq!(r, 32_f32); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxvq_f64() { + let r = vmaxvq_f64(transmute(f64x2::new(1., 4.))); + assert_eq!(r, 4_f64); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminv_s8() { + let r = vminv_s8(transmute(i8x8::new(1, 2, 3, 4, -8, 6, 7, 5))); + assert_eq!(r, -8_i8); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminvq_s8() { + #[rustfmt::skip] + let r = vminvq_s8(transmute(i8x16::new( + 1, 2, 3, 4, + -16, 6, 7, 5, + 8, 1, 1, 1, + 1, 1, 1, 1, + ))); + assert_eq!(r, -16_i8); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminv_s16() { + let r = vminv_s16(transmute(i16x4::new(1, 2, -4, 3))); + assert_eq!(r, -4_i16); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminvq_s16() { + let r = vminvq_s16(transmute(i16x8::new(1, 2, 7, 4, -16, 6, 7, 5))); + assert_eq!(r, -16_i16); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminv_s32() { + let r = vminv_s32(transmute(i32x2::new(1, -4))); + assert_eq!(r, -4_i32); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminvq_s32() { + let r = vminvq_s32(transmute(i32x4::new(1, 2, -32, 4))); + assert_eq!(r, -32_i32); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminv_u8() { + let r = vminv_u8(transmute(u8x8::new(1, 2, 3, 4, 8, 6, 7, 5))); + assert_eq!(r, 1_u8); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminvq_u8() { + #[rustfmt::skip] + let r = vminvq_u8(transmute(u8x16::new( + 1, 2, 3, 4, + 16, 6, 7, 5, + 8, 1, 1, 1, + 1, 1, 1, 1, + ))); + assert_eq!(r, 1_u8); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminv_u16() { + let r = vminv_u16(transmute(u16x4::new(1, 2, 4, 3))); + assert_eq!(r, 1_u16); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminvq_u16() { + let r = vminvq_u16(transmute(u16x8::new(1, 2, 7, 4, 16, 6, 7, 5))); + assert_eq!(r, 1_u16); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminv_u32() { + let r = vminv_u32(transmute(u32x2::new(1, 4))); + assert_eq!(r, 1_u32); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminvq_u32() { + let r = vminvq_u32(transmute(u32x4::new(1, 2, 32, 4))); + assert_eq!(r, 1_u32); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminv_f32() { + let r = vminv_f32(transmute(f32x2::new(1., 4.))); + assert_eq!(r, 1_f32); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminvq_f32() { + let r = vminvq_f32(transmute(f32x4::new(1., 2., 32., 4.))); + assert_eq!(r, 1_f32); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminvq_f64() { + let r = vminvq_f64(transmute(f64x2::new(1., 4.))); + assert_eq!(r, 1_f64); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpminq_s8() { + #[cfg_attr(rustfmt, skip)] + let a = i8x16::new(1, -2, 3, -4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); + #[cfg_attr(rustfmt, skip)] + let b = i8x16::new(0, 3, 2, 5, 4, 7, 6, 9, 0, 3, 2, 5, 4, 7, 6, 9); + #[cfg_attr(rustfmt, skip)] + let e = i8x16::new(-2, -4, 5, 7, 1, 3, 5, 7, 0, 2, 4, 6, 0, 2, 4, 6); + let r: i8x16 = transmute(vpminq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpminq_s16() { + let a = i16x8::new(1, -2, 3, 4, 5, 6, 7, 8); + let b = i16x8::new(0, 3, 2, 5, 4, 7, 6, 9); + let e = i16x8::new(-2, 3, 5, 7, 0, 2, 4, 6); + let r: i16x8 = transmute(vpminq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpminq_s32() { + let a = i32x4::new(1, -2, 3, 4); + let b = i32x4::new(0, 3, 2, 5); + let e = i32x4::new(-2, 3, 0, 2); + let r: i32x4 = transmute(vpminq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpminq_u8() { + #[cfg_attr(rustfmt, skip)] + let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); + #[cfg_attr(rustfmt, skip)] + let b = u8x16::new(0, 3, 2, 5, 4, 7, 6, 9, 0, 3, 2, 5, 4, 7, 6, 9); + #[cfg_attr(rustfmt, skip)] + let e = u8x16::new(1, 3, 5, 7, 1, 3, 5, 7, 0, 2, 4, 6, 0, 2, 4, 6); + let r: u8x16 = transmute(vpminq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpminq_u16() { + let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b = u16x8::new(0, 3, 2, 5, 4, 7, 6, 9); + let e = u16x8::new(1, 3, 5, 7, 0, 2, 4, 6); + let r: u16x8 = transmute(vpminq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpminq_u32() { + let a = u32x4::new(1, 2, 3, 4); + let b = u32x4::new(0, 3, 2, 5); + let e = u32x4::new(1, 3, 0, 2); + let r: u32x4 = transmute(vpminq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmin_f32() { + let a = f32x4::new(1., -2., 3., 4.); + let b = f32x4::new(0., 3., 2., 5.); + let e = f32x4::new(-2., 3., 0., 2.); + let r: f32x4 = transmute(vpminq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmin_f64() { + let a = f64x2::new(1., -2.); + let b = f64x2::new(0., 3.); + let e = f64x2::new(-2., 0.); + let r: f64x2 = transmute(vpminq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmaxq_s8() { + #[cfg_attr(rustfmt, skip)] + let a = i8x16::new(1, -2, 3, -4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); + #[cfg_attr(rustfmt, skip)] + let b = i8x16::new(0, 3, 2, 5, 4, 7, 6, 9, 0, 3, 2, 5, 4, 7, 6, 9); + #[cfg_attr(rustfmt, skip)] + let e = i8x16::new(1, 3, 6, 8, 2, 4, 6, 8, 3, 5, 7, 9, 3, 5, 7, 9); + let r: i8x16 = transmute(vpmaxq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmaxq_s16() { + let a = i16x8::new(1, -2, 3, 4, 5, 6, 7, 8); + let b = i16x8::new(0, 3, 2, 5, 4, 7, 6, 9); + let e = i16x8::new(1, 4, 6, 8, 3, 5, 7, 9); + let r: i16x8 = transmute(vpmaxq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmaxq_s32() { + let a = i32x4::new(1, -2, 3, 4); + let b = i32x4::new(0, 3, 2, 5); + let e = i32x4::new(1, 4, 3, 5); + let r: i32x4 = transmute(vpmaxq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmaxq_u8() { + #[cfg_attr(rustfmt, skip)] + let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); + #[cfg_attr(rustfmt, skip)] + let b = u8x16::new(0, 3, 2, 5, 4, 7, 6, 9, 0, 3, 2, 5, 4, 7, 6, 9); + #[cfg_attr(rustfmt, skip)] + let e = u8x16::new(2, 4, 6, 8, 2, 4, 6, 8, 3, 5, 7, 9, 3, 5, 7, 9); + let r: u8x16 = transmute(vpmaxq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmaxq_u16() { + let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b = u16x8::new(0, 3, 2, 5, 4, 7, 6, 9); + let e = u16x8::new(2, 4, 6, 8, 3, 5, 7, 9); + let r: u16x8 = transmute(vpmaxq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmaxq_u32() { + let a = u32x4::new(1, 2, 3, 4); + let b = u32x4::new(0, 3, 2, 5); + let e = u32x4::new(2, 4, 3, 5); + let r: u32x4 = transmute(vpmaxq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmax_f32() { + let a = f32x4::new(1., -2., 3., 4.); + let b = f32x4::new(0., 3., 2., 5.); + let e = f32x4::new(1., 4., 3., 5.); + let r: f32x4 = transmute(vpmaxq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmax_f64() { + let a = f64x2::new(1., -2.); + let b = f64x2::new(0., 3.); + let e = f64x2::new(1., 3.); + let r: f64x2 = transmute(vpmaxq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_p64() { + let a: i64x1 = i64x1::new(0); + let b: i64x1 = i64x1::new(1); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vext_p64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_f64() { + let a: f64x1 = f64x1::new(0.); + let b: f64x1 = f64x1::new(1.); + let e: f64x1 = f64x1::new(0.); + let r: f64x1 = transmute(vext_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshld_n_s64() { + let a: i64 = 1; + let e: i64 = 4; + let r: i64 = vshld_n_s64::<2>(a); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshld_n_u64() { + let a: u64 = 1; + let e: u64 = 4; + let r: u64 = vshld_n_u64::<2>(a); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrd_n_s64() { + let a: i64 = 4; + let e: i64 = 1; + let r: i64 = vshrd_n_s64::<2>(a); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrd_n_u64() { + let a: u64 = 4; + let e: u64 = 1; + let r: u64 = vshrd_n_u64::<2>(a); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsrad_n_s64() { + let a: i64 = 1; + let b: i64 = 4; + let e: i64 = 2; + let r: i64 = vsrad_n_s64::<2>(a, b); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsrad_n_u64() { + let a: u64 = 1; + let b: u64 = 4; + let e: u64 = 2; + let r: u64 = vsrad_n_u64::<2>(a, b); + assert_eq!(r, e); + } + + macro_rules! test_vcombine { + ($test_id:ident => $fn_id:ident ([$($a:expr),*], [$($b:expr),*])) => { + #[allow(unused_assignments)] + #[simd_test(enable = "neon")] + unsafe fn $test_id() { + let a = [$($a),*]; + let b = [$($b),*]; + let e = [$($a),* $(, $b)*]; + let c = $fn_id(transmute(a), transmute(b)); + let mut d = e; + d = transmute(c); + assert_eq!(d, e); + } + } + } + + test_vcombine!(test_vcombine_s8 => vcombine_s8([3_i8, -4, 5, -6, 7, 8, 9, 10], [13_i8, -14, 15, -16, 17, 18, 19, 110])); + test_vcombine!(test_vcombine_u8 => vcombine_u8([3_u8, 4, 5, 6, 7, 8, 9, 10], [13_u8, 14, 15, 16, 17, 18, 19, 110])); + test_vcombine!(test_vcombine_p8 => vcombine_p8([3_u8, 4, 5, 6, 7, 8, 9, 10], [13_u8, 14, 15, 16, 17, 18, 19, 110])); + + test_vcombine!(test_vcombine_s16 => vcombine_s16([3_i16, -4, 5, -6], [13_i16, -14, 15, -16])); + test_vcombine!(test_vcombine_u16 => vcombine_u16([3_u16, 4, 5, 6], [13_u16, 14, 15, 16])); + test_vcombine!(test_vcombine_p16 => vcombine_p16([3_u16, 4, 5, 6], [13_u16, 14, 15, 16])); + // FIXME: 16-bit floats + // test_vcombine!(test_vcombine_f16 => vcombine_f16([3_f16, 4., 5., 6.], + // [13_f16, 14., 15., 16.])); + + test_vcombine!(test_vcombine_s32 => vcombine_s32([3_i32, -4], [13_i32, -14])); + test_vcombine!(test_vcombine_u32 => vcombine_u32([3_u32, 4], [13_u32, 14])); + // note: poly32x4 does not exist, and neither does vcombine_p32 + test_vcombine!(test_vcombine_f32 => vcombine_f32([3_f32, -4.], [13_f32, -14.])); + + test_vcombine!(test_vcombine_s64 => vcombine_s64([-3_i64], [13_i64])); + test_vcombine!(test_vcombine_u64 => vcombine_u64([3_u64], [13_u64])); + test_vcombine!(test_vcombine_p64 => vcombine_p64([3_u64], [13_u64])); + test_vcombine!(test_vcombine_f64 => vcombine_f64([-3_f64], [13_f64])); + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_f64() { + let a: f64 = 3.3; + let e = f64x1::new(3.3); + let r: f64x1 = transmute(vdup_n_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_p64() { + let a: u64 = 3; + let e = u64x1::new(3); + let r: u64x1 = transmute(vdup_n_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_f64() { + let a: f64 = 3.3; + let e = f64x2::new(3.3, 3.3); + let r: f64x2 = transmute(vdupq_n_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_p64() { + let a: u64 = 3; + let e = u64x2::new(3, 3); + let r: u64x2 = transmute(vdupq_n_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_p64() { + let a: u64 = 3; + let e = u64x1::new(3); + let r: u64x1 = transmute(vmov_n_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_f64() { + let a: f64 = 3.3; + let e = f64x1::new(3.3); + let r: f64x1 = transmute(vmov_n_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovq_n_p64() { + let a: u64 = 3; + let e = u64x2::new(3, 3); + let r: u64x2 = transmute(vmovq_n_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovq_n_f64() { + let a: f64 = 3.3; + let e = f64x2::new(3.3, 3.3); + let r: f64x2 = transmute(vmovq_n_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_f64() { + let a = f64x2::new(1.0, 2.0); + let e = f64x1::new(2.0); + let r: f64x1 = transmute(vget_high_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_p64() { + let a = u64x2::new(1, 2); + let e = u64x1::new(2); + let r: u64x1 = transmute(vget_high_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_f64() { + let a = f64x2::new(1.0, 2.0); + let e = f64x1::new(1.0); + let r: f64x1 = transmute(vget_low_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_p64() { + let a = u64x2::new(1, 2); + let e = u64x1::new(1); + let r: u64x1 = transmute(vget_low_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_f64() { + let v = f64x1::new(1.0); + let r = vget_lane_f64::<0>(transmute(v)); + assert_eq!(r, 1.0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_f64() { + let v = f64x2::new(0.0, 1.0); + let r = vgetq_lane_f64::<1>(transmute(v)); + assert_eq!(r, 1.0); + let r = vgetq_lane_f64::<0>(transmute(v)); + assert_eq!(r, 0.0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_s64() { + let a: i64x1 = i64x1::new(1); + let b: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); + let r: i64x1 = transmute(vcopy_lane_s64::<0, 0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_u64() { + let a: u64x1 = u64x1::new(1); + let b: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcopy_lane_u64::<0, 0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_p64() { + let a: i64x1 = i64x1::new(1); + let b: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); + let r: i64x1 = transmute(vcopy_lane_p64::<0, 0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_f64() { + let a: f64 = 1.; + let b: f64 = 0.; + let e: f64 = 0.; + let r: f64 = transmute(vcopy_lane_f64::<0, 0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_s64() { + let a: i64x1 = i64x1::new(1); + let b: i64x2 = i64x2::new(0, 0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); + let r: i64x1 = transmute(vcopy_laneq_s64::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_u64() { + let a: u64x1 = u64x1::new(1); + let b: u64x2 = u64x2::new(0, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcopy_laneq_u64::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_p64() { + let a: i64x1 = i64x1::new(1); + let b: i64x2 = i64x2::new(0, 0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); + let r: i64x1 = transmute(vcopy_laneq_p64::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_f64() { + let a: f64 = 1.; + let b: f64x2 = f64x2::new(0., 0.5); + let e: f64 = 0.5; + let r: f64 = transmute(vcopy_laneq_f64::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_u64() { + test_cmp_u64( + |i, j| vceq_u64(i, j), + |a: u64, b: u64| -> u64 { + if a == b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_u64() { + testq_cmp_u64( + |i, j| vceqq_u64(i, j), + |a: u64, b: u64| -> u64 { + if a == b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_s64() { + test_cmp_s64( + |i, j| vceq_s64(i, j), + |a: i64, b: i64| -> u64 { + if a == b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_s64() { + testq_cmp_s64( + |i, j| vceqq_s64(i, j), + |a: i64, b: i64| -> u64 { + if a == b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_p64() { + test_cmp_p64( + |i, j| vceq_p64(i, j), + |a: u64, b: u64| -> u64 { + if a == b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_p64() { + testq_cmp_p64( + |i, j| vceqq_p64(i, j), + |a: u64, b: u64| -> u64 { + if a == b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_f64() { + test_cmp_f64( + |i, j| vceq_f64(i, j), + |a: f64, b: f64| -> u64 { + if a == b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_f64() { + testq_cmp_f64( + |i, j| vceqq_f64(i, j), + |a: f64, b: f64| -> u64 { + if a == b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_s64() { + test_cmp_s64( + |i, j| vcgt_s64(i, j), + |a: i64, b: i64| -> u64 { + if a > b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_s64() { + testq_cmp_s64( + |i, j| vcgtq_s64(i, j), + |a: i64, b: i64| -> u64 { + if a > b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_u64() { + test_cmp_u64( + |i, j| vcgt_u64(i, j), + |a: u64, b: u64| -> u64 { + if a > b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_u64() { + testq_cmp_u64( + |i, j| vcgtq_u64(i, j), + |a: u64, b: u64| -> u64 { + if a > b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_f64() { + test_cmp_f64( + |i, j| vcgt_f64(i, j), + |a: f64, b: f64| -> u64 { + if a > b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_f64() { + testq_cmp_f64( + |i, j| vcgtq_f64(i, j), + |a: f64, b: f64| -> u64 { + if a > b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_s64() { + test_cmp_s64( + |i, j| vclt_s64(i, j), + |a: i64, b: i64| -> u64 { + if a < b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_s64() { + testq_cmp_s64( + |i, j| vcltq_s64(i, j), + |a: i64, b: i64| -> u64 { + if a < b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_u64() { + test_cmp_u64( + |i, j| vclt_u64(i, j), + |a: u64, b: u64| -> u64 { + if a < b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_u64() { + testq_cmp_u64( + |i, j| vcltq_u64(i, j), + |a: u64, b: u64| -> u64 { + if a < b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vltq_f64() { + test_cmp_f64( + |i, j| vclt_f64(i, j), + |a: f64, b: f64| -> u64 { + if a < b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_f64() { + testq_cmp_f64( + |i, j| vcltq_f64(i, j), + |a: f64, b: f64| -> u64 { + if a < b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_s64() { + test_cmp_s64( + |i, j| vcle_s64(i, j), + |a: i64, b: i64| -> u64 { + if a <= b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_s64() { + testq_cmp_s64( + |i, j| vcleq_s64(i, j), + |a: i64, b: i64| -> u64 { + if a <= b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_u64() { + test_cmp_u64( + |i, j| vcle_u64(i, j), + |a: u64, b: u64| -> u64 { + if a <= b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_u64() { + testq_cmp_u64( + |i, j| vcleq_u64(i, j), + |a: u64, b: u64| -> u64 { + if a <= b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vleq_f64() { + test_cmp_f64( + |i, j| vcle_f64(i, j), + |a: f64, b: f64| -> u64 { + if a <= b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_f64() { + testq_cmp_f64( + |i, j| vcleq_f64(i, j), + |a: f64, b: f64| -> u64 { + if a <= b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_s64() { + test_cmp_s64( + |i, j| vcge_s64(i, j), + |a: i64, b: i64| -> u64 { + if a >= b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_s64() { + testq_cmp_s64( + |i, j| vcgeq_s64(i, j), + |a: i64, b: i64| -> u64 { + if a >= b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_u64() { + test_cmp_u64( + |i, j| vcge_u64(i, j), + |a: u64, b: u64| -> u64 { + if a >= b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_u64() { + testq_cmp_u64( + |i, j| vcgeq_u64(i, j), + |a: u64, b: u64| -> u64 { + if a >= b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgeq_f64() { + test_cmp_f64( + |i, j| vcge_f64(i, j), + |a: f64, b: f64| -> u64 { + if a >= b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_f64() { + testq_cmp_f64( + |i, j| vcgeq_f64(i, j), + |a: f64, b: f64| -> u64 { + if a >= b { + 0xFFFFFFFFFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_f64() { + test_ari_f64(|i, j| vmul_f64(i, j), |a: f64, b: f64| -> f64 { a * b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_f64() { + testq_ari_f64(|i, j| vmulq_f64(i, j), |a: f64, b: f64| -> f64 { a * b }); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_f64() { + test_ari_f64(|i, j| vsub_f64(i, j), |a: f64, b: f64| -> f64 { a - b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_f64() { + testq_ari_f64(|i, j| vsubq_f64(i, j), |a: f64, b: f64| -> f64 { a - b }); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabsd_s64() { + assert_eq!(vabsd_s64(-1), 1); + assert_eq!(vabsd_s64(0), 0); + assert_eq!(vabsd_s64(1), 1); + assert_eq!(vabsd_s64(i64::MIN), i64::MIN); + assert_eq!(vabsd_s64(i64::MIN + 1), i64::MAX); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabs_s64() { + let a = i64x1::new(i64::MIN); + let r: i64x1 = transmute(vabs_s64(transmute(a))); + let e = i64x1::new(i64::MIN); + assert_eq!(r, e); + let a = i64x1::new(i64::MIN + 1); + let r: i64x1 = transmute(vabs_s64(transmute(a))); + let e = i64x1::new(i64::MAX); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabsq_s64() { + let a = i64x2::new(i64::MIN, i64::MIN + 1); + let r: i64x2 = transmute(vabsq_s64(transmute(a))); + let e = i64x2::new(i64::MIN, i64::MAX); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_f64() { + let a = u64x1::new(0x8000000000000000); + let b = f64x1::new(-1.23f64); + let c = f64x1::new(2.34f64); + let e = f64x1::new(-2.34f64); + let r: f64x1 = transmute(vbsl_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_p64() { + let a = u64x1::new(1); + let b = u64x1::new(u64::MAX); + let c = u64x1::new(u64::MIN); + let e = u64x1::new(1); + let r: u64x1 = transmute(vbsl_p64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbslq_f64() { + let a = u64x2::new(1, 0x8000000000000000); + let b = f64x2::new(f64::MAX, -1.23f64); + let c = f64x2::new(f64::MIN, 2.34f64); + let e = f64x2::new(f64::MIN, -2.34f64); + let r: f64x2 = transmute(vbslq_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbslq_p64() { + let a = u64x2::new(u64::MAX, 1); + let b = u64x2::new(u64::MAX, u64::MAX); + let c = u64x2::new(u64::MIN, u64::MIN); + let e = u64x2::new(u64::MAX, 1); + let r: u64x2 = transmute(vbslq_p64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddv_s16() { + let a = i16x4::new(1, 2, 3, -4); + let r: i16 = transmute(vaddv_s16(transmute(a))); + let e = 2_i16; + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddv_u16() { + let a = u16x4::new(1, 2, 3, 4); + let r: u16 = transmute(vaddv_u16(transmute(a))); + let e = 10_u16; + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddv_s32() { + let a = i32x2::new(1, -2); + let r: i32 = transmute(vaddv_s32(transmute(a))); + let e = -1_i32; + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddv_u32() { + let a = u32x2::new(1, 2); + let r: u32 = transmute(vaddv_u32(transmute(a))); + let e = 3_u32; + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddv_s8() { + let a = i8x8::new(1, 2, 3, 4, 5, 6, 7, -8); + let r: i8 = transmute(vaddv_s8(transmute(a))); + let e = 20_i8; + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddv_u8() { + let a = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u8 = transmute(vaddv_u8(transmute(a))); + let e = 36_u8; + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddvq_s16() { + let a = i16x8::new(1, 2, 3, 4, 5, 6, 7, -8); + let r: i16 = transmute(vaddvq_s16(transmute(a))); + let e = 20_i16; + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddvq_u16() { + let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u16 = transmute(vaddvq_u16(transmute(a))); + let e = 36_u16; + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddvq_s32() { + let a = i32x4::new(1, 2, 3, -4); + let r: i32 = transmute(vaddvq_s32(transmute(a))); + let e = 2_i32; + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddvq_u32() { + let a = u32x4::new(1, 2, 3, 4); + let r: u32 = transmute(vaddvq_u32(transmute(a))); + let e = 10_u32; + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddvq_s8() { + let a = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, -16); + let r: i8 = transmute(vaddvq_s8(transmute(a))); + let e = 104_i8; + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddvq_u8() { + let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: u8 = transmute(vaddvq_u8(transmute(a))); + let e = 136_u8; + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddvq_s64() { + let a = i64x2::new(1, -2); + let r: i64 = transmute(vaddvq_s64(transmute(a))); + let e = -1_i64; + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddvq_u64() { + let a = u64x2::new(1, 2); + let r: u64 = transmute(vaddvq_u64(transmute(a))); + let e = 3_u64; + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddlv_s8() { + let a = i8x8::new(1, 2, 3, 4, 5, 6, 7, -8); + let r: i16 = vaddlv_s8(transmute(a)); + let e = 20_i16; + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddlv_u8() { + let a = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u16 = vaddlv_u8(transmute(a)); + let e = 36_u16; + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddlvq_s8() { + let a = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, -16); + let r: i16 = vaddlvq_s8(transmute(a)); + let e = 104_i16; + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddlvq_u8() { + let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: u16 = vaddlvq_u8(transmute(a)); + let e = 136_u16; + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_f64() { + let a: [f64; 2] = [0., 1.]; + let e = f64x1::new(1.); + let r: f64x1 = transmute(vld1_f64(a[1..].as_ptr())); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_f64() { + let a: [f64; 3] = [0., 1., 2.]; + let e = f64x2::new(1., 2.); + let r: f64x2 = transmute(vld1q_f64(a[1..].as_ptr())); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_f64() { + let a: [f64; 2] = [1., 42.]; + let e = f64x1::new(42.); + let r: f64x1 = transmute(vld1_dup_f64(a[1..].as_ptr())); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_f64() { + let elem: f64 = 42.; + let e = f64x2::new(42., 42.); + let r: f64x2 = transmute(vld1q_dup_f64(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_f64() { + let a = f64x1::new(0.); + let elem: f64 = 42.; + let e = f64x1::new(42.); + let r: f64x1 = transmute(vld1_lane_f64::<0>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_f64() { + let a = f64x2::new(0., 1.); + let elem: f64 = 42.; + let e = f64x2::new(0., 42.); + let r: f64x2 = transmute(vld1q_lane_f64::<1>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_f64() { + let mut vals = [0_f64; 2]; + let a = f64x1::new(1.); + + vst1_f64(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0.); + assert_eq!(vals[1], 1.); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_f64() { + let mut vals = [0_f64; 3]; + let a = f64x2::new(1., 2.); + + vst1q_f64(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0.); + assert_eq!(vals[1], 1.); + assert_eq!(vals[2], 2.); + } + + #[simd_test(enable = "neon,sm4")] + unsafe fn test_vsm3tt1aq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let c: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(2, 1536, 4, 16395); + let r: u32x4 = transmute(vsm3tt1aq_u32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sm4")] + unsafe fn test_vsm3tt1bq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let c: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(2, 1536, 4, 16392); + let r: u32x4 = transmute(vsm3tt1bq_u32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sm4")] + unsafe fn test_vsm3tt2aq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let c: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(2, 1572864, 4, 1447435); + let r: u32x4 = transmute(vsm3tt2aq_u32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sm4")] + unsafe fn test_vsm3tt2bq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let c: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(2, 1572864, 4, 1052680); + let r: u32x4 = transmute(vsm3tt2bq_u32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,sha3")] + unsafe fn test_vxarq_u64() { + let a: u64x2 = u64x2::new(1, 2); + let b: u64x2 = u64x2::new(3, 4); + let e: u64x2 = u64x2::new(2, 6); + let r: u64x2 = transmute(vxarq_u64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } +} + +#[cfg(test)] +#[cfg(target_endian = "little")] +#[path = "../../arm_shared/neon/table_lookup_tests.rs"] +mod table_lookup_tests; + +#[cfg(test)] +#[path = "../../arm_shared/neon/shift_and_insert_tests.rs"] +mod shift_and_insert_tests; + +#[cfg(test)] +#[path = "../../arm_shared/neon/load_tests.rs"] +mod load_tests; + +#[cfg(test)] +#[path = "../../arm_shared/neon/store_tests.rs"] +mod store_tests; diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/prefetch.rs b/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/prefetch.rs new file mode 100644 index 000000000..3ae0ef506 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/prefetch.rs @@ -0,0 +1,73 @@ +#[cfg(test)] +use stdarch_test::assert_instr; + +extern "unadjusted" { + #[link_name = "llvm.prefetch"] + fn prefetch(p: *const i8, rw: i32, loc: i32, ty: i32); +} + +/// See [`prefetch`](fn._prefetch.html). +pub const _PREFETCH_READ: i32 = 0; + +/// See [`prefetch`](fn._prefetch.html). +pub const _PREFETCH_WRITE: i32 = 1; + +/// See [`prefetch`](fn._prefetch.html). +pub const _PREFETCH_LOCALITY0: i32 = 0; + +/// See [`prefetch`](fn._prefetch.html). +pub const _PREFETCH_LOCALITY1: i32 = 1; + +/// See [`prefetch`](fn._prefetch.html). +pub const _PREFETCH_LOCALITY2: i32 = 2; + +/// See [`prefetch`](fn._prefetch.html). +pub const _PREFETCH_LOCALITY3: i32 = 3; + +/// Fetch the cache line that contains address `p` using the given `RW` and `LOCALITY`. +/// +/// The `RW` must be one of: +/// +/// * [`_PREFETCH_READ`](constant._PREFETCH_READ.html): the prefetch is preparing +/// for a read. +/// +/// * [`_PREFETCH_WRITE`](constant._PREFETCH_WRITE.html): the prefetch is preparing +/// for a write. +/// +/// The `LOCALITY` must be one of: +/// +/// * [`_PREFETCH_LOCALITY0`](constant._PREFETCH_LOCALITY0.html): Streaming or +/// non-temporal prefetch, for data that is used only once. +/// +/// * [`_PREFETCH_LOCALITY1`](constant._PREFETCH_LOCALITY1.html): Fetch into level 3 cache. +/// +/// * [`_PREFETCH_LOCALITY2`](constant._PREFETCH_LOCALITY2.html): Fetch into level 2 cache. +/// +/// * [`_PREFETCH_LOCALITY3`](constant._PREFETCH_LOCALITY3.html): Fetch into level 1 cache. +/// +/// The prefetch memory instructions signal to the memory system that memory accesses +/// from a specified address are likely to occur in the near future. The memory system +/// can respond by taking actions that are expected to speed up the memory access when +/// they do occur, such as preloading the specified address into one or more caches. +/// Because these signals are only hints, it is valid for a particular CPU to treat +/// any or all prefetch instructions as a NOP. +/// +/// +/// [Arm's documentation](https://developer.arm.com/documentation/den0024/a/the-a64-instruction-set/memory-access-instructions/prefetching-memory?lang=en) +#[inline(always)] +#[cfg_attr(test, assert_instr("prfm pldl1strm", RW = _PREFETCH_READ, LOCALITY = _PREFETCH_LOCALITY0))] +#[cfg_attr(test, assert_instr("prfm pldl3keep", RW = _PREFETCH_READ, LOCALITY = _PREFETCH_LOCALITY1))] +#[cfg_attr(test, assert_instr("prfm pldl2keep", RW = _PREFETCH_READ, LOCALITY = _PREFETCH_LOCALITY2))] +#[cfg_attr(test, assert_instr("prfm pldl1keep", RW = _PREFETCH_READ, LOCALITY = _PREFETCH_LOCALITY3))] +#[cfg_attr(test, assert_instr("prfm pstl1strm", RW = _PREFETCH_WRITE, LOCALITY = _PREFETCH_LOCALITY0))] +#[cfg_attr(test, assert_instr("prfm pstl3keep", RW = _PREFETCH_WRITE, LOCALITY = _PREFETCH_LOCALITY1))] +#[cfg_attr(test, assert_instr("prfm pstl2keep", RW = _PREFETCH_WRITE, LOCALITY = _PREFETCH_LOCALITY2))] +#[cfg_attr(test, assert_instr("prfm pstl1keep", RW = _PREFETCH_WRITE, LOCALITY = _PREFETCH_LOCALITY3))] +#[rustc_legacy_const_generics(1, 2)] +// FIXME: Replace this with the standard ACLE __pld/__pldx/__pli/__plix intrinsics +pub unsafe fn _prefetch(p: *const i8) { + // We use the `llvm.prefetch` intrinsic with `cache type` = 1 (data cache). + static_assert_imm1!(RW); + static_assert_imm2!(LOCALITY); + prefetch(p, RW, LOCALITY, 1); +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/test_support.rs b/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/test_support.rs new file mode 100644 index 000000000..9c5994b15 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/test_support.rs @@ -0,0 +1,184 @@ +use crate::core_arch::{aarch64::neon::*, arm_shared::*, simd::*}; +use std::{i16, i32, i8, mem::transmute, u16, u32, u8, vec::Vec}; + +macro_rules! V_u64 { + () => { + vec![ + 0x0000000000000000u64, + 0x0101010101010101u64, + 0x0202020202020202u64, + 0x0F0F0F0F0F0F0F0Fu64, + 0x8080808080808080u64, + 0xF0F0F0F0F0F0F0F0u64, + 0xFFFFFFFFFFFFFFFFu64, + ] + }; +} + +macro_rules! V_f64 { + () => { + vec![ + 0.0f64, + 1.0f64, + -1.0f64, + 1.2f64, + 2.4f64, + std::f64::MAX, + std::f64::MIN, + std::f64::INFINITY, + std::f64::NEG_INFINITY, + std::f64::NAN, + ] + }; +} + +macro_rules! to64 { + ($t : ident) => { + |v: $t| -> u64 { transmute(v) } + }; +} + +macro_rules! to128 { + ($t : ident) => { + |v: $t| -> u128 { transmute(v) } + }; +} + +pub(crate) fn test( + vals: Vec, + fill1: fn(T) -> V, + fill2: fn(U) -> W, + cast: fn(W) -> X, + test_fun: fn(V, V) -> W, + verify_fun: fn(T, T) -> U, +) where + T: Copy + core::fmt::Debug, + U: Copy + core::fmt::Debug + std::cmp::PartialEq, + V: Copy + core::fmt::Debug, + W: Copy + core::fmt::Debug, + X: Copy + core::fmt::Debug + std::cmp::PartialEq, +{ + let pairs = vals.iter().zip(vals.iter()); + + for (i, j) in pairs { + let a: V = fill1(*i); + let b: V = fill1(*j); + + let actual_pre: W = test_fun(a, b); + let expected_pre: W = fill2(verify_fun(*i, *j)); + + let actual: X = cast(actual_pre); + let expected: X = cast(expected_pre); + + assert_eq!( + actual, expected, + "[{:?}:{:?}] :\nf({:?}, {:?}) = {:?}\ng({:?}, {:?}) = {:?}\n", + *i, *j, &a, &b, actual_pre, &a, &b, expected_pre + ); + } +} + +macro_rules! gen_test_fn { + ($n: ident, $t: ident, $u: ident, $v: ident, $w: ident, $x: ident, $vals: expr, $fill1: expr, $fill2: expr, $cast: expr) => { + pub(crate) fn $n(test_fun: fn($v, $v) -> $w, verify_fun: fn($t, $t) -> $u) { + unsafe { + test::<$t, $u, $v, $w, $x>($vals, $fill1, $fill2, $cast, test_fun, verify_fun) + }; + } + }; +} + +macro_rules! gen_fill_fn { + ($id: ident, $el_width: expr, $num_els: expr, $in_t : ident, $out_t: ident, $cmp_t: ident) => { + pub(crate) fn $id(val: $in_t) -> $out_t { + let initial: [$in_t; $num_els] = [val; $num_els]; + let result: $cmp_t = unsafe { transmute(initial) }; + let result_out: $out_t = unsafe { transmute(result) }; + + // println!("FILL: {:016x} as {} x {}: {:016x}", val.reverse_bits(), $el_width, $num_els, (result as u64).reverse_bits()); + + result_out + } + }; +} + +gen_fill_fn!(fill_u64, 64, 1, u64, uint64x1_t, u64); +gen_fill_fn!(fillq_u64, 64, 2, u64, uint64x2_t, u128); +gen_fill_fn!(fill_f64, 64, 1, f64, float64x1_t, u64); +gen_fill_fn!(fillq_f64, 64, 2, f64, float64x2_t, u128); +gen_fill_fn!(fill_p64, 64, 1, u64, poly64x1_t, u64); +gen_fill_fn!(fillq_p64, 64, 2, u64, poly64x2_t, u128); + +gen_test_fn!( + test_ari_f64, + f64, + f64, + float64x1_t, + float64x1_t, + u64, + V_f64!(), + fill_f64, + fill_f64, + to64!(float64x1_t) +); +gen_test_fn!( + test_cmp_f64, + f64, + u64, + float64x1_t, + uint64x1_t, + u64, + V_f64!(), + fill_f64, + fill_u64, + to64!(uint64x1_t) +); +gen_test_fn!( + testq_ari_f64, + f64, + f64, + float64x2_t, + float64x2_t, + u128, + V_f64!(), + fillq_f64, + fillq_f64, + to128!(float64x2_t) +); +gen_test_fn!( + testq_cmp_f64, + f64, + u64, + float64x2_t, + uint64x2_t, + u128, + V_f64!(), + fillq_f64, + fillq_u64, + to128!(uint64x2_t) +); + +gen_test_fn!( + test_cmp_p64, + u64, + u64, + poly64x1_t, + uint64x1_t, + u64, + V_u64!(), + fill_p64, + fill_u64, + to64!(uint64x1_t) +); +gen_test_fn!( + testq_cmp_p64, + u64, + u64, + poly64x2_t, + uint64x2_t, + u128, + V_u64!(), + fillq_p64, + fillq_u64, + to128!(uint64x2_t) +); diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/tme.rs b/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/tme.rs new file mode 100644 index 000000000..d1b2cf334 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/tme.rs @@ -0,0 +1,179 @@ +//! ARM's Transactional Memory Extensions (TME). +//! +//! This CPU feature is available on Aarch64 - A architecture profile. +//! This feature is in the non-neon feature set. TME specific vendor documentation can +//! be found [TME Intrinsics Introduction][tme_intrinsics_intro]. +//! +//! The reference is [ACLE Q4 2019][acle_q4_2019_ref]. +//! +//! ACLE has a section for TME extensions and state masks for aborts and failure codes. +//! [ARM A64 Architecture Register Datasheet][a_profile_future] also describes possible failure code scenarios. +//! +//! [acle_q4_2019_ref]: https://static.docs.arm.com/101028/0010/ACLE_2019Q4_release-0010.pdf +//! [tme_intrinsics_intro]: https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics +//! [llvm_aarch64_int]: https://github.com/llvm/llvm-project/commit/a36d31478c182903523e04eb271bbf102bfab2cc#diff-ff24e1c35f4d54f1110ce5d90c709319R626-R646 +//! [a_profile_future]: https://static.docs.arm.com/ddi0601/a/SysReg_xml_futureA-2019-04.pdf?_ga=2.116560387.441514988.1590524918-1110153136.1588469296 + +#[cfg(test)] +use stdarch_test::assert_instr; + +extern "unadjusted" { + #[link_name = "llvm.aarch64.tstart"] + fn aarch64_tstart() -> u64; + #[link_name = "llvm.aarch64.tcommit"] + fn aarch64_tcommit() -> (); + #[link_name = "llvm.aarch64.tcancel"] + fn aarch64_tcancel(imm0: u64) -> (); + #[link_name = "llvm.aarch64.ttest"] + fn aarch64_ttest() -> u64; +} + +/// Transaction successfully started. +pub const _TMSTART_SUCCESS: u64 = 0x00_u64; + +/// Extraction mask for failure reason +pub const _TMFAILURE_REASON: u64 = 0x00007FFF_u64; + +/// Transaction retry is possible. +pub const _TMFAILURE_RTRY: u64 = 1 << 15; + +/// Transaction executed a TCANCEL instruction +pub const _TMFAILURE_CNCL: u64 = 1 << 16; + +/// Transaction aborted because a conflict occurred +pub const _TMFAILURE_MEM: u64 = 1 << 17; + +/// Fallback error type for any other reason +pub const _TMFAILURE_IMP: u64 = 1 << 18; + +/// Transaction aborted because a non-permissible operation was attempted +pub const _TMFAILURE_ERR: u64 = 1 << 19; + +/// Transaction aborted due to read or write set limit was exceeded +pub const _TMFAILURE_SIZE: u64 = 1 << 20; + +/// Transaction aborted due to transactional nesting level was exceeded +pub const _TMFAILURE_NEST: u64 = 1 << 21; + +/// Transaction aborted due to a debug trap. +pub const _TMFAILURE_DBG: u64 = 1 << 22; + +/// Transaction failed from interrupt +pub const _TMFAILURE_INT: u64 = 1 << 23; + +/// Indicates a TRIVIAL version of TM is available +pub const _TMFAILURE_TRIVIAL: u64 = 1 << 24; + +/// Starts a new transaction. When the transaction starts successfully the return value is 0. +/// If the transaction fails, all state modifications are discarded and a cause of the failure +/// is encoded in the return value. +/// +/// [ARM TME Intrinsics](https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics). +#[inline] +#[target_feature(enable = "tme")] +#[cfg_attr(test, assert_instr(tstart))] +pub unsafe fn __tstart() -> u64 { + aarch64_tstart() +} + +/// Commits the current transaction. For a nested transaction, the only effect is that the +/// transactional nesting depth is decreased. For an outer transaction, the state modifications +/// performed transactionally are committed to the architectural state. +/// +/// [ARM TME Intrinsics](https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics). +#[inline] +#[target_feature(enable = "tme")] +#[cfg_attr(test, assert_instr(tcommit))] +pub unsafe fn __tcommit() { + aarch64_tcommit() +} + +/// Cancels the current transaction and discards all state modifications that were performed transactionally. +/// +/// [ARM TME Intrinsics](https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics). +#[inline] +#[target_feature(enable = "tme")] +#[cfg_attr(test, assert_instr(tcancel, IMM16 = 0x0))] +#[rustc_legacy_const_generics(0)] +pub unsafe fn __tcancel() { + static_assert!(IMM16: u64 where IMM16 <= 65535); + aarch64_tcancel(IMM16); +} + +/// Tests if executing inside a transaction. If no transaction is currently executing, +/// the return value is 0. Otherwise, this intrinsic returns the depth of the transaction. +/// +/// [ARM TME Intrinsics](https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics). +#[inline] +#[target_feature(enable = "tme")] +#[cfg_attr(test, assert_instr(ttest))] +pub unsafe fn __ttest() -> u64 { + aarch64_ttest() +} + +#[cfg(test)] +mod tests { + use stdarch_test::simd_test; + + use crate::core_arch::aarch64::*; + + const CANCEL_CODE: u64 = (0 | (0x123 & _TMFAILURE_REASON) as u64) as u64; + + #[simd_test(enable = "tme")] + unsafe fn test_tstart() { + let mut x = 0; + for i in 0..10 { + let code = tme::__tstart(); + if code == _TMSTART_SUCCESS { + x += 1; + assert_eq!(x, i + 1); + break; + } + assert_eq!(x, 0); + } + } + + #[simd_test(enable = "tme")] + unsafe fn test_tcommit() { + let mut x = 0; + for i in 0..10 { + let code = tme::__tstart(); + if code == _TMSTART_SUCCESS { + x += 1; + assert_eq!(x, i + 1); + tme::__tcommit(); + } + assert_eq!(x, i + 1); + } + } + + #[simd_test(enable = "tme")] + unsafe fn test_tcancel() { + let mut x = 0; + + for i in 0..10 { + let code = tme::__tstart(); + if code == _TMSTART_SUCCESS { + x += 1; + assert_eq!(x, i + 1); + tme::__tcancel::(); + break; + } + } + + assert_eq!(x, 0); + } + + #[simd_test(enable = "tme")] + unsafe fn test_ttest() { + for _ in 0..10 { + let code = tme::__tstart(); + if code == _TMSTART_SUCCESS { + if tme::__ttest() == 2 { + tme::__tcancel::(); + break; + } + } + } + } +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/arm/armclang.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm/armclang.rs index 2e0a82ade..e68c02d02 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/arm/armclang.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/arm/armclang.rs @@ -11,58 +11,25 @@ use stdarch_test::assert_instr; /// Inserts a breakpoint instruction. /// -/// `val` is a compile-time constant integer in range `[0, 255]`. +/// `VAL` is a compile-time constant integer in range `[0, 255]`. /// -/// The breakpoint instruction inserted is: -/// -/// * `BKPT` when compiling as T32, -/// * `BRK` when compiling as A32 or A64. -/// -/// # Safety -/// -/// If `val` is out-of-range the behavior is **undefined**. +/// The breakpoint instruction inserted is `BKPT` on A32/T32. /// /// # Note /// /// [ARM's documentation][arm_docs] defines that `__breakpoint` accepts the -/// following values for `val`: +/// following values for `VAL`: /// -/// - `0...65535` when compiling as A32 or A64, +/// - `0...65535` when compiling as A32, /// - `0...255` when compiling as T32. /// -/// The current implementation only accepts values in range `[0, 255]` - if the -/// value is out-of-range the behavior is **undefined**. +/// The current implementation only accepts values in range `[0, 255]`. /// /// [arm_docs]: https://developer.arm.com/docs/100067/latest/compiler-specific-intrinsics/__breakpoint-intrinsic -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(bkpt, val = 0))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(brk, val = 0))] +#[cfg_attr(test, assert_instr(bkpt, VAL = 0))] #[inline(always)] -#[rustc_args_required_const(0)] -pub unsafe fn __breakpoint(val: i32) { - // Ensure that this compiles correctly on non-arm architectures, so libstd - // doc builds work. The proper macro will shadow this definition below. - #[allow(unused_macros)] - macro_rules! call { - ($e:expr) => { - () - }; - } - - #[cfg(target_arch = "arm")] - macro_rules! call { - ($imm8:expr) => { - asm!(concat!("BKPT ", stringify!($imm8)) : : : : "volatile") - } - } - - #[cfg(target_arch = "aarch64")] - macro_rules! call { - ($imm8:expr) => { - asm!(concat!("BRK ", stringify!($imm8)) : : : : "volatile") - } - } - - // We can't `panic!` inside this intrinsic, so we can't really validate the - // arguments here. If `val` is out-of-range this macro uses `val == 255`: - constify_imm8!(val, call); +#[rustc_legacy_const_generics(0)] +pub unsafe fn __breakpoint() { + static_assert_imm8!(VAL); + crate::arch::asm!("bkpt #{}", const VAL); } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/acle/dsp.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm/dsp.rs similarity index 99% rename from crux-mir/lib/stdarch/crates/core_arch/src/acle/dsp.rs rename to crux-mir/lib/stdarch/crates/core_arch/src/arm/dsp.rs index 49986fa6e..6720f97a5 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/acle/dsp.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/arm/dsp.rs @@ -32,7 +32,7 @@ types! { pub struct uint16x2_t(u16, u16); } -extern "C" { +extern "unadjusted" { #[link_name = "llvm.arm.smulbb"] fn arm_smulbb(a: i32, b: i32) -> i32; @@ -240,7 +240,7 @@ mod tests { arm::*, simd::{i16x2, i8x4, u8x4}, }; - use std::{i32, mem::transmute}; + use std::mem::transmute; use stdarch_test::simd_test; #[test] diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/acle/ex.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm/ex.rs similarity index 80% rename from crux-mir/lib/stdarch/crates/core_arch/src/acle/ex.rs rename to crux-mir/lib/stdarch/crates/core_arch/src/arm/ex.rs index 0426c6518..75f378642 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/acle/ex.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/arm/ex.rs @@ -8,9 +8,10 @@ #[cfg(any( all(target_feature = "v6k", not(target_feature = "mclass")), // excludes v6-M all(target_feature = "v7", target_feature = "mclass"), // v7-M + doc ))] pub unsafe fn __clrex() { - extern "C" { + extern "unadjusted" { #[link_name = "llvm.arm.clrex"] fn clrex(); } @@ -18,14 +19,15 @@ pub unsafe fn __clrex() { clrex() } -/// Executes a exclusive LDR instruction for 8 bit value. +/// Executes an exclusive LDR instruction for 8 bit value. // Supported: v6K, v7-M, v7-A, v7-R // Not supported: v5, v6, v6-M -#[cfg( +#[cfg(any( target_feature = "v6k", // includes v7-M but excludes v6-M -)] + doc +))] pub unsafe fn __ldrexb(p: *const u8) -> u8 { - extern "C" { + extern "unadjusted" { #[link_name = "llvm.arm.ldrex.p0i8"] fn ldrex8(p: *const u8) -> u32; } @@ -33,14 +35,15 @@ pub unsafe fn __ldrexb(p: *const u8) -> u8 { ldrex8(p) as u8 } -/// Executes a exclusive LDR instruction for 16 bit value. +/// Executes an exclusive LDR instruction for 16 bit value. // Supported: v6K, v7-M, v7-A, v7-R, v8 // Not supported: v5, v6, v6-M -#[cfg( +#[cfg(any( target_feature = "v6k", // includes v7-M but excludes v6-M -)] + doc +))] pub unsafe fn __ldrexh(p: *const u16) -> u16 { - extern "C" { + extern "unadjusted" { #[link_name = "llvm.arm.ldrex.p0i16"] fn ldrex16(p: *const u16) -> u32; } @@ -48,15 +51,16 @@ pub unsafe fn __ldrexh(p: *const u16) -> u16 { ldrex16(p) as u16 } -/// Executes a exclusive LDR instruction for 32 bit value. +/// Executes an exclusive LDR instruction for 32 bit value. // Supported: v6, v7-M, v6K, v7-A, v7-R, v8 // Not supported: v5, v6-M #[cfg(any( all(target_feature = "v6", not(target_feature = "mclass")), // excludes v6-M all(target_feature = "v7", target_feature = "mclass"), // v7-M + doc ))] pub unsafe fn __ldrex(p: *const u32) -> u32 { - extern "C" { + extern "unadjusted" { #[link_name = "llvm.arm.ldrex.p0i32"] fn ldrex32(p: *const u32) -> u32; } @@ -64,16 +68,17 @@ pub unsafe fn __ldrex(p: *const u32) -> u32 { ldrex32(p) } -/// Executes a exclusive STR instruction for 8 bit values +/// Executes an exclusive STR instruction for 8 bit values /// /// Returns `0` if the operation succeeded, or `1` if it failed // supported: v6K, v7-M, v7-A, v7-R // Not supported: v5, v6, v6-M -#[cfg( +#[cfg(any( target_feature = "v6k", // includes v7-M but excludes v6-M -)] + doc +))] pub unsafe fn __strexb(value: u32, addr: *mut u8) -> u32 { - extern "C" { + extern "unadjusted" { #[link_name = "llvm.arm.strex.p0i8"] fn strex8(value: u32, addr: *mut u8) -> u32; } @@ -81,16 +86,18 @@ pub unsafe fn __strexb(value: u32, addr: *mut u8) -> u32 { strex8(value, addr) } -/// Executes a exclusive STR instruction for 16 bit values +/// Executes an exclusive STR instruction for 16 bit values /// /// Returns `0` if the operation succeeded, or `1` if it failed // Supported: v6K, v7-M, v7-A, v7-R, v8 // Not supported: v5, v6, v6-M -#[cfg( +#[cfg(target_feature = "aarch64")] +#[cfg(any( target_feature = "v6k", // includes v7-M but excludes v6-M -)] + doc +))] pub unsafe fn __strexh(value: u16, addr: *mut u16) -> u32 { - extern "C" { + extern "unadjusted" { #[link_name = "llvm.arm.strex.p0i16"] fn strex16(value: u32, addr: *mut u16) -> u32; } @@ -98,7 +105,7 @@ pub unsafe fn __strexh(value: u16, addr: *mut u16) -> u32 { strex16(value as u32, addr) } -/// Executes a exclusive STR instruction for 32 bit values +/// Executes an exclusive STR instruction for 32 bit values /// /// Returns `0` if the operation succeeded, or `1` if it failed // Supported: v6, v7-M, v6K, v7-A, v7-R, v8 @@ -106,9 +113,10 @@ pub unsafe fn __strexh(value: u16, addr: *mut u16) -> u32 { #[cfg(any( all(target_feature = "v6", not(target_feature = "mclass")), // excludes v6-M all(target_feature = "v7", target_feature = "mclass"), // v7-M + doc ))] pub unsafe fn __strex(value: u32, addr: *mut u32) -> u32 { - extern "C" { + extern "unadjusted" { #[link_name = "llvm.arm.strex.p0i32"] fn strex32(value: u32, addr: *mut u32) -> u32; } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/arm/mod.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm/mod.rs index a467c2ce8..efe0068d4 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/arm/mod.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/arm/mod.rs @@ -5,40 +5,81 @@ //! //! [arm_ref]: http://infocenter.arm.com/help/topic/com.arm.doc.ihi0073a/IHI0073A_arm_neon_intrinsics_ref.pdf //! [arm_dat]: https://developer.arm.com/technologies/neon/intrinsics -#![allow(non_camel_case_types)] mod armclang; - pub use self::armclang::*; mod v6; pub use self::v6::*; -#[cfg(any(target_arch = "aarch64", target_feature = "v7"))] -mod v7; -#[cfg(any(target_arch = "aarch64", target_feature = "v7"))] -pub use self::v7::*; +// Supported arches: 6, 7-M. See Section 10.1 of ACLE (e.g. SSAT) +#[cfg(any(target_feature = "v6", doc))] +mod sat; + +#[cfg(any(target_feature = "v6", doc))] +pub use self::sat::*; + +// Supported arches: 5TE, 7E-M. See Section 10.1 of ACLE (e.g. QADD) +// We also include the A profile even though DSP is deprecated on that profile as of ACLE 2.0 (see +// section 5.4.7) +// Here we workaround the difference between LLVM's +dsp and ACLE's __ARM_FEATURE_DSP by gating on +// '+v5te' rather than on '+dsp' +#[cfg(any( + // >= v5TE but excludes v7-M + all(target_feature = "v5te", not(target_feature = "mclass")), + // v7E-M + all(target_feature = "mclass", target_feature = "dsp"), + doc, +))] +pub mod dsp; + +#[cfg(any( + // >= v5TE but excludes v7-M + all(target_feature = "v5te", not(target_feature = "mclass")), + // v7E-M + all(target_feature = "mclass", target_feature = "dsp"), + doc, +))] +pub use self::dsp::*; -// NEON is supported on AArch64, and on ARM when built with the v7 and neon -// features. Building ARM without neon produces incorrect codegen. +// Deprecated in ACLE 2.0 for the A profile but fully supported on the M and R profiles, says +// Section 5.4.9 of ACLE. We'll expose these for the A profile even if deprecated #[cfg(any( - target_arch = "aarch64", - all(target_feature = "v7", target_feature = "neon"), - dox + // v7-A, v7-R + all(target_feature = "v6", not(target_feature = "mclass")), + // v7E-M + all(target_feature = "mclass", target_feature = "dsp"), + doc, ))] -mod neon; +mod simd32; + #[cfg(any( - target_arch = "aarch64", - all(target_feature = "v7", target_feature = "neon"), - dox + // v7-A, v7-R + all(target_feature = "v6", not(target_feature = "mclass")), + // v7E-M + all(target_feature = "mclass", target_feature = "dsp"), + doc, ))] -pub use self::neon::*; +pub use self::simd32::*; + +#[cfg(any(target_feature = "v7", doc))] +mod v7; +#[cfg(any(target_feature = "v7", doc))] +pub use self::v7::*; + +mod ex; +pub use self::ex::*; -pub use crate::core_arch::acle::*; +pub use crate::core_arch::arm_shared::*; #[cfg(test)] use stdarch_test::assert_instr; +#[cfg(any(target_feature = "v7", doc))] +pub(crate) mod neon; +#[cfg(any(target_feature = "v7", doc))] +pub use neon::*; + /// Generates the trap instruction `UDF` #[cfg(target_arch = "arm")] #[cfg_attr(test, assert_instr(udf))] @@ -46,3 +87,27 @@ use stdarch_test::assert_instr; pub unsafe fn udf() -> ! { crate::intrinsics::abort() } + +/// Generates a DBG instruction. +/// +/// This provides a hint to debugging and related systems. The argument must be +/// a constant integer from 0 to 15 inclusive. See implementation documentation +/// for the effect (if any) of this instruction and the meaning of the +/// argument. This is available only when compiling for AArch32. +// Section 10.1 of ACLE says that the supported arches are: 7, 7-M +// "The DBG hint instruction is added in ARMv7. It is UNDEFINED in the ARMv6 base architecture, and +// executes as a NOP instruction in ARMv6K and ARMv6T2." - ARM Architecture Reference Manual ARMv7-A +// and ARMv7-R edition (ARM DDI 0406C.c) sections D12.4.1 "ARM instruction set support" and D12.4.2 +// "Thumb instruction set support" +#[cfg(any(target_feature = "v7", doc))] +#[inline(always)] +#[rustc_legacy_const_generics(0)] +pub unsafe fn __dbg() { + static_assert_imm4!(IMM4); + dbg(IMM4); +} + +extern "unadjusted" { + #[link_name = "llvm.arm.dbg"] + fn dbg(_: i32); +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/arm/neon.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm/neon.rs index 237bdaeb0..a6291c95c 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/arm/neon.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/arm/neon.rs @@ -1,156 +1,48 @@ -//! ARMv7 NEON intrinsics - +use crate::core_arch::arm_shared::neon::*; +use crate::core_arch::simd::{f32x4, i32x4, u32x4}; use crate::core_arch::simd_llvm::*; -#[cfg(target_arch = "arm")] -use crate::mem::transmute; +use crate::mem::{align_of, transmute}; + #[cfg(test)] use stdarch_test::assert_instr; -types! { - /// ARM-specific 64-bit wide vector of eight packed `i8`. - pub struct int8x8_t(i8, i8, i8, i8, i8, i8, i8, i8); - /// ARM-specific 64-bit wide vector of eight packed `u8`. - pub struct uint8x8_t(u8, u8, u8, u8, u8, u8, u8, u8); - /// ARM-specific 64-bit wide polynomial vector of eight packed `u8`. - pub struct poly8x8_t(u8, u8, u8, u8, u8, u8, u8, u8); - /// ARM-specific 64-bit wide vector of four packed `i16`. - pub struct int16x4_t(i16, i16, i16, i16); - /// ARM-specific 64-bit wide vector of four packed `u16`. - pub struct uint16x4_t(u16, u16, u16, u16); - // FIXME: ARM-specific 64-bit wide vector of four packed `f16`. - // pub struct float16x4_t(f16, f16, f16, f16); - /// ARM-specific 64-bit wide vector of four packed `u16`. - pub struct poly16x4_t(u16, u16, u16, u16); - /// ARM-specific 64-bit wide vector of two packed `i32`. - pub struct int32x2_t(i32, i32); - /// ARM-specific 64-bit wide vector of two packed `u32`. - pub struct uint32x2_t(u32, u32); - /// ARM-specific 64-bit wide vector of two packed `f32`. - pub struct float32x2_t(f32, f32); - /// ARM-specific 64-bit wide vector of one packed `i64`. - pub struct int64x1_t(i64); - /// ARM-specific 64-bit wide vector of one packed `u64`. - pub struct uint64x1_t(u64); - - /// ARM-specific 128-bit wide vector of sixteen packed `i8`. - pub struct int8x16_t( - i8, i8 ,i8, i8, i8, i8 ,i8, i8, - i8, i8 ,i8, i8, i8, i8 ,i8, i8, - ); - /// ARM-specific 128-bit wide vector of sixteen packed `u8`. - pub struct uint8x16_t( - u8, u8 ,u8, u8, u8, u8 ,u8, u8, - u8, u8 ,u8, u8, u8, u8 ,u8, u8, - ); - /// ARM-specific 128-bit wide vector of sixteen packed `u8`. - pub struct poly8x16_t( - u8, u8, u8, u8, u8, u8, u8, u8, - u8, u8, u8, u8, u8, u8, u8, u8 - ); - /// ARM-specific 128-bit wide vector of eight packed `i16`. - pub struct int16x8_t(i16, i16, i16, i16, i16, i16, i16, i16); - /// ARM-specific 128-bit wide vector of eight packed `u16`. - pub struct uint16x8_t(u16, u16, u16, u16, u16, u16, u16, u16); - // FIXME: ARM-specific 128-bit wide vector of eight packed `f16`. - // pub struct float16x8_t(f16, f16, f16, f16, f16, f16, f16); - /// ARM-specific 128-bit wide vector of eight packed `u16`. - pub struct poly16x8_t(u16, u16, u16, u16, u16, u16, u16, u16); - /// ARM-specific 128-bit wide vector of four packed `i32`. - pub struct int32x4_t(i32, i32, i32, i32); - /// ARM-specific 128-bit wide vector of four packed `u32`. - pub struct uint32x4_t(u32, u32, u32, u32); - /// ARM-specific 128-bit wide vector of four packed `f32`. - pub struct float32x4_t(f32, f32, f32, f32); - /// ARM-specific 128-bit wide vector of two packed `i64`. - pub struct int64x2_t(i64, i64); - /// ARM-specific 128-bit wide vector of two packed `u64`. - pub struct uint64x2_t(u64, u64); -} - -/// ARM-specific type containing two `int8x8_t` vectors. -#[derive(Copy, Clone)] -pub struct int8x8x2_t(pub int8x8_t, pub int8x8_t); -/// ARM-specific type containing three `int8x8_t` vectors. -#[derive(Copy, Clone)] -pub struct int8x8x3_t(pub int8x8_t, pub int8x8_t, pub int8x8_t); -/// ARM-specific type containing four `int8x8_t` vectors. -#[derive(Copy, Clone)] -pub struct int8x8x4_t(pub int8x8_t, pub int8x8_t, pub int8x8_t, pub int8x8_t); - -/// ARM-specific type containing two `uint8x8_t` vectors. -#[derive(Copy, Clone)] -pub struct uint8x8x2_t(pub uint8x8_t, pub uint8x8_t); -/// ARM-specific type containing three `uint8x8_t` vectors. -#[derive(Copy, Clone)] -pub struct uint8x8x3_t(pub uint8x8_t, pub uint8x8_t, pub uint8x8_t); -/// ARM-specific type containing four `uint8x8_t` vectors. -#[derive(Copy, Clone)] -pub struct uint8x8x4_t(pub uint8x8_t, pub uint8x8_t, pub uint8x8_t, pub uint8x8_t); - -/// ARM-specific type containing two `poly8x8_t` vectors. -#[derive(Copy, Clone)] -pub struct poly8x8x2_t(pub poly8x8_t, pub poly8x8_t); -/// ARM-specific type containing three `poly8x8_t` vectors. -#[derive(Copy, Clone)] -pub struct poly8x8x3_t(pub poly8x8_t, pub poly8x8_t, pub poly8x8_t); -/// ARM-specific type containing four `poly8x8_t` vectors. -#[derive(Copy, Clone)] -pub struct poly8x8x4_t(pub poly8x8_t, pub poly8x8_t, pub poly8x8_t, pub poly8x8_t); +#[allow(non_camel_case_types)] +pub(crate) type p8 = u8; +#[allow(non_camel_case_types)] +pub(crate) type p16 = u16; #[allow(improper_ctypes)] -extern "C" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frsqrte.v2f32")] - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrsqrte.v2f32")] - fn frsqrte_v2f32(a: float32x2_t) -> float32x2_t; - - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmins.v8i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sminp.v8i8")] - fn vpmins_v8i8(a: int8x8_t, b: int8x8_t) -> int8x8_t; - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmins.v4i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sminp.v4i16")] - fn vpmins_v4i16(a: int16x4_t, b: int16x4_t) -> int16x4_t; - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmins.v2i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sminp.v2i32")] - fn vpmins_v2i32(a: int32x2_t, b: int32x2_t) -> int32x2_t; - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpminu.v8i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uminp.v8i8")] - fn vpminu_v8i8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpminu.v4i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uminp.v4i16")] - fn vpminu_v4i16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpminu.v2i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uminp.v2i32")] - fn vpminu_v2i32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmins.v2f32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminp.v2f32")] - fn vpminf_v2f32(a: float32x2_t, b: float32x2_t) -> float32x2_t; - - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmaxs.v8i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smaxp.v8i8")] - fn vpmaxs_v8i8(a: int8x8_t, b: int8x8_t) -> int8x8_t; - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmaxs.v4i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smaxp.v4i16")] - fn vpmaxs_v4i16(a: int16x4_t, b: int16x4_t) -> int16x4_t; - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmaxs.v2i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smaxp.v2i32")] - fn vpmaxs_v2i32(a: int32x2_t, b: int32x2_t) -> int32x2_t; - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmaxu.v8i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umaxp.v8i8")] - fn vpmaxu_v8i8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmaxu.v4i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umaxp.v4i16")] - fn vpmaxu_v4i16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmaxu.v2i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umaxp.v2i32")] - fn vpmaxu_v2i32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmaxs.v2f32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxp.v2f32")] - fn vpmaxf_v2f32(a: float32x2_t, b: float32x2_t) -> float32x2_t; -} - -#[cfg(target_arch = "arm")] -#[allow(improper_ctypes)] -extern "C" { +extern "unadjusted" { + #[link_name = "llvm.arm.neon.vbsl.v8i8"] + fn vbsl_s8_(a: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t; + #[link_name = "llvm.arm.neon.vbsl.v16i8"] + fn vbslq_s8_(a: int8x16_t, b: int8x16_t, c: int8x16_t) -> int8x16_t; + #[link_name = "llvm.arm.neon.vpadals.v4i16.v8i8"] + pub(crate) fn vpadal_s8_(a: int16x4_t, b: int8x8_t) -> int16x4_t; + #[link_name = "llvm.arm.neon.vpadals.v2i32.v4i16"] + pub(crate) fn vpadal_s16_(a: int32x2_t, b: int16x4_t) -> int32x2_t; + #[link_name = "llvm.arm.neon.vpadals.v1i64.v2i32"] + pub(crate) fn vpadal_s32_(a: int64x1_t, b: int32x2_t) -> int64x1_t; + #[link_name = "llvm.arm.neon.vpadals.v8i16.v16i8"] + pub(crate) fn vpadalq_s8_(a: int16x8_t, b: int8x16_t) -> int16x8_t; + #[link_name = "llvm.arm.neon.vpadals.v4i32.v8i16"] + pub(crate) fn vpadalq_s16_(a: int32x4_t, b: int16x8_t) -> int32x4_t; + #[link_name = "llvm.arm.neon.vpadals.v2i64.v4i32"] + pub(crate) fn vpadalq_s32_(a: int64x2_t, b: int32x4_t) -> int64x2_t; + + #[link_name = "llvm.arm.neon.vpadalu.v4i16.v8i8"] + pub(crate) fn vpadal_u8_(a: uint16x4_t, b: uint8x8_t) -> uint16x4_t; + #[link_name = "llvm.arm.neon.vpadalu.v2i32.v4i16"] + pub(crate) fn vpadal_u16_(a: uint32x2_t, b: uint16x4_t) -> uint32x2_t; + #[link_name = "llvm.arm.neon.vpadalu.v1i64.v2i32"] + pub(crate) fn vpadal_u32_(a: uint64x1_t, b: uint32x2_t) -> uint64x1_t; + #[link_name = "llvm.arm.neon.vpadalu.v8i16.v16i8"] + pub(crate) fn vpadalq_u8_(a: uint16x8_t, b: uint8x16_t) -> uint16x8_t; + #[link_name = "llvm.arm.neon.vpadalu.v4i32.v8i16"] + pub(crate) fn vpadalq_u16_(a: uint32x4_t, b: uint16x8_t) -> uint32x4_t; + #[link_name = "llvm.arm.neon.vpadalu.v2i64.v4i32"] + pub(crate) fn vpadalq_u32_(a: uint64x2_t, b: uint32x4_t) -> uint64x2_t; + #[link_name = "llvm.arm.neon.vtbl1"] fn vtbl1(a: int8x8_t, b: int8x8_t) -> int8x8_t; #[link_name = "llvm.arm.neon.vtbl2"] @@ -175,674 +67,461 @@ extern "C" { d: int8x8_t, e: int8x8_t, ) -> int8x8_t; -} - -/// Vector add. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] -pub unsafe fn vadd_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { - simd_add(a, b) -} - -/// Vector add. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] -pub unsafe fn vaddq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { - simd_add(a, b) -} - -/// Vector add. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] -pub unsafe fn vadd_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { - simd_add(a, b) -} - -/// Vector add. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] -pub unsafe fn vaddq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { - simd_add(a, b) -} - -/// Vector add. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] -pub unsafe fn vadd_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { - simd_add(a, b) -} - -/// Vector add. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] -pub unsafe fn vaddq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { - simd_add(a, b) -} - -/// Vector add. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] -pub unsafe fn vaddq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { - simd_add(a, b) -} - -/// Vector add. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] -pub unsafe fn vadd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - simd_add(a, b) -} - -/// Vector add. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] -pub unsafe fn vaddq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { - simd_add(a, b) -} - -/// Vector add. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] -pub unsafe fn vadd_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - simd_add(a, b) -} - -/// Vector add. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] -pub unsafe fn vaddq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { - simd_add(a, b) -} - -/// Vector add. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] -pub unsafe fn vadd_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - simd_add(a, b) -} - -/// Vector add. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] -pub unsafe fn vaddq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { - simd_add(a, b) -} - -/// Vector add. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] -pub unsafe fn vaddq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { - simd_add(a, b) -} - -/// Vector add. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fadd))] -pub unsafe fn vadd_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { - simd_add(a, b) -} -/// Vector add. + #[link_name = "llvm.arm.neon.vshiftins.v8i8"] + fn vshiftins_v8i8(a: int8x8_t, b: int8x8_t, shift: int8x8_t) -> int8x8_t; + #[link_name = "llvm.arm.neon.vshiftins.v16i8"] + fn vshiftins_v16i8(a: int8x16_t, b: int8x16_t, shift: int8x16_t) -> int8x16_t; + #[link_name = "llvm.arm.neon.vshiftins.v4i16"] + fn vshiftins_v4i16(a: int16x4_t, b: int16x4_t, shift: int16x4_t) -> int16x4_t; + #[link_name = "llvm.arm.neon.vshiftins.v8i16"] + fn vshiftins_v8i16(a: int16x8_t, b: int16x8_t, shift: int16x8_t) -> int16x8_t; + #[link_name = "llvm.arm.neon.vshiftins.v2i32"] + fn vshiftins_v2i32(a: int32x2_t, b: int32x2_t, shift: int32x2_t) -> int32x2_t; + #[link_name = "llvm.arm.neon.vshiftins.v4i32"] + fn vshiftins_v4i32(a: int32x4_t, b: int32x4_t, shift: int32x4_t) -> int32x4_t; + #[link_name = "llvm.arm.neon.vshiftins.v1i64"] + fn vshiftins_v1i64(a: int64x1_t, b: int64x1_t, shift: int64x1_t) -> int64x1_t; + #[link_name = "llvm.arm.neon.vshiftins.v2i64"] + fn vshiftins_v2i64(a: int64x2_t, b: int64x2_t, shift: int64x2_t) -> int64x2_t; + + #[link_name = "llvm.arm.neon.vld1.v8i8.p0i8"] + fn vld1_v8i8(addr: *const i8, align: i32) -> int8x8_t; + #[link_name = "llvm.arm.neon.vld1.v16i8.p0i8"] + fn vld1q_v16i8(addr: *const i8, align: i32) -> int8x16_t; + #[link_name = "llvm.arm.neon.vld1.v4i16.p0i8"] + fn vld1_v4i16(addr: *const i8, align: i32) -> int16x4_t; + #[link_name = "llvm.arm.neon.vld1.v8i16.p0i8"] + fn vld1q_v8i16(addr: *const i8, align: i32) -> int16x8_t; + #[link_name = "llvm.arm.neon.vld1.v2i32.p0i8"] + fn vld1_v2i32(addr: *const i8, align: i32) -> int32x2_t; + #[link_name = "llvm.arm.neon.vld1.v4i32.p0i8"] + fn vld1q_v4i32(addr: *const i8, align: i32) -> int32x4_t; + #[link_name = "llvm.arm.neon.vld1.v1i64.p0i8"] + fn vld1_v1i64(addr: *const i8, align: i32) -> int64x1_t; + #[link_name = "llvm.arm.neon.vld1.v2i64.p0i8"] + fn vld1q_v2i64(addr: *const i8, align: i32) -> int64x2_t; + #[link_name = "llvm.arm.neon.vld1.v2f32.p0i8"] + fn vld1_v2f32(addr: *const i8, align: i32) -> float32x2_t; + #[link_name = "llvm.arm.neon.vld1.v4f32.p0i8"] + fn vld1q_v4f32(addr: *const i8, align: i32) -> float32x4_t; + + #[link_name = "llvm.arm.neon.vst1.p0i8.v8i8"] + fn vst1_v8i8(addr: *const i8, val: int8x8_t, align: i32); + #[link_name = "llvm.arm.neon.vst1.p0i8.v16i8"] + fn vst1q_v16i8(addr: *const i8, val: int8x16_t, align: i32); + #[link_name = "llvm.arm.neon.vst1.p0i8.v4i16"] + fn vst1_v4i16(addr: *const i8, val: int16x4_t, align: i32); + #[link_name = "llvm.arm.neon.vst1.p0i8.v8i16"] + fn vst1q_v8i16(addr: *const i8, val: int16x8_t, align: i32); + #[link_name = "llvm.arm.neon.vst1.p0i8.v2i32"] + fn vst1_v2i32(addr: *const i8, val: int32x2_t, align: i32); + #[link_name = "llvm.arm.neon.vst1.p0i8.v4i32"] + fn vst1q_v4i32(addr: *const i8, val: int32x4_t, align: i32); + #[link_name = "llvm.arm.neon.vst1.p0i8.v1i64"] + fn vst1_v1i64(addr: *const i8, val: int64x1_t, align: i32); + #[link_name = "llvm.arm.neon.vst1.p0i8.v2i64"] + fn vst1q_v2i64(addr: *const i8, val: int64x2_t, align: i32); + #[link_name = "llvm.arm.neon.vst1.p0i8.v2f32"] + fn vst1_v2f32(addr: *const i8, val: float32x2_t, align: i32); + #[link_name = "llvm.arm.neon.vst1.p0i8.v4f32"] + fn vst1q_v4f32(addr: *const i8, val: float32x4_t, align: i32); +} + +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fadd))] -pub unsafe fn vaddq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { - simd_add(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.8"))] +pub unsafe fn vld1_s8(ptr: *const i8) -> int8x8_t { + vld1_v8i8(ptr as *const i8, align_of::() as i32) } -/// Vector long add. +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddl))] -pub unsafe fn vaddl_s8(a: int8x8_t, b: int8x8_t) -> int16x8_t { - let a: int16x8_t = simd_cast(a); - let b: int16x8_t = simd_cast(b); - simd_add(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.8"))] +pub unsafe fn vld1q_s8(ptr: *const i8) -> int8x16_t { + vld1q_v16i8(ptr as *const i8, align_of::() as i32) } -/// Vector long add. +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddl))] -pub unsafe fn vaddl_s16(a: int16x4_t, b: int16x4_t) -> int32x4_t { - let a: int32x4_t = simd_cast(a); - let b: int32x4_t = simd_cast(b); - simd_add(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.16"))] +pub unsafe fn vld1_s16(ptr: *const i16) -> int16x4_t { + vld1_v4i16(ptr as *const i8, align_of::() as i32) } -/// Vector long add. +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddl))] -pub unsafe fn vaddl_s32(a: int32x2_t, b: int32x2_t) -> int64x2_t { - let a: int64x2_t = simd_cast(a); - let b: int64x2_t = simd_cast(b); - simd_add(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.16"))] +pub unsafe fn vld1q_s16(ptr: *const i16) -> int16x8_t { + vld1q_v8i16(ptr as *const i8, align_of::() as i32) } -/// Vector long add. +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddl))] -pub unsafe fn vaddl_u8(a: uint8x8_t, b: uint8x8_t) -> uint16x8_t { - let a: uint16x8_t = simd_cast(a); - let b: uint16x8_t = simd_cast(b); - simd_add(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vldr))] +pub unsafe fn vld1_s32(ptr: *const i32) -> int32x2_t { + vld1_v2i32(ptr as *const i8, align_of::() as i32) } -/// Vector long add. +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddl))] -pub unsafe fn vaddl_u16(a: uint16x4_t, b: uint16x4_t) -> uint32x4_t { - let a: uint32x4_t = simd_cast(a); - let b: uint32x4_t = simd_cast(b); - simd_add(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.32"))] +pub unsafe fn vld1q_s32(ptr: *const i32) -> int32x4_t { + vld1q_v4i32(ptr as *const i8, align_of::() as i32) } -/// Vector long add. +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddl))] -pub unsafe fn vaddl_u32(a: uint32x2_t, b: uint32x2_t) -> uint64x2_t { - let a: uint64x2_t = simd_cast(a); - let b: uint64x2_t = simd_cast(b); - simd_add(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vldr))] +pub unsafe fn vld1_s64(ptr: *const i64) -> int64x1_t { + vld1_v1i64(ptr as *const i8, align_of::() as i32) } -/// Vector narrow integer. +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] -pub unsafe fn vmovn_s16(a: int16x8_t) -> int8x8_t { - simd_cast(a) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.64"))] +pub unsafe fn vld1q_s64(ptr: *const i64) -> int64x2_t { + vld1q_v2i64(ptr as *const i8, align_of::() as i32) } -/// Vector narrow integer. +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] -pub unsafe fn vmovn_s32(a: int32x4_t) -> int16x4_t { - simd_cast(a) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.8"))] +pub unsafe fn vld1_u8(ptr: *const u8) -> uint8x8_t { + transmute(vld1_v8i8(ptr as *const i8, align_of::() as i32)) } -/// Vector narrow integer. +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] -pub unsafe fn vmovn_s64(a: int64x2_t) -> int32x2_t { - simd_cast(a) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.8"))] +pub unsafe fn vld1q_u8(ptr: *const u8) -> uint8x16_t { + transmute(vld1q_v16i8(ptr as *const i8, align_of::() as i32)) } -/// Vector narrow integer. +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] -pub unsafe fn vmovn_u16(a: uint16x8_t) -> uint8x8_t { - simd_cast(a) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.16"))] +pub unsafe fn vld1_u16(ptr: *const u16) -> uint16x4_t { + transmute(vld1_v4i16(ptr as *const i8, align_of::() as i32)) } -/// Vector narrow integer. +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] -pub unsafe fn vmovn_u32(a: uint32x4_t) -> uint16x4_t { - simd_cast(a) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.16"))] +pub unsafe fn vld1q_u16(ptr: *const u16) -> uint16x8_t { + transmute(vld1q_v8i16(ptr as *const i8, align_of::() as i32)) } -/// Vector narrow integer. +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] -pub unsafe fn vmovn_u64(a: uint64x2_t) -> uint32x2_t { - simd_cast(a) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vldr))] +pub unsafe fn vld1_u32(ptr: *const u32) -> uint32x2_t { + transmute(vld1_v2i32(ptr as *const i8, align_of::() as i32)) } -/// Vector long move. +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovl))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sxtl))] -pub unsafe fn vmovl_s8(a: int8x8_t) -> int16x8_t { - simd_cast(a) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.32"))] +pub unsafe fn vld1q_u32(ptr: *const u32) -> uint32x4_t { + transmute(vld1q_v4i32(ptr as *const i8, align_of::() as i32)) } -/// Vector long move. +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovl))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sxtl))] -pub unsafe fn vmovl_s16(a: int16x4_t) -> int32x4_t { - simd_cast(a) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vldr))] +pub unsafe fn vld1_u64(ptr: *const u64) -> uint64x1_t { + transmute(vld1_v1i64(ptr as *const i8, align_of::() as i32)) } -/// Vector long move. +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovl))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sxtl))] -pub unsafe fn vmovl_s32(a: int32x2_t) -> int64x2_t { - simd_cast(a) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.64"))] +pub unsafe fn vld1q_u64(ptr: *const u64) -> uint64x2_t { + transmute(vld1q_v2i64(ptr as *const i8, align_of::() as i32)) } -/// Vector long move. +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovl))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uxtl))] -pub unsafe fn vmovl_u8(a: uint8x8_t) -> uint16x8_t { - simd_cast(a) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.8"))] +pub unsafe fn vld1_p8(ptr: *const p8) -> poly8x8_t { + transmute(vld1_v8i8(ptr as *const i8, align_of::() as i32)) } -/// Vector long move. +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovl))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uxtl))] -pub unsafe fn vmovl_u16(a: uint16x4_t) -> uint32x4_t { - simd_cast(a) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.8"))] +pub unsafe fn vld1q_p8(ptr: *const p8) -> poly8x16_t { + transmute(vld1q_v16i8(ptr as *const i8, align_of::() as i32)) } -/// Vector long move. +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovl))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uxtl))] -pub unsafe fn vmovl_u32(a: uint32x2_t) -> uint64x2_t { - simd_cast(a) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.16"))] +pub unsafe fn vld1_p16(ptr: *const p16) -> poly16x4_t { + transmute(vld1_v4i16(ptr as *const i8, align_of::() as i32)) } -/// Reciprocal square-root estimate. +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(frsqrte))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsqrte))] -pub unsafe fn vrsqrte_f32(a: float32x2_t) -> float32x2_t { - frsqrte_v2f32(a) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.16"))] +pub unsafe fn vld1q_p16(ptr: *const p16) -> poly16x8_t { + transmute(vld1q_v8i16(ptr as *const i8, align_of::() as i32)) } -/// Vector bitwise not. +/// Load multiple single-element structures to one, two, three, or four registers. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p64) #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] -pub unsafe fn vmvn_s8(a: int8x8_t) -> int8x8_t { - let b = int8x8_t(-1, -1, -1, -1, -1, -1, -1, -1); - simd_xor(a, b) +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr(vldr))] +pub unsafe fn vld1_p64(ptr: *const p64) -> poly64x1_t { + transmute(vld1_v1i64(ptr as *const i8, align_of::() as i32)) } -/// Vector bitwise not. +/// Load multiple single-element structures to one, two, three, or four registers. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p64) #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] -pub unsafe fn vmvnq_s8(a: int8x16_t) -> int8x16_t { - let b = int8x16_t( - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - ); - simd_xor(a, b) +#[target_feature(enable = "neon,aes")] +#[cfg_attr(test, assert_instr("vld1.64"))] +pub unsafe fn vld1q_p64(ptr: *const p64) -> poly64x2_t { + transmute(vld1q_v2i64(ptr as *const i8, align_of::() as i32)) } -/// Vector bitwise not. +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] -pub unsafe fn vmvn_s16(a: int16x4_t) -> int16x4_t { - let b = int16x4_t(-1, -1, -1, -1); - simd_xor(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vldr))] +pub unsafe fn vld1_f32(ptr: *const f32) -> float32x2_t { + vld1_v2f32(ptr as *const i8, align_of::() as i32) } -/// Vector bitwise not. +/// Load multiple single-element structures to one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] -pub unsafe fn vmvnq_s16(a: int16x8_t) -> int16x8_t { - let b = int16x8_t(-1, -1, -1, -1, -1, -1, -1, -1); - simd_xor(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.32"))] +pub unsafe fn vld1q_f32(ptr: *const f32) -> float32x4_t { + vld1q_v4f32(ptr as *const i8, align_of::() as i32) } -/// Vector bitwise not. +/// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] -pub unsafe fn vmvn_s32(a: int32x2_t) -> int32x2_t { - let b = int32x2_t(-1, -1); - simd_xor(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.8"))] +pub unsafe fn vst1_s8(ptr: *mut i8, a: int8x8_t) { + vst1_v8i8(ptr as *const i8, a, align_of::() as i32) } -/// Vector bitwise not. +/// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] -pub unsafe fn vmvnq_s32(a: int32x4_t) -> int32x4_t { - let b = int32x4_t(-1, -1, -1, -1); - simd_xor(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.8"))] +pub unsafe fn vst1q_s8(ptr: *mut i8, a: int8x16_t) { + vst1q_v16i8(ptr as *const i8, a, align_of::() as i32) } -/// Vector bitwise not. +/// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] -pub unsafe fn vmvn_u8(a: uint8x8_t) -> uint8x8_t { - let b = uint8x8_t(255, 255, 255, 255, 255, 255, 255, 255); - simd_xor(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.16"))] +pub unsafe fn vst1_s16(ptr: *mut i16, a: int16x4_t) { + vst1_v4i16(ptr as *const i8, a, align_of::() as i32) } -/// Vector bitwise not. +/// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] -pub unsafe fn vmvnq_u8(a: uint8x16_t) -> uint8x16_t { - let b = uint8x16_t( - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ); - simd_xor(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.16"))] +pub unsafe fn vst1q_s16(ptr: *mut i16, a: int16x8_t) { + vst1q_v8i16(ptr as *const i8, a, align_of::() as i32) } -/// Vector bitwise not. +/// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] -pub unsafe fn vmvn_u16(a: uint16x4_t) -> uint16x4_t { - let b = uint16x4_t(65_535, 65_535, 65_535, 65_535); - simd_xor(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.32"))] +pub unsafe fn vst1_s32(ptr: *mut i32, a: int32x2_t) { + vst1_v2i32(ptr as *const i8, a, align_of::() as i32) } -/// Vector bitwise not. +/// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] -pub unsafe fn vmvnq_u16(a: uint16x8_t) -> uint16x8_t { - let b = uint16x8_t( - 65_535, 65_535, 65_535, 65_535, 65_535, 65_535, 65_535, 65_535, - ); - simd_xor(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.32"))] +pub unsafe fn vst1q_s32(ptr: *mut i32, a: int32x4_t) { + vst1q_v4i32(ptr as *const i8, a, align_of::() as i32) } -/// Vector bitwise not. +/// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] -pub unsafe fn vmvn_u32(a: uint32x2_t) -> uint32x2_t { - let b = uint32x2_t(4_294_967_295, 4_294_967_295); - simd_xor(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.64"))] +pub unsafe fn vst1_s64(ptr: *mut i64, a: int64x1_t) { + vst1_v1i64(ptr as *const i8, a, align_of::() as i32) } -/// Vector bitwise not. +/// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] -pub unsafe fn vmvnq_u32(a: uint32x4_t) -> uint32x4_t { - let b = uint32x4_t(4_294_967_295, 4_294_967_295, 4_294_967_295, 4_294_967_295); - simd_xor(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.64"))] +pub unsafe fn vst1q_s64(ptr: *mut i64, a: int64x2_t) { + vst1q_v2i64(ptr as *const i8, a, align_of::() as i32) } -/// Vector bitwise not. +/// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] -pub unsafe fn vmvn_p8(a: poly8x8_t) -> poly8x8_t { - let b = poly8x8_t(255, 255, 255, 255, 255, 255, 255, 255); - simd_xor(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.8"))] +pub unsafe fn vst1_u8(ptr: *mut u8, a: uint8x8_t) { + vst1_v8i8(ptr as *const i8, transmute(a), align_of::() as i32) } -/// Vector bitwise not. +/// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] -pub unsafe fn vmvnq_p8(a: poly8x16_t) -> poly8x16_t { - let b = poly8x16_t( - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ); - simd_xor(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.8"))] +pub unsafe fn vst1q_u8(ptr: *mut u8, a: uint8x16_t) { + vst1q_v16i8(ptr as *const i8, transmute(a), align_of::() as i32) } -/// Folding minimum of adjacent pairs +/// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sminp))] -pub unsafe fn vpmin_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { - vpmins_v8i8(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.16"))] +pub unsafe fn vst1_u16(ptr: *mut u16, a: uint16x4_t) { + vst1_v4i16(ptr as *const i8, transmute(a), align_of::() as i32) } -/// Folding minimum of adjacent pairs +/// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sminp))] -pub unsafe fn vpmin_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { - vpmins_v4i16(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.16"))] +pub unsafe fn vst1q_u16(ptr: *mut u16, a: uint16x8_t) { + vst1q_v8i16(ptr as *const i8, transmute(a), align_of::() as i32) } -/// Folding minimum of adjacent pairs +/// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sminp))] -pub unsafe fn vpmin_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { - vpmins_v2i32(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.32"))] +pub unsafe fn vst1_u32(ptr: *mut u32, a: uint32x2_t) { + vst1_v2i32(ptr as *const i8, transmute(a), align_of::() as i32) } -/// Folding minimum of adjacent pairs +/// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uminp))] -pub unsafe fn vpmin_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - vpminu_v8i8(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.32"))] +pub unsafe fn vst1q_u32(ptr: *mut u32, a: uint32x4_t) { + vst1q_v4i32(ptr as *const i8, transmute(a), align_of::() as i32) } -/// Folding minimum of adjacent pairs +/// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uminp))] -pub unsafe fn vpmin_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - vpminu_v4i16(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.64"))] +pub unsafe fn vst1_u64(ptr: *mut u64, a: uint64x1_t) { + vst1_v1i64(ptr as *const i8, transmute(a), align_of::() as i32) } -/// Folding minimum of adjacent pairs +/// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uminp))] -pub unsafe fn vpmin_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - vpminu_v2i32(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.64"))] +pub unsafe fn vst1q_u64(ptr: *mut u64, a: uint64x2_t) { + vst1q_v2i64(ptr as *const i8, transmute(a), align_of::() as i32) } -/// Folding minimum of adjacent pairs +/// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fminp))] -pub unsafe fn vpmin_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { - vpminf_v2f32(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.8"))] +pub unsafe fn vst1_p8(ptr: *mut p8, a: poly8x8_t) { + vst1_v8i8(ptr as *const i8, transmute(a), align_of::() as i32) } -/// Folding maximum of adjacent pairs +/// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smaxp))] -pub unsafe fn vpmax_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { - vpmaxs_v8i8(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.8"))] +pub unsafe fn vst1q_p8(ptr: *mut p8, a: poly8x16_t) { + vst1q_v16i8(ptr as *const i8, transmute(a), align_of::() as i32) } -/// Folding maximum of adjacent pairs +/// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smaxp))] -pub unsafe fn vpmax_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { - vpmaxs_v4i16(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.16"))] +pub unsafe fn vst1_p16(ptr: *mut p16, a: poly16x4_t) { + vst1_v4i16(ptr as *const i8, transmute(a), align_of::() as i32) } -/// Folding maximum of adjacent pairs +/// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smaxp))] -pub unsafe fn vpmax_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { - vpmaxs_v2i32(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.16"))] +pub unsafe fn vst1q_p16(ptr: *mut p16, a: poly16x8_t) { + vst1q_v8i16(ptr as *const i8, transmute(a), align_of::() as i32) } -/// Folding maximum of adjacent pairs +/// Store multiple single-element structures from one, two, three, or four registers. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p64) #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umaxp))] -pub unsafe fn vpmax_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - vpmaxu_v8i8(a, b) +#[target_feature(enable = "neon,aes,v8")] +#[cfg_attr(test, assert_instr("vst1.64"))] +pub unsafe fn vst1_p64(ptr: *mut p64, a: poly64x1_t) { + vst1_v1i64(ptr as *const i8, transmute(a), align_of::() as i32) } -/// Folding maximum of adjacent pairs +/// Store multiple single-element structures from one, two, three, or four registers. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p64) #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umaxp))] -pub unsafe fn vpmax_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - vpmaxu_v4i16(a, b) +#[target_feature(enable = "neon,aes,v8")] +#[cfg_attr(test, assert_instr("vst1.64"))] +pub unsafe fn vst1q_p64(ptr: *mut p64, a: poly64x2_t) { + vst1q_v2i64(ptr as *const i8, transmute(a), align_of::() as i32) } -/// Folding maximum of adjacent pairs +// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umaxp))] -pub unsafe fn vpmax_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - vpmaxu_v2i32(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.32"))] +pub unsafe fn vst1_f32(ptr: *mut f32, a: float32x2_t) { + vst1_v2f32(ptr as *const i8, a, align_of::() as i32) } -/// Folding maximum of adjacent pairs +// Store multiple single-element structures from one, two, three, or four registers. #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmaxp))] -pub unsafe fn vpmax_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { - vpmaxf_v2f32(a, b) +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vst1.32"))] +pub unsafe fn vst1q_f32(ptr: *mut f32, a: float32x4_t) { + vst1q_v4f32(ptr as *const i8, a, align_of::() as i32) } /// Table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbl))] @@ -852,7 +531,6 @@ pub unsafe fn vtbl1_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { /// Table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbl))] @@ -862,7 +540,6 @@ pub unsafe fn vtbl1_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { /// Table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbl))] @@ -872,7 +549,6 @@ pub unsafe fn vtbl1_p8(a: poly8x8_t, b: uint8x8_t) -> poly8x8_t { /// Table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbl))] @@ -882,7 +558,6 @@ pub unsafe fn vtbl2_s8(a: int8x8x2_t, b: int8x8_t) -> int8x8_t { /// Table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbl))] @@ -892,7 +567,6 @@ pub unsafe fn vtbl2_u8(a: uint8x8x2_t, b: uint8x8_t) -> uint8x8_t { /// Table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbl))] @@ -902,7 +576,6 @@ pub unsafe fn vtbl2_p8(a: poly8x8x2_t, b: uint8x8_t) -> poly8x8_t { /// Table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbl))] @@ -912,7 +585,6 @@ pub unsafe fn vtbl3_s8(a: int8x8x3_t, b: int8x8_t) -> int8x8_t { /// Table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbl))] @@ -927,7 +599,6 @@ pub unsafe fn vtbl3_u8(a: uint8x8x3_t, b: uint8x8_t) -> uint8x8_t { /// Table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbl))] @@ -942,7 +613,6 @@ pub unsafe fn vtbl3_p8(a: poly8x8x3_t, b: uint8x8_t) -> poly8x8_t { /// Table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbl))] @@ -952,7 +622,6 @@ pub unsafe fn vtbl4_s8(a: int8x8x4_t, b: int8x8_t) -> int8x8_t { /// Table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbl))] @@ -968,7 +637,6 @@ pub unsafe fn vtbl4_u8(a: uint8x8x4_t, b: uint8x8_t) -> uint8x8_t { /// Table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbl))] @@ -984,7 +652,6 @@ pub unsafe fn vtbl4_p8(a: poly8x8x4_t, b: uint8x8_t) -> poly8x8_t { /// Extended table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbx))] @@ -994,7 +661,6 @@ pub unsafe fn vtbx1_s8(a: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t { /// Extended table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbx))] @@ -1004,7 +670,6 @@ pub unsafe fn vtbx1_u8(a: uint8x8_t, b: uint8x8_t, c: uint8x8_t) -> uint8x8_t { /// Extended table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbx))] @@ -1014,7 +679,6 @@ pub unsafe fn vtbx1_p8(a: poly8x8_t, b: poly8x8_t, c: uint8x8_t) -> poly8x8_t { /// Extended table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbx))] @@ -1024,7 +688,6 @@ pub unsafe fn vtbx2_s8(a: int8x8_t, b: int8x8x2_t, c: int8x8_t) -> int8x8_t { /// Extended table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbx))] @@ -1039,7 +702,6 @@ pub unsafe fn vtbx2_u8(a: uint8x8_t, b: uint8x8x2_t, c: uint8x8_t) -> uint8x8_t /// Extended table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbx))] @@ -1054,7 +716,6 @@ pub unsafe fn vtbx2_p8(a: poly8x8_t, b: poly8x8x2_t, c: uint8x8_t) -> poly8x8_t /// Extended table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbx))] @@ -1064,7 +725,6 @@ pub unsafe fn vtbx3_s8(a: int8x8_t, b: int8x8x3_t, c: int8x8_t) -> int8x8_t { /// Extended table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbx))] @@ -1080,7 +740,6 @@ pub unsafe fn vtbx3_u8(a: uint8x8_t, b: uint8x8x3_t, c: uint8x8_t) -> uint8x8_t /// Extended table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbx))] @@ -1096,7 +755,6 @@ pub unsafe fn vtbx3_p8(a: poly8x8_t, b: poly8x8x3_t, c: uint8x8_t) -> poly8x8_t /// Extended table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbx))] @@ -1106,7 +764,6 @@ pub unsafe fn vtbx4_s8(a: int8x8_t, b: int8x8x4_t, c: int8x8_t) -> int8x8_t { /// Extended table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbx))] @@ -1123,7 +780,6 @@ pub unsafe fn vtbx4_u8(a: uint8x8_t, b: uint8x8x4_t, c: uint8x8_t) -> uint8x8_t /// Extended table look-up #[inline] -#[cfg(target_arch = "arm")] #[cfg(target_endian = "little")] #[target_feature(enable = "neon,v7")] #[cfg_attr(test, assert_instr(vtbx))] @@ -1138,550 +794,597 @@ pub unsafe fn vtbx4_p8(a: poly8x8_t, b: poly8x8x4_t, c: uint8x8_t) -> poly8x8_t )) } -#[cfg(test)] -mod tests { - use crate::core_arch::{arm::*, simd::*}; - use std::{i16, i32, i8, mem::transmute, u16, u32, u8}; - use stdarch_test::simd_test; - - #[simd_test(enable = "neon")] - unsafe fn test_vadd_s8() { - let a = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b = i8x8::new(8, 7, 6, 5, 4, 3, 2, 1); - let e = i8x8::new(9, 9, 9, 9, 9, 9, 9, 9); - let r: i8x8 = transmute(vadd_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vaddq_s8() { - let a = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); - let b = i8x16::new(8, 7, 6, 5, 4, 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, 1); - let e = i8x16::new(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9); - let r: i8x16 = transmute(vaddq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vadd_s16() { - let a = i16x4::new(1, 2, 3, 4); - let b = i16x4::new(8, 7, 6, 5); - let e = i16x4::new(9, 9, 9, 9); - let r: i16x4 = transmute(vadd_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vaddq_s16() { - let a = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b = i16x8::new(8, 7, 6, 5, 4, 3, 2, 1); - let e = i16x8::new(9, 9, 9, 9, 9, 9, 9, 9); - let r: i16x8 = transmute(vaddq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vadd_s32() { - let a = i32x2::new(1, 2); - let b = i32x2::new(8, 7); - let e = i32x2::new(9, 9); - let r: i32x2 = transmute(vadd_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vaddq_s32() { - let a = i32x4::new(1, 2, 3, 4); - let b = i32x4::new(8, 7, 6, 5); - let e = i32x4::new(9, 9, 9, 9); - let r: i32x4 = transmute(vaddq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vadd_u8() { - let a = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b = u8x8::new(8, 7, 6, 5, 4, 3, 2, 1); - let e = u8x8::new(9, 9, 9, 9, 9, 9, 9, 9); - let r: u8x8 = transmute(vadd_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vaddq_u8() { - let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8); - let b = u8x16::new(8, 7, 6, 5, 4, 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, 1); - let e = u8x16::new(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9); - let r: u8x16 = transmute(vaddq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vadd_u16() { - let a = u16x4::new(1, 2, 3, 4); - let b = u16x4::new(8, 7, 6, 5); - let e = u16x4::new(9, 9, 9, 9); - let r: u16x4 = transmute(vadd_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vaddq_u16() { - let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b = u16x8::new(8, 7, 6, 5, 4, 3, 2, 1); - let e = u16x8::new(9, 9, 9, 9, 9, 9, 9, 9); - let r: u16x8 = transmute(vaddq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vadd_u32() { - let a = u32x2::new(1, 2); - let b = u32x2::new(8, 7); - let e = u32x2::new(9, 9); - let r: u32x2 = transmute(vadd_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vaddq_u32() { - let a = u32x4::new(1, 2, 3, 4); - let b = u32x4::new(8, 7, 6, 5); - let e = u32x4::new(9, 9, 9, 9); - let r: u32x4 = transmute(vaddq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vadd_f32() { - let a = f32x2::new(1., 2.); - let b = f32x2::new(8., 7.); - let e = f32x2::new(9., 9.); - let r: f32x2 = transmute(vadd_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vaddq_f32() { - let a = f32x4::new(1., 2., 3., 4.); - let b = f32x4::new(8., 7., 6., 5.); - let e = f32x4::new(9., 9., 9., 9.); - let r: f32x4 = transmute(vaddq_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vaddl_s8() { - let v = i8::MAX; - let a = i8x8::new(v, v, v, v, v, v, v, v); - let v = 2 * (v as i16); - let e = i16x8::new(v, v, v, v, v, v, v, v); - let r: i16x8 = transmute(vaddl_s8(transmute(a), transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vaddl_s16() { - let v = i16::MAX; - let a = i16x4::new(v, v, v, v); - let v = 2 * (v as i32); - let e = i32x4::new(v, v, v, v); - let r: i32x4 = transmute(vaddl_s16(transmute(a), transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vaddl_s32() { - let v = i32::MAX; - let a = i32x2::new(v, v); - let v = 2 * (v as i64); - let e = i64x2::new(v, v); - let r: i64x2 = transmute(vaddl_s32(transmute(a), transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vaddl_u8() { - let v = u8::MAX; - let a = u8x8::new(v, v, v, v, v, v, v, v); - let v = 2 * (v as u16); - let e = u16x8::new(v, v, v, v, v, v, v, v); - let r: u16x8 = transmute(vaddl_u8(transmute(a), transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vaddl_u16() { - let v = u16::MAX; - let a = u16x4::new(v, v, v, v); - let v = 2 * (v as u32); - let e = u32x4::new(v, v, v, v); - let r: u32x4 = transmute(vaddl_u16(transmute(a), transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vaddl_u32() { - let v = u32::MAX; - let a = u32x2::new(v, v); - let v = 2 * (v as u64); - let e = u64x2::new(v, v); - let r: u64x2 = transmute(vaddl_u32(transmute(a), transmute(a))); - assert_eq!(r, e); - } +// These float-to-int implementations have undefined behaviour when `a` overflows +// the destination type. Clang has the same problem: https://llvm.org/PR47510 - #[simd_test(enable = "neon")] - unsafe fn test_vmvn_s8() { - let a = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let e = i8x8::new(-1, -2, -3, -4, -5, -6, -7, -8); - let r: i8x8 = transmute(vmvn_s8(transmute(a))); - assert_eq!(r, e); - } +/// Floating-point Convert to Signed fixed-point, rounding toward Zero (vector) +#[inline] +#[target_feature(enable = "neon")] +#[target_feature(enable = "v7")] +#[cfg_attr(test, assert_instr("vcvt.s32.f32"))] +pub unsafe fn vcvtq_s32_f32(a: float32x4_t) -> int32x4_t { + transmute(simd_cast::<_, i32x4>(transmute::<_, f32x4>(a))) +} - #[simd_test(enable = "neon")] - unsafe fn test_vmvnq_s8() { - let a = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let e = i8x16::new( - -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, - ); - let r: i8x16 = transmute(vmvnq_s8(transmute(a))); - assert_eq!(r, e); - } +/// Floating-point Convert to Unsigned fixed-point, rounding toward Zero (vector) +#[inline] +#[target_feature(enable = "neon")] +#[target_feature(enable = "v7")] +#[cfg_attr(test, assert_instr("vcvt.u32.f32"))] +pub unsafe fn vcvtq_u32_f32(a: float32x4_t) -> uint32x4_t { + transmute(simd_cast::<_, u32x4>(transmute::<_, f32x4>(a))) +} - #[simd_test(enable = "neon")] - unsafe fn test_vmvn_s16() { - let a = i16x4::new(0, 1, 2, 3); - let e = i16x4::new(-1, -2, -3, -4); - let r: i16x4 = transmute(vmvn_s16(transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmvnq_s16() { - let a = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let e = i16x8::new(-1, -2, -3, -4, -5, -6, -7, -8); - let r: i16x8 = transmute(vmvnq_s16(transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmvn_s32() { - let a = i32x2::new(0, 1); - let e = i32x2::new(-1, -2); - let r: i32x2 = transmute(vmvn_s32(transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmvnq_s32() { - let a = i32x4::new(0, 1, 2, 3); - let e = i32x4::new(-1, -2, -3, -4); - let r: i32x4 = transmute(vmvnq_s32(transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmvn_u8() { - let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let e = u8x8::new(255, 254, 253, 252, 251, 250, 249, 248); - let r: u8x8 = transmute(vmvn_u8(transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmvnq_u8() { - let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let e = u8x16::new( - 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, - ); - let r: u8x16 = transmute(vmvnq_u8(transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmvn_u16() { - let a = u16x4::new(0, 1, 2, 3); - let e = u16x4::new(65_535, 65_534, 65_533, 65_532); - let r: u16x4 = transmute(vmvn_u16(transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmvnq_u16() { - let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let e = u16x8::new( - 65_535, 65_534, 65_533, 65_532, 65_531, 65_530, 65_529, 65_528, - ); - let r: u16x8 = transmute(vmvnq_u16(transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmvn_u32() { - let a = u32x2::new(0, 1); - let e = u32x2::new(4_294_967_295, 4_294_967_294); - let r: u32x2 = transmute(vmvn_u32(transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmvnq_u32() { - let a = u32x4::new(0, 1, 2, 3); - let e = u32x4::new(4_294_967_295, 4_294_967_294, 4_294_967_293, 4_294_967_292); - let r: u32x4 = transmute(vmvnq_u32(transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmvn_p8() { - let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let e = u8x8::new(255, 254, 253, 252, 251, 250, 249, 248); - let r: u8x8 = transmute(vmvn_p8(transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmvnq_p8() { - let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let e = u8x16::new( - 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, - ); - let r: u8x16 = transmute(vmvnq_p8(transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmovn_s16() { - let a = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r: i8x8 = transmute(vmovn_s16(transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmovn_s32() { - let a = i32x4::new(1, 2, 3, 4); - let e = i16x4::new(1, 2, 3, 4); - let r: i16x4 = transmute(vmovn_s32(transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmovn_s64() { - let a = i64x2::new(1, 2); - let e = i32x2::new(1, 2); - let r: i32x2 = transmute(vmovn_s64(transmute(a))); - assert_eq!(r, e); - } +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + static_assert_imm3!(N); + let n = N as i8; + vshiftins_v8i8(a, b, int8x8_t(n, n, n, n, n, n, n, n)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + static_assert_imm3!(N); + let n = N as i8; + vshiftins_v16i8( + a, + b, + int8x16_t(n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n), + ) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + static_assert_imm4!(N); + let n = N as i16; + vshiftins_v4i16(a, b, int16x4_t(n, n, n, n)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + static_assert_imm4!(N); + let n = N as i16; + vshiftins_v8i16(a, b, int16x8_t(n, n, n, n, n, n, n, n)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.32", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + static_assert!(N: i32 where N >= 0 && N <= 31); + vshiftins_v2i32(a, b, int32x2_t(N, N)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.32", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + static_assert!(N: i32 where N >= 0 && N <= 31); + vshiftins_v4i32(a, b, int32x4_t(N, N, N, N)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.64", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + static_assert!(N : i32 where 0 <= N && N <= 63); + vshiftins_v1i64(a, b, int64x1_t(N as i64)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.64", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + static_assert!(N : i32 where 0 <= N && N <= 63); + vshiftins_v2i64(a, b, int64x2_t(N as i64, N as i64)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + static_assert_imm3!(N); + let n = N as i8; + transmute(vshiftins_v8i8( + transmute(a), + transmute(b), + int8x8_t(n, n, n, n, n, n, n, n), + )) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + static_assert_imm3!(N); + let n = N as i8; + transmute(vshiftins_v16i8( + transmute(a), + transmute(b), + int8x16_t(n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n), + )) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + static_assert_imm4!(N); + let n = N as i16; + transmute(vshiftins_v4i16( + transmute(a), + transmute(b), + int16x4_t(n, n, n, n), + )) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + static_assert_imm4!(N); + let n = N as i16; + transmute(vshiftins_v8i16( + transmute(a), + transmute(b), + int16x8_t(n, n, n, n, n, n, n, n), + )) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.32", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + static_assert!(N: i32 where N >= 0 && N <= 31); + transmute(vshiftins_v2i32(transmute(a), transmute(b), int32x2_t(N, N))) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.32", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + static_assert!(N: i32 where N >= 0 && N <= 31); + transmute(vshiftins_v4i32( + transmute(a), + transmute(b), + int32x4_t(N, N, N, N), + )) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.64", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + static_assert!(N : i32 where 0 <= N && N <= 63); + transmute(vshiftins_v1i64( + transmute(a), + transmute(b), + int64x1_t(N as i64), + )) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.64", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + static_assert!(N : i32 where 0 <= N && N <= 63); + transmute(vshiftins_v2i64( + transmute(a), + transmute(b), + int64x2_t(N as i64, N as i64), + )) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + static_assert_imm3!(N); + let n = N as i8; + transmute(vshiftins_v8i8( + transmute(a), + transmute(b), + int8x8_t(n, n, n, n, n, n, n, n), + )) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + static_assert_imm3!(N); + let n = N as i8; + transmute(vshiftins_v16i8( + transmute(a), + transmute(b), + int8x16_t(n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n), + )) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + static_assert_imm4!(N); + let n = N as i16; + transmute(vshiftins_v4i16( + transmute(a), + transmute(b), + int16x4_t(n, n, n, n), + )) +} - #[simd_test(enable = "neon")] - unsafe fn test_vmovn_u16() { - let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r: u8x8 = transmute(vmovn_u16(transmute(a))); - assert_eq!(r, e); - } +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + static_assert_imm4!(N); + let n = N as i16; + transmute(vshiftins_v8i16( + transmute(a), + transmute(b), + int16x8_t(n, n, n, n, n, n, n, n), + )) +} - #[simd_test(enable = "neon")] - unsafe fn test_vmovn_u32() { - let a = u32x4::new(1, 2, 3, 4); - let e = u16x4::new(1, 2, 3, 4); - let r: u16x4 = transmute(vmovn_u32(transmute(a))); - assert_eq!(r, e); - } +/// Shift Left and Insert (immediate) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_p64) +#[inline] +#[target_feature(enable = "neon,v7,aes")] +#[cfg_attr(test, assert_instr("vsli.64", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_p64(a: poly64x1_t, b: poly64x1_t) -> poly64x1_t { + static_assert!(N : i32 where 0 <= N && N <= 63); + transmute(vshiftins_v1i64( + transmute(a), + transmute(b), + int64x1_t(N as i64), + )) +} - #[simd_test(enable = "neon")] - unsafe fn test_vmovn_u64() { - let a = u64x2::new(1, 2); - let e = u32x2::new(1, 2); - let r: u32x2 = transmute(vmovn_u64(transmute(a))); - assert_eq!(r, e); - } +/// Shift Left and Insert (immediate) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_p64) +#[inline] +#[target_feature(enable = "neon,v7,aes")] +#[cfg_attr(test, assert_instr("vsli.64", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t { + static_assert!(N : i32 where 0 <= N && N <= 63); + transmute(vshiftins_v2i64( + transmute(a), + transmute(b), + int64x2_t(N as i64, N as i64), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + static_assert!(N : i32 where 1 <= N && N <= 8); + let n = -N as i8; + vshiftins_v8i8(a, b, int8x8_t(n, n, n, n, n, n, n, n)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + static_assert!(N : i32 where 1 <= N && N <= 8); + let n = -N as i8; + vshiftins_v16i8( + a, + b, + int8x16_t(n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n), + ) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + static_assert!(N : i32 where 1 <= N && N <= 16); + let n = -N as i16; + vshiftins_v4i16(a, b, int16x4_t(n, n, n, n)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + static_assert!(N : i32 where 1 <= N && N <= 16); + let n = -N as i16; + vshiftins_v8i16(a, b, int16x8_t(n, n, n, n, n, n, n, n)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.32", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + static_assert!(N : i32 where 1 <= N && N <= 32); + vshiftins_v2i32(a, b, int32x2_t(-N, -N)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.32", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + static_assert!(N : i32 where 1 <= N && N <= 32); + vshiftins_v4i32(a, b, int32x4_t(-N, -N, -N, -N)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.64", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + static_assert!(N : i32 where 1 <= N && N <= 64); + vshiftins_v1i64(a, b, int64x1_t(-N as i64)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.64", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + static_assert!(N : i32 where 1 <= N && N <= 64); + vshiftins_v2i64(a, b, int64x2_t(-N as i64, -N as i64)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + static_assert!(N : i32 where 1 <= N && N <= 8); + let n = -N as i8; + transmute(vshiftins_v8i8( + transmute(a), + transmute(b), + int8x8_t(n, n, n, n, n, n, n, n), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + static_assert!(N : i32 where 1 <= N && N <= 8); + let n = -N as i8; + transmute(vshiftins_v16i8( + transmute(a), + transmute(b), + int8x16_t(n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + static_assert!(N : i32 where 1 <= N && N <= 16); + let n = -N as i16; + transmute(vshiftins_v4i16( + transmute(a), + transmute(b), + int16x4_t(n, n, n, n), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + static_assert!(N : i32 where 1 <= N && N <= 16); + let n = -N as i16; + transmute(vshiftins_v8i16( + transmute(a), + transmute(b), + int16x8_t(n, n, n, n, n, n, n, n), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.32", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + static_assert!(N : i32 where 1 <= N && N <= 32); + transmute(vshiftins_v2i32( + transmute(a), + transmute(b), + int32x2_t(-N, -N), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.32", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + static_assert!(N : i32 where 1 <= N && N <= 32); + transmute(vshiftins_v4i32( + transmute(a), + transmute(b), + int32x4_t(-N, -N, -N, -N), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.64", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + static_assert!(N : i32 where 1 <= N && N <= 64); + transmute(vshiftins_v1i64( + transmute(a), + transmute(b), + int64x1_t(-N as i64), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.64", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + static_assert!(N : i32 where 1 <= N && N <= 64); + transmute(vshiftins_v2i64( + transmute(a), + transmute(b), + int64x2_t(-N as i64, -N as i64), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + static_assert!(N : i32 where 1 <= N && N <= 8); + let n = -N as i8; + transmute(vshiftins_v8i8( + transmute(a), + transmute(b), + int8x8_t(n, n, n, n, n, n, n, n), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + static_assert!(N : i32 where 1 <= N && N <= 8); + let n = -N as i8; + transmute(vshiftins_v16i8( + transmute(a), + transmute(b), + int8x16_t(n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + static_assert!(N : i32 where 1 <= N && N <= 16); + let n = -N as i16; + transmute(vshiftins_v4i16( + transmute(a), + transmute(b), + int16x4_t(n, n, n, n), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + static_assert!(N : i32 where 1 <= N && N <= 16); + let n = -N as i16; + transmute(vshiftins_v8i16( + transmute(a), + transmute(b), + int16x8_t(n, n, n, n, n, n, n, n), + )) +} - #[simd_test(enable = "neon")] - unsafe fn test_vmovl_s8() { - let e = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let a = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r: i16x8 = transmute(vmovl_s8(transmute(a))); - assert_eq!(r, e); - } +/// Shift Right and Insert (immediate) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_p64) +#[inline] +#[target_feature(enable = "neon,v7,aes")] +#[cfg_attr(test, assert_instr("vsri.64", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_p64(a: poly64x1_t, b: poly64x1_t) -> poly64x1_t { + static_assert!(N : i32 where 1 <= N && N <= 64); + transmute(vshiftins_v1i64( + transmute(a), + transmute(b), + int64x1_t(-N as i64), + )) +} - #[simd_test(enable = "neon")] - unsafe fn test_vmovl_s16() { - let e = i32x4::new(1, 2, 3, 4); - let a = i16x4::new(1, 2, 3, 4); - let r: i32x4 = transmute(vmovl_s16(transmute(a))); - assert_eq!(r, e); - } +/// Shift Right and Insert (immediate) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_p64) +#[inline] +#[target_feature(enable = "neon,v7,aes")] +#[cfg_attr(test, assert_instr("vsri.64", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t { + static_assert!(N : i32 where 1 <= N && N <= 64); + transmute(vshiftins_v2i64( + transmute(a), + transmute(b), + int64x2_t(-N as i64, -N as i64), + )) +} - #[simd_test(enable = "neon")] - unsafe fn test_vmovl_s32() { - let e = i64x2::new(1, 2); - let a = i32x2::new(1, 2); - let r: i64x2 = transmute(vmovl_s32(transmute(a))); - assert_eq!(r, e); - } +#[cfg(test)] +mod tests { + use super::*; + use crate::core_arch::{arm::*, simd::*}; + use crate::mem::transmute; + use stdarch_test::simd_test; #[simd_test(enable = "neon")] - unsafe fn test_vmovl_u8() { - let e = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let a = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r: u16x8 = transmute(vmovl_u8(transmute(a))); + unsafe fn test_vcvtq_s32_f32() { + let f = f32x4::new(-1., 2., 3., 4.); + let e = i32x4::new(-1, 2, 3, 4); + let r: i32x4 = transmute(vcvtq_s32_f32(transmute(f))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovl_u16() { + unsafe fn test_vcvtq_u32_f32() { + let f = f32x4::new(1., 2., 3., 4.); let e = u32x4::new(1, 2, 3, 4); - let a = u16x4::new(1, 2, 3, 4); - let r: u32x4 = transmute(vmovl_u16(transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmovl_u32() { - let e = u64x2::new(1, 2); - let a = u32x2::new(1, 2); - let r: u64x2 = transmute(vmovl_u32(transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vrsqrt_f32() { - let a = f32x2::new(1.0, 2.0); - let e = f32x2::new(0.9980469, 0.7050781); - let r: f32x2 = transmute(vrsqrte_f32(transmute(a))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmin_s8() { - let a = i8x8::new(1, -2, 3, -4, 5, 6, 7, 8); - let b = i8x8::new(0, 3, 2, 5, 4, 7, 6, 9); - let e = i8x8::new(-2, -4, 5, 7, 0, 2, 4, 6); - let r: i8x8 = transmute(vpmin_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmin_s16() { - let a = i16x4::new(1, 2, 3, -4); - let b = i16x4::new(0, 3, 2, 5); - let e = i16x4::new(1, -4, 0, 2); - let r: i16x4 = transmute(vpmin_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmin_s32() { - let a = i32x2::new(1, -2); - let b = i32x2::new(0, 3); - let e = i32x2::new(-2, 0); - let r: i32x2 = transmute(vpmin_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmin_u8() { - let a = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b = u8x8::new(0, 3, 2, 5, 4, 7, 6, 9); - let e = u8x8::new(1, 3, 5, 7, 0, 2, 4, 6); - let r: u8x8 = transmute(vpmin_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmin_u16() { - let a = u16x4::new(1, 2, 3, 4); - let b = u16x4::new(0, 3, 2, 5); - let e = u16x4::new(1, 3, 0, 2); - let r: u16x4 = transmute(vpmin_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmin_u32() { - let a = u32x2::new(1, 2); - let b = u32x2::new(0, 3); - let e = u32x2::new(1, 0); - let r: u32x2 = transmute(vpmin_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmin_f32() { - let a = f32x2::new(1., -2.); - let b = f32x2::new(0., 3.); - let e = f32x2::new(-2., 0.); - let r: f32x2 = transmute(vpmin_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmax_s8() { - let a = i8x8::new(1, -2, 3, -4, 5, 6, 7, 8); - let b = i8x8::new(0, 3, 2, 5, 4, 7, 6, 9); - let e = i8x8::new(1, 3, 6, 8, 3, 5, 7, 9); - let r: i8x8 = transmute(vpmax_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmax_s16() { - let a = i16x4::new(1, 2, 3, -4); - let b = i16x4::new(0, 3, 2, 5); - let e = i16x4::new(2, 3, 3, 5); - let r: i16x4 = transmute(vpmax_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmax_s32() { - let a = i32x2::new(1, -2); - let b = i32x2::new(0, 3); - let e = i32x2::new(1, 3); - let r: i32x2 = transmute(vpmax_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmax_u8() { - let a = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b = u8x8::new(0, 3, 2, 5, 4, 7, 6, 9); - let e = u8x8::new(2, 4, 6, 8, 3, 5, 7, 9); - let r: u8x8 = transmute(vpmax_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmax_u16() { - let a = u16x4::new(1, 2, 3, 4); - let b = u16x4::new(0, 3, 2, 5); - let e = u16x4::new(2, 4, 3, 5); - let r: u16x4 = transmute(vpmax_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmax_u32() { - let a = u32x2::new(1, 2); - let b = u32x2::new(0, 3); - let e = u32x2::new(2, 3); - let r: u32x2 = transmute(vpmax_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vpmax_f32() { - let a = f32x2::new(1., -2.); - let b = f32x2::new(0., 3.); - let e = f32x2::new(1., 3.); - let r: f32x2 = transmute(vpmax_f32(transmute(a), transmute(b))); + let r: u32x4 = transmute(vcvtq_u32_f32(transmute(f))); assert_eq!(r, e); } } - -#[cfg(test)] -#[cfg(target_endian = "little")] -#[path = "table_lookup_tests.rs"] -mod table_lookup_tests; diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/acle/sat.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm/sat.rs similarity index 100% rename from crux-mir/lib/stdarch/crates/core_arch/src/acle/sat.rs rename to crux-mir/lib/stdarch/crates/core_arch/src/arm/sat.rs diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/acle/simd32.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm/simd32.rs similarity index 99% rename from crux-mir/lib/stdarch/crates/core_arch/src/acle/simd32.rs rename to crux-mir/lib/stdarch/crates/core_arch/src/arm/simd32.rs index 678324f10..2d867acc8 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/acle/simd32.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/arm/simd32.rs @@ -65,7 +65,7 @@ #[cfg(test)] use stdarch_test::assert_instr; -use crate::{core_arch::acle::dsp::int16x2_t, mem::transmute}; +use crate::{core_arch::arm::dsp::int16x2_t, mem::transmute}; types! { /// ARM-specific 32-bit wide vector of four packed `i8`. @@ -80,7 +80,7 @@ macro_rules! dsp_call { }; } -extern "C" { +extern "unadjusted" { #[link_name = "llvm.arm.qadd8"] fn arm_qadd8(a: i32, b: i32) -> i32; @@ -464,7 +464,7 @@ pub unsafe fn __usada8(a: int8x4_t, b: int8x4_t, c: u32) -> u32 { #[cfg(test)] mod tests { use crate::core_arch::simd::{i16x2, i8x4, u8x4}; - use std::{i16, i8, mem::transmute}; + use std::mem::transmute; use stdarch_test::simd_test; #[test] diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/arm/v7.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm/v7.rs index e7507f9b9..59beaf722 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/arm/v7.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/arm/v7.rs @@ -76,7 +76,6 @@ mod tests { } #[test] - #[cfg(dont_compile_me)] // FIXME need to add `v7` upstream in rustc fn _rbit_u32() { unsafe { assert_eq!( diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/acle/barrier/common.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/barrier/common.rs similarity index 100% rename from crux-mir/lib/stdarch/crates/core_arch/src/acle/barrier/common.rs rename to crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/barrier/common.rs diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/acle/barrier/cp15.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/barrier/cp15.rs similarity index 60% rename from crux-mir/lib/stdarch/crates/core_arch/src/acle/barrier/cp15.rs rename to crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/barrier/cp15.rs index 7938acbbb..6faae0fee 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/acle/barrier/cp15.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/barrier/cp15.rs @@ -1,6 +1,8 @@ // Reference: ARM11 MPCore Processor Technical Reference Manual (ARM DDI 0360E) Section 3.5 "Summary // of CP15 instructions" +use crate::arch::asm; + /// Full system is the required shareability domain, reads and writes are the /// required access types pub struct SY; @@ -8,20 +10,29 @@ pub struct SY; impl super::super::sealed::Dmb for SY { #[inline(always)] unsafe fn __dmb(&self) { - asm!("mcr p15, 0, r0, c7, c10, 5" : : : "memory" : "volatile") + asm!( + "mcr p15, 0, r0, c7, c10, 5", + options(preserves_flags, nostack) + ) } } impl super::super::sealed::Dsb for SY { #[inline(always)] unsafe fn __dsb(&self) { - asm!("mcr p15, 0, r0, c7, c10, 4" : : : "memory" : "volatile") + asm!( + "mcr p15, 0, r0, c7, c10, 4", + options(preserves_flags, nostack) + ) } } impl super::super::sealed::Isb for SY { #[inline(always)] unsafe fn __isb(&self) { - asm!("mcr p15, 0, r0, c7, c5, 4" : : : "memory" : "volatile") + asm!( + "mcr p15, 0, r0, c7, c5, 4", + options(preserves_flags, nostack) + ) } } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/acle/barrier/mod.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/barrier/mod.rs similarity index 99% rename from crux-mir/lib/stdarch/crates/core_arch/src/acle/barrier/mod.rs rename to crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/barrier/mod.rs index b3cbf44d2..6ccced00e 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/acle/barrier/mod.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/barrier/mod.rs @@ -122,7 +122,7 @@ where arg.__isb() } -extern "C" { +extern "unadjusted" { #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.dmb")] #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.dmb")] fn dmb(_: i32); diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/acle/barrier/not_mclass.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/barrier/not_mclass.rs similarity index 100% rename from crux-mir/lib/stdarch/crates/core_arch/src/acle/barrier/not_mclass.rs rename to crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/barrier/not_mclass.rs diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/acle/barrier/v8.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/barrier/v8.rs similarity index 84% rename from crux-mir/lib/stdarch/crates/core_arch/src/acle/barrier/v8.rs rename to crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/barrier/v8.rs index 2951a5a67..db15da805 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/acle/barrier/v8.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/barrier/v8.rs @@ -16,7 +16,7 @@ pub struct NSHLD; dmb_dsb!(NSHLD); -/// Outher Shareable is the required shareability domain, reads are the required +/// Outer Shareable is the required shareability domain, reads are the required /// access type pub struct OSHLD; diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/crc.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/crc.rs new file mode 100644 index 000000000..779d1ed42 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/crc.rs @@ -0,0 +1,133 @@ +extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crc32b")] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.crc32b")] + fn crc32b_(crc: u32, data: u32) -> u32; + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crc32h")] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.crc32h")] + fn crc32h_(crc: u32, data: u32) -> u32; + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crc32w")] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.crc32w")] + fn crc32w_(crc: u32, data: u32) -> u32; + + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crc32cb")] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.crc32cb")] + fn crc32cb_(crc: u32, data: u32) -> u32; + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crc32ch")] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.crc32ch")] + fn crc32ch_(crc: u32, data: u32) -> u32; + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crc32cw")] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.crc32cw")] + fn crc32cw_(crc: u32, data: u32) -> u32; +} + +#[cfg(test)] +use stdarch_test::assert_instr; + +/// CRC32 single round checksum for bytes (8 bits). +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/__crc32b) +#[inline] +#[target_feature(enable = "crc")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr(test, assert_instr(crc32b))] +pub unsafe fn __crc32b(crc: u32, data: u8) -> u32 { + crc32b_(crc, data as u32) +} + +/// CRC32 single round checksum for half words (16 bits). +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/__crc32h) +#[inline] +#[target_feature(enable = "crc")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr(test, assert_instr(crc32h))] +pub unsafe fn __crc32h(crc: u32, data: u16) -> u32 { + crc32h_(crc, data as u32) +} + +/// CRC32 single round checksum for words (32 bits). +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/__crc32w) +#[inline] +#[target_feature(enable = "crc")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr(test, assert_instr(crc32w))] +pub unsafe fn __crc32w(crc: u32, data: u32) -> u32 { + crc32w_(crc, data) +} + +/// CRC32-C single round checksum for bytes (8 bits). +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/__crc32cb) +#[inline] +#[target_feature(enable = "crc")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr(test, assert_instr(crc32cb))] +pub unsafe fn __crc32cb(crc: u32, data: u8) -> u32 { + crc32cb_(crc, data as u32) +} + +/// CRC32-C single round checksum for half words (16 bits). +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/__crc32ch) +#[inline] +#[target_feature(enable = "crc")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr(test, assert_instr(crc32ch))] +pub unsafe fn __crc32ch(crc: u32, data: u16) -> u32 { + crc32ch_(crc, data as u32) +} + +/// CRC32-C single round checksum for words (32 bits). +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/__crc32cw) +#[inline] +#[target_feature(enable = "crc")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr(test, assert_instr(crc32cw))] +pub unsafe fn __crc32cw(crc: u32, data: u32) -> u32 { + crc32cw_(crc, data) +} + +#[cfg(test)] +mod tests { + use crate::core_arch::{arm_shared::*, simd::*}; + use std::mem; + use stdarch_test::simd_test; + + #[simd_test(enable = "crc")] + unsafe fn test_crc32b() { + assert_eq!(__crc32b(0, 0), 0); + assert_eq!(__crc32b(0, 255), 755167117); + } + + #[simd_test(enable = "crc")] + unsafe fn test_crc32h() { + assert_eq!(__crc32h(0, 0), 0); + assert_eq!(__crc32h(0, 16384), 1994146192); + } + + #[simd_test(enable = "crc")] + unsafe fn test_crc32w() { + assert_eq!(__crc32w(0, 0), 0); + assert_eq!(__crc32w(0, 4294967295), 3736805603); + } + + #[simd_test(enable = "crc")] + unsafe fn test_crc32cb() { + assert_eq!(__crc32cb(0, 0), 0); + assert_eq!(__crc32cb(0, 255), 2910671697); + } + + #[simd_test(enable = "crc")] + unsafe fn test_crc32ch() { + assert_eq!(__crc32ch(0, 0), 0); + assert_eq!(__crc32ch(0, 16384), 1098587580); + } + + #[simd_test(enable = "crc")] + unsafe fn test_crc32cw() { + assert_eq!(__crc32cw(0, 0), 0); + assert_eq!(__crc32cw(0, 4294967295), 3080238136); + } +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/crypto.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/crypto.rs similarity index 55% rename from crux-mir/lib/stdarch/crates/core_arch/src/aarch64/crypto.rs rename to crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/crypto.rs index 91269d805..060091136 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/aarch64/crypto.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/crypto.rs @@ -1,36 +1,50 @@ -use crate::core_arch::arm::{uint32x4_t, uint8x16_t}; +use crate::core_arch::arm_shared::{uint32x4_t, uint8x16_t}; #[allow(improper_ctypes)] -extern "C" { - #[link_name = "llvm.aarch64.crypto.aese"] +extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.aese")] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.aese")] fn vaeseq_u8_(data: uint8x16_t, key: uint8x16_t) -> uint8x16_t; - #[link_name = "llvm.aarch64.crypto.aesd"] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.aesd")] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.aesd")] fn vaesdq_u8_(data: uint8x16_t, key: uint8x16_t) -> uint8x16_t; - #[link_name = "llvm.aarch64.crypto.aesmc"] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.aesmc")] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.aesmc")] fn vaesmcq_u8_(data: uint8x16_t) -> uint8x16_t; - #[link_name = "llvm.aarch64.crypto.aesimc"] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.aesimc")] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.aesimc")] fn vaesimcq_u8_(data: uint8x16_t) -> uint8x16_t; - #[link_name = "llvm.aarch64.crypto.sha1h"] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sha1h")] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.sha1h")] fn vsha1h_u32_(hash_e: u32) -> u32; - #[link_name = "llvm.aarch64.crypto.sha1su0"] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sha1su0")] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.sha1su0")] fn vsha1su0q_u32_(w0_3: uint32x4_t, w4_7: uint32x4_t, w8_11: uint32x4_t) -> uint32x4_t; - #[link_name = "llvm.aarch64.crypto.sha1su1"] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sha1su1")] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.sha1su1")] fn vsha1su1q_u32_(tw0_3: uint32x4_t, w12_15: uint32x4_t) -> uint32x4_t; - #[link_name = "llvm.aarch64.crypto.sha1c"] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sha1c")] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.sha1c")] fn vsha1cq_u32_(hash_abcd: uint32x4_t, hash_e: u32, wk: uint32x4_t) -> uint32x4_t; - #[link_name = "llvm.aarch64.crypto.sha1p"] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sha1p")] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.sha1p")] fn vsha1pq_u32_(hash_abcd: uint32x4_t, hash_e: u32, wk: uint32x4_t) -> uint32x4_t; - #[link_name = "llvm.aarch64.crypto.sha1m"] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sha1m")] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.sha1m")] fn vsha1mq_u32_(hash_abcd: uint32x4_t, hash_e: u32, wk: uint32x4_t) -> uint32x4_t; - #[link_name = "llvm.aarch64.crypto.sha256h"] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sha256h")] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.sha256h")] fn vsha256hq_u32_(hash_abcd: uint32x4_t, hash_efgh: uint32x4_t, wk: uint32x4_t) -> uint32x4_t; - #[link_name = "llvm.aarch64.crypto.sha256h2"] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sha256h2")] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.sha256h2")] fn vsha256h2q_u32_(hash_efgh: uint32x4_t, hash_abcd: uint32x4_t, wk: uint32x4_t) -> uint32x4_t; - #[link_name = "llvm.aarch64.crypto.sha256su0"] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sha256su0")] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.sha256su0")] fn vsha256su0q_u32_(w0_3: uint32x4_t, w4_7: uint32x4_t) -> uint32x4_t; - #[link_name = "llvm.aarch64.crypto.sha256su1"] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.crypto.sha256su1")] + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.sha256su1")] fn vsha256su1q_u32_(tw0_3: uint32x4_t, w8_11: uint32x4_t, w12_15: uint32x4_t) -> uint32x4_t; } @@ -38,88 +52,121 @@ extern "C" { use stdarch_test::assert_instr; /// AES single round encryption. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaeseq_u8) #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(not(target_arch = "arm"), target_feature(enable = "aes"))] +#[cfg_attr(target_arch = "arm", target_feature(enable = "crypto,v8"))] #[cfg_attr(test, assert_instr(aese))] pub unsafe fn vaeseq_u8(data: uint8x16_t, key: uint8x16_t) -> uint8x16_t { vaeseq_u8_(data, key) } /// AES single round decryption. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaesdq_u8) #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(not(target_arch = "arm"), target_feature(enable = "aes"))] +#[cfg_attr(target_arch = "arm", target_feature(enable = "crypto,v8"))] #[cfg_attr(test, assert_instr(aesd))] pub unsafe fn vaesdq_u8(data: uint8x16_t, key: uint8x16_t) -> uint8x16_t { vaesdq_u8_(data, key) } /// AES mix columns. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaesmcq_u8) #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(not(target_arch = "arm"), target_feature(enable = "aes"))] +#[cfg_attr(target_arch = "arm", target_feature(enable = "crypto,v8"))] #[cfg_attr(test, assert_instr(aesmc))] pub unsafe fn vaesmcq_u8(data: uint8x16_t) -> uint8x16_t { vaesmcq_u8_(data) } /// AES inverse mix columns. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaesimcq_u8) #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(not(target_arch = "arm"), target_feature(enable = "aes"))] +#[cfg_attr(target_arch = "arm", target_feature(enable = "crypto,v8"))] #[cfg_attr(test, assert_instr(aesimc))] pub unsafe fn vaesimcq_u8(data: uint8x16_t) -> uint8x16_t { vaesimcq_u8_(data) } /// SHA1 fixed rotate. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha1h_u32) #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(not(target_arch = "arm"), target_feature(enable = "sha2"))] +#[cfg_attr(target_arch = "arm", target_feature(enable = "crypto,v8"))] #[cfg_attr(test, assert_instr(sha1h))] pub unsafe fn vsha1h_u32(hash_e: u32) -> u32 { vsha1h_u32_(hash_e) } /// SHA1 hash update accelerator, choose. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha1cq_u32) #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(not(target_arch = "arm"), target_feature(enable = "sha2"))] +#[cfg_attr(target_arch = "arm", target_feature(enable = "crypto,v8"))] #[cfg_attr(test, assert_instr(sha1c))] pub unsafe fn vsha1cq_u32(hash_abcd: uint32x4_t, hash_e: u32, wk: uint32x4_t) -> uint32x4_t { vsha1cq_u32_(hash_abcd, hash_e, wk) } /// SHA1 hash update accelerator, majority. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha1mq_u32) #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(not(target_arch = "arm"), target_feature(enable = "sha2"))] +#[cfg_attr(target_arch = "arm", target_feature(enable = "crypto,v8"))] #[cfg_attr(test, assert_instr(sha1m))] pub unsafe fn vsha1mq_u32(hash_abcd: uint32x4_t, hash_e: u32, wk: uint32x4_t) -> uint32x4_t { vsha1mq_u32_(hash_abcd, hash_e, wk) } /// SHA1 hash update accelerator, parity. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha1pq_u32) #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(not(target_arch = "arm"), target_feature(enable = "sha2"))] +#[cfg_attr(target_arch = "arm", target_feature(enable = "crypto,v8"))] #[cfg_attr(test, assert_instr(sha1p))] pub unsafe fn vsha1pq_u32(hash_abcd: uint32x4_t, hash_e: u32, wk: uint32x4_t) -> uint32x4_t { vsha1pq_u32_(hash_abcd, hash_e, wk) } /// SHA1 schedule update accelerator, first part. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha1su0q_u32) #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(not(target_arch = "arm"), target_feature(enable = "sha2"))] +#[cfg_attr(target_arch = "arm", target_feature(enable = "crypto,v8"))] #[cfg_attr(test, assert_instr(sha1su0))] pub unsafe fn vsha1su0q_u32(w0_3: uint32x4_t, w4_7: uint32x4_t, w8_11: uint32x4_t) -> uint32x4_t { vsha1su0q_u32_(w0_3, w4_7, w8_11) } /// SHA1 schedule update accelerator, second part. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha1su1q_u32) #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(not(target_arch = "arm"), target_feature(enable = "sha2"))] +#[cfg_attr(target_arch = "arm", target_feature(enable = "crypto,v8"))] #[cfg_attr(test, assert_instr(sha1su1))] pub unsafe fn vsha1su1q_u32(tw0_3: uint32x4_t, w12_15: uint32x4_t) -> uint32x4_t { vsha1su1q_u32_(tw0_3, w12_15) } /// SHA256 hash update accelerator. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha256hq_u32) #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(not(target_arch = "arm"), target_feature(enable = "sha2"))] +#[cfg_attr(target_arch = "arm", target_feature(enable = "crypto,v8"))] #[cfg_attr(test, assert_instr(sha256h))] pub unsafe fn vsha256hq_u32( hash_abcd: uint32x4_t, @@ -130,8 +177,11 @@ pub unsafe fn vsha256hq_u32( } /// SHA256 hash update accelerator, upper part. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha256h2q_u32) #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(not(target_arch = "arm"), target_feature(enable = "sha2"))] +#[cfg_attr(target_arch = "arm", target_feature(enable = "crypto,v8"))] #[cfg_attr(test, assert_instr(sha256h2))] pub unsafe fn vsha256h2q_u32( hash_efgh: uint32x4_t, @@ -142,16 +192,22 @@ pub unsafe fn vsha256h2q_u32( } /// SHA256 schedule update accelerator, first part. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha256su0q_u32) #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(not(target_arch = "arm"), target_feature(enable = "sha2"))] +#[cfg_attr(target_arch = "arm", target_feature(enable = "crypto,v8"))] #[cfg_attr(test, assert_instr(sha256su0))] pub unsafe fn vsha256su0q_u32(w0_3: uint32x4_t, w4_7: uint32x4_t) -> uint32x4_t { vsha256su0q_u32_(w0_3, w4_7) } /// SHA256 schedule update accelerator, second part. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha256su1q_u32) #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(not(target_arch = "arm"), target_feature(enable = "sha2"))] +#[cfg_attr(target_arch = "arm", target_feature(enable = "crypto,v8"))] #[cfg_attr(test, assert_instr(sha256su1))] pub unsafe fn vsha256su1q_u32( tw0_3: uint32x4_t, @@ -163,11 +219,13 @@ pub unsafe fn vsha256su1q_u32( #[cfg(test)] mod tests { - use crate::core_arch::{aarch64::*, simd::*}; + use super::*; + use crate::core_arch::{arm_shared::*, simd::*}; use std::mem; use stdarch_test::simd_test; - #[simd_test(enable = "crypto")] + #[cfg_attr(target_arch = "arm", simd_test(enable = "crypto"))] + #[cfg_attr(not(target_arch = "arm"), simd_test(enable = "aes"))] unsafe fn test_vaeseq_u8() { let data = mem::transmute(u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8)); let key = mem::transmute(u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7)); @@ -180,7 +238,8 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(target_arch = "arm", simd_test(enable = "crypto"))] + #[cfg_attr(not(target_arch = "arm"), simd_test(enable = "aes"))] unsafe fn test_vaesdq_u8() { let data = mem::transmute(u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8)); let key = mem::transmute(u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7)); @@ -191,7 +250,8 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(target_arch = "arm", simd_test(enable = "crypto"))] + #[cfg_attr(not(target_arch = "arm"), simd_test(enable = "aes"))] unsafe fn test_vaesmcq_u8() { let data = mem::transmute(u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8)); let r: u8x16 = mem::transmute(vaesmcq_u8(data)); @@ -201,7 +261,8 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(target_arch = "arm", simd_test(enable = "crypto"))] + #[cfg_attr(not(target_arch = "arm"), simd_test(enable = "aes"))] unsafe fn test_vaesimcq_u8() { let data = mem::transmute(u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8)); let r: u8x16 = mem::transmute(vaesimcq_u8(data)); @@ -211,13 +272,15 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(target_arch = "arm", simd_test(enable = "crypto"))] + #[cfg_attr(not(target_arch = "arm"), simd_test(enable = "sha2"))] unsafe fn test_vsha1h_u32() { assert_eq!(vsha1h_u32(0x1234), 0x048d); assert_eq!(vsha1h_u32(0x5678), 0x159e); } - #[simd_test(enable = "crypto")] + #[cfg_attr(target_arch = "arm", simd_test(enable = "crypto"))] + #[cfg_attr(not(target_arch = "arm"), simd_test(enable = "sha2"))] unsafe fn test_vsha1su0q_u32() { let r: u32x4 = mem::transmute(vsha1su0q_u32( mem::transmute(u32x4::new(0x1234_u32, 0x5678_u32, 0x9abc_u32, 0xdef0_u32)), @@ -227,7 +290,8 @@ mod tests { assert_eq!(r, u32x4::new(0x9abc, 0xdef0, 0x1234, 0x5678)); } - #[simd_test(enable = "crypto")] + #[cfg_attr(target_arch = "arm", simd_test(enable = "crypto"))] + #[cfg_attr(not(target_arch = "arm"), simd_test(enable = "sha2"))] unsafe fn test_vsha1su1q_u32() { let r: u32x4 = mem::transmute(vsha1su1q_u32( mem::transmute(u32x4::new(0x1234, 0x5678, 0x9abc, 0xdef0)), @@ -239,7 +303,8 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(target_arch = "arm", simd_test(enable = "crypto"))] + #[cfg_attr(not(target_arch = "arm"), simd_test(enable = "sha2"))] unsafe fn test_vsha1cq_u32() { let r: u32x4 = mem::transmute(vsha1cq_u32( mem::transmute(u32x4::new(0x1234, 0x5678, 0x9abc, 0xdef0)), @@ -252,7 +317,8 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(target_arch = "arm", simd_test(enable = "crypto"))] + #[cfg_attr(not(target_arch = "arm"), simd_test(enable = "sha2"))] unsafe fn test_vsha1pq_u32() { let r: u32x4 = mem::transmute(vsha1pq_u32( mem::transmute(u32x4::new(0x1234, 0x5678, 0x9abc, 0xdef0)), @@ -265,7 +331,8 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(target_arch = "arm", simd_test(enable = "crypto"))] + #[cfg_attr(not(target_arch = "arm"), simd_test(enable = "sha2"))] unsafe fn test_vsha1mq_u32() { let r: u32x4 = mem::transmute(vsha1mq_u32( mem::transmute(u32x4::new(0x1234, 0x5678, 0x9abc, 0xdef0)), @@ -278,7 +345,8 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(target_arch = "arm", simd_test(enable = "crypto"))] + #[cfg_attr(not(target_arch = "arm"), simd_test(enable = "sha2"))] unsafe fn test_vsha256hq_u32() { let r: u32x4 = mem::transmute(vsha256hq_u32( mem::transmute(u32x4::new(0x1234, 0x5678, 0x9abc, 0xdef0)), @@ -291,7 +359,8 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(target_arch = "arm", simd_test(enable = "crypto"))] + #[cfg_attr(not(target_arch = "arm"), simd_test(enable = "sha2"))] unsafe fn test_vsha256h2q_u32() { let r: u32x4 = mem::transmute(vsha256h2q_u32( mem::transmute(u32x4::new(0x1234, 0x5678, 0x9abc, 0xdef0)), @@ -304,7 +373,8 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(target_arch = "arm", simd_test(enable = "crypto"))] + #[cfg_attr(not(target_arch = "arm"), simd_test(enable = "sha2"))] unsafe fn test_vsha256su0q_u32() { let r: u32x4 = mem::transmute(vsha256su0q_u32( mem::transmute(u32x4::new(0x1234, 0x5678, 0x9abc, 0xdef0)), @@ -316,7 +386,8 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(target_arch = "arm", simd_test(enable = "crypto"))] + #[cfg_attr(not(target_arch = "arm"), simd_test(enable = "sha2"))] unsafe fn test_vsha256su1q_u32() { let r: u32x4 = mem::transmute(vsha256su1q_u32( mem::transmute(u32x4::new(0x1234, 0x5678, 0x9abc, 0xdef0)), diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/acle/hints.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/hints.rs similarity index 63% rename from crux-mir/lib/stdarch/crates/core_arch/src/acle/hints.rs rename to crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/hints.rs index 20faed69c..d7e43f551 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/acle/hints.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/hints.rs @@ -9,7 +9,7 @@ /// low-power state until one of a number of asynchronous events occurs. // Section 10.1 of ACLE says that the supported arches are: 8, 6K, 6-M // LLVM says "instruction requires: armv6k" -#[cfg(any(target_feature = "v6", target_arch = "aarch64"))] +#[cfg(any(target_feature = "v6", target_arch = "aarch64", doc))] #[inline(always)] pub unsafe fn __wfi() { hint(HINT_WFI); @@ -22,7 +22,7 @@ pub unsafe fn __wfi() { /// another processor. // Section 10.1 of ACLE says that the supported arches are: 8, 6K, 6-M // LLVM says "instruction requires: armv6k" -#[cfg(any(target_feature = "v6", target_arch = "aarch64"))] +#[cfg(any(target_feature = "v6", target_arch = "aarch64", doc))] #[inline(always)] pub unsafe fn __wfe() { hint(HINT_WFE); @@ -34,7 +34,7 @@ pub unsafe fn __wfe() { /// system. It is a NOP on a uniprocessor system. // Section 10.1 of ACLE says that the supported arches are: 8, 6K, 6-M, 7-M // LLVM says "instruction requires: armv6k" -#[cfg(any(target_feature = "v6", target_arch = "aarch64"))] +#[cfg(any(target_feature = "v6", target_arch = "aarch64", doc))] #[inline(always)] pub unsafe fn __sev() { hint(HINT_SEV); @@ -49,6 +49,7 @@ pub unsafe fn __sev() { #[cfg(any( target_feature = "v8", // 32-bit ARMv8 target_arch = "aarch64", // AArch64 + doc, ))] #[inline(always)] pub unsafe fn __sevl() { @@ -62,53 +63,12 @@ pub unsafe fn __sevl() { /// improve overall system performance. // Section 10.1 of ACLE says that the supported arches are: 8, 6K, 6-M // LLVM says "instruction requires: armv6k" -#[cfg(any(target_feature = "v6", target_arch = "aarch64"))] +#[cfg(any(target_feature = "v6", target_arch = "aarch64", doc))] #[inline(always)] pub unsafe fn __yield() { hint(HINT_YIELD); } -/// Generates a DBG instruction. -/// -/// This provides a hint to debugging and related systems. The argument must be -/// a constant integer from 0 to 15 inclusive. See implementation documentation -/// for the effect (if any) of this instruction and the meaning of the -/// argument. This is available only when compliling for AArch32. -// Section 10.1 of ACLE says that the supported arches are: 7, 7-M -// "The DBG hint instruction is added in ARMv7. It is UNDEFINED in the ARMv6 base architecture, and -// executes as a NOP instruction in ARMv6K and ARMv6T2." - ARM Architecture Reference Manual ARMv7-A -// and ARMv7-R edition (ARM DDI 0406C.c) sections D12.4.1 "ARM instruction set support" and D12.4.2 -// "Thumb instruction set support" -#[cfg(target_feature = "v7")] -#[inline(always)] -#[rustc_args_required_const(0)] -pub unsafe fn __dbg(imm4: u32) { - macro_rules! call { - ($imm4:expr) => { - asm!(concat!("DBG ", stringify!($imm4)) : : : : "volatile") - } - } - - match imm4 & 0b1111 { - 0 => call!(0), - 1 => call!(1), - 2 => call!(2), - 3 => call!(3), - 4 => call!(4), - 5 => call!(5), - 6 => call!(6), - 7 => call!(7), - 8 => call!(8), - 9 => call!(9), - 10 => call!(10), - 11 => call!(11), - 12 => call!(12), - 13 => call!(13), - 14 => call!(14), - _ => call!(15), - } -} - /// Generates an unspecified no-op instruction. /// /// Note that not all architectures provide a distinguished NOP instruction. On @@ -117,10 +77,10 @@ pub unsafe fn __dbg(imm4: u32) { /// will increase execution time. #[inline(always)] pub unsafe fn __nop() { - asm!("NOP" : : : : "volatile") + crate::arch::asm!("nop", options(nomem, nostack, preserves_flags)); } -extern "C" { +extern "unadjusted" { #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.hint")] #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.hint")] fn hint(_: i32); diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/acle/mod.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/mod.rs similarity index 65% rename from crux-mir/lib/stdarch/crates/core_arch/src/acle/mod.rs rename to crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/mod.rs index 5f29decf5..4c8d19854 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/acle/mod.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/mod.rs @@ -47,6 +47,9 @@ //! //! - [ACLE Q2 2018](https://developer.arm.com/docs/101028/latest) +// Only for 'neon' submodule +#![allow(non_camel_case_types)] + // 8, 7 and 6-M are supported via dedicated instructions like DMB. All other arches are supported // via CP15 instructions. See Section 10.1 of ACLE mod barrier; @@ -54,70 +57,29 @@ mod barrier; pub use self::barrier::*; mod hints; - pub use self::hints::*; mod registers; - pub use self::registers::*; -mod ex; - -pub use self::ex::*; - -// Supported arches: 5TE, 7E-M. See Section 10.1 of ACLE (e.g. QADD) -// We also include the A profile even though DSP is deprecated on that profile as of ACLE 2.0 (see -// section 5.4.7) -// Here we workaround the difference between LLVM's +dsp and ACLE's __ARM_FEATURE_DSP by gating on -// '+v5te' rather than on '+dsp' -#[cfg(all( - not(target_arch = "aarch64"), - any( - // >= v5TE but excludes v7-M - all(target_feature = "v5te", not(target_feature = "mclass")), - // v7E-M - all(target_feature = "mclass", target_feature = "dsp"), - ) -))] -mod dsp; - -#[cfg(all( - not(target_arch = "aarch64"), - any( - all(target_feature = "v5te", not(target_feature = "mclass")), - all(target_feature = "mclass", target_feature = "dsp"), - ) -))] -pub use self::dsp::*; - -// Supported arches: 6, 7-M. See Section 10.1 of ACLE (e.g. SSAT) -#[cfg(all(not(target_arch = "aarch64"), target_feature = "v6",))] -mod sat; - -#[cfg(all(not(target_arch = "aarch64"), target_feature = "v6",))] -pub use self::sat::*; - -// Deprecated in ACLE 2.0 for the A profile but fully supported on the M and R profiles, says -// Section 5.4.9 of ACLE. We'll expose these for the A profile even if deprecated -#[cfg(all( - not(target_arch = "aarch64"), - any( - // v7-A, v7-R - all(target_feature = "v6", not(target_feature = "mclass")), - // v7E-M - all(target_feature = "mclass", target_feature = "dsp") - ) -))] -mod simd32; - -#[cfg(all( - not(target_arch = "aarch64"), - any( - all(target_feature = "v6", not(target_feature = "mclass")), - all(target_feature = "mclass", target_feature = "dsp") - ) -))] -pub use self::simd32::*; +#[cfg(any(target_arch = "aarch64", target_feature = "v7", doc))] +mod crc; +#[cfg(any(target_arch = "aarch64", target_feature = "v7", doc))] +pub use crc::*; + +#[cfg(any(target_arch = "aarch64", target_feature = "v7", doc))] +mod crypto; +#[cfg(any(target_arch = "aarch64", target_feature = "v7", doc))] +pub use self::crypto::*; + +#[cfg(any(target_arch = "aarch64", target_feature = "v7", doc))] +pub(crate) mod neon; +#[cfg(any(target_arch = "aarch64", target_feature = "v7", doc))] +pub use self::neon::*; + +#[cfg(test)] +#[cfg(any(target_arch = "aarch64", target_feature = "v7", doc))] +pub(crate) mod test_support; mod sealed { pub trait Dmb { diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs new file mode 100644 index 000000000..ac2709744 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs @@ -0,0 +1,44738 @@ +// This code is automatically generated. DO NOT MODIFY. +// +// Instead, modify `crates/stdarch-gen/neon.spec` and run the following command to re-generate this file: +// +// ``` +// OUT_DIR=`pwd`/crates/core_arch cargo run -p stdarch-gen -- crates/stdarch-gen/neon.spec +// ``` +use super::*; +#[cfg(test)] +use stdarch_test::assert_instr; + +/// Vector bitwise and +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vand_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vand_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_and(a, b) +} + +/// Vector bitwise and +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vandq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vandq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_and(a, b) +} + +/// Vector bitwise and +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vand_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vand_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_and(a, b) +} + +/// Vector bitwise and +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vandq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vandq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_and(a, b) +} + +/// Vector bitwise and +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vand_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vand_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_and(a, b) +} + +/// Vector bitwise and +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vandq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vandq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_and(a, b) +} + +/// Vector bitwise and +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vand_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vand_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_and(a, b) +} + +/// Vector bitwise and +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vandq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vandq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_and(a, b) +} + +/// Vector bitwise and +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vand_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vand_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_and(a, b) +} + +/// Vector bitwise and +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vandq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vandq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_and(a, b) +} + +/// Vector bitwise and +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vand_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vand_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_and(a, b) +} + +/// Vector bitwise and +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vandq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vandq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_and(a, b) +} + +/// Vector bitwise and +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vand_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vand_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + simd_and(a, b) +} + +/// Vector bitwise and +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vandq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vandq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + simd_and(a, b) +} + +/// Vector bitwise and +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vand_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vand_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + simd_and(a, b) +} + +/// Vector bitwise and +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vandq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vandq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_and(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorr_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vorr_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorrq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vorrq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorr_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vorr_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorrq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vorrq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorr_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vorr_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorrq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vorrq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorr_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vorr_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorrq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vorrq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorr_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vorr_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorrq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vorrq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorr_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vorr_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorrq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vorrq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorr_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vorr_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorrq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vorrq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorr_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vorr_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorrq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vorrq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_or(a, b) +} + +/// Vector bitwise exclusive or (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn veor_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veorq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn veorq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn veor_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veorq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn veorq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn veor_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veorq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn veorq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn veor_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veorq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn veorq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn veor_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veorq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn veorq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn veor_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veorq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn veorq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn veor_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veorq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn veorq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn veor_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veorq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn veorq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_xor(a, b) +} + +/// Absolute difference between the arguments +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabd_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabd_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabds.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sabd.v8i8")] + fn vabd_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vabd_s8_(a, b) +} + +/// Absolute difference between the arguments +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabdq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabds.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sabd.v16i8")] + fn vabdq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vabdq_s8_(a, b) +} + +/// Absolute difference between the arguments +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabd_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabd_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabds.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sabd.v4i16")] + fn vabd_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vabd_s16_(a, b) +} + +/// Absolute difference between the arguments +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabdq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabds.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sabd.v8i16")] + fn vabdq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vabdq_s16_(a, b) +} + +/// Absolute difference between the arguments +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabd_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabd_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabds.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sabd.v2i32")] + fn vabd_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vabd_s32_(a, b) +} + +/// Absolute difference between the arguments +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabdq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabds.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sabd.v4i32")] + fn vabdq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vabdq_s32_(a, b) +} + +/// Absolute difference between the arguments +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabd_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabdu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uabd.v8i8")] + fn vabd_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; + } +vabd_u8_(a, b) +} + +/// Absolute difference between the arguments +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabdq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabdu.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uabd.v16i8")] + fn vabdq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; + } +vabdq_u8_(a, b) +} + +/// Absolute difference between the arguments +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabd_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabd_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabdu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uabd.v4i16")] + fn vabd_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; + } +vabd_u16_(a, b) +} + +/// Absolute difference between the arguments +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabdq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabdu.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uabd.v8i16")] + fn vabdq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; + } +vabdq_u16_(a, b) +} + +/// Absolute difference between the arguments +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabd_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabd_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabdu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uabd.v2i32")] + fn vabd_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; + } +vabd_u32_(a, b) +} + +/// Absolute difference between the arguments +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabdq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabdu.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uabd.v4i32")] + fn vabdq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; + } +vabdq_u32_(a, b) +} + +/// Absolute difference between the arguments of Floating +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabd_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fabd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabd_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabds.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fabd.v2f32")] + fn vabd_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; + } +vabd_f32_(a, b) +} + +/// Absolute difference between the arguments of Floating +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fabd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabdq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabds.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fabd.v4f32")] + fn vabdq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + } +vabdq_f32_(a, b) +} + +/// Unsigned Absolute difference Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabdl.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabdl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabdl_u8(a: uint8x8_t, b: uint8x8_t) -> uint16x8_t { + simd_cast(vabd_u8(a, b)) +} + +/// Unsigned Absolute difference Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabdl.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabdl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabdl_u16(a: uint16x4_t, b: uint16x4_t) -> uint32x4_t { + simd_cast(vabd_u16(a, b)) +} + +/// Unsigned Absolute difference Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabdl.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabdl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabdl_u32(a: uint32x2_t, b: uint32x2_t) -> uint64x2_t { + simd_cast(vabd_u32(a, b)) +} + +/// Signed Absolute difference Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabdl.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabdl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabdl_s8(a: int8x8_t, b: int8x8_t) -> int16x8_t { + let c: uint8x8_t = simd_cast(vabd_s8(a, b)); + simd_cast(c) +} + +/// Signed Absolute difference Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabdl.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabdl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabdl_s16(a: int16x4_t, b: int16x4_t) -> int32x4_t { + let c: uint16x4_t = simd_cast(vabd_s16(a, b)); + simd_cast(c) +} + +/// Signed Absolute difference Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabdl.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabdl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabdl_s32(a: int32x2_t, b: int32x2_t) -> int64x2_t { + let c: uint32x2_t = simd_cast(vabd_s32(a, b)); + simd_cast(c) +} + +/// Compare bitwise Equal (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vceq_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vceqq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vceq_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vceqq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vceq_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vceqq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vceq_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vceqq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vceq_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vceqq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vceq_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vceqq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vceq_p8(a: poly8x8_t, b: poly8x8_t) -> uint8x8_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vceqq_p8(a: poly8x16_t, b: poly8x16_t) -> uint8x16_t { + simd_eq(a, b) +} + +/// Floating-point compare equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmeq))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vceq_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { + simd_eq(a, b) +} + +/// Floating-point compare equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmeq))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vceqq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { + simd_eq(a, b) +} + +/// Signed compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtst_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t { + let c: int8x8_t = simd_and(a, b); + let d: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtstq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t { + let c: int8x16_t = simd_and(a, b); + let d: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtst_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t { + let c: int16x4_t = simd_and(a, b); + let d: i16x4 = i16x4::new(0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtstq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t { + let c: int16x8_t = simd_and(a, b); + let d: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtst_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t { + let c: int32x2_t = simd_and(a, b); + let d: i32x2 = i32x2::new(0, 0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtstq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t { + let c: int32x4_t = simd_and(a, b); + let d: i32x4 = i32x4::new(0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtst_p8(a: poly8x8_t, b: poly8x8_t) -> uint8x8_t { + let c: poly8x8_t = simd_and(a, b); + let d: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtstq_p8(a: poly8x16_t, b: poly8x16_t) -> uint8x16_t { + let c: poly8x16_t = simd_and(a, b); + let d: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtst_p16(a: poly16x4_t, b: poly16x4_t) -> uint16x4_t { + let c: poly16x4_t = simd_and(a, b); + let d: i16x4 = i16x4::new(0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtstq_p16(a: poly16x8_t, b: poly16x8_t) -> uint16x8_t { + let c: poly16x8_t = simd_and(a, b); + let d: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Unsigned compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtst_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + let c: uint8x8_t = simd_and(a, b); + let d: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Unsigned compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtstq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + let c: uint8x16_t = simd_and(a, b); + let d: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Unsigned compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtst_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + let c: uint16x4_t = simd_and(a, b); + let d: u16x4 = u16x4::new(0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Unsigned compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtstq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + let c: uint16x8_t = simd_and(a, b); + let d: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Unsigned compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtst_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + let c: uint32x2_t = simd_and(a, b); + let d: u32x2 = u32x2::new(0, 0); + simd_ne(c, transmute(d)) +} + +/// Unsigned compare bitwise Test bits nonzero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtstq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + let c: uint32x4_t = simd_and(a, b); + let d: u32x4 = u32x4::new(0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Floating-point absolute value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabs_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fabs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabs_f32(a: float32x2_t) -> float32x2_t { + simd_fabs(a) +} + +/// Floating-point absolute value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabsq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fabs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabsq_f32(a: float32x4_t) -> float32x4_t { + simd_fabs(a) +} + +/// Compare signed greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgt_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcgt_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t { + simd_gt(a, b) +} + +/// Compare signed greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcgtq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t { + simd_gt(a, b) +} + +/// Compare signed greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgt_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcgt_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t { + simd_gt(a, b) +} + +/// Compare signed greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcgtq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t { + simd_gt(a, b) +} + +/// Compare signed greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgt_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcgt_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t { + simd_gt(a, b) +} + +/// Compare signed greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcgtq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t { + simd_gt(a, b) +} + +/// Compare unsigned highe +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgt_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcgt_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_gt(a, b) +} + +/// Compare unsigned highe +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcgtq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_gt(a, b) +} + +/// Compare unsigned highe +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgt_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcgt_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_gt(a, b) +} + +/// Compare unsigned highe +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcgtq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_gt(a, b) +} + +/// Compare unsigned highe +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgt_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcgt_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_gt(a, b) +} + +/// Compare unsigned highe +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcgtq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_gt(a, b) +} + +/// Floating-point compare greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgt_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmgt))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcgt_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { + simd_gt(a, b) +} + +/// Floating-point compare greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmgt))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcgtq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { + simd_gt(a, b) +} + +/// Compare signed less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclt_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclt_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t { + simd_lt(a, b) +} + +/// Compare signed less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcltq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t { + simd_lt(a, b) +} + +/// Compare signed less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclt_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclt_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t { + simd_lt(a, b) +} + +/// Compare signed less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcltq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t { + simd_lt(a, b) +} + +/// Compare signed less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclt_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclt_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t { + simd_lt(a, b) +} + +/// Compare signed less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcltq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t { + simd_lt(a, b) +} + +/// Compare unsigned less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclt_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclt_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_lt(a, b) +} + +/// Compare unsigned less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcltq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_lt(a, b) +} + +/// Compare unsigned less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclt_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclt_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_lt(a, b) +} + +/// Compare unsigned less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcltq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_lt(a, b) +} + +/// Compare unsigned less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclt_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclt_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_lt(a, b) +} + +/// Compare unsigned less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcltq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_lt(a, b) +} + +/// Floating-point compare less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclt_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmgt))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclt_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { + simd_lt(a, b) +} + +/// Floating-point compare less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmgt))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcltq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { + simd_lt(a, b) +} + +/// Compare signed less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcle_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcle_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t { + simd_le(a, b) +} + +/// Compare signed less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcleq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t { + simd_le(a, b) +} + +/// Compare signed less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcle_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcle_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t { + simd_le(a, b) +} + +/// Compare signed less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcleq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t { + simd_le(a, b) +} + +/// Compare signed less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcle_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcle_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t { + simd_le(a, b) +} + +/// Compare signed less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcleq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t { + simd_le(a, b) +} + +/// Compare unsigned less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcle_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcle_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_le(a, b) +} + +/// Compare unsigned less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcleq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_le(a, b) +} + +/// Compare unsigned less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcle_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcle_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_le(a, b) +} + +/// Compare unsigned less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcleq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_le(a, b) +} + +/// Compare unsigned less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcle_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcle_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_le(a, b) +} + +/// Compare unsigned less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcleq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_le(a, b) +} + +/// Floating-point compare less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcle_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmge))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcle_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { + simd_le(a, b) +} + +/// Floating-point compare less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmge))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcleq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { + simd_le(a, b) +} + +/// Compare signed greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcge_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcge_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t { + simd_ge(a, b) +} + +/// Compare signed greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcgeq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t { + simd_ge(a, b) +} + +/// Compare signed greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcge_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcge_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t { + simd_ge(a, b) +} + +/// Compare signed greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcgeq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t { + simd_ge(a, b) +} + +/// Compare signed greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcge_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcge_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t { + simd_ge(a, b) +} + +/// Compare signed greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcgeq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t { + simd_ge(a, b) +} + +/// Compare unsigned greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcge_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcge_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_ge(a, b) +} + +/// Compare unsigned greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcgeq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_ge(a, b) +} + +/// Compare unsigned greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcge_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcge_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_ge(a, b) +} + +/// Compare unsigned greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcgeq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_ge(a, b) +} + +/// Compare unsigned greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcge_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcge_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_ge(a, b) +} + +/// Compare unsigned greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcgeq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_ge(a, b) +} + +/// Floating-point compare greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcge_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmge))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcge_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { + simd_ge(a, b) +} + +/// Floating-point compare greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmge))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcgeq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { + simd_ge(a, b) +} + +/// Count leading sign bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcls_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcls.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcls_s8(a: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcls.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.cls.v8i8")] + fn vcls_s8_(a: int8x8_t) -> int8x8_t; + } +vcls_s8_(a) +} + +/// Count leading sign bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclsq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcls.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclsq_s8(a: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcls.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.cls.v16i8")] + fn vclsq_s8_(a: int8x16_t) -> int8x16_t; + } +vclsq_s8_(a) +} + +/// Count leading sign bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcls_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcls.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcls_s16(a: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcls.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.cls.v4i16")] + fn vcls_s16_(a: int16x4_t) -> int16x4_t; + } +vcls_s16_(a) +} + +/// Count leading sign bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclsq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcls.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclsq_s16(a: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcls.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.cls.v8i16")] + fn vclsq_s16_(a: int16x8_t) -> int16x8_t; + } +vclsq_s16_(a) +} + +/// Count leading sign bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcls_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcls.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcls_s32(a: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcls.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.cls.v2i32")] + fn vcls_s32_(a: int32x2_t) -> int32x2_t; + } +vcls_s32_(a) +} + +/// Count leading sign bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclsq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcls.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclsq_s32(a: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcls.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.cls.v4i32")] + fn vclsq_s32_(a: int32x4_t) -> int32x4_t; + } +vclsq_s32_(a) +} + +/// Count leading sign bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcls_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcls))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcls_u8(a: uint8x8_t) -> int8x8_t { + transmute(vcls_s8(transmute(a))) +} + +/// Count leading sign bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclsq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcls))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclsq_u8(a: uint8x16_t) -> int8x16_t { + transmute(vclsq_s8(transmute(a))) +} + +/// Count leading sign bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcls_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcls))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcls_u16(a: uint16x4_t) -> int16x4_t { + transmute(vcls_s16(transmute(a))) +} + +/// Count leading sign bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclsq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcls))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclsq_u16(a: uint16x8_t) -> int16x8_t { + transmute(vclsq_s16(transmute(a))) +} + +/// Count leading sign bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcls_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcls))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcls_u32(a: uint32x2_t) -> int32x2_t { + transmute(vcls_s32(transmute(a))) +} + +/// Count leading sign bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclsq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcls))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclsq_u32(a: uint32x4_t) -> int32x4_t { + transmute(vclsq_s32(transmute(a))) +} + +/// Count leading zero bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclz_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclz_s8(a: int8x8_t) -> int8x8_t { + vclz_s8_(a) +} + +/// Count leading zero bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclzq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclzq_s8(a: int8x16_t) -> int8x16_t { + vclzq_s8_(a) +} + +/// Count leading zero bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclz_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclz_s16(a: int16x4_t) -> int16x4_t { + vclz_s16_(a) +} + +/// Count leading zero bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclzq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclzq_s16(a: int16x8_t) -> int16x8_t { + vclzq_s16_(a) +} + +/// Count leading zero bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclz_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclz_s32(a: int32x2_t) -> int32x2_t { + vclz_s32_(a) +} + +/// Count leading zero bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclzq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclzq_s32(a: int32x4_t) -> int32x4_t { + vclzq_s32_(a) +} + +/// Count leading zero bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclz_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclz_u8(a: uint8x8_t) -> uint8x8_t { + transmute(vclz_s8_(transmute(a))) +} + +/// Count leading zero bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclzq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclzq_u8(a: uint8x16_t) -> uint8x16_t { + transmute(vclzq_s8_(transmute(a))) +} + +/// Count leading zero bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclz_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclz_u16(a: uint16x4_t) -> uint16x4_t { + transmute(vclz_s16_(transmute(a))) +} + +/// Count leading zero bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclzq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclzq_u16(a: uint16x8_t) -> uint16x8_t { + transmute(vclzq_s16_(transmute(a))) +} + +/// Count leading zero bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclz_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclz_u32(a: uint32x2_t) -> uint32x2_t { + transmute(vclz_s32_(transmute(a))) +} + +/// Count leading zero bits +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclzq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vclzq_u32(a: uint32x4_t) -> uint32x4_t { + transmute(vclzq_s32_(transmute(a))) +} + +/// Floating-point absolute compare greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcagt_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacgt.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(facgt))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcagt_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vacgt.v2i32.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.facgt.v2i32.v2f32")] + fn vcagt_f32_(a: float32x2_t, b: float32x2_t) -> uint32x2_t; + } +vcagt_f32_(a, b) +} + +/// Floating-point absolute compare greater than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcagtq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacgt.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(facgt))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcagtq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vacgt.v4i32.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.facgt.v4i32.v4f32")] + fn vcagtq_f32_(a: float32x4_t, b: float32x4_t) -> uint32x4_t; + } +vcagtq_f32_(a, b) +} + +/// Floating-point absolute compare greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcage_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacge.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(facge))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcage_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vacge.v2i32.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.facge.v2i32.v2f32")] + fn vcage_f32_(a: float32x2_t, b: float32x2_t) -> uint32x2_t; + } +vcage_f32_(a, b) +} + +/// Floating-point absolute compare greater than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcageq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacge.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(facge))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcageq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vacge.v4i32.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.facge.v4i32.v4f32")] + fn vcageq_f32_(a: float32x4_t, b: float32x4_t) -> uint32x4_t; + } +vcageq_f32_(a, b) +} + +/// Floating-point absolute compare less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcalt_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacgt.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(facgt))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcalt_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { + vcagt_f32(b, a) +} + +/// Floating-point absolute compare less than +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaltq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacgt.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(facgt))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcaltq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { + vcagtq_f32(b, a) +} + +/// Floating-point absolute compare less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcale_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacge.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(facge))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcale_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { + vcage_f32(b, a) +} + +/// Floating-point absolute compare less than or equal +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaleq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacge.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(facge))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcaleq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { + vcageq_f32(b, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcreate_s8(a: u64) -> int8x8_t { + transmute(a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcreate_s16(a: u64) -> int16x4_t { + transmute(a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcreate_s32(a: u64) -> int32x2_t { + transmute(a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcreate_s64(a: u64) -> int64x1_t { + transmute(a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcreate_u8(a: u64) -> uint8x8_t { + transmute(a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcreate_u16(a: u64) -> uint16x4_t { + transmute(a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcreate_u32(a: u64) -> uint32x2_t { + transmute(a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcreate_u64(a: u64) -> uint64x1_t { + transmute(a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcreate_p8(a: u64) -> poly8x8_t { + transmute(a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcreate_p16(a: u64) -> poly16x4_t { + transmute(a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcreate_p64(a: u64) -> poly64x1_t { + transmute(a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcreate_f32(a: u64) -> float32x2_t { + transmute(a) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_f32_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(scvtf))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcvt_f32_s32(a: int32x2_t) -> float32x2_t { + simd_cast(a) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_f32_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(scvtf))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcvtq_f32_s32(a: int32x4_t) -> float32x4_t { + simd_cast(a) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_f32_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ucvtf))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcvt_f32_u32(a: uint32x2_t) -> float32x2_t { + simd_cast(a) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_f32_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ucvtf))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcvtq_f32_u32(a: uint32x4_t) -> float32x4_t { + simd_cast(a) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_f32_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vcvt, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvt_n_f32_s32(a: int32x2_t) -> float32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcvtfxs2fp.v2f32.v2i32")] + fn vcvt_n_f32_s32_(a: int32x2_t, n: i32) -> float32x2_t; + } +vcvt_n_f32_s32_(a, N) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_f32_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(scvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvt_n_f32_s32(a: int32x2_t) -> float32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxs2fp.v2f32.v2i32")] + fn vcvt_n_f32_s32_(a: int32x2_t, n: i32) -> float32x2_t; + } +vcvt_n_f32_s32_(a, N) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_f32_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vcvt, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvtq_n_f32_s32(a: int32x4_t) -> float32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcvtfxs2fp.v4f32.v4i32")] + fn vcvtq_n_f32_s32_(a: int32x4_t, n: i32) -> float32x4_t; + } +vcvtq_n_f32_s32_(a, N) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_f32_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(scvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtq_n_f32_s32(a: int32x4_t) -> float32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxs2fp.v4f32.v4i32")] + fn vcvtq_n_f32_s32_(a: int32x4_t, n: i32) -> float32x4_t; + } +vcvtq_n_f32_s32_(a, N) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_f32_u32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vcvt, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvt_n_f32_u32(a: uint32x2_t) -> float32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcvtfxu2fp.v2f32.v2i32")] + fn vcvt_n_f32_u32_(a: uint32x2_t, n: i32) -> float32x2_t; + } +vcvt_n_f32_u32_(a, N) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_f32_u32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ucvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvt_n_f32_u32(a: uint32x2_t) -> float32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxu2fp.v2f32.v2i32")] + fn vcvt_n_f32_u32_(a: uint32x2_t, n: i32) -> float32x2_t; + } +vcvt_n_f32_u32_(a, N) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_f32_u32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vcvt, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvtq_n_f32_u32(a: uint32x4_t) -> float32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcvtfxu2fp.v4f32.v4i32")] + fn vcvtq_n_f32_u32_(a: uint32x4_t, n: i32) -> float32x4_t; + } +vcvtq_n_f32_u32_(a, N) +} + +/// Fixed-point convert to floating-point +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_f32_u32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ucvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtq_n_f32_u32(a: uint32x4_t) -> float32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxu2fp.v4f32.v4i32")] + fn vcvtq_n_f32_u32_(a: uint32x4_t, n: i32) -> float32x4_t; + } +vcvtq_n_f32_u32_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_s32_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vcvt, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvt_n_s32_f32(a: float32x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcvtfp2fxs.v2i32.v2f32")] + fn vcvt_n_s32_f32_(a: float32x2_t, n: i32) -> int32x2_t; + } +vcvt_n_s32_f32_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_s32_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzs, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvt_n_s32_f32(a: float32x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxs.v2i32.v2f32")] + fn vcvt_n_s32_f32_(a: float32x2_t, n: i32) -> int32x2_t; + } +vcvt_n_s32_f32_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_s32_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vcvt, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvtq_n_s32_f32(a: float32x4_t) -> int32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcvtfp2fxs.v4i32.v4f32")] + fn vcvtq_n_s32_f32_(a: float32x4_t, n: i32) -> int32x4_t; + } +vcvtq_n_s32_f32_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_s32_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzs, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtq_n_s32_f32(a: float32x4_t) -> int32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxs.v4i32.v4f32")] + fn vcvtq_n_s32_f32_(a: float32x4_t, n: i32) -> int32x4_t; + } +vcvtq_n_s32_f32_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_u32_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vcvt, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvt_n_u32_f32(a: float32x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcvtfp2fxu.v2i32.v2f32")] + fn vcvt_n_u32_f32_(a: float32x2_t, n: i32) -> uint32x2_t; + } +vcvt_n_u32_f32_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_u32_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzu, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvt_n_u32_f32(a: float32x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxu.v2i32.v2f32")] + fn vcvt_n_u32_f32_(a: float32x2_t, n: i32) -> uint32x2_t; + } +vcvt_n_u32_f32_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_u32_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vcvt, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvtq_n_u32_f32(a: float32x4_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcvtfp2fxu.v4i32.v4f32")] + fn vcvtq_n_u32_f32_(a: float32x4_t, n: i32) -> uint32x4_t; + } +vcvtq_n_u32_f32_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_u32_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzu, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vcvtq_n_u32_f32(a: float32x4_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxu.v4i32.v4f32")] + fn vcvtq_n_u32_f32_(a: float32x4_t, n: i32) -> uint32x4_t; + } +vcvtq_n_u32_f32_(a, N) +} + +/// Floating-point convert to signed fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_s32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcvtzs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcvt_s32_f32(a: float32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.fptosi.sat.v2i32.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fptosi.sat.v2i32.v2f32")] + fn vcvt_s32_f32_(a: float32x2_t) -> int32x2_t; + } +vcvt_s32_f32_(a) +} + +/// Floating-point convert to signed fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_s32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcvtzs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcvtq_s32_f32(a: float32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.fptosi.sat.v4i32.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fptosi.sat.v4i32.v4f32")] + fn vcvtq_s32_f32_(a: float32x4_t) -> int32x4_t; + } +vcvtq_s32_f32_(a) +} + +/// Floating-point convert to unsigned fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_u32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcvtzu))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcvt_u32_f32(a: float32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.fptoui.sat.v2i32.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fptoui.sat.v2i32.v2f32")] + fn vcvt_u32_f32_(a: float32x2_t) -> uint32x2_t; + } +vcvt_u32_f32_(a) +} + +/// Floating-point convert to unsigned fixed-point, rounding toward zero +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_u32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcvtzu))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vcvtq_u32_f32(a: float32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.fptoui.sat.v4i32.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fptoui.sat.v4i32.v4f32")] + fn vcvtq_u32_f32_(a: float32x4_t) -> uint32x4_t; + } +vcvtq_u32_f32_(a) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_lane_s8(a: int8x8_t) -> int8x8_t { + static_assert_imm3!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 8))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_laneq_s8(a: int8x16_t) -> int8x16_t { + static_assert_imm4!(N); + simd_shuffle16!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_lane_s16(a: int16x4_t) -> int16x4_t { + static_assert_imm2!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_laneq_s16(a: int16x8_t) -> int16x8_t { + static_assert_imm3!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 1))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_lane_s32(a: int32x2_t) -> int32x2_t { + static_assert_imm1!(N); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_laneq_s32(a: int32x4_t) -> int32x4_t { + static_assert_imm2!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 8))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_laneq_s8(a: int8x16_t) -> int8x8_t { + static_assert_imm4!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_laneq_s16(a: int16x8_t) -> int16x4_t { + static_assert_imm3!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_laneq_s32(a: int32x4_t) -> int32x2_t { + static_assert_imm2!(N); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_lane_s8(a: int8x8_t) -> int8x16_t { + static_assert_imm3!(N); + simd_shuffle16!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_lane_s16(a: int16x4_t) -> int16x8_t { + static_assert_imm2!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 1))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_lane_s32(a: int32x2_t) -> int32x4_t { + static_assert_imm1!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_lane_u8(a: uint8x8_t) -> uint8x8_t { + static_assert_imm3!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 8))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_laneq_u8(a: uint8x16_t) -> uint8x16_t { + static_assert_imm4!(N); + simd_shuffle16!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_lane_u16(a: uint16x4_t) -> uint16x4_t { + static_assert_imm2!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_laneq_u16(a: uint16x8_t) -> uint16x8_t { + static_assert_imm3!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 1))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_lane_u32(a: uint32x2_t) -> uint32x2_t { + static_assert_imm1!(N); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_laneq_u32(a: uint32x4_t) -> uint32x4_t { + static_assert_imm2!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 8))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_laneq_u8(a: uint8x16_t) -> uint8x8_t { + static_assert_imm4!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_laneq_u16(a: uint16x8_t) -> uint16x4_t { + static_assert_imm3!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_laneq_u32(a: uint32x4_t) -> uint32x2_t { + static_assert_imm2!(N); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_lane_u8(a: uint8x8_t) -> uint8x16_t { + static_assert_imm3!(N); + simd_shuffle16!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_lane_u16(a: uint16x4_t) -> uint16x8_t { + static_assert_imm2!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 1))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_lane_u32(a: uint32x2_t) -> uint32x4_t { + static_assert_imm1!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_lane_p8(a: poly8x8_t) -> poly8x8_t { + static_assert_imm3!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 8))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_laneq_p8(a: poly8x16_t) -> poly8x16_t { + static_assert_imm4!(N); + simd_shuffle16!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_lane_p16(a: poly16x4_t) -> poly16x4_t { + static_assert_imm2!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_laneq_p16(a: poly16x8_t) -> poly16x8_t { + static_assert_imm3!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 8))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_laneq_p8(a: poly8x16_t) -> poly8x8_t { + static_assert_imm4!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_laneq_p16(a: poly16x8_t) -> poly16x4_t { + static_assert_imm3!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_lane_p8(a: poly8x8_t) -> poly8x16_t { + static_assert_imm3!(N); + simd_shuffle16!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_lane_p16(a: poly16x4_t) -> poly16x8_t { + static_assert_imm2!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 1))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_laneq_s64(a: int64x2_t) -> int64x2_t { + static_assert_imm1!(N); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 0))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_lane_s64(a: int64x1_t) -> int64x2_t { + static_assert!(N : i32 where N == 0); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 1))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_laneq_u64(a: uint64x2_t) -> uint64x2_t { + static_assert_imm1!(N); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 0))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_lane_u64(a: uint64x1_t) -> uint64x2_t { + static_assert!(N : i32 where N == 0); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 1))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_lane_f32(a: float32x2_t) -> float32x2_t { + static_assert_imm1!(N); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_laneq_f32(a: float32x4_t) -> float32x4_t { + static_assert_imm2!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_laneq_f32(a: float32x4_t) -> float32x2_t { + static_assert_imm2!(N); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 1))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdupq_lane_f32(a: float32x2_t) -> float32x4_t { + static_assert_imm1!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, N = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, N = 0))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_lane_s64(a: int64x1_t) -> int64x1_t { + static_assert!(N : i32 where N == 0); + a +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, N = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, N = 0))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_lane_u64(a: uint64x1_t) -> uint64x1_t { + static_assert!(N : i32 where N == 0); + a +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, N = 1))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_laneq_s64(a: int64x2_t) -> int64x1_t { + static_assert_imm1!(N); + transmute::(simd_extract(a, N as u32)) +} + +/// Set all vector lanes to the same value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, N = 1))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vdup_laneq_u64(a: uint64x2_t) -> uint64x1_t { + static_assert_imm1!(N); + transmute::(simd_extract(a, N as u32)) +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vext_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 7))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 7))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vext_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + static_assert_imm3!(N); + match N & 0b111 { + 0 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), + 2 => simd_shuffle8!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), + 3 => simd_shuffle8!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), + 4 => simd_shuffle8!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), + 5 => simd_shuffle8!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), + 6 => simd_shuffle8!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), + 7 => simd_shuffle8!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 15))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 15))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vextq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + static_assert_imm4!(N); + match N & 0b1111 { + 0 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle16!(a, b, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]), + 2 => simd_shuffle16!(a, b, [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]), + 3 => simd_shuffle16!(a, b, [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18]), + 4 => simd_shuffle16!(a, b, [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]), + 5 => simd_shuffle16!(a, b, [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]), + 6 => simd_shuffle16!(a, b, [6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]), + 7 => simd_shuffle16!(a, b, [7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22]), + 8 => simd_shuffle16!(a, b, [8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]), + 9 => simd_shuffle16!(a, b, [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]), + 10 => simd_shuffle16!(a, b, [10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]), + 11 => simd_shuffle16!(a, b, [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]), + 12 => simd_shuffle16!(a, b, [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]), + 13 => simd_shuffle16!(a, b, [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]), + 14 => simd_shuffle16!(a, b, [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]), + 15 => simd_shuffle16!(a, b, [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vext_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 3))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vext_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + static_assert_imm2!(N); + match N & 0b11 { + 0 => simd_shuffle4!(a, b, [0, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [1, 2, 3, 4]), + 2 => simd_shuffle4!(a, b, [2, 3, 4, 5]), + 3 => simd_shuffle4!(a, b, [3, 4, 5, 6]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 7))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 7))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vextq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + static_assert_imm3!(N); + match N & 0b111 { + 0 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), + 2 => simd_shuffle8!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), + 3 => simd_shuffle8!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), + 4 => simd_shuffle8!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), + 5 => simd_shuffle8!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), + 6 => simd_shuffle8!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), + 7 => simd_shuffle8!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vext_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vext_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + static_assert_imm1!(N); + match N & 0b1 { + 0 => simd_shuffle2!(a, b, [0, 1]), + 1 => simd_shuffle2!(a, b, [1, 2]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 3))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vextq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + static_assert_imm2!(N); + match N & 0b11 { + 0 => simd_shuffle4!(a, b, [0, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [1, 2, 3, 4]), + 2 => simd_shuffle4!(a, b, [2, 3, 4, 5]), + 3 => simd_shuffle4!(a, b, [3, 4, 5, 6]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vext_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 7))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 7))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vext_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + static_assert_imm3!(N); + match N & 0b111 { + 0 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), + 2 => simd_shuffle8!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), + 3 => simd_shuffle8!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), + 4 => simd_shuffle8!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), + 5 => simd_shuffle8!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), + 6 => simd_shuffle8!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), + 7 => simd_shuffle8!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 15))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 15))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vextq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + static_assert_imm4!(N); + match N & 0b1111 { + 0 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle16!(a, b, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]), + 2 => simd_shuffle16!(a, b, [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]), + 3 => simd_shuffle16!(a, b, [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18]), + 4 => simd_shuffle16!(a, b, [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]), + 5 => simd_shuffle16!(a, b, [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]), + 6 => simd_shuffle16!(a, b, [6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]), + 7 => simd_shuffle16!(a, b, [7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22]), + 8 => simd_shuffle16!(a, b, [8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]), + 9 => simd_shuffle16!(a, b, [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]), + 10 => simd_shuffle16!(a, b, [10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]), + 11 => simd_shuffle16!(a, b, [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]), + 12 => simd_shuffle16!(a, b, [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]), + 13 => simd_shuffle16!(a, b, [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]), + 14 => simd_shuffle16!(a, b, [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]), + 15 => simd_shuffle16!(a, b, [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vext_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 3))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vext_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + static_assert_imm2!(N); + match N & 0b11 { + 0 => simd_shuffle4!(a, b, [0, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [1, 2, 3, 4]), + 2 => simd_shuffle4!(a, b, [2, 3, 4, 5]), + 3 => simd_shuffle4!(a, b, [3, 4, 5, 6]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 7))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 7))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vextq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + static_assert_imm3!(N); + match N & 0b111 { + 0 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), + 2 => simd_shuffle8!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), + 3 => simd_shuffle8!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), + 4 => simd_shuffle8!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), + 5 => simd_shuffle8!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), + 6 => simd_shuffle8!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), + 7 => simd_shuffle8!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vext_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vext_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + static_assert_imm1!(N); + match N & 0b1 { + 0 => simd_shuffle2!(a, b, [0, 1]), + 1 => simd_shuffle2!(a, b, [1, 2]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 3))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vextq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + static_assert_imm2!(N); + match N & 0b11 { + 0 => simd_shuffle4!(a, b, [0, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [1, 2, 3, 4]), + 2 => simd_shuffle4!(a, b, [2, 3, 4, 5]), + 3 => simd_shuffle4!(a, b, [3, 4, 5, 6]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vext_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 7))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 7))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vext_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + static_assert_imm3!(N); + match N & 0b111 { + 0 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), + 2 => simd_shuffle8!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), + 3 => simd_shuffle8!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), + 4 => simd_shuffle8!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), + 5 => simd_shuffle8!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), + 6 => simd_shuffle8!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), + 7 => simd_shuffle8!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 15))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 15))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vextq_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + static_assert_imm4!(N); + match N & 0b1111 { + 0 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle16!(a, b, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]), + 2 => simd_shuffle16!(a, b, [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]), + 3 => simd_shuffle16!(a, b, [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18]), + 4 => simd_shuffle16!(a, b, [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]), + 5 => simd_shuffle16!(a, b, [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]), + 6 => simd_shuffle16!(a, b, [6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]), + 7 => simd_shuffle16!(a, b, [7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22]), + 8 => simd_shuffle16!(a, b, [8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]), + 9 => simd_shuffle16!(a, b, [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]), + 10 => simd_shuffle16!(a, b, [10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]), + 11 => simd_shuffle16!(a, b, [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]), + 12 => simd_shuffle16!(a, b, [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]), + 13 => simd_shuffle16!(a, b, [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]), + 14 => simd_shuffle16!(a, b, [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]), + 15 => simd_shuffle16!(a, b, [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vext_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 3))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vext_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + static_assert_imm2!(N); + match N & 0b11 { + 0 => simd_shuffle4!(a, b, [0, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [1, 2, 3, 4]), + 2 => simd_shuffle4!(a, b, [2, 3, 4, 5]), + 3 => simd_shuffle4!(a, b, [3, 4, 5, 6]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 7))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 7))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vextq_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + static_assert_imm3!(N); + match N & 0b111 { + 0 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), + 2 => simd_shuffle8!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), + 3 => simd_shuffle8!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), + 4 => simd_shuffle8!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), + 5 => simd_shuffle8!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), + 6 => simd_shuffle8!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), + 7 => simd_shuffle8!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vextq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + static_assert_imm1!(N); + match N & 0b1 { + 0 => simd_shuffle2!(a, b, [0, 1]), + 1 => simd_shuffle2!(a, b, [1, 2]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vextq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + static_assert_imm1!(N); + match N & 0b1 { + 0 => simd_shuffle2!(a, b, [0, 1]), + 1 => simd_shuffle2!(a, b, [1, 2]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vext_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vext_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + static_assert_imm1!(N); + match N & 0b1 { + 0 => simd_shuffle2!(a, b, [0, 1]), + 1 => simd_shuffle2!(a, b, [1, 2]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 3))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vextq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + static_assert_imm2!(N); + match N & 0b11 { + 0 => simd_shuffle4!(a, b, [0, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [1, 2, 3, 4]), + 2 => simd_shuffle4!(a, b, [2, 3, 4, 5]), + 3 => simd_shuffle4!(a, b, [3, 4, 5, 6]), + _ => unreachable_unchecked(), + } +} + +/// Multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_s8(a: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_s8(a: int8x16_t, b: int8x16_t, c: int8x16_t) -> int8x16_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_u8(a: uint8x8_t, b: uint8x8_t, c: uint8x8_t) -> uint8x8_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_u8(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x4_t) -> uint16x4_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x2_t) -> uint32x2_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t { + simd_add(a, simd_mul(b, c)) +} + +/// Floating-point multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + simd_add(a, simd_mul(b, c)) +} + +/// Floating-point multiply-add to accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + simd_add(a, simd_mul(b, c)) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_n_s16(a: int16x4_t, b: int16x4_t, c: i16) -> int16x4_t { + vmla_s16(a, b, vdup_n_s16(c)) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_n_s16(a: int16x8_t, b: int16x8_t, c: i16) -> int16x8_t { + vmlaq_s16(a, b, vdupq_n_s16(c)) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_n_s32(a: int32x2_t, b: int32x2_t, c: i32) -> int32x2_t { + vmla_s32(a, b, vdup_n_s32(c)) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_n_s32(a: int32x4_t, b: int32x4_t, c: i32) -> int32x4_t { + vmlaq_s32(a, b, vdupq_n_s32(c)) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_n_u16(a: uint16x4_t, b: uint16x4_t, c: u16) -> uint16x4_t { + vmla_u16(a, b, vdup_n_u16(c)) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_n_u16(a: uint16x8_t, b: uint16x8_t, c: u16) -> uint16x8_t { + vmlaq_u16(a, b, vdupq_n_u16(c)) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_n_u32(a: uint32x2_t, b: uint32x2_t, c: u32) -> uint32x2_t { + vmla_u32(a, b, vdup_n_u32(c)) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_n_u32(a: uint32x4_t, b: uint32x4_t, c: u32) -> uint32x4_t { + vmlaq_u32(a, b, vdupq_n_u32(c)) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_n_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_n_f32(a: float32x2_t, b: float32x2_t, c: f32) -> float32x2_t { + vmla_f32(a, b, vdup_n_f32(c)) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_n_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_n_f32(a: float32x4_t, b: float32x4_t, c: f32) -> float32x4_t { + vmlaq_f32(a, b, vdupq_n_f32(c)) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_lane_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t { + static_assert_imm2!(LANE); + vmla_s16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_laneq_s16(a: int16x4_t, b: int16x4_t, c: int16x8_t) -> int16x4_t { + static_assert_imm3!(LANE); + vmla_s16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_lane_s16(a: int16x8_t, b: int16x8_t, c: int16x4_t) -> int16x8_t { + static_assert_imm2!(LANE); + vmlaq_s16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_laneq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + static_assert_imm3!(LANE); + vmlaq_s16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_lane_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t { + static_assert_imm1!(LANE); + vmla_s32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_laneq_s32(a: int32x2_t, b: int32x2_t, c: int32x4_t) -> int32x2_t { + static_assert_imm2!(LANE); + vmla_s32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_lane_s32(a: int32x4_t, b: int32x4_t, c: int32x2_t) -> int32x4_t { + static_assert_imm1!(LANE); + vmlaq_s32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_laneq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + vmlaq_s32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_lane_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x4_t) -> uint16x4_t { + static_assert_imm2!(LANE); + vmla_u16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_laneq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_laneq_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x8_t) -> uint16x4_t { + static_assert_imm3!(LANE); + vmla_u16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_lane_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x4_t) -> uint16x8_t { + static_assert_imm2!(LANE); + vmlaq_u16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_laneq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_laneq_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t { + static_assert_imm3!(LANE); + vmlaq_u16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_lane_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x2_t) -> uint32x2_t { + static_assert_imm1!(LANE); + vmla_u32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_laneq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_laneq_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x4_t) -> uint32x2_t { + static_assert_imm2!(LANE); + vmla_u32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_lane_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x2_t) -> uint32x4_t { + static_assert_imm1!(LANE); + vmlaq_u32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_laneq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_laneq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + vmlaq_u32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_lane_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + static_assert_imm1!(LANE); + vmla_f32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_laneq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmla_laneq_f32(a: float32x2_t, b: float32x2_t, c: float32x4_t) -> float32x2_t { + static_assert_imm2!(LANE); + vmla_f32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_lane_f32(a: float32x4_t, b: float32x4_t, c: float32x2_t) -> float32x4_t { + static_assert_imm1!(LANE); + vmlaq_f32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_laneq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlaq_laneq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + static_assert_imm2!(LANE); + vmlaq_f32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Signed multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlal))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlal_s8(a: int16x8_t, b: int8x8_t, c: int8x8_t) -> int16x8_t { + simd_add(a, vmull_s8(b, c)) +} + +/// Signed multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlal))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlal_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t { + simd_add(a, vmull_s16(b, c)) +} + +/// Signed multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlal))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlal_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t { + simd_add(a, vmull_s32(b, c)) +} + +/// Unsigned multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlal))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlal_u8(a: uint16x8_t, b: uint8x8_t, c: uint8x8_t) -> uint16x8_t { + simd_add(a, vmull_u8(b, c)) +} + +/// Unsigned multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlal))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlal_u16(a: uint32x4_t, b: uint16x4_t, c: uint16x4_t) -> uint32x4_t { + simd_add(a, vmull_u16(b, c)) +} + +/// Unsigned multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlal))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlal_u32(a: uint64x2_t, b: uint32x2_t, c: uint32x2_t) -> uint64x2_t { + simd_add(a, vmull_u32(b, c)) +} + +/// Vector widening multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlal))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlal_n_s16(a: int32x4_t, b: int16x4_t, c: i16) -> int32x4_t { + vmlal_s16(a, b, vdup_n_s16(c)) +} + +/// Vector widening multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlal))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlal_n_s32(a: int64x2_t, b: int32x2_t, c: i32) -> int64x2_t { + vmlal_s32(a, b, vdup_n_s32(c)) +} + +/// Vector widening multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlal))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlal_n_u16(a: uint32x4_t, b: uint16x4_t, c: u16) -> uint32x4_t { + vmlal_u16(a, b, vdup_n_u16(c)) +} + +/// Vector widening multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlal))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlal_n_u32(a: uint64x2_t, b: uint32x2_t, c: u32) -> uint64x2_t { + vmlal_u32(a, b, vdup_n_u32(c)) +} + +/// Vector widening multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlal, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlal_lane_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + vmlal_s16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector widening multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlal, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlal_laneq_s16(a: int32x4_t, b: int16x4_t, c: int16x8_t) -> int32x4_t { + static_assert_imm3!(LANE); + vmlal_s16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector widening multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlal, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlal_lane_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t { + static_assert_imm1!(LANE); + vmlal_s32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector widening multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlal, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlal_laneq_s32(a: int64x2_t, b: int32x2_t, c: int32x4_t) -> int64x2_t { + static_assert_imm2!(LANE); + vmlal_s32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector widening multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlal, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlal_lane_u16(a: uint32x4_t, b: uint16x4_t, c: uint16x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + vmlal_u16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector widening multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_laneq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlal, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlal_laneq_u16(a: uint32x4_t, b: uint16x4_t, c: uint16x8_t) -> uint32x4_t { + static_assert_imm3!(LANE); + vmlal_u16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector widening multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlal, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlal_lane_u32(a: uint64x2_t, b: uint32x2_t, c: uint32x2_t) -> uint64x2_t { + static_assert_imm1!(LANE); + vmlal_u32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector widening multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_laneq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlal, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlal_laneq_u32(a: uint64x2_t, b: uint32x2_t, c: uint32x4_t) -> uint64x2_t { + static_assert_imm2!(LANE); + vmlal_u32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Multiply-subtract from accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_s8(a: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_s8(a: int8x16_t, b: int8x16_t, c: int8x16_t) -> int8x16_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_u8(a: uint8x8_t, b: uint8x8_t, c: uint8x8_t) -> uint8x8_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_u8(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x4_t) -> uint16x4_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x2_t) -> uint32x2_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Floating-point multiply-subtract from accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Floating-point multiply-subtract from accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_n_s16(a: int16x4_t, b: int16x4_t, c: i16) -> int16x4_t { + vmls_s16(a, b, vdup_n_s16(c)) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_n_s16(a: int16x8_t, b: int16x8_t, c: i16) -> int16x8_t { + vmlsq_s16(a, b, vdupq_n_s16(c)) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_n_s32(a: int32x2_t, b: int32x2_t, c: i32) -> int32x2_t { + vmls_s32(a, b, vdup_n_s32(c)) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_n_s32(a: int32x4_t, b: int32x4_t, c: i32) -> int32x4_t { + vmlsq_s32(a, b, vdupq_n_s32(c)) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_n_u16(a: uint16x4_t, b: uint16x4_t, c: u16) -> uint16x4_t { + vmls_u16(a, b, vdup_n_u16(c)) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_n_u16(a: uint16x8_t, b: uint16x8_t, c: u16) -> uint16x8_t { + vmlsq_u16(a, b, vdupq_n_u16(c)) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_n_u32(a: uint32x2_t, b: uint32x2_t, c: u32) -> uint32x2_t { + vmls_u32(a, b, vdup_n_u32(c)) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_n_u32(a: uint32x4_t, b: uint32x4_t, c: u32) -> uint32x4_t { + vmlsq_u32(a, b, vdupq_n_u32(c)) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_n_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_n_f32(a: float32x2_t, b: float32x2_t, c: f32) -> float32x2_t { + vmls_f32(a, b, vdup_n_f32(c)) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_n_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_n_f32(a: float32x4_t, b: float32x4_t, c: f32) -> float32x4_t { + vmlsq_f32(a, b, vdupq_n_f32(c)) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_lane_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t { + static_assert_imm2!(LANE); + vmls_s16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_laneq_s16(a: int16x4_t, b: int16x4_t, c: int16x8_t) -> int16x4_t { + static_assert_imm3!(LANE); + vmls_s16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_lane_s16(a: int16x8_t, b: int16x8_t, c: int16x4_t) -> int16x8_t { + static_assert_imm2!(LANE); + vmlsq_s16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_laneq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + static_assert_imm3!(LANE); + vmlsq_s16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_lane_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t { + static_assert_imm1!(LANE); + vmls_s32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_laneq_s32(a: int32x2_t, b: int32x2_t, c: int32x4_t) -> int32x2_t { + static_assert_imm2!(LANE); + vmls_s32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_lane_s32(a: int32x4_t, b: int32x4_t, c: int32x2_t) -> int32x4_t { + static_assert_imm1!(LANE); + vmlsq_s32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_laneq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + vmlsq_s32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_lane_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x4_t) -> uint16x4_t { + static_assert_imm2!(LANE); + vmls_u16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_laneq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_laneq_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x8_t) -> uint16x4_t { + static_assert_imm3!(LANE); + vmls_u16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_lane_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x4_t) -> uint16x8_t { + static_assert_imm2!(LANE); + vmlsq_u16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_laneq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_laneq_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t { + static_assert_imm3!(LANE); + vmlsq_u16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_lane_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x2_t) -> uint32x2_t { + static_assert_imm1!(LANE); + vmls_u32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_laneq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_laneq_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x4_t) -> uint32x2_t { + static_assert_imm2!(LANE); + vmls_u32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_lane_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x2_t) -> uint32x4_t { + static_assert_imm1!(LANE); + vmlsq_u32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_laneq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_laneq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + vmlsq_u32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_lane_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + static_assert_imm1!(LANE); + vmls_f32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_laneq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmls_laneq_f32(a: float32x2_t, b: float32x2_t, c: float32x4_t) -> float32x2_t { + static_assert_imm2!(LANE); + vmls_f32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_lane_f32(a: float32x4_t, b: float32x4_t, c: float32x2_t) -> float32x4_t { + static_assert_imm1!(LANE); + vmlsq_f32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_laneq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsq_laneq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + static_assert_imm2!(LANE); + vmlsq_f32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Signed multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlsl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsl_s8(a: int16x8_t, b: int8x8_t, c: int8x8_t) -> int16x8_t { + simd_sub(a, vmull_s8(b, c)) +} + +/// Signed multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlsl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsl_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t { + simd_sub(a, vmull_s16(b, c)) +} + +/// Signed multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlsl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsl_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t { + simd_sub(a, vmull_s32(b, c)) +} + +/// Unsigned multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlsl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsl_u8(a: uint16x8_t, b: uint8x8_t, c: uint8x8_t) -> uint16x8_t { + simd_sub(a, vmull_u8(b, c)) +} + +/// Unsigned multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlsl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsl_u16(a: uint32x4_t, b: uint16x4_t, c: uint16x4_t) -> uint32x4_t { + simd_sub(a, vmull_u16(b, c)) +} + +/// Unsigned multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlsl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsl_u32(a: uint64x2_t, b: uint32x2_t, c: uint32x2_t) -> uint64x2_t { + simd_sub(a, vmull_u32(b, c)) +} + +/// Vector widening multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlsl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsl_n_s16(a: int32x4_t, b: int16x4_t, c: i16) -> int32x4_t { + vmlsl_s16(a, b, vdup_n_s16(c)) +} + +/// Vector widening multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlsl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsl_n_s32(a: int64x2_t, b: int32x2_t, c: i32) -> int64x2_t { + vmlsl_s32(a, b, vdup_n_s32(c)) +} + +/// Vector widening multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlsl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsl_n_u16(a: uint32x4_t, b: uint16x4_t, c: u16) -> uint32x4_t { + vmlsl_u16(a, b, vdup_n_u16(c)) +} + +/// Vector widening multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlsl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsl_n_u32(a: uint64x2_t, b: uint32x2_t, c: u32) -> uint64x2_t { + vmlsl_u32(a, b, vdup_n_u32(c)) +} + +/// Vector widening multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlsl, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsl_lane_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + vmlsl_s16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector widening multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlsl, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsl_laneq_s16(a: int32x4_t, b: int16x4_t, c: int16x8_t) -> int32x4_t { + static_assert_imm3!(LANE); + vmlsl_s16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector widening multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlsl, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsl_lane_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t { + static_assert_imm1!(LANE); + vmlsl_s32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector widening multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlsl, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsl_laneq_s32(a: int64x2_t, b: int32x2_t, c: int32x4_t) -> int64x2_t { + static_assert_imm2!(LANE); + vmlsl_s32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector widening multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlsl, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsl_lane_u16(a: uint32x4_t, b: uint16x4_t, c: uint16x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + vmlsl_u16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector widening multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_laneq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlsl, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsl_laneq_u16(a: uint32x4_t, b: uint16x4_t, c: uint16x8_t) -> uint32x4_t { + static_assert_imm3!(LANE); + vmlsl_u16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector widening multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlsl, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsl_lane_u32(a: uint64x2_t, b: uint32x2_t, c: uint32x2_t) -> uint64x2_t { + static_assert_imm1!(LANE); + vmlsl_u32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector widening multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_laneq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlsl, LANE = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmlsl_laneq_u32(a: uint64x2_t, b: uint32x2_t, c: uint32x4_t) -> uint64x2_t { + static_assert_imm2!(LANE); + vmlsl_u32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vneg_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(neg))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vneg_s8(a: int8x8_t) -> int8x8_t { + simd_neg(a) +} + +/// Negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vnegq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(neg))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vnegq_s8(a: int8x16_t) -> int8x16_t { + simd_neg(a) +} + +/// Negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vneg_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(neg))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vneg_s16(a: int16x4_t) -> int16x4_t { + simd_neg(a) +} + +/// Negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vnegq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(neg))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vnegq_s16(a: int16x8_t) -> int16x8_t { + simd_neg(a) +} + +/// Negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vneg_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(neg))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vneg_s32(a: int32x2_t) -> int32x2_t { + simd_neg(a) +} + +/// Negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vnegq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(neg))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vnegq_s32(a: int32x4_t) -> int32x4_t { + simd_neg(a) +} + +/// Negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vneg_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fneg))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vneg_f32(a: float32x2_t) -> float32x2_t { + simd_neg(a) +} + +/// Negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vnegq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fneg))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vnegq_f32(a: float32x4_t) -> float32x4_t { + simd_neg(a) +} + +/// Signed saturating negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqneg_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqneg.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqneg))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqneg_s8(a: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqneg.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqneg.v8i8")] + fn vqneg_s8_(a: int8x8_t) -> int8x8_t; + } +vqneg_s8_(a) +} + +/// Signed saturating negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqnegq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqneg.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqneg))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqnegq_s8(a: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqneg.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqneg.v16i8")] + fn vqnegq_s8_(a: int8x16_t) -> int8x16_t; + } +vqnegq_s8_(a) +} + +/// Signed saturating negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqneg_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqneg.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqneg))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqneg_s16(a: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqneg.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqneg.v4i16")] + fn vqneg_s16_(a: int16x4_t) -> int16x4_t; + } +vqneg_s16_(a) +} + +/// Signed saturating negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqnegq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqneg.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqneg))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqnegq_s16(a: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqneg.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqneg.v8i16")] + fn vqnegq_s16_(a: int16x8_t) -> int16x8_t; + } +vqnegq_s16_(a) +} + +/// Signed saturating negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqneg_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqneg.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqneg))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqneg_s32(a: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqneg.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqneg.v2i32")] + fn vqneg_s32_(a: int32x2_t) -> int32x2_t; + } +vqneg_s32_(a) +} + +/// Signed saturating negate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqnegq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqneg.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqneg))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqnegq_s32(a: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqneg.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqneg.v4i32")] + fn vqnegq_s32_(a: int32x4_t) -> int32x4_t; + } +vqnegq_s32_(a) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsub_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqsub_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.usub.sat.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v8i8")] + fn vqsub_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; + } +vqsub_u8_(a, b) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqsubq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.usub.sat.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v16i8")] + fn vqsubq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; + } +vqsubq_u8_(a, b) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsub_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqsub_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.usub.sat.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v4i16")] + fn vqsub_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; + } +vqsub_u16_(a, b) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqsubq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.usub.sat.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v8i16")] + fn vqsubq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; + } +vqsubq_u16_(a, b) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsub_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqsub_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.usub.sat.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v2i32")] + fn vqsub_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; + } +vqsub_u32_(a, b) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqsubq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.usub.sat.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v4i32")] + fn vqsubq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; + } +vqsubq_u32_(a, b) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsub_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqsub_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.usub.sat.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v1i64")] + fn vqsub_u64_(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t; + } +vqsub_u64_(a, b) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqsubq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.usub.sat.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v2i64")] + fn vqsubq_u64_(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t; + } +vqsubq_u64_(a, b) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsub_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqsub_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.ssub.sat.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v8i8")] + fn vqsub_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vqsub_s8_(a, b) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqsubq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.ssub.sat.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v16i8")] + fn vqsubq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vqsubq_s8_(a, b) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsub_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqsub_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.ssub.sat.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v4i16")] + fn vqsub_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vqsub_s16_(a, b) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqsubq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.ssub.sat.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v8i16")] + fn vqsubq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vqsubq_s16_(a, b) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsub_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqsub_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.ssub.sat.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v2i32")] + fn vqsub_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vqsub_s32_(a, b) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqsubq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.ssub.sat.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v4i32")] + fn vqsubq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vqsubq_s32_(a, b) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsub_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqsub_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.ssub.sat.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v1i64")] + fn vqsub_s64_(a: int64x1_t, b: int64x1_t) -> int64x1_t; + } +vqsub_s64_(a, b) +} + +/// Saturating subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqsubq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.ssub.sat.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v2i64")] + fn vqsubq_s64_(a: int64x2_t, b: int64x2_t) -> int64x2_t; + } +vqsubq_s64_(a, b) +} + +/// Halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhadd_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhadd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhaddu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhadd.v8i8")] + fn vhadd_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; + } +vhadd_u8_(a, b) +} + +/// Halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhaddq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhaddq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhaddu.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhadd.v16i8")] + fn vhaddq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; + } +vhaddq_u8_(a, b) +} + +/// Halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhadd_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhadd_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhaddu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhadd.v4i16")] + fn vhadd_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; + } +vhadd_u16_(a, b) +} + +/// Halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhaddq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhaddq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhaddu.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhadd.v8i16")] + fn vhaddq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; + } +vhaddq_u16_(a, b) +} + +/// Halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhadd_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhadd_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhaddu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhadd.v2i32")] + fn vhadd_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; + } +vhadd_u32_(a, b) +} + +/// Halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhaddq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhaddq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhaddu.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhadd.v4i32")] + fn vhaddq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; + } +vhaddq_u32_(a, b) +} + +/// Halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhadd_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhadd_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhadds.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shadd.v8i8")] + fn vhadd_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vhadd_s8_(a, b) +} + +/// Halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhaddq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhaddq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhadds.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shadd.v16i8")] + fn vhaddq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vhaddq_s8_(a, b) +} + +/// Halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhadd_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhadd_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhadds.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shadd.v4i16")] + fn vhadd_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vhadd_s16_(a, b) +} + +/// Halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhaddq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhaddq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhadds.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shadd.v8i16")] + fn vhaddq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vhaddq_s16_(a, b) +} + +/// Halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhadd_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhadd_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhadds.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shadd.v2i32")] + fn vhadd_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vhadd_s32_(a, b) +} + +/// Halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhaddq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhaddq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhadds.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shadd.v4i32")] + fn vhaddq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vhaddq_s32_(a, b) +} + +/// Rounding halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhadd_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urhadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrhadd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhaddu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urhadd.v8i8")] + fn vrhadd_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; + } +vrhadd_u8_(a, b) +} + +/// Rounding halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhaddq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urhadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrhaddq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhaddu.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urhadd.v16i8")] + fn vrhaddq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; + } +vrhaddq_u8_(a, b) +} + +/// Rounding halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhadd_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urhadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrhadd_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhaddu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urhadd.v4i16")] + fn vrhadd_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; + } +vrhadd_u16_(a, b) +} + +/// Rounding halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhaddq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urhadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrhaddq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhaddu.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urhadd.v8i16")] + fn vrhaddq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; + } +vrhaddq_u16_(a, b) +} + +/// Rounding halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhadd_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urhadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrhadd_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhaddu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urhadd.v2i32")] + fn vrhadd_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; + } +vrhadd_u32_(a, b) +} + +/// Rounding halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhaddq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urhadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrhaddq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhaddu.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urhadd.v4i32")] + fn vrhaddq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; + } +vrhaddq_u32_(a, b) +} + +/// Rounding halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhadd_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srhadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrhadd_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhadds.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srhadd.v8i8")] + fn vrhadd_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vrhadd_s8_(a, b) +} + +/// Rounding halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhaddq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srhadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrhaddq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhadds.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srhadd.v16i8")] + fn vrhaddq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vrhaddq_s8_(a, b) +} + +/// Rounding halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhadd_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srhadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrhadd_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhadds.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srhadd.v4i16")] + fn vrhadd_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vrhadd_s16_(a, b) +} + +/// Rounding halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhaddq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srhadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrhaddq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhadds.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srhadd.v8i16")] + fn vrhaddq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vrhaddq_s16_(a, b) +} + +/// Rounding halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhadd_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srhadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrhadd_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhadds.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srhadd.v2i32")] + fn vrhadd_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vrhadd_s32_(a, b) +} + +/// Rounding halving add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhaddq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srhadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrhaddq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhadds.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srhadd.v4i32")] + fn vrhaddq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vrhaddq_s32_(a, b) +} + +/// Floating-point round to integral, to nearest with ties to even +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndn_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrintn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(frintn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrndn_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrintn.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frintn.v2f32")] + fn vrndn_f32_(a: float32x2_t) -> float32x2_t; + } +vrndn_f32_(a) +} + +/// Floating-point round to integral, to nearest with ties to even +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndnq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrintn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(frintn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrndnq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrintn.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frintn.v4f32")] + fn vrndnq_f32_(a: float32x4_t) -> float32x4_t; + } +vrndnq_f32_(a) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqadd_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqadd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.uadd.sat.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v8i8")] + fn vqadd_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; + } +vqadd_u8_(a, b) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqaddq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.uadd.sat.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v16i8")] + fn vqaddq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; + } +vqaddq_u8_(a, b) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqadd_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqadd_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.uadd.sat.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v4i16")] + fn vqadd_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; + } +vqadd_u16_(a, b) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqaddq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.uadd.sat.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v8i16")] + fn vqaddq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; + } +vqaddq_u16_(a, b) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqadd_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqadd_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.uadd.sat.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v2i32")] + fn vqadd_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; + } +vqadd_u32_(a, b) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqaddq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.uadd.sat.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v4i32")] + fn vqaddq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; + } +vqaddq_u32_(a, b) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqadd_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqadd_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.uadd.sat.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v1i64")] + fn vqadd_u64_(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t; + } +vqadd_u64_(a, b) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqaddq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.uadd.sat.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v2i64")] + fn vqaddq_u64_(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t; + } +vqaddq_u64_(a, b) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqadd_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqadd_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.sadd.sat.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v8i8")] + fn vqadd_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vqadd_s8_(a, b) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqaddq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.sadd.sat.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v16i8")] + fn vqaddq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vqaddq_s8_(a, b) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqadd_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqadd_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.sadd.sat.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v4i16")] + fn vqadd_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vqadd_s16_(a, b) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqaddq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.sadd.sat.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v8i16")] + fn vqaddq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vqaddq_s16_(a, b) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqadd_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqadd_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.sadd.sat.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v2i32")] + fn vqadd_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vqadd_s32_(a, b) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqaddq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.sadd.sat.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v4i32")] + fn vqaddq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vqaddq_s32_(a, b) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqadd_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqadd_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.sadd.sat.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v1i64")] + fn vqadd_s64_(a: int64x1_t, b: int64x1_t) -> int64x1_t; + } +vqadd_s64_(a, b) +} + +/// Saturating add +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqaddq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.sadd.sat.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v2i64")] + fn vqaddq_s64_(a: int64x2_t, b: int64x2_t) -> int64x2_t; + } +vqaddq_s64_(a, b) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s8_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_s8_x2(a: *const i8) -> int8x8x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x2.v8i8.p0i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x2.v8i8.p0i8")] + fn vld1_s8_x2_(a: *const i8) -> int8x8x2_t; + } +vld1_s8_x2_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s16_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_s16_x2(a: *const i16) -> int16x4x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x2.v4i16.p0i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x2.v4i16.p0i16")] + fn vld1_s16_x2_(a: *const i16) -> int16x4x2_t; + } +vld1_s16_x2_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s32_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_s32_x2(a: *const i32) -> int32x2x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x2.v2i32.p0i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x2.v2i32.p0i32")] + fn vld1_s32_x2_(a: *const i32) -> int32x2x2_t; + } +vld1_s32_x2_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s64_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_s64_x2(a: *const i64) -> int64x1x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x2.v1i64.p0i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x2.v1i64.p0i64")] + fn vld1_s64_x2_(a: *const i64) -> int64x1x2_t; + } +vld1_s64_x2_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s8_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_s8_x2(a: *const i8) -> int8x16x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x2.v16i8.p0i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x2.v16i8.p0i8")] + fn vld1q_s8_x2_(a: *const i8) -> int8x16x2_t; + } +vld1q_s8_x2_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s16_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_s16_x2(a: *const i16) -> int16x8x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x2.v8i16.p0i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x2.v8i16.p0i16")] + fn vld1q_s16_x2_(a: *const i16) -> int16x8x2_t; + } +vld1q_s16_x2_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s32_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_s32_x2(a: *const i32) -> int32x4x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x2.v4i32.p0i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x2.v4i32.p0i32")] + fn vld1q_s32_x2_(a: *const i32) -> int32x4x2_t; + } +vld1q_s32_x2_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s64_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_s64_x2(a: *const i64) -> int64x2x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x2.v2i64.p0i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x2.v2i64.p0i64")] + fn vld1q_s64_x2_(a: *const i64) -> int64x2x2_t; + } +vld1q_s64_x2_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s8_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_s8_x3(a: *const i8) -> int8x8x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x3.v8i8.p0i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x3.v8i8.p0i8")] + fn vld1_s8_x3_(a: *const i8) -> int8x8x3_t; + } +vld1_s8_x3_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s16_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_s16_x3(a: *const i16) -> int16x4x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x3.v4i16.p0i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x3.v4i16.p0i16")] + fn vld1_s16_x3_(a: *const i16) -> int16x4x3_t; + } +vld1_s16_x3_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s32_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_s32_x3(a: *const i32) -> int32x2x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x3.v2i32.p0i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x3.v2i32.p0i32")] + fn vld1_s32_x3_(a: *const i32) -> int32x2x3_t; + } +vld1_s32_x3_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s64_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_s64_x3(a: *const i64) -> int64x1x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x3.v1i64.p0i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x3.v1i64.p0i64")] + fn vld1_s64_x3_(a: *const i64) -> int64x1x3_t; + } +vld1_s64_x3_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s8_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_s8_x3(a: *const i8) -> int8x16x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x3.v16i8.p0i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x3.v16i8.p0i8")] + fn vld1q_s8_x3_(a: *const i8) -> int8x16x3_t; + } +vld1q_s8_x3_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s16_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_s16_x3(a: *const i16) -> int16x8x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x3.v8i16.p0i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x3.v8i16.p0i16")] + fn vld1q_s16_x3_(a: *const i16) -> int16x8x3_t; + } +vld1q_s16_x3_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s32_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_s32_x3(a: *const i32) -> int32x4x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x3.v4i32.p0i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x3.v4i32.p0i32")] + fn vld1q_s32_x3_(a: *const i32) -> int32x4x3_t; + } +vld1q_s32_x3_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s64_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_s64_x3(a: *const i64) -> int64x2x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x3.v2i64.p0i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x3.v2i64.p0i64")] + fn vld1q_s64_x3_(a: *const i64) -> int64x2x3_t; + } +vld1q_s64_x3_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s8_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_s8_x4(a: *const i8) -> int8x8x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x4.v8i8.p0i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x4.v8i8.p0i8")] + fn vld1_s8_x4_(a: *const i8) -> int8x8x4_t; + } +vld1_s8_x4_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s16_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_s16_x4(a: *const i16) -> int16x4x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x4.v4i16.p0i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x4.v4i16.p0i16")] + fn vld1_s16_x4_(a: *const i16) -> int16x4x4_t; + } +vld1_s16_x4_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s32_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_s32_x4(a: *const i32) -> int32x2x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x4.v2i32.p0i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x4.v2i32.p0i32")] + fn vld1_s32_x4_(a: *const i32) -> int32x2x4_t; + } +vld1_s32_x4_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s64_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_s64_x4(a: *const i64) -> int64x1x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x4.v1i64.p0i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x4.v1i64.p0i64")] + fn vld1_s64_x4_(a: *const i64) -> int64x1x4_t; + } +vld1_s64_x4_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s8_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_s8_x4(a: *const i8) -> int8x16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x4.v16i8.p0i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x4.v16i8.p0i8")] + fn vld1q_s8_x4_(a: *const i8) -> int8x16x4_t; + } +vld1q_s8_x4_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s16_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_s16_x4(a: *const i16) -> int16x8x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x4.v8i16.p0i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x4.v8i16.p0i16")] + fn vld1q_s16_x4_(a: *const i16) -> int16x8x4_t; + } +vld1q_s16_x4_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s32_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_s32_x4(a: *const i32) -> int32x4x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x4.v4i32.p0i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x4.v4i32.p0i32")] + fn vld1q_s32_x4_(a: *const i32) -> int32x4x4_t; + } +vld1q_s32_x4_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s64_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_s64_x4(a: *const i64) -> int64x2x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x4.v2i64.p0i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x4.v2i64.p0i64")] + fn vld1q_s64_x4_(a: *const i64) -> int64x2x4_t; + } +vld1q_s64_x4_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u8_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_u8_x2(a: *const u8) -> uint8x8x2_t { + transmute(vld1_s8_x2(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u16_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_u16_x2(a: *const u16) -> uint16x4x2_t { + transmute(vld1_s16_x2(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u32_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_u32_x2(a: *const u32) -> uint32x2x2_t { + transmute(vld1_s32_x2(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u64_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_u64_x2(a: *const u64) -> uint64x1x2_t { + transmute(vld1_s64_x2(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u8_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_u8_x2(a: *const u8) -> uint8x16x2_t { + transmute(vld1q_s8_x2(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u16_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_u16_x2(a: *const u16) -> uint16x8x2_t { + transmute(vld1q_s16_x2(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u32_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_u32_x2(a: *const u32) -> uint32x4x2_t { + transmute(vld1q_s32_x2(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u64_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_u64_x2(a: *const u64) -> uint64x2x2_t { + transmute(vld1q_s64_x2(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u8_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_u8_x3(a: *const u8) -> uint8x8x3_t { + transmute(vld1_s8_x3(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u16_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_u16_x3(a: *const u16) -> uint16x4x3_t { + transmute(vld1_s16_x3(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u32_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_u32_x3(a: *const u32) -> uint32x2x3_t { + transmute(vld1_s32_x3(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u64_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_u64_x3(a: *const u64) -> uint64x1x3_t { + transmute(vld1_s64_x3(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u8_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_u8_x3(a: *const u8) -> uint8x16x3_t { + transmute(vld1q_s8_x3(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u16_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_u16_x3(a: *const u16) -> uint16x8x3_t { + transmute(vld1q_s16_x3(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u32_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_u32_x3(a: *const u32) -> uint32x4x3_t { + transmute(vld1q_s32_x3(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u64_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_u64_x3(a: *const u64) -> uint64x2x3_t { + transmute(vld1q_s64_x3(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u8_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_u8_x4(a: *const u8) -> uint8x8x4_t { + transmute(vld1_s8_x4(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u16_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_u16_x4(a: *const u16) -> uint16x4x4_t { + transmute(vld1_s16_x4(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u32_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_u32_x4(a: *const u32) -> uint32x2x4_t { + transmute(vld1_s32_x4(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u64_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_u64_x4(a: *const u64) -> uint64x1x4_t { + transmute(vld1_s64_x4(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u8_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_u8_x4(a: *const u8) -> uint8x16x4_t { + transmute(vld1q_s8_x4(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u16_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_u16_x4(a: *const u16) -> uint16x8x4_t { + transmute(vld1q_s16_x4(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u32_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_u32_x4(a: *const u32) -> uint32x4x4_t { + transmute(vld1q_s32_x4(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u64_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_u64_x4(a: *const u64) -> uint64x2x4_t { + transmute(vld1q_s64_x4(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p8_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_p8_x2(a: *const p8) -> poly8x8x2_t { + transmute(vld1_s8_x2(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p8_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_p8_x3(a: *const p8) -> poly8x8x3_t { + transmute(vld1_s8_x3(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p8_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_p8_x4(a: *const p8) -> poly8x8x4_t { + transmute(vld1_s8_x4(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p8_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_p8_x2(a: *const p8) -> poly8x16x2_t { + transmute(vld1q_s8_x2(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p8_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_p8_x3(a: *const p8) -> poly8x16x3_t { + transmute(vld1q_s8_x3(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p8_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_p8_x4(a: *const p8) -> poly8x16x4_t { + transmute(vld1q_s8_x4(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p16_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_p16_x2(a: *const p16) -> poly16x4x2_t { + transmute(vld1_s16_x2(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p16_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_p16_x3(a: *const p16) -> poly16x4x3_t { + transmute(vld1_s16_x3(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p16_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_p16_x4(a: *const p16) -> poly16x4x4_t { + transmute(vld1_s16_x4(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p16_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_p16_x2(a: *const p16) -> poly16x8x2_t { + transmute(vld1q_s16_x2(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p16_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_p16_x3(a: *const p16) -> poly16x8x3_t { + transmute(vld1q_s16_x3(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p16_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_p16_x4(a: *const p16) -> poly16x8x4_t { + transmute(vld1q_s16_x4(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p64_x2) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_p64_x2(a: *const p64) -> poly64x1x2_t { + transmute(vld1_s64_x2(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p64_x3) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_p64_x3(a: *const p64) -> poly64x1x3_t { + transmute(vld1_s64_x3(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p64_x4) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_p64_x4(a: *const p64) -> poly64x1x4_t { + transmute(vld1_s64_x4(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p64_x2) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_p64_x2(a: *const p64) -> poly64x2x2_t { + transmute(vld1q_s64_x2(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p64_x3) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_p64_x3(a: *const p64) -> poly64x2x3_t { + transmute(vld1q_s64_x3(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p64_x4) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_p64_x4(a: *const p64) -> poly64x2x4_t { + transmute(vld1q_s64_x4(transmute(a))) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_f32_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_f32_x2(a: *const f32) -> float32x2x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x2.v2f32.p0f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x2.v2f32.p0f32")] + fn vld1_f32_x2_(a: *const f32) -> float32x2x2_t; + } +vld1_f32_x2_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_f32_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_f32_x2(a: *const f32) -> float32x4x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x2.v4f32.p0f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x2.v4f32.p0f32")] + fn vld1q_f32_x2_(a: *const f32) -> float32x4x2_t; + } +vld1q_f32_x2_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_f32_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_f32_x3(a: *const f32) -> float32x2x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x3.v2f32.p0f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x3.v2f32.p0f32")] + fn vld1_f32_x3_(a: *const f32) -> float32x2x3_t; + } +vld1_f32_x3_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_f32_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_f32_x3(a: *const f32) -> float32x4x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x3.v4f32.p0f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x3.v4f32.p0f32")] + fn vld1q_f32_x3_(a: *const f32) -> float32x4x3_t; + } +vld1q_f32_x3_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_f32_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1_f32_x4(a: *const f32) -> float32x2x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x4.v2f32.p0f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x4.v2f32.p0f32")] + fn vld1_f32_x4_(a: *const f32) -> float32x2x4_t; + } +vld1_f32_x4_(a) +} + +/// Load multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_f32_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld1q_f32_x4(a: *const f32) -> float32x4x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld1x4.v4f32.p0f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld1x4.v4f32.p0f32")] + fn vld1q_f32_x4_(a: *const f32) -> float32x4x4_t; + } +vld1q_f32_x4_(a) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2))] +pub unsafe fn vld2_s8(a: *const i8) -> int8x8x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2.v8i8.p0i8")] + fn vld2_s8_(ptr: *const i8, size: i32) -> int8x8x2_t; + } +vld2_s8_(a as *const i8, 1) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2_s8(a: *const i8) -> int8x8x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2.v8i8.p0v8i8")] + fn vld2_s8_(ptr: *const int8x8_t) -> int8x8x2_t; + } +vld2_s8_(a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2))] +pub unsafe fn vld2_s16(a: *const i16) -> int16x4x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2.v4i16.p0i8")] + fn vld2_s16_(ptr: *const i8, size: i32) -> int16x4x2_t; + } +vld2_s16_(a as *const i8, 2) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2_s16(a: *const i16) -> int16x4x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2.v4i16.p0v4i16")] + fn vld2_s16_(ptr: *const int16x4_t) -> int16x4x2_t; + } +vld2_s16_(a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2))] +pub unsafe fn vld2_s32(a: *const i32) -> int32x2x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2.v2i32.p0i8")] + fn vld2_s32_(ptr: *const i8, size: i32) -> int32x2x2_t; + } +vld2_s32_(a as *const i8, 4) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2_s32(a: *const i32) -> int32x2x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2.v2i32.p0v2i32")] + fn vld2_s32_(ptr: *const int32x2_t) -> int32x2x2_t; + } +vld2_s32_(a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2))] +pub unsafe fn vld2q_s8(a: *const i8) -> int8x16x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2.v16i8.p0i8")] + fn vld2q_s8_(ptr: *const i8, size: i32) -> int8x16x2_t; + } +vld2q_s8_(a as *const i8, 1) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_s8(a: *const i8) -> int8x16x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2.v16i8.p0v16i8")] + fn vld2q_s8_(ptr: *const int8x16_t) -> int8x16x2_t; + } +vld2q_s8_(a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2))] +pub unsafe fn vld2q_s16(a: *const i16) -> int16x8x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2.v8i16.p0i8")] + fn vld2q_s16_(ptr: *const i8, size: i32) -> int16x8x2_t; + } +vld2q_s16_(a as *const i8, 2) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_s16(a: *const i16) -> int16x8x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2.v8i16.p0v8i16")] + fn vld2q_s16_(ptr: *const int16x8_t) -> int16x8x2_t; + } +vld2q_s16_(a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2))] +pub unsafe fn vld2q_s32(a: *const i32) -> int32x4x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2.v4i32.p0i8")] + fn vld2q_s32_(ptr: *const i8, size: i32) -> int32x4x2_t; + } +vld2q_s32_(a as *const i8, 4) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_s32(a: *const i32) -> int32x4x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2.v4i32.p0v4i32")] + fn vld2q_s32_(ptr: *const int32x4_t) -> int32x4x2_t; + } +vld2q_s32_(a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_s64) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(nop))] +pub unsafe fn vld2_s64(a: *const i64) -> int64x1x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2.v1i64.p0i8")] + fn vld2_s64_(ptr: *const i8, size: i32) -> int64x1x2_t; + } +vld2_s64_(a as *const i8, 8) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_s64) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2_s64(a: *const i64) -> int64x1x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2.v1i64.p0v1i64")] + fn vld2_s64_(ptr: *const int64x1_t) -> int64x1x2_t; + } +vld2_s64_(a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2_u8(a: *const u8) -> uint8x8x2_t { + transmute(vld2_s8(transmute(a))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2_u16(a: *const u16) -> uint16x4x2_t { + transmute(vld2_s16(transmute(a))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2_u32(a: *const u32) -> uint32x2x2_t { + transmute(vld2_s32(transmute(a))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2q_u8(a: *const u8) -> uint8x16x2_t { + transmute(vld2q_s8(transmute(a))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2q_u16(a: *const u16) -> uint16x8x2_t { + transmute(vld2q_s16(transmute(a))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2q_u32(a: *const u32) -> uint32x4x2_t { + transmute(vld2q_s32(transmute(a))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2_p8(a: *const p8) -> poly8x8x2_t { + transmute(vld2_s8(transmute(a))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2_p16(a: *const p16) -> poly16x4x2_t { + transmute(vld2_s16(transmute(a))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2q_p8(a: *const p8) -> poly8x16x2_t { + transmute(vld2q_s8(transmute(a))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2q_p16(a: *const p16) -> poly16x8x2_t { + transmute(vld2q_s16(transmute(a))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2_u64(a: *const u64) -> uint64x1x2_t { + transmute(vld2_s64(transmute(a))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2_p64(a: *const p64) -> poly64x1x2_t { + transmute(vld2_s64(transmute(a))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2))] +pub unsafe fn vld2_f32(a: *const f32) -> float32x2x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2.v2f32.p0i8")] + fn vld2_f32_(ptr: *const i8, size: i32) -> float32x2x2_t; + } +vld2_f32_(a as *const i8, 4) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2_f32(a: *const f32) -> float32x2x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2.v2f32.p0v2f32")] + fn vld2_f32_(ptr: *const float32x2_t) -> float32x2x2_t; + } +vld2_f32_(a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2))] +pub unsafe fn vld2q_f32(a: *const f32) -> float32x4x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2.v4f32.p0i8")] + fn vld2q_f32_(ptr: *const i8, size: i32) -> float32x4x2_t; + } +vld2q_f32_(a as *const i8, 4) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_f32(a: *const f32) -> float32x4x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2.v4f32.p0v4f32")] + fn vld2q_f32_(ptr: *const float32x4_t) -> float32x4x2_t; + } +vld2q_f32_(a as _) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2))] +pub unsafe fn vld2_dup_s8(a: *const i8) -> int8x8x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2dup.v8i8.p0i8")] + fn vld2_dup_s8_(ptr: *const i8, size: i32) -> int8x8x2_t; + } +vld2_dup_s8_(a as *const i8, 1) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2_dup_s8(a: *const i8) -> int8x8x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2r.v8i8.p0i8")] + fn vld2_dup_s8_(ptr: *const i8) -> int8x8x2_t; + } +vld2_dup_s8_(a as _) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2))] +pub unsafe fn vld2_dup_s16(a: *const i16) -> int16x4x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2dup.v4i16.p0i8")] + fn vld2_dup_s16_(ptr: *const i8, size: i32) -> int16x4x2_t; + } +vld2_dup_s16_(a as *const i8, 2) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2_dup_s16(a: *const i16) -> int16x4x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2r.v4i16.p0i16")] + fn vld2_dup_s16_(ptr: *const i16) -> int16x4x2_t; + } +vld2_dup_s16_(a as _) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2))] +pub unsafe fn vld2_dup_s32(a: *const i32) -> int32x2x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2dup.v2i32.p0i8")] + fn vld2_dup_s32_(ptr: *const i8, size: i32) -> int32x2x2_t; + } +vld2_dup_s32_(a as *const i8, 4) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2_dup_s32(a: *const i32) -> int32x2x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2r.v2i32.p0i32")] + fn vld2_dup_s32_(ptr: *const i32) -> int32x2x2_t; + } +vld2_dup_s32_(a as _) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2))] +pub unsafe fn vld2q_dup_s8(a: *const i8) -> int8x16x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2dup.v16i8.p0i8")] + fn vld2q_dup_s8_(ptr: *const i8, size: i32) -> int8x16x2_t; + } +vld2q_dup_s8_(a as *const i8, 1) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_dup_s8(a: *const i8) -> int8x16x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2r.v16i8.p0i8")] + fn vld2q_dup_s8_(ptr: *const i8) -> int8x16x2_t; + } +vld2q_dup_s8_(a as _) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2))] +pub unsafe fn vld2q_dup_s16(a: *const i16) -> int16x8x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2dup.v8i16.p0i8")] + fn vld2q_dup_s16_(ptr: *const i8, size: i32) -> int16x8x2_t; + } +vld2q_dup_s16_(a as *const i8, 2) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_dup_s16(a: *const i16) -> int16x8x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2r.v8i16.p0i16")] + fn vld2q_dup_s16_(ptr: *const i16) -> int16x8x2_t; + } +vld2q_dup_s16_(a as _) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2))] +pub unsafe fn vld2q_dup_s32(a: *const i32) -> int32x4x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2dup.v4i32.p0i8")] + fn vld2q_dup_s32_(ptr: *const i8, size: i32) -> int32x4x2_t; + } +vld2q_dup_s32_(a as *const i8, 4) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_dup_s32(a: *const i32) -> int32x4x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2r.v4i32.p0i32")] + fn vld2q_dup_s32_(ptr: *const i32) -> int32x4x2_t; + } +vld2q_dup_s32_(a as _) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_s64) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(nop))] +pub unsafe fn vld2_dup_s64(a: *const i64) -> int64x1x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2dup.v1i64.p0i8")] + fn vld2_dup_s64_(ptr: *const i8, size: i32) -> int64x1x2_t; + } +vld2_dup_s64_(a as *const i8, 8) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_s64) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2_dup_s64(a: *const i64) -> int64x1x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2r.v1i64.p0i64")] + fn vld2_dup_s64_(ptr: *const i64) -> int64x1x2_t; + } +vld2_dup_s64_(a as _) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2_dup_u8(a: *const u8) -> uint8x8x2_t { + transmute(vld2_dup_s8(transmute(a))) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2_dup_u16(a: *const u16) -> uint16x4x2_t { + transmute(vld2_dup_s16(transmute(a))) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2_dup_u32(a: *const u32) -> uint32x2x2_t { + transmute(vld2_dup_s32(transmute(a))) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2q_dup_u8(a: *const u8) -> uint8x16x2_t { + transmute(vld2q_dup_s8(transmute(a))) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2q_dup_u16(a: *const u16) -> uint16x8x2_t { + transmute(vld2q_dup_s16(transmute(a))) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2q_dup_u32(a: *const u32) -> uint32x4x2_t { + transmute(vld2q_dup_s32(transmute(a))) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2_dup_p8(a: *const p8) -> poly8x8x2_t { + transmute(vld2_dup_s8(transmute(a))) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2_dup_p16(a: *const p16) -> poly16x4x2_t { + transmute(vld2_dup_s16(transmute(a))) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2q_dup_p8(a: *const p8) -> poly8x16x2_t { + transmute(vld2q_dup_s8(transmute(a))) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2q_dup_p16(a: *const p16) -> poly16x8x2_t { + transmute(vld2q_dup_s16(transmute(a))) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2_dup_u64(a: *const u64) -> uint64x1x2_t { + transmute(vld2_dup_s64(transmute(a))) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2_dup_p64(a: *const p64) -> poly64x1x2_t { + transmute(vld2_dup_s64(transmute(a))) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2))] +pub unsafe fn vld2_dup_f32(a: *const f32) -> float32x2x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2dup.v2f32.p0i8")] + fn vld2_dup_f32_(ptr: *const i8, size: i32) -> float32x2x2_t; + } +vld2_dup_f32_(a as *const i8, 4) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2_dup_f32(a: *const f32) -> float32x2x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2r.v2f32.p0f32")] + fn vld2_dup_f32_(ptr: *const f32) -> float32x2x2_t; + } +vld2_dup_f32_(a as _) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2))] +pub unsafe fn vld2q_dup_f32(a: *const f32) -> float32x4x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2dup.v4f32.p0i8")] + fn vld2q_dup_f32_(ptr: *const i8, size: i32) -> float32x4x2_t; + } +vld2q_dup_f32_(a as *const i8, 4) +} + +/// Load single 2-element structure and replicate to all lanes of two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_dup_f32(a: *const f32) -> float32x4x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2r.v4f32.p0f32")] + fn vld2q_dup_f32_(ptr: *const f32) -> float32x4x2_t; + } +vld2q_dup_f32_(a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vld2_lane_s8(a: *const i8, b: int8x8x2_t) -> int8x8x2_t { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2lane.v8i8.p0i8")] + fn vld2_lane_s8_(ptr: *const i8, a: int8x8_t, b: int8x8_t, n: i32, size: i32) -> int8x8x2_t; + } +vld2_lane_s8_(a as _, b.0, b.1, LANE, 1) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2_lane_s8(a: *const i8, b: int8x8x2_t) -> int8x8x2_t { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2lane.v8i8.p0i8")] + fn vld2_lane_s8_(a: int8x8_t, b: int8x8_t, n: i64, ptr: *const i8) -> int8x8x2_t; + } +vld2_lane_s8_(b.0, b.1, LANE as i64, a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vld2_lane_s16(a: *const i16, b: int16x4x2_t) -> int16x4x2_t { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2lane.v4i16.p0i8")] + fn vld2_lane_s16_(ptr: *const i8, a: int16x4_t, b: int16x4_t, n: i32, size: i32) -> int16x4x2_t; + } +vld2_lane_s16_(a as _, b.0, b.1, LANE, 2) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2_lane_s16(a: *const i16, b: int16x4x2_t) -> int16x4x2_t { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2lane.v4i16.p0i8")] + fn vld2_lane_s16_(a: int16x4_t, b: int16x4_t, n: i64, ptr: *const i8) -> int16x4x2_t; + } +vld2_lane_s16_(b.0, b.1, LANE as i64, a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vld2_lane_s32(a: *const i32, b: int32x2x2_t) -> int32x2x2_t { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2lane.v2i32.p0i8")] + fn vld2_lane_s32_(ptr: *const i8, a: int32x2_t, b: int32x2_t, n: i32, size: i32) -> int32x2x2_t; + } +vld2_lane_s32_(a as _, b.0, b.1, LANE, 4) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2_lane_s32(a: *const i32, b: int32x2x2_t) -> int32x2x2_t { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2lane.v2i32.p0i8")] + fn vld2_lane_s32_(a: int32x2_t, b: int32x2_t, n: i64, ptr: *const i8) -> int32x2x2_t; + } +vld2_lane_s32_(b.0, b.1, LANE as i64, a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vld2q_lane_s16(a: *const i16, b: int16x8x2_t) -> int16x8x2_t { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2lane.v8i16.p0i8")] + fn vld2q_lane_s16_(ptr: *const i8, a: int16x8_t, b: int16x8_t, n: i32, size: i32) -> int16x8x2_t; + } +vld2q_lane_s16_(a as _, b.0, b.1, LANE, 2) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_lane_s16(a: *const i16, b: int16x8x2_t) -> int16x8x2_t { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2lane.v8i16.p0i8")] + fn vld2q_lane_s16_(a: int16x8_t, b: int16x8_t, n: i64, ptr: *const i8) -> int16x8x2_t; + } +vld2q_lane_s16_(b.0, b.1, LANE as i64, a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vld2q_lane_s32(a: *const i32, b: int32x4x2_t) -> int32x4x2_t { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2lane.v4i32.p0i8")] + fn vld2q_lane_s32_(ptr: *const i8, a: int32x4_t, b: int32x4_t, n: i32, size: i32) -> int32x4x2_t; + } +vld2q_lane_s32_(a as _, b.0, b.1, LANE, 4) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_lane_s32(a: *const i32, b: int32x4x2_t) -> int32x4x2_t { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2lane.v4i32.p0i8")] + fn vld2q_lane_s32_(a: int32x4_t, b: int32x4_t, n: i64, ptr: *const i8) -> int32x4x2_t; + } +vld2q_lane_s32_(b.0, b.1, LANE as i64, a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2_lane_u8(a: *const u8, b: uint8x8x2_t) -> uint8x8x2_t { + static_assert_imm3!(LANE); + transmute(vld2_lane_s8::(transmute(a), transmute(b))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2_lane_u16(a: *const u16, b: uint16x4x2_t) -> uint16x4x2_t { + static_assert_imm2!(LANE); + transmute(vld2_lane_s16::(transmute(a), transmute(b))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2_lane_u32(a: *const u32, b: uint32x2x2_t) -> uint32x2x2_t { + static_assert_imm1!(LANE); + transmute(vld2_lane_s32::(transmute(a), transmute(b))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2q_lane_u16(a: *const u16, b: uint16x8x2_t) -> uint16x8x2_t { + static_assert_imm3!(LANE); + transmute(vld2q_lane_s16::(transmute(a), transmute(b))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2q_lane_u32(a: *const u32, b: uint32x4x2_t) -> uint32x4x2_t { + static_assert_imm2!(LANE); + transmute(vld2q_lane_s32::(transmute(a), transmute(b))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2_lane_p8(a: *const p8, b: poly8x8x2_t) -> poly8x8x2_t { + static_assert_imm3!(LANE); + transmute(vld2_lane_s8::(transmute(a), transmute(b))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2_lane_p16(a: *const p16, b: poly16x4x2_t) -> poly16x4x2_t { + static_assert_imm2!(LANE); + transmute(vld2_lane_s16::(transmute(a), transmute(b))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld2q_lane_p16(a: *const p16, b: poly16x8x2_t) -> poly16x8x2_t { + static_assert_imm3!(LANE); + transmute(vld2q_lane_s16::(transmute(a), transmute(b))) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vld2_lane_f32(a: *const f32, b: float32x2x2_t) -> float32x2x2_t { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2lane.v2f32.p0i8")] + fn vld2_lane_f32_(ptr: *const i8, a: float32x2_t, b: float32x2_t, n: i32, size: i32) -> float32x2x2_t; + } +vld2_lane_f32_(a as _, b.0, b.1, LANE, 4) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2_lane_f32(a: *const f32, b: float32x2x2_t) -> float32x2x2_t { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2lane.v2f32.p0i8")] + fn vld2_lane_f32_(a: float32x2_t, b: float32x2_t, n: i64, ptr: *const i8) -> float32x2x2_t; + } +vld2_lane_f32_(b.0, b.1, LANE as i64, a as _) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vld2q_lane_f32(a: *const f32, b: float32x4x2_t) -> float32x4x2_t { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld2lane.v4f32.p0i8")] + fn vld2q_lane_f32_(ptr: *const i8, a: float32x4_t, b: float32x4_t, n: i32, size: i32) -> float32x4x2_t; + } +vld2q_lane_f32_(a as _, b.0, b.1, LANE, 4) +} + +/// Load multiple 2-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld2q_lane_f32(a: *const f32, b: float32x4x2_t) -> float32x4x2_t { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld2lane.v4f32.p0i8")] + fn vld2q_lane_f32_(a: float32x4_t, b: float32x4_t, n: i64, ptr: *const i8) -> float32x4x2_t; + } +vld2q_lane_f32_(b.0, b.1, LANE as i64, a as _) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3))] +pub unsafe fn vld3_s8(a: *const i8) -> int8x8x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3.v8i8.p0i8")] + fn vld3_s8_(ptr: *const i8, size: i32) -> int8x8x3_t; + } +vld3_s8_(a as *const i8, 1) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3_s8(a: *const i8) -> int8x8x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3.v8i8.p0v8i8")] + fn vld3_s8_(ptr: *const int8x8_t) -> int8x8x3_t; + } +vld3_s8_(a as _) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3))] +pub unsafe fn vld3_s16(a: *const i16) -> int16x4x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3.v4i16.p0i8")] + fn vld3_s16_(ptr: *const i8, size: i32) -> int16x4x3_t; + } +vld3_s16_(a as *const i8, 2) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3_s16(a: *const i16) -> int16x4x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3.v4i16.p0v4i16")] + fn vld3_s16_(ptr: *const int16x4_t) -> int16x4x3_t; + } +vld3_s16_(a as _) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3))] +pub unsafe fn vld3_s32(a: *const i32) -> int32x2x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3.v2i32.p0i8")] + fn vld3_s32_(ptr: *const i8, size: i32) -> int32x2x3_t; + } +vld3_s32_(a as *const i8, 4) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3_s32(a: *const i32) -> int32x2x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3.v2i32.p0v2i32")] + fn vld3_s32_(ptr: *const int32x2_t) -> int32x2x3_t; + } +vld3_s32_(a as _) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3))] +pub unsafe fn vld3q_s8(a: *const i8) -> int8x16x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3.v16i8.p0i8")] + fn vld3q_s8_(ptr: *const i8, size: i32) -> int8x16x3_t; + } +vld3q_s8_(a as *const i8, 1) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_s8(a: *const i8) -> int8x16x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3.v16i8.p0v16i8")] + fn vld3q_s8_(ptr: *const int8x16_t) -> int8x16x3_t; + } +vld3q_s8_(a as _) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3))] +pub unsafe fn vld3q_s16(a: *const i16) -> int16x8x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3.v8i16.p0i8")] + fn vld3q_s16_(ptr: *const i8, size: i32) -> int16x8x3_t; + } +vld3q_s16_(a as *const i8, 2) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_s16(a: *const i16) -> int16x8x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3.v8i16.p0v8i16")] + fn vld3q_s16_(ptr: *const int16x8_t) -> int16x8x3_t; + } +vld3q_s16_(a as _) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3))] +pub unsafe fn vld3q_s32(a: *const i32) -> int32x4x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3.v4i32.p0i8")] + fn vld3q_s32_(ptr: *const i8, size: i32) -> int32x4x3_t; + } +vld3q_s32_(a as *const i8, 4) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_s32(a: *const i32) -> int32x4x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3.v4i32.p0v4i32")] + fn vld3q_s32_(ptr: *const int32x4_t) -> int32x4x3_t; + } +vld3q_s32_(a as _) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_s64) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(nop))] +pub unsafe fn vld3_s64(a: *const i64) -> int64x1x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3.v1i64.p0i8")] + fn vld3_s64_(ptr: *const i8, size: i32) -> int64x1x3_t; + } +vld3_s64_(a as *const i8, 8) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_s64) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3_s64(a: *const i64) -> int64x1x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3.v1i64.p0v1i64")] + fn vld3_s64_(ptr: *const int64x1_t) -> int64x1x3_t; + } +vld3_s64_(a as _) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3_u8(a: *const u8) -> uint8x8x3_t { + transmute(vld3_s8(transmute(a))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3_u16(a: *const u16) -> uint16x4x3_t { + transmute(vld3_s16(transmute(a))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3_u32(a: *const u32) -> uint32x2x3_t { + transmute(vld3_s32(transmute(a))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3q_u8(a: *const u8) -> uint8x16x3_t { + transmute(vld3q_s8(transmute(a))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3q_u16(a: *const u16) -> uint16x8x3_t { + transmute(vld3q_s16(transmute(a))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3q_u32(a: *const u32) -> uint32x4x3_t { + transmute(vld3q_s32(transmute(a))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3_p8(a: *const p8) -> poly8x8x3_t { + transmute(vld3_s8(transmute(a))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3_p16(a: *const p16) -> poly16x4x3_t { + transmute(vld3_s16(transmute(a))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3q_p8(a: *const p8) -> poly8x16x3_t { + transmute(vld3q_s8(transmute(a))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3q_p16(a: *const p16) -> poly16x8x3_t { + transmute(vld3q_s16(transmute(a))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3_u64(a: *const u64) -> uint64x1x3_t { + transmute(vld3_s64(transmute(a))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3_p64(a: *const p64) -> poly64x1x3_t { + transmute(vld3_s64(transmute(a))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3))] +pub unsafe fn vld3_f32(a: *const f32) -> float32x2x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3.v2f32.p0i8")] + fn vld3_f32_(ptr: *const i8, size: i32) -> float32x2x3_t; + } +vld3_f32_(a as *const i8, 4) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3_f32(a: *const f32) -> float32x2x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3.v2f32.p0v2f32")] + fn vld3_f32_(ptr: *const float32x2_t) -> float32x2x3_t; + } +vld3_f32_(a as _) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3))] +pub unsafe fn vld3q_f32(a: *const f32) -> float32x4x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3.v4f32.p0i8")] + fn vld3q_f32_(ptr: *const i8, size: i32) -> float32x4x3_t; + } +vld3q_f32_(a as *const i8, 4) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_f32(a: *const f32) -> float32x4x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3.v4f32.p0v4f32")] + fn vld3q_f32_(ptr: *const float32x4_t) -> float32x4x3_t; + } +vld3q_f32_(a as _) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3))] +pub unsafe fn vld3_dup_s8(a: *const i8) -> int8x8x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3dup.v8i8.p0i8")] + fn vld3_dup_s8_(ptr: *const i8, size: i32) -> int8x8x3_t; + } +vld3_dup_s8_(a as *const i8, 1) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3_dup_s8(a: *const i8) -> int8x8x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3r.v8i8.p0i8")] + fn vld3_dup_s8_(ptr: *const i8) -> int8x8x3_t; + } +vld3_dup_s8_(a as _) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3))] +pub unsafe fn vld3_dup_s16(a: *const i16) -> int16x4x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3dup.v4i16.p0i8")] + fn vld3_dup_s16_(ptr: *const i8, size: i32) -> int16x4x3_t; + } +vld3_dup_s16_(a as *const i8, 2) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3_dup_s16(a: *const i16) -> int16x4x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3r.v4i16.p0i16")] + fn vld3_dup_s16_(ptr: *const i16) -> int16x4x3_t; + } +vld3_dup_s16_(a as _) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3))] +pub unsafe fn vld3_dup_s32(a: *const i32) -> int32x2x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3dup.v2i32.p0i8")] + fn vld3_dup_s32_(ptr: *const i8, size: i32) -> int32x2x3_t; + } +vld3_dup_s32_(a as *const i8, 4) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3_dup_s32(a: *const i32) -> int32x2x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3r.v2i32.p0i32")] + fn vld3_dup_s32_(ptr: *const i32) -> int32x2x3_t; + } +vld3_dup_s32_(a as _) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3))] +pub unsafe fn vld3q_dup_s8(a: *const i8) -> int8x16x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3dup.v16i8.p0i8")] + fn vld3q_dup_s8_(ptr: *const i8, size: i32) -> int8x16x3_t; + } +vld3q_dup_s8_(a as *const i8, 1) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_dup_s8(a: *const i8) -> int8x16x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3r.v16i8.p0i8")] + fn vld3q_dup_s8_(ptr: *const i8) -> int8x16x3_t; + } +vld3q_dup_s8_(a as _) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3))] +pub unsafe fn vld3q_dup_s16(a: *const i16) -> int16x8x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3dup.v8i16.p0i8")] + fn vld3q_dup_s16_(ptr: *const i8, size: i32) -> int16x8x3_t; + } +vld3q_dup_s16_(a as *const i8, 2) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_dup_s16(a: *const i16) -> int16x8x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3r.v8i16.p0i16")] + fn vld3q_dup_s16_(ptr: *const i16) -> int16x8x3_t; + } +vld3q_dup_s16_(a as _) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3))] +pub unsafe fn vld3q_dup_s32(a: *const i32) -> int32x4x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3dup.v4i32.p0i8")] + fn vld3q_dup_s32_(ptr: *const i8, size: i32) -> int32x4x3_t; + } +vld3q_dup_s32_(a as *const i8, 4) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_dup_s32(a: *const i32) -> int32x4x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3r.v4i32.p0i32")] + fn vld3q_dup_s32_(ptr: *const i32) -> int32x4x3_t; + } +vld3q_dup_s32_(a as _) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_s64) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(nop))] +pub unsafe fn vld3_dup_s64(a: *const i64) -> int64x1x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3dup.v1i64.p0i8")] + fn vld3_dup_s64_(ptr: *const i8, size: i32) -> int64x1x3_t; + } +vld3_dup_s64_(a as *const i8, 8) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_s64) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3_dup_s64(a: *const i64) -> int64x1x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3r.v1i64.p0i64")] + fn vld3_dup_s64_(ptr: *const i64) -> int64x1x3_t; + } +vld3_dup_s64_(a as _) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3_dup_u8(a: *const u8) -> uint8x8x3_t { + transmute(vld3_dup_s8(transmute(a))) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3_dup_u16(a: *const u16) -> uint16x4x3_t { + transmute(vld3_dup_s16(transmute(a))) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3_dup_u32(a: *const u32) -> uint32x2x3_t { + transmute(vld3_dup_s32(transmute(a))) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3q_dup_u8(a: *const u8) -> uint8x16x3_t { + transmute(vld3q_dup_s8(transmute(a))) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3q_dup_u16(a: *const u16) -> uint16x8x3_t { + transmute(vld3q_dup_s16(transmute(a))) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3q_dup_u32(a: *const u32) -> uint32x4x3_t { + transmute(vld3q_dup_s32(transmute(a))) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3_dup_p8(a: *const p8) -> poly8x8x3_t { + transmute(vld3_dup_s8(transmute(a))) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3_dup_p16(a: *const p16) -> poly16x4x3_t { + transmute(vld3_dup_s16(transmute(a))) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3q_dup_p8(a: *const p8) -> poly8x16x3_t { + transmute(vld3q_dup_s8(transmute(a))) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3q_dup_p16(a: *const p16) -> poly16x8x3_t { + transmute(vld3q_dup_s16(transmute(a))) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3_dup_u64(a: *const u64) -> uint64x1x3_t { + transmute(vld3_dup_s64(transmute(a))) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3_dup_p64(a: *const p64) -> poly64x1x3_t { + transmute(vld3_dup_s64(transmute(a))) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3))] +pub unsafe fn vld3_dup_f32(a: *const f32) -> float32x2x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3dup.v2f32.p0i8")] + fn vld3_dup_f32_(ptr: *const i8, size: i32) -> float32x2x3_t; + } +vld3_dup_f32_(a as *const i8, 4) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3_dup_f32(a: *const f32) -> float32x2x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3r.v2f32.p0f32")] + fn vld3_dup_f32_(ptr: *const f32) -> float32x2x3_t; + } +vld3_dup_f32_(a as _) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3))] +pub unsafe fn vld3q_dup_f32(a: *const f32) -> float32x4x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3dup.v4f32.p0i8")] + fn vld3q_dup_f32_(ptr: *const i8, size: i32) -> float32x4x3_t; + } +vld3q_dup_f32_(a as *const i8, 4) +} + +/// Load single 3-element structure and replicate to all lanes of three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_dup_f32(a: *const f32) -> float32x4x3_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3r.v4f32.p0f32")] + fn vld3q_dup_f32_(ptr: *const f32) -> float32x4x3_t; + } +vld3q_dup_f32_(a as _) +} + +/// Load multiple 3-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vld3_lane_s8(a: *const i8, b: int8x8x3_t) -> int8x8x3_t { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3lane.v8i8.p0i8")] + fn vld3_lane_s8_(ptr: *const i8, a: int8x8_t, b: int8x8_t, c: int8x8_t, n: i32, size: i32) -> int8x8x3_t; + } +vld3_lane_s8_(a as _, b.0, b.1, b.2, LANE, 1) +} + +/// Load multiple 3-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3_lane_s8(a: *const i8, b: int8x8x3_t) -> int8x8x3_t { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3lane.v8i8.p0i8")] + fn vld3_lane_s8_(a: int8x8_t, b: int8x8_t, c: int8x8_t, n: i64, ptr: *const i8) -> int8x8x3_t; + } +vld3_lane_s8_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Load multiple 3-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vld3_lane_s16(a: *const i16, b: int16x4x3_t) -> int16x4x3_t { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3lane.v4i16.p0i8")] + fn vld3_lane_s16_(ptr: *const i8, a: int16x4_t, b: int16x4_t, c: int16x4_t, n: i32, size: i32) -> int16x4x3_t; + } +vld3_lane_s16_(a as _, b.0, b.1, b.2, LANE, 2) +} + +/// Load multiple 3-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3_lane_s16(a: *const i16, b: int16x4x3_t) -> int16x4x3_t { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3lane.v4i16.p0i8")] + fn vld3_lane_s16_(a: int16x4_t, b: int16x4_t, c: int16x4_t, n: i64, ptr: *const i8) -> int16x4x3_t; + } +vld3_lane_s16_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Load multiple 3-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vld3_lane_s32(a: *const i32, b: int32x2x3_t) -> int32x2x3_t { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3lane.v2i32.p0i8")] + fn vld3_lane_s32_(ptr: *const i8, a: int32x2_t, b: int32x2_t, c: int32x2_t, n: i32, size: i32) -> int32x2x3_t; + } +vld3_lane_s32_(a as _, b.0, b.1, b.2, LANE, 4) +} + +/// Load multiple 3-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3_lane_s32(a: *const i32, b: int32x2x3_t) -> int32x2x3_t { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3lane.v2i32.p0i8")] + fn vld3_lane_s32_(a: int32x2_t, b: int32x2_t, c: int32x2_t, n: i64, ptr: *const i8) -> int32x2x3_t; + } +vld3_lane_s32_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Load multiple 3-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vld3q_lane_s16(a: *const i16, b: int16x8x3_t) -> int16x8x3_t { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3lane.v8i16.p0i8")] + fn vld3q_lane_s16_(ptr: *const i8, a: int16x8_t, b: int16x8_t, c: int16x8_t, n: i32, size: i32) -> int16x8x3_t; + } +vld3q_lane_s16_(a as _, b.0, b.1, b.2, LANE, 2) +} + +/// Load multiple 3-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_lane_s16(a: *const i16, b: int16x8x3_t) -> int16x8x3_t { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3lane.v8i16.p0i8")] + fn vld3q_lane_s16_(a: int16x8_t, b: int16x8_t, c: int16x8_t, n: i64, ptr: *const i8) -> int16x8x3_t; + } +vld3q_lane_s16_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Load multiple 3-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vld3q_lane_s32(a: *const i32, b: int32x4x3_t) -> int32x4x3_t { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3lane.v4i32.p0i8")] + fn vld3q_lane_s32_(ptr: *const i8, a: int32x4_t, b: int32x4_t, c: int32x4_t, n: i32, size: i32) -> int32x4x3_t; + } +vld3q_lane_s32_(a as _, b.0, b.1, b.2, LANE, 4) +} + +/// Load multiple 3-element structures to two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_lane_s32(a: *const i32, b: int32x4x3_t) -> int32x4x3_t { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3lane.v4i32.p0i8")] + fn vld3q_lane_s32_(a: int32x4_t, b: int32x4_t, c: int32x4_t, n: i64, ptr: *const i8) -> int32x4x3_t; + } +vld3q_lane_s32_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3_lane_u8(a: *const u8, b: uint8x8x3_t) -> uint8x8x3_t { + static_assert_imm3!(LANE); + transmute(vld3_lane_s8::(transmute(a), transmute(b))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3_lane_u16(a: *const u16, b: uint16x4x3_t) -> uint16x4x3_t { + static_assert_imm2!(LANE); + transmute(vld3_lane_s16::(transmute(a), transmute(b))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3_lane_u32(a: *const u32, b: uint32x2x3_t) -> uint32x2x3_t { + static_assert_imm1!(LANE); + transmute(vld3_lane_s32::(transmute(a), transmute(b))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3q_lane_u16(a: *const u16, b: uint16x8x3_t) -> uint16x8x3_t { + static_assert_imm3!(LANE); + transmute(vld3q_lane_s16::(transmute(a), transmute(b))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3q_lane_u32(a: *const u32, b: uint32x4x3_t) -> uint32x4x3_t { + static_assert_imm2!(LANE); + transmute(vld3q_lane_s32::(transmute(a), transmute(b))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3_lane_p8(a: *const p8, b: poly8x8x3_t) -> poly8x8x3_t { + static_assert_imm3!(LANE); + transmute(vld3_lane_s8::(transmute(a), transmute(b))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3_lane_p16(a: *const p16, b: poly16x4x3_t) -> poly16x4x3_t { + static_assert_imm2!(LANE); + transmute(vld3_lane_s16::(transmute(a), transmute(b))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld3q_lane_p16(a: *const p16, b: poly16x8x3_t) -> poly16x8x3_t { + static_assert_imm3!(LANE); + transmute(vld3q_lane_s16::(transmute(a), transmute(b))) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vld3_lane_f32(a: *const f32, b: float32x2x3_t) -> float32x2x3_t { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3lane.v2f32.p0i8")] + fn vld3_lane_f32_(ptr: *const i8, a: float32x2_t, b: float32x2_t, c: float32x2_t, n: i32, size: i32) -> float32x2x3_t; + } +vld3_lane_f32_(a as _, b.0, b.1, b.2, LANE, 4) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3_lane_f32(a: *const f32, b: float32x2x3_t) -> float32x2x3_t { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3lane.v2f32.p0i8")] + fn vld3_lane_f32_(a: float32x2_t, b: float32x2_t, c: float32x2_t, n: i64, ptr: *const i8) -> float32x2x3_t; + } +vld3_lane_f32_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vld3q_lane_f32(a: *const f32, b: float32x4x3_t) -> float32x4x3_t { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld3lane.v4f32.p0i8")] + fn vld3q_lane_f32_(ptr: *const i8, a: float32x4_t, b: float32x4_t, c: float32x4_t, n: i32, size: i32) -> float32x4x3_t; + } +vld3q_lane_f32_(a as _, b.0, b.1, b.2, LANE, 4) +} + +/// Load multiple 3-element structures to three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld3q_lane_f32(a: *const f32, b: float32x4x3_t) -> float32x4x3_t { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld3lane.v4f32.p0i8")] + fn vld3q_lane_f32_(a: float32x4_t, b: float32x4_t, c: float32x4_t, n: i64, ptr: *const i8) -> float32x4x3_t; + } +vld3q_lane_f32_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4))] +pub unsafe fn vld4_s8(a: *const i8) -> int8x8x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4.v8i8.p0i8")] + fn vld4_s8_(ptr: *const i8, size: i32) -> int8x8x4_t; + } +vld4_s8_(a as *const i8, 1) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4_s8(a: *const i8) -> int8x8x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4.v8i8.p0v8i8")] + fn vld4_s8_(ptr: *const int8x8_t) -> int8x8x4_t; + } +vld4_s8_(a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4))] +pub unsafe fn vld4_s16(a: *const i16) -> int16x4x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4.v4i16.p0i8")] + fn vld4_s16_(ptr: *const i8, size: i32) -> int16x4x4_t; + } +vld4_s16_(a as *const i8, 2) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4_s16(a: *const i16) -> int16x4x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4.v4i16.p0v4i16")] + fn vld4_s16_(ptr: *const int16x4_t) -> int16x4x4_t; + } +vld4_s16_(a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4))] +pub unsafe fn vld4_s32(a: *const i32) -> int32x2x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4.v2i32.p0i8")] + fn vld4_s32_(ptr: *const i8, size: i32) -> int32x2x4_t; + } +vld4_s32_(a as *const i8, 4) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4_s32(a: *const i32) -> int32x2x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4.v2i32.p0v2i32")] + fn vld4_s32_(ptr: *const int32x2_t) -> int32x2x4_t; + } +vld4_s32_(a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4))] +pub unsafe fn vld4q_s8(a: *const i8) -> int8x16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4.v16i8.p0i8")] + fn vld4q_s8_(ptr: *const i8, size: i32) -> int8x16x4_t; + } +vld4q_s8_(a as *const i8, 1) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_s8(a: *const i8) -> int8x16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4.v16i8.p0v16i8")] + fn vld4q_s8_(ptr: *const int8x16_t) -> int8x16x4_t; + } +vld4q_s8_(a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4))] +pub unsafe fn vld4q_s16(a: *const i16) -> int16x8x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4.v8i16.p0i8")] + fn vld4q_s16_(ptr: *const i8, size: i32) -> int16x8x4_t; + } +vld4q_s16_(a as *const i8, 2) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_s16(a: *const i16) -> int16x8x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4.v8i16.p0v8i16")] + fn vld4q_s16_(ptr: *const int16x8_t) -> int16x8x4_t; + } +vld4q_s16_(a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4))] +pub unsafe fn vld4q_s32(a: *const i32) -> int32x4x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4.v4i32.p0i8")] + fn vld4q_s32_(ptr: *const i8, size: i32) -> int32x4x4_t; + } +vld4q_s32_(a as *const i8, 4) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_s32(a: *const i32) -> int32x4x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4.v4i32.p0v4i32")] + fn vld4q_s32_(ptr: *const int32x4_t) -> int32x4x4_t; + } +vld4q_s32_(a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_s64) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(nop))] +pub unsafe fn vld4_s64(a: *const i64) -> int64x1x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4.v1i64.p0i8")] + fn vld4_s64_(ptr: *const i8, size: i32) -> int64x1x4_t; + } +vld4_s64_(a as *const i8, 8) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_s64) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4_s64(a: *const i64) -> int64x1x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4.v1i64.p0v1i64")] + fn vld4_s64_(ptr: *const int64x1_t) -> int64x1x4_t; + } +vld4_s64_(a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4_u8(a: *const u8) -> uint8x8x4_t { + transmute(vld4_s8(transmute(a))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4_u16(a: *const u16) -> uint16x4x4_t { + transmute(vld4_s16(transmute(a))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4_u32(a: *const u32) -> uint32x2x4_t { + transmute(vld4_s32(transmute(a))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4q_u8(a: *const u8) -> uint8x16x4_t { + transmute(vld4q_s8(transmute(a))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4q_u16(a: *const u16) -> uint16x8x4_t { + transmute(vld4q_s16(transmute(a))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4q_u32(a: *const u32) -> uint32x4x4_t { + transmute(vld4q_s32(transmute(a))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4_p8(a: *const p8) -> poly8x8x4_t { + transmute(vld4_s8(transmute(a))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4_p16(a: *const p16) -> poly16x4x4_t { + transmute(vld4_s16(transmute(a))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4q_p8(a: *const p8) -> poly8x16x4_t { + transmute(vld4q_s8(transmute(a))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4q_p16(a: *const p16) -> poly16x8x4_t { + transmute(vld4q_s16(transmute(a))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4_u64(a: *const u64) -> uint64x1x4_t { + transmute(vld4_s64(transmute(a))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4_p64(a: *const p64) -> poly64x1x4_t { + transmute(vld4_s64(transmute(a))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4))] +pub unsafe fn vld4_f32(a: *const f32) -> float32x2x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4.v2f32.p0i8")] + fn vld4_f32_(ptr: *const i8, size: i32) -> float32x2x4_t; + } +vld4_f32_(a as *const i8, 4) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4_f32(a: *const f32) -> float32x2x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4.v2f32.p0v2f32")] + fn vld4_f32_(ptr: *const float32x2_t) -> float32x2x4_t; + } +vld4_f32_(a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4))] +pub unsafe fn vld4q_f32(a: *const f32) -> float32x4x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4.v4f32.p0i8")] + fn vld4q_f32_(ptr: *const i8, size: i32) -> float32x4x4_t; + } +vld4q_f32_(a as *const i8, 4) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_f32(a: *const f32) -> float32x4x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4.v4f32.p0v4f32")] + fn vld4q_f32_(ptr: *const float32x4_t) -> float32x4x4_t; + } +vld4q_f32_(a as _) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4))] +pub unsafe fn vld4_dup_s8(a: *const i8) -> int8x8x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4dup.v8i8.p0i8")] + fn vld4_dup_s8_(ptr: *const i8, size: i32) -> int8x8x4_t; + } +vld4_dup_s8_(a as *const i8, 1) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4_dup_s8(a: *const i8) -> int8x8x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4r.v8i8.p0i8")] + fn vld4_dup_s8_(ptr: *const i8) -> int8x8x4_t; + } +vld4_dup_s8_(a as _) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4))] +pub unsafe fn vld4_dup_s16(a: *const i16) -> int16x4x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4dup.v4i16.p0i8")] + fn vld4_dup_s16_(ptr: *const i8, size: i32) -> int16x4x4_t; + } +vld4_dup_s16_(a as *const i8, 2) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4_dup_s16(a: *const i16) -> int16x4x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4r.v4i16.p0i16")] + fn vld4_dup_s16_(ptr: *const i16) -> int16x4x4_t; + } +vld4_dup_s16_(a as _) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4))] +pub unsafe fn vld4_dup_s32(a: *const i32) -> int32x2x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4dup.v2i32.p0i8")] + fn vld4_dup_s32_(ptr: *const i8, size: i32) -> int32x2x4_t; + } +vld4_dup_s32_(a as *const i8, 4) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4_dup_s32(a: *const i32) -> int32x2x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4r.v2i32.p0i32")] + fn vld4_dup_s32_(ptr: *const i32) -> int32x2x4_t; + } +vld4_dup_s32_(a as _) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4))] +pub unsafe fn vld4q_dup_s8(a: *const i8) -> int8x16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4dup.v16i8.p0i8")] + fn vld4q_dup_s8_(ptr: *const i8, size: i32) -> int8x16x4_t; + } +vld4q_dup_s8_(a as *const i8, 1) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_dup_s8(a: *const i8) -> int8x16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4r.v16i8.p0i8")] + fn vld4q_dup_s8_(ptr: *const i8) -> int8x16x4_t; + } +vld4q_dup_s8_(a as _) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4))] +pub unsafe fn vld4q_dup_s16(a: *const i16) -> int16x8x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4dup.v8i16.p0i8")] + fn vld4q_dup_s16_(ptr: *const i8, size: i32) -> int16x8x4_t; + } +vld4q_dup_s16_(a as *const i8, 2) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_dup_s16(a: *const i16) -> int16x8x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4r.v8i16.p0i16")] + fn vld4q_dup_s16_(ptr: *const i16) -> int16x8x4_t; + } +vld4q_dup_s16_(a as _) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4))] +pub unsafe fn vld4q_dup_s32(a: *const i32) -> int32x4x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4dup.v4i32.p0i8")] + fn vld4q_dup_s32_(ptr: *const i8, size: i32) -> int32x4x4_t; + } +vld4q_dup_s32_(a as *const i8, 4) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_dup_s32(a: *const i32) -> int32x4x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4r.v4i32.p0i32")] + fn vld4q_dup_s32_(ptr: *const i32) -> int32x4x4_t; + } +vld4q_dup_s32_(a as _) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_s64) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(nop))] +pub unsafe fn vld4_dup_s64(a: *const i64) -> int64x1x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4dup.v1i64.p0i8")] + fn vld4_dup_s64_(ptr: *const i8, size: i32) -> int64x1x4_t; + } +vld4_dup_s64_(a as *const i8, 8) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_s64) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4_dup_s64(a: *const i64) -> int64x1x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4r.v1i64.p0i64")] + fn vld4_dup_s64_(ptr: *const i64) -> int64x1x4_t; + } +vld4_dup_s64_(a as _) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4_dup_u8(a: *const u8) -> uint8x8x4_t { + transmute(vld4_dup_s8(transmute(a))) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4_dup_u16(a: *const u16) -> uint16x4x4_t { + transmute(vld4_dup_s16(transmute(a))) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4_dup_u32(a: *const u32) -> uint32x2x4_t { + transmute(vld4_dup_s32(transmute(a))) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4q_dup_u8(a: *const u8) -> uint8x16x4_t { + transmute(vld4q_dup_s8(transmute(a))) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4q_dup_u16(a: *const u16) -> uint16x8x4_t { + transmute(vld4q_dup_s16(transmute(a))) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4q_dup_u32(a: *const u32) -> uint32x4x4_t { + transmute(vld4q_dup_s32(transmute(a))) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4_dup_p8(a: *const p8) -> poly8x8x4_t { + transmute(vld4_dup_s8(transmute(a))) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4_dup_p16(a: *const p16) -> poly16x4x4_t { + transmute(vld4_dup_s16(transmute(a))) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4q_dup_p8(a: *const p8) -> poly8x16x4_t { + transmute(vld4q_dup_s8(transmute(a))) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4q_dup_p16(a: *const p16) -> poly16x8x4_t { + transmute(vld4q_dup_s16(transmute(a))) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4_dup_u64(a: *const u64) -> uint64x1x4_t { + transmute(vld4_dup_s64(transmute(a))) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4r))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4_dup_p64(a: *const p64) -> poly64x1x4_t { + transmute(vld4_dup_s64(transmute(a))) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4))] +pub unsafe fn vld4_dup_f32(a: *const f32) -> float32x2x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4dup.v2f32.p0i8")] + fn vld4_dup_f32_(ptr: *const i8, size: i32) -> float32x2x4_t; + } +vld4_dup_f32_(a as *const i8, 4) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4_dup_f32(a: *const f32) -> float32x2x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4r.v2f32.p0f32")] + fn vld4_dup_f32_(ptr: *const f32) -> float32x2x4_t; + } +vld4_dup_f32_(a as _) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4))] +pub unsafe fn vld4q_dup_f32(a: *const f32) -> float32x4x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4dup.v4f32.p0i8")] + fn vld4q_dup_f32_(ptr: *const i8, size: i32) -> float32x4x4_t; + } +vld4q_dup_f32_(a as *const i8, 4) +} + +/// Load single 4-element structure and replicate to all lanes of four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4r))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_dup_f32(a: *const f32) -> float32x4x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4r.v4f32.p0f32")] + fn vld4q_dup_f32_(ptr: *const f32) -> float32x4x4_t; + } +vld4q_dup_f32_(a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vld4_lane_s8(a: *const i8, b: int8x8x4_t) -> int8x8x4_t { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4lane.v8i8.p0i8")] + fn vld4_lane_s8_(ptr: *const i8, a: int8x8_t, b: int8x8_t, c: int8x8_t, d: int8x8_t, n: i32, size: i32) -> int8x8x4_t; + } +vld4_lane_s8_(a as _, b.0, b.1, b.2, b.3, LANE, 1) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4_lane_s8(a: *const i8, b: int8x8x4_t) -> int8x8x4_t { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4lane.v8i8.p0i8")] + fn vld4_lane_s8_(a: int8x8_t, b: int8x8_t, c: int8x8_t, d: int8x8_t, n: i64, ptr: *const i8) -> int8x8x4_t; + } +vld4_lane_s8_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vld4_lane_s16(a: *const i16, b: int16x4x4_t) -> int16x4x4_t { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4lane.v4i16.p0i8")] + fn vld4_lane_s16_(ptr: *const i8, a: int16x4_t, b: int16x4_t, c: int16x4_t, d: int16x4_t, n: i32, size: i32) -> int16x4x4_t; + } +vld4_lane_s16_(a as _, b.0, b.1, b.2, b.3, LANE, 2) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4_lane_s16(a: *const i16, b: int16x4x4_t) -> int16x4x4_t { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4lane.v4i16.p0i8")] + fn vld4_lane_s16_(a: int16x4_t, b: int16x4_t, c: int16x4_t, d: int16x4_t, n: i64, ptr: *const i8) -> int16x4x4_t; + } +vld4_lane_s16_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vld4_lane_s32(a: *const i32, b: int32x2x4_t) -> int32x2x4_t { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4lane.v2i32.p0i8")] + fn vld4_lane_s32_(ptr: *const i8, a: int32x2_t, b: int32x2_t, c: int32x2_t, d: int32x2_t, n: i32, size: i32) -> int32x2x4_t; + } +vld4_lane_s32_(a as _, b.0, b.1, b.2, b.3, LANE, 4) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4_lane_s32(a: *const i32, b: int32x2x4_t) -> int32x2x4_t { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4lane.v2i32.p0i8")] + fn vld4_lane_s32_(a: int32x2_t, b: int32x2_t, c: int32x2_t, d: int32x2_t, n: i64, ptr: *const i8) -> int32x2x4_t; + } +vld4_lane_s32_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vld4q_lane_s16(a: *const i16, b: int16x8x4_t) -> int16x8x4_t { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4lane.v8i16.p0i8")] + fn vld4q_lane_s16_(ptr: *const i8, a: int16x8_t, b: int16x8_t, c: int16x8_t, d: int16x8_t, n: i32, size: i32) -> int16x8x4_t; + } +vld4q_lane_s16_(a as _, b.0, b.1, b.2, b.3, LANE, 2) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_lane_s16(a: *const i16, b: int16x8x4_t) -> int16x8x4_t { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4lane.v8i16.p0i8")] + fn vld4q_lane_s16_(a: int16x8_t, b: int16x8_t, c: int16x8_t, d: int16x8_t, n: i64, ptr: *const i8) -> int16x8x4_t; + } +vld4q_lane_s16_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vld4q_lane_s32(a: *const i32, b: int32x4x4_t) -> int32x4x4_t { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4lane.v4i32.p0i8")] + fn vld4q_lane_s32_(ptr: *const i8, a: int32x4_t, b: int32x4_t, c: int32x4_t, d: int32x4_t, n: i32, size: i32) -> int32x4x4_t; + } +vld4q_lane_s32_(a as _, b.0, b.1, b.2, b.3, LANE, 4) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_lane_s32(a: *const i32, b: int32x4x4_t) -> int32x4x4_t { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4lane.v4i32.p0i8")] + fn vld4q_lane_s32_(a: int32x4_t, b: int32x4_t, c: int32x4_t, d: int32x4_t, n: i64, ptr: *const i8) -> int32x4x4_t; + } +vld4q_lane_s32_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4_lane_u8(a: *const u8, b: uint8x8x4_t) -> uint8x8x4_t { + static_assert_imm3!(LANE); + transmute(vld4_lane_s8::(transmute(a), transmute(b))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4_lane_u16(a: *const u16, b: uint16x4x4_t) -> uint16x4x4_t { + static_assert_imm2!(LANE); + transmute(vld4_lane_s16::(transmute(a), transmute(b))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4_lane_u32(a: *const u32, b: uint32x2x4_t) -> uint32x2x4_t { + static_assert_imm1!(LANE); + transmute(vld4_lane_s32::(transmute(a), transmute(b))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4q_lane_u16(a: *const u16, b: uint16x8x4_t) -> uint16x8x4_t { + static_assert_imm3!(LANE); + transmute(vld4q_lane_s16::(transmute(a), transmute(b))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4q_lane_u32(a: *const u32, b: uint32x4x4_t) -> uint32x4x4_t { + static_assert_imm2!(LANE); + transmute(vld4q_lane_s32::(transmute(a), transmute(b))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4_lane_p8(a: *const p8, b: poly8x8x4_t) -> poly8x8x4_t { + static_assert_imm3!(LANE); + transmute(vld4_lane_s8::(transmute(a), transmute(b))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4_lane_p16(a: *const p16, b: poly16x4x4_t) -> poly16x4x4_t { + static_assert_imm2!(LANE); + transmute(vld4_lane_s16::(transmute(a), transmute(b))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vld4q_lane_p16(a: *const p16, b: poly16x8x4_t) -> poly16x8x4_t { + static_assert_imm3!(LANE); + transmute(vld4q_lane_s16::(transmute(a), transmute(b))) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vld4_lane_f32(a: *const f32, b: float32x2x4_t) -> float32x2x4_t { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4lane.v2f32.p0i8")] + fn vld4_lane_f32_(ptr: *const i8, a: float32x2_t, b: float32x2_t, c: float32x2_t, d: float32x2_t, n: i32, size: i32) -> float32x2x4_t; + } +vld4_lane_f32_(a as _, b.0, b.1, b.2, b.3, LANE, 4) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4_lane_f32(a: *const f32, b: float32x2x4_t) -> float32x2x4_t { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4lane.v2f32.p0i8")] + fn vld4_lane_f32_(a: float32x2_t, b: float32x2_t, c: float32x2_t, d: float32x2_t, n: i64, ptr: *const i8) -> float32x2x4_t; + } +vld4_lane_f32_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vld4q_lane_f32(a: *const f32, b: float32x4x4_t) -> float32x4x4_t { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vld4lane.v4f32.p0i8")] + fn vld4q_lane_f32_(ptr: *const i8, a: float32x4_t, b: float32x4_t, c: float32x4_t, d: float32x4_t, n: i32, size: i32) -> float32x4x4_t; + } +vld4q_lane_f32_(a as _, b.0, b.1, b.2, b.3, LANE, 4) +} + +/// Load multiple 4-element structures to four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ld4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vld4q_lane_f32(a: *const f32, b: float32x4x4_t) -> float32x4x4_t { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ld4lane.v4f32.p0i8")] + fn vld4q_lane_f32_(a: float32x4_t, b: float32x4_t, c: float32x4_t, d: float32x4_t, n: i64, ptr: *const i8) -> float32x4x4_t; + } +vld4q_lane_f32_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_lane_s8(a: *mut i8, b: int8x8_t) { + static_assert_imm3!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_lane_s16(a: *mut i16, b: int16x4_t) { + static_assert_imm2!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_lane_s32(a: *mut i32, b: int32x2_t) { + static_assert_imm1!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_lane_s64(a: *mut i64, b: int64x1_t) { + static_assert!(LANE : i32 where LANE == 0); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_lane_s8(a: *mut i8, b: int8x16_t) { + static_assert_imm4!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_lane_s16(a: *mut i16, b: int16x8_t) { + static_assert_imm3!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_lane_s32(a: *mut i32, b: int32x4_t) { + static_assert_imm2!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_lane_s64(a: *mut i64, b: int64x2_t) { + static_assert_imm1!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_lane_u8(a: *mut u8, b: uint8x8_t) { + static_assert_imm3!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_lane_u16(a: *mut u16, b: uint16x4_t) { + static_assert_imm2!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_lane_u32(a: *mut u32, b: uint32x2_t) { + static_assert_imm1!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_lane_u64(a: *mut u64, b: uint64x1_t) { + static_assert!(LANE : i32 where LANE == 0); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_lane_u8(a: *mut u8, b: uint8x16_t) { + static_assert_imm4!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_lane_u16(a: *mut u16, b: uint16x8_t) { + static_assert_imm3!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_lane_u32(a: *mut u32, b: uint32x4_t) { + static_assert_imm2!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_lane_u64(a: *mut u64, b: uint64x2_t) { + static_assert_imm1!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_lane_p8(a: *mut p8, b: poly8x8_t) { + static_assert_imm3!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_lane_p16(a: *mut p16, b: poly16x4_t) { + static_assert_imm2!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_lane_p8(a: *mut p8, b: poly8x16_t) { + static_assert_imm4!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_lane_p16(a: *mut p16, b: poly16x8_t) { + static_assert_imm3!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_lane_p64(a: *mut p64, b: poly64x1_t) { + static_assert!(LANE : i32 where LANE == 0); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_lane_p64(a: *mut p64, b: poly64x2_t) { + static_assert_imm1!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_lane_f32(a: *mut f32, b: float32x2_t) { + static_assert_imm1!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_lane_f32(a: *mut f32, b: float32x4_t) { + static_assert_imm2!(LANE); + *a = simd_extract(b, LANE as u32); +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s8_x2) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1_s8_x2(a: *mut i8, b: int8x8x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x2.p0i8.v8i8")] + fn vst1_s8_x2_(ptr: *mut i8, a: int8x8_t, b: int8x8_t); + } +vst1_s8_x2_(a, b.0, b.1) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s8_x2) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_s8_x2(a: *mut i8, b: int8x8x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x2.v8i8.p0i8")] + fn vst1_s8_x2_(a: int8x8_t, b: int8x8_t, ptr: *mut i8); + } +vst1_s8_x2_(b.0, b.1, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s16_x2) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1_s16_x2(a: *mut i16, b: int16x4x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x2.p0i16.v4i16")] + fn vst1_s16_x2_(ptr: *mut i16, a: int16x4_t, b: int16x4_t); + } +vst1_s16_x2_(a, b.0, b.1) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s16_x2) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_s16_x2(a: *mut i16, b: int16x4x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x2.v4i16.p0i16")] + fn vst1_s16_x2_(a: int16x4_t, b: int16x4_t, ptr: *mut i16); + } +vst1_s16_x2_(b.0, b.1, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s32_x2) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1_s32_x2(a: *mut i32, b: int32x2x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x2.p0i32.v2i32")] + fn vst1_s32_x2_(ptr: *mut i32, a: int32x2_t, b: int32x2_t); + } +vst1_s32_x2_(a, b.0, b.1) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s32_x2) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_s32_x2(a: *mut i32, b: int32x2x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x2.v2i32.p0i32")] + fn vst1_s32_x2_(a: int32x2_t, b: int32x2_t, ptr: *mut i32); + } +vst1_s32_x2_(b.0, b.1, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s64_x2) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1_s64_x2(a: *mut i64, b: int64x1x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x2.p0i64.v1i64")] + fn vst1_s64_x2_(ptr: *mut i64, a: int64x1_t, b: int64x1_t); + } +vst1_s64_x2_(a, b.0, b.1) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s64_x2) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_s64_x2(a: *mut i64, b: int64x1x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x2.v1i64.p0i64")] + fn vst1_s64_x2_(a: int64x1_t, b: int64x1_t, ptr: *mut i64); + } +vst1_s64_x2_(b.0, b.1, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s8_x2) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1q_s8_x2(a: *mut i8, b: int8x16x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x2.p0i8.v16i8")] + fn vst1q_s8_x2_(ptr: *mut i8, a: int8x16_t, b: int8x16_t); + } +vst1q_s8_x2_(a, b.0, b.1) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s8_x2) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_s8_x2(a: *mut i8, b: int8x16x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x2.v16i8.p0i8")] + fn vst1q_s8_x2_(a: int8x16_t, b: int8x16_t, ptr: *mut i8); + } +vst1q_s8_x2_(b.0, b.1, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s16_x2) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1q_s16_x2(a: *mut i16, b: int16x8x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x2.p0i16.v8i16")] + fn vst1q_s16_x2_(ptr: *mut i16, a: int16x8_t, b: int16x8_t); + } +vst1q_s16_x2_(a, b.0, b.1) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s16_x2) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_s16_x2(a: *mut i16, b: int16x8x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x2.v8i16.p0i16")] + fn vst1q_s16_x2_(a: int16x8_t, b: int16x8_t, ptr: *mut i16); + } +vst1q_s16_x2_(b.0, b.1, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s32_x2) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1q_s32_x2(a: *mut i32, b: int32x4x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x2.p0i32.v4i32")] + fn vst1q_s32_x2_(ptr: *mut i32, a: int32x4_t, b: int32x4_t); + } +vst1q_s32_x2_(a, b.0, b.1) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s32_x2) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_s32_x2(a: *mut i32, b: int32x4x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x2.v4i32.p0i32")] + fn vst1q_s32_x2_(a: int32x4_t, b: int32x4_t, ptr: *mut i32); + } +vst1q_s32_x2_(b.0, b.1, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s64_x2) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1q_s64_x2(a: *mut i64, b: int64x2x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x2.p0i64.v2i64")] + fn vst1q_s64_x2_(ptr: *mut i64, a: int64x2_t, b: int64x2_t); + } +vst1q_s64_x2_(a, b.0, b.1) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s64_x2) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_s64_x2(a: *mut i64, b: int64x2x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x2.v2i64.p0i64")] + fn vst1q_s64_x2_(a: int64x2_t, b: int64x2_t, ptr: *mut i64); + } +vst1q_s64_x2_(b.0, b.1, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s8_x3) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1_s8_x3(a: *mut i8, b: int8x8x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x3.p0i8.v8i8")] + fn vst1_s8_x3_(ptr: *mut i8, a: int8x8_t, b: int8x8_t, c: int8x8_t); + } +vst1_s8_x3_(a, b.0, b.1, b.2) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s8_x3) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_s8_x3(a: *mut i8, b: int8x8x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x3.v8i8.p0i8")] + fn vst1_s8_x3_(a: int8x8_t, b: int8x8_t, c: int8x8_t, ptr: *mut i8); + } +vst1_s8_x3_(b.0, b.1, b.2, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s16_x3) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1_s16_x3(a: *mut i16, b: int16x4x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x3.p0i16.v4i16")] + fn vst1_s16_x3_(ptr: *mut i16, a: int16x4_t, b: int16x4_t, c: int16x4_t); + } +vst1_s16_x3_(a, b.0, b.1, b.2) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s16_x3) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_s16_x3(a: *mut i16, b: int16x4x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x3.v4i16.p0i16")] + fn vst1_s16_x3_(a: int16x4_t, b: int16x4_t, c: int16x4_t, ptr: *mut i16); + } +vst1_s16_x3_(b.0, b.1, b.2, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s32_x3) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1_s32_x3(a: *mut i32, b: int32x2x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x3.p0i32.v2i32")] + fn vst1_s32_x3_(ptr: *mut i32, a: int32x2_t, b: int32x2_t, c: int32x2_t); + } +vst1_s32_x3_(a, b.0, b.1, b.2) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s32_x3) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_s32_x3(a: *mut i32, b: int32x2x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x3.v2i32.p0i32")] + fn vst1_s32_x3_(a: int32x2_t, b: int32x2_t, c: int32x2_t, ptr: *mut i32); + } +vst1_s32_x3_(b.0, b.1, b.2, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s64_x3) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1_s64_x3(a: *mut i64, b: int64x1x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x3.p0i64.v1i64")] + fn vst1_s64_x3_(ptr: *mut i64, a: int64x1_t, b: int64x1_t, c: int64x1_t); + } +vst1_s64_x3_(a, b.0, b.1, b.2) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s64_x3) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_s64_x3(a: *mut i64, b: int64x1x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x3.v1i64.p0i64")] + fn vst1_s64_x3_(a: int64x1_t, b: int64x1_t, c: int64x1_t, ptr: *mut i64); + } +vst1_s64_x3_(b.0, b.1, b.2, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s8_x3) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1q_s8_x3(a: *mut i8, b: int8x16x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x3.p0i8.v16i8")] + fn vst1q_s8_x3_(ptr: *mut i8, a: int8x16_t, b: int8x16_t, c: int8x16_t); + } +vst1q_s8_x3_(a, b.0, b.1, b.2) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s8_x3) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_s8_x3(a: *mut i8, b: int8x16x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x3.v16i8.p0i8")] + fn vst1q_s8_x3_(a: int8x16_t, b: int8x16_t, c: int8x16_t, ptr: *mut i8); + } +vst1q_s8_x3_(b.0, b.1, b.2, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s16_x3) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1q_s16_x3(a: *mut i16, b: int16x8x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x3.p0i16.v8i16")] + fn vst1q_s16_x3_(ptr: *mut i16, a: int16x8_t, b: int16x8_t, c: int16x8_t); + } +vst1q_s16_x3_(a, b.0, b.1, b.2) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s16_x3) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_s16_x3(a: *mut i16, b: int16x8x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x3.v8i16.p0i16")] + fn vst1q_s16_x3_(a: int16x8_t, b: int16x8_t, c: int16x8_t, ptr: *mut i16); + } +vst1q_s16_x3_(b.0, b.1, b.2, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s32_x3) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1q_s32_x3(a: *mut i32, b: int32x4x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x3.p0i32.v4i32")] + fn vst1q_s32_x3_(ptr: *mut i32, a: int32x4_t, b: int32x4_t, c: int32x4_t); + } +vst1q_s32_x3_(a, b.0, b.1, b.2) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s32_x3) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_s32_x3(a: *mut i32, b: int32x4x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x3.v4i32.p0i32")] + fn vst1q_s32_x3_(a: int32x4_t, b: int32x4_t, c: int32x4_t, ptr: *mut i32); + } +vst1q_s32_x3_(b.0, b.1, b.2, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s64_x3) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1q_s64_x3(a: *mut i64, b: int64x2x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x3.p0i64.v2i64")] + fn vst1q_s64_x3_(ptr: *mut i64, a: int64x2_t, b: int64x2_t, c: int64x2_t); + } +vst1q_s64_x3_(a, b.0, b.1, b.2) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s64_x3) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_s64_x3(a: *mut i64, b: int64x2x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x3.v2i64.p0i64")] + fn vst1q_s64_x3_(a: int64x2_t, b: int64x2_t, c: int64x2_t, ptr: *mut i64); + } +vst1q_s64_x3_(b.0, b.1, b.2, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s8_x4) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1_s8_x4(a: *mut i8, b: int8x8x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x4.p0i8.v8i8")] + fn vst1_s8_x4_(ptr: *mut i8, a: int8x8_t, b: int8x8_t, c: int8x8_t, d: int8x8_t); + } +vst1_s8_x4_(a, b.0, b.1, b.2, b.3) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s8_x4) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_s8_x4(a: *mut i8, b: int8x8x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x4.v8i8.p0i8")] + fn vst1_s8_x4_(a: int8x8_t, b: int8x8_t, c: int8x8_t, d: int8x8_t, ptr: *mut i8); + } +vst1_s8_x4_(b.0, b.1, b.2, b.3, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s16_x4) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1_s16_x4(a: *mut i16, b: int16x4x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x4.p0i16.v4i16")] + fn vst1_s16_x4_(ptr: *mut i16, a: int16x4_t, b: int16x4_t, c: int16x4_t, d: int16x4_t); + } +vst1_s16_x4_(a, b.0, b.1, b.2, b.3) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s16_x4) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_s16_x4(a: *mut i16, b: int16x4x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x4.v4i16.p0i16")] + fn vst1_s16_x4_(a: int16x4_t, b: int16x4_t, c: int16x4_t, d: int16x4_t, ptr: *mut i16); + } +vst1_s16_x4_(b.0, b.1, b.2, b.3, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s32_x4) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1_s32_x4(a: *mut i32, b: int32x2x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x4.p0i32.v2i32")] + fn vst1_s32_x4_(ptr: *mut i32, a: int32x2_t, b: int32x2_t, c: int32x2_t, d: int32x2_t); + } +vst1_s32_x4_(a, b.0, b.1, b.2, b.3) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s32_x4) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_s32_x4(a: *mut i32, b: int32x2x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x4.v2i32.p0i32")] + fn vst1_s32_x4_(a: int32x2_t, b: int32x2_t, c: int32x2_t, d: int32x2_t, ptr: *mut i32); + } +vst1_s32_x4_(b.0, b.1, b.2, b.3, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s64_x4) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1_s64_x4(a: *mut i64, b: int64x1x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x4.p0i64.v1i64")] + fn vst1_s64_x4_(ptr: *mut i64, a: int64x1_t, b: int64x1_t, c: int64x1_t, d: int64x1_t); + } +vst1_s64_x4_(a, b.0, b.1, b.2, b.3) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s64_x4) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_s64_x4(a: *mut i64, b: int64x1x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x4.v1i64.p0i64")] + fn vst1_s64_x4_(a: int64x1_t, b: int64x1_t, c: int64x1_t, d: int64x1_t, ptr: *mut i64); + } +vst1_s64_x4_(b.0, b.1, b.2, b.3, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s8_x4) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1q_s8_x4(a: *mut i8, b: int8x16x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x4.p0i8.v16i8")] + fn vst1q_s8_x4_(ptr: *mut i8, a: int8x16_t, b: int8x16_t, c: int8x16_t, d: int8x16_t); + } +vst1q_s8_x4_(a, b.0, b.1, b.2, b.3) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s8_x4) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_s8_x4(a: *mut i8, b: int8x16x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x4.v16i8.p0i8")] + fn vst1q_s8_x4_(a: int8x16_t, b: int8x16_t, c: int8x16_t, d: int8x16_t, ptr: *mut i8); + } +vst1q_s8_x4_(b.0, b.1, b.2, b.3, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s16_x4) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1q_s16_x4(a: *mut i16, b: int16x8x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x4.p0i16.v8i16")] + fn vst1q_s16_x4_(ptr: *mut i16, a: int16x8_t, b: int16x8_t, c: int16x8_t, d: int16x8_t); + } +vst1q_s16_x4_(a, b.0, b.1, b.2, b.3) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s16_x4) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_s16_x4(a: *mut i16, b: int16x8x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x4.v8i16.p0i16")] + fn vst1q_s16_x4_(a: int16x8_t, b: int16x8_t, c: int16x8_t, d: int16x8_t, ptr: *mut i16); + } +vst1q_s16_x4_(b.0, b.1, b.2, b.3, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s32_x4) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1q_s32_x4(a: *mut i32, b: int32x4x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x4.p0i32.v4i32")] + fn vst1q_s32_x4_(ptr: *mut i32, a: int32x4_t, b: int32x4_t, c: int32x4_t, d: int32x4_t); + } +vst1q_s32_x4_(a, b.0, b.1, b.2, b.3) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s32_x4) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_s32_x4(a: *mut i32, b: int32x4x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x4.v4i32.p0i32")] + fn vst1q_s32_x4_(a: int32x4_t, b: int32x4_t, c: int32x4_t, d: int32x4_t, ptr: *mut i32); + } +vst1q_s32_x4_(b.0, b.1, b.2, b.3, a) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s64_x4) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1q_s64_x4(a: *mut i64, b: int64x2x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x4.p0i64.v2i64")] + fn vst1q_s64_x4_(ptr: *mut i64, a: int64x2_t, b: int64x2_t, c: int64x2_t, d: int64x2_t); + } +vst1q_s64_x4_(a, b.0, b.1, b.2, b.3) +} + +/// Store multiple single-element structures from one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s64_x4) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_s64_x4(a: *mut i64, b: int64x2x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x4.v2i64.p0i64")] + fn vst1q_s64_x4_(a: int64x2_t, b: int64x2_t, c: int64x2_t, d: int64x2_t, ptr: *mut i64); + } +vst1q_s64_x4_(b.0, b.1, b.2, b.3, a) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u8_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_u8_x2(a: *mut u8, b: uint8x8x2_t) { + vst1_s8_x2(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u16_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_u16_x2(a: *mut u16, b: uint16x4x2_t) { + vst1_s16_x2(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u32_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_u32_x2(a: *mut u32, b: uint32x2x2_t) { + vst1_s32_x2(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u64_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_u64_x2(a: *mut u64, b: uint64x1x2_t) { + vst1_s64_x2(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u8_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_u8_x2(a: *mut u8, b: uint8x16x2_t) { + vst1q_s8_x2(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u16_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_u16_x2(a: *mut u16, b: uint16x8x2_t) { + vst1q_s16_x2(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u32_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_u32_x2(a: *mut u32, b: uint32x4x2_t) { + vst1q_s32_x2(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u64_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_u64_x2(a: *mut u64, b: uint64x2x2_t) { + vst1q_s64_x2(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u8_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_u8_x3(a: *mut u8, b: uint8x8x3_t) { + vst1_s8_x3(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u16_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_u16_x3(a: *mut u16, b: uint16x4x3_t) { + vst1_s16_x3(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u32_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_u32_x3(a: *mut u32, b: uint32x2x3_t) { + vst1_s32_x3(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u64_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_u64_x3(a: *mut u64, b: uint64x1x3_t) { + vst1_s64_x3(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u8_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_u8_x3(a: *mut u8, b: uint8x16x3_t) { + vst1q_s8_x3(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u16_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_u16_x3(a: *mut u16, b: uint16x8x3_t) { + vst1q_s16_x3(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u32_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_u32_x3(a: *mut u32, b: uint32x4x3_t) { + vst1q_s32_x3(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u64_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_u64_x3(a: *mut u64, b: uint64x2x3_t) { + vst1q_s64_x3(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u8_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_u8_x4(a: *mut u8, b: uint8x8x4_t) { + vst1_s8_x4(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u16_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_u16_x4(a: *mut u16, b: uint16x4x4_t) { + vst1_s16_x4(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u32_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_u32_x4(a: *mut u32, b: uint32x2x4_t) { + vst1_s32_x4(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u64_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_u64_x4(a: *mut u64, b: uint64x1x4_t) { + vst1_s64_x4(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u8_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_u8_x4(a: *mut u8, b: uint8x16x4_t) { + vst1q_s8_x4(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u16_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_u16_x4(a: *mut u16, b: uint16x8x4_t) { + vst1q_s16_x4(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u32_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_u32_x4(a: *mut u32, b: uint32x4x4_t) { + vst1q_s32_x4(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u64_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_u64_x4(a: *mut u64, b: uint64x2x4_t) { + vst1q_s64_x4(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p8_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_p8_x2(a: *mut p8, b: poly8x8x2_t) { + vst1_s8_x2(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p8_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_p8_x3(a: *mut p8, b: poly8x8x3_t) { + vst1_s8_x3(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p8_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_p8_x4(a: *mut p8, b: poly8x8x4_t) { + vst1_s8_x4(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p8_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_p8_x2(a: *mut p8, b: poly8x16x2_t) { + vst1q_s8_x2(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p8_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_p8_x3(a: *mut p8, b: poly8x16x3_t) { + vst1q_s8_x3(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p8_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_p8_x4(a: *mut p8, b: poly8x16x4_t) { + vst1q_s8_x4(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p16_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_p16_x2(a: *mut p16, b: poly16x4x2_t) { + vst1_s16_x2(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p16_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_p16_x3(a: *mut p16, b: poly16x4x3_t) { + vst1_s16_x3(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p16_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_p16_x4(a: *mut p16, b: poly16x4x4_t) { + vst1_s16_x4(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p16_x2) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_p16_x2(a: *mut p16, b: poly16x8x2_t) { + vst1q_s16_x2(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p16_x3) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_p16_x3(a: *mut p16, b: poly16x8x3_t) { + vst1q_s16_x3(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p16_x4) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_p16_x4(a: *mut p16, b: poly16x8x4_t) { + vst1q_s16_x4(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p64_x2) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_p64_x2(a: *mut p64, b: poly64x1x2_t) { + vst1_s64_x2(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p64_x3) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_p64_x3(a: *mut p64, b: poly64x1x3_t) { + vst1_s64_x3(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p64_x4) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1_p64_x4(a: *mut p64, b: poly64x1x4_t) { + vst1_s64_x4(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p64_x2) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_p64_x2(a: *mut p64, b: poly64x2x2_t) { + vst1q_s64_x2(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p64_x3) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_p64_x3(a: *mut p64, b: poly64x2x3_t) { + vst1q_s64_x3(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p64_x4) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st1))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst1q_p64_x4(a: *mut p64, b: poly64x2x4_t) { + vst1q_s64_x4(transmute(a), transmute(b)) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f32_x2) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1_f32_x2(a: *mut f32, b: float32x2x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x2.p0f32.v2f32")] + fn vst1_f32_x2_(ptr: *mut f32, a: float32x2_t, b: float32x2_t); + } +vst1_f32_x2_(a, b.0, b.1) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f32_x2) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_f32_x2(a: *mut f32, b: float32x2x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x2.v2f32.p0f32")] + fn vst1_f32_x2_(a: float32x2_t, b: float32x2_t, ptr: *mut f32); + } +vst1_f32_x2_(b.0, b.1, a) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f32_x2) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1q_f32_x2(a: *mut f32, b: float32x4x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x2.p0f32.v4f32")] + fn vst1q_f32_x2_(ptr: *mut f32, a: float32x4_t, b: float32x4_t); + } +vst1q_f32_x2_(a, b.0, b.1) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f32_x2) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_f32_x2(a: *mut f32, b: float32x4x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x2.v4f32.p0f32")] + fn vst1q_f32_x2_(a: float32x4_t, b: float32x4_t, ptr: *mut f32); + } +vst1q_f32_x2_(b.0, b.1, a) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f32_x3) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1_f32_x3(a: *mut f32, b: float32x2x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x3.p0f32.v2f32")] + fn vst1_f32_x3_(ptr: *mut f32, a: float32x2_t, b: float32x2_t, c: float32x2_t); + } +vst1_f32_x3_(a, b.0, b.1, b.2) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f32_x3) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_f32_x3(a: *mut f32, b: float32x2x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x3.v2f32.p0f32")] + fn vst1_f32_x3_(a: float32x2_t, b: float32x2_t, c: float32x2_t, ptr: *mut f32); + } +vst1_f32_x3_(b.0, b.1, b.2, a) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f32_x3) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1q_f32_x3(a: *mut f32, b: float32x4x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x3.p0f32.v4f32")] + fn vst1q_f32_x3_(ptr: *mut f32, a: float32x4_t, b: float32x4_t, c: float32x4_t); + } +vst1q_f32_x3_(a, b.0, b.1, b.2) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f32_x3) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_f32_x3(a: *mut f32, b: float32x4x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x3.v4f32.p0f32")] + fn vst1q_f32_x3_(a: float32x4_t, b: float32x4_t, c: float32x4_t, ptr: *mut f32); + } +vst1q_f32_x3_(b.0, b.1, b.2, a) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f32_x4) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1_f32_x4(a: *mut f32, b: float32x2x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x4.p0f32.v2f32")] + fn vst1_f32_x4_(ptr: *mut f32, a: float32x2_t, b: float32x2_t, c: float32x2_t, d: float32x2_t); + } +vst1_f32_x4_(a, b.0, b.1, b.2, b.3) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f32_x4) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1_f32_x4(a: *mut f32, b: float32x2x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x4.v2f32.p0f32")] + fn vst1_f32_x4_(a: float32x2_t, b: float32x2_t, c: float32x2_t, d: float32x2_t, ptr: *mut f32); + } +vst1_f32_x4_(b.0, b.1, b.2, b.3, a) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f32_x4) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst1))] +pub unsafe fn vst1q_f32_x4(a: *mut f32, b: float32x4x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst1x4.p0f32.v4f32")] + fn vst1q_f32_x4_(ptr: *mut f32, a: float32x4_t, b: float32x4_t, c: float32x4_t, d: float32x4_t); + } +vst1q_f32_x4_(a, b.0, b.1, b.2, b.3) +} + +/// Store multiple single-element structures to one, two, three, or four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f32_x4) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st1))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst1q_f32_x4(a: *mut f32, b: float32x4x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st1x4.v4f32.p0f32")] + fn vst1q_f32_x4_(a: float32x4_t, b: float32x4_t, c: float32x4_t, d: float32x4_t, ptr: *mut f32); + } +vst1q_f32_x4_(b.0, b.1, b.2, b.3, a) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst2))] +pub unsafe fn vst2_s8(a: *mut i8, b: int8x8x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2.p0i8.v8i8")] + fn vst2_s8_(ptr: *mut i8, a: int8x8_t, b: int8x8_t, size: i32); + } +vst2_s8_(a as _, b.0, b.1, 1) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2_s8(a: *mut i8, b: int8x8x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2.v8i8.p0i8")] + fn vst2_s8_(a: int8x8_t, b: int8x8_t, ptr: *mut i8); + } +vst2_s8_(b.0, b.1, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst2))] +pub unsafe fn vst2_s16(a: *mut i16, b: int16x4x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2.p0i8.v4i16")] + fn vst2_s16_(ptr: *mut i8, a: int16x4_t, b: int16x4_t, size: i32); + } +vst2_s16_(a as _, b.0, b.1, 2) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2_s16(a: *mut i16, b: int16x4x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2.v4i16.p0i8")] + fn vst2_s16_(a: int16x4_t, b: int16x4_t, ptr: *mut i8); + } +vst2_s16_(b.0, b.1, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst2))] +pub unsafe fn vst2_s32(a: *mut i32, b: int32x2x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2.p0i8.v2i32")] + fn vst2_s32_(ptr: *mut i8, a: int32x2_t, b: int32x2_t, size: i32); + } +vst2_s32_(a as _, b.0, b.1, 4) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2_s32(a: *mut i32, b: int32x2x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2.v2i32.p0i8")] + fn vst2_s32_(a: int32x2_t, b: int32x2_t, ptr: *mut i8); + } +vst2_s32_(b.0, b.1, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst2))] +pub unsafe fn vst2q_s8(a: *mut i8, b: int8x16x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2.p0i8.v16i8")] + fn vst2q_s8_(ptr: *mut i8, a: int8x16_t, b: int8x16_t, size: i32); + } +vst2q_s8_(a as _, b.0, b.1, 1) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2q_s8(a: *mut i8, b: int8x16x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2.v16i8.p0i8")] + fn vst2q_s8_(a: int8x16_t, b: int8x16_t, ptr: *mut i8); + } +vst2q_s8_(b.0, b.1, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst2))] +pub unsafe fn vst2q_s16(a: *mut i16, b: int16x8x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2.p0i8.v8i16")] + fn vst2q_s16_(ptr: *mut i8, a: int16x8_t, b: int16x8_t, size: i32); + } +vst2q_s16_(a as _, b.0, b.1, 2) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2q_s16(a: *mut i16, b: int16x8x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2.v8i16.p0i8")] + fn vst2q_s16_(a: int16x8_t, b: int16x8_t, ptr: *mut i8); + } +vst2q_s16_(b.0, b.1, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst2))] +pub unsafe fn vst2q_s32(a: *mut i32, b: int32x4x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2.p0i8.v4i32")] + fn vst2q_s32_(ptr: *mut i8, a: int32x4_t, b: int32x4_t, size: i32); + } +vst2q_s32_(a as _, b.0, b.1, 4) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2q_s32(a: *mut i32, b: int32x4x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2.v4i32.p0i8")] + fn vst2q_s32_(a: int32x4_t, b: int32x4_t, ptr: *mut i8); + } +vst2q_s32_(b.0, b.1, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_s64) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(nop))] +pub unsafe fn vst2_s64(a: *mut i64, b: int64x1x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2.p0i8.v1i64")] + fn vst2_s64_(ptr: *mut i8, a: int64x1_t, b: int64x1_t, size: i32); + } +vst2_s64_(a as _, b.0, b.1, 8) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_s64) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2_s64(a: *mut i64, b: int64x1x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2.v1i64.p0i8")] + fn vst2_s64_(a: int64x1_t, b: int64x1_t, ptr: *mut i8); + } +vst2_s64_(b.0, b.1, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst2_u8(a: *mut u8, b: uint8x8x2_t) { + transmute(vst2_s8(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst2_u16(a: *mut u16, b: uint16x4x2_t) { + transmute(vst2_s16(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst2_u32(a: *mut u32, b: uint32x2x2_t) { + transmute(vst2_s32(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst2q_u8(a: *mut u8, b: uint8x16x2_t) { + transmute(vst2q_s8(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst2q_u16(a: *mut u16, b: uint16x8x2_t) { + transmute(vst2q_s16(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst2q_u32(a: *mut u32, b: uint32x4x2_t) { + transmute(vst2q_s32(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst2_p8(a: *mut p8, b: poly8x8x2_t) { + transmute(vst2_s8(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst2_p16(a: *mut p16, b: poly16x4x2_t) { + transmute(vst2_s16(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst2q_p8(a: *mut p8, b: poly8x16x2_t) { + transmute(vst2q_s8(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst2q_p16(a: *mut p16, b: poly16x8x2_t) { + transmute(vst2q_s16(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst2_u64(a: *mut u64, b: uint64x1x2_t) { + transmute(vst2_s64(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst2_p64(a: *mut p64, b: poly64x1x2_t) { + transmute(vst2_s64(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst2))] +pub unsafe fn vst2_f32(a: *mut f32, b: float32x2x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2.p0i8.v2f32")] + fn vst2_f32_(ptr: *mut i8, a: float32x2_t, b: float32x2_t, size: i32); + } +vst2_f32_(a as _, b.0, b.1, 4) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2_f32(a: *mut f32, b: float32x2x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2.v2f32.p0i8")] + fn vst2_f32_(a: float32x2_t, b: float32x2_t, ptr: *mut i8); + } +vst2_f32_(b.0, b.1, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst2))] +pub unsafe fn vst2q_f32(a: *mut f32, b: float32x4x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2.p0i8.v4f32")] + fn vst2q_f32_(ptr: *mut i8, a: float32x4_t, b: float32x4_t, size: i32); + } +vst2q_f32_(a as _, b.0, b.1, 4) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2q_f32(a: *mut f32, b: float32x4x2_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2.v4f32.p0i8")] + fn vst2q_f32_(a: float32x4_t, b: float32x4_t, ptr: *mut i8); + } +vst2q_f32_(b.0, b.1, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vst2_lane_s8(a: *mut i8, b: int8x8x2_t) { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2lane.p0i8.v8i8")] + fn vst2_lane_s8_(ptr: *mut i8, a: int8x8_t, b: int8x8_t, n: i32, size: i32); + } +vst2_lane_s8_(a as _, b.0, b.1, LANE, 1) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2_lane_s8(a: *mut i8, b: int8x8x2_t) { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2lane.v8i8.p0i8")] + fn vst2_lane_s8_(a: int8x8_t, b: int8x8_t, n: i64, ptr: *mut i8); + } +vst2_lane_s8_(b.0, b.1, LANE as i64, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vst2_lane_s16(a: *mut i16, b: int16x4x2_t) { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2lane.p0i8.v4i16")] + fn vst2_lane_s16_(ptr: *mut i8, a: int16x4_t, b: int16x4_t, n: i32, size: i32); + } +vst2_lane_s16_(a as _, b.0, b.1, LANE, 2) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2_lane_s16(a: *mut i16, b: int16x4x2_t) { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2lane.v4i16.p0i8")] + fn vst2_lane_s16_(a: int16x4_t, b: int16x4_t, n: i64, ptr: *mut i8); + } +vst2_lane_s16_(b.0, b.1, LANE as i64, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vst2_lane_s32(a: *mut i32, b: int32x2x2_t) { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2lane.p0i8.v2i32")] + fn vst2_lane_s32_(ptr: *mut i8, a: int32x2_t, b: int32x2_t, n: i32, size: i32); + } +vst2_lane_s32_(a as _, b.0, b.1, LANE, 4) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2_lane_s32(a: *mut i32, b: int32x2x2_t) { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2lane.v2i32.p0i8")] + fn vst2_lane_s32_(a: int32x2_t, b: int32x2_t, n: i64, ptr: *mut i8); + } +vst2_lane_s32_(b.0, b.1, LANE as i64, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vst2q_lane_s16(a: *mut i16, b: int16x8x2_t) { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2lane.p0i8.v8i16")] + fn vst2q_lane_s16_(ptr: *mut i8, a: int16x8_t, b: int16x8_t, n: i32, size: i32); + } +vst2q_lane_s16_(a as _, b.0, b.1, LANE, 2) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2q_lane_s16(a: *mut i16, b: int16x8x2_t) { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2lane.v8i16.p0i8")] + fn vst2q_lane_s16_(a: int16x8_t, b: int16x8_t, n: i64, ptr: *mut i8); + } +vst2q_lane_s16_(b.0, b.1, LANE as i64, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vst2q_lane_s32(a: *mut i32, b: int32x4x2_t) { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2lane.p0i8.v4i32")] + fn vst2q_lane_s32_(ptr: *mut i8, a: int32x4_t, b: int32x4_t, n: i32, size: i32); + } +vst2q_lane_s32_(a as _, b.0, b.1, LANE, 4) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2q_lane_s32(a: *mut i32, b: int32x4x2_t) { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2lane.v4i32.p0i8")] + fn vst2q_lane_s32_(a: int32x4_t, b: int32x4_t, n: i64, ptr: *mut i8); + } +vst2q_lane_s32_(b.0, b.1, LANE as i64, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst2_lane_u8(a: *mut u8, b: uint8x8x2_t) { + static_assert_imm3!(LANE); + transmute(vst2_lane_s8::(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst2_lane_u16(a: *mut u16, b: uint16x4x2_t) { + static_assert_imm2!(LANE); + transmute(vst2_lane_s16::(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst2_lane_u32(a: *mut u32, b: uint32x2x2_t) { + static_assert_imm1!(LANE); + transmute(vst2_lane_s32::(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst2q_lane_u16(a: *mut u16, b: uint16x8x2_t) { + static_assert_imm3!(LANE); + transmute(vst2q_lane_s16::(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst2q_lane_u32(a: *mut u32, b: uint32x4x2_t) { + static_assert_imm2!(LANE); + transmute(vst2q_lane_s32::(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst2_lane_p8(a: *mut p8, b: poly8x8x2_t) { + static_assert_imm3!(LANE); + transmute(vst2_lane_s8::(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst2_lane_p16(a: *mut p16, b: poly16x4x2_t) { + static_assert_imm2!(LANE); + transmute(vst2_lane_s16::(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst2q_lane_p16(a: *mut p16, b: poly16x8x2_t) { + static_assert_imm3!(LANE); + transmute(vst2q_lane_s16::(transmute(a), transmute(b))) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vst2_lane_f32(a: *mut f32, b: float32x2x2_t) { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2lane.p0i8.v2f32")] + fn vst2_lane_f32_(ptr: *mut i8, a: float32x2_t, b: float32x2_t, n: i32, size: i32); + } +vst2_lane_f32_(a as _, b.0, b.1, LANE, 4) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2_lane_f32(a: *mut f32, b: float32x2x2_t) { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2lane.v2f32.p0i8")] + fn vst2_lane_f32_(a: float32x2_t, b: float32x2_t, n: i64, ptr: *mut i8); + } +vst2_lane_f32_(b.0, b.1, LANE as i64, a as _) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vst2q_lane_f32(a: *mut f32, b: float32x4x2_t) { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2lane.p0i8.v4f32")] + fn vst2q_lane_f32_(ptr: *mut i8, a: float32x4_t, b: float32x4_t, n: i32, size: i32); + } +vst2q_lane_f32_(a as _, b.0, b.1, LANE, 4) +} + +/// Store multiple 2-element structures from two registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st2, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst2q_lane_f32(a: *mut f32, b: float32x4x2_t) { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st2lane.v4f32.p0i8")] + fn vst2q_lane_f32_(a: float32x4_t, b: float32x4_t, n: i64, ptr: *mut i8); + } +vst2q_lane_f32_(b.0, b.1, LANE as i64, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst3))] +pub unsafe fn vst3_s8(a: *mut i8, b: int8x8x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst3.p0i8.v8i8")] + fn vst3_s8_(ptr: *mut i8, a: int8x8_t, b: int8x8_t, c: int8x8_t, size: i32); + } +vst3_s8_(a as _, b.0, b.1, b.2, 1) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3_s8(a: *mut i8, b: int8x8x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3.v8i8.p0i8")] + fn vst3_s8_(a: int8x8_t, b: int8x8_t, c: int8x8_t, ptr: *mut i8); + } +vst3_s8_(b.0, b.1, b.2, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst3))] +pub unsafe fn vst3_s16(a: *mut i16, b: int16x4x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst3.p0i8.v4i16")] + fn vst3_s16_(ptr: *mut i8, a: int16x4_t, b: int16x4_t, c: int16x4_t, size: i32); + } +vst3_s16_(a as _, b.0, b.1, b.2, 2) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3_s16(a: *mut i16, b: int16x4x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3.v4i16.p0i8")] + fn vst3_s16_(a: int16x4_t, b: int16x4_t, c: int16x4_t, ptr: *mut i8); + } +vst3_s16_(b.0, b.1, b.2, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst3))] +pub unsafe fn vst3_s32(a: *mut i32, b: int32x2x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst3.p0i8.v2i32")] + fn vst3_s32_(ptr: *mut i8, a: int32x2_t, b: int32x2_t, c: int32x2_t, size: i32); + } +vst3_s32_(a as _, b.0, b.1, b.2, 4) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3_s32(a: *mut i32, b: int32x2x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3.v2i32.p0i8")] + fn vst3_s32_(a: int32x2_t, b: int32x2_t, c: int32x2_t, ptr: *mut i8); + } +vst3_s32_(b.0, b.1, b.2, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst3))] +pub unsafe fn vst3q_s8(a: *mut i8, b: int8x16x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst3.p0i8.v16i8")] + fn vst3q_s8_(ptr: *mut i8, a: int8x16_t, b: int8x16_t, c: int8x16_t, size: i32); + } +vst3q_s8_(a as _, b.0, b.1, b.2, 1) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3q_s8(a: *mut i8, b: int8x16x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3.v16i8.p0i8")] + fn vst3q_s8_(a: int8x16_t, b: int8x16_t, c: int8x16_t, ptr: *mut i8); + } +vst3q_s8_(b.0, b.1, b.2, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst3))] +pub unsafe fn vst3q_s16(a: *mut i16, b: int16x8x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst3.p0i8.v8i16")] + fn vst3q_s16_(ptr: *mut i8, a: int16x8_t, b: int16x8_t, c: int16x8_t, size: i32); + } +vst3q_s16_(a as _, b.0, b.1, b.2, 2) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3q_s16(a: *mut i16, b: int16x8x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3.v8i16.p0i8")] + fn vst3q_s16_(a: int16x8_t, b: int16x8_t, c: int16x8_t, ptr: *mut i8); + } +vst3q_s16_(b.0, b.1, b.2, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst3))] +pub unsafe fn vst3q_s32(a: *mut i32, b: int32x4x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst3.p0i8.v4i32")] + fn vst3q_s32_(ptr: *mut i8, a: int32x4_t, b: int32x4_t, c: int32x4_t, size: i32); + } +vst3q_s32_(a as _, b.0, b.1, b.2, 4) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3q_s32(a: *mut i32, b: int32x4x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3.v4i32.p0i8")] + fn vst3q_s32_(a: int32x4_t, b: int32x4_t, c: int32x4_t, ptr: *mut i8); + } +vst3q_s32_(b.0, b.1, b.2, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_s64) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(nop))] +pub unsafe fn vst3_s64(a: *mut i64, b: int64x1x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst3.p0i8.v1i64")] + fn vst3_s64_(ptr: *mut i8, a: int64x1_t, b: int64x1_t, c: int64x1_t, size: i32); + } +vst3_s64_(a as _, b.0, b.1, b.2, 8) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_s64) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3_s64(a: *mut i64, b: int64x1x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3.v1i64.p0i8")] + fn vst3_s64_(a: int64x1_t, b: int64x1_t, c: int64x1_t, ptr: *mut i8); + } +vst3_s64_(b.0, b.1, b.2, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st3))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst3_u8(a: *mut u8, b: uint8x8x3_t) { + transmute(vst3_s8(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st3))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst3_u16(a: *mut u16, b: uint16x4x3_t) { + transmute(vst3_s16(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st3))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst3_u32(a: *mut u32, b: uint32x2x3_t) { + transmute(vst3_s32(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st3))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst3q_u8(a: *mut u8, b: uint8x16x3_t) { + transmute(vst3q_s8(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st3))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst3q_u16(a: *mut u16, b: uint16x8x3_t) { + transmute(vst3q_s16(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st3))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst3q_u32(a: *mut u32, b: uint32x4x3_t) { + transmute(vst3q_s32(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st3))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst3_p8(a: *mut p8, b: poly8x8x3_t) { + transmute(vst3_s8(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st3))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst3_p16(a: *mut p16, b: poly16x4x3_t) { + transmute(vst3_s16(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st3))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst3q_p8(a: *mut p8, b: poly8x16x3_t) { + transmute(vst3q_s8(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st3))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst3q_p16(a: *mut p16, b: poly16x8x3_t) { + transmute(vst3q_s16(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst3_u64(a: *mut u64, b: uint64x1x3_t) { + transmute(vst3_s64(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst3_p64(a: *mut p64, b: poly64x1x3_t) { + transmute(vst3_s64(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst3))] +pub unsafe fn vst3_f32(a: *mut f32, b: float32x2x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst3.p0i8.v2f32")] + fn vst3_f32_(ptr: *mut i8, a: float32x2_t, b: float32x2_t, c: float32x2_t, size: i32); + } +vst3_f32_(a as _, b.0, b.1, b.2, 4) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3_f32(a: *mut f32, b: float32x2x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3.v2f32.p0i8")] + fn vst3_f32_(a: float32x2_t, b: float32x2_t, c: float32x2_t, ptr: *mut i8); + } +vst3_f32_(b.0, b.1, b.2, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst3))] +pub unsafe fn vst3q_f32(a: *mut f32, b: float32x4x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst3.p0i8.v4f32")] + fn vst3q_f32_(ptr: *mut i8, a: float32x4_t, b: float32x4_t, c: float32x4_t, size: i32); + } +vst3q_f32_(a as _, b.0, b.1, b.2, 4) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3q_f32(a: *mut f32, b: float32x4x3_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3.v4f32.p0i8")] + fn vst3q_f32_(a: float32x4_t, b: float32x4_t, c: float32x4_t, ptr: *mut i8); + } +vst3q_f32_(b.0, b.1, b.2, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vst3_lane_s8(a: *mut i8, b: int8x8x3_t) { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst3lane.p0i8.v8i8")] + fn vst3_lane_s8_(ptr: *mut i8, a: int8x8_t, b: int8x8_t, c: int8x8_t, n: i32, size: i32); + } +vst3_lane_s8_(a as _, b.0, b.1, b.2, LANE, 1) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3_lane_s8(a: *mut i8, b: int8x8x3_t) { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3lane.v8i8.p0i8")] + fn vst3_lane_s8_(a: int8x8_t, b: int8x8_t, c: int8x8_t, n: i64, ptr: *mut i8); + } +vst3_lane_s8_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vst3_lane_s16(a: *mut i16, b: int16x4x3_t) { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst3lane.p0i8.v4i16")] + fn vst3_lane_s16_(ptr: *mut i8, a: int16x4_t, b: int16x4_t, c: int16x4_t, n: i32, size: i32); + } +vst3_lane_s16_(a as _, b.0, b.1, b.2, LANE, 2) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3_lane_s16(a: *mut i16, b: int16x4x3_t) { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3lane.v4i16.p0i8")] + fn vst3_lane_s16_(a: int16x4_t, b: int16x4_t, c: int16x4_t, n: i64, ptr: *mut i8); + } +vst3_lane_s16_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vst3_lane_s32(a: *mut i32, b: int32x2x3_t) { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst3lane.p0i8.v2i32")] + fn vst3_lane_s32_(ptr: *mut i8, a: int32x2_t, b: int32x2_t, c: int32x2_t, n: i32, size: i32); + } +vst3_lane_s32_(a as _, b.0, b.1, b.2, LANE, 4) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3_lane_s32(a: *mut i32, b: int32x2x3_t) { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3lane.v2i32.p0i8")] + fn vst3_lane_s32_(a: int32x2_t, b: int32x2_t, c: int32x2_t, n: i64, ptr: *mut i8); + } +vst3_lane_s32_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vst3q_lane_s16(a: *mut i16, b: int16x8x3_t) { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst3lane.p0i8.v8i16")] + fn vst3q_lane_s16_(ptr: *mut i8, a: int16x8_t, b: int16x8_t, c: int16x8_t, n: i32, size: i32); + } +vst3q_lane_s16_(a as _, b.0, b.1, b.2, LANE, 2) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3q_lane_s16(a: *mut i16, b: int16x8x3_t) { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3lane.v8i16.p0i8")] + fn vst3q_lane_s16_(a: int16x8_t, b: int16x8_t, c: int16x8_t, n: i64, ptr: *mut i8); + } +vst3q_lane_s16_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vst3q_lane_s32(a: *mut i32, b: int32x4x3_t) { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst3lane.p0i8.v4i32")] + fn vst3q_lane_s32_(ptr: *mut i8, a: int32x4_t, b: int32x4_t, c: int32x4_t, n: i32, size: i32); + } +vst3q_lane_s32_(a as _, b.0, b.1, b.2, LANE, 4) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3q_lane_s32(a: *mut i32, b: int32x4x3_t) { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3lane.v4i32.p0i8")] + fn vst3q_lane_s32_(a: int32x4_t, b: int32x4_t, c: int32x4_t, n: i64, ptr: *mut i8); + } +vst3q_lane_s32_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst3_lane_u8(a: *mut u8, b: uint8x8x3_t) { + static_assert_imm3!(LANE); + transmute(vst3_lane_s8::(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst3_lane_u16(a: *mut u16, b: uint16x4x3_t) { + static_assert_imm2!(LANE); + transmute(vst3_lane_s16::(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst3_lane_u32(a: *mut u32, b: uint32x2x3_t) { + static_assert_imm1!(LANE); + transmute(vst3_lane_s32::(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst3q_lane_u16(a: *mut u16, b: uint16x8x3_t) { + static_assert_imm3!(LANE); + transmute(vst3q_lane_s16::(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst3q_lane_u32(a: *mut u32, b: uint32x4x3_t) { + static_assert_imm2!(LANE); + transmute(vst3q_lane_s32::(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst3_lane_p8(a: *mut p8, b: poly8x8x3_t) { + static_assert_imm3!(LANE); + transmute(vst3_lane_s8::(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst3_lane_p16(a: *mut p16, b: poly16x4x3_t) { + static_assert_imm2!(LANE); + transmute(vst3_lane_s16::(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst3q_lane_p16(a: *mut p16, b: poly16x8x3_t) { + static_assert_imm3!(LANE); + transmute(vst3q_lane_s16::(transmute(a), transmute(b))) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vst3_lane_f32(a: *mut f32, b: float32x2x3_t) { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst3lane.p0i8.v2f32")] + fn vst3_lane_f32_(ptr: *mut i8, a: float32x2_t, b: float32x2_t, c: float32x2_t, n: i32, size: i32); + } +vst3_lane_f32_(a as _, b.0, b.1, b.2, LANE, 4) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3_lane_f32(a: *mut f32, b: float32x2x3_t) { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3lane.v2f32.p0i8")] + fn vst3_lane_f32_(a: float32x2_t, b: float32x2_t, c: float32x2_t, n: i64, ptr: *mut i8); + } +vst3_lane_f32_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vst3q_lane_f32(a: *mut f32, b: float32x4x3_t) { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst3lane.p0i8.v4f32")] + fn vst3q_lane_f32_(ptr: *mut i8, a: float32x4_t, b: float32x4_t, c: float32x4_t, n: i32, size: i32); + } +vst3q_lane_f32_(a as _, b.0, b.1, b.2, LANE, 4) +} + +/// Store multiple 3-element structures from three registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st3, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst3q_lane_f32(a: *mut f32, b: float32x4x3_t) { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st3lane.v4f32.p0i8")] + fn vst3q_lane_f32_(a: float32x4_t, b: float32x4_t, c: float32x4_t, n: i64, ptr: *mut i8); + } +vst3q_lane_f32_(b.0, b.1, b.2, LANE as i64, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst4))] +pub unsafe fn vst4_s8(a: *mut i8, b: int8x8x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst4.p0i8.v8i8")] + fn vst4_s8_(ptr: *mut i8, a: int8x8_t, b: int8x8_t, c: int8x8_t, d: int8x8_t, size: i32); + } +vst4_s8_(a as _, b.0, b.1, b.2, b.3, 1) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4_s8(a: *mut i8, b: int8x8x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4.v8i8.p0i8")] + fn vst4_s8_(a: int8x8_t, b: int8x8_t, c: int8x8_t, d: int8x8_t, ptr: *mut i8); + } +vst4_s8_(b.0, b.1, b.2, b.3, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst4))] +pub unsafe fn vst4_s16(a: *mut i16, b: int16x4x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst4.p0i8.v4i16")] + fn vst4_s16_(ptr: *mut i8, a: int16x4_t, b: int16x4_t, c: int16x4_t, d: int16x4_t, size: i32); + } +vst4_s16_(a as _, b.0, b.1, b.2, b.3, 2) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4_s16(a: *mut i16, b: int16x4x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4.v4i16.p0i8")] + fn vst4_s16_(a: int16x4_t, b: int16x4_t, c: int16x4_t, d: int16x4_t, ptr: *mut i8); + } +vst4_s16_(b.0, b.1, b.2, b.3, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst4))] +pub unsafe fn vst4_s32(a: *mut i32, b: int32x2x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst4.p0i8.v2i32")] + fn vst4_s32_(ptr: *mut i8, a: int32x2_t, b: int32x2_t, c: int32x2_t, d: int32x2_t, size: i32); + } +vst4_s32_(a as _, b.0, b.1, b.2, b.3, 4) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4_s32(a: *mut i32, b: int32x2x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4.v2i32.p0i8")] + fn vst4_s32_(a: int32x2_t, b: int32x2_t, c: int32x2_t, d: int32x2_t, ptr: *mut i8); + } +vst4_s32_(b.0, b.1, b.2, b.3, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst4))] +pub unsafe fn vst4q_s8(a: *mut i8, b: int8x16x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst4.p0i8.v16i8")] + fn vst4q_s8_(ptr: *mut i8, a: int8x16_t, b: int8x16_t, c: int8x16_t, d: int8x16_t, size: i32); + } +vst4q_s8_(a as _, b.0, b.1, b.2, b.3, 1) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4q_s8(a: *mut i8, b: int8x16x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4.v16i8.p0i8")] + fn vst4q_s8_(a: int8x16_t, b: int8x16_t, c: int8x16_t, d: int8x16_t, ptr: *mut i8); + } +vst4q_s8_(b.0, b.1, b.2, b.3, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst4))] +pub unsafe fn vst4q_s16(a: *mut i16, b: int16x8x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst4.p0i8.v8i16")] + fn vst4q_s16_(ptr: *mut i8, a: int16x8_t, b: int16x8_t, c: int16x8_t, d: int16x8_t, size: i32); + } +vst4q_s16_(a as _, b.0, b.1, b.2, b.3, 2) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4q_s16(a: *mut i16, b: int16x8x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4.v8i16.p0i8")] + fn vst4q_s16_(a: int16x8_t, b: int16x8_t, c: int16x8_t, d: int16x8_t, ptr: *mut i8); + } +vst4q_s16_(b.0, b.1, b.2, b.3, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst4))] +pub unsafe fn vst4q_s32(a: *mut i32, b: int32x4x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst4.p0i8.v4i32")] + fn vst4q_s32_(ptr: *mut i8, a: int32x4_t, b: int32x4_t, c: int32x4_t, d: int32x4_t, size: i32); + } +vst4q_s32_(a as _, b.0, b.1, b.2, b.3, 4) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4q_s32(a: *mut i32, b: int32x4x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4.v4i32.p0i8")] + fn vst4q_s32_(a: int32x4_t, b: int32x4_t, c: int32x4_t, d: int32x4_t, ptr: *mut i8); + } +vst4q_s32_(b.0, b.1, b.2, b.3, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_s64) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(nop))] +pub unsafe fn vst4_s64(a: *mut i64, b: int64x1x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst4.p0i8.v1i64")] + fn vst4_s64_(ptr: *mut i8, a: int64x1_t, b: int64x1_t, c: int64x1_t, d: int64x1_t, size: i32); + } +vst4_s64_(a as _, b.0, b.1, b.2, b.3, 8) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_s64) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4_s64(a: *mut i64, b: int64x1x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4.v1i64.p0i8")] + fn vst4_s64_(a: int64x1_t, b: int64x1_t, c: int64x1_t, d: int64x1_t, ptr: *mut i8); + } +vst4_s64_(b.0, b.1, b.2, b.3, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st4))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst4_u8(a: *mut u8, b: uint8x8x4_t) { + transmute(vst4_s8(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st4))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst4_u16(a: *mut u16, b: uint16x4x4_t) { + transmute(vst4_s16(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st4))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst4_u32(a: *mut u32, b: uint32x2x4_t) { + transmute(vst4_s32(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st4))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst4q_u8(a: *mut u8, b: uint8x16x4_t) { + transmute(vst4q_s8(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st4))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst4q_u16(a: *mut u16, b: uint16x8x4_t) { + transmute(vst4q_s16(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st4))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst4q_u32(a: *mut u32, b: uint32x4x4_t) { + transmute(vst4q_s32(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st4))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst4_p8(a: *mut p8, b: poly8x8x4_t) { + transmute(vst4_s8(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st4))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst4_p16(a: *mut p16, b: poly16x4x4_t) { + transmute(vst4_s16(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st4))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst4q_p8(a: *mut p8, b: poly8x16x4_t) { + transmute(vst4q_s8(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st4))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst4q_p16(a: *mut p16, b: poly16x8x4_t) { + transmute(vst4q_s16(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst4_u64(a: *mut u64, b: uint64x1x4_t) { + transmute(vst4_s64(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst4_p64(a: *mut p64, b: poly64x1x4_t) { + transmute(vst4_s64(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst4))] +pub unsafe fn vst4_f32(a: *mut f32, b: float32x2x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst4.p0i8.v2f32")] + fn vst4_f32_(ptr: *mut i8, a: float32x2_t, b: float32x2_t, c: float32x2_t, d: float32x2_t, size: i32); + } +vst4_f32_(a as _, b.0, b.1, b.2, b.3, 4) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4_f32(a: *mut f32, b: float32x2x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4.v2f32.p0i8")] + fn vst4_f32_(a: float32x2_t, b: float32x2_t, c: float32x2_t, d: float32x2_t, ptr: *mut i8); + } +vst4_f32_(b.0, b.1, b.2, b.3, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst4))] +pub unsafe fn vst4q_f32(a: *mut f32, b: float32x4x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst4.p0i8.v4f32")] + fn vst4q_f32_(ptr: *mut i8, a: float32x4_t, b: float32x4_t, c: float32x4_t, d: float32x4_t, size: i32); + } +vst4q_f32_(a as _, b.0, b.1, b.2, b.3, 4) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4))] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4q_f32(a: *mut f32, b: float32x4x4_t) { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4.v4f32.p0i8")] + fn vst4q_f32_(a: float32x4_t, b: float32x4_t, c: float32x4_t, d: float32x4_t, ptr: *mut i8); + } +vst4q_f32_(b.0, b.1, b.2, b.3, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vst4_lane_s8(a: *mut i8, b: int8x8x4_t) { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst4lane.p0i8.v8i8")] + fn vst4_lane_s8_(ptr: *mut i8, a: int8x8_t, b: int8x8_t, c: int8x8_t, d: int8x8_t, n: i32, size: i32); + } +vst4_lane_s8_(a as _, b.0, b.1, b.2, b.3, LANE, 1) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4_lane_s8(a: *mut i8, b: int8x8x4_t) { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4lane.v8i8.p0i8")] + fn vst4_lane_s8_(a: int8x8_t, b: int8x8_t, c: int8x8_t, d: int8x8_t, n: i64, ptr: *mut i8); + } +vst4_lane_s8_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vst4_lane_s16(a: *mut i16, b: int16x4x4_t) { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst4lane.p0i8.v4i16")] + fn vst4_lane_s16_(ptr: *mut i8, a: int16x4_t, b: int16x4_t, c: int16x4_t, d: int16x4_t, n: i32, size: i32); + } +vst4_lane_s16_(a as _, b.0, b.1, b.2, b.3, LANE, 2) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4_lane_s16(a: *mut i16, b: int16x4x4_t) { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4lane.v4i16.p0i8")] + fn vst4_lane_s16_(a: int16x4_t, b: int16x4_t, c: int16x4_t, d: int16x4_t, n: i64, ptr: *mut i8); + } +vst4_lane_s16_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vst4_lane_s32(a: *mut i32, b: int32x2x4_t) { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst4lane.p0i8.v2i32")] + fn vst4_lane_s32_(ptr: *mut i8, a: int32x2_t, b: int32x2_t, c: int32x2_t, d: int32x2_t, n: i32, size: i32); + } +vst4_lane_s32_(a as _, b.0, b.1, b.2, b.3, LANE, 4) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4_lane_s32(a: *mut i32, b: int32x2x4_t) { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4lane.v2i32.p0i8")] + fn vst4_lane_s32_(a: int32x2_t, b: int32x2_t, c: int32x2_t, d: int32x2_t, n: i64, ptr: *mut i8); + } +vst4_lane_s32_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vst4q_lane_s16(a: *mut i16, b: int16x8x4_t) { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst4lane.p0i8.v8i16")] + fn vst4q_lane_s16_(ptr: *mut i8, a: int16x8_t, b: int16x8_t, c: int16x8_t, d: int16x8_t, n: i32, size: i32); + } +vst4q_lane_s16_(a as _, b.0, b.1, b.2, b.3, LANE, 2) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4q_lane_s16(a: *mut i16, b: int16x8x4_t) { + static_assert_imm3!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4lane.v8i16.p0i8")] + fn vst4q_lane_s16_(a: int16x8_t, b: int16x8_t, c: int16x8_t, d: int16x8_t, n: i64, ptr: *mut i8); + } +vst4q_lane_s16_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vst4q_lane_s32(a: *mut i32, b: int32x4x4_t) { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst4lane.p0i8.v4i32")] + fn vst4q_lane_s32_(ptr: *mut i8, a: int32x4_t, b: int32x4_t, c: int32x4_t, d: int32x4_t, n: i32, size: i32); + } +vst4q_lane_s32_(a as _, b.0, b.1, b.2, b.3, LANE, 4) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4q_lane_s32(a: *mut i32, b: int32x4x4_t) { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4lane.v4i32.p0i8")] + fn vst4q_lane_s32_(a: int32x4_t, b: int32x4_t, c: int32x4_t, d: int32x4_t, n: i64, ptr: *mut i8); + } +vst4q_lane_s32_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst4_lane_u8(a: *mut u8, b: uint8x8x4_t) { + static_assert_imm3!(LANE); + transmute(vst4_lane_s8::(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst4_lane_u16(a: *mut u16, b: uint16x4x4_t) { + static_assert_imm2!(LANE); + transmute(vst4_lane_s16::(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst4_lane_u32(a: *mut u32, b: uint32x2x4_t) { + static_assert_imm1!(LANE); + transmute(vst4_lane_s32::(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst4q_lane_u16(a: *mut u16, b: uint16x8x4_t) { + static_assert_imm3!(LANE); + transmute(vst4q_lane_s16::(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst4q_lane_u32(a: *mut u32, b: uint32x4x4_t) { + static_assert_imm2!(LANE); + transmute(vst4q_lane_s32::(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst4_lane_p8(a: *mut p8, b: poly8x8x4_t) { + static_assert_imm3!(LANE); + transmute(vst4_lane_s8::(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst4_lane_p16(a: *mut p16, b: poly16x4x4_t) { + static_assert_imm2!(LANE); + transmute(vst4_lane_s16::(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vst4q_lane_p16(a: *mut p16, b: poly16x8x4_t) { + static_assert_imm3!(LANE); + transmute(vst4q_lane_s16::(transmute(a), transmute(b))) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vst4_lane_f32(a: *mut f32, b: float32x2x4_t) { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst4lane.p0i8.v2f32")] + fn vst4_lane_f32_(ptr: *mut i8, a: float32x2_t, b: float32x2_t, c: float32x2_t, d: float32x2_t, n: i32, size: i32); + } +vst4_lane_f32_(a as _, b.0, b.1, b.2, b.3, LANE, 4) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4_lane_f32(a: *mut f32, b: float32x2x4_t) { + static_assert_imm1!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4lane.v2f32.p0i8")] + fn vst4_lane_f32_(a: float32x2_t, b: float32x2_t, c: float32x2_t, d: float32x2_t, n: i64, ptr: *mut i8); + } +vst4_lane_f32_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_f32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vst4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vst4q_lane_f32(a: *mut f32, b: float32x4x4_t) { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst4lane.p0i8.v4f32")] + fn vst4q_lane_f32_(ptr: *mut i8, a: float32x4_t, b: float32x4_t, c: float32x4_t, d: float32x4_t, n: i32, size: i32); + } +vst4q_lane_f32_(a as _, b.0, b.1, b.2, b.3, LANE, 4) +} + +/// Store multiple 4-element structures from four registers +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_f32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(st4, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vst4q_lane_f32(a: *mut f32, b: float32x4x4_t) { + static_assert_imm2!(LANE); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.st4lane.v4f32.p0i8")] + fn vst4q_lane_f32_(a: float32x4_t, b: float32x4_t, c: float32x4_t, d: float32x4_t, n: i64, ptr: *mut i8); + } +vst4q_lane_f32_(b.0, b.1, b.2, b.3, LANE as i64, a as _) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_mul(a, b) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_mul(a, b) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_mul(a, b) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_mul(a, b) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_mul(a, b) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_mul(a, b) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_mul(a, b) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_mul(a, b) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_mul(a, b) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_mul(a, b) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_mul(a, b) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_mul(a, b) +} + +/// Polynomial multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(pmul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmulp.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.pmul.v8i8")] + fn vmul_p8_(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t; + } +vmul_p8_(a, b) +} + +/// Polynomial multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(pmul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmulp.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.pmul.v16i8")] + fn vmulq_p8_(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t; + } +vmulq_p8_(a, b) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + simd_mul(a, b) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + simd_mul(a, b) +} + +/// Vector multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_n_s16(a: int16x4_t, b: i16) -> int16x4_t { + simd_mul(a, vdup_n_s16(b)) +} + +/// Vector multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_n_s16(a: int16x8_t, b: i16) -> int16x8_t { + simd_mul(a, vdupq_n_s16(b)) +} + +/// Vector multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_n_s32(a: int32x2_t, b: i32) -> int32x2_t { + simd_mul(a, vdup_n_s32(b)) +} + +/// Vector multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_n_s32(a: int32x4_t, b: i32) -> int32x4_t { + simd_mul(a, vdupq_n_s32(b)) +} + +/// Vector multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_n_u16(a: uint16x4_t, b: u16) -> uint16x4_t { + simd_mul(a, vdup_n_u16(b)) +} + +/// Vector multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_n_u16(a: uint16x8_t, b: u16) -> uint16x8_t { + simd_mul(a, vdupq_n_u16(b)) +} + +/// Vector multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_n_u32(a: uint32x2_t, b: u32) -> uint32x2_t { + simd_mul(a, vdup_n_u32(b)) +} + +/// Vector multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_n_u32(a: uint32x4_t, b: u32) -> uint32x4_t { + simd_mul(a, vdupq_n_u32(b)) +} + +/// Vector multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_n_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_n_f32(a: float32x2_t, b: f32) -> float32x2_t { + simd_mul(a, vdup_n_f32(b)) +} + +/// Vector multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_n_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_n_f32(a: float32x4_t, b: f32) -> float32x4_t { + simd_mul(a, vdupq_n_f32(b)) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_lane_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + static_assert_imm2!(LANE); + simd_mul(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_laneq_s16(a: int16x4_t, b: int16x8_t) -> int16x4_t { + static_assert_imm3!(LANE); + simd_mul(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_lane_s16(a: int16x8_t, b: int16x4_t) -> int16x8_t { + static_assert_imm2!(LANE); + simd_mul(a, simd_shuffle8!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_laneq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + static_assert_imm3!(LANE); + simd_mul(a, simd_shuffle8!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_lane_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + static_assert_imm1!(LANE); + simd_mul(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_laneq_s32(a: int32x2_t, b: int32x4_t) -> int32x2_t { + static_assert_imm2!(LANE); + simd_mul(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_lane_s32(a: int32x4_t, b: int32x2_t) -> int32x4_t { + static_assert_imm1!(LANE); + simd_mul(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_laneq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + simd_mul(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_lane_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + static_assert_imm2!(LANE); + simd_mul(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_laneq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_laneq_u16(a: uint16x4_t, b: uint16x8_t) -> uint16x4_t { + static_assert_imm3!(LANE); + simd_mul(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_lane_u16(a: uint16x8_t, b: uint16x4_t) -> uint16x8_t { + static_assert_imm2!(LANE); + simd_mul(a, simd_shuffle8!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_laneq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_laneq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + static_assert_imm3!(LANE); + simd_mul(a, simd_shuffle8!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_lane_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + static_assert_imm1!(LANE); + simd_mul(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_laneq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_laneq_u32(a: uint32x2_t, b: uint32x4_t) -> uint32x2_t { + static_assert_imm2!(LANE); + simd_mul(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_lane_u32(a: uint32x4_t, b: uint32x2_t) -> uint32x4_t { + static_assert_imm1!(LANE); + simd_mul(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_laneq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_laneq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + simd_mul(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Floating-point multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_lane_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + static_assert_imm1!(LANE); + simd_mul(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Floating-point multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_laneq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmul_laneq_f32(a: float32x2_t, b: float32x4_t) -> float32x2_t { + static_assert_imm2!(LANE); + simd_mul(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Floating-point multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_lane_f32(a: float32x4_t, b: float32x2_t) -> float32x4_t { + static_assert_imm1!(LANE); + simd_mul(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Floating-point multiply +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_laneq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmulq_laneq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + static_assert_imm2!(LANE); + simd_mul(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Signed multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmull.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smull))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmull_s8(a: int8x8_t, b: int8x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmulls.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smull.v8i8")] + fn vmull_s8_(a: int8x8_t, b: int8x8_t) -> int16x8_t; + } +vmull_s8_(a, b) +} + +/// Signed multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmull.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smull))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmull_s16(a: int16x4_t, b: int16x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmulls.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smull.v4i16")] + fn vmull_s16_(a: int16x4_t, b: int16x4_t) -> int32x4_t; + } +vmull_s16_(a, b) +} + +/// Signed multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmull.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smull))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmull_s32(a: int32x2_t, b: int32x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmulls.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smull.v2i32")] + fn vmull_s32_(a: int32x2_t, b: int32x2_t) -> int64x2_t; + } +vmull_s32_(a, b) +} + +/// Unsigned multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmull.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umull))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmull_u8(a: uint8x8_t, b: uint8x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmullu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umull.v8i8")] + fn vmull_u8_(a: uint8x8_t, b: uint8x8_t) -> uint16x8_t; + } +vmull_u8_(a, b) +} + +/// Unsigned multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmull.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umull))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmull_u16(a: uint16x4_t, b: uint16x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmullu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umull.v4i16")] + fn vmull_u16_(a: uint16x4_t, b: uint16x4_t) -> uint32x4_t; + } +vmull_u16_(a, b) +} + +/// Unsigned multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmull.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umull))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmull_u32(a: uint32x2_t, b: uint32x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmullu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umull.v2i32")] + fn vmull_u32_(a: uint32x2_t, b: uint32x2_t) -> uint64x2_t; + } +vmull_u32_(a, b) +} + +/// Polynomial multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmull.p8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(pmull))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmull_p8(a: poly8x8_t, b: poly8x8_t) -> poly16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmullp.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.pmull.v8i8")] + fn vmull_p8_(a: poly8x8_t, b: poly8x8_t) -> poly16x8_t; + } +vmull_p8_(a, b) +} + +/// Vector long multiply with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smull))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmull_n_s16(a: int16x4_t, b: i16) -> int32x4_t { + vmull_s16(a, vdup_n_s16(b)) +} + +/// Vector long multiply with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smull))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmull_n_s32(a: int32x2_t, b: i32) -> int64x2_t { + vmull_s32(a, vdup_n_s32(b)) +} + +/// Vector long multiply with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umull))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmull_n_u16(a: uint16x4_t, b: u16) -> uint32x4_t { + vmull_u16(a, vdup_n_u16(b)) +} + +/// Vector long multiply with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umull))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmull_n_u32(a: uint32x2_t, b: u32) -> uint64x2_t { + vmull_u32(a, vdup_n_u32(b)) +} + +/// Vector long multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smull, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmull_lane_s16(a: int16x4_t, b: int16x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + vmull_s16(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector long multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smull, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmull_laneq_s16(a: int16x4_t, b: int16x8_t) -> int32x4_t { + static_assert_imm3!(LANE); + vmull_s16(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector long multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smull, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmull_lane_s32(a: int32x2_t, b: int32x2_t) -> int64x2_t { + static_assert_imm1!(LANE); + vmull_s32(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Vector long multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smull, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmull_laneq_s32(a: int32x2_t, b: int32x4_t) -> int64x2_t { + static_assert_imm2!(LANE); + vmull_s32(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Vector long multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umull, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmull_lane_u16(a: uint16x4_t, b: uint16x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + vmull_u16(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector long multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_laneq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umull, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmull_laneq_u16(a: uint16x4_t, b: uint16x8_t) -> uint32x4_t { + static_assert_imm3!(LANE); + vmull_u16(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector long multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umull, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmull_lane_u32(a: uint32x2_t, b: uint32x2_t) -> uint64x2_t { + static_assert_imm1!(LANE); + vmull_u32(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Vector long multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_laneq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umull, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmull_laneq_u32(a: uint32x2_t, b: uint32x4_t) -> uint64x2_t { + static_assert_imm2!(LANE); + vmull_u32(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Floating-point fused Multiply-Add to accumulator(vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfma_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfma))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vfma_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.fma.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fma.v2f32")] + fn vfma_f32_(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t; + } +vfma_f32_(b, c, a) +} + +/// Floating-point fused Multiply-Add to accumulator(vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmaq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfma))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vfmaq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.fma.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fma.v4f32")] + fn vfmaq_f32_(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t; + } +vfmaq_f32_(b, c, a) +} + +/// Floating-point fused Multiply-Add to accumulator(vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfma_n_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfma))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vfma_n_f32(a: float32x2_t, b: float32x2_t, c: f32) -> float32x2_t { + vfma_f32(a, b, vdup_n_f32_vfp4(c)) +} + +/// Floating-point fused Multiply-Add to accumulator(vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmaq_n_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfma))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmla))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vfmaq_n_f32(a: float32x4_t, b: float32x4_t, c: f32) -> float32x4_t { + vfmaq_f32(a, b, vdupq_n_f32_vfp4(c)) +} + +/// Floating-point fused multiply-subtract from accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfms_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfms))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vfms_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + let b: float32x2_t = simd_neg(b); + vfma_f32(a, b, c) +} + +/// Floating-point fused multiply-subtract from accumulator +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfms))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vfmsq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + let b: float32x4_t = simd_neg(b); + vfmaq_f32(a, b, c) +} + +/// Floating-point fused Multiply-subtract to accumulator(vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfms_n_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfms))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vfms_n_f32(a: float32x2_t, b: float32x2_t, c: f32) -> float32x2_t { + vfms_f32(a, b, vdup_n_f32_vfp4(c)) +} + +/// Floating-point fused Multiply-subtract to accumulator(vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsq_n_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfms))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmls))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vfmsq_n_f32(a: float32x4_t, b: float32x4_t, c: f32) -> float32x4_t { + vfmsq_f32(a, b, vdupq_n_f32_vfp4(c)) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsub_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsub_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_sub(a, b) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_sub(a, b) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsub_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsub_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_sub(a, b) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_sub(a, b) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsub_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsub_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_sub(a, b) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_sub(a, b) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsub_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsub_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_sub(a, b) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_sub(a, b) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsub_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsub_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_sub(a, b) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_sub(a, b) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsub_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsub_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_sub(a, b) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_sub(a, b) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsub_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsub_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + simd_sub(a, b) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + simd_sub(a, b) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsub_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsub_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + simd_sub(a, b) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_sub(a, b) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsub_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsub_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + simd_sub(a, b) +} + +/// Subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + simd_sub(a, b) +} + +/// Bitwise exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vadd_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vadd_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + simd_xor(a, b) +} + +/// Bitwise exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vadd_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vadd_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + simd_xor(a, b) +} + +/// Bitwise exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vaddq_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + simd_xor(a, b) +} + +/// Bitwise exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vaddq_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + simd_xor(a, b) +} + +/// Bitwise exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vadd_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vadd_p64(a: poly64x1_t, b: poly64x1_t) -> poly64x1_t { + simd_xor(a, b) +} + +/// Bitwise exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_p64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vaddq_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t { + simd_xor(a, b) +} + +/// Bitwise exclusive OR +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_p128) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vaddq_p128(a: p128, b: p128) -> p128 { + a ^ b +} + +/// Subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubhn_s16(a: int16x8_t, b: int16x8_t) -> int8x8_t { + let c: i16x8 = i16x8::new(8, 8, 8, 8, 8, 8, 8, 8); + simd_cast(simd_shr(simd_sub(a, b), transmute(c))) +} + +/// Subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubhn_s32(a: int32x4_t, b: int32x4_t) -> int16x4_t { + let c: i32x4 = i32x4::new(16, 16, 16, 16); + simd_cast(simd_shr(simd_sub(a, b), transmute(c))) +} + +/// Subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubhn_s64(a: int64x2_t, b: int64x2_t) -> int32x2_t { + let c: i64x2 = i64x2::new(32, 32); + simd_cast(simd_shr(simd_sub(a, b), transmute(c))) +} + +/// Subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubhn_u16(a: uint16x8_t, b: uint16x8_t) -> uint8x8_t { + let c: u16x8 = u16x8::new(8, 8, 8, 8, 8, 8, 8, 8); + simd_cast(simd_shr(simd_sub(a, b), transmute(c))) +} + +/// Subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubhn_u32(a: uint32x4_t, b: uint32x4_t) -> uint16x4_t { + let c: u32x4 = u32x4::new(16, 16, 16, 16); + simd_cast(simd_shr(simd_sub(a, b), transmute(c))) +} + +/// Subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubhn_u64(a: uint64x2_t, b: uint64x2_t) -> uint32x2_t { + let c: u64x2 = u64x2::new(32, 32); + simd_cast(simd_shr(simd_sub(a, b), transmute(c))) +} + +/// Subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_high_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubhn_high_s16(a: int8x8_t, b: int16x8_t, c: int16x8_t) -> int8x16_t { + let d: int8x8_t = vsubhn_s16(b, c); + simd_shuffle16!(a, d, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_high_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubhn_high_s32(a: int16x4_t, b: int32x4_t, c: int32x4_t) -> int16x8_t { + let d: int16x4_t = vsubhn_s32(b, c); + simd_shuffle8!(a, d, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_high_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubhn_high_s64(a: int32x2_t, b: int64x2_t, c: int64x2_t) -> int32x4_t { + let d: int32x2_t = vsubhn_s64(b, c); + simd_shuffle4!(a, d, [0, 1, 2, 3]) +} + +/// Subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_high_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubhn_high_u16(a: uint8x8_t, b: uint16x8_t, c: uint16x8_t) -> uint8x16_t { + let d: uint8x8_t = vsubhn_u16(b, c); + simd_shuffle16!(a, d, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_high_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubhn_high_u32(a: uint16x4_t, b: uint32x4_t, c: uint32x4_t) -> uint16x8_t { + let d: uint16x4_t = vsubhn_u32(b, c); + simd_shuffle8!(a, d, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_high_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn2))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubhn_high_u64(a: uint32x2_t, b: uint64x2_t, c: uint64x2_t) -> uint32x4_t { + let d: uint32x2_t = vsubhn_u64(b, c); + simd_shuffle4!(a, d, [0, 1, 2, 3]) +} + +/// Signed halving subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsub_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhsub_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhsub.v8i8")] + fn vhsub_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; + } +vhsub_u8_(a, b) +} + +/// Signed halving subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsubq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhsubq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubu.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhsub.v16i8")] + fn vhsubq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; + } +vhsubq_u8_(a, b) +} + +/// Signed halving subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsub_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhsub_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhsub.v4i16")] + fn vhsub_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; + } +vhsub_u16_(a, b) +} + +/// Signed halving subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsubq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhsubq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubu.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhsub.v8i16")] + fn vhsubq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; + } +vhsubq_u16_(a, b) +} + +/// Signed halving subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsub_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhsub_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhsub.v2i32")] + fn vhsub_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; + } +vhsub_u32_(a, b) +} + +/// Signed halving subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsubq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhsubq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubu.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhsub.v4i32")] + fn vhsubq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; + } +vhsubq_u32_(a, b) +} + +/// Signed halving subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsub_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhsub_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubs.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shsub.v8i8")] + fn vhsub_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vhsub_s8_(a, b) +} + +/// Signed halving subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsubq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhsubq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubs.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shsub.v16i8")] + fn vhsubq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vhsubq_s8_(a, b) +} + +/// Signed halving subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsub_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhsub_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubs.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shsub.v4i16")] + fn vhsub_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vhsub_s16_(a, b) +} + +/// Signed halving subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsubq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhsubq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubs.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shsub.v8i16")] + fn vhsubq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vhsubq_s16_(a, b) +} + +/// Signed halving subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsub_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhsub_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubs.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shsub.v2i32")] + fn vhsub_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vhsub_s32_(a, b) +} + +/// Signed halving subtract +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsubq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shsub))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vhsubq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubs.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shsub.v4i32")] + fn vhsubq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vhsubq_s32_(a, b) +} + +/// Signed Subtract Wide +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssubw))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubw_s8(a: int16x8_t, b: int8x8_t) -> int16x8_t { + simd_sub(a, simd_cast(b)) +} + +/// Signed Subtract Wide +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssubw))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubw_s16(a: int32x4_t, b: int16x4_t) -> int32x4_t { + simd_sub(a, simd_cast(b)) +} + +/// Signed Subtract Wide +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssubw))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubw_s32(a: int64x2_t, b: int32x2_t) -> int64x2_t { + simd_sub(a, simd_cast(b)) +} + +/// Unsigned Subtract Wide +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usubw))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubw_u8(a: uint16x8_t, b: uint8x8_t) -> uint16x8_t { + simd_sub(a, simd_cast(b)) +} + +/// Unsigned Subtract Wide +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usubw))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubw_u16(a: uint32x4_t, b: uint16x4_t) -> uint32x4_t { + simd_sub(a, simd_cast(b)) +} + +/// Unsigned Subtract Wide +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usubw))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubw_u32(a: uint64x2_t, b: uint32x2_t) -> uint64x2_t { + simd_sub(a, simd_cast(b)) +} + +/// Signed Subtract Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssubl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubl_s8(a: int8x8_t, b: int8x8_t) -> int16x8_t { + let c: int16x8_t = simd_cast(a); + let d: int16x8_t = simd_cast(b); + simd_sub(c, d) +} + +/// Signed Subtract Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssubl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubl_s16(a: int16x4_t, b: int16x4_t) -> int32x4_t { + let c: int32x4_t = simd_cast(a); + let d: int32x4_t = simd_cast(b); + simd_sub(c, d) +} + +/// Signed Subtract Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssubl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubl_s32(a: int32x2_t, b: int32x2_t) -> int64x2_t { + let c: int64x2_t = simd_cast(a); + let d: int64x2_t = simd_cast(b); + simd_sub(c, d) +} + +/// Unsigned Subtract Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usubl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubl_u8(a: uint8x8_t, b: uint8x8_t) -> uint16x8_t { + let c: uint16x8_t = simd_cast(a); + let d: uint16x8_t = simd_cast(b); + simd_sub(c, d) +} + +/// Unsigned Subtract Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usubl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubl_u16(a: uint16x4_t, b: uint16x4_t) -> uint32x4_t { + let c: uint32x4_t = simd_cast(a); + let d: uint32x4_t = simd_cast(b); + simd_sub(c, d) +} + +/// Unsigned Subtract Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usubl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsubl_u32(a: uint32x2_t, b: uint32x2_t) -> uint64x2_t { + let c: uint64x2_t = simd_cast(a); + let d: uint64x2_t = simd_cast(b); + simd_sub(c, d) +} + +/// Maximum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmax_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smax))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmax_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smax.v8i8")] + fn vmax_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vmax_s8_(a, b) +} + +/// Maximum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smax))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmaxq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smax.v16i8")] + fn vmaxq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vmaxq_s8_(a, b) +} + +/// Maximum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmax_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smax))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmax_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smax.v4i16")] + fn vmax_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vmax_s16_(a, b) +} + +/// Maximum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smax))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmaxq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smax.v8i16")] + fn vmaxq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vmaxq_s16_(a, b) +} + +/// Maximum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmax_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smax))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmax_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smax.v2i32")] + fn vmax_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vmax_s32_(a, b) +} + +/// Maximum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smax))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmaxq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smax.v4i32")] + fn vmaxq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vmaxq_s32_(a, b) +} + +/// Maximum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmax_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umax))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmax_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umax.v8i8")] + fn vmax_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; + } +vmax_u8_(a, b) +} + +/// Maximum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umax))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmaxq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxu.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umax.v16i8")] + fn vmaxq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; + } +vmaxq_u8_(a, b) +} + +/// Maximum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmax_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umax))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmax_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umax.v4i16")] + fn vmax_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; + } +vmax_u16_(a, b) +} + +/// Maximum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umax))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmaxq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxu.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umax.v8i16")] + fn vmaxq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; + } +vmaxq_u16_(a, b) +} + +/// Maximum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmax_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umax))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmax_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umax.v2i32")] + fn vmax_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; + } +vmax_u32_(a, b) +} + +/// Maximum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umax))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmaxq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxu.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umax.v4i32")] + fn vmaxq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; + } +vmaxq_u32_(a, b) +} + +/// Maximum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmax_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmax))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmax_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmax.v2f32")] + fn vmax_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; + } +vmax_f32_(a, b) +} + +/// Maximum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmax))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmaxq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmax.v4f32")] + fn vmaxq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + } +vmaxq_f32_(a, b) +} + +/// Floating-point Maximum Number (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxnm_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmaxnm))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmaxnm))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmaxnm_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxnm.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxnm.v2f32")] + fn vmaxnm_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; + } +vmaxnm_f32_(a, b) +} + +/// Floating-point Maximum Number (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxnmq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmaxnm))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmaxnm))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmaxnmq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxnm.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxnm.v4f32")] + fn vmaxnmq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + } +vmaxnmq_f32_(a, b) +} + +/// Minimum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmin_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smin))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmin_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smin.v8i8")] + fn vmin_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vmin_s8_(a, b) +} + +/// Minimum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smin))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vminq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smin.v16i8")] + fn vminq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vminq_s8_(a, b) +} + +/// Minimum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmin_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smin))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmin_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smin.v4i16")] + fn vmin_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vmin_s16_(a, b) +} + +/// Minimum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smin))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vminq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smin.v8i16")] + fn vminq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vminq_s16_(a, b) +} + +/// Minimum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmin_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smin))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmin_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smin.v2i32")] + fn vmin_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vmin_s32_(a, b) +} + +/// Minimum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smin))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vminq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smin.v4i32")] + fn vminq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vminq_s32_(a, b) +} + +/// Minimum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmin_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umin))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmin_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umin.v8i8")] + fn vmin_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; + } +vmin_u8_(a, b) +} + +/// Minimum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umin))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vminq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminu.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umin.v16i8")] + fn vminq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; + } +vminq_u8_(a, b) +} + +/// Minimum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmin_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umin))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmin_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umin.v4i16")] + fn vmin_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; + } +vmin_u16_(a, b) +} + +/// Minimum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umin))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vminq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminu.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umin.v8i16")] + fn vminq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; + } +vminq_u16_(a, b) +} + +/// Minimum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmin_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umin))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmin_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umin.v2i32")] + fn vmin_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; + } +vmin_u32_(a, b) +} + +/// Minimum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umin))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vminq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminu.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umin.v4i32")] + fn vminq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; + } +vminq_u32_(a, b) +} + +/// Minimum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmin_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmin))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vmin_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmin.v2f32")] + fn vmin_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; + } +vmin_f32_(a, b) +} + +/// Minimum (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmin))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vminq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmin.v4f32")] + fn vminq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + } +vminq_f32_(a, b) +} + +/// Floating-point Minimum Number (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminnm_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vminnm))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fminnm))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vminnm_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminnm.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminnm.v2f32")] + fn vminnm_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; + } +vminnm_f32_(a, b) +} + +/// Floating-point Minimum Number (vector) +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminnmq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vminnm))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fminnm))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vminnmq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminnm.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminnm.v4f32")] + fn vminnmq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + } +vminnmq_f32_(a, b) +} + +/// Floating-point add pairwise +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadd_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(faddp))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vpadd_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpadd.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.faddp.v2f32")] + fn vpadd_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; + } +vpadd_f32_(a, b) +} + +/// Signed saturating doubling multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmull))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmull))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmull_s16(a: int16x4_t, b: int16x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqdmull.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqdmull.v4i32")] + fn vqdmull_s16_(a: int16x4_t, b: int16x4_t) -> int32x4_t; + } +vqdmull_s16_(a, b) +} + +/// Signed saturating doubling multiply long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmull))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmull))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmull_s32(a: int32x2_t, b: int32x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqdmull.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqdmull.v2i64")] + fn vqdmull_s32_(a: int32x2_t, b: int32x2_t) -> int64x2_t; + } +vqdmull_s32_(a, b) +} + +/// Vector saturating doubling long multiply with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmull))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmull))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmull_n_s16(a: int16x4_t, b: i16) -> int32x4_t { + vqdmull_s16(a, vdup_n_s16(b)) +} + +/// Vector saturating doubling long multiply with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmull))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmull))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmull_n_s32(a: int32x2_t, b: i32) -> int64x2_t { + vqdmull_s32(a, vdup_n_s32(b)) +} + +/// Vector saturating doubling long multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmull, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmull, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmull_lane_s16(a: int16x4_t, b: int16x4_t) -> int32x4_t { + static_assert_imm2!(N); + let b: int16x4_t = simd_shuffle4!(b, b, [N as u32, N as u32, N as u32, N as u32]); + vqdmull_s16(a, b) +} + +/// Vector saturating doubling long multiply by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmull, N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmull, N = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmull_lane_s32(a: int32x2_t, b: int32x2_t) -> int64x2_t { + static_assert_imm1!(N); + let b: int32x2_t = simd_shuffle2!(b, b, [N as u32, N as u32]); + vqdmull_s32(a, b) +} + +/// Signed saturating doubling multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlal))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlal))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmlal_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t { + vqaddq_s32(a, vqdmull_s16(b, c)) +} + +/// Signed saturating doubling multiply-add long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlal))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlal))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmlal_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t { + vqaddq_s64(a, vqdmull_s32(b, c)) +} + +/// Vector widening saturating doubling multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlal))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlal))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmlal_n_s16(a: int32x4_t, b: int16x4_t, c: i16) -> int32x4_t { + vqaddq_s32(a, vqdmull_n_s16(b, c)) +} + +/// Vector widening saturating doubling multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlal))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlal))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmlal_n_s32(a: int64x2_t, b: int32x2_t, c: i32) -> int64x2_t { + vqaddq_s64(a, vqdmull_n_s32(b, c)) +} + +/// Vector widening saturating doubling multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlal, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlal, N = 2))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmlal_lane_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t { + static_assert_imm2!(N); + vqaddq_s32(a, vqdmull_lane_s16::(b, c)) +} + +/// Vector widening saturating doubling multiply accumulate with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlal, N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlal, N = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmlal_lane_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t { + static_assert_imm1!(N); + vqaddq_s64(a, vqdmull_lane_s32::(b, c)) +} + +/// Signed saturating doubling multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlsl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmlsl_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t { + vqsubq_s32(a, vqdmull_s16(b, c)) +} + +/// Signed saturating doubling multiply-subtract long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlsl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmlsl_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t { + vqsubq_s64(a, vqdmull_s32(b, c)) +} + +/// Vector widening saturating doubling multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlsl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmlsl_n_s16(a: int32x4_t, b: int16x4_t, c: i16) -> int32x4_t { + vqsubq_s32(a, vqdmull_n_s16(b, c)) +} + +/// Vector widening saturating doubling multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlsl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmlsl_n_s32(a: int64x2_t, b: int32x2_t, c: i32) -> int64x2_t { + vqsubq_s64(a, vqdmull_n_s32(b, c)) +} + +/// Vector widening saturating doubling multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlsl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlsl, N = 2))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmlsl_lane_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t { + static_assert_imm2!(N); + vqsubq_s32(a, vqdmull_lane_s16::(b, c)) +} + +/// Vector widening saturating doubling multiply subtract with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlsl, N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlsl, N = 1))] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmlsl_lane_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t { + static_assert_imm1!(N); + vqsubq_s64(a, vqdmull_lane_s32::(b, c)) +} + +/// Signed saturating doubling multiply returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulh_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmulh))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmulh_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqdmulh.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqdmulh.v4i16")] + fn vqdmulh_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vqdmulh_s16_(a, b) +} + +/// Signed saturating doubling multiply returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmulh))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmulhq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqdmulh.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqdmulh.v8i16")] + fn vqdmulhq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vqdmulhq_s16_(a, b) +} + +/// Signed saturating doubling multiply returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulh_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmulh))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmulh_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqdmulh.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqdmulh.v2i32")] + fn vqdmulh_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vqdmulh_s32_(a, b) +} + +/// Signed saturating doubling multiply returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmulh))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmulhq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqdmulh.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqdmulh.v4i32")] + fn vqdmulhq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vqdmulhq_s32_(a, b) +} + +/// Vector saturating doubling multiply high with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulh_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmulh))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmulh_n_s16(a: int16x4_t, b: i16) -> int16x4_t { + let b: int16x4_t = vdup_n_s16(b); + vqdmulh_s16(a, b) +} + +/// Vector saturating doubling multiply high with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulh_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmulh))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmulh_n_s32(a: int32x2_t, b: i32) -> int32x2_t { + let b: int32x2_t = vdup_n_s32(b); + vqdmulh_s32(a, b) +} + +/// Vector saturating doubling multiply high with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhq_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmulh))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmulhq_n_s16(a: int16x8_t, b: i16) -> int16x8_t { + let b: int16x8_t = vdupq_n_s16(b); + vqdmulhq_s16(a, b) +} + +/// Vector saturating doubling multiply high with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhq_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmulh))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmulhq_n_s32(a: int32x4_t, b: i32) -> int32x4_t { + let b: int32x4_t = vdupq_n_s32(b); + vqdmulhq_s32(a, b) +} + +/// Vector saturating doubling multiply high by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhq_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmulh, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmulhq_laneq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + static_assert_imm3!(LANE); + vqdmulhq_s16(a, vdupq_n_s16(simd_extract(b, LANE as u32))) +} + +/// Vector saturating doubling multiply high by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulh_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmulh, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmulh_laneq_s16(a: int16x4_t, b: int16x8_t) -> int16x4_t { + static_assert_imm3!(LANE); + vqdmulh_s16(a, vdup_n_s16(simd_extract(b, LANE as u32))) +} + +/// Vector saturating doubling multiply high by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhq_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmulh, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmulhq_laneq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + vqdmulhq_s32(a, vdupq_n_s32(simd_extract(b, LANE as u32))) +} + +/// Vector saturating doubling multiply high by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulh_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmulh, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqdmulh_laneq_s32(a: int32x2_t, b: int32x4_t) -> int32x2_t { + static_assert_imm2!(LANE); + vqdmulh_s32(a, vdup_n_s32(simd_extract(b, LANE as u32))) +} + +/// Signed saturating extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqxtn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqmovn_s16(a: int16x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqmovns.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqxtn.v8i8")] + fn vqmovn_s16_(a: int16x8_t) -> int8x8_t; + } +vqmovn_s16_(a) +} + +/// Signed saturating extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqxtn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqmovn_s32(a: int32x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqmovns.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqxtn.v4i16")] + fn vqmovn_s32_(a: int32x4_t) -> int16x4_t; + } +vqmovn_s32_(a) +} + +/// Signed saturating extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqxtn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqmovn_s64(a: int64x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqmovns.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqxtn.v2i32")] + fn vqmovn_s64_(a: int64x2_t) -> int32x2_t; + } +vqmovn_s64_(a) +} + +/// Unsigned saturating extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqxtn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqmovn_u16(a: uint16x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqmovnu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqxtn.v8i8")] + fn vqmovn_u16_(a: uint16x8_t) -> uint8x8_t; + } +vqmovn_u16_(a) +} + +/// Unsigned saturating extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqxtn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqmovn_u32(a: uint32x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqmovnu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqxtn.v4i16")] + fn vqmovn_u32_(a: uint32x4_t) -> uint16x4_t; + } +vqmovn_u32_(a) +} + +/// Unsigned saturating extract narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqxtn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqmovn_u64(a: uint64x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqmovnu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqxtn.v2i32")] + fn vqmovn_u64_(a: uint64x2_t) -> uint32x2_t; + } +vqmovn_u64_(a) +} + +/// Signed saturating extract unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovun_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovun))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqxtun))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqmovun_s16(a: int16x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqmovnsu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqxtun.v8i8")] + fn vqmovun_s16_(a: int16x8_t) -> uint8x8_t; + } +vqmovun_s16_(a) +} + +/// Signed saturating extract unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovun_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovun))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqxtun))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqmovun_s32(a: int32x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqmovnsu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqxtun.v4i16")] + fn vqmovun_s32_(a: int32x4_t) -> uint16x4_t; + } +vqmovun_s32_(a) +} + +/// Signed saturating extract unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovun_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovun))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqxtun))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqmovun_s64(a: int64x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqmovnsu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqxtun.v2i32")] + fn vqmovun_s64_(a: int64x2_t) -> uint32x2_t; + } +vqmovun_s64_(a) +} + +/// Signed saturating rounding doubling multiply returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulh_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrdmulh_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrdmulh.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrdmulh.v4i16")] + fn vqrdmulh_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vqrdmulh_s16_(a, b) +} + +/// Signed saturating rounding doubling multiply returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrdmulhq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrdmulh.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrdmulh.v8i16")] + fn vqrdmulhq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vqrdmulhq_s16_(a, b) +} + +/// Signed saturating rounding doubling multiply returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulh_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrdmulh_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrdmulh.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrdmulh.v2i32")] + fn vqrdmulh_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vqrdmulh_s32_(a, b) +} + +/// Signed saturating rounding doubling multiply returning high half +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrdmulhq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrdmulh.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrdmulh.v4i32")] + fn vqrdmulhq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vqrdmulhq_s32_(a, b) +} + +/// Vector saturating rounding doubling multiply high with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulh_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrdmulh_n_s16(a: int16x4_t, b: i16) -> int16x4_t { + vqrdmulh_s16(a, vdup_n_s16(b)) +} + +/// Vector saturating rounding doubling multiply high with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhq_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrdmulhq_n_s16(a: int16x8_t, b: i16) -> int16x8_t { + vqrdmulhq_s16(a, vdupq_n_s16(b)) +} + +/// Vector saturating rounding doubling multiply high with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulh_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrdmulh_n_s32(a: int32x2_t, b: i32) -> int32x2_t { + vqrdmulh_s32(a, vdup_n_s32(b)) +} + +/// Vector saturating rounding doubling multiply high with scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhq_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrdmulhq_n_s32(a: int32x4_t, b: i32) -> int32x4_t { + vqrdmulhq_s32(a, vdupq_n_s32(b)) +} + +/// Vector rounding saturating doubling multiply high by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulh_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrdmulh_lane_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + static_assert_imm2!(LANE); + let b: int16x4_t = simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmulh_s16(a, b) +} + +/// Vector rounding saturating doubling multiply high by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulh_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrdmulh_laneq_s16(a: int16x4_t, b: int16x8_t) -> int16x4_t { + static_assert_imm3!(LANE); + let b: int16x4_t = simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmulh_s16(a, b) +} + +/// Vector rounding saturating doubling multiply high by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhq_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrdmulhq_lane_s16(a: int16x8_t, b: int16x4_t) -> int16x8_t { + static_assert_imm2!(LANE); + let b: int16x8_t = simd_shuffle8!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmulhq_s16(a, b) +} + +/// Vector rounding saturating doubling multiply high by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhq_laneq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrdmulhq_laneq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + static_assert_imm3!(LANE); + let b: int16x8_t = simd_shuffle8!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmulhq_s16(a, b) +} + +/// Vector rounding saturating doubling multiply high by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulh_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrdmulh_lane_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + static_assert_imm1!(LANE); + let b: int32x2_t = simd_shuffle2!(b, b, [LANE as u32, LANE as u32]); + vqrdmulh_s32(a, b) +} + +/// Vector rounding saturating doubling multiply high by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulh_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrdmulh_laneq_s32(a: int32x2_t, b: int32x4_t) -> int32x2_t { + static_assert_imm2!(LANE); + let b: int32x2_t = simd_shuffle2!(b, b, [LANE as u32, LANE as u32]); + vqrdmulh_s32(a, b) +} + +/// Vector rounding saturating doubling multiply high by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhq_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrdmulhq_lane_s32(a: int32x4_t, b: int32x2_t) -> int32x4_t { + static_assert_imm1!(LANE); + let b: int32x4_t = simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmulhq_s32(a, b) +} + +/// Vector rounding saturating doubling multiply high by scalar +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhq_laneq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrdmulhq_laneq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + let b: int32x4_t = simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmulhq_s32(a, b) +} + +/// Signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshl_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrshl_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshifts.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshl.v8i8")] + fn vqrshl_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vqrshl_s8_(a, b) +} + +/// Signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrshlq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshifts.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshl.v16i8")] + fn vqrshlq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vqrshlq_s8_(a, b) +} + +/// Signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshl_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrshl_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshifts.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshl.v4i16")] + fn vqrshl_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vqrshl_s16_(a, b) +} + +/// Signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrshlq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshifts.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshl.v8i16")] + fn vqrshlq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vqrshlq_s16_(a, b) +} + +/// Signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshl_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrshl_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshifts.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshl.v2i32")] + fn vqrshl_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vqrshl_s32_(a, b) +} + +/// Signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrshlq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshifts.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshl.v4i32")] + fn vqrshlq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vqrshlq_s32_(a, b) +} + +/// Signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshl_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrshl_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshifts.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshl.v1i64")] + fn vqrshl_s64_(a: int64x1_t, b: int64x1_t) -> int64x1_t; + } +vqrshl_s64_(a, b) +} + +/// Signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrshlq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshifts.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshl.v2i64")] + fn vqrshlq_s64_(a: int64x2_t, b: int64x2_t) -> int64x2_t; + } +vqrshlq_s64_(a, b) +} + +/// Unsigned signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshl_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqrshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrshl_u8(a: uint8x8_t, b: int8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshl.v8i8")] + fn vqrshl_u8_(a: uint8x8_t, b: int8x8_t) -> uint8x8_t; + } +vqrshl_u8_(a, b) +} + +/// Unsigned signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqrshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrshlq_u8(a: uint8x16_t, b: int8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftu.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshl.v16i8")] + fn vqrshlq_u8_(a: uint8x16_t, b: int8x16_t) -> uint8x16_t; + } +vqrshlq_u8_(a, b) +} + +/// Unsigned signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshl_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqrshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrshl_u16(a: uint16x4_t, b: int16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshl.v4i16")] + fn vqrshl_u16_(a: uint16x4_t, b: int16x4_t) -> uint16x4_t; + } +vqrshl_u16_(a, b) +} + +/// Unsigned signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqrshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrshlq_u16(a: uint16x8_t, b: int16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftu.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshl.v8i16")] + fn vqrshlq_u16_(a: uint16x8_t, b: int16x8_t) -> uint16x8_t; + } +vqrshlq_u16_(a, b) +} + +/// Unsigned signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshl_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqrshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrshl_u32(a: uint32x2_t, b: int32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshl.v2i32")] + fn vqrshl_u32_(a: uint32x2_t, b: int32x2_t) -> uint32x2_t; + } +vqrshl_u32_(a, b) +} + +/// Unsigned signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqrshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrshlq_u32(a: uint32x4_t, b: int32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftu.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshl.v4i32")] + fn vqrshlq_u32_(a: uint32x4_t, b: int32x4_t) -> uint32x4_t; + } +vqrshlq_u32_(a, b) +} + +/// Unsigned signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshl_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqrshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrshl_u64(a: uint64x1_t, b: int64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftu.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshl.v1i64")] + fn vqrshl_u64_(a: uint64x1_t, b: int64x1_t) -> uint64x1_t; + } +vqrshl_u64_(a, b) +} + +/// Unsigned signed saturating rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqrshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqrshlq_u64(a: uint64x2_t, b: int64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftu.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshl.v2i64")] + fn vqrshlq_u64_(a: uint64x2_t, b: int64x2_t) -> uint64x2_t; + } +vqrshlq_u64_(a, b) +} + +/// Signed saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrn_n_s16(a: int16x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftns.v8i8")] + fn vqrshrn_n_s16_(a: int16x8_t, n: int16x8_t) -> int8x8_t; + } +vqrshrn_n_s16_(a, int16x8_t(-N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16)) +} + +/// Signed saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrn_n_s16(a: int16x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshrn.v8i8")] + fn vqrshrn_n_s16_(a: int16x8_t, n: i32) -> int8x8_t; + } +vqrshrn_n_s16_(a, N) +} + +/// Signed saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrn_n_s32(a: int32x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftns.v4i16")] + fn vqrshrn_n_s32_(a: int32x4_t, n: int32x4_t) -> int16x4_t; + } +vqrshrn_n_s32_(a, int32x4_t(-N as i32, -N as i32, -N as i32, -N as i32)) +} + +/// Signed saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrn_n_s32(a: int32x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshrn.v4i16")] + fn vqrshrn_n_s32_(a: int32x4_t, n: i32) -> int16x4_t; + } +vqrshrn_n_s32_(a, N) +} + +/// Signed saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_s64) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrn_n_s64(a: int64x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftns.v2i32")] + fn vqrshrn_n_s64_(a: int64x2_t, n: int64x2_t) -> int32x2_t; + } +vqrshrn_n_s64_(a, int64x2_t(-N as i64, -N as i64)) +} + +/// Signed saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_s64) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrn_n_s64(a: int64x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshrn.v2i32")] + fn vqrshrn_n_s64_(a: int64x2_t, n: i32) -> int32x2_t; + } +vqrshrn_n_s64_(a, N) +} + +/// Unsigned signed saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_u16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrn_n_u16(a: uint16x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftnu.v8i8")] + fn vqrshrn_n_u16_(a: uint16x8_t, n: uint16x8_t) -> uint8x8_t; + } +vqrshrn_n_u16_(a, uint16x8_t(-N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16)) +} + +/// Unsigned signed saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_u16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrn_n_u16(a: uint16x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshrn.v8i8")] + fn vqrshrn_n_u16_(a: uint16x8_t, n: i32) -> uint8x8_t; + } +vqrshrn_n_u16_(a, N) +} + +/// Unsigned signed saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_u32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrn_n_u32(a: uint32x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftnu.v4i16")] + fn vqrshrn_n_u32_(a: uint32x4_t, n: uint32x4_t) -> uint16x4_t; + } +vqrshrn_n_u32_(a, uint32x4_t(-N as u32, -N as u32, -N as u32, -N as u32)) +} + +/// Unsigned signed saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_u32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrn_n_u32(a: uint32x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshrn.v4i16")] + fn vqrshrn_n_u32_(a: uint32x4_t, n: i32) -> uint16x4_t; + } +vqrshrn_n_u32_(a, N) +} + +/// Unsigned signed saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_u64) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrn_n_u64(a: uint64x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftnu.v2i32")] + fn vqrshrn_n_u64_(a: uint64x2_t, n: uint64x2_t) -> uint32x2_t; + } +vqrshrn_n_u64_(a, uint64x2_t(-N as u64, -N as u64)) +} + +/// Unsigned signed saturating rounded shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_u64) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrn_n_u64(a: uint64x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshrn.v2i32")] + fn vqrshrn_n_u64_(a: uint64x2_t, n: i32) -> uint32x2_t; + } +vqrshrn_n_u64_(a, N) +} + +/// Signed saturating rounded shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrun_n_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqrshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrun_n_s16(a: int16x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftnsu.v8i8")] + fn vqrshrun_n_s16_(a: int16x8_t, n: int16x8_t) -> uint8x8_t; + } +vqrshrun_n_s16_(a, int16x8_t(-N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16)) +} + +/// Signed saturating rounded shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrun_n_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrun_n_s16(a: int16x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshrun.v8i8")] + fn vqrshrun_n_s16_(a: int16x8_t, n: i32) -> uint8x8_t; + } +vqrshrun_n_s16_(a, N) +} + +/// Signed saturating rounded shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrun_n_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqrshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrun_n_s32(a: int32x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftnsu.v4i16")] + fn vqrshrun_n_s32_(a: int32x4_t, n: int32x4_t) -> uint16x4_t; + } +vqrshrun_n_s32_(a, int32x4_t(-N as i32, -N as i32, -N as i32, -N as i32)) +} + +/// Signed saturating rounded shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrun_n_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrun_n_s32(a: int32x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshrun.v4i16")] + fn vqrshrun_n_s32_(a: int32x4_t, n: i32) -> uint16x4_t; + } +vqrshrun_n_s32_(a, N) +} + +/// Signed saturating rounded shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrun_n_s64) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqrshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrun_n_s64(a: int64x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftnsu.v2i32")] + fn vqrshrun_n_s64_(a: int64x2_t, n: int64x2_t) -> uint32x2_t; + } +vqrshrun_n_s64_(a, int64x2_t(-N as i64, -N as i64)) +} + +/// Signed saturating rounded shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrun_n_s64) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqrshrun_n_s64(a: int64x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshrun.v2i32")] + fn vqrshrun_n_s64_(a: int64x2_t, n: i32) -> uint32x2_t; + } +vqrshrun_n_s64_(a, N) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshl_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshifts.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshl.v8i8")] + fn vqshl_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vqshl_s8_(a, b) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshlq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshifts.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshl.v16i8")] + fn vqshlq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vqshlq_s8_(a, b) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshl_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshifts.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshl.v4i16")] + fn vqshl_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vqshl_s16_(a, b) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshlq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshifts.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshl.v8i16")] + fn vqshlq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vqshlq_s16_(a, b) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshl_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshifts.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshl.v2i32")] + fn vqshl_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vqshl_s32_(a, b) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshlq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshifts.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshl.v4i32")] + fn vqshlq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vqshlq_s32_(a, b) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshl_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshifts.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshl.v1i64")] + fn vqshl_s64_(a: int64x1_t, b: int64x1_t) -> int64x1_t; + } +vqshl_s64_(a, b) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshlq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshifts.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshl.v2i64")] + fn vqshlq_s64_(a: int64x2_t, b: int64x2_t) -> int64x2_t; + } +vqshlq_s64_(a, b) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshl_u8(a: uint8x8_t, b: int8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshl.v8i8")] + fn vqshl_u8_(a: uint8x8_t, b: int8x8_t) -> uint8x8_t; + } +vqshl_u8_(a, b) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshlq_u8(a: uint8x16_t, b: int8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftu.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshl.v16i8")] + fn vqshlq_u8_(a: uint8x16_t, b: int8x16_t) -> uint8x16_t; + } +vqshlq_u8_(a, b) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshl_u16(a: uint16x4_t, b: int16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshl.v4i16")] + fn vqshl_u16_(a: uint16x4_t, b: int16x4_t) -> uint16x4_t; + } +vqshl_u16_(a, b) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshlq_u16(a: uint16x8_t, b: int16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftu.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshl.v8i16")] + fn vqshlq_u16_(a: uint16x8_t, b: int16x8_t) -> uint16x8_t; + } +vqshlq_u16_(a, b) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshl_u32(a: uint32x2_t, b: int32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshl.v2i32")] + fn vqshl_u32_(a: uint32x2_t, b: int32x2_t) -> uint32x2_t; + } +vqshl_u32_(a, b) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshlq_u32(a: uint32x4_t, b: int32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftu.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshl.v4i32")] + fn vqshlq_u32_(a: uint32x4_t, b: int32x4_t) -> uint32x4_t; + } +vqshlq_u32_(a, b) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshl_u64(a: uint64x1_t, b: int64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftu.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshl.v1i64")] + fn vqshl_u64_(a: uint64x1_t, b: int64x1_t) -> uint64x1_t; + } +vqshl_u64_(a, b) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshlq_u64(a: uint64x2_t, b: int64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftu.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshl.v2i64")] + fn vqshlq_u64_(a: uint64x2_t, b: int64x2_t) -> uint64x2_t; + } +vqshlq_u64_(a, b) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_n_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshl_n_s8(a: int8x8_t) -> int8x8_t { + static_assert_imm3!(N); + vqshl_s8(a, vdup_n_s8(N as _)) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_n_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshlq_n_s8(a: int8x16_t) -> int8x16_t { + static_assert_imm3!(N); + vqshlq_s8(a, vdupq_n_s8(N as _)) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshl_n_s16(a: int16x4_t) -> int16x4_t { + static_assert_imm4!(N); + vqshl_s16(a, vdup_n_s16(N as _)) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshlq_n_s16(a: int16x8_t) -> int16x8_t { + static_assert_imm4!(N); + vqshlq_s16(a, vdupq_n_s16(N as _)) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshl_n_s32(a: int32x2_t) -> int32x2_t { + static_assert_imm5!(N); + vqshl_s32(a, vdup_n_s32(N as _)) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshlq_n_s32(a: int32x4_t) -> int32x4_t { + static_assert_imm5!(N); + vqshlq_s32(a, vdupq_n_s32(N as _)) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshl_n_s64(a: int64x1_t) -> int64x1_t { + static_assert_imm6!(N); + vqshl_s64(a, vdup_n_s64(N as _)) +} + +/// Signed saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshlq_n_s64(a: int64x2_t) -> int64x2_t { + static_assert_imm6!(N); + vqshlq_s64(a, vdupq_n_s64(N as _)) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_n_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshl_n_u8(a: uint8x8_t) -> uint8x8_t { + static_assert_imm3!(N); + vqshl_u8(a, vdup_n_s8(N as _)) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_n_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshlq_n_u8(a: uint8x16_t) -> uint8x16_t { + static_assert_imm3!(N); + vqshlq_u8(a, vdupq_n_s8(N as _)) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshl_n_u16(a: uint16x4_t) -> uint16x4_t { + static_assert_imm4!(N); + vqshl_u16(a, vdup_n_s16(N as _)) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshlq_n_u16(a: uint16x8_t) -> uint16x8_t { + static_assert_imm4!(N); + vqshlq_u16(a, vdupq_n_s16(N as _)) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshl_n_u32(a: uint32x2_t) -> uint32x2_t { + static_assert_imm5!(N); + vqshl_u32(a, vdup_n_s32(N as _)) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshlq_n_u32(a: uint32x4_t) -> uint32x4_t { + static_assert_imm5!(N); + vqshlq_u32(a, vdupq_n_s32(N as _)) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshl_n_u64(a: uint64x1_t) -> uint64x1_t { + static_assert_imm6!(N); + vqshl_u64(a, vdup_n_s64(N as _)) +} + +/// Unsigned saturating shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqshlq_n_u64(a: uint64x2_t) -> uint64x2_t { + static_assert_imm6!(N); + vqshlq_u64(a, vdupq_n_s64(N as _)) +} + +/// Signed saturating shift left unsigned +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlu_n_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqshlu, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshlu_n_s8(a: int8x8_t) -> uint8x8_t { + static_assert_imm3!(N); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftsu.v8i8")] + fn vqshlu_n_s8_(a: int8x8_t, n: int8x8_t) -> uint8x8_t; + } +vqshlu_n_s8_(a, int8x8_t(N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8)) +} + +/// Signed saturating shift left unsigned +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlu_n_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshlu, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshlu_n_s8(a: int8x8_t) -> uint8x8_t { + static_assert_imm3!(N); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshlu.v8i8")] + fn vqshlu_n_s8_(a: int8x8_t, n: int8x8_t) -> uint8x8_t; + } +vqshlu_n_s8_(a, int8x8_t(N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8)) +} + +/// Signed saturating shift left unsigned +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlu_n_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqshlu, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshlu_n_s16(a: int16x4_t) -> uint16x4_t { + static_assert_imm4!(N); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftsu.v4i16")] + fn vqshlu_n_s16_(a: int16x4_t, n: int16x4_t) -> uint16x4_t; + } +vqshlu_n_s16_(a, int16x4_t(N as i16, N as i16, N as i16, N as i16)) +} + +/// Signed saturating shift left unsigned +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlu_n_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshlu, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshlu_n_s16(a: int16x4_t) -> uint16x4_t { + static_assert_imm4!(N); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshlu.v4i16")] + fn vqshlu_n_s16_(a: int16x4_t, n: int16x4_t) -> uint16x4_t; + } +vqshlu_n_s16_(a, int16x4_t(N as i16, N as i16, N as i16, N as i16)) +} + +/// Signed saturating shift left unsigned +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlu_n_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqshlu, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshlu_n_s32(a: int32x2_t) -> uint32x2_t { + static_assert_imm5!(N); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftsu.v2i32")] + fn vqshlu_n_s32_(a: int32x2_t, n: int32x2_t) -> uint32x2_t; + } +vqshlu_n_s32_(a, int32x2_t(N as i32, N as i32)) +} + +/// Signed saturating shift left unsigned +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlu_n_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshlu, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshlu_n_s32(a: int32x2_t) -> uint32x2_t { + static_assert_imm5!(N); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshlu.v2i32")] + fn vqshlu_n_s32_(a: int32x2_t, n: int32x2_t) -> uint32x2_t; + } +vqshlu_n_s32_(a, int32x2_t(N as i32, N as i32)) +} + +/// Signed saturating shift left unsigned +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlu_n_s64) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqshlu, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshlu_n_s64(a: int64x1_t) -> uint64x1_t { + static_assert_imm6!(N); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftsu.v1i64")] + fn vqshlu_n_s64_(a: int64x1_t, n: int64x1_t) -> uint64x1_t; + } +vqshlu_n_s64_(a, int64x1_t(N as i64)) +} + +/// Signed saturating shift left unsigned +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlu_n_s64) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshlu, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshlu_n_s64(a: int64x1_t) -> uint64x1_t { + static_assert_imm6!(N); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshlu.v1i64")] + fn vqshlu_n_s64_(a: int64x1_t, n: int64x1_t) -> uint64x1_t; + } +vqshlu_n_s64_(a, int64x1_t(N as i64)) +} + +/// Signed saturating shift left unsigned +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshluq_n_s8) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqshlu, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshluq_n_s8(a: int8x16_t) -> uint8x16_t { + static_assert_imm3!(N); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftsu.v16i8")] + fn vqshluq_n_s8_(a: int8x16_t, n: int8x16_t) -> uint8x16_t; + } +vqshluq_n_s8_(a, int8x16_t(N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8)) +} + +/// Signed saturating shift left unsigned +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshluq_n_s8) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshlu, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshluq_n_s8(a: int8x16_t) -> uint8x16_t { + static_assert_imm3!(N); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshlu.v16i8")] + fn vqshluq_n_s8_(a: int8x16_t, n: int8x16_t) -> uint8x16_t; + } +vqshluq_n_s8_(a, int8x16_t(N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8)) +} + +/// Signed saturating shift left unsigned +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshluq_n_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqshlu, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshluq_n_s16(a: int16x8_t) -> uint16x8_t { + static_assert_imm4!(N); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftsu.v8i16")] + fn vqshluq_n_s16_(a: int16x8_t, n: int16x8_t) -> uint16x8_t; + } +vqshluq_n_s16_(a, int16x8_t(N as i16, N as i16, N as i16, N as i16, N as i16, N as i16, N as i16, N as i16)) +} + +/// Signed saturating shift left unsigned +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshluq_n_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshlu, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshluq_n_s16(a: int16x8_t) -> uint16x8_t { + static_assert_imm4!(N); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshlu.v8i16")] + fn vqshluq_n_s16_(a: int16x8_t, n: int16x8_t) -> uint16x8_t; + } +vqshluq_n_s16_(a, int16x8_t(N as i16, N as i16, N as i16, N as i16, N as i16, N as i16, N as i16, N as i16)) +} + +/// Signed saturating shift left unsigned +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshluq_n_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqshlu, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshluq_n_s32(a: int32x4_t) -> uint32x4_t { + static_assert_imm5!(N); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftsu.v4i32")] + fn vqshluq_n_s32_(a: int32x4_t, n: int32x4_t) -> uint32x4_t; + } +vqshluq_n_s32_(a, int32x4_t(N as i32, N as i32, N as i32, N as i32)) +} + +/// Signed saturating shift left unsigned +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshluq_n_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshlu, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshluq_n_s32(a: int32x4_t) -> uint32x4_t { + static_assert_imm5!(N); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshlu.v4i32")] + fn vqshluq_n_s32_(a: int32x4_t, n: int32x4_t) -> uint32x4_t; + } +vqshluq_n_s32_(a, int32x4_t(N as i32, N as i32, N as i32, N as i32)) +} + +/// Signed saturating shift left unsigned +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshluq_n_s64) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqshlu, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshluq_n_s64(a: int64x2_t) -> uint64x2_t { + static_assert_imm6!(N); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftsu.v2i64")] + fn vqshluq_n_s64_(a: int64x2_t, n: int64x2_t) -> uint64x2_t; + } +vqshluq_n_s64_(a, int64x2_t(N as i64, N as i64)) +} + +/// Signed saturating shift left unsigned +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshluq_n_s64) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshlu, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshluq_n_s64(a: int64x2_t) -> uint64x2_t { + static_assert_imm6!(N); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshlu.v2i64")] + fn vqshluq_n_s64_(a: int64x2_t, n: int64x2_t) -> uint64x2_t; + } +vqshluq_n_s64_(a, int64x2_t(N as i64, N as i64)) +} + +/// Signed saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrn_n_s16(a: int16x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftns.v8i8")] + fn vqshrn_n_s16_(a: int16x8_t, n: int16x8_t) -> int8x8_t; + } +vqshrn_n_s16_(a, int16x8_t(-N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16)) +} + +/// Signed saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrn_n_s16(a: int16x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshrn.v8i8")] + fn vqshrn_n_s16_(a: int16x8_t, n: i32) -> int8x8_t; + } +vqshrn_n_s16_(a, N) +} + +/// Signed saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrn_n_s32(a: int32x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftns.v4i16")] + fn vqshrn_n_s32_(a: int32x4_t, n: int32x4_t) -> int16x4_t; + } +vqshrn_n_s32_(a, int32x4_t(-N as i32, -N as i32, -N as i32, -N as i32)) +} + +/// Signed saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrn_n_s32(a: int32x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshrn.v4i16")] + fn vqshrn_n_s32_(a: int32x4_t, n: i32) -> int16x4_t; + } +vqshrn_n_s32_(a, N) +} + +/// Signed saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_s64) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrn_n_s64(a: int64x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftns.v2i32")] + fn vqshrn_n_s64_(a: int64x2_t, n: int64x2_t) -> int32x2_t; + } +vqshrn_n_s64_(a, int64x2_t(-N as i64, -N as i64)) +} + +/// Signed saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_s64) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrn_n_s64(a: int64x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshrn.v2i32")] + fn vqshrn_n_s64_(a: int64x2_t, n: i32) -> int32x2_t; + } +vqshrn_n_s64_(a, N) +} + +/// Unsigned saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_u16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrn_n_u16(a: uint16x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftnu.v8i8")] + fn vqshrn_n_u16_(a: uint16x8_t, n: uint16x8_t) -> uint8x8_t; + } +vqshrn_n_u16_(a, uint16x8_t(-N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16)) +} + +/// Unsigned saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_u16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrn_n_u16(a: uint16x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshrn.v8i8")] + fn vqshrn_n_u16_(a: uint16x8_t, n: i32) -> uint8x8_t; + } +vqshrn_n_u16_(a, N) +} + +/// Unsigned saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_u32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrn_n_u32(a: uint32x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftnu.v4i16")] + fn vqshrn_n_u32_(a: uint32x4_t, n: uint32x4_t) -> uint16x4_t; + } +vqshrn_n_u32_(a, uint32x4_t(-N as u32, -N as u32, -N as u32, -N as u32)) +} + +/// Unsigned saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_u32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrn_n_u32(a: uint32x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshrn.v4i16")] + fn vqshrn_n_u32_(a: uint32x4_t, n: i32) -> uint16x4_t; + } +vqshrn_n_u32_(a, N) +} + +/// Unsigned saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_u64) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrn_n_u64(a: uint64x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftnu.v2i32")] + fn vqshrn_n_u64_(a: uint64x2_t, n: uint64x2_t) -> uint32x2_t; + } +vqshrn_n_u64_(a, uint64x2_t(-N as u64, -N as u64)) +} + +/// Unsigned saturating shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_u64) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrn_n_u64(a: uint64x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshrn.v2i32")] + fn vqshrn_n_u64_(a: uint64x2_t, n: i32) -> uint32x2_t; + } +vqshrn_n_u64_(a, N) +} + +/// Signed saturating shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrun_n_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrun_n_s16(a: int16x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftnsu.v8i8")] + fn vqshrun_n_s16_(a: int16x8_t, n: int16x8_t) -> uint8x8_t; + } +vqshrun_n_s16_(a, int16x8_t(-N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16)) +} + +/// Signed saturating shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrun_n_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrun_n_s16(a: int16x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshrun.v8i8")] + fn vqshrun_n_s16_(a: int16x8_t, n: i32) -> uint8x8_t; + } +vqshrun_n_s16_(a, N) +} + +/// Signed saturating shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrun_n_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrun_n_s32(a: int32x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftnsu.v4i16")] + fn vqshrun_n_s32_(a: int32x4_t, n: int32x4_t) -> uint16x4_t; + } +vqshrun_n_s32_(a, int32x4_t(-N as i32, -N as i32, -N as i32, -N as i32)) +} + +/// Signed saturating shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrun_n_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrun_n_s32(a: int32x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshrun.v4i16")] + fn vqshrun_n_s32_(a: int32x4_t, n: i32) -> uint16x4_t; + } +vqshrun_n_s32_(a, N) +} + +/// Signed saturating shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrun_n_s64) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vqshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrun_n_s64(a: int64x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftnsu.v2i32")] + fn vqshrun_n_s64_(a: int64x2_t, n: int64x2_t) -> uint32x2_t; + } +vqshrun_n_s64_(a, int64x2_t(-N as i64, -N as i64)) +} + +/// Signed saturating shift right unsigned narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrun_n_s64) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vqshrun_n_s64(a: int64x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshrun.v2i32")] + fn vqshrun_n_s64_(a: int64x2_t, n: i32) -> uint32x2_t; + } +vqshrun_n_s64_(a, N) +} + +/// Reciprocal square-root estimate. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrte_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsqrte))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(frsqrte))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsqrte_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrsqrte.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frsqrte.v2f32")] + fn vrsqrte_f32_(a: float32x2_t) -> float32x2_t; + } +vrsqrte_f32_(a) +} + +/// Reciprocal square-root estimate. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrteq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsqrte))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(frsqrte))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsqrteq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrsqrte.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frsqrte.v4f32")] + fn vrsqrteq_f32_(a: float32x4_t) -> float32x4_t; + } +vrsqrteq_f32_(a) +} + +/// Unsigned reciprocal square root estimate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrte_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsqrte))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ursqrte))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsqrte_u32(a: uint32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrsqrte.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ursqrte.v2i32")] + fn vrsqrte_u32_(a: uint32x2_t) -> uint32x2_t; + } +vrsqrte_u32_(a) +} + +/// Unsigned reciprocal square root estimate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrteq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsqrte))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ursqrte))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsqrteq_u32(a: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrsqrte.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ursqrte.v4i32")] + fn vrsqrteq_u32_(a: uint32x4_t) -> uint32x4_t; + } +vrsqrteq_u32_(a) +} + +/// Floating-point reciprocal square root step +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrts_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsqrts))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(frsqrts))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsqrts_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrsqrts.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frsqrts.v2f32")] + fn vrsqrts_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; + } +vrsqrts_f32_(a, b) +} + +/// Floating-point reciprocal square root step +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrtsq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsqrts))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(frsqrts))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsqrtsq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrsqrts.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frsqrts.v4f32")] + fn vrsqrtsq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + } +vrsqrtsq_f32_(a, b) +} + +/// Reciprocal estimate. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpe_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrecpe))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(frecpe))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrecpe_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrecpe.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frecpe.v2f32")] + fn vrecpe_f32_(a: float32x2_t) -> float32x2_t; + } +vrecpe_f32_(a) +} + +/// Reciprocal estimate. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpeq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrecpe))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(frecpe))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrecpeq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrecpe.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frecpe.v4f32")] + fn vrecpeq_f32_(a: float32x4_t) -> float32x4_t; + } +vrecpeq_f32_(a) +} + +/// Unsigned reciprocal estimate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpe_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrecpe))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urecpe))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrecpe_u32(a: uint32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrecpe.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urecpe.v2i32")] + fn vrecpe_u32_(a: uint32x2_t) -> uint32x2_t; + } +vrecpe_u32_(a) +} + +/// Unsigned reciprocal estimate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpeq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrecpe))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urecpe))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrecpeq_u32(a: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrecpe.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urecpe.v4i32")] + fn vrecpeq_u32_(a: uint32x4_t) -> uint32x4_t; + } +vrecpeq_u32_(a) +} + +/// Floating-point reciprocal step +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecps_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrecps))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(frecps))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrecps_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrecps.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frecps.v2f32")] + fn vrecps_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; + } +vrecps_f32_(a, b) +} + +/// Floating-point reciprocal step +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpsq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrecps))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(frecps))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrecpsq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrecps.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frecps.v4f32")] + fn vrecpsq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + } +vrecpsq_f32_(a, b) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s8_u8(a: uint8x8_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s8_p8(a: poly8x8_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s16_p16(a: poly16x4_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s16_u16(a: uint16x4_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s32_u32(a: uint32x2_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s64_u64(a: uint64x1_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s8_u8(a: uint8x16_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s8_p8(a: poly8x16_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s16_p16(a: poly16x8_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s16_u16(a: uint16x8_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s32_u32(a: uint32x4_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s64_u64(a: uint64x2_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u8_p8(a: poly8x8_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u8_s8(a: int8x8_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u16_p16(a: poly16x4_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u16_s16(a: int16x4_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u32_s32(a: int32x2_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u64_s64(a: int64x1_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u8_p8(a: poly8x16_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u8_s8(a: int8x16_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u16_p16(a: poly16x8_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u16_s16(a: int16x8_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u32_s32(a: int32x4_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u64_s64(a: int64x2_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p8_s8(a: int8x8_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p8_u8(a: uint8x8_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p16_s16(a: int16x4_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p16_u16(a: uint16x4_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p8_s8(a: int8x16_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p8_u8(a: uint8x16_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p16_s16(a: int16x8_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p16_u16(a: uint16x8_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s8_s16(a: int16x4_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s8_u16(a: uint16x4_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s8_p16(a: poly16x4_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s16_s32(a: int32x2_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s16_u32(a: uint32x2_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s32_s64(a: int64x1_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s32_u64(a: uint64x1_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s8_s16(a: int16x8_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s8_u16(a: uint16x8_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s8_p16(a: poly16x8_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s16_s32(a: int32x4_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s16_u32(a: uint32x4_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s32_s64(a: int64x2_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s32_u64(a: uint64x2_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u8_p16(a: poly16x4_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u8_s16(a: int16x4_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u8_u16(a: uint16x4_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u16_s32(a: int32x2_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u16_u32(a: uint32x2_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u32_s64(a: int64x1_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u32_u64(a: uint64x1_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u8_p16(a: poly16x8_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u8_s16(a: int16x8_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u8_u16(a: uint16x8_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u16_s32(a: int32x4_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u16_u32(a: uint32x4_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u32_s64(a: int64x2_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u32_u64(a: uint64x2_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p8_p16(a: poly16x4_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p8_s16(a: int16x4_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p8_u16(a: uint16x4_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p16_s32(a: int32x2_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p16_u32(a: uint32x2_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p8_p16(a: poly16x8_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p8_s16(a: int16x8_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p8_u16(a: uint16x8_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p16_s32(a: int32x4_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p16_u32(a: uint32x4_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s32_p64(a: poly64x1_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u32_p64(a: poly64x1_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s32_p64(a: poly64x2_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u32_p64(a: poly64x2_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_p128) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s64_p128(a: p128) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_p128) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u64_p128(a: p128) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_p128) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p64_p128(a: p128) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s16_p8(a: poly8x8_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s16_s8(a: int8x8_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s16_u8(a: uint8x8_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s32_p16(a: poly16x4_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s32_s16(a: int16x4_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s32_u16(a: uint16x4_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s64_s32(a: int32x2_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s64_u32(a: uint32x2_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s16_p8(a: poly8x16_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s16_s8(a: int8x16_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s16_u8(a: uint8x16_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s32_p16(a: poly16x8_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s32_s16(a: int16x8_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s32_u16(a: uint16x8_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s64_s32(a: int32x4_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s64_u32(a: uint32x4_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u16_p8(a: poly8x8_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u16_s8(a: int8x8_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u16_u8(a: uint8x8_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u32_p16(a: poly16x4_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u32_s16(a: int16x4_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u32_u16(a: uint16x4_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u64_s32(a: int32x2_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u64_u32(a: uint32x2_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u16_p8(a: poly8x16_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u16_s8(a: int8x16_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u16_u8(a: uint8x16_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u32_p16(a: poly16x8_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u32_s16(a: int16x8_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u32_u16(a: uint16x8_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u64_s32(a: int32x4_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u64_u32(a: uint32x4_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p16_p8(a: poly8x8_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p16_s8(a: int8x8_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p16_u8(a: uint8x8_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p16_p8(a: poly8x16_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p16_s8(a: int8x16_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p16_u8(a: uint8x16_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_s32) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p64_s32(a: int32x2_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_u32) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p64_u32(a: uint32x2_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_s32) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p64_s32(a: int32x4_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_u32) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p64_u32(a: uint32x4_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_s64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p128_s64(a: int64x2_t) -> p128 { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_u64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p128_u64(a: uint64x2_t) -> p128 { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p128_p64(a: poly64x2_t) -> p128 { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s8_s32(a: int32x2_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s8_u32(a: uint32x2_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s16_s64(a: int64x1_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s16_u64(a: uint64x1_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s8_s32(a: int32x4_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s8_u32(a: uint32x4_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s16_s64(a: int64x2_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s16_u64(a: uint64x2_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u8_s32(a: int32x2_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u8_u32(a: uint32x2_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u16_s64(a: int64x1_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u16_u64(a: uint64x1_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u8_s32(a: int32x4_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u8_u32(a: uint32x4_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u16_s64(a: int64x2_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u16_u64(a: uint64x2_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p8_s32(a: int32x2_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p8_u32(a: uint32x2_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p16_s64(a: int64x1_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p16_u64(a: uint64x1_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p8_s32(a: int32x4_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p8_u32(a: uint32x4_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p16_s64(a: int64x2_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p16_u64(a: uint64x2_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s16_p64(a: poly64x1_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u16_p64(a: poly64x1_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p16_p64(a: poly64x1_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s16_p64(a: poly64x2_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u16_p64(a: poly64x2_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p16_p64(a: poly64x2_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_p128) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s32_p128(a: p128) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_p128) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u32_p128(a: p128) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s32_p8(a: poly8x8_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s32_s8(a: int8x8_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s32_u8(a: uint8x8_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s64_p16(a: poly16x4_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s64_s16(a: int16x4_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s64_u16(a: uint16x4_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s32_p8(a: poly8x16_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s32_s8(a: int8x16_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s32_u8(a: uint8x16_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s64_p16(a: poly16x8_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s64_s16(a: int16x8_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s64_u16(a: uint16x8_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u32_p8(a: poly8x8_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u32_s8(a: int8x8_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u32_u8(a: uint8x8_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u64_p16(a: poly16x4_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u64_s16(a: int16x4_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u64_u16(a: uint16x4_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u32_p8(a: poly8x16_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u32_s8(a: int8x16_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u32_u8(a: uint8x16_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u64_p16(a: poly16x8_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u64_s16(a: int16x8_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u64_u16(a: uint16x8_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_p16) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p64_p16(a: poly16x4_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_s16) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p64_s16(a: int16x4_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_u16) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p64_u16(a: uint16x4_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_p16) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p64_p16(a: poly16x8_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_s16) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p64_s16(a: int16x8_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_u16) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p64_u16(a: uint16x8_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_s32) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p128_s32(a: int32x4_t) -> p128 { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_u32) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p128_u32(a: uint32x4_t) -> p128 { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s8_s64(a: int64x1_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s8_u64(a: uint64x1_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u8_s64(a: int64x1_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u8_u64(a: uint64x1_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p8_s64(a: int64x1_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p8_u64(a: uint64x1_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s8_s64(a: int64x2_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s8_u64(a: uint64x2_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u8_s64(a: int64x2_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u8_u64(a: uint64x2_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p8_s64(a: int64x2_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p8_u64(a: uint64x2_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s8_p64(a: poly64x1_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u8_p64(a: poly64x1_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p8_p64(a: poly64x1_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s8_p64(a: poly64x2_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u8_p64(a: poly64x2_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p8_p64(a: poly64x2_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_p128) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s16_p128(a: p128) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_p128) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u16_p128(a: p128) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_p128) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p16_p128(a: p128) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s64_p8(a: poly8x8_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s64_s8(a: int8x8_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s64_u8(a: uint8x8_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u64_p8(a: poly8x8_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u64_s8(a: int8x8_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u64_u8(a: uint8x8_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s64_p8(a: poly8x16_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s64_s8(a: int8x16_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s64_u8(a: uint8x16_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u64_p8(a: poly8x16_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u64_s8(a: int8x16_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u64_u8(a: uint8x16_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_p8) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p64_p8(a: poly8x8_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_s8) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p64_s8(a: int8x8_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_u8) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p64_u8(a: uint8x8_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_p8) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p64_p8(a: poly8x16_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_s8) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p64_s8(a: int8x16_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_u8) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p64_u8(a: uint8x16_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_s16) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p128_s16(a: int16x8_t) -> p128 { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_u16) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p128_u16(a: uint16x8_t) -> p128 { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_p16) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p128_p16(a: poly16x8_t) -> p128 { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_s8) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p128_s8(a: int8x16_t) -> p128 { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_u8) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p128_u8(a: uint8x16_t) -> p128 { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_p8) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p128_p8(a: poly8x16_t) -> p128 { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_p128) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s8_p128(a: p128) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_p128) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u8_p128(a: p128) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_p128) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p8_p128(a: p128) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s8_f32(a: float32x2_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s16_f32(a: float32x2_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s32_f32(a: float32x2_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_s64_f32(a: float32x2_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s8_f32(a: float32x4_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s16_f32(a: float32x4_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s32_f32(a: float32x4_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_s64_f32(a: float32x4_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u8_f32(a: float32x2_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u16_f32(a: float32x2_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u32_f32(a: float32x2_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_u64_f32(a: float32x2_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u8_f32(a: float32x4_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u16_f32(a: float32x4_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u32_f32(a: float32x4_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_u64_f32(a: float32x4_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p8_f32(a: float32x2_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_p16_f32(a: float32x2_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p8_f32(a: float32x4_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p16_f32(a: float32x4_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_p128_f32(a: float32x4_t) -> p128 { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_f32_s8(a: int8x8_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_f32_s16(a: int16x4_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_f32_s32(a: int32x2_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_f32_s64(a: int64x1_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_f32_s8(a: int8x16_t) -> float32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_f32_s16(a: int16x8_t) -> float32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_f32_s32(a: int32x4_t) -> float32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_f32_s64(a: int64x2_t) -> float32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_f32_u8(a: uint8x8_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_f32_u16(a: uint16x4_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_f32_u32(a: uint32x2_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_f32_u64(a: uint64x1_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_f32_u8(a: uint8x16_t) -> float32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_f32_u16(a: uint16x8_t) -> float32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_f32_u32(a: uint32x4_t) -> float32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_f32_u64(a: uint64x2_t) -> float32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_f32_p8(a: poly8x8_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpret_f32_p16(a: poly16x4_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_f32_p8(a: poly8x16_t) -> float32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_f32_p16(a: poly16x8_t) -> float32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_p128) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vreinterpretq_f32_p128(a: p128) -> float32x4_t { + transmute(a) +} + +/// Signed rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshl_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshl_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshifts.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srshl.v8i8")] + fn vrshl_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vrshl_s8_(a, b) +} + +/// Signed rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshlq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshlq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshifts.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srshl.v16i8")] + fn vrshlq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vrshlq_s8_(a, b) +} + +/// Signed rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshl_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshl_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshifts.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srshl.v4i16")] + fn vrshl_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vrshl_s16_(a, b) +} + +/// Signed rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshlq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshlq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshifts.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srshl.v8i16")] + fn vrshlq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vrshlq_s16_(a, b) +} + +/// Signed rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshl_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshl_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshifts.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srshl.v2i32")] + fn vrshl_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vrshl_s32_(a, b) +} + +/// Signed rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshlq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshlq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshifts.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srshl.v4i32")] + fn vrshlq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vrshlq_s32_(a, b) +} + +/// Signed rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshl_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshl_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshifts.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srshl.v1i64")] + fn vrshl_s64_(a: int64x1_t, b: int64x1_t) -> int64x1_t; + } +vrshl_s64_(a, b) +} + +/// Signed rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshlq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshlq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshifts.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srshl.v2i64")] + fn vrshlq_s64_(a: int64x2_t, b: int64x2_t) -> int64x2_t; + } +vrshlq_s64_(a, b) +} + +/// Unsigned rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshl_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshl_u8(a: uint8x8_t, b: int8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urshl.v8i8")] + fn vrshl_u8_(a: uint8x8_t, b: int8x8_t) -> uint8x8_t; + } +vrshl_u8_(a, b) +} + +/// Unsigned rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshlq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshlq_u8(a: uint8x16_t, b: int8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftu.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urshl.v16i8")] + fn vrshlq_u8_(a: uint8x16_t, b: int8x16_t) -> uint8x16_t; + } +vrshlq_u8_(a, b) +} + +/// Unsigned rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshl_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshl_u16(a: uint16x4_t, b: int16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urshl.v4i16")] + fn vrshl_u16_(a: uint16x4_t, b: int16x4_t) -> uint16x4_t; + } +vrshl_u16_(a, b) +} + +/// Unsigned rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshlq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshlq_u16(a: uint16x8_t, b: int16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftu.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urshl.v8i16")] + fn vrshlq_u16_(a: uint16x8_t, b: int16x8_t) -> uint16x8_t; + } +vrshlq_u16_(a, b) +} + +/// Unsigned rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshl_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshl_u32(a: uint32x2_t, b: int32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urshl.v2i32")] + fn vrshl_u32_(a: uint32x2_t, b: int32x2_t) -> uint32x2_t; + } +vrshl_u32_(a, b) +} + +/// Unsigned rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshlq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshlq_u32(a: uint32x4_t, b: int32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftu.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urshl.v4i32")] + fn vrshlq_u32_(a: uint32x4_t, b: int32x4_t) -> uint32x4_t; + } +vrshlq_u32_(a, b) +} + +/// Unsigned rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshl_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshl_u64(a: uint64x1_t, b: int64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftu.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urshl.v1i64")] + fn vrshl_u64_(a: uint64x1_t, b: int64x1_t) -> uint64x1_t; + } +vrshl_u64_(a, b) +} + +/// Unsigned rounding shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshlq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshlq_u64(a: uint64x2_t, b: int64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftu.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urshl.v2i64")] + fn vrshlq_u64_(a: uint64x2_t, b: int64x2_t) -> uint64x2_t; + } +vrshlq_u64_(a, b) +} + +/// Signed rounding shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshr_n_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshr_n_s8(a: int8x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + vrshl_s8(a, vdup_n_s8((-N) as _)) +} + +/// Signed rounding shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrq_n_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshrq_n_s8(a: int8x16_t) -> int8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + vrshlq_s8(a, vdupq_n_s8((-N) as _)) +} + +/// Signed rounding shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshr_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshr_n_s16(a: int16x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + vrshl_s16(a, vdup_n_s16((-N) as _)) +} + +/// Signed rounding shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrq_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshrq_n_s16(a: int16x8_t) -> int16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + vrshlq_s16(a, vdupq_n_s16((-N) as _)) +} + +/// Signed rounding shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshr_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshr_n_s32(a: int32x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + vrshl_s32(a, vdup_n_s32((-N) as _)) +} + +/// Signed rounding shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrq_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshrq_n_s32(a: int32x4_t) -> int32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + vrshlq_s32(a, vdupq_n_s32((-N) as _)) +} + +/// Signed rounding shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshr_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshr_n_s64(a: int64x1_t) -> int64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + vrshl_s64(a, vdup_n_s64((-N) as _)) +} + +/// Signed rounding shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrq_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshrq_n_s64(a: int64x2_t) -> int64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + vrshlq_s64(a, vdupq_n_s64((-N) as _)) +} + +/// Unsigned rounding shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshr_n_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshr_n_u8(a: uint8x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + vrshl_u8(a, vdup_n_s8((-N) as _)) +} + +/// Unsigned rounding shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrq_n_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshrq_n_u8(a: uint8x16_t) -> uint8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + vrshlq_u8(a, vdupq_n_s8((-N) as _)) +} + +/// Unsigned rounding shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshr_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshr_n_u16(a: uint16x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + vrshl_u16(a, vdup_n_s16((-N) as _)) +} + +/// Unsigned rounding shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrq_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshrq_n_u16(a: uint16x8_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + vrshlq_u16(a, vdupq_n_s16((-N) as _)) +} + +/// Unsigned rounding shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshr_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshr_n_u32(a: uint32x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + vrshl_u32(a, vdup_n_s32((-N) as _)) +} + +/// Unsigned rounding shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrq_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshrq_n_u32(a: uint32x4_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + vrshlq_u32(a, vdupq_n_s32((-N) as _)) +} + +/// Unsigned rounding shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshr_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshr_n_u64(a: uint64x1_t) -> uint64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + vrshl_u64(a, vdup_n_s64((-N) as _)) +} + +/// Unsigned rounding shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrq_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshrq_n_u64(a: uint64x2_t) -> uint64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + vrshlq_u64(a, vdupq_n_s64((-N) as _)) +} + +/// Rounding shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_n_s16) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrn_n_s16(a: int16x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftn.v8i8")] + fn vrshrn_n_s16_(a: int16x8_t, n: int16x8_t) -> int8x8_t; + } +vrshrn_n_s16_(a, int16x8_t(-N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16)) +} + +/// Rounding shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_n_s16) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrshrn_n_s16(a: int16x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.rshrn.v8i8")] + fn vrshrn_n_s16_(a: int16x8_t, n: i32) -> int8x8_t; + } +vrshrn_n_s16_(a, N) +} + +/// Rounding shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_n_s32) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrn_n_s32(a: int32x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftn.v4i16")] + fn vrshrn_n_s32_(a: int32x4_t, n: int32x4_t) -> int16x4_t; + } +vrshrn_n_s32_(a, int32x4_t(-N as i32, -N as i32, -N as i32, -N as i32)) +} + +/// Rounding shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_n_s32) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrshrn_n_s32(a: int32x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.rshrn.v4i16")] + fn vrshrn_n_s32_(a: int32x4_t, n: i32) -> int16x4_t; + } +vrshrn_n_s32_(a, N) +} + +/// Rounding shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_n_s64) +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrn_n_s64(a: int64x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftn.v2i32")] + fn vrshrn_n_s64_(a: int64x2_t, n: int64x2_t) -> int32x2_t; + } +vrshrn_n_s64_(a, int64x2_t(-N as i64, -N as i64)) +} + +/// Rounding shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_n_s64) +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "neon_intrinsics", since = "1.59.0")] +pub unsafe fn vrshrn_n_s64(a: int64x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.rshrn.v2i32")] + fn vrshrn_n_s64_(a: int64x2_t, n: i32) -> int32x2_t; + } +vrshrn_n_s64_(a, N) +} + +/// Rounding shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshrn, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshrn_n_u16(a: uint16x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + transmute(vrshrn_n_s16::(transmute(a))) +} + +/// Rounding shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshrn, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshrn_n_u32(a: uint32x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + transmute(vrshrn_n_s32::(transmute(a))) +} + +/// Rounding shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshrn, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrshrn_n_u64(a: uint64x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + transmute(vrshrn_n_s64::(transmute(a))) +} + +/// Signed rounding shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsra_n_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srsra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsra_n_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_add(a, vrshr_n_s8::(b)) +} + +/// Signed rounding shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsraq_n_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srsra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsraq_n_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_add(a, vrshrq_n_s8::(b)) +} + +/// Signed rounding shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsra_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srsra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsra_n_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_add(a, vrshr_n_s16::(b)) +} + +/// Signed rounding shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsraq_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srsra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsraq_n_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_add(a, vrshrq_n_s16::(b)) +} + +/// Signed rounding shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsra_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srsra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsra_n_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_add(a, vrshr_n_s32::(b)) +} + +/// Signed rounding shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsraq_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srsra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsraq_n_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_add(a, vrshrq_n_s32::(b)) +} + +/// Signed rounding shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsra_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srsra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsra_n_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + simd_add(a, vrshr_n_s64::(b)) +} + +/// Signed rounding shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsraq_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srsra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsraq_n_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + simd_add(a, vrshrq_n_s64::(b)) +} + +/// Unsigned rounding shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsra_n_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ursra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsra_n_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_add(a, vrshr_n_u8::(b)) +} + +/// Unsigned rounding shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsraq_n_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ursra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsraq_n_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_add(a, vrshrq_n_u8::(b)) +} + +/// Unsigned rounding shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsra_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ursra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsra_n_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_add(a, vrshr_n_u16::(b)) +} + +/// Unsigned rounding shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsraq_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ursra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsraq_n_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_add(a, vrshrq_n_u16::(b)) +} + +/// Unsigned rounding shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsra_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ursra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsra_n_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_add(a, vrshr_n_u32::(b)) +} + +/// Unsigned rounding shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsraq_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ursra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsraq_n_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_add(a, vrshrq_n_u32::(b)) +} + +/// Unsigned rounding shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsra_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ursra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsra_n_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + simd_add(a, vrshr_n_u64::(b)) +} + +/// Unsigned rounding shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsraq_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ursra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsraq_n_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + simd_add(a, vrshrq_n_u64::(b)) +} + +/// Rounding subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rsubhn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsubhn_s16(a: int16x8_t, b: int16x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrsubhn.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.rsubhn.v8i8")] + fn vrsubhn_s16_(a: int16x8_t, b: int16x8_t) -> int8x8_t; + } +vrsubhn_s16_(a, b) +} + +/// Rounding subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rsubhn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsubhn_s32(a: int32x4_t, b: int32x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrsubhn.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.rsubhn.v4i16")] + fn vrsubhn_s32_(a: int32x4_t, b: int32x4_t) -> int16x4_t; + } +vrsubhn_s32_(a, b) +} + +/// Rounding subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rsubhn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsubhn_s64(a: int64x2_t, b: int64x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrsubhn.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.rsubhn.v2i32")] + fn vrsubhn_s64_(a: int64x2_t, b: int64x2_t) -> int32x2_t; + } +vrsubhn_s64_(a, b) +} + +/// Rounding subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rsubhn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsubhn_u16(a: uint16x8_t, b: uint16x8_t) -> uint8x8_t { + transmute(vrsubhn_s16(transmute(a), transmute(b))) +} + +/// Rounding subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rsubhn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsubhn_u32(a: uint32x4_t, b: uint32x4_t) -> uint16x4_t { + transmute(vrsubhn_s32(transmute(a), transmute(b))) +} + +/// Rounding subtract returning high narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rsubhn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vrsubhn_u64(a: uint64x2_t, b: uint64x2_t) -> uint32x2_t { + transmute(vrsubhn_s64(transmute(a), transmute(b))) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vset_lane_s8(a: i8, b: int8x8_t) -> int8x8_t { + static_assert_imm3!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vset_lane_s16(a: i16, b: int16x4_t) -> int16x4_t { + static_assert_imm2!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vset_lane_s32(a: i32, b: int32x2_t) -> int32x2_t { + static_assert_imm1!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vset_lane_s64(a: i64, b: int64x1_t) -> int64x1_t { + static_assert!(LANE : i32 where LANE == 0); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vset_lane_u8(a: u8, b: uint8x8_t) -> uint8x8_t { + static_assert_imm3!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vset_lane_u16(a: u16, b: uint16x4_t) -> uint16x4_t { + static_assert_imm2!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vset_lane_u32(a: u32, b: uint32x2_t) -> uint32x2_t { + static_assert_imm1!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vset_lane_u64(a: u64, b: uint64x1_t) -> uint64x1_t { + static_assert!(LANE : i32 where LANE == 0); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vset_lane_p8(a: p8, b: poly8x8_t) -> poly8x8_t { + static_assert_imm3!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vset_lane_p16(a: p16, b: poly16x4_t) -> poly16x4_t { + static_assert_imm2!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vset_lane_p64(a: p64, b: poly64x1_t) -> poly64x1_t { + static_assert!(LANE : i32 where LANE == 0); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsetq_lane_s8(a: i8, b: int8x16_t) -> int8x16_t { + static_assert_imm4!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsetq_lane_s16(a: i16, b: int16x8_t) -> int16x8_t { + static_assert_imm3!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsetq_lane_s32(a: i32, b: int32x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsetq_lane_s64(a: i64, b: int64x2_t) -> int64x2_t { + static_assert_imm1!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsetq_lane_u8(a: u8, b: uint8x16_t) -> uint8x16_t { + static_assert_imm4!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsetq_lane_u16(a: u16, b: uint16x8_t) -> uint16x8_t { + static_assert_imm3!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsetq_lane_u32(a: u32, b: uint32x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsetq_lane_u64(a: u64, b: uint64x2_t) -> uint64x2_t { + static_assert_imm1!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsetq_lane_p8(a: p8, b: poly8x16_t) -> poly8x16_t { + static_assert_imm4!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsetq_lane_p16(a: p16, b: poly16x8_t) -> poly16x8_t { + static_assert_imm3!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "aes,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsetq_lane_p64(a: p64, b: poly64x2_t) -> poly64x2_t { + static_assert_imm1!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vset_lane_f32(a: f32, b: float32x2_t) -> float32x2_t { + static_assert_imm1!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsetq_lane_f32(a: f32, b: float32x4_t) -> float32x4_t { + static_assert_imm2!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Signed Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshl_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshifts.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sshl.v8i8")] + fn vshl_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vshl_s8_(a, b) +} + +/// Signed Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshlq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshifts.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sshl.v16i8")] + fn vshlq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vshlq_s8_(a, b) +} + +/// Signed Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshl_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshifts.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sshl.v4i16")] + fn vshl_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vshl_s16_(a, b) +} + +/// Signed Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshlq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshifts.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sshl.v8i16")] + fn vshlq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vshlq_s16_(a, b) +} + +/// Signed Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshl_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshifts.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sshl.v2i32")] + fn vshl_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vshl_s32_(a, b) +} + +/// Signed Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshlq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshifts.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sshl.v4i32")] + fn vshlq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vshlq_s32_(a, b) +} + +/// Signed Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshl_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshifts.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sshl.v1i64")] + fn vshl_s64_(a: int64x1_t, b: int64x1_t) -> int64x1_t; + } +vshl_s64_(a, b) +} + +/// Signed Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshlq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshifts.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sshl.v2i64")] + fn vshlq_s64_(a: int64x2_t, b: int64x2_t) -> int64x2_t; + } +vshlq_s64_(a, b) +} + +/// Unsigned Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshl_u8(a: uint8x8_t, b: int8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshiftu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ushl.v8i8")] + fn vshl_u8_(a: uint8x8_t, b: int8x8_t) -> uint8x8_t; + } +vshl_u8_(a, b) +} + +/// Unsigned Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshlq_u8(a: uint8x16_t, b: int8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshiftu.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ushl.v16i8")] + fn vshlq_u8_(a: uint8x16_t, b: int8x16_t) -> uint8x16_t; + } +vshlq_u8_(a, b) +} + +/// Unsigned Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshl_u16(a: uint16x4_t, b: int16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshiftu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ushl.v4i16")] + fn vshl_u16_(a: uint16x4_t, b: int16x4_t) -> uint16x4_t; + } +vshl_u16_(a, b) +} + +/// Unsigned Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshlq_u16(a: uint16x8_t, b: int16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshiftu.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ushl.v8i16")] + fn vshlq_u16_(a: uint16x8_t, b: int16x8_t) -> uint16x8_t; + } +vshlq_u16_(a, b) +} + +/// Unsigned Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshl_u32(a: uint32x2_t, b: int32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshiftu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ushl.v2i32")] + fn vshl_u32_(a: uint32x2_t, b: int32x2_t) -> uint32x2_t; + } +vshl_u32_(a, b) +} + +/// Unsigned Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshlq_u32(a: uint32x4_t, b: int32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshiftu.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ushl.v4i32")] + fn vshlq_u32_(a: uint32x4_t, b: int32x4_t) -> uint32x4_t; + } +vshlq_u32_(a, b) +} + +/// Unsigned Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshl_u64(a: uint64x1_t, b: int64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshiftu.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ushl.v1i64")] + fn vshl_u64_(a: uint64x1_t, b: int64x1_t) -> uint64x1_t; + } +vshl_u64_(a, b) +} + +/// Unsigned Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushl))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshlq_u64(a: uint64x2_t, b: int64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshiftu.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ushl.v2i64")] + fn vshlq_u64_(a: uint64x2_t, b: int64x2_t) -> uint64x2_t; + } +vshlq_u64_(a, b) +} + +/// Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_n_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshl_n_s8(a: int8x8_t) -> int8x8_t { + static_assert_imm3!(N); + simd_shl(a, vdup_n_s8(N as _)) +} + +/// Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_n_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshlq_n_s8(a: int8x16_t) -> int8x16_t { + static_assert_imm3!(N); + simd_shl(a, vdupq_n_s8(N as _)) +} + +/// Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshl_n_s16(a: int16x4_t) -> int16x4_t { + static_assert_imm4!(N); + simd_shl(a, vdup_n_s16(N as _)) +} + +/// Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshlq_n_s16(a: int16x8_t) -> int16x8_t { + static_assert_imm4!(N); + simd_shl(a, vdupq_n_s16(N as _)) +} + +/// Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshl_n_s32(a: int32x2_t) -> int32x2_t { + static_assert_imm5!(N); + simd_shl(a, vdup_n_s32(N as _)) +} + +/// Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshlq_n_s32(a: int32x4_t) -> int32x4_t { + static_assert_imm5!(N); + simd_shl(a, vdupq_n_s32(N as _)) +} + +/// Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_n_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshl_n_u8(a: uint8x8_t) -> uint8x8_t { + static_assert_imm3!(N); + simd_shl(a, vdup_n_u8(N as _)) +} + +/// Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_n_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshlq_n_u8(a: uint8x16_t) -> uint8x16_t { + static_assert_imm3!(N); + simd_shl(a, vdupq_n_u8(N as _)) +} + +/// Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshl_n_u16(a: uint16x4_t) -> uint16x4_t { + static_assert_imm4!(N); + simd_shl(a, vdup_n_u16(N as _)) +} + +/// Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshlq_n_u16(a: uint16x8_t) -> uint16x8_t { + static_assert_imm4!(N); + simd_shl(a, vdupq_n_u16(N as _)) +} + +/// Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshl_n_u32(a: uint32x2_t) -> uint32x2_t { + static_assert_imm5!(N); + simd_shl(a, vdup_n_u32(N as _)) +} + +/// Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshlq_n_u32(a: uint32x4_t) -> uint32x4_t { + static_assert_imm5!(N); + simd_shl(a, vdupq_n_u32(N as _)) +} + +/// Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshl_n_s64(a: int64x1_t) -> int64x1_t { + static_assert_imm6!(N); + simd_shl(a, vdup_n_s64(N as _)) +} + +/// Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshlq_n_s64(a: int64x2_t) -> int64x2_t { + static_assert_imm6!(N); + simd_shl(a, vdupq_n_s64(N as _)) +} + +/// Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshl_n_u64(a: uint64x1_t) -> uint64x1_t { + static_assert_imm6!(N); + simd_shl(a, vdup_n_u64(N as _)) +} + +/// Shift left +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshlq_n_u64(a: uint64x2_t) -> uint64x2_t { + static_assert_imm6!(N); + simd_shl(a, vdupq_n_u64(N as _)) +} + +/// Signed shift left long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_n_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshll.s8", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshll, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshll_n_s8(a: int8x8_t) -> int16x8_t { + static_assert!(N : i32 where N >= 0 && N <= 8); + simd_shl(simd_cast(a), vdupq_n_s16(N as _)) +} + +/// Signed shift left long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshll.s16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshll, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshll_n_s16(a: int16x4_t) -> int32x4_t { + static_assert!(N : i32 where N >= 0 && N <= 16); + simd_shl(simd_cast(a), vdupq_n_s32(N as _)) +} + +/// Signed shift left long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshll.s32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshll, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshll_n_s32(a: int32x2_t) -> int64x2_t { + static_assert!(N : i32 where N >= 0 && N <= 32); + simd_shl(simd_cast(a), vdupq_n_s64(N as _)) +} + +/// Signed shift left long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_n_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshll.u8", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushll, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshll_n_u8(a: uint8x8_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 0 && N <= 8); + simd_shl(simd_cast(a), vdupq_n_u16(N as _)) +} + +/// Signed shift left long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshll.u16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushll, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshll_n_u16(a: uint16x4_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 0 && N <= 16); + simd_shl(simd_cast(a), vdupq_n_u32(N as _)) +} + +/// Signed shift left long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshll.u32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushll, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshll_n_u32(a: uint32x2_t) -> uint64x2_t { + static_assert!(N : i32 where N >= 0 && N <= 32); + simd_shl(simd_cast(a), vdupq_n_u64(N as _)) +} + +/// Shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshr_n_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s8", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshr_n_s8(a: int8x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + let n: i32 = if N == 8 { 7 } else { N }; + simd_shr(a, vdup_n_s8(n as _)) +} + +/// Shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrq_n_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s8", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshrq_n_s8(a: int8x16_t) -> int8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + let n: i32 = if N == 8 { 7 } else { N }; + simd_shr(a, vdupq_n_s8(n as _)) +} + +/// Shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshr_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshr_n_s16(a: int16x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + let n: i32 = if N == 16 { 15 } else { N }; + simd_shr(a, vdup_n_s16(n as _)) +} + +/// Shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrq_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshrq_n_s16(a: int16x8_t) -> int16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + let n: i32 = if N == 16 { 15 } else { N }; + simd_shr(a, vdupq_n_s16(n as _)) +} + +/// Shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshr_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshr_n_s32(a: int32x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + let n: i32 = if N == 32 { 31 } else { N }; + simd_shr(a, vdup_n_s32(n as _)) +} + +/// Shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrq_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshrq_n_s32(a: int32x4_t) -> int32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + let n: i32 = if N == 32 { 31 } else { N }; + simd_shr(a, vdupq_n_s32(n as _)) +} + +/// Shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshr_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s64", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshr_n_s64(a: int64x1_t) -> int64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + let n: i32 = if N == 64 { 63 } else { N }; + simd_shr(a, vdup_n_s64(n as _)) +} + +/// Shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrq_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s64", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshrq_n_s64(a: int64x2_t) -> int64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + let n: i32 = if N == 64 { 63 } else { N }; + simd_shr(a, vdupq_n_s64(n as _)) +} + +/// Shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshr_n_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u8", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshr_n_u8(a: uint8x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + let n: i32 = if N == 8 { return vdup_n_u8(0); } else { N }; + simd_shr(a, vdup_n_u8(n as _)) +} + +/// Shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrq_n_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u8", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshrq_n_u8(a: uint8x16_t) -> uint8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + let n: i32 = if N == 8 { return vdupq_n_u8(0); } else { N }; + simd_shr(a, vdupq_n_u8(n as _)) +} + +/// Shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshr_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshr_n_u16(a: uint16x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + let n: i32 = if N == 16 { return vdup_n_u16(0); } else { N }; + simd_shr(a, vdup_n_u16(n as _)) +} + +/// Shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrq_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshrq_n_u16(a: uint16x8_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + let n: i32 = if N == 16 { return vdupq_n_u16(0); } else { N }; + simd_shr(a, vdupq_n_u16(n as _)) +} + +/// Shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshr_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshr_n_u32(a: uint32x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + let n: i32 = if N == 32 { return vdup_n_u32(0); } else { N }; + simd_shr(a, vdup_n_u32(n as _)) +} + +/// Shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrq_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshrq_n_u32(a: uint32x4_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + let n: i32 = if N == 32 { return vdupq_n_u32(0); } else { N }; + simd_shr(a, vdupq_n_u32(n as _)) +} + +/// Shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshr_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u64", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshr_n_u64(a: uint64x1_t) -> uint64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + let n: i32 = if N == 64 { return vdup_n_u64(0); } else { N }; + simd_shr(a, vdup_n_u64(n as _)) +} + +/// Shift right +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrq_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u64", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushr, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshrq_n_u64(a: uint64x2_t) -> uint64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + let n: i32 = if N == 64 { return vdupq_n_u64(0); } else { N }; + simd_shr(a, vdupq_n_u64(n as _)) +} + +/// Shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshrn.i16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshrn_n_s16(a: int16x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_cast(simd_shr(a, vdupq_n_s16(N as _))) +} + +/// Shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshrn.i32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshrn_n_s32(a: int32x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_cast(simd_shr(a, vdupq_n_s32(N as _))) +} + +/// Shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshrn.i64", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshrn_n_s64(a: int64x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_cast(simd_shr(a, vdupq_n_s64(N as _))) +} + +/// Shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshrn.i16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshrn_n_u16(a: uint16x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_cast(simd_shr(a, vdupq_n_u16(N as _))) +} + +/// Shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshrn.i32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshrn_n_u32(a: uint32x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_cast(simd_shr(a, vdupq_n_u32(N as _))) +} + +/// Shift right narrow +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshrn.i64", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shrn, N = 2))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vshrn_n_u64(a: uint64x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_cast(simd_shr(a, vdupq_n_u64(N as _))) +} + +/// Signed shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsra_n_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsra_n_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_add(a, vshr_n_s8::(b)) +} + +/// Signed shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsraq_n_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsraq_n_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_add(a, vshrq_n_s8::(b)) +} + +/// Signed shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsra_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsra_n_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_add(a, vshr_n_s16::(b)) +} + +/// Signed shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsraq_n_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsraq_n_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_add(a, vshrq_n_s16::(b)) +} + +/// Signed shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsra_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsra_n_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_add(a, vshr_n_s32::(b)) +} + +/// Signed shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsraq_n_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsraq_n_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_add(a, vshrq_n_s32::(b)) +} + +/// Signed shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsra_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsra_n_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + simd_add(a, vshr_n_s64::(b)) +} + +/// Signed shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsraq_n_s64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsraq_n_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + simd_add(a, vshrq_n_s64::(b)) +} + +/// Unsigned shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsra_n_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsra_n_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_add(a, vshr_n_u8::(b)) +} + +/// Unsigned shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsraq_n_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsraq_n_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_add(a, vshrq_n_u8::(b)) +} + +/// Unsigned shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsra_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsra_n_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_add(a, vshr_n_u16::(b)) +} + +/// Unsigned shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsraq_n_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsraq_n_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_add(a, vshrq_n_u16::(b)) +} + +/// Unsigned shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsra_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsra_n_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_add(a, vshr_n_u32::(b)) +} + +/// Unsigned shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsraq_n_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsraq_n_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_add(a, vshrq_n_u32::(b)) +} + +/// Unsigned shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsra_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsra_n_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + simd_add(a, vshr_n_u64::(b)) +} + +/// Unsigned shift right and accumulate +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsraq_n_u64) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usra, N = 2))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vsraq_n_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + simd_add(a, vshrq_n_u64::(b)) +} + +/// Transpose elements +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(trn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtrn_s8(a: int8x8_t, b: int8x8_t) -> int8x8x2_t { + let a1: int8x8_t = simd_shuffle8!(a, b, [0, 8, 2, 10, 4, 12, 6, 14]); + let b1: int8x8_t = simd_shuffle8!(a, b, [1, 9, 3, 11, 5, 13, 7, 15]); + transmute((a1, b1)) +} + +/// Transpose elements +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(trn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtrn_s16(a: int16x4_t, b: int16x4_t) -> int16x4x2_t { + let a1: int16x4_t = simd_shuffle4!(a, b, [0, 4, 2, 6]); + let b1: int16x4_t = simd_shuffle4!(a, b, [1, 5, 3, 7]); + transmute((a1, b1)) +} + +/// Transpose elements +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrnq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(trn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtrnq_s8(a: int8x16_t, b: int8x16_t) -> int8x16x2_t { + let a1: int8x16_t = simd_shuffle16!(a, b, [0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]); + let b1: int8x16_t = simd_shuffle16!(a, b, [1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]); + transmute((a1, b1)) +} + +/// Transpose elements +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrnq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(trn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtrnq_s16(a: int16x8_t, b: int16x8_t) -> int16x8x2_t { + let a1: int16x8_t = simd_shuffle8!(a, b, [0, 8, 2, 10, 4, 12, 6, 14]); + let b1: int16x8_t = simd_shuffle8!(a, b, [1, 9, 3, 11, 5, 13, 7, 15]); + transmute((a1, b1)) +} + +/// Transpose elements +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrnq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(trn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtrnq_s32(a: int32x4_t, b: int32x4_t) -> int32x4x2_t { + let a1: int32x4_t = simd_shuffle4!(a, b, [0, 4, 2, 6]); + let b1: int32x4_t = simd_shuffle4!(a, b, [1, 5, 3, 7]); + transmute((a1, b1)) +} + +/// Transpose elements +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(trn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtrn_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8x2_t { + let a1: uint8x8_t = simd_shuffle8!(a, b, [0, 8, 2, 10, 4, 12, 6, 14]); + let b1: uint8x8_t = simd_shuffle8!(a, b, [1, 9, 3, 11, 5, 13, 7, 15]); + transmute((a1, b1)) +} + +/// Transpose elements +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(trn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtrn_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4x2_t { + let a1: uint16x4_t = simd_shuffle4!(a, b, [0, 4, 2, 6]); + let b1: uint16x4_t = simd_shuffle4!(a, b, [1, 5, 3, 7]); + transmute((a1, b1)) +} + +/// Transpose elements +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrnq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(trn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtrnq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16x2_t { + let a1: uint8x16_t = simd_shuffle16!(a, b, [0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]); + let b1: uint8x16_t = simd_shuffle16!(a, b, [1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]); + transmute((a1, b1)) +} + +/// Transpose elements +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrnq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(trn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtrnq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8x2_t { + let a1: uint16x8_t = simd_shuffle8!(a, b, [0, 8, 2, 10, 4, 12, 6, 14]); + let b1: uint16x8_t = simd_shuffle8!(a, b, [1, 9, 3, 11, 5, 13, 7, 15]); + transmute((a1, b1)) +} + +/// Transpose elements +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrnq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(trn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtrnq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4x2_t { + let a1: uint32x4_t = simd_shuffle4!(a, b, [0, 4, 2, 6]); + let b1: uint32x4_t = simd_shuffle4!(a, b, [1, 5, 3, 7]); + transmute((a1, b1)) +} + +/// Transpose elements +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(trn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtrn_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8x2_t { + let a1: poly8x8_t = simd_shuffle8!(a, b, [0, 8, 2, 10, 4, 12, 6, 14]); + let b1: poly8x8_t = simd_shuffle8!(a, b, [1, 9, 3, 11, 5, 13, 7, 15]); + transmute((a1, b1)) +} + +/// Transpose elements +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(trn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtrn_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4x2_t { + let a1: poly16x4_t = simd_shuffle4!(a, b, [0, 4, 2, 6]); + let b1: poly16x4_t = simd_shuffle4!(a, b, [1, 5, 3, 7]); + transmute((a1, b1)) +} + +/// Transpose elements +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrnq_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(trn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtrnq_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16x2_t { + let a1: poly8x16_t = simd_shuffle16!(a, b, [0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]); + let b1: poly8x16_t = simd_shuffle16!(a, b, [1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]); + transmute((a1, b1)) +} + +/// Transpose elements +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrnq_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(trn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtrnq_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8x2_t { + let a1: poly16x8_t = simd_shuffle8!(a, b, [0, 8, 2, 10, 4, 12, 6, 14]); + let b1: poly16x8_t = simd_shuffle8!(a, b, [1, 9, 3, 11, 5, 13, 7, 15]); + transmute((a1, b1)) +} + +/// Transpose elements +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(zip))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtrn_s32(a: int32x2_t, b: int32x2_t) -> int32x2x2_t { + let a1: int32x2_t = simd_shuffle2!(a, b, [0, 2]); + let b1: int32x2_t = simd_shuffle2!(a, b, [1, 3]); + transmute((a1, b1)) +} + +/// Transpose elements +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(zip))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtrn_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2x2_t { + let a1: uint32x2_t = simd_shuffle2!(a, b, [0, 2]); + let b1: uint32x2_t = simd_shuffle2!(a, b, [1, 3]); + transmute((a1, b1)) +} + +/// Transpose elements +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(zip))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtrn_f32(a: float32x2_t, b: float32x2_t) -> float32x2x2_t { + let a1: float32x2_t = simd_shuffle2!(a, b, [0, 2]); + let b1: float32x2_t = simd_shuffle2!(a, b, [1, 3]); + transmute((a1, b1)) +} + +/// Transpose elements +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrnq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(trn))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vtrnq_f32(a: float32x4_t, b: float32x4_t) -> float32x4x2_t { + let a1: float32x4_t = simd_shuffle4!(a, b, [0, 4, 2, 6]); + let b1: float32x4_t = simd_shuffle4!(a, b, [1, 5, 3, 7]); + transmute((a1, b1)) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(zip))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vzip_s8(a: int8x8_t, b: int8x8_t) -> int8x8x2_t { + let a0: int8x8_t = simd_shuffle8!(a, b, [0, 8, 1, 9, 2, 10, 3, 11]); + let b0: int8x8_t = simd_shuffle8!(a, b, [4, 12, 5, 13, 6, 14, 7, 15]); + transmute((a0, b0)) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(zip))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vzip_s16(a: int16x4_t, b: int16x4_t) -> int16x4x2_t { + let a0: int16x4_t = simd_shuffle4!(a, b, [0, 4, 1, 5]); + let b0: int16x4_t = simd_shuffle4!(a, b, [2, 6, 3, 7]); + transmute((a0, b0)) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(zip))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vzip_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8x2_t { + let a0: uint8x8_t = simd_shuffle8!(a, b, [0, 8, 1, 9, 2, 10, 3, 11]); + let b0: uint8x8_t = simd_shuffle8!(a, b, [4, 12, 5, 13, 6, 14, 7, 15]); + transmute((a0, b0)) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(zip))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vzip_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4x2_t { + let a0: uint16x4_t = simd_shuffle4!(a, b, [0, 4, 1, 5]); + let b0: uint16x4_t = simd_shuffle4!(a, b, [2, 6, 3, 7]); + transmute((a0, b0)) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(zip))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vzip_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8x2_t { + let a0: poly8x8_t = simd_shuffle8!(a, b, [0, 8, 1, 9, 2, 10, 3, 11]); + let b0: poly8x8_t = simd_shuffle8!(a, b, [4, 12, 5, 13, 6, 14, 7, 15]); + transmute((a0, b0)) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(zip))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vzip_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4x2_t { + let a0: poly16x4_t = simd_shuffle4!(a, b, [0, 4, 1, 5]); + let b0: poly16x4_t = simd_shuffle4!(a, b, [2, 6, 3, 7]); + transmute((a0, b0)) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(zip))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vzip_s32(a: int32x2_t, b: int32x2_t) -> int32x2x2_t { + let a0: int32x2_t = simd_shuffle2!(a, b, [0, 2]); + let b0: int32x2_t = simd_shuffle2!(a, b, [1, 3]); + transmute((a0, b0)) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(zip))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vzip_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2x2_t { + let a0: uint32x2_t = simd_shuffle2!(a, b, [0, 2]); + let b0: uint32x2_t = simd_shuffle2!(a, b, [1, 3]); + transmute((a0, b0)) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzipq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vzipq_s8(a: int8x16_t, b: int8x16_t) -> int8x16x2_t { + let a0: int8x16_t = simd_shuffle16!(a, b, [0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]); + let b0: int8x16_t = simd_shuffle16!(a, b, [8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]); + transmute((a0, b0)) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzipq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vzipq_s16(a: int16x8_t, b: int16x8_t) -> int16x8x2_t { + let a0: int16x8_t = simd_shuffle8!(a, b, [0, 8, 1, 9, 2, 10, 3, 11]); + let b0: int16x8_t = simd_shuffle8!(a, b, [4, 12, 5, 13, 6, 14, 7, 15]); + transmute((a0, b0)) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzipq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vzipq_s32(a: int32x4_t, b: int32x4_t) -> int32x4x2_t { + let a0: int32x4_t = simd_shuffle4!(a, b, [0, 4, 1, 5]); + let b0: int32x4_t = simd_shuffle4!(a, b, [2, 6, 3, 7]); + transmute((a0, b0)) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzipq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vzipq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16x2_t { + let a0: uint8x16_t = simd_shuffle16!(a, b, [0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]); + let b0: uint8x16_t = simd_shuffle16!(a, b, [8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]); + transmute((a0, b0)) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzipq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vzipq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8x2_t { + let a0: uint16x8_t = simd_shuffle8!(a, b, [0, 8, 1, 9, 2, 10, 3, 11]); + let b0: uint16x8_t = simd_shuffle8!(a, b, [4, 12, 5, 13, 6, 14, 7, 15]); + transmute((a0, b0)) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzipq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vzipq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4x2_t { + let a0: uint32x4_t = simd_shuffle4!(a, b, [0, 4, 1, 5]); + let b0: uint32x4_t = simd_shuffle4!(a, b, [2, 6, 3, 7]); + transmute((a0, b0)) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzipq_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vzipq_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16x2_t { + let a0: poly8x16_t = simd_shuffle16!(a, b, [0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]); + let b0: poly8x16_t = simd_shuffle16!(a, b, [8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]); + transmute((a0, b0)) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzipq_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vzipq_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8x2_t { + let a0: poly16x8_t = simd_shuffle8!(a, b, [0, 8, 1, 9, 2, 10, 3, 11]); + let b0: poly16x8_t = simd_shuffle8!(a, b, [4, 12, 5, 13, 6, 14, 7, 15]); + transmute((a0, b0)) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(zip))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vzip_f32(a: float32x2_t, b: float32x2_t) -> float32x2x2_t { + let a0: float32x2_t = simd_shuffle2!(a, b, [0, 2]); + let b0: float32x2_t = simd_shuffle2!(a, b, [1, 3]); + transmute((a0, b0)) +} + +/// Zip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzipq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vzipq_f32(a: float32x4_t, b: float32x4_t) -> float32x4x2_t { + let a0: float32x4_t = simd_shuffle4!(a, b, [0, 4, 1, 5]); + let b0: float32x4_t = simd_shuffle4!(a, b, [2, 6, 3, 7]); + transmute((a0, b0)) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uzp))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vuzp_s8(a: int8x8_t, b: int8x8_t) -> int8x8x2_t { + let a0: int8x8_t = simd_shuffle8!(a, b, [0, 2, 4, 6, 8, 10, 12, 14]); + let b0: int8x8_t = simd_shuffle8!(a, b, [1, 3, 5, 7, 9, 11, 13, 15]); + transmute((a0, b0)) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uzp))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vuzp_s16(a: int16x4_t, b: int16x4_t) -> int16x4x2_t { + let a0: int16x4_t = simd_shuffle4!(a, b, [0, 2, 4, 6]); + let b0: int16x4_t = simd_shuffle4!(a, b, [1, 3, 5, 7]); + transmute((a0, b0)) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzpq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uzp))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vuzpq_s8(a: int8x16_t, b: int8x16_t) -> int8x16x2_t { + let a0: int8x16_t = simd_shuffle16!(a, b, [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]); + let b0: int8x16_t = simd_shuffle16!(a, b, [1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]); + transmute((a0, b0)) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzpq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uzp))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vuzpq_s16(a: int16x8_t, b: int16x8_t) -> int16x8x2_t { + let a0: int16x8_t = simd_shuffle8!(a, b, [0, 2, 4, 6, 8, 10, 12, 14]); + let b0: int16x8_t = simd_shuffle8!(a, b, [1, 3, 5, 7, 9, 11, 13, 15]); + transmute((a0, b0)) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzpq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uzp))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vuzpq_s32(a: int32x4_t, b: int32x4_t) -> int32x4x2_t { + let a0: int32x4_t = simd_shuffle4!(a, b, [0, 2, 4, 6]); + let b0: int32x4_t = simd_shuffle4!(a, b, [1, 3, 5, 7]); + transmute((a0, b0)) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uzp))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vuzp_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8x2_t { + let a0: uint8x8_t = simd_shuffle8!(a, b, [0, 2, 4, 6, 8, 10, 12, 14]); + let b0: uint8x8_t = simd_shuffle8!(a, b, [1, 3, 5, 7, 9, 11, 13, 15]); + transmute((a0, b0)) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uzp))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vuzp_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4x2_t { + let a0: uint16x4_t = simd_shuffle4!(a, b, [0, 2, 4, 6]); + let b0: uint16x4_t = simd_shuffle4!(a, b, [1, 3, 5, 7]); + transmute((a0, b0)) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzpq_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uzp))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vuzpq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16x2_t { + let a0: uint8x16_t = simd_shuffle16!(a, b, [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]); + let b0: uint8x16_t = simd_shuffle16!(a, b, [1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]); + transmute((a0, b0)) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzpq_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uzp))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vuzpq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8x2_t { + let a0: uint16x8_t = simd_shuffle8!(a, b, [0, 2, 4, 6, 8, 10, 12, 14]); + let b0: uint16x8_t = simd_shuffle8!(a, b, [1, 3, 5, 7, 9, 11, 13, 15]); + transmute((a0, b0)) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzpq_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uzp))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vuzpq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4x2_t { + let a0: uint32x4_t = simd_shuffle4!(a, b, [0, 2, 4, 6]); + let b0: uint32x4_t = simd_shuffle4!(a, b, [1, 3, 5, 7]); + transmute((a0, b0)) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uzp))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vuzp_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8x2_t { + let a0: poly8x8_t = simd_shuffle8!(a, b, [0, 2, 4, 6, 8, 10, 12, 14]); + let b0: poly8x8_t = simd_shuffle8!(a, b, [1, 3, 5, 7, 9, 11, 13, 15]); + transmute((a0, b0)) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uzp))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vuzp_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4x2_t { + let a0: poly16x4_t = simd_shuffle4!(a, b, [0, 2, 4, 6]); + let b0: poly16x4_t = simd_shuffle4!(a, b, [1, 3, 5, 7]); + transmute((a0, b0)) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzpq_p8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uzp))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vuzpq_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16x2_t { + let a0: poly8x16_t = simd_shuffle16!(a, b, [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]); + let b0: poly8x16_t = simd_shuffle16!(a, b, [1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]); + transmute((a0, b0)) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzpq_p16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uzp))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vuzpq_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8x2_t { + let a0: poly16x8_t = simd_shuffle8!(a, b, [0, 2, 4, 6, 8, 10, 12, 14]); + let b0: poly16x8_t = simd_shuffle8!(a, b, [1, 3, 5, 7, 9, 11, 13, 15]); + transmute((a0, b0)) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(zip))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vuzp_s32(a: int32x2_t, b: int32x2_t) -> int32x2x2_t { + let a0: int32x2_t = simd_shuffle2!(a, b, [0, 2]); + let b0: int32x2_t = simd_shuffle2!(a, b, [1, 3]); + transmute((a0, b0)) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(zip))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vuzp_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2x2_t { + let a0: uint32x2_t = simd_shuffle2!(a, b, [0, 2]); + let b0: uint32x2_t = simd_shuffle2!(a, b, [1, 3]); + transmute((a0, b0)) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(zip))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vuzp_f32(a: float32x2_t, b: float32x2_t) -> float32x2x2_t { + let a0: float32x2_t = simd_shuffle2!(a, b, [0, 2]); + let b0: float32x2_t = simd_shuffle2!(a, b, [1, 3]); + transmute((a0, b0)) +} + +/// Unzip vectors +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzpq_f32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uzp))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vuzpq_f32(a: float32x4_t, b: float32x4_t) -> float32x4x2_t { + let a0: float32x4_t = simd_shuffle4!(a, b, [0, 2, 4, 6]); + let b0: float32x4_t = simd_shuffle4!(a, b, [1, 3, 5, 7]); + transmute((a0, b0)) +} + +/// Unsigned Absolute difference and Accumulate Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_u8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabal.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabal))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabal_u8(a: uint16x8_t, b: uint8x8_t, c: uint8x8_t) -> uint16x8_t { + let d: uint8x8_t = vabd_u8(b, c); + simd_add(a, simd_cast(d)) +} + +/// Unsigned Absolute difference and Accumulate Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_u16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabal.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabal))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabal_u16(a: uint32x4_t, b: uint16x4_t, c: uint16x4_t) -> uint32x4_t { + let d: uint16x4_t = vabd_u16(b, c); + simd_add(a, simd_cast(d)) +} + +/// Unsigned Absolute difference and Accumulate Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_u32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabal.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabal))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabal_u32(a: uint64x2_t, b: uint32x2_t, c: uint32x2_t) -> uint64x2_t { + let d: uint32x2_t = vabd_u32(b, c); + simd_add(a, simd_cast(d)) +} + +/// Signed Absolute difference and Accumulate Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabal.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabal))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabal_s8(a: int16x8_t, b: int8x8_t, c: int8x8_t) -> int16x8_t { + let d: int8x8_t = vabd_s8(b, c); + let e: uint8x8_t = simd_cast(d); + simd_add(a, simd_cast(e)) +} + +/// Signed Absolute difference and Accumulate Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabal.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabal))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabal_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t { + let d: int16x4_t = vabd_s16(b, c); + let e: uint16x4_t = simd_cast(d); + simd_add(a, simd_cast(e)) +} + +/// Signed Absolute difference and Accumulate Long +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabal.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabal))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vabal_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t { + let d: int32x2_t = vabd_s32(b, c); + let e: uint32x2_t = simd_cast(d); + simd_add(a, simd_cast(e)) +} + +/// Singned saturating Absolute value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabs_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqabs.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqabs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqabs_s8(a: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqabs.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqabs.v8i8")] + fn vqabs_s8_(a: int8x8_t) -> int8x8_t; + } +vqabs_s8_(a) +} + +/// Singned saturating Absolute value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabsq_s8) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqabs.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqabs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqabsq_s8(a: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqabs.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqabs.v16i8")] + fn vqabsq_s8_(a: int8x16_t) -> int8x16_t; + } +vqabsq_s8_(a) +} + +/// Singned saturating Absolute value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabs_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqabs.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqabs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqabs_s16(a: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqabs.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqabs.v4i16")] + fn vqabs_s16_(a: int16x4_t) -> int16x4_t; + } +vqabs_s16_(a) +} + +/// Singned saturating Absolute value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabsq_s16) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqabs.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqabs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqabsq_s16(a: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqabs.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqabs.v8i16")] + fn vqabsq_s16_(a: int16x8_t) -> int16x8_t; + } +vqabsq_s16_(a) +} + +/// Singned saturating Absolute value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabs_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqabs.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqabs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqabs_s32(a: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqabs.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqabs.v2i32")] + fn vqabs_s32_(a: int32x2_t) -> int32x2_t; + } +vqabs_s32_(a) +} + +/// Singned saturating Absolute value +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabsq_s32) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqabs.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqabs))] +#[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] +pub unsafe fn vqabsq_s32(a: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqabs.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqabs.v4i32")] + fn vqabsq_s32_(a: int32x4_t) -> int32x4_t; + } +vqabsq_s32_(a) +} + +#[cfg(test)] +#[allow(overflowing_literals)] +mod test { + use super::*; + use crate::core_arch::simd::*; + use std::mem::transmute; + use stdarch_test::simd_test; + + #[simd_test(enable = "neon")] + unsafe fn test_vand_s8() { + let a: i8x8 = i8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i8x8 = i8x8::new(0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F); + let e: i8x8 = i8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: i8x8 = transmute(vand_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i8x8 = i8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i8x8 = i8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i8x8 = i8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let r: i8x8 = transmute(vand_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vandq_s8() { + let a: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x00); + let b: i8x16 = i8x16::new(0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F); + let e: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x00); + let r: i8x16 = transmute(vandq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x00); + let b: i8x16 = i8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i8x16 = i8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let r: i8x16 = transmute(vandq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vand_s16() { + let a: i16x4 = i16x4::new(0x00, 0x01, 0x02, 0x03); + let b: i16x4 = i16x4::new(0x0F, 0x0F, 0x0F, 0x0F); + let e: i16x4 = i16x4::new(0x00, 0x01, 0x02, 0x03); + let r: i16x4 = transmute(vand_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i16x4 = i16x4::new(0x00, 0x01, 0x02, 0x03); + let b: i16x4 = i16x4::new(0x00, 0x00, 0x00, 0x00); + let e: i16x4 = i16x4::new(0x00, 0x00, 0x00, 0x00); + let r: i16x4 = transmute(vand_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vandq_s16() { + let a: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i16x8 = i16x8::new(0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F); + let e: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: i16x8 = transmute(vandq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i16x8 = i16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i16x8 = i16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let r: i16x8 = transmute(vandq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vand_s32() { + let a: i32x2 = i32x2::new(0x00, 0x01); + let b: i32x2 = i32x2::new(0x0F, 0x0F); + let e: i32x2 = i32x2::new(0x00, 0x01); + let r: i32x2 = transmute(vand_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i32x2 = i32x2::new(0x00, 0x01); + let b: i32x2 = i32x2::new(0x00, 0x00); + let e: i32x2 = i32x2::new(0x00, 0x00); + let r: i32x2 = transmute(vand_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vandq_s32() { + let a: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); + let b: i32x4 = i32x4::new(0x0F, 0x0F, 0x0F, 0x0F); + let e: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); + let r: i32x4 = transmute(vandq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); + let b: i32x4 = i32x4::new(0x00, 0x00, 0x00, 0x00); + let e: i32x4 = i32x4::new(0x00, 0x00, 0x00, 0x00); + let r: i32x4 = transmute(vandq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vand_u8() { + let a: u8x8 = u8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u8x8 = u8x8::new(0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F); + let e: u8x8 = u8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: u8x8 = transmute(vand_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u8x8 = u8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u8x8 = u8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: u8x8 = u8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let r: u8x8 = transmute(vand_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vandq_u8() { + let a: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x00); + let b: u8x16 = u8x16::new(0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F); + let e: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x00); + let r: u8x16 = transmute(vandq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x00); + let b: u8x16 = u8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: u8x16 = u8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let r: u8x16 = transmute(vandq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vand_u16() { + let a: u16x4 = u16x4::new(0x00, 0x01, 0x02, 0x03); + let b: u16x4 = u16x4::new(0x0F, 0x0F, 0x0F, 0x0F); + let e: u16x4 = u16x4::new(0x00, 0x01, 0x02, 0x03); + let r: u16x4 = transmute(vand_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u16x4 = u16x4::new(0x00, 0x01, 0x02, 0x03); + let b: u16x4 = u16x4::new(0x00, 0x00, 0x00, 0x00); + let e: u16x4 = u16x4::new(0x00, 0x00, 0x00, 0x00); + let r: u16x4 = transmute(vand_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vandq_u16() { + let a: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u16x8 = u16x8::new(0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F); + let e: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: u16x8 = transmute(vandq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u16x8 = u16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: u16x8 = u16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let r: u16x8 = transmute(vandq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vand_u32() { + let a: u32x2 = u32x2::new(0x00, 0x01); + let b: u32x2 = u32x2::new(0x0F, 0x0F); + let e: u32x2 = u32x2::new(0x00, 0x01); + let r: u32x2 = transmute(vand_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u32x2 = u32x2::new(0x00, 0x01); + let b: u32x2 = u32x2::new(0x00, 0x00); + let e: u32x2 = u32x2::new(0x00, 0x00); + let r: u32x2 = transmute(vand_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vandq_u32() { + let a: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); + let b: u32x4 = u32x4::new(0x0F, 0x0F, 0x0F, 0x0F); + let e: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); + let r: u32x4 = transmute(vandq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); + let b: u32x4 = u32x4::new(0x00, 0x00, 0x00, 0x00); + let e: u32x4 = u32x4::new(0x00, 0x00, 0x00, 0x00); + let r: u32x4 = transmute(vandq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vand_s64() { + let a: i64x1 = i64x1::new(0x00); + let b: i64x1 = i64x1::new(0x0F); + let e: i64x1 = i64x1::new(0x00); + let r: i64x1 = transmute(vand_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i64x1 = i64x1::new(0x00); + let b: i64x1 = i64x1::new(0x00); + let e: i64x1 = i64x1::new(0x00); + let r: i64x1 = transmute(vand_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vandq_s64() { + let a: i64x2 = i64x2::new(0x00, 0x01); + let b: i64x2 = i64x2::new(0x0F, 0x0F); + let e: i64x2 = i64x2::new(0x00, 0x01); + let r: i64x2 = transmute(vandq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i64x2 = i64x2::new(0x00, 0x01); + let b: i64x2 = i64x2::new(0x00, 0x00); + let e: i64x2 = i64x2::new(0x00, 0x00); + let r: i64x2 = transmute(vandq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vand_u64() { + let a: u64x1 = u64x1::new(0x00); + let b: u64x1 = u64x1::new(0x0F); + let e: u64x1 = u64x1::new(0x00); + let r: u64x1 = transmute(vand_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u64x1 = u64x1::new(0x00); + let b: u64x1 = u64x1::new(0x00); + let e: u64x1 = u64x1::new(0x00); + let r: u64x1 = transmute(vand_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vandq_u64() { + let a: u64x2 = u64x2::new(0x00, 0x01); + let b: u64x2 = u64x2::new(0x0F, 0x0F); + let e: u64x2 = u64x2::new(0x00, 0x01); + let r: u64x2 = transmute(vandq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u64x2 = u64x2::new(0x00, 0x01); + let b: u64x2 = u64x2::new(0x00, 0x00); + let e: u64x2 = u64x2::new(0x00, 0x00); + let r: u64x2 = transmute(vandq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorr_s8() { + let a: i8x8 = i8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i8x8 = i8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i8x8 = i8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: i8x8 = transmute(vorr_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_s8() { + let a: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); + let b: i8x16 = i8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); + let r: i8x16 = transmute(vorrq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorr_s16() { + let a: i16x4 = i16x4::new(0x00, 0x01, 0x02, 0x03); + let b: i16x4 = i16x4::new(0x00, 0x00, 0x00, 0x00); + let e: i16x4 = i16x4::new(0x00, 0x01, 0x02, 0x03); + let r: i16x4 = transmute(vorr_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_s16() { + let a: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i16x8 = i16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: i16x8 = transmute(vorrq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorr_s32() { + let a: i32x2 = i32x2::new(0x00, 0x01); + let b: i32x2 = i32x2::new(0x00, 0x00); + let e: i32x2 = i32x2::new(0x00, 0x01); + let r: i32x2 = transmute(vorr_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_s32() { + let a: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); + let b: i32x4 = i32x4::new(0x00, 0x00, 0x00, 0x00); + let e: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); + let r: i32x4 = transmute(vorrq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorr_u8() { + let a: u8x8 = u8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u8x8 = u8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: u8x8 = u8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: u8x8 = transmute(vorr_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_u8() { + let a: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); + let b: u8x16 = u8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); + let r: u8x16 = transmute(vorrq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorr_u16() { + let a: u16x4 = u16x4::new(0x00, 0x01, 0x02, 0x03); + let b: u16x4 = u16x4::new(0x00, 0x00, 0x00, 0x00); + let e: u16x4 = u16x4::new(0x00, 0x01, 0x02, 0x03); + let r: u16x4 = transmute(vorr_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_u16() { + let a: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u16x8 = u16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: u16x8 = transmute(vorrq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorr_u32() { + let a: u32x2 = u32x2::new(0x00, 0x01); + let b: u32x2 = u32x2::new(0x00, 0x00); + let e: u32x2 = u32x2::new(0x00, 0x01); + let r: u32x2 = transmute(vorr_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_u32() { + let a: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); + let b: u32x4 = u32x4::new(0x00, 0x00, 0x00, 0x00); + let e: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); + let r: u32x4 = transmute(vorrq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorr_s64() { + let a: i64x1 = i64x1::new(0x00); + let b: i64x1 = i64x1::new(0x00); + let e: i64x1 = i64x1::new(0x00); + let r: i64x1 = transmute(vorr_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_s64() { + let a: i64x2 = i64x2::new(0x00, 0x01); + let b: i64x2 = i64x2::new(0x00, 0x00); + let e: i64x2 = i64x2::new(0x00, 0x01); + let r: i64x2 = transmute(vorrq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorr_u64() { + let a: u64x1 = u64x1::new(0x00); + let b: u64x1 = u64x1::new(0x00); + let e: u64x1 = u64x1::new(0x00); + let r: u64x1 = transmute(vorr_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_u64() { + let a: u64x2 = u64x2::new(0x00, 0x01); + let b: u64x2 = u64x2::new(0x00, 0x00); + let e: u64x2 = u64x2::new(0x00, 0x01); + let r: u64x2 = transmute(vorrq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veor_s8() { + let a: i8x8 = i8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i8x8 = i8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i8x8 = i8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: i8x8 = transmute(veor_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veorq_s8() { + let a: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); + let b: i8x16 = i8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); + let r: i8x16 = transmute(veorq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veor_s16() { + let a: i16x4 = i16x4::new(0x00, 0x01, 0x02, 0x03); + let b: i16x4 = i16x4::new(0x00, 0x00, 0x00, 0x00); + let e: i16x4 = i16x4::new(0x00, 0x01, 0x02, 0x03); + let r: i16x4 = transmute(veor_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veorq_s16() { + let a: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i16x8 = i16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: i16x8 = transmute(veorq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veor_s32() { + let a: i32x2 = i32x2::new(0x00, 0x01); + let b: i32x2 = i32x2::new(0x00, 0x00); + let e: i32x2 = i32x2::new(0x00, 0x01); + let r: i32x2 = transmute(veor_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veorq_s32() { + let a: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); + let b: i32x4 = i32x4::new(0x00, 0x00, 0x00, 0x00); + let e: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); + let r: i32x4 = transmute(veorq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veor_u8() { + let a: u8x8 = u8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u8x8 = u8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: u8x8 = u8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: u8x8 = transmute(veor_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veorq_u8() { + let a: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); + let b: u8x16 = u8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); + let r: u8x16 = transmute(veorq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veor_u16() { + let a: u16x4 = u16x4::new(0x00, 0x01, 0x02, 0x03); + let b: u16x4 = u16x4::new(0x00, 0x00, 0x00, 0x00); + let e: u16x4 = u16x4::new(0x00, 0x01, 0x02, 0x03); + let r: u16x4 = transmute(veor_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veorq_u16() { + let a: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u16x8 = u16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: u16x8 = transmute(veorq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veor_u32() { + let a: u32x2 = u32x2::new(0x00, 0x01); + let b: u32x2 = u32x2::new(0x00, 0x00); + let e: u32x2 = u32x2::new(0x00, 0x01); + let r: u32x2 = transmute(veor_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veorq_u32() { + let a: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); + let b: u32x4 = u32x4::new(0x00, 0x00, 0x00, 0x00); + let e: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); + let r: u32x4 = transmute(veorq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veor_s64() { + let a: i64x1 = i64x1::new(0x00); + let b: i64x1 = i64x1::new(0x00); + let e: i64x1 = i64x1::new(0x00); + let r: i64x1 = transmute(veor_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veorq_s64() { + let a: i64x2 = i64x2::new(0x00, 0x01); + let b: i64x2 = i64x2::new(0x00, 0x00); + let e: i64x2 = i64x2::new(0x00, 0x01); + let r: i64x2 = transmute(veorq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veor_u64() { + let a: u64x1 = u64x1::new(0x00); + let b: u64x1 = u64x1::new(0x00); + let e: u64x1 = u64x1::new(0x00); + let r: u64x1 = transmute(veor_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veorq_u64() { + let a: u64x2 = u64x2::new(0x00, 0x01); + let b: u64x2 = u64x2::new(0x00, 0x00); + let e: u64x2 = u64x2::new(0x00, 0x01); + let r: u64x2 = transmute(veorq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabd_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: i8x8 = i8x8::new(15, 13, 11, 9, 7, 5, 3, 1); + let r: i8x8 = transmute(vabd_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdq_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + let e: i8x16 = i8x16::new(15, 13, 11, 9, 7, 5, 3, 1, 1, 3, 5, 7, 9, 11, 13, 15); + let r: i8x16 = transmute(vabdq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabd_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(16, 15, 14, 13); + let e: i16x4 = i16x4::new(15, 13, 11, 9); + let r: i16x4 = transmute(vabd_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: i16x8 = i16x8::new(15, 13, 11, 9, 7, 5, 3, 1); + let r: i16x8 = transmute(vabdq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabd_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(16, 15); + let e: i32x2 = i32x2::new(15, 13); + let r: i32x2 = transmute(vabd_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(16, 15, 14, 13); + let e: i32x4 = i32x4::new(15, 13, 11, 9); + let r: i32x4 = transmute(vabdq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabd_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x8 = u8x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: u8x8 = u8x8::new(15, 13, 11, 9, 7, 5, 3, 1); + let r: u8x8 = transmute(vabd_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdq_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + let e: u8x16 = u8x16::new(15, 13, 11, 9, 7, 5, 3, 1, 1, 3, 5, 7, 9, 11, 13, 15); + let r: u8x16 = transmute(vabdq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabd_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(16, 15, 14, 13); + let e: u16x4 = u16x4::new(15, 13, 11, 9); + let r: u16x4 = transmute(vabd_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x8 = u16x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: u16x8 = u16x8::new(15, 13, 11, 9, 7, 5, 3, 1); + let r: u16x8 = transmute(vabdq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabd_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(16, 15); + let e: u32x2 = u32x2::new(15, 13); + let r: u32x2 = transmute(vabd_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(16, 15, 14, 13); + let e: u32x4 = u32x4::new(15, 13, 11, 9); + let r: u32x4 = transmute(vabdq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabd_f32() { + let a: f32x2 = f32x2::new(1.0, 2.0); + let b: f32x2 = f32x2::new(9.0, 3.0); + let e: f32x2 = f32x2::new(8.0, 1.0); + let r: f32x2 = transmute(vabd_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdq_f32() { + let a: f32x4 = f32x4::new(1.0, 2.0, 5.0, -4.0); + let b: f32x4 = f32x4::new(9.0, 3.0, 2.0, 8.0); + let e: f32x4 = f32x4::new(8.0, 1.0, 3.0, 12.0); + let r: f32x4 = transmute(vabdq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 4, 3, 2, 1); + let b: u8x8 = u8x8::new(10, 10, 10, 10, 10, 10, 10, 10); + let e: u16x8 = u16x8::new(9, 8, 7, 6, 6, 7, 8, 9); + let r: u16x8 = transmute(vabdl_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(10, 10, 10, 10); + let e: u32x4 = u32x4::new(9, 8, 7, 6); + let r: u32x4 = transmute(vabdl_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(10, 10); + let e: u64x2 = u64x2::new(9, 8); + let r: u64x2 = transmute(vabdl_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 4, 3, 2, 1); + let b: i8x8 = i8x8::new(10, 10, 10, 10, 10, 10, 10, 10); + let e: i16x8 = i16x8::new(9, 8, 7, 6, 6, 7, 8, 9); + let r: i16x8 = transmute(vabdl_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_s16() { + let a: i16x4 = i16x4::new(1, 2, 11, 12); + let b: i16x4 = i16x4::new(10, 10, 10, 10); + let e: i32x4 = i32x4::new(9, 8, 1, 2); + let r: i32x4 = transmute(vabdl_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_s32() { + let a: i32x2 = i32x2::new(1, 11); + let b: i32x2 = i32x2::new(10, 10); + let e: i64x2 = i64x2::new(9, 1); + let r: i64x2 = transmute(vabdl_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_u8() { + let a: u8x8 = u8x8::new(0, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u8x8 = u8x8::new(0, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vceq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u8x8 = u8x8::new(0, 0, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u8x8 = u8x8::new(0, 0xFF, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08); + let e: u8x8 = u8x8::new(0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0); + let r: u8x8 = transmute(vceq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_u8() { + let a: u8x16 = u8x16::new(0, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0xFF); + let b: u8x16 = u8x16::new(0, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0xFF); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vceqq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u8x16 = u8x16::new(0, 0, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0xCC, 0x0D, 0xEE, 0xFF); + let b: u8x16 = u8x16::new(0, 0xFF, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08, 0x08, 0x00, 0x0A, 0x0A, 0xCC, 0xD0, 0xEE, 0); + let e: u8x16 = u8x16::new(0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0); + let r: u8x16 = transmute(vceqq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_u16() { + let a: u16x4 = u16x4::new(0, 0x01, 0x02, 0x03); + let b: u16x4 = u16x4::new(0, 0x01, 0x02, 0x03); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vceq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u16x4 = u16x4::new(0, 0, 0x02, 0x03); + let b: u16x4 = u16x4::new(0, 0xFF_FF, 0x02, 0x04); + let e: u16x4 = u16x4::new(0xFF_FF, 0, 0xFF_FF, 0); + let r: u16x4 = transmute(vceq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_u16() { + let a: u16x8 = u16x8::new(0, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u16x8 = u16x8::new(0, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vceqq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u16x8 = u16x8::new(0, 0, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u16x8 = u16x8::new(0, 0xFF_FF, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08); + let e: u16x8 = u16x8::new(0xFF_FF, 0, 0xFF_FF, 0, 0xFF_FF, 0, 0xFF_FF, 0); + let r: u16x8 = transmute(vceqq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_u32() { + let a: u32x2 = u32x2::new(0, 0x01); + let b: u32x2 = u32x2::new(0, 0x01); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vceq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u32x2 = u32x2::new(0, 0); + let b: u32x2 = u32x2::new(0, 0xFF_FF_FF_FF); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); + let r: u32x2 = transmute(vceq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_u32() { + let a: u32x4 = u32x4::new(0, 0x01, 0x02, 0x03); + let b: u32x4 = u32x4::new(0, 0x01, 0x02, 0x03); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vceqq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u32x4 = u32x4::new(0, 0, 0x02, 0x03); + let b: u32x4 = u32x4::new(0, 0xFF_FF_FF_FF, 0x02, 0x04); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0, 0xFF_FF_FF_FF, 0); + let r: u32x4 = transmute(vceqq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_s8() { + let a: i8x8 = i8x8::new(-128, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i8x8 = i8x8::new(-128, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vceq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i8x8 = i8x8::new(-128, -128, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i8x8 = i8x8::new(-128, 0x7F, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08); + let e: u8x8 = u8x8::new(0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0); + let r: u8x8 = transmute(vceq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_s8() { + let a: i8x16 = i8x16::new(-128, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x7F); + let b: i8x16 = i8x16::new(-128, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x7F); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vceqq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i8x16 = i8x16::new(-128, -128, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0xCC, 0x0D, 0xEE, 0x7F); + let b: i8x16 = i8x16::new(-128, 0x7F, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08, 0x08, 0x00, 0x0A, 0x0A, 0xCC, 0xD0, 0xEE, -128); + let e: u8x16 = u8x16::new(0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0); + let r: u8x16 = transmute(vceqq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_s16() { + let a: i16x4 = i16x4::new(-32768, 0x01, 0x02, 0x03); + let b: i16x4 = i16x4::new(-32768, 0x01, 0x02, 0x03); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vceq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i16x4 = i16x4::new(-32768, -32768, 0x02, 0x03); + let b: i16x4 = i16x4::new(-32768, 0x7F_FF, 0x02, 0x04); + let e: u16x4 = u16x4::new(0xFF_FF, 0, 0xFF_FF, 0); + let r: u16x4 = transmute(vceq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_s16() { + let a: i16x8 = i16x8::new(-32768, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i16x8 = i16x8::new(-32768, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vceqq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i16x8 = i16x8::new(-32768, -32768, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i16x8 = i16x8::new(-32768, 0x7F_FF, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08); + let e: u16x8 = u16x8::new(0xFF_FF, 0, 0xFF_FF, 0, 0xFF_FF, 0, 0xFF_FF, 0); + let r: u16x8 = transmute(vceqq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_s32() { + let a: i32x2 = i32x2::new(-2147483648, 0x01); + let b: i32x2 = i32x2::new(-2147483648, 0x01); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vceq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i32x2 = i32x2::new(-2147483648, -2147483648); + let b: i32x2 = i32x2::new(-2147483648, 0x7F_FF_FF_FF); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); + let r: u32x2 = transmute(vceq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_s32() { + let a: i32x4 = i32x4::new(-2147483648, 0x01, 0x02, 0x03); + let b: i32x4 = i32x4::new(-2147483648, 0x01, 0x02, 0x03); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vceqq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i32x4 = i32x4::new(-2147483648, -2147483648, 0x02, 0x03); + let b: i32x4 = i32x4::new(-2147483648, 0x7F_FF_FF_FF, 0x02, 0x04); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0, 0xFF_FF_FF_FF, 0); + let r: u32x4 = transmute(vceqq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_p8() { + let a: i8x8 = i8x8::new(-128, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i8x8 = i8x8::new(-128, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vceq_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i8x8 = i8x8::new(-128, -128, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i8x8 = i8x8::new(-128, 0x7F, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08); + let e: u8x8 = u8x8::new(0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0); + let r: u8x8 = transmute(vceq_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_p8() { + let a: i8x16 = i8x16::new(-128, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x7F); + let b: i8x16 = i8x16::new(-128, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x7F); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vceqq_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i8x16 = i8x16::new(-128, -128, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0xCC, 0x0D, 0xEE, 0x7F); + let b: i8x16 = i8x16::new(-128, 0x7F, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08, 0x08, 0x00, 0x0A, 0x0A, 0xCC, 0xD0, 0xEE, -128); + let e: u8x16 = u8x16::new(0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0); + let r: u8x16 = transmute(vceqq_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_f32() { + let a: f32x2 = f32x2::new(1.2, 3.4); + let b: f32x2 = f32x2::new(1.2, 3.4); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vceq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_f32() { + let a: f32x4 = f32x4::new(1.2, 3.4, 5.6, 7.8); + let b: f32x4 = f32x4::new(1.2, 3.4, 5.6, 7.8); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vceqq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtst_s8() { + let a: i8x8 = i8x8::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let b: i8x8 = i8x8::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let e: u8x8 = u8x8::new(0xFF, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vtst_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstq_s8() { + let a: i8x16 = i8x16::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x7F); + let b: i8x16 = i8x16::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x7F); + let e: u8x16 = u8x16::new(0xFF, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vtstq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtst_s16() { + let a: i16x4 = i16x4::new(-32768, 0x00, 0x01, 0x02); + let b: i16x4 = i16x4::new(-32768, 0x00, 0x01, 0x02); + let e: u16x4 = u16x4::new(0xFF_FF, 0, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vtst_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstq_s16() { + let a: i16x8 = i16x8::new(-32768, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let b: i16x8 = i16x8::new(-32768, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let e: u16x8 = u16x8::new(0xFF_FF, 0, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vtstq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtst_s32() { + let a: i32x2 = i32x2::new(-2147483648, 0x00); + let b: i32x2 = i32x2::new(-2147483648, 0x00); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); + let r: u32x2 = transmute(vtst_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstq_s32() { + let a: i32x4 = i32x4::new(-2147483648, 0x00, 0x01, 0x02); + let b: i32x4 = i32x4::new(-2147483648, 0x00, 0x01, 0x02); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vtstq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtst_p8() { + let a: i8x8 = i8x8::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let b: i8x8 = i8x8::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let e: u8x8 = u8x8::new(0xFF, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vtst_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstq_p8() { + let a: i8x16 = i8x16::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x7F); + let b: i8x16 = i8x16::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x7F); + let e: u8x16 = u8x16::new(0xFF, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vtstq_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtst_p16() { + let a: i16x4 = i16x4::new(-32768, 0x00, 0x01, 0x02); + let b: i16x4 = i16x4::new(-32768, 0x00, 0x01, 0x02); + let e: u16x4 = u16x4::new(0xFF_FF, 0, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vtst_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstq_p16() { + let a: i16x8 = i16x8::new(-32768, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let b: i16x8 = i16x8::new(-32768, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let e: u16x8 = u16x8::new(0xFF_FF, 0, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vtstq_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtst_u8() { + let a: u8x8 = u8x8::new(0, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let b: u8x8 = u8x8::new(0, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let e: u8x8 = u8x8::new(0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vtst_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstq_u8() { + let a: u8x16 = u8x16::new(0, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0xFF); + let b: u8x16 = u8x16::new(0, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0xFF); + let e: u8x16 = u8x16::new(0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vtstq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtst_u16() { + let a: u16x4 = u16x4::new(0, 0x00, 0x01, 0x02); + let b: u16x4 = u16x4::new(0, 0x00, 0x01, 0x02); + let e: u16x4 = u16x4::new(0, 0, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vtst_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstq_u16() { + let a: u16x8 = u16x8::new(0, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let b: u16x8 = u16x8::new(0, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let e: u16x8 = u16x8::new(0, 0, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vtstq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtst_u32() { + let a: u32x2 = u32x2::new(0, 0x00); + let b: u32x2 = u32x2::new(0, 0x00); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vtst_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstq_u32() { + let a: u32x4 = u32x4::new(0, 0x00, 0x01, 0x02); + let b: u32x4 = u32x4::new(0, 0x00, 0x01, 0x02); + let e: u32x4 = u32x4::new(0, 0, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vtstq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabs_f32() { + let a: f32x2 = f32x2::new(-0.1, -2.2); + let e: f32x2 = f32x2::new(0.1, 2.2); + let r: f32x2 = transmute(vabs_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabsq_f32() { + let a: f32x4 = f32x4::new(-0.1, -2.2, -3.3, -6.6); + let e: f32x4 = f32x4::new(0.1, 2.2, 3.3, 6.6); + let r: f32x4 = transmute(vabsq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vcgt_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vcgtq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(0, 1, 2, 3); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vcgt_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vcgtq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(0, 1); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcgt_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(0, 1, 2, 3); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcgtq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vcgt_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vcgtq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(0, 1, 2, 3); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vcgt_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vcgtq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(0, 1); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcgt_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(0, 1, 2, 3); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcgtq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_f32() { + let a: f32x2 = f32x2::new(1.2, 2.3); + let b: f32x2 = f32x2::new(0.1, 1.2); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcgt_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_f32() { + let a: f32x4 = f32x4::new(1.2, 2.3, 3.4, 4.5); + let b: f32x4 = f32x4::new(0.1, 1.2, 2.3, 3.4); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcgtq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_s8() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vclt_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_s8() { + let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vcltq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vclt_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vcltq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_s32() { + let a: i32x2 = i32x2::new(0, 1); + let b: i32x2 = i32x2::new(1, 2); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vclt_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcltq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_u8() { + let a: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vclt_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_u8() { + let a: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vcltq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(1, 2, 3, 4); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vclt_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vcltq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_u32() { + let a: u32x2 = u32x2::new(0, 1); + let b: u32x2 = u32x2::new(1, 2); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vclt_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcltq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_f32() { + let a: f32x2 = f32x2::new(0.1, 1.2); + let b: f32x2 = f32x2::new(1.2, 2.3); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vclt_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_f32() { + let a: f32x4 = f32x4::new(0.1, 1.2, 2.3, 3.4); + let b: f32x4 = f32x4::new(1.2, 2.3, 3.4, 4.5); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcltq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_s8() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vcle_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_s8() { + let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vcleq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vcle_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vcleq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_s32() { + let a: i32x2 = i32x2::new(0, 1); + let b: i32x2 = i32x2::new(1, 2); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcle_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcleq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_u8() { + let a: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vcle_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_u8() { + let a: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vcleq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(1, 2, 3, 4); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vcle_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vcleq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_u32() { + let a: u32x2 = u32x2::new(0, 1); + let b: u32x2 = u32x2::new(1, 2); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcle_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcleq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_f32() { + let a: f32x2 = f32x2::new(0.1, 1.2); + let b: f32x2 = f32x2::new(1.2, 2.3); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcle_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_f32() { + let a: f32x4 = f32x4::new(0.1, 1.2, 2.3, 3.4); + let b: f32x4 = f32x4::new(1.2, 2.3, 3.4, 4.5); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcleq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vcge_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vcgeq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(0, 1, 2, 3); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vcge_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vcgeq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(0, 1); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcge_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(0, 1, 2, 3); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcgeq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vcge_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vcgeq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(0, 1, 2, 3); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vcge_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vcgeq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(0, 1); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcge_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(0, 1, 2, 3); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcgeq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_f32() { + let a: f32x2 = f32x2::new(1.2, 2.3); + let b: f32x2 = f32x2::new(0.1, 1.2); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcge_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_f32() { + let a: f32x4 = f32x4::new(1.2, 2.3, 3.4, 4.5); + let b: f32x4 = f32x4::new(0.1, 1.2, 2.3, 3.4); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcgeq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcls_s8() { + let a: i8x8 = i8x8::new(-128, -1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i8x8 = i8x8::new(0, 7, 7, 7, 7, 7, 7, 7); + let r: i8x8 = transmute(vcls_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclsq_s8() { + let a: i8x16 = i8x16::new(-128, -1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7F); + let e: i8x16 = i8x16::new(0, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 0); + let r: i8x16 = transmute(vclsq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcls_s16() { + let a: i16x4 = i16x4::new(-32768, -1, 0x00, 0x00); + let e: i16x4 = i16x4::new(0, 15, 15, 15); + let r: i16x4 = transmute(vcls_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclsq_s16() { + let a: i16x8 = i16x8::new(-32768, -1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i16x8 = i16x8::new(0, 15, 15, 15, 15, 15, 15, 15); + let r: i16x8 = transmute(vclsq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcls_s32() { + let a: i32x2 = i32x2::new(-2147483648, -1); + let e: i32x2 = i32x2::new(0, 31); + let r: i32x2 = transmute(vcls_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclsq_s32() { + let a: i32x4 = i32x4::new(-2147483648, -1, 0x00, 0x00); + let e: i32x4 = i32x4::new(0, 31, 31, 31); + let r: i32x4 = transmute(vclsq_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcls_u8() { + let a: u8x8 = u8x8::new(0, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i8x8 = i8x8::new(7, 7, 7, 7, 7, 7, 7, 7); + let r: i8x8 = transmute(vcls_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclsq_u8() { + let a: u8x16 = u8x16::new(0, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF); + let e: i8x16 = i8x16::new(7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7); + let r: i8x16 = transmute(vclsq_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcls_u16() { + let a: u16x4 = u16x4::new(0, 0xFF_FF, 0x00, 0x00); + let e: i16x4 = i16x4::new(15, 15, 15, 15); + let r: i16x4 = transmute(vcls_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclsq_u16() { + let a: u16x8 = u16x8::new(0, 0xFF_FF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i16x8 = i16x8::new(15, 15, 15, 15, 15, 15, 15, 15); + let r: i16x8 = transmute(vclsq_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcls_u32() { + let a: u32x2 = u32x2::new(0, 0xFF_FF_FF_FF); + let e: i32x2 = i32x2::new(31, 31); + let r: i32x2 = transmute(vcls_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclsq_u32() { + let a: u32x4 = u32x4::new(0, 0xFF_FF_FF_FF, 0x00, 0x00); + let e: i32x4 = i32x4::new(31, 31, 31, 31); + let r: i32x4 = transmute(vclsq_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclz_s8() { + let a: i8x8 = i8x8::new(-128, -1, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01); + let e: i8x8 = i8x8::new(0, 0, 8, 7, 7, 7, 7, 7); + let r: i8x8 = transmute(vclz_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclzq_s8() { + let a: i8x16 = i8x16::new(-128, -1, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x7F); + let e: i8x16 = i8x16::new(0, 0, 8, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 1); + let r: i8x16 = transmute(vclzq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclz_s16() { + let a: i16x4 = i16x4::new(-32768, -1, 0x00, 0x01); + let e: i16x4 = i16x4::new(0, 0, 16, 15); + let r: i16x4 = transmute(vclz_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclzq_s16() { + let a: i16x8 = i16x8::new(-32768, -1, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01); + let e: i16x8 = i16x8::new(0, 0, 16, 15, 15, 15, 15, 15); + let r: i16x8 = transmute(vclzq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclz_s32() { + let a: i32x2 = i32x2::new(-2147483648, -1); + let e: i32x2 = i32x2::new(0, 0); + let r: i32x2 = transmute(vclz_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclzq_s32() { + let a: i32x4 = i32x4::new(-2147483648, -1, 0x00, 0x01); + let e: i32x4 = i32x4::new(0, 0, 32, 31); + let r: i32x4 = transmute(vclzq_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclz_u8() { + let a: u8x8 = u8x8::new(0, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01); + let e: u8x8 = u8x8::new(8, 8, 7, 7, 7, 7, 7, 7); + let r: u8x8 = transmute(vclz_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclzq_u8() { + let a: u8x16 = u8x16::new(0, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0xFF); + let e: u8x16 = u8x16::new(8, 8, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 0); + let r: u8x16 = transmute(vclzq_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclz_u16() { + let a: u16x4 = u16x4::new(0, 0x00, 0x01, 0x01); + let e: u16x4 = u16x4::new(16, 16, 15, 15); + let r: u16x4 = transmute(vclz_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclzq_u16() { + let a: u16x8 = u16x8::new(0, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01); + let e: u16x8 = u16x8::new(16, 16, 15, 15, 15, 15, 15, 15); + let r: u16x8 = transmute(vclzq_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclz_u32() { + let a: u32x2 = u32x2::new(0, 0x00); + let e: u32x2 = u32x2::new(32, 32); + let r: u32x2 = transmute(vclz_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclzq_u32() { + let a: u32x4 = u32x4::new(0, 0x00, 0x01, 0x01); + let e: u32x4 = u32x4::new(32, 32, 31, 31); + let r: u32x4 = transmute(vclzq_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcagt_f32() { + let a: f32x2 = f32x2::new(-1.2, 0.0); + let b: f32x2 = f32x2::new(-1.1, 0.0); + let e: u32x2 = u32x2::new(!0, 0); + let r: u32x2 = transmute(vcagt_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcagtq_f32() { + let a: f32x4 = f32x4::new(-1.2, 0.0, 1.2, 2.3); + let b: f32x4 = f32x4::new(-1.1, 0.0, 1.1, 2.4); + let e: u32x4 = u32x4::new(!0, 0, 0xFF_FF_FF_FF, 0); + let r: u32x4 = transmute(vcagtq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcage_f32() { + let a: f32x2 = f32x2::new(-1.2, 0.0); + let b: f32x2 = f32x2::new(-1.1, 0.0); + let e: u32x2 = u32x2::new(!0, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcage_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcageq_f32() { + let a: f32x4 = f32x4::new(-1.2, 0.0, 1.2, 2.3); + let b: f32x4 = f32x4::new(-1.1, 0.0, 1.1, 2.4); + let e: u32x4 = u32x4::new(!0, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0); + let r: u32x4 = transmute(vcageq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcalt_f32() { + let a: f32x2 = f32x2::new(-1.2, 0.0); + let b: f32x2 = f32x2::new(-1.1, 0.0); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vcalt_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcaltq_f32() { + let a: f32x4 = f32x4::new(-1.2, 0.0, 1.2, 2.3); + let b: f32x4 = f32x4::new(-1.1, 0.0, 1.1, 2.4); + let e: u32x4 = u32x4::new(0, 0, 0, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcaltq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcale_f32() { + let a: f32x2 = f32x2::new(-1.2, 0.0); + let b: f32x2 = f32x2::new(-1.1, 0.0); + let e: u32x2 = u32x2::new(0, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcale_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcaleq_f32() { + let a: f32x4 = f32x4::new(-1.2, 0.0, 1.2, 2.3); + let b: f32x4 = f32x4::new(-1.1, 0.0, 1.1, 2.4); + let e: u32x4 = u32x4::new(0, 0xFF_FF_FF_FF, 0, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcaleq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_s8() { + let a: u64 = 1; + let e: i8x8 = i8x8::new(1, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vcreate_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_s16() { + let a: u64 = 1; + let e: i16x4 = i16x4::new(1, 0, 0, 0); + let r: i16x4 = transmute(vcreate_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_s32() { + let a: u64 = 1; + let e: i32x2 = i32x2::new(1, 0); + let r: i32x2 = transmute(vcreate_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_s64() { + let a: u64 = 1; + let e: i64x1 = i64x1::new(1); + let r: i64x1 = transmute(vcreate_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_u8() { + let a: u64 = 1; + let e: u8x8 = u8x8::new(1, 0, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vcreate_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_u16() { + let a: u64 = 1; + let e: u16x4 = u16x4::new(1, 0, 0, 0); + let r: u16x4 = transmute(vcreate_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_u32() { + let a: u64 = 1; + let e: u32x2 = u32x2::new(1, 0); + let r: u32x2 = transmute(vcreate_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_u64() { + let a: u64 = 1; + let e: u64x1 = u64x1::new(1); + let r: u64x1 = transmute(vcreate_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_p8() { + let a: u64 = 1; + let e: i8x8 = i8x8::new(1, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vcreate_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_p16() { + let a: u64 = 1; + let e: i16x4 = i16x4::new(1, 0, 0, 0); + let r: i16x4 = transmute(vcreate_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_p64() { + let a: u64 = 1; + let e: i64x1 = i64x1::new(1); + let r: i64x1 = transmute(vcreate_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_f32() { + let a: u64 = 0; + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vcreate_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_f32_s32() { + let a: i32x2 = i32x2::new(1, 2); + let e: f32x2 = f32x2::new(1., 2.); + let r: f32x2 = transmute(vcvt_f32_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_f32_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let e: f32x4 = f32x4::new(1., 2., 3., 4.); + let r: f32x4 = transmute(vcvtq_f32_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_f32_u32() { + let a: u32x2 = u32x2::new(1, 2); + let e: f32x2 = f32x2::new(1., 2.); + let r: f32x2 = transmute(vcvt_f32_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_f32_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let e: f32x4 = f32x4::new(1., 2., 3., 4.); + let r: f32x4 = transmute(vcvtq_f32_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_n_f32_s32() { + let a: i32x2 = i32x2::new(1, 2); + let e: f32x2 = f32x2::new(0.25, 0.5); + let r: f32x2 = transmute(vcvt_n_f32_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_n_f32_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let e: f32x4 = f32x4::new(0.25, 0.5, 0.75, 1.); + let r: f32x4 = transmute(vcvtq_n_f32_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_n_f32_u32() { + let a: u32x2 = u32x2::new(1, 2); + let e: f32x2 = f32x2::new(0.25, 0.5); + let r: f32x2 = transmute(vcvt_n_f32_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_n_f32_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let e: f32x4 = f32x4::new(0.25, 0.5, 0.75, 1.); + let r: f32x4 = transmute(vcvtq_n_f32_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_n_s32_f32() { + let a: f32x2 = f32x2::new(0.25, 0.5); + let e: i32x2 = i32x2::new(1, 2); + let r: i32x2 = transmute(vcvt_n_s32_f32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_n_s32_f32() { + let a: f32x4 = f32x4::new(0.25, 0.5, 0.75, 1.); + let e: i32x4 = i32x4::new(1, 2, 3, 4); + let r: i32x4 = transmute(vcvtq_n_s32_f32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_n_u32_f32() { + let a: f32x2 = f32x2::new(0.25, 0.5); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vcvt_n_u32_f32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_n_u32_f32() { + let a: f32x4 = f32x4::new(0.25, 0.5, 0.75, 1.); + let e: u32x4 = u32x4::new(1, 2, 3, 4); + let r: u32x4 = transmute(vcvtq_n_u32_f32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_s32_f32() { + let a: f32x2 = f32x2::new(-1.1, 2.1); + let e: i32x2 = i32x2::new(-1, 2); + let r: i32x2 = transmute(vcvt_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_s32_f32() { + let a: f32x4 = f32x4::new(-1.1, 2.1, -2.9, 3.9); + let e: i32x4 = i32x4::new(-1, 2, -2, 3); + let r: i32x4 = transmute(vcvtq_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_u32_f32() { + let a: f32x2 = f32x2::new(1.1, 2.1); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vcvt_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_u32_f32() { + let a: f32x4 = f32x4::new(1.1, 2.1, 2.9, 3.9); + let e: u32x4 = u32x4::new(1, 2, 2, 3); + let r: u32x4 = transmute(vcvtq_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_s8() { + let a: i8x8 = i8x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: i8x8 = i8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i8x8 = transmute(vdup_lane_s8::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_s8() { + let a: i8x16 = i8x16::new(1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let r: i8x16 = transmute(vdupq_laneq_s8::<8>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 4); + let e: i16x4 = i16x4::new(1, 1, 1, 1); + let r: i16x4 = transmute(vdup_lane_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i16x8 = transmute(vdupq_laneq_s16::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_s32() { + let a: i32x2 = i32x2::new(1, 1); + let e: i32x2 = i32x2::new(1, 1); + let r: i32x2 = transmute(vdup_lane_s32::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 4); + let e: i32x4 = i32x4::new(1, 1, 1, 1); + let r: i32x4 = transmute(vdupq_laneq_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_s8() { + let a: i8x16 = i8x16::new(1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16); + let e: i8x8 = i8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i8x8 = transmute(vdup_laneq_s8::<8>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: i16x4 = i16x4::new(1, 1, 1, 1); + let r: i16x4 = transmute(vdup_laneq_s16::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 4); + let e: i32x2 = i32x2::new(1, 1); + let r: i32x2 = transmute(vdup_laneq_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_s8() { + let a: i8x8 = i8x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: i8x16 = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let r: i8x16 = transmute(vdupq_lane_s8::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 4); + let e: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i16x8 = transmute(vdupq_lane_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_s32() { + let a: i32x2 = i32x2::new(1, 1); + let e: i32x4 = i32x4::new(1, 1, 1, 1); + let r: i32x4 = transmute(vdupq_lane_s32::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_u8() { + let a: u8x8 = u8x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: u8x8 = u8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: u8x8 = transmute(vdup_lane_u8::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_u8() { + let a: u8x16 = u8x16::new(1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let r: u8x16 = transmute(vdupq_laneq_u8::<8>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_u16() { + let a: u16x4 = u16x4::new(1, 1, 1, 4); + let e: u16x4 = u16x4::new(1, 1, 1, 1); + let r: u16x4 = transmute(vdup_lane_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_u16() { + let a: u16x8 = u16x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: u16x8 = u16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: u16x8 = transmute(vdupq_laneq_u16::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_u32() { + let a: u32x2 = u32x2::new(1, 1); + let e: u32x2 = u32x2::new(1, 1); + let r: u32x2 = transmute(vdup_lane_u32::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_u32() { + let a: u32x4 = u32x4::new(1, 1, 1, 4); + let e: u32x4 = u32x4::new(1, 1, 1, 1); + let r: u32x4 = transmute(vdupq_laneq_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_u8() { + let a: u8x16 = u8x16::new(1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16); + let e: u8x8 = u8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: u8x8 = transmute(vdup_laneq_u8::<8>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_u16() { + let a: u16x8 = u16x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: u16x4 = u16x4::new(1, 1, 1, 1); + let r: u16x4 = transmute(vdup_laneq_u16::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_u32() { + let a: u32x4 = u32x4::new(1, 1, 1, 4); + let e: u32x2 = u32x2::new(1, 1); + let r: u32x2 = transmute(vdup_laneq_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_u8() { + let a: u8x8 = u8x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: u8x16 = u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let r: u8x16 = transmute(vdupq_lane_u8::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_u16() { + let a: u16x4 = u16x4::new(1, 1, 1, 4); + let e: u16x8 = u16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: u16x8 = transmute(vdupq_lane_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_u32() { + let a: u32x2 = u32x2::new(1, 1); + let e: u32x4 = u32x4::new(1, 1, 1, 1); + let r: u32x4 = transmute(vdupq_lane_u32::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_p8() { + let a: i8x8 = i8x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: i8x8 = i8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i8x8 = transmute(vdup_lane_p8::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_p8() { + let a: i8x16 = i8x16::new(1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let r: i8x16 = transmute(vdupq_laneq_p8::<8>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_p16() { + let a: i16x4 = i16x4::new(1, 1, 1, 4); + let e: i16x4 = i16x4::new(1, 1, 1, 1); + let r: i16x4 = transmute(vdup_lane_p16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_p16() { + let a: i16x8 = i16x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i16x8 = transmute(vdupq_laneq_p16::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_p8() { + let a: i8x16 = i8x16::new(1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16); + let e: i8x8 = i8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i8x8 = transmute(vdup_laneq_p8::<8>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_p16() { + let a: i16x8 = i16x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: i16x4 = i16x4::new(1, 1, 1, 1); + let r: i16x4 = transmute(vdup_laneq_p16::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_p8() { + let a: i8x8 = i8x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: i8x16 = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let r: i8x16 = transmute(vdupq_lane_p8::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_p16() { + let a: i16x4 = i16x4::new(1, 1, 1, 4); + let e: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i16x8 = transmute(vdupq_lane_p16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_s64() { + let a: i64x2 = i64x2::new(1, 1); + let e: i64x2 = i64x2::new(1, 1); + let r: i64x2 = transmute(vdupq_laneq_s64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_s64() { + let a: i64x1 = i64x1::new(1); + let e: i64x2 = i64x2::new(1, 1); + let r: i64x2 = transmute(vdupq_lane_s64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_u64() { + let a: u64x2 = u64x2::new(1, 1); + let e: u64x2 = u64x2::new(1, 1); + let r: u64x2 = transmute(vdupq_laneq_u64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_u64() { + let a: u64x1 = u64x1::new(1); + let e: u64x2 = u64x2::new(1, 1); + let r: u64x2 = transmute(vdupq_lane_u64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_f32() { + let a: f32x2 = f32x2::new(1., 1.); + let e: f32x2 = f32x2::new(1., 1.); + let r: f32x2 = transmute(vdup_lane_f32::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_f32() { + let a: f32x4 = f32x4::new(1., 1., 1., 4.); + let e: f32x4 = f32x4::new(1., 1., 1., 1.); + let r: f32x4 = transmute(vdupq_laneq_f32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_f32() { + let a: f32x4 = f32x4::new(1., 1., 1., 4.); + let e: f32x2 = f32x2::new(1., 1.); + let r: f32x2 = transmute(vdup_laneq_f32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_f32() { + let a: f32x2 = f32x2::new(1., 1.); + let e: f32x4 = f32x4::new(1., 1., 1., 1.); + let r: f32x4 = transmute(vdupq_lane_f32::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_s64() { + let a: i64x1 = i64x1::new(0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vdup_lane_s64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_u64() { + let a: u64x1 = u64x1::new(0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vdup_lane_u64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i64x1 = i64x1::new(1); + let r: i64x1 = transmute(vdup_laneq_s64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: u64x1 = u64x1::new(1); + let r: u64x1 = transmute(vdup_laneq_u64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_s8() { + let a: i8x8 = i8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i8x8 = i8x8::new(1, 2, 2, 2, 2, 2, 2, 2); + let r: i8x8 = transmute(vext_s8::<7>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_s8() { + let a: i8x16 = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let e: i8x16 = i8x16::new(1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let r: i8x16 = transmute(vextq_s8::<15>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i16x4 = i16x4::new(1, 2, 2, 2); + let r: i16x4 = transmute(vext_s16::<3>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i16x8 = i16x8::new(1, 2, 2, 2, 2, 2, 2, 2); + let r: i16x8 = transmute(vextq_s16::<7>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_s32() { + let a: i32x2 = i32x2::new(1, 1); + let b: i32x2 = i32x2::new(2, 2); + let e: i32x2 = i32x2::new(1, 2); + let r: i32x2 = transmute(vext_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 1); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let e: i32x4 = i32x4::new(1, 2, 2, 2); + let r: i32x4 = transmute(vextq_s32::<3>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_u8() { + let a: u8x8 = u8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: u8x8 = u8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: u8x8 = u8x8::new(1, 2, 2, 2, 2, 2, 2, 2); + let r: u8x8 = transmute(vext_u8::<7>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_u8() { + let a: u8x16 = u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let b: u8x16 = u8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let e: u8x16 = u8x16::new(1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let r: u8x16 = transmute(vextq_u8::<15>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_u16() { + let a: u16x4 = u16x4::new(1, 1, 1, 1); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let e: u16x4 = u16x4::new(1, 2, 2, 2); + let r: u16x4 = transmute(vext_u16::<3>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_u16() { + let a: u16x8 = u16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: u16x8 = u16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: u16x8 = u16x8::new(1, 2, 2, 2, 2, 2, 2, 2); + let r: u16x8 = transmute(vextq_u16::<7>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_u32() { + let a: u32x2 = u32x2::new(1, 1); + let b: u32x2 = u32x2::new(2, 2); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vext_u32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_u32() { + let a: u32x4 = u32x4::new(1, 1, 1, 1); + let b: u32x4 = u32x4::new(2, 2, 2, 2); + let e: u32x4 = u32x4::new(1, 2, 2, 2); + let r: u32x4 = transmute(vextq_u32::<3>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_p8() { + let a: i8x8 = i8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i8x8 = i8x8::new(1, 2, 2, 2, 2, 2, 2, 2); + let r: i8x8 = transmute(vext_p8::<7>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_p8() { + let a: i8x16 = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let e: i8x16 = i8x16::new(1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let r: i8x16 = transmute(vextq_p8::<15>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_p16() { + let a: i16x4 = i16x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i16x4 = i16x4::new(1, 2, 2, 2); + let r: i16x4 = transmute(vext_p16::<3>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_p16() { + let a: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i16x8 = i16x8::new(1, 2, 2, 2, 2, 2, 2, 2); + let r: i16x8 = transmute(vextq_p16::<7>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_s64() { + let a: i64x2 = i64x2::new(1, 1); + let b: i64x2 = i64x2::new(2, 2); + let e: i64x2 = i64x2::new(1, 2); + let r: i64x2 = transmute(vextq_s64::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_u64() { + let a: u64x2 = u64x2::new(1, 1); + let b: u64x2 = u64x2::new(2, 2); + let e: u64x2 = u64x2::new(1, 2); + let r: u64x2 = transmute(vextq_u64::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_f32() { + let a: f32x2 = f32x2::new(1., 1.); + let b: f32x2 = f32x2::new(2., 2.); + let e: f32x2 = f32x2::new(1., 2.); + let r: f32x2 = transmute(vext_f32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_f32() { + let a: f32x4 = f32x4::new(1., 1., 1., 1.); + let b: f32x4 = f32x4::new(2., 2., 2., 2.); + let e: f32x4 = f32x4::new(1., 2., 2., 2.); + let r: f32x4 = transmute(vextq_f32::<3>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_s8() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i8x8 = i8x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: i8x8 = i8x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: i8x8 = transmute(vmla_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_s8() { + let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let c: i8x16 = i8x16::new(3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3); + let e: i8x16 = i8x16::new(6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21); + let r: i8x16 = transmute(vmlaq_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x4 = i16x4::new(3, 3, 3, 3); + let e: i16x4 = i16x4::new(6, 7, 8, 9); + let r: i16x4 = transmute(vmla_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i16x8 = i16x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: i16x8 = transmute(vmlaq_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_s32() { + let a: i32x2 = i32x2::new(0, 1); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x2 = i32x2::new(3, 3); + let e: i32x2 = i32x2::new(6, 7); + let r: i32x2 = transmute(vmla_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let c: i32x4 = i32x4::new(3, 3, 3, 3); + let e: i32x4 = i32x4::new(6, 7, 8, 9); + let r: i32x4 = transmute(vmlaq_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_u8() { + let a: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u8x8 = u8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u8x8 = u8x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: u8x8 = u8x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: u8x8 = transmute(vmla_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_u8() { + let a: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: u8x16 = u8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let c: u8x16 = u8x16::new(3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3); + let e: u8x16 = u8x16::new(6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21); + let r: u8x16 = transmute(vmlaq_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x4 = u16x4::new(3, 3, 3, 3); + let e: u16x4 = u16x4::new(6, 7, 8, 9); + let r: u16x4 = transmute(vmla_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u16x8 = u16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u16x8 = u16x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: u16x8 = transmute(vmlaq_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_u32() { + let a: u32x2 = u32x2::new(0, 1); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x2 = u32x2::new(3, 3); + let e: u32x2 = u32x2::new(6, 7); + let r: u32x2 = transmute(vmla_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u32x4 = u32x4::new(2, 2, 2, 2); + let c: u32x4 = u32x4::new(3, 3, 3, 3); + let e: u32x4 = u32x4::new(6, 7, 8, 9); + let r: u32x4 = transmute(vmlaq_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_f32() { + let a: f32x2 = f32x2::new(0., 1.); + let b: f32x2 = f32x2::new(2., 2.); + let c: f32x2 = f32x2::new(3., 3.); + let e: f32x2 = f32x2::new(6., 7.); + let r: f32x2 = transmute(vmla_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_f32() { + let a: f32x4 = f32x4::new(0., 1., 2., 3.); + let b: f32x4 = f32x4::new(2., 2., 2., 2.); + let c: f32x4 = f32x4::new(3., 3., 3., 3.); + let e: f32x4 = f32x4::new(6., 7., 8., 9.); + let r: f32x4 = transmute(vmlaq_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_n_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16 = 3; + let e: i16x4 = i16x4::new(6, 7, 8, 9); + let r: i16x4 = transmute(vmla_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_n_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i16 = 3; + let e: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: i16x8 = transmute(vmlaq_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_n_s32() { + let a: i32x2 = i32x2::new(0, 1); + let b: i32x2 = i32x2::new(2, 2); + let c: i32 = 3; + let e: i32x2 = i32x2::new(6, 7); + let r: i32x2 = transmute(vmla_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_n_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let c: i32 = 3; + let e: i32x4 = i32x4::new(6, 7, 8, 9); + let r: i32x4 = transmute(vmlaq_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_n_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16 = 3; + let e: u16x4 = u16x4::new(6, 7, 8, 9); + let r: u16x4 = transmute(vmla_n_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_n_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u16x8 = u16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u16 = 3; + let e: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: u16x8 = transmute(vmlaq_n_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_n_u32() { + let a: u32x2 = u32x2::new(0, 1); + let b: u32x2 = u32x2::new(2, 2); + let c: u32 = 3; + let e: u32x2 = u32x2::new(6, 7); + let r: u32x2 = transmute(vmla_n_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_n_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u32x4 = u32x4::new(2, 2, 2, 2); + let c: u32 = 3; + let e: u32x4 = u32x4::new(6, 7, 8, 9); + let r: u32x4 = transmute(vmlaq_n_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_n_f32() { + let a: f32x2 = f32x2::new(0., 1.); + let b: f32x2 = f32x2::new(2., 2.); + let c: f32 = 3.; + let e: f32x2 = f32x2::new(6., 7.); + let r: f32x2 = transmute(vmla_n_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_n_f32() { + let a: f32x4 = f32x4::new(0., 1., 2., 3.); + let b: f32x4 = f32x4::new(2., 2., 2., 2.); + let c: f32 = 3.; + let e: f32x4 = f32x4::new(6., 7., 8., 9.); + let r: f32x4 = transmute(vmlaq_n_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_lane_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x4 = i16x4::new(0, 3, 0, 0); + let e: i16x4 = i16x4::new(6, 7, 8, 9); + let r: i16x4 = transmute(vmla_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_laneq_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x8 = i16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: i16x4 = i16x4::new(6, 7, 8, 9); + let r: i16x4 = transmute(vmla_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_lane_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i16x4 = i16x4::new(0, 3, 0, 0); + let e: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: i16x8 = transmute(vmlaq_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_laneq_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i16x8 = i16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: i16x8 = transmute(vmlaq_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_lane_s32() { + let a: i32x2 = i32x2::new(0, 1); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x2 = i32x2::new(0, 3); + let e: i32x2 = i32x2::new(6, 7); + let r: i32x2 = transmute(vmla_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_laneq_s32() { + let a: i32x2 = i32x2::new(0, 1); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x4 = i32x4::new(0, 3, 0, 0); + let e: i32x2 = i32x2::new(6, 7); + let r: i32x2 = transmute(vmla_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_lane_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let c: i32x2 = i32x2::new(0, 3); + let e: i32x4 = i32x4::new(6, 7, 8, 9); + let r: i32x4 = transmute(vmlaq_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_laneq_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let c: i32x4 = i32x4::new(0, 3, 0, 0); + let e: i32x4 = i32x4::new(6, 7, 8, 9); + let r: i32x4 = transmute(vmlaq_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_lane_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x4 = u16x4::new(0, 3, 0, 0); + let e: u16x4 = u16x4::new(6, 7, 8, 9); + let r: u16x4 = transmute(vmla_lane_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_laneq_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x8 = u16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: u16x4 = u16x4::new(6, 7, 8, 9); + let r: u16x4 = transmute(vmla_laneq_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_lane_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u16x8 = u16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u16x4 = u16x4::new(0, 3, 0, 0); + let e: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: u16x8 = transmute(vmlaq_lane_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_laneq_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u16x8 = u16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u16x8 = u16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: u16x8 = transmute(vmlaq_laneq_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_lane_u32() { + let a: u32x2 = u32x2::new(0, 1); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x2 = u32x2::new(0, 3); + let e: u32x2 = u32x2::new(6, 7); + let r: u32x2 = transmute(vmla_lane_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_laneq_u32() { + let a: u32x2 = u32x2::new(0, 1); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x4 = u32x4::new(0, 3, 0, 0); + let e: u32x2 = u32x2::new(6, 7); + let r: u32x2 = transmute(vmla_laneq_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_lane_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u32x4 = u32x4::new(2, 2, 2, 2); + let c: u32x2 = u32x2::new(0, 3); + let e: u32x4 = u32x4::new(6, 7, 8, 9); + let r: u32x4 = transmute(vmlaq_lane_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_laneq_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u32x4 = u32x4::new(2, 2, 2, 2); + let c: u32x4 = u32x4::new(0, 3, 0, 0); + let e: u32x4 = u32x4::new(6, 7, 8, 9); + let r: u32x4 = transmute(vmlaq_laneq_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_lane_f32() { + let a: f32x2 = f32x2::new(0., 1.); + let b: f32x2 = f32x2::new(2., 2.); + let c: f32x2 = f32x2::new(0., 3.); + let e: f32x2 = f32x2::new(6., 7.); + let r: f32x2 = transmute(vmla_lane_f32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_laneq_f32() { + let a: f32x2 = f32x2::new(0., 1.); + let b: f32x2 = f32x2::new(2., 2.); + let c: f32x4 = f32x4::new(0., 3., 0., 0.); + let e: f32x2 = f32x2::new(6., 7.); + let r: f32x2 = transmute(vmla_laneq_f32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_lane_f32() { + let a: f32x4 = f32x4::new(0., 1., 2., 3.); + let b: f32x4 = f32x4::new(2., 2., 2., 2.); + let c: f32x2 = f32x2::new(0., 3.); + let e: f32x4 = f32x4::new(6., 7., 8., 9.); + let r: f32x4 = transmute(vmlaq_lane_f32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_laneq_f32() { + let a: f32x4 = f32x4::new(0., 1., 2., 3.); + let b: f32x4 = f32x4::new(2., 2., 2., 2.); + let c: f32x4 = f32x4::new(0., 3., 0., 0.); + let e: f32x4 = f32x4::new(6., 7., 8., 9.); + let r: f32x4 = transmute(vmlaq_laneq_f32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_s8() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i8x8 = i8x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: i16x8 = transmute(vmlal_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_s16() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x4 = i16x4::new(3, 3, 3, 3); + let e: i32x4 = i32x4::new(6, 7, 8, 9); + let r: i32x4 = transmute(vmlal_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_s32() { + let a: i64x2 = i64x2::new(0, 1); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x2 = i32x2::new(3, 3); + let e: i64x2 = i64x2::new(6, 7); + let r: i64x2 = transmute(vmlal_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_u8() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u8x8 = u8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u8x8 = u8x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: u16x8 = transmute(vmlal_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_u16() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x4 = u16x4::new(3, 3, 3, 3); + let e: u32x4 = u32x4::new(6, 7, 8, 9); + let r: u32x4 = transmute(vmlal_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_u32() { + let a: u64x2 = u64x2::new(0, 1); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x2 = u32x2::new(3, 3); + let e: u64x2 = u64x2::new(6, 7); + let r: u64x2 = transmute(vmlal_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_n_s16() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16 = 3; + let e: i32x4 = i32x4::new(6, 7, 8, 9); + let r: i32x4 = transmute(vmlal_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_n_s32() { + let a: i64x2 = i64x2::new(0, 1); + let b: i32x2 = i32x2::new(2, 2); + let c: i32 = 3; + let e: i64x2 = i64x2::new(6, 7); + let r: i64x2 = transmute(vmlal_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_n_u16() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16 = 3; + let e: u32x4 = u32x4::new(6, 7, 8, 9); + let r: u32x4 = transmute(vmlal_n_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_n_u32() { + let a: u64x2 = u64x2::new(0, 1); + let b: u32x2 = u32x2::new(2, 2); + let c: u32 = 3; + let e: u64x2 = u64x2::new(6, 7); + let r: u64x2 = transmute(vmlal_n_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_lane_s16() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x4 = i16x4::new(0, 3, 0, 0); + let e: i32x4 = i32x4::new(6, 7, 8, 9); + let r: i32x4 = transmute(vmlal_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_laneq_s16() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x8 = i16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: i32x4 = i32x4::new(6, 7, 8, 9); + let r: i32x4 = transmute(vmlal_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_lane_s32() { + let a: i64x2 = i64x2::new(0, 1); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x2 = i32x2::new(0, 3); + let e: i64x2 = i64x2::new(6, 7); + let r: i64x2 = transmute(vmlal_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_laneq_s32() { + let a: i64x2 = i64x2::new(0, 1); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x4 = i32x4::new(0, 3, 0, 0); + let e: i64x2 = i64x2::new(6, 7); + let r: i64x2 = transmute(vmlal_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_lane_u16() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x4 = u16x4::new(0, 3, 0, 0); + let e: u32x4 = u32x4::new(6, 7, 8, 9); + let r: u32x4 = transmute(vmlal_lane_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_laneq_u16() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x8 = u16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: u32x4 = u32x4::new(6, 7, 8, 9); + let r: u32x4 = transmute(vmlal_laneq_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_lane_u32() { + let a: u64x2 = u64x2::new(0, 1); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x2 = u32x2::new(0, 3); + let e: u64x2 = u64x2::new(6, 7); + let r: u64x2 = transmute(vmlal_lane_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_laneq_u32() { + let a: u64x2 = u64x2::new(0, 1); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x4 = u32x4::new(0, 3, 0, 0); + let e: u64x2 = u64x2::new(6, 7); + let r: u64x2 = transmute(vmlal_laneq_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_s8() { + let a: i8x8 = i8x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i8x8 = i8x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i8x8 = transmute(vmls_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_s8() { + let a: i8x16 = i8x16::new(6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let c: i8x16 = i8x16::new(3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3); + let e: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: i8x16 = transmute(vmlsq_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_s16() { + let a: i16x4 = i16x4::new(6, 7, 8, 9); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x4 = i16x4::new(3, 3, 3, 3); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vmls_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_s16() { + let a: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i16x8 = i16x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vmlsq_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_s32() { + let a: i32x2 = i32x2::new(6, 7); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x2 = i32x2::new(3, 3); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vmls_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_s32() { + let a: i32x4 = i32x4::new(6, 7, 8, 9); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let c: i32x4 = i32x4::new(3, 3, 3, 3); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vmlsq_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_u8() { + let a: u8x8 = u8x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: u8x8 = u8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u8x8 = u8x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u8x8 = transmute(vmls_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_u8() { + let a: u8x16 = u8x16::new(6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21); + let b: u8x16 = u8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let c: u8x16 = u8x16::new(3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3); + let e: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: u8x16 = transmute(vmlsq_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_u16() { + let a: u16x4 = u16x4::new(6, 7, 8, 9); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x4 = u16x4::new(3, 3, 3, 3); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vmls_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_u16() { + let a: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: u16x8 = u16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u16x8 = u16x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vmlsq_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_u32() { + let a: u32x2 = u32x2::new(6, 7); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x2 = u32x2::new(3, 3); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vmls_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_u32() { + let a: u32x4 = u32x4::new(6, 7, 8, 9); + let b: u32x4 = u32x4::new(2, 2, 2, 2); + let c: u32x4 = u32x4::new(3, 3, 3, 3); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vmlsq_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_f32() { + let a: f32x2 = f32x2::new(6., 7.); + let b: f32x2 = f32x2::new(2., 2.); + let c: f32x2 = f32x2::new(3., 3.); + let e: f32x2 = f32x2::new(0., 1.); + let r: f32x2 = transmute(vmls_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_f32() { + let a: f32x4 = f32x4::new(6., 7., 8., 9.); + let b: f32x4 = f32x4::new(2., 2., 2., 2.); + let c: f32x4 = f32x4::new(3., 3., 3., 3.); + let e: f32x4 = f32x4::new(0., 1., 2., 3.); + let r: f32x4 = transmute(vmlsq_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_n_s16() { + let a: i16x4 = i16x4::new(6, 7, 8, 9); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16 = 3; + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vmls_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_n_s16() { + let a: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i16 = 3; + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vmlsq_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_n_s32() { + let a: i32x2 = i32x2::new(6, 7); + let b: i32x2 = i32x2::new(2, 2); + let c: i32 = 3; + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vmls_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_n_s32() { + let a: i32x4 = i32x4::new(6, 7, 8, 9); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let c: i32 = 3; + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vmlsq_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_n_u16() { + let a: u16x4 = u16x4::new(6, 7, 8, 9); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16 = 3; + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vmls_n_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_n_u16() { + let a: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: u16x8 = u16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u16 = 3; + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vmlsq_n_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_n_u32() { + let a: u32x2 = u32x2::new(6, 7); + let b: u32x2 = u32x2::new(2, 2); + let c: u32 = 3; + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vmls_n_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_n_u32() { + let a: u32x4 = u32x4::new(6, 7, 8, 9); + let b: u32x4 = u32x4::new(2, 2, 2, 2); + let c: u32 = 3; + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vmlsq_n_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_n_f32() { + let a: f32x2 = f32x2::new(6., 7.); + let b: f32x2 = f32x2::new(2., 2.); + let c: f32 = 3.; + let e: f32x2 = f32x2::new(0., 1.); + let r: f32x2 = transmute(vmls_n_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_n_f32() { + let a: f32x4 = f32x4::new(6., 7., 8., 9.); + let b: f32x4 = f32x4::new(2., 2., 2., 2.); + let c: f32 = 3.; + let e: f32x4 = f32x4::new(0., 1., 2., 3.); + let r: f32x4 = transmute(vmlsq_n_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_lane_s16() { + let a: i16x4 = i16x4::new(6, 7, 8, 9); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x4 = i16x4::new(0, 3, 0, 0); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vmls_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_laneq_s16() { + let a: i16x4 = i16x4::new(6, 7, 8, 9); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x8 = i16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vmls_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_lane_s16() { + let a: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i16x4 = i16x4::new(0, 3, 0, 0); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vmlsq_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_laneq_s16() { + let a: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i16x8 = i16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vmlsq_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_lane_s32() { + let a: i32x2 = i32x2::new(6, 7); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x2 = i32x2::new(0, 3); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vmls_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_laneq_s32() { + let a: i32x2 = i32x2::new(6, 7); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x4 = i32x4::new(0, 3, 0, 0); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vmls_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_lane_s32() { + let a: i32x4 = i32x4::new(6, 7, 8, 9); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let c: i32x2 = i32x2::new(0, 3); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vmlsq_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_laneq_s32() { + let a: i32x4 = i32x4::new(6, 7, 8, 9); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let c: i32x4 = i32x4::new(0, 3, 0, 0); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vmlsq_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_lane_u16() { + let a: u16x4 = u16x4::new(6, 7, 8, 9); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x4 = u16x4::new(0, 3, 0, 0); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vmls_lane_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_laneq_u16() { + let a: u16x4 = u16x4::new(6, 7, 8, 9); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x8 = u16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vmls_laneq_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_lane_u16() { + let a: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: u16x8 = u16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u16x4 = u16x4::new(0, 3, 0, 0); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vmlsq_lane_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_laneq_u16() { + let a: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: u16x8 = u16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u16x8 = u16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vmlsq_laneq_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_lane_u32() { + let a: u32x2 = u32x2::new(6, 7); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x2 = u32x2::new(0, 3); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vmls_lane_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_laneq_u32() { + let a: u32x2 = u32x2::new(6, 7); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x4 = u32x4::new(0, 3, 0, 0); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vmls_laneq_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_lane_u32() { + let a: u32x4 = u32x4::new(6, 7, 8, 9); + let b: u32x4 = u32x4::new(2, 2, 2, 2); + let c: u32x2 = u32x2::new(0, 3); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vmlsq_lane_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_laneq_u32() { + let a: u32x4 = u32x4::new(6, 7, 8, 9); + let b: u32x4 = u32x4::new(2, 2, 2, 2); + let c: u32x4 = u32x4::new(0, 3, 0, 0); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vmlsq_laneq_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_lane_f32() { + let a: f32x2 = f32x2::new(6., 7.); + let b: f32x2 = f32x2::new(2., 2.); + let c: f32x2 = f32x2::new(0., 3.); + let e: f32x2 = f32x2::new(0., 1.); + let r: f32x2 = transmute(vmls_lane_f32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_laneq_f32() { + let a: f32x2 = f32x2::new(6., 7.); + let b: f32x2 = f32x2::new(2., 2.); + let c: f32x4 = f32x4::new(0., 3., 0., 0.); + let e: f32x2 = f32x2::new(0., 1.); + let r: f32x2 = transmute(vmls_laneq_f32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_lane_f32() { + let a: f32x4 = f32x4::new(6., 7., 8., 9.); + let b: f32x4 = f32x4::new(2., 2., 2., 2.); + let c: f32x2 = f32x2::new(0., 3.); + let e: f32x4 = f32x4::new(0., 1., 2., 3.); + let r: f32x4 = transmute(vmlsq_lane_f32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_laneq_f32() { + let a: f32x4 = f32x4::new(6., 7., 8., 9.); + let b: f32x4 = f32x4::new(2., 2., 2., 2.); + let c: f32x4 = f32x4::new(0., 3., 0., 0.); + let e: f32x4 = f32x4::new(0., 1., 2., 3.); + let r: f32x4 = transmute(vmlsq_laneq_f32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_s8() { + let a: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i8x8 = i8x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vmlsl_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_s16() { + let a: i32x4 = i32x4::new(6, 7, 8, 9); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x4 = i16x4::new(3, 3, 3, 3); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vmlsl_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_s32() { + let a: i64x2 = i64x2::new(6, 7); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x2 = i32x2::new(3, 3); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vmlsl_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_u8() { + let a: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: u8x8 = u8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u8x8 = u8x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vmlsl_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_u16() { + let a: u32x4 = u32x4::new(6, 7, 8, 9); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x4 = u16x4::new(3, 3, 3, 3); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vmlsl_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_u32() { + let a: u64x2 = u64x2::new(6, 7); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x2 = u32x2::new(3, 3); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vmlsl_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_n_s16() { + let a: i32x4 = i32x4::new(6, 7, 8, 9); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16 = 3; + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vmlsl_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_n_s32() { + let a: i64x2 = i64x2::new(6, 7); + let b: i32x2 = i32x2::new(2, 2); + let c: i32 = 3; + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vmlsl_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_n_u16() { + let a: u32x4 = u32x4::new(6, 7, 8, 9); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16 = 3; + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vmlsl_n_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_n_u32() { + let a: u64x2 = u64x2::new(6, 7); + let b: u32x2 = u32x2::new(2, 2); + let c: u32 = 3; + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vmlsl_n_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_lane_s16() { + let a: i32x4 = i32x4::new(6, 7, 8, 9); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x4 = i16x4::new(0, 3, 0, 0); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vmlsl_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_laneq_s16() { + let a: i32x4 = i32x4::new(6, 7, 8, 9); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x8 = i16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vmlsl_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_lane_s32() { + let a: i64x2 = i64x2::new(6, 7); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x2 = i32x2::new(0, 3); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vmlsl_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_laneq_s32() { + let a: i64x2 = i64x2::new(6, 7); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x4 = i32x4::new(0, 3, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vmlsl_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_lane_u16() { + let a: u32x4 = u32x4::new(6, 7, 8, 9); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x4 = u16x4::new(0, 3, 0, 0); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vmlsl_lane_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_laneq_u16() { + let a: u32x4 = u32x4::new(6, 7, 8, 9); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x8 = u16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vmlsl_laneq_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_lane_u32() { + let a: u64x2 = u64x2::new(6, 7); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x2 = u32x2::new(0, 3); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vmlsl_lane_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_laneq_u32() { + let a: u64x2 = u64x2::new(6, 7); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x4 = u32x4::new(0, 3, 0, 0); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vmlsl_laneq_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vneg_s8() { + let a: i8x8 = i8x8::new(0, 1, -1, 2, -2, 3, -3, 4); + let e: i8x8 = i8x8::new(0, -1, 1, -2, 2, -3, 3, -4); + let r: i8x8 = transmute(vneg_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vnegq_s8() { + let a: i8x16 = i8x16::new(0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 7, -7, 8); + let e: i8x16 = i8x16::new(0, -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7, -8); + let r: i8x16 = transmute(vnegq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vneg_s16() { + let a: i16x4 = i16x4::new(0, 1, -1, 2); + let e: i16x4 = i16x4::new(0, -1, 1, -2); + let r: i16x4 = transmute(vneg_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vnegq_s16() { + let a: i16x8 = i16x8::new(0, 1, -1, 2, -2, 3, -3, 4); + let e: i16x8 = i16x8::new(0, -1, 1, -2, 2, -3, 3, -4); + let r: i16x8 = transmute(vnegq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vneg_s32() { + let a: i32x2 = i32x2::new(0, 1); + let e: i32x2 = i32x2::new(0, -1); + let r: i32x2 = transmute(vneg_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vnegq_s32() { + let a: i32x4 = i32x4::new(0, 1, -1, 2); + let e: i32x4 = i32x4::new(0, -1, 1, -2); + let r: i32x4 = transmute(vnegq_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vneg_f32() { + let a: f32x2 = f32x2::new(0., 1.); + let e: f32x2 = f32x2::new(0., -1.); + let r: f32x2 = transmute(vneg_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vnegq_f32() { + let a: f32x4 = f32x4::new(0., 1., -1., 2.); + let e: f32x4 = f32x4::new(0., -1., 1., -2.); + let r: f32x4 = transmute(vnegq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqneg_s8() { + let a: i8x8 = i8x8::new(-128, 0, 1, -1, 2, -2, 3, -3); + let e: i8x8 = i8x8::new(0x7F, 0, -1, 1, -2, 2, -3, 3); + let r: i8x8 = transmute(vqneg_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqnegq_s8() { + let a: i8x16 = i8x16::new(-128, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 7, -7); + let e: i8x16 = i8x16::new(0x7F, 0, -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7); + let r: i8x16 = transmute(vqnegq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqneg_s16() { + let a: i16x4 = i16x4::new(-32768, 0, 1, -1); + let e: i16x4 = i16x4::new(0x7F_FF, 0, -1, 1); + let r: i16x4 = transmute(vqneg_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqnegq_s16() { + let a: i16x8 = i16x8::new(-32768, 0, 1, -1, 2, -2, 3, -3); + let e: i16x8 = i16x8::new(0x7F_FF, 0, -1, 1, -2, 2, -3, 3); + let r: i16x8 = transmute(vqnegq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqneg_s32() { + let a: i32x2 = i32x2::new(-2147483648, 0); + let e: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0); + let r: i32x2 = transmute(vqneg_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqnegq_s32() { + let a: i32x4 = i32x4::new(-2147483648, 0, 1, -1); + let e: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0, -1, 1); + let r: i32x4 = transmute(vqnegq_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_u8() { + let a: u8x8 = u8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(41, 40, 39, 38, 37, 36, 35, 34); + let r: u8x8 = transmute(vqsub_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_u8() { + let a: u8x16 = u8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); + let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26); + let r: u8x16 = transmute(vqsubq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_u16() { + let a: u16x4 = u16x4::new(42, 42, 42, 42); + let b: u16x4 = u16x4::new(1, 2, 3, 4); + let e: u16x4 = u16x4::new(41, 40, 39, 38); + let r: u16x4 = transmute(vqsub_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_u16() { + let a: u16x8 = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(41, 40, 39, 38, 37, 36, 35, 34); + let r: u16x8 = transmute(vqsubq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_u32() { + let a: u32x2 = u32x2::new(42, 42); + let b: u32x2 = u32x2::new(1, 2); + let e: u32x2 = u32x2::new(41, 40); + let r: u32x2 = transmute(vqsub_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_u32() { + let a: u32x4 = u32x4::new(42, 42, 42, 42); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(41, 40, 39, 38); + let r: u32x4 = transmute(vqsubq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_u64() { + let a: u64x1 = u64x1::new(42); + let b: u64x1 = u64x1::new(1); + let e: u64x1 = u64x1::new(41); + let r: u64x1 = transmute(vqsub_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_u64() { + let a: u64x2 = u64x2::new(42, 42); + let b: u64x2 = u64x2::new(1, 2); + let e: u64x2 = u64x2::new(41, 40); + let r: u64x2 = transmute(vqsubq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_s8() { + let a: i8x8 = i8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i8x8 = i8x8::new(41, 40, 39, 38, 37, 36, 35, 34); + let r: i8x8 = transmute(vqsub_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_s8() { + let a: i8x16 = i8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); + let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26); + let r: i8x16 = transmute(vqsubq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_s16() { + let a: i16x4 = i16x4::new(42, 42, 42, 42); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let e: i16x4 = i16x4::new(41, 40, 39, 38); + let r: i16x4 = transmute(vqsub_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_s16() { + let a: i16x8 = i16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i16x8 = i16x8::new(41, 40, 39, 38, 37, 36, 35, 34); + let r: i16x8 = transmute(vqsubq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_s32() { + let a: i32x2 = i32x2::new(42, 42); + let b: i32x2 = i32x2::new(1, 2); + let e: i32x2 = i32x2::new(41, 40); + let r: i32x2 = transmute(vqsub_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_s32() { + let a: i32x4 = i32x4::new(42, 42, 42, 42); + let b: i32x4 = i32x4::new(1, 2, 3, 4); + let e: i32x4 = i32x4::new(41, 40, 39, 38); + let r: i32x4 = transmute(vqsubq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_s64() { + let a: i64x1 = i64x1::new(42); + let b: i64x1 = i64x1::new(1); + let e: i64x1 = i64x1::new(41); + let r: i64x1 = transmute(vqsub_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_s64() { + let a: i64x2 = i64x2::new(42, 42); + let b: i64x2 = i64x2::new(1, 2); + let e: i64x2 = i64x2::new(41, 40); + let r: i64x2 = transmute(vqsubq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhadd_u8() { + let a: u8x8 = u8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(21, 22, 22, 23, 23, 24, 24, 25); + let r: u8x8 = transmute(vhadd_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhaddq_u8() { + let a: u8x16 = u8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); + let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(21, 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29); + let r: u8x16 = transmute(vhaddq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhadd_u16() { + let a: u16x4 = u16x4::new(42, 42, 42, 42); + let b: u16x4 = u16x4::new(1, 2, 3, 4); + let e: u16x4 = u16x4::new(21, 22, 22, 23); + let r: u16x4 = transmute(vhadd_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhaddq_u16() { + let a: u16x8 = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(21, 22, 22, 23, 23, 24, 24, 25); + let r: u16x8 = transmute(vhaddq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhadd_u32() { + let a: u32x2 = u32x2::new(42, 42); + let b: u32x2 = u32x2::new(1, 2); + let e: u32x2 = u32x2::new(21, 22); + let r: u32x2 = transmute(vhadd_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhaddq_u32() { + let a: u32x4 = u32x4::new(42, 42, 42, 42); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(21, 22, 22, 23); + let r: u32x4 = transmute(vhaddq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhadd_s8() { + let a: i8x8 = i8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i8x8 = i8x8::new(21, 22, 22, 23, 23, 24, 24, 25); + let r: i8x8 = transmute(vhadd_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhaddq_s8() { + let a: i8x16 = i8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); + let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(21, 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29); + let r: i8x16 = transmute(vhaddq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhadd_s16() { + let a: i16x4 = i16x4::new(42, 42, 42, 42); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let e: i16x4 = i16x4::new(21, 22, 22, 23); + let r: i16x4 = transmute(vhadd_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhaddq_s16() { + let a: i16x8 = i16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i16x8 = i16x8::new(21, 22, 22, 23, 23, 24, 24, 25); + let r: i16x8 = transmute(vhaddq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhadd_s32() { + let a: i32x2 = i32x2::new(42, 42); + let b: i32x2 = i32x2::new(1, 2); + let e: i32x2 = i32x2::new(21, 22); + let r: i32x2 = transmute(vhadd_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhaddq_s32() { + let a: i32x4 = i32x4::new(42, 42, 42, 42); + let b: i32x4 = i32x4::new(1, 2, 3, 4); + let e: i32x4 = i32x4::new(21, 22, 22, 23); + let r: i32x4 = transmute(vhaddq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhadd_u8() { + let a: u8x8 = u8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(22, 22, 23, 23, 24, 24, 25, 25); + let r: u8x8 = transmute(vrhadd_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhaddq_u8() { + let a: u8x16 = u8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); + let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29, 29); + let r: u8x16 = transmute(vrhaddq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhadd_u16() { + let a: u16x4 = u16x4::new(42, 42, 42, 42); + let b: u16x4 = u16x4::new(1, 2, 3, 4); + let e: u16x4 = u16x4::new(22, 22, 23, 23); + let r: u16x4 = transmute(vrhadd_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhaddq_u16() { + let a: u16x8 = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(22, 22, 23, 23, 24, 24, 25, 25); + let r: u16x8 = transmute(vrhaddq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhadd_u32() { + let a: u32x2 = u32x2::new(42, 42); + let b: u32x2 = u32x2::new(1, 2); + let e: u32x2 = u32x2::new(22, 22); + let r: u32x2 = transmute(vrhadd_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhaddq_u32() { + let a: u32x4 = u32x4::new(42, 42, 42, 42); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(22, 22, 23, 23); + let r: u32x4 = transmute(vrhaddq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhadd_s8() { + let a: i8x8 = i8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i8x8 = i8x8::new(22, 22, 23, 23, 24, 24, 25, 25); + let r: i8x8 = transmute(vrhadd_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhaddq_s8() { + let a: i8x16 = i8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); + let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29, 29); + let r: i8x16 = transmute(vrhaddq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhadd_s16() { + let a: i16x4 = i16x4::new(42, 42, 42, 42); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let e: i16x4 = i16x4::new(22, 22, 23, 23); + let r: i16x4 = transmute(vrhadd_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhaddq_s16() { + let a: i16x8 = i16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i16x8 = i16x8::new(22, 22, 23, 23, 24, 24, 25, 25); + let r: i16x8 = transmute(vrhaddq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhadd_s32() { + let a: i32x2 = i32x2::new(42, 42); + let b: i32x2 = i32x2::new(1, 2); + let e: i32x2 = i32x2::new(22, 22); + let r: i32x2 = transmute(vrhadd_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhaddq_s32() { + let a: i32x4 = i32x4::new(42, 42, 42, 42); + let b: i32x4 = i32x4::new(1, 2, 3, 4); + let e: i32x4 = i32x4::new(22, 22, 23, 23); + let r: i32x4 = transmute(vrhaddq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndn_f32() { + let a: f32x2 = f32x2::new(-1.5, 0.5); + let e: f32x2 = f32x2::new(-2.0, 0.0); + let r: f32x2 = transmute(vrndn_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndnq_f32() { + let a: f32x4 = f32x4::new(-1.5, 0.5, 1.5, 2.5); + let e: f32x4 = f32x4::new(-2.0, 0.0, 2.0, 2.0); + let r: f32x4 = transmute(vrndnq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_u8() { + let a: u8x8 = u8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(43, 44, 45, 46, 47, 48, 49, 50); + let r: u8x8 = transmute(vqadd_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_u8() { + let a: u8x16 = u8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); + let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58); + let r: u8x16 = transmute(vqaddq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_u16() { + let a: u16x4 = u16x4::new(42, 42, 42, 42); + let b: u16x4 = u16x4::new(1, 2, 3, 4); + let e: u16x4 = u16x4::new(43, 44, 45, 46); + let r: u16x4 = transmute(vqadd_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_u16() { + let a: u16x8 = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(43, 44, 45, 46, 47, 48, 49, 50); + let r: u16x8 = transmute(vqaddq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_u32() { + let a: u32x2 = u32x2::new(42, 42); + let b: u32x2 = u32x2::new(1, 2); + let e: u32x2 = u32x2::new(43, 44); + let r: u32x2 = transmute(vqadd_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_u32() { + let a: u32x4 = u32x4::new(42, 42, 42, 42); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(43, 44, 45, 46); + let r: u32x4 = transmute(vqaddq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_u64() { + let a: u64x1 = u64x1::new(42); + let b: u64x1 = u64x1::new(1); + let e: u64x1 = u64x1::new(43); + let r: u64x1 = transmute(vqadd_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_u64() { + let a: u64x2 = u64x2::new(42, 42); + let b: u64x2 = u64x2::new(1, 2); + let e: u64x2 = u64x2::new(43, 44); + let r: u64x2 = transmute(vqaddq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_s8() { + let a: i8x8 = i8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i8x8 = i8x8::new(43, 44, 45, 46, 47, 48, 49, 50); + let r: i8x8 = transmute(vqadd_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_s8() { + let a: i8x16 = i8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); + let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58); + let r: i8x16 = transmute(vqaddq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_s16() { + let a: i16x4 = i16x4::new(42, 42, 42, 42); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let e: i16x4 = i16x4::new(43, 44, 45, 46); + let r: i16x4 = transmute(vqadd_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_s16() { + let a: i16x8 = i16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i16x8 = i16x8::new(43, 44, 45, 46, 47, 48, 49, 50); + let r: i16x8 = transmute(vqaddq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_s32() { + let a: i32x2 = i32x2::new(42, 42); + let b: i32x2 = i32x2::new(1, 2); + let e: i32x2 = i32x2::new(43, 44); + let r: i32x2 = transmute(vqadd_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_s32() { + let a: i32x4 = i32x4::new(42, 42, 42, 42); + let b: i32x4 = i32x4::new(1, 2, 3, 4); + let e: i32x4 = i32x4::new(43, 44, 45, 46); + let r: i32x4 = transmute(vqaddq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_s64() { + let a: i64x1 = i64x1::new(42); + let b: i64x1 = i64x1::new(1); + let e: i64x1 = i64x1::new(43); + let r: i64x1 = transmute(vqadd_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_s64() { + let a: i64x2 = i64x2::new(42, 42); + let b: i64x2 = i64x2::new(1, 2); + let e: i64x2 = i64x2::new(43, 44); + let r: i64x2 = transmute(vqaddq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_s8_x2() { + let a: [i8; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [i8x8; 2] = [i8x8::new(1, 2, 3, 4, 5, 6, 7, 8), i8x8::new(9, 10, 11, 12, 13, 14, 15, 16)]; + let r: [i8x8; 2] = transmute(vld1_s8_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_s16_x2() { + let a: [i16; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [i16x4; 2] = [i16x4::new(1, 2, 3, 4), i16x4::new(5, 6, 7, 8)]; + let r: [i16x4; 2] = transmute(vld1_s16_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_s32_x2() { + let a: [i32; 5] = [0, 1, 2, 3, 4]; + let e: [i32x2; 2] = [i32x2::new(1, 2), i32x2::new(3, 4)]; + let r: [i32x2; 2] = transmute(vld1_s32_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_s64_x2() { + let a: [i64; 3] = [0, 1, 2]; + let e: [i64x1; 2] = [i64x1::new(1), i64x1::new(2)]; + let r: [i64x1; 2] = transmute(vld1_s64_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_s8_x2() { + let a: [i8; 33] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [i8x16; 2] = [i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), i8x16::new(17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32)]; + let r: [i8x16; 2] = transmute(vld1q_s8_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_s16_x2() { + let a: [i16; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [i16x8; 2] = [i16x8::new(1, 2, 3, 4, 5, 6, 7, 8), i16x8::new(9, 10, 11, 12, 13, 14, 15, 16)]; + let r: [i16x8; 2] = transmute(vld1q_s16_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_s32_x2() { + let a: [i32; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [i32x4; 2] = [i32x4::new(1, 2, 3, 4), i32x4::new(5, 6, 7, 8)]; + let r: [i32x4; 2] = transmute(vld1q_s32_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_s64_x2() { + let a: [i64; 5] = [0, 1, 2, 3, 4]; + let e: [i64x2; 2] = [i64x2::new(1, 2), i64x2::new(3, 4)]; + let r: [i64x2; 2] = transmute(vld1q_s64_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_s8_x3() { + let a: [i8; 25] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]; + let e: [i8x8; 3] = [i8x8::new(1, 2, 3, 4, 5, 6, 7, 8), i8x8::new(9, 10, 11, 12, 13, 14, 15, 16), i8x8::new(17, 18, 19, 20, 21, 22, 23, 24)]; + let r: [i8x8; 3] = transmute(vld1_s8_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_s16_x3() { + let a: [i16; 13] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + let e: [i16x4; 3] = [i16x4::new(1, 2, 3, 4), i16x4::new(5, 6, 7, 8), i16x4::new(9, 10, 11, 12)]; + let r: [i16x4; 3] = transmute(vld1_s16_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_s32_x3() { + let a: [i32; 7] = [0, 1, 2, 3, 4, 5, 6]; + let e: [i32x2; 3] = [i32x2::new(1, 2), i32x2::new(3, 4), i32x2::new(5, 6)]; + let r: [i32x2; 3] = transmute(vld1_s32_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_s64_x3() { + let a: [i64; 4] = [0, 1, 2, 3]; + let e: [i64x1; 3] = [i64x1::new(1), i64x1::new(2), i64x1::new(3)]; + let r: [i64x1; 3] = transmute(vld1_s64_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_s8_x3() { + let a: [i8; 49] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [i8x16; 3] = [i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), i8x16::new(17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32), i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16)]; + let r: [i8x16; 3] = transmute(vld1q_s8_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_s16_x3() { + let a: [i16; 25] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]; + let e: [i16x8; 3] = [i16x8::new(1, 2, 3, 4, 5, 6, 7, 8), i16x8::new(9, 10, 11, 12, 13, 14, 15, 16), i16x8::new(17, 18, 19, 20, 21, 22, 23, 24)]; + let r: [i16x8; 3] = transmute(vld1q_s16_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_s32_x3() { + let a: [i32; 13] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + let e: [i32x4; 3] = [i32x4::new(1, 2, 3, 4), i32x4::new(5, 6, 7, 8), i32x4::new(9, 10, 11, 12)]; + let r: [i32x4; 3] = transmute(vld1q_s32_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_s64_x3() { + let a: [i64; 7] = [0, 1, 2, 3, 4, 5, 6]; + let e: [i64x2; 3] = [i64x2::new(1, 2), i64x2::new(3, 4), i64x2::new(5, 6)]; + let r: [i64x2; 3] = transmute(vld1q_s64_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_s8_x4() { + let a: [i8; 33] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [i8x8; 4] = [i8x8::new(1, 2, 3, 4, 5, 6, 7, 8), i8x8::new(9, 10, 11, 12, 13, 14, 15, 16), i8x8::new(17, 18, 19, 20, 21, 22, 23, 24), i8x8::new(25, 26, 27, 28, 29, 30, 31, 32)]; + let r: [i8x8; 4] = transmute(vld1_s8_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_s16_x4() { + let a: [i16; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [i16x4; 4] = [i16x4::new(1, 2, 3, 4), i16x4::new(5, 6, 7, 8), i16x4::new(9, 10, 11, 12), i16x4::new(13, 14, 15, 16)]; + let r: [i16x4; 4] = transmute(vld1_s16_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_s32_x4() { + let a: [i32; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [i32x2; 4] = [i32x2::new(1, 2), i32x2::new(3, 4), i32x2::new(5, 6), i32x2::new(7, 8)]; + let r: [i32x2; 4] = transmute(vld1_s32_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_s64_x4() { + let a: [i64; 5] = [0, 1, 2, 3, 4]; + let e: [i64x1; 4] = [i64x1::new(1), i64x1::new(2), i64x1::new(3), i64x1::new(4)]; + let r: [i64x1; 4] = transmute(vld1_s64_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_s8_x4() { + let a: [i8; 65] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [i8x16; 4] = [i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), i8x16::new(17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32), i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), i8x16::new(17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32)]; + let r: [i8x16; 4] = transmute(vld1q_s8_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_s16_x4() { + let a: [i16; 33] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [i16x8; 4] = [i16x8::new(1, 2, 3, 4, 5, 6, 7, 8), i16x8::new(9, 10, 11, 12, 13, 14, 15, 16), i16x8::new(17, 18, 19, 20, 21, 22, 23, 24), i16x8::new(25, 26, 27, 28, 29, 30, 31, 32)]; + let r: [i16x8; 4] = transmute(vld1q_s16_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_s32_x4() { + let a: [i32; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [i32x4; 4] = [i32x4::new(1, 2, 3, 4), i32x4::new(5, 6, 7, 8), i32x4::new(9, 10, 11, 12), i32x4::new(13, 14, 15, 16)]; + let r: [i32x4; 4] = transmute(vld1q_s32_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_s64_x4() { + let a: [i64; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [i64x2; 4] = [i64x2::new(1, 2), i64x2::new(3, 4), i64x2::new(5, 6), i64x2::new(7, 8)]; + let r: [i64x2; 4] = transmute(vld1q_s64_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_u8_x2() { + let a: [u8; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [u8x8; 2] = [u8x8::new(1, 2, 3, 4, 5, 6, 7, 8), u8x8::new(9, 10, 11, 12, 13, 14, 15, 16)]; + let r: [u8x8; 2] = transmute(vld1_u8_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_u16_x2() { + let a: [u16; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [u16x4; 2] = [u16x4::new(1, 2, 3, 4), u16x4::new(5, 6, 7, 8)]; + let r: [u16x4; 2] = transmute(vld1_u16_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_u32_x2() { + let a: [u32; 5] = [0, 1, 2, 3, 4]; + let e: [u32x2; 2] = [u32x2::new(1, 2), u32x2::new(3, 4)]; + let r: [u32x2; 2] = transmute(vld1_u32_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_u64_x2() { + let a: [u64; 3] = [0, 1, 2]; + let e: [u64x1; 2] = [u64x1::new(1), u64x1::new(2)]; + let r: [u64x1; 2] = transmute(vld1_u64_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_u8_x2() { + let a: [u8; 33] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [u8x16; 2] = [u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), u8x16::new(17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32)]; + let r: [u8x16; 2] = transmute(vld1q_u8_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_u16_x2() { + let a: [u16; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [u16x8; 2] = [u16x8::new(1, 2, 3, 4, 5, 6, 7, 8), u16x8::new(9, 10, 11, 12, 13, 14, 15, 16)]; + let r: [u16x8; 2] = transmute(vld1q_u16_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_u32_x2() { + let a: [u32; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [u32x4; 2] = [u32x4::new(1, 2, 3, 4), u32x4::new(5, 6, 7, 8)]; + let r: [u32x4; 2] = transmute(vld1q_u32_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_u64_x2() { + let a: [u64; 5] = [0, 1, 2, 3, 4]; + let e: [u64x2; 2] = [u64x2::new(1, 2), u64x2::new(3, 4)]; + let r: [u64x2; 2] = transmute(vld1q_u64_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_u8_x3() { + let a: [u8; 25] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]; + let e: [u8x8; 3] = [u8x8::new(1, 2, 3, 4, 5, 6, 7, 8), u8x8::new(9, 10, 11, 12, 13, 14, 15, 16), u8x8::new(17, 18, 19, 20, 21, 22, 23, 24)]; + let r: [u8x8; 3] = transmute(vld1_u8_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_u16_x3() { + let a: [u16; 13] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + let e: [u16x4; 3] = [u16x4::new(1, 2, 3, 4), u16x4::new(5, 6, 7, 8), u16x4::new(9, 10, 11, 12)]; + let r: [u16x4; 3] = transmute(vld1_u16_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_u32_x3() { + let a: [u32; 7] = [0, 1, 2, 3, 4, 5, 6]; + let e: [u32x2; 3] = [u32x2::new(1, 2), u32x2::new(3, 4), u32x2::new(5, 6)]; + let r: [u32x2; 3] = transmute(vld1_u32_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_u64_x3() { + let a: [u64; 4] = [0, 1, 2, 3]; + let e: [u64x1; 3] = [u64x1::new(1), u64x1::new(2), u64x1::new(3)]; + let r: [u64x1; 3] = transmute(vld1_u64_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_u8_x3() { + let a: [u8; 49] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [u8x16; 3] = [u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), u8x16::new(17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32), u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16)]; + let r: [u8x16; 3] = transmute(vld1q_u8_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_u16_x3() { + let a: [u16; 25] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]; + let e: [u16x8; 3] = [u16x8::new(1, 2, 3, 4, 5, 6, 7, 8), u16x8::new(9, 10, 11, 12, 13, 14, 15, 16), u16x8::new(17, 18, 19, 20, 21, 22, 23, 24)]; + let r: [u16x8; 3] = transmute(vld1q_u16_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_u32_x3() { + let a: [u32; 13] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + let e: [u32x4; 3] = [u32x4::new(1, 2, 3, 4), u32x4::new(5, 6, 7, 8), u32x4::new(9, 10, 11, 12)]; + let r: [u32x4; 3] = transmute(vld1q_u32_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_u64_x3() { + let a: [u64; 7] = [0, 1, 2, 3, 4, 5, 6]; + let e: [u64x2; 3] = [u64x2::new(1, 2), u64x2::new(3, 4), u64x2::new(5, 6)]; + let r: [u64x2; 3] = transmute(vld1q_u64_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_u8_x4() { + let a: [u8; 33] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [u8x8; 4] = [u8x8::new(1, 2, 3, 4, 5, 6, 7, 8), u8x8::new(9, 10, 11, 12, 13, 14, 15, 16), u8x8::new(17, 18, 19, 20, 21, 22, 23, 24), u8x8::new(25, 26, 27, 28, 29, 30, 31, 32)]; + let r: [u8x8; 4] = transmute(vld1_u8_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_u16_x4() { + let a: [u16; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [u16x4; 4] = [u16x4::new(1, 2, 3, 4), u16x4::new(5, 6, 7, 8), u16x4::new(9, 10, 11, 12), u16x4::new(13, 14, 15, 16)]; + let r: [u16x4; 4] = transmute(vld1_u16_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_u32_x4() { + let a: [u32; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [u32x2; 4] = [u32x2::new(1, 2), u32x2::new(3, 4), u32x2::new(5, 6), u32x2::new(7, 8)]; + let r: [u32x2; 4] = transmute(vld1_u32_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_u64_x4() { + let a: [u64; 5] = [0, 1, 2, 3, 4]; + let e: [u64x1; 4] = [u64x1::new(1), u64x1::new(2), u64x1::new(3), u64x1::new(4)]; + let r: [u64x1; 4] = transmute(vld1_u64_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_u8_x4() { + let a: [u8; 65] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [u8x16; 4] = [u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), u8x16::new(17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32), u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), u8x16::new(17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32)]; + let r: [u8x16; 4] = transmute(vld1q_u8_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_u16_x4() { + let a: [u16; 33] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [u16x8; 4] = [u16x8::new(1, 2, 3, 4, 5, 6, 7, 8), u16x8::new(9, 10, 11, 12, 13, 14, 15, 16), u16x8::new(17, 18, 19, 20, 21, 22, 23, 24), u16x8::new(25, 26, 27, 28, 29, 30, 31, 32)]; + let r: [u16x8; 4] = transmute(vld1q_u16_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_u32_x4() { + let a: [u32; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [u32x4; 4] = [u32x4::new(1, 2, 3, 4), u32x4::new(5, 6, 7, 8), u32x4::new(9, 10, 11, 12), u32x4::new(13, 14, 15, 16)]; + let r: [u32x4; 4] = transmute(vld1q_u32_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_u64_x4() { + let a: [u64; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [u64x2; 4] = [u64x2::new(1, 2), u64x2::new(3, 4), u64x2::new(5, 6), u64x2::new(7, 8)]; + let r: [u64x2; 4] = transmute(vld1q_u64_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_p8_x2() { + let a: [u8; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [i8x8; 2] = [i8x8::new(1, 2, 3, 4, 5, 6, 7, 8), i8x8::new(9, 10, 11, 12, 13, 14, 15, 16)]; + let r: [i8x8; 2] = transmute(vld1_p8_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_p8_x3() { + let a: [u8; 25] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]; + let e: [i8x8; 3] = [i8x8::new(1, 2, 3, 4, 5, 6, 7, 8), i8x8::new(9, 10, 11, 12, 13, 14, 15, 16), i8x8::new(17, 18, 19, 20, 21, 22, 23, 24)]; + let r: [i8x8; 3] = transmute(vld1_p8_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_p8_x4() { + let a: [u8; 33] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [i8x8; 4] = [i8x8::new(1, 2, 3, 4, 5, 6, 7, 8), i8x8::new(9, 10, 11, 12, 13, 14, 15, 16), i8x8::new(17, 18, 19, 20, 21, 22, 23, 24), i8x8::new(25, 26, 27, 28, 29, 30, 31, 32)]; + let r: [i8x8; 4] = transmute(vld1_p8_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_p8_x2() { + let a: [u8; 33] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [i8x16; 2] = [i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), i8x16::new(17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32)]; + let r: [i8x16; 2] = transmute(vld1q_p8_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_p8_x3() { + let a: [u8; 49] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [i8x16; 3] = [i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), i8x16::new(17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32), i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16)]; + let r: [i8x16; 3] = transmute(vld1q_p8_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_p8_x4() { + let a: [u8; 65] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [i8x16; 4] = [i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), i8x16::new(17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32), i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), i8x16::new(17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32)]; + let r: [i8x16; 4] = transmute(vld1q_p8_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_p16_x2() { + let a: [u16; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [i16x4; 2] = [i16x4::new(1, 2, 3, 4), i16x4::new(5, 6, 7, 8)]; + let r: [i16x4; 2] = transmute(vld1_p16_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_p16_x3() { + let a: [u16; 13] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + let e: [i16x4; 3] = [i16x4::new(1, 2, 3, 4), i16x4::new(5, 6, 7, 8), i16x4::new(9, 10, 11, 12)]; + let r: [i16x4; 3] = transmute(vld1_p16_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_p16_x4() { + let a: [u16; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [i16x4; 4] = [i16x4::new(1, 2, 3, 4), i16x4::new(5, 6, 7, 8), i16x4::new(9, 10, 11, 12), i16x4::new(13, 14, 15, 16)]; + let r: [i16x4; 4] = transmute(vld1_p16_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_p16_x2() { + let a: [u16; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [i16x8; 2] = [i16x8::new(1, 2, 3, 4, 5, 6, 7, 8), i16x8::new(9, 10, 11, 12, 13, 14, 15, 16)]; + let r: [i16x8; 2] = transmute(vld1q_p16_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_p16_x3() { + let a: [u16; 25] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]; + let e: [i16x8; 3] = [i16x8::new(1, 2, 3, 4, 5, 6, 7, 8), i16x8::new(9, 10, 11, 12, 13, 14, 15, 16), i16x8::new(17, 18, 19, 20, 21, 22, 23, 24)]; + let r: [i16x8; 3] = transmute(vld1q_p16_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_p16_x4() { + let a: [u16; 33] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [i16x8; 4] = [i16x8::new(1, 2, 3, 4, 5, 6, 7, 8), i16x8::new(9, 10, 11, 12, 13, 14, 15, 16), i16x8::new(17, 18, 19, 20, 21, 22, 23, 24), i16x8::new(25, 26, 27, 28, 29, 30, 31, 32)]; + let r: [i16x8; 4] = transmute(vld1q_p16_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_p64_x2() { + let a: [u64; 3] = [0, 1, 2]; + let e: [i64x1; 2] = [i64x1::new(1), i64x1::new(2)]; + let r: [i64x1; 2] = transmute(vld1_p64_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_p64_x3() { + let a: [u64; 4] = [0, 1, 2, 3]; + let e: [i64x1; 3] = [i64x1::new(1), i64x1::new(2), i64x1::new(3)]; + let r: [i64x1; 3] = transmute(vld1_p64_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_p64_x4() { + let a: [u64; 5] = [0, 1, 2, 3, 4]; + let e: [i64x1; 4] = [i64x1::new(1), i64x1::new(2), i64x1::new(3), i64x1::new(4)]; + let r: [i64x1; 4] = transmute(vld1_p64_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_p64_x2() { + let a: [u64; 5] = [0, 1, 2, 3, 4]; + let e: [i64x2; 2] = [i64x2::new(1, 2), i64x2::new(3, 4)]; + let r: [i64x2; 2] = transmute(vld1q_p64_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_p64_x3() { + let a: [u64; 7] = [0, 1, 2, 3, 4, 5, 6]; + let e: [i64x2; 3] = [i64x2::new(1, 2), i64x2::new(3, 4), i64x2::new(5, 6)]; + let r: [i64x2; 3] = transmute(vld1q_p64_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_p64_x4() { + let a: [u64; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [i64x2; 4] = [i64x2::new(1, 2), i64x2::new(3, 4), i64x2::new(5, 6), i64x2::new(7, 8)]; + let r: [i64x2; 4] = transmute(vld1q_p64_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_f32_x2() { + let a: [f32; 5] = [0., 1., 2., 3., 4.]; + let e: [f32x2; 2] = [f32x2::new(1., 2.), f32x2::new(3., 4.)]; + let r: [f32x2; 2] = transmute(vld1_f32_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_f32_x2() { + let a: [f32; 9] = [0., 1., 2., 3., 4., 5., 6., 7., 8.]; + let e: [f32x4; 2] = [f32x4::new(1., 2., 3., 4.), f32x4::new(5., 6., 7., 8.)]; + let r: [f32x4; 2] = transmute(vld1q_f32_x2(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_f32_x3() { + let a: [f32; 7] = [0., 1., 2., 3., 4., 5., 6.]; + let e: [f32x2; 3] = [f32x2::new(1., 2.), f32x2::new(3., 4.), f32x2::new(5., 6.)]; + let r: [f32x2; 3] = transmute(vld1_f32_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_f32_x3() { + let a: [f32; 13] = [0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12.]; + let e: [f32x4; 3] = [f32x4::new(1., 2., 3., 4.), f32x4::new(5., 6., 7., 8.), f32x4::new(9., 10., 11., 12.)]; + let r: [f32x4; 3] = transmute(vld1q_f32_x3(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_f32_x4() { + let a: [f32; 9] = [0., 1., 2., 3., 4., 5., 6., 7., 8.]; + let e: [f32x2; 4] = [f32x2::new(1., 2.), f32x2::new(3., 4.), f32x2::new(5., 6.), f32x2::new(7., 8.)]; + let r: [f32x2; 4] = transmute(vld1_f32_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_f32_x4() { + let a: [f32; 17] = [0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16.]; + let e: [f32x4; 4] = [f32x4::new(1., 2., 3., 4.), f32x4::new(5., 6., 7., 8.), f32x4::new(9., 10., 11., 12.), f32x4::new(13., 14., 15., 16.)]; + let r: [f32x4; 4] = transmute(vld1q_f32_x4(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_s8() { + let a: [i8; 17] = [0, 1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9]; + let e: [i8x8; 2] = [i8x8::new(1, 2, 2, 3, 2, 3, 4, 5), i8x8::new(2, 3, 4, 5, 6, 7, 8, 9)]; + let r: [i8x8; 2] = transmute(vld2_s8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_s16() { + let a: [i16; 9] = [0, 1, 2, 2, 3, 2, 4, 3, 5]; + let e: [i16x4; 2] = [i16x4::new(1, 2, 2, 3), i16x4::new(2, 3, 4, 5)]; + let r: [i16x4; 2] = transmute(vld2_s16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_s32() { + let a: [i32; 5] = [0, 1, 2, 2, 3]; + let e: [i32x2; 2] = [i32x2::new(1, 2), i32x2::new(2, 3)]; + let r: [i32x2; 2] = transmute(vld2_s32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_s8() { + let a: [i8; 33] = [0, 1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15, 8, 16, 9, 17]; + let e: [i8x16; 2] = [i8x16::new(1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9), i8x16::new(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17)]; + let r: [i8x16; 2] = transmute(vld2q_s8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_s16() { + let a: [i16; 17] = [0, 1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9]; + let e: [i16x8; 2] = [i16x8::new(1, 2, 2, 3, 2, 3, 4, 5), i16x8::new(2, 3, 4, 5, 6, 7, 8, 9)]; + let r: [i16x8; 2] = transmute(vld2q_s16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_s32() { + let a: [i32; 9] = [0, 1, 2, 2, 3, 2, 4, 3, 5]; + let e: [i32x4; 2] = [i32x4::new(1, 2, 2, 3), i32x4::new(2, 3, 4, 5)]; + let r: [i32x4; 2] = transmute(vld2q_s32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_s64() { + let a: [i64; 3] = [0, 1, 2]; + let e: [i64x1; 2] = [i64x1::new(1), i64x1::new(2)]; + let r: [i64x1; 2] = transmute(vld2_s64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_u8() { + let a: [u8; 17] = [0, 1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9]; + let e: [u8x8; 2] = [u8x8::new(1, 2, 2, 3, 2, 3, 4, 5), u8x8::new(2, 3, 4, 5, 6, 7, 8, 9)]; + let r: [u8x8; 2] = transmute(vld2_u8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_u16() { + let a: [u16; 9] = [0, 1, 2, 2, 3, 2, 4, 3, 5]; + let e: [u16x4; 2] = [u16x4::new(1, 2, 2, 3), u16x4::new(2, 3, 4, 5)]; + let r: [u16x4; 2] = transmute(vld2_u16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_u32() { + let a: [u32; 5] = [0, 1, 2, 2, 3]; + let e: [u32x2; 2] = [u32x2::new(1, 2), u32x2::new(2, 3)]; + let r: [u32x2; 2] = transmute(vld2_u32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_u8() { + let a: [u8; 33] = [0, 1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15, 8, 16, 9, 17]; + let e: [u8x16; 2] = [u8x16::new(1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9), u8x16::new(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17)]; + let r: [u8x16; 2] = transmute(vld2q_u8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_u16() { + let a: [u16; 17] = [0, 1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9]; + let e: [u16x8; 2] = [u16x8::new(1, 2, 2, 3, 2, 3, 4, 5), u16x8::new(2, 3, 4, 5, 6, 7, 8, 9)]; + let r: [u16x8; 2] = transmute(vld2q_u16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_u32() { + let a: [u32; 9] = [0, 1, 2, 2, 3, 2, 4, 3, 5]; + let e: [u32x4; 2] = [u32x4::new(1, 2, 2, 3), u32x4::new(2, 3, 4, 5)]; + let r: [u32x4; 2] = transmute(vld2q_u32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_p8() { + let a: [u8; 17] = [0, 1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9]; + let e: [i8x8; 2] = [i8x8::new(1, 2, 2, 3, 2, 3, 4, 5), i8x8::new(2, 3, 4, 5, 6, 7, 8, 9)]; + let r: [i8x8; 2] = transmute(vld2_p8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_p16() { + let a: [u16; 9] = [0, 1, 2, 2, 3, 2, 4, 3, 5]; + let e: [i16x4; 2] = [i16x4::new(1, 2, 2, 3), i16x4::new(2, 3, 4, 5)]; + let r: [i16x4; 2] = transmute(vld2_p16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_p8() { + let a: [u8; 33] = [0, 1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15, 8, 16, 9, 17]; + let e: [i8x16; 2] = [i8x16::new(1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9), i8x16::new(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17)]; + let r: [i8x16; 2] = transmute(vld2q_p8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_p16() { + let a: [u16; 17] = [0, 1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9]; + let e: [i16x8; 2] = [i16x8::new(1, 2, 2, 3, 2, 3, 4, 5), i16x8::new(2, 3, 4, 5, 6, 7, 8, 9)]; + let r: [i16x8; 2] = transmute(vld2q_p16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_u64() { + let a: [u64; 3] = [0, 1, 2]; + let e: [u64x1; 2] = [u64x1::new(1), u64x1::new(2)]; + let r: [u64x1; 2] = transmute(vld2_u64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_p64() { + let a: [u64; 3] = [0, 1, 2]; + let e: [i64x1; 2] = [i64x1::new(1), i64x1::new(2)]; + let r: [i64x1; 2] = transmute(vld2_p64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_f32() { + let a: [f32; 5] = [0., 1., 2., 2., 3.]; + let e: [f32x2; 2] = [f32x2::new(1., 2.), f32x2::new(2., 3.)]; + let r: [f32x2; 2] = transmute(vld2_f32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_f32() { + let a: [f32; 9] = [0., 1., 2., 2., 3., 2., 4., 3., 5.]; + let e: [f32x4; 2] = [f32x4::new(1., 2., 2., 3.), f32x4::new(2., 3., 4., 5.)]; + let r: [f32x4; 2] = transmute(vld2q_f32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_dup_s8() { + let a: [i8; 17] = [0, 1, 1, 2, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9]; + let e: [i8x8; 2] = [i8x8::new(1, 1, 1, 1, 1, 1, 1, 1), i8x8::new(1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [i8x8; 2] = transmute(vld2_dup_s8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_dup_s16() { + let a: [i16; 9] = [0, 1, 1, 2, 3, 1, 4, 3, 5]; + let e: [i16x4; 2] = [i16x4::new(1, 1, 1, 1), i16x4::new(1, 1, 1, 1)]; + let r: [i16x4; 2] = transmute(vld2_dup_s16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_dup_s32() { + let a: [i32; 5] = [0, 1, 1, 2, 3]; + let e: [i32x2; 2] = [i32x2::new(1, 1), i32x2::new(1, 1)]; + let r: [i32x2; 2] = transmute(vld2_dup_s32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_dup_s8() { + let a: [i8; 33] = [0, 1, 1, 2, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15, 8, 16, 9, 17]; + let e: [i8x16; 2] = [i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [i8x16; 2] = transmute(vld2q_dup_s8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_dup_s16() { + let a: [i16; 17] = [0, 1, 1, 2, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9]; + let e: [i16x8; 2] = [i16x8::new(1, 1, 1, 1, 1, 1, 1, 1), i16x8::new(1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [i16x8; 2] = transmute(vld2q_dup_s16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_dup_s32() { + let a: [i32; 9] = [0, 1, 1, 2, 3, 1, 4, 3, 5]; + let e: [i32x4; 2] = [i32x4::new(1, 1, 1, 1), i32x4::new(1, 1, 1, 1)]; + let r: [i32x4; 2] = transmute(vld2q_dup_s32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_dup_s64() { + let a: [i64; 3] = [0, 1, 1]; + let e: [i64x1; 2] = [i64x1::new(1), i64x1::new(1)]; + let r: [i64x1; 2] = transmute(vld2_dup_s64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_dup_u8() { + let a: [u8; 17] = [0, 1, 1, 2, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9]; + let e: [u8x8; 2] = [u8x8::new(1, 1, 1, 1, 1, 1, 1, 1), u8x8::new(1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [u8x8; 2] = transmute(vld2_dup_u8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_dup_u16() { + let a: [u16; 9] = [0, 1, 1, 2, 3, 1, 4, 3, 5]; + let e: [u16x4; 2] = [u16x4::new(1, 1, 1, 1), u16x4::new(1, 1, 1, 1)]; + let r: [u16x4; 2] = transmute(vld2_dup_u16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_dup_u32() { + let a: [u32; 5] = [0, 1, 1, 2, 3]; + let e: [u32x2; 2] = [u32x2::new(1, 1), u32x2::new(1, 1)]; + let r: [u32x2; 2] = transmute(vld2_dup_u32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_dup_u8() { + let a: [u8; 33] = [0, 1, 1, 2, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15, 8, 16, 9, 17]; + let e: [u8x16; 2] = [u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [u8x16; 2] = transmute(vld2q_dup_u8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_dup_u16() { + let a: [u16; 17] = [0, 1, 1, 2, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9]; + let e: [u16x8; 2] = [u16x8::new(1, 1, 1, 1, 1, 1, 1, 1), u16x8::new(1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [u16x8; 2] = transmute(vld2q_dup_u16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_dup_u32() { + let a: [u32; 9] = [0, 1, 1, 2, 3, 1, 4, 3, 5]; + let e: [u32x4; 2] = [u32x4::new(1, 1, 1, 1), u32x4::new(1, 1, 1, 1)]; + let r: [u32x4; 2] = transmute(vld2q_dup_u32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_dup_p8() { + let a: [u8; 17] = [0, 1, 1, 2, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9]; + let e: [i8x8; 2] = [i8x8::new(1, 1, 1, 1, 1, 1, 1, 1), i8x8::new(1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [i8x8; 2] = transmute(vld2_dup_p8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_dup_p16() { + let a: [u16; 9] = [0, 1, 1, 2, 3, 1, 4, 3, 5]; + let e: [i16x4; 2] = [i16x4::new(1, 1, 1, 1), i16x4::new(1, 1, 1, 1)]; + let r: [i16x4; 2] = transmute(vld2_dup_p16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_dup_p8() { + let a: [u8; 33] = [0, 1, 1, 2, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15, 8, 16, 9, 17]; + let e: [i8x16; 2] = [i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [i8x16; 2] = transmute(vld2q_dup_p8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_dup_p16() { + let a: [u16; 17] = [0, 1, 1, 2, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9]; + let e: [i16x8; 2] = [i16x8::new(1, 1, 1, 1, 1, 1, 1, 1), i16x8::new(1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [i16x8; 2] = transmute(vld2q_dup_p16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_dup_u64() { + let a: [u64; 3] = [0, 1, 1]; + let e: [u64x1; 2] = [u64x1::new(1), u64x1::new(1)]; + let r: [u64x1; 2] = transmute(vld2_dup_u64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_dup_p64() { + let a: [u64; 3] = [0, 1, 1]; + let e: [i64x1; 2] = [i64x1::new(1), i64x1::new(1)]; + let r: [i64x1; 2] = transmute(vld2_dup_p64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_dup_f32() { + let a: [f32; 5] = [0., 1., 1., 2., 3.]; + let e: [f32x2; 2] = [f32x2::new(1., 1.), f32x2::new(1., 1.)]; + let r: [f32x2; 2] = transmute(vld2_dup_f32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_dup_f32() { + let a: [f32; 9] = [0., 1., 1., 2., 3., 1., 4., 3., 5.]; + let e: [f32x4; 2] = [f32x4::new(1., 1., 1., 1.), f32x4::new(1., 1., 1., 1.)]; + let r: [f32x4; 2] = transmute(vld2q_dup_f32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_lane_s8() { + let a: [i8; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i8x8; 2] = [i8x8::new(0, 2, 2, 14, 2, 16, 17, 18), i8x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let e: [i8x8; 2] = [i8x8::new(1, 2, 2, 14, 2, 16, 17, 18), i8x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let r: [i8x8; 2] = transmute(vld2_lane_s8::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_lane_s16() { + let a: [i16; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i16x4; 2] = [i16x4::new(0, 2, 2, 14), i16x4::new(2, 16, 17, 18)]; + let e: [i16x4; 2] = [i16x4::new(1, 2, 2, 14), i16x4::new(2, 16, 17, 18)]; + let r: [i16x4; 2] = transmute(vld2_lane_s16::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_lane_s32() { + let a: [i32; 5] = [0, 1, 2, 3, 4]; + let b: [i32x2; 2] = [i32x2::new(0, 2), i32x2::new(2, 14)]; + let e: [i32x2; 2] = [i32x2::new(1, 2), i32x2::new(2, 14)]; + let r: [i32x2; 2] = transmute(vld2_lane_s32::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_lane_s16() { + let a: [i16; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i16x8; 2] = [i16x8::new(0, 2, 2, 14, 2, 16, 17, 18), i16x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let e: [i16x8; 2] = [i16x8::new(1, 2, 2, 14, 2, 16, 17, 18), i16x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let r: [i16x8; 2] = transmute(vld2q_lane_s16::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_lane_s32() { + let a: [i32; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i32x4; 2] = [i32x4::new(0, 2, 2, 14), i32x4::new(2, 16, 17, 18)]; + let e: [i32x4; 2] = [i32x4::new(1, 2, 2, 14), i32x4::new(2, 16, 17, 18)]; + let r: [i32x4; 2] = transmute(vld2q_lane_s32::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_lane_u8() { + let a: [u8; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [u8x8; 2] = [u8x8::new(0, 2, 2, 14, 2, 16, 17, 18), u8x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let e: [u8x8; 2] = [u8x8::new(1, 2, 2, 14, 2, 16, 17, 18), u8x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let r: [u8x8; 2] = transmute(vld2_lane_u8::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_lane_u16() { + let a: [u16; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [u16x4; 2] = [u16x4::new(0, 2, 2, 14), u16x4::new(2, 16, 17, 18)]; + let e: [u16x4; 2] = [u16x4::new(1, 2, 2, 14), u16x4::new(2, 16, 17, 18)]; + let r: [u16x4; 2] = transmute(vld2_lane_u16::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_lane_u32() { + let a: [u32; 5] = [0, 1, 2, 3, 4]; + let b: [u32x2; 2] = [u32x2::new(0, 2), u32x2::new(2, 14)]; + let e: [u32x2; 2] = [u32x2::new(1, 2), u32x2::new(2, 14)]; + let r: [u32x2; 2] = transmute(vld2_lane_u32::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_lane_u16() { + let a: [u16; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [u16x8; 2] = [u16x8::new(0, 2, 2, 14, 2, 16, 17, 18), u16x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let e: [u16x8; 2] = [u16x8::new(1, 2, 2, 14, 2, 16, 17, 18), u16x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let r: [u16x8; 2] = transmute(vld2q_lane_u16::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_lane_u32() { + let a: [u32; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [u32x4; 2] = [u32x4::new(0, 2, 2, 14), u32x4::new(2, 16, 17, 18)]; + let e: [u32x4; 2] = [u32x4::new(1, 2, 2, 14), u32x4::new(2, 16, 17, 18)]; + let r: [u32x4; 2] = transmute(vld2q_lane_u32::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_lane_p8() { + let a: [u8; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i8x8; 2] = [i8x8::new(0, 2, 2, 14, 2, 16, 17, 18), i8x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let e: [i8x8; 2] = [i8x8::new(1, 2, 2, 14, 2, 16, 17, 18), i8x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let r: [i8x8; 2] = transmute(vld2_lane_p8::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_lane_p16() { + let a: [u16; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i16x4; 2] = [i16x4::new(0, 2, 2, 14), i16x4::new(2, 16, 17, 18)]; + let e: [i16x4; 2] = [i16x4::new(1, 2, 2, 14), i16x4::new(2, 16, 17, 18)]; + let r: [i16x4; 2] = transmute(vld2_lane_p16::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_lane_p16() { + let a: [u16; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i16x8; 2] = [i16x8::new(0, 2, 2, 14, 2, 16, 17, 18), i16x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let e: [i16x8; 2] = [i16x8::new(1, 2, 2, 14, 2, 16, 17, 18), i16x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let r: [i16x8; 2] = transmute(vld2q_lane_p16::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2_lane_f32() { + let a: [f32; 5] = [0., 1., 2., 3., 4.]; + let b: [f32x2; 2] = [f32x2::new(0., 2.), f32x2::new(2., 14.)]; + let e: [f32x2; 2] = [f32x2::new(1., 2.), f32x2::new(2., 14.)]; + let r: [f32x2; 2] = transmute(vld2_lane_f32::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld2q_lane_f32() { + let a: [f32; 9] = [0., 1., 2., 3., 4., 5., 6., 7., 8.]; + let b: [f32x4; 2] = [f32x4::new(0., 2., 2., 14.), f32x4::new(2., 16., 17., 18.)]; + let e: [f32x4; 2] = [f32x4::new(1., 2., 2., 14.), f32x4::new(2., 16., 17., 18.)]; + let r: [f32x4; 2] = transmute(vld2q_lane_f32::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_s8() { + let a: [i8; 25] = [0, 1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16]; + let e: [i8x8; 3] = [i8x8::new(1, 2, 2, 4, 2, 4, 7, 8), i8x8::new(2, 4, 7, 8, 13, 14, 15, 16), i8x8::new(2, 4, 7, 8, 13, 14, 15, 16)]; + let r: [i8x8; 3] = transmute(vld3_s8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_s16() { + let a: [i16; 13] = [0, 1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8]; + let e: [i16x4; 3] = [i16x4::new(1, 2, 2, 4), i16x4::new(2, 4, 7, 8), i16x4::new(2, 4, 7, 8)]; + let r: [i16x4; 3] = transmute(vld3_s16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_s32() { + let a: [i32; 7] = [0, 1, 2, 2, 2, 4, 4]; + let e: [i32x2; 3] = [i32x2::new(1, 2), i32x2::new(2, 4), i32x2::new(2, 4)]; + let r: [i32x2; 3] = transmute(vld3_s32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_s8() { + let a: [i8; 49] = [0, 1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16, 2, 25, 41, 4, 26, 42, 7, 27, 43, 8, 28, 44, 13, 29, 45, 14, 30, 46, 15, 31, 47, 16, 32, 48]; + let e: [i8x16; 3] = [i8x16::new(1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16), i8x16::new(2, 4, 7, 8, 13, 14, 15, 16, 25, 26, 27, 28, 29, 30, 31, 32), i8x16::new(2, 4, 7, 8, 13, 14, 15, 16, 41, 42, 43, 44, 45, 46, 47, 48)]; + let r: [i8x16; 3] = transmute(vld3q_s8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_s16() { + let a: [i16; 25] = [0, 1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16]; + let e: [i16x8; 3] = [i16x8::new(1, 2, 2, 4, 2, 4, 7, 8), i16x8::new(2, 4, 7, 8, 13, 14, 15, 16), i16x8::new(2, 4, 7, 8, 13, 14, 15, 16)]; + let r: [i16x8; 3] = transmute(vld3q_s16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_s32() { + let a: [i32; 13] = [0, 1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8]; + let e: [i32x4; 3] = [i32x4::new(1, 2, 2, 4), i32x4::new(2, 4, 7, 8), i32x4::new(2, 4, 7, 8)]; + let r: [i32x4; 3] = transmute(vld3q_s32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_s64() { + let a: [i64; 4] = [0, 1, 2, 2]; + let e: [i64x1; 3] = [i64x1::new(1), i64x1::new(2), i64x1::new(2)]; + let r: [i64x1; 3] = transmute(vld3_s64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_u8() { + let a: [u8; 25] = [0, 1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16]; + let e: [u8x8; 3] = [u8x8::new(1, 2, 2, 4, 2, 4, 7, 8), u8x8::new(2, 4, 7, 8, 13, 14, 15, 16), u8x8::new(2, 4, 7, 8, 13, 14, 15, 16)]; + let r: [u8x8; 3] = transmute(vld3_u8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_u16() { + let a: [u16; 13] = [0, 1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8]; + let e: [u16x4; 3] = [u16x4::new(1, 2, 2, 4), u16x4::new(2, 4, 7, 8), u16x4::new(2, 4, 7, 8)]; + let r: [u16x4; 3] = transmute(vld3_u16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_u32() { + let a: [u32; 7] = [0, 1, 2, 2, 2, 4, 4]; + let e: [u32x2; 3] = [u32x2::new(1, 2), u32x2::new(2, 4), u32x2::new(2, 4)]; + let r: [u32x2; 3] = transmute(vld3_u32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_u8() { + let a: [u8; 49] = [0, 1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16, 2, 25, 41, 4, 26, 42, 7, 27, 43, 8, 28, 44, 13, 29, 45, 14, 30, 46, 15, 31, 47, 16, 32, 48]; + let e: [u8x16; 3] = [u8x16::new(1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16), u8x16::new(2, 4, 7, 8, 13, 14, 15, 16, 25, 26, 27, 28, 29, 30, 31, 32), u8x16::new(2, 4, 7, 8, 13, 14, 15, 16, 41, 42, 43, 44, 45, 46, 47, 48)]; + let r: [u8x16; 3] = transmute(vld3q_u8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_u16() { + let a: [u16; 25] = [0, 1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16]; + let e: [u16x8; 3] = [u16x8::new(1, 2, 2, 4, 2, 4, 7, 8), u16x8::new(2, 4, 7, 8, 13, 14, 15, 16), u16x8::new(2, 4, 7, 8, 13, 14, 15, 16)]; + let r: [u16x8; 3] = transmute(vld3q_u16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_u32() { + let a: [u32; 13] = [0, 1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8]; + let e: [u32x4; 3] = [u32x4::new(1, 2, 2, 4), u32x4::new(2, 4, 7, 8), u32x4::new(2, 4, 7, 8)]; + let r: [u32x4; 3] = transmute(vld3q_u32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_p8() { + let a: [u8; 25] = [0, 1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16]; + let e: [i8x8; 3] = [i8x8::new(1, 2, 2, 4, 2, 4, 7, 8), i8x8::new(2, 4, 7, 8, 13, 14, 15, 16), i8x8::new(2, 4, 7, 8, 13, 14, 15, 16)]; + let r: [i8x8; 3] = transmute(vld3_p8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_p16() { + let a: [u16; 13] = [0, 1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8]; + let e: [i16x4; 3] = [i16x4::new(1, 2, 2, 4), i16x4::new(2, 4, 7, 8), i16x4::new(2, 4, 7, 8)]; + let r: [i16x4; 3] = transmute(vld3_p16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_p8() { + let a: [u8; 49] = [0, 1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16, 2, 25, 41, 4, 26, 42, 7, 27, 43, 8, 28, 44, 13, 29, 45, 14, 30, 46, 15, 31, 47, 16, 32, 48]; + let e: [i8x16; 3] = [i8x16::new(1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16), i8x16::new(2, 4, 7, 8, 13, 14, 15, 16, 25, 26, 27, 28, 29, 30, 31, 32), i8x16::new(2, 4, 7, 8, 13, 14, 15, 16, 41, 42, 43, 44, 45, 46, 47, 48)]; + let r: [i8x16; 3] = transmute(vld3q_p8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_p16() { + let a: [u16; 25] = [0, 1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16]; + let e: [i16x8; 3] = [i16x8::new(1, 2, 2, 4, 2, 4, 7, 8), i16x8::new(2, 4, 7, 8, 13, 14, 15, 16), i16x8::new(2, 4, 7, 8, 13, 14, 15, 16)]; + let r: [i16x8; 3] = transmute(vld3q_p16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_u64() { + let a: [u64; 4] = [0, 1, 2, 2]; + let e: [u64x1; 3] = [u64x1::new(1), u64x1::new(2), u64x1::new(2)]; + let r: [u64x1; 3] = transmute(vld3_u64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_p64() { + let a: [u64; 4] = [0, 1, 2, 2]; + let e: [i64x1; 3] = [i64x1::new(1), i64x1::new(2), i64x1::new(2)]; + let r: [i64x1; 3] = transmute(vld3_p64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_f32() { + let a: [f32; 7] = [0., 1., 2., 2., 2., 4., 4.]; + let e: [f32x2; 3] = [f32x2::new(1., 2.), f32x2::new(2., 4.), f32x2::new(2., 4.)]; + let r: [f32x2; 3] = transmute(vld3_f32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_f32() { + let a: [f32; 13] = [0., 1., 2., 2., 2., 4., 4., 2., 7., 7., 4., 8., 8.]; + let e: [f32x4; 3] = [f32x4::new(1., 2., 2., 4.), f32x4::new(2., 4., 7., 8.), f32x4::new(2., 4., 7., 8.)]; + let r: [f32x4; 3] = transmute(vld3q_f32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_dup_s8() { + let a: [i8; 25] = [0, 1, 1, 1, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13]; + let e: [i8x8; 3] = [i8x8::new(1, 1, 1, 1, 1, 1, 1, 1), i8x8::new(1, 1, 1, 1, 1, 1, 1, 1), i8x8::new(1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [i8x8; 3] = transmute(vld3_dup_s8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_dup_s16() { + let a: [i16; 13] = [0, 1, 1, 1, 3, 1, 4, 3, 5, 1, 6, 3, 7]; + let e: [i16x4; 3] = [i16x4::new(1, 1, 1, 1), i16x4::new(1, 1, 1, 1), i16x4::new(1, 1, 1, 1)]; + let r: [i16x4; 3] = transmute(vld3_dup_s16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_dup_s32() { + let a: [i32; 7] = [0, 1, 1, 1, 3, 1, 4]; + let e: [i32x2; 3] = [i32x2::new(1, 1), i32x2::new(1, 1), i32x2::new(1, 1)]; + let r: [i32x2; 3] = transmute(vld3_dup_s32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_dup_s8() { + let a: [i8; 49] = [0, 1, 1, 1, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15, 8, 16, 9, 17, 6, 14, 7, 15, 8, 16, 9, 17, 6, 14, 7, 15, 8, 16, 9, 17]; + let e: [i8x16; 3] = [i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [i8x16; 3] = transmute(vld3q_dup_s8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_dup_s16() { + let a: [i16; 25] = [0, 1, 1, 1, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13]; + let e: [i16x8; 3] = [i16x8::new(1, 1, 1, 1, 1, 1, 1, 1), i16x8::new(1, 1, 1, 1, 1, 1, 1, 1), i16x8::new(1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [i16x8; 3] = transmute(vld3q_dup_s16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_dup_s32() { + let a: [i32; 13] = [0, 1, 1, 1, 3, 1, 4, 3, 5, 1, 6, 3, 7]; + let e: [i32x4; 3] = [i32x4::new(1, 1, 1, 1), i32x4::new(1, 1, 1, 1), i32x4::new(1, 1, 1, 1)]; + let r: [i32x4; 3] = transmute(vld3q_dup_s32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_dup_s64() { + let a: [i64; 4] = [0, 1, 1, 1]; + let e: [i64x1; 3] = [i64x1::new(1), i64x1::new(1), i64x1::new(1)]; + let r: [i64x1; 3] = transmute(vld3_dup_s64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_dup_u8() { + let a: [u8; 25] = [0, 1, 1, 1, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13]; + let e: [u8x8; 3] = [u8x8::new(1, 1, 1, 1, 1, 1, 1, 1), u8x8::new(1, 1, 1, 1, 1, 1, 1, 1), u8x8::new(1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [u8x8; 3] = transmute(vld3_dup_u8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_dup_u16() { + let a: [u16; 13] = [0, 1, 1, 1, 3, 1, 4, 3, 5, 1, 6, 3, 7]; + let e: [u16x4; 3] = [u16x4::new(1, 1, 1, 1), u16x4::new(1, 1, 1, 1), u16x4::new(1, 1, 1, 1)]; + let r: [u16x4; 3] = transmute(vld3_dup_u16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_dup_u32() { + let a: [u32; 7] = [0, 1, 1, 1, 3, 1, 4]; + let e: [u32x2; 3] = [u32x2::new(1, 1), u32x2::new(1, 1), u32x2::new(1, 1)]; + let r: [u32x2; 3] = transmute(vld3_dup_u32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_dup_u8() { + let a: [u8; 49] = [0, 1, 1, 1, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15, 8, 16, 9, 17, 6, 14, 7, 15, 8, 16, 9, 17, 6, 14, 7, 15, 8, 16, 9, 17]; + let e: [u8x16; 3] = [u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [u8x16; 3] = transmute(vld3q_dup_u8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_dup_u16() { + let a: [u16; 25] = [0, 1, 1, 1, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13]; + let e: [u16x8; 3] = [u16x8::new(1, 1, 1, 1, 1, 1, 1, 1), u16x8::new(1, 1, 1, 1, 1, 1, 1, 1), u16x8::new(1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [u16x8; 3] = transmute(vld3q_dup_u16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_dup_u32() { + let a: [u32; 13] = [0, 1, 1, 1, 3, 1, 4, 3, 5, 1, 6, 3, 7]; + let e: [u32x4; 3] = [u32x4::new(1, 1, 1, 1), u32x4::new(1, 1, 1, 1), u32x4::new(1, 1, 1, 1)]; + let r: [u32x4; 3] = transmute(vld3q_dup_u32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_dup_p8() { + let a: [u8; 25] = [0, 1, 1, 1, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13]; + let e: [i8x8; 3] = [i8x8::new(1, 1, 1, 1, 1, 1, 1, 1), i8x8::new(1, 1, 1, 1, 1, 1, 1, 1), i8x8::new(1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [i8x8; 3] = transmute(vld3_dup_p8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_dup_p16() { + let a: [u16; 13] = [0, 1, 1, 1, 3, 1, 4, 3, 5, 1, 6, 3, 7]; + let e: [i16x4; 3] = [i16x4::new(1, 1, 1, 1), i16x4::new(1, 1, 1, 1), i16x4::new(1, 1, 1, 1)]; + let r: [i16x4; 3] = transmute(vld3_dup_p16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_dup_p8() { + let a: [u8; 49] = [0, 1, 1, 1, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15, 8, 16, 9, 17, 6, 14, 7, 15, 8, 16, 9, 17, 6, 14, 7, 15, 8, 16, 9, 17]; + let e: [i8x16; 3] = [i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [i8x16; 3] = transmute(vld3q_dup_p8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_dup_p16() { + let a: [u16; 25] = [0, 1, 1, 1, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13]; + let e: [i16x8; 3] = [i16x8::new(1, 1, 1, 1, 1, 1, 1, 1), i16x8::new(1, 1, 1, 1, 1, 1, 1, 1), i16x8::new(1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [i16x8; 3] = transmute(vld3q_dup_p16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_dup_u64() { + let a: [u64; 4] = [0, 1, 1, 1]; + let e: [u64x1; 3] = [u64x1::new(1), u64x1::new(1), u64x1::new(1)]; + let r: [u64x1; 3] = transmute(vld3_dup_u64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_dup_p64() { + let a: [u64; 4] = [0, 1, 1, 1]; + let e: [i64x1; 3] = [i64x1::new(1), i64x1::new(1), i64x1::new(1)]; + let r: [i64x1; 3] = transmute(vld3_dup_p64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_dup_f32() { + let a: [f32; 7] = [0., 1., 1., 1., 3., 1., 4.]; + let e: [f32x2; 3] = [f32x2::new(1., 1.), f32x2::new(1., 1.), f32x2::new(1., 1.)]; + let r: [f32x2; 3] = transmute(vld3_dup_f32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_dup_f32() { + let a: [f32; 13] = [0., 1., 1., 1., 3., 1., 4., 3., 5., 1., 4., 3., 5.]; + let e: [f32x4; 3] = [f32x4::new(1., 1., 1., 1.), f32x4::new(1., 1., 1., 1.), f32x4::new(1., 1., 1., 1.)]; + let r: [f32x4; 3] = transmute(vld3q_dup_f32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_lane_s8() { + let a: [i8; 25] = [0, 1, 2, 2, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i8x8; 3] = [i8x8::new(0, 2, 2, 14, 2, 16, 17, 18), i8x8::new(2, 20, 21, 22, 23, 24, 25, 26), i8x8::new(11, 12, 13, 14, 15, 16, 17, 18)]; + let e: [i8x8; 3] = [i8x8::new(1, 2, 2, 14, 2, 16, 17, 18), i8x8::new(2, 20, 21, 22, 23, 24, 25, 26), i8x8::new(2, 12, 13, 14, 15, 16, 17, 18)]; + let r: [i8x8; 3] = transmute(vld3_lane_s8::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_lane_s16() { + let a: [i16; 13] = [0, 1, 2, 2, 4, 5, 6, 7, 8, 1, 2, 3, 4]; + let b: [i16x4; 3] = [i16x4::new(0, 2, 2, 14), i16x4::new(2, 16, 17, 18), i16x4::new(2, 20, 21, 22)]; + let e: [i16x4; 3] = [i16x4::new(1, 2, 2, 14), i16x4::new(2, 16, 17, 18), i16x4::new(2, 20, 21, 22)]; + let r: [i16x4; 3] = transmute(vld3_lane_s16::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_lane_s32() { + let a: [i32; 7] = [0, 1, 2, 2, 4, 5, 6]; + let b: [i32x2; 3] = [i32x2::new(0, 2), i32x2::new(2, 14), i32x2::new(2, 16)]; + let e: [i32x2; 3] = [i32x2::new(1, 2), i32x2::new(2, 14), i32x2::new(2, 16)]; + let r: [i32x2; 3] = transmute(vld3_lane_s32::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_lane_s16() { + let a: [i16; 25] = [0, 1, 2, 2, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i16x8; 3] = [i16x8::new(0, 2, 2, 14, 2, 16, 17, 18), i16x8::new(2, 20, 21, 22, 23, 24, 25, 26), i16x8::new(11, 12, 13, 14, 15, 16, 17, 18)]; + let e: [i16x8; 3] = [i16x8::new(1, 2, 2, 14, 2, 16, 17, 18), i16x8::new(2, 20, 21, 22, 23, 24, 25, 26), i16x8::new(2, 12, 13, 14, 15, 16, 17, 18)]; + let r: [i16x8; 3] = transmute(vld3q_lane_s16::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_lane_s32() { + let a: [i32; 13] = [0, 1, 2, 2, 4, 5, 6, 7, 8, 1, 2, 3, 4]; + let b: [i32x4; 3] = [i32x4::new(0, 2, 2, 14), i32x4::new(2, 16, 17, 18), i32x4::new(2, 20, 21, 22)]; + let e: [i32x4; 3] = [i32x4::new(1, 2, 2, 14), i32x4::new(2, 16, 17, 18), i32x4::new(2, 20, 21, 22)]; + let r: [i32x4; 3] = transmute(vld3q_lane_s32::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_lane_u8() { + let a: [u8; 25] = [0, 1, 2, 2, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [u8x8; 3] = [u8x8::new(0, 2, 2, 14, 2, 16, 17, 18), u8x8::new(2, 20, 21, 22, 23, 24, 25, 26), u8x8::new(11, 12, 13, 14, 15, 16, 17, 18)]; + let e: [u8x8; 3] = [u8x8::new(1, 2, 2, 14, 2, 16, 17, 18), u8x8::new(2, 20, 21, 22, 23, 24, 25, 26), u8x8::new(2, 12, 13, 14, 15, 16, 17, 18)]; + let r: [u8x8; 3] = transmute(vld3_lane_u8::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_lane_u16() { + let a: [u16; 13] = [0, 1, 2, 2, 4, 5, 6, 7, 8, 1, 2, 3, 4]; + let b: [u16x4; 3] = [u16x4::new(0, 2, 2, 14), u16x4::new(2, 16, 17, 18), u16x4::new(2, 20, 21, 22)]; + let e: [u16x4; 3] = [u16x4::new(1, 2, 2, 14), u16x4::new(2, 16, 17, 18), u16x4::new(2, 20, 21, 22)]; + let r: [u16x4; 3] = transmute(vld3_lane_u16::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_lane_u32() { + let a: [u32; 7] = [0, 1, 2, 2, 4, 5, 6]; + let b: [u32x2; 3] = [u32x2::new(0, 2), u32x2::new(2, 14), u32x2::new(2, 16)]; + let e: [u32x2; 3] = [u32x2::new(1, 2), u32x2::new(2, 14), u32x2::new(2, 16)]; + let r: [u32x2; 3] = transmute(vld3_lane_u32::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_lane_u16() { + let a: [u16; 25] = [0, 1, 2, 2, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [u16x8; 3] = [u16x8::new(0, 2, 2, 14, 2, 16, 17, 18), u16x8::new(2, 20, 21, 22, 23, 24, 25, 26), u16x8::new(11, 12, 13, 14, 15, 16, 17, 18)]; + let e: [u16x8; 3] = [u16x8::new(1, 2, 2, 14, 2, 16, 17, 18), u16x8::new(2, 20, 21, 22, 23, 24, 25, 26), u16x8::new(2, 12, 13, 14, 15, 16, 17, 18)]; + let r: [u16x8; 3] = transmute(vld3q_lane_u16::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_lane_u32() { + let a: [u32; 13] = [0, 1, 2, 2, 4, 5, 6, 7, 8, 1, 2, 3, 4]; + let b: [u32x4; 3] = [u32x4::new(0, 2, 2, 14), u32x4::new(2, 16, 17, 18), u32x4::new(2, 20, 21, 22)]; + let e: [u32x4; 3] = [u32x4::new(1, 2, 2, 14), u32x4::new(2, 16, 17, 18), u32x4::new(2, 20, 21, 22)]; + let r: [u32x4; 3] = transmute(vld3q_lane_u32::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_lane_p8() { + let a: [u8; 25] = [0, 1, 2, 2, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i8x8; 3] = [i8x8::new(0, 2, 2, 14, 2, 16, 17, 18), i8x8::new(2, 20, 21, 22, 23, 24, 25, 26), i8x8::new(11, 12, 13, 14, 15, 16, 17, 18)]; + let e: [i8x8; 3] = [i8x8::new(1, 2, 2, 14, 2, 16, 17, 18), i8x8::new(2, 20, 21, 22, 23, 24, 25, 26), i8x8::new(2, 12, 13, 14, 15, 16, 17, 18)]; + let r: [i8x8; 3] = transmute(vld3_lane_p8::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_lane_p16() { + let a: [u16; 13] = [0, 1, 2, 2, 4, 5, 6, 7, 8, 1, 2, 3, 4]; + let b: [i16x4; 3] = [i16x4::new(0, 2, 2, 14), i16x4::new(2, 16, 17, 18), i16x4::new(2, 20, 21, 22)]; + let e: [i16x4; 3] = [i16x4::new(1, 2, 2, 14), i16x4::new(2, 16, 17, 18), i16x4::new(2, 20, 21, 22)]; + let r: [i16x4; 3] = transmute(vld3_lane_p16::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_lane_p16() { + let a: [u16; 25] = [0, 1, 2, 2, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i16x8; 3] = [i16x8::new(0, 2, 2, 14, 2, 16, 17, 18), i16x8::new(2, 20, 21, 22, 23, 24, 25, 26), i16x8::new(11, 12, 13, 14, 15, 16, 17, 18)]; + let e: [i16x8; 3] = [i16x8::new(1, 2, 2, 14, 2, 16, 17, 18), i16x8::new(2, 20, 21, 22, 23, 24, 25, 26), i16x8::new(2, 12, 13, 14, 15, 16, 17, 18)]; + let r: [i16x8; 3] = transmute(vld3q_lane_p16::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3_lane_f32() { + let a: [f32; 7] = [0., 1., 2., 2., 4., 5., 6.]; + let b: [f32x2; 3] = [f32x2::new(0., 2.), f32x2::new(2., 14.), f32x2::new(9., 16.)]; + let e: [f32x2; 3] = [f32x2::new(1., 2.), f32x2::new(2., 14.), f32x2::new(2., 16.)]; + let r: [f32x2; 3] = transmute(vld3_lane_f32::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld3q_lane_f32() { + let a: [f32; 13] = [0., 1., 2., 2., 4., 5., 6., 7., 8., 5., 6., 7., 8.]; + let b: [f32x4; 3] = [f32x4::new(0., 2., 2., 14.), f32x4::new(9., 16., 17., 18.), f32x4::new(5., 6., 7., 8.)]; + let e: [f32x4; 3] = [f32x4::new(1., 2., 2., 14.), f32x4::new(2., 16., 17., 18.), f32x4::new(2., 6., 7., 8.)]; + let r: [f32x4; 3] = transmute(vld3q_lane_f32::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_s8() { + let a: [i8; 33] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let e: [i8x8; 4] = [i8x8::new(1, 2, 2, 6, 2, 6, 6, 8), i8x8::new(2, 6, 6, 8, 6, 8, 8, 16), i8x8::new(2, 6, 6, 8, 6, 8, 8, 16), i8x8::new(6, 8, 8, 16, 8, 16, 16, 32)]; + let r: [i8x8; 4] = transmute(vld4_s8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_s16() { + let a: [i16; 17] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16]; + let e: [i16x4; 4] = [i16x4::new(1, 2, 2, 6), i16x4::new(2, 6, 6, 8), i16x4::new(2, 6, 6, 8), i16x4::new(6, 8, 8, 16)]; + let r: [i16x4; 4] = transmute(vld4_s16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_s32() { + let a: [i32; 9] = [0, 1, 2, 2, 6, 2, 6, 6, 8]; + let e: [i32x2; 4] = [i32x2::new(1, 2), i32x2::new(2, 6), i32x2::new(2, 6), i32x2::new(6, 8)]; + let r: [i32x2; 4] = transmute(vld4_s32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_s8() { + let a: [i8; 65] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64]; + let e: [i8x16; 4] = [i8x16::new(1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16), i8x16::new(2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32), i8x16::new(2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48), i8x16::new(6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64)]; + let r: [i8x16; 4] = transmute(vld4q_s8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_s16() { + let a: [i16; 33] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let e: [i16x8; 4] = [i16x8::new(1, 2, 2, 6, 2, 6, 6, 8), i16x8::new(2, 6, 6, 8, 6, 8, 8, 16), i16x8::new(2, 6, 6, 8, 6, 8, 8, 16), i16x8::new(6, 8, 8, 16, 8, 16, 16, 32)]; + let r: [i16x8; 4] = transmute(vld4q_s16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_s32() { + let a: [i32; 17] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16]; + let e: [i32x4; 4] = [i32x4::new(1, 2, 2, 6), i32x4::new(2, 6, 6, 8), i32x4::new(2, 6, 6, 8), i32x4::new(6, 8, 8, 16)]; + let r: [i32x4; 4] = transmute(vld4q_s32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_s64() { + let a: [i64; 5] = [0, 1, 2, 2, 6]; + let e: [i64x1; 4] = [i64x1::new(1), i64x1::new(2), i64x1::new(2), i64x1::new(6)]; + let r: [i64x1; 4] = transmute(vld4_s64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_u8() { + let a: [u8; 33] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let e: [u8x8; 4] = [u8x8::new(1, 2, 2, 6, 2, 6, 6, 8), u8x8::new(2, 6, 6, 8, 6, 8, 8, 16), u8x8::new(2, 6, 6, 8, 6, 8, 8, 16), u8x8::new(6, 8, 8, 16, 8, 16, 16, 32)]; + let r: [u8x8; 4] = transmute(vld4_u8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_u16() { + let a: [u16; 17] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16]; + let e: [u16x4; 4] = [u16x4::new(1, 2, 2, 6), u16x4::new(2, 6, 6, 8), u16x4::new(2, 6, 6, 8), u16x4::new(6, 8, 8, 16)]; + let r: [u16x4; 4] = transmute(vld4_u16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_u32() { + let a: [u32; 9] = [0, 1, 2, 2, 6, 2, 6, 6, 8]; + let e: [u32x2; 4] = [u32x2::new(1, 2), u32x2::new(2, 6), u32x2::new(2, 6), u32x2::new(6, 8)]; + let r: [u32x2; 4] = transmute(vld4_u32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_u8() { + let a: [u8; 65] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64]; + let e: [u8x16; 4] = [u8x16::new(1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16), u8x16::new(2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32), u8x16::new(2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48), u8x16::new(6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64)]; + let r: [u8x16; 4] = transmute(vld4q_u8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_u16() { + let a: [u16; 33] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let e: [u16x8; 4] = [u16x8::new(1, 2, 2, 6, 2, 6, 6, 8), u16x8::new(2, 6, 6, 8, 6, 8, 8, 16), u16x8::new(2, 6, 6, 8, 6, 8, 8, 16), u16x8::new(6, 8, 8, 16, 8, 16, 16, 32)]; + let r: [u16x8; 4] = transmute(vld4q_u16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_u32() { + let a: [u32; 17] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16]; + let e: [u32x4; 4] = [u32x4::new(1, 2, 2, 6), u32x4::new(2, 6, 6, 8), u32x4::new(2, 6, 6, 8), u32x4::new(6, 8, 8, 16)]; + let r: [u32x4; 4] = transmute(vld4q_u32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_p8() { + let a: [u8; 33] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let e: [i8x8; 4] = [i8x8::new(1, 2, 2, 6, 2, 6, 6, 8), i8x8::new(2, 6, 6, 8, 6, 8, 8, 16), i8x8::new(2, 6, 6, 8, 6, 8, 8, 16), i8x8::new(6, 8, 8, 16, 8, 16, 16, 32)]; + let r: [i8x8; 4] = transmute(vld4_p8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_p16() { + let a: [u16; 17] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16]; + let e: [i16x4; 4] = [i16x4::new(1, 2, 2, 6), i16x4::new(2, 6, 6, 8), i16x4::new(2, 6, 6, 8), i16x4::new(6, 8, 8, 16)]; + let r: [i16x4; 4] = transmute(vld4_p16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_p8() { + let a: [u8; 65] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64]; + let e: [i8x16; 4] = [i8x16::new(1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16), i8x16::new(2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32), i8x16::new(2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48), i8x16::new(6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64)]; + let r: [i8x16; 4] = transmute(vld4q_p8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_p16() { + let a: [u16; 33] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let e: [i16x8; 4] = [i16x8::new(1, 2, 2, 6, 2, 6, 6, 8), i16x8::new(2, 6, 6, 8, 6, 8, 8, 16), i16x8::new(2, 6, 6, 8, 6, 8, 8, 16), i16x8::new(6, 8, 8, 16, 8, 16, 16, 32)]; + let r: [i16x8; 4] = transmute(vld4q_p16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_u64() { + let a: [u64; 5] = [0, 1, 2, 2, 6]; + let e: [u64x1; 4] = [u64x1::new(1), u64x1::new(2), u64x1::new(2), u64x1::new(6)]; + let r: [u64x1; 4] = transmute(vld4_u64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_p64() { + let a: [u64; 5] = [0, 1, 2, 2, 6]; + let e: [i64x1; 4] = [i64x1::new(1), i64x1::new(2), i64x1::new(2), i64x1::new(6)]; + let r: [i64x1; 4] = transmute(vld4_p64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_f32() { + let a: [f32; 9] = [0., 1., 2., 2., 6., 2., 6., 6., 8.]; + let e: [f32x2; 4] = [f32x2::new(1., 2.), f32x2::new(2., 6.), f32x2::new(2., 6.), f32x2::new(6., 8.)]; + let r: [f32x2; 4] = transmute(vld4_f32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_f32() { + let a: [f32; 17] = [0., 1., 2., 2., 6., 2., 6., 6., 8., 2., 6., 6., 8., 6., 8., 15., 16.]; + let e: [f32x4; 4] = [f32x4::new(1., 2., 2., 6.), f32x4::new(2., 6., 6., 8.), f32x4::new(2., 6., 6., 15.), f32x4::new(6., 8., 8., 16.)]; + let r: [f32x4; 4] = transmute(vld4q_f32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_dup_s8() { + let a: [i8; 33] = [0, 1, 1, 1, 1, 2, 4, 3, 5, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9]; + let e: [i8x8; 4] = [i8x8::new(1, 1, 1, 1, 1, 1, 1, 1), i8x8::new(1, 1, 1, 1, 1, 1, 1, 1), i8x8::new(1, 1, 1, 1, 1, 1, 1, 1), i8x8::new(1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [i8x8; 4] = transmute(vld4_dup_s8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_dup_s16() { + let a: [i16; 17] = [0, 1, 1, 1, 1, 2, 4, 3, 5, 8, 6, 3, 7, 4, 8, 5, 9]; + let e: [i16x4; 4] = [i16x4::new(1, 1, 1, 1), i16x4::new(1, 1, 1, 1), i16x4::new(1, 1, 1, 1), i16x4::new(1, 1, 1, 1)]; + let r: [i16x4; 4] = transmute(vld4_dup_s16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_dup_s32() { + let a: [i32; 9] = [0, 1, 1, 1, 1, 2, 4, 3, 5]; + let e: [i32x2; 4] = [i32x2::new(1, 1), i32x2::new(1, 1), i32x2::new(1, 1), i32x2::new(1, 1)]; + let r: [i32x2; 4] = transmute(vld4_dup_s32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_dup_s8() { + let a: [i8; 65] = [0, 1, 1, 1, 1, 2, 4, 3, 5, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9]; + let e: [i8x16; 4] = [i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [i8x16; 4] = transmute(vld4q_dup_s8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_dup_s16() { + let a: [i16; 33] = [0, 1, 1, 1, 1, 2, 4, 3, 5, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9]; + let e: [i16x8; 4] = [i16x8::new(1, 1, 1, 1, 1, 1, 1, 1), i16x8::new(1, 1, 1, 1, 1, 1, 1, 1), i16x8::new(1, 1, 1, 1, 1, 1, 1, 1), i16x8::new(1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [i16x8; 4] = transmute(vld4q_dup_s16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_dup_s32() { + let a: [i32; 17] = [0, 1, 1, 1, 1, 2, 4, 3, 5, 8, 6, 3, 7, 4, 8, 5, 9]; + let e: [i32x4; 4] = [i32x4::new(1, 1, 1, 1), i32x4::new(1, 1, 1, 1), i32x4::new(1, 1, 1, 1), i32x4::new(1, 1, 1, 1)]; + let r: [i32x4; 4] = transmute(vld4q_dup_s32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_dup_s64() { + let a: [i64; 5] = [0, 1, 1, 1, 1]; + let e: [i64x1; 4] = [i64x1::new(1), i64x1::new(1), i64x1::new(1), i64x1::new(1)]; + let r: [i64x1; 4] = transmute(vld4_dup_s64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_dup_u8() { + let a: [u8; 33] = [0, 1, 1, 1, 1, 2, 4, 3, 5, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9]; + let e: [u8x8; 4] = [u8x8::new(1, 1, 1, 1, 1, 1, 1, 1), u8x8::new(1, 1, 1, 1, 1, 1, 1, 1), u8x8::new(1, 1, 1, 1, 1, 1, 1, 1), u8x8::new(1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [u8x8; 4] = transmute(vld4_dup_u8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_dup_u16() { + let a: [u16; 17] = [0, 1, 1, 1, 1, 2, 4, 3, 5, 8, 6, 3, 7, 4, 8, 5, 9]; + let e: [u16x4; 4] = [u16x4::new(1, 1, 1, 1), u16x4::new(1, 1, 1, 1), u16x4::new(1, 1, 1, 1), u16x4::new(1, 1, 1, 1)]; + let r: [u16x4; 4] = transmute(vld4_dup_u16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_dup_u32() { + let a: [u32; 9] = [0, 1, 1, 1, 1, 2, 4, 3, 5]; + let e: [u32x2; 4] = [u32x2::new(1, 1), u32x2::new(1, 1), u32x2::new(1, 1), u32x2::new(1, 1)]; + let r: [u32x2; 4] = transmute(vld4_dup_u32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_dup_u8() { + let a: [u8; 65] = [0, 1, 1, 1, 1, 2, 4, 3, 5, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9]; + let e: [u8x16; 4] = [u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [u8x16; 4] = transmute(vld4q_dup_u8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_dup_u16() { + let a: [u16; 33] = [0, 1, 1, 1, 1, 2, 4, 3, 5, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9]; + let e: [u16x8; 4] = [u16x8::new(1, 1, 1, 1, 1, 1, 1, 1), u16x8::new(1, 1, 1, 1, 1, 1, 1, 1), u16x8::new(1, 1, 1, 1, 1, 1, 1, 1), u16x8::new(1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [u16x8; 4] = transmute(vld4q_dup_u16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_dup_u32() { + let a: [u32; 17] = [0, 1, 1, 1, 1, 2, 4, 3, 5, 8, 6, 3, 7, 4, 8, 5, 9]; + let e: [u32x4; 4] = [u32x4::new(1, 1, 1, 1), u32x4::new(1, 1, 1, 1), u32x4::new(1, 1, 1, 1), u32x4::new(1, 1, 1, 1)]; + let r: [u32x4; 4] = transmute(vld4q_dup_u32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_dup_p8() { + let a: [u8; 33] = [0, 1, 1, 1, 1, 2, 4, 3, 5, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9]; + let e: [i8x8; 4] = [i8x8::new(1, 1, 1, 1, 1, 1, 1, 1), i8x8::new(1, 1, 1, 1, 1, 1, 1, 1), i8x8::new(1, 1, 1, 1, 1, 1, 1, 1), i8x8::new(1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [i8x8; 4] = transmute(vld4_dup_p8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_dup_p16() { + let a: [u16; 17] = [0, 1, 1, 1, 1, 2, 4, 3, 5, 8, 6, 3, 7, 4, 8, 5, 9]; + let e: [i16x4; 4] = [i16x4::new(1, 1, 1, 1), i16x4::new(1, 1, 1, 1), i16x4::new(1, 1, 1, 1), i16x4::new(1, 1, 1, 1)]; + let r: [i16x4; 4] = transmute(vld4_dup_p16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_dup_p8() { + let a: [u8; 65] = [0, 1, 1, 1, 1, 2, 4, 3, 5, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9]; + let e: [i8x16; 4] = [i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [i8x16; 4] = transmute(vld4q_dup_p8(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_dup_p16() { + let a: [u16; 33] = [0, 1, 1, 1, 1, 2, 4, 3, 5, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9]; + let e: [i16x8; 4] = [i16x8::new(1, 1, 1, 1, 1, 1, 1, 1), i16x8::new(1, 1, 1, 1, 1, 1, 1, 1), i16x8::new(1, 1, 1, 1, 1, 1, 1, 1), i16x8::new(1, 1, 1, 1, 1, 1, 1, 1)]; + let r: [i16x8; 4] = transmute(vld4q_dup_p16(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_dup_u64() { + let a: [u64; 5] = [0, 1, 1, 1, 1]; + let e: [u64x1; 4] = [u64x1::new(1), u64x1::new(1), u64x1::new(1), u64x1::new(1)]; + let r: [u64x1; 4] = transmute(vld4_dup_u64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_dup_p64() { + let a: [u64; 5] = [0, 1, 1, 1, 1]; + let e: [i64x1; 4] = [i64x1::new(1), i64x1::new(1), i64x1::new(1), i64x1::new(1)]; + let r: [i64x1; 4] = transmute(vld4_dup_p64(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_dup_f32() { + let a: [f32; 9] = [0., 1., 1., 1., 1., 6., 4., 3., 5.]; + let e: [f32x2; 4] = [f32x2::new(1., 1.), f32x2::new(1., 1.), f32x2::new(1., 1.), f32x2::new(1., 1.)]; + let r: [f32x2; 4] = transmute(vld4_dup_f32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_dup_f32() { + let a: [f32; 17] = [0., 1., 1., 1., 1., 6., 4., 3., 5., 7., 4., 3., 5., 8., 4., 3., 5.]; + let e: [f32x4; 4] = [f32x4::new(1., 1., 1., 1.), f32x4::new(1., 1., 1., 1.), f32x4::new(1., 1., 1., 1.), f32x4::new(1., 1., 1., 1.)]; + let r: [f32x4; 4] = transmute(vld4q_dup_f32(a[1..].as_ptr())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_lane_s8() { + let a: [i8; 33] = [0, 1, 2, 2, 2, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i8x8; 4] = [i8x8::new(0, 2, 2, 2, 2, 16, 2, 18), i8x8::new(2, 20, 21, 22, 2, 24, 25, 26), i8x8::new(11, 12, 13, 14, 15, 16, 2, 18), i8x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let e: [i8x8; 4] = [i8x8::new(1, 2, 2, 2, 2, 16, 2, 18), i8x8::new(2, 20, 21, 22, 2, 24, 25, 26), i8x8::new(2, 12, 13, 14, 15, 16, 2, 18), i8x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let r: [i8x8; 4] = transmute(vld4_lane_s8::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_lane_s16() { + let a: [i16; 17] = [0, 1, 2, 2, 2, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i16x4; 4] = [i16x4::new(0, 2, 2, 2), i16x4::new(2, 16, 2, 18), i16x4::new(2, 20, 21, 22), i16x4::new(2, 24, 25, 26)]; + let e: [i16x4; 4] = [i16x4::new(1, 2, 2, 2), i16x4::new(2, 16, 2, 18), i16x4::new(2, 20, 21, 22), i16x4::new(2, 24, 25, 26)]; + let r: [i16x4; 4] = transmute(vld4_lane_s16::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_lane_s32() { + let a: [i32; 9] = [0, 1, 2, 2, 2, 5, 6, 7, 8]; + let b: [i32x2; 4] = [i32x2::new(0, 2), i32x2::new(2, 2), i32x2::new(2, 16), i32x2::new(2, 18)]; + let e: [i32x2; 4] = [i32x2::new(1, 2), i32x2::new(2, 2), i32x2::new(2, 16), i32x2::new(2, 18)]; + let r: [i32x2; 4] = transmute(vld4_lane_s32::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_lane_s16() { + let a: [i16; 33] = [0, 1, 2, 2, 2, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i16x8; 4] = [i16x8::new(0, 2, 2, 2, 2, 16, 2, 18), i16x8::new(2, 20, 21, 22, 2, 24, 25, 26), i16x8::new(11, 12, 13, 14, 15, 16, 2, 18), i16x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let e: [i16x8; 4] = [i16x8::new(1, 2, 2, 2, 2, 16, 2, 18), i16x8::new(2, 20, 21, 22, 2, 24, 25, 26), i16x8::new(2, 12, 13, 14, 15, 16, 2, 18), i16x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let r: [i16x8; 4] = transmute(vld4q_lane_s16::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_lane_s32() { + let a: [i32; 17] = [0, 1, 2, 2, 2, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i32x4; 4] = [i32x4::new(0, 2, 2, 2), i32x4::new(2, 16, 2, 18), i32x4::new(2, 20, 21, 22), i32x4::new(2, 24, 25, 26)]; + let e: [i32x4; 4] = [i32x4::new(1, 2, 2, 2), i32x4::new(2, 16, 2, 18), i32x4::new(2, 20, 21, 22), i32x4::new(2, 24, 25, 26)]; + let r: [i32x4; 4] = transmute(vld4q_lane_s32::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_lane_u8() { + let a: [u8; 33] = [0, 1, 2, 2, 2, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [u8x8; 4] = [u8x8::new(0, 2, 2, 2, 2, 16, 2, 18), u8x8::new(2, 20, 21, 22, 2, 24, 25, 26), u8x8::new(11, 12, 13, 14, 15, 16, 2, 18), u8x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let e: [u8x8; 4] = [u8x8::new(1, 2, 2, 2, 2, 16, 2, 18), u8x8::new(2, 20, 21, 22, 2, 24, 25, 26), u8x8::new(2, 12, 13, 14, 15, 16, 2, 18), u8x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let r: [u8x8; 4] = transmute(vld4_lane_u8::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_lane_u16() { + let a: [u16; 17] = [0, 1, 2, 2, 2, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [u16x4; 4] = [u16x4::new(0, 2, 2, 2), u16x4::new(2, 16, 2, 18), u16x4::new(2, 20, 21, 22), u16x4::new(2, 24, 25, 26)]; + let e: [u16x4; 4] = [u16x4::new(1, 2, 2, 2), u16x4::new(2, 16, 2, 18), u16x4::new(2, 20, 21, 22), u16x4::new(2, 24, 25, 26)]; + let r: [u16x4; 4] = transmute(vld4_lane_u16::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_lane_u32() { + let a: [u32; 9] = [0, 1, 2, 2, 2, 5, 6, 7, 8]; + let b: [u32x2; 4] = [u32x2::new(0, 2), u32x2::new(2, 2), u32x2::new(2, 16), u32x2::new(2, 18)]; + let e: [u32x2; 4] = [u32x2::new(1, 2), u32x2::new(2, 2), u32x2::new(2, 16), u32x2::new(2, 18)]; + let r: [u32x2; 4] = transmute(vld4_lane_u32::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_lane_u16() { + let a: [u16; 33] = [0, 1, 2, 2, 2, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [u16x8; 4] = [u16x8::new(0, 2, 2, 2, 2, 16, 2, 18), u16x8::new(2, 20, 21, 22, 2, 24, 25, 26), u16x8::new(11, 12, 13, 14, 15, 16, 2, 18), u16x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let e: [u16x8; 4] = [u16x8::new(1, 2, 2, 2, 2, 16, 2, 18), u16x8::new(2, 20, 21, 22, 2, 24, 25, 26), u16x8::new(2, 12, 13, 14, 15, 16, 2, 18), u16x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let r: [u16x8; 4] = transmute(vld4q_lane_u16::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_lane_u32() { + let a: [u32; 17] = [0, 1, 2, 2, 2, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [u32x4; 4] = [u32x4::new(0, 2, 2, 2), u32x4::new(2, 16, 2, 18), u32x4::new(2, 20, 21, 22), u32x4::new(2, 24, 25, 26)]; + let e: [u32x4; 4] = [u32x4::new(1, 2, 2, 2), u32x4::new(2, 16, 2, 18), u32x4::new(2, 20, 21, 22), u32x4::new(2, 24, 25, 26)]; + let r: [u32x4; 4] = transmute(vld4q_lane_u32::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_lane_p8() { + let a: [u8; 33] = [0, 1, 2, 2, 2, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i8x8; 4] = [i8x8::new(0, 2, 2, 2, 2, 16, 2, 18), i8x8::new(2, 20, 21, 22, 2, 24, 25, 26), i8x8::new(11, 12, 13, 14, 15, 16, 2, 18), i8x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let e: [i8x8; 4] = [i8x8::new(1, 2, 2, 2, 2, 16, 2, 18), i8x8::new(2, 20, 21, 22, 2, 24, 25, 26), i8x8::new(2, 12, 13, 14, 15, 16, 2, 18), i8x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let r: [i8x8; 4] = transmute(vld4_lane_p8::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_lane_p16() { + let a: [u16; 17] = [0, 1, 2, 2, 2, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i16x4; 4] = [i16x4::new(0, 2, 2, 2), i16x4::new(2, 16, 2, 18), i16x4::new(2, 20, 21, 22), i16x4::new(2, 24, 25, 26)]; + let e: [i16x4; 4] = [i16x4::new(1, 2, 2, 2), i16x4::new(2, 16, 2, 18), i16x4::new(2, 20, 21, 22), i16x4::new(2, 24, 25, 26)]; + let r: [i16x4; 4] = transmute(vld4_lane_p16::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_lane_p16() { + let a: [u16; 33] = [0, 1, 2, 2, 2, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8]; + let b: [i16x8; 4] = [i16x8::new(0, 2, 2, 2, 2, 16, 2, 18), i16x8::new(2, 20, 21, 22, 2, 24, 25, 26), i16x8::new(11, 12, 13, 14, 15, 16, 2, 18), i16x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let e: [i16x8; 4] = [i16x8::new(1, 2, 2, 2, 2, 16, 2, 18), i16x8::new(2, 20, 21, 22, 2, 24, 25, 26), i16x8::new(2, 12, 13, 14, 15, 16, 2, 18), i16x8::new(2, 20, 21, 22, 23, 24, 25, 26)]; + let r: [i16x8; 4] = transmute(vld4q_lane_p16::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4_lane_f32() { + let a: [f32; 9] = [0., 1., 2., 2., 2., 5., 6., 7., 8.]; + let b: [f32x2; 4] = [f32x2::new(0., 2.), f32x2::new(2., 2.), f32x2::new(2., 16.), f32x2::new(2., 18.)]; + let e: [f32x2; 4] = [f32x2::new(1., 2.), f32x2::new(2., 2.), f32x2::new(2., 16.), f32x2::new(2., 18.)]; + let r: [f32x2; 4] = transmute(vld4_lane_f32::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld4q_lane_f32() { + let a: [f32; 17] = [0., 1., 2., 2., 2., 5., 6., 7., 8., 5., 6., 7., 8., 1., 4., 3., 5.]; + let b: [f32x4; 4] = [f32x4::new(0., 2., 2., 2.), f32x4::new(2., 16., 2., 18.), f32x4::new(5., 6., 7., 8.), f32x4::new(1., 4., 3., 5.)]; + let e: [f32x4; 4] = [f32x4::new(1., 2., 2., 2.), f32x4::new(2., 16., 2., 18.), f32x4::new(2., 6., 7., 8.), f32x4::new(2., 4., 3., 5.)]; + let r: [f32x4; 4] = transmute(vld4q_lane_f32::<0>(a[1..].as_ptr(), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_lane_s8() { + let a: [i8; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [i8; 8] = [1, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [i8; 8] = [0i8; 8]; + vst1_lane_s8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_lane_s16() { + let a: [i16; 5] = [0, 1, 2, 3, 4]; + let e: [i16; 4] = [1, 0, 0, 0]; + let mut r: [i16; 4] = [0i16; 4]; + vst1_lane_s16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_lane_s32() { + let a: [i32; 3] = [0, 1, 2]; + let e: [i32; 2] = [1, 0]; + let mut r: [i32; 2] = [0i32; 2]; + vst1_lane_s32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_lane_s64() { + let a: [i64; 2] = [0, 1]; + let e: [i64; 1] = [1]; + let mut r: [i64; 1] = [0i64; 1]; + vst1_lane_s64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_lane_s8() { + let a: [i8; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [i8; 16] = [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [i8; 16] = [0i8; 16]; + vst1q_lane_s8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_lane_s16() { + let a: [i16; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [i16; 8] = [1, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [i16; 8] = [0i16; 8]; + vst1q_lane_s16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_lane_s32() { + let a: [i32; 5] = [0, 1, 2, 3, 4]; + let e: [i32; 4] = [1, 0, 0, 0]; + let mut r: [i32; 4] = [0i32; 4]; + vst1q_lane_s32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_lane_s64() { + let a: [i64; 3] = [0, 1, 2]; + let e: [i64; 2] = [1, 0]; + let mut r: [i64; 2] = [0i64; 2]; + vst1q_lane_s64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_lane_u8() { + let a: [u8; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [u8; 8] = [1, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u8; 8] = [0u8; 8]; + vst1_lane_u8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_lane_u16() { + let a: [u16; 5] = [0, 1, 2, 3, 4]; + let e: [u16; 4] = [1, 0, 0, 0]; + let mut r: [u16; 4] = [0u16; 4]; + vst1_lane_u16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_lane_u32() { + let a: [u32; 3] = [0, 1, 2]; + let e: [u32; 2] = [1, 0]; + let mut r: [u32; 2] = [0u32; 2]; + vst1_lane_u32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_lane_u64() { + let a: [u64; 2] = [0, 1]; + let e: [u64; 1] = [1]; + let mut r: [u64; 1] = [0u64; 1]; + vst1_lane_u64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_lane_u8() { + let a: [u8; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [u8; 16] = [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u8; 16] = [0u8; 16]; + vst1q_lane_u8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_lane_u16() { + let a: [u16; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [u16; 8] = [1, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u16; 8] = [0u16; 8]; + vst1q_lane_u16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_lane_u32() { + let a: [u32; 5] = [0, 1, 2, 3, 4]; + let e: [u32; 4] = [1, 0, 0, 0]; + let mut r: [u32; 4] = [0u32; 4]; + vst1q_lane_u32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_lane_u64() { + let a: [u64; 3] = [0, 1, 2]; + let e: [u64; 2] = [1, 0]; + let mut r: [u64; 2] = [0u64; 2]; + vst1q_lane_u64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_lane_p8() { + let a: [u8; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [u8; 8] = [1, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u8; 8] = [0u8; 8]; + vst1_lane_p8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_lane_p16() { + let a: [u16; 5] = [0, 1, 2, 3, 4]; + let e: [u16; 4] = [1, 0, 0, 0]; + let mut r: [u16; 4] = [0u16; 4]; + vst1_lane_p16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_lane_p8() { + let a: [u8; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [u8; 16] = [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u8; 16] = [0u8; 16]; + vst1q_lane_p8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_lane_p16() { + let a: [u16; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [u16; 8] = [1, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u16; 8] = [0u16; 8]; + vst1q_lane_p16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_lane_p64() { + let a: [u64; 2] = [0, 1]; + let e: [u64; 1] = [1]; + let mut r: [u64; 1] = [0u64; 1]; + vst1_lane_p64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_lane_p64() { + let a: [u64; 3] = [0, 1, 2]; + let e: [u64; 2] = [1, 0]; + let mut r: [u64; 2] = [0u64; 2]; + vst1q_lane_p64::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_lane_f32() { + let a: [f32; 3] = [0., 1., 2.]; + let e: [f32; 2] = [1., 0.]; + let mut r: [f32; 2] = [0f32; 2]; + vst1_lane_f32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_lane_f32() { + let a: [f32; 5] = [0., 1., 2., 3., 4.]; + let e: [f32; 4] = [1., 0., 0., 0.]; + let mut r: [f32; 4] = [0f32; 4]; + vst1q_lane_f32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_s8_x2() { + let a: [i8; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [i8; 16] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let mut r: [i8; 16] = [0i8; 16]; + vst1_s8_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_s16_x2() { + let a: [i16; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [i16; 8] = [1, 2, 3, 4, 5, 6, 7, 8]; + let mut r: [i16; 8] = [0i16; 8]; + vst1_s16_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_s32_x2() { + let a: [i32; 5] = [0, 1, 2, 3, 4]; + let e: [i32; 4] = [1, 2, 3, 4]; + let mut r: [i32; 4] = [0i32; 4]; + vst1_s32_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_s64_x2() { + let a: [i64; 3] = [0, 1, 2]; + let e: [i64; 2] = [1, 2]; + let mut r: [i64; 2] = [0i64; 2]; + vst1_s64_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_s8_x2() { + let a: [i8; 33] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [i8; 32] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let mut r: [i8; 32] = [0i8; 32]; + vst1q_s8_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_s16_x2() { + let a: [i16; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [i16; 16] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let mut r: [i16; 16] = [0i16; 16]; + vst1q_s16_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_s32_x2() { + let a: [i32; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [i32; 8] = [1, 2, 3, 4, 5, 6, 7, 8]; + let mut r: [i32; 8] = [0i32; 8]; + vst1q_s32_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_s64_x2() { + let a: [i64; 5] = [0, 1, 2, 3, 4]; + let e: [i64; 4] = [1, 2, 3, 4]; + let mut r: [i64; 4] = [0i64; 4]; + vst1q_s64_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_s8_x3() { + let a: [i8; 25] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]; + let e: [i8; 24] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]; + let mut r: [i8; 24] = [0i8; 24]; + vst1_s8_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_s16_x3() { + let a: [i16; 13] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + let e: [i16; 12] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + let mut r: [i16; 12] = [0i16; 12]; + vst1_s16_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_s32_x3() { + let a: [i32; 7] = [0, 1, 2, 3, 4, 5, 6]; + let e: [i32; 6] = [1, 2, 3, 4, 5, 6]; + let mut r: [i32; 6] = [0i32; 6]; + vst1_s32_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_s64_x3() { + let a: [i64; 4] = [0, 1, 2, 3]; + let e: [i64; 3] = [1, 2, 3]; + let mut r: [i64; 3] = [0i64; 3]; + vst1_s64_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_s8_x3() { + let a: [i8; 49] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [i8; 48] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let mut r: [i8; 48] = [0i8; 48]; + vst1q_s8_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_s16_x3() { + let a: [i16; 25] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]; + let e: [i16; 24] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]; + let mut r: [i16; 24] = [0i16; 24]; + vst1q_s16_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_s32_x3() { + let a: [i32; 13] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + let e: [i32; 12] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + let mut r: [i32; 12] = [0i32; 12]; + vst1q_s32_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_s64_x3() { + let a: [i64; 7] = [0, 1, 2, 3, 4, 5, 6]; + let e: [i64; 6] = [1, 2, 3, 4, 5, 6]; + let mut r: [i64; 6] = [0i64; 6]; + vst1q_s64_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_s8_x4() { + let a: [i8; 33] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [i8; 32] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let mut r: [i8; 32] = [0i8; 32]; + vst1_s8_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_s16_x4() { + let a: [i16; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [i16; 16] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let mut r: [i16; 16] = [0i16; 16]; + vst1_s16_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_s32_x4() { + let a: [i32; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [i32; 8] = [1, 2, 3, 4, 5, 6, 7, 8]; + let mut r: [i32; 8] = [0i32; 8]; + vst1_s32_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_s64_x4() { + let a: [i64; 5] = [0, 1, 2, 3, 4]; + let e: [i64; 4] = [1, 2, 3, 4]; + let mut r: [i64; 4] = [0i64; 4]; + vst1_s64_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_s8_x4() { + let a: [i8; 65] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [i8; 64] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let mut r: [i8; 64] = [0i8; 64]; + vst1q_s8_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_s16_x4() { + let a: [i16; 33] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [i16; 32] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let mut r: [i16; 32] = [0i16; 32]; + vst1q_s16_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_s32_x4() { + let a: [i32; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [i32; 16] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let mut r: [i32; 16] = [0i32; 16]; + vst1q_s32_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_s64_x4() { + let a: [i64; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [i64; 8] = [1, 2, 3, 4, 5, 6, 7, 8]; + let mut r: [i64; 8] = [0i64; 8]; + vst1q_s64_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_u8_x2() { + let a: [u8; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [u8; 16] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let mut r: [u8; 16] = [0u8; 16]; + vst1_u8_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_u16_x2() { + let a: [u16; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [u16; 8] = [1, 2, 3, 4, 5, 6, 7, 8]; + let mut r: [u16; 8] = [0u16; 8]; + vst1_u16_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_u32_x2() { + let a: [u32; 5] = [0, 1, 2, 3, 4]; + let e: [u32; 4] = [1, 2, 3, 4]; + let mut r: [u32; 4] = [0u32; 4]; + vst1_u32_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_u64_x2() { + let a: [u64; 3] = [0, 1, 2]; + let e: [u64; 2] = [1, 2]; + let mut r: [u64; 2] = [0u64; 2]; + vst1_u64_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_u8_x2() { + let a: [u8; 33] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [u8; 32] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let mut r: [u8; 32] = [0u8; 32]; + vst1q_u8_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_u16_x2() { + let a: [u16; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [u16; 16] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let mut r: [u16; 16] = [0u16; 16]; + vst1q_u16_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_u32_x2() { + let a: [u32; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [u32; 8] = [1, 2, 3, 4, 5, 6, 7, 8]; + let mut r: [u32; 8] = [0u32; 8]; + vst1q_u32_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_u64_x2() { + let a: [u64; 5] = [0, 1, 2, 3, 4]; + let e: [u64; 4] = [1, 2, 3, 4]; + let mut r: [u64; 4] = [0u64; 4]; + vst1q_u64_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_u8_x3() { + let a: [u8; 25] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]; + let e: [u8; 24] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]; + let mut r: [u8; 24] = [0u8; 24]; + vst1_u8_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_u16_x3() { + let a: [u16; 13] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + let e: [u16; 12] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + let mut r: [u16; 12] = [0u16; 12]; + vst1_u16_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_u32_x3() { + let a: [u32; 7] = [0, 1, 2, 3, 4, 5, 6]; + let e: [u32; 6] = [1, 2, 3, 4, 5, 6]; + let mut r: [u32; 6] = [0u32; 6]; + vst1_u32_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_u64_x3() { + let a: [u64; 4] = [0, 1, 2, 3]; + let e: [u64; 3] = [1, 2, 3]; + let mut r: [u64; 3] = [0u64; 3]; + vst1_u64_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_u8_x3() { + let a: [u8; 49] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [u8; 48] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let mut r: [u8; 48] = [0u8; 48]; + vst1q_u8_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_u16_x3() { + let a: [u16; 25] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]; + let e: [u16; 24] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]; + let mut r: [u16; 24] = [0u16; 24]; + vst1q_u16_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_u32_x3() { + let a: [u32; 13] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + let e: [u32; 12] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + let mut r: [u32; 12] = [0u32; 12]; + vst1q_u32_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_u64_x3() { + let a: [u64; 7] = [0, 1, 2, 3, 4, 5, 6]; + let e: [u64; 6] = [1, 2, 3, 4, 5, 6]; + let mut r: [u64; 6] = [0u64; 6]; + vst1q_u64_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_u8_x4() { + let a: [u8; 33] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [u8; 32] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let mut r: [u8; 32] = [0u8; 32]; + vst1_u8_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_u16_x4() { + let a: [u16; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [u16; 16] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let mut r: [u16; 16] = [0u16; 16]; + vst1_u16_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_u32_x4() { + let a: [u32; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [u32; 8] = [1, 2, 3, 4, 5, 6, 7, 8]; + let mut r: [u32; 8] = [0u32; 8]; + vst1_u32_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_u64_x4() { + let a: [u64; 5] = [0, 1, 2, 3, 4]; + let e: [u64; 4] = [1, 2, 3, 4]; + let mut r: [u64; 4] = [0u64; 4]; + vst1_u64_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_u8_x4() { + let a: [u8; 65] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [u8; 64] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let mut r: [u8; 64] = [0u8; 64]; + vst1q_u8_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_u16_x4() { + let a: [u16; 33] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [u16; 32] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let mut r: [u16; 32] = [0u16; 32]; + vst1q_u16_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_u32_x4() { + let a: [u32; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [u32; 16] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let mut r: [u32; 16] = [0u32; 16]; + vst1q_u32_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_u64_x4() { + let a: [u64; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [u64; 8] = [1, 2, 3, 4, 5, 6, 7, 8]; + let mut r: [u64; 8] = [0u64; 8]; + vst1q_u64_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_p8_x2() { + let a: [u8; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [u8; 16] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let mut r: [u8; 16] = [0u8; 16]; + vst1_p8_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_p8_x3() { + let a: [u8; 25] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]; + let e: [u8; 24] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]; + let mut r: [u8; 24] = [0u8; 24]; + vst1_p8_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_p8_x4() { + let a: [u8; 33] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [u8; 32] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let mut r: [u8; 32] = [0u8; 32]; + vst1_p8_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_p8_x2() { + let a: [u8; 33] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [u8; 32] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let mut r: [u8; 32] = [0u8; 32]; + vst1q_p8_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_p8_x3() { + let a: [u8; 49] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [u8; 48] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let mut r: [u8; 48] = [0u8; 48]; + vst1q_p8_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_p8_x4() { + let a: [u8; 65] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [u8; 64] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let mut r: [u8; 64] = [0u8; 64]; + vst1q_p8_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_p16_x2() { + let a: [u16; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [u16; 8] = [1, 2, 3, 4, 5, 6, 7, 8]; + let mut r: [u16; 8] = [0u16; 8]; + vst1_p16_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_p16_x3() { + let a: [u16; 13] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + let e: [u16; 12] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + let mut r: [u16; 12] = [0u16; 12]; + vst1_p16_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_p16_x4() { + let a: [u16; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [u16; 16] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let mut r: [u16; 16] = [0u16; 16]; + vst1_p16_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_p16_x2() { + let a: [u16; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e: [u16; 16] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let mut r: [u16; 16] = [0u16; 16]; + vst1q_p16_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_p16_x3() { + let a: [u16; 25] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]; + let e: [u16; 24] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]; + let mut r: [u16; 24] = [0u16; 24]; + vst1q_p16_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_p16_x4() { + let a: [u16; 33] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let e: [u16; 32] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let mut r: [u16; 32] = [0u16; 32]; + vst1q_p16_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_p64_x2() { + let a: [u64; 3] = [0, 1, 2]; + let e: [u64; 2] = [1, 2]; + let mut r: [u64; 2] = [0u64; 2]; + vst1_p64_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_p64_x3() { + let a: [u64; 4] = [0, 1, 2, 3]; + let e: [u64; 3] = [1, 2, 3]; + let mut r: [u64; 3] = [0u64; 3]; + vst1_p64_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_p64_x4() { + let a: [u64; 5] = [0, 1, 2, 3, 4]; + let e: [u64; 4] = [1, 2, 3, 4]; + let mut r: [u64; 4] = [0u64; 4]; + vst1_p64_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_p64_x2() { + let a: [u64; 5] = [0, 1, 2, 3, 4]; + let e: [u64; 4] = [1, 2, 3, 4]; + let mut r: [u64; 4] = [0u64; 4]; + vst1q_p64_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_p64_x3() { + let a: [u64; 7] = [0, 1, 2, 3, 4, 5, 6]; + let e: [u64; 6] = [1, 2, 3, 4, 5, 6]; + let mut r: [u64; 6] = [0u64; 6]; + vst1q_p64_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_p64_x4() { + let a: [u64; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e: [u64; 8] = [1, 2, 3, 4, 5, 6, 7, 8]; + let mut r: [u64; 8] = [0u64; 8]; + vst1q_p64_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_f32_x2() { + let a: [f32; 5] = [0., 1., 2., 3., 4.]; + let e: [f32; 4] = [1., 2., 3., 4.]; + let mut r: [f32; 4] = [0f32; 4]; + vst1_f32_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_f32_x2() { + let a: [f32; 9] = [0., 1., 2., 3., 4., 5., 6., 7., 8.]; + let e: [f32; 8] = [1., 2., 3., 4., 5., 6., 7., 8.]; + let mut r: [f32; 8] = [0f32; 8]; + vst1q_f32_x2(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_f32_x3() { + let a: [f32; 7] = [0., 1., 2., 3., 4., 5., 6.]; + let e: [f32; 6] = [1., 2., 3., 4., 5., 6.]; + let mut r: [f32; 6] = [0f32; 6]; + vst1_f32_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_f32_x3() { + let a: [f32; 13] = [0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12.]; + let e: [f32; 12] = [1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12.]; + let mut r: [f32; 12] = [0f32; 12]; + vst1q_f32_x3(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_f32_x4() { + let a: [f32; 9] = [0., 1., 2., 3., 4., 5., 6., 7., 8.]; + let e: [f32; 8] = [1., 2., 3., 4., 5., 6., 7., 8.]; + let mut r: [f32; 8] = [0f32; 8]; + vst1_f32_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_f32_x4() { + let a: [f32; 17] = [0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16.]; + let e: [f32; 16] = [1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16.]; + let mut r: [f32; 16] = [0f32; 16]; + vst1q_f32_x4(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_s8() { + let a: [i8; 17] = [0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9]; + let e: [i8; 16] = [1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9]; + let mut r: [i8; 16] = [0i8; 16]; + vst2_s8(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_s16() { + let a: [i16; 9] = [0, 1, 2, 2, 3, 2, 3, 4, 5]; + let e: [i16; 8] = [1, 2, 2, 3, 2, 4, 3, 5]; + let mut r: [i16; 8] = [0i16; 8]; + vst2_s16(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_s32() { + let a: [i32; 5] = [0, 1, 2, 2, 3]; + let e: [i32; 4] = [1, 2, 2, 3]; + let mut r: [i32; 4] = [0i32; 4]; + vst2_s32(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_s8() { + let a: [i8; 33] = [0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]; + let e: [i8; 32] = [1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15, 8, 16, 9, 17]; + let mut r: [i8; 32] = [0i8; 32]; + vst2q_s8(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_s16() { + let a: [i16; 17] = [0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9]; + let e: [i16; 16] = [1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9]; + let mut r: [i16; 16] = [0i16; 16]; + vst2q_s16(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_s32() { + let a: [i32; 9] = [0, 1, 2, 2, 3, 2, 3, 4, 5]; + let e: [i32; 8] = [1, 2, 2, 3, 2, 4, 3, 5]; + let mut r: [i32; 8] = [0i32; 8]; + vst2q_s32(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_s64() { + let a: [i64; 3] = [0, 1, 2]; + let e: [i64; 2] = [1, 2]; + let mut r: [i64; 2] = [0i64; 2]; + vst2_s64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_u8() { + let a: [u8; 17] = [0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9]; + let e: [u8; 16] = [1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9]; + let mut r: [u8; 16] = [0u8; 16]; + vst2_u8(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_u16() { + let a: [u16; 9] = [0, 1, 2, 2, 3, 2, 3, 4, 5]; + let e: [u16; 8] = [1, 2, 2, 3, 2, 4, 3, 5]; + let mut r: [u16; 8] = [0u16; 8]; + vst2_u16(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_u32() { + let a: [u32; 5] = [0, 1, 2, 2, 3]; + let e: [u32; 4] = [1, 2, 2, 3]; + let mut r: [u32; 4] = [0u32; 4]; + vst2_u32(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_u8() { + let a: [u8; 33] = [0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]; + let e: [u8; 32] = [1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15, 8, 16, 9, 17]; + let mut r: [u8; 32] = [0u8; 32]; + vst2q_u8(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_u16() { + let a: [u16; 17] = [0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9]; + let e: [u16; 16] = [1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9]; + let mut r: [u16; 16] = [0u16; 16]; + vst2q_u16(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_u32() { + let a: [u32; 9] = [0, 1, 2, 2, 3, 2, 3, 4, 5]; + let e: [u32; 8] = [1, 2, 2, 3, 2, 4, 3, 5]; + let mut r: [u32; 8] = [0u32; 8]; + vst2q_u32(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_p8() { + let a: [u8; 17] = [0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9]; + let e: [u8; 16] = [1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9]; + let mut r: [u8; 16] = [0u8; 16]; + vst2_p8(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_p16() { + let a: [u16; 9] = [0, 1, 2, 2, 3, 2, 3, 4, 5]; + let e: [u16; 8] = [1, 2, 2, 3, 2, 4, 3, 5]; + let mut r: [u16; 8] = [0u16; 8]; + vst2_p16(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_p8() { + let a: [u8; 33] = [0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]; + let e: [u8; 32] = [1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15, 8, 16, 9, 17]; + let mut r: [u8; 32] = [0u8; 32]; + vst2q_p8(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_p16() { + let a: [u16; 17] = [0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9]; + let e: [u16; 16] = [1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9]; + let mut r: [u16; 16] = [0u16; 16]; + vst2q_p16(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_u64() { + let a: [u64; 3] = [0, 1, 2]; + let e: [u64; 2] = [1, 2]; + let mut r: [u64; 2] = [0u64; 2]; + vst2_u64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_p64() { + let a: [u64; 3] = [0, 1, 2]; + let e: [u64; 2] = [1, 2]; + let mut r: [u64; 2] = [0u64; 2]; + vst2_p64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_f32() { + let a: [f32; 5] = [0., 1., 2., 2., 3.]; + let e: [f32; 4] = [1., 2., 2., 3.]; + let mut r: [f32; 4] = [0f32; 4]; + vst2_f32(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_f32() { + let a: [f32; 9] = [0., 1., 2., 2., 3., 2., 3., 4., 5.]; + let e: [f32; 8] = [1., 2., 2., 3., 2., 4., 3., 5.]; + let mut r: [f32; 8] = [0f32; 8]; + vst2q_f32(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_lane_s8() { + let a: [i8; 17] = [0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9]; + let e: [i8; 16] = [1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [i8; 16] = [0i8; 16]; + vst2_lane_s8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_lane_s16() { + let a: [i16; 9] = [0, 1, 2, 2, 3, 2, 3, 4, 5]; + let e: [i16; 8] = [1, 2, 0, 0, 0, 0, 0, 0]; + let mut r: [i16; 8] = [0i16; 8]; + vst2_lane_s16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_lane_s32() { + let a: [i32; 5] = [0, 1, 2, 2, 3]; + let e: [i32; 4] = [1, 2, 0, 0]; + let mut r: [i32; 4] = [0i32; 4]; + vst2_lane_s32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_lane_s16() { + let a: [i16; 17] = [0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9]; + let e: [i16; 16] = [1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [i16; 16] = [0i16; 16]; + vst2q_lane_s16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_lane_s32() { + let a: [i32; 9] = [0, 1, 2, 2, 3, 2, 3, 4, 5]; + let e: [i32; 8] = [1, 2, 0, 0, 0, 0, 0, 0]; + let mut r: [i32; 8] = [0i32; 8]; + vst2q_lane_s32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_lane_u8() { + let a: [u8; 17] = [0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9]; + let e: [u8; 16] = [1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u8; 16] = [0u8; 16]; + vst2_lane_u8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_lane_u16() { + let a: [u16; 9] = [0, 1, 2, 2, 3, 2, 3, 4, 5]; + let e: [u16; 8] = [1, 2, 0, 0, 0, 0, 0, 0]; + let mut r: [u16; 8] = [0u16; 8]; + vst2_lane_u16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_lane_u32() { + let a: [u32; 5] = [0, 1, 2, 2, 3]; + let e: [u32; 4] = [1, 2, 0, 0]; + let mut r: [u32; 4] = [0u32; 4]; + vst2_lane_u32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_lane_u16() { + let a: [u16; 17] = [0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9]; + let e: [u16; 16] = [1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u16; 16] = [0u16; 16]; + vst2q_lane_u16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_lane_u32() { + let a: [u32; 9] = [0, 1, 2, 2, 3, 2, 3, 4, 5]; + let e: [u32; 8] = [1, 2, 0, 0, 0, 0, 0, 0]; + let mut r: [u32; 8] = [0u32; 8]; + vst2q_lane_u32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_lane_p8() { + let a: [u8; 17] = [0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9]; + let e: [u8; 16] = [1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u8; 16] = [0u8; 16]; + vst2_lane_p8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_lane_p16() { + let a: [u16; 9] = [0, 1, 2, 2, 3, 2, 3, 4, 5]; + let e: [u16; 8] = [1, 2, 0, 0, 0, 0, 0, 0]; + let mut r: [u16; 8] = [0u16; 8]; + vst2_lane_p16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_lane_p16() { + let a: [u16; 17] = [0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9]; + let e: [u16; 16] = [1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u16; 16] = [0u16; 16]; + vst2q_lane_p16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2_lane_f32() { + let a: [f32; 5] = [0., 1., 2., 2., 3.]; + let e: [f32; 4] = [1., 2., 0., 0.]; + let mut r: [f32; 4] = [0f32; 4]; + vst2_lane_f32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst2q_lane_f32() { + let a: [f32; 9] = [0., 1., 2., 2., 3., 2., 3., 4., 5.]; + let e: [f32; 8] = [1., 2., 0., 0., 0., 0., 0., 0.]; + let mut r: [f32; 8] = [0f32; 8]; + vst2q_lane_f32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_s8() { + let a: [i8; 25] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16]; + let e: [i8; 24] = [1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16]; + let mut r: [i8; 24] = [0i8; 24]; + vst3_s8(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_s16() { + let a: [i16; 13] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8]; + let e: [i16; 12] = [1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8]; + let mut r: [i16; 12] = [0i16; 12]; + vst3_s16(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_s32() { + let a: [i32; 7] = [0, 1, 2, 2, 4, 2, 4]; + let e: [i32; 6] = [1, 2, 2, 2, 4, 4]; + let mut r: [i32; 6] = [0i32; 6]; + vst3_s32(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_s8() { + let a: [i8; 49] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16, 25, 26, 27, 28, 29, 30, 31, 32, 2, 4, 7, 8, 13, 14, 15, 16, 41, 42, 43, 44, 45, 46, 47, 48]; + let e: [i8; 48] = [1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16, 2, 25, 41, 4, 26, 42, 7, 27, 43, 8, 28, 44, 13, 29, 45, 14, 30, 46, 15, 31, 47, 16, 32, 48]; + let mut r: [i8; 48] = [0i8; 48]; + vst3q_s8(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_s16() { + let a: [i16; 25] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16]; + let e: [i16; 24] = [1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16]; + let mut r: [i16; 24] = [0i16; 24]; + vst3q_s16(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_s32() { + let a: [i32; 13] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8]; + let e: [i32; 12] = [1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8]; + let mut r: [i32; 12] = [0i32; 12]; + vst3q_s32(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_s64() { + let a: [i64; 4] = [0, 1, 2, 2]; + let e: [i64; 3] = [1, 2, 2]; + let mut r: [i64; 3] = [0i64; 3]; + vst3_s64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_u8() { + let a: [u8; 25] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16]; + let e: [u8; 24] = [1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16]; + let mut r: [u8; 24] = [0u8; 24]; + vst3_u8(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_u16() { + let a: [u16; 13] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8]; + let e: [u16; 12] = [1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8]; + let mut r: [u16; 12] = [0u16; 12]; + vst3_u16(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_u32() { + let a: [u32; 7] = [0, 1, 2, 2, 4, 2, 4]; + let e: [u32; 6] = [1, 2, 2, 2, 4, 4]; + let mut r: [u32; 6] = [0u32; 6]; + vst3_u32(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_u8() { + let a: [u8; 49] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16, 25, 26, 27, 28, 29, 30, 31, 32, 2, 4, 7, 8, 13, 14, 15, 16, 41, 42, 43, 44, 45, 46, 47, 48]; + let e: [u8; 48] = [1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16, 2, 25, 41, 4, 26, 42, 7, 27, 43, 8, 28, 44, 13, 29, 45, 14, 30, 46, 15, 31, 47, 16, 32, 48]; + let mut r: [u8; 48] = [0u8; 48]; + vst3q_u8(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_u16() { + let a: [u16; 25] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16]; + let e: [u16; 24] = [1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16]; + let mut r: [u16; 24] = [0u16; 24]; + vst3q_u16(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_u32() { + let a: [u32; 13] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8]; + let e: [u32; 12] = [1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8]; + let mut r: [u32; 12] = [0u32; 12]; + vst3q_u32(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_p8() { + let a: [u8; 25] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16]; + let e: [u8; 24] = [1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16]; + let mut r: [u8; 24] = [0u8; 24]; + vst3_p8(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_p16() { + let a: [u16; 13] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8]; + let e: [u16; 12] = [1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8]; + let mut r: [u16; 12] = [0u16; 12]; + vst3_p16(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_p8() { + let a: [u8; 49] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16, 25, 26, 27, 28, 29, 30, 31, 32, 2, 4, 7, 8, 13, 14, 15, 16, 41, 42, 43, 44, 45, 46, 47, 48]; + let e: [u8; 48] = [1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16, 2, 25, 41, 4, 26, 42, 7, 27, 43, 8, 28, 44, 13, 29, 45, 14, 30, 46, 15, 31, 47, 16, 32, 48]; + let mut r: [u8; 48] = [0u8; 48]; + vst3q_p8(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_p16() { + let a: [u16; 25] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16]; + let e: [u16; 24] = [1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16]; + let mut r: [u16; 24] = [0u16; 24]; + vst3q_p16(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_u64() { + let a: [u64; 4] = [0, 1, 2, 2]; + let e: [u64; 3] = [1, 2, 2]; + let mut r: [u64; 3] = [0u64; 3]; + vst3_u64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_p64() { + let a: [u64; 4] = [0, 1, 2, 2]; + let e: [u64; 3] = [1, 2, 2]; + let mut r: [u64; 3] = [0u64; 3]; + vst3_p64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_f32() { + let a: [f32; 7] = [0., 1., 2., 2., 4., 2., 4.]; + let e: [f32; 6] = [1., 2., 2., 2., 4., 4.]; + let mut r: [f32; 6] = [0f32; 6]; + vst3_f32(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_f32() { + let a: [f32; 13] = [0., 1., 2., 2., 4., 2., 4., 7., 8., 2., 4., 7., 8.]; + let e: [f32; 12] = [1., 2., 2., 2., 4., 4., 2., 7., 7., 4., 8., 8.]; + let mut r: [f32; 12] = [0f32; 12]; + vst3q_f32(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_lane_s8() { + let a: [i8; 25] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16]; + let e: [i8; 24] = [1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [i8; 24] = [0i8; 24]; + vst3_lane_s8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_lane_s16() { + let a: [i16; 13] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8]; + let e: [i16; 12] = [1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [i16; 12] = [0i16; 12]; + vst3_lane_s16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_lane_s32() { + let a: [i32; 7] = [0, 1, 2, 2, 4, 2, 4]; + let e: [i32; 6] = [1, 2, 2, 0, 0, 0]; + let mut r: [i32; 6] = [0i32; 6]; + vst3_lane_s32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_lane_s16() { + let a: [i16; 25] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16]; + let e: [i16; 24] = [1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [i16; 24] = [0i16; 24]; + vst3q_lane_s16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_lane_s32() { + let a: [i32; 13] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8]; + let e: [i32; 12] = [1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [i32; 12] = [0i32; 12]; + vst3q_lane_s32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_lane_u8() { + let a: [u8; 25] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16]; + let e: [u8; 24] = [1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u8; 24] = [0u8; 24]; + vst3_lane_u8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_lane_u16() { + let a: [u16; 13] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8]; + let e: [u16; 12] = [1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u16; 12] = [0u16; 12]; + vst3_lane_u16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_lane_u32() { + let a: [u32; 7] = [0, 1, 2, 2, 4, 2, 4]; + let e: [u32; 6] = [1, 2, 2, 0, 0, 0]; + let mut r: [u32; 6] = [0u32; 6]; + vst3_lane_u32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_lane_u16() { + let a: [u16; 25] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16]; + let e: [u16; 24] = [1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u16; 24] = [0u16; 24]; + vst3q_lane_u16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_lane_u32() { + let a: [u32; 13] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8]; + let e: [u32; 12] = [1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u32; 12] = [0u32; 12]; + vst3q_lane_u32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_lane_p8() { + let a: [u8; 25] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16]; + let e: [u8; 24] = [1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u8; 24] = [0u8; 24]; + vst3_lane_p8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_lane_p16() { + let a: [u16; 13] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8]; + let e: [u16; 12] = [1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u16; 12] = [0u16; 12]; + vst3_lane_p16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_lane_p16() { + let a: [u16; 25] = [0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16]; + let e: [u16; 24] = [1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u16; 24] = [0u16; 24]; + vst3q_lane_p16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3_lane_f32() { + let a: [f32; 7] = [0., 1., 2., 2., 3., 2., 3.]; + let e: [f32; 6] = [1., 2., 2., 0., 0., 0.]; + let mut r: [f32; 6] = [0f32; 6]; + vst3_lane_f32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst3q_lane_f32() { + let a: [f32; 13] = [0., 1., 2., 2., 3., 2., 3., 4., 5., 2., 3., 4., 5.]; + let e: [f32; 12] = [1., 2., 2., 0., 0., 0., 0., 0., 0., 0., 0., 0.]; + let mut r: [f32; 12] = [0f32; 12]; + vst3q_lane_f32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_s8() { + let a: [i8; 33] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let e: [i8; 32] = [1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let mut r: [i8; 32] = [0i8; 32]; + vst4_s8(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_s16() { + let a: [i16; 17] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16]; + let e: [i16; 16] = [1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16]; + let mut r: [i16; 16] = [0i16; 16]; + vst4_s16(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_s32() { + let a: [i32; 9] = [0, 1, 2, 2, 6, 2, 6, 6, 8]; + let e: [i32; 8] = [1, 2, 2, 6, 2, 6, 6, 8]; + let mut r: [i32; 8] = [0i32; 8]; + vst4_s32(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_s8() { + let a: [i8; 65] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64]; + let e: [i8; 64] = [1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64]; + let mut r: [i8; 64] = [0i8; 64]; + vst4q_s8(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_s16() { + let a: [i16; 33] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let e: [i16; 32] = [1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let mut r: [i16; 32] = [0i16; 32]; + vst4q_s16(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_s32() { + let a: [i32; 17] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16]; + let e: [i32; 16] = [1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16]; + let mut r: [i32; 16] = [0i32; 16]; + vst4q_s32(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_s64() { + let a: [i64; 5] = [0, 1, 2, 2, 6]; + let e: [i64; 4] = [1, 2, 2, 6]; + let mut r: [i64; 4] = [0i64; 4]; + vst4_s64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_u8() { + let a: [u8; 33] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let e: [u8; 32] = [1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let mut r: [u8; 32] = [0u8; 32]; + vst4_u8(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_u16() { + let a: [u16; 17] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16]; + let e: [u16; 16] = [1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16]; + let mut r: [u16; 16] = [0u16; 16]; + vst4_u16(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_u32() { + let a: [u32; 9] = [0, 1, 2, 2, 6, 2, 6, 6, 8]; + let e: [u32; 8] = [1, 2, 2, 6, 2, 6, 6, 8]; + let mut r: [u32; 8] = [0u32; 8]; + vst4_u32(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_u8() { + let a: [u8; 65] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64]; + let e: [u8; 64] = [1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64]; + let mut r: [u8; 64] = [0u8; 64]; + vst4q_u8(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_u16() { + let a: [u16; 33] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let e: [u16; 32] = [1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let mut r: [u16; 32] = [0u16; 32]; + vst4q_u16(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_u32() { + let a: [u32; 17] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16]; + let e: [u32; 16] = [1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16]; + let mut r: [u32; 16] = [0u32; 16]; + vst4q_u32(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_p8() { + let a: [u8; 33] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let e: [u8; 32] = [1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let mut r: [u8; 32] = [0u8; 32]; + vst4_p8(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_p16() { + let a: [u16; 17] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16]; + let e: [u16; 16] = [1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16]; + let mut r: [u16; 16] = [0u16; 16]; + vst4_p16(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_p8() { + let a: [u8; 65] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64]; + let e: [u8; 64] = [1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64]; + let mut r: [u8; 64] = [0u8; 64]; + vst4q_p8(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_p16() { + let a: [u16; 33] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let e: [u16; 32] = [1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let mut r: [u16; 32] = [0u16; 32]; + vst4q_p16(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_u64() { + let a: [u64; 5] = [0, 1, 2, 2, 6]; + let e: [u64; 4] = [1, 2, 2, 6]; + let mut r: [u64; 4] = [0u64; 4]; + vst4_u64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_p64() { + let a: [u64; 5] = [0, 1, 2, 2, 6]; + let e: [u64; 4] = [1, 2, 2, 6]; + let mut r: [u64; 4] = [0u64; 4]; + vst4_p64(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_f32() { + let a: [f32; 9] = [0., 1., 2., 2., 6., 2., 6., 6., 8.]; + let e: [f32; 8] = [1., 2., 2., 6., 2., 6., 6., 8.]; + let mut r: [f32; 8] = [0f32; 8]; + vst4_f32(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_f32() { + let a: [f32; 17] = [0., 1., 2., 2., 6., 2., 6., 6., 8., 2., 6., 6., 8., 6., 8., 8., 16.]; + let e: [f32; 16] = [1., 2., 2., 6., 2., 6., 6., 8., 2., 6., 6., 8., 6., 8., 8., 16.]; + let mut r: [f32; 16] = [0f32; 16]; + vst4q_f32(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_lane_s8() { + let a: [i8; 33] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let e: [i8; 32] = [1, 2, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [i8; 32] = [0i8; 32]; + vst4_lane_s8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_lane_s16() { + let a: [i16; 17] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16]; + let e: [i16; 16] = [1, 2, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [i16; 16] = [0i16; 16]; + vst4_lane_s16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_lane_s32() { + let a: [i32; 9] = [0, 1, 2, 2, 6, 2, 6, 6, 8]; + let e: [i32; 8] = [1, 2, 2, 6, 0, 0, 0, 0]; + let mut r: [i32; 8] = [0i32; 8]; + vst4_lane_s32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_lane_s16() { + let a: [i16; 33] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let e: [i16; 32] = [1, 2, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [i16; 32] = [0i16; 32]; + vst4q_lane_s16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_lane_s32() { + let a: [i32; 17] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16]; + let e: [i32; 16] = [1, 2, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [i32; 16] = [0i32; 16]; + vst4q_lane_s32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_lane_u8() { + let a: [u8; 33] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let e: [u8; 32] = [1, 2, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u8; 32] = [0u8; 32]; + vst4_lane_u8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_lane_u16() { + let a: [u16; 17] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16]; + let e: [u16; 16] = [1, 2, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u16; 16] = [0u16; 16]; + vst4_lane_u16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_lane_u32() { + let a: [u32; 9] = [0, 1, 2, 2, 6, 2, 6, 6, 8]; + let e: [u32; 8] = [1, 2, 2, 6, 0, 0, 0, 0]; + let mut r: [u32; 8] = [0u32; 8]; + vst4_lane_u32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_lane_u16() { + let a: [u16; 33] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let e: [u16; 32] = [1, 2, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u16; 32] = [0u16; 32]; + vst4q_lane_u16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_lane_u32() { + let a: [u32; 17] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16]; + let e: [u32; 16] = [1, 2, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u32; 16] = [0u32; 16]; + vst4q_lane_u32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_lane_p8() { + let a: [u8; 33] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let e: [u8; 32] = [1, 2, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u8; 32] = [0u8; 32]; + vst4_lane_p8::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_lane_p16() { + let a: [u16; 17] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16]; + let e: [u16; 16] = [1, 2, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u16; 16] = [0u16; 16]; + vst4_lane_p16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_lane_p16() { + let a: [u16; 33] = [0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32]; + let e: [u16; 32] = [1, 2, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut r: [u16; 32] = [0u16; 32]; + vst4q_lane_p16::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4_lane_f32() { + let a: [f32; 9] = [0., 1., 2., 2., 6., 2., 6., 6., 8.]; + let e: [f32; 8] = [1., 2., 2., 6., 0., 0., 0., 0.]; + let mut r: [f32; 8] = [0f32; 8]; + vst4_lane_f32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst4q_lane_f32() { + let a: [f32; 17] = [0., 1., 2., 2., 6., 2., 6., 6., 8., 2., 6., 6., 8., 6., 8., 8., 16.]; + let e: [f32; 16] = [1., 2., 2., 6., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]; + let mut r: [f32; 16] = [0f32; 16]; + vst4q_lane_f32::<0>(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_s8() { + let a: i8x8 = i8x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i8x8 = i8x8::new(1, 4, 3, 8, 5, 12, 7, 16); + let r: i8x8 = transmute(vmul_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_s8() { + let a: i8x16 = i8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); + let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(1, 4, 3, 8, 5, 12, 7, 16, 9, 20, 11, 24, 13, 28, 15, 32); + let r: i8x16 = transmute(vmulq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_s16() { + let a: i16x4 = i16x4::new(1, 2, 1, 2); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let e: i16x4 = i16x4::new(1, 4, 3, 8); + let r: i16x4 = transmute(vmul_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_s16() { + let a: i16x8 = i16x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i16x8 = i16x8::new(1, 4, 3, 8, 5, 12, 7, 16); + let r: i16x8 = transmute(vmulq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(1, 2); + let e: i32x2 = i32x2::new(1, 4); + let r: i32x2 = transmute(vmul_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_s32() { + let a: i32x4 = i32x4::new(1, 2, 1, 2); + let b: i32x4 = i32x4::new(1, 2, 3, 4); + let e: i32x4 = i32x4::new(1, 4, 3, 8); + let r: i32x4 = transmute(vmulq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_u8() { + let a: u8x8 = u8x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(1, 4, 3, 8, 5, 12, 7, 16); + let r: u8x8 = transmute(vmul_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_u8() { + let a: u8x16 = u8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); + let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(1, 4, 3, 8, 5, 12, 7, 16, 9, 20, 11, 24, 13, 28, 15, 32); + let r: u8x16 = transmute(vmulq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_u16() { + let a: u16x4 = u16x4::new(1, 2, 1, 2); + let b: u16x4 = u16x4::new(1, 2, 3, 4); + let e: u16x4 = u16x4::new(1, 4, 3, 8); + let r: u16x4 = transmute(vmul_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_u16() { + let a: u16x8 = u16x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(1, 4, 3, 8, 5, 12, 7, 16); + let r: u16x8 = transmute(vmulq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(1, 2); + let e: u32x2 = u32x2::new(1, 4); + let r: u32x2 = transmute(vmul_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_u32() { + let a: u32x4 = u32x4::new(1, 2, 1, 2); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(1, 4, 3, 8); + let r: u32x4 = transmute(vmulq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_p8() { + let a: i8x8 = i8x8::new(1, 3, 1, 3, 1, 3, 1, 3); + let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i8x8 = i8x8::new(1, 6, 3, 12, 5, 10, 7, 24); + let r: i8x8 = transmute(vmul_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_p8() { + let a: i8x16 = i8x16::new(1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3); + let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(1, 6, 3, 12, 5, 10, 7, 24, 9, 30, 11, 20, 13, 18, 15, 48); + let r: i8x16 = transmute(vmulq_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_f32() { + let a: f32x2 = f32x2::new(1.0, 2.0); + let b: f32x2 = f32x2::new(2.0, 3.0); + let e: f32x2 = f32x2::new(2.0, 6.0); + let r: f32x2 = transmute(vmul_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_f32() { + let a: f32x4 = f32x4::new(1.0, 2.0, 1.0, 2.0); + let b: f32x4 = f32x4::new(2.0, 3.0, 4.0, 5.0); + let e: f32x4 = f32x4::new(2.0, 6.0, 4.0, 10.0); + let r: f32x4 = transmute(vmulq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_n_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16 = 2; + let e: i16x4 = i16x4::new(2, 4, 6, 8); + let r: i16x4 = transmute(vmul_n_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_n_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16 = 2; + let e: i16x8 = i16x8::new(2, 4, 6, 8, 10, 12, 14, 16); + let r: i16x8 = transmute(vmulq_n_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_n_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32 = 2; + let e: i32x2 = i32x2::new(2, 4); + let r: i32x2 = transmute(vmul_n_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_n_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32 = 2; + let e: i32x4 = i32x4::new(2, 4, 6, 8); + let r: i32x4 = transmute(vmulq_n_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_n_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16 = 2; + let e: u16x4 = u16x4::new(2, 4, 6, 8); + let r: u16x4 = transmute(vmul_n_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_n_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16 = 2; + let e: u16x8 = u16x8::new(2, 4, 6, 8, 10, 12, 14, 16); + let r: u16x8 = transmute(vmulq_n_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_n_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32 = 2; + let e: u32x2 = u32x2::new(2, 4); + let r: u32x2 = transmute(vmul_n_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_n_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32 = 2; + let e: u32x4 = u32x4::new(2, 4, 6, 8); + let r: u32x4 = transmute(vmulq_n_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_n_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let b: f32 = 2.; + let e: f32x2 = f32x2::new(2., 4.); + let r: f32x2 = transmute(vmul_n_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_n_f32() { + let a: f32x4 = f32x4::new(1., 2., 3., 4.); + let b: f32 = 2.; + let e: f32x4 = f32x4::new(2., 4., 6., 8.); + let r: f32x4 = transmute(vmulq_n_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_lane_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16x4 = i16x4::new(2, 4, 6, 8); + let r: i16x4 = transmute(vmul_lane_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_laneq_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16x4 = i16x4::new(2, 4, 6, 8); + let r: i16x4 = transmute(vmul_laneq_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_lane_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16x8 = i16x8::new(2, 4, 6, 8, 10, 12, 14, 16); + let r: i16x8 = transmute(vmulq_lane_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_laneq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16x8 = i16x8::new(2, 4, 6, 8, 10, 12, 14, 16); + let r: i16x8 = transmute(vmulq_laneq_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_lane_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(0, 2); + let e: i32x2 = i32x2::new(2, 4); + let r: i32x2 = transmute(vmul_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_laneq_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32x2 = i32x2::new(2, 4); + let r: i32x2 = transmute(vmul_laneq_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_lane_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x2 = i32x2::new(0, 2); + let e: i32x4 = i32x4::new(2, 4, 6, 8); + let r: i32x4 = transmute(vmulq_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_laneq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32x4 = i32x4::new(2, 4, 6, 8); + let r: i32x4 = transmute(vmulq_laneq_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_lane_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(0, 2, 0, 0); + let e: u16x4 = u16x4::new(2, 4, 6, 8); + let r: u16x4 = transmute(vmul_lane_u16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_laneq_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x8 = u16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: u16x4 = u16x4::new(2, 4, 6, 8); + let r: u16x4 = transmute(vmul_laneq_u16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_lane_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x4 = u16x4::new(0, 2, 0, 0); + let e: u16x8 = u16x8::new(2, 4, 6, 8, 10, 12, 14, 16); + let r: u16x8 = transmute(vmulq_lane_u16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_laneq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x8 = u16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: u16x8 = u16x8::new(2, 4, 6, 8, 10, 12, 14, 16); + let r: u16x8 = transmute(vmulq_laneq_u16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_lane_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(0, 2); + let e: u32x2 = u32x2::new(2, 4); + let r: u32x2 = transmute(vmul_lane_u32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_laneq_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x4 = u32x4::new(0, 2, 0, 0); + let e: u32x2 = u32x2::new(2, 4); + let r: u32x2 = transmute(vmul_laneq_u32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_lane_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x2 = u32x2::new(0, 2); + let e: u32x4 = u32x4::new(2, 4, 6, 8); + let r: u32x4 = transmute(vmulq_lane_u32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_laneq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(0, 2, 0, 0); + let e: u32x4 = u32x4::new(2, 4, 6, 8); + let r: u32x4 = transmute(vmulq_laneq_u32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_lane_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let b: f32x2 = f32x2::new(2., 0.); + let e: f32x2 = f32x2::new(2., 4.); + let r: f32x2 = transmute(vmul_lane_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_laneq_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let b: f32x4 = f32x4::new(2., 0., 0., 0.); + let e: f32x2 = f32x2::new(2., 4.); + let r: f32x2 = transmute(vmul_laneq_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_lane_f32() { + let a: f32x4 = f32x4::new(1., 2., 3., 4.); + let b: f32x2 = f32x2::new(2., 0.); + let e: f32x4 = f32x4::new(2., 4., 6., 8.); + let r: f32x4 = transmute(vmulq_lane_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_laneq_f32() { + let a: f32x4 = f32x4::new(1., 2., 3., 4.); + let b: f32x4 = f32x4::new(2., 0., 0., 0.); + let e: f32x4 = f32x4::new(2., 4., 6., 8.); + let r: f32x4 = transmute(vmulq_laneq_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: i16x8 = i16x8::new(1, 4, 3, 8, 5, 12, 7, 16); + let r: i16x8 = transmute(vmull_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(1, 2, 1, 2); + let e: i32x4 = i32x4::new(1, 4, 3, 8); + let r: i32x4 = transmute(vmull_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(1, 2); + let e: i64x2 = i64x2::new(1, 4); + let r: i64x2 = transmute(vmull_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x8 = u8x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: u16x8 = u16x8::new(1, 4, 3, 8, 5, 12, 7, 16); + let r: u16x8 = transmute(vmull_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(1, 2, 1, 2); + let e: u32x4 = u32x4::new(1, 4, 3, 8); + let r: u32x4 = transmute(vmull_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(1, 2); + let e: u64x2 = u64x2::new(1, 4); + let r: u64x2 = transmute(vmull_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_p8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(1, 3, 1, 3, 1, 3, 1, 3); + let e: i16x8 = i16x8::new(1, 6, 3, 12, 5, 10, 7, 24); + let r: i16x8 = transmute(vmull_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_n_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16 = 2; + let e: i32x4 = i32x4::new(2, 4, 6, 8); + let r: i32x4 = transmute(vmull_n_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_n_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32 = 2; + let e: i64x2 = i64x2::new(2, 4); + let r: i64x2 = transmute(vmull_n_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_n_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16 = 2; + let e: u32x4 = u32x4::new(2, 4, 6, 8); + let r: u32x4 = transmute(vmull_n_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_n_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32 = 2; + let e: u64x2 = u64x2::new(2, 4); + let r: u64x2 = transmute(vmull_n_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_lane_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i32x4 = i32x4::new(2, 4, 6, 8); + let r: i32x4 = transmute(vmull_lane_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_laneq_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i32x4 = i32x4::new(2, 4, 6, 8); + let r: i32x4 = transmute(vmull_laneq_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_lane_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(0, 2); + let e: i64x2 = i64x2::new(2, 4); + let r: i64x2 = transmute(vmull_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_laneq_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i64x2 = i64x2::new(2, 4); + let r: i64x2 = transmute(vmull_laneq_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_lane_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(0, 2, 0, 0); + let e: u32x4 = u32x4::new(2, 4, 6, 8); + let r: u32x4 = transmute(vmull_lane_u16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_laneq_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x8 = u16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: u32x4 = u32x4::new(2, 4, 6, 8); + let r: u32x4 = transmute(vmull_laneq_u16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_lane_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(0, 2); + let e: u64x2 = u64x2::new(2, 4); + let r: u64x2 = transmute(vmull_lane_u32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_laneq_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x4 = u32x4::new(0, 2, 0, 0); + let e: u64x2 = u64x2::new(2, 4); + let r: u64x2 = transmute(vmull_laneq_u32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfma_f32() { + let a: f32x2 = f32x2::new(8.0, 18.0); + let b: f32x2 = f32x2::new(6.0, 4.0); + let c: f32x2 = f32x2::new(2.0, 3.0); + let e: f32x2 = f32x2::new(20.0, 30.0); + let r: f32x2 = transmute(vfma_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmaq_f32() { + let a: f32x4 = f32x4::new(8.0, 18.0, 12.0, 10.0); + let b: f32x4 = f32x4::new(6.0, 4.0, 7.0, 8.0); + let c: f32x4 = f32x4::new(2.0, 3.0, 4.0, 5.0); + let e: f32x4 = f32x4::new(20.0, 30.0, 40.0, 50.0); + let r: f32x4 = transmute(vfmaq_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfma_n_f32() { + let a: f32x2 = f32x2::new(2.0, 3.0); + let b: f32x2 = f32x2::new(6.0, 4.0); + let c: f32 = 8.0; + let e: f32x2 = f32x2::new(50.0, 35.0); + let r: f32x2 = transmute(vfma_n_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmaq_n_f32() { + let a: f32x4 = f32x4::new(2.0, 3.0, 4.0, 5.0); + let b: f32x4 = f32x4::new(6.0, 4.0, 7.0, 8.0); + let c: f32 = 8.0; + let e: f32x4 = f32x4::new(50.0, 35.0, 60.0, 69.0); + let r: f32x4 = transmute(vfmaq_n_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfms_f32() { + let a: f32x2 = f32x2::new(20.0, 30.0); + let b: f32x2 = f32x2::new(6.0, 4.0); + let c: f32x2 = f32x2::new(2.0, 3.0); + let e: f32x2 = f32x2::new(8.0, 18.0); + let r: f32x2 = transmute(vfms_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmsq_f32() { + let a: f32x4 = f32x4::new(20.0, 30.0, 40.0, 50.0); + let b: f32x4 = f32x4::new(6.0, 4.0, 7.0, 8.0); + let c: f32x4 = f32x4::new(2.0, 3.0, 4.0, 5.0); + let e: f32x4 = f32x4::new(8.0, 18.0, 12.0, 10.0); + let r: f32x4 = transmute(vfmsq_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfms_n_f32() { + let a: f32x2 = f32x2::new(50.0, 35.0); + let b: f32x2 = f32x2::new(6.0, 4.0); + let c: f32 = 8.0; + let e: f32x2 = f32x2::new(2.0, 3.0); + let r: f32x2 = transmute(vfms_n_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmsq_n_f32() { + let a: f32x4 = f32x4::new(50.0, 35.0, 60.0, 69.0); + let b: f32x4 = f32x4::new(6.0, 4.0, 7.0, 8.0); + let c: f32 = 8.0; + let e: f32x4 = f32x4::new(2.0, 3.0, 4.0, 5.0); + let r: f32x4 = transmute(vfmsq_n_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: i8x8 = i8x8::new(0, 0, 2, 2, 4, 4, 6, 6); + let r: i8x8 = transmute(vsub_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); + let e: i8x16 = i8x16::new(0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14); + let r: i8x16 = transmute(vsubq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(1, 2, 1, 2); + let e: i16x4 = i16x4::new(0, 0, 2, 2); + let r: i16x4 = transmute(vsub_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: i16x8 = i16x8::new(0, 0, 2, 2, 4, 4, 6, 6); + let r: i16x8 = transmute(vsubq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(1, 2); + let e: i32x2 = i32x2::new(0, 0); + let r: i32x2 = transmute(vsub_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(1, 2, 1, 2); + let e: i32x4 = i32x4::new(0, 0, 2, 2); + let r: i32x4 = transmute(vsubq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x8 = u8x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: u8x8 = u8x8::new(0, 0, 2, 2, 4, 4, 6, 6); + let r: u8x8 = transmute(vsub_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); + let e: u8x16 = u8x16::new(0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14); + let r: u8x16 = transmute(vsubq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(1, 2, 1, 2); + let e: u16x4 = u16x4::new(0, 0, 2, 2); + let r: u16x4 = transmute(vsub_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x8 = u16x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: u16x8 = u16x8::new(0, 0, 2, 2, 4, 4, 6, 6); + let r: u16x8 = transmute(vsubq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(1, 2); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vsub_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(1, 2, 1, 2); + let e: u32x4 = u32x4::new(0, 0, 2, 2); + let r: u32x4 = transmute(vsubq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_s64() { + let a: i64x1 = i64x1::new(1); + let b: i64x1 = i64x1::new(1); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vsub_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_s64() { + let a: i64x2 = i64x2::new(1, 2); + let b: i64x2 = i64x2::new(1, 2); + let e: i64x2 = i64x2::new(0, 0); + let r: i64x2 = transmute(vsubq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_u64() { + let a: u64x1 = u64x1::new(1); + let b: u64x1 = u64x1::new(1); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vsub_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_u64() { + let a: u64x2 = u64x2::new(1, 2); + let b: u64x2 = u64x2::new(1, 2); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vsubq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_f32() { + let a: f32x2 = f32x2::new(1.0, 4.0); + let b: f32x2 = f32x2::new(1.0, 2.0); + let e: f32x2 = f32x2::new(0.0, 2.0); + let r: f32x2 = transmute(vsub_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_f32() { + let a: f32x4 = f32x4::new(1.0, 4.0, 3.0, 8.0); + let b: f32x4 = f32x4::new(1.0, 2.0, 3.0, 4.0); + let e: f32x4 = f32x4::new(0.0, 2.0, 0.0, 4.0); + let r: f32x4 = transmute(vsubq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vadd_p8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let e: i8x8 = i8x8::new(0, 3, 2, 5, 4, 7, 6, 9); + let r: i8x8 = transmute(vadd_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vadd_p16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(1, 1, 1, 1); + let e: i16x4 = i16x4::new(0, 3, 2, 5); + let r: i16x4 = transmute(vadd_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddq_p8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let e: i8x16 = i8x16::new(0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, 17); + let r: i8x16 = transmute(vaddq_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddq_p16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let e: i16x8 = i16x8::new(0, 3, 2, 5, 4, 7, 6, 9); + let r: i16x8 = transmute(vaddq_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vadd_p64() { + let a: i64x1 = i64x1::new(1); + let b: i64x1 = i64x1::new(1); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vadd_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddq_p64() { + let a: i64x2 = i64x2::new(1, 2); + let b: i64x2 = i64x2::new(1, 1); + let e: i64x2 = i64x2::new(0, 3); + let r: i64x2 = transmute(vaddq_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddq_p128() { + let a: p128 = 16; + let b: p128 = 1; + let e: p128 = 17; + let r: p128 = transmute(vaddq_p128(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_s16() { + let a: i16x8 = i16x8::new(0x7F_FF, -32768, 1, 1, 0x7F_FF, -32768, 1, 1); + let b: i16x8 = i16x8::new(1, 0, 0, 0, 1, 0, 0, 0); + let e: i8x8 = i8x8::new(0x7F, -128, 0, 0, 0x7F, -128, 0, 0); + let r: i8x8 = transmute(vsubhn_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_s32() { + let a: i32x4 = i32x4::new(0x7F_FF_FF_FF, -2147483648, 1, 1); + let b: i32x4 = i32x4::new(1, 0, 0, 0); + let e: i16x4 = i16x4::new(0x7F_FF, -32768, 0, 0); + let r: i16x4 = transmute(vsubhn_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_s64() { + let a: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, -9223372036854775808); + let b: i64x2 = i64x2::new(1, 0); + let e: i32x2 = i32x2::new(0x7F_FF_FF_FF, -2147483648); + let r: i32x2 = transmute(vsubhn_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_u16() { + let a: u16x8 = u16x8::new(0xFF_FF, 0, 1, 1, 0xFF_FF, 0, 1, 1); + let b: u16x8 = u16x8::new(1, 0, 0, 0, 1, 0, 0, 0); + let e: u8x8 = u8x8::new(0xFF, 0, 0, 0, 0xFF, 0, 0, 0); + let r: u8x8 = transmute(vsubhn_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_u32() { + let a: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0, 1, 1); + let b: u32x4 = u32x4::new(1, 0, 0, 0); + let e: u16x4 = u16x4::new(0xFF_FF, 0, 0, 0); + let r: u16x4 = transmute(vsubhn_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_u64() { + let a: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); + let b: u64x2 = u64x2::new(1, 0); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); + let r: u32x2 = transmute(vsubhn_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_high_s16() { + let a: i8x8 = i8x8::new(0x7F, 0, 0x7F, 0, 0x7F, 0, 0x7F, 0); + let b: i16x8 = i16x8::new(0x7F_FF, 1, 0x7F_FF, 1, 0x7F_FF, 1, 0x7F_FF, 1); + let c: i16x8 = i16x8::new(1, 0, 1, 0, 1, 0, 1, 0); + let e: i8x16 = i8x16::new(0x7F, 0, 0x7F, 0, 0x7F, 0, 0x7F, 0, 0x7F, 0, 0x7F, 0, 0x7F, 0, 0x7F, 0); + let r: i8x16 = transmute(vsubhn_high_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_high_s32() { + let a: i16x4 = i16x4::new(0x7F_FF, 0, 0x7F_FF, 0); + let b: i32x4 = i32x4::new(0x7F_FF_FF_FF, 1, 0x7F_FF_FF_FF, 1); + let c: i32x4 = i32x4::new(1, 0, 1, 0); + let e: i16x8 = i16x8::new(0x7F_FF, 0, 0x7F_FF, 0, 0x7F_FF, 0, 0x7F_FF, 0); + let r: i16x8 = transmute(vsubhn_high_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_high_s64() { + let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0); + let b: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, 1); + let c: i64x2 = i64x2::new(1, 0); + let e: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0, 0x7F_FF_FF_FF, 0); + let r: i32x4 = transmute(vsubhn_high_s64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_high_u16() { + let a: u8x8 = u8x8::new(0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0); + let b: u16x8 = u16x8::new(0xFF_FF, 1, 0xFF_FF, 1, 0xFF_FF, 1, 0xFF_FF, 1); + let c: u16x8 = u16x8::new(1, 0, 1, 0, 1, 0, 1, 0); + let e: u8x16 = u8x16::new(0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0); + let r: u8x16 = transmute(vsubhn_high_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_high_u32() { + let a: u16x4 = u16x4::new(0xFF_FF, 0, 0xFF_FF, 0); + let b: u32x4 = u32x4::new(0xFF_FF_FF_FF, 1, 0xFF_FF_FF_FF, 1); + let c: u32x4 = u32x4::new(1, 0, 1, 0); + let e: u16x8 = u16x8::new(0xFF_FF, 0, 0xFF_FF, 0, 0xFF_FF, 0, 0xFF_FF, 0); + let r: u16x8 = transmute(vsubhn_high_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_high_u64() { + let a: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); + let b: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 1); + let c: u64x2 = u64x2::new(1, 0); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0, 0xFF_FF_FF_FF, 0); + let r: u32x4 = transmute(vsubhn_high_u64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsub_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x8 = u8x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: u8x8 = u8x8::new(0, 0, 1, 1, 2, 2, 3, 3); + let r: u8x8 = transmute(vhsub_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsubq_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); + let e: u8x16 = u8x16::new(0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7); + let r: u8x16 = transmute(vhsubq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsub_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(1, 2, 1, 2); + let e: u16x4 = u16x4::new(0, 0, 1, 1); + let r: u16x4 = transmute(vhsub_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsubq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x8 = u16x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: u16x8 = u16x8::new(0, 0, 1, 1, 2, 2, 3, 3); + let r: u16x8 = transmute(vhsubq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsub_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(1, 2); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vhsub_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsubq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(1, 2, 1, 2); + let e: u32x4 = u32x4::new(0, 0, 1, 1); + let r: u32x4 = transmute(vhsubq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsub_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: i8x8 = i8x8::new(0, 0, 1, 1, 2, 2, 3, 3); + let r: i8x8 = transmute(vhsub_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsubq_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); + let e: i8x16 = i8x16::new(0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7); + let r: i8x16 = transmute(vhsubq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsub_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(1, 2, 1, 2); + let e: i16x4 = i16x4::new(0, 0, 1, 1); + let r: i16x4 = transmute(vhsub_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsubq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: i16x8 = i16x8::new(0, 0, 1, 1, 2, 2, 3, 3); + let r: i16x8 = transmute(vhsubq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsub_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(1, 2); + let e: i32x2 = i32x2::new(0, 0); + let r: i32x2 = transmute(vhsub_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsubq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(1, 2, 1, 2); + let e: i32x4 = i32x4::new(0, 0, 1, 1); + let r: i32x4 = transmute(vhsubq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_s8() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i16x8 = transmute(vsubw_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_s16() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(0, 1, 2, 3); + let e: i32x4 = i32x4::new(0, 0, 0, 0); + let r: i32x4 = transmute(vsubw_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_s32() { + let a: i64x2 = i64x2::new(0, 1); + let b: i32x2 = i32x2::new(0, 1); + let e: i64x2 = i64x2::new(0, 0); + let r: i64x2 = transmute(vsubw_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_u8() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vsubw_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_u16() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(0, 1, 2, 3); + let e: u32x4 = u32x4::new(0, 0, 0, 0); + let r: u32x4 = transmute(vsubw_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_u32() { + let a: u64x2 = u64x2::new(0, 1); + let b: u32x2 = u32x2::new(0, 1); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vsubw_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_s8() { + let a: i8x8 = i8x8::new(0x7F, -128, 2, 3, 4, 5, 6, 7); + let b: i8x8 = i8x8::new(0x7F, -128, 2, 3, 4, 5, 6, 7); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i16x8 = transmute(vsubl_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_s16() { + let a: i16x4 = i16x4::new(0x7F_FF, -32768, 2, 3); + let b: i16x4 = i16x4::new(0x7F_FF, -32768, 2, 3); + let e: i32x4 = i32x4::new(0, 0, 0, 0); + let r: i32x4 = transmute(vsubl_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_s32() { + let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, -2147483648); + let b: i32x2 = i32x2::new(0x7F_FF_FF_FF, -2147483648); + let e: i64x2 = i64x2::new(0, 0); + let r: i64x2 = transmute(vsubl_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_u8() { + let a: u8x8 = u8x8::new(0xFF, 0, 2, 3, 4, 5, 6, 7); + let b: u8x8 = u8x8::new(0xFF, 0, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vsubl_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_u16() { + let a: u16x4 = u16x4::new(0xFF_FF, 0, 2, 3); + let b: u16x4 = u16x4::new(0xFF_FF, 0, 2, 3); + let e: u32x4 = u32x4::new(0, 0, 0, 0); + let r: u32x4 = transmute(vsubl_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_u32() { + let a: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); + let b: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vsubl_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmax_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: i8x8 = i8x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let r: i8x8 = transmute(vmax_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxq_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + let e: i8x16 = i8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 9, 10, 11, 12, 13, 14, 15, 16); + let r: i8x16 = transmute(vmaxq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmax_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(16, 15, 14, 13); + let e: i16x4 = i16x4::new(16, 15, 14, 13); + let r: i16x4 = transmute(vmax_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: i16x8 = i16x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let r: i16x8 = transmute(vmaxq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmax_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(16, 15); + let e: i32x2 = i32x2::new(16, 15); + let r: i32x2 = transmute(vmax_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(16, 15, 14, 13); + let e: i32x4 = i32x4::new(16, 15, 14, 13); + let r: i32x4 = transmute(vmaxq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmax_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x8 = u8x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: u8x8 = u8x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let r: u8x8 = transmute(vmax_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxq_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + let e: u8x16 = u8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 9, 10, 11, 12, 13, 14, 15, 16); + let r: u8x16 = transmute(vmaxq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmax_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(16, 15, 14, 13); + let e: u16x4 = u16x4::new(16, 15, 14, 13); + let r: u16x4 = transmute(vmax_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x8 = u16x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: u16x8 = u16x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let r: u16x8 = transmute(vmaxq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmax_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(16, 15); + let e: u32x2 = u32x2::new(16, 15); + let r: u32x2 = transmute(vmax_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(16, 15, 14, 13); + let e: u32x4 = u32x4::new(16, 15, 14, 13); + let r: u32x4 = transmute(vmaxq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmax_f32() { + let a: f32x2 = f32x2::new(1.0, -2.0); + let b: f32x2 = f32x2::new(0.0, 3.0); + let e: f32x2 = f32x2::new(1.0, 3.0); + let r: f32x2 = transmute(vmax_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxq_f32() { + let a: f32x4 = f32x4::new(1.0, -2.0, 3.0, -4.0); + let b: f32x4 = f32x4::new(0.0, 3.0, 2.0, 8.0); + let e: f32x4 = f32x4::new(1.0, 3.0, 3.0, 8.0); + let r: f32x4 = transmute(vmaxq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxnm_f32() { + let a: f32x2 = f32x2::new(1.0, 2.0); + let b: f32x2 = f32x2::new(8.0, 16.0); + let e: f32x2 = f32x2::new(8.0, 16.0); + let r: f32x2 = transmute(vmaxnm_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxnmq_f32() { + let a: f32x4 = f32x4::new(1.0, 2.0, 3.0, -4.0); + let b: f32x4 = f32x4::new(8.0, 16.0, -1.0, 6.0); + let e: f32x4 = f32x4::new(8.0, 16.0, 3.0, 6.0); + let r: f32x4 = transmute(vmaxnmq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmin_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vmin_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminq_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + let e: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 8, 7, 6, 5, 4, 3, 2, 1); + let r: i8x16 = transmute(vminq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmin_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(16, 15, 14, 13); + let e: i16x4 = i16x4::new(1, 2, 3, 4); + let r: i16x4 = transmute(vmin_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i16x8 = transmute(vminq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmin_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(16, 15); + let e: i32x2 = i32x2::new(1, 2); + let r: i32x2 = transmute(vmin_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(16, 15, 14, 13); + let e: i32x4 = i32x4::new(1, 2, 3, 4); + let r: i32x4 = transmute(vminq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmin_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x8 = u8x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vmin_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminq_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + let e: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 8, 7, 6, 5, 4, 3, 2, 1); + let r: u8x16 = transmute(vminq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmin_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(16, 15, 14, 13); + let e: u16x4 = u16x4::new(1, 2, 3, 4); + let r: u16x4 = transmute(vmin_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x8 = u16x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u16x8 = transmute(vminq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmin_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(16, 15); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vmin_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(16, 15, 14, 13); + let e: u32x4 = u32x4::new(1, 2, 3, 4); + let r: u32x4 = transmute(vminq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmin_f32() { + let a: f32x2 = f32x2::new(1.0, -2.0); + let b: f32x2 = f32x2::new(0.0, 3.0); + let e: f32x2 = f32x2::new(0.0, -2.0); + let r: f32x2 = transmute(vmin_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminq_f32() { + let a: f32x4 = f32x4::new(1.0, -2.0, 3.0, -4.0); + let b: f32x4 = f32x4::new(0.0, 3.0, 2.0, 8.0); + let e: f32x4 = f32x4::new(0.0, -2.0, 2.0, -4.0); + let r: f32x4 = transmute(vminq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminnm_f32() { + let a: f32x2 = f32x2::new(1.0, 2.0); + let b: f32x2 = f32x2::new(8.0, 16.0); + let e: f32x2 = f32x2::new(1.0, 2.0); + let r: f32x2 = transmute(vminnm_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminnmq_f32() { + let a: f32x4 = f32x4::new(1.0, 2.0, 3.0, -4.0); + let b: f32x4 = f32x4::new(8.0, 16.0, -1.0, 6.0); + let e: f32x4 = f32x4::new(1.0, 2.0, -1.0, -4.0); + let r: f32x4 = transmute(vminnmq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpadd_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let b: f32x2 = f32x2::new(3., 4.); + let e: f32x2 = f32x2::new(3., 7.); + let r: f32x2 = transmute(vpadd_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let e: i32x4 = i32x4::new(0, 4, 12, 24); + let r: i32x4 = transmute(vqdmull_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_s32() { + let a: i32x2 = i32x2::new(0, 1); + let b: i32x2 = i32x2::new(1, 2); + let e: i64x2 = i64x2::new(0, 4); + let r: i64x2 = transmute(vqdmull_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_n_s16() { + let a: i16x4 = i16x4::new(2, 4, 6, 8); + let b: i16 = 2; + let e: i32x4 = i32x4::new(8, 16, 24, 32); + let r: i32x4 = transmute(vqdmull_n_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_n_s32() { + let a: i32x2 = i32x2::new(2, 4); + let b: i32 = 2; + let e: i64x2 = i64x2::new(8, 16); + let r: i64x2 = transmute(vqdmull_n_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_lane_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(0, 2, 2, 0); + let e: i32x4 = i32x4::new(4, 8, 12, 16); + let r: i32x4 = transmute(vqdmull_lane_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_lane_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(0, 2); + let e: i64x2 = i64x2::new(4, 8); + let r: i64x2 = transmute(vqdmull_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_s16() { + let a: i32x4 = i32x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let c: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i32x4 = i32x4::new(5, 9, 13, 17); + let r: i32x4 = transmute(vqdmlal_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_s32() { + let a: i64x2 = i64x2::new(1, 1); + let b: i32x2 = i32x2::new(1, 2); + let c: i32x2 = i32x2::new(2, 2); + let e: i64x2 = i64x2::new(5, 9); + let r: i64x2 = transmute(vqdmlal_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_n_s16() { + let a: i32x4 = i32x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let c: i16 = 2; + let e: i32x4 = i32x4::new(5, 9, 13, 17); + let r: i32x4 = transmute(vqdmlal_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_n_s32() { + let a: i64x2 = i64x2::new(1, 1); + let b: i32x2 = i32x2::new(1, 2); + let c: i32 = 2; + let e: i64x2 = i64x2::new(5, 9); + let r: i64x2 = transmute(vqdmlal_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_lane_s16() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let c: i16x4 = i16x4::new(0, 2, 2, 0); + let e: i32x4 = i32x4::new(5, 10, 15, 20); + let r: i32x4 = transmute(vqdmlal_lane_s16::<2>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_lane_s32() { + let a: i64x2 = i64x2::new(1, 2); + let b: i32x2 = i32x2::new(1, 2); + let c: i32x2 = i32x2::new(0, 2); + let e: i64x2 = i64x2::new(5, 10); + let r: i64x2 = transmute(vqdmlal_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_s16() { + let a: i32x4 = i32x4::new(3, 7, 11, 15); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let c: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i32x4 = i32x4::new(-1, -1, -1, -1); + let r: i32x4 = transmute(vqdmlsl_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_s32() { + let a: i64x2 = i64x2::new(3, 7); + let b: i32x2 = i32x2::new(1, 2); + let c: i32x2 = i32x2::new(2, 2); + let e: i64x2 = i64x2::new(-1, -1); + let r: i64x2 = transmute(vqdmlsl_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_n_s16() { + let a: i32x4 = i32x4::new(3, 7, 11, 15); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let c: i16 = 2; + let e: i32x4 = i32x4::new(-1, -1, -1, -1); + let r: i32x4 = transmute(vqdmlsl_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_n_s32() { + let a: i64x2 = i64x2::new(3, 7); + let b: i32x2 = i32x2::new(1, 2); + let c: i32 = 2; + let e: i64x2 = i64x2::new(-1, -1); + let r: i64x2 = transmute(vqdmlsl_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_lane_s16() { + let a: i32x4 = i32x4::new(3, 6, 9, 12); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let c: i16x4 = i16x4::new(0, 2, 2, 0); + let e: i32x4 = i32x4::new(-1, -2, -3, -4); + let r: i32x4 = transmute(vqdmlsl_lane_s16::<2>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_lane_s32() { + let a: i64x2 = i64x2::new(3, 6); + let b: i32x2 = i32x2::new(1, 2); + let c: i32x2 = i32x2::new(0, 2); + let e: i64x2 = i64x2::new(-1, -2); + let r: i64x2 = transmute(vqdmlsl_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulh_s16() { + let a: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i16x4 = i16x4::new(1, 1, 1, 1); + let r: i16x4 = transmute(vqdmulh_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhq_s16() { + let a: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i16x8 = transmute(vqdmulhq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulh_s32() { + let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32x2 = i32x2::new(2, 2); + let e: i32x2 = i32x2::new(1, 1); + let r: i32x2 = transmute(vqdmulh_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhq_s32() { + let a: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let e: i32x4 = i32x4::new(1, 1, 1, 1); + let r: i32x4 = transmute(vqdmulhq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulh_n_s16() { + let a: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16 = 2; + let e: i16x4 = i16x4::new(1, 1, 1, 1); + let r: i16x4 = transmute(vqdmulh_n_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulh_n_s32() { + let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32 = 2; + let e: i32x2 = i32x2::new(1, 1); + let r: i32x2 = transmute(vqdmulh_n_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhq_n_s16() { + let a: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16 = 2; + let e: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i16x8 = transmute(vqdmulhq_n_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhq_n_s32() { + let a: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32 = 2; + let e: i32x4 = i32x4::new(1, 1, 1, 1); + let r: i32x4 = transmute(vqdmulhq_n_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhq_laneq_s16() { + let a: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16x8 = i16x8::new(2, 1, 1, 1, 1, 1, 1, 1); + let e: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i16x8 = transmute(vqdmulhq_laneq_s16::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulh_laneq_s16() { + let a: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16x8 = i16x8::new(2, 1, 1, 1, 1, 1, 1, 1); + let e: i16x4 = i16x4::new(1, 1, 1, 1); + let r: i16x4 = transmute(vqdmulh_laneq_s16::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhq_laneq_s32() { + let a: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32x4 = i32x4::new(2, 1, 1, 1); + let e: i32x4 = i32x4::new(1, 1, 1, 1); + let r: i32x4 = transmute(vqdmulhq_laneq_s32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulh_laneq_s32() { + let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32x4 = i32x4::new(2, 1, 1, 1); + let e: i32x2 = i32x2::new(1, 1); + let r: i32x2 = transmute(vqdmulh_laneq_s32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_s16() { + let a: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let e: i8x8 = i8x8::new(0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F); + let r: i8x8 = transmute(vqmovn_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_s32() { + let a: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let e: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let r: i16x4 = transmute(vqmovn_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_s64() { + let a: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, 0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let r: i32x2 = transmute(vqmovn_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_u16() { + let a: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vqmovn_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_u32() { + let a: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vqmovn_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_u64() { + let a: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vqmovn_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovun_s16() { + let a: i16x8 = i16x8::new(-1, -1, -1, -1, -1, -1, -1, -1); + let e: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vqmovun_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovun_s32() { + let a: i32x4 = i32x4::new(-1, -1, -1, -1); + let e: u16x4 = u16x4::new(0, 0, 0, 0); + let r: u16x4 = transmute(vqmovun_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovun_s64() { + let a: i64x2 = i64x2::new(-1, -1); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vqmovun_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulh_s16() { + let a: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i16x4 = i16x4::new(2, 2, 2, 2); + let r: i16x4 = transmute(vqrdmulh_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhq_s16() { + let a: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let r: i16x8 = transmute(vqrdmulhq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulh_s32() { + let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32x2 = i32x2::new(2, 2); + let e: i32x2 = i32x2::new(2, 2); + let r: i32x2 = transmute(vqrdmulh_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhq_s32() { + let a: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let e: i32x4 = i32x4::new(2, 2, 2, 2); + let r: i32x4 = transmute(vqrdmulhq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulh_n_s16() { + let a: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16 = 2; + let e: i16x4 = i16x4::new(2, 2, 2, 2); + let r: i16x4 = transmute(vqrdmulh_n_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhq_n_s16() { + let a: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16 = 2; + let e: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let r: i16x8 = transmute(vqrdmulhq_n_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulh_n_s32() { + let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32 = 2; + let e: i32x2 = i32x2::new(2, 2); + let r: i32x2 = transmute(vqrdmulh_n_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhq_n_s32() { + let a: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32 = 2; + let e: i32x4 = i32x4::new(2, 2, 2, 2); + let r: i32x4 = transmute(vqrdmulhq_n_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulh_lane_s16() { + let a: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16x4 = i16x4::new(2, 2, 2, 2); + let r: i16x4 = transmute(vqrdmulh_lane_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulh_laneq_s16() { + let a: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16x4 = i16x4::new(2, 2, 2, 2); + let r: i16x4 = transmute(vqrdmulh_laneq_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhq_lane_s16() { + let a: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let r: i16x8 = transmute(vqrdmulhq_lane_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhq_laneq_s16() { + let a: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let r: i16x8 = transmute(vqrdmulhq_laneq_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulh_lane_s32() { + let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32x2 = i32x2::new(0, 2); + let e: i32x2 = i32x2::new(2, 2); + let r: i32x2 = transmute(vqrdmulh_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulh_laneq_s32() { + let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32x2 = i32x2::new(2, 2); + let r: i32x2 = transmute(vqrdmulh_laneq_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhq_lane_s32() { + let a: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32x2 = i32x2::new(0, 2); + let e: i32x4 = i32x4::new(2, 2, 2, 2); + let r: i32x4 = transmute(vqrdmulhq_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhq_laneq_s32() { + let a: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32x4 = i32x4::new(2, 2, 2, 2); + let r: i32x4 = transmute(vqrdmulhq_laneq_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshl_s8() { + let a: i8x8 = i8x8::new(2, -128, 0x7F, 3, 4, 5, 6, 7); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i8x8 = i8x8::new(8, -128, 0x7F, 12, 16, 20, 24, 28); + let r: i8x8 = transmute(vqrshl_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlq_s8() { + let a: i8x16 = i8x16::new(2, -128, 0x7F, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let e: i8x16 = i8x16::new(8, -128, 0x7F, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60); + let r: i8x16 = transmute(vqrshlq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshl_s16() { + let a: i16x4 = i16x4::new(2, -32768, 0x7F_FF, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i16x4 = i16x4::new(8, -32768, 0x7F_FF, 12); + let r: i16x4 = transmute(vqrshl_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlq_s16() { + let a: i16x8 = i16x8::new(2, -32768, 0x7F_FF, 3, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i16x8 = i16x8::new(8, -32768, 0x7F_FF, 12, 16, 20, 24, 28); + let r: i16x8 = transmute(vqrshlq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshl_s32() { + let a: i32x2 = i32x2::new(2, -2147483648); + let b: i32x2 = i32x2::new(2, 2); + let e: i32x2 = i32x2::new(8, -2147483648); + let r: i32x2 = transmute(vqrshl_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlq_s32() { + let a: i32x4 = i32x4::new(2, -2147483648, 0x7F_FF_FF_FF, 3); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let e: i32x4 = i32x4::new(8, -2147483648, 0x7F_FF_FF_FF, 12); + let r: i32x4 = transmute(vqrshlq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshl_s64() { + let a: i64x1 = i64x1::new(2); + let b: i64x1 = i64x1::new(2); + let e: i64x1 = i64x1::new(8); + let r: i64x1 = transmute(vqrshl_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlq_s64() { + let a: i64x2 = i64x2::new(2, -9223372036854775808); + let b: i64x2 = i64x2::new(2, 2); + let e: i64x2 = i64x2::new(8, -9223372036854775808); + let r: i64x2 = transmute(vqrshlq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshl_u8() { + let a: u8x8 = u8x8::new(2, 0, 0xFF, 3, 4, 5, 6, 7); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: u8x8 = u8x8::new(8, 0, 0xFF, 12, 16, 20, 24, 28); + let r: u8x8 = transmute(vqrshl_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlq_u8() { + let a: u8x16 = u8x16::new(2, 0, 0xFF, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let e: u8x16 = u8x16::new(8, 0, 0xFF, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60); + let r: u8x16 = transmute(vqrshlq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshl_u16() { + let a: u16x4 = u16x4::new(2, 0, 0xFF_FF, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: u16x4 = u16x4::new(8, 0, 0xFF_FF, 12); + let r: u16x4 = transmute(vqrshl_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlq_u16() { + let a: u16x8 = u16x8::new(2, 0, 0xFF_FF, 3, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: u16x8 = u16x8::new(8, 0, 0xFF_FF, 12, 16, 20, 24, 28); + let r: u16x8 = transmute(vqrshlq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshl_u32() { + let a: u32x2 = u32x2::new(2, 0); + let b: i32x2 = i32x2::new(2, 2); + let e: u32x2 = u32x2::new(8, 0); + let r: u32x2 = transmute(vqrshl_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlq_u32() { + let a: u32x4 = u32x4::new(2, 0, 0xFF_FF_FF_FF, 3); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let e: u32x4 = u32x4::new(8, 0, 0xFF_FF_FF_FF, 12); + let r: u32x4 = transmute(vqrshlq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshl_u64() { + let a: u64x1 = u64x1::new(2); + let b: i64x1 = i64x1::new(2); + let e: u64x1 = u64x1::new(8); + let r: u64x1 = transmute(vqrshl_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlq_u64() { + let a: u64x2 = u64x2::new(2, 0); + let b: i64x2 = i64x2::new(2, 2); + let e: u64x2 = u64x2::new(8, 0); + let r: u64x2 = transmute(vqrshlq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_n_s16() { + let a: i16x8 = i16x8::new(-32768, 4, 8, 12, 16, 20, 24, 28); + let e: i8x8 = i8x8::new(-128, 1, 2, 3, 4, 5, 6, 7); + let r: i8x8 = transmute(vqrshrn_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_n_s32() { + let a: i32x4 = i32x4::new(-2147483648, 4, 8, 12); + let e: i16x4 = i16x4::new(-32768, 1, 2, 3); + let r: i16x4 = transmute(vqrshrn_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_n_s64() { + let a: i64x2 = i64x2::new(-9223372036854775808, 4); + let e: i32x2 = i32x2::new(-2147483648, 1); + let r: i32x2 = transmute(vqrshrn_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_n_u16() { + let a: u16x8 = u16x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let e: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u8x8 = transmute(vqrshrn_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_n_u32() { + let a: u32x4 = u32x4::new(0, 4, 8, 12); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vqrshrn_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_n_u64() { + let a: u64x2 = u64x2::new(0, 4); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vqrshrn_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrun_n_s16() { + let a: i16x8 = i16x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let e: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u8x8 = transmute(vqrshrun_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrun_n_s32() { + let a: i32x4 = i32x4::new(0, 4, 8, 12); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vqrshrun_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrun_n_s64() { + let a: i64x2 = i64x2::new(0, 4); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vqrshrun_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_s8() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i8x8 = i8x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let r: i8x8 = transmute(vqshl_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_s8() { + let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let e: i8x16 = i8x16::new(0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60); + let r: i8x16 = transmute(vqshlq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i16x4 = i16x4::new(0, 4, 8, 12); + let r: i16x4 = transmute(vqshl_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i16x8 = i16x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let r: i16x8 = transmute(vqshlq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_s32() { + let a: i32x2 = i32x2::new(0, 1); + let b: i32x2 = i32x2::new(2, 2); + let e: i32x2 = i32x2::new(0, 4); + let r: i32x2 = transmute(vqshl_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let e: i32x4 = i32x4::new(0, 4, 8, 12); + let r: i32x4 = transmute(vqshlq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_s64() { + let a: i64x1 = i64x1::new(0); + let b: i64x1 = i64x1::new(2); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vqshl_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_s64() { + let a: i64x2 = i64x2::new(0, 1); + let b: i64x2 = i64x2::new(2, 2); + let e: i64x2 = i64x2::new(0, 4); + let r: i64x2 = transmute(vqshlq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_u8() { + let a: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: u8x8 = u8x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let r: u8x8 = transmute(vqshl_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_u8() { + let a: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let e: u8x16 = u8x16::new(0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60); + let r: u8x16 = transmute(vqshlq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: u16x4 = u16x4::new(0, 4, 8, 12); + let r: u16x4 = transmute(vqshl_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: u16x8 = u16x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let r: u16x8 = transmute(vqshlq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_u32() { + let a: u32x2 = u32x2::new(0, 1); + let b: i32x2 = i32x2::new(2, 2); + let e: u32x2 = u32x2::new(0, 4); + let r: u32x2 = transmute(vqshl_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let e: u32x4 = u32x4::new(0, 4, 8, 12); + let r: u32x4 = transmute(vqshlq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_u64() { + let a: u64x1 = u64x1::new(0); + let b: i64x1 = i64x1::new(2); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vqshl_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_u64() { + let a: u64x2 = u64x2::new(0, 1); + let b: i64x2 = i64x2::new(2, 2); + let e: u64x2 = u64x2::new(0, 4); + let r: u64x2 = transmute(vqshlq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_n_s8() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x8 = i8x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let r: i8x8 = transmute(vqshl_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_n_s8() { + let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: i8x16 = i8x16::new(0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60); + let r: i8x16 = transmute(vqshlq_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_n_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: i16x4 = i16x4::new(0, 4, 8, 12); + let r: i16x4 = transmute(vqshl_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_n_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i16x8 = i16x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let r: i16x8 = transmute(vqshlq_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_n_s32() { + let a: i32x2 = i32x2::new(0, 1); + let e: i32x2 = i32x2::new(0, 4); + let r: i32x2 = transmute(vqshl_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_n_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let e: i32x4 = i32x4::new(0, 4, 8, 12); + let r: i32x4 = transmute(vqshlq_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_n_s64() { + let a: i64x1 = i64x1::new(0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vqshl_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_n_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i64x2 = i64x2::new(0, 4); + let r: i64x2 = transmute(vqshlq_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_n_u8() { + let a: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u8x8 = u8x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let r: u8x8 = transmute(vqshl_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_n_u8() { + let a: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: u8x16 = u8x16::new(0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60); + let r: u8x16 = transmute(vqshlq_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_n_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let e: u16x4 = u16x4::new(0, 4, 8, 12); + let r: u16x4 = transmute(vqshl_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_n_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let r: u16x8 = transmute(vqshlq_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_n_u32() { + let a: u32x2 = u32x2::new(0, 1); + let e: u32x2 = u32x2::new(0, 4); + let r: u32x2 = transmute(vqshl_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_n_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let e: u32x4 = u32x4::new(0, 4, 8, 12); + let r: u32x4 = transmute(vqshlq_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_n_u64() { + let a: u64x1 = u64x1::new(0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vqshl_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_n_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: u64x2 = u64x2::new(0, 4); + let r: u64x2 = transmute(vqshlq_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlu_n_s8() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u8x8 = u8x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let r: u8x8 = transmute(vqshlu_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlu_n_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: u16x4 = u16x4::new(0, 4, 8, 12); + let r: u16x4 = transmute(vqshlu_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlu_n_s32() { + let a: i32x2 = i32x2::new(0, 1); + let e: u32x2 = u32x2::new(0, 4); + let r: u32x2 = transmute(vqshlu_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlu_n_s64() { + let a: i64x1 = i64x1::new(0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vqshlu_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshluq_n_s8() { + let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: u8x16 = u8x16::new(0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60); + let r: u8x16 = transmute(vqshluq_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshluq_n_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let r: u16x8 = transmute(vqshluq_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshluq_n_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let e: u32x4 = u32x4::new(0, 4, 8, 12); + let r: u32x4 = transmute(vqshluq_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshluq_n_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: u64x2 = u64x2::new(0, 4); + let r: u64x2 = transmute(vqshluq_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_n_s16() { + let a: i16x8 = i16x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let e: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i8x8 = transmute(vqshrn_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_n_s32() { + let a: i32x4 = i32x4::new(0, 4, 8, 12); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vqshrn_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_n_s64() { + let a: i64x2 = i64x2::new(0, 4); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vqshrn_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_n_u16() { + let a: u16x8 = u16x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let e: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u8x8 = transmute(vqshrn_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_n_u32() { + let a: u32x4 = u32x4::new(0, 4, 8, 12); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vqshrn_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_n_u64() { + let a: u64x2 = u64x2::new(0, 4); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vqshrn_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrun_n_s16() { + let a: i16x8 = i16x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let e: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u8x8 = transmute(vqshrun_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrun_n_s32() { + let a: i32x4 = i32x4::new(0, 4, 8, 12); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vqshrun_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrun_n_s64() { + let a: i64x2 = i64x2::new(0, 4); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vqshrun_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsqrte_f32() { + let a: f32x2 = f32x2::new(1.0, 2.0); + let e: f32x2 = f32x2::new(0.998046875, 0.705078125); + let r: f32x2 = transmute(vrsqrte_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsqrteq_f32() { + let a: f32x4 = f32x4::new(1.0, 2.0, 3.0, 4.0); + let e: f32x4 = f32x4::new(0.998046875, 0.705078125, 0.576171875, 0.4990234375); + let r: f32x4 = transmute(vrsqrteq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsqrte_u32() { + let a: u32x2 = u32x2::new(1, 2); + let e: u32x2 = u32x2::new(4294967295, 4294967295); + let r: u32x2 = transmute(vrsqrte_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsqrteq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(4294967295, 4294967295, 4294967295, 4294967295); + let r: u32x4 = transmute(vrsqrteq_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsqrts_f32() { + let a: f32x2 = f32x2::new(1.0, 2.0); + let b: f32x2 = f32x2::new(1.0, 2.0); + let e: f32x2 = f32x2::new(1., -0.5); + let r: f32x2 = transmute(vrsqrts_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsqrtsq_f32() { + let a: f32x4 = f32x4::new(1.0, 2.0, 3.0, 4.0); + let b: f32x4 = f32x4::new(1.0, 2.0, 3.0, 4.0); + let e: f32x4 = f32x4::new(1., -0.5, -3.0, -6.5); + let r: f32x4 = transmute(vrsqrtsq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrecpe_f32() { + let a: f32x2 = f32x2::new(4.0, 3.0); + let e: f32x2 = f32x2::new(0.24951171875, 0.3330078125); + let r: f32x2 = transmute(vrecpe_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrecpeq_f32() { + let a: f32x4 = f32x4::new(4.0, 3.0, 2.0, 1.0); + let e: f32x4 = f32x4::new(0.24951171875, 0.3330078125, 0.4990234375, 0.998046875); + let r: f32x4 = transmute(vrecpeq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrecpe_u32() { + let a: u32x2 = u32x2::new(4, 3); + let e: u32x2 = u32x2::new(4294967295, 4294967295); + let r: u32x2 = transmute(vrecpe_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrecpeq_u32() { + let a: u32x4 = u32x4::new(4, 3, 2, 1); + let e: u32x4 = u32x4::new(4294967295, 4294967295, 4294967295, 4294967295); + let r: u32x4 = transmute(vrecpeq_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrecps_f32() { + let a: f32x2 = f32x2::new(4.0, 3.0); + let b: f32x2 = f32x2::new(4.0, 3.0); + let e: f32x2 = f32x2::new(-14., -7.); + let r: f32x2 = transmute(vrecps_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrecpsq_f32() { + let a: f32x4 = f32x4::new(4.0, 3.0, 2.0, 1.0); + let b: f32x4 = f32x4::new(4.0, 3.0, 2.0, 1.0); + let e: f32x4 = f32x4::new(-14., -7., -2., 1.); + let r: f32x4 = transmute(vrecpsq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_u8() { + let a: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i8x8 = transmute(vreinterpret_s8_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_p8() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i8x8 = transmute(vreinterpret_s8_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_p16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vreinterpret_s16_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vreinterpret_s16_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_u32() { + let a: u32x2 = u32x2::new(0, 1); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vreinterpret_s32_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_u64() { + let a: u64x1 = u64x1::new(0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_u8() { + let a: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: i8x16 = transmute(vreinterpretq_s8_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_p8() { + let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: i8x16 = transmute(vreinterpretq_s8_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_p16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vreinterpretq_s16_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vreinterpretq_s16_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vreinterpretq_s32_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_s64_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_p8() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u8x8 = transmute(vreinterpret_u8_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_s8() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u8x8 = transmute(vreinterpret_u8_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_p16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vreinterpret_u16_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vreinterpret_u16_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_s32() { + let a: i32x2 = i32x2::new(0, 1); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vreinterpret_u32_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_s64() { + let a: i64x1 = i64x1::new(0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_p8() { + let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: u8x16 = transmute(vreinterpretq_u8_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_s8() { + let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: u8x16 = transmute(vreinterpretq_u8_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_p16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vreinterpretq_u16_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vreinterpretq_u16_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vreinterpretq_u32_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vreinterpretq_u64_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_s8() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i8x8 = transmute(vreinterpret_p8_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_u8() { + let a: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i8x8 = transmute(vreinterpret_p8_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vreinterpret_p16_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vreinterpret_p16_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_s8() { + let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: i8x16 = transmute(vreinterpretq_p8_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_u8() { + let a: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: i8x16 = transmute(vreinterpretq_p8_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vreinterpretq_p16_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vreinterpretq_p16_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: i8x8 = transmute(vreinterpret_s8_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let e: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: i8x8 = transmute(vreinterpret_s8_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_p16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: i8x8 = transmute(vreinterpret_s8_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_s32() { + let a: i32x2 = i32x2::new(0, 1); + let e: i16x4 = i16x4::new(0, 0, 1, 0); + let r: i16x4 = transmute(vreinterpret_s16_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_u32() { + let a: u32x2 = u32x2::new(0, 1); + let e: i16x4 = i16x4::new(0, 0, 1, 0); + let r: i16x4 = transmute(vreinterpret_s16_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_s64() { + let a: i64x1 = i64x1::new(0); + let e: i32x2 = i32x2::new(0, 0); + let r: i32x2 = transmute(vreinterpret_s32_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_u64() { + let a: u64x1 = u64x1::new(0); + let e: i32x2 = i32x2::new(0, 0); + let r: i32x2 = transmute(vreinterpret_s32_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let r: i8x16 = transmute(vreinterpretq_s8_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let r: i8x16 = transmute(vreinterpretq_s8_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_p16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let r: i8x16 = transmute(vreinterpretq_s8_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let e: i16x8 = i16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: i16x8 = transmute(vreinterpretq_s16_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let e: i16x8 = i16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: i16x8 = transmute(vreinterpretq_s16_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i32x4 = i32x4::new(0, 0, 1, 0); + let r: i32x4 = transmute(vreinterpretq_s32_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: i32x4 = i32x4::new(0, 0, 1, 0); + let r: i32x4 = transmute(vreinterpretq_s32_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_p16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: u8x8 = u8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: u8x8 = transmute(vreinterpret_u8_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: u8x8 = u8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: u8x8 = transmute(vreinterpret_u8_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let e: u8x8 = u8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: u8x8 = transmute(vreinterpret_u8_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_s32() { + let a: i32x2 = i32x2::new(0, 1); + let e: u16x4 = u16x4::new(0, 0, 1, 0); + let r: u16x4 = transmute(vreinterpret_u16_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_u32() { + let a: u32x2 = u32x2::new(0, 1); + let e: u16x4 = u16x4::new(0, 0, 1, 0); + let r: u16x4 = transmute(vreinterpret_u16_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_s64() { + let a: i64x1 = i64x1::new(0); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vreinterpret_u32_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_u64() { + let a: u64x1 = u64x1::new(0); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vreinterpret_u32_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_p16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u8x16 = u8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let r: u8x16 = transmute(vreinterpretq_u8_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u8x16 = u8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let r: u8x16 = transmute(vreinterpretq_u8_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u8x16 = u8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let r: u8x16 = transmute(vreinterpretq_u8_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let e: u16x8 = u16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: u16x8 = transmute(vreinterpretq_u16_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let e: u16x8 = u16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: u16x8 = transmute(vreinterpretq_u16_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: u32x4 = u32x4::new(0, 0, 1, 0); + let r: u32x4 = transmute(vreinterpretq_u32_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: u32x4 = u32x4::new(0, 0, 1, 0); + let r: u32x4 = transmute(vreinterpretq_u32_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_p16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: i8x8 = transmute(vreinterpret_p8_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: i8x8 = transmute(vreinterpret_p8_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let e: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: i8x8 = transmute(vreinterpret_p8_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_s32() { + let a: i32x2 = i32x2::new(0, 1); + let e: i16x4 = i16x4::new(0, 0, 1, 0); + let r: i16x4 = transmute(vreinterpret_p16_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_u32() { + let a: u32x2 = u32x2::new(0, 1); + let e: i16x4 = i16x4::new(0, 0, 1, 0); + let r: i16x4 = transmute(vreinterpret_p16_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_p16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let r: i8x16 = transmute(vreinterpretq_p8_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let r: i8x16 = transmute(vreinterpretq_p8_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let r: i8x16 = transmute(vreinterpretq_p8_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let e: i16x8 = i16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: i16x8 = transmute(vreinterpretq_p16_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let e: i16x8 = i16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: i16x8 = transmute(vreinterpretq_p16_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_p64() { + let a: i64x1 = i64x1::new(0); + let e: i32x2 = i32x2::new(0, 0); + let r: i32x2 = transmute(vreinterpret_s32_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_p64() { + let a: i64x1 = i64x1::new(0); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vreinterpret_u32_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i32x4 = i32x4::new(0, 0, 1, 0); + let r: i32x4 = transmute(vreinterpretq_s32_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: u32x4 = u32x4::new(0, 0, 1, 0); + let r: u32x4 = transmute(vreinterpretq_u32_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_p128() { + let a: p128 = 0; + let e: i64x2 = i64x2::new(0, 0); + let r: i64x2 = transmute(vreinterpretq_s64_p128(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_p128() { + let a: p128 = 0; + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vreinterpretq_u64_p128(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_p128() { + let a: p128 = 0; + let e: i64x2 = i64x2::new(0, 0); + let r: i64x2 = transmute(vreinterpretq_p64_p128(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_p8() { + let a: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vreinterpret_s16_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_s8() { + let a: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vreinterpret_s16_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_u8() { + let a: u8x8 = u8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vreinterpret_s16_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_p16() { + let a: i16x4 = i16x4::new(0, 0, 1, 0); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vreinterpret_s32_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_s16() { + let a: i16x4 = i16x4::new(0, 0, 1, 0); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vreinterpret_s32_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_u16() { + let a: u16x4 = u16x4::new(0, 0, 1, 0); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vreinterpret_s32_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_s32() { + let a: i32x2 = i32x2::new(0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_u32() { + let a: u32x2 = u32x2::new(0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_p8() { + let a: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vreinterpretq_s16_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_s8() { + let a: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vreinterpretq_s16_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_u8() { + let a: u8x16 = u8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vreinterpretq_s16_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_p16() { + let a: i16x8 = i16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vreinterpretq_s32_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_s16() { + let a: i16x8 = i16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vreinterpretq_s32_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_u16() { + let a: u16x8 = u16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vreinterpretq_s32_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_s32() { + let a: i32x4 = i32x4::new(0, 0, 1, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_s64_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_u32() { + let a: u32x4 = u32x4::new(0, 0, 1, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_s64_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_p8() { + let a: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vreinterpret_u16_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_s8() { + let a: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vreinterpret_u16_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_u8() { + let a: u8x8 = u8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vreinterpret_u16_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_p16() { + let a: i16x4 = i16x4::new(0, 0, 1, 0); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vreinterpret_u32_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_s16() { + let a: i16x4 = i16x4::new(0, 0, 1, 0); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vreinterpret_u32_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_u16() { + let a: u16x4 = u16x4::new(0, 0, 1, 0); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vreinterpret_u32_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_s32() { + let a: i32x2 = i32x2::new(0, 0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_u32() { + let a: u32x2 = u32x2::new(0, 0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_p8() { + let a: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vreinterpretq_u16_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_s8() { + let a: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vreinterpretq_u16_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_u8() { + let a: u8x16 = u8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vreinterpretq_u16_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_p16() { + let a: i16x8 = i16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vreinterpretq_u32_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_s16() { + let a: i16x8 = i16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vreinterpretq_u32_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_u16() { + let a: u16x8 = u16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vreinterpretq_u32_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_s32() { + let a: i32x4 = i32x4::new(0, 0, 1, 0); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vreinterpretq_u64_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_u32() { + let a: u32x4 = u32x4::new(0, 0, 1, 0); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vreinterpretq_u64_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_p8() { + let a: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vreinterpret_p16_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_s8() { + let a: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vreinterpret_p16_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_u8() { + let a: u8x8 = u8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vreinterpret_p16_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_p8() { + let a: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vreinterpretq_p16_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_s8() { + let a: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vreinterpretq_p16_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_u8() { + let a: u8x16 = u8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vreinterpretq_p16_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_s32() { + let a: i32x2 = i32x2::new(0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_u32() { + let a: u32x2 = u32x2::new(0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_s32() { + let a: i32x4 = i32x4::new(0, 0, 1, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_p64_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_u32() { + let a: u32x4 = u32x4::new(0, 0, 1, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_p64_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p128_s64() { + let a: i64x2 = i64x2::new(0, 0); + let e: p128 = 0; + let r: p128 = transmute(vreinterpretq_p128_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p128_u64() { + let a: u64x2 = u64x2::new(0, 0); + let e: p128 = 0; + let r: p128 = transmute(vreinterpretq_p128_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p128_p64() { + let a: i64x2 = i64x2::new(0, 0); + let e: p128 = 0; + let r: p128 = transmute(vreinterpretq_p128_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_s32() { + let a: i32x2 = i32x2::new(0, 1); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_s8_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_u32() { + let a: u32x2 = u32x2::new(0, 1); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_s8_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_s64() { + let a: i64x1 = i64x1::new(0); + let e: i16x4 = i16x4::new(0, 0, 0, 0); + let r: i16x4 = transmute(vreinterpret_s16_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_u64() { + let a: u64x1 = u64x1::new(0); + let e: i16x4 = i16x4::new(0, 0, 0, 0); + let r: i16x4 = transmute(vreinterpret_s16_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_s8_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_s8_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_s16_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_s16_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_s32() { + let a: i32x2 = i32x2::new(0, 1); + let e: u8x8 = u8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: u8x8 = transmute(vreinterpret_u8_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_u32() { + let a: u32x2 = u32x2::new(0, 1); + let e: u8x8 = u8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: u8x8 = transmute(vreinterpret_u8_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_s64() { + let a: i64x1 = i64x1::new(0); + let e: u16x4 = u16x4::new(0, 0, 0, 0); + let r: u16x4 = transmute(vreinterpret_u16_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_u64() { + let a: u64x1 = u64x1::new(0); + let e: u16x4 = u16x4::new(0, 0, 0, 0); + let r: u16x4 = transmute(vreinterpret_u16_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let e: u8x16 = u8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let r: u8x16 = transmute(vreinterpretq_u8_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let e: u8x16 = u8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let r: u8x16 = transmute(vreinterpretq_u8_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: u16x8 = u16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: u16x8 = transmute(vreinterpretq_u16_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: u16x8 = u16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: u16x8 = transmute(vreinterpretq_u16_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_s32() { + let a: i32x2 = i32x2::new(0, 1); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_p8_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_u32() { + let a: u32x2 = u32x2::new(0, 1); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_p8_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_s64() { + let a: i64x1 = i64x1::new(0); + let e: i16x4 = i16x4::new(0, 0, 0, 0); + let r: i16x4 = transmute(vreinterpret_p16_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_u64() { + let a: u64x1 = u64x1::new(0); + let e: i16x4 = i16x4::new(0, 0, 0, 0); + let r: i16x4 = transmute(vreinterpret_p16_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_p8_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_p8_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_p16_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_p16_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_p64() { + let a: i64x1 = i64x1::new(0); + let e: i16x4 = i16x4::new(0, 0, 0, 0); + let r: i16x4 = transmute(vreinterpret_s16_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_p64() { + let a: i64x1 = i64x1::new(0); + let e: u16x4 = u16x4::new(0, 0, 0, 0); + let r: u16x4 = transmute(vreinterpret_u16_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_p64() { + let a: i64x1 = i64x1::new(0); + let e: i16x4 = i16x4::new(0, 0, 0, 0); + let r: i16x4 = transmute(vreinterpret_p16_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_s16_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: u16x8 = u16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: u16x8 = transmute(vreinterpretq_u16_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_p16_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_p128() { + let a: p128 = 0; + let e: i32x4 = i32x4::new(0, 0, 0, 0); + let r: i32x4 = transmute(vreinterpretq_s32_p128(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_p128() { + let a: p128 = 0; + let e: u32x4 = u32x4::new(0, 0, 0, 0); + let r: u32x4 = transmute(vreinterpretq_u32_p128(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_p8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vreinterpret_s32_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_s8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vreinterpret_s32_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_u8() { + let a: u8x8 = u8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vreinterpret_s32_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_p16() { + let a: i16x4 = i16x4::new(0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_s16() { + let a: i16x4 = i16x4::new(0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_u16() { + let a: u16x4 = u16x4::new(0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_p8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vreinterpretq_s32_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_s8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vreinterpretq_s32_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_u8() { + let a: u8x16 = u8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vreinterpretq_s32_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_p16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_s64_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_s16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_s64_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_u16() { + let a: u16x8 = u16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_s64_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_p8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vreinterpret_u32_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_s8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vreinterpret_u32_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_u8() { + let a: u8x8 = u8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vreinterpret_u32_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_p16() { + let a: i16x4 = i16x4::new(0, 0, 0, 0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_s16() { + let a: i16x4 = i16x4::new(0, 0, 0, 0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_u16() { + let a: u16x4 = u16x4::new(0, 0, 0, 0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_p8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vreinterpretq_u32_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_s8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vreinterpretq_u32_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_u8() { + let a: u8x16 = u8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vreinterpretq_u32_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_p16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vreinterpretq_u64_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_s16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vreinterpretq_u64_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_u16() { + let a: u16x8 = u16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vreinterpretq_u64_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_p16() { + let a: i16x4 = i16x4::new(0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_s16() { + let a: i16x4 = i16x4::new(0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_u16() { + let a: u16x4 = u16x4::new(0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_p16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_p64_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_s16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_p64_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_u16() { + let a: u16x8 = u16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_p64_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p128_s32() { + let a: i32x4 = i32x4::new(0, 0, 0, 0); + let e: p128 = 0; + let r: p128 = transmute(vreinterpretq_p128_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p128_u32() { + let a: u32x4 = u32x4::new(0, 0, 0, 0); + let e: p128 = 0; + let r: p128 = transmute(vreinterpretq_p128_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_s64() { + let a: i64x1 = i64x1::new(0); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_s8_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_u64() { + let a: u64x1 = u64x1::new(0); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_s8_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_s64() { + let a: i64x1 = i64x1::new(0); + let e: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vreinterpret_u8_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_u64() { + let a: u64x1 = u64x1::new(0); + let e: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vreinterpret_u8_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_s64() { + let a: i64x1 = i64x1::new(0); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_p8_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_u64() { + let a: u64x1 = u64x1::new(0); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_p8_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_s8_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_s8_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vreinterpretq_u8_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vreinterpretq_u8_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_p8_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_p8_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_p64() { + let a: i64x1 = i64x1::new(0); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_s8_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_p64() { + let a: i64x1 = i64x1::new(0); + let e: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vreinterpret_u8_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_p64() { + let a: i64x1 = i64x1::new(0); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_p8_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_s8_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vreinterpretq_u8_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_p8_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_p128() { + let a: p128 = 0; + let e: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_s16_p128(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_p128() { + let a: p128 = 0; + let e: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vreinterpretq_u16_p128(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_p128() { + let a: p128 = 0; + let e: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_p16_p128(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_p8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_s8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_u8() { + let a: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_p8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_s8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_u8() { + let a: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_p8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_s64_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_s8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_s64_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_u8() { + let a: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_s64_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_p8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vreinterpretq_u64_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_s8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vreinterpretq_u64_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_u8() { + let a: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vreinterpretq_u64_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_p8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_s8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_u8() { + let a: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_p8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_p64_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_s8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_p64_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_u8() { + let a: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_p64_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p128_s16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: p128 = 0; + let r: p128 = transmute(vreinterpretq_p128_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p128_u16() { + let a: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: p128 = 0; + let r: p128 = transmute(vreinterpretq_p128_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p128_p16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: p128 = 0; + let r: p128 = transmute(vreinterpretq_p128_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p128_s8() { + let a: i8x16 = i8x16::new(1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: p128 = 1; + let r: p128 = transmute(vreinterpretq_p128_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p128_u8() { + let a: u8x16 = u8x16::new(1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: p128 = 1; + let r: p128 = transmute(vreinterpretq_p128_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p128_p8() { + let a: i8x16 = i8x16::new(1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: p128 = 1; + let r: p128 = transmute(vreinterpretq_p128_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_p128() { + let a: p128 = 1; + let e: i8x16 = i8x16::new(1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_s8_p128(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_p128() { + let a: p128 = 1; + let e: u8x16 = u8x16::new(1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vreinterpretq_u8_p128(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_p128() { + let a: p128 = 1; + let e: i8x16 = i8x16::new(1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_p8_p128(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_s8_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: i16x4 = i16x4::new(0, 0, 0, 0); + let r: i16x4 = transmute(vreinterpret_s16_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: i32x2 = i32x2::new(0, 0); + let r: i32x2 = transmute(vreinterpret_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_s8_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_s16_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: i32x4 = i32x4::new(0, 0, 0, 0); + let r: i32x4 = transmute(vreinterpretq_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: i64x2 = i64x2::new(0, 0); + let r: i64x2 = transmute(vreinterpretq_s64_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vreinterpret_u8_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: u16x4 = u16x4::new(0, 0, 0, 0); + let r: u16x4 = transmute(vreinterpret_u16_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vreinterpret_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vreinterpretq_u8_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vreinterpretq_u16_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: u32x4 = u32x4::new(0, 0, 0, 0); + let r: u32x4 = transmute(vreinterpretq_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vreinterpretq_u64_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_p8_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: i16x4 = i16x4::new(0, 0, 0, 0); + let r: i16x4 = transmute(vreinterpret_p16_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_p8_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_p16_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p128_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: p128 = 0; + let r: p128 = transmute(vreinterpretq_p128_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_s8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_s16() { + let a: i16x4 = i16x4::new(0, 0, 0, 0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_s32() { + let a: i32x2 = i32x2::new(0, 0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_s64() { + let a: i64x1 = i64x1::new(0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_s8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_s16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_s32() { + let a: i32x4 = i32x4::new(0, 0, 0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_s64() { + let a: i64x2 = i64x2::new(0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_u8() { + let a: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_u16() { + let a: u16x4 = u16x4::new(0, 0, 0, 0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_u32() { + let a: u32x2 = u32x2::new(0, 0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_u64() { + let a: u64x1 = u64x1::new(0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_u8() { + let a: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_u16() { + let a: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_u32() { + let a: u32x4 = u32x4::new(0, 0, 0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_u64() { + let a: u64x2 = u64x2::new(0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_p8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_p16() { + let a: i16x4 = i16x4::new(0, 0, 0, 0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_p8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_p16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_p128() { + let a: p128 = 0; + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_p128(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshl_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i8x8 = i8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: i8x8 = transmute(vrshl_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshlq_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let e: i8x16 = i8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let r: i8x16 = transmute(vrshlq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshl_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i16x4 = i16x4::new(4, 8, 12, 16); + let r: i16x4 = transmute(vrshl_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshlq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: i16x8 = transmute(vrshlq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshl_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(2, 2); + let e: i32x2 = i32x2::new(4, 8); + let r: i32x2 = transmute(vrshl_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshlq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let e: i32x4 = i32x4::new(4, 8, 12, 16); + let r: i32x4 = transmute(vrshlq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshl_s64() { + let a: i64x1 = i64x1::new(1); + let b: i64x1 = i64x1::new(2); + let e: i64x1 = i64x1::new(4); + let r: i64x1 = transmute(vrshl_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshlq_s64() { + let a: i64x2 = i64x2::new(1, 2); + let b: i64x2 = i64x2::new(2, 2); + let e: i64x2 = i64x2::new(4, 8); + let r: i64x2 = transmute(vrshlq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshl_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: u8x8 = u8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: u8x8 = transmute(vrshl_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshlq_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let e: u8x16 = u8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let r: u8x16 = transmute(vrshlq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshl_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: u16x4 = u16x4::new(4, 8, 12, 16); + let r: u16x4 = transmute(vrshl_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshlq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: u16x8 = transmute(vrshlq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshl_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: i32x2 = i32x2::new(2, 2); + let e: u32x2 = u32x2::new(4, 8); + let r: u32x2 = transmute(vrshl_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshlq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let e: u32x4 = u32x4::new(4, 8, 12, 16); + let r: u32x4 = transmute(vrshlq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshl_u64() { + let a: u64x1 = u64x1::new(1); + let b: i64x1 = i64x1::new(2); + let e: u64x1 = u64x1::new(4); + let r: u64x1 = transmute(vrshl_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshlq_u64() { + let a: u64x2 = u64x2::new(1, 2); + let b: i64x2 = i64x2::new(2, 2); + let e: u64x2 = u64x2::new(4, 8); + let r: u64x2 = transmute(vrshlq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshr_n_s8() { + let a: i8x8 = i8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vrshr_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrq_n_s8() { + let a: i8x16 = i8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let e: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: i8x16 = transmute(vrshrq_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshr_n_s16() { + let a: i16x4 = i16x4::new(4, 8, 12, 16); + let e: i16x4 = i16x4::new(1, 2, 3, 4); + let r: i16x4 = transmute(vrshr_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrq_n_s16() { + let a: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i16x8 = transmute(vrshrq_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshr_n_s32() { + let a: i32x2 = i32x2::new(4, 8); + let e: i32x2 = i32x2::new(1, 2); + let r: i32x2 = transmute(vrshr_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrq_n_s32() { + let a: i32x4 = i32x4::new(4, 8, 12, 16); + let e: i32x4 = i32x4::new(1, 2, 3, 4); + let r: i32x4 = transmute(vrshrq_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshr_n_s64() { + let a: i64x1 = i64x1::new(4); + let e: i64x1 = i64x1::new(1); + let r: i64x1 = transmute(vrshr_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrq_n_s64() { + let a: i64x2 = i64x2::new(4, 8); + let e: i64x2 = i64x2::new(1, 2); + let r: i64x2 = transmute(vrshrq_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshr_n_u8() { + let a: u8x8 = u8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vrshr_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrq_n_u8() { + let a: u8x16 = u8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let e: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: u8x16 = transmute(vrshrq_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshr_n_u16() { + let a: u16x4 = u16x4::new(4, 8, 12, 16); + let e: u16x4 = u16x4::new(1, 2, 3, 4); + let r: u16x4 = transmute(vrshr_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrq_n_u16() { + let a: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u16x8 = transmute(vrshrq_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshr_n_u32() { + let a: u32x2 = u32x2::new(4, 8); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vrshr_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrq_n_u32() { + let a: u32x4 = u32x4::new(4, 8, 12, 16); + let e: u32x4 = u32x4::new(1, 2, 3, 4); + let r: u32x4 = transmute(vrshrq_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshr_n_u64() { + let a: u64x1 = u64x1::new(4); + let e: u64x1 = u64x1::new(1); + let r: u64x1 = transmute(vrshr_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrq_n_u64() { + let a: u64x2 = u64x2::new(4, 8); + let e: u64x2 = u64x2::new(1, 2); + let r: u64x2 = transmute(vrshrq_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_n_s16() { + let a: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vrshrn_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_n_s32() { + let a: i32x4 = i32x4::new(4, 8, 12, 16); + let e: i16x4 = i16x4::new(1, 2, 3, 4); + let r: i16x4 = transmute(vrshrn_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_n_s64() { + let a: i64x2 = i64x2::new(4, 8); + let e: i32x2 = i32x2::new(1, 2); + let r: i32x2 = transmute(vrshrn_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_n_u16() { + let a: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vrshrn_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_n_u32() { + let a: u32x4 = u32x4::new(4, 8, 12, 16); + let e: u16x4 = u16x4::new(1, 2, 3, 4); + let r: u16x4 = transmute(vrshrn_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_n_u64() { + let a: u64x2 = u64x2::new(4, 8); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vrshrn_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsra_n_s8() { + let a: i8x8 = i8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i8x8 = i8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: i8x8 = i8x8::new(2, 3, 4, 5, 6, 7, 8, 9); + let r: i8x8 = transmute(vrsra_n_s8::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsraq_n_s8() { + let a: i8x16 = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let b: i8x16 = i8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let e: i8x16 = i8x16::new(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17); + let r: i8x16 = transmute(vrsraq_n_s8::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsra_n_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(4, 8, 12, 16); + let e: i16x4 = i16x4::new(2, 3, 4, 5); + let r: i16x4 = transmute(vrsra_n_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsraq_n_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: i16x8 = i16x8::new(2, 3, 4, 5, 6, 7, 8, 9); + let r: i16x8 = transmute(vrsraq_n_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsra_n_s32() { + let a: i32x2 = i32x2::new(1, 1); + let b: i32x2 = i32x2::new(4, 8); + let e: i32x2 = i32x2::new(2, 3); + let r: i32x2 = transmute(vrsra_n_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsraq_n_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 1); + let b: i32x4 = i32x4::new(4, 8, 12, 16); + let e: i32x4 = i32x4::new(2, 3, 4, 5); + let r: i32x4 = transmute(vrsraq_n_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsra_n_s64() { + let a: i64x1 = i64x1::new(1); + let b: i64x1 = i64x1::new(4); + let e: i64x1 = i64x1::new(2); + let r: i64x1 = transmute(vrsra_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsraq_n_s64() { + let a: i64x2 = i64x2::new(1, 1); + let b: i64x2 = i64x2::new(4, 8); + let e: i64x2 = i64x2::new(2, 3); + let r: i64x2 = transmute(vrsraq_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsra_n_u8() { + let a: u8x8 = u8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: u8x8 = u8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: u8x8 = u8x8::new(2, 3, 4, 5, 6, 7, 8, 9); + let r: u8x8 = transmute(vrsra_n_u8::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsraq_n_u8() { + let a: u8x16 = u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let b: u8x16 = u8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let e: u8x16 = u8x16::new(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17); + let r: u8x16 = transmute(vrsraq_n_u8::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsra_n_u16() { + let a: u16x4 = u16x4::new(1, 1, 1, 1); + let b: u16x4 = u16x4::new(4, 8, 12, 16); + let e: u16x4 = u16x4::new(2, 3, 4, 5); + let r: u16x4 = transmute(vrsra_n_u16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsraq_n_u16() { + let a: u16x8 = u16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: u16x8 = u16x8::new(2, 3, 4, 5, 6, 7, 8, 9); + let r: u16x8 = transmute(vrsraq_n_u16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsra_n_u32() { + let a: u32x2 = u32x2::new(1, 1); + let b: u32x2 = u32x2::new(4, 8); + let e: u32x2 = u32x2::new(2, 3); + let r: u32x2 = transmute(vrsra_n_u32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsraq_n_u32() { + let a: u32x4 = u32x4::new(1, 1, 1, 1); + let b: u32x4 = u32x4::new(4, 8, 12, 16); + let e: u32x4 = u32x4::new(2, 3, 4, 5); + let r: u32x4 = transmute(vrsraq_n_u32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsra_n_u64() { + let a: u64x1 = u64x1::new(1); + let b: u64x1 = u64x1::new(4); + let e: u64x1 = u64x1::new(2); + let r: u64x1 = transmute(vrsra_n_u64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsraq_n_u64() { + let a: u64x2 = u64x2::new(1, 1); + let b: u64x2 = u64x2::new(4, 8); + let e: u64x2 = u64x2::new(2, 3); + let r: u64x2 = transmute(vrsraq_n_u64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsubhn_s16() { + let a: i16x8 = i16x8::new(0x7F_FF, -32768, 0, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i8x8 = i8x8::new(-128, -128, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vrsubhn_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsubhn_s32() { + let a: i32x4 = i32x4::new(0x7F_FF_FF_FF, -2147483648, 0, 4); + let b: i32x4 = i32x4::new(1, 2, 3, 4); + let e: i16x4 = i16x4::new(-32768, -32768, 0, 0); + let r: i16x4 = transmute(vrsubhn_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsubhn_s64() { + let a: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, -9223372036854775808); + let b: i64x2 = i64x2::new(1, 2); + let e: i32x2 = i32x2::new(-2147483648, -2147483648); + let r: i32x2 = transmute(vrsubhn_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsubhn_u16() { + let a: u16x8 = u16x8::new(0xFF_FF, 0, 3, 4, 5, 6, 7, 8); + let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vrsubhn_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsubhn_u32() { + let a: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0, 3, 4); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u16x4 = u16x4::new(0, 0, 0, 0); + let r: u16x4 = transmute(vrsubhn_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsubhn_u64() { + let a: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); + let b: u64x2 = u64x2::new(1, 2); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vrsubhn_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_s8() { + let a: i8 = 1; + let b: i8x8 = i8x8::new(0, 2, 3, 4, 5, 6, 7, 8); + let e: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vset_lane_s8::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_s16() { + let a: i16 = 1; + let b: i16x4 = i16x4::new(0, 2, 3, 4); + let e: i16x4 = i16x4::new(1, 2, 3, 4); + let r: i16x4 = transmute(vset_lane_s16::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_s32() { + let a: i32 = 1; + let b: i32x2 = i32x2::new(0, 2); + let e: i32x2 = i32x2::new(1, 2); + let r: i32x2 = transmute(vset_lane_s32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_s64() { + let a: i64 = 1; + let b: i64x1 = i64x1::new(0); + let e: i64x1 = i64x1::new(1); + let r: i64x1 = transmute(vset_lane_s64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_u8() { + let a: u8 = 1; + let b: u8x8 = u8x8::new(0, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vset_lane_u8::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_u16() { + let a: u16 = 1; + let b: u16x4 = u16x4::new(0, 2, 3, 4); + let e: u16x4 = u16x4::new(1, 2, 3, 4); + let r: u16x4 = transmute(vset_lane_u16::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_u32() { + let a: u32 = 1; + let b: u32x2 = u32x2::new(0, 2); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vset_lane_u32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_u64() { + let a: u64 = 1; + let b: u64x1 = u64x1::new(0); + let e: u64x1 = u64x1::new(1); + let r: u64x1 = transmute(vset_lane_u64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_p8() { + let a: p8 = 1; + let b: i8x8 = i8x8::new(0, 2, 3, 4, 5, 6, 7, 8); + let e: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vset_lane_p8::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_p16() { + let a: p16 = 1; + let b: i16x4 = i16x4::new(0, 2, 3, 4); + let e: i16x4 = i16x4::new(1, 2, 3, 4); + let r: i16x4 = transmute(vset_lane_p16::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_p64() { + let a: p64 = 1; + let b: i64x1 = i64x1::new(0); + let e: i64x1 = i64x1::new(1); + let r: i64x1 = transmute(vset_lane_p64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_s8() { + let a: i8 = 1; + let b: i8x16 = i8x16::new(0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: i8x16 = transmute(vsetq_lane_s8::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_s16() { + let a: i16 = 1; + let b: i16x8 = i16x8::new(0, 2, 3, 4, 5, 6, 7, 8); + let e: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i16x8 = transmute(vsetq_lane_s16::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_s32() { + let a: i32 = 1; + let b: i32x4 = i32x4::new(0, 2, 3, 4); + let e: i32x4 = i32x4::new(1, 2, 3, 4); + let r: i32x4 = transmute(vsetq_lane_s32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_s64() { + let a: i64 = 1; + let b: i64x2 = i64x2::new(0, 2); + let e: i64x2 = i64x2::new(1, 2); + let r: i64x2 = transmute(vsetq_lane_s64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_u8() { + let a: u8 = 1; + let b: u8x16 = u8x16::new(0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: u8x16 = transmute(vsetq_lane_u8::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_u16() { + let a: u16 = 1; + let b: u16x8 = u16x8::new(0, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u16x8 = transmute(vsetq_lane_u16::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_u32() { + let a: u32 = 1; + let b: u32x4 = u32x4::new(0, 2, 3, 4); + let e: u32x4 = u32x4::new(1, 2, 3, 4); + let r: u32x4 = transmute(vsetq_lane_u32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_u64() { + let a: u64 = 1; + let b: u64x2 = u64x2::new(0, 2); + let e: u64x2 = u64x2::new(1, 2); + let r: u64x2 = transmute(vsetq_lane_u64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_p8() { + let a: p8 = 1; + let b: i8x16 = i8x16::new(0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: i8x16 = transmute(vsetq_lane_p8::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_p16() { + let a: p16 = 1; + let b: i16x8 = i16x8::new(0, 2, 3, 4, 5, 6, 7, 8); + let e: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i16x8 = transmute(vsetq_lane_p16::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_p64() { + let a: p64 = 1; + let b: i64x2 = i64x2::new(0, 2); + let e: i64x2 = i64x2::new(1, 2); + let r: i64x2 = transmute(vsetq_lane_p64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_f32() { + let a: f32 = 1.; + let b: f32x2 = f32x2::new(0., 2.); + let e: f32x2 = f32x2::new(1., 2.); + let r: f32x2 = transmute(vset_lane_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_f32() { + let a: f32 = 1.; + let b: f32x4 = f32x4::new(0., 2., 3., 4.); + let e: f32x4 = f32x4::new(1., 2., 3., 4.); + let r: f32x4 = transmute(vsetq_lane_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i8x8 = i8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: i8x8 = transmute(vshl_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let e: i8x16 = i8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let r: i8x16 = transmute(vshlq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i16x4 = i16x4::new(4, 8, 12, 16); + let r: i16x4 = transmute(vshl_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: i16x8 = transmute(vshlq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(2, 2); + let e: i32x2 = i32x2::new(4, 8); + let r: i32x2 = transmute(vshl_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let e: i32x4 = i32x4::new(4, 8, 12, 16); + let r: i32x4 = transmute(vshlq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_s64() { + let a: i64x1 = i64x1::new(1); + let b: i64x1 = i64x1::new(2); + let e: i64x1 = i64x1::new(4); + let r: i64x1 = transmute(vshl_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_s64() { + let a: i64x2 = i64x2::new(1, 2); + let b: i64x2 = i64x2::new(2, 2); + let e: i64x2 = i64x2::new(4, 8); + let r: i64x2 = transmute(vshlq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: u8x8 = u8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: u8x8 = transmute(vshl_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let e: u8x16 = u8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let r: u8x16 = transmute(vshlq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: u16x4 = u16x4::new(4, 8, 12, 16); + let r: u16x4 = transmute(vshl_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: u16x8 = transmute(vshlq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: i32x2 = i32x2::new(2, 2); + let e: u32x2 = u32x2::new(4, 8); + let r: u32x2 = transmute(vshl_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let e: u32x4 = u32x4::new(4, 8, 12, 16); + let r: u32x4 = transmute(vshlq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_u64() { + let a: u64x1 = u64x1::new(1); + let b: i64x1 = i64x1::new(2); + let e: u64x1 = u64x1::new(4); + let r: u64x1 = transmute(vshl_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_u64() { + let a: u64x2 = u64x2::new(1, 2); + let b: i64x2 = i64x2::new(2, 2); + let e: u64x2 = u64x2::new(4, 8); + let r: u64x2 = transmute(vshlq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_n_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i8x8 = i8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: i8x8 = transmute(vshl_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_n_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let r: i8x16 = transmute(vshlq_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_n_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let e: i16x4 = i16x4::new(4, 8, 12, 16); + let r: i16x4 = transmute(vshl_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_n_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: i16x8 = transmute(vshlq_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_n_s32() { + let a: i32x2 = i32x2::new(1, 2); + let e: i32x2 = i32x2::new(4, 8); + let r: i32x2 = transmute(vshl_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_n_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let e: i32x4 = i32x4::new(4, 8, 12, 16); + let r: i32x4 = transmute(vshlq_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_n_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: u8x8 = transmute(vshl_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_n_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let r: u8x16 = transmute(vshlq_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_n_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let e: u16x4 = u16x4::new(4, 8, 12, 16); + let r: u16x4 = transmute(vshl_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_n_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: u16x8 = transmute(vshlq_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_n_u32() { + let a: u32x2 = u32x2::new(1, 2); + let e: u32x2 = u32x2::new(4, 8); + let r: u32x2 = transmute(vshl_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_n_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(4, 8, 12, 16); + let r: u32x4 = transmute(vshlq_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_n_s64() { + let a: i64x1 = i64x1::new(1); + let e: i64x1 = i64x1::new(4); + let r: i64x1 = transmute(vshl_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_n_s64() { + let a: i64x2 = i64x2::new(1, 2); + let e: i64x2 = i64x2::new(4, 8); + let r: i64x2 = transmute(vshlq_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_n_u64() { + let a: u64x1 = u64x1::new(1); + let e: u64x1 = u64x1::new(4); + let r: u64x1 = transmute(vshl_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_n_u64() { + let a: u64x2 = u64x2::new(1, 2); + let e: u64x2 = u64x2::new(4, 8); + let r: u64x2 = transmute(vshlq_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_n_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: i16x8 = transmute(vshll_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_n_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let e: i32x4 = i32x4::new(4, 8, 12, 16); + let r: i32x4 = transmute(vshll_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_n_s32() { + let a: i32x2 = i32x2::new(1, 2); + let e: i64x2 = i64x2::new(4, 8); + let r: i64x2 = transmute(vshll_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_n_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: u16x8 = transmute(vshll_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_n_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(4, 8, 12, 16); + let r: u32x4 = transmute(vshll_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_n_u32() { + let a: u32x2 = u32x2::new(1, 2); + let e: u64x2 = u64x2::new(4, 8); + let r: u64x2 = transmute(vshll_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshr_n_s8() { + let a: i8x8 = i8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vshr_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrq_n_s8() { + let a: i8x16 = i8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let e: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: i8x16 = transmute(vshrq_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshr_n_s16() { + let a: i16x4 = i16x4::new(4, 8, 12, 16); + let e: i16x4 = i16x4::new(1, 2, 3, 4); + let r: i16x4 = transmute(vshr_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrq_n_s16() { + let a: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i16x8 = transmute(vshrq_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshr_n_s32() { + let a: i32x2 = i32x2::new(4, 8); + let e: i32x2 = i32x2::new(1, 2); + let r: i32x2 = transmute(vshr_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrq_n_s32() { + let a: i32x4 = i32x4::new(4, 8, 12, 16); + let e: i32x4 = i32x4::new(1, 2, 3, 4); + let r: i32x4 = transmute(vshrq_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshr_n_s64() { + let a: i64x1 = i64x1::new(4); + let e: i64x1 = i64x1::new(1); + let r: i64x1 = transmute(vshr_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrq_n_s64() { + let a: i64x2 = i64x2::new(4, 8); + let e: i64x2 = i64x2::new(1, 2); + let r: i64x2 = transmute(vshrq_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshr_n_u8() { + let a: u8x8 = u8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vshr_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrq_n_u8() { + let a: u8x16 = u8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let e: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: u8x16 = transmute(vshrq_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshr_n_u16() { + let a: u16x4 = u16x4::new(4, 8, 12, 16); + let e: u16x4 = u16x4::new(1, 2, 3, 4); + let r: u16x4 = transmute(vshr_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrq_n_u16() { + let a: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u16x8 = transmute(vshrq_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshr_n_u32() { + let a: u32x2 = u32x2::new(4, 8); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vshr_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrq_n_u32() { + let a: u32x4 = u32x4::new(4, 8, 12, 16); + let e: u32x4 = u32x4::new(1, 2, 3, 4); + let r: u32x4 = transmute(vshrq_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshr_n_u64() { + let a: u64x1 = u64x1::new(4); + let e: u64x1 = u64x1::new(1); + let r: u64x1 = transmute(vshr_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrq_n_u64() { + let a: u64x2 = u64x2::new(4, 8); + let e: u64x2 = u64x2::new(1, 2); + let r: u64x2 = transmute(vshrq_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_n_s16() { + let a: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vshrn_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_n_s32() { + let a: i32x4 = i32x4::new(4, 8, 12, 16); + let e: i16x4 = i16x4::new(1, 2, 3, 4); + let r: i16x4 = transmute(vshrn_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_n_s64() { + let a: i64x2 = i64x2::new(4, 8); + let e: i32x2 = i32x2::new(1, 2); + let r: i32x2 = transmute(vshrn_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_n_u16() { + let a: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vshrn_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_n_u32() { + let a: u32x4 = u32x4::new(4, 8, 12, 16); + let e: u16x4 = u16x4::new(1, 2, 3, 4); + let r: u16x4 = transmute(vshrn_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_n_u64() { + let a: u64x2 = u64x2::new(4, 8); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vshrn_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsra_n_s8() { + let a: i8x8 = i8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i8x8 = i8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: i8x8 = i8x8::new(2, 3, 4, 5, 6, 7, 8, 9); + let r: i8x8 = transmute(vsra_n_s8::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsraq_n_s8() { + let a: i8x16 = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let b: i8x16 = i8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let e: i8x16 = i8x16::new(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17); + let r: i8x16 = transmute(vsraq_n_s8::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsra_n_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(4, 8, 12, 16); + let e: i16x4 = i16x4::new(2, 3, 4, 5); + let r: i16x4 = transmute(vsra_n_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsraq_n_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: i16x8 = i16x8::new(2, 3, 4, 5, 6, 7, 8, 9); + let r: i16x8 = transmute(vsraq_n_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsra_n_s32() { + let a: i32x2 = i32x2::new(1, 1); + let b: i32x2 = i32x2::new(4, 8); + let e: i32x2 = i32x2::new(2, 3); + let r: i32x2 = transmute(vsra_n_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsraq_n_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 1); + let b: i32x4 = i32x4::new(4, 8, 12, 16); + let e: i32x4 = i32x4::new(2, 3, 4, 5); + let r: i32x4 = transmute(vsraq_n_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsra_n_s64() { + let a: i64x1 = i64x1::new(1); + let b: i64x1 = i64x1::new(4); + let e: i64x1 = i64x1::new(2); + let r: i64x1 = transmute(vsra_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsraq_n_s64() { + let a: i64x2 = i64x2::new(1, 1); + let b: i64x2 = i64x2::new(4, 8); + let e: i64x2 = i64x2::new(2, 3); + let r: i64x2 = transmute(vsraq_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsra_n_u8() { + let a: u8x8 = u8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: u8x8 = u8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: u8x8 = u8x8::new(2, 3, 4, 5, 6, 7, 8, 9); + let r: u8x8 = transmute(vsra_n_u8::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsraq_n_u8() { + let a: u8x16 = u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let b: u8x16 = u8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let e: u8x16 = u8x16::new(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17); + let r: u8x16 = transmute(vsraq_n_u8::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsra_n_u16() { + let a: u16x4 = u16x4::new(1, 1, 1, 1); + let b: u16x4 = u16x4::new(4, 8, 12, 16); + let e: u16x4 = u16x4::new(2, 3, 4, 5); + let r: u16x4 = transmute(vsra_n_u16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsraq_n_u16() { + let a: u16x8 = u16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: u16x8 = u16x8::new(2, 3, 4, 5, 6, 7, 8, 9); + let r: u16x8 = transmute(vsraq_n_u16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsra_n_u32() { + let a: u32x2 = u32x2::new(1, 1); + let b: u32x2 = u32x2::new(4, 8); + let e: u32x2 = u32x2::new(2, 3); + let r: u32x2 = transmute(vsra_n_u32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsraq_n_u32() { + let a: u32x4 = u32x4::new(1, 1, 1, 1); + let b: u32x4 = u32x4::new(4, 8, 12, 16); + let e: u32x4 = u32x4::new(2, 3, 4, 5); + let r: u32x4 = transmute(vsraq_n_u32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsra_n_u64() { + let a: u64x1 = u64x1::new(1); + let b: u64x1 = u64x1::new(4); + let e: u64x1 = u64x1::new(2); + let r: u64x1 = transmute(vsra_n_u64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsraq_n_u64() { + let a: u64x2 = u64x2::new(1, 1); + let b: u64x2 = u64x2::new(4, 8); + let e: u64x2 = u64x2::new(2, 3); + let r: u64x2 = transmute(vsraq_n_u64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn_s8() { + let a: i8x8 = i8x8::new(0, 2, 2, 6, 2, 10, 6, 14); + let b: i8x8 = i8x8::new(1, 3, 3, 7, 3, 1, 7, 15); + let e: [i8; 16] = [0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 10, 1, 14, 15]; + let r: [i8; 16] = transmute(vtrn_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn_s16() { + let a: i16x4 = i16x4::new(0, 2, 2, 6); + let b: i16x4 = i16x4::new(1, 3, 3, 7); + let e: [i16; 8] = [0, 1, 2, 3, 2, 3, 6, 7]; + let r: [i16; 8] = transmute(vtrn_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrnq_s8() { + let a: i8x16 = i8x16::new(0, 2, 2, 6, 2, 10, 6, 14, 2, 18, 6, 22, 10, 26, 14, 30); + let b: i8x16 = i8x16::new(1, 3, 3, 7, 3, 1, 7, 15, 3, 19, 7, 23, 1, 27, 15, 31); + let e: [i8; 32] = [0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 10, 1, 14, 15, 2, 3, 6, 7, 10, 1, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31]; + let r: [i8; 32] = transmute(vtrnq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrnq_s16() { + let a: i16x8 = i16x8::new(0, 2, 2, 6, 2, 10, 6, 14); + let b: i16x8 = i16x8::new(1, 3, 3, 7, 3, 1, 7, 15); + let e: [i16; 16] = [0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 10, 1, 14, 15]; + let r: [i16; 16] = transmute(vtrnq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrnq_s32() { + let a: i32x4 = i32x4::new(0, 2, 2, 6); + let b: i32x4 = i32x4::new(1, 3, 3, 7); + let e: [i32; 8] = [0, 1, 2, 3, 2, 3, 6, 7]; + let r: [i32; 8] = transmute(vtrnq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn_u8() { + let a: u8x8 = u8x8::new(0, 2, 2, 6, 2, 10, 6, 14); + let b: u8x8 = u8x8::new(1, 3, 3, 7, 3, 1, 7, 15); + let e: [u8; 16] = [0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 10, 1, 14, 15]; + let r: [u8; 16] = transmute(vtrn_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn_u16() { + let a: u16x4 = u16x4::new(0, 2, 2, 6); + let b: u16x4 = u16x4::new(1, 3, 3, 7); + let e: [u16; 8] = [0, 1, 2, 3, 2, 3, 6, 7]; + let r: [u16; 8] = transmute(vtrn_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrnq_u8() { + let a: u8x16 = u8x16::new(0, 2, 2, 6, 2, 10, 6, 14, 2, 18, 6, 22, 10, 26, 14, 30); + let b: u8x16 = u8x16::new(1, 3, 3, 7, 3, 1, 7, 15, 3, 19, 7, 23, 1, 27, 15, 31); + let e: [u8; 32] = [0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 10, 1, 14, 15, 2, 3, 6, 7, 10, 1, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31]; + let r: [u8; 32] = transmute(vtrnq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrnq_u16() { + let a: u16x8 = u16x8::new(0, 2, 2, 6, 2, 10, 6, 14); + let b: u16x8 = u16x8::new(1, 3, 3, 7, 3, 1, 7, 15); + let e: [u16; 16] = [0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 10, 1, 14, 15]; + let r: [u16; 16] = transmute(vtrnq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrnq_u32() { + let a: u32x4 = u32x4::new(0, 2, 2, 6); + let b: u32x4 = u32x4::new(1, 3, 3, 7); + let e: [u32; 8] = [0, 1, 2, 3, 2, 3, 6, 7]; + let r: [u32; 8] = transmute(vtrnq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn_p8() { + let a: i8x8 = i8x8::new(0, 2, 2, 6, 2, 10, 6, 14); + let b: i8x8 = i8x8::new(1, 3, 3, 7, 3, 1, 7, 15); + let e: [u8; 16] = [0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 10, 1, 14, 15]; + let r: [u8; 16] = transmute(vtrn_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn_p16() { + let a: i16x4 = i16x4::new(0, 2, 2, 6); + let b: i16x4 = i16x4::new(1, 3, 3, 7); + let e: [u16; 8] = [0, 1, 2, 3, 2, 3, 6, 7]; + let r: [u16; 8] = transmute(vtrn_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrnq_p8() { + let a: i8x16 = i8x16::new(0, 2, 2, 6, 2, 10, 6, 14, 2, 18, 6, 22, 10, 26, 14, 30); + let b: i8x16 = i8x16::new(1, 3, 3, 7, 3, 1, 7, 15, 3, 19, 7, 23, 1, 27, 15, 31); + let e: [u8; 32] = [0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 10, 1, 14, 15, 2, 3, 6, 7, 10, 1, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31]; + let r: [u8; 32] = transmute(vtrnq_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrnq_p16() { + let a: i16x8 = i16x8::new(0, 2, 2, 6, 2, 10, 6, 14); + let b: i16x8 = i16x8::new(1, 3, 3, 7, 3, 1, 7, 15); + let e: [u16; 16] = [0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 10, 1, 14, 15]; + let r: [u16; 16] = transmute(vtrnq_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn_s32() { + let a: i32x2 = i32x2::new(0, 2); + let b: i32x2 = i32x2::new(1, 3); + let e: [i32; 4] = [0, 1, 2, 3]; + let r: [i32; 4] = transmute(vtrn_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn_u32() { + let a: u32x2 = u32x2::new(0, 2); + let b: u32x2 = u32x2::new(1, 3); + let e: [u32; 4] = [0, 1, 2, 3]; + let r: [u32; 4] = transmute(vtrn_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn_f32() { + let a: f32x2 = f32x2::new(0., 2.); + let b: f32x2 = f32x2::new(1., 3.); + let e: [f32; 4] = [0., 1., 2., 3.]; + let r: [f32; 4] = transmute(vtrn_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrnq_f32() { + let a: f32x4 = f32x4::new(0., 2., 2., 6.); + let b: f32x4 = f32x4::new(1., 3., 3., 7.); + let e: [f32; 8] = [0., 1., 2., 3., 2., 3., 6., 7.]; + let r: [f32; 8] = transmute(vtrnq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip_s8() { + let a: i8x8 = i8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i8x8 = i8x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: [i8; 16] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; + let r: [i8; 16] = transmute(vzip_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip_s16() { + let a: i16x4 = i16x4::new(0, 2, 4, 6); + let b: i16x4 = i16x4::new(1, 3, 5, 7); + let e: [i16; 8] = [0, 1, 2, 3, 4, 5, 6, 7]; + let r: [i16; 8] = transmute(vzip_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip_u8() { + let a: u8x8 = u8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: u8x8 = u8x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: [u8; 16] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; + let r: [u8; 16] = transmute(vzip_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip_u16() { + let a: u16x4 = u16x4::new(0, 2, 4, 6); + let b: u16x4 = u16x4::new(1, 3, 5, 7); + let e: [u16; 8] = [0, 1, 2, 3, 4, 5, 6, 7]; + let r: [u16; 8] = transmute(vzip_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip_p8() { + let a: i8x8 = i8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i8x8 = i8x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: [u8; 16] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; + let r: [u8; 16] = transmute(vzip_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip_p16() { + let a: i16x4 = i16x4::new(0, 2, 4, 6); + let b: i16x4 = i16x4::new(1, 3, 5, 7); + let e: [u16; 8] = [0, 1, 2, 3, 4, 5, 6, 7]; + let r: [u16; 8] = transmute(vzip_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip_s32() { + let a: i32x2 = i32x2::new(0, 2); + let b: i32x2 = i32x2::new(1, 3); + let e: [i32; 4] = [0, 1, 2, 3]; + let r: [i32; 4] = transmute(vzip_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip_u32() { + let a: u32x2 = u32x2::new(0, 2); + let b: u32x2 = u32x2::new(1, 3); + let e: [u32; 4] = [0, 1, 2, 3]; + let r: [u32; 4] = transmute(vzip_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzipq_s8() { + let a: i8x16 = i8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let b: i8x16 = i8x16::new(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + let e: [i8; 32] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]; + let r: [i8; 32] = transmute(vzipq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzipq_s16() { + let a: i16x8 = i16x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i16x8 = i16x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: [i16; 16] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; + let r: [i16; 16] = transmute(vzipq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzipq_s32() { + let a: i32x4 = i32x4::new(0, 2, 4, 6); + let b: i32x4 = i32x4::new(1, 3, 5, 7); + let e: [i32; 8] = [0, 1, 2, 3, 4, 5, 6, 7]; + let r: [i32; 8] = transmute(vzipq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzipq_u8() { + let a: u8x16 = u8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let b: u8x16 = u8x16::new(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + let e: [u8; 32] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]; + let r: [u8; 32] = transmute(vzipq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzipq_u16() { + let a: u16x8 = u16x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: u16x8 = u16x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: [u16; 16] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; + let r: [u16; 16] = transmute(vzipq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzipq_u32() { + let a: u32x4 = u32x4::new(0, 2, 4, 6); + let b: u32x4 = u32x4::new(1, 3, 5, 7); + let e: [u32; 8] = [0, 1, 2, 3, 4, 5, 6, 7]; + let r: [u32; 8] = transmute(vzipq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzipq_p8() { + let a: i8x16 = i8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let b: i8x16 = i8x16::new(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + let e: [u8; 32] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]; + let r: [u8; 32] = transmute(vzipq_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzipq_p16() { + let a: i16x8 = i16x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i16x8 = i16x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: [u16; 16] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; + let r: [u16; 16] = transmute(vzipq_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let b: f32x2 = f32x2::new(5., 6.); + let e: [f32; 4] = [1., 5., 2., 6.]; + let r: [f32; 4] = transmute(vzip_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzipq_f32() { + let a: f32x4 = f32x4::new(1., 2., 3., 4.); + let b: f32x4 = f32x4::new(5., 6., 7., 8.); + let e: [f32; 8] = [1., 5., 2., 6., 3., 7., 4., 8.]; + let r: [f32; 8] = transmute(vzipq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp_s8() { + let a: i8x8 = i8x8::new(1, 2, 2, 3, 2, 3, 3, 8); + let b: i8x8 = i8x8::new(2, 3, 3, 8, 3, 15, 8, 16); + let e: [i8; 16] = [1, 2, 2, 3, 2, 3, 3, 8, 2, 3, 3, 8, 3, 8, 15, 16]; + let r: [i8; 16] = transmute(vuzp_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp_s16() { + let a: i16x4 = i16x4::new(1, 2, 2, 3); + let b: i16x4 = i16x4::new(2, 3, 3, 8); + let e: [i16; 8] = [1, 2, 2, 3, 2, 3, 3, 8]; + let r: [i16; 8] = transmute(vuzp_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzpq_s8() { + let a: i8x16 = i8x16::new(1, 2, 2, 3, 2, 3, 3, 8, 2, 3, 3, 8, 3, 15, 8, 16); + let b: i8x16 = i8x16::new(2, 3, 3, 8, 3, 15, 8, 16, 3, 29, 8, 30, 15, 31, 16, 32); + let e: [i8; 32] = [1, 2, 2, 3, 2, 3, 3, 8, 2, 3, 3, 8, 3, 8, 15, 16, 2, 3, 3, 8, 3, 8, 15, 16, 3, 8, 15, 16, 29, 30, 31, 32]; + let r: [i8; 32] = transmute(vuzpq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzpq_s16() { + let a: i16x8 = i16x8::new(1, 2, 2, 3, 2, 3, 3, 8); + let b: i16x8 = i16x8::new(2, 3, 3, 8, 3, 15, 8, 16); + let e: [i16; 16] = [1, 2, 2, 3, 2, 3, 3, 8, 2, 3, 3, 8, 3, 8, 15, 16]; + let r: [i16; 16] = transmute(vuzpq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzpq_s32() { + let a: i32x4 = i32x4::new(1, 2, 2, 3); + let b: i32x4 = i32x4::new(2, 3, 3, 8); + let e: [i32; 8] = [1, 2, 2, 3, 2, 3, 3, 8]; + let r: [i32; 8] = transmute(vuzpq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp_u8() { + let a: u8x8 = u8x8::new(1, 2, 2, 3, 2, 3, 3, 8); + let b: u8x8 = u8x8::new(2, 3, 3, 8, 3, 15, 8, 16); + let e: [u8; 16] = [1, 2, 2, 3, 2, 3, 3, 8, 2, 3, 3, 8, 3, 8, 15, 16]; + let r: [u8; 16] = transmute(vuzp_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp_u16() { + let a: u16x4 = u16x4::new(1, 2, 2, 3); + let b: u16x4 = u16x4::new(2, 3, 3, 8); + let e: [u16; 8] = [1, 2, 2, 3, 2, 3, 3, 8]; + let r: [u16; 8] = transmute(vuzp_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzpq_u8() { + let a: u8x16 = u8x16::new(1, 2, 2, 3, 2, 3, 3, 8, 2, 3, 3, 8, 3, 15, 8, 16); + let b: u8x16 = u8x16::new(2, 3, 3, 8, 3, 15, 8, 16, 3, 29, 8, 30, 15, 31, 16, 32); + let e: [u8; 32] = [1, 2, 2, 3, 2, 3, 3, 8, 2, 3, 3, 8, 3, 8, 15, 16, 2, 3, 3, 8, 3, 8, 15, 16, 3, 8, 15, 16, 29, 30, 31, 32]; + let r: [u8; 32] = transmute(vuzpq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzpq_u16() { + let a: u16x8 = u16x8::new(1, 2, 2, 3, 2, 3, 3, 8); + let b: u16x8 = u16x8::new(2, 3, 3, 8, 3, 15, 8, 16); + let e: [u16; 16] = [1, 2, 2, 3, 2, 3, 3, 8, 2, 3, 3, 8, 3, 8, 15, 16]; + let r: [u16; 16] = transmute(vuzpq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzpq_u32() { + let a: u32x4 = u32x4::new(1, 2, 2, 3); + let b: u32x4 = u32x4::new(2, 3, 3, 8); + let e: [u32; 8] = [1, 2, 2, 3, 2, 3, 3, 8]; + let r: [u32; 8] = transmute(vuzpq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp_p8() { + let a: i8x8 = i8x8::new(1, 2, 2, 3, 2, 3, 3, 8); + let b: i8x8 = i8x8::new(2, 3, 3, 8, 3, 15, 8, 16); + let e: [u8; 16] = [1, 2, 2, 3, 2, 3, 3, 8, 2, 3, 3, 8, 3, 8, 15, 16]; + let r: [u8; 16] = transmute(vuzp_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp_p16() { + let a: i16x4 = i16x4::new(1, 2, 2, 3); + let b: i16x4 = i16x4::new(2, 3, 3, 8); + let e: [u16; 8] = [1, 2, 2, 3, 2, 3, 3, 8]; + let r: [u16; 8] = transmute(vuzp_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzpq_p8() { + let a: i8x16 = i8x16::new(1, 2, 2, 3, 2, 3, 3, 8, 2, 3, 3, 8, 3, 15, 8, 16); + let b: i8x16 = i8x16::new(2, 3, 3, 8, 3, 15, 8, 16, 3, 29, 8, 30, 15, 31, 16, 32); + let e: [u8; 32] = [1, 2, 2, 3, 2, 3, 3, 8, 2, 3, 3, 8, 3, 8, 15, 16, 2, 3, 3, 8, 3, 8, 15, 16, 3, 8, 15, 16, 29, 30, 31, 32]; + let r: [u8; 32] = transmute(vuzpq_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzpq_p16() { + let a: i16x8 = i16x8::new(1, 2, 2, 3, 2, 3, 3, 8); + let b: i16x8 = i16x8::new(2, 3, 3, 8, 3, 15, 8, 16); + let e: [u16; 16] = [1, 2, 2, 3, 2, 3, 3, 8, 2, 3, 3, 8, 3, 8, 15, 16]; + let r: [u16; 16] = transmute(vuzpq_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(2, 3); + let e: [i32; 4] = [1, 2, 2, 3]; + let r: [i32; 4] = transmute(vuzp_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(2, 3); + let e: [u32; 4] = [1, 2, 2, 3]; + let r: [u32; 4] = transmute(vuzp_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let b: f32x2 = f32x2::new(2., 6.); + let e: [f32; 4] = [1., 2., 2., 6.]; + let r: [f32; 4] = transmute(vuzp_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzpq_f32() { + let a: f32x4 = f32x4::new(1., 2., 2., 4.); + let b: f32x4 = f32x4::new(2., 6., 6., 8.); + let e: [f32; 8] = [1., 2., 2., 6., 2., 4., 6., 8.]; + let r: [f32; 8] = transmute(vuzpq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_u8() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let c: u8x8 = u8x8::new(10, 10, 10, 10, 10, 10, 10, 10); + let e: u16x8 = u16x8::new(10, 10, 10, 10, 10, 10, 10, 10); + let r: u16x8 = transmute(vabal_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_u16() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(1, 2, 3, 4); + let c: u16x4 = u16x4::new(10, 10, 10, 10); + let e: u32x4 = u32x4::new(10, 10, 10, 10); + let r: u32x4 = transmute(vabal_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_u32() { + let a: u64x2 = u64x2::new(1, 2); + let b: u32x2 = u32x2::new(1, 2); + let c: u32x2 = u32x2::new(10, 10); + let e: u64x2 = u64x2::new(10, 10); + let r: u64x2 = transmute(vabal_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_s8() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let c: i8x8 = i8x8::new(10, 10, 10, 10, 10, 10, 10, 10); + let e: i16x8 = i16x8::new(10, 10, 10, 10, 10, 10, 10, 10); + let r: i16x8 = transmute(vabal_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_s16() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let c: i16x4 = i16x4::new(10, 10, 10, 10); + let e: i32x4 = i32x4::new(10, 10, 10, 10); + let r: i32x4 = transmute(vabal_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_s32() { + let a: i64x2 = i64x2::new(1, 2); + let b: i32x2 = i32x2::new(1, 2); + let c: i32x2 = i32x2::new(10, 10); + let e: i64x2 = i64x2::new(10, 10); + let r: i64x2 = transmute(vabal_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqabs_s8() { + let a: i8x8 = i8x8::new(-128, 0x7F, -6, -5, -4, -3, -2, -1); + let e: i8x8 = i8x8::new(0x7F, 0x7F, 6, 5, 4, 3, 2, 1); + let r: i8x8 = transmute(vqabs_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqabsq_s8() { + let a: i8x16 = i8x16::new(-128, 0x7F, -6, -5, -4, -3, -2, -1, 0, -127, 127, 1, 2, 3, 4, 5); + let e: i8x16 = i8x16::new(0x7F, 0x7F, 6, 5, 4, 3, 2, 1, 0, 127, 127, 1, 2, 3, 4, 5); + let r: i8x16 = transmute(vqabsq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqabs_s16() { + let a: i16x4 = i16x4::new(-32768, 0x7F_FF, -6, -5); + let e: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 6, 5); + let r: i16x4 = transmute(vqabs_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqabsq_s16() { + let a: i16x8 = i16x8::new(-32768, 0x7F_FF, -6, -5, -4, -3, -2, -1); + let e: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 6, 5, 4, 3, 2, 1); + let r: i16x8 = transmute(vqabsq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqabs_s32() { + let a: i32x2 = i32x2::new(-2147483648, 0x7F_FF_FF_FF); + let e: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let r: i32x2 = transmute(vqabs_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqabsq_s32() { + let a: i32x4 = i32x4::new(-2147483648, 0x7F_FF_FF_FF, -6, -5); + let e: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 6, 5); + let r: i32x4 = transmute(vqabsq_s32(transmute(a))); + assert_eq!(r, e); + } +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs new file mode 100644 index 000000000..bbee29ae7 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs @@ -0,0 +1,206 @@ +//! Tests for ARM+v7+neon load (vld1) intrinsics. +//! +//! These are included in `{arm, aarch64}::neon`. + +use super::*; + +#[cfg(target_arch = "arm")] +use crate::core_arch::arm::*; + +#[cfg(target_arch = "aarch64")] +use crate::core_arch::aarch64::*; + +use crate::core_arch::simd::*; +use std::mem; +use stdarch_test::simd_test; +#[simd_test(enable = "neon")] +unsafe fn test_vld1_s8() { + let a: [i8; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vld1_s8(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon")] +unsafe fn test_vld1q_s8() { + let a: [i8; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: i8x16 = transmute(vld1q_s8(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon")] +unsafe fn test_vld1_s16() { + let a: [i16; 5] = [0, 1, 2, 3, 4]; + let e = i16x4::new(1, 2, 3, 4); + let r: i16x4 = transmute(vld1_s16(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon")] +unsafe fn test_vld1q_s16() { + let a: [i16; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i16x8 = transmute(vld1q_s16(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon")] +unsafe fn test_vld1_s32() { + let a: [i32; 3] = [0, 1, 2]; + let e = i32x2::new(1, 2); + let r: i32x2 = transmute(vld1_s32(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon")] +unsafe fn test_vld1q_s32() { + let a: [i32; 5] = [0, 1, 2, 3, 4]; + let e = i32x4::new(1, 2, 3, 4); + let r: i32x4 = transmute(vld1q_s32(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon")] +unsafe fn test_vld1_s64() { + let a: [i64; 2] = [0, 1]; + let e = i64x1::new(1); + let r: i64x1 = transmute(vld1_s64(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon")] +unsafe fn test_vld1q_s64() { + let a: [i64; 3] = [0, 1, 2]; + let e = i64x2::new(1, 2); + let r: i64x2 = transmute(vld1q_s64(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon")] +unsafe fn test_vld1_u8() { + let a: [u8; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vld1_u8(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon")] +unsafe fn test_vld1q_u8() { + let a: [u8; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: u8x16 = transmute(vld1q_u8(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon")] +unsafe fn test_vld1_u16() { + let a: [u16; 5] = [0, 1, 2, 3, 4]; + let e = u16x4::new(1, 2, 3, 4); + let r: u16x4 = transmute(vld1_u16(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon")] +unsafe fn test_vld1q_u16() { + let a: [u16; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u16x8 = transmute(vld1q_u16(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon")] +unsafe fn test_vld1_u32() { + let a: [u32; 3] = [0, 1, 2]; + let e = u32x2::new(1, 2); + let r: u32x2 = transmute(vld1_u32(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon")] +unsafe fn test_vld1q_u32() { + let a: [u32; 5] = [0, 1, 2, 3, 4]; + let e = u32x4::new(1, 2, 3, 4); + let r: u32x4 = transmute(vld1q_u32(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon")] +unsafe fn test_vld1_u64() { + let a: [u64; 2] = [0, 1]; + let e = u64x1::new(1); + let r: u64x1 = transmute(vld1_u64(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon")] +unsafe fn test_vld1q_u64() { + let a: [u64; 3] = [0, 1, 2]; + let e = u64x2::new(1, 2); + let r: u64x2 = transmute(vld1q_u64(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon")] +unsafe fn test_vld1_p8() { + let a: [p8; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vld1_p8(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon")] +unsafe fn test_vld1q_p8() { + let a: [p8; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let e = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: u8x16 = transmute(vld1q_p8(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon")] +unsafe fn test_vld1_p16() { + let a: [p16; 5] = [0, 1, 2, 3, 4]; + let e = u16x4::new(1, 2, 3, 4); + let r: u16x4 = transmute(vld1_p16(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon")] +unsafe fn test_vld1q_p16() { + let a: [p16; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u16x8 = transmute(vld1q_p16(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon,aes")] +unsafe fn test_vld1_p64() { + let a: [p64; 2] = [0, 1]; + let e = u64x1::new(1); + let r: u64x1 = transmute(vld1_p64(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon,aes")] +unsafe fn test_vld1q_p64() { + let a: [p64; 3] = [0, 1, 2]; + let e = u64x2::new(1, 2); + let r: u64x2 = transmute(vld1q_p64(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon")] +unsafe fn test_vld1_f32() { + let a: [f32; 3] = [0., 1., 2.]; + let e = f32x2::new(1., 2.); + let r: f32x2 = transmute(vld1_f32(a[1..].as_ptr())); + assert_eq!(r, e) +} + +#[simd_test(enable = "neon")] +unsafe fn test_vld1q_f32() { + let a: [f32; 5] = [0., 1., 2., 3., 4.]; + let e = f32x4::new(1., 2., 3., 4.); + let r: f32x4 = transmute(vld1q_f32(a[1..].as_ptr())); + assert_eq!(r, e) +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/mod.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/mod.rs new file mode 100644 index 000000000..0559aea83 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/mod.rs @@ -0,0 +1,12503 @@ +//! ARMv7 NEON intrinsics + +#[rustfmt::skip] +mod generated; +#[rustfmt::skip] +pub use self::generated::*; + +use crate::{ + core_arch::simd::*, core_arch::simd_llvm::*, hint::unreachable_unchecked, mem::transmute, +}; +#[cfg(test)] +use stdarch_test::assert_instr; + +pub(crate) type p8 = u8; +pub(crate) type p16 = u16; +pub(crate) type p64 = u64; +pub(crate) type p128 = u128; + +types! { + /// ARM-specific 64-bit wide vector of eight packed `i8`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct int8x8_t(pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8); + /// ARM-specific 64-bit wide vector of eight packed `u8`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct uint8x8_t(pub(crate) u8, pub(crate) u8, pub(crate) u8, pub(crate) u8, pub(crate) u8, pub(crate) u8, pub(crate) u8, pub(crate) u8); + /// ARM-specific 64-bit wide polynomial vector of eight packed `p8`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct poly8x8_t(pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8); + /// ARM-specific 64-bit wide vector of four packed `i16`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct int16x4_t(pub(crate) i16, pub(crate) i16, pub(crate) i16, pub(crate) i16); + /// ARM-specific 64-bit wide vector of four packed `u16`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct uint16x4_t(pub(crate) u16, pub(crate) u16, pub(crate) u16, pub(crate) u16); + // FIXME: ARM-specific 64-bit wide vector of four packed `f16`. + // pub struct float16x4_t(f16, f16, f16, f16); + /// ARM-specific 64-bit wide vector of four packed `p16`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct poly16x4_t(pub(crate) p16, pub(crate) p16, pub(crate) p16, pub(crate) p16); + /// ARM-specific 64-bit wide vector of two packed `i32`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct int32x2_t(pub(crate) i32, pub(crate) i32); + /// ARM-specific 64-bit wide vector of two packed `u32`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct uint32x2_t(pub(crate) u32, pub(crate) u32); + /// ARM-specific 64-bit wide vector of two packed `f32`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct float32x2_t(pub(crate) f32, pub(crate) f32); + /// ARM-specific 64-bit wide vector of one packed `i64`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct int64x1_t(pub(crate) i64); + /// ARM-specific 64-bit wide vector of one packed `u64`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct uint64x1_t(pub(crate) u64); + /// ARM-specific 64-bit wide vector of one packed `p64`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct poly64x1_t(pub(crate) p64); + + /// ARM-specific 128-bit wide vector of sixteen packed `i8`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct int8x16_t( + pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8 , pub(crate) i8, pub(crate) i8, + pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8 , pub(crate) i8, pub(crate) i8, + ); + /// ARM-specific 128-bit wide vector of sixteen packed `u8`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct uint8x16_t( + pub(crate) u8, pub(crate) u8 , pub(crate) u8, pub(crate) u8, pub(crate) u8, pub(crate) u8 , pub(crate) u8, pub(crate) u8, + pub(crate) u8, pub(crate) u8 , pub(crate) u8, pub(crate) u8, pub(crate) u8, pub(crate) u8 , pub(crate) u8, pub(crate) u8, + ); + /// ARM-specific 128-bit wide vector of sixteen packed `p8`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct poly8x16_t( + pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, + pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, + ); + /// ARM-specific 128-bit wide vector of eight packed `i16`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct int16x8_t(pub(crate) i16, pub(crate) i16, pub(crate) i16, pub(crate) i16, pub(crate) i16, pub(crate) i16, pub(crate) i16, pub(crate) i16); + /// ARM-specific 128-bit wide vector of eight packed `u16`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct uint16x8_t(pub(crate) u16, pub(crate) u16, pub(crate) u16, pub(crate) u16, pub(crate) u16, pub(crate) u16, pub(crate) u16, pub(crate) u16); + // FIXME: ARM-specific 128-bit wide vector of eight packed `f16`. + // pub struct float16x8_t(f16, f16, f16, f16, f16, f16, f16); + /// ARM-specific 128-bit wide vector of eight packed `p16`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct poly16x8_t(pub(crate) p16, pub(crate) p16, pub(crate) p16, pub(crate) p16, pub(crate) p16, pub(crate) p16, pub(crate) p16, pub(crate) p16); + /// ARM-specific 128-bit wide vector of four packed `i32`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct int32x4_t(pub(crate) i32, pub(crate) i32, pub(crate) i32, pub(crate) i32); + /// ARM-specific 128-bit wide vector of four packed `u32`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct uint32x4_t(pub(crate) u32, pub(crate) u32, pub(crate) u32, pub(crate) u32); + /// ARM-specific 128-bit wide vector of four packed `f32`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct float32x4_t(pub(crate) f32, pub(crate) f32, pub(crate) f32, pub(crate) f32); + /// ARM-specific 128-bit wide vector of two packed `i64`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct int64x2_t(pub(crate) i64, pub(crate) i64); + /// ARM-specific 128-bit wide vector of two packed `u64`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct uint64x2_t(pub(crate) u64, pub(crate) u64); + /// ARM-specific 128-bit wide vector of two packed `p64`. + #[cfg_attr(target_arch = "aarch64", stable(feature = "neon_intrinsics", since = "1.59.0"))] + pub struct poly64x2_t(pub(crate) p64, pub(crate) p64); +} + +/// ARM-specific type containing two `int8x8_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int8x8x2_t(pub int8x8_t, pub int8x8_t); +/// ARM-specific type containing three `int8x8_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int8x8x3_t(pub int8x8_t, pub int8x8_t, pub int8x8_t); +/// ARM-specific type containing four `int8x8_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int8x8x4_t(pub int8x8_t, pub int8x8_t, pub int8x8_t, pub int8x8_t); + +/// ARM-specific type containing two `int8x16_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int8x16x2_t(pub int8x16_t, pub int8x16_t); +/// ARM-specific type containing three `int8x16_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int8x16x3_t(pub int8x16_t, pub int8x16_t, pub int8x16_t); +/// ARM-specific type containing four `int8x16_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int8x16x4_t(pub int8x16_t, pub int8x16_t, pub int8x16_t, pub int8x16_t); + +/// ARM-specific type containing two `uint8x8_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint8x8x2_t(pub uint8x8_t, pub uint8x8_t); +/// ARM-specific type containing three `uint8x8_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint8x8x3_t(pub uint8x8_t, pub uint8x8_t, pub uint8x8_t); +/// ARM-specific type containing four `uint8x8_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint8x8x4_t(pub uint8x8_t, pub uint8x8_t, pub uint8x8_t, pub uint8x8_t); + +/// ARM-specific type containing two `uint8x16_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint8x16x2_t(pub uint8x16_t, pub uint8x16_t); +/// ARM-specific type containing three `uint8x16_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint8x16x3_t(pub uint8x16_t, pub uint8x16_t, pub uint8x16_t); +/// ARM-specific type containing four `uint8x16_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint8x16x4_t( + pub uint8x16_t, + pub uint8x16_t, + pub uint8x16_t, + pub uint8x16_t, +); + +/// ARM-specific type containing two `poly8x8_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct poly8x8x2_t(pub poly8x8_t, pub poly8x8_t); +/// ARM-specific type containing three `poly8x8_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct poly8x8x3_t(pub poly8x8_t, pub poly8x8_t, pub poly8x8_t); +/// ARM-specific type containing four `poly8x8_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct poly8x8x4_t(pub poly8x8_t, pub poly8x8_t, pub poly8x8_t, pub poly8x8_t); + +/// ARM-specific type containing two `poly8x16_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct poly8x16x2_t(pub poly8x16_t, pub poly8x16_t); +/// ARM-specific type containing three `poly8x16_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct poly8x16x3_t(pub poly8x16_t, pub poly8x16_t, pub poly8x16_t); +/// ARM-specific type containing four `poly8x16_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct poly8x16x4_t( + pub poly8x16_t, + pub poly8x16_t, + pub poly8x16_t, + pub poly8x16_t, +); + +/// ARM-specific type containing two `int16x4_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int16x4x2_t(pub int16x4_t, pub int16x4_t); +/// ARM-specific type containing three `int16x4_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int16x4x3_t(pub int16x4_t, pub int16x4_t, pub int16x4_t); +/// ARM-specific type containing four `int16x4_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int16x4x4_t(pub int16x4_t, pub int16x4_t, pub int16x4_t, pub int16x4_t); + +/// ARM-specific type containing two `int16x8_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int16x8x2_t(pub int16x8_t, pub int16x8_t); +/// ARM-specific type containing three `int16x8_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int16x8x3_t(pub int16x8_t, pub int16x8_t, pub int16x8_t); +/// ARM-specific type containing four `int16x8_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int16x8x4_t(pub int16x8_t, pub int16x8_t, pub int16x8_t, pub int16x8_t); + +/// ARM-specific type containing two `uint16x4_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint16x4x2_t(pub uint16x4_t, pub uint16x4_t); +/// ARM-specific type containing three `uint16x4_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint16x4x3_t(pub uint16x4_t, pub uint16x4_t, pub uint16x4_t); +/// ARM-specific type containing four `uint16x4_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint16x4x4_t( + pub uint16x4_t, + pub uint16x4_t, + pub uint16x4_t, + pub uint16x4_t, +); + +/// ARM-specific type containing two `uint16x8_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint16x8x2_t(pub uint16x8_t, pub uint16x8_t); +/// ARM-specific type containing three `uint16x8_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint16x8x3_t(pub uint16x8_t, pub uint16x8_t, pub uint16x8_t); +/// ARM-specific type containing four `uint16x8_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint16x8x4_t( + pub uint16x8_t, + pub uint16x8_t, + pub uint16x8_t, + pub uint16x8_t, +); + +/// ARM-specific type containing two `poly16x4_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct poly16x4x2_t(pub poly16x4_t, pub poly16x4_t); +/// ARM-specific type containing three `poly16x4_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct poly16x4x3_t(pub poly16x4_t, pub poly16x4_t, pub poly16x4_t); +/// ARM-specific type containing four `poly16x4_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct poly16x4x4_t( + pub poly16x4_t, + pub poly16x4_t, + pub poly16x4_t, + pub poly16x4_t, +); + +/// ARM-specific type containing two `poly16x8_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct poly16x8x2_t(pub poly16x8_t, pub poly16x8_t); +/// ARM-specific type containing three `poly16x8_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct poly16x8x3_t(pub poly16x8_t, pub poly16x8_t, pub poly16x8_t); +/// ARM-specific type containing four `poly16x8_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct poly16x8x4_t( + pub poly16x8_t, + pub poly16x8_t, + pub poly16x8_t, + pub poly16x8_t, +); + +/// ARM-specific type containing two `int32x2_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int32x2x2_t(pub int32x2_t, pub int32x2_t); +/// ARM-specific type containing three `int32x2_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int32x2x3_t(pub int32x2_t, pub int32x2_t, pub int32x2_t); +/// ARM-specific type containing four `int32x2_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int32x2x4_t(pub int32x2_t, pub int32x2_t, pub int32x2_t, pub int32x2_t); + +/// ARM-specific type containing two `int32x4_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int32x4x2_t(pub int32x4_t, pub int32x4_t); +/// ARM-specific type containing three `int32x4_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int32x4x3_t(pub int32x4_t, pub int32x4_t, pub int32x4_t); +/// ARM-specific type containing four `int32x4_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int32x4x4_t(pub int32x4_t, pub int32x4_t, pub int32x4_t, pub int32x4_t); + +/// ARM-specific type containing two `uint32x2_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint32x2x2_t(pub uint32x2_t, pub uint32x2_t); +/// ARM-specific type containing three `uint32x2_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint32x2x3_t(pub uint32x2_t, pub uint32x2_t, pub uint32x2_t); +/// ARM-specific type containing four `uint32x2_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint32x2x4_t( + pub uint32x2_t, + pub uint32x2_t, + pub uint32x2_t, + pub uint32x2_t, +); + +/// ARM-specific type containing two `uint32x4_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint32x4x2_t(pub uint32x4_t, pub uint32x4_t); +/// ARM-specific type containing three `uint32x4_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint32x4x3_t(pub uint32x4_t, pub uint32x4_t, pub uint32x4_t); +/// ARM-specific type containing four `uint32x4_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint32x4x4_t( + pub uint32x4_t, + pub uint32x4_t, + pub uint32x4_t, + pub uint32x4_t, +); + +/// ARM-specific type containing two `float32x2_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct float32x2x2_t(pub float32x2_t, pub float32x2_t); +/// ARM-specific type containing three `float32x2_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct float32x2x3_t(pub float32x2_t, pub float32x2_t, pub float32x2_t); +/// ARM-specific type containing four `float32x2_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct float32x2x4_t( + pub float32x2_t, + pub float32x2_t, + pub float32x2_t, + pub float32x2_t, +); + +/// ARM-specific type containing two `float32x4_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct float32x4x2_t(pub float32x4_t, pub float32x4_t); +/// ARM-specific type containing three `float32x4_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct float32x4x3_t(pub float32x4_t, pub float32x4_t, pub float32x4_t); +/// ARM-specific type containing four `float32x4_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct float32x4x4_t( + pub float32x4_t, + pub float32x4_t, + pub float32x4_t, + pub float32x4_t, +); + +/// ARM-specific type containing four `int64x1_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int64x1x2_t(pub int64x1_t, pub int64x1_t); +/// ARM-specific type containing four `int64x1_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int64x1x3_t(pub int64x1_t, pub int64x1_t, pub int64x1_t); +/// ARM-specific type containing four `int64x1_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int64x1x4_t(pub int64x1_t, pub int64x1_t, pub int64x1_t, pub int64x1_t); + +/// ARM-specific type containing four `int64x2_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int64x2x2_t(pub int64x2_t, pub int64x2_t); +/// ARM-specific type containing four `int64x2_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int64x2x3_t(pub int64x2_t, pub int64x2_t, pub int64x2_t); +/// ARM-specific type containing four `int64x2_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct int64x2x4_t(pub int64x2_t, pub int64x2_t, pub int64x2_t, pub int64x2_t); + +/// ARM-specific type containing four `uint64x1_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint64x1x2_t(pub uint64x1_t, pub uint64x1_t); +/// ARM-specific type containing four `uint64x1_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint64x1x3_t(pub uint64x1_t, pub uint64x1_t, pub uint64x1_t); +/// ARM-specific type containing four `uint64x1_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint64x1x4_t( + pub uint64x1_t, + pub uint64x1_t, + pub uint64x1_t, + pub uint64x1_t, +); + +/// ARM-specific type containing four `uint64x2_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint64x2x2_t(pub uint64x2_t, pub uint64x2_t); +/// ARM-specific type containing four `uint64x2_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint64x2x3_t(pub uint64x2_t, pub uint64x2_t, pub uint64x2_t); +/// ARM-specific type containing four `uint64x2_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct uint64x2x4_t( + pub uint64x2_t, + pub uint64x2_t, + pub uint64x2_t, + pub uint64x2_t, +); + +/// ARM-specific type containing four `poly64x1_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct poly64x1x2_t(pub poly64x1_t, pub poly64x1_t); +/// ARM-specific type containing four `poly64x1_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct poly64x1x3_t(pub poly64x1_t, pub poly64x1_t, pub poly64x1_t); +/// ARM-specific type containing four `poly64x1_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct poly64x1x4_t( + pub poly64x1_t, + pub poly64x1_t, + pub poly64x1_t, + pub poly64x1_t, +); + +/// ARM-specific type containing four `poly64x2_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct poly64x2x2_t(pub poly64x2_t, pub poly64x2_t); +/// ARM-specific type containing four `poly64x2_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct poly64x2x3_t(pub poly64x2_t, pub poly64x2_t, pub poly64x2_t); +/// ARM-specific type containing four `poly64x2_t` vectors. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub struct poly64x2x4_t( + pub poly64x2_t, + pub poly64x2_t, + pub poly64x2_t, + pub poly64x2_t, +); + +#[allow(improper_ctypes)] +extern "unadjusted" { + // absolute value (64-bit) + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabs.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.abs.v8i8")] + fn vabs_s8_(a: int8x8_t) -> int8x8_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabs.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.abs.v4i16")] + fn vabs_s16_(a: int16x4_t) -> int16x4_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabs.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.abs.v2i32")] + fn vabs_s32_(a: int32x2_t) -> int32x2_t; + // absolute value (128-bit) + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabs.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.abs.v16i8")] + fn vabsq_s8_(a: int8x16_t) -> int8x16_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabs.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.abs.v8i16")] + fn vabsq_s16_(a: int16x8_t) -> int16x8_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabs.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.abs.v4i32")] + fn vabsq_s32_(a: int32x4_t) -> int32x4_t; + + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmins.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sminp.v8i8")] + fn vpmins_v8i8(a: int8x8_t, b: int8x8_t) -> int8x8_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmins.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sminp.v4i16")] + fn vpmins_v4i16(a: int16x4_t, b: int16x4_t) -> int16x4_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmins.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sminp.v2i32")] + fn vpmins_v2i32(a: int32x2_t, b: int32x2_t) -> int32x2_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpminu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uminp.v8i8")] + fn vpminu_v8i8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpminu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uminp.v4i16")] + fn vpminu_v4i16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpminu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uminp.v2i32")] + fn vpminu_v2i32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmins.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminp.v2f32")] + fn vpminf_v2f32(a: float32x2_t, b: float32x2_t) -> float32x2_t; + + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmaxs.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smaxp.v8i8")] + fn vpmaxs_v8i8(a: int8x8_t, b: int8x8_t) -> int8x8_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmaxs.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smaxp.v4i16")] + fn vpmaxs_v4i16(a: int16x4_t, b: int16x4_t) -> int16x4_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmaxs.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smaxp.v2i32")] + fn vpmaxs_v2i32(a: int32x2_t, b: int32x2_t) -> int32x2_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmaxu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umaxp.v8i8")] + fn vpmaxu_v8i8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmaxu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umaxp.v4i16")] + fn vpmaxu_v4i16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmaxu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umaxp.v2i32")] + fn vpmaxu_v2i32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmaxs.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxp.v2f32")] + fn vpmaxf_v2f32(a: float32x2_t, b: float32x2_t) -> float32x2_t; + + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vraddhn.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.raddhn.v8i8")] + fn vraddhn_s16_(a: int16x8_t, b: int16x8_t) -> int8x8_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vraddhn.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.raddhn.v4i16")] + fn vraddhn_s32_(a: int32x4_t, b: int32x4_t) -> int16x4_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vraddhn.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.raddhn.v2i32")] + fn vraddhn_s64_(a: int64x2_t, b: int64x2_t) -> int32x2_t; + + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpadd.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.addp.v4i16")] + fn vpadd_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpadd.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.addp.v2i32")] + fn vpadd_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpadd.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.addp.v8i8")] + fn vpadd_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddls.v4i16.v8i8")] + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.neon.saddlp.v4i16.v8i8" + )] + pub(crate) fn vpaddl_s8_(a: int8x8_t) -> int16x4_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddls.v2i32.v4i16")] + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.neon.saddlp.v2i32.v4i16" + )] + pub(crate) fn vpaddl_s16_(a: int16x4_t) -> int32x2_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddls.v1i64.v2i32")] + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.neon.saddlp.v1i64.v2i32" + )] + pub(crate) fn vpaddl_s32_(a: int32x2_t) -> int64x1_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddls.v8i16.v16i8")] + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.neon.saddlp.v8i16.v16i8" + )] + pub(crate) fn vpaddlq_s8_(a: int8x16_t) -> int16x8_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddls.v4i32.v8i16")] + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.neon.saddlp.v4i32.v8i16" + )] + pub(crate) fn vpaddlq_s16_(a: int16x8_t) -> int32x4_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddls.v2i64.v4i32")] + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.neon.saddlp.v2i64.v4i32" + )] + pub(crate) fn vpaddlq_s32_(a: int32x4_t) -> int64x2_t; + + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddlu.v4i16.v8i8")] + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.neon.uaddlp.v4i16.v8i8" + )] + pub(crate) fn vpaddl_u8_(a: uint8x8_t) -> uint16x4_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddlu.v2i32.v4i16")] + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.neon.uaddlp.v2i32.v4i16" + )] + pub(crate) fn vpaddl_u16_(a: uint16x4_t) -> uint32x2_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddlu.v1i64.v2i32")] + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.neon.uaddlp.v1i64.v2i32" + )] + pub(crate) fn vpaddl_u32_(a: uint32x2_t) -> uint64x1_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddlu.v8i16.v16i8")] + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.neon.uaddlp.v8i16.v16i8" + )] + pub(crate) fn vpaddlq_u8_(a: uint8x16_t) -> uint16x8_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddlu.v4i32.v8i16")] + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.neon.uaddlp.v4i32.v8i16" + )] + pub(crate) fn vpaddlq_u16_(a: uint16x8_t) -> uint32x4_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddlu.v2i64.v4i32")] + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.neon.uaddlp.v2i64.v4i32" + )] + pub(crate) fn vpaddlq_u32_(a: uint32x4_t) -> uint64x2_t; + + #[cfg_attr(target_arch = "arm", link_name = "llvm.ctpop.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ctpop.v8i8")] + fn vcnt_s8_(a: int8x8_t) -> int8x8_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.ctpop.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ctpop.v16i8")] + fn vcntq_s8_(a: int8x16_t) -> int8x16_t; + + #[cfg_attr(target_arch = "arm", link_name = "llvm.ctlz.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ctlz.v8i8")] + fn vclz_s8_(a: int8x8_t) -> int8x8_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.ctlz.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ctlz.v16i8")] + fn vclzq_s8_(a: int8x16_t) -> int8x16_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.ctlz.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ctlz.v4i16")] + fn vclz_s16_(a: int16x4_t) -> int16x4_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.ctlz.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ctlz.v8i16")] + fn vclzq_s16_(a: int16x8_t) -> int16x8_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.ctlz.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ctlz.v2i32")] + fn vclz_s32_(a: int32x2_t) -> int32x2_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.ctlz.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ctlz.v4i32")] + fn vclzq_s32_(a: int32x4_t) -> int32x4_t; +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", LANE = 7))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 7))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_lane_s8(ptr: *const i8, src: int8x8_t) -> int8x8_t { + static_assert_imm3!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", LANE = 15))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 15))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_lane_s8(ptr: *const i8, src: int8x16_t) -> int8x16_t { + static_assert_imm4!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16", LANE = 3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 3))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_lane_s16(ptr: *const i16, src: int16x4_t) -> int16x4_t { + static_assert_imm2!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16", LANE = 7))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 7))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_lane_s16(ptr: *const i16, src: int16x8_t) -> int16x8_t { + static_assert_imm3!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 1))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_lane_s32(ptr: *const i32, src: int32x2_t) -> int32x2_t { + static_assert_imm1!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32", LANE = 3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 3))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_lane_s32(ptr: *const i32, src: int32x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr", LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr, LANE = 0))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_lane_s64(ptr: *const i64, src: int64x1_t) -> int64x1_t { + static_assert!(LANE : i32 where LANE == 0); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 1))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_lane_s64(ptr: *const i64, src: int64x2_t) -> int64x2_t { + static_assert_imm1!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", LANE = 7))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 7))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_lane_u8(ptr: *const u8, src: uint8x8_t) -> uint8x8_t { + static_assert_imm3!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", LANE = 15))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 15))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_lane_u8(ptr: *const u8, src: uint8x16_t) -> uint8x16_t { + static_assert_imm4!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16", LANE = 3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 3))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_lane_u16(ptr: *const u16, src: uint16x4_t) -> uint16x4_t { + static_assert_imm2!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16", LANE = 7))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 7))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_lane_u16(ptr: *const u16, src: uint16x8_t) -> uint16x8_t { + static_assert_imm3!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 1))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_lane_u32(ptr: *const u32, src: uint32x2_t) -> uint32x2_t { + static_assert_imm1!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32", LANE = 3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 3))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_lane_u32(ptr: *const u32, src: uint32x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr", LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr, LANE = 0))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_lane_u64(ptr: *const u64, src: uint64x1_t) -> uint64x1_t { + static_assert!(LANE : i32 where LANE == 0); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 1))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_lane_u64(ptr: *const u64, src: uint64x2_t) -> uint64x2_t { + static_assert_imm1!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", LANE = 7))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 7))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_lane_p8(ptr: *const p8, src: poly8x8_t) -> poly8x8_t { + static_assert_imm3!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", LANE = 15))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 15))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_lane_p8(ptr: *const p8, src: poly8x16_t) -> poly8x16_t { + static_assert_imm4!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16", LANE = 3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 3))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_lane_p16(ptr: *const p16, src: poly16x4_t) -> poly16x4_t { + static_assert_imm2!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16", LANE = 7))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 7))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_lane_p16(ptr: *const p16, src: poly16x8_t) -> poly16x8_t { + static_assert_imm3!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr", LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr, LANE = 0))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_lane_p64(ptr: *const p64, src: poly64x1_t) -> poly64x1_t { + static_assert!(LANE : i32 where LANE == 0); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 1))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_lane_p64(ptr: *const p64, src: poly64x2_t) -> poly64x2_t { + static_assert_imm1!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 1))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_lane_f32(ptr: *const f32, src: float32x2_t) -> float32x2_t { + static_assert_imm1!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure to one lane of one register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32", LANE = 3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 3))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_lane_f32(ptr: *const f32, src: float32x4_t) -> float32x4_t { + static_assert_imm2!(LANE); + simd_insert(src, LANE as u32, *ptr) +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_dup_s8(ptr: *const i8) -> int8x8_t { + let x = vld1_lane_s8::<0>(ptr, transmute(i8x8::splat(0))); + simd_shuffle8!(x, x, [0, 0, 0, 0, 0, 0, 0, 0]) +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_dup_s8(ptr: *const i8) -> int8x16_t { + let x = vld1q_lane_s8::<0>(ptr, transmute(i8x16::splat(0))); + simd_shuffle16!(x, x, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_dup_s16(ptr: *const i16) -> int16x4_t { + let x = vld1_lane_s16::<0>(ptr, transmute(i16x4::splat(0))); + simd_shuffle4!(x, x, [0, 0, 0, 0]) +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_dup_s16(ptr: *const i16) -> int16x8_t { + let x = vld1q_lane_s16::<0>(ptr, transmute(i16x8::splat(0))); + simd_shuffle8!(x, x, [0, 0, 0, 0, 0, 0, 0, 0]) +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_dup_s32(ptr: *const i32) -> int32x2_t { + let x = vld1_lane_s32::<0>(ptr, transmute(i32x2::splat(0))); + simd_shuffle2!(x, x, [0, 0]) +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_dup_s32(ptr: *const i32) -> int32x4_t { + let x = vld1q_lane_s32::<0>(ptr, transmute(i32x4::splat(0))); + simd_shuffle4!(x, x, [0, 0, 0, 0]) +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_dup_s64(ptr: *const i64) -> int64x1_t { + #[cfg(target_arch = "aarch64")] + { + crate::core_arch::aarch64::vld1_s64(ptr) + } + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::vld1_s64(ptr) + } +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_dup_s64(ptr: *const i64) -> int64x2_t { + let x = vld1q_lane_s64::<0>(ptr, transmute(i64x2::splat(0))); + simd_shuffle2!(x, x, [0, 0]) +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_dup_u8(ptr: *const u8) -> uint8x8_t { + let x = vld1_lane_u8::<0>(ptr, transmute(u8x8::splat(0))); + simd_shuffle8!(x, x, [0, 0, 0, 0, 0, 0, 0, 0]) +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_dup_u8(ptr: *const u8) -> uint8x16_t { + let x = vld1q_lane_u8::<0>(ptr, transmute(u8x16::splat(0))); + simd_shuffle16!(x, x, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_dup_u16(ptr: *const u16) -> uint16x4_t { + let x = vld1_lane_u16::<0>(ptr, transmute(u16x4::splat(0))); + simd_shuffle4!(x, x, [0, 0, 0, 0]) +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_dup_u16(ptr: *const u16) -> uint16x8_t { + let x = vld1q_lane_u16::<0>(ptr, transmute(u16x8::splat(0))); + simd_shuffle8!(x, x, [0, 0, 0, 0, 0, 0, 0, 0]) +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_dup_u32(ptr: *const u32) -> uint32x2_t { + let x = vld1_lane_u32::<0>(ptr, transmute(u32x2::splat(0))); + simd_shuffle2!(x, x, [0, 0]) +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_dup_u32(ptr: *const u32) -> uint32x4_t { + let x = vld1q_lane_u32::<0>(ptr, transmute(u32x4::splat(0))); + simd_shuffle4!(x, x, [0, 0, 0, 0]) +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_dup_u64(ptr: *const u64) -> uint64x1_t { + #[cfg(target_arch = "aarch64")] + { + crate::core_arch::aarch64::vld1_u64(ptr) + } + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::vld1_u64(ptr) + } +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_dup_u64(ptr: *const u64) -> uint64x2_t { + let x = vld1q_lane_u64::<0>(ptr, transmute(u64x2::splat(0))); + simd_shuffle2!(x, x, [0, 0]) +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_dup_p8(ptr: *const p8) -> poly8x8_t { + let x = vld1_lane_p8::<0>(ptr, transmute(u8x8::splat(0))); + simd_shuffle8!(x, x, [0, 0, 0, 0, 0, 0, 0, 0]) +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_dup_p8(ptr: *const p8) -> poly8x16_t { + let x = vld1q_lane_p8::<0>(ptr, transmute(u8x16::splat(0))); + simd_shuffle16!(x, x, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_dup_p16(ptr: *const p16) -> poly16x4_t { + let x = vld1_lane_p16::<0>(ptr, transmute(u16x4::splat(0))); + simd_shuffle4!(x, x, [0, 0, 0, 0]) +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_dup_p16(ptr: *const p16) -> poly16x8_t { + let x = vld1q_lane_p16::<0>(ptr, transmute(u16x8::splat(0))); + simd_shuffle8!(x, x, [0, 0, 0, 0, 0, 0, 0, 0]) +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_dup_f32(ptr: *const f32) -> float32x2_t { + let x = vld1_lane_f32::<0>(ptr, transmute(f32x2::splat(0.))); + simd_shuffle2!(x, x, [0, 0]) +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1_dup_p64(ptr: *const p64) -> poly64x1_t { + #[cfg(target_arch = "aarch64")] + { + crate::core_arch::aarch64::vld1_p64(ptr) + } + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::vld1_p64(ptr) + } +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_p64) +#[inline] +#[target_feature(enable = "neon,aes")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_dup_p64(ptr: *const p64) -> poly64x2_t { + let x = vld1q_lane_p64::<0>(ptr, transmute(u64x2::splat(0))); + simd_shuffle2!(x, x, [0, 0]) +} + +/// Load one single-element structure and Replicate to all lanes (of one register). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vld1q_dup_f32(ptr: *const f32) -> float32x4_t { + let x = vld1q_lane_f32::<0>(ptr, transmute(f32x4::splat(0.))); + simd_shuffle4!(x, x, [0, 0, 0, 0]) +} + +// signed absolute difference and accumulate (64-bit) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("saba"))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaba_s8(a: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t { + simd_add(a, vabd_s8(b, c)) +} +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("saba"))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaba_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t { + simd_add(a, vabd_s16(b, c)) +} +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("saba"))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaba_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t { + simd_add(a, vabd_s32(b, c)) +} +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("uaba"))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaba_u8(a: uint8x8_t, b: uint8x8_t, c: uint8x8_t) -> uint8x8_t { + simd_add(a, vabd_u8(b, c)) +} +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("uaba"))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaba_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x4_t) -> uint16x4_t { + simd_add(a, vabd_u16(b, c)) +} +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("uaba"))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaba_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x2_t) -> uint32x2_t { + simd_add(a, vabd_u32(b, c)) +} +// signed absolute difference and accumulate (128-bit) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("saba"))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vabaq_s8(a: int8x16_t, b: int8x16_t, c: int8x16_t) -> int8x16_t { + simd_add(a, vabdq_s8(b, c)) +} +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("saba"))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vabaq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + simd_add(a, vabdq_s16(b, c)) +} +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("saba"))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vabaq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + simd_add(a, vabdq_s32(b, c)) +} +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("uaba"))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vabaq_u8(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t { + simd_add(a, vabdq_u8(b, c)) +} +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("uaba"))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vabaq_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t { + simd_add(a, vabdq_u16(b, c)) +} +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("uaba"))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vabaq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t { + simd_add(a, vabdq_u32(b, c)) +} + +/// Absolute value (wrapping). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(abs))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vabs_s8(a: int8x8_t) -> int8x8_t { + vabs_s8_(a) +} +/// Absolute value (wrapping). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(abs))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vabs_s16(a: int16x4_t) -> int16x4_t { + vabs_s16_(a) +} +/// Absolute value (wrapping). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(abs))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vabs_s32(a: int32x2_t) -> int32x2_t { + vabs_s32_(a) +} +/// Absolute value (wrapping). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(abs))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vabsq_s8(a: int8x16_t) -> int8x16_t { + vabsq_s8_(a) +} +/// Absolute value (wrapping). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(abs))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vabsq_s16(a: int16x8_t) -> int16x8_t { + vabsq_s16_(a) +} +/// Absolute value (wrapping). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(abs))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vabsq_s32(a: int32x4_t) -> int32x4_t { + vabsq_s32_(a) +} + +/// Add pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpadd_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + vpadd_s16_(a, b) +} +/// Add pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpadd_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + vpadd_s32_(a, b) +} +/// Add pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpadd_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + vpadd_s8_(a, b) +} +/// Add pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpadd_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + transmute(vpadd_s16_(transmute(a), transmute(b))) +} +/// Add pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpadd_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + transmute(vpadd_s32_(transmute(a), transmute(b))) +} +/// Add pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpadd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + transmute(vpadd_s8_(transmute(a), transmute(b))) +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vadd_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_add(a, b) +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_add(a, b) +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vadd_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_add(a, b) +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_add(a, b) +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vadd_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_add(a, b) +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_add(a, b) +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + simd_add(a, b) +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vadd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_add(a, b) +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_add(a, b) +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vadd_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_add(a, b) +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_add(a, b) +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vadd_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_add(a, b) +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_add(a, b) +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(add))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_add(a, b) +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fadd))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vadd_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + simd_add(a, b) +} + +/// Vector add. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fadd))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + simd_add(a, b) +} + +/// Signed Add Long (vector). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddl_s8(a: int8x8_t, b: int8x8_t) -> int16x8_t { + let a: int16x8_t = simd_cast(a); + let b: int16x8_t = simd_cast(b); + simd_add(a, b) +} + +/// Signed Add Long (vector). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddl_s16(a: int16x4_t, b: int16x4_t) -> int32x4_t { + let a: int32x4_t = simd_cast(a); + let b: int32x4_t = simd_cast(b); + simd_add(a, b) +} + +/// Signed Add Long (vector). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddl_s32(a: int32x2_t, b: int32x2_t) -> int64x2_t { + let a: int64x2_t = simd_cast(a); + let b: int64x2_t = simd_cast(b); + simd_add(a, b) +} + +/// Unsigned Add Long (vector). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddl_u8(a: uint8x8_t, b: uint8x8_t) -> uint16x8_t { + let a: uint16x8_t = simd_cast(a); + let b: uint16x8_t = simd_cast(b); + simd_add(a, b) +} + +/// Unsigned Add Long (vector). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddl_u16(a: uint16x4_t, b: uint16x4_t) -> uint32x4_t { + let a: uint32x4_t = simd_cast(a); + let b: uint32x4_t = simd_cast(b); + simd_add(a, b) +} + +/// Unsigned Add Long (vector). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddl_u32(a: uint32x2_t, b: uint32x2_t) -> uint64x2_t { + let a: uint64x2_t = simd_cast(a); + let b: uint64x2_t = simd_cast(b); + simd_add(a, b) +} + +/// Signed Add Long (vector, high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddl2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddl_high_s8(a: int8x16_t, b: int8x16_t) -> int16x8_t { + let a: int8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + let b: int8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let a: int16x8_t = simd_cast(a); + let b: int16x8_t = simd_cast(b); + simd_add(a, b) +} + +/// Signed Add Long (vector, high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddl2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddl_high_s16(a: int16x8_t, b: int16x8_t) -> int32x4_t { + let a: int16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let b: int16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let a: int32x4_t = simd_cast(a); + let b: int32x4_t = simd_cast(b); + simd_add(a, b) +} + +/// Signed Add Long (vector, high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddl2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddl_high_s32(a: int32x4_t, b: int32x4_t) -> int64x2_t { + let a: int32x2_t = simd_shuffle2!(a, a, [2, 3]); + let b: int32x2_t = simd_shuffle2!(b, b, [2, 3]); + let a: int64x2_t = simd_cast(a); + let b: int64x2_t = simd_cast(b); + simd_add(a, b) +} + +/// Unsigned Add Long (vector, high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddl2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddl_high_u8(a: uint8x16_t, b: uint8x16_t) -> uint16x8_t { + let a: uint8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + let b: uint8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let a: uint16x8_t = simd_cast(a); + let b: uint16x8_t = simd_cast(b); + simd_add(a, b) +} + +/// Unsigned Add Long (vector, high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddl2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddl_high_u16(a: uint16x8_t, b: uint16x8_t) -> uint32x4_t { + let a: uint16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let b: uint16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let a: uint32x4_t = simd_cast(a); + let b: uint32x4_t = simd_cast(b); + simd_add(a, b) +} + +/// Unsigned Add Long (vector, high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddl2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddl_high_u32(a: uint32x4_t, b: uint32x4_t) -> uint64x2_t { + let a: uint32x2_t = simd_shuffle2!(a, a, [2, 3]); + let b: uint32x2_t = simd_shuffle2!(b, b, [2, 3]); + let a: uint64x2_t = simd_cast(a); + let b: uint64x2_t = simd_cast(b); + simd_add(a, b) +} + +/// Signed Add Wide. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddw))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddw_s8(a: int16x8_t, b: int8x8_t) -> int16x8_t { + let b: int16x8_t = simd_cast(b); + simd_add(a, b) +} + +/// Signed Add Wide. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddw))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddw_s16(a: int32x4_t, b: int16x4_t) -> int32x4_t { + let b: int32x4_t = simd_cast(b); + simd_add(a, b) +} + +/// Signed Add Wide. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddw))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddw_s32(a: int64x2_t, b: int32x2_t) -> int64x2_t { + let b: int64x2_t = simd_cast(b); + simd_add(a, b) +} + +/// Unsigned Add Wide. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddw))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddw_u8(a: uint16x8_t, b: uint8x8_t) -> uint16x8_t { + let b: uint16x8_t = simd_cast(b); + simd_add(a, b) +} + +/// Unsigned Add Wide. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddw))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddw_u16(a: uint32x4_t, b: uint16x4_t) -> uint32x4_t { + let b: uint32x4_t = simd_cast(b); + simd_add(a, b) +} + +/// Unsigned Add Wide. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddw))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddw_u32(a: uint64x2_t, b: uint32x2_t) -> uint64x2_t { + let b: uint64x2_t = simd_cast(b); + simd_add(a, b) +} + +/// Signed Add Wide (high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddw2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddw_high_s8(a: int16x8_t, b: int8x16_t) -> int16x8_t { + let b: int8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let b: int16x8_t = simd_cast(b); + simd_add(a, b) +} + +/// Signed Add Wide (high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddw2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddw_high_s16(a: int32x4_t, b: int16x8_t) -> int32x4_t { + let b: int16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let b: int32x4_t = simd_cast(b); + simd_add(a, b) +} + +/// Signed Add Wide (high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddw2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddw_high_s32(a: int64x2_t, b: int32x4_t) -> int64x2_t { + let b: int32x2_t = simd_shuffle2!(b, b, [2, 3]); + let b: int64x2_t = simd_cast(b); + simd_add(a, b) +} + +/// Unsigned Add Wide (high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddw2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddw_high_u8(a: uint16x8_t, b: uint8x16_t) -> uint16x8_t { + let b: uint8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let b: uint16x8_t = simd_cast(b); + simd_add(a, b) +} + +/// Unsigned Add Wide (high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddw2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddw_high_u16(a: uint32x4_t, b: uint16x8_t) -> uint32x4_t { + let b: uint16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let b: uint32x4_t = simd_cast(b); + simd_add(a, b) +} + +/// Unsigned Add Wide (high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddw2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddw_high_u32(a: uint64x2_t, b: uint32x4_t) -> uint64x2_t { + let b: uint32x2_t = simd_shuffle2!(b, b, [2, 3]); + let b: uint64x2_t = simd_cast(b); + simd_add(a, b) +} + +/// Add returning High Narrow. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addhn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddhn_s16(a: int16x8_t, b: int16x8_t) -> int8x8_t { + simd_cast(simd_shr(simd_add(a, b), int16x8_t(8, 8, 8, 8, 8, 8, 8, 8))) +} + +/// Add returning High Narrow. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addhn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddhn_s32(a: int32x4_t, b: int32x4_t) -> int16x4_t { + simd_cast(simd_shr(simd_add(a, b), int32x4_t(16, 16, 16, 16))) +} + +/// Add returning High Narrow. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addhn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddhn_s64(a: int64x2_t, b: int64x2_t) -> int32x2_t { + simd_cast(simd_shr(simd_add(a, b), int64x2_t(32, 32))) +} + +/// Add returning High Narrow. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addhn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddhn_u16(a: uint16x8_t, b: uint16x8_t) -> uint8x8_t { + simd_cast(simd_shr(simd_add(a, b), uint16x8_t(8, 8, 8, 8, 8, 8, 8, 8))) +} + +/// Add returning High Narrow. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addhn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddhn_u32(a: uint32x4_t, b: uint32x4_t) -> uint16x4_t { + simd_cast(simd_shr(simd_add(a, b), uint32x4_t(16, 16, 16, 16))) +} + +/// Add returning High Narrow. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addhn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddhn_u64(a: uint64x2_t, b: uint64x2_t) -> uint32x2_t { + simd_cast(simd_shr(simd_add(a, b), uint64x2_t(32, 32))) +} + +/// Add returning High Narrow (high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addhn2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddhn_high_s16(r: int8x8_t, a: int16x8_t, b: int16x8_t) -> int8x16_t { + let x = simd_cast(simd_shr(simd_add(a, b), int16x8_t(8, 8, 8, 8, 8, 8, 8, 8))); + simd_shuffle16!(r, x, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Add returning High Narrow (high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addhn2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddhn_high_s32(r: int16x4_t, a: int32x4_t, b: int32x4_t) -> int16x8_t { + let x = simd_cast(simd_shr(simd_add(a, b), int32x4_t(16, 16, 16, 16))); + simd_shuffle8!(r, x, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Add returning High Narrow (high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addhn2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddhn_high_s64(r: int32x2_t, a: int64x2_t, b: int64x2_t) -> int32x4_t { + let x = simd_cast(simd_shr(simd_add(a, b), int64x2_t(32, 32))); + simd_shuffle4!(r, x, [0, 1, 2, 3]) +} + +/// Add returning High Narrow (high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addhn2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddhn_high_u16(r: uint8x8_t, a: uint16x8_t, b: uint16x8_t) -> uint8x16_t { + let x = simd_cast(simd_shr(simd_add(a, b), uint16x8_t(8, 8, 8, 8, 8, 8, 8, 8))); + simd_shuffle16!(r, x, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Add returning High Narrow (high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addhn2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddhn_high_u32(r: uint16x4_t, a: uint32x4_t, b: uint32x4_t) -> uint16x8_t { + let x = simd_cast(simd_shr(simd_add(a, b), uint32x4_t(16, 16, 16, 16))); + simd_shuffle8!(r, x, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Add returning High Narrow (high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addhn2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vaddhn_high_u64(r: uint32x2_t, a: uint64x2_t, b: uint64x2_t) -> uint32x4_t { + let x = simd_cast(simd_shr(simd_add(a, b), uint64x2_t(32, 32))); + simd_shuffle4!(r, x, [0, 1, 2, 3]) +} + +/// Rounding Add returning High Narrow. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vraddhn.i16))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(raddhn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vraddhn_s16(a: int16x8_t, b: int16x8_t) -> int8x8_t { + vraddhn_s16_(a, b) +} + +/// Rounding Add returning High Narrow. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vraddhn.i32))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(raddhn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vraddhn_s32(a: int32x4_t, b: int32x4_t) -> int16x4_t { + vraddhn_s32_(a, b) +} + +/// Rounding Add returning High Narrow. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vraddhn.i64))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(raddhn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vraddhn_s64(a: int64x2_t, b: int64x2_t) -> int32x2_t { + vraddhn_s64_(a, b) +} + +/// Rounding Add returning High Narrow. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vraddhn.i16))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(raddhn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vraddhn_u16(a: uint16x8_t, b: uint16x8_t) -> uint8x8_t { + transmute(vraddhn_s16_(transmute(a), transmute(b))) +} + +/// Rounding Add returning High Narrow. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vraddhn.i32))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(raddhn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vraddhn_u32(a: uint32x4_t, b: uint32x4_t) -> uint16x4_t { + transmute(vraddhn_s32_(transmute(a), transmute(b))) +} + +/// Rounding Add returning High Narrow. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vraddhn.i64))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(raddhn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vraddhn_u64(a: uint64x2_t, b: uint64x2_t) -> uint32x2_t { + transmute(vraddhn_s64_(transmute(a), transmute(b))) +} + +/// Rounding Add returning High Narrow (high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vraddhn.i16))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(raddhn2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vraddhn_high_s16(r: int8x8_t, a: int16x8_t, b: int16x8_t) -> int8x16_t { + let x = vraddhn_s16_(a, b); + simd_shuffle16!(r, x, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Rounding Add returning High Narrow (high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vraddhn.i32))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(raddhn2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vraddhn_high_s32(r: int16x4_t, a: int32x4_t, b: int32x4_t) -> int16x8_t { + let x = vraddhn_s32_(a, b); + simd_shuffle8!(r, x, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Rounding Add returning High Narrow (high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vraddhn.i64))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(raddhn2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vraddhn_high_s64(r: int32x2_t, a: int64x2_t, b: int64x2_t) -> int32x4_t { + let x = vraddhn_s64_(a, b); + simd_shuffle4!(r, x, [0, 1, 2, 3]) +} + +/// Rounding Add returning High Narrow (high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vraddhn.i16))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(raddhn2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vraddhn_high_u16(r: uint8x8_t, a: uint16x8_t, b: uint16x8_t) -> uint8x16_t { + let x: uint8x8_t = transmute(vraddhn_s16_(transmute(a), transmute(b))); + simd_shuffle16!(r, x, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Rounding Add returning High Narrow (high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vraddhn.i32))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(raddhn2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vraddhn_high_u32(r: uint16x4_t, a: uint32x4_t, b: uint32x4_t) -> uint16x8_t { + let x: uint16x4_t = transmute(vraddhn_s32_(transmute(a), transmute(b))); + simd_shuffle8!(r, x, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Rounding Add returning High Narrow (high half). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vraddhn.i64))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(raddhn2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vraddhn_high_u64(r: uint32x2_t, a: uint64x2_t, b: uint64x2_t) -> uint32x4_t { + let x: uint32x2_t = transmute(vraddhn_s64_(transmute(a), transmute(b))); + simd_shuffle4!(r, x, [0, 1, 2, 3]) +} + +/// Signed Add Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpaddl.s8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddlp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpaddl_s8(a: int8x8_t) -> int16x4_t { + vpaddl_s8_(a) +} + +/// Signed Add Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpaddl.s16))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddlp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpaddl_s16(a: int16x4_t) -> int32x2_t { + vpaddl_s16_(a) +} + +/// Signed Add Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpaddl.s32))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddlp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpaddl_s32(a: int32x2_t) -> int64x1_t { + vpaddl_s32_(a) +} + +/// Signed Add Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpaddl.s8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddlp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpaddlq_s8(a: int8x16_t) -> int16x8_t { + vpaddlq_s8_(a) +} + +/// Signed Add Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpaddl.s16))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddlp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpaddlq_s16(a: int16x8_t) -> int32x4_t { + vpaddlq_s16_(a) +} + +/// Signed Add Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpaddl.s32))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddlp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpaddlq_s32(a: int32x4_t) -> int64x2_t { + vpaddlq_s32_(a) +} + +/// Unsigned Add Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpaddl.u8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddlp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpaddl_u8(a: uint8x8_t) -> uint16x4_t { + vpaddl_u8_(a) +} + +/// Unsigned Add Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpaddl.u16))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddlp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpaddl_u16(a: uint16x4_t) -> uint32x2_t { + vpaddl_u16_(a) +} + +/// Unsigned Add Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpaddl.u32))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddlp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpaddl_u32(a: uint32x2_t) -> uint64x1_t { + vpaddl_u32_(a) +} + +/// Unsigned Add Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpaddl.u8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddlp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpaddlq_u8(a: uint8x16_t) -> uint16x8_t { + vpaddlq_u8_(a) +} + +/// Unsigned Add Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpaddl.u16))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddlp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpaddlq_u16(a: uint16x8_t) -> uint32x4_t { + vpaddlq_u16_(a) +} + +/// Unsigned Add Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpaddl.u32))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddlp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpaddlq_u32(a: uint32x4_t) -> uint64x2_t { + vpaddlq_u32_(a) +} + +/// Vector narrow integer. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovn_s16(a: int16x8_t) -> int8x8_t { + simd_cast(a) +} + +/// Vector narrow integer. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovn_s32(a: int32x4_t) -> int16x4_t { + simd_cast(a) +} + +/// Vector narrow integer. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovn_s64(a: int64x2_t) -> int32x2_t { + simd_cast(a) +} + +/// Vector narrow integer. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovn_u16(a: uint16x8_t) -> uint8x8_t { + simd_cast(a) +} + +/// Vector narrow integer. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovn_u32(a: uint32x4_t) -> uint16x4_t { + simd_cast(a) +} + +/// Vector narrow integer. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovn_u64(a: uint64x2_t) -> uint32x2_t { + simd_cast(a) +} + +/// Vector long move. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sxtl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovl_s8(a: int8x8_t) -> int16x8_t { + simd_cast(a) +} + +/// Vector long move. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sxtl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovl_s16(a: int16x4_t) -> int32x4_t { + simd_cast(a) +} + +/// Vector long move. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sxtl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovl_s32(a: int32x2_t) -> int64x2_t { + simd_cast(a) +} + +/// Vector long move. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uxtl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovl_u8(a: uint8x8_t) -> uint16x8_t { + simd_cast(a) +} + +/// Vector long move. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uxtl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovl_u16(a: uint16x4_t) -> uint32x4_t { + simd_cast(a) +} + +/// Vector long move. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uxtl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovl_u32(a: uint32x2_t) -> uint64x2_t { + simd_cast(a) +} + +/// Vector bitwise not. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmvn_s8(a: int8x8_t) -> int8x8_t { + let b = int8x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + simd_xor(a, b) +} + +/// Vector bitwise not. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmvnq_s8(a: int8x16_t) -> int8x16_t { + let b = int8x16_t( + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + ); + simd_xor(a, b) +} + +/// Vector bitwise not. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmvn_s16(a: int16x4_t) -> int16x4_t { + let b = int16x4_t(-1, -1, -1, -1); + simd_xor(a, b) +} + +/// Vector bitwise not. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmvnq_s16(a: int16x8_t) -> int16x8_t { + let b = int16x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + simd_xor(a, b) +} + +/// Vector bitwise not. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmvn_s32(a: int32x2_t) -> int32x2_t { + let b = int32x2_t(-1, -1); + simd_xor(a, b) +} + +/// Vector bitwise not. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmvnq_s32(a: int32x4_t) -> int32x4_t { + let b = int32x4_t(-1, -1, -1, -1); + simd_xor(a, b) +} + +/// Vector bitwise not. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmvn_u8(a: uint8x8_t) -> uint8x8_t { + let b = uint8x8_t(255, 255, 255, 255, 255, 255, 255, 255); + simd_xor(a, b) +} + +/// Vector bitwise not. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmvnq_u8(a: uint8x16_t) -> uint8x16_t { + let b = uint8x16_t( + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + ); + simd_xor(a, b) +} + +/// Vector bitwise not. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmvn_u16(a: uint16x4_t) -> uint16x4_t { + let b = uint16x4_t(65_535, 65_535, 65_535, 65_535); + simd_xor(a, b) +} + +/// Vector bitwise not. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmvnq_u16(a: uint16x8_t) -> uint16x8_t { + let b = uint16x8_t( + 65_535, 65_535, 65_535, 65_535, 65_535, 65_535, 65_535, 65_535, + ); + simd_xor(a, b) +} + +/// Vector bitwise not. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmvn_u32(a: uint32x2_t) -> uint32x2_t { + let b = uint32x2_t(4_294_967_295, 4_294_967_295); + simd_xor(a, b) +} + +/// Vector bitwise not. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmvnq_u32(a: uint32x4_t) -> uint32x4_t { + let b = uint32x4_t(4_294_967_295, 4_294_967_295, 4_294_967_295, 4_294_967_295); + simd_xor(a, b) +} + +/// Vector bitwise not. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmvn_p8(a: poly8x8_t) -> poly8x8_t { + let b = poly8x8_t(255, 255, 255, 255, 255, 255, 255, 255); + simd_xor(a, b) +} + +/// Vector bitwise not. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mvn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmvnq_p8(a: poly8x16_t) -> poly8x16_t { + let b = poly8x16_t( + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + ); + simd_xor(a, b) +} + +/// Vector bitwise bit clear +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbic_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + let c = int8x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + simd_and(simd_xor(b, c), a) +} + +/// Vector bitwise bit clear +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbicq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + let c = int8x16_t( + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + ); + simd_and(simd_xor(b, c), a) +} + +/// Vector bitwise bit clear +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbic_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + let c = int16x4_t(-1, -1, -1, -1); + simd_and(simd_xor(b, c), a) +} + +/// Vector bitwise bit clear +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbicq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + let c = int16x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + simd_and(simd_xor(b, c), a) +} + +/// Vector bitwise bit clear +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbic_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + let c = int32x2_t(-1, -1); + simd_and(simd_xor(b, c), a) +} + +/// Vector bitwise bit clear +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbicq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + let c = int32x4_t(-1, -1, -1, -1); + simd_and(simd_xor(b, c), a) +} + +/// Vector bitwise bit clear +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbic_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + let c = int64x1_t(-1); + simd_and(simd_xor(b, c), a) +} + +/// Vector bitwise bit clear +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbicq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + let c = int64x2_t(-1, -1); + simd_and(simd_xor(b, c), a) +} + +/// Vector bitwise bit clear +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbic_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + let c = int8x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + simd_and(simd_xor(b, transmute(c)), a) +} + +/// Vector bitwise bit clear +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbicq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + let c = int8x16_t( + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + ); + simd_and(simd_xor(b, transmute(c)), a) +} + +/// Vector bitwise bit clear +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbic_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + let c = int16x4_t(-1, -1, -1, -1); + simd_and(simd_xor(b, transmute(c)), a) +} + +/// Vector bitwise bit clear +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbicq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + let c = int16x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + simd_and(simd_xor(b, transmute(c)), a) +} + +/// Vector bitwise bit clear +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbic_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + let c = int32x2_t(-1, -1); + simd_and(simd_xor(b, transmute(c)), a) +} + +/// Vector bitwise bit clear +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbicq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + let c = int32x4_t(-1, -1, -1, -1); + simd_and(simd_xor(b, transmute(c)), a) +} + +/// Vector bitwise bit clear +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbic_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + let c = int64x1_t(-1); + simd_and(simd_xor(b, transmute(c)), a) +} + +/// Vector bitwise bit clear +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbicq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + let c = int64x2_t(-1, -1); + simd_and(simd_xor(b, transmute(c)), a) +} + +/// Bitwise Select instructions. This instruction sets each bit in the destination SIMD&FP register +/// to the corresponding bit from the first source SIMD&FP register when the original +/// destination bit was 1, otherwise from the second source SIMD&FP register. + +/// Bitwise Select. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbsl_s8(a: uint8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t { + let not = int8x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Bitwise Select. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbsl_s16(a: uint16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t { + let not = int16x4_t(-1, -1, -1, -1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Bitwise Select. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbsl_s32(a: uint32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t { + let not = int32x2_t(-1, -1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Bitwise Select. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbsl_s64(a: uint64x1_t, b: int64x1_t, c: int64x1_t) -> int64x1_t { + let not = int64x1_t(-1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Bitwise Select. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbsl_u8(a: uint8x8_t, b: uint8x8_t, c: uint8x8_t) -> uint8x8_t { + let not = int8x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Bitwise Select. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbsl_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x4_t) -> uint16x4_t { + let not = int16x4_t(-1, -1, -1, -1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Bitwise Select. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbsl_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x2_t) -> uint32x2_t { + let not = int32x2_t(-1, -1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Bitwise Select. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbsl_u64(a: uint64x1_t, b: uint64x1_t, c: uint64x1_t) -> uint64x1_t { + let not = int64x1_t(-1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Bitwise Select. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbsl_f32(a: uint32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + let not = int32x2_t(-1, -1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Bitwise Select. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbsl_p8(a: uint8x8_t, b: poly8x8_t, c: poly8x8_t) -> poly8x8_t { + let not = int8x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Bitwise Select. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbsl_p16(a: uint16x4_t, b: poly16x4_t, c: poly16x4_t) -> poly16x4_t { + let not = int16x4_t(-1, -1, -1, -1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Bitwise Select. (128-bit) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbslq_s8(a: uint8x16_t, b: int8x16_t, c: int8x16_t) -> int8x16_t { + let not = int8x16_t( + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + ); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Bitwise Select. (128-bit) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbslq_s16(a: uint16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + let not = int16x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Bitwise Select. (128-bit) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbslq_s32(a: uint32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + let not = int32x4_t(-1, -1, -1, -1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Bitwise Select. (128-bit) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbslq_s64(a: uint64x2_t, b: int64x2_t, c: int64x2_t) -> int64x2_t { + let not = int64x2_t(-1, -1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Bitwise Select. (128-bit) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbslq_u8(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t { + let not = int8x16_t( + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + ); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Bitwise Select. (128-bit) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbslq_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t { + let not = int16x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Bitwise Select. (128-bit) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbslq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t { + let not = int32x4_t(-1, -1, -1, -1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Bitwise Select. (128-bit) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbslq_u64(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t { + let not = int64x2_t(-1, -1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Bitwise Select. (128-bit) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbslq_p8(a: uint8x16_t, b: poly8x16_t, c: poly8x16_t) -> poly8x16_t { + let not = int8x16_t( + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + ); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Bitwise Select. (128-bit) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbslq_p16(a: uint16x8_t, b: poly16x8_t, c: poly16x8_t) -> poly16x8_t { + let not = int16x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Bitwise Select. (128-bit) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vbslq_f32(a: uint32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + let not = int32x4_t(-1, -1, -1, -1); + transmute(simd_or( + simd_and(a, transmute(b)), + simd_and(simd_xor(a, transmute(not)), transmute(c)), + )) +} + +/// Vector bitwise inclusive OR NOT +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vorn_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + let c = int8x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + simd_or(simd_xor(b, c), a) +} + +/// Vector bitwise inclusive OR NOT +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vornq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + let c = int8x16_t( + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + ); + simd_or(simd_xor(b, c), a) +} + +/// Vector bitwise inclusive OR NOT +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vorn_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + let c = int16x4_t(-1, -1, -1, -1); + simd_or(simd_xor(b, c), a) +} + +/// Vector bitwise inclusive OR NOT +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vornq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + let c = int16x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + simd_or(simd_xor(b, c), a) +} + +/// Vector bitwise inclusive OR NOT +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vorn_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + let c = int32x2_t(-1, -1); + simd_or(simd_xor(b, c), a) +} + +/// Vector bitwise inclusive OR NOT +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vornq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + let c = int32x4_t(-1, -1, -1, -1); + simd_or(simd_xor(b, c), a) +} + +/// Vector bitwise inclusive OR NOT +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vorn_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + let c = int64x1_t(-1); + simd_or(simd_xor(b, c), a) +} + +/// Vector bitwise inclusive OR NOT +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vornq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + let c = int64x2_t(-1, -1); + simd_or(simd_xor(b, c), a) +} + +/// Vector bitwise inclusive OR NOT +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vorn_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + let c = int8x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + simd_or(simd_xor(b, transmute(c)), a) +} + +/// Vector bitwise inclusive OR NOT +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vornq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + let c = int8x16_t( + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + ); + simd_or(simd_xor(b, transmute(c)), a) +} + +/// Vector bitwise inclusive OR NOT +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vorn_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + let c = int16x4_t(-1, -1, -1, -1); + simd_or(simd_xor(b, transmute(c)), a) +} + +/// Vector bitwise inclusive OR NOT +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vornq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + let c = int16x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + simd_or(simd_xor(b, transmute(c)), a) +} + +/// Vector bitwise inclusive OR NOT +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vorn_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + let c = int32x2_t(-1, -1); + simd_or(simd_xor(b, transmute(c)), a) +} + +/// Vector bitwise inclusive OR NOT +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vornq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + let c = int32x4_t(-1, -1, -1, -1); + simd_or(simd_xor(b, transmute(c)), a) +} + +/// Vector bitwise inclusive OR NOT +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vorn_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + let c = int64x1_t(-1); + simd_or(simd_xor(b, transmute(c)), a) +} + +/// Vector bitwise inclusive OR NOT +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vornq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + let c = int64x2_t(-1, -1); + simd_or(simd_xor(b, transmute(c)), a) +} + +/// Folding minimum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sminp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpmin_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + vpmins_v8i8(a, b) +} + +/// Folding minimum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sminp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpmin_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + vpmins_v4i16(a, b) +} + +/// Folding minimum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sminp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpmin_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + vpmins_v2i32(a, b) +} + +/// Folding minimum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uminp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpmin_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + vpminu_v8i8(a, b) +} + +/// Folding minimum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uminp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpmin_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + vpminu_v4i16(a, b) +} + +/// Folding minimum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uminp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpmin_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + vpminu_v2i32(a, b) +} + +/// Folding minimum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fminp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpmin_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + vpminf_v2f32(a, b) +} + +/// Folding maximum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smaxp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpmax_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + vpmaxs_v8i8(a, b) +} + +/// Folding maximum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smaxp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpmax_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + vpmaxs_v4i16(a, b) +} + +/// Folding maximum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smaxp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpmax_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + vpmaxs_v2i32(a, b) +} + +/// Folding maximum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umaxp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpmax_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + vpmaxu_v8i8(a, b) +} + +/// Folding maximum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umaxp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpmax_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + vpmaxu_v4i16(a, b) +} + +/// Folding maximum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umaxp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpmax_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + vpmaxu_v2i32(a, b) +} + +/// Folding maximum of adjacent pairs +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmaxp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpmax_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + vpmaxf_v2f32(a, b) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 1))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vgetq_lane_u64(v: uint64x2_t) -> u64 { + static_assert_imm1!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 0))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_lane_u64(v: uint64x1_t) -> u64 { + static_assert!(IMM5 : i32 where IMM5 == 0); + simd_extract(v, 0) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_lane_u16(v: uint16x4_t) -> u16 { + static_assert_imm2!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_lane_s16(v: int16x4_t) -> i16 { + static_assert_imm2!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_lane_p16(v: poly16x4_t) -> p16 { + static_assert_imm2!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 1))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_lane_u32(v: uint32x2_t) -> u32 { + static_assert_imm1!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 1))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_lane_s32(v: int32x2_t) -> i32 { + static_assert_imm1!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 1))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_lane_f32(v: float32x2_t) -> f32 { + static_assert_imm1!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 1))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vgetq_lane_f32(v: float32x4_t) -> f32 { + static_assert_imm2!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 0))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_lane_p64(v: poly64x1_t) -> p64 { + static_assert!(IMM5 : i32 where IMM5 == 0); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 0))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vgetq_lane_p64(v: poly64x2_t) -> p64 { + static_assert_imm1!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 0))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_lane_s64(v: int64x1_t) -> i64 { + static_assert!(IMM5 : i32 where IMM5 == 0); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 0))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vgetq_lane_s64(v: int64x2_t) -> i64 { + static_assert_imm1!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vgetq_lane_u16(v: uint16x8_t) -> u16 { + static_assert_imm3!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vgetq_lane_u32(v: uint32x4_t) -> u32 { + static_assert_imm2!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vgetq_lane_s16(v: int16x8_t) -> i16 { + static_assert_imm3!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vgetq_lane_p16(v: poly16x8_t) -> p16 { + static_assert_imm3!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vgetq_lane_s32(v: int32x4_t) -> i32 { + static_assert_imm2!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_lane_u8(v: uint8x8_t) -> u8 { + static_assert_imm3!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_lane_s8(v: int8x8_t) -> i8 { + static_assert_imm3!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_lane_p8(v: poly8x8_t) -> p8 { + static_assert_imm3!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vgetq_lane_u8(v: uint8x16_t) -> u8 { + static_assert_imm4!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vgetq_lane_s8(v: int8x16_t) -> i8 { + static_assert_imm4!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(test, assert_instr(nop, IMM5 = 2))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vgetq_lane_p8(v: poly8x16_t) -> p8 { + static_assert_imm4!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_high_s8(a: int8x16_t) -> int8x8_t { + simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_high_s16(a: int16x8_t) -> int16x4_t { + simd_shuffle4!(a, a, [4, 5, 6, 7]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_high_s32(a: int32x4_t) -> int32x2_t { + simd_shuffle2!(a, a, [2, 3]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_high_s64(a: int64x2_t) -> int64x1_t { + int64x1_t(simd_extract(a, 1)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_high_u8(a: uint8x16_t) -> uint8x8_t { + simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_high_u16(a: uint16x8_t) -> uint16x4_t { + simd_shuffle4!(a, a, [4, 5, 6, 7]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_high_u32(a: uint32x4_t) -> uint32x2_t { + simd_shuffle2!(a, a, [2, 3]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_high_u64(a: uint64x2_t) -> uint64x1_t { + uint64x1_t(simd_extract(a, 1)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_high_p8(a: poly8x16_t) -> poly8x8_t { + simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_high_p16(a: poly16x8_t) -> poly16x4_t { + simd_shuffle4!(a, a, [4, 5, 6, 7]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_high_f32(a: float32x4_t) -> float32x2_t { + simd_shuffle2!(a, a, [2, 3]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(test, assert_instr(nop))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "vget_low_s8", since = "1.60.0") +)] +pub unsafe fn vget_low_s8(a: int8x16_t) -> int8x8_t { + simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(test, assert_instr(nop))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_low_s16(a: int16x8_t) -> int16x4_t { + simd_shuffle4!(a, a, [0, 1, 2, 3]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(test, assert_instr(nop))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_low_s32(a: int32x4_t) -> int32x2_t { + simd_shuffle2!(a, a, [0, 1]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(test, assert_instr(nop))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_low_s64(a: int64x2_t) -> int64x1_t { + int64x1_t(simd_extract(a, 0)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(test, assert_instr(nop))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_low_u8(a: uint8x16_t) -> uint8x8_t { + simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(test, assert_instr(nop))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_low_u16(a: uint16x8_t) -> uint16x4_t { + simd_shuffle4!(a, a, [0, 1, 2, 3]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(test, assert_instr(nop))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_low_u32(a: uint32x4_t) -> uint32x2_t { + simd_shuffle2!(a, a, [0, 1]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(test, assert_instr(nop))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_low_u64(a: uint64x2_t) -> uint64x1_t { + uint64x1_t(simd_extract(a, 0)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(test, assert_instr(nop))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_low_p8(a: poly8x16_t) -> poly8x8_t { + simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(test, assert_instr(nop))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_low_p16(a: poly16x8_t) -> poly16x4_t { + simd_shuffle4!(a, a, [0, 1, 2, 3]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(test, assert_instr(nop))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vget_low_f32(a: float32x4_t) -> float32x2_t { + simd_shuffle2!(a, a, [0, 1]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdupq_n_s8(value: i8) -> int8x16_t { + int8x16_t( + value, value, value, value, value, value, value, value, value, value, value, value, value, + value, value, value, + ) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdupq_n_s16(value: i16) -> int16x8_t { + int16x8_t(value, value, value, value, value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdupq_n_s32(value: i32) -> int32x4_t { + int32x4_t(value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdupq_n_s64(value: i64) -> int64x2_t { + int64x2_t(value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdupq_n_u8(value: u8) -> uint8x16_t { + uint8x16_t( + value, value, value, value, value, value, value, value, value, value, value, value, value, + value, value, value, + ) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdupq_n_u16(value: u16) -> uint16x8_t { + uint16x8_t(value, value, value, value, value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdupq_n_u32(value: u32) -> uint32x4_t { + uint32x4_t(value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdupq_n_u64(value: u64) -> uint64x2_t { + uint64x2_t(value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdupq_n_p8(value: p8) -> poly8x16_t { + poly8x16_t( + value, value, value, value, value, value, value, value, value, value, value, value, value, + value, value, value, + ) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdupq_n_p16(value: p16) -> poly16x8_t { + poly16x8_t(value, value, value, value, value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdupq_n_f32(value: f32) -> float32x4_t { + float32x4_t(value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +/// +/// Private vfp4 version used by FMA intriniscs because LLVM does +/// not inline the non-vfp4 version in vfp4 functions. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +unsafe fn vdupq_n_f32_vfp4(value: f32) -> float32x4_t { + float32x4_t(value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdup_n_s8(value: i8) -> int8x8_t { + int8x8_t(value, value, value, value, value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdup_n_s16(value: i16) -> int16x4_t { + int16x4_t(value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdup_n_s32(value: i32) -> int32x2_t { + int32x2_t(value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmov))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdup_n_s64(value: i64) -> int64x1_t { + int64x1_t(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdup_n_u8(value: u8) -> uint8x8_t { + uint8x8_t(value, value, value, value, value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdup_n_u16(value: u16) -> uint16x4_t { + uint16x4_t(value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdup_n_u32(value: u32) -> uint32x2_t { + uint32x2_t(value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmov))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdup_n_u64(value: u64) -> uint64x1_t { + uint64x1_t(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdup_n_p8(value: p8) -> poly8x8_t { + poly8x8_t(value, value, value, value, value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdup_n_p16(value: p16) -> poly16x4_t { + poly16x4_t(value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vdup_n_f32(value: f32) -> float32x2_t { + float32x2_t(value, value) +} + +/// Duplicate vector element to vector or scalar +/// +/// Private vfp4 version used by FMA intriniscs because LLVM does +/// not inline the non-vfp4 version in vfp4 functions. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +unsafe fn vdup_n_f32_vfp4(value: f32) -> float32x2_t { + float32x2_t(value, value) +} + +/// Load SIMD&FP register (immediate offset) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vldrq_p128(a: *const p128) -> p128 { + *a +} + +/// Store SIMD&FP register (immediate offset) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vstrq_p128(a: *mut p128, b: p128) { + *a = b; +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmov_n_s8(value: i8) -> int8x8_t { + vdup_n_s8(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmov_n_s16(value: i16) -> int16x4_t { + vdup_n_s16(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmov_n_s32(value: i32) -> int32x2_t { + vdup_n_s32(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmov))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmov_n_s64(value: i64) -> int64x1_t { + vdup_n_s64(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmov_n_u8(value: u8) -> uint8x8_t { + vdup_n_u8(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmov_n_u16(value: u16) -> uint16x4_t { + vdup_n_u16(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmov_n_u32(value: u32) -> uint32x2_t { + vdup_n_u32(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmov))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmov_n_u64(value: u64) -> uint64x1_t { + vdup_n_u64(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmov_n_p8(value: p8) -> poly8x8_t { + vdup_n_p8(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmov_n_p16(value: p16) -> poly16x4_t { + vdup_n_p16(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmov_n_f32(value: f32) -> float32x2_t { + vdup_n_f32(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovq_n_s8(value: i8) -> int8x16_t { + vdupq_n_s8(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovq_n_s16(value: i16) -> int16x8_t { + vdupq_n_s16(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovq_n_s32(value: i32) -> int32x4_t { + vdupq_n_s32(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovq_n_s64(value: i64) -> int64x2_t { + vdupq_n_s64(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovq_n_u8(value: u8) -> uint8x16_t { + vdupq_n_u8(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovq_n_u16(value: u16) -> uint16x8_t { + vdupq_n_u16(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovq_n_u32(value: u32) -> uint32x4_t { + vdupq_n_u32(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovq_n_u64(value: u64) -> uint64x2_t { + vdupq_n_u64(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovq_n_p8(value: p8) -> poly8x16_t { + vdupq_n_p8(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovq_n_p16(value: p16) -> poly16x8_t { + vdupq_n_p16(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vmovq_n_f32(value: f32) -> float32x4_t { + vdupq_n_f32(value) +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("nop", N = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("nop", N = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vext_s64(a: int64x1_t, _b: int64x1_t) -> int64x1_t { + static_assert!(N : i32 where N == 0); + a +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("nop", N = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("nop", N = 0))] +#[rustc_legacy_const_generics(2)] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vext_u64(a: uint64x1_t, _b: uint64x1_t) -> uint64x1_t { + static_assert!(N : i32 where N == 0); + a +} + +/// Population count per byte. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcnt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cnt))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vcnt_s8(a: int8x8_t) -> int8x8_t { + vcnt_s8_(a) +} +/// Population count per byte. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcnt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cnt))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vcntq_s8(a: int8x16_t) -> int8x16_t { + vcntq_s8_(a) +} +/// Population count per byte. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcnt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cnt))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vcnt_u8(a: uint8x8_t) -> uint8x8_t { + transmute(vcnt_s8_(transmute(a))) +} +/// Population count per byte. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcnt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cnt))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vcntq_u8(a: uint8x16_t) -> uint8x16_t { + transmute(vcntq_s8_(transmute(a))) +} +/// Population count per byte. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcnt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cnt))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vcnt_p8(a: poly8x8_t) -> poly8x8_t { + transmute(vcnt_s8_(transmute(a))) +} +/// Population count per byte. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcnt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cnt))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vcntq_p8(a: poly8x16_t) -> poly8x16_t { + transmute(vcntq_s8_(transmute(a))) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev16))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev16_s8(a: int8x8_t) -> int8x8_t { + simd_shuffle8!(a, a, [1, 0, 3, 2, 5, 4, 7, 6]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev16))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev16q_s8(a: int8x16_t) -> int8x16_t { + simd_shuffle16!(a, a, [1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev16))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev16_u8(a: uint8x8_t) -> uint8x8_t { + simd_shuffle8!(a, a, [1, 0, 3, 2, 5, 4, 7, 6]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev16))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev16q_u8(a: uint8x16_t) -> uint8x16_t { + simd_shuffle16!(a, a, [1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev16))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev16_p8(a: poly8x8_t) -> poly8x8_t { + simd_shuffle8!(a, a, [1, 0, 3, 2, 5, 4, 7, 6]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev16))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev16q_p8(a: poly8x16_t) -> poly8x16_t { + simd_shuffle16!(a, a, [1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev32_s8(a: int8x8_t) -> int8x8_t { + simd_shuffle8!(a, a, [3, 2, 1, 0, 7, 6, 5, 4]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev32q_s8(a: int8x16_t) -> int8x16_t { + simd_shuffle16!(a, a, [3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev32_u8(a: uint8x8_t) -> uint8x8_t { + simd_shuffle8!(a, a, [3, 2, 1, 0, 7, 6, 5, 4]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev32q_u8(a: uint8x16_t) -> uint8x16_t { + simd_shuffle16!(a, a, [3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev32_s16(a: int16x4_t) -> int16x4_t { + simd_shuffle4!(a, a, [1, 0, 3, 2]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev32q_s16(a: int16x8_t) -> int16x8_t { + simd_shuffle8!(a, a, [1, 0, 3, 2, 5, 4, 7, 6]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev32_p16(a: poly16x4_t) -> poly16x4_t { + simd_shuffle4!(a, a, [1, 0, 3, 2]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev32q_p16(a: poly16x8_t) -> poly16x8_t { + simd_shuffle8!(a, a, [1, 0, 3, 2, 5, 4, 7, 6]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev32_u16(a: uint16x4_t) -> uint16x4_t { + simd_shuffle4!(a, a, [1, 0, 3, 2]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev32q_u16(a: uint16x8_t) -> uint16x8_t { + simd_shuffle8!(a, a, [1, 0, 3, 2, 5, 4, 7, 6]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev32_p8(a: poly8x8_t) -> poly8x8_t { + simd_shuffle8!(a, a, [3, 2, 1, 0, 7, 6, 5, 4]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev32q_p8(a: poly8x16_t) -> poly8x16_t { + simd_shuffle16!(a, a, [3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev64_s8(a: int8x8_t) -> int8x8_t { + simd_shuffle8!(a, a, [7, 6, 5, 4, 3, 2, 1, 0]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev64q_s8(a: int8x16_t) -> int8x16_t { + simd_shuffle16!(a, a, [7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev64_s16(a: int16x4_t) -> int16x4_t { + simd_shuffle4!(a, a, [3, 2, 1, 0]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev64q_s16(a: int16x8_t) -> int16x8_t { + simd_shuffle8!(a, a, [3, 2, 1, 0, 7, 6, 5, 4]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev64_s32(a: int32x2_t) -> int32x2_t { + simd_shuffle2!(a, a, [1, 0]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev64q_s32(a: int32x4_t) -> int32x4_t { + simd_shuffle4!(a, a, [1, 0, 3, 2]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev64_u8(a: uint8x8_t) -> uint8x8_t { + simd_shuffle8!(a, a, [7, 6, 5, 4, 3, 2, 1, 0]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev64q_u8(a: uint8x16_t) -> uint8x16_t { + simd_shuffle16!(a, a, [7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev64_u16(a: uint16x4_t) -> uint16x4_t { + simd_shuffle4!(a, a, [3, 2, 1, 0]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev64q_u16(a: uint16x8_t) -> uint16x8_t { + simd_shuffle8!(a, a, [3, 2, 1, 0, 7, 6, 5, 4]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev64_u32(a: uint32x2_t) -> uint32x2_t { + simd_shuffle2!(a, a, [1, 0]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev64q_u32(a: uint32x4_t) -> uint32x4_t { + simd_shuffle4!(a, a, [1, 0, 3, 2]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev64_f32(a: float32x2_t) -> float32x2_t { + simd_shuffle2!(a, a, [1, 0]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev64q_f32(a: float32x4_t) -> float32x4_t { + simd_shuffle4!(a, a, [1, 0, 3, 2]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev64_p8(a: poly8x8_t) -> poly8x8_t { + simd_shuffle8!(a, a, [7, 6, 5, 4, 3, 2, 1, 0]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev64q_p8(a: poly8x16_t) -> poly8x16_t { + simd_shuffle16!(a, a, [7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev64_p16(a: poly16x4_t) -> poly16x4_t { + simd_shuffle4!(a, a, [3, 2, 1, 0]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vrev64q_p16(a: poly16x8_t) -> poly16x8_t { + simd_shuffle8!(a, a, [3, 2, 1, 0, 7, 6, 5, 4]) +} + +/// Signed Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.s8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sadalp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpadal_s8(a: int16x4_t, b: int8x8_t) -> int16x4_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadal_s8_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddl_s8_(b), a) + } +} + +/// Signed Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.s16))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sadalp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpadal_s16(a: int32x2_t, b: int16x4_t) -> int32x2_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadal_s16_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddl_s16_(b), a) + } +} + +/// Signed Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.s32))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sadalp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpadal_s32(a: int64x1_t, b: int32x2_t) -> int64x1_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadal_s32_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddl_s32_(b), a) + } +} + +/// Signed Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.s8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sadalp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpadalq_s8(a: int16x8_t, b: int8x16_t) -> int16x8_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadalq_s8_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddlq_s8_(b), a) + } +} + +/// Signed Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.s16))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sadalp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpadalq_s16(a: int32x4_t, b: int16x8_t) -> int32x4_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadalq_s16_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddlq_s16_(b), a) + } +} + +/// Signed Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.s32))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sadalp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpadalq_s32(a: int64x2_t, b: int32x4_t) -> int64x2_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadalq_s32_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddlq_s32_(b), a) + } +} + +/// Unsigned Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.u8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uadalp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpadal_u8(a: uint16x4_t, b: uint8x8_t) -> uint16x4_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadal_u8_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddl_u8_(b), a) + } +} + +/// Unsigned Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.u16))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uadalp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpadal_u16(a: uint32x2_t, b: uint16x4_t) -> uint32x2_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadal_u16_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddl_u16_(b), a) + } +} + +/// Unsigned Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.u32))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uadalp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpadal_u32(a: uint64x1_t, b: uint32x2_t) -> uint64x1_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadal_u32_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddl_u32_(b), a) + } +} + +/// Unsigned Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.u8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uadalp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpadalq_u8(a: uint16x8_t, b: uint8x16_t) -> uint16x8_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadalq_u8_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddlq_u8_(b), a) + } +} + +/// Unsigned Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.u16))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uadalp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpadalq_u16(a: uint32x4_t, b: uint16x8_t) -> uint32x4_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadalq_u16_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddlq_u16_(b), a) + } +} + +/// Unsigned Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.u32))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uadalp))] +#[cfg_attr( + target_arch = "aarch64", + stable(feature = "neon_intrinsics", since = "1.59.0") +)] +pub unsafe fn vpadalq_u32(a: uint64x2_t, b: uint32x4_t) -> uint64x2_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadalq_u32_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddlq_u32_(b), a) + } +} + +/// 8-bit integer matrix multiply-accumulate +#[inline] +#[cfg_attr(not(bootstrap), target_feature(enable = "i8mm"))] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smmla))] +pub unsafe fn vmmlaq_s32(a: int32x4_t, b: int8x16_t, c: int8x16_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.smmla.v4i32.v16i8")] + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.neon.smmla.v4i32.v16i8" + )] + fn vmmlaq_s32_(a: int32x4_t, b: int8x16_t, c: int8x16_t) -> int32x4_t; + } + vmmlaq_s32_(a, b, c) +} + +/// 8-bit integer matrix multiply-accumulate +#[inline] +#[cfg_attr(not(bootstrap), target_feature(enable = "i8mm"))] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ummla))] +pub unsafe fn vmmlaq_u32(a: uint32x4_t, b: uint8x16_t, c: uint8x16_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.ummla.v4i32.v16i8")] + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.neon.ummla.v4i32.v16i8" + )] + fn vmmlaq_u32_(a: uint32x4_t, b: uint8x16_t, c: uint8x16_t) -> uint32x4_t; + } + vmmlaq_u32_(a, b, c) +} + +/// Unsigned and signed 8-bit integer matrix multiply-accumulate +#[inline] +#[cfg_attr(not(bootstrap), target_feature(enable = "i8mm"))] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usmmla))] +pub unsafe fn vusmmlaq_s32(a: int32x4_t, b: uint8x16_t, c: int8x16_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "unadjusted" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.usmmla.v4i32.v16i8")] + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.neon.usmmla.v4i32.v16i8" + )] + fn vusmmlaq_s32_(a: int32x4_t, b: uint8x16_t, c: int8x16_t) -> int32x4_t; + } + vusmmlaq_s32_(a, b, c) +} + +#[cfg(test)] +mod tests { + use super::*; + #[cfg(target_arch = "aarch64")] + use crate::core_arch::aarch64::*; + #[cfg(target_arch = "arm")] + use crate::core_arch::arm::*; + use crate::core_arch::arm_shared::test_support::*; + use crate::core_arch::simd::*; + use std::{i16, i32, i8, mem::transmute, u16, u32, u8, vec::Vec}; + use stdarch_test::simd_test; + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_s8() { + let a = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let elem: i8 = 42; + let e = i8x8::new(0, 1, 2, 3, 4, 5, 6, 42); + let r: i8x8 = transmute(vld1_lane_s8::<7>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_s8() { + let a = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let elem: i8 = 42; + let e = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 42); + let r: i8x16 = transmute(vld1q_lane_s8::<15>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_s16() { + let a = i16x4::new(0, 1, 2, 3); + let elem: i16 = 42; + let e = i16x4::new(0, 1, 2, 42); + let r: i16x4 = transmute(vld1_lane_s16::<3>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_s16() { + let a = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let elem: i16 = 42; + let e = i16x8::new(0, 1, 2, 3, 4, 5, 6, 42); + let r: i16x8 = transmute(vld1q_lane_s16::<7>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_s32() { + let a = i32x2::new(0, 1); + let elem: i32 = 42; + let e = i32x2::new(0, 42); + let r: i32x2 = transmute(vld1_lane_s32::<1>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_s32() { + let a = i32x4::new(0, 1, 2, 3); + let elem: i32 = 42; + let e = i32x4::new(0, 1, 2, 42); + let r: i32x4 = transmute(vld1q_lane_s32::<3>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_s64() { + let a = i64x1::new(0); + let elem: i64 = 42; + let e = i64x1::new(42); + let r: i64x1 = transmute(vld1_lane_s64::<0>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_s64() { + let a = i64x2::new(0, 1); + let elem: i64 = 42; + let e = i64x2::new(0, 42); + let r: i64x2 = transmute(vld1q_lane_s64::<1>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_u8() { + let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let elem: u8 = 42; + let e = u8x8::new(0, 1, 2, 3, 4, 5, 6, 42); + let r: u8x8 = transmute(vld1_lane_u8::<7>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_u8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let elem: u8 = 42; + let e = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 42); + let r: u8x16 = transmute(vld1q_lane_u8::<15>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_u16() { + let a = u16x4::new(0, 1, 2, 3); + let elem: u16 = 42; + let e = u16x4::new(0, 1, 2, 42); + let r: u16x4 = transmute(vld1_lane_u16::<3>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_u16() { + let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let elem: u16 = 42; + let e = u16x8::new(0, 1, 2, 3, 4, 5, 6, 42); + let r: u16x8 = transmute(vld1q_lane_u16::<7>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_u32() { + let a = u32x2::new(0, 1); + let elem: u32 = 42; + let e = u32x2::new(0, 42); + let r: u32x2 = transmute(vld1_lane_u32::<1>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_u32() { + let a = u32x4::new(0, 1, 2, 3); + let elem: u32 = 42; + let e = u32x4::new(0, 1, 2, 42); + let r: u32x4 = transmute(vld1q_lane_u32::<3>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_u64() { + let a = u64x1::new(0); + let elem: u64 = 42; + let e = u64x1::new(42); + let r: u64x1 = transmute(vld1_lane_u64::<0>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_u64() { + let a = u64x2::new(0, 1); + let elem: u64 = 42; + let e = u64x2::new(0, 42); + let r: u64x2 = transmute(vld1q_lane_u64::<1>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_p8() { + let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let elem: p8 = 42; + let e = u8x8::new(0, 1, 2, 3, 4, 5, 6, 42); + let r: u8x8 = transmute(vld1_lane_p8::<7>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_p8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let elem: p8 = 42; + let e = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 42); + let r: u8x16 = transmute(vld1q_lane_p8::<15>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_p16() { + let a = u16x4::new(0, 1, 2, 3); + let elem: p16 = 42; + let e = u16x4::new(0, 1, 2, 42); + let r: u16x4 = transmute(vld1_lane_p16::<3>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_p16() { + let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let elem: p16 = 42; + let e = u16x8::new(0, 1, 2, 3, 4, 5, 6, 42); + let r: u16x8 = transmute(vld1q_lane_p16::<7>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon,aes")] + unsafe fn test_vld1_lane_p64() { + let a = u64x1::new(0); + let elem: u64 = 42; + let e = u64x1::new(42); + let r: u64x1 = transmute(vld1_lane_p64::<0>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon,aes")] + unsafe fn test_vld1q_lane_p64() { + let a = u64x2::new(0, 1); + let elem: u64 = 42; + let e = u64x2::new(0, 42); + let r: u64x2 = transmute(vld1q_lane_p64::<1>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_f32() { + let a = f32x2::new(0., 1.); + let elem: f32 = 42.; + let e = f32x2::new(0., 42.); + let r: f32x2 = transmute(vld1_lane_f32::<1>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_f32() { + let a = f32x4::new(0., 1., 2., 3.); + let elem: f32 = 42.; + let e = f32x4::new(0., 1., 2., 42.); + let r: f32x4 = transmute(vld1q_lane_f32::<3>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_s8() { + let elem: i8 = 42; + let e = i8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let r: i8x8 = transmute(vld1_dup_s8(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_s8() { + let elem: i8 = 42; + let e = i8x16::new( + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + ); + let r: i8x16 = transmute(vld1q_dup_s8(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_s16() { + let elem: i16 = 42; + let e = i16x4::new(42, 42, 42, 42); + let r: i16x4 = transmute(vld1_dup_s16(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_s16() { + let elem: i16 = 42; + let e = i16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let r: i16x8 = transmute(vld1q_dup_s16(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_s32() { + let elem: i32 = 42; + let e = i32x2::new(42, 42); + let r: i32x2 = transmute(vld1_dup_s32(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_s32() { + let elem: i32 = 42; + let e = i32x4::new(42, 42, 42, 42); + let r: i32x4 = transmute(vld1q_dup_s32(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_s64() { + let elem: i64 = 42; + let e = i64x1::new(42); + let r: i64x1 = transmute(vld1_dup_s64(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_s64() { + let elem: i64 = 42; + let e = i64x2::new(42, 42); + let r: i64x2 = transmute(vld1q_dup_s64(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_u8() { + let elem: u8 = 42; + let e = u8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let r: u8x8 = transmute(vld1_dup_u8(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_u8() { + let elem: u8 = 42; + let e = u8x16::new( + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + ); + let r: u8x16 = transmute(vld1q_dup_u8(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_u16() { + let elem: u16 = 42; + let e = u16x4::new(42, 42, 42, 42); + let r: u16x4 = transmute(vld1_dup_u16(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_u16() { + let elem: u16 = 42; + let e = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let r: u16x8 = transmute(vld1q_dup_u16(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_u32() { + let elem: u32 = 42; + let e = u32x2::new(42, 42); + let r: u32x2 = transmute(vld1_dup_u32(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_u32() { + let elem: u32 = 42; + let e = u32x4::new(42, 42, 42, 42); + let r: u32x4 = transmute(vld1q_dup_u32(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_u64() { + let elem: u64 = 42; + let e = u64x1::new(42); + let r: u64x1 = transmute(vld1_dup_u64(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_u64() { + let elem: u64 = 42; + let e = u64x2::new(42, 42); + let r: u64x2 = transmute(vld1q_dup_u64(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_p8() { + let elem: p8 = 42; + let e = u8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let r: u8x8 = transmute(vld1_dup_p8(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_p8() { + let elem: p8 = 42; + let e = u8x16::new( + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + ); + let r: u8x16 = transmute(vld1q_dup_p8(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_p16() { + let elem: p16 = 42; + let e = u16x4::new(42, 42, 42, 42); + let r: u16x4 = transmute(vld1_dup_p16(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_p16() { + let elem: p16 = 42; + let e = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let r: u16x8 = transmute(vld1q_dup_p16(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon,aes")] + unsafe fn test_vld1_dup_p64() { + let elem: u64 = 42; + let e = u64x1::new(42); + let r: u64x1 = transmute(vld1_dup_p64(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon,aes")] + unsafe fn test_vld1q_dup_p64() { + let elem: u64 = 42; + let e = u64x2::new(42, 42); + let r: u64x2 = transmute(vld1q_dup_p64(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_f32() { + let elem: f32 = 42.; + let e = f32x2::new(42., 42.); + let r: f32x2 = transmute(vld1_dup_f32(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_f32() { + let elem: f32 = 42.; + let e = f32x4::new(42., 42., 42., 42.); + let r: f32x4 = transmute(vld1q_dup_f32(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_u8() { + let v = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r = vget_lane_u8::<1>(transmute(v)); + assert_eq!(r, 2); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_u32() { + let v = i32x4::new(1, 2, 3, 4); + let r = vgetq_lane_u32::<1>(transmute(v)); + assert_eq!(r, 2); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_s32() { + let v = i32x4::new(1, 2, 3, 4); + let r = vgetq_lane_s32::<1>(transmute(v)); + assert_eq!(r, 2); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_u64() { + let v: u64 = 1; + let r = vget_lane_u64::<0>(transmute(v)); + assert_eq!(r, 1); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_u16() { + let v = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r = vgetq_lane_u16::<1>(transmute(v)); + assert_eq!(r, 2); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_s8() { + let v = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = vget_lane_s8::<2>(transmute(v)); + assert_eq!(r, 2); + let r = vget_lane_s8::<4>(transmute(v)); + assert_eq!(r, 4); + let r = vget_lane_s8::<5>(transmute(v)); + assert_eq!(r, 5); + } + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_p8() { + let v = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = vget_lane_p8::<2>(transmute(v)); + assert_eq!(r, 2); + let r = vget_lane_p8::<3>(transmute(v)); + assert_eq!(r, 3); + let r = vget_lane_p8::<5>(transmute(v)); + assert_eq!(r, 5); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_p16() { + let v = u16x4::new(0, 1, 2, 3); + let r = vget_lane_p16::<2>(transmute(v)); + assert_eq!(r, 2); + let r = vget_lane_p16::<3>(transmute(v)); + assert_eq!(r, 3); + let r = vget_lane_p16::<0>(transmute(v)); + assert_eq!(r, 0); + let r = vget_lane_p16::<1>(transmute(v)); + assert_eq!(r, 1); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_s16() { + let v = i16x4::new(0, 1, 2, 3); + let r = vget_lane_s16::<2>(transmute(v)); + assert_eq!(r, 2); + let r = vget_lane_s16::<3>(transmute(v)); + assert_eq!(r, 3); + let r = vget_lane_s16::<0>(transmute(v)); + assert_eq!(r, 0); + let r = vget_lane_s16::<1>(transmute(v)); + assert_eq!(r, 1); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_u16() { + let v = u16x4::new(0, 1, 2, 3); + let r = vget_lane_u16::<2>(transmute(v)); + assert_eq!(r, 2); + let r = vget_lane_u16::<3>(transmute(v)); + assert_eq!(r, 3); + let r = vget_lane_u16::<0>(transmute(v)); + assert_eq!(r, 0); + let r = vget_lane_u16::<1>(transmute(v)); + assert_eq!(r, 1); + } + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_f32() { + let v = f32x2::new(0.0, 1.0); + let r = vget_lane_f32::<1>(transmute(v)); + assert_eq!(r, 1.0); + let r = vget_lane_f32::<0>(transmute(v)); + assert_eq!(r, 0.0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_s32() { + let v = i32x2::new(0, 1); + let r = vget_lane_s32::<1>(transmute(v)); + assert_eq!(r, 1); + let r = vget_lane_s32::<0>(transmute(v)); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_u32() { + let v = u32x2::new(0, 1); + let r = vget_lane_u32::<1>(transmute(v)); + assert_eq!(r, 1); + let r = vget_lane_u32::<0>(transmute(v)); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_s64() { + let v = i64x1::new(1); + let r = vget_lane_s64::<0>(transmute(v)); + assert_eq!(r, 1); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_p64() { + let v = u64x1::new(1); + let r = vget_lane_p64::<0>(transmute(v)); + assert_eq!(r, 1); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_s8() { + let v = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = vgetq_lane_s8::<7>(transmute(v)); + assert_eq!(r, 7); + let r = vgetq_lane_s8::<13>(transmute(v)); + assert_eq!(r, 13); + let r = vgetq_lane_s8::<3>(transmute(v)); + assert_eq!(r, 3); + let r = vgetq_lane_s8::<0>(transmute(v)); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_p8() { + let v = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = vgetq_lane_p8::<7>(transmute(v)); + assert_eq!(r, 7); + let r = vgetq_lane_p8::<13>(transmute(v)); + assert_eq!(r, 13); + let r = vgetq_lane_p8::<3>(transmute(v)); + assert_eq!(r, 3); + let r = vgetq_lane_p8::<0>(transmute(v)); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_u8() { + let v = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = vgetq_lane_u8::<7>(transmute(v)); + assert_eq!(r, 7); + let r = vgetq_lane_u8::<13>(transmute(v)); + assert_eq!(r, 13); + let r = vgetq_lane_u8::<3>(transmute(v)); + assert_eq!(r, 3); + let r = vgetq_lane_u8::<0>(transmute(v)); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_s16() { + let v = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = vgetq_lane_s16::<3>(transmute(v)); + assert_eq!(r, 3); + let r = vgetq_lane_s16::<6>(transmute(v)); + assert_eq!(r, 6); + let r = vgetq_lane_s16::<0>(transmute(v)); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_p16() { + let v = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = vgetq_lane_p16::<3>(transmute(v)); + assert_eq!(r, 3); + let r = vgetq_lane_p16::<7>(transmute(v)); + assert_eq!(r, 7); + let r = vgetq_lane_p16::<1>(transmute(v)); + assert_eq!(r, 1); + } + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_f32() { + let v = f32x4::new(0.0, 1.0, 2.0, 3.0); + let r = vgetq_lane_f32::<3>(transmute(v)); + assert_eq!(r, 3.0); + let r = vgetq_lane_f32::<0>(transmute(v)); + assert_eq!(r, 0.0); + let r = vgetq_lane_f32::<2>(transmute(v)); + assert_eq!(r, 2.0); + let r = vgetq_lane_f32::<1>(transmute(v)); + assert_eq!(r, 1.0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_s64() { + let v = i64x2::new(0, 1); + let r = vgetq_lane_s64::<1>(transmute(v)); + assert_eq!(r, 1); + let r = vgetq_lane_s64::<0>(transmute(v)); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_p64() { + let v = u64x2::new(0, 1); + let r = vgetq_lane_p64::<1>(transmute(v)); + assert_eq!(r, 1); + let r = vgetq_lane_p64::<0>(transmute(v)); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_s64() { + let a: i64x1 = i64x1::new(0); + let b: i64x1 = i64x1::new(1); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vext_s64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_u64() { + let a: u64x1 = u64x1::new(0); + let b: u64x1 = u64x1::new(1); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vext_u64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_s8() { + let a = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e = i8x8::new(9, 10, 11, 12, 13, 14, 15, 16); + let r: i8x8 = transmute(vget_high_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_s16() { + let a = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e = i16x4::new(5, 6, 7, 8); + let r: i16x4 = transmute(vget_high_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_s32() { + let a = i32x4::new(1, 2, 3, 4); + let e = i32x2::new(3, 4); + let r: i32x2 = transmute(vget_high_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_s64() { + let a = i64x2::new(1, 2); + let e = i64x1::new(2); + let r: i64x1 = transmute(vget_high_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_u8() { + let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e = u8x8::new(9, 10, 11, 12, 13, 14, 15, 16); + let r: u8x8 = transmute(vget_high_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_u16() { + let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e = u16x4::new(5, 6, 7, 8); + let r: u16x4 = transmute(vget_high_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_u32() { + let a = u32x4::new(1, 2, 3, 4); + let e = u32x2::new(3, 4); + let r: u32x2 = transmute(vget_high_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_u64() { + let a = u64x2::new(1, 2); + let e = u64x1::new(2); + let r: u64x1 = transmute(vget_high_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_p8() { + let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e = u8x8::new(9, 10, 11, 12, 13, 14, 15, 16); + let r: u8x8 = transmute(vget_high_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_p16() { + let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e = u16x4::new(5, 6, 7, 8); + let r: u16x4 = transmute(vget_high_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_f32() { + let a = f32x4::new(1.0, 2.0, 3.0, 4.0); + let e = f32x2::new(3.0, 4.0); + let r: f32x2 = transmute(vget_high_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_s8() { + let a = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vget_low_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_s16() { + let a = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e = i16x4::new(1, 2, 3, 4); + let r: i16x4 = transmute(vget_low_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_s32() { + let a = i32x4::new(1, 2, 3, 4); + let e = i32x2::new(1, 2); + let r: i32x2 = transmute(vget_low_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_s64() { + let a = i64x2::new(1, 2); + let e = i64x1::new(1); + let r: i64x1 = transmute(vget_low_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_u8() { + let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vget_low_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_u16() { + let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e = u16x4::new(1, 2, 3, 4); + let r: u16x4 = transmute(vget_low_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_u32() { + let a = u32x4::new(1, 2, 3, 4); + let e = u32x2::new(1, 2); + let r: u32x2 = transmute(vget_low_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_u64() { + let a = u64x2::new(1, 2); + let e = u64x1::new(1); + let r: u64x1 = transmute(vget_low_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_p8() { + let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vget_low_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_p16() { + let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e = u16x4::new(1, 2, 3, 4); + let r: u16x4 = transmute(vget_low_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_f32() { + let a = f32x4::new(1.0, 2.0, 3.0, 4.0); + let e = f32x2::new(1.0, 2.0); + let r: f32x2 = transmute(vget_low_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_s8() { + let v: i8 = 42; + let e = i8x16::new( + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + ); + let r: i8x16 = transmute(vdupq_n_s8(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_s16() { + let v: i16 = 64; + let e = i16x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: i16x8 = transmute(vdupq_n_s16(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_s32() { + let v: i32 = 64; + let e = i32x4::new(64, 64, 64, 64); + let r: i32x4 = transmute(vdupq_n_s32(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_s64() { + let v: i64 = 64; + let e = i64x2::new(64, 64); + let r: i64x2 = transmute(vdupq_n_s64(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_u8() { + let v: u8 = 64; + let e = u8x16::new( + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + ); + let r: u8x16 = transmute(vdupq_n_u8(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_u16() { + let v: u16 = 64; + let e = u16x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: u16x8 = transmute(vdupq_n_u16(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_u32() { + let v: u32 = 64; + let e = u32x4::new(64, 64, 64, 64); + let r: u32x4 = transmute(vdupq_n_u32(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_u64() { + let v: u64 = 64; + let e = u64x2::new(64, 64); + let r: u64x2 = transmute(vdupq_n_u64(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_p8() { + let v: p8 = 64; + let e = u8x16::new( + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + ); + let r: u8x16 = transmute(vdupq_n_p8(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_p16() { + let v: p16 = 64; + let e = u16x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: u16x8 = transmute(vdupq_n_p16(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_f32() { + let v: f32 = 64.0; + let e = f32x4::new(64.0, 64.0, 64.0, 64.0); + let r: f32x4 = transmute(vdupq_n_f32(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_s8() { + let v: i8 = 64; + let e = i8x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: i8x8 = transmute(vdup_n_s8(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_s16() { + let v: i16 = 64; + let e = i16x4::new(64, 64, 64, 64); + let r: i16x4 = transmute(vdup_n_s16(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_s32() { + let v: i32 = 64; + let e = i32x2::new(64, 64); + let r: i32x2 = transmute(vdup_n_s32(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_s64() { + let v: i64 = 64; + let e = i64x1::new(64); + let r: i64x1 = transmute(vdup_n_s64(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_u8() { + let v: u8 = 64; + let e = u8x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: u8x8 = transmute(vdup_n_u8(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_u16() { + let v: u16 = 64; + let e = u16x4::new(64, 64, 64, 64); + let r: u16x4 = transmute(vdup_n_u16(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_u32() { + let v: u32 = 64; + let e = u32x2::new(64, 64); + let r: u32x2 = transmute(vdup_n_u32(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_u64() { + let v: u64 = 64; + let e = u64x1::new(64); + let r: u64x1 = transmute(vdup_n_u64(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_p8() { + let v: p8 = 64; + let e = u8x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: u8x8 = transmute(vdup_n_p8(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_p16() { + let v: p16 = 64; + let e = u16x4::new(64, 64, 64, 64); + let r: u16x4 = transmute(vdup_n_p16(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_f32() { + let v: f32 = 64.0; + let e = f32x2::new(64.0, 64.0); + let r: f32x2 = transmute(vdup_n_f32(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vldrq_p128() { + let v: [p128; 2] = [1, 2]; + let e: p128 = 2; + let r: p128 = vldrq_p128(v[1..].as_ptr()); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vstrq_p128() { + let v: [p128; 2] = [1, 2]; + let e: p128 = 2; + let mut r: p128 = 1; + vstrq_p128(&mut r, v[1]); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_s8() { + let v: i8 = 64; + let e = i8x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: i8x8 = transmute(vmov_n_s8(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_s16() { + let v: i16 = 64; + let e = i16x4::new(64, 64, 64, 64); + let r: i16x4 = transmute(vmov_n_s16(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_s32() { + let v: i32 = 64; + let e = i32x2::new(64, 64); + let r: i32x2 = transmute(vmov_n_s32(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_s64() { + let v: i64 = 64; + let e = i64x1::new(64); + let r: i64x1 = transmute(vmov_n_s64(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_u8() { + let v: u8 = 64; + let e = u8x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: u8x8 = transmute(vmov_n_u8(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_u16() { + let v: u16 = 64; + let e = u16x4::new(64, 64, 64, 64); + let r: u16x4 = transmute(vmov_n_u16(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_u32() { + let v: u32 = 64; + let e = u32x2::new(64, 64); + let r: u32x2 = transmute(vmov_n_u32(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_u64() { + let v: u64 = 64; + let e = u64x1::new(64); + let r: u64x1 = transmute(vmov_n_u64(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_p8() { + let v: p8 = 64; + let e = u8x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: u8x8 = transmute(vmov_n_p8(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_p16() { + let v: p16 = 64; + let e = u16x4::new(64, 64, 64, 64); + let r: u16x4 = transmute(vmov_n_p16(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_f32() { + let v: f32 = 64.0; + let e = f32x2::new(64.0, 64.0); + let r: f32x2 = transmute(vmov_n_f32(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovq_n_s8() { + let v: i8 = 64; + let e = i8x16::new( + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + ); + let r: i8x16 = transmute(vmovq_n_s8(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovq_n_s16() { + let v: i16 = 64; + let e = i16x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: i16x8 = transmute(vmovq_n_s16(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovq_n_s32() { + let v: i32 = 64; + let e = i32x4::new(64, 64, 64, 64); + let r: i32x4 = transmute(vmovq_n_s32(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovq_n_s64() { + let v: i64 = 64; + let e = i64x2::new(64, 64); + let r: i64x2 = transmute(vmovq_n_s64(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovq_n_u8() { + let v: u8 = 64; + let e = u8x16::new( + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + ); + let r: u8x16 = transmute(vmovq_n_u8(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovq_n_u16() { + let v: u16 = 64; + let e = u16x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: u16x8 = transmute(vmovq_n_u16(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovq_n_u32() { + let v: u32 = 64; + let e = u32x4::new(64, 64, 64, 64); + let r: u32x4 = transmute(vmovq_n_u32(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovq_n_u64() { + let v: u64 = 64; + let e = u64x2::new(64, 64); + let r: u64x2 = transmute(vmovq_n_u64(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovq_n_p8() { + let v: p8 = 64; + let e = u8x16::new( + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + ); + let r: u8x16 = transmute(vmovq_n_p8(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovq_n_p16() { + let v: p16 = 64; + let e = u16x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: u16x8 = transmute(vmovq_n_p16(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovq_n_f32() { + let v: f32 = 64.0; + let e = f32x4::new(64.0, 64.0, 64.0, 64.0); + let r: f32x4 = transmute(vmovq_n_f32(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_u64() { + let v = i64x2::new(1, 2); + let r = vgetq_lane_u64::<1>(transmute(v)); + assert_eq!(r, 2); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vadd_s8() { + test_ari_s8( + |i, j| vadd_s8(i, j), + |a: i8, b: i8| -> i8 { a.overflowing_add(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddq_s8() { + testq_ari_s8( + |i, j| vaddq_s8(i, j), + |a: i8, b: i8| -> i8 { a.overflowing_add(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vadd_s16() { + test_ari_s16( + |i, j| vadd_s16(i, j), + |a: i16, b: i16| -> i16 { a.overflowing_add(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddq_s16() { + testq_ari_s16( + |i, j| vaddq_s16(i, j), + |a: i16, b: i16| -> i16 { a.overflowing_add(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vadd_s32() { + test_ari_s32( + |i, j| vadd_s32(i, j), + |a: i32, b: i32| -> i32 { a.overflowing_add(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddq_s32() { + testq_ari_s32( + |i, j| vaddq_s32(i, j), + |a: i32, b: i32| -> i32 { a.overflowing_add(b).0 }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vadd_u8() { + test_ari_u8( + |i, j| vadd_u8(i, j), + |a: u8, b: u8| -> u8 { a.overflowing_add(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddq_u8() { + testq_ari_u8( + |i, j| vaddq_u8(i, j), + |a: u8, b: u8| -> u8 { a.overflowing_add(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vadd_u16() { + test_ari_u16( + |i, j| vadd_u16(i, j), + |a: u16, b: u16| -> u16 { a.overflowing_add(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddq_u16() { + testq_ari_u16( + |i, j| vaddq_u16(i, j), + |a: u16, b: u16| -> u16 { a.overflowing_add(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vadd_u32() { + test_ari_u32( + |i, j| vadd_u32(i, j), + |a: u32, b: u32| -> u32 { a.overflowing_add(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddq_u32() { + testq_ari_u32( + |i, j| vaddq_u32(i, j), + |a: u32, b: u32| -> u32 { a.overflowing_add(b).0 }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vadd_f32() { + test_ari_f32(|i, j| vadd_f32(i, j), |a: f32, b: f32| -> f32 { a + b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddq_f32() { + testq_ari_f32(|i, j| vaddq_f32(i, j), |a: f32, b: f32| -> f32 { a + b }); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddl_s8() { + let v = i8::MAX; + let a = i8x8::new(v, v, v, v, v, v, v, v); + let v = 2 * (v as i16); + let e = i16x8::new(v, v, v, v, v, v, v, v); + let r: i16x8 = transmute(vaddl_s8(transmute(a), transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddl_s16() { + let v = i16::MAX; + let a = i16x4::new(v, v, v, v); + let v = 2 * (v as i32); + let e = i32x4::new(v, v, v, v); + let r: i32x4 = transmute(vaddl_s16(transmute(a), transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddl_s32() { + let v = i32::MAX; + let a = i32x2::new(v, v); + let v = 2 * (v as i64); + let e = i64x2::new(v, v); + let r: i64x2 = transmute(vaddl_s32(transmute(a), transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddl_u8() { + let v = u8::MAX; + let a = u8x8::new(v, v, v, v, v, v, v, v); + let v = 2 * (v as u16); + let e = u16x8::new(v, v, v, v, v, v, v, v); + let r: u16x8 = transmute(vaddl_u8(transmute(a), transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddl_u16() { + let v = u16::MAX; + let a = u16x4::new(v, v, v, v); + let v = 2 * (v as u32); + let e = u32x4::new(v, v, v, v); + let r: u32x4 = transmute(vaddl_u16(transmute(a), transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddl_u32() { + let v = u32::MAX; + let a = u32x2::new(v, v); + let v = 2 * (v as u64); + let e = u64x2::new(v, v); + let r: u64x2 = transmute(vaddl_u32(transmute(a), transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddl_high_s8() { + let a = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let x = i8::MAX; + let b = i8x16::new(x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x); + let x = x as i16; + let e = i16x8::new(x + 8, x + 9, x + 10, x + 11, x + 12, x + 13, x + 14, x + 15); + let r: i16x8 = transmute(vaddl_high_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddl_high_s16() { + let a = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let x = i16::MAX; + let b = i16x8::new(x, x, x, x, x, x, x, x); + let x = x as i32; + let e = i32x4::new(x + 4, x + 5, x + 6, x + 7); + let r: i32x4 = transmute(vaddl_high_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddl_high_s32() { + let a = i32x4::new(0, 1, 2, 3); + let x = i32::MAX; + let b = i32x4::new(x, x, x, x); + let x = x as i64; + let e = i64x2::new(x + 2, x + 3); + let r: i64x2 = transmute(vaddl_high_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddl_high_u8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let x = u8::MAX; + let b = u8x16::new(x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x); + let x = x as u16; + let e = u16x8::new(x + 8, x + 9, x + 10, x + 11, x + 12, x + 13, x + 14, x + 15); + let r: u16x8 = transmute(vaddl_high_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddl_high_u16() { + let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let x = u16::MAX; + let b = u16x8::new(x, x, x, x, x, x, x, x); + let x = x as u32; + let e = u32x4::new(x + 4, x + 5, x + 6, x + 7); + let r: u32x4 = transmute(vaddl_high_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddl_high_u32() { + let a = u32x4::new(0, 1, 2, 3); + let x = u32::MAX; + let b = u32x4::new(x, x, x, x); + let x = x as u64; + let e = u64x2::new(x + 2, x + 3); + let r: u64x2 = transmute(vaddl_high_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddw_s8() { + let x = i16::MAX; + let a = i16x8::new(x, 1, 2, 3, 4, 5, 6, 7); + let y = i8::MAX; + let b = i8x8::new(y, y, y, y, y, y, y, y); + let y = y as i16; + let e = i16x8::new( + x.wrapping_add(y), + 1 + y, + 2 + y, + 3 + y, + 4 + y, + 5 + y, + 6 + y, + 7 + y, + ); + let r: i16x8 = transmute(vaddw_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddw_s16() { + let x = i32::MAX; + let a = i32x4::new(x, 1, 2, 3); + let y = i16::MAX; + let b = i16x4::new(y, y, y, y); + let y = y as i32; + let e = i32x4::new(x.wrapping_add(y), 1 + y, 2 + y, 3 + y); + let r: i32x4 = transmute(vaddw_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddw_s32() { + let x = i64::MAX; + let a = i64x2::new(x, 1); + let y = i32::MAX; + let b = i32x2::new(y, y); + let y = y as i64; + let e = i64x2::new(x.wrapping_add(y), 1 + y); + let r: i64x2 = transmute(vaddw_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddw_u8() { + let x = u16::MAX; + let a = u16x8::new(x, 1, 2, 3, 4, 5, 6, 7); + let y = u8::MAX; + let b = u8x8::new(y, y, y, y, y, y, y, y); + let y = y as u16; + let e = u16x8::new( + x.wrapping_add(y), + 1 + y, + 2 + y, + 3 + y, + 4 + y, + 5 + y, + 6 + y, + 7 + y, + ); + let r: u16x8 = transmute(vaddw_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddw_u16() { + let x = u32::MAX; + let a = u32x4::new(x, 1, 2, 3); + let y = u16::MAX; + let b = u16x4::new(y, y, y, y); + let y = y as u32; + let e = u32x4::new(x.wrapping_add(y), 1 + y, 2 + y, 3 + y); + let r: u32x4 = transmute(vaddw_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddw_u32() { + let x = u64::MAX; + let a = u64x2::new(x, 1); + let y = u32::MAX; + let b = u32x2::new(y, y); + let y = y as u64; + let e = u64x2::new(x.wrapping_add(y), 1 + y); + let r: u64x2 = transmute(vaddw_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddw_high_s8() { + let x = i16::MAX; + let a = i16x8::new(x, 1, 2, 3, 4, 5, 6, 7); + let y = i8::MAX; + let b = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, y, y, y, y, y, y, y, y); + let y = y as i16; + let e = i16x8::new( + x.wrapping_add(y), + 1 + y, + 2 + y, + 3 + y, + 4 + y, + 5 + y, + 6 + y, + 7 + y, + ); + let r: i16x8 = transmute(vaddw_high_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddw_high_s16() { + let x = i32::MAX; + let a = i32x4::new(x, 1, 2, 3); + let y = i16::MAX; + let b = i16x8::new(0, 0, 0, 0, y, y, y, y); + let y = y as i32; + let e = i32x4::new(x.wrapping_add(y), 1 + y, 2 + y, 3 + y); + let r: i32x4 = transmute(vaddw_high_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddw_high_s32() { + let x = i64::MAX; + let a = i64x2::new(x, 1); + let y = i32::MAX; + let b = i32x4::new(0, 0, y, y); + let y = y as i64; + let e = i64x2::new(x.wrapping_add(y), 1 + y); + let r: i64x2 = transmute(vaddw_high_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddw_high_u8() { + let x = u16::MAX; + let a = u16x8::new(x, 1, 2, 3, 4, 5, 6, 7); + let y = u8::MAX; + let b = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, y, y, y, y, y, y, y, y); + let y = y as u16; + let e = u16x8::new( + x.wrapping_add(y), + 1 + y, + 2 + y, + 3 + y, + 4 + y, + 5 + y, + 6 + y, + 7 + y, + ); + let r: u16x8 = transmute(vaddw_high_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddw_high_u16() { + let x = u32::MAX; + let a = u32x4::new(x, 1, 2, 3); + let y = u16::MAX; + let b = u16x8::new(0, 0, 0, 0, y, y, y, y); + let y = y as u32; + let e = u32x4::new(x.wrapping_add(y), 1 + y, 2 + y, 3 + y); + let r: u32x4 = transmute(vaddw_high_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddw_high_u32() { + let x = u64::MAX; + let a = u64x2::new(x, 1); + let y = u32::MAX; + let b = u32x4::new(0, 0, y, y); + let y = y as u64; + let e = u64x2::new(x.wrapping_add(y), 1 + y); + let r: u64x2 = transmute(vaddw_high_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddhn_s16() { + let a = i16x8::new( + (0 << 8) + 1, + (1 << 8) + 1, + (2 << 8) + 1, + (3 << 8) + 1, + (4 << 8) + 1, + (5 << 8) + 1, + (6 << 8) + 1, + (7 << 8) + 1, + ); + let e = i8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let r: i8x8 = transmute(vaddhn_s16(transmute(a), transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddhn_s32() { + let a = i32x4::new((0 << 16) + 1, (1 << 16) + 1, (2 << 16) + 1, (3 << 16) + 1); + let e = i16x4::new(0, 2, 4, 6); + let r: i16x4 = transmute(vaddhn_s32(transmute(a), transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddhn_s64() { + let a = i64x2::new((0 << 32) + 1, (1 << 32) + 1); + let e = i32x2::new(0, 2); + let r: i32x2 = transmute(vaddhn_s64(transmute(a), transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddhn_u16() { + let a = u16x8::new( + (0 << 8) + 1, + (1 << 8) + 1, + (2 << 8) + 1, + (3 << 8) + 1, + (4 << 8) + 1, + (5 << 8) + 1, + (6 << 8) + 1, + (7 << 8) + 1, + ); + let e = u8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let r: u8x8 = transmute(vaddhn_u16(transmute(a), transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddhn_u32() { + let a = u32x4::new((0 << 16) + 1, (1 << 16) + 1, (2 << 16) + 1, (3 << 16) + 1); + let e = u16x4::new(0, 2, 4, 6); + let r: u16x4 = transmute(vaddhn_u32(transmute(a), transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddhn_u64() { + let a = u64x2::new((0 << 32) + 1, (1 << 32) + 1); + let e = u32x2::new(0, 2); + let r: u32x2 = transmute(vaddhn_u64(transmute(a), transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddhn_high_s16() { + let r = i8x8::splat(42); + let a = i16x8::new( + (0 << 8) + 1, + (1 << 8) + 1, + (2 << 8) + 1, + (3 << 8) + 1, + (4 << 8) + 1, + (5 << 8) + 1, + (6 << 8) + 1, + (7 << 8) + 1, + ); + let e = i8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 0, 2, 4, 6, 8, 10, 12, 14); + let r: i8x16 = transmute(vaddhn_high_s16(transmute(r), transmute(a), transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddhn_high_s32() { + let r = i16x4::splat(42); + let a = i32x4::new((0 << 16) + 1, (1 << 16) + 1, (2 << 16) + 1, (3 << 16) + 1); + let e = i16x8::new(42, 42, 42, 42, 0, 2, 4, 6); + let r: i16x8 = transmute(vaddhn_high_s32(transmute(r), transmute(a), transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddhn_high_s64() { + let r = i32x2::splat(42); + let a = i64x2::new((0 << 32) + 1, (1 << 32) + 1); + let e = i32x4::new(42, 42, 0, 2); + let r: i32x4 = transmute(vaddhn_high_s64(transmute(r), transmute(a), transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddhn_high_u16() { + let r = u8x8::splat(42); + let a = u16x8::new( + (0 << 8) + 1, + (1 << 8) + 1, + (2 << 8) + 1, + (3 << 8) + 1, + (4 << 8) + 1, + (5 << 8) + 1, + (6 << 8) + 1, + (7 << 8) + 1, + ); + let e = u8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 0, 2, 4, 6, 8, 10, 12, 14); + let r: u8x16 = transmute(vaddhn_high_u16(transmute(r), transmute(a), transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddhn_high_u32() { + let r = u16x4::splat(42); + let a = u32x4::new((0 << 16) + 1, (1 << 16) + 1, (2 << 16) + 1, (3 << 16) + 1); + let e = u16x8::new(42, 42, 42, 42, 0, 2, 4, 6); + let r: u16x8 = transmute(vaddhn_high_u32(transmute(r), transmute(a), transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddhn_high_u64() { + let r = u32x2::splat(42); + let a = u64x2::new((0 << 32) + 1, (1 << 32) + 1); + let e = u32x4::new(42, 42, 0, 2); + let r: u32x4 = transmute(vaddhn_high_u64(transmute(r), transmute(a), transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vraddhn_s16() { + let round_constant: i16 = (1 << 8) - 1; + let a = i16x8::new( + 0 << 8, + 1 << 8, + 2 << 8, + 3 << 8, + 4 << 8, + 5 << 8, + 6 << 8, + 7 << 8, + ); + let b = i16x8::new( + 0 << 8, + (1 << 8) + round_constant, + 2 << 8, + (3 << 8) + round_constant, + 4 << 8, + (5 << 8) + round_constant, + 6 << 8, + (7 << 8) + round_constant, + ); + let e = i8x8::new(0, 3, 4, 7, 8, 11, 12, 15); + let r: i8x8 = transmute(vraddhn_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vraddhn_s32() { + let round_constant: i32 = (1 << 16) - 1; + let a = i32x4::new(0 << 16, 1 << 16, 2 << 16, 3 << 16); + let b = i32x4::new( + 0 << 16, + (1 << 16) + round_constant, + 2 << 16, + (3 << 16) + round_constant, + ); + let e = i16x4::new(0, 3, 4, 7); + let r: i16x4 = transmute(vraddhn_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vraddhn_s64() { + let round_constant: i64 = (1 << 32) - 1; + let a = i64x2::new(0 << 32, 1 << 32); + let b = i64x2::new(0 << 32, (1 << 32) + round_constant); + let e = i32x2::new(0, 3); + let r: i32x2 = transmute(vraddhn_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vraddhn_u16() { + let round_constant: u16 = (1 << 8) - 1; + let a = u16x8::new( + 0 << 8, + 1 << 8, + 2 << 8, + 3 << 8, + 4 << 8, + 5 << 8, + 6 << 8, + 7 << 8, + ); + let b = u16x8::new( + 0 << 8, + (1 << 8) + round_constant, + 2 << 8, + (3 << 8) + round_constant, + 4 << 8, + (5 << 8) + round_constant, + 6 << 8, + (7 << 8) + round_constant, + ); + let e = u8x8::new(0, 3, 4, 7, 8, 11, 12, 15); + let r: u8x8 = transmute(vraddhn_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vraddhn_u32() { + let round_constant: u32 = (1 << 16) - 1; + let a = u32x4::new(0 << 16, 1 << 16, 2 << 16, 3 << 16); + let b = u32x4::new( + 0 << 16, + (1 << 16) + round_constant, + 2 << 16, + (3 << 16) + round_constant, + ); + let e = u16x4::new(0, 3, 4, 7); + let r: u16x4 = transmute(vraddhn_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vraddhn_u64() { + let round_constant: u64 = (1 << 32) - 1; + let a = u64x2::new(0 << 32, 1 << 32); + let b = u64x2::new(0 << 32, (1 << 32) + round_constant); + let e = u32x2::new(0, 3); + let r: u32x2 = transmute(vraddhn_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vraddhn_high_s16() { + let r = i8x8::splat(42); + let round_constant: i16 = (1 << 8) - 1; + let a = i16x8::new( + 0 << 8, + 1 << 8, + 2 << 8, + 3 << 8, + 4 << 8, + 5 << 8, + 6 << 8, + 7 << 8, + ); + let b = i16x8::new( + 0 << 8, + (1 << 8) + round_constant, + 2 << 8, + (3 << 8) + round_constant, + 4 << 8, + (5 << 8) + round_constant, + 6 << 8, + (7 << 8) + round_constant, + ); + let e = i8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 0, 3, 4, 7, 8, 11, 12, 15); + let r: i8x16 = transmute(vraddhn_high_s16(transmute(r), transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vraddhn_high_s32() { + let r = i16x4::splat(42); + let round_constant: i32 = (1 << 16) - 1; + let a = i32x4::new(0 << 16, 1 << 16, 2 << 16, 3 << 16); + let b = i32x4::new( + 0 << 16, + (1 << 16) + round_constant, + 2 << 16, + (3 << 16) + round_constant, + ); + let e = i16x8::new(42, 42, 42, 42, 0, 3, 4, 7); + let r: i16x8 = transmute(vraddhn_high_s32(transmute(r), transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vraddhn_high_s64() { + let r = i32x2::splat(42); + let round_constant: i64 = (1 << 32) - 1; + let a = i64x2::new(0 << 32, 1 << 32); + let b = i64x2::new(0 << 32, (1 << 32) + round_constant); + let e = i32x4::new(42, 42, 0, 3); + let r: i32x4 = transmute(vraddhn_high_s64(transmute(r), transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vraddhn_high_u16() { + let r = u8x8::splat(42); + let round_constant: u16 = (1 << 8) - 1; + let a = u16x8::new( + 0 << 8, + 1 << 8, + 2 << 8, + 3 << 8, + 4 << 8, + 5 << 8, + 6 << 8, + 7 << 8, + ); + let b = u16x8::new( + 0 << 8, + (1 << 8) + round_constant, + 2 << 8, + (3 << 8) + round_constant, + 4 << 8, + (5 << 8) + round_constant, + 6 << 8, + (7 << 8) + round_constant, + ); + let e = u8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 0, 3, 4, 7, 8, 11, 12, 15); + let r: u8x16 = transmute(vraddhn_high_u16(transmute(r), transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vraddhn_high_u32() { + let r = u16x4::splat(42); + let round_constant: u32 = (1 << 16) - 1; + let a = u32x4::new(0 << 16, 1 << 16, 2 << 16, 3 << 16); + let b = u32x4::new( + 0 << 16, + (1 << 16) + round_constant, + 2 << 16, + (3 << 16) + round_constant, + ); + let e = u16x8::new(42, 42, 42, 42, 0, 3, 4, 7); + let r: u16x8 = transmute(vraddhn_high_s32(transmute(r), transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vraddhn_high_u64() { + let r = u32x2::splat(42); + let round_constant: u64 = (1 << 32) - 1; + let a = u64x2::new(0 << 32, 1 << 32); + let b = u64x2::new(0 << 32, (1 << 32) + round_constant); + let e = u32x4::new(42, 42, 0, 3); + let r: u32x4 = transmute(vraddhn_high_s64(transmute(r), transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpaddl_s8() { + let a = i8x8::new(-4, -3, -2, -1, 0, 1, 2, 3); + let r: i16x4 = transmute(vpaddl_s8(transmute(a))); + let e = i16x4::new(-7, -3, 1, 5); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpaddl_s16() { + let a = i16x4::new(-2, -1, 0, 1); + let r: i32x2 = transmute(vpaddl_s16(transmute(a))); + let e = i32x2::new(-3, 1); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpaddl_s32() { + let a = i32x2::new(-1, 0); + let r: i64x1 = transmute(vpaddl_s32(transmute(a))); + let e = i64x1::new(-1); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpaddlq_s8() { + let a = i8x16::new(-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vpaddlq_s8(transmute(a))); + let e = i16x8::new(-15, -11, -7, -3, 1, 5, 9, 13); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpaddlq_s16() { + let a = i16x8::new(-4, -3, -2, -1, 0, 1, 2, 3); + let r: i32x4 = transmute(vpaddlq_s16(transmute(a))); + let e = i32x4::new(-7, -3, 1, 5); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpaddlq_s32() { + let a = i32x4::new(-2, -1, 0, 1); + let r: i64x2 = transmute(vpaddlq_s32(transmute(a))); + let e = i64x2::new(-3, 1); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpaddl_u8() { + let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, u8::MAX); + let r: u16x4 = transmute(vpaddl_u8(transmute(a))); + let e = u16x4::new(1, 5, 9, u8::MAX as u16 + 6); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpaddl_u16() { + let a = u16x4::new(0, 1, 2, u16::MAX); + let r: u32x2 = transmute(vpaddl_u16(transmute(a))); + let e = u32x2::new(1, u16::MAX as u32 + 2); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpaddl_u32() { + let a = u32x2::new(1, u32::MAX); + let r: u64x1 = transmute(vpaddl_u32(transmute(a))); + let e = u64x1::new(u32::MAX as u64 + 1); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpaddlq_u8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, u8::MAX); + let r: u16x8 = transmute(vpaddlq_u8(transmute(a))); + let e = u16x8::new(1, 5, 9, 13, 17, 21, 25, u8::MAX as u16 + 14); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpaddlq_u16() { + let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, u16::MAX); + let r: u32x4 = transmute(vpaddlq_u16(transmute(a))); + let e = u32x4::new(1, 5, 9, u16::MAX as u32 + 6); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpaddlq_u32() { + let a = u32x4::new(0, 1, 2, u32::MAX); + let r: u64x2 = transmute(vpaddlq_u32(transmute(a))); + let e = u64x2::new(1, u32::MAX as u64 + 2); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpadal_s8() { + let a = i16x4::new(42, 42, 42, 42); + let b = i8x8::new(-4, -3, -2, -1, 0, 1, 2, 3); + let r: i16x4 = transmute(vpadal_s8(transmute(a), transmute(b))); + let e = i16x4::new(35, 39, 43, 47); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpadal_s16() { + let a = i32x2::new(42, 42); + let b = i16x4::new(-2, -1, 0, 1); + let r: i32x2 = transmute(vpadal_s16(transmute(a), transmute(b))); + let e = i32x2::new(39, 43); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpadal_s32() { + let a = i64x1::new(42); + let b = i32x2::new(-1, 0); + let r: i64x1 = transmute(vpadal_s32(transmute(a), transmute(b))); + let e = i64x1::new(41); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpadalq_s8() { + let a = i16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b = i8x16::new(-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vpadalq_s8(transmute(a), transmute(b))); + let e = i16x8::new(27, 31, 35, 39, 43, 47, 51, 55); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpadalq_s16() { + let a = i32x4::new(42, 42, 42, 42); + let b = i16x8::new(-4, -3, -2, -1, 0, 1, 2, 3); + let r: i32x4 = transmute(vpadalq_s16(transmute(a), transmute(b))); + let e = i32x4::new(35, 39, 43, 47); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpadalq_s32() { + let a = i64x2::new(42, 42); + let b = i32x4::new(-2, -1, 0, 1); + let r: i64x2 = transmute(vpadalq_s32(transmute(a), transmute(b))); + let e = i64x2::new(39, 43); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpadal_u8() { + let a = u16x4::new(42, 42, 42, 42); + let b = u8x8::new(0, 1, 2, 3, 4, 5, 6, u8::MAX); + let r: u16x4 = transmute(vpadal_u8(transmute(a), transmute(b))); + let e = u16x4::new(43, 47, 51, u8::MAX as u16 + 48); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpadal_u16() { + let a = u32x2::new(42, 42); + let b = u16x4::new(0, 1, 2, u16::MAX); + let r: u32x2 = transmute(vpadal_u16(transmute(a), transmute(b))); + let e = u32x2::new(43, u16::MAX as u32 + 44); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpadal_u32() { + let a = u64x1::new(42); + let b = u32x2::new(1, u32::MAX); + let r: u64x1 = transmute(vpadal_u32(transmute(a), transmute(b))); + let e = u64x1::new(u32::MAX as u64 + 43); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpadalq_u8() { + let a = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, u8::MAX); + let r: u16x8 = transmute(vpadalq_u8(transmute(a), transmute(b))); + let e = u16x8::new(43, 47, 51, 55, 59, 63, 67, u8::MAX as u16 + 56); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpadalq_u16() { + let a = u32x4::new(42, 42, 42, 42); + let b = u16x8::new(0, 1, 2, 3, 4, 5, 6, u16::MAX); + let r: u32x4 = transmute(vpadalq_u16(transmute(a), transmute(b))); + let e = u32x4::new(43, 47, 51, u16::MAX as u32 + 48); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpadalq_u32() { + let a = u64x2::new(42, 42); + let b = u32x4::new(0, 1, 2, u32::MAX); + let r: u64x2 = transmute(vpadalq_u32(transmute(a), transmute(b))); + let e = u64x2::new(43, u32::MAX as u64 + 44); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmvn_s8() { + let a = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e = i8x8::new(-1, -2, -3, -4, -5, -6, -7, -8); + let r: i8x8 = transmute(vmvn_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmvnq_s8() { + let a = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e = i8x16::new( + -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, + ); + let r: i8x16 = transmute(vmvnq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmvn_s16() { + let a = i16x4::new(0, 1, 2, 3); + let e = i16x4::new(-1, -2, -3, -4); + let r: i16x4 = transmute(vmvn_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmvnq_s16() { + let a = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e = i16x8::new(-1, -2, -3, -4, -5, -6, -7, -8); + let r: i16x8 = transmute(vmvnq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmvn_s32() { + let a = i32x2::new(0, 1); + let e = i32x2::new(-1, -2); + let r: i32x2 = transmute(vmvn_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmvnq_s32() { + let a = i32x4::new(0, 1, 2, 3); + let e = i32x4::new(-1, -2, -3, -4); + let r: i32x4 = transmute(vmvnq_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmvn_u8() { + let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e = u8x8::new(255, 254, 253, 252, 251, 250, 249, 248); + let r: u8x8 = transmute(vmvn_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmvnq_u8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e = u8x16::new( + 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, + ); + let r: u8x16 = transmute(vmvnq_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmvn_u16() { + let a = u16x4::new(0, 1, 2, 3); + let e = u16x4::new(65_535, 65_534, 65_533, 65_532); + let r: u16x4 = transmute(vmvn_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmvnq_u16() { + let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e = u16x8::new( + 65_535, 65_534, 65_533, 65_532, 65_531, 65_530, 65_529, 65_528, + ); + let r: u16x8 = transmute(vmvnq_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmvn_u32() { + let a = u32x2::new(0, 1); + let e = u32x2::new(4_294_967_295, 4_294_967_294); + let r: u32x2 = transmute(vmvn_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmvnq_u32() { + let a = u32x4::new(0, 1, 2, 3); + let e = u32x4::new(4_294_967_295, 4_294_967_294, 4_294_967_293, 4_294_967_292); + let r: u32x4 = transmute(vmvnq_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmvn_p8() { + let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e = u8x8::new(255, 254, 253, 252, 251, 250, 249, 248); + let r: u8x8 = transmute(vmvn_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmvnq_p8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e = u8x16::new( + 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, + ); + let r: u8x16 = transmute(vmvnq_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vbic_s8() { + let a = i8x8::new(0, -1, -2, -3, -4, -5, -6, -7); + let b = i8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let e = i8x8::new(0, -2, -2, -4, -4, -6, -6, -8); + let r: i8x8 = transmute(vbic_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vbicq_s8() { + let a = i8x16::new( + 0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, + ); + let b = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let e = i8x16::new( + 0, -2, -2, -4, -4, -6, -6, -8, -8, -10, -10, -12, -12, -14, -14, -16, + ); + let r: i8x16 = transmute(vbicq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vbic_s16() { + let a = i16x4::new(0, -1, -2, -3); + let b = i16x4::new(1, 1, 1, 1); + let e = i16x4::new(0, -2, -2, -4); + let r: i16x4 = transmute(vbic_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vbicq_s16() { + let a = i16x8::new(0, -1, -2, -3, -4, -5, -6, -7); + let b = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let e = i16x8::new(0, -2, -2, -4, -4, -6, -6, -8); + let r: i16x8 = transmute(vbicq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vbic_s32() { + let a = i32x2::new(0, -1); + let b = i32x2::new(1, 1); + let e = i32x2::new(0, -2); + let r: i32x2 = transmute(vbic_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vbicq_s32() { + let a = i32x4::new(0, -1, -2, -3); + let b = i32x4::new(1, 1, 1, 1); + let e = i32x4::new(0, -2, -2, -4); + let r: i32x4 = transmute(vbicq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vbic_s64() { + let a = i64x1::new(-1); + let b = i64x1::new(1); + let e = i64x1::new(-2); + let r: i64x1 = transmute(vbic_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vbicq_s64() { + let a = i64x2::new(0, -1); + let b = i64x2::new(1, 1); + let e = i64x2::new(0, -2); + let r: i64x2 = transmute(vbicq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vbic_u8() { + let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b = u8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let e = u8x8::new(0, 0, 2, 2, 4, 4, 6, 6); + let r: u8x8 = transmute(vbic_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vbicq_u8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let e = u8x16::new(0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14); + let r: u8x16 = transmute(vbicq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vbic_u16() { + let a = u16x4::new(0, 1, 2, 3); + let b = u16x4::new(1, 1, 1, 1); + let e = u16x4::new(0, 0, 2, 2); + let r: u16x4 = transmute(vbic_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vbicq_u16() { + let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b = u16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let e = u16x8::new(0, 0, 2, 2, 4, 4, 6, 6); + let r: u16x8 = transmute(vbicq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vbic_u32() { + let a = u32x2::new(0, 1); + let b = u32x2::new(1, 1); + let e = u32x2::new(0, 0); + let r: u32x2 = transmute(vbic_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vbicq_u32() { + let a = u32x4::new(0, 1, 2, 3); + let b = u32x4::new(1, 1, 1, 1); + let e = u32x4::new(0, 0, 2, 2); + let r: u32x4 = transmute(vbicq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vbic_u64() { + let a = u64x1::new(1); + let b = u64x1::new(1); + let e = u64x1::new(0); + let r: u64x1 = transmute(vbic_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vbicq_u64() { + let a = u64x2::new(0, 1); + let b = u64x2::new(1, 1); + let e = u64x2::new(0, 0); + let r: u64x2 = transmute(vbicq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_s8() { + let a = u8x8::new(u8::MAX, 1, u8::MAX, 2, u8::MAX, 0, u8::MAX, 0); + let b = i8x8::new( + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + ); + let c = i8x8::new( + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + ); + let e = i8x8::new( + i8::MAX, + i8::MIN | 1, + i8::MAX, + i8::MIN | 2, + i8::MAX, + i8::MIN, + i8::MAX, + i8::MIN, + ); + let r: i8x8 = transmute(vbsl_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_s16() { + let a = u16x4::new(u16::MAX, 0, 1, 2); + let b = i16x4::new(i16::MAX, i16::MAX, i16::MAX, i16::MAX); + let c = i16x4::new(i16::MIN, i16::MIN, i16::MIN, i16::MIN); + let e = i16x4::new(i16::MAX, i16::MIN, i16::MIN | 1, i16::MIN | 2); + let r: i16x4 = transmute(vbsl_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_s32() { + let a = u32x2::new(u32::MAX, 1); + let b = i32x2::new(i32::MAX, i32::MAX); + let c = i32x2::new(i32::MIN, i32::MIN); + let e = i32x2::new(i32::MAX, i32::MIN | 1); + let r: i32x2 = transmute(vbsl_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_s64() { + let a = u64x1::new(1); + let b = i64x1::new(i64::MAX); + let c = i64x1::new(i64::MIN); + let e = i64x1::new(i64::MIN | 1); + let r: i64x1 = transmute(vbsl_s64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_u8() { + let a = u8x8::new(u8::MAX, 1, u8::MAX, 2, u8::MAX, 0, u8::MAX, 0); + let b = u8x8::new( + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + ); + let c = u8x8::new( + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + ); + let e = u8x8::new(u8::MAX, 1, u8::MAX, 2, u8::MAX, u8::MIN, u8::MAX, u8::MIN); + let r: u8x8 = transmute(vbsl_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_u16() { + let a = u16x4::new(u16::MAX, 0, 1, 2); + let b = u16x4::new(u16::MAX, u16::MAX, u16::MAX, u16::MAX); + let c = u16x4::new(u16::MIN, u16::MIN, u16::MIN, u16::MIN); + let e = u16x4::new(u16::MAX, 0, 1, 2); + let r: u16x4 = transmute(vbsl_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_u32() { + let a = u32x2::new(u32::MAX, 2); + let b = u32x2::new(u32::MAX, u32::MAX); + let c = u32x2::new(u32::MIN, u32::MIN); + let e = u32x2::new(u32::MAX, 2); + let r: u32x2 = transmute(vbsl_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_u64() { + let a = u64x1::new(2); + let b = u64x1::new(u64::MAX); + let c = u64x1::new(u64::MIN); + let e = u64x1::new(2); + let r: u64x1 = transmute(vbsl_u64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_f32() { + let a = u32x2::new(1, 0x80000000); + let b = f32x2::new(8388609f32, -1.23f32); + let c = f32x2::new(2097152f32, 2.34f32); + let e = f32x2::new(2097152.25f32, -2.34f32); + let r: f32x2 = transmute(vbsl_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_p8() { + let a = u8x8::new(u8::MAX, 1, u8::MAX, 2, u8::MAX, 0, u8::MAX, 0); + let b = u8x8::new( + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + ); + let c = u8x8::new( + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + ); + let e = u8x8::new(u8::MAX, 1, u8::MAX, 2, u8::MAX, u8::MIN, u8::MAX, u8::MIN); + let r: u8x8 = transmute(vbsl_p8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_p16() { + let a = u16x4::new(u16::MAX, 0, 1, 2); + let b = u16x4::new(u16::MAX, u16::MAX, u16::MAX, u16::MAX); + let c = u16x4::new(u16::MIN, u16::MIN, u16::MIN, u16::MIN); + let e = u16x4::new(u16::MAX, 0, 1, 2); + let r: u16x4 = transmute(vbsl_p16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbslq_s8() { + let a = u8x16::new( + u8::MAX, + 1, + u8::MAX, + 2, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + ); + let b = i8x16::new( + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + ); + let c = i8x16::new( + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + ); + let e = i8x16::new( + i8::MAX, + i8::MIN | 1, + i8::MAX, + i8::MIN | 2, + i8::MAX, + i8::MIN, + i8::MAX, + i8::MIN, + i8::MAX, + i8::MIN, + i8::MAX, + i8::MIN, + i8::MAX, + i8::MIN, + i8::MAX, + i8::MIN, + ); + let r: i8x16 = transmute(vbslq_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbslq_s16() { + let a = u16x8::new(u16::MAX, 1, u16::MAX, 2, u16::MAX, 0, u16::MAX, 0); + let b = i16x8::new( + i16::MAX, + i16::MAX, + i16::MAX, + i16::MAX, + i16::MAX, + i16::MAX, + i16::MAX, + i16::MAX, + ); + let c = i16x8::new( + i16::MIN, + i16::MIN, + i16::MIN, + i16::MIN, + i16::MIN, + i16::MIN, + i16::MIN, + i16::MIN, + ); + let e = i16x8::new( + i16::MAX, + i16::MIN | 1, + i16::MAX, + i16::MIN | 2, + i16::MAX, + i16::MIN, + i16::MAX, + i16::MIN, + ); + let r: i16x8 = transmute(vbslq_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbslq_s32() { + let a = u32x4::new(u32::MAX, 1, u32::MAX, 2); + let b = i32x4::new(i32::MAX, i32::MAX, i32::MAX, i32::MAX); + let c = i32x4::new(i32::MIN, i32::MIN, i32::MIN, i32::MIN); + let e = i32x4::new(i32::MAX, i32::MIN | 1, i32::MAX, i32::MIN | 2); + let r: i32x4 = transmute(vbslq_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbslq_s64() { + let a = u64x2::new(u64::MAX, 1); + let b = i64x2::new(i64::MAX, i64::MAX); + let c = i64x2::new(i64::MIN, i64::MIN); + let e = i64x2::new(i64::MAX, i64::MIN | 1); + let r: i64x2 = transmute(vbslq_s64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbslq_u8() { + let a = u8x16::new( + u8::MAX, + 1, + u8::MAX, + 2, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + ); + let b = u8x16::new( + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + ); + let c = u8x16::new( + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + ); + let e = u8x16::new( + u8::MAX, + 1, + u8::MAX, + 2, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + ); + let r: u8x16 = transmute(vbslq_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbslq_u16() { + let a = u16x8::new(u16::MAX, 1, u16::MAX, 2, u16::MAX, 0, u16::MAX, 0); + let b = u16x8::new( + u16::MAX, + u16::MAX, + u16::MAX, + u16::MAX, + u16::MAX, + u16::MAX, + u16::MAX, + u16::MAX, + ); + let c = u16x8::new( + u16::MIN, + u16::MIN, + u16::MIN, + u16::MIN, + u16::MIN, + u16::MIN, + u16::MIN, + u16::MIN, + ); + let e = u16x8::new( + u16::MAX, + 1, + u16::MAX, + 2, + u16::MAX, + u16::MIN, + u16::MAX, + u16::MIN, + ); + let r: u16x8 = transmute(vbslq_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbslq_u32() { + let a = u32x4::new(u32::MAX, 1, u32::MAX, 2); + let b = u32x4::new(u32::MAX, u32::MAX, u32::MAX, u32::MAX); + let c = u32x4::new(u32::MIN, u32::MIN, u32::MIN, u32::MIN); + let e = u32x4::new(u32::MAX, 1, u32::MAX, 2); + let r: u32x4 = transmute(vbslq_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbslq_u64() { + let a = u64x2::new(u64::MAX, 1); + let b = u64x2::new(u64::MAX, u64::MAX); + let c = u64x2::new(u64::MIN, u64::MIN); + let e = u64x2::new(u64::MAX, 1); + let r: u64x2 = transmute(vbslq_u64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbslq_f32() { + let a = u32x4::new(u32::MAX, 0, 1, 0x80000000); + let b = f32x4::new(-1.23f32, -1.23f32, 8388609f32, -1.23f32); + let c = f32x4::new(2.34f32, 2.34f32, 2097152f32, 2.34f32); + let e = f32x4::new(-1.23f32, 2.34f32, 2097152.25f32, -2.34f32); + let r: f32x4 = transmute(vbslq_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbslq_p8() { + let a = u8x16::new( + u8::MAX, + 1, + u8::MAX, + 2, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + ); + let b = u8x16::new( + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + ); + let c = u8x16::new( + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + ); + let e = u8x16::new( + u8::MAX, + 1, + u8::MAX, + 2, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + ); + let r: u8x16 = transmute(vbslq_p8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbslq_p16() { + let a = u16x8::new(u16::MAX, 1, u16::MAX, 2, u16::MAX, 0, u16::MAX, 0); + let b = u16x8::new( + u16::MAX, + u16::MAX, + u16::MAX, + u16::MAX, + u16::MAX, + u16::MAX, + u16::MAX, + u16::MAX, + ); + let c = u16x8::new( + u16::MIN, + u16::MIN, + u16::MIN, + u16::MIN, + u16::MIN, + u16::MIN, + u16::MIN, + u16::MIN, + ); + let e = u16x8::new( + u16::MAX, + 1, + u16::MAX, + 2, + u16::MAX, + u16::MIN, + u16::MAX, + u16::MIN, + ); + let r: u16x8 = transmute(vbslq_p16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorn_s8() { + let a = i8x8::new(0, -1, -2, -3, -4, -5, -6, -7); + let b = i8x8::new(-2, -2, -2, -2, -2, -2, -2, -2); + let e = i8x8::new(1, -1, -1, -3, -3, -5, -5, -7); + let r: i8x8 = transmute(vorn_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vornq_s8() { + let a = i8x16::new( + 0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, + ); + let b = i8x16::new( + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + ); + let e = i8x16::new( + 1, -1, -1, -3, -3, -5, -5, -7, -7, -9, -9, -11, -11, -13, -13, -15, + ); + let r: i8x16 = transmute(vornq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorn_s16() { + let a = i16x4::new(0, -1, -2, -3); + let b = i16x4::new(-2, -2, -2, -2); + let e = i16x4::new(1, -1, -1, -3); + let r: i16x4 = transmute(vorn_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vornq_s16() { + let a = i16x8::new(0, -1, -2, -3, -4, -5, -6, -7); + let b = i16x8::new(-2, -2, -2, -2, -2, -2, -2, -2); + let e = i16x8::new(1, -1, -1, -3, -3, -5, -5, -7); + let r: i16x8 = transmute(vornq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorn_s32() { + let a = i32x2::new(0, -1); + let b = i32x2::new(-2, -2); + let e = i32x2::new(1, -1); + let r: i32x2 = transmute(vorn_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vornq_s32() { + let a = i32x4::new(0, -1, -2, -3); + let b = i32x4::new(-2, -2, -2, -2); + let e = i32x4::new(1, -1, -1, -3); + let r: i32x4 = transmute(vornq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorn_s64() { + let a = i64x1::new(0); + let b = i64x1::new(-2); + let e = i64x1::new(1); + let r: i64x1 = transmute(vorn_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vornq_s64() { + let a = i64x2::new(0, -1); + let b = i64x2::new(-2, -2); + let e = i64x2::new(1, -1); + let r: i64x2 = transmute(vornq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorn_u8() { + let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let t = u8::MAX - 1; + let b = u8x8::new(t, t, t, t, t, t, t, t); + let e = u8x8::new(1, 1, 3, 3, 5, 5, 7, 7); + let r: u8x8 = transmute(vorn_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vornq_u8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let t = u8::MAX - 1; + let b = u8x16::new(t, t, t, t, t, t, t, t, t, t, t, t, t, t, t, t); + let e = u8x16::new(1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15); + let r: u8x16 = transmute(vornq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorn_u16() { + let a = u16x4::new(0, 1, 2, 3); + let t = u16::MAX - 1; + let b = u16x4::new(t, t, t, t); + let e = u16x4::new(1, 1, 3, 3); + let r: u16x4 = transmute(vorn_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vornq_u16() { + let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let t = u16::MAX - 1; + let b = u16x8::new(t, t, t, t, t, t, t, t); + let e = u16x8::new(1, 1, 3, 3, 5, 5, 7, 7); + let r: u16x8 = transmute(vornq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorn_u32() { + let a = u32x2::new(0, 1); + let t = u32::MAX - 1; + let b = u32x2::new(t, t); + let e = u32x2::new(1, 1); + let r: u32x2 = transmute(vorn_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vornq_u32() { + let a = u32x4::new(0, 1, 2, 3); + let t = u32::MAX - 1; + let b = u32x4::new(t, t, t, t); + let e = u32x4::new(1, 1, 3, 3); + let r: u32x4 = transmute(vornq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorn_u64() { + let a = u64x1::new(0); + let t = u64::MAX - 1; + let b = u64x1::new(t); + let e = u64x1::new(1); + let r: u64x1 = transmute(vorn_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vornq_u64() { + let a = u64x2::new(0, 1); + let t = u64::MAX - 1; + let b = u64x2::new(t, t); + let e = u64x2::new(1, 1); + let r: u64x2 = transmute(vornq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovn_s16() { + let a = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vmovn_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovn_s32() { + let a = i32x4::new(1, 2, 3, 4); + let e = i16x4::new(1, 2, 3, 4); + let r: i16x4 = transmute(vmovn_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovn_s64() { + let a = i64x2::new(1, 2); + let e = i32x2::new(1, 2); + let r: i32x2 = transmute(vmovn_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovn_u16() { + let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vmovn_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovn_u32() { + let a = u32x4::new(1, 2, 3, 4); + let e = u16x4::new(1, 2, 3, 4); + let r: u16x4 = transmute(vmovn_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovn_u64() { + let a = u64x2::new(1, 2); + let e = u32x2::new(1, 2); + let r: u32x2 = transmute(vmovn_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovl_s8() { + let e = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let a = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i16x8 = transmute(vmovl_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovl_s16() { + let e = i32x4::new(1, 2, 3, 4); + let a = i16x4::new(1, 2, 3, 4); + let r: i32x4 = transmute(vmovl_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovl_s32() { + let e = i64x2::new(1, 2); + let a = i32x2::new(1, 2); + let r: i64x2 = transmute(vmovl_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovl_u8() { + let e = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let a = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u16x8 = transmute(vmovl_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovl_u16() { + let e = u32x4::new(1, 2, 3, 4); + let a = u16x4::new(1, 2, 3, 4); + let r: u32x4 = transmute(vmovl_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovl_u32() { + let e = u64x2::new(1, 2); + let a = u32x2::new(1, 2); + let r: u64x2 = transmute(vmovl_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmin_s8() { + let a = i8x8::new(1, -2, 3, -4, 5, 6, 7, 8); + let b = i8x8::new(0, 3, 2, 5, 4, 7, 6, 9); + let e = i8x8::new(-2, -4, 5, 7, 0, 2, 4, 6); + let r: i8x8 = transmute(vpmin_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmin_s16() { + let a = i16x4::new(1, 2, 3, -4); + let b = i16x4::new(0, 3, 2, 5); + let e = i16x4::new(1, -4, 0, 2); + let r: i16x4 = transmute(vpmin_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmin_s32() { + let a = i32x2::new(1, -2); + let b = i32x2::new(0, 3); + let e = i32x2::new(-2, 0); + let r: i32x2 = transmute(vpmin_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmin_u8() { + let a = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b = u8x8::new(0, 3, 2, 5, 4, 7, 6, 9); + let e = u8x8::new(1, 3, 5, 7, 0, 2, 4, 6); + let r: u8x8 = transmute(vpmin_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmin_u16() { + let a = u16x4::new(1, 2, 3, 4); + let b = u16x4::new(0, 3, 2, 5); + let e = u16x4::new(1, 3, 0, 2); + let r: u16x4 = transmute(vpmin_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmin_u32() { + let a = u32x2::new(1, 2); + let b = u32x2::new(0, 3); + let e = u32x2::new(1, 0); + let r: u32x2 = transmute(vpmin_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmin_f32() { + let a = f32x2::new(1., -2.); + let b = f32x2::new(0., 3.); + let e = f32x2::new(-2., 0.); + let r: f32x2 = transmute(vpmin_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmax_s8() { + let a = i8x8::new(1, -2, 3, -4, 5, 6, 7, 8); + let b = i8x8::new(0, 3, 2, 5, 4, 7, 6, 9); + let e = i8x8::new(1, 3, 6, 8, 3, 5, 7, 9); + let r: i8x8 = transmute(vpmax_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmax_s16() { + let a = i16x4::new(1, 2, 3, -4); + let b = i16x4::new(0, 3, 2, 5); + let e = i16x4::new(2, 3, 3, 5); + let r: i16x4 = transmute(vpmax_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmax_s32() { + let a = i32x2::new(1, -2); + let b = i32x2::new(0, 3); + let e = i32x2::new(1, 3); + let r: i32x2 = transmute(vpmax_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmax_u8() { + let a = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b = u8x8::new(0, 3, 2, 5, 4, 7, 6, 9); + let e = u8x8::new(2, 4, 6, 8, 3, 5, 7, 9); + let r: u8x8 = transmute(vpmax_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmax_u16() { + let a = u16x4::new(1, 2, 3, 4); + let b = u16x4::new(0, 3, 2, 5); + let e = u16x4::new(2, 4, 3, 5); + let r: u16x4 = transmute(vpmax_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmax_u32() { + let a = u32x2::new(1, 2); + let b = u32x2::new(0, 3); + let e = u32x2::new(2, 3); + let r: u32x2 = transmute(vpmax_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmax_f32() { + let a = f32x2::new(1., -2.); + let b = f32x2::new(0., 3.); + let e = f32x2::new(1., 3.); + let r: f32x2 = transmute(vpmax_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vand_s8() { + test_bit_s8(|i, j| vand_s8(i, j), |a: i8, b: i8| -> i8 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vandq_s8() { + testq_bit_s8(|i, j| vandq_s8(i, j), |a: i8, b: i8| -> i8 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vand_s16() { + test_bit_s16(|i, j| vand_s16(i, j), |a: i16, b: i16| -> i16 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vandq_s16() { + testq_bit_s16(|i, j| vandq_s16(i, j), |a: i16, b: i16| -> i16 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vand_s32() { + test_bit_s32(|i, j| vand_s32(i, j), |a: i32, b: i32| -> i32 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vandq_s32() { + testq_bit_s32(|i, j| vandq_s32(i, j), |a: i32, b: i32| -> i32 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vand_s64() { + test_bit_s64(|i, j| vand_s64(i, j), |a: i64, b: i64| -> i64 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vandq_s64() { + testq_bit_s64(|i, j| vandq_s64(i, j), |a: i64, b: i64| -> i64 { a & b }); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vand_u8() { + test_bit_u8(|i, j| vand_u8(i, j), |a: u8, b: u8| -> u8 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vandq_u8() { + testq_bit_u8(|i, j| vandq_u8(i, j), |a: u8, b: u8| -> u8 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vand_u16() { + test_bit_u16(|i, j| vand_u16(i, j), |a: u16, b: u16| -> u16 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vandq_u16() { + testq_bit_u16(|i, j| vandq_u16(i, j), |a: u16, b: u16| -> u16 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vand_u32() { + test_bit_u32(|i, j| vand_u32(i, j), |a: u32, b: u32| -> u32 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vandq_u32() { + testq_bit_u32(|i, j| vandq_u32(i, j), |a: u32, b: u32| -> u32 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vand_u64() { + test_bit_u64(|i, j| vand_u64(i, j), |a: u64, b: u64| -> u64 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vandq_u64() { + testq_bit_u64(|i, j| vandq_u64(i, j), |a: u64, b: u64| -> u64 { a & b }); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorr_s8() { + test_bit_s8(|i, j| vorr_s8(i, j), |a: i8, b: i8| -> i8 { a | b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_s8() { + testq_bit_s8(|i, j| vorrq_s8(i, j), |a: i8, b: i8| -> i8 { a | b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vorr_s16() { + test_bit_s16(|i, j| vorr_s16(i, j), |a: i16, b: i16| -> i16 { a | b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_s16() { + testq_bit_s16(|i, j| vorrq_s16(i, j), |a: i16, b: i16| -> i16 { a | b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vorr_s32() { + test_bit_s32(|i, j| vorr_s32(i, j), |a: i32, b: i32| -> i32 { a | b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_s32() { + testq_bit_s32(|i, j| vorrq_s32(i, j), |a: i32, b: i32| -> i32 { a | b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vorr_s64() { + test_bit_s64(|i, j| vorr_s64(i, j), |a: i64, b: i64| -> i64 { a | b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_s64() { + testq_bit_s64(|i, j| vorrq_s64(i, j), |a: i64, b: i64| -> i64 { a | b }); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorr_u8() { + test_bit_u8(|i, j| vorr_u8(i, j), |a: u8, b: u8| -> u8 { a | b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_u8() { + testq_bit_u8(|i, j| vorrq_u8(i, j), |a: u8, b: u8| -> u8 { a | b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vorr_u16() { + test_bit_u16(|i, j| vorr_u16(i, j), |a: u16, b: u16| -> u16 { a | b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_u16() { + testq_bit_u16(|i, j| vorrq_u16(i, j), |a: u16, b: u16| -> u16 { a | b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vorr_u32() { + test_bit_u32(|i, j| vorr_u32(i, j), |a: u32, b: u32| -> u32 { a | b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_u32() { + testq_bit_u32(|i, j| vorrq_u32(i, j), |a: u32, b: u32| -> u32 { a | b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vorr_u64() { + test_bit_u64(|i, j| vorr_u64(i, j), |a: u64, b: u64| -> u64 { a | b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_u64() { + testq_bit_u64(|i, j| vorrq_u64(i, j), |a: u64, b: u64| -> u64 { a | b }); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veor_s8() { + test_bit_s8(|i, j| veor_s8(i, j), |a: i8, b: i8| -> i8 { a ^ b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_veorq_s8() { + testq_bit_s8(|i, j| veorq_s8(i, j), |a: i8, b: i8| -> i8 { a ^ b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_veor_s16() { + test_bit_s16(|i, j| veor_s16(i, j), |a: i16, b: i16| -> i16 { a ^ b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_veorq_s16() { + testq_bit_s16(|i, j| veorq_s16(i, j), |a: i16, b: i16| -> i16 { a ^ b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_veor_s32() { + test_bit_s32(|i, j| veor_s32(i, j), |a: i32, b: i32| -> i32 { a ^ b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_veorq_s32() { + testq_bit_s32(|i, j| veorq_s32(i, j), |a: i32, b: i32| -> i32 { a ^ b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_veor_s64() { + test_bit_s64(|i, j| veor_s64(i, j), |a: i64, b: i64| -> i64 { a ^ b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_veorq_s64() { + testq_bit_s64(|i, j| veorq_s64(i, j), |a: i64, b: i64| -> i64 { a ^ b }); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veor_u8() { + test_bit_u8(|i, j| veor_u8(i, j), |a: u8, b: u8| -> u8 { a ^ b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_veorq_u8() { + testq_bit_u8(|i, j| veorq_u8(i, j), |a: u8, b: u8| -> u8 { a ^ b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_veor_u16() { + test_bit_u16(|i, j| veor_u16(i, j), |a: u16, b: u16| -> u16 { a ^ b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_veorq_u16() { + testq_bit_u16(|i, j| veorq_u16(i, j), |a: u16, b: u16| -> u16 { a ^ b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_veor_u32() { + test_bit_u32(|i, j| veor_u32(i, j), |a: u32, b: u32| -> u32 { a ^ b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_veorq_u32() { + testq_bit_u32(|i, j| veorq_u32(i, j), |a: u32, b: u32| -> u32 { a ^ b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_veor_u64() { + test_bit_u64(|i, j| veor_u64(i, j), |a: u64, b: u64| -> u64 { a ^ b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_veorq_u64() { + testq_bit_u64(|i, j| veorq_u64(i, j), |a: u64, b: u64| -> u64 { a ^ b }); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_s8() { + test_cmp_s8( + |i, j| vceq_s8(i, j), + |a: i8, b: i8| -> u8 { + if a == b { + 0xFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_s8() { + testq_cmp_s8( + |i, j| vceqq_s8(i, j), + |a: i8, b: i8| -> u8 { + if a == b { + 0xFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vceq_s16() { + test_cmp_s16( + |i, j| vceq_s16(i, j), + |a: i16, b: i16| -> u16 { + if a == b { + 0xFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_s16() { + testq_cmp_s16( + |i, j| vceqq_s16(i, j), + |a: i16, b: i16| -> u16 { + if a == b { + 0xFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vceq_s32() { + test_cmp_s32( + |i, j| vceq_s32(i, j), + |a: i32, b: i32| -> u32 { + if a == b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_s32() { + testq_cmp_s32( + |i, j| vceqq_s32(i, j), + |a: i32, b: i32| -> u32 { + if a == b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_u8() { + test_cmp_u8( + |i, j| vceq_u8(i, j), + |a: u8, b: u8| -> u8 { + if a == b { + 0xFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_u8() { + testq_cmp_u8( + |i, j| vceqq_u8(i, j), + |a: u8, b: u8| -> u8 { + if a == b { + 0xFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vceq_u16() { + test_cmp_u16( + |i, j| vceq_u16(i, j), + |a: u16, b: u16| -> u16 { + if a == b { + 0xFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_u16() { + testq_cmp_u16( + |i, j| vceqq_u16(i, j), + |a: u16, b: u16| -> u16 { + if a == b { + 0xFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vceq_u32() { + test_cmp_u32( + |i, j| vceq_u32(i, j), + |a: u32, b: u32| -> u32 { + if a == b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_u32() { + testq_cmp_u32( + |i, j| vceqq_u32(i, j), + |a: u32, b: u32| -> u32 { + if a == b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_f32() { + test_cmp_f32( + |i, j| vcge_f32(i, j), + |a: f32, b: f32| -> u32 { + if a == b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_f32() { + testq_cmp_f32( + |i, j| vcgeq_f32(i, j), + |a: f32, b: f32| -> u32 { + if a == b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_s8() { + test_cmp_s8( + |i, j| vcgt_s8(i, j), + |a: i8, b: i8| -> u8 { + if a > b { + 0xFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_s8() { + testq_cmp_s8( + |i, j| vcgtq_s8(i, j), + |a: i8, b: i8| -> u8 { + if a > b { + 0xFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_s16() { + test_cmp_s16( + |i, j| vcgt_s16(i, j), + |a: i16, b: i16| -> u16 { + if a > b { + 0xFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_s16() { + testq_cmp_s16( + |i, j| vcgtq_s16(i, j), + |a: i16, b: i16| -> u16 { + if a > b { + 0xFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_s32() { + test_cmp_s32( + |i, j| vcgt_s32(i, j), + |a: i32, b: i32| -> u32 { + if a > b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_s32() { + testq_cmp_s32( + |i, j| vcgtq_s32(i, j), + |a: i32, b: i32| -> u32 { + if a > b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_u8() { + test_cmp_u8( + |i, j| vcgt_u8(i, j), + |a: u8, b: u8| -> u8 { + if a > b { + 0xFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_u8() { + testq_cmp_u8( + |i, j| vcgtq_u8(i, j), + |a: u8, b: u8| -> u8 { + if a > b { + 0xFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_u16() { + test_cmp_u16( + |i, j| vcgt_u16(i, j), + |a: u16, b: u16| -> u16 { + if a > b { + 0xFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_u16() { + testq_cmp_u16( + |i, j| vcgtq_u16(i, j), + |a: u16, b: u16| -> u16 { + if a > b { + 0xFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_u32() { + test_cmp_u32( + |i, j| vcgt_u32(i, j), + |a: u32, b: u32| -> u32 { + if a > b { + 0xFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_u32() { + testq_cmp_u32( + |i, j| vcgtq_u32(i, j), + |a: u32, b: u32| -> u32 { + if a > b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_f32() { + test_cmp_f32( + |i, j| vcgt_f32(i, j), + |a: f32, b: f32| -> u32 { + if a > b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_f32() { + testq_cmp_f32( + |i, j| vcgtq_f32(i, j), + |a: f32, b: f32| -> u32 { + if a > b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_s8() { + test_cmp_s8( + |i, j| vclt_s8(i, j), + |a: i8, b: i8| -> u8 { + if a < b { + 0xFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_s8() { + testq_cmp_s8( + |i, j| vcltq_s8(i, j), + |a: i8, b: i8| -> u8 { + if a < b { + 0xFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vclt_s16() { + test_cmp_s16( + |i, j| vclt_s16(i, j), + |a: i16, b: i16| -> u16 { + if a < b { + 0xFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_s16() { + testq_cmp_s16( + |i, j| vcltq_s16(i, j), + |a: i16, b: i16| -> u16 { + if a < b { + 0xFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vclt_s32() { + test_cmp_s32( + |i, j| vclt_s32(i, j), + |a: i32, b: i32| -> u32 { + if a < b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_s32() { + testq_cmp_s32( + |i, j| vcltq_s32(i, j), + |a: i32, b: i32| -> u32 { + if a < b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_u8() { + test_cmp_u8( + |i, j| vclt_u8(i, j), + |a: u8, b: u8| -> u8 { + if a < b { + 0xFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_u8() { + testq_cmp_u8( + |i, j| vcltq_u8(i, j), + |a: u8, b: u8| -> u8 { + if a < b { + 0xFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vclt_u16() { + test_cmp_u16( + |i, j| vclt_u16(i, j), + |a: u16, b: u16| -> u16 { + if a < b { + 0xFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_u16() { + testq_cmp_u16( + |i, j| vcltq_u16(i, j), + |a: u16, b: u16| -> u16 { + if a < b { + 0xFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vclt_u32() { + test_cmp_u32( + |i, j| vclt_u32(i, j), + |a: u32, b: u32| -> u32 { + if a < b { + 0xFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_u32() { + testq_cmp_u32( + |i, j| vcltq_u32(i, j), + |a: u32, b: u32| -> u32 { + if a < b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_f32() { + test_cmp_f32( + |i, j| vclt_f32(i, j), + |a: f32, b: f32| -> u32 { + if a < b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_f32() { + testq_cmp_f32( + |i, j| vcltq_f32(i, j), + |a: f32, b: f32| -> u32 { + if a < b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_s8() { + test_cmp_s8( + |i, j| vcle_s8(i, j), + |a: i8, b: i8| -> u8 { + if a <= b { + 0xFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_s8() { + testq_cmp_s8( + |i, j| vcleq_s8(i, j), + |a: i8, b: i8| -> u8 { + if a <= b { + 0xFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcle_s16() { + test_cmp_s16( + |i, j| vcle_s16(i, j), + |a: i16, b: i16| -> u16 { + if a <= b { + 0xFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_s16() { + testq_cmp_s16( + |i, j| vcleq_s16(i, j), + |a: i16, b: i16| -> u16 { + if a <= b { + 0xFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcle_s32() { + test_cmp_s32( + |i, j| vcle_s32(i, j), + |a: i32, b: i32| -> u32 { + if a <= b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_s32() { + testq_cmp_s32( + |i, j| vcleq_s32(i, j), + |a: i32, b: i32| -> u32 { + if a <= b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_u8() { + test_cmp_u8( + |i, j| vcle_u8(i, j), + |a: u8, b: u8| -> u8 { + if a <= b { + 0xFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_u8() { + testq_cmp_u8( + |i, j| vcleq_u8(i, j), + |a: u8, b: u8| -> u8 { + if a <= b { + 0xFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcle_u16() { + test_cmp_u16( + |i, j| vcle_u16(i, j), + |a: u16, b: u16| -> u16 { + if a <= b { + 0xFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_u16() { + testq_cmp_u16( + |i, j| vcleq_u16(i, j), + |a: u16, b: u16| -> u16 { + if a <= b { + 0xFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcle_u32() { + test_cmp_u32( + |i, j| vcle_u32(i, j), + |a: u32, b: u32| -> u32 { + if a <= b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_u32() { + testq_cmp_u32( + |i, j| vcleq_u32(i, j), + |a: u32, b: u32| -> u32 { + if a <= b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_f32() { + test_cmp_f32( + |i, j| vcle_f32(i, j), + |a: f32, b: f32| -> u32 { + if a <= b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_f32() { + testq_cmp_f32( + |i, j| vcleq_f32(i, j), + |a: f32, b: f32| -> u32 { + if a <= b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_s8() { + test_cmp_s8( + |i, j| vcge_s8(i, j), + |a: i8, b: i8| -> u8 { + if a >= b { + 0xFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_s8() { + testq_cmp_s8( + |i, j| vcgeq_s8(i, j), + |a: i8, b: i8| -> u8 { + if a >= b { + 0xFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcge_s16() { + test_cmp_s16( + |i, j| vcge_s16(i, j), + |a: i16, b: i16| -> u16 { + if a >= b { + 0xFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_s16() { + testq_cmp_s16( + |i, j| vcgeq_s16(i, j), + |a: i16, b: i16| -> u16 { + if a >= b { + 0xFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcge_s32() { + test_cmp_s32( + |i, j| vcge_s32(i, j), + |a: i32, b: i32| -> u32 { + if a >= b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_s32() { + testq_cmp_s32( + |i, j| vcgeq_s32(i, j), + |a: i32, b: i32| -> u32 { + if a >= b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_u8() { + test_cmp_u8( + |i, j| vcge_u8(i, j), + |a: u8, b: u8| -> u8 { + if a >= b { + 0xFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_u8() { + testq_cmp_u8( + |i, j| vcgeq_u8(i, j), + |a: u8, b: u8| -> u8 { + if a >= b { + 0xFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcge_u16() { + test_cmp_u16( + |i, j| vcge_u16(i, j), + |a: u16, b: u16| -> u16 { + if a >= b { + 0xFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_u16() { + testq_cmp_u16( + |i, j| vcgeq_u16(i, j), + |a: u16, b: u16| -> u16 { + if a >= b { + 0xFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcge_u32() { + test_cmp_u32( + |i, j| vcge_u32(i, j), + |a: u32, b: u32| -> u32 { + if a >= b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_u32() { + testq_cmp_u32( + |i, j| vcgeq_u32(i, j), + |a: u32, b: u32| -> u32 { + if a >= b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_f32() { + test_cmp_f32( + |i, j| vcge_f32(i, j), + |a: f32, b: f32| -> u32 { + if a >= b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_f32() { + testq_cmp_f32( + |i, j| vcgeq_f32(i, j), + |a: f32, b: f32| -> u32 { + if a >= b { + 0xFFFFFFFF + } else { + 0 + } + }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_s8() { + test_ari_s8( + |i, j| vqsub_s8(i, j), + |a: i8, b: i8| -> i8 { a.saturating_sub(b) }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_s8() { + testq_ari_s8( + |i, j| vqsubq_s8(i, j), + |a: i8, b: i8| -> i8 { a.saturating_sub(b) }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_s16() { + test_ari_s16( + |i, j| vqsub_s16(i, j), + |a: i16, b: i16| -> i16 { a.saturating_sub(b) }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_s16() { + testq_ari_s16( + |i, j| vqsubq_s16(i, j), + |a: i16, b: i16| -> i16 { a.saturating_sub(b) }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_s32() { + test_ari_s32( + |i, j| vqsub_s32(i, j), + |a: i32, b: i32| -> i32 { a.saturating_sub(b) }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_s32() { + testq_ari_s32( + |i, j| vqsubq_s32(i, j), + |a: i32, b: i32| -> i32 { a.saturating_sub(b) }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_u8() { + test_ari_u8( + |i, j| vqsub_u8(i, j), + |a: u8, b: u8| -> u8 { a.saturating_sub(b) }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_u8() { + testq_ari_u8( + |i, j| vqsubq_u8(i, j), + |a: u8, b: u8| -> u8 { a.saturating_sub(b) }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_u16() { + test_ari_u16( + |i, j| vqsub_u16(i, j), + |a: u16, b: u16| -> u16 { a.saturating_sub(b) }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_u16() { + testq_ari_u16( + |i, j| vqsubq_u16(i, j), + |a: u16, b: u16| -> u16 { a.saturating_sub(b) }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_u32() { + test_ari_u32( + |i, j| vqsub_u32(i, j), + |a: u32, b: u32| -> u32 { a.saturating_sub(b) }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_u32() { + testq_ari_u32( + |i, j| vqsubq_u32(i, j), + |a: u32, b: u32| -> u32 { a.saturating_sub(b) }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhadd_s8() { + test_ari_s8(|i, j| vhadd_s8(i, j), |a: i8, b: i8| -> i8 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vhaddq_s8() { + testq_ari_s8(|i, j| vhaddq_s8(i, j), |a: i8, b: i8| -> i8 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vhadd_s16() { + test_ari_s16(|i, j| vhadd_s16(i, j), |a: i16, b: i16| -> i16 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vhaddq_s16() { + testq_ari_s16(|i, j| vhaddq_s16(i, j), |a: i16, b: i16| -> i16 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vhadd_s32() { + test_ari_s32(|i, j| vhadd_s32(i, j), |a: i32, b: i32| -> i32 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vhaddq_s32() { + testq_ari_s32(|i, j| vhaddq_s32(i, j), |a: i32, b: i32| -> i32 { a & b }); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhadd_u8() { + test_ari_u8(|i, j| vhadd_u8(i, j), |a: u8, b: u8| -> u8 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vhaddq_u8() { + testq_ari_u8(|i, j| vhaddq_u8(i, j), |a: u8, b: u8| -> u8 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vhadd_u16() { + test_ari_u16(|i, j| vhadd_u16(i, j), |a: u16, b: u16| -> u16 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vhaddq_u16() { + testq_ari_u16(|i, j| vhaddq_u16(i, j), |a: u16, b: u16| -> u16 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vhadd_u32() { + test_ari_u32(|i, j| vhadd_u32(i, j), |a: u32, b: u32| -> u32 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vhaddq_u32() { + testq_ari_u32(|i, j| vhaddq_u32(i, j), |a: u32, b: u32| -> u32 { a & b }); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhadd_s8() { + test_ari_s8(|i, j| vrhadd_s8(i, j), |a: i8, b: i8| -> i8 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrhaddq_s8() { + testq_ari_s8(|i, j| vrhaddq_s8(i, j), |a: i8, b: i8| -> i8 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrhadd_s16() { + test_ari_s16(|i, j| vrhadd_s16(i, j), |a: i16, b: i16| -> i16 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrhaddq_s16() { + testq_ari_s16(|i, j| vrhaddq_s16(i, j), |a: i16, b: i16| -> i16 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrhadd_s32() { + test_ari_s32(|i, j| vrhadd_s32(i, j), |a: i32, b: i32| -> i32 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrhaddq_s32() { + testq_ari_s32(|i, j| vrhaddq_s32(i, j), |a: i32, b: i32| -> i32 { a & b }); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhadd_u8() { + test_ari_u8(|i, j| vrhadd_u8(i, j), |a: u8, b: u8| -> u8 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrhaddq_u8() { + testq_ari_u8(|i, j| vrhaddq_u8(i, j), |a: u8, b: u8| -> u8 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrhadd_u16() { + test_ari_u16(|i, j| vrhadd_u16(i, j), |a: u16, b: u16| -> u16 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrhaddq_u16() { + testq_ari_u16(|i, j| vrhaddq_u16(i, j), |a: u16, b: u16| -> u16 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrhadd_u32() { + test_ari_u32(|i, j| vrhadd_u32(i, j), |a: u32, b: u32| -> u32 { a & b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrhaddq_u32() { + testq_ari_u32(|i, j| vrhaddq_u32(i, j), |a: u32, b: u32| -> u32 { a & b }); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_s8() { + test_ari_s8( + |i, j| vqadd_s8(i, j), + |a: i8, b: i8| -> i8 { a.saturating_add(b) }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_s8() { + testq_ari_s8( + |i, j| vqaddq_s8(i, j), + |a: i8, b: i8| -> i8 { a.saturating_add(b) }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_s16() { + test_ari_s16( + |i, j| vqadd_s16(i, j), + |a: i16, b: i16| -> i16 { a.saturating_add(b) }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_s16() { + testq_ari_s16( + |i, j| vqaddq_s16(i, j), + |a: i16, b: i16| -> i16 { a.saturating_add(b) }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_s32() { + test_ari_s32( + |i, j| vqadd_s32(i, j), + |a: i32, b: i32| -> i32 { a.saturating_add(b) }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_s32() { + testq_ari_s32( + |i, j| vqaddq_s32(i, j), + |a: i32, b: i32| -> i32 { a.saturating_add(b) }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_u8() { + test_ari_u8( + |i, j| vqadd_u8(i, j), + |a: u8, b: u8| -> u8 { a.saturating_add(b) }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_u8() { + testq_ari_u8( + |i, j| vqaddq_u8(i, j), + |a: u8, b: u8| -> u8 { a.saturating_add(b) }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_u16() { + test_ari_u16( + |i, j| vqadd_u16(i, j), + |a: u16, b: u16| -> u16 { a.saturating_add(b) }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_u16() { + testq_ari_u16( + |i, j| vqaddq_u16(i, j), + |a: u16, b: u16| -> u16 { a.saturating_add(b) }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_u32() { + test_ari_u32( + |i, j| vqadd_u32(i, j), + |a: u32, b: u32| -> u32 { a.saturating_add(b) }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_u32() { + testq_ari_u32( + |i, j| vqaddq_u32(i, j), + |a: u32, b: u32| -> u32 { a.saturating_add(b) }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_s8() { + test_ari_s8( + |i, j| vmul_s8(i, j), + |a: i8, b: i8| -> i8 { a.overflowing_mul(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_s8() { + testq_ari_s8( + |i, j| vmulq_s8(i, j), + |a: i8, b: i8| -> i8 { a.overflowing_mul(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vmul_s16() { + test_ari_s16( + |i, j| vmul_s16(i, j), + |a: i16, b: i16| -> i16 { a.overflowing_mul(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_s16() { + testq_ari_s16( + |i, j| vmulq_s16(i, j), + |a: i16, b: i16| -> i16 { a.overflowing_mul(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vmul_s32() { + test_ari_s32( + |i, j| vmul_s32(i, j), + |a: i32, b: i32| -> i32 { a.overflowing_mul(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_s32() { + testq_ari_s32( + |i, j| vmulq_s32(i, j), + |a: i32, b: i32| -> i32 { a.overflowing_mul(b).0 }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_u8() { + test_ari_u8( + |i, j| vmul_u8(i, j), + |a: u8, b: u8| -> u8 { a.overflowing_mul(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_u8() { + testq_ari_u8( + |i, j| vmulq_u8(i, j), + |a: u8, b: u8| -> u8 { a.overflowing_mul(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vmul_u16() { + test_ari_u16( + |i, j| vmul_u16(i, j), + |a: u16, b: u16| -> u16 { a.overflowing_mul(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_u16() { + testq_ari_u16( + |i, j| vmulq_u16(i, j), + |a: u16, b: u16| -> u16 { a.overflowing_mul(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vmul_u32() { + test_ari_u32( + |i, j| vmul_u32(i, j), + |a: u32, b: u32| -> u32 { a.overflowing_mul(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_u32() { + testq_ari_u32( + |i, j| vmulq_u32(i, j), + |a: u32, b: u32| -> u32 { a.overflowing_mul(b).0 }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_f32() { + test_ari_f32(|i, j| vmul_f32(i, j), |a: f32, b: f32| -> f32 { a * b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_f32() { + testq_ari_f32(|i, j| vmulq_f32(i, j), |a: f32, b: f32| -> f32 { a * b }); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_s8() { + test_ari_s8(|i, j| vsub_s8(i, j), |a: i8, b: i8| -> i8 { a - b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_s8() { + testq_ari_s8(|i, j| vsubq_s8(i, j), |a: i8, b: i8| -> i8 { a - b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vsub_s16() { + test_ari_s16(|i, j| vsub_s16(i, j), |a: i16, b: i16| -> i16 { a - b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_s16() { + testq_ari_s16(|i, j| vsubq_s16(i, j), |a: i16, b: i16| -> i16 { a - b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vsub_s32() { + test_ari_s32(|i, j| vsub_s32(i, j), |a: i32, b: i32| -> i32 { a - b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_s32() { + testq_ari_s32(|i, j| vsubq_s32(i, j), |a: i32, b: i32| -> i32 { a - b }); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_u8() { + test_ari_u8(|i, j| vsub_u8(i, j), |a: u8, b: u8| -> u8 { a - b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_u8() { + testq_ari_u8(|i, j| vsubq_u8(i, j), |a: u8, b: u8| -> u8 { a - b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vsub_u16() { + test_ari_u16(|i, j| vsub_u16(i, j), |a: u16, b: u16| -> u16 { a - b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_u16() { + testq_ari_u16(|i, j| vsubq_u16(i, j), |a: u16, b: u16| -> u16 { a - b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vsub_u32() { + test_ari_u32(|i, j| vsub_u32(i, j), |a: u32, b: u32| -> u32 { a - b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_u32() { + testq_ari_u32(|i, j| vsubq_u32(i, j), |a: u32, b: u32| -> u32 { a - b }); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_f32() { + test_ari_f32(|i, j| vsub_f32(i, j), |a: f32, b: f32| -> f32 { a - b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_f32() { + testq_ari_f32(|i, j| vsubq_f32(i, j), |a: f32, b: f32| -> f32 { a - b }); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsub_s8() { + test_ari_s8( + |i, j| vhsub_s8(i, j), + |a: i8, b: i8| -> i8 { (((a as i16) - (b as i16)) / 2) as i8 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vhsubq_s8() { + testq_ari_s8( + |i, j| vhsubq_s8(i, j), + |a: i8, b: i8| -> i8 { (((a as i16) - (b as i16)) / 2) as i8 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vhsub_s16() { + test_ari_s16( + |i, j| vhsub_s16(i, j), + |a: i16, b: i16| -> i16 { (((a as i32) - (b as i32)) / 2) as i16 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vhsubq_s16() { + testq_ari_s16( + |i, j| vhsubq_s16(i, j), + |a: i16, b: i16| -> i16 { (((a as i32) - (b as i32)) / 2) as i16 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vhsub_s32() { + test_ari_s32( + |i, j| vhsub_s32(i, j), + |a: i32, b: i32| -> i32 { (((a as i64) - (b as i64)) / 2) as i32 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vhsubq_s32() { + testq_ari_s32( + |i, j| vhsubq_s32(i, j), + |a: i32, b: i32| -> i32 { (((a as i64) - (b as i64)) / 2) as i32 }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsub_u8() { + test_ari_u8( + |i, j| vhsub_u8(i, j), + |a: u8, b: u8| -> u8 { (((a as u16) - (b as u16)) / 2) as u8 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vhsubq_u8() { + testq_ari_u8( + |i, j| vhsubq_u8(i, j), + |a: u8, b: u8| -> u8 { (((a as u16) - (b as u16)) / 2) as u8 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vhsub_u16() { + test_ari_u16( + |i, j| vhsub_u16(i, j), + |a: u16, b: u16| -> u16 { (((a as u16) - (b as u16)) / 2) as u16 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vhsubq_u16() { + testq_ari_u16( + |i, j| vhsubq_u16(i, j), + |a: u16, b: u16| -> u16 { (((a as u16) - (b as u16)) / 2) as u16 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vhsub_u32() { + test_ari_u32( + |i, j| vhsub_u32(i, j), + |a: u32, b: u32| -> u32 { (((a as u64) - (b as u64)) / 2) as u32 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vhsubq_u32() { + testq_ari_u32( + |i, j| vhsubq_u32(i, j), + |a: u32, b: u32| -> u32 { (((a as u64) - (b as u64)) / 2) as u32 }, + ); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabs_s8() { + let a = i8x8::new(-1, 0, 1, -2, 0, 2, -128, 127); + let r: i8x8 = transmute(vabs_s8(transmute(a))); + let e = i8x8::new(1, 0, 1, 2, 0, 2, -128, 127); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabsq_s8() { + let a = i8x16::new(-1, 0, 1, -2, 0, 2, -128, 127, -1, 0, 1, -2, 0, 2, -128, 127); + let r: i8x16 = transmute(vabsq_s8(transmute(a))); + let e = i8x16::new(1, 0, 1, 2, 0, 2, -128, 127, 1, 0, 1, 2, 0, 2, -128, 127); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabs_s16() { + let a = i16x4::new(-1, 0, i16::MIN, i16::MAX); + let r: i16x4 = transmute(vabs_s16(transmute(a))); + let e = i16x4::new(1, 0, i16::MIN, i16::MAX); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabsq_s16() { + let a = i16x8::new(-1, 0, i16::MIN, i16::MAX, -1, 0, i16::MIN, i16::MAX); + let r: i16x8 = transmute(vabsq_s16(transmute(a))); + let e = i16x8::new(1, 0, i16::MIN, i16::MAX, 1, 0, i16::MIN, i16::MAX); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabs_s32() { + let a = i32x2::new(i32::MIN, i32::MIN + 1); + let r: i32x2 = transmute(vabs_s32(transmute(a))); + let e = i32x2::new(i32::MIN, i32::MAX); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabsq_s32() { + let a = i32x4::new(i32::MIN, i32::MIN + 1, 0, -1); + let r: i32x4 = transmute(vabsq_s32(transmute(a))); + let e = i32x4::new(i32::MIN, i32::MAX, 0, 1); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaba_s8() { + let a = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b = i8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let c = i8x8::new(10, 9, 8, 7, 6, 5, 4, 3); + let r: i8x8 = transmute(vaba_s8(transmute(a), transmute(b), transmute(c))); + let e = i8x8::new(10, 10, 10, 10, 10, 10, 10, 10); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaba_s16() { + let a = i16x4::new(1, 2, 3, 4); + let b = i16x4::new(1, 1, 1, 1); + let c = i16x4::new(10, 9, 8, 7); + let r: i16x4 = transmute(vaba_s16(transmute(a), transmute(b), transmute(c))); + let e = i16x4::new(10, 10, 10, 10); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaba_s32() { + let a = i32x2::new(1, 2); + let b = i32x2::new(1, 1); + let c = i32x2::new(10, 9); + let r: i32x2 = transmute(vaba_s32(transmute(a), transmute(b), transmute(c))); + let e = i32x2::new(10, 10); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaba_u8() { + let a = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b = u8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let c = u8x8::new(10, 9, 8, 7, 6, 5, 4, 3); + let r: u8x8 = transmute(vaba_u8(transmute(a), transmute(b), transmute(c))); + let e = u8x8::new(10, 10, 10, 10, 10, 10, 10, 10); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaba_u16() { + let a = u16x4::new(1, 2, 3, 4); + let b = u16x4::new(1, 1, 1, 1); + let c = u16x4::new(10, 9, 8, 7); + let r: u16x4 = transmute(vaba_u16(transmute(a), transmute(b), transmute(c))); + let e = u16x4::new(10, 10, 10, 10); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaba_u32() { + let a = u32x2::new(1, 2); + let b = u32x2::new(1, 1); + let c = u32x2::new(10, 9); + let r: u32x2 = transmute(vaba_u32(transmute(a), transmute(b), transmute(c))); + let e = u32x2::new(10, 10); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabaq_s8() { + let a = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3, 2); + let b = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let c = i8x16::new(10, 9, 8, 7, 6, 5, 4, 3, 12, 13, 14, 15, 16, 17, 18, 19); + let r: i8x16 = transmute(vabaq_s8(transmute(a), transmute(b), transmute(c))); + let e = i8x16::new( + 10, 10, 10, 10, 10, 10, 10, 10, 20, 20, 20, 20, 20, 20, 20, 20, + ); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabaq_s16() { + let a = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let c = i16x8::new(10, 9, 8, 7, 6, 5, 4, 3); + let r: i16x8 = transmute(vabaq_s16(transmute(a), transmute(b), transmute(c))); + let e = i16x8::new(10, 10, 10, 10, 10, 10, 10, 10); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabaq_s32() { + let a = i32x4::new(1, 2, 3, 4); + let b = i32x4::new(1, 1, 1, 1); + let c = i32x4::new(10, 9, 8, 7); + let r: i32x4 = transmute(vabaq_s32(transmute(a), transmute(b), transmute(c))); + let e = i32x4::new(10, 10, 10, 10); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabaq_u8() { + let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3, 2); + let b = u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let c = u8x16::new(10, 9, 8, 7, 6, 5, 4, 3, 12, 13, 14, 15, 16, 17, 18, 19); + let r: u8x16 = transmute(vabaq_u8(transmute(a), transmute(b), transmute(c))); + let e = u8x16::new( + 10, 10, 10, 10, 10, 10, 10, 10, 20, 20, 20, 20, 20, 20, 20, 20, + ); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabaq_u16() { + let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b = u16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let c = u16x8::new(10, 9, 8, 7, 6, 5, 4, 3); + let r: u16x8 = transmute(vabaq_u16(transmute(a), transmute(b), transmute(c))); + let e = u16x8::new(10, 10, 10, 10, 10, 10, 10, 10); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabaq_u32() { + let a = u32x4::new(1, 2, 3, 4); + let b = u32x4::new(1, 1, 1, 1); + let c = u32x4::new(10, 9, 8, 7); + let r: u32x4 = transmute(vabaq_u32(transmute(a), transmute(b), transmute(c))); + let e = u32x4::new(10, 10, 10, 10); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpadd_s16() { + let a = i16x4::new(1, 2, 3, 4); + let b = i16x4::new(0, -1, -2, -3); + let r: i16x4 = transmute(vpadd_s16(transmute(a), transmute(b))); + let e = i16x4::new(3, 7, -1, -5); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vpadd_s32() { + let a = i32x2::new(1, 2); + let b = i32x2::new(0, -1); + let r: i32x2 = transmute(vpadd_s32(transmute(a), transmute(b))); + let e = i32x2::new(3, -1); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vpadd_s8() { + let a = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b = i8x8::new(0, -1, -2, -3, -4, -5, -6, -7); + let r: i8x8 = transmute(vpadd_s8(transmute(a), transmute(b))); + let e = i8x8::new(3, 7, 11, 15, -1, -5, -9, -13); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vpadd_u16() { + let a = u16x4::new(1, 2, 3, 4); + let b = u16x4::new(30, 31, 32, 33); + let r: u16x4 = transmute(vpadd_u16(transmute(a), transmute(b))); + let e = u16x4::new(3, 7, 61, 65); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vpadd_u32() { + let a = u32x2::new(1, 2); + let b = u32x2::new(30, 31); + let r: u32x2 = transmute(vpadd_u32(transmute(a), transmute(b))); + let e = u32x2::new(3, 61); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vpadd_u8() { + let a = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b = u8x8::new(30, 31, 32, 33, 34, 35, 36, 37); + let r: u8x8 = transmute(vpadd_u8(transmute(a), transmute(b))); + let e = u8x8::new(3, 7, 11, 15, 61, 65, 69, 73); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcnt_s8() { + let a: i8x8 = transmute(u8x8::new( + 0b11001000, 0b11111111, 0b00000000, 0b11011111, 0b10000001, 0b10101001, 0b00001000, + 0b00111111, + )); + let e = i8x8::new(3, 8, 0, 7, 2, 4, 1, 6); + let r: i8x8 = transmute(vcnt_s8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcntq_s8() { + let a: i8x16 = transmute(u8x16::new( + 0b11001000, 0b11111111, 0b00000000, 0b11011111, 0b10000001, 0b10101001, 0b00001000, + 0b00111111, 0b11101110, 0b00000000, 0b11111111, 0b00100001, 0b11111111, 0b10010111, + 0b11100000, 0b00010000, + )); + let e = i8x16::new(3, 8, 0, 7, 2, 4, 1, 6, 6, 0, 8, 2, 8, 5, 3, 1); + let r: i8x16 = transmute(vcntq_s8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcnt_u8() { + let a = u8x8::new( + 0b11001000, 0b11111111, 0b00000000, 0b11011111, 0b10000001, 0b10101001, 0b00001000, + 0b00111111, + ); + let e = u8x8::new(3, 8, 0, 7, 2, 4, 1, 6); + let r: u8x8 = transmute(vcnt_u8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcntq_u8() { + let a = u8x16::new( + 0b11001000, 0b11111111, 0b00000000, 0b11011111, 0b10000001, 0b10101001, 0b00001000, + 0b00111111, 0b11101110, 0b00000000, 0b11111111, 0b00100001, 0b11111111, 0b10010111, + 0b11100000, 0b00010000, + ); + let e = u8x16::new(3, 8, 0, 7, 2, 4, 1, 6, 6, 0, 8, 2, 8, 5, 3, 1); + let r: u8x16 = transmute(vcntq_u8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcnt_p8() { + let a = u8x8::new( + 0b11001000, 0b11111111, 0b00000000, 0b11011111, 0b10000001, 0b10101001, 0b00001000, + 0b00111111, + ); + let e = u8x8::new(3, 8, 0, 7, 2, 4, 1, 6); + let r: u8x8 = transmute(vcnt_p8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vcntq_p8() { + let a = u8x16::new( + 0b11001000, 0b11111111, 0b00000000, 0b11011111, 0b10000001, 0b10101001, 0b00001000, + 0b00111111, 0b11101110, 0b00000000, 0b11111111, 0b00100001, 0b11111111, 0b10010111, + 0b11100000, 0b00010000, + ); + let e = u8x16::new(3, 8, 0, 7, 2, 4, 1, 6, 6, 0, 8, 2, 8, 5, 3, 1); + let r: u8x16 = transmute(vcntq_p8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev16_s8() { + let a = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = i8x8::new(1, 0, 3, 2, 5, 4, 7, 6); + let e: i8x8 = transmute(vrev16_s8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev16q_s8() { + let a = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = i8x16::new(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + let e: i8x16 = transmute(vrev16q_s8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev16_u8() { + let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = u8x8::new(1, 0, 3, 2, 5, 4, 7, 6); + let e: u8x8 = transmute(vrev16_u8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev16q_u8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = u8x16::new(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + let e: u8x16 = transmute(vrev16q_u8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev16_p8() { + let a = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = i8x8::new(1, 0, 3, 2, 5, 4, 7, 6); + let e: i8x8 = transmute(vrev16_p8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev16q_p8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = u8x16::new(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + let e: u8x16 = transmute(vrev16q_p8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev32_s8() { + let a = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = i8x8::new(3, 2, 1, 0, 7, 6, 5, 4); + let e: i8x8 = transmute(vrev32_s8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev32q_s8() { + let a = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = i8x16::new(3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); + let e: i8x16 = transmute(vrev32q_s8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev32_u8() { + let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = u8x8::new(3, 2, 1, 0, 7, 6, 5, 4); + let e: u8x8 = transmute(vrev32_u8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev32q_u8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = u8x16::new(3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); + let e: u8x16 = transmute(vrev32q_u8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev32_s16() { + let a = i16x4::new(0, 1, 2, 3); + let r = i16x4::new(1, 0, 3, 2); + let e: i16x4 = transmute(vrev32_s16(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev32q_s16() { + let a = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = i16x8::new(1, 0, 3, 2, 5, 4, 7, 6); + let e: i16x8 = transmute(vrev32q_s16(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev32_p16() { + let a = i16x4::new(0, 1, 2, 3); + let r = i16x4::new(1, 0, 3, 2); + let e: i16x4 = transmute(vrev32_p16(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev32q_p16() { + let a = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = i16x8::new(1, 0, 3, 2, 5, 4, 7, 6); + let e: i16x8 = transmute(vrev32q_p16(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev32_u16() { + let a = u16x4::new(0, 1, 2, 3); + let r = u16x4::new(1, 0, 3, 2); + let e: u16x4 = transmute(vrev32_u16(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev32q_u16() { + let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = u16x8::new(1, 0, 3, 2, 5, 4, 7, 6); + let e: u16x8 = transmute(vrev32q_u16(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev32_p8() { + let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = u8x8::new(3, 2, 1, 0, 7, 6, 5, 4); + let e: u8x8 = transmute(vrev32_p8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev32q_p8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = u8x16::new(3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); + let e: u8x16 = transmute(vrev32q_p8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev64_s8() { + let a = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = i8x8::new(7, 6, 5, 4, 3, 2, 1, 0); + let e: i8x8 = transmute(vrev64_s8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev64q_s8() { + let a = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = i8x16::new(7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); + let e: i8x16 = transmute(vrev64q_s8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev64_s16() { + let a = i16x4::new(0, 1, 2, 3); + let r = i16x4::new(3, 2, 1, 0); + let e: i16x4 = transmute(vrev64_s16(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev64q_s16() { + let a = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = i16x8::new(3, 2, 1, 0, 7, 6, 5, 4); + let e: i16x8 = transmute(vrev64q_s16(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev64_s32() { + let a = i32x2::new(0, 1); + let r = i32x2::new(1, 0); + let e: i32x2 = transmute(vrev64_s32(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev64q_s32() { + let a = i32x4::new(0, 1, 2, 3); + let r = i32x4::new(1, 0, 3, 2); + let e: i32x4 = transmute(vrev64q_s32(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev64_u8() { + let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = u8x8::new(7, 6, 5, 4, 3, 2, 1, 0); + let e: u8x8 = transmute(vrev64_u8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev64q_u8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = u8x16::new(7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); + let e: u8x16 = transmute(vrev64q_u8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev64_u16() { + let a = u16x4::new(0, 1, 2, 3); + let r = u16x4::new(3, 2, 1, 0); + let e: u16x4 = transmute(vrev64_u16(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev64q_u16() { + let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = u16x8::new(3, 2, 1, 0, 7, 6, 5, 4); + let e: u16x8 = transmute(vrev64q_u16(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev64_u32() { + let a = u32x2::new(0, 1); + let r = u32x2::new(1, 0); + let e: u32x2 = transmute(vrev64_u32(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev64q_u32() { + let a = u32x4::new(0, 1, 2, 3); + let r = u32x4::new(1, 0, 3, 2); + let e: u32x4 = transmute(vrev64q_u32(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev64_f32() { + let a = f32x2::new(1.0, 2.0); + let r = f32x2::new(2.0, 1.0); + let e: f32x2 = transmute(vrev64_f32(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev64q_f32() { + let a = f32x4::new(1.0, 2.0, -2.0, -1.0); + let r = f32x4::new(2.0, 1.0, -1.0, -2.0); + let e: f32x4 = transmute(vrev64q_f32(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev64_p8() { + let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = u8x8::new(7, 6, 5, 4, 3, 2, 1, 0); + let e: u8x8 = transmute(vrev64_p8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev64q_p8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = u8x16::new(7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); + let e: u8x16 = transmute(vrev64q_p8(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev64_p16() { + let a = u16x4::new(0, 1, 2, 3); + let r = u16x4::new(3, 2, 1, 0); + let e: u16x4 = transmute(vrev64_p16(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev64q_p16() { + let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = u16x8::new(3, 2, 1, 0, 7, 6, 5, 4); + let e: u16x8 = transmute(vrev64q_p16(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon,i8mm")] + unsafe fn test_vmmlaq_s32() { + let a = i32x4::new(1, 3, 4, -0x10000); + let b = i8x16::new(1, 21, 31, 14, 5, 6, -128, 8, 9, 13, 15, 12, 13, -1, 20, 16); + let c = i8x16::new(12, 22, 3, 4, -1, 56, 7, 8, 91, 10, -128, 15, 13, 14, 17, 16); + let e = i32x4::new(123, -5353, 690, -65576); + let r: i32x4 = transmute(vmmlaq_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,i8mm")] + unsafe fn test_vmmlaq_u32() { + let a = u32x4::new(1, 3, 4, 0xffff0000); + let b = u8x16::new(1, 21, 31, 14, 5, 6, 128, 8, 9, 13, 15, 12, 13, 255, 20, 16); + let c = u8x16::new(12, 22, 3, 4, 255, 56, 7, 8, 91, 10, 128, 15, 13, 14, 17, 16); + let e = u32x4::new(3195, 6935, 18354, 4294909144); + let r: u32x4 = transmute(vmmlaq_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon,i8mm")] + unsafe fn test_vusmmlaq_s32() { + let a = i32x4::new(1, 3, 4, -0x10000); + let b = u8x16::new(1, 21, 31, 14, 5, 6, 128, 8, 9, 13, 15, 12, 13, 255, 20, 16); + let c = i8x16::new(12, 22, 3, 4, -1, 56, 7, 8, 91, 10, -128, 15, 13, 14, 17, 16); + let e = i32x4::new(1915, -1001, 15026, -61992); + let r: i32x4 = transmute(vusmmlaq_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } +} + +#[cfg(all(test, target_arch = "arm", target_endian = "little"))] +mod table_lookup_tests; + +#[cfg(all(test, target_arch = "arm"))] +mod shift_and_insert_tests; + +#[cfg(all(test, target_arch = "arm"))] +mod load_tests; + +#[cfg(all(test, target_arch = "arm"))] +mod store_tests; diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/shift_and_insert_tests.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/shift_and_insert_tests.rs new file mode 100644 index 000000000..ebb8b7b9e --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/shift_and_insert_tests.rs @@ -0,0 +1,93 @@ +//! Tests for ARM+v7+neon shift and insert (vsli[q]_n, vsri[q]_n) intrinsics. +//! +//! These are included in `{arm, aarch64}::neon`. + +use super::*; + +#[cfg(target_arch = "aarch64")] +use crate::core_arch::aarch64::*; + +#[cfg(target_arch = "arm")] +use crate::core_arch::arm::*; + +use crate::core_arch::simd::*; +use std::mem::transmute; +use stdarch_test::simd_test; + +macro_rules! test_vsli { + ($test_id:ident, $t:ty => $fn_id:ident ([$($a:expr),*], [$($b:expr),*], $n:expr)) => { + #[simd_test(enable = "neon")] + #[allow(unused_assignments)] + unsafe fn $test_id() { + let a = [$($a as $t),*]; + let b = [$($b as $t),*]; + let n_bit_mask: $t = (1 << $n) - 1; + let e = [$(($a as $t & n_bit_mask) | ($b as $t << $n)),*]; + let r = $fn_id::<$n>(transmute(a), transmute(b)); + let mut d = e; + d = transmute(r); + assert_eq!(d, e); + } + } +} +test_vsli!(test_vsli_n_s8, i8 => vsli_n_s8([3, -44, 127, -56, 0, 24, -97, 10], [-128, -14, 125, -77, 27, 8, -1, 110], 5)); +test_vsli!(test_vsliq_n_s8, i8 => vsliq_n_s8([3, -44, 127, -56, 0, 24, -97, 10, -33, 1, -6, -39, 15, 101, -80, -1], [-128, -14, 125, -77, 27, 8, -1, 110, -4, -92, 111, 32, 1, -4, -29, 99], 2)); +test_vsli!(test_vsli_n_s16, i16 => vsli_n_s16([3304, -44, 2300, -546], [-1208, -140, 1225, -707], 7)); +test_vsli!(test_vsliq_n_s16, i16 => vsliq_n_s16([3304, -44, 2300, -20046, 0, 9924, -907, 1190], [-1208, -140, 4225, -707, 2701, 804, -71, 2110], 14)); +test_vsli!(test_vsli_n_s32, i32 => vsli_n_s32([125683, -78901], [-128, -112944], 23)); +test_vsli!(test_vsliq_n_s32, i32 => vsliq_n_s32([125683, -78901, 127, -12009], [-128, -112944, 125, -707], 15)); +test_vsli!(test_vsli_n_s64, i64 => vsli_n_s64([-333333], [1028], 45)); +test_vsli!(test_vsliq_n_s64, i64 => vsliq_n_s64([-333333, -52023], [1028, -99814], 33)); +test_vsli!(test_vsli_n_u8, u8 => vsli_n_u8([3, 44, 127, 56, 0, 24, 97, 10], [127, 14, 125, 77, 27, 8, 1, 110], 5)); +test_vsli!(test_vsliq_n_u8, u8 => vsliq_n_u8([3, 44, 127, 56, 0, 24, 97, 10, 33, 1, 6, 39, 15, 101, 80, 1], [127, 14, 125, 77, 27, 8, 1, 110, 4, 92, 111, 32, 1, 4, 29, 99], 2)); +test_vsli!(test_vsli_n_u16, u16 => vsli_n_u16([3304, 44, 2300, 546], [1208, 140, 1225, 707], 7)); +test_vsli!(test_vsliq_n_u16, u16 => vsliq_n_u16([3304, 44, 2300, 20046, 0, 9924, 907, 1190], [1208, 140, 4225, 707, 2701, 804, 71, 2110], 14)); +test_vsli!(test_vsli_n_u32, u32 => vsli_n_u32([125683, 78901], [128, 112944], 23)); +test_vsli!(test_vsliq_n_u32, u32 => vsliq_n_u32([125683, 78901, 127, 12009], [128, 112944, 125, 707], 15)); +test_vsli!(test_vsli_n_u64, u64 => vsli_n_u64([333333], [1028], 45)); +test_vsli!(test_vsliq_n_u64, u64 => vsliq_n_u64([333333, 52023], [1028, 99814], 33)); +test_vsli!(test_vsli_n_p8, i8 => vsli_n_p8([3, 44, 127, 56, 0, 24, 97, 10], [127, 14, 125, 77, 27, 8, 1, 110], 5)); +test_vsli!(test_vsliq_n_p8, i8 => vsliq_n_p8([3, 44, 127, 56, 0, 24, 97, 10, 33, 1, 6, 39, 15, 101, 80, 1], [127, 14, 125, 77, 27, 8, 1, 110, 4, 92, 111, 32, 1, 4, 29, 99], 2)); +test_vsli!(test_vsli_n_p16, i16 => vsli_n_p16([3304, 44, 2300, 546], [1208, 140, 1225, 707], 7)); +test_vsli!(test_vsliq_n_p16, i16 => vsliq_n_p16([3304, 44, 2300, 20046, 0, 9924, 907, 1190], [1208, 140, 4225, 707, 2701, 804, 71, 2110], 14)); +test_vsli!(test_vsli_n_p64, i64 => vsli_n_p64([333333], [1028], 45)); +test_vsli!(test_vsliq_n_p64, i64 => vsliq_n_p64([333333, 52023], [1028, 99814], 33)); + +macro_rules! test_vsri { + ($test_id:ident, $t:ty => $fn_id:ident ([$($a:expr),*], [$($b:expr),*], $n:expr)) => { + #[simd_test(enable = "neon")] + #[allow(unused_assignments)] + unsafe fn $test_id() { + let a = [$($a as $t),*]; + let b = [$($b as $t),*]; + let n_bit_mask = ((1 as $t << $n) - 1).rotate_right($n); + let e = [$(($a as $t & n_bit_mask) | (($b as $t >> $n) & !n_bit_mask)),*]; + let r = $fn_id::<$n>(transmute(a), transmute(b)); + let mut d = e; + d = transmute(r); + assert_eq!(d, e); + } + } +} +test_vsri!(test_vsri_n_s8, i8 => vsri_n_s8([3, -44, 127, -56, 0, 24, -97, 10], [-128, -14, 125, -77, 27, 8, -1, 110], 5)); +test_vsri!(test_vsriq_n_s8, i8 => vsriq_n_s8([3, -44, 127, -56, 0, 24, -97, 10, -33, 1, -6, -39, 15, 101, -80, -1], [-128, -14, 125, -77, 27, 8, -1, 110, -4, -92, 111, 32, 1, -4, -29, 99], 2)); +test_vsri!(test_vsri_n_s16, i16 => vsri_n_s16([3304, -44, 2300, -546], [-1208, -140, 1225, -707], 7)); +test_vsri!(test_vsriq_n_s16, i16 => vsriq_n_s16([3304, -44, 2300, -20046, 0, 9924, -907, 1190], [-1208, -140, 4225, -707, 2701, 804, -71, 2110], 14)); +test_vsri!(test_vsri_n_s32, i32 => vsri_n_s32([125683, -78901], [-128, -112944], 23)); +test_vsri!(test_vsriq_n_s32, i32 => vsriq_n_s32([125683, -78901, 127, -12009], [-128, -112944, 125, -707], 15)); +test_vsri!(test_vsri_n_s64, i64 => vsri_n_s64([-333333], [1028], 45)); +test_vsri!(test_vsriq_n_s64, i64 => vsriq_n_s64([-333333, -52023], [1028, -99814], 33)); +test_vsri!(test_vsri_n_u8, u8 => vsri_n_u8([3, 44, 127, 56, 0, 24, 97, 10], [127, 14, 125, 77, 27, 8, 1, 110], 5)); +test_vsri!(test_vsriq_n_u8, u8 => vsriq_n_u8([3, 44, 127, 56, 0, 24, 97, 10, 33, 1, 6, 39, 15, 101, 80, 1], [127, 14, 125, 77, 27, 8, 1, 110, 4, 92, 111, 32, 1, 4, 29, 99], 2)); +test_vsri!(test_vsri_n_u16, u16 => vsri_n_u16([3304, 44, 2300, 546], [1208, 140, 1225, 707], 7)); +test_vsri!(test_vsriq_n_u16, u16 => vsriq_n_u16([3304, 44, 2300, 20046, 0, 9924, 907, 1190], [1208, 140, 4225, 707, 2701, 804, 71, 2110], 14)); +test_vsri!(test_vsri_n_u32, u32 => vsri_n_u32([125683, 78901], [128, 112944], 23)); +test_vsri!(test_vsriq_n_u32, u32 => vsriq_n_u32([125683, 78901, 127, 12009], [128, 112944, 125, 707], 15)); +test_vsri!(test_vsri_n_u64, u64 => vsri_n_u64([333333], [1028], 45)); +test_vsri!(test_vsriq_n_u64, u64 => vsriq_n_u64([333333, 52023], [1028, 99814], 33)); +test_vsri!(test_vsri_n_p8, i8 => vsri_n_p8([3, 44, 127, 56, 0, 24, 97, 10], [127, 14, 125, 77, 27, 8, 1, 110], 5)); +test_vsri!(test_vsriq_n_p8, i8 => vsriq_n_p8([3, 44, 127, 56, 0, 24, 97, 10, 33, 1, 6, 39, 15, 101, 80, 1], [127, 14, 125, 77, 27, 8, 1, 110, 4, 92, 111, 32, 1, 4, 29, 99], 2)); +test_vsri!(test_vsri_n_p16, i16 => vsri_n_p16([3304, 44, 2300, 546], [1208, 140, 1225, 707], 7)); +test_vsri!(test_vsriq_n_p16, i16 => vsriq_n_p16([3304, 44, 2300, 20046, 0, 9924, 907, 1190], [1208, 140, 4225, 707, 2701, 804, 71, 2110], 14)); +test_vsri!(test_vsri_n_p64, i64 => vsri_n_p64([333333], [1028], 45)); +test_vsri!(test_vsriq_n_p64, i64 => vsriq_n_p64([333333, 52023], [1028, 99814], 33)); diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/store_tests.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/store_tests.rs new file mode 100644 index 000000000..cad660e87 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/store_tests.rs @@ -0,0 +1,389 @@ +//! Tests for ARM+v7+neon store (vst1) intrinsics. +//! +//! These are included in `{arm, aarch64}::neon`. + +use super::*; + +#[cfg(target_arch = "arm")] +use crate::core_arch::arm::*; + +#[cfg(target_arch = "aarch64")] +use crate::core_arch::aarch64::*; + +use crate::core_arch::simd::*; +use stdarch_test::simd_test; + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_s8() { + let mut vals = [0_i8; 9]; + let a = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + + vst1_s8(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); + assert_eq!(vals[5], 5); + assert_eq!(vals[6], 6); + assert_eq!(vals[7], 7); + assert_eq!(vals[8], 8); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_s8() { + let mut vals = [0_i8; 17]; + let a = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + + vst1q_s8(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); + assert_eq!(vals[5], 5); + assert_eq!(vals[6], 6); + assert_eq!(vals[7], 7); + assert_eq!(vals[8], 8); + assert_eq!(vals[9], 9); + assert_eq!(vals[10], 10); + assert_eq!(vals[11], 11); + assert_eq!(vals[12], 12); + assert_eq!(vals[13], 13); + assert_eq!(vals[14], 14); + assert_eq!(vals[15], 15); + assert_eq!(vals[16], 16); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_s16() { + let mut vals = [0_i16; 5]; + let a = i16x4::new(1, 2, 3, 4); + + vst1_s16(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_s16() { + let mut vals = [0_i16; 9]; + let a = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + + vst1q_s16(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); + assert_eq!(vals[5], 5); + assert_eq!(vals[6], 6); + assert_eq!(vals[7], 7); + assert_eq!(vals[8], 8); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_s32() { + let mut vals = [0_i32; 3]; + let a = i32x2::new(1, 2); + + vst1_s32(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_s32() { + let mut vals = [0_i32; 5]; + let a = i32x4::new(1, 2, 3, 4); + + vst1q_s32(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_s64() { + let mut vals = [0_i64; 2]; + let a = i64x1::new(1); + + vst1_s64(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_s64() { + let mut vals = [0_i64; 3]; + let a = i64x2::new(1, 2); + + vst1q_s64(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_u8() { + let mut vals = [0_u8; 9]; + let a = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + + vst1_u8(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); + assert_eq!(vals[5], 5); + assert_eq!(vals[6], 6); + assert_eq!(vals[7], 7); + assert_eq!(vals[8], 8); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_u8() { + let mut vals = [0_u8; 17]; + let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + + vst1q_u8(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); + assert_eq!(vals[5], 5); + assert_eq!(vals[6], 6); + assert_eq!(vals[7], 7); + assert_eq!(vals[8], 8); + assert_eq!(vals[9], 9); + assert_eq!(vals[10], 10); + assert_eq!(vals[11], 11); + assert_eq!(vals[12], 12); + assert_eq!(vals[13], 13); + assert_eq!(vals[14], 14); + assert_eq!(vals[15], 15); + assert_eq!(vals[16], 16); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_u16() { + let mut vals = [0_u16; 5]; + let a = u16x4::new(1, 2, 3, 4); + + vst1_u16(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_u16() { + let mut vals = [0_u16; 9]; + let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + + vst1q_u16(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); + assert_eq!(vals[5], 5); + assert_eq!(vals[6], 6); + assert_eq!(vals[7], 7); + assert_eq!(vals[8], 8); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_u32() { + let mut vals = [0_u32; 3]; + let a = u32x2::new(1, 2); + + vst1_u32(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_u32() { + let mut vals = [0_u32; 5]; + let a = u32x4::new(1, 2, 3, 4); + + vst1q_u32(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_u64() { + let mut vals = [0_u64; 2]; + let a = u64x1::new(1); + + vst1_u64(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_u64() { + let mut vals = [0_u64; 3]; + let a = u64x2::new(1, 2); + + vst1q_u64(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_p8() { + let mut vals = [0_u8; 9]; + let a = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + + vst1_p8(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); + assert_eq!(vals[5], 5); + assert_eq!(vals[6], 6); + assert_eq!(vals[7], 7); + assert_eq!(vals[8], 8); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_p8() { + let mut vals = [0_u8; 17]; + let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + + vst1q_p8(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); + assert_eq!(vals[5], 5); + assert_eq!(vals[6], 6); + assert_eq!(vals[7], 7); + assert_eq!(vals[8], 8); + assert_eq!(vals[9], 9); + assert_eq!(vals[10], 10); + assert_eq!(vals[11], 11); + assert_eq!(vals[12], 12); + assert_eq!(vals[13], 13); + assert_eq!(vals[14], 14); + assert_eq!(vals[15], 15); + assert_eq!(vals[16], 16); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_p16() { + let mut vals = [0_u16; 5]; + let a = u16x4::new(1, 2, 3, 4); + + vst1_p16(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_p16() { + let mut vals = [0_u16; 9]; + let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + + vst1q_p16(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); + assert_eq!(vals[5], 5); + assert_eq!(vals[6], 6); + assert_eq!(vals[7], 7); + assert_eq!(vals[8], 8); +} + +#[simd_test(enable = "neon,aes")] +unsafe fn test_vst1_p64() { + let mut vals = [0_u64; 2]; + let a = u64x1::new(1); + + vst1_p64(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); +} + +#[simd_test(enable = "neon,aes")] +unsafe fn test_vst1q_p64() { + let mut vals = [0_u64; 3]; + let a = u64x2::new(1, 2); + + vst1q_p64(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_f32() { + let mut vals = [0_f32; 3]; + let a = f32x2::new(1., 2.); + + vst1_f32(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0.); + assert_eq!(vals[1], 1.); + assert_eq!(vals[2], 2.); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_f32() { + let mut vals = [0_f32; 5]; + let a = f32x4::new(1., 2., 3., 4.); + + vst1q_f32(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0.); + assert_eq!(vals[1], 1.); + assert_eq!(vals[2], 2.); + assert_eq!(vals[3], 3.); + assert_eq!(vals[4], 4.); +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/arm/table_lookup_tests.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/table_lookup_tests.rs similarity index 100% rename from crux-mir/lib/stdarch/crates/core_arch/src/arm/table_lookup_tests.rs rename to crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/neon/table_lookup_tests.rs diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/acle/registers/aarch32.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/registers/aarch32.rs similarity index 100% rename from crux-mir/lib/stdarch/crates/core_arch/src/acle/registers/aarch32.rs rename to crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/registers/aarch32.rs diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/acle/registers/mod.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/registers/mod.rs similarity index 83% rename from crux-mir/lib/stdarch/crates/core_arch/src/acle/registers/mod.rs rename to crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/registers/mod.rs index 73fcc2c7b..621efe2f5 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/acle/registers/mod.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/registers/mod.rs @@ -4,7 +4,7 @@ macro_rules! rsr { impl super::super::sealed::Rsr for $R { unsafe fn __rsr(&self) -> u32 { let r: u32; - asm!(concat!("mrs $0,", stringify!($R)) : "=r"(r) : : : "volatile"); + crate::arch::asm!(concat!("mrs {},", stringify!($R)), out(reg) r, options(nomem, nostack)); r } } @@ -17,7 +17,7 @@ macro_rules! rsrp { impl super::super::sealed::Rsrp for $R { unsafe fn __rsrp(&self) -> *const u8 { let r: *const u8; - asm!(concat!("mrs $0,", stringify!($R)) : "=r"(r) : : : "volatile"); + crate::arch::asm!(concat!("mrs {},", stringify!($R)), out(reg) r, options(nomem, nostack)); r } } @@ -29,7 +29,7 @@ macro_rules! wsr { ($R:ident) => { impl super::super::sealed::Wsr for $R { unsafe fn __wsr(&self, value: u32) { - asm!(concat!("msr ", stringify!($R), ",$0") : : "r"(value) : : "volatile"); + crate::arch::asm!(concat!("msr ", stringify!($R), ", {}"), in(reg) value, options(nomem, nostack)); } } }; @@ -40,7 +40,7 @@ macro_rules! wsrp { ($R:ident) => { impl super::super::sealed::Wsrp for $R { unsafe fn __wsrp(&self, value: *const u8) { - asm!(concat!("msr ", stringify!($R), ",$0") : : "r"(value) : : "volatile"); + crate::arch::asm!(concat!("msr ", stringify!($R), ", {}"), in(reg) value, options(nomem, nostack)); } } }; diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/acle/registers/v6m.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/registers/v6m.rs similarity index 100% rename from crux-mir/lib/stdarch/crates/core_arch/src/acle/registers/v6m.rs rename to crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/registers/v6m.rs diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/acle/registers/v7m.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/registers/v7m.rs similarity index 100% rename from crux-mir/lib/stdarch/crates/core_arch/src/acle/registers/v7m.rs rename to crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/registers/v7m.rs diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/test_support.rs b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/test_support.rs new file mode 100644 index 000000000..ff752f25b --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/arm_shared/test_support.rs @@ -0,0 +1,836 @@ +#[cfg(target_arch = "arm")] +use crate::core_arch::arm::*; + +#[cfg(target_arch = "aarch64")] +use crate::core_arch::aarch64::*; + +use crate::core_arch::simd::*; +use std::{i16, i32, i8, mem::transmute, u16, u32, u8, vec::Vec}; + +macro_rules! V_u8 { + () => { + vec![0x00u8, 0x01u8, 0x02u8, 0x0Fu8, 0x80u8, 0xF0u8, 0xFFu8] + }; +} +macro_rules! V_u16 { + () => { + vec![ + 0x0000u16, 0x0101u16, 0x0202u16, 0x0F0Fu16, 0x8000u16, 0xF0F0u16, 0xFFFFu16, + ] + }; +} +macro_rules! V_u32 { + () => { + vec![ + 0x00000000u32, + 0x01010101u32, + 0x02020202u32, + 0x0F0F0F0Fu32, + 0x80000000u32, + 0xF0F0F0F0u32, + 0xFFFFFFFFu32, + ] + }; +} +macro_rules! V_u64 { + () => { + vec![ + 0x0000000000000000u64, + 0x0101010101010101u64, + 0x0202020202020202u64, + 0x0F0F0F0F0F0F0F0Fu64, + 0x8080808080808080u64, + 0xF0F0F0F0F0F0F0F0u64, + 0xFFFFFFFFFFFFFFFFu64, + ] + }; +} + +macro_rules! V_i8 { + () => { + vec![ + 0x00i8, 0x01i8, 0x02i8, 0x0Fi8, -128i8, /* 0x80 */ + -16i8, /* 0xF0 */ + -1i8, /* 0xFF */ + ] + }; +} +macro_rules! V_i16 { + () => { + vec![ + 0x0000i16, 0x0101i16, 0x0202i16, 0x0F0Fi16, -32768i16, /* 0x8000 */ + -3856i16, /* 0xF0F0 */ + -1i16, /* 0xFFF */ + ] + }; +} +macro_rules! V_i32 { + () => { + vec![ + 0x00000000i32, + 0x01010101i32, + 0x02020202i32, + 0x0F0F0F0Fi32, + -2139062144i32, /* 0x80000000 */ + -252645136i32, /* 0xF0F0F0F0 */ + -1i32, /* 0xFFFFFFFF */ + ] + }; +} + +macro_rules! V_i64 { + () => { + vec![ + 0x0000000000000000i64, + 0x0101010101010101i64, + 0x0202020202020202i64, + 0x0F0F0F0F0F0F0F0Fi64, + -9223372036854775808i64, /* 0x8000000000000000 */ + -1152921504606846976i64, /* 0xF000000000000000 */ + -1i64, /* 0xFFFFFFFFFFFFFFFF */ + ] + }; +} + +macro_rules! V_f32 { + () => { + vec![ + 0.0f32, + 1.0f32, + -1.0f32, + 1.2f32, + 2.4f32, + std::f32::MAX, + std::f32::MIN, + std::f32::INFINITY, + std::f32::NEG_INFINITY, + std::f32::NAN, + ] + }; +} + +macro_rules! to64 { + ($t : ident) => { + |v: $t| -> u64 { transmute(v) } + }; +} + +macro_rules! to128 { + ($t : ident) => { + |v: $t| -> u128 { transmute(v) } + }; +} + +pub(crate) fn test( + vals: Vec, + fill1: fn(T) -> V, + fill2: fn(U) -> W, + cast: fn(W) -> X, + test_fun: fn(V, V) -> W, + verify_fun: fn(T, T) -> U, +) where + T: Copy + core::fmt::Debug + std::cmp::PartialEq, + U: Copy + core::fmt::Debug + std::cmp::PartialEq, + V: Copy + core::fmt::Debug, + W: Copy + core::fmt::Debug, + X: Copy + core::fmt::Debug + std::cmp::PartialEq, +{ + let pairs = vals.iter().zip(vals.iter()); + + for (i, j) in pairs { + let a: V = fill1(*i); + let b: V = fill1(*j); + + let actual_pre: W = test_fun(a, b); + let expected_pre: W = fill2(verify_fun(*i, *j)); + + let actual: X = cast(actual_pre); + let expected: X = cast(expected_pre); + + assert_eq!( + actual, expected, + "[{:?}:{:?}] :\nf({:?}, {:?}) = {:?}\ng({:?}, {:?}) = {:?}\n", + *i, *j, &a, &b, actual_pre, &a, &b, expected_pre + ); + } +} + +macro_rules! gen_test_fn { + ($n: ident, $t: ident, $u: ident, $v: ident, $w: ident, $x: ident, $vals: expr, $fill1: expr, $fill2: expr, $cast: expr) => { + pub(crate) fn $n(test_fun: fn($v, $v) -> $w, verify_fun: fn($t, $t) -> $u) { + unsafe { + test::<$t, $u, $v, $w, $x>($vals, $fill1, $fill2, $cast, test_fun, verify_fun) + }; + } + }; +} + +macro_rules! gen_fill_fn { + ($id: ident, $el_width: expr, $num_els: expr, $in_t : ident, $out_t: ident, $cmp_t: ident) => { + pub(crate) fn $id(val: $in_t) -> $out_t { + let initial: [$in_t; $num_els] = [val; $num_els]; + let result: $cmp_t = unsafe { transmute(initial) }; + let result_out: $out_t = unsafe { transmute(result) }; + + // println!("FILL: {:016x} as {} x {}: {:016x}", val.reverse_bits(), $el_width, $num_els, (result as u64).reverse_bits()); + + result_out + } + }; +} + +gen_fill_fn!(fill_u8, 8, 8, u8, uint8x8_t, u64); +gen_fill_fn!(fill_s8, 8, 8, i8, int8x8_t, u64); +gen_fill_fn!(fillq_u8, 8, 16, u8, uint8x16_t, u128); +gen_fill_fn!(fillq_s8, 8, 16, i8, int8x16_t, u128); + +gen_fill_fn!(fill_u16, 16, 4, u16, uint16x4_t, u64); +gen_fill_fn!(fill_s16, 16, 4, i16, int16x4_t, u64); +gen_fill_fn!(fillq_u16, 16, 8, u16, uint16x8_t, u128); +gen_fill_fn!(fillq_s16, 16, 8, i16, int16x8_t, u128); + +gen_fill_fn!(fill_u32, 32, 2, u32, uint32x2_t, u64); +gen_fill_fn!(fill_s32, 32, 2, i32, int32x2_t, u64); +gen_fill_fn!(fillq_u32, 32, 4, u32, uint32x4_t, u128); +gen_fill_fn!(fillq_s32, 32, 4, i32, int32x4_t, u128); + +gen_fill_fn!(fill_u64, 64, 1, u64, uint64x1_t, u64); +gen_fill_fn!(fill_s64, 64, 1, i64, int64x1_t, u64); +gen_fill_fn!(fillq_u64, 64, 2, u64, uint64x2_t, u128); +gen_fill_fn!(fillq_s64, 64, 2, i64, int64x2_t, u128); + +gen_fill_fn!(fill_f32, 32, 2, f32, float32x2_t, u64); +gen_fill_fn!(fillq_f32, 32, 4, f32, float32x4_t, u128); + +gen_test_fn!( + test_ari_u8, + u8, + u8, + uint8x8_t, + uint8x8_t, + u64, + V_u8!(), + fill_u8, + fill_u8, + to64!(uint8x8_t) +); +gen_test_fn!( + test_bit_u8, + u8, + u8, + uint8x8_t, + uint8x8_t, + u64, + V_u8!(), + fill_u8, + fill_u8, + to64!(uint8x8_t) +); +gen_test_fn!( + test_cmp_u8, + u8, + u8, + uint8x8_t, + uint8x8_t, + u64, + V_u8!(), + fill_u8, + fill_u8, + to64!(uint8x8_t) +); +gen_test_fn!( + testq_ari_u8, + u8, + u8, + uint8x16_t, + uint8x16_t, + u128, + V_u8!(), + fillq_u8, + fillq_u8, + to128!(uint8x16_t) +); +gen_test_fn!( + testq_bit_u8, + u8, + u8, + uint8x16_t, + uint8x16_t, + u128, + V_u8!(), + fillq_u8, + fillq_u8, + to128!(uint8x16_t) +); +gen_test_fn!( + testq_cmp_u8, + u8, + u8, + uint8x16_t, + uint8x16_t, + u128, + V_u8!(), + fillq_u8, + fillq_u8, + to128!(uint8x16_t) +); + +gen_test_fn!( + test_ari_s8, + i8, + i8, + int8x8_t, + int8x8_t, + u64, + V_i8!(), + fill_s8, + fill_s8, + to64!(int8x8_t) +); +gen_test_fn!( + test_bit_s8, + i8, + i8, + int8x8_t, + int8x8_t, + u64, + V_i8!(), + fill_s8, + fill_s8, + to64!(int8x8_t) +); +gen_test_fn!( + test_cmp_s8, + i8, + u8, + int8x8_t, + uint8x8_t, + u64, + V_i8!(), + fill_s8, + fill_u8, + to64!(uint8x8_t) +); +gen_test_fn!( + testq_ari_s8, + i8, + i8, + int8x16_t, + int8x16_t, + u128, + V_i8!(), + fillq_s8, + fillq_s8, + to128!(int8x16_t) +); +gen_test_fn!( + testq_bit_s8, + i8, + i8, + int8x16_t, + int8x16_t, + u128, + V_i8!(), + fillq_s8, + fillq_s8, + to128!(int8x16_t) +); +gen_test_fn!( + testq_cmp_s8, + i8, + u8, + int8x16_t, + uint8x16_t, + u128, + V_i8!(), + fillq_s8, + fillq_u8, + to128!(uint8x16_t) +); + +gen_test_fn!( + test_ari_u16, + u16, + u16, + uint16x4_t, + uint16x4_t, + u64, + V_u16!(), + fill_u16, + fill_u16, + to64!(uint16x4_t) +); +gen_test_fn!( + test_bit_u16, + u16, + u16, + uint16x4_t, + uint16x4_t, + u64, + V_u16!(), + fill_u16, + fill_u16, + to64!(uint16x4_t) +); +gen_test_fn!( + test_cmp_u16, + u16, + u16, + uint16x4_t, + uint16x4_t, + u64, + V_u16!(), + fill_u16, + fill_u16, + to64!(uint16x4_t) +); +gen_test_fn!( + testq_ari_u16, + u16, + u16, + uint16x8_t, + uint16x8_t, + u128, + V_u16!(), + fillq_u16, + fillq_u16, + to128!(uint16x8_t) +); +gen_test_fn!( + testq_bit_u16, + u16, + u16, + uint16x8_t, + uint16x8_t, + u128, + V_u16!(), + fillq_u16, + fillq_u16, + to128!(uint16x8_t) +); +gen_test_fn!( + testq_cmp_u16, + u16, + u16, + uint16x8_t, + uint16x8_t, + u128, + V_u16!(), + fillq_u16, + fillq_u16, + to128!(uint16x8_t) +); + +gen_test_fn!( + test_ari_s16, + i16, + i16, + int16x4_t, + int16x4_t, + u64, + V_i16!(), + fill_s16, + fill_s16, + to64!(int16x4_t) +); +gen_test_fn!( + test_bit_s16, + i16, + i16, + int16x4_t, + int16x4_t, + u64, + V_i16!(), + fill_s16, + fill_s16, + to64!(int16x4_t) +); +gen_test_fn!( + test_cmp_s16, + i16, + u16, + int16x4_t, + uint16x4_t, + u64, + V_i16!(), + fill_s16, + fill_u16, + to64!(uint16x4_t) +); +gen_test_fn!( + testq_ari_s16, + i16, + i16, + int16x8_t, + int16x8_t, + u128, + V_i16!(), + fillq_s16, + fillq_s16, + to128!(int16x8_t) +); +gen_test_fn!( + testq_bit_s16, + i16, + i16, + int16x8_t, + int16x8_t, + u128, + V_i16!(), + fillq_s16, + fillq_s16, + to128!(int16x8_t) +); +gen_test_fn!( + testq_cmp_s16, + i16, + u16, + int16x8_t, + uint16x8_t, + u128, + V_i16!(), + fillq_s16, + fillq_u16, + to128!(uint16x8_t) +); + +gen_test_fn!( + test_ari_u32, + u32, + u32, + uint32x2_t, + uint32x2_t, + u64, + V_u32!(), + fill_u32, + fill_u32, + to64!(uint32x2_t) +); +gen_test_fn!( + test_bit_u32, + u32, + u32, + uint32x2_t, + uint32x2_t, + u64, + V_u32!(), + fill_u32, + fill_u32, + to64!(uint32x2_t) +); +gen_test_fn!( + test_cmp_u32, + u32, + u32, + uint32x2_t, + uint32x2_t, + u64, + V_u32!(), + fill_u32, + fill_u32, + to64!(uint32x2_t) +); +gen_test_fn!( + testq_ari_u32, + u32, + u32, + uint32x4_t, + uint32x4_t, + u128, + V_u32!(), + fillq_u32, + fillq_u32, + to128!(uint32x4_t) +); +gen_test_fn!( + testq_bit_u32, + u32, + u32, + uint32x4_t, + uint32x4_t, + u128, + V_u32!(), + fillq_u32, + fillq_u32, + to128!(uint32x4_t) +); +gen_test_fn!( + testq_cmp_u32, + u32, + u32, + uint32x4_t, + uint32x4_t, + u128, + V_u32!(), + fillq_u32, + fillq_u32, + to128!(uint32x4_t) +); + +gen_test_fn!( + test_ari_s32, + i32, + i32, + int32x2_t, + int32x2_t, + u64, + V_i32!(), + fill_s32, + fill_s32, + to64!(int32x2_t) +); +gen_test_fn!( + test_bit_s32, + i32, + i32, + int32x2_t, + int32x2_t, + u64, + V_i32!(), + fill_s32, + fill_s32, + to64!(int32x2_t) +); +gen_test_fn!( + test_cmp_s32, + i32, + u32, + int32x2_t, + uint32x2_t, + u64, + V_i32!(), + fill_s32, + fill_u32, + to64!(uint32x2_t) +); +gen_test_fn!( + testq_ari_s32, + i32, + i32, + int32x4_t, + int32x4_t, + u128, + V_i32!(), + fillq_s32, + fillq_s32, + to128!(int32x4_t) +); +gen_test_fn!( + testq_bit_s32, + i32, + i32, + int32x4_t, + int32x4_t, + u128, + V_i32!(), + fillq_s32, + fillq_s32, + to128!(int32x4_t) +); +gen_test_fn!( + testq_cmp_s32, + i32, + u32, + int32x4_t, + uint32x4_t, + u128, + V_i32!(), + fillq_s32, + fillq_u32, + to128!(uint32x4_t) +); + +gen_test_fn!( + test_ari_u64, + u64, + u64, + uint64x1_t, + uint64x1_t, + u64, + V_u64!(), + fill_u64, + fill_u64, + to64!(uint64x1_t) +); +gen_test_fn!( + test_bit_u64, + u64, + u64, + uint64x1_t, + uint64x1_t, + u64, + V_u64!(), + fill_u64, + fill_u64, + to64!(uint64x1_t) +); +gen_test_fn!( + test_cmp_u64, + u64, + u64, + uint64x1_t, + uint64x1_t, + u64, + V_u64!(), + fill_u64, + fill_u64, + to64!(uint64x1_t) +); +gen_test_fn!( + testq_ari_u64, + u64, + u64, + uint64x2_t, + uint64x2_t, + u128, + V_u64!(), + fillq_u64, + fillq_u64, + to128!(uint64x2_t) +); +gen_test_fn!( + testq_bit_u64, + u64, + u64, + uint64x2_t, + uint64x2_t, + u128, + V_u64!(), + fillq_u64, + fillq_u64, + to128!(uint64x2_t) +); +gen_test_fn!( + testq_cmp_u64, + u64, + u64, + uint64x2_t, + uint64x2_t, + u128, + V_u64!(), + fillq_u64, + fillq_u64, + to128!(uint64x2_t) +); + +gen_test_fn!( + test_ari_s64, + i64, + i64, + int64x1_t, + int64x1_t, + u64, + V_i64!(), + fill_s64, + fill_s64, + to64!(int64x1_t) +); +gen_test_fn!( + test_bit_s64, + i64, + i64, + int64x1_t, + int64x1_t, + u64, + V_i64!(), + fill_s64, + fill_s64, + to64!(int64x1_t) +); +gen_test_fn!( + test_cmp_s64, + i64, + u64, + int64x1_t, + uint64x1_t, + u64, + V_i64!(), + fill_s64, + fill_u64, + to64!(uint64x1_t) +); +gen_test_fn!( + testq_ari_s64, + i64, + i64, + int64x2_t, + int64x2_t, + u128, + V_i64!(), + fillq_s64, + fillq_s64, + to128!(int64x2_t) +); +gen_test_fn!( + testq_bit_s64, + i64, + i64, + int64x2_t, + int64x2_t, + u128, + V_i64!(), + fillq_s64, + fillq_s64, + to128!(int64x2_t) +); +gen_test_fn!( + testq_cmp_s64, + i64, + u64, + int64x2_t, + uint64x2_t, + u128, + V_i64!(), + fillq_s64, + fillq_u64, + to128!(uint64x2_t) +); + +gen_test_fn!( + test_ari_f32, + f32, + f32, + float32x2_t, + float32x2_t, + u64, + V_f32!(), + fill_f32, + fill_f32, + to64!(float32x2_t) +); +gen_test_fn!( + test_cmp_f32, + f32, + u32, + float32x2_t, + uint32x2_t, + u64, + V_f32!(), + fill_f32, + fill_u32, + to64!(uint32x2_t) +); +gen_test_fn!( + testq_ari_f32, + f32, + f32, + float32x4_t, + float32x4_t, + u128, + V_f32!(), + fillq_f32, + fillq_f32, + to128!(float32x4_t) +); +gen_test_fn!( + testq_cmp_f32, + f32, + u32, + float32x4_t, + uint32x4_t, + u128, + V_f32!(), + fillq_f32, + fillq_u32, + to128!(uint32x4_t) +); diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/core_arch_docs.md b/crux-mir/lib/stdarch/crates/core_arch/src/core_arch_docs.md index 35f05f33a..eddd1fc0c 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/core_arch_docs.md +++ b/crux-mir/lib/stdarch/crates/core_arch/src/core_arch_docs.md @@ -185,6 +185,8 @@ others at: * [`x86_64`] * [`arm`] * [`aarch64`] +* [`riscv32`] +* [`riscv64`] * [`mips`] * [`mips64`] * [`powerpc`] @@ -192,16 +194,18 @@ others at: * [`nvptx`] * [`wasm32`] -[`x86`]: x86/index.html -[`x86_64`]: x86_64/index.html -[`arm`]: arm/index.html -[`aarch64`]: aarch64/index.html -[`mips`]: mips/index.html -[`mips64`]: mips64/index.html -[`powerpc`]: powerpc/index.html -[`powerpc64`]: powerpc64/index.html -[`nvptx`]: nvptx/index.html -[`wasm32`]: wasm32/index.html +[`x86`]: ../../core/arch/x86/index.html +[`x86_64`]: ../../core/arch/x86_64/index.html +[`arm`]: ../../core/arch/arm/index.html +[`aarch64`]: ../../core/arch/aarch64/index.html +[`riscv32`]: ../../core/arch/riscv32/index.html +[`riscv64`]: ../../core/arch/riscv64/index.html +[`mips`]: ../../core/arch/mips/index.html +[`mips64`]: ../../core/arch/mips64/index.html +[`powerpc`]: ../../core/arch/powerpc/index.html +[`powerpc64`]: ../../core/arch/powerpc64/index.html +[`nvptx`]: ../../core/arch/nvptx/index.html +[`wasm32`]: ../../core/arch/wasm32/index.html # Examples @@ -210,12 +214,6 @@ using LLVM's auto-vectorization to produce optimized vectorized code for AVX2 and also for the default platform. ```rust -# #![cfg_attr(not(dox),feature(stdsimd))] -# #[allow(unused_imports)] -# #[cfg(not(dox))] -# #[macro_use(is_x86_feature_detected)] -# extern crate std_detect; - fn main() { let mut dst = [0]; add_quickly(&[1], &[2], &mut dst); @@ -280,7 +278,7 @@ pub fn hex_encode(src: &[u8], dst: &mut [u8]) { } // translated from -// https://github.com/Matherunner/bin2hex-sse/blob/master/base16_sse4.cpp +// #[target_feature(enable = "sse4.1")] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] unsafe fn hex_encode_sse41(mut src: &[u8], dst: &mut [u8]) { diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/lib.rs b/crux-mir/lib/stdarch/crates/core_arch/src/lib.rs index 3be585967..5a9727a0a 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/lib.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/lib.rs @@ -1,15 +1,14 @@ -#![doc(include = "core_arch_docs.md")] +#![doc = include_str!("core_arch_docs.md")] +#![allow(improper_ctypes_definitions)] #![allow(dead_code)] #![allow(unused_features)] +#![deny(rust_2018_idioms)] #![feature( - const_fn, - const_fn_union, custom_inner_attributes, link_llvm_intrinsics, platform_intrinsics, repr_simd, simd_ffi, - asm, proc_macro_hygiene, stmt_expr_attributes, core_intrinsics, @@ -18,23 +17,24 @@ stdsimd, staged_api, doc_cfg, - mmx_target_feature, tbm_target_feature, sse4a_target_feature, + riscv_target_feature, arm_target_feature, - aarch64_target_feature, cmpxchg16b_target_feature, avx512_target_feature, mips_target_feature, powerpc_target_feature, wasm_target_feature, abi_unadjusted, - adx_target_feature, rtm_target_feature, f16c_target_feature, - external_doc + allow_internal_unstable, + decl_macro, + asm_const, + target_feature_11 )] -#![cfg_attr(test, feature(test, abi_vectorcall, untagged_unions))] +#![cfg_attr(test, feature(test, abi_vectorcall))] #![deny(clippy::missing_inline_in_public_items)] #![allow( clippy::inline_always, @@ -63,16 +63,13 @@ extern crate std; #[cfg(test)] #[macro_use] extern crate std_detect; -#[cfg(test)] -extern crate stdarch_test; - -#[cfg(all(test, target_arch = "wasm32"))] -extern crate wasm_bindgen_test; - #[path = "mod.rs"] mod core_arch; -pub use self::core_arch::arch::*; +pub mod arch { + pub use crate::core_arch::arch::*; + pub use core::arch::asm; +} #[allow(unused_imports)] -use core::{ffi, hint, intrinsics, marker, mem, ops, ptr, sync}; +use core::{convert, ffi, hint, intrinsics, marker, mem, ops, ptr, sync}; diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/macros.rs b/crux-mir/lib/stdarch/crates/core_arch/src/macros.rs index 00d369d99..1c917c52b 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/macros.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/macros.rs @@ -1,352 +1,81 @@ //! Utility macros. -#[allow(unused)] -macro_rules! constify_imm8 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1111_1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - 15 => $expand!(15), - 16 => $expand!(16), - 17 => $expand!(17), - 18 => $expand!(18), - 19 => $expand!(19), - 20 => $expand!(20), - 21 => $expand!(21), - 22 => $expand!(22), - 23 => $expand!(23), - 24 => $expand!(24), - 25 => $expand!(25), - 26 => $expand!(26), - 27 => $expand!(27), - 28 => $expand!(28), - 29 => $expand!(29), - 30 => $expand!(30), - 31 => $expand!(31), - 32 => $expand!(32), - 33 => $expand!(33), - 34 => $expand!(34), - 35 => $expand!(35), - 36 => $expand!(36), - 37 => $expand!(37), - 38 => $expand!(38), - 39 => $expand!(39), - 40 => $expand!(40), - 41 => $expand!(41), - 42 => $expand!(42), - 43 => $expand!(43), - 44 => $expand!(44), - 45 => $expand!(45), - 46 => $expand!(46), - 47 => $expand!(47), - 48 => $expand!(48), - 49 => $expand!(49), - 50 => $expand!(50), - 51 => $expand!(51), - 52 => $expand!(52), - 53 => $expand!(53), - 54 => $expand!(54), - 55 => $expand!(55), - 56 => $expand!(56), - 57 => $expand!(57), - 58 => $expand!(58), - 59 => $expand!(59), - 60 => $expand!(60), - 61 => $expand!(61), - 62 => $expand!(62), - 63 => $expand!(63), - 64 => $expand!(64), - 65 => $expand!(65), - 66 => $expand!(66), - 67 => $expand!(67), - 68 => $expand!(68), - 69 => $expand!(69), - 70 => $expand!(70), - 71 => $expand!(71), - 72 => $expand!(72), - 73 => $expand!(73), - 74 => $expand!(74), - 75 => $expand!(75), - 76 => $expand!(76), - 77 => $expand!(77), - 78 => $expand!(78), - 79 => $expand!(79), - 80 => $expand!(80), - 81 => $expand!(81), - 82 => $expand!(82), - 83 => $expand!(83), - 84 => $expand!(84), - 85 => $expand!(85), - 86 => $expand!(86), - 87 => $expand!(87), - 88 => $expand!(88), - 89 => $expand!(89), - 90 => $expand!(90), - 91 => $expand!(91), - 92 => $expand!(92), - 93 => $expand!(93), - 94 => $expand!(94), - 95 => $expand!(95), - 96 => $expand!(96), - 97 => $expand!(97), - 98 => $expand!(98), - 99 => $expand!(99), - 100 => $expand!(100), - 101 => $expand!(101), - 102 => $expand!(102), - 103 => $expand!(103), - 104 => $expand!(104), - 105 => $expand!(105), - 106 => $expand!(106), - 107 => $expand!(107), - 108 => $expand!(108), - 109 => $expand!(109), - 110 => $expand!(110), - 111 => $expand!(111), - 112 => $expand!(112), - 113 => $expand!(113), - 114 => $expand!(114), - 115 => $expand!(115), - 116 => $expand!(116), - 117 => $expand!(117), - 118 => $expand!(118), - 119 => $expand!(119), - 120 => $expand!(120), - 121 => $expand!(121), - 122 => $expand!(122), - 123 => $expand!(123), - 124 => $expand!(124), - 125 => $expand!(125), - 126 => $expand!(126), - 127 => $expand!(127), - 128 => $expand!(128), - 129 => $expand!(129), - 130 => $expand!(130), - 131 => $expand!(131), - 132 => $expand!(132), - 133 => $expand!(133), - 134 => $expand!(134), - 135 => $expand!(135), - 136 => $expand!(136), - 137 => $expand!(137), - 138 => $expand!(138), - 139 => $expand!(139), - 140 => $expand!(140), - 141 => $expand!(141), - 142 => $expand!(142), - 143 => $expand!(143), - 144 => $expand!(144), - 145 => $expand!(145), - 146 => $expand!(146), - 147 => $expand!(147), - 148 => $expand!(148), - 149 => $expand!(149), - 150 => $expand!(150), - 151 => $expand!(151), - 152 => $expand!(152), - 153 => $expand!(153), - 154 => $expand!(154), - 155 => $expand!(155), - 156 => $expand!(156), - 157 => $expand!(157), - 158 => $expand!(158), - 159 => $expand!(159), - 160 => $expand!(160), - 161 => $expand!(161), - 162 => $expand!(162), - 163 => $expand!(163), - 164 => $expand!(164), - 165 => $expand!(165), - 166 => $expand!(166), - 167 => $expand!(167), - 168 => $expand!(168), - 169 => $expand!(169), - 170 => $expand!(170), - 171 => $expand!(171), - 172 => $expand!(172), - 173 => $expand!(173), - 174 => $expand!(174), - 175 => $expand!(175), - 176 => $expand!(176), - 177 => $expand!(177), - 178 => $expand!(178), - 179 => $expand!(179), - 180 => $expand!(180), - 181 => $expand!(181), - 182 => $expand!(182), - 183 => $expand!(183), - 184 => $expand!(184), - 185 => $expand!(185), - 186 => $expand!(186), - 187 => $expand!(187), - 188 => $expand!(188), - 189 => $expand!(189), - 190 => $expand!(190), - 191 => $expand!(191), - 192 => $expand!(192), - 193 => $expand!(193), - 194 => $expand!(194), - 195 => $expand!(195), - 196 => $expand!(196), - 197 => $expand!(197), - 198 => $expand!(198), - 199 => $expand!(199), - 200 => $expand!(200), - 201 => $expand!(201), - 202 => $expand!(202), - 203 => $expand!(203), - 204 => $expand!(204), - 205 => $expand!(205), - 206 => $expand!(206), - 207 => $expand!(207), - 208 => $expand!(208), - 209 => $expand!(209), - 210 => $expand!(210), - 211 => $expand!(211), - 212 => $expand!(212), - 213 => $expand!(213), - 214 => $expand!(214), - 215 => $expand!(215), - 216 => $expand!(216), - 217 => $expand!(217), - 218 => $expand!(218), - 219 => $expand!(219), - 220 => $expand!(220), - 221 => $expand!(221), - 222 => $expand!(222), - 223 => $expand!(223), - 224 => $expand!(224), - 225 => $expand!(225), - 226 => $expand!(226), - 227 => $expand!(227), - 228 => $expand!(228), - 229 => $expand!(229), - 230 => $expand!(230), - 231 => $expand!(231), - 232 => $expand!(232), - 233 => $expand!(233), - 234 => $expand!(234), - 235 => $expand!(235), - 236 => $expand!(236), - 237 => $expand!(237), - 238 => $expand!(238), - 239 => $expand!(239), - 240 => $expand!(240), - 241 => $expand!(241), - 242 => $expand!(242), - 243 => $expand!(243), - 244 => $expand!(244), - 245 => $expand!(245), - 246 => $expand!(246), - 247 => $expand!(247), - 248 => $expand!(248), - 249 => $expand!(249), - 250 => $expand!(250), - 251 => $expand!(251), - 252 => $expand!(252), - 253 => $expand!(253), - 254 => $expand!(254), - _ => $expand!(255), - } +// Helper struct used to trigger const eval errors when the const generic immediate value `IMM` is +// out of `[MIN-MAX]` range. +pub(crate) struct ValidateConstImm; +impl ValidateConstImm { + pub(crate) const VALID: () = { + assert!(IMM >= MIN && IMM <= MAX, "IMM value not in expected range"); }; } -//immediate value: 0:31 -#[allow(unused)] -macro_rules! constify_imm5 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1_1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - 15 => $expand!(15), - 16 => $expand!(16), - 17 => $expand!(17), - 18 => $expand!(18), - 19 => $expand!(19), - 20 => $expand!(20), - 21 => $expand!(21), - 22 => $expand!(22), - 23 => $expand!(23), - 24 => $expand!(24), - 25 => $expand!(25), - 26 => $expand!(26), - 27 => $expand!(27), - 28 => $expand!(28), - 29 => $expand!(29), - 30 => $expand!(30), - _ => $expand!(31), - } +#[allow(unused_macros)] +macro_rules! static_assert_imm1 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, 0, { (1 << 1) - 1 }>::VALID; + }; +} + +#[allow(unused_macros)] +macro_rules! static_assert_imm2 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, 0, { (1 << 2) - 1 }>::VALID; + }; +} + +#[allow(unused_macros)] +macro_rules! static_assert_imm3 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, 0, { (1 << 3) - 1 }>::VALID; + }; +} + +#[allow(unused_macros)] +macro_rules! static_assert_imm4 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, 0, { (1 << 4) - 1 }>::VALID; + }; +} + +#[allow(unused_macros)] +macro_rules! static_assert_imm5 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, 0, { (1 << 5) - 1 }>::VALID; + }; +} + +#[allow(unused_macros)] +macro_rules! static_assert_imm6 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, 0, { (1 << 6) - 1 }>::VALID; + }; +} + +#[allow(unused_macros)] +macro_rules! static_assert_imm8 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, 0, { (1 << 8) - 1 }>::VALID; + }; +} + +#[allow(unused_macros)] +macro_rules! static_assert_imm16 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, 0, { (1 << 16) - 1 }>::VALID; }; } -//immediate value: -16:15 #[allow(unused)] -macro_rules! constify_imm5 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1_1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - 15 => $expand!(15), - 16 => $expand!(16), - 17 => $expand!(17), - 18 => $expand!(18), - 19 => $expand!(19), - 20 => $expand!(20), - 21 => $expand!(21), - 22 => $expand!(22), - 23 => $expand!(23), - 24 => $expand!(24), - 25 => $expand!(25), - 26 => $expand!(26), - 27 => $expand!(27), - 28 => $expand!(28), - 29 => $expand!(29), - 30 => $expand!(30), - _ => $expand!(31), +macro_rules! static_assert { + ($imm:ident : $ty:ty where $e:expr) => {{ + struct Validate(); + impl Validate<$imm> { + const VALID: () = { + assert!($e, concat!("Assertion failed: ", stringify!($e))); + }; } - }; + let _ = Validate::<$imm>::VALID; + }}; } #[allow(unused)] @@ -363,3 +92,99 @@ macro_rules! types { pub struct $name($($fields)*); )*) } + +#[allow(unused)] +macro_rules! simd_shuffle2 { + ($x:expr, $y:expr, <$(const $imm:ident : $ty:ty),+ $(,)?> $idx:expr $(,)?) => {{ + struct ConstParam<$(const $imm: $ty),+>; + impl<$(const $imm: $ty),+> ConstParam<$($imm),+> { + const IDX: [u32; 2] = $idx; + } + + simd_shuffle($x, $y, ConstParam::<$($imm),+>::IDX) + }}; + ($x:expr, $y:expr, $idx:expr $(,)?) => {{ + const IDX: [u32; 2] = $idx; + simd_shuffle($x, $y, IDX) + }}; +} + +#[allow(unused_macros)] +macro_rules! simd_shuffle4 { + ($x:expr, $y:expr, <$(const $imm:ident : $ty:ty),+ $(,)?> $idx:expr $(,)?) => {{ + struct ConstParam<$(const $imm: $ty),+>; + impl<$(const $imm: $ty),+> ConstParam<$($imm),+> { + const IDX: [u32; 4] = $idx; + } + + simd_shuffle($x, $y, ConstParam::<$($imm),+>::IDX) + }}; + ($x:expr, $y:expr, $idx:expr $(,)?) => {{ + const IDX: [u32; 4] = $idx; + simd_shuffle($x, $y, IDX) + }}; +} + +#[allow(unused_macros)] +macro_rules! simd_shuffle8 { + ($x:expr, $y:expr, <$(const $imm:ident : $ty:ty),+ $(,)?> $idx:expr $(,)?) => {{ + struct ConstParam<$(const $imm: $ty),+>; + impl<$(const $imm: $ty),+> ConstParam<$($imm),+> { + const IDX: [u32; 8] = $idx; + } + + simd_shuffle($x, $y, ConstParam::<$($imm),+>::IDX) + }}; + ($x:expr, $y:expr, $idx:expr $(,)?) => {{ + const IDX: [u32; 8] = $idx; + simd_shuffle($x, $y, IDX) + }}; +} + +#[allow(unused)] +macro_rules! simd_shuffle16 { + ($x:expr, $y:expr, <$(const $imm:ident : $ty:ty),+ $(,)?> $idx:expr $(,)?) => {{ + struct ConstParam<$(const $imm: $ty),+>; + impl<$(const $imm: $ty),+> ConstParam<$($imm),+> { + const IDX: [u32; 16] = $idx; + } + + simd_shuffle($x, $y, ConstParam::<$($imm),+>::IDX) + }}; + ($x:expr, $y:expr, $idx:expr $(,)?) => {{ + const IDX: [u32; 16] = $idx; + simd_shuffle($x, $y, IDX) + }}; +} + +#[allow(unused_macros)] +macro_rules! simd_shuffle32 { + ($x:expr, $y:expr, <$(const $imm:ident : $ty:ty),+> $(,)? $idx:expr $(,)?) => {{ + struct ConstParam<$(const $imm: $ty),+>; + impl<$(const $imm: $ty),+> ConstParam<$($imm),+> { + const IDX: [u32; 32] = $idx; + } + + simd_shuffle($x, $y, ConstParam::<$($imm),+>::IDX) + }}; + ($x:expr, $y:expr, $idx:expr $(,)?) => {{ + const IDX: [u32; 32] = $idx; + simd_shuffle($x, $y, IDX) + }}; +} + +#[allow(unused_macros)] +macro_rules! simd_shuffle64 { + ($x:expr, $y:expr, <$(const $imm:ident : $ty:ty),+ $(,)?> $idx:expr $(,)?) => {{ + struct ConstParam<$(const $imm: $ty),+>; + impl<$(const $imm: $ty),+> ConstParam<$($imm),+> { + const IDX: [u32; 64] = $idx; + } + + simd_shuffle($x, $y, ConstParam::<$($imm),+>::IDX) + }}; + ($x:expr, $y:expr, $idx:expr $(,)?) => {{ + const IDX: [u32; 64] = $idx; + simd_shuffle($x, $y, IDX) + }}; +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/mips/mod.rs b/crux-mir/lib/stdarch/crates/core_arch/src/mips/mod.rs index efde97f4d..96905aedc 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/mips/mod.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/mips/mod.rs @@ -1,6 +1,10 @@ //! MIPS +// Building this module (even if unused) for non-fp64 targets fails with an LLVM +// error. +#[cfg(target_feature = "fp64")] mod msa; +#[cfg(target_feature = "fp64")] pub use self::msa::*; #[cfg(test)] diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/mips/msa.rs b/crux-mir/lib/stdarch/crates/core_arch/src/mips/msa.rs index 29009b5d7..85ed30d18 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/mips/msa.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/mips/msa.rs @@ -161,13 +161,13 @@ extern "C" { fn msa_aver_s_w(a: v4i32, b: v4i32) -> v4i32; #[link_name = "llvm.mips.aver.s.d"] fn msa_aver_s_d(a: v2i64, b: v2i64) -> v2i64; - #[link_name = "llvm.mips.aver.s.b"] + #[link_name = "llvm.mips.aver.u.b"] fn msa_aver_u_b(a: v16u8, b: v16u8) -> v16u8; - #[link_name = "llvm.mips.aver.s.h"] + #[link_name = "llvm.mips.aver.u.h"] fn msa_aver_u_h(a: v8u16, b: v8u16) -> v8u16; - #[link_name = "llvm.mips.aver.s.w"] + #[link_name = "llvm.mips.aver.u.w"] fn msa_aver_u_w(a: v4u32, b: v4u32) -> v4u32; - #[link_name = "llvm.mips.aver.s.d"] + #[link_name = "llvm.mips.aver.u.d"] fn msa_aver_u_d(a: v2u64, b: v2u64) -> v2u64; #[link_name = "llvm.mips.bclr.b"] fn msa_bclr_b(a: v16u8, b: v16u8) -> v16u8; @@ -415,7 +415,7 @@ extern "C" { fn msa_dpadd_s_w(a: v4i32, b: v8i16, c: v8i16) -> v4i32; #[link_name = "llvm.mips.dpadd.s.d"] fn msa_dpadd_s_d(a: v2i64, b: v4i32, c: v4i32) -> v2i64; - #[link_name = "llvm.mips.dpadd.s.h"] + #[link_name = "llvm.mips.dpadd.u.h"] fn msa_dpadd_u_h(a: v8u16, b: v16u8, c: v16u8) -> v8u16; #[link_name = "llvm.mips.dpadd.u.w"] fn msa_dpadd_u_w(a: v4u32, b: v8u16, c: v8u16) -> v4u32; @@ -498,8 +498,8 @@ extern "C" { fn msa_fexp2_w(a: v4f32, b: v4i32) -> v4f32; #[link_name = "llvm.mips.fexp2.d"] fn msa_fexp2_d(a: v2f64, b: v2i64) -> v2f64; - #[link_name = "llvm.mips.fexupl.w"] // FIXME: 16-bit floats + // #[link_name = "llvm.mips.fexupl.w"] // fn msa_fexupl_w(a: f16x8) -> v4f32; #[link_name = "llvm.mips.fexupl.d"] fn msa_fexupl_d(a: v4f32) -> v2f64; @@ -1411,14 +1411,10 @@ pub unsafe fn __msa_addv_d(a: v2i64, b: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(addvi.b, imm5 = 0b10111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_addvi_b(a: v16i8, imm5: i32) -> v16i8 { - macro_rules! call { - ($imm5:expr) => { - msa_addvi_b(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_addvi_b(a: v16i8) -> v16i8 { + static_assert_imm5!(IMM5); + msa_addvi_b(a, IMM5) } /// Immediate Add @@ -1430,14 +1426,10 @@ pub unsafe fn __msa_addvi_b(a: v16i8, imm5: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(addvi.h, imm5 = 0b10111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_addvi_h(a: v8i16, imm5: i32) -> v8i16 { - macro_rules! call { - ($imm5:expr) => { - msa_addvi_h(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_addvi_h(a: v8i16) -> v8i16 { + static_assert_imm5!(IMM5); + msa_addvi_h(a, IMM5) } /// Immediate Add @@ -1449,14 +1441,10 @@ pub unsafe fn __msa_addvi_h(a: v8i16, imm5: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(addvi.w, imm5 = 0b10111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_addvi_w(a: v4i32, imm5: i32) -> v4i32 { - macro_rules! call { - ($imm5:expr) => { - msa_addvi_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_addvi_w(a: v4i32) -> v4i32 { + static_assert_imm5!(IMM5); + msa_addvi_w(a, IMM5) } /// Immediate Add @@ -1468,14 +1456,10 @@ pub unsafe fn __msa_addvi_w(a: v4i32, imm5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(addvi.d, imm5 = 0b10111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_addvi_d(a: v2i64, imm5: i32) -> v2i64 { - macro_rules! call { - ($imm5:expr) => { - msa_addvi_d(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_addvi_d(a: v2i64) -> v2i64 { + static_assert_imm5!(IMM5); + msa_addvi_d(a, IMM5) } /// Vector Logical And @@ -1501,14 +1485,10 @@ pub unsafe fn __msa_and_v(a: v16u8, b: v16u8) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(andi.b, imm8 = 0b10010111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_andi_b(a: v16u8, imm8: i32) -> v16u8 { - macro_rules! call { - ($imm8:expr) => { - msa_andi_b(a, $imm8) - }; - } - constify_imm8!(imm8, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_andi_b(a: v16u8) -> v16u8 { + static_assert_imm8!(IMM8); + msa_andi_b(a, IMM8) } /// Vector Absolute Values of Signed Subtract @@ -1928,14 +1908,10 @@ pub unsafe fn __msa_bclr_d(a: v2u64, b: v2u64) -> v2u64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bclri.b, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bclri_b(a: v16u8, imm3: i32) -> v16u8 { - macro_rules! call { - ($imm3:expr) => { - msa_bclri_b(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bclri_b(a: v16u8) -> v16u8 { + static_assert_imm3!(IMM3); + msa_bclri_b(a, IMM3) } /// Immediate Bit Clear @@ -1947,14 +1923,10 @@ pub unsafe fn __msa_bclri_b(a: v16u8, imm3: i32) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bclri.h, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bclri_h(a: v8u16, imm4: i32) -> v8u16 { - macro_rules! call { - ($imm4:expr) => { - msa_bclri_h(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bclri_h(a: v8u16) -> v8u16 { + static_assert_imm4!(IMM4); + msa_bclri_h(a, IMM4) } /// Immediate Bit Clear @@ -1966,14 +1938,10 @@ pub unsafe fn __msa_bclri_h(a: v8u16, imm4: i32) -> v8u16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bclri.w, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bclri_w(a: v4u32, imm5: i32) -> v4u32 { - macro_rules! call { - ($imm5:expr) => { - msa_bclri_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bclri_w(a: v4u32) -> v4u32 { + static_assert_imm5!(IMM5); + msa_bclri_w(a, IMM5) } /// Immediate Bit Clear @@ -1985,14 +1953,10 @@ pub unsafe fn __msa_bclri_w(a: v4u32, imm5: i32) -> v4u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bclri.d, imm6 = 0b111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bclri_d(a: v2u64, imm6: i32) -> v2u64 { - macro_rules! call { - ($imm6:expr) => { - msa_bclri_d(a, $imm6) - }; - } - constify_imm6!(imm6, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bclri_d(a: v2u64) -> v2u64 { + static_assert_imm6!(IMM6); + msa_bclri_d(a, IMM6) } /// Vector Bit Insert Left @@ -2060,14 +2024,10 @@ pub unsafe fn __msa_binsl_d(a: v2u64, b: v2u64, c: v2u64) -> v2u64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(binsli.b, imm3 = 0b111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_binsli_b(a: v16u8, b: v16u8, imm3: i32) -> v16u8 { - macro_rules! call { - ($imm3:expr) => { - msa_binsli_b(a, mem::transmute(b), $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_binsli_b(a: v16u8, b: v16u8) -> v16u8 { + static_assert_imm3!(IMM3); + msa_binsli_b(a, mem::transmute(b), IMM3) } /// Immediate Bit Insert Left @@ -2079,14 +2039,10 @@ pub unsafe fn __msa_binsli_b(a: v16u8, b: v16u8, imm3: i32) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(binsli.h, imm4 = 0b1111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_binsli_h(a: v8u16, b: v8u16, imm4: i32) -> v8u16 { - macro_rules! call { - ($imm4:expr) => { - msa_binsli_h(a, mem::transmute(b), $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_binsli_h(a: v8u16, b: v8u16) -> v8u16 { + static_assert_imm4!(IMM4); + msa_binsli_h(a, mem::transmute(b), IMM4) } /// Immediate Bit Insert Left @@ -2098,14 +2054,10 @@ pub unsafe fn __msa_binsli_h(a: v8u16, b: v8u16, imm4: i32) -> v8u16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(binsli.w, imm5 = 0b11111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_binsli_w(a: v4u32, b: v4u32, imm5: i32) -> v4u32 { - macro_rules! call { - ($imm5:expr) => { - msa_binsli_w(a, mem::transmute(b), $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_binsli_w(a: v4u32, b: v4u32) -> v4u32 { + static_assert_imm5!(IMM5); + msa_binsli_w(a, mem::transmute(b), IMM5) } /// Immediate Bit Insert Left @@ -2117,14 +2069,10 @@ pub unsafe fn __msa_binsli_w(a: v4u32, b: v4u32, imm5: i32) -> v4u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(binsli.d, imm6 = 0b111111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_binsli_d(a: v2u64, b: v2u64, imm6: i32) -> v2u64 { - macro_rules! call { - ($imm6:expr) => { - msa_binsli_d(a, mem::transmute(b), $imm6) - }; - } - constify_imm6!(imm6, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_binsli_d(a: v2u64, b: v2u64) -> v2u64 { + static_assert_imm6!(IMM6); + msa_binsli_d(a, mem::transmute(b), IMM6) } /// Vector Bit Insert Right @@ -2192,14 +2140,10 @@ pub unsafe fn __msa_binsr_d(a: v2u64, b: v2u64, c: v2u64) -> v2u64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(binsri.b, imm3 = 0b111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_binsri_b(a: v16u8, b: v16u8, imm3: i32) -> v16u8 { - macro_rules! call { - ($imm3:expr) => { - msa_binsri_b(a, mem::transmute(b), $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_binsri_b(a: v16u8, b: v16u8) -> v16u8 { + static_assert_imm3!(IMM3); + msa_binsri_b(a, mem::transmute(b), IMM3) } /// Immediate Bit Insert Right @@ -2211,14 +2155,10 @@ pub unsafe fn __msa_binsri_b(a: v16u8, b: v16u8, imm3: i32) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(binsri.h, imm4 = 0b1111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_binsri_h(a: v8u16, b: v8u16, imm4: i32) -> v8u16 { - macro_rules! call { - ($imm4:expr) => { - msa_binsri_h(a, mem::transmute(b), $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_binsri_h(a: v8u16, b: v8u16) -> v8u16 { + static_assert_imm4!(IMM4); + msa_binsri_h(a, mem::transmute(b), IMM4) } /// Immediate Bit Insert Right @@ -2230,14 +2170,10 @@ pub unsafe fn __msa_binsri_h(a: v8u16, b: v8u16, imm4: i32) -> v8u16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(binsri.w, imm5 = 0b11111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_binsri_w(a: v4u32, b: v4u32, imm5: i32) -> v4u32 { - macro_rules! call { - ($imm5:expr) => { - msa_binsri_w(a, mem::transmute(b), $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_binsri_w(a: v4u32, b: v4u32) -> v4u32 { + static_assert_imm5!(IMM5); + msa_binsri_w(a, mem::transmute(b), IMM5) } /// Immediate Bit Insert Right @@ -2249,14 +2185,10 @@ pub unsafe fn __msa_binsri_w(a: v4u32, b: v4u32, imm5: i32) -> v4u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(binsri.d, imm6 = 0b111111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_binsri_d(a: v2u64, b: v2u64, imm6: i32) -> v2u64 { - macro_rules! call { - ($imm6:expr) => { - msa_binsri_d(a, mem::transmute(b), $imm6) - }; - } - constify_imm6!(imm6, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_binsri_d(a: v2u64, b: v2u64) -> v2u64 { + static_assert_imm6!(IMM6); + msa_binsri_d(a, mem::transmute(b), IMM6) } /// Vector Bit Move If Not Zero @@ -2282,14 +2214,10 @@ pub unsafe fn __msa_bmnz_v(a: v16u8, b: v16u8, c: v16u8) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bmnzi.b, imm8 = 0b11111111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_bmnzi_b(a: v16u8, b: v16u8, imm8: i32) -> v16u8 { - macro_rules! call { - ($imm8:expr) => { - msa_bmnzi_b(a, mem::transmute(b), $imm8) - }; - } - constify_imm8!(imm8, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_bmnzi_b(a: v16u8, b: v16u8) -> v16u8 { + static_assert_imm8!(IMM8); + msa_bmnzi_b(a, mem::transmute(b), IMM8) } /// Vector Bit Move If Zero @@ -2315,14 +2243,10 @@ pub unsafe fn __msa_bmz_v(a: v16u8, b: v16u8, c: v16u8) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bmzi.b, imm8 = 0b11111111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_bmzi_b(a: v16u8, b: v16u8, imm8: i32) -> v16u8 { - macro_rules! call { - ($imm8:expr) => { - msa_bmzi_b(a, mem::transmute(b), $imm8) - }; - } - constify_imm8!(imm8, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_bmzi_b(a: v16u8, b: v16u8) -> v16u8 { + static_assert_imm8!(IMM8); + msa_bmzi_b(a, mem::transmute(b), IMM8) } /// Vector Bit Negate @@ -2390,14 +2314,10 @@ pub unsafe fn __msa_bneg_d(a: v2u64, b: v2u64) -> v2u64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bnegi.b, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bnegi_b(a: v16u8, imm3: i32) -> v16u8 { - macro_rules! call { - ($imm3:expr) => { - msa_bnegi_b(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bnegi_b(a: v16u8) -> v16u8 { + static_assert_imm3!(IMM3); + msa_bnegi_b(a, IMM3) } /// Immediate Bit Negate @@ -2409,14 +2329,10 @@ pub unsafe fn __msa_bnegi_b(a: v16u8, imm3: i32) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bnegi.h, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bnegi_h(a: v8u16, imm4: i32) -> v8u16 { - macro_rules! call { - ($imm4:expr) => { - msa_bnegi_h(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bnegi_h(a: v8u16) -> v8u16 { + static_assert_imm4!(IMM4); + msa_bnegi_h(a, IMM4) } /// Immediate Bit Negate @@ -2428,14 +2344,10 @@ pub unsafe fn __msa_bnegi_h(a: v8u16, imm4: i32) -> v8u16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bnegi.w, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bnegi_w(a: v4u32, imm5: i32) -> v4u32 { - macro_rules! call { - ($imm5:expr) => { - msa_bnegi_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bnegi_w(a: v4u32) -> v4u32 { + static_assert_imm5!(IMM5); + msa_bnegi_w(a, IMM5) } /// Immediate Bit Negate @@ -2447,14 +2359,10 @@ pub unsafe fn __msa_bnegi_w(a: v4u32, imm5: i32) -> v4u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bnegi.d, imm6 = 0b111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bnegi_d(a: v2u64, imm6: i32) -> v2u64 { - macro_rules! call { - ($imm6:expr) => { - msa_bnegi_d(a, $imm6) - }; - } - constify_imm6!(imm6, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bnegi_d(a: v2u64) -> v2u64 { + static_assert_imm6!(IMM6); + msa_bnegi_d(a, IMM6) } /// Immediate Branch If All Elements Are Not Zero @@ -2536,14 +2444,10 @@ pub unsafe fn __msa_bsel_v(a: v16u8, b: v16u8, c: v16u8) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bseli.b, imm8 = 0b11111111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_bseli_b(a: v16u8, b: v16u8, imm8: i32) -> v16u8 { - macro_rules! call { - ($imm8:expr) => { - msa_bseli_b(a, mem::transmute(b), $imm8) - }; - } - constify_imm8!(imm8, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_bseli_b(a: v16u8, b: v16u8) -> v16u8 { + static_assert_imm8!(IMM8); + msa_bseli_b(a, mem::transmute(b), IMM8) } /// Vector Bit Set @@ -2611,14 +2515,10 @@ pub unsafe fn __msa_bset_d(a: v2u64, b: v2u64) -> v2u64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bseti.b, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bseti_b(a: v16u8, imm3: i32) -> v16u8 { - macro_rules! call { - ($imm3:expr) => { - msa_bseti_b(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bseti_b(a: v16u8) -> v16u8 { + static_assert_imm3!(IMM3); + msa_bseti_b(a, IMM3) } /// Immediate Bit Set @@ -2630,14 +2530,10 @@ pub unsafe fn __msa_bseti_b(a: v16u8, imm3: i32) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bseti.h, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bseti_h(a: v8u16, imm4: i32) -> v8u16 { - macro_rules! call { - ($imm4:expr) => { - msa_bseti_h(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bseti_h(a: v8u16) -> v8u16 { + static_assert_imm4!(IMM4); + msa_bseti_h(a, IMM4) } /// Immediate Bit Set @@ -2649,14 +2545,10 @@ pub unsafe fn __msa_bseti_h(a: v8u16, imm4: i32) -> v8u16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bseti.w, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bseti_w(a: v4u32, imm5: i32) -> v4u32 { - macro_rules! call { - ($imm5:expr) => { - msa_bseti_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bseti_w(a: v4u32) -> v4u32 { + static_assert_imm5!(IMM5); + msa_bseti_w(a, IMM5) } /// Immediate Bit Set @@ -2668,14 +2560,10 @@ pub unsafe fn __msa_bseti_w(a: v4u32, imm5: i32) -> v4u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bseti.d, imm6 = 0b111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bseti_d(a: v2u64, imm6: i32) -> v2u64 { - macro_rules! call { - ($imm6:expr) => { - msa_bseti_d(a, $imm6) - }; - } - constify_imm6!(imm6, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bseti_d(a: v2u64) -> v2u64 { + static_assert_imm6!(IMM6); + msa_bseti_d(a, IMM6) } /// Immediate Branch If At Least One Element Is Zero @@ -2795,14 +2683,10 @@ pub unsafe fn __msa_ceq_d(a: v2i64, b: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ceqi.b, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_ceqi_b(a: v16i8, imm_s5: i32) -> v16i8 { - macro_rules! call { - ($imm_s5:expr) => { - msa_ceqi_b(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_ceqi_b(a: v16i8) -> v16i8 { + static_assert_imm_s5!(IMM_S5); + msa_ceqi_b(a, IMM_S5) } /// Immediate Compare Equal @@ -2814,14 +2698,10 @@ pub unsafe fn __msa_ceqi_b(a: v16i8, imm_s5: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ceqi.h, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_ceqi_h(a: v8i16, imm_s5: i32) -> v8i16 { - macro_rules! call { - ($imm_s5:expr) => { - msa_ceqi_h(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_ceqi_h(a: v8i16) -> v8i16 { + static_assert_imm_s5!(IMM_S5); + msa_ceqi_h(a, IMM_S5) } /// Immediate Compare Equal @@ -2833,14 +2713,10 @@ pub unsafe fn __msa_ceqi_h(a: v8i16, imm_s5: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ceqi.w, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_ceqi_w(a: v4i32, imm_s5: i32) -> v4i32 { - macro_rules! call { - ($imm_s5:expr) => { - msa_ceqi_w(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_ceqi_w(a: v4i32) -> v4i32 { + static_assert_imm_s5!(IMM_S5); + msa_ceqi_w(a, IMM_S5) } /// Immediate Compare Equal @@ -2852,14 +2728,10 @@ pub unsafe fn __msa_ceqi_w(a: v4i32, imm_s5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ceqi.d, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_ceqi_d(a: v2i64, imm_s5: i32) -> v2i64 { - macro_rules! call { - ($imm_s5:expr) => { - msa_ceqi_d(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_ceqi_d(a: v2i64) -> v2i64 { + static_assert_imm_s5!(IMM_S5); + msa_ceqi_d(a, IMM_S5) } /// GPR Copy from MSA Control Register @@ -2870,14 +2742,10 @@ pub unsafe fn __msa_ceqi_d(a: v2i64, imm_s5: i32) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(cfcmsa, imm5 = 0b11111))] -#[rustc_args_required_const(0)] -pub unsafe fn __msa_cfcmsa(imm5: i32) -> i32 { - macro_rules! call { - ($imm5:expr) => { - msa_cfcmsa($imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(0)] +pub unsafe fn __msa_cfcmsa() -> i32 { + static_assert_imm5!(IMM5); + msa_cfcmsa(IMM5) } /// Vector Compare Signed Less Than or Equal @@ -3002,14 +2870,10 @@ pub unsafe fn __msa_cle_u_d(a: v2u64, b: v2u64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clei_s.b, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clei_s_b(a: v16i8, imm_s5: i32) -> v16i8 { - macro_rules! call { - ($imm_s5:expr) => { - msa_clei_s_b(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clei_s_b(a: v16i8) -> v16i8 { + static_assert_imm_s5!(IMM_S5); + msa_clei_s_b(a, IMM_S5) } /// Immediate Compare Signed Less Than or Equal @@ -3022,14 +2886,10 @@ pub unsafe fn __msa_clei_s_b(a: v16i8, imm_s5: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clei_s.h, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clei_s_h(a: v8i16, imm_s5: i32) -> v8i16 { - macro_rules! call { - ($imm_s5:expr) => { - msa_clei_s_h(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clei_s_h(a: v8i16) -> v8i16 { + static_assert_imm_s5!(IMM_S5); + msa_clei_s_h(a, IMM_S5) } /// Immediate Compare Signed Less Than or Equal @@ -3042,14 +2902,10 @@ pub unsafe fn __msa_clei_s_h(a: v8i16, imm_s5: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clei_s.w, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clei_s_w(a: v4i32, imm_s5: i32) -> v4i32 { - macro_rules! call { - ($imm_s5:expr) => { - msa_clei_s_w(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clei_s_w(a: v4i32) -> v4i32 { + static_assert_imm_s5!(IMM_S5); + msa_clei_s_w(a, IMM_S5) } /// Immediate Compare Signed Less Than or Equal @@ -3062,14 +2918,10 @@ pub unsafe fn __msa_clei_s_w(a: v4i32, imm_s5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clei_s.d, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clei_s_d(a: v2i64, imm_s5: i32) -> v2i64 { - macro_rules! call { - ($imm_s5:expr) => { - msa_clei_s_d(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clei_s_d(a: v2i64) -> v2i64 { + static_assert_imm_s5!(IMM_S5); + msa_clei_s_d(a, IMM_S5) } /// Immediate Compare Unsigned Less Than or Equal @@ -3082,14 +2934,10 @@ pub unsafe fn __msa_clei_s_d(a: v2i64, imm_s5: i32) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clei_u.b, imm5 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clei_u_b(a: v16u8, imm5: i32) -> v16i8 { - macro_rules! call { - ($imm5:expr) => { - msa_clei_u_b(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clei_u_b(a: v16u8) -> v16i8 { + static_assert_imm5!(IMM5); + msa_clei_u_b(a, IMM5) } /// Immediate Compare Unsigned Less Than or Equal @@ -3102,14 +2950,10 @@ pub unsafe fn __msa_clei_u_b(a: v16u8, imm5: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clei_u.h, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clei_u_h(a: v8u16, imm5: i32) -> v8i16 { - macro_rules! call { - ($imm5:expr) => { - msa_clei_u_h(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clei_u_h(a: v8u16) -> v8i16 { + static_assert_imm5!(IMM5); + msa_clei_u_h(a, IMM5) } /// Immediate Compare Unsigned Less Than or Equal @@ -3122,14 +2966,10 @@ pub unsafe fn __msa_clei_u_h(a: v8u16, imm5: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clei_u.w, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clei_u_w(a: v4u32, imm5: i32) -> v4i32 { - macro_rules! call { - ($imm5:expr) => { - msa_clei_u_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clei_u_w(a: v4u32) -> v4i32 { + static_assert_imm5!(IMM5); + msa_clei_u_w(a, IMM5) } /// Immediate Compare Unsigned Less Than or Equal @@ -3142,14 +2982,10 @@ pub unsafe fn __msa_clei_u_w(a: v4u32, imm5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clei_u.d, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clei_u_d(a: v2u64, imm5: i32) -> v2i64 { - macro_rules! call { - ($imm5:expr) => { - msa_clei_u_d(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clei_u_d(a: v2u64) -> v2i64 { + static_assert_imm5!(IMM5); + msa_clei_u_d(a, IMM5) } /// Vector Compare Signed Less Than @@ -3274,14 +3110,10 @@ pub unsafe fn __msa_clt_u_d(a: v2u64, b: v2u64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clti_s.b, imm_s5 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clti_s_b(a: v16i8, imm_s5: i32) -> v16i8 { - macro_rules! call { - ($imm_s5:expr) => { - msa_clti_s_b(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clti_s_b(a: v16i8) -> v16i8 { + static_assert_imm_s5!(IMM_S5); + msa_clti_s_b(a, IMM_S5) } /// Immediate Compare Signed Less Than @@ -3294,14 +3126,10 @@ pub unsafe fn __msa_clti_s_b(a: v16i8, imm_s5: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clti_s.h, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clti_s_h(a: v8i16, imm_s5: i32) -> v8i16 { - macro_rules! call { - ($imm_s5:expr) => { - msa_clti_s_h(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clti_s_h(a: v8i16) -> v8i16 { + static_assert_imm_s5!(IMM_S5); + msa_clti_s_h(a, IMM_S5) } /// Immediate Compare Signed Less Than @@ -3314,14 +3142,10 @@ pub unsafe fn __msa_clti_s_h(a: v8i16, imm_s5: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clti_s.w, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clti_s_w(a: v4i32, imm_s5: i32) -> v4i32 { - macro_rules! call { - ($imm_s5:expr) => { - msa_clti_s_w(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clti_s_w(a: v4i32) -> v4i32 { + static_assert_imm_s5!(IMM_S5); + msa_clti_s_w(a, IMM_S5) } /// Immediate Compare Signed Less Than @@ -3334,14 +3158,10 @@ pub unsafe fn __msa_clti_s_w(a: v4i32, imm_s5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clti_s.d, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clti_s_d(a: v2i64, imm_s5: i32) -> v2i64 { - macro_rules! call { - ($imm_s5:expr) => { - msa_clti_s_d(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clti_s_d(a: v2i64) -> v2i64 { + static_assert_imm_s5!(IMM_S5); + msa_clti_s_d(a, IMM_S5) } /// Immediate Compare Unsigned Less Than @@ -3354,14 +3174,10 @@ pub unsafe fn __msa_clti_s_d(a: v2i64, imm_s5: i32) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clti_u.b, imm5 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clti_u_b(a: v16u8, imm5: i32) -> v16i8 { - macro_rules! call { - ($imm5:expr) => { - msa_clti_u_b(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clti_u_b(a: v16u8) -> v16i8 { + static_assert_imm5!(IMM5); + msa_clti_u_b(a, IMM5) } /// Immediate Compare Unsigned Less Than @@ -3374,14 +3190,10 @@ pub unsafe fn __msa_clti_u_b(a: v16u8, imm5: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clti_u.h, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clti_u_h(a: v8u16, imm5: i32) -> v8i16 { - macro_rules! call { - ($imm5:expr) => { - msa_clti_u_h(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clti_u_h(a: v8u16) -> v8i16 { + static_assert_imm5!(IMM5); + msa_clti_u_h(a, IMM5) } /// Immediate Compare Unsigned Less Than @@ -3394,14 +3206,10 @@ pub unsafe fn __msa_clti_u_h(a: v8u16, imm5: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clti_u.w, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clti_u_w(a: v4u32, imm5: i32) -> v4i32 { - macro_rules! call { - ($imm5:expr) => { - msa_clti_u_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clti_u_w(a: v4u32) -> v4i32 { + static_assert_imm5!(IMM5); + msa_clti_u_w(a, IMM5) } /// Immediate Compare Unsigned Less Than @@ -3414,14 +3222,10 @@ pub unsafe fn __msa_clti_u_w(a: v4u32, imm5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clti_u.d, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clti_u_d(a: v2u64, imm5: i32) -> v2i64 { - macro_rules! call { - ($imm5:expr) => { - msa_clti_u_d(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clti_u_d(a: v2u64) -> v2i64 { + static_assert_imm5!(IMM5); + msa_clti_u_d(a, IMM5) } /// Element Copy to GPR Signed @@ -3432,14 +3236,10 @@ pub unsafe fn __msa_clti_u_d(a: v2u64, imm5: i32) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(copy_s.b, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_copy_s_b(a: v16i8, imm4: i32) -> i32 { - macro_rules! call { - ($imm4:expr) => { - msa_copy_s_b(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_copy_s_b(a: v16i8) -> i32 { + static_assert_imm4!(IMM4); + msa_copy_s_b(a, IMM4) } /// Element Copy to GPR Signed @@ -3450,14 +3250,10 @@ pub unsafe fn __msa_copy_s_b(a: v16i8, imm4: i32) -> i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(copy_s.h, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_copy_s_h(a: v8i16, imm3: i32) -> i32 { - macro_rules! call { - ($imm3:expr) => { - msa_copy_s_h(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_copy_s_h(a: v8i16) -> i32 { + static_assert_imm3!(IMM3); + msa_copy_s_h(a, IMM3) } /// Element Copy to GPR Signed @@ -3468,14 +3264,10 @@ pub unsafe fn __msa_copy_s_h(a: v8i16, imm3: i32) -> i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(copy_s.w, imm2 = 0b11))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_copy_s_w(a: v4i32, imm2: i32) -> i32 { - macro_rules! call { - ($imm2:expr) => { - msa_copy_s_w(a, $imm2) - }; - } - constify_imm2!(imm2, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_copy_s_w(a: v4i32) -> i32 { + static_assert_imm2!(IMM2); + msa_copy_s_w(a, IMM2) } /// Element Copy to GPR Signed @@ -3486,14 +3278,10 @@ pub unsafe fn __msa_copy_s_w(a: v4i32, imm2: i32) -> i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(copy_s.d, imm1 = 0b1))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_copy_s_d(a: v2i64, imm1: i32) -> i64 { - macro_rules! call { - ($imm1:expr) => { - msa_copy_s_d(a, $imm1) - }; - } - constify_imm1!(imm1, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_copy_s_d(a: v2i64) -> i64 { + static_assert_imm1!(IMM1); + msa_copy_s_d(a, IMM1) } /// Element Copy to GPR Unsigned @@ -3504,14 +3292,10 @@ pub unsafe fn __msa_copy_s_d(a: v2i64, imm1: i32) -> i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(copy_u.b, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_copy_u_b(a: v16i8, imm4: i32) -> u32 { - macro_rules! call { - ($imm4:expr) => { - msa_copy_u_b(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_copy_u_b(a: v16i8) -> u32 { + static_assert_imm4!(IMM4); + msa_copy_u_b(a, IMM4) } /// Element Copy to GPR Unsigned @@ -3522,14 +3306,10 @@ pub unsafe fn __msa_copy_u_b(a: v16i8, imm4: i32) -> u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(copy_u.h, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_copy_u_h(a: v8i16, imm3: i32) -> u32 { - macro_rules! call { - ($imm3:expr) => { - msa_copy_u_h(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_copy_u_h(a: v8i16) -> u32 { + static_assert_imm3!(IMM3); + msa_copy_u_h(a, IMM3) } /// Element Copy to GPR Unsigned @@ -3540,14 +3320,10 @@ pub unsafe fn __msa_copy_u_h(a: v8i16, imm3: i32) -> u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(copy_u.w, imm2 = 0b11))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_copy_u_w(a: v4i32, imm2: i32) -> u32 { - macro_rules! call { - ($imm2:expr) => { - msa_copy_u_w(a, $imm2) - }; - } - constify_imm2!(imm2, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_copy_u_w(a: v4i32) -> u32 { + static_assert_imm2!(IMM2); + msa_copy_u_w(a, IMM2) } /// Element Copy to GPR Unsigned @@ -3558,14 +3334,10 @@ pub unsafe fn __msa_copy_u_w(a: v4i32, imm2: i32) -> u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(copy_u.d, imm1 = 0b1))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_copy_u_d(a: v2i64, imm1: i32) -> u64 { - macro_rules! call { - ($imm1:expr) => { - msa_copy_u_d(a, $imm1) - }; - } - constify_imm1!(imm1, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_copy_u_d(a: v2i64) -> u64 { + static_assert_imm1!(IMM1); + msa_copy_u_d(a, IMM1) } /// GPR Copy to MSA Control Register @@ -3578,14 +3350,10 @@ pub unsafe fn __msa_copy_u_d(a: v2i64, imm1: i32) -> u64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ctcmsa, imm1 = 0b1))] -#[rustc_args_required_const(0)] -pub unsafe fn __msa_ctcmsa(imm5: i32, a: i32) -> () { - macro_rules! call { - ($imm5:expr) => { - msa_ctcmsa($imm5, a) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(0)] +pub unsafe fn __msa_ctcmsa(a: i32) -> () { + static_assert_imm5!(IMM5); + msa_ctcmsa(IMM5, a) } /// Vector Signed Divide @@ -5798,14 +5566,10 @@ pub unsafe fn __msa_ilvr_d(a: v2i64, b: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(insert.b, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_insert_b(a: v16i8, imm4: i32, c: i32) -> v16i8 { - macro_rules! call { - ($imm4:expr) => { - msa_insert_b(a, $imm4, c) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_insert_b(a: v16i8, c: i32) -> v16i8 { + static_assert_imm4!(IMM4); + msa_insert_b(a, IMM4, c) } /// GPR Insert Element @@ -5817,14 +5581,10 @@ pub unsafe fn __msa_insert_b(a: v16i8, imm4: i32, c: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(insert.h, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_insert_h(a: v8i16, imm3: i32, c: i32) -> v8i16 { - macro_rules! call { - ($imm3:expr) => { - msa_insert_h(a, $imm3, c) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_insert_h(a: v8i16, c: i32) -> v8i16 { + static_assert_imm3!(IMM3); + msa_insert_h(a, IMM3, c) } /// GPR Insert Element @@ -5836,14 +5596,10 @@ pub unsafe fn __msa_insert_h(a: v8i16, imm3: i32, c: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(insert.w, imm2 = 0b11))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_insert_w(a: v4i32, imm2: i32, c: i32) -> v4i32 { - macro_rules! call { - ($imm2:expr) => { - msa_insert_w(a, $imm2, c) - }; - } - constify_imm2!(imm2, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_insert_w(a: v4i32, c: i32) -> v4i32 { + static_assert_imm2!(IMM2); + msa_insert_w(a, IMM2, c) } /// GPR Insert Element @@ -5855,14 +5611,10 @@ pub unsafe fn __msa_insert_w(a: v4i32, imm2: i32, c: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(insert.d, imm1 = 0b1))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_insert_d(a: v2i64, imm1: i32, c: i64) -> v2i64 { - macro_rules! call { - ($imm1:expr) => { - msa_insert_d(a, $imm1, c) - }; - } - constify_imm1!(imm1, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_insert_d(a: v2i64, c: i64) -> v2i64 { + static_assert_imm1!(IMM1); + msa_insert_d(a, IMM1, c) } /// Element Insert Element @@ -5874,14 +5626,10 @@ pub unsafe fn __msa_insert_d(a: v2i64, imm1: i32, c: i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(insve.b, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_insve_b(a: v16i8, imm4: i32, c: v16i8) -> v16i8 { - macro_rules! call { - ($imm4:expr) => { - msa_insve_b(a, $imm4, c) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_insve_b(a: v16i8, c: v16i8) -> v16i8 { + static_assert_imm4!(IMM4); + msa_insve_b(a, IMM4, c) } /// Element Insert Element @@ -5893,14 +5641,10 @@ pub unsafe fn __msa_insve_b(a: v16i8, imm4: i32, c: v16i8) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(insve.h, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_insve_h(a: v8i16, imm3: i32, c: v8i16) -> v8i16 { - macro_rules! call { - ($imm3:expr) => { - msa_insve_h(a, $imm3, c) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_insve_h(a: v8i16, c: v8i16) -> v8i16 { + static_assert_imm3!(IMM3); + msa_insve_h(a, IMM3, c) } /// Element Insert Element @@ -5912,14 +5656,10 @@ pub unsafe fn __msa_insve_h(a: v8i16, imm3: i32, c: v8i16) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(insve.w, imm2 = 0b11))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_insve_w(a: v4i32, imm2: i32, c: v4i32) -> v4i32 { - macro_rules! call { - ($imm2:expr) => { - msa_insve_w(a, $imm2, c) - }; - } - constify_imm2!(imm2, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_insve_w(a: v4i32, c: v4i32) -> v4i32 { + static_assert_imm2!(IMM2); + msa_insve_w(a, IMM2, c) } /// Element Insert Element @@ -5931,14 +5671,10 @@ pub unsafe fn __msa_insve_w(a: v4i32, imm2: i32, c: v4i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(insve.d, imm1 = 0b1))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_insve_d(a: v2i64, imm1: i32, c: v2i64) -> v2i64 { - macro_rules! call { - ($imm1:expr) => { - msa_insve_d(a, $imm1, c) - }; - } - constify_imm1!(imm1, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_insve_d(a: v2i64, c: v2i64) -> v2i64 { + static_assert_imm1!(IMM1); + msa_insve_d(a, IMM1, c) } /// Vector Load @@ -5950,14 +5686,10 @@ pub unsafe fn __msa_insve_d(a: v2i64, imm1: i32, c: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ld.b, imm_s10 = 0b1111111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_ld_b(mem_addr: *mut u8, imm_s10: i32) -> v16i8 { - macro_rules! call { - ($imm_s10:expr) => { - msa_ld_b(mem_addr, $imm_s10) - }; - } - constify_imm_s10!(imm_s10, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_ld_b(mem_addr: *mut u8) -> v16i8 { + static_assert_imm_s10!(IMM_S10); + msa_ld_b(mem_addr, IMM_S10) } /// Vector Load @@ -5969,14 +5701,11 @@ pub unsafe fn __msa_ld_b(mem_addr: *mut u8, imm_s10: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ld.h, imm_s11 = 0b11111111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_ld_h(mem_addr: *mut u8, imm_s11: i32) -> v8i16 { - macro_rules! call { - ($imm_s11:expr) => { - msa_ld_h(mem_addr, $imm_s11) - }; - } - constify_imm_s11!(imm_s11, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_ld_h(mem_addr: *mut u8) -> v8i16 { + static_assert_imm_s11!(IMM_S11); + static_assert!(IMM_S11: i32 where IMM_S11 % 2 == 0); + msa_ld_h(mem_addr, IMM_S11) } /// Vector Load @@ -5988,14 +5717,11 @@ pub unsafe fn __msa_ld_h(mem_addr: *mut u8, imm_s11: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ld.w, imm_s12 = 0b111111111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_ld_w(mem_addr: *mut u8, imm_s12: i32) -> v4i32 { - macro_rules! call { - ($imm_s12:expr) => { - msa_ld_w(mem_addr, $imm_s12) - }; - } - constify_imm_s12!(imm_s12, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_ld_w(mem_addr: *mut u8) -> v4i32 { + static_assert_imm_s12!(IMM_S12); + static_assert!(IMM_S12: i32 where IMM_S12 % 4 == 0); + msa_ld_w(mem_addr, IMM_S12) } /// Vector Load @@ -6007,14 +5733,11 @@ pub unsafe fn __msa_ld_w(mem_addr: *mut u8, imm_s12: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ld.d, imm_s13 = 0b1111111111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_ld_d(mem_addr: *mut u8, imm_s13: i32) -> v2i64 { - macro_rules! call { - ($imm_s13:expr) => { - msa_ld_d(mem_addr, $imm_s13) - }; - } - constify_imm_s13!(imm_s13, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_ld_d(mem_addr: *mut u8) -> v2i64 { + static_assert_imm_s13!(IMM_S13); + static_assert!(IMM_S13: i32 where IMM_S13 % 8 == 0); + msa_ld_d(mem_addr, IMM_S13) } /// Immediate Load @@ -6026,14 +5749,10 @@ pub unsafe fn __msa_ld_d(mem_addr: *mut u8, imm_s13: i32) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ldi.b, imm_s10 = 0b1111111111))] -#[rustc_args_required_const(0)] -pub unsafe fn __msa_ldi_b(imm_s10: i32) -> v16i8 { - macro_rules! call { - ($imm_s10:expr) => { - msa_ldi_b($imm_s10) - }; - } - constify_imm_s10!(imm_s10, call) +#[rustc_legacy_const_generics(0)] +pub unsafe fn __msa_ldi_b() -> v16i8 { + static_assert_imm_s10!(IMM_S10); + msa_ldi_b(IMM_S10) } /// Immediate Load @@ -6045,14 +5764,10 @@ pub unsafe fn __msa_ldi_b(imm_s10: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ldi.h, imm_s10 = 0b1111111111))] -#[rustc_args_required_const(0)] -pub unsafe fn __msa_ldi_h(imm_s10: i32) -> v8i16 { - macro_rules! call { - ($imm_s10:expr) => { - msa_ldi_h($imm_s10) - }; - } - constify_imm_s10!(imm_s10, call) +#[rustc_legacy_const_generics(0)] +pub unsafe fn __msa_ldi_h() -> v8i16 { + static_assert_imm_s10!(IMM_S10); + msa_ldi_h(IMM_S10) } /// Immediate Load @@ -6064,14 +5779,10 @@ pub unsafe fn __msa_ldi_h(imm_s10: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ldi.w, imm_s10 = 0b1111111111))] -#[rustc_args_required_const(0)] -pub unsafe fn __msa_ldi_w(imm_s10: i32) -> v4i32 { - macro_rules! call { - ($imm_s10:expr) => { - msa_ldi_w($imm_s10) - }; - } - constify_imm_s10!(imm_s10, call) +#[rustc_legacy_const_generics(0)] +pub unsafe fn __msa_ldi_w() -> v4i32 { + static_assert_imm_s10!(IMM_S10); + msa_ldi_w(IMM_S10) } /// Immediate Load @@ -6083,14 +5794,10 @@ pub unsafe fn __msa_ldi_w(imm_s10: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ldi.d, imm_s10 = 0b1111111111))] -#[rustc_args_required_const(0)] -pub unsafe fn __msa_ldi_d(imm_s10: i32) -> v2i64 { - macro_rules! call { - ($imm_s10:expr) => { - msa_ldi_d($imm_s10) - }; - } - constify_imm_s10!(imm_s10, call) +#[rustc_legacy_const_generics(0)] +pub unsafe fn __msa_ldi_d() -> v2i64 { + static_assert_imm_s10!(IMM_S10); + msa_ldi_d(IMM_S10) } /// Vector Fixed-Point Multiply and Add @@ -6378,14 +6085,10 @@ pub unsafe fn __msa_max_u_d(a: v2u64, b: v2u64) -> v2u64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(maxi_s.b, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_maxi_s_b(a: v16i8, imm_s5: i32) -> v16i8 { - macro_rules! call { - ($imm_s5:expr) => { - msa_maxi_s_b(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_maxi_s_b(a: v16i8) -> v16i8 { + static_assert_imm_s5!(IMM_S5); + msa_maxi_s_b(a, IMM_S5) } /// Immediate Signed Maximum @@ -6397,14 +6100,10 @@ pub unsafe fn __msa_maxi_s_b(a: v16i8, imm_s5: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(maxi_s.h, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_maxi_s_h(a: v8i16, imm_s5: i32) -> v8i16 { - macro_rules! call { - ($imm_s5:expr) => { - msa_maxi_s_h(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_maxi_s_h(a: v8i16) -> v8i16 { + static_assert_imm_s5!(IMM_S5); + msa_maxi_s_h(a, IMM_S5) } /// Immediate Signed Maximum @@ -6416,14 +6115,10 @@ pub unsafe fn __msa_maxi_s_h(a: v8i16, imm_s5: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(maxi_s.w, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_maxi_s_w(a: v4i32, imm_s5: i32) -> v4i32 { - macro_rules! call { - ($imm_s5:expr) => { - msa_maxi_s_w(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_maxi_s_w(a: v4i32) -> v4i32 { + static_assert_imm_s5!(IMM_S5); + msa_maxi_s_w(a, IMM_S5) } /// Immediate Signed Maximum @@ -6435,14 +6130,10 @@ pub unsafe fn __msa_maxi_s_w(a: v4i32, imm_s5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(maxi_s.d, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_maxi_s_d(a: v2i64, imm_s5: i32) -> v2i64 { - macro_rules! call { - ($imm_s5:expr) => { - msa_maxi_s_d(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_maxi_s_d(a: v2i64) -> v2i64 { + static_assert_imm_s5!(IMM_S5); + msa_maxi_s_d(a, IMM_S5) } /// Immediate Unsigned Maximum @@ -6454,14 +6145,10 @@ pub unsafe fn __msa_maxi_s_d(a: v2i64, imm_s5: i32) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(maxi_u.b, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_maxi_u_b(a: v16u8, imm5: i32) -> v16u8 { - macro_rules! call { - ($imm5:expr) => { - msa_maxi_u_b(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_maxi_u_b(a: v16u8) -> v16u8 { + static_assert_imm5!(IMM5); + msa_maxi_u_b(a, IMM5) } /// Immediate Unsigned Maximum @@ -6473,14 +6160,10 @@ pub unsafe fn __msa_maxi_u_b(a: v16u8, imm5: i32) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(maxi_u.h, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_maxi_u_h(a: v8u16, imm5: i32) -> v8u16 { - macro_rules! call { - ($imm5:expr) => { - msa_maxi_u_h(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_maxi_u_h(a: v8u16) -> v8u16 { + static_assert_imm5!(IMM5); + msa_maxi_u_h(a, IMM5) } /// Immediate Unsigned Maximum @@ -6492,14 +6175,10 @@ pub unsafe fn __msa_maxi_u_h(a: v8u16, imm5: i32) -> v8u16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(maxi_u.w, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_maxi_u_w(a: v4u32, imm5: i32) -> v4u32 { - macro_rules! call { - ($imm5:expr) => { - msa_maxi_u_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_maxi_u_w(a: v4u32) -> v4u32 { + static_assert_imm5!(IMM5); + msa_maxi_u_w(a, IMM5) } /// Immediate Unsigned Maximum @@ -6511,14 +6190,10 @@ pub unsafe fn __msa_maxi_u_w(a: v4u32, imm5: i32) -> v4u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(maxi_u.d, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_maxi_u_d(a: v2u64, imm5: i32) -> v2u64 { - macro_rules! call { - ($imm5:expr) => { - msa_maxi_u_d(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_maxi_u_d(a: v2u64) -> v2u64 { + static_assert_imm5!(IMM5); + msa_maxi_u_d(a, IMM5) } /// Vector Minimum Based on Absolute Value @@ -6638,14 +6313,10 @@ pub unsafe fn __msa_min_s_d(a: v2i64, b: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(mini_s.b, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_mini_s_b(a: v16i8, imm_s5: i32) -> v16i8 { - macro_rules! call { - ($imm_s5:expr) => { - msa_mini_s_b(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_mini_s_b(a: v16i8) -> v16i8 { + static_assert_imm_s5!(IMM_S5); + msa_mini_s_b(a, IMM_S5) } /// Immediate Signed Minimum @@ -6657,14 +6328,10 @@ pub unsafe fn __msa_mini_s_b(a: v16i8, imm_s5: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(mini_s.h, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_mini_s_h(a: v8i16, imm_s5: i32) -> v8i16 { - macro_rules! call { - ($imm_s5:expr) => { - msa_mini_s_h(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_mini_s_h(a: v8i16) -> v8i16 { + static_assert_imm_s5!(IMM_S5); + msa_mini_s_h(a, IMM_S5) } /// Immediate Signed Minimum @@ -6676,14 +6343,10 @@ pub unsafe fn __msa_mini_s_h(a: v8i16, imm_s5: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(mini_s.w, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_mini_s_w(a: v4i32, imm_s5: i32) -> v4i32 { - macro_rules! call { - ($imm_s5:expr) => { - msa_mini_s_w(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_mini_s_w(a: v4i32) -> v4i32 { + static_assert_imm_s5!(IMM_S5); + msa_mini_s_w(a, IMM_S5) } /// Immediate Signed Minimum @@ -6695,14 +6358,10 @@ pub unsafe fn __msa_mini_s_w(a: v4i32, imm_s5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(mini_s.d, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_mini_s_d(a: v2i64, imm_s5: i32) -> v2i64 { - macro_rules! call { - ($imm_s5:expr) => { - msa_mini_s_d(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_mini_s_d(a: v2i64) -> v2i64 { + static_assert_imm_s5!(IMM_S5); + msa_mini_s_d(a, IMM_S5) } /// Vector Unsigned Minimum @@ -6766,14 +6425,10 @@ pub unsafe fn __msa_min_u_d(a: v2u64, b: v2u64) -> v2u64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(mini_u.b, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_mini_u_b(a: v16u8, imm5: i32) -> v16u8 { - macro_rules! call { - ($imm5:expr) => { - msa_mini_u_b(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_mini_u_b(a: v16u8) -> v16u8 { + static_assert_imm5!(IMM5); + msa_mini_u_b(a, IMM5) } /// Immediate Unsigned Minimum @@ -6785,14 +6440,10 @@ pub unsafe fn __msa_mini_u_b(a: v16u8, imm5: i32) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(mini_u.h, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_mini_u_h(a: v8u16, imm5: i32) -> v8u16 { - macro_rules! call { - ($imm5:expr) => { - msa_mini_u_h(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_mini_u_h(a: v8u16) -> v8u16 { + static_assert_imm5!(IMM5); + msa_mini_u_h(a, IMM5) } /// Immediate Unsigned Minimum @@ -6804,14 +6455,10 @@ pub unsafe fn __msa_mini_u_h(a: v8u16, imm5: i32) -> v8u16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(mini_u.w, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_mini_u_w(a: v4u32, imm5: i32) -> v4u32 { - macro_rules! call { - ($imm5:expr) => { - msa_mini_u_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_mini_u_w(a: v4u32) -> v4u32 { + static_assert_imm5!(IMM5); + msa_mini_u_w(a, IMM5) } /// Immediate Unsigned Minimum @@ -6823,14 +6470,10 @@ pub unsafe fn __msa_mini_u_w(a: v4u32, imm5: i32) -> v4u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(mini_u.d, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_mini_u_d(a: v2u64, imm5: i32) -> v2u64 { - macro_rules! call { - ($imm5:expr) => { - msa_mini_u_d(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_mini_u_d(a: v2u64) -> v2u64 { + static_assert_imm5!(IMM5); + msa_mini_u_d(a, IMM5) } /// Vector Signed Modulo @@ -7313,14 +6956,10 @@ pub unsafe fn __msa_nor_v(a: v16u8, b: v16u8) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(nori.b, imm8 = 0b11111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_nori_b(a: v16u8, imm8: i32) -> v16u8 { - macro_rules! call { - ($imm8:expr) => { - msa_nori_b(a, $imm8) - }; - } - constify_imm8!(imm8, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_nori_b(a: v16u8) -> v16u8 { + static_assert_imm8!(IMM8); + msa_nori_b(a, IMM8) } /// Vector Logical Or @@ -7347,14 +6986,10 @@ pub unsafe fn __msa_or_v(a: v16u8, b: v16u8) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ori.b, imm8 = 0b11111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_ori_b(a: v16u8, imm8: i32) -> v16u8 { - macro_rules! call { - ($imm8:expr) => { - msa_ori_b(a, $imm8) - }; - } - constify_imm8!(imm8, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_ori_b(a: v16u8) -> v16u8 { + static_assert_imm8!(IMM8); + msa_ori_b(a, IMM8) } /// Vector Pack Even @@ -7518,14 +7153,10 @@ pub unsafe fn __msa_pcnt_d(a: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sat_s.b, imm4 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_sat_s_b(a: v16i8, imm3: i32) -> v16i8 { - macro_rules! call { - ($imm3:expr) => { - msa_sat_s_b(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_sat_s_b(a: v16i8) -> v16i8 { + static_assert_imm3!(IMM3); + msa_sat_s_b(a, IMM3) } /// Immediate Signed Saturate @@ -7537,14 +7168,10 @@ pub unsafe fn __msa_sat_s_b(a: v16i8, imm3: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sat_s.h, imm3 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_sat_s_h(a: v8i16, imm4: i32) -> v8i16 { - macro_rules! call { - ($imm4:expr) => { - msa_sat_s_h(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_sat_s_h(a: v8i16) -> v8i16 { + static_assert_imm4!(IMM4); + msa_sat_s_h(a, IMM4) } /// Immediate Signed Saturate @@ -7556,14 +7183,10 @@ pub unsafe fn __msa_sat_s_h(a: v8i16, imm4: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sat_s.w, imm2 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_sat_s_w(a: v4i32, imm5: i32) -> v4i32 { - macro_rules! call { - ($imm5:expr) => { - msa_sat_s_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_sat_s_w(a: v4i32) -> v4i32 { + static_assert_imm5!(IMM5); + msa_sat_s_w(a, IMM5) } /// Immediate Signed Saturate @@ -7575,14 +7198,10 @@ pub unsafe fn __msa_sat_s_w(a: v4i32, imm5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sat_s.d, imm1 = 0b111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_sat_s_d(a: v2i64, imm6: i32) -> v2i64 { - macro_rules! call { - ($imm6:expr) => { - msa_sat_s_d(a, $imm6) - }; - } - constify_imm6!(imm6, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_sat_s_d(a: v2i64) -> v2i64 { + static_assert_imm6!(IMM6); + msa_sat_s_d(a, IMM6) } /// Immediate Unsigned Saturate @@ -7594,14 +7213,10 @@ pub unsafe fn __msa_sat_s_d(a: v2i64, imm6: i32) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sat_u.b, imm4 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_sat_u_b(a: v16u8, imm3: i32) -> v16u8 { - macro_rules! call { - ($imm3:expr) => { - msa_sat_u_b(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_sat_u_b(a: v16u8) -> v16u8 { + static_assert_imm3!(IMM3); + msa_sat_u_b(a, IMM3) } /// Immediate Unsigned Saturate @@ -7613,14 +7228,10 @@ pub unsafe fn __msa_sat_u_b(a: v16u8, imm3: i32) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sat_u.h, imm3 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_sat_u_h(a: v8u16, imm4: i32) -> v8u16 { - macro_rules! call { - ($imm4:expr) => { - msa_sat_u_h(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_sat_u_h(a: v8u16) -> v8u16 { + static_assert_imm4!(IMM4); + msa_sat_u_h(a, IMM4) } /// Immediate Unsigned Saturate @@ -7632,14 +7243,10 @@ pub unsafe fn __msa_sat_u_h(a: v8u16, imm4: i32) -> v8u16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sat_u.w, imm2 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_sat_u_w(a: v4u32, imm5: i32) -> v4u32 { - macro_rules! call { - ($imm5:expr) => { - msa_sat_u_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_sat_u_w(a: v4u32) -> v4u32 { + static_assert_imm5!(IMM5); + msa_sat_u_w(a, IMM5) } /// Immediate Unsigned Saturate @@ -7651,14 +7258,10 @@ pub unsafe fn __msa_sat_u_w(a: v4u32, imm5: i32) -> v4u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sat_u.d, imm1 = 0b111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_sat_u_d(a: v2u64, imm6: i32) -> v2u64 { - macro_rules! call { - ($imm6:expr) => { - msa_sat_u_d(a, $imm6) - }; - } - constify_imm6!(imm6, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_sat_u_d(a: v2u64) -> v2u64 { + static_assert_imm6!(IMM6); + msa_sat_u_d(a, IMM6) } /// Immediate Set Shuffle Elements @@ -7671,14 +7274,10 @@ pub unsafe fn __msa_sat_u_d(a: v2u64, imm6: i32) -> v2u64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(shf.b, imm8 = 0b11111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_shf_b(a: v16i8, imm8: i32) -> v16i8 { - macro_rules! call { - ($imm8:expr) => { - msa_shf_b(a, $imm8) - }; - } - constify_imm8!(imm8, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_shf_b(a: v16i8) -> v16i8 { + static_assert_imm8!(IMM8); + msa_shf_b(a, IMM8) } /// Immediate Set Shuffle Elements @@ -7691,14 +7290,10 @@ pub unsafe fn __msa_shf_b(a: v16i8, imm8: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(shf.h, imm8 = 0b11111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_shf_h(a: v8i16, imm8: i32) -> v8i16 { - macro_rules! call { - ($imm8:expr) => { - msa_shf_h(a, $imm8) - }; - } - constify_imm8!(imm8, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_shf_h(a: v8i16) -> v8i16 { + static_assert_imm8!(IMM8); + msa_shf_h(a, IMM8) } /// Immediate Set Shuffle Elements @@ -7711,14 +7306,10 @@ pub unsafe fn __msa_shf_h(a: v8i16, imm8: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(shf.w, imm8 = 0b11111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_shf_w(a: v4i32, imm8: i32) -> v4i32 { - macro_rules! call { - ($imm8:expr) => { - msa_shf_w(a, $imm8) - }; - } - constify_imm8!(imm8, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_shf_w(a: v4i32) -> v4i32 { + static_assert_imm8!(IMM8); + msa_shf_w(a, IMM8) } /// GPR Columns Slide @@ -7815,14 +7406,10 @@ pub unsafe fn __msa_sld_d(a: v2i64, b: v2i64, c: i32) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sldi.b, imm4 = 0b1111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_sldi_b(a: v16i8, b: v16i8, imm4: i32) -> v16i8 { - macro_rules! call { - ($imm4:expr) => { - msa_sldi_b(a, mem::transmute(b), $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_sldi_b(a: v16i8, b: v16i8) -> v16i8 { + static_assert_imm4!(IMM4); + msa_sldi_b(a, mem::transmute(b), IMM4) } /// Immediate Columns Slide @@ -7839,14 +7426,10 @@ pub unsafe fn __msa_sldi_b(a: v16i8, b: v16i8, imm4: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sldi.h, imm3 = 0b111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_sldi_h(a: v8i16, b: v8i16, imm3: i32) -> v8i16 { - macro_rules! call { - ($imm3:expr) => { - msa_sldi_h(a, mem::transmute(b), $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_sldi_h(a: v8i16, b: v8i16) -> v8i16 { + static_assert_imm3!(IMM3); + msa_sldi_h(a, mem::transmute(b), IMM3) } /// Immediate Columns Slide @@ -7863,14 +7446,10 @@ pub unsafe fn __msa_sldi_h(a: v8i16, b: v8i16, imm3: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sldi.w, imm2 = 0b11))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_sldi_w(a: v4i32, b: v4i32, imm2: i32) -> v4i32 { - macro_rules! call { - ($imm2:expr) => { - msa_sldi_w(a, mem::transmute(b), $imm2) - }; - } - constify_imm2!(imm2, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_sldi_w(a: v4i32, b: v4i32) -> v4i32 { + static_assert_imm2!(IMM2); + msa_sldi_w(a, mem::transmute(b), IMM2) } /// Immediate Columns Slide @@ -7887,14 +7466,10 @@ pub unsafe fn __msa_sldi_w(a: v4i32, b: v4i32, imm2: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sldi.d, imm1 = 0b1))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_sldi_d(a: v2i64, b: v2i64, imm1: i32) -> v2i64 { - macro_rules! call { - ($imm1:expr) => { - msa_sldi_d(a, mem::transmute(b), $imm1) - }; - } - constify_imm1!(imm1, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_sldi_d(a: v2i64, b: v2i64) -> v2i64 { + static_assert_imm1!(IMM1); + msa_sldi_d(a, mem::transmute(b), IMM1) } /// Vector Shift Left @@ -7962,14 +7537,10 @@ pub unsafe fn __msa_sll_d(a: v2i64, b: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(slli.b, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_slli_b(a: v16i8, imm4: i32) -> v16i8 { - macro_rules! call { - ($imm4:expr) => { - msa_slli_b(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_slli_b(a: v16i8) -> v16i8 { + static_assert_imm4!(IMM4); + msa_slli_b(a, IMM4) } /// Immediate Shift Left @@ -7981,14 +7552,10 @@ pub unsafe fn __msa_slli_b(a: v16i8, imm4: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(slli.h, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_slli_h(a: v8i16, imm3: i32) -> v8i16 { - macro_rules! call { - ($imm3:expr) => { - msa_slli_h(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_slli_h(a: v8i16) -> v8i16 { + static_assert_imm3!(IMM3); + msa_slli_h(a, IMM3) } /// Immediate Shift Left @@ -8000,14 +7567,10 @@ pub unsafe fn __msa_slli_h(a: v8i16, imm3: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(slli.w, imm2 = 0b11))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_slli_w(a: v4i32, imm2: i32) -> v4i32 { - macro_rules! call { - ($imm2:expr) => { - msa_slli_w(a, $imm2) - }; - } - constify_imm2!(imm2, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_slli_w(a: v4i32) -> v4i32 { + static_assert_imm2!(IMM2); + msa_slli_w(a, IMM2) } /// Immediate Shift Left @@ -8019,14 +7582,10 @@ pub unsafe fn __msa_slli_w(a: v4i32, imm2: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(slli.d, imm1 = 0b1))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_slli_d(a: v2i64, imm1: i32) -> v2i64 { - macro_rules! call { - ($imm1:expr) => { - msa_slli_d(a, $imm1) - }; - } - constify_imm1!(imm1, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_slli_d(a: v2i64) -> v2i64 { + static_assert_imm1!(IMM1); + msa_slli_d(a, IMM1) } /// GPR Element Splat @@ -8093,14 +7652,10 @@ pub unsafe fn __msa_splat_d(a: v2i64, b: i32) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(splati.b, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_splati_b(a: v16i8, imm4: i32) -> v16i8 { - macro_rules! call { - ($imm4:expr) => { - msa_splati_b(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_splati_b(a: v16i8) -> v16i8 { + static_assert_imm4!(IMM4); + msa_splati_b(a, IMM4) } /// Immediate Element Splat @@ -8111,14 +7666,10 @@ pub unsafe fn __msa_splati_b(a: v16i8, imm4: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(splati.h, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_splati_h(a: v8i16, imm3: i32) -> v8i16 { - macro_rules! call { - ($imm3:expr) => { - msa_splati_h(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_splati_h(a: v8i16) -> v8i16 { + static_assert_imm3!(IMM3); + msa_splati_h(a, IMM3) } /// Immediate Element Splat @@ -8129,14 +7680,10 @@ pub unsafe fn __msa_splati_h(a: v8i16, imm3: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(splati.w, imm2 = 0b11))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_splati_w(a: v4i32, imm2: i32) -> v4i32 { - macro_rules! call { - ($imm2:expr) => { - msa_splati_w(a, $imm2) - }; - } - constify_imm2!(imm2, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_splati_w(a: v4i32) -> v4i32 { + static_assert_imm2!(IMM2); + msa_splati_w(a, IMM2) } /// Immediate Element Splat @@ -8147,14 +7694,10 @@ pub unsafe fn __msa_splati_w(a: v4i32, imm2: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(splati.d, imm1 = 0b1))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_splati_d(a: v2i64, imm1: i32) -> v2i64 { - macro_rules! call { - ($imm1:expr) => { - msa_splati_d(a, $imm1) - }; - } - constify_imm1!(imm1, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_splati_d(a: v2i64) -> v2i64 { + static_assert_imm1!(IMM1); + msa_splati_d(a, IMM1) } /// Vector Shift Right Arithmetic @@ -8222,14 +7765,10 @@ pub unsafe fn __msa_sra_d(a: v2i64, b: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srai.b, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srai_b(a: v16i8, imm3: i32) -> v16i8 { - macro_rules! call { - ($imm3:expr) => { - msa_srai_b(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srai_b(a: v16i8) -> v16i8 { + static_assert_imm3!(IMM3); + msa_srai_b(a, IMM3) } /// Immediate Shift Right Arithmetic @@ -8241,14 +7780,10 @@ pub unsafe fn __msa_srai_b(a: v16i8, imm3: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srai.h, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srai_h(a: v8i16, imm4: i32) -> v8i16 { - macro_rules! call { - ($imm4:expr) => { - msa_srai_h(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srai_h(a: v8i16) -> v8i16 { + static_assert_imm4!(IMM4); + msa_srai_h(a, IMM4) } /// Immediate Shift Right Arithmetic @@ -8260,14 +7795,10 @@ pub unsafe fn __msa_srai_h(a: v8i16, imm4: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srai.w, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srai_w(a: v4i32, imm5: i32) -> v4i32 { - macro_rules! call { - ($imm5:expr) => { - msa_srai_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srai_w(a: v4i32) -> v4i32 { + static_assert_imm5!(IMM5); + msa_srai_w(a, IMM5) } /// Immediate Shift Right Arithmetic @@ -8279,14 +7810,10 @@ pub unsafe fn __msa_srai_w(a: v4i32, imm5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srai.d, imm6 = 0b111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srai_d(a: v2i64, imm6: i32) -> v2i64 { - macro_rules! call { - ($imm6:expr) => { - msa_srai_d(a, $imm6) - }; - } - constify_imm6!(imm6, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srai_d(a: v2i64) -> v2i64 { + static_assert_imm6!(IMM6); + msa_srai_d(a, IMM6) } /// Vector Shift Right Arithmetic Rounded @@ -8359,14 +7886,10 @@ pub unsafe fn __msa_srar_d(a: v2i64, b: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srari.b, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srari_b(a: v16i8, imm3: i32) -> v16i8 { - macro_rules! call { - ($imm3:expr) => { - msa_srari_b(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srari_b(a: v16i8) -> v16i8 { + static_assert_imm3!(IMM3); + msa_srari_b(a, IMM3) } /// Immediate Shift Right Arithmetic Rounded @@ -8379,14 +7902,10 @@ pub unsafe fn __msa_srari_b(a: v16i8, imm3: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srari.h, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srari_h(a: v8i16, imm4: i32) -> v8i16 { - macro_rules! call { - ($imm4:expr) => { - msa_srari_h(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srari_h(a: v8i16) -> v8i16 { + static_assert_imm4!(IMM4); + msa_srari_h(a, IMM4) } /// Immediate Shift Right Arithmetic Rounded @@ -8399,14 +7918,10 @@ pub unsafe fn __msa_srari_h(a: v8i16, imm4: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srari.w, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srari_w(a: v4i32, imm5: i32) -> v4i32 { - macro_rules! call { - ($imm5:expr) => { - msa_srari_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srari_w(a: v4i32) -> v4i32 { + static_assert_imm5!(IMM5); + msa_srari_w(a, IMM5) } /// Immediate Shift Right Arithmetic Rounded @@ -8419,14 +7934,10 @@ pub unsafe fn __msa_srari_w(a: v4i32, imm5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srari.d, imm6 = 0b111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srari_d(a: v2i64, imm6: i32) -> v2i64 { - macro_rules! call { - ($imm6:expr) => { - msa_srari_d(a, $imm6) - }; - } - constify_imm6!(imm6, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srari_d(a: v2i64) -> v2i64 { + static_assert_imm6!(IMM6); + msa_srari_d(a, IMM6) } /// Vector Shift Right Logical @@ -8494,14 +8005,10 @@ pub unsafe fn __msa_srl_d(a: v2i64, b: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srli.b, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srli_b(a: v16i8, imm4: i32) -> v16i8 { - macro_rules! call { - ($imm4:expr) => { - msa_srli_b(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srli_b(a: v16i8) -> v16i8 { + static_assert_imm4!(IMM4); + msa_srli_b(a, IMM4) } /// Immediate Shift Right Logical @@ -8513,14 +8020,10 @@ pub unsafe fn __msa_srli_b(a: v16i8, imm4: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srli.h, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srli_h(a: v8i16, imm3: i32) -> v8i16 { - macro_rules! call { - ($imm3:expr) => { - msa_srli_h(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srli_h(a: v8i16) -> v8i16 { + static_assert_imm3!(IMM3); + msa_srli_h(a, IMM3) } /// Immediate Shift Right Logical @@ -8532,14 +8035,10 @@ pub unsafe fn __msa_srli_h(a: v8i16, imm3: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srli.w, imm2 = 0b11))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srli_w(a: v4i32, imm2: i32) -> v4i32 { - macro_rules! call { - ($imm2:expr) => { - msa_srli_w(a, $imm2) - }; - } - constify_imm2!(imm2, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srli_w(a: v4i32) -> v4i32 { + static_assert_imm2!(IMM2); + msa_srli_w(a, IMM2) } /// Immediate Shift Right Logical @@ -8551,14 +8050,10 @@ pub unsafe fn __msa_srli_w(a: v4i32, imm2: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srli.d, imm1 = 0b1))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srli_d(a: v2i64, imm1: i32) -> v2i64 { - macro_rules! call { - ($imm1:expr) => { - msa_srli_d(a, $imm1) - }; - } - constify_imm1!(imm1, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srli_d(a: v2i64) -> v2i64 { + static_assert_imm1!(IMM1); + msa_srli_d(a, IMM1) } /// Vector Shift Right Logical Rounded @@ -8631,14 +8126,10 @@ pub unsafe fn __msa_srlr_d(a: v2i64, b: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srlri.b, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srlri_b(a: v16i8, imm3: i32) -> v16i8 { - macro_rules! call { - ($imm3:expr) => { - msa_srlri_b(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srlri_b(a: v16i8) -> v16i8 { + static_assert_imm3!(IMM3); + msa_srlri_b(a, IMM3) } /// Immediate Shift Right Logical Rounded @@ -8651,14 +8142,10 @@ pub unsafe fn __msa_srlri_b(a: v16i8, imm3: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srlri.h, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srlri_h(a: v8i16, imm4: i32) -> v8i16 { - macro_rules! call { - ($imm4:expr) => { - msa_srlri_h(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srlri_h(a: v8i16) -> v8i16 { + static_assert_imm4!(IMM4); + msa_srlri_h(a, IMM4) } /// Immediate Shift Right Logical Rounded @@ -8671,14 +8158,10 @@ pub unsafe fn __msa_srlri_h(a: v8i16, imm4: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srlri.w, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srlri_w(a: v4i32, imm5: i32) -> v4i32 { - macro_rules! call { - ($imm5:expr) => { - msa_srlri_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srlri_w(a: v4i32) -> v4i32 { + static_assert_imm5!(IMM5); + msa_srlri_w(a, IMM5) } /// Immediate Shift Right Logical Rounded @@ -8691,14 +8174,10 @@ pub unsafe fn __msa_srlri_w(a: v4i32, imm5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srlri.d, imm6 = 0b111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srlri_d(a: v2i64, imm6: i32) -> v2i64 { - macro_rules! call { - ($imm6:expr) => { - msa_srlri_d(a, $imm6) - }; - } - constify_imm6!(imm6, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srlri_d(a: v2i64) -> v2i64 { + static_assert_imm6!(IMM6); + msa_srlri_d(a, IMM6) } /// Vector Store @@ -8710,14 +8189,10 @@ pub unsafe fn __msa_srlri_d(a: v2i64, imm6: i32) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(st.b, imm_s10 = 0b1111111111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_st_b(a: v16i8, mem_addr: *mut u8, imm_s10: i32) -> () { - macro_rules! call { - ($imm_s10:expr) => { - msa_st_b(a, mem_addr, $imm_s10) - }; - } - constify_imm_s10!(imm_s10, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_st_b(a: v16i8, mem_addr: *mut u8) -> () { + static_assert_imm_s10!(IMM_S10); + msa_st_b(a, mem_addr, IMM_S10) } /// Vector Store @@ -8729,14 +8204,11 @@ pub unsafe fn __msa_st_b(a: v16i8, mem_addr: *mut u8, imm_s10: i32) -> () { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(st.h, imm_s11 = 0b11111111111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_st_h(a: v8i16, mem_addr: *mut u8, imm_s11: i32) -> () { - macro_rules! call { - ($imm_s11:expr) => { - msa_st_h(a, mem_addr, $imm_s11) - }; - } - constify_imm_s11!(imm_s11, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_st_h(a: v8i16, mem_addr: *mut u8) -> () { + static_assert_imm_s11!(IMM_S11); + static_assert!(IMM_S11: i32 where IMM_S11 % 2 == 0); + msa_st_h(a, mem_addr, IMM_S11) } /// Vector Store @@ -8748,14 +8220,11 @@ pub unsafe fn __msa_st_h(a: v8i16, mem_addr: *mut u8, imm_s11: i32) -> () { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(st.w, imm_s12 = 0b111111111111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_st_w(a: v4i32, mem_addr: *mut u8, imm_s12: i32) -> () { - macro_rules! call { - ($imm_s12:expr) => { - msa_st_w(a, mem_addr, $imm_s12) - }; - } - constify_imm_s12!(imm_s12, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_st_w(a: v4i32, mem_addr: *mut u8) -> () { + static_assert_imm_s12!(IMM_S12); + static_assert!(IMM_S12: i32 where IMM_S12 % 4 == 0); + msa_st_w(a, mem_addr, IMM_S12) } /// Vector Store @@ -8767,14 +8236,11 @@ pub unsafe fn __msa_st_w(a: v4i32, mem_addr: *mut u8, imm_s12: i32) -> () { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(st.d, imm_s13 = 0b1111111111111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_st_d(a: v2i64, mem_addr: *mut u8, imm_s13: i32) -> () { - macro_rules! call { - ($imm_s13:expr) => { - msa_st_d(a, mem_addr, $imm_s13) - }; - } - constify_imm_s13!(imm_s13, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_st_d(a: v2i64, mem_addr: *mut u8) -> () { + static_assert_imm_s13!(IMM_S13); + static_assert!(IMM_S13: i32 where IMM_S13 % 8 == 0); + msa_st_d(a, mem_addr, IMM_S13) } /// Vector Signed Saturated Subtract of Signed Values @@ -9062,14 +8528,10 @@ pub unsafe fn __msa_subv_d(a: v2i64, b: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(subvi.b, imm5 = 0b10111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_subvi_b(a: v16i8, imm5: i32) -> v16i8 { - macro_rules! call { - ($imm5:expr) => { - msa_subvi_b(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_subvi_b(a: v16i8) -> v16i8 { + static_assert_imm5!(IMM5); + msa_subvi_b(a, IMM5) } /// Immediate Subtract @@ -9081,14 +8543,10 @@ pub unsafe fn __msa_subvi_b(a: v16i8, imm5: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(subvi.h, imm5 = 0b10111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_subvi_h(a: v8i16, imm5: i32) -> v8i16 { - macro_rules! call { - ($imm5:expr) => { - msa_subvi_h(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_subvi_h(a: v8i16) -> v8i16 { + static_assert_imm5!(IMM5); + msa_subvi_h(a, IMM5) } /// Immediate Subtract @@ -9100,14 +8558,10 @@ pub unsafe fn __msa_subvi_h(a: v8i16, imm5: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(subvi.w, imm5 = 0b10111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_subvi_w(a: v4i32, imm5: i32) -> v4i32 { - macro_rules! call { - ($imm5:expr) => { - msa_subvi_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_subvi_w(a: v4i32) -> v4i32 { + static_assert_imm5!(IMM5); + msa_subvi_w(a, IMM5) } /// Immediate Subtract @@ -9119,14 +8573,10 @@ pub unsafe fn __msa_subvi_w(a: v4i32, imm5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(subvi.d, imm5 = 0b10111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_subvi_d(a: v2i64, imm5: i32) -> v2i64 { - macro_rules! call { - ($imm5:expr) => { - msa_subvi_d(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_subvi_d(a: v2i64) -> v2i64 { + static_assert_imm5!(IMM5); + msa_subvi_d(a, IMM5) } /// Vector Data Preserving Shuffle @@ -9221,14 +8671,10 @@ pub unsafe fn __msa_xor_v(a: v16u8, b: v16u8) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(xori.b, imm8 = 0b11111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_xori_b(a: v16u8, imm8: i32) -> v16u8 { - macro_rules! call { - ($imm8:expr) => { - msa_xori_b(a, $imm8) - }; - } - constify_imm8!(imm8, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_xori_b(a: v16u8) -> v16u8 { + static_assert_imm8!(IMM8); + msa_xori_b(a, IMM8) } #[cfg(test)] @@ -9319,10 +8765,10 @@ mod tests { unsafe fn test_msa_adds_a_b() { #[rustfmt::skip] let a = i8x16::new( - 100, i8::max_value(), 100, i8::max_value(), - 100, i8::max_value(), 100, i8::max_value(), - 100, i8::max_value(), 100, i8::max_value(), - 100, i8::max_value(), 100, i8::max_value() + 100, i8::MAX, 100, i8::MAX, + 100, i8::MAX, 100, i8::MAX, + 100, i8::MAX, 100, i8::MAX, + 100, i8::MAX, 100, i8::MAX ); #[rustfmt::skip] let b = i8x16::new( @@ -9349,15 +8795,15 @@ mod tests { unsafe fn test_msa_adds_a_h() { #[rustfmt::skip] let a = i16x8::new( - 100, i16::max_value(), 100, i16::max_value(), - 100, i16::max_value(), 100, i16::max_value() + 100, i16::MAX, 100, i16::MAX, + 100, i16::MAX, 100, i16::MAX ); #[rustfmt::skip] let b = i16x8::new(-4, -3, -2, -1, -4, -3, -2, -1); #[rustfmt::skip] let r = i16x8::new( - 104, i16::max_value(), 102, i16::max_value(), - 104, i16::max_value(), 102, i16::max_value() + 104, i16::MAX, 102, i16::MAX, + 104, i16::MAX, 102, i16::MAX ); assert_eq!( @@ -9369,11 +8815,11 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_adds_a_w() { #[rustfmt::skip] - let a = i32x4::new(100, i32::max_value(), 100, i32::max_value()); + let a = i32x4::new(100, i32::MAX, 100, i32::MAX); #[rustfmt::skip] let b = i32x4::new(-4, -3, -2, -1); #[rustfmt::skip] - let r = i32x4::new(104, i32::max_value(), 102, i32::max_value()); + let r = i32x4::new(104, i32::MAX, 102, i32::MAX); assert_eq!( r, @@ -9384,11 +8830,11 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_adds_a_d() { #[rustfmt::skip] - let a = i64x2::new(100, i64::max_value()); + let a = i64x2::new(100, i64::MAX); #[rustfmt::skip] let b = i64x2::new(-4, -3); #[rustfmt::skip] - let r = i64x2::new(104, i64::max_value()); + let r = i64x2::new(104, i64::MAX); assert_eq!( r, @@ -9400,10 +8846,10 @@ mod tests { unsafe fn test_msa_adds_s_b() { #[rustfmt::skip] let a = i8x16::new( - 100, i8::min_value(), 100, i8::max_value(), - 100, i8::min_value(), 100, i8::max_value(), - 100, i8::min_value(), 100, i8::max_value(), - 100, i8::min_value(), 100, i8::max_value() + 100, i8::MIN, 100, i8::MAX, + 100, i8::MIN, 100, i8::MAX, + 100, i8::MIN, 100, i8::MAX, + 100, i8::MIN, 100, i8::MAX ); #[rustfmt::skip] let b = i8x16::new( @@ -9414,10 +8860,10 @@ mod tests { ); #[rustfmt::skip] let r = i8x16::new( - 96, i8::min_value(), 98, i8::max_value(), - 96, i8::min_value(), 98, i8::max_value(), - 96, i8::min_value(), 98, i8::max_value(), - 96, i8::min_value(), 98, i8::max_value() + 96, i8::MIN, 98, i8::MAX, + 96, i8::MIN, 98, i8::MAX, + 96, i8::MIN, 98, i8::MAX, + 96, i8::MIN, 98, i8::MAX ); assert_eq!( @@ -9430,15 +8876,15 @@ mod tests { unsafe fn test_msa_adds_s_h() { #[rustfmt::skip] let a = i16x8::new( - 100, i16::min_value(), 100, i16::max_value(), - 100, i16::min_value(), 100, i16::max_value() + 100, i16::MIN, 100, i16::MAX, + 100, i16::MIN, 100, i16::MAX ); #[rustfmt::skip] let b = i16x8::new(-4, -3, -2, 1, -4, -3, -2, 1); #[rustfmt::skip] let r = i16x8::new( - 96, i16::min_value(), 98, i16::max_value(), - 96, i16::min_value(), 98, i16::max_value() + 96, i16::MIN, 98, i16::MAX, + 96, i16::MIN, 98, i16::MAX ); assert_eq!( @@ -9450,11 +8896,11 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_adds_s_w() { #[rustfmt::skip] - let a = i32x4::new(100, i32::max_value(), 100, i32::min_value()); + let a = i32x4::new(100, i32::MAX, 100, i32::MIN); #[rustfmt::skip] let b = i32x4::new(-4, 3, -2, -1); #[rustfmt::skip] - let r = i32x4::new(96, i32::max_value(), 98, i32::min_value()); + let r = i32x4::new(96, i32::MAX, 98, i32::MIN); assert_eq!( r, @@ -9465,11 +8911,11 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_adds_s_d() { #[rustfmt::skip] - let a = i64x2::new(100, i64::min_value()); + let a = i64x2::new(100, i64::MIN); #[rustfmt::skip] let b = i64x2::new(-4, -3); #[rustfmt::skip] - let r = i64x2::new(96, i64::min_value()); + let r = i64x2::new(96, i64::MIN); assert_eq!( r, @@ -9481,10 +8927,10 @@ mod tests { unsafe fn test_msa_adds_u_b() { #[rustfmt::skip] let a = u8x16::new( - 100, u8::max_value(), 100, u8::max_value(), - 100, u8::max_value(), 100, u8::max_value(), - 100, u8::max_value(), 100, u8::max_value(), - 100, u8::max_value(), 100, u8::max_value() + 100, u8::MAX, 100, u8::MAX, + 100, u8::MAX, 100, u8::MAX, + 100, u8::MAX, 100, u8::MAX, + 100, u8::MAX, 100, u8::MAX ); #[rustfmt::skip] let b = u8x16::new( @@ -9495,10 +8941,10 @@ mod tests { ); #[rustfmt::skip] let r = u8x16::new( - 104, u8::max_value(), 102, u8::max_value(), - 104, u8::max_value(), 102, u8::max_value(), - 104, u8::max_value(), 102, u8::max_value(), - 104, u8::max_value(), 102, u8::max_value() + 104, u8::MAX, 102, u8::MAX, + 104, u8::MAX, 102, u8::MAX, + 104, u8::MAX, 102, u8::MAX, + 104, u8::MAX, 102, u8::MAX ); assert_eq!( @@ -9511,15 +8957,15 @@ mod tests { unsafe fn test_msa_adds_u_h() { #[rustfmt::skip] let a = u16x8::new( - 100, u16::max_value(), 100, u16::max_value(), - 100, u16::max_value(), 100, u16::max_value() + 100, u16::MAX, 100, u16::MAX, + 100, u16::MAX, 100, u16::MAX ); #[rustfmt::skip] let b = u16x8::new(4, 3, 2, 1, 4, 3, 2, 1); #[rustfmt::skip] let r = u16x8::new( - 104, u16::max_value(), 102, u16::max_value(), - 104, u16::max_value(), 102, u16::max_value() + 104, u16::MAX, 102, u16::MAX, + 104, u16::MAX, 102, u16::MAX ); assert_eq!( @@ -9531,11 +8977,11 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_adds_u_w() { #[rustfmt::skip] - let a = u32x4::new(100, u32::max_value(), 100, u32::max_value()); + let a = u32x4::new(100, u32::MAX, 100, u32::MAX); #[rustfmt::skip] let b = u32x4::new(4, 3, 2, 1); #[rustfmt::skip] - let r = u32x4::new(104, u32::max_value(), 102, u32::max_value()); + let r = u32x4::new(104, u32::MAX, 102, u32::MAX); assert_eq!( r, @@ -9546,11 +8992,11 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_adds_u_d() { #[rustfmt::skip] - let a = u64x2::new(100, u64::max_value()); + let a = u64x2::new(100, u64::MAX); #[rustfmt::skip] let b = u64x2::new(4, 3); #[rustfmt::skip] - let r = u64x2::new(104, u64::max_value()); + let r = u64x2::new(104, u64::MAX); assert_eq!( r, @@ -9562,10 +9008,10 @@ mod tests { unsafe fn test_msa_addv_b() { #[rustfmt::skip] let a = i8x16::new( - 100, i8::min_value(), 100, i8::max_value(), - 100, i8::min_value(), 100, i8::max_value(), - 100, i8::min_value(), 100, i8::max_value(), - 100, i8::min_value(), 100, i8::max_value() + 100, i8::MIN, 100, i8::MAX, + 100, i8::MIN, 100, i8::MAX, + 100, i8::MIN, 100, i8::MAX, + 100, i8::MIN, 100, i8::MAX ); #[rustfmt::skip] let b = i8x16::new( @@ -9592,8 +9038,8 @@ mod tests { unsafe fn test_msa_addv_h() { #[rustfmt::skip] let a = i16x8::new( - 100, i16::min_value(), 100, i16::max_value(), - 100, i16::min_value(), 100, i16::max_value() + 100, i16::MIN, 100, i16::MAX, + 100, i16::MIN, 100, i16::MAX ); #[rustfmt::skip] let b = i16x8::new(-4, -3, -2, 1, -4, -3, -2, 1); @@ -9609,7 +9055,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_addv_w() { #[rustfmt::skip] - let a = i32x4::new(100, i32::max_value(), 100, i32::min_value()); + let a = i32x4::new(100, i32::MAX, 100, i32::MIN); #[rustfmt::skip] let b = i32x4::new(-4, 3, -2, -1); #[rustfmt::skip] @@ -9624,7 +9070,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_addv_d() { #[rustfmt::skip] - let a = i64x2::new(100, i64::min_value()); + let a = i64x2::new(100, i64::MIN); #[rustfmt::skip] let b = i64x2::new(-4, -3); #[rustfmt::skip] @@ -9640,10 +9086,10 @@ mod tests { unsafe fn test_msa_addvi_b() { #[rustfmt::skip] let a = i8x16::new( - 100, i8::max_value(), 100, i8::max_value(), - 100, i8::max_value(), 100, i8::max_value(), - 100, i8::max_value(), 100, i8::max_value(), - 100, i8::max_value(), 100, i8::max_value() + 100, i8::MAX, 100, i8::MAX, + 100, i8::MAX, 100, i8::MAX, + 100, i8::MAX, 100, i8::MAX, + 100, i8::MAX, 100, i8::MAX ); #[rustfmt::skip] let r = i8x16::new( @@ -9660,8 +9106,8 @@ mod tests { unsafe fn test_msa_addvi_h() { #[rustfmt::skip] let a = i16x8::new( - i16::max_value(), 3276, -100, -127, - i16::max_value(), 3276, -100, -127 + i16::MAX, 3276, -100, -127, + i16::MAX, 3276, -100, -127 ); #[rustfmt::skip] let r = i16x8::new( @@ -9675,7 +9121,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_addvi_w() { #[rustfmt::skip] - let a = i32x4::new(100, i32::max_value(), 100, i32::min_value()); + let a = i32x4::new(100, i32::MAX, 100, i32::MIN); #[rustfmt::skip] let r = i32x4::new(103, -2147483646, 103, -2147483645); @@ -9685,7 +9131,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_addvi_d() { #[rustfmt::skip] - let a = i64x2::new(100, i64::min_value()); + let a = i64x2::new(100, i64::MIN); #[rustfmt::skip] let r = i64x2::new(117, -9223372036854775791); @@ -9696,10 +9142,10 @@ mod tests { unsafe fn test_msa_and_v() { #[rustfmt::skip] let a = u8x16::new( - 100, u8::max_value(), 100, u8::max_value(), - 100, u8::max_value(), 100, u8::max_value(), - 100, u8::max_value(), 100, u8::max_value(), - 100, u8::max_value(), 100, u8::max_value() + 100, u8::MAX, 100, u8::MAX, + 100, u8::MAX, 100, u8::MAX, + 100, u8::MAX, 100, u8::MAX, + 100, u8::MAX, 100, u8::MAX ); #[rustfmt::skip] let b = u8x16::new( @@ -9726,10 +9172,10 @@ mod tests { unsafe fn test_msa_andi_b() { #[rustfmt::skip] let a = u8x16::new( - 100, u8::max_value(), 100, u8::max_value(), - 100, u8::max_value(), 100, u8::max_value(), - 100, u8::max_value(), 100, u8::max_value(), - 100, u8::max_value(), 100, u8::max_value() + 100, u8::MAX, 100, u8::MAX, + 100, u8::MAX, 100, u8::MAX, + 100, u8::MAX, 100, u8::MAX, + 100, u8::MAX, 100, u8::MAX ); #[rustfmt::skip] let r = u8x16::new( @@ -10437,10 +9883,10 @@ mod tests { unsafe fn test_msa_binsli_b() { #[rustfmt::skip] let a = u8x16::new( - u8::max_value(), 155, 55, 1, - u8::max_value(), 155, 55, 1, - u8::max_value(), 155, 55, 1, - u8::max_value(), 155, 55, 1 + u8::MAX, 155, 55, 1, + u8::MAX, 155, 55, 1, + u8::MAX, 155, 55, 1, + u8::MAX, 155, 55, 1 ); #[rustfmt::skip] let b = u8x16::new( @@ -10721,10 +10167,10 @@ mod tests { unsafe fn test_msa_bmnz_v() { #[rustfmt::skip] let a = u8x16::new( - u8::max_value(), 155, 55, 1, - u8::max_value(), 155, 55, 1, - u8::max_value(), 155, 55, 1, - u8::max_value(), 155, 55, 1 + u8::MAX, 155, 55, 1, + u8::MAX, 155, 55, 1, + u8::MAX, 155, 55, 1, + u8::MAX, 155, 55, 1 ); #[rustfmt::skip] let b = u8x16::new( @@ -10762,17 +10208,17 @@ mod tests { unsafe fn test_msa_bmnzi_b() { #[rustfmt::skip] let a = u8x16::new( - u8::max_value(), 155, 55, 1, - u8::max_value(), 155, 55, 1, - u8::max_value(), 155, 55, 1, - u8::max_value(), 155, 55, 1 + u8::MAX, 155, 55, 1, + u8::MAX, 155, 55, 1, + u8::MAX, 155, 55, 1, + u8::MAX, 155, 55, 1 ); #[rustfmt::skip] let b = u8x16::new( - 1, u8::max_value(), 155, 55, - 1, u8::max_value(), 155, 55, - 1, u8::max_value(), 155, 55, - 1, u8::max_value(), 155, 55 + 1, u8::MAX, 155, 55, + 1, u8::MAX, 155, 55, + 1, u8::MAX, 155, 55, + 1, u8::MAX, 155, 55 ); #[rustfmt::skip] let r = u8x16::new( @@ -10792,10 +10238,10 @@ mod tests { unsafe fn test_msa_bmz_v() { #[rustfmt::skip] let a = u8x16::new( - u8::max_value(), 155, 55, 1, - u8::max_value(), 155, 55, 1, - u8::max_value(), 155, 55, 1, - u8::max_value(), 155, 55, 1 + u8::MAX, 155, 55, 1, + u8::MAX, 155, 55, 1, + u8::MAX, 155, 55, 1, + u8::MAX, 155, 55, 1 ); #[rustfmt::skip] let b = u8x16::new( @@ -10833,10 +10279,10 @@ mod tests { unsafe fn test_msa_bmzi_b() { #[rustfmt::skip] let a = u8x16::new( - u8::max_value(), 155, 55, 1, - u8::max_value(), 155, 55, 1, - u8::max_value(), 155, 55, 1, - u8::max_value(), 155, 55, 1 + u8::MAX, 155, 55, 1, + u8::MAX, 155, 55, 1, + u8::MAX, 155, 55, 1, + u8::MAX, 155, 55, 1 ); #[rustfmt::skip] let b = u8x16::new( @@ -10938,10 +10384,10 @@ mod tests { unsafe fn test_msa_bnegi_b() { #[rustfmt::skip] let a = u8x16::new( - 50, 100, 127, u8::max_value(), - 50, 100, 127, u8::max_value(), - 50, 100, 127, u8::max_value(), - 50, 100, 127, u8::max_value() + 50, 100, 127, u8::MAX, + 50, 100, 127, u8::MAX, + 50, 100, 127, u8::MAX, + 50, 100, 127, u8::MAX ); #[rustfmt::skip] let r = u8x16::new( @@ -11518,17 +10964,17 @@ mod tests { unsafe fn test_msa_cle_u_b() { #[rustfmt::skip] let a = u8x16::new( - u8::max_value(), 127, 55, 2, - u8::max_value(), 127, 55, 2, - u8::max_value(), 127, 55, 2, - u8::max_value(), 127, 55, 2 + u8::MAX, 127, 55, 2, + u8::MAX, 127, 55, 2, + u8::MAX, 127, 55, 2, + u8::MAX, 127, 55, 2 ); #[rustfmt::skip] let b = u8x16::new( - u8::max_value(), 126, 55, 1, - u8::max_value(), 126, 55, 1, - u8::max_value(), 126, 55, 1, - u8::max_value(), 126, 55, 1 + u8::MAX, 126, 55, 1, + u8::MAX, 126, 55, 1, + u8::MAX, 126, 55, 1, + u8::MAX, 126, 55, 1 ); #[rustfmt::skip] let r = i8x16::new(-1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0); @@ -11543,13 +10989,13 @@ mod tests { unsafe fn test_msa_cle_u_h() { #[rustfmt::skip] let a = u16x8::new( - u16::max_value(), 155, 55, 2, - u16::max_value(), 155, 55, 2 + u16::MAX, 155, 55, 2, + u16::MAX, 155, 55, 2 ); #[rustfmt::skip] let b = u16x8::new( - u16::max_value(), 155, 56, 1, - u16::max_value(), 155, 56, 1 + u16::MAX, 155, 56, 1, + u16::MAX, 155, 56, 1 ); #[rustfmt::skip] let r = i16x8::new(-1, -1, -1, 0, -1, -1, -1, 0); @@ -11563,9 +11009,9 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_cle_u_w() { #[rustfmt::skip] - let a = u32x4::new(u32::max_value(), 155, 55, 2); + let a = u32x4::new(u32::MAX, 155, 55, 2); #[rustfmt::skip] - let b = u32x4::new(u32::max_value(), 156, 55, 1); + let b = u32x4::new(u32::MAX, 156, 55, 1); #[rustfmt::skip] let r = i32x4::new(-1, -1, -1, 0); @@ -11578,9 +11024,9 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_cle_u_d() { #[rustfmt::skip] - let a = u64x2::new(u64::max_value(), 155); + let a = u64x2::new(u64::MAX, 155); #[rustfmt::skip] - let b = u64x2::new(u64::max_value(), 156); + let b = u64x2::new(u64::MAX, 156); #[rustfmt::skip] let r = i64x2::new(-1, -1); @@ -12041,7 +11487,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_copy_u_d() { #[rustfmt::skip] - let a = i64x2::new(3, i64::max_value()); + let a = i64x2::new(3, i64::MAX); #[rustfmt::skip] let r = 9223372036854775807 as u64; @@ -14701,7 +14147,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_insert_d() { #[rustfmt::skip] - let a = i64x2::new(3, i64::max_value()); + let a = i64x2::new(3, i64::MAX); #[rustfmt::skip] let r = i64x2::new(3, 100); @@ -14712,10 +14158,10 @@ mod tests { unsafe fn test_msa_insve_b() { #[rustfmt::skip] let a = i8x16::new( - -100, i8::max_value(), 4, i8::max_value(), - -100, i8::max_value(), 4, i8::max_value(), - -100, i8::max_value(), 4, i8::max_value(), - -100, i8::max_value(), 4, i8::max_value() + -100, i8::MAX, 4, i8::MAX, + -100, i8::MAX, 4, i8::MAX, + -100, i8::MAX, 4, i8::MAX, + -100, i8::MAX, 4, i8::MAX ); #[rustfmt::skip] let b = i8x16::new( @@ -14742,8 +14188,8 @@ mod tests { unsafe fn test_msa_insve_h() { #[rustfmt::skip] let a = i16x8::new( - i16::max_value(), 3276, 100, 11, - i16::max_value(), 3276, 100, 11 + i16::MAX, 3276, 100, 11, + i16::MAX, 3276, 100, 11 ); #[rustfmt::skip] let b = i16x8::new( @@ -14780,7 +14226,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_insve_d() { #[rustfmt::skip] - let a = i64x2::new(3, i64::max_value()); + let a = i64x2::new(3, i64::MAX); #[rustfmt::skip] let b = i64x2::new(1, 2); #[rustfmt::skip] @@ -14895,7 +14341,7 @@ mod tests { unsafe fn test_msa_madd_q_h() { #[rustfmt::skip] let a = i16x8::new( - i16::max_value(), 1024, i16::min_value(), -1024, + i16::MAX, 1024, i16::MIN, -1024, 1, 2, 3, 4 ); #[rustfmt::skip] @@ -14905,7 +14351,7 @@ mod tests { ); #[rustfmt::skip] let c = i16x8::new( - i16::max_value(), i16::max_value(), 1, -1, + i16::MAX, i16::MAX, 1, -1, 33, 66, 99, 132 ); #[rustfmt::skip] @@ -14924,7 +14370,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_madd_q_w() { #[rustfmt::skip] - let a = i32x4::new(i32::max_value(), i32::min_value(), 1, 2); + let a = i32x4::new(i32::MAX, i32::MIN, 1, 2); #[rustfmt::skip] let b = i32x4::new(102401, 102401, 102401, 102401); #[rustfmt::skip] @@ -14975,7 +14421,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_maddr_q_w() { #[rustfmt::skip] - let a = i32x4::new(i32::max_value(), i32::min_value(), 1, 2); + let a = i32x4::new(i32::MAX, i32::MIN, 1, 2); #[rustfmt::skip] let b = i32x4::new(102401, 102401, 102401, 102401); #[rustfmt::skip] @@ -16007,7 +15453,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_msubr_q_w() { #[rustfmt::skip] - let a = i32x4::new(i32::max_value(), -2147483647, 1, 2); + let a = i32x4::new(i32::MAX, -2147483647, 1, 2); #[rustfmt::skip] let b = i32x4::new(10240, 10240, 10240, 10240); #[rustfmt::skip] @@ -16154,8 +15600,8 @@ mod tests { unsafe fn test_msa_mul_q_w() { #[rustfmt::skip] let a = i32x4::new( - i32::max_value(), i32::max_value(), - i32::min_value(), i32::min_value() + i32::MAX, i32::MAX, + i32::MIN, i32::MIN ); #[rustfmt::skip] let b = i32x4::new(30, 60, 30, 60); @@ -16193,8 +15639,8 @@ mod tests { unsafe fn test_msa_mulr_q_w() { #[rustfmt::skip] let a = i32x4::new( - i32::max_value(), i32::max_value(), - i32::min_value(), i32::min_value() + i32::MAX, i32::MAX, + i32::MIN, i32::MIN ); #[rustfmt::skip] let b = i32x4::new(30, 60, 30, 60); @@ -16325,8 +15771,8 @@ mod tests { unsafe fn test_msa_nloc_w() { #[rustfmt::skip] let a = i32x4::new( - i32::min_value(), -1073741824, - 1073741824, i32::max_value() + i32::MIN, -1073741824, + 1073741824, i32::MAX ); #[rustfmt::skip] let r = i32x4::new(1, 2, 0, 0); @@ -16337,7 +15783,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_nloc_d() { #[rustfmt::skip] - let a = i64x2::new(i64::min_value(), i64::max_value()); + let a = i64x2::new(i64::MIN, i64::MAX); #[rustfmt::skip] let r = i64x2::new(1, 0); @@ -16684,8 +16130,8 @@ mod tests { unsafe fn test_msa_pcnt_w() { #[rustfmt::skip] let a = i32x4::new( - i32::min_value(), -1073741824, - 1073741824, i32::max_value() + i32::MIN, -1073741824, + 1073741824, i32::MAX ); #[rustfmt::skip] let r = i32x4::new(1, 2, 1, 31); @@ -16707,10 +16153,10 @@ mod tests { unsafe fn test_msa_sat_s_b() { #[rustfmt::skip] let a = i8x16::new( - i8::max_value(), 105, 30, 1, - i8::max_value(), 105, 30, 1, - i8::max_value(), 105, 30, 1, - i8::max_value(), 105, 30, 1 + i8::MAX, 105, 30, 1, + i8::MAX, 105, 30, 1, + i8::MAX, 105, 30, 1, + i8::MAX, 105, 30, 1 ); #[rustfmt::skip] let r = i8x16::new( @@ -16727,8 +16173,8 @@ mod tests { unsafe fn test_msa_sat_s_h() { #[rustfmt::skip] let a = i16x8::new( - i16::max_value(), 1155, 155, 1, - i16::max_value(), 1155, 155, 1 + i16::MAX, 1155, 155, 1, + i16::MAX, 1155, 155, 1 ); #[rustfmt::skip] let r = i16x8::new(127, 127, 127, 1, 127, 127, 127, 1); @@ -16739,7 +16185,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_sat_s_w() { #[rustfmt::skip] - let a = i32x4::new(i32::max_value(), 111111155, i32::max_value(), 1); + let a = i32x4::new(i32::MAX, 111111155, i32::MAX, 1); #[rustfmt::skip] let r = i32x4::new(131071, 131071, 131071, 1); @@ -16749,7 +16195,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_sat_s_d() { #[rustfmt::skip] - let a = i64x2::new(i64::max_value(), 1); + let a = i64x2::new(i64::MAX, 1); #[rustfmt::skip] let r = i64x2::new(137438953471, 1); @@ -16760,10 +16206,10 @@ mod tests { unsafe fn test_msa_sat_u_b() { #[rustfmt::skip] let a = u8x16::new( - u8::max_value(), 105, 30, 1, - u8::max_value(), 105, 30, 1, - u8::max_value(), 105, 30, 1, - u8::max_value(), 105, 30, 1 + u8::MAX, 105, 30, 1, + u8::MAX, 105, 30, 1, + u8::MAX, 105, 30, 1, + u8::MAX, 105, 30, 1 ); #[rustfmt::skip] let r = u8x16::new( @@ -16780,8 +16226,8 @@ mod tests { unsafe fn test_msa_sat_u_h() { #[rustfmt::skip] let a = u16x8::new( - u16::max_value(), 1155, 155, 1, - u16::max_value(), 1155, 155, 1 + u16::MAX, 1155, 155, 1, + u16::MAX, 1155, 155, 1 ); #[rustfmt::skip] let r = u16x8::new(255, 255, 155, 1, 255, 255, 155, 1); @@ -16792,7 +16238,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_sat_u_w() { #[rustfmt::skip] - let a = u32x4::new(u32::max_value(), 111111155, u32::max_value(), 1); + let a = u32x4::new(u32::MAX, 111111155, u32::MAX, 1); #[rustfmt::skip] let r = u32x4::new(262143, 262143, 262143, 1); @@ -16802,7 +16248,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_sat_u_d() { #[rustfmt::skip] - let a = u64x2::new(u64::max_value(), 1); + let a = u64x2::new(u64::MAX, 1); #[rustfmt::skip] let r = u64x2::new(274877906943, 1); @@ -17293,7 +16739,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_sra_w() { #[rustfmt::skip] - let a = i32x4::new(i32::min_value(), -1073741824, 1, 2); + let a = i32x4::new(i32::MIN, -1073741824, 1, 2); #[rustfmt::skip] let b = i32x4::new(16, 15, 16, 15); #[rustfmt::skip] @@ -17308,7 +16754,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_sra_d() { #[rustfmt::skip] - let a = i64x2::new(i64::min_value(), i64::max_value()); + let a = i64x2::new(i64::MIN, i64::MAX); #[rustfmt::skip] let b = i64x2::new(32, 31); #[rustfmt::skip] @@ -17324,10 +16770,10 @@ mod tests { unsafe fn test_msa_srai_b() { #[rustfmt::skip] let a = i8x16::new( - i8::max_value(), 125, 55, 1, - i8::max_value(), 125, 55, 1, - i8::max_value(), 125, 55, 1, - i8::max_value(), 125, 55, 1 + i8::MAX, 125, 55, 1, + i8::MAX, 125, 55, 1, + i8::MAX, 125, 55, 1, + i8::MAX, 125, 55, 1 ); #[rustfmt::skip] let r = i8x16::new( @@ -17344,8 +16790,8 @@ mod tests { unsafe fn test_msa_srai_h() { #[rustfmt::skip] let a = i16x8::new( - i16::max_value(), 125, 55, 1, - i16::max_value(), 125, 55, 1 + i16::MAX, 125, 55, 1, + i16::MAX, 125, 55, 1 ); #[rustfmt::skip] let r = i16x8::new(8191, 31, 13, 0, 8191, 31, 13, 0); @@ -17356,7 +16802,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_srai_w() { #[rustfmt::skip] - let a = i32x4::new(i32::max_value(), 125, 55, 1); + let a = i32x4::new(i32::MAX, 125, 55, 1); let r = i32x4::new(536870911, 31, 13, 0); assert_eq!(r, mem::transmute(__msa_srai_w(mem::transmute(a), 2))); @@ -17365,7 +16811,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_srai_d() { #[rustfmt::skip] - let a = i64x2::new(i64::max_value(), 55); + let a = i64x2::new(i64::MAX, 55); #[rustfmt::skip] let r = i64x2::new(2305843009213693951, 13); @@ -17406,7 +16852,7 @@ mod tests { unsafe fn test_msa_srar_h() { #[rustfmt::skip] let a = i16x8::new( - i16::min_value(), -16384, -8192, -4096, + i16::MIN, -16384, -8192, -4096, 150, 50, 25, 15 ); #[rustfmt::skip] @@ -17429,7 +16875,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_srar_w() { #[rustfmt::skip] - let a = i32x4::new(i32::min_value(), -1073741824, 100, 50); + let a = i32x4::new(i32::MIN, -1073741824, 100, 50); #[rustfmt::skip] let b = i32x4::new(16, 15, 1, 2); #[rustfmt::skip] @@ -17444,7 +16890,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_srar_d() { #[rustfmt::skip] - let a = i64x2::new(i64::min_value(), i64::max_value()); + let a = i64x2::new(i64::MIN, i64::MAX); #[rustfmt::skip] let b = i64x2::new(32, 31); #[rustfmt::skip] @@ -17460,10 +16906,10 @@ mod tests { unsafe fn test_msa_srari_b() { #[rustfmt::skip] let a = i8x16::new( - 125, i8::max_value(), 55, 1, - 125, i8::max_value(), 55, 1, - 125, i8::max_value(), 55, 1, - 125, i8::max_value(), 55, 1 + 125, i8::MAX, 55, 1, + 125, i8::MAX, 55, 1, + 125, i8::MAX, 55, 1, + 125, i8::MAX, 55, 1 ); #[rustfmt::skip] let r = i8x16::new( @@ -17560,7 +17006,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_srl_w() { #[rustfmt::skip] - let a = i32x4::new(i32::min_value(), -1073741824, 1, 2); + let a = i32x4::new(i32::MIN, -1073741824, 1, 2); #[rustfmt::skip] let b = i32x4::new(16, 15, 16, 15); #[rustfmt::skip] @@ -17575,7 +17021,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_srl_d() { #[rustfmt::skip] - let a = i64x2::new(i64::min_value(), i64::max_value()); + let a = i64x2::new(i64::MIN, i64::MAX); #[rustfmt::skip] let b = i64x2::new(32, 31); #[rustfmt::skip] @@ -17611,8 +17057,8 @@ mod tests { unsafe fn test_msa_srli_h() { #[rustfmt::skip] let a = i16x8::new( - i16::max_value(), 3276, 100, 127, - i16::max_value(), 3276, 100, 127 + i16::MAX, 3276, 100, 127, + i16::MAX, 3276, 100, 127 ); #[rustfmt::skip] let r = i16x8::new( @@ -17626,7 +17072,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_srli_w() { #[rustfmt::skip] - let a = i32x4::new(100, i32::max_value(), 100, i32::max_value()); + let a = i32x4::new(100, i32::MAX, 100, i32::MAX); #[rustfmt::skip] let r = i32x4::new(25, 536870911, 25, 536870911); @@ -17636,7 +17082,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_srli_d() { #[rustfmt::skip] - let a = i64x2::new(100, i64::max_value()); + let a = i64x2::new(100, i64::MAX); #[rustfmt::skip] let r = i64x2::new(50, 4611686018427387903); @@ -17697,7 +17143,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_srlr_w() { #[rustfmt::skip] - let a = i32x4::new(i32::min_value(), -1073741824, 1, 2); + let a = i32x4::new(i32::MIN, -1073741824, 1, 2); #[rustfmt::skip] let b = i32x4::new(16, 15, 16, 15); let r = i32x4::new(32768, 98304, 0, 0); @@ -17711,7 +17157,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_srlr_d() { #[rustfmt::skip] - let a = i64x2::new(i64::min_value(), i64::max_value()); + let a = i64x2::new(i64::MIN, i64::MAX); #[rustfmt::skip] let b = i64x2::new(32, 31); #[rustfmt::skip] @@ -17727,10 +17173,10 @@ mod tests { unsafe fn test_msa_srlri_b() { #[rustfmt::skip] let a = i8x16::new( - 25, 50, 100, i8::max_value(), - 25, 50, 100, i8::max_value(), - 25, 50, 100, i8::max_value(), - 25, 50, 100, i8::max_value() + 25, 50, 100, i8::MAX, + 25, 50, 100, i8::MAX, + 25, 50, 100, i8::MAX, + 25, 50, 100, i8::MAX ); #[rustfmt::skip] let r = i8x16::new( @@ -17747,8 +17193,8 @@ mod tests { unsafe fn test_msa_srlri_h() { #[rustfmt::skip] let a = i16x8::new( - i16::max_value(), 3276, 100, 127, - i16::max_value(), 3276, 100, 127 + i16::MAX, 3276, 100, 127, + i16::MAX, 3276, 100, 127 ); let r = i16x8::new(8192, 819, 25, 32, 8192, 819, 25, 32); @@ -17758,7 +17204,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_srlri_w() { #[rustfmt::skip] - let a = i32x4::new(100, 150, 200, i32::max_value()); + let a = i32x4::new(100, 150, 200, i32::MAX); #[rustfmt::skip] let r = i32x4::new(25, 38, 50, 536870912); @@ -17768,7 +17214,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_srlri_d() { #[rustfmt::skip] - let a = i64x2::new(100, i64::max_value()); + let a = i64x2::new(100, i64::MAX); #[rustfmt::skip] let r = i64x2::new(50, 4611686018427387904); @@ -17839,10 +17285,10 @@ mod tests { unsafe fn test_msa_subs_s_b() { #[rustfmt::skip] let a = i8x16::new( - i8::min_value(), -2, -3, -4, - i8::min_value(), -2, -3, -4, - i8::min_value(), -2, -3, -4, - i8::min_value(), -2, -3, -4 + i8::MIN, -2, -3, -4, + i8::MIN, -2, -3, -4, + i8::MIN, -2, -3, -4, + i8::MIN, -2, -3, -4 ); #[rustfmt::skip] let b = i8x16::new( @@ -17853,10 +17299,10 @@ mod tests { ); #[rustfmt::skip] let r = i8x16::new( - i8::min_value(), 5, -11, 5, - i8::min_value(), 5, -11, 5, - i8::min_value(), 5, -11, 5, - i8::min_value(), 5, -11, 5 + i8::MIN, 5, -11, 5, + i8::MIN, 5, -11, 5, + i8::MIN, 5, -11, 5, + i8::MIN, 5, -11, 5 ); assert_eq!( @@ -17869,15 +17315,15 @@ mod tests { unsafe fn test_msa_subs_s_h() { #[rustfmt::skip] let a = i16x8::new( - i16::min_value(), -2, -3, -4, - i16::min_value(), -2, -3, -4 + i16::MIN, -2, -3, -4, + i16::MIN, -2, -3, -4 ); #[rustfmt::skip] let b = i16x8::new(6, -7, 8, -9, 6, -7, 8, -9); #[rustfmt::skip] let r = i16x8::new( - i16::min_value(), 5, -11, 5, - i16::min_value(), 5, -11, 5 + i16::MIN, 5, -11, 5, + i16::MIN, 5, -11, 5 ); assert_eq!( @@ -17889,11 +17335,11 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_subs_s_w() { #[rustfmt::skip] - let a = i32x4::new(i32::min_value(), -2, -3, -4); + let a = i32x4::new(i32::MIN, -2, -3, -4); #[rustfmt::skip] let b = i32x4::new(6, -7, 8, -9); #[rustfmt::skip] - let r = i32x4::new(i32::min_value(), 5, -11, 5); + let r = i32x4::new(i32::MIN, 5, -11, 5); assert_eq!( r, @@ -17904,11 +17350,11 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_subs_s_d() { #[rustfmt::skip] - let a = i64x2::new(i64::min_value(), -2); + let a = i64x2::new(i64::MIN, -2); #[rustfmt::skip] let b = i64x2::new(6, -7); #[rustfmt::skip] - let r = i64x2::new(i64::min_value(), 5); + let r = i64x2::new(i64::MIN, 5); assert_eq!( r, @@ -17920,10 +17366,10 @@ mod tests { unsafe fn test_msa_subs_u_b() { #[rustfmt::skip] let a = u8x16::new( - u8::max_value(), 2, 3, 4, - u8::max_value(), 2, 3, 4, - u8::max_value(), 2, 3, 4, - u8::max_value(), 2, 3, 4 + u8::MAX, 2, 3, 4, + u8::MAX, 2, 3, 4, + u8::MAX, 2, 3, 4, + u8::MAX, 2, 3, 4 ); #[rustfmt::skip] let b = u8x16::new( @@ -17950,8 +17396,8 @@ mod tests { unsafe fn test_msa_subs_u_h() { #[rustfmt::skip] let a = u16x8::new( - u16::max_value(), 2, 3, 4, - u16::max_value(), 2, 3, 4 + u16::MAX, 2, 3, 4, + u16::MAX, 2, 3, 4 ); #[rustfmt::skip] let b = u16x8::new(6, 7, 8, 9, 6, 7, 8, 9); @@ -17967,7 +17413,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_subs_u_w() { #[rustfmt::skip] - let a = u32x4::new(u32::max_value(), 2, 3, 4); + let a = u32x4::new(u32::MAX, 2, 3, 4); #[rustfmt::skip] let b = u32x4::new(6, 7, 8, 9); #[rustfmt::skip] @@ -17982,7 +17428,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_subs_u_d() { #[rustfmt::skip] - let a = u64x2::new(u64::max_value(), 2); + let a = u64x2::new(u64::MAX, 2); #[rustfmt::skip] let b = u64x2::new(6, 7); #[rustfmt::skip] @@ -17998,10 +17444,10 @@ mod tests { unsafe fn test_msa_subsus_u_b() { #[rustfmt::skip] let a = u8x16::new( - u8::max_value(), 2, 3, 4, - u8::max_value(), 2, 3, 4, - u8::max_value(), 2, 3, 4, - u8::max_value(), 2, 3, 4 + u8::MAX, 2, 3, 4, + u8::MAX, 2, 3, 4, + u8::MAX, 2, 3, 4, + u8::MAX, 2, 3, 4 ); #[rustfmt::skip] let b = i8x16::new( @@ -18028,8 +17474,8 @@ mod tests { unsafe fn test_msa_subsus_u_h() { #[rustfmt::skip] let a = u16x8::new( - u16::max_value(), 2, 3, 4, - u16::max_value(), 2, 3, 4 + u16::MAX, 2, 3, 4, + u16::MAX, 2, 3, 4 ); #[rustfmt::skip] let b = i16x8::new(-6, -7, -8, -9, -6, -7, -8, -9); @@ -18045,7 +17491,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_subsus_u_w() { #[rustfmt::skip] - let a = u32x4::new(u32::max_value(), 2, 3, 4); + let a = u32x4::new(u32::MAX, 2, 3, 4); #[rustfmt::skip] let b = i32x4::new(-6, -7, -8, -9); #[rustfmt::skip] @@ -18060,7 +17506,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_subsus_u_d() { #[rustfmt::skip] - let a = u64x2::new(u64::max_value(), 2); + let a = u64x2::new(u64::MAX, 2); #[rustfmt::skip] let b = i64x2::new(-6, -7); #[rustfmt::skip] @@ -18076,17 +17522,17 @@ mod tests { unsafe fn test_msa_subsuu_s_b() { #[rustfmt::skip] let a = u8x16::new( - u8::max_value(), 2, 3, 4, - u8::max_value(), 2, 3, 4, - u8::max_value(), 2, 3, 4, - u8::max_value(), 2, 3, 4 + u8::MAX, 2, 3, 4, + u8::MAX, 2, 3, 4, + u8::MAX, 2, 3, 4, + u8::MAX, 2, 3, 4 ); #[rustfmt::skip] let b = u8x16::new( - 6, 7, 8, u8::max_value(), - 6, 7, 8, u8::max_value(), - 6, 7, 8, u8::max_value(), - 6, 7, 8, u8::max_value() + 6, 7, 8, u8::MAX, + 6, 7, 8, u8::MAX, + 6, 7, 8, u8::MAX, + 6, 7, 8, u8::MAX ); #[rustfmt::skip] let r = i8x16::new( @@ -18106,8 +17552,8 @@ mod tests { unsafe fn test_msa_subsuu_s_h() { #[rustfmt::skip] let a = u16x8::new( - u16::max_value(), 2, 3, - 4, u16::max_value(), 2, 3, 4 + u16::MAX, 2, 3, + 4, u16::MAX, 2, 3, 4 ); #[rustfmt::skip] let b = u16x8::new(6, 7, 8, 65535, 6, 7, 8, 65535); @@ -18123,7 +17569,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_subsuu_s_w() { #[rustfmt::skip] - let a = u32x4::new(u32::max_value(), 2, 3, 4); + let a = u32x4::new(u32::MAX, 2, 3, 4); #[rustfmt::skip] let b = u32x4::new(6, 7, 8, 4294967295); #[rustfmt::skip] @@ -18138,11 +17584,11 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_subsuu_s_d() { #[rustfmt::skip] - let a = u64x2::new(u64::max_value(), 2); + let a = u64x2::new(u64::MAX, 2); #[rustfmt::skip] let b = u64x2::new(6, 7); #[rustfmt::skip] - let r = i64x2::new(i64::max_value(), -5); + let r = i64x2::new(i64::MAX, -5); assert_eq!( r, @@ -18154,10 +17600,10 @@ mod tests { unsafe fn test_msa_subv_b() { #[rustfmt::skip] let a = i8x16::new( - i8::min_value(), -2, -3, -4, - i8::min_value(), -2, -3, -4, - i8::min_value(), -2, -3, -4, - i8::min_value(), -2, -3, -4 + i8::MIN, -2, -3, -4, + i8::MIN, -2, -3, -4, + i8::MIN, -2, -3, -4, + i8::MIN, -2, -3, -4 ); #[rustfmt::skip] let b = i8x16::new( @@ -18184,8 +17630,8 @@ mod tests { unsafe fn test_msa_subv_h() { #[rustfmt::skip] let a = i16x8::new( - i16::min_value(), -2, -3, -4, - i16::min_value(), -2, -3, -4 + i16::MIN, -2, -3, -4, + i16::MIN, -2, -3, -4 ); #[rustfmt::skip] let b = i16x8::new(6, -7, 8, -9, 6, -7, 8, -9); @@ -18201,7 +17647,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_subv_w() { #[rustfmt::skip] - let a = i32x4::new(i32::min_value(), -2, -3, -4); + let a = i32x4::new(i32::MIN, -2, -3, -4); #[rustfmt::skip] let b = i32x4::new(6, -7, 8, -9); #[rustfmt::skip] @@ -18216,7 +17662,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_subv_d() { #[rustfmt::skip] - let a = i64x2::new(i64::max_value(), -2); + let a = i64x2::new(i64::MAX, -2); #[rustfmt::skip] let b = i64x2::new(6, -7); #[rustfmt::skip] @@ -18232,10 +17678,10 @@ mod tests { unsafe fn test_msa_subvi_b() { #[rustfmt::skip] let a = i8x16::new( - 100, i8::max_value(), 50, i8::min_value(), - 100, i8::max_value(), 50, i8::min_value(), - 100, i8::max_value(), 50, i8::min_value(), - 100, i8::max_value(), 50, i8::min_value() + 100, i8::MAX, 50, i8::MIN, + 100, i8::MAX, 50, i8::MIN, + 100, i8::MAX, 50, i8::MIN, + 100, i8::MAX, 50, i8::MIN ); #[rustfmt::skip] let r = i8x16::new( @@ -18252,8 +17698,8 @@ mod tests { unsafe fn test_msa_subvi_h() { #[rustfmt::skip] let a = i16x8::new( - i16::max_value(), 3276, -100, i16::min_value(), - i16::max_value(), 3276, -100, i16::min_value() + i16::MAX, 3276, -100, i16::MIN, + i16::MAX, 3276, -100, i16::MIN ); #[rustfmt::skip] let r = i16x8::new( @@ -18267,7 +17713,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_subvi_w() { #[rustfmt::skip] - let a = i32x4::new(100, 150, 200, i32::max_value()); + let a = i32x4::new(100, 150, 200, i32::MAX); #[rustfmt::skip] let r = i32x4::new(95, 145, 195, 2147483642); @@ -18277,7 +17723,7 @@ mod tests { #[simd_test(enable = "msa")] unsafe fn test_msa_subvi_d() { #[rustfmt::skip] - let a = i64x2::new(100, i64::max_value()); + let a = i64x2::new(100, i64::MAX); #[rustfmt::skip] let r = i64x2::new(95, 9223372036854775802); diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/mips/msa/macros.rs b/crux-mir/lib/stdarch/crates/core_arch/src/mips/msa/macros.rs index a16a58dc1..de8905840 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/mips/msa/macros.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/mips/msa/macros.rs @@ -1,4353 +1,31 @@ //! Utility macros. -//immediate value: -4096:4088 -macro_rules! constify_imm_s13 { - ($imm_s13:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm_s13) & 0b1_1111_1111_1111 { - 0 => $expand!(0), - 8 => $expand!(8), - 16 => $expand!(16), - 24 => $expand!(24), - 32 => $expand!(32), - 40 => $expand!(40), - 48 => $expand!(48), - 56 => $expand!(56), - 64 => $expand!(64), - 72 => $expand!(72), - 80 => $expand!(80), - 88 => $expand!(88), - 96 => $expand!(96), - 104 => $expand!(104), - 112 => $expand!(112), - 120 => $expand!(120), - 128 => $expand!(128), - 136 => $expand!(136), - 144 => $expand!(144), - 152 => $expand!(152), - 160 => $expand!(160), - 168 => $expand!(168), - 176 => $expand!(176), - 184 => $expand!(184), - 192 => $expand!(192), - 200 => $expand!(200), - 208 => $expand!(208), - 216 => $expand!(216), - 224 => $expand!(224), - 232 => $expand!(232), - 240 => $expand!(240), - 248 => $expand!(248), - 256 => $expand!(256), - 264 => $expand!(264), - 272 => $expand!(272), - 280 => $expand!(280), - 288 => $expand!(288), - 296 => $expand!(296), - 304 => $expand!(304), - 312 => $expand!(312), - 320 => $expand!(320), - 328 => $expand!(328), - 336 => $expand!(336), - 344 => $expand!(344), - 352 => $expand!(352), - 360 => $expand!(360), - 368 => $expand!(368), - 376 => $expand!(376), - 384 => $expand!(384), - 392 => $expand!(392), - 400 => $expand!(400), - 408 => $expand!(408), - 416 => $expand!(416), - 424 => $expand!(424), - 432 => $expand!(432), - 440 => $expand!(440), - 448 => $expand!(448), - 456 => $expand!(456), - 464 => $expand!(464), - 472 => $expand!(472), - 480 => $expand!(480), - 488 => $expand!(488), - 496 => $expand!(496), - 504 => $expand!(504), - 512 => $expand!(512), - 520 => $expand!(520), - 528 => $expand!(528), - 536 => $expand!(536), - 544 => $expand!(544), - 552 => $expand!(552), - 560 => $expand!(560), - 568 => $expand!(568), - 576 => $expand!(576), - 584 => $expand!(584), - 592 => $expand!(592), - 600 => $expand!(600), - 608 => $expand!(608), - 616 => $expand!(616), - 624 => $expand!(624), - 632 => $expand!(632), - 640 => $expand!(640), - 648 => $expand!(648), - 656 => $expand!(656), - 664 => $expand!(664), - 672 => $expand!(672), - 680 => $expand!(680), - 688 => $expand!(688), - 696 => $expand!(696), - 704 => $expand!(704), - 712 => $expand!(712), - 720 => $expand!(720), - 728 => $expand!(728), - 736 => $expand!(736), - 744 => $expand!(744), - 752 => $expand!(752), - 760 => $expand!(760), - 768 => $expand!(768), - 776 => $expand!(776), - 784 => $expand!(784), - 792 => $expand!(792), - 800 => $expand!(800), - 808 => $expand!(808), - 816 => $expand!(816), - 824 => $expand!(824), - 832 => $expand!(832), - 840 => $expand!(840), - 848 => $expand!(848), - 856 => $expand!(856), - 864 => $expand!(864), - 872 => $expand!(872), - 880 => $expand!(880), - 888 => $expand!(888), - 896 => $expand!(896), - 904 => $expand!(904), - 912 => $expand!(912), - 920 => $expand!(920), - 928 => $expand!(928), - 936 => $expand!(936), - 944 => $expand!(944), - 952 => $expand!(952), - 960 => $expand!(960), - 968 => $expand!(968), - 976 => $expand!(976), - 984 => $expand!(984), - 992 => $expand!(992), - 1000 => $expand!(1000), - 1008 => $expand!(1008), - 1016 => $expand!(1016), - 1024 => $expand!(1024), - 1032 => $expand!(1032), - 1040 => $expand!(1040), - 1048 => $expand!(1048), - 1056 => $expand!(1056), - 1064 => $expand!(1064), - 1072 => $expand!(1072), - 1080 => $expand!(1080), - 1088 => $expand!(1088), - 1096 => $expand!(1096), - 1104 => $expand!(1104), - 1112 => $expand!(1112), - 1120 => $expand!(1120), - 1128 => $expand!(1128), - 1136 => $expand!(1136), - 1144 => $expand!(1144), - 1152 => $expand!(1152), - 1160 => $expand!(1160), - 1168 => $expand!(1168), - 1176 => $expand!(1176), - 1184 => $expand!(1184), - 1192 => $expand!(1192), - 1200 => $expand!(1200), - 1208 => $expand!(1208), - 1216 => $expand!(1216), - 1224 => $expand!(1224), - 1232 => $expand!(1232), - 1240 => $expand!(1240), - 1248 => $expand!(1248), - 1256 => $expand!(1256), - 1264 => $expand!(1264), - 1272 => $expand!(1272), - 1280 => $expand!(1280), - 1288 => $expand!(1288), - 1296 => $expand!(1296), - 1304 => $expand!(1304), - 1312 => $expand!(1312), - 1320 => $expand!(1320), - 1328 => $expand!(1328), - 1336 => $expand!(1336), - 1344 => $expand!(1344), - 1352 => $expand!(1352), - 1360 => $expand!(1360), - 1368 => $expand!(1368), - 1376 => $expand!(1376), - 1384 => $expand!(1384), - 1392 => $expand!(1392), - 1400 => $expand!(1400), - 1408 => $expand!(1408), - 1416 => $expand!(1416), - 1424 => $expand!(1424), - 1432 => $expand!(1432), - 1440 => $expand!(1440), - 1448 => $expand!(1448), - 1456 => $expand!(1456), - 1464 => $expand!(1464), - 1472 => $expand!(1472), - 1480 => $expand!(1480), - 1488 => $expand!(1488), - 1496 => $expand!(1496), - 1504 => $expand!(1504), - 1512 => $expand!(1512), - 1520 => $expand!(1520), - 1528 => $expand!(1528), - 1536 => $expand!(1536), - 1544 => $expand!(1544), - 1552 => $expand!(1552), - 1560 => $expand!(1560), - 1568 => $expand!(1568), - 1576 => $expand!(1576), - 1584 => $expand!(1584), - 1592 => $expand!(1592), - 1600 => $expand!(1600), - 1608 => $expand!(1608), - 1616 => $expand!(1616), - 1624 => $expand!(1624), - 1632 => $expand!(1632), - 1640 => $expand!(1640), - 1648 => $expand!(1648), - 1656 => $expand!(1656), - 1664 => $expand!(1664), - 1672 => $expand!(1672), - 1680 => $expand!(1680), - 1688 => $expand!(1688), - 1696 => $expand!(1696), - 1704 => $expand!(1704), - 1712 => $expand!(1712), - 1720 => $expand!(1720), - 1728 => $expand!(1728), - 1736 => $expand!(1736), - 1744 => $expand!(1744), - 1752 => $expand!(1752), - 1760 => $expand!(1760), - 1768 => $expand!(1768), - 1776 => $expand!(1776), - 1784 => $expand!(1784), - 1792 => $expand!(1792), - 1800 => $expand!(1800), - 1808 => $expand!(1808), - 1816 => $expand!(1816), - 1824 => $expand!(1824), - 1832 => $expand!(1832), - 1840 => $expand!(1840), - 1848 => $expand!(1848), - 1856 => $expand!(1856), - 1864 => $expand!(1864), - 1872 => $expand!(1872), - 1880 => $expand!(1880), - 1888 => $expand!(1888), - 1896 => $expand!(1896), - 1904 => $expand!(1904), - 1912 => $expand!(1912), - 1920 => $expand!(1920), - 1928 => $expand!(1928), - 1936 => $expand!(1936), - 1944 => $expand!(1944), - 1952 => $expand!(1952), - 1960 => $expand!(1960), - 1968 => $expand!(1968), - 1976 => $expand!(1976), - 1984 => $expand!(1984), - 1992 => $expand!(1992), - 2000 => $expand!(2000), - 2008 => $expand!(2008), - 2016 => $expand!(2016), - 2024 => $expand!(2024), - 2032 => $expand!(2032), - 2040 => $expand!(2040), - 2048 => $expand!(2048), - 2056 => $expand!(2056), - 2064 => $expand!(2064), - 2072 => $expand!(2072), - 2080 => $expand!(2080), - 2088 => $expand!(2088), - 2096 => $expand!(2096), - 2104 => $expand!(2104), - 2112 => $expand!(2112), - 2120 => $expand!(2120), - 2128 => $expand!(2128), - 2136 => $expand!(2136), - 2144 => $expand!(2144), - 2152 => $expand!(2152), - 2160 => $expand!(2160), - 2168 => $expand!(2168), - 2176 => $expand!(2176), - 2184 => $expand!(2184), - 2192 => $expand!(2192), - 2200 => $expand!(2200), - 2208 => $expand!(2208), - 2216 => $expand!(2216), - 2224 => $expand!(2224), - 2232 => $expand!(2232), - 2240 => $expand!(2240), - 2248 => $expand!(2248), - 2256 => $expand!(2256), - 2264 => $expand!(2264), - 2272 => $expand!(2272), - 2280 => $expand!(2280), - 2288 => $expand!(2288), - 2296 => $expand!(2296), - 2304 => $expand!(2304), - 2312 => $expand!(2312), - 2320 => $expand!(2320), - 2328 => $expand!(2328), - 2336 => $expand!(2336), - 2344 => $expand!(2344), - 2352 => $expand!(2352), - 2360 => $expand!(2360), - 2368 => $expand!(2368), - 2376 => $expand!(2376), - 2384 => $expand!(2384), - 2392 => $expand!(2392), - 2400 => $expand!(2400), - 2408 => $expand!(2408), - 2416 => $expand!(2416), - 2424 => $expand!(2424), - 2432 => $expand!(2432), - 2440 => $expand!(2440), - 2448 => $expand!(2448), - 2456 => $expand!(2456), - 2464 => $expand!(2464), - 2472 => $expand!(2472), - 2480 => $expand!(2480), - 2488 => $expand!(2488), - 2496 => $expand!(2496), - 2504 => $expand!(2504), - 2512 => $expand!(2512), - 2520 => $expand!(2520), - 2528 => $expand!(2528), - 2536 => $expand!(2536), - 2544 => $expand!(2544), - 2552 => $expand!(2552), - 2560 => $expand!(2560), - 2568 => $expand!(2568), - 2576 => $expand!(2576), - 2584 => $expand!(2584), - 2592 => $expand!(2592), - 2600 => $expand!(2600), - 2608 => $expand!(2608), - 2616 => $expand!(2616), - 2624 => $expand!(2624), - 2632 => $expand!(2632), - 2640 => $expand!(2640), - 2648 => $expand!(2648), - 2656 => $expand!(2656), - 2664 => $expand!(2664), - 2672 => $expand!(2672), - 2680 => $expand!(2680), - 2688 => $expand!(2688), - 2696 => $expand!(2696), - 2704 => $expand!(2704), - 2712 => $expand!(2712), - 2720 => $expand!(2720), - 2728 => $expand!(2728), - 2736 => $expand!(2736), - 2744 => $expand!(2744), - 2752 => $expand!(2752), - 2760 => $expand!(2760), - 2768 => $expand!(2768), - 2776 => $expand!(2776), - 2784 => $expand!(2784), - 2792 => $expand!(2792), - 2800 => $expand!(2800), - 2808 => $expand!(2808), - 2816 => $expand!(2816), - 2824 => $expand!(2824), - 2832 => $expand!(2832), - 2840 => $expand!(2840), - 2848 => $expand!(2848), - 2856 => $expand!(2856), - 2864 => $expand!(2864), - 2872 => $expand!(2872), - 2880 => $expand!(2880), - 2888 => $expand!(2888), - 2896 => $expand!(2896), - 2904 => $expand!(2904), - 2912 => $expand!(2912), - 2920 => $expand!(2920), - 2928 => $expand!(2928), - 2936 => $expand!(2936), - 2944 => $expand!(2944), - 2952 => $expand!(2952), - 2960 => $expand!(2960), - 2968 => $expand!(2968), - 2976 => $expand!(2976), - 2984 => $expand!(2984), - 2992 => $expand!(2992), - 3000 => $expand!(3000), - 3008 => $expand!(3008), - 3016 => $expand!(3016), - 3024 => $expand!(3024), - 3032 => $expand!(3032), - 3040 => $expand!(3040), - 3048 => $expand!(3048), - 3056 => $expand!(3056), - 3064 => $expand!(3064), - 3072 => $expand!(3072), - 3080 => $expand!(3080), - 3088 => $expand!(3088), - 3096 => $expand!(3096), - 3104 => $expand!(3104), - 3112 => $expand!(3112), - 3120 => $expand!(3120), - 3128 => $expand!(3128), - 3136 => $expand!(3136), - 3144 => $expand!(3144), - 3152 => $expand!(3152), - 3160 => $expand!(3160), - 3168 => $expand!(3168), - 3176 => $expand!(3176), - 3184 => $expand!(3184), - 3192 => $expand!(3192), - 3200 => $expand!(3200), - 3208 => $expand!(3208), - 3216 => $expand!(3216), - 3224 => $expand!(3224), - 3232 => $expand!(3232), - 3240 => $expand!(3240), - 3248 => $expand!(3248), - 3256 => $expand!(3256), - 3264 => $expand!(3264), - 3272 => $expand!(3272), - 3280 => $expand!(3280), - 3288 => $expand!(3288), - 3296 => $expand!(3296), - 3304 => $expand!(3304), - 3312 => $expand!(3312), - 3320 => $expand!(3320), - 3328 => $expand!(3328), - 3336 => $expand!(3336), - 3344 => $expand!(3344), - 3352 => $expand!(3352), - 3360 => $expand!(3360), - 3368 => $expand!(3368), - 3376 => $expand!(3376), - 3384 => $expand!(3384), - 3392 => $expand!(3392), - 3400 => $expand!(3400), - 3408 => $expand!(3408), - 3416 => $expand!(3416), - 3424 => $expand!(3424), - 3432 => $expand!(3432), - 3440 => $expand!(3440), - 3448 => $expand!(3448), - 3456 => $expand!(3456), - 3464 => $expand!(3464), - 3472 => $expand!(3472), - 3480 => $expand!(3480), - 3488 => $expand!(3488), - 3496 => $expand!(3496), - 3504 => $expand!(3504), - 3512 => $expand!(3512), - 3520 => $expand!(3520), - 3528 => $expand!(3528), - 3536 => $expand!(3536), - 3544 => $expand!(3544), - 3552 => $expand!(3552), - 3560 => $expand!(3560), - 3568 => $expand!(3568), - 3576 => $expand!(3576), - 3584 => $expand!(3584), - 3592 => $expand!(3592), - 3600 => $expand!(3600), - 3608 => $expand!(3608), - 3616 => $expand!(3616), - 3624 => $expand!(3624), - 3632 => $expand!(3632), - 3640 => $expand!(3640), - 3648 => $expand!(3648), - 3656 => $expand!(3656), - 3664 => $expand!(3664), - 3672 => $expand!(3672), - 3680 => $expand!(3680), - 3688 => $expand!(3688), - 3696 => $expand!(3696), - 3704 => $expand!(3704), - 3712 => $expand!(3712), - 3720 => $expand!(3720), - 3728 => $expand!(3728), - 3736 => $expand!(3736), - 3744 => $expand!(3744), - 3752 => $expand!(3752), - 3760 => $expand!(3760), - 3768 => $expand!(3768), - 3776 => $expand!(3776), - 3784 => $expand!(3784), - 3792 => $expand!(3792), - 3700 => $expand!(3700), - 3808 => $expand!(3808), - 3816 => $expand!(3816), - 3824 => $expand!(3824), - 3832 => $expand!(3832), - 3840 => $expand!(3840), - 3848 => $expand!(3848), - 3856 => $expand!(3856), - 3864 => $expand!(3864), - 3872 => $expand!(3872), - 3880 => $expand!(3880), - 3888 => $expand!(3888), - 3896 => $expand!(3896), - 3904 => $expand!(3904), - 3912 => $expand!(3912), - 3920 => $expand!(3920), - 3928 => $expand!(3928), - 3936 => $expand!(3936), - 3944 => $expand!(3944), - 3952 => $expand!(3952), - 3960 => $expand!(3960), - 3968 => $expand!(3968), - 3976 => $expand!(3976), - 3984 => $expand!(3984), - 3992 => $expand!(3992), - 4000 => $expand!(4000), - 4008 => $expand!(4008), - 4016 => $expand!(4016), - 4024 => $expand!(4024), - 4032 => $expand!(4032), - 4040 => $expand!(4040), - 4048 => $expand!(4048), - 4056 => $expand!(4056), - 4064 => $expand!(4064), - 4072 => $expand!(4072), - 4080 => $expand!(4080), - 4096 => $expand!(-4096), - 4104 => $expand!(-4088), - 4112 => $expand!(-4080), - 4120 => $expand!(-4072), - 4128 => $expand!(-4064), - 4136 => $expand!(-4056), - 4144 => $expand!(-4048), - 4152 => $expand!(-4040), - 4160 => $expand!(-4032), - 4168 => $expand!(-4024), - 4176 => $expand!(-4016), - 4184 => $expand!(-4008), - 4192 => $expand!(-4000), - 4200 => $expand!(-3992), - 4208 => $expand!(-3984), - 4216 => $expand!(-3976), - 4224 => $expand!(-3968), - 4232 => $expand!(-3960), - 4240 => $expand!(-3952), - 4248 => $expand!(-3944), - 4256 => $expand!(-3936), - 4264 => $expand!(-3928), - 4272 => $expand!(-3920), - 4280 => $expand!(-3912), - 4288 => $expand!(-3904), - 4296 => $expand!(-3896), - 4304 => $expand!(-3888), - 4312 => $expand!(-3880), - 4320 => $expand!(-3872), - 4328 => $expand!(-3864), - 4336 => $expand!(-3856), - 4344 => $expand!(-3848), - 4352 => $expand!(-3840), - 4360 => $expand!(-3832), - 4368 => $expand!(-3824), - 4376 => $expand!(-3816), - 4384 => $expand!(-3808), - 4392 => $expand!(-3800), - 4400 => $expand!(-3792), - 4408 => $expand!(-3784), - 4416 => $expand!(-3776), - 4424 => $expand!(-3768), - 4432 => $expand!(-3760), - 4440 => $expand!(-3752), - 4448 => $expand!(-3744), - 4456 => $expand!(-3736), - 4464 => $expand!(-3728), - 4472 => $expand!(-3720), - 4480 => $expand!(-3712), - 4488 => $expand!(-3704), - 4496 => $expand!(-3696), - 4504 => $expand!(-3688), - 4512 => $expand!(-3680), - 4520 => $expand!(-3672), - 4528 => $expand!(-3664), - 4536 => $expand!(-3656), - 4544 => $expand!(-3648), - 4552 => $expand!(-3640), - 4560 => $expand!(-3632), - 4568 => $expand!(-3624), - 4576 => $expand!(-3616), - 4584 => $expand!(-3608), - 4592 => $expand!(-3600), - 4600 => $expand!(-3592), - 4608 => $expand!(-3584), - 4616 => $expand!(-3576), - 4624 => $expand!(-3568), - 4632 => $expand!(-3560), - 4640 => $expand!(-3552), - 4648 => $expand!(-3544), - 4656 => $expand!(-3536), - 4664 => $expand!(-3528), - 4672 => $expand!(-3520), - 4680 => $expand!(-3512), - 4688 => $expand!(-3504), - 4696 => $expand!(-3496), - 4704 => $expand!(-3488), - 4712 => $expand!(-3480), - 4720 => $expand!(-3472), - 4728 => $expand!(-3464), - 4736 => $expand!(-3456), - 4744 => $expand!(-3448), - 4752 => $expand!(-3440), - 4760 => $expand!(-3432), - 4768 => $expand!(-3424), - 4776 => $expand!(-3416), - 4784 => $expand!(-3408), - 4792 => $expand!(-3400), - 4800 => $expand!(-3392), - 4808 => $expand!(-3384), - 4816 => $expand!(-3376), - 4824 => $expand!(-3368), - 4832 => $expand!(-3360), - 4840 => $expand!(-3352), - 4848 => $expand!(-3344), - 4856 => $expand!(-3336), - 4864 => $expand!(-3328), - 4872 => $expand!(-3320), - 4880 => $expand!(-3312), - 4888 => $expand!(-3304), - 4896 => $expand!(-3296), - 4904 => $expand!(-3288), - 4912 => $expand!(-3280), - 4920 => $expand!(-3272), - 4928 => $expand!(-3264), - 4936 => $expand!(-3256), - 4944 => $expand!(-3248), - 4952 => $expand!(-3240), - 4960 => $expand!(-3232), - 4968 => $expand!(-3224), - 4976 => $expand!(-3216), - 4984 => $expand!(-3208), - 4992 => $expand!(-3200), - 5000 => $expand!(-3192), - 5008 => $expand!(-3184), - 5016 => $expand!(-3176), - 5024 => $expand!(-3168), - 5032 => $expand!(-3160), - 5040 => $expand!(-3152), - 5048 => $expand!(-3144), - 5056 => $expand!(-3136), - 5064 => $expand!(-3128), - 5072 => $expand!(-3120), - 5080 => $expand!(-3112), - 5088 => $expand!(-3104), - 5096 => $expand!(-3096), - 5104 => $expand!(-3088), - 5112 => $expand!(-3080), - 5120 => $expand!(-3072), - 5128 => $expand!(-3064), - 5136 => $expand!(-3056), - 5144 => $expand!(-3048), - 5152 => $expand!(-3040), - 5160 => $expand!(-3032), - 5168 => $expand!(-3024), - 5176 => $expand!(-3016), - 5184 => $expand!(-3008), - 5192 => $expand!(-3000), - 5200 => $expand!(-2992), - 5208 => $expand!(-2984), - 5216 => $expand!(-2976), - 5224 => $expand!(-2968), - 5232 => $expand!(-2960), - 5240 => $expand!(-2952), - 5248 => $expand!(-2944), - 5256 => $expand!(-2936), - 5264 => $expand!(-2928), - 5272 => $expand!(-2920), - 5280 => $expand!(-2912), - 5288 => $expand!(-2904), - 5296 => $expand!(-2896), - 5304 => $expand!(-2888), - 5312 => $expand!(-2880), - 5320 => $expand!(-2872), - 5328 => $expand!(-2864), - 5336 => $expand!(-2856), - 5344 => $expand!(-2848), - 5352 => $expand!(-2840), - 5360 => $expand!(-2832), - 5368 => $expand!(-2824), - 5376 => $expand!(-2816), - 5384 => $expand!(-2808), - 5392 => $expand!(-2800), - 5400 => $expand!(-2792), - 5408 => $expand!(-2784), - 5416 => $expand!(-2776), - 5424 => $expand!(-2768), - 5432 => $expand!(-2760), - 5440 => $expand!(-2752), - 5448 => $expand!(-2744), - 5456 => $expand!(-2736), - 5464 => $expand!(-2728), - 5472 => $expand!(-2720), - 5480 => $expand!(-2712), - 5488 => $expand!(-2704), - 5496 => $expand!(-2696), - 5504 => $expand!(-2688), - 5512 => $expand!(-2680), - 5520 => $expand!(-2672), - 5528 => $expand!(-2664), - 5536 => $expand!(-2656), - 5544 => $expand!(-2648), - 5552 => $expand!(-2640), - 5560 => $expand!(-2632), - 5568 => $expand!(-2624), - 5576 => $expand!(-2616), - 5584 => $expand!(-2608), - 5592 => $expand!(-2600), - 5600 => $expand!(-2592), - 5608 => $expand!(-2584), - 5616 => $expand!(-2576), - 5624 => $expand!(-2568), - 5632 => $expand!(-2560), - 5640 => $expand!(-2552), - 5648 => $expand!(-2544), - 5656 => $expand!(-2536), - 5664 => $expand!(-2528), - 5672 => $expand!(-2520), - 5680 => $expand!(-2512), - 5688 => $expand!(-2504), - 5696 => $expand!(-2496), - 5704 => $expand!(-2488), - 5712 => $expand!(-2480), - 5720 => $expand!(-2472), - 5728 => $expand!(-2464), - 5736 => $expand!(-2456), - 5744 => $expand!(-2448), - 5752 => $expand!(-2440), - 5760 => $expand!(-2432), - 5768 => $expand!(-2424), - 5776 => $expand!(-2416), - 5784 => $expand!(-2408), - 5792 => $expand!(-2400), - 5800 => $expand!(-2392), - 5808 => $expand!(-2384), - 5816 => $expand!(-2376), - 5824 => $expand!(-2368), - 5832 => $expand!(-2360), - 5840 => $expand!(-2352), - 5848 => $expand!(-2344), - 5856 => $expand!(-2336), - 5864 => $expand!(-2328), - 5872 => $expand!(-2320), - 5880 => $expand!(-2312), - 5888 => $expand!(-2304), - 5896 => $expand!(-2296), - 5904 => $expand!(-2288), - 5912 => $expand!(-2280), - 5920 => $expand!(-2272), - 5928 => $expand!(-2264), - 5936 => $expand!(-2256), - 5944 => $expand!(-2248), - 5952 => $expand!(-2240), - 5960 => $expand!(-2232), - 5968 => $expand!(-2224), - 5976 => $expand!(-2216), - 5984 => $expand!(-2208), - 5992 => $expand!(-2200), - 6000 => $expand!(-2192), - 6008 => $expand!(-2184), - 6016 => $expand!(-2176), - 6024 => $expand!(-2168), - 6032 => $expand!(-2160), - 6040 => $expand!(-2152), - 6048 => $expand!(-2144), - 6056 => $expand!(-2136), - 6064 => $expand!(-2128), - 6072 => $expand!(-2120), - 6080 => $expand!(-2112), - 6088 => $expand!(-2104), - 6096 => $expand!(-2096), - 6104 => $expand!(-2088), - 6112 => $expand!(-2080), - 6120 => $expand!(-2072), - 6128 => $expand!(-2064), - 6136 => $expand!(-2056), - 6144 => $expand!(-2048), - 6152 => $expand!(-2040), - 6160 => $expand!(-2032), - 6168 => $expand!(-2024), - 6176 => $expand!(-2016), - 6184 => $expand!(-2008), - 6192 => $expand!(-2000), - 6200 => $expand!(-1992), - 6208 => $expand!(-1984), - 6216 => $expand!(-1976), - 6224 => $expand!(-1968), - 6232 => $expand!(-1960), - 6240 => $expand!(-1952), - 6248 => $expand!(-1944), - 6256 => $expand!(-1936), - 6264 => $expand!(-1928), - 6272 => $expand!(-1920), - 6280 => $expand!(-1912), - 6288 => $expand!(-1904), - 6296 => $expand!(-1896), - 6304 => $expand!(-1888), - 6312 => $expand!(-1880), - 6320 => $expand!(-1872), - 6328 => $expand!(-1864), - 6336 => $expand!(-1856), - 6344 => $expand!(-1848), - 6352 => $expand!(-1840), - 6360 => $expand!(-1832), - 6368 => $expand!(-1824), - 6376 => $expand!(-1816), - 6384 => $expand!(-1808), - 6392 => $expand!(-1800), - 6400 => $expand!(-1792), - 6408 => $expand!(-1784), - 6416 => $expand!(-1776), - 6424 => $expand!(-1768), - 6432 => $expand!(-1760), - 6440 => $expand!(-1752), - 6448 => $expand!(-1744), - 6456 => $expand!(-1736), - 6464 => $expand!(-1728), - 6472 => $expand!(-1720), - 6480 => $expand!(-1712), - 6488 => $expand!(-1704), - 6496 => $expand!(-1696), - 6504 => $expand!(-1688), - 6512 => $expand!(-1680), - 6520 => $expand!(-1672), - 6528 => $expand!(-1664), - 6536 => $expand!(-1656), - 6544 => $expand!(-1648), - 6552 => $expand!(-1640), - 6560 => $expand!(-1632), - 6568 => $expand!(-1624), - 6576 => $expand!(-1616), - 6584 => $expand!(-1608), - 6592 => $expand!(-1600), - 6600 => $expand!(-1592), - 6608 => $expand!(-1584), - 6616 => $expand!(-1576), - 6624 => $expand!(-1568), - 6632 => $expand!(-1560), - 6640 => $expand!(-1552), - 6648 => $expand!(-1544), - 6656 => $expand!(-1536), - 6664 => $expand!(-1528), - 6672 => $expand!(-1520), - 6680 => $expand!(-1512), - 6688 => $expand!(-1504), - 6696 => $expand!(-1496), - 6704 => $expand!(-1488), - 6712 => $expand!(-1480), - 6720 => $expand!(-1472), - 6728 => $expand!(-1464), - 6736 => $expand!(-1456), - 6744 => $expand!(-1448), - 6752 => $expand!(-1440), - 6760 => $expand!(-1432), - 6768 => $expand!(-1424), - 6776 => $expand!(-1416), - 6784 => $expand!(-1408), - 6792 => $expand!(-1400), - 6800 => $expand!(-1392), - 6808 => $expand!(-1384), - 6816 => $expand!(-1376), - 6824 => $expand!(-1368), - 6832 => $expand!(-1360), - 6840 => $expand!(-1352), - 6848 => $expand!(-1344), - 6856 => $expand!(-1336), - 6864 => $expand!(-1328), - 6872 => $expand!(-1320), - 6880 => $expand!(-1312), - 6888 => $expand!(-1304), - 6896 => $expand!(-1296), - 6904 => $expand!(-1288), - 6912 => $expand!(-1280), - 6920 => $expand!(-1272), - 6928 => $expand!(-1264), - 6936 => $expand!(-1256), - 6944 => $expand!(-1248), - 6952 => $expand!(-1240), - 6960 => $expand!(-1232), - 6968 => $expand!(-1224), - 6976 => $expand!(-1216), - 6984 => $expand!(-1208), - 6992 => $expand!(-1200), - 6900 => $expand!(-1192), - 7008 => $expand!(-1184), - 7016 => $expand!(-1176), - 7024 => $expand!(-1168), - 7032 => $expand!(-1160), - 7040 => $expand!(-1152), - 7048 => $expand!(-1144), - 7056 => $expand!(-1136), - 7064 => $expand!(-1128), - 7072 => $expand!(-1120), - 7080 => $expand!(-1112), - 7088 => $expand!(-1104), - 7096 => $expand!(-1096), - 7104 => $expand!(-1088), - 7112 => $expand!(-1080), - 7120 => $expand!(-1072), - 7128 => $expand!(-1064), - 7136 => $expand!(-1056), - 7144 => $expand!(-1048), - 7152 => $expand!(-1040), - 7160 => $expand!(-1032), - 7168 => $expand!(-1024), - 7176 => $expand!(-1016), - 7184 => $expand!(-1008), - 7192 => $expand!(-1000), - 7100 => $expand!(-992), - 7208 => $expand!(-984), - 7216 => $expand!(-976), - 7224 => $expand!(-968), - 7232 => $expand!(-960), - 7240 => $expand!(-952), - 7248 => $expand!(-944), - 7256 => $expand!(-936), - 7264 => $expand!(-928), - 7272 => $expand!(-920), - 7280 => $expand!(-912), - 7288 => $expand!(-904), - 7296 => $expand!(-896), - 7304 => $expand!(-888), - 7312 => $expand!(-880), - 7320 => $expand!(-872), - 7328 => $expand!(-864), - 7336 => $expand!(-856), - 7344 => $expand!(-848), - 7352 => $expand!(-840), - 7360 => $expand!(-832), - 7368 => $expand!(-824), - 7376 => $expand!(-816), - 7384 => $expand!(-808), - 7392 => $expand!(-800), - 7400 => $expand!(-792), - 7408 => $expand!(-784), - 7416 => $expand!(-776), - 7424 => $expand!(-768), - 7432 => $expand!(-760), - 7440 => $expand!(-752), - 7448 => $expand!(-744), - 7456 => $expand!(-736), - 7464 => $expand!(-728), - 7472 => $expand!(-720), - 7480 => $expand!(-712), - 7488 => $expand!(-704), - 7496 => $expand!(-696), - 7504 => $expand!(-688), - 7512 => $expand!(-680), - 7520 => $expand!(-672), - 7528 => $expand!(-664), - 7536 => $expand!(-656), - 7544 => $expand!(-648), - 7552 => $expand!(-640), - 7560 => $expand!(-632), - 7568 => $expand!(-624), - 7576 => $expand!(-616), - 7584 => $expand!(-608), - 7592 => $expand!(-600), - 7600 => $expand!(-592), - 7608 => $expand!(-584), - 7616 => $expand!(-576), - 7624 => $expand!(-568), - 7632 => $expand!(-560), - 7640 => $expand!(-552), - 7648 => $expand!(-544), - 7656 => $expand!(-536), - 7664 => $expand!(-528), - 7672 => $expand!(-520), - 7680 => $expand!(-512), - 7688 => $expand!(-504), - 7696 => $expand!(-496), - 7704 => $expand!(-488), - 7712 => $expand!(-480), - 7720 => $expand!(-472), - 7728 => $expand!(-464), - 7736 => $expand!(-456), - 7744 => $expand!(-448), - 7752 => $expand!(-440), - 7760 => $expand!(-432), - 7768 => $expand!(-424), - 7776 => $expand!(-416), - 7784 => $expand!(-408), - 7792 => $expand!(-400), - 7700 => $expand!(-392), - 7808 => $expand!(-384), - 7816 => $expand!(-376), - 7824 => $expand!(-368), - 7832 => $expand!(-360), - 7840 => $expand!(-352), - 7848 => $expand!(-344), - 7856 => $expand!(-336), - 7864 => $expand!(-328), - 7872 => $expand!(-320), - 7880 => $expand!(-312), - 7888 => $expand!(-304), - 7896 => $expand!(-296), - 7904 => $expand!(-288), - 7912 => $expand!(-280), - 7920 => $expand!(-272), - 7928 => $expand!(-264), - 7936 => $expand!(-256), - 7944 => $expand!(-248), - 7952 => $expand!(-240), - 7960 => $expand!(-232), - 7968 => $expand!(-224), - 7976 => $expand!(-216), - 7984 => $expand!(-208), - 7992 => $expand!(-200), - 8000 => $expand!(-192), - 8008 => $expand!(-184), - 8016 => $expand!(-176), - 8024 => $expand!(-168), - 8032 => $expand!(-160), - 8040 => $expand!(-152), - 8048 => $expand!(-144), - 8056 => $expand!(-136), - 8064 => $expand!(-128), - 8072 => $expand!(-120), - 8080 => $expand!(-112), - 8088 => $expand!(-104), - 8096 => $expand!(-96), - 8104 => $expand!(-88), - 8112 => $expand!(-80), - 8120 => $expand!(-72), - 8128 => $expand!(-64), - 8136 => $expand!(-56), - 8144 => $expand!(-48), - 8152 => $expand!(-40), - 8160 => $expand!(-32), - 8168 => $expand!(-24), - 8176 => $expand!(-16), - 8184 => $expand!(-8), - _ => $expand!(4088), - } +macro_rules! static_assert_imm_s5 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, -16, 15>::VALID; }; } -//immediate value: -2048:2044 -macro_rules! constify_imm_s12 { - ($imm_s12:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm_s12) & 0b1111_1111_1111 { - 0 => $expand!(0), - 4 => $expand!(4), - 8 => $expand!(8), - 12 => $expand!(12), - 16 => $expand!(16), - 20 => $expand!(20), - 24 => $expand!(24), - 28 => $expand!(28), - 32 => $expand!(32), - 36 => $expand!(36), - 40 => $expand!(40), - 44 => $expand!(44), - 48 => $expand!(48), - 52 => $expand!(52), - 56 => $expand!(56), - 60 => $expand!(60), - 64 => $expand!(64), - 68 => $expand!(68), - 72 => $expand!(72), - 76 => $expand!(76), - 80 => $expand!(80), - 84 => $expand!(84), - 88 => $expand!(88), - 92 => $expand!(92), - 96 => $expand!(96), - 100 => $expand!(100), - 104 => $expand!(104), - 108 => $expand!(108), - 112 => $expand!(112), - 116 => $expand!(116), - 120 => $expand!(120), - 124 => $expand!(124), - 128 => $expand!(128), - 132 => $expand!(132), - 136 => $expand!(136), - 140 => $expand!(140), - 144 => $expand!(144), - 148 => $expand!(148), - 152 => $expand!(152), - 156 => $expand!(156), - 160 => $expand!(160), - 164 => $expand!(164), - 168 => $expand!(168), - 172 => $expand!(172), - 176 => $expand!(176), - 180 => $expand!(180), - 184 => $expand!(184), - 188 => $expand!(188), - 192 => $expand!(192), - 196 => $expand!(196), - 200 => $expand!(200), - 204 => $expand!(204), - 208 => $expand!(208), - 212 => $expand!(212), - 216 => $expand!(216), - 220 => $expand!(220), - 224 => $expand!(224), - 228 => $expand!(228), - 232 => $expand!(232), - 236 => $expand!(236), - 240 => $expand!(240), - 244 => $expand!(244), - 248 => $expand!(248), - 252 => $expand!(252), - 256 => $expand!(256), - 260 => $expand!(260), - 264 => $expand!(264), - 268 => $expand!(268), - 272 => $expand!(272), - 276 => $expand!(276), - 280 => $expand!(280), - 284 => $expand!(284), - 288 => $expand!(288), - 292 => $expand!(292), - 296 => $expand!(296), - 300 => $expand!(300), - 304 => $expand!(304), - 308 => $expand!(308), - 312 => $expand!(312), - 316 => $expand!(316), - 320 => $expand!(320), - 324 => $expand!(324), - 328 => $expand!(328), - 332 => $expand!(332), - 336 => $expand!(336), - 340 => $expand!(340), - 344 => $expand!(344), - 348 => $expand!(348), - 352 => $expand!(352), - 356 => $expand!(356), - 360 => $expand!(360), - 364 => $expand!(364), - 368 => $expand!(368), - 372 => $expand!(372), - 376 => $expand!(376), - 380 => $expand!(380), - 384 => $expand!(384), - 388 => $expand!(388), - 392 => $expand!(392), - 396 => $expand!(396), - 400 => $expand!(400), - 404 => $expand!(404), - 408 => $expand!(408), - 412 => $expand!(412), - 416 => $expand!(416), - 420 => $expand!(420), - 424 => $expand!(424), - 428 => $expand!(428), - 432 => $expand!(432), - 436 => $expand!(436), - 440 => $expand!(440), - 444 => $expand!(444), - 448 => $expand!(448), - 452 => $expand!(452), - 456 => $expand!(456), - 460 => $expand!(460), - 464 => $expand!(464), - 468 => $expand!(468), - 472 => $expand!(472), - 476 => $expand!(476), - 480 => $expand!(480), - 484 => $expand!(484), - 488 => $expand!(488), - 492 => $expand!(492), - 496 => $expand!(496), - 500 => $expand!(500), - 504 => $expand!(504), - 508 => $expand!(508), - 512 => $expand!(512), - 516 => $expand!(516), - 520 => $expand!(520), - 524 => $expand!(524), - 528 => $expand!(528), - 532 => $expand!(532), - 536 => $expand!(536), - 540 => $expand!(540), - 544 => $expand!(544), - 548 => $expand!(548), - 552 => $expand!(552), - 556 => $expand!(556), - 560 => $expand!(560), - 564 => $expand!(564), - 568 => $expand!(568), - 572 => $expand!(572), - 576 => $expand!(576), - 580 => $expand!(580), - 584 => $expand!(584), - 588 => $expand!(588), - 592 => $expand!(592), - 596 => $expand!(596), - 600 => $expand!(600), - 604 => $expand!(604), - 608 => $expand!(608), - 612 => $expand!(612), - 616 => $expand!(616), - 620 => $expand!(620), - 624 => $expand!(624), - 628 => $expand!(628), - 632 => $expand!(632), - 636 => $expand!(636), - 640 => $expand!(640), - 644 => $expand!(644), - 648 => $expand!(648), - 652 => $expand!(652), - 656 => $expand!(656), - 660 => $expand!(660), - 664 => $expand!(664), - 668 => $expand!(668), - 672 => $expand!(672), - 676 => $expand!(676), - 680 => $expand!(680), - 684 => $expand!(684), - 688 => $expand!(688), - 692 => $expand!(692), - 696 => $expand!(696), - 700 => $expand!(700), - 704 => $expand!(704), - 708 => $expand!(708), - 712 => $expand!(712), - 716 => $expand!(716), - 720 => $expand!(720), - 724 => $expand!(724), - 728 => $expand!(728), - 732 => $expand!(732), - 736 => $expand!(736), - 740 => $expand!(740), - 744 => $expand!(744), - 748 => $expand!(748), - 752 => $expand!(752), - 756 => $expand!(756), - 760 => $expand!(760), - 764 => $expand!(764), - 768 => $expand!(768), - 772 => $expand!(772), - 776 => $expand!(776), - 780 => $expand!(780), - 784 => $expand!(784), - 788 => $expand!(788), - 792 => $expand!(792), - 800 => $expand!(800), - 804 => $expand!(804), - 808 => $expand!(808), - 812 => $expand!(812), - 816 => $expand!(816), - 820 => $expand!(820), - 824 => $expand!(824), - 828 => $expand!(828), - 832 => $expand!(832), - 836 => $expand!(836), - 840 => $expand!(840), - 844 => $expand!(844), - 848 => $expand!(848), - 852 => $expand!(852), - 856 => $expand!(856), - 860 => $expand!(860), - 864 => $expand!(864), - 868 => $expand!(868), - 872 => $expand!(872), - 876 => $expand!(876), - 880 => $expand!(880), - 884 => $expand!(884), - 888 => $expand!(888), - 892 => $expand!(892), - 896 => $expand!(896), - 900 => $expand!(900), - 904 => $expand!(904), - 908 => $expand!(908), - 912 => $expand!(912), - 916 => $expand!(916), - 920 => $expand!(920), - 924 => $expand!(924), - 928 => $expand!(928), - 932 => $expand!(932), - 936 => $expand!(936), - 940 => $expand!(940), - 944 => $expand!(944), - 948 => $expand!(948), - 952 => $expand!(952), - 956 => $expand!(956), - 960 => $expand!(960), - 964 => $expand!(964), - 968 => $expand!(968), - 972 => $expand!(972), - 976 => $expand!(976), - 980 => $expand!(980), - 984 => $expand!(984), - 988 => $expand!(988), - 992 => $expand!(992), - 996 => $expand!(996), - 1000 => $expand!(1000), - 1004 => $expand!(1004), - 1008 => $expand!(1008), - 1012 => $expand!(1012), - 1016 => $expand!(1016), - 1020 => $expand!(1020), - 1024 => $expand!(1024), - 1028 => $expand!(1028), - 1032 => $expand!(1032), - 1036 => $expand!(1036), - 1040 => $expand!(1040), - 1044 => $expand!(1044), - 1048 => $expand!(1048), - 1052 => $expand!(1052), - 1056 => $expand!(1056), - 1060 => $expand!(1060), - 1064 => $expand!(1064), - 1068 => $expand!(1068), - 1072 => $expand!(1072), - 1076 => $expand!(1076), - 1080 => $expand!(1080), - 1084 => $expand!(1084), - 1088 => $expand!(1088), - 1092 => $expand!(1092), - 1096 => $expand!(1096), - 1100 => $expand!(1100), - 1104 => $expand!(1104), - 1108 => $expand!(1108), - 1112 => $expand!(1112), - 1116 => $expand!(1116), - 1120 => $expand!(1120), - 1124 => $expand!(1124), - 1128 => $expand!(1128), - 1132 => $expand!(1132), - 1136 => $expand!(1136), - 1140 => $expand!(1140), - 1144 => $expand!(1144), - 1148 => $expand!(1148), - 1152 => $expand!(1152), - 1156 => $expand!(1156), - 1160 => $expand!(1160), - 1164 => $expand!(1164), - 1168 => $expand!(1168), - 1172 => $expand!(1172), - 1176 => $expand!(1176), - 1180 => $expand!(1180), - 1184 => $expand!(1184), - 1188 => $expand!(1188), - 1192 => $expand!(1192), - 1196 => $expand!(1196), - 1200 => $expand!(1200), - 1204 => $expand!(1204), - 1208 => $expand!(1208), - 1212 => $expand!(1212), - 1216 => $expand!(1216), - 1220 => $expand!(1220), - 1224 => $expand!(1224), - 1228 => $expand!(1228), - 1232 => $expand!(1232), - 1236 => $expand!(1236), - 1240 => $expand!(1240), - 1244 => $expand!(1244), - 1248 => $expand!(1248), - 1252 => $expand!(1252), - 1256 => $expand!(1256), - 1260 => $expand!(1260), - 1264 => $expand!(1264), - 1268 => $expand!(1268), - 1272 => $expand!(1272), - 1276 => $expand!(1276), - 1280 => $expand!(1280), - 1284 => $expand!(1284), - 1288 => $expand!(1288), - 1292 => $expand!(1292), - 1296 => $expand!(1296), - 1300 => $expand!(1300), - 1304 => $expand!(1304), - 1308 => $expand!(1308), - 1312 => $expand!(1312), - 1316 => $expand!(1316), - 1320 => $expand!(1320), - 1324 => $expand!(1324), - 1328 => $expand!(1328), - 1332 => $expand!(1332), - 1336 => $expand!(1336), - 1340 => $expand!(1340), - 1344 => $expand!(1344), - 1348 => $expand!(1348), - 1352 => $expand!(1352), - 1356 => $expand!(1356), - 1360 => $expand!(1360), - 1364 => $expand!(1364), - 1368 => $expand!(1368), - 1372 => $expand!(1372), - 1376 => $expand!(1376), - 1380 => $expand!(1380), - 1384 => $expand!(1384), - 1388 => $expand!(1388), - 1392 => $expand!(1392), - 1396 => $expand!(1396), - 1400 => $expand!(1400), - 1404 => $expand!(1404), - 1408 => $expand!(1408), - 1412 => $expand!(1412), - 1416 => $expand!(1416), - 1420 => $expand!(1420), - 1424 => $expand!(1424), - 1428 => $expand!(1428), - 1432 => $expand!(1432), - 1436 => $expand!(1436), - 1440 => $expand!(1440), - 1444 => $expand!(1444), - 1448 => $expand!(1448), - 1452 => $expand!(1452), - 1456 => $expand!(1456), - 1460 => $expand!(1460), - 1464 => $expand!(1464), - 1468 => $expand!(1468), - 1472 => $expand!(1472), - 1476 => $expand!(1476), - 1480 => $expand!(1480), - 1484 => $expand!(1484), - 1488 => $expand!(1488), - 1492 => $expand!(1492), - 1496 => $expand!(1496), - 1500 => $expand!(1500), - 1504 => $expand!(1504), - 1508 => $expand!(1508), - 1512 => $expand!(1512), - 1516 => $expand!(1516), - 1520 => $expand!(1520), - 1524 => $expand!(1524), - 1528 => $expand!(1528), - 1532 => $expand!(1532), - 1536 => $expand!(1536), - 1540 => $expand!(1540), - 1544 => $expand!(1544), - 1548 => $expand!(1548), - 1552 => $expand!(1552), - 1556 => $expand!(1556), - 1560 => $expand!(1560), - 1564 => $expand!(1564), - 1568 => $expand!(1568), - 1572 => $expand!(1572), - 1576 => $expand!(1576), - 1580 => $expand!(1580), - 1584 => $expand!(1584), - 1588 => $expand!(1588), - 1592 => $expand!(1592), - 1596 => $expand!(1596), - 1600 => $expand!(1600), - 1604 => $expand!(1604), - 1608 => $expand!(1608), - 1612 => $expand!(1612), - 1616 => $expand!(1616), - 1620 => $expand!(1620), - 1624 => $expand!(1624), - 1628 => $expand!(1628), - 1632 => $expand!(1632), - 1636 => $expand!(1636), - 1640 => $expand!(1640), - 1644 => $expand!(1644), - 1648 => $expand!(1648), - 1652 => $expand!(1652), - 1656 => $expand!(1656), - 1660 => $expand!(1660), - 1664 => $expand!(1664), - 1668 => $expand!(1668), - 1672 => $expand!(1672), - 1676 => $expand!(1676), - 1680 => $expand!(1680), - 1684 => $expand!(1684), - 1688 => $expand!(1688), - 1692 => $expand!(1692), - 1696 => $expand!(1696), - 1700 => $expand!(1700), - 1704 => $expand!(1704), - 1708 => $expand!(1708), - 1712 => $expand!(1712), - 1716 => $expand!(1716), - 1720 => $expand!(1720), - 1724 => $expand!(1724), - 1728 => $expand!(1728), - 1732 => $expand!(1732), - 1736 => $expand!(1736), - 1740 => $expand!(1740), - 1744 => $expand!(1744), - 1748 => $expand!(1748), - 1752 => $expand!(1752), - 1756 => $expand!(1756), - 1760 => $expand!(1760), - 1764 => $expand!(1764), - 1768 => $expand!(1768), - 1772 => $expand!(1772), - 1776 => $expand!(1776), - 1780 => $expand!(1780), - 1784 => $expand!(1784), - 1788 => $expand!(1788), - 1792 => $expand!(1792), - 1796 => $expand!(1796), - 1800 => $expand!(1800), - 1804 => $expand!(1804), - 1808 => $expand!(1808), - 1812 => $expand!(1812), - 1816 => $expand!(1816), - 1820 => $expand!(1820), - 1824 => $expand!(1824), - 1828 => $expand!(1828), - 1832 => $expand!(1832), - 1836 => $expand!(1836), - 1840 => $expand!(1840), - 1844 => $expand!(1844), - 1848 => $expand!(1848), - 1852 => $expand!(1852), - 1856 => $expand!(1856), - 1860 => $expand!(1860), - 1864 => $expand!(1864), - 1868 => $expand!(1868), - 1872 => $expand!(1872), - 1876 => $expand!(1876), - 1880 => $expand!(1880), - 1884 => $expand!(1884), - 1888 => $expand!(1888), - 1892 => $expand!(1892), - 1896 => $expand!(1896), - 1900 => $expand!(1900), - 1904 => $expand!(1904), - 1908 => $expand!(1908), - 1912 => $expand!(1912), - 1916 => $expand!(1916), - 1920 => $expand!(1920), - 1924 => $expand!(1924), - 1928 => $expand!(1928), - 1932 => $expand!(1932), - 1936 => $expand!(1936), - 1940 => $expand!(1940), - 1944 => $expand!(1944), - 1948 => $expand!(1948), - 1952 => $expand!(1952), - 1956 => $expand!(1956), - 1960 => $expand!(1960), - 1964 => $expand!(1964), - 1968 => $expand!(1968), - 1972 => $expand!(1972), - 1976 => $expand!(1976), - 1980 => $expand!(1980), - 1984 => $expand!(1984), - 1988 => $expand!(1988), - 1992 => $expand!(1992), - 1996 => $expand!(1996), - 2000 => $expand!(2000), - 2004 => $expand!(2004), - 2008 => $expand!(2008), - 2012 => $expand!(2012), - 2016 => $expand!(2016), - 2020 => $expand!(2020), - 2024 => $expand!(2024), - 2028 => $expand!(2028), - 2032 => $expand!(2032), - 2036 => $expand!(2036), - 2040 => $expand!(2040), - 2048 => $expand!(-2048), - 2052 => $expand!(-2044), - 2056 => $expand!(-2040), - 2060 => $expand!(-2036), - 2064 => $expand!(-2032), - 2068 => $expand!(-2028), - 2072 => $expand!(-2024), - 2076 => $expand!(-2020), - 2080 => $expand!(-2016), - 2084 => $expand!(-2012), - 2088 => $expand!(-2008), - 2092 => $expand!(-2004), - 2096 => $expand!(-2000), - 2100 => $expand!(-1996), - 2104 => $expand!(-1992), - 2108 => $expand!(-1988), - 2112 => $expand!(-1984), - 2116 => $expand!(-1980), - 2120 => $expand!(-1976), - 2124 => $expand!(-1972), - 2128 => $expand!(-1968), - 2132 => $expand!(-1964), - 2136 => $expand!(-1960), - 2140 => $expand!(-1956), - 2144 => $expand!(-1952), - 2148 => $expand!(-1948), - 2152 => $expand!(-1944), - 2156 => $expand!(-1940), - 2160 => $expand!(-1936), - 2164 => $expand!(-1932), - 2168 => $expand!(-1928), - 2172 => $expand!(-1924), - 2176 => $expand!(-1920), - 2180 => $expand!(-1916), - 2184 => $expand!(-1912), - 2188 => $expand!(-1908), - 2192 => $expand!(-1904), - 2196 => $expand!(-1900), - 2200 => $expand!(-1896), - 2204 => $expand!(-1892), - 2208 => $expand!(-1888), - 2212 => $expand!(-1884), - 2216 => $expand!(-1880), - 2220 => $expand!(-1876), - 2224 => $expand!(-1872), - 2228 => $expand!(-1868), - 2232 => $expand!(-1864), - 2236 => $expand!(-1860), - 2240 => $expand!(-1856), - 2244 => $expand!(-1852), - 2248 => $expand!(-1848), - 2252 => $expand!(-1844), - 2256 => $expand!(-1840), - 2260 => $expand!(-1836), - 2264 => $expand!(-1832), - 2268 => $expand!(-1828), - 2272 => $expand!(-1824), - 2276 => $expand!(-1820), - 2280 => $expand!(-1816), - 2284 => $expand!(-1812), - 2288 => $expand!(-1808), - 2292 => $expand!(-1804), - 2296 => $expand!(-1800), - 2300 => $expand!(-1796), - 2304 => $expand!(-1792), - 2308 => $expand!(-1788), - 2312 => $expand!(-1784), - 2316 => $expand!(-1780), - 2320 => $expand!(-1776), - 2324 => $expand!(-1772), - 2328 => $expand!(-1768), - 2332 => $expand!(-1764), - 2336 => $expand!(-1760), - 2340 => $expand!(-1756), - 2344 => $expand!(-1752), - 2348 => $expand!(-1748), - 2352 => $expand!(-1744), - 2356 => $expand!(-1740), - 2360 => $expand!(-1736), - 2364 => $expand!(-1732), - 2368 => $expand!(-1728), - 2372 => $expand!(-1724), - 2376 => $expand!(-1720), - 2380 => $expand!(-1716), - 2384 => $expand!(-1712), - 2388 => $expand!(-1708), - 2392 => $expand!(-1704), - 2396 => $expand!(-1700), - 2400 => $expand!(-1696), - 2404 => $expand!(-1692), - 2408 => $expand!(-1688), - 2412 => $expand!(-1684), - 2416 => $expand!(-1680), - 2420 => $expand!(-1676), - 2424 => $expand!(-1672), - 2428 => $expand!(-1668), - 2432 => $expand!(-1664), - 2436 => $expand!(-1660), - 2440 => $expand!(-1656), - 2444 => $expand!(-1652), - 2448 => $expand!(-1648), - 2452 => $expand!(-1644), - 2456 => $expand!(-1640), - 2460 => $expand!(-1636), - 2464 => $expand!(-1632), - 2468 => $expand!(-1628), - 2472 => $expand!(-1624), - 2476 => $expand!(-1620), - 2480 => $expand!(-1616), - 2484 => $expand!(-1612), - 2488 => $expand!(-1608), - 2492 => $expand!(-1604), - 2496 => $expand!(-1600), - 2500 => $expand!(-1596), - 2504 => $expand!(-1592), - 2508 => $expand!(-1588), - 2512 => $expand!(-1584), - 2516 => $expand!(-1580), - 2520 => $expand!(-1576), - 2524 => $expand!(-1572), - 2528 => $expand!(-1568), - 2532 => $expand!(-1564), - 2536 => $expand!(-1560), - 2540 => $expand!(-1556), - 2544 => $expand!(-1552), - 2548 => $expand!(-1548), - 2552 => $expand!(-1544), - 2556 => $expand!(-1540), - 2560 => $expand!(-1536), - 2564 => $expand!(-1532), - 2568 => $expand!(-1528), - 2572 => $expand!(-1524), - 2576 => $expand!(-1520), - 2580 => $expand!(-1516), - 2584 => $expand!(-1512), - 2588 => $expand!(-1508), - 2592 => $expand!(-1504), - 2596 => $expand!(-1500), - 2600 => $expand!(-1496), - 2604 => $expand!(-1492), - 2608 => $expand!(-1488), - 2612 => $expand!(-1484), - 2616 => $expand!(-1480), - 2620 => $expand!(-1476), - 2624 => $expand!(-1472), - 2628 => $expand!(-1468), - 2632 => $expand!(-1464), - 2636 => $expand!(-1460), - 2640 => $expand!(-1456), - 2644 => $expand!(-1452), - 2648 => $expand!(-1448), - 2652 => $expand!(-1444), - 2656 => $expand!(-1440), - 2660 => $expand!(-1436), - 2664 => $expand!(-1432), - 2668 => $expand!(-1428), - 2672 => $expand!(-1424), - 2676 => $expand!(-1420), - 2680 => $expand!(-1416), - 2684 => $expand!(-1412), - 2688 => $expand!(-1408), - 2692 => $expand!(-1404), - 2696 => $expand!(-1400), - 2700 => $expand!(-1396), - 2704 => $expand!(-1392), - 2708 => $expand!(-1388), - 2712 => $expand!(-1384), - 2716 => $expand!(-1380), - 2720 => $expand!(-1376), - 2724 => $expand!(-1372), - 2728 => $expand!(-1368), - 2732 => $expand!(-1364), - 2736 => $expand!(-1360), - 2740 => $expand!(-1356), - 2744 => $expand!(-1352), - 2748 => $expand!(-1348), - 2752 => $expand!(-1344), - 2756 => $expand!(-1340), - 2760 => $expand!(-1336), - 2764 => $expand!(-1332), - 2768 => $expand!(-1328), - 2772 => $expand!(-1324), - 2776 => $expand!(-1320), - 2780 => $expand!(-1316), - 2784 => $expand!(-1312), - 2788 => $expand!(-1308), - 2792 => $expand!(-1304), - 2796 => $expand!(-1300), - 2800 => $expand!(-1296), - 2804 => $expand!(-1292), - 2808 => $expand!(-1288), - 2812 => $expand!(-1284), - 2816 => $expand!(-1280), - 2820 => $expand!(-1276), - 2824 => $expand!(-1272), - 2828 => $expand!(-1268), - 2832 => $expand!(-1264), - 2836 => $expand!(-1260), - 2840 => $expand!(-1256), - 2844 => $expand!(-1252), - 2848 => $expand!(-1248), - 2852 => $expand!(-1244), - 2856 => $expand!(-1240), - 2860 => $expand!(-1236), - 2864 => $expand!(-1232), - 2868 => $expand!(-1228), - 2872 => $expand!(-1224), - 2876 => $expand!(-1220), - 2880 => $expand!(-1216), - 2884 => $expand!(-1212), - 2888 => $expand!(-1208), - 2892 => $expand!(-1204), - 2896 => $expand!(-1200), - 2900 => $expand!(-1196), - 2904 => $expand!(-1192), - 2908 => $expand!(-1188), - 2912 => $expand!(-1184), - 2916 => $expand!(-1180), - 2920 => $expand!(-1176), - 2924 => $expand!(-1172), - 2928 => $expand!(-1168), - 2932 => $expand!(-1164), - 2936 => $expand!(-1160), - 2940 => $expand!(-1156), - 2944 => $expand!(-1152), - 2948 => $expand!(-1148), - 2952 => $expand!(-1144), - 2956 => $expand!(-1140), - 2960 => $expand!(-1136), - 2964 => $expand!(-1132), - 2968 => $expand!(-1128), - 2972 => $expand!(-1124), - 2976 => $expand!(-1120), - 2980 => $expand!(-1116), - 2984 => $expand!(-1112), - 2988 => $expand!(-1108), - 2992 => $expand!(-1104), - 2996 => $expand!(-1100), - 3000 => $expand!(-1096), - 3004 => $expand!(-1092), - 3008 => $expand!(-1088), - 3012 => $expand!(-1084), - 3016 => $expand!(-1080), - 3020 => $expand!(-1076), - 3024 => $expand!(-1072), - 3028 => $expand!(-1068), - 3032 => $expand!(-1064), - 3036 => $expand!(-1060), - 3040 => $expand!(-1056), - 3044 => $expand!(-1052), - 3048 => $expand!(-1048), - 3052 => $expand!(-1044), - 3056 => $expand!(-1040), - 3060 => $expand!(-1036), - 3064 => $expand!(-1032), - 3068 => $expand!(-1028), - 3072 => $expand!(-1024), - 3076 => $expand!(-1020), - 3080 => $expand!(-1016), - 3084 => $expand!(-1012), - 3088 => $expand!(-1008), - 3092 => $expand!(-1004), - 3096 => $expand!(-1000), - 3100 => $expand!(-996), - 3104 => $expand!(-992), - 3108 => $expand!(-988), - 3112 => $expand!(-984), - 3116 => $expand!(-980), - 3120 => $expand!(-976), - 3124 => $expand!(-972), - 3128 => $expand!(-968), - 3132 => $expand!(-964), - 3136 => $expand!(-960), - 3140 => $expand!(-956), - 3144 => $expand!(-952), - 3148 => $expand!(-948), - 3152 => $expand!(-944), - 3156 => $expand!(-940), - 3160 => $expand!(-936), - 3164 => $expand!(-932), - 3168 => $expand!(-928), - 3172 => $expand!(-924), - 3176 => $expand!(-920), - 3180 => $expand!(-916), - 3184 => $expand!(-912), - 3188 => $expand!(-908), - 3192 => $expand!(-904), - 3196 => $expand!(-900), - 3200 => $expand!(-896), - 3204 => $expand!(-892), - 3208 => $expand!(-888), - 3212 => $expand!(-884), - 3216 => $expand!(-880), - 3220 => $expand!(-876), - 3224 => $expand!(-872), - 3228 => $expand!(-868), - 3232 => $expand!(-864), - 3236 => $expand!(-860), - 3240 => $expand!(-856), - 3244 => $expand!(-852), - 3248 => $expand!(-848), - 3252 => $expand!(-844), - 3256 => $expand!(-840), - 3260 => $expand!(-836), - 3264 => $expand!(-832), - 3268 => $expand!(-828), - 3272 => $expand!(-824), - 3276 => $expand!(-820), - 3280 => $expand!(-816), - 3284 => $expand!(-812), - 3288 => $expand!(-808), - 3292 => $expand!(-804), - 3296 => $expand!(-800), - 3300 => $expand!(-796), - 3304 => $expand!(-792), - 3308 => $expand!(-788), - 3312 => $expand!(-784), - 3316 => $expand!(-780), - 3320 => $expand!(-776), - 3324 => $expand!(-772), - 3328 => $expand!(-768), - 3332 => $expand!(-764), - 3336 => $expand!(-760), - 3340 => $expand!(-756), - 3344 => $expand!(-752), - 3348 => $expand!(-748), - 3352 => $expand!(-744), - 3356 => $expand!(-740), - 3360 => $expand!(-736), - 3364 => $expand!(-732), - 3368 => $expand!(-728), - 3372 => $expand!(-724), - 3376 => $expand!(-720), - 3380 => $expand!(-716), - 3384 => $expand!(-712), - 3388 => $expand!(-708), - 3392 => $expand!(-704), - 3396 => $expand!(-700), - 3400 => $expand!(-696), - 3404 => $expand!(-692), - 3408 => $expand!(-688), - 3412 => $expand!(-684), - 3416 => $expand!(-680), - 3420 => $expand!(-676), - 3424 => $expand!(-672), - 3428 => $expand!(-668), - 3432 => $expand!(-664), - 3436 => $expand!(-660), - 3440 => $expand!(-656), - 3444 => $expand!(-652), - 3448 => $expand!(-648), - 3452 => $expand!(-644), - 3456 => $expand!(-640), - 3460 => $expand!(-636), - 3464 => $expand!(-632), - 3468 => $expand!(-628), - 3472 => $expand!(-624), - 3476 => $expand!(-620), - 3480 => $expand!(-616), - 3484 => $expand!(-612), - 3488 => $expand!(-608), - 3492 => $expand!(-604), - 3496 => $expand!(-600), - 3500 => $expand!(-596), - 3504 => $expand!(-592), - 3508 => $expand!(-588), - 3512 => $expand!(-584), - 3516 => $expand!(-580), - 3520 => $expand!(-576), - 3524 => $expand!(-572), - 3528 => $expand!(-568), - 3532 => $expand!(-564), - 3536 => $expand!(-560), - 3540 => $expand!(-556), - 3544 => $expand!(-552), - 3548 => $expand!(-548), - 3552 => $expand!(-544), - 3556 => $expand!(-540), - 3560 => $expand!(-536), - 3564 => $expand!(-532), - 3568 => $expand!(-528), - 3572 => $expand!(-524), - 3576 => $expand!(-520), - 3580 => $expand!(-516), - 3584 => $expand!(-512), - 3588 => $expand!(-508), - 3592 => $expand!(-504), - 3596 => $expand!(-500), - 3600 => $expand!(-496), - 3604 => $expand!(-492), - 3608 => $expand!(-488), - 3612 => $expand!(-484), - 3616 => $expand!(-480), - 3620 => $expand!(-476), - 3624 => $expand!(-472), - 3628 => $expand!(-468), - 3632 => $expand!(-464), - 3636 => $expand!(-460), - 3640 => $expand!(-456), - 3644 => $expand!(-452), - 3648 => $expand!(-448), - 3652 => $expand!(-444), - 3656 => $expand!(-440), - 3660 => $expand!(-436), - 3664 => $expand!(-432), - 3668 => $expand!(-428), - 3672 => $expand!(-424), - 3676 => $expand!(-420), - 3680 => $expand!(-416), - 3684 => $expand!(-412), - 3688 => $expand!(-408), - 3692 => $expand!(-404), - 3696 => $expand!(-400), - 3700 => $expand!(-396), - 3704 => $expand!(-392), - 3708 => $expand!(-388), - 3712 => $expand!(-384), - 3716 => $expand!(-380), - 3720 => $expand!(-376), - 3724 => $expand!(-372), - 3728 => $expand!(-368), - 3732 => $expand!(-364), - 3736 => $expand!(-360), - 3740 => $expand!(-356), - 3744 => $expand!(-352), - 3748 => $expand!(-348), - 3752 => $expand!(-344), - 3756 => $expand!(-340), - 3760 => $expand!(-336), - 3764 => $expand!(-332), - 3768 => $expand!(-328), - 3772 => $expand!(-324), - 3776 => $expand!(-320), - 3780 => $expand!(-316), - 3784 => $expand!(-312), - 3788 => $expand!(-308), - 3792 => $expand!(-304), - 3796 => $expand!(-300), - 3800 => $expand!(-296), - 3804 => $expand!(-292), - 3808 => $expand!(-288), - 3812 => $expand!(-284), - 3816 => $expand!(-280), - 3820 => $expand!(-276), - 3824 => $expand!(-272), - 3828 => $expand!(-268), - 3832 => $expand!(-264), - 3836 => $expand!(-260), - 3840 => $expand!(-256), - 3844 => $expand!(-252), - 3848 => $expand!(-248), - 3852 => $expand!(-244), - 3856 => $expand!(-240), - 3860 => $expand!(-236), - 3864 => $expand!(-232), - 3868 => $expand!(-228), - 3872 => $expand!(-224), - 3876 => $expand!(-220), - 3880 => $expand!(-216), - 3884 => $expand!(-212), - 3888 => $expand!(-208), - 3892 => $expand!(-204), - 3896 => $expand!(-200), - 3900 => $expand!(-196), - 3904 => $expand!(-192), - 3908 => $expand!(-188), - 3912 => $expand!(-184), - 3916 => $expand!(-180), - 3920 => $expand!(-176), - 3924 => $expand!(-172), - 3928 => $expand!(-168), - 3932 => $expand!(-164), - 3936 => $expand!(-160), - 3940 => $expand!(-156), - 3944 => $expand!(-152), - 3948 => $expand!(-148), - 3952 => $expand!(-144), - 3956 => $expand!(-140), - 3960 => $expand!(-136), - 3964 => $expand!(-132), - 3968 => $expand!(-128), - 3972 => $expand!(-124), - 3976 => $expand!(-120), - 3980 => $expand!(-116), - 3984 => $expand!(-112), - 3988 => $expand!(-108), - 3992 => $expand!(-104), - 3996 => $expand!(-100), - 4000 => $expand!(-96), - 4004 => $expand!(-92), - 4008 => $expand!(-88), - 4012 => $expand!(-84), - 4016 => $expand!(-80), - 4020 => $expand!(-76), - 4024 => $expand!(-72), - 4028 => $expand!(-68), - 4032 => $expand!(-64), - 4036 => $expand!(-60), - 4040 => $expand!(-56), - 4044 => $expand!(-52), - 4048 => $expand!(-48), - 4052 => $expand!(-44), - 4056 => $expand!(-40), - 4060 => $expand!(-36), - 4064 => $expand!(-32), - 4068 => $expand!(-28), - 4072 => $expand!(-24), - 4076 => $expand!(-20), - 4080 => $expand!(-16), - 4084 => $expand!(-12), - 4088 => $expand!(-8), - 4092 => $expand!(-4), - _ => $expand!(2044), - } +macro_rules! static_assert_imm_s10 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, -512, 511>::VALID; }; } -//immediate value: -1024:1022 -macro_rules! constify_imm_s11 { - ($imm_s11:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm_s11) & 0b111_1111_1111 { - 0 => $expand!(0), - 2 => $expand!(2), - 4 => $expand!(4), - 6 => $expand!(6), - 8 => $expand!(8), - 10 => $expand!(10), - 12 => $expand!(12), - 14 => $expand!(14), - 16 => $expand!(16), - 18 => $expand!(18), - 20 => $expand!(20), - 22 => $expand!(22), - 24 => $expand!(24), - 26 => $expand!(26), - 28 => $expand!(28), - 30 => $expand!(30), - 32 => $expand!(32), - 34 => $expand!(34), - 36 => $expand!(36), - 38 => $expand!(38), - 40 => $expand!(40), - 42 => $expand!(42), - 44 => $expand!(44), - 46 => $expand!(46), - 48 => $expand!(48), - 50 => $expand!(50), - 52 => $expand!(52), - 54 => $expand!(54), - 56 => $expand!(56), - 58 => $expand!(58), - 60 => $expand!(60), - 62 => $expand!(62), - 64 => $expand!(64), - 66 => $expand!(66), - 68 => $expand!(68), - 70 => $expand!(70), - 72 => $expand!(72), - 74 => $expand!(74), - 76 => $expand!(76), - 78 => $expand!(78), - 80 => $expand!(80), - 82 => $expand!(82), - 84 => $expand!(84), - 86 => $expand!(86), - 88 => $expand!(88), - 90 => $expand!(90), - 92 => $expand!(92), - 94 => $expand!(94), - 96 => $expand!(96), - 98 => $expand!(98), - 100 => $expand!(100), - 102 => $expand!(102), - 104 => $expand!(104), - 106 => $expand!(106), - 108 => $expand!(108), - 110 => $expand!(110), - 112 => $expand!(112), - 114 => $expand!(114), - 116 => $expand!(116), - 118 => $expand!(118), - 120 => $expand!(120), - 122 => $expand!(122), - 124 => $expand!(124), - 126 => $expand!(126), - 128 => $expand!(128), - 130 => $expand!(130), - 132 => $expand!(132), - 134 => $expand!(134), - 136 => $expand!(136), - 138 => $expand!(138), - 140 => $expand!(140), - 142 => $expand!(142), - 144 => $expand!(144), - 146 => $expand!(146), - 148 => $expand!(148), - 150 => $expand!(150), - 152 => $expand!(152), - 154 => $expand!(154), - 156 => $expand!(156), - 158 => $expand!(158), - 160 => $expand!(160), - 162 => $expand!(162), - 164 => $expand!(164), - 166 => $expand!(166), - 168 => $expand!(168), - 170 => $expand!(170), - 172 => $expand!(172), - 174 => $expand!(174), - 176 => $expand!(176), - 178 => $expand!(178), - 180 => $expand!(180), - 182 => $expand!(182), - 184 => $expand!(184), - 186 => $expand!(186), - 188 => $expand!(188), - 190 => $expand!(190), - 192 => $expand!(192), - 194 => $expand!(194), - 196 => $expand!(196), - 198 => $expand!(198), - 200 => $expand!(200), - 202 => $expand!(202), - 204 => $expand!(204), - 206 => $expand!(206), - 208 => $expand!(208), - 210 => $expand!(210), - 212 => $expand!(212), - 214 => $expand!(214), - 216 => $expand!(216), - 218 => $expand!(218), - 220 => $expand!(220), - 222 => $expand!(222), - 224 => $expand!(224), - 226 => $expand!(226), - 228 => $expand!(228), - 230 => $expand!(230), - 232 => $expand!(232), - 234 => $expand!(234), - 236 => $expand!(236), - 238 => $expand!(238), - 240 => $expand!(240), - 242 => $expand!(242), - 244 => $expand!(244), - 246 => $expand!(246), - 248 => $expand!(248), - 250 => $expand!(250), - 252 => $expand!(252), - 254 => $expand!(254), - 256 => $expand!(256), - 258 => $expand!(258), - 260 => $expand!(260), - 262 => $expand!(262), - 264 => $expand!(264), - 266 => $expand!(266), - 268 => $expand!(268), - 270 => $expand!(270), - 272 => $expand!(272), - 274 => $expand!(274), - 276 => $expand!(276), - 278 => $expand!(278), - 280 => $expand!(280), - 282 => $expand!(282), - 284 => $expand!(284), - 286 => $expand!(286), - 288 => $expand!(288), - 290 => $expand!(290), - 292 => $expand!(292), - 294 => $expand!(294), - 296 => $expand!(296), - 298 => $expand!(298), - 300 => $expand!(300), - 302 => $expand!(302), - 304 => $expand!(304), - 306 => $expand!(306), - 308 => $expand!(308), - 310 => $expand!(310), - 312 => $expand!(312), - 314 => $expand!(314), - 316 => $expand!(316), - 318 => $expand!(318), - 320 => $expand!(320), - 322 => $expand!(322), - 324 => $expand!(324), - 326 => $expand!(326), - 328 => $expand!(328), - 330 => $expand!(330), - 332 => $expand!(332), - 334 => $expand!(334), - 336 => $expand!(336), - 338 => $expand!(338), - 340 => $expand!(340), - 342 => $expand!(342), - 344 => $expand!(344), - 346 => $expand!(346), - 348 => $expand!(348), - 350 => $expand!(350), - 352 => $expand!(352), - 354 => $expand!(354), - 356 => $expand!(356), - 358 => $expand!(358), - 360 => $expand!(360), - 362 => $expand!(362), - 364 => $expand!(364), - 366 => $expand!(366), - 368 => $expand!(368), - 370 => $expand!(370), - 372 => $expand!(372), - 374 => $expand!(374), - 376 => $expand!(376), - 378 => $expand!(378), - 380 => $expand!(380), - 382 => $expand!(382), - 384 => $expand!(384), - 386 => $expand!(386), - 388 => $expand!(388), - 390 => $expand!(390), - 392 => $expand!(392), - 394 => $expand!(394), - 396 => $expand!(396), - 398 => $expand!(398), - 400 => $expand!(400), - 402 => $expand!(402), - 404 => $expand!(404), - 406 => $expand!(406), - 408 => $expand!(408), - 410 => $expand!(410), - 412 => $expand!(412), - 414 => $expand!(414), - 416 => $expand!(416), - 418 => $expand!(418), - 420 => $expand!(420), - 422 => $expand!(422), - 424 => $expand!(424), - 426 => $expand!(426), - 428 => $expand!(428), - 430 => $expand!(430), - 432 => $expand!(432), - 434 => $expand!(434), - 436 => $expand!(436), - 438 => $expand!(438), - 440 => $expand!(440), - 442 => $expand!(442), - 444 => $expand!(444), - 446 => $expand!(446), - 448 => $expand!(448), - 450 => $expand!(450), - 452 => $expand!(452), - 454 => $expand!(454), - 456 => $expand!(456), - 458 => $expand!(458), - 460 => $expand!(460), - 462 => $expand!(462), - 464 => $expand!(464), - 466 => $expand!(466), - 468 => $expand!(468), - 470 => $expand!(470), - 472 => $expand!(472), - 474 => $expand!(474), - 476 => $expand!(476), - 478 => $expand!(478), - 480 => $expand!(480), - 482 => $expand!(482), - 484 => $expand!(484), - 486 => $expand!(486), - 488 => $expand!(488), - 490 => $expand!(490), - 492 => $expand!(492), - 494 => $expand!(494), - 496 => $expand!(496), - 498 => $expand!(498), - 500 => $expand!(500), - 502 => $expand!(502), - 504 => $expand!(504), - 506 => $expand!(506), - 508 => $expand!(508), - 510 => $expand!(510), - 512 => $expand!(512), - 514 => $expand!(514), - 516 => $expand!(516), - 518 => $expand!(518), - 520 => $expand!(520), - 522 => $expand!(522), - 524 => $expand!(524), - 526 => $expand!(526), - 528 => $expand!(528), - 530 => $expand!(530), - 532 => $expand!(532), - 534 => $expand!(534), - 536 => $expand!(536), - 538 => $expand!(538), - 540 => $expand!(540), - 542 => $expand!(542), - 544 => $expand!(544), - 546 => $expand!(546), - 548 => $expand!(548), - 550 => $expand!(550), - 552 => $expand!(552), - 554 => $expand!(554), - 556 => $expand!(556), - 558 => $expand!(558), - 560 => $expand!(560), - 562 => $expand!(562), - 564 => $expand!(564), - 566 => $expand!(566), - 568 => $expand!(568), - 570 => $expand!(570), - 572 => $expand!(572), - 574 => $expand!(574), - 576 => $expand!(576), - 578 => $expand!(578), - 580 => $expand!(580), - 582 => $expand!(582), - 584 => $expand!(584), - 586 => $expand!(586), - 588 => $expand!(588), - 590 => $expand!(590), - 592 => $expand!(592), - 594 => $expand!(594), - 596 => $expand!(596), - 598 => $expand!(598), - 600 => $expand!(600), - 602 => $expand!(602), - 604 => $expand!(604), - 606 => $expand!(606), - 608 => $expand!(608), - 610 => $expand!(610), - 612 => $expand!(612), - 614 => $expand!(614), - 616 => $expand!(616), - 618 => $expand!(618), - 620 => $expand!(620), - 622 => $expand!(622), - 624 => $expand!(624), - 626 => $expand!(626), - 628 => $expand!(628), - 630 => $expand!(630), - 632 => $expand!(632), - 634 => $expand!(634), - 636 => $expand!(636), - 638 => $expand!(638), - 640 => $expand!(640), - 642 => $expand!(642), - 644 => $expand!(644), - 646 => $expand!(646), - 648 => $expand!(648), - 650 => $expand!(650), - 652 => $expand!(652), - 654 => $expand!(654), - 656 => $expand!(656), - 658 => $expand!(658), - 660 => $expand!(660), - 662 => $expand!(662), - 664 => $expand!(664), - 666 => $expand!(666), - 668 => $expand!(668), - 670 => $expand!(670), - 672 => $expand!(672), - 674 => $expand!(674), - 676 => $expand!(676), - 678 => $expand!(678), - 680 => $expand!(680), - 682 => $expand!(682), - 684 => $expand!(684), - 686 => $expand!(686), - 688 => $expand!(688), - 690 => $expand!(690), - 692 => $expand!(692), - 694 => $expand!(694), - 696 => $expand!(696), - 698 => $expand!(698), - 700 => $expand!(700), - 702 => $expand!(702), - 704 => $expand!(704), - 706 => $expand!(706), - 708 => $expand!(708), - 710 => $expand!(710), - 712 => $expand!(712), - 714 => $expand!(714), - 716 => $expand!(716), - 718 => $expand!(718), - 720 => $expand!(720), - 722 => $expand!(722), - 724 => $expand!(724), - 726 => $expand!(726), - 728 => $expand!(728), - 730 => $expand!(730), - 732 => $expand!(732), - 734 => $expand!(734), - 736 => $expand!(736), - 738 => $expand!(738), - 740 => $expand!(740), - 742 => $expand!(742), - 744 => $expand!(744), - 746 => $expand!(746), - 748 => $expand!(748), - 750 => $expand!(750), - 752 => $expand!(752), - 754 => $expand!(754), - 756 => $expand!(756), - 758 => $expand!(758), - 760 => $expand!(760), - 762 => $expand!(762), - 764 => $expand!(764), - 766 => $expand!(766), - 768 => $expand!(768), - 770 => $expand!(770), - 772 => $expand!(772), - 774 => $expand!(774), - 776 => $expand!(776), - 778 => $expand!(778), - 780 => $expand!(780), - 782 => $expand!(782), - 784 => $expand!(784), - 786 => $expand!(786), - 788 => $expand!(788), - 790 => $expand!(790), - 792 => $expand!(792), - 794 => $expand!(794), - 796 => $expand!(796), - 798 => $expand!(798), - 800 => $expand!(800), - 802 => $expand!(802), - 804 => $expand!(804), - 806 => $expand!(806), - 808 => $expand!(808), - 810 => $expand!(810), - 812 => $expand!(812), - 814 => $expand!(814), - 816 => $expand!(816), - 818 => $expand!(818), - 820 => $expand!(820), - 822 => $expand!(822), - 824 => $expand!(824), - 826 => $expand!(826), - 828 => $expand!(828), - 830 => $expand!(830), - 832 => $expand!(832), - 834 => $expand!(834), - 836 => $expand!(836), - 838 => $expand!(838), - 840 => $expand!(840), - 842 => $expand!(842), - 844 => $expand!(844), - 846 => $expand!(846), - 848 => $expand!(848), - 850 => $expand!(850), - 852 => $expand!(852), - 854 => $expand!(854), - 856 => $expand!(856), - 858 => $expand!(858), - 860 => $expand!(860), - 862 => $expand!(862), - 864 => $expand!(864), - 866 => $expand!(866), - 868 => $expand!(868), - 870 => $expand!(870), - 872 => $expand!(872), - 874 => $expand!(874), - 876 => $expand!(876), - 878 => $expand!(878), - 880 => $expand!(880), - 882 => $expand!(882), - 884 => $expand!(884), - 886 => $expand!(886), - 888 => $expand!(888), - 890 => $expand!(890), - 892 => $expand!(892), - 894 => $expand!(894), - 896 => $expand!(896), - 898 => $expand!(898), - 900 => $expand!(900), - 902 => $expand!(902), - 904 => $expand!(904), - 906 => $expand!(906), - 908 => $expand!(908), - 910 => $expand!(910), - 912 => $expand!(912), - 914 => $expand!(914), - 916 => $expand!(916), - 918 => $expand!(918), - 920 => $expand!(920), - 922 => $expand!(922), - 924 => $expand!(924), - 926 => $expand!(926), - 928 => $expand!(928), - 930 => $expand!(930), - 932 => $expand!(932), - 934 => $expand!(934), - 936 => $expand!(936), - 938 => $expand!(938), - 940 => $expand!(940), - 942 => $expand!(942), - 944 => $expand!(944), - 946 => $expand!(946), - 948 => $expand!(948), - 950 => $expand!(950), - 952 => $expand!(952), - 954 => $expand!(954), - 956 => $expand!(956), - 958 => $expand!(958), - 960 => $expand!(960), - 962 => $expand!(962), - 964 => $expand!(964), - 966 => $expand!(966), - 968 => $expand!(968), - 970 => $expand!(970), - 972 => $expand!(972), - 974 => $expand!(974), - 976 => $expand!(976), - 978 => $expand!(978), - 980 => $expand!(980), - 982 => $expand!(982), - 984 => $expand!(984), - 986 => $expand!(986), - 988 => $expand!(988), - 990 => $expand!(990), - 992 => $expand!(992), - 994 => $expand!(994), - 996 => $expand!(996), - 998 => $expand!(998), - 1000 => $expand!(1000), - 1002 => $expand!(1002), - 1004 => $expand!(1004), - 1006 => $expand!(1006), - 1008 => $expand!(1008), - 1010 => $expand!(1010), - 1012 => $expand!(1012), - 1014 => $expand!(1014), - 1016 => $expand!(1016), - 1018 => $expand!(1018), - 1020 => $expand!(1020), - 1024 => $expand!(-1024), - 1026 => $expand!(-1022), - 1028 => $expand!(-1020), - 1030 => $expand!(-1018), - 1032 => $expand!(-1016), - 1034 => $expand!(-1014), - 1036 => $expand!(-1012), - 1038 => $expand!(-1010), - 1040 => $expand!(-1008), - 1042 => $expand!(-1006), - 1044 => $expand!(-1004), - 1046 => $expand!(-1002), - 1048 => $expand!(-1000), - 1050 => $expand!(-998), - 1052 => $expand!(-996), - 1054 => $expand!(-994), - 1056 => $expand!(-992), - 1058 => $expand!(-990), - 1060 => $expand!(-988), - 1062 => $expand!(-986), - 1064 => $expand!(-984), - 1066 => $expand!(-982), - 1068 => $expand!(-980), - 1070 => $expand!(-978), - 1072 => $expand!(-976), - 1074 => $expand!(-974), - 1076 => $expand!(-972), - 1078 => $expand!(-970), - 1080 => $expand!(-968), - 1082 => $expand!(-966), - 1084 => $expand!(-964), - 1086 => $expand!(-962), - 1088 => $expand!(-960), - 1090 => $expand!(-958), - 1092 => $expand!(-956), - 1094 => $expand!(-954), - 1096 => $expand!(-952), - 1098 => $expand!(-950), - 1100 => $expand!(-948), - 1102 => $expand!(-946), - 1104 => $expand!(-944), - 1106 => $expand!(-942), - 1108 => $expand!(-940), - 1110 => $expand!(-938), - 1112 => $expand!(-936), - 1114 => $expand!(-934), - 1116 => $expand!(-932), - 1118 => $expand!(-930), - 1120 => $expand!(-928), - 1122 => $expand!(-926), - 1124 => $expand!(-924), - 1126 => $expand!(-922), - 1128 => $expand!(-920), - 1130 => $expand!(-918), - 1132 => $expand!(-916), - 1134 => $expand!(-914), - 1136 => $expand!(-912), - 1138 => $expand!(-910), - 1140 => $expand!(-908), - 1142 => $expand!(-906), - 1144 => $expand!(-904), - 1146 => $expand!(-902), - 1148 => $expand!(-900), - 1150 => $expand!(-898), - 1152 => $expand!(-896), - 1154 => $expand!(-894), - 1156 => $expand!(-892), - 1158 => $expand!(-890), - 1160 => $expand!(-888), - 1162 => $expand!(-886), - 1164 => $expand!(-884), - 1166 => $expand!(-882), - 1168 => $expand!(-880), - 1170 => $expand!(-878), - 1172 => $expand!(-876), - 1174 => $expand!(-874), - 1176 => $expand!(-872), - 1178 => $expand!(-870), - 1180 => $expand!(-868), - 1182 => $expand!(-866), - 1184 => $expand!(-864), - 1186 => $expand!(-862), - 1188 => $expand!(-860), - 1190 => $expand!(-858), - 1192 => $expand!(-856), - 1194 => $expand!(-854), - 1196 => $expand!(-852), - 1198 => $expand!(-850), - 1200 => $expand!(-848), - 1202 => $expand!(-846), - 1204 => $expand!(-844), - 1206 => $expand!(-842), - 1208 => $expand!(-840), - 1210 => $expand!(-838), - 1212 => $expand!(-836), - 1214 => $expand!(-834), - 1216 => $expand!(-832), - 1218 => $expand!(-830), - 1220 => $expand!(-828), - 1222 => $expand!(-826), - 1224 => $expand!(-824), - 1226 => $expand!(-822), - 1228 => $expand!(-820), - 1230 => $expand!(-818), - 1232 => $expand!(-816), - 1234 => $expand!(-814), - 1236 => $expand!(-812), - 1238 => $expand!(-810), - 1240 => $expand!(-808), - 1242 => $expand!(-806), - 1244 => $expand!(-804), - 1246 => $expand!(-802), - 1248 => $expand!(-800), - 1250 => $expand!(-798), - 1252 => $expand!(-796), - 1254 => $expand!(-794), - 1256 => $expand!(-792), - 1258 => $expand!(-790), - 1260 => $expand!(-788), - 1262 => $expand!(-786), - 1264 => $expand!(-784), - 1266 => $expand!(-782), - 1268 => $expand!(-780), - 1270 => $expand!(-778), - 1272 => $expand!(-776), - 1274 => $expand!(-774), - 1276 => $expand!(-772), - 1278 => $expand!(-770), - 1280 => $expand!(-768), - 1282 => $expand!(-766), - 1284 => $expand!(-764), - 1286 => $expand!(-762), - 1288 => $expand!(-760), - 1290 => $expand!(-758), - 1292 => $expand!(-756), - 1294 => $expand!(-754), - 1296 => $expand!(-752), - 1298 => $expand!(-750), - 1300 => $expand!(-748), - 1302 => $expand!(-746), - 1304 => $expand!(-744), - 1306 => $expand!(-742), - 1308 => $expand!(-740), - 1310 => $expand!(-738), - 1312 => $expand!(-736), - 1314 => $expand!(-734), - 1316 => $expand!(-732), - 1318 => $expand!(-730), - 1320 => $expand!(-728), - 1322 => $expand!(-726), - 1324 => $expand!(-724), - 1326 => $expand!(-722), - 1328 => $expand!(-720), - 1330 => $expand!(-718), - 1332 => $expand!(-716), - 1334 => $expand!(-714), - 1336 => $expand!(-712), - 1338 => $expand!(-710), - 1340 => $expand!(-708), - 1342 => $expand!(-706), - 1344 => $expand!(-704), - 1346 => $expand!(-702), - 1348 => $expand!(-700), - 1350 => $expand!(-698), - 1352 => $expand!(-696), - 1354 => $expand!(-694), - 1356 => $expand!(-692), - 1358 => $expand!(-690), - 1360 => $expand!(-688), - 1362 => $expand!(-686), - 1364 => $expand!(-684), - 1366 => $expand!(-682), - 1368 => $expand!(-680), - 1370 => $expand!(-678), - 1372 => $expand!(-676), - 1374 => $expand!(-674), - 1376 => $expand!(-672), - 1378 => $expand!(-670), - 1380 => $expand!(-668), - 1382 => $expand!(-666), - 1384 => $expand!(-664), - 1386 => $expand!(-662), - 1388 => $expand!(-660), - 1390 => $expand!(-658), - 1392 => $expand!(-656), - 1394 => $expand!(-654), - 1396 => $expand!(-652), - 1398 => $expand!(-650), - 1400 => $expand!(-648), - 1402 => $expand!(-646), - 1404 => $expand!(-644), - 1406 => $expand!(-642), - 1408 => $expand!(-640), - 1410 => $expand!(-638), - 1412 => $expand!(-636), - 1414 => $expand!(-634), - 1416 => $expand!(-632), - 1418 => $expand!(-630), - 1420 => $expand!(-628), - 1422 => $expand!(-626), - 1424 => $expand!(-624), - 1426 => $expand!(-622), - 1428 => $expand!(-620), - 1430 => $expand!(-618), - 1432 => $expand!(-616), - 1434 => $expand!(-614), - 1436 => $expand!(-612), - 1438 => $expand!(-610), - 1440 => $expand!(-608), - 1442 => $expand!(-606), - 1444 => $expand!(-604), - 1446 => $expand!(-602), - 1448 => $expand!(-600), - 1450 => $expand!(-598), - 1452 => $expand!(-596), - 1454 => $expand!(-594), - 1456 => $expand!(-592), - 1458 => $expand!(-590), - 1460 => $expand!(-588), - 1462 => $expand!(-586), - 1464 => $expand!(-584), - 1466 => $expand!(-582), - 1468 => $expand!(-580), - 1470 => $expand!(-578), - 1472 => $expand!(-576), - 1474 => $expand!(-574), - 1476 => $expand!(-572), - 1478 => $expand!(-570), - 1480 => $expand!(-568), - 1482 => $expand!(-566), - 1484 => $expand!(-564), - 1486 => $expand!(-562), - 1488 => $expand!(-560), - 1490 => $expand!(-558), - 1492 => $expand!(-556), - 1494 => $expand!(-554), - 1496 => $expand!(-552), - 1498 => $expand!(-550), - 1500 => $expand!(-548), - 1502 => $expand!(-546), - 1504 => $expand!(-544), - 1506 => $expand!(-542), - 1508 => $expand!(-540), - 1510 => $expand!(-538), - 1512 => $expand!(-536), - 1514 => $expand!(-534), - 1516 => $expand!(-532), - 1518 => $expand!(-530), - 1520 => $expand!(-528), - 1522 => $expand!(-526), - 1524 => $expand!(-524), - 1526 => $expand!(-522), - 1528 => $expand!(-520), - 1530 => $expand!(-518), - 1532 => $expand!(-516), - 1534 => $expand!(-514), - 1536 => $expand!(-512), - 1538 => $expand!(-510), - 1540 => $expand!(-508), - 1542 => $expand!(-506), - 1544 => $expand!(-504), - 1546 => $expand!(-502), - 1548 => $expand!(-500), - 1550 => $expand!(-498), - 1552 => $expand!(-496), - 1554 => $expand!(-494), - 1556 => $expand!(-492), - 1558 => $expand!(-490), - 1560 => $expand!(-488), - 1562 => $expand!(-486), - 1564 => $expand!(-484), - 1566 => $expand!(-482), - 1568 => $expand!(-480), - 1570 => $expand!(-478), - 1572 => $expand!(-476), - 1574 => $expand!(-474), - 1576 => $expand!(-472), - 1578 => $expand!(-470), - 1580 => $expand!(-468), - 1582 => $expand!(-466), - 1584 => $expand!(-464), - 1586 => $expand!(-462), - 1588 => $expand!(-460), - 1590 => $expand!(-458), - 1592 => $expand!(-456), - 1594 => $expand!(-454), - 1596 => $expand!(-452), - 1598 => $expand!(-450), - 1600 => $expand!(-448), - 1602 => $expand!(-446), - 1604 => $expand!(-444), - 1606 => $expand!(-442), - 1608 => $expand!(-440), - 1610 => $expand!(-438), - 1612 => $expand!(-436), - 1614 => $expand!(-434), - 1616 => $expand!(-432), - 1618 => $expand!(-430), - 1620 => $expand!(-428), - 1622 => $expand!(-426), - 1624 => $expand!(-424), - 1626 => $expand!(-422), - 1628 => $expand!(-420), - 1630 => $expand!(-418), - 1632 => $expand!(-416), - 1634 => $expand!(-414), - 1636 => $expand!(-412), - 1638 => $expand!(-410), - 1640 => $expand!(-408), - 1642 => $expand!(-406), - 1644 => $expand!(-404), - 1646 => $expand!(-402), - 1648 => $expand!(-400), - 1650 => $expand!(-398), - 1652 => $expand!(-396), - 1654 => $expand!(-394), - 1656 => $expand!(-392), - 1658 => $expand!(-390), - 1660 => $expand!(-388), - 1662 => $expand!(-386), - 1664 => $expand!(-384), - 1666 => $expand!(-382), - 1668 => $expand!(-380), - 1670 => $expand!(-378), - 1672 => $expand!(-376), - 1674 => $expand!(-374), - 1676 => $expand!(-372), - 1678 => $expand!(-370), - 1680 => $expand!(-368), - 1682 => $expand!(-366), - 1684 => $expand!(-364), - 1686 => $expand!(-362), - 1688 => $expand!(-360), - 1690 => $expand!(-358), - 1692 => $expand!(-356), - 1694 => $expand!(-354), - 1696 => $expand!(-352), - 1698 => $expand!(-350), - 1700 => $expand!(-348), - 1702 => $expand!(-346), - 1704 => $expand!(-344), - 1706 => $expand!(-342), - 1708 => $expand!(-340), - 1710 => $expand!(-338), - 1712 => $expand!(-336), - 1714 => $expand!(-334), - 1716 => $expand!(-332), - 1718 => $expand!(-330), - 1720 => $expand!(-328), - 1722 => $expand!(-326), - 1724 => $expand!(-324), - 1726 => $expand!(-322), - 1728 => $expand!(-320), - 1730 => $expand!(-318), - 1732 => $expand!(-316), - 1734 => $expand!(-314), - 1736 => $expand!(-312), - 1738 => $expand!(-310), - 1740 => $expand!(-308), - 1742 => $expand!(-306), - 1744 => $expand!(-304), - 1746 => $expand!(-302), - 1748 => $expand!(-300), - 1750 => $expand!(-298), - 1752 => $expand!(-296), - 1754 => $expand!(-294), - 1756 => $expand!(-292), - 1758 => $expand!(-290), - 1760 => $expand!(-288), - 1762 => $expand!(-286), - 1764 => $expand!(-284), - 1766 => $expand!(-282), - 1768 => $expand!(-280), - 1770 => $expand!(-278), - 1772 => $expand!(-276), - 1774 => $expand!(-274), - 1776 => $expand!(-272), - 1778 => $expand!(-270), - 1780 => $expand!(-268), - 1782 => $expand!(-266), - 1784 => $expand!(-264), - 1786 => $expand!(-262), - 1788 => $expand!(-260), - 1790 => $expand!(-258), - 1792 => $expand!(-256), - 1794 => $expand!(-254), - 1796 => $expand!(-252), - 1798 => $expand!(-250), - 1800 => $expand!(-248), - 1802 => $expand!(-246), - 1804 => $expand!(-244), - 1806 => $expand!(-242), - 1808 => $expand!(-240), - 1810 => $expand!(-238), - 1812 => $expand!(-236), - 1814 => $expand!(-234), - 1816 => $expand!(-232), - 1818 => $expand!(-230), - 1820 => $expand!(-228), - 1822 => $expand!(-226), - 1824 => $expand!(-224), - 1826 => $expand!(-222), - 1828 => $expand!(-220), - 1830 => $expand!(-218), - 1832 => $expand!(-216), - 1834 => $expand!(-214), - 1836 => $expand!(-212), - 1838 => $expand!(-210), - 1840 => $expand!(-208), - 1842 => $expand!(-206), - 1844 => $expand!(-204), - 1846 => $expand!(-202), - 1848 => $expand!(-200), - 1850 => $expand!(-198), - 1852 => $expand!(-196), - 1854 => $expand!(-194), - 1856 => $expand!(-192), - 1858 => $expand!(-190), - 1860 => $expand!(-188), - 1862 => $expand!(-186), - 1864 => $expand!(-184), - 1866 => $expand!(-182), - 1868 => $expand!(-180), - 1870 => $expand!(-178), - 1872 => $expand!(-176), - 1874 => $expand!(-174), - 1876 => $expand!(-172), - 1878 => $expand!(-170), - 1880 => $expand!(-168), - 1882 => $expand!(-166), - 1884 => $expand!(-164), - 1886 => $expand!(-162), - 1888 => $expand!(-160), - 1890 => $expand!(-158), - 1892 => $expand!(-156), - 1894 => $expand!(-154), - 1896 => $expand!(-152), - 1898 => $expand!(-150), - 1900 => $expand!(-148), - 1902 => $expand!(-146), - 1904 => $expand!(-144), - 1906 => $expand!(-142), - 1908 => $expand!(-140), - 1910 => $expand!(-138), - 1912 => $expand!(-136), - 1914 => $expand!(-134), - 1916 => $expand!(-132), - 1918 => $expand!(-130), - 1920 => $expand!(-128), - 1922 => $expand!(-126), - 1924 => $expand!(-124), - 1926 => $expand!(-122), - 1928 => $expand!(-120), - 1930 => $expand!(-118), - 1932 => $expand!(-116), - 1934 => $expand!(-114), - 1936 => $expand!(-112), - 1938 => $expand!(-110), - 1940 => $expand!(-108), - 1942 => $expand!(-106), - 1944 => $expand!(-104), - 1946 => $expand!(-102), - 1948 => $expand!(-100), - 1950 => $expand!(-98), - 1952 => $expand!(-96), - 1954 => $expand!(-94), - 1956 => $expand!(-92), - 1958 => $expand!(-90), - 1960 => $expand!(-88), - 1962 => $expand!(-86), - 1964 => $expand!(-84), - 1966 => $expand!(-82), - 1968 => $expand!(-80), - 1970 => $expand!(-78), - 1972 => $expand!(-76), - 1974 => $expand!(-74), - 1976 => $expand!(-72), - 1978 => $expand!(-70), - 1980 => $expand!(-68), - 1982 => $expand!(-66), - 1984 => $expand!(-64), - 1986 => $expand!(-62), - 1988 => $expand!(-60), - 1990 => $expand!(-58), - 1992 => $expand!(-56), - 1994 => $expand!(-54), - 1996 => $expand!(-52), - 1998 => $expand!(-50), - 2000 => $expand!(-48), - 2002 => $expand!(-46), - 2004 => $expand!(-44), - 2006 => $expand!(-42), - 2008 => $expand!(-40), - 2010 => $expand!(-38), - 2012 => $expand!(-36), - 2014 => $expand!(-34), - 2016 => $expand!(-32), - 2018 => $expand!(-30), - 2020 => $expand!(-28), - 2022 => $expand!(-26), - 2024 => $expand!(-24), - 2026 => $expand!(-22), - 2028 => $expand!(-20), - 2030 => $expand!(-18), - 2032 => $expand!(-16), - 2034 => $expand!(-14), - 2036 => $expand!(-12), - 2038 => $expand!(-10), - 2040 => $expand!(-8), - 2042 => $expand!(-6), - 2044 => $expand!(-4), - 2046 => $expand!(-2), - _ => $expand!(1022), - } +macro_rules! static_assert_imm_s11 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, -1024, 1023>::VALID; }; } -//immediate value: -512:511 -macro_rules! constify_imm_s10 { - ($imm_s10:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm_s10) & 0b11_1111_1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - 15 => $expand!(15), - 16 => $expand!(16), - 17 => $expand!(17), - 18 => $expand!(18), - 19 => $expand!(19), - 20 => $expand!(20), - 21 => $expand!(21), - 22 => $expand!(22), - 23 => $expand!(23), - 24 => $expand!(24), - 25 => $expand!(25), - 26 => $expand!(26), - 27 => $expand!(27), - 28 => $expand!(28), - 29 => $expand!(29), - 30 => $expand!(30), - 31 => $expand!(31), - 32 => $expand!(32), - 33 => $expand!(33), - 34 => $expand!(34), - 35 => $expand!(35), - 36 => $expand!(36), - 37 => $expand!(37), - 38 => $expand!(38), - 39 => $expand!(39), - 40 => $expand!(40), - 41 => $expand!(41), - 42 => $expand!(42), - 43 => $expand!(43), - 44 => $expand!(44), - 45 => $expand!(45), - 46 => $expand!(46), - 47 => $expand!(47), - 48 => $expand!(48), - 49 => $expand!(49), - 50 => $expand!(50), - 51 => $expand!(51), - 52 => $expand!(52), - 53 => $expand!(53), - 54 => $expand!(54), - 55 => $expand!(55), - 56 => $expand!(56), - 57 => $expand!(57), - 58 => $expand!(58), - 59 => $expand!(59), - 60 => $expand!(60), - 61 => $expand!(61), - 62 => $expand!(62), - 63 => $expand!(63), - 64 => $expand!(64), - 65 => $expand!(65), - 66 => $expand!(66), - 67 => $expand!(67), - 68 => $expand!(68), - 69 => $expand!(69), - 70 => $expand!(70), - 71 => $expand!(71), - 72 => $expand!(72), - 73 => $expand!(73), - 74 => $expand!(74), - 75 => $expand!(75), - 76 => $expand!(76), - 77 => $expand!(77), - 78 => $expand!(78), - 79 => $expand!(79), - 80 => $expand!(80), - 81 => $expand!(81), - 82 => $expand!(82), - 83 => $expand!(83), - 84 => $expand!(84), - 85 => $expand!(85), - 86 => $expand!(86), - 87 => $expand!(87), - 88 => $expand!(88), - 89 => $expand!(89), - 90 => $expand!(90), - 91 => $expand!(91), - 92 => $expand!(92), - 93 => $expand!(93), - 94 => $expand!(94), - 95 => $expand!(95), - 96 => $expand!(96), - 97 => $expand!(97), - 98 => $expand!(98), - 99 => $expand!(99), - 100 => $expand!(100), - 101 => $expand!(101), - 102 => $expand!(102), - 103 => $expand!(103), - 104 => $expand!(104), - 105 => $expand!(105), - 106 => $expand!(106), - 107 => $expand!(107), - 108 => $expand!(108), - 109 => $expand!(109), - 110 => $expand!(110), - 111 => $expand!(111), - 112 => $expand!(112), - 113 => $expand!(113), - 114 => $expand!(114), - 115 => $expand!(115), - 116 => $expand!(116), - 117 => $expand!(117), - 118 => $expand!(118), - 119 => $expand!(119), - 120 => $expand!(120), - 121 => $expand!(121), - 122 => $expand!(122), - 123 => $expand!(123), - 124 => $expand!(124), - 125 => $expand!(125), - 126 => $expand!(126), - 127 => $expand!(127), - 128 => $expand!(128), - 129 => $expand!(129), - 130 => $expand!(130), - 131 => $expand!(131), - 132 => $expand!(132), - 133 => $expand!(133), - 134 => $expand!(134), - 135 => $expand!(135), - 136 => $expand!(136), - 137 => $expand!(137), - 138 => $expand!(138), - 139 => $expand!(139), - 140 => $expand!(140), - 141 => $expand!(141), - 142 => $expand!(142), - 143 => $expand!(143), - 144 => $expand!(144), - 145 => $expand!(145), - 146 => $expand!(146), - 147 => $expand!(147), - 148 => $expand!(148), - 149 => $expand!(149), - 150 => $expand!(150), - 151 => $expand!(151), - 152 => $expand!(152), - 153 => $expand!(153), - 154 => $expand!(154), - 155 => $expand!(155), - 156 => $expand!(156), - 157 => $expand!(157), - 158 => $expand!(158), - 159 => $expand!(159), - 160 => $expand!(160), - 161 => $expand!(161), - 162 => $expand!(162), - 163 => $expand!(163), - 164 => $expand!(164), - 165 => $expand!(165), - 166 => $expand!(166), - 167 => $expand!(167), - 168 => $expand!(168), - 169 => $expand!(169), - 170 => $expand!(170), - 171 => $expand!(171), - 172 => $expand!(172), - 173 => $expand!(173), - 174 => $expand!(174), - 175 => $expand!(175), - 176 => $expand!(176), - 177 => $expand!(177), - 178 => $expand!(178), - 179 => $expand!(179), - 180 => $expand!(180), - 181 => $expand!(181), - 182 => $expand!(182), - 183 => $expand!(183), - 184 => $expand!(184), - 185 => $expand!(185), - 186 => $expand!(186), - 187 => $expand!(187), - 188 => $expand!(188), - 189 => $expand!(189), - 190 => $expand!(190), - 191 => $expand!(191), - 192 => $expand!(192), - 193 => $expand!(193), - 194 => $expand!(194), - 195 => $expand!(195), - 196 => $expand!(196), - 197 => $expand!(197), - 198 => $expand!(198), - 199 => $expand!(199), - 200 => $expand!(200), - 201 => $expand!(201), - 202 => $expand!(202), - 203 => $expand!(203), - 204 => $expand!(204), - 205 => $expand!(205), - 206 => $expand!(206), - 207 => $expand!(207), - 208 => $expand!(208), - 209 => $expand!(209), - 210 => $expand!(210), - 211 => $expand!(211), - 212 => $expand!(212), - 213 => $expand!(213), - 214 => $expand!(214), - 215 => $expand!(215), - 216 => $expand!(216), - 217 => $expand!(217), - 218 => $expand!(218), - 219 => $expand!(219), - 220 => $expand!(220), - 221 => $expand!(221), - 222 => $expand!(222), - 223 => $expand!(223), - 224 => $expand!(224), - 225 => $expand!(225), - 226 => $expand!(226), - 227 => $expand!(227), - 228 => $expand!(228), - 229 => $expand!(229), - 230 => $expand!(230), - 231 => $expand!(231), - 232 => $expand!(232), - 233 => $expand!(233), - 234 => $expand!(234), - 235 => $expand!(235), - 236 => $expand!(236), - 237 => $expand!(237), - 238 => $expand!(238), - 239 => $expand!(239), - 240 => $expand!(240), - 241 => $expand!(241), - 242 => $expand!(242), - 243 => $expand!(243), - 244 => $expand!(244), - 245 => $expand!(245), - 246 => $expand!(246), - 247 => $expand!(247), - 248 => $expand!(248), - 249 => $expand!(249), - 250 => $expand!(250), - 251 => $expand!(251), - 252 => $expand!(252), - 253 => $expand!(253), - 254 => $expand!(254), - 255 => $expand!(255), - 256 => $expand!(256), - 257 => $expand!(257), - 258 => $expand!(258), - 259 => $expand!(259), - 260 => $expand!(260), - 261 => $expand!(261), - 262 => $expand!(262), - 263 => $expand!(263), - 264 => $expand!(264), - 265 => $expand!(265), - 266 => $expand!(266), - 267 => $expand!(267), - 268 => $expand!(268), - 269 => $expand!(269), - 270 => $expand!(270), - 271 => $expand!(271), - 272 => $expand!(272), - 273 => $expand!(273), - 274 => $expand!(274), - 275 => $expand!(275), - 276 => $expand!(276), - 277 => $expand!(277), - 278 => $expand!(278), - 279 => $expand!(279), - 280 => $expand!(280), - 281 => $expand!(281), - 282 => $expand!(282), - 283 => $expand!(283), - 284 => $expand!(284), - 285 => $expand!(285), - 286 => $expand!(286), - 287 => $expand!(287), - 288 => $expand!(288), - 289 => $expand!(289), - 290 => $expand!(290), - 291 => $expand!(291), - 292 => $expand!(292), - 293 => $expand!(293), - 294 => $expand!(294), - 295 => $expand!(295), - 296 => $expand!(296), - 297 => $expand!(297), - 298 => $expand!(298), - 299 => $expand!(299), - 300 => $expand!(300), - 301 => $expand!(301), - 302 => $expand!(302), - 303 => $expand!(303), - 304 => $expand!(304), - 305 => $expand!(305), - 306 => $expand!(306), - 307 => $expand!(307), - 308 => $expand!(308), - 309 => $expand!(309), - 310 => $expand!(310), - 311 => $expand!(311), - 312 => $expand!(312), - 313 => $expand!(313), - 314 => $expand!(314), - 315 => $expand!(315), - 316 => $expand!(316), - 317 => $expand!(317), - 318 => $expand!(318), - 319 => $expand!(319), - 320 => $expand!(320), - 321 => $expand!(321), - 322 => $expand!(322), - 323 => $expand!(323), - 324 => $expand!(324), - 325 => $expand!(325), - 326 => $expand!(326), - 327 => $expand!(327), - 328 => $expand!(328), - 329 => $expand!(329), - 330 => $expand!(330), - 331 => $expand!(331), - 332 => $expand!(332), - 333 => $expand!(333), - 334 => $expand!(334), - 335 => $expand!(335), - 336 => $expand!(336), - 337 => $expand!(337), - 338 => $expand!(338), - 339 => $expand!(339), - 340 => $expand!(340), - 341 => $expand!(341), - 342 => $expand!(342), - 343 => $expand!(343), - 344 => $expand!(344), - 345 => $expand!(345), - 346 => $expand!(346), - 347 => $expand!(347), - 348 => $expand!(348), - 349 => $expand!(349), - 350 => $expand!(350), - 351 => $expand!(351), - 352 => $expand!(352), - 353 => $expand!(353), - 354 => $expand!(354), - 355 => $expand!(355), - 356 => $expand!(356), - 357 => $expand!(357), - 358 => $expand!(358), - 359 => $expand!(359), - 360 => $expand!(360), - 361 => $expand!(361), - 362 => $expand!(362), - 363 => $expand!(363), - 364 => $expand!(364), - 365 => $expand!(365), - 366 => $expand!(366), - 367 => $expand!(367), - 368 => $expand!(368), - 369 => $expand!(369), - 370 => $expand!(370), - 371 => $expand!(371), - 372 => $expand!(372), - 373 => $expand!(373), - 374 => $expand!(374), - 375 => $expand!(375), - 376 => $expand!(376), - 377 => $expand!(377), - 378 => $expand!(378), - 379 => $expand!(379), - 380 => $expand!(380), - 381 => $expand!(381), - 382 => $expand!(382), - 383 => $expand!(383), - 384 => $expand!(384), - 385 => $expand!(385), - 386 => $expand!(386), - 387 => $expand!(387), - 388 => $expand!(388), - 389 => $expand!(389), - 390 => $expand!(390), - 391 => $expand!(391), - 392 => $expand!(392), - 393 => $expand!(393), - 394 => $expand!(394), - 395 => $expand!(395), - 396 => $expand!(396), - 397 => $expand!(397), - 398 => $expand!(398), - 399 => $expand!(399), - 400 => $expand!(400), - 401 => $expand!(401), - 402 => $expand!(402), - 403 => $expand!(403), - 404 => $expand!(404), - 405 => $expand!(405), - 406 => $expand!(406), - 407 => $expand!(407), - 408 => $expand!(408), - 409 => $expand!(409), - 410 => $expand!(410), - 411 => $expand!(411), - 412 => $expand!(412), - 413 => $expand!(413), - 414 => $expand!(414), - 415 => $expand!(415), - 416 => $expand!(416), - 417 => $expand!(417), - 418 => $expand!(418), - 419 => $expand!(419), - 420 => $expand!(420), - 421 => $expand!(421), - 422 => $expand!(422), - 423 => $expand!(423), - 424 => $expand!(424), - 425 => $expand!(425), - 426 => $expand!(426), - 427 => $expand!(427), - 428 => $expand!(428), - 429 => $expand!(429), - 430 => $expand!(430), - 431 => $expand!(431), - 432 => $expand!(432), - 433 => $expand!(433), - 434 => $expand!(434), - 435 => $expand!(435), - 436 => $expand!(436), - 437 => $expand!(437), - 438 => $expand!(438), - 439 => $expand!(439), - 440 => $expand!(440), - 441 => $expand!(441), - 442 => $expand!(442), - 443 => $expand!(443), - 444 => $expand!(444), - 445 => $expand!(445), - 446 => $expand!(446), - 447 => $expand!(447), - 448 => $expand!(448), - 449 => $expand!(449), - 450 => $expand!(450), - 451 => $expand!(451), - 452 => $expand!(452), - 453 => $expand!(453), - 454 => $expand!(454), - 455 => $expand!(455), - 456 => $expand!(456), - 457 => $expand!(457), - 458 => $expand!(458), - 459 => $expand!(459), - 460 => $expand!(460), - 461 => $expand!(461), - 462 => $expand!(462), - 463 => $expand!(463), - 464 => $expand!(464), - 465 => $expand!(465), - 466 => $expand!(466), - 467 => $expand!(467), - 468 => $expand!(468), - 469 => $expand!(469), - 470 => $expand!(470), - 471 => $expand!(471), - 472 => $expand!(472), - 473 => $expand!(473), - 474 => $expand!(474), - 475 => $expand!(475), - 476 => $expand!(476), - 477 => $expand!(477), - 478 => $expand!(478), - 479 => $expand!(479), - 480 => $expand!(480), - 481 => $expand!(481), - 482 => $expand!(482), - 483 => $expand!(483), - 484 => $expand!(484), - 485 => $expand!(485), - 486 => $expand!(486), - 487 => $expand!(487), - 488 => $expand!(488), - 489 => $expand!(489), - 490 => $expand!(490), - 491 => $expand!(491), - 492 => $expand!(492), - 493 => $expand!(493), - 494 => $expand!(494), - 495 => $expand!(495), - 496 => $expand!(496), - 497 => $expand!(497), - 498 => $expand!(498), - 499 => $expand!(499), - 500 => $expand!(500), - 501 => $expand!(501), - 502 => $expand!(502), - 503 => $expand!(503), - 504 => $expand!(504), - 505 => $expand!(505), - 506 => $expand!(506), - 507 => $expand!(507), - 508 => $expand!(508), - 509 => $expand!(509), - 510 => $expand!(510), - 512 => $expand!(-512), - 513 => $expand!(-511), - 514 => $expand!(-510), - 515 => $expand!(-509), - 516 => $expand!(-508), - 517 => $expand!(-507), - 518 => $expand!(-506), - 519 => $expand!(-505), - 520 => $expand!(-504), - 521 => $expand!(-503), - 522 => $expand!(-502), - 523 => $expand!(-501), - 524 => $expand!(-500), - 525 => $expand!(-499), - 526 => $expand!(-498), - 527 => $expand!(-497), - 528 => $expand!(-496), - 529 => $expand!(-495), - 530 => $expand!(-494), - 531 => $expand!(-493), - 532 => $expand!(-492), - 533 => $expand!(-491), - 534 => $expand!(-490), - 535 => $expand!(-489), - 536 => $expand!(-488), - 537 => $expand!(-487), - 538 => $expand!(-486), - 539 => $expand!(-485), - 540 => $expand!(-484), - 541 => $expand!(-483), - 542 => $expand!(-482), - 543 => $expand!(-481), - 544 => $expand!(-480), - 545 => $expand!(-479), - 546 => $expand!(-478), - 547 => $expand!(-477), - 548 => $expand!(-476), - 549 => $expand!(-475), - 550 => $expand!(-474), - 551 => $expand!(-473), - 552 => $expand!(-472), - 553 => $expand!(-471), - 554 => $expand!(-470), - 555 => $expand!(-469), - 556 => $expand!(-468), - 557 => $expand!(-467), - 558 => $expand!(-466), - 559 => $expand!(-465), - 560 => $expand!(-464), - 561 => $expand!(-463), - 562 => $expand!(-462), - 563 => $expand!(-461), - 564 => $expand!(-460), - 565 => $expand!(-459), - 566 => $expand!(-458), - 567 => $expand!(-457), - 568 => $expand!(-456), - 569 => $expand!(-455), - 570 => $expand!(-454), - 571 => $expand!(-453), - 572 => $expand!(-452), - 573 => $expand!(-451), - 574 => $expand!(-450), - 575 => $expand!(-449), - 576 => $expand!(-448), - 577 => $expand!(-447), - 578 => $expand!(-446), - 579 => $expand!(-445), - 580 => $expand!(-444), - 581 => $expand!(-443), - 582 => $expand!(-442), - 583 => $expand!(-441), - 584 => $expand!(-440), - 585 => $expand!(-439), - 586 => $expand!(-438), - 587 => $expand!(-437), - 588 => $expand!(-436), - 589 => $expand!(-435), - 590 => $expand!(-434), - 591 => $expand!(-433), - 592 => $expand!(-432), - 593 => $expand!(-431), - 594 => $expand!(-430), - 595 => $expand!(-429), - 596 => $expand!(-428), - 597 => $expand!(-427), - 598 => $expand!(-426), - 599 => $expand!(-425), - 600 => $expand!(-424), - 601 => $expand!(-423), - 602 => $expand!(-422), - 603 => $expand!(-421), - 604 => $expand!(-420), - 605 => $expand!(-419), - 606 => $expand!(-418), - 607 => $expand!(-417), - 608 => $expand!(-416), - 609 => $expand!(-415), - 610 => $expand!(-414), - 611 => $expand!(-413), - 612 => $expand!(-412), - 613 => $expand!(-411), - 614 => $expand!(-410), - 615 => $expand!(-409), - 616 => $expand!(-408), - 617 => $expand!(-407), - 618 => $expand!(-406), - 619 => $expand!(-405), - 620 => $expand!(-404), - 621 => $expand!(-403), - 622 => $expand!(-402), - 623 => $expand!(-401), - 624 => $expand!(-400), - 625 => $expand!(-399), - 626 => $expand!(-398), - 627 => $expand!(-397), - 628 => $expand!(-396), - 629 => $expand!(-395), - 630 => $expand!(-394), - 631 => $expand!(-393), - 632 => $expand!(-392), - 633 => $expand!(-391), - 634 => $expand!(-390), - 635 => $expand!(-389), - 636 => $expand!(-388), - 637 => $expand!(-387), - 638 => $expand!(-386), - 639 => $expand!(-385), - 640 => $expand!(-384), - 641 => $expand!(-383), - 642 => $expand!(-382), - 643 => $expand!(-381), - 644 => $expand!(-380), - 645 => $expand!(-379), - 646 => $expand!(-378), - 647 => $expand!(-377), - 648 => $expand!(-376), - 649 => $expand!(-375), - 650 => $expand!(-374), - 651 => $expand!(-373), - 652 => $expand!(-372), - 653 => $expand!(-371), - 654 => $expand!(-370), - 655 => $expand!(-369), - 656 => $expand!(-368), - 657 => $expand!(-367), - 658 => $expand!(-366), - 659 => $expand!(-365), - 660 => $expand!(-364), - 661 => $expand!(-363), - 662 => $expand!(-362), - 663 => $expand!(-361), - 664 => $expand!(-360), - 665 => $expand!(-359), - 666 => $expand!(-358), - 667 => $expand!(-357), - 668 => $expand!(-356), - 669 => $expand!(-355), - 670 => $expand!(-354), - 671 => $expand!(-353), - 672 => $expand!(-352), - 673 => $expand!(-351), - 674 => $expand!(-350), - 675 => $expand!(-349), - 676 => $expand!(-348), - 677 => $expand!(-347), - 678 => $expand!(-346), - 679 => $expand!(-345), - 680 => $expand!(-344), - 681 => $expand!(-343), - 682 => $expand!(-342), - 683 => $expand!(-341), - 684 => $expand!(-340), - 685 => $expand!(-339), - 686 => $expand!(-338), - 687 => $expand!(-337), - 688 => $expand!(-336), - 689 => $expand!(-335), - 690 => $expand!(-334), - 691 => $expand!(-333), - 692 => $expand!(-332), - 693 => $expand!(-331), - 694 => $expand!(-330), - 695 => $expand!(-329), - 696 => $expand!(-328), - 697 => $expand!(-327), - 698 => $expand!(-326), - 699 => $expand!(-325), - 700 => $expand!(-324), - 701 => $expand!(-323), - 702 => $expand!(-322), - 703 => $expand!(-321), - 704 => $expand!(-320), - 705 => $expand!(-319), - 706 => $expand!(-318), - 707 => $expand!(-317), - 708 => $expand!(-316), - 709 => $expand!(-315), - 710 => $expand!(-314), - 711 => $expand!(-313), - 712 => $expand!(-312), - 713 => $expand!(-311), - 714 => $expand!(-310), - 715 => $expand!(-309), - 716 => $expand!(-308), - 717 => $expand!(-307), - 718 => $expand!(-306), - 719 => $expand!(-305), - 720 => $expand!(-304), - 721 => $expand!(-303), - 722 => $expand!(-302), - 723 => $expand!(-301), - 724 => $expand!(-300), - 725 => $expand!(-299), - 726 => $expand!(-298), - 727 => $expand!(-297), - 728 => $expand!(-296), - 729 => $expand!(-295), - 730 => $expand!(-294), - 731 => $expand!(-293), - 732 => $expand!(-292), - 733 => $expand!(-291), - 734 => $expand!(-290), - 735 => $expand!(-289), - 736 => $expand!(-288), - 737 => $expand!(-287), - 738 => $expand!(-286), - 739 => $expand!(-285), - 740 => $expand!(-284), - 741 => $expand!(-283), - 742 => $expand!(-282), - 743 => $expand!(-281), - 744 => $expand!(-280), - 745 => $expand!(-279), - 746 => $expand!(-278), - 747 => $expand!(-277), - 748 => $expand!(-276), - 749 => $expand!(-275), - 750 => $expand!(-274), - 751 => $expand!(-273), - 752 => $expand!(-272), - 753 => $expand!(-271), - 754 => $expand!(-270), - 755 => $expand!(-269), - 756 => $expand!(-268), - 757 => $expand!(-267), - 758 => $expand!(-266), - 759 => $expand!(-265), - 760 => $expand!(-264), - 761 => $expand!(-263), - 762 => $expand!(-262), - 763 => $expand!(-261), - 764 => $expand!(-260), - 765 => $expand!(-259), - 766 => $expand!(-258), - 767 => $expand!(-257), - 768 => $expand!(-256), - 769 => $expand!(-255), - 770 => $expand!(-254), - 771 => $expand!(-253), - 772 => $expand!(-252), - 773 => $expand!(-251), - 774 => $expand!(-250), - 775 => $expand!(-249), - 776 => $expand!(-248), - 777 => $expand!(-247), - 778 => $expand!(-246), - 779 => $expand!(-245), - 780 => $expand!(-244), - 781 => $expand!(-243), - 782 => $expand!(-242), - 783 => $expand!(-241), - 784 => $expand!(-240), - 785 => $expand!(-239), - 786 => $expand!(-238), - 787 => $expand!(-237), - 788 => $expand!(-236), - 789 => $expand!(-235), - 790 => $expand!(-234), - 791 => $expand!(-233), - 792 => $expand!(-232), - 793 => $expand!(-231), - 794 => $expand!(-230), - 795 => $expand!(-229), - 796 => $expand!(-228), - 797 => $expand!(-227), - 798 => $expand!(-226), - 799 => $expand!(-225), - 800 => $expand!(-224), - 801 => $expand!(-223), - 802 => $expand!(-222), - 803 => $expand!(-221), - 804 => $expand!(-220), - 805 => $expand!(-219), - 806 => $expand!(-218), - 807 => $expand!(-217), - 808 => $expand!(-216), - 809 => $expand!(-215), - 810 => $expand!(-214), - 811 => $expand!(-213), - 812 => $expand!(-212), - 813 => $expand!(-211), - 814 => $expand!(-210), - 815 => $expand!(-209), - 816 => $expand!(-208), - 817 => $expand!(-207), - 818 => $expand!(-206), - 819 => $expand!(-205), - 820 => $expand!(-204), - 821 => $expand!(-203), - 822 => $expand!(-202), - 823 => $expand!(-201), - 824 => $expand!(-200), - 825 => $expand!(-199), - 826 => $expand!(-198), - 827 => $expand!(-197), - 828 => $expand!(-196), - 829 => $expand!(-195), - 830 => $expand!(-194), - 831 => $expand!(-193), - 832 => $expand!(-192), - 833 => $expand!(-191), - 834 => $expand!(-190), - 835 => $expand!(-189), - 836 => $expand!(-188), - 837 => $expand!(-187), - 838 => $expand!(-186), - 839 => $expand!(-185), - 840 => $expand!(-184), - 841 => $expand!(-183), - 842 => $expand!(-182), - 843 => $expand!(-181), - 844 => $expand!(-180), - 845 => $expand!(-179), - 846 => $expand!(-178), - 847 => $expand!(-177), - 848 => $expand!(-176), - 849 => $expand!(-175), - 850 => $expand!(-174), - 851 => $expand!(-173), - 852 => $expand!(-172), - 853 => $expand!(-171), - 854 => $expand!(-170), - 855 => $expand!(-169), - 856 => $expand!(-168), - 857 => $expand!(-167), - 858 => $expand!(-166), - 859 => $expand!(-165), - 860 => $expand!(-164), - 861 => $expand!(-163), - 862 => $expand!(-162), - 863 => $expand!(-161), - 864 => $expand!(-160), - 865 => $expand!(-159), - 866 => $expand!(-158), - 867 => $expand!(-157), - 868 => $expand!(-156), - 869 => $expand!(-155), - 870 => $expand!(-154), - 871 => $expand!(-153), - 872 => $expand!(-152), - 873 => $expand!(-151), - 874 => $expand!(-150), - 875 => $expand!(-149), - 876 => $expand!(-148), - 877 => $expand!(-147), - 878 => $expand!(-146), - 879 => $expand!(-145), - 880 => $expand!(-144), - 881 => $expand!(-143), - 882 => $expand!(-142), - 883 => $expand!(-141), - 884 => $expand!(-140), - 885 => $expand!(-139), - 886 => $expand!(-138), - 887 => $expand!(-137), - 888 => $expand!(-136), - 889 => $expand!(-135), - 890 => $expand!(-134), - 891 => $expand!(-133), - 892 => $expand!(-132), - 893 => $expand!(-131), - 894 => $expand!(-130), - 895 => $expand!(-129), - 896 => $expand!(-128), - 897 => $expand!(-127), - 898 => $expand!(-126), - 899 => $expand!(-125), - 900 => $expand!(-124), - 901 => $expand!(-123), - 902 => $expand!(-122), - 903 => $expand!(-121), - 904 => $expand!(-120), - 905 => $expand!(-119), - 906 => $expand!(-118), - 907 => $expand!(-117), - 908 => $expand!(-116), - 909 => $expand!(-115), - 910 => $expand!(-114), - 911 => $expand!(-113), - 912 => $expand!(-112), - 913 => $expand!(-111), - 914 => $expand!(-110), - 915 => $expand!(-109), - 916 => $expand!(-108), - 917 => $expand!(-107), - 918 => $expand!(-106), - 919 => $expand!(-105), - 920 => $expand!(-104), - 921 => $expand!(-103), - 922 => $expand!(-102), - 923 => $expand!(-101), - 924 => $expand!(-100), - 925 => $expand!(-99), - 926 => $expand!(-98), - 927 => $expand!(-97), - 928 => $expand!(-96), - 929 => $expand!(-95), - 930 => $expand!(-94), - 931 => $expand!(-93), - 932 => $expand!(-92), - 933 => $expand!(-91), - 934 => $expand!(-90), - 935 => $expand!(-89), - 936 => $expand!(-88), - 937 => $expand!(-87), - 938 => $expand!(-86), - 939 => $expand!(-85), - 940 => $expand!(-84), - 941 => $expand!(-83), - 942 => $expand!(-82), - 943 => $expand!(-81), - 944 => $expand!(-80), - 945 => $expand!(-79), - 946 => $expand!(-78), - 947 => $expand!(-77), - 948 => $expand!(-76), - 949 => $expand!(-75), - 950 => $expand!(-74), - 951 => $expand!(-73), - 952 => $expand!(-72), - 953 => $expand!(-71), - 954 => $expand!(-70), - 955 => $expand!(-69), - 956 => $expand!(-68), - 957 => $expand!(-67), - 958 => $expand!(-66), - 959 => $expand!(-65), - 960 => $expand!(-64), - 961 => $expand!(-63), - 962 => $expand!(-62), - 963 => $expand!(-61), - 964 => $expand!(-60), - 965 => $expand!(-59), - 966 => $expand!(-58), - 967 => $expand!(-57), - 968 => $expand!(-56), - 969 => $expand!(-55), - 970 => $expand!(-54), - 971 => $expand!(-53), - 972 => $expand!(-52), - 973 => $expand!(-51), - 974 => $expand!(-50), - 975 => $expand!(-49), - 976 => $expand!(-48), - 977 => $expand!(-47), - 978 => $expand!(-46), - 979 => $expand!(-45), - 980 => $expand!(-44), - 981 => $expand!(-43), - 982 => $expand!(-42), - 983 => $expand!(-41), - 984 => $expand!(-40), - 985 => $expand!(-39), - 986 => $expand!(-38), - 987 => $expand!(-37), - 988 => $expand!(-36), - 989 => $expand!(-35), - 990 => $expand!(-34), - 991 => $expand!(-33), - 992 => $expand!(-32), - 993 => $expand!(-31), - 994 => $expand!(-30), - 995 => $expand!(-29), - 996 => $expand!(-28), - 997 => $expand!(-27), - 998 => $expand!(-26), - 999 => $expand!(-25), - 1000 => $expand!(-24), - 1001 => $expand!(-23), - 1002 => $expand!(-22), - 1003 => $expand!(-21), - 1004 => $expand!(-20), - 1005 => $expand!(-19), - 1006 => $expand!(-18), - 1007 => $expand!(-17), - 1008 => $expand!(-16), - 1009 => $expand!(-15), - 1010 => $expand!(-14), - 1011 => $expand!(-13), - 1012 => $expand!(-12), - 1013 => $expand!(-11), - 1014 => $expand!(-10), - 1015 => $expand!(-9), - 1016 => $expand!(-8), - 1017 => $expand!(-7), - 1018 => $expand!(-6), - 1019 => $expand!(-5), - 1020 => $expand!(-4), - 1021 => $expand!(-3), - 1022 => $expand!(-2), - 1023 => $expand!(-1), - _ => $expand!(511), - } +macro_rules! static_assert_imm_s12 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, -2048, 2047>::VALID; }; } -//immediate value: 0:63 -macro_rules! constify_imm6 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b11_1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - 15 => $expand!(15), - 16 => $expand!(16), - 17 => $expand!(17), - 18 => $expand!(18), - 19 => $expand!(19), - 20 => $expand!(20), - 21 => $expand!(21), - 22 => $expand!(22), - 23 => $expand!(23), - 24 => $expand!(24), - 25 => $expand!(25), - 26 => $expand!(26), - 27 => $expand!(27), - 28 => $expand!(28), - 29 => $expand!(29), - 30 => $expand!(30), - 31 => $expand!(31), - 32 => $expand!(32), - 33 => $expand!(33), - 34 => $expand!(34), - 35 => $expand!(35), - 36 => $expand!(36), - 37 => $expand!(37), - 38 => $expand!(38), - 39 => $expand!(39), - 40 => $expand!(40), - 41 => $expand!(41), - 42 => $expand!(42), - 43 => $expand!(43), - 44 => $expand!(44), - 45 => $expand!(45), - 46 => $expand!(46), - 47 => $expand!(47), - 48 => $expand!(48), - 49 => $expand!(49), - 50 => $expand!(50), - 51 => $expand!(51), - 52 => $expand!(52), - 53 => $expand!(53), - 54 => $expand!(54), - 55 => $expand!(55), - 56 => $expand!(56), - 57 => $expand!(57), - 58 => $expand!(58), - 59 => $expand!(59), - 60 => $expand!(60), - 61 => $expand!(61), - 62 => $expand!(62), - _ => $expand!(63), - } - }; -} - -//immediate value: 0:31 -macro_rules! constify_imm5 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1_1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - 15 => $expand!(15), - 16 => $expand!(16), - 17 => $expand!(17), - 18 => $expand!(18), - 19 => $expand!(19), - 20 => $expand!(20), - 21 => $expand!(21), - 22 => $expand!(22), - 23 => $expand!(23), - 24 => $expand!(24), - 25 => $expand!(25), - 26 => $expand!(26), - 27 => $expand!(27), - 28 => $expand!(28), - 29 => $expand!(29), - 30 => $expand!(30), - _ => $expand!(31), - } - }; -} - -//immediate value: -16:15 -macro_rules! constify_imm_s5 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1_1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - 16 => $expand!(-16), - 17 => $expand!(-15), - 18 => $expand!(-14), - 19 => $expand!(-13), - 20 => $expand!(-12), - 21 => $expand!(-11), - 22 => $expand!(-10), - 23 => $expand!(-9), - 24 => $expand!(-8), - 25 => $expand!(-7), - 26 => $expand!(-6), - 27 => $expand!(-5), - 28 => $expand!(-4), - 29 => $expand!(-3), - 30 => $expand!(-2), - 31 => $expand!(-1), - _ => $expand!(15), - } - }; -} - -//immediate value: 0:15 -macro_rules! constify_imm4 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - _ => $expand!(15), - } - }; -} - -//immediate value: 0:7 -macro_rules! constify_imm3 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - _ => $expand!(7), - } - }; -} - -//immediate value: 0:3 -macro_rules! constify_imm2 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b11 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - _ => $expand!(3), - } - }; -} - -//immediate value: 0:1 -macro_rules! constify_imm1 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1 { - 0 => $expand!(0), - _ => $expand!(1), - } +macro_rules! static_assert_imm_s13 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, -4096, 4095>::VALID; }; } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/mod.rs b/crux-mir/lib/stdarch/crates/core_arch/src/mod.rs index eecfaed36..2f7af22cb 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/mod.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/mod.rs @@ -3,18 +3,21 @@ #[macro_use] mod macros; -#[cfg(any(target_arch = "arm", target_arch = "aarch64", dox))] -mod acle; +#[cfg(any(target_arch = "riscv32", target_arch = "riscv64", doc))] +mod riscv_shared; + +#[cfg(any(target_arch = "arm", target_arch = "aarch64", doc))] +mod arm_shared; mod simd; -#[doc(include = "core_arch_docs.md")] +#[doc = include_str!("core_arch_docs.md")] #[stable(feature = "simd_arch", since = "1.27.0")] pub mod arch { /// Platform-specific intrinsics for the `x86` platform. /// /// See the [module documentation](../index.html) for more details. - #[cfg(any(target_arch = "x86", dox))] + #[cfg(any(target_arch = "x86", doc))] #[doc(cfg(target_arch = "x86"))] #[stable(feature = "simd_x86", since = "1.27.0")] pub mod x86 { @@ -25,7 +28,7 @@ pub mod arch { /// Platform-specific intrinsics for the `x86_64` platform. /// /// See the [module documentation](../index.html) for more details. - #[cfg(any(target_arch = "x86_64", dox))] + #[cfg(any(target_arch = "x86_64", doc))] #[doc(cfg(target_arch = "x86_64"))] #[stable(feature = "simd_x86", since = "1.27.0")] pub mod x86_64 { @@ -38,7 +41,7 @@ pub mod arch { /// Platform-specific intrinsics for the `arm` platform. /// /// See the [module documentation](../index.html) for more details. - #[cfg(any(target_arch = "arm", dox))] + #[cfg(any(target_arch = "arm", doc))] #[doc(cfg(target_arch = "arm"))] #[unstable(feature = "stdsimd", issue = "27731")] pub mod arm { @@ -48,17 +51,138 @@ pub mod arch { /// Platform-specific intrinsics for the `aarch64` platform. /// /// See the [module documentation](../index.html) for more details. - #[cfg(any(target_arch = "aarch64", dox))] + #[cfg(any(target_arch = "aarch64", doc))] #[doc(cfg(target_arch = "aarch64"))] - #[unstable(feature = "stdsimd", issue = "27731")] + #[stable(feature = "neon_intrinsics", since = "1.59.0")] pub mod aarch64 { - pub use crate::core_arch::{aarch64::*, arm::*}; + #[stable(feature = "neon_intrinsics", since = "1.59.0")] + pub use crate::core_arch::aarch64::*; } - /// Platform-specific intrinsics for the `wasm32` platform. + /// Platform-specific intrinsics for the `riscv32` platform. /// /// See the [module documentation](../index.html) for more details. - #[cfg(any(target_arch = "wasm32", dox))] + #[cfg(any(target_arch = "riscv32", doc))] + #[doc(cfg(any(target_arch = "riscv32")))] + #[unstable(feature = "stdsimd", issue = "27731")] + pub mod riscv32 { + pub use crate::core_arch::riscv_shared::*; + } + + /// Platform-specific intrinsics for the `riscv64` platform. + /// + /// See the [module documentation](../index.html) for more details. + #[cfg(any(target_arch = "riscv64", doc))] + #[doc(cfg(any(target_arch = "riscv64")))] + #[unstable(feature = "stdsimd", issue = "27731")] + pub mod riscv64 { + pub use crate::core_arch::riscv64::*; + // RISC-V RV64 supports all RV32 instructions as well in current specifications (2022-01-05). + // Module `riscv_shared` includes instructions available under all RISC-V platforms, + // i.e. RISC-V RV32 instructions. + pub use crate::core_arch::riscv_shared::*; + } + + /// Platform-specific intrinsics for the `wasm32` platform. + /// + /// This module provides intrinsics specific to the WebAssembly + /// architecture. Here you'll find intrinsics specific to WebAssembly that + /// aren't otherwise surfaced somewhere in a cross-platform abstraction of + /// `std`, and you'll also find functions for leveraging WebAssembly + /// proposals such as [atomics] and [simd]. + /// + /// Intrinsics in the `wasm32` module are modeled after the WebAssembly + /// instructions that they represent. Most functions are named after the + /// instruction they intend to correspond to, and the arguments/results + /// correspond to the type signature of the instruction itself. Stable + /// WebAssembly instructions are [documented online][instrdoc]. + /// + /// [instrdoc]: https://webassembly.github.io/spec/core/valid/instructions.html + /// + /// If a proposal is not yet stable in WebAssembly itself then the functions + /// within this function may be unstable and require the nightly channel of + /// Rust to use. As the proposal itself stabilizes the intrinsics in this + /// module should stabilize as well. + /// + /// [atomics]: https://github.com/webassembly/threads + /// [simd]: https://github.com/webassembly/simd + /// + /// See the [module documentation](../index.html) for general information + /// about the `arch` module and platform intrinsics. + /// + /// ## Atomics + /// + /// The [threads proposal][atomics] for WebAssembly adds a number of + /// instructions for dealing with multithreaded programs. Most instructions + /// added in the [atomics] proposal are exposed in Rust through the + /// `std::sync::atomic` module. Some instructions, however, don't have + /// direct equivalents in Rust so they're exposed here instead. + /// + /// Note that the instructions added in the [atomics] proposal can work in + /// either a context with a shared wasm memory and without. These intrinsics + /// are always available in the standard library, but you likely won't be + /// able to use them too productively unless you recompile the standard + /// library (and all your code) with `-Ctarget-feature=+atomics`. + /// + /// It's also worth pointing out that multi-threaded WebAssembly and its + /// story in Rust is still in a somewhat "early days" phase as of the time + /// of this writing. Pieces should mostly work but it generally requires a + /// good deal of manual setup. At this time it's not as simple as "just call + /// `std::thread::spawn`", but it will hopefully get there one day! + /// + /// ## SIMD + /// + /// The [simd proposal][simd] for WebAssembly added a new `v128` type for a + /// 128-bit SIMD register. It also added a large array of instructions to + /// operate on the `v128` type to perform data processing. Using SIMD on + /// wasm is intended to be similar to as you would on `x86_64`, for example. + /// You'd write a function such as: + /// + /// ```rust,ignore + /// #[cfg(target_arch = "wasm32")] + /// #[target_feature(enable = "simd128")] + /// unsafe fn uses_simd() { + /// use std::arch::wasm32::*; + /// // ... + /// } + /// ``` + /// + /// Unlike `x86_64`, however, WebAssembly does not currently have dynamic + /// detection at runtime as to whether SIMD is supported (this is one of the + /// motivators for the [conditional sections][condsections] and [feature + /// detection] proposals, but that is still pretty early days). This means + /// that your binary will either have SIMD and can only run on engines + /// which support SIMD, or it will not have SIMD at all. For compatibility + /// the standard library itself does not use any SIMD internally. + /// Determining how best to ship your WebAssembly binary with SIMD is + /// largely left up to you as it can can be pretty nuanced depending on + /// your situation. + /// + /// [condsections]: https://github.com/webassembly/conditional-sections + /// [feature detection]: https://github.com/WebAssembly/feature-detection + /// + /// To enable SIMD support at compile time you need to do one of two things: + /// + /// * First you can annotate functions with `#[target_feature(enable = + /// "simd128")]`. This causes just that one function to have SIMD support + /// available to it, and intrinsics will get inlined as usual in this + /// situation. + /// + /// * Second you can compile your program with `-Ctarget-feature=+simd128`. + /// This compilation flag blanket enables SIMD support for your entire + /// compilation. Note that this does not include the standard library + /// unless you [recompile the standard library][buildstd]. + /// + /// [buildstd]: https://doc.rust-lang.org/nightly/cargo/reference/unstable.html#build-std + /// + /// If you enable SIMD via either of these routes then you'll have a + /// WebAssembly binary that uses SIMD instructions, and you'll need to ship + /// that accordingly. Also note that if you call SIMD intrinsics but don't + /// enable SIMD via either of these mechanisms, you'll still have SIMD + /// generated in your program. This means to generate a binary without SIMD + /// you'll need to avoid both options above plus calling into any intrinsics + /// in this module. + #[cfg(any(target_arch = "wasm32", doc))] #[doc(cfg(target_arch = "wasm32"))] #[stable(feature = "simd_wasm32", since = "1.33.0")] pub mod wasm32 { @@ -66,10 +190,32 @@ pub mod arch { pub use crate::core_arch::wasm32::*; } + /// Platform-specific intrinsics for the `wasm64` platform. + /// + /// See the [module documentation](../index.html) for more details. + #[cfg(any(target_arch = "wasm64", doc))] + #[doc(cfg(target_arch = "wasm64"))] + #[unstable(feature = "simd_wasm64", issue = "90599")] + pub mod wasm64 { + #[unstable(feature = "simd_wasm64", issue = "90599")] + pub use crate::core_arch::wasm32::*; + } + + /// Platform-specific intrinsics for the `wasm` target family. + /// + /// See the [module documentation](../index.html) for more details. + #[cfg(any(target_family = "wasm", doc))] + #[doc(cfg(target_family = "wasm"))] + #[unstable(feature = "simd_wasm64", issue = "90599")] + pub mod wasm { + #[unstable(feature = "simd_wasm64", issue = "90599")] + pub use crate::core_arch::wasm32::*; + } + /// Platform-specific intrinsics for the `mips` platform. /// /// See the [module documentation](../index.html) for more details. - #[cfg(any(target_arch = "mips", dox))] + #[cfg(any(target_arch = "mips", doc))] #[doc(cfg(target_arch = "mips"))] #[unstable(feature = "stdsimd", issue = "27731")] pub mod mips { @@ -79,7 +225,7 @@ pub mod arch { /// Platform-specific intrinsics for the `mips64` platform. /// /// See the [module documentation](../index.html) for more details. - #[cfg(any(target_arch = "mips64", dox))] + #[cfg(any(target_arch = "mips64", doc))] #[doc(cfg(target_arch = "mips64"))] #[unstable(feature = "stdsimd", issue = "27731")] pub mod mips64 { @@ -89,7 +235,7 @@ pub mod arch { /// Platform-specific intrinsics for the `PowerPC` platform. /// /// See the [module documentation](../index.html) for more details. - #[cfg(any(target_arch = "powerpc", dox))] + #[cfg(any(target_arch = "powerpc", doc))] #[doc(cfg(target_arch = "powerpc"))] #[unstable(feature = "stdsimd", issue = "27731")] pub mod powerpc { @@ -99,7 +245,7 @@ pub mod arch { /// Platform-specific intrinsics for the `PowerPC64` platform. /// /// See the [module documentation](../index.html) for more details. - #[cfg(any(target_arch = "powerpc64", dox))] + #[cfg(any(target_arch = "powerpc64", doc))] #[doc(cfg(target_arch = "powerpc64"))] #[unstable(feature = "stdsimd", issue = "27731")] pub mod powerpc64 { @@ -109,7 +255,7 @@ pub mod arch { /// Platform-specific intrinsics for the `NVPTX` platform. /// /// See the [module documentation](../index.html) for more details. - #[cfg(any(target_arch = "nvptx", target_arch = "nvptx64", dox))] + #[cfg(any(target_arch = "nvptx", target_arch = "nvptx64", doc))] #[doc(cfg(any(target_arch = "nvptx", target_arch = "nvptx64")))] #[unstable(feature = "stdsimd", issue = "27731")] pub mod nvptx { @@ -119,36 +265,40 @@ pub mod arch { mod simd_llvm; -#[cfg(any(target_arch = "x86", target_arch = "x86_64", dox))] +#[cfg(any(target_arch = "x86", target_arch = "x86_64", doc))] #[doc(cfg(any(target_arch = "x86", target_arch = "x86_64")))] mod x86; -#[cfg(any(target_arch = "x86_64", dox))] +#[cfg(any(target_arch = "x86_64", doc))] #[doc(cfg(target_arch = "x86_64"))] mod x86_64; -#[cfg(any(target_arch = "aarch64", dox))] +#[cfg(any(target_arch = "aarch64", doc))] #[doc(cfg(target_arch = "aarch64"))] mod aarch64; -#[cfg(any(target_arch = "arm", target_arch = "aarch64", dox))] -#[doc(cfg(any(target_arch = "arm", target_arch = "aarch64")))] +#[cfg(any(target_arch = "arm", doc))] +#[doc(cfg(any(target_arch = "arm")))] mod arm; -#[cfg(any(target_arch = "wasm32", dox))] -#[doc(cfg(target_arch = "wasm32"))] +#[cfg(any(target_arch = "riscv64", doc))] +#[doc(cfg(any(target_arch = "riscv64")))] +mod riscv64; + +#[cfg(any(target_family = "wasm", doc))] +#[doc(cfg(target_family = "wasm"))] mod wasm32; -#[cfg(any(target_arch = "mips", target_arch = "mips64", dox))] +#[cfg(any(target_arch = "mips", target_arch = "mips64", doc))] #[doc(cfg(any(target_arch = "mips", target_arch = "mips64")))] mod mips; -#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64", dox))] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64", doc))] #[doc(cfg(any(target_arch = "powerpc", target_arch = "powerpc64")))] mod powerpc; -#[cfg(any(target_arch = "powerpc64", dox))] +#[cfg(any(target_arch = "powerpc64", doc))] #[doc(cfg(target_arch = "powerpc64"))] mod powerpc64; -#[cfg(any(target_arch = "nvptx", target_arch = "nvptx64", dox))] +#[cfg(any(target_arch = "nvptx", target_arch = "nvptx64", doc))] #[doc(cfg(any(target_arch = "nvptx", target_arch = "nvptx64")))] mod nvptx; diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/powerpc/altivec.rs b/crux-mir/lib/stdarch/crates/core_arch/src/powerpc/altivec.rs index 3b6ee2ea1..70344c88b 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/powerpc/altivec.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/powerpc/altivec.rs @@ -51,6 +51,8 @@ types! { #[allow(improper_ctypes)] extern "C" { + #[link_name = "llvm.ppc.altivec.lvx"] + fn lvx(p: *const i8) -> vector_unsigned_int; #[link_name = "llvm.ppc.altivec.vperm"] fn vperm( a: vector_signed_int, @@ -356,6 +358,7 @@ mod sealed { } + #[allow(unknown_lints, unused_macro_rules)] macro_rules! impl_vec_trait { ([$Trait:ident $m:ident] $fun:ident ($a:ty)) => { impl $Trait for $a { @@ -441,8 +444,7 @@ mod sealed { #[inline(always)] unsafe fn load(off: i32, p: *const i8) -> u32x4 { let addr = p.offset(off as isize); - - *(addr as *const u32x4) + transmute(lvx(addr)) } pub trait VectorLd { @@ -2045,32 +2047,32 @@ mod tests { } test_vec_avg! { test_vec_avg_i32x4, i32x4, - [i32::min_value(), i32::max_value(), 1, -1], + [i32::MIN, i32::MAX, 1, -1], [-1, 1, 1, -1], [-1073741824, 1073741824, 1, -1] } test_vec_avg! { test_vec_avg_u32x4, u32x4, - [u32::max_value(), 0, 1, 2], + [u32::MAX, 0, 1, 2], [2, 1, 0, 0], [2147483649, 1, 1, 1] } test_vec_avg! { test_vec_avg_i16x8, i16x8, - [i16::min_value(), i16::max_value(), 1, -1, 0, 0, 0, 0], + [i16::MIN, i16::MAX, 1, -1, 0, 0, 0, 0], [-1, 1, 1, -1, 0, 0, 0, 0], [-16384, 16384, 1, -1, 0, 0, 0, 0] } test_vec_avg! { test_vec_avg_u16x8, u16x8, - [u16::max_value(), 0, 1, 2, 0, 0, 0, 0], + [u16::MAX, 0, 1, 2, 0, 0, 0, 0], [2, 1, 0, 0, 0, 0, 0, 0], [32769, 1, 1, 1, 0, 0, 0, 0] } test_vec_avg! { test_vec_avg_i8x16, i8x16, - [i8::min_value(), i8::max_value(), 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [i8::MIN, i8::MAX, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [-1, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [-64, 64, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } test_vec_avg! { test_vec_avg_u8x16, u8x16, - [u8::max_value(), 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [u8::MAX, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [129, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } @@ -2081,36 +2083,36 @@ mod tests { } test_vec_adds! { test_vec_adds_i32x4, i32x4, - [i32::min_value(), i32::max_value(), 1, -1], + [i32::MIN, i32::MAX, 1, -1], [-1, 1, 1, -1], - [i32::min_value(), i32::max_value(), 2, -2] } + [i32::MIN, i32::MAX, 2, -2] } test_vec_adds! { test_vec_adds_u32x4, u32x4, - [u32::max_value(), 0, 1, 2], + [u32::MAX, 0, 1, 2], [2, 1, 0, 0], - [u32::max_value(), 1, 1, 2] } + [u32::MAX, 1, 1, 2] } test_vec_adds! { test_vec_adds_i16x8, i16x8, - [i16::min_value(), i16::max_value(), 1, -1, 0, 0, 0, 0], + [i16::MIN, i16::MAX, 1, -1, 0, 0, 0, 0], [-1, 1, 1, -1, 0, 0, 0, 0], - [i16::min_value(), i16::max_value(), 2, -2, 0, 0, 0, 0] } + [i16::MIN, i16::MAX, 2, -2, 0, 0, 0, 0] } test_vec_adds! { test_vec_adds_u16x8, u16x8, - [u16::max_value(), 0, 1, 2, 0, 0, 0, 0], + [u16::MAX, 0, 1, 2, 0, 0, 0, 0], [2, 1, 0, 0, 0, 0, 0, 0], - [u16::max_value(), 1, 1, 2, 0, 0, 0, 0] } + [u16::MAX, 1, 1, 2, 0, 0, 0, 0] } test_vec_adds! { test_vec_adds_i8x16, i8x16, - [i8::min_value(), i8::max_value(), 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [i8::MIN, i8::MAX, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [-1, 1, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [i8::min_value(), i8::max_value(), 2, -2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + [i8::MIN, i8::MAX, 2, -2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } test_vec_adds! { test_vec_adds_u8x16, u8x16, - [u8::max_value(), 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [u8::MAX, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [u8::max_value(), 1, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + [u8::MAX, 1, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } - test_vec_2! { test_vec_addc, vec_addc, u32x4, [u32::max_value(), 0, 0, 0], [1, 1, 1, 1], [1, 0, 0, 0] } + test_vec_2! { test_vec_addc, vec_addc, u32x4, [u32::MAX, 0, 0, 0], [1, 1, 1, 1], [1, 0, 0, 0] } macro_rules! test_vec_abs { { $name: ident, $ty: ident, $a: expr, $d: expr } => { @@ -2481,10 +2483,9 @@ mod tests { 7 * 256, )); let b: vector_signed_short = transmute(i16x8::new(256, 256, 256, 256, 256, 256, 256, 256)); - let c: vector_signed_short = - transmute(i16x8::new(0, 1, 2, 3, 4, 5, 6, i16::max_value() - 1)); + let c: vector_signed_short = transmute(i16x8::new(0, 1, 2, 3, 4, 5, 6, i16::MAX - 1)); - let d = i16x8::new(0, 3, 6, 9, 12, 15, 18, i16::max_value()); + let d = i16x8::new(0, 3, 6, 9, 12, 15, 18, i16::MAX); assert_eq!(d, transmute(vec_mradds(a, b, c))); } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/powerpc/vsx.rs b/crux-mir/lib/stdarch/crates/core_arch/src/powerpc/vsx.rs index 394c9a704..3141bc8bc 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/powerpc/vsx.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/powerpc/vsx.rs @@ -21,7 +21,7 @@ types! { pub struct vector_signed_long(i64, i64); /// PowerPC-specific 128-bit wide vector of two packed `u64` pub struct vector_unsigned_long(u64, u64); - /// PowerPC-specific 128-bit wide vector mask of two elements + /// PowerPC-specific 128-bit wide vector mask of two `i64` pub struct vector_bool_long(i64, i64); /// PowerPC-specific 128-bit wide vector of two packed `f64` pub struct vector_double(f64, f64); @@ -47,10 +47,10 @@ mod sealed { #[cfg_attr(all(test, target_endian = "big"), assert_instr(xxspltd, dm = 0x0))] unsafe fn xxpermdi(a: i64x2, b: i64x2, dm: u8) -> i64x2 { match dm & 0b11 { - 0 => simd_shuffle2(a, b, [0b00, 0b10]), - 1 => simd_shuffle2(a, b, [0b01, 0b10]), - 2 => simd_shuffle2(a, b, [0b00, 0b11]), - _ => simd_shuffle2(a, b, [0b01, 0b11]), + 0 => simd_shuffle2!(a, b, [0b00, 0b10]), + 1 => simd_shuffle2!(a, b, [0b01, 0b10]), + 2 => simd_shuffle2!(a, b, [0b00, 0b11]), + _ => simd_shuffle2!(a, b, [0b01, 0b11]), } } @@ -75,12 +75,13 @@ mod sealed { /// Vector permute. #[inline] #[target_feature(enable = "vsx")] -#[rustc_args_required_const(2)] -pub unsafe fn vec_xxpermdi(a: T, b: T, dm: u8) -> T +//#[rustc_legacy_const_generics(2)] +pub unsafe fn vec_xxpermdi(a: T, b: T) -> T where T: sealed::VectorPermDI, { - a.vec_xxpermdi(b, dm) + static_assert_imm2!(DM); + a.vec_xxpermdi(b, DM as u8) } #[cfg(test)] @@ -102,10 +103,10 @@ mod tests { let a: $longtype = mem::transmute($shorttype::new($($a),+, $($b),+)); let b = mem::transmute($shorttype::new($($c),+, $($d),+)); - assert_eq!($shorttype::new($($a),+, $($c),+), mem::transmute(vec_xxpermdi(a, b, 0))); - assert_eq!($shorttype::new($($b),+, $($c),+), mem::transmute(vec_xxpermdi(a, b, 1))); - assert_eq!($shorttype::new($($a),+, $($d),+), mem::transmute(vec_xxpermdi(a, b, 2))); - assert_eq!($shorttype::new($($b),+, $($d),+), mem::transmute(vec_xxpermdi(a, b, 3))); + assert_eq!($shorttype::new($($a),+, $($c),+), mem::transmute(vec_xxpermdi::<_, 0>(a, b))); + assert_eq!($shorttype::new($($b),+, $($c),+), mem::transmute(vec_xxpermdi::<_, 1>(a, b))); + assert_eq!($shorttype::new($($a),+, $($d),+), mem::transmute(vec_xxpermdi::<_, 2>(a, b))); + assert_eq!($shorttype::new($($b),+, $($d),+), mem::transmute(vec_xxpermdi::<_, 3>(a, b))); } } } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/riscv64/mod.rs b/crux-mir/lib/stdarch/crates/core_arch/src/riscv64/mod.rs new file mode 100644 index 000000000..751b9a860 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/riscv64/mod.rs @@ -0,0 +1,49 @@ +//! RISC-V RV64 specific intrinsics +use crate::arch::asm; + +/// Loads virtual machine memory by unsigned word integer +/// +/// This instruction performs an explicit memory access as though `V=1`; +/// i.e., with the address translation and protection, and the endianness, that apply to memory +/// accesses in either VS-mode or VU-mode. +/// +/// This operation is not available under RV32 base instruction set. +/// +/// This function is unsafe for it accesses the virtual supervisor or user via a `HLV.WU` +/// instruction which is effectively a dereference to any memory address. +#[inline] +pub unsafe fn hlv_wu(src: *const u32) -> u32 { + let value: u32; + asm!(".insn i 0x73, 0x4, {}, {}, 0x681", out(reg) value, in(reg) src, options(readonly, nostack)); + value +} + +/// Loads virtual machine memory by double integer +/// +/// This instruction performs an explicit memory access as though `V=1`; +/// i.e., with the address translation and protection, and the endianness, that apply to memory +/// accesses in either VS-mode or VU-mode. +/// +/// This operation is not available under RV32 base instruction set. +/// +/// This function is unsafe for it accesses the virtual supervisor or user via a `HLV.D` +/// instruction which is effectively a dereference to any memory address. +#[inline] +pub unsafe fn hlv_d(src: *const i64) -> i64 { + let value: i64; + asm!(".insn i 0x73, 0x4, {}, {}, 0x6C0", out(reg) value, in(reg) src, options(readonly, nostack)); + value +} + +/// Stores virtual machine memory by double integer +/// +/// This instruction performs an explicit memory access as though `V=1`; +/// i.e., with the address translation and protection, and the endianness, that apply to memory +/// accesses in either VS-mode or VU-mode. +/// +/// This function is unsafe for it accesses the virtual supervisor or user via a `HSV.D` +/// instruction which is effectively a dereference to any memory address. +#[inline] +pub unsafe fn hsv_d(dst: *mut i64, src: i64) { + asm!(".insn r 0x73, 0x4, 0x37, x0, {}, {}", in(reg) dst, in(reg) src, options(nostack)); +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/riscv_shared/mod.rs b/crux-mir/lib/stdarch/crates/core_arch/src/riscv_shared/mod.rs new file mode 100644 index 000000000..0e35fe1f1 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/riscv_shared/mod.rs @@ -0,0 +1,760 @@ +//! Shared RISC-V intrinsics +mod p; + +pub use p::*; + +use crate::arch::asm; + +/// Generates the `PAUSE` instruction +/// +/// The PAUSE instruction is a HINT that indicates the current hart's rate of instruction retirement +/// should be temporarily reduced or paused. The duration of its effect must be bounded and may be zero. +#[inline] +pub fn pause() { + unsafe { asm!(".insn i 0x0F, 0, x0, x0, 0x010", options(nomem, nostack)) } +} + +/// Generates the `NOP` instruction +/// +/// The NOP instruction does not change any architecturally visible state, except for +/// advancing the `pc` and incrementing any applicable performance counters. +#[inline] +pub fn nop() { + unsafe { asm!("nop", options(nomem, nostack)) } +} + +/// Generates the `WFI` instruction +/// +/// The WFI instruction provides a hint to the implementation that the current hart can be stalled +/// until an interrupt might need servicing. This instruction is a hint, +/// and a legal implementation is to simply implement WFI as a NOP. +#[inline] +pub unsafe fn wfi() { + asm!("wfi", options(nomem, nostack)) +} + +/// Generates the `FENCE.I` instruction +/// +/// A FENCE.I instruction ensures that a subsequent instruction fetch on a RISC-V hart will see +/// any previous data stores already visible to the same RISC-V hart. +/// +/// FENCE.I does not ensure that other RISC-V harts' instruction fetches will observe the +/// local hart's stores in a multiprocessor system. +#[inline] +pub unsafe fn fence_i() { + asm!("fence.i", options(nostack)) +} + +/// Supervisor memory management fence for given virtual address and address space +/// +/// The fence orders only reads and writes made to leaf page table entries corresponding to +/// the virtual address in parameter `vaddr`, for the address space identified by integer parameter +/// `asid`. Accesses to global mappings are not ordered. The fence also invalidates all +/// address-translation cache entries that contain leaf page table entries corresponding to the +/// virtual address in parameter `vaddr` and that match the address space identified by integer +/// parameter `asid`, except for entries containing global mappings. +#[inline] +pub unsafe fn sfence_vma(vaddr: usize, asid: usize) { + asm!("sfence.vma {}, {}", in(reg) vaddr, in(reg) asid, options(nostack)) +} + +/// Supervisor memory management fence for given virtual address +/// +/// The fence orders only reads and writes made to leaf page table entries corresponding to +/// the virtual address in parameter `vaddr`, for all address spaces. +/// The fence also invalidates all address-translation cache entries that contain leaf page +/// table entries corresponding to the virtual address in parameter `vaddr`, for all address spaces. +#[inline] +pub unsafe fn sfence_vma_vaddr(vaddr: usize) { + asm!("sfence.vma {}, x0", in(reg) vaddr, options(nostack)) +} + +/// Supervisor memory management fence for given address space +/// +/// The fence orders all reads and writes made to any level of the page tables, +/// but only for the address space identified by integer parameter `asid`. +/// +/// Accesses to global mappings are not ordered. The fence also invalidates all +/// address-translation cache entries matching the address space identified by integer +/// parameter `asid`, except for entries containing global mappings. +#[inline] +pub unsafe fn sfence_vma_asid(asid: usize) { + asm!("sfence.vma x0, {}", in(reg) asid, options(nostack)) +} + +/// Supervisor memory management fence for all address spaces and virtual addresses +/// +/// The fence orders all reads and writes made to any level of the page +/// tables, for all address spaces. The fence also invalidates all address-translation cache entries, +/// for all address spaces. +#[inline] +pub unsafe fn sfence_vma_all() { + asm!("sfence.vma", options(nostack)) +} + +/// Invalidate supervisor translation cache for given virtual address and address space +/// +/// This instruction invalidates any address-translation cache entries that an +/// `SFENCE.VMA` instruction with the same values of `vaddr` and `asid` would invalidate. +#[inline] +pub unsafe fn sinval_vma(vaddr: usize, asid: usize) { + // asm!("sinval.vma {}, {}", in(reg) vaddr, in(reg) asid, options(nostack)) + asm!(".insn r 0x73, 0, 0x0B, x0, {}, {}", in(reg) vaddr, in(reg) asid, options(nostack)) +} + +/// Invalidate supervisor translation cache for given virtual address +/// +/// This instruction invalidates any address-translation cache entries that an +/// `SFENCE.VMA` instruction with the same values of `vaddr` and `asid` would invalidate. +#[inline] +pub unsafe fn sinval_vma_vaddr(vaddr: usize) { + asm!(".insn r 0x73, 0, 0x0B, x0, {}, x0", in(reg) vaddr, options(nostack)) +} + +/// Invalidate supervisor translation cache for given address space +/// +/// This instruction invalidates any address-translation cache entries that an +/// `SFENCE.VMA` instruction with the same values of `vaddr` and `asid` would invalidate. +#[inline] +pub unsafe fn sinval_vma_asid(asid: usize) { + asm!(".insn r 0x73, 0, 0x0B, x0, x0, {}", in(reg) asid, options(nostack)) +} + +/// Invalidate supervisor translation cache for all address spaces and virtual addresses +/// +/// This instruction invalidates any address-translation cache entries that an +/// `SFENCE.VMA` instruction with the same values of `vaddr` and `asid` would invalidate. +#[inline] +pub unsafe fn sinval_vma_all() { + asm!(".insn r 0x73, 0, 0x0B, x0, x0, x0", options(nostack)) +} + +/// Generates the `SFENCE.W.INVAL` instruction +/// +/// This instruction guarantees that any previous stores already visible to the current RISC-V hart +/// are ordered before subsequent `SINVAL.VMA` instructions executed by the same hart. +#[inline] +pub unsafe fn sfence_w_inval() { + // asm!("sfence.w.inval", options(nostack)) + asm!(".insn i 0x73, 0, x0, x0, 0x180", options(nostack)) +} + +/// Generates the `SFENCE.INVAL.IR` instruction +/// +/// This instruction guarantees that any previous SINVAL.VMA instructions executed by the current hart +/// are ordered before subsequent implicit references by that hart to the memory-management data structures. +#[inline] +pub unsafe fn sfence_inval_ir() { + // asm!("sfence.inval.ir", options(nostack)) + asm!(".insn i 0x73, 0, x0, x0, 0x181", options(nostack)) +} + +/// Loads virtual machine memory by signed byte integer +/// +/// This instruction performs an explicit memory access as though `V=1`; +/// i.e., with the address translation and protection, and the endianness, that apply to memory +/// accesses in either VS-mode or VU-mode. +/// +/// This function is unsafe for it accesses the virtual supervisor or user via a `HLV.B` +/// instruction which is effectively a dereference to any memory address. +#[inline] +pub unsafe fn hlv_b(src: *const i8) -> i8 { + let value: i8; + asm!(".insn i 0x73, 0x4, {}, {}, 0x600", out(reg) value, in(reg) src, options(readonly, nostack)); + value +} + +/// Loads virtual machine memory by unsigned byte integer +/// +/// This instruction performs an explicit memory access as though `V=1`; +/// i.e., with the address translation and protection, and the endianness, that apply to memory +/// accesses in either VS-mode or VU-mode. +/// +/// This function is unsafe for it accesses the virtual supervisor or user via a `HLV.BU` +/// instruction which is effectively a dereference to any memory address. +#[inline] +pub unsafe fn hlv_bu(src: *const u8) -> u8 { + let value: u8; + asm!(".insn i 0x73, 0x4, {}, {}, 0x601", out(reg) value, in(reg) src, options(readonly, nostack)); + value +} + +/// Loads virtual machine memory by signed half integer +/// +/// This instruction performs an explicit memory access as though `V=1`; +/// i.e., with the address translation and protection, and the endianness, that apply to memory +/// accesses in either VS-mode or VU-mode. +/// +/// This function is unsafe for it accesses the virtual supervisor or user via a `HLV.H` +/// instruction which is effectively a dereference to any memory address. +#[inline] +pub unsafe fn hlv_h(src: *const i16) -> i16 { + let value: i16; + asm!(".insn i 0x73, 0x4, {}, {}, 0x640", out(reg) value, in(reg) src, options(readonly, nostack)); + value +} + +/// Loads virtual machine memory by unsigned half integer +/// +/// This instruction performs an explicit memory access as though `V=1`; +/// i.e., with the address translation and protection, and the endianness, that apply to memory +/// accesses in either VS-mode or VU-mode. +/// +/// This function is unsafe for it accesses the virtual supervisor or user via a `HLV.HU` +/// instruction which is effectively a dereference to any memory address. +#[inline] +pub unsafe fn hlv_hu(src: *const u16) -> u16 { + let value: u16; + asm!(".insn i 0x73, 0x4, {}, {}, 0x641", out(reg) value, in(reg) src, options(readonly, nostack)); + value +} + +/// Accesses virtual machine instruction by unsigned half integer +/// +/// This instruction performs an explicit memory access as though `V=1`; +/// the memory being read must be executable in both stages of address translation, +/// but read permission is not required. +/// +/// This function is unsafe for it accesses the virtual supervisor or user via a `HLVX.HU` +/// instruction which is effectively a dereference to any memory address. +#[inline] +pub unsafe fn hlvx_hu(src: *const u16) -> u16 { + let insn: u16; + asm!(".insn i 0x73, 0x4, {}, {}, 0x643", out(reg) insn, in(reg) src, options(readonly, nostack)); + insn +} + +/// Loads virtual machine memory by signed word integer +/// +/// This instruction performs an explicit memory access as though `V=1`; +/// i.e., with the address translation and protection, and the endianness, that apply to memory +/// accesses in either VS-mode or VU-mode. +/// +/// This function is unsafe for it accesses the virtual supervisor or user via a `HLV.W` +/// instruction which is effectively a dereference to any memory address. +#[inline] +pub unsafe fn hlv_w(src: *const i32) -> i32 { + let value: i32; + asm!(".insn i 0x73, 0x4, {}, {}, 0x680", out(reg) value, in(reg) src, options(readonly, nostack)); + value +} + +/// Accesses virtual machine instruction by unsigned word integer +/// +/// This instruction performs an explicit memory access as though `V=1`; +/// the memory being read must be executable in both stages of address translation, +/// but read permission is not required. +/// +/// This function is unsafe for it accesses the virtual supervisor or user via a `HLVX.WU` +/// instruction which is effectively a dereference to any memory address. +#[inline] +pub unsafe fn hlvx_wu(src: *const u32) -> u32 { + let insn: u32; + asm!(".insn i 0x73, 0x4, {}, {}, 0x683", out(reg) insn, in(reg) src, options(readonly, nostack)); + insn +} + +/// Stores virtual machine memory by byte integer +/// +/// This instruction performs an explicit memory access as though `V=1`; +/// i.e., with the address translation and protection, and the endianness, that apply to memory +/// accesses in either VS-mode or VU-mode. +/// +/// This function is unsafe for it accesses the virtual supervisor or user via a `HSV.B` +/// instruction which is effectively a dereference to any memory address. +#[inline] +pub unsafe fn hsv_b(dst: *mut i8, src: i8) { + asm!(".insn r 0x73, 0x4, 0x31, x0, {}, {}", in(reg) dst, in(reg) src, options(nostack)); +} + +/// Stores virtual machine memory by half integer +/// +/// This instruction performs an explicit memory access as though `V=1`; +/// i.e., with the address translation and protection, and the endianness, that apply to memory +/// accesses in either VS-mode or VU-mode. +/// +/// This function is unsafe for it accesses the virtual supervisor or user via a `HSV.H` +/// instruction which is effectively a dereference to any memory address. +#[inline] +pub unsafe fn hsv_h(dst: *mut i16, src: i16) { + asm!(".insn r 0x73, 0x4, 0x33, x0, {}, {}", in(reg) dst, in(reg) src, options(nostack)); +} + +/// Stores virtual machine memory by word integer +/// +/// This instruction performs an explicit memory access as though `V=1`; +/// i.e., with the address translation and protection, and the endianness, that apply to memory +/// accesses in either VS-mode or VU-mode. +/// +/// This function is unsafe for it accesses the virtual supervisor or user via a `HSV.W` +/// instruction which is effectively a dereference to any memory address. +#[inline] +pub unsafe fn hsv_w(dst: *mut i32, src: i32) { + asm!(".insn r 0x73, 0x4, 0x35, x0, {}, {}", in(reg) dst, in(reg) src, options(nostack)); +} + +/// Hypervisor memory management fence for given guest virtual address and guest address space +/// +/// Guarantees that any previous stores already visible to the current hart are ordered before all +/// implicit reads by that hart done for VS-stage address translation for instructions that: +/// - are subsequent to the `HFENCE.VVMA`, and +/// - execute when `hgatp.VMID` has the same setting as it did when `HFENCE.VVMA` executed. +/// +/// This fence specifies a single guest virtual address, and a single guest address-space identifier. +#[inline] +pub unsafe fn hfence_vvma(vaddr: usize, asid: usize) { + // asm!("hfence.vvma {}, {}", in(reg) vaddr, in(reg) asid) + asm!(".insn r 0x73, 0, 0x11, x0, {}, {}", in(reg) vaddr, in(reg) asid, options(nostack)) +} + +/// Hypervisor memory management fence for given guest virtual address +/// +/// Guarantees that any previous stores already visible to the current hart are ordered before all +/// implicit reads by that hart done for VS-stage address translation for instructions that: +/// - are subsequent to the `HFENCE.VVMA`, and +/// - execute when `hgatp.VMID` has the same setting as it did when `HFENCE.VVMA` executed. +/// +/// This fence specifies a single guest virtual address. +#[inline] +pub unsafe fn hfence_vvma_vaddr(vaddr: usize) { + asm!(".insn r 0x73, 0, 0x11, x0, {}, x0", in(reg) vaddr, options(nostack)) +} + +/// Hypervisor memory management fence for given guest address space +/// +/// Guarantees that any previous stores already visible to the current hart are ordered before all +/// implicit reads by that hart done for VS-stage address translation for instructions that: +/// - are subsequent to the `HFENCE.VVMA`, and +/// - execute when `hgatp.VMID` has the same setting as it did when `HFENCE.VVMA` executed. +/// +/// This fence specifies a single guest address-space identifier. +#[inline] +pub unsafe fn hfence_vvma_asid(asid: usize) { + asm!(".insn r 0x73, 0, 0x11, x0, x0, {}", in(reg) asid, options(nostack)) +} + +/// Hypervisor memory management fence for all guest address spaces and guest virtual addresses +/// +/// Guarantees that any previous stores already visible to the current hart are ordered before all +/// implicit reads by that hart done for VS-stage address translation for instructions that: +/// - are subsequent to the `HFENCE.VVMA`, and +/// - execute when `hgatp.VMID` has the same setting as it did when `HFENCE.VVMA` executed. +/// +/// This fence applies to any guest address spaces and guest virtual addresses. +#[inline] +pub unsafe fn hfence_vvma_all() { + asm!(".insn r 0x73, 0, 0x11, x0, x0, x0", options(nostack)) +} + +/// Hypervisor memory management fence for guest physical address and virtual machine +/// +/// Guarantees that any previous stores already visible to the current hart are ordered before all implicit reads +/// by that hart done for G-stage address translation for instructions that follow the HFENCE.GVMA. +/// +/// This fence specifies a single guest physical address, **shifted right by 2 bits**, and a single virtual machine +/// by virtual machine identifier (VMID). +#[inline] +pub unsafe fn hfence_gvma(gaddr: usize, vmid: usize) { + // asm!("hfence.gvma {}, {}", in(reg) gaddr, in(reg) vmid, options(nostack)) + asm!(".insn r 0x73, 0, 0x31, x0, {}, {}", in(reg) gaddr, in(reg) vmid, options(nostack)) +} + +/// Hypervisor memory management fence for guest physical address +/// +/// Guarantees that any previous stores already visible to the current hart are ordered before all implicit reads +/// by that hart done for G-stage address translation for instructions that follow the HFENCE.GVMA. +/// +/// This fence specifies a single guest physical address; **the physical address should be shifted right by 2 bits**. +#[inline] +pub unsafe fn hfence_gvma_gaddr(gaddr: usize) { + asm!(".insn r 0x73, 0, 0x31, x0, {}, x0", in(reg) gaddr, options(nostack)) +} + +/// Hypervisor memory management fence for given virtual machine +/// +/// Guarantees that any previous stores already visible to the current hart are ordered before all implicit reads +/// by that hart done for G-stage address translation for instructions that follow the HFENCE.GVMA. +/// +/// This fence specifies a single virtual machine by virtual machine identifier (VMID). +#[inline] +pub unsafe fn hfence_gvma_vmid(vmid: usize) { + asm!(".insn r 0x73, 0, 0x31, x0, x0, {}", in(reg) vmid, options(nostack)) +} + +/// Hypervisor memory management fence for all virtual machines and guest physical addresses +/// +/// Guarantees that any previous stores already visible to the current hart are ordered before all implicit reads +/// by that hart done for G-stage address translation for instructions that follow the HFENCE.GVMA. +/// +/// This fence specifies all guest physical addresses and all virtual machines. +#[inline] +pub unsafe fn hfence_gvma_all() { + asm!(".insn r 0x73, 0, 0x31, x0, x0, x0", options(nostack)) +} + +/// Invalidate hypervisor translation cache for given guest virtual address and guest address space +/// +/// This instruction invalidates any address-translation cache entries that an +/// `HFENCE.VVMA` instruction with the same values of `vaddr` and `asid` would invalidate. +/// +/// This fence specifies a single guest virtual address, and a single guest address-space identifier. +#[inline] +pub unsafe fn hinval_vvma(vaddr: usize, asid: usize) { + // asm!("hinval.vvma {}, {}", in(reg) vaddr, in(reg) asid, options(nostack)) + asm!(".insn r 0x73, 0, 0x13, x0, {}, {}", in(reg) vaddr, in(reg) asid, options(nostack)) +} + +/// Invalidate hypervisor translation cache for given guest virtual address +/// +/// This instruction invalidates any address-translation cache entries that an +/// `HFENCE.VVMA` instruction with the same values of `vaddr` and `asid` would invalidate. +/// +/// This fence specifies a single guest virtual address. +#[inline] +pub unsafe fn hinval_vvma_vaddr(vaddr: usize) { + asm!(".insn r 0x73, 0, 0x13, x0, {}, x0", in(reg) vaddr, options(nostack)) +} + +/// Invalidate hypervisor translation cache for given guest address space +/// +/// This instruction invalidates any address-translation cache entries that an +/// `HFENCE.VVMA` instruction with the same values of `vaddr` and `asid` would invalidate. +/// +/// This fence specifies a single guest address-space identifier. +#[inline] +pub unsafe fn hinval_vvma_asid(asid: usize) { + asm!(".insn r 0x73, 0, 0x13, x0, x0, {}", in(reg) asid, options(nostack)) +} + +/// Invalidate hypervisor translation cache for all guest address spaces and guest virtual addresses +/// +/// This instruction invalidates any address-translation cache entries that an +/// `HFENCE.VVMA` instruction with the same values of `vaddr` and `asid` would invalidate. +/// +/// This fence applies to any guest address spaces and guest virtual addresses. +#[inline] +pub unsafe fn hinval_vvma_all() { + asm!(".insn r 0x73, 0, 0x13, x0, x0, x0", options(nostack)) +} + +/// Invalidate hypervisor translation cache for guest physical address and virtual machine +/// +/// This instruction invalidates any address-translation cache entries that an +/// `HFENCE.GVMA` instruction with the same values of `gaddr` and `vmid` would invalidate. +/// +/// This fence specifies a single guest physical address, **shifted right by 2 bits**, and a single virtual machine +/// by virtual machine identifier (VMID). +#[inline] +pub unsafe fn hinval_gvma(gaddr: usize, vmid: usize) { + // asm!("hinval.gvma {}, {}", in(reg) gaddr, in(reg) vmid, options(nostack)) + asm!(".insn r 0x73, 0, 0x33, x0, {}, {}", in(reg) gaddr, in(reg) vmid, options(nostack)) +} + +/// Invalidate hypervisor translation cache for guest physical address +/// +/// This instruction invalidates any address-translation cache entries that an +/// `HFENCE.GVMA` instruction with the same values of `gaddr` and `vmid` would invalidate. +/// +/// This fence specifies a single guest physical address; **the physical address should be shifted right by 2 bits**. +#[inline] +pub unsafe fn hinval_gvma_gaddr(gaddr: usize) { + asm!(".insn r 0x73, 0, 0x33, x0, {}, x0", in(reg) gaddr, options(nostack)) +} + +/// Invalidate hypervisor translation cache for given virtual machine +/// +/// This instruction invalidates any address-translation cache entries that an +/// `HFENCE.GVMA` instruction with the same values of `gaddr` and `vmid` would invalidate. +/// +/// This fence specifies a single virtual machine by virtual machine identifier (VMID). +#[inline] +pub unsafe fn hinval_gvma_vmid(vmid: usize) { + asm!(".insn r 0x73, 0, 0x33, x0, x0, {}", in(reg) vmid, options(nostack)) +} + +/// Invalidate hypervisor translation cache for all virtual machines and guest physical addresses +/// +/// This instruction invalidates any address-translation cache entries that an +/// `HFENCE.GVMA` instruction with the same values of `gaddr` and `vmid` would invalidate. +/// +/// This fence specifies all guest physical addresses and all virtual machines. +#[inline] +pub unsafe fn hinval_gvma_all() { + asm!(".insn r 0x73, 0, 0x33, x0, x0, x0", options(nostack)) +} + +/// Reads the floating-point control and status register `fcsr` +/// +/// Register `fcsr` is a 32-bit read/write register that selects the dynamic rounding mode +/// for floating-point arithmetic operations and holds the accrued exception flag. +/// +/// Accoding to "F" Standard Extension for Single-Precision Floating-Point, Version 2.2, +/// register `fcsr` is defined as: +/// +/// | Bit index | Meaning | +/// |:----------|:--------| +/// | 0..=4 | Accrued Exceptions (`fflags`) | +/// | 5..=7 | Rounding Mode (`frm`) | +/// | 8..=31 | _Reserved_ | +/// +/// For definition of each field, visit [`frrm`] and [`frflags`]. +/// +/// [`frrm`]: fn.frrm.html +/// [`frflags`]: fn.frflags.html +#[inline] +pub fn frcsr() -> u32 { + let value: u32; + unsafe { asm!("frcsr {}", out(reg) value, options(nomem, nostack)) }; + value +} + +/// Swaps the floating-point control and status register `fcsr` +/// +/// This function swaps the value in `fcsr` by copying the original value to be returned, +/// and then writing a new value obtained from input variable `value` into `fcsr`. +#[inline] +pub fn fscsr(value: u32) -> u32 { + let original: u32; + unsafe { asm!("fscsr {}, {}", out(reg) original, in(reg) value, options(nomem, nostack)) } + original +} + +/// Reads the floating-point rounding mode register `frm` +/// +/// Accoding to "F" Standard Extension for Single-Precision Floating-Point, Version 2.2, +/// the rounding mode field is defined as listed in the table below: +/// +/// | Rounding Mode | Mnemonic | Meaning | +/// |:-------------|:----------|:---------| +/// | 000 | RNE | Round to Nearest, ties to Even | +/// | 001 | RTZ | Round towards Zero | +/// | 010 | RDN | Round Down (towards −∞) | +/// | 011 | RUP | Round Up (towards +∞) | +/// | 100 | RMM | Round to Nearest, ties to Max Magnitude | +/// | 101 | | _Reserved for future use._ | +/// | 110 | | _Reserved for future use._ | +/// | 111 | DYN | In Rounding Mode register, _reserved_. | +#[inline] +pub fn frrm() -> u32 { + let value: u32; + unsafe { asm!("frrm {}", out(reg) value, options(nomem, nostack)) }; + value +} + +/// Swaps the floating-point rounding mode register `frm` +/// +/// This function swaps the value in `frm` by copying the original value to be returned, +/// and then writing a new value obtained from the three least-significant bits of +/// input variable `value` into `frm`. +#[inline] +pub fn fsrm(value: u32) -> u32 { + let original: u32; + unsafe { asm!("fsrm {}, {}", out(reg) original, in(reg) value, options(nomem, nostack)) } + original +} + +/// Reads the floating-point accrued exception flags register `fflags` +/// +/// The accrued exception flags indicate the exception conditions that have arisen +/// on any floating-point arithmetic instruction since the field was last reset by software. +/// +/// Accoding to "F" Standard Extension for Single-Precision Floating-Point, Version 2.2, +/// the accured exception flags is defined as a bit vector of 5 bits. +/// The meaning of each binary bit is listed in the table below. +/// +/// | Bit index | Mnemonic | Meaning | +/// |:--|:---|:-----------------| +/// | 4 | NV | Invalid Operation | +/// | 3 | DZ | Divide by Zero | +/// | 2 | OF | Overflow | +/// | 1 | UF | Underflow | +/// | 0 | NX | Inexact | +#[inline] +pub fn frflags() -> u32 { + let value: u32; + unsafe { asm!("frflags {}", out(reg) value, options(nomem, nostack)) }; + value +} + +/// Swaps the floating-point accrued exception flags register `fflags` +/// +/// This function swaps the value in `fflags` by copying the original value to be returned, +/// and then writing a new value obtained from the five least-significant bits of +/// input variable `value` into `fflags`. +#[inline] +pub fn fsflags(value: u32) -> u32 { + let original: u32; + unsafe { asm!("fsflags {}, {}", out(reg) original, in(reg) value, options(nomem, nostack)) } + original +} + +/// `P0` transformation function as is used in the SM3 hash algorithm +/// +/// This function is included in `Zksh` extension. It's defined as: +/// +/// ```text +/// P0(X) = X ⊕ (X ≪ 9) ⊕ (X ≪ 17) +/// ``` +/// +/// where `⊕` represents 32-bit xor, and `≪ k` represents rotate left by `k` bits. +/// +/// In the SM3 algorithm, the `P0` transformation is used as `E ← P0(TT2)` when the +/// compression function `CF` uses the intermediate value `TT2` to calculate +/// the variable `E` in one iteration for subsequent processes. +/// +/// According to RISC-V Cryptography Extensions, Volume I, the execution latency of +/// this instruction must always be independent from the data it operates on. +#[inline] +#[target_feature(enable = "zksh")] +pub fn sm3p0(x: u32) -> u32 { + let ans: u32; + unsafe { asm!("sm3p0 {}, {}", lateout(reg) ans, in(reg) x, options(pure, nomem, nostack)) }; + ans +} + +/// `P1` transformation function as is used in the SM3 hash algorithm +/// +/// This function is included in `Zksh` extension. It's defined as: +/// +/// ```text +/// P1(X) = X ⊕ (X ≪ 15) ⊕ (X ≪ 23) +/// ``` +/// +/// where `⊕` represents 32-bit xor, and `≪ k` represents rotate left by `k` bits. +/// +/// In the SM3 algorithm, the `P1` transformation is used to expand message, +/// where expanded word `Wj` can be generated from the previous words. +/// The whole process can be described as the following pseudocode: +/// +/// ```text +/// FOR j=16 TO 67 +/// Wj ← P1(Wj−16 ⊕ Wj−9 ⊕ (Wj−3 ≪ 15)) ⊕ (Wj−13 ≪ 7) ⊕ Wj−6 +/// ENDFOR +/// ``` +/// +/// According to RISC-V Cryptography Extensions, Volume I, the execution latency of +/// this instruction must always be independent from the data it operates on. +#[inline] +#[target_feature(enable = "zksh")] +pub fn sm3p1(x: u32) -> u32 { + let ans: u32; + unsafe { asm!("sm3p1 {}, {}", lateout(reg) ans, in(reg) x, options(pure, nomem, nostack)) }; + ans +} + +/// Accelerates the round function `F` in the SM4 block cipher algorithm +/// +/// This instruction is included in extension `Zksed`. It's defined as: +/// +/// ```text +/// SM4ED(x, a, BS) = x ⊕ T(ai) +/// ... where +/// ai = a.bytes[BS] +/// T(ai) = L(τ(ai)) +/// bi = τ(ai) = SM4-S-Box(ai) +/// ci = L(bi) = bi ⊕ (bi ≪ 2) ⊕ (bi ≪ 10) ⊕ (bi ≪ 18) ⊕ (bi ≪ 24) +/// SM4ED = (ci ≪ (BS * 8)) ⊕ x +/// ``` +/// +/// where `⊕` represents 32-bit xor, and `≪ k` represents rotate left by `k` bits. +/// As is defined above, `T` is a combined transformation of non linear S-Box transform `τ` +/// and linear layer transform `L`. +/// +/// In the SM4 algorithm, the round function `F` is defined as: +/// +/// ```text +/// F(x0, x1, x2, x3, rk) = x0 ⊕ T(x1 ⊕ x2 ⊕ x3 ⊕ rk) +/// ... where +/// T(A) = L(τ(A)) +/// B = τ(A) = (SM4-S-Box(a0), SM4-S-Box(a1), SM4-S-Box(a2), SM4-S-Box(a3)) +/// C = L(B) = B ⊕ (B ≪ 2) ⊕ (B ≪ 10) ⊕ (B ≪ 18) ⊕ (B ≪ 24) +/// ``` +/// +/// It can be implemented by `sm4ed` instruction like: +/// +/// ```no_run +/// # #[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] +/// # fn round_function(x0: u32, x1: u32, x2: u32, x3: u32, rk: u32) -> u32 { +/// # #[cfg(target_arch = "riscv32")] use core::arch::riscv32::sm4ed; +/// # #[cfg(target_arch = "riscv64")] use core::arch::riscv64::sm4ed; +/// let a = x1 ^ x2 ^ x3 ^ rk; +/// let c0 = sm4ed::<0>(x0, a); +/// let c1 = sm4ed::<1>(c0, a); // c1 represents c[0..=1], etc. +/// let c2 = sm4ed::<2>(c1, a); +/// let c3 = sm4ed::<3>(c2, a); +/// return c3; // c3 represents c[0..=3] +/// # } +/// ``` +/// +/// According to RISC-V Cryptography Extensions, Volume I, the execution latency of +/// this instruction must always be independent from the data it operates on. +#[inline] +#[target_feature(enable = "zksed")] +pub fn sm4ed(x: u32, a: u32) -> u32 { + static_assert!(BS: u8 where BS <= 3); + let ans: u32; + unsafe { + asm!("sm4ed {}, {}, {}, {}", lateout(reg) ans, in(reg) x, in(reg) a, const BS, options(pure, nomem, nostack)) + }; + ans +} + +/// Accelerates the key schedule operation in the SM4 block cipher algorithm +/// +/// This instruction is included in extension `Zksed`. It's defined as: +/// +/// ```text +/// SM4KS(x, k, BS) = x ⊕ T'(ki) +/// ... where +/// ki = k.bytes[BS] +/// T'(ki) = L'(τ(ki)) +/// bi = τ(ki) = SM4-S-Box(ki) +/// ci = L'(bi) = bi ⊕ (bi ≪ 13) ⊕ (bi ≪ 23) +/// SM4KS = (ci ≪ (BS * 8)) ⊕ x +/// ``` +/// +/// where `⊕` represents 32-bit xor, and `≪ k` represents rotate left by `k` bits. +/// As is defined above, `T'` is a combined transformation of non linear S-Box transform `τ` +/// and the replaced linear layer transform `L'`. +/// +/// In the SM4 algorithm, the key schedule is defined as: +/// +/// ```text +/// rk[i] = K[i+4] = K[i] ⊕ T'(K[i+1] ⊕ K[i+2] ⊕ K[i+3] ⊕ CK[i]) +/// ... where +/// K[0..=3] = MK[0..=3] ⊕ FK[0..=3] +/// T'(K) = L'(τ(K)) +/// B = τ(K) = (SM4-S-Box(k0), SM4-S-Box(k1), SM4-S-Box(k2), SM4-S-Box(k3)) +/// C = L'(B) = B ⊕ (B ≪ 13) ⊕ (B ≪ 23) +/// ``` +/// +/// where `MK` represents the input 128-bit encryption key, +/// constants `FK` and `CK` are fixed system configuration constant values defined by the SM4 algorithm. +/// Hence, the key schedule operation can be implemented by `sm4ks` instruction like: +/// +/// ```no_run +/// # #[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] +/// # fn key_schedule(k0: u32, k1: u32, k2: u32, k3: u32, ck_i: u32) -> u32 { +/// # #[cfg(target_arch = "riscv32")] use core::arch::riscv32::sm4ks; +/// # #[cfg(target_arch = "riscv64")] use core::arch::riscv64::sm4ks; +/// let k = k1 ^ k2 ^ k3 ^ ck_i; +/// let c0 = sm4ks::<0>(k0, k); +/// let c1 = sm4ks::<1>(c0, k); // c1 represents c[0..=1], etc. +/// let c2 = sm4ks::<2>(c1, k); +/// let c3 = sm4ks::<3>(c2, k); +/// return c3; // c3 represents c[0..=3] +/// # } +/// ``` +/// +/// According to RISC-V Cryptography Extensions, Volume I, the execution latency of +/// this instruction must always be independent from the data it operates on. +#[inline] +#[target_feature(enable = "zksed")] +pub fn sm4ks(x: u32, k: u32) -> u32 { + static_assert!(BS: u8 where BS <= 3); + let ans: u32; + unsafe { + asm!("sm4ks {}, {}, {}, {}", lateout(reg) ans, in(reg) x, in(reg) k, const BS, options(pure, nomem, nostack)) + }; + ans +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/riscv_shared/p.rs b/crux-mir/lib/stdarch/crates/core_arch/src/riscv_shared/p.rs new file mode 100644 index 000000000..a26044aee --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/riscv_shared/p.rs @@ -0,0 +1,1061 @@ +//! RISC-V Packed SIMD intrinsics; shared part. +//! +//! RV64 only part is placed in riscv64 folder. +use crate::arch::asm; + +/// Adds packed 16-bit signed numbers, discarding overflow bits +#[inline] +pub fn add16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x20, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Halves the sum of packed 16-bit signed numbers, dropping least bits +#[inline] +pub fn radd16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x00, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Halves the sum of packed 16-bit unsigned numbers, dropping least bits +#[inline] +pub fn uradd16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x10, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Adds packed 16-bit signed numbers, saturating at the numeric bounds +#[inline] +pub fn kadd16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x08, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Adds packed 16-bit unsigned numbers, saturating at the numeric bounds +#[inline] +pub fn ukadd16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x18, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Subtracts packed 16-bit signed numbers, discarding overflow bits +#[inline] +pub fn sub16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x21, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Halves the subtraction result of packed 16-bit signed numbers, dropping least bits +#[inline] +pub fn rsub16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x01, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Halves the subtraction result of packed 16-bit unsigned numbers, dropping least bits +#[inline] +pub fn ursub16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x11, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Subtracts packed 16-bit signed numbers, saturating at the numeric bounds +#[inline] +pub fn ksub16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x09, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Subtracts packed 16-bit unsigned numbers, saturating at the numeric bounds +#[inline] +pub fn uksub16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x19, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Cross adds and subtracts packed 16-bit signed numbers, discarding overflow bits +#[inline] +pub fn cras16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x22, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Cross halves of adds and subtracts packed 16-bit signed numbers, dropping least bits +#[inline] +pub fn rcras16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x02, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Cross halves of adds and subtracts packed 16-bit unsigned numbers, dropping least bits +#[inline] +pub fn urcras16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x12, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Cross adds and subtracts packed 16-bit signed numbers, saturating at the numeric bounds +#[inline] +pub fn kcras16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x0A, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Cross adds and subtracts packed 16-bit unsigned numbers, saturating at the numeric bounds +#[inline] +pub fn ukcras16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x1A, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Cross subtracts and adds packed 16-bit signed numbers, discarding overflow bits +#[inline] +pub fn crsa16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x23, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Cross halves of subtracts and adds packed 16-bit signed numbers, dropping least bits +#[inline] +pub fn rcrsa16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x03, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Cross halves of subtracts and adds packed 16-bit unsigned numbers, dropping least bits +#[inline] +pub fn urcrsa16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x13, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Cross subtracts and adds packed 16-bit signed numbers, saturating at the numeric bounds +#[inline] +pub fn kcrsa16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x0B, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Cross subtracts and adds packed 16-bit unsigned numbers, saturating at the numeric bounds +#[inline] +pub fn ukcrsa16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x1B, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Straight adds and subtracts packed 16-bit signed numbers, discarding overflow bits +#[inline] +pub fn stas16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x7A, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Straight halves of adds and subtracts packed 16-bit signed numbers, dropping least bits +#[inline] +pub fn rstas16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x5A, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Straight halves of adds and subtracts packed 16-bit unsigned numbers, dropping least bits +#[inline] +pub fn urstas16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x6A, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Straight adds and subtracts packed 16-bit signed numbers, saturating at the numeric bounds +#[inline] +pub fn kstas16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x62, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Straight adds and subtracts packed 16-bit unsigned numbers, saturating at the numeric bounds +#[inline] +pub fn ukstas16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x72, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Straight subtracts and adds packed 16-bit signed numbers, discarding overflow bits +#[inline] +pub fn stsa16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x7B, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Straight halves of subtracts and adds packed 16-bit signed numbers, dropping least bits +#[inline] +pub fn rstsa16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x5B, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Straight halves of subtracts and adds packed 16-bit unsigned numbers, dropping least bits +#[inline] +pub fn urstsa16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x6B, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Straight subtracts and adds packed 16-bit signed numbers, saturating at the numeric bounds +#[inline] +pub fn kstsa16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x63, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Straight subtracts and adds packed 16-bit unsigned numbers, saturating at the numeric bounds +#[inline] +pub fn ukstsa16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x73, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Adds packed 8-bit signed numbers, discarding overflow bits +#[inline] +pub fn add8(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x24, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Halves the sum of packed 8-bit signed numbers, dropping least bits +#[inline] +pub fn radd8(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x04, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Halves the sum of packed 8-bit unsigned numbers, dropping least bits +#[inline] +pub fn uradd8(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x14, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Adds packed 8-bit signed numbers, saturating at the numeric bounds +#[inline] +pub fn kadd8(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x0C, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Adds packed 8-bit unsigned numbers, saturating at the numeric bounds +#[inline] +pub fn ukadd8(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x1C, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Subtracts packed 8-bit signed numbers, discarding overflow bits +#[inline] +pub fn sub8(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x25, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Halves the subtraction result of packed 8-bit signed numbers, dropping least bits +#[inline] +pub fn rsub8(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x05, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Halves the subtraction result of packed 8-bit unsigned numbers, dropping least bits +#[inline] +pub fn ursub8(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x15, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Subtracts packed 8-bit signed numbers, saturating at the numeric bounds +#[inline] +pub fn ksub8(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x0D, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Subtracts packed 8-bit unsigned numbers, saturating at the numeric bounds +#[inline] +pub fn uksub8(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x1D, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Arithmetic right shift packed 16-bit elements without rounding up +#[inline] +pub fn sra16(a: usize, b: u32) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x28, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Arithmetic right shift packed 16-bit elements with rounding up +#[inline] +pub fn sra16u(a: usize, b: u32) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x30, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Logical right shift packed 16-bit elements without rounding up +#[inline] +pub fn srl16(a: usize, b: u32) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x29, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Logical right shift packed 16-bit elements with rounding up +#[inline] +pub fn srl16u(a: usize, b: u32) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x31, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Logical left shift packed 16-bit elements, discarding overflow bits +#[inline] +pub fn sll16(a: usize, b: u32) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x2A, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Logical left shift packed 16-bit elements, saturating at the numeric bounds +#[inline] +pub fn ksll16(a: usize, b: u32) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x32, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Logical saturating left then arithmetic right shift packed 16-bit elements +#[inline] +pub fn kslra16(a: usize, b: i32) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x2B, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Logical saturating left then arithmetic right shift packed 16-bit elements +#[inline] +pub fn kslra16u(a: usize, b: i32) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x33, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Arithmetic right shift packed 8-bit elements without rounding up +#[inline] +pub fn sra8(a: usize, b: u32) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x2C, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Arithmetic right shift packed 8-bit elements with rounding up +#[inline] +pub fn sra8u(a: usize, b: u32) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x34, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Logical right shift packed 8-bit elements without rounding up +#[inline] +pub fn srl8(a: usize, b: u32) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x2D, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Logical right shift packed 8-bit elements with rounding up +#[inline] +pub fn srl8u(a: usize, b: u32) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x35, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Logical left shift packed 8-bit elements, discarding overflow bits +#[inline] +pub fn sll8(a: usize, b: u32) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x2E, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Logical left shift packed 8-bit elements, saturating at the numeric bounds +#[inline] +pub fn ksll8(a: usize, b: u32) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x36, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Logical saturating left then arithmetic right shift packed 8-bit elements +#[inline] +pub fn kslra8(a: usize, b: i32) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x2F, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Logical saturating left then arithmetic right shift packed 8-bit elements +#[inline] +pub fn kslra8u(a: usize, b: i32) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x37, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Compare equality for packed 16-bit elements +#[inline] +pub fn cmpeq16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x26, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Compare whether 16-bit packed signed integers are less than the others +#[inline] +pub fn scmplt16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x06, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Compare whether 16-bit packed signed integers are less than or equal to the others +#[inline] +pub fn scmple16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x0E, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Compare whether 16-bit packed unsigned integers are less than the others +#[inline] +pub fn ucmplt16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x16, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Compare whether 16-bit packed unsigned integers are less than or equal to the others +#[inline] +pub fn ucmple16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x1E, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Compare equality for packed 8-bit elements +#[inline] +pub fn cmpeq8(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x27, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Compare whether 8-bit packed signed integers are less than the others +#[inline] +pub fn scmplt8(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x07, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Compare whether 8-bit packed signed integers are less than or equal to the others +#[inline] +pub fn scmple8(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x0F, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Compare whether 8-bit packed unsigned integers are less than the others +#[inline] +pub fn ucmplt8(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x17, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Compare whether 8-bit packed unsigned integers are less than or equal to the others +#[inline] +pub fn ucmple8(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x1F, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Get minimum values from 16-bit packed signed integers +#[inline] +pub fn smin16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x40, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Get minimum values from 16-bit packed unsigned integers +#[inline] +pub fn umin16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x48, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Get maximum values from 16-bit packed signed integers +#[inline] +pub fn smax16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x41, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Get maximum values from 16-bit packed unsigned integers +#[inline] +pub fn umax16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x49, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/* todo: sclip16, uclip16 */ + +/// Compute the absolute value of packed 16-bit signed integers +#[inline] +pub fn kabs16(a: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn i 0x77, 0x0, {}, {}, 0xAD1", lateout(reg) value, in(reg) a, options(pure, nomem, nostack)) + } + value +} + +/// Count the number of redundant sign bits of the packed 16-bit elements +#[inline] +pub fn clrs16(a: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn i 0x77, 0x0, {}, {}, 0xAE8", lateout(reg) value, in(reg) a, options(pure, nomem, nostack)) + } + value +} + +/// Count the number of leading zero bits of the packed 16-bit elements +#[inline] +pub fn clz16(a: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn i 0x77, 0x0, {}, {}, 0xAE9", lateout(reg) value, in(reg) a, options(pure, nomem, nostack)) + } + value +} + +/// Swap the 16-bit halfwords within each 32-bit word of a register +#[inline] +pub fn swap16(a: usize) -> usize { + let value: usize; + // this instruction is an alias for `pkbt rd, rs1, rs1`. + unsafe { + asm!(".insn r 0x77, 0x0, 0x0F, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) a, options(pure, nomem, nostack)) + } + value +} + +/// Get minimum values from 8-bit packed signed integers +#[inline] +pub fn smin8(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x44, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Get minimum values from 8-bit packed unsigned integers +#[inline] +pub fn umin8(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x4C, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Get maximum values from 8-bit packed signed integers +#[inline] +pub fn smax8(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x45, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Get maximum values from 8-bit packed unsigned integers +#[inline] +pub fn umax8(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x4D, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/* todo: sclip8, uclip8 */ + +/// Compute the absolute value of packed 8-bit signed integers +#[inline] +pub fn kabs8(a: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn i 0x77, 0x0, {}, {}, 0xAD0", lateout(reg) value, in(reg) a, options(pure, nomem, nostack)) + } + value +} + +/// Count the number of redundant sign bits of the packed 8-bit elements +#[inline] +pub fn clrs8(a: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn i 0x77, 0x0, {}, {}, 0xAE0", lateout(reg) value, in(reg) a, options(pure, nomem, nostack)) + } + value +} + +/// Count the number of leading zero bits of the packed 8-bit elements +#[inline] +pub fn clz8(a: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn i 0x77, 0x0, {}, {}, 0xAE1", lateout(reg) value, in(reg) a, options(pure, nomem, nostack)) + } + value +} + +/// Swap the 8-bit bytes within each 16-bit halfword of a register. +#[inline] +pub fn swap8(a: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn i 0x77, 0x0, {}, {}, 0xAD8", lateout(reg) value, in(reg) a, options(pure, nomem, nostack)) + } + value +} + +/// Unpack first and zeroth into two 16-bit signed halfwords in each 32-bit chunk +#[inline] +pub fn sunpkd810(a: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn i 0x77, 0x0, {}, {}, 0xAC8", lateout(reg) value, in(reg) a, options(pure, nomem, nostack)) + } + value +} + +/// Unpack second and zeroth into two 16-bit signed halfwords in each 32-bit chunk +#[inline] +pub fn sunpkd820(a: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn i 0x77, 0x0, {}, {}, 0xAC9", lateout(reg) value, in(reg) a, options(pure, nomem, nostack)) + } + value +} + +/// Unpack third and zeroth into two 16-bit signed halfwords in each 32-bit chunk +#[inline] +pub fn sunpkd830(a: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn i 0x77, 0x0, {}, {}, 0xACA", lateout(reg) value, in(reg) a, options(pure, nomem, nostack)) + } + value +} + +/// Unpack third and first into two 16-bit signed halfwords in each 32-bit chunk +#[inline] +pub fn sunpkd831(a: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn i 0x77, 0x0, {}, {}, 0xACB", lateout(reg) value, in(reg) a, options(pure, nomem, nostack)) + } + value +} + +/// Unpack third and second into two 16-bit signed halfwords in each 32-bit chunk +#[inline] +pub fn sunpkd832(a: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn i 0x77, 0x0, {}, {}, 0xAD3", lateout(reg) value, in(reg) a, options(pure, nomem, nostack)) + } + value +} + +/// Unpack first and zeroth into two 16-bit unsigned halfwords in each 32-bit chunk +#[inline] +pub fn zunpkd810(a: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn i 0x77, 0x0, {}, {}, 0xACC", lateout(reg) value, in(reg) a, options(pure, nomem, nostack)) + } + value +} + +/// Unpack second and zeroth into two 16-bit unsigned halfwords in each 32-bit chunk +#[inline] +pub fn zunpkd820(a: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn i 0x77, 0x0, {}, {}, 0xACD", lateout(reg) value, in(reg) a, options(pure, nomem, nostack)) + } + value +} + +/// Unpack third and zeroth into two 16-bit unsigned halfwords in each 32-bit chunk +#[inline] +pub fn zunpkd830(a: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn i 0x77, 0x0, {}, {}, 0xACE", lateout(reg) value, in(reg) a, options(pure, nomem, nostack)) + } + value +} + +/// Unpack third and first into two 16-bit unsigned halfwords in each 32-bit chunk +#[inline] +pub fn zunpkd831(a: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn i 0x77, 0x0, {}, {}, 0xACF", lateout(reg) value, in(reg) a, options(pure, nomem, nostack)) + } + value +} + +/// Unpack third and second into two 16-bit unsigned halfwords in each 32-bit chunk +#[inline] +pub fn zunpkd832(a: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn i 0x77, 0x0, {}, {}, 0xAD7", lateout(reg) value, in(reg) a, options(pure, nomem, nostack)) + } + value +} + +// todo: pkbb16, pktt16 + +/// Pack two 16-bit data from bottom and top half from 32-bit chunks +#[inline] +pub fn pkbt16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x1, 0x0F, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Pack two 16-bit data from top and bottom half from 32-bit chunks +#[inline] +pub fn pktb16(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x1, 0x1F, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Count the number of redundant sign bits of the packed 32-bit elements +#[inline] +pub fn clrs32(a: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn i 0x77, 0x0, {}, {}, 0xAF8", lateout(reg) value, in(reg) a, options(pure, nomem, nostack)) + } + value +} + +/// Count the number of leading zero bits of the packed 32-bit elements +#[inline] +pub fn clz32(a: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn i 0x77, 0x0, {}, {}, 0xAF9", lateout(reg) value, in(reg) a, options(pure, nomem, nostack)) + } + value +} + +/// Calculate the sum of absolute difference of unsigned 8-bit data elements +#[inline] +pub fn pbsad(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x7E, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Calculate and accumulate the sum of absolute difference of unsigned 8-bit data elements +#[inline] +pub fn pbsada(t: usize, a: usize, b: usize) -> usize { + let mut value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x7F, {}, {}, {}", inlateout(reg) t => value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Multiply signed 8-bit elements and add 16-bit elements on results for packed 32-bit chunks +#[inline] +pub fn smaqa(t: usize, a: usize, b: usize) -> usize { + let mut value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x64, {}, {}, {}", inlateout(reg) t => value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Multiply unsigned 8-bit elements and add 16-bit elements on results for packed 32-bit chunks +#[inline] +pub fn umaqa(t: usize, a: usize, b: usize) -> usize { + let mut value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x66, {}, {}, {}", inlateout(reg) t => value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Multiply signed to unsigned 8-bit and add 16-bit elements on results for packed 32-bit chunks +#[inline] +pub fn smaqasu(t: usize, a: usize, b: usize) -> usize { + let mut value: usize; + unsafe { + asm!(".insn r 0x77, 0x0, 0x65, {}, {}, {}", inlateout(reg) t => value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Adds signed lower 16-bit content of two registers with Q15 saturation +#[inline] +pub fn kaddh(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x1, 0x02, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Subtracts signed lower 16-bit content of two registers with Q15 saturation +#[inline] +pub fn ksubh(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x1, 0x03, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Adds signed lower 16-bit content of two registers with U16 saturation +#[inline] +pub fn ukaddh(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x1, 0x0A, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} + +/// Subtracts signed lower 16-bit content of two registers with U16 saturation +#[inline] +pub fn uksubh(a: usize, b: usize) -> usize { + let value: usize; + unsafe { + asm!(".insn r 0x77, 0x1, 0x0B, {}, {}, {}", lateout(reg) value, in(reg) a, in(reg) b, options(pure, nomem, nostack)) + } + value +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/simd.rs b/crux-mir/lib/stdarch/crates/core_arch/src/simd.rs index 746f08464..281fefba4 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/simd.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/simd.rs @@ -10,7 +10,7 @@ macro_rules! simd_ty { #[allow(clippy::use_self)] impl $id { - #[inline] + #[inline(always)] pub(crate) const fn new($($elem_name: $elem_ty),*) -> Self { $id($($elem_name),*) } @@ -43,12 +43,12 @@ macro_rules! simd_m_ty { #[allow(clippy::use_self)] impl $id { - #[inline] + #[inline(always)] const fn bool_to_internal(x: bool) -> $ety { [0 as $ety, !(0 as $ety)][x as usize] } - #[inline] + #[inline(always)] pub(crate) const fn new($($elem_name: bool),*) -> Self { $id($(Self::bool_to_internal($elem_name)),*) } @@ -90,114 +90,1016 @@ simd_ty!(i16x2[i16]: i16, i16 | x0, x1); // 64-bit wide types: -simd_ty!(u8x8[u8]: - u8, u8, u8, u8, u8, u8, u8, u8 - | x0, x1, x2, x3, x4, x5, x6, x7); +simd_ty!( + u8x8[u8]: u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7 +); simd_ty!(u16x4[u16]: u16, u16, u16, u16 | x0, x1, x2, x3); simd_ty!(u32x2[u32]: u32, u32 | x0, x1); simd_ty!(u64x1[u64]: u64 | x1); -simd_ty!(i8x8[i8]: - i8, i8, i8, i8, i8, i8, i8, i8 - | x0, x1, x2, x3, x4, x5, x6, x7); +simd_ty!( + i8x8[i8]: i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7 +); simd_ty!(i16x4[i16]: i16, i16, i16, i16 | x0, x1, x2, x3); simd_ty!(i32x2[i32]: i32, i32 | x0, x1); simd_ty!(i64x1[i64]: i64 | x1); simd_ty!(f32x2[f32]: f32, f32 | x0, x1); +simd_ty!(f64x1[f64]: f64 | x1); // 128-bit wide types: -simd_ty!(u8x16[u8]: - u8, u8, u8, u8, u8, u8, u8, u8, - u8, u8, u8, u8, u8, u8, u8, u8 - | x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15 +simd_ty!( + u8x16[u8]: u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7, + x8, + x9, + x10, + x11, + x12, + x13, + x14, + x15 +); +simd_ty!( + u16x8[u16]: u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7 ); -simd_ty!(u16x8[u16]: - u16, u16, u16, u16, u16, u16, u16, u16 - | x0, x1, x2, x3, x4, x5, x6, x7); simd_ty!(u32x4[u32]: u32, u32, u32, u32 | x0, x1, x2, x3); simd_ty!(u64x2[u64]: u64, u64 | x0, x1); -simd_ty!(i8x16[i8]: - i8, i8, i8, i8, i8, i8, i8, i8, - i8, i8, i8, i8, i8, i8, i8, i8 - | x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15 +simd_ty!( + i8x16[i8]: i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7, + x8, + x9, + x10, + x11, + x12, + x13, + x14, + x15 +); +simd_ty!( + i16x8[i16]: i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7 ); -simd_ty!(i16x8[i16]: - i16, i16, i16, i16, i16, i16, i16, i16 - | x0, x1, x2, x3, x4, x5, x6, x7); simd_ty!(i32x4[i32]: i32, i32, i32, i32 | x0, x1, x2, x3); simd_ty!(i64x2[i64]: i64, i64 | x0, x1); simd_ty!(f32x4[f32]: f32, f32, f32, f32 | x0, x1, x2, x3); simd_ty!(f64x2[f64]: f64, f64 | x0, x1); +simd_ty!(f64x4[f64]: f64, f64, f64, f64 | x0, x1, x2, x3); -simd_m_ty!(m8x16[i8]: - i8, i8, i8, i8, i8, i8, i8, i8, - i8, i8, i8, i8, i8, i8, i8, i8 - | x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15 +simd_m_ty!( + m8x16[i8]: i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7, + x8, + x9, + x10, + x11, + x12, + x13, + x14, + x15 +); +simd_m_ty!( + m16x8[i16]: i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7 ); -simd_m_ty!(m16x8[i16]: - i16, i16, i16, i16, i16, i16, i16, i16 - | x0, x1, x2, x3, x4, x5, x6, x7); simd_m_ty!(m32x4[i32]: i32, i32, i32, i32 | x0, x1, x2, x3); simd_m_ty!(m64x2[i64]: i64, i64 | x0, x1); // 256-bit wide types: -simd_ty!(u8x32[u8]: - u8, u8, u8, u8, u8, u8, u8, u8, - u8, u8, u8, u8, u8, u8, u8, u8, - u8, u8, u8, u8, u8, u8, u8, u8, - u8, u8, u8, u8, u8, u8, u8, u8 - | x0, x1, x2, x3, x4, x5, x6, x7, - x8, x9, x10, x11, x12, x13, x14, x15, - x16, x17, x18, x19, x20, x21, x22, x23, - x24, x25, x26, x27, x28, x29, x30, x31 -); -simd_ty!(u16x16[u16]: - u16, u16, u16, u16, u16, u16, u16, u16, - u16, u16, u16, u16, u16, u16, u16, u16 - | x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15 -); -simd_ty!(u32x8[u32]: - u32, u32, u32, u32, u32, u32, u32, u32 - | x0, x1, x2, x3, x4, x5, x6, x7); +simd_ty!( + u8x32[u8]: u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7, + x8, + x9, + x10, + x11, + x12, + x13, + x14, + x15, + x16, + x17, + x18, + x19, + x20, + x21, + x22, + x23, + x24, + x25, + x26, + x27, + x28, + x29, + x30, + x31 +); +simd_ty!( + u16x16[u16]: u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7, + x8, + x9, + x10, + x11, + x12, + x13, + x14, + x15 +); +simd_ty!( + u32x8[u32]: u32, + u32, + u32, + u32, + u32, + u32, + u32, + u32 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7 +); simd_ty!(u64x4[u64]: u64, u64, u64, u64 | x0, x1, x2, x3); -simd_ty!(i8x32[i8]: - i8, i8, i8, i8, i8, i8, i8, i8, - i8, i8, i8, i8, i8, i8, i8, i8, - i8, i8, i8, i8, i8, i8, i8, i8, - i8, i8, i8, i8, i8, i8, i8, i8 - | x0, x1, x2, x3, x4, x5, x6, x7, - x8, x9, x10, x11, x12, x13, x14, x15, - x16, x17, x18, x19, x20, x21, x22, x23, - x24, x25, x26, x27, x28, x29, x30, x31 -); -simd_ty!(i16x16[i16]: - i16, i16, i16, i16, i16, i16, i16, i16, - i16, i16, i16, i16, i16, i16, i16, i16 - | x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15 -); -simd_ty!(i32x8[i32]: - i32, i32, i32, i32, i32, i32, i32, i32 - | x0, x1, x2, x3, x4, x5, x6, x7); +simd_ty!( + i8x32[i8]: i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7, + x8, + x9, + x10, + x11, + x12, + x13, + x14, + x15, + x16, + x17, + x18, + x19, + x20, + x21, + x22, + x23, + x24, + x25, + x26, + x27, + x28, + x29, + x30, + x31 +); +simd_ty!( + i16x16[i16]: i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7, + x8, + x9, + x10, + x11, + x12, + x13, + x14, + x15 +); +simd_ty!( + i32x8[i32]: i32, + i32, + i32, + i32, + i32, + i32, + i32, + i32 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7 +); simd_ty!(i64x4[i64]: i64, i64, i64, i64 | x0, x1, x2, x3); -simd_ty!(f32x8[f32]: - f32, f32, f32, f32, f32, f32, f32, f32 | - x0, x1, x2, x3, x4, x5, x6, x7); +simd_ty!( + f32x8[f32]: f32, + f32, + f32, + f32, + f32, + f32, + f32, + f32 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7 +); // 512-bit wide types: -simd_ty!(i32x16[i32]: - i32, i32, i32, i32, i32, i32, i32, i32, - i32, i32, i32, i32, i32, i32, i32, i32 - | x0, x1, x2, x3, x4, x5, x6, x7, - x8, x9, x10, x11, x12, x13, x14, x15); +simd_ty!( + i8x64[i8]: i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8, + i8 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7, + x8, + x9, + x10, + x11, + x12, + x13, + x14, + x15, + x16, + x17, + x18, + x19, + x20, + x21, + x22, + x23, + x24, + x25, + x26, + x27, + x28, + x29, + x30, + x31, + x32, + x33, + x34, + x35, + x36, + x37, + x38, + x39, + x40, + x41, + x42, + x43, + x44, + x45, + x46, + x47, + x48, + x49, + x50, + x51, + x52, + x53, + x54, + x55, + x56, + x57, + x58, + x59, + x60, + x61, + x62, + x63 +); -simd_ty!(i64x8[i64]: - i64, i64, i64, i64, i64, i64, i64, i64 - | x0, x1, x2, x3, x4, x5, x6, x7); +simd_ty!( + u8x64[u8]: u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8, + u8 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7, + x8, + x9, + x10, + x11, + x12, + x13, + x14, + x15, + x16, + x17, + x18, + x19, + x20, + x21, + x22, + x23, + x24, + x25, + x26, + x27, + x28, + x29, + x30, + x31, + x32, + x33, + x34, + x35, + x36, + x37, + x38, + x39, + x40, + x41, + x42, + x43, + x44, + x45, + x46, + x47, + x48, + x49, + x50, + x51, + x52, + x53, + x54, + x55, + x56, + x57, + x58, + x59, + x60, + x61, + x62, + x63 +); + +simd_ty!( + i16x32[i16]: i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16, + i16 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7, + x8, + x9, + x10, + x11, + x12, + x13, + x14, + x15, + x16, + x17, + x18, + x19, + x20, + x21, + x22, + x23, + x24, + x25, + x26, + x27, + x28, + x29, + x30, + x31 +); + +simd_ty!( + u16x32[u16]: u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16, + u16 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7, + x8, + x9, + x10, + x11, + x12, + x13, + x14, + x15, + x16, + x17, + x18, + x19, + x20, + x21, + x22, + x23, + x24, + x25, + x26, + x27, + x28, + x29, + x30, + x31 +); + +simd_ty!( + i32x16[i32]: i32, + i32, + i32, + i32, + i32, + i32, + i32, + i32, + i32, + i32, + i32, + i32, + i32, + i32, + i32, + i32 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7, + x8, + x9, + x10, + x11, + x12, + x13, + x14, + x15 +); + +simd_ty!( + u32x16[u32]: u32, + u32, + u32, + u32, + u32, + u32, + u32, + u32, + u32, + u32, + u32, + u32, + u32, + u32, + u32, + u32 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7, + x8, + x9, + x10, + x11, + x12, + x13, + x14, + x15 +); + +simd_ty!( + f32x16[f32]: f32, + f32, + f32, + f32, + f32, + f32, + f32, + f32, + f32, + f32, + f32, + f32, + f32, + f32, + f32, + f32 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7, + x8, + x9, + x10, + x11, + x12, + x13, + x14, + x15 +); + +simd_ty!( + i64x8[i64]: i64, + i64, + i64, + i64, + i64, + i64, + i64, + i64 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7 +); + +simd_ty!( + u64x8[u64]: u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7 +); + +simd_ty!( + f64x8[f64]: f64, + f64, + f64, + f64, + f64, + f64, + f64, + f64 | x0, + x1, + x2, + x3, + x4, + x5, + x6, + x7 +); diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/simd_llvm.rs b/crux-mir/lib/stdarch/crates/core_arch/src/simd_llvm.rs index 02934f7cb..decdecaaf 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/simd_llvm.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/simd_llvm.rs @@ -1,6 +1,4 @@ -//! LLVM's simd platform intrinsics -//! -//! TODO: should use `link_llvm_intrinsic` instead: issue #112 +//! LLVM's SIMD platform intrinsics extern "platform-intrinsic" { //pub fn simd_select_bitmask @@ -11,20 +9,7 @@ extern "platform-intrinsic" { pub fn simd_gt(x: T, y: T) -> U; pub fn simd_ge(x: T, y: T) -> U; - #[rustc_args_required_const(2)] - pub fn simd_shuffle2(x: T, y: T, idx: [u32; 2]) -> U; - #[rustc_args_required_const(2)] - pub fn simd_shuffle4(x: T, y: T, idx: [u32; 4]) -> U; - #[rustc_args_required_const(2)] - pub fn simd_shuffle8(x: T, y: T, idx: [u32; 8]) -> U; - #[rustc_args_required_const(2)] - pub fn simd_shuffle16(x: T, y: T, idx: [u32; 16]) -> U; - #[rustc_args_required_const(2)] - pub fn simd_shuffle32(x: T, y: T, idx: [u32; 32]) -> U; - #[rustc_args_required_const(2)] - pub fn simd_shuffle64(x: T, y: T, idx: [u32; 64]) -> U; - #[rustc_args_required_const(2)] - pub fn simd_shuffle128(x: T, y: T, idx: [u32; 128]) -> U; + pub fn simd_shuffle(x: T, y: T, idx: U) -> V; #[rustc_const_unstable(feature = "const_simd_insert", issue = "none")] pub fn simd_insert(x: T, idx: u32, val: U) -> T; @@ -45,6 +30,8 @@ extern "platform-intrinsic" { pub fn simd_or(x: T, y: T) -> T; pub fn simd_xor(x: T, y: T) -> T; + pub fn simd_neg(x: T) -> T; + pub fn simd_saturating_add(x: T, y: T) -> T; pub fn simd_saturating_sub(x: T, y: T) -> T; diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/wasm32/atomic.rs b/crux-mir/lib/stdarch/crates/core_arch/src/wasm32/atomic.rs index b8ffaeac0..52d4bea87 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/wasm32/atomic.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/wasm32/atomic.rs @@ -1,34 +1,22 @@ -//! Intrinsics associated with WebAssembly's upcoming threads proposal. -//! -//! These intrinsics are all unstable because they're not actually stable in -//! WebAssembly itself yet. The signatures may change as [the -//! specification][spec] is updated. -//! -//! [spec]: https://github.com/WebAssembly/threads - -#![cfg(any(target_feature = "atomics", dox))] - #[cfg(test)] use stdarch_test::assert_instr; -#[cfg(test)] -use wasm_bindgen_test::wasm_bindgen_test; extern "C" { - #[link_name = "llvm.wasm.atomic.wait.i32"] + #[link_name = "llvm.wasm.memory.atomic.wait32"] fn llvm_atomic_wait_i32(ptr: *mut i32, exp: i32, timeout: i64) -> i32; - #[link_name = "llvm.wasm.atomic.wait.i64"] + #[link_name = "llvm.wasm.memory.atomic.wait64"] fn llvm_atomic_wait_i64(ptr: *mut i64, exp: i64, timeout: i64) -> i32; - #[link_name = "llvm.wasm.atomic.notify"] + #[link_name = "llvm.wasm.memory.atomic.notify"] fn llvm_atomic_notify(ptr: *mut i32, cnt: i32) -> i32; } -/// Corresponding intrinsic to wasm's [`i32.atomic.wait` instruction][instr] +/// Corresponding intrinsic to wasm's [`memory.atomic.wait32` instruction][instr] /// /// This function, when called, will block the current thread if the memory /// pointed to by `ptr` is equal to `expression` (performing this action /// atomically). /// -/// The argument `timeout_ns` is a maxinum number of nanoseconds the calling +/// The argument `timeout_ns` is a maximum number of nanoseconds the calling /// thread will be blocked for, if it blocks. If the timeout is negative then /// the calling thread will be blocked forever. /// @@ -43,27 +31,22 @@ extern "C" { /// didn't block /// * 2 - the thread blocked, but the timeout expired. /// -/// # Availability -/// -/// This intrinsic is only available **when the standard library itself is -/// compiled with the `atomics` target feature**. This version of the standard -/// library is not obtainable via `rustup`, but rather will require the -/// standard library to be compiled from source. -/// -/// [instr]: https://github.com/WebAssembly/threads/blob/master/proposals/threads/Overview.md#wait +/// [instr]: https://webassembly.github.io/threads/core/syntax/instructions.html#syntax-instr-atomic-memory #[inline] -#[cfg_attr(test, assert_instr("i32.atomic.wait"))] -pub unsafe fn i32_atomic_wait(ptr: *mut i32, expression: i32, timeout_ns: i64) -> i32 { +#[cfg_attr(test, assert_instr(memory.atomic.wait32))] +#[target_feature(enable = "atomics")] +#[doc(alias("memory.atomic.wait32"))] +pub unsafe fn memory_atomic_wait32(ptr: *mut i32, expression: i32, timeout_ns: i64) -> i32 { llvm_atomic_wait_i32(ptr, expression, timeout_ns) } -/// Corresponding intrinsic to wasm's [`i64.atomic.wait` instruction][instr] +/// Corresponding intrinsic to wasm's [`memory.atomic.wait64` instruction][instr] /// /// This function, when called, will block the current thread if the memory /// pointed to by `ptr` is equal to `expression` (performing this action /// atomically). /// -/// The argument `timeout_ns` is a maxinum number of nanoseconds the calling +/// The argument `timeout_ns` is a maximum number of nanoseconds the calling /// thread will be blocked for, if it blocks. If the timeout is negative then /// the calling thread will be blocked forever. /// @@ -78,21 +61,16 @@ pub unsafe fn i32_atomic_wait(ptr: *mut i32, expression: i32, timeout_ns: i64) - /// didn't block /// * 2 - the thread blocked, but the timeout expired. /// -/// # Availability -/// -/// This intrinsic is only available **when the standard library itself is -/// compiled with the `atomics` target feature**. This version of the standard -/// library is not obtainable via `rustup`, but rather will require the -/// standard library to be compiled from source. -/// -/// [instr]: https://github.com/WebAssembly/threads/blob/master/proposals/threads/Overview.md#wait +/// [instr]: https://webassembly.github.io/threads/core/syntax/instructions.html#syntax-instr-atomic-memory #[inline] -#[cfg_attr(test, assert_instr("i64.atomic.wait"))] -pub unsafe fn i64_atomic_wait(ptr: *mut i64, expression: i64, timeout_ns: i64) -> i32 { +#[cfg_attr(test, assert_instr(memory.atomic.wait64))] +#[target_feature(enable = "atomics")] +#[doc(alias("memory.atomic.wait64"))] +pub unsafe fn memory_atomic_wait64(ptr: *mut i64, expression: i64, timeout_ns: i64) -> i32 { llvm_atomic_wait_i64(ptr, expression, timeout_ns) } -/// Corresponding intrinsic to wasm's [`atomic.notify` instruction][instr] +/// Corresponding intrinsic to wasm's [`memory.atomic.notify` instruction][instr] /// /// This function will notify a number of threads blocked on the address /// indicated by `ptr`. Threads previously blocked with the `i32_atomic_wait` @@ -105,16 +83,11 @@ pub unsafe fn i64_atomic_wait(ptr: *mut i64, expression: i64, timeout_ns: i64) - /// /// Returns the number of waiters which were actually notified. /// -/// # Availability -/// -/// This intrinsic is only available **when the standard library itself is -/// compiled with the `atomics` target feature**. This version of the standard -/// library is not obtainable via `rustup`, but rather will require the -/// standard library to be compiled from source. -/// -/// [instr]: https://github.com/WebAssembly/threads/blob/master/proposals/threads/Overview.md#wake +/// [instr]: https://webassembly.github.io/threads/core/syntax/instructions.html#syntax-instr-atomic-memory #[inline] -#[cfg_attr(test, assert_instr("atomic.wake"))] -pub unsafe fn atomic_notify(ptr: *mut i32, waiters: u32) -> u32 { +#[cfg_attr(test, assert_instr(memory.atomic.notify))] +#[target_feature(enable = "atomics")] +#[doc(alias("memory.atomic.notify"))] +pub unsafe fn memory_atomic_notify(ptr: *mut i32, waiters: u32) -> u32 { llvm_atomic_notify(ptr, waiters as i32) as u32 } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/wasm32/memory.rs b/crux-mir/lib/stdarch/crates/core_arch/src/wasm32/memory.rs index f2c7fa54c..b5cf13e98 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/wasm32/memory.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/wasm32/memory.rs @@ -1,13 +1,11 @@ #[cfg(test)] use stdarch_test::assert_instr; -#[cfg(test)] -use wasm_bindgen_test::wasm_bindgen_test; extern "C" { - #[link_name = "llvm.wasm.memory.grow.i32"] - fn llvm_memory_grow(mem: i32, pages: i32) -> i32; - #[link_name = "llvm.wasm.memory.size.i32"] - fn llvm_memory_size(mem: i32) -> i32; + #[link_name = "llvm.wasm.memory.grow"] + fn llvm_memory_grow(mem: u32, pages: usize) -> usize; + #[link_name = "llvm.wasm.memory.size"] + fn llvm_memory_size(mem: u32) -> usize; } /// Corresponding intrinsic to wasm's [`memory.size` instruction][instr] @@ -15,7 +13,7 @@ extern "C" { /// This function, when called, will return the current memory size in units of /// pages. The current WebAssembly page size is 65536 bytes (64 KB). /// -/// The argument `mem` is the numerical index of which memory to return the +/// The argument `MEM` is the numerical index of which memory to return the /// size of. Note that currently the WebAssembly specification only supports one /// memory, so it is required that zero is passed in. The argument is present to /// be forward-compatible with future WebAssembly revisions. If a nonzero @@ -23,16 +21,13 @@ extern "C" { /// /// [instr]: http://webassembly.github.io/spec/core/exec/instructions.html#exec-memory-size #[inline] -#[cfg_attr(test, assert_instr("memory.size", mem = 0))] -#[rustc_args_required_const(0)] +#[cfg_attr(test, assert_instr("memory.size", MEM = 0))] +#[rustc_legacy_const_generics(0)] #[stable(feature = "simd_wasm32", since = "1.33.0")] -pub fn memory_size(mem: u32) -> usize { - unsafe { - if mem != 0 { - crate::intrinsics::abort(); - } - llvm_memory_size(0) as usize - } +#[doc(alias("memory.size"))] +pub fn memory_size() -> usize { + static_assert!(MEM: u32 where MEM == 0); + unsafe { llvm_memory_size(MEM) } } /// Corresponding intrinsic to wasm's [`memory.grow` instruction][instr] @@ -41,9 +36,9 @@ pub fn memory_size(mem: u32) -> usize { /// by the specified `delta` of pages. The current WebAssembly page size is /// 65536 bytes (64 KB). If memory is successfully grown then the previous size /// of memory, in pages, is returned. If memory cannot be grown then -/// `usize::max_value()` is returned. +/// `usize::MAX` is returned. /// -/// The argument `mem` is the numerical index of which memory to return the +/// The argument `MEM` is the numerical index of which memory to return the /// size of. Note that currently the WebAssembly specification only supports one /// memory, so it is required that zero is passed in. The argument is present to /// be forward-compatible with future WebAssembly revisions. If a nonzero @@ -51,14 +46,13 @@ pub fn memory_size(mem: u32) -> usize { /// /// [instr]: http://webassembly.github.io/spec/core/exec/instructions.html#exec-memory-grow #[inline] -#[cfg_attr(test, assert_instr("memory.grow", mem = 0))] -#[rustc_args_required_const(0)] +#[cfg_attr(test, assert_instr("memory.grow", MEM = 0))] +#[rustc_legacy_const_generics(0)] #[stable(feature = "simd_wasm32", since = "1.33.0")] -pub fn memory_grow(mem: u32, delta: usize) -> usize { +#[doc(alias("memory.grow"))] +pub fn memory_grow(delta: usize) -> usize { unsafe { - if mem != 0 { - crate::intrinsics::abort(); - } - llvm_memory_grow(0, delta as i32) as isize as usize + static_assert!(MEM: u32 where MEM == 0); + llvm_memory_grow(MEM, delta) } } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/wasm32/mod.rs b/crux-mir/lib/stdarch/crates/core_arch/src/wasm32/mod.rs index 5e7a9d85f..2fbe80e99 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/wasm32/mod.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/wasm32/mod.rs @@ -2,26 +2,25 @@ #[cfg(test)] use stdarch_test::assert_instr; -#[cfg(test)] -use wasm_bindgen_test::wasm_bindgen_test; -#[cfg(any(target_feature = "atomics", dox))] mod atomic; -#[cfg(any(target_feature = "atomics", dox))] pub use self::atomic::*; -#[cfg(any(target_feature = "simd128", dox))] mod simd128; -#[cfg(any(target_feature = "simd128", dox))] pub use self::simd128::*; mod memory; pub use self::memory::*; -/// Generates the trap instruction `UNREACHABLE` +/// Generates the [`unreachable`] instruction, which causes an unconditional [trap]. +/// +/// This function is safe to call and immediately aborts the execution. +/// +/// [`unreachable`]: https://webassembly.github.io/spec/core/syntax/instructions.html#syntax-instr-control +/// [trap]: https://webassembly.github.io/spec/core/intro/overview.html#trap #[cfg_attr(test, assert_instr(unreachable))] #[inline] #[stable(feature = "unreachable_wasm32", since = "1.37.0")] -pub unsafe fn unreachable() -> ! { +pub fn unreachable() -> ! { crate::intrinsics::abort() } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/wasm32/simd128.rs b/crux-mir/lib/stdarch/crates/core_arch/src/wasm32/simd128.rs index c7377e23a..c0025696b 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/wasm32/simd128.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/wasm32/simd128.rs @@ -4,1149 +4,3235 @@ //! https://github.com/WebAssembly/simd/blob/master/proposals/simd/SIMD.md #![allow(non_camel_case_types)] +#![allow(unused_imports)] use crate::{ - core_arch::{simd::*, simd_llvm::*}, + core_arch::{simd, simd_llvm::*}, marker::Sized, - mem::transmute, - ptr, + mem, ptr, }; #[cfg(test)] use stdarch_test::assert_instr; -#[cfg(test)] -use wasm_bindgen_test::wasm_bindgen_test; types! { /// WASM-specific 128-bit wide SIMD vector type. + /// + /// This type corresponds to the `v128` type in the [WebAssembly SIMD + /// proposal](https://github.com/webassembly/simd). This type is 128-bits + /// large and the meaning of all the bits is defined within the context of + /// how this value is used. + /// + /// This same type is used simultaneously for all 128-bit-wide SIMD types, + /// for example: + /// + /// * sixteen 8-bit integers (both `i8` and `u8`) + /// * eight 16-bit integers (both `i16` and `u16`) + /// * four 32-bit integers (both `i32` and `u32`) + /// * two 64-bit integers (both `i64` and `u64`) + /// * four 32-bit floats (`f32`) + /// * two 64-bit floats (`f64`) + /// + /// The `v128` type in Rust is intended to be quite analogous to the `v128` + /// type in WebAssembly. Operations on `v128` can only be performed with the + /// functions in this module. // N.B., internals here are arbitrary. + #[stable(feature = "wasm_simd", since = "1.54.0")] pub struct v128(i32, i32, i32, i32); } -#[allow(non_camel_case_types)] -#[unstable(feature = "stdimd_internal", issue = "none")] -pub(crate) trait v128Ext: Sized { - fn as_v128(self) -> v128; - - #[inline] - fn as_u8x16(self) -> u8x16 { - unsafe { transmute(self.as_v128()) } - } - - #[inline] - fn as_u16x8(self) -> u16x8 { - unsafe { transmute(self.as_v128()) } - } - - #[inline] - fn as_u32x4(self) -> u32x4 { - unsafe { transmute(self.as_v128()) } - } - - #[inline] - fn as_u64x2(self) -> u64x2 { - unsafe { transmute(self.as_v128()) } - } - - #[inline] - fn as_i8x16(self) -> i8x16 { - unsafe { transmute(self.as_v128()) } - } - - #[inline] - fn as_i16x8(self) -> i16x8 { - unsafe { transmute(self.as_v128()) } - } - - #[inline] - fn as_i32x4(self) -> i32x4 { - unsafe { transmute(self.as_v128()) } - } - - #[inline] - fn as_i64x2(self) -> i64x2 { - unsafe { transmute(self.as_v128()) } - } - - #[inline] - fn as_f32x4(self) -> f32x4 { - unsafe { transmute(self.as_v128()) } - } - - #[inline] - fn as_f64x2(self) -> f64x2 { - unsafe { transmute(self.as_v128()) } +macro_rules! conversions { + ($(($name:ident = $ty:ty))*) => { + impl v128 { + $( + #[inline(always)] + fn $name(self) -> $ty { + unsafe { mem::transmute(self) } + } + )* + } + $( + impl $ty { + #[inline(always)] + #[rustc_const_stable(feature = "wasm_simd_const", since = "1.56.0")] + const fn v128(self) -> v128 { + unsafe { mem::transmute(self) } + } + } + )* } } -impl v128Ext for v128 { - #[inline] - fn as_v128(self) -> Self { - self - } +conversions! { + (as_u8x16 = simd::u8x16) + (as_u16x8 = simd::u16x8) + (as_u32x4 = simd::u32x4) + (as_u64x2 = simd::u64x2) + (as_i8x16 = simd::i8x16) + (as_i16x8 = simd::i16x8) + (as_i32x4 = simd::i32x4) + (as_i64x2 = simd::i64x2) + (as_f32x4 = simd::f32x4) + (as_f64x2 = simd::f64x2) } #[allow(improper_ctypes)] extern "C" { + #[link_name = "llvm.wasm.swizzle"] + fn llvm_swizzle(a: simd::i8x16, b: simd::i8x16) -> simd::i8x16; + + #[link_name = "llvm.wasm.bitselect.v16i8"] + fn llvm_bitselect(a: simd::i8x16, b: simd::i8x16, c: simd::i8x16) -> simd::i8x16; #[link_name = "llvm.wasm.anytrue.v16i8"] - fn llvm_i8x16_any_true(x: i8x16) -> i32; + fn llvm_any_true_i8x16(x: simd::i8x16) -> i32; + #[link_name = "llvm.wasm.alltrue.v16i8"] - fn llvm_i8x16_all_true(x: i8x16) -> i32; + fn llvm_i8x16_all_true(x: simd::i8x16) -> i32; + #[link_name = "llvm.ctpop.v16i8"] + fn llvm_popcnt(a: simd::i8x16) -> simd::i8x16; + #[link_name = "llvm.wasm.bitmask.v16i8"] + fn llvm_bitmask_i8x16(a: simd::i8x16) -> i32; + #[link_name = "llvm.wasm.narrow.signed.v16i8.v8i16"] + fn llvm_narrow_i8x16_s(a: simd::i16x8, b: simd::i16x8) -> simd::i8x16; + #[link_name = "llvm.wasm.narrow.unsigned.v16i8.v8i16"] + fn llvm_narrow_i8x16_u(a: simd::i16x8, b: simd::i16x8) -> simd::i8x16; #[link_name = "llvm.sadd.sat.v16i8"] - fn llvm_i8x16_add_saturate_s(a: i8x16, b: i8x16) -> i8x16; + fn llvm_i8x16_add_sat_s(a: simd::i8x16, b: simd::i8x16) -> simd::i8x16; #[link_name = "llvm.uadd.sat.v16i8"] - fn llvm_i8x16_add_saturate_u(a: i8x16, b: i8x16) -> i8x16; - #[link_name = "llvm.wasm.sub.saturate.signed.v16i8"] - fn llvm_i8x16_sub_saturate_s(a: i8x16, b: i8x16) -> i8x16; - #[link_name = "llvm.wasm.sub.saturate.unsigned.v16i8"] - fn llvm_i8x16_sub_saturate_u(a: i8x16, b: i8x16) -> i8x16; - - #[link_name = "llvm.wasm.anytrue.v8i16"] - fn llvm_i16x8_any_true(x: i16x8) -> i32; + fn llvm_i8x16_add_sat_u(a: simd::i8x16, b: simd::i8x16) -> simd::i8x16; + #[link_name = "llvm.wasm.sub.sat.signed.v16i8"] + fn llvm_i8x16_sub_sat_s(a: simd::i8x16, b: simd::i8x16) -> simd::i8x16; + #[link_name = "llvm.wasm.sub.sat.unsigned.v16i8"] + fn llvm_i8x16_sub_sat_u(a: simd::i8x16, b: simd::i8x16) -> simd::i8x16; + #[link_name = "llvm.wasm.avgr.unsigned.v16i8"] + fn llvm_avgr_u_i8x16(a: simd::i8x16, b: simd::i8x16) -> simd::i8x16; + + #[link_name = "llvm.wasm.extadd.pairwise.signed.v8i16"] + fn llvm_i16x8_extadd_pairwise_i8x16_s(x: simd::i8x16) -> simd::i16x8; + #[link_name = "llvm.wasm.extadd.pairwise.unsigned.v8i16"] + fn llvm_i16x8_extadd_pairwise_i8x16_u(x: simd::i8x16) -> simd::i16x8; + #[link_name = "llvm.wasm.q15mulr.sat.signed"] + fn llvm_q15mulr(a: simd::i16x8, b: simd::i16x8) -> simd::i16x8; #[link_name = "llvm.wasm.alltrue.v8i16"] - fn llvm_i16x8_all_true(x: i16x8) -> i32; + fn llvm_i16x8_all_true(x: simd::i16x8) -> i32; + #[link_name = "llvm.wasm.bitmask.v8i16"] + fn llvm_bitmask_i16x8(a: simd::i16x8) -> i32; + #[link_name = "llvm.wasm.narrow.signed.v8i16.v4i32"] + fn llvm_narrow_i16x8_s(a: simd::i32x4, b: simd::i32x4) -> simd::i16x8; + #[link_name = "llvm.wasm.narrow.unsigned.v8i16.v4i32"] + fn llvm_narrow_i16x8_u(a: simd::i32x4, b: simd::i32x4) -> simd::i16x8; #[link_name = "llvm.sadd.sat.v8i16"] - fn llvm_i16x8_add_saturate_s(a: i16x8, b: i16x8) -> i16x8; + fn llvm_i16x8_add_sat_s(a: simd::i16x8, b: simd::i16x8) -> simd::i16x8; #[link_name = "llvm.uadd.sat.v8i16"] - fn llvm_i16x8_add_saturate_u(a: i16x8, b: i16x8) -> i16x8; - #[link_name = "llvm.wasm.sub.saturate.signed.v8i16"] - fn llvm_i16x8_sub_saturate_s(a: i16x8, b: i16x8) -> i16x8; - #[link_name = "llvm.wasm.sub.saturate.unsigned.v8i16"] - fn llvm_i16x8_sub_saturate_u(a: i16x8, b: i16x8) -> i16x8; - - #[link_name = "llvm.wasm.anytrue.v4i32"] - fn llvm_i32x4_any_true(x: i32x4) -> i32; + fn llvm_i16x8_add_sat_u(a: simd::i16x8, b: simd::i16x8) -> simd::i16x8; + #[link_name = "llvm.wasm.sub.sat.signed.v8i16"] + fn llvm_i16x8_sub_sat_s(a: simd::i16x8, b: simd::i16x8) -> simd::i16x8; + #[link_name = "llvm.wasm.sub.sat.unsigned.v8i16"] + fn llvm_i16x8_sub_sat_u(a: simd::i16x8, b: simd::i16x8) -> simd::i16x8; + #[link_name = "llvm.wasm.avgr.unsigned.v8i16"] + fn llvm_avgr_u_i16x8(a: simd::i16x8, b: simd::i16x8) -> simd::i16x8; + + #[link_name = "llvm.wasm.extadd.pairwise.signed.v16i8"] + fn llvm_i32x4_extadd_pairwise_i16x8_s(x: simd::i16x8) -> simd::i32x4; + #[link_name = "llvm.wasm.extadd.pairwise.unsigned.v16i8"] + fn llvm_i32x4_extadd_pairwise_i16x8_u(x: simd::i16x8) -> simd::i32x4; #[link_name = "llvm.wasm.alltrue.v4i32"] - fn llvm_i32x4_all_true(x: i32x4) -> i32; + fn llvm_i32x4_all_true(x: simd::i32x4) -> i32; + #[link_name = "llvm.wasm.bitmask.v4i32"] + fn llvm_bitmask_i32x4(a: simd::i32x4) -> i32; + #[link_name = "llvm.wasm.dot"] + fn llvm_i32x4_dot_i16x8_s(a: simd::i16x8, b: simd::i16x8) -> simd::i32x4; - #[link_name = "llvm.wasm.anytrue.v2i64"] - fn llvm_i64x2_any_true(x: i64x2) -> i32; #[link_name = "llvm.wasm.alltrue.v2i64"] - fn llvm_i64x2_all_true(x: i64x2) -> i32; - + fn llvm_i64x2_all_true(x: simd::i64x2) -> i32; + #[link_name = "llvm.wasm.bitmask.v2i64"] + fn llvm_bitmask_i64x2(a: simd::i64x2) -> i32; + + #[link_name = "llvm.ceil.v4f32"] + fn llvm_f32x4_ceil(x: simd::f32x4) -> simd::f32x4; + #[link_name = "llvm.floor.v4f32"] + fn llvm_f32x4_floor(x: simd::f32x4) -> simd::f32x4; + #[link_name = "llvm.trunc.v4f32"] + fn llvm_f32x4_trunc(x: simd::f32x4) -> simd::f32x4; + #[link_name = "llvm.nearbyint.v4f32"] + fn llvm_f32x4_nearest(x: simd::f32x4) -> simd::f32x4; #[link_name = "llvm.fabs.v4f32"] - fn llvm_f32x4_abs(x: f32x4) -> f32x4; + fn llvm_f32x4_abs(x: simd::f32x4) -> simd::f32x4; #[link_name = "llvm.sqrt.v4f32"] - fn llvm_f32x4_sqrt(x: f32x4) -> f32x4; + fn llvm_f32x4_sqrt(x: simd::f32x4) -> simd::f32x4; #[link_name = "llvm.minimum.v4f32"] - fn llvm_f32x4_min(x: f32x4, y: f32x4) -> f32x4; + fn llvm_f32x4_min(x: simd::f32x4, y: simd::f32x4) -> simd::f32x4; #[link_name = "llvm.maximum.v4f32"] - fn llvm_f32x4_max(x: f32x4, y: f32x4) -> f32x4; + fn llvm_f32x4_max(x: simd::f32x4, y: simd::f32x4) -> simd::f32x4; + + #[link_name = "llvm.ceil.v2f64"] + fn llvm_f64x2_ceil(x: simd::f64x2) -> simd::f64x2; + #[link_name = "llvm.floor.v2f64"] + fn llvm_f64x2_floor(x: simd::f64x2) -> simd::f64x2; + #[link_name = "llvm.trunc.v2f64"] + fn llvm_f64x2_trunc(x: simd::f64x2) -> simd::f64x2; + #[link_name = "llvm.nearbyint.v2f64"] + fn llvm_f64x2_nearest(x: simd::f64x2) -> simd::f64x2; #[link_name = "llvm.fabs.v2f64"] - fn llvm_f64x2_abs(x: f64x2) -> f64x2; + fn llvm_f64x2_abs(x: simd::f64x2) -> simd::f64x2; #[link_name = "llvm.sqrt.v2f64"] - fn llvm_f64x2_sqrt(x: f64x2) -> f64x2; + fn llvm_f64x2_sqrt(x: simd::f64x2) -> simd::f64x2; #[link_name = "llvm.minimum.v2f64"] - fn llvm_f64x2_min(x: f64x2, y: f64x2) -> f64x2; + fn llvm_f64x2_min(x: simd::f64x2, y: simd::f64x2) -> simd::f64x2; #[link_name = "llvm.maximum.v2f64"] - fn llvm_f64x2_max(x: f64x2, y: f64x2) -> f64x2; + fn llvm_f64x2_max(x: simd::f64x2, y: simd::f64x2) -> simd::f64x2; - #[link_name = "llvm.wasm.bitselect.v16i8"] - fn llvm_bitselect(a: i8x16, b: i8x16, c: i8x16) -> i8x16; + #[link_name = "llvm.fptosi.sat.v4i32.v4f32"] + fn llvm_i32x4_trunc_sat_f32x4_s(x: simd::f32x4) -> simd::i32x4; + #[link_name = "llvm.fptoui.sat.v4i32.v4f32"] + fn llvm_i32x4_trunc_sat_f32x4_u(x: simd::f32x4) -> simd::i32x4; + #[link_name = "llvm.fptosi.sat.v2i32.v2f64"] + fn llvm_i32x2_trunc_sat_f64x2_s(x: simd::f64x2) -> simd::i32x2; + #[link_name = "llvm.fptoui.sat.v2i32.v2f64"] + fn llvm_i32x2_trunc_sat_f64x2_u(x: simd::f64x2) -> simd::i32x2; +} + +#[repr(packed)] +#[derive(Copy)] +struct Unaligned(T); + +impl Clone for Unaligned { + fn clone(&self) -> Unaligned { + *self + } } /// Loads a `v128` vector from the given heap address. +/// +/// This intrinsic will emit a load with an alignment of 1. While this is +/// provided for completeness it is not strictly necessary, you can also load +/// the pointer directly: +/// +/// ```rust,ignore +/// let a: &v128 = ...; +/// let value = unsafe { v128_load(a) }; +/// // .. is the same as .. +/// let value = *a; +/// ``` +/// +/// The alignment of the load can be configured by doing a manual load without +/// this intrinsic. +/// +/// # Unsafety +/// +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to load 16 bytes from. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned load. #[inline] #[cfg_attr(test, assert_instr(v128.load))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub unsafe fn v128_load(m: *const v128) -> v128 { - ptr::read(m) + (*(m as *const Unaligned)).0 } -/// Stores a `v128` vector to the given heap address. +/// Load eight 8-bit integers and sign extend each one to a 16-bit lane +/// +/// # Unsafety +/// +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to load 8 bytes from. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned load. #[inline] -#[cfg_attr(test, assert_instr(v128.store))] -pub unsafe fn v128_store(m: *mut v128, a: v128) { - ptr::write(m, a) -} - -/// Materializes a constant SIMD value from the immediate operands. -/// -/// The `v128.const` instruction is encoded with 16 immediate bytes -/// `imm` which provide the bits of the vector directly. -#[inline] -#[cfg(not(only_node_compatible_functions))] -#[rustc_args_required_const(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)] -#[cfg_attr(test, assert_instr( - v128.const, - a0 = 0, - a1 = 1, - a2 = 2, - a3 = 3, - a4 = 4, - a5 = 5, - a6 = 6, - a7 = 7, - a8 = 8, - a9 = 9, - a10 = 10, - a11 = 11, - a12 = 12, - a13 = 13, - a14 = 14, - a15 = 15, -))] -pub const fn v128_const( - a0: u8, - a1: u8, - a2: u8, - a3: u8, - a4: u8, - a5: u8, - a6: u8, - a7: u8, - a8: u8, - a9: u8, - a10: u8, - a11: u8, - a12: u8, - a13: u8, - a14: u8, - a15: u8, -) -> v128 { - union U { - imm: [u8; 16], - vec: v128, - } - unsafe { - U { - imm: [ - a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, - ], - } - .vec - } +#[cfg_attr(test, assert_instr(v128.load8x8_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load8x8_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn i16x8_load_extend_i8x8(m: *const i8) -> v128 { + let m = *(m as *const Unaligned); + simd_cast::<_, simd::i16x8>(m.0).v128() } -/// Creates a vector with identical lanes. +/// Load eight 8-bit integers and zero extend each one to a 16-bit lane /// -/// Constructs a vector with `x` replicated to all 16 lanes. +/// # Unsafety +/// +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to load 8 bytes from. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned load. #[inline] -#[cfg_attr(test, assert_instr(i8x16.splat))] -pub fn i8x16_splat(a: i8) -> v128 { - unsafe { transmute(i8x16::splat(a)) } +#[cfg_attr(test, assert_instr(v128.load8x8_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load8x8_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn i16x8_load_extend_u8x8(m: *const u8) -> v128 { + let m = *(m as *const Unaligned); + simd_cast::<_, simd::u16x8>(m.0).v128() } -/// Extracts a lane from a 128-bit vector interpreted as 16 packed i8 numbers. -/// -/// Extracts the scalar value of lane specified in the immediate mode operand -/// `imm` from `a`. +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_load_extend_u8x8 as u16x8_load_extend_u8x8; + +/// Load four 16-bit integers and sign extend each one to a 32-bit lane /// /// # Unsafety /// -/// This function has undefined behavior if `imm` is greater than or equal to -/// 16. +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to load 8 bytes from. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned load. #[inline] -#[rustc_args_required_const(1)] -pub unsafe fn i8x16_extract_lane(a: v128, imm: usize) -> i8 { - #[cfg(test)] - #[assert_instr(i8x16.extract_lane_s)] - fn extract_lane_s(a: v128) -> i32 { - unsafe { i8x16_extract_lane(a, 0) as i32 } - } - #[cfg(test)] - #[cfg(not(only_node_compatible_functions))] - #[assert_instr(i8x16.extract_lane_u)] - fn extract_lane_u(a: v128) -> u32 { - unsafe { i8x16_extract_lane(a, 0) as u32 } - } - simd_extract(a.as_i8x16(), imm as u32) +#[cfg_attr(test, assert_instr(v128.load16x4_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load16x4_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn i32x4_load_extend_i16x4(m: *const i16) -> v128 { + let m = *(m as *const Unaligned); + simd_cast::<_, simd::i32x4>(m.0).v128() } -/// Replaces a lane from a 128-bit vector interpreted as 16 packed i8 numbers. -/// -/// Replaces the scalar value of lane specified in the immediate mode operand -/// `imm` with `a`. +/// Load four 16-bit integers and zero extend each one to a 32-bit lane /// /// # Unsafety /// -/// This function has undefined behavior if `imm` is greater than or equal to -/// 16. +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to load 8 bytes from. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned load. #[inline] -#[cfg_attr(test, assert_instr(i8x16.replace_lane, imm = 0))] -#[rustc_args_required_const(1)] -pub unsafe fn i8x16_replace_lane(a: v128, imm: usize, val: i8) -> v128 { - transmute(simd_insert(a.as_i8x16(), imm as u32, val)) +#[cfg_attr(test, assert_instr(v128.load16x4_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load16x4_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn i32x4_load_extend_u16x4(m: *const u16) -> v128 { + let m = *(m as *const Unaligned); + simd_cast::<_, simd::u32x4>(m.0).v128() } -/// Creates a vector with identical lanes. +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_load_extend_u16x4 as u32x4_load_extend_u16x4; + +/// Load two 32-bit integers and sign extend each one to a 64-bit lane /// -/// Construct a vector with `x` replicated to all 8 lanes. +/// # Unsafety +/// +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to load 8 bytes from. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned load. #[inline] -#[cfg_attr(test, assert_instr(i16x8.splat))] -pub fn i16x8_splat(a: i16) -> v128 { - unsafe { transmute(i16x8::splat(a)) } +#[cfg_attr(test, assert_instr(v128.load32x2_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load32x2_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn i64x2_load_extend_i32x2(m: *const i32) -> v128 { + let m = *(m as *const Unaligned); + simd_cast::<_, simd::i64x2>(m.0).v128() } -/// Extracts a lane from a 128-bit vector interpreted as 8 packed i16 numbers. -/// -/// Extracts a the scalar value of lane specified in the immediate mode operand -/// `imm` from `a`. +/// Load two 32-bit integers and zero extend each one to a 64-bit lane /// /// # Unsafety /// -/// This function has undefined behavior if `imm` is greater than or equal to -/// 8. +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to load 8 bytes from. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned load. #[inline] -#[rustc_args_required_const(1)] -pub unsafe fn i16x8_extract_lane(a: v128, imm: usize) -> i16 { - #[cfg(test)] - #[assert_instr(i16x8.extract_lane_s)] - fn extract_lane_s(a: v128) -> i32 { - unsafe { i16x8_extract_lane(a, 0) as i32 } - } - #[cfg(test)] - #[cfg(not(only_node_compatible_functions))] - #[assert_instr(i16x8.extract_lane_u)] - fn extract_lane_u(a: v128) -> u32 { - unsafe { i16x8_extract_lane(a, 0) as u32 } - } - simd_extract(a.as_i16x8(), imm as u32) +#[cfg_attr(test, assert_instr(v128.load32x2_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load32x2_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn i64x2_load_extend_u32x2(m: *const u32) -> v128 { + let m = *(m as *const Unaligned); + simd_cast::<_, simd::u64x2>(m.0).v128() } -/// Replaces a lane from a 128-bit vector interpreted as 8 packed i16 numbers. +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_load_extend_u32x2 as u64x2_load_extend_u32x2; + +/// Load a single element and splat to all lanes of a v128 vector. /// -/// Replaces the scalar value of lane specified in the immediate mode operand -/// `imm` with `a`. +/// While this intrinsic is provided for completeness it can also be replaced +/// with `u8x16_splat(*m)` and it should generate equivalent code (and also not +/// require `unsafe`). /// /// # Unsafety /// -/// This function has undefined behavior if `imm` is greater than or equal to -/// 8. +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to load 1 byte from. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned load. #[inline] -#[cfg_attr(test, assert_instr(i16x8.replace_lane, imm = 0))] -#[rustc_args_required_const(1)] -pub unsafe fn i16x8_replace_lane(a: v128, imm: usize, val: i16) -> v128 { - transmute(simd_insert(a.as_i16x8(), imm as u32, val)) +#[cfg_attr(test, assert_instr(v128.load8_splat))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load8_splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_load8_splat(m: *const u8) -> v128 { + u8x16_splat(*m) } -/// Creates a vector with identical lanes. +/// Load a single element and splat to all lanes of a v128 vector. /// -/// Constructs a vector with `x` replicated to all 4 lanes. +/// While this intrinsic is provided for completeness it can also be replaced +/// with `u16x8_splat(*m)` and it should generate equivalent code (and also not +/// require `unsafe`). +/// +/// # Unsafety +/// +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to load 2 bytes from. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned load. #[inline] -#[cfg_attr(test, assert_instr(i32x4.splat))] -pub fn i32x4_splat(a: i32) -> v128 { - unsafe { transmute(i32x4::splat(a)) } +#[cfg_attr(test, assert_instr(v128.load16_splat))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load16_splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_load16_splat(m: *const u16) -> v128 { + u16x8_splat(ptr::read_unaligned(m)) } -/// Extracts a lane from a 128-bit vector interpreted as 4 packed i32 numbers. +/// Load a single element and splat to all lanes of a v128 vector. /// -/// Extracts the scalar value of lane specified in the immediate mode operand -/// `imm` from `a`. +/// While this intrinsic is provided for completeness it can also be replaced +/// with `u32x4_splat(*m)` and it should generate equivalent code (and also not +/// require `unsafe`). /// /// # Unsafety /// -/// This function has undefined behavior if `imm` is greater than or equal to -/// 4. +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to load 4 bytes from. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned load. #[inline] -#[cfg_attr(test, assert_instr(i32x4.extract_lane, imm = 0))] -#[rustc_args_required_const(1)] -pub unsafe fn i32x4_extract_lane(a: v128, imm: usize) -> i32 { - simd_extract(a.as_i32x4(), imm as u32) +#[cfg_attr(test, assert_instr(v128.load32_splat))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load32_splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_load32_splat(m: *const u32) -> v128 { + u32x4_splat(ptr::read_unaligned(m)) } -/// Replaces a lane from a 128-bit vector interpreted as 4 packed i32 numbers. +/// Load a single element and splat to all lanes of a v128 vector. /// -/// Replaces the scalar value of lane specified in the immediate mode operand -/// `imm` with `a`. +/// While this intrinsic is provided for completeness it can also be replaced +/// with `u64x2_splat(*m)` and it should generate equivalent code (and also not +/// require `unsafe`). /// /// # Unsafety /// -/// This function has undefined behavior if `imm` is greater than or equal to -/// 4. +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to load 8 bytes from. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned load. #[inline] -#[cfg_attr(test, assert_instr(i32x4.replace_lane, imm = 0))] -#[rustc_args_required_const(1)] -pub unsafe fn i32x4_replace_lane(a: v128, imm: usize, val: i32) -> v128 { - transmute(simd_insert(a.as_i32x4(), imm as u32, val)) +#[cfg_attr(test, assert_instr(v128.load64_splat))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load64_splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_load64_splat(m: *const u64) -> v128 { + u64x2_splat(ptr::read_unaligned(m)) } -/// Creates a vector with identical lanes. +/// Load a 32-bit element into the low bits of the vector and sets all other +/// bits to zero. /// -/// Construct a vector with `x` replicated to all 2 lanes. +/// This intrinsic is provided for completeness and is equivalent to `u32x4(*m, +/// 0, 0, 0)` (which doesn't require `unsafe`). +/// +/// # Unsafety +/// +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to load 4 bytes from. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned load. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(i8x16.splat))] -pub fn i64x2_splat(a: i64) -> v128 { - unsafe { transmute(i64x2::splat(a)) } +#[cfg_attr(test, assert_instr(v128.load32_zero))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load32_zero"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_load32_zero(m: *const u32) -> v128 { + u32x4(ptr::read_unaligned(m), 0, 0, 0) } -/// Extracts a lane from a 128-bit vector interpreted as 2 packed i64 numbers. +/// Load a 64-bit element into the low bits of the vector and sets all other +/// bits to zero. /// -/// Extracts the scalar value of lane specified in the immediate mode operand -/// `imm` from `a`. +/// This intrinsic is provided for completeness and is equivalent to +/// `u64x2_replace_lane::<0>(u64x2(0, 0), *m)` (which doesn't require `unsafe`). /// /// # Unsafety /// -/// This function has undefined behavior if `imm` is greater than or equal to -/// 2. +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to load 8 bytes from. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned load. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(i64x2.extract_lane_s, imm = 0))] -#[rustc_args_required_const(1)] -pub unsafe fn i64x2_extract_lane(a: v128, imm: usize) -> i64 { - simd_extract(a.as_i64x2(), imm as u32) +#[cfg_attr(test, assert_instr(v128.load64_zero))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load64_zero"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_load64_zero(m: *const u64) -> v128 { + u64x2_replace_lane::<0>(u64x2(0, 0), ptr::read_unaligned(m)) } -/// Replaces a lane from a 128-bit vector interpreted as 2 packed i64 numbers. +/// Stores a `v128` vector to the given heap address. /// -/// Replaces the scalar value of lane specified in the immediate mode operand -/// `imm` with `a`. +/// This intrinsic will emit a store with an alignment of 1. While this is +/// provided for completeness it is not strictly necessary, you can also store +/// the pointer directly: +/// +/// ```rust,ignore +/// let a: &mut v128 = ...; +/// unsafe { v128_store(a, value) }; +/// // .. is the same as .. +/// *a = value; +/// ``` +/// +/// The alignment of the store can be configured by doing a manual store without +/// this intrinsic. /// /// # Unsafety /// -/// This function has undefined behavior if `imm` is greater than or equal to -/// 2. +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to store 16 bytes to. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned store. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(i64x2.replace_lane, imm = 0))] -#[rustc_args_required_const(1)] -pub unsafe fn i64x2_replace_lane(a: v128, imm: usize, val: i64) -> v128 { - transmute(simd_insert(a.as_i64x2(), imm as u32, val)) +#[cfg_attr(test, assert_instr(v128.store))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.store"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_store(m: *mut v128, a: v128) { + *(m as *mut Unaligned) = Unaligned(a); } -/// Creates a vector with identical lanes. +/// Loads an 8-bit value from `m` and sets lane `L` of `v` to that value. /// -/// Constructs a vector with `x` replicated to all 4 lanes. +/// This intrinsic is provided for completeness and is equivalent to +/// `u8x16_replace_lane::(v, *m)` (which doesn't require `unsafe`). +/// +/// # Unsafety +/// +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to load 1 byte from. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned load. #[inline] -#[cfg_attr(test, assert_instr(f32x4.splat))] -pub fn f32x4_splat(a: f32) -> v128 { - unsafe { transmute(f32x4::splat(a)) } +#[cfg_attr(test, assert_instr(v128.load8_lane, L = 0))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load8_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_load8_lane(v: v128, m: *const u8) -> v128 { + u8x16_replace_lane::(v, *m) } -/// Extracts a lane from a 128-bit vector interpreted as 4 packed f32 numbers. +/// Loads a 16-bit value from `m` and sets lane `L` of `v` to that value. /// -/// Extracts the scalar value of lane specified in the immediate mode operand -/// `imm` from `a`. +/// This intrinsic is provided for completeness and is equivalent to +/// `u16x8_replace_lane::(v, *m)` (which doesn't require `unsafe`). /// /// # Unsafety /// -/// This function has undefined behavior if `imm` is greater than or equal to -/// 4. +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to load 2 bytes from. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned load. #[inline] -#[cfg_attr(test, assert_instr(f32x4.extract_lane, imm = 0))] -#[rustc_args_required_const(1)] -pub unsafe fn f32x4_extract_lane(a: v128, imm: usize) -> f32 { - simd_extract(a.as_f32x4(), imm as u32) +#[cfg_attr(test, assert_instr(v128.load16_lane, L = 0))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load16_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_load16_lane(v: v128, m: *const u16) -> v128 { + u16x8_replace_lane::(v, ptr::read_unaligned(m)) } -/// Replaces a lane from a 128-bit vector interpreted as 4 packed f32 numbers. +/// Loads a 32-bit value from `m` and sets lane `L` of `v` to that value. /// -/// Replaces the scalar value of lane specified in the immediate mode operand -/// `imm` with `a`. +/// This intrinsic is provided for completeness and is equivalent to +/// `u32x4_replace_lane::(v, *m)` (which doesn't require `unsafe`). /// /// # Unsafety /// -/// This function has undefined behavior if `imm` is greater than or equal to -/// 4. +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to load 4 bytes from. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned load. #[inline] -#[cfg_attr(test, assert_instr(f32x4.replace_lane, imm = 0))] -#[rustc_args_required_const(1)] -pub unsafe fn f32x4_replace_lane(a: v128, imm: usize, val: f32) -> v128 { - transmute(simd_insert(a.as_f32x4(), imm as u32, val)) +#[cfg_attr(test, assert_instr(v128.load32_lane, L = 0))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load32_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_load32_lane(v: v128, m: *const u32) -> v128 { + u32x4_replace_lane::(v, ptr::read_unaligned(m)) } -/// Creates a vector with identical lanes. +/// Loads a 64-bit value from `m` and sets lane `L` of `v` to that value. /// -/// Constructs a vector with `x` replicated to all 2 lanes. +/// This intrinsic is provided for completeness and is equivalent to +/// `u64x2_replace_lane::(v, *m)` (which doesn't require `unsafe`). +/// +/// # Unsafety +/// +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to load 8 bytes from. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned load. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(f64x2.splat))] -pub fn f64x2_splat(a: f64) -> v128 { - unsafe { transmute(f64x2::splat(a)) } +#[cfg_attr(test, assert_instr(v128.load64_lane, L = 0))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load64_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_load64_lane(v: v128, m: *const u64) -> v128 { + u64x2_replace_lane::(v, ptr::read_unaligned(m)) } -/// Extracts lane from a 128-bit vector interpreted as 2 packed f64 numbers. +/// Stores the 8-bit value from lane `L` of `v` into `m` /// -/// Extracts the scalar value of lane specified in the immediate mode operand -/// `imm` from `a`. +/// This intrinsic is provided for completeness and is equivalent to +/// `*m = u8x16_extract_lane::(v)` (which doesn't require `unsafe`). /// /// # Unsafety /// -/// This function has undefined behavior if `imm` is greater than or equal to -/// 2. +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to store 1 byte to. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned store. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(f64x2.extract_lane_s, imm = 0))] -#[rustc_args_required_const(1)] -pub unsafe fn f64x2_extract_lane(a: v128, imm: usize) -> f64 { - simd_extract(a.as_f64x2(), imm as u32) +#[cfg_attr(test, assert_instr(v128.store8_lane, L = 0))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.store8_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_store8_lane(v: v128, m: *mut u8) { + *m = u8x16_extract_lane::(v); } -/// Replaces a lane from a 128-bit vector interpreted as 2 packed f64 numbers. +/// Stores the 16-bit value from lane `L` of `v` into `m` /// -/// Replaces the scalar value of lane specified in the immediate mode operand -/// `imm` with `a`. +/// This intrinsic is provided for completeness and is equivalent to +/// `*m = u16x8_extract_lane::(v)` (which doesn't require `unsafe`). /// /// # Unsafety /// -/// This function has undefined behavior if `imm` is greater than or equal to -/// 2. +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to store 2 bytes to. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned store. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(f64x2.replace_lane, imm = 0))] -#[rustc_args_required_const(1)] -pub unsafe fn f64x2_replace_lane(a: v128, imm: usize, val: f64) -> v128 { - transmute(simd_insert(a.as_f64x2(), imm as u32, val)) +#[cfg_attr(test, assert_instr(v128.store16_lane, L = 0))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.store16_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_store16_lane(v: v128, m: *mut u16) { + ptr::write_unaligned(m, u16x8_extract_lane::(v)) } -/// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit -/// integers. +/// Stores the 32-bit value from lane `L` of `v` into `m` /// -/// Returns a new vector where each lane is all ones if the pairwise elements -/// were equal, or all zeros if the elements were not equal. +/// This intrinsic is provided for completeness and is equivalent to +/// `*m = u32x4_extract_lane::(v)` (which doesn't require `unsafe`). +/// +/// # Unsafety +/// +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to store 4 bytes to. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned store. #[inline] -#[cfg_attr(test, assert_instr(i8x16.eq))] -pub fn i8x16_eq(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_eq::<_, i8x16>(a.as_i8x16(), b.as_i8x16())) } +#[cfg_attr(test, assert_instr(v128.store32_lane, L = 0))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.store32_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_store32_lane(v: v128, m: *mut u32) { + ptr::write_unaligned(m, u32x4_extract_lane::(v)) } -/// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit -/// integers. +/// Stores the 64-bit value from lane `L` of `v` into `m` +/// +/// This intrinsic is provided for completeness and is equivalent to +/// `*m = u64x2_extract_lane::(v)` (which doesn't require `unsafe`). /// -/// Returns a new vector where each lane is all ones if the pairwise elements -/// were not equal, or all zeros if the elements were equal. +/// # Unsafety +/// +/// This intrinsic is unsafe because it takes a raw pointer as an argument, and +/// the pointer must be valid to store 8 bytes to. Note that there is no +/// alignment requirement on this pointer since this intrinsic performs a +/// 1-aligned store. #[inline] -#[cfg_attr(test, assert_instr(i8x16.ne))] -pub fn i8x16_ne(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_ne::<_, i8x16>(a.as_i8x16(), b.as_i8x16())) } +#[cfg_attr(test, assert_instr(v128.store64_lane, L = 0))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.store64_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_store64_lane(v: v128, m: *mut u64) { + ptr::write_unaligned(m, u64x2_extract_lane::(v)) } -/// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit -/// signed integers. +/// Materializes a SIMD value from the provided operands. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is less than the pairwise right element, or all zeros otherwise. -#[inline] -#[cfg_attr(test, assert_instr(i8x16.lt_s))] -pub fn i8x16_lt_s(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_lt::<_, i8x16>(a.as_i8x16(), b.as_i8x16())) } +/// If possible this will generate a `v128.const` instruction, otherwise it may +/// be lowered to a sequence of instructions to materialize the vector value. +#[inline] +#[target_feature(enable = "simd128")] +#[cfg_attr( + test, + assert_instr( + v128.const, + a0 = 0, + a1 = 1, + a2 = 2, + a3 = 3, + a4 = 4, + a5 = 5, + a6 = 6, + a7 = 7, + a8 = 8, + a9 = 9, + a10 = 10, + a11 = 11, + a12 = 12, + a13 = 13, + a14 = 14, + a15 = 15, + ) +)] +#[doc(alias("v128.const"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +#[rustc_const_stable(feature = "wasm_simd", since = "1.54.0")] +pub const fn i8x16( + a0: i8, + a1: i8, + a2: i8, + a3: i8, + a4: i8, + a5: i8, + a6: i8, + a7: i8, + a8: i8, + a9: i8, + a10: i8, + a11: i8, + a12: i8, + a13: i8, + a14: i8, + a15: i8, +) -> v128 { + simd::i8x16( + a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, + ) + .v128() } -/// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit -/// unsigned integers. +/// Materializes a SIMD value from the provided operands. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is less than the pairwise right element, or all zeros otherwise. -#[inline] -#[cfg_attr(test, assert_instr(i8x16.lt_u))] -pub fn i8x16_lt_u(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_lt::<_, i8x16>(a.as_u8x16(), b.as_u8x16())) } +/// If possible this will generate a `v128.const` instruction, otherwise it may +/// be lowered to a sequence of instructions to materialize the vector value. +#[inline] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.const"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +#[rustc_const_stable(feature = "wasm_simd", since = "1.54.0")] +pub const fn u8x16( + a0: u8, + a1: u8, + a2: u8, + a3: u8, + a4: u8, + a5: u8, + a6: u8, + a7: u8, + a8: u8, + a9: u8, + a10: u8, + a11: u8, + a12: u8, + a13: u8, + a14: u8, + a15: u8, +) -> v128 { + simd::u8x16( + a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, + ) + .v128() } -/// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit -/// signed integers. +/// Materializes a SIMD value from the provided operands. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is greater than the pairwise right element, or all zeros otherwise. +/// If possible this will generate a `v128.const` instruction, otherwise it may +/// be lowered to a sequence of instructions to materialize the vector value. +#[inline] +#[target_feature(enable = "simd128")] +#[cfg_attr( + test, + assert_instr( + v128.const, + a0 = 0, + a1 = 1, + a2 = 2, + a3 = 3, + a4 = 4, + a5 = 5, + a6 = 6, + a7 = 7, + ) +)] +#[doc(alias("v128.const"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +#[rustc_const_stable(feature = "wasm_simd", since = "1.54.0")] +pub const fn i16x8(a0: i16, a1: i16, a2: i16, a3: i16, a4: i16, a5: i16, a6: i16, a7: i16) -> v128 { + simd::i16x8(a0, a1, a2, a3, a4, a5, a6, a7).v128() +} + +/// Materializes a SIMD value from the provided operands. +/// +/// If possible this will generate a `v128.const` instruction, otherwise it may +/// be lowered to a sequence of instructions to materialize the vector value. #[inline] -#[cfg_attr(test, assert_instr(i8x16.gt_s))] -pub fn i8x16_gt_s(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_gt::<_, i8x16>(a.as_i8x16(), b.as_i8x16())) } +#[target_feature(enable = "simd128")] +#[doc(alias("v128.const"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +#[rustc_const_stable(feature = "wasm_simd", since = "1.54.0")] +pub const fn u16x8(a0: u16, a1: u16, a2: u16, a3: u16, a4: u16, a5: u16, a6: u16, a7: u16) -> v128 { + simd::u16x8(a0, a1, a2, a3, a4, a5, a6, a7).v128() } -/// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit -/// unsigned integers. +/// Materializes a SIMD value from the provided operands. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is greater than the pairwise right element, or all zeros otherwise. +/// If possible this will generate a `v128.const` instruction, otherwise it may +/// be lowered to a sequence of instructions to materialize the vector value. #[inline] -#[cfg_attr(test, assert_instr(i8x16.gt_u))] -pub fn i8x16_gt_u(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_gt::<_, i8x16>(a.as_u8x16(), b.as_u8x16())) } +#[target_feature(enable = "simd128")] +#[cfg_attr(test, assert_instr(v128.const, a0 = 0, a1 = 1, a2 = 2, a3 = 3))] +#[doc(alias("v128.const"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +#[rustc_const_stable(feature = "wasm_simd", since = "1.54.0")] +pub const fn i32x4(a0: i32, a1: i32, a2: i32, a3: i32) -> v128 { + simd::i32x4(a0, a1, a2, a3).v128() } -/// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit -/// signed integers. +/// Materializes a SIMD value from the provided operands. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is less than the pairwise right element, or all zeros otherwise. +/// If possible this will generate a `v128.const` instruction, otherwise it may +/// be lowered to a sequence of instructions to materialize the vector value. #[inline] -#[cfg_attr(test, assert_instr(i8x16.le_s))] -pub fn i8x16_le_s(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_le::<_, i8x16>(a.as_i8x16(), b.as_i8x16())) } +#[target_feature(enable = "simd128")] +#[doc(alias("v128.const"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +#[rustc_const_stable(feature = "wasm_simd", since = "1.54.0")] +pub const fn u32x4(a0: u32, a1: u32, a2: u32, a3: u32) -> v128 { + simd::u32x4(a0, a1, a2, a3).v128() } -/// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit -/// unsigned integers. +/// Materializes a SIMD value from the provided operands. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is less than the pairwise right element, or all zeros otherwise. +/// If possible this will generate a `v128.const` instruction, otherwise it may +/// be lowered to a sequence of instructions to materialize the vector value. #[inline] -#[cfg_attr(test, assert_instr(i8x16.le_u))] -pub fn i8x16_le_u(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_le::<_, i8x16>(a.as_u8x16(), b.as_u8x16())) } +#[target_feature(enable = "simd128")] +#[cfg_attr(test, assert_instr(v128.const, a0 = 1, a1 = 2))] +#[doc(alias("v128.const"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +#[rustc_const_stable(feature = "wasm_simd", since = "1.54.0")] +pub const fn i64x2(a0: i64, a1: i64) -> v128 { + simd::i64x2(a0, a1).v128() } -/// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit -/// signed integers. +/// Materializes a SIMD value from the provided operands. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is greater than the pairwise right element, or all zeros otherwise. +/// If possible this will generate a `v128.const` instruction, otherwise it may +/// be lowered to a sequence of instructions to materialize the vector value. #[inline] -#[cfg_attr(test, assert_instr(i8x16.ge_s))] -pub fn i8x16_ge_s(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_ge::<_, i8x16>(a.as_i8x16(), b.as_i8x16())) } +#[target_feature(enable = "simd128")] +#[doc(alias("v128.const"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +#[rustc_const_stable(feature = "wasm_simd", since = "1.54.0")] +pub const fn u64x2(a0: u64, a1: u64) -> v128 { + simd::u64x2(a0, a1).v128() } -/// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit -/// unsigned integers. +/// Materializes a SIMD value from the provided operands. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is greater than the pairwise right element, or all zeros otherwise. +/// If possible this will generate a `v128.const` instruction, otherwise it may +/// be lowered to a sequence of instructions to materialize the vector value. #[inline] -#[cfg_attr(test, assert_instr(i8x16.ge_u))] -pub fn i8x16_ge_u(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_ge::<_, i8x16>(a.as_u8x16(), b.as_u8x16())) } +#[target_feature(enable = "simd128")] +#[cfg_attr(test, assert_instr(v128.const, a0 = 0.0, a1 = 1.0, a2 = 2.0, a3 = 3.0))] +#[doc(alias("v128.const"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +#[rustc_const_stable(feature = "wasm_simd_const", since = "1.56.0")] +pub const fn f32x4(a0: f32, a1: f32, a2: f32, a3: f32) -> v128 { + simd::f32x4(a0, a1, a2, a3).v128() } -/// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit -/// integers. +/// Materializes a SIMD value from the provided operands. /// -/// Returns a new vector where each lane is all ones if the pairwise elements -/// were equal, or all zeros if the elements were not equal. +/// If possible this will generate a `v128.const` instruction, otherwise it may +/// be lowered to a sequence of instructions to materialize the vector value. #[inline] -#[cfg_attr(test, assert_instr(i16x8.eq))] -pub fn i16x8_eq(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_eq::<_, i16x8>(a.as_i16x8(), b.as_i16x8())) } +#[target_feature(enable = "simd128")] +#[cfg_attr(test, assert_instr(v128.const, a0 = 0.0, a1 = 1.0))] +#[doc(alias("v128.const"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +#[rustc_const_stable(feature = "wasm_simd_const", since = "1.56.0")] +pub const fn f64x2(a0: f64, a1: f64) -> v128 { + simd::f64x2(a0, a1).v128() } -/// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit -/// integers. +/// Returns a new vector with lanes selected from the lanes of the two input +/// vectors `$a` and `$b` specified in the 16 immediate operands. /// -/// Returns a new vector where each lane is all ones if the pairwise elements -/// were not equal, or all zeros if the elements were equal. -#[inline] -#[cfg_attr(test, assert_instr(i16x8.ne))] -pub fn i16x8_ne(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_ne::<_, i16x8>(a.as_i16x8(), b.as_i16x8())) } +/// The `$a` and `$b` expressions must have type `v128`, and this function +/// generates a wasm instruction that is encoded with 16 bytes providing the +/// indices of the elements to return. The indices `i` in range [0, 15] select +/// the `i`-th element of `a`. The indices in range [16, 31] select the `i - +/// 16`-th element of `b`. +/// +/// Note that this is a macro due to the codegen requirements of all of the +/// index expressions `$i*` must be constant. A compiler error will be +/// generated if any of the expressions are not constant. +/// +/// All indexes `$i*` must have the type `u32`. +#[inline] +#[cfg_attr(test, + assert_instr( + i8x16.shuffle, + I0 = 0, + I1 = 2, + I2 = 4, + I3 = 6, + I4 = 8, + I5 = 10, + I6 = 12, + I7 = 14, + I8 = 16, + I9 = 18, + I10 = 20, + I11 = 22, + I12 = 24, + I13 = 26, + I14 = 28, + I15 = 30, + ) +)] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.shuffle"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_shuffle< + const I0: usize, + const I1: usize, + const I2: usize, + const I3: usize, + const I4: usize, + const I5: usize, + const I6: usize, + const I7: usize, + const I8: usize, + const I9: usize, + const I10: usize, + const I11: usize, + const I12: usize, + const I13: usize, + const I14: usize, + const I15: usize, +>( + a: v128, + b: v128, +) -> v128 { + static_assert!(I0: usize where I0 < 32); + static_assert!(I1: usize where I1 < 32); + static_assert!(I2: usize where I2 < 32); + static_assert!(I3: usize where I3 < 32); + static_assert!(I4: usize where I4 < 32); + static_assert!(I5: usize where I5 < 32); + static_assert!(I6: usize where I6 < 32); + static_assert!(I7: usize where I7 < 32); + static_assert!(I8: usize where I8 < 32); + static_assert!(I9: usize where I9 < 32); + static_assert!(I10: usize where I10 < 32); + static_assert!(I11: usize where I11 < 32); + static_assert!(I12: usize where I12 < 32); + static_assert!(I13: usize where I13 < 32); + static_assert!(I14: usize where I14 < 32); + static_assert!(I15: usize where I15 < 32); + let shuf: simd::u8x16 = unsafe { + simd_shuffle16!( + a.as_u8x16(), + b.as_u8x16(), + < + const I0: usize, + const I1: usize, + const I2: usize, + const I3: usize, + const I4: usize, + const I5: usize, + const I6: usize, + const I7: usize, + const I8: usize, + const I9: usize, + const I10: usize, + const I11: usize, + const I12: usize, + const I13: usize, + const I14: usize, + const I15: usize, + > [ + I0 as u32, I1 as u32, I2 as u32, I3 as u32, I4 as u32, I5 as u32, I6 as u32, I7 as u32, + I8 as u32, I9 as u32, I10 as u32, I11 as u32, I12 as u32, I13 as u32, I14 as u32, + I15 as u32, + ], + ) + }; + shuf.v128() } -/// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit -/// signed integers. +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i8x16_shuffle as u8x16_shuffle; + +/// Same as [`i8x16_shuffle`], except operates as if the inputs were eight +/// 16-bit integers, only taking 8 indices to shuffle. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is less than the pairwise right element, or all zeros otherwise. -#[inline] -#[cfg_attr(test, assert_instr(i16x8.lt_s))] -pub fn i16x8_lt_s(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_lt::<_, i16x8>(a.as_i16x8(), b.as_i16x8())) } +/// Indices in the range [0, 7] select from `a` while [8, 15] select from `b`. +/// Note that this will generate the `i8x16.shuffle` instruction, since there +/// is no native `i16x8.shuffle` instruction (there is no need for one since +/// `i8x16.shuffle` suffices). +#[inline] +#[cfg_attr(test, + assert_instr( + i8x16.shuffle, + I0 = 0, + I1 = 2, + I2 = 4, + I3 = 6, + I4 = 8, + I5 = 10, + I6 = 12, + I7 = 14, + ) +)] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.shuffle"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_shuffle< + const I0: usize, + const I1: usize, + const I2: usize, + const I3: usize, + const I4: usize, + const I5: usize, + const I6: usize, + const I7: usize, +>( + a: v128, + b: v128, +) -> v128 { + static_assert!(I0: usize where I0 < 16); + static_assert!(I1: usize where I1 < 16); + static_assert!(I2: usize where I2 < 16); + static_assert!(I3: usize where I3 < 16); + static_assert!(I4: usize where I4 < 16); + static_assert!(I5: usize where I5 < 16); + static_assert!(I6: usize where I6 < 16); + static_assert!(I7: usize where I7 < 16); + let shuf: simd::u16x8 = unsafe { + simd_shuffle8!( + a.as_u16x8(), + b.as_u16x8(), + < + const I0: usize, + const I1: usize, + const I2: usize, + const I3: usize, + const I4: usize, + const I5: usize, + const I6: usize, + const I7: usize, + > [ + I0 as u32, I1 as u32, I2 as u32, I3 as u32, I4 as u32, I5 as u32, I6 as u32, I7 as u32, + ], + ) + }; + shuf.v128() } -/// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit -/// unsigned integers. +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_shuffle as u16x8_shuffle; + +/// Same as [`i8x16_shuffle`], except operates as if the inputs were four +/// 32-bit integers, only taking 4 indices to shuffle. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is less than the pairwise right element, or all zeros otherwise. -#[inline] -#[cfg_attr(test, assert_instr(i16x8.lt_u))] -pub fn i16x8_lt_u(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_lt::<_, i16x8>(a.as_u16x8(), b.as_u16x8())) } -} +/// Indices in the range [0, 3] select from `a` while [4, 7] select from `b`. +/// Note that this will generate the `i8x16.shuffle` instruction, since there +/// is no native `i32x4.shuffle` instruction (there is no need for one since +/// `i8x16.shuffle` suffices). +#[inline] +#[cfg_attr(test, assert_instr(i8x16.shuffle, I0 = 0, I1 = 2, I2 = 4, I3 = 6))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.shuffle"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_shuffle( + a: v128, + b: v128, +) -> v128 { + static_assert!(I0: usize where I0 < 8); + static_assert!(I1: usize where I1 < 8); + static_assert!(I2: usize where I2 < 8); + static_assert!(I3: usize where I3 < 8); + let shuf: simd::u32x4 = unsafe { + simd_shuffle4!( + a.as_u32x4(), + b.as_u32x4(), + [I0 as u32, I1 as u32, I2 as u32, I3 as u32], + ) + }; + shuf.v128() +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_shuffle as u32x4_shuffle; + +/// Same as [`i8x16_shuffle`], except operates as if the inputs were two +/// 64-bit integers, only taking 2 indices to shuffle. +/// +/// Indices in the range [0, 1] select from `a` while [2, 3] select from `b`. +/// Note that this will generate the `v8x16.shuffle` instruction, since there +/// is no native `i64x2.shuffle` instruction (there is no need for one since +/// `i8x16.shuffle` suffices). +#[inline] +#[cfg_attr(test, assert_instr(i8x16.shuffle, I0 = 0, I1 = 2))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.shuffle"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_shuffle(a: v128, b: v128) -> v128 { + static_assert!(I0: usize where I0 < 4); + static_assert!(I1: usize where I1 < 4); + let shuf: simd::u64x2 = unsafe { + simd_shuffle2!( + a.as_u64x2(), + b.as_u64x2(), + [I0 as u32, I1 as u32], + ) + }; + shuf.v128() +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_shuffle as u64x2_shuffle; -/// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit -/// signed integers. +/// Extracts a lane from a 128-bit vector interpreted as 16 packed i8 numbers. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is greater than the pairwise right element, or all zeros otherwise. +/// Extracts the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(i16x8.gt_s))] -pub fn i16x8_gt_s(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_gt::<_, i16x8>(a.as_i16x8(), b.as_i16x8())) } +#[cfg_attr(test, assert_instr(i8x16.extract_lane_s, N = 3))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.extract_lane_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_extract_lane(a: v128) -> i8 { + static_assert!(N: usize where N < 16); + unsafe { simd_extract(a.as_i8x16(), N as u32) } } -/// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit -/// unsigned integers. +/// Extracts a lane from a 128-bit vector interpreted as 16 packed u8 numbers. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is greater than the pairwise right element, or all zeros otherwise. +/// Extracts the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(i16x8.gt_u))] -pub fn i16x8_gt_u(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_gt::<_, i16x8>(a.as_u16x8(), b.as_u16x8())) } +#[cfg_attr(test, assert_instr(i8x16.extract_lane_u, N = 3))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.extract_lane_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_extract_lane(a: v128) -> u8 { + static_assert!(N: usize where N < 16); + unsafe { simd_extract(a.as_u8x16(), N as u32) } } -/// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit -/// signed integers. +/// Replaces a lane from a 128-bit vector interpreted as 16 packed i8 numbers. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is less than the pairwise right element, or all zeros otherwise. +/// Replaces the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(i16x8.le_s))] -pub fn i16x8_le_s(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_le::<_, i16x8>(a.as_i16x8(), b.as_i16x8())) } +#[cfg_attr(test, assert_instr(i8x16.replace_lane, N = 2))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.replace_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_replace_lane(a: v128, val: i8) -> v128 { + static_assert!(N: usize where N < 16); + unsafe { simd_insert(a.as_i8x16(), N as u32, val).v128() } } -/// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit -/// unsigned integers. +/// Replaces a lane from a 128-bit vector interpreted as 16 packed u8 numbers. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is less than the pairwise right element, or all zeros otherwise. +/// Replaces the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(i16x8.le_u))] -pub fn i16x8_le_u(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_le::<_, i16x8>(a.as_u16x8(), b.as_u16x8())) } +#[cfg_attr(test, assert_instr(i8x16.replace_lane, N = 2))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.replace_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_replace_lane(a: v128, val: u8) -> v128 { + static_assert!(N: usize where N < 16); + unsafe { simd_insert(a.as_u8x16(), N as u32, val).v128() } } -/// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit -/// signed integers. +/// Extracts a lane from a 128-bit vector interpreted as 8 packed i16 numbers. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is greater than the pairwise right element, or all zeros otherwise. +/// Extracts a the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(i16x8.ge_s))] -pub fn i16x8_ge_s(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_ge::<_, i16x8>(a.as_i16x8(), b.as_i16x8())) } +#[cfg_attr(test, assert_instr(i16x8.extract_lane_s, N = 2))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extract_lane_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extract_lane(a: v128) -> i16 { + static_assert!(N: usize where N < 8); + unsafe { simd_extract(a.as_i16x8(), N as u32) } } -/// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit -/// unsigned integers. +/// Extracts a lane from a 128-bit vector interpreted as 8 packed u16 numbers. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is greater than the pairwise right element, or all zeros otherwise. +/// Extracts a the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(i16x8.ge_u))] -pub fn i16x8_ge_u(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_ge::<_, i16x8>(a.as_u16x8(), b.as_u16x8())) } +#[cfg_attr(test, assert_instr(i16x8.extract_lane_u, N = 2))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extract_lane_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_extract_lane(a: v128) -> u16 { + static_assert!(N: usize where N < 8); + unsafe { simd_extract(a.as_u16x8(), N as u32) } } -/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit -/// integers. +/// Replaces a lane from a 128-bit vector interpreted as 8 packed i16 numbers. /// -/// Returns a new vector where each lane is all ones if the pairwise elements -/// were equal, or all zeros if the elements were not equal. +/// Replaces the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(i32x4.eq))] -pub fn i32x4_eq(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_eq::<_, i32x4>(a.as_i32x4(), b.as_i32x4())) } +#[cfg_attr(test, assert_instr(i16x8.replace_lane, N = 2))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.replace_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_replace_lane(a: v128, val: i16) -> v128 { + static_assert!(N: usize where N < 8); + unsafe { simd_insert(a.as_i16x8(), N as u32, val).v128() } } -/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit -/// integers. +/// Replaces a lane from a 128-bit vector interpreted as 8 packed u16 numbers. /// -/// Returns a new vector where each lane is all ones if the pairwise elements -/// were not equal, or all zeros if the elements were equal. +/// Replaces the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(i32x4.ne))] -pub fn i32x4_ne(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_ne::<_, i32x4>(a.as_i32x4(), b.as_i32x4())) } +#[cfg_attr(test, assert_instr(i16x8.replace_lane, N = 2))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.replace_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_replace_lane(a: v128, val: u16) -> v128 { + static_assert!(N: usize where N < 8); + unsafe { simd_insert(a.as_u16x8(), N as u32, val).v128() } } -/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit -/// signed integers. +/// Extracts a lane from a 128-bit vector interpreted as 4 packed i32 numbers. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is less than the pairwise right element, or all zeros otherwise. +/// Extracts the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(i32x4.lt_s))] -pub fn i32x4_lt_s(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_lt::<_, i32x4>(a.as_i32x4(), b.as_i32x4())) } +#[cfg_attr(test, assert_instr(i32x4.extract_lane, N = 2))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.extract_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extract_lane(a: v128) -> i32 { + static_assert!(N: usize where N < 4); + unsafe { simd_extract(a.as_i32x4(), N as u32) } } -/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit -/// unsigned integers. +/// Extracts a lane from a 128-bit vector interpreted as 4 packed u32 numbers. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is less than the pairwise right element, or all zeros otherwise. +/// Extracts the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(i32x4.lt_u))] -pub fn i32x4_lt_u(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_lt::<_, i32x4>(a.as_u32x4(), b.as_u32x4())) } +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.extract_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_extract_lane(a: v128) -> u32 { + i32x4_extract_lane::(a) as u32 } -/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit -/// signed integers. +/// Replaces a lane from a 128-bit vector interpreted as 4 packed i32 numbers. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is greater than the pairwise right element, or all zeros otherwise. +/// Replaces the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(i32x4.gt_s))] -pub fn i32x4_gt_s(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_gt::<_, i32x4>(a.as_i32x4(), b.as_i32x4())) } +#[cfg_attr(test, assert_instr(i32x4.replace_lane, N = 2))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.replace_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_replace_lane(a: v128, val: i32) -> v128 { + static_assert!(N: usize where N < 4); + unsafe { simd_insert(a.as_i32x4(), N as u32, val).v128() } } -/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit -/// unsigned integers. +/// Replaces a lane from a 128-bit vector interpreted as 4 packed u32 numbers. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is greater than the pairwise right element, or all zeros otherwise. +/// Replaces the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(i32x4.gt_u))] -pub fn i32x4_gt_u(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_gt::<_, i32x4>(a.as_u32x4(), b.as_u32x4())) } +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.replace_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_replace_lane(a: v128, val: u32) -> v128 { + i32x4_replace_lane::(a, val as i32) } -/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit -/// signed integers. +/// Extracts a lane from a 128-bit vector interpreted as 2 packed i64 numbers. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is less than the pairwise right element, or all zeros otherwise. +/// Extracts the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(i32x4.le_s))] -pub fn i32x4_le_s(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_le::<_, i32x4>(a.as_i32x4(), b.as_i32x4())) } +#[cfg_attr(test, assert_instr(i64x2.extract_lane, N = 1))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.extract_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_extract_lane(a: v128) -> i64 { + static_assert!(N: usize where N < 2); + unsafe { simd_extract(a.as_i64x2(), N as u32) } } -/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit -/// unsigned integers. +/// Extracts a lane from a 128-bit vector interpreted as 2 packed u64 numbers. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is less than the pairwise right element, or all zeros otherwise. +/// Extracts the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(i32x4.le_u))] -pub fn i32x4_le_u(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_le::<_, i32x4>(a.as_u32x4(), b.as_u32x4())) } +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.extract_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u64x2_extract_lane(a: v128) -> u64 { + i64x2_extract_lane::(a) as u64 } -/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit -/// signed integers. +/// Replaces a lane from a 128-bit vector interpreted as 2 packed i64 numbers. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is greater than the pairwise right element, or all zeros otherwise. +/// Replaces the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(i32x4.ge_s))] -pub fn i32x4_ge_s(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_ge::<_, i32x4>(a.as_i32x4(), b.as_i32x4())) } +#[cfg_attr(test, assert_instr(i64x2.replace_lane, N = 0))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.replace_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_replace_lane(a: v128, val: i64) -> v128 { + static_assert!(N: usize where N < 2); + unsafe { simd_insert(a.as_i64x2(), N as u32, val).v128() } } -/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit -/// unsigned integers. +/// Replaces a lane from a 128-bit vector interpreted as 2 packed u64 numbers. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is greater than the pairwise right element, or all zeros otherwise. +/// Replaces the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(i32x4.ge_u))] -pub fn i32x4_ge_u(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_ge::<_, i32x4>(a.as_u32x4(), b.as_u32x4())) } +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.replace_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u64x2_replace_lane(a: v128, val: u64) -> v128 { + i64x2_replace_lane::(a, val as i64) } -/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit -/// floating point numbers. +/// Extracts a lane from a 128-bit vector interpreted as 4 packed f32 numbers. /// -/// Returns a new vector where each lane is all ones if the pairwise elements -/// were equal, or all zeros if the elements were not equal. +/// Extracts the scalar value of lane specified fn the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(f32x4.eq))] -pub fn f32x4_eq(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_eq::<_, i32x4>(a.as_f32x4(), b.as_f32x4())) } +#[cfg_attr(test, assert_instr(f32x4.extract_lane, N = 1))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.extract_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_extract_lane(a: v128) -> f32 { + static_assert!(N: usize where N < 4); + unsafe { simd_extract(a.as_f32x4(), N as u32) } } -/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit -/// floating point numbers. +/// Replaces a lane from a 128-bit vector interpreted as 4 packed f32 numbers. /// -/// Returns a new vector where each lane is all ones if the pairwise elements -/// were not equal, or all zeros if the elements were equal. +/// Replaces the scalar value of lane specified fn the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(f32x4.ne))] -pub fn f32x4_ne(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_ne::<_, i32x4>(a.as_f32x4(), b.as_f32x4())) } +#[cfg_attr(test, assert_instr(f32x4.replace_lane, N = 1))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.replace_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_replace_lane(a: v128, val: f32) -> v128 { + static_assert!(N: usize where N < 4); + unsafe { simd_insert(a.as_f32x4(), N as u32, val).v128() } } -/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit -/// floating point numbers. +/// Extracts a lane from a 128-bit vector interpreted as 2 packed f64 numbers. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is less than the pairwise right element, or all zeros otherwise. +/// Extracts the scalar value of lane specified fn the immediate mode operand +/// `N` from `a`. If `N` fs out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(f32x4.lt))] -pub fn f32x4_lt(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_lt::<_, i32x4>(a.as_f32x4(), b.as_f32x4())) } +#[cfg_attr(test, assert_instr(f64x2.extract_lane, N = 1))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.extract_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_extract_lane(a: v128) -> f64 { + static_assert!(N: usize where N < 2); + unsafe { simd_extract(a.as_f64x2(), N as u32) } } -/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit -/// floating point numbers. +/// Replaces a lane from a 128-bit vector interpreted as 2 packed f64 numbers. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is greater than the pairwise right element, or all zeros otherwise. +/// Replaces the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(f32x4.gt))] -pub fn f32x4_gt(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_gt::<_, i32x4>(a.as_f32x4(), b.as_f32x4())) } +#[cfg_attr(test, assert_instr(f64x2.replace_lane, N = 1))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.replace_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_replace_lane(a: v128, val: f64) -> v128 { + static_assert!(N: usize where N < 2); + unsafe { simd_insert(a.as_f64x2(), N as u32, val).v128() } } -/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit -/// floating point numbers. +/// Returns a new vector with lanes selected from the lanes of the first input +/// vector `a` specified in the second input vector `s`. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is less than the pairwise right element, or all zeros otherwise. +/// The indices `i` in range [0, 15] select the `i`-th element of `a`. For +/// indices outside of the range the resulting lane is 0. #[inline] -#[cfg_attr(test, assert_instr(f32x4.le))] -pub fn f32x4_le(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_le::<_, i32x4>(a.as_f32x4(), b.as_f32x4())) } +#[cfg_attr(test, assert_instr(i8x16.swizzle))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.swizzle"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_swizzle(a: v128, s: v128) -> v128 { + unsafe { llvm_swizzle(a.as_i8x16(), s.as_i8x16()).v128() } } -/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit -/// floating point numbers. +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i8x16_swizzle as u8x16_swizzle; + +/// Creates a vector with identical lanes. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is greater than the pairwise right element, or all zeros otherwise. +/// Constructs a vector with `x` replicated to all 16 lanes. #[inline] -#[cfg_attr(test, assert_instr(f32x4.ge))] -pub fn f32x4_ge(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_ge::<_, i32x4>(a.as_f32x4(), b.as_f32x4())) } +#[cfg_attr(test, assert_instr(i8x16.splat))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_splat(a: i8) -> v128 { + simd::i8x16::splat(a).v128() } -/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit -/// floating point numbers. +/// Creates a vector with identical lanes. /// -/// Returns a new vector where each lane is all ones if the pairwise elements -/// were equal, or all zeros if the elements were not equal. +/// Constructs a vector with `x` replicated to all 16 lanes. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(f64x2.eq))] -pub fn f64x2_eq(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_eq::<_, i64x2>(a.as_f64x2(), b.as_f64x2())) } +#[cfg_attr(test, assert_instr(i8x16.splat))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_splat(a: u8) -> v128 { + simd::u8x16::splat(a).v128() } -/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit -/// floating point numbers. +/// Creates a vector with identical lanes. /// -/// Returns a new vector where each lane is all ones if the pairwise elements -/// were not equal, or all zeros if the elements were equal. +/// Construct a vector with `x` replicated to all 8 lanes. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(f64x2.ne))] -pub fn f64x2_ne(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_ne::<_, i64x2>(a.as_f64x2(), b.as_f64x2())) } +#[cfg_attr(test, assert_instr(i16x8.splat))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_splat(a: i16) -> v128 { + simd::i16x8::splat(a).v128() } -/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit -/// floating point numbers. +/// Creates a vector with identical lanes. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is less than the pairwise right element, or all zeros otherwise. +/// Construct a vector with `x` replicated to all 8 lanes. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(f64x2.lt))] -pub fn f64x2_lt(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_lt::<_, i64x2>(a.as_f64x2(), b.as_f64x2())) } +#[cfg_attr(test, assert_instr(i16x8.splat))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_splat(a: u16) -> v128 { + simd::u16x8::splat(a).v128() } -/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit -/// floating point numbers. +/// Creates a vector with identical lanes. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is greater than the pairwise right element, or all zeros otherwise. +/// Constructs a vector with `x` replicated to all 4 lanes. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(f64x2.gt))] -pub fn f64x2_gt(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_gt::<_, i64x2>(a.as_f64x2(), b.as_f64x2())) } +#[cfg_attr(test, assert_instr(i32x4.splat))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_splat(a: i32) -> v128 { + simd::i32x4::splat(a).v128() } -/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit -/// floating point numbers. +/// Creates a vector with identical lanes. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is less than the pairwise right element, or all zeros otherwise. +/// Constructs a vector with `x` replicated to all 4 lanes. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(f64x2.le))] -pub fn f64x2_le(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_le::<_, i64x2>(a.as_f64x2(), b.as_f64x2())) } +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_splat(a: u32) -> v128 { + i32x4_splat(a as i32) } -/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit -/// floating point numbers. +/// Creates a vector with identical lanes. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is greater than the pairwise right element, or all zeros otherwise. +/// Construct a vector with `x` replicated to all 2 lanes. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(f64x2.ge))] -pub fn f64x2_ge(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_ge::<_, i64x2>(a.as_f64x2(), b.as_f64x2())) } +#[cfg_attr(test, assert_instr(i64x2.splat))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_splat(a: i64) -> v128 { + simd::i64x2::splat(a).v128() } -/// Flips each bit of the 128-bit input vector. +/// Creates a vector with identical lanes. +/// +/// Construct a vector with `x` replicated to all 2 lanes. #[inline] -#[cfg_attr(test, assert_instr(v128.not))] -pub fn v128_not(a: v128) -> v128 { - unsafe { transmute(simd_xor(a.as_i64x2(), i64x2(!0, !0))) } +#[target_feature(enable = "simd128")] +#[doc(alias("u64x2.splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u64x2_splat(a: u64) -> v128 { + i64x2_splat(a as i64) } -/// Performs a bitwise and of the two input 128-bit vectors, returning the -/// resulting vector. +/// Creates a vector with identical lanes. +/// +/// Constructs a vector with `x` replicated to all 4 lanes. #[inline] -#[cfg_attr(test, assert_instr(v128.and))] -pub fn v128_and(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_and(a.as_i64x2(), b.as_i64x2())) } +#[cfg_attr(test, assert_instr(f32x4.splat))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_splat(a: f32) -> v128 { + simd::f32x4::splat(a).v128() } -/// Performs a bitwise or of the two input 128-bit vectors, returning the -/// resulting vector. +/// Creates a vector with identical lanes. +/// +/// Constructs a vector with `x` replicated to all 2 lanes. #[inline] -#[cfg_attr(test, assert_instr(v128.or))] -pub fn v128_or(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_or(a.as_i64x2(), b.as_i64x2())) } +#[cfg_attr(test, assert_instr(f64x2.splat))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_splat(a: f64) -> v128 { + simd::f64x2::splat(a).v128() } -/// Performs a bitwise xor of the two input 128-bit vectors, returning the -/// resulting vector. +/// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit +/// integers. +/// +/// Returns a new vector where each lane is all ones if the corresponding input elements +/// were equal, or all zeros otherwise. #[inline] -#[cfg_attr(test, assert_instr(v128.xor))] -pub fn v128_xor(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_xor(a.as_i64x2(), b.as_i64x2())) } +#[cfg_attr(test, assert_instr(i8x16.eq))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.eq"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_eq(a: v128, b: v128) -> v128 { + unsafe { simd_eq::<_, simd::i8x16>(a.as_i8x16(), b.as_i8x16()).v128() } } -/// Use the bitmask in `c` to select bits from `v1` when 1 and `v2` when 0. +/// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit +/// integers. +/// +/// Returns a new vector where each lane is all ones if the corresponding input elements +/// were not equal, or all zeros otherwise. #[inline] -#[cfg_attr(test, assert_instr(v128.bitselect))] -pub fn v128_bitselect(v1: v128, v2: v128, c: v128) -> v128 { - unsafe { transmute(llvm_bitselect(c.as_i8x16(), v1.as_i8x16(), v2.as_i8x16())) } +#[cfg_attr(test, assert_instr(i8x16.ne))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.ne"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_ne(a: v128, b: v128) -> v128 { + unsafe { simd_ne::<_, simd::i8x16>(a.as_i8x16(), b.as_i8x16()).v128() } } -/// Negates a 128-bit vectors intepreted as sixteen 8-bit signed integers -#[inline] -#[cfg_attr(test, assert_instr(i8x16.neg))] -pub fn i8x16_neg(a: v128) -> v128 { - unsafe { transmute(simd_mul(a.as_i8x16(), i8x16::splat(-1))) } -} +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i8x16_eq as u8x16_eq; +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i8x16_ne as u8x16_ne; -/// Returns 1 if any lane is nonzero or 0 if all lanes are zero. +/// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit +/// signed integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is less than the right element, or all zeros otherwise. #[inline] -#[cfg_attr(test, assert_instr(i8x16.any_true))] -pub fn i8x16_any_true(a: v128) -> i32 { - unsafe { llvm_i8x16_any_true(a.as_i8x16()) } +#[cfg_attr(test, assert_instr(i8x16.lt_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.lt_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_lt(a: v128, b: v128) -> v128 { + unsafe { simd_lt::<_, simd::i8x16>(a.as_i8x16(), b.as_i8x16()).v128() } } -/// Returns 1 if all lanes are nonzero or 0 if any lane is nonzero. +/// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit +/// unsigned integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is less than the right element, or all zeros otherwise. #[inline] -#[cfg_attr(test, assert_instr(i8x16.all_true))] -pub fn i8x16_all_true(a: v128) -> i32 { - unsafe { llvm_i8x16_all_true(a.as_i8x16()) } +#[cfg_attr(test, assert_instr(i8x16.lt_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.lt_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_lt(a: v128, b: v128) -> v128 { + unsafe { simd_lt::<_, simd::i8x16>(a.as_u8x16(), b.as_u8x16()).v128() } } -/// Shifts each lane to the left by the specified number of bits. +/// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit +/// signed integers. /// -/// Only the low bits of the shift amount are used if the shift amount is -/// greater than the lane width. +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is greater than the right element, or all zeros otherwise. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(i8x16.shl))] -pub fn i8x16_shl(a: v128, amt: u32) -> v128 { - unsafe { transmute(simd_shl(a.as_i8x16(), i8x16::splat(amt as i8))) } +#[cfg_attr(test, assert_instr(i8x16.gt_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.gt_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_gt(a: v128, b: v128) -> v128 { + unsafe { simd_gt::<_, simd::i8x16>(a.as_i8x16(), b.as_i8x16()).v128() } } -/// Shifts each lane to the right by the specified number of bits, sign -/// extending. +/// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit +/// unsigned integers. /// -/// Only the low bits of the shift amount are used if the shift amount is -/// greater than the lane width. +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is greater than the right element, or all zeros otherwise. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(i8x16.shl))] -pub fn i8x16_shr_s(a: v128, amt: u32) -> v128 { - unsafe { transmute(simd_shr(a.as_i8x16(), i8x16::splat(amt as i8))) } +#[cfg_attr(test, assert_instr(i8x16.gt_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.gt_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_gt(a: v128, b: v128) -> v128 { + unsafe { simd_gt::<_, simd::i8x16>(a.as_u8x16(), b.as_u8x16()).v128() } } -/// Shifts each lane to the right by the specified number of bits, shifting in -/// zeros. +/// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit +/// signed integers. /// -/// Only the low bits of the shift amount are used if the shift amount is -/// greater than the lane width. +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is less than the right element, or all zeros otherwise. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(i8x16.shl))] -pub fn i8x16_shr_u(a: v128, amt: u32) -> v128 { - unsafe { transmute(simd_shr(a.as_u8x16(), u8x16::splat(amt as u8))) } +#[cfg_attr(test, assert_instr(i8x16.le_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.le_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_le(a: v128, b: v128) -> v128 { + unsafe { simd_le::<_, simd::i8x16>(a.as_i8x16(), b.as_i8x16()).v128() } } -/// Adds two 128-bit vectors as if they were two packed sixteen 8-bit integers. +/// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit +/// unsigned integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is less than the right element, or all zeros otherwise. #[inline] -#[cfg_attr(test, assert_instr(i8x16.add))] -pub fn i8x16_add(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_add(a.as_i8x16(), b.as_i8x16())) } +#[cfg_attr(test, assert_instr(i8x16.le_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.le_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_le(a: v128, b: v128) -> v128 { + unsafe { simd_le::<_, simd::i8x16>(a.as_u8x16(), b.as_u8x16()).v128() } } -/// Adds two 128-bit vectors as if they were two packed sixteen 8-bit signed -/// integers, saturating on overflow to `i8::max_value()`. +/// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit +/// signed integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is greater than the right element, or all zeros otherwise. #[inline] -#[cfg_attr(test, assert_instr(i8x16.add_saturate_s))] -pub fn i8x16_add_saturate_s(a: v128, b: v128) -> v128 { - unsafe { transmute(llvm_i8x16_add_saturate_s(a.as_i8x16(), b.as_i8x16())) } +#[cfg_attr(test, assert_instr(i8x16.ge_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.ge_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_ge(a: v128, b: v128) -> v128 { + unsafe { simd_ge::<_, simd::i8x16>(a.as_i8x16(), b.as_i8x16()).v128() } } -/// Adds two 128-bit vectors as if they were two packed sixteen 8-bit unsigned -/// integers, saturating on overflow to `u8::max_value()`. +/// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit +/// unsigned integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is greater than the right element, or all zeros otherwise. #[inline] -#[cfg_attr(test, assert_instr(i8x16.add_saturate_u))] -pub fn i8x16_add_saturate_u(a: v128, b: v128) -> v128 { - unsafe { transmute(llvm_i8x16_add_saturate_u(a.as_i8x16(), b.as_i8x16())) } +#[cfg_attr(test, assert_instr(i8x16.ge_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.ge_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_ge(a: v128, b: v128) -> v128 { + unsafe { simd_ge::<_, simd::i8x16>(a.as_u8x16(), b.as_u8x16()).v128() } } -/// Subtracts two 128-bit vectors as if they were two packed sixteen 8-bit integers. +/// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit +/// integers. +/// +/// Returns a new vector where each lane is all ones if the corresponding input elements +/// were equal, or all zeros otherwise. #[inline] -#[cfg_attr(test, assert_instr(i8x16.sub))] -pub fn i8x16_sub(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_sub(a.as_i8x16(), b.as_i8x16())) } +#[cfg_attr(test, assert_instr(i16x8.eq))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.eq"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_eq(a: v128, b: v128) -> v128 { + unsafe { simd_eq::<_, simd::i16x8>(a.as_i16x8(), b.as_i16x8()).v128() } } -/// Subtracts two 128-bit vectors as if they were two packed sixteen 8-bit -/// signed integers, saturating on overflow to `i8::min_value()`. +/// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit +/// integers. +/// +/// Returns a new vector where each lane is all ones if the corresponding input elements +/// were not equal, or all zeros otherwise. #[inline] -#[cfg_attr(test, assert_instr(i8x16.sub_saturate_s))] -pub fn i8x16_sub_saturate_s(a: v128, b: v128) -> v128 { - unsafe { transmute(llvm_i8x16_sub_saturate_s(a.as_i8x16(), b.as_i8x16())) } +#[cfg_attr(test, assert_instr(i16x8.ne))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.ne"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_ne(a: v128, b: v128) -> v128 { + unsafe { simd_ne::<_, simd::i16x8>(a.as_i16x8(), b.as_i16x8()).v128() } } -/// Subtracts two 128-bit vectors as if they were two packed sixteen 8-bit -/// unsigned integers, saturating on overflow to 0. -#[inline] -#[cfg_attr(test, assert_instr(i8x16.sub_saturate_u))] -pub fn i8x16_sub_saturate_u(a: v128, b: v128) -> v128 { - unsafe { transmute(llvm_i8x16_sub_saturate_u(a.as_i8x16(), b.as_i8x16())) } -} +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_eq as u16x8_eq; +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_ne as u16x8_ne; -/// Multiplies two 128-bit vectors as if they were two packed sixteen 8-bit +/// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit /// signed integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is less than the right element, or all zeros otherwise. #[inline] -#[cfg_attr(test, assert_instr(i8x16.mul))] -pub fn i8x16_mul(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_mul(a.as_i8x16(), b.as_i8x16())) } +#[cfg_attr(test, assert_instr(i16x8.lt_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.lt_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_lt(a: v128, b: v128) -> v128 { + unsafe { simd_lt::<_, simd::i16x8>(a.as_i16x8(), b.as_i16x8()).v128() } } -/// Negates a 128-bit vectors intepreted as eight 16-bit signed integers +/// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit +/// unsigned integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is less than the right element, or all zeros otherwise. #[inline] -#[cfg_attr(test, assert_instr(i16x8.neg))] -pub fn i16x8_neg(a: v128) -> v128 { - unsafe { transmute(simd_mul(a.as_i16x8(), i16x8::splat(-1))) } +#[cfg_attr(test, assert_instr(i16x8.lt_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.lt_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_lt(a: v128, b: v128) -> v128 { + unsafe { simd_lt::<_, simd::i16x8>(a.as_u16x8(), b.as_u16x8()).v128() } } -/// Returns 1 if any lane is nonzero or 0 if all lanes are zero. +/// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit +/// signed integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is greater than the right element, or all zeros otherwise. #[inline] -#[cfg_attr(test, assert_instr(i16x8.any_true))] -pub fn i16x8_any_true(a: v128) -> i32 { - unsafe { llvm_i16x8_any_true(a.as_i16x8()) } +#[cfg_attr(test, assert_instr(i16x8.gt_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.gt_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_gt(a: v128, b: v128) -> v128 { + unsafe { simd_gt::<_, simd::i16x8>(a.as_i16x8(), b.as_i16x8()).v128() } } -/// Returns 1 if all lanes are nonzero or 0 if any lane is nonzero. +/// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit +/// unsigned integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is greater than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.gt_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.gt_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_gt(a: v128, b: v128) -> v128 { + unsafe { simd_gt::<_, simd::i16x8>(a.as_u16x8(), b.as_u16x8()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit +/// signed integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is less than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.le_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.le_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_le(a: v128, b: v128) -> v128 { + unsafe { simd_le::<_, simd::i16x8>(a.as_i16x8(), b.as_i16x8()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit +/// unsigned integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is less than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.le_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.le_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_le(a: v128, b: v128) -> v128 { + unsafe { simd_le::<_, simd::i16x8>(a.as_u16x8(), b.as_u16x8()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit +/// signed integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is greater than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.ge_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.ge_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_ge(a: v128, b: v128) -> v128 { + unsafe { simd_ge::<_, simd::i16x8>(a.as_i16x8(), b.as_i16x8()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit +/// unsigned integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is greater than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.ge_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.ge_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_ge(a: v128, b: v128) -> v128 { + unsafe { simd_ge::<_, simd::i16x8>(a.as_u16x8(), b.as_u16x8()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +/// integers. +/// +/// Returns a new vector where each lane is all ones if the corresponding input elements +/// were equal, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.eq))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.eq"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_eq(a: v128, b: v128) -> v128 { + unsafe { simd_eq::<_, simd::i32x4>(a.as_i32x4(), b.as_i32x4()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +/// integers. +/// +/// Returns a new vector where each lane is all ones if the corresponding input elements +/// were not equal, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.ne))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.ne"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_ne(a: v128, b: v128) -> v128 { + unsafe { simd_ne::<_, simd::i32x4>(a.as_i32x4(), b.as_i32x4()).v128() } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_eq as u32x4_eq; +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_ne as u32x4_ne; + +/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +/// signed integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is less than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.lt_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.lt_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_lt(a: v128, b: v128) -> v128 { + unsafe { simd_lt::<_, simd::i32x4>(a.as_i32x4(), b.as_i32x4()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +/// unsigned integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is less than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.lt_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.lt_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_lt(a: v128, b: v128) -> v128 { + unsafe { simd_lt::<_, simd::i32x4>(a.as_u32x4(), b.as_u32x4()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +/// signed integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is greater than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.gt_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.gt_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_gt(a: v128, b: v128) -> v128 { + unsafe { simd_gt::<_, simd::i32x4>(a.as_i32x4(), b.as_i32x4()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +/// unsigned integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is greater than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.gt_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.gt_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_gt(a: v128, b: v128) -> v128 { + unsafe { simd_gt::<_, simd::i32x4>(a.as_u32x4(), b.as_u32x4()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +/// signed integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is less than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.le_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.le_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_le(a: v128, b: v128) -> v128 { + unsafe { simd_le::<_, simd::i32x4>(a.as_i32x4(), b.as_i32x4()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +/// unsigned integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is less than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.le_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.le_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_le(a: v128, b: v128) -> v128 { + unsafe { simd_le::<_, simd::i32x4>(a.as_u32x4(), b.as_u32x4()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +/// signed integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is greater than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.ge_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.ge_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_ge(a: v128, b: v128) -> v128 { + unsafe { simd_ge::<_, simd::i32x4>(a.as_i32x4(), b.as_i32x4()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +/// unsigned integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is greater than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.ge_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.ge_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_ge(a: v128, b: v128) -> v128 { + unsafe { simd_ge::<_, simd::i32x4>(a.as_u32x4(), b.as_u32x4()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit +/// integers. +/// +/// Returns a new vector where each lane is all ones if the corresponding input elements +/// were equal, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i64x2.eq))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.eq"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_eq(a: v128, b: v128) -> v128 { + unsafe { simd_eq::<_, simd::i64x2>(a.as_i64x2(), b.as_i64x2()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit +/// integers. +/// +/// Returns a new vector where each lane is all ones if the corresponding input elements +/// were not equal, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i64x2.ne))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.ne"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_ne(a: v128, b: v128) -> v128 { + unsafe { simd_ne::<_, simd::i64x2>(a.as_i64x2(), b.as_i64x2()).v128() } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_eq as u64x2_eq; +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_ne as u64x2_ne; + +/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit +/// signed integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is less than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i64x2.lt_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.lt_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_lt(a: v128, b: v128) -> v128 { + unsafe { simd_lt::<_, simd::i64x2>(a.as_i64x2(), b.as_i64x2()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit +/// signed integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is greater than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i64x2.gt_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.gt_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_gt(a: v128, b: v128) -> v128 { + unsafe { simd_gt::<_, simd::i64x2>(a.as_i64x2(), b.as_i64x2()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit +/// signed integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is less than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i64x2.le_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.le_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_le(a: v128, b: v128) -> v128 { + unsafe { simd_le::<_, simd::i64x2>(a.as_i64x2(), b.as_i64x2()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit +/// signed integers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is greater than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i64x2.ge_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.ge_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_ge(a: v128, b: v128) -> v128 { + unsafe { simd_ge::<_, simd::i64x2>(a.as_i64x2(), b.as_i64x2()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +/// floating point numbers. +/// +/// Returns a new vector where each lane is all ones if the corresponding input elements +/// were equal, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(f32x4.eq))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.eq"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_eq(a: v128, b: v128) -> v128 { + unsafe { simd_eq::<_, simd::i32x4>(a.as_f32x4(), b.as_f32x4()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +/// floating point numbers. +/// +/// Returns a new vector where each lane is all ones if the corresponding input elements +/// were not equal, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(f32x4.ne))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.ne"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_ne(a: v128, b: v128) -> v128 { + unsafe { simd_ne::<_, simd::i32x4>(a.as_f32x4(), b.as_f32x4()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +/// floating point numbers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is less than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(f32x4.lt))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.lt"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_lt(a: v128, b: v128) -> v128 { + unsafe { simd_lt::<_, simd::i32x4>(a.as_f32x4(), b.as_f32x4()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +/// floating point numbers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is greater than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(f32x4.gt))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.gt"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_gt(a: v128, b: v128) -> v128 { + unsafe { simd_gt::<_, simd::i32x4>(a.as_f32x4(), b.as_f32x4()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +/// floating point numbers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is less than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(f32x4.le))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.le"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_le(a: v128, b: v128) -> v128 { + unsafe { simd_le::<_, simd::i32x4>(a.as_f32x4(), b.as_f32x4()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +/// floating point numbers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is greater than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(f32x4.ge))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.ge"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_ge(a: v128, b: v128) -> v128 { + unsafe { simd_ge::<_, simd::i32x4>(a.as_f32x4(), b.as_f32x4()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit +/// floating point numbers. +/// +/// Returns a new vector where each lane is all ones if the corresponding input elements +/// were equal, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(f64x2.eq))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.eq"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_eq(a: v128, b: v128) -> v128 { + unsafe { simd_eq::<_, simd::i64x2>(a.as_f64x2(), b.as_f64x2()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit +/// floating point numbers. +/// +/// Returns a new vector where each lane is all ones if the corresponding input elements +/// were not equal, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(f64x2.ne))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.ne"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_ne(a: v128, b: v128) -> v128 { + unsafe { simd_ne::<_, simd::i64x2>(a.as_f64x2(), b.as_f64x2()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit +/// floating point numbers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is less than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(f64x2.lt))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.lt"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_lt(a: v128, b: v128) -> v128 { + unsafe { simd_lt::<_, simd::i64x2>(a.as_f64x2(), b.as_f64x2()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit +/// floating point numbers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is greater than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(f64x2.gt))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.gt"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_gt(a: v128, b: v128) -> v128 { + unsafe { simd_gt::<_, simd::i64x2>(a.as_f64x2(), b.as_f64x2()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit +/// floating point numbers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is less than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(f64x2.le))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.le"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_le(a: v128, b: v128) -> v128 { + unsafe { simd_le::<_, simd::i64x2>(a.as_f64x2(), b.as_f64x2()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit +/// floating point numbers. +/// +/// Returns a new vector where each lane is all ones if the lane-wise left +/// element is greater than the right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(f64x2.ge))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.ge"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_ge(a: v128, b: v128) -> v128 { + unsafe { simd_ge::<_, simd::i64x2>(a.as_f64x2(), b.as_f64x2()).v128() } +} + +/// Flips each bit of the 128-bit input vector. +#[inline] +#[cfg_attr(test, assert_instr(v128.not))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.not"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn v128_not(a: v128) -> v128 { + unsafe { simd_xor(a.as_i64x2(), simd::i64x2(!0, !0)).v128() } +} + +/// Performs a bitwise and of the two input 128-bit vectors, returning the +/// resulting vector. +#[inline] +#[cfg_attr(test, assert_instr(v128.and))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.and"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn v128_and(a: v128, b: v128) -> v128 { + unsafe { simd_and(a.as_i64x2(), b.as_i64x2()).v128() } +} + +/// Bitwise AND of bits of `a` and the logical inverse of bits of `b`. +/// +/// This operation is equivalent to `v128.and(a, v128.not(b))` +#[inline] +#[cfg_attr(test, assert_instr(v128.andnot))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.andnot"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn v128_andnot(a: v128, b: v128) -> v128 { + unsafe { simd_and(a.as_i64x2(), simd_xor(b.as_i64x2(), simd::i64x2(-1, -1))).v128() } +} + +/// Performs a bitwise or of the two input 128-bit vectors, returning the +/// resulting vector. +#[inline] +#[cfg_attr(test, assert_instr(v128.or))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.or"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn v128_or(a: v128, b: v128) -> v128 { + unsafe { simd_or(a.as_i64x2(), b.as_i64x2()).v128() } +} + +/// Performs a bitwise xor of the two input 128-bit vectors, returning the +/// resulting vector. +#[inline] +#[cfg_attr(test, assert_instr(v128.xor))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.xor"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn v128_xor(a: v128, b: v128) -> v128 { + unsafe { simd_xor(a.as_i64x2(), b.as_i64x2()).v128() } +} + +/// Use the bitmask in `c` to select bits from `v1` when 1 and `v2` when 0. +#[inline] +#[cfg_attr(test, assert_instr(v128.bitselect))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.bitselect"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn v128_bitselect(v1: v128, v2: v128, c: v128) -> v128 { + unsafe { llvm_bitselect(v1.as_i8x16(), v2.as_i8x16(), c.as_i8x16()).v128() } +} + +/// Returns `true` if any bit in `a` is set, or `false` otherwise. +#[inline] +#[cfg_attr(test, assert_instr(v128.any_true))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.any_true"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn v128_any_true(a: v128) -> bool { + unsafe { llvm_any_true_i8x16(a.as_i8x16()) != 0 } +} + +/// Lane-wise wrapping absolute value. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.abs))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.abs"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_abs(a: v128) -> v128 { + unsafe { + let a = a.as_i8x16(); + let zero = simd::i8x16::splat(0); + simd_select::(simd_lt(a, zero), simd_sub(zero, a), a).v128() + } +} + +/// Negates a 128-bit vectors interpreted as sixteen 8-bit signed integers +#[inline] +#[cfg_attr(test, assert_instr(i8x16.neg))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.neg"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_neg(a: v128) -> v128 { + unsafe { simd_mul(a.as_i8x16(), simd::i8x16::splat(-1)).v128() } +} + +/// Count the number of bits set to one within each lane. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.popcnt))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.popcnt"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_popcnt(v: v128) -> v128 { + unsafe { llvm_popcnt(v.as_i8x16()).v128() } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i8x16_popcnt as u8x16_popcnt; + +/// Returns true if all lanes are non-zero, false otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.all_true))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.all_true"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_all_true(a: v128) -> bool { + unsafe { llvm_i8x16_all_true(a.as_i8x16()) != 0 } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i8x16_all_true as u8x16_all_true; + +/// Extracts the high bit for each lane in `a` and produce a scalar mask with +/// all bits concatenated. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.bitmask))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.bitmask"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_bitmask(a: v128) -> u16 { + // FIXME(https://bugs.llvm.org/show_bug.cgi?id=50507) - this produces an + // extraneous `i32.and` instruction against a mask of 65535 when converting + // from the native intrinsic's i32 return value to our desired u16. This + // shouldn't be necessary, though, but requires upstream LLVM changes. + unsafe { llvm_bitmask_i8x16(a.as_i8x16()) as u16 } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i8x16_bitmask as u8x16_bitmask; + +/// Converts two input vectors into a smaller lane vector by narrowing each +/// lane. +/// +/// Signed saturation to 0x7f or 0x80 is used and the input lanes are always +/// interpreted as signed integers. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.narrow_i16x8_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.narrow_i16x8_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_narrow_i16x8(a: v128, b: v128) -> v128 { + unsafe { llvm_narrow_i8x16_s(a.as_i16x8(), b.as_i16x8()).v128() } +} + +/// Converts two input vectors into a smaller lane vector by narrowing each +/// lane. +/// +/// Signed saturation to 0x00 or 0xff is used and the input lanes are always +/// interpreted as signed integers. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.narrow_i16x8_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.narrow_i16x8_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_narrow_i16x8(a: v128, b: v128) -> v128 { + unsafe { llvm_narrow_i8x16_u(a.as_i16x8(), b.as_i16x8()).v128() } +} + +/// Shifts each lane to the left by the specified number of bits. +/// +/// Only the low bits of the shift amount are used if the shift amount is +/// greater than the lane width. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.shl))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.shl"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_shl(a: v128, amt: u32) -> v128 { + unsafe { simd_shl(a.as_i8x16(), simd::i8x16::splat(amt as i8)).v128() } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i8x16_shl as u8x16_shl; + +/// Shifts each lane to the right by the specified number of bits, sign +/// extending. +/// +/// Only the low bits of the shift amount are used if the shift amount is +/// greater than the lane width. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.shr_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.shr_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_shr(a: v128, amt: u32) -> v128 { + unsafe { simd_shr(a.as_i8x16(), simd::i8x16::splat(amt as i8)).v128() } +} + +/// Shifts each lane to the right by the specified number of bits, shifting in +/// zeros. +/// +/// Only the low bits of the shift amount are used if the shift amount is +/// greater than the lane width. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.shr_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.shr_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_shr(a: v128, amt: u32) -> v128 { + unsafe { simd_shr(a.as_u8x16(), simd::u8x16::splat(amt as u8)).v128() } +} + +/// Adds two 128-bit vectors as if they were two packed sixteen 8-bit integers. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.add))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.add"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_add(a: v128, b: v128) -> v128 { + unsafe { simd_add(a.as_i8x16(), b.as_i8x16()).v128() } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i8x16_add as u8x16_add; + +/// Adds two 128-bit vectors as if they were two packed sixteen 8-bit signed +/// integers, saturating on overflow to `i8::MAX`. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.add_sat_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.add_sat_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_add_sat(a: v128, b: v128) -> v128 { + unsafe { llvm_i8x16_add_sat_s(a.as_i8x16(), b.as_i8x16()).v128() } +} + +/// Adds two 128-bit vectors as if they were two packed sixteen 8-bit unsigned +/// integers, saturating on overflow to `u8::MAX`. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.add_sat_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.add_sat_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_add_sat(a: v128, b: v128) -> v128 { + unsafe { llvm_i8x16_add_sat_u(a.as_i8x16(), b.as_i8x16()).v128() } +} + +/// Subtracts two 128-bit vectors as if they were two packed sixteen 8-bit integers. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.sub))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.sub"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_sub(a: v128, b: v128) -> v128 { + unsafe { simd_sub(a.as_i8x16(), b.as_i8x16()).v128() } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i8x16_sub as u8x16_sub; + +/// Subtracts two 128-bit vectors as if they were two packed sixteen 8-bit +/// signed integers, saturating on overflow to `i8::MIN`. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.sub_sat_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.sub_sat_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_sub_sat(a: v128, b: v128) -> v128 { + unsafe { llvm_i8x16_sub_sat_s(a.as_i8x16(), b.as_i8x16()).v128() } +} + +/// Subtracts two 128-bit vectors as if they were two packed sixteen 8-bit +/// unsigned integers, saturating on overflow to 0. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.sub_sat_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.sub_sat_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_sub_sat(a: v128, b: v128) -> v128 { + unsafe { llvm_i8x16_sub_sat_u(a.as_i8x16(), b.as_i8x16()).v128() } +} + +/// Compares lane-wise signed integers, and returns the minimum of +/// each pair. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.min_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.min_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_min(a: v128, b: v128) -> v128 { + let a = a.as_i8x16(); + let b = b.as_i8x16(); + unsafe { simd_select::(simd_lt(a, b), a, b).v128() } +} + +/// Compares lane-wise unsigned integers, and returns the minimum of +/// each pair. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.min_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.min_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_min(a: v128, b: v128) -> v128 { + let a = a.as_u8x16(); + let b = b.as_u8x16(); + unsafe { simd_select::(simd_lt(a, b), a, b).v128() } +} + +/// Compares lane-wise signed integers, and returns the maximum of +/// each pair. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.max_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.max_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_max(a: v128, b: v128) -> v128 { + let a = a.as_i8x16(); + let b = b.as_i8x16(); + unsafe { simd_select::(simd_gt(a, b), a, b).v128() } +} + +/// Compares lane-wise unsigned integers, and returns the maximum of +/// each pair. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.max_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.max_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_max(a: v128, b: v128) -> v128 { + let a = a.as_u8x16(); + let b = b.as_u8x16(); + unsafe { simd_select::(simd_gt(a, b), a, b).v128() } +} + +/// Lane-wise rounding average. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.avgr_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.avgr_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_avgr(a: v128, b: v128) -> v128 { + unsafe { llvm_avgr_u_i8x16(a.as_i8x16(), b.as_i8x16()).v128() } +} + +/// Integer extended pairwise addition producing extended results +/// (twice wider results than the inputs). +#[inline] +#[cfg_attr(test, assert_instr(i16x8.extadd_pairwise_i8x16_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extadd_pairwise_i8x16_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extadd_pairwise_i8x16(a: v128) -> v128 { + unsafe { llvm_i16x8_extadd_pairwise_i8x16_s(a.as_i8x16()).v128() } +} + +/// Integer extended pairwise addition producing extended results +/// (twice wider results than the inputs). +#[inline] +#[cfg_attr(test, assert_instr(i16x8.extadd_pairwise_i8x16_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extadd_pairwise_i8x16_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extadd_pairwise_u8x16(a: v128) -> v128 { + unsafe { llvm_i16x8_extadd_pairwise_i8x16_u(a.as_i8x16()).v128() } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_extadd_pairwise_u8x16 as u16x8_extadd_pairwise_u8x16; + +/// Lane-wise wrapping absolute value. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.abs))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.abs"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_abs(a: v128) -> v128 { + let a = a.as_i16x8(); + let zero = simd::i16x8::splat(0); + unsafe { + simd_select::(simd_lt(a, zero), simd_sub(zero, a), a).v128() + } +} + +/// Negates a 128-bit vectors interpreted as eight 16-bit signed integers +#[inline] +#[cfg_attr(test, assert_instr(i16x8.neg))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.neg"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_neg(a: v128) -> v128 { + unsafe { simd_mul(a.as_i16x8(), simd::i16x8::splat(-1)).v128() } +} + +/// Lane-wise saturating rounding multiplication in Q15 format. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.q15mulr_sat_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.q15mulr_sat_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_q15mulr_sat(a: v128, b: v128) -> v128 { + unsafe { llvm_q15mulr(a.as_i16x8(), b.as_i16x8()).v128() } +} + +/// Returns true if all lanes are non-zero, false otherwise. #[inline] #[cfg_attr(test, assert_instr(i16x8.all_true))] -pub fn i16x8_all_true(a: v128) -> i32 { - unsafe { llvm_i16x8_all_true(a.as_i16x8()) } +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.all_true"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_all_true(a: v128) -> bool { + unsafe { llvm_i16x8_all_true(a.as_i16x8()) != 0 } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_all_true as u16x8_all_true; + +/// Extracts the high bit for each lane in `a` and produce a scalar mask with +/// all bits concatenated. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.bitmask))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.bitmask"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_bitmask(a: v128) -> u8 { + unsafe { llvm_bitmask_i16x8(a.as_i16x8()) as u8 } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_bitmask as u16x8_bitmask; + +/// Converts two input vectors into a smaller lane vector by narrowing each +/// lane. +/// +/// Signed saturation to 0x7fff or 0x8000 is used and the input lanes are always +/// interpreted as signed integers. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.narrow_i32x4_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.narrow_i32x4_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_narrow_i32x4(a: v128, b: v128) -> v128 { + unsafe { llvm_narrow_i16x8_s(a.as_i32x4(), b.as_i32x4()).v128() } +} + +/// Converts two input vectors into a smaller lane vector by narrowing each +/// lane. +/// +/// Signed saturation to 0x0000 or 0xffff is used and the input lanes are always +/// interpreted as signed integers. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.narrow_i32x4_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.narrow_i32x4_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_narrow_i32x4(a: v128, b: v128) -> v128 { + unsafe { llvm_narrow_i16x8_u(a.as_i32x4(), b.as_i32x4()).v128() } +} + +/// Converts low half of the smaller lane vector to a larger lane +/// vector, sign extended. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.extend_low_i8x16_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extend_low_i8x16_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extend_low_i8x16(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle8!( + a.as_i8x16(), + a.as_i8x16(), + [0, 1, 2, 3, 4, 5, 6, 7], + )) + .v128() + } +} + +/// Converts high half of the smaller lane vector to a larger lane +/// vector, sign extended. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.extend_high_i8x16_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extend_high_i8x16_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extend_high_i8x16(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle8!( + a.as_i8x16(), + a.as_i8x16(), + [8, 9, 10, 11, 12, 13, 14, 15], + )) + .v128() + } +} + +/// Converts low half of the smaller lane vector to a larger lane +/// vector, zero extended. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.extend_low_i8x16_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extend_low_i8x16_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extend_low_u8x16(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle8!( + a.as_u8x16(), + a.as_u8x16(), + [0, 1, 2, 3, 4, 5, 6, 7], + )) + .v128() + } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_extend_low_u8x16 as u16x8_extend_low_u8x16; + +/// Converts high half of the smaller lane vector to a larger lane +/// vector, zero extended. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.extend_high_i8x16_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extend_high_i8x16_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extend_high_u8x16(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle8!( + a.as_u8x16(), + a.as_u8x16(), + [8, 9, 10, 11, 12, 13, 14, 15], + )) + .v128() + } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_extend_high_u8x16 as u16x8_extend_high_u8x16; + +/// Shifts each lane to the left by the specified number of bits. +/// +/// Only the low bits of the shift amount are used if the shift amount is +/// greater than the lane width. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.shl))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.shl"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_shl(a: v128, amt: u32) -> v128 { + unsafe { simd_shl(a.as_i16x8(), simd::i16x8::splat(amt as i16)).v128() } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_shl as u16x8_shl; + +/// Shifts each lane to the right by the specified number of bits, sign +/// extending. +/// +/// Only the low bits of the shift amount are used if the shift amount is +/// greater than the lane width. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.shr_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.shr_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_shr(a: v128, amt: u32) -> v128 { + unsafe { simd_shr(a.as_i16x8(), simd::i16x8::splat(amt as i16)).v128() } +} + +/// Shifts each lane to the right by the specified number of bits, shifting in +/// zeros. +/// +/// Only the low bits of the shift amount are used if the shift amount is +/// greater than the lane width. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.shr_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.shr_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_shr(a: v128, amt: u32) -> v128 { + unsafe { simd_shr(a.as_u16x8(), simd::u16x8::splat(amt as u16)).v128() } +} + +/// Adds two 128-bit vectors as if they were two packed eight 16-bit integers. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.add))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.add"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_add(a: v128, b: v128) -> v128 { + unsafe { simd_add(a.as_i16x8(), b.as_i16x8()).v128() } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_add as u16x8_add; + +/// Adds two 128-bit vectors as if they were two packed eight 16-bit signed +/// integers, saturating on overflow to `i16::MAX`. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.add_sat_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.add_sat_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_add_sat(a: v128, b: v128) -> v128 { + unsafe { llvm_i16x8_add_sat_s(a.as_i16x8(), b.as_i16x8()).v128() } +} + +/// Adds two 128-bit vectors as if they were two packed eight 16-bit unsigned +/// integers, saturating on overflow to `u16::MAX`. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.add_sat_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.add_sat_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_add_sat(a: v128, b: v128) -> v128 { + unsafe { llvm_i16x8_add_sat_u(a.as_i16x8(), b.as_i16x8()).v128() } +} + +/// Subtracts two 128-bit vectors as if they were two packed eight 16-bit integers. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.sub))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.sub"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_sub(a: v128, b: v128) -> v128 { + unsafe { simd_sub(a.as_i16x8(), b.as_i16x8()).v128() } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_sub as u16x8_sub; + +/// Subtracts two 128-bit vectors as if they were two packed eight 16-bit +/// signed integers, saturating on overflow to `i16::MIN`. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.sub_sat_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.sub_sat_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_sub_sat(a: v128, b: v128) -> v128 { + unsafe { llvm_i16x8_sub_sat_s(a.as_i16x8(), b.as_i16x8()).v128() } +} + +/// Subtracts two 128-bit vectors as if they were two packed eight 16-bit +/// unsigned integers, saturating on overflow to 0. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.sub_sat_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.sub_sat_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_sub_sat(a: v128, b: v128) -> v128 { + unsafe { llvm_i16x8_sub_sat_u(a.as_i16x8(), b.as_i16x8()).v128() } +} + +/// Multiplies two 128-bit vectors as if they were two packed eight 16-bit +/// signed integers. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.mul))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.mul"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_mul(a: v128, b: v128) -> v128 { + unsafe { simd_mul(a.as_i16x8(), b.as_i16x8()).v128() } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_mul as u16x8_mul; + +/// Compares lane-wise signed integers, and returns the minimum of +/// each pair. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.min_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.min_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_min(a: v128, b: v128) -> v128 { + let a = a.as_i16x8(); + let b = b.as_i16x8(); + unsafe { simd_select::(simd_lt(a, b), a, b).v128() } +} + +/// Compares lane-wise unsigned integers, and returns the minimum of +/// each pair. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.min_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.min_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_min(a: v128, b: v128) -> v128 { + let a = a.as_u16x8(); + let b = b.as_u16x8(); + unsafe { simd_select::(simd_lt(a, b), a, b).v128() } +} + +/// Compares lane-wise signed integers, and returns the maximum of +/// each pair. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.max_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.max_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_max(a: v128, b: v128) -> v128 { + let a = a.as_i16x8(); + let b = b.as_i16x8(); + unsafe { simd_select::(simd_gt(a, b), a, b).v128() } +} + +/// Compares lane-wise unsigned integers, and returns the maximum of +/// each pair. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.max_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.max_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_max(a: v128, b: v128) -> v128 { + let a = a.as_u16x8(); + let b = b.as_u16x8(); + unsafe { simd_select::(simd_gt(a, b), a, b).v128() } +} + +/// Lane-wise rounding average. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.avgr_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.avgr_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_avgr(a: v128, b: v128) -> v128 { + unsafe { llvm_avgr_u_i16x8(a.as_i16x8(), b.as_i16x8()).v128() } +} + +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. +/// +/// Equivalent of `i16x8_mul(i16x8_extend_low_i8x16(a), i16x8_extend_low_i8x16(b))` +#[inline] +#[cfg_attr(test, assert_instr(i16x8.extmul_low_i8x16_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extmul_low_i8x16_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extmul_low_i8x16(a: v128, b: v128) -> v128 { + unsafe { + let lhs = simd_cast::(simd_shuffle8!( + a.as_i8x16(), + a.as_i8x16(), + [0, 1, 2, 3, 4, 5, 6, 7], + )); + let rhs = simd_cast::(simd_shuffle8!( + b.as_i8x16(), + b.as_i8x16(), + [0, 1, 2, 3, 4, 5, 6, 7], + )); + simd_mul(lhs, rhs).v128() + } +} + +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. +/// +/// Equivalent of `i16x8_mul(i16x8_extend_high_i8x16(a), i16x8_extend_high_i8x16(b))` +#[inline] +#[cfg_attr(test, assert_instr(i16x8.extmul_high_i8x16_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extmul_high_i8x16_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extmul_high_i8x16(a: v128, b: v128) -> v128 { + unsafe { + let lhs = simd_cast::(simd_shuffle8!( + a.as_i8x16(), + a.as_i8x16(), + [8, 9, 10, 11, 12, 13, 14, 15], + )); + let rhs = simd_cast::(simd_shuffle8!( + b.as_i8x16(), + b.as_i8x16(), + [8, 9, 10, 11, 12, 13, 14, 15], + )); + simd_mul(lhs, rhs).v128() + } +} + +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. +/// +/// Equivalent of `i16x8_mul(i16x8_extend_low_u8x16(a), i16x8_extend_low_u8x16(b))` +#[inline] +#[cfg_attr(test, assert_instr(i16x8.extmul_low_i8x16_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extmul_low_i8x16_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extmul_low_u8x16(a: v128, b: v128) -> v128 { + unsafe { + let lhs = simd_cast::(simd_shuffle8!( + a.as_u8x16(), + a.as_u8x16(), + [0, 1, 2, 3, 4, 5, 6, 7], + )); + let rhs = simd_cast::(simd_shuffle8!( + b.as_u8x16(), + b.as_u8x16(), + [0, 1, 2, 3, 4, 5, 6, 7], + )); + simd_mul(lhs, rhs).v128() + } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_extmul_low_u8x16 as u16x8_extmul_low_u8x16; + +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. +/// +/// Equivalent of `i16x8_mul(i16x8_extend_high_u8x16(a), i16x8_extend_high_u8x16(b))` +#[inline] +#[cfg_attr(test, assert_instr(i16x8.extmul_high_i8x16_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extmul_high_i8x16_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extmul_high_u8x16(a: v128, b: v128) -> v128 { + unsafe { + let lhs = simd_cast::(simd_shuffle8!( + a.as_u8x16(), + a.as_u8x16(), + [8, 9, 10, 11, 12, 13, 14, 15], + )); + let rhs = simd_cast::(simd_shuffle8!( + b.as_u8x16(), + b.as_u8x16(), + [8, 9, 10, 11, 12, 13, 14, 15], + )); + simd_mul(lhs, rhs).v128() + } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_extmul_high_u8x16 as u16x8_extmul_high_u8x16; + +/// Integer extended pairwise addition producing extended results +/// (twice wider results than the inputs). +#[inline] +#[cfg_attr(test, assert_instr(i32x4.extadd_pairwise_i16x8_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.extadd_pairwise_i16x8_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extadd_pairwise_i16x8(a: v128) -> v128 { + unsafe { llvm_i32x4_extadd_pairwise_i16x8_s(a.as_i16x8()).v128() } +} + +/// Integer extended pairwise addition producing extended results +/// (twice wider results than the inputs). +#[inline] +#[cfg_attr(test, assert_instr(i32x4.extadd_pairwise_i16x8_u))] +#[doc(alias("i32x4.extadd_pairwise_i16x8_u"))] +#[target_feature(enable = "simd128")] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extadd_pairwise_u16x8(a: v128) -> v128 { + unsafe { llvm_i32x4_extadd_pairwise_i16x8_u(a.as_i16x8()).v128() } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_extadd_pairwise_u16x8 as u32x4_extadd_pairwise_u16x8; + +/// Lane-wise wrapping absolute value. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.abs))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.abs"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_abs(a: v128) -> v128 { + let a = a.as_i32x4(); + let zero = simd::i32x4::splat(0); + unsafe { + simd_select::(simd_lt(a, zero), simd_sub(zero, a), a).v128() + } +} + +/// Negates a 128-bit vectors interpreted as four 32-bit signed integers +#[inline] +#[cfg_attr(test, assert_instr(i32x4.neg))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.neg"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_neg(a: v128) -> v128 { + unsafe { simd_mul(a.as_i32x4(), simd::i32x4::splat(-1)).v128() } +} + +/// Returns true if all lanes are non-zero, false otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.all_true))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.all_true"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_all_true(a: v128) -> bool { + unsafe { llvm_i32x4_all_true(a.as_i32x4()) != 0 } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_all_true as u32x4_all_true; + +/// Extracts the high bit for each lane in `a` and produce a scalar mask with +/// all bits concatenated. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.bitmask))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.bitmask"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_bitmask(a: v128) -> u8 { + unsafe { llvm_bitmask_i32x4(a.as_i32x4()) as u8 } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_bitmask as u32x4_bitmask; + +/// Converts low half of the smaller lane vector to a larger lane +/// vector, sign extended. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.extend_low_i16x8_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.extend_low_i16x8_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extend_low_i16x8(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle4!( + a.as_i16x8(), + a.as_i16x8(), + [0, 1, 2, 3] + )) + .v128() + } +} + +/// Converts high half of the smaller lane vector to a larger lane +/// vector, sign extended. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.extend_high_i16x8_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.extend_high_i16x8_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extend_high_i16x8(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle4!( + a.as_i16x8(), + a.as_i16x8(), + [4, 5, 6, 7] + )) + .v128() + } +} + +/// Converts low half of the smaller lane vector to a larger lane +/// vector, zero extended. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.extend_low_i16x8_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.extend_low_i16x8_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extend_low_u16x8(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle4!( + a.as_u16x8(), + a.as_u16x8(), + [0, 1, 2, 3] + )) + .v128() + } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_extend_low_u16x8 as u32x4_extend_low_u16x8; + +/// Converts high half of the smaller lane vector to a larger lane +/// vector, zero extended. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.extend_high_i16x8_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.extend_high_i16x8_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extend_high_u16x8(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle4!( + a.as_u16x8(), + a.as_u16x8(), + [4, 5, 6, 7] + )) + .v128() + } } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_extend_high_u16x8 as u32x4_extend_high_u16x8; + /// Shifts each lane to the left by the specified number of bits. /// /// Only the low bits of the shift amount are used if the shift amount is /// greater than the lane width. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(i16x8.shl))] -pub fn i16x8_shl(a: v128, amt: u32) -> v128 { - unsafe { transmute(simd_shl(a.as_i16x8(), i16x8::splat(amt as i16))) } +#[cfg_attr(test, assert_instr(i32x4.shl))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.shl"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_shl(a: v128, amt: u32) -> v128 { + unsafe { simd_shl(a.as_i32x4(), simd::i32x4::splat(amt as i32)).v128() } } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_shl as u32x4_shl; + /// Shifts each lane to the right by the specified number of bits, sign /// extending. /// /// Only the low bits of the shift amount are used if the shift amount is /// greater than the lane width. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(i16x8.shl))] -pub fn i16x8_shr_s(a: v128, amt: u32) -> v128 { - unsafe { transmute(simd_shr(a.as_i16x8(), i16x8::splat(amt as i16))) } +#[cfg_attr(test, assert_instr(i32x4.shr_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.shr_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_shr(a: v128, amt: u32) -> v128 { + unsafe { simd_shr(a.as_i32x4(), simd::i32x4::splat(amt as i32)).v128() } } /// Shifts each lane to the right by the specified number of bits, shifting in @@ -1155,189 +3241,364 @@ pub fn i16x8_shr_s(a: v128, amt: u32) -> v128 { /// Only the low bits of the shift amount are used if the shift amount is /// greater than the lane width. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(i16x8.shl))] -pub fn i16x8_shr_u(a: v128, amt: u32) -> v128 { - unsafe { transmute(simd_shr(a.as_u16x8(), u16x8::splat(amt as u16))) } +#[cfg_attr(test, assert_instr(i32x4.shr_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.shr_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_shr(a: v128, amt: u32) -> v128 { + unsafe { simd_shr(a.as_u32x4(), simd::u32x4::splat(amt as u32)).v128() } } -/// Adds two 128-bit vectors as if they were two packed eight 16-bit integers. +/// Adds two 128-bit vectors as if they were two packed four 32-bit integers. #[inline] -#[cfg_attr(test, assert_instr(i16x8.add))] -pub fn i16x8_add(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_add(a.as_i16x8(), b.as_i16x8())) } +#[cfg_attr(test, assert_instr(i32x4.add))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.add"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_add(a: v128, b: v128) -> v128 { + unsafe { simd_add(a.as_i32x4(), b.as_i32x4()).v128() } } -/// Adds two 128-bit vectors as if they were two packed eight 16-bit signed -/// integers, saturating on overflow to `i16::max_value()`. +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_add as u32x4_add; + +/// Subtracts two 128-bit vectors as if they were two packed four 32-bit integers. #[inline] -#[cfg_attr(test, assert_instr(i16x8.add_saturate_s))] -pub fn i16x8_add_saturate_s(a: v128, b: v128) -> v128 { - unsafe { transmute(llvm_i16x8_add_saturate_s(a.as_i16x8(), b.as_i16x8())) } +#[cfg_attr(test, assert_instr(i32x4.sub))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.sub"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_sub(a: v128, b: v128) -> v128 { + unsafe { simd_sub(a.as_i32x4(), b.as_i32x4()).v128() } } -/// Adds two 128-bit vectors as if they were two packed eight 16-bit unsigned -/// integers, saturating on overflow to `u16::max_value()`. +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_sub as u32x4_sub; + +/// Multiplies two 128-bit vectors as if they were two packed four 32-bit +/// signed integers. #[inline] -#[cfg_attr(test, assert_instr(i16x8.add_saturate_u))] -pub fn i16x8_add_saturate_u(a: v128, b: v128) -> v128 { - unsafe { transmute(llvm_i16x8_add_saturate_u(a.as_i16x8(), b.as_i16x8())) } +#[cfg_attr(test, assert_instr(i32x4.mul))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.mul"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_mul(a: v128, b: v128) -> v128 { + unsafe { simd_mul(a.as_i32x4(), b.as_i32x4()).v128() } } -/// Subtracts two 128-bit vectors as if they were two packed eight 16-bit integers. +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_mul as u32x4_mul; + +/// Compares lane-wise signed integers, and returns the minimum of +/// each pair. #[inline] -#[cfg_attr(test, assert_instr(i16x8.sub))] -pub fn i16x8_sub(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_sub(a.as_i16x8(), b.as_i16x8())) } +#[cfg_attr(test, assert_instr(i32x4.min_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.min_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_min(a: v128, b: v128) -> v128 { + let a = a.as_i32x4(); + let b = b.as_i32x4(); + unsafe { simd_select::(simd_lt(a, b), a, b).v128() } } -/// Subtracts two 128-bit vectors as if they were two packed eight 16-bit -/// signed integers, saturating on overflow to `i16::min_value()`. +/// Compares lane-wise unsigned integers, and returns the minimum of +/// each pair. #[inline] -#[cfg_attr(test, assert_instr(i16x8.sub_saturate_s))] -pub fn i16x8_sub_saturate_s(a: v128, b: v128) -> v128 { - unsafe { transmute(llvm_i16x8_sub_saturate_s(a.as_i16x8(), b.as_i16x8())) } +#[cfg_attr(test, assert_instr(i32x4.min_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.min_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_min(a: v128, b: v128) -> v128 { + let a = a.as_u32x4(); + let b = b.as_u32x4(); + unsafe { simd_select::(simd_lt(a, b), a, b).v128() } } -/// Subtracts two 128-bit vectors as if they were two packed eight 16-bit -/// unsigned integers, saturating on overflow to 0. +/// Compares lane-wise signed integers, and returns the maximum of +/// each pair. #[inline] -#[cfg_attr(test, assert_instr(i16x8.sub_saturate_u))] -pub fn i16x8_sub_saturate_u(a: v128, b: v128) -> v128 { - unsafe { transmute(llvm_i16x8_sub_saturate_u(a.as_i16x8(), b.as_i16x8())) } +#[cfg_attr(test, assert_instr(i32x4.max_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.max_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_max(a: v128, b: v128) -> v128 { + let a = a.as_i32x4(); + let b = b.as_i32x4(); + unsafe { simd_select::(simd_gt(a, b), a, b).v128() } } -/// Multiplies two 128-bit vectors as if they were two packed eight 16-bit -/// signed integers. +/// Compares lane-wise unsigned integers, and returns the maximum of +/// each pair. #[inline] -#[cfg_attr(test, assert_instr(i16x8.mul))] -pub fn i16x8_mul(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_mul(a.as_i16x8(), b.as_i16x8())) } +#[cfg_attr(test, assert_instr(i32x4.max_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.max_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_max(a: v128, b: v128) -> v128 { + let a = a.as_u32x4(); + let b = b.as_u32x4(); + unsafe { simd_select::(simd_gt(a, b), a, b).v128() } } -/// Negates a 128-bit vectors intepreted as four 32-bit signed integers +/// Lane-wise multiply signed 16-bit integers in the two input vectors and add +/// adjacent pairs of the full 32-bit results. #[inline] -#[cfg_attr(test, assert_instr(i32x4.neg))] -pub fn i32x4_neg(a: v128) -> v128 { - unsafe { transmute(simd_mul(a.as_i32x4(), i32x4::splat(-1))) } +#[cfg_attr(test, assert_instr(i32x4.dot_i16x8_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.dot_i16x8_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_dot_i16x8(a: v128, b: v128) -> v128 { + unsafe { llvm_i32x4_dot_i16x8_s(a.as_i16x8(), b.as_i16x8()).v128() } } -/// Returns 1 if any lane is nonzero or 0 if all lanes are zero. +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. +/// +/// Equivalent of `i32x4_mul(i32x4_extend_low_i16x8_s(a), i32x4_extend_low_i16x8_s(b))` #[inline] -#[cfg_attr(test, assert_instr(i32x4.any_true))] -pub fn i32x4_any_true(a: v128) -> i32 { - unsafe { llvm_i32x4_any_true(a.as_i32x4()) } +#[cfg_attr(test, assert_instr(i32x4.extmul_low_i16x8_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.extmul_low_i16x8_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extmul_low_i16x8(a: v128, b: v128) -> v128 { + unsafe { + let lhs = simd_cast::(simd_shuffle4!( + a.as_i16x8(), + a.as_i16x8(), + [0, 1, 2, 3] + )); + let rhs = simd_cast::(simd_shuffle4!( + b.as_i16x8(), + b.as_i16x8(), + [0, 1, 2, 3] + )); + simd_mul(lhs, rhs).v128() + } } -/// Returns 1 if all lanes are nonzero or 0 if any lane is nonzero. +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. +/// +/// Equivalent of `i32x4_mul(i32x4_extend_high_i16x8_s(a), i32x4_extend_high_i16x8_s(b))` #[inline] -#[cfg_attr(test, assert_instr(i32x4.all_true))] -pub fn i32x4_all_true(a: v128) -> i32 { - unsafe { llvm_i32x4_all_true(a.as_i32x4()) } +#[cfg_attr(test, assert_instr(i32x4.extmul_high_i16x8_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.extmul_high_i16x8_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extmul_high_i16x8(a: v128, b: v128) -> v128 { + unsafe { + let lhs = simd_cast::(simd_shuffle4!( + a.as_i16x8(), + a.as_i16x8(), + [4, 5, 6, 7] + )); + let rhs = simd_cast::(simd_shuffle4!( + b.as_i16x8(), + b.as_i16x8(), + [4, 5, 6, 7] + )); + simd_mul(lhs, rhs).v128() + } } -/// Shifts each lane to the left by the specified number of bits. +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. /// -/// Only the low bits of the shift amount are used if the shift amount is -/// greater than the lane width. +/// Equivalent of `i32x4_mul(i32x4_extend_low_u16x8(a), i32x4_extend_low_u16x8(b))` #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(i32x4.shl))] -pub fn i32x4_shl(a: v128, amt: u32) -> v128 { - unsafe { transmute(simd_shl(a.as_i32x4(), i32x4::splat(amt as i32))) } +#[cfg_attr(test, assert_instr(i32x4.extmul_low_i16x8_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.extmul_low_i16x8_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extmul_low_u16x8(a: v128, b: v128) -> v128 { + unsafe { + let lhs = simd_cast::(simd_shuffle4!( + a.as_u16x8(), + a.as_u16x8(), + [0, 1, 2, 3] + )); + let rhs = simd_cast::(simd_shuffle4!( + b.as_u16x8(), + b.as_u16x8(), + [0, 1, 2, 3] + )); + simd_mul(lhs, rhs).v128() + } } -/// Shifts each lane to the right by the specified number of bits, sign -/// extending. +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_extmul_low_u16x8 as u32x4_extmul_low_u16x8; + +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. /// -/// Only the low bits of the shift amount are used if the shift amount is -/// greater than the lane width. +/// Equivalent of `i32x4_mul(i32x4_extend_high_u16x8(a), i32x4_extend_high_u16x8(b))` #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(i32x4.shl))] -pub fn i32x4_shr_s(a: v128, amt: u32) -> v128 { - unsafe { transmute(simd_shr(a.as_i32x4(), i32x4::splat(amt as i32))) } +#[cfg_attr(test, assert_instr(i32x4.extmul_high_i16x8_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.extmul_high_i16x8_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extmul_high_u16x8(a: v128, b: v128) -> v128 { + unsafe { + let lhs = simd_cast::(simd_shuffle4!( + a.as_u16x8(), + a.as_u16x8(), + [4, 5, 6, 7] + )); + let rhs = simd_cast::(simd_shuffle4!( + b.as_u16x8(), + b.as_u16x8(), + [4, 5, 6, 7] + )); + simd_mul(lhs, rhs).v128() + } } -/// Shifts each lane to the right by the specified number of bits, shifting in -/// zeros. -/// -/// Only the low bits of the shift amount are used if the shift amount is -/// greater than the lane width. +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_extmul_high_u16x8 as u32x4_extmul_high_u16x8; + +/// Lane-wise wrapping absolute value. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(i32x4.shl))] -pub fn i32x4_shr_u(a: v128, amt: u32) -> v128 { - unsafe { transmute(simd_shr(a.as_u32x4(), u32x4::splat(amt as u32))) } +// #[cfg_attr(test, assert_instr(i64x2.abs))] // FIXME llvm +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.abs"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_abs(a: v128) -> v128 { + let a = a.as_i64x2(); + let zero = simd::i64x2::splat(0); + unsafe { + simd_select::(simd_lt(a, zero), simd_sub(zero, a), a).v128() + } } -/// Adds two 128-bit vectors as if they were two packed four 32-bit integers. +/// Negates a 128-bit vectors interpreted as two 64-bit signed integers #[inline] -#[cfg_attr(test, assert_instr(i32x4.add))] -pub fn i32x4_add(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_add(a.as_i32x4(), b.as_i32x4())) } +#[cfg_attr(test, assert_instr(i64x2.neg))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.neg"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_neg(a: v128) -> v128 { + unsafe { simd_mul(a.as_i64x2(), simd::i64x2::splat(-1)).v128() } } -/// Subtracts two 128-bit vectors as if they were two packed four 32-bit integers. +/// Returns true if all lanes are non-zero, false otherwise. #[inline] -#[cfg_attr(test, assert_instr(i32x4.sub))] -pub fn i32x4_sub(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_sub(a.as_i32x4(), b.as_i32x4())) } +#[cfg_attr(test, assert_instr(i64x2.all_true))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.all_true"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_all_true(a: v128) -> bool { + unsafe { llvm_i64x2_all_true(a.as_i64x2()) != 0 } } -/// Multiplies two 128-bit vectors as if they were two packed four 32-bit -/// signed integers. +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_all_true as u64x2_all_true; + +/// Extracts the high bit for each lane in `a` and produce a scalar mask with +/// all bits concatenated. #[inline] -#[cfg_attr(test, assert_instr(i32x4.mul))] -pub fn i32x4_mul(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_mul(a.as_i32x4(), b.as_i32x4())) } +#[cfg_attr(test, assert_instr(i64x2.bitmask))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.bitmask"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_bitmask(a: v128) -> u8 { + unsafe { llvm_bitmask_i64x2(a.as_i64x2()) as u8 } } -/// Negates a 128-bit vectors intepreted as two 64-bit signed integers +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_bitmask as u64x2_bitmask; + +/// Converts low half of the smaller lane vector to a larger lane +/// vector, sign extended. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(i32x4.neg))] -pub fn i64x2_neg(a: v128) -> v128 { - unsafe { transmute(simd_mul(a.as_i64x2(), i64x2::splat(-1))) } +#[cfg_attr(test, assert_instr(i64x2.extend_low_i32x4_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.extend_low_i32x4_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_extend_low_i32x4(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle2!(a.as_i32x4(), a.as_i32x4(), [0, 1])) + .v128() + } } -/// Returns 1 if any lane is nonzero or 0 if all lanes are zero. +/// Converts high half of the smaller lane vector to a larger lane +/// vector, sign extended. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(i64x2.any_true))] -pub fn i64x2_any_true(a: v128) -> i32 { - unsafe { llvm_i64x2_any_true(a.as_i64x2()) } +#[cfg_attr(test, assert_instr(i64x2.extend_high_i32x4_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.extend_high_i32x4_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_extend_high_i32x4(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle2!(a.as_i32x4(), a.as_i32x4(), [2, 3])) + .v128() + } } -/// Returns 1 if all lanes are nonzero or 0 if any lane is nonzero. +/// Converts low half of the smaller lane vector to a larger lane +/// vector, zero extended. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(i64x2.all_true))] -pub fn i64x2_all_true(a: v128) -> i32 { - unsafe { llvm_i64x2_all_true(a.as_i64x2()) } +#[cfg_attr(test, assert_instr(i64x2.extend_low_i32x4_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.extend_low_i32x4_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_extend_low_u32x4(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle2!(a.as_u32x4(), a.as_u32x4(), [0, 1])) + .v128() + } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_extend_low_u32x4 as u64x2_extend_low_u32x4; + +/// Converts high half of the smaller lane vector to a larger lane +/// vector, zero extended. +#[inline] +#[cfg_attr(test, assert_instr(i64x2.extend_high_i32x4_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.extend_high_i32x4_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_extend_high_u32x4(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle2!(a.as_u32x4(), a.as_u32x4(), [2, 3])) + .v128() + } } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_extend_high_u32x4 as u64x2_extend_high_u32x4; + /// Shifts each lane to the left by the specified number of bits. /// /// Only the low bits of the shift amount are used if the shift amount is /// greater than the lane width. #[inline] -#[cfg(not(only_node_compatible_functions))] #[cfg_attr(test, assert_instr(i64x2.shl))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.shl"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub fn i64x2_shl(a: v128, amt: u32) -> v128 { - unsafe { transmute(simd_shl(a.as_i64x2(), i64x2::splat(amt as i64))) } + unsafe { simd_shl(a.as_i64x2(), simd::i64x2::splat(amt as i64)).v128() } } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_shl as u64x2_shl; + /// Shifts each lane to the right by the specified number of bits, sign /// extending. /// /// Only the low bits of the shift amount are used if the shift amount is /// greater than the lane width. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(i64x2.shl))] -pub fn i64x2_shr_s(a: v128, amt: u32) -> v128 { - unsafe { transmute(simd_shr(a.as_i64x2(), i64x2::splat(amt as i64))) } +#[cfg_attr(test, assert_instr(i64x2.shr_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.shr_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_shr(a: v128, amt: u32) -> v128 { + unsafe { simd_shr(a.as_i64x2(), simd::i64x2::splat(amt as i64)).v128() } } /// Shifts each lane to the right by the specified number of bits, shifting in @@ -1346,271 +3607,677 @@ pub fn i64x2_shr_s(a: v128, amt: u32) -> v128 { /// Only the low bits of the shift amount are used if the shift amount is /// greater than the lane width. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(i64x2.shl))] -pub fn i64x2_shr_u(a: v128, amt: u32) -> v128 { - unsafe { transmute(simd_shr(a.as_u64x2(), u64x2::splat(amt as u64))) } +#[cfg_attr(test, assert_instr(i64x2.shr_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.shr_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u64x2_shr(a: v128, amt: u32) -> v128 { + unsafe { simd_shr(a.as_u64x2(), simd::u64x2::splat(amt as u64)).v128() } } /// Adds two 128-bit vectors as if they were two packed two 64-bit integers. #[inline] -#[cfg(not(only_node_compatible_functions))] #[cfg_attr(test, assert_instr(i64x2.add))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.add"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub fn i64x2_add(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_add(a.as_i64x2(), b.as_i64x2())) } + unsafe { simd_add(a.as_i64x2(), b.as_i64x2()).v128() } } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_add as u64x2_add; + /// Subtracts two 128-bit vectors as if they were two packed two 64-bit integers. #[inline] -#[cfg(not(only_node_compatible_functions))] #[cfg_attr(test, assert_instr(i64x2.sub))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.sub"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub fn i64x2_sub(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_sub(a.as_i64x2(), b.as_i64x2())) } + unsafe { simd_sub(a.as_i64x2(), b.as_i64x2()).v128() } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_sub as u64x2_sub; + +/// Multiplies two 128-bit vectors as if they were two packed two 64-bit integers. +#[inline] +#[cfg_attr(test, assert_instr(i64x2.mul))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.mul"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_mul(a: v128, b: v128) -> v128 { + unsafe { simd_mul(a.as_i64x2(), b.as_i64x2()).v128() } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_mul as u64x2_mul; + +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. +/// +/// Equivalent of `i64x2_mul(i64x2_extend_low_i32x4_s(a), i64x2_extend_low_i32x4_s(b))` +#[inline] +#[cfg_attr(test, assert_instr(i64x2.extmul_low_i32x4_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.extmul_low_i32x4_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_extmul_low_i32x4(a: v128, b: v128) -> v128 { + unsafe { + let lhs = simd_cast::(simd_shuffle2!( + a.as_i32x4(), + a.as_i32x4(), + [0, 1] + )); + let rhs = simd_cast::(simd_shuffle2!( + b.as_i32x4(), + b.as_i32x4(), + [0, 1] + )); + simd_mul(lhs, rhs).v128() + } +} + +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. +/// +/// Equivalent of `i64x2_mul(i64x2_extend_high_i32x4_s(a), i64x2_extend_high_i32x4_s(b))` +#[inline] +#[cfg_attr(test, assert_instr(i64x2.extmul_high_i32x4_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.extmul_high_i32x4_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_extmul_high_i32x4(a: v128, b: v128) -> v128 { + unsafe { + let lhs = simd_cast::(simd_shuffle2!( + a.as_i32x4(), + a.as_i32x4(), + [2, 3] + )); + let rhs = simd_cast::(simd_shuffle2!( + b.as_i32x4(), + b.as_i32x4(), + [2, 3] + )); + simd_mul(lhs, rhs).v128() + } +} + +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. +/// +/// Equivalent of `i64x2_mul(i64x2_extend_low_i32x4_u(a), i64x2_extend_low_i32x4_u(b))` +#[inline] +#[cfg_attr(test, assert_instr(i64x2.extmul_low_i32x4_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.extmul_low_i32x4_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_extmul_low_u32x4(a: v128, b: v128) -> v128 { + unsafe { + let lhs = simd_cast::(simd_shuffle2!( + a.as_u32x4(), + a.as_u32x4(), + [0, 1] + )); + let rhs = simd_cast::(simd_shuffle2!( + b.as_u32x4(), + b.as_u32x4(), + [0, 1] + )); + simd_mul(lhs, rhs).v128() + } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_extmul_low_u32x4 as u64x2_extmul_low_u32x4; + +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. +/// +/// Equivalent of `i64x2_mul(i64x2_extend_high_i32x4_u(a), i64x2_extend_high_i32x4_u(b))` +#[inline] +#[cfg_attr(test, assert_instr(i64x2.extmul_high_i32x4_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.extmul_high_i32x4_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_extmul_high_u32x4(a: v128, b: v128) -> v128 { + unsafe { + let lhs = simd_cast::(simd_shuffle2!( + a.as_u32x4(), + a.as_u32x4(), + [2, 3] + )); + let rhs = simd_cast::(simd_shuffle2!( + b.as_u32x4(), + b.as_u32x4(), + [2, 3] + )); + simd_mul(lhs, rhs).v128() + } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_extmul_high_u32x4 as u64x2_extmul_high_u32x4; + +/// Lane-wise rounding to the nearest integral value not smaller than the input. +#[inline] +#[cfg_attr(test, assert_instr(f32x4.ceil))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.ceil"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_ceil(a: v128) -> v128 { + unsafe { llvm_f32x4_ceil(a.as_f32x4()).v128() } +} + +/// Lane-wise rounding to the nearest integral value not greater than the input. +#[inline] +#[cfg_attr(test, assert_instr(f32x4.floor))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.floor"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_floor(a: v128) -> v128 { + unsafe { llvm_f32x4_floor(a.as_f32x4()).v128() } +} + +/// Lane-wise rounding to the nearest integral value with the magnitude not +/// larger than the input. +#[inline] +#[cfg_attr(test, assert_instr(f32x4.trunc))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.trunc"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_trunc(a: v128) -> v128 { + unsafe { llvm_f32x4_trunc(a.as_f32x4()).v128() } +} + +/// Lane-wise rounding to the nearest integral value; if two values are equally +/// near, rounds to the even one. +#[inline] +#[cfg_attr(test, assert_instr(f32x4.nearest))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.nearest"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_nearest(a: v128) -> v128 { + unsafe { llvm_f32x4_nearest(a.as_f32x4()).v128() } } /// Calculates the absolute value of each lane of a 128-bit vector interpreted /// as four 32-bit floating point numbers. #[inline] #[cfg_attr(test, assert_instr(f32x4.abs))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.abs"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub fn f32x4_abs(a: v128) -> v128 { - unsafe { transmute(llvm_f32x4_abs(a.as_f32x4())) } + unsafe { llvm_f32x4_abs(a.as_f32x4()).v128() } } /// Negates each lane of a 128-bit vector interpreted as four 32-bit floating /// point numbers. #[inline] #[cfg_attr(test, assert_instr(f32x4.neg))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.neg"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub fn f32x4_neg(a: v128) -> v128 { - unsafe { f32x4_mul(a, transmute(f32x4(-1.0, -1.0, -1.0, -1.0))) } + f32x4_mul(a, f32x4_splat(-1.)) } /// Calculates the square root of each lane of a 128-bit vector interpreted as /// four 32-bit floating point numbers. #[inline] -#[cfg(not(only_node_compatible_functions))] #[cfg_attr(test, assert_instr(f32x4.sqrt))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.sqrt"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub fn f32x4_sqrt(a: v128) -> v128 { - unsafe { transmute(llvm_f32x4_sqrt(a.as_f32x4())) } + unsafe { llvm_f32x4_sqrt(a.as_f32x4()).v128() } } -/// Adds pairwise lanes of two 128-bit vectors interpreted as four 32-bit +/// Lane-wise addition of two 128-bit vectors interpreted as four 32-bit /// floating point numbers. #[inline] #[cfg_attr(test, assert_instr(f32x4.add))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.add"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub fn f32x4_add(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_add(a.as_f32x4(), b.as_f32x4())) } + unsafe { simd_add(a.as_f32x4(), b.as_f32x4()).v128() } } -/// Subtracts pairwise lanes of two 128-bit vectors interpreted as four 32-bit +/// Lane-wise subtraction of two 128-bit vectors interpreted as four 32-bit /// floating point numbers. #[inline] #[cfg_attr(test, assert_instr(f32x4.sub))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.sub"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub fn f32x4_sub(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_sub(a.as_f32x4(), b.as_f32x4())) } + unsafe { simd_sub(a.as_f32x4(), b.as_f32x4()).v128() } } -/// Multiplies pairwise lanes of two 128-bit vectors interpreted as four 32-bit +/// Lane-wise multiplication of two 128-bit vectors interpreted as four 32-bit /// floating point numbers. #[inline] #[cfg_attr(test, assert_instr(f32x4.mul))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.mul"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub fn f32x4_mul(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_mul(a.as_f32x4(), b.as_f32x4())) } + unsafe { simd_mul(a.as_f32x4(), b.as_f32x4()).v128() } } -/// Divides pairwise lanes of two 128-bit vectors interpreted as four 32-bit +/// Lane-wise division of two 128-bit vectors interpreted as four 32-bit /// floating point numbers. #[inline] -#[cfg(not(only_node_compatible_functions))] #[cfg_attr(test, assert_instr(f32x4.div))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.div"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub fn f32x4_div(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_div(a.as_f32x4(), b.as_f32x4())) } + unsafe { simd_div(a.as_f32x4(), b.as_f32x4()).v128() } } -/// Calculates the minimum of pairwise lanes of two 128-bit vectors interpreted +/// Calculates the lane-wise minimum of two 128-bit vectors interpreted /// as four 32-bit floating point numbers. #[inline] #[cfg_attr(test, assert_instr(f32x4.min))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.min"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub fn f32x4_min(a: v128, b: v128) -> v128 { - unsafe { transmute(llvm_f32x4_min(a.as_f32x4(), b.as_f32x4())) } + unsafe { llvm_f32x4_min(a.as_f32x4(), b.as_f32x4()).v128() } } -/// Calculates the maximum of pairwise lanes of two 128-bit vectors interpreted +/// Calculates the lane-wise minimum of two 128-bit vectors interpreted /// as four 32-bit floating point numbers. #[inline] #[cfg_attr(test, assert_instr(f32x4.max))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.max"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub fn f32x4_max(a: v128, b: v128) -> v128 { - unsafe { transmute(llvm_f32x4_max(a.as_f32x4(), b.as_f32x4())) } + unsafe { llvm_f32x4_max(a.as_f32x4(), b.as_f32x4()).v128() } +} + +/// Lane-wise minimum value, defined as `b < a ? b : a` +#[inline] +#[cfg_attr(test, assert_instr(f32x4.pmin))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.pmin"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_pmin(a: v128, b: v128) -> v128 { + unsafe { + simd_select::( + simd_lt(b.as_f32x4(), a.as_f32x4()), + b.as_f32x4(), + a.as_f32x4(), + ) + .v128() + } +} + +/// Lane-wise maximum value, defined as `a < b ? b : a` +#[inline] +#[cfg_attr(test, assert_instr(f32x4.pmax))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.pmax"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_pmax(a: v128, b: v128) -> v128 { + unsafe { + simd_select::( + simd_lt(a.as_f32x4(), b.as_f32x4()), + b.as_f32x4(), + a.as_f32x4(), + ) + .v128() + } +} + +/// Lane-wise rounding to the nearest integral value not smaller than the input. +#[inline] +#[cfg_attr(test, assert_instr(f64x2.ceil))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.ceil"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_ceil(a: v128) -> v128 { + unsafe { llvm_f64x2_ceil(a.as_f64x2()).v128() } +} + +/// Lane-wise rounding to the nearest integral value not greater than the input. +#[inline] +#[cfg_attr(test, assert_instr(f64x2.floor))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.floor"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_floor(a: v128) -> v128 { + unsafe { llvm_f64x2_floor(a.as_f64x2()).v128() } +} + +/// Lane-wise rounding to the nearest integral value with the magnitude not +/// larger than the input. +#[inline] +#[cfg_attr(test, assert_instr(f64x2.trunc))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.trunc"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_trunc(a: v128) -> v128 { + unsafe { llvm_f64x2_trunc(a.as_f64x2()).v128() } +} + +/// Lane-wise rounding to the nearest integral value; if two values are equally +/// near, rounds to the even one. +#[inline] +#[cfg_attr(test, assert_instr(f64x2.nearest))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.nearest"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_nearest(a: v128) -> v128 { + unsafe { llvm_f64x2_nearest(a.as_f64x2()).v128() } } /// Calculates the absolute value of each lane of a 128-bit vector interpreted /// as two 64-bit floating point numbers. #[inline] -#[cfg(not(only_node_compatible_functions))] #[cfg_attr(test, assert_instr(f64x2.abs))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.abs"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub fn f64x2_abs(a: v128) -> v128 { - unsafe { transmute(llvm_f64x2_abs(a.as_f64x2())) } + unsafe { llvm_f64x2_abs(a.as_f64x2()).v128() } } /// Negates each lane of a 128-bit vector interpreted as two 64-bit floating /// point numbers. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr(f64x2.abs))] +#[cfg_attr(test, assert_instr(f64x2.neg))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.neg"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub fn f64x2_neg(a: v128) -> v128 { - unsafe { f64x2_mul(a, transmute(f64x2(-1.0, -1.0))) } + f64x2_mul(a, f64x2_splat(-1.0)) } /// Calculates the square root of each lane of a 128-bit vector interpreted as /// two 64-bit floating point numbers. #[inline] -#[cfg(not(only_node_compatible_functions))] #[cfg_attr(test, assert_instr(f64x2.sqrt))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.sqrt"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub fn f64x2_sqrt(a: v128) -> v128 { - unsafe { transmute(llvm_f64x2_sqrt(a.as_f64x2())) } + unsafe { llvm_f64x2_sqrt(a.as_f64x2()).v128() } } -/// Adds pairwise lanes of two 128-bit vectors interpreted as two 64-bit +/// Lane-wise add of two 128-bit vectors interpreted as two 64-bit /// floating point numbers. #[inline] -#[cfg(not(only_node_compatible_functions))] #[cfg_attr(test, assert_instr(f64x2.add))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.add"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub fn f64x2_add(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_add(a.as_f64x2(), b.as_f64x2())) } + unsafe { simd_add(a.as_f64x2(), b.as_f64x2()).v128() } } -/// Subtracts pairwise lanes of two 128-bit vectors interpreted as two 64-bit +/// Lane-wise subtract of two 128-bit vectors interpreted as two 64-bit /// floating point numbers. #[inline] -#[cfg(not(only_node_compatible_functions))] #[cfg_attr(test, assert_instr(f64x2.sub))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.sub"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub fn f64x2_sub(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_sub(a.as_f64x2(), b.as_f64x2())) } + unsafe { simd_sub(a.as_f64x2(), b.as_f64x2()).v128() } } -/// Multiplies pairwise lanes of two 128-bit vectors interpreted as two 64-bit +/// Lane-wise multiply of two 128-bit vectors interpreted as two 64-bit /// floating point numbers. #[inline] -#[cfg(not(only_node_compatible_functions))] #[cfg_attr(test, assert_instr(f64x2.mul))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.mul"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub fn f64x2_mul(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_mul(a.as_f64x2(), b.as_f64x2())) } + unsafe { simd_mul(a.as_f64x2(), b.as_f64x2()).v128() } } -/// Divides pairwise lanes of two 128-bit vectors interpreted as two 64-bit +/// Lane-wise divide of two 128-bit vectors interpreted as two 64-bit /// floating point numbers. #[inline] -#[cfg(not(only_node_compatible_functions))] #[cfg_attr(test, assert_instr(f64x2.div))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.div"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub fn f64x2_div(a: v128, b: v128) -> v128 { - unsafe { transmute(simd_div(a.as_f64x2(), b.as_f64x2())) } + unsafe { simd_div(a.as_f64x2(), b.as_f64x2()).v128() } } -/// Calculates the minimum of pairwise lanes of two 128-bit vectors interpreted +/// Calculates the lane-wise minimum of two 128-bit vectors interpreted /// as two 64-bit floating point numbers. #[inline] -#[cfg(not(only_node_compatible_functions))] #[cfg_attr(test, assert_instr(f64x2.min))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.min"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub fn f64x2_min(a: v128, b: v128) -> v128 { - unsafe { transmute(llvm_f64x2_min(a.as_f64x2(), b.as_f64x2())) } + unsafe { llvm_f64x2_min(a.as_f64x2(), b.as_f64x2()).v128() } } -/// Calculates the maximum of pairwise lanes of two 128-bit vectors interpreted +/// Calculates the lane-wise maximum of two 128-bit vectors interpreted /// as two 64-bit floating point numbers. #[inline] -#[cfg(not(only_node_compatible_functions))] #[cfg_attr(test, assert_instr(f64x2.max))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.max"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub fn f64x2_max(a: v128, b: v128) -> v128 { - unsafe { transmute(llvm_f64x2_max(a.as_f64x2(), b.as_f64x2())) } + unsafe { llvm_f64x2_max(a.as_f64x2(), b.as_f64x2()).v128() } } -/// Converts a 128-bit vector interpreted as four 32-bit floating point numbers -/// into a 128-bit vector of four 32-bit signed integers. -/// -/// NaN is converted to 0 and if it's out of bounds it becomes the nearest -/// representable intger. +/// Lane-wise minimum value, defined as `b < a ? b : a` #[inline] -#[cfg_attr(test, assert_instr("i32x4.trunc_sat_f32x4_s"))] -pub fn i32x4_trunc_s_f32x4_sat(a: v128) -> v128 { - unsafe { transmute(simd_cast::<_, i32x4>(a.as_f32x4())) } +#[cfg_attr(test, assert_instr(f64x2.pmin))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.pmin"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_pmin(a: v128, b: v128) -> v128 { + unsafe { + simd_select::( + simd_lt(b.as_f64x2(), a.as_f64x2()), + b.as_f64x2(), + a.as_f64x2(), + ) + .v128() + } } -/// Converts a 128-bit vector interpreted as four 32-bit floating point numbers -/// into a 128-bit vector of four 32-bit unsigned integers. -/// -/// NaN is converted to 0 and if it's out of bounds it becomes the nearest -/// representable intger. +/// Lane-wise maximum value, defined as `a < b ? b : a` #[inline] -#[cfg_attr(test, assert_instr("i32x4.trunc_sat_f32x4_u"))] -pub fn i32x4_trunc_u_f32x4_sat(a: v128) -> v128 { - unsafe { transmute(simd_cast::<_, u32x4>(a.as_f32x4())) } +#[cfg_attr(test, assert_instr(f64x2.pmax))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.pmax"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_pmax(a: v128, b: v128) -> v128 { + unsafe { + simd_select::( + simd_lt(a.as_f64x2(), b.as_f64x2()), + b.as_f64x2(), + a.as_f64x2(), + ) + .v128() + } } -/// Converts a 128-bit vector interpreted as two 64-bit floating point numbers -/// into a 128-bit vector of two 64-bit signed integers. +/// Converts a 128-bit vector interpreted as four 32-bit floating point numbers +/// into a 128-bit vector of four 32-bit signed integers. /// /// NaN is converted to 0 and if it's out of bounds it becomes the nearest /// representable intger. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr("i64x2.trunc_s/f64x2:sat"))] -pub fn i64x2_trunc_s_f64x2_sat(a: v128) -> v128 { - unsafe { transmute(simd_cast::<_, i64x2>(a.as_f64x2())) } +#[cfg_attr(test, assert_instr(i32x4.trunc_sat_f32x4_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.trunc_sat_f32x4_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_trunc_sat_f32x4(a: v128) -> v128 { + unsafe { llvm_i32x4_trunc_sat_f32x4_s(a.as_f32x4()).v128() } } -/// Converts a 128-bit vector interpreted as two 64-bit floating point numbers -/// into a 128-bit vector of two 64-bit unsigned integers. +/// Converts a 128-bit vector interpreted as four 32-bit floating point numbers +/// into a 128-bit vector of four 32-bit unsigned integers. /// /// NaN is converted to 0 and if it's out of bounds it becomes the nearest /// representable intger. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr("i64x2.trunc_u/f64x2:sat"))] -pub fn i64x2_trunc_u_f64x2_sat(a: v128) -> v128 { - unsafe { transmute(simd_cast::<_, u64x2>(a.as_f64x2())) } +#[cfg_attr(test, assert_instr(i32x4.trunc_sat_f32x4_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.trunc_sat_f32x4_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_trunc_sat_f32x4(a: v128) -> v128 { + unsafe { llvm_i32x4_trunc_sat_f32x4_u(a.as_f32x4()).v128() } } /// Converts a 128-bit vector interpreted as four 32-bit signed integers into a /// 128-bit vector of four 32-bit floating point numbers. #[inline] -#[cfg_attr(test, assert_instr("f32x4.convert_i32x4_s"))] -pub fn f32x4_convert_i32x4_s(a: v128) -> v128 { - unsafe { transmute(simd_cast::<_, f32x4>(a.as_i32x4())) } +#[cfg_attr(test, assert_instr(f32x4.convert_i32x4_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.convert_i32x4_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_convert_i32x4(a: v128) -> v128 { + unsafe { simd_cast::<_, simd::f32x4>(a.as_i32x4()).v128() } } /// Converts a 128-bit vector interpreted as four 32-bit unsigned integers into a /// 128-bit vector of four 32-bit floating point numbers. #[inline] -#[cfg_attr(test, assert_instr("f32x4.convert_i32x4_u"))] -pub fn f32x4_convert_i32x4_u(a: v128) -> v128 { - unsafe { transmute(simd_cast::<_, f32x4>(a.as_u32x4())) } +#[cfg_attr(test, assert_instr(f32x4.convert_i32x4_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.convert_i32x4_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_convert_u32x4(a: v128) -> v128 { + unsafe { simd_cast::<_, simd::f32x4>(a.as_u32x4()).v128() } +} + +/// Saturating conversion of the two double-precision floating point lanes to +/// two lower integer lanes using the IEEE `convertToIntegerTowardZero` +/// function. +/// +/// The two higher lanes of the result are initialized to zero. If any input +/// lane is a NaN, the resulting lane is 0. If the rounded integer value of a +/// lane is outside the range of the destination type, the result is saturated +/// to the nearest representable integer value. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.trunc_sat_f64x2_s_zero))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.trunc_sat_f64x2_s_zero"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_trunc_sat_f64x2_zero(a: v128) -> v128 { + let ret: simd::i32x4 = unsafe { + simd_shuffle4!( + llvm_i32x2_trunc_sat_f64x2_s(a.as_f64x2()), + simd::i32x2::splat(0), + [0, 1, 2, 3], + ) + }; + ret.v128() +} + +/// Saturating conversion of the two double-precision floating point lanes to +/// two lower integer lanes using the IEEE `convertToIntegerTowardZero` +/// function. +/// +/// The two higher lanes of the result are initialized to zero. If any input +/// lane is a NaN, the resulting lane is 0. If the rounded integer value of a +/// lane is outside the range of the destination type, the result is saturated +/// to the nearest representable integer value. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.trunc_sat_f64x2_u_zero))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.trunc_sat_f64x2_u_zero"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_trunc_sat_f64x2_zero(a: v128) -> v128 { + let ret: simd::i32x4 = unsafe { + simd_shuffle4!( + llvm_i32x2_trunc_sat_f64x2_u(a.as_f64x2()), + simd::i32x2::splat(0), + [0, 1, 2, 3], + ) + }; + ret.v128() +} + +/// Lane-wise conversion from integer to floating point. +#[inline] +#[cfg_attr(test, assert_instr(f64x2.convert_low_i32x4_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.convert_low_i32x4_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_convert_low_i32x4(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle2!(a.as_i32x4(), a.as_i32x4(), [0, 1],)) + .v128() + } +} + +/// Lane-wise conversion from integer to floating point. +#[inline] +#[cfg_attr(test, assert_instr(f64x2.convert_low_i32x4_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.convert_low_i32x4_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_convert_low_u32x4(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle2!(a.as_u32x4(), a.as_u32x4(), [0, 1],)) + .v128() + } } -/// Converts a 128-bit vector interpreted as two 64-bit signed integers into a -/// 128-bit vector of two 64-bit floating point numbers. +/// Conversion of the two double-precision floating point lanes to two lower +/// single-precision lanes of the result. The two higher lanes of the result are +/// initialized to zero. If the conversion result is not representable as a +/// single-precision floating point number, it is rounded to the nearest-even +/// representable number. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr("f64x2.convert_s/i64x2"))] -pub fn f64x2_convert_s_i64x2(a: v128) -> v128 { - unsafe { transmute(simd_cast::<_, f64x2>(a.as_i64x2())) } +#[cfg_attr(test, assert_instr(f32x4.demote_f64x2_zero))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.demote_f64x2_zero"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_demote_f64x2_zero(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle4!( + a.as_f64x2(), + simd::f64x2::splat(0.0), + [0, 1, 2, 3] + )) + .v128() + } } -/// Converts a 128-bit vector interpreted as two 64-bit unsigned integers into a -/// 128-bit vector of two 64-bit floating point numbers. +/// Conversion of the two lower single-precision floating point lanes to the two +/// double-precision lanes of the result. #[inline] -#[cfg(not(only_node_compatible_functions))] -#[cfg_attr(test, assert_instr("f64x2.convert_u/i64x2"))] -pub fn f64x2_convert_u_i64x2(a: v128) -> v128 { - unsafe { transmute(simd_cast::<_, f64x2>(a.as_u64x2())) } +#[cfg_attr(test, assert_instr(f64x2.promote_low_f32x4))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.promote_low_f32x4"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_promote_low_f32x4(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle2!(a.as_f32x4(), a.as_f32x4(), [0, 1])) + .v128() + } } #[cfg(test)] pub mod tests { use super::*; + use core::ops::{Add, Div, Mul, Neg, Sub}; use std; - use std::mem; + use std::fmt::Debug; + use std::mem::transmute; use std::num::Wrapping; use std::prelude::v1::*; - use wasm_bindgen_test::*; fn compare_bytes(a: v128, b: v128) { let a: [u8; 16] = unsafe { transmute(a) }; @@ -1618,35 +4285,168 @@ pub mod tests { assert_eq!(a, b); } - #[wasm_bindgen_test] - #[cfg(not(only_node_compatible_functions))] - fn test_v128_const() { - const A: v128 = - unsafe { super::v128_const(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15) }; - compare_bytes(A, A); + #[test] + fn test_load() { + unsafe { + let arr: [i32; 4] = [0, 1, 2, 3]; + let vec = v128_load(arr.as_ptr() as *const v128); + compare_bytes(vec, i32x4(0, 1, 2, 3)); + } } - macro_rules! test_splat { - ($test_id:ident: $val:expr => $($vals:expr),*) => { - #[wasm_bindgen_test] - fn $test_id() { - let a = super::$test_id($val); - let b: v128 = unsafe { - transmute([$($vals as u8),*]) - }; - compare_bytes(a, b); - } + #[test] + fn test_load_extend() { + unsafe { + let arr: [i8; 8] = [-3, -2, -1, 0, 1, 2, 3, 4]; + let vec = i16x8_load_extend_i8x8(arr.as_ptr()); + compare_bytes(vec, i16x8(-3, -2, -1, 0, 1, 2, 3, 4)); + let vec = i16x8_load_extend_u8x8(arr.as_ptr() as *const u8); + compare_bytes(vec, i16x8(253, 254, 255, 0, 1, 2, 3, 4)); + + let arr: [i16; 4] = [-1, 0, 1, 2]; + let vec = i32x4_load_extend_i16x4(arr.as_ptr()); + compare_bytes(vec, i32x4(-1, 0, 1, 2)); + let vec = i32x4_load_extend_u16x4(arr.as_ptr() as *const u16); + compare_bytes(vec, i32x4(65535, 0, 1, 2)); + + let arr: [i32; 2] = [-1, 1]; + let vec = i64x2_load_extend_i32x2(arr.as_ptr()); + compare_bytes(vec, i64x2(-1, 1)); + let vec = i64x2_load_extend_u32x2(arr.as_ptr() as *const u32); + compare_bytes(vec, i64x2(u32::max_value().into(), 1)); + } + } + + #[test] + fn test_load_splat() { + unsafe { + compare_bytes(v128_load8_splat(&8), i8x16_splat(8)); + compare_bytes(v128_load16_splat(&9), i16x8_splat(9)); + compare_bytes(v128_load32_splat(&10), i32x4_splat(10)); + compare_bytes(v128_load64_splat(&11), i64x2_splat(11)); + } + } + + #[test] + fn test_load_zero() { + unsafe { + compare_bytes(v128_load32_zero(&10), i32x4(10, 0, 0, 0)); + compare_bytes(v128_load64_zero(&11), i64x2(11, 0)); + } + } + + #[test] + fn test_store() { + unsafe { + let mut spot = i8x16_splat(0); + v128_store(&mut spot, i8x16_splat(1)); + compare_bytes(spot, i8x16_splat(1)); + } + } + + #[test] + fn test_load_lane() { + unsafe { + let zero = i8x16_splat(0); + compare_bytes( + v128_load8_lane::<2>(zero, &1), + i8x16_replace_lane::<2>(zero, 1), + ); + + compare_bytes( + v128_load16_lane::<2>(zero, &1), + i16x8_replace_lane::<2>(zero, 1), + ); + + compare_bytes( + v128_load32_lane::<2>(zero, &1), + i32x4_replace_lane::<2>(zero, 1), + ); + + compare_bytes( + v128_load64_lane::<1>(zero, &1), + i64x2_replace_lane::<1>(zero, 1), + ); } } - test_splat!(i8x16_splat: 42 => 42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42); - test_splat!(i16x8_splat: 42 => 42, 0, 42, 0, 42, 0, 42, 0, 42, 0, 42, 0, 42, 0, 42, 0); - test_splat!(i32x4_splat: 42 => 42, 0, 0, 0, 42, 0, 0, 0, 42, 0, 0, 0, 42, 0, 0, 0); - #[cfg(not(only_node_compatible_functions))] - test_splat!(i64x2_splat: 42 => 42, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0); - test_splat!(f32x4_splat: 42. => 0, 0, 40, 66, 0, 0, 40, 66, 0, 0, 40, 66, 0, 0, 40, 66); - #[cfg(not(only_node_compatible_functions))] - test_splat!(f64x2_splat: 42. => 0, 0, 0, 0, 0, 0, 69, 64, 0, 0, 0, 0, 0, 0, 69, 64); + #[test] + fn test_store_lane() { + unsafe { + let mut spot = 0; + let zero = i8x16_splat(0); + v128_store8_lane::<5>(i8x16_replace_lane::<5>(zero, 7), &mut spot); + assert_eq!(spot, 7); + + let mut spot = 0; + v128_store16_lane::<5>(i16x8_replace_lane::<5>(zero, 7), &mut spot); + assert_eq!(spot, 7); + + let mut spot = 0; + v128_store32_lane::<3>(i32x4_replace_lane::<3>(zero, 7), &mut spot); + assert_eq!(spot, 7); + + let mut spot = 0; + v128_store64_lane::<0>(i64x2_replace_lane::<0>(zero, 7), &mut spot); + assert_eq!(spot, 7); + } + } + + #[test] + fn test_i8x16() { + const A: v128 = super::i8x16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + compare_bytes(A, A); + + const _: v128 = i16x8(0, 1, 2, 3, 4, 5, 6, 7); + const _: v128 = i32x4(0, 1, 2, 3); + const _: v128 = i64x2(0, 1); + const _: v128 = f32x4(0., 1., 2., 3.); + const _: v128 = f64x2(0., 1.); + + let bytes: [i16; 8] = unsafe { mem::transmute(i16x8(-1, -2, -3, -4, -5, -6, -7, -8)) }; + assert_eq!(bytes, [-1, -2, -3, -4, -5, -6, -7, -8]); + let bytes: [i8; 16] = unsafe { + mem::transmute(i8x16( + -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, + )) + }; + assert_eq!( + bytes, + [-1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16] + ); + } + + #[test] + fn test_shuffle() { + let vec_a = i8x16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let vec_b = i8x16( + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ); + + let vec_r = i8x16_shuffle::<0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30>( + vec_a, vec_b, + ); + let vec_e = i8x16(0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30); + compare_bytes(vec_r, vec_e); + + let vec_a = i16x8(0, 1, 2, 3, 4, 5, 6, 7); + let vec_b = i16x8(8, 9, 10, 11, 12, 13, 14, 15); + let vec_r = i16x8_shuffle::<0, 8, 2, 10, 4, 12, 6, 14>(vec_a, vec_b); + let vec_e = i16x8(0, 8, 2, 10, 4, 12, 6, 14); + compare_bytes(vec_r, vec_e); + + let vec_a = i32x4(0, 1, 2, 3); + let vec_b = i32x4(4, 5, 6, 7); + let vec_r = i32x4_shuffle::<0, 4, 2, 6>(vec_a, vec_b); + let vec_e = i32x4(0, 4, 2, 6); + compare_bytes(vec_r, vec_e); + + let vec_a = i64x2(0, 1); + let vec_b = i64x2(2, 3); + let vec_r = i64x2_shuffle::<0, 2>(vec_a, vec_b); + let vec_e = i64x2(0, 2); + compare_bytes(vec_r, vec_e); + } // tests extract and replace lanes macro_rules! test_extract { @@ -1658,25 +4458,25 @@ pub mod tests { count: $count:expr, indices: [$($idx:expr),*], ) => { - #[wasm_bindgen_test] + #[test] fn $test_id() { unsafe { let arr: [$elem; $count] = [123 as $elem; $count]; let vec: v128 = transmute(arr); $( - assert_eq!($extract(vec, $idx), 123 as $elem); - )*; + assert_eq!($extract::<$idx>(vec), 123 as $elem); + )* // create a vector from array and check that the indices contain // the same values as in the array: let arr: [$elem; $count] = [$($idx as $elem),*]; let vec: v128 = transmute(arr); $( - assert_eq!($extract(vec, $idx), $idx as $elem); + assert_eq!($extract::<$idx>(vec), $idx as $elem); - let tmp = $replace(vec, $idx, 124 as $elem); - assert_eq!($extract(tmp, $idx), 124 as $elem); - )*; + let tmp = $replace::<$idx>(vec, 124 as $elem); + assert_eq!($extract::<$idx>(tmp), 124 as $elem); + )* } } } @@ -1706,7 +4506,6 @@ pub mod tests { count: 4, indices: [0, 1, 2, 3], } - #[cfg(not(only_node_compatible_functions))] test_extract! { name: test_i64x2_extract_replace, extract: i64x2_extract_lane, @@ -1723,23 +4522,228 @@ pub mod tests { count: 4, indices: [0, 1, 2, 3], } - #[cfg(not(only_node_compatible_functions))] - test_extract! { - name: test_f64x2_extract_replace, - extract: f64x2_extract_lane, - replace: f64x2_replace_lane, - elem: f64, - count: 2, - indices: [0, 1], + test_extract! { + name: test_f64x2_extract_replace, + extract: f64x2_extract_lane, + replace: f64x2_replace_lane, + elem: f64, + count: 2, + indices: [0, 1], + } + + #[test] + #[rustfmt::skip] + fn test_swizzle() { + compare_bytes( + i8x16_swizzle( + i32x4(1, 2, 3, 4), + i8x16( + 32, 31, 30, 29, + 0, 1, 2, 3, + 12, 13, 14, 15, + 0, 4, 8, 12), + ), + i32x4(0, 1, 4, 0x04030201), + ); + } + + macro_rules! test_splat { + ($test_id:ident: $val:expr => $($vals:expr),*) => { + #[test] + fn $test_id() { + let a = super::$test_id($val); + let b = u8x16($($vals as u8),*); + compare_bytes(a, b); + } + } + } + + mod splats { + use super::*; + test_splat!(i8x16_splat: 42 => 42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42); + test_splat!(i16x8_splat: 42 => 42, 0, 42, 0, 42, 0, 42, 0, 42, 0, 42, 0, 42, 0, 42, 0); + test_splat!(i32x4_splat: 42 => 42, 0, 0, 0, 42, 0, 0, 0, 42, 0, 0, 0, 42, 0, 0, 0); + test_splat!(i64x2_splat: 42 => 42, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0); + test_splat!(f32x4_splat: 42. => 0, 0, 40, 66, 0, 0, 40, 66, 0, 0, 40, 66, 0, 0, 40, 66); + test_splat!(f64x2_splat: 42. => 0, 0, 0, 0, 0, 0, 69, 64, 0, 0, 0, 0, 0, 0, 69, 64); + } + + #[test] + fn test_bitmasks() { + let zero = i8x16_splat(0); + let ones = i8x16_splat(!0); + + assert_eq!(i8x16_bitmask(zero), 0); + assert_eq!(i8x16_bitmask(ones), 0xffff); + assert_eq!(i8x16_bitmask(i8x16_splat(i8::MAX)), 0); + assert_eq!(i8x16_bitmask(i8x16_splat(i8::MIN)), 0xffff); + assert_eq!(i8x16_bitmask(i8x16_replace_lane::<1>(zero, -1)), 0b10); + + assert_eq!(i16x8_bitmask(zero), 0); + assert_eq!(i16x8_bitmask(ones), 0xff); + assert_eq!(i16x8_bitmask(i16x8_splat(i16::MAX)), 0); + assert_eq!(i16x8_bitmask(i16x8_splat(i16::MIN)), 0xff); + assert_eq!(i16x8_bitmask(i16x8_replace_lane::<1>(zero, -1)), 0b10); + + assert_eq!(i32x4_bitmask(zero), 0); + assert_eq!(i32x4_bitmask(ones), 0b1111); + assert_eq!(i32x4_bitmask(i32x4_splat(i32::MAX)), 0); + assert_eq!(i32x4_bitmask(i32x4_splat(i32::MIN)), 0b1111); + assert_eq!(i32x4_bitmask(i32x4_replace_lane::<1>(zero, -1)), 0b10); + + assert_eq!(i64x2_bitmask(zero), 0); + assert_eq!(i64x2_bitmask(ones), 0b11); + assert_eq!(i64x2_bitmask(i64x2_splat(i64::MAX)), 0); + assert_eq!(i64x2_bitmask(i64x2_splat(i64::MIN)), 0b11); + assert_eq!(i64x2_bitmask(i64x2_replace_lane::<1>(zero, -1)), 0b10); + } + + #[test] + fn test_narrow() { + let zero = i8x16_splat(0); + let ones = i8x16_splat(!0); + + compare_bytes(i8x16_narrow_i16x8(zero, zero), zero); + compare_bytes(u8x16_narrow_i16x8(zero, zero), zero); + compare_bytes(i8x16_narrow_i16x8(ones, ones), ones); + compare_bytes(u8x16_narrow_i16x8(ones, ones), zero); + + compare_bytes( + i8x16_narrow_i16x8( + i16x8( + 0, + 1, + 2, + -1, + i8::MIN.into(), + i8::MAX.into(), + u8::MIN.into(), + u8::MAX.into(), + ), + i16x8( + i16::MIN.into(), + i16::MAX.into(), + u16::MIN as i16, + u16::MAX as i16, + 0, + 0, + 0, + 0, + ), + ), + i8x16(0, 1, 2, -1, -128, 127, 0, 127, -128, 127, 0, -1, 0, 0, 0, 0), + ); + + compare_bytes( + u8x16_narrow_i16x8( + i16x8( + 0, + 1, + 2, + -1, + i8::MIN.into(), + i8::MAX.into(), + u8::MIN.into(), + u8::MAX.into(), + ), + i16x8( + i16::MIN.into(), + i16::MAX.into(), + u16::MIN as i16, + u16::MAX as i16, + 0, + 0, + 0, + 0, + ), + ), + i8x16(0, 1, 2, 0, 0, 127, 0, -1, 0, -1, 0, 0, 0, 0, 0, 0), + ); + + compare_bytes(i16x8_narrow_i32x4(zero, zero), zero); + compare_bytes(u16x8_narrow_i32x4(zero, zero), zero); + compare_bytes(i16x8_narrow_i32x4(ones, ones), ones); + compare_bytes(u16x8_narrow_i32x4(ones, ones), zero); + + compare_bytes( + i16x8_narrow_i32x4( + i32x4(0, -1, i16::MIN.into(), i16::MAX.into()), + i32x4( + i32::MIN.into(), + i32::MAX.into(), + u32::MIN as i32, + u32::MAX as i32, + ), + ), + i16x8(0, -1, i16::MIN, i16::MAX, i16::MIN, i16::MAX, 0, -1), + ); + + compare_bytes( + u16x8_narrow_i32x4( + i32x4(u16::MAX.into(), -1, i16::MIN.into(), i16::MAX.into()), + i32x4( + i32::MIN.into(), + i32::MAX.into(), + u32::MIN as i32, + u32::MAX as i32, + ), + ), + i16x8(-1, 0, 0, i16::MAX, 0, -1, 0, 0), + ); + } + + #[test] + fn test_extend() { + let zero = i8x16_splat(0); + let ones = i8x16_splat(!0); + + compare_bytes(i16x8_extend_low_i8x16(zero), zero); + compare_bytes(i16x8_extend_high_i8x16(zero), zero); + compare_bytes(i16x8_extend_low_u8x16(zero), zero); + compare_bytes(i16x8_extend_high_u8x16(zero), zero); + compare_bytes(i16x8_extend_low_i8x16(ones), ones); + compare_bytes(i16x8_extend_high_i8x16(ones), ones); + let halves = u16x8_splat(u8::MAX.into()); + compare_bytes(i16x8_extend_low_u8x16(ones), halves); + compare_bytes(i16x8_extend_high_u8x16(ones), halves); + + compare_bytes(i32x4_extend_low_i16x8(zero), zero); + compare_bytes(i32x4_extend_high_i16x8(zero), zero); + compare_bytes(i32x4_extend_low_u16x8(zero), zero); + compare_bytes(i32x4_extend_high_u16x8(zero), zero); + compare_bytes(i32x4_extend_low_i16x8(ones), ones); + compare_bytes(i32x4_extend_high_i16x8(ones), ones); + let halves = u32x4_splat(u16::MAX.into()); + compare_bytes(i32x4_extend_low_u16x8(ones), halves); + compare_bytes(i32x4_extend_high_u16x8(ones), halves); + + compare_bytes(i64x2_extend_low_i32x4(zero), zero); + compare_bytes(i64x2_extend_high_i32x4(zero), zero); + compare_bytes(i64x2_extend_low_u32x4(zero), zero); + compare_bytes(i64x2_extend_high_u32x4(zero), zero); + compare_bytes(i64x2_extend_low_i32x4(ones), ones); + compare_bytes(i64x2_extend_high_i32x4(ones), ones); + let halves = i64x2_splat(u32::MAX.into()); + compare_bytes(u64x2_extend_low_u32x4(ones), halves); + compare_bytes(u64x2_extend_high_u32x4(ones), halves); + } + + #[test] + fn test_dot() { + let zero = i8x16_splat(0); + let ones = i8x16_splat(!0); + let two = i32x4_splat(2); + compare_bytes(i32x4_dot_i16x8(zero, zero), zero); + compare_bytes(i32x4_dot_i16x8(ones, ones), two); } macro_rules! test_binop { ( $($name:ident => { - $([$($vec1:tt)*] ($op:tt | $f:ident) [$($vec2:tt)*],)* + $([$($vec1:tt)*] ($op:ident | $f:ident) [$($vec2:tt)*],)* })* ) => ($( - #[wasm_bindgen_test] + #[test] fn $name() { unsafe { $( @@ -1753,7 +4757,7 @@ pub mod tests { v3 = mem::transmute(v3_v128); for (i, actual) in v3.iter().enumerate() { - let expected = (Wrapping(v1[i]) $op Wrapping(v2[i])).0; + let expected = v1[i].$op(v2[i]); assert_eq!(*actual, expected); } )* @@ -1765,10 +4769,10 @@ pub mod tests { macro_rules! test_unop { ( $($name:ident => { - $(($op:tt | $f:ident) [$($vec1:tt)*],)* + $(($op:ident | $f:ident) [$($vec1:tt)*],)* })* ) => ($( - #[wasm_bindgen_test] + #[test] fn $name() { unsafe { $( @@ -1780,7 +4784,7 @@ pub mod tests { v2 = mem::transmute(v2_v128); for (i, actual) in v2.iter().enumerate() { - let expected = ($op Wrapping(v1[i])).0; + let expected = v1[i].$op(); assert_eq!(*actual, expected); } )* @@ -1789,546 +4793,1344 @@ pub mod tests { )*) } + trait Avgr: Sized { + fn avgr(self, other: Self) -> Self; + } + + macro_rules! impl_avgr { + ($($i:ident)*) => ($(impl Avgr for $i { + fn avgr(self, other: Self) -> Self { + ((self as u64 + other as u64 + 1) / 2) as $i + } + })*) + } + + impl_avgr!(u8 u16); + test_binop! { test_i8x16_add => { [0i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - (+ | i8x16_add) + (wrapping_add | i8x16_add) + [1i8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + + [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (wrapping_add | i8x16_add) + [-2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, -18], + + [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (wrapping_add | i8x16_add) + [127, -44, 43, 126, 4, 2, 9, -3, -59, -43, 39, -69, 79, -3, 9, -24], + } + + test_i8x16_add_sat_s => { + [0i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + (saturating_add | i8x16_add_sat) [1i8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] - (+ | i8x16_add) + (saturating_add | i8x16_add_sat) [-2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, -18], [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] - (+ | i8x16_add) + (saturating_add | i8x16_add_sat) [127, -44, 43, 126, 4, 2, 9, -3, -59, -43, 39, -69, 79, -3, 9, -24], } + + test_i8x16_add_sat_u => { + [0u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + (saturating_add | u8x16_add_sat) + [1u8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + + [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (saturating_add | u8x16_add_sat) + [255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240], + + [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (saturating_add | u8x16_add_sat) + [127, -44i8 as u8, 43, 126, 4, 2, 9, -3i8 as u8, -59i8 as u8, -43i8 as u8, 39, -69i8 as u8, 79, -3i8 as u8, 9, -24i8 as u8], + } + test_i8x16_sub => { [0i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - (- | i8x16_sub) + (wrapping_sub | i8x16_sub) + [1i8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + + [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (wrapping_sub | i8x16_sub) + [-2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, -18], + + [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (wrapping_sub | i8x16_sub) + [-127, -44, 43, 126, 4, 2, 9, -3, -59, -43, 39, -69, 79, -3, 4, 8], + } + + test_i8x16_sub_sat_s => { + [0i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + (saturating_sub | i8x16_sub_sat) + [1i8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + + [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (saturating_sub | i8x16_sub_sat) + [-2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, -18], + + [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (saturating_sub | i8x16_sub_sat) + [-127, -44, 43, 126, 4, 2, 9, -3, -59, -43, 39, -69, 79, -3, 4, 8], + } + + test_i8x16_sub_sat_u => { + [0u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + (saturating_sub | u8x16_sub_sat) + [1u8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + + [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (saturating_sub | u8x16_sub_sat) + [255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240], + + [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (saturating_sub | u8x16_sub_sat) + [127, -44i8 as u8, 43, 126, 4, 2, 9, -3i8 as u8, -59i8 as u8, -43i8 as u8, 39, -69i8 as u8, 79, -3i8 as u8, 9, -24i8 as u8], + } + + test_i8x16_min_s => { + [0i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + (min | i8x16_min) [1i8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] - (- | i8x16_sub) + (min | i8x16_min) [-2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, -18], [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] - (- | i8x16_sub) + (min | i8x16_min) [-127, -44, 43, 126, 4, 2, 9, -3, -59, -43, 39, -69, 79, -3, 4, 8], } - test_i8x16_mul => { + + test_i8x16_min_u => { + [0u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + (min | u8x16_min) + [1u8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + + [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (min | u8x16_min) + [255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240], + + [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (min | u8x16_min) + [127, -44i8 as u8, 43, 126, 4, 2, 9, -3i8 as u8, -59i8 as u8, -43i8 as u8, 39, -69i8 as u8, 79, -3i8 as u8, 9, -24i8 as u8], + } + + test_i8x16_max_s => { [0i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - (* | i8x16_mul) + (max | i8x16_max) [1i8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] - (* | i8x16_mul) + (max | i8x16_max) [-2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, -18], [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] - (* | i8x16_mul) - [-127, -44, 43, 126, 4, 2, 9, -3, -59, -43, 39, -69, 79, -3, 30, 3], + (max | i8x16_max) + [-127, -44, 43, 126, 4, 2, 9, -3, -59, -43, 39, -69, 79, -3, 4, 8], + } + + test_i8x16_max_u => { + [0u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + (max | u8x16_max) + [1u8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + + [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (max | u8x16_max) + [255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240], + + [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (max | u8x16_max) + [127, -44i8 as u8, 43, 126, 4, 2, 9, -3i8 as u8, -59i8 as u8, -43i8 as u8, 39, -69i8 as u8, 79, -3i8 as u8, 9, -24i8 as u8], + } + + test_i8x16_avgr_u => { + [0u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + (avgr | u8x16_avgr) + [1u8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + + [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (avgr | u8x16_avgr) + [255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240], + + [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (avgr | u8x16_avgr) + [127, -44i8 as u8, 43, 126, 4, 2, 9, -3i8 as u8, -59i8 as u8, -43i8 as u8, 39, -69i8 as u8, 79, -3i8 as u8, 9, -24i8 as u8], } test_i16x8_add => { [0i16, 0, 0, 0, 0, 0, 0, 0] - (+ | i16x8_add) + (wrapping_add | i16x8_add) [1i16, 1, 1, 1, 1, 1, 1, 1], [1i16, 2, 3, 4, 5, 6, 7, 8] - (+ | i16x8_add) + (wrapping_add | i16x8_add) [32767, 8, -2494,-4, 4882, -4, 848, 3830], } + test_i16x8_add_sat_s => { + [0i16, 0, 0, 0, 0, 0, 0, 0] + (saturating_add | i16x8_add_sat) + [1i16, 1, 1, 1, 1, 1, 1, 1], + + [1i16, 2, 3, 4, 5, 6, 7, 8] + (saturating_add | i16x8_add_sat) + [32767, 8, -2494,-4, 4882, -4, 848, 3830], + } + + test_i16x8_add_sat_u => { + [0u16, 0, 0, 0, 0, 0, 0, 0] + (saturating_add | u16x8_add_sat) + [1u16, 1, 1, 1, 1, 1, 1, 1], + + [1u16, 2, 3, 4, 5, 6, 7, 8] + (saturating_add | u16x8_add_sat) + [32767, 8, -2494i16 as u16,-4i16 as u16, 4882, -4i16 as u16, 848, 3830], + } + test_i16x8_sub => { [0i16, 0, 0, 0, 0, 0, 0, 0] - (- | i16x8_sub) + (wrapping_sub | i16x8_sub) + [1i16, 1, 1, 1, 1, 1, 1, 1], + + [1i16, 2, 3, 4, 5, 6, 7, 8] + (wrapping_sub | i16x8_sub) + [32767, 8, -2494,-4, 4882, -4, 848, 3830], + } + + test_i16x8_sub_sat_s => { + [0i16, 0, 0, 0, 0, 0, 0, 0] + (saturating_sub | i16x8_sub_sat) [1i16, 1, 1, 1, 1, 1, 1, 1], [1i16, 2, 3, 4, 5, 6, 7, 8] - (- | i16x8_sub) + (saturating_sub | i16x8_sub_sat) [32767, 8, -2494,-4, 4882, -4, 848, 3830], } + test_i16x8_sub_sat_u => { + [0u16, 0, 0, 0, 0, 0, 0, 0] + (saturating_sub | u16x8_sub_sat) + [1u16, 1, 1, 1, 1, 1, 1, 1], + + [1u16, 2, 3, 4, 5, 6, 7, 8] + (saturating_sub | u16x8_sub_sat) + [32767, 8, -2494i16 as u16,-4i16 as u16, 4882, -4i16 as u16, 848, 3830], + } + test_i16x8_mul => { [0i16, 0, 0, 0, 0, 0, 0, 0] - (* | i16x8_mul) + (wrapping_mul | i16x8_mul) + [1i16, 1, 1, 1, 1, 1, 1, 1], + + [1i16, 2, 3, 4, 5, 6, 7, 8] + (wrapping_mul | i16x8_mul) + [32767, 8, -2494,-4, 4882, -4, 848, 3830], + } + + test_i16x8_min_s => { + [0i16, 0, 0, 0, 0, 0, 0, 0] + (min | i16x8_min) + [1i16, 1, 1, 1, 1, 1, 1, 1], + + [1i16, 2, 3, 4, 5, 6, 7, 8] + (min | i16x8_min) + [32767, 8, -2494,-4, 4882, -4, 848, 3830], + } + + test_i16x8_min_u => { + [0u16, 0, 0, 0, 0, 0, 0, 0] + (min | u16x8_min) + [1u16, 1, 1, 1, 1, 1, 1, 1], + + [1u16, 2, 3, 4, 5, 6, 7, 8] + (min | u16x8_min) + [32767, 8, -2494i16 as u16,-4i16 as u16, 4882, -4i16 as u16, 848, 3830], + } + + test_i16x8_max_s => { + [0i16, 0, 0, 0, 0, 0, 0, 0] + (max | i16x8_max) [1i16, 1, 1, 1, 1, 1, 1, 1], [1i16, 2, 3, 4, 5, 6, 7, 8] - (* | i16x8_mul) + (max | i16x8_max) [32767, 8, -2494,-4, 4882, -4, 848, 3830], } + test_i16x8_max_u => { + [0u16, 0, 0, 0, 0, 0, 0, 0] + (max | u16x8_max) + [1u16, 1, 1, 1, 1, 1, 1, 1], + + [1u16, 2, 3, 4, 5, 6, 7, 8] + (max | u16x8_max) + [32767, 8, -2494i16 as u16,-4i16 as u16, 4882, -4i16 as u16, 848, 3830], + } + + test_i16x8_avgr_u => { + [0u16, 0, 0, 0, 0, 0, 0, 0] + (avgr | u16x8_avgr) + [1u16, 1, 1, 1, 1, 1, 1, 1], + + [1u16, 2, 3, 4, 5, 6, 7, 8] + (avgr | u16x8_avgr) + [32767, 8, -2494i16 as u16,-4i16 as u16, 4882, -4i16 as u16, 848, 3830], + } + test_i32x4_add => { - [0i32, 0, 0, 0] (+ | i32x4_add) [1, 2, 3, 4], - [1i32, 1283, i32::max_value(), i32::min_value()] - (+ | i32x4_add) - [i32::max_value(); 4], + [0i32, 0, 0, 0] (wrapping_add | i32x4_add) [1, 2, 3, 4], + [1i32, 1283, i32::MAX, i32::MIN] + (wrapping_add | i32x4_add) + [i32::MAX; 4], } test_i32x4_sub => { - [0i32, 0, 0, 0] (- | i32x4_sub) [1, 2, 3, 4], - [1i32, 1283, i32::max_value(), i32::min_value()] - (- | i32x4_sub) - [i32::max_value(); 4], + [0i32, 0, 0, 0] (wrapping_sub | i32x4_sub) [1, 2, 3, 4], + [1i32, 1283, i32::MAX, i32::MIN] + (wrapping_sub | i32x4_sub) + [i32::MAX; 4], } test_i32x4_mul => { - [0i32, 0, 0, 0] (* | i32x4_mul) [1, 2, 3, 4], - [1i32, 1283, i32::max_value(), i32::min_value()] - (* | i32x4_mul) - [i32::max_value(); 4], + [0i32, 0, 0, 0] (wrapping_mul | i32x4_mul) [1, 2, 3, 4], + [1i32, 1283, i32::MAX, i32::MIN] + (wrapping_mul | i32x4_mul) + [i32::MAX; 4], + } + + test_i32x4_min_s => { + [0i32, 0, 0, 0] (min | i32x4_min) [1, 2, 3, 4], + [1i32, 1283, i32::MAX, i32::MIN] + (min | i32x4_min) + [i32::MAX; 4], + } + + test_i32x4_min_u => { + [0u32, 0, 0, 0] (min | u32x4_min) [1, 2, 3, 4], + [1u32, 1283, i32::MAX as u32, i32::MIN as u32] + (min | u32x4_min) + [i32::MAX as u32; 4], + } + + test_i32x4_max_s => { + [0i32, 0, 0, 0] (max | i32x4_max) [1, 2, 3, 4], + [1i32, 1283, i32::MAX, i32::MIN] + (max | i32x4_max) + [i32::MAX; 4], + } + + test_i32x4_max_u => { + [0u32, 0, 0, 0] (max | u32x4_max) [1, 2, 3, 4], + [1u32, 1283, i32::MAX as u32, i32::MIN as u32] + (max | u32x4_max) + [i32::MAX as u32; 4], + } + + test_i64x2_add => { + [0i64, 0] (wrapping_add | i64x2_add) [1, 2], + [i64::MIN, i64::MAX] (wrapping_add | i64x2_add) [i64::MAX, i64::MIN], + [i64::MAX; 2] (wrapping_add | i64x2_add) [i64::MAX; 2], + [-4i64, -4] (wrapping_add | i64x2_add) [800, 939], + } + + test_i64x2_sub => { + [0i64, 0] (wrapping_sub | i64x2_sub) [1, 2], + [i64::MIN, i64::MAX] (wrapping_sub | i64x2_sub) [i64::MAX, i64::MIN], + [i64::MAX; 2] (wrapping_sub | i64x2_sub) [i64::MAX; 2], + [-4i64, -4] (wrapping_sub | i64x2_sub) [800, 939], + } + + test_i64x2_mul => { + [0i64, 0] (wrapping_mul | i64x2_mul) [1, 2], + [i64::MIN, i64::MAX] (wrapping_mul | i64x2_mul) [i64::MAX, i64::MIN], + [i64::MAX; 2] (wrapping_mul | i64x2_mul) [i64::MAX; 2], + [-4i64, -4] (wrapping_mul | i64x2_mul) [800, 939], + } + + test_f32x4_add => { + [-1.0f32, 2.0, 3.0, 4.0] (add | f32x4_add) [1., 2., 0., 0.], + [f32::INFINITY, -0.0, f32::NEG_INFINITY, 3.0] + (add | f32x4_add) + [1., 2., 0., 0.], + } + + test_f32x4_sub => { + [-1.0f32, 2.0, 3.0, 4.0] (sub | f32x4_sub) [1., 2., 0., 0.], + [f32::INFINITY, -0.0, f32::NEG_INFINITY, 3.0] + (sub | f32x4_sub) + [1., 2., 0., 0.], + } + + test_f32x4_mul => { + [-1.0f32, 2.0, 3.0, 4.0] (mul | f32x4_mul) [1., 2., 0., 0.], + [f32::INFINITY, -0.0, f32::NEG_INFINITY, 3.0] + (mul | f32x4_mul) + [1., 2., 1., 0.], + } + + test_f32x4_div => { + [-1.0f32, 2.0, 3.0, 4.0] (div | f32x4_div) [1., 2., 0., 0.], + [f32::INFINITY, -0.0, f32::NEG_INFINITY, 3.0] + (div | f32x4_div) + [1., 2., 0., 0.], + } + + test_f32x4_min => { + [-1.0f32, 2.0, 3.0, 4.0] (min | f32x4_min) [1., 2., 0., 0.], + [f32::INFINITY, -0.0, f32::NEG_INFINITY, 3.0] + (min | f32x4_min) + [1., 2., 0., 0.], + } + + test_f32x4_max => { + [-1.0f32, 2.0, 3.0, 4.0] (max | f32x4_max) [1., 2., 0., 0.], + [f32::INFINITY, -0.0, f32::NEG_INFINITY, 3.0] + (max | f32x4_max) + [1., 2., 0., 0.], + } + + test_f32x4_pmin => { + [-1.0f32, 2.0, 3.0, 4.0] (min | f32x4_pmin) [1., 2., 0., 0.], + [f32::INFINITY, -0.0, f32::NEG_INFINITY, 3.0] + (min | f32x4_pmin) + [1., 2., 0., 0.], + } + + test_f32x4_pmax => { + [-1.0f32, 2.0, 3.0, 4.0] (max | f32x4_pmax) [1., 2., 0., 0.], + [f32::INFINITY, -0.0, f32::NEG_INFINITY, 3.0] + (max | f32x4_pmax) + [1., 2., 0., 0.], + } + + test_f64x2_add => { + [-1.0f64, 2.0] (add | f64x2_add) [1., 2.], + [f64::INFINITY, f64::NEG_INFINITY] (add | f64x2_add) [1., 2.], + } + + test_f64x2_sub => { + [-1.0f64, 2.0] (sub | f64x2_sub) [1., 2.], + [f64::INFINITY, f64::NEG_INFINITY] (sub | f64x2_sub) [1., 2.], + } + + test_f64x2_mul => { + [-1.0f64, 2.0] (mul | f64x2_mul) [1., 2.], + [f64::INFINITY, f64::NEG_INFINITY] (mul | f64x2_mul) [1., 2.], + } + + test_f64x2_div => { + [-1.0f64, 2.0] (div | f64x2_div) [1., 2.], + [f64::INFINITY, f64::NEG_INFINITY] (div | f64x2_div) [1., 2.], + } + + test_f64x2_min => { + [-1.0f64, 2.0] (min | f64x2_min) [1., 2.], + [f64::INFINITY, f64::NEG_INFINITY] (min | f64x2_min) [1., 2.], + } + + test_f64x2_max => { + [-1.0f64, 2.0] (max | f64x2_max) [1., 2.], + [f64::INFINITY, f64::NEG_INFINITY] (max | f64x2_max) [1., 2.], + } + + test_f64x2_pmin => { + [-1.0f64, 2.0] (min | f64x2_pmin) [1., 2.], + [f64::INFINITY, f64::NEG_INFINITY] (min | f64x2_pmin) [1., 2.], } - // TODO: test_i64x2_add - // TODO: test_i64x2_sub + test_f64x2_pmax => { + [-1.0f64, 2.0] (max | f64x2_pmax) [1., 2.], + [f64::INFINITY, f64::NEG_INFINITY] (max | f64x2_pmax) [1., 2.], + } } test_unop! { + test_i8x16_abs => { + (wrapping_abs | i8x16_abs) + [1i8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + + (wrapping_abs | i8x16_abs) + [-2i8, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, -18], + + (wrapping_abs | i8x16_abs) + [-127i8, -44, 43, 126, 4, -128, 127, -59, -43, 39, -69, 79, -3, 35, 83, 13], + } + test_i8x16_neg => { - (- | i8x16_neg) + (wrapping_neg | i8x16_neg) [1i8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], - (- | i8x16_neg) + (wrapping_neg | i8x16_neg) [-2i8, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, -18], - (- | i8x16_neg) + (wrapping_neg | i8x16_neg) [-127i8, -44, 43, 126, 4, -128, 127, -59, -43, 39, -69, 79, -3, 35, 83, 13], } + test_i16x8_abs => { + (wrapping_abs | i16x8_abs) [1i16, 1, 1, 1, 1, 1, 1, 1], + (wrapping_abs | i16x8_abs) [2i16, 0x7fff, !0, 4, 42, -5, 33, -4847], + } + test_i16x8_neg => { - (- | i16x8_neg) [1i16, 1, 1, 1, 1, 1, 1, 1], - (- | i16x8_neg) [2i16, 0x7fff, !0, 4, 42, -5, 33, -4847], + (wrapping_neg | i16x8_neg) [1i16, 1, 1, 1, 1, 1, 1, 1], + (wrapping_neg | i16x8_neg) [2i16, 0x7fff, !0, 4, 42, -5, 33, -4847], + } + + test_i32x4_abs => { + (wrapping_abs | i32x4_abs) [1i32, 2, 3, 4], + (wrapping_abs | i32x4_abs) [i32::MIN, i32::MAX, 0, 4], } test_i32x4_neg => { - (- | i32x4_neg) [1i32, 2, 3, 4], - (- | i32x4_neg) [i32::min_value(), i32::max_value(), 0, 4], - } - - // TODO: test_i64x2_neg - } - - // #[wasm_bindgen_test] - // fn v8x16_shuffle() { - // unsafe { - // let a = [0_u8, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; - // let b = [ - // 16_u8, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, - // 31, - // ]; - // - // let vec_a: v128 = transmute(a); - // let vec_b: v128 = transmute(b); - // - // let vec_r = v8x16_shuffle!( - // vec_a, - // vec_b, - // [0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30] - // ); - // - // let e = - // [0_u8, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]; - // let vec_e: v128 = transmute(e); - // compare_bytes(vec_r, vec_e); - // } - // } - // - // macro_rules! floating_point { - // (f32) => { - // true - // }; - // (f64) => { - // true - // }; - // ($id:ident) => { - // false - // }; - // } - // - // trait IsNan: Sized { - // fn is_nan(self) -> bool { - // false - // } - // } - // impl IsNan for i8 {} - // impl IsNan for i16 {} - // impl IsNan for i32 {} - // impl IsNan for i64 {} - // - // macro_rules! test_bop { - // ($id:ident[$ety:ident; $ecount:expr] | - // $binary_op:ident [$op_test_id:ident] : - // ([$($in_a:expr),*], [$($in_b:expr),*]) => [$($out:expr),*]) => { - // test_bop!( - // $id[$ety; $ecount] => $ety | $binary_op [ $op_test_id ]: - // ([$($in_a),*], [$($in_b),*]) => [$($out),*] - // ); - // - // }; - // ($id:ident[$ety:ident; $ecount:expr] => $oty:ident | - // $binary_op:ident [$op_test_id:ident] : - // ([$($in_a:expr),*], [$($in_b:expr),*]) => [$($out:expr),*]) => { - // #[wasm_bindgen_test] - // fn $op_test_id() { - // unsafe { - // let a_input: [$ety; $ecount] = [$($in_a),*]; - // let b_input: [$ety; $ecount] = [$($in_b),*]; - // let output: [$oty; $ecount] = [$($out),*]; - // - // let a_vec_in: v128 = transmute(a_input); - // let b_vec_in: v128 = transmute(b_input); - // let vec_res: v128 = $id::$binary_op(a_vec_in, b_vec_in); - // - // let res: [$oty; $ecount] = transmute(vec_res); - // - // if !floating_point!($ety) { - // assert_eq!(res, output); - // } else { - // for i in 0..$ecount { - // let r = res[i]; - // let o = output[i]; - // assert_eq!(r.is_nan(), o.is_nan()); - // if !r.is_nan() { - // assert_eq!(r, o); - // } - // } - // } - // } - // } - // } - // } - // - // macro_rules! test_bops { - // ($id:ident[$ety:ident; $ecount:expr] | - // $binary_op:ident [$op_test_id:ident]: - // ([$($in_a:expr),*], $in_b:expr) => [$($out:expr),*]) => { - // #[wasm_bindgen_test] - // fn $op_test_id() { - // unsafe { - // let a_input: [$ety; $ecount] = [$($in_a),*]; - // let output: [$ety; $ecount] = [$($out),*]; - // - // let a_vec_in: v128 = transmute(a_input); - // let vec_res: v128 = $id::$binary_op(a_vec_in, $in_b); - // - // let res: [$ety; $ecount] = transmute(vec_res); - // assert_eq!(res, output); - // } - // } - // } - // } - // - // macro_rules! test_uop { - // ($id:ident[$ety:ident; $ecount:expr] | - // $unary_op:ident [$op_test_id:ident]: [$($in_a:expr),*] => [$($out:expr),*]) => { - // #[wasm_bindgen_test] - // fn $op_test_id() { - // unsafe { - // let a_input: [$ety; $ecount] = [$($in_a),*]; - // let output: [$ety; $ecount] = [$($out),*]; - // - // let a_vec_in: v128 = transmute(a_input); - // let vec_res: v128 = $id::$unary_op(a_vec_in); - // - // let res: [$ety; $ecount] = transmute(vec_res); - // assert_eq!(res, output); - // } - // } - // } - // } - // - // - // - // test_bops!(i8x16[i8; 16] | shl[i8x16_shl_test]: - // ([0, -1, 2, 3, 4, 5, 6, i8::max_value(), 1, 1, 1, 1, 1, 1, 1, 1], 1) => - // [0, -2, 4, 6, 8, 10, 12, -2, 2, 2, 2, 2, 2, 2, 2, 2]); - // test_bops!(i16x8[i16; 8] | shl[i16x8_shl_test]: - // ([0, -1, 2, 3, 4, 5, 6, i16::max_value()], 1) => - // [0, -2, 4, 6, 8, 10, 12, -2]); - // test_bops!(i32x4[i32; 4] | shl[i32x4_shl_test]: - // ([0, -1, 2, 3], 1) => [0, -2, 4, 6]); - // test_bops!(i64x2[i64; 2] | shl[i64x2_shl_test]: - // ([0, -1], 1) => [0, -2]); - // - // test_bops!(i8x16[i8; 16] | shr_s[i8x16_shr_s_test]: - // ([0, -1, 2, 3, 4, 5, 6, i8::max_value(), 1, 1, 1, 1, 1, 1, 1, 1], 1) => - // [0, -1, 1, 1, 2, 2, 3, 63, 0, 0, 0, 0, 0, 0, 0, 0]); - // test_bops!(i16x8[i16; 8] | shr_s[i16x8_shr_s_test]: - // ([0, -1, 2, 3, 4, 5, 6, i16::max_value()], 1) => - // [0, -1, 1, 1, 2, 2, 3, i16::max_value() / 2]); - // test_bops!(i32x4[i32; 4] | shr_s[i32x4_shr_s_test]: - // ([0, -1, 2, 3], 1) => [0, -1, 1, 1]); - // test_bops!(i64x2[i64; 2] | shr_s[i64x2_shr_s_test]: - // ([0, -1], 1) => [0, -1]); - // - // test_bops!(i8x16[i8; 16] | shr_u[i8x16_uhr_u_test]: - // ([0, -1, 2, 3, 4, 5, 6, i8::max_value(), 1, 1, 1, 1, 1, 1, 1, 1], 1) => - // [0, i8::max_value(), 1, 1, 2, 2, 3, 63, 0, 0, 0, 0, 0, 0, 0, 0]); - // test_bops!(i16x8[i16; 8] | shr_u[i16x8_uhr_u_test]: - // ([0, -1, 2, 3, 4, 5, 6, i16::max_value()], 1) => - // [0, i16::max_value(), 1, 1, 2, 2, 3, i16::max_value() / 2]); - // test_bops!(i32x4[i32; 4] | shr_u[i32x4_uhr_u_test]: - // ([0, -1, 2, 3], 1) => [0, i32::max_value(), 1, 1]); - // test_bops!(i64x2[i64; 2] | shr_u[i64x2_uhr_u_test]: - // ([0, -1], 1) => [0, i64::max_value()]); - // - // #[wasm_bindgen_test] - // fn v128_bitwise_logical_ops() { - // unsafe { - // let a: [u32; 4] = [u32::max_value(), 0, u32::max_value(), 0]; - // let b: [u32; 4] = [u32::max_value(); 4]; - // let c: [u32; 4] = [0; 4]; - // - // let vec_a: v128 = transmute(a); - // let vec_b: v128 = transmute(b); - // let vec_c: v128 = transmute(c); - // - // let r: v128 = v128::and(vec_a, vec_a); - // compare_bytes(r, vec_a); - // let r: v128 = v128::and(vec_a, vec_b); - // compare_bytes(r, vec_a); - // let r: v128 = v128::or(vec_a, vec_b); - // compare_bytes(r, vec_b); - // let r: v128 = v128::not(vec_b); - // compare_bytes(r, vec_c); - // let r: v128 = v128::xor(vec_a, vec_c); - // compare_bytes(r, vec_a); - // - // let r: v128 = v128::bitselect(vec_b, vec_c, vec_b); - // compare_bytes(r, vec_b); - // let r: v128 = v128::bitselect(vec_b, vec_c, vec_c); - // compare_bytes(r, vec_c); - // let r: v128 = v128::bitselect(vec_b, vec_c, vec_a); - // compare_bytes(r, vec_a); - // } - // } - // - // macro_rules! test_bool_red { - // ($id:ident[$test_id:ident] | [$($true:expr),*] | [$($false:expr),*] | [$($alt:expr),*]) => { - // #[wasm_bindgen_test] - // fn $test_id() { - // unsafe { - // let vec_a: v128 = transmute([$($true),*]); // true - // let vec_b: v128 = transmute([$($false),*]); // false - // let vec_c: v128 = transmute([$($alt),*]); // alternating - // - // assert_eq!($id::any_true(vec_a), 1); - // assert_eq!($id::any_true(vec_b), 0); - // assert_eq!($id::any_true(vec_c), 1); - // - // assert_eq!($id::all_true(vec_a), 1); - // assert_eq!($id::all_true(vec_b), 0); - // assert_eq!($id::all_true(vec_c), 0); - // } - // } - // } - // } - // - // test_bool_red!( - // i8x16[i8x16_boolean_reductions] - // | [1_i8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] - // | [0_i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - // | [1_i8, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0] - // ); - // test_bool_red!( - // i16x8[i16x8_boolean_reductions] - // | [1_i16, 1, 1, 1, 1, 1, 1, 1] - // | [0_i16, 0, 0, 0, 0, 0, 0, 0] - // | [1_i16, 0, 1, 0, 1, 0, 1, 0] - // ); - // test_bool_red!( - // i32x4[i32x4_boolean_reductions] - // | [1_i32, 1, 1, 1] - // | [0_i32, 0, 0, 0] - // | [1_i32, 0, 1, 0] - // ); - // test_bool_red!( - // i64x2[i64x2_boolean_reductions] | [1_i64, 1] | [0_i64, 0] | [1_i64, 0] - // ); - // - // test_bop!(i8x16[i8; 16] | eq[i8x16_eq_test]: - // ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - // [0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, 15]) => - // [-1, 0, -1, 0 ,-1, 0, -1, -1, -1, 0, -1, 0 ,-1, 0, -1, -1]); - // test_bop!(i16x8[i16; 8] | eq[i16x8_eq_test]: - // ([0, 1, 2, 3, 4, 5, 6, 7], [0, 2, 2, 4, 4, 6, 6, 7]) => - // [-1, 0, -1, 0 ,-1, 0, -1, -1]); - // test_bop!(i32x4[i32; 4] | eq[i32x4_eq_test]: - // ([0, 1, 2, 3], [0, 2, 2, 4]) => [-1, 0, -1, 0]); - // test_bop!(i64x2[i64; 2] | eq[i64x2_eq_test]: ([0, 1], [0, 2]) => [-1, 0]); - // test_bop!(f32x4[f32; 4] => i32 | eq[f32x4_eq_test]: - // ([0., 1., 2., 3.], [0., 2., 2., 4.]) => [-1, 0, -1, 0]); - // test_bop!(f64x2[f64; 2] => i64 | eq[f64x2_eq_test]: ([0., 1.], [0., 2.]) => [-1, 0]); - // - // test_bop!(i8x16[i8; 16] | ne[i8x16_ne_test]: - // ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - // [0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, 15]) => - // [0, -1, 0, -1 ,0, -1, 0, 0, 0, -1, 0, -1 ,0, -1, 0, 0]); - // test_bop!(i16x8[i16; 8] | ne[i16x8_ne_test]: - // ([0, 1, 2, 3, 4, 5, 6, 7], [0, 2, 2, 4, 4, 6, 6, 7]) => - // [0, -1, 0, -1 ,0, -1, 0, 0]); - // test_bop!(i32x4[i32; 4] | ne[i32x4_ne_test]: - // ([0, 1, 2, 3], [0, 2, 2, 4]) => [0, -1, 0, -1]); - // test_bop!(i64x2[i64; 2] | ne[i64x2_ne_test]: ([0, 1], [0, 2]) => [0, -1]); - // test_bop!(f32x4[f32; 4] => i32 | ne[f32x4_ne_test]: - // ([0., 1., 2., 3.], [0., 2., 2., 4.]) => [0, -1, 0, -1]); - // test_bop!(f64x2[f64; 2] => i64 | ne[f64x2_ne_test]: ([0., 1.], [0., 2.]) => [0, -1]); - // - // test_bop!(i8x16[i8; 16] | lt[i8x16_lt_test]: - // ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - // [0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, 15]) => - // [0, -1, 0, -1 ,0, -1, 0, 0, 0, -1, 0, -1 ,0, -1, 0, 0]); - // test_bop!(i16x8[i16; 8] | lt[i16x8_lt_test]: - // ([0, 1, 2, 3, 4, 5, 6, 7], [0, 2, 2, 4, 4, 6, 6, 7]) => - // [0, -1, 0, -1 ,0, -1, 0, 0]); - // test_bop!(i32x4[i32; 4] | lt[i32x4_lt_test]: - // ([0, 1, 2, 3], [0, 2, 2, 4]) => [0, -1, 0, -1]); - // test_bop!(i64x2[i64; 2] | lt[i64x2_lt_test]: ([0, 1], [0, 2]) => [0, -1]); - // test_bop!(f32x4[f32; 4] => i32 | lt[f32x4_lt_test]: - // ([0., 1., 2., 3.], [0., 2., 2., 4.]) => [0, -1, 0, -1]); - // test_bop!(f64x2[f64; 2] => i64 | lt[f64x2_lt_test]: ([0., 1.], [0., 2.]) => [0, -1]); - // - // test_bop!(i8x16[i8; 16] | gt[i8x16_gt_test]: - // ([0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, 15], - // [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) => - // [0, -1, 0, -1 ,0, -1, 0, 0, 0, -1, 0, -1 ,0, -1, 0, 0]); - // test_bop!(i16x8[i16; 8] | gt[i16x8_gt_test]: - // ([0, 2, 2, 4, 4, 6, 6, 7], [0, 1, 2, 3, 4, 5, 6, 7]) => - // [0, -1, 0, -1 ,0, -1, 0, 0]); - // test_bop!(i32x4[i32; 4] | gt[i32x4_gt_test]: - // ([0, 2, 2, 4], [0, 1, 2, 3]) => [0, -1, 0, -1]); - // test_bop!(i64x2[i64; 2] | gt[i64x2_gt_test]: ([0, 2], [0, 1]) => [0, -1]); - // test_bop!(f32x4[f32; 4] => i32 | gt[f32x4_gt_test]: - // ([0., 2., 2., 4.], [0., 1., 2., 3.]) => [0, -1, 0, -1]); - // test_bop!(f64x2[f64; 2] => i64 | gt[f64x2_gt_test]: ([0., 2.], [0., 1.]) => [0, -1]); - // - // test_bop!(i8x16[i8; 16] | ge[i8x16_ge_test]: - // ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - // [0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, 15]) => - // [-1, 0, -1, 0 ,-1, 0, -1, -1, -1, 0, -1, 0 ,-1, 0, -1, -1]); - // test_bop!(i16x8[i16; 8] | ge[i16x8_ge_test]: - // ([0, 1, 2, 3, 4, 5, 6, 7], [0, 2, 2, 4, 4, 6, 6, 7]) => - // [-1, 0, -1, 0 ,-1, 0, -1, -1]); - // test_bop!(i32x4[i32; 4] | ge[i32x4_ge_test]: - // ([0, 1, 2, 3], [0, 2, 2, 4]) => [-1, 0, -1, 0]); - // test_bop!(i64x2[i64; 2] | ge[i64x2_ge_test]: ([0, 1], [0, 2]) => [-1, 0]); - // test_bop!(f32x4[f32; 4] => i32 | ge[f32x4_ge_test]: - // ([0., 1., 2., 3.], [0., 2., 2., 4.]) => [-1, 0, -1, 0]); - // test_bop!(f64x2[f64; 2] => i64 | ge[f64x2_ge_test]: ([0., 1.], [0., 2.]) => [-1, 0]); - // - // test_bop!(i8x16[i8; 16] | le[i8x16_le_test]: - // ([0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, 15], - // [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] - // ) => - // [-1, 0, -1, 0 ,-1, 0, -1, -1, -1, 0, -1, 0 ,-1, 0, -1, -1]); - // test_bop!(i16x8[i16; 8] | le[i16x8_le_test]: - // ([0, 2, 2, 4, 4, 6, 6, 7], [0, 1, 2, 3, 4, 5, 6, 7]) => - // [-1, 0, -1, 0 ,-1, 0, -1, -1]); - // test_bop!(i32x4[i32; 4] | le[i32x4_le_test]: - // ([0, 2, 2, 4], [0, 1, 2, 3]) => [-1, 0, -1, 0]); - // test_bop!(i64x2[i64; 2] | le[i64x2_le_test]: ([0, 2], [0, 1]) => [-1, 0]); - // test_bop!(f32x4[f32; 4] => i32 | le[f32x4_le_test]: - // ([0., 2., 2., 4.], [0., 1., 2., 3.]) => [-1, 0, -1, -0]); - // test_bop!(f64x2[f64; 2] => i64 | le[f64x2_le_test]: ([0., 2.], [0., 1.]) => [-1, 0]); - // - // #[wasm_bindgen_test] - // fn v128_bitwise_load_store() { - // unsafe { - // let mut arr: [i32; 4] = [0, 1, 2, 3]; - // - // let vec = v128::load(arr.as_ptr() as *const v128); - // let vec = i32x4::add(vec, vec); - // v128::store(arr.as_mut_ptr() as *mut v128, vec); - // - // assert_eq!(arr, [0, 2, 4, 6]); - // } - // } - // - // test_uop!(f32x4[f32; 4] | neg[f32x4_neg_test]: [0., 1., 2., 3.] => [ 0., -1., -2., -3.]); - // test_uop!(f32x4[f32; 4] | abs[f32x4_abs_test]: [0., -1., 2., -3.] => [ 0., 1., 2., 3.]); - // test_bop!(f32x4[f32; 4] | min[f32x4_min_test]: - // ([0., -1., 7., 8.], [1., -3., -4., 10.]) => [0., -3., -4., 8.]); - // test_bop!(f32x4[f32; 4] | min[f32x4_min_test_nan]: - // ([0., -1., 7., 8.], [1., -3., -4., std::f32::NAN]) - // => [0., -3., -4., std::f32::NAN]); - // test_bop!(f32x4[f32; 4] | max[f32x4_max_test]: - // ([0., -1., 7., 8.], [1., -3., -4., 10.]) => [1., -1., 7., 10.]); - // test_bop!(f32x4[f32; 4] | max[f32x4_max_test_nan]: - // ([0., -1., 7., 8.], [1., -3., -4., std::f32::NAN]) - // => [1., -1., 7., std::f32::NAN]); - // test_bop!(f32x4[f32; 4] | add[f32x4_add_test]: - // ([0., -1., 7., 8.], [1., -3., -4., 10.]) => [1., -4., 3., 18.]); - // test_bop!(f32x4[f32; 4] | sub[f32x4_sub_test]: - // ([0., -1., 7., 8.], [1., -3., -4., 10.]) => [-1., 2., 11., -2.]); - // test_bop!(f32x4[f32; 4] | mul[f32x4_mul_test]: - // ([0., -1., 7., 8.], [1., -3., -4., 10.]) => [0., 3., -28., 80.]); - // test_bop!(f32x4[f32; 4] | div[f32x4_div_test]: - // ([0., -8., 70., 8.], [1., 4., 10., 2.]) => [0., -2., 7., 4.]); - // - // test_uop!(f64x2[f64; 2] | neg[f64x2_neg_test]: [0., 1.] => [ 0., -1.]); - // test_uop!(f64x2[f64; 2] | abs[f64x2_abs_test]: [0., -1.] => [ 0., 1.]); - // test_bop!(f64x2[f64; 2] | min[f64x2_min_test]: - // ([0., -1.], [1., -3.]) => [0., -3.]); - // test_bop!(f64x2[f64; 2] | min[f64x2_min_test_nan]: - // ([7., 8.], [-4., std::f64::NAN]) - // => [ -4., std::f64::NAN]); - // test_bop!(f64x2[f64; 2] | max[f64x2_max_test]: - // ([0., -1.], [1., -3.]) => [1., -1.]); - // test_bop!(f64x2[f64; 2] | max[f64x2_max_test_nan]: - // ([7., 8.], [ -4., std::f64::NAN]) - // => [7., std::f64::NAN]); - // test_bop!(f64x2[f64; 2] | add[f64x2_add_test]: - // ([0., -1.], [1., -3.]) => [1., -4.]); - // test_bop!(f64x2[f64; 2] | sub[f64x2_sub_test]: - // ([0., -1.], [1., -3.]) => [-1., 2.]); - // test_bop!(f64x2[f64; 2] | mul[f64x2_mul_test]: - // ([0., -1.], [1., -3.]) => [0., 3.]); - // test_bop!(f64x2[f64; 2] | div[f64x2_div_test]: - // ([0., -8.], [1., 4.]) => [0., -2.]); - // - // macro_rules! test_conv { - // ($test_id:ident | $conv_id:ident | $to_ty:ident | $from:expr, $to:expr) => { - // #[wasm_bindgen_test] - // fn $test_id() { - // unsafe { - // let from: v128 = transmute($from); - // let to: v128 = transmute($to); - // - // let r: v128 = $to_ty::$conv_id(from); - // - // compare_bytes(r, to); - // } - // } - // }; - // } - // - // test_conv!( - // f32x4_convert_s_i32x4 | convert_s_i32x4 | f32x4 | [1_i32, 2, 3, 4], - // [1_f32, 2., 3., 4.] - // ); - // test_conv!( - // f32x4_convert_u_i32x4 - // | convert_u_i32x4 - // | f32x4 - // | [u32::max_value(), 2, 3, 4], - // [u32::max_value() as f32, 2., 3., 4.] - // ); - // test_conv!( - // f64x2_convert_s_i64x2 | convert_s_i64x2 | f64x2 | [1_i64, 2], - // [1_f64, 2.] - // ); - // test_conv!( - // f64x2_convert_u_i64x2 - // | convert_u_i64x2 - // | f64x2 - // | [u64::max_value(), 2], - // [18446744073709552000.0, 2.] - // ); - // - // // FIXME: this fails, and produces -2147483648 instead of saturating at - // // i32::max_value() test_conv!(i32x4_trunc_s_f32x4_sat | trunc_s_f32x4_sat - // // | i32x4 | [1_f32, 2., (i32::max_value() as f32 + 1.), 4.], - // // [1_i32, 2, i32::max_value(), 4]); FIXME: add other saturating tests + (wrapping_neg | i32x4_neg) [1i32, 2, 3, 4], + (wrapping_neg | i32x4_neg) [i32::MIN, i32::MAX, 0, 4], + } + + test_i64x2_abs => { + (wrapping_abs | i64x2_abs) [1i64, 2], + (wrapping_abs | i64x2_abs) [i64::MIN, i64::MAX], + } + + test_i64x2_neg => { + (wrapping_neg | i64x2_neg) [1i64, 2], + (wrapping_neg | i64x2_neg) [i64::MIN, i64::MAX], + } + + test_f32x4_ceil => { + (ceil | f32x4_ceil) [1.0f32, 2., 2.5, 3.3], + (ceil | f32x4_ceil) [0.0, -0.3, f32::INFINITY, -0.0], + } + + test_f32x4_floor => { + (floor | f32x4_floor) [1.0f32, 2., 2.5, 3.3], + (floor | f32x4_floor) [0.0, -0.3, f32::INFINITY, -0.0], + } + + test_f32x4_trunc => { + (trunc | f32x4_trunc) [1.0f32, 2., 2.5, 3.3], + (trunc | f32x4_trunc) [0.0, -0.3, f32::INFINITY, -0.0], + } + + test_f32x4_nearest => { + (round | f32x4_nearest) [1.0f32, 2., 2.6, 3.3], + (round | f32x4_nearest) [0.0, -0.3, f32::INFINITY, -0.0], + } + + test_f32x4_abs => { + (abs | f32x4_abs) [1.0f32, 2., 2.6, 3.3], + (abs | f32x4_abs) [0.0, -0.3, f32::INFINITY, -0.0], + } + + test_f32x4_neg => { + (neg | f32x4_neg) [1.0f32, 2., 2.6, 3.3], + (neg | f32x4_neg) [0.0, -0.3, f32::INFINITY, -0.0], + } + + test_f32x4_sqrt => { + (sqrt | f32x4_sqrt) [1.0f32, 2., 2.6, 3.3], + (sqrt | f32x4_sqrt) [0.0, 0.3, f32::INFINITY, 0.1], + } + + test_f64x2_ceil => { + (ceil | f64x2_ceil) [1.0f64, 2.3], + (ceil | f64x2_ceil) [f64::INFINITY, -0.1], + } + + test_f64x2_floor => { + (floor | f64x2_floor) [1.0f64, 2.3], + (floor | f64x2_floor) [f64::INFINITY, -0.1], + } + + test_f64x2_trunc => { + (trunc | f64x2_trunc) [1.0f64, 2.3], + (trunc | f64x2_trunc) [f64::INFINITY, -0.1], + } + + test_f64x2_nearest => { + (round | f64x2_nearest) [1.0f64, 2.3], + (round | f64x2_nearest) [f64::INFINITY, -0.1], + } + + test_f64x2_abs => { + (abs | f64x2_abs) [1.0f64, 2.3], + (abs | f64x2_abs) [f64::INFINITY, -0.1], + } + + test_f64x2_neg => { + (neg | f64x2_neg) [1.0f64, 2.3], + (neg | f64x2_neg) [f64::INFINITY, -0.1], + } + + test_f64x2_sqrt => { + (sqrt | f64x2_sqrt) [1.0f64, 2.3], + (sqrt | f64x2_sqrt) [f64::INFINITY, 0.1], + } + } + + macro_rules! floating_point { + (f32) => { + true + }; + (f64) => { + true + }; + ($id:ident) => { + false + }; + } + + trait IsNan: Sized { + fn is_nan(self) -> bool { + false + } + } + impl IsNan for i8 {} + impl IsNan for i16 {} + impl IsNan for i32 {} + impl IsNan for i64 {} + + macro_rules! test_bop { + ($id:ident[$ety:ident; $ecount:expr] | + $binary_op:ident [$op_test_id:ident] : + ([$($in_a:expr),*], [$($in_b:expr),*]) => [$($out:expr),*]) => { + test_bop!( + $id[$ety; $ecount] => $ety | $binary_op [ $op_test_id ]: + ([$($in_a),*], [$($in_b),*]) => [$($out),*] + ); + + }; + ($id:ident[$ety:ident; $ecount:expr] => $oty:ident | + $binary_op:ident [$op_test_id:ident] : + ([$($in_a:expr),*], [$($in_b:expr),*]) => [$($out:expr),*]) => { + #[test] + fn $op_test_id() { + unsafe { + let a_input: [$ety; $ecount] = [$($in_a),*]; + let b_input: [$ety; $ecount] = [$($in_b),*]; + let output: [$oty; $ecount] = [$($out),*]; + + let a_vec_in: v128 = transmute(a_input); + let b_vec_in: v128 = transmute(b_input); + let vec_res: v128 = $binary_op(a_vec_in, b_vec_in); + + let res: [$oty; $ecount] = transmute(vec_res); + + if !floating_point!($ety) { + assert_eq!(res, output); + } else { + for i in 0..$ecount { + let r = res[i]; + let o = output[i]; + assert_eq!(r.is_nan(), o.is_nan()); + if !r.is_nan() { + assert_eq!(r, o); + } + } + } + } + } + } + } + + macro_rules! test_bops { + ($id:ident[$ety:ident; $ecount:expr] | + $binary_op:ident [$op_test_id:ident]: + ([$($in_a:expr),*], $in_b:expr) => [$($out:expr),*]) => { + #[test] + fn $op_test_id() { + unsafe { + let a_input: [$ety; $ecount] = [$($in_a),*]; + let output: [$ety; $ecount] = [$($out),*]; + + let a_vec_in: v128 = transmute(a_input); + let vec_res: v128 = $binary_op(a_vec_in, $in_b); + + let res: [$ety; $ecount] = transmute(vec_res); + assert_eq!(res, output); + } + } + } + } + + macro_rules! test_uop { + ($id:ident[$ety:ident; $ecount:expr] | + $unary_op:ident [$op_test_id:ident]: [$($in_a:expr),*] => [$($out:expr),*]) => { + #[test] + fn $op_test_id() { + unsafe { + let a_input: [$ety; $ecount] = [$($in_a),*]; + let output: [$ety; $ecount] = [$($out),*]; + + let a_vec_in: v128 = transmute(a_input); + let vec_res: v128 = $unary_op(a_vec_in); + + let res: [$ety; $ecount] = transmute(vec_res); + assert_eq!(res, output); + } + } + } + } + + test_bops!(i8x16[i8; 16] | i8x16_shl[i8x16_shl_test]: + ([0, -1, 2, 3, 4, 5, 6, i8::MAX, 1, 1, 1, 1, 1, 1, 1, 1], 1) => + [0, -2, 4, 6, 8, 10, 12, -2, 2, 2, 2, 2, 2, 2, 2, 2]); + test_bops!(i16x8[i16; 8] | i16x8_shl[i16x8_shl_test]: + ([0, -1, 2, 3, 4, 5, 6, i16::MAX], 1) => + [0, -2, 4, 6, 8, 10, 12, -2]); + test_bops!(i32x4[i32; 4] | i32x4_shl[i32x4_shl_test]: + ([0, -1, 2, 3], 1) => [0, -2, 4, 6]); + test_bops!(i64x2[i64; 2] | i64x2_shl[i64x2_shl_test]: + ([0, -1], 1) => [0, -2]); + + test_bops!(i8x16[i8; 16] | i8x16_shr[i8x16_shr_s_test]: + ([0, -1, 2, 3, 4, 5, 6, i8::MAX, 1, 1, 1, 1, 1, 1, 1, 1], 1) => + [0, -1, 1, 1, 2, 2, 3, 63, 0, 0, 0, 0, 0, 0, 0, 0]); + test_bops!(i16x8[i16; 8] | i16x8_shr[i16x8_shr_s_test]: + ([0, -1, 2, 3, 4, 5, 6, i16::MAX], 1) => + [0, -1, 1, 1, 2, 2, 3, i16::MAX / 2]); + test_bops!(i32x4[i32; 4] | i32x4_shr[i32x4_shr_s_test]: + ([0, -1, 2, 3], 1) => [0, -1, 1, 1]); + test_bops!(i64x2[i64; 2] | i64x2_shr[i64x2_shr_s_test]: + ([0, -1], 1) => [0, -1]); + + test_bops!(i8x16[i8; 16] | u8x16_shr[i8x16_uhr_u_test]: + ([0, -1, 2, 3, 4, 5, 6, i8::MAX, 1, 1, 1, 1, 1, 1, 1, 1], 1) => + [0, i8::MAX, 1, 1, 2, 2, 3, 63, 0, 0, 0, 0, 0, 0, 0, 0]); + test_bops!(i16x8[i16; 8] | u16x8_shr[i16x8_uhr_u_test]: + ([0, -1, 2, 3, 4, 5, 6, i16::MAX], 1) => + [0, i16::MAX, 1, 1, 2, 2, 3, i16::MAX / 2]); + test_bops!(i32x4[i32; 4] | u32x4_shr[i32x4_uhr_u_test]: + ([0, -1, 2, 3], 1) => [0, i32::MAX, 1, 1]); + test_bops!(i64x2[i64; 2] | u64x2_shr[i64x2_uhr_u_test]: + ([0, -1], 1) => [0, i64::MAX]); + + #[test] + fn v128_bitwise_logical_ops() { + unsafe { + let a: [u32; 4] = [u32::MAX, 0, u32::MAX, 0]; + let b: [u32; 4] = [u32::MAX; 4]; + let c: [u32; 4] = [0; 4]; + + let vec_a: v128 = transmute(a); + let vec_b: v128 = transmute(b); + let vec_c: v128 = transmute(c); + + let r: v128 = v128_and(vec_a, vec_a); + compare_bytes(r, vec_a); + let r: v128 = v128_and(vec_a, vec_b); + compare_bytes(r, vec_a); + let r: v128 = v128_andnot(vec_a, vec_b); + compare_bytes(r, vec_c); + let r: v128 = v128_andnot(vec_a, vec_a); + compare_bytes(r, vec_c); + let r: v128 = v128_andnot(vec_a, vec_c); + compare_bytes(r, vec_a); + let r: v128 = v128_or(vec_a, vec_b); + compare_bytes(r, vec_b); + let r: v128 = v128_not(vec_b); + compare_bytes(r, vec_c); + let r: v128 = v128_xor(vec_a, vec_c); + compare_bytes(r, vec_a); + + let r: v128 = v128_bitselect(vec_b, vec_c, vec_b); + compare_bytes(r, vec_b); + let r: v128 = v128_bitselect(vec_b, vec_c, vec_c); + compare_bytes(r, vec_c); + let r: v128 = v128_bitselect(vec_b, vec_c, vec_a); + compare_bytes(r, vec_a); + } + } + + macro_rules! test_bool_red { + ([$test_id:ident, $any:ident, $all:ident] | [$($true:expr),*] | [$($false:expr),*] | [$($alt:expr),*]) => { + #[test] + fn $test_id() { + unsafe { + let vec_a: v128 = transmute([$($true),*]); // true + let vec_b: v128 = transmute([$($false),*]); // false + let vec_c: v128 = transmute([$($alt),*]); // alternating + + // TODO + // assert_eq!($any(vec_a), true); + // assert_eq!($any(vec_b), false); + // assert_eq!($any(vec_c), true); + + assert_eq!($all(vec_a), true); + assert_eq!($all(vec_b), false); + assert_eq!($all(vec_c), false); + } + } + } + } + + test_bool_red!( + [i8x16_boolean_reductions, v128_any_true, i8x16_all_true] + | [1_i8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + | [0_i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + | [1_i8, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0] + ); + test_bool_red!( + [i16x8_boolean_reductions, v128_any_true, i16x8_all_true] + | [1_i16, 1, 1, 1, 1, 1, 1, 1] + | [0_i16, 0, 0, 0, 0, 0, 0, 0] + | [1_i16, 0, 1, 0, 1, 0, 1, 0] + ); + test_bool_red!( + [i32x4_boolean_reductions, v128_any_true, i32x4_all_true] + | [1_i32, 1, 1, 1] + | [0_i32, 0, 0, 0] + | [1_i32, 0, 1, 0] + ); + test_bool_red!( + [i64x2_boolean_reductions, v128_any_true, i64x2_all_true] + | [1_i64, 1] + | [0_i64, 0] + | [1_i64, 0] + ); + + test_bop!(i8x16[i8; 16] | i8x16_eq[i8x16_eq_test]: + ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + [0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, 15]) => + [-1, 0, -1, 0 ,-1, 0, -1, -1, -1, 0, -1, 0 ,-1, 0, -1, -1]); + test_bop!(i16x8[i16; 8] | i16x8_eq[i16x8_eq_test]: + ([0, 1, 2, 3, 4, 5, 6, 7], [0, 2, 2, 4, 4, 6, 6, 7]) => + [-1, 0, -1, 0 ,-1, 0, -1, -1]); + test_bop!(i32x4[i32; 4] | i32x4_eq[i32x4_eq_test]: + ([0, 1, 2, 3], [0, 2, 2, 4]) => [-1, 0, -1, 0]); + test_bop!(i64x2[i64; 2] | i64x2_eq[i64x2_eq_test]: + ([0, 1], [0, 2]) => [-1, 0]); + test_bop!(f32x4[f32; 4] => i32 | f32x4_eq[f32x4_eq_test]: + ([0., 1., 2., 3.], [0., 2., 2., 4.]) => [-1, 0, -1, 0]); + test_bop!(f64x2[f64; 2] => i64 | f64x2_eq[f64x2_eq_test]: ([0., 1.], [0., 2.]) => [-1, 0]); + + test_bop!(i8x16[i8; 16] | i8x16_ne[i8x16_ne_test]: + ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + [0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, 15]) => + [0, -1, 0, -1 ,0, -1, 0, 0, 0, -1, 0, -1 ,0, -1, 0, 0]); + test_bop!(i16x8[i16; 8] | i16x8_ne[i16x8_ne_test]: + ([0, 1, 2, 3, 4, 5, 6, 7], [0, 2, 2, 4, 4, 6, 6, 7]) => + [0, -1, 0, -1 ,0, -1, 0, 0]); + test_bop!(i32x4[i32; 4] | i32x4_ne[i32x4_ne_test]: + ([0, 1, 2, 3], [0, 2, 2, 4]) => [0, -1, 0, -1]); + test_bop!(i64x2[i64; 2] | i64x2_ne[i64x2_ne_test]: + ([0, 1], [0, 2]) => [0, -1]); + test_bop!(f32x4[f32; 4] => i32 | f32x4_ne[f32x4_ne_test]: + ([0., 1., 2., 3.], [0., 2., 2., 4.]) => [0, -1, 0, -1]); + test_bop!(f64x2[f64; 2] => i64 | f64x2_ne[f64x2_ne_test]: ([0., 1.], [0., 2.]) => [0, -1]); + + test_bop!(i8x16[i8; 16] | i8x16_lt[i8x16_lt_s_test]: + ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, -12, 13, 14, 15], + [0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, 15]) => + [0, -1, 0, -1 ,0, -1, 0, 0, 0, -1, 0, -1, -1, -1, 0, 0]); + test_bop!(i8x16[i8; 16] | u8x16_lt[i8x16_lt_u_test]: + ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, -12, 13, 14, 15], + [0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, 15]) => + [0, -1, 0, -1 ,0, -1, 0, 0, 0, -1, 0, -1 ,0, -1, 0, 0]); + test_bop!(i16x8[i16; 8] | i16x8_lt[i16x8_lt_s_test]: + ([0, 1, 2, 3, 4, 5, 6, -7], [0, 2, 2, 4, 4, 6, 6, 7]) => + [0, -1, 0, -1 ,0, -1, 0, -1]); + test_bop!(i16x8[i16; 8] | u16x8_lt[i16x8_lt_u_test]: + ([0, 1, 2, 3, 4, 5, 6, -7], [0, 2, 2, 4, 4, 6, 6, 7]) => + [0, -1, 0, -1 ,0, -1, 0, 0]); + test_bop!(i32x4[i32; 4] | i32x4_lt[i32x4_lt_s_test]: + ([-1, 1, 2, 3], [0, 2, 2, 4]) => [-1, -1, 0, -1]); + test_bop!(i32x4[i32; 4] | u32x4_lt[i32x4_lt_u_test]: + ([-1, 1, 2, 3], [0, 2, 2, 4]) => [0, -1, 0, -1]); + test_bop!(i64x2[i64; 2] | i64x2_lt[i64x2_lt_s_test]: + ([-1, 3], [0, 2]) => [-1, 0]); + test_bop!(f32x4[f32; 4] => i32 | f32x4_lt[f32x4_lt_test]: + ([0., 1., 2., 3.], [0., 2., 2., 4.]) => [0, -1, 0, -1]); + test_bop!(f64x2[f64; 2] => i64 | f64x2_lt[f64x2_lt_test]: ([0., 1.], [0., 2.]) => [0, -1]); + + test_bop!(i8x16[i8; 16] | i8x16_gt[i8x16_gt_s_test]: + ([0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, -15], + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) => + [0, -1, 0, -1 ,0, -1, 0, 0, 0, -1, 0, -1 ,0, -1, 0, 0]); + test_bop!(i8x16[i8; 16] | u8x16_gt[i8x16_gt_u_test]: + ([0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, -15], + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) => + [0, -1, 0, -1 ,0, -1, 0, 0, 0, -1, 0, -1 ,0, -1, 0, -1]); + test_bop!(i16x8[i16; 8] | i16x8_gt[i16x8_gt_s_test]: + ([0, 2, 2, 4, 4, 6, 6, -7], [0, 1, 2, 3, 4, 5, 6, 7]) => + [0, -1, 0, -1 ,0, -1, 0, 0]); + test_bop!(i16x8[i16; 8] | u16x8_gt[i16x8_gt_u_test]: + ([0, 2, 2, 4, 4, 6, 6, -7], [0, 1, 2, 3, 4, 5, 6, 7]) => + [0, -1, 0, -1 ,0, -1, 0, -1]); + test_bop!(i32x4[i32; 4] | i32x4_gt[i32x4_gt_s_test]: + ([0, 2, 2, -4], [0, 1, 2, 3]) => [0, -1, 0, 0]); + test_bop!(i32x4[i32; 4] | u32x4_gt[i32x4_gt_u_test]: + ([0, 2, 2, -4], [0, 1, 2, 3]) => [0, -1, 0, -1]); + test_bop!(i64x2[i64; 2] | i64x2_gt[i64x2_gt_s_test]: + ([-1, 2], [0, 1]) => [0, -1]); + test_bop!(f32x4[f32; 4] => i32 | f32x4_gt[f32x4_gt_test]: + ([0., 2., 2., 4.], [0., 1., 2., 3.]) => [0, -1, 0, -1]); + test_bop!(f64x2[f64; 2] => i64 | f64x2_gt[f64x2_gt_test]: ([0., 2.], [0., 1.]) => [0, -1]); + + test_bop!(i8x16[i8; 16] | i8x16_ge[i8x16_ge_s_test]: + ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, -15], + [0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, 15]) => + [-1, 0, -1, 0 ,-1, 0, -1, -1, -1, 0, -1, 0 ,-1, 0, -1, 0]); + test_bop!(i8x16[i8; 16] | u8x16_ge[i8x16_ge_u_test]: + ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, -15], + [0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, 15]) => + [-1, 0, -1, 0 ,-1, 0, -1, -1, -1, 0, -1, 0 ,-1, 0, -1, -1]); + test_bop!(i16x8[i16; 8] | i16x8_ge[i16x8_ge_s_test]: + ([0, 1, 2, 3, 4, 5, 6, -7], [0, 2, 2, 4, 4, 6, 6, 7]) => + [-1, 0, -1, 0 ,-1, 0, -1, 0]); + test_bop!(i16x8[i16; 8] | u16x8_ge[i16x8_ge_u_test]: + ([0, 1, 2, 3, 4, 5, 6, -7], [0, 2, 2, 4, 4, 6, 6, 7]) => + [-1, 0, -1, 0 ,-1, 0, -1, -1]); + test_bop!(i32x4[i32; 4] | i32x4_ge[i32x4_ge_s_test]: + ([0, 1, 2, -3], [0, 2, 2, 4]) => [-1, 0, -1, 0]); + test_bop!(i32x4[i32; 4] | u32x4_ge[i32x4_ge_u_test]: + ([0, 1, 2, -3], [0, 2, 2, 4]) => [-1, 0, -1, -1]); + test_bop!(i64x2[i64; 2] | i64x2_ge[i64x2_ge_s_test]: + ([0, 1], [-1, 2]) => [-1, 0]); + test_bop!(f32x4[f32; 4] => i32 | f32x4_ge[f32x4_ge_test]: + ([0., 1., 2., 3.], [0., 2., 2., 4.]) => [-1, 0, -1, 0]); + test_bop!(f64x2[f64; 2] => i64 | f64x2_ge[f64x2_ge_test]: ([0., 1.], [0., 2.]) => [-1, 0]); + + test_bop!(i8x16[i8; 16] | i8x16_le[i8x16_le_s_test]: + ([0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, -15], + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + ) => + [-1, 0, -1, 0 ,-1, 0, -1, -1, -1, 0, -1, 0 ,-1, 0, -1, -1]); + test_bop!(i8x16[i8; 16] | u8x16_le[i8x16_le_u_test]: + ([0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, -15], + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + ) => + [-1, 0, -1, 0 ,-1, 0, -1, -1, -1, 0, -1, 0 ,-1, 0, -1, 0]); + test_bop!(i16x8[i16; 8] | i16x8_le[i16x8_le_s_test]: + ([0, 2, 2, 4, 4, 6, 6, -7], [0, 1, 2, 3, 4, 5, 6, 7]) => + [-1, 0, -1, 0 ,-1, 0, -1, -1]); + test_bop!(i16x8[i16; 8] | u16x8_le[i16x8_le_u_test]: + ([0, 2, 2, 4, 4, 6, 6, -7], [0, 1, 2, 3, 4, 5, 6, 7]) => + [-1, 0, -1, 0 ,-1, 0, -1, 0]); + test_bop!(i32x4[i32; 4] | i32x4_le[i32x4_le_s_test]: + ([0, 2, 2, -4], [0, 1, 2, 3]) => [-1, 0, -1, -1]); + test_bop!(i32x4[i32; 4] | u32x4_le[i32x4_le_u_test]: + ([0, 2, 2, -4], [0, 1, 2, 3]) => [-1, 0, -1, 0]); + test_bop!(i64x2[i64; 2] | i64x2_le[i64x2_le_s_test]: + ([0, 2], [0, 1]) => [-1, 0]); + test_bop!(f32x4[f32; 4] => i32 | f32x4_le[f32x4_le_test]: + ([0., 2., 2., 4.], [0., 1., 2., 3.]) => [-1, 0, -1, -0]); + test_bop!(f64x2[f64; 2] => i64 | f64x2_le[f64x2_le_test]: ([0., 2.], [0., 1.]) => [-1, 0]); + + test_uop!(f32x4[f32; 4] | f32x4_neg[f32x4_neg_test]: [0., 1., 2., 3.] => [ 0., -1., -2., -3.]); + test_uop!(f32x4[f32; 4] | f32x4_abs[f32x4_abs_test]: [0., -1., 2., -3.] => [ 0., 1., 2., 3.]); + test_bop!(f32x4[f32; 4] | f32x4_min[f32x4_min_test]: + ([0., -1., 7., 8.], [1., -3., -4., 10.]) => [0., -3., -4., 8.]); + test_bop!(f32x4[f32; 4] | f32x4_min[f32x4_min_test_nan]: + ([0., -1., 7., 8.], [1., -3., -4., std::f32::NAN]) + => [0., -3., -4., std::f32::NAN]); + test_bop!(f32x4[f32; 4] | f32x4_max[f32x4_max_test]: + ([0., -1., 7., 8.], [1., -3., -4., 10.]) => [1., -1., 7., 10.]); + test_bop!(f32x4[f32; 4] | f32x4_max[f32x4_max_test_nan]: + ([0., -1., 7., 8.], [1., -3., -4., std::f32::NAN]) + => [1., -1., 7., std::f32::NAN]); + test_bop!(f32x4[f32; 4] | f32x4_add[f32x4_add_test]: + ([0., -1., 7., 8.], [1., -3., -4., 10.]) => [1., -4., 3., 18.]); + test_bop!(f32x4[f32; 4] | f32x4_sub[f32x4_sub_test]: + ([0., -1., 7., 8.], [1., -3., -4., 10.]) => [-1., 2., 11., -2.]); + test_bop!(f32x4[f32; 4] | f32x4_mul[f32x4_mul_test]: + ([0., -1., 7., 8.], [1., -3., -4., 10.]) => [0., 3., -28., 80.]); + test_bop!(f32x4[f32; 4] | f32x4_div[f32x4_div_test]: + ([0., -8., 70., 8.], [1., 4., 10., 2.]) => [0., -2., 7., 4.]); + + test_uop!(f64x2[f64; 2] | f64x2_neg[f64x2_neg_test]: [0., 1.] => [ 0., -1.]); + test_uop!(f64x2[f64; 2] | f64x2_abs[f64x2_abs_test]: [0., -1.] => [ 0., 1.]); + test_bop!(f64x2[f64; 2] | f64x2_min[f64x2_min_test]: + ([0., -1.], [1., -3.]) => [0., -3.]); + test_bop!(f64x2[f64; 2] | f64x2_min[f64x2_min_test_nan]: + ([7., 8.], [-4., std::f64::NAN]) + => [ -4., std::f64::NAN]); + test_bop!(f64x2[f64; 2] | f64x2_max[f64x2_max_test]: + ([0., -1.], [1., -3.]) => [1., -1.]); + test_bop!(f64x2[f64; 2] | f64x2_max[f64x2_max_test_nan]: + ([7., 8.], [ -4., std::f64::NAN]) + => [7., std::f64::NAN]); + test_bop!(f64x2[f64; 2] | f64x2_add[f64x2_add_test]: + ([0., -1.], [1., -3.]) => [1., -4.]); + test_bop!(f64x2[f64; 2] | f64x2_sub[f64x2_sub_test]: + ([0., -1.], [1., -3.]) => [-1., 2.]); + test_bop!(f64x2[f64; 2] | f64x2_mul[f64x2_mul_test]: + ([0., -1.], [1., -3.]) => [0., 3.]); + test_bop!(f64x2[f64; 2] | f64x2_div[f64x2_div_test]: + ([0., -8.], [1., 4.]) => [0., -2.]); + + macro_rules! test_conv { + ($test_id:ident | $conv_id:ident | $to_ty:ident | $from:expr, $to:expr) => { + #[test] + fn $test_id() { + unsafe { + let from: v128 = transmute($from); + let to: v128 = transmute($to); + + let r: v128 = $conv_id(from); + + compare_bytes(r, to); + } + } + }; + } + + test_conv!( + f32x4_convert_s_i32x4 | f32x4_convert_i32x4 | f32x4 | [1_i32, 2, 3, 4], + [1_f32, 2., 3., 4.] + ); + test_conv!( + f32x4_convert_u_i32x4 | f32x4_convert_u32x4 | f32x4 | [u32::MAX, 2, 3, 4], + [u32::MAX as f32, 2., 3., 4.] + ); + + #[test] + fn test_conversions() { + compare_bytes( + i32x4_trunc_sat_f32x4(f32x4(1., f32::NEG_INFINITY, f32::INFINITY, f32::NAN)), + i32x4(1, i32::MIN, i32::MAX, 0), + ); + compare_bytes( + u32x4_trunc_sat_f32x4(f32x4(1., f32::NEG_INFINITY, f32::INFINITY, f32::NAN)), + u32x4(1, 0, u32::MAX, 0), + ); + compare_bytes(f64x2_convert_low_i32x4(i32x4(1, 2, 3, 4)), f64x2(1., 2.)); + compare_bytes( + f64x2_convert_low_i32x4(i32x4(i32::MIN, i32::MAX, 3, 4)), + f64x2(f64::from(i32::MIN), f64::from(i32::MAX)), + ); + compare_bytes(f64x2_convert_low_u32x4(u32x4(1, 2, 3, 4)), f64x2(1., 2.)); + compare_bytes( + f64x2_convert_low_u32x4(u32x4(u32::MIN, u32::MAX, 3, 4)), + f64x2(f64::from(u32::MIN), f64::from(u32::MAX)), + ); + + compare_bytes( + i32x4_trunc_sat_f64x2_zero(f64x2(1., f64::NEG_INFINITY)), + i32x4(1, i32::MIN, 0, 0), + ); + compare_bytes( + i32x4_trunc_sat_f64x2_zero(f64x2(f64::NAN, f64::INFINITY)), + i32x4(0, i32::MAX, 0, 0), + ); + compare_bytes( + u32x4_trunc_sat_f64x2_zero(f64x2(1., f64::NEG_INFINITY)), + u32x4(1, 0, 0, 0), + ); + compare_bytes( + u32x4_trunc_sat_f64x2_zero(f64x2(f64::NAN, f64::INFINITY)), + u32x4(0, u32::MAX, 0, 0), + ); + } + + #[test] + fn test_popcnt() { + unsafe { + for i in 0..=255 { + compare_bytes( + i8x16_popcnt(u8x16_splat(i)), + u8x16_splat(i.count_ones() as u8), + ) + } + + let vectors = [ + [0u8, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + [ + 100, 200, 50, 0, 10, 7, 38, 185, 192, 3, 34, 85, 93, 7, 31, 99, + ], + ]; + + for vector in vectors.iter() { + compare_bytes( + i8x16_popcnt(transmute(*vector)), + i8x16( + vector[0].count_ones() as i8, + vector[1].count_ones() as i8, + vector[2].count_ones() as i8, + vector[3].count_ones() as i8, + vector[4].count_ones() as i8, + vector[5].count_ones() as i8, + vector[6].count_ones() as i8, + vector[7].count_ones() as i8, + vector[8].count_ones() as i8, + vector[9].count_ones() as i8, + vector[10].count_ones() as i8, + vector[11].count_ones() as i8, + vector[12].count_ones() as i8, + vector[13].count_ones() as i8, + vector[14].count_ones() as i8, + vector[15].count_ones() as i8, + ), + ) + } + } + } + + #[test] + fn test_promote_demote() { + let tests = [ + [1., 2.], + [f64::NAN, f64::INFINITY], + [100., 201.], + [0., -0.], + [f64::NEG_INFINITY, 0.], + ]; + + for [a, b] in tests { + compare_bytes( + f32x4_demote_f64x2_zero(f64x2(a, b)), + f32x4(a as f32, b as f32, 0., 0.), + ); + compare_bytes( + f64x2_promote_low_f32x4(f32x4(a as f32, b as f32, 0., 0.)), + f64x2(a, b), + ); + } + } + + #[test] + fn test_extmul() { + macro_rules! test { + ($( + $ctor:ident { + from: $from:ident, + to: $to:ident, + low: $low:ident, + high: $high:ident, + } => { + $(([$($a:tt)*] * [$($b:tt)*]))* + } + )*) => ($( + $(unsafe { + let a: [$from; 16 / mem::size_of::<$from>()] = [$($a)*]; + let b: [$from; 16 / mem::size_of::<$from>()] = [$($b)*]; + let low = mem::transmute::<_, [$to; 16 / mem::size_of::<$to>()]>($low($ctor($($a)*), $ctor($($b)*))); + let high = mem::transmute::<_, [$to; 16 / mem::size_of::<$to>()]>($high($ctor($($a)*), $ctor($($b)*))); + + let half = a.len() / 2; + for i in 0..half { + assert_eq!( + (a[i] as $to).wrapping_mul((b[i] as $to)), + low[i], + "expected {} * {}", a[i] as $to, b[i] as $to, + ); + assert_eq!( + (a[half + i] as $to).wrapping_mul((b[half + i] as $to)), + high[i], + "expected {} * {}", a[half + i] as $to, b[half + i] as $to, + ); + } + })* + )*) + } + test! { + i8x16 { + from: i8, + to: i16, + low: i16x8_extmul_low_i8x16, + high: i16x8_extmul_high_i8x16, + } => { + ( + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + * + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + ) + ( + [-1, -2, 3, 100, 124, -38, 33, 87, 92, 108, 22, 8, -43, -128, 22, 0] + * + [-5, -2, 6, 10, 45, -4, 4, -2, 0, 88, 92, -102, -98, 83, 73, 54] + ) + } + u8x16 { + from: u8, + to: u16, + low: u16x8_extmul_low_u8x16, + high: u16x8_extmul_high_u8x16, + } => { + ( + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + * + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + ) + ( + [1, 2, 3, 100, 124, 38, 33, 87, 92, 198, 22, 8, 43, 128, 22, 0] + * + [5, 200, 6, 10, 45, 248, 4, 2, 0, 2, 92, 102, 234, 83, 73, 54] + ) + } + i16x8 { + from: i16, + to: i32, + low: i32x4_extmul_low_i16x8, + high: i32x4_extmul_high_i16x8, + } => { + ( + [0, 0, 0, 0, 0, 0, 0, 0] + * + [0, 0, 0, 0, 0, 0, 0, 0] + ) + ( + [-1, 0, i16::MAX, 19931, -2259, 64, 200, 87] + * + [1, 1, i16::MIN, 29391, 105, 2, 100, -2] + ) + } + u16x8 { + from: u16, + to: u32, + low: u32x4_extmul_low_u16x8, + high: u32x4_extmul_high_u16x8, + } => { + ( + [0, 0, 0, 0, 0, 0, 0, 0] + * + [0, 0, 0, 0, 0, 0, 0, 0] + ) + ( + [1, 0, u16::MAX, 19931, 2259, 64, 200, 87] + * + [1, 1, 3, 29391, 105, 2, 100, 2] + ) + } + i32x4 { + from: i32, + to: i64, + low: i64x2_extmul_low_i32x4, + high: i64x2_extmul_high_i32x4, + } => { + ( + [0, 0, 0, 0] + * + [0, 0, 0, 0] + ) + ( + [-1, 0, i32::MAX, 19931] + * + [1, 1, i32::MIN, 29391] + ) + ( + [i32::MAX, 3003183, 3 << 20, 0xffffff] + * + [i32::MAX, i32::MIN, -40042, 300] + ) + } + u32x4 { + from: u32, + to: u64, + low: u64x2_extmul_low_u32x4, + high: u64x2_extmul_high_u32x4, + } => { + ( + [0, 0, 0, 0] + * + [0, 0, 0, 0] + ) + ( + [1, 0, u32::MAX, 19931] + * + [1, 1, 3, 29391] + ) + ( + [u32::MAX, 3003183, 3 << 20, 0xffffff] + * + [u32::MAX, 3000, 40042, 300] + ) + } + } + } + + #[test] + fn test_q15mulr_sat_s() { + fn test(a: [i16; 8], b: [i16; 8]) { + let a_v = i16x8(a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7]); + let b_v = i16x8(b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]); + let result = i16x8_q15mulr_sat(a_v, b_v); + let result = unsafe { mem::transmute::(result) }; + + for (i, (a, b)) in a.iter().zip(&b).enumerate() { + assert_eq!( + result[i], + (((*a as i32) * (*b as i32) + 0x4000) >> 15) as i16 + ); + } + } + + test([0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0]); + test([1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1]); + test( + [-1, 100, 2003, -29494, 12, 128, 994, 1], + [-4049, 8494, -10483, 0, 5, 2222, 883, -9], + ); + } + + #[test] + fn test_extadd() { + macro_rules! test { + ($( + $func:ident { + from: $from:ident, + to: $to:ident, + } => { + $([$($a:tt)*])* + } + )*) => ($( + $(unsafe { + let a: [$from; 16 / mem::size_of::<$from>()] = [$($a)*]; + let a_v = mem::transmute::<_, v128>(a); + let r = mem::transmute::()]>($func(a_v)); + + let half = a.len() / 2; + for i in 0..half { + assert_eq!( + (a[2 * i] as $to).wrapping_add((a[2 * i + 1] as $to)), + r[i], + "failed {} + {} != {}", + a[2 * i] as $to, + a[2 * i + 1] as $to, + r[i], + ); + } + })* + )*) + } + test! { + i16x8_extadd_pairwise_i8x16 { + from: i8, + to: i16, + } => { + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + [-1, -2, 3, 100, 124, -38, 33, 87, 92, 108, 22, 8, -43, -128, 22, 0] + [-5, -2, 6, 10, 45, -4, 4, -2, 0, 88, 92, -102, -98, 83, 73, 54] + } + i16x8_extadd_pairwise_u8x16 { + from: u8, + to: i16, + } => { + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + [1, 2, 3, 100, 124, 38, 33, 87, 92, 198, 22, 8, 43, 128, 22, 0] + [5, 200, 6, 10, 45, 248, 4, 2, 0, 2, 92, 102, 234, 83, 73, 54] + } + i32x4_extadd_pairwise_i16x8 { + from: i16, + to: i32, + } => { + [0, 0, 0, 0, 0, 0, 0, 0] + [-1, 0, i16::MAX, 19931, -2259, 64, 200, 87] + [1, 1, i16::MIN, 29391, 105, 2, 100, -2] + } + i32x4_extadd_pairwise_u16x8 { + from: u16, + to: i32, + } => { + [0, 0, 0, 0, 0, 0, 0, 0] + [1, 0, u16::MAX, 19931, 2259, 64, 200, 87] + [1, 1, 3, 29391, 105, 2, 100, 2] + } + } + } } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/adx.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/adx.rs index 6df321c04..15fcec8a5 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/adx.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/adx.rs @@ -55,7 +55,7 @@ mod tests { #[test] fn test_addcarry_u32() { unsafe { - let a = u32::max_value(); + let a = u32::MAX; let mut out = 0; let r = _addcarry_u32(0, a, 1, &mut out); @@ -86,7 +86,7 @@ mod tests { #[simd_test(enable = "adx")] unsafe fn test_addcarryx_u32() { - let a = u32::max_value(); + let a = u32::MAX; let mut out = 0; let r = _addcarryx_u32(0, a, 1, &mut out); @@ -127,7 +127,7 @@ mod tests { #[test] fn test_subborrow_u32() { unsafe { - let a = u32::max_value(); + let a = u32::MAX; let mut out = 0; let r = _subborrow_u32(0, 0, 1, &mut out); diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/aes.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/aes.rs index 603744aef..ffded1a0d 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/aes.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/aes.rs @@ -87,21 +87,17 @@ pub unsafe fn _mm_aesimc_si128(a: __m128i) -> __m128i { /// /// Assist in expanding the AES cipher key by computing steps towards /// generating a round key for encryption cipher using data from `a` and an -/// 8-bit round constant `imm8`. +/// 8-bit round constant `IMM8`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_aeskeygenassist_si128) #[inline] #[target_feature(enable = "aes")] -#[cfg_attr(test, assert_instr(aeskeygenassist, imm8 = 0))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(aeskeygenassist, IMM8 = 0))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_aeskeygenassist_si128(a: __m128i, imm8: i32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - aeskeygenassist(a, $imm8) - }; - } - constify_imm8!(imm8, call) +pub unsafe fn _mm_aeskeygenassist_si128(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + aeskeygenassist(a, IMM8 as u8) } #[cfg(test)] @@ -169,7 +165,7 @@ mod tests { // Constants taken from https://msdn.microsoft.com/en-us/library/cc714138.aspx. let a = _mm_set_epi64x(0x0123456789abcdef, 0x8899aabbccddeeff); let e = _mm_set_epi64x(0x857c266b7c266e85, 0xeac4eea9c4eeacea); - let r = _mm_aeskeygenassist_si128(a, 5); + let r = _mm_aeskeygenassist_si128::<5>(a); assert_eq_m128i(r, e); } } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx.rs index 68896e0aa..ad9e68db6 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx.rs @@ -53,7 +53,7 @@ pub unsafe fn _mm256_add_ps(a: __m256, b: __m256) -> __m256 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_and_pd) #[inline] #[target_feature(enable = "avx")] -// FIXME: Should be 'vandpd' instuction. +// FIXME: Should be 'vandpd' instruction. // See https://github.com/rust-lang/stdarch/issues/71 #[cfg_attr(test, assert_instr(vandps))] #[stable(feature = "simd_x86", since = "1.27.0")] @@ -83,7 +83,7 @@ pub unsafe fn _mm256_and_ps(a: __m256, b: __m256) -> __m256 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_or_pd) #[inline] #[target_feature(enable = "avx")] -// FIXME: should be `vorpd` instuction. +// FIXME: should be `vorpd` instruction. // See . #[cfg_attr(test, assert_instr(vorps))] #[stable(feature = "simd_x86", since = "1.27.0")] @@ -113,44 +113,21 @@ pub unsafe fn _mm256_or_ps(a: __m256, b: __m256) -> __m256 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shuffle_pd) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vshufpd, imm8 = 0x1))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_shuffle_pd(a: __m256d, b: __m256d, imm8: i32) -> __m256d { - let imm8 = (imm8 & 0xFF) as u8; - macro_rules! shuffle4 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - simd_shuffle4(a, b, [$a, $b, $c, $d]); - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr) => { - match (imm8 >> 3) & 0x1 { - 0 => shuffle4!($a, $b, $c, 6), - _ => shuffle4!($a, $b, $c, 7), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr) => { - match (imm8 >> 2) & 0x1 { - 0 => shuffle3!($a, $b, 2), - _ => shuffle3!($a, $b, 3), - } - }; - } - macro_rules! shuffle1 { - ($a:expr) => { - match (imm8 >> 1) & 0x1 { - 0 => shuffle2!($a, 4), - _ => shuffle2!($a, 5), - } - }; - } - match imm8 & 0x1 { - 0 => shuffle1!(0), - _ => shuffle1!(1), - } +#[cfg_attr(test, assert_instr(vshufpd, MASK = 3))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_shuffle_pd(a: __m256d, b: __m256d) -> __m256d { + static_assert_imm8!(MASK); + simd_shuffle4!( + a, + b, + [ + MASK as u32 & 0b1, + ((MASK as u32 >> 1) & 0b1) + 4, + ((MASK as u32 >> 2) & 0b1) + 2, + ((MASK as u32 >> 3) & 0b1) + 6, + ], + ) } /// Shuffles single-precision (32-bit) floating-point elements in `a` within @@ -159,61 +136,25 @@ pub unsafe fn _mm256_shuffle_pd(a: __m256d, b: __m256d, imm8: i32) -> __m256d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shuffle_ps) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vshufps, imm8 = 0x0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_shuffle_ps(a: __m256, b: __m256, imm8: i32) -> __m256 { - let imm8 = (imm8 & 0xFF) as u8; - macro_rules! shuffle4 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr, - $e:expr, - $f:expr, - $g:expr, - $h:expr - ) => { - simd_shuffle8(a, b, [$a, $b, $c, $d, $e, $f, $g, $h]); - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr, $e:expr, $f:expr, $g:expr) => { - match (imm8 >> 6) & 0x3 { - 0 => shuffle4!($a, $b, $c, 8, $e, $f, $g, 12), - 1 => shuffle4!($a, $b, $c, 9, $e, $f, $g, 13), - 2 => shuffle4!($a, $b, $c, 10, $e, $f, $g, 14), - _ => shuffle4!($a, $b, $c, 11, $e, $f, $g, 15), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr, $e:expr, $f:expr) => { - match (imm8 >> 4) & 0x3 { - 0 => shuffle3!($a, $b, 8, $e, $f, 12), - 1 => shuffle3!($a, $b, 9, $e, $f, 13), - 2 => shuffle3!($a, $b, 10, $e, $f, 14), - _ => shuffle3!($a, $b, 11, $e, $f, 15), - } - }; - } - macro_rules! shuffle1 { - ($a:expr, $e:expr) => { - match (imm8 >> 2) & 0x3 { - 0 => shuffle2!($a, 0, $e, 4), - 1 => shuffle2!($a, 1, $e, 5), - 2 => shuffle2!($a, 2, $e, 6), - _ => shuffle2!($a, 3, $e, 7), - } - }; - } - match imm8 & 0x3 { - 0 => shuffle1!(0, 4), - 1 => shuffle1!(1, 5), - 2 => shuffle1!(2, 6), - _ => shuffle1!(3, 7), - } +#[cfg_attr(test, assert_instr(vshufps, MASK = 3))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_shuffle_ps(a: __m256, b: __m256) -> __m256 { + static_assert_imm8!(MASK); + simd_shuffle8!( + a, + b, + [ + MASK as u32 & 0b11, + (MASK as u32 >> 2) & 0b11, + ((MASK as u32 >> 4) & 0b11) + 8, + ((MASK as u32 >> 6) & 0b11) + 8, + (MASK as u32 & 0b11) + 4, + ((MASK as u32 >> 2) & 0b11) + 4, + ((MASK as u32 >> 4) & 0b11) + 12, + ((MASK as u32 >> 6) & 0b11) + 12, + ], + ) } /// Computes the bitwise NOT of packed double-precision (64-bit) floating-point @@ -255,7 +196,7 @@ pub unsafe fn _mm256_andnot_ps(a: __m256, b: __m256) -> __m256 { #[cfg_attr(test, assert_instr(vmaxpd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_max_pd(a: __m256d, b: __m256d) -> __m256d { - simd_fmax(a, b) + vmaxpd(a, b) } /// Compares packed single-precision (32-bit) floating-point elements in `a` @@ -267,7 +208,7 @@ pub unsafe fn _mm256_max_pd(a: __m256d, b: __m256d) -> __m256d { #[cfg_attr(test, assert_instr(vmaxps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_max_ps(a: __m256, b: __m256) -> __m256 { - simd_fmax(a, b) + vmaxps(a, b) } /// Compares packed double-precision (64-bit) floating-point elements @@ -279,7 +220,7 @@ pub unsafe fn _mm256_max_ps(a: __m256, b: __m256) -> __m256 { #[cfg_attr(test, assert_instr(vminpd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_min_pd(a: __m256d, b: __m256d) -> __m256d { - simd_fmin(a, b) + vminpd(a, b) } /// Compares packed single-precision (32-bit) floating-point elements in `a` @@ -291,7 +232,7 @@ pub unsafe fn _mm256_min_pd(a: __m256d, b: __m256d) -> __m256d { #[cfg_attr(test, assert_instr(vminps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_min_ps(a: __m256, b: __m256) -> __m256 { - simd_fmin(a, b) + vminps(a, b) } /// Multiplies packed double-precision (64-bit) floating-point elements @@ -391,7 +332,7 @@ pub unsafe fn _mm256_div_pd(a: __m256d, b: __m256d) -> __m256d { } /// Rounds packed double-precision (64-bit) floating point elements in `a` -/// according to the flag `b`. The value of `b` may be as follows: +/// according to the flag `ROUNDING`. The value of `ROUNDING` may be as follows: /// /// - `0x00`: Round to the nearest whole number. /// - `0x01`: Round down, toward negative infinity. @@ -405,16 +346,12 @@ pub unsafe fn _mm256_div_pd(a: __m256d, b: __m256d) -> __m256d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_round_pd) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vroundpd, b = 0x3))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(vroundpd, ROUNDING = 0x3))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_round_pd(a: __m256d, b: i32) -> __m256d { - macro_rules! call { - ($imm8:expr) => { - roundpd256(a, $imm8) - }; - } - constify_imm8!(b, call) +pub unsafe fn _mm256_round_pd(a: __m256d) -> __m256d { + static_assert_imm4!(ROUNDING); + roundpd256(a, ROUNDING) } /// Rounds packed double-precision (64-bit) floating point elements in `a` @@ -442,7 +379,7 @@ pub unsafe fn _mm256_floor_pd(a: __m256d) -> __m256d { } /// Rounds packed single-precision (32-bit) floating point elements in `a` -/// according to the flag `b`. The value of `b` may be as follows: +/// according to the flag `ROUNDING`. The value of `ROUNDING` may be as follows: /// /// - `0x00`: Round to the nearest whole number. /// - `0x01`: Round down, toward negative infinity. @@ -456,16 +393,12 @@ pub unsafe fn _mm256_floor_pd(a: __m256d) -> __m256d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_round_ps) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vroundps, b = 0x00))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(vroundps, ROUNDING = 0x00))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_round_ps(a: __m256, b: i32) -> __m256 { - macro_rules! call { - ($imm8:expr) => { - roundps256(a, $imm8) - }; - } - constify_imm8!(b, call) +pub unsafe fn _mm256_round_ps(a: __m256) -> __m256 { + static_assert_imm4!(ROUNDING); + roundps256(a, ROUNDING) } /// Rounds packed single-precision (32-bit) floating point elements in `a` @@ -525,44 +458,21 @@ pub unsafe fn _mm256_sqrt_pd(a: __m256d) -> __m256d { // Note: LLVM7 prefers single-precision blend instructions when // possible, see: https://bugs.llvm.org/show_bug.cgi?id=38194 // #[cfg_attr(test, assert_instr(vblendpd, imm8 = 9))] -#[cfg_attr(test, assert_instr(vblendps, imm8 = 9))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_blend_pd(a: __m256d, b: __m256d, imm8: i32) -> __m256d { - let imm8 = (imm8 & 0xFF) as u8; - macro_rules! blend4 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - simd_shuffle4(a, b, [$a, $b, $c, $d]); - }; - } - macro_rules! blend3 { - ($a:expr, $b:expr, $c:expr) => { - match imm8 & 0x8 { - 0 => blend4!($a, $b, $c, 3), - _ => blend4!($a, $b, $c, 7), - } - }; - } - macro_rules! blend2 { - ($a:expr, $b:expr) => { - match imm8 & 0x4 { - 0 => blend3!($a, $b, 2), - _ => blend3!($a, $b, 6), - } - }; - } - macro_rules! blend1 { - ($a:expr) => { - match imm8 & 0x2 { - 0 => blend2!($a, 1), - _ => blend2!($a, 5), - } - }; - } - match imm8 & 0x1 { - 0 => blend1!(0), - _ => blend1!(4), - } +#[cfg_attr(test, assert_instr(vblendps, IMM4 = 9))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_blend_pd(a: __m256d, b: __m256d) -> __m256d { + static_assert_imm4!(IMM4); + simd_shuffle4!( + a, + b, + [ + ((IMM4 as u32 >> 0) & 1) * 4 + 0, + ((IMM4 as u32 >> 1) & 1) * 4 + 1, + ((IMM4 as u32 >> 2) & 1) * 4 + 2, + ((IMM4 as u32 >> 3) & 1) * 4 + 3, + ], + ) } /// Blends packed single-precision (32-bit) floating-point elements from @@ -571,61 +481,25 @@ pub unsafe fn _mm256_blend_pd(a: __m256d, b: __m256d, imm8: i32) -> __m256d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_blend_ps) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vblendps, imm8 = 9))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_blend_ps(a: __m256, b: __m256, imm8: i32) -> __m256 { - let imm8 = (imm8 & 0xFF) as u8; - macro_rules! blend4 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr, - $e:expr, - $f:expr, - $g:expr, - $h:expr - ) => { - simd_shuffle8(a, b, [$a, $b, $c, $d, $e, $f, $g, $h]); - }; - } - macro_rules! blend3 { - ($a:expr, $b:expr, $c:expr, $d:expr, $e:expr, $f:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => blend4!($a, $b, $c, $d, $e, $f, 6, 7), - 0b01 => blend4!($a, $b, $c, $d, $e, $f, 14, 7), - 0b10 => blend4!($a, $b, $c, $d, $e, $f, 6, 15), - _ => blend4!($a, $b, $c, $d, $e, $f, 14, 15), - } - }; - } - macro_rules! blend2 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => blend3!($a, $b, $c, $d, 4, 5), - 0b01 => blend3!($a, $b, $c, $d, 12, 5), - 0b10 => blend3!($a, $b, $c, $d, 4, 13), - _ => blend3!($a, $b, $c, $d, 12, 13), - } - }; - } - macro_rules! blend1 { - ($a:expr, $b:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => blend2!($a, $b, 2, 3), - 0b01 => blend2!($a, $b, 10, 3), - 0b10 => blend2!($a, $b, 2, 11), - _ => blend2!($a, $b, 10, 11), - } - }; - } - match imm8 & 0b11 { - 0b00 => blend1!(0, 1), - 0b01 => blend1!(8, 1), - 0b10 => blend1!(0, 9), - _ => blend1!(8, 9), - } +#[cfg_attr(test, assert_instr(vblendps, IMM8 = 9))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_blend_ps(a: __m256, b: __m256) -> __m256 { + static_assert_imm8!(IMM8); + simd_shuffle8!( + a, + b, + [ + ((IMM8 as u32 >> 0) & 1) * 8 + 0, + ((IMM8 as u32 >> 1) & 1) * 8 + 1, + ((IMM8 as u32 >> 2) & 1) * 8 + 2, + ((IMM8 as u32 >> 3) & 1) * 8 + 3, + ((IMM8 as u32 >> 4) & 1) * 8 + 4, + ((IMM8 as u32 >> 5) & 1) * 8 + 5, + ((IMM8 as u32 >> 6) & 1) * 8 + 6, + ((IMM8 as u32 >> 7) & 1) * 8 + 7, + ], + ) } /// Blends packed double-precision (64-bit) floating-point elements from @@ -660,16 +534,12 @@ pub unsafe fn _mm256_blendv_ps(a: __m256, b: __m256, c: __m256) -> __m256 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_dp_ps) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vdpps, imm8 = 0x0))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vdpps, IMM8 = 0x0))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_dp_ps(a: __m256, b: __m256, imm8: i32) -> __m256 { - macro_rules! call { - ($imm8:expr) => { - vdpps(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +pub unsafe fn _mm256_dp_ps(a: __m256, b: __m256) -> __m256 { + static_assert_imm8!(IMM8); + vdpps(a, b, IMM8) } /// Horizontal addition of adjacent pairs in the two packed vectors @@ -858,82 +728,66 @@ pub const _CMP_TRUE_US: i32 = 0x1f; /// Compares packed double-precision (64-bit) floating-point /// elements in `a` and `b` based on the comparison operand -/// specified by `imm8`. +/// specified by `IMM5`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_pd) #[inline] #[target_feature(enable = "avx,sse2")] -#[cfg_attr(test, assert_instr(vcmpeqpd, imm8 = 0))] // TODO Validate vcmppd -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vcmpeqpd, IMM5 = 0))] // TODO Validate vcmppd +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmp_pd(a: __m128d, b: __m128d, imm8: i32) -> __m128d { - macro_rules! call { - ($imm8:expr) => { - vcmppd(a, b, $imm8) - }; - } - constify_imm6!(imm8, call) +pub unsafe fn _mm_cmp_pd(a: __m128d, b: __m128d) -> __m128d { + static_assert_imm5!(IMM5); + vcmppd(a, b, IMM5 as i8) } /// Compares packed double-precision (64-bit) floating-point /// elements in `a` and `b` based on the comparison operand -/// specified by `imm8`. +/// specified by `IMM5`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_pd) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vcmpeqpd, imm8 = 0))] // TODO Validate vcmppd -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vcmpeqpd, IMM5 = 0))] // TODO Validate vcmppd +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_cmp_pd(a: __m256d, b: __m256d, imm8: i32) -> __m256d { - macro_rules! call { - ($imm8:expr) => { - vcmppd256(a, b, $imm8) - }; - } - constify_imm6!(imm8, call) +pub unsafe fn _mm256_cmp_pd(a: __m256d, b: __m256d) -> __m256d { + static_assert_imm5!(IMM5); + vcmppd256(a, b, IMM5 as u8) } /// Compares packed single-precision (32-bit) floating-point /// elements in `a` and `b` based on the comparison operand -/// specified by `imm8`. +/// specified by `IMM5`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_ps) #[inline] #[target_feature(enable = "avx,sse")] -#[cfg_attr(test, assert_instr(vcmpeqps, imm8 = 0))] // TODO Validate vcmpps -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vcmpeqps, IMM5 = 0))] // TODO Validate vcmpps +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmp_ps(a: __m128, b: __m128, imm8: i32) -> __m128 { - macro_rules! call { - ($imm8:expr) => { - vcmpps(a, b, $imm8) - }; - } - constify_imm6!(imm8, call) +pub unsafe fn _mm_cmp_ps(a: __m128, b: __m128) -> __m128 { + static_assert_imm5!(IMM5); + vcmpps(a, b, IMM5 as i8) } /// Compares packed single-precision (32-bit) floating-point /// elements in `a` and `b` based on the comparison operand -/// specified by `imm8`. +/// specified by `IMM5`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_ps) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vcmpeqps, imm8 = 0))] // TODO Validate vcmpps -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vcmpeqps, IMM5 = 0))] // TODO Validate vcmpps +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_cmp_ps(a: __m256, b: __m256, imm8: i32) -> __m256 { - macro_rules! call { - ($imm8:expr) => { - vcmpps256(a, b, $imm8) - }; - } - constify_imm6!(imm8, call) +pub unsafe fn _mm256_cmp_ps(a: __m256, b: __m256) -> __m256 { + static_assert_imm5!(IMM5); + vcmpps256(a, b, IMM5 as u8) } /// Compares the lower double-precision (64-bit) floating-point element in -/// `a` and `b` based on the comparison operand specified by `imm8`, +/// `a` and `b` based on the comparison operand specified by `IMM5`, /// store the result in the lower element of returned vector, /// and copies the upper element from `a` to the upper element of returned /// vector. @@ -941,20 +795,16 @@ pub unsafe fn _mm256_cmp_ps(a: __m256, b: __m256, imm8: i32) -> __m256 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_sd) #[inline] #[target_feature(enable = "avx,sse2")] -#[cfg_attr(test, assert_instr(vcmpeqsd, imm8 = 0))] // TODO Validate vcmpsd -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vcmpeqsd, IMM5 = 0))] // TODO Validate vcmpsd +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmp_sd(a: __m128d, b: __m128d, imm8: i32) -> __m128d { - macro_rules! call { - ($imm8:expr) => { - vcmpsd(a, b, $imm8) - }; - } - constify_imm6!(imm8, call) +pub unsafe fn _mm_cmp_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_imm5!(IMM5); + vcmpsd(a, b, IMM5 as i8) } /// Compares the lower single-precision (32-bit) floating-point element in -/// `a` and `b` based on the comparison operand specified by `imm8`, +/// `a` and `b` based on the comparison operand specified by `IMM5`, /// store the result in the lower element of returned vector, /// and copies the upper 3 packed elements from `a` to the upper elements of /// returned vector. @@ -962,16 +812,12 @@ pub unsafe fn _mm_cmp_sd(a: __m128d, b: __m128d, imm8: i32) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_ss) #[inline] #[target_feature(enable = "avx,sse")] -#[cfg_attr(test, assert_instr(vcmpeqss, imm8 = 0))] // TODO Validate vcmpss -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vcmpeqss, IMM5 = 0))] // TODO Validate vcmpss +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmp_ss(a: __m128, b: __m128, imm8: i32) -> __m128 { - macro_rules! call { - ($imm8:expr) => { - vcmpss(a, b, $imm8) - }; - } - constify_imm6!(imm8, call) +pub unsafe fn _mm_cmp_ss(a: __m128, b: __m128) -> __m128 { + static_assert_imm5!(IMM5); + vcmpss(a, b, IMM5 as i8) } /// Converts packed 32-bit integers in `a` to packed double-precision (64-bit) @@ -1078,15 +924,17 @@ pub unsafe fn _mm256_cvttps_epi32(a: __m256) -> __m256i { #[target_feature(enable = "avx")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextractf128, imm8 = 1) + assert_instr(vextractf128, IMM1 = 1) )] -#[rustc_args_required_const(1)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_extractf128_ps(a: __m256, imm8: i32) -> __m128 { - match imm8 & 1 { - 0 => simd_shuffle4(a, _mm256_undefined_ps(), [0, 1, 2, 3]), - _ => simd_shuffle4(a, _mm256_undefined_ps(), [4, 5, 6, 7]), - } +#[rustc_legacy_const_generics(1)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_extractf128_ps(a: __m256) -> __m128 { + static_assert_imm1!(IMM1); + simd_shuffle4!( + a, + _mm256_undefined_ps(), + [[0, 1, 2, 3], [4, 5, 6, 7]][IMM1 as usize], + ) } /// Extracts 128 bits (composed of 2 packed double-precision (64-bit) @@ -1097,15 +945,13 @@ pub unsafe fn _mm256_extractf128_ps(a: __m256, imm8: i32) -> __m128 { #[target_feature(enable = "avx")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextractf128, imm8 = 1) + assert_instr(vextractf128, IMM1 = 1) )] -#[rustc_args_required_const(1)] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_extractf128_pd(a: __m256d, imm8: i32) -> __m128d { - match imm8 & 1 { - 0 => simd_shuffle2(a, _mm256_undefined_pd(), [0, 1]), - _ => simd_shuffle2(a, _mm256_undefined_pd(), [2, 3]), - } +pub unsafe fn _mm256_extractf128_pd(a: __m256d) -> __m128d { + static_assert_imm1!(IMM1); + simd_shuffle2!(a, _mm256_undefined_pd(), [[0, 1], [2, 3]][IMM1 as usize]) } /// Extracts 128 bits (composed of integer data) from `a`, selected with `imm8`. @@ -1115,16 +961,17 @@ pub unsafe fn _mm256_extractf128_pd(a: __m256d, imm8: i32) -> __m128d { #[target_feature(enable = "avx")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextractf128, imm8 = 1) + assert_instr(vextractf128, IMM1 = 1) )] -#[rustc_args_required_const(1)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_extractf128_si256(a: __m256i, imm8: i32) -> __m128i { - let b = _mm256_undefined_si256().as_i64x4(); - let dst: i64x2 = match imm8 & 1 { - 0 => simd_shuffle2(a.as_i64x4(), b, [0, 1]), - _ => simd_shuffle2(a.as_i64x4(), b, [2, 3]), - }; +#[rustc_legacy_const_generics(1)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_extractf128_si256(a: __m256i) -> __m128i { + static_assert_imm1!(IMM1); + let dst: i64x2 = simd_shuffle2!( + a.as_i64x4(), + _mm256_undefined_si256().as_i64x4(), + [[0, 1], [2, 3]][IMM1 as usize], + ); transmute(dst) } @@ -1181,56 +1028,25 @@ pub unsafe fn _mm_permutevar_ps(a: __m128, b: __m128i) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permute_ps) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vpermilps, imm8 = 9))] -#[rustc_args_required_const(1)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_permute_ps(a: __m256, imm8: i32) -> __m256 { - let imm8 = (imm8 & 0xFF) as u8; - macro_rules! shuffle4 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - simd_shuffle8( - a, - _mm256_undefined_ps(), - [$a, $b, $c, $d, $a + 4, $b + 4, $c + 4, $d + 4], - ) - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle4!($a, $b, $c, 0), - 0b01 => shuffle4!($a, $b, $c, 1), - 0b10 => shuffle4!($a, $b, $c, 2), - _ => shuffle4!($a, $b, $c, 3), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle3!($a, $b, 0), - 0b01 => shuffle3!($a, $b, 1), - 0b10 => shuffle3!($a, $b, 2), - _ => shuffle3!($a, $b, 3), - } - }; - } - macro_rules! shuffle1 { - ($a:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle2!($a, 0), - 0b01 => shuffle2!($a, 1), - 0b10 => shuffle2!($a, 2), - _ => shuffle2!($a, 3), - } - }; - } - match imm8 & 0b11 { - 0b00 => shuffle1!(0), - 0b01 => shuffle1!(1), - 0b10 => shuffle1!(2), - _ => shuffle1!(3), - } +#[cfg_attr(test, assert_instr(vpermilps, IMM8 = 9))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_permute_ps(a: __m256) -> __m256 { + static_assert_imm8!(IMM8); + simd_shuffle8!( + a, + _mm256_undefined_ps(), + [ + (IMM8 as u32 >> 0) & 0b11, + (IMM8 as u32 >> 2) & 0b11, + (IMM8 as u32 >> 4) & 0b11, + (IMM8 as u32 >> 6) & 0b11, + ((IMM8 as u32 >> 0) & 0b11) + 4, + ((IMM8 as u32 >> 2) & 0b11) + 4, + ((IMM8 as u32 >> 4) & 0b11) + 4, + ((IMM8 as u32 >> 6) & 0b11) + 4, + ], + ) } /// Shuffles single-precision (32-bit) floating-point elements in `a` @@ -1239,52 +1055,21 @@ pub unsafe fn _mm256_permute_ps(a: __m256, imm8: i32) -> __m256 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_permute_ps) #[inline] #[target_feature(enable = "avx,sse")] -#[cfg_attr(test, assert_instr(vpermilps, imm8 = 9))] -#[rustc_args_required_const(1)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_permute_ps(a: __m128, imm8: i32) -> __m128 { - let imm8 = (imm8 & 0xFF) as u8; - macro_rules! shuffle4 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - simd_shuffle4(a, _mm_undefined_ps(), [$a, $b, $c, $d]) - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle4!($a, $b, $c, 0), - 0b01 => shuffle4!($a, $b, $c, 1), - 0b10 => shuffle4!($a, $b, $c, 2), - _ => shuffle4!($a, $b, $c, 3), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle3!($a, $b, 0), - 0b01 => shuffle3!($a, $b, 1), - 0b10 => shuffle3!($a, $b, 2), - _ => shuffle3!($a, $b, 3), - } - }; - } - macro_rules! shuffle1 { - ($a:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle2!($a, 0), - 0b01 => shuffle2!($a, 1), - 0b10 => shuffle2!($a, 2), - _ => shuffle2!($a, 3), - } - }; - } - match imm8 & 0b11 { - 0b00 => shuffle1!(0), - 0b01 => shuffle1!(1), - 0b10 => shuffle1!(2), - _ => shuffle1!(3), - } +#[cfg_attr(test, assert_instr(vpermilps, IMM8 = 9))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_permute_ps(a: __m128) -> __m128 { + static_assert_imm8!(IMM8); + simd_shuffle4!( + a, + _mm_undefined_ps(), + [ + (IMM8 as u32 >> 0) & 0b11, + (IMM8 as u32 >> 2) & 0b11, + (IMM8 as u32 >> 4) & 0b11, + (IMM8 as u32 >> 6) & 0b11, + ], + ) } /// Shuffles double-precision (64-bit) floating-point elements in `a` @@ -1317,44 +1102,21 @@ pub unsafe fn _mm_permutevar_pd(a: __m128d, b: __m128i) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permute_pd) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vpermilpd, imm8 = 0x1))] -#[rustc_args_required_const(1)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_permute_pd(a: __m256d, imm8: i32) -> __m256d { - let imm8 = (imm8 & 0xFF) as u8; - macro_rules! shuffle4 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - simd_shuffle4(a, _mm256_undefined_pd(), [$a, $b, $c, $d]); - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr) => { - match (imm8 >> 3) & 0x1 { - 0 => shuffle4!($a, $b, $c, 2), - _ => shuffle4!($a, $b, $c, 3), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr) => { - match (imm8 >> 2) & 0x1 { - 0 => shuffle3!($a, $b, 2), - _ => shuffle3!($a, $b, 3), - } - }; - } - macro_rules! shuffle1 { - ($a:expr) => { - match (imm8 >> 1) & 0x1 { - 0 => shuffle2!($a, 0), - _ => shuffle2!($a, 1), - } - }; - } - match imm8 & 0x1 { - 0 => shuffle1!(0), - _ => shuffle1!(1), - } +#[cfg_attr(test, assert_instr(vpermilpd, IMM4 = 0x1))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_permute_pd(a: __m256d) -> __m256d { + static_assert_imm4!(IMM4); + simd_shuffle4!( + a, + _mm256_undefined_pd(), + [ + ((IMM4 as u32 >> 0) & 1), + ((IMM4 as u32 >> 1) & 1), + ((IMM4 as u32 >> 2) & 1) + 2, + ((IMM4 as u32 >> 3) & 1) + 2, + ], + ) } /// Shuffles double-precision (64-bit) floating-point elements in `a` @@ -1363,28 +1125,16 @@ pub unsafe fn _mm256_permute_pd(a: __m256d, imm8: i32) -> __m256d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_permute_pd) #[inline] #[target_feature(enable = "avx,sse2")] -#[cfg_attr(test, assert_instr(vpermilpd, imm8 = 0x1))] -#[rustc_args_required_const(1)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_permute_pd(a: __m128d, imm8: i32) -> __m128d { - let imm8 = (imm8 & 0xFF) as u8; - macro_rules! shuffle2 { - ($a:expr, $b:expr) => { - simd_shuffle2(a, _mm_undefined_pd(), [$a, $b]); - }; - } - macro_rules! shuffle1 { - ($a:expr) => { - match (imm8 >> 1) & 0x1 { - 0 => shuffle2!($a, 0), - _ => shuffle2!($a, 1), - } - }; - } - match imm8 & 0x1 { - 0 => shuffle1!(0), - _ => shuffle1!(1), - } +#[cfg_attr(test, assert_instr(vpermilpd, IMM2 = 0x1))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_permute_pd(a: __m128d) -> __m128d { + static_assert_imm2!(IMM2); + simd_shuffle2!( + a, + _mm_undefined_pd(), + [(IMM2 as u32) & 1, (IMM2 as u32 >> 1) & 1], + ) } /// Shuffles 256 bits (composed of 8 packed single-precision (32-bit) @@ -1393,16 +1143,12 @@ pub unsafe fn _mm_permute_pd(a: __m128d, imm8: i32) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permute2f128_ps) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vperm2f128, imm8 = 0x5))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vperm2f128, IMM8 = 0x5))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_permute2f128_ps(a: __m256, b: __m256, imm8: i32) -> __m256 { - macro_rules! call { - ($imm8:expr) => { - vperm2f128ps256(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +pub unsafe fn _mm256_permute2f128_ps(a: __m256, b: __m256) -> __m256 { + static_assert_imm8!(IMM8); + vperm2f128ps256(a, b, IMM8 as i8) } /// Shuffles 256 bits (composed of 4 packed double-precision (64-bit) @@ -1411,37 +1157,26 @@ pub unsafe fn _mm256_permute2f128_ps(a: __m256, b: __m256, imm8: i32) -> __m256 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permute2f128_pd) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vperm2f128, imm8 = 0x31))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vperm2f128, IMM8 = 0x31))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_permute2f128_pd(a: __m256d, b: __m256d, imm8: i32) -> __m256d { - macro_rules! call { - ($imm8:expr) => { - vperm2f128pd256(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +pub unsafe fn _mm256_permute2f128_pd(a: __m256d, b: __m256d) -> __m256d { + static_assert_imm8!(IMM8); + vperm2f128pd256(a, b, IMM8 as i8) } -/// Shuffles 258-bits (composed of integer data) selected by `imm8` +/// Shuffles 128-bits (composed of integer data) selected by `imm8` /// from `a` and `b`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permute2f128_si256) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vperm2f128, imm8 = 0x31))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vperm2f128, IMM8 = 0x31))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_permute2f128_si256(a: __m256i, b: __m256i, imm8: i32) -> __m256i { - let a = a.as_i32x8(); - let b = b.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vperm2f128si256(a, b, $imm8) - }; - } - let r = constify_imm8!(imm8, call); - transmute(r) +pub unsafe fn _mm256_permute2f128_si256(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(vperm2f128si256(a.as_i32x8(), b.as_i32x8(), IMM8 as i8)) } /// Broadcasts a single-precision (32-bit) floating-point element from memory @@ -1516,16 +1251,17 @@ pub unsafe fn _mm256_broadcast_pd(a: &__m128d) -> __m256d { #[target_feature(enable = "avx")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vinsertf128, imm8 = 1) + assert_instr(vinsertf128, IMM1 = 1) )] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_insertf128_ps(a: __m256, b: __m128, imm8: i32) -> __m256 { - let b = _mm256_castps128_ps256(b); - match imm8 & 1 { - 0 => simd_shuffle8(a, b, [8, 9, 10, 11, 4, 5, 6, 7]), - _ => simd_shuffle8(a, b, [0, 1, 2, 3, 8, 9, 10, 11]), - } +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_insertf128_ps(a: __m256, b: __m128) -> __m256 { + static_assert_imm1!(IMM1); + simd_shuffle8!( + a, + _mm256_castps128_ps256(b), + [[8, 9, 10, 11, 4, 5, 6, 7], [0, 1, 2, 3, 8, 9, 10, 11]][IMM1 as usize], + ) } /// Copies `a` to result, then inserts 128 bits (composed of 2 packed @@ -1537,15 +1273,17 @@ pub unsafe fn _mm256_insertf128_ps(a: __m256, b: __m128, imm8: i32) -> __m256 { #[target_feature(enable = "avx")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vinsertf128, imm8 = 1) + assert_instr(vinsertf128, IMM1 = 1) )] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_insertf128_pd(a: __m256d, b: __m128d, imm8: i32) -> __m256d { - match imm8 & 1 { - 0 => simd_shuffle4(a, _mm256_castpd128_pd256(b), [4, 5, 2, 3]), - _ => simd_shuffle4(a, _mm256_castpd128_pd256(b), [0, 1, 4, 5]), - } +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_insertf128_pd(a: __m256d, b: __m128d) -> __m256d { + static_assert_imm1!(IMM1); + simd_shuffle4!( + a, + _mm256_castpd128_pd256(b), + [[4, 5, 2, 3], [0, 1, 4, 5]][IMM1 as usize], + ) } /// Copies `a` to result, then inserts 128 bits from `b` into result @@ -1556,16 +1294,17 @@ pub unsafe fn _mm256_insertf128_pd(a: __m256d, b: __m128d, imm8: i32) -> __m256d #[target_feature(enable = "avx")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vinsertf128, imm8 = 1) + assert_instr(vinsertf128, IMM1 = 1) )] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_insertf128_si256(a: __m256i, b: __m128i, imm8: i32) -> __m256i { - let b = _mm256_castsi128_si256(b).as_i64x4(); - let dst: i64x4 = match imm8 & 1 { - 0 => simd_shuffle4(a.as_i64x4(), b, [4, 5, 2, 3]), - _ => simd_shuffle4(a.as_i64x4(), b, [0, 1, 4, 5]), - }; +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_insertf128_si256(a: __m256i, b: __m128i) -> __m256i { + static_assert_imm1!(IMM1); + let dst: i64x4 = simd_shuffle4!( + a.as_i64x4(), + _mm256_castsi128_si256(b).as_i64x4(), + [[4, 5, 2, 3], [0, 1, 4, 5]][IMM1 as usize], + ); transmute(dst) } @@ -1576,10 +1315,11 @@ pub unsafe fn _mm256_insertf128_si256(a: __m256i, b: __m128i, imm8: i32) -> __m2 #[inline] #[target_feature(enable = "avx")] // This intrinsic has no corresponding instruction. -#[rustc_args_required_const(2)] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_insert_epi8(a: __m256i, i: i8, index: i32) -> __m256i { - transmute(simd_insert(a.as_i8x32(), (index as u32) & 31, i)) +pub unsafe fn _mm256_insert_epi8(a: __m256i, i: i8) -> __m256i { + static_assert_imm5!(INDEX); + transmute(simd_insert(a.as_i8x32(), INDEX as u32, i)) } /// Copies `a` to result, and inserts the 16-bit integer `i` into result @@ -1589,10 +1329,11 @@ pub unsafe fn _mm256_insert_epi8(a: __m256i, i: i8, index: i32) -> __m256i { #[inline] #[target_feature(enable = "avx")] // This intrinsic has no corresponding instruction. -#[rustc_args_required_const(2)] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_insert_epi16(a: __m256i, i: i16, index: i32) -> __m256i { - transmute(simd_insert(a.as_i16x16(), (index as u32) & 15, i)) +pub unsafe fn _mm256_insert_epi16(a: __m256i, i: i16) -> __m256i { + static_assert_imm4!(INDEX); + transmute(simd_insert(a.as_i16x16(), INDEX as u32, i)) } /// Copies `a` to result, and inserts the 32-bit integer `i` into result @@ -1602,10 +1343,11 @@ pub unsafe fn _mm256_insert_epi16(a: __m256i, i: i16, index: i32) -> __m256i { #[inline] #[target_feature(enable = "avx")] // This intrinsic has no corresponding instruction. -#[rustc_args_required_const(2)] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_insert_epi32(a: __m256i, i: i32, index: i32) -> __m256i { - transmute(simd_insert(a.as_i32x8(), (index as u32) & 7, i)) +pub unsafe fn _mm256_insert_epi32(a: __m256i, i: i32) -> __m256i { + static_assert_imm3!(INDEX); + transmute(simd_insert(a.as_i32x8(), INDEX as u32, i)) } /// Loads 256-bits (composed of 4 packed double-precision (64-bit) @@ -1897,7 +1639,7 @@ pub unsafe fn _mm_maskstore_ps(mem_addr: *mut f32, mask: __m128i, a: __m128) { #[cfg_attr(test, assert_instr(vmovshdup))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_movehdup_ps(a: __m256) -> __m256 { - simd_shuffle8(a, a, [1, 1, 3, 3, 5, 5, 7, 7]) + simd_shuffle8!(a, a, [1, 1, 3, 3, 5, 5, 7, 7]) } /// Duplicate even-indexed single-precision (32-bit) floating-point elements @@ -1909,7 +1651,7 @@ pub unsafe fn _mm256_movehdup_ps(a: __m256) -> __m256 { #[cfg_attr(test, assert_instr(vmovsldup))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_moveldup_ps(a: __m256) -> __m256 { - simd_shuffle8(a, a, [0, 0, 2, 2, 4, 4, 6, 6]) + simd_shuffle8!(a, a, [0, 0, 2, 2, 4, 4, 6, 6]) } /// Duplicate even-indexed double-precision (64-bit) floating-point elements @@ -1921,7 +1663,7 @@ pub unsafe fn _mm256_moveldup_ps(a: __m256) -> __m256 { #[cfg_attr(test, assert_instr(vmovddup))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_movedup_pd(a: __m256d) -> __m256d { - simd_shuffle4(a, a, [0, 0, 2, 2]) + simd_shuffle4!(a, a, [0, 0, 2, 2]) } /// Loads 256-bits of integer data from unaligned memory into result. @@ -2014,7 +1756,7 @@ pub unsafe fn _mm256_rsqrt_ps(a: __m256) -> __m256 { #[cfg_attr(test, assert_instr(vunpckhpd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpackhi_pd(a: __m256d, b: __m256d) -> __m256d { - simd_shuffle4(a, b, [1, 5, 3, 7]) + simd_shuffle4!(a, b, [1, 5, 3, 7]) } /// Unpacks and interleave single-precision (32-bit) floating-point elements @@ -2026,7 +1768,7 @@ pub unsafe fn _mm256_unpackhi_pd(a: __m256d, b: __m256d) -> __m256d { #[cfg_attr(test, assert_instr(vunpckhps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpackhi_ps(a: __m256, b: __m256) -> __m256 { - simd_shuffle8(a, b, [2, 10, 3, 11, 6, 14, 7, 15]) + simd_shuffle8!(a, b, [2, 10, 3, 11, 6, 14, 7, 15]) } /// Unpacks and interleave double-precision (64-bit) floating-point elements @@ -2038,7 +1780,7 @@ pub unsafe fn _mm256_unpackhi_ps(a: __m256, b: __m256) -> __m256 { #[cfg_attr(test, assert_instr(vunpcklpd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpacklo_pd(a: __m256d, b: __m256d) -> __m256d { - simd_shuffle4(a, b, [0, 4, 2, 6]) + simd_shuffle4!(a, b, [0, 4, 2, 6]) } /// Unpacks and interleave single-precision (32-bit) floating-point elements @@ -2050,7 +1792,7 @@ pub unsafe fn _mm256_unpacklo_pd(a: __m256d, b: __m256d) -> __m256d { #[cfg_attr(test, assert_instr(vunpcklps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpacklo_ps(a: __m256, b: __m256) -> __m256 { - simd_shuffle8(a, b, [0, 8, 1, 9, 4, 12, 5, 13]) + simd_shuffle8!(a, b, [0, 8, 1, 9, 4, 12, 5, 13]) } /// Computes the bitwise AND of 256 bits (representing integer data) in `a` and @@ -2397,8 +2139,7 @@ pub unsafe fn _mm256_set_ps( _mm256_setr_ps(h, g, f, e, d, c, b, a) } -/// Sets packed 8-bit integers in returned vector with the supplied values in -/// reverse order. +/// Sets packed 8-bit integers in returned vector with the supplied values. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_set_epi8) #[inline] @@ -2830,7 +2571,7 @@ pub unsafe fn _mm256_castsi256_pd(a: __m256i) -> __m256d { // instructions, thus it has zero latency. #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_castps256_ps128(a: __m256) -> __m128 { - simd_shuffle4(a, a, [0, 1, 2, 3]) + simd_shuffle4!(a, a, [0, 1, 2, 3]) } /// Casts vector of type __m256d to type __m128d. @@ -2842,7 +2583,7 @@ pub unsafe fn _mm256_castps256_ps128(a: __m256) -> __m128 { // instructions, thus it has zero latency. #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_castpd256_pd128(a: __m256d) -> __m128d { - simd_shuffle2(a, a, [0, 1]) + simd_shuffle2!(a, a, [0, 1]) } /// Casts vector of type __m256i to type __m128i. @@ -2855,7 +2596,7 @@ pub unsafe fn _mm256_castpd256_pd128(a: __m256d) -> __m128d { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_castsi256_si128(a: __m256i) -> __m128i { let a = a.as_i64x4(); - let dst: i64x2 = simd_shuffle2(a, a, [0, 1]); + let dst: i64x2 = simd_shuffle2!(a, a, [0, 1]); transmute(dst) } @@ -2869,8 +2610,8 @@ pub unsafe fn _mm256_castsi256_si128(a: __m256i) -> __m128i { // instructions, thus it has zero latency. #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_castps128_ps256(a: __m128) -> __m256 { - // FIXME simd_shuffle8(a, a, [0, 1, 2, 3, -1, -1, -1, -1]) - simd_shuffle8(a, a, [0, 1, 2, 3, 0, 0, 0, 0]) + // FIXME simd_shuffle8!(a, a, [0, 1, 2, 3, -1, -1, -1, -1]) + simd_shuffle8!(a, a, [0, 1, 2, 3, 0, 0, 0, 0]) } /// Casts vector of type __m128d to type __m256d; @@ -2883,8 +2624,8 @@ pub unsafe fn _mm256_castps128_ps256(a: __m128) -> __m256 { // instructions, thus it has zero latency. #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_castpd128_pd256(a: __m128d) -> __m256d { - // FIXME simd_shuffle4(a, a, [0, 1, -1, -1]) - simd_shuffle4(a, a, [0, 1, 0, 0]) + // FIXME simd_shuffle4!(a, a, [0, 1, -1, -1]) + simd_shuffle4!(a, a, [0, 1, 0, 0]) } /// Casts vector of type __m128i to type __m256i; @@ -2898,8 +2639,8 @@ pub unsafe fn _mm256_castpd128_pd256(a: __m128d) -> __m256d { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_castsi128_si256(a: __m128i) -> __m256i { let a = a.as_i64x2(); - // FIXME simd_shuffle4(a, a, [0, 1, -1, -1]) - let dst: i64x4 = simd_shuffle4(a, a, [0, 1, 0, 0]); + // FIXME simd_shuffle4!(a, a, [0, 1, -1, -1]) + let dst: i64x4 = simd_shuffle4!(a, a, [0, 1, 0, 0]); transmute(dst) } @@ -2914,7 +2655,7 @@ pub unsafe fn _mm256_castsi128_si256(a: __m128i) -> __m256i { // instructions, thus it has zero latency. #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_zextps128_ps256(a: __m128) -> __m256 { - simd_shuffle8(a, _mm_setzero_ps(), [0, 1, 2, 3, 4, 5, 6, 7]) + simd_shuffle8!(a, _mm_setzero_ps(), [0, 1, 2, 3, 4, 5, 6, 7]) } /// Constructs a 256-bit integer vector from a 128-bit integer vector. @@ -2929,7 +2670,7 @@ pub unsafe fn _mm256_zextps128_ps256(a: __m128) -> __m256 { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_zextsi128_si256(a: __m128i) -> __m256i { let b = _mm_setzero_si128().as_i64x2(); - let dst: i64x4 = simd_shuffle4(a.as_i64x2(), b, [0, 1, 2, 3]); + let dst: i64x4 = simd_shuffle4!(a.as_i64x2(), b, [0, 1, 2, 3]); transmute(dst) } @@ -2945,7 +2686,7 @@ pub unsafe fn _mm256_zextsi128_si256(a: __m128i) -> __m256i { // instructions, thus it has zero latency. #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_zextpd128_pd256(a: __m128d) -> __m256d { - simd_shuffle4(a, _mm_setzero_pd(), [0, 1, 2, 3]) + simd_shuffle4!(a, _mm_setzero_pd(), [0, 1, 2, 3]) } /// Returns vector of type `__m256` with undefined elements. @@ -2956,8 +2697,7 @@ pub unsafe fn _mm256_zextpd128_pd256(a: __m128d) -> __m256d { // This intrinsic has no corresponding instruction. #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_undefined_ps() -> __m256 { - // FIXME: this function should return MaybeUninit<__m256> - mem::MaybeUninit::<__m256>::uninit().assume_init() + _mm256_set1_ps(0.0) } /// Returns vector of type `__m256d` with undefined elements. @@ -2968,8 +2708,7 @@ pub unsafe fn _mm256_undefined_ps() -> __m256 { // This intrinsic has no corresponding instruction. #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_undefined_pd() -> __m256d { - // FIXME: this function should return MaybeUninit<__m256d> - mem::MaybeUninit::<__m256d>::uninit().assume_init() + _mm256_set1_pd(0.0) } /// Returns vector of type __m256i with undefined elements. @@ -2980,8 +2719,7 @@ pub unsafe fn _mm256_undefined_pd() -> __m256d { // This intrinsic has no corresponding instruction. #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_undefined_si256() -> __m256i { - // FIXME: this function should return MaybeUninit<__m256i> - mem::MaybeUninit::<__m256i>::uninit().assume_init() + __m256i(0, 0, 0, 0) } /// Sets packed __m256 returned vector with the supplied values. @@ -2992,7 +2730,7 @@ pub unsafe fn _mm256_undefined_si256() -> __m256i { #[cfg_attr(test, assert_instr(vinsertf128))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_set_m128(hi: __m128, lo: __m128) -> __m256 { - simd_shuffle8(lo, hi, [0, 1, 2, 3, 4, 5, 6, 7]) + simd_shuffle8!(lo, hi, [0, 1, 2, 3, 4, 5, 6, 7]) } /// Sets packed __m256d returned vector with the supplied values. @@ -3066,7 +2804,7 @@ pub unsafe fn _mm256_setr_m128i(lo: __m128i, hi: __m128i) -> __m256i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_loadu2_m128(hiaddr: *const f32, loaddr: *const f32) -> __m256 { let a = _mm256_castps128_ps256(_mm_loadu_ps(loaddr)); - _mm256_insertf128_ps(a, _mm_loadu_ps(hiaddr), 1) + _mm256_insertf128_ps::<1>(a, _mm_loadu_ps(hiaddr)) } /// Loads two 128-bit values (composed of 2 packed double-precision (64-bit) @@ -3081,7 +2819,7 @@ pub unsafe fn _mm256_loadu2_m128(hiaddr: *const f32, loaddr: *const f32) -> __m2 #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_loadu2_m128d(hiaddr: *const f64, loaddr: *const f64) -> __m256d { let a = _mm256_castpd128_pd256(_mm_loadu_pd(loaddr)); - _mm256_insertf128_pd(a, _mm_loadu_pd(hiaddr), 1) + _mm256_insertf128_pd::<1>(a, _mm_loadu_pd(hiaddr)) } /// Loads two 128-bit values (composed of integer data) from memory, and combine @@ -3095,7 +2833,7 @@ pub unsafe fn _mm256_loadu2_m128d(hiaddr: *const f64, loaddr: *const f64) -> __m #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_loadu2_m128i(hiaddr: *const __m128i, loaddr: *const __m128i) -> __m256i { let a = _mm256_castsi128_si256(_mm_loadu_si128(loaddr)); - _mm256_insertf128_si256(a, _mm_loadu_si128(hiaddr), 1) + _mm256_insertf128_si256::<1>(a, _mm_loadu_si128(hiaddr)) } /// Stores the high and low 128-bit halves (each composed of 4 packed @@ -3111,7 +2849,7 @@ pub unsafe fn _mm256_loadu2_m128i(hiaddr: *const __m128i, loaddr: *const __m128i pub unsafe fn _mm256_storeu2_m128(hiaddr: *mut f32, loaddr: *mut f32, a: __m256) { let lo = _mm256_castps256_ps128(a); _mm_storeu_ps(loaddr, lo); - let hi = _mm256_extractf128_ps(a, 1); + let hi = _mm256_extractf128_ps::<1>(a); _mm_storeu_ps(hiaddr, hi); } @@ -3128,7 +2866,7 @@ pub unsafe fn _mm256_storeu2_m128(hiaddr: *mut f32, loaddr: *mut f32, a: __m256) pub unsafe fn _mm256_storeu2_m128d(hiaddr: *mut f64, loaddr: *mut f64, a: __m256d) { let lo = _mm256_castpd256_pd128(a); _mm_storeu_pd(loaddr, lo); - let hi = _mm256_extractf128_pd(a, 1); + let hi = _mm256_extractf128_pd::<1>(a); _mm_storeu_pd(hiaddr, hi); } @@ -3144,7 +2882,7 @@ pub unsafe fn _mm256_storeu2_m128d(hiaddr: *mut f64, loaddr: *mut f64, a: __m256 pub unsafe fn _mm256_storeu2_m128i(hiaddr: *mut __m128i, loaddr: *mut __m128i, a: __m256i) { let lo = _mm256_castsi256_si128(a); _mm_storeu_si128(loaddr, lo); - let hi = _mm256_extractf128_si256(a, 1); + let hi = _mm256_extractf128_si256::<1>(a); _mm_storeu_si128(hiaddr, hi); } @@ -3159,7 +2897,7 @@ pub unsafe fn _mm256_cvtss_f32(a: __m256) -> f32 { simd_extract(a, 0) } -/// LLVM intrinsics used in the above functions +// LLVM intrinsics used in the above functions #[allow(improper_ctypes)] extern "C" { #[link_name = "llvm.x86.avx.addsub.pd.256"] @@ -3187,17 +2925,17 @@ extern "C" { #[link_name = "llvm.x86.avx.hsub.ps.256"] fn vhsubps(a: __m256, b: __m256) -> __m256; #[link_name = "llvm.x86.sse2.cmp.pd"] - fn vcmppd(a: __m128d, b: __m128d, imm8: u8) -> __m128d; + fn vcmppd(a: __m128d, b: __m128d, imm8: i8) -> __m128d; #[link_name = "llvm.x86.avx.cmp.pd.256"] fn vcmppd256(a: __m256d, b: __m256d, imm8: u8) -> __m256d; #[link_name = "llvm.x86.sse.cmp.ps"] - fn vcmpps(a: __m128, b: __m128, imm8: u8) -> __m128; + fn vcmpps(a: __m128, b: __m128, imm8: i8) -> __m128; #[link_name = "llvm.x86.avx.cmp.ps.256"] fn vcmpps256(a: __m256, b: __m256, imm8: u8) -> __m256; #[link_name = "llvm.x86.sse2.cmp.sd"] - fn vcmpsd(a: __m128d, b: __m128d, imm8: u8) -> __m128d; + fn vcmpsd(a: __m128d, b: __m128d, imm8: i8) -> __m128d; #[link_name = "llvm.x86.sse.cmp.ss"] - fn vcmpss(a: __m128, b: __m128, imm8: u8) -> __m128; + fn vcmpss(a: __m128, b: __m128, imm8: i8) -> __m128; #[link_name = "llvm.x86.avx.cvtdq2.ps.256"] fn vcvtdq2ps(a: i32x8) -> __m256; #[link_name = "llvm.x86.avx.cvt.pd2.ps.256"] @@ -3294,6 +3032,14 @@ extern "C" { fn movmskpd256(a: __m256d) -> i32; #[link_name = "llvm.x86.avx.movmsk.ps.256"] fn movmskps256(a: __m256) -> i32; + #[link_name = "llvm.x86.avx.min.ps.256"] + fn vminps(a: __m256, b: __m256) -> __m256; + #[link_name = "llvm.x86.avx.max.ps.256"] + fn vmaxps(a: __m256, b: __m256) -> __m256; + #[link_name = "llvm.x86.avx.min.pd.256"] + fn vminpd(a: __m256d, b: __m256d) -> __m256d; + #[link_name = "llvm.x86.avx.max.pd.256"] + fn vmaxpd(a: __m256d, b: __m256d) -> __m256d; } #[cfg(test)] @@ -3361,7 +3107,7 @@ mod tests { unsafe fn test_mm256_shuffle_pd() { let a = _mm256_setr_pd(1., 4., 5., 8.); let b = _mm256_setr_pd(2., 3., 6., 7.); - let r = _mm256_shuffle_pd(a, b, 0xF); + let r = _mm256_shuffle_pd::<0b11_11_11_11>(a, b); let e = _mm256_setr_pd(4., 3., 8., 7.); assert_eq_m256d(r, e); } @@ -3370,7 +3116,7 @@ mod tests { unsafe fn test_mm256_shuffle_ps() { let a = _mm256_setr_ps(1., 4., 5., 8., 9., 12., 13., 16.); let b = _mm256_setr_ps(2., 3., 6., 7., 10., 11., 14., 15.); - let r = _mm256_shuffle_ps(a, b, 0x0F); + let r = _mm256_shuffle_ps::<0b00_00_11_11>(a, b); let e = _mm256_setr_ps(8., 8., 2., 2., 16., 16., 10., 10.); assert_eq_m256(r, e); } @@ -3398,6 +3144,23 @@ mod tests { let r = _mm256_max_pd(a, b); let e = _mm256_setr_pd(2., 4., 6., 8.); assert_eq_m256d(r, e); + // > If the values being compared are both 0.0s (of either sign), the + // > value in the second operand (source operand) is returned. + let w = _mm256_max_pd(_mm256_set1_pd(0.0), _mm256_set1_pd(-0.0)); + let x = _mm256_max_pd(_mm256_set1_pd(-0.0), _mm256_set1_pd(0.0)); + let wu: [u64; 4] = transmute(w); + let xu: [u64; 4] = transmute(x); + assert_eq!(wu, [0x8000_0000_0000_0000u64; 4]); + assert_eq!(xu, [0u64; 4]); + // > If only one value is a NaN (SNaN or QNaN) for this instruction, the + // > second operand (source operand), either a NaN or a valid + // > floating-point value, is written to the result. + let y = _mm256_max_pd(_mm256_set1_pd(f64::NAN), _mm256_set1_pd(0.0)); + let z = _mm256_max_pd(_mm256_set1_pd(0.0), _mm256_set1_pd(f64::NAN)); + let yf: [f64; 4] = transmute(y); + let zf: [f64; 4] = transmute(z); + assert_eq!(yf, [0.0; 4]); + assert!(zf.iter().all(|f| f.is_nan()), "{:?}", zf); } #[simd_test(enable = "avx")] @@ -3407,6 +3170,23 @@ mod tests { let r = _mm256_max_ps(a, b); let e = _mm256_setr_ps(2., 4., 6., 8., 10., 12., 14., 16.); assert_eq_m256(r, e); + // > If the values being compared are both 0.0s (of either sign), the + // > value in the second operand (source operand) is returned. + let w = _mm256_max_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(-0.0)); + let x = _mm256_max_ps(_mm256_set1_ps(-0.0), _mm256_set1_ps(0.0)); + let wu: [u32; 8] = transmute(w); + let xu: [u32; 8] = transmute(x); + assert_eq!(wu, [0x8000_0000u32; 8]); + assert_eq!(xu, [0u32; 8]); + // > If only one value is a NaN (SNaN or QNaN) for this instruction, the + // > second operand (source operand), either a NaN or a valid + // > floating-point value, is written to the result. + let y = _mm256_max_ps(_mm256_set1_ps(f32::NAN), _mm256_set1_ps(0.0)); + let z = _mm256_max_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(f32::NAN)); + let yf: [f32; 8] = transmute(y); + let zf: [f32; 8] = transmute(z); + assert_eq!(yf, [0.0; 8]); + assert!(zf.iter().all(|f| f.is_nan()), "{:?}", zf); } #[simd_test(enable = "avx")] @@ -3416,6 +3196,23 @@ mod tests { let r = _mm256_min_pd(a, b); let e = _mm256_setr_pd(1., 3., 5., 7.); assert_eq_m256d(r, e); + // > If the values being compared are both 0.0s (of either sign), the + // > value in the second operand (source operand) is returned. + let w = _mm256_min_pd(_mm256_set1_pd(0.0), _mm256_set1_pd(-0.0)); + let x = _mm256_min_pd(_mm256_set1_pd(-0.0), _mm256_set1_pd(0.0)); + let wu: [u64; 4] = transmute(w); + let xu: [u64; 4] = transmute(x); + assert_eq!(wu, [0x8000_0000_0000_0000u64; 4]); + assert_eq!(xu, [0u64; 4]); + // > If only one value is a NaN (SNaN or QNaN) for this instruction, the + // > second operand (source operand), either a NaN or a valid + // > floating-point value, is written to the result. + let y = _mm256_min_pd(_mm256_set1_pd(f64::NAN), _mm256_set1_pd(0.0)); + let z = _mm256_min_pd(_mm256_set1_pd(0.0), _mm256_set1_pd(f64::NAN)); + let yf: [f64; 4] = transmute(y); + let zf: [f64; 4] = transmute(z); + assert_eq!(yf, [0.0; 4]); + assert!(zf.iter().all(|f| f.is_nan()), "{:?}", zf); } #[simd_test(enable = "avx")] @@ -3425,6 +3222,23 @@ mod tests { let r = _mm256_min_ps(a, b); let e = _mm256_setr_ps(1., 3., 5., 7., 9., 11., 13., 15.); assert_eq_m256(r, e); + // > If the values being compared are both 0.0s (of either sign), the + // > value in the second operand (source operand) is returned. + let w = _mm256_min_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(-0.0)); + let x = _mm256_min_ps(_mm256_set1_ps(-0.0), _mm256_set1_ps(0.0)); + let wu: [u32; 8] = transmute(w); + let xu: [u32; 8] = transmute(x); + assert_eq!(wu, [0x8000_0000u32; 8]); + assert_eq!(xu, [0u32; 8]); + // > If only one value is a NaN (SNaN or QNaN) for this instruction, the + // > second operand (source operand), either a NaN or a valid + // > floating-point value, is written to the result. + let y = _mm256_min_ps(_mm256_set1_ps(f32::NAN), _mm256_set1_ps(0.0)); + let z = _mm256_min_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(f32::NAN)); + let yf: [f32; 8] = transmute(y); + let zf: [f32; 8] = transmute(z); + assert_eq!(yf, [0.0; 8]); + assert!(zf.iter().all(|f| f.is_nan()), "{:?}", zf); } #[simd_test(enable = "avx")] @@ -3484,9 +3298,9 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm256_round_pd() { let a = _mm256_setr_pd(1.55, 2.2, 3.99, -1.2); - let result_closest = _mm256_round_pd(a, 0b00000000); - let result_down = _mm256_round_pd(a, 0b00000001); - let result_up = _mm256_round_pd(a, 0b00000010); + let result_closest = _mm256_round_pd::<0b0000>(a); + let result_down = _mm256_round_pd::<0b0001>(a); + let result_up = _mm256_round_pd::<0b0010>(a); let expected_closest = _mm256_setr_pd(2., 2., 4., -1.); let expected_down = _mm256_setr_pd(1., 2., 3., -2.); let expected_up = _mm256_setr_pd(2., 3., 4., -1.); @@ -3514,9 +3328,9 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm256_round_ps() { let a = _mm256_setr_ps(1.55, 2.2, 3.99, -1.2, 1.55, 2.2, 3.99, -1.2); - let result_closest = _mm256_round_ps(a, 0b00000000); - let result_down = _mm256_round_ps(a, 0b00000001); - let result_up = _mm256_round_ps(a, 0b00000010); + let result_closest = _mm256_round_ps::<0b0000>(a); + let result_down = _mm256_round_ps::<0b0001>(a); + let result_up = _mm256_round_ps::<0b0010>(a); let expected_closest = _mm256_setr_ps(2., 2., 4., -1., 2., 2., 4., -1.); let expected_down = _mm256_setr_ps(1., 2., 3., -2., 1., 2., 3., -2.); let expected_up = _mm256_setr_ps(2., 3., 4., -1., 2., 3., 4., -1.); @@ -3579,11 +3393,11 @@ mod tests { unsafe fn test_mm256_blend_pd() { let a = _mm256_setr_pd(4., 9., 16., 25.); let b = _mm256_setr_pd(4., 3., 2., 5.); - let r = _mm256_blend_pd(a, b, 0x0); + let r = _mm256_blend_pd::<0x0>(a, b); assert_eq_m256d(r, _mm256_setr_pd(4., 9., 16., 25.)); - let r = _mm256_blend_pd(a, b, 0x3); + let r = _mm256_blend_pd::<0x3>(a, b); assert_eq_m256d(r, _mm256_setr_pd(4., 3., 16., 25.)); - let r = _mm256_blend_pd(a, b, 0xF); + let r = _mm256_blend_pd::<0xF>(a, b); assert_eq_m256d(r, _mm256_setr_pd(4., 3., 2., 5.)); } @@ -3591,11 +3405,11 @@ mod tests { unsafe fn test_mm256_blend_ps() { let a = _mm256_setr_ps(1., 4., 5., 8., 9., 12., 13., 16.); let b = _mm256_setr_ps(2., 3., 6., 7., 10., 11., 14., 15.); - let r = _mm256_blend_ps(a, b, 0x0); + let r = _mm256_blend_ps::<0x0>(a, b); assert_eq_m256(r, _mm256_setr_ps(1., 4., 5., 8., 9., 12., 13., 16.)); - let r = _mm256_blend_ps(a, b, 0x3); + let r = _mm256_blend_ps::<0x3>(a, b); assert_eq_m256(r, _mm256_setr_ps(2., 3., 5., 8., 9., 12., 13., 16.)); - let r = _mm256_blend_ps(a, b, 0xF); + let r = _mm256_blend_ps::<0xF>(a, b); assert_eq_m256(r, _mm256_setr_ps(2., 3., 6., 7., 9., 12., 13., 16.)); } @@ -3626,7 +3440,7 @@ mod tests { unsafe fn test_mm256_dp_ps() { let a = _mm256_setr_ps(4., 9., 16., 25., 4., 9., 16., 25.); let b = _mm256_setr_ps(4., 3., 2., 5., 8., 9., 64., 50.); - let r = _mm256_dp_ps(a, b, 0xFF); + let r = _mm256_dp_ps::<0xFF>(a, b); let e = _mm256_setr_ps(200., 200., 200., 200., 2387., 2387., 2387., 2387.); assert_eq_m256(r, e); } @@ -3711,7 +3525,7 @@ mod tests { unsafe fn test_mm_cmp_pd() { let a = _mm_setr_pd(4., 9.); let b = _mm_setr_pd(4., 3.); - let r = _mm_cmp_pd(a, b, _CMP_GE_OS); + let r = _mm_cmp_pd::<_CMP_GE_OS>(a, b); assert!(get_m128d(r, 0).is_nan()); assert!(get_m128d(r, 1).is_nan()); } @@ -3720,7 +3534,7 @@ mod tests { unsafe fn test_mm256_cmp_pd() { let a = _mm256_setr_pd(1., 2., 3., 4.); let b = _mm256_setr_pd(5., 6., 7., 8.); - let r = _mm256_cmp_pd(a, b, _CMP_GE_OS); + let r = _mm256_cmp_pd::<_CMP_GE_OS>(a, b); let e = _mm256_set1_pd(0.); assert_eq_m256d(r, e); } @@ -3729,7 +3543,7 @@ mod tests { unsafe fn test_mm_cmp_ps() { let a = _mm_setr_ps(4., 3., 2., 5.); let b = _mm_setr_ps(4., 9., 16., 25.); - let r = _mm_cmp_ps(a, b, _CMP_GE_OS); + let r = _mm_cmp_ps::<_CMP_GE_OS>(a, b); assert!(get_m128(r, 0).is_nan()); assert_eq!(get_m128(r, 1), 0.); assert_eq!(get_m128(r, 2), 0.); @@ -3740,7 +3554,7 @@ mod tests { unsafe fn test_mm256_cmp_ps() { let a = _mm256_setr_ps(1., 2., 3., 4., 1., 2., 3., 4.); let b = _mm256_setr_ps(5., 6., 7., 8., 5., 6., 7., 8.); - let r = _mm256_cmp_ps(a, b, _CMP_GE_OS); + let r = _mm256_cmp_ps::<_CMP_GE_OS>(a, b); let e = _mm256_set1_ps(0.); assert_eq_m256(r, e); } @@ -3749,7 +3563,7 @@ mod tests { unsafe fn test_mm_cmp_sd() { let a = _mm_setr_pd(4., 9.); let b = _mm_setr_pd(4., 3.); - let r = _mm_cmp_sd(a, b, _CMP_GE_OS); + let r = _mm_cmp_sd::<_CMP_GE_OS>(a, b); assert!(get_m128d(r, 0).is_nan()); assert_eq!(get_m128d(r, 1), 9.); } @@ -3758,7 +3572,7 @@ mod tests { unsafe fn test_mm_cmp_ss() { let a = _mm_setr_ps(4., 3., 2., 5.); let b = _mm_setr_ps(4., 9., 16., 25.); - let r = _mm_cmp_ss(a, b, _CMP_GE_OS); + let r = _mm_cmp_ss::<_CMP_GE_OS>(a, b); assert!(get_m128(r, 0).is_nan()); assert_eq!(get_m128(r, 1), 3.); assert_eq!(get_m128(r, 2), 2.); @@ -3832,7 +3646,7 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm256_extractf128_ps() { let a = _mm256_setr_ps(4., 3., 2., 5., 8., 9., 64., 50.); - let r = _mm256_extractf128_ps(a, 0); + let r = _mm256_extractf128_ps::<0>(a); let e = _mm_setr_ps(4., 3., 2., 5.); assert_eq_m128(r, e); } @@ -3840,7 +3654,7 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm256_extractf128_pd() { let a = _mm256_setr_pd(4., 3., 2., 5.); - let r = _mm256_extractf128_pd(a, 0); + let r = _mm256_extractf128_pd::<0>(a); let e = _mm_setr_pd(4., 3.); assert_eq_m128d(r, e); } @@ -3848,7 +3662,7 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm256_extractf128_si256() { let a = _mm256_setr_epi64x(4, 3, 2, 5); - let r = _mm256_extractf128_si256(a, 0); + let r = _mm256_extractf128_si256::<0>(a); let e = _mm_setr_epi64x(4, 3); assert_eq_m128i(r, e); } @@ -3884,7 +3698,7 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm256_permute_ps() { let a = _mm256_setr_ps(4., 3., 2., 5., 8., 9., 64., 50.); - let r = _mm256_permute_ps(a, 0x1b); + let r = _mm256_permute_ps::<0x1b>(a); let e = _mm256_setr_ps(5., 2., 3., 4., 50., 64., 9., 8.); assert_eq_m256(r, e); } @@ -3892,7 +3706,7 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm_permute_ps() { let a = _mm_setr_ps(4., 3., 2., 5.); - let r = _mm_permute_ps(a, 0x1b); + let r = _mm_permute_ps::<0x1b>(a); let e = _mm_setr_ps(5., 2., 3., 4.); assert_eq_m128(r, e); } @@ -3918,7 +3732,7 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm256_permute_pd() { let a = _mm256_setr_pd(4., 3., 2., 5.); - let r = _mm256_permute_pd(a, 5); + let r = _mm256_permute_pd::<5>(a); let e = _mm256_setr_pd(3., 4., 5., 2.); assert_eq_m256d(r, e); } @@ -3926,7 +3740,7 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm_permute_pd() { let a = _mm_setr_pd(4., 3.); - let r = _mm_permute_pd(a, 1); + let r = _mm_permute_pd::<1>(a); let e = _mm_setr_pd(3., 4.); assert_eq_m128d(r, e); } @@ -3935,7 +3749,7 @@ mod tests { unsafe fn test_mm256_permute2f128_ps() { let a = _mm256_setr_ps(1., 2., 3., 4., 1., 2., 3., 4.); let b = _mm256_setr_ps(5., 6., 7., 8., 5., 6., 7., 8.); - let r = _mm256_permute2f128_ps(a, b, 0x13); + let r = _mm256_permute2f128_ps::<0x13>(a, b); let e = _mm256_setr_ps(5., 6., 7., 8., 1., 2., 3., 4.); assert_eq_m256(r, e); } @@ -3944,7 +3758,7 @@ mod tests { unsafe fn test_mm256_permute2f128_pd() { let a = _mm256_setr_pd(1., 2., 3., 4.); let b = _mm256_setr_pd(5., 6., 7., 8.); - let r = _mm256_permute2f128_pd(a, b, 0x31); + let r = _mm256_permute2f128_pd::<0x31>(a, b); let e = _mm256_setr_pd(3., 4., 7., 8.); assert_eq_m256d(r, e); } @@ -3953,7 +3767,7 @@ mod tests { unsafe fn test_mm256_permute2f128_si256() { let a = _mm256_setr_epi32(1, 2, 3, 4, 1, 2, 3, 4); let b = _mm256_setr_epi32(5, 6, 7, 8, 5, 6, 7, 8); - let r = _mm256_permute2f128_si256(a, b, 0x20); + let r = _mm256_permute2f128_si256::<0x20>(a, b); let e = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); assert_eq_m256i(r, e); } @@ -3999,7 +3813,7 @@ mod tests { unsafe fn test_mm256_insertf128_ps() { let a = _mm256_setr_ps(4., 3., 2., 5., 8., 9., 64., 50.); let b = _mm_setr_ps(4., 9., 16., 25.); - let r = _mm256_insertf128_ps(a, b, 0); + let r = _mm256_insertf128_ps::<0>(a, b); let e = _mm256_setr_ps(4., 9., 16., 25., 8., 9., 64., 50.); assert_eq_m256(r, e); } @@ -4008,7 +3822,7 @@ mod tests { unsafe fn test_mm256_insertf128_pd() { let a = _mm256_setr_pd(1., 2., 3., 4.); let b = _mm_setr_pd(5., 6.); - let r = _mm256_insertf128_pd(a, b, 0); + let r = _mm256_insertf128_pd::<0>(a, b); let e = _mm256_setr_pd(5., 6., 3., 4.); assert_eq_m256d(r, e); } @@ -4017,7 +3831,7 @@ mod tests { unsafe fn test_mm256_insertf128_si256() { let a = _mm256_setr_epi64x(1, 2, 3, 4); let b = _mm_setr_epi64x(5, 6); - let r = _mm256_insertf128_si256(a, b, 0); + let r = _mm256_insertf128_si256::<0>(a, b); let e = _mm256_setr_epi64x(5, 6, 3, 4); assert_eq_m256i(r, e); } @@ -4031,7 +3845,7 @@ mod tests { 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, ); - let r = _mm256_insert_epi8(a, 0, 31); + let r = _mm256_insert_epi8::<31>(a, 0); #[rustfmt::skip] let e = _mm256_setr_epi8( 1, 2, 3, 4, 5, 6, 7, 8, @@ -4049,7 +3863,7 @@ mod tests { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, ); - let r = _mm256_insert_epi16(a, 0, 15); + let r = _mm256_insert_epi16::<15>(a, 0); #[rustfmt::skip] let e = _mm256_setr_epi16( 0, 1, 2, 3, 4, 5, 6, 7, @@ -4061,7 +3875,7 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm256_insert_epi32() { let a = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); - let r = _mm256_insert_epi32(a, 0, 7); + let r = _mm256_insert_epi32::<7>(a, 0); let e = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 0); assert_eq_m256i(r, e); } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx2.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx2.rs index 193e1ca3a..16add3dbb 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx2.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx2.rs @@ -153,29 +153,29 @@ pub unsafe fn _mm256_adds_epu16(a: __m256i, b: __m256i) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_alignr_epi8) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpalignr, n = 7))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vpalignr, IMM8 = 7))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { - let n = n as u32; - // If `palignr` is shifting the pair of vectors more than the size of two +pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + // If palignr is shifting the pair of vectors more than the size of two // lanes, emit zero. - if n > 32 { + if IMM8 > 32 { return _mm256_set1_epi8(0); } - // If `palignr` is shifting the pair of input vectors more than one lane, + // If palignr is shifting the pair of input vectors more than one lane, // but less than two lanes, convert to shifting in zeroes. - let (a, b, n) = if n > 16 { - (_mm256_set1_epi8(0), a, n - 16) + let (a, b) = if IMM8 > 16 { + (_mm256_set1_epi8(0), a) } else { - (a, b, n) + (a, b) }; let a = a.as_i8x32(); let b = b.as_i8x32(); - let r: i8x32 = match n { - 0 => simd_shuffle32( + let r: i8x32 = match IMM8 % 16 { + 0 => simd_shuffle32!( b, a, [ @@ -183,7 +183,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 23, 24, 25, 26, 27, 28, 29, 30, 31, ], ), - 1 => simd_shuffle32( + 1 => simd_shuffle32!( b, a, [ @@ -191,7 +191,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 24, 25, 26, 27, 28, 29, 30, 31, 48, ], ), - 2 => simd_shuffle32( + 2 => simd_shuffle32!( b, a, [ @@ -199,7 +199,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 25, 26, 27, 28, 29, 30, 31, 48, 49, ], ), - 3 => simd_shuffle32( + 3 => simd_shuffle32!( b, a, [ @@ -207,7 +207,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 25, 26, 27, 28, 29, 30, 31, 48, 49, 50, ], ), - 4 => simd_shuffle32( + 4 => simd_shuffle32!( b, a, [ @@ -215,7 +215,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 26, 27, 28, 29, 30, 31, 48, 49, 50, 51, ], ), - 5 => simd_shuffle32( + 5 => simd_shuffle32!( b, a, [ @@ -223,7 +223,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 27, 28, 29, 30, 31, 48, 49, 50, 51, 52, ], ), - 6 => simd_shuffle32( + 6 => simd_shuffle32!( b, a, [ @@ -231,7 +231,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 28, 29, 30, 31, 48, 49, 50, 51, 52, 53, ], ), - 7 => simd_shuffle32( + 7 => simd_shuffle32!( b, a, [ @@ -239,7 +239,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 28, 29, 30, 31, 48, 49, 50, 51, 52, 53, 54, ], ), - 8 => simd_shuffle32( + 8 => simd_shuffle32!( b, a, [ @@ -247,7 +247,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 29, 30, 31, 48, 49, 50, 51, 52, 53, 54, 55, ], ), - 9 => simd_shuffle32( + 9 => simd_shuffle32!( b, a, [ @@ -255,7 +255,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 30, 31, 48, 49, 50, 51, 52, 53, 54, 55, 56, ], ), - 10 => simd_shuffle32( + 10 => simd_shuffle32!( b, a, [ @@ -263,7 +263,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 31, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, ], ), - 11 => simd_shuffle32( + 11 => simd_shuffle32!( b, a, [ @@ -271,7 +271,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, ], ), - 12 => simd_shuffle32( + 12 => simd_shuffle32!( b, a, [ @@ -279,7 +279,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, ], ), - 13 => simd_shuffle32( + 13 => simd_shuffle32!( b, a, [ @@ -287,7 +287,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, ], ), - 14 => simd_shuffle32( + 14 => simd_shuffle32!( b, a, [ @@ -295,7 +295,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, ], ), - 15 => simd_shuffle32( + 15 => simd_shuffle32!( b, a, [ @@ -358,209 +358,95 @@ pub unsafe fn _mm256_avg_epu8(a: __m256i, b: __m256i) -> __m256i { transmute(pavgb(a.as_u8x32(), b.as_u8x32())) } -/// Blends packed 32-bit integers from `a` and `b` using control mask `imm8`. +/// Blends packed 32-bit integers from `a` and `b` using control mask `IMM4`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_blend_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vblendps, imm8 = 9))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vblendps, IMM4 = 9))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_blend_epi32(a: __m128i, b: __m128i, imm8: i32) -> __m128i { - let imm8 = (imm8 & 0xFF) as u8; +pub unsafe fn _mm_blend_epi32(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm4!(IMM4); let a = a.as_i32x4(); let b = b.as_i32x4(); - macro_rules! blend2 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - simd_shuffle4(a, b, [$a, $b, $c, $d]); - }; - } - macro_rules! blend1 { - ($a:expr, $b:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => blend2!($a, $b, 2, 3), - 0b01 => blend2!($a, $b, 6, 3), - 0b10 => blend2!($a, $b, 2, 7), - _ => blend2!($a, $b, 6, 7), - } - }; - } - let r: i32x4 = match imm8 & 0b11 { - 0b00 => blend1!(0, 1), - 0b01 => blend1!(4, 1), - 0b10 => blend1!(0, 5), - _ => blend1!(4, 5), - }; + let r: i32x4 = simd_shuffle4!( + a, + b, + [ + [0, 4, 0, 4][IMM4 as usize & 0b11], + [1, 1, 5, 5][IMM4 as usize & 0b11], + [2, 6, 2, 6][(IMM4 as usize >> 2) & 0b11], + [3, 3, 7, 7][(IMM4 as usize >> 2) & 0b11], + ], + ); transmute(r) } -/// Blends packed 32-bit integers from `a` and `b` using control mask `imm8`. +/// Blends packed 32-bit integers from `a` and `b` using control mask `IMM8`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_blend_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vblendps, imm8 = 9))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vblendps, IMM8 = 9))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_blend_epi32(a: __m256i, b: __m256i, imm8: i32) -> __m256i { - let imm8 = (imm8 & 0xFF) as u8; +pub unsafe fn _mm256_blend_epi32(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i32x8(); let b = b.as_i32x8(); - macro_rules! blend4 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr, - $e:expr, - $f:expr, - $g:expr, - $h:expr - ) => { - simd_shuffle8(a, b, [$a, $b, $c, $d, $e, $f, $g, $h]); - }; - } - macro_rules! blend3 { - ($a:expr, $b:expr, $c:expr, $d:expr, $e:expr, $f:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => blend4!($a, $b, $c, $d, $e, $f, 6, 7), - 0b01 => blend4!($a, $b, $c, $d, $e, $f, 14, 7), - 0b10 => blend4!($a, $b, $c, $d, $e, $f, 6, 15), - _ => blend4!($a, $b, $c, $d, $e, $f, 14, 15), - } - }; - } - macro_rules! blend2 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => blend3!($a, $b, $c, $d, 4, 5), - 0b01 => blend3!($a, $b, $c, $d, 12, 5), - 0b10 => blend3!($a, $b, $c, $d, 4, 13), - _ => blend3!($a, $b, $c, $d, 12, 13), - } - }; - } - macro_rules! blend1 { - ($a:expr, $b:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => blend2!($a, $b, 2, 3), - 0b01 => blend2!($a, $b, 10, 3), - 0b10 => blend2!($a, $b, 2, 11), - _ => blend2!($a, $b, 10, 11), - } - }; - } - let r: i32x8 = match imm8 & 0b11 { - 0b00 => blend1!(0, 1), - 0b01 => blend1!(8, 1), - 0b10 => blend1!(0, 9), - _ => blend1!(8, 9), - }; + let r: i32x8 = simd_shuffle8!( + a, + b, + [ + [0, 8, 0, 8][IMM8 as usize & 0b11], + [1, 1, 9, 9][IMM8 as usize & 0b11], + [2, 10, 2, 10][(IMM8 as usize >> 2) & 0b11], + [3, 3, 11, 11][(IMM8 as usize >> 2) & 0b11], + [4, 12, 4, 12][(IMM8 as usize >> 4) & 0b11], + [5, 5, 13, 13][(IMM8 as usize >> 4) & 0b11], + [6, 14, 6, 14][(IMM8 as usize >> 6) & 0b11], + [7, 7, 15, 15][(IMM8 as usize >> 6) & 0b11], + ], + ); transmute(r) } -/// Blends packed 16-bit integers from `a` and `b` using control mask `imm8`. +/// Blends packed 16-bit integers from `a` and `b` using control mask `IMM8`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_blend_epi16) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpblendw, imm8 = 9))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vpblendw, IMM8 = 9))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_blend_epi16(a: __m256i, b: __m256i, imm8: i32) -> __m256i { - let imm8 = (imm8 & 0xFF) as u8; +pub unsafe fn _mm256_blend_epi16(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i16x16(); let b = b.as_i16x16(); - macro_rules! blend4 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr, - $e:expr, - $f:expr, - $g:expr, - $h:expr, - $i:expr, - $j:expr, - $k:expr, - $l:expr, - $m:expr, - $n:expr, - $o:expr, - $p:expr - ) => { - simd_shuffle16( - a, - b, - [ - $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k, $l, $m, $n, $o, $p, - ], - ) - }; - } - macro_rules! blend3 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr, - $e:expr, - $f:expr, - $a2:expr, - $b2:expr, - $c2:expr, - $d2:expr, - $e2:expr, - $f2:expr - ) => { - match (imm8 >> 6) & 0b11 { - 0b00 => blend4!($a, $b, $c, $d, $e, $f, 6, 7, $a2, $b2, $c2, $d2, $e2, $f2, 14, 15), - 0b01 => { - blend4!($a, $b, $c, $d, $e, $f, 22, 7, $a2, $b2, $c2, $d2, $e2, $f2, 30, 15) - } - 0b10 => { - blend4!($a, $b, $c, $d, $e, $f, 6, 23, $a2, $b2, $c2, $d2, $e2, $f2, 14, 31) - } - _ => blend4!($a, $b, $c, $d, $e, $f, 22, 23, $a2, $b2, $c2, $d2, $e2, $f2, 30, 31), - } - }; - } - macro_rules! blend2 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr, - $a2:expr, - $b2:expr, - $c2:expr, - $d2:expr - ) => { - match (imm8 >> 4) & 0b11 { - 0b00 => blend3!($a, $b, $c, $d, 4, 5, $a2, $b2, $c2, $d2, 12, 13), - 0b01 => blend3!($a, $b, $c, $d, 20, 5, $a2, $b2, $c2, $d2, 28, 13), - 0b10 => blend3!($a, $b, $c, $d, 4, 21, $a2, $b2, $c2, $d2, 12, 29), - _ => blend3!($a, $b, $c, $d, 20, 21, $a2, $b2, $c2, $d2, 28, 29), - } - }; - } - macro_rules! blend1 { - ($a1:expr, $b1:expr, $a2:expr, $b2:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => blend2!($a1, $b1, 2, 3, $a2, $b2, 10, 11), - 0b01 => blend2!($a1, $b1, 18, 3, $a2, $b2, 26, 11), - 0b10 => blend2!($a1, $b1, 2, 19, $a2, $b2, 10, 27), - _ => blend2!($a1, $b1, 18, 19, $a2, $b2, 26, 27), - } - }; - } - let r: i16x16 = match imm8 & 0b11 { - 0b00 => blend1!(0, 1, 8, 9), - 0b01 => blend1!(16, 1, 24, 9), - 0b10 => blend1!(0, 17, 8, 25), - _ => blend1!(16, 17, 24, 25), - }; + + let r: i16x16 = simd_shuffle16!( + a, + b, + [ + [0, 16, 0, 16][IMM8 as usize & 0b11], + [1, 1, 17, 17][IMM8 as usize & 0b11], + [2, 18, 2, 18][(IMM8 as usize >> 2) & 0b11], + [3, 3, 19, 19][(IMM8 as usize >> 2) & 0b11], + [4, 20, 4, 20][(IMM8 as usize >> 4) & 0b11], + [5, 5, 21, 21][(IMM8 as usize >> 4) & 0b11], + [6, 22, 6, 22][(IMM8 as usize >> 6) & 0b11], + [7, 7, 23, 23][(IMM8 as usize >> 6) & 0b11], + [8, 24, 8, 24][IMM8 as usize & 0b11], + [9, 9, 25, 25][IMM8 as usize & 0b11], + [10, 26, 10, 26][(IMM8 as usize >> 2) & 0b11], + [11, 11, 27, 27][(IMM8 as usize >> 2) & 0b11], + [12, 28, 12, 28][(IMM8 as usize >> 4) & 0b11], + [13, 13, 29, 29][(IMM8 as usize >> 4) & 0b11], + [14, 30, 14, 30][(IMM8 as usize >> 6) & 0b11], + [15, 15, 31, 31][(IMM8 as usize >> 6) & 0b11], + ], + ); transmute(r) } @@ -585,7 +471,7 @@ pub unsafe fn _mm256_blendv_epi8(a: __m256i, b: __m256i, mask: __m256i) -> __m25 #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_broadcastb_epi8(a: __m128i) -> __m128i { let zero = _mm_setzero_si128(); - let ret = simd_shuffle16(a.as_i8x16(), zero.as_i8x16(), [0_u32; 16]); + let ret = simd_shuffle16!(a.as_i8x16(), zero.as_i8x16(), [0_u32; 16]); transmute::(ret) } @@ -599,7 +485,7 @@ pub unsafe fn _mm_broadcastb_epi8(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_broadcastb_epi8(a: __m128i) -> __m256i { let zero = _mm_setzero_si128(); - let ret = simd_shuffle32(a.as_i8x16(), zero.as_i8x16(), [0_u32; 32]); + let ret = simd_shuffle32!(a.as_i8x16(), zero.as_i8x16(), [0_u32; 32]); transmute::(ret) } @@ -615,7 +501,7 @@ pub unsafe fn _mm256_broadcastb_epi8(a: __m128i) -> __m256i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_broadcastd_epi32(a: __m128i) -> __m128i { let zero = _mm_setzero_si128(); - let ret = simd_shuffle4(a.as_i32x4(), zero.as_i32x4(), [0_u32; 4]); + let ret = simd_shuffle4!(a.as_i32x4(), zero.as_i32x4(), [0_u32; 4]); transmute::(ret) } @@ -631,7 +517,7 @@ pub unsafe fn _mm_broadcastd_epi32(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_broadcastd_epi32(a: __m128i) -> __m256i { let zero = _mm_setzero_si128(); - let ret = simd_shuffle8(a.as_i32x4(), zero.as_i32x4(), [0_u32; 8]); + let ret = simd_shuffle8!(a.as_i32x4(), zero.as_i32x4(), [0_u32; 8]); transmute::(ret) } @@ -645,7 +531,7 @@ pub unsafe fn _mm256_broadcastd_epi32(a: __m128i) -> __m256i { #[cfg_attr(test, assert_instr(vmovddup))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_broadcastq_epi64(a: __m128i) -> __m128i { - let ret = simd_shuffle2(a.as_i64x2(), a.as_i64x2(), [0_u32; 2]); + let ret = simd_shuffle2!(a.as_i64x2(), a.as_i64x2(), [0_u32; 2]); transmute::(ret) } @@ -658,7 +544,7 @@ pub unsafe fn _mm_broadcastq_epi64(a: __m128i) -> __m128i { #[cfg_attr(test, assert_instr(vbroadcastsd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_broadcastq_epi64(a: __m128i) -> __m256i { - let ret = simd_shuffle4(a.as_i64x2(), a.as_i64x2(), [0_u32; 4]); + let ret = simd_shuffle4!(a.as_i64x2(), a.as_i64x2(), [0_u32; 4]); transmute::(ret) } @@ -671,7 +557,7 @@ pub unsafe fn _mm256_broadcastq_epi64(a: __m128i) -> __m256i { #[cfg_attr(test, assert_instr(vmovddup))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_broadcastsd_pd(a: __m128d) -> __m128d { - simd_shuffle2(a, _mm_setzero_pd(), [0_u32; 2]) + simd_shuffle2!(a, _mm_setzero_pd(), [0_u32; 2]) } /// Broadcasts the low double-precision (64-bit) floating-point element @@ -683,7 +569,7 @@ pub unsafe fn _mm_broadcastsd_pd(a: __m128d) -> __m128d { #[cfg_attr(test, assert_instr(vbroadcastsd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_broadcastsd_pd(a: __m128d) -> __m256d { - simd_shuffle4(a, _mm_setzero_pd(), [0_u32; 4]) + simd_shuffle4!(a, _mm_setzero_pd(), [0_u32; 4]) } // N.B., `broadcastsi128_si256` is often compiled to `vinsertf128` or @@ -697,7 +583,7 @@ pub unsafe fn _mm256_broadcastsd_pd(a: __m128d) -> __m256d { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_broadcastsi128_si256(a: __m128i) -> __m256i { let zero = _mm_setzero_si128(); - let ret = simd_shuffle4(a.as_i64x2(), zero.as_i64x2(), [0, 1, 0, 1]); + let ret = simd_shuffle4!(a.as_i64x2(), zero.as_i64x2(), [0, 1, 0, 1]); transmute::(ret) } @@ -710,7 +596,7 @@ pub unsafe fn _mm256_broadcastsi128_si256(a: __m128i) -> __m256i { #[cfg_attr(test, assert_instr(vbroadcastss))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_broadcastss_ps(a: __m128) -> __m128 { - simd_shuffle4(a, _mm_setzero_ps(), [0_u32; 4]) + simd_shuffle4!(a, _mm_setzero_ps(), [0_u32; 4]) } /// Broadcasts the low single-precision (32-bit) floating-point element @@ -722,7 +608,7 @@ pub unsafe fn _mm_broadcastss_ps(a: __m128) -> __m128 { #[cfg_attr(test, assert_instr(vbroadcastss))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_broadcastss_ps(a: __m128) -> __m256 { - simd_shuffle8(a, _mm_setzero_ps(), [0_u32; 8]) + simd_shuffle8!(a, _mm_setzero_ps(), [0_u32; 8]) } /// Broadcasts the low packed 16-bit integer from a to all elements of @@ -735,7 +621,7 @@ pub unsafe fn _mm256_broadcastss_ps(a: __m128) -> __m256 { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_broadcastw_epi16(a: __m128i) -> __m128i { let zero = _mm_setzero_si128(); - let ret = simd_shuffle8(a.as_i16x8(), zero.as_i16x8(), [0_u32; 8]); + let ret = simd_shuffle8!(a.as_i16x8(), zero.as_i16x8(), [0_u32; 8]); transmute::(ret) } @@ -749,7 +635,7 @@ pub unsafe fn _mm_broadcastw_epi16(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_broadcastw_epi16(a: __m128i) -> __m256i { let zero = _mm_setzero_si128(); - let ret = simd_shuffle16(a.as_i16x8(), zero.as_i16x8(), [0_u32; 16]); + let ret = simd_shuffle16!(a.as_i16x8(), zero.as_i16x8(), [0_u32; 16]); transmute::(ret) } @@ -861,7 +747,7 @@ pub unsafe fn _mm256_cvtepi16_epi32(a: __m128i) -> __m256i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_cvtepi16_epi64(a: __m128i) -> __m256i { let a = a.as_i16x8(); - let v64: i16x4 = simd_shuffle4(a, a, [0, 1, 2, 3]); + let v64: i16x4 = simd_shuffle4!(a, a, [0, 1, 2, 3]); transmute::(simd_cast(v64)) } @@ -896,7 +782,7 @@ pub unsafe fn _mm256_cvtepi8_epi16(a: __m128i) -> __m256i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_cvtepi8_epi32(a: __m128i) -> __m256i { let a = a.as_i8x16(); - let v64: i8x8 = simd_shuffle8(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); + let v64: i8x8 = simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); transmute::(simd_cast(v64)) } @@ -909,7 +795,7 @@ pub unsafe fn _mm256_cvtepi8_epi32(a: __m128i) -> __m256i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_cvtepi8_epi64(a: __m128i) -> __m256i { let a = a.as_i8x16(); - let v32: i8x4 = simd_shuffle4(a, a, [0, 1, 2, 3]); + let v32: i8x4 = simd_shuffle4!(a, a, [0, 1, 2, 3]); transmute::(simd_cast(v32)) } @@ -935,7 +821,7 @@ pub unsafe fn _mm256_cvtepu16_epi32(a: __m128i) -> __m256i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_cvtepu16_epi64(a: __m128i) -> __m256i { let a = a.as_u16x8(); - let v64: u16x4 = simd_shuffle4(a, a, [0, 1, 2, 3]); + let v64: u16x4 = simd_shuffle4!(a, a, [0, 1, 2, 3]); transmute::(simd_cast(v64)) } @@ -971,7 +857,7 @@ pub unsafe fn _mm256_cvtepu8_epi16(a: __m128i) -> __m256i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_cvtepu8_epi32(a: __m128i) -> __m256i { let a = a.as_u8x16(); - let v64: u8x8 = simd_shuffle8(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); + let v64: u8x8 = simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); transmute::(simd_cast(v64)) } @@ -985,28 +871,26 @@ pub unsafe fn _mm256_cvtepu8_epi32(a: __m128i) -> __m256i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_cvtepu8_epi64(a: __m128i) -> __m256i { let a = a.as_u8x16(); - let v32: u8x4 = simd_shuffle4(a, a, [0, 1, 2, 3]); + let v32: u8x4 = simd_shuffle4!(a, a, [0, 1, 2, 3]); transmute::(simd_cast(v32)) } -/// Extracts 128 bits (of integer data) from `a` selected with `imm8`. +/// Extracts 128 bits (of integer data) from `a` selected with `IMM1`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_extracti128_si256) #[inline] #[target_feature(enable = "avx2")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextractf128, imm8 = 1) + assert_instr(vextractf128, IMM1 = 1) )] -#[rustc_args_required_const(1)] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_extracti128_si256(a: __m256i, imm8: i32) -> __m128i { +pub unsafe fn _mm256_extracti128_si256(a: __m256i) -> __m128i { + static_assert_imm1!(IMM1); let a = a.as_i64x4(); let b = _mm256_undefined_si256().as_i64x4(); - let dst: i64x2 = match imm8 & 0b01 { - 0 => simd_shuffle2(a, b, [0, 1]), - _ => simd_shuffle2(a, b, [2, 3]), - }; + let dst: i64x2 = simd_shuffle2!(a, b, [[0, 1], [2, 3]][IMM1 as usize]); transmute(dst) } @@ -1080,871 +964,756 @@ pub unsafe fn _mm256_hsubs_epi16(a: __m256i, b: __m256i) -> __m256i { /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. +/// `scale` should be 1, 2, 4 or 8. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_i32gather_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherdd, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vpgatherdd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_i32gather_epi32(slice: *const i32, offsets: __m128i, scale: i32) -> __m128i { +pub unsafe fn _mm_i32gather_epi32( + slice: *const i32, + offsets: __m128i, +) -> __m128i { + static_assert_imm8_scale!(SCALE); let zero = _mm_setzero_si128().as_i32x4(); let neg_one = _mm_set1_epi32(-1).as_i32x4(); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherdd(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8!(scale, call); + let r = pgatherdd(zero, slice, offsets, neg_one, SCALE as i8); transmute(r) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. If mask is set, load the value from `src` in +/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in /// that position instead. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_i32gather_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherdd, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vpgatherdd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_mask_i32gather_epi32( +pub unsafe fn _mm_mask_i32gather_epi32( src: __m128i, slice: *const i32, offsets: __m128i, mask: __m128i, - scale: i32, ) -> __m128i { + static_assert_imm8_scale!(SCALE); let src = src.as_i32x4(); let mask = mask.as_i32x4(); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherdd(src, slice, offsets, mask, $imm8) - }; - } - let r = constify_imm8!(scale, call); + let r = pgatherdd(src, slice, offsets, mask, SCALE as i8); transmute(r) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. +/// `scale` should be 1, 2, 4 or 8. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_i32gather_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherdd, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vpgatherdd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_i32gather_epi32(slice: *const i32, offsets: __m256i, scale: i32) -> __m256i { +pub unsafe fn _mm256_i32gather_epi32( + slice: *const i32, + offsets: __m256i, +) -> __m256i { + static_assert_imm8_scale!(SCALE); let zero = _mm256_setzero_si256().as_i32x8(); let neg_one = _mm256_set1_epi32(-1).as_i32x8(); let offsets = offsets.as_i32x8(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherdd(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8!(scale, call); + let r = vpgatherdd(zero, slice, offsets, neg_one, SCALE as i8); transmute(r) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. If mask is set, load the value from `src` in +/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in /// that position instead. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_i32gather_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherdd, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vpgatherdd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_mask_i32gather_epi32( +pub unsafe fn _mm256_mask_i32gather_epi32( src: __m256i, slice: *const i32, offsets: __m256i, mask: __m256i, - scale: i32, ) -> __m256i { + static_assert_imm8_scale!(SCALE); let src = src.as_i32x8(); let mask = mask.as_i32x8(); let offsets = offsets.as_i32x8(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherdd(src, slice, offsets, mask, $imm8) - }; - } - let r = constify_imm8!(scale, call); + let r = vpgatherdd(src, slice, offsets, mask, SCALE as i8); transmute(r) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. +/// `scale` should be 1, 2, 4 or 8. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_i32gather_ps) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherdps, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vgatherdps, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_i32gather_ps(slice: *const f32, offsets: __m128i, scale: i32) -> __m128 { +pub unsafe fn _mm_i32gather_ps(slice: *const f32, offsets: __m128i) -> __m128 { + static_assert_imm8_scale!(SCALE); let zero = _mm_setzero_ps(); let neg_one = _mm_set1_ps(-1.0); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherdps(zero, slice, offsets, neg_one, $imm8) - }; - } - constify_imm8!(scale, call) + pgatherdps(zero, slice, offsets, neg_one, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. If mask is set, load the value from `src` in +/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in /// that position instead. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_i32gather_ps) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherdps, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vgatherdps, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_mask_i32gather_ps( +pub unsafe fn _mm_mask_i32gather_ps( src: __m128, slice: *const f32, offsets: __m128i, mask: __m128, - scale: i32, ) -> __m128 { + static_assert_imm8_scale!(SCALE); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherdps(src, slice, offsets, mask, $imm8) - }; - } - constify_imm8!(scale, call) + pgatherdps(src, slice, offsets, mask, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. +/// `scale` should be 1, 2, 4 or 8. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_i32gather_ps) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherdps, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vgatherdps, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_i32gather_ps(slice: *const f32, offsets: __m256i, scale: i32) -> __m256 { +pub unsafe fn _mm256_i32gather_ps(slice: *const f32, offsets: __m256i) -> __m256 { + static_assert_imm8_scale!(SCALE); let zero = _mm256_setzero_ps(); let neg_one = _mm256_set1_ps(-1.0); let offsets = offsets.as_i32x8(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherdps(zero, slice, offsets, neg_one, $imm8) - }; - } - constify_imm8!(scale, call) + vpgatherdps(zero, slice, offsets, neg_one, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. If mask is set, load the value from `src` in +/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in /// that position instead. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_i32gather_ps) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherdps, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vgatherdps, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_mask_i32gather_ps( +pub unsafe fn _mm256_mask_i32gather_ps( src: __m256, slice: *const f32, offsets: __m256i, mask: __m256, - scale: i32, ) -> __m256 { + static_assert_imm8_scale!(SCALE); let offsets = offsets.as_i32x8(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherdps(src, slice, offsets, mask, $imm8) - }; - } - constify_imm8!(scale, call) + vpgatherdps(src, slice, offsets, mask, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. +/// `scale` should be 1, 2, 4 or 8. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_i32gather_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherdq, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vpgatherdq, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_i32gather_epi64(slice: *const i64, offsets: __m128i, scale: i32) -> __m128i { +pub unsafe fn _mm_i32gather_epi64( + slice: *const i64, + offsets: __m128i, +) -> __m128i { + static_assert_imm8_scale!(SCALE); let zero = _mm_setzero_si128().as_i64x2(); let neg_one = _mm_set1_epi64x(-1).as_i64x2(); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherdq(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8!(scale, call); + let r = pgatherdq(zero, slice, offsets, neg_one, SCALE as i8); transmute(r) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. If mask is set, load the value from `src` in +/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in /// that position instead. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_i32gather_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherdq, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vpgatherdq, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_mask_i32gather_epi64( +pub unsafe fn _mm_mask_i32gather_epi64( src: __m128i, slice: *const i64, offsets: __m128i, mask: __m128i, - scale: i32, ) -> __m128i { + static_assert_imm8_scale!(SCALE); let src = src.as_i64x2(); let mask = mask.as_i64x2(); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherdq(src, slice, offsets, mask, $imm8) - }; - } - let r = constify_imm8!(scale, call); + let r = pgatherdq(src, slice, offsets, mask, SCALE as i8); transmute(r) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. +/// `scale` should be 1, 2, 4 or 8. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_i32gather_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherdq, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vpgatherdq, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_i32gather_epi64(slice: *const i64, offsets: __m128i, scale: i32) -> __m256i { +pub unsafe fn _mm256_i32gather_epi64( + slice: *const i64, + offsets: __m128i, +) -> __m256i { + static_assert_imm8_scale!(SCALE); let zero = _mm256_setzero_si256().as_i64x4(); let neg_one = _mm256_set1_epi64x(-1).as_i64x4(); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherdq(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8!(scale, call); + let r = vpgatherdq(zero, slice, offsets, neg_one, SCALE as i8); transmute(r) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. If mask is set, load the value from `src` in +/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in /// that position instead. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_i32gather_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherdq, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vpgatherdq, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_mask_i32gather_epi64( +pub unsafe fn _mm256_mask_i32gather_epi64( src: __m256i, slice: *const i64, offsets: __m128i, mask: __m256i, - scale: i32, ) -> __m256i { + static_assert_imm8_scale!(SCALE); let src = src.as_i64x4(); let mask = mask.as_i64x4(); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherdq(src, slice, offsets, mask, $imm8) - }; - } - let r = constify_imm8!(scale, call); + let r = vpgatherdq(src, slice, offsets, mask, SCALE as i8); transmute(r) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. +/// `scale` should be 1, 2, 4 or 8. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_i32gather_pd) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherdpd, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vgatherdpd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_i32gather_pd(slice: *const f64, offsets: __m128i, scale: i32) -> __m128d { +pub unsafe fn _mm_i32gather_pd(slice: *const f64, offsets: __m128i) -> __m128d { + static_assert_imm8_scale!(SCALE); let zero = _mm_setzero_pd(); let neg_one = _mm_set1_pd(-1.0); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherdpd(zero, slice, offsets, neg_one, $imm8) - }; - } - constify_imm8!(scale, call) + pgatherdpd(zero, slice, offsets, neg_one, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. If mask is set, load the value from `src` in +/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in /// that position instead. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_i32gather_pd) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherdpd, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vgatherdpd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_mask_i32gather_pd( +pub unsafe fn _mm_mask_i32gather_pd( src: __m128d, slice: *const f64, offsets: __m128i, mask: __m128d, - scale: i32, ) -> __m128d { + static_assert_imm8_scale!(SCALE); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherdpd(src, slice, offsets, mask, $imm8) - }; - } - constify_imm8!(scale, call) + pgatherdpd(src, slice, offsets, mask, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. +/// `scale` should be 1, 2, 4 or 8. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_i32gather_pd) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherdpd, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vgatherdpd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_i32gather_pd(slice: *const f64, offsets: __m128i, scale: i32) -> __m256d { +pub unsafe fn _mm256_i32gather_pd( + slice: *const f64, + offsets: __m128i, +) -> __m256d { + static_assert_imm8_scale!(SCALE); let zero = _mm256_setzero_pd(); let neg_one = _mm256_set1_pd(-1.0); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherdpd(zero, slice, offsets, neg_one, $imm8) - }; - } - constify_imm8!(scale, call) + vpgatherdpd(zero, slice, offsets, neg_one, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. If mask is set, load the value from `src` in +/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in /// that position instead. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_i32gather_pd) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherdpd, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vgatherdpd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_mask_i32gather_pd( +pub unsafe fn _mm256_mask_i32gather_pd( src: __m256d, slice: *const f64, offsets: __m128i, mask: __m256d, - scale: i32, ) -> __m256d { + static_assert_imm8_scale!(SCALE); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherdpd(src, slice, offsets, mask, $imm8) - }; - } - constify_imm8!(scale, call) + vpgatherdpd(src, slice, offsets, mask, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. +/// `scale` should be 1, 2, 4 or 8. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_i64gather_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherqd, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vpgatherqd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_i64gather_epi32(slice: *const i32, offsets: __m128i, scale: i32) -> __m128i { +pub unsafe fn _mm_i64gather_epi32( + slice: *const i32, + offsets: __m128i, +) -> __m128i { + static_assert_imm8_scale!(SCALE); let zero = _mm_setzero_si128().as_i32x4(); let neg_one = _mm_set1_epi64x(-1).as_i32x4(); let offsets = offsets.as_i64x2(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherqd(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8!(scale, call); + let r = pgatherqd(zero, slice, offsets, neg_one, SCALE as i8); transmute(r) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. If mask is set, load the value from `src` in +/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in /// that position instead. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_i64gather_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherqd, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vpgatherqd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_mask_i64gather_epi32( +pub unsafe fn _mm_mask_i64gather_epi32( src: __m128i, slice: *const i32, offsets: __m128i, mask: __m128i, - scale: i32, ) -> __m128i { + static_assert_imm8_scale!(SCALE); let src = src.as_i32x4(); let mask = mask.as_i32x4(); let offsets = offsets.as_i64x2(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherqd(src, slice, offsets, mask, $imm8) - }; - } - let r = constify_imm8!(scale, call); + let r = pgatherqd(src, slice, offsets, mask, SCALE as i8); transmute(r) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. +/// `scale` should be 1, 2, 4 or 8. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_i64gather_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherqd, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vpgatherqd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_i64gather_epi32(slice: *const i32, offsets: __m256i, scale: i32) -> __m128i { +pub unsafe fn _mm256_i64gather_epi32( + slice: *const i32, + offsets: __m256i, +) -> __m128i { + static_assert_imm8_scale!(SCALE); let zero = _mm_setzero_si128().as_i32x4(); let neg_one = _mm_set1_epi64x(-1).as_i32x4(); let offsets = offsets.as_i64x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherqd(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8!(scale, call); + let r = vpgatherqd(zero, slice, offsets, neg_one, SCALE as i8); transmute(r) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. If mask is set, load the value from `src` in +/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in /// that position instead. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_i64gather_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherqd, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vpgatherqd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_mask_i64gather_epi32( +pub unsafe fn _mm256_mask_i64gather_epi32( src: __m128i, slice: *const i32, offsets: __m256i, mask: __m128i, - scale: i32, ) -> __m128i { + static_assert_imm8_scale!(SCALE); let src = src.as_i32x4(); let mask = mask.as_i32x4(); let offsets = offsets.as_i64x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherqd(src, slice, offsets, mask, $imm8) - }; - } - let r = constify_imm8!(scale, call); + let r = vpgatherqd(src, slice, offsets, mask, SCALE as i8); transmute(r) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. +/// `scale` should be 1, 2, 4 or 8. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_i64gather_ps) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherqps, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vgatherqps, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_i64gather_ps(slice: *const f32, offsets: __m128i, scale: i32) -> __m128 { +pub unsafe fn _mm_i64gather_ps(slice: *const f32, offsets: __m128i) -> __m128 { + static_assert_imm8_scale!(SCALE); let zero = _mm_setzero_ps(); let neg_one = _mm_set1_ps(-1.0); let offsets = offsets.as_i64x2(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherqps(zero, slice, offsets, neg_one, $imm8) - }; - } - constify_imm8!(scale, call) + pgatherqps(zero, slice, offsets, neg_one, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. If mask is set, load the value from `src` in +/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in /// that position instead. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_i64gather_ps) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherqps, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vgatherqps, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_mask_i64gather_ps( +pub unsafe fn _mm_mask_i64gather_ps( src: __m128, slice: *const f32, offsets: __m128i, mask: __m128, - scale: i32, ) -> __m128 { + static_assert_imm8_scale!(SCALE); let offsets = offsets.as_i64x2(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherqps(src, slice, offsets, mask, $imm8) - }; - } - constify_imm8!(scale, call) + pgatherqps(src, slice, offsets, mask, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. +/// `scale` should be 1, 2, 4 or 8. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_i64gather_ps) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherqps, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vgatherqps, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_i64gather_ps(slice: *const f32, offsets: __m256i, scale: i32) -> __m128 { +pub unsafe fn _mm256_i64gather_ps(slice: *const f32, offsets: __m256i) -> __m128 { + static_assert_imm8_scale!(SCALE); let zero = _mm_setzero_ps(); let neg_one = _mm_set1_ps(-1.0); let offsets = offsets.as_i64x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherqps(zero, slice, offsets, neg_one, $imm8) - }; - } - constify_imm8!(scale, call) + vpgatherqps(zero, slice, offsets, neg_one, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. If mask is set, load the value from `src` in +/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in /// that position instead. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_i64gather_ps) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherqps, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vgatherqps, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_mask_i64gather_ps( +pub unsafe fn _mm256_mask_i64gather_ps( src: __m128, slice: *const f32, offsets: __m256i, mask: __m128, - scale: i32, ) -> __m128 { + static_assert_imm8_scale!(SCALE); let offsets = offsets.as_i64x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherqps(src, slice, offsets, mask, $imm8) - }; - } - constify_imm8!(scale, call) + vpgatherqps(src, slice, offsets, mask, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. +/// `scale` should be 1, 2, 4 or 8. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_i64gather_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherqq, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vpgatherqq, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_i64gather_epi64(slice: *const i64, offsets: __m128i, scale: i32) -> __m128i { +pub unsafe fn _mm_i64gather_epi64( + slice: *const i64, + offsets: __m128i, +) -> __m128i { + static_assert_imm8_scale!(SCALE); let zero = _mm_setzero_si128().as_i64x2(); let neg_one = _mm_set1_epi64x(-1).as_i64x2(); let slice = slice as *const i8; let offsets = offsets.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - pgatherqq(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8!(scale, call); + let r = pgatherqq(zero, slice, offsets, neg_one, SCALE as i8); transmute(r) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. If mask is set, load the value from `src` in +/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in /// that position instead. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_i64gather_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherqq, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vpgatherqq, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_mask_i64gather_epi64( +pub unsafe fn _mm_mask_i64gather_epi64( src: __m128i, slice: *const i64, offsets: __m128i, mask: __m128i, - scale: i32, ) -> __m128i { + static_assert_imm8_scale!(SCALE); let src = src.as_i64x2(); let mask = mask.as_i64x2(); let offsets = offsets.as_i64x2(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherqq(src, slice, offsets, mask, $imm8) - }; - } - let r = constify_imm8!(scale, call); + let r = pgatherqq(src, slice, offsets, mask, SCALE as i8); transmute(r) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. +/// `scale` should be 1, 2, 4 or 8. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_i64gather_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherqq, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vpgatherqq, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_i64gather_epi64(slice: *const i64, offsets: __m256i, scale: i32) -> __m256i { +pub unsafe fn _mm256_i64gather_epi64( + slice: *const i64, + offsets: __m256i, +) -> __m256i { + static_assert_imm8_scale!(SCALE); let zero = _mm256_setzero_si256().as_i64x4(); let neg_one = _mm256_set1_epi64x(-1).as_i64x4(); let slice = slice as *const i8; let offsets = offsets.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vpgatherqq(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8!(scale, call); + let r = vpgatherqq(zero, slice, offsets, neg_one, SCALE as i8); transmute(r) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. If mask is set, load the value from `src` in +/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in /// that position instead. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_i64gather_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherqq, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vpgatherqq, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_mask_i64gather_epi64( +pub unsafe fn _mm256_mask_i64gather_epi64( src: __m256i, slice: *const i64, offsets: __m256i, mask: __m256i, - scale: i32, ) -> __m256i { + static_assert_imm8_scale!(SCALE); let src = src.as_i64x4(); let mask = mask.as_i64x4(); let offsets = offsets.as_i64x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherqq(src, slice, offsets, mask, $imm8) - }; - } - let r = constify_imm8!(scale, call); + let r = vpgatherqq(src, slice, offsets, mask, SCALE as i8); transmute(r) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. +/// `scale` should be 1, 2, 4 or 8. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_i64gather_pd) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherqpd, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vgatherqpd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_i64gather_pd(slice: *const f64, offsets: __m128i, scale: i32) -> __m128d { +pub unsafe fn _mm_i64gather_pd(slice: *const f64, offsets: __m128i) -> __m128d { + static_assert_imm8_scale!(SCALE); let zero = _mm_setzero_pd(); let neg_one = _mm_set1_pd(-1.0); let slice = slice as *const i8; let offsets = offsets.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - pgatherqpd(zero, slice, offsets, neg_one, $imm8) - }; - } - constify_imm8!(scale, call) + pgatherqpd(zero, slice, offsets, neg_one, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. If mask is set, load the value from `src` in +/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in /// that position instead. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_i64gather_pd) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherqpd, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vgatherqpd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_mask_i64gather_pd( +pub unsafe fn _mm_mask_i64gather_pd( src: __m128d, slice: *const f64, offsets: __m128i, mask: __m128d, - scale: i32, ) -> __m128d { + static_assert_imm8_scale!(SCALE); let slice = slice as *const i8; let offsets = offsets.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - pgatherqpd(src, slice, offsets, mask, $imm8) - }; - } - constify_imm8!(scale, call) + pgatherqpd(src, slice, offsets, mask, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. +/// `scale` should be 1, 2, 4 or 8. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_i64gather_pd) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherqpd, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vgatherqpd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_i64gather_pd(slice: *const f64, offsets: __m256i, scale: i32) -> __m256d { +pub unsafe fn _mm256_i64gather_pd( + slice: *const f64, + offsets: __m256i, +) -> __m256d { + static_assert_imm8_scale!(SCALE); let zero = _mm256_setzero_pd(); let neg_one = _mm256_set1_pd(-1.0); let slice = slice as *const i8; let offsets = offsets.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vpgatherqpd(zero, slice, offsets, neg_one, $imm8) - }; - } - constify_imm8!(scale, call) + vpgatherqpd(zero, slice, offsets, neg_one, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, /// where -/// `scale` is between 1 and 8. If mask is set, load the value from `src` in +/// `scale` should be 1, 2, 4 or 8. If mask is set, load the value from `src` in /// that position instead. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_i64gather_pd) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherqpd, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vgatherqpd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_mask_i64gather_pd( +pub unsafe fn _mm256_mask_i64gather_pd( src: __m256d, slice: *const f64, offsets: __m256i, mask: __m256d, - scale: i32, ) -> __m256d { + static_assert_imm8_scale!(SCALE); let slice = slice as *const i8; let offsets = offsets.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vpgatherqpd(src, slice, offsets, mask, $imm8) - }; - } - constify_imm8!(scale, call) + vpgatherqpd(src, slice, offsets, mask, SCALE as i8) } /// Copies `a` to `dst`, then insert 128 bits (of integer data) from `b` at the -/// location specified by `imm8`. +/// location specified by `IMM1`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_inserti128_si256) #[inline] #[target_feature(enable = "avx2")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vinsertf128, imm8 = 1) + assert_instr(vinsertf128, IMM1 = 1) )] -#[rustc_args_required_const(2)] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_inserti128_si256(a: __m256i, b: __m128i, imm8: i32) -> __m256i { +pub unsafe fn _mm256_inserti128_si256(a: __m256i, b: __m128i) -> __m256i { + static_assert_imm1!(IMM1); let a = a.as_i64x4(); let b = _mm256_castsi128_si256(b).as_i64x4(); - let dst: i64x4 = match imm8 & 0b01 { - 0 => simd_shuffle4(a, b, [4, 5, 2, 3]), - _ => simd_shuffle4(a, b, [0, 1, 4, 5]), - }; + let dst: i64x4 = + simd_shuffle4!(a, b, [[4, 5, 2, 3], [0, 1, 4, 5]][IMM1 as usize]); transmute(dst) } @@ -2232,7 +2001,7 @@ pub unsafe fn _mm256_min_epu8(a: __m256i, b: __m256i) -> __m256i { #[cfg_attr(test, assert_instr(vpmovmskb))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_movemask_epi8(a: __m256i) -> i32 { - pmovmskb(a.as_i8x32()) + simd_bitmask::<_, u32>(a.as_i8x32()) as i32 } /// Computes the sum of absolute differences (SADs) of quadruplets of unsigned @@ -2246,19 +2015,12 @@ pub unsafe fn _mm256_movemask_epi8(a: __m256i) -> i32 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mpsadbw_epu8) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vmpsadbw, imm8 = 0))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vmpsadbw, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_mpsadbw_epu8(a: __m256i, b: __m256i, imm8: i32) -> __m256i { - let a = a.as_u8x32(); - let b = b.as_u8x32(); - macro_rules! call { - ($imm8:expr) => { - mpsadbw(a, b, $imm8) - }; - } - let r = constify_imm8!(imm8, call); - transmute(r) +pub unsafe fn _mm256_mpsadbw_epu8(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(mpsadbw(a.as_u8x32(), b.as_u8x32(), IMM8)) } /// Multiplies the low 32-bit integers from each packed 64-bit element in @@ -2434,54 +2196,22 @@ pub unsafe fn _mm256_permutevar8x32_epi32(a: __m256i, b: __m256i) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permute4x64_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpermpd, imm8 = 9))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(vpermpd, IMM8 = 9))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_permute4x64_epi64(a: __m256i, imm8: i32) -> __m256i { - let imm8 = (imm8 & 0xFF) as u8; +pub unsafe fn _mm256_permute4x64_epi64(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); let zero = _mm256_setzero_si256().as_i64x4(); - let a = a.as_i64x4(); - macro_rules! permute4 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - simd_shuffle4(a, zero, [$a, $b, $c, $d]); - }; - } - macro_rules! permute3 { - ($a:expr, $b:expr, $c:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => permute4!($a, $b, $c, 0), - 0b01 => permute4!($a, $b, $c, 1), - 0b10 => permute4!($a, $b, $c, 2), - _ => permute4!($a, $b, $c, 3), - } - }; - } - macro_rules! permute2 { - ($a:expr, $b:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => permute3!($a, $b, 0), - 0b01 => permute3!($a, $b, 1), - 0b10 => permute3!($a, $b, 2), - _ => permute3!($a, $b, 3), - } - }; - } - macro_rules! permute1 { - ($a:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => permute2!($a, 0), - 0b01 => permute2!($a, 1), - 0b10 => permute2!($a, 2), - _ => permute2!($a, 3), - } - }; - } - let r: i64x4 = match imm8 & 0b11 { - 0b00 => permute1!(0), - 0b01 => permute1!(1), - 0b10 => permute1!(2), - _ => permute1!(3), - }; + let r: i64x4 = simd_shuffle4!( + a.as_i64x4(), + zero, + [ + IMM8 as u32 & 0b11, + (IMM8 as u32 >> 2) & 0b11, + (IMM8 as u32 >> 4) & 0b11, + (IMM8 as u32 >> 6) & 0b11, + ], + ); transmute(r) } @@ -2490,18 +2220,12 @@ pub unsafe fn _mm256_permute4x64_epi64(a: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permute2x128_si256) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vperm2f128, imm8 = 9))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vperm2f128, IMM8 = 9))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_permute2x128_si256(a: __m256i, b: __m256i, imm8: i32) -> __m256i { - let a = a.as_i64x4(); - let b = b.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vperm2i128(a, b, $imm8) - }; - } - transmute(constify_imm8!(imm8, call)) +pub unsafe fn _mm256_permute2x128_si256(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(vperm2i128(a.as_i64x4(), b.as_i64x4(), IMM8 as i8)) } /// Shuffles 64-bit floating-point elements in `a` across lanes using the @@ -2510,53 +2234,21 @@ pub unsafe fn _mm256_permute2x128_si256(a: __m256i, b: __m256i, imm8: i32) -> __ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permute4x64_pd) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpermpd, imm8 = 1))] -#[rustc_args_required_const(1)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_permute4x64_pd(a: __m256d, imm8: i32) -> __m256d { - let imm8 = (imm8 & 0xFF) as u8; - let undef = _mm256_undefined_pd(); - macro_rules! shuffle_done { - ($x01:expr, $x23:expr, $x45:expr, $x67:expr) => { - simd_shuffle4(a, undef, [$x01, $x23, $x45, $x67]) - }; - } - macro_rules! shuffle_x67 { - ($x01:expr, $x23:expr, $x45:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle_done!($x01, $x23, $x45, 0), - 0b01 => shuffle_done!($x01, $x23, $x45, 1), - 0b10 => shuffle_done!($x01, $x23, $x45, 2), - _ => shuffle_done!($x01, $x23, $x45, 3), - } - }; - } - macro_rules! shuffle_x45 { - ($x01:expr, $x23:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle_x67!($x01, $x23, 0), - 0b01 => shuffle_x67!($x01, $x23, 1), - 0b10 => shuffle_x67!($x01, $x23, 2), - _ => shuffle_x67!($x01, $x23, 3), - } - }; - } - macro_rules! shuffle_x23 { - ($x01:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle_x45!($x01, 0), - 0b01 => shuffle_x45!($x01, 1), - 0b10 => shuffle_x45!($x01, 2), - _ => shuffle_x45!($x01, 3), - } - }; - } - match imm8 & 0b11 { - 0b00 => shuffle_x23!(0), - 0b01 => shuffle_x23!(1), - 0b10 => shuffle_x23!(2), - _ => shuffle_x23!(3), - } +#[cfg_attr(test, assert_instr(vpermpd, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_permute4x64_pd(a: __m256d) -> __m256d { + static_assert_imm8!(IMM8); + simd_shuffle4!( + a, + _mm256_undefined_pd(), + [ + IMM8 as u32 & 0b11, + (IMM8 as u32 >> 2) & 0b11, + (IMM8 as u32 >> 4) & 0b11, + (IMM8 as u32 >> 6) & 0b11, + ], + ) } /// Shuffles eight 32-bit foating-point elements in `a` across lanes using @@ -2587,14 +2279,13 @@ pub unsafe fn _mm256_sad_epu8(a: __m256i, b: __m256i) -> __m256i { /// Shuffles bytes from `a` according to the content of `b`. /// -/// The last 4 bits of each byte of `b` are used as addresses into the 32 bytes -/// of `a`. +/// For each of the 128-bit low and high halves of the vectors, the last +/// 4 bits of each byte of `b` are used as addresses into the respective +/// low or high 16 bytes of `a`. That is, the halves are shuffled separately. /// /// In addition, if the highest significant bit of a byte of `b` is set, the /// respective destination byte is set to 0. /// -/// The low and high halves of the vectors are shuffled separately. -/// /// Picturing `a` and `b` as `[u8; 32]`, `_mm256_shuffle_epi8` is logically /// equivalent to: /// @@ -2656,74 +2347,25 @@ pub unsafe fn _mm256_shuffle_epi8(a: __m256i, b: __m256i) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shuffle_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpermilps, imm8 = 9))] -#[rustc_args_required_const(1)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_shuffle_epi32(a: __m256i, imm8: i32) -> __m256i { - // simd_shuffleX requires that its selector parameter be made up of - // constant values, but we can't enforce that here. In spirit, we need - // to write a `match` on all possible values of a byte, and for each value, - // hard-code the correct `simd_shuffleX` call using only constants. We - // then hope for LLVM to do the rest. - // - // Of course, that's... awful. So we try to use macros to do it for us. - let imm8 = (imm8 & 0xFF) as u8; - - let a = a.as_i32x8(); - macro_rules! shuffle_done { - ($x01:expr, $x23:expr, $x45:expr, $x67:expr) => { - simd_shuffle8( - a, - a, - [ - $x01, - $x23, - $x45, - $x67, - 4 + $x01, - 4 + $x23, - 4 + $x45, - 4 + $x67, - ], - ) - }; - } - macro_rules! shuffle_x67 { - ($x01:expr, $x23:expr, $x45:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle_done!($x01, $x23, $x45, 0), - 0b01 => shuffle_done!($x01, $x23, $x45, 1), - 0b10 => shuffle_done!($x01, $x23, $x45, 2), - _ => shuffle_done!($x01, $x23, $x45, 3), - } - }; - } - macro_rules! shuffle_x45 { - ($x01:expr, $x23:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle_x67!($x01, $x23, 0), - 0b01 => shuffle_x67!($x01, $x23, 1), - 0b10 => shuffle_x67!($x01, $x23, 2), - _ => shuffle_x67!($x01, $x23, 3), - } - }; - } - macro_rules! shuffle_x23 { - ($x01:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle_x45!($x01, 0), - 0b01 => shuffle_x45!($x01, 1), - 0b10 => shuffle_x45!($x01, 2), - _ => shuffle_x45!($x01, 3), - } - }; - } - let r: i32x8 = match imm8 & 0b11 { - 0b00 => shuffle_x23!(0), - 0b01 => shuffle_x23!(1), - 0b10 => shuffle_x23!(2), - _ => shuffle_x23!(3), - }; +#[cfg_attr(test, assert_instr(vpermilps, MASK = 9))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_shuffle_epi32(a: __m256i) -> __m256i { + static_assert_imm8!(MASK); + let r: i32x8 = simd_shuffle8!( + a.as_i32x8(), + a.as_i32x8(), + [ + MASK as u32 & 0b11, + (MASK as u32 >> 2) & 0b11, + (MASK as u32 >> 4) & 0b11, + (MASK as u32 >> 6) & 0b11, + (MASK as u32 & 0b11) + 4, + ((MASK as u32 >> 2) & 0b11) + 4, + ((MASK as u32 >> 4) & 0b11) + 4, + ((MASK as u32 >> 6) & 0b11) + 4, + ], + ); transmute(r) } @@ -2734,57 +2376,34 @@ pub unsafe fn _mm256_shuffle_epi32(a: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shufflehi_epi16) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpshufhw, imm8 = 9))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(vpshufhw, IMM8 = 9))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_shufflehi_epi16(a: __m256i, imm8: i32) -> __m256i { - let imm8 = (imm8 & 0xFF) as u8; +pub unsafe fn _mm256_shufflehi_epi16(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i16x16(); - macro_rules! shuffle_done { - ($x01:expr, $x23:expr, $x45:expr, $x67:expr) => { - #[rustfmt::skip] - simd_shuffle16(a, a, [ - 0, 1, 2, 3, 4+$x01, 4+$x23, 4+$x45, 4+$x67, - 8, 9, 10, 11, 12+$x01, 12+$x23, 12+$x45, 12+$x67 - ]); - }; - } - macro_rules! shuffle_x67 { - ($x01:expr, $x23:expr, $x45:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle_done!($x01, $x23, $x45, 0), - 0b01 => shuffle_done!($x01, $x23, $x45, 1), - 0b10 => shuffle_done!($x01, $x23, $x45, 2), - _ => shuffle_done!($x01, $x23, $x45, 3), - } - }; - } - macro_rules! shuffle_x45 { - ($x01:expr, $x23:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle_x67!($x01, $x23, 0), - 0b01 => shuffle_x67!($x01, $x23, 1), - 0b10 => shuffle_x67!($x01, $x23, 2), - _ => shuffle_x67!($x01, $x23, 3), - } - }; - } - macro_rules! shuffle_x23 { - ($x01:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle_x45!($x01, 0), - 0b01 => shuffle_x45!($x01, 1), - 0b10 => shuffle_x45!($x01, 2), - _ => shuffle_x45!($x01, 3), - } - }; - } - let r: i16x16 = match imm8 & 0b11 { - 0b00 => shuffle_x23!(0), - 0b01 => shuffle_x23!(1), - 0b10 => shuffle_x23!(2), - _ => shuffle_x23!(3), - }; + let r: i16x16 = simd_shuffle16!( + a, + a, + [ + 0, + 1, + 2, + 3, + 4 + (IMM8 as u32 & 0b11), + 4 + ((IMM8 as u32 >> 2) & 0b11), + 4 + ((IMM8 as u32 >> 4) & 0b11), + 4 + ((IMM8 as u32 >> 6) & 0b11), + 8, + 9, + 10, + 11, + 12 + (IMM8 as u32 & 0b11), + 12 + ((IMM8 as u32 >> 2) & 0b11), + 12 + ((IMM8 as u32 >> 4) & 0b11), + 12 + ((IMM8 as u32 >> 6) & 0b11), + ], + ); transmute(r) } @@ -2795,57 +2414,34 @@ pub unsafe fn _mm256_shufflehi_epi16(a: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shufflelo_epi16) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpshuflw, imm8 = 9))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(vpshuflw, IMM8 = 9))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_shufflelo_epi16(a: __m256i, imm8: i32) -> __m256i { - let imm8 = (imm8 & 0xFF) as u8; +pub unsafe fn _mm256_shufflelo_epi16(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i16x16(); - macro_rules! shuffle_done { - ($x01: expr, $x23: expr, $x45: expr, $x67: expr) => { - #[rustfmt::skip] - simd_shuffle16(a, a, [ - 0+$x01, 0+$x23, 0+$x45, 0+$x67, 4, 5, 6, 7, - 8+$x01, 8+$x23, 8+$x45, 8+$x67, 12, 13, 14, 15, - ]); - }; - } - macro_rules! shuffle_x67 { - ($x01:expr, $x23:expr, $x45:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle_done!($x01, $x23, $x45, 0), - 0b01 => shuffle_done!($x01, $x23, $x45, 1), - 0b10 => shuffle_done!($x01, $x23, $x45, 2), - _ => shuffle_done!($x01, $x23, $x45, 3), - } - }; - } - macro_rules! shuffle_x45 { - ($x01:expr, $x23:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle_x67!($x01, $x23, 0), - 0b01 => shuffle_x67!($x01, $x23, 1), - 0b10 => shuffle_x67!($x01, $x23, 2), - _ => shuffle_x67!($x01, $x23, 3), - } - }; - } - macro_rules! shuffle_x23 { - ($x01:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle_x45!($x01, 0), - 0b01 => shuffle_x45!($x01, 1), - 0b10 => shuffle_x45!($x01, 2), - _ => shuffle_x45!($x01, 3), - } - }; - } - let r: i16x16 = match imm8 & 0b11 { - 0b00 => shuffle_x23!(0), - 0b01 => shuffle_x23!(1), - 0b10 => shuffle_x23!(2), - _ => shuffle_x23!(3), - }; + let r: i16x16 = simd_shuffle16!( + a, + a, + [ + 0 + (IMM8 as u32 & 0b11), + 0 + ((IMM8 as u32 >> 2) & 0b11), + 0 + ((IMM8 as u32 >> 4) & 0b11), + 0 + ((IMM8 as u32 >> 6) & 0b11), + 4, + 5, + 6, + 7, + 8 + (IMM8 as u32 & 0b11), + 8 + ((IMM8 as u32 >> 2) & 0b11), + 8 + ((IMM8 as u32 >> 4) & 0b11), + 8 + ((IMM8 as u32 >> 6) & 0b11), + 12, + 13, + 14, + 15, + ], + ); transmute(r) } @@ -2924,40 +2520,46 @@ pub unsafe fn _mm256_sll_epi64(a: __m256i, count: __m128i) -> __m256i { transmute(psllq(a.as_i64x4(), count.as_i64x2())) } -/// Shifts packed 16-bit integers in `a` left by `imm8` while +/// Shifts packed 16-bit integers in `a` left by `IMM8` while /// shifting in zeros, return the results; /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_slli_epi16) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpsllw))] +#[cfg_attr(test, assert_instr(vpsllw, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_slli_epi16(a: __m256i, imm8: i32) -> __m256i { - transmute(pslliw(a.as_i16x16(), imm8)) +pub unsafe fn _mm256_slli_epi16(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(pslliw(a.as_i16x16(), IMM8)) } -/// Shifts packed 32-bit integers in `a` left by `imm8` while +/// Shifts packed 32-bit integers in `a` left by `IMM8` while /// shifting in zeros, return the results; /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_slli_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpslld))] +#[cfg_attr(test, assert_instr(vpslld, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_slli_epi32(a: __m256i, imm8: i32) -> __m256i { - transmute(psllid(a.as_i32x8(), imm8)) +pub unsafe fn _mm256_slli_epi32(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(psllid(a.as_i32x8(), IMM8)) } -/// Shifts packed 64-bit integers in `a` left by `imm8` while +/// Shifts packed 64-bit integers in `a` left by `IMM8` while /// shifting in zeros, return the results; /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_slli_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpsllq))] +#[cfg_attr(test, assert_instr(vpsllq, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_slli_epi64(a: __m256i, imm8: i32) -> __m256i { - transmute(pslliq(a.as_i64x4(), imm8)) +pub unsafe fn _mm256_slli_epi64(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(pslliq(a.as_i64x4(), IMM8)) } /// Shifts 128-bit lanes in `a` left by `imm8` bytes while shifting in zeros. @@ -2965,17 +2567,12 @@ pub unsafe fn _mm256_slli_epi64(a: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_slli_si256) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpslldq, imm8 = 3))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(vpslldq, IMM8 = 3))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_slli_si256(a: __m256i, imm8: i32) -> __m256i { - let a = a.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vpslldq(a, $imm8) - }; - } - transmute(constify_imm8!(imm8 * 8, call)) +pub unsafe fn _mm256_slli_si256(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + _mm256_bslli_epi128::(a) } /// Shifts 128-bit lanes in `a` left by `imm8` bytes while shifting in zeros. @@ -2983,17 +2580,60 @@ pub unsafe fn _mm256_slli_si256(a: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_bslli_epi128) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpslldq, imm8 = 3))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(vpslldq, IMM8 = 3))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_bslli_epi128(a: __m256i, imm8: i32) -> __m256i { - let a = a.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vpslldq(a, $imm8) - }; +pub unsafe fn _mm256_bslli_epi128(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + const fn mask(shift: i32, i: u32) -> u32 { + let shift = shift as u32 & 0xff; + if shift > 15 || i % 16 < shift { + 0 + } else { + 32 + (i - shift) + } } - transmute(constify_imm8!(imm8 * 8, call)) + let a = a.as_i8x32(); + let zero = _mm256_setzero_si256().as_i8x32(); + let r: i8x32 = simd_shuffle32!( + zero, + a, + [ + mask(IMM8, 0), + mask(IMM8, 1), + mask(IMM8, 2), + mask(IMM8, 3), + mask(IMM8, 4), + mask(IMM8, 5), + mask(IMM8, 6), + mask(IMM8, 7), + mask(IMM8, 8), + mask(IMM8, 9), + mask(IMM8, 10), + mask(IMM8, 11), + mask(IMM8, 12), + mask(IMM8, 13), + mask(IMM8, 14), + mask(IMM8, 15), + mask(IMM8, 16), + mask(IMM8, 17), + mask(IMM8, 18), + mask(IMM8, 19), + mask(IMM8, 20), + mask(IMM8, 21), + mask(IMM8, 22), + mask(IMM8, 23), + mask(IMM8, 24), + mask(IMM8, 25), + mask(IMM8, 26), + mask(IMM8, 27), + mask(IMM8, 28), + mask(IMM8, 29), + mask(IMM8, 30), + mask(IMM8, 31), + ], + ); + transmute(r) } /// Shifts packed 32-bit integers in `a` left by the amount @@ -3072,28 +2712,32 @@ pub unsafe fn _mm256_sra_epi32(a: __m256i, count: __m128i) -> __m256i { transmute(psrad(a.as_i32x8(), count.as_i32x4())) } -/// Shifts packed 16-bit integers in `a` right by `imm8` while +/// Shifts packed 16-bit integers in `a` right by `IMM8` while /// shifting in sign bits. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_srai_epi16) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpsraw))] +#[cfg_attr(test, assert_instr(vpsraw, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_srai_epi16(a: __m256i, imm8: i32) -> __m256i { - transmute(psraiw(a.as_i16x16(), imm8)) +pub unsafe fn _mm256_srai_epi16(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(psraiw(a.as_i16x16(), IMM8)) } -/// Shifts packed 32-bit integers in `a` right by `imm8` while +/// Shifts packed 32-bit integers in `a` right by `IMM8` while /// shifting in sign bits. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_srai_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpsrad))] +#[cfg_attr(test, assert_instr(vpsrad, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_srai_epi32(a: __m256i, imm8: i32) -> __m256i { - transmute(psraid(a.as_i32x8(), imm8)) +pub unsafe fn _mm256_srai_epi32(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(psraid(a.as_i32x8(), IMM8)) } /// Shifts packed 32-bit integers in `a` right by the amount specified by the @@ -3125,17 +2769,12 @@ pub unsafe fn _mm256_srav_epi32(a: __m256i, count: __m256i) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_srli_si256) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpsrldq, imm8 = 3))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(vpsrldq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_srli_si256(a: __m256i, imm8: i32) -> __m256i { - let a = a.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vpsrldq(a, $imm8) - }; - } - transmute(constify_imm8!(imm8 * 8, call)) +pub unsafe fn _mm256_srli_si256(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + _mm256_bsrli_epi128::(a) } /// Shifts 128-bit lanes in `a` right by `imm8` bytes while shifting in zeros. @@ -3143,17 +2782,145 @@ pub unsafe fn _mm256_srli_si256(a: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_bsrli_epi128) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpsrldq, imm8 = 3))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(vpsrldq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_bsrli_epi128(a: __m256i, imm8: i32) -> __m256i { - let a = a.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vpsrldq(a, $imm8) - }; - } - transmute(constify_imm8!(imm8 * 8, call)) +pub unsafe fn _mm256_bsrli_epi128(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_i8x32(); + let zero = _mm256_setzero_si256().as_i8x32(); + let r: i8x32 = match IMM8 % 16 { + 0 => simd_shuffle32!( + a, + zero, + [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, + 23, 24, 25, 26, 27, 28, 29, 30, 31, + ], + ), + 1 => simd_shuffle32!( + a, + zero, + [ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 32, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, + ], + ), + 2 => simd_shuffle32!( + a, + zero, + [ + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 32, 32, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 32, + ], + ), + 3 => simd_shuffle32!( + a, + zero, + [ + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 32, 32, 32, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 32, 32, + ], + ), + 4 => simd_shuffle32!( + a, + zero, + [ + 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 32, 32, 32, 32, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 32, 32, 32, + ], + ), + 5 => simd_shuffle32!( + a, + zero, + [ + 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 32, 32, 32, 32, 32, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 32, 32, 32, 32, + ], + ), + 6 => simd_shuffle32!( + a, + zero, + [ + 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 32, 32, 32, 32, 32, 32, 22, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 32, 32, 32, 32, 32, 32, + ], + ), + 7 => simd_shuffle32!( + a, + zero, + [ + 7, 8, 9, 10, 11, 12, 13, 14, 15, 32, 32, 32, 32, 32, 32, 32, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 32, 32, 32, 32, 32, 32, 32, + ], + ), + 8 => simd_shuffle32!( + a, + zero, + [ + 8, 9, 10, 11, 12, 13, 14, 15, 32, 32, 32, 32, 32, 32, 32, 32, 24, 25, 26, 27, 28, + 29, 30, 31, 32, 32, 32, 32, 32, 32, 32, 32, + ], + ), + 9 => simd_shuffle32!( + a, + zero, + [ + 9, 10, 11, 12, 13, 14, 15, 32, 32, 32, 32, 32, 32, 32, 32, 32, 25, 26, 27, 28, 29, + 30, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, + ], + ), + 10 => simd_shuffle32!( + a, + zero, + [ + 10, 11, 12, 13, 14, 15, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 26, 27, 28, 29, 30, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + ], + ), + 11 => simd_shuffle32!( + a, + zero, + [ + 11, 12, 13, 14, 15, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 27, 28, 29, 30, 31, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + ], + ), + 12 => simd_shuffle32!( + a, + zero, + [ + 12, 13, 14, 15, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 28, 29, 30, 31, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + ], + ), + 13 => simd_shuffle32!( + a, + zero, + [ + 13, 14, 15, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 29, 30, 31, 32, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + ], + ), + 14 => simd_shuffle32!( + a, + zero, + [ + 14, 15, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 30, 31, 32, 32, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + ], + ), + 15 => simd_shuffle32!( + a, + zero, + [ + 14, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 31, 32, 32, 32, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + ], + ), + _ => zero, + }; + transmute(r) } /// Shifts packed 16-bit integers in `a` right by `count` while shifting in @@ -3192,40 +2959,46 @@ pub unsafe fn _mm256_srl_epi64(a: __m256i, count: __m128i) -> __m256i { transmute(psrlq(a.as_i64x4(), count.as_i64x2())) } -/// Shifts packed 16-bit integers in `a` right by `imm8` while shifting in +/// Shifts packed 16-bit integers in `a` right by `IMM8` while shifting in /// zeros /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_srli_epi16) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpsrlw))] +#[cfg_attr(test, assert_instr(vpsrlw, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_srli_epi16(a: __m256i, imm8: i32) -> __m256i { - transmute(psrliw(a.as_i16x16(), imm8)) +pub unsafe fn _mm256_srli_epi16(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(psrliw(a.as_i16x16(), IMM8)) } -/// Shifts packed 32-bit integers in `a` right by `imm8` while shifting in +/// Shifts packed 32-bit integers in `a` right by `IMM8` while shifting in /// zeros /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_srli_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpsrld))] +#[cfg_attr(test, assert_instr(vpsrld, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_srli_epi32(a: __m256i, imm8: i32) -> __m256i { - transmute(psrlid(a.as_i32x8(), imm8)) +pub unsafe fn _mm256_srli_epi32(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(psrlid(a.as_i32x8(), IMM8)) } -/// Shifts packed 64-bit integers in `a` right by `imm8` while shifting in +/// Shifts packed 64-bit integers in `a` right by `IMM8` while shifting in /// zeros /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_srli_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpsrlq))] +#[cfg_attr(test, assert_instr(vpsrlq, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_srli_epi64(a: __m256i, imm8: i32) -> __m256i { - transmute(psrliq(a.as_i64x4(), imm8)) +pub unsafe fn _mm256_srli_epi64(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(psrliq(a.as_i64x4(), IMM8)) } /// Shifts packed 32-bit integers in `a` right by the amount specified by @@ -3289,7 +3062,7 @@ pub unsafe fn _mm256_sub_epi16(a: __m256i, b: __m256i) -> __m256i { transmute(simd_sub(a.as_i16x16(), b.as_i16x16())) } -/// Subtract packed 32-bit integers in `b` from packed 16-bit integers in `a` +/// Subtract packed 32-bit integers in `b` from packed 32-bit integers in `a` /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_sub_epi32) #[inline] @@ -3300,7 +3073,7 @@ pub unsafe fn _mm256_sub_epi32(a: __m256i, b: __m256i) -> __m256i { transmute(simd_sub(a.as_i32x8(), b.as_i32x8())) } -/// Subtract packed 64-bit integers in `b` from packed 16-bit integers in `a` +/// Subtract packed 64-bit integers in `b` from packed 64-bit integers in `a` /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_sub_epi64) #[inline] @@ -3311,7 +3084,7 @@ pub unsafe fn _mm256_sub_epi64(a: __m256i, b: __m256i) -> __m256i { transmute(simd_sub(a.as_i64x4(), b.as_i64x4())) } -/// Subtract packed 8-bit integers in `b` from packed 16-bit integers in `a` +/// Subtract packed 8-bit integers in `b` from packed 8-bit integers in `a` /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_sub_epi8) #[inline] @@ -3415,7 +3188,7 @@ pub unsafe fn _mm256_subs_epu8(a: __m256i, b: __m256i) -> __m256i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpackhi_epi8(a: __m256i, b: __m256i) -> __m256i { #[rustfmt::skip] - let r: i8x32 = simd_shuffle32(a.as_i8x32(), b.as_i8x32(), [ + let r: i8x32 = simd_shuffle32!(a.as_i8x32(), b.as_i8x32(), [ 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47, 24, 56, 25, 57, 26, 58, 27, 59, @@ -3468,7 +3241,7 @@ pub unsafe fn _mm256_unpackhi_epi8(a: __m256i, b: __m256i) -> __m256i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpacklo_epi8(a: __m256i, b: __m256i) -> __m256i { #[rustfmt::skip] - let r: i8x32 = simd_shuffle32(a.as_i8x32(), b.as_i8x32(), [ + let r: i8x32 = simd_shuffle32!(a.as_i8x32(), b.as_i8x32(), [ 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, 16, 48, 17, 49, 18, 50, 19, 51, @@ -3516,7 +3289,7 @@ pub unsafe fn _mm256_unpacklo_epi8(a: __m256i, b: __m256i) -> __m256i { #[cfg_attr(test, assert_instr(vpunpckhwd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpackhi_epi16(a: __m256i, b: __m256i) -> __m256i { - let r: i16x16 = simd_shuffle16( + let r: i16x16 = simd_shuffle16!( a.as_i16x16(), b.as_i16x16(), [4, 20, 5, 21, 6, 22, 7, 23, 12, 28, 13, 29, 14, 30, 15, 31], @@ -3564,7 +3337,7 @@ pub unsafe fn _mm256_unpackhi_epi16(a: __m256i, b: __m256i) -> __m256i { #[cfg_attr(test, assert_instr(vpunpcklwd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpacklo_epi16(a: __m256i, b: __m256i) -> __m256i { - let r: i16x16 = simd_shuffle16( + let r: i16x16 = simd_shuffle16!( a.as_i16x16(), b.as_i16x16(), [0, 16, 1, 17, 2, 18, 3, 19, 8, 24, 9, 25, 10, 26, 11, 27], @@ -3605,7 +3378,7 @@ pub unsafe fn _mm256_unpacklo_epi16(a: __m256i, b: __m256i) -> __m256i { #[cfg_attr(test, assert_instr(vunpckhps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpackhi_epi32(a: __m256i, b: __m256i) -> __m256i { - let r: i32x8 = simd_shuffle8(a.as_i32x8(), b.as_i32x8(), [2, 10, 3, 11, 6, 14, 7, 15]); + let r: i32x8 = simd_shuffle8!(a.as_i32x8(), b.as_i32x8(), [2, 10, 3, 11, 6, 14, 7, 15]); transmute(r) } @@ -3642,7 +3415,7 @@ pub unsafe fn _mm256_unpackhi_epi32(a: __m256i, b: __m256i) -> __m256i { #[cfg_attr(test, assert_instr(vunpcklps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpacklo_epi32(a: __m256i, b: __m256i) -> __m256i { - let r: i32x8 = simd_shuffle8(a.as_i32x8(), b.as_i32x8(), [0, 8, 1, 9, 4, 12, 5, 13]); + let r: i32x8 = simd_shuffle8!(a.as_i32x8(), b.as_i32x8(), [0, 8, 1, 9, 4, 12, 5, 13]); transmute(r) } @@ -3679,7 +3452,7 @@ pub unsafe fn _mm256_unpacklo_epi32(a: __m256i, b: __m256i) -> __m256i { #[cfg_attr(test, assert_instr(vunpckhpd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpackhi_epi64(a: __m256i, b: __m256i) -> __m256i { - let r: i64x4 = simd_shuffle4(a.as_i64x4(), b.as_i64x4(), [1, 5, 3, 7]); + let r: i64x4 = simd_shuffle4!(a.as_i64x4(), b.as_i64x4(), [1, 5, 3, 7]); transmute(r) } @@ -3716,7 +3489,7 @@ pub unsafe fn _mm256_unpackhi_epi64(a: __m256i, b: __m256i) -> __m256i { #[cfg_attr(test, assert_instr(vunpcklpd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpacklo_epi64(a: __m256i, b: __m256i) -> __m256i { - let r: i64x4 = simd_shuffle4(a.as_i64x4(), b.as_i64x4(), [0, 4, 2, 6]); + let r: i64x4 = simd_shuffle4!(a.as_i64x4(), b.as_i64x4(), [0, 4, 2, 6]); transmute(r) } @@ -3732,49 +3505,49 @@ pub unsafe fn _mm256_xor_si256(a: __m256i, b: __m256i) -> __m256i { transmute(simd_xor(a.as_i64x4(), b.as_i64x4())) } -/// Extracts an 8-bit integer from `a`, selected with `imm8`. Returns a 32-bit +/// Extracts an 8-bit integer from `a`, selected with `INDEX`. Returns a 32-bit /// integer containing the zero-extended integer data. /// -/// See [LLVM commit D20468][https://reviews.llvm.org/D20468]. +/// See [LLVM commit D20468](https://reviews.llvm.org/D20468). /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_extract_epi8) #[inline] #[target_feature(enable = "avx2")] // This intrinsic has no corresponding instruction. -#[rustc_args_required_const(1)] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_extract_epi8(a: __m256i, imm8: i32) -> i8 { - let imm8 = (imm8 & 31) as u32; - simd_extract(a.as_i8x32(), imm8) +pub unsafe fn _mm256_extract_epi8(a: __m256i) -> i32 { + static_assert_imm5!(INDEX); + simd_extract::<_, u8>(a.as_u8x32(), INDEX as u32) as i32 } -/// Extracts a 16-bit integer from `a`, selected with `imm8`. Returns a 32-bit +/// Extracts a 16-bit integer from `a`, selected with `INDEX`. Returns a 32-bit /// integer containing the zero-extended integer data. /// -/// See [LLVM commit D20468][https://reviews.llvm.org/D20468]. +/// See [LLVM commit D20468](https://reviews.llvm.org/D20468). /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_extract_epi16) #[inline] #[target_feature(enable = "avx2")] // This intrinsic has no corresponding instruction. -#[rustc_args_required_const(1)] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_extract_epi16(a: __m256i, imm8: i32) -> i16 { - let imm8 = (imm8 & 15) as u32; - simd_extract(a.as_i16x16(), imm8) +pub unsafe fn _mm256_extract_epi16(a: __m256i) -> i32 { + static_assert_imm4!(INDEX); + simd_extract::<_, u16>(a.as_u16x16(), INDEX as u32) as i32 } -/// Extracts a 32-bit integer from `a`, selected with `imm8`. +/// Extracts a 32-bit integer from `a`, selected with `INDEX`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_extract_epi32) #[inline] #[target_feature(enable = "avx2")] // This intrinsic has no corresponding instruction. -#[rustc_args_required_const(1)] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_extract_epi32(a: __m256i, imm8: i32) -> i32 { - let imm8 = (imm8 & 7) as u32; - simd_extract(a.as_i32x8(), imm8) +pub unsafe fn _mm256_extract_epi32(a: __m256i) -> i32 { + static_assert_imm3!(INDEX); + simd_extract(a.as_i32x8(), INDEX as u32) } /// Returns the first element of the input vector of `[4 x double]`. @@ -3869,8 +3642,6 @@ extern "C" { fn pminud(a: u32x8, b: u32x8) -> u32x8; #[link_name = "llvm.x86.avx2.pminu.b"] fn pminub(a: u8x32, b: u8x32) -> u8x32; - #[link_name = "llvm.x86.avx2.pmovmskb"] - fn pmovmskb(a: i8x32) -> i32; #[link_name = "llvm.x86.avx2.mpsadbw"] fn mpsadbw(a: u8x32, b: u8x32, imm8: i32) -> u16x16; #[link_name = "llvm.x86.avx2.pmulhu.w"] @@ -4037,7 +3808,7 @@ extern "C" { #[cfg(test)] mod tests { - use std; + use stdarch_test::simd_test; use crate::core_arch::x86::*; @@ -4046,14 +3817,14 @@ mod tests { unsafe fn test_mm256_abs_epi32() { #[rustfmt::skip] let a = _mm256_setr_epi32( - 0, 1, -1, std::i32::MAX, - std::i32::MIN, 100, -100, -32, + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, ); let r = _mm256_abs_epi32(a); #[rustfmt::skip] let e = _mm256_setr_epi32( - 0, 1, 1, std::i32::MAX, - std::i32::MAX.wrapping_add(1), 100, 100, 32, + 0, 1, 1, i32::MAX, + i32::MAX.wrapping_add(1), 100, 100, 32, ); assert_eq_m256i(r, e); } @@ -4063,13 +3834,13 @@ mod tests { #[rustfmt::skip] let a = _mm256_setr_epi16( 0, 1, -1, 2, -2, 3, -3, 4, - -4, 5, -5, std::i16::MAX, std::i16::MIN, 100, -100, -32, + -4, 5, -5, i16::MAX, i16::MIN, 100, -100, -32, ); let r = _mm256_abs_epi16(a); #[rustfmt::skip] let e = _mm256_setr_epi16( 0, 1, 1, 2, 2, 3, 3, 4, - 4, 5, 5, std::i16::MAX, std::i16::MAX.wrapping_add(1), 100, 100, 32, + 4, 5, 5, i16::MAX, i16::MAX.wrapping_add(1), 100, 100, 32, ); assert_eq_m256i(r, e); } @@ -4079,17 +3850,17 @@ mod tests { #[rustfmt::skip] let a = _mm256_setr_epi8( 0, 1, -1, 2, -2, 3, -3, 4, - -4, 5, -5, std::i8::MAX, std::i8::MIN, 100, -100, -32, + -4, 5, -5, i8::MAX, i8::MIN, 100, -100, -32, 0, 1, -1, 2, -2, 3, -3, 4, - -4, 5, -5, std::i8::MAX, std::i8::MIN, 100, -100, -32, + -4, 5, -5, i8::MAX, i8::MIN, 100, -100, -32, ); let r = _mm256_abs_epi8(a); #[rustfmt::skip] let e = _mm256_setr_epi8( 0, 1, 1, 2, 2, 3, 3, 4, - 4, 5, 5, std::i8::MAX, std::i8::MAX.wrapping_add(1), 100, 100, 32, + 4, 5, 5, i8::MAX, i8::MAX.wrapping_add(1), 100, 100, 32, 0, 1, 1, 2, 2, 3, 3, 4, - 4, 5, 5, std::i8::MAX, std::i8::MAX.wrapping_add(1), 100, 100, 32, + 4, 5, 5, i8::MAX, i8::MAX.wrapping_add(1), 100, 100, 32, ); assert_eq_m256i(r, e); } @@ -4340,10 +4111,10 @@ mod tests { unsafe fn test_mm_blend_epi32() { let (a, b) = (_mm_set1_epi32(3), _mm_set1_epi32(9)); let e = _mm_setr_epi32(9, 3, 3, 3); - let r = _mm_blend_epi32(a, b, 0x01 as i32); + let r = _mm_blend_epi32::<0x01>(a, b); assert_eq_m128i(r, e); - let r = _mm_blend_epi32(b, a, 0x0E as i32); + let r = _mm_blend_epi32::<0x0E>(b, a); assert_eq_m128i(r, e); } @@ -4351,15 +4122,15 @@ mod tests { unsafe fn test_mm256_blend_epi32() { let (a, b) = (_mm256_set1_epi32(3), _mm256_set1_epi32(9)); let e = _mm256_setr_epi32(9, 3, 3, 3, 3, 3, 3, 3); - let r = _mm256_blend_epi32(a, b, 0x01 as i32); + let r = _mm256_blend_epi32::<0x01>(a, b); assert_eq_m256i(r, e); let e = _mm256_setr_epi32(3, 9, 3, 3, 3, 3, 3, 9); - let r = _mm256_blend_epi32(a, b, 0x82 as i32); + let r = _mm256_blend_epi32::<0x82>(a, b); assert_eq_m256i(r, e); let e = _mm256_setr_epi32(3, 3, 9, 9, 9, 9, 9, 3); - let r = _mm256_blend_epi32(a, b, 0x7C as i32); + let r = _mm256_blend_epi32::<0x7C>(a, b); assert_eq_m256i(r, e); } @@ -4367,32 +4138,32 @@ mod tests { unsafe fn test_mm256_blend_epi16() { let (a, b) = (_mm256_set1_epi16(3), _mm256_set1_epi16(9)); let e = _mm256_setr_epi16(9, 3, 3, 3, 3, 3, 3, 3, 9, 3, 3, 3, 3, 3, 3, 3); - let r = _mm256_blend_epi16(a, b, 0x01 as i32); + let r = _mm256_blend_epi16::<0x01>(a, b); assert_eq_m256i(r, e); - let r = _mm256_blend_epi16(b, a, 0xFE as i32); + let r = _mm256_blend_epi16::<0xFE>(b, a); assert_eq_m256i(r, e); } #[simd_test(enable = "avx2")] unsafe fn test_mm256_blendv_epi8() { let (a, b) = (_mm256_set1_epi8(4), _mm256_set1_epi8(2)); - let mask = _mm256_insert_epi8(_mm256_set1_epi8(0), -1, 2); - let e = _mm256_insert_epi8(_mm256_set1_epi8(4), 2, 2); + let mask = _mm256_insert_epi8::<2>(_mm256_set1_epi8(0), -1); + let e = _mm256_insert_epi8::<2>(_mm256_set1_epi8(4), 2); let r = _mm256_blendv_epi8(a, b, mask); assert_eq_m256i(r, e); } #[simd_test(enable = "avx2")] unsafe fn test_mm_broadcastb_epi8() { - let a = _mm_insert_epi8(_mm_set1_epi8(0x00), 0x2a, 0); + let a = _mm_insert_epi8::<0>(_mm_set1_epi8(0x00), 0x2a); let res = _mm_broadcastb_epi8(a); assert_eq_m128i(res, _mm_set1_epi8(0x2a)); } #[simd_test(enable = "avx2")] unsafe fn test_mm256_broadcastb_epi8() { - let a = _mm_insert_epi8(_mm_set1_epi8(0x00), 0x2a, 0); + let a = _mm_insert_epi8::<0>(_mm_set1_epi8(0x00), 0x2a); let res = _mm256_broadcastb_epi8(a); assert_eq_m256i(res, _mm256_set1_epi8(0x2a)); } @@ -4468,14 +4239,14 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm_broadcastw_epi16() { - let a = _mm_insert_epi16(_mm_set1_epi16(0x2a), 0x22b, 0); + let a = _mm_insert_epi16::<0>(_mm_set1_epi16(0x2a), 0x22b); let res = _mm_broadcastw_epi16(a); assert_eq_m128i(res, _mm_set1_epi16(0x22b)); } #[simd_test(enable = "avx2")] unsafe fn test_mm256_broadcastw_epi16() { - let a = _mm_insert_epi16(_mm_set1_epi16(0x2a), 0x22b, 0); + let a = _mm_insert_epi16::<0>(_mm_set1_epi16(0x2a), 0x22b); let res = _mm256_broadcastw_epi16(a); assert_eq_m256i(res, _mm256_set1_epi16(0x22b)); } @@ -4497,7 +4268,7 @@ mod tests { 7, 6, 5, 4, 3, 2, 1, 0, ); let r = _mm256_cmpeq_epi8(a, b); - assert_eq_m256i(r, _mm256_insert_epi8(_mm256_set1_epi8(0), !0, 2)); + assert_eq_m256i(r, _mm256_insert_epi8::<2>(_mm256_set1_epi8(0), !0)); } #[simd_test(enable = "avx2")] @@ -4513,7 +4284,7 @@ mod tests { 7, 6, 5, 4, 3, 2, 1, 0, ); let r = _mm256_cmpeq_epi16(a, b); - assert_eq_m256i(r, _mm256_insert_epi16(_mm256_set1_epi16(0), !0, 2)); + assert_eq_m256i(r, _mm256_insert_epi16::<2>(_mm256_set1_epi16(0), !0)); } #[simd_test(enable = "avx2")] @@ -4522,7 +4293,7 @@ mod tests { let b = _mm256_setr_epi32(7, 6, 2, 4, 3, 2, 1, 0); let r = _mm256_cmpeq_epi32(a, b); let e = _mm256_set1_epi32(0); - let e = _mm256_insert_epi32(e, !0, 2); + let e = _mm256_insert_epi32::<2>(e, !0); assert_eq_m256i(r, e); } @@ -4531,39 +4302,39 @@ mod tests { let a = _mm256_setr_epi64x(0, 1, 2, 3); let b = _mm256_setr_epi64x(3, 2, 2, 0); let r = _mm256_cmpeq_epi64(a, b); - assert_eq_m256i(r, _mm256_insert_epi64(_mm256_set1_epi64x(0), !0, 2)); + assert_eq_m256i(r, _mm256_insert_epi64::<2>(_mm256_set1_epi64x(0), !0)); } #[simd_test(enable = "avx2")] unsafe fn test_mm256_cmpgt_epi8() { - let a = _mm256_insert_epi8(_mm256_set1_epi8(0), 5, 0); + let a = _mm256_insert_epi8::<0>(_mm256_set1_epi8(0), 5); let b = _mm256_set1_epi8(0); let r = _mm256_cmpgt_epi8(a, b); - assert_eq_m256i(r, _mm256_insert_epi8(_mm256_set1_epi8(0), !0, 0)); + assert_eq_m256i(r, _mm256_insert_epi8::<0>(_mm256_set1_epi8(0), !0)); } #[simd_test(enable = "avx2")] unsafe fn test_mm256_cmpgt_epi16() { - let a = _mm256_insert_epi16(_mm256_set1_epi16(0), 5, 0); + let a = _mm256_insert_epi16::<0>(_mm256_set1_epi16(0), 5); let b = _mm256_set1_epi16(0); let r = _mm256_cmpgt_epi16(a, b); - assert_eq_m256i(r, _mm256_insert_epi16(_mm256_set1_epi16(0), !0, 0)); + assert_eq_m256i(r, _mm256_insert_epi16::<0>(_mm256_set1_epi16(0), !0)); } #[simd_test(enable = "avx2")] unsafe fn test_mm256_cmpgt_epi32() { - let a = _mm256_insert_epi32(_mm256_set1_epi32(0), 5, 0); + let a = _mm256_insert_epi32::<0>(_mm256_set1_epi32(0), 5); let b = _mm256_set1_epi32(0); let r = _mm256_cmpgt_epi32(a, b); - assert_eq_m256i(r, _mm256_insert_epi32(_mm256_set1_epi32(0), !0, 0)); + assert_eq_m256i(r, _mm256_insert_epi32::<0>(_mm256_set1_epi32(0), !0)); } #[simd_test(enable = "avx2")] unsafe fn test_mm256_cmpgt_epi64() { - let a = _mm256_insert_epi64(_mm256_set1_epi64x(0), 5, 0); + let a = _mm256_insert_epi64::<0>(_mm256_set1_epi64x(0), 5); let b = _mm256_set1_epi64x(0); let r = _mm256_cmpgt_epi64(a, b); - assert_eq_m256i(r, _mm256_insert_epi64(_mm256_set1_epi64x(0), !0, 0)); + assert_eq_m256i(r, _mm256_insert_epi64::<0>(_mm256_set1_epi64x(0), !0)); } #[simd_test(enable = "avx2")] @@ -4685,7 +4456,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_extracti128_si256() { let a = _mm256_setr_epi64x(1, 2, 3, 4); - let r = _mm256_extracti128_si256(a, 0b01); + let r = _mm256_extracti128_si256::<1>(a); let e = _mm_setr_epi64x(3, 4); assert_eq_m128i(r, e); } @@ -4711,8 +4482,8 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_hadds_epi16() { let a = _mm256_set1_epi16(2); - let a = _mm256_insert_epi16(a, 0x7fff, 0); - let a = _mm256_insert_epi16(a, 1, 1); + let a = _mm256_insert_epi16::<0>(a, 0x7fff); + let a = _mm256_insert_epi16::<1>(a, 1); let b = _mm256_set1_epi16(4); let r = _mm256_hadds_epi16(a, b); #[rustfmt::skip] @@ -4744,11 +4515,11 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_hsubs_epi16() { let a = _mm256_set1_epi16(2); - let a = _mm256_insert_epi16(a, 0x7fff, 0); - let a = _mm256_insert_epi16(a, -1, 1); + let a = _mm256_insert_epi16::<0>(a, 0x7fff); + let a = _mm256_insert_epi16::<1>(a, -1); let b = _mm256_set1_epi16(4); let r = _mm256_hsubs_epi16(a, b); - let e = _mm256_insert_epi16(_mm256_set1_epi16(0), 0x7FFF, 0); + let e = _mm256_insert_epi16::<0>(_mm256_set1_epi16(0), 0x7FFF); assert_eq_m256i(r, e); } @@ -4765,7 +4536,7 @@ mod tests { unsafe fn test_mm256_inserti128_si256() { let a = _mm256_setr_epi64x(1, 2, 3, 4); let b = _mm_setr_epi64x(7, 8); - let r = _mm256_inserti128_si256(a, b, 0b01); + let r = _mm256_inserti128_si256::<1>(a, b); let e = _mm256_setr_epi64x(1, 2, 7, 8); assert_eq_m256i(r, e); } @@ -4967,7 +4738,7 @@ mod tests { unsafe fn test_mm256_mpsadbw_epu8() { let a = _mm256_set1_epi8(2); let b = _mm256_set1_epi8(4); - let r = _mm256_mpsadbw_epu8(a, b, 0); + let r = _mm256_mpsadbw_epu8::<0>(a, b); let e = _mm256_set1_epi16(8); assert_eq_m256i(r, e); } @@ -5116,7 +4887,7 @@ mod tests { 0, 1, 2, 3, 44, 22, 22, 11, 4, 5, 6, 7, 88, 66, 66, 55, ); - let r = _mm256_shufflehi_epi16(a, 0b00_01_01_11); + let r = _mm256_shufflehi_epi16::<0b00_01_01_11>(a); assert_eq_m256i(r, e); } @@ -5132,7 +4903,7 @@ mod tests { 44, 22, 22, 11, 0, 1, 2, 3, 88, 66, 66, 55, 4, 5, 6, 7, ); - let r = _mm256_shufflelo_epi16(a, 0b00_01_01_11); + let r = _mm256_shufflelo_epi16::<0b00_01_01_11>(a); assert_eq_m256i(r, e); } @@ -5166,7 +4937,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_sll_epi16() { let a = _mm256_set1_epi16(0xFF); - let b = _mm_insert_epi16(_mm_set1_epi16(0), 4, 0); + let b = _mm_insert_epi16::<0>(_mm_set1_epi16(0), 4); let r = _mm256_sll_epi16(a, b); assert_eq_m256i(r, _mm256_set1_epi16(0xFF0)); } @@ -5174,7 +4945,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_sll_epi32() { let a = _mm256_set1_epi32(0xFFFF); - let b = _mm_insert_epi32(_mm_set1_epi32(0), 4, 0); + let b = _mm_insert_epi32::<0>(_mm_set1_epi32(0), 4); let r = _mm256_sll_epi32(a, b); assert_eq_m256i(r, _mm256_set1_epi32(0xFFFF0)); } @@ -5182,7 +4953,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_sll_epi64() { let a = _mm256_set1_epi64x(0xFFFFFFFF); - let b = _mm_insert_epi64(_mm_set1_epi64x(0), 4, 0); + let b = _mm_insert_epi64::<0>(_mm_set1_epi64x(0), 4); let r = _mm256_sll_epi64(a, b); assert_eq_m256i(r, _mm256_set1_epi64x(0xFFFFFFFF0)); } @@ -5190,7 +4961,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_slli_epi16() { assert_eq_m256i( - _mm256_slli_epi16(_mm256_set1_epi16(0xFF), 4), + _mm256_slli_epi16::<4>(_mm256_set1_epi16(0xFF)), _mm256_set1_epi16(0xFF0), ); } @@ -5198,7 +4969,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_slli_epi32() { assert_eq_m256i( - _mm256_slli_epi32(_mm256_set1_epi32(0xFFFF), 4), + _mm256_slli_epi32::<4>(_mm256_set1_epi32(0xFFFF)), _mm256_set1_epi32(0xFFFF0), ); } @@ -5206,7 +4977,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_slli_epi64() { assert_eq_m256i( - _mm256_slli_epi64(_mm256_set1_epi64x(0xFFFFFFFF), 4), + _mm256_slli_epi64::<4>(_mm256_set1_epi64x(0xFFFFFFFF)), _mm256_set1_epi64x(0xFFFFFFFF0), ); } @@ -5214,7 +4985,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_slli_si256() { let a = _mm256_set1_epi64x(0xFFFFFFFF); - let r = _mm256_slli_si256(a, 3); + let r = _mm256_slli_si256::<3>(a); assert_eq_m256i(r, _mm256_set1_epi64x(0xFFFFFFFF000000)); } @@ -5265,7 +5036,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_sra_epi32() { let a = _mm256_set1_epi32(-1); - let b = _mm_insert_epi32(_mm_set1_epi32(0), 1, 0); + let b = _mm_insert_epi32::<0>(_mm_set1_epi32(0), 1); let r = _mm256_sra_epi32(a, b); assert_eq_m256i(r, _mm256_set1_epi32(-1)); } @@ -5273,7 +5044,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_srai_epi16() { assert_eq_m256i( - _mm256_srai_epi16(_mm256_set1_epi16(-1), 1), + _mm256_srai_epi16::<1>(_mm256_set1_epi16(-1)), _mm256_set1_epi16(-1), ); } @@ -5281,7 +5052,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_srai_epi32() { assert_eq_m256i( - _mm256_srai_epi32(_mm256_set1_epi32(-1), 1), + _mm256_srai_epi32::<1>(_mm256_set1_epi32(-1)), _mm256_set1_epi32(-1), ); } @@ -5313,7 +5084,7 @@ mod tests { 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, ); - let r = _mm256_srli_si256(a, 3); + let r = _mm256_srli_si256::<3>(a); #[rustfmt::skip] let e = _mm256_setr_epi8( 4, 5, 6, 7, 8, 9, 10, 11, @@ -5327,7 +5098,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_srl_epi16() { let a = _mm256_set1_epi16(0xFF); - let b = _mm_insert_epi16(_mm_set1_epi16(0), 4, 0); + let b = _mm_insert_epi16::<0>(_mm_set1_epi16(0), 4); let r = _mm256_srl_epi16(a, b); assert_eq_m256i(r, _mm256_set1_epi16(0xF)); } @@ -5335,7 +5106,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_srl_epi32() { let a = _mm256_set1_epi32(0xFFFF); - let b = _mm_insert_epi32(_mm_set1_epi32(0), 4, 0); + let b = _mm_insert_epi32::<0>(_mm_set1_epi32(0), 4); let r = _mm256_srl_epi32(a, b); assert_eq_m256i(r, _mm256_set1_epi32(0xFFF)); } @@ -5351,7 +5122,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_srli_epi16() { assert_eq_m256i( - _mm256_srli_epi16(_mm256_set1_epi16(0xFF), 4), + _mm256_srli_epi16::<4>(_mm256_set1_epi16(0xFF)), _mm256_set1_epi16(0xF), ); } @@ -5359,7 +5130,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_srli_epi32() { assert_eq_m256i( - _mm256_srli_epi32(_mm256_set1_epi32(0xFFFF), 4), + _mm256_srli_epi32::<4>(_mm256_set1_epi32(0xFFFF)), _mm256_set1_epi32(0xFFF), ); } @@ -5367,7 +5138,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_srli_epi64() { assert_eq_m256i( - _mm256_srli_epi64(_mm256_set1_epi64x(0xFFFFFFFF), 4), + _mm256_srli_epi64::<4>(_mm256_set1_epi64x(0xFFFFFFFF)), _mm256_set1_epi64x(0xFFFFFFF), ); } @@ -5496,10 +5267,10 @@ mod tests { -17, -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28, -29, -30, -31, -32, ); - let r = _mm256_alignr_epi8(a, b, 33); + let r = _mm256_alignr_epi8::<33>(a, b); assert_eq_m256i(r, _mm256_set1_epi8(0)); - let r = _mm256_alignr_epi8(a, b, 17); + let r = _mm256_alignr_epi8::<17>(a, b); #[rustfmt::skip] let expected = _mm256_setr_epi8( 2, 3, 4, 5, 6, 7, 8, 9, @@ -5509,7 +5280,7 @@ mod tests { ); assert_eq_m256i(r, expected); - let r = _mm256_alignr_epi8(a, b, 4); + let r = _mm256_alignr_epi8::<4>(a, b); #[rustfmt::skip] let expected = _mm256_setr_epi8( -5, -6, -7, -8, -9, -10, -11, -12, @@ -5526,10 +5297,10 @@ mod tests { -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28, -29, -30, -31, -32, ); - let r = _mm256_alignr_epi8(a, b, 16); + let r = _mm256_alignr_epi8::<16>(a, b); assert_eq_m256i(r, expected); - let r = _mm256_alignr_epi8(a, b, 15); + let r = _mm256_alignr_epi8::<15>(a, b); #[rustfmt::skip] let expected = _mm256_setr_epi8( -16, 1, 2, 3, 4, 5, 6, 7, @@ -5539,7 +5310,7 @@ mod tests { ); assert_eq_m256i(r, expected); - let r = _mm256_alignr_epi8(a, b, 0); + let r = _mm256_alignr_epi8::<0>(a, b); assert_eq_m256i(r, b); } @@ -5583,7 +5354,7 @@ mod tests { unsafe fn test_mm256_permute4x64_epi64() { let a = _mm256_setr_epi64x(100, 200, 300, 400); let expected = _mm256_setr_epi64x(400, 100, 200, 100); - let r = _mm256_permute4x64_epi64(a, 0b00010011); + let r = _mm256_permute4x64_epi64::<0b00010011>(a); assert_eq_m256i(r, expected); } @@ -5591,7 +5362,7 @@ mod tests { unsafe fn test_mm256_permute2x128_si256() { let a = _mm256_setr_epi64x(100, 200, 500, 600); let b = _mm256_setr_epi64x(300, 400, 700, 800); - let r = _mm256_permute2x128_si256(a, b, 0b00_01_00_11); + let r = _mm256_permute2x128_si256::<0b00_01_00_11>(a, b); let e = _mm256_setr_epi64x(700, 800, 500, 600); assert_eq_m256i(r, e); } @@ -5599,7 +5370,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_permute4x64_pd() { let a = _mm256_setr_pd(1., 2., 3., 4.); - let r = _mm256_permute4x64_pd(a, 0b00_01_00_11); + let r = _mm256_permute4x64_pd::<0b00_01_00_11>(a); let e = _mm256_setr_pd(4., 1., 2., 1.); assert_eq_m256d(r, e); } @@ -5620,7 +5391,7 @@ mod tests { arr[i as usize] = i; } // A multiplier of 4 is word-addressing - let r = _mm_i32gather_epi32(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48), 4); + let r = _mm_i32gather_epi32::<4>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)); assert_eq_m128i(r, _mm_setr_epi32(0, 16, 32, 48)); } @@ -5631,12 +5402,11 @@ mod tests { arr[i as usize] = i; } // A multiplier of 4 is word-addressing - let r = _mm_mask_i32gather_epi32( + let r = _mm_mask_i32gather_epi32::<4>( _mm_set1_epi32(256), arr.as_ptr(), _mm_setr_epi32(0, 16, 64, 96), _mm_setr_epi32(-1, -1, -1, 0), - 4, ); assert_eq_m128i(r, _mm_setr_epi32(0, 16, 64, 256)); } @@ -5648,11 +5418,8 @@ mod tests { arr[i as usize] = i; } // A multiplier of 4 is word-addressing - let r = _mm256_i32gather_epi32( - arr.as_ptr(), - _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4), - 4, - ); + let r = + _mm256_i32gather_epi32::<4>(arr.as_ptr(), _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4)); assert_eq_m256i(r, _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4)); } @@ -5663,12 +5430,11 @@ mod tests { arr[i as usize] = i; } // A multiplier of 4 is word-addressing - let r = _mm256_mask_i32gather_epi32( + let r = _mm256_mask_i32gather_epi32::<4>( _mm256_set1_epi32(256), arr.as_ptr(), _mm256_setr_epi32(0, 16, 64, 96, 0, 0, 0, 0), _mm256_setr_epi32(-1, -1, -1, 0, 0, 0, 0, 0), - 4, ); assert_eq_m256i(r, _mm256_setr_epi32(0, 16, 64, 256, 256, 256, 256, 256)); } @@ -5682,7 +5448,7 @@ mod tests { j += 1.0; } // A multiplier of 4 is word-addressing for f32s - let r = _mm_i32gather_ps(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48), 4); + let r = _mm_i32gather_ps::<4>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)); assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 32.0, 48.0)); } @@ -5695,12 +5461,11 @@ mod tests { j += 1.0; } // A multiplier of 4 is word-addressing for f32s - let r = _mm_mask_i32gather_ps( + let r = _mm_mask_i32gather_ps::<4>( _mm_set1_ps(256.0), arr.as_ptr(), _mm_setr_epi32(0, 16, 64, 96), _mm_setr_ps(-1.0, -1.0, -1.0, 0.0), - 4, ); assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 64.0, 256.0)); } @@ -5714,11 +5479,8 @@ mod tests { j += 1.0; } // A multiplier of 4 is word-addressing for f32s - let r = _mm256_i32gather_ps( - arr.as_ptr(), - _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4), - 4, - ); + let r = + _mm256_i32gather_ps::<4>(arr.as_ptr(), _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4)); assert_eq_m256(r, _mm256_setr_ps(0.0, 16.0, 32.0, 48.0, 1.0, 2.0, 3.0, 4.0)); } @@ -5731,12 +5493,11 @@ mod tests { j += 1.0; } // A multiplier of 4 is word-addressing for f32s - let r = _mm256_mask_i32gather_ps( + let r = _mm256_mask_i32gather_ps::<4>( _mm256_set1_ps(256.0), arr.as_ptr(), _mm256_setr_epi32(0, 16, 64, 96, 0, 0, 0, 0), _mm256_setr_ps(-1.0, -1.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0), - 4, ); assert_eq_m256( r, @@ -5751,7 +5512,7 @@ mod tests { arr[i as usize] = i; } // A multiplier of 8 is word-addressing for i64s - let r = _mm_i32gather_epi64(arr.as_ptr(), _mm_setr_epi32(0, 16, 0, 0), 8); + let r = _mm_i32gather_epi64::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 0, 0)); assert_eq_m128i(r, _mm_setr_epi64x(0, 16)); } @@ -5762,12 +5523,11 @@ mod tests { arr[i as usize] = i; } // A multiplier of 8 is word-addressing for i64s - let r = _mm_mask_i32gather_epi64( + let r = _mm_mask_i32gather_epi64::<8>( _mm_set1_epi64x(256), arr.as_ptr(), _mm_setr_epi32(16, 16, 16, 16), _mm_setr_epi64x(-1, 0), - 8, ); assert_eq_m128i(r, _mm_setr_epi64x(16, 256)); } @@ -5779,7 +5539,7 @@ mod tests { arr[i as usize] = i; } // A multiplier of 8 is word-addressing for i64s - let r = _mm256_i32gather_epi64(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48), 8); + let r = _mm256_i32gather_epi64::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)); assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 32, 48)); } @@ -5790,12 +5550,11 @@ mod tests { arr[i as usize] = i; } // A multiplier of 8 is word-addressing for i64s - let r = _mm256_mask_i32gather_epi64( + let r = _mm256_mask_i32gather_epi64::<8>( _mm256_set1_epi64x(256), arr.as_ptr(), _mm_setr_epi32(0, 16, 64, 96), _mm256_setr_epi64x(-1, -1, -1, 0), - 8, ); assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 64, 256)); } @@ -5809,7 +5568,7 @@ mod tests { j += 1.0; } // A multiplier of 8 is word-addressing for f64s - let r = _mm_i32gather_pd(arr.as_ptr(), _mm_setr_epi32(0, 16, 0, 0), 8); + let r = _mm_i32gather_pd::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 0, 0)); assert_eq_m128d(r, _mm_setr_pd(0.0, 16.0)); } @@ -5822,12 +5581,11 @@ mod tests { j += 1.0; } // A multiplier of 8 is word-addressing for f64s - let r = _mm_mask_i32gather_pd( + let r = _mm_mask_i32gather_pd::<8>( _mm_set1_pd(256.0), arr.as_ptr(), _mm_setr_epi32(16, 16, 16, 16), _mm_setr_pd(-1.0, 0.0), - 8, ); assert_eq_m128d(r, _mm_setr_pd(16.0, 256.0)); } @@ -5841,7 +5599,7 @@ mod tests { j += 1.0; } // A multiplier of 8 is word-addressing for f64s - let r = _mm256_i32gather_pd(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48), 8); + let r = _mm256_i32gather_pd::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)); assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 32.0, 48.0)); } @@ -5854,12 +5612,11 @@ mod tests { j += 1.0; } // A multiplier of 8 is word-addressing for f64s - let r = _mm256_mask_i32gather_pd( + let r = _mm256_mask_i32gather_pd::<8>( _mm256_set1_pd(256.0), arr.as_ptr(), _mm_setr_epi32(0, 16, 64, 96), _mm256_setr_pd(-1.0, -1.0, -1.0, 0.0), - 8, ); assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 64.0, 256.0)); } @@ -5871,7 +5628,7 @@ mod tests { arr[i as usize] = i; } // A multiplier of 4 is word-addressing - let r = _mm_i64gather_epi32(arr.as_ptr(), _mm_setr_epi64x(0, 16), 4); + let r = _mm_i64gather_epi32::<4>(arr.as_ptr(), _mm_setr_epi64x(0, 16)); assert_eq_m128i(r, _mm_setr_epi32(0, 16, 0, 0)); } @@ -5882,12 +5639,11 @@ mod tests { arr[i as usize] = i; } // A multiplier of 4 is word-addressing - let r = _mm_mask_i64gather_epi32( + let r = _mm_mask_i64gather_epi32::<4>( _mm_set1_epi32(256), arr.as_ptr(), _mm_setr_epi64x(0, 16), _mm_setr_epi32(-1, 0, -1, 0), - 4, ); assert_eq_m128i(r, _mm_setr_epi32(0, 256, 0, 0)); } @@ -5899,7 +5655,7 @@ mod tests { arr[i as usize] = i; } // A multiplier of 4 is word-addressing - let r = _mm256_i64gather_epi32(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48), 4); + let r = _mm256_i64gather_epi32::<4>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)); assert_eq_m128i(r, _mm_setr_epi32(0, 16, 32, 48)); } @@ -5910,12 +5666,11 @@ mod tests { arr[i as usize] = i; } // A multiplier of 4 is word-addressing - let r = _mm256_mask_i64gather_epi32( + let r = _mm256_mask_i64gather_epi32::<4>( _mm_set1_epi32(256), arr.as_ptr(), _mm256_setr_epi64x(0, 16, 64, 96), _mm_setr_epi32(-1, -1, -1, 0), - 4, ); assert_eq_m128i(r, _mm_setr_epi32(0, 16, 64, 256)); } @@ -5929,7 +5684,7 @@ mod tests { j += 1.0; } // A multiplier of 4 is word-addressing for f32s - let r = _mm_i64gather_ps(arr.as_ptr(), _mm_setr_epi64x(0, 16), 4); + let r = _mm_i64gather_ps::<4>(arr.as_ptr(), _mm_setr_epi64x(0, 16)); assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 0.0, 0.0)); } @@ -5942,12 +5697,11 @@ mod tests { j += 1.0; } // A multiplier of 4 is word-addressing for f32s - let r = _mm_mask_i64gather_ps( + let r = _mm_mask_i64gather_ps::<4>( _mm_set1_ps(256.0), arr.as_ptr(), _mm_setr_epi64x(0, 16), _mm_setr_ps(-1.0, 0.0, -1.0, 0.0), - 4, ); assert_eq_m128(r, _mm_setr_ps(0.0, 256.0, 0.0, 0.0)); } @@ -5961,7 +5715,7 @@ mod tests { j += 1.0; } // A multiplier of 4 is word-addressing for f32s - let r = _mm256_i64gather_ps(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48), 4); + let r = _mm256_i64gather_ps::<4>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)); assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 32.0, 48.0)); } @@ -5974,12 +5728,11 @@ mod tests { j += 1.0; } // A multiplier of 4 is word-addressing for f32s - let r = _mm256_mask_i64gather_ps( + let r = _mm256_mask_i64gather_ps::<4>( _mm_set1_ps(256.0), arr.as_ptr(), _mm256_setr_epi64x(0, 16, 64, 96), _mm_setr_ps(-1.0, -1.0, -1.0, 0.0), - 4, ); assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 64.0, 256.0)); } @@ -5991,7 +5744,7 @@ mod tests { arr[i as usize] = i; } // A multiplier of 8 is word-addressing for i64s - let r = _mm_i64gather_epi64(arr.as_ptr(), _mm_setr_epi64x(0, 16), 8); + let r = _mm_i64gather_epi64::<8>(arr.as_ptr(), _mm_setr_epi64x(0, 16)); assert_eq_m128i(r, _mm_setr_epi64x(0, 16)); } @@ -6002,12 +5755,11 @@ mod tests { arr[i as usize] = i; } // A multiplier of 8 is word-addressing for i64s - let r = _mm_mask_i64gather_epi64( + let r = _mm_mask_i64gather_epi64::<8>( _mm_set1_epi64x(256), arr.as_ptr(), _mm_setr_epi64x(16, 16), _mm_setr_epi64x(-1, 0), - 8, ); assert_eq_m128i(r, _mm_setr_epi64x(16, 256)); } @@ -6019,7 +5771,7 @@ mod tests { arr[i as usize] = i; } // A multiplier of 8 is word-addressing for i64s - let r = _mm256_i64gather_epi64(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48), 8); + let r = _mm256_i64gather_epi64::<8>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)); assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 32, 48)); } @@ -6030,12 +5782,11 @@ mod tests { arr[i as usize] = i; } // A multiplier of 8 is word-addressing for i64s - let r = _mm256_mask_i64gather_epi64( + let r = _mm256_mask_i64gather_epi64::<8>( _mm256_set1_epi64x(256), arr.as_ptr(), _mm256_setr_epi64x(0, 16, 64, 96), _mm256_setr_epi64x(-1, -1, -1, 0), - 8, ); assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 64, 256)); } @@ -6049,7 +5800,7 @@ mod tests { j += 1.0; } // A multiplier of 8 is word-addressing for f64s - let r = _mm_i64gather_pd(arr.as_ptr(), _mm_setr_epi64x(0, 16), 8); + let r = _mm_i64gather_pd::<8>(arr.as_ptr(), _mm_setr_epi64x(0, 16)); assert_eq_m128d(r, _mm_setr_pd(0.0, 16.0)); } @@ -6062,12 +5813,11 @@ mod tests { j += 1.0; } // A multiplier of 8 is word-addressing for f64s - let r = _mm_mask_i64gather_pd( + let r = _mm_mask_i64gather_pd::<8>( _mm_set1_pd(256.0), arr.as_ptr(), _mm_setr_epi64x(16, 16), _mm_setr_pd(-1.0, 0.0), - 8, ); assert_eq_m128d(r, _mm_setr_pd(16.0, 256.0)); } @@ -6081,7 +5831,7 @@ mod tests { j += 1.0; } // A multiplier of 8 is word-addressing for f64s - let r = _mm256_i64gather_pd(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48), 8); + let r = _mm256_i64gather_pd::<8>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)); assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 32.0, 48.0)); } @@ -6094,12 +5844,11 @@ mod tests { j += 1.0; } // A multiplier of 8 is word-addressing for f64s - let r = _mm256_mask_i64gather_pd( + let r = _mm256_mask_i64gather_pd::<8>( _mm256_set1_pd(256.0), arr.as_ptr(), _mm256_setr_epi64x(0, 16, 64, 96), _mm256_setr_pd(-1.0, -1.0, -1.0, 0.0), - 8, ); assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 64.0, 256.0)); } @@ -6113,9 +5862,9 @@ mod tests { 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 ); - let r1 = _mm256_extract_epi8(a, 0); - let r2 = _mm256_extract_epi8(a, 35); - assert_eq!(r1, -1); + let r1 = _mm256_extract_epi8::<0>(a); + let r2 = _mm256_extract_epi8::<3>(a); + assert_eq!(r1, 0xFF); assert_eq!(r2, 3); } @@ -6126,17 +5875,17 @@ mod tests { -1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, ); - let r1 = _mm256_extract_epi16(a, 0); - let r2 = _mm256_extract_epi16(a, 19); - assert_eq!(r1, -1); + let r1 = _mm256_extract_epi16::<0>(a); + let r2 = _mm256_extract_epi16::<3>(a); + assert_eq!(r1, 0xFFFF); assert_eq!(r2, 3); } #[simd_test(enable = "avx2")] unsafe fn test_mm256_extract_epi32() { let a = _mm256_setr_epi32(-1, 1, 2, 3, 4, 5, 6, 7); - let r1 = _mm256_extract_epi32(a, 0); - let r2 = _mm256_extract_epi32(a, 11); + let r1 = _mm256_extract_epi32::<0>(a); + let r2 = _mm256_extract_epi32::<3>(a); assert_eq!(r1, -1); assert_eq!(r2, 3); } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512bf16.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512bf16.rs new file mode 100644 index 000000000..e9977e018 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512bf16.rs @@ -0,0 +1,1573 @@ +//! [AVX512BF16 intrinsics]. +//! +//! [AVX512BF16 intrinsics]: https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769&avx512techs=AVX512_BF16 + +use crate::{ + core_arch::{simd::*, simd_llvm::*, x86::*}, + mem::transmute, +}; + +#[cfg(test)] +use stdarch_test::assert_instr; + +#[allow(improper_ctypes)] +extern "C" { + #[link_name = "llvm.x86.avx512bf16.cvtne2ps2bf16.128"] + fn cvtne2ps2bf16(a: f32x4, b: f32x4) -> i16x8; + #[link_name = "llvm.x86.avx512bf16.cvtne2ps2bf16.256"] + fn cvtne2ps2bf16_256(a: f32x8, b: f32x8) -> i16x16; + #[link_name = "llvm.x86.avx512bf16.cvtne2ps2bf16.512"] + fn cvtne2ps2bf16_512(a: f32x16, b: f32x16) -> i16x32; + #[link_name = "llvm.x86.avx512bf16.cvtneps2bf16.256"] + fn cvtneps2bf16_256(a: f32x8) -> i16x8; + #[link_name = "llvm.x86.avx512bf16.cvtneps2bf16.512"] + fn cvtneps2bf16_512(a: f32x16) -> i16x16; + #[link_name = "llvm.x86.avx512bf16.dpbf16ps.128"] + fn dpbf16ps(a: f32x4, b: i32x4, c: i32x4) -> f32x4; + #[link_name = "llvm.x86.avx512bf16.dpbf16ps.256"] + fn dpbf16ps_256(a: f32x8, b: i32x8, c: i32x8) -> f32x8; + #[link_name = "llvm.x86.avx512bf16.dpbf16ps.512"] + fn dpbf16ps_512(a: f32x16, b: i32x16, c: i32x16) -> f32x16; +} + +/// Convert packed single-precision (32-bit) floating-point elements in two 128-bit vectors +/// a and b to packed BF16 (16-bit) floating-point elements, and store the results in a +/// 128-bit wide vector. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651&avx512techs=AVX512_BF16&text=_mm_cvtne2ps_pbh) +#[inline] +#[target_feature(enable = "avx512bf16,avx512vl")] +#[cfg_attr(test, assert_instr("vcvtne2ps2bf16"))] +pub unsafe fn _mm_cvtne2ps_pbh(a: __m128, b: __m128) -> __m128bh { + transmute(cvtne2ps2bf16(a.as_f32x4(), b.as_f32x4())) +} + +/// Convert packed single-precision (32-bit) floating-point elements in two vectors +/// a and b to packed BF16 (16-bit) floating-point elements, and store the results +/// in single vector dst using writemask k (elements are copied from src when the +/// corresponding mask bit is not set). +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651&avx512techs=AVX512_BF16&text=_mm_mask_cvtne2ps_pbh) +#[inline] +#[target_feature(enable = "avx512bf16,avx512vl")] +#[cfg_attr(test, assert_instr("vcvtne2ps2bf16"))] +pub unsafe fn _mm_mask_cvtne2ps_pbh(src: __m128bh, k: __mmask8, a: __m128, b: __m128) -> __m128bh { + let cvt = _mm_cvtne2ps_pbh(a, b).as_u16x8(); + transmute(simd_select_bitmask(k, cvt, src.as_u16x8())) +} + +/// Convert packed single-precision (32-bit) floating-point elements in two vectors +/// a and b to packed BF16 (16-bit) floating-point elements, and store the results +/// in single vector dst using zeromask k (elements are zeroed out when the corresponding +/// mask bit is not set). +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651&avx512techs=AVX512_BF16&text=_mm_maskz_cvtne2ps_pbh) +#[inline] +#[target_feature(enable = "avx512bf16,avx512vl")] +#[cfg_attr(test, assert_instr("vcvtne2ps2bf16"))] +pub unsafe fn _mm_maskz_cvtne2ps_pbh(k: __mmask8, a: __m128, b: __m128) -> __m128bh { + let cvt = _mm_cvtne2ps_pbh(a, b).as_u16x8(); + let zero = _mm_setzero_si128().as_u16x8(); + transmute(simd_select_bitmask(k, cvt, zero)) +} + +/// Convert packed single-precision (32-bit) floating-point elements in two 256-bit vectors +/// a and b to packed BF16 (16-bit) floating-point elements, and store the results in a +/// 256-bit wide vector. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651,1654&avx512techs=AVX512_BF16&text=_mm256_cvtne2ps_pbh) +#[inline] +#[target_feature(enable = "avx512bf16,avx512vl")] +#[cfg_attr(test, assert_instr("vcvtne2ps2bf16"))] +pub unsafe fn _mm256_cvtne2ps_pbh(a: __m256, b: __m256) -> __m256bh { + transmute(cvtne2ps2bf16_256(a.as_f32x8(), b.as_f32x8())) +} + +/// Convert packed single-precision (32-bit) floating-point elements in two vectors a and b +/// to packed BF16 (16-bit) floating-point elements and and store the results in single vector +/// dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651,1654&avx512techs=AVX512_BF16&text=_mm256_mask_cvtne2ps_pbh) +#[inline] +#[target_feature(enable = "avx512bf16,avx512vl")] +#[cfg_attr(test, assert_instr("vcvtne2ps2bf16"))] +pub unsafe fn _mm256_mask_cvtne2ps_pbh( + src: __m256bh, + k: __mmask16, + a: __m256, + b: __m256, +) -> __m256bh { + let cvt = _mm256_cvtne2ps_pbh(a, b).as_u16x16(); + transmute(simd_select_bitmask(k, cvt, src.as_u16x16())) +} + +/// Convert packed single-precision (32-bit) floating-point elements in two vectors a and b +/// to packed BF16 (16-bit) floating-point elements, and store the results in single vector +/// dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651,1654&avx512techs=AVX512_BF16&text=_mm256_maskz_cvtne2ps_pbh) +#[inline] +#[target_feature(enable = "avx512bf16,avx512vl")] +#[cfg_attr(test, assert_instr("vcvtne2ps2bf16"))] +pub unsafe fn _mm256_maskz_cvtne2ps_pbh(k: __mmask16, a: __m256, b: __m256) -> __m256bh { + let cvt = _mm256_cvtne2ps_pbh(a, b).as_u16x16(); + let zero = _mm256_setzero_si256().as_u16x16(); + transmute(simd_select_bitmask(k, cvt, zero)) +} + +/// Convert packed single-precision (32-bit) floating-point elements in two 512-bit vectors +/// a and b to packed BF16 (16-bit) floating-point elements, and store the results in a +/// 512-bit wide vector. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651,1654,1657&avx512techs=AVX512_BF16&text=_mm512_cvtne2ps_pbh) +#[inline] +#[target_feature(enable = "avx512bf16,avx512f")] +#[cfg_attr(test, assert_instr("vcvtne2ps2bf16"))] +pub unsafe fn _mm512_cvtne2ps_pbh(a: __m512, b: __m512) -> __m512bh { + transmute(cvtne2ps2bf16_512(a.as_f32x16(), b.as_f32x16())) +} + +/// Convert packed single-precision (32-bit) floating-point elements in two vectors +/// a and b to packed BF16 (16-bit) floating-point elements, and store the results +/// in single vector dst using writemask k (elements are copied from src when the +/// corresponding mask bit is not set). +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651,1654,1657&avx512techs=AVX512_BF16&text=_mm512_mask_cvtne2ps_pbh) +#[inline] +#[target_feature(enable = "avx512bf16,avx512f")] +#[cfg_attr(test, assert_instr("vcvtne2ps2bf16"))] +pub unsafe fn _mm512_mask_cvtne2ps_pbh( + src: __m512bh, + k: __mmask32, + a: __m512, + b: __m512, +) -> __m512bh { + let cvt = _mm512_cvtne2ps_pbh(a, b).as_u16x32(); + transmute(simd_select_bitmask(k, cvt, src.as_u16x32())) +} + +/// Convert packed single-precision (32-bit) floating-point elements in two vectors +/// a and b to packed BF16 (16-bit) floating-point elements, and store the results +/// in single vector dst using zeromask k (elements are zeroed out when the corresponding +/// mask bit is not set). +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651,1654,1657&avx512techs=AVX512_BF16&text=_mm512_maskz_cvtne2ps_pbh) +#[inline] +#[target_feature(enable = "avx512bf16,avx512f")] +#[cfg_attr(test, assert_instr("vcvtne2ps2bf16"))] +pub unsafe fn _mm512_maskz_cvtne2ps_pbh(k: __mmask32, a: __m512, b: __m512) -> __m512bh { + let cvt = _mm512_cvtne2ps_pbh(a, b).as_u16x32(); + let zero = _mm512_setzero_si512().as_u16x32(); + transmute(simd_select_bitmask(k, cvt, zero)) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed BF16 (16-bit) +/// floating-point elements, and store the results in dst. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651,1654,1657,1660&avx512techs=AVX512_BF16&text=_mm256_cvtneps_pbh) +#[inline] +#[target_feature(enable = "avx512bf16,avx512vl")] +#[cfg_attr(test, assert_instr("vcvtneps2bf16"))] +pub unsafe fn _mm256_cvtneps_pbh(a: __m256) -> __m128bh { + transmute(cvtneps2bf16_256(a.as_f32x8())) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed BF16 (16-bit) +/// floating-point elements, and store the results in dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651,1654,1657,1660&avx512techs=AVX512_BF16&text=_mm256_mask_cvtneps_pbh) +#[inline] +#[target_feature(enable = "avx512bf16,avx512vl")] +#[cfg_attr(test, assert_instr("vcvtneps2bf16"))] +pub unsafe fn _mm256_mask_cvtneps_pbh(src: __m128bh, k: __mmask8, a: __m256) -> __m128bh { + let cvt = _mm256_cvtneps_pbh(a).as_u16x8(); + transmute(simd_select_bitmask(k, cvt, src.as_u16x8())) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed BF16 (16-bit) +/// floating-point elements, and store the results in dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651,1654,1657,1660&avx512techs=AVX512_BF16&text=_mm256_maskz_cvtneps_pbh) +#[inline] +#[target_feature(enable = "avx512bf16,avx512vl")] +#[cfg_attr(test, assert_instr("vcvtneps2bf16"))] +pub unsafe fn _mm256_maskz_cvtneps_pbh(k: __mmask8, a: __m256) -> __m128bh { + let cvt = _mm256_cvtneps_pbh(a).as_u16x8(); + let zero = _mm_setzero_si128().as_u16x8(); + transmute(simd_select_bitmask(k, cvt, zero)) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed BF16 (16-bit) +/// floating-point elements, and store the results in dst. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651,1654,1657,1660&avx512techs=AVX512_BF16&text=_mm512_cvtneps_pbh) +#[inline] +#[target_feature(enable = "avx512bf16,avx512f")] +#[cfg_attr(test, assert_instr("vcvtneps2bf16"))] +pub unsafe fn _mm512_cvtneps_pbh(a: __m512) -> __m256bh { + transmute(cvtneps2bf16_512(a.as_f32x16())) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed BF16 (16-bit) +/// floating-point elements, and store the results in dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651,1654,1657,1660&avx512techs=AVX512_BF16&text=_mm512_mask_cvtneps_pbh) +#[inline] +#[target_feature(enable = "avx512bf16,avx512f")] +#[cfg_attr(test, assert_instr("vcvtneps2bf16"))] +pub unsafe fn _mm512_mask_cvtneps_pbh(src: __m256bh, k: __mmask16, a: __m512) -> __m256bh { + let cvt = _mm512_cvtneps_pbh(a).as_u16x16(); + transmute(simd_select_bitmask(k, cvt, src.as_u16x16())) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed BF16 (16-bit) +/// floating-point elements, and store the results in dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651,1654,1657,1660&avx512techs=AVX512_BF16&text=_mm512_maskz_cvtneps_pbh) +#[inline] +#[target_feature(enable = "avx512bf16,avx512f")] +#[cfg_attr(test, assert_instr("vcvtneps2bf16"))] +pub unsafe fn _mm512_maskz_cvtneps_pbh(k: __mmask16, a: __m512) -> __m256bh { + let cvt = _mm512_cvtneps_pbh(a).as_u16x16(); + let zero = _mm256_setzero_si256().as_u16x16(); + transmute(simd_select_bitmask(k, cvt, zero)) +} + +/// Compute dot-product of BF16 (16-bit) floating-point pairs in a and b, +/// accumulating the intermediate single-precision (32-bit) floating-point elements +/// with elements in src, and store the results in dst. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651,1654,1657,1660&avx512techs=AVX512_BF16&text=_mm_dpbf16_ps) +#[inline] +#[target_feature(enable = "avx512bf16,avx512vl")] +#[cfg_attr(test, assert_instr("vdpbf16ps"))] +pub unsafe fn _mm_dpbf16_ps(src: __m128, a: __m128bh, b: __m128bh) -> __m128 { + transmute(dpbf16ps(src.as_f32x4(), a.as_i32x4(), b.as_i32x4())) +} + +/// Compute dot-product of BF16 (16-bit) floating-point pairs in a and b, +/// accumulating the intermediate single-precision (32-bit) floating-point elements +/// with elements in src, and store the results in dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651,1654,1657,1660&avx512techs=AVX512_BF16&text=_mm_mask_dpbf16_ps) +#[inline] +#[target_feature(enable = "avx512bf16,avx512vl")] +#[cfg_attr(test, assert_instr("vdpbf16ps"))] +pub unsafe fn _mm_mask_dpbf16_ps(src: __m128, k: __mmask8, a: __m128bh, b: __m128bh) -> __m128 { + let rst = _mm_dpbf16_ps(src, a, b).as_f32x4(); + transmute(simd_select_bitmask(k, rst, src.as_f32x4())) +} + +/// Compute dot-product of BF16 (16-bit) floating-point pairs in a and b, +/// accumulating the intermediate single-precision (32-bit) floating-point elements +/// with elements in src, and store the results in dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651,1654,1657,1660&avx512techs=AVX512_BF16&text=_mm_maskz_dpbf16_ps) +#[inline] +#[target_feature(enable = "avx512bf16,avx512vl")] +#[cfg_attr(test, assert_instr("vdpbf16ps"))] +pub unsafe fn _mm_maskz_dpbf16_ps(k: __mmask8, src: __m128, a: __m128bh, b: __m128bh) -> __m128 { + let rst = _mm_dpbf16_ps(src, a, b).as_f32x4(); + let zero = _mm_set1_ps(0.0_f32).as_f32x4(); + transmute(simd_select_bitmask(k, rst, zero)) +} + +/// Compute dot-product of BF16 (16-bit) floating-point pairs in a and b, +/// accumulating the intermediate single-precision (32-bit) floating-point elements +/// with elements in src, and store the results in dst. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651,1654,1657,1660&avx512techs=AVX512_BF16&text=_mm256_dpbf16_ps) +#[inline] +#[target_feature(enable = "avx512bf16,avx512vl")] +#[cfg_attr(test, assert_instr("vdpbf16ps"))] +pub unsafe fn _mm256_dpbf16_ps(src: __m256, a: __m256bh, b: __m256bh) -> __m256 { + transmute(dpbf16ps_256(src.as_f32x8(), a.as_i32x8(), b.as_i32x8())) +} + +/// Compute dot-product of BF16 (16-bit) floating-point pairs in a and b, +/// accumulating the intermediate single-precision (32-bit) floating-point elements +/// with elements in src, and store the results in dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651,1654,1657,1660&avx512techs=AVX512_BF16&text=_mm256_mask_dpbf16_ps) +#[inline] +#[target_feature(enable = "avx512bf16,avx512vl")] +#[cfg_attr(test, assert_instr("vdpbf16ps"))] +pub unsafe fn _mm256_mask_dpbf16_ps(src: __m256, k: __mmask8, a: __m256bh, b: __m256bh) -> __m256 { + let rst = _mm256_dpbf16_ps(src, a, b).as_f32x8(); + transmute(simd_select_bitmask(k, rst, src.as_f32x8())) +} + +/// Compute dot-product of BF16 (16-bit) floating-point pairs in a and b, +/// accumulating the intermediate single-precision (32-bit) floating-point elements +/// with elements in src, and store the results in dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651,1654,1657,1660&avx512techs=AVX512_BF16&text=_mm256_maskz_dpbf16_ps) +#[inline] +#[target_feature(enable = "avx512bf16,avx512vl")] +#[cfg_attr(test, assert_instr("vdpbf16ps"))] +pub unsafe fn _mm256_maskz_dpbf16_ps(k: __mmask8, src: __m256, a: __m256bh, b: __m256bh) -> __m256 { + let rst = _mm256_dpbf16_ps(src, a, b).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, rst, zero)) +} + +/// Compute dot-product of BF16 (16-bit) floating-point pairs in a and b, +/// accumulating the intermediate single-precision (32-bit) floating-point elements +/// with elements in src, and store the results in dst.Compute dot-product of BF16 (16-bit) +/// floating-point pairs in a and b, accumulating the intermediate single-precision (32-bit) +/// floating-point elements with elements in src, and store the results in dst. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651,1654,1657,1660&avx512techs=AVX512_BF16&text=_mm512_dpbf16_ps) +#[inline] +#[target_feature(enable = "avx512bf16,avx512f")] +#[cfg_attr(test, assert_instr("vdpbf16ps"))] +pub unsafe fn _mm512_dpbf16_ps(src: __m512, a: __m512bh, b: __m512bh) -> __m512 { + transmute(dpbf16ps_512(src.as_f32x16(), a.as_i32x16(), b.as_i32x16())) +} + +/// Compute dot-product of BF16 (16-bit) floating-point pairs in a and b, +/// accumulating the intermediate single-precision (32-bit) floating-point elements +/// with elements in src, and store the results in dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651,1654,1657,1660&avx512techs=AVX512_BF16&text=_mm512_mask_dpbf16_ps) +#[inline] +#[target_feature(enable = "avx512bf16,avx512f")] +#[cfg_attr(test, assert_instr("vdpbf16ps"))] +pub unsafe fn _mm512_mask_dpbf16_ps(src: __m512, k: __mmask16, a: __m512bh, b: __m512bh) -> __m512 { + let rst = _mm512_dpbf16_ps(src, a, b).as_f32x16(); + transmute(simd_select_bitmask(k, rst, src.as_f32x16())) +} + +/// Compute dot-product of BF16 (16-bit) floating-point pairs in a and b, +/// accumulating the intermediate single-precision (32-bit) floating-point elements +/// with elements in src, and store the results in dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=1769,1651,1654,1657,1660&avx512techs=AVX512_BF16&text=_mm512_maskz_dpbf16_ps) +#[inline] +#[target_feature(enable = "avx512bf16,avx512f")] +#[cfg_attr(test, assert_instr("vdpbf16ps"))] +pub unsafe fn _mm512_maskz_dpbf16_ps( + k: __mmask16, + src: __m512, + a: __m512bh, + b: __m512bh, +) -> __m512 { + let rst = _mm512_dpbf16_ps(src, a, b).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, rst, zero)) +} + +#[cfg(test)] +mod tests { + use crate::{core_arch::x86::*, mem::transmute}; + use stdarch_test::simd_test; + + #[simd_test(enable = "avx512bf16,avx512vl")] + unsafe fn test_mm_cvtne2ps_pbh() { + let a_array = [178.125_f32, 10.5_f32, 3.75_f32, 50.25_f32]; + let b_array = [-178.125_f32, -10.5_f32, -3.75_f32, -50.25_f32]; + let a: __m128 = transmute(a_array); + let b: __m128 = transmute(b_array); + let c: __m128bh = _mm_cvtne2ps_pbh(a, b); + let result: [u16; 8] = transmute(c.as_u16x8()); + #[rustfmt::skip] + let expected_result: [u16; 8] = [ + 0b1_10000110_0110010, + 0b1_10000010_0101000, + 0b1_10000000_1110000, + 0b1_10000100_1001001, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + ]; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512vl")] + unsafe fn test_mm_mask_cvtne2ps_pbh() { + let a_array = [178.125_f32, 10.5_f32, 3.75_f32, 50.25_f32]; + let b_array = [-178.125_f32, -10.5_f32, -3.75_f32, -50.25_f32]; + #[rustfmt::skip] + let src_array: [u16; 8] = [ + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + ]; + let src: __m128bh = transmute(src_array); + let a: __m128 = transmute(a_array); + let b: __m128 = transmute(b_array); + let k: __mmask8 = 0b1111_1111; + let c: __m128bh = _mm_mask_cvtne2ps_pbh(src, k, a, b); + let result: [u16; 8] = transmute(c.as_u16x8()); + #[rustfmt::skip] + let expected_result: [u16; 8] = [ + 0b1_10000110_0110010, + 0b1_10000010_0101000, + 0b1_10000000_1110000, + 0b1_10000100_1001001, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + ]; + assert_eq!(result, expected_result); + let k = 0b0000_0000; + let c = _mm_mask_cvtne2ps_pbh(src, k, a, b); + let result: [u16; 8] = transmute(c.as_u16x8()); + let expected_result = src_array; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512vl")] + unsafe fn test_mm_maskz_cvtne2ps_pbh() { + let a_array = [178.125_f32, 10.5_f32, 3.75_f32, 50.25_f32]; + let b_array = [-178.125_f32, -10.5_f32, -3.75_f32, -50.25_f32]; + let a: __m128 = transmute(a_array); + let b: __m128 = transmute(b_array); + let k: __mmask8 = 0b1111_1111; + let c: __m128bh = _mm_maskz_cvtne2ps_pbh(k, a, b); + let result: [u16; 8] = transmute(c.as_u16x8()); + #[rustfmt::skip] + let expected_result: [u16; 8] = [ + 0b1_10000110_0110010, + 0b1_10000010_0101000, + 0b1_10000000_1110000, + 0b1_10000100_1001001, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + ]; + assert_eq!(result, expected_result); + let k = 0b0011_1100; + let c = _mm_maskz_cvtne2ps_pbh(k, a, b); + let result: [u16; 8] = transmute(c.as_u16x8()); + #[rustfmt::skip] + let expected_result: [u16; 8] = [ + 0, + 0, + 0b1_10000000_1110000, + 0b1_10000100_1001001, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0, + 0, + ]; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512vl")] + unsafe fn test_mm256_cvtne2ps_pbh() { + #[rustfmt::skip] + let a_array = [ + 178.125_f32, + 10.5_f32, + 3.75_f32, + 50.25_f32, + 16.5_f32, + 255.11_f32, + 1000.158_f32, + 575.575_f32, + ]; + let b_array = [ + -178.125_f32, + -10.5_f32, + -3.75_f32, + -50.25_f32, + -16.5_f32, + -255.11_f32, + -1000.158_f32, + -575.575_f32, + ]; + let a: __m256 = transmute(a_array); + let b: __m256 = transmute(b_array); + let c: __m256bh = _mm256_cvtne2ps_pbh(a, b); + let result: [u16; 16] = transmute(c.as_u16x16()); + #[rustfmt::skip] + let expected_result: [u16; 16] = [ + 0b1_10000110_0110010, + 0b1_10000010_0101000, + 0b1_10000000_1110000, + 0b1_10000100_1001001, + 0b1_10000011_0000100, + 0b1_10000110_1111111, + 0b1_10001000_1111010, + 0b1_10001000_0010000, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000011_0000100, + 0b0_10000110_1111111, + 0b0_10001000_1111010, + 0b0_10001000_0010000, + ]; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512vl")] + unsafe fn test_mm256_mask_cvtne2ps_pbh() { + #[rustfmt::skip] + let a_array = [ + 178.125_f32, + 10.5_f32, + 3.75_f32, + 50.25_f32, + 16.5_f32, + 255.11_f32, + 1000.158_f32, + 575.575_f32, + ]; + let b_array = [ + -178.125_f32, + -10.5_f32, + -3.75_f32, + -50.25_f32, + -16.5_f32, + -255.11_f32, + -1000.158_f32, + -575.575_f32, + ]; + let src_array: [u16; 16] = [ + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + ]; + let src: __m256bh = transmute(src_array); + let a: __m256 = transmute(a_array); + let b: __m256 = transmute(b_array); + let k: __mmask16 = 0xffff; + let c: __m256bh = _mm256_mask_cvtne2ps_pbh(src, k, a, b); + let result: [u16; 16] = transmute(c.as_u16x16()); + #[rustfmt::skip] + let expected_result: [u16; 16] = [ + 0b1_10000110_0110010, + 0b1_10000010_0101000, + 0b1_10000000_1110000, + 0b1_10000100_1001001, + 0b1_10000011_0000100, + 0b1_10000110_1111111, + 0b1_10001000_1111010, + 0b1_10001000_0010000, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000011_0000100, + 0b0_10000110_1111111, + 0b0_10001000_1111010, + 0b0_10001000_0010000, + ]; + assert_eq!(result, expected_result); + let k: __mmask16 = 0; + let c: __m256bh = _mm256_mask_cvtne2ps_pbh(src, k, a, b); + let result: [u16; 16] = transmute(c.as_u16x16()); + let expected_result = src_array; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512vl")] + unsafe fn test_mm256_maskz_cvtne2ps_pbh() { + #[rustfmt::skip] + let a_array = [ + 178.125_f32, + 10.5_f32, + 3.75_f32, + 50.25_f32, + 16.5_f32, + 255.11_f32, + 1000.158_f32, + 575.575_f32, + ]; + let b_array = [ + -178.125_f32, + -10.5_f32, + -3.75_f32, + -50.25_f32, + -16.5_f32, + -255.11_f32, + -1000.158_f32, + -575.575_f32, + ]; + let a: __m256 = transmute(a_array); + let b: __m256 = transmute(b_array); + let k: __mmask16 = 0xffff; + let c: __m256bh = _mm256_maskz_cvtne2ps_pbh(k, a, b); + let result: [u16; 16] = transmute(c.as_u16x16()); + #[rustfmt::skip] + let expected_result: [u16; 16] = [ + 0b1_10000110_0110010, + 0b1_10000010_0101000, + 0b1_10000000_1110000, + 0b1_10000100_1001001, + 0b1_10000011_0000100, + 0b1_10000110_1111111, + 0b1_10001000_1111010, + 0b1_10001000_0010000, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000011_0000100, + 0b0_10000110_1111111, + 0b0_10001000_1111010, + 0b0_10001000_0010000, + ]; + assert_eq!(result, expected_result); + let k: __mmask16 = 0b0110_1100_0011_0110; + let c: __m256bh = _mm256_maskz_cvtne2ps_pbh(k, a, b); + let result: [u16; 16] = transmute(c.as_u16x16()); + #[rustfmt::skip] + let expected_result: [u16; 16] = [ + 0, + 0b1_10000010_0101000, + 0b1_10000000_1110000, + 0, + 0b1_10000011_0000100, + 0b1_10000110_1111111, + 0, + 0, + 0, + 0, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0, + 0b0_10000110_1111111, + 0b0_10001000_1111010, + 0, + ]; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512f")] + unsafe fn test_mm512_cvtne2ps_pbh() { + #[rustfmt::skip] + let a_array = [ + 178.125_f32, + 10.5_f32, + 3.75_f32, + 50.25_f32, + 16.5_f32, + 255.11_f32, + 1000.158_f32, + 575.575_f32, + 178.125_f32, + 10.5_f32, + 3.75_f32, + 50.25_f32, + 16.5_f32, + 255.11_f32, + 1000.158_f32, + 575.575_f32, + ]; + let b_array = [ + -178.125_f32, + -10.5_f32, + -3.75_f32, + -50.25_f32, + -16.5_f32, + -255.11_f32, + -1000.158_f32, + -575.575_f32, + -178.125_f32, + -10.5_f32, + -3.75_f32, + -50.25_f32, + -16.5_f32, + -255.11_f32, + -1000.158_f32, + -575.575_f32, + ]; + let a: __m512 = transmute(a_array); + let b: __m512 = transmute(b_array); + let c: __m512bh = _mm512_cvtne2ps_pbh(a, b); + let result: [u16; 32] = transmute(c.as_u16x32()); + #[rustfmt::skip] + let expected_result: [u16; 32] = [ + 0b1_10000110_0110010, + 0b1_10000010_0101000, + 0b1_10000000_1110000, + 0b1_10000100_1001001, + 0b1_10000011_0000100, + 0b1_10000110_1111111, + 0b1_10001000_1111010, + 0b1_10001000_0010000, + 0b1_10000110_0110010, + 0b1_10000010_0101000, + 0b1_10000000_1110000, + 0b1_10000100_1001001, + 0b1_10000011_0000100, + 0b1_10000110_1111111, + 0b1_10001000_1111010, + 0b1_10001000_0010000, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000011_0000100, + 0b0_10000110_1111111, + 0b0_10001000_1111010, + 0b0_10001000_0010000, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000011_0000100, + 0b0_10000110_1111111, + 0b0_10001000_1111010, + 0b0_10001000_0010000, + ]; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512f")] + unsafe fn test_mm512_mask_cvtne2ps_pbh() { + #[rustfmt::skip] + let a_array = [ + 178.125_f32, + 10.5_f32, + 3.75_f32, + 50.25_f32, + 16.5_f32, + 255.11_f32, + 1000.158_f32, + 575.575_f32, + 178.125_f32, + 10.5_f32, + 3.75_f32, + 50.25_f32, + 16.5_f32, + 255.11_f32, + 1000.158_f32, + 575.575_f32, + ]; + let b_array = [ + -178.125_f32, + -10.5_f32, + -3.75_f32, + -50.25_f32, + -16.5_f32, + -255.11_f32, + -1000.158_f32, + -575.575_f32, + -178.125_f32, + -10.5_f32, + -3.75_f32, + -50.25_f32, + -16.5_f32, + -255.11_f32, + -1000.158_f32, + -575.575_f32, + ]; + let src_array: [u16; 32] = [ + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + ]; + let src: __m512bh = transmute(src_array); + let a: __m512 = transmute(a_array); + let b: __m512 = transmute(b_array); + let k: __mmask32 = 0xffffffff; + let c: __m512bh = _mm512_mask_cvtne2ps_pbh(src, k, a, b); + let result: [u16; 32] = transmute(c.as_u16x32()); + #[rustfmt::skip] + let expected_result: [u16; 32] = [ + 0b1_10000110_0110010, + 0b1_10000010_0101000, + 0b1_10000000_1110000, + 0b1_10000100_1001001, + 0b1_10000011_0000100, + 0b1_10000110_1111111, + 0b1_10001000_1111010, + 0b1_10001000_0010000, + 0b1_10000110_0110010, + 0b1_10000010_0101000, + 0b1_10000000_1110000, + 0b1_10000100_1001001, + 0b1_10000011_0000100, + 0b1_10000110_1111111, + 0b1_10001000_1111010, + 0b1_10001000_0010000, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000011_0000100, + 0b0_10000110_1111111, + 0b0_10001000_1111010, + 0b0_10001000_0010000, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000011_0000100, + 0b0_10000110_1111111, + 0b0_10001000_1111010, + 0b0_10001000_0010000, + ]; + assert_eq!(result, expected_result); + let k: __mmask32 = 0; + let c: __m512bh = _mm512_mask_cvtne2ps_pbh(src, k, a, b); + let result: [u16; 32] = transmute(c.as_u16x32()); + let expected_result = src_array; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512f")] + unsafe fn test_mm512_maskz_cvtne2ps_pbh() { + #[rustfmt::skip] + let a_array = [ + 178.125_f32, + 10.5_f32, + 3.75_f32, + 50.25_f32, + 16.5_f32, + 255.11_f32, + 1000.158_f32, + 575.575_f32, + 178.125_f32, + 10.5_f32, + 3.75_f32, + 50.25_f32, + 16.5_f32, + 255.11_f32, + 1000.158_f32, + 575.575_f32, + ]; + let b_array = [ + -178.125_f32, + -10.5_f32, + -3.75_f32, + -50.25_f32, + -16.5_f32, + -255.11_f32, + -1000.158_f32, + -575.575_f32, + -178.125_f32, + -10.5_f32, + -3.75_f32, + -50.25_f32, + -16.5_f32, + -255.11_f32, + -1000.158_f32, + -575.575_f32, + ]; + let a: __m512 = transmute(a_array); + let b: __m512 = transmute(b_array); + let k: __mmask32 = 0xffffffff; + let c: __m512bh = _mm512_maskz_cvtne2ps_pbh(k, a, b); + let result: [u16; 32] = transmute(c.as_u16x32()); + #[rustfmt::skip] + let expected_result: [u16; 32] = [ + 0b1_10000110_0110010, + 0b1_10000010_0101000, + 0b1_10000000_1110000, + 0b1_10000100_1001001, + 0b1_10000011_0000100, + 0b1_10000110_1111111, + 0b1_10001000_1111010, + 0b1_10001000_0010000, + 0b1_10000110_0110010, + 0b1_10000010_0101000, + 0b1_10000000_1110000, + 0b1_10000100_1001001, + 0b1_10000011_0000100, + 0b1_10000110_1111111, + 0b1_10001000_1111010, + 0b1_10001000_0010000, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000011_0000100, + 0b0_10000110_1111111, + 0b0_10001000_1111010, + 0b0_10001000_0010000, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000011_0000100, + 0b0_10000110_1111111, + 0b0_10001000_1111010, + 0b0_10001000_0010000, + ]; + assert_eq!(result, expected_result); + let k: __mmask32 = 0b1100_1010_1001_0110_1010_0011_0101_0110; + let c: __m512bh = _mm512_maskz_cvtne2ps_pbh(k, a, b); + let result: [u16; 32] = transmute(c.as_u16x32()); + #[rustfmt::skip] + let expected_result: [u16; 32] = [ + 0, + 0b1_10000010_0101000, + 0b1_10000000_1110000, + 0, + 0b1_10000011_0000100, + 0, + 0b1_10001000_1111010, + 0, + 0b1_10000110_0110010, + 0b1_10000010_0101000, + 0, + 0, + 0, + 0b1_10000110_1111111, + 0, + 0b1_10001000_0010000, + 0, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0, + 0b0_10000011_0000100, + 0, + 0, + 0b0_10001000_0010000, + 0, + 0b0_10000010_0101000, + 0, + 0b0_10000100_1001001, + 0, + 0, + 0b0_10001000_1111010, + 0b0_10001000_0010000, + ]; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512vl")] + unsafe fn test_mm256_cvtneps_pbh() { + #[rustfmt::skip] + let a_array = [ + 178.125_f32, + 10.5_f32, + 3.75_f32, + 50.25_f32, + 16.5_f32, + 255.11_f32, + 1000.158_f32, + 575.575_f32, + ]; + let a: __m256 = transmute(a_array); + let c: __m128bh = _mm256_cvtneps_pbh(a); + let result: [u16; 8] = transmute(c.as_u16x8()); + #[rustfmt::skip] + let expected_result: [u16; 8] = [ + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000011_0000100, + 0b0_10000110_1111111, + 0b0_10001000_1111010, + 0b0_10001000_0010000, + ]; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512vl")] + unsafe fn test_mm256_mask_cvtneps_pbh() { + #[rustfmt::skip] + let a_array = [ + 178.125_f32, + 10.5_f32, + 3.75_f32, + 50.25_f32, + 16.5_f32, + 255.11_f32, + 1000.158_f32, + 575.575_f32, + ]; + let src_array: [u16; 8] = [ + 0b1_10000110_0110010, + 0b1_10000010_0101000, + 0b1_10000000_1110000, + 0b1_10000100_1001001, + 0b1_10000011_0000100, + 0b1_10000110_1111111, + 0b1_10001000_1111010, + 0b1_10001000_0010000, + ]; + let src: __m128bh = transmute(src_array); + let a: __m256 = transmute(a_array); + let k: __mmask8 = 0xff; + let b = _mm256_mask_cvtneps_pbh(src, k, a); + let result: [u16; 8] = transmute(b.as_u16x8()); + #[rustfmt::skip] + let expected_result: [u16; 8] = [ + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000011_0000100, + 0b0_10000110_1111111, + 0b0_10001000_1111010, + 0b0_10001000_0010000, + ]; + assert_eq!(result, expected_result); + let k: __mmask8 = 0x0; + let b: __m128bh = _mm256_mask_cvtneps_pbh(src, k, a); + let result: [u16; 8] = transmute(b.as_u16x8()); + let expected_result: [u16; 8] = src_array; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512vl")] + unsafe fn test_mm256_maskz_cvtneps_pbh() { + #[rustfmt::skip] + let a_array = [ + 178.125_f32, + 10.5_f32, + 3.75_f32, + 50.25_f32, + 16.5_f32, + 255.11_f32, + 1000.158_f32, + 575.575_f32, + ]; + let a: __m256 = transmute(a_array); + let k: __mmask8 = 0xff; + let b = _mm256_maskz_cvtneps_pbh(k, a); + let result: [u16; 8] = transmute(b.as_u16x8()); + #[rustfmt::skip] + let expected_result: [u16; 8] = [ + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000011_0000100, + 0b0_10000110_1111111, + 0b0_10001000_1111010, + 0b0_10001000_0010000, + ]; + assert_eq!(result, expected_result); + let k: __mmask8 = 0x6; + let b: __m128bh = _mm256_maskz_cvtneps_pbh(k, a); + let result: [u16; 8] = transmute(b.as_u16x8()); + let expected_result: [u16; 8] = + [0, 0b0_10000010_0101000, 0b0_10000000_1110000, 0, 0, 0, 0, 0]; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512f")] + unsafe fn test_mm512_cvtneps_pbh() { + #[rustfmt::skip] + let a_array = [ + 178.125_f32, + 10.5_f32, + 3.75_f32, + 50.25_f32, + 16.5_f32, + 255.11_f32, + 1000.158_f32, + 575.575_f32, + 178.125_f32, + 10.5_f32, + 3.75_f32, + 50.25_f32, + 16.5_f32, + 255.11_f32, + 1000.158_f32, + 575.575_f32, + ]; + let a: __m512 = transmute(a_array); + let c: __m256bh = _mm512_cvtneps_pbh(a); + let result: [u16; 16] = transmute(c.as_u16x16()); + #[rustfmt::skip] + let expected_result: [u16; 16] = [ + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000011_0000100, + 0b0_10000110_1111111, + 0b0_10001000_1111010, + 0b0_10001000_0010000, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000011_0000100, + 0b0_10000110_1111111, + 0b0_10001000_1111010, + 0b0_10001000_0010000, + ]; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512f")] + unsafe fn test_mm512_mask_cvtneps_pbh() { + #[rustfmt::skip] + let a_array = [ + 178.125_f32, + 10.5_f32, + 3.75_f32, + 50.25_f32, + 16.5_f32, + 255.11_f32, + 1000.158_f32, + 575.575_f32, + 178.125_f32, + 10.5_f32, + 3.75_f32, + 50.25_f32, + 16.5_f32, + 255.11_f32, + 1000.158_f32, + 575.575_f32, + ]; + let src_array: [u16; 16] = [ + 0b1_10000110_0110010, + 0b1_10000010_0101000, + 0b1_10000000_1110000, + 0b1_10000100_1001001, + 0b1_10000011_0000100, + 0b1_10000110_1111111, + 0b1_10001000_1111010, + 0b1_10001000_0010000, + 0b1_10000110_0110010, + 0b1_10000010_0101000, + 0b1_10000000_1110000, + 0b1_10000100_1001001, + 0b1_10000011_0000100, + 0b1_10000110_1111111, + 0b1_10001000_1111010, + 0b1_10001000_0010000, + ]; + let src: __m256bh = transmute(src_array); + let a: __m512 = transmute(a_array); + let k: __mmask16 = 0xffff; + let c: __m256bh = _mm512_mask_cvtneps_pbh(src, k, a); + let result: [u16; 16] = transmute(c.as_u16x16()); + #[rustfmt::skip] + let expected_result: [u16; 16] = [ + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000011_0000100, + 0b0_10000110_1111111, + 0b0_10001000_1111010, + 0b0_10001000_0010000, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000011_0000100, + 0b0_10000110_1111111, + 0b0_10001000_1111010, + 0b0_10001000_0010000, + ]; + assert_eq!(result, expected_result); + let k: __mmask16 = 0; + let c: __m256bh = _mm512_mask_cvtneps_pbh(src, k, a); + let result: [u16; 16] = transmute(c.as_u16x16()); + let expected_result = src_array; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512f")] + unsafe fn test_mm512_maskz_cvtneps_pbh() { + #[rustfmt::skip] + let a_array = [ + 178.125_f32, + 10.5_f32, + 3.75_f32, + 50.25_f32, + 16.5_f32, + 255.11_f32, + 1000.158_f32, + 575.575_f32, + 178.125_f32, + 10.5_f32, + 3.75_f32, + 50.25_f32, + 16.5_f32, + 255.11_f32, + 1000.158_f32, + 575.575_f32, + ]; + let a: __m512 = transmute(a_array); + let k: __mmask16 = 0xffff; + let c: __m256bh = _mm512_maskz_cvtneps_pbh(k, a); + let result: [u16; 16] = transmute(c.as_u16x16()); + #[rustfmt::skip] + let expected_result: [u16; 16] = [ + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000011_0000100, + 0b0_10000110_1111111, + 0b0_10001000_1111010, + 0b0_10001000_0010000, + 0b0_10000110_0110010, + 0b0_10000010_0101000, + 0b0_10000000_1110000, + 0b0_10000100_1001001, + 0b0_10000011_0000100, + 0b0_10000110_1111111, + 0b0_10001000_1111010, + 0b0_10001000_0010000, + ]; + assert_eq!(result, expected_result); + let k: __mmask16 = 0x653a; + let c: __m256bh = _mm512_maskz_cvtneps_pbh(k, a); + let result: [u16; 16] = transmute(c.as_u16x16()); + #[rustfmt::skip] + let expected_result: [u16; 16] = [ + 0, + 0b0_10000010_0101000, + 0, + 0b0_10000100_1001001, + 0b0_10000011_0000100, + 0b0_10000110_1111111, + 0, + 0, + 0b0_10000110_0110010, + 0, + 0b0_10000000_1110000, + 0, + 0, + 0b0_10000110_1111111, + 0b0_10001000_1111010, + 0, + ]; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512vl")] + unsafe fn test_mm_dpbf16_ps() { + let a_array = [8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32]; + let b_array = [-1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32]; + let a1: __m128 = transmute(a_array); + let b1: __m128 = transmute(b_array); + let src: __m128 = transmute([1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32]); + let a: __m128bh = _mm_cvtne2ps_pbh(a1, a1); + let b: __m128bh = _mm_cvtne2ps_pbh(b1, b1); + let c: __m128 = _mm_dpbf16_ps(src, a, b); + let result: [f32; 4] = transmute(c.as_f32x4()); + let expected_result: [f32; 4] = [-18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32]; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512vl")] + unsafe fn test_mm_mask_dpbf16_ps() { + let a_array = [8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32]; + let b_array = [-1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32]; + let a1: __m128 = transmute(a_array); + let b1: __m128 = transmute(b_array); + let k: __mmask8 = 0xf3; + let src: __m128 = transmute([1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32]); + let a: __m128bh = _mm_cvtne2ps_pbh(a1, a1); + let b: __m128bh = _mm_cvtne2ps_pbh(b1, b1); + let c: __m128 = _mm_mask_dpbf16_ps(src, k, a, b); + let result: [f32; 4] = transmute(c.as_f32x4()); + let expected_result: [f32; 4] = [-18.0_f32, -52.0_f32, 3.0_f32, 4.0_f32]; + assert_eq!(result, expected_result); + let k: __mmask8 = 0xff; + let c: __m128 = _mm_mask_dpbf16_ps(src, k, a, b); + let result: [f32; 4] = transmute(c.as_f32x4()); + let expected_result: [f32; 4] = [-18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32]; + assert_eq!(result, expected_result); + let k: __mmask8 = 0; + let c: __m128 = _mm_mask_dpbf16_ps(src, k, a, b); + let result: [f32; 4] = transmute(c.as_f32x4()); + let expected_result: [f32; 4] = [1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32]; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512vl")] + unsafe fn test_mm_maskz_dpbf16_ps() { + let a_array = [8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32]; + let b_array = [-1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32]; + let a1: __m128 = transmute(a_array); + let b1: __m128 = transmute(b_array); + let k: __mmask8 = 0xf3; + let src: __m128 = transmute([1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32]); + let a: __m128bh = _mm_cvtne2ps_pbh(a1, a1); + let b: __m128bh = _mm_cvtne2ps_pbh(b1, b1); + let c: __m128 = _mm_maskz_dpbf16_ps(k, src, a, b); + let result: [f32; 4] = transmute(c.as_f32x4()); + let expected_result: [f32; 4] = [-18.0_f32, -52.0_f32, 0.0, 0.0]; + assert_eq!(result, expected_result); + let k: __mmask8 = 0xff; + let c: __m128 = _mm_maskz_dpbf16_ps(k, src, a, b); + let result: [f32; 4] = transmute(c.as_f32x4()); + let expected_result: [f32; 4] = [-18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32]; + assert_eq!(result, expected_result); + let k: __mmask8 = 0; + let c: __m128 = _mm_maskz_dpbf16_ps(k, src, a, b); + let result: [f32; 4] = transmute(c.as_f32x4()); + let expected_result: [f32; 4] = [0.0, 0.0, 0.0, 0.0]; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512vl")] + unsafe fn test_mm256_dpbf16_ps() { + #[rustfmt::skip] + let a_array = [ + 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, + ]; + let b_array = [ + -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, + ]; + let a1: __m256 = transmute(a_array); + let b1: __m256 = transmute(b_array); + #[rustfmt::skip] + let src: __m256 = transmute([ + 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, + ]); + let a: __m256bh = _mm256_cvtne2ps_pbh(a1, a1); + let b: __m256bh = _mm256_cvtne2ps_pbh(b1, b1); + let c: __m256 = _mm256_dpbf16_ps(src, a, b); + let result: [f32; 8] = transmute(c.as_f32x8()); + #[rustfmt::skip] + let expected_result: [f32; 8] = [ + -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, + ]; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512vl")] + unsafe fn test_mm256_mask_dpbf16_ps() { + #[rustfmt::skip] + let a_array = [ + 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, + ]; + let b_array = [ + -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, + ]; + let a1: __m256 = transmute(a_array); + let b1: __m256 = transmute(b_array); + let k: __mmask8 = 0x33; + #[rustfmt::skip] + let src: __m256 = transmute([ + 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, + ]); + let a: __m256bh = _mm256_cvtne2ps_pbh(a1, a1); + let b: __m256bh = _mm256_cvtne2ps_pbh(b1, b1); + let c: __m256 = _mm256_mask_dpbf16_ps(src, k, a, b); + let result: [f32; 8] = transmute(c.as_f32x8()); + #[rustfmt::skip] + let expected_result: [f32; 8] = [ + -18.0_f32, -52.0_f32, 3.0_f32, 4.0_f32, -18.0_f32, -52.0_f32, 3.0_f32, 4.0_f32, + ]; + assert_eq!(result, expected_result); + let k: __mmask8 = 0xff; + let c: __m256 = _mm256_mask_dpbf16_ps(src, k, a, b); + let result: [f32; 8] = transmute(c.as_f32x8()); + #[rustfmt::skip] + let expected_result: [f32; 8] = [ + -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, + ]; + assert_eq!(result, expected_result); + let k: __mmask8 = 0; + let c: __m256 = _mm256_mask_dpbf16_ps(src, k, a, b); + let result: [f32; 8] = transmute(c.as_f32x8()); + #[rustfmt::skip] + let expected_result: [f32; 8] = [ + 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, + ]; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512vl")] + unsafe fn test_mm256_maskz_dpbf16_ps() { + #[rustfmt::skip] + let a_array = [ + 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, + ]; + let b_array = [ + -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, + ]; + let a1: __m256 = transmute(a_array); + let b1: __m256 = transmute(b_array); + let k: __mmask8 = 0x33; + #[rustfmt::skip] + let src: __m256 = transmute([ + 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, + ]); + let a: __m256bh = _mm256_cvtne2ps_pbh(a1, a1); + let b: __m256bh = _mm256_cvtne2ps_pbh(b1, b1); + let c: __m256 = _mm256_maskz_dpbf16_ps(k, src, a, b); + let result: [f32; 8] = transmute(c.as_f32x8()); + #[rustfmt::skip] + let expected_result: [f32; 8] = [ + -18.0_f32, -52.0_f32, 0.0, 0.0, -18.0_f32, -52.0_f32, 0.0, 0.0, + ]; + assert_eq!(result, expected_result); + let k: __mmask8 = 0xff; + let c: __m256 = _mm256_maskz_dpbf16_ps(k, src, a, b); + let result: [f32; 8] = transmute(c.as_f32x8()); + #[rustfmt::skip] + let expected_result: [f32; 8] = [ + -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, + ]; + assert_eq!(result, expected_result); + let k: __mmask8 = 0; + let c: __m256 = _mm256_maskz_dpbf16_ps(k, src, a, b); + let result: [f32; 8] = transmute(c.as_f32x8()); + let expected_result: [f32; 8] = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512f")] + unsafe fn test_mm512_dpbf16_ps() { + #[rustfmt::skip] + let a_array = [ + 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, + 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, + ]; + let b_array = [ + -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, + -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, + ]; + let a1: __m512 = transmute(a_array); + let b1: __m512 = transmute(b_array); + let src: __m512 = transmute([ + 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, + 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, + ]); + let a: __m512bh = _mm512_cvtne2ps_pbh(a1, a1); + let b: __m512bh = _mm512_cvtne2ps_pbh(b1, b1); + let c: __m512 = _mm512_dpbf16_ps(src, a, b); + let result: [f32; 16] = transmute(c.as_f32x16()); + #[rustfmt::skip] + let expected_result: [f32; 16] = [ + -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, + -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, + ]; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512f")] + unsafe fn test_mm512_mask_dpbf16_ps() { + #[rustfmt::skip] + let a_array = [ + 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, + 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, + ]; + let b_array = [ + -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, + -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, + ]; + let a1: __m512 = transmute(a_array); + let b1: __m512 = transmute(b_array); + let k: __mmask16 = 0x3333; + #[rustfmt::skip] + let src: __m512 = transmute([ + 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, + 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, + ]); + let a: __m512bh = _mm512_cvtne2ps_pbh(a1, a1); + let b: __m512bh = _mm512_cvtne2ps_pbh(b1, b1); + let c: __m512 = _mm512_mask_dpbf16_ps(src, k, a, b); + let result: [f32; 16] = transmute(c.as_f32x16()); + #[rustfmt::skip] + let expected_result: [f32; 16] = [ + -18.0_f32, -52.0_f32, 3.0_f32, 4.0_f32, -18.0_f32, -52.0_f32, 3.0_f32, 4.0_f32, + -18.0_f32, -52.0_f32, 3.0_f32, 4.0_f32, -18.0_f32, -52.0_f32, 3.0_f32, 4.0_f32, + ]; + assert_eq!(result, expected_result); + let k: __mmask16 = 0xffff; + let c: __m512 = _mm512_mask_dpbf16_ps(src, k, a, b); + let result: [f32; 16] = transmute(c.as_f32x16()); + #[rustfmt::skip] + let expected_result: [f32; 16] = [ + -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, + -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, + ]; + assert_eq!(result, expected_result); + let k: __mmask16 = 0; + let c: __m512 = _mm512_mask_dpbf16_ps(src, k, a, b); + let result: [f32; 16] = transmute(c.as_f32x16()); + #[rustfmt::skip] + let expected_result: [f32; 16] = [ + 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, + 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, + ]; + assert_eq!(result, expected_result); + } + + #[simd_test(enable = "avx512bf16,avx512f")] + unsafe fn test_mm512_maskz_dpbf16_ps() { + #[rustfmt::skip] + let a_array = [ + 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, + 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, + ]; + let b_array = [ + -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, + -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, + ]; + let a1: __m512 = transmute(a_array); + let b1: __m512 = transmute(b_array); + let k: __mmask16 = 0x3333; + #[rustfmt::skip] + let src: __m512 = transmute([ + 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, + 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, + ]); + let a: __m512bh = _mm512_cvtne2ps_pbh(a1, a1); + let b: __m512bh = _mm512_cvtne2ps_pbh(b1, b1); + let c: __m512 = _mm512_maskz_dpbf16_ps(k, src, a, b); + let result: [f32; 16] = transmute(c.as_f32x16()); + #[rustfmt::skip] + let expected_result: [f32; 16] = [ + -18.0_f32, -52.0_f32, 0.0, 0.0, -18.0_f32, -52.0_f32, 0.0, 0.0, -18.0_f32, -52.0_f32, + 0.0, 0.0, -18.0_f32, -52.0_f32, 0.0, 0.0, + ]; + assert_eq!(result, expected_result); + let k: __mmask16 = 0xffff; + let c: __m512 = _mm512_maskz_dpbf16_ps(k, src, a, b); + let result: [f32; 16] = transmute(c.as_f32x16()); + #[rustfmt::skip] + let expected_result: [f32; 16] = [ + -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, + -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, + ]; + assert_eq!(result, expected_result); + let k: __mmask16 = 0; + let c: __m512 = _mm512_maskz_dpbf16_ps(k, src, a, b); + let result: [f32; 16] = transmute(c.as_f32x16()); + #[rustfmt::skip] + let expected_result: [f32; 16] = [ + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + ]; + assert_eq!(result, expected_result); + } +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512bitalg.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512bitalg.rs new file mode 100644 index 000000000..3c9df3912 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512bitalg.rs @@ -0,0 +1,760 @@ +//! Bit-oriented Algorithms (BITALG) +//! +//! The intrinsics here correspond to those in the `immintrin.h` C header. +//! +//! The reference is [Intel 64 and IA-32 Architectures Software Developer's +//! Manual Volume 2: Instruction Set Reference, A-Z][intel64_ref]. +//! +//! [intel64_ref]: http://www.intel.de/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf + +use crate::core_arch::simd::i16x16; +use crate::core_arch::simd::i16x32; +use crate::core_arch::simd::i16x8; +use crate::core_arch::simd::i8x16; +use crate::core_arch::simd::i8x32; +use crate::core_arch::simd::i8x64; +use crate::core_arch::simd_llvm::simd_select_bitmask; +use crate::core_arch::x86::__m128i; +use crate::core_arch::x86::__m256i; +use crate::core_arch::x86::__m512i; +use crate::core_arch::x86::__mmask16; +use crate::core_arch::x86::__mmask32; +use crate::core_arch::x86::__mmask64; +use crate::core_arch::x86::__mmask8; +use crate::core_arch::x86::_mm256_setzero_si256; +use crate::core_arch::x86::_mm512_setzero_si512; +use crate::core_arch::x86::_mm_setzero_si128; +use crate::core_arch::x86::m128iExt; +use crate::core_arch::x86::m256iExt; +use crate::core_arch::x86::m512iExt; +use crate::mem::transmute; + +#[cfg(test)] +use stdarch_test::assert_instr; + +#[allow(improper_ctypes)] +extern "C" { + #[link_name = "llvm.ctpop.v32i16"] + fn popcnt_v32i16(x: i16x32) -> i16x32; + #[link_name = "llvm.ctpop.v16i16"] + fn popcnt_v16i16(x: i16x16) -> i16x16; + #[link_name = "llvm.ctpop.v8i16"] + fn popcnt_v8i16(x: i16x8) -> i16x8; + + #[link_name = "llvm.ctpop.v64i8"] + fn popcnt_v64i8(x: i8x64) -> i8x64; + #[link_name = "llvm.ctpop.v32i8"] + fn popcnt_v32i8(x: i8x32) -> i8x32; + #[link_name = "llvm.ctpop.v16i8"] + fn popcnt_v16i8(x: i8x16) -> i8x16; + + #[link_name = "llvm.x86.avx512.mask.vpshufbitqmb.512"] + fn bitshuffle_512(data: i8x64, indices: i8x64, mask: __mmask64) -> __mmask64; + #[link_name = "llvm.x86.avx512.mask.vpshufbitqmb.256"] + fn bitshuffle_256(data: i8x32, indices: i8x32, mask: __mmask32) -> __mmask32; + #[link_name = "llvm.x86.avx512.mask.vpshufbitqmb.128"] + fn bitshuffle_128(data: i8x16, indices: i8x16, mask: __mmask16) -> __mmask16; +} + +/// For each packed 16-bit integer maps the value to the number of logical 1 bits. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_popcnt_epi16) +#[inline] +#[target_feature(enable = "avx512bitalg")] +#[cfg_attr(test, assert_instr(vpopcntw))] +pub unsafe fn _mm512_popcnt_epi16(a: __m512i) -> __m512i { + transmute(popcnt_v32i16(a.as_i16x32())) +} + +/// For each packed 16-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_popcnt_epi16) +#[inline] +#[target_feature(enable = "avx512bitalg")] +#[cfg_attr(test, assert_instr(vpopcntw))] +pub unsafe fn _mm512_maskz_popcnt_epi16(k: __mmask32, a: __m512i) -> __m512i { + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, popcnt_v32i16(a.as_i16x32()), zero)) +} + +/// For each packed 16-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are copied from src if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_popcnt_epi16) +#[inline] +#[target_feature(enable = "avx512bitalg")] +#[cfg_attr(test, assert_instr(vpopcntw))] +pub unsafe fn _mm512_mask_popcnt_epi16(src: __m512i, k: __mmask32, a: __m512i) -> __m512i { + transmute(simd_select_bitmask( + k, + popcnt_v32i16(a.as_i16x32()), + src.as_i16x32(), + )) +} + +/// For each packed 16-bit integer maps the value to the number of logical 1 bits. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_popcnt_epi16) +#[inline] +#[target_feature(enable = "avx512bitalg,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntw))] +pub unsafe fn _mm256_popcnt_epi16(a: __m256i) -> __m256i { + transmute(popcnt_v16i16(a.as_i16x16())) +} + +/// For each packed 16-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_popcnt_epi16) +#[inline] +#[target_feature(enable = "avx512bitalg,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntw))] +pub unsafe fn _mm256_maskz_popcnt_epi16(k: __mmask16, a: __m256i) -> __m256i { + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, popcnt_v16i16(a.as_i16x16()), zero)) +} + +/// For each packed 16-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are copied from src if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_popcnt_epi16) +#[inline] +#[target_feature(enable = "avx512bitalg,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntw))] +pub unsafe fn _mm256_mask_popcnt_epi16(src: __m256i, k: __mmask16, a: __m256i) -> __m256i { + transmute(simd_select_bitmask( + k, + popcnt_v16i16(a.as_i16x16()), + src.as_i16x16(), + )) +} + +/// For each packed 16-bit integer maps the value to the number of logical 1 bits. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_popcnt_epi16) +#[inline] +#[target_feature(enable = "avx512bitalg,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntw))] +pub unsafe fn _mm_popcnt_epi16(a: __m128i) -> __m128i { + transmute(popcnt_v8i16(a.as_i16x8())) +} + +/// For each packed 16-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_popcnt_epi16) +#[inline] +#[target_feature(enable = "avx512bitalg,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntw))] +pub unsafe fn _mm_maskz_popcnt_epi16(k: __mmask8, a: __m128i) -> __m128i { + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, popcnt_v8i16(a.as_i16x8()), zero)) +} + +/// For each packed 16-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are copied from src if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_popcnt_epi16) +#[inline] +#[target_feature(enable = "avx512bitalg,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntw))] +pub unsafe fn _mm_mask_popcnt_epi16(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(simd_select_bitmask( + k, + popcnt_v8i16(a.as_i16x8()), + src.as_i16x8(), + )) +} + +/// For each packed 8-bit integer maps the value to the number of logical 1 bits. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_popcnt_epi8) +#[inline] +#[target_feature(enable = "avx512bitalg")] +#[cfg_attr(test, assert_instr(vpopcntb))] +pub unsafe fn _mm512_popcnt_epi8(a: __m512i) -> __m512i { + transmute(popcnt_v64i8(a.as_i8x64())) +} + +/// For each packed 8-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_popcnt_epi8) +#[inline] +#[target_feature(enable = "avx512bitalg")] +#[cfg_attr(test, assert_instr(vpopcntb))] +pub unsafe fn _mm512_maskz_popcnt_epi8(k: __mmask64, a: __m512i) -> __m512i { + let zero = _mm512_setzero_si512().as_i8x64(); + transmute(simd_select_bitmask(k, popcnt_v64i8(a.as_i8x64()), zero)) +} + +/// For each packed 8-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are copied from src if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_popcnt_epi8) +#[inline] +#[target_feature(enable = "avx512bitalg")] +#[cfg_attr(test, assert_instr(vpopcntb))] +pub unsafe fn _mm512_mask_popcnt_epi8(src: __m512i, k: __mmask64, a: __m512i) -> __m512i { + transmute(simd_select_bitmask( + k, + popcnt_v64i8(a.as_i8x64()), + src.as_i8x64(), + )) +} + +/// For each packed 8-bit integer maps the value to the number of logical 1 bits. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_popcnt_epi8) +#[inline] +#[target_feature(enable = "avx512bitalg,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntb))] +pub unsafe fn _mm256_popcnt_epi8(a: __m256i) -> __m256i { + transmute(popcnt_v32i8(a.as_i8x32())) +} + +/// For each packed 8-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_popcnt_epi8) +#[inline] +#[target_feature(enable = "avx512bitalg,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntb))] +pub unsafe fn _mm256_maskz_popcnt_epi8(k: __mmask32, a: __m256i) -> __m256i { + let zero = _mm256_setzero_si256().as_i8x32(); + transmute(simd_select_bitmask(k, popcnt_v32i8(a.as_i8x32()), zero)) +} + +/// For each packed 8-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are copied from src if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_popcnt_epi8) +#[inline] +#[target_feature(enable = "avx512bitalg,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntb))] +pub unsafe fn _mm256_mask_popcnt_epi8(src: __m256i, k: __mmask32, a: __m256i) -> __m256i { + transmute(simd_select_bitmask( + k, + popcnt_v32i8(a.as_i8x32()), + src.as_i8x32(), + )) +} + +/// For each packed 8-bit integer maps the value to the number of logical 1 bits. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_popcnt_epi8) +#[inline] +#[target_feature(enable = "avx512bitalg,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntb))] +pub unsafe fn _mm_popcnt_epi8(a: __m128i) -> __m128i { + transmute(popcnt_v16i8(a.as_i8x16())) +} + +/// For each packed 8-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_popcnt_epi8) +#[inline] +#[target_feature(enable = "avx512bitalg,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntb))] +pub unsafe fn _mm_maskz_popcnt_epi8(k: __mmask16, a: __m128i) -> __m128i { + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask(k, popcnt_v16i8(a.as_i8x16()), zero)) +} + +/// For each packed 8-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are copied from src if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_popcnt_epi8) +#[inline] +#[target_feature(enable = "avx512bitalg,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntb))] +pub unsafe fn _mm_mask_popcnt_epi8(src: __m128i, k: __mmask16, a: __m128i) -> __m128i { + transmute(simd_select_bitmask( + k, + popcnt_v16i8(a.as_i8x16()), + src.as_i8x16(), + )) +} + +/// Considers the input `b` as packed 64-bit integers and `c` as packed 8-bit integers. +/// Then groups 8 8-bit values from `c`as indices into the the bits of the corresponding 64-bit integer. +/// It then selects these bits and packs them into the output. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_bitshuffle_epi64_mask) +#[inline] +#[target_feature(enable = "avx512bitalg")] +#[cfg_attr(test, assert_instr(vpshufbitqmb))] +pub unsafe fn _mm512_bitshuffle_epi64_mask(b: __m512i, c: __m512i) -> __mmask64 { + transmute(bitshuffle_512(b.as_i8x64(), c.as_i8x64(), !0)) +} + +/// Considers the input `b` as packed 64-bit integers and `c` as packed 8-bit integers. +/// Then groups 8 8-bit values from `c`as indices into the the bits of the corresponding 64-bit integer. +/// It then selects these bits and packs them into the output. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_bitshuffle_epi64_mask) +#[inline] +#[target_feature(enable = "avx512bitalg")] +#[cfg_attr(test, assert_instr(vpshufbitqmb))] +pub unsafe fn _mm512_mask_bitshuffle_epi64_mask(k: __mmask64, b: __m512i, c: __m512i) -> __mmask64 { + transmute(bitshuffle_512(b.as_i8x64(), c.as_i8x64(), k)) +} + +/// Considers the input `b` as packed 64-bit integers and `c` as packed 8-bit integers. +/// Then groups 8 8-bit values from `c`as indices into the the bits of the corresponding 64-bit integer. +/// It then selects these bits and packs them into the output. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_bitshuffle_epi64_mask) +#[inline] +#[target_feature(enable = "avx512bitalg,avx512vl")] +#[cfg_attr(test, assert_instr(vpshufbitqmb))] +pub unsafe fn _mm256_bitshuffle_epi64_mask(b: __m256i, c: __m256i) -> __mmask32 { + transmute(bitshuffle_256(b.as_i8x32(), c.as_i8x32(), !0)) +} + +/// Considers the input `b` as packed 64-bit integers and `c` as packed 8-bit integers. +/// Then groups 8 8-bit values from `c`as indices into the the bits of the corresponding 64-bit integer. +/// It then selects these bits and packs them into the output. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_bitshuffle_epi64_mask) +#[inline] +#[target_feature(enable = "avx512bitalg,avx512vl")] +#[cfg_attr(test, assert_instr(vpshufbitqmb))] +pub unsafe fn _mm256_mask_bitshuffle_epi64_mask(k: __mmask32, b: __m256i, c: __m256i) -> __mmask32 { + transmute(bitshuffle_256(b.as_i8x32(), c.as_i8x32(), k)) +} + +/// Considers the input `b` as packed 64-bit integers and `c` as packed 8-bit integers. +/// Then groups 8 8-bit values from `c`as indices into the the bits of the corresponding 64-bit integer. +/// It then selects these bits and packs them into the output. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_bitshuffle_epi64_mask) +#[inline] +#[target_feature(enable = "avx512bitalg,avx512vl")] +#[cfg_attr(test, assert_instr(vpshufbitqmb))] +pub unsafe fn _mm_bitshuffle_epi64_mask(b: __m128i, c: __m128i) -> __mmask16 { + transmute(bitshuffle_128(b.as_i8x16(), c.as_i8x16(), !0)) +} + +/// Considers the input `b` as packed 64-bit integers and `c` as packed 8-bit integers. +/// Then groups 8 8-bit values from `c`as indices into the the bits of the corresponding 64-bit integer. +/// It then selects these bits and packs them into the output. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_bitshuffle_epi64_mask) +#[inline] +#[target_feature(enable = "avx512bitalg,avx512vl")] +#[cfg_attr(test, assert_instr(vpshufbitqmb))] +pub unsafe fn _mm_mask_bitshuffle_epi64_mask(k: __mmask16, b: __m128i, c: __m128i) -> __mmask16 { + transmute(bitshuffle_128(b.as_i8x16(), c.as_i8x16(), k)) +} + +#[cfg(test)] +mod tests { + // Some of the constants in the tests below are just bit patterns. They should not + // be interpreted as integers; signedness does not make sense for them, but + // __mXXXi happens to be defined in terms of signed integers. + #![allow(overflowing_literals)] + + use stdarch_test::simd_test; + + use crate::core_arch::x86::*; + + #[simd_test(enable = "avx512bitalg,avx512f")] + unsafe fn test_mm512_popcnt_epi16() { + let test_data = _mm512_set_epi16( + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1_FF, 0x3_FF, 0x7_FF, 0xF_FF, 0x1F_FF, + 0x3F_FF, 0x7F_FF, 0xFF_FF, -1, -100, 255, 256, 2, 4, 8, 16, 32, 64, 128, 256, 512, + 1024, 2048, + ); + let actual_result = _mm512_popcnt_epi16(test_data); + let reference_result = _mm512_set_epi16( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 12, 8, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, + ); + assert_eq_m512i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f")] + unsafe fn test_mm512_maskz_popcnt_epi16() { + let test_data = _mm512_set_epi16( + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1_FF, 0x3_FF, 0x7_FF, 0xF_FF, 0x1F_FF, + 0x3F_FF, 0x7F_FF, 0xFF_FF, -1, -100, 255, 256, 2, 4, 8, 16, 32, 64, 128, 256, 512, + 1024, 2048, + ); + let mask = 0xFF_FF_00_00; + let actual_result = _mm512_maskz_popcnt_epi16(mask, test_data); + let reference_result = _mm512_set_epi16( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + ); + assert_eq_m512i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f")] + unsafe fn test_mm512_mask_popcnt_epi16() { + let test_data = _mm512_set_epi16( + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1_FF, 0x3_FF, 0x7_FF, 0xF_FF, 0x1F_FF, + 0x3F_FF, 0x7F_FF, 0xFF_FF, -1, -100, 255, 256, 2, 4, 8, 16, 32, 64, 128, 256, 512, + 1024, 2048, + ); + let mask = 0xFF_FF_00_00; + let actual_result = _mm512_mask_popcnt_epi16(test_data, mask, test_data); + let reference_result = _mm512_set_epi16( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0xFF_FF, -1, -100, 255, 256, 2, + 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, + ); + assert_eq_m512i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f,avx512vl")] + unsafe fn test_mm256_popcnt_epi16() { + let test_data = _mm256_set_epi16( + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1_FF, 0x3_FF, 0x7_FF, 0xF_FF, 0x1F_FF, + 0x3F_FF, 0x7F_FF, + ); + let actual_result = _mm256_popcnt_epi16(test_data); + let reference_result = + _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f,avx512vl")] + unsafe fn test_mm256_maskz_popcnt_epi16() { + let test_data = _mm256_set_epi16( + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1_FF, 0x3_FF, 0x7_FF, 0xF_FF, 0x1F_FF, + 0x3F_FF, 0x7F_FF, + ); + let mask = 0xFF_00; + let actual_result = _mm256_maskz_popcnt_epi16(mask, test_data); + let reference_result = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f,avx512vl")] + unsafe fn test_mm256_mask_popcnt_epi16() { + let test_data = _mm256_set_epi16( + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1_FF, 0x3_FF, 0x7_FF, 0xF_FF, 0x1F_FF, + 0x3F_FF, 0x7F_FF, + ); + let mask = 0xFF_00; + let actual_result = _mm256_mask_popcnt_epi16(test_data, mask, test_data); + let reference_result = _mm256_set_epi16( + 0, 1, 2, 3, 4, 5, 6, 7, 0xFF, 0x1_FF, 0x3_FF, 0x7_FF, 0xF_FF, 0x1F_FF, 0x3F_FF, 0x7F_FF, + ); + assert_eq_m256i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f,avx512vl")] + unsafe fn test_mm_popcnt_epi16() { + let test_data = _mm_set_epi16(0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F); + let actual_result = _mm_popcnt_epi16(test_data); + let reference_result = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + assert_eq_m128i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f,avx512vl")] + unsafe fn test_mm_maskz_popcnt_epi16() { + let test_data = _mm_set_epi16(0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F); + let mask = 0xF0; + let actual_result = _mm_maskz_popcnt_epi16(mask, test_data); + let reference_result = _mm_set_epi16(0, 1, 2, 3, 0, 0, 0, 0); + assert_eq_m128i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f,avx512vl")] + unsafe fn test_mm_mask_popcnt_epi16() { + let test_data = _mm_set_epi16(0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F); + let mask = 0xF0; + let actual_result = _mm_mask_popcnt_epi16(test_data, mask, test_data); + let reference_result = _mm_set_epi16(0, 1, 2, 3, 0xF, 0x1F, 0x3F, 0x7F); + assert_eq_m128i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f")] + unsafe fn test_mm512_popcnt_epi8() { + let test_data = _mm512_set_epi8( + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, -1, 2, 4, 8, 16, 32, 64, 128, 171, 206, 100, + 217, 109, 253, 190, 177, 254, 179, 215, 230, 68, 201, 172, 183, 154, 84, 56, 227, 189, + 140, 35, 117, 219, 169, 226, 170, 13, 22, 159, 251, 73, 121, 143, 145, 85, 91, 137, 90, + 225, 21, 249, 211, 155, 228, 70, + ); + let actual_result = _mm512_popcnt_epi8(test_data); + let reference_result = _mm512_set_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 1, 1, 1, 1, 1, 1, 1, 5, 5, 3, 5, 5, 7, 6, 4, 7, 5, 6, 5, + 2, 4, 4, 6, 4, 3, 3, 5, 6, 3, 3, 5, 6, 4, 4, 4, 3, 3, 6, 7, 3, 5, 5, 3, 4, 5, 3, 4, 4, + 3, 6, 5, 5, 4, 3, + ); + assert_eq_m512i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f")] + unsafe fn test_mm512_maskz_popcnt_epi8() { + let test_data = _mm512_set_epi8( + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, -1, 2, 4, 8, 16, 32, 64, 128, 171, 206, 100, + 217, 109, 253, 190, 177, 254, 179, 215, 230, 68, 201, 172, 183, 154, 84, 56, 227, 189, + 140, 35, 117, 219, 169, 226, 170, 13, 22, 159, 251, 73, 121, 143, 145, 85, 91, 137, 90, + 225, 21, 249, 211, 155, 228, 70, + ); + let mask = 0xFF_FF_FF_FF_00_00_00_00; + let actual_result = _mm512_maskz_popcnt_epi8(mask, test_data); + let reference_result = _mm512_set_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 1, 1, 1, 1, 1, 1, 1, 5, 5, 3, 5, 5, 7, 6, 4, 7, 5, 6, 5, + 2, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + ); + assert_eq_m512i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f")] + unsafe fn test_mm512_mask_popcnt_epi8() { + let test_data = _mm512_set_epi8( + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, -1, 2, 4, 8, 16, 32, 64, 128, 171, 206, 100, + 217, 109, 253, 190, 177, 254, 179, 215, 230, 68, 201, 172, 183, 154, 84, 56, 227, 189, + 140, 35, 117, 219, 169, 226, 170, 13, 22, 159, 251, 73, 121, 143, 145, 85, 91, 137, 90, + 225, 21, 249, 211, 155, 228, 70, + ); + let mask = 0xFF_FF_FF_FF_00_00_00_00; + let actual_result = _mm512_mask_popcnt_epi8(test_data, mask, test_data); + let reference_result = _mm512_set_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 1, 1, 1, 1, 1, 1, 1, 5, 5, 3, 5, 5, 7, 6, 4, 7, 5, 6, 5, + 2, 4, 4, 183, 154, 84, 56, 227, 189, 140, 35, 117, 219, 169, 226, 170, 13, 22, 159, + 251, 73, 121, 143, 145, 85, 91, 137, 90, 225, 21, 249, 211, 155, 228, 70, + ); + assert_eq_m512i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f,avx512vl")] + unsafe fn test_mm256_popcnt_epi8() { + let test_data = _mm256_set_epi8( + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, -1, 2, 4, 8, 16, 32, 64, 128, 171, 206, 100, + 217, 109, 253, 190, 177, 254, 179, 215, 230, 68, 201, 172, + ); + let actual_result = _mm256_popcnt_epi8(test_data); + let reference_result = _mm256_set_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 1, 1, 1, 1, 1, 1, 1, 5, 5, 3, 5, 5, 7, 6, 4, 7, 5, 6, 5, + 2, 4, 4, + ); + assert_eq_m256i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f,avx512vl")] + unsafe fn test_mm256_maskz_popcnt_epi8() { + let test_data = _mm256_set_epi8( + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, -1, 2, 4, 8, 16, 32, 64, 251, 73, 121, 143, + 145, 85, 91, 137, 90, 225, 21, 249, 211, 155, 228, 70, + ); + let mask = 0xFF_FF_00_00; + let actual_result = _mm256_maskz_popcnt_epi8(mask, test_data); + let reference_result = _mm256_set_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, + ); + assert_eq_m256i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f,avx512vl")] + unsafe fn test_mm256_mask_popcnt_epi8() { + let test_data = _mm256_set_epi8( + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, -1, 2, 4, 8, 16, 32, 64, 251, 73, 121, 143, + 145, 85, 91, 137, 90, 225, 21, 249, 211, 155, 228, 70, + ); + let mask = 0xFF_FF_00_00; + let actual_result = _mm256_mask_popcnt_epi8(test_data, mask, test_data); + let reference_result = _mm256_set_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 1, 1, 1, 1, 1, 1, 251, 73, 121, 143, 145, 85, 91, 137, + 90, 225, 21, 249, 211, 155, 228, 70, + ); + assert_eq_m256i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f,avx512vl")] + unsafe fn test_mm_popcnt_epi8() { + let test_data = _mm_set_epi8( + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, -1, 2, 4, 8, 16, 32, 64, + ); + let actual_result = _mm_popcnt_epi8(test_data); + let reference_result = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 1, 1, 1, 1, 1, 1); + assert_eq_m128i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f,avx512vl")] + unsafe fn test_mm_maskz_popcnt_epi8() { + let test_data = _mm_set_epi8( + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 90, 225, 21, 249, 211, 155, 228, 70, + ); + let mask = 0xFF_00; + let actual_result = _mm_maskz_popcnt_epi8(mask, test_data); + let reference_result = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m128i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f,avx512vl")] + unsafe fn test_mm_mask_popcnt_epi8() { + let test_data = _mm_set_epi8( + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 90, 225, 21, 249, 211, 155, 228, 70, + ); + let mask = 0xFF_00; + let actual_result = _mm_mask_popcnt_epi8(test_data, mask, test_data); + let reference_result = + _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 90, 225, 21, 249, 211, 155, 228, 70); + assert_eq_m128i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f")] + unsafe fn test_mm512_bitshuffle_epi64_mask() { + let test_indices = _mm512_set_epi8( + 63, 62, 61, 60, 59, 58, 57, 56, 63, 62, 61, 60, 59, 58, 57, 56, 32, 32, 16, 16, 0, 0, + 8, 8, 56, 48, 40, 32, 24, 16, 8, 0, 63, 62, 61, 60, 59, 58, 57, 56, 63, 62, 61, 60, 59, + 58, 57, 56, 32, 32, 16, 16, 0, 0, 8, 8, 56, 48, 40, 32, 24, 16, 8, 0, + ); + let test_data = _mm512_setr_epi64( + 0xFF_FF_FF_FF_00_00_00_00, + 0xFF_00_FF_00_FF_00_FF_00, + 0xFF_00_00_00_00_00_00_00, + 0xAC_00_00_00_00_00_00_00, + 0xFF_FF_FF_FF_00_00_00_00, + 0xFF_00_FF_00_FF_00_FF_00, + 0xFF_00_00_00_00_00_00_00, + 0xAC_00_00_00_00_00_00_00, + ); + let actual_result = _mm512_bitshuffle_epi64_mask(test_data, test_indices); + let reference_result = 0xF0 << 0 + | 0x03 << 8 + | 0xFF << 16 + | 0xAC << 24 + | 0xF0 << 32 + | 0x03 << 40 + | 0xFF << 48 + | 0xAC << 56; + + assert_eq!(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f")] + unsafe fn test_mm512_mask_bitshuffle_epi64_mask() { + let test_indices = _mm512_set_epi8( + 63, 62, 61, 60, 59, 58, 57, 56, 63, 62, 61, 60, 59, 58, 57, 56, 32, 32, 16, 16, 0, 0, + 8, 8, 56, 48, 40, 32, 24, 16, 8, 0, 63, 62, 61, 60, 59, 58, 57, 56, 63, 62, 61, 60, 59, + 58, 57, 56, 32, 32, 16, 16, 0, 0, 8, 8, 56, 48, 40, 32, 24, 16, 8, 0, + ); + let test_data = _mm512_setr_epi64( + 0xFF_FF_FF_FF_00_00_00_00, + 0xFF_00_FF_00_FF_00_FF_00, + 0xFF_00_00_00_00_00_00_00, + 0xAC_00_00_00_00_00_00_00, + 0xFF_FF_FF_FF_00_00_00_00, + 0xFF_00_FF_00_FF_00_FF_00, + 0xFF_00_00_00_00_00_00_00, + 0xAC_00_00_00_00_00_00_00, + ); + let mask = 0xFF_FF_FF_FF_00_00_00_00; + let actual_result = _mm512_mask_bitshuffle_epi64_mask(mask, test_data, test_indices); + let reference_result = 0x00 << 0 + | 0x00 << 8 + | 0x00 << 16 + | 0x00 << 24 + | 0xF0 << 32 + | 0x03 << 40 + | 0xFF << 48 + | 0xAC << 56; + + assert_eq!(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f,avx512vl")] + unsafe fn test_mm256_bitshuffle_epi64_mask() { + let test_indices = _mm256_set_epi8( + 63, 62, 61, 60, 59, 58, 57, 56, 63, 62, 61, 60, 59, 58, 57, 56, 32, 32, 16, 16, 0, 0, + 8, 8, 56, 48, 40, 32, 24, 16, 8, 0, + ); + let test_data = _mm256_setr_epi64x( + 0xFF_FF_FF_FF_00_00_00_00, + 0xFF_00_FF_00_FF_00_FF_00, + 0xFF_00_00_00_00_00_00_00, + 0xAC_00_00_00_00_00_00_00, + ); + let actual_result = _mm256_bitshuffle_epi64_mask(test_data, test_indices); + let reference_result = 0xF0 << 0 | 0x03 << 8 | 0xFF << 16 | 0xAC << 24; + + assert_eq!(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f,avx512vl")] + unsafe fn test_mm256_mask_bitshuffle_epi64_mask() { + let test_indices = _mm256_set_epi8( + 63, 62, 61, 60, 59, 58, 57, 56, 63, 62, 61, 60, 59, 58, 57, 56, 32, 32, 16, 16, 0, 0, + 8, 8, 56, 48, 40, 32, 24, 16, 8, 0, + ); + let test_data = _mm256_setr_epi64x( + 0xFF_FF_FF_FF_00_00_00_00, + 0xFF_00_FF_00_FF_00_FF_00, + 0xFF_00_00_00_00_00_00_00, + 0xAC_00_00_00_00_00_00_00, + ); + let mask = 0xFF_FF_00_00; + let actual_result = _mm256_mask_bitshuffle_epi64_mask(mask, test_data, test_indices); + let reference_result = 0x00 << 0 | 0x00 << 8 | 0xFF << 16 | 0xAC << 24; + + assert_eq!(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f,avx512vl")] + unsafe fn test_mm_bitshuffle_epi64_mask() { + let test_indices = _mm_set_epi8( + 63, 62, 61, 60, 59, 58, 57, 56, 63, 62, 61, 60, 59, 58, 57, 56, + ); + let test_data = _mm_setr_epi64x(0xFF_00_00_00_00_00_00_00, 0xAC_00_00_00_00_00_00_00); + let actual_result = _mm_bitshuffle_epi64_mask(test_data, test_indices); + let reference_result = 0xFF << 0 | 0xAC << 8; + + assert_eq!(actual_result, reference_result); + } + + #[simd_test(enable = "avx512bitalg,avx512f,avx512vl")] + unsafe fn test_mm_mask_bitshuffle_epi64_mask() { + let test_indices = _mm_set_epi8( + 63, 62, 61, 60, 59, 58, 57, 56, 63, 62, 61, 60, 59, 58, 57, 56, + ); + let test_data = _mm_setr_epi64x(0xFF_00_00_00_00_00_00_00, 0xAC_00_00_00_00_00_00_00); + let mask = 0xFF_00; + let actual_result = _mm_mask_bitshuffle_epi64_mask(mask, test_data, test_indices); + let reference_result = 0x00 << 0 | 0xAC << 8; + + assert_eq!(actual_result, reference_result); + } +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512bw.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512bw.rs new file mode 100644 index 000000000..49d78ed60 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512bw.rs @@ -0,0 +1,18769 @@ +use crate::{ + arch::asm, + core_arch::{simd::*, simd_llvm::*, x86::*}, + mem::{self, transmute}, + ptr, +}; + +#[cfg(test)] +use stdarch_test::assert_instr; + +use super::avx512f::{vpl, vps}; + +/// Compute the absolute value of packed signed 16-bit integers in a, and store the unsigned results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_abs_epi16&expand=30) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpabsw))] +pub unsafe fn _mm512_abs_epi16(a: __m512i) -> __m512i { + let a = a.as_i16x32(); + // all-0 is a properly initialized i16x32 + let zero: i16x32 = mem::zeroed(); + let sub = simd_sub(zero, a); + let cmp: i16x32 = simd_gt(a, zero); + transmute(simd_select(cmp, a, sub)) +} + +/// Compute the absolute value of packed signed 16-bit integers in a, and store the unsigned results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_abs_epi16&expand=31) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpabsw))] +pub unsafe fn _mm512_mask_abs_epi16(src: __m512i, k: __mmask32, a: __m512i) -> __m512i { + let abs = _mm512_abs_epi16(a).as_i16x32(); + transmute(simd_select_bitmask(k, abs, src.as_i16x32())) +} + +/// Compute the absolute value of packed signed 16-bit integers in a, and store the unsigned results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_abs_epi16&expand=32) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpabsw))] +pub unsafe fn _mm512_maskz_abs_epi16(k: __mmask32, a: __m512i) -> __m512i { + let abs = _mm512_abs_epi16(a).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, abs, zero)) +} + +/// Compute the absolute value of packed signed 16-bit integers in a, and store the unsigned results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_abs_epi16&expand=28) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpabsw))] +pub unsafe fn _mm256_mask_abs_epi16(src: __m256i, k: __mmask16, a: __m256i) -> __m256i { + let abs = _mm256_abs_epi16(a).as_i16x16(); + transmute(simd_select_bitmask(k, abs, src.as_i16x16())) +} + +/// Compute the absolute value of packed signed 16-bit integers in a, and store the unsigned results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_abs_epi16&expand=29) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpabsw))] +pub unsafe fn _mm256_maskz_abs_epi16(k: __mmask16, a: __m256i) -> __m256i { + let abs = _mm256_abs_epi16(a).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, abs, zero)) +} + +/// Compute the absolute value of packed signed 16-bit integers in a, and store the unsigned results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_abs_epi16&expand=25) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpabsw))] +pub unsafe fn _mm_mask_abs_epi16(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let abs = _mm_abs_epi16(a).as_i16x8(); + transmute(simd_select_bitmask(k, abs, src.as_i16x8())) +} + +/// Compute the absolute value of packed signed 16-bit integers in a, and store the unsigned results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_abs_epi16&expand=26) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpabsw))] +pub unsafe fn _mm_maskz_abs_epi16(k: __mmask8, a: __m128i) -> __m128i { + let abs = _mm_abs_epi16(a).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, abs, zero)) +} + +/// Compute the absolute value of packed signed 8-bit integers in a, and store the unsigned results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_abs_epi8&expand=57) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpabsb))] +pub unsafe fn _mm512_abs_epi8(a: __m512i) -> __m512i { + let a = a.as_i8x64(); + // all-0 is a properly initialized i8x64 + let zero: i8x64 = mem::zeroed(); + let sub = simd_sub(zero, a); + let cmp: i8x64 = simd_gt(a, zero); + transmute(simd_select(cmp, a, sub)) +} + +/// Compute the absolute value of packed signed 8-bit integers in a, and store the unsigned results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_abs_epi8&expand=58) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpabsb))] +pub unsafe fn _mm512_mask_abs_epi8(src: __m512i, k: __mmask64, a: __m512i) -> __m512i { + let abs = _mm512_abs_epi8(a).as_i8x64(); + transmute(simd_select_bitmask(k, abs, src.as_i8x64())) +} + +/// Compute the absolute value of packed signed 8-bit integers in a, and store the unsigned results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_abs_epi8&expand=59) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpabsb))] +pub unsafe fn _mm512_maskz_abs_epi8(k: __mmask64, a: __m512i) -> __m512i { + let abs = _mm512_abs_epi8(a).as_i8x64(); + let zero = _mm512_setzero_si512().as_i8x64(); + transmute(simd_select_bitmask(k, abs, zero)) +} + +/// Compute the absolute value of packed signed 8-bit integers in a, and store the unsigned results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_abs_epi8&expand=55) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpabsb))] +pub unsafe fn _mm256_mask_abs_epi8(src: __m256i, k: __mmask32, a: __m256i) -> __m256i { + let abs = _mm256_abs_epi8(a).as_i8x32(); + transmute(simd_select_bitmask(k, abs, src.as_i8x32())) +} + +/// Compute the absolute value of packed signed 8-bit integers in a, and store the unsigned results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_abs_epi8&expand=56) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpabsb))] +pub unsafe fn _mm256_maskz_abs_epi8(k: __mmask32, a: __m256i) -> __m256i { + let abs = _mm256_abs_epi8(a).as_i8x32(); + let zero = _mm256_setzero_si256().as_i8x32(); + transmute(simd_select_bitmask(k, abs, zero)) +} + +/// Compute the absolute value of packed signed 8-bit integers in a, and store the unsigned results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set) +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_abs_epi8&expand=52) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpabsb))] +pub unsafe fn _mm_mask_abs_epi8(src: __m128i, k: __mmask16, a: __m128i) -> __m128i { + let abs = _mm_abs_epi8(a).as_i8x16(); + transmute(simd_select_bitmask(k, abs, src.as_i8x16())) +} + +/// Compute the absolute value of packed signed 8-bit integers in a, and store the unsigned results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_abs_epi8&expand=53) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpabsb))] +pub unsafe fn _mm_maskz_abs_epi8(k: __mmask16, a: __m128i) -> __m128i { + let abs = _mm_abs_epi8(a).as_i8x16(); + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask(k, abs, zero)) +} + +/// Add packed 16-bit integers in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_add_epi16&expand=91) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpaddw))] +pub unsafe fn _mm512_add_epi16(a: __m512i, b: __m512i) -> __m512i { + transmute(simd_add(a.as_i16x32(), b.as_i16x32())) +} + +/// Add packed 16-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_add_epi16&expand=92) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpaddw))] +pub unsafe fn _mm512_mask_add_epi16(src: __m512i, k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let add = _mm512_add_epi16(a, b).as_i16x32(); + transmute(simd_select_bitmask(k, add, src.as_i16x32())) +} + +/// Add packed 16-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_add_epi16&expand=93) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpaddw))] +pub unsafe fn _mm512_maskz_add_epi16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let add = _mm512_add_epi16(a, b).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, add, zero)) +} + +/// Add packed 16-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_add_epi&expand=89) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddw))] +pub unsafe fn _mm256_mask_add_epi16(src: __m256i, k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let add = _mm256_add_epi16(a, b).as_i16x16(); + transmute(simd_select_bitmask(k, add, src.as_i16x16())) +} + +/// Add packed 16-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_add_epi16&expand=90) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddw))] +pub unsafe fn _mm256_maskz_add_epi16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let add = _mm256_add_epi16(a, b).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, add, zero)) +} + +/// Add packed 16-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_add_epi16&expand=86) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddw))] +pub unsafe fn _mm_mask_add_epi16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let add = _mm_add_epi16(a, b).as_i16x8(); + transmute(simd_select_bitmask(k, add, src.as_i16x8())) +} + +/// Add packed 16-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_add_epi16&expand=87) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddw))] +pub unsafe fn _mm_maskz_add_epi16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let add = _mm_add_epi16(a, b).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, add, zero)) +} + +/// Add packed 8-bit integers in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_add_epi8&expand=118) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpaddb))] +pub unsafe fn _mm512_add_epi8(a: __m512i, b: __m512i) -> __m512i { + transmute(simd_add(a.as_i8x64(), b.as_i8x64())) +} + +/// Add packed 8-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_add_epi8&expand=119) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpaddb))] +pub unsafe fn _mm512_mask_add_epi8(src: __m512i, k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + let add = _mm512_add_epi8(a, b).as_i8x64(); + transmute(simd_select_bitmask(k, add, src.as_i8x64())) +} + +/// Add packed 8-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_add_epi8&expand=120) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpaddb))] +pub unsafe fn _mm512_maskz_add_epi8(k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + let add = _mm512_add_epi8(a, b).as_i8x64(); + let zero = _mm512_setzero_si512().as_i8x64(); + transmute(simd_select_bitmask(k, add, zero)) +} + +/// Add packed 8-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_add_epi8&expand=116) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddb))] +pub unsafe fn _mm256_mask_add_epi8(src: __m256i, k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + let add = _mm256_add_epi8(a, b).as_i8x32(); + transmute(simd_select_bitmask(k, add, src.as_i8x32())) +} + +/// Add packed 8-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_add_epi8&expand=117) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddb))] +pub unsafe fn _mm256_maskz_add_epi8(k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + let add = _mm256_add_epi8(a, b).as_i8x32(); + let zero = _mm256_setzero_si256().as_i8x32(); + transmute(simd_select_bitmask(k, add, zero)) +} + +/// Add packed 8-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_add_epi8&expand=113) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddb))] +pub unsafe fn _mm_mask_add_epi8(src: __m128i, k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let add = _mm_add_epi8(a, b).as_i8x16(); + transmute(simd_select_bitmask(k, add, src.as_i8x16())) +} + +/// Add packed 8-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_add_epi8&expand=114) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddb))] +pub unsafe fn _mm_maskz_add_epi8(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let add = _mm_add_epi8(a, b).as_i8x16(); + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask(k, add, zero)) +} + +/// Add packed unsigned 16-bit integers in a and b using saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_adds_epu16&expand=197) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpaddusw))] +pub unsafe fn _mm512_adds_epu16(a: __m512i, b: __m512i) -> __m512i { + transmute(vpaddusw( + a.as_u16x32(), + b.as_u16x32(), + _mm512_setzero_si512().as_u16x32(), + 0b11111111_11111111_11111111_11111111, + )) +} + +/// Add packed unsigned 16-bit integers in a and b using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_adds_epu16&expand=198) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpaddusw))] +pub unsafe fn _mm512_mask_adds_epu16( + src: __m512i, + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + transmute(vpaddusw(a.as_u16x32(), b.as_u16x32(), src.as_u16x32(), k)) +} + +/// Add packed unsigned 16-bit integers in a and b using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_adds_epu16&expand=199) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpaddusw))] +pub unsafe fn _mm512_maskz_adds_epu16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + transmute(vpaddusw( + a.as_u16x32(), + b.as_u16x32(), + _mm512_setzero_si512().as_u16x32(), + k, + )) +} + +/// Add packed unsigned 16-bit integers in a and b using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_adds_epu16&expand=195) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddusw))] +pub unsafe fn _mm256_mask_adds_epu16( + src: __m256i, + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + transmute(vpaddusw256( + a.as_u16x16(), + b.as_u16x16(), + src.as_u16x16(), + k, + )) +} + +/// Add packed unsigned 16-bit integers in a and b using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_adds_epu16&expand=196) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddusw))] +pub unsafe fn _mm256_maskz_adds_epu16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + transmute(vpaddusw256( + a.as_u16x16(), + b.as_u16x16(), + _mm256_setzero_si256().as_u16x16(), + k, + )) +} + +/// Add packed unsigned 16-bit integers in a and b using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_adds_epu16&expand=192) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddusw))] +pub unsafe fn _mm_mask_adds_epu16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + transmute(vpaddusw128(a.as_u16x8(), b.as_u16x8(), src.as_u16x8(), k)) +} + +/// Add packed unsigned 16-bit integers in a and b using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_adds_epu16&expand=193) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddusw))] +pub unsafe fn _mm_maskz_adds_epu16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + transmute(vpaddusw128( + a.as_u16x8(), + b.as_u16x8(), + _mm_setzero_si128().as_u16x8(), + k, + )) +} + +/// Add packed unsigned 8-bit integers in a and b using saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_adds_epu8&expand=206) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpaddusb))] +pub unsafe fn _mm512_adds_epu8(a: __m512i, b: __m512i) -> __m512i { + transmute(vpaddusb( + a.as_u8x64(), + b.as_u8x64(), + _mm512_setzero_si512().as_u8x64(), + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + )) +} + +/// Add packed unsigned 8-bit integers in a and b using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_adds_epu8&expand=207) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpaddusb))] +pub unsafe fn _mm512_mask_adds_epu8(src: __m512i, k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + transmute(vpaddusb(a.as_u8x64(), b.as_u8x64(), src.as_u8x64(), k)) +} + +/// Add packed unsigned 8-bit integers in a and b using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_adds_epu8&expand=208) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpaddusb))] +pub unsafe fn _mm512_maskz_adds_epu8(k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + transmute(vpaddusb( + a.as_u8x64(), + b.as_u8x64(), + _mm512_setzero_si512().as_u8x64(), + k, + )) +} + +/// Add packed unsigned 8-bit integers in a and b using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_adds_epu8&expand=204) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddusb))] +pub unsafe fn _mm256_mask_adds_epu8(src: __m256i, k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + transmute(vpaddusb256(a.as_u8x32(), b.as_u8x32(), src.as_u8x32(), k)) +} + +/// Add packed unsigned 8-bit integers in a and b using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_adds_epu8&expand=205) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddusb))] +pub unsafe fn _mm256_maskz_adds_epu8(k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + transmute(vpaddusb256( + a.as_u8x32(), + b.as_u8x32(), + _mm256_setzero_si256().as_u8x32(), + k, + )) +} + +/// Add packed unsigned 8-bit integers in a and b using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_adds_epu8&expand=201) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddusb))] +pub unsafe fn _mm_mask_adds_epu8(src: __m128i, k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + transmute(vpaddusb128(a.as_u8x16(), b.as_u8x16(), src.as_u8x16(), k)) +} + +/// Add packed unsigned 8-bit integers in a and b using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_adds_epu8&expand=202) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddusb))] +pub unsafe fn _mm_maskz_adds_epu8(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + transmute(vpaddusb128( + a.as_u8x16(), + b.as_u8x16(), + _mm_setzero_si128().as_u8x16(), + k, + )) +} + +/// Add packed signed 16-bit integers in a and b using saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_adds_epi16&expand=179) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpaddsw))] +pub unsafe fn _mm512_adds_epi16(a: __m512i, b: __m512i) -> __m512i { + transmute(vpaddsw( + a.as_i16x32(), + b.as_i16x32(), + _mm512_setzero_si512().as_i16x32(), + 0b11111111_11111111_11111111_11111111, + )) +} + +/// Add packed signed 16-bit integers in a and b using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_adds_epi16&expand=180) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpaddsw))] +pub unsafe fn _mm512_mask_adds_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + transmute(vpaddsw(a.as_i16x32(), b.as_i16x32(), src.as_i16x32(), k)) +} + +/// Add packed signed 16-bit integers in a and b using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_adds_epi16&expand=181) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpaddsw))] +pub unsafe fn _mm512_maskz_adds_epi16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + transmute(vpaddsw( + a.as_i16x32(), + b.as_i16x32(), + _mm512_setzero_si512().as_i16x32(), + k, + )) +} + +/// Add packed signed 16-bit integers in a and b using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_adds_epi16&expand=177) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddsw))] +pub unsafe fn _mm256_mask_adds_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + transmute(vpaddsw256(a.as_i16x16(), b.as_i16x16(), src.as_i16x16(), k)) +} + +/// Add packed signed 16-bit integers in a and b using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_adds_epi16&expand=178) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddsw))] +pub unsafe fn _mm256_maskz_adds_epi16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + transmute(vpaddsw256( + a.as_i16x16(), + b.as_i16x16(), + _mm256_setzero_si256().as_i16x16(), + k, + )) +} + +/// Add packed signed 16-bit integers in a and b using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_adds_epi16&expand=174) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddsw))] +pub unsafe fn _mm_mask_adds_epi16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + transmute(vpaddsw128(a.as_i16x8(), b.as_i16x8(), src.as_i16x8(), k)) +} + +/// Add packed signed 16-bit integers in a and b using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_adds_epi16&expand=175) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddsw))] +pub unsafe fn _mm_maskz_adds_epi16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + transmute(vpaddsw128( + a.as_i16x8(), + b.as_i16x8(), + _mm_setzero_si128().as_i16x8(), + k, + )) +} + +/// Add packed signed 8-bit integers in a and b using saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_adds_epi8&expand=188) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpaddsb))] +pub unsafe fn _mm512_adds_epi8(a: __m512i, b: __m512i) -> __m512i { + transmute(vpaddsb( + a.as_i8x64(), + b.as_i8x64(), + _mm512_setzero_si512().as_i8x64(), + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + )) +} + +/// Add packed signed 8-bit integers in a and b using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_adds_epi8&expand=189) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpaddsb))] +pub unsafe fn _mm512_mask_adds_epi8(src: __m512i, k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + transmute(vpaddsb(a.as_i8x64(), b.as_i8x64(), src.as_i8x64(), k)) +} + +/// Add packed signed 8-bit integers in a and b using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_adds_epi8&expand=190) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpaddsb))] +pub unsafe fn _mm512_maskz_adds_epi8(k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + transmute(vpaddsb( + a.as_i8x64(), + b.as_i8x64(), + _mm512_setzero_si512().as_i8x64(), + k, + )) +} + +/// Add packed signed 8-bit integers in a and b using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_adds_epi8&expand=186) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddsb))] +pub unsafe fn _mm256_mask_adds_epi8(src: __m256i, k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + transmute(vpaddsb256(a.as_i8x32(), b.as_i8x32(), src.as_i8x32(), k)) +} + +/// Add packed signed 8-bit integers in a and b using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_adds_epi8&expand=187) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddsb))] +pub unsafe fn _mm256_maskz_adds_epi8(k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + transmute(vpaddsb256( + a.as_i8x32(), + b.as_i8x32(), + _mm256_setzero_si256().as_i8x32(), + k, + )) +} + +/// Add packed signed 8-bit integers in a and b using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_adds_epi8&expand=183) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddsb))] +pub unsafe fn _mm_mask_adds_epi8(src: __m128i, k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + transmute(vpaddsb128(a.as_i8x16(), b.as_i8x16(), src.as_i8x16(), k)) +} + +/// Add packed signed 8-bit integers in a and b using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_adds_epi8&expand=184) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddsb))] +pub unsafe fn _mm_maskz_adds_epi8(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + transmute(vpaddsb128( + a.as_i8x16(), + b.as_i8x16(), + _mm_setzero_si128().as_i8x16(), + k, + )) +} + +/// Subtract packed 16-bit integers in b from packed 16-bit integers in a, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sub_epi16&expand=5685) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsubw))] +pub unsafe fn _mm512_sub_epi16(a: __m512i, b: __m512i) -> __m512i { + transmute(simd_sub(a.as_i16x32(), b.as_i16x32())) +} + +/// Subtract packed 16-bit integers in b from packed 16-bit integers in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sub_epi16&expand=5683) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsubw))] +pub unsafe fn _mm512_mask_sub_epi16(src: __m512i, k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let sub = _mm512_sub_epi16(a, b).as_i16x32(); + transmute(simd_select_bitmask(k, sub, src.as_i16x32())) +} + +/// Subtract packed 16-bit integers in b from packed 16-bit integers in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sub_epi16&expand=5684) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsubw))] +pub unsafe fn _mm512_maskz_sub_epi16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let sub = _mm512_sub_epi16(a, b).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, sub, zero)) +} + +/// Subtract packed 16-bit integers in b from packed 16-bit integers in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_sub_epi16&expand=5680) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubw))] +pub unsafe fn _mm256_mask_sub_epi16(src: __m256i, k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let sub = _mm256_sub_epi16(a, b).as_i16x16(); + transmute(simd_select_bitmask(k, sub, src.as_i16x16())) +} + +/// Subtract packed 16-bit integers in b from packed 16-bit integers in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_sub_epi16&expand=5681) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubw))] +pub unsafe fn _mm256_maskz_sub_epi16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let sub = _mm256_sub_epi16(a, b).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, sub, zero)) +} + +/// Subtract packed 16-bit integers in b from packed 16-bit integers in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_sub_epi16&expand=5677) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubw))] +pub unsafe fn _mm_mask_sub_epi16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let sub = _mm_sub_epi16(a, b).as_i16x8(); + transmute(simd_select_bitmask(k, sub, src.as_i16x8())) +} + +/// Subtract packed 16-bit integers in b from packed 16-bit integers in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_sub_epi16&expand=5678) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubw))] +pub unsafe fn _mm_maskz_sub_epi16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let sub = _mm_sub_epi16(a, b).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, sub, zero)) +} + +/// Subtract packed 8-bit integers in b from packed 8-bit integers in a, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sub_epi8&expand=5712) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsubb))] +pub unsafe fn _mm512_sub_epi8(a: __m512i, b: __m512i) -> __m512i { + transmute(simd_sub(a.as_i8x64(), b.as_i8x64())) +} + +/// Subtract packed 8-bit integers in b from packed 8-bit integers in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sub_epi8&expand=5710) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsubb))] +pub unsafe fn _mm512_mask_sub_epi8(src: __m512i, k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + let sub = _mm512_sub_epi8(a, b).as_i8x64(); + transmute(simd_select_bitmask(k, sub, src.as_i8x64())) +} + +/// Subtract packed 8-bit integers in b from packed 8-bit integers in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sub_epi8&expand=5711) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsubb))] +pub unsafe fn _mm512_maskz_sub_epi8(k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + let sub = _mm512_sub_epi8(a, b).as_i8x64(); + let zero = _mm512_setzero_si512().as_i8x64(); + transmute(simd_select_bitmask(k, sub, zero)) +} + +/// Subtract packed 8-bit integers in b from packed 8-bit integers in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_sub_epi8&expand=5707) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubb))] +pub unsafe fn _mm256_mask_sub_epi8(src: __m256i, k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + let sub = _mm256_sub_epi8(a, b).as_i8x32(); + transmute(simd_select_bitmask(k, sub, src.as_i8x32())) +} + +/// Subtract packed 8-bit integers in b from packed 8-bit integers in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_sub_epi8&expand=5708) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubb))] +pub unsafe fn _mm256_maskz_sub_epi8(k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + let sub = _mm256_sub_epi8(a, b).as_i8x32(); + let zero = _mm256_setzero_si256().as_i8x32(); + transmute(simd_select_bitmask(k, sub, zero)) +} + +/// Subtract packed 8-bit integers in b from packed 8-bit integers in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_sub_epi8&expand=5704) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubb))] +pub unsafe fn _mm_mask_sub_epi8(src: __m128i, k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let sub = _mm_sub_epi8(a, b).as_i8x16(); + transmute(simd_select_bitmask(k, sub, src.as_i8x16())) +} + +/// Subtract packed 8-bit integers in b from packed 8-bit integers in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_sub_epi8&expand=5705) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubb))] +pub unsafe fn _mm_maskz_sub_epi8(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let sub = _mm_sub_epi8(a, b).as_i8x16(); + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask(k, sub, zero)) +} + +/// Subtract packed unsigned 16-bit integers in b from packed unsigned 16-bit integers in a using saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_subs_epu16&expand=5793) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsubusw))] +pub unsafe fn _mm512_subs_epu16(a: __m512i, b: __m512i) -> __m512i { + transmute(vpsubusw( + a.as_u16x32(), + b.as_u16x32(), + _mm512_setzero_si512().as_u16x32(), + 0b11111111_11111111_11111111_11111111, + )) +} + +/// Subtract packed unsigned 16-bit integers in b from packed unsigned 16-bit integers in a using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_subs_epu16&expand=5791) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsubusw))] +pub unsafe fn _mm512_mask_subs_epu16( + src: __m512i, + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + transmute(vpsubusw(a.as_u16x32(), b.as_u16x32(), src.as_u16x32(), k)) +} + +/// Subtract packed unsigned 16-bit integers in b from packed unsigned 16-bit integers in a using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_subs_epu16&expand=5792) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsubusw))] +pub unsafe fn _mm512_maskz_subs_epu16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + transmute(vpsubusw( + a.as_u16x32(), + b.as_u16x32(), + _mm512_setzero_si512().as_u16x32(), + k, + )) +} + +/// Subtract packed unsigned 16-bit integers in b from packed unsigned 16-bit integers in a using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_subs_epu16&expand=5788) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubusw))] +pub unsafe fn _mm256_mask_subs_epu16( + src: __m256i, + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + transmute(vpsubusw256( + a.as_u16x16(), + b.as_u16x16(), + src.as_u16x16(), + k, + )) +} + +/// Subtract packed unsigned 16-bit integers in b from packed unsigned 16-bit integers in a using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_subs_epu16&expand=5789) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubusw))] +pub unsafe fn _mm256_maskz_subs_epu16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + transmute(vpsubusw256( + a.as_u16x16(), + b.as_u16x16(), + _mm256_setzero_si256().as_u16x16(), + k, + )) +} + +/// Subtract packed unsigned 16-bit integers in b from packed unsigned 16-bit integers in a using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_subs_epu16&expand=5785) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubusw))] +pub unsafe fn _mm_mask_subs_epu16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + transmute(vpsubusw128(a.as_u16x8(), b.as_u16x8(), src.as_u16x8(), k)) +} + +/// Subtract packed unsigned 16-bit integers in b from packed unsigned 16-bit integers in a using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_subs_epu16&expand=5786) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubusw))] +pub unsafe fn _mm_maskz_subs_epu16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + transmute(vpsubusw128( + a.as_u16x8(), + b.as_u16x8(), + _mm_setzero_si128().as_u16x8(), + k, + )) +} + +/// Subtract packed unsigned 8-bit integers in b from packed unsigned 8-bit integers in a using saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_subs_epu8&expand=5802) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsubusb))] +pub unsafe fn _mm512_subs_epu8(a: __m512i, b: __m512i) -> __m512i { + transmute(vpsubusb( + a.as_u8x64(), + b.as_u8x64(), + _mm512_setzero_si512().as_u8x64(), + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + )) +} + +/// Subtract packed unsigned 8-bit integers in b from packed unsigned 8-bit integers in a using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_subs_epu8&expand=5800) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsubusb))] +pub unsafe fn _mm512_mask_subs_epu8(src: __m512i, k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + transmute(vpsubusb(a.as_u8x64(), b.as_u8x64(), src.as_u8x64(), k)) +} + +/// Subtract packed unsigned 8-bit integers in b from packed unsigned 8-bit integers in a using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_subs_epu8&expand=5801) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsubusb))] +pub unsafe fn _mm512_maskz_subs_epu8(k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + transmute(vpsubusb( + a.as_u8x64(), + b.as_u8x64(), + _mm512_setzero_si512().as_u8x64(), + k, + )) +} + +/// Subtract packed unsigned 8-bit integers in b from packed unsigned 8-bit integers in a using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_subs_epu8&expand=5797) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubusb))] +pub unsafe fn _mm256_mask_subs_epu8(src: __m256i, k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + transmute(vpsubusb256(a.as_u8x32(), b.as_u8x32(), src.as_u8x32(), k)) +} + +/// Subtract packed unsigned 8-bit integers in b from packed unsigned 8-bit integers in a using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_subs_epu8&expand=5798) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubusb))] +pub unsafe fn _mm256_maskz_subs_epu8(k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + transmute(vpsubusb256( + a.as_u8x32(), + b.as_u8x32(), + _mm256_setzero_si256().as_u8x32(), + k, + )) +} + +/// Subtract packed unsigned 8-bit integers in b from packed unsigned 8-bit integers in a using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_subs_epu8&expand=5794) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubusb))] +pub unsafe fn _mm_mask_subs_epu8(src: __m128i, k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + transmute(vpsubusb128(a.as_u8x16(), b.as_u8x16(), src.as_u8x16(), k)) +} + +/// Subtract packed unsigned 8-bit integers in b from packed unsigned 8-bit integers in a using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_subs_epu8&expand=5795) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubusb))] +pub unsafe fn _mm_maskz_subs_epu8(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + transmute(vpsubusb128( + a.as_u8x16(), + b.as_u8x16(), + _mm_setzero_si128().as_u8x16(), + k, + )) +} + +/// Subtract packed signed 16-bit integers in b from packed 16-bit integers in a using saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_subs_epi16&expand=5775) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsubsw))] +pub unsafe fn _mm512_subs_epi16(a: __m512i, b: __m512i) -> __m512i { + transmute(vpsubsw( + a.as_i16x32(), + b.as_i16x32(), + _mm512_setzero_si512().as_i16x32(), + 0b11111111_11111111_11111111_11111111, + )) +} + +/// Subtract packed signed 16-bit integers in b from packed 16-bit integers in a using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_subs_epi16&expand=5773) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsubsw))] +pub unsafe fn _mm512_mask_subs_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + transmute(vpsubsw(a.as_i16x32(), b.as_i16x32(), src.as_i16x32(), k)) +} + +/// Subtract packed signed 16-bit integers in b from packed 16-bit integers in a using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_subs_epi16&expand=5774) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsubsw))] +pub unsafe fn _mm512_maskz_subs_epi16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + transmute(vpsubsw( + a.as_i16x32(), + b.as_i16x32(), + _mm512_setzero_si512().as_i16x32(), + k, + )) +} + +/// Subtract packed signed 16-bit integers in b from packed 16-bit integers in a using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_subs_epi16&expand=5770) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubsw))] +pub unsafe fn _mm256_mask_subs_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + transmute(vpsubsw256(a.as_i16x16(), b.as_i16x16(), src.as_i16x16(), k)) +} + +/// Subtract packed signed 16-bit integers in b from packed 16-bit integers in a using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_subs_epi16&expand=5771) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubsw))] +pub unsafe fn _mm256_maskz_subs_epi16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + transmute(vpsubsw256( + a.as_i16x16(), + b.as_i16x16(), + _mm256_setzero_si256().as_i16x16(), + k, + )) +} + +/// Subtract packed signed 16-bit integers in b from packed 16-bit integers in a using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_subs_epi16&expand=5767) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubsw))] +pub unsafe fn _mm_mask_subs_epi16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + transmute(vpsubsw128(a.as_i16x8(), b.as_i16x8(), src.as_i16x8(), k)) +} + +/// Subtract packed signed 16-bit integers in b from packed 16-bit integers in a using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_subs_epi16&expand=5768) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubsw))] +pub unsafe fn _mm_maskz_subs_epi16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + transmute(vpsubsw128( + a.as_i16x8(), + b.as_i16x8(), + _mm_setzero_si128().as_i16x8(), + k, + )) +} + +/// Subtract packed signed 8-bit integers in b from packed 8-bit integers in a using saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_subs_epi8&expand=5784) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsubsb))] +pub unsafe fn _mm512_subs_epi8(a: __m512i, b: __m512i) -> __m512i { + transmute(vpsubsb( + a.as_i8x64(), + b.as_i8x64(), + _mm512_setzero_si512().as_i8x64(), + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + )) +} + +/// Subtract packed signed 8-bit integers in b from packed 8-bit integers in a using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_subs_epi8&expand=5782) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsubsb))] +pub unsafe fn _mm512_mask_subs_epi8(src: __m512i, k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + transmute(vpsubsb(a.as_i8x64(), b.as_i8x64(), src.as_i8x64(), k)) +} + +/// Subtract packed signed 8-bit integers in b from packed 8-bit integers in a using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_subs_epi8&expand=5783) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsubsb))] +pub unsafe fn _mm512_maskz_subs_epi8(k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + transmute(vpsubsb( + a.as_i8x64(), + b.as_i8x64(), + _mm512_setzero_si512().as_i8x64(), + k, + )) +} + +/// Subtract packed signed 8-bit integers in b from packed 8-bit integers in a using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_subs_epi8&expand=5779) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubsb))] +pub unsafe fn _mm256_mask_subs_epi8(src: __m256i, k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + transmute(vpsubsb256(a.as_i8x32(), b.as_i8x32(), src.as_i8x32(), k)) +} + +/// Subtract packed signed 8-bit integers in b from packed 8-bit integers in a using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_subs_epi8&expand=5780) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubsb))] +pub unsafe fn _mm256_maskz_subs_epi8(k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + transmute(vpsubsb256( + a.as_i8x32(), + b.as_i8x32(), + _mm256_setzero_si256().as_i8x32(), + k, + )) +} + +/// Subtract packed signed 8-bit integers in b from packed 8-bit integers in a using saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_subs_epi8&expand=5776) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubsb))] +pub unsafe fn _mm_mask_subs_epi8(src: __m128i, k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + transmute(vpsubsb128(a.as_i8x16(), b.as_i8x16(), src.as_i8x16(), k)) +} + +/// Subtract packed signed 8-bit integers in b from packed 8-bit integers in a using saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_subs_epi8&expand=5777) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubsb))] +pub unsafe fn _mm_maskz_subs_epi8(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + transmute(vpsubsb128( + a.as_i8x16(), + b.as_i8x16(), + _mm_setzero_si128().as_i8x16(), + k, + )) +} + +/// Multiply the packed unsigned 16-bit integers in a and b, producing intermediate 32-bit integers, and store the high 16 bits of the intermediate integers in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mulhi_epu16&expand=3973) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmulhuw))] +pub unsafe fn _mm512_mulhi_epu16(a: __m512i, b: __m512i) -> __m512i { + transmute(vpmulhuw(a.as_u16x32(), b.as_u16x32())) +} + +/// Multiply the packed unsigned 16-bit integers in a and b, producing intermediate 32-bit integers, and store the high 16 bits of the intermediate integers in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_mulhi_epu16&expand=3971) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmulhuw))] +pub unsafe fn _mm512_mask_mulhi_epu16( + src: __m512i, + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + let mul = _mm512_mulhi_epu16(a, b).as_u16x32(); + transmute(simd_select_bitmask(k, mul, src.as_u16x32())) +} + +/// Multiply the packed unsigned 16-bit integers in a and b, producing intermediate 32-bit integers, and store the high 16 bits of the intermediate integers in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_mulhi_epu16&expand=3972) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmulhuw))] +pub unsafe fn _mm512_maskz_mulhi_epu16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let mul = _mm512_mulhi_epu16(a, b).as_u16x32(); + let zero = _mm512_setzero_si512().as_u16x32(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply the packed unsigned 16-bit integers in a and b, producing intermediate 32-bit integers, and store the high 16 bits of the intermediate integers in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_mulhi_epu16&expand=3968) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmulhuw))] +pub unsafe fn _mm256_mask_mulhi_epu16( + src: __m256i, + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + let mul = _mm256_mulhi_epu16(a, b).as_u16x16(); + transmute(simd_select_bitmask(k, mul, src.as_u16x16())) +} + +/// Multiply the packed unsigned 16-bit integers in a and b, producing intermediate 32-bit integers, and store the high 16 bits of the intermediate integers in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_mulhi_epu16&expand=3969) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmulhuw))] +pub unsafe fn _mm256_maskz_mulhi_epu16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let mul = _mm256_mulhi_epu16(a, b).as_u16x16(); + let zero = _mm256_setzero_si256().as_u16x16(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply the packed unsigned 16-bit integers in a and b, producing intermediate 32-bit integers, and store the high 16 bits of the intermediate integers in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_mulhi_epu16&expand=3965) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmulhuw))] +pub unsafe fn _mm_mask_mulhi_epu16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let mul = _mm_mulhi_epu16(a, b).as_u16x8(); + transmute(simd_select_bitmask(k, mul, src.as_u16x8())) +} + +/// Multiply the packed unsigned 16-bit integers in a and b, producing intermediate 32-bit integers, and store the high 16 bits of the intermediate integers in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_mulhi_epu16&expand=3966) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmulhuw))] +pub unsafe fn _mm_maskz_mulhi_epu16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let mul = _mm_mulhi_epu16(a, b).as_u16x8(); + let zero = _mm_setzero_si128().as_u16x8(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply the packed signed 16-bit integers in a and b, producing intermediate 32-bit integers, and store the high 16 bits of the intermediate integers in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mulhi_epi16&expand=3962) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmulhw))] +pub unsafe fn _mm512_mulhi_epi16(a: __m512i, b: __m512i) -> __m512i { + transmute(vpmulhw(a.as_i16x32(), b.as_i16x32())) +} + +/// Multiply the packed signed 16-bit integers in a and b, producing intermediate 32-bit integers, and store the high 16 bits of the intermediate integers in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_mulhi_epi16&expand=3960) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmulhw))] +pub unsafe fn _mm512_mask_mulhi_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + let mul = _mm512_mulhi_epi16(a, b).as_i16x32(); + transmute(simd_select_bitmask(k, mul, src.as_i16x32())) +} + +/// Multiply the packed signed 16-bit integers in a and b, producing intermediate 32-bit integers, and store the high 16 bits of the intermediate integers in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_mulhi_epi16&expand=3961) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmulhw))] +pub unsafe fn _mm512_maskz_mulhi_epi16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let mul = _mm512_mulhi_epi16(a, b).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply the packed signed 16-bit integers in a and b, producing intermediate 32-bit integers, and store the high 16 bits of the intermediate integers in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_mulhi_epi16&expand=3957) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmulhw))] +pub unsafe fn _mm256_mask_mulhi_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + let mul = _mm256_mulhi_epi16(a, b).as_i16x16(); + transmute(simd_select_bitmask(k, mul, src.as_i16x16())) +} + +/// Multiply the packed signed 16-bit integers in a and b, producing intermediate 32-bit integers, and store the high 16 bits of the intermediate integers in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_mulhi_epi16&expand=3958) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmulhw))] +pub unsafe fn _mm256_maskz_mulhi_epi16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let mul = _mm256_mulhi_epi16(a, b).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply the packed signed 16-bit integers in a and b, producing intermediate 32-bit integers, and store the high 16 bits of the intermediate integers in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_mulhi_epi16&expand=3954) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmulhw))] +pub unsafe fn _mm_mask_mulhi_epi16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let mul = _mm_mulhi_epi16(a, b).as_i16x8(); + transmute(simd_select_bitmask(k, mul, src.as_i16x8())) +} + +/// Multiply the packed signed 16-bit integers in a and b, producing intermediate 32-bit integers, and store the high 16 bits of the intermediate integers in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_mulhi_epi16&expand=3955) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmulhw))] +pub unsafe fn _mm_maskz_mulhi_epi16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let mul = _mm_mulhi_epi16(a, b).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply packed signed 16-bit integers in a and b, producing intermediate signed 32-bit integers. Truncate each intermediate integer to the 18 most significant bits, round by adding 1, and store bits \[16:1\] to dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mulhrs_epi16&expand=3986) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmulhrsw))] +pub unsafe fn _mm512_mulhrs_epi16(a: __m512i, b: __m512i) -> __m512i { + transmute(vpmulhrsw(a.as_i16x32(), b.as_i16x32())) +} + +/// Multiply packed signed 16-bit integers in a and b, producing intermediate signed 32-bit integers. Truncate each intermediate integer to the 18 most significant bits, round by adding 1, and store bits \[16:1\] to dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_mulhrs_epi16&expand=3984) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmulhrsw))] +pub unsafe fn _mm512_mask_mulhrs_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + let mul = _mm512_mulhrs_epi16(a, b).as_i16x32(); + transmute(simd_select_bitmask(k, mul, src.as_i16x32())) +} + +/// Multiply packed signed 16-bit integers in a and b, producing intermediate signed 32-bit integers. Truncate each intermediate integer to the 18 most significant bits, round by adding 1, and store bits \[16:1\] to dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_mulhrs_epi16&expand=3985) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmulhrsw))] +pub unsafe fn _mm512_maskz_mulhrs_epi16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let mul = _mm512_mulhrs_epi16(a, b).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply packed signed 16-bit integers in a and b, producing intermediate signed 32-bit integers. Truncate each intermediate integer to the 18 most significant bits, round by adding 1, and store bits \[16:1\] to dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_mulhrs_epi16&expand=3981) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmulhrsw))] +pub unsafe fn _mm256_mask_mulhrs_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + let mul = _mm256_mulhrs_epi16(a, b).as_i16x16(); + transmute(simd_select_bitmask(k, mul, src.as_i16x16())) +} + +/// Multiply packed signed 16-bit integers in a and b, producing intermediate signed 32-bit integers. Truncate each intermediate integer to the 18 most significant bits, round by adding 1, and store bits \[16:1\] to dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_mulhrs_epi16&expand=3982) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmulhrsw))] +pub unsafe fn _mm256_maskz_mulhrs_epi16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let mul = _mm256_mulhrs_epi16(a, b).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply packed signed 16-bit integers in a and b, producing intermediate signed 32-bit integers. Truncate each intermediate integer to the 18 most significant bits, round by adding 1, and store bits \[16:1\] to dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_mulhrs_epi16&expand=3978) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmulhrsw))] +pub unsafe fn _mm_mask_mulhrs_epi16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let mul = _mm_mulhrs_epi16(a, b).as_i16x8(); + transmute(simd_select_bitmask(k, mul, src.as_i16x8())) +} + +/// Multiply packed signed 16-bit integers in a and b, producing intermediate signed 32-bit integers. Truncate each intermediate integer to the 18 most significant bits, round by adding 1, and store bits \[16:1\] to dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_mulhrs_epi16&expand=3979) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmulhrsw))] +pub unsafe fn _mm_maskz_mulhrs_epi16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let mul = _mm_mulhrs_epi16(a, b).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply the packed 16-bit integers in a and b, producing intermediate 32-bit integers, and store the low 16 bits of the intermediate integers in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mullo_epi16&expand=3996) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmullw))] +pub unsafe fn _mm512_mullo_epi16(a: __m512i, b: __m512i) -> __m512i { + transmute(simd_mul(a.as_i16x32(), b.as_i16x32())) +} + +/// Multiply the packed 16-bit integers in a and b, producing intermediate 32-bit integers, and store the low 16 bits of the intermediate integers in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_mullo_epi16&expand=3994) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmullw))] +pub unsafe fn _mm512_mask_mullo_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + let mul = _mm512_mullo_epi16(a, b).as_i16x32(); + transmute(simd_select_bitmask(k, mul, src.as_i16x32())) +} + +/// Multiply the packed 16-bit integers in a and b, producing intermediate 32-bit integers, and store the low 16 bits of the intermediate integers in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_mullo_epi16&expand=3995) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmullw))] +pub unsafe fn _mm512_maskz_mullo_epi16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let mul = _mm512_mullo_epi16(a, b).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply the packed 16-bit integers in a and b, producing intermediate 32-bit integers, and store the low 16 bits of the intermediate integers in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_mullo_epi16&expand=3991) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmullw))] +pub unsafe fn _mm256_mask_mullo_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + let mul = _mm256_mullo_epi16(a, b).as_i16x16(); + transmute(simd_select_bitmask(k, mul, src.as_i16x16())) +} + +/// Multiply the packed 16-bit integers in a and b, producing intermediate 32-bit integers, and store the low 16 bits of the intermediate integers in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_mullo_epi16&expand=3992) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmullw))] +pub unsafe fn _mm256_maskz_mullo_epi16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let mul = _mm256_mullo_epi16(a, b).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply the packed 16-bit integers in a and b, producing intermediate 32-bit integers, and store the low 16 bits of the intermediate integers in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_mullo_epi16&expand=3988) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmullw))] +pub unsafe fn _mm_mask_mullo_epi16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let mul = _mm_mullo_epi16(a, b).as_i16x8(); + transmute(simd_select_bitmask(k, mul, src.as_i16x8())) +} + +/// Multiply the packed 16-bit integers in a and b, producing intermediate 32-bit integers, and store the low 16 bits of the intermediate integers in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_mullo_epi16&expand=3989) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmullw))] +pub unsafe fn _mm_maskz_mullo_epi16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let mul = _mm_mullo_epi16(a, b).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Compare packed unsigned 16-bit integers in a and b, and store packed maximum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_max_epu16&expand=3609) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmaxuw))] +pub unsafe fn _mm512_max_epu16(a: __m512i, b: __m512i) -> __m512i { + transmute(vpmaxuw(a.as_u16x32(), b.as_u16x32())) +} + +/// Compare packed unsigned 16-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_max_epu16&expand=3607) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmaxuw))] +pub unsafe fn _mm512_mask_max_epu16(src: __m512i, k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let max = _mm512_max_epu16(a, b).as_u16x32(); + transmute(simd_select_bitmask(k, max, src.as_u16x32())) +} + +/// Compare packed unsigned 16-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_max_epu16&expand=3608) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmaxuw))] +pub unsafe fn _mm512_maskz_max_epu16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let max = _mm512_max_epu16(a, b).as_u16x32(); + let zero = _mm512_setzero_si512().as_u16x32(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed unsigned 16-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_max_epu16&expand=3604) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxuw))] +pub unsafe fn _mm256_mask_max_epu16(src: __m256i, k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let max = _mm256_max_epu16(a, b).as_u16x16(); + transmute(simd_select_bitmask(k, max, src.as_u16x16())) +} + +/// Compare packed unsigned 16-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_max_epu16&expand=3605) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxuw))] +pub unsafe fn _mm256_maskz_max_epu16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let max = _mm256_max_epu16(a, b).as_u16x16(); + let zero = _mm256_setzero_si256().as_u16x16(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed unsigned 16-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_max_epu16&expand=3601) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxuw))] +pub unsafe fn _mm_mask_max_epu16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let max = _mm_max_epu16(a, b).as_u16x8(); + transmute(simd_select_bitmask(k, max, src.as_u16x8())) +} + +/// Compare packed unsigned 16-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_max_epu16&expand=3602) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxuw))] +pub unsafe fn _mm_maskz_max_epu16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let max = _mm_max_epu16(a, b).as_u16x8(); + let zero = _mm_setzero_si128().as_u16x8(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed unsigned 8-bit integers in a and b, and store packed maximum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_max_epu8&expand=3636) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmaxub))] +pub unsafe fn _mm512_max_epu8(a: __m512i, b: __m512i) -> __m512i { + transmute(vpmaxub(a.as_u8x64(), b.as_u8x64())) +} + +/// Compare packed unsigned 8-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_max_epu8&expand=3634) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmaxub))] +pub unsafe fn _mm512_mask_max_epu8(src: __m512i, k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + let max = _mm512_max_epu8(a, b).as_u8x64(); + transmute(simd_select_bitmask(k, max, src.as_u8x64())) +} + +/// Compare packed unsigned 8-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_max_epu8&expand=3635) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmaxub))] +pub unsafe fn _mm512_maskz_max_epu8(k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + let max = _mm512_max_epu8(a, b).as_u8x64(); + let zero = _mm512_setzero_si512().as_u8x64(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed unsigned 8-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_max_epu8&expand=3631) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxub))] +pub unsafe fn _mm256_mask_max_epu8(src: __m256i, k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + let max = _mm256_max_epu8(a, b).as_u8x32(); + transmute(simd_select_bitmask(k, max, src.as_u8x32())) +} + +/// Compare packed unsigned 8-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_max_epu8&expand=3632) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxub))] +pub unsafe fn _mm256_maskz_max_epu8(k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + let max = _mm256_max_epu8(a, b).as_u8x32(); + let zero = _mm256_setzero_si256().as_u8x32(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed unsigned 8-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_max_epu8&expand=3628) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxub))] +pub unsafe fn _mm_mask_max_epu8(src: __m128i, k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let max = _mm_max_epu8(a, b).as_u8x16(); + transmute(simd_select_bitmask(k, max, src.as_u8x16())) +} + +/// Compare packed unsigned 8-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_max_epu8&expand=3629) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxub))] +pub unsafe fn _mm_maskz_max_epu8(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let max = _mm_max_epu8(a, b).as_u8x16(); + let zero = _mm_setzero_si128().as_u8x16(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed signed 16-bit integers in a and b, and store packed maximum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_max_epi16&expand=3573) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmaxsw))] +pub unsafe fn _mm512_max_epi16(a: __m512i, b: __m512i) -> __m512i { + transmute(vpmaxsw(a.as_i16x32(), b.as_i16x32())) +} + +/// Compare packed signed 16-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_max_epi16&expand=3571) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmaxsw))] +pub unsafe fn _mm512_mask_max_epi16(src: __m512i, k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let max = _mm512_max_epi16(a, b).as_i16x32(); + transmute(simd_select_bitmask(k, max, src.as_i16x32())) +} + +/// Compare packed signed 16-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_max_epi16&expand=3572) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmaxsw))] +pub unsafe fn _mm512_maskz_max_epi16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let max = _mm512_max_epi16(a, b).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed signed 16-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_max_epi16&expand=3568) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxsw))] +pub unsafe fn _mm256_mask_max_epi16(src: __m256i, k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let max = _mm256_max_epi16(a, b).as_i16x16(); + transmute(simd_select_bitmask(k, max, src.as_i16x16())) +} + +/// Compare packed signed 16-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_max_epi16&expand=3569) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxsw))] +pub unsafe fn _mm256_maskz_max_epi16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let max = _mm256_max_epi16(a, b).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed signed 16-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_max_epi16&expand=3565) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxsw))] +pub unsafe fn _mm_mask_max_epi16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let max = _mm_max_epi16(a, b).as_i16x8(); + transmute(simd_select_bitmask(k, max, src.as_i16x8())) +} + +/// Compare packed signed 16-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_max_epi16&expand=3566) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxsw))] +pub unsafe fn _mm_maskz_max_epi16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let max = _mm_max_epi16(a, b).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed signed 8-bit integers in a and b, and store packed maximum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_max_epi8&expand=3600) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmaxsb))] +pub unsafe fn _mm512_max_epi8(a: __m512i, b: __m512i) -> __m512i { + transmute(vpmaxsb(a.as_i8x64(), b.as_i8x64())) +} + +/// Compare packed signed 8-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_max_epi8&expand=3598) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmaxsb))] +pub unsafe fn _mm512_mask_max_epi8(src: __m512i, k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + let max = _mm512_max_epi8(a, b).as_i8x64(); + transmute(simd_select_bitmask(k, max, src.as_i8x64())) +} + +/// Compare packed signed 8-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_max_epi8&expand=3599) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmaxsb))] +pub unsafe fn _mm512_maskz_max_epi8(k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + let max = _mm512_max_epi8(a, b).as_i8x64(); + let zero = _mm512_setzero_si512().as_i8x64(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed signed 8-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_max_epi8&expand=3595) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxsb))] +pub unsafe fn _mm256_mask_max_epi8(src: __m256i, k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + let max = _mm256_max_epi8(a, b).as_i8x32(); + transmute(simd_select_bitmask(k, max, src.as_i8x32())) +} + +/// Compare packed signed 8-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_max_epi8&expand=3596) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxsb))] +pub unsafe fn _mm256_maskz_max_epi8(k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + let max = _mm256_max_epi8(a, b).as_i8x32(); + let zero = _mm256_setzero_si256().as_i8x32(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed signed 8-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_max_epi8&expand=3592) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxsb))] +pub unsafe fn _mm_mask_max_epi8(src: __m128i, k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let max = _mm_max_epi8(a, b).as_i8x16(); + transmute(simd_select_bitmask(k, max, src.as_i8x16())) +} + +/// Compare packed signed 8-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_max_epi8&expand=3593) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxsb))] +pub unsafe fn _mm_maskz_max_epi8(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let max = _mm_max_epi8(a, b).as_i8x16(); + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed unsigned 16-bit integers in a and b, and store packed minimum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_min_epu16&expand=3723) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpminuw))] +pub unsafe fn _mm512_min_epu16(a: __m512i, b: __m512i) -> __m512i { + transmute(vpminuw(a.as_u16x32(), b.as_u16x32())) +} + +/// Compare packed unsigned 16-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_min_epu16&expand=3721) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpminuw))] +pub unsafe fn _mm512_mask_min_epu16(src: __m512i, k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let min = _mm512_min_epu16(a, b).as_u16x32(); + transmute(simd_select_bitmask(k, min, src.as_u16x32())) +} + +/// Compare packed unsigned 16-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_min_epu16&expand=3722) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpminuw))] +pub unsafe fn _mm512_maskz_min_epu16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let min = _mm512_min_epu16(a, b).as_u16x32(); + let zero = _mm512_setzero_si512().as_u16x32(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed unsigned 16-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_min_epu16&expand=3718) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpminuw))] +pub unsafe fn _mm256_mask_min_epu16(src: __m256i, k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let min = _mm256_min_epu16(a, b).as_u16x16(); + transmute(simd_select_bitmask(k, min, src.as_u16x16())) +} + +/// Compare packed unsigned 16-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_min_epu16&expand=3719) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpminuw))] +pub unsafe fn _mm256_maskz_min_epu16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let min = _mm256_min_epu16(a, b).as_u16x16(); + let zero = _mm256_setzero_si256().as_u16x16(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed unsigned 16-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_min_epu16&expand=3715) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpminuw))] +pub unsafe fn _mm_mask_min_epu16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let min = _mm_min_epu16(a, b).as_u16x8(); + transmute(simd_select_bitmask(k, min, src.as_u16x8())) +} + +/// Compare packed unsigned 16-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_min_epu16&expand=3716) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpminuw))] +pub unsafe fn _mm_maskz_min_epu16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let min = _mm_min_epu16(a, b).as_u16x8(); + let zero = _mm_setzero_si128().as_u16x8(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed unsigned 8-bit integers in a and b, and store packed minimum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_min_epu8&expand=3750) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpminub))] +pub unsafe fn _mm512_min_epu8(a: __m512i, b: __m512i) -> __m512i { + transmute(vpminub(a.as_u8x64(), b.as_u8x64())) +} + +/// Compare packed unsigned 8-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_min_epu8&expand=3748) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpminub))] +pub unsafe fn _mm512_mask_min_epu8(src: __m512i, k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + let min = _mm512_min_epu8(a, b).as_u8x64(); + transmute(simd_select_bitmask(k, min, src.as_u8x64())) +} + +/// Compare packed unsigned 8-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_min_epu8&expand=3749) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpminub))] +pub unsafe fn _mm512_maskz_min_epu8(k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + let min = _mm512_min_epu8(a, b).as_u8x64(); + let zero = _mm512_setzero_si512().as_u8x64(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed unsigned 8-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_min_epu8&expand=3745) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpminub))] +pub unsafe fn _mm256_mask_min_epu8(src: __m256i, k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + let min = _mm256_min_epu8(a, b).as_u8x32(); + transmute(simd_select_bitmask(k, min, src.as_u8x32())) +} + +/// Compare packed unsigned 8-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_min_epu8&expand=3746) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpminub))] +pub unsafe fn _mm256_maskz_min_epu8(k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + let min = _mm256_min_epu8(a, b).as_u8x32(); + let zero = _mm256_setzero_si256().as_u8x32(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed unsigned 8-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_min_epu8&expand=3742) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpminub))] +pub unsafe fn _mm_mask_min_epu8(src: __m128i, k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let min = _mm_min_epu8(a, b).as_u8x16(); + transmute(simd_select_bitmask(k, min, src.as_u8x16())) +} + +/// Compare packed unsigned 8-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_min_epu8&expand=3743) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpminub))] +pub unsafe fn _mm_maskz_min_epu8(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let min = _mm_min_epu8(a, b).as_u8x16(); + let zero = _mm_setzero_si128().as_u8x16(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed signed 16-bit integers in a and b, and store packed minimum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_min_epi16&expand=3687) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpminsw))] +pub unsafe fn _mm512_min_epi16(a: __m512i, b: __m512i) -> __m512i { + transmute(vpminsw(a.as_i16x32(), b.as_i16x32())) +} + +/// Compare packed signed 16-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_min_epi16&expand=3685) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpminsw))] +pub unsafe fn _mm512_mask_min_epi16(src: __m512i, k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let min = _mm512_min_epi16(a, b).as_i16x32(); + transmute(simd_select_bitmask(k, min, src.as_i16x32())) +} + +/// Compare packed signed 16-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_min_epi16&expand=3686) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpminsw))] +pub unsafe fn _mm512_maskz_min_epi16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let min = _mm512_min_epi16(a, b).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed signed 16-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_min_epi16&expand=3682) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpminsw))] +pub unsafe fn _mm256_mask_min_epi16(src: __m256i, k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let min = _mm256_min_epi16(a, b).as_i16x16(); + transmute(simd_select_bitmask(k, min, src.as_i16x16())) +} + +/// Compare packed signed 16-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_min_epi16&expand=3683) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpminsw))] +pub unsafe fn _mm256_maskz_min_epi16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let min = _mm256_min_epi16(a, b).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed signed 16-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_min_epi16&expand=3679) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpminsw))] +pub unsafe fn _mm_mask_min_epi16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let min = _mm_min_epi16(a, b).as_i16x8(); + transmute(simd_select_bitmask(k, min, src.as_i16x8())) +} + +/// Compare packed signed 16-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_min_epi16&expand=3680) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpminsw))] +pub unsafe fn _mm_maskz_min_epi16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let min = _mm_min_epi16(a, b).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed signed 8-bit integers in a and b, and store packed minimum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_min_epi8&expand=3714) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpminsb))] +pub unsafe fn _mm512_min_epi8(a: __m512i, b: __m512i) -> __m512i { + transmute(vpminsb(a.as_i8x64(), b.as_i8x64())) +} + +/// Compare packed signed 8-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_min_epi8&expand=3712) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpminsb))] +pub unsafe fn _mm512_mask_min_epi8(src: __m512i, k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + let min = _mm512_min_epi8(a, b).as_i8x64(); + transmute(simd_select_bitmask(k, min, src.as_i8x64())) +} + +/// Compare packed signed 8-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_min_epi8&expand=3713) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpminsb))] +pub unsafe fn _mm512_maskz_min_epi8(k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + let min = _mm512_min_epi8(a, b).as_i8x64(); + let zero = _mm512_setzero_si512().as_i8x64(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed signed 8-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_min_epi8&expand=3709) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpminsb))] +pub unsafe fn _mm256_mask_min_epi8(src: __m256i, k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + let min = _mm256_min_epi8(a, b).as_i8x32(); + transmute(simd_select_bitmask(k, min, src.as_i8x32())) +} + +/// Compare packed signed 8-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_min_epi8&expand=3710) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpminsb))] +pub unsafe fn _mm256_maskz_min_epi8(k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + let min = _mm256_min_epi8(a, b).as_i8x32(); + let zero = _mm256_setzero_si256().as_i8x32(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed signed 8-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_min_epi8&expand=3706) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpminsb))] +pub unsafe fn _mm_mask_min_epi8(src: __m128i, k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let min = _mm_min_epi8(a, b).as_i8x16(); + transmute(simd_select_bitmask(k, min, src.as_i8x16())) +} + +/// Compare packed signed 8-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_min_epi8&expand=3707) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpminsb))] +pub unsafe fn _mm_maskz_min_epi8(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let min = _mm_min_epi8(a, b).as_i8x16(); + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed unsigned 16-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_cmplt_epu16_mask&expand=1050) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmplt_epu16_mask(a: __m512i, b: __m512i) -> __mmask32 { + simd_bitmask::(simd_lt(a.as_u16x32(), b.as_u16x32())) +} + +/// Compare packed unsigned 16-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmplt_epu16_mask&expand=1051) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmplt_epu16_mask(k1: __mmask32, a: __m512i, b: __m512i) -> __mmask32 { + _mm512_cmplt_epu16_mask(a, b) & k1 +} + +/// Compare packed unsigned 16-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_cmplt_epu16_mask&expand=1050) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmplt_epu16_mask(a: __m256i, b: __m256i) -> __mmask16 { + simd_bitmask::(simd_lt(a.as_u16x16(), b.as_u16x16())) +} + +/// Compare packed unsigned 16-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmplt_epu16_mask&expand=1049) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmplt_epu16_mask(k1: __mmask16, a: __m256i, b: __m256i) -> __mmask16 { + _mm256_cmplt_epu16_mask(a, b) & k1 +} + +/// Compare packed unsigned 16-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmplt_epi16_mask&expand=1018) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmplt_epu16_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_lt(a.as_u16x8(), b.as_u16x8())) +} + +/// Compare packed unsigned 16-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmplt_epi16_mask&expand=1019) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmplt_epu16_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmplt_epu16_mask(a, b) & k1 +} + +/// Compare packed unsigned 8-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm512_cmplt_epu8_mask&expand=1068) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmplt_epu8_mask(a: __m512i, b: __m512i) -> __mmask64 { + simd_bitmask::(simd_lt(a.as_u8x64(), b.as_u8x64())) +} + +/// Compare packed unsigned 8-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmplt_epu8_mask&expand=1069) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmplt_epu8_mask(k1: __mmask64, a: __m512i, b: __m512i) -> __mmask64 { + _mm512_cmplt_epu8_mask(a, b) & k1 +} + +/// Compare packed unsigned 8-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmplt_epu8_mask&expand=1066) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmplt_epu8_mask(a: __m256i, b: __m256i) -> __mmask32 { + simd_bitmask::(simd_lt(a.as_u8x32(), b.as_u8x32())) +} + +/// Compare packed unsigned 8-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmplt_epu8_mask&expand=1067) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmplt_epu8_mask(k1: __mmask32, a: __m256i, b: __m256i) -> __mmask32 { + _mm256_cmplt_epu8_mask(a, b) & k1 +} + +/// Compare packed unsigned 8-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmplt_epu8_mask&expand=1064) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmplt_epu8_mask(a: __m128i, b: __m128i) -> __mmask16 { + simd_bitmask::(simd_lt(a.as_u8x16(), b.as_u8x16())) +} + +/// Compare packed unsigned 8-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmplt_epu8_mask&expand=1065) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmplt_epu8_mask(k1: __mmask16, a: __m128i, b: __m128i) -> __mmask16 { + _mm_cmplt_epu8_mask(a, b) & k1 +} + +/// Compare packed signed 16-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmplt_epi16_mask&expand=1022) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmplt_epi16_mask(a: __m512i, b: __m512i) -> __mmask32 { + simd_bitmask::(simd_lt(a.as_i16x32(), b.as_i16x32())) +} + +/// Compare packed signed 16-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmplt_epi16_mask&expand=1023) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmplt_epi16_mask(k1: __mmask32, a: __m512i, b: __m512i) -> __mmask32 { + _mm512_cmplt_epi16_mask(a, b) & k1 +} + +/// Compare packed signed 16-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmplt_epi16_mask&expand=1020) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmplt_epi16_mask(a: __m256i, b: __m256i) -> __mmask16 { + simd_bitmask::(simd_lt(a.as_i16x16(), b.as_i16x16())) +} + +/// Compare packed signed 16-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmplt_epi16_mask&expand=1021) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmplt_epi16_mask(k1: __mmask16, a: __m256i, b: __m256i) -> __mmask16 { + _mm256_cmplt_epi16_mask(a, b) & k1 +} + +/// Compare packed signed 16-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmplt_epi16_mask&expand=1018) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmplt_epi16_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_lt(a.as_i16x8(), b.as_i16x8())) +} + +/// Compare packed signed 16-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmplt_epi16_mask&expand=1019) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmplt_epi16_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmplt_epi16_mask(a, b) & k1 +} + +/// Compare packed signed 8-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmplt_epi8_mask&expand=1044) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmplt_epi8_mask(a: __m512i, b: __m512i) -> __mmask64 { + simd_bitmask::(simd_lt(a.as_i8x64(), b.as_i8x64())) +} + +/// Compare packed signed 8-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmplt_epi8_mask&expand=1045) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmplt_epi8_mask(k1: __mmask64, a: __m512i, b: __m512i) -> __mmask64 { + _mm512_cmplt_epi8_mask(a, b) & k1 +} + +/// Compare packed signed 8-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmplt_epi8_mask&expand=1042) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmplt_epi8_mask(a: __m256i, b: __m256i) -> __mmask32 { + simd_bitmask::(simd_lt(a.as_i8x32(), b.as_i8x32())) +} + +/// Compare packed signed 8-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmplt_epi8_mask&expand=1043) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmplt_epi8_mask(k1: __mmask32, a: __m256i, b: __m256i) -> __mmask32 { + _mm256_cmplt_epi8_mask(a, b) & k1 +} + +/// Compare packed signed 8-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmplt_epi8_mask&expand=1040) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmplt_epi8_mask(a: __m128i, b: __m128i) -> __mmask16 { + simd_bitmask::(simd_lt(a.as_i8x16(), b.as_i8x16())) +} + +/// Compare packed signed 8-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmplt_epi8_mask&expand=1041) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmplt_epi8_mask(k1: __mmask16, a: __m128i, b: __m128i) -> __mmask16 { + _mm_cmplt_epi8_mask(a, b) & k1 +} + +/// Compare packed unsigned 16-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpgt_epu16_mask&expand=927) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmpgt_epu16_mask(a: __m512i, b: __m512i) -> __mmask32 { + simd_bitmask::(simd_gt(a.as_u16x32(), b.as_u16x32())) +} + +/// Compare packed unsigned 16-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpgt_epu16_mask&expand=928) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmpgt_epu16_mask(k1: __mmask32, a: __m512i, b: __m512i) -> __mmask32 { + _mm512_cmpgt_epu16_mask(a, b) & k1 +} + +/// Compare packed unsigned 16-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpgt_epu16_mask&expand=925) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmpgt_epu16_mask(a: __m256i, b: __m256i) -> __mmask16 { + simd_bitmask::(simd_gt(a.as_u16x16(), b.as_u16x16())) +} + +/// Compare packed unsigned 16-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpgt_epu16_mask&expand=926) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmpgt_epu16_mask(k1: __mmask16, a: __m256i, b: __m256i) -> __mmask16 { + _mm256_cmpgt_epu16_mask(a, b) & k1 +} + +/// Compare packed unsigned 16-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpgt_epu16_mask&expand=923) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmpgt_epu16_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_gt(a.as_u16x8(), b.as_u16x8())) +} + +/// Compare packed unsigned 16-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpgt_epu16_mask&expand=924) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmpgt_epu16_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpgt_epu16_mask(a, b) & k1 +} + +/// Compare packed unsigned 8-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpgt_epu8_mask&expand=945) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmpgt_epu8_mask(a: __m512i, b: __m512i) -> __mmask64 { + simd_bitmask::(simd_gt(a.as_u8x64(), b.as_u8x64())) +} + +/// Compare packed unsigned 8-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpgt_epu8_mask&expand=946) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmpgt_epu8_mask(k1: __mmask64, a: __m512i, b: __m512i) -> __mmask64 { + _mm512_cmpgt_epu8_mask(a, b) & k1 +} + +/// Compare packed unsigned 8-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpgt_epu8_mask&expand=943) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmpgt_epu8_mask(a: __m256i, b: __m256i) -> __mmask32 { + simd_bitmask::(simd_gt(a.as_u8x32(), b.as_u8x32())) +} + +/// Compare packed unsigned 8-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpgt_epu8_mask&expand=944) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmpgt_epu8_mask(k1: __mmask32, a: __m256i, b: __m256i) -> __mmask32 { + _mm256_cmpgt_epu8_mask(a, b) & k1 +} + +/// Compare packed unsigned 8-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpgt_epu8_mask&expand=941) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmpgt_epu8_mask(a: __m128i, b: __m128i) -> __mmask16 { + simd_bitmask::(simd_gt(a.as_u8x16(), b.as_u8x16())) +} + +/// Compare packed unsigned 8-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpgt_epu8_mask&expand=942) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmpgt_epu8_mask(k1: __mmask16, a: __m128i, b: __m128i) -> __mmask16 { + _mm_cmpgt_epu8_mask(a, b) & k1 +} + +/// Compare packed signed 16-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpgt_epi16_mask&expand=897) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmpgt_epi16_mask(a: __m512i, b: __m512i) -> __mmask32 { + simd_bitmask::(simd_gt(a.as_i16x32(), b.as_i16x32())) +} + +/// Compare packed signed 16-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpgt_epi16_mask&expand=898) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmpgt_epi16_mask(k1: __mmask32, a: __m512i, b: __m512i) -> __mmask32 { + _mm512_cmpgt_epi16_mask(a, b) & k1 +} + +/// Compare packed signed 16-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpgt_epi16_mask&expand=895) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmpgt_epi16_mask(a: __m256i, b: __m256i) -> __mmask16 { + simd_bitmask::(simd_gt(a.as_i16x16(), b.as_i16x16())) +} + +/// Compare packed signed 16-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpgt_epi16_mask&expand=896) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmpgt_epi16_mask(k1: __mmask16, a: __m256i, b: __m256i) -> __mmask16 { + _mm256_cmpgt_epi16_mask(a, b) & k1 +} + +/// Compare packed signed 16-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpgt_epi16_mask&expand=893) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmpgt_epi16_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_gt(a.as_i16x8(), b.as_i16x8())) +} + +/// Compare packed signed 16-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpgt_epi16_mask&expand=894) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmpgt_epi16_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpgt_epi16_mask(a, b) & k1 +} + +/// Compare packed signed 8-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpgt_epi8_mask&expand=921) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmpgt_epi8_mask(a: __m512i, b: __m512i) -> __mmask64 { + simd_bitmask::(simd_gt(a.as_i8x64(), b.as_i8x64())) +} + +/// Compare packed signed 8-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpgt_epi8_mask&expand=922) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmpgt_epi8_mask(k1: __mmask64, a: __m512i, b: __m512i) -> __mmask64 { + _mm512_cmpgt_epi8_mask(a, b) & k1 +} + +/// Compare packed signed 8-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpgt_epi8_mask&expand=919) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmpgt_epi8_mask(a: __m256i, b: __m256i) -> __mmask32 { + simd_bitmask::(simd_gt(a.as_i8x32(), b.as_i8x32())) +} + +/// Compare packed signed 8-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpgt_epi8_mask&expand=920) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmpgt_epi8_mask(k1: __mmask32, a: __m256i, b: __m256i) -> __mmask32 { + _mm256_cmpgt_epi8_mask(a, b) & k1 +} + +/// Compare packed signed 8-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpgt_epi8_mask&expand=917) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmpgt_epi8_mask(a: __m128i, b: __m128i) -> __mmask16 { + simd_bitmask::(simd_gt(a.as_i8x16(), b.as_i8x16())) +} + +/// Compare packed signed 8-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpgt_epi8_mask&expand=918) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmpgt_epi8_mask(k1: __mmask16, a: __m128i, b: __m128i) -> __mmask16 { + _mm_cmpgt_epi8_mask(a, b) & k1 +} + +/// Compare packed unsigned 16-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmple_epu16_mask&expand=989) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmple_epu16_mask(a: __m512i, b: __m512i) -> __mmask32 { + simd_bitmask::(simd_le(a.as_u16x32(), b.as_u16x32())) +} + +/// Compare packed unsigned 16-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmple_epu16_mask&expand=990) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmple_epu16_mask(k1: __mmask32, a: __m512i, b: __m512i) -> __mmask32 { + _mm512_cmple_epu16_mask(a, b) & k1 +} + +/// Compare packed unsigned 16-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmple_epu16_mask&expand=987) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmple_epu16_mask(a: __m256i, b: __m256i) -> __mmask16 { + simd_bitmask::(simd_le(a.as_u16x16(), b.as_u16x16())) +} + +/// Compare packed unsigned 16-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmple_epu16_mask&expand=988) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmple_epu16_mask(k1: __mmask16, a: __m256i, b: __m256i) -> __mmask16 { + _mm256_cmple_epu16_mask(a, b) & k1 +} + +/// Compare packed unsigned 16-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmple_epu16_mask&expand=985) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmple_epu16_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_le(a.as_u16x8(), b.as_u16x8())) +} + +/// Compare packed unsigned 16-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmple_epu16_mask&expand=986) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmple_epu16_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmple_epu16_mask(a, b) & k1 +} + +/// Compare packed unsigned 8-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmple_epu8_mask&expand=1007) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmple_epu8_mask(a: __m512i, b: __m512i) -> __mmask64 { + simd_bitmask::(simd_le(a.as_u8x64(), b.as_u8x64())) +} + +/// Compare packed unsigned 8-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmple_epu8_mask&expand=1008) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmple_epu8_mask(k1: __mmask64, a: __m512i, b: __m512i) -> __mmask64 { + _mm512_cmple_epu8_mask(a, b) & k1 +} + +/// Compare packed unsigned 8-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmple_epu8_mask&expand=1005) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmple_epu8_mask(a: __m256i, b: __m256i) -> __mmask32 { + simd_bitmask::(simd_le(a.as_u8x32(), b.as_u8x32())) +} + +/// Compare packed unsigned 8-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmple_epu8_mask&expand=1006) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmple_epu8_mask(k1: __mmask32, a: __m256i, b: __m256i) -> __mmask32 { + _mm256_cmple_epu8_mask(a, b) & k1 +} + +/// Compare packed unsigned 8-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmple_epu8_mask&expand=1003) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmple_epu8_mask(a: __m128i, b: __m128i) -> __mmask16 { + simd_bitmask::(simd_le(a.as_u8x16(), b.as_u8x16())) +} + +/// Compare packed unsigned 8-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmple_epu8_mask&expand=1004) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmple_epu8_mask(k1: __mmask16, a: __m128i, b: __m128i) -> __mmask16 { + _mm_cmple_epu8_mask(a, b) & k1 +} + +/// Compare packed signed 16-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmple_epi16_mask&expand=965) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmple_epi16_mask(a: __m512i, b: __m512i) -> __mmask32 { + simd_bitmask::(simd_le(a.as_i16x32(), b.as_i16x32())) +} + +/// Compare packed signed 16-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmple_epi16_mask&expand=966) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmple_epi16_mask(k1: __mmask32, a: __m512i, b: __m512i) -> __mmask32 { + _mm512_cmple_epi16_mask(a, b) & k1 +} + +/// Compare packed signed 16-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmple_epi16_mask&expand=963) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmple_epi16_mask(a: __m256i, b: __m256i) -> __mmask16 { + simd_bitmask::(simd_le(a.as_i16x16(), b.as_i16x16())) +} + +/// Compare packed signed 16-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmple_epi16_mask&expand=964) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmple_epi16_mask(k1: __mmask16, a: __m256i, b: __m256i) -> __mmask16 { + _mm256_cmple_epi16_mask(a, b) & k1 +} + +/// Compare packed signed 16-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmple_epi16_mask&expand=961) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmple_epi16_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_le(a.as_i16x8(), b.as_i16x8())) +} + +/// Compare packed signed 16-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmple_epi16_mask&expand=962) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmple_epi16_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmple_epi16_mask(a, b) & k1 +} + +/// Compare packed signed 8-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmple_epi8_mask&expand=983) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmple_epi8_mask(a: __m512i, b: __m512i) -> __mmask64 { + simd_bitmask::(simd_le(a.as_i8x64(), b.as_i8x64())) +} + +/// Compare packed signed 8-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmple_epi8_mask&expand=984) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmple_epi8_mask(k1: __mmask64, a: __m512i, b: __m512i) -> __mmask64 { + _mm512_cmple_epi8_mask(a, b) & k1 +} + +/// Compare packed signed 8-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmple_epi8_mask&expand=981) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmple_epi8_mask(a: __m256i, b: __m256i) -> __mmask32 { + simd_bitmask::(simd_le(a.as_i8x32(), b.as_i8x32())) +} + +/// Compare packed signed 8-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmple_epi8_mask&expand=982) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmple_epi8_mask(k1: __mmask32, a: __m256i, b: __m256i) -> __mmask32 { + _mm256_cmple_epi8_mask(a, b) & k1 +} + +/// Compare packed signed 8-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmple_epi8_mask&expand=979) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmple_epi8_mask(a: __m128i, b: __m128i) -> __mmask16 { + simd_bitmask::(simd_le(a.as_i8x16(), b.as_i8x16())) +} + +/// Compare packed signed 8-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmple_epi8_mask&expand=980) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmple_epi8_mask(k1: __mmask16, a: __m128i, b: __m128i) -> __mmask16 { + _mm_cmple_epi8_mask(a, b) & k1 +} + +/// Compare packed unsigned 16-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpge_epu16_mask&expand=867) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmpge_epu16_mask(a: __m512i, b: __m512i) -> __mmask32 { + simd_bitmask::(simd_ge(a.as_u16x32(), b.as_u16x32())) +} + +/// Compare packed unsigned 16-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpge_epu16_mask&expand=868) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmpge_epu16_mask(k1: __mmask32, a: __m512i, b: __m512i) -> __mmask32 { + _mm512_cmpge_epu16_mask(a, b) & k1 +} + +/// Compare packed unsigned 16-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpge_epu16_mask&expand=865) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmpge_epu16_mask(a: __m256i, b: __m256i) -> __mmask16 { + simd_bitmask::(simd_ge(a.as_u16x16(), b.as_u16x16())) +} + +/// Compare packed unsigned 16-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpge_epu16_mask&expand=866) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmpge_epu16_mask(k1: __mmask16, a: __m256i, b: __m256i) -> __mmask16 { + _mm256_cmpge_epu16_mask(a, b) & k1 +} + +/// Compare packed unsigned 16-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpge_epu16_mask&expand=863) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmpge_epu16_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_ge(a.as_u16x8(), b.as_u16x8())) +} + +/// Compare packed unsigned 16-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpge_epu16_mask&expand=864) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmpge_epu16_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpge_epu16_mask(a, b) & k1 +} + +/// Compare packed unsigned 8-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpge_epu8_mask&expand=885) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmpge_epu8_mask(a: __m512i, b: __m512i) -> __mmask64 { + simd_bitmask::(simd_ge(a.as_u8x64(), b.as_u8x64())) +} + +/// Compare packed unsigned 8-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpge_epu8_mask&expand=886) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmpge_epu8_mask(k1: __mmask64, a: __m512i, b: __m512i) -> __mmask64 { + _mm512_cmpge_epu8_mask(a, b) & k1 +} + +/// Compare packed unsigned 8-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpge_epu8_mask&expand=883) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmpge_epu8_mask(a: __m256i, b: __m256i) -> __mmask32 { + simd_bitmask::(simd_ge(a.as_u8x32(), b.as_u8x32())) +} + +/// Compare packed unsigned 8-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpge_epu8_mask&expand=884) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmpge_epu8_mask(k1: __mmask32, a: __m256i, b: __m256i) -> __mmask32 { + _mm256_cmpge_epu8_mask(a, b) & k1 +} + +/// Compare packed unsigned 8-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpge_epu8_mask&expand=881) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmpge_epu8_mask(a: __m128i, b: __m128i) -> __mmask16 { + simd_bitmask::(simd_ge(a.as_u8x16(), b.as_u8x16())) +} + +/// Compare packed unsigned 8-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpge_epu8_mask&expand=882) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmpge_epu8_mask(k1: __mmask16, a: __m128i, b: __m128i) -> __mmask16 { + _mm_cmpge_epu8_mask(a, b) & k1 +} + +/// Compare packed signed 16-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpge_epi16_mask&expand=843) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmpge_epi16_mask(a: __m512i, b: __m512i) -> __mmask32 { + simd_bitmask::(simd_ge(a.as_i16x32(), b.as_i16x32())) +} + +/// Compare packed signed 16-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpge_epi16_mask&expand=844) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmpge_epi16_mask(k1: __mmask32, a: __m512i, b: __m512i) -> __mmask32 { + _mm512_cmpge_epi16_mask(a, b) & k1 +} + +/// Compare packed signed 16-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpge_epi16_mask&expand=841) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmpge_epi16_mask(a: __m256i, b: __m256i) -> __mmask16 { + simd_bitmask::(simd_ge(a.as_i16x16(), b.as_i16x16())) +} + +/// Compare packed signed 16-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpge_epi16_mask&expand=842) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmpge_epi16_mask(k1: __mmask16, a: __m256i, b: __m256i) -> __mmask16 { + _mm256_cmpge_epi16_mask(a, b) & k1 +} + +/// Compare packed signed 16-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpge_epi16_mask&expand=839) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmpge_epi16_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_ge(a.as_i16x8(), b.as_i16x8())) +} + +/// Compare packed signed 16-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpge_epi16_mask&expand=840) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmpge_epi16_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpge_epi16_mask(a, b) & k1 +} + +/// Compare packed signed 8-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpge_epi8_mask&expand=861) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmpge_epi8_mask(a: __m512i, b: __m512i) -> __mmask64 { + simd_bitmask::(simd_ge(a.as_i8x64(), b.as_i8x64())) +} + +/// Compare packed signed 8-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpge_epi8_mask&expand=862) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmpge_epi8_mask(k1: __mmask64, a: __m512i, b: __m512i) -> __mmask64 { + _mm512_cmpge_epi8_mask(a, b) & k1 +} + +/// Compare packed signed 8-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpge_epi8_mask&expand=859) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmpge_epi8_mask(a: __m256i, b: __m256i) -> __mmask32 { + simd_bitmask::(simd_ge(a.as_i8x32(), b.as_i8x32())) +} + +/// Compare packed signed 8-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpge_epi8_mask&expand=860) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmpge_epi8_mask(k1: __mmask32, a: __m256i, b: __m256i) -> __mmask32 { + _mm256_cmpge_epi8_mask(a, b) & k1 +} + +/// Compare packed signed 8-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpge_epi8_mask&expand=857) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmpge_epi8_mask(a: __m128i, b: __m128i) -> __mmask16 { + simd_bitmask::(simd_ge(a.as_i8x16(), b.as_i8x16())) +} + +/// Compare packed signed 8-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpge_epi8_mask&expand=858) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmpge_epi8_mask(k1: __mmask16, a: __m128i, b: __m128i) -> __mmask16 { + _mm_cmpge_epi8_mask(a, b) & k1 +} + +/// Compare packed unsigned 16-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpeq_epu16_mask&expand=801) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmpeq_epu16_mask(a: __m512i, b: __m512i) -> __mmask32 { + simd_bitmask::(simd_eq(a.as_u16x32(), b.as_u16x32())) +} + +/// Compare packed unsigned 16-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpeq_epu16_mask&expand=802) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmpeq_epu16_mask(k1: __mmask32, a: __m512i, b: __m512i) -> __mmask32 { + _mm512_cmpeq_epu16_mask(a, b) & k1 +} + +/// Compare packed unsigned 16-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpeq_epu16_mask&expand=799) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmpeq_epu16_mask(a: __m256i, b: __m256i) -> __mmask16 { + simd_bitmask::(simd_eq(a.as_u16x16(), b.as_u16x16())) +} + +/// Compare packed unsigned 16-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpeq_epu16_mask&expand=800) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmpeq_epu16_mask(k1: __mmask16, a: __m256i, b: __m256i) -> __mmask16 { + _mm256_cmpeq_epu16_mask(a, b) & k1 +} + +/// Compare packed unsigned 16-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpeq_epu16_mask&expand=797) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmpeq_epu16_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_eq(a.as_u16x8(), b.as_u16x8())) +} + +/// Compare packed unsigned 16-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpeq_epu16_mask&expand=798) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmpeq_epu16_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpeq_epu16_mask(a, b) & k1 +} + +/// Compare packed unsigned 8-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpeq_epu8_mask&expand=819) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmpeq_epu8_mask(a: __m512i, b: __m512i) -> __mmask64 { + simd_bitmask::(simd_eq(a.as_u8x64(), b.as_u8x64())) +} + +/// Compare packed unsigned 8-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpeq_epu8_mask&expand=820) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmpeq_epu8_mask(k1: __mmask64, a: __m512i, b: __m512i) -> __mmask64 { + _mm512_cmpeq_epu8_mask(a, b) & k1 +} + +/// Compare packed unsigned 8-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpeq_epu8_mask&expand=817) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmpeq_epu8_mask(a: __m256i, b: __m256i) -> __mmask32 { + simd_bitmask::(simd_eq(a.as_u8x32(), b.as_u8x32())) +} + +/// Compare packed unsigned 8-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpeq_epu8_mask&expand=818) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmpeq_epu8_mask(k1: __mmask32, a: __m256i, b: __m256i) -> __mmask32 { + _mm256_cmpeq_epu8_mask(a, b) & k1 +} + +/// Compare packed unsigned 8-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpeq_epu8_mask&expand=815) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmpeq_epu8_mask(a: __m128i, b: __m128i) -> __mmask16 { + simd_bitmask::(simd_eq(a.as_u8x16(), b.as_u8x16())) +} + +/// Compare packed unsigned 8-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpeq_epu8_mask&expand=816) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmpeq_epu8_mask(k1: __mmask16, a: __m128i, b: __m128i) -> __mmask16 { + _mm_cmpeq_epu8_mask(a, b) & k1 +} + +/// Compare packed signed 16-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpeq_epi16_mask&expand=771) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmpeq_epi16_mask(a: __m512i, b: __m512i) -> __mmask32 { + simd_bitmask::(simd_eq(a.as_i16x32(), b.as_i16x32())) +} + +/// Compare packed signed 16-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpeq_epi16_mask&expand=772) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmpeq_epi16_mask(k1: __mmask32, a: __m512i, b: __m512i) -> __mmask32 { + _mm512_cmpeq_epi16_mask(a, b) & k1 +} + +/// Compare packed signed 16-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpeq_epi16_mask&expand=769) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmpeq_epi16_mask(a: __m256i, b: __m256i) -> __mmask16 { + simd_bitmask::(simd_eq(a.as_i16x16(), b.as_i16x16())) +} + +/// Compare packed signed 16-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpeq_epi16_mask&expand=770) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmpeq_epi16_mask(k1: __mmask16, a: __m256i, b: __m256i) -> __mmask16 { + _mm256_cmpeq_epi16_mask(a, b) & k1 +} + +/// Compare packed signed 16-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpeq_epi16_mask&expand=767) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmpeq_epi16_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_eq(a.as_i16x8(), b.as_i16x8())) +} + +/// Compare packed signed 16-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpeq_epi16_mask&expand=768) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmpeq_epi16_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpeq_epi16_mask(a, b) & k1 +} + +/// Compare packed signed 8-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpeq_epi8_mask&expand=795) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmpeq_epi8_mask(a: __m512i, b: __m512i) -> __mmask64 { + simd_bitmask::(simd_eq(a.as_i8x64(), b.as_i8x64())) +} + +/// Compare packed signed 8-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpeq_epi8_mask&expand=796) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmpeq_epi8_mask(k1: __mmask64, a: __m512i, b: __m512i) -> __mmask64 { + _mm512_cmpeq_epi8_mask(a, b) & k1 +} + +/// Compare packed signed 8-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpeq_epi8_mask&expand=793) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmpeq_epi8_mask(a: __m256i, b: __m256i) -> __mmask32 { + simd_bitmask::(simd_eq(a.as_i8x32(), b.as_i8x32())) +} + +/// Compare packed signed 8-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpeq_epi8_mask&expand=794) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmpeq_epi8_mask(k1: __mmask32, a: __m256i, b: __m256i) -> __mmask32 { + _mm256_cmpeq_epi8_mask(a, b) & k1 +} + +/// Compare packed signed 8-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpeq_epi8_mask&expand=791) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmpeq_epi8_mask(a: __m128i, b: __m128i) -> __mmask16 { + simd_bitmask::(simd_eq(a.as_i8x16(), b.as_i8x16())) +} + +/// Compare packed signed 8-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpeq_epi8_mask&expand=792) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmpeq_epi8_mask(k1: __mmask16, a: __m128i, b: __m128i) -> __mmask16 { + _mm_cmpeq_epi8_mask(a, b) & k1 +} + +/// Compare packed unsigned 16-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpneq_epu16_mask&expand=1106) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmpneq_epu16_mask(a: __m512i, b: __m512i) -> __mmask32 { + simd_bitmask::(simd_ne(a.as_u16x32(), b.as_u16x32())) +} + +/// Compare packed unsigned 16-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpneq_epu16_mask&expand=1107) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmpneq_epu16_mask(k1: __mmask32, a: __m512i, b: __m512i) -> __mmask32 { + _mm512_cmpneq_epu16_mask(a, b) & k1 +} + +/// Compare packed unsigned 16-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpneq_epu16_mask&expand=1104) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmpneq_epu16_mask(a: __m256i, b: __m256i) -> __mmask16 { + simd_bitmask::(simd_ne(a.as_u16x16(), b.as_u16x16())) +} + +/// Compare packed unsigned 16-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpneq_epu16_mask&expand=1105) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmpneq_epu16_mask(k1: __mmask16, a: __m256i, b: __m256i) -> __mmask16 { + _mm256_cmpneq_epu16_mask(a, b) & k1 +} + +/// Compare packed unsigned 16-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpneq_epu16_mask&expand=1102) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmpneq_epu16_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_ne(a.as_u16x8(), b.as_u16x8())) +} + +/// Compare packed unsigned 16-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpneq_epu16_mask&expand=1103) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmpneq_epu16_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpneq_epu16_mask(a, b) & k1 +} + +/// Compare packed unsigned 8-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpneq_epu8_mask&expand=1124) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmpneq_epu8_mask(a: __m512i, b: __m512i) -> __mmask64 { + simd_bitmask::(simd_ne(a.as_u8x64(), b.as_u8x64())) +} + +/// Compare packed unsigned 8-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpneq_epu8_mask&expand=1125) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmpneq_epu8_mask(k1: __mmask64, a: __m512i, b: __m512i) -> __mmask64 { + _mm512_cmpneq_epu8_mask(a, b) & k1 +} + +/// Compare packed unsigned 8-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpneq_epu8_mask&expand=1122) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmpneq_epu8_mask(a: __m256i, b: __m256i) -> __mmask32 { + simd_bitmask::(simd_ne(a.as_u8x32(), b.as_u8x32())) +} + +/// Compare packed unsigned 8-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpneq_epu8_mask&expand=1123) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmpneq_epu8_mask(k1: __mmask32, a: __m256i, b: __m256i) -> __mmask32 { + _mm256_cmpneq_epu8_mask(a, b) & k1 +} + +/// Compare packed unsigned 8-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpneq_epu8_mask&expand=1120) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmpneq_epu8_mask(a: __m128i, b: __m128i) -> __mmask16 { + simd_bitmask::(simd_ne(a.as_u8x16(), b.as_u8x16())) +} + +/// Compare packed unsigned 8-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpneq_epu8_mask&expand=1121) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmpneq_epu8_mask(k1: __mmask16, a: __m128i, b: __m128i) -> __mmask16 { + _mm_cmpneq_epu8_mask(a, b) & k1 +} + +/// Compare packed signed 16-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpneq_epi16_mask&expand=1082) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmpneq_epi16_mask(a: __m512i, b: __m512i) -> __mmask32 { + simd_bitmask::(simd_ne(a.as_i16x32(), b.as_i16x32())) +} + +/// Compare packed signed 16-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpneq_epi16_mask&expand=1083) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmpneq_epi16_mask(k1: __mmask32, a: __m512i, b: __m512i) -> __mmask32 { + _mm512_cmpneq_epi16_mask(a, b) & k1 +} + +/// Compare packed signed 16-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpneq_epi16_mask&expand=1080) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmpneq_epi16_mask(a: __m256i, b: __m256i) -> __mmask16 { + simd_bitmask::(simd_ne(a.as_i16x16(), b.as_i16x16())) +} + +/// Compare packed signed 16-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpneq_epi16_mask&expand=1081) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmpneq_epi16_mask(k1: __mmask16, a: __m256i, b: __m256i) -> __mmask16 { + _mm256_cmpneq_epi16_mask(a, b) & k1 +} + +/// Compare packed signed 16-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpneq_epi16_mask&expand=1078) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmpneq_epi16_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_ne(a.as_i16x8(), b.as_i16x8())) +} + +/// Compare packed signed 16-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpneq_epi16_mask&expand=1079) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmpneq_epi16_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpneq_epi16_mask(a, b) & k1 +} + +/// Compare packed signed 8-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpneq_epi8_mask&expand=1100) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_cmpneq_epi8_mask(a: __m512i, b: __m512i) -> __mmask64 { + simd_bitmask::(simd_ne(a.as_i8x64(), b.as_i8x64())) +} + +/// Compare packed signed 8-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpneq_epi8_mask&expand=1101) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm512_mask_cmpneq_epi8_mask(k1: __mmask64, a: __m512i, b: __m512i) -> __mmask64 { + _mm512_cmpneq_epi8_mask(a, b) & k1 +} + +/// Compare packed signed 8-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpneq_epi8_mask&expand=1098) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_cmpneq_epi8_mask(a: __m256i, b: __m256i) -> __mmask32 { + simd_bitmask::(simd_ne(a.as_i8x32(), b.as_i8x32())) +} + +/// Compare packed signed 8-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpneq_epi8_mask&expand=1099) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm256_mask_cmpneq_epi8_mask(k1: __mmask32, a: __m256i, b: __m256i) -> __mmask32 { + _mm256_cmpneq_epi8_mask(a, b) & k1 +} + +/// Compare packed signed 8-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpneq_epi8_mask&expand=1096) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_cmpneq_epi8_mask(a: __m128i, b: __m128i) -> __mmask16 { + simd_bitmask::(simd_ne(a.as_i8x16(), b.as_i8x16())) +} + +/// Compare packed signed 8-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpneq_epi8_mask&expand=1097) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] +pub unsafe fn _mm_mask_cmpneq_epi8_mask(k1: __mmask16, a: __m128i, b: __m128i) -> __mmask16 { + _mm_cmpneq_epi8_mask(a, b) & k1 +} + +/// Compare packed unsigned 16-bit integers in a and b based on the comparison operand specified by `IMM8`, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_epu16_mask&expand=715) +#[inline] +#[target_feature(enable = "avx512bw")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm512_cmp_epu16_mask(a: __m512i, b: __m512i) -> __mmask32 { + static_assert_imm3!(IMM8); + let a = a.as_u16x32(); + let b = b.as_u16x32(); + let r = vpcmpuw(a, b, IMM8, 0b11111111_11111111_11111111_11111111); + transmute(r) +} + +/// Compare packed unsigned 16-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_epu16_mask&expand=716) +#[inline] +#[target_feature(enable = "avx512bw")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm512_mask_cmp_epu16_mask( + k1: __mmask32, + a: __m512i, + b: __m512i, +) -> __mmask32 { + static_assert_imm3!(IMM8); + let a = a.as_u16x32(); + let b = b.as_u16x32(); + let r = vpcmpuw(a, b, IMM8, k1); + transmute(r) +} + +/// Compare packed unsigned 16-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_epu16_mask&expand=713) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm256_cmp_epu16_mask(a: __m256i, b: __m256i) -> __mmask16 { + static_assert_imm3!(IMM8); + let a = a.as_u16x16(); + let b = b.as_u16x16(); + let r = vpcmpuw256(a, b, IMM8, 0b11111111_11111111); + transmute(r) +} + +/// Compare packed unsigned 16-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmp_epu16_mask&expand=714) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm256_mask_cmp_epu16_mask( + k1: __mmask16, + a: __m256i, + b: __m256i, +) -> __mmask16 { + static_assert_imm3!(IMM8); + let a = a.as_u16x16(); + let b = b.as_u16x16(); + let r = vpcmpuw256(a, b, IMM8, k1); + transmute(r) +} + +/// Compare packed unsigned 16-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_epu16_mask&expand=711) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm_cmp_epu16_mask(a: __m128i, b: __m128i) -> __mmask8 { + static_assert_imm3!(IMM8); + let a = a.as_u16x8(); + let b = b.as_u16x8(); + let r = vpcmpuw128(a, b, IMM8, 0b11111111); + transmute(r) +} + +/// Compare packed unsigned 16-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_epu16_mask&expand=712) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm_mask_cmp_epu16_mask( + k1: __mmask8, + a: __m128i, + b: __m128i, +) -> __mmask8 { + static_assert_imm3!(IMM8); + let a = a.as_u16x8(); + let b = b.as_u16x8(); + let r = vpcmpuw128(a, b, IMM8, k1); + transmute(r) +} + +/// Compare packed unsigned 8-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_epu8_mask&expand=733) +#[inline] +#[target_feature(enable = "avx512bw")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm512_cmp_epu8_mask(a: __m512i, b: __m512i) -> __mmask64 { + static_assert_imm3!(IMM8); + let a = a.as_u8x64(); + let b = b.as_u8x64(); + let r = vpcmpub( + a, + b, + IMM8, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + ); + transmute(r) +} + +/// Compare packed unsigned 8-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_epu8_mask&expand=734) +#[inline] +#[target_feature(enable = "avx512bw")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm512_mask_cmp_epu8_mask( + k1: __mmask64, + a: __m512i, + b: __m512i, +) -> __mmask64 { + static_assert_imm3!(IMM8); + let a = a.as_u8x64(); + let b = b.as_u8x64(); + let r = vpcmpub(a, b, IMM8, k1); + transmute(r) +} + +/// Compare packed unsigned 8-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_epu8_mask&expand=731) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm256_cmp_epu8_mask(a: __m256i, b: __m256i) -> __mmask32 { + static_assert_imm3!(IMM8); + let a = a.as_u8x32(); + let b = b.as_u8x32(); + let r = vpcmpub256(a, b, IMM8, 0b11111111_11111111_11111111_11111111); + transmute(r) +} + +/// Compare packed unsigned 8-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmp_epu8_mask&expand=732) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm256_mask_cmp_epu8_mask( + k1: __mmask32, + a: __m256i, + b: __m256i, +) -> __mmask32 { + static_assert_imm3!(IMM8); + let a = a.as_u8x32(); + let b = b.as_u8x32(); + let r = vpcmpub256(a, b, IMM8, k1); + transmute(r) +} + +/// Compare packed unsigned 8-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_epu8_mask&expand=729) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm_cmp_epu8_mask(a: __m128i, b: __m128i) -> __mmask16 { + static_assert_imm3!(IMM8); + let a = a.as_u8x16(); + let b = b.as_u8x16(); + let r = vpcmpub128(a, b, IMM8, 0b11111111_11111111); + transmute(r) +} + +/// Compare packed unsigned 8-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_epu8_mask&expand=730) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm_mask_cmp_epu8_mask( + k1: __mmask16, + a: __m128i, + b: __m128i, +) -> __mmask16 { + static_assert_imm3!(IMM8); + let a = a.as_u8x16(); + let b = b.as_u8x16(); + let r = vpcmpub128(a, b, IMM8, k1); + transmute(r) +} + +/// Compare packed signed 16-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_epi16_mask&expand=691) +#[inline] +#[target_feature(enable = "avx512bw")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm512_cmp_epi16_mask(a: __m512i, b: __m512i) -> __mmask32 { + static_assert_imm3!(IMM8); + let a = a.as_i16x32(); + let b = b.as_i16x32(); + let r = vpcmpw(a, b, IMM8, 0b11111111_11111111_11111111_11111111); + transmute(r) +} + +/// Compare packed signed 16-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_epi16_mask&expand=692) +#[inline] +#[target_feature(enable = "avx512bw")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm512_mask_cmp_epi16_mask( + k1: __mmask32, + a: __m512i, + b: __m512i, +) -> __mmask32 { + static_assert_imm3!(IMM8); + let a = a.as_i16x32(); + let b = b.as_i16x32(); + let r = vpcmpw(a, b, IMM8, k1); + transmute(r) +} + +/// Compare packed signed 16-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_epi16_mask&expand=689) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm256_cmp_epi16_mask(a: __m256i, b: __m256i) -> __mmask16 { + static_assert_imm3!(IMM8); + let a = a.as_i16x16(); + let b = b.as_i16x16(); + let r = vpcmpw256(a, b, IMM8, 0b11111111_11111111); + transmute(r) +} + +/// Compare packed signed 16-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmp_epi16_mask&expand=690) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm256_mask_cmp_epi16_mask( + k1: __mmask16, + a: __m256i, + b: __m256i, +) -> __mmask16 { + static_assert_imm3!(IMM8); + let a = a.as_i16x16(); + let b = b.as_i16x16(); + let r = vpcmpw256(a, b, IMM8, k1); + transmute(r) +} + +/// Compare packed signed 16-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_epi16_mask&expand=687) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm_cmp_epi16_mask(a: __m128i, b: __m128i) -> __mmask8 { + static_assert_imm3!(IMM8); + let a = a.as_i16x8(); + let b = b.as_i16x8(); + let r = vpcmpw128(a, b, IMM8, 0b11111111); + transmute(r) +} + +/// Compare packed signed 16-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_epi16_mask&expand=688) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm_mask_cmp_epi16_mask( + k1: __mmask8, + a: __m128i, + b: __m128i, +) -> __mmask8 { + static_assert_imm3!(IMM8); + let a = a.as_i16x8(); + let b = b.as_i16x8(); + let r = vpcmpw128(a, b, IMM8, k1); + transmute(r) +} + +/// Compare packed signed 8-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_epi8_mask&expand=709) +#[inline] +#[target_feature(enable = "avx512bw")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm512_cmp_epi8_mask(a: __m512i, b: __m512i) -> __mmask64 { + static_assert_imm3!(IMM8); + let a = a.as_i8x64(); + let b = b.as_i8x64(); + let r = vpcmpb( + a, + b, + IMM8, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + ); + transmute(r) +} + +/// Compare packed signed 8-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_epi8_mask&expand=710) +#[inline] +#[target_feature(enable = "avx512bw")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm512_mask_cmp_epi8_mask( + k1: __mmask64, + a: __m512i, + b: __m512i, +) -> __mmask64 { + static_assert_imm3!(IMM8); + let a = a.as_i8x64(); + let b = b.as_i8x64(); + let r = vpcmpb(a, b, IMM8, k1); + transmute(r) +} + +/// Compare packed signed 8-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_epi8_mask&expand=707) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm256_cmp_epi8_mask(a: __m256i, b: __m256i) -> __mmask32 { + static_assert_imm3!(IMM8); + let a = a.as_i8x32(); + let b = b.as_i8x32(); + let r = vpcmpb256(a, b, IMM8, 0b11111111_11111111_11111111_11111111); + transmute(r) +} + +/// Compare packed signed 8-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmp_epi8_mask&expand=708) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm256_mask_cmp_epi8_mask( + k1: __mmask32, + a: __m256i, + b: __m256i, +) -> __mmask32 { + static_assert_imm3!(IMM8); + let a = a.as_i8x32(); + let b = b.as_i8x32(); + let r = vpcmpb256(a, b, IMM8, k1); + transmute(r) +} + +/// Compare packed signed 8-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_epi8_mask&expand=705) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm_cmp_epi8_mask(a: __m128i, b: __m128i) -> __mmask16 { + static_assert_imm3!(IMM8); + let a = a.as_i8x16(); + let b = b.as_i8x16(); + let r = vpcmpb128(a, b, IMM8, 0b11111111_11111111); + transmute(r) +} + +/// Compare packed signed 8-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_epi8_mask&expand=706) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm_mask_cmp_epi8_mask( + k1: __mmask16, + a: __m128i, + b: __m128i, +) -> __mmask16 { + static_assert_imm3!(IMM8); + let a = a.as_i8x16(); + let b = b.as_i8x16(); + let r = vpcmpb128(a, b, IMM8, k1); + transmute(r) +} + +/// Load 512-bits (composed of 32 packed 16-bit integers) from memory into dst. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_loadu_epi16&expand=3368) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu16 +pub unsafe fn _mm512_loadu_epi16(mem_addr: *const i16) -> __m512i { + ptr::read_unaligned(mem_addr as *const __m512i) +} + +/// Load 256-bits (composed of 16 packed 16-bit integers) from memory into dst. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_loadu_epi16&expand=3365) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu16 +pub unsafe fn _mm256_loadu_epi16(mem_addr: *const i16) -> __m256i { + ptr::read_unaligned(mem_addr as *const __m256i) +} + +/// Load 128-bits (composed of 8 packed 16-bit integers) from memory into dst. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadu_epi16&expand=3362) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu16 +pub unsafe fn _mm_loadu_epi16(mem_addr: *const i16) -> __m128i { + ptr::read_unaligned(mem_addr as *const __m128i) +} + +/// Load 512-bits (composed of 64 packed 8-bit integers) from memory into dst. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_loadu_epi8&expand=3395) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu8 +pub unsafe fn _mm512_loadu_epi8(mem_addr: *const i8) -> __m512i { + ptr::read_unaligned(mem_addr as *const __m512i) +} + +/// Load 256-bits (composed of 32 packed 8-bit integers) from memory into dst. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_loadu_epi8&expand=3392) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu8 +pub unsafe fn _mm256_loadu_epi8(mem_addr: *const i8) -> __m256i { + ptr::read_unaligned(mem_addr as *const __m256i) +} + +/// Load 128-bits (composed of 16 packed 8-bit integers) from memory into dst. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadu_epi8&expand=3389) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu8 +pub unsafe fn _mm_loadu_epi8(mem_addr: *const i8) -> __m128i { + ptr::read_unaligned(mem_addr as *const __m128i) +} + +/// Store 512-bits (composed of 32 packed 16-bit integers) from a into memory. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_storeu_epi16&expand=5622) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu16 +pub unsafe fn _mm512_storeu_epi16(mem_addr: *mut i16, a: __m512i) { + ptr::write_unaligned(mem_addr as *mut __m512i, a); +} + +/// Store 256-bits (composed of 16 packed 16-bit integers) from a into memory. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_storeu_epi16&expand=5620) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu16 +pub unsafe fn _mm256_storeu_epi16(mem_addr: *mut i16, a: __m256i) { + ptr::write_unaligned(mem_addr as *mut __m256i, a); +} + +/// Store 128-bits (composed of 8 packed 16-bit integers) from a into memory. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_storeu_epi16&expand=5618) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu16 +pub unsafe fn _mm_storeu_epi16(mem_addr: *mut i16, a: __m128i) { + ptr::write_unaligned(mem_addr as *mut __m128i, a); +} + +/// Store 512-bits (composed of 64 packed 8-bit integers) from a into memory. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_storeu_epi8&expand=5640) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu8 +pub unsafe fn _mm512_storeu_epi8(mem_addr: *mut i8, a: __m512i) { + ptr::write_unaligned(mem_addr as *mut __m512i, a); +} + +/// Store 256-bits (composed of 32 packed 8-bit integers) from a into memory. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_storeu_epi8&expand=5638) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu8 +pub unsafe fn _mm256_storeu_epi8(mem_addr: *mut i8, a: __m256i) { + ptr::write_unaligned(mem_addr as *mut __m256i, a); +} + +/// Store 128-bits (composed of 16 packed 8-bit integers) from a into memory. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_storeu_epi8&expand=5636) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu8 +pub unsafe fn _mm_storeu_epi8(mem_addr: *mut i8, a: __m128i) { + ptr::write_unaligned(mem_addr as *mut __m128i, a); +} + +/// Load packed 16-bit integers from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_loadu_epi16) +#[inline] +#[target_feature(enable = "avx512f,avx512bw")] +pub unsafe fn _mm512_mask_loadu_epi16(src: __m512i, k: __mmask32, mem_addr: *const i16) -> __m512i { + let mut dst: __m512i = src; + asm!( + vpl!("vmovdqu16 {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 16-bit integers from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_loadu_epi16) +#[inline] +#[target_feature(enable = "avx512f,avx512bw")] +pub unsafe fn _mm512_maskz_loadu_epi16(k: __mmask32, mem_addr: *const i16) -> __m512i { + let mut dst: __m512i; + asm!( + vpl!("vmovdqu16 {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 8-bit integers from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_loadu_epi8) +#[inline] +#[target_feature(enable = "avx512f,avx512bw")] +pub unsafe fn _mm512_mask_loadu_epi8(src: __m512i, k: __mmask64, mem_addr: *const i8) -> __m512i { + let mut dst: __m512i = src; + asm!( + vpl!("vmovdqu8 {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 8-bit integers from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_loadu_epi8) +#[inline] +#[target_feature(enable = "avx512f,avx512bw")] +pub unsafe fn _mm512_maskz_loadu_epi8(k: __mmask64, mem_addr: *const i8) -> __m512i { + let mut dst: __m512i; + asm!( + vpl!("vmovdqu8 {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 16-bit integers from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_loadu_epi16) +#[inline] +#[target_feature(enable = "avx512f,avx512bw,avx512vl,avx")] +pub unsafe fn _mm256_mask_loadu_epi16(src: __m256i, k: __mmask16, mem_addr: *const i16) -> __m256i { + let mut dst: __m256i = src; + asm!( + vpl!("vmovdqu16 {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 16-bit integers from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_loadu_epi16) +#[inline] +#[target_feature(enable = "avx512f,avx512bw,avx512vl,avx")] +pub unsafe fn _mm256_maskz_loadu_epi16(k: __mmask16, mem_addr: *const i16) -> __m256i { + let mut dst: __m256i; + asm!( + vpl!("vmovdqu16 {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 8-bit integers from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_loadu_epi8) +#[inline] +#[target_feature(enable = "avx512f,avx512bw,avx512vl,avx")] +pub unsafe fn _mm256_mask_loadu_epi8(src: __m256i, k: __mmask32, mem_addr: *const i8) -> __m256i { + let mut dst: __m256i = src; + asm!( + vpl!("vmovdqu8 {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 8-bit integers from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_loadu_epi8) +#[inline] +#[target_feature(enable = "avx512f,avx512bw,avx512vl,avx")] +pub unsafe fn _mm256_maskz_loadu_epi8(k: __mmask32, mem_addr: *const i8) -> __m256i { + let mut dst: __m256i; + asm!( + vpl!("vmovdqu8 {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 16-bit integers from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_loadu_epi16) +#[inline] +#[target_feature(enable = "avx512f,avx512bw,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_loadu_epi16(src: __m128i, k: __mmask8, mem_addr: *const i16) -> __m128i { + let mut dst: __m128i = src; + asm!( + vpl!("vmovdqu16 {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 16-bit integers from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_loadu_epi16) +#[inline] +#[target_feature(enable = "avx512f,avx512bw,avx512vl,avx,sse")] +pub unsafe fn _mm_maskz_loadu_epi16(k: __mmask8, mem_addr: *const i16) -> __m128i { + let mut dst: __m128i; + asm!( + vpl!("vmovdqu16 {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 8-bit integers from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_loadu_epi8) +#[inline] +#[target_feature(enable = "avx512f,avx512bw,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_loadu_epi8(src: __m128i, k: __mmask16, mem_addr: *const i8) -> __m128i { + let mut dst: __m128i = src; + asm!( + vpl!("vmovdqu8 {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 8-bit integers from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_loadu_epi8) +#[inline] +#[target_feature(enable = "avx512f,avx512bw,avx512vl,avx,sse")] +pub unsafe fn _mm_maskz_loadu_epi8(k: __mmask16, mem_addr: *const i8) -> __m128i { + let mut dst: __m128i; + asm!( + vpl!("vmovdqu8 {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Store packed 16-bit integers from a into memory using writemask k. +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_storeu_epi16) +#[inline] +#[target_feature(enable = "avx512f,avx512bw")] +pub unsafe fn _mm512_mask_storeu_epi16(mem_addr: *mut i16, mask: __mmask32, a: __m512i) { + asm!( + vps!("vmovdqu16", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(zmm_reg) a, + options(nostack) + ); +} + +/// Store packed 8-bit integers from a into memory using writemask k. +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_storeu_epi8) +#[inline] +#[target_feature(enable = "avx512f,avx512bw")] +pub unsafe fn _mm512_mask_storeu_epi8(mem_addr: *mut i8, mask: __mmask64, a: __m512i) { + asm!( + vps!("vmovdqu8", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(zmm_reg) a, + options(nostack) + ); +} + +/// Store packed 16-bit integers from a into memory using writemask k. +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_storeu_epi16) +#[inline] +#[target_feature(enable = "avx512f,avx512bw,avx512vl,avx")] +pub unsafe fn _mm256_mask_storeu_epi16(mem_addr: *mut i16, mask: __mmask16, a: __m256i) { + asm!( + vps!("vmovdqu16", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(ymm_reg) a, + options(nostack) + ); +} + +/// Store packed 8-bit integers from a into memory using writemask k. +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_storeu_epi8) +#[inline] +#[target_feature(enable = "avx512f,avx512bw,avx512vl,avx")] +pub unsafe fn _mm256_mask_storeu_epi8(mem_addr: *mut i8, mask: __mmask32, a: __m256i) { + asm!( + vps!("vmovdqu8", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(ymm_reg) a, + options(nostack) + ); +} + +/// Store packed 16-bit integers from a into memory using writemask k. +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_storeu_epi16) +#[inline] +#[target_feature(enable = "avx512f,avx512bw,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_storeu_epi16(mem_addr: *mut i16, mask: __mmask8, a: __m128i) { + asm!( + vps!("vmovdqu16", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(xmm_reg) a, + options(nostack) + ); +} + +/// Store packed 8-bit integers from a into memory using writemask k. +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_storeu_epi8) +#[inline] +#[target_feature(enable = "avx512f,avx512bw,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_storeu_epi8(mem_addr: *mut i8, mask: __mmask16, a: __m128i) { + asm!( + vps!("vmovdqu8", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(xmm_reg) a, + options(nostack) + ); +} + +/// Multiply packed signed 16-bit integers in a and b, producing intermediate signed 32-bit integers. Horizontally add adjacent pairs of intermediate 32-bit integers, and pack the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_madd_epi16&expand=3511) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmaddwd))] +pub unsafe fn _mm512_madd_epi16(a: __m512i, b: __m512i) -> __m512i { + transmute(vpmaddwd(a.as_i16x32(), b.as_i16x32())) +} + +/// Multiply packed signed 16-bit integers in a and b, producing intermediate signed 32-bit integers. Horizontally add adjacent pairs of intermediate 32-bit integers, and pack the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_madd_epi16&expand=3512) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmaddwd))] +pub unsafe fn _mm512_mask_madd_epi16( + src: __m512i, + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + let madd = _mm512_madd_epi16(a, b).as_i32x16(); + transmute(simd_select_bitmask(k, madd, src.as_i32x16())) +} + +/// Multiply packed signed 16-bit integers in a and b, producing intermediate signed 32-bit integers. Horizontally add adjacent pairs of intermediate 32-bit integers, and pack the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_madd_epi16&expand=3513) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmaddwd))] +pub unsafe fn _mm512_maskz_madd_epi16(k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let madd = _mm512_madd_epi16(a, b).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, madd, zero)) +} + +/// Multiply packed signed 16-bit integers in a and b, producing intermediate signed 32-bit integers. Horizontally add adjacent pairs of intermediate 32-bit integers, and pack the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_madd_epi16&expand=3509) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaddwd))] +pub unsafe fn _mm256_mask_madd_epi16(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let madd = _mm256_madd_epi16(a, b).as_i32x8(); + transmute(simd_select_bitmask(k, madd, src.as_i32x8())) +} + +/// Multiply packed signed 16-bit integers in a and b, producing intermediate signed 32-bit integers. Horizontally add adjacent pairs of intermediate 32-bit integers, and pack the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_madd_epi16&expand=3510) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaddwd))] +pub unsafe fn _mm256_maskz_madd_epi16(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let madd = _mm256_madd_epi16(a, b).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, madd, zero)) +} + +/// Multiply packed signed 16-bit integers in a and b, producing intermediate signed 32-bit integers. Horizontally add adjacent pairs of intermediate 32-bit integers, and pack the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_madd_epi16&expand=3506) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaddwd))] +pub unsafe fn _mm_mask_madd_epi16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let madd = _mm_madd_epi16(a, b).as_i32x4(); + transmute(simd_select_bitmask(k, madd, src.as_i32x4())) +} + +/// Multiply packed signed 16-bit integers in a and b, producing intermediate signed 32-bit integers. Horizontally add adjacent pairs of intermediate 32-bit integers, and pack the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_madd_epi16&expand=3507) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaddwd))] +pub unsafe fn _mm_maskz_madd_epi16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let madd = _mm_madd_epi16(a, b).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, madd, zero)) +} + +/// Vertically multiply each unsigned 8-bit integer from a with the corresponding signed 8-bit integer from b, producing intermediate signed 16-bit integers. Horizontally add adjacent pairs of intermediate signed 16-bit integers, and pack the saturated results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maddubs_epi16&expand=3539) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmaddubsw))] +pub unsafe fn _mm512_maddubs_epi16(a: __m512i, b: __m512i) -> __m512i { + transmute(vpmaddubsw(a.as_i8x64(), b.as_i8x64())) +} + +/// Multiply packed unsigned 8-bit integers in a by packed signed 8-bit integers in b, producing intermediate signed 16-bit integers. Horizontally add adjacent pairs of intermediate signed 16-bit integers, and pack the saturated results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_maddubs_epi16&expand=3540) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmaddubsw))] +pub unsafe fn _mm512_mask_maddubs_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + let madd = _mm512_maddubs_epi16(a, b).as_i16x32(); + transmute(simd_select_bitmask(k, madd, src.as_i16x32())) +} + +/// Multiply packed unsigned 8-bit integers in a by packed signed 8-bit integers in b, producing intermediate signed 16-bit integers. Horizontally add adjacent pairs of intermediate signed 16-bit integers, and pack the saturated results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_maddubs_epi16&expand=3541) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmaddubsw))] +pub unsafe fn _mm512_maskz_maddubs_epi16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let madd = _mm512_maddubs_epi16(a, b).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, madd, zero)) +} + +/// Multiply packed unsigned 8-bit integers in a by packed signed 8-bit integers in b, producing intermediate signed 16-bit integers. Horizontally add adjacent pairs of intermediate signed 16-bit integers, and pack the saturated results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_maddubs_epi16&expand=3537) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaddubsw))] +pub unsafe fn _mm256_mask_maddubs_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + let madd = _mm256_maddubs_epi16(a, b).as_i16x16(); + transmute(simd_select_bitmask(k, madd, src.as_i16x16())) +} + +/// Multiply packed unsigned 8-bit integers in a by packed signed 8-bit integers in b, producing intermediate signed 16-bit integers. Horizontally add adjacent pairs of intermediate signed 16-bit integers, and pack the saturated results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_maddubs_epi16&expand=3538) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaddubsw))] +pub unsafe fn _mm256_maskz_maddubs_epi16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let madd = _mm256_maddubs_epi16(a, b).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, madd, zero)) +} + +/// Multiply packed unsigned 8-bit integers in a by packed signed 8-bit integers in b, producing intermediate signed 16-bit integers. Horizontally add adjacent pairs of intermediate signed 16-bit integers, and pack the saturated results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_maddubs_epi16&expand=3534) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaddubsw))] +pub unsafe fn _mm_mask_maddubs_epi16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let madd = _mm_maddubs_epi16(a, b).as_i16x8(); + transmute(simd_select_bitmask(k, madd, src.as_i16x8())) +} + +/// Multiply packed unsigned 8-bit integers in a by packed signed 8-bit integers in b, producing intermediate signed 16-bit integers. Horizontally add adjacent pairs of intermediate signed 16-bit integers, and pack the saturated results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_maddubs_epi16&expand=3535) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaddubsw))] +pub unsafe fn _mm_maskz_maddubs_epi16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let madd = _mm_maddubs_epi16(a, b).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, madd, zero)) +} + +/// Convert packed signed 32-bit integers from a and b to packed 16-bit integers using signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_packs_epi32&expand=4091) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpackssdw))] +pub unsafe fn _mm512_packs_epi32(a: __m512i, b: __m512i) -> __m512i { + transmute(vpackssdw(a.as_i32x16(), b.as_i32x16())) +} + +/// Convert packed signed 32-bit integers from a and b to packed 16-bit integers using signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_packs_epi32&expand=4089) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpackssdw))] +pub unsafe fn _mm512_mask_packs_epi32( + src: __m512i, + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + let pack = _mm512_packs_epi32(a, b).as_i16x32(); + transmute(simd_select_bitmask(k, pack, src.as_i16x32())) +} + +/// Convert packed signed 32-bit integers from a and b to packed 16-bit integers using signed saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_packs_epi32&expand=4090) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpackssdw))] +pub unsafe fn _mm512_maskz_packs_epi32(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let pack = _mm512_packs_epi32(a, b).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, pack, zero)) +} + +/// Convert packed signed 32-bit integers from a and b to packed 16-bit integers using signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_packs_epi32&expand=4086) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpackssdw))] +pub unsafe fn _mm256_mask_packs_epi32( + src: __m256i, + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + let pack = _mm256_packs_epi32(a, b).as_i16x16(); + transmute(simd_select_bitmask(k, pack, src.as_i16x16())) +} + +/// Convert packed signed 32-bit integers from a and b to packed 16-bit integers using signed saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_packs_epi32&expand=4087) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpackssdw))] +pub unsafe fn _mm256_maskz_packs_epi32(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let pack = _mm256_packs_epi32(a, b).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, pack, zero)) +} + +/// Convert packed signed 32-bit integers from a and b to packed 16-bit integers using signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_packs_epi32&expand=4083) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpackssdw))] +pub unsafe fn _mm_mask_packs_epi32(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let pack = _mm_packs_epi32(a, b).as_i16x8(); + transmute(simd_select_bitmask(k, pack, src.as_i16x8())) +} + +/// Convert packed signed 32-bit integers from a and b to packed 16-bit integers using signed saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_packs_epi32&expand=4084) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpackssdw))] +pub unsafe fn _mm_maskz_packs_epi32(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let pack = _mm_packs_epi32(a, b).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, pack, zero)) +} + +/// Convert packed signed 16-bit integers from a and b to packed 8-bit integers using signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_packs_epi16&expand=4082) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpacksswb))] +pub unsafe fn _mm512_packs_epi16(a: __m512i, b: __m512i) -> __m512i { + transmute(vpacksswb(a.as_i16x32(), b.as_i16x32())) +} + +/// Convert packed signed 16-bit integers from a and b to packed 8-bit integers using signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_packs_epi16&expand=4080) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpacksswb))] +pub unsafe fn _mm512_mask_packs_epi16( + src: __m512i, + k: __mmask64, + a: __m512i, + b: __m512i, +) -> __m512i { + let pack = _mm512_packs_epi16(a, b).as_i8x64(); + transmute(simd_select_bitmask(k, pack, src.as_i8x64())) +} + +/// Convert packed signed 16-bit integers from a and b to packed 8-bit integers using signed saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_packs_epi16&expand=4081) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpacksswb))] +pub unsafe fn _mm512_maskz_packs_epi16(k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + let pack = _mm512_packs_epi16(a, b).as_i8x64(); + let zero = _mm512_setzero_si512().as_i8x64(); + transmute(simd_select_bitmask(k, pack, zero)) +} + +/// Convert packed signed 16-bit integers from a and b to packed 8-bit integers using signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_packs_epi16&expand=4077) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpacksswb))] +pub unsafe fn _mm256_mask_packs_epi16( + src: __m256i, + k: __mmask32, + a: __m256i, + b: __m256i, +) -> __m256i { + let pack = _mm256_packs_epi16(a, b).as_i8x32(); + transmute(simd_select_bitmask(k, pack, src.as_i8x32())) +} + +/// Convert packed signed 16-bit integers from a and b to packed 8-bit integers using signed saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=#text=_mm256_maskz_packs_epi16&expand=4078) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpacksswb))] +pub unsafe fn _mm256_maskz_packs_epi16(k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + let pack = _mm256_packs_epi16(a, b).as_i8x32(); + let zero = _mm256_setzero_si256().as_i8x32(); + transmute(simd_select_bitmask(k, pack, zero)) +} + +/// Convert packed signed 16-bit integers from a and b to packed 8-bit integers using signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_packs_epi16&expand=4074) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpacksswb))] +pub unsafe fn _mm_mask_packs_epi16(src: __m128i, k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let pack = _mm_packs_epi16(a, b).as_i8x16(); + transmute(simd_select_bitmask(k, pack, src.as_i8x16())) +} + +/// Convert packed signed 16-bit integers from a and b to packed 8-bit integers using signed saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_packs_epi16&expand=4075) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpacksswb))] +pub unsafe fn _mm_maskz_packs_epi16(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let pack = _mm_packs_epi16(a, b).as_i8x16(); + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask(k, pack, zero)) +} + +/// Convert packed signed 32-bit integers from a and b to packed 16-bit integers using unsigned saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_packus_epi32&expand=4130) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpackusdw))] +pub unsafe fn _mm512_packus_epi32(a: __m512i, b: __m512i) -> __m512i { + transmute(vpackusdw(a.as_i32x16(), b.as_i32x16())) +} + +/// Convert packed signed 32-bit integers from a and b to packed 16-bit integers using unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_packus_epi32&expand=4128) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpackusdw))] +pub unsafe fn _mm512_mask_packus_epi32( + src: __m512i, + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + let pack = _mm512_packus_epi32(a, b).as_i16x32(); + transmute(simd_select_bitmask(k, pack, src.as_i16x32())) +} + +/// Convert packed signed 32-bit integers from a and b to packed 16-bit integers using unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_packus_epi32&expand=4129) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpackusdw))] +pub unsafe fn _mm512_maskz_packus_epi32(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let pack = _mm512_packus_epi32(a, b).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, pack, zero)) +} + +/// Convert packed signed 32-bit integers from a and b to packed 16-bit integers using unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_packus_epi32&expand=4125) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpackusdw))] +pub unsafe fn _mm256_mask_packus_epi32( + src: __m256i, + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + let pack = _mm256_packus_epi32(a, b).as_i16x16(); + transmute(simd_select_bitmask(k, pack, src.as_i16x16())) +} + +/// Convert packed signed 32-bit integers from a and b to packed 16-bit integers using unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_packus_epi32&expand=4126) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpackusdw))] +pub unsafe fn _mm256_maskz_packus_epi32(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let pack = _mm256_packus_epi32(a, b).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, pack, zero)) +} + +/// Convert packed signed 32-bit integers from a and b to packed 16-bit integers using unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_packus_epi32&expand=4122) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpackusdw))] +pub unsafe fn _mm_mask_packus_epi32(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let pack = _mm_packus_epi32(a, b).as_i16x8(); + transmute(simd_select_bitmask(k, pack, src.as_i16x8())) +} + +/// Convert packed signed 32-bit integers from a and b to packed 16-bit integers using unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_packus_epi32&expand=4123) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpackusdw))] +pub unsafe fn _mm_maskz_packus_epi32(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let pack = _mm_packus_epi32(a, b).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, pack, zero)) +} + +/// Convert packed signed 16-bit integers from a and b to packed 8-bit integers using unsigned saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_packus_epi16&expand=4121) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpackuswb))] +pub unsafe fn _mm512_packus_epi16(a: __m512i, b: __m512i) -> __m512i { + transmute(vpackuswb(a.as_i16x32(), b.as_i16x32())) +} + +/// Convert packed signed 16-bit integers from a and b to packed 8-bit integers using unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_packus_epi16&expand=4119) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpackuswb))] +pub unsafe fn _mm512_mask_packus_epi16( + src: __m512i, + k: __mmask64, + a: __m512i, + b: __m512i, +) -> __m512i { + let pack = _mm512_packus_epi16(a, b).as_i8x64(); + transmute(simd_select_bitmask(k, pack, src.as_i8x64())) +} + +/// Convert packed signed 16-bit integers from a and b to packed 8-bit integers using unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_packus_epi16&expand=4120) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpackuswb))] +pub unsafe fn _mm512_maskz_packus_epi16(k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + let pack = _mm512_packus_epi16(a, b).as_i8x64(); + let zero = _mm512_setzero_si512().as_i8x64(); + transmute(simd_select_bitmask(k, pack, zero)) +} + +/// Convert packed signed 16-bit integers from a and b to packed 8-bit integers using unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_packus_epi16&expand=4116) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpackuswb))] +pub unsafe fn _mm256_mask_packus_epi16( + src: __m256i, + k: __mmask32, + a: __m256i, + b: __m256i, +) -> __m256i { + let pack = _mm256_packus_epi16(a, b).as_i8x32(); + transmute(simd_select_bitmask(k, pack, src.as_i8x32())) +} + +/// Convert packed signed 16-bit integers from a and b to packed 8-bit integers using unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_packus_epi16&expand=4117) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpackuswb))] +pub unsafe fn _mm256_maskz_packus_epi16(k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + let pack = _mm256_packus_epi16(a, b).as_i8x32(); + let zero = _mm256_setzero_si256().as_i8x32(); + transmute(simd_select_bitmask(k, pack, zero)) +} + +/// Convert packed signed 16-bit integers from a and b to packed 8-bit integers using unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_packus_epi16&expand=4113) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpackuswb))] +pub unsafe fn _mm_mask_packus_epi16(src: __m128i, k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let pack = _mm_packus_epi16(a, b).as_i8x16(); + transmute(simd_select_bitmask(k, pack, src.as_i8x16())) +} + +/// Convert packed signed 16-bit integers from a and b to packed 8-bit integers using unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_packus_epi16&expand=4114) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpackuswb))] +pub unsafe fn _mm_maskz_packus_epi16(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let pack = _mm_packus_epi16(a, b).as_i8x16(); + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask(k, pack, zero)) +} + +/// Average packed unsigned 16-bit integers in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_avg_epu16&expand=388) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpavgw))] +pub unsafe fn _mm512_avg_epu16(a: __m512i, b: __m512i) -> __m512i { + transmute(vpavgw(a.as_u16x32(), b.as_u16x32())) +} + +/// Average packed unsigned 16-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_avg_epu16&expand=389) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpavgw))] +pub unsafe fn _mm512_mask_avg_epu16(src: __m512i, k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let avg = _mm512_avg_epu16(a, b).as_u16x32(); + transmute(simd_select_bitmask(k, avg, src.as_u16x32())) +} + +/// Average packed unsigned 16-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_avg_epu16&expand=390) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpavgw))] +pub unsafe fn _mm512_maskz_avg_epu16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let avg = _mm512_avg_epu16(a, b).as_u16x32(); + let zero = _mm512_setzero_si512().as_u16x32(); + transmute(simd_select_bitmask(k, avg, zero)) +} + +/// Average packed unsigned 16-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_avg_epu16&expand=386) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpavgw))] +pub unsafe fn _mm256_mask_avg_epu16(src: __m256i, k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let avg = _mm256_avg_epu16(a, b).as_u16x16(); + transmute(simd_select_bitmask(k, avg, src.as_u16x16())) +} + +/// Average packed unsigned 16-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_avg_epu16&expand=387) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpavgw))] +pub unsafe fn _mm256_maskz_avg_epu16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let avg = _mm256_avg_epu16(a, b).as_u16x16(); + let zero = _mm256_setzero_si256().as_u16x16(); + transmute(simd_select_bitmask(k, avg, zero)) +} + +/// Average packed unsigned 16-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_avg_epu16&expand=383) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpavgw))] +pub unsafe fn _mm_mask_avg_epu16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let avg = _mm_avg_epu16(a, b).as_u16x8(); + transmute(simd_select_bitmask(k, avg, src.as_u16x8())) +} + +/// Average packed unsigned 16-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_avg_epu16&expand=384) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpavgw))] +pub unsafe fn _mm_maskz_avg_epu16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let avg = _mm_avg_epu16(a, b).as_u16x8(); + let zero = _mm_setzero_si128().as_u16x8(); + transmute(simd_select_bitmask(k, avg, zero)) +} + +/// Average packed unsigned 8-bit integers in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_avg_epu8&expand=397) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpavgb))] +pub unsafe fn _mm512_avg_epu8(a: __m512i, b: __m512i) -> __m512i { + transmute(vpavgb(a.as_u8x64(), b.as_u8x64())) +} + +/// Average packed unsigned 8-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_avg_epu8&expand=398) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpavgb))] +pub unsafe fn _mm512_mask_avg_epu8(src: __m512i, k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + let avg = _mm512_avg_epu8(a, b).as_u8x64(); + transmute(simd_select_bitmask(k, avg, src.as_u8x64())) +} + +/// Average packed unsigned 8-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_avg_epu8&expand=399) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpavgb))] +pub unsafe fn _mm512_maskz_avg_epu8(k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + let avg = _mm512_avg_epu8(a, b).as_u8x64(); + let zero = _mm512_setzero_si512().as_u8x64(); + transmute(simd_select_bitmask(k, avg, zero)) +} + +/// Average packed unsigned 8-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_avg_epu8&expand=395) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpavgb))] +pub unsafe fn _mm256_mask_avg_epu8(src: __m256i, k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + let avg = _mm256_avg_epu8(a, b).as_u8x32(); + transmute(simd_select_bitmask(k, avg, src.as_u8x32())) +} + +/// Average packed unsigned 8-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_avg_epu8&expand=396) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpavgb))] +pub unsafe fn _mm256_maskz_avg_epu8(k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + let avg = _mm256_avg_epu8(a, b).as_u8x32(); + let zero = _mm256_setzero_si256().as_u8x32(); + transmute(simd_select_bitmask(k, avg, zero)) +} + +/// Average packed unsigned 8-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_avg_epu8&expand=392) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpavgb))] +pub unsafe fn _mm_mask_avg_epu8(src: __m128i, k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let avg = _mm_avg_epu8(a, b).as_u8x16(); + transmute(simd_select_bitmask(k, avg, src.as_u8x16())) +} + +/// Average packed unsigned 8-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_avg_epu8&expand=393) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpavgb))] +pub unsafe fn _mm_maskz_avg_epu8(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let avg = _mm_avg_epu8(a, b).as_u8x16(); + let zero = _mm_setzero_si128().as_u8x16(); + transmute(simd_select_bitmask(k, avg, zero)) +} + +/// Shift packed 16-bit integers in a left by count while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sll_epi16&expand=5271) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsllw))] +pub unsafe fn _mm512_sll_epi16(a: __m512i, count: __m128i) -> __m512i { + transmute(vpsllw(a.as_i16x32(), count.as_i16x8())) +} + +/// Shift packed 16-bit integers in a left by count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sll_epi16&expand=5269) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsllw))] +pub unsafe fn _mm512_mask_sll_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, + count: __m128i, +) -> __m512i { + let shf = _mm512_sll_epi16(a, count).as_i16x32(); + transmute(simd_select_bitmask(k, shf, src.as_i16x32())) +} + +/// Shift packed 16-bit integers in a left by count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sll_epi16&expand=5270) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsllw))] +pub unsafe fn _mm512_maskz_sll_epi16(k: __mmask32, a: __m512i, count: __m128i) -> __m512i { + let shf = _mm512_sll_epi16(a, count).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 16-bit integers in a left by count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_sll_epi16&expand=5266) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllw))] +pub unsafe fn _mm256_mask_sll_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, + count: __m128i, +) -> __m256i { + let shf = _mm256_sll_epi16(a, count).as_i16x16(); + transmute(simd_select_bitmask(k, shf, src.as_i16x16())) +} + +/// Shift packed 16-bit integers in a left by count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_sll_epi16&expand=5267) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllw))] +pub unsafe fn _mm256_maskz_sll_epi16(k: __mmask16, a: __m256i, count: __m128i) -> __m256i { + let shf = _mm256_sll_epi16(a, count).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 16-bit integers in a left by count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_sll_epi16&expand=5263) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllw))] +pub unsafe fn _mm_mask_sll_epi16(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_sll_epi16(a, count).as_i16x8(); + transmute(simd_select_bitmask(k, shf, src.as_i16x8())) +} + +/// Shift packed 16-bit integers in a left by count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_sll_epi16&expand=5264) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllw))] +pub unsafe fn _mm_maskz_sll_epi16(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_sll_epi16(a, count).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 16-bit integers in a left by imm8 while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_slli_epi16&expand=5301) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsllw, IMM8 = 5))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_slli_epi16(a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i16x32(); + let r = vpslliw(a, IMM8); + transmute(r) +} + +/// Shift packed 16-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_slli_epi16&expand=5299) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsllw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_slli_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, +) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i16x32(); + let shf = vpslliw(a, IMM8); + transmute(simd_select_bitmask(k, shf, src.as_i16x32())) +} + +/// Shift packed 16-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_slli_epi16&expand=5300) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsllw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_slli_epi16(k: __mmask32, a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i16x32(); + let shf = vpslliw(a, IMM8); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 16-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_slli_epi16&expand=5296) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_slli_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, +) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = pslliw256(a.as_i16x16(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i16x16())) +} + +/// Shift packed 16-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_slli_epi16&expand=5297) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_slli_epi16(k: __mmask16, a: __m256i) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = pslliw256(a.as_i16x16(), imm8); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Shift packed 16-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_slli_epi16&expand=5293) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_slli_epi16( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = pslliw128(a.as_i16x8(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i16x8())) +} + +/// Shift packed 16-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_slli_epi16&expand=5294) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_slli_epi16(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = pslliw128(a.as_i16x8(), imm8); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Shift packed 16-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sllv_epi16&expand=5333) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsllvw))] +pub unsafe fn _mm512_sllv_epi16(a: __m512i, count: __m512i) -> __m512i { + transmute(vpsllvw(a.as_i16x32(), count.as_i16x32())) +} + +/// Shift packed 16-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sllv_epi16&expand=5331) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsllvw))] +pub unsafe fn _mm512_mask_sllv_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, + count: __m512i, +) -> __m512i { + let shf = _mm512_sllv_epi16(a, count).as_i16x32(); + transmute(simd_select_bitmask(k, shf, src.as_i16x32())) +} + +/// Shift packed 16-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sllv_epi16&expand=5332) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsllvw))] +pub unsafe fn _mm512_maskz_sllv_epi16(k: __mmask32, a: __m512i, count: __m512i) -> __m512i { + let shf = _mm512_sllv_epi16(a, count).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 16-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_sllv_epi16&expand=5330) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllvw))] +pub unsafe fn _mm256_sllv_epi16(a: __m256i, count: __m256i) -> __m256i { + transmute(vpsllvw256(a.as_i16x16(), count.as_i16x16())) +} + +/// Shift packed 16-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_sllv_epi16&expand=5328) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllvw))] +pub unsafe fn _mm256_mask_sllv_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, + count: __m256i, +) -> __m256i { + let shf = _mm256_sllv_epi16(a, count).as_i16x16(); + transmute(simd_select_bitmask(k, shf, src.as_i16x16())) +} + +/// Shift packed 16-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_sllv_epi16&expand=5329) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllvw))] +pub unsafe fn _mm256_maskz_sllv_epi16(k: __mmask16, a: __m256i, count: __m256i) -> __m256i { + let shf = _mm256_sllv_epi16(a, count).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 16-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_sllv_epi16&expand=5327) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllvw))] +pub unsafe fn _mm_sllv_epi16(a: __m128i, count: __m128i) -> __m128i { + transmute(vpsllvw128(a.as_i16x8(), count.as_i16x8())) +} + +/// Shift packed 16-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_sllv_epi16&expand=5325) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllvw))] +pub unsafe fn _mm_mask_sllv_epi16( + src: __m128i, + k: __mmask8, + a: __m128i, + count: __m128i, +) -> __m128i { + let shf = _mm_sllv_epi16(a, count).as_i16x8(); + transmute(simd_select_bitmask(k, shf, src.as_i16x8())) +} + +/// Shift packed 16-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_sllv_epi16&expand=5326) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllvw))] +pub unsafe fn _mm_maskz_sllv_epi16(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_sllv_epi16(a, count).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 16-bit integers in a right by count while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_srl_epi16&expand=5483) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsrlw))] +pub unsafe fn _mm512_srl_epi16(a: __m512i, count: __m128i) -> __m512i { + transmute(vpsrlw(a.as_i16x32(), count.as_i16x8())) +} + +/// Shift packed 16-bit integers in a right by count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_srl_epi16&expand=5481) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsrlw))] +pub unsafe fn _mm512_mask_srl_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, + count: __m128i, +) -> __m512i { + let shf = _mm512_srl_epi16(a, count).as_i16x32(); + transmute(simd_select_bitmask(k, shf, src.as_i16x32())) +} + +/// Shift packed 16-bit integers in a right by count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_srl_epi16&expand=5482) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsrlw))] +pub unsafe fn _mm512_maskz_srl_epi16(k: __mmask32, a: __m512i, count: __m128i) -> __m512i { + let shf = _mm512_srl_epi16(a, count).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 16-bit integers in a right by count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_srl_epi16&expand=5478) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlw))] +pub unsafe fn _mm256_mask_srl_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, + count: __m128i, +) -> __m256i { + let shf = _mm256_srl_epi16(a, count).as_i16x16(); + transmute(simd_select_bitmask(k, shf, src.as_i16x16())) +} + +/// Shift packed 16-bit integers in a right by count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_srl_epi16&expand=5479) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlw))] +pub unsafe fn _mm256_maskz_srl_epi16(k: __mmask16, a: __m256i, count: __m128i) -> __m256i { + let shf = _mm256_srl_epi16(a, count).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 16-bit integers in a right by count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_srl_epi16&expand=5475) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlw))] +pub unsafe fn _mm_mask_srl_epi16(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_srl_epi16(a, count).as_i16x8(); + transmute(simd_select_bitmask(k, shf, src.as_i16x8())) +} + +/// Shift packed 16-bit integers in a right by count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_srl_epi16&expand=5476) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlw))] +pub unsafe fn _mm_maskz_srl_epi16(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_srl_epi16(a, count).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 16-bit integers in a right by imm8 while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_srli_epi16&expand=5513) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsrlw, IMM8 = 5))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_srli_epi16(a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i16x32(); + let r = vpsrliw(a, IMM8); + transmute(r) +} + +/// Shift packed 16-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_srli_epi16&expand=5511) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsrlw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_srli_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, +) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i16x32(); + let shf = vpsrliw(a, IMM8); + transmute(simd_select_bitmask(k, shf, src.as_i16x32())) +} + +/// Shift packed 16-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_srli_epi16&expand=5512) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsrlw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_srli_epi16(k: __mmask32, a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + //imm8 should be u32, it seems the document to verify is incorrect + let a = a.as_i16x32(); + let shf = vpsrliw(a, IMM8 as u32); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 16-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_srli_epi16&expand=5508) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_srli_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let shf = _mm256_srli_epi16::(a); + transmute(simd_select_bitmask(k, shf.as_i16x16(), src.as_i16x16())) +} + +/// Shift packed 16-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_srli_epi16&expand=5509) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_srli_epi16(k: __mmask16, a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let shf = _mm256_srli_epi16::(a); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, shf.as_i16x16(), zero)) +} + +/// Shift packed 16-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_srli_epi16&expand=5505) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_srli_epi16( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let shf = _mm_srli_epi16::(a); + transmute(simd_select_bitmask(k, shf.as_i16x8(), src.as_i16x8())) +} + +/// Shift packed 16-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_srli_epi16&expand=5506) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_srli_epi16(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let shf = _mm_srli_epi16::(a); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, shf.as_i16x8(), zero)) +} + +/// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_srlv_epi16&expand=5545) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsrlvw))] +pub unsafe fn _mm512_srlv_epi16(a: __m512i, count: __m512i) -> __m512i { + transmute(vpsrlvw(a.as_i16x32(), count.as_i16x32())) +} + +/// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_srlv_epi16&expand=5543) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsrlvw))] +pub unsafe fn _mm512_mask_srlv_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, + count: __m512i, +) -> __m512i { + let shf = _mm512_srlv_epi16(a, count).as_i16x32(); + transmute(simd_select_bitmask(k, shf, src.as_i16x32())) +} + +/// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_srlv_epi16&expand=5544) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsrlvw))] +pub unsafe fn _mm512_maskz_srlv_epi16(k: __mmask32, a: __m512i, count: __m512i) -> __m512i { + let shf = _mm512_srlv_epi16(a, count).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_srlv_epi16&expand=5542) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlvw))] +pub unsafe fn _mm256_srlv_epi16(a: __m256i, count: __m256i) -> __m256i { + transmute(vpsrlvw256(a.as_i16x16(), count.as_i16x16())) +} + +/// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_srlv_epi16&expand=5540) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlvw))] +pub unsafe fn _mm256_mask_srlv_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, + count: __m256i, +) -> __m256i { + let shf = _mm256_srlv_epi16(a, count).as_i16x16(); + transmute(simd_select_bitmask(k, shf, src.as_i16x16())) +} + +/// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_srlv_epi16&expand=5541) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlvw))] +pub unsafe fn _mm256_maskz_srlv_epi16(k: __mmask16, a: __m256i, count: __m256i) -> __m256i { + let shf = _mm256_srlv_epi16(a, count).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srlv_epi16&expand=5539) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlvw))] +pub unsafe fn _mm_srlv_epi16(a: __m128i, count: __m128i) -> __m128i { + transmute(vpsrlvw128(a.as_i16x8(), count.as_i16x8())) +} + +/// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_srlv_epi16&expand=5537) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlvw))] +pub unsafe fn _mm_mask_srlv_epi16( + src: __m128i, + k: __mmask8, + a: __m128i, + count: __m128i, +) -> __m128i { + let shf = _mm_srlv_epi16(a, count).as_i16x8(); + transmute(simd_select_bitmask(k, shf, src.as_i16x8())) +} + +/// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_srlv_epi16&expand=5538) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlvw))] +pub unsafe fn _mm_maskz_srlv_epi16(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_srlv_epi16(a, count).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 16-bit integers in a right by count while shifting in sign bits, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sra_epi16&expand=5398) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsraw))] +pub unsafe fn _mm512_sra_epi16(a: __m512i, count: __m128i) -> __m512i { + transmute(vpsraw(a.as_i16x32(), count.as_i16x8())) +} + +/// Shift packed 16-bit integers in a right by count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sra_epi16&expand=5396) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsraw))] +pub unsafe fn _mm512_mask_sra_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, + count: __m128i, +) -> __m512i { + let shf = _mm512_sra_epi16(a, count).as_i16x32(); + transmute(simd_select_bitmask(k, shf, src.as_i16x32())) +} + +/// Shift packed 16-bit integers in a right by count while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sra_epi16&expand=5397) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsraw))] +pub unsafe fn _mm512_maskz_sra_epi16(k: __mmask32, a: __m512i, count: __m128i) -> __m512i { + let shf = _mm512_sra_epi16(a, count).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 16-bit integers in a right by count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_sra_epi16&expand=5393) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsraw))] +pub unsafe fn _mm256_mask_sra_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, + count: __m128i, +) -> __m256i { + let shf = _mm256_sra_epi16(a, count).as_i16x16(); + transmute(simd_select_bitmask(k, shf, src.as_i16x16())) +} + +/// Shift packed 16-bit integers in a right by count while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_sra_epi16&expand=5394) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsraw))] +pub unsafe fn _mm256_maskz_sra_epi16(k: __mmask16, a: __m256i, count: __m128i) -> __m256i { + let shf = _mm256_sra_epi16(a, count).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 16-bit integers in a right by count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_sra_epi16&expand=5390) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsraw))] +pub unsafe fn _mm_mask_sra_epi16(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_sra_epi16(a, count).as_i16x8(); + transmute(simd_select_bitmask(k, shf, src.as_i16x8())) +} + +/// Shift packed 16-bit integers in a right by count while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_sra_epi16&expand=5391) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsraw))] +pub unsafe fn _mm_maskz_sra_epi16(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_sra_epi16(a, count).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 16-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_srai_epi16&expand=5427) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsraw, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_srai_epi16(a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i16x32(); + let r = vpsraiw(a, IMM8); + transmute(r) +} + +/// Shift packed 16-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_srai_epi16&expand=5425) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsraw, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_srai_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, +) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i16x32(); + let shf = vpsraiw(a, IMM8); + transmute(simd_select_bitmask(k, shf, src.as_i16x32())) +} + +/// Shift packed 16-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_srai_epi16&expand=5426) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsraw, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_srai_epi16(k: __mmask32, a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i16x32(); + let shf = vpsraiw(a, IMM8); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 16-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_srai_epi16&expand=5422) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsraw, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_srai_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, +) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psraiw256(a.as_i16x16(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i16x16())) +} + +/// Shift packed 16-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_srai_epi16&expand=5423) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsraw, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_srai_epi16(k: __mmask16, a: __m256i) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psraiw256(a.as_i16x16(), imm8); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Shift packed 16-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_srai_epi16&expand=5419) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsraw, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_srai_epi16( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psraiw128(a.as_i16x8(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i16x8())) +} + +/// Shift packed 16-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_srai_epi16&expand=5420) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsraw, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_srai_epi16(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psraiw128(a.as_i16x8(), imm8); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_srav_epi16&expand=5456) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsravw))] +pub unsafe fn _mm512_srav_epi16(a: __m512i, count: __m512i) -> __m512i { + transmute(vpsravw(a.as_i16x32(), count.as_i16x32())) +} + +/// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_srav_epi16&expand=5454) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsravw))] +pub unsafe fn _mm512_mask_srav_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, + count: __m512i, +) -> __m512i { + let shf = _mm512_srav_epi16(a, count).as_i16x32(); + transmute(simd_select_bitmask(k, shf, src.as_i16x32())) +} + +/// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_srav_epi16&expand=5455) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsravw))] +pub unsafe fn _mm512_maskz_srav_epi16(k: __mmask32, a: __m512i, count: __m512i) -> __m512i { + let shf = _mm512_srav_epi16(a, count).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_srav_epi16&expand=5453) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsravw))] +pub unsafe fn _mm256_srav_epi16(a: __m256i, count: __m256i) -> __m256i { + transmute(vpsravw256(a.as_i16x16(), count.as_i16x16())) +} + +/// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_srav_epi16&expand=5451) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsravw))] +pub unsafe fn _mm256_mask_srav_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, + count: __m256i, +) -> __m256i { + let shf = _mm256_srav_epi16(a, count).as_i16x16(); + transmute(simd_select_bitmask(k, shf, src.as_i16x16())) +} + +/// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_srav_epi16&expand=5452) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsravw))] +pub unsafe fn _mm256_maskz_srav_epi16(k: __mmask16, a: __m256i, count: __m256i) -> __m256i { + let shf = _mm256_srav_epi16(a, count).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srav_epi16&expand=5450) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsravw))] +pub unsafe fn _mm_srav_epi16(a: __m128i, count: __m128i) -> __m128i { + transmute(vpsravw128(a.as_i16x8(), count.as_i16x8())) +} + +/// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_srav_epi16&expand=5448) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsravw))] +pub unsafe fn _mm_mask_srav_epi16( + src: __m128i, + k: __mmask8, + a: __m128i, + count: __m128i, +) -> __m128i { + let shf = _mm_srav_epi16(a, count).as_i16x8(); + transmute(simd_select_bitmask(k, shf, src.as_i16x8())) +} + +/// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_srav_epi16&expand=5449) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpsravw))] +pub unsafe fn _mm_maskz_srav_epi16(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_srav_epi16(a, count).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shuffle 16-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permutex2var_epi16&expand=4226) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2w or vpermt2w +pub unsafe fn _mm512_permutex2var_epi16(a: __m512i, idx: __m512i, b: __m512i) -> __m512i { + transmute(vpermi2w(a.as_i16x32(), idx.as_i16x32(), b.as_i16x32())) +} + +/// Shuffle 16-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permutex2var_epi16&expand=4223) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpermt2w))] +pub unsafe fn _mm512_mask_permutex2var_epi16( + a: __m512i, + k: __mmask32, + idx: __m512i, + b: __m512i, +) -> __m512i { + let permute = _mm512_permutex2var_epi16(a, idx, b).as_i16x32(); + transmute(simd_select_bitmask(k, permute, a.as_i16x32())) +} + +/// Shuffle 16-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permutex2var_epi16&expand=4225) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2w or vpermt2w +pub unsafe fn _mm512_maskz_permutex2var_epi16( + k: __mmask32, + a: __m512i, + idx: __m512i, + b: __m512i, +) -> __m512i { + let permute = _mm512_permutex2var_epi16(a, idx, b).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle 16-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from idx when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask2_permutex2var_epi16&expand=4224) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpermi2w))] +pub unsafe fn _mm512_mask2_permutex2var_epi16( + a: __m512i, + idx: __m512i, + k: __mmask32, + b: __m512i, +) -> __m512i { + let permute = _mm512_permutex2var_epi16(a, idx, b).as_i16x32(); + transmute(simd_select_bitmask(k, permute, idx.as_i16x32())) +} + +/// Shuffle 16-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permutex2var_epi16&expand=4222) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2w or vpermt2w +pub unsafe fn _mm256_permutex2var_epi16(a: __m256i, idx: __m256i, b: __m256i) -> __m256i { + transmute(vpermi2w256(a.as_i16x16(), idx.as_i16x16(), b.as_i16x16())) +} + +/// Shuffle 16-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_permutex2var_epi16&expand=4219) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpermt2w))] +pub unsafe fn _mm256_mask_permutex2var_epi16( + a: __m256i, + k: __mmask16, + idx: __m256i, + b: __m256i, +) -> __m256i { + let permute = _mm256_permutex2var_epi16(a, idx, b).as_i16x16(); + transmute(simd_select_bitmask(k, permute, a.as_i16x16())) +} + +/// Shuffle 16-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permutex2var_epi16&expand=4221) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2w or vpermt2w +pub unsafe fn _mm256_maskz_permutex2var_epi16( + k: __mmask16, + a: __m256i, + idx: __m256i, + b: __m256i, +) -> __m256i { + let permute = _mm256_permutex2var_epi16(a, idx, b).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle 16-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from idx when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask2_permutex2var_epi16&expand=4220) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpermi2w))] +pub unsafe fn _mm256_mask2_permutex2var_epi16( + a: __m256i, + idx: __m256i, + k: __mmask16, + b: __m256i, +) -> __m256i { + let permute = _mm256_permutex2var_epi16(a, idx, b).as_i16x16(); + transmute(simd_select_bitmask(k, permute, idx.as_i16x16())) +} + +/// Shuffle 16-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_permutex2var_epi16&expand=4218) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2w or vpermt2w +pub unsafe fn _mm_permutex2var_epi16(a: __m128i, idx: __m128i, b: __m128i) -> __m128i { + transmute(vpermi2w128(a.as_i16x8(), idx.as_i16x8(), b.as_i16x8())) +} + +/// Shuffle 16-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_permutex2var_epi16&expand=4215) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpermt2w))] +pub unsafe fn _mm_mask_permutex2var_epi16( + a: __m128i, + k: __mmask8, + idx: __m128i, + b: __m128i, +) -> __m128i { + let permute = _mm_permutex2var_epi16(a, idx, b).as_i16x8(); + transmute(simd_select_bitmask(k, permute, a.as_i16x8())) +} + +/// Shuffle 16-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_permutex2var_epi16&expand=4217) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2w or vpermt2w +pub unsafe fn _mm_maskz_permutex2var_epi16( + k: __mmask8, + a: __m128i, + idx: __m128i, + b: __m128i, +) -> __m128i { + let permute = _mm_permutex2var_epi16(a, idx, b).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle 16-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from idx when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask2_permutex2var_epi16&expand=4216) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpermi2w))] +pub unsafe fn _mm_mask2_permutex2var_epi16( + a: __m128i, + idx: __m128i, + k: __mmask8, + b: __m128i, +) -> __m128i { + let permute = _mm_permutex2var_epi16(a, idx, b).as_i16x8(); + transmute(simd_select_bitmask(k, permute, idx.as_i16x8())) +} + +/// Shuffle 16-bit integers in a across lanes using the corresponding index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permutexvar_epi16&expand=4295) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpermw))] +pub unsafe fn _mm512_permutexvar_epi16(idx: __m512i, a: __m512i) -> __m512i { + transmute(vpermw(a.as_i16x32(), idx.as_i16x32())) +} + +/// Shuffle 16-bit integers in a across lanes using the corresponding index in idx, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permutexvar_epi16&expand=4293) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpermw))] +pub unsafe fn _mm512_mask_permutexvar_epi16( + src: __m512i, + k: __mmask32, + idx: __m512i, + a: __m512i, +) -> __m512i { + let permute = _mm512_permutexvar_epi16(idx, a).as_i16x32(); + transmute(simd_select_bitmask(k, permute, src.as_i16x32())) +} + +/// Shuffle 16-bit integers in a across lanes using the corresponding index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permutexvar_epi16&expand=4294) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpermw))] +pub unsafe fn _mm512_maskz_permutexvar_epi16(k: __mmask32, idx: __m512i, a: __m512i) -> __m512i { + let permute = _mm512_permutexvar_epi16(idx, a).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle 16-bit integers in a across lanes using the corresponding index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permutexvar_epi16&expand=4292) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpermw))] +pub unsafe fn _mm256_permutexvar_epi16(idx: __m256i, a: __m256i) -> __m256i { + transmute(vpermw256(a.as_i16x16(), idx.as_i16x16())) +} + +/// Shuffle 16-bit integers in a across lanes using the corresponding index in idx, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_permutexvar_epi16&expand=4290) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpermw))] +pub unsafe fn _mm256_mask_permutexvar_epi16( + src: __m256i, + k: __mmask16, + idx: __m256i, + a: __m256i, +) -> __m256i { + let permute = _mm256_permutexvar_epi16(idx, a).as_i16x16(); + transmute(simd_select_bitmask(k, permute, src.as_i16x16())) +} + +/// Shuffle 16-bit integers in a across lanes using the corresponding index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permutexvar_epi16&expand=4291) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpermw))] +pub unsafe fn _mm256_maskz_permutexvar_epi16(k: __mmask16, idx: __m256i, a: __m256i) -> __m256i { + let permute = _mm256_permutexvar_epi16(idx, a).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle 16-bit integers in a across lanes using the corresponding index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_permutexvar_epi16&expand=4289) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpermw))] +pub unsafe fn _mm_permutexvar_epi16(idx: __m128i, a: __m128i) -> __m128i { + transmute(vpermw128(a.as_i16x8(), idx.as_i16x8())) +} + +/// Shuffle 16-bit integers in a across lanes using the corresponding index in idx, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_permutexvar_epi16&expand=4287) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpermw))] +pub unsafe fn _mm_mask_permutexvar_epi16( + src: __m128i, + k: __mmask8, + idx: __m128i, + a: __m128i, +) -> __m128i { + let permute = _mm_permutexvar_epi16(idx, a).as_i16x8(); + transmute(simd_select_bitmask(k, permute, src.as_i16x8())) +} + +/// Shuffle 16-bit integers in a across lanes using the corresponding index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_permutexvar_epi16&expand=4288) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpermw))] +pub unsafe fn _mm_maskz_permutexvar_epi16(k: __mmask8, idx: __m128i, a: __m128i) -> __m128i { + let permute = _mm_permutexvar_epi16(idx, a).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Blend packed 16-bit integers from a and b using control mask k, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_blend_epi16&expand=430) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vmovdqu16))] //should be vpblendmw +pub unsafe fn _mm512_mask_blend_epi16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + transmute(simd_select_bitmask(k, b.as_i16x32(), a.as_i16x32())) +} + +/// Blend packed 16-bit integers from a and b using control mask k, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_blend_epi16&expand=429) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqu16))] //should be vpblendmw +pub unsafe fn _mm256_mask_blend_epi16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + transmute(simd_select_bitmask(k, b.as_i16x16(), a.as_i16x16())) +} + +/// Blend packed 16-bit integers from a and b using control mask k, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_blend_epi16&expand=427) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqu16))] //should be vpblendmw +pub unsafe fn _mm_mask_blend_epi16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + transmute(simd_select_bitmask(k, b.as_i16x8(), a.as_i16x8())) +} + +/// Blend packed 8-bit integers from a and b using control mask k, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_blend_epi8&expand=441) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vmovdqu8))] //should be vpblendmb +pub unsafe fn _mm512_mask_blend_epi8(k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + transmute(simd_select_bitmask(k, b.as_i8x64(), a.as_i8x64())) +} + +/// Blend packed 8-bit integers from a and b using control mask k, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_blend_epi8&expand=440) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqu8))] //should be vpblendmb +pub unsafe fn _mm256_mask_blend_epi8(k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + transmute(simd_select_bitmask(k, b.as_i8x32(), a.as_i8x32())) +} + +/// Blend packed 8-bit integers from a and b using control mask k, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_blend_epi8&expand=439) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqu8))] //should be vpblendmb +pub unsafe fn _mm_mask_blend_epi8(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + transmute(simd_select_bitmask(k, b.as_i8x16(), a.as_i8x16())) +} + +/// Broadcast the low packed 16-bit integer from a to all elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_broadcastw_epi16&expand=587) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpbroadcastw))] +pub unsafe fn _mm512_broadcastw_epi16(a: __m128i) -> __m512i { + let a = _mm512_castsi128_si512(a).as_i16x32(); + let ret: i16x32 = simd_shuffle32!( + a, + a, + [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, + ], + ); + transmute(ret) +} + +/// Broadcast the low packed 16-bit integer from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_broadcastw_epi16&expand=588) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpbroadcastw))] +pub unsafe fn _mm512_mask_broadcastw_epi16(src: __m512i, k: __mmask32, a: __m128i) -> __m512i { + let broadcast = _mm512_broadcastw_epi16(a).as_i16x32(); + transmute(simd_select_bitmask(k, broadcast, src.as_i16x32())) +} + +/// Broadcast the low packed 16-bit integer from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_broadcastw_epi16&expand=589) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpbroadcastw))] +pub unsafe fn _mm512_maskz_broadcastw_epi16(k: __mmask32, a: __m128i) -> __m512i { + let broadcast = _mm512_broadcastw_epi16(a).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Broadcast the low packed 16-bit integer from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_broadcastw_epi16&expand=585) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastw))] +pub unsafe fn _mm256_mask_broadcastw_epi16(src: __m256i, k: __mmask16, a: __m128i) -> __m256i { + let broadcast = _mm256_broadcastw_epi16(a).as_i16x16(); + transmute(simd_select_bitmask(k, broadcast, src.as_i16x16())) +} + +/// Broadcast the low packed 16-bit integer from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_broadcastw_epi16&expand=586) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastw))] +pub unsafe fn _mm256_maskz_broadcastw_epi16(k: __mmask16, a: __m128i) -> __m256i { + let broadcast = _mm256_broadcastw_epi16(a).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Broadcast the low packed 16-bit integer from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_broadcastw_epi16&expand=582) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastw))] +pub unsafe fn _mm_mask_broadcastw_epi16(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let broadcast = _mm_broadcastw_epi16(a).as_i16x8(); + transmute(simd_select_bitmask(k, broadcast, src.as_i16x8())) +} + +/// Broadcast the low packed 16-bit integer from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_broadcastw_epi16&expand=583) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastw))] +pub unsafe fn _mm_maskz_broadcastw_epi16(k: __mmask8, a: __m128i) -> __m128i { + let broadcast = _mm_broadcastw_epi16(a).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Broadcast the low packed 8-bit integer from a to all elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_broadcastb_epi8&expand=536) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpbroadcastb))] +pub unsafe fn _mm512_broadcastb_epi8(a: __m128i) -> __m512i { + let a = _mm512_castsi128_si512(a).as_i8x64(); + let ret: i8x64 = simd_shuffle64!( + a, + a, + [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + ], + ); + transmute(ret) +} + +/// Broadcast the low packed 8-bit integer from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_broadcastb_epi8&expand=537) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpbroadcastb))] +pub unsafe fn _mm512_mask_broadcastb_epi8(src: __m512i, k: __mmask64, a: __m128i) -> __m512i { + let broadcast = _mm512_broadcastb_epi8(a).as_i8x64(); + transmute(simd_select_bitmask(k, broadcast, src.as_i8x64())) +} + +/// Broadcast the low packed 8-bit integer from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_broadcastb_epi8&expand=538) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpbroadcastb))] +pub unsafe fn _mm512_maskz_broadcastb_epi8(k: __mmask64, a: __m128i) -> __m512i { + let broadcast = _mm512_broadcastb_epi8(a).as_i8x64(); + let zero = _mm512_setzero_si512().as_i8x64(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Broadcast the low packed 8-bit integer from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_broadcastb_epi8&expand=534) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastb))] +pub unsafe fn _mm256_mask_broadcastb_epi8(src: __m256i, k: __mmask32, a: __m128i) -> __m256i { + let broadcast = _mm256_broadcastb_epi8(a).as_i8x32(); + transmute(simd_select_bitmask(k, broadcast, src.as_i8x32())) +} + +/// Broadcast the low packed 8-bit integer from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_broadcastb_epi8&expand=535) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastb))] +pub unsafe fn _mm256_maskz_broadcastb_epi8(k: __mmask32, a: __m128i) -> __m256i { + let broadcast = _mm256_broadcastb_epi8(a).as_i8x32(); + let zero = _mm256_setzero_si256().as_i8x32(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Broadcast the low packed 8-bit integer from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_broadcastb_epi8&expand=531) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastb))] +pub unsafe fn _mm_mask_broadcastb_epi8(src: __m128i, k: __mmask16, a: __m128i) -> __m128i { + let broadcast = _mm_broadcastb_epi8(a).as_i8x16(); + transmute(simd_select_bitmask(k, broadcast, src.as_i8x16())) +} + +/// Broadcast the low packed 8-bit integer from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_broadcastb_epi8&expand=532) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastb))] +pub unsafe fn _mm_maskz_broadcastb_epi8(k: __mmask16, a: __m128i) -> __m128i { + let broadcast = _mm_broadcastb_epi8(a).as_i8x16(); + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Unpack and interleave 16-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_unpackhi_epi16&expand=6012) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpunpckhwd))] +pub unsafe fn _mm512_unpackhi_epi16(a: __m512i, b: __m512i) -> __m512i { + let a = a.as_i16x32(); + let b = b.as_i16x32(); + #[rustfmt::skip] + let r: i16x32 = simd_shuffle32!( + a, + b, + [ + 4, 32 + 4, 5, 32 + 5, + 6, 32 + 6, 7, 32 + 7, + 12, 32 + 12, 13, 32 + 13, + 14, 32 + 14, 15, 32 + 15, + 20, 32 + 20, 21, 32 + 21, + 22, 32 + 22, 23, 32 + 23, + 28, 32 + 28, 29, 32 + 29, + 30, 32 + 30, 31, 32 + 31, + ], + ); + transmute(r) +} + +/// Unpack and interleave 16-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_unpackhi_epi16&expand=6010) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpunpckhwd))] +pub unsafe fn _mm512_mask_unpackhi_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + let unpackhi = _mm512_unpackhi_epi16(a, b).as_i16x32(); + transmute(simd_select_bitmask(k, unpackhi, src.as_i16x32())) +} + +/// Unpack and interleave 16-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_unpackhi_epi16&expand=6011) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpunpckhwd))] +pub unsafe fn _mm512_maskz_unpackhi_epi16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let unpackhi = _mm512_unpackhi_epi16(a, b).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, unpackhi, zero)) +} + +/// Unpack and interleave 16-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_unpackhi_epi16&expand=6007) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpckhwd))] +pub unsafe fn _mm256_mask_unpackhi_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + let unpackhi = _mm256_unpackhi_epi16(a, b).as_i16x16(); + transmute(simd_select_bitmask(k, unpackhi, src.as_i16x16())) +} + +/// Unpack and interleave 16-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_unpackhi_epi16&expand=6008) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpckhwd))] +pub unsafe fn _mm256_maskz_unpackhi_epi16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let unpackhi = _mm256_unpackhi_epi16(a, b).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, unpackhi, zero)) +} + +/// Unpack and interleave 16-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_unpackhi_epi16&expand=6004) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpckhwd))] +pub unsafe fn _mm_mask_unpackhi_epi16( + src: __m128i, + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + let unpackhi = _mm_unpackhi_epi16(a, b).as_i16x8(); + transmute(simd_select_bitmask(k, unpackhi, src.as_i16x8())) +} + +/// Unpack and interleave 16-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_unpackhi_epi16&expand=6005) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpckhwd))] +pub unsafe fn _mm_maskz_unpackhi_epi16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let unpackhi = _mm_unpackhi_epi16(a, b).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, unpackhi, zero)) +} + +/// Unpack and interleave 8-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_unpackhi_epi8&expand=6039) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpunpckhbw))] +pub unsafe fn _mm512_unpackhi_epi8(a: __m512i, b: __m512i) -> __m512i { + let a = a.as_i8x64(); + let b = b.as_i8x64(); + #[rustfmt::skip] + let r: i8x64 = simd_shuffle64!( + a, + b, + [ + 8, 64+8, 9, 64+9, + 10, 64+10, 11, 64+11, + 12, 64+12, 13, 64+13, + 14, 64+14, 15, 64+15, + 24, 64+24, 25, 64+25, + 26, 64+26, 27, 64+27, + 28, 64+28, 29, 64+29, + 30, 64+30, 31, 64+31, + 40, 64+40, 41, 64+41, + 42, 64+42, 43, 64+43, + 44, 64+44, 45, 64+45, + 46, 64+46, 47, 64+47, + 56, 64+56, 57, 64+57, + 58, 64+58, 59, 64+59, + 60, 64+60, 61, 64+61, + 62, 64+62, 63, 64+63, + ], + ); + transmute(r) +} + +/// Unpack and interleave 8-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_unpackhi_epi8&expand=6037) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpunpckhbw))] +pub unsafe fn _mm512_mask_unpackhi_epi8( + src: __m512i, + k: __mmask64, + a: __m512i, + b: __m512i, +) -> __m512i { + let unpackhi = _mm512_unpackhi_epi8(a, b).as_i8x64(); + transmute(simd_select_bitmask(k, unpackhi, src.as_i8x64())) +} + +/// Unpack and interleave 8-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_unpackhi_epi8&expand=6038) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpunpckhbw))] +pub unsafe fn _mm512_maskz_unpackhi_epi8(k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + let unpackhi = _mm512_unpackhi_epi8(a, b).as_i8x64(); + let zero = _mm512_setzero_si512().as_i8x64(); + transmute(simd_select_bitmask(k, unpackhi, zero)) +} + +/// Unpack and interleave 8-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_unpackhi_epi8&expand=6034) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpckhbw))] +pub unsafe fn _mm256_mask_unpackhi_epi8( + src: __m256i, + k: __mmask32, + a: __m256i, + b: __m256i, +) -> __m256i { + let unpackhi = _mm256_unpackhi_epi8(a, b).as_i8x32(); + transmute(simd_select_bitmask(k, unpackhi, src.as_i8x32())) +} + +/// Unpack and interleave 8-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_unpackhi_epi8&expand=6035) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpckhbw))] +pub unsafe fn _mm256_maskz_unpackhi_epi8(k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + let unpackhi = _mm256_unpackhi_epi8(a, b).as_i8x32(); + let zero = _mm256_setzero_si256().as_i8x32(); + transmute(simd_select_bitmask(k, unpackhi, zero)) +} + +/// Unpack and interleave 8-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_unpackhi_epi8&expand=6031) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpckhbw))] +pub unsafe fn _mm_mask_unpackhi_epi8( + src: __m128i, + k: __mmask16, + a: __m128i, + b: __m128i, +) -> __m128i { + let unpackhi = _mm_unpackhi_epi8(a, b).as_i8x16(); + transmute(simd_select_bitmask(k, unpackhi, src.as_i8x16())) +} + +/// Unpack and interleave 8-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_unpackhi_epi8&expand=6032) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpckhbw))] +pub unsafe fn _mm_maskz_unpackhi_epi8(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let unpackhi = _mm_unpackhi_epi8(a, b).as_i8x16(); + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask(k, unpackhi, zero)) +} + +/// Unpack and interleave 16-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_unpacklo_epi16&expand=6069) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpunpcklwd))] +pub unsafe fn _mm512_unpacklo_epi16(a: __m512i, b: __m512i) -> __m512i { + let a = a.as_i16x32(); + let b = b.as_i16x32(); + #[rustfmt::skip] + let r: i16x32 = simd_shuffle32!( + a, + b, + [ + 0, 32+0, 1, 32+1, + 2, 32+2, 3, 32+3, + 8, 32+8, 9, 32+9, + 10, 32+10, 11, 32+11, + 16, 32+16, 17, 32+17, + 18, 32+18, 19, 32+19, + 24, 32+24, 25, 32+25, + 26, 32+26, 27, 32+27 + ], + ); + transmute(r) +} + +/// Unpack and interleave 16-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_unpacklo_epi16&expand=6067) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpunpcklwd))] +pub unsafe fn _mm512_mask_unpacklo_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + let unpacklo = _mm512_unpacklo_epi16(a, b).as_i16x32(); + transmute(simd_select_bitmask(k, unpacklo, src.as_i16x32())) +} + +/// Unpack and interleave 16-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_unpacklo_epi16&expand=6068) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpunpcklwd))] +pub unsafe fn _mm512_maskz_unpacklo_epi16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { + let unpacklo = _mm512_unpacklo_epi16(a, b).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, unpacklo, zero)) +} + +/// Unpack and interleave 16-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_unpacklo_epi16&expand=6064) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpcklwd))] +pub unsafe fn _mm256_mask_unpacklo_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + let unpacklo = _mm256_unpacklo_epi16(a, b).as_i16x16(); + transmute(simd_select_bitmask(k, unpacklo, src.as_i16x16())) +} + +/// Unpack and interleave 16-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_unpacklo_epi16&expand=6065) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpcklwd))] +pub unsafe fn _mm256_maskz_unpacklo_epi16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { + let unpacklo = _mm256_unpacklo_epi16(a, b).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, unpacklo, zero)) +} + +/// Unpack and interleave 16-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_unpacklo_epi16&expand=6061) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpcklwd))] +pub unsafe fn _mm_mask_unpacklo_epi16( + src: __m128i, + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + let unpacklo = _mm_unpacklo_epi16(a, b).as_i16x8(); + transmute(simd_select_bitmask(k, unpacklo, src.as_i16x8())) +} + +/// Unpack and interleave 16-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_unpacklo_epi16&expand=6062) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpcklwd))] +pub unsafe fn _mm_maskz_unpacklo_epi16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let unpacklo = _mm_unpacklo_epi16(a, b).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, unpacklo, zero)) +} + +/// Unpack and interleave 8-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_unpacklo_epi8&expand=6096) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpunpcklbw))] +pub unsafe fn _mm512_unpacklo_epi8(a: __m512i, b: __m512i) -> __m512i { + let a = a.as_i8x64(); + let b = b.as_i8x64(); + #[rustfmt::skip] + let r: i8x64 = simd_shuffle64!( + a, + b, + [ + 0, 64+0, 1, 64+1, + 2, 64+2, 3, 64+3, + 4, 64+4, 5, 64+5, + 6, 64+6, 7, 64+7, + 16, 64+16, 17, 64+17, + 18, 64+18, 19, 64+19, + 20, 64+20, 21, 64+21, + 22, 64+22, 23, 64+23, + 32, 64+32, 33, 64+33, + 34, 64+34, 35, 64+35, + 36, 64+36, 37, 64+37, + 38, 64+38, 39, 64+39, + 48, 64+48, 49, 64+49, + 50, 64+50, 51, 64+51, + 52, 64+52, 53, 64+53, + 54, 64+54, 55, 64+55, + ], + ); + transmute(r) +} + +/// Unpack and interleave 8-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_unpacklo_epi8&expand=6094) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpunpcklbw))] +pub unsafe fn _mm512_mask_unpacklo_epi8( + src: __m512i, + k: __mmask64, + a: __m512i, + b: __m512i, +) -> __m512i { + let unpacklo = _mm512_unpacklo_epi8(a, b).as_i8x64(); + transmute(simd_select_bitmask(k, unpacklo, src.as_i8x64())) +} + +/// Unpack and interleave 8-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_unpacklo_epi8&expand=6095) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpunpcklbw))] +pub unsafe fn _mm512_maskz_unpacklo_epi8(k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + let unpacklo = _mm512_unpacklo_epi8(a, b).as_i8x64(); + let zero = _mm512_setzero_si512().as_i8x64(); + transmute(simd_select_bitmask(k, unpacklo, zero)) +} + +/// Unpack and interleave 8-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_unpacklo_epi8&expand=6091) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpcklbw))] +pub unsafe fn _mm256_mask_unpacklo_epi8( + src: __m256i, + k: __mmask32, + a: __m256i, + b: __m256i, +) -> __m256i { + let unpacklo = _mm256_unpacklo_epi8(a, b).as_i8x32(); + transmute(simd_select_bitmask(k, unpacklo, src.as_i8x32())) +} + +/// Unpack and interleave 8-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_unpacklo_epi8&expand=6092) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpcklbw))] +pub unsafe fn _mm256_maskz_unpacklo_epi8(k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + let unpacklo = _mm256_unpacklo_epi8(a, b).as_i8x32(); + let zero = _mm256_setzero_si256().as_i8x32(); + transmute(simd_select_bitmask(k, unpacklo, zero)) +} + +/// Unpack and interleave 8-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_unpacklo_epi8&expand=6088) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpcklbw))] +pub unsafe fn _mm_mask_unpacklo_epi8( + src: __m128i, + k: __mmask16, + a: __m128i, + b: __m128i, +) -> __m128i { + let unpacklo = _mm_unpacklo_epi8(a, b).as_i8x16(); + transmute(simd_select_bitmask(k, unpacklo, src.as_i8x16())) +} + +/// Unpack and interleave 8-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_unpacklo_epi8&expand=6089) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpcklbw))] +pub unsafe fn _mm_maskz_unpacklo_epi8(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let unpacklo = _mm_unpacklo_epi8(a, b).as_i8x16(); + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask(k, unpacklo, zero)) +} + +/// Move packed 16-bit integers from a into dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_mov_epi16&expand=3795) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vmovdqu16))] +pub unsafe fn _mm512_mask_mov_epi16(src: __m512i, k: __mmask32, a: __m512i) -> __m512i { + let mov = a.as_i16x32(); + transmute(simd_select_bitmask(k, mov, src.as_i16x32())) +} + +/// Move packed 16-bit integers from a into dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_mov_epi16&expand=3796) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vmovdqu16))] +pub unsafe fn _mm512_maskz_mov_epi16(k: __mmask32, a: __m512i) -> __m512i { + let mov = a.as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, mov, zero)) +} + +/// Move packed 16-bit integers from a into dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_mov_epi16&expand=3793) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqu16))] +pub unsafe fn _mm256_mask_mov_epi16(src: __m256i, k: __mmask16, a: __m256i) -> __m256i { + let mov = a.as_i16x16(); + transmute(simd_select_bitmask(k, mov, src.as_i16x16())) +} + +/// Move packed 16-bit integers from a into dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_mov_epi16&expand=3794) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqu16))] +pub unsafe fn _mm256_maskz_mov_epi16(k: __mmask16, a: __m256i) -> __m256i { + let mov = a.as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, mov, zero)) +} + +/// Move packed 16-bit integers from a into dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_mov_epi16&expand=3791) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqu16))] +pub unsafe fn _mm_mask_mov_epi16(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let mov = a.as_i16x8(); + transmute(simd_select_bitmask(k, mov, src.as_i16x8())) +} + +/// Move packed 16-bit integers from a into dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_mov_epi16&expand=3792) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqu16))] +pub unsafe fn _mm_maskz_mov_epi16(k: __mmask8, a: __m128i) -> __m128i { + let mov = a.as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, mov, zero)) +} + +/// Move packed 8-bit integers from a into dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_mov_epi8&expand=3813) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vmovdqu8))] +pub unsafe fn _mm512_mask_mov_epi8(src: __m512i, k: __mmask64, a: __m512i) -> __m512i { + let mov = a.as_i8x64(); + transmute(simd_select_bitmask(k, mov, src.as_i8x64())) +} + +/// Move packed 8-bit integers from a into dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_mov_epi8&expand=3814) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vmovdqu8))] +pub unsafe fn _mm512_maskz_mov_epi8(k: __mmask64, a: __m512i) -> __m512i { + let mov = a.as_i8x64(); + let zero = _mm512_setzero_si512().as_i8x64(); + transmute(simd_select_bitmask(k, mov, zero)) +} + +/// Move packed 8-bit integers from a into dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_mov_epi8&expand=3811) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqu8))] +pub unsafe fn _mm256_mask_mov_epi8(src: __m256i, k: __mmask32, a: __m256i) -> __m256i { + let mov = a.as_i8x32(); + transmute(simd_select_bitmask(k, mov, src.as_i8x32())) +} + +/// Move packed 8-bit integers from a into dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_mov_epi8&expand=3812) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqu8))] +pub unsafe fn _mm256_maskz_mov_epi8(k: __mmask32, a: __m256i) -> __m256i { + let mov = a.as_i8x32(); + let zero = _mm256_setzero_si256().as_i8x32(); + transmute(simd_select_bitmask(k, mov, zero)) +} + +/// Move packed 8-bit integers from a into dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_mov_epi8&expand=3809) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqu8))] +pub unsafe fn _mm_mask_mov_epi8(src: __m128i, k: __mmask16, a: __m128i) -> __m128i { + let mov = a.as_i8x16(); + transmute(simd_select_bitmask(k, mov, src.as_i8x16())) +} + +/// Move packed 8-bit integers from a into dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_mov_epi8&expand=3810) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqu8))] +pub unsafe fn _mm_maskz_mov_epi8(k: __mmask16, a: __m128i) -> __m128i { + let mov = a.as_i8x16(); + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask(k, mov, zero)) +} + +/// Broadcast 16-bit integer a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_set1_epi16&expand=4942) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpbroadcastw))] +pub unsafe fn _mm512_mask_set1_epi16(src: __m512i, k: __mmask32, a: i16) -> __m512i { + let r = _mm512_set1_epi16(a).as_i16x32(); + transmute(simd_select_bitmask(k, r, src.as_i16x32())) +} + +/// Broadcast the low packed 16-bit integer from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_set1_epi16&expand=4943) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpbroadcastw))] +pub unsafe fn _mm512_maskz_set1_epi16(k: __mmask32, a: i16) -> __m512i { + let r = _mm512_set1_epi16(a).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Broadcast 16-bit integer a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_set1_epi16&expand=4939) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastw))] +pub unsafe fn _mm256_mask_set1_epi16(src: __m256i, k: __mmask16, a: i16) -> __m256i { + let r = _mm256_set1_epi16(a).as_i16x16(); + transmute(simd_select_bitmask(k, r, src.as_i16x16())) +} + +/// Broadcast the low packed 16-bit integer from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_set1_epi16&expand=4940) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastw))] +pub unsafe fn _mm256_maskz_set1_epi16(k: __mmask16, a: i16) -> __m256i { + let r = _mm256_set1_epi16(a).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Broadcast 16-bit integer a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_set1_epi16&expand=4936) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastw))] +pub unsafe fn _mm_mask_set1_epi16(src: __m128i, k: __mmask8, a: i16) -> __m128i { + let r = _mm_set1_epi16(a).as_i16x8(); + transmute(simd_select_bitmask(k, r, src.as_i16x8())) +} + +/// Broadcast the low packed 16-bit integer from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_set1_epi16&expand=4937) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastw))] +pub unsafe fn _mm_maskz_set1_epi16(k: __mmask8, a: i16) -> __m128i { + let r = _mm_set1_epi16(a).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Broadcast 8-bit integer a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_set1_epi8&expand=4970) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpbroadcastb))] +pub unsafe fn _mm512_mask_set1_epi8(src: __m512i, k: __mmask64, a: i8) -> __m512i { + let r = _mm512_set1_epi8(a).as_i8x64(); + transmute(simd_select_bitmask(k, r, src.as_i8x64())) +} + +/// Broadcast 8-bit integer a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_set1_epi8&expand=4971) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpbroadcastb))] +pub unsafe fn _mm512_maskz_set1_epi8(k: __mmask64, a: i8) -> __m512i { + let r = _mm512_set1_epi8(a).as_i8x64(); + let zero = _mm512_setzero_si512().as_i8x64(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Broadcast 8-bit integer a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_set1_epi8&expand=4967) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastb))] +pub unsafe fn _mm256_mask_set1_epi8(src: __m256i, k: __mmask32, a: i8) -> __m256i { + let r = _mm256_set1_epi8(a).as_i8x32(); + transmute(simd_select_bitmask(k, r, src.as_i8x32())) +} + +/// Broadcast 8-bit integer a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_set1_epi8&expand=4968) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastb))] +pub unsafe fn _mm256_maskz_set1_epi8(k: __mmask32, a: i8) -> __m256i { + let r = _mm256_set1_epi8(a).as_i8x32(); + let zero = _mm256_setzero_si256().as_i8x32(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Broadcast 8-bit integer a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_set1_epi8&expand=4964) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastb))] +pub unsafe fn _mm_mask_set1_epi8(src: __m128i, k: __mmask16, a: i8) -> __m128i { + let r = _mm_set1_epi8(a).as_i8x16(); + transmute(simd_select_bitmask(k, r, src.as_i8x16())) +} + +/// Broadcast 8-bit integer a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_set1_epi8&expand=4965) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastb))] +pub unsafe fn _mm_maskz_set1_epi8(k: __mmask16, a: i8) -> __m128i { + let r = _mm_set1_epi8(a).as_i8x16(); + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Shuffle 16-bit integers in the low 64 bits of 128-bit lanes of a using the control in imm8. Store the results in the low 64 bits of 128-bit lanes of dst, with the high 64 bits of 128-bit lanes being copied from from a to dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shufflelo_epi16&expand=5221) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpshuflw, IMM8 = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_shufflelo_epi16(a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_i16x32(); + let r: i16x32 = simd_shuffle32!( + a, + a, + [ + IMM8 as u32 & 0b11, + (IMM8 as u32 >> 2) & 0b11, + (IMM8 as u32 >> 4) & 0b11, + (IMM8 as u32 >> 6) & 0b11, + 4, + 5, + 6, + 7, + (IMM8 as u32 & 0b11) + 8, + ((IMM8 as u32 >> 2) & 0b11) + 8, + ((IMM8 as u32 >> 4) & 0b11) + 8, + ((IMM8 as u32 >> 6) & 0b11) + 8, + 12, + 13, + 14, + 15, + (IMM8 as u32 & 0b11) + 16, + ((IMM8 as u32 >> 2) & 0b11) + 16, + ((IMM8 as u32 >> 4) & 0b11) + 16, + ((IMM8 as u32 >> 6) & 0b11) + 16, + 20, + 21, + 22, + 23, + (IMM8 as u32 & 0b11) + 24, + ((IMM8 as u32 >> 2) & 0b11) + 24, + ((IMM8 as u32 >> 4) & 0b11) + 24, + ((IMM8 as u32 >> 6) & 0b11) + 24, + 28, + 29, + 30, + 31, + ], + ); + transmute(r) +} + +/// Shuffle 16-bit integers in the low 64 bits of 128-bit lanes of a using the control in imm8. Store the results in the low 64 bits of 128-bit lanes of dst, with the high 64 bits of 128-bit lanes being copied from from a to dst, using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shufflelo_epi16&expand=5219) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpshuflw, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_shufflelo_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let r = _mm512_shufflelo_epi16::(a); + transmute(simd_select_bitmask(k, r.as_i16x32(), src.as_i16x32())) +} + +/// Shuffle 16-bit integers in the low 64 bits of 128-bit lanes of a using the control in imm8. Store the results in the low 64 bits of 128-bit lanes of dst, with the high 64 bits of 128-bit lanes being copied from from a to dst, using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shufflelo_epi16&expand=5220) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpshuflw, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_shufflelo_epi16(k: __mmask32, a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let r = _mm512_shufflelo_epi16::(a); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, r.as_i16x32(), zero)) +} + +/// Shuffle 16-bit integers in the low 64 bits of 128-bit lanes of a using the control in imm8. Store the results in the low 64 bits of 128-bit lanes of dst, with the high 64 bits of 128-bit lanes being copied from from a to dst, using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shufflelo_epi16&expand=5216) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpshuflw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_shufflelo_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let shuffle = _mm256_shufflelo_epi16::(a); + transmute(simd_select_bitmask(k, shuffle.as_i16x16(), src.as_i16x16())) +} + +/// Shuffle 16-bit integers in the low 64 bits of 128-bit lanes of a using the control in imm8. Store the results in the low 64 bits of 128-bit lanes of dst, with the high 64 bits of 128-bit lanes being copied from from a to dst, using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shufflelo_epi16&expand=5217) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpshuflw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_shufflelo_epi16(k: __mmask16, a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let shuffle = _mm256_shufflelo_epi16::(a); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, shuffle.as_i16x16(), zero)) +} + +/// Shuffle 16-bit integers in the low 64 bits of 128-bit lanes of a using the control in imm8. Store the results in the low 64 bits of 128-bit lanes of dst, with the high 64 bits of 128-bit lanes being copied from from a to dst, using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shufflelo_epi16&expand=5213) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpshuflw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_shufflelo_epi16( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let shuffle = _mm_shufflelo_epi16::(a); + transmute(simd_select_bitmask(k, shuffle.as_i16x8(), src.as_i16x8())) +} + +/// Shuffle 16-bit integers in the low 64 bits of 128-bit lanes of a using the control in imm8. Store the results in the low 64 bits of 128-bit lanes of dst, with the high 64 bits of 128-bit lanes being copied from from a to dst, using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shufflelo_epi16&expand=5214) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpshuflw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_shufflelo_epi16(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let shuffle = _mm_shufflelo_epi16::(a); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, shuffle.as_i16x8(), zero)) +} + +/// Shuffle 16-bit integers in the high 64 bits of 128-bit lanes of a using the control in imm8. Store the results in the high 64 bits of 128-bit lanes of dst, with the low 64 bits of 128-bit lanes being copied from from a to dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shufflehi_epi16&expand=5212) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpshufhw, IMM8 = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_shufflehi_epi16(a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_i16x32(); + let r: i16x32 = simd_shuffle32!( + a, + a, + [ + 0, + 1, + 2, + 3, + (IMM8 as u32 & 0b11) + 4, + ((IMM8 as u32 >> 2) & 0b11) + 4, + ((IMM8 as u32 >> 4) & 0b11) + 4, + ((IMM8 as u32 >> 6) & 0b11) + 4, + 8, + 9, + 10, + 11, + (IMM8 as u32 & 0b11) + 12, + ((IMM8 as u32 >> 2) & 0b11) + 12, + ((IMM8 as u32 >> 4) & 0b11) + 12, + ((IMM8 as u32 >> 6) & 0b11) + 12, + 16, + 17, + 18, + 19, + (IMM8 as u32 & 0b11) + 20, + ((IMM8 as u32 >> 2) & 0b11) + 20, + ((IMM8 as u32 >> 4) & 0b11) + 20, + ((IMM8 as u32 >> 6) & 0b11) + 20, + 24, + 25, + 26, + 27, + (IMM8 as u32 & 0b11) + 28, + ((IMM8 as u32 >> 2) & 0b11) + 28, + ((IMM8 as u32 >> 4) & 0b11) + 28, + ((IMM8 as u32 >> 6) & 0b11) + 28, + ], + ); + transmute(r) +} + +/// Shuffle 16-bit integers in the high 64 bits of 128-bit lanes of a using the control in imm8. Store the results in the high 64 bits of 128-bit lanes of dst, with the low 64 bits of 128-bit lanes being copied from from a to dst, using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shufflehi_epi16&expand=5210) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpshufhw, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_shufflehi_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let r = _mm512_shufflehi_epi16::(a); + transmute(simd_select_bitmask(k, r.as_i16x32(), src.as_i16x32())) +} + +/// Shuffle 16-bit integers in the high 64 bits of 128-bit lanes of a using the control in imm8. Store the results in the high 64 bits of 128-bit lanes of dst, with the low 64 bits of 128-bit lanes being copied from from a to dst, using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shufflehi_epi16&expand=5211) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpshufhw, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_shufflehi_epi16(k: __mmask32, a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let r = _mm512_shufflehi_epi16::(a); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, r.as_i16x32(), zero)) +} + +/// Shuffle 16-bit integers in the high 64 bits of 128-bit lanes of a using the control in imm8. Store the results in the high 64 bits of 128-bit lanes of dst, with the low 64 bits of 128-bit lanes being copied from from a to dst, using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shufflehi_epi16&expand=5207) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpshufhw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_shufflehi_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let shuffle = _mm256_shufflehi_epi16::(a); + transmute(simd_select_bitmask(k, shuffle.as_i16x16(), src.as_i16x16())) +} + +/// Shuffle 16-bit integers in the high 64 bits of 128-bit lanes of a using the control in imm8. Store the results in the high 64 bits of 128-bit lanes of dst, with the low 64 bits of 128-bit lanes being copied from from a to dst, using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shufflehi_epi16&expand=5208) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpshufhw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_shufflehi_epi16(k: __mmask16, a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let shuffle = _mm256_shufflehi_epi16::(a); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, shuffle.as_i16x16(), zero)) +} + +/// Shuffle 16-bit integers in the high 64 bits of 128-bit lanes of a using the control in imm8. Store the results in the high 64 bits of 128-bit lanes of dst, with the low 64 bits of 128-bit lanes being copied from from a to dst, using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shufflehi_epi16&expand=5204) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpshufhw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_shufflehi_epi16( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let shuffle = _mm_shufflehi_epi16::(a); + transmute(simd_select_bitmask(k, shuffle.as_i16x8(), src.as_i16x8())) +} + +/// Shuffle 16-bit integers in the high 64 bits of 128-bit lanes of a using the control in imm8. Store the results in the high 64 bits of 128-bit lanes of dst, with the low 64 bits of 128-bit lanes being copied from from a to dst, using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shufflehi_epi16&expand=5205) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpshufhw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_shufflehi_epi16(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let shuffle = _mm_shufflehi_epi16::(a); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, shuffle.as_i16x8(), zero)) +} + +/// Shuffle packed 8-bit integers in a according to shuffle control mask in the corresponding 8-bit element of b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shuffle_epi8&expand=5159) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpshufb))] +pub unsafe fn _mm512_shuffle_epi8(a: __m512i, b: __m512i) -> __m512i { + transmute(vpshufb(a.as_i8x64(), b.as_i8x64())) +} + +/// Shuffle 8-bit integers in a within 128-bit lanes using the control in the corresponding 8-bit element of b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shuffle_epi8&expand=5157) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpshufb))] +pub unsafe fn _mm512_mask_shuffle_epi8( + src: __m512i, + k: __mmask64, + a: __m512i, + b: __m512i, +) -> __m512i { + let shuffle = _mm512_shuffle_epi8(a, b).as_i8x64(); + transmute(simd_select_bitmask(k, shuffle, src.as_i8x64())) +} + +/// Shuffle packed 8-bit integers in a according to shuffle control mask in the corresponding 8-bit element of b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shuffle_epi8&expand=5158) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpshufb))] +pub unsafe fn _mm512_maskz_shuffle_epi8(k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + let shuffle = _mm512_shuffle_epi8(a, b).as_i8x64(); + let zero = _mm512_setzero_si512().as_i8x64(); + transmute(simd_select_bitmask(k, shuffle, zero)) +} + +/// Shuffle 8-bit integers in a within 128-bit lanes using the control in the corresponding 8-bit element of b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shuffle_epi8&expand=5154) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpshufb))] +pub unsafe fn _mm256_mask_shuffle_epi8( + src: __m256i, + k: __mmask32, + a: __m256i, + b: __m256i, +) -> __m256i { + let shuffle = _mm256_shuffle_epi8(a, b).as_i8x32(); + transmute(simd_select_bitmask(k, shuffle, src.as_i8x32())) +} + +/// Shuffle packed 8-bit integers in a according to shuffle control mask in the corresponding 8-bit element of b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shuffle_epi8&expand=5155) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpshufb))] +pub unsafe fn _mm256_maskz_shuffle_epi8(k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + let shuffle = _mm256_shuffle_epi8(a, b).as_i8x32(); + let zero = _mm256_setzero_si256().as_i8x32(); + transmute(simd_select_bitmask(k, shuffle, zero)) +} + +/// Shuffle 8-bit integers in a within 128-bit lanes using the control in the corresponding 8-bit element of b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shuffle_epi8&expand=5151) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpshufb))] +pub unsafe fn _mm_mask_shuffle_epi8(src: __m128i, k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let shuffle = _mm_shuffle_epi8(a, b).as_i8x16(); + transmute(simd_select_bitmask(k, shuffle, src.as_i8x16())) +} + +/// Shuffle packed 8-bit integers in a according to shuffle control mask in the corresponding 8-bit element of b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shuffle_epi8&expand=5152) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpshufb))] +pub unsafe fn _mm_maskz_shuffle_epi8(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let shuffle = _mm_shuffle_epi8(a, b).as_i8x16(); + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask(k, shuffle, zero)) +} + +/// Compute the bitwise AND of packed 16-bit integers in a and b, producing intermediate 16-bit values, and set the corresponding bit in result mask k if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_test_epi16_mask&expand=5884) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vptestmw))] +pub unsafe fn _mm512_test_epi16_mask(a: __m512i, b: __m512i) -> __mmask32 { + let and = _mm512_and_si512(a, b); + let zero = _mm512_setzero_si512(); + _mm512_cmpneq_epi16_mask(and, zero) +} + +/// Compute the bitwise AND of packed 16-bit integers in a and b, producing intermediate 16-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_test_epi16_mask&expand=5883) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vptestmw))] +pub unsafe fn _mm512_mask_test_epi16_mask(k: __mmask32, a: __m512i, b: __m512i) -> __mmask32 { + let and = _mm512_and_si512(a, b); + let zero = _mm512_setzero_si512(); + _mm512_mask_cmpneq_epi16_mask(k, and, zero) +} + +/// Compute the bitwise AND of packed 16-bit integers in a and b, producing intermediate 16-bit values, and set the corresponding bit in result mask k if the intermediate value is non-zero. +/// +// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_test_epi16_mask&expand=5882) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vptestmw))] +pub unsafe fn _mm256_test_epi16_mask(a: __m256i, b: __m256i) -> __mmask16 { + let and = _mm256_and_si256(a, b); + let zero = _mm256_setzero_si256(); + _mm256_cmpneq_epi16_mask(and, zero) +} + +/// Compute the bitwise AND of packed 16-bit integers in a and b, producing intermediate 16-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_test_epi16_mask&expand=5881) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vptestmw))] +pub unsafe fn _mm256_mask_test_epi16_mask(k: __mmask16, a: __m256i, b: __m256i) -> __mmask16 { + let and = _mm256_and_si256(a, b); + let zero = _mm256_setzero_si256(); + _mm256_mask_cmpneq_epi16_mask(k, and, zero) +} + +/// Compute the bitwise AND of packed 16-bit integers in a and b, producing intermediate 16-bit values, and set the corresponding bit in result mask k if the intermediate value is non-zero. +/// +// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_test_epi16_mask&expand=5880) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vptestmw))] +pub unsafe fn _mm_test_epi16_mask(a: __m128i, b: __m128i) -> __mmask8 { + let and = _mm_and_si128(a, b); + let zero = _mm_setzero_si128(); + _mm_cmpneq_epi16_mask(and, zero) +} + +/// Compute the bitwise AND of packed 16-bit integers in a and b, producing intermediate 16-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_test_epi16_mask&expand=5879) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vptestmw))] +pub unsafe fn _mm_mask_test_epi16_mask(k: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + let and = _mm_and_si128(a, b); + let zero = _mm_setzero_si128(); + _mm_mask_cmpneq_epi16_mask(k, and, zero) +} + +/// Compute the bitwise AND of packed 8-bit integers in a and b, producing intermediate 8-bit values, and set the corresponding bit in result mask k if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_test_epi8_mask&expand=5902) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vptestmb))] +pub unsafe fn _mm512_test_epi8_mask(a: __m512i, b: __m512i) -> __mmask64 { + let and = _mm512_and_si512(a, b); + let zero = _mm512_setzero_si512(); + _mm512_cmpneq_epi8_mask(and, zero) +} + +/// Compute the bitwise AND of packed 8-bit integers in a and b, producing intermediate 8-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_test_epi8_mask&expand=5901) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vptestmb))] +pub unsafe fn _mm512_mask_test_epi8_mask(k: __mmask64, a: __m512i, b: __m512i) -> __mmask64 { + let and = _mm512_and_si512(a, b); + let zero = _mm512_setzero_si512(); + _mm512_mask_cmpneq_epi8_mask(k, and, zero) +} + +/// Compute the bitwise AND of packed 8-bit integers in a and b, producing intermediate 8-bit values, and set the corresponding bit in result mask k if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_test_epi8_mask&expand=5900) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vptestmb))] +pub unsafe fn _mm256_test_epi8_mask(a: __m256i, b: __m256i) -> __mmask32 { + let and = _mm256_and_si256(a, b); + let zero = _mm256_setzero_si256(); + _mm256_cmpneq_epi8_mask(and, zero) +} + +/// Compute the bitwise AND of packed 8-bit integers in a and b, producing intermediate 8-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_test_epi8_mask&expand=5899) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vptestmb))] +pub unsafe fn _mm256_mask_test_epi8_mask(k: __mmask32, a: __m256i, b: __m256i) -> __mmask32 { + let and = _mm256_and_si256(a, b); + let zero = _mm256_setzero_si256(); + _mm256_mask_cmpneq_epi8_mask(k, and, zero) +} + +/// Compute the bitwise AND of packed 8-bit integers in a and b, producing intermediate 8-bit values, and set the corresponding bit in result mask k if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_test_epi8_mask&expand=5898) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vptestmb))] +pub unsafe fn _mm_test_epi8_mask(a: __m128i, b: __m128i) -> __mmask16 { + let and = _mm_and_si128(a, b); + let zero = _mm_setzero_si128(); + _mm_cmpneq_epi8_mask(and, zero) +} + +/// Compute the bitwise AND of packed 8-bit integers in a and b, producing intermediate 8-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_test_epi8_mask&expand=5897) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vptestmb))] +pub unsafe fn _mm_mask_test_epi8_mask(k: __mmask16, a: __m128i, b: __m128i) -> __mmask16 { + let and = _mm_and_si128(a, b); + let zero = _mm_setzero_si128(); + _mm_mask_cmpneq_epi8_mask(k, and, zero) +} + +/// Compute the bitwise NAND of packed 16-bit integers in a and b, producing intermediate 16-bit values, and set the corresponding bit in result mask k if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_testn_epi16_mask&expand=5915) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vptestnmw))] +pub unsafe fn _mm512_testn_epi16_mask(a: __m512i, b: __m512i) -> __mmask32 { + let and = _mm512_and_si512(a, b); + let zero = _mm512_setzero_si512(); + _mm512_cmpeq_epi16_mask(and, zero) +} + +/// Compute the bitwise NAND of packed 16-bit integers in a and b, producing intermediate 16-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_testn_epi16&expand=5914) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vptestnmw))] +pub unsafe fn _mm512_mask_testn_epi16_mask(k: __mmask32, a: __m512i, b: __m512i) -> __mmask32 { + let and = _mm512_and_si512(a, b); + let zero = _mm512_setzero_si512(); + _mm512_mask_cmpeq_epi16_mask(k, and, zero) +} + +/// Compute the bitwise NAND of packed 16-bit integers in a and b, producing intermediate 16-bit values, and set the corresponding bit in result mask k if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_testn_epi16_mask&expand=5913) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vptestnmw))] +pub unsafe fn _mm256_testn_epi16_mask(a: __m256i, b: __m256i) -> __mmask16 { + let and = _mm256_and_si256(a, b); + let zero = _mm256_setzero_si256(); + _mm256_cmpeq_epi16_mask(and, zero) +} + +/// Compute the bitwise NAND of packed 16-bit integers in a and b, producing intermediate 16-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_testn_epi16_mask&expand=5912) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vptestnmw))] +pub unsafe fn _mm256_mask_testn_epi16_mask(k: __mmask16, a: __m256i, b: __m256i) -> __mmask16 { + let and = _mm256_and_si256(a, b); + let zero = _mm256_setzero_si256(); + _mm256_mask_cmpeq_epi16_mask(k, and, zero) +} + +/// Compute the bitwise NAND of packed 16-bit integers in a and b, producing intermediate 16-bit values, and set the corresponding bit in result mask k if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_testn_epi16_mask&expand=5911) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vptestnmw))] +pub unsafe fn _mm_testn_epi16_mask(a: __m128i, b: __m128i) -> __mmask8 { + let and = _mm_and_si128(a, b); + let zero = _mm_setzero_si128(); + _mm_cmpeq_epi16_mask(and, zero) +} + +/// Compute the bitwise NAND of packed 16-bit integers in a and b, producing intermediate 16-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_testn_epi16_mask&expand=5910) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vptestnmw))] +pub unsafe fn _mm_mask_testn_epi16_mask(k: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + let and = _mm_and_si128(a, b); + let zero = _mm_setzero_si128(); + _mm_mask_cmpeq_epi16_mask(k, and, zero) +} + +/// Compute the bitwise NAND of packed 8-bit integers in a and b, producing intermediate 8-bit values, and set the corresponding bit in result mask k if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_testn_epi8_mask&expand=5933) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vptestnmb))] +pub unsafe fn _mm512_testn_epi8_mask(a: __m512i, b: __m512i) -> __mmask64 { + let and = _mm512_and_si512(a, b); + let zero = _mm512_setzero_si512(); + _mm512_cmpeq_epi8_mask(and, zero) +} + +/// Compute the bitwise NAND of packed 8-bit integers in a and b, producing intermediate 8-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_testn_epi8_mask&expand=5932) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vptestnmb))] +pub unsafe fn _mm512_mask_testn_epi8_mask(k: __mmask64, a: __m512i, b: __m512i) -> __mmask64 { + let and = _mm512_and_si512(a, b); + let zero = _mm512_setzero_si512(); + _mm512_mask_cmpeq_epi8_mask(k, and, zero) +} + +/// Compute the bitwise NAND of packed 8-bit integers in a and b, producing intermediate 8-bit values, and set the corresponding bit in result mask k if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_testn_epi8_mask&expand=5931) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vptestnmb))] +pub unsafe fn _mm256_testn_epi8_mask(a: __m256i, b: __m256i) -> __mmask32 { + let and = _mm256_and_si256(a, b); + let zero = _mm256_setzero_si256(); + _mm256_cmpeq_epi8_mask(and, zero) +} + +/// Compute the bitwise NAND of packed 8-bit integers in a and b, producing intermediate 8-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_testn_epi8_mask&expand=5930) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vptestnmb))] +pub unsafe fn _mm256_mask_testn_epi8_mask(k: __mmask32, a: __m256i, b: __m256i) -> __mmask32 { + let and = _mm256_and_si256(a, b); + let zero = _mm256_setzero_si256(); + _mm256_mask_cmpeq_epi8_mask(k, and, zero) +} + +/// Compute the bitwise NAND of packed 8-bit integers in a and b, producing intermediate 8-bit values, and set the corresponding bit in result mask k if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_testn_epi8_mask&expand=5929) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vptestnmb))] +pub unsafe fn _mm_testn_epi8_mask(a: __m128i, b: __m128i) -> __mmask16 { + let and = _mm_and_si128(a, b); + let zero = _mm_setzero_si128(); + _mm_cmpeq_epi8_mask(and, zero) +} + +/// Compute the bitwise NAND of packed 8-bit integers in a and b, producing intermediate 8-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_testn_epi8_mask&expand=5928) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vptestnmb))] +pub unsafe fn _mm_mask_testn_epi8_mask(k: __mmask16, a: __m128i, b: __m128i) -> __mmask16 { + let and = _mm_and_si128(a, b); + let zero = _mm_setzero_si128(); + _mm_mask_cmpeq_epi8_mask(k, and, zero) +} + +/// Store 64-bit mask from a into memory. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_store_mask64&expand=5578) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(mov))] //should be kmovq +pub unsafe fn _store_mask64(mem_addr: *mut u64, a: __mmask64) { + ptr::write(mem_addr as *mut __mmask64, a); +} + +/// Store 32-bit mask from a into memory. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_store_mask32&expand=5577) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(mov))] //should be kmovd +pub unsafe fn _store_mask32(mem_addr: *mut u32, a: __mmask32) { + ptr::write(mem_addr as *mut __mmask32, a); +} + +/// Load 64-bit mask from memory into k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_load_mask64&expand=3318) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(mov))] //should be kmovq +pub unsafe fn _load_mask64(mem_addr: *const u64) -> __mmask64 { + ptr::read(mem_addr as *const __mmask64) +} + +/// Load 32-bit mask from memory into k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_load_mask32&expand=3317) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(mov))] //should be kmovd +pub unsafe fn _load_mask32(mem_addr: *const u32) -> __mmask32 { + ptr::read(mem_addr as *const __mmask32) +} + +/// Compute the absolute differences of packed unsigned 8-bit integers in a and b, then horizontally sum each consecutive 8 differences to produce eight unsigned 16-bit integers, and pack these unsigned 16-bit integers in the low 16 bits of 64-bit elements in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sad_epu8&expand=4855) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsadbw))] +pub unsafe fn _mm512_sad_epu8(a: __m512i, b: __m512i) -> __m512i { + transmute(vpsadbw(a.as_u8x64(), b.as_u8x64())) +} + +/// Compute the sum of absolute differences (SADs) of quadruplets of unsigned 8-bit integers in a compared to those in b, and store the 16-bit results in dst. Four SADs are performed on four 8-bit quadruplets for each 64-bit lane. The first two SADs use the lower 8-bit quadruplet of the lane from a, and the last two SADs use the uppper 8-bit quadruplet of the lane from a. Quadruplets from b are selected from within 128-bit lanes according to the control in imm8, and each SAD in each 64-bit lane uses the selected quadruplet at 8-bit offsets. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_dbsad_epu8&expand=2114) +#[inline] +#[target_feature(enable = "avx512bw")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vdbpsadbw, IMM8 = 0))] +pub unsafe fn _mm512_dbsad_epu8(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_u8x64(); + let b = b.as_u8x64(); + let r = vdbpsadbw(a, b, IMM8); + transmute(r) +} + +/// Compute the sum of absolute differences (SADs) of quadruplets of unsigned 8-bit integers in a compared to those in b, and store the 16-bit results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). Four SADs are performed on four 8-bit quadruplets for each 64-bit lane. The first two SADs use the lower 8-bit quadruplet of the lane from a, and the last two SADs use the uppper 8-bit quadruplet of the lane from a. Quadruplets from b are selected from within 128-bit lanes according to the control in imm8, and each SAD in each 64-bit lane uses the selected quadruplet at 8-bit offsets. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_dbsad_epu8&expand=2115) +#[inline] +#[target_feature(enable = "avx512bw")] +#[rustc_legacy_const_generics(4)] +#[cfg_attr(test, assert_instr(vdbpsadbw, IMM8 = 0))] +pub unsafe fn _mm512_mask_dbsad_epu8( + src: __m512i, + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_u8x64(); + let b = b.as_u8x64(); + let r = vdbpsadbw(a, b, IMM8); + transmute(simd_select_bitmask(k, r, src.as_u16x32())) +} + +/// Compute the sum of absolute differences (SADs) of quadruplets of unsigned 8-bit integers in a compared to those in b, and store the 16-bit results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). Four SADs are performed on four 8-bit quadruplets for each 64-bit lane. The first two SADs use the lower 8-bit quadruplet of the lane from a, and the last two SADs use the uppper 8-bit quadruplet of the lane from a. Quadruplets from b are selected from within 128-bit lanes according to the control in imm8, and each SAD in each 64-bit lane uses the selected quadruplet at 8-bit offsets. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_dbsad_epu8&expand=2116) +#[inline] +#[target_feature(enable = "avx512bw")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vdbpsadbw, IMM8 = 0))] +pub unsafe fn _mm512_maskz_dbsad_epu8( + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_u8x64(); + let b = b.as_u8x64(); + let r = vdbpsadbw(a, b, IMM8); + transmute(simd_select_bitmask( + k, + r, + _mm512_setzero_si512().as_u16x32(), + )) +} + +/// Compute the sum of absolute differences (SADs) of quadruplets of unsigned 8-bit integers in a compared to those in b, and store the 16-bit results in dst. Four SADs are performed on four 8-bit quadruplets for each 64-bit lane. The first two SADs use the lower 8-bit quadruplet of the lane from a, and the last two SADs use the uppper 8-bit quadruplet of the lane from a. Quadruplets from b are selected from within 128-bit lanes according to the control in imm8, and each SAD in each 64-bit lane uses the selected quadruplet at 8-bit offsets. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_dbsad_epu8&expand=2111) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vdbpsadbw, IMM8 = 0))] +pub unsafe fn _mm256_dbsad_epu8(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_u8x32(); + let b = b.as_u8x32(); + let r = vdbpsadbw256(a, b, IMM8); + transmute(r) +} + +/// Compute the sum of absolute differences (SADs) of quadruplets of unsigned 8-bit integers in a compared to those in b, and store the 16-bit results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). Four SADs are performed on four 8-bit quadruplets for each 64-bit lane. The first two SADs use the lower 8-bit quadruplet of the lane from a, and the last two SADs use the uppper 8-bit quadruplet of the lane from a. Quadruplets from b are selected from within 128-bit lanes according to the control in imm8, and each SAD in each 64-bit lane uses the selected quadruplet at 8-bit offsets. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_dbsad_epu8&expand=2112) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(4)] +#[cfg_attr(test, assert_instr(vdbpsadbw, IMM8 = 0))] +pub unsafe fn _mm256_mask_dbsad_epu8( + src: __m256i, + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_u8x32(); + let b = b.as_u8x32(); + let r = vdbpsadbw256(a, b, IMM8); + transmute(simd_select_bitmask(k, r, src.as_u16x16())) +} + +/// Compute the sum of absolute differences (SADs) of quadruplets of unsigned 8-bit integers in a compared to those in b, and store the 16-bit results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). Four SADs are performed on four 8-bit quadruplets for each 64-bit lane. The first two SADs use the lower 8-bit quadruplet of the lane from a, and the last two SADs use the uppper 8-bit quadruplet of the lane from a. Quadruplets from b are selected from within 128-bit lanes according to the control in imm8, and each SAD in each 64-bit lane uses the selected quadruplet at 8-bit offsets. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_dbsad_epu8&expand=2113) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vdbpsadbw, IMM8 = 0))] +pub unsafe fn _mm256_maskz_dbsad_epu8( + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_u8x32(); + let b = b.as_u8x32(); + let r = vdbpsadbw256(a, b, IMM8); + transmute(simd_select_bitmask( + k, + r, + _mm256_setzero_si256().as_u16x16(), + )) +} + +/// Compute the sum of absolute differences (SADs) of quadruplets of unsigned 8-bit integers in a compared to those in b, and store the 16-bit results in dst. Four SADs are performed on four 8-bit quadruplets for each 64-bit lane. The first two SADs use the lower 8-bit quadruplet of the lane from a, and the last two SADs use the uppper 8-bit quadruplet of the lane from a. Quadruplets from b are selected from within 128-bit lanes according to the control in imm8, and each SAD in each 64-bit lane uses the selected quadruplet at 8-bit offsets. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_dbsad_epu8&expand=2108) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vdbpsadbw, IMM8 = 0))] +pub unsafe fn _mm_dbsad_epu8(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_u8x16(); + let b = b.as_u8x16(); + let r = vdbpsadbw128(a, b, IMM8); + transmute(r) +} + +/// Compute the sum of absolute differences (SADs) of quadruplets of unsigned 8-bit integers in a compared to those in b, and store the 16-bit results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). Four SADs are performed on four 8-bit quadruplets for each 64-bit lane. The first two SADs use the lower 8-bit quadruplet of the lane from a, and the last two SADs use the uppper 8-bit quadruplet of the lane from a. Quadruplets from b are selected from within 128-bit lanes according to the control in imm8, and each SAD in each 64-bit lane uses the selected quadruplet at 8-bit offsets. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_dbsad_epu8&expand=2109) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(4)] +#[cfg_attr(test, assert_instr(vdbpsadbw, IMM8 = 0))] +pub unsafe fn _mm_mask_dbsad_epu8( + src: __m128i, + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_u8x16(); + let b = b.as_u8x16(); + let r = vdbpsadbw128(a, b, IMM8); + transmute(simd_select_bitmask(k, r, src.as_u16x8())) +} + +/// Compute the sum of absolute differences (SADs) of quadruplets of unsigned 8-bit integers in a compared to those in b, and store the 16-bit results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). Four SADs are performed on four 8-bit quadruplets for each 64-bit lane. The first two SADs use the lower 8-bit quadruplet of the lane from a, and the last two SADs use the uppper 8-bit quadruplet of the lane from a. Quadruplets from b are selected from within 128-bit lanes according to the control in imm8, and each SAD in each 64-bit lane uses the selected quadruplet at 8-bit offsets. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_dbsad_epu8&expand=2110) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vdbpsadbw, IMM8 = 0))] +pub unsafe fn _mm_maskz_dbsad_epu8( + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_u8x16(); + let b = b.as_u8x16(); + let r = vdbpsadbw128(a, b, IMM8); + transmute(simd_select_bitmask(k, r, _mm_setzero_si128().as_u16x8())) +} + +/// Set each bit of mask register k based on the most significant bit of the corresponding packed 16-bit integer in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_movepi16_mask&expand=3873) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovw2m))] +pub unsafe fn _mm512_movepi16_mask(a: __m512i) -> __mmask32 { + let filter = _mm512_set1_epi16(1 << 15); + let a = _mm512_and_si512(a, filter); + _mm512_cmpeq_epi16_mask(a, filter) +} + +/// Set each bit of mask register k based on the most significant bit of the corresponding packed 16-bit integer in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_movepi16_mask&expand=3872) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovw2m))] +pub unsafe fn _mm256_movepi16_mask(a: __m256i) -> __mmask16 { + let filter = _mm256_set1_epi16(1 << 15); + let a = _mm256_and_si256(a, filter); + _mm256_cmpeq_epi16_mask(a, filter) +} + +/// Set each bit of mask register k based on the most significant bit of the corresponding packed 16-bit integer in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_movepi16_mask&expand=3871) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovw2m))] +pub unsafe fn _mm_movepi16_mask(a: __m128i) -> __mmask8 { + let filter = _mm_set1_epi16(1 << 15); + let a = _mm_and_si128(a, filter); + _mm_cmpeq_epi16_mask(a, filter) +} + +/// Set each bit of mask register k based on the most significant bit of the corresponding packed 8-bit integer in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_movepi8_mask&expand=3883) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovb2m))] +pub unsafe fn _mm512_movepi8_mask(a: __m512i) -> __mmask64 { + let filter = _mm512_set1_epi8(1 << 7); + let a = _mm512_and_si512(a, filter); + _mm512_cmpeq_epi8_mask(a, filter) +} + +/// Set each bit of mask register k based on the most significant bit of the corresponding packed 8-bit integer in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_movepi8_mask&expand=3882) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovmskb))] // should be vpmovb2m but compiled to vpmovmskb in the test shim because that takes less cycles than + // using vpmovb2m plus converting the mask register to a standard register. +pub unsafe fn _mm256_movepi8_mask(a: __m256i) -> __mmask32 { + let filter = _mm256_set1_epi8(1 << 7); + let a = _mm256_and_si256(a, filter); + _mm256_cmpeq_epi8_mask(a, filter) +} + +/// Set each bit of mask register k based on the most significant bit of the corresponding packed 8-bit integer in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_movepi8_mask&expand=3881) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovmskb))] // should be vpmovb2m but compiled to vpmovmskb in the test shim because that takes less cycles than + // using vpmovb2m plus converting the mask register to a standard register. +pub unsafe fn _mm_movepi8_mask(a: __m128i) -> __mmask16 { + let filter = _mm_set1_epi8(1 << 7); + let a = _mm_and_si128(a, filter); + _mm_cmpeq_epi8_mask(a, filter) +} + +/// Set each packed 16-bit integer in dst to all ones or all zeros based on the value of the corresponding bit in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_movm_epi16&expand=3886) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovm2w))] +pub unsafe fn _mm512_movm_epi16(k: __mmask32) -> __m512i { + let one = _mm512_set1_epi16( + 1 << 15 + | 1 << 14 + | 1 << 13 + | 1 << 12 + | 1 << 11 + | 1 << 10 + | 1 << 9 + | 1 << 8 + | 1 << 7 + | 1 << 6 + | 1 << 5 + | 1 << 4 + | 1 << 3 + | 1 << 2 + | 1 << 1 + | 1 << 0, + ) + .as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, one, zero)) +} + +/// Set each packed 16-bit integer in dst to all ones or all zeros based on the value of the corresponding bit in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_movm_epi16&expand=3885) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovm2w))] +pub unsafe fn _mm256_movm_epi16(k: __mmask16) -> __m256i { + let one = _mm256_set1_epi16( + 1 << 15 + | 1 << 14 + | 1 << 13 + | 1 << 12 + | 1 << 11 + | 1 << 10 + | 1 << 9 + | 1 << 8 + | 1 << 7 + | 1 << 6 + | 1 << 5 + | 1 << 4 + | 1 << 3 + | 1 << 2 + | 1 << 1 + | 1 << 0, + ) + .as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, one, zero)) +} + +/// Set each packed 16-bit integer in dst to all ones or all zeros based on the value of the corresponding bit in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_movm_epi16&expand=3884) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovm2w))] +pub unsafe fn _mm_movm_epi16(k: __mmask8) -> __m128i { + let one = _mm_set1_epi16( + 1 << 15 + | 1 << 14 + | 1 << 13 + | 1 << 12 + | 1 << 11 + | 1 << 10 + | 1 << 9 + | 1 << 8 + | 1 << 7 + | 1 << 6 + | 1 << 5 + | 1 << 4 + | 1 << 3 + | 1 << 2 + | 1 << 1 + | 1 << 0, + ) + .as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, one, zero)) +} + +/// Set each packed 8-bit integer in dst to all ones or all zeros based on the value of the corresponding bit in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_movm_epi8&expand=3895) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovm2b))] +pub unsafe fn _mm512_movm_epi8(k: __mmask64) -> __m512i { + let one = + _mm512_set1_epi8(1 << 7 | 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0) + .as_i8x64(); + let zero = _mm512_setzero_si512().as_i8x64(); + transmute(simd_select_bitmask(k, one, zero)) +} + +/// Set each packed 8-bit integer in dst to all ones or all zeros based on the value of the corresponding bit in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_movm_epi8&expand=3894) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovm2b))] +pub unsafe fn _mm256_movm_epi8(k: __mmask32) -> __m256i { + let one = + _mm256_set1_epi8(1 << 7 | 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0) + .as_i8x32(); + let zero = _mm256_setzero_si256().as_i8x32(); + transmute(simd_select_bitmask(k, one, zero)) +} + +/// Set each packed 8-bit integer in dst to all ones or all zeros based on the value of the corresponding bit in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_movm_epi8&expand=3893) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovm2b))] +pub unsafe fn _mm_movm_epi8(k: __mmask16) -> __m128i { + let one = _mm_set1_epi8(1 << 7 | 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0) + .as_i8x16(); + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask(k, one, zero)) +} + +/// Add 32-bit masks in a and b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_kadd_mask32&expand=3207) +#[inline] +#[target_feature(enable = "avx512bw")] +pub unsafe fn _kadd_mask32(a: __mmask32, b: __mmask32) -> __mmask32 { + transmute(a + b) +} + +/// Add 64-bit masks in a and b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_kadd_mask64&expand=3208) +#[inline] +#[target_feature(enable = "avx512bw")] +pub unsafe fn _kadd_mask64(a: __mmask64, b: __mmask64) -> __mmask64 { + transmute(a + b) +} + +/// Compute the bitwise AND of 32-bit masks a and b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_kand_mask32&expand=3213) +#[inline] +#[target_feature(enable = "avx512bw")] +pub unsafe fn _kand_mask32(a: __mmask32, b: __mmask32) -> __mmask32 { + transmute(a & b) +} + +/// Compute the bitwise AND of 64-bit masks a and b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_kand_mask64&expand=3214) +#[inline] +#[target_feature(enable = "avx512bw")] +pub unsafe fn _kand_mask64(a: __mmask64, b: __mmask64) -> __mmask64 { + transmute(a & b) +} + +/// Compute the bitwise NOT of 32-bit mask a, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_knot_mask32&expand=3234) +#[inline] +#[target_feature(enable = "avx512bw")] +pub unsafe fn _knot_mask32(a: __mmask32) -> __mmask32 { + transmute(a ^ 0b11111111_11111111_11111111_11111111) +} + +/// Compute the bitwise NOT of 64-bit mask a, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_knot_mask64&expand=3235) +#[inline] +#[target_feature(enable = "avx512bw")] +pub unsafe fn _knot_mask64(a: __mmask64) -> __mmask64 { + transmute(a ^ 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111) +} + +/// Compute the bitwise NOT of 32-bit masks a and then AND with b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_kandn_mask32&expand=3219) +#[inline] +#[target_feature(enable = "avx512bw")] +pub unsafe fn _kandn_mask32(a: __mmask32, b: __mmask32) -> __mmask32 { + transmute(_knot_mask32(a) & b) +} + +/// Compute the bitwise NOT of 64-bit masks a and then AND with b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_kandn_mask64&expand=3220) +#[inline] +#[target_feature(enable = "avx512bw")] +pub unsafe fn _kandn_mask64(a: __mmask64, b: __mmask64) -> __mmask64 { + transmute(_knot_mask64(a) & b) +} + +/// Compute the bitwise OR of 32-bit masks a and b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_kor_mask32&expand=3240) +#[inline] +#[target_feature(enable = "avx512bw")] +pub unsafe fn _kor_mask32(a: __mmask32, b: __mmask32) -> __mmask32 { + transmute(a | b) +} + +/// Compute the bitwise OR of 64-bit masks a and b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_kor_mask64&expand=3241) +#[inline] +#[target_feature(enable = "avx512bw")] +pub unsafe fn _kor_mask64(a: __mmask64, b: __mmask64) -> __mmask64 { + transmute(a | b) +} + +/// Compute the bitwise XOR of 32-bit masks a and b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_kxor_mask32&expand=3292) +#[inline] +#[target_feature(enable = "avx512bw")] +pub unsafe fn _kxor_mask32(a: __mmask32, b: __mmask32) -> __mmask32 { + transmute(a ^ b) +} + +/// Compute the bitwise XOR of 64-bit masks a and b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_kxor_mask64&expand=3293) +#[inline] +#[target_feature(enable = "avx512bw")] +pub unsafe fn _kxor_mask64(a: __mmask64, b: __mmask64) -> __mmask64 { + transmute(a ^ b) +} + +/// Compute the bitwise XNOR of 32-bit masks a and b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_kxnor_mask32&expand=3286) +#[inline] +#[target_feature(enable = "avx512bw")] +pub unsafe fn _kxnor_mask32(a: __mmask32, b: __mmask32) -> __mmask32 { + transmute(_knot_mask32(a ^ b)) +} + +/// Compute the bitwise XNOR of 64-bit masks a and b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_kxnor_mask64&expand=3287) +#[inline] +#[target_feature(enable = "avx512bw")] +pub unsafe fn _kxnor_mask64(a: __mmask64, b: __mmask64) -> __mmask64 { + transmute(_knot_mask64(a ^ b)) +} + +/// Convert packed 16-bit integers in a to packed 8-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepi16_epi8&expand=1407) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovwb))] +pub unsafe fn _mm512_cvtepi16_epi8(a: __m512i) -> __m256i { + let a = a.as_i16x32(); + transmute::(simd_cast(a)) +} + +/// Convert packed 16-bit integers in a to packed 8-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepi16_epi8&expand=1408) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovwb))] +pub unsafe fn _mm512_mask_cvtepi16_epi8(src: __m256i, k: __mmask32, a: __m512i) -> __m256i { + let convert = _mm512_cvtepi16_epi8(a).as_i8x32(); + transmute(simd_select_bitmask(k, convert, src.as_i8x32())) +} + +/// Convert packed 16-bit integers in a to packed 8-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepi16_epi8&expand=1409) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovwb))] +pub unsafe fn _mm512_maskz_cvtepi16_epi8(k: __mmask32, a: __m512i) -> __m256i { + let convert = _mm512_cvtepi16_epi8(a).as_i8x32(); + transmute(simd_select_bitmask( + k, + convert, + _mm256_setzero_si256().as_i8x32(), + )) +} + +/// Convert packed 16-bit integers in a to packed 8-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvtepi16_epi8&expand=1404) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovwb))] +pub unsafe fn _mm256_cvtepi16_epi8(a: __m256i) -> __m128i { + let a = a.as_i16x16(); + transmute::(simd_cast(a)) +} + +/// Convert packed 16-bit integers in a to packed 8-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepi16_epi8&expand=1405) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovwb))] +pub unsafe fn _mm256_mask_cvtepi16_epi8(src: __m128i, k: __mmask16, a: __m256i) -> __m128i { + let convert = _mm256_cvtepi16_epi8(a).as_i8x16(); + transmute(simd_select_bitmask(k, convert, src.as_i8x16())) +} + +/// Convert packed 16-bit integers in a to packed 8-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtepi16_epi8&expand=1406) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovwb))] +pub unsafe fn _mm256_maskz_cvtepi16_epi8(k: __mmask16, a: __m256i) -> __m128i { + let convert = _mm256_cvtepi16_epi8(a).as_i8x16(); + transmute(simd_select_bitmask( + k, + convert, + _mm_setzero_si128().as_i8x16(), + )) +} + +/// Convert packed 16-bit integers in a to packed 8-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtepi16_epi8&expand=1401) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovwb))] +pub unsafe fn _mm_cvtepi16_epi8(a: __m128i) -> __m128i { + let a = a.as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + let v256: i16x16 = simd_shuffle16!(a, zero, [0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 8, 8, 8, 8, 8, 8]); + transmute::(simd_cast(v256)) +} + +/// Convert packed 16-bit integers in a to packed 8-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepi16_epi8&expand=1402) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovwb))] +pub unsafe fn _mm_mask_cvtepi16_epi8(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepi16_epi8(a).as_i8x16(); + let k: __mmask16 = 0b11111111_11111111 & k as __mmask16; + transmute(simd_select_bitmask(k, convert, src.as_i8x16())) +} + +/// Convert packed 16-bit integers in a to packed 8-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtepi16_epi8&expand=1403) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovwb))] +pub unsafe fn _mm_maskz_cvtepi16_epi8(k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepi16_epi8(a).as_i8x16(); + let k: __mmask16 = 0b11111111_11111111 & k as __mmask16; + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Convert packed signed 16-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtsepi16_epi8&expand=1807) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovswb))] +pub unsafe fn _mm512_cvtsepi16_epi8(a: __m512i) -> __m256i { + transmute(vpmovswb( + a.as_i16x32(), + _mm256_setzero_si256().as_i8x32(), + 0b11111111_11111111_11111111_11111111, + )) +} + +/// Convert packed signed 16-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtsepi16_epi8&expand=1808) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovswb))] +pub unsafe fn _mm512_mask_cvtsepi16_epi8(src: __m256i, k: __mmask32, a: __m512i) -> __m256i { + transmute(vpmovswb(a.as_i16x32(), src.as_i8x32(), k)) +} + +/// Convert packed signed 16-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtsepi16_epi8&expand=1809) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovswb))] +pub unsafe fn _mm512_maskz_cvtsepi16_epi8(k: __mmask32, a: __m512i) -> __m256i { + transmute(vpmovswb( + a.as_i16x32(), + _mm256_setzero_si256().as_i8x32(), + k, + )) +} + +/// Convert packed signed 16-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvtsepi16_epi8&expand=1804) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovswb))] +pub unsafe fn _mm256_cvtsepi16_epi8(a: __m256i) -> __m128i { + transmute(vpmovswb256( + a.as_i16x16(), + _mm_setzero_si128().as_i8x16(), + 0b11111111_11111111, + )) +} + +/// Convert packed signed 16-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtsepi16_epi8&expand=1805) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovswb))] +pub unsafe fn _mm256_mask_cvtsepi16_epi8(src: __m128i, k: __mmask16, a: __m256i) -> __m128i { + transmute(vpmovswb256(a.as_i16x16(), src.as_i8x16(), k)) +} + +/// Convert packed signed 16-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtsepi16_epi8&expand=1806) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovswb))] +pub unsafe fn _mm256_maskz_cvtsepi16_epi8(k: __mmask16, a: __m256i) -> __m128i { + transmute(vpmovswb256( + a.as_i16x16(), + _mm_setzero_si128().as_i8x16(), + k, + )) +} + +/// Convert packed signed 16-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtsepi16_epi8&expand=1801) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovswb))] +pub unsafe fn _mm_cvtsepi16_epi8(a: __m128i) -> __m128i { + transmute(vpmovswb128( + a.as_i16x8(), + _mm_setzero_si128().as_i8x16(), + 0b11111111, + )) +} + +/// Convert packed signed 16-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtsepi16_epi8&expand=1802) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovswb))] +pub unsafe fn _mm_mask_cvtsepi16_epi8(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovswb128(a.as_i16x8(), src.as_i8x16(), k)) +} + +/// Convert packed signed 16-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtsepi16_epi8&expand=1803) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovswb))] +pub unsafe fn _mm_maskz_cvtsepi16_epi8(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovswb128(a.as_i16x8(), _mm_setzero_si128().as_i8x16(), k)) +} + +/// Convert packed unsigned 16-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtusepi16_epi8&expand=2042) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovuswb))] +pub unsafe fn _mm512_cvtusepi16_epi8(a: __m512i) -> __m256i { + transmute(vpmovuswb( + a.as_u16x32(), + _mm256_setzero_si256().as_u8x32(), + 0b11111111_11111111_11111111_11111111, + )) +} + +/// Convert packed unsigned 16-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtusepi16_epi8&expand=2043) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovuswb))] +pub unsafe fn _mm512_mask_cvtusepi16_epi8(src: __m256i, k: __mmask32, a: __m512i) -> __m256i { + transmute(vpmovuswb(a.as_u16x32(), src.as_u8x32(), k)) +} + +/// Convert packed unsigned 16-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtusepi16_epi8&expand=2044) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovuswb))] +pub unsafe fn _mm512_maskz_cvtusepi16_epi8(k: __mmask32, a: __m512i) -> __m256i { + transmute(vpmovuswb( + a.as_u16x32(), + _mm256_setzero_si256().as_u8x32(), + k, + )) +} + +/// Convert packed unsigned 16-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvtusepi16_epi8&expand=2039) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovuswb))] +pub unsafe fn _mm256_cvtusepi16_epi8(a: __m256i) -> __m128i { + transmute(vpmovuswb256( + a.as_u16x16(), + _mm_setzero_si128().as_u8x16(), + 0b11111111_11111111, + )) +} + +/// Convert packed unsigned 16-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtusepi16_epi8&expand=2040) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovuswb))] +pub unsafe fn _mm256_mask_cvtusepi16_epi8(src: __m128i, k: __mmask16, a: __m256i) -> __m128i { + transmute(vpmovuswb256(a.as_u16x16(), src.as_u8x16(), k)) +} + +/// Convert packed unsigned 16-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtusepi16_epi8&expand=2041) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovuswb))] +pub unsafe fn _mm256_maskz_cvtusepi16_epi8(k: __mmask16, a: __m256i) -> __m128i { + transmute(vpmovuswb256( + a.as_u16x16(), + _mm_setzero_si128().as_u8x16(), + k, + )) +} + +/// Convert packed unsigned 16-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtusepi16_epi8&expand=2036) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovuswb))] +pub unsafe fn _mm_cvtusepi16_epi8(a: __m128i) -> __m128i { + transmute(vpmovuswb128( + a.as_u16x8(), + _mm_setzero_si128().as_u8x16(), + 0b11111111, + )) +} + +/// Convert packed unsigned 16-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtusepi16_epi8&expand=2037) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovuswb))] +pub unsafe fn _mm_mask_cvtusepi16_epi8(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovuswb128(a.as_u16x8(), src.as_u8x16(), k)) +} + +/// Convert packed unsigned 16-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtusepi16_epi8&expand=2038) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovuswb))] +pub unsafe fn _mm_maskz_cvtusepi16_epi8(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovuswb128( + a.as_u16x8(), + _mm_setzero_si128().as_u8x16(), + k, + )) +} + +/// Sign extend packed 8-bit integers in a to packed 16-bit integers, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepi8_epi16&expand=1526) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovsxbw))] +pub unsafe fn _mm512_cvtepi8_epi16(a: __m256i) -> __m512i { + let a = a.as_i8x32(); + transmute::(simd_cast(a)) +} + +/// Sign extend packed 8-bit integers in a to packed 16-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepi8_epi16&expand=1527) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovsxbw))] +pub unsafe fn _mm512_mask_cvtepi8_epi16(src: __m512i, k: __mmask32, a: __m256i) -> __m512i { + let convert = _mm512_cvtepi8_epi16(a).as_i16x32(); + transmute(simd_select_bitmask(k, convert, src.as_i16x32())) +} + +/// Sign extend packed 8-bit integers in a to packed 16-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepi8_epi16&expand=1528) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovsxbw))] +pub unsafe fn _mm512_maskz_cvtepi8_epi16(k: __mmask32, a: __m256i) -> __m512i { + let convert = _mm512_cvtepi8_epi16(a).as_i16x32(); + transmute(simd_select_bitmask( + k, + convert, + _mm512_setzero_si512().as_i16x32(), + )) +} + +/// Sign extend packed 8-bit integers in a to packed 16-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepi8_epi16&expand=1524) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxbw))] +pub unsafe fn _mm256_mask_cvtepi8_epi16(src: __m256i, k: __mmask16, a: __m128i) -> __m256i { + let convert = _mm256_cvtepi8_epi16(a).as_i16x16(); + transmute(simd_select_bitmask(k, convert, src.as_i16x16())) +} + +/// Sign extend packed 8-bit integers in a to packed 16-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtepi8_epi16&expand=1525) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxbw))] +pub unsafe fn _mm256_maskz_cvtepi8_epi16(k: __mmask16, a: __m128i) -> __m256i { + let convert = _mm256_cvtepi8_epi16(a).as_i16x16(); + transmute(simd_select_bitmask( + k, + convert, + _mm256_setzero_si256().as_i16x16(), + )) +} + +/// Sign extend packed 8-bit integers in a to packed 16-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepi8_epi16&expand=1521) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxbw))] +pub unsafe fn _mm_mask_cvtepi8_epi16(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepi8_epi16(a).as_i16x8(); + transmute(simd_select_bitmask(k, convert, src.as_i16x8())) +} + +/// Sign extend packed 8-bit integers in a to packed 16-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtepi8_epi16&expand=1522) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxbw))] +pub unsafe fn _mm_maskz_cvtepi8_epi16(k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepi8_epi16(a).as_i16x8(); + transmute(simd_select_bitmask( + k, + convert, + _mm_setzero_si128().as_i16x8(), + )) +} + +/// Zero extend packed unsigned 8-bit integers in a to packed 16-bit integers, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepu8_epi16&expand=1612) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovzxbw))] +pub unsafe fn _mm512_cvtepu8_epi16(a: __m256i) -> __m512i { + let a = a.as_u8x32(); + transmute::(simd_cast(a)) +} + +/// Zero extend packed unsigned 8-bit integers in a to packed 16-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepu8_epi16&expand=1613) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovzxbw))] +pub unsafe fn _mm512_mask_cvtepu8_epi16(src: __m512i, k: __mmask32, a: __m256i) -> __m512i { + let convert = _mm512_cvtepu8_epi16(a).as_i16x32(); + transmute(simd_select_bitmask(k, convert, src.as_i16x32())) +} + +/// Zero extend packed unsigned 8-bit integers in a to packed 16-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepu8_epi16&expand=1614) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovzxbw))] +pub unsafe fn _mm512_maskz_cvtepu8_epi16(k: __mmask32, a: __m256i) -> __m512i { + let convert = _mm512_cvtepu8_epi16(a).as_i16x32(); + transmute(simd_select_bitmask( + k, + convert, + _mm512_setzero_si512().as_i16x32(), + )) +} + +/// Zero extend packed unsigned 8-bit integers in a to packed 16-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepu8_epi16&expand=1610) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxbw))] +pub unsafe fn _mm256_mask_cvtepu8_epi16(src: __m256i, k: __mmask16, a: __m128i) -> __m256i { + let convert = _mm256_cvtepu8_epi16(a).as_i16x16(); + transmute(simd_select_bitmask(k, convert, src.as_i16x16())) +} + +/// Zero extend packed unsigned 8-bit integers in a to packed 16-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtepu8_epi16&expand=1611) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxbw))] +pub unsafe fn _mm256_maskz_cvtepu8_epi16(k: __mmask16, a: __m128i) -> __m256i { + let convert = _mm256_cvtepu8_epi16(a).as_i16x16(); + transmute(simd_select_bitmask( + k, + convert, + _mm256_setzero_si256().as_i16x16(), + )) +} + +/// Zero extend packed unsigned 8-bit integers in a to packed 16-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepu8_epi16&expand=1607) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxbw))] +pub unsafe fn _mm_mask_cvtepu8_epi16(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepu8_epi16(a).as_i16x8(); + transmute(simd_select_bitmask(k, convert, src.as_i16x8())) +} + +/// Zero extend packed unsigned 8-bit integers in a to packed 16-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtepu8_epi16&expand=1608) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxbw))] +pub unsafe fn _mm_maskz_cvtepu8_epi16(k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepu8_epi16(a).as_i16x8(); + transmute(simd_select_bitmask( + k, + convert, + _mm_setzero_si128().as_i16x8(), + )) +} + +/// Shift 128-bit lanes in a left by imm8 bytes while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_bslli_epi128&expand=591) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpslldq, IMM8 = 3))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_bslli_epi128(a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + const fn mask(shift: i32, i: u32) -> u32 { + let shift = shift as u32 & 0xff; + if shift > 15 || i % 16 < shift { + 0 + } else { + 64 + (i - shift) + } + } + let a = a.as_i8x64(); + let zero = _mm512_setzero_si512().as_i8x64(); + let r: i8x64 = simd_shuffle64!( + zero, + a, + [ + mask(IMM8, 0), + mask(IMM8, 1), + mask(IMM8, 2), + mask(IMM8, 3), + mask(IMM8, 4), + mask(IMM8, 5), + mask(IMM8, 6), + mask(IMM8, 7), + mask(IMM8, 8), + mask(IMM8, 9), + mask(IMM8, 10), + mask(IMM8, 11), + mask(IMM8, 12), + mask(IMM8, 13), + mask(IMM8, 14), + mask(IMM8, 15), + mask(IMM8, 16), + mask(IMM8, 17), + mask(IMM8, 18), + mask(IMM8, 19), + mask(IMM8, 20), + mask(IMM8, 21), + mask(IMM8, 22), + mask(IMM8, 23), + mask(IMM8, 24), + mask(IMM8, 25), + mask(IMM8, 26), + mask(IMM8, 27), + mask(IMM8, 28), + mask(IMM8, 29), + mask(IMM8, 30), + mask(IMM8, 31), + mask(IMM8, 32), + mask(IMM8, 33), + mask(IMM8, 34), + mask(IMM8, 35), + mask(IMM8, 36), + mask(IMM8, 37), + mask(IMM8, 38), + mask(IMM8, 39), + mask(IMM8, 40), + mask(IMM8, 41), + mask(IMM8, 42), + mask(IMM8, 43), + mask(IMM8, 44), + mask(IMM8, 45), + mask(IMM8, 46), + mask(IMM8, 47), + mask(IMM8, 48), + mask(IMM8, 49), + mask(IMM8, 50), + mask(IMM8, 51), + mask(IMM8, 52), + mask(IMM8, 53), + mask(IMM8, 54), + mask(IMM8, 55), + mask(IMM8, 56), + mask(IMM8, 57), + mask(IMM8, 58), + mask(IMM8, 59), + mask(IMM8, 60), + mask(IMM8, 61), + mask(IMM8, 62), + mask(IMM8, 63), + ], + ); + transmute(r) +} + +/// Shift 128-bit lanes in a right by imm8 bytes while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_bsrli_epi128&expand=594) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpsrldq, IMM8 = 3))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_bsrli_epi128(a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_i8x64(); + let zero = _mm512_setzero_si512().as_i8x64(); + let r: i8x64 = match IMM8 % 16 { + 0 => simd_shuffle64!( + a, + zero, + [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, + 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, + 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + ], + ), + 1 => simd_shuffle64!( + a, + zero, + [ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 80, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 96, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, + ], + ), + 2 => simd_shuffle64!( + a, + zero, + [ + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 80, 81, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 46, 47, 96, 97, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, + ], + ), + 3 => simd_shuffle64!( + a, + zero, + [ + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 80, 81, 82, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 46, 47, 96, 97, 98, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, + 114, + ], + ), + 4 => simd_shuffle64!( + a, + zero, + [ + 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 30, 31, 80, 81, 82, 83, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, + 47, 96, 97, 98, 99, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, + 115, + ], + ), + 5 => simd_shuffle64!( + a, + zero, + [ + 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 80, 81, 82, 83, 84, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 96, 97, 98, 99, 100, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, + 115, 116, + ], + ), + 6 => simd_shuffle64!( + a, + zero, + [ + 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 22, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 80, 81, 82, 83, 84, 85, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 96, + 97, 98, 99, 100, 101, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, 115, + 116, 117, + ], + ), + 7 => simd_shuffle64!( + a, + zero, + [ + 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 80, 81, 82, 83, 84, 85, 86, 39, 40, 41, 42, 43, 44, 45, 46, 47, 96, + 97, 98, 99, 100, 101, 102, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, 115, + 116, 117, 118, + ], + ), + 8 => simd_shuffle64!( + a, + zero, + [ + 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 24, 25, 26, 27, 28, + 29, 30, 31, 80, 81, 82, 83, 84, 85, 86, 87, 40, 41, 42, 43, 44, 45, 46, 47, 96, 97, + 98, 99, 100, 101, 102, 103, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, 115, + 116, 117, 118, 119, + ], + ), + 9 => simd_shuffle64!( + a, + zero, + [ + 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 25, 26, 27, 28, 29, + 30, 31, 80, 81, 82, 83, 84, 85, 86, 87, 88, 41, 42, 43, 44, 45, 46, 47, 96, 97, 98, + 99, 100, 101, 102, 103, 104, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, 115, 116, + 117, 118, 119, 120, + ], + ), + 10 => simd_shuffle64!( + a, + zero, + [ + 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 26, 27, 28, 29, 30, + 31, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 42, 43, 44, 45, 46, 47, 96, 97, 98, 99, + 100, 101, 102, 103, 104, 105, 58, 59, 60, 61, 62, 63, 112, 113, 114, 115, 116, 117, + 118, 119, 120, 121, + ], + ), + 11 => simd_shuffle64!( + a, + zero, + [ + 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 27, 28, 29, 30, 31, + 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 43, 44, 45, 46, 47, 96, 97, 98, 99, + 100, 101, 102, 103, 104, 105, 106, 59, 60, 61, 62, 63, 112, 113, 114, 115, 116, + 117, 118, 119, 120, 121, 122, + ], + ), + 12 => simd_shuffle64!( + a, + zero, + [ + 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 28, 29, 30, 31, 80, + 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 44, 45, 46, 47, 96, 97, 98, 99, 100, + 101, 102, 103, 104, 105, 106, 107, 60, 61, 62, 63, 112, 113, 114, 115, 116, 117, + 118, 119, 120, 121, 122, 123, + ], + ), + 13 => simd_shuffle64!( + a, + zero, + [ + 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 29, 30, 31, 80, 81, + 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 45, 46, 47, 96, 97, 98, 99, 100, 101, + 102, 103, 104, 105, 106, 107, 108, 61, 62, 63, 112, 113, 114, 115, 116, 117, 118, + 119, 120, 121, 122, 123, 124, + ], + ), + 14 => simd_shuffle64!( + a, + zero, + [ + 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 30, 31, 80, 81, 82, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 46, 47, 96, 97, 98, 99, 100, 101, 102, + 103, 104, 105, 106, 107, 108, 109, 62, 63, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, + ], + ), + 15 => simd_shuffle64!( + a, + zero, + [ + 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 31, 80, 81, 82, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 47, 96, 97, 98, 99, 100, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 63, 112, 113, 114, 115, 116, 117, 118, 119, 120, + 121, 122, 123, 124, 125, 126, + ], + ), + _ => zero, + }; + transmute(r) +} + +/// Concatenate pairs of 16-byte blocks in a and b into a 32-byte temporary result, shift the result right by imm8 bytes, and store the low 16 bytes in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_alignr_epi8&expand=263) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpalignr, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_alignr_epi8(a: __m512i, b: __m512i) -> __m512i { + // If palignr is shifting the pair of vectors more than the size of two + // lanes, emit zero. + if IMM8 > 32 { + return _mm512_set1_epi8(0); + } + // If palignr is shifting the pair of input vectors more than one lane, + // but less than two lanes, convert to shifting in zeroes. + let (a, b) = if IMM8 > 16 { + (_mm512_set1_epi8(0), a) + } else { + (a, b) + }; + let a = a.as_i8x64(); + let b = b.as_i8x64(); + + let r: i8x64 = match IMM8 % 16 { + 0 => simd_shuffle64!( + b, + a, + [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, + 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, + 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + ], + ), + 1 => simd_shuffle64!( + b, + a, + [ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 80, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 96, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, + ], + ), + 2 => simd_shuffle64!( + b, + a, + [ + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 80, 81, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 46, 47, 96, 97, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, + ], + ), + 3 => simd_shuffle64!( + b, + a, + [ + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 80, 81, 82, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 46, 47, 96, 97, 98, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, + 114, + ], + ), + 4 => simd_shuffle64!( + b, + a, + [ + 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 30, 31, 80, 81, 82, 83, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, + 47, 96, 97, 98, 99, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, + 115, + ], + ), + 5 => simd_shuffle64!( + b, + a, + [ + 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 80, 81, 82, 83, 84, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 96, 97, 98, 99, 100, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, + 115, 116, + ], + ), + 6 => simd_shuffle64!( + b, + a, + [ + 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 22, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 80, 81, 82, 83, 84, 85, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 96, + 97, 98, 99, 100, 101, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, 115, + 116, 117, + ], + ), + 7 => simd_shuffle64!( + b, + a, + [ + 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 80, 81, 82, 83, 84, 85, 86, 39, 40, 41, 42, 43, 44, 45, 46, 47, 96, + 97, 98, 99, 100, 101, 102, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, 115, + 116, 117, 118, + ], + ), + 8 => simd_shuffle64!( + b, + a, + [ + 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 24, 25, 26, 27, 28, + 29, 30, 31, 80, 81, 82, 83, 84, 85, 86, 87, 40, 41, 42, 43, 44, 45, 46, 47, 96, 97, + 98, 99, 100, 101, 102, 103, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, 115, + 116, 117, 118, 119, + ], + ), + 9 => simd_shuffle64!( + b, + a, + [ + 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 25, 26, 27, 28, 29, + 30, 31, 80, 81, 82, 83, 84, 85, 86, 87, 88, 41, 42, 43, 44, 45, 46, 47, 96, 97, 98, + 99, 100, 101, 102, 103, 104, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, 115, 116, + 117, 118, 119, 120, + ], + ), + 10 => simd_shuffle64!( + b, + a, + [ + 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 26, 27, 28, 29, 30, + 31, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 42, 43, 44, 45, 46, 47, 96, 97, 98, 99, + 100, 101, 102, 103, 104, 105, 58, 59, 60, 61, 62, 63, 112, 113, 114, 115, 116, 117, + 118, 119, 120, 121, + ], + ), + 11 => simd_shuffle64!( + b, + a, + [ + 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 27, 28, 29, 30, 31, + 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 43, 44, 45, 46, 47, 96, 97, 98, 99, + 100, 101, 102, 103, 104, 105, 106, 59, 60, 61, 62, 63, 112, 113, 114, 115, 116, + 117, 118, 119, 120, 121, 122, + ], + ), + 12 => simd_shuffle64!( + b, + a, + [ + 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 28, 29, 30, 31, 80, + 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 44, 45, 46, 47, 96, 97, 98, 99, 100, + 101, 102, 103, 104, 105, 106, 107, 60, 61, 62, 63, 112, 113, 114, 115, 116, 117, + 118, 119, 120, 121, 122, 123, + ], + ), + 13 => simd_shuffle64!( + b, + a, + [ + 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 29, 30, 31, 80, 81, + 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 45, 46, 47, 96, 97, 98, 99, 100, 101, + 102, 103, 104, 105, 106, 107, 108, 61, 62, 63, 112, 113, 114, 115, 116, 117, 118, + 119, 120, 121, 122, 123, 124, + ], + ), + 14 => simd_shuffle64!( + b, + a, + [ + 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 30, 31, 80, 81, 82, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 46, 47, 96, 97, 98, 99, 100, 101, 102, + 103, 104, 105, 106, 107, 108, 109, 62, 63, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, + ], + ), + 15 => simd_shuffle64!( + b, + a, + [ + 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 31, 80, 81, 82, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 47, 96, 97, 98, 99, 100, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 63, 112, 113, 114, 115, 116, 117, 118, 119, 120, + 121, 122, 123, 124, 125, 126, + ], + ), + _ => b, + }; + transmute(r) +} + +/// Concatenate pairs of 16-byte blocks in a and b into a 32-byte temporary result, shift the result right by imm8 bytes, and store the low 16 bytes in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_alignr_epi8&expand=264) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpalignr, IMM8 = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_alignr_epi8( + src: __m512i, + k: __mmask64, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let r = _mm512_alignr_epi8::(a, b); + transmute(simd_select_bitmask(k, r.as_i8x64(), src.as_i8x64())) +} + +/// Concatenate pairs of 16-byte blocks in a and b into a 32-byte temporary result, shift the result right by imm8 bytes, and store the low 16 bytes in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_alignr_epi8&expand=265) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpalignr, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_alignr_epi8( + k: __mmask64, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let r = _mm512_alignr_epi8::(a, b); + let zero = _mm512_setzero_si512().as_i8x64(); + transmute(simd_select_bitmask(k, r.as_i8x64(), zero)) +} + +/// Concatenate pairs of 16-byte blocks in a and b into a 32-byte temporary result, shift the result right by imm8 bytes, and store the low 16 bytes in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_alignr_epi8&expand=261) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(4)] +#[cfg_attr(test, assert_instr(vpalignr, IMM8 = 5))] +pub unsafe fn _mm256_mask_alignr_epi8( + src: __m256i, + k: __mmask32, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let r = _mm256_alignr_epi8::(a, b); + transmute(simd_select_bitmask(k, r.as_i8x32(), src.as_i8x32())) +} + +/// Concatenate pairs of 16-byte blocks in a and b into a 32-byte temporary result, shift the result right by imm8 bytes, and store the low 16 bytes in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_alignr_epi8&expand=262) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpalignr, IMM8 = 5))] +pub unsafe fn _mm256_maskz_alignr_epi8( + k: __mmask32, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let r = _mm256_alignr_epi8::(a, b); + transmute(simd_select_bitmask( + k, + r.as_i8x32(), + _mm256_setzero_si256().as_i8x32(), + )) +} + +/// Concatenate pairs of 16-byte blocks in a and b into a 32-byte temporary result, shift the result right by imm8 bytes, and store the low 16 bytes in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_alignr_epi8&expand=258) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(4)] +#[cfg_attr(test, assert_instr(vpalignr, IMM8 = 5))] +pub unsafe fn _mm_mask_alignr_epi8( + src: __m128i, + k: __mmask16, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let r = _mm_alignr_epi8::(a, b); + transmute(simd_select_bitmask(k, r.as_i8x16(), src.as_i8x16())) +} + +/// Concatenate pairs of 16-byte blocks in a and b into a 32-byte temporary result, shift the result right by imm8 bytes, and store the low 16 bytes in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_alignr_epi8&expand=259) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpalignr, IMM8 = 5))] +pub unsafe fn _mm_maskz_alignr_epi8( + k: __mmask16, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let r = _mm_alignr_epi8::(a, b); + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask(k, r.as_i8x16(), zero)) +} + +/// Convert packed signed 16-bit integers in a to packed 8-bit integers with signed saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtsepi16_storeu_epi8&expand=1812) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovswb))] +pub unsafe fn _mm512_mask_cvtsepi16_storeu_epi8(mem_addr: *mut i8, k: __mmask32, a: __m512i) { + vpmovswbmem(mem_addr as *mut i8, a.as_i16x32(), k); +} + +/// Convert packed signed 16-bit integers in a to packed 8-bit integers with signed saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtsepi16_storeu_epi8&expand=1811) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovswb))] +pub unsafe fn _mm256_mask_cvtsepi16_storeu_epi8(mem_addr: *mut i8, k: __mmask16, a: __m256i) { + vpmovswbmem256(mem_addr as *mut i8, a.as_i16x16(), k); +} + +/// Convert packed signed 16-bit integers in a to packed 8-bit integers with signed saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtsepi16_storeu_epi8&expand=1810) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovswb))] +pub unsafe fn _mm_mask_cvtsepi16_storeu_epi8(mem_addr: *mut i8, k: __mmask8, a: __m128i) { + vpmovswbmem128(mem_addr as *mut i8, a.as_i16x8(), k); +} + +/// Convert packed 16-bit integers in a to packed 8-bit integers with truncation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepi16_storeu_epi8&expand=1412) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovwb))] +pub unsafe fn _mm512_mask_cvtepi16_storeu_epi8(mem_addr: *mut i8, k: __mmask32, a: __m512i) { + vpmovwbmem(mem_addr as *mut i8, a.as_i16x32(), k); +} + +/// Convert packed 16-bit integers in a to packed 8-bit integers with truncation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepi16_storeu_epi8&expand=1411) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovwb))] +pub unsafe fn _mm256_mask_cvtepi16_storeu_epi8(mem_addr: *mut i8, k: __mmask16, a: __m256i) { + vpmovwbmem256(mem_addr as *mut i8, a.as_i16x16(), k); +} + +/// Convert packed 16-bit integers in a to packed 8-bit integers with truncation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepi16_storeu_epi8&expand=1410) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovwb))] +pub unsafe fn _mm_mask_cvtepi16_storeu_epi8(mem_addr: *mut i8, k: __mmask8, a: __m128i) { + vpmovwbmem128(mem_addr as *mut i8, a.as_i16x8(), k); +} + +/// Convert packed unsigned 16-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtusepi16_storeu_epi8&expand=2047) +#[inline] +#[target_feature(enable = "avx512bw")] +#[cfg_attr(test, assert_instr(vpmovuswb))] +pub unsafe fn _mm512_mask_cvtusepi16_storeu_epi8(mem_addr: *mut i8, k: __mmask32, a: __m512i) { + vpmovuswbmem(mem_addr as *mut i8, a.as_i16x32(), k); +} + +/// Convert packed unsigned 16-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtusepi16_storeu_epi8&expand=2046) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovuswb))] +pub unsafe fn _mm256_mask_cvtusepi16_storeu_epi8(mem_addr: *mut i8, k: __mmask16, a: __m256i) { + vpmovuswbmem256(mem_addr as *mut i8, a.as_i16x16(), k); +} + +/// Convert packed unsigned 16-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtusepi16_storeu_epi8&expand=2045) +#[inline] +#[target_feature(enable = "avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovuswb))] +pub unsafe fn _mm_mask_cvtusepi16_storeu_epi8(mem_addr: *mut i8, k: __mmask8, a: __m128i) { + vpmovuswbmem128(mem_addr as *mut i8, a.as_i16x8(), k); +} + +#[allow(improper_ctypes)] +extern "C" { + #[link_name = "llvm.x86.avx512.mask.paddus.w.512"] + fn vpaddusw(a: u16x32, b: u16x32, src: u16x32, mask: u32) -> u16x32; + #[link_name = "llvm.x86.avx512.mask.paddus.w.256"] + fn vpaddusw256(a: u16x16, b: u16x16, src: u16x16, mask: u16) -> u16x16; + #[link_name = "llvm.x86.avx512.mask.paddus.w.128"] + fn vpaddusw128(a: u16x8, b: u16x8, src: u16x8, mask: u8) -> u16x8; + + #[link_name = "llvm.x86.avx512.mask.paddus.b.512"] + fn vpaddusb(a: u8x64, b: u8x64, src: u8x64, mask: u64) -> u8x64; + #[link_name = "llvm.x86.avx512.mask.paddus.b.256"] + fn vpaddusb256(a: u8x32, b: u8x32, src: u8x32, mask: u32) -> u8x32; + #[link_name = "llvm.x86.avx512.mask.paddus.b.128"] + fn vpaddusb128(a: u8x16, b: u8x16, src: u8x16, mask: u16) -> u8x16; + + #[link_name = "llvm.x86.avx512.mask.padds.w.512"] + fn vpaddsw(a: i16x32, b: i16x32, src: i16x32, mask: u32) -> i16x32; + #[link_name = "llvm.x86.avx512.mask.padds.w.256"] + fn vpaddsw256(a: i16x16, b: i16x16, src: i16x16, mask: u16) -> i16x16; + #[link_name = "llvm.x86.avx512.mask.padds.w.128"] + fn vpaddsw128(a: i16x8, b: i16x8, src: i16x8, mask: u8) -> i16x8; + + #[link_name = "llvm.x86.avx512.mask.padds.b.512"] + fn vpaddsb(a: i8x64, b: i8x64, src: i8x64, mask: u64) -> i8x64; + #[link_name = "llvm.x86.avx512.mask.padds.b.256"] + fn vpaddsb256(a: i8x32, b: i8x32, src: i8x32, mask: u32) -> i8x32; + #[link_name = "llvm.x86.avx512.mask.padds.b.128"] + fn vpaddsb128(a: i8x16, b: i8x16, src: i8x16, mask: u16) -> i8x16; + + #[link_name = "llvm.x86.avx512.mask.psubus.w.512"] + fn vpsubusw(a: u16x32, b: u16x32, src: u16x32, mask: u32) -> u16x32; + #[link_name = "llvm.x86.avx512.mask.psubus.w.256"] + fn vpsubusw256(a: u16x16, b: u16x16, src: u16x16, mask: u16) -> u16x16; + #[link_name = "llvm.x86.avx512.mask.psubus.w.128"] + fn vpsubusw128(a: u16x8, b: u16x8, src: u16x8, mask: u8) -> u16x8; + + #[link_name = "llvm.x86.avx512.mask.psubus.b.512"] + fn vpsubusb(a: u8x64, b: u8x64, src: u8x64, mask: u64) -> u8x64; + #[link_name = "llvm.x86.avx512.mask.psubus.b.256"] + fn vpsubusb256(a: u8x32, b: u8x32, src: u8x32, mask: u32) -> u8x32; + #[link_name = "llvm.x86.avx512.mask.psubus.b.128"] + fn vpsubusb128(a: u8x16, b: u8x16, src: u8x16, mask: u16) -> u8x16; + + #[link_name = "llvm.x86.avx512.mask.psubs.w.512"] + fn vpsubsw(a: i16x32, b: i16x32, src: i16x32, mask: u32) -> i16x32; + #[link_name = "llvm.x86.avx512.mask.psubs.w.256"] + fn vpsubsw256(a: i16x16, b: i16x16, src: i16x16, mask: u16) -> i16x16; + #[link_name = "llvm.x86.avx512.mask.psubs.w.128"] + fn vpsubsw128(a: i16x8, b: i16x8, src: i16x8, mask: u8) -> i16x8; + + #[link_name = "llvm.x86.avx512.mask.psubs.b.512"] + fn vpsubsb(a: i8x64, b: i8x64, src: i8x64, mask: u64) -> i8x64; + #[link_name = "llvm.x86.avx512.mask.psubs.b.256"] + fn vpsubsb256(a: i8x32, b: i8x32, src: i8x32, mask: u32) -> i8x32; + #[link_name = "llvm.x86.avx512.mask.psubs.b.128"] + fn vpsubsb128(a: i8x16, b: i8x16, src: i8x16, mask: u16) -> i8x16; + + #[link_name = "llvm.x86.avx512.pmulhu.w.512"] + fn vpmulhuw(a: u16x32, b: u16x32) -> u16x32; + #[link_name = "llvm.x86.avx512.pmulh.w.512"] + fn vpmulhw(a: i16x32, b: i16x32) -> i16x32; + #[link_name = "llvm.x86.avx512.pmul.hr.sw.512"] + fn vpmulhrsw(a: i16x32, b: i16x32) -> i16x32; + + #[link_name = "llvm.x86.avx512.mask.ucmp.w.512"] + fn vpcmpuw(a: u16x32, b: u16x32, op: i32, mask: u32) -> u32; + #[link_name = "llvm.x86.avx512.mask.ucmp.w.256"] + fn vpcmpuw256(a: u16x16, b: u16x16, op: i32, mask: u16) -> u16; + #[link_name = "llvm.x86.avx512.mask.ucmp.w.128"] + fn vpcmpuw128(a: u16x8, b: u16x8, op: i32, mask: u8) -> u8; + + #[link_name = "llvm.x86.avx512.mask.ucmp.b.512"] + fn vpcmpub(a: u8x64, b: u8x64, op: i32, mask: u64) -> u64; + #[link_name = "llvm.x86.avx512.mask.ucmp.b.256"] + fn vpcmpub256(a: u8x32, b: u8x32, op: i32, mask: u32) -> u32; + #[link_name = "llvm.x86.avx512.mask.ucmp.b.128"] + fn vpcmpub128(a: u8x16, b: u8x16, op: i32, mask: u16) -> u16; + + #[link_name = "llvm.x86.avx512.mask.cmp.w.512"] + fn vpcmpw(a: i16x32, b: i16x32, op: i32, mask: u32) -> u32; + #[link_name = "llvm.x86.avx512.mask.cmp.w.256"] + fn vpcmpw256(a: i16x16, b: i16x16, op: i32, mask: u16) -> u16; + #[link_name = "llvm.x86.avx512.mask.cmp.w.128"] + fn vpcmpw128(a: i16x8, b: i16x8, op: i32, mask: u8) -> u8; + + #[link_name = "llvm.x86.avx512.mask.cmp.b.512"] + fn vpcmpb(a: i8x64, b: i8x64, op: i32, mask: u64) -> u64; + #[link_name = "llvm.x86.avx512.mask.cmp.b.256"] + fn vpcmpb256(a: i8x32, b: i8x32, op: i32, mask: u32) -> u32; + #[link_name = "llvm.x86.avx512.mask.cmp.b.128"] + fn vpcmpb128(a: i8x16, b: i8x16, op: i32, mask: u16) -> u16; + + #[link_name = "llvm.x86.avx512.mask.pmaxu.w.512"] + fn vpmaxuw(a: u16x32, b: u16x32) -> u16x32; + #[link_name = "llvm.x86.avx512.mask.pmaxu.b.512"] + fn vpmaxub(a: u8x64, b: u8x64) -> u8x64; + #[link_name = "llvm.x86.avx512.mask.pmaxs.w.512"] + fn vpmaxsw(a: i16x32, b: i16x32) -> i16x32; + #[link_name = "llvm.x86.avx512.mask.pmaxs.b.512"] + fn vpmaxsb(a: i8x64, b: i8x64) -> i8x64; + + #[link_name = "llvm.x86.avx512.mask.pminu.w.512"] + fn vpminuw(a: u16x32, b: u16x32) -> u16x32; + #[link_name = "llvm.x86.avx512.mask.pminu.b.512"] + fn vpminub(a: u8x64, b: u8x64) -> u8x64; + #[link_name = "llvm.x86.avx512.mask.pmins.w.512"] + fn vpminsw(a: i16x32, b: i16x32) -> i16x32; + #[link_name = "llvm.x86.avx512.mask.pmins.b.512"] + fn vpminsb(a: i8x64, b: i8x64) -> i8x64; + + #[link_name = "llvm.x86.avx512.pmaddw.d.512"] + fn vpmaddwd(a: i16x32, b: i16x32) -> i32x16; + #[link_name = "llvm.x86.avx512.pmaddubs.w.512"] + fn vpmaddubsw(a: i8x64, b: i8x64) -> i16x32; + + #[link_name = "llvm.x86.avx512.packssdw.512"] + fn vpackssdw(a: i32x16, b: i32x16) -> i16x32; + #[link_name = "llvm.x86.avx512.packsswb.512"] + fn vpacksswb(a: i16x32, b: i16x32) -> i8x64; + #[link_name = "llvm.x86.avx512.packusdw.512"] + fn vpackusdw(a: i32x16, b: i32x16) -> u16x32; + #[link_name = "llvm.x86.avx512.packuswb.512"] + fn vpackuswb(a: i16x32, b: i16x32) -> u8x64; + + #[link_name = "llvm.x86.avx512.pavg.w.512"] + fn vpavgw(a: u16x32, b: u16x32) -> u16x32; + #[link_name = "llvm.x86.avx512.pavg.b.512"] + fn vpavgb(a: u8x64, b: u8x64) -> u8x64; + + #[link_name = "llvm.x86.avx512.psll.w.512"] + fn vpsllw(a: i16x32, count: i16x8) -> i16x32; + #[link_name = "llvm.x86.avx512.pslli.w.512"] + fn vpslliw(a: i16x32, imm8: u32) -> i16x32; + + #[link_name = "llvm.x86.avx2.pslli.w"] + fn pslliw256(a: i16x16, imm8: i32) -> i16x16; + #[link_name = "llvm.x86.sse2.pslli.w"] + fn pslliw128(a: i16x8, imm8: i32) -> i16x8; + + #[link_name = "llvm.x86.avx512.psllv.w.512"] + fn vpsllvw(a: i16x32, b: i16x32) -> i16x32; + #[link_name = "llvm.x86.avx512.psllv.w.256"] + fn vpsllvw256(a: i16x16, b: i16x16) -> i16x16; + #[link_name = "llvm.x86.avx512.psllv.w.128"] + fn vpsllvw128(a: i16x8, b: i16x8) -> i16x8; + + #[link_name = "llvm.x86.avx512.psrl.w.512"] + fn vpsrlw(a: i16x32, count: i16x8) -> i16x32; + #[link_name = "llvm.x86.avx512.psrli.w.512"] + fn vpsrliw(a: i16x32, imm8: u32) -> i16x32; + + #[link_name = "llvm.x86.avx512.psrlv.w.512"] + fn vpsrlvw(a: i16x32, b: i16x32) -> i16x32; + #[link_name = "llvm.x86.avx512.psrlv.w.256"] + fn vpsrlvw256(a: i16x16, b: i16x16) -> i16x16; + #[link_name = "llvm.x86.avx512.psrlv.w.128"] + fn vpsrlvw128(a: i16x8, b: i16x8) -> i16x8; + + #[link_name = "llvm.x86.avx512.psra.w.512"] + fn vpsraw(a: i16x32, count: i16x8) -> i16x32; + #[link_name = "llvm.x86.avx512.psrai.w.512"] + fn vpsraiw(a: i16x32, imm8: u32) -> i16x32; + + #[link_name = "llvm.x86.avx2.psrai.w"] + fn psraiw256(a: i16x16, imm8: i32) -> i16x16; + #[link_name = "llvm.x86.sse2.psrai.w"] + fn psraiw128(a: i16x8, imm8: i32) -> i16x8; + + #[link_name = "llvm.x86.avx512.psrav.w.512"] + fn vpsravw(a: i16x32, count: i16x32) -> i16x32; + #[link_name = "llvm.x86.avx512.psrav.w.256"] + fn vpsravw256(a: i16x16, count: i16x16) -> i16x16; + #[link_name = "llvm.x86.avx512.psrav.w.128"] + fn vpsravw128(a: i16x8, count: i16x8) -> i16x8; + + #[link_name = "llvm.x86.avx512.vpermi2var.hi.512"] + fn vpermi2w(a: i16x32, idx: i16x32, b: i16x32) -> i16x32; + #[link_name = "llvm.x86.avx512.vpermi2var.hi.256"] + fn vpermi2w256(a: i16x16, idx: i16x16, b: i16x16) -> i16x16; + #[link_name = "llvm.x86.avx512.vpermi2var.hi.128"] + fn vpermi2w128(a: i16x8, idx: i16x8, b: i16x8) -> i16x8; + + #[link_name = "llvm.x86.avx512.permvar.hi.512"] + fn vpermw(a: i16x32, idx: i16x32) -> i16x32; + #[link_name = "llvm.x86.avx512.permvar.hi.256"] + fn vpermw256(a: i16x16, idx: i16x16) -> i16x16; + #[link_name = "llvm.x86.avx512.permvar.hi.128"] + fn vpermw128(a: i16x8, idx: i16x8) -> i16x8; + + #[link_name = "llvm.x86.avx512.pshuf.b.512"] + fn vpshufb(a: i8x64, b: i8x64) -> i8x64; + + #[link_name = "llvm.x86.avx512.psad.bw.512"] + fn vpsadbw(a: u8x64, b: u8x64) -> u64x8; + + #[link_name = "llvm.x86.avx512.dbpsadbw.512"] + fn vdbpsadbw(a: u8x64, b: u8x64, imm8: i32) -> u16x32; + #[link_name = "llvm.x86.avx512.dbpsadbw.256"] + fn vdbpsadbw256(a: u8x32, b: u8x32, imm8: i32) -> u16x16; + #[link_name = "llvm.x86.avx512.dbpsadbw.128"] + fn vdbpsadbw128(a: u8x16, b: u8x16, imm8: i32) -> u16x8; + + #[link_name = "llvm.x86.avx512.mask.pmovs.wb.512"] + fn vpmovswb(a: i16x32, src: i8x32, mask: u32) -> i8x32; + #[link_name = "llvm.x86.avx512.mask.pmovs.wb.256"] + fn vpmovswb256(a: i16x16, src: i8x16, mask: u16) -> i8x16; + #[link_name = "llvm.x86.avx512.mask.pmovs.wb.128"] + fn vpmovswb128(a: i16x8, src: i8x16, mask: u8) -> i8x16; + + #[link_name = "llvm.x86.avx512.mask.pmovus.wb.512"] + fn vpmovuswb(a: u16x32, src: u8x32, mask: u32) -> u8x32; + #[link_name = "llvm.x86.avx512.mask.pmovus.wb.256"] + fn vpmovuswb256(a: u16x16, src: u8x16, mask: u16) -> u8x16; + #[link_name = "llvm.x86.avx512.mask.pmovus.wb.128"] + fn vpmovuswb128(a: u16x8, src: u8x16, mask: u8) -> u8x16; + + #[link_name = "llvm.x86.avx512.mask.pmovs.wb.mem.512"] + fn vpmovswbmem(mem_addr: *mut i8, a: i16x32, mask: u32); + #[link_name = "llvm.x86.avx512.mask.pmovs.wb.mem.256"] + fn vpmovswbmem256(mem_addr: *mut i8, a: i16x16, mask: u16); + #[link_name = "llvm.x86.avx512.mask.pmovs.wb.mem.128"] + fn vpmovswbmem128(mem_addr: *mut i8, a: i16x8, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.pmov.wb.mem.512"] + fn vpmovwbmem(mem_addr: *mut i8, a: i16x32, mask: u32); + #[link_name = "llvm.x86.avx512.mask.pmov.wb.mem.256"] + fn vpmovwbmem256(mem_addr: *mut i8, a: i16x16, mask: u16); + #[link_name = "llvm.x86.avx512.mask.pmov.wb.mem.128"] + fn vpmovwbmem128(mem_addr: *mut i8, a: i16x8, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.pmovus.wb.mem.512"] + fn vpmovuswbmem(mem_addr: *mut i8, a: i16x32, mask: u32); + #[link_name = "llvm.x86.avx512.mask.pmovus.wb.mem.256"] + fn vpmovuswbmem256(mem_addr: *mut i8, a: i16x16, mask: u16); + #[link_name = "llvm.x86.avx512.mask.pmovus.wb.mem.128"] + fn vpmovuswbmem128(mem_addr: *mut i8, a: i16x8, mask: u8); +} + +#[cfg(test)] +mod tests { + + use stdarch_test::simd_test; + + use crate::core_arch::x86::*; + use crate::hint::black_box; + use crate::mem::{self}; + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_abs_epi16() { + let a = _mm512_set1_epi16(-1); + let r = _mm512_abs_epi16(a); + let e = _mm512_set1_epi16(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_abs_epi16() { + let a = _mm512_set1_epi16(-1); + let r = _mm512_mask_abs_epi16(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_abs_epi16(a, 0b00000000_11111111_00000000_11111111, a); + #[rustfmt::skip] + let e = _mm512_set_epi16(-1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_abs_epi16() { + let a = _mm512_set1_epi16(-1); + let r = _mm512_maskz_abs_epi16(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_abs_epi16(0b00000000_11111111_00000000_11111111, a); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_abs_epi16() { + let a = _mm256_set1_epi16(-1); + let r = _mm256_mask_abs_epi16(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_abs_epi16(a, 0b00000000_11111111, a); + let e = _mm256_set_epi16(-1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_abs_epi16() { + let a = _mm256_set1_epi16(-1); + let r = _mm256_maskz_abs_epi16(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_abs_epi16(0b00000000_11111111, a); + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_abs_epi16() { + let a = _mm_set1_epi16(-1); + let r = _mm_mask_abs_epi16(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_abs_epi16(a, 0b00001111, a); + let e = _mm_set_epi16(-1, -1, -1, -1, 1, 1, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_abs_epi16() { + let a = _mm_set1_epi16(-1); + let r = _mm_maskz_abs_epi16(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_abs_epi16(0b00001111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 1, 1, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_abs_epi8() { + let a = _mm512_set1_epi8(-1); + let r = _mm512_abs_epi8(a); + let e = _mm512_set1_epi8(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_abs_epi8() { + let a = _mm512_set1_epi8(-1); + let r = _mm512_mask_abs_epi8(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_abs_epi8( + a, + 0b00000000_11111111_00000000_11111111_00000000_11111111_00000000_11111111, + a, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_abs_epi8() { + let a = _mm512_set1_epi8(-1); + let r = _mm512_maskz_abs_epi8(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_abs_epi8( + 0b00000000_11111111_00000000_11111111_00000000_11111111_00000000_11111111, + a, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_abs_epi8() { + let a = _mm256_set1_epi8(-1); + let r = _mm256_mask_abs_epi8(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_abs_epi8(a, 0b00000000_11111111_00000000_11111111, a); + #[rustfmt::skip] + let e = _mm256_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_abs_epi8() { + let a = _mm256_set1_epi8(-1); + let r = _mm256_maskz_abs_epi8(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_abs_epi8(0b00000000_11111111_00000000_11111111, a); + #[rustfmt::skip] + let e = _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_abs_epi8() { + let a = _mm_set1_epi8(-1); + let r = _mm_mask_abs_epi8(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_abs_epi8(a, 0b00000000_11111111, a); + let e = _mm_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_abs_epi8() { + let a = _mm_set1_epi8(-1); + let r = _mm_maskz_abs_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_abs_epi8(0b00000000_11111111, a); + #[rustfmt::skip] + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_add_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(2); + let r = _mm512_add_epi16(a, b); + let e = _mm512_set1_epi16(3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_add_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(2); + let r = _mm512_mask_add_epi16(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_add_epi16(a, 0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3, + 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_add_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(2); + let r = _mm512_maskz_add_epi16(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_add_epi16(0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 3, 3, 3, 3, + 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 3, 3, 3, 3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_add_epi16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(2); + let r = _mm256_mask_add_epi16(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_add_epi16(a, 0b00000000_11111111, a, b); + let e = _mm256_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_add_epi16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(2); + let r = _mm256_maskz_add_epi16(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_add_epi16(0b00000000_11111111, a, b); + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 3, 3, 3, 3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_add_epi16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(2); + let r = _mm_mask_add_epi16(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_add_epi16(a, 0b00001111, a, b); + let e = _mm_set_epi16(1, 1, 1, 1, 3, 3, 3, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_add_epi16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(2); + let r = _mm_maskz_add_epi16(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_add_epi16(0b00001111, a, b); + let e = _mm_set_epi16(0, 0, 0, 0, 3, 3, 3, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_add_epi8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(2); + let r = _mm512_add_epi8(a, b); + let e = _mm512_set1_epi8(3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_add_epi8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(2); + let r = _mm512_mask_add_epi8(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_add_epi8( + a, + 0b00000000_11111111_00000000_11111111_00000000_11111111_00000000_11111111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3, + 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3, + 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3, + 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_add_epi8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(2); + let r = _mm512_maskz_add_epi8(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_add_epi8( + 0b00000000_11111111_00000000_11111111_00000000_11111111_00000000_11111111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 3, 3, 3, 3, + 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 3, 3, 3, 3, + 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 3, 3, 3, 3, + 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 3, 3, 3, 3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_add_epi8() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(2); + let r = _mm256_mask_add_epi8(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_add_epi8(a, 0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3, + 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_add_epi8() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(2); + let r = _mm256_maskz_add_epi8(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_add_epi8(0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 3, 3, 3, 3, + 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 3, 3, 3, 3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_add_epi8() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(2); + let r = _mm_mask_add_epi8(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_add_epi8(a, 0b00000000_11111111, a, b); + let e = _mm_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_add_epi8() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(2); + let r = _mm_maskz_add_epi8(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_add_epi8(0b00000000_11111111, a, b); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 3, 3, 3, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_adds_epu16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(u16::MAX as i16); + let r = _mm512_adds_epu16(a, b); + let e = _mm512_set1_epi16(u16::MAX as i16); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_adds_epu16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(u16::MAX as i16); + let r = _mm512_mask_adds_epu16(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_adds_epu16(a, 0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, u16::MAX as i16, u16::MAX as i16, u16::MAX as i16, u16::MAX as i16); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_adds_epu16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(u16::MAX as i16); + let r = _mm512_maskz_adds_epu16(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_adds_epu16(0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, u16::MAX as i16, u16::MAX as i16, u16::MAX as i16, u16::MAX as i16); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_adds_epu16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(u16::MAX as i16); + let r = _mm256_mask_adds_epu16(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_adds_epu16(a, 0b00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, u16::MAX as i16, u16::MAX as i16, u16::MAX as i16, u16::MAX as i16); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_adds_epu16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(u16::MAX as i16); + let r = _mm256_maskz_adds_epu16(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_adds_epu16(0b00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, u16::MAX as i16, u16::MAX as i16, u16::MAX as i16, u16::MAX as i16); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_adds_epu16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(u16::MAX as i16); + let r = _mm_mask_adds_epu16(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_adds_epu16(a, 0b00001111, a, b); + #[rustfmt::skip] + let e = _mm_set_epi16(1, 1, 1, 1, u16::MAX as i16, u16::MAX as i16, u16::MAX as i16, u16::MAX as i16); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_adds_epu16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(u16::MAX as i16); + let r = _mm_maskz_adds_epu16(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_adds_epu16(0b00001111, a, b); + #[rustfmt::skip] + let e = _mm_set_epi16(0, 0, 0, 0, u16::MAX as i16, u16::MAX as i16, u16::MAX as i16, u16::MAX as i16); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_adds_epu8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(u8::MAX as i8); + let r = _mm512_adds_epu8(a, b); + let e = _mm512_set1_epi8(u8::MAX as i8); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_adds_epu8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(u8::MAX as i8); + let r = _mm512_mask_adds_epu8(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_adds_epu8( + a, + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00001111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, u8::MAX as i8); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_adds_epu8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(u8::MAX as i8); + let r = _mm512_maskz_adds_epu8(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_adds_epu8( + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00001111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, u8::MAX as i8); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_adds_epu8() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(u8::MAX as i8); + let r = _mm256_mask_adds_epu8(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_adds_epu8(a, 0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, u8::MAX as i8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_adds_epu8() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(u8::MAX as i8); + let r = _mm256_maskz_adds_epu8(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_adds_epu8(0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, u8::MAX as i8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_adds_epu8() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(u8::MAX as i8); + let r = _mm_mask_adds_epu8(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_adds_epu8(a, 0b00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, u8::MAX as i8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_adds_epu8() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(u8::MAX as i8); + let r = _mm_maskz_adds_epu8(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_adds_epu8(0b00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, u8::MAX as i8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_adds_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(i16::MAX); + let r = _mm512_adds_epi16(a, b); + let e = _mm512_set1_epi16(i16::MAX); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_adds_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(i16::MAX); + let r = _mm512_mask_adds_epi16(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_adds_epi16(a, 0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, i16::MAX, i16::MAX, i16::MAX, i16::MAX); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_adds_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(i16::MAX); + let r = _mm512_maskz_adds_epi16(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_adds_epi16(0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, i16::MAX, i16::MAX, i16::MAX, i16::MAX); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_adds_epi16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(i16::MAX); + let r = _mm256_mask_adds_epi16(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_adds_epi16(a, 0b00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, i16::MAX, i16::MAX, i16::MAX, i16::MAX); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_adds_epi16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(i16::MAX); + let r = _mm256_maskz_adds_epi16(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_adds_epi16(0b00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, i16::MAX, i16::MAX, i16::MAX, i16::MAX); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_adds_epi16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(i16::MAX); + let r = _mm_mask_adds_epi16(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_adds_epi16(a, 0b00001111, a, b); + let e = _mm_set_epi16(1, 1, 1, 1, i16::MAX, i16::MAX, i16::MAX, i16::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_adds_epi16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(i16::MAX); + let r = _mm_maskz_adds_epi16(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_adds_epi16(0b00001111, a, b); + let e = _mm_set_epi16(0, 0, 0, 0, i16::MAX, i16::MAX, i16::MAX, i16::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_adds_epi8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(i8::MAX); + let r = _mm512_adds_epi8(a, b); + let e = _mm512_set1_epi8(i8::MAX); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_adds_epi8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(i8::MAX); + let r = _mm512_mask_adds_epi8(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_adds_epi8( + a, + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00001111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, i8::MAX, i8::MAX, i8::MAX, i8::MAX); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_adds_epi8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(i8::MAX); + let r = _mm512_maskz_adds_epi8(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_adds_epi8( + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00001111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, i8::MAX, i8::MAX, i8::MAX, i8::MAX); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_adds_epi8() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(i8::MAX); + let r = _mm256_mask_adds_epi8(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_adds_epi8(a, 0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, i8::MAX, i8::MAX, i8::MAX, i8::MAX); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_adds_epi8() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(i8::MAX); + let r = _mm256_maskz_adds_epi8(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_adds_epi8(0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, i8::MAX, i8::MAX, i8::MAX, i8::MAX); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_adds_epi8() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(i8::MAX); + let r = _mm_mask_adds_epi8(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_adds_epi8(a, 0b00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, i8::MAX, i8::MAX, i8::MAX, i8::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_adds_epi8() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(i8::MAX); + let r = _mm_maskz_adds_epi8(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_adds_epi8(0b00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, i8::MAX, i8::MAX, i8::MAX, i8::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_sub_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(2); + let r = _mm512_sub_epi16(a, b); + let e = _mm512_set1_epi16(-1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_sub_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(2); + let r = _mm512_mask_sub_epi16(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_sub_epi16(a, 0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1, -1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_sub_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(2); + let r = _mm512_maskz_sub_epi16(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_sub_epi16(0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_sub_epi16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(2); + let r = _mm256_mask_sub_epi16(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_sub_epi16(a, 0b00000000_11111111, a, b); + let e = _mm256_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1, -1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_sub_epi16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(2); + let r = _mm256_maskz_sub_epi16(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_sub_epi16(0b00000000_11111111, a, b); + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_sub_epi16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(2); + let r = _mm_mask_sub_epi16(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_sub_epi16(a, 0b00001111, a, b); + let e = _mm_set_epi16(1, 1, 1, 1, -1, -1, -1, -1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_sub_epi16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(2); + let r = _mm_maskz_sub_epi16(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_sub_epi16(0b00001111, a, b); + let e = _mm_set_epi16(0, 0, 0, 0, -1, -1, -1, -1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_sub_epi8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(2); + let r = _mm512_sub_epi8(a, b); + let e = _mm512_set1_epi8(-1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_sub_epi8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(2); + let r = _mm512_mask_sub_epi8(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_sub_epi8( + a, + 0b00000000_11111111_00000000_11111111_00000000_11111111_00000000_11111111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1, -1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_sub_epi8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(2); + let r = _mm512_maskz_sub_epi8(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_sub_epi8( + 0b00000000_11111111_00000000_11111111_00000000_11111111_00000000_11111111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_sub_epi8() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(2); + let r = _mm256_mask_sub_epi8(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_sub_epi8(a, 0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1, -1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_sub_epi8() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(2); + let r = _mm256_maskz_sub_epi8(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_sub_epi8(0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_sub_epi8() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(2); + let r = _mm_mask_sub_epi8(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_sub_epi8(a, 0b00000000_11111111, a, b); + let e = _mm_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1, -1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_sub_epi8() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(2); + let r = _mm_maskz_sub_epi8(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_sub_epi8(0b00000000_11111111, a, b); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_subs_epu16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(u16::MAX as i16); + let r = _mm512_subs_epu16(a, b); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_subs_epu16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(u16::MAX as i16); + let r = _mm512_mask_subs_epu16(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_subs_epu16(a, 0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_subs_epu16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(u16::MAX as i16); + let r = _mm512_maskz_subs_epu16(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_subs_epu16(0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_subs_epu16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(u16::MAX as i16); + let r = _mm256_mask_subs_epu16(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_subs_epu16(a, 0b00000000_00001111, a, b); + let e = _mm256_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_subs_epu16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(u16::MAX as i16); + let r = _mm256_maskz_subs_epu16(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_subs_epu16(0b00000000_00001111, a, b); + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_subs_epu16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(u16::MAX as i16); + let r = _mm_mask_subs_epu16(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_subs_epu16(a, 0b00001111, a, b); + let e = _mm_set_epi16(1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_subs_epu16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(u16::MAX as i16); + let r = _mm_maskz_subs_epu16(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_subs_epu16(0b00001111, a, b); + let e = _mm_set_epi16(0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_subs_epu8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(u8::MAX as i8); + let r = _mm512_subs_epu8(a, b); + let e = _mm512_set1_epi8(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_subs_epu8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(u8::MAX as i8); + let r = _mm512_mask_subs_epu8(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_subs_epu8( + a, + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00001111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_subs_epu8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(u8::MAX as i8); + let r = _mm512_maskz_subs_epu8(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_subs_epu8( + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00001111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_subs_epu8() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(u8::MAX as i8); + let r = _mm256_mask_subs_epu8(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_subs_epu8(a, 0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_subs_epu8() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(u8::MAX as i8); + let r = _mm256_maskz_subs_epu8(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_subs_epu8(0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_subs_epu8() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(u8::MAX as i8); + let r = _mm_mask_subs_epu8(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_subs_epu8(a, 0b00000000_00001111, a, b); + let e = _mm_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_subs_epu8() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(u8::MAX as i8); + let r = _mm_maskz_subs_epu8(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_subs_epu8(0b00000000_00001111, a, b); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_subs_epi16() { + let a = _mm512_set1_epi16(-1); + let b = _mm512_set1_epi16(i16::MAX); + let r = _mm512_subs_epi16(a, b); + let e = _mm512_set1_epi16(i16::MIN); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_subs_epi16() { + let a = _mm512_set1_epi16(-1); + let b = _mm512_set1_epi16(i16::MAX); + let r = _mm512_mask_subs_epi16(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_subs_epi16(a, 0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, i16::MIN, i16::MIN, i16::MIN, i16::MIN); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_subs_epi16() { + let a = _mm512_set1_epi16(-1); + let b = _mm512_set1_epi16(i16::MAX); + let r = _mm512_maskz_subs_epi16(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_subs_epi16(0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, i16::MIN, i16::MIN, i16::MIN, i16::MIN); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_subs_epi16() { + let a = _mm256_set1_epi16(-1); + let b = _mm256_set1_epi16(i16::MAX); + let r = _mm256_mask_subs_epi16(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_subs_epi16(a, 0b00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi16(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, i16::MIN, i16::MIN, i16::MIN, i16::MIN); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_subs_epi16() { + let a = _mm256_set1_epi16(-1); + let b = _mm256_set1_epi16(i16::MAX); + let r = _mm256_maskz_subs_epi16(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_subs_epi16(0b00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, i16::MIN, i16::MIN, i16::MIN, i16::MIN); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_subs_epi16() { + let a = _mm_set1_epi16(-1); + let b = _mm_set1_epi16(i16::MAX); + let r = _mm_mask_subs_epi16(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_subs_epi16(a, 0b00001111, a, b); + let e = _mm_set_epi16(-1, -1, -1, -1, i16::MIN, i16::MIN, i16::MIN, i16::MIN); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_subs_epi16() { + let a = _mm_set1_epi16(-1); + let b = _mm_set1_epi16(i16::MAX); + let r = _mm_maskz_subs_epi16(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_subs_epi16(0b00001111, a, b); + let e = _mm_set_epi16(0, 0, 0, 0, i16::MIN, i16::MIN, i16::MIN, i16::MIN); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_subs_epi8() { + let a = _mm512_set1_epi8(-1); + let b = _mm512_set1_epi8(i8::MAX); + let r = _mm512_subs_epi8(a, b); + let e = _mm512_set1_epi8(i8::MIN); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_subs_epi8() { + let a = _mm512_set1_epi8(-1); + let b = _mm512_set1_epi8(i8::MAX); + let r = _mm512_mask_subs_epi8(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_subs_epi8( + a, + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00001111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, i8::MIN, i8::MIN, i8::MIN, i8::MIN); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_subs_epi8() { + let a = _mm512_set1_epi8(-1); + let b = _mm512_set1_epi8(i8::MAX); + let r = _mm512_maskz_subs_epi8(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_subs_epi8( + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00001111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, i8::MIN, i8::MIN, i8::MIN, i8::MIN); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_subs_epi8() { + let a = _mm256_set1_epi8(-1); + let b = _mm256_set1_epi8(i8::MAX); + let r = _mm256_mask_subs_epi8(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_subs_epi8(a, 0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, i8::MIN, i8::MIN, i8::MIN, i8::MIN); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_subs_epi8() { + let a = _mm256_set1_epi8(-1); + let b = _mm256_set1_epi8(i8::MAX); + let r = _mm256_maskz_subs_epi8(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_subs_epi8(0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, i8::MIN, i8::MIN, i8::MIN, i8::MIN); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_subs_epi8() { + let a = _mm_set1_epi8(-1); + let b = _mm_set1_epi8(i8::MAX); + let r = _mm_mask_subs_epi8(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_subs_epi8(a, 0b00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, i8::MIN, i8::MIN, i8::MIN, i8::MIN); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_subs_epi8() { + let a = _mm_set1_epi8(-1); + let b = _mm_set1_epi8(i8::MAX); + let r = _mm_maskz_subs_epi8(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_subs_epi8(0b00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, i8::MIN, i8::MIN, i8::MIN, i8::MIN); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mulhi_epu16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let r = _mm512_mulhi_epu16(a, b); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_mulhi_epu16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let r = _mm512_mask_mulhi_epu16(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_mulhi_epu16(a, 0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_mulhi_epu16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let r = _mm512_maskz_mulhi_epu16(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_mulhi_epu16(0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_mulhi_epu16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1); + let r = _mm256_mask_mulhi_epu16(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_mulhi_epu16(a, 0b00000000_00001111, a, b); + let e = _mm256_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_mulhi_epu16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1); + let r = _mm256_maskz_mulhi_epu16(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_mulhi_epu16(0b00000000_00001111, a, b); + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_mulhi_epu16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1); + let r = _mm_mask_mulhi_epu16(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_mulhi_epu16(a, 0b00001111, a, b); + let e = _mm_set_epi16(1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_mulhi_epu16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1); + let r = _mm_maskz_mulhi_epu16(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_mulhi_epu16(0b00001111, a, b); + let e = _mm_set_epi16(0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mulhi_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let r = _mm512_mulhi_epi16(a, b); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_mulhi_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let r = _mm512_mask_mulhi_epi16(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_mulhi_epi16(a, 0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_mulhi_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let r = _mm512_maskz_mulhi_epi16(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_mulhi_epi16(0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_mulhi_epi16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1); + let r = _mm256_mask_mulhi_epi16(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_mulhi_epi16(a, 0b00000000_00001111, a, b); + let e = _mm256_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_mulhi_epi16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1); + let r = _mm256_maskz_mulhi_epi16(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_mulhi_epi16(0b00000000_00001111, a, b); + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_mulhi_epi16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1); + let r = _mm_mask_mulhi_epi16(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_mulhi_epi16(a, 0b00001111, a, b); + let e = _mm_set_epi16(1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_mulhi_epi16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1); + let r = _mm_maskz_mulhi_epi16(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_mulhi_epi16(0b00001111, a, b); + let e = _mm_set_epi16(0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mulhrs_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let r = _mm512_mulhrs_epi16(a, b); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_mulhrs_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let r = _mm512_mask_mulhrs_epi16(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_mulhrs_epi16(a, 0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_mulhrs_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let r = _mm512_maskz_mulhrs_epi16(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_mulhrs_epi16(0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_mulhrs_epi16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1); + let r = _mm256_mask_mulhrs_epi16(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_mulhrs_epi16(a, 0b00000000_00001111, a, b); + let e = _mm256_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_mulhrs_epi16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1); + let r = _mm256_maskz_mulhrs_epi16(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_mulhrs_epi16(0b00000000_00001111, a, b); + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_mulhrs_epi16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1); + let r = _mm_mask_mulhrs_epi16(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_mulhrs_epi16(a, 0b00001111, a, b); + let e = _mm_set_epi16(1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_mulhrs_epi16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1); + let r = _mm_maskz_mulhrs_epi16(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_mulhrs_epi16(0b00001111, a, b); + let e = _mm_set_epi16(0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mullo_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let r = _mm512_mullo_epi16(a, b); + let e = _mm512_set1_epi16(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_mullo_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let r = _mm512_mask_mullo_epi16(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_mullo_epi16(a, 0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_mullo_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let r = _mm512_maskz_mullo_epi16(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_mullo_epi16(0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_mullo_epi16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1); + let r = _mm256_mask_mullo_epi16(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_mullo_epi16(a, 0b00000000_00001111, a, b); + let e = _mm256_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_mullo_epi16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1); + let r = _mm256_maskz_mullo_epi16(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_mullo_epi16(0b00000000_00001111, a, b); + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_mullo_epi16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1); + let r = _mm_mask_mullo_epi16(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_mullo_epi16(a, 0b00001111, a, b); + let e = _mm_set_epi16(1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_mullo_epi16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1); + let r = _mm_maskz_mullo_epi16(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_mullo_epi16(0b00001111, a, b); + let e = _mm_set_epi16(0, 0, 0, 0, 1, 1, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_max_epu16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_max_epu16(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 8, 9, 10, 11, 12, 13, 14, 15, + 15, 14, 13, 12, 11, 10, 9, 8, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_max_epu16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_mask_max_epu16(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_max_epu16(a, 0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_max_epu16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_maskz_max_epu16(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_max_epu16(0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_max_epu16() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm256_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_mask_max_epu16(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_max_epu16(a, 0b00000000_11111111, a, b); + let e = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_max_epu16() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm256_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_maskz_max_epu16(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_max_epu16(0b00000000_11111111, a, b); + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_max_epu16() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm_set_epi16(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm_mask_max_epu16(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_max_epu16(a, 0b00001111, a, b); + let e = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_max_epu16() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm_set_epi16(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm_maskz_max_epu16(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_max_epu16(0b00001111, a, b); + let e = _mm_set_epi16(0, 0, 0, 0, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_max_epu8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_max_epu8(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 8, 9, 10, 11, 12, 13, 14, 15, + 15, 14, 13, 12, 11, 10, 9, 8, 8, 9, 10, 11, 12, 13, 14, 15, + 15, 14, 13, 12, 11, 10, 9, 8, 8, 9, 10, 11, 12, 13, 14, 15, + 15, 14, 13, 12, 11, 10, 9, 8, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_max_epu8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_mask_max_epu8(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_max_epu8( + a, + 0b00000000_11111111_00000000_11111111_00000000_11111111_00000000_11111111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_max_epu8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_maskz_max_epu8(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_max_epu8( + 0b00000000_11111111_00000000_11111111_00000000_11111111_00000000_11111111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_max_epu8() { + #[rustfmt::skip] + let a = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm256_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_mask_max_epu8(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_max_epu8(a, 0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_max_epu8() { + #[rustfmt::skip] + let a = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm256_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_maskz_max_epu8(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_max_epu8(0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_max_epu8() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm_mask_max_epu8(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_max_epu8(a, 0b00000000_11111111, a, b); + let e = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_max_epu8() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm_maskz_max_epu8(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_max_epu8(0b00000000_11111111, a, b); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_max_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_max_epi16(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 8, 9, 10, 11, 12, 13, 14, 15, + 15, 14, 13, 12, 11, 10, 9, 8, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_max_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_mask_max_epi16(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_max_epi16(a, 0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_max_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_maskz_max_epi16(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_max_epi16(0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_max_epi16() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm256_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_mask_max_epi16(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_max_epi16(a, 0b00000000_11111111, a, b); + let e = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_max_epi16() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm256_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_maskz_max_epi16(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_max_epi16(0b00000000_11111111, a, b); + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_max_epi16() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm_set_epi16(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm_mask_max_epi16(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_max_epi16(a, 0b00001111, a, b); + let e = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_max_epi16() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm_set_epi16(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm_maskz_max_epi16(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_max_epi16(0b00001111, a, b); + let e = _mm_set_epi16(0, 0, 0, 0, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_max_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_max_epi8(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 8, 9, 10, 11, 12, 13, 14, 15, + 15, 14, 13, 12, 11, 10, 9, 8, 8, 9, 10, 11, 12, 13, 14, 15, + 15, 14, 13, 12, 11, 10, 9, 8, 8, 9, 10, 11, 12, 13, 14, 15, + 15, 14, 13, 12, 11, 10, 9, 8, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_max_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_mask_max_epi8(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_max_epi8( + a, + 0b00000000_11111111_00000000_11111111_00000000_11111111_00000000_11111111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_max_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_maskz_max_epi8(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_max_epi8( + 0b00000000_11111111_00000000_11111111_00000000_11111111_00000000_11111111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_max_epi8() { + #[rustfmt::skip] + let a = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm256_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_mask_max_epi8(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_max_epi8(a, 0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_max_epi8() { + #[rustfmt::skip] + let a = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm256_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_maskz_max_epi8(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_max_epi8(0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_max_epi8() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm_mask_max_epi8(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_max_epi8(a, 0b00000000_11111111, a, b); + let e = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_max_epi8() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm_maskz_max_epi8(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_max_epi8(0b00000000_11111111, a, b); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_min_epu16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_min_epu16(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_min_epu16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_mask_min_epu16(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_min_epu16(a, 0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_min_epu16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_maskz_min_epu16(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_min_epu16(0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_min_epu16() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm256_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_mask_min_epu16(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_min_epu16(a, 0b00000000_11111111, a, b); + let e = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_min_epu16() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm256_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_maskz_min_epu16(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_min_epu16(0b00000000_11111111, a, b); + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_min_epu16() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm_set_epi16(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm_mask_min_epu16(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_min_epu16(a, 0b00001111, a, b); + let e = _mm_set_epi16(0, 1, 2, 3, 3, 2, 1, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_min_epu16() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm_set_epi16(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm_maskz_min_epu16(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_min_epu16(0b00001111, a, b); + let e = _mm_set_epi16(0, 0, 0, 0, 3, 2, 1, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_min_epu8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_min_epu8(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_min_epu8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_mask_min_epu8(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_min_epu8( + a, + 0b00000000_11111111_00000000_11111111_00000000_11111111_00000000_11111111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_min_epu8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_maskz_min_epu8(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_min_epu8( + 0b00000000_11111111_00000000_11111111_00000000_11111111_00000000_11111111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_min_epu8() { + #[rustfmt::skip] + let a = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm256_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_mask_min_epu8(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_min_epu8(a, 0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_min_epu8() { + #[rustfmt::skip] + let a = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm256_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_maskz_min_epu8(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_min_epu8(0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_min_epu8() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm_mask_min_epu8(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_min_epu8(a, 0b00000000_11111111, a, b); + let e = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_min_epu8() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm_maskz_min_epu8(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_min_epu8(0b00000000_11111111, a, b); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_min_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_min_epi16(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_min_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_mask_min_epi16(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_min_epi16(a, 0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_min_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_maskz_min_epi16(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_min_epi16(0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_min_epi16() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm256_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_mask_min_epi16(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_min_epi16(a, 0b00000000_11111111, a, b); + let e = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_min_epi16() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm256_set_epi16(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_maskz_min_epi16(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_min_epi16(0b00000000_11111111, a, b); + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_min_epi16() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm_set_epi16(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm_mask_min_epi16(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_min_epi16(a, 0b00001111, a, b); + let e = _mm_set_epi16(0, 1, 2, 3, 3, 2, 1, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_min_epi16() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm_set_epi16(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm_maskz_min_epi16(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_min_epi16(0b00001111, a, b); + let e = _mm_set_epi16(0, 0, 0, 0, 3, 2, 1, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_min_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_min_epi8(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_min_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_mask_min_epi8(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_min_epi8( + a, + 0b00000000_11111111_00000000_11111111_00000000_11111111_00000000_11111111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_min_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm512_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_maskz_min_epi8(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_min_epi8( + 0b00000000_11111111_00000000_11111111_00000000_11111111_00000000_11111111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_min_epi8() { + #[rustfmt::skip] + let a = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm256_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_mask_min_epi8(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_min_epi8(a, 0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_min_epi8() { + #[rustfmt::skip] + let a = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm256_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_maskz_min_epi8(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_min_epi8(0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 5, 4, 3, 2, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_min_epi8() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm_mask_min_epi8(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_min_epi8(a, 0b00000000_11111111, a, b); + let e = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_min_epi8() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm_maskz_min_epi8(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_min_epi8(0b00000000_11111111, a, b); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmplt_epu16_mask() { + let a = _mm512_set1_epi16(-2); + let b = _mm512_set1_epi16(-1); + let m = _mm512_cmplt_epu16_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmplt_epu16_mask() { + let a = _mm512_set1_epi16(-2); + let b = _mm512_set1_epi16(-1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmplt_epu16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmplt_epu16_mask() { + let a = _mm256_set1_epi16(-2); + let b = _mm256_set1_epi16(-1); + let m = _mm256_cmplt_epu16_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmplt_epu16_mask() { + let a = _mm256_set1_epi16(-2); + let b = _mm256_set1_epi16(-1); + let mask = 0b01010101_01010101; + let r = _mm256_mask_cmplt_epu16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmplt_epu16_mask() { + let a = _mm_set1_epi16(-2); + let b = _mm_set1_epi16(-1); + let m = _mm_cmplt_epu16_mask(a, b); + assert_eq!(m, 0b11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmplt_epu16_mask() { + let a = _mm_set1_epi16(-2); + let b = _mm_set1_epi16(-1); + let mask = 0b01010101; + let r = _mm_mask_cmplt_epu16_mask(mask, a, b); + assert_eq!(r, 0b01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmplt_epu8_mask() { + let a = _mm512_set1_epi8(-2); + let b = _mm512_set1_epi8(-1); + let m = _mm512_cmplt_epu8_mask(a, b); + assert_eq!( + m, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111 + ); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmplt_epu8_mask() { + let a = _mm512_set1_epi8(-2); + let b = _mm512_set1_epi8(-1); + let mask = 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmplt_epu8_mask(mask, a, b); + assert_eq!( + r, + 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101 + ); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmplt_epu8_mask() { + let a = _mm256_set1_epi8(-2); + let b = _mm256_set1_epi8(-1); + let m = _mm256_cmplt_epu8_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmplt_epu8_mask() { + let a = _mm256_set1_epi8(-2); + let b = _mm256_set1_epi8(-1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm256_mask_cmplt_epu8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmplt_epu8_mask() { + let a = _mm_set1_epi8(-2); + let b = _mm_set1_epi8(-1); + let m = _mm_cmplt_epu8_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmplt_epu8_mask() { + let a = _mm_set1_epi8(-2); + let b = _mm_set1_epi8(-1); + let mask = 0b01010101_01010101; + let r = _mm_mask_cmplt_epu8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmplt_epi16_mask() { + let a = _mm512_set1_epi16(-2); + let b = _mm512_set1_epi16(-1); + let m = _mm512_cmplt_epi16_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmplt_epi16_mask() { + let a = _mm512_set1_epi16(-2); + let b = _mm512_set1_epi16(-1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmplt_epi16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmplt_epi16_mask() { + let a = _mm256_set1_epi16(-2); + let b = _mm256_set1_epi16(-1); + let m = _mm256_cmplt_epi16_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmplt_epi16_mask() { + let a = _mm256_set1_epi16(-2); + let b = _mm256_set1_epi16(-1); + let mask = 0b01010101_01010101; + let r = _mm256_mask_cmplt_epi16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmplt_epi16_mask() { + let a = _mm_set1_epi16(-2); + let b = _mm_set1_epi16(-1); + let m = _mm_cmplt_epi16_mask(a, b); + assert_eq!(m, 0b11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmplt_epi16_mask() { + let a = _mm_set1_epi16(-2); + let b = _mm_set1_epi16(-1); + let mask = 0b01010101; + let r = _mm_mask_cmplt_epi16_mask(mask, a, b); + assert_eq!(r, 0b01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmplt_epi8_mask() { + let a = _mm512_set1_epi8(-2); + let b = _mm512_set1_epi8(-1); + let m = _mm512_cmplt_epi8_mask(a, b); + assert_eq!( + m, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111 + ); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmplt_epi8_mask() { + let a = _mm512_set1_epi8(-2); + let b = _mm512_set1_epi8(-1); + let mask = 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmplt_epi8_mask(mask, a, b); + assert_eq!( + r, + 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101 + ); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmplt_epi8_mask() { + let a = _mm256_set1_epi8(-2); + let b = _mm256_set1_epi8(-1); + let m = _mm256_cmplt_epi8_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmplt_epi8_mask() { + let a = _mm256_set1_epi8(-2); + let b = _mm256_set1_epi8(-1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm256_mask_cmplt_epi8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmplt_epi8_mask() { + let a = _mm_set1_epi8(-2); + let b = _mm_set1_epi8(-1); + let m = _mm_cmplt_epi8_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmplt_epi8_mask() { + let a = _mm_set1_epi8(-2); + let b = _mm_set1_epi8(-1); + let mask = 0b01010101_01010101; + let r = _mm_mask_cmplt_epi8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmpgt_epu16_mask() { + let a = _mm512_set1_epi16(2); + let b = _mm512_set1_epi16(1); + let m = _mm512_cmpgt_epu16_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmpgt_epu16_mask() { + let a = _mm512_set1_epi16(2); + let b = _mm512_set1_epi16(1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmpgt_epu16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmpgt_epu16_mask() { + let a = _mm256_set1_epi16(2); + let b = _mm256_set1_epi16(1); + let m = _mm256_cmpgt_epu16_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmpgt_epu16_mask() { + let a = _mm256_set1_epi16(2); + let b = _mm256_set1_epi16(1); + let mask = 0b01010101_01010101; + let r = _mm256_mask_cmpgt_epu16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmpgt_epu16_mask() { + let a = _mm_set1_epi16(2); + let b = _mm_set1_epi16(1); + let m = _mm_cmpgt_epu16_mask(a, b); + assert_eq!(m, 0b11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmpgt_epu16_mask() { + let a = _mm_set1_epi16(2); + let b = _mm_set1_epi16(1); + let mask = 0b01010101; + let r = _mm_mask_cmpgt_epu16_mask(mask, a, b); + assert_eq!(r, 0b01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmpgt_epu8_mask() { + let a = _mm512_set1_epi8(2); + let b = _mm512_set1_epi8(1); + let m = _mm512_cmpgt_epu8_mask(a, b); + assert_eq!( + m, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111 + ); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmpgt_epu8_mask() { + let a = _mm512_set1_epi8(2); + let b = _mm512_set1_epi8(1); + let mask = 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmpgt_epu8_mask(mask, a, b); + assert_eq!( + r, + 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101 + ); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmpgt_epu8_mask() { + let a = _mm256_set1_epi8(2); + let b = _mm256_set1_epi8(1); + let m = _mm256_cmpgt_epu8_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmpgt_epu8_mask() { + let a = _mm256_set1_epi8(2); + let b = _mm256_set1_epi8(1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm256_mask_cmpgt_epu8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmpgt_epu8_mask() { + let a = _mm_set1_epi8(2); + let b = _mm_set1_epi8(1); + let m = _mm_cmpgt_epu8_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmpgt_epu8_mask() { + let a = _mm_set1_epi8(2); + let b = _mm_set1_epi8(1); + let mask = 0b01010101_01010101; + let r = _mm_mask_cmpgt_epu8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmpgt_epi16_mask() { + let a = _mm512_set1_epi16(2); + let b = _mm512_set1_epi16(-1); + let m = _mm512_cmpgt_epi16_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmpgt_epi16_mask() { + let a = _mm512_set1_epi16(2); + let b = _mm512_set1_epi16(-1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmpgt_epi16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmpgt_epi16_mask() { + let a = _mm256_set1_epi16(2); + let b = _mm256_set1_epi16(-1); + let m = _mm256_cmpgt_epi16_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmpgt_epi16_mask() { + let a = _mm256_set1_epi16(2); + let b = _mm256_set1_epi16(-1); + let mask = 0b001010101_01010101; + let r = _mm256_mask_cmpgt_epi16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmpgt_epi16_mask() { + let a = _mm_set1_epi16(2); + let b = _mm_set1_epi16(-1); + let m = _mm_cmpgt_epi16_mask(a, b); + assert_eq!(m, 0b11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmpgt_epi16_mask() { + let a = _mm_set1_epi16(2); + let b = _mm_set1_epi16(-1); + let mask = 0b01010101; + let r = _mm_mask_cmpgt_epi16_mask(mask, a, b); + assert_eq!(r, 0b01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmpgt_epi8_mask() { + let a = _mm512_set1_epi8(2); + let b = _mm512_set1_epi8(-1); + let m = _mm512_cmpgt_epi8_mask(a, b); + assert_eq!( + m, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111 + ); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmpgt_epi8_mask() { + let a = _mm512_set1_epi8(2); + let b = _mm512_set1_epi8(-1); + let mask = 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmpgt_epi8_mask(mask, a, b); + assert_eq!( + r, + 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101 + ); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmpgt_epi8_mask() { + let a = _mm256_set1_epi8(2); + let b = _mm256_set1_epi8(-1); + let m = _mm256_cmpgt_epi8_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmpgt_epi8_mask() { + let a = _mm256_set1_epi8(2); + let b = _mm256_set1_epi8(-1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm256_mask_cmpgt_epi8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmpgt_epi8_mask() { + let a = _mm_set1_epi8(2); + let b = _mm_set1_epi8(-1); + let m = _mm_cmpgt_epi8_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmpgt_epi8_mask() { + let a = _mm_set1_epi8(2); + let b = _mm_set1_epi8(-1); + let mask = 0b01010101_01010101; + let r = _mm_mask_cmpgt_epi8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmple_epu16_mask() { + let a = _mm512_set1_epi16(-1); + let b = _mm512_set1_epi16(-1); + let m = _mm512_cmple_epu16_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmple_epu16_mask() { + let a = _mm512_set1_epi16(-1); + let b = _mm512_set1_epi16(-1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmple_epu16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmple_epu16_mask() { + let a = _mm256_set1_epi16(-1); + let b = _mm256_set1_epi16(-1); + let m = _mm256_cmple_epu16_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmple_epu16_mask() { + let a = _mm256_set1_epi16(-1); + let b = _mm256_set1_epi16(-1); + let mask = 0b01010101_01010101; + let r = _mm256_mask_cmple_epu16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmple_epu16_mask() { + let a = _mm_set1_epi16(-1); + let b = _mm_set1_epi16(-1); + let m = _mm_cmple_epu16_mask(a, b); + assert_eq!(m, 0b11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmple_epu16_mask() { + let a = _mm_set1_epi16(-1); + let b = _mm_set1_epi16(-1); + let mask = 0b01010101; + let r = _mm_mask_cmple_epu16_mask(mask, a, b); + assert_eq!(r, 0b01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmple_epu8_mask() { + let a = _mm512_set1_epi8(-1); + let b = _mm512_set1_epi8(-1); + let m = _mm512_cmple_epu8_mask(a, b); + assert_eq!( + m, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111 + ); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmple_epu8_mask() { + let a = _mm512_set1_epi8(-1); + let b = _mm512_set1_epi8(-1); + let mask = 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmple_epu8_mask(mask, a, b); + assert_eq!( + r, + 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101 + ); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmple_epu8_mask() { + let a = _mm256_set1_epi8(-1); + let b = _mm256_set1_epi8(-1); + let m = _mm256_cmple_epu8_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmple_epu8_mask() { + let a = _mm256_set1_epi8(-1); + let b = _mm256_set1_epi8(-1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm256_mask_cmple_epu8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmple_epu8_mask() { + let a = _mm_set1_epi8(-1); + let b = _mm_set1_epi8(-1); + let m = _mm_cmple_epu8_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmple_epu8_mask() { + let a = _mm_set1_epi8(-1); + let b = _mm_set1_epi8(-1); + let mask = 0b01010101_01010101; + let r = _mm_mask_cmple_epu8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmple_epi16_mask() { + let a = _mm512_set1_epi16(-1); + let b = _mm512_set1_epi16(-1); + let m = _mm512_cmple_epi16_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmple_epi16_mask() { + let a = _mm512_set1_epi16(-1); + let b = _mm512_set1_epi16(-1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmple_epi16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmple_epi16_mask() { + let a = _mm256_set1_epi16(-1); + let b = _mm256_set1_epi16(-1); + let m = _mm256_cmple_epi16_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmple_epi16_mask() { + let a = _mm256_set1_epi16(-1); + let b = _mm256_set1_epi16(-1); + let mask = 0b01010101_01010101; + let r = _mm256_mask_cmple_epi16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmple_epi16_mask() { + let a = _mm_set1_epi16(-1); + let b = _mm_set1_epi16(-1); + let m = _mm_cmple_epi16_mask(a, b); + assert_eq!(m, 0b11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmple_epi16_mask() { + let a = _mm_set1_epi16(-1); + let b = _mm_set1_epi16(-1); + let mask = 0b01010101; + let r = _mm_mask_cmple_epi16_mask(mask, a, b); + assert_eq!(r, 0b01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmple_epi8_mask() { + let a = _mm512_set1_epi8(-1); + let b = _mm512_set1_epi8(-1); + let m = _mm512_cmple_epi8_mask(a, b); + assert_eq!( + m, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111 + ); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmple_epi8_mask() { + let a = _mm512_set1_epi8(-1); + let b = _mm512_set1_epi8(-1); + let mask = 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmple_epi8_mask(mask, a, b); + assert_eq!( + r, + 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101 + ); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmple_epi8_mask() { + let a = _mm256_set1_epi8(-1); + let b = _mm256_set1_epi8(-1); + let m = _mm256_cmple_epi8_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmple_epi8_mask() { + let a = _mm256_set1_epi8(-1); + let b = _mm256_set1_epi8(-1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm256_mask_cmple_epi8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmple_epi8_mask() { + let a = _mm_set1_epi8(-1); + let b = _mm_set1_epi8(-1); + let m = _mm_cmple_epi8_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmple_epi8_mask() { + let a = _mm_set1_epi8(-1); + let b = _mm_set1_epi8(-1); + let mask = 0b01010101_01010101; + let r = _mm_mask_cmple_epi8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmpge_epu16_mask() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let m = _mm512_cmpge_epu16_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmpge_epu16_mask() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmpge_epu16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmpge_epu16_mask() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1); + let m = _mm256_cmpge_epu16_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmpge_epu16_mask() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1); + let mask = 0b01010101_01010101; + let r = _mm256_mask_cmpge_epu16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmpge_epu16_mask() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1); + let m = _mm_cmpge_epu16_mask(a, b); + assert_eq!(m, 0b11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmpge_epu16_mask() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1); + let mask = 0b01010101; + let r = _mm_mask_cmpge_epu16_mask(mask, a, b); + assert_eq!(r, 0b01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmpge_epu8_mask() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(1); + let m = _mm512_cmpge_epu8_mask(a, b); + assert_eq!( + m, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111 + ); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmpge_epu8_mask() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(1); + let mask = 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmpge_epu8_mask(mask, a, b); + assert_eq!( + r, + 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101 + ); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmpge_epu8_mask() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(1); + let m = _mm256_cmpge_epu8_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmpge_epu8_mask() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm256_mask_cmpge_epu8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmpge_epu8_mask() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(1); + let m = _mm_cmpge_epu8_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmpge_epu8_mask() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(1); + let mask = 0b01010101_01010101; + let r = _mm_mask_cmpge_epu8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmpge_epi16_mask() { + let a = _mm512_set1_epi16(-1); + let b = _mm512_set1_epi16(-1); + let m = _mm512_cmpge_epi16_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmpge_epi16_mask() { + let a = _mm512_set1_epi16(-1); + let b = _mm512_set1_epi16(-1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmpge_epi16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmpge_epi16_mask() { + let a = _mm256_set1_epi16(-1); + let b = _mm256_set1_epi16(-1); + let m = _mm256_cmpge_epi16_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmpge_epi16_mask() { + let a = _mm256_set1_epi16(-1); + let b = _mm256_set1_epi16(-1); + let mask = 0b01010101_01010101; + let r = _mm256_mask_cmpge_epi16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmpge_epi16_mask() { + let a = _mm_set1_epi16(-1); + let b = _mm_set1_epi16(-1); + let m = _mm_cmpge_epi16_mask(a, b); + assert_eq!(m, 0b11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmpge_epi16_mask() { + let a = _mm_set1_epi16(-1); + let b = _mm_set1_epi16(-1); + let mask = 0b01010101; + let r = _mm_mask_cmpge_epi16_mask(mask, a, b); + assert_eq!(r, 0b01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmpge_epi8_mask() { + let a = _mm512_set1_epi8(-1); + let b = _mm512_set1_epi8(-1); + let m = _mm512_cmpge_epi8_mask(a, b); + assert_eq!( + m, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111 + ); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmpge_epi8_mask() { + let a = _mm512_set1_epi8(-1); + let b = _mm512_set1_epi8(-1); + let mask = 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmpge_epi8_mask(mask, a, b); + assert_eq!( + r, + 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101 + ); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmpge_epi8_mask() { + let a = _mm256_set1_epi8(-1); + let b = _mm256_set1_epi8(-1); + let m = _mm256_cmpge_epi8_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmpge_epi8_mask() { + let a = _mm256_set1_epi8(-1); + let b = _mm256_set1_epi8(-1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm256_mask_cmpge_epi8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmpge_epi8_mask() { + let a = _mm_set1_epi8(-1); + let b = _mm_set1_epi8(-1); + let m = _mm_cmpge_epi8_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmpge_epi8_mask() { + let a = _mm_set1_epi8(-1); + let b = _mm_set1_epi8(-1); + let mask = 0b01010101_01010101; + let r = _mm_mask_cmpge_epi8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmpeq_epu16_mask() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let m = _mm512_cmpeq_epu16_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmpeq_epu16_mask() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmpeq_epu16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmpeq_epu16_mask() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1); + let m = _mm256_cmpeq_epu16_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmpeq_epu16_mask() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1); + let mask = 0b01010101_01010101; + let r = _mm256_mask_cmpeq_epu16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmpeq_epu16_mask() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1); + let m = _mm_cmpeq_epu16_mask(a, b); + assert_eq!(m, 0b11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmpeq_epu16_mask() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1); + let mask = 0b01010101; + let r = _mm_mask_cmpeq_epu16_mask(mask, a, b); + assert_eq!(r, 0b01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmpeq_epu8_mask() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(1); + let m = _mm512_cmpeq_epu8_mask(a, b); + assert_eq!( + m, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111 + ); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmpeq_epu8_mask() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(1); + let mask = 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmpeq_epu8_mask(mask, a, b); + assert_eq!( + r, + 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101 + ); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmpeq_epu8_mask() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(1); + let m = _mm256_cmpeq_epu8_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmpeq_epu8_mask() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm256_mask_cmpeq_epu8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmpeq_epu8_mask() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(1); + let m = _mm_cmpeq_epu8_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmpeq_epu8_mask() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(1); + let mask = 0b01010101_01010101; + let r = _mm_mask_cmpeq_epu8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmpeq_epi16_mask() { + let a = _mm512_set1_epi16(-1); + let b = _mm512_set1_epi16(-1); + let m = _mm512_cmpeq_epi16_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmpeq_epi16_mask() { + let a = _mm512_set1_epi16(-1); + let b = _mm512_set1_epi16(-1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmpeq_epi16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmpeq_epi16_mask() { + let a = _mm256_set1_epi16(-1); + let b = _mm256_set1_epi16(-1); + let m = _mm256_cmpeq_epi16_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmpeq_epi16_mask() { + let a = _mm256_set1_epi16(-1); + let b = _mm256_set1_epi16(-1); + let mask = 0b01010101_01010101; + let r = _mm256_mask_cmpeq_epi16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmpeq_epi16_mask() { + let a = _mm_set1_epi16(-1); + let b = _mm_set1_epi16(-1); + let m = _mm_cmpeq_epi16_mask(a, b); + assert_eq!(m, 0b11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmpeq_epi16_mask() { + let a = _mm_set1_epi16(-1); + let b = _mm_set1_epi16(-1); + let mask = 0b01010101; + let r = _mm_mask_cmpeq_epi16_mask(mask, a, b); + assert_eq!(r, 0b01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmpeq_epi8_mask() { + let a = _mm512_set1_epi8(-1); + let b = _mm512_set1_epi8(-1); + let m = _mm512_cmpeq_epi8_mask(a, b); + assert_eq!( + m, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111 + ); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmpeq_epi8_mask() { + let a = _mm512_set1_epi8(-1); + let b = _mm512_set1_epi8(-1); + let mask = 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmpeq_epi8_mask(mask, a, b); + assert_eq!( + r, + 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101 + ); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmpeq_epi8_mask() { + let a = _mm256_set1_epi8(-1); + let b = _mm256_set1_epi8(-1); + let m = _mm256_cmpeq_epi8_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmpeq_epi8_mask() { + let a = _mm256_set1_epi8(-1); + let b = _mm256_set1_epi8(-1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm256_mask_cmpeq_epi8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmpeq_epi8_mask() { + let a = _mm_set1_epi8(-1); + let b = _mm_set1_epi8(-1); + let m = _mm_cmpeq_epi8_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmpeq_epi8_mask() { + let a = _mm_set1_epi8(-1); + let b = _mm_set1_epi8(-1); + let mask = 0b01010101_01010101; + let r = _mm_mask_cmpeq_epi8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmpneq_epu16_mask() { + let a = _mm512_set1_epi16(2); + let b = _mm512_set1_epi16(1); + let m = _mm512_cmpneq_epu16_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmpneq_epu16_mask() { + let a = _mm512_set1_epi16(2); + let b = _mm512_set1_epi16(1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmpneq_epu16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmpneq_epu16_mask() { + let a = _mm256_set1_epi16(2); + let b = _mm256_set1_epi16(1); + let m = _mm256_cmpneq_epu16_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmpneq_epu16_mask() { + let a = _mm256_set1_epi16(2); + let b = _mm256_set1_epi16(1); + let mask = 0b01010101_01010101; + let r = _mm256_mask_cmpneq_epu16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmpneq_epu16_mask() { + let a = _mm_set1_epi16(2); + let b = _mm_set1_epi16(1); + let m = _mm_cmpneq_epu16_mask(a, b); + assert_eq!(m, 0b11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmpneq_epu16_mask() { + let a = _mm_set1_epi16(2); + let b = _mm_set1_epi16(1); + let mask = 0b01010101; + let r = _mm_mask_cmpneq_epu16_mask(mask, a, b); + assert_eq!(r, 0b01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmpneq_epu8_mask() { + let a = _mm512_set1_epi8(2); + let b = _mm512_set1_epi8(1); + let m = _mm512_cmpneq_epu8_mask(a, b); + assert_eq!( + m, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111 + ); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmpneq_epu8_mask() { + let a = _mm512_set1_epi8(2); + let b = _mm512_set1_epi8(1); + let mask = 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmpneq_epu8_mask(mask, a, b); + assert_eq!( + r, + 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101 + ); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmpneq_epu8_mask() { + let a = _mm256_set1_epi8(2); + let b = _mm256_set1_epi8(1); + let m = _mm256_cmpneq_epu8_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmpneq_epu8_mask() { + let a = _mm256_set1_epi8(2); + let b = _mm256_set1_epi8(1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm256_mask_cmpneq_epu8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmpneq_epu8_mask() { + let a = _mm_set1_epi8(2); + let b = _mm_set1_epi8(1); + let m = _mm_cmpneq_epu8_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmpneq_epu8_mask() { + let a = _mm_set1_epi8(2); + let b = _mm_set1_epi8(1); + let mask = 0b01010101_01010101; + let r = _mm_mask_cmpneq_epu8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmpneq_epi16_mask() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(-1); + let m = _mm512_cmpneq_epi16_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmpneq_epi16_mask() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(-1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmpneq_epi16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmpneq_epi16_mask() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(-1); + let m = _mm256_cmpneq_epi16_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmpneq_epi16_mask() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(-1); + let mask = 0b01010101_01010101; + let r = _mm256_mask_cmpneq_epi16_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmpneq_epi16_mask() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(-1); + let m = _mm_cmpneq_epi16_mask(a, b); + assert_eq!(m, 0b11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmpneq_epi16_mask() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(-1); + let mask = 0b01010101; + let r = _mm_mask_cmpneq_epi16_mask(mask, a, b); + assert_eq!(r, 0b01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmpneq_epi8_mask() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(-1); + let m = _mm512_cmpneq_epi8_mask(a, b); + assert_eq!( + m, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111 + ); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmpneq_epi8_mask() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(-1); + let mask = 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmpneq_epi8_mask(mask, a, b); + assert_eq!( + r, + 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101 + ); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmpneq_epi8_mask() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(-1); + let m = _mm256_cmpneq_epi8_mask(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmpneq_epi8_mask() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(-1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm256_mask_cmpneq_epi8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmpneq_epi8_mask() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(-1); + let m = _mm_cmpneq_epi8_mask(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmpneq_epi8_mask() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(-1); + let mask = 0b01010101_01010101; + let r = _mm_mask_cmpneq_epi8_mask(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmp_epu16_mask() { + let a = _mm512_set1_epi16(0); + let b = _mm512_set1_epi16(1); + let m = _mm512_cmp_epu16_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmp_epu16_mask() { + let a = _mm512_set1_epi16(0); + let b = _mm512_set1_epi16(1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmp_epu16_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmp_epu16_mask() { + let a = _mm256_set1_epi16(0); + let b = _mm256_set1_epi16(1); + let m = _mm256_cmp_epu16_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmp_epu16_mask() { + let a = _mm256_set1_epi16(0); + let b = _mm256_set1_epi16(1); + let mask = 0b01010101_01010101; + let r = _mm256_mask_cmp_epu16_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmp_epu16_mask() { + let a = _mm_set1_epi16(0); + let b = _mm_set1_epi16(1); + let m = _mm_cmp_epu16_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmp_epu16_mask() { + let a = _mm_set1_epi16(0); + let b = _mm_set1_epi16(1); + let mask = 0b01010101; + let r = _mm_mask_cmp_epu16_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmp_epu8_mask() { + let a = _mm512_set1_epi8(0); + let b = _mm512_set1_epi8(1); + let m = _mm512_cmp_epu8_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!( + m, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111 + ); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmp_epu8_mask() { + let a = _mm512_set1_epi8(0); + let b = _mm512_set1_epi8(1); + let mask = 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmp_epu8_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!( + r, + 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101 + ); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmp_epu8_mask() { + let a = _mm256_set1_epi8(0); + let b = _mm256_set1_epi8(1); + let m = _mm256_cmp_epu8_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmp_epu8_mask() { + let a = _mm256_set1_epi8(0); + let b = _mm256_set1_epi8(1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm256_mask_cmp_epu8_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmp_epu8_mask() { + let a = _mm_set1_epi8(0); + let b = _mm_set1_epi8(1); + let m = _mm_cmp_epu8_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmp_epu8_mask() { + let a = _mm_set1_epi8(0); + let b = _mm_set1_epi8(1); + let mask = 0b01010101_01010101; + let r = _mm_mask_cmp_epu8_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmp_epi16_mask() { + let a = _mm512_set1_epi16(0); + let b = _mm512_set1_epi16(1); + let m = _mm512_cmp_epi16_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmp_epi16_mask() { + let a = _mm512_set1_epi16(0); + let b = _mm512_set1_epi16(1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmp_epi16_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmp_epi16_mask() { + let a = _mm256_set1_epi16(0); + let b = _mm256_set1_epi16(1); + let m = _mm256_cmp_epi16_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmp_epi16_mask() { + let a = _mm256_set1_epi16(0); + let b = _mm256_set1_epi16(1); + let mask = 0b01010101_01010101; + let r = _mm256_mask_cmp_epi16_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmp_epi16_mask() { + let a = _mm_set1_epi16(0); + let b = _mm_set1_epi16(1); + let m = _mm_cmp_epi16_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmp_epi16_mask() { + let a = _mm_set1_epi16(0); + let b = _mm_set1_epi16(1); + let mask = 0b01010101; + let r = _mm_mask_cmp_epi16_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cmp_epi8_mask() { + let a = _mm512_set1_epi8(0); + let b = _mm512_set1_epi8(1); + let m = _mm512_cmp_epi8_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!( + m, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111 + ); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cmp_epi8_mask() { + let a = _mm512_set1_epi8(0); + let b = _mm512_set1_epi8(1); + let mask = 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101; + let r = _mm512_mask_cmp_epi8_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!( + r, + 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101 + ); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cmp_epi8_mask() { + let a = _mm256_set1_epi8(0); + let b = _mm256_set1_epi8(1); + let m = _mm256_cmp_epi8_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b11111111_11111111_11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cmp_epi8_mask() { + let a = _mm256_set1_epi8(0); + let b = _mm256_set1_epi8(1); + let mask = 0b01010101_01010101_01010101_01010101; + let r = _mm256_mask_cmp_epi8_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b01010101_01010101_01010101_01010101); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cmp_epi8_mask() { + let a = _mm_set1_epi8(0); + let b = _mm_set1_epi8(1); + let m = _mm_cmp_epi8_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b11111111_11111111); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cmp_epi8_mask() { + let a = _mm_set1_epi8(0); + let b = _mm_set1_epi8(1); + let mask = 0b01010101_01010101; + let r = _mm_mask_cmp_epi8_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b01010101_01010101); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_loadu_epi16() { + #[rustfmt::skip] + let a: [i16; 32] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let r = _mm512_loadu_epi16(&a[0]); + #[rustfmt::skip] + let e = _mm512_set_epi16(32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_loadu_epi16() { + let a: [i16; 16] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let r = _mm256_loadu_epi16(&a[0]); + let e = _mm256_set_epi16(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_loadu_epi16() { + let a: [i16; 8] = [1, 2, 3, 4, 5, 6, 7, 8]; + let r = _mm_loadu_epi16(&a[0]); + let e = _mm_set_epi16(8, 7, 6, 5, 4, 3, 2, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_loadu_epi8() { + #[rustfmt::skip] + let a: [i8; 64] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let r = _mm512_loadu_epi8(&a[0]); + #[rustfmt::skip] + let e = _mm512_set_epi8(32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, + 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_loadu_epi8() { + #[rustfmt::skip] + let a: [i8; 32] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; + let r = _mm256_loadu_epi8(&a[0]); + #[rustfmt::skip] + let e = _mm256_set_epi8(32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_loadu_epi8() { + let a: [i8; 16] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let r = _mm_loadu_epi8(&a[0]); + let e = _mm_set_epi8(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_storeu_epi16() { + let a = _mm512_set1_epi16(9); + let mut r = _mm512_undefined_epi32(); + _mm512_storeu_epi16(&mut r as *mut _ as *mut i16, a); + assert_eq_m512i(r, a); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_storeu_epi16() { + let a = _mm256_set1_epi16(9); + let mut r = _mm256_set1_epi32(0); + _mm256_storeu_epi16(&mut r as *mut _ as *mut i16, a); + assert_eq_m256i(r, a); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_storeu_epi16() { + let a = _mm_set1_epi16(9); + let mut r = _mm_set1_epi32(0); + _mm_storeu_epi16(&mut r as *mut _ as *mut i16, a); + assert_eq_m128i(r, a); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_storeu_epi8() { + let a = _mm512_set1_epi8(9); + let mut r = _mm512_undefined_epi32(); + _mm512_storeu_epi8(&mut r as *mut _ as *mut i8, a); + assert_eq_m512i(r, a); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_storeu_epi8() { + let a = _mm256_set1_epi8(9); + let mut r = _mm256_set1_epi32(0); + _mm256_storeu_epi8(&mut r as *mut _ as *mut i8, a); + assert_eq_m256i(r, a); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_storeu_epi8() { + let a = _mm_set1_epi8(9); + let mut r = _mm_set1_epi32(0); + _mm_storeu_epi8(&mut r as *mut _ as *mut i8, a); + assert_eq_m128i(r, a); + } + + #[simd_test(enable = "avx512f,avx512bw")] + unsafe fn test_mm512_mask_loadu_epi16() { + let src = _mm512_set1_epi16(42); + let a = &[ + 1_i16, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]; + let p = a.as_ptr(); + let m = 0b10101010_11001100_11101000_11001010; + let r = _mm512_mask_loadu_epi16(src, m, black_box(p)); + let e = &[ + 42_i16, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16, 42, 42, 19, 20, 42, 42, + 23, 24, 42, 26, 42, 28, 42, 30, 42, 32, + ]; + let e = _mm512_loadu_epi16(e.as_ptr()); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512bw")] + unsafe fn test_mm512_maskz_loadu_epi16() { + let a = &[ + 1_i16, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]; + let p = a.as_ptr(); + let m = 0b10101010_11001100_11101000_11001010; + let r = _mm512_maskz_loadu_epi16(m, black_box(p)); + let e = &[ + 0_i16, 2, 0, 4, 0, 0, 7, 8, 0, 0, 0, 12, 0, 14, 15, 16, 0, 0, 19, 20, 0, 0, 23, 24, 0, + 26, 0, 28, 0, 30, 0, 32, + ]; + let e = _mm512_loadu_epi16(e.as_ptr()); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512bw")] + unsafe fn test_mm512_mask_storeu_epi16() { + let mut r = [42_i16; 32]; + let a = &[ + 1_i16, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]; + let a = _mm512_loadu_epi16(a.as_ptr()); + let m = 0b10101010_11001100_11101000_11001010; + _mm512_mask_storeu_epi16(r.as_mut_ptr(), m, a); + let e = &[ + 42_i16, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16, 42, 42, 19, 20, 42, 42, + 23, 24, 42, 26, 42, 28, 42, 30, 42, 32, + ]; + let e = _mm512_loadu_epi16(e.as_ptr()); + assert_eq_m512i(_mm512_loadu_epi16(r.as_ptr()), e); + } + + #[simd_test(enable = "avx512f,avx512bw")] + unsafe fn test_mm512_mask_loadu_epi8() { + let src = _mm512_set1_epi8(42); + let a = &[ + 1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + ]; + let p = a.as_ptr(); + let m = 0b00000000_11111111_11111111_00000000_10101010_11001100_11101000_11001010; + let r = _mm512_mask_loadu_epi8(src, m, black_box(p)); + let e = &[ + 42_i8, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16, 42, 42, 19, 20, 42, 42, + 23, 24, 42, 26, 42, 28, 42, 30, 42, 32, 42, 42, 42, 42, 42, 42, 42, 42, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 42, 42, 42, 42, 42, 42, 42, 42, + ]; + let e = _mm512_loadu_epi8(e.as_ptr()); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512bw")] + unsafe fn test_mm512_maskz_loadu_epi8() { + let a = &[ + 1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + ]; + let p = a.as_ptr(); + let m = 0b00000000_11111111_11111111_00000000_10101010_11001100_11101000_11001010; + let r = _mm512_maskz_loadu_epi8(m, black_box(p)); + let e = &[ + 0_i8, 2, 0, 4, 0, 0, 7, 8, 0, 0, 0, 12, 0, 14, 15, 16, 0, 0, 19, 20, 0, 0, 23, 24, 0, + 26, 0, 28, 0, 30, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 41, 42, 43, 44, 45, 46, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + let e = _mm512_loadu_epi8(e.as_ptr()); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512bw")] + unsafe fn test_mm512_mask_storeu_epi8() { + let mut r = [42_i8; 64]; + let a = &[ + 1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + ]; + let a = _mm512_loadu_epi8(a.as_ptr()); + let m = 0b00000000_11111111_11111111_00000000_10101010_11001100_11101000_11001010; + _mm512_mask_storeu_epi8(r.as_mut_ptr(), m, a); + let e = &[ + 42_i8, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16, 42, 42, 19, 20, 42, 42, + 23, 24, 42, 26, 42, 28, 42, 30, 42, 32, 42, 42, 42, 42, 42, 42, 42, 42, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 42, 42, 42, 42, 42, 42, 42, 42, + ]; + let e = _mm512_loadu_epi8(e.as_ptr()); + assert_eq_m512i(_mm512_loadu_epi8(r.as_ptr()), e); + } + + #[simd_test(enable = "avx512f,avx512bw,avx512vl")] + unsafe fn test_mm256_mask_loadu_epi16() { + let src = _mm256_set1_epi16(42); + let a = &[1_i16, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let p = a.as_ptr(); + let m = 0b11101000_11001010; + let r = _mm256_mask_loadu_epi16(src, m, black_box(p)); + let e = &[ + 42_i16, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16, + ]; + let e = _mm256_loadu_epi16(e.as_ptr()); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_loadu_epi16() { + let a = &[1_i16, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let p = a.as_ptr(); + let m = 0b11101000_11001010; + let r = _mm256_maskz_loadu_epi16(m, black_box(p)); + let e = &[0_i16, 2, 0, 4, 0, 0, 7, 8, 0, 0, 0, 12, 0, 14, 15, 16]; + let e = _mm256_loadu_epi16(e.as_ptr()); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512bw,avx512vl")] + unsafe fn test_mm256_mask_storeu_epi16() { + let mut r = [42_i16; 16]; + let a = &[1_i16, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let a = _mm256_loadu_epi16(a.as_ptr()); + let m = 0b11101000_11001010; + _mm256_mask_storeu_epi16(r.as_mut_ptr(), m, a); + let e = &[ + 42_i16, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16, + ]; + let e = _mm256_loadu_epi16(e.as_ptr()); + assert_eq_m256i(_mm256_loadu_epi16(r.as_ptr()), e); + } + + #[simd_test(enable = "avx512f,avx512bw,avx512vl")] + unsafe fn test_mm256_mask_loadu_epi8() { + let src = _mm256_set1_epi8(42); + let a = &[ + 1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]; + let p = a.as_ptr(); + let m = 0b10101010_11001100_11101000_11001010; + let r = _mm256_mask_loadu_epi8(src, m, black_box(p)); + let e = &[ + 42_i8, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16, 42, 42, 19, 20, 42, 42, + 23, 24, 42, 26, 42, 28, 42, 30, 42, 32, + ]; + let e = _mm256_loadu_epi8(e.as_ptr()); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_loadu_epi8() { + let a = &[ + 1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]; + let p = a.as_ptr(); + let m = 0b10101010_11001100_11101000_11001010; + let r = _mm256_maskz_loadu_epi8(m, black_box(p)); + let e = &[ + 0_i8, 2, 0, 4, 0, 0, 7, 8, 0, 0, 0, 12, 0, 14, 15, 16, 0, 0, 19, 20, 0, 0, 23, 24, 0, + 26, 0, 28, 0, 30, 0, 32, + ]; + let e = _mm256_loadu_epi8(e.as_ptr()); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512bw,avx512vl")] + unsafe fn test_mm256_mask_storeu_epi8() { + let mut r = [42_i8; 32]; + let a = &[ + 1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]; + let a = _mm256_loadu_epi8(a.as_ptr()); + let m = 0b10101010_11001100_11101000_11001010; + _mm256_mask_storeu_epi8(r.as_mut_ptr(), m, a); + let e = &[ + 42_i8, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16, 42, 42, 19, 20, 42, 42, + 23, 24, 42, 26, 42, 28, 42, 30, 42, 32, + ]; + let e = _mm256_loadu_epi8(e.as_ptr()); + assert_eq_m256i(_mm256_loadu_epi8(r.as_ptr()), e); + } + + #[simd_test(enable = "avx512f,avx512bw,avx512vl")] + unsafe fn test_mm_mask_loadu_epi16() { + let src = _mm_set1_epi16(42); + let a = &[1_i16, 2, 3, 4, 5, 6, 7, 8]; + let p = a.as_ptr(); + let m = 0b11001010; + let r = _mm_mask_loadu_epi16(src, m, black_box(p)); + let e = &[42_i16, 2, 42, 4, 42, 42, 7, 8]; + let e = _mm_loadu_epi16(e.as_ptr()); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512bw,avx512vl")] + unsafe fn test_mm_maskz_loadu_epi16() { + let a = &[1_i16, 2, 3, 4, 5, 6, 7, 8]; + let p = a.as_ptr(); + let m = 0b11001010; + let r = _mm_maskz_loadu_epi16(m, black_box(p)); + let e = &[0_i16, 2, 0, 4, 0, 0, 7, 8]; + let e = _mm_loadu_epi16(e.as_ptr()); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512bw,avx512vl")] + unsafe fn test_mm_mask_storeu_epi16() { + let mut r = [42_i16; 8]; + let a = &[1_i16, 2, 3, 4, 5, 6, 7, 8]; + let a = _mm_loadu_epi16(a.as_ptr()); + let m = 0b11001010; + _mm_mask_storeu_epi16(r.as_mut_ptr(), m, a); + let e = &[42_i16, 2, 42, 4, 42, 42, 7, 8]; + let e = _mm_loadu_epi16(e.as_ptr()); + assert_eq_m128i(_mm_loadu_epi16(r.as_ptr()), e); + } + + #[simd_test(enable = "avx512f,avx512bw,avx512vl")] + unsafe fn test_mm_mask_loadu_epi8() { + let src = _mm_set1_epi8(42); + let a = &[1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let p = a.as_ptr(); + let m = 0b11101000_11001010; + let r = _mm_mask_loadu_epi8(src, m, black_box(p)); + let e = &[ + 42_i8, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16, + ]; + let e = _mm_loadu_epi8(e.as_ptr()); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512bw,avx512vl")] + unsafe fn test_mm_maskz_loadu_epi8() { + let a = &[1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let p = a.as_ptr(); + let m = 0b11101000_11001010; + let r = _mm_maskz_loadu_epi8(m, black_box(p)); + let e = &[0_i8, 2, 0, 4, 0, 0, 7, 8, 0, 0, 0, 12, 0, 14, 15, 16]; + let e = _mm_loadu_epi8(e.as_ptr()); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512bw,avx512vl")] + unsafe fn test_mm_mask_storeu_epi8() { + let mut r = [42_i8; 16]; + let a = &[1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let a = _mm_loadu_epi8(a.as_ptr()); + let m = 0b11101000_11001010; + _mm_mask_storeu_epi8(r.as_mut_ptr(), m, a); + let e = &[ + 42_i8, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16, + ]; + let e = _mm_loadu_epi8(e.as_ptr()); + assert_eq_m128i(_mm_loadu_epi8(r.as_ptr()), e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_madd_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let r = _mm512_madd_epi16(a, b); + let e = _mm512_set1_epi32(2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_madd_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let r = _mm512_mask_madd_epi16(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_madd_epi16(a, 0b00000000_00001111, a, b); + let e = _mm512_set_epi32( + 1 << 16 | 1, + 1 << 16 | 1, + 1 << 16 | 1, + 1 << 16 | 1, + 1 << 16 | 1, + 1 << 16 | 1, + 1 << 16 | 1, + 1 << 16 | 1, + 1 << 16 | 1, + 1 << 16 | 1, + 1 << 16 | 1, + 1 << 16 | 1, + 2, + 2, + 2, + 2, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_madd_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let r = _mm512_maskz_madd_epi16(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_madd_epi16(0b00000000_00001111, a, b); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_madd_epi16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1); + let r = _mm256_mask_madd_epi16(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_madd_epi16(a, 0b00001111, a, b); + let e = _mm256_set_epi32( + 1 << 16 | 1, + 1 << 16 | 1, + 1 << 16 | 1, + 1 << 16 | 1, + 2, + 2, + 2, + 2, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_madd_epi16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1); + let r = _mm256_maskz_madd_epi16(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_madd_epi16(0b00001111, a, b); + let e = _mm256_set_epi32(0, 0, 0, 0, 2, 2, 2, 2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_madd_epi16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1); + let r = _mm_mask_madd_epi16(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_madd_epi16(a, 0b00001111, a, b); + let e = _mm_set_epi32(2, 2, 2, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_madd_epi16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1); + let r = _mm_maskz_madd_epi16(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_madd_epi16(0b00001111, a, b); + let e = _mm_set_epi32(2, 2, 2, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maddubs_epi16() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(1); + let r = _mm512_maddubs_epi16(a, b); + let e = _mm512_set1_epi16(2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_maddubs_epi16() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(1); + let src = _mm512_set1_epi16(1); + let r = _mm512_mask_maddubs_epi16(src, 0, a, b); + assert_eq_m512i(r, src); + let r = _mm512_mask_add_epi16(src, 0b00000000_00000000_00000000_00000001, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1<<9|2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_maddubs_epi16() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(1); + let r = _mm512_maskz_maddubs_epi16(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_maddubs_epi16(0b00000000_11111111_00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, + 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_maddubs_epi16() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(1); + let src = _mm256_set1_epi16(1); + let r = _mm256_mask_maddubs_epi16(src, 0, a, b); + assert_eq_m256i(r, src); + let r = _mm256_mask_add_epi16(src, 0b00000000_00000001, a, b); + let e = _mm256_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 << 9 | 2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_maddubs_epi16() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(1); + let r = _mm256_maskz_maddubs_epi16(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_maddubs_epi16(0b00000000_11111111, a, b); + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_maddubs_epi16() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(1); + let src = _mm_set1_epi16(1); + let r = _mm_mask_maddubs_epi16(src, 0, a, b); + assert_eq_m128i(r, src); + let r = _mm_mask_add_epi16(src, 0b00000001, a, b); + let e = _mm_set_epi16(1, 1, 1, 1, 1, 1, 1, 1 << 9 | 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_maddubs_epi16() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(1); + let r = _mm_maskz_maddubs_epi16(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_maddubs_epi16(0b00001111, a, b); + let e = _mm_set_epi16(0, 0, 0, 0, 2, 2, 2, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_packs_epi32() { + let a = _mm512_set1_epi32(i32::MAX); + let b = _mm512_set1_epi32(1); + let r = _mm512_packs_epi32(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(1, 1, 1, 1, i16::MAX, i16::MAX, i16::MAX, i16::MAX, 1, 1, 1, 1, i16::MAX, i16::MAX, i16::MAX, i16::MAX, + 1, 1, 1, 1, i16::MAX, i16::MAX, i16::MAX, i16::MAX, 1, 1, 1, 1, i16::MAX, i16::MAX, i16::MAX, i16::MAX); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_packs_epi32() { + let a = _mm512_set1_epi32(i32::MAX); + let b = _mm512_set1_epi32(1 << 16 | 1); + let r = _mm512_mask_packs_epi32(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_packs_epi32(b, 0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, i16::MAX, i16::MAX, i16::MAX, i16::MAX); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_packs_epi32() { + let a = _mm512_set1_epi32(i32::MAX); + let b = _mm512_set1_epi32(1); + let r = _mm512_maskz_packs_epi32(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_packs_epi32(0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, i16::MAX, i16::MAX, i16::MAX, i16::MAX); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_packs_epi32() { + let a = _mm256_set1_epi32(i32::MAX); + let b = _mm256_set1_epi32(1 << 16 | 1); + let r = _mm256_mask_packs_epi32(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_packs_epi32(b, 0b00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, i16::MAX, i16::MAX, i16::MAX, i16::MAX); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_packs_epi32() { + let a = _mm256_set1_epi32(i32::MAX); + let b = _mm256_set1_epi32(1); + let r = _mm256_maskz_packs_epi32(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_packs_epi32(0b00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, i16::MAX, i16::MAX, i16::MAX, i16::MAX); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_packs_epi32() { + let a = _mm_set1_epi32(i32::MAX); + let b = _mm_set1_epi32(1 << 16 | 1); + let r = _mm_mask_packs_epi32(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_packs_epi32(b, 0b00001111, a, b); + let e = _mm_set_epi16(1, 1, 1, 1, i16::MAX, i16::MAX, i16::MAX, i16::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_packs_epi32() { + let a = _mm_set1_epi32(i32::MAX); + let b = _mm_set1_epi32(1); + let r = _mm_maskz_packs_epi32(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_packs_epi32(0b00001111, a, b); + let e = _mm_set_epi16(0, 0, 0, 0, i16::MAX, i16::MAX, i16::MAX, i16::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_packs_epi16() { + let a = _mm512_set1_epi16(i16::MAX); + let b = _mm512_set1_epi16(1); + let r = _mm512_packs_epi16(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, + 1, 1, 1, 1, 1, 1, 1, 1, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, + 1, 1, 1, 1, 1, 1, 1, 1, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, + 1, 1, 1, 1, 1, 1, 1, 1, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_packs_epi16() { + let a = _mm512_set1_epi16(i16::MAX); + let b = _mm512_set1_epi16(1 << 8 | 1); + let r = _mm512_mask_packs_epi16(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_packs_epi16( + b, + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00001111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, i8::MAX, i8::MAX, i8::MAX, i8::MAX); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_packs_epi16() { + let a = _mm512_set1_epi16(i16::MAX); + let b = _mm512_set1_epi16(1); + let r = _mm512_maskz_packs_epi16(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_packs_epi16( + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00001111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, i8::MAX, i8::MAX, i8::MAX, i8::MAX); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_packs_epi16() { + let a = _mm256_set1_epi16(i16::MAX); + let b = _mm256_set1_epi16(1 << 8 | 1); + let r = _mm256_mask_packs_epi16(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_packs_epi16(b, 0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, i8::MAX, i8::MAX, i8::MAX, i8::MAX); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_packs_epi16() { + let a = _mm256_set1_epi16(i16::MAX); + let b = _mm256_set1_epi16(1); + let r = _mm256_maskz_packs_epi16(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_packs_epi16(0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, i8::MAX, i8::MAX, i8::MAX, i8::MAX); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_packs_epi16() { + let a = _mm_set1_epi16(i16::MAX); + let b = _mm_set1_epi16(1 << 8 | 1); + let r = _mm_mask_packs_epi16(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_packs_epi16(b, 0b00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, i8::MAX, i8::MAX, i8::MAX, i8::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_packs_epi16() { + let a = _mm_set1_epi16(i16::MAX); + let b = _mm_set1_epi16(1); + let r = _mm_maskz_packs_epi16(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_packs_epi16(0b00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, i8::MAX, i8::MAX, i8::MAX, i8::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_packus_epi32() { + let a = _mm512_set1_epi32(-1); + let b = _mm512_set1_epi32(1); + let r = _mm512_packus_epi32(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, + 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_packus_epi32() { + let a = _mm512_set1_epi32(-1); + let b = _mm512_set1_epi32(1 << 16 | 1); + let r = _mm512_mask_packus_epi32(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_packus_epi32(b, 0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_packus_epi32() { + let a = _mm512_set1_epi32(-1); + let b = _mm512_set1_epi32(1); + let r = _mm512_maskz_packus_epi32(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_packus_epi32(0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_packus_epi32() { + let a = _mm256_set1_epi32(-1); + let b = _mm256_set1_epi32(1 << 16 | 1); + let r = _mm256_mask_packus_epi32(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_packus_epi32(b, 0b00000000_00001111, a, b); + let e = _mm256_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_packus_epi32() { + let a = _mm256_set1_epi32(-1); + let b = _mm256_set1_epi32(1); + let r = _mm256_maskz_packus_epi32(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_packus_epi32(0b00000000_00001111, a, b); + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_packus_epi32() { + let a = _mm_set1_epi32(-1); + let b = _mm_set1_epi32(1 << 16 | 1); + let r = _mm_mask_packus_epi32(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_packus_epi32(b, 0b00001111, a, b); + let e = _mm_set_epi16(1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_packus_epi32() { + let a = _mm_set1_epi32(-1); + let b = _mm_set1_epi32(1); + let r = _mm_maskz_packus_epi32(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_packus_epi32(0b00001111, a, b); + let e = _mm_set_epi16(0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_packus_epi16() { + let a = _mm512_set1_epi16(-1); + let b = _mm512_set1_epi16(1); + let r = _mm512_packus_epi16(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_packus_epi16() { + let a = _mm512_set1_epi16(-1); + let b = _mm512_set1_epi16(1 << 8 | 1); + let r = _mm512_mask_packus_epi16(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_packus_epi16( + b, + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00001111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_packus_epi16() { + let a = _mm512_set1_epi16(-1); + let b = _mm512_set1_epi16(1); + let r = _mm512_maskz_packus_epi16(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_packus_epi16( + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00001111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_packus_epi16() { + let a = _mm256_set1_epi16(-1); + let b = _mm256_set1_epi16(1 << 8 | 1); + let r = _mm256_mask_packus_epi16(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_packus_epi16(b, 0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_packus_epi16() { + let a = _mm256_set1_epi16(-1); + let b = _mm256_set1_epi16(1); + let r = _mm256_maskz_packus_epi16(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_packus_epi16(0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_packus_epi16() { + let a = _mm_set1_epi16(-1); + let b = _mm_set1_epi16(1 << 8 | 1); + let r = _mm_mask_packus_epi16(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_packus_epi16(b, 0b00000000_00001111, a, b); + let e = _mm_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_packus_epi16() { + let a = _mm_set1_epi16(-1); + let b = _mm_set1_epi16(1); + let r = _mm_maskz_packus_epi16(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_packus_epi16(0b00000000_00001111, a, b); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_avg_epu16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let r = _mm512_avg_epu16(a, b); + let e = _mm512_set1_epi16(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_avg_epu16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let r = _mm512_mask_avg_epu16(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_avg_epu16(a, 0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_avg_epu16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1); + let r = _mm512_maskz_avg_epu16(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_avg_epu16(0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_avg_epu16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1); + let r = _mm256_mask_avg_epu16(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_avg_epu16(a, 0b00000000_00001111, a, b); + let e = _mm256_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_avg_epu16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1); + let r = _mm256_maskz_avg_epu16(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_avg_epu16(0b00000000_00001111, a, b); + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_avg_epu16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1); + let r = _mm_mask_avg_epu16(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_avg_epu16(a, 0b00001111, a, b); + let e = _mm_set_epi16(1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_avg_epu16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1); + let r = _mm_maskz_avg_epu16(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_avg_epu16(0b00001111, a, b); + let e = _mm_set_epi16(0, 0, 0, 0, 1, 1, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_avg_epu8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(1); + let r = _mm512_avg_epu8(a, b); + let e = _mm512_set1_epi8(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_avg_epu8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(1); + let r = _mm512_mask_avg_epu8(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_avg_epu8( + a, + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00001111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_avg_epu8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(1); + let r = _mm512_maskz_avg_epu8(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_avg_epu8( + 0b00000000_000000000_00000000_00000000_00000000_0000000_00000000_00001111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_avg_epu8() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(1); + let r = _mm256_mask_avg_epu8(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_avg_epu8(a, 0b00000000_00000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_avg_epu8() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(1); + let r = _mm256_maskz_avg_epu8(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_avg_epu8(0b00000000_0000000_00000000_00001111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_avg_epu8() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(1); + let r = _mm_mask_avg_epu8(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_avg_epu8(a, 0b00000000_00001111, a, b); + let e = _mm_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_avg_epu8() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(1); + let r = _mm_maskz_avg_epu8(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_avg_epu8(0b00000000_00001111, a, b); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_sll_epi16() { + let a = _mm512_set1_epi16(1 << 15); + let count = _mm_set1_epi16(2); + let r = _mm512_sll_epi16(a, count); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_sll_epi16() { + let a = _mm512_set1_epi16(1 << 15); + let count = _mm_set1_epi16(2); + let r = _mm512_mask_sll_epi16(a, 0, a, count); + assert_eq_m512i(r, a); + let r = _mm512_mask_sll_epi16(a, 0b11111111_11111111_11111111_11111111, a, count); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_sll_epi16() { + let a = _mm512_set1_epi16(1 << 15); + let count = _mm_set1_epi16(2); + let r = _mm512_maskz_sll_epi16(0, a, count); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_sll_epi16(0b11111111_11111111_11111111_11111111, a, count); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_sll_epi16() { + let a = _mm256_set1_epi16(1 << 15); + let count = _mm_set1_epi16(2); + let r = _mm256_mask_sll_epi16(a, 0, a, count); + assert_eq_m256i(r, a); + let r = _mm256_mask_sll_epi16(a, 0b11111111_11111111, a, count); + let e = _mm256_set1_epi16(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_sll_epi16() { + let a = _mm256_set1_epi16(1 << 15); + let count = _mm_set1_epi16(2); + let r = _mm256_maskz_sll_epi16(0, a, count); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_sll_epi16(0b11111111_11111111, a, count); + let e = _mm256_set1_epi16(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_sll_epi16() { + let a = _mm_set1_epi16(1 << 15); + let count = _mm_set1_epi16(2); + let r = _mm_mask_sll_epi16(a, 0, a, count); + assert_eq_m128i(r, a); + let r = _mm_mask_sll_epi16(a, 0b11111111, a, count); + let e = _mm_set1_epi16(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_sll_epi16() { + let a = _mm_set1_epi16(1 << 15); + let count = _mm_set1_epi16(2); + let r = _mm_maskz_sll_epi16(0, a, count); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_sll_epi16(0b11111111, a, count); + let e = _mm_set1_epi16(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_slli_epi16() { + let a = _mm512_set1_epi16(1 << 15); + let r = _mm512_slli_epi16::<1>(a); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_slli_epi16() { + let a = _mm512_set1_epi16(1 << 15); + let r = _mm512_mask_slli_epi16::<1>(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_slli_epi16::<1>(a, 0b11111111_11111111_11111111_11111111, a); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_slli_epi16() { + let a = _mm512_set1_epi16(1 << 15); + let r = _mm512_maskz_slli_epi16::<1>(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_slli_epi16::<1>(0b11111111_11111111_11111111_11111111, a); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_slli_epi16() { + let a = _mm256_set1_epi16(1 << 15); + let r = _mm256_mask_slli_epi16::<1>(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_slli_epi16::<1>(a, 0b11111111_11111111, a); + let e = _mm256_set1_epi16(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_slli_epi16() { + let a = _mm256_set1_epi16(1 << 15); + let r = _mm256_maskz_slli_epi16::<1>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_slli_epi16::<1>(0b11111111_11111111, a); + let e = _mm256_set1_epi16(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_slli_epi16() { + let a = _mm_set1_epi16(1 << 15); + let r = _mm_mask_slli_epi16::<1>(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_slli_epi16::<1>(a, 0b11111111, a); + let e = _mm_set1_epi16(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_slli_epi16() { + let a = _mm_set1_epi16(1 << 15); + let r = _mm_maskz_slli_epi16::<1>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_slli_epi16::<1>(0b11111111, a); + let e = _mm_set1_epi16(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_sllv_epi16() { + let a = _mm512_set1_epi16(1 << 15); + let count = _mm512_set1_epi16(2); + let r = _mm512_sllv_epi16(a, count); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_sllv_epi16() { + let a = _mm512_set1_epi16(1 << 15); + let count = _mm512_set1_epi16(2); + let r = _mm512_mask_sllv_epi16(a, 0, a, count); + assert_eq_m512i(r, a); + let r = _mm512_mask_sllv_epi16(a, 0b11111111_11111111_11111111_11111111, a, count); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_sllv_epi16() { + let a = _mm512_set1_epi16(1 << 15); + let count = _mm512_set1_epi16(2); + let r = _mm512_maskz_sllv_epi16(0, a, count); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_sllv_epi16(0b11111111_11111111_11111111_11111111, a, count); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_sllv_epi16() { + let a = _mm256_set1_epi16(1 << 15); + let count = _mm256_set1_epi16(2); + let r = _mm256_sllv_epi16(a, count); + let e = _mm256_set1_epi16(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_sllv_epi16() { + let a = _mm256_set1_epi16(1 << 15); + let count = _mm256_set1_epi16(2); + let r = _mm256_mask_sllv_epi16(a, 0, a, count); + assert_eq_m256i(r, a); + let r = _mm256_mask_sllv_epi16(a, 0b11111111_11111111, a, count); + let e = _mm256_set1_epi16(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_sllv_epi16() { + let a = _mm256_set1_epi16(1 << 15); + let count = _mm256_set1_epi16(2); + let r = _mm256_maskz_sllv_epi16(0, a, count); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_sllv_epi16(0b11111111_11111111, a, count); + let e = _mm256_set1_epi16(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_sllv_epi16() { + let a = _mm_set1_epi16(1 << 15); + let count = _mm_set1_epi16(2); + let r = _mm_sllv_epi16(a, count); + let e = _mm_set1_epi16(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_sllv_epi16() { + let a = _mm_set1_epi16(1 << 15); + let count = _mm_set1_epi16(2); + let r = _mm_mask_sllv_epi16(a, 0, a, count); + assert_eq_m128i(r, a); + let r = _mm_mask_sllv_epi16(a, 0b11111111, a, count); + let e = _mm_set1_epi16(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_sllv_epi16() { + let a = _mm_set1_epi16(1 << 15); + let count = _mm_set1_epi16(2); + let r = _mm_maskz_sllv_epi16(0, a, count); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_sllv_epi16(0b11111111, a, count); + let e = _mm_set1_epi16(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_srl_epi16() { + let a = _mm512_set1_epi16(1 << 1); + let count = _mm_set1_epi16(2); + let r = _mm512_srl_epi16(a, count); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_srl_epi16() { + let a = _mm512_set1_epi16(1 << 1); + let count = _mm_set1_epi16(2); + let r = _mm512_mask_srl_epi16(a, 0, a, count); + assert_eq_m512i(r, a); + let r = _mm512_mask_srl_epi16(a, 0b11111111_11111111_11111111_11111111, a, count); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_srl_epi16() { + let a = _mm512_set1_epi16(1 << 1); + let count = _mm_set1_epi16(2); + let r = _mm512_maskz_srl_epi16(0, a, count); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_srl_epi16(0b11111111_11111111_11111111_11111111, a, count); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_srl_epi16() { + let a = _mm256_set1_epi16(1 << 1); + let count = _mm_set1_epi16(2); + let r = _mm256_mask_srl_epi16(a, 0, a, count); + assert_eq_m256i(r, a); + let r = _mm256_mask_srl_epi16(a, 0b11111111_11111111, a, count); + let e = _mm256_set1_epi16(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_srl_epi16() { + let a = _mm256_set1_epi16(1 << 1); + let count = _mm_set1_epi16(2); + let r = _mm256_maskz_srl_epi16(0, a, count); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_srl_epi16(0b11111111_11111111, a, count); + let e = _mm256_set1_epi16(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_srl_epi16() { + let a = _mm_set1_epi16(1 << 1); + let count = _mm_set1_epi16(2); + let r = _mm_mask_srl_epi16(a, 0, a, count); + assert_eq_m128i(r, a); + let r = _mm_mask_srl_epi16(a, 0b11111111, a, count); + let e = _mm_set1_epi16(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_srl_epi16() { + let a = _mm_set1_epi16(1 << 1); + let count = _mm_set1_epi16(2); + let r = _mm_maskz_srl_epi16(0, a, count); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_srl_epi16(0b11111111, a, count); + let e = _mm_set1_epi16(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_srli_epi16() { + let a = _mm512_set1_epi16(1 << 1); + let r = _mm512_srli_epi16::<2>(a); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_srli_epi16() { + let a = _mm512_set1_epi16(1 << 1); + let r = _mm512_mask_srli_epi16::<2>(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_srli_epi16::<2>(a, 0b11111111_11111111_11111111_11111111, a); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_srli_epi16() { + let a = _mm512_set1_epi16(1 << 1); + let r = _mm512_maskz_srli_epi16::<2>(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_srli_epi16::<2>(0b11111111_11111111_11111111_11111111, a); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_srli_epi16() { + let a = _mm256_set1_epi16(1 << 1); + let r = _mm256_mask_srli_epi16::<2>(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_srli_epi16::<2>(a, 0b11111111_11111111, a); + let e = _mm256_set1_epi16(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_srli_epi16() { + let a = _mm256_set1_epi16(1 << 1); + let r = _mm256_maskz_srli_epi16::<2>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_srli_epi16::<2>(0b11111111_11111111, a); + let e = _mm256_set1_epi16(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_srli_epi16() { + let a = _mm_set1_epi16(1 << 1); + let r = _mm_mask_srli_epi16::<2>(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_srli_epi16::<2>(a, 0b11111111, a); + let e = _mm_set1_epi16(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_srli_epi16() { + let a = _mm_set1_epi16(1 << 1); + let r = _mm_maskz_srli_epi16::<2>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_srli_epi16::<2>(0b11111111, a); + let e = _mm_set1_epi16(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_srlv_epi16() { + let a = _mm512_set1_epi16(1 << 1); + let count = _mm512_set1_epi16(2); + let r = _mm512_srlv_epi16(a, count); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_srlv_epi16() { + let a = _mm512_set1_epi16(1 << 1); + let count = _mm512_set1_epi16(2); + let r = _mm512_mask_srlv_epi16(a, 0, a, count); + assert_eq_m512i(r, a); + let r = _mm512_mask_srlv_epi16(a, 0b11111111_11111111_11111111_11111111, a, count); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_srlv_epi16() { + let a = _mm512_set1_epi16(1 << 1); + let count = _mm512_set1_epi16(2); + let r = _mm512_maskz_srlv_epi16(0, a, count); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_srlv_epi16(0b11111111_11111111_11111111_11111111, a, count); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_srlv_epi16() { + let a = _mm256_set1_epi16(1 << 1); + let count = _mm256_set1_epi16(2); + let r = _mm256_srlv_epi16(a, count); + let e = _mm256_set1_epi16(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_srlv_epi16() { + let a = _mm256_set1_epi16(1 << 1); + let count = _mm256_set1_epi16(2); + let r = _mm256_mask_srlv_epi16(a, 0, a, count); + assert_eq_m256i(r, a); + let r = _mm256_mask_srlv_epi16(a, 0b11111111_11111111, a, count); + let e = _mm256_set1_epi16(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_srlv_epi16() { + let a = _mm256_set1_epi16(1 << 1); + let count = _mm256_set1_epi16(2); + let r = _mm256_maskz_srlv_epi16(0, a, count); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_srlv_epi16(0b11111111_11111111, a, count); + let e = _mm256_set1_epi16(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_srlv_epi16() { + let a = _mm_set1_epi16(1 << 1); + let count = _mm_set1_epi16(2); + let r = _mm_srlv_epi16(a, count); + let e = _mm_set1_epi16(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_srlv_epi16() { + let a = _mm_set1_epi16(1 << 1); + let count = _mm_set1_epi16(2); + let r = _mm_mask_srlv_epi16(a, 0, a, count); + assert_eq_m128i(r, a); + let r = _mm_mask_srlv_epi16(a, 0b11111111, a, count); + let e = _mm_set1_epi16(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_srlv_epi16() { + let a = _mm_set1_epi16(1 << 1); + let count = _mm_set1_epi16(2); + let r = _mm_maskz_srlv_epi16(0, a, count); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_srlv_epi16(0b11111111, a, count); + let e = _mm_set1_epi16(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_sra_epi16() { + let a = _mm512_set1_epi16(8); + let count = _mm_set1_epi16(1); + let r = _mm512_sra_epi16(a, count); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_sra_epi16() { + let a = _mm512_set1_epi16(8); + let count = _mm_set1_epi16(1); + let r = _mm512_mask_sra_epi16(a, 0, a, count); + assert_eq_m512i(r, a); + let r = _mm512_mask_sra_epi16(a, 0b11111111_11111111_11111111_11111111, a, count); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_sra_epi16() { + let a = _mm512_set1_epi16(8); + let count = _mm_set1_epi16(1); + let r = _mm512_maskz_sra_epi16(0, a, count); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_sra_epi16(0b11111111_11111111_11111111_11111111, a, count); + let e = _mm512_set1_epi16(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_sra_epi16() { + let a = _mm256_set1_epi16(8); + let count = _mm_set1_epi16(1); + let r = _mm256_mask_sra_epi16(a, 0, a, count); + assert_eq_m256i(r, a); + let r = _mm256_mask_sra_epi16(a, 0b11111111_11111111, a, count); + let e = _mm256_set1_epi16(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_sra_epi16() { + let a = _mm256_set1_epi16(8); + let count = _mm_set1_epi16(1); + let r = _mm256_maskz_sra_epi16(0, a, count); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_sra_epi16(0b11111111_11111111, a, count); + let e = _mm256_set1_epi16(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_sra_epi16() { + let a = _mm_set1_epi16(8); + let count = _mm_set1_epi16(1); + let r = _mm_mask_sra_epi16(a, 0, a, count); + assert_eq_m128i(r, a); + let r = _mm_mask_sra_epi16(a, 0b11111111, a, count); + let e = _mm_set1_epi16(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_sra_epi16() { + let a = _mm_set1_epi16(8); + let count = _mm_set1_epi16(1); + let r = _mm_maskz_sra_epi16(0, a, count); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_sra_epi16(0b11111111, a, count); + let e = _mm_set1_epi16(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_srai_epi16() { + let a = _mm512_set1_epi16(8); + let r = _mm512_srai_epi16::<2>(a); + let e = _mm512_set1_epi16(2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_srai_epi16() { + let a = _mm512_set1_epi16(8); + let r = _mm512_mask_srai_epi16::<2>(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_srai_epi16::<2>(a, 0b11111111_11111111_11111111_11111111, a); + let e = _mm512_set1_epi16(2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_srai_epi16() { + let a = _mm512_set1_epi16(8); + let r = _mm512_maskz_srai_epi16::<2>(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_srai_epi16::<2>(0b11111111_11111111_11111111_11111111, a); + let e = _mm512_set1_epi16(2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_srai_epi16() { + let a = _mm256_set1_epi16(8); + let r = _mm256_mask_srai_epi16::<2>(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_srai_epi16::<2>(a, 0b11111111_11111111, a); + let e = _mm256_set1_epi16(2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_srai_epi16() { + let a = _mm256_set1_epi16(8); + let r = _mm256_maskz_srai_epi16::<2>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_srai_epi16::<2>(0b11111111_11111111, a); + let e = _mm256_set1_epi16(2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_srai_epi16() { + let a = _mm_set1_epi16(8); + let r = _mm_mask_srai_epi16::<2>(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_srai_epi16::<2>(a, 0b11111111, a); + let e = _mm_set1_epi16(2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_srai_epi16() { + let a = _mm_set1_epi16(8); + let r = _mm_maskz_srai_epi16::<2>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_srai_epi16::<2>(0b11111111, a); + let e = _mm_set1_epi16(2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_srav_epi16() { + let a = _mm512_set1_epi16(8); + let count = _mm512_set1_epi16(2); + let r = _mm512_srav_epi16(a, count); + let e = _mm512_set1_epi16(2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_srav_epi16() { + let a = _mm512_set1_epi16(8); + let count = _mm512_set1_epi16(2); + let r = _mm512_mask_srav_epi16(a, 0, a, count); + assert_eq_m512i(r, a); + let r = _mm512_mask_srav_epi16(a, 0b11111111_11111111_11111111_11111111, a, count); + let e = _mm512_set1_epi16(2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_srav_epi16() { + let a = _mm512_set1_epi16(8); + let count = _mm512_set1_epi16(2); + let r = _mm512_maskz_srav_epi16(0, a, count); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_srav_epi16(0b11111111_11111111_11111111_11111111, a, count); + let e = _mm512_set1_epi16(2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_srav_epi16() { + let a = _mm256_set1_epi16(8); + let count = _mm256_set1_epi16(2); + let r = _mm256_srav_epi16(a, count); + let e = _mm256_set1_epi16(2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_srav_epi16() { + let a = _mm256_set1_epi16(8); + let count = _mm256_set1_epi16(2); + let r = _mm256_mask_srav_epi16(a, 0, a, count); + assert_eq_m256i(r, a); + let r = _mm256_mask_srav_epi16(a, 0b11111111_11111111, a, count); + let e = _mm256_set1_epi16(2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_srav_epi16() { + let a = _mm256_set1_epi16(8); + let count = _mm256_set1_epi16(2); + let r = _mm256_maskz_srav_epi16(0, a, count); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_srav_epi16(0b11111111_11111111, a, count); + let e = _mm256_set1_epi16(2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_srav_epi16() { + let a = _mm_set1_epi16(8); + let count = _mm_set1_epi16(2); + let r = _mm_srav_epi16(a, count); + let e = _mm_set1_epi16(2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_srav_epi16() { + let a = _mm_set1_epi16(8); + let count = _mm_set1_epi16(2); + let r = _mm_mask_srav_epi16(a, 0, a, count); + assert_eq_m128i(r, a); + let r = _mm_mask_srav_epi16(a, 0b11111111, a, count); + let e = _mm_set1_epi16(2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_srav_epi16() { + let a = _mm_set1_epi16(8); + let count = _mm_set1_epi16(2); + let r = _mm_maskz_srav_epi16(0, a, count); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_srav_epi16(0b11111111, a, count); + let e = _mm_set1_epi16(2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_permutex2var_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + #[rustfmt::skip] + let idx = _mm512_set_epi16(1, 1<<5, 2, 1<<5, 3, 1<<5, 4, 1<<5, 5, 1<<5, 6, 1<<5, 7, 1<<5, 8, 1<<5, + 9, 1<<5, 10, 1<<5, 11, 1<<5, 12, 1<<5, 13, 1<<5, 14, 1<<5, 15, 1<<5, 16, 1<<5); + let b = _mm512_set1_epi16(100); + let r = _mm512_permutex2var_epi16(a, idx, b); + #[rustfmt::skip] + let e = _mm512_set_epi16( + 30, 100, 29, 100, 28, 100, 27, 100, 26, 100, 25, 100, 24, 100, 23, 100, + 22, 100, 21, 100, 20, 100, 19, 100, 18, 100, 17, 100, 16, 100, 15, 100, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_permutex2var_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + #[rustfmt::skip] + let idx = _mm512_set_epi16(1, 1<<5, 2, 1<<5, 3, 1<<5, 4, 1<<5, 5, 1<<5, 6, 1<<5, 7, 1<<5, 8, 1<<5, + 9, 1<<5, 10, 1<<5, 11, 1<<5, 12, 1<<5, 13, 1<<5, 14, 1<<5, 15, 1<<5, 16, 1<<5); + let b = _mm512_set1_epi16(100); + let r = _mm512_mask_permutex2var_epi16(a, 0, idx, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_permutex2var_epi16(a, 0b11111111_11111111_11111111_11111111, idx, b); + #[rustfmt::skip] + let e = _mm512_set_epi16( + 30, 100, 29, 100, 28, 100, 27, 100, 26, 100, 25, 100, 24, 100, 23, 100, + 22, 100, 21, 100, 20, 100, 19, 100, 18, 100, 17, 100, 16, 100, 15, 100, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_permutex2var_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + #[rustfmt::skip] + let idx = _mm512_set_epi16(1, 1<<5, 2, 1<<5, 3, 1<<5, 4, 1<<5, 5, 1<<5, 6, 1<<5, 7, 1<<5, 8, 1<<5, + 9, 1<<5, 10, 1<<5, 11, 1<<5, 12, 1<<5, 13, 1<<5, 14, 1<<5, 15, 1<<5, 16, 1<<5); + let b = _mm512_set1_epi16(100); + let r = _mm512_maskz_permutex2var_epi16(0, a, idx, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_permutex2var_epi16(0b11111111_11111111_11111111_11111111, a, idx, b); + #[rustfmt::skip] + let e = _mm512_set_epi16( + 30, 100, 29, 100, 28, 100, 27, 100, 26, 100, 25, 100, 24, 100, 23, 100, + 22, 100, 21, 100, 20, 100, 19, 100, 18, 100, 17, 100, 16, 100, 15, 100, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask2_permutex2var_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + #[rustfmt::skip] + let idx = _mm512_set_epi16(1, 1<<5, 2, 1<<5, 3, 1<<5, 4, 1<<5, 5, 1<<5, 6, 1<<5, 7, 1<<5, 8, 1<<5, + 9, 1<<5, 10, 1<<5, 11, 1<<5, 12, 1<<5, 13, 1<<5, 14, 1<<5, 15, 1<<5, 16, 1<<5); + let b = _mm512_set1_epi16(100); + let r = _mm512_mask2_permutex2var_epi16(a, idx, 0, b); + assert_eq_m512i(r, idx); + let r = _mm512_mask2_permutex2var_epi16(a, idx, 0b11111111_11111111_11111111_11111111, b); + #[rustfmt::skip] + let e = _mm512_set_epi16( + 30, 100, 29, 100, 28, 100, 27, 100, 26, 100, 25, 100, 24, 100, 23, 100, + 22, 100, 21, 100, 20, 100, 19, 100, 18, 100, 17, 100, 16, 100, 15, 100, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_permutex2var_epi16() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let idx = _mm256_set_epi16(1, 1<<4, 2, 1<<4, 3, 1<<4, 4, 1<<4, 5, 1<<4, 6, 1<<4, 7, 1<<4, 8, 1<<4); + let b = _mm256_set1_epi16(100); + let r = _mm256_permutex2var_epi16(a, idx, b); + let e = _mm256_set_epi16( + 14, 100, 13, 100, 12, 100, 11, 100, 10, 100, 9, 100, 8, 100, 7, 100, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_permutex2var_epi16() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let idx = _mm256_set_epi16(1, 1<<4, 2, 1<<4, 3, 1<<4, 4, 1<<4, 5, 1<<4, 6, 1<<4, 7, 1<<4, 8, 1<<4); + let b = _mm256_set1_epi16(100); + let r = _mm256_mask_permutex2var_epi16(a, 0, idx, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_permutex2var_epi16(a, 0b11111111_11111111, idx, b); + let e = _mm256_set_epi16( + 14, 100, 13, 100, 12, 100, 11, 100, 10, 100, 9, 100, 8, 100, 7, 100, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_permutex2var_epi16() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let idx = _mm256_set_epi16(1, 1<<4, 2, 1<<4, 3, 1<<4, 4, 1<<4, 5, 1<<4, 6, 1<<4, 7, 1<<4, 8, 1<<4); + let b = _mm256_set1_epi16(100); + let r = _mm256_maskz_permutex2var_epi16(0, a, idx, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_permutex2var_epi16(0b11111111_11111111, a, idx, b); + let e = _mm256_set_epi16( + 14, 100, 13, 100, 12, 100, 11, 100, 10, 100, 9, 100, 8, 100, 7, 100, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask2_permutex2var_epi16() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let idx = _mm256_set_epi16(1, 1<<4, 2, 1<<4, 3, 1<<4, 4, 1<<4, 5, 1<<4, 6, 1<<4, 7, 1<<4, 8, 1<<4); + let b = _mm256_set1_epi16(100); + let r = _mm256_mask2_permutex2var_epi16(a, idx, 0, b); + assert_eq_m256i(r, idx); + let r = _mm256_mask2_permutex2var_epi16(a, idx, 0b11111111_11111111, b); + #[rustfmt::skip] + let e = _mm256_set_epi16( + 14, 100, 13, 100, 12, 100, 11, 100, 10, 100, 9, 100, 8, 100, 7, 100, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_permutex2var_epi16() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let idx = _mm_set_epi16(1, 1 << 3, 2, 1 << 3, 3, 1 << 3, 4, 1 << 3); + let b = _mm_set1_epi16(100); + let r = _mm_permutex2var_epi16(a, idx, b); + let e = _mm_set_epi16(6, 100, 5, 100, 4, 100, 3, 100); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_permutex2var_epi16() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let idx = _mm_set_epi16(1, 1 << 3, 2, 1 << 3, 3, 1 << 3, 4, 1 << 3); + let b = _mm_set1_epi16(100); + let r = _mm_mask_permutex2var_epi16(a, 0, idx, b); + assert_eq_m128i(r, a); + let r = _mm_mask_permutex2var_epi16(a, 0b11111111, idx, b); + let e = _mm_set_epi16(6, 100, 5, 100, 4, 100, 3, 100); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_permutex2var_epi16() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let idx = _mm_set_epi16(1, 1 << 3, 2, 1 << 3, 3, 1 << 3, 4, 1 << 3); + let b = _mm_set1_epi16(100); + let r = _mm_maskz_permutex2var_epi16(0, a, idx, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_permutex2var_epi16(0b11111111, a, idx, b); + let e = _mm_set_epi16(6, 100, 5, 100, 4, 100, 3, 100); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask2_permutex2var_epi16() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let idx = _mm_set_epi16(1, 1 << 3, 2, 1 << 3, 3, 1 << 3, 4, 1 << 3); + let b = _mm_set1_epi16(100); + let r = _mm_mask2_permutex2var_epi16(a, idx, 0, b); + assert_eq_m128i(r, idx); + let r = _mm_mask2_permutex2var_epi16(a, idx, 0b11111111, b); + let e = _mm_set_epi16(6, 100, 5, 100, 4, 100, 3, 100); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_permutexvar_epi16() { + let idx = _mm512_set1_epi16(1); + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let r = _mm512_permutexvar_epi16(idx, a); + let e = _mm512_set1_epi16(30); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_permutexvar_epi16() { + let idx = _mm512_set1_epi16(1); + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let r = _mm512_mask_permutexvar_epi16(a, 0, idx, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_permutexvar_epi16(a, 0b11111111_11111111_11111111_11111111, idx, a); + let e = _mm512_set1_epi16(30); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_permutexvar_epi16() { + let idx = _mm512_set1_epi16(1); + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let r = _mm512_maskz_permutexvar_epi16(0, idx, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_permutexvar_epi16(0b11111111_11111111_11111111_11111111, idx, a); + let e = _mm512_set1_epi16(30); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_permutexvar_epi16() { + let idx = _mm256_set1_epi16(1); + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm256_permutexvar_epi16(idx, a); + let e = _mm256_set1_epi16(14); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_permutexvar_epi16() { + let idx = _mm256_set1_epi16(1); + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm256_mask_permutexvar_epi16(a, 0, idx, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_permutexvar_epi16(a, 0b11111111_11111111, idx, a); + let e = _mm256_set1_epi16(14); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_permutexvar_epi16() { + let idx = _mm256_set1_epi16(1); + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm256_maskz_permutexvar_epi16(0, idx, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_permutexvar_epi16(0b11111111_11111111, idx, a); + let e = _mm256_set1_epi16(14); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_permutexvar_epi16() { + let idx = _mm_set1_epi16(1); + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm_permutexvar_epi16(idx, a); + let e = _mm_set1_epi16(6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_permutexvar_epi16() { + let idx = _mm_set1_epi16(1); + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm_mask_permutexvar_epi16(a, 0, idx, a); + assert_eq_m128i(r, a); + let r = _mm_mask_permutexvar_epi16(a, 0b11111111, idx, a); + let e = _mm_set1_epi16(6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_permutexvar_epi16() { + let idx = _mm_set1_epi16(1); + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm_maskz_permutexvar_epi16(0, idx, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_permutexvar_epi16(0b11111111, idx, a); + let e = _mm_set1_epi16(6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_blend_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(2); + let r = _mm512_mask_blend_epi16(0b11111111_00000000_11111111_00000000, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_blend_epi16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(2); + let r = _mm256_mask_blend_epi16(0b11111111_00000000, a, b); + let e = _mm256_set_epi16(2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_blend_epi16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(2); + let r = _mm_mask_blend_epi16(0b11110000, a, b); + let e = _mm_set_epi16(2, 2, 2, 2, 1, 1, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_blend_epi8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(2); + let r = _mm512_mask_blend_epi8( + 0b11111111_00000000_11111111_00000000_11111111_00000000_11111111_00000000, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_blend_epi8() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(2); + let r = _mm256_mask_blend_epi8(0b11111111_00000000_11111111_00000000, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_blend_epi8() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(2); + let r = _mm_mask_blend_epi8(0b11111111_00000000, a, b); + let e = _mm_set_epi8(2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_broadcastw_epi16() { + let a = _mm_set_epi16(17, 18, 19, 20, 21, 22, 23, 24); + let r = _mm512_broadcastw_epi16(a); + let e = _mm512_set1_epi16(24); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_broadcastw_epi16() { + let src = _mm512_set1_epi16(1); + let a = _mm_set_epi16(17, 18, 19, 20, 21, 22, 23, 24); + let r = _mm512_mask_broadcastw_epi16(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_broadcastw_epi16(src, 0b11111111_11111111_11111111_11111111, a); + let e = _mm512_set1_epi16(24); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_broadcastw_epi16() { + let a = _mm_set_epi16(17, 18, 19, 20, 21, 22, 23, 24); + let r = _mm512_maskz_broadcastw_epi16(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_broadcastw_epi16(0b11111111_11111111_11111111_11111111, a); + let e = _mm512_set1_epi16(24); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_broadcastw_epi16() { + let src = _mm256_set1_epi16(1); + let a = _mm_set_epi16(17, 18, 19, 20, 21, 22, 23, 24); + let r = _mm256_mask_broadcastw_epi16(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_broadcastw_epi16(src, 0b11111111_11111111, a); + let e = _mm256_set1_epi16(24); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_broadcastw_epi16() { + let a = _mm_set_epi16(17, 18, 19, 20, 21, 22, 23, 24); + let r = _mm256_maskz_broadcastw_epi16(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_broadcastw_epi16(0b11111111_11111111, a); + let e = _mm256_set1_epi16(24); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_broadcastw_epi16() { + let src = _mm_set1_epi16(1); + let a = _mm_set_epi16(17, 18, 19, 20, 21, 22, 23, 24); + let r = _mm_mask_broadcastw_epi16(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_broadcastw_epi16(src, 0b11111111, a); + let e = _mm_set1_epi16(24); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_broadcastw_epi16() { + let a = _mm_set_epi16(17, 18, 19, 20, 21, 22, 23, 24); + let r = _mm_maskz_broadcastw_epi16(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_broadcastw_epi16(0b11111111, a); + let e = _mm_set1_epi16(24); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_broadcastb_epi8() { + let a = _mm_set_epi8( + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ); + let r = _mm512_broadcastb_epi8(a); + let e = _mm512_set1_epi8(32); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_broadcastb_epi8() { + let src = _mm512_set1_epi8(1); + let a = _mm_set_epi8( + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ); + let r = _mm512_mask_broadcastb_epi8(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_broadcastb_epi8( + src, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + a, + ); + let e = _mm512_set1_epi8(32); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_broadcastb_epi8() { + let a = _mm_set_epi8( + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ); + let r = _mm512_maskz_broadcastb_epi8(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_broadcastb_epi8( + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + a, + ); + let e = _mm512_set1_epi8(32); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_broadcastb_epi8() { + let src = _mm256_set1_epi8(1); + let a = _mm_set_epi8( + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ); + let r = _mm256_mask_broadcastb_epi8(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_broadcastb_epi8(src, 0b11111111_11111111_11111111_11111111, a); + let e = _mm256_set1_epi8(32); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_broadcastb_epi8() { + let a = _mm_set_epi8( + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ); + let r = _mm256_maskz_broadcastb_epi8(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_broadcastb_epi8(0b11111111_11111111_11111111_11111111, a); + let e = _mm256_set1_epi8(32); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_broadcastb_epi8() { + let src = _mm_set1_epi8(1); + let a = _mm_set_epi8( + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ); + let r = _mm_mask_broadcastb_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_broadcastb_epi8(src, 0b11111111_11111111, a); + let e = _mm_set1_epi8(32); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_broadcastb_epi8() { + let a = _mm_set_epi8( + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ); + let r = _mm_maskz_broadcastb_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_broadcastb_epi8(0b11111111_11111111, a); + let e = _mm_set1_epi8(32); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_unpackhi_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32); + #[rustfmt::skip] + let b = _mm512_set_epi16(33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64); + let r = _mm512_unpackhi_epi16(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(33, 1, 34, 2, 35, 3, 36, 4, 41, 9, 42, 10, 43, 11, 44, 12, + 49, 17, 50, 18, 51, 19, 52, 20, 57, 25, 58, 26, 59, 27, 60, 28); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_unpackhi_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32); + #[rustfmt::skip] + let b = _mm512_set_epi16(33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64); + let r = _mm512_mask_unpackhi_epi16(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_unpackhi_epi16(a, 0b11111111_11111111_11111111_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(33, 1, 34, 2, 35, 3, 36, 4, 41, 9, 42, 10, 43, 11, 44, 12, + 49, 17, 50, 18, 51, 19, 52, 20, 57, 25, 58, 26, 59, 27, 60, 28); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_unpackhi_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32); + #[rustfmt::skip] + let b = _mm512_set_epi16(33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64); + let r = _mm512_maskz_unpackhi_epi16(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_unpackhi_epi16(0b11111111_11111111_11111111_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(33, 1, 34, 2, 35, 3, 36, 4, 41, 9, 42, 10, 43, 11, 44, 12, + 49, 17, 50, 18, 51, 19, 52, 20, 57, 25, 58, 26, 59, 27, 60, 28); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_unpackhi_epi16() { + let a = _mm256_set_epi16(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b = _mm256_set_epi16( + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + ); + let r = _mm256_mask_unpackhi_epi16(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_unpackhi_epi16(a, 0b11111111_11111111, a, b); + let e = _mm256_set_epi16(33, 1, 34, 2, 35, 3, 36, 4, 41, 9, 42, 10, 43, 11, 44, 12); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_unpackhi_epi16() { + let a = _mm256_set_epi16(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b = _mm256_set_epi16( + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + ); + let r = _mm256_maskz_unpackhi_epi16(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_unpackhi_epi16(0b11111111_11111111, a, b); + let e = _mm256_set_epi16(33, 1, 34, 2, 35, 3, 36, 4, 41, 9, 42, 10, 43, 11, 44, 12); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_unpackhi_epi16() { + let a = _mm_set_epi16(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm_set_epi16(33, 34, 35, 36, 37, 38, 39, 40); + let r = _mm_mask_unpackhi_epi16(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_unpackhi_epi16(a, 0b11111111, a, b); + let e = _mm_set_epi16(33, 1, 34, 2, 35, 3, 36, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_unpackhi_epi16() { + let a = _mm_set_epi16(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm_set_epi16(33, 34, 35, 36, 37, 38, 39, 40); + let r = _mm_maskz_unpackhi_epi16(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_unpackhi_epi16(0b11111111, a, b); + let e = _mm_set_epi16(33, 1, 34, 2, 35, 3, 36, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_unpackhi_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64); + #[rustfmt::skip] + let b = _mm512_set_epi8(65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, + 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, + 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 0); + let r = _mm512_unpackhi_epi8(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi8(65, 1, 66, 2, 67, 3, 68, 4, 69, 5, 70, 6, 71, 7, 72, 8, + 81, 17, 82, 18, 83, 19, 84, 20, 85, 21, 86, 22, 87, 23, 88, 24, + 97, 33, 98, 34, 99, 35, 100, 36, 101, 37, 102, 38, 103, 39, 104, 40, + 113, 49, 114, 50, 115, 51, 116, 52, 117, 53, 118, 54, 119, 55, 120, 56); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_unpackhi_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64); + #[rustfmt::skip] + let b = _mm512_set_epi8(65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, + 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, + 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 0); + let r = _mm512_mask_unpackhi_epi8(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_unpackhi_epi8( + a, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(65, 1, 66, 2, 67, 3, 68, 4, 69, 5, 70, 6, 71, 7, 72, 8, + 81, 17, 82, 18, 83, 19, 84, 20, 85, 21, 86, 22, 87, 23, 88, 24, + 97, 33, 98, 34, 99, 35, 100, 36, 101, 37, 102, 38, 103, 39, 104, 40, + 113, 49, 114, 50, 115, 51, 116, 52, 117, 53, 118, 54, 119, 55, 120, 56); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_unpackhi_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64); + #[rustfmt::skip] + let b = _mm512_set_epi8(65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, + 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, + 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 0); + let r = _mm512_maskz_unpackhi_epi8(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_unpackhi_epi8( + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(65, 1, 66, 2, 67, 3, 68, 4, 69, 5, 70, 6, 71, 7, 72, 8, + 81, 17, 82, 18, 83, 19, 84, 20, 85, 21, 86, 22, 87, 23, 88, 24, + 97, 33, 98, 34, 99, 35, 100, 36, 101, 37, 102, 38, 103, 39, 104, 40, + 113, 49, 114, 50, 115, 51, 116, 52, 117, 53, 118, 54, 119, 55, 120, 56); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_unpackhi_epi8() { + #[rustfmt::skip] + let a = _mm256_set_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32); + #[rustfmt::skip] + let b = _mm256_set_epi8(65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, + 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96); + let r = _mm256_mask_unpackhi_epi8(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_unpackhi_epi8(a, 0b11111111_11111111_11111111_11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(65, 1, 66, 2, 67, 3, 68, 4, 69, 5, 70, 6, 71, 7, 72, 8, + 81, 17, 82, 18, 83, 19, 84, 20, 85, 21, 86, 22, 87, 23, 88, 24); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_unpackhi_epi8() { + #[rustfmt::skip] + let a = _mm256_set_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32); + #[rustfmt::skip] + let b = _mm256_set_epi8(65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, + 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96); + let r = _mm256_maskz_unpackhi_epi8(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_unpackhi_epi8(0b11111111_11111111_11111111_11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(65, 1, 66, 2, 67, 3, 68, 4, 69, 5, 70, 6, 71, 7, 72, 8, + 81, 17, 82, 18, 83, 19, 84, 20, 85, 21, 86, 22, 87, 23, 88, 24); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_unpackhi_epi8() { + let a = _mm_set_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b = _mm_set_epi8( + 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, + ); + let r = _mm_mask_unpackhi_epi8(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_unpackhi_epi8(a, 0b11111111_11111111, a, b); + let e = _mm_set_epi8(65, 1, 66, 2, 67, 3, 68, 4, 69, 5, 70, 6, 71, 7, 72, 8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_unpackhi_epi8() { + let a = _mm_set_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b = _mm_set_epi8( + 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, + ); + let r = _mm_maskz_unpackhi_epi8(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_unpackhi_epi8(0b11111111_11111111, a, b); + let e = _mm_set_epi8(65, 1, 66, 2, 67, 3, 68, 4, 69, 5, 70, 6, 71, 7, 72, 8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_unpacklo_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32); + #[rustfmt::skip] + let b = _mm512_set_epi16(33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64); + let r = _mm512_unpacklo_epi16(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(37, 5, 38, 6, 39, 7, 40, 8, 45, 13, 46, 14, 47, 15, 48, 16, + 53, 21, 54, 22, 55, 23, 56, 24, 61, 29, 62, 30, 63, 31, 64, 32); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_unpacklo_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32); + #[rustfmt::skip] + let b = _mm512_set_epi16(33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64); + let r = _mm512_mask_unpacklo_epi16(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_unpacklo_epi16(a, 0b11111111_11111111_11111111_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(37, 5, 38, 6, 39, 7, 40, 8, 45, 13, 46, 14, 47, 15, 48, 16, + 53, 21, 54, 22, 55, 23, 56, 24, 61, 29, 62, 30, 63, 31, 64, 32); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_unpacklo_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32); + #[rustfmt::skip] + let b = _mm512_set_epi16(33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64); + let r = _mm512_maskz_unpacklo_epi16(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_unpacklo_epi16(0b11111111_11111111_11111111_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi16(37, 5, 38, 6, 39, 7, 40, 8, 45, 13, 46, 14, 47, 15, 48, 16, + 53, 21, 54, 22, 55, 23, 56, 24, 61, 29, 62, 30, 63, 31, 64, 32); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_unpacklo_epi16() { + let a = _mm256_set_epi16(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b = _mm256_set_epi16( + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + ); + let r = _mm256_mask_unpacklo_epi16(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_unpacklo_epi16(a, 0b11111111_11111111, a, b); + let e = _mm256_set_epi16(37, 5, 38, 6, 39, 7, 40, 8, 45, 13, 46, 14, 47, 15, 48, 16); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_unpacklo_epi16() { + let a = _mm256_set_epi16(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b = _mm256_set_epi16( + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + ); + let r = _mm256_maskz_unpacklo_epi16(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_unpacklo_epi16(0b11111111_11111111, a, b); + let e = _mm256_set_epi16(37, 5, 38, 6, 39, 7, 40, 8, 45, 13, 46, 14, 47, 15, 48, 16); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_unpacklo_epi16() { + let a = _mm_set_epi16(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm_set_epi16(33, 34, 35, 36, 37, 38, 39, 40); + let r = _mm_mask_unpacklo_epi16(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_unpacklo_epi16(a, 0b11111111, a, b); + let e = _mm_set_epi16(37, 5, 38, 6, 39, 7, 40, 8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_unpacklo_epi16() { + let a = _mm_set_epi16(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm_set_epi16(33, 34, 35, 36, 37, 38, 39, 40); + let r = _mm_maskz_unpacklo_epi16(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_unpacklo_epi16(0b11111111, a, b); + let e = _mm_set_epi16(37, 5, 38, 6, 39, 7, 40, 8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_unpacklo_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64); + #[rustfmt::skip] + let b = _mm512_set_epi8(65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, + 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, + 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 0); + let r = _mm512_unpacklo_epi8(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi8(73, 9, 74, 10, 75, 11, 76, 12, 77, 13, 78, 14, 79, 15, 80, 16, + 89, 25, 90, 26, 91, 27, 92, 28, 93, 29, 94, 30, 95, 31, 96, 32, + 105, 41, 106, 42, 107, 43, 108, 44, 109, 45, 110, 46, 111, 47, 112, 48, + 121, 57, 122, 58, 123, 59, 124, 60, 125, 61, 126, 62, 127, 63, 0, 64); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_unpacklo_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64); + #[rustfmt::skip] + let b = _mm512_set_epi8(65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, + 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, + 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 0); + let r = _mm512_mask_unpacklo_epi8(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_unpacklo_epi8( + a, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(73, 9, 74, 10, 75, 11, 76, 12, 77, 13, 78, 14, 79, 15, 80, 16, + 89, 25, 90, 26, 91, 27, 92, 28, 93, 29, 94, 30, 95, 31, 96, 32, + 105, 41, 106, 42, 107, 43, 108, 44, 109, 45, 110, 46, 111, 47, 112, 48, + 121, 57, 122, 58, 123, 59, 124, 60, 125, 61, 126, 62, 127, 63, 0, 64); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_unpacklo_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64); + #[rustfmt::skip] + let b = _mm512_set_epi8(65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, + 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, + 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 0); + let r = _mm512_maskz_unpacklo_epi8(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_unpacklo_epi8( + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(73, 9, 74, 10, 75, 11, 76, 12, 77, 13, 78, 14, 79, 15, 80, 16, + 89, 25, 90, 26, 91, 27, 92, 28, 93, 29, 94, 30, 95, 31, 96, 32, + 105, 41, 106, 42, 107, 43, 108, 44, 109, 45, 110, 46, 111, 47, 112, 48, + 121, 57, 122, 58, 123, 59, 124, 60, 125, 61, 126, 62, 127, 63, 0, 64); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_unpacklo_epi8() { + #[rustfmt::skip] + let a = _mm256_set_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32); + #[rustfmt::skip] + let b = _mm256_set_epi8(65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, + 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96); + let r = _mm256_mask_unpacklo_epi8(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_unpacklo_epi8(a, 0b11111111_11111111_11111111_11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(73, 9, 74, 10, 75, 11, 76, 12, 77, 13, 78, 14, 79, 15, 80, 16, + 89, 25, 90, 26, 91, 27, 92, 28, 93, 29, 94, 30, 95, 31, 96, 32); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_unpacklo_epi8() { + #[rustfmt::skip] + let a = _mm256_set_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32); + #[rustfmt::skip] + let b = _mm256_set_epi8(65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, + 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96); + let r = _mm256_maskz_unpacklo_epi8(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_unpacklo_epi8(0b11111111_11111111_11111111_11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(73, 9, 74, 10, 75, 11, 76, 12, 77, 13, 78, 14, 79, 15, 80, 16, + 89, 25, 90, 26, 91, 27, 92, 28, 93, 29, 94, 30, 95, 31, 96, 32); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_unpacklo_epi8() { + let a = _mm_set_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b = _mm_set_epi8( + 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, + ); + let r = _mm_mask_unpacklo_epi8(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_unpacklo_epi8(a, 0b11111111_11111111, a, b); + let e = _mm_set_epi8( + 73, 9, 74, 10, 75, 11, 76, 12, 77, 13, 78, 14, 79, 15, 80, 16, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_unpacklo_epi8() { + let a = _mm_set_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b = _mm_set_epi8( + 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, + ); + let r = _mm_maskz_unpacklo_epi8(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_unpacklo_epi8(0b11111111_11111111, a, b); + let e = _mm_set_epi8( + 73, 9, 74, 10, 75, 11, 76, 12, 77, 13, 78, 14, 79, 15, 80, 16, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_mov_epi16() { + let src = _mm512_set1_epi16(1); + let a = _mm512_set1_epi16(2); + let r = _mm512_mask_mov_epi16(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_mov_epi16(src, 0b11111111_11111111_11111111_11111111, a); + assert_eq_m512i(r, a); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_mov_epi16() { + let a = _mm512_set1_epi16(2); + let r = _mm512_maskz_mov_epi16(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_mov_epi16(0b11111111_11111111_11111111_11111111, a); + assert_eq_m512i(r, a); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_mov_epi16() { + let src = _mm256_set1_epi16(1); + let a = _mm256_set1_epi16(2); + let r = _mm256_mask_mov_epi16(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_mov_epi16(src, 0b11111111_11111111, a); + assert_eq_m256i(r, a); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_mov_epi16() { + let a = _mm256_set1_epi16(2); + let r = _mm256_maskz_mov_epi16(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_mov_epi16(0b11111111_11111111, a); + assert_eq_m256i(r, a); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_mov_epi16() { + let src = _mm_set1_epi16(1); + let a = _mm_set1_epi16(2); + let r = _mm_mask_mov_epi16(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_mov_epi16(src, 0b11111111, a); + assert_eq_m128i(r, a); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_mov_epi16() { + let a = _mm_set1_epi16(2); + let r = _mm_maskz_mov_epi16(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_mov_epi16(0b11111111, a); + assert_eq_m128i(r, a); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_mov_epi8() { + let src = _mm512_set1_epi8(1); + let a = _mm512_set1_epi8(2); + let r = _mm512_mask_mov_epi8(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_mov_epi8( + src, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + a, + ); + assert_eq_m512i(r, a); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_mov_epi8() { + let a = _mm512_set1_epi8(2); + let r = _mm512_maskz_mov_epi8(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_mov_epi8( + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + a, + ); + assert_eq_m512i(r, a); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_mov_epi8() { + let src = _mm256_set1_epi8(1); + let a = _mm256_set1_epi8(2); + let r = _mm256_mask_mov_epi8(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_mov_epi8(src, 0b11111111_11111111_11111111_11111111, a); + assert_eq_m256i(r, a); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_mov_epi8() { + let a = _mm256_set1_epi8(2); + let r = _mm256_maskz_mov_epi8(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_mov_epi8(0b11111111_11111111_11111111_11111111, a); + assert_eq_m256i(r, a); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_mov_epi8() { + let src = _mm_set1_epi8(1); + let a = _mm_set1_epi8(2); + let r = _mm_mask_mov_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_mov_epi8(src, 0b11111111_11111111, a); + assert_eq_m128i(r, a); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_mov_epi8() { + let a = _mm_set1_epi8(2); + let r = _mm_maskz_mov_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_mov_epi8(0b11111111_11111111, a); + assert_eq_m128i(r, a); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_set1_epi16() { + let src = _mm512_set1_epi16(2); + let a: i16 = 11; + let r = _mm512_mask_set1_epi16(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_set1_epi16(src, 0b11111111_11111111_11111111_11111111, a); + let e = _mm512_set1_epi16(11); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_set1_epi16() { + let a: i16 = 11; + let r = _mm512_maskz_set1_epi16(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_set1_epi16(0b11111111_11111111_11111111_11111111, a); + let e = _mm512_set1_epi16(11); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_set1_epi16() { + let src = _mm256_set1_epi16(2); + let a: i16 = 11; + let r = _mm256_mask_set1_epi16(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_set1_epi16(src, 0b11111111_11111111, a); + let e = _mm256_set1_epi16(11); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_set1_epi16() { + let a: i16 = 11; + let r = _mm256_maskz_set1_epi16(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_set1_epi16(0b11111111_11111111, a); + let e = _mm256_set1_epi16(11); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_set1_epi16() { + let src = _mm_set1_epi16(2); + let a: i16 = 11; + let r = _mm_mask_set1_epi16(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_set1_epi16(src, 0b11111111, a); + let e = _mm_set1_epi16(11); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_set1_epi16() { + let a: i16 = 11; + let r = _mm_maskz_set1_epi16(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_set1_epi16(0b11111111, a); + let e = _mm_set1_epi16(11); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_set1_epi8() { + let src = _mm512_set1_epi8(2); + let a: i8 = 11; + let r = _mm512_mask_set1_epi8(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_set1_epi8( + src, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + a, + ); + let e = _mm512_set1_epi8(11); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_set1_epi8() { + let a: i8 = 11; + let r = _mm512_maskz_set1_epi8(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_set1_epi8( + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + a, + ); + let e = _mm512_set1_epi8(11); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_set1_epi8() { + let src = _mm256_set1_epi8(2); + let a: i8 = 11; + let r = _mm256_mask_set1_epi8(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_set1_epi8(src, 0b11111111_11111111_11111111_11111111, a); + let e = _mm256_set1_epi8(11); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_set1_epi8() { + let a: i8 = 11; + let r = _mm256_maskz_set1_epi8(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_set1_epi8(0b11111111_11111111_11111111_11111111, a); + let e = _mm256_set1_epi8(11); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_set1_epi8() { + let src = _mm_set1_epi8(2); + let a: i8 = 11; + let r = _mm_mask_set1_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_set1_epi8(src, 0b11111111_11111111, a); + let e = _mm_set1_epi8(11); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_set1_epi8() { + let a: i8 = 11; + let r = _mm_maskz_set1_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_set1_epi8(0b11111111_11111111, a); + let e = _mm_set1_epi8(11); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_shufflelo_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ); + #[rustfmt::skip] + let e = _mm512_set_epi16( + 0, 1, 2, 3, 7, 6, 6, 4, 8, 9, 10, 11, 15, 14, 14, 12, + 16, 17, 18, 19, 23, 22, 22, 20, 24, 25, 26, 27, 31, 30, 30, 28, + ); + let r = _mm512_shufflelo_epi16::<0b00_01_01_11>(a); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_shufflelo_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ); + let r = _mm512_mask_shufflelo_epi16::<0b00_01_01_11>(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_shufflelo_epi16::<0b00_01_01_11>( + a, + 0b11111111_11111111_11111111_11111111, + a, + ); + #[rustfmt::skip] + let e = _mm512_set_epi16( + 0, 1, 2, 3, 7, 6, 6, 4, 8, 9, 10, 11, 15, 14, 14, 12, + 16, 17, 18, 19, 23, 22, 22, 20, 24, 25, 26, 27, 31, 30, 30, 28, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_shufflelo_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ); + let r = _mm512_maskz_shufflelo_epi16::<0b00_01_01_11>(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = + _mm512_maskz_shufflelo_epi16::<0b00_01_01_11>(0b11111111_11111111_11111111_11111111, a); + #[rustfmt::skip] + let e = _mm512_set_epi16( + 0, 1, 2, 3, 7, 6, 6, 4, 8, 9, 10, 11, 15, 14, 14, 12, + 16, 17, 18, 19, 23, 22, 22, 20, 24, 25, 26, 27, 31, 30, 30, 28, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_shufflelo_epi16() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm256_mask_shufflelo_epi16::<0b00_01_01_11>(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_shufflelo_epi16::<0b00_01_01_11>(a, 0b11111111_11111111, a); + let e = _mm256_set_epi16(0, 1, 2, 3, 7, 6, 6, 4, 8, 9, 10, 11, 15, 14, 14, 12); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_shufflelo_epi16() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm256_maskz_shufflelo_epi16::<0b00_01_01_11>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_shufflelo_epi16::<0b00_01_01_11>(0b11111111_11111111, a); + let e = _mm256_set_epi16(0, 1, 2, 3, 7, 6, 6, 4, 8, 9, 10, 11, 15, 14, 14, 12); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_shufflelo_epi16() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm_mask_shufflelo_epi16::<0b00_01_01_11>(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_shufflelo_epi16::<0b00_01_01_11>(a, 0b11111111, a); + let e = _mm_set_epi16(0, 1, 2, 3, 7, 6, 6, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_shufflelo_epi16() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm_maskz_shufflelo_epi16::<0b00_01_01_11>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_shufflelo_epi16::<0b00_01_01_11>(0b11111111, a); + let e = _mm_set_epi16(0, 1, 2, 3, 7, 6, 6, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_shufflehi_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ); + #[rustfmt::skip] + let e = _mm512_set_epi16( + 3, 2, 2, 0, 4, 5, 6, 7, 11, 10, 10, 8, 12, 13, 14, 15, + 19, 18, 18, 16, 20, 21, 22, 23, 27, 26, 26, 24, 28, 29, 30, 31, + ); + let r = _mm512_shufflehi_epi16::<0b00_01_01_11>(a); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_shufflehi_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ); + let r = _mm512_mask_shufflehi_epi16::<0b00_01_01_11>(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_shufflehi_epi16::<0b00_01_01_11>( + a, + 0b11111111_11111111_11111111_11111111, + a, + ); + #[rustfmt::skip] + let e = _mm512_set_epi16( + 3, 2, 2, 0, 4, 5, 6, 7, 11, 10, 10, 8, 12, 13, 14, 15, + 19, 18, 18, 16, 20, 21, 22, 23, 27, 26, 26, 24, 28, 29, 30, 31, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_shufflehi_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ); + let r = _mm512_maskz_shufflehi_epi16::<0b00_01_01_11>(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = + _mm512_maskz_shufflehi_epi16::<0b00_01_01_11>(0b11111111_11111111_11111111_11111111, a); + #[rustfmt::skip] + let e = _mm512_set_epi16( + 3, 2, 2, 0, 4, 5, 6, 7, 11, 10, 10, 8, 12, 13, 14, 15, + 19, 18, 18, 16, 20, 21, 22, 23, 27, 26, 26, 24, 28, 29, 30, 31, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_shufflehi_epi16() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm256_mask_shufflehi_epi16::<0b00_01_01_11>(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_shufflehi_epi16::<0b00_01_01_11>(a, 0b11111111_11111111, a); + let e = _mm256_set_epi16(3, 2, 2, 0, 4, 5, 6, 7, 11, 10, 10, 8, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_shufflehi_epi16() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm256_maskz_shufflehi_epi16::<0b00_01_01_11>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_shufflehi_epi16::<0b00_01_01_11>(0b11111111_11111111, a); + let e = _mm256_set_epi16(3, 2, 2, 0, 4, 5, 6, 7, 11, 10, 10, 8, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_shufflehi_epi16() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm_mask_shufflehi_epi16::<0b00_01_01_11>(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_shufflehi_epi16::<0b00_01_01_11>(a, 0b11111111, a); + let e = _mm_set_epi16(3, 2, 2, 0, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_shufflehi_epi16() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm_maskz_shufflehi_epi16::<0b00_01_01_11>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_shufflehi_epi16::<0b00_01_01_11>(0b11111111, a); + let e = _mm_set_epi16(3, 2, 2, 0, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_shuffle_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63); + let b = _mm512_set1_epi8(1); + let r = _mm512_shuffle_epi8(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi8(14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, + 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_shuffle_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63); + let b = _mm512_set1_epi8(1); + let r = _mm512_mask_shuffle_epi8(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_shuffle_epi8( + a, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, + 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_shuffle_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63); + let b = _mm512_set1_epi8(1); + let r = _mm512_maskz_shuffle_epi8(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_shuffle_epi8( + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8(14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, + 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_shuffle_epi8() { + #[rustfmt::skip] + let a = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let b = _mm256_set1_epi8(1); + let r = _mm256_mask_shuffle_epi8(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_shuffle_epi8(a, 0b11111111_11111111_11111111_11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_shuffle_epi8() { + #[rustfmt::skip] + let a = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let b = _mm256_set1_epi8(1); + let r = _mm256_maskz_shuffle_epi8(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_shuffle_epi8(0b11111111_11111111_11111111_11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8(14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_shuffle_epi8() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm_set1_epi8(1); + let r = _mm_mask_shuffle_epi8(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_shuffle_epi8(a, 0b11111111_11111111, a, b); + let e = _mm_set_epi8( + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_shuffle_epi8() { + #[rustfmt::skip] + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm_set1_epi8(1); + let r = _mm_maskz_shuffle_epi8(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_shuffle_epi8(0b11111111_11111111, a, b); + let e = _mm_set_epi8( + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_test_epi16_mask() { + let a = _mm512_set1_epi16(1 << 0); + let b = _mm512_set1_epi16(1 << 0 | 1 << 1); + let r = _mm512_test_epi16_mask(a, b); + let e: __mmask32 = 0b11111111_11111111_11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_test_epi16_mask() { + let a = _mm512_set1_epi16(1 << 0); + let b = _mm512_set1_epi16(1 << 0 | 1 << 1); + let r = _mm512_mask_test_epi16_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm512_mask_test_epi16_mask(0b11111111_11111111_11111111_11111111, a, b); + let e: __mmask32 = 0b11111111_11111111_11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_test_epi16_mask() { + let a = _mm256_set1_epi16(1 << 0); + let b = _mm256_set1_epi16(1 << 0 | 1 << 1); + let r = _mm256_test_epi16_mask(a, b); + let e: __mmask16 = 0b11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_test_epi16_mask() { + let a = _mm256_set1_epi16(1 << 0); + let b = _mm256_set1_epi16(1 << 0 | 1 << 1); + let r = _mm256_mask_test_epi16_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm256_mask_test_epi16_mask(0b11111111_11111111, a, b); + let e: __mmask16 = 0b11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_test_epi16_mask() { + let a = _mm_set1_epi16(1 << 0); + let b = _mm_set1_epi16(1 << 0 | 1 << 1); + let r = _mm_test_epi16_mask(a, b); + let e: __mmask8 = 0b11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_test_epi16_mask() { + let a = _mm_set1_epi16(1 << 0); + let b = _mm_set1_epi16(1 << 0 | 1 << 1); + let r = _mm_mask_test_epi16_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm_mask_test_epi16_mask(0b11111111, a, b); + let e: __mmask8 = 0b11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_test_epi8_mask() { + let a = _mm512_set1_epi8(1 << 0); + let b = _mm512_set1_epi8(1 << 0 | 1 << 1); + let r = _mm512_test_epi8_mask(a, b); + let e: __mmask64 = + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_test_epi8_mask() { + let a = _mm512_set1_epi8(1 << 0); + let b = _mm512_set1_epi8(1 << 0 | 1 << 1); + let r = _mm512_mask_test_epi8_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm512_mask_test_epi8_mask( + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + a, + b, + ); + let e: __mmask64 = + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_test_epi8_mask() { + let a = _mm256_set1_epi8(1 << 0); + let b = _mm256_set1_epi8(1 << 0 | 1 << 1); + let r = _mm256_test_epi8_mask(a, b); + let e: __mmask32 = 0b11111111_11111111_11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_test_epi8_mask() { + let a = _mm256_set1_epi8(1 << 0); + let b = _mm256_set1_epi8(1 << 0 | 1 << 1); + let r = _mm256_mask_test_epi8_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm256_mask_test_epi8_mask(0b11111111_11111111_11111111_11111111, a, b); + let e: __mmask32 = 0b11111111_11111111_11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_test_epi8_mask() { + let a = _mm_set1_epi8(1 << 0); + let b = _mm_set1_epi8(1 << 0 | 1 << 1); + let r = _mm_test_epi8_mask(a, b); + let e: __mmask16 = 0b11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_test_epi8_mask() { + let a = _mm_set1_epi8(1 << 0); + let b = _mm_set1_epi8(1 << 0 | 1 << 1); + let r = _mm_mask_test_epi8_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm_mask_test_epi8_mask(0b11111111_11111111, a, b); + let e: __mmask16 = 0b11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_testn_epi16_mask() { + let a = _mm512_set1_epi16(1 << 0); + let b = _mm512_set1_epi16(1 << 0 | 1 << 1); + let r = _mm512_testn_epi16_mask(a, b); + let e: __mmask32 = 0b00000000_00000000_00000000_00000000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_testn_epi16_mask() { + let a = _mm512_set1_epi16(1 << 0); + let b = _mm512_set1_epi16(1 << 0 | 1 << 1); + let r = _mm512_mask_testn_epi16_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm512_mask_testn_epi16_mask(0b11111111_11111111_11111111_11111111, a, b); + let e: __mmask32 = 0b00000000_00000000_00000000_00000000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_testn_epi16_mask() { + let a = _mm256_set1_epi16(1 << 0); + let b = _mm256_set1_epi16(1 << 0 | 1 << 1); + let r = _mm256_testn_epi16_mask(a, b); + let e: __mmask16 = 0b00000000_00000000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_testn_epi16_mask() { + let a = _mm256_set1_epi16(1 << 0); + let b = _mm256_set1_epi16(1 << 0 | 1 << 1); + let r = _mm256_mask_testn_epi16_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm256_mask_testn_epi16_mask(0b11111111_11111111, a, b); + let e: __mmask16 = 0b00000000_00000000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_testn_epi16_mask() { + let a = _mm_set1_epi16(1 << 0); + let b = _mm_set1_epi16(1 << 0 | 1 << 1); + let r = _mm_testn_epi16_mask(a, b); + let e: __mmask8 = 0b00000000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_testn_epi16_mask() { + let a = _mm_set1_epi16(1 << 0); + let b = _mm_set1_epi16(1 << 0 | 1 << 1); + let r = _mm_mask_testn_epi16_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm_mask_testn_epi16_mask(0b11111111, a, b); + let e: __mmask8 = 0b00000000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_testn_epi8_mask() { + let a = _mm512_set1_epi8(1 << 0); + let b = _mm512_set1_epi8(1 << 0 | 1 << 1); + let r = _mm512_testn_epi8_mask(a, b); + let e: __mmask64 = + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_testn_epi8_mask() { + let a = _mm512_set1_epi8(1 << 0); + let b = _mm512_set1_epi8(1 << 0 | 1 << 1); + let r = _mm512_mask_testn_epi8_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm512_mask_testn_epi8_mask( + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + a, + b, + ); + let e: __mmask64 = + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_testn_epi8_mask() { + let a = _mm256_set1_epi8(1 << 0); + let b = _mm256_set1_epi8(1 << 0 | 1 << 1); + let r = _mm256_testn_epi8_mask(a, b); + let e: __mmask32 = 0b00000000_00000000_00000000_00000000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_testn_epi8_mask() { + let a = _mm256_set1_epi8(1 << 0); + let b = _mm256_set1_epi8(1 << 0 | 1 << 1); + let r = _mm256_mask_testn_epi8_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm256_mask_testn_epi8_mask(0b11111111_11111111_11111111_11111111, a, b); + let e: __mmask32 = 0b00000000_00000000_00000000_00000000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_testn_epi8_mask() { + let a = _mm_set1_epi8(1 << 0); + let b = _mm_set1_epi8(1 << 0 | 1 << 1); + let r = _mm_testn_epi8_mask(a, b); + let e: __mmask16 = 0b00000000_00000000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_testn_epi8_mask() { + let a = _mm_set1_epi8(1 << 0); + let b = _mm_set1_epi8(1 << 0 | 1 << 1); + let r = _mm_mask_testn_epi8_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm_mask_testn_epi8_mask(0b11111111_11111111, a, b); + let e: __mmask16 = 0b00000000_00000000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_store_mask64() { + let a: __mmask64 = + 0b11111111_00000000_11111111_00000000_11111111_00000000_11111111_00000000; + let mut r = 0; + _store_mask64(&mut r as *mut _ as *mut u64, a); + assert_eq!(r, a); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_store_mask32() { + let a: __mmask32 = 0b11111111_00000000_11111111_00000000; + let mut r = 0; + _store_mask32(&mut r as *mut _ as *mut u32, a); + assert_eq!(r, a); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_load_mask64() { + let p: __mmask64 = + 0b11111111_00000000_11111111_00000000_11111111_00000000_11111111_00000000; + let r = _load_mask64(&p); + let e: __mmask64 = + 0b11111111_00000000_11111111_00000000_11111111_00000000_11111111_00000000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_load_mask32() { + let p: __mmask32 = 0b11111111_00000000_11111111_00000000; + let r = _load_mask32(&p); + let e: __mmask32 = 0b11111111_00000000_11111111_00000000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_sad_epu8() { + let a = _mm512_set1_epi8(2); + let b = _mm512_set1_epi8(4); + let r = _mm512_sad_epu8(a, b); + let e = _mm512_set1_epi64(16); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_dbsad_epu8() { + let a = _mm512_set1_epi8(2); + let b = _mm512_set1_epi8(4); + let r = _mm512_dbsad_epu8::<0>(a, b); + let e = _mm512_set1_epi16(8); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_dbsad_epu8() { + let src = _mm512_set1_epi16(1); + let a = _mm512_set1_epi8(2); + let b = _mm512_set1_epi8(4); + let r = _mm512_mask_dbsad_epu8::<0>(src, 0, a, b); + assert_eq_m512i(r, src); + let r = _mm512_mask_dbsad_epu8::<0>(src, 0b11111111_11111111_11111111_11111111, a, b); + let e = _mm512_set1_epi16(8); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_dbsad_epu8() { + let a = _mm512_set1_epi8(2); + let b = _mm512_set1_epi8(4); + let r = _mm512_maskz_dbsad_epu8::<0>(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_dbsad_epu8::<0>(0b11111111_11111111_11111111_11111111, a, b); + let e = _mm512_set1_epi16(8); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_dbsad_epu8() { + let a = _mm256_set1_epi8(2); + let b = _mm256_set1_epi8(4); + let r = _mm256_dbsad_epu8::<0>(a, b); + let e = _mm256_set1_epi16(8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_dbsad_epu8() { + let src = _mm256_set1_epi16(1); + let a = _mm256_set1_epi8(2); + let b = _mm256_set1_epi8(4); + let r = _mm256_mask_dbsad_epu8::<0>(src, 0, a, b); + assert_eq_m256i(r, src); + let r = _mm256_mask_dbsad_epu8::<0>(src, 0b11111111_11111111, a, b); + let e = _mm256_set1_epi16(8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_dbsad_epu8() { + let a = _mm256_set1_epi8(2); + let b = _mm256_set1_epi8(4); + let r = _mm256_maskz_dbsad_epu8::<0>(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_dbsad_epu8::<0>(0b11111111_11111111, a, b); + let e = _mm256_set1_epi16(8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_dbsad_epu8() { + let a = _mm_set1_epi8(2); + let b = _mm_set1_epi8(4); + let r = _mm_dbsad_epu8::<0>(a, b); + let e = _mm_set1_epi16(8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_dbsad_epu8() { + let src = _mm_set1_epi16(1); + let a = _mm_set1_epi8(2); + let b = _mm_set1_epi8(4); + let r = _mm_mask_dbsad_epu8::<0>(src, 0, a, b); + assert_eq_m128i(r, src); + let r = _mm_mask_dbsad_epu8::<0>(src, 0b11111111, a, b); + let e = _mm_set1_epi16(8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_dbsad_epu8() { + let a = _mm_set1_epi8(2); + let b = _mm_set1_epi8(4); + let r = _mm_maskz_dbsad_epu8::<0>(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_dbsad_epu8::<0>(0b11111111, a, b); + let e = _mm_set1_epi16(8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_movepi16_mask() { + let a = _mm512_set1_epi16(1 << 15); + let r = _mm512_movepi16_mask(a); + let e: __mmask32 = 0b11111111_11111111_11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_movepi16_mask() { + let a = _mm256_set1_epi16(1 << 15); + let r = _mm256_movepi16_mask(a); + let e: __mmask16 = 0b11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_movepi16_mask() { + let a = _mm_set1_epi16(1 << 15); + let r = _mm_movepi16_mask(a); + let e: __mmask8 = 0b11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_movepi8_mask() { + let a = _mm512_set1_epi8(1 << 7); + let r = _mm512_movepi8_mask(a); + let e: __mmask64 = + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_movepi8_mask() { + let a = _mm256_set1_epi8(1 << 7); + let r = _mm256_movepi8_mask(a); + let e: __mmask32 = 0b11111111_11111111_11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_movepi8_mask() { + let a = _mm_set1_epi8(1 << 7); + let r = _mm_movepi8_mask(a); + let e: __mmask16 = 0b11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_movm_epi16() { + let a: __mmask32 = 0b11111111_11111111_11111111_11111111; + let r = _mm512_movm_epi16(a); + let e = _mm512_set1_epi16( + 1 << 15 + | 1 << 14 + | 1 << 13 + | 1 << 12 + | 1 << 11 + | 1 << 10 + | 1 << 9 + | 1 << 8 + | 1 << 7 + | 1 << 6 + | 1 << 5 + | 1 << 4 + | 1 << 3 + | 1 << 2 + | 1 << 1 + | 1 << 0, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_movm_epi16() { + let a: __mmask16 = 0b11111111_11111111; + let r = _mm256_movm_epi16(a); + let e = _mm256_set1_epi16( + 1 << 15 + | 1 << 14 + | 1 << 13 + | 1 << 12 + | 1 << 11 + | 1 << 10 + | 1 << 9 + | 1 << 8 + | 1 << 7 + | 1 << 6 + | 1 << 5 + | 1 << 4 + | 1 << 3 + | 1 << 2 + | 1 << 1 + | 1 << 0, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_movm_epi16() { + let a: __mmask8 = 0b11111111; + let r = _mm_movm_epi16(a); + let e = _mm_set1_epi16( + 1 << 15 + | 1 << 14 + | 1 << 13 + | 1 << 12 + | 1 << 11 + | 1 << 10 + | 1 << 9 + | 1 << 8 + | 1 << 7 + | 1 << 6 + | 1 << 5 + | 1 << 4 + | 1 << 3 + | 1 << 2 + | 1 << 1 + | 1 << 0, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_movm_epi8() { + let a: __mmask64 = + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111; + let r = _mm512_movm_epi8(a); + let e = + _mm512_set1_epi8(1 << 7 | 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_movm_epi8() { + let a: __mmask32 = 0b11111111_11111111_11111111_11111111; + let r = _mm256_movm_epi8(a); + let e = + _mm256_set1_epi8(1 << 7 | 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_movm_epi8() { + let a: __mmask16 = 0b11111111_11111111; + let r = _mm_movm_epi8(a); + let e = + _mm_set1_epi8(1 << 7 | 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_kadd_mask32() { + let a: __mmask32 = 11; + let b: __mmask32 = 22; + let r = _kadd_mask32(a, b); + let e: __mmask32 = 33; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_kadd_mask64() { + let a: __mmask64 = 11; + let b: __mmask64 = 22; + let r = _kadd_mask64(a, b); + let e: __mmask64 = 33; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_kand_mask32() { + let a: __mmask32 = 0b11001100_00110011_11001100_00110011; + let b: __mmask32 = 0b11001100_00110011_11001100_00110011; + let r = _kand_mask32(a, b); + let e: __mmask32 = 0b11001100_00110011_11001100_00110011; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_kand_mask64() { + let a: __mmask64 = + 0b11001100_00110011_11001100_00110011_11001100_00110011_11001100_00110011; + let b: __mmask64 = + 0b11001100_00110011_11001100_00110011_11001100_00110011_11001100_00110011; + let r = _kand_mask64(a, b); + let e: __mmask64 = + 0b11001100_00110011_11001100_00110011_11001100_00110011_11001100_00110011; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_knot_mask32() { + let a: __mmask32 = 0b11001100_00110011_11001100_00110011; + let r = _knot_mask32(a); + let e: __mmask32 = 0b00110011_11001100_00110011_11001100; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_knot_mask64() { + let a: __mmask64 = + 0b11001100_00110011_11001100_00110011_11001100_00110011_11001100_00110011; + let r = _knot_mask64(a); + let e: __mmask64 = + 0b00110011_11001100_00110011_11001100_00110011_11001100_00110011_11001100; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_kandn_mask32() { + let a: __mmask32 = 0b11001100_00110011_11001100_00110011; + let b: __mmask32 = 0b11001100_00110011_11001100_00110011; + let r = _kandn_mask32(a, b); + let e: __mmask32 = 0b00000000_00000000_00000000_00000000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_kandn_mask64() { + let a: __mmask64 = + 0b11001100_00110011_11001100_00110011_11001100_00110011_11001100_00110011; + let b: __mmask64 = + 0b11001100_00110011_11001100_00110011_11001100_00110011_11001100_00110011; + let r = _kandn_mask64(a, b); + let e: __mmask64 = + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_kor_mask32() { + let a: __mmask32 = 0b00110011_11001100_00110011_11001100; + let b: __mmask32 = 0b11001100_00110011_11001100_00110011; + let r = _kor_mask32(a, b); + let e: __mmask32 = 0b11111111_11111111_11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_kor_mask64() { + let a: __mmask64 = + 0b00110011_11001100_00110011_11001100_00110011_11001100_00110011_11001100; + let b: __mmask64 = + 0b11001100_00110011_11001100_00110011_11001100_00110011_11001100_00110011; + let r = _kor_mask64(a, b); + let e: __mmask64 = + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_kxor_mask32() { + let a: __mmask32 = 0b00110011_11001100_00110011_11001100; + let b: __mmask32 = 0b11001100_00110011_11001100_00110011; + let r = _kxor_mask32(a, b); + let e: __mmask32 = 0b11111111_11111111_11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_kxor_mask64() { + let a: __mmask64 = + 0b00110011_11001100_00110011_11001100_00110011_11001100_00110011_11001100; + let b: __mmask64 = + 0b11001100_00110011_11001100_00110011_11001100_00110011_11001100_00110011; + let r = _kxor_mask64(a, b); + let e: __mmask64 = + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_kxnor_mask32() { + let a: __mmask32 = 0b00110011_11001100_00110011_11001100; + let b: __mmask32 = 0b11001100_00110011_11001100_00110011; + let r = _kxnor_mask32(a, b); + let e: __mmask32 = 0b00000000_00000000_00000000_00000000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_kxnor_mask64() { + let a: __mmask64 = + 0b00110011_11001100_00110011_11001100_00110011_11001100_00110011_11001100; + let b: __mmask64 = + 0b11001100_00110011_11001100_00110011_11001100_00110011_11001100_00110011; + let r = _kxnor_mask64(a, b); + let e: __mmask64 = + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cvtepi16_epi8() { + let a = _mm512_set1_epi16(2); + let r = _mm512_cvtepi16_epi8(a); + let e = _mm256_set1_epi8(2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cvtepi16_epi8() { + let src = _mm256_set1_epi8(1); + let a = _mm512_set1_epi16(2); + let r = _mm512_mask_cvtepi16_epi8(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm512_mask_cvtepi16_epi8(src, 0b11111111_11111111_11111111_11111111, a); + let e = _mm256_set1_epi8(2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_cvtepi16_epi8() { + let a = _mm512_set1_epi16(2); + let r = _mm512_maskz_cvtepi16_epi8(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm512_maskz_cvtepi16_epi8(0b11111111_11111111_11111111_11111111, a); + let e = _mm256_set1_epi8(2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cvtepi16_epi8() { + let a = _mm256_set1_epi16(2); + let r = _mm256_cvtepi16_epi8(a); + let e = _mm_set1_epi8(2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cvtepi16_epi8() { + let src = _mm_set1_epi8(1); + let a = _mm256_set1_epi16(2); + let r = _mm256_mask_cvtepi16_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvtepi16_epi8(src, 0b11111111_11111111, a); + let e = _mm_set1_epi8(2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_cvtepi16_epi8() { + let a = _mm256_set1_epi16(2); + let r = _mm256_maskz_cvtepi16_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvtepi16_epi8(0b11111111_11111111, a); + let e = _mm_set1_epi8(2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cvtepi16_epi8() { + let a = _mm_set1_epi16(2); + let r = _mm_cvtepi16_epi8(a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cvtepi16_epi8() { + let src = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1); + let a = _mm_set1_epi16(2); + let r = _mm_mask_cvtepi16_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtepi16_epi8(src, 0b11111111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_cvtepi16_epi8() { + let a = _mm_set1_epi16(2); + let r = _mm_maskz_cvtepi16_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtepi16_epi8(0b11111111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cvtsepi16_epi8() { + let a = _mm512_set1_epi16(i16::MAX); + let r = _mm512_cvtsepi16_epi8(a); + let e = _mm256_set1_epi8(i8::MAX); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cvtsepi16_epi8() { + let src = _mm256_set1_epi8(1); + let a = _mm512_set1_epi16(i16::MAX); + let r = _mm512_mask_cvtsepi16_epi8(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm512_mask_cvtsepi16_epi8(src, 0b11111111_11111111_11111111_11111111, a); + let e = _mm256_set1_epi8(i8::MAX); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cvtsepi16_epi8() { + let a = _mm256_set1_epi16(i16::MAX); + let r = _mm256_cvtsepi16_epi8(a); + let e = _mm_set1_epi8(i8::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cvtsepi16_epi8() { + let src = _mm_set1_epi8(1); + let a = _mm256_set1_epi16(i16::MAX); + let r = _mm256_mask_cvtsepi16_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvtsepi16_epi8(src, 0b11111111_11111111, a); + let e = _mm_set1_epi8(i8::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_cvtsepi16_epi8() { + let a = _mm256_set1_epi16(i16::MAX); + let r = _mm256_maskz_cvtsepi16_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvtsepi16_epi8(0b11111111_11111111, a); + let e = _mm_set1_epi8(i8::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cvtsepi16_epi8() { + let a = _mm_set1_epi16(i16::MAX); + let r = _mm_cvtsepi16_epi8(a); + #[rustfmt::skip] + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cvtsepi16_epi8() { + let src = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1); + let a = _mm_set1_epi16(i16::MAX); + let r = _mm_mask_cvtsepi16_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtsepi16_epi8(src, 0b11111111, a); + #[rustfmt::skip] + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_cvtsepi16_epi8() { + let a = _mm_set1_epi16(i16::MAX); + let r = _mm_maskz_cvtsepi16_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtsepi16_epi8(0b11111111, a); + #[rustfmt::skip] + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_cvtsepi16_epi8() { + let a = _mm512_set1_epi16(i16::MAX); + let r = _mm512_maskz_cvtsepi16_epi8(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm512_maskz_cvtsepi16_epi8(0b11111111_11111111_11111111_11111111, a); + let e = _mm256_set1_epi8(i8::MAX); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cvtusepi16_epi8() { + let a = _mm512_set1_epi16(i16::MIN); + let r = _mm512_cvtusepi16_epi8(a); + let e = _mm256_set1_epi8(-1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cvtusepi16_epi8() { + let src = _mm256_set1_epi8(1); + let a = _mm512_set1_epi16(i16::MIN); + let r = _mm512_mask_cvtusepi16_epi8(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm512_mask_cvtusepi16_epi8(src, 0b11111111_11111111_11111111_11111111, a); + let e = _mm256_set1_epi8(-1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_cvtusepi16_epi8() { + let a = _mm512_set1_epi16(i16::MIN); + let r = _mm512_maskz_cvtusepi16_epi8(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm512_maskz_cvtusepi16_epi8(0b11111111_11111111_11111111_11111111, a); + let e = _mm256_set1_epi8(-1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_cvtusepi16_epi8() { + let a = _mm256_set1_epi16(i16::MIN); + let r = _mm256_cvtusepi16_epi8(a); + let e = _mm_set1_epi8(-1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cvtusepi16_epi8() { + let src = _mm_set1_epi8(1); + let a = _mm256_set1_epi16(i16::MIN); + let r = _mm256_mask_cvtusepi16_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvtusepi16_epi8(src, 0b11111111_11111111, a); + let e = _mm_set1_epi8(-1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_cvtusepi16_epi8() { + let a = _mm256_set1_epi16(i16::MIN); + let r = _mm256_maskz_cvtusepi16_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvtusepi16_epi8(0b11111111_11111111, a); + let e = _mm_set1_epi8(-1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_cvtusepi16_epi8() { + let a = _mm_set1_epi16(i16::MIN); + let r = _mm_cvtusepi16_epi8(a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cvtusepi16_epi8() { + let src = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1); + let a = _mm_set1_epi16(i16::MIN); + let r = _mm_mask_cvtusepi16_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtusepi16_epi8(src, 0b11111111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_cvtusepi16_epi8() { + let a = _mm_set1_epi16(i16::MIN); + let r = _mm_maskz_cvtusepi16_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtusepi16_epi8(0b11111111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cvtepi8_epi16() { + let a = _mm256_set1_epi8(2); + let r = _mm512_cvtepi8_epi16(a); + let e = _mm512_set1_epi16(2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cvtepi8_epi16() { + let src = _mm512_set1_epi16(1); + let a = _mm256_set1_epi8(2); + let r = _mm512_mask_cvtepi8_epi16(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_cvtepi8_epi16(src, 0b11111111_11111111_11111111_11111111, a); + let e = _mm512_set1_epi16(2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_cvtepi8_epi16() { + let a = _mm256_set1_epi8(2); + let r = _mm512_maskz_cvtepi8_epi16(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_cvtepi8_epi16(0b11111111_11111111_11111111_11111111, a); + let e = _mm512_set1_epi16(2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cvtepi8_epi16() { + let src = _mm256_set1_epi16(1); + let a = _mm_set1_epi8(2); + let r = _mm256_mask_cvtepi8_epi16(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_cvtepi8_epi16(src, 0b11111111_11111111, a); + let e = _mm256_set1_epi16(2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_cvtepi8_epi16() { + let a = _mm_set1_epi8(2); + let r = _mm256_maskz_cvtepi8_epi16(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_cvtepi8_epi16(0b11111111_11111111, a); + let e = _mm256_set1_epi16(2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cvtepi8_epi16() { + let src = _mm_set1_epi16(1); + let a = _mm_set1_epi8(2); + let r = _mm_mask_cvtepi8_epi16(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtepi8_epi16(src, 0b11111111, a); + let e = _mm_set1_epi16(2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_cvtepi8_epi16() { + let a = _mm_set1_epi8(2); + let r = _mm_maskz_cvtepi8_epi16(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtepi8_epi16(0b11111111, a); + let e = _mm_set1_epi16(2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_cvtepu8_epi16() { + let a = _mm256_set1_epi8(2); + let r = _mm512_cvtepu8_epi16(a); + let e = _mm512_set1_epi16(2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cvtepu8_epi16() { + let src = _mm512_set1_epi16(1); + let a = _mm256_set1_epi8(2); + let r = _mm512_mask_cvtepu8_epi16(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_cvtepu8_epi16(src, 0b11111111_11111111_11111111_11111111, a); + let e = _mm512_set1_epi16(2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_cvtepu8_epi16() { + let a = _mm256_set1_epi8(2); + let r = _mm512_maskz_cvtepu8_epi16(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_cvtepu8_epi16(0b11111111_11111111_11111111_11111111, a); + let e = _mm512_set1_epi16(2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cvtepu8_epi16() { + let src = _mm256_set1_epi16(1); + let a = _mm_set1_epi8(2); + let r = _mm256_mask_cvtepu8_epi16(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_cvtepu8_epi16(src, 0b11111111_11111111, a); + let e = _mm256_set1_epi16(2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_cvtepu8_epi16() { + let a = _mm_set1_epi8(2); + let r = _mm256_maskz_cvtepu8_epi16(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_cvtepu8_epi16(0b11111111_11111111, a); + let e = _mm256_set1_epi16(2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cvtepu8_epi16() { + let src = _mm_set1_epi16(1); + let a = _mm_set1_epi8(2); + let r = _mm_mask_cvtepu8_epi16(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtepu8_epi16(src, 0b11111111, a); + let e = _mm_set1_epi16(2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_cvtepu8_epi16() { + let a = _mm_set1_epi8(2); + let r = _mm_maskz_cvtepu8_epi16(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtepu8_epi16(0b11111111, a); + let e = _mm_set1_epi16(2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_bslli_epi128() { + #[rustfmt::skip] + let a = _mm512_set_epi8( + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + ); + let r = _mm512_bslli_epi128::<9>(a); + #[rustfmt::skip] + let e = _mm512_set_epi8( + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_bsrli_epi128() { + #[rustfmt::skip] + let a = _mm512_set_epi8( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + ); + let r = _mm512_bsrli_epi128::<3>(a); + #[rustfmt::skip] + let e = _mm512_set_epi8( + 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, + 0, 0, 0, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 0, 0, 0, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 0, 0, 0, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_alignr_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8( + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + ); + let b = _mm512_set1_epi8(1); + let r = _mm512_alignr_epi8::<14>(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi8( + 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, + 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, + 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, + 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_alignr_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8( + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + ); + let b = _mm512_set1_epi8(1); + let r = _mm512_mask_alignr_epi8::<14>(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_alignr_epi8::<14>( + a, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8( + 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, + 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, + 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, + 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_maskz_alignr_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8( + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + ); + let b = _mm512_set1_epi8(1); + let r = _mm512_maskz_alignr_epi8::<14>(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_alignr_epi8::<14>( + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8( + 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, + 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, + 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, + 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_alignr_epi8() { + #[rustfmt::skip] + let a = _mm256_set_epi8( + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + ); + let b = _mm256_set1_epi8(1); + let r = _mm256_mask_alignr_epi8::<14>(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_alignr_epi8::<14>(a, 0b11111111_11111111_11111111_11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8( + 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, + 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_alignr_epi8() { + #[rustfmt::skip] + let a = _mm256_set_epi8( + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + ); + let b = _mm256_set1_epi8(1); + let r = _mm256_maskz_alignr_epi8::<14>(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_alignr_epi8::<14>(0b11111111_11111111_11111111_11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_epi8( + 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, + 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_alignr_epi8() { + let a = _mm_set_epi8(1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0); + let b = _mm_set1_epi8(1); + let r = _mm_mask_alignr_epi8::<14>(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_alignr_epi8::<14>(a, 0b11111111_11111111, a, b); + let e = _mm_set_epi8(0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_maskz_alignr_epi8() { + let a = _mm_set_epi8(1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0); + let b = _mm_set1_epi8(1); + let r = _mm_maskz_alignr_epi8::<14>(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_alignr_epi8::<14>(0b11111111_11111111, a, b); + let e = _mm_set_epi8(0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cvtsepi16_storeu_epi8() { + let a = _mm512_set1_epi16(i16::MAX); + let mut r = _mm256_undefined_si256(); + _mm512_mask_cvtsepi16_storeu_epi8( + &mut r as *mut _ as *mut i8, + 0b11111111_11111111_11111111_11111111, + a, + ); + let e = _mm256_set1_epi8(i8::MAX); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cvtsepi16_storeu_epi8() { + let a = _mm256_set1_epi16(i16::MAX); + let mut r = _mm_undefined_si128(); + _mm256_mask_cvtsepi16_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111_11111111, a); + let e = _mm_set1_epi8(i8::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cvtsepi16_storeu_epi8() { + let a = _mm_set1_epi16(i16::MAX); + let mut r = _mm_set1_epi8(0); + _mm_mask_cvtsepi16_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + #[rustfmt::skip] + let e = _mm_set_epi8( + 0, 0, 0, 0, 0, 0, 0, 0, + i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cvtepi16_storeu_epi8() { + let a = _mm512_set1_epi16(8); + let mut r = _mm256_undefined_si256(); + _mm512_mask_cvtepi16_storeu_epi8( + &mut r as *mut _ as *mut i8, + 0b11111111_11111111_11111111_11111111, + a, + ); + let e = _mm256_set1_epi8(8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cvtepi16_storeu_epi8() { + let a = _mm256_set1_epi16(8); + let mut r = _mm_undefined_si128(); + _mm256_mask_cvtepi16_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111_11111111, a); + let e = _mm_set1_epi8(8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cvtepi16_storeu_epi8() { + let a = _mm_set1_epi16(8); + let mut r = _mm_set1_epi8(0); + _mm_mask_cvtepi16_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8, 8, 8, 8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw")] + unsafe fn test_mm512_mask_cvtusepi16_storeu_epi8() { + let a = _mm512_set1_epi16(i16::MAX); + let mut r = _mm256_undefined_si256(); + _mm512_mask_cvtusepi16_storeu_epi8( + &mut r as *mut _ as *mut i8, + 0b11111111_11111111_11111111_11111111, + a, + ); + let e = _mm256_set1_epi8(u8::MAX as i8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm256_mask_cvtusepi16_storeu_epi8() { + let a = _mm256_set1_epi16(i16::MAX); + let mut r = _mm_undefined_si128(); + _mm256_mask_cvtusepi16_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111_11111111, a); + let e = _mm_set1_epi8(u8::MAX as i8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512bw,avx512vl")] + unsafe fn test_mm_mask_cvtusepi16_storeu_epi8() { + let a = _mm_set1_epi16(i16::MAX); + let mut r = _mm_set1_epi8(0); + _mm_mask_cvtusepi16_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + #[rustfmt::skip] + let e = _mm_set_epi8( + 0, 0, 0, 0, + 0, 0, 0, 0, + u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, + u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, + ); + assert_eq_m128i(r, e); + } +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512cd.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512cd.rs new file mode 100644 index 000000000..ac9d3aed3 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512cd.rs @@ -0,0 +1,1170 @@ +use crate::{ + core_arch::{simd::*, simd_llvm::*, x86::*}, + mem::transmute, +}; + +#[cfg(test)] +use stdarch_test::assert_instr; + +/// Broadcast the low 16-bits from input mask k to all 32-bit elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_broadcastmw_epi32&expand=553) +#[inline] +#[target_feature(enable = "avx512cd")] +#[cfg_attr(test, assert_instr(vpbroadcast))] // should be vpbroadcastmw2d +pub unsafe fn _mm512_broadcastmw_epi32(k: __mmask16) -> __m512i { + _mm512_set1_epi32(k as i32) +} + +/// Broadcast the low 16-bits from input mask k to all 32-bit elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_broadcastmw_epi32&expand=552) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcast))] // should be vpbroadcastmw2d +pub unsafe fn _mm256_broadcastmw_epi32(k: __mmask16) -> __m256i { + _mm256_set1_epi32(k as i32) +} + +/// Broadcast the low 16-bits from input mask k to all 32-bit elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_broadcastmw_epi32&expand=551) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcast))] // should be vpbroadcastmw2d +pub unsafe fn _mm_broadcastmw_epi32(k: __mmask16) -> __m128i { + _mm_set1_epi32(k as i32) +} + +/// Broadcast the low 8-bits from input mask k to all 64-bit elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_broadcastmb_epi64&expand=550) +#[inline] +#[target_feature(enable = "avx512cd")] +#[cfg_attr(test, assert_instr(vpbroadcast))] // should be vpbroadcastmb2q +pub unsafe fn _mm512_broadcastmb_epi64(k: __mmask8) -> __m512i { + _mm512_set1_epi64(k as i64) +} + +/// Broadcast the low 8-bits from input mask k to all 64-bit elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_broadcastmb_epi64&expand=549) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcast))] // should be vpbroadcastmb2q +pub unsafe fn _mm256_broadcastmb_epi64(k: __mmask8) -> __m256i { + _mm256_set1_epi64x(k as i64) +} + +/// Broadcast the low 8-bits from input mask k to all 64-bit elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_broadcastmb_epi64&expand=548) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcast))] // should be vpbroadcastmb2q +pub unsafe fn _mm_broadcastmb_epi64(k: __mmask8) -> __m128i { + _mm_set1_epi64x(k as i64) +} + +/// Test each 32-bit element of a for equality with all other elements in a closer to the least significant bit. Each element's comparison forms a zero extended bit vector in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_conflict_epi32&expand=1248) +#[inline] +#[target_feature(enable = "avx512cd")] +#[cfg_attr(test, assert_instr(vpconflictd))] +pub unsafe fn _mm512_conflict_epi32(a: __m512i) -> __m512i { + transmute(vpconflictd(a.as_i32x16())) +} + +/// Test each 32-bit element of a for equality with all other elements in a closer to the least significant bit using writemask k (elements are copied from src when the corresponding mask bit is not set). Each element's comparison forms a zero extended bit vector in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_conflict_epi32&expand=1249) +#[inline] +#[target_feature(enable = "avx512cd")] +#[cfg_attr(test, assert_instr(vpconflictd))] +pub unsafe fn _mm512_mask_conflict_epi32(src: __m512i, k: __mmask16, a: __m512i) -> __m512i { + let conflict = _mm512_conflict_epi32(a).as_i32x16(); + transmute(simd_select_bitmask(k, conflict, src.as_i32x16())) +} + +/// Test each 32-bit element of a for equality with all other elements in a closer to the least significant bit using zeromask k (elements are zeroed out when the corresponding mask bit is not set). Each element's comparison forms a zero extended bit vector in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_conflict_epi32&expand=1250) +#[inline] +#[target_feature(enable = "avx512cd")] +#[cfg_attr(test, assert_instr(vpconflictd))] +pub unsafe fn _mm512_maskz_conflict_epi32(k: __mmask16, a: __m512i) -> __m512i { + let conflict = _mm512_conflict_epi32(a).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, conflict, zero)) +} + +/// Test each 32-bit element of a for equality with all other elements in a closer to the least significant bit. Each element's comparison forms a zero extended bit vector in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_conflict_epi32&expand=1245) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vpconflictd))] +pub unsafe fn _mm256_conflict_epi32(a: __m256i) -> __m256i { + transmute(vpconflictd256(a.as_i32x8())) +} + +/// Test each 32-bit element of a for equality with all other elements in a closer to the least significant bit using writemask k (elements are copied from src when the corresponding mask bit is not set). Each element's comparison forms a zero extended bit vector in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_conflict_epi32&expand=1246) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vpconflictd))] +pub unsafe fn _mm256_mask_conflict_epi32(src: __m256i, k: __mmask8, a: __m256i) -> __m256i { + let conflict = _mm256_conflict_epi32(a).as_i32x8(); + transmute(simd_select_bitmask(k, conflict, src.as_i32x8())) +} + +/// Test each 32-bit element of a for equality with all other elements in a closer to the least significant bit using zeromask k (elements are zeroed out when the corresponding mask bit is not set). Each element's comparison forms a zero extended bit vector in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_conflict_epi32&expand=1247) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vpconflictd))] +pub unsafe fn _mm256_maskz_conflict_epi32(k: __mmask8, a: __m256i) -> __m256i { + let conflict = _mm256_conflict_epi32(a).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, conflict, zero)) +} + +/// Test each 32-bit element of a for equality with all other elements in a closer to the least significant bit. Each element's comparison forms a zero extended bit vector in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_conflict_epi32&expand=1242) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vpconflictd))] +pub unsafe fn _mm_conflict_epi32(a: __m128i) -> __m128i { + transmute(vpconflictd128(a.as_i32x4())) +} + +/// Test each 32-bit element of a for equality with all other elements in a closer to the least significant bit using writemask k (elements are copied from src when the corresponding mask bit is not set). Each element's comparison forms a zero extended bit vector in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_conflict_epi32&expand=1243) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vpconflictd))] +pub unsafe fn _mm_mask_conflict_epi32(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let conflict = _mm_conflict_epi32(a).as_i32x4(); + transmute(simd_select_bitmask(k, conflict, src.as_i32x4())) +} + +/// Test each 32-bit element of a for equality with all other elements in a closer to the least significant bit using zeromask k (elements are zeroed out when the corresponding mask bit is not set). Each element's comparison forms a zero extended bit vector in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_conflict_epi32&expand=1244) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vpconflictd))] +pub unsafe fn _mm_maskz_conflict_epi32(k: __mmask8, a: __m128i) -> __m128i { + let conflict = _mm_conflict_epi32(a).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, conflict, zero)) +} + +/// Test each 64-bit element of a for equality with all other elements in a closer to the least significant bit. Each element's comparison forms a zero extended bit vector in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_conflict_epi64&expand=1257) +#[inline] +#[target_feature(enable = "avx512cd")] +#[cfg_attr(test, assert_instr(vpconflictq))] +pub unsafe fn _mm512_conflict_epi64(a: __m512i) -> __m512i { + transmute(vpconflictq(a.as_i64x8())) +} + +/// Test each 64-bit element of a for equality with all other elements in a closer to the least significant bit using writemask k (elements are copied from src when the corresponding mask bit is not set). Each element's comparison forms a zero extended bit vector in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_conflict_epi64&expand=1258) +#[inline] +#[target_feature(enable = "avx512cd")] +#[cfg_attr(test, assert_instr(vpconflictq))] +pub unsafe fn _mm512_mask_conflict_epi64(src: __m512i, k: __mmask8, a: __m512i) -> __m512i { + let conflict = _mm512_conflict_epi64(a).as_i64x8(); + transmute(simd_select_bitmask(k, conflict, src.as_i64x8())) +} + +/// Test each 64-bit element of a for equality with all other elements in a closer to the least significant bit using zeromask k (elements are zeroed out when the corresponding mask bit is not set). Each element's comparison forms a zero extended bit vector in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_conflict_epi64&expand=1259) +#[inline] +#[target_feature(enable = "avx512cd")] +#[cfg_attr(test, assert_instr(vpconflictq))] +pub unsafe fn _mm512_maskz_conflict_epi64(k: __mmask8, a: __m512i) -> __m512i { + let conflict = _mm512_conflict_epi64(a).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, conflict, zero)) +} + +/// Test each 64-bit element of a for equality with all other elements in a closer to the least significant bit. Each element's comparison forms a zero extended bit vector in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_conflict_epi64&expand=1254) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vpconflictq))] +pub unsafe fn _mm256_conflict_epi64(a: __m256i) -> __m256i { + transmute(vpconflictq256(a.as_i64x4())) +} + +/// Test each 64-bit element of a for equality with all other elements in a closer to the least significant bit using writemask k (elements are copied from src when the corresponding mask bit is not set). Each element's comparison forms a zero extended bit vector in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_conflict_epi64&expand=1255) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vpconflictq))] +pub unsafe fn _mm256_mask_conflict_epi64(src: __m256i, k: __mmask8, a: __m256i) -> __m256i { + let conflict = _mm256_conflict_epi64(a).as_i64x4(); + transmute(simd_select_bitmask(k, conflict, src.as_i64x4())) +} + +/// Test each 64-bit element of a for equality with all other elements in a closer to the least significant bit using zeromask k (elements are zeroed out when the corresponding mask bit is not set). Each element's comparison forms a zero extended bit vector in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_conflict_epi64&expand=1256) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vpconflictq))] +pub unsafe fn _mm256_maskz_conflict_epi64(k: __mmask8, a: __m256i) -> __m256i { + let conflict = _mm256_conflict_epi64(a).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, conflict, zero)) +} + +/// Test each 64-bit element of a for equality with all other elements in a closer to the least significant bit. Each element's comparison forms a zero extended bit vector in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_conflict_epi64&expand=1251) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vpconflictq))] +pub unsafe fn _mm_conflict_epi64(a: __m128i) -> __m128i { + transmute(vpconflictq128(a.as_i64x2())) +} + +/// Test each 64-bit element of a for equality with all other elements in a closer to the least significant bit using writemask k (elements are copied from src when the corresponding mask bit is not set). Each element's comparison forms a zero extended bit vector in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_conflict_epi64&expand=1252) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vpconflictq))] +pub unsafe fn _mm_mask_conflict_epi64(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let conflict = _mm_conflict_epi64(a).as_i64x2(); + transmute(simd_select_bitmask(k, conflict, src.as_i64x2())) +} + +/// Test each 64-bit element of a for equality with all other elements in a closer to the least significant bit using zeromask k (elements are zeroed out when the corresponding mask bit is not set). Each element's comparison forms a zero extended bit vector in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_conflict_epi64&expand=1253) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vpconflictq))] +pub unsafe fn _mm_maskz_conflict_epi64(k: __mmask8, a: __m128i) -> __m128i { + let conflict = _mm_conflict_epi64(a).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, conflict, zero)) +} + +/// Counts the number of leading zero bits in each packed 32-bit integer in a, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_lzcnt_epi32&expand=3491) +#[inline] +#[target_feature(enable = "avx512cd")] +#[cfg_attr(test, assert_instr(vplzcntd))] +pub unsafe fn _mm512_lzcnt_epi32(a: __m512i) -> __m512i { + transmute(vplzcntd(a.as_i32x16(), false)) +} + +/// Counts the number of leading zero bits in each packed 32-bit integer in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_lzcnt_epi32&expand=3492) +#[inline] +#[target_feature(enable = "avx512cd")] +#[cfg_attr(test, assert_instr(vplzcntd))] +pub unsafe fn _mm512_mask_lzcnt_epi32(src: __m512i, k: __mmask16, a: __m512i) -> __m512i { + let zerocount = _mm512_lzcnt_epi32(a).as_i32x16(); + transmute(simd_select_bitmask(k, zerocount, src.as_i32x16())) +} + +/// Counts the number of leading zero bits in each packed 32-bit integer in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_lzcnt_epi32&expand=3493) +#[inline] +#[target_feature(enable = "avx512cd")] +#[cfg_attr(test, assert_instr(vplzcntd))] +pub unsafe fn _mm512_maskz_lzcnt_epi32(k: __mmask16, a: __m512i) -> __m512i { + let zerocount = _mm512_lzcnt_epi32(a).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, zerocount, zero)) +} + +/// Counts the number of leading zero bits in each packed 32-bit integer in a, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_lzcnt_epi32&expand=3488) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vplzcntd))] +pub unsafe fn _mm256_lzcnt_epi32(a: __m256i) -> __m256i { + transmute(vplzcntd256(a.as_i32x8(), false)) +} + +/// Counts the number of leading zero bits in each packed 32-bit integer in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_lzcnt_epi32&expand=3489) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vplzcntd))] +pub unsafe fn _mm256_mask_lzcnt_epi32(src: __m256i, k: __mmask8, a: __m256i) -> __m256i { + let zerocount = _mm256_lzcnt_epi32(a).as_i32x8(); + transmute(simd_select_bitmask(k, zerocount, src.as_i32x8())) +} + +/// Counts the number of leading zero bits in each packed 32-bit integer in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_lzcnt_epi32&expand=3490) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vplzcntd))] +pub unsafe fn _mm256_maskz_lzcnt_epi32(k: __mmask8, a: __m256i) -> __m256i { + let zerocount = _mm256_lzcnt_epi32(a).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, zerocount, zero)) +} + +/// Counts the number of leading zero bits in each packed 32-bit integer in a, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_lzcnt_epi32&expand=3485) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vplzcntd))] +pub unsafe fn _mm_lzcnt_epi32(a: __m128i) -> __m128i { + transmute(vplzcntd128(a.as_i32x4(), false)) +} + +/// Counts the number of leading zero bits in each packed 32-bit integer in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_lzcnt_epi32&expand=3486) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vplzcntd))] +pub unsafe fn _mm_mask_lzcnt_epi32(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let zerocount = _mm_lzcnt_epi32(a).as_i32x4(); + transmute(simd_select_bitmask(k, zerocount, src.as_i32x4())) +} + +/// Counts the number of leading zero bits in each packed 32-bit integer in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_lzcnt_epi32&expand=3487) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vplzcntd))] +pub unsafe fn _mm_maskz_lzcnt_epi32(k: __mmask8, a: __m128i) -> __m128i { + let zerocount = _mm_lzcnt_epi32(a).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, zerocount, zero)) +} + +/// Counts the number of leading zero bits in each packed 64-bit integer in a, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_lzcnt_epi64&expand=3500) +#[inline] +#[target_feature(enable = "avx512cd")] +#[cfg_attr(test, assert_instr(vplzcntq))] +pub unsafe fn _mm512_lzcnt_epi64(a: __m512i) -> __m512i { + transmute(vplzcntq(a.as_i64x8(), false)) +} + +/// Counts the number of leading zero bits in each packed 64-bit integer in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_lzcnt_epi64&expand=3501) +#[inline] +#[target_feature(enable = "avx512cd")] +#[cfg_attr(test, assert_instr(vplzcntq))] +pub unsafe fn _mm512_mask_lzcnt_epi64(src: __m512i, k: __mmask8, a: __m512i) -> __m512i { + let zerocount = _mm512_lzcnt_epi64(a).as_i64x8(); + transmute(simd_select_bitmask(k, zerocount, src.as_i64x8())) +} + +/// Counts the number of leading zero bits in each packed 64-bit integer in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_lzcnt_epi64&expand=3502) +#[inline] +#[target_feature(enable = "avx512cd")] +#[cfg_attr(test, assert_instr(vplzcntq))] +pub unsafe fn _mm512_maskz_lzcnt_epi64(k: __mmask8, a: __m512i) -> __m512i { + let zerocount = _mm512_lzcnt_epi64(a).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, zerocount, zero)) +} + +/// Counts the number of leading zero bits in each packed 64-bit integer in a, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_lzcnt_epi64&expand=3497) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vplzcntq))] +pub unsafe fn _mm256_lzcnt_epi64(a: __m256i) -> __m256i { + transmute(vplzcntq256(a.as_i64x4(), false)) +} + +/// Counts the number of leading zero bits in each packed 64-bit integer in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_lzcnt_epi64&expand=3498) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vplzcntq))] +pub unsafe fn _mm256_mask_lzcnt_epi64(src: __m256i, k: __mmask8, a: __m256i) -> __m256i { + let zerocount = _mm256_lzcnt_epi64(a).as_i64x4(); + transmute(simd_select_bitmask(k, zerocount, src.as_i64x4())) +} + +/// Counts the number of leading zero bits in each packed 64-bit integer in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_lzcnt_epi64&expand=3499) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vplzcntq))] +pub unsafe fn _mm256_maskz_lzcnt_epi64(k: __mmask8, a: __m256i) -> __m256i { + let zerocount = _mm256_lzcnt_epi64(a).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, zerocount, zero)) +} + +/// Counts the number of leading zero bits in each packed 64-bit integer in a, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_lzcnt_epi64&expand=3494) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vplzcntq))] +pub unsafe fn _mm_lzcnt_epi64(a: __m128i) -> __m128i { + transmute(vplzcntq128(a.as_i64x2(), false)) +} + +/// Counts the number of leading zero bits in each packed 64-bit integer in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_lzcnt_epi64&expand=3495) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vplzcntq))] +pub unsafe fn _mm_mask_lzcnt_epi64(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let zerocount = _mm_lzcnt_epi64(a).as_i64x2(); + transmute(simd_select_bitmask(k, zerocount, src.as_i64x2())) +} + +/// Counts the number of leading zero bits in each packed 64-bit integer in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_lzcnt_epi64&expand=3496) +#[inline] +#[target_feature(enable = "avx512cd,avx512vl")] +#[cfg_attr(test, assert_instr(vplzcntq))] +pub unsafe fn _mm_maskz_lzcnt_epi64(k: __mmask8, a: __m128i) -> __m128i { + let zerocount = _mm_lzcnt_epi64(a).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, zerocount, zero)) +} + +#[allow(improper_ctypes)] +extern "C" { + #[link_name = "llvm.x86.avx512.conflict.d.512"] + fn vpconflictd(a: i32x16) -> i32x16; + #[link_name = "llvm.x86.avx512.conflict.d.256"] + fn vpconflictd256(a: i32x8) -> i32x8; + #[link_name = "llvm.x86.avx512.conflict.d.128"] + fn vpconflictd128(a: i32x4) -> i32x4; + + #[link_name = "llvm.x86.avx512.conflict.q.512"] + fn vpconflictq(a: i64x8) -> i64x8; + #[link_name = "llvm.x86.avx512.conflict.q.256"] + fn vpconflictq256(a: i64x4) -> i64x4; + #[link_name = "llvm.x86.avx512.conflict.q.128"] + fn vpconflictq128(a: i64x2) -> i64x2; + + #[link_name = "llvm.ctlz.v16i32"] + fn vplzcntd(a: i32x16, nonzero: bool) -> i32x16; + #[link_name = "llvm.ctlz.v8i32"] + fn vplzcntd256(a: i32x8, nonzero: bool) -> i32x8; + #[link_name = "llvm.ctlz.v4i32"] + fn vplzcntd128(a: i32x4, nonzero: bool) -> i32x4; + + #[link_name = "llvm.ctlz.v8i64"] + fn vplzcntq(a: i64x8, nonzero: bool) -> i64x8; + #[link_name = "llvm.ctlz.v4i64"] + fn vplzcntq256(a: i64x4, nonzero: bool) -> i64x4; + #[link_name = "llvm.ctlz.v2i64"] + fn vplzcntq128(a: i64x2, nonzero: bool) -> i64x2; +} + +#[cfg(test)] +mod tests { + + use crate::core_arch::x86::*; + use stdarch_test::simd_test; + + #[simd_test(enable = "avx512cd")] + unsafe fn test_mm512_broadcastmw_epi32() { + let a: __mmask16 = 2; + let r = _mm512_broadcastmw_epi32(a); + let e = _mm512_set1_epi32(2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm256_broadcastmw_epi32() { + let a: __mmask16 = 2; + let r = _mm256_broadcastmw_epi32(a); + let e = _mm256_set1_epi32(2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm_broadcastmw_epi32() { + let a: __mmask16 = 2; + let r = _mm_broadcastmw_epi32(a); + let e = _mm_set1_epi32(2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512cd")] + unsafe fn test_mm512_broadcastmb_epi64() { + let a: __mmask8 = 2; + let r = _mm512_broadcastmb_epi64(a); + let e = _mm512_set1_epi64(2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm256_broadcastmb_epi64() { + let a: __mmask8 = 2; + let r = _mm256_broadcastmb_epi64(a); + let e = _mm256_set1_epi64x(2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm_broadcastmb_epi64() { + let a: __mmask8 = 2; + let r = _mm_broadcastmb_epi64(a); + let e = _mm_set1_epi64x(2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512cd")] + unsafe fn test_mm512_conflict_epi32() { + let a = _mm512_set1_epi32(1); + let r = _mm512_conflict_epi32(a); + let e = _mm512_set_epi32( + 1 << 14 + | 1 << 13 + | 1 << 12 + | 1 << 11 + | 1 << 10 + | 1 << 9 + | 1 << 8 + | 1 << 7 + | 1 << 6 + | 1 << 5 + | 1 << 4 + | 1 << 3 + | 1 << 2 + | 1 << 1 + | 1 << 0, + 1 << 13 + | 1 << 12 + | 1 << 11 + | 1 << 10 + | 1 << 9 + | 1 << 8 + | 1 << 7 + | 1 << 6 + | 1 << 5 + | 1 << 4 + | 1 << 3 + | 1 << 2 + | 1 << 1 + | 1 << 0, + 1 << 12 + | 1 << 11 + | 1 << 10 + | 1 << 9 + | 1 << 8 + | 1 << 7 + | 1 << 6 + | 1 << 5 + | 1 << 4 + | 1 << 3 + | 1 << 2 + | 1 << 1 + | 1 << 0, + 1 << 11 + | 1 << 10 + | 1 << 9 + | 1 << 8 + | 1 << 7 + | 1 << 6 + | 1 << 5 + | 1 << 4 + | 1 << 3 + | 1 << 2 + | 1 << 1 + | 1 << 0, + 1 << 10 + | 1 << 9 + | 1 << 8 + | 1 << 7 + | 1 << 6 + | 1 << 5 + | 1 << 4 + | 1 << 3 + | 1 << 2 + | 1 << 1 + | 1 << 0, + 1 << 9 | 1 << 8 | 1 << 7 | 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 8 | 1 << 7 | 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 7 | 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 2 | 1 << 1 | 1 << 0, + 1 << 1 | 1 << 0, + 1 << 0, + 0, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512cd")] + unsafe fn test_mm512_mask_conflict_epi32() { + let a = _mm512_set1_epi32(1); + let r = _mm512_mask_conflict_epi32(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_conflict_epi32(a, 0b11111111_11111111, a); + let e = _mm512_set_epi32( + 1 << 14 + | 1 << 13 + | 1 << 12 + | 1 << 11 + | 1 << 10 + | 1 << 9 + | 1 << 8 + | 1 << 7 + | 1 << 6 + | 1 << 5 + | 1 << 4 + | 1 << 3 + | 1 << 2 + | 1 << 1 + | 1 << 0, + 1 << 13 + | 1 << 12 + | 1 << 11 + | 1 << 10 + | 1 << 9 + | 1 << 8 + | 1 << 7 + | 1 << 6 + | 1 << 5 + | 1 << 4 + | 1 << 3 + | 1 << 2 + | 1 << 1 + | 1 << 0, + 1 << 12 + | 1 << 11 + | 1 << 10 + | 1 << 9 + | 1 << 8 + | 1 << 7 + | 1 << 6 + | 1 << 5 + | 1 << 4 + | 1 << 3 + | 1 << 2 + | 1 << 1 + | 1 << 0, + 1 << 11 + | 1 << 10 + | 1 << 9 + | 1 << 8 + | 1 << 7 + | 1 << 6 + | 1 << 5 + | 1 << 4 + | 1 << 3 + | 1 << 2 + | 1 << 1 + | 1 << 0, + 1 << 10 + | 1 << 9 + | 1 << 8 + | 1 << 7 + | 1 << 6 + | 1 << 5 + | 1 << 4 + | 1 << 3 + | 1 << 2 + | 1 << 1 + | 1 << 0, + 1 << 9 | 1 << 8 | 1 << 7 | 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 8 | 1 << 7 | 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 7 | 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 2 | 1 << 1 | 1 << 0, + 1 << 1 | 1 << 0, + 1 << 0, + 0, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512cd")] + unsafe fn test_mm512_maskz_conflict_epi32() { + let a = _mm512_set1_epi32(1); + let r = _mm512_maskz_conflict_epi32(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_conflict_epi32(0b11111111_11111111, a); + let e = _mm512_set_epi32( + 1 << 14 + | 1 << 13 + | 1 << 12 + | 1 << 11 + | 1 << 10 + | 1 << 9 + | 1 << 8 + | 1 << 7 + | 1 << 6 + | 1 << 5 + | 1 << 4 + | 1 << 3 + | 1 << 2 + | 1 << 1 + | 1 << 0, + 1 << 13 + | 1 << 12 + | 1 << 11 + | 1 << 10 + | 1 << 9 + | 1 << 8 + | 1 << 7 + | 1 << 6 + | 1 << 5 + | 1 << 4 + | 1 << 3 + | 1 << 2 + | 1 << 1 + | 1 << 0, + 1 << 12 + | 1 << 11 + | 1 << 10 + | 1 << 9 + | 1 << 8 + | 1 << 7 + | 1 << 6 + | 1 << 5 + | 1 << 4 + | 1 << 3 + | 1 << 2 + | 1 << 1 + | 1 << 0, + 1 << 11 + | 1 << 10 + | 1 << 9 + | 1 << 8 + | 1 << 7 + | 1 << 6 + | 1 << 5 + | 1 << 4 + | 1 << 3 + | 1 << 2 + | 1 << 1 + | 1 << 0, + 1 << 10 + | 1 << 9 + | 1 << 8 + | 1 << 7 + | 1 << 6 + | 1 << 5 + | 1 << 4 + | 1 << 3 + | 1 << 2 + | 1 << 1 + | 1 << 0, + 1 << 9 | 1 << 8 | 1 << 7 | 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 8 | 1 << 7 | 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 7 | 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 2 | 1 << 1 | 1 << 0, + 1 << 1 | 1 << 0, + 1 << 0, + 0, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm256_conflict_epi32() { + let a = _mm256_set1_epi32(1); + let r = _mm256_conflict_epi32(a); + let e = _mm256_set_epi32( + 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 2 | 1 << 1 | 1 << 0, + 1 << 1 | 1 << 0, + 1 << 0, + 0, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm256_mask_conflict_epi32() { + let a = _mm256_set1_epi32(1); + let r = _mm256_mask_conflict_epi32(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_conflict_epi32(a, 0b11111111, a); + let e = _mm256_set_epi32( + 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 2 | 1 << 1 | 1 << 0, + 1 << 1 | 1 << 0, + 1 << 0, + 0, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm256_maskz_conflict_epi32() { + let a = _mm256_set1_epi32(1); + let r = _mm256_maskz_conflict_epi32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_conflict_epi32(0b11111111, a); + let e = _mm256_set_epi32( + 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 2 | 1 << 1 | 1 << 0, + 1 << 1 | 1 << 0, + 1 << 0, + 0, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm_conflict_epi32() { + let a = _mm_set1_epi32(1); + let r = _mm_conflict_epi32(a); + let e = _mm_set_epi32(1 << 2 | 1 << 1 | 1 << 0, 1 << 1 | 1 << 0, 1 << 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm_mask_conflict_epi32() { + let a = _mm_set1_epi32(1); + let r = _mm_mask_conflict_epi32(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_conflict_epi32(a, 0b00001111, a); + let e = _mm_set_epi32(1 << 2 | 1 << 1 | 1 << 0, 1 << 1 | 1 << 0, 1 << 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm_maskz_conflict_epi32() { + let a = _mm_set1_epi32(1); + let r = _mm_maskz_conflict_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_conflict_epi32(0b00001111, a); + let e = _mm_set_epi32(1 << 2 | 1 << 1 | 1 << 0, 1 << 1 | 1 << 0, 1 << 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512cd")] + unsafe fn test_mm512_conflict_epi64() { + let a = _mm512_set1_epi64(1); + let r = _mm512_conflict_epi64(a); + let e = _mm512_set_epi64( + 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 2 | 1 << 1 | 1 << 0, + 1 << 1 | 1 << 0, + 1 << 0, + 0, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512cd")] + unsafe fn test_mm512_mask_conflict_epi64() { + let a = _mm512_set1_epi64(1); + let r = _mm512_mask_conflict_epi64(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_conflict_epi64(a, 0b11111111, a); + let e = _mm512_set_epi64( + 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 2 | 1 << 1 | 1 << 0, + 1 << 1 | 1 << 0, + 1 << 0, + 0, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512cd")] + unsafe fn test_mm512_maskz_conflict_epi64() { + let a = _mm512_set1_epi64(1); + let r = _mm512_maskz_conflict_epi64(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_conflict_epi64(0b11111111, a); + let e = _mm512_set_epi64( + 1 << 6 | 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 5 | 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 4 | 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 3 | 1 << 2 | 1 << 1 | 1 << 0, + 1 << 2 | 1 << 1 | 1 << 0, + 1 << 1 | 1 << 0, + 1 << 0, + 0, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm256_conflict_epi64() { + let a = _mm256_set1_epi64x(1); + let r = _mm256_conflict_epi64(a); + let e = _mm256_set_epi64x(1 << 2 | 1 << 1 | 1 << 0, 1 << 1 | 1 << 0, 1 << 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm256_mask_conflict_epi64() { + let a = _mm256_set1_epi64x(1); + let r = _mm256_mask_conflict_epi64(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_conflict_epi64(a, 0b00001111, a); + let e = _mm256_set_epi64x(1 << 2 | 1 << 1 | 1 << 0, 1 << 1 | 1 << 0, 1 << 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm256_maskz_conflict_epi64() { + let a = _mm256_set1_epi64x(1); + let r = _mm256_maskz_conflict_epi64(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_conflict_epi64(0b00001111, a); + let e = _mm256_set_epi64x(1 << 2 | 1 << 1 | 1 << 0, 1 << 1 | 1 << 0, 1 << 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm_conflict_epi64() { + let a = _mm_set1_epi64x(1); + let r = _mm_conflict_epi64(a); + let e = _mm_set_epi64x(1 << 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm_mask_conflict_epi64() { + let a = _mm_set1_epi64x(1); + let r = _mm_mask_conflict_epi64(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_conflict_epi64(a, 0b00000011, a); + let e = _mm_set_epi64x(1 << 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm_maskz_conflict_epi64() { + let a = _mm_set1_epi64x(1); + let r = _mm_maskz_conflict_epi64(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_conflict_epi64(0b00000011, a); + let e = _mm_set_epi64x(1 << 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512cd")] + unsafe fn test_mm512_lzcnt_epi32() { + let a = _mm512_set1_epi32(1); + let r = _mm512_lzcnt_epi32(a); + let e = _mm512_set1_epi32(31); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512cd")] + unsafe fn test_mm512_mask_lzcnt_epi32() { + let a = _mm512_set1_epi32(1); + let r = _mm512_mask_lzcnt_epi32(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_lzcnt_epi32(a, 0b11111111_11111111, a); + let e = _mm512_set1_epi32(31); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512cd")] + unsafe fn test_mm512_maskz_lzcnt_epi32() { + let a = _mm512_set1_epi32(2); + let r = _mm512_maskz_lzcnt_epi32(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_lzcnt_epi32(0b11111111_11111111, a); + let e = _mm512_set1_epi32(30); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm256_lzcnt_epi32() { + let a = _mm256_set1_epi32(1); + let r = _mm256_lzcnt_epi32(a); + let e = _mm256_set1_epi32(31); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm256_mask_lzcnt_epi32() { + let a = _mm256_set1_epi32(1); + let r = _mm256_mask_lzcnt_epi32(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_lzcnt_epi32(a, 0b11111111, a); + let e = _mm256_set1_epi32(31); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm256_maskz_lzcnt_epi32() { + let a = _mm256_set1_epi32(1); + let r = _mm256_maskz_lzcnt_epi32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_lzcnt_epi32(0b11111111, a); + let e = _mm256_set1_epi32(31); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm_lzcnt_epi32() { + let a = _mm_set1_epi32(1); + let r = _mm_lzcnt_epi32(a); + let e = _mm_set1_epi32(31); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm_mask_lzcnt_epi32() { + let a = _mm_set1_epi32(1); + let r = _mm_mask_lzcnt_epi32(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_lzcnt_epi32(a, 0b00001111, a); + let e = _mm_set1_epi32(31); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm_maskz_lzcnt_epi32() { + let a = _mm_set1_epi32(1); + let r = _mm_maskz_lzcnt_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_lzcnt_epi32(0b00001111, a); + let e = _mm_set1_epi32(31); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512cd")] + unsafe fn test_mm512_lzcnt_epi64() { + let a = _mm512_set1_epi64(1); + let r = _mm512_lzcnt_epi64(a); + let e = _mm512_set1_epi64(63); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512cd")] + unsafe fn test_mm512_mask_lzcnt_epi64() { + let a = _mm512_set1_epi64(1); + let r = _mm512_mask_lzcnt_epi64(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_lzcnt_epi64(a, 0b11111111, a); + let e = _mm512_set1_epi64(63); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512cd")] + unsafe fn test_mm512_maskz_lzcnt_epi64() { + let a = _mm512_set1_epi64(2); + let r = _mm512_maskz_lzcnt_epi64(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_lzcnt_epi64(0b11111111, a); + let e = _mm512_set1_epi64(62); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm256_lzcnt_epi64() { + let a = _mm256_set1_epi64x(1); + let r = _mm256_lzcnt_epi64(a); + let e = _mm256_set1_epi64x(63); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm256_mask_lzcnt_epi64() { + let a = _mm256_set1_epi64x(1); + let r = _mm256_mask_lzcnt_epi64(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_lzcnt_epi64(a, 0b00001111, a); + let e = _mm256_set1_epi64x(63); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm256_maskz_lzcnt_epi64() { + let a = _mm256_set1_epi64x(1); + let r = _mm256_maskz_lzcnt_epi64(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_lzcnt_epi64(0b00001111, a); + let e = _mm256_set1_epi64x(63); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm_lzcnt_epi64() { + let a = _mm_set1_epi64x(1); + let r = _mm_lzcnt_epi64(a); + let e = _mm_set1_epi64x(63); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm_mask_lzcnt_epi64() { + let a = _mm_set1_epi64x(1); + let r = _mm_mask_lzcnt_epi64(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_lzcnt_epi64(a, 0b00001111, a); + let e = _mm_set1_epi64x(63); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512cd,avx512vl")] + unsafe fn test_mm_maskz_lzcnt_epi64() { + let a = _mm_set1_epi64x(1); + let r = _mm_maskz_lzcnt_epi64(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_lzcnt_epi64(0b00001111, a); + let e = _mm_set1_epi64x(63); + assert_eq_m128i(r, e); + } +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512f.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512f.rs index f38583102..f70a28466 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512f.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512f.rs @@ -1,14 +1,47 @@ use crate::{ + arch::asm, core_arch::{simd::*, simd_llvm::*, x86::*}, mem::{self, transmute}, + ptr, }; +// x86-32 wants to use a 32-bit address size, but asm! defaults to using the full +// register name (e.g. rax). We have to explicitly override the placeholder to +// use the 32-bit register name in that case. + +#[cfg(target_pointer_width = "32")] +macro_rules! vpl { + ($inst:expr) => { + concat!($inst, ", [{p:e}]") + }; +} +#[cfg(target_pointer_width = "64")] +macro_rules! vpl { + ($inst:expr) => { + concat!($inst, ", [{p}]") + }; +} +#[cfg(target_pointer_width = "32")] +macro_rules! vps { + ($inst1:expr, $inst2:expr) => { + concat!($inst1, " [{p:e}]", $inst2) + }; +} +#[cfg(target_pointer_width = "64")] +macro_rules! vps { + ($inst1:expr, $inst2:expr) => { + concat!($inst1, " [{p}]", $inst2) + }; +} + +pub(crate) use {vpl, vps}; + #[cfg(test)] use stdarch_test::assert_instr; /// Computes the absolute values of packed 32-bit integers in `a`. /// -/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#avx512techs=AVX512F&expand=33,34,4990,33&text=_mm512_abs_epi32) +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_abs_epi32&expand=39) #[inline] #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vpabsd))] @@ -25,7 +58,7 @@ pub unsafe fn _mm512_abs_epi32(a: __m512i) -> __m512i { /// unsigned results in `dst` using writemask `k` (elements are copied from /// `src` when the corresponding mask bit is not set). /// -/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#avx512techs=AVX512F&expand=33,34,4990,33&text=_mm512_abs_epi32) +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_abs_epi32&expand=40) #[inline] #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vpabsd))] @@ -38,7 +71,7 @@ pub unsafe fn _mm512_mask_abs_epi32(src: __m512i, k: __mmask16, a: __m512i) -> _ /// unsigned results in `dst` using zeromask `k` (elements are zeroed out when /// the corresponding mask bit is not set). /// -/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#avx512techs=AVX512F&expand=33,34,4990,33,34,35,35&text=_mm512_maskz_abs_epi32) +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_abs_epi32&expand=41) #[inline] #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vpabsd))] @@ -48,153 +81,55750 @@ pub unsafe fn _mm512_maskz_abs_epi32(k: __mmask16, a: __m512i) -> __m512i { transmute(simd_select_bitmask(k, abs, zero)) } -/// Returns vector of type `__m512i` with all elements set to zero. +/// Compute the absolute value of packed signed 32-bit integers in a, and store the unsigned results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_abs_epi32&expand=37) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpabsd))] +pub unsafe fn _mm256_mask_abs_epi32(src: __m256i, k: __mmask8, a: __m256i) -> __m256i { + let abs = _mm256_abs_epi32(a).as_i32x8(); + transmute(simd_select_bitmask(k, abs, src.as_i32x8())) +} + +/// Compute the absolute value of packed signed 32-bit integers in a, and store the unsigned results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_abs_epi32&expand=38) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpabsd))] +pub unsafe fn _mm256_maskz_abs_epi32(k: __mmask8, a: __m256i) -> __m256i { + let abs = _mm256_abs_epi32(a).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, abs, zero)) +} + +/// Compute the absolute value of packed signed 32-bit integers in a, and store the unsigned results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_abs_epi32&expand=34) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpabsd))] +pub unsafe fn _mm_mask_abs_epi32(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let abs = _mm_abs_epi32(a).as_i32x4(); + transmute(simd_select_bitmask(k, abs, src.as_i32x4())) +} + +/// Compute the absolute value of packed signed 32-bit integers in a, and store the unsigned results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). /// -/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#avx512techs=AVX512F&expand=33,34,4990&text=_mm512_setzero_si512) +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_abs_epi32&expand=35) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpabsd))] +pub unsafe fn _mm_maskz_abs_epi32(k: __mmask8, a: __m128i) -> __m128i { + let abs = _mm_abs_epi32(a).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, abs, zero)) +} + +/// Compute the absolute value of packed signed 64-bit integers in a, and store the unsigned results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_abs_epi64&expand=48) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vxorps))] -pub unsafe fn _mm512_setzero_si512() -> __m512i { - // All-0 is a properly initialized __m512i - mem::zeroed() +#[cfg_attr(test, assert_instr(vpabsq))] +pub unsafe fn _mm512_abs_epi64(a: __m512i) -> __m512i { + let a = a.as_i64x8(); + // all-0 is a properly initialized i64x8 + let zero: i64x8 = mem::zeroed(); + let sub = simd_sub(zero, a); + let cmp: i64x8 = simd_gt(a, zero); + transmute(simd_select(cmp, a, sub)) } -/// Sets packed 32-bit integers in `dst` with the supplied values in reverse -/// order. +/// Compute the absolute value of packed signed 64-bit integers in a, and store the unsigned results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_abs_epi64&expand=49) #[inline] #[target_feature(enable = "avx512f")] -pub unsafe fn _mm512_setr_epi32( - e15: i32, - e14: i32, - e13: i32, - e12: i32, - e11: i32, - e10: i32, - e9: i32, - e8: i32, - e7: i32, - e6: i32, - e5: i32, - e4: i32, - e3: i32, - e2: i32, - e1: i32, - e0: i32, -) -> __m512i { - let r = i32x16( - e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0, - ); - transmute(r) +#[cfg_attr(test, assert_instr(vpabsq))] +pub unsafe fn _mm512_mask_abs_epi64(src: __m512i, k: __mmask8, a: __m512i) -> __m512i { + let abs = _mm512_abs_epi64(a).as_i64x8(); + transmute(simd_select_bitmask(k, abs, src.as_i64x8())) } -/// Broadcast 64-bit integer `a` to all elements of `dst`. +/// Compute the absolute value of packed signed 64-bit integers in a, and store the unsigned results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_abs_epi64&expand=50) #[inline] #[target_feature(enable = "avx512f")] -pub unsafe fn _mm512_set1_epi64(a: i64) -> __m512i { - transmute(i64x8::splat(a)) +#[cfg_attr(test, assert_instr(vpabsq))] +pub unsafe fn _mm512_maskz_abs_epi64(k: __mmask8, a: __m512i) -> __m512i { + let abs = _mm512_abs_epi64(a).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, abs, zero)) } -#[cfg(test)] -mod tests { - use std; - use stdarch_test::simd_test; +/// Compute the absolute value of packed signed 64-bit integers in a, and store the unsigned results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_abs_epi64&expand=45) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpabsq))] +pub unsafe fn _mm256_abs_epi64(a: __m256i) -> __m256i { + let a = a.as_i64x4(); + // all-0 is a properly initialized i64x4 + let zero: i64x4 = mem::zeroed(); + let sub = simd_sub(zero, a); + let cmp: i64x4 = simd_gt(a, zero); + transmute(simd_select(cmp, a, sub)) +} - use crate::core_arch::x86::*; +/// Compute the absolute value of packed signed 64-bit integers in a, and store the unsigned results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_abs_epi64&expand=46) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpabsq))] +pub unsafe fn _mm256_mask_abs_epi64(src: __m256i, k: __mmask8, a: __m256i) -> __m256i { + let abs = _mm256_abs_epi64(a).as_i64x4(); + transmute(simd_select_bitmask(k, abs, src.as_i64x4())) +} - #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_abs_epi32() { - #[rustfmt::skip] - let a = _mm512_setr_epi32( - 0, 1, -1, std::i32::MAX, - std::i32::MIN, 100, -100, -32, - 0, 1, -1, std::i32::MAX, - std::i32::MIN, 100, -100, -32, - ); - let r = _mm512_abs_epi32(a); - let e = _mm512_setr_epi32( - 0, - 1, - 1, - std::i32::MAX, - std::i32::MAX.wrapping_add(1), - 100, - 100, - 32, - 0, - 1, - 1, - std::i32::MAX, - std::i32::MAX.wrapping_add(1), - 100, - 100, - 32, - ); - assert_eq_m512i(r, e); - } +/// Compute the absolute value of packed signed 64-bit integers in a, and store the unsigned results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_abs_epi64&expand=45) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpabsq))] +pub unsafe fn _mm256_maskz_abs_epi64(k: __mmask8, a: __m256i) -> __m256i { + let abs = _mm256_abs_epi64(a).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, abs, zero)) +} - #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_abs_epi32() { - #[rustfmt::skip] - let a = _mm512_setr_epi32( - 0, 1, -1, std::i32::MAX, - std::i32::MIN, 100, -100, -32, - 0, 1, -1, std::i32::MAX, - std::i32::MIN, 100, -100, -32, - ); - let r = _mm512_mask_abs_epi32(a, 0, a); - assert_eq_m512i(r, a); - let r = _mm512_mask_abs_epi32(a, 0b11111111, a); - let e = _mm512_setr_epi32( - 0, - 1, - 1, - std::i32::MAX, - std::i32::MAX.wrapping_add(1), - 100, - 100, - 32, - 0, - 1, - -1, - std::i32::MAX, - std::i32::MIN, - 100, - -100, - -32, - ); - assert_eq_m512i(r, e); - } +/// Finds the absolute value of each packed single-precision (32-bit) floating-point element in v2, storing the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_abs_ps&expand=65) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpandq))] +pub unsafe fn _mm512_abs_ps(v2: __m512) -> __m512 { + let a = _mm512_set1_epi32(0x7FFFFFFF); // from LLVM code + let b = transmute::(v2.as_f32x16()); + let abs = _mm512_and_epi32(a, b); + transmute(abs) +} - #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_maskz_abs_epi32() { - #[rustfmt::skip] - let a = _mm512_setr_epi32( - 0, 1, -1, std::i32::MAX, - std::i32::MIN, 100, -100, -32, - 0, 1, -1, std::i32::MAX, - std::i32::MIN, 100, -100, -32, - ); - let r = _mm512_maskz_abs_epi32(0, a); - assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_abs_epi32(0b11111111, a); - let e = _mm512_setr_epi32( - 0, - 1, - 1, - std::i32::MAX, - std::i32::MAX.wrapping_add(1), - 100, - 100, - 32, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - ); - assert_eq_m512i(r, e); +/// Finds the absolute value of each packed single-precision (32-bit) floating-point element in v2, storing the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_abs_ps&expand=66) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpandd))] +pub unsafe fn _mm512_mask_abs_ps(src: __m512, k: __mmask16, v2: __m512) -> __m512 { + let abs = _mm512_abs_ps(v2).as_f32x16(); + transmute(simd_select_bitmask(k, abs, src.as_f32x16())) +} + +/// Finds the absolute value of each packed double-precision (64-bit) floating-point element in v2, storing the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_abs_pd&expand=60) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpandq))] +pub unsafe fn _mm512_abs_pd(v2: __m512d) -> __m512d { + let a = _mm512_set1_epi64(0x7FFFFFFFFFFFFFFF); // from LLVM code + let b = transmute::(v2.as_f64x8()); + let abs = _mm512_and_epi64(a, b); + transmute(abs) +} + +/// Finds the absolute value of each packed double-precision (64-bit) floating-point element in v2, storing the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_abs_pd&expand=61) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpandq))] +pub unsafe fn _mm512_mask_abs_pd(src: __m512d, k: __mmask8, v2: __m512d) -> __m512d { + let abs = _mm512_abs_pd(v2).as_f64x8(); + transmute(simd_select_bitmask(k, abs, src.as_f64x8())) +} + +/// Move packed 32-bit integers from a to dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_mov_epi32&expand=3801) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovdqa32))] +pub unsafe fn _mm512_mask_mov_epi32(src: __m512i, k: __mmask16, a: __m512i) -> __m512i { + let mov = a.as_i32x16(); + transmute(simd_select_bitmask(k, mov, src.as_i32x16())) +} + +/// Move packed 32-bit integers from a into dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_mov_epi32&expand=3802) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovdqa32))] +pub unsafe fn _mm512_maskz_mov_epi32(k: __mmask16, a: __m512i) -> __m512i { + let mov = a.as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, mov, zero)) +} + +/// Move packed 32-bit integers from a to dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_mov_epi32&expand=3799) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqa32))] +pub unsafe fn _mm256_mask_mov_epi32(src: __m256i, k: __mmask8, a: __m256i) -> __m256i { + let mov = a.as_i32x8(); + transmute(simd_select_bitmask(k, mov, src.as_i32x8())) +} + +/// Move packed 32-bit integers from a into dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_mov_epi32&expand=3800) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqa32))] +pub unsafe fn _mm256_maskz_mov_epi32(k: __mmask8, a: __m256i) -> __m256i { + let mov = a.as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, mov, zero)) +} + +/// Move packed 32-bit integers from a to dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_mov_epi32&expand=3797) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqa32))] +pub unsafe fn _mm_mask_mov_epi32(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let mov = a.as_i32x4(); + transmute(simd_select_bitmask(k, mov, src.as_i32x4())) +} + +/// Move packed 32-bit integers from a into dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_mov_epi32&expand=3798) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqa32))] +pub unsafe fn _mm_maskz_mov_epi32(k: __mmask8, a: __m128i) -> __m128i { + let mov = a.as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, mov, zero)) +} + +/// Move packed 64-bit integers from a to dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_mov_epi64&expand=3807) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovdqa64))] +pub unsafe fn _mm512_mask_mov_epi64(src: __m512i, k: __mmask8, a: __m512i) -> __m512i { + let mov = a.as_i64x8(); + transmute(simd_select_bitmask(k, mov, src.as_i64x8())) +} + +/// Move packed 64-bit integers from a into dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_mov_epi64&expand=3808) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovdqa64))] +pub unsafe fn _mm512_maskz_mov_epi64(k: __mmask8, a: __m512i) -> __m512i { + let mov = a.as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, mov, zero)) +} + +/// Move packed 64-bit integers from a to dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_mov_epi64&expand=3805) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqa64))] +pub unsafe fn _mm256_mask_mov_epi64(src: __m256i, k: __mmask8, a: __m256i) -> __m256i { + let mov = a.as_i64x4(); + transmute(simd_select_bitmask(k, mov, src.as_i64x4())) +} + +/// Move packed 64-bit integers from a into dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_mov_epi64&expand=3806) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqa64))] +pub unsafe fn _mm256_maskz_mov_epi64(k: __mmask8, a: __m256i) -> __m256i { + let mov = a.as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, mov, zero)) +} + +/// Move packed 64-bit integers from a to dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_mov_epi64&expand=3803) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqa64))] +pub unsafe fn _mm_mask_mov_epi64(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let mov = a.as_i64x2(); + transmute(simd_select_bitmask(k, mov, src.as_i64x2())) +} + +/// Move packed 64-bit integers from a into dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_mov_epi64&expand=3804) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqa64))] +pub unsafe fn _mm_maskz_mov_epi64(k: __mmask8, a: __m128i) -> __m128i { + let mov = a.as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, mov, zero)) +} + +/// Move packed single-precision (32-bit) floating-point elements from a to dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_mov_ps&expand=3825) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovaps))] +pub unsafe fn _mm512_mask_mov_ps(src: __m512, k: __mmask16, a: __m512) -> __m512 { + let mov = a.as_f32x16(); + transmute(simd_select_bitmask(k, mov, src.as_f32x16())) +} + +/// Move packed single-precision (32-bit) floating-point elements from a into dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_mov_ps&expand=3826) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovaps))] +pub unsafe fn _mm512_maskz_mov_ps(k: __mmask16, a: __m512) -> __m512 { + let mov = a.as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, mov, zero)) +} + +/// Move packed single-precision (32-bit) floating-point elements from a to dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_mov_ps&expand=3823) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovaps))] +pub unsafe fn _mm256_mask_mov_ps(src: __m256, k: __mmask8, a: __m256) -> __m256 { + let mov = a.as_f32x8(); + transmute(simd_select_bitmask(k, mov, src.as_f32x8())) +} + +/// Move packed single-precision (32-bit) floating-point elements from a into dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_mov_ps&expand=3824) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovaps))] +pub unsafe fn _mm256_maskz_mov_ps(k: __mmask8, a: __m256) -> __m256 { + let mov = a.as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, mov, zero)) +} + +/// Move packed single-precision (32-bit) floating-point elements from a to dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_mov_ps&expand=3821) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovaps))] +pub unsafe fn _mm_mask_mov_ps(src: __m128, k: __mmask8, a: __m128) -> __m128 { + let mov = a.as_f32x4(); + transmute(simd_select_bitmask(k, mov, src.as_f32x4())) +} + +/// Move packed single-precision (32-bit) floating-point elements from a into dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_mov_ps&expand=3822) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovaps))] +pub unsafe fn _mm_maskz_mov_ps(k: __mmask8, a: __m128) -> __m128 { + let mov = a.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, mov, zero)) +} + +/// Move packed double-precision (64-bit) floating-point elements from a to dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_mov_pd&expand=3819) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovapd))] +pub unsafe fn _mm512_mask_mov_pd(src: __m512d, k: __mmask8, a: __m512d) -> __m512d { + let mov = a.as_f64x8(); + transmute(simd_select_bitmask(k, mov, src.as_f64x8())) +} + +/// Move packed double-precision (64-bit) floating-point elements from a into dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_mov_pd&expand=3820) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovapd))] +pub unsafe fn _mm512_maskz_mov_pd(k: __mmask8, a: __m512d) -> __m512d { + let mov = a.as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, mov, zero)) +} + +/// Move packed double-precision (64-bit) floating-point elements from a to dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_mov_pd&expand=3817) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovapd))] +pub unsafe fn _mm256_mask_mov_pd(src: __m256d, k: __mmask8, a: __m256d) -> __m256d { + let mov = a.as_f64x4(); + transmute(simd_select_bitmask(k, mov, src.as_f64x4())) +} + +/// Move packed double-precision (64-bit) floating-point elements from a into dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_mov_pd&expand=3818) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovapd))] +pub unsafe fn _mm256_maskz_mov_pd(k: __mmask8, a: __m256d) -> __m256d { + let mov = a.as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, mov, zero)) +} + +/// Move packed double-precision (64-bit) floating-point elements from a to dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_mov_pd&expand=3815) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovapd))] +pub unsafe fn _mm_mask_mov_pd(src: __m128d, k: __mmask8, a: __m128d) -> __m128d { + let mov = a.as_f64x2(); + transmute(simd_select_bitmask(k, mov, src.as_f64x2())) +} + +/// Move packed double-precision (64-bit) floating-point elements from a into dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_mov_pd&expand=3816) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovapd))] +pub unsafe fn _mm_maskz_mov_pd(k: __mmask8, a: __m128d) -> __m128d { + let mov = a.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, mov, zero)) +} + +/// Add packed 32-bit integers in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_add_epi32&expand=100) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpaddd))] +pub unsafe fn _mm512_add_epi32(a: __m512i, b: __m512i) -> __m512i { + transmute(simd_add(a.as_i32x16(), b.as_i32x16())) +} + +/// Add packed 32-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_add_epi32&expand=101) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpaddd))] +pub unsafe fn _mm512_mask_add_epi32(src: __m512i, k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let add = _mm512_add_epi32(a, b).as_i32x16(); + transmute(simd_select_bitmask(k, add, src.as_i32x16())) +} + +/// Add packed 32-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_add_epi32&expand=102) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpaddd))] +pub unsafe fn _mm512_maskz_add_epi32(k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let add = _mm512_add_epi32(a, b).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, add, zero)) +} + +/// Add packed 32-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_add_epi32&expand=98) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddd))] +pub unsafe fn _mm256_mask_add_epi32(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let add = _mm256_add_epi32(a, b).as_i32x8(); + transmute(simd_select_bitmask(k, add, src.as_i32x8())) +} + +/// Add packed 32-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_add_epi32&expand=99) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddd))] +pub unsafe fn _mm256_maskz_add_epi32(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let add = _mm256_add_epi32(a, b).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, add, zero)) +} + +/// Add packed 32-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_add_epi32&expand=95) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddd))] +pub unsafe fn _mm_mask_add_epi32(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let add = _mm_add_epi32(a, b).as_i32x4(); + transmute(simd_select_bitmask(k, add, src.as_i32x4())) +} + +/// Add packed 32-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_add_epi32&expand=96) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddd))] +pub unsafe fn _mm_maskz_add_epi32(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let add = _mm_add_epi32(a, b).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, add, zero)) +} + +/// Add packed 64-bit integers in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_add_epi64&expand=109) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpaddq))] +pub unsafe fn _mm512_add_epi64(a: __m512i, b: __m512i) -> __m512i { + transmute(simd_add(a.as_i64x8(), b.as_i64x8())) +} + +/// Add packed 64-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_add_epi64&expand=110) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpaddq))] +pub unsafe fn _mm512_mask_add_epi64(src: __m512i, k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let add = _mm512_add_epi64(a, b).as_i64x8(); + transmute(simd_select_bitmask(k, add, src.as_i64x8())) +} + +/// Add packed 64-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_add_epi64&expand=111) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpaddq))] +pub unsafe fn _mm512_maskz_add_epi64(k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let add = _mm512_add_epi64(a, b).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, add, zero)) +} + +/// Add packed 64-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_add_epi64&expand=107) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddq))] +pub unsafe fn _mm256_mask_add_epi64(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let add = _mm256_add_epi64(a, b).as_i64x4(); + transmute(simd_select_bitmask(k, add, src.as_i64x4())) +} + +/// Add packed 64-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_add_epi64&expand=108) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddq))] +pub unsafe fn _mm256_maskz_add_epi64(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let add = _mm256_add_epi64(a, b).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, add, zero)) +} + +/// Add packed 64-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_add_epi64&expand=104) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddq))] +pub unsafe fn _mm_mask_add_epi64(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let add = _mm_add_epi64(a, b).as_i64x2(); + transmute(simd_select_bitmask(k, add, src.as_i64x2())) +} + +/// Add packed 64-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_add_epi64&expand=105) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpaddq))] +pub unsafe fn _mm_maskz_add_epi64(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let add = _mm_add_epi64(a, b).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, add, zero)) +} + +/// Add packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_add_ps&expand=139) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddps))] +pub unsafe fn _mm512_add_ps(a: __m512, b: __m512) -> __m512 { + transmute(simd_add(a.as_f32x16(), b.as_f32x16())) +} + +/// Add packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_add_ps&expand=140) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddps))] +pub unsafe fn _mm512_mask_add_ps(src: __m512, k: __mmask16, a: __m512, b: __m512) -> __m512 { + let add = _mm512_add_ps(a, b).as_f32x16(); + transmute(simd_select_bitmask(k, add, src.as_f32x16())) +} + +/// Add packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_add_ps&expand=141) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddps))] +pub unsafe fn _mm512_maskz_add_ps(k: __mmask16, a: __m512, b: __m512) -> __m512 { + let add = _mm512_add_ps(a, b).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, add, zero)) +} + +/// Add packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_add_ps&expand=137) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vaddps))] +pub unsafe fn _mm256_mask_add_ps(src: __m256, k: __mmask8, a: __m256, b: __m256) -> __m256 { + let add = _mm256_add_ps(a, b).as_f32x8(); + transmute(simd_select_bitmask(k, add, src.as_f32x8())) +} + +/// Add packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_add_ps&expand=138) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vaddps))] +pub unsafe fn _mm256_maskz_add_ps(k: __mmask8, a: __m256, b: __m256) -> __m256 { + let add = _mm256_add_ps(a, b).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, add, zero)) +} + +/// Add packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_add_ps&expand=134) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vaddps))] +pub unsafe fn _mm_mask_add_ps(src: __m128, k: __mmask8, a: __m128, b: __m128) -> __m128 { + let add = _mm_add_ps(a, b).as_f32x4(); + transmute(simd_select_bitmask(k, add, src.as_f32x4())) +} + +/// Add packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_add_ps&expand=135) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vaddps))] +pub unsafe fn _mm_maskz_add_ps(k: __mmask8, a: __m128, b: __m128) -> __m128 { + let add = _mm_add_ps(a, b).as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, add, zero)) +} + +/// Add packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_add_pd&expand=127) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddpd))] +pub unsafe fn _mm512_add_pd(a: __m512d, b: __m512d) -> __m512d { + transmute(simd_add(a.as_f64x8(), b.as_f64x8())) +} + +/// Add packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_add_pd&expand=128) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddpd))] +pub unsafe fn _mm512_mask_add_pd(src: __m512d, k: __mmask8, a: __m512d, b: __m512d) -> __m512d { + let add = _mm512_add_pd(a, b).as_f64x8(); + transmute(simd_select_bitmask(k, add, src.as_f64x8())) +} + +/// Add packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_add_pd&expand=129) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddpd))] +pub unsafe fn _mm512_maskz_add_pd(k: __mmask8, a: __m512d, b: __m512d) -> __m512d { + let add = _mm512_add_pd(a, b).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, add, zero)) +} + +/// Add packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_add_pd&expand=125) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vaddpd))] +pub unsafe fn _mm256_mask_add_pd(src: __m256d, k: __mmask8, a: __m256d, b: __m256d) -> __m256d { + let add = _mm256_add_pd(a, b).as_f64x4(); + transmute(simd_select_bitmask(k, add, src.as_f64x4())) +} + +/// Add packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_add_pd&expand=126) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vaddpd))] +pub unsafe fn _mm256_maskz_add_pd(k: __mmask8, a: __m256d, b: __m256d) -> __m256d { + let add = _mm256_add_pd(a, b).as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, add, zero)) +} + +/// Add packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_add_pd&expand=122) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vaddpd))] +pub unsafe fn _mm_mask_add_pd(src: __m128d, k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let add = _mm_add_pd(a, b).as_f64x2(); + transmute(simd_select_bitmask(k, add, src.as_f64x2())) +} + +/// Add packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_add_pd&expand=123) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vaddpd))] +pub unsafe fn _mm_maskz_add_pd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let add = _mm_add_pd(a, b).as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, add, zero)) +} + +/// Subtract packed 32-bit integers in b from packed 32-bit integers in a, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sub_epi32&expand=5694) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsubd))] +pub unsafe fn _mm512_sub_epi32(a: __m512i, b: __m512i) -> __m512i { + transmute(simd_sub(a.as_i32x16(), b.as_i32x16())) +} + +/// Subtract packed 32-bit integers in b from packed 32-bit integers in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sub_epi32&expand=5692) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsubd))] +pub unsafe fn _mm512_mask_sub_epi32(src: __m512i, k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let sub = _mm512_sub_epi32(a, b).as_i32x16(); + transmute(simd_select_bitmask(k, sub, src.as_i32x16())) +} + +/// Subtract packed 32-bit integers in b from packed 32-bit integers in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sub_epi32&expand=5693) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsubd))] +pub unsafe fn _mm512_maskz_sub_epi32(k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let sub = _mm512_sub_epi32(a, b).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, sub, zero)) +} + +/// Subtract packed 32-bit integers in b from packed 32-bit integers in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_sub_epi32&expand=5689) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubd))] +pub unsafe fn _mm256_mask_sub_epi32(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let sub = _mm256_sub_epi32(a, b).as_i32x8(); + transmute(simd_select_bitmask(k, sub, src.as_i32x8())) +} + +/// Subtract packed 32-bit integers in b from packed 32-bit integers in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_sub_epi32&expand=5690) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubd))] +pub unsafe fn _mm256_maskz_sub_epi32(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let sub = _mm256_sub_epi32(a, b).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, sub, zero)) +} + +/// Subtract packed 32-bit integers in b from packed 32-bit integers in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_sub_epi32&expand=5686) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubd))] +pub unsafe fn _mm_mask_sub_epi32(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let sub = _mm_sub_epi32(a, b).as_i32x4(); + transmute(simd_select_bitmask(k, sub, src.as_i32x4())) +} + +/// Subtract packed 32-bit integers in b from packed 32-bit integers in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_sub_epi32&expand=5687) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubd))] +pub unsafe fn _mm_maskz_sub_epi32(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let sub = _mm_sub_epi32(a, b).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, sub, zero)) +} + +/// Subtract packed 64-bit integers in b from packed 64-bit integers in a, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sub_epi64&expand=5703) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsubq))] +pub unsafe fn _mm512_sub_epi64(a: __m512i, b: __m512i) -> __m512i { + transmute(simd_sub(a.as_i64x8(), b.as_i64x8())) +} + +/// Subtract packed 64-bit integers in b from packed 64-bit integers in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sub_epi64&expand=5701) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsubq))] +pub unsafe fn _mm512_mask_sub_epi64(src: __m512i, k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let sub = _mm512_sub_epi64(a, b).as_i64x8(); + transmute(simd_select_bitmask(k, sub, src.as_i64x8())) +} + +/// Subtract packed 64-bit integers in b from packed 64-bit integers in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sub_epi64&expand=5702) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsubq))] +pub unsafe fn _mm512_maskz_sub_epi64(k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let sub = _mm512_sub_epi64(a, b).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, sub, zero)) +} + +/// Subtract packed 64-bit integers in b from packed 64-bit integers in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_sub_epi64&expand=5698) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubq))] +pub unsafe fn _mm256_mask_sub_epi64(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let sub = _mm256_sub_epi64(a, b).as_i64x4(); + transmute(simd_select_bitmask(k, sub, src.as_i64x4())) +} + +/// Subtract packed 64-bit integers in b from packed 64-bit integers in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_sub_epi64&expand=5699) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubq))] +pub unsafe fn _mm256_maskz_sub_epi64(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let sub = _mm256_sub_epi64(a, b).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, sub, zero)) +} + +/// Subtract packed 64-bit integers in b from packed 64-bit integers in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_sub_epi64&expand=5695) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubq))] +pub unsafe fn _mm_mask_sub_epi64(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let sub = _mm_sub_epi64(a, b).as_i64x2(); + transmute(simd_select_bitmask(k, sub, src.as_i64x2())) +} + +/// Subtract packed 64-bit integers in b from packed 64-bit integers in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_sub_epi64&expand=5696) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsubq))] +pub unsafe fn _mm_maskz_sub_epi64(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let sub = _mm_sub_epi64(a, b).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, sub, zero)) +} + +/// Subtract packed single-precision (32-bit) floating-point elements in b from packed single-precision (32-bit) floating-point elements in a, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sub_ps&expand=5733) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubps))] +pub unsafe fn _mm512_sub_ps(a: __m512, b: __m512) -> __m512 { + transmute(simd_sub(a.as_f32x16(), b.as_f32x16())) +} + +/// Subtract packed single-precision (32-bit) floating-point elements in b from packed single-precision (32-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sub_ps&expand=5731) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubps))] +pub unsafe fn _mm512_mask_sub_ps(src: __m512, k: __mmask16, a: __m512, b: __m512) -> __m512 { + let sub = _mm512_sub_ps(a, b).as_f32x16(); + transmute(simd_select_bitmask(k, sub, src.as_f32x16())) +} + +/// Subtract packed single-precision (32-bit) floating-point elements in b from packed single-precision (32-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sub_ps&expand=5732) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubps))] +pub unsafe fn _mm512_maskz_sub_ps(k: __mmask16, a: __m512, b: __m512) -> __m512 { + let sub = _mm512_sub_ps(a, b).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, sub, zero)) +} + +/// Subtract packed single-precision (32-bit) floating-point elements in b from packed single-precision (32-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_sub_ps&expand=5728) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vsubps))] +pub unsafe fn _mm256_mask_sub_ps(src: __m256, k: __mmask8, a: __m256, b: __m256) -> __m256 { + let sub = _mm256_sub_ps(a, b).as_f32x8(); + transmute(simd_select_bitmask(k, sub, src.as_f32x8())) +} + +/// Subtract packed single-precision (32-bit) floating-point elements in b from packed single-precision (32-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_sub_ps&expand=5729) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vsubps))] +pub unsafe fn _mm256_maskz_sub_ps(k: __mmask8, a: __m256, b: __m256) -> __m256 { + let sub = _mm256_sub_ps(a, b).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, sub, zero)) +} + +/// Subtract packed single-precision (32-bit) floating-point elements in b from packed single-precision (32-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_sub_ps&expand=5725) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vsubps))] +pub unsafe fn _mm_mask_sub_ps(src: __m128, k: __mmask8, a: __m128, b: __m128) -> __m128 { + let sub = _mm_sub_ps(a, b).as_f32x4(); + transmute(simd_select_bitmask(k, sub, src.as_f32x4())) +} + +/// Subtract packed single-precision (32-bit) floating-point elements in b from packed single-precision (32-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_sub_ps&expand=5726) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vsubps))] +pub unsafe fn _mm_maskz_sub_ps(k: __mmask8, a: __m128, b: __m128) -> __m128 { + let sub = _mm_sub_ps(a, b).as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, sub, zero)) +} + +/// Subtract packed double-precision (64-bit) floating-point elements in b from packed double-precision (64-bit) floating-point elements in a, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sub_pd&expand=5721) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubpd))] +pub unsafe fn _mm512_sub_pd(a: __m512d, b: __m512d) -> __m512d { + transmute(simd_sub(a.as_f64x8(), b.as_f64x8())) +} + +/// Subtract packed double-precision (64-bit) floating-point elements in b from packed double-precision (64-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sub_pd&expand=5719) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubpd))] +pub unsafe fn _mm512_mask_sub_pd(src: __m512d, k: __mmask8, a: __m512d, b: __m512d) -> __m512d { + let sub = _mm512_sub_pd(a, b).as_f64x8(); + transmute(simd_select_bitmask(k, sub, src.as_f64x8())) +} + +/// Subtract packed double-precision (64-bit) floating-point elements in b from packed double-precision (64-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sub_pd&expand=5720) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubpd))] +pub unsafe fn _mm512_maskz_sub_pd(k: __mmask8, a: __m512d, b: __m512d) -> __m512d { + let sub = _mm512_sub_pd(a, b).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, sub, zero)) +} + +/// Subtract packed double-precision (64-bit) floating-point elements in b from packed double-precision (64-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_sub_pd&expand=5716) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vsubpd))] +pub unsafe fn _mm256_mask_sub_pd(src: __m256d, k: __mmask8, a: __m256d, b: __m256d) -> __m256d { + let sub = _mm256_sub_pd(a, b).as_f64x4(); + transmute(simd_select_bitmask(k, sub, src.as_f64x4())) +} + +/// Subtract packed double-precision (64-bit) floating-point elements in b from packed double-precision (64-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_sub_pd&expand=5717) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vsubpd))] +pub unsafe fn _mm256_maskz_sub_pd(k: __mmask8, a: __m256d, b: __m256d) -> __m256d { + let sub = _mm256_sub_pd(a, b).as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, sub, zero)) +} + +/// Subtract packed double-precision (64-bit) floating-point elements in b from packed double-precision (64-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_sub_pd&expand=5713) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vsubpd))] +pub unsafe fn _mm_mask_sub_pd(src: __m128d, k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let sub = _mm_sub_pd(a, b).as_f64x2(); + transmute(simd_select_bitmask(k, sub, src.as_f64x2())) +} + +/// Subtract packed double-precision (64-bit) floating-point elements in b from packed double-precision (64-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_sub_pd&expand=5714) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vsubpd))] +pub unsafe fn _mm_maskz_sub_pd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let sub = _mm_sub_pd(a, b).as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, sub, zero)) +} + +/// Multiply the low signed 32-bit integers from each packed 64-bit element in a and b, and store the signed 64-bit results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mul_epi32&expand=3907) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmuldq))] +pub unsafe fn _mm512_mul_epi32(a: __m512i, b: __m512i) -> __m512i { + transmute(vpmuldq(a.as_i32x16(), b.as_i32x16())) +} + +/// Multiply the low signed 32-bit integers from each packed 64-bit element in a and b, and store the signed 64-bit results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_mul_epi32&expand=3905) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmuldq))] +pub unsafe fn _mm512_mask_mul_epi32(src: __m512i, k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let mul = _mm512_mul_epi32(a, b).as_i64x8(); + transmute(simd_select_bitmask(k, mul, src.as_i64x8())) +} + +/// Multiply the low signed 32-bit integers from each packed 64-bit element in a and b, and store the signed 64-bit results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_mul_epi32&expand=3906) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmuldq))] +pub unsafe fn _mm512_maskz_mul_epi32(k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let mul = _mm512_mul_epi32(a, b).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply the low signed 32-bit integers from each packed 64-bit element in a and b, and store the signed 64-bit results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_mul_epi32&expand=3902) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmuldq))] +pub unsafe fn _mm256_mask_mul_epi32(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let mul = _mm256_mul_epi32(a, b).as_i64x4(); + transmute(simd_select_bitmask(k, mul, src.as_i64x4())) +} + +/// Multiply the low signed 32-bit integers from each packed 64-bit element in a and b, and store the signed 64-bit results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_mul_epi32&expand=3903) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmuldq))] +pub unsafe fn _mm256_maskz_mul_epi32(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let mul = _mm256_mul_epi32(a, b).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply the low signed 32-bit integers from each packed 64-bit element in a and b, and store the signed 64-bit results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_mul_epi32&expand=3899) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmuldq))] +pub unsafe fn _mm_mask_mul_epi32(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let mul = _mm_mul_epi32(a, b).as_i64x2(); + transmute(simd_select_bitmask(k, mul, src.as_i64x2())) +} + +/// Multiply the low signed 32-bit integers from each packed 64-bit element in a and b, and store the signed 64-bit results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_mul_epi32&expand=3900) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmuldq))] +pub unsafe fn _mm_maskz_mul_epi32(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let mul = _mm_mul_epi32(a, b).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply the packed 32-bit integers in a and b, producing intermediate 64-bit integers, and store the low 32 bits of the intermediate integers in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mullo_epi&expand=4005) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmulld))] +pub unsafe fn _mm512_mullo_epi32(a: __m512i, b: __m512i) -> __m512i { + transmute(simd_mul(a.as_i32x16(), b.as_i32x16())) +} + +/// Multiply the packed 32-bit integers in a and b, producing intermediate 64-bit integers, and store the low 32 bits of the intermediate integers in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_mullo_epi32&expand=4003) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmulld))] +pub unsafe fn _mm512_mask_mullo_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + let mul = _mm512_mullo_epi32(a, b).as_i32x16(); + transmute(simd_select_bitmask(k, mul, src.as_i32x16())) +} + +/// Multiply the packed 32-bit integers in a and b, producing intermediate 64-bit integers, and store the low 32 bits of the intermediate integers in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_mullo_epi32&expand=4004) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmulld))] +pub unsafe fn _mm512_maskz_mullo_epi32(k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let mul = _mm512_mullo_epi32(a, b).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply the packed 32-bit integers in a and b, producing intermediate 64-bit integers, and store the low 32 bits of the intermediate integers in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_mullo_epi32&expand=4000) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmulld))] +pub unsafe fn _mm256_mask_mullo_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + let mul = _mm256_mullo_epi32(a, b).as_i32x8(); + transmute(simd_select_bitmask(k, mul, src.as_i32x8())) +} + +/// Multiply the packed 32-bit integers in a and b, producing intermediate 64-bit integers, and store the low 32 bits of the intermediate integers in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_mullo_epi32&expand=4001) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmulld))] +pub unsafe fn _mm256_maskz_mullo_epi32(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let mul = _mm256_mullo_epi32(a, b).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply the packed 32-bit integers in a and b, producing intermediate 64-bit integers, and store the low 32 bits of the intermediate integers in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_mullo_epi32&expand=3997) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmulld))] +pub unsafe fn _mm_mask_mullo_epi32(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let mul = _mm_mullo_epi32(a, b).as_i32x4(); + transmute(simd_select_bitmask(k, mul, src.as_i32x4())) +} + +/// Multiply the packed 32-bit integers in a and b, producing intermediate 64-bit integers, and store the low 32 bits of the intermediate integers in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_mullo_epi32&expand=3998) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmulld))] +pub unsafe fn _mm_maskz_mullo_epi32(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let mul = _mm_mullo_epi32(a, b).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiplies elements in packed 64-bit integer vectors a and b together, storing the lower 64 bits of the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_mullox_epi64&expand=4017) +/// +/// This intrinsic generates a sequence of instructions, which may perform worse than a native instruction. Consider the performance impact of this intrinsic. +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mullox_epi64(a: __m512i, b: __m512i) -> __m512i { + transmute(simd_mul(a.as_i64x8(), b.as_i64x8())) +} + +/// Multiplies elements in packed 64-bit integer vectors a and b together, storing the lower 64 bits of the result in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_mask_mullox&expand=4016) +/// +/// This intrinsic generates a sequence of instructions, which may perform worse than a native instruction. Consider the performance impact of this intrinsic. +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_mullox_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, + b: __m512i, +) -> __m512i { + let mul = _mm512_mullox_epi64(a, b).as_i64x8(); + transmute(simd_select_bitmask(k, mul, src.as_i64x8())) +} + +/// Multiply the low unsigned 32-bit integers from each packed 64-bit element in a and b, and store the unsigned 64-bit results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_mul_epu32&expand=3916) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmuludq))] +pub unsafe fn _mm512_mul_epu32(a: __m512i, b: __m512i) -> __m512i { + transmute(vpmuludq(a.as_u32x16(), b.as_u32x16())) +} + +/// Multiply the low unsigned 32-bit integers from each packed 64-bit element in a and b, and store the unsigned 64-bit results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_mask_mul_epu32&expand=3914) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmuludq))] +pub unsafe fn _mm512_mask_mul_epu32(src: __m512i, k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let mul = _mm512_mul_epu32(a, b).as_u64x8(); + transmute(simd_select_bitmask(k, mul, src.as_u64x8())) +} + +/// Multiply the low unsigned 32-bit integers from each packed 64-bit element in a and b, and store the unsigned 64-bit results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_maskz_mul_epu32&expand=3915) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmuludq))] +pub unsafe fn _mm512_maskz_mul_epu32(k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let mul = _mm512_mul_epu32(a, b).as_u64x8(); + let zero = _mm512_setzero_si512().as_u64x8(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply the low unsigned 32-bit integers from each packed 64-bit element in a and b, and store the unsigned 64-bit results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_mul_epu32&expand=3911) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmuludq))] +pub unsafe fn _mm256_mask_mul_epu32(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let mul = _mm256_mul_epu32(a, b).as_u64x4(); + transmute(simd_select_bitmask(k, mul, src.as_u64x4())) +} + +/// Multiply the low unsigned 32-bit integers from each packed 64-bit element in a and b, and store the unsigned 64-bit results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_mul_epu32&expand=3912) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmuludq))] +pub unsafe fn _mm256_maskz_mul_epu32(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let mul = _mm256_mul_epu32(a, b).as_u64x4(); + let zero = _mm256_setzero_si256().as_u64x4(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply the low unsigned 32-bit integers from each packed 64-bit element in a and b, and store the unsigned 64-bit results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_mul_epu32&expand=3908) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmuludq))] +pub unsafe fn _mm_mask_mul_epu32(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let mul = _mm_mul_epu32(a, b).as_u64x2(); + transmute(simd_select_bitmask(k, mul, src.as_u64x2())) +} + +/// Multiply the low unsigned 32-bit integers from each packed 64-bit element in a and b, and store the unsigned 64-bit results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_mul_epu32&expand=3909) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmuludq))] +pub unsafe fn _mm_maskz_mul_epu32(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let mul = _mm_mul_epu32(a, b).as_u64x2(); + let zero = _mm_setzero_si128().as_u64x2(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mul_ps&expand=3934) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulps))] +pub unsafe fn _mm512_mul_ps(a: __m512, b: __m512) -> __m512 { + transmute(simd_mul(a.as_f32x16(), b.as_f32x16())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_mul_ps&expand=3932) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulps))] +pub unsafe fn _mm512_mask_mul_ps(src: __m512, k: __mmask16, a: __m512, b: __m512) -> __m512 { + let mul = _mm512_mul_ps(a, b).as_f32x16(); + transmute(simd_select_bitmask(k, mul, src.as_f32x16())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_mul_ps&expand=3933) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulps))] +pub unsafe fn _mm512_maskz_mul_ps(k: __mmask16, a: __m512, b: __m512) -> __m512 { + let mul = _mm512_mul_ps(a, b).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_mul_ps&expand=3929) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmulps))] +pub unsafe fn _mm256_mask_mul_ps(src: __m256, k: __mmask8, a: __m256, b: __m256) -> __m256 { + let mul = _mm256_mul_ps(a, b).as_f32x8(); + transmute(simd_select_bitmask(k, mul, src.as_f32x8())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_mul_ps&expand=3930) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmulps))] +pub unsafe fn _mm256_maskz_mul_ps(k: __mmask8, a: __m256, b: __m256) -> __m256 { + let mul = _mm256_mul_ps(a, b).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_mul_ps&expand=3926) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmulps))] +pub unsafe fn _mm_mask_mul_ps(src: __m128, k: __mmask8, a: __m128, b: __m128) -> __m128 { + let mul = _mm_mul_ps(a, b).as_f32x4(); + transmute(simd_select_bitmask(k, mul, src.as_f32x4())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_mul_ps&expand=3927) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmulps))] +pub unsafe fn _mm_maskz_mul_ps(k: __mmask8, a: __m128, b: __m128) -> __m128 { + let mul = _mm_mul_ps(a, b).as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mul_pd&expand=3925) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulpd))] +pub unsafe fn _mm512_mul_pd(a: __m512d, b: __m512d) -> __m512d { + transmute(simd_mul(a.as_f64x8(), b.as_f64x8())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_mul_pd&expand=3923) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulpd))] +pub unsafe fn _mm512_mask_mul_pd(src: __m512d, k: __mmask8, a: __m512d, b: __m512d) -> __m512d { + let mul = _mm512_mul_pd(a, b).as_f64x8(); + transmute(simd_select_bitmask(k, mul, src.as_f64x8())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_mul_pd&expand=3924) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulpd))] +pub unsafe fn _mm512_maskz_mul_pd(k: __mmask8, a: __m512d, b: __m512d) -> __m512d { + let mul = _mm512_mul_pd(a, b).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_mul_pd&expand=3920) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmulpd))] +pub unsafe fn _mm256_mask_mul_pd(src: __m256d, k: __mmask8, a: __m256d, b: __m256d) -> __m256d { + let mul = _mm256_mul_pd(a, b).as_f64x4(); + transmute(simd_select_bitmask(k, mul, src.as_f64x4())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_mul_pd&expand=3921) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmulpd))] +pub unsafe fn _mm256_maskz_mul_pd(k: __mmask8, a: __m256d, b: __m256d) -> __m256d { + let mul = _mm256_mul_pd(a, b).as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_mul_pd&expand=3917) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmulpd))] +pub unsafe fn _mm_mask_mul_pd(src: __m128d, k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let mul = _mm_mul_pd(a, b).as_f64x2(); + transmute(simd_select_bitmask(k, mul, src.as_f64x2())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_mul_pd&expand=3918) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmulpd))] +pub unsafe fn _mm_maskz_mul_pd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let mul = _mm_mul_pd(a, b).as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, mul, zero)) +} + +/// Divide packed single-precision (32-bit) floating-point elements in a by packed elements in b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_div_ps&expand=2162) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivps))] +pub unsafe fn _mm512_div_ps(a: __m512, b: __m512) -> __m512 { + transmute(simd_div(a.as_f32x16(), b.as_f32x16())) +} + +/// Divide packed single-precision (32-bit) floating-point elements in a by packed elements in b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_div_ps&expand=2163) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivps))] +pub unsafe fn _mm512_mask_div_ps(src: __m512, k: __mmask16, a: __m512, b: __m512) -> __m512 { + let div = _mm512_div_ps(a, b).as_f32x16(); + transmute(simd_select_bitmask(k, div, src.as_f32x16())) +} + +/// Divide packed single-precision (32-bit) floating-point elements in a by packed elements in b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_div_ps&expand=2164) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivps))] +pub unsafe fn _mm512_maskz_div_ps(k: __mmask16, a: __m512, b: __m512) -> __m512 { + let div = _mm512_div_ps(a, b).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, div, zero)) +} + +/// Divide packed single-precision (32-bit) floating-point elements in a by packed elements in b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_div_ps&expand=2160) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vdivps))] +pub unsafe fn _mm256_mask_div_ps(src: __m256, k: __mmask8, a: __m256, b: __m256) -> __m256 { + let div = _mm256_div_ps(a, b).as_f32x8(); + transmute(simd_select_bitmask(k, div, src.as_f32x8())) +} + +/// Divide packed single-precision (32-bit) floating-point elements in a by packed elements in b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_div_ps&expand=2161) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vdivps))] +pub unsafe fn _mm256_maskz_div_ps(k: __mmask8, a: __m256, b: __m256) -> __m256 { + let div = _mm256_div_ps(a, b).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, div, zero)) +} + +/// Divide packed single-precision (32-bit) floating-point elements in a by packed elements in b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_div_ps&expand=2157) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vdivps))] +pub unsafe fn _mm_mask_div_ps(src: __m128, k: __mmask8, a: __m128, b: __m128) -> __m128 { + let div = _mm_div_ps(a, b).as_f32x4(); + transmute(simd_select_bitmask(k, div, src.as_f32x4())) +} + +/// Divide packed single-precision (32-bit) floating-point elements in a by packed elements in b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_div_ps&expand=2158) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vdivps))] +pub unsafe fn _mm_maskz_div_ps(k: __mmask8, a: __m128, b: __m128) -> __m128 { + let div = _mm_div_ps(a, b).as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, div, zero)) +} + +/// Divide packed double-precision (64-bit) floating-point elements in a by packed elements in b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_div_pd&expand=2153) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivpd))] +pub unsafe fn _mm512_div_pd(a: __m512d, b: __m512d) -> __m512d { + transmute(simd_div(a.as_f64x8(), b.as_f64x8())) +} + +/// Divide packed double-precision (64-bit) floating-point elements in a by packed elements in b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_div_pd&expand=2154) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivpd))] +pub unsafe fn _mm512_mask_div_pd(src: __m512d, k: __mmask8, a: __m512d, b: __m512d) -> __m512d { + let div = _mm512_div_pd(a, b).as_f64x8(); + transmute(simd_select_bitmask(k, div, src.as_f64x8())) +} + +/// Divide packed double-precision (64-bit) floating-point elements in a by packed elements in b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_div_pd&expand=2155) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivpd))] +pub unsafe fn _mm512_maskz_div_pd(k: __mmask8, a: __m512d, b: __m512d) -> __m512d { + let div = _mm512_div_pd(a, b).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, div, zero)) +} + +/// Divide packed double-precision (64-bit) floating-point elements in a by packed elements in b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_div_pd&expand=2151) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vdivpd))] +pub unsafe fn _mm256_mask_div_pd(src: __m256d, k: __mmask8, a: __m256d, b: __m256d) -> __m256d { + let div = _mm256_div_pd(a, b).as_f64x4(); + transmute(simd_select_bitmask(k, div, src.as_f64x4())) +} + +/// Divide packed double-precision (64-bit) floating-point elements in a by packed elements in b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_div_pd&expand=2152) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vdivpd))] +pub unsafe fn _mm256_maskz_div_pd(k: __mmask8, a: __m256d, b: __m256d) -> __m256d { + let div = _mm256_div_pd(a, b).as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, div, zero)) +} + +/// Divide packed double-precision (64-bit) floating-point elements in a by packed elements in b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_div_pd&expand=2148) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vdivpd))] +pub unsafe fn _mm_mask_div_pd(src: __m128d, k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let div = _mm_div_pd(a, b).as_f64x2(); + transmute(simd_select_bitmask(k, div, src.as_f64x2())) +} + +/// Divide packed double-precision (64-bit) floating-point elements in a by packed elements in b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_div_pd&expand=2149) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vdivpd))] +pub unsafe fn _mm_maskz_div_pd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let div = _mm_div_pd(a, b).as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, div, zero)) +} + +/// Compare packed signed 32-bit integers in a and b, and store packed maximum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_max_epi32&expand=3582) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmaxsd))] +pub unsafe fn _mm512_max_epi32(a: __m512i, b: __m512i) -> __m512i { + transmute(vpmaxsd(a.as_i32x16(), b.as_i32x16())) +} + +/// Compare packed signed 32-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_max_epi32&expand=3580) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmaxsd))] +pub unsafe fn _mm512_mask_max_epi32(src: __m512i, k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let max = _mm512_max_epi32(a, b).as_i32x16(); + transmute(simd_select_bitmask(k, max, src.as_i32x16())) +} + +/// Compare packed signed 32-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_max_epi32&expand=3581) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmaxsd))] +pub unsafe fn _mm512_maskz_max_epi32(k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let max = _mm512_max_epi32(a, b).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed signed 32-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_max_epi32&expand=3577) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxsd))] +pub unsafe fn _mm256_mask_max_epi32(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let max = _mm256_max_epi32(a, b).as_i32x8(); + transmute(simd_select_bitmask(k, max, src.as_i32x8())) +} + +/// Compare packed signed 32-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_max_epi32&expand=3578) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxsd))] +pub unsafe fn _mm256_maskz_max_epi32(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let max = _mm256_max_epi32(a, b).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed signed 32-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_max_epi32&expand=3574) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxsd))] +pub unsafe fn _mm_mask_max_epi32(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let max = _mm_max_epi32(a, b).as_i32x4(); + transmute(simd_select_bitmask(k, max, src.as_i32x4())) +} + +/// Compare packed signed 32-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_max_epi32&expand=3575) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxsd))] +pub unsafe fn _mm_maskz_max_epi32(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let max = _mm_max_epi32(a, b).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed signed 64-bit integers in a and b, and store packed maximum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_max_epi64&expand=3591) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmaxsq))] +pub unsafe fn _mm512_max_epi64(a: __m512i, b: __m512i) -> __m512i { + transmute(vpmaxsq(a.as_i64x8(), b.as_i64x8())) +} + +/// Compare packed signed 64-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_max_epi64&expand=3589) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmaxsq))] +pub unsafe fn _mm512_mask_max_epi64(src: __m512i, k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let max = _mm512_max_epi64(a, b).as_i64x8(); + transmute(simd_select_bitmask(k, max, src.as_i64x8())) +} + +/// Compare packed signed 64-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_max_epi64&expand=3590) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmaxsq))] +pub unsafe fn _mm512_maskz_max_epi64(k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let max = _mm512_max_epi64(a, b).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed signed 64-bit integers in a and b, and store packed maximum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_max_epi64&expand=3588) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxsq))] +pub unsafe fn _mm256_max_epi64(a: __m256i, b: __m256i) -> __m256i { + transmute(vpmaxsq256(a.as_i64x4(), b.as_i64x4())) +} + +/// Compare packed signed 64-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_max_epi64&expand=3586) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxsq))] +pub unsafe fn _mm256_mask_max_epi64(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let max = _mm256_max_epi64(a, b).as_i64x4(); + transmute(simd_select_bitmask(k, max, src.as_i64x4())) +} + +/// Compare packed signed 64-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_max_epi64&expand=3587) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxsq))] +pub unsafe fn _mm256_maskz_max_epi64(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let max = _mm256_max_epi64(a, b).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed signed 64-bit integers in a and b, and store packed maximum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_max_epi64&expand=3585) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxsq))] +pub unsafe fn _mm_max_epi64(a: __m128i, b: __m128i) -> __m128i { + transmute(vpmaxsq128(a.as_i64x2(), b.as_i64x2())) +} + +/// Compare packed signed 64-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_max_epi64&expand=3583) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxsq))] +pub unsafe fn _mm_mask_max_epi64(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let max = _mm_max_epi64(a, b).as_i64x2(); + transmute(simd_select_bitmask(k, max, src.as_i64x2())) +} + +/// Compare packed signed 64-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_max_epi64&expand=3584) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxsq))] +pub unsafe fn _mm_maskz_max_epi64(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let max = _mm_max_epi64(a, b).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed maximum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_max_ps&expand=3655) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxps))] +pub unsafe fn _mm512_max_ps(a: __m512, b: __m512) -> __m512 { + transmute(vmaxps( + a.as_f32x16(), + b.as_f32x16(), + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_max_ps&expand=3653) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxps))] +pub unsafe fn _mm512_mask_max_ps(src: __m512, k: __mmask16, a: __m512, b: __m512) -> __m512 { + let max = _mm512_max_ps(a, b).as_f32x16(); + transmute(simd_select_bitmask(k, max, src.as_f32x16())) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_max_ps&expand=3654) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxps))] +pub unsafe fn _mm512_maskz_max_ps(k: __mmask16, a: __m512, b: __m512) -> __m512 { + let max = _mm512_max_ps(a, b).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_max_ps&expand=3650) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmaxps))] +pub unsafe fn _mm256_mask_max_ps(src: __m256, k: __mmask8, a: __m256, b: __m256) -> __m256 { + let max = _mm256_max_ps(a, b).as_f32x8(); + transmute(simd_select_bitmask(k, max, src.as_f32x8())) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_max_ps&expand=3651) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmaxps))] +pub unsafe fn _mm256_maskz_max_ps(k: __mmask8, a: __m256, b: __m256) -> __m256 { + let max = _mm256_max_ps(a, b).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_max_ps&expand=3647) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmaxps))] +pub unsafe fn _mm_mask_max_ps(src: __m128, k: __mmask8, a: __m128, b: __m128) -> __m128 { + let max = _mm_max_ps(a, b).as_f32x4(); + transmute(simd_select_bitmask(k, max, src.as_f32x4())) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_max_ps&expand=3648) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmaxps))] +pub unsafe fn _mm_maskz_max_ps(k: __mmask8, a: __m128, b: __m128) -> __m128 { + let max = _mm_max_ps(a, b).as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed maximum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_max_pd&expand=3645) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxpd))] +pub unsafe fn _mm512_max_pd(a: __m512d, b: __m512d) -> __m512d { + transmute(vmaxpd(a.as_f64x8(), b.as_f64x8(), _MM_FROUND_CUR_DIRECTION)) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_max_pd&expand=3643) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxpd))] +pub unsafe fn _mm512_mask_max_pd(src: __m512d, k: __mmask8, a: __m512d, b: __m512d) -> __m512d { + let max = _mm512_max_pd(a, b).as_f64x8(); + transmute(simd_select_bitmask(k, max, src.as_f64x8())) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_max_pd&expand=3644) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxpd))] +pub unsafe fn _mm512_maskz_max_pd(k: __mmask8, a: __m512d, b: __m512d) -> __m512d { + let max = _mm512_max_pd(a, b).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_max_pd&expand=3640) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmaxpd))] +pub unsafe fn _mm256_mask_max_pd(src: __m256d, k: __mmask8, a: __m256d, b: __m256d) -> __m256d { + let max = _mm256_max_pd(a, b).as_f64x4(); + transmute(simd_select_bitmask(k, max, src.as_f64x4())) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_max_pd&expand=3641) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmaxpd))] +pub unsafe fn _mm256_maskz_max_pd(k: __mmask8, a: __m256d, b: __m256d) -> __m256d { + let max = _mm256_max_pd(a, b).as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_max_pd&expand=3637) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmaxpd))] +pub unsafe fn _mm_mask_max_pd(src: __m128d, k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let max = _mm_max_pd(a, b).as_f64x2(); + transmute(simd_select_bitmask(k, max, src.as_f64x2())) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_max_pd&expand=3638) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmaxpd))] +pub unsafe fn _mm_maskz_max_pd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let max = _mm_max_pd(a, b).as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed unsigned 32-bit integers in a and b, and store packed maximum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_max_epu32&expand=3618) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmaxud))] +pub unsafe fn _mm512_max_epu32(a: __m512i, b: __m512i) -> __m512i { + transmute(vpmaxud(a.as_u32x16(), b.as_u32x16())) +} + +/// Compare packed unsigned 32-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_max_epu32&expand=3616) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmaxud))] +pub unsafe fn _mm512_mask_max_epu32(src: __m512i, k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let max = _mm512_max_epu32(a, b).as_u32x16(); + transmute(simd_select_bitmask(k, max, src.as_u32x16())) +} + +/// Compare packed unsigned 32-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_max_epu32&expand=3617) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmaxud))] +pub unsafe fn _mm512_maskz_max_epu32(k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let max = _mm512_max_epu32(a, b).as_u32x16(); + let zero = _mm512_setzero_si512().as_u32x16(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed unsigned 32-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_max_epu32&expand=3613) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxud))] +pub unsafe fn _mm256_mask_max_epu32(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let max = _mm256_max_epu32(a, b).as_u32x8(); + transmute(simd_select_bitmask(k, max, src.as_u32x8())) +} + +/// Compare packed unsigned 32-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_max_epu32&expand=3614) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxud))] +pub unsafe fn _mm256_maskz_max_epu32(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let max = _mm256_max_epu32(a, b).as_u32x8(); + let zero = _mm256_setzero_si256().as_u32x8(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed unsigned 32-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_max_epu32&expand=3610) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxud))] +pub unsafe fn _mm_mask_max_epu32(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let max = _mm_max_epu32(a, b).as_u32x4(); + transmute(simd_select_bitmask(k, max, src.as_u32x4())) +} + +/// Compare packed unsigned 32-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_max_epu32&expand=3611) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxud))] +pub unsafe fn _mm_maskz_max_epu32(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let max = _mm_max_epu32(a, b).as_u32x4(); + let zero = _mm_setzero_si128().as_u32x4(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed unsigned 64-bit integers in a and b, and store packed maximum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_max_epu64&expand=3627) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmaxuq))] +pub unsafe fn _mm512_max_epu64(a: __m512i, b: __m512i) -> __m512i { + transmute(vpmaxuq(a.as_u64x8(), b.as_u64x8())) +} + +/// Compare packed unsigned 64-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_max_epu64&expand=3625) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmaxuq))] +pub unsafe fn _mm512_mask_max_epu64(src: __m512i, k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let max = _mm512_max_epu64(a, b).as_u64x8(); + transmute(simd_select_bitmask(k, max, src.as_u64x8())) +} + +/// Compare packed unsigned 64-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_max_epu&expand=3626) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmaxuq))] +pub unsafe fn _mm512_maskz_max_epu64(k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let max = _mm512_max_epu64(a, b).as_u64x8(); + let zero = _mm512_setzero_si512().as_u64x8(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed unsigned 64-bit integers in a and b, and store packed maximum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_max_epu64&expand=3624) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxuq))] +pub unsafe fn _mm256_max_epu64(a: __m256i, b: __m256i) -> __m256i { + transmute(vpmaxuq256(a.as_u64x4(), b.as_u64x4())) +} + +/// Compare packed unsigned 64-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_max_epu64&expand=3622) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxuq))] +pub unsafe fn _mm256_mask_max_epu64(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let max = _mm256_max_epu64(a, b).as_u64x4(); + transmute(simd_select_bitmask(k, max, src.as_u64x4())) +} + +/// Compare packed unsigned 64-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_max_epu64&expand=3623) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxuq))] +pub unsafe fn _mm256_maskz_max_epu64(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let max = _mm256_max_epu64(a, b).as_u64x4(); + let zero = _mm256_setzero_si256().as_u64x4(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed unsigned 64-bit integers in a and b, and store packed maximum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_max_epu64&expand=3621) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxuq))] +pub unsafe fn _mm_max_epu64(a: __m128i, b: __m128i) -> __m128i { + transmute(vpmaxuq128(a.as_u64x2(), b.as_u64x2())) +} + +/// Compare packed unsigned 64-bit integers in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_max_epu64&expand=3619) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxuq))] +pub unsafe fn _mm_mask_max_epu64(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let max = _mm_max_epu64(a, b).as_u64x2(); + transmute(simd_select_bitmask(k, max, src.as_u64x2())) +} + +/// Compare packed unsigned 64-bit integers in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_max_epu64&expand=3620) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmaxuq))] +pub unsafe fn _mm_maskz_max_epu64(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let max = _mm_max_epu64(a, b).as_u64x2(); + let zero = _mm_setzero_si128().as_u64x2(); + transmute(simd_select_bitmask(k, max, zero)) +} + +/// Compare packed signed 32-bit integers in a and b, and store packed minimum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_min_epi32&expand=3696) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpminsd))] +pub unsafe fn _mm512_min_epi32(a: __m512i, b: __m512i) -> __m512i { + transmute(vpminsd(a.as_i32x16(), b.as_i32x16())) +} + +/// Compare packed signed 32-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_min_epi32&expand=3694) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpminsd))] +pub unsafe fn _mm512_mask_min_epi32(src: __m512i, k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let min = _mm512_min_epi32(a, b).as_i32x16(); + transmute(simd_select_bitmask(k, min, src.as_i32x16())) +} + +/// Compare packed signed 32-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_min_epi32&expand=3695) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpminsd))] +pub unsafe fn _mm512_maskz_min_epi32(k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let min = _mm512_min_epi32(a, b).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed signed 32-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_min_epi32&expand=3691) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpminsd))] +pub unsafe fn _mm256_mask_min_epi32(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let min = _mm256_min_epi32(a, b).as_i32x8(); + transmute(simd_select_bitmask(k, min, src.as_i32x8())) +} + +/// Compare packed signed 32-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_min_epi32&expand=3692) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpminsd))] +pub unsafe fn _mm256_maskz_min_epi32(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let min = _mm256_min_epi32(a, b).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed signed 32-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_min_epi32&expand=3688) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpminsd))] +pub unsafe fn _mm_mask_min_epi32(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let min = _mm_min_epi32(a, b).as_i32x4(); + transmute(simd_select_bitmask(k, min, src.as_i32x4())) +} + +/// Compare packed signed 32-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_min_epi32&expand=3689) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpminsd))] +pub unsafe fn _mm_maskz_min_epi32(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let min = _mm_min_epi32(a, b).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed signed 64-bit integers in a and b, and store packed minimum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_min_epi64&expand=3705) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpminsq))] +pub unsafe fn _mm512_min_epi64(a: __m512i, b: __m512i) -> __m512i { + transmute(vpminsq(a.as_i64x8(), b.as_i64x8())) +} + +/// Compare packed signed 64-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_min_epi64&expand=3703) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpminsq))] +pub unsafe fn _mm512_mask_min_epi64(src: __m512i, k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let min = _mm512_min_epi64(a, b).as_i64x8(); + transmute(simd_select_bitmask(k, min, src.as_i64x8())) +} + +/// Compare packed signed 64-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_maskz_min_epi64&expand=3704) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpminsq))] +pub unsafe fn _mm512_maskz_min_epi64(k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let min = _mm512_min_epi64(a, b).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed signed 64-bit integers in a and b, and store packed minimum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_min_epi64&expand=3702) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpminsq))] +pub unsafe fn _mm256_min_epi64(a: __m256i, b: __m256i) -> __m256i { + transmute(vpminsq256(a.as_i64x4(), b.as_i64x4())) +} + +/// Compare packed signed 64-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_min_epi64&expand=3700) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpminsq))] +pub unsafe fn _mm256_mask_min_epi64(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let min = _mm256_min_epi64(a, b).as_i64x4(); + transmute(simd_select_bitmask(k, min, src.as_i64x4())) +} + +/// Compare packed signed 64-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_min_epi64&expand=3701) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpminsq))] +pub unsafe fn _mm256_maskz_min_epi64(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let min = _mm256_min_epi64(a, b).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed minimum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_min_ps&expand=3769) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminps))] +pub unsafe fn _mm512_min_ps(a: __m512, b: __m512) -> __m512 { + transmute(vminps( + a.as_f32x16(), + b.as_f32x16(), + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_min_ps&expand=3767) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminps))] +pub unsafe fn _mm512_mask_min_ps(src: __m512, k: __mmask16, a: __m512, b: __m512) -> __m512 { + let min = _mm512_min_ps(a, b).as_f32x16(); + transmute(simd_select_bitmask(k, min, src.as_f32x16())) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_min_ps&expand=3768) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminps))] +pub unsafe fn _mm512_maskz_min_ps(k: __mmask16, a: __m512, b: __m512) -> __m512 { + let min = _mm512_min_ps(a, b).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_min_ps&expand=3764) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vminps))] +pub unsafe fn _mm256_mask_min_ps(src: __m256, k: __mmask8, a: __m256, b: __m256) -> __m256 { + let min = _mm256_min_ps(a, b).as_f32x8(); + transmute(simd_select_bitmask(k, min, src.as_f32x8())) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_min_ps&expand=3765) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vminps))] +pub unsafe fn _mm256_maskz_min_ps(k: __mmask8, a: __m256, b: __m256) -> __m256 { + let min = _mm256_min_ps(a, b).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_min_ps&expand=3761) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vminps))] +pub unsafe fn _mm_mask_min_ps(src: __m128, k: __mmask8, a: __m128, b: __m128) -> __m128 { + let min = _mm_min_ps(a, b).as_f32x4(); + transmute(simd_select_bitmask(k, min, src.as_f32x4())) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_min_ps&expand=3762) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vminps))] +pub unsafe fn _mm_maskz_min_ps(k: __mmask8, a: __m128, b: __m128) -> __m128 { + let min = _mm_min_ps(a, b).as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed minimum values in dst. + +/// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed minimum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_min_pd&expand=3759) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminpd))] +pub unsafe fn _mm512_min_pd(a: __m512d, b: __m512d) -> __m512d { + transmute(vminpd(a.as_f64x8(), b.as_f64x8(), _MM_FROUND_CUR_DIRECTION)) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_mask_min_pd&expand=3757) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminpd))] +pub unsafe fn _mm512_mask_min_pd(src: __m512d, k: __mmask8, a: __m512d, b: __m512d) -> __m512d { + let min = _mm512_min_pd(a, b).as_f64x8(); + transmute(simd_select_bitmask(k, min, src.as_f64x8())) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_maskz_min_pd&expand=3758) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminpd))] +pub unsafe fn _mm512_maskz_min_pd(k: __mmask8, a: __m512d, b: __m512d) -> __m512d { + let min = _mm512_min_pd(a, b).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_min_pd&expand=3754) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vminpd))] +pub unsafe fn _mm256_mask_min_pd(src: __m256d, k: __mmask8, a: __m256d, b: __m256d) -> __m256d { + let min = _mm256_min_pd(a, b).as_f64x4(); + transmute(simd_select_bitmask(k, min, src.as_f64x4())) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_min_pd&expand=3755) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vminpd))] +pub unsafe fn _mm256_maskz_min_pd(k: __mmask8, a: __m256d, b: __m256d) -> __m256d { + let min = _mm256_min_pd(a, b).as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_min_pd&expand=3751) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vminpd))] +pub unsafe fn _mm_mask_min_pd(src: __m128d, k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let min = _mm_min_pd(a, b).as_f64x2(); + transmute(simd_select_bitmask(k, min, src.as_f64x2())) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_min_pd&expand=3752) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vminpd))] +pub unsafe fn _mm_maskz_min_pd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let min = _mm_min_pd(a, b).as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed unsigned 32-bit integers in a and b, and store packed minimum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_min_epu32&expand=3732) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpminud))] +pub unsafe fn _mm512_min_epu32(a: __m512i, b: __m512i) -> __m512i { + transmute(vpminud(a.as_u32x16(), b.as_u32x16())) +} + +/// Compare packed unsigned 32-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_min_epu32&expand=3730) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpminud))] +pub unsafe fn _mm512_mask_min_epu32(src: __m512i, k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let min = _mm512_min_epu32(a, b).as_u32x16(); + transmute(simd_select_bitmask(k, min, src.as_u32x16())) +} + +/// Compare packed unsigned 32-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_min_epu32&expand=3731) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpminud))] +pub unsafe fn _mm512_maskz_min_epu32(k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let min = _mm512_min_epu32(a, b).as_u32x16(); + let zero = _mm512_setzero_si512().as_u32x16(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed unsigned 32-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_min_epu32&expand=3727) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpminud))] +pub unsafe fn _mm256_mask_min_epu32(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let min = _mm256_min_epu32(a, b).as_u32x8(); + transmute(simd_select_bitmask(k, min, src.as_u32x8())) +} + +/// Compare packed unsigned 32-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_min_epu32&expand=3728) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpminud))] +pub unsafe fn _mm256_maskz_min_epu32(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let min = _mm256_min_epu32(a, b).as_u32x8(); + let zero = _mm256_setzero_si256().as_u32x8(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed unsigned 32-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_min_epu32&expand=3724) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpminud))] +pub unsafe fn _mm_mask_min_epu32(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let min = _mm_min_epu32(a, b).as_u32x4(); + transmute(simd_select_bitmask(k, min, src.as_u32x4())) +} + +/// Compare packed unsigned 32-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_min_epu32&expand=3725) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpminud))] +pub unsafe fn _mm_maskz_min_epu32(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let min = _mm_min_epu32(a, b).as_u32x4(); + let zero = _mm_setzero_si128().as_u32x4(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed unsigned 64-bit integers in a and b, and store packed minimum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_min_epu64&expand=3741) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpminuq))] +pub unsafe fn _mm512_min_epu64(a: __m512i, b: __m512i) -> __m512i { + transmute(vpminuq(a.as_u64x8(), b.as_u64x8())) +} + +/// Compare packed unsigned 64-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_min_epu64&expand=3739) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpminuq))] +pub unsafe fn _mm512_mask_min_epu64(src: __m512i, k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let min = _mm512_min_epu64(a, b).as_u64x8(); + transmute(simd_select_bitmask(k, min, src.as_u64x8())) +} + +/// Compare packed unsigned 64-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_min_epu64&expand=3740) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpminuq))] +pub unsafe fn _mm512_maskz_min_epu64(k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let min = _mm512_min_epu64(a, b).as_u64x8(); + let zero = _mm512_setzero_si512().as_u64x8(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed unsigned 64-bit integers in a and b, and store packed minimum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_min_epu64&expand=3738) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpminuq))] +pub unsafe fn _mm256_min_epu64(a: __m256i, b: __m256i) -> __m256i { + transmute(vpminuq256(a.as_u64x4(), b.as_u64x4())) +} + +/// Compare packed unsigned 64-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_min_epu64&expand=3736) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpminuq))] +pub unsafe fn _mm256_mask_min_epu64(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let min = _mm256_min_epu64(a, b).as_u64x4(); + transmute(simd_select_bitmask(k, min, src.as_u64x4())) +} + +/// Compare packed unsigned 64-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_min_epu64&expand=3737) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpminuq))] +pub unsafe fn _mm256_maskz_min_epu64(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let min = _mm256_min_epu64(a, b).as_u64x4(); + let zero = _mm256_setzero_si256().as_u64x4(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compare packed unsigned 64-bit integers in a and b, and store packed minimum values in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_min_epu64&expand=3735) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpminuq))] +pub unsafe fn _mm_min_epu64(a: __m128i, b: __m128i) -> __m128i { + transmute(vpminuq128(a.as_u64x2(), b.as_u64x2())) +} + +/// Compare packed unsigned 64-bit integers in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_min_epu64&expand=3733) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpminuq))] +pub unsafe fn _mm_mask_min_epu64(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let min = _mm_min_epu64(a, b).as_u64x2(); + transmute(simd_select_bitmask(k, min, src.as_u64x2())) +} + +/// Compare packed unsigned 64-bit integers in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_min_epu64&expand=3734) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpminuq))] +pub unsafe fn _mm_maskz_min_epu64(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let min = _mm_min_epu64(a, b).as_u64x2(); + let zero = _mm_setzero_si128().as_u64x2(); + transmute(simd_select_bitmask(k, min, zero)) +} + +/// Compute the square root of packed single-precision (32-bit) floating-point elements in a, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sqrt_ps&expand=5371) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtps))] +pub unsafe fn _mm512_sqrt_ps(a: __m512) -> __m512 { + transmute(vsqrtps(a.as_f32x16(), _MM_FROUND_CUR_DIRECTION)) +} + +/// Compute the square root of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sqrt_ps&expand=5369) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtps))] +pub unsafe fn _mm512_mask_sqrt_ps(src: __m512, k: __mmask16, a: __m512) -> __m512 { + let sqrt = _mm512_sqrt_ps(a).as_f32x16(); + transmute(simd_select_bitmask(k, sqrt, src.as_f32x16())) +} + +/// Compute the square root of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sqrt_ps&expand=5370) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtps))] +pub unsafe fn _mm512_maskz_sqrt_ps(k: __mmask16, a: __m512) -> __m512 { + let sqrt = _mm512_sqrt_ps(a).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, sqrt, zero)) +} + +/// Compute the square root of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_sqrt_ps&expand=5366) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vsqrtps))] +pub unsafe fn _mm256_mask_sqrt_ps(src: __m256, k: __mmask8, a: __m256) -> __m256 { + let sqrt = _mm256_sqrt_ps(a).as_f32x8(); + transmute(simd_select_bitmask(k, sqrt, src.as_f32x8())) +} + +/// Compute the square root of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_sqrt_ps&expand=5367) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vsqrtps))] +pub unsafe fn _mm256_maskz_sqrt_ps(k: __mmask8, a: __m256) -> __m256 { + let sqrt = _mm256_sqrt_ps(a).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, sqrt, zero)) +} + +/// Compute the square root of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_sqrt_ps&expand=5363) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vsqrtps))] +pub unsafe fn _mm_mask_sqrt_ps(src: __m128, k: __mmask8, a: __m128) -> __m128 { + let sqrt = _mm_sqrt_ps(a).as_f32x4(); + transmute(simd_select_bitmask(k, sqrt, src.as_f32x4())) +} + +/// Compute the square root of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_sqrt_ps&expand=5364) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vsqrtps))] +pub unsafe fn _mm_maskz_sqrt_ps(k: __mmask8, a: __m128) -> __m128 { + let sqrt = _mm_sqrt_ps(a).as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, sqrt, zero)) +} + +/// Compute the square root of packed double-precision (64-bit) floating-point elements in a, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sqrt_pd&expand=5362) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtpd))] +pub unsafe fn _mm512_sqrt_pd(a: __m512d) -> __m512d { + transmute(vsqrtpd(a.as_f64x8(), _MM_FROUND_CUR_DIRECTION)) +} + +/// Compute the square root of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sqrt_pd&expand=5360) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtpd))] +pub unsafe fn _mm512_mask_sqrt_pd(src: __m512d, k: __mmask8, a: __m512d) -> __m512d { + let sqrt = _mm512_sqrt_pd(a).as_f64x8(); + transmute(simd_select_bitmask(k, sqrt, src.as_f64x8())) +} + +/// Compute the square root of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sqrt_pd&expand=5361) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtpd))] +pub unsafe fn _mm512_maskz_sqrt_pd(k: __mmask8, a: __m512d) -> __m512d { + let sqrt = _mm512_sqrt_pd(a).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, sqrt, zero)) +} + +/// Compute the square root of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_sqrt_pd&expand=5357) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vsqrtpd))] +pub unsafe fn _mm256_mask_sqrt_pd(src: __m256d, k: __mmask8, a: __m256d) -> __m256d { + let sqrt = _mm256_sqrt_pd(a).as_f64x4(); + transmute(simd_select_bitmask(k, sqrt, src.as_f64x4())) +} + +/// Compute the square root of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_sqrt_pd&expand=5358) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vsqrtpd))] +pub unsafe fn _mm256_maskz_sqrt_pd(k: __mmask8, a: __m256d) -> __m256d { + let sqrt = _mm256_sqrt_pd(a).as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, sqrt, zero)) +} + +/// Compute the square root of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_sqrt_pd&expand=5354) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vsqrtpd))] +pub unsafe fn _mm_mask_sqrt_pd(src: __m128d, k: __mmask8, a: __m128d) -> __m128d { + let sqrt = _mm_sqrt_pd(a).as_f64x2(); + transmute(simd_select_bitmask(k, sqrt, src.as_f64x2())) +} + +/// Compute the square root of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_sqrt_pd&expand=5355) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vsqrtpd))] +pub unsafe fn _mm_maskz_sqrt_pd(k: __mmask8, a: __m128d) -> __m128d { + let sqrt = _mm_sqrt_pd(a).as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, sqrt, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmadd_ps&expand=2557) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmadd132ps or vfmadd213ps or vfmadd231ps +pub unsafe fn _mm512_fmadd_ps(a: __m512, b: __m512, c: __m512) -> __m512 { + transmute(vfmadd132ps(a.as_f32x16(), b.as_f32x16(), c.as_f32x16())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmadd_ps&expand=2558) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmadd132ps or vfmadd213ps or vfmadd231ps +pub unsafe fn _mm512_mask_fmadd_ps(a: __m512, k: __mmask16, b: __m512, c: __m512) -> __m512 { + let fmadd = _mm512_fmadd_ps(a, b, c).as_f32x16(); + transmute(simd_select_bitmask(k, fmadd, a.as_f32x16())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmadd_ps&expand=2560) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmadd132ps or vfmadd213ps or vfmadd231ps +pub unsafe fn _mm512_maskz_fmadd_ps(k: __mmask16, a: __m512, b: __m512, c: __m512) -> __m512 { + let fmadd = _mm512_fmadd_ps(a, b, c).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, fmadd, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmadd_ps&expand=2559) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmadd132ps or vfmadd213ps or vfmadd231ps +pub unsafe fn _mm512_mask3_fmadd_ps(a: __m512, b: __m512, c: __m512, k: __mmask16) -> __m512 { + let fmadd = _mm512_fmadd_ps(a, b, c).as_f32x16(); + transmute(simd_select_bitmask(k, fmadd, c.as_f32x16())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_fmadd_ps&expand=2554) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmadd132ps or vfmadd213ps or vfmadd231ps +pub unsafe fn _mm256_mask_fmadd_ps(a: __m256, k: __mmask8, b: __m256, c: __m256) -> __m256 { + let fmadd = _mm256_fmadd_ps(a, b, c).as_f32x8(); + transmute(simd_select_bitmask(k, fmadd, a.as_f32x8())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_fmadd_ps&expand=2556) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmadd132ps or vfmadd213ps or vfmadd231ps +pub unsafe fn _mm256_maskz_fmadd_ps(k: __mmask8, a: __m256, b: __m256, c: __m256) -> __m256 { + let fmadd = _mm256_fmadd_ps(a, b, c).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, fmadd, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask3_fmadd_ps&expand=2555) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmadd132ps or vfmadd213ps or vfmadd231ps +pub unsafe fn _mm256_mask3_fmadd_ps(a: __m256, b: __m256, c: __m256, k: __mmask8) -> __m256 { + let fmadd = _mm256_fmadd_ps(a, b, c).as_f32x8(); + transmute(simd_select_bitmask(k, fmadd, c.as_f32x8())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_fmadd_ps&expand=2550) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmadd132ps or vfmadd213ps or vfmadd231ps +pub unsafe fn _mm_mask_fmadd_ps(a: __m128, k: __mmask8, b: __m128, c: __m128) -> __m128 { + let fmadd = _mm_fmadd_ps(a, b, c).as_f32x4(); + transmute(simd_select_bitmask(k, fmadd, a.as_f32x4())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_fmadd_ps&expand=2552) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmadd132ps or vfmadd213ps or vfmadd231ps +pub unsafe fn _mm_maskz_fmadd_ps(k: __mmask8, a: __m128, b: __m128, c: __m128) -> __m128 { + let fmadd = _mm_fmadd_ps(a, b, c).as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, fmadd, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask3_fmadd_ps&expand=2551) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmadd132ps or vfmadd213ps or vfmadd231ps +pub unsafe fn _mm_mask3_fmadd_ps(a: __m128, b: __m128, c: __m128, k: __mmask8) -> __m128 { + let fmadd = _mm_fmadd_ps(a, b, c).as_f32x4(); + transmute(simd_select_bitmask(k, fmadd, c.as_f32x4())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmadd_pd&expand=2545) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmadd132pd or vfmadd213pd or vfmadd231pd +pub unsafe fn _mm512_fmadd_pd(a: __m512d, b: __m512d, c: __m512d) -> __m512d { + transmute(vfmadd132pd(a.as_f64x8(), b.as_f64x8(), c.as_f64x8())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmadd_pd&expand=2546) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmadd132pd or vfmadd213pd or vfmadd231pd +pub unsafe fn _mm512_mask_fmadd_pd(a: __m512d, k: __mmask8, b: __m512d, c: __m512d) -> __m512d { + let fmadd = _mm512_fmadd_pd(a, b, c).as_f64x8(); + transmute(simd_select_bitmask(k, fmadd, a.as_f64x8())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmadd_pd&expand=2548) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmadd132pd or vfmadd213pd or vfmadd231pd +pub unsafe fn _mm512_maskz_fmadd_pd(k: __mmask8, a: __m512d, b: __m512d, c: __m512d) -> __m512d { + let fmadd = _mm512_fmadd_pd(a, b, c).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, fmadd, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmadd_pd&expand=2547) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmadd132pd or vfmadd213pd or vfmadd231pd +pub unsafe fn _mm512_mask3_fmadd_pd(a: __m512d, b: __m512d, c: __m512d, k: __mmask8) -> __m512d { + let fmadd = _mm512_fmadd_pd(a, b, c).as_f64x8(); + transmute(simd_select_bitmask(k, fmadd, c.as_f64x8())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_fmadd_pd&expand=2542) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmadd132pd or vfmadd213pd or vfmadd231pd +pub unsafe fn _mm256_mask_fmadd_pd(a: __m256d, k: __mmask8, b: __m256d, c: __m256d) -> __m256d { + let fmadd = _mm256_fmadd_pd(a, b, c).as_f64x4(); + transmute(simd_select_bitmask(k, fmadd, a.as_f64x4())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_fmadd_pd&expand=2544) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmadd132pd or vfmadd213pd or vfmadd231pd +pub unsafe fn _mm256_maskz_fmadd_pd(k: __mmask8, a: __m256d, b: __m256d, c: __m256d) -> __m256d { + let fmadd = _mm256_fmadd_pd(a, b, c).as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, fmadd, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask3_fmadd_pd&expand=2543) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmadd132pd or vfmadd213pd or vfmadd231pd +pub unsafe fn _mm256_mask3_fmadd_pd(a: __m256d, b: __m256d, c: __m256d, k: __mmask8) -> __m256d { + let fmadd = _mm256_fmadd_pd(a, b, c).as_f64x4(); + transmute(simd_select_bitmask(k, fmadd, c.as_f64x4())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_fmadd_pd&expand=2538) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmadd132pd or vfmadd213pd or vfmadd231pd +pub unsafe fn _mm_mask_fmadd_pd(a: __m128d, k: __mmask8, b: __m128d, c: __m128d) -> __m128d { + let fmadd = _mm_fmadd_pd(a, b, c).as_f64x2(); + transmute(simd_select_bitmask(k, fmadd, a.as_f64x2())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_fmadd_pd&expand=2540) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmadd132pd or vfmadd213pd or vfmadd231pd +pub unsafe fn _mm_maskz_fmadd_pd(k: __mmask8, a: __m128d, b: __m128d, c: __m128d) -> __m128d { + let fmadd = _mm_fmadd_pd(a, b, c).as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, fmadd, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask3_fmadd_pd&expand=2539) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmadd132pd or vfmadd213pd or vfmadd231pd +pub unsafe fn _mm_mask3_fmadd_pd(a: __m128d, b: __m128d, c: __m128d, k: __mmask8) -> __m128d { + let fmadd = _mm_fmadd_pd(a, b, c).as_f64x2(); + transmute(simd_select_bitmask(k, fmadd, c.as_f64x2())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmsub_ps&expand=2643) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generate vfmadd, gcc generate vfmsub +pub unsafe fn _mm512_fmsub_ps(a: __m512, b: __m512, c: __m512) -> __m512 { + let zero: f32x16 = mem::zeroed(); + let sub = simd_sub(zero, c.as_f32x16()); + transmute(vfmadd132ps(a.as_f32x16(), b.as_f32x16(), sub)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmsub_ps&expand=2644) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generate vfmadd, gcc generate vfmsub +pub unsafe fn _mm512_mask_fmsub_ps(a: __m512, k: __mmask16, b: __m512, c: __m512) -> __m512 { + let fmsub = _mm512_fmsub_ps(a, b, c).as_f32x16(); + transmute(simd_select_bitmask(k, fmsub, a.as_f32x16())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmsub_ps&expand=2646) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generate vfmadd, gcc generate vfmsub +pub unsafe fn _mm512_maskz_fmsub_ps(k: __mmask16, a: __m512, b: __m512, c: __m512) -> __m512 { + let fmsub = _mm512_fmsub_ps(a, b, c).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, fmsub, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmsub_ps&expand=2645) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generate vfmadd, gcc generate vfmsub +pub unsafe fn _mm512_mask3_fmsub_ps(a: __m512, b: __m512, c: __m512, k: __mmask16) -> __m512 { + let fmsub = _mm512_fmsub_ps(a, b, c).as_f32x16(); + transmute(simd_select_bitmask(k, fmsub, c.as_f32x16())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_fmsub_ps&expand=2640) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsub))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generate vfmadd, gcc generate vfmsub +pub unsafe fn _mm256_mask_fmsub_ps(a: __m256, k: __mmask8, b: __m256, c: __m256) -> __m256 { + let fmsub = _mm256_fmsub_ps(a, b, c).as_f32x8(); + transmute(simd_select_bitmask(k, fmsub, a.as_f32x8())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_fmsub_ps&expand=2642) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsub))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generate vfmadd, gcc generate vfmsub +pub unsafe fn _mm256_maskz_fmsub_ps(k: __mmask8, a: __m256, b: __m256, c: __m256) -> __m256 { + let fmsub = _mm256_fmsub_ps(a, b, c).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, fmsub, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask3_fmsub_ps&expand=2641) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsub))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generate vfmadd, gcc generate vfmsub +pub unsafe fn _mm256_mask3_fmsub_ps(a: __m256, b: __m256, c: __m256, k: __mmask8) -> __m256 { + let fmsub = _mm256_fmsub_ps(a, b, c).as_f32x8(); + transmute(simd_select_bitmask(k, fmsub, c.as_f32x8())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_fmsub_ps&expand=2636) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsub))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generate vfmadd, gcc generate vfmsub +pub unsafe fn _mm_mask_fmsub_ps(a: __m128, k: __mmask8, b: __m128, c: __m128) -> __m128 { + let fmsub = _mm_fmsub_ps(a, b, c).as_f32x4(); + transmute(simd_select_bitmask(k, fmsub, a.as_f32x4())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_fmsub_ps&expand=2638) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsub))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generate vfmadd, gcc generate vfmsub +pub unsafe fn _mm_maskz_fmsub_ps(k: __mmask8, a: __m128, b: __m128, c: __m128) -> __m128 { + let fmsub = _mm_fmsub_ps(a, b, c).as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, fmsub, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask3_fmsub_ps&expand=2637) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsub))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generate vfmadd, gcc generate vfmsub +pub unsafe fn _mm_mask3_fmsub_ps(a: __m128, b: __m128, c: __m128, k: __mmask8) -> __m128 { + let fmsub = _mm_fmsub_ps(a, b, c).as_f32x4(); + transmute(simd_select_bitmask(k, fmsub, c.as_f32x4())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmsub_pd&expand=2631) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang fmadd, gcc fmsub +pub unsafe fn _mm512_fmsub_pd(a: __m512d, b: __m512d, c: __m512d) -> __m512d { + let zero: f64x8 = mem::zeroed(); + let sub = simd_sub(zero, c.as_f64x8()); + transmute(vfmadd132pd(a.as_f64x8(), b.as_f64x8(), sub)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmsub_pd&expand=2632) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang fmadd, gcc fmsub +pub unsafe fn _mm512_mask_fmsub_pd(a: __m512d, k: __mmask8, b: __m512d, c: __m512d) -> __m512d { + let fmsub = _mm512_fmsub_pd(a, b, c).as_f64x8(); + transmute(simd_select_bitmask(k, fmsub, a.as_f64x8())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmsub_pd&expand=2634) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang fmadd, gcc fmsub +pub unsafe fn _mm512_maskz_fmsub_pd(k: __mmask8, a: __m512d, b: __m512d, c: __m512d) -> __m512d { + let fmsub = _mm512_fmsub_pd(a, b, c).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, fmsub, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmsub_pd&expand=2633) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang fmadd, gcc fmsub +pub unsafe fn _mm512_mask3_fmsub_pd(a: __m512d, b: __m512d, c: __m512d, k: __mmask8) -> __m512d { + let fmsub = _mm512_fmsub_pd(a, b, c).as_f64x8(); + transmute(simd_select_bitmask(k, fmsub, c.as_f64x8())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_fmsub_pd&expand=2628) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsub))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang fmadd, gcc fmsub +pub unsafe fn _mm256_mask_fmsub_pd(a: __m256d, k: __mmask8, b: __m256d, c: __m256d) -> __m256d { + let fmsub = _mm256_fmsub_pd(a, b, c).as_f64x4(); + transmute(simd_select_bitmask(k, fmsub, a.as_f64x4())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_fmsub_pd&expand=2630) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsub))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang fmadd, gcc fmsub +pub unsafe fn _mm256_maskz_fmsub_pd(k: __mmask8, a: __m256d, b: __m256d, c: __m256d) -> __m256d { + let fmsub = _mm256_fmsub_pd(a, b, c).as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, fmsub, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask3_fmsub_pd&expand=2629) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsub))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang fmadd, gcc fmsub +pub unsafe fn _mm256_mask3_fmsub_pd(a: __m256d, b: __m256d, c: __m256d, k: __mmask8) -> __m256d { + let fmsub = _mm256_fmsub_pd(a, b, c).as_f64x4(); + transmute(simd_select_bitmask(k, fmsub, c.as_f64x4())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_fmsub_pd&expand=2624) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsub))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang fmadd, gcc fmsub +pub unsafe fn _mm_mask_fmsub_pd(a: __m128d, k: __mmask8, b: __m128d, c: __m128d) -> __m128d { + let fmsub = _mm_fmsub_pd(a, b, c).as_f64x2(); + transmute(simd_select_bitmask(k, fmsub, a.as_f64x2())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_fmsub_pd&expand=2626) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsub))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang fmadd, gcc fmsub +pub unsafe fn _mm_maskz_fmsub_pd(k: __mmask8, a: __m128d, b: __m128d, c: __m128d) -> __m128d { + let fmsub = _mm_fmsub_pd(a, b, c).as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, fmsub, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask3_fmsub_pd&expand=2625) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsub))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang fmadd, gcc fmsub +pub unsafe fn _mm_mask3_fmsub_pd(a: __m128d, b: __m128d, c: __m128d, k: __mmask8) -> __m128d { + let fmsub = _mm_fmsub_pd(a, b, c).as_f64x2(); + transmute(simd_select_bitmask(k, fmsub, c.as_f64x2())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmaddsub_ps&expand=2611) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps +pub unsafe fn _mm512_fmaddsub_ps(a: __m512, b: __m512, c: __m512) -> __m512 { + transmute(vfmaddsub213ps( + a.as_f32x16(), + b.as_f32x16(), + c.as_f32x16(), + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmaddsub_ps&expand=2612) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps +pub unsafe fn _mm512_mask_fmaddsub_ps(a: __m512, k: __mmask16, b: __m512, c: __m512) -> __m512 { + let fmaddsub = _mm512_fmaddsub_ps(a, b, c).as_f32x16(); + transmute(simd_select_bitmask(k, fmaddsub, a.as_f32x16())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmaddsub_ps&expand=2614) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps +pub unsafe fn _mm512_maskz_fmaddsub_ps(k: __mmask16, a: __m512, b: __m512, c: __m512) -> __m512 { + let fmaddsub = _mm512_fmaddsub_ps(a, b, c).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, fmaddsub, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmaddsub_ps&expand=2613) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps +pub unsafe fn _mm512_mask3_fmaddsub_ps(a: __m512, b: __m512, c: __m512, k: __mmask16) -> __m512 { + let fmaddsub = _mm512_fmaddsub_ps(a, b, c).as_f32x16(); + transmute(simd_select_bitmask(k, fmaddsub, c.as_f32x16())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_fmaddsub_ps&expand=2608) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps +pub unsafe fn _mm256_mask_fmaddsub_ps(a: __m256, k: __mmask8, b: __m256, c: __m256) -> __m256 { + let fmaddsub = _mm256_fmaddsub_ps(a, b, c).as_f32x8(); + transmute(simd_select_bitmask(k, fmaddsub, a.as_f32x8())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_fmaddsub_ps&expand=2610) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps +pub unsafe fn _mm256_maskz_fmaddsub_ps(k: __mmask8, a: __m256, b: __m256, c: __m256) -> __m256 { + let fmaddsub = _mm256_fmaddsub_ps(a, b, c).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, fmaddsub, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask3_fmaddsub_ps&expand=2609) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps +pub unsafe fn _mm256_mask3_fmaddsub_ps(a: __m256, b: __m256, c: __m256, k: __mmask8) -> __m256 { + let fmaddsub = _mm256_fmaddsub_ps(a, b, c).as_f32x8(); + transmute(simd_select_bitmask(k, fmaddsub, c.as_f32x8())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_fmaddsub_ps&expand=2604) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps +pub unsafe fn _mm_mask_fmaddsub_ps(a: __m128, k: __mmask8, b: __m128, c: __m128) -> __m128 { + let fmaddsub = _mm_fmaddsub_ps(a, b, c).as_f32x4(); + transmute(simd_select_bitmask(k, fmaddsub, a.as_f32x4())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/IntrinsicsGuide/#text=_mm_maskz_fmaddsub_ps&expand=2606) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps +pub unsafe fn _mm_maskz_fmaddsub_ps(k: __mmask8, a: __m128, b: __m128, c: __m128) -> __m128 { + let fmaddsub = _mm_fmaddsub_ps(a, b, c).as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, fmaddsub, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask3_fmaddsub_ps&expand=2605) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps +pub unsafe fn _mm_mask3_fmaddsub_ps(a: __m128, b: __m128, c: __m128, k: __mmask8) -> __m128 { + let fmaddsub = _mm_fmaddsub_ps(a, b, c).as_f32x4(); + transmute(simd_select_bitmask(k, fmaddsub, c.as_f32x4())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmaddsub_pd&expand=2599) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd +pub unsafe fn _mm512_fmaddsub_pd(a: __m512d, b: __m512d, c: __m512d) -> __m512d { + transmute(vfmaddsub213pd( + a.as_f64x8(), + b.as_f64x8(), + c.as_f64x8(), + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmaddsub_pd&expand=2600) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd +pub unsafe fn _mm512_mask_fmaddsub_pd(a: __m512d, k: __mmask8, b: __m512d, c: __m512d) -> __m512d { + let fmaddsub = _mm512_fmaddsub_pd(a, b, c).as_f64x8(); + transmute(simd_select_bitmask(k, fmaddsub, a.as_f64x8())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmaddsub_pd&expand=2602) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd +pub unsafe fn _mm512_maskz_fmaddsub_pd(k: __mmask8, a: __m512d, b: __m512d, c: __m512d) -> __m512d { + let fmaddsub = _mm512_fmaddsub_pd(a, b, c).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, fmaddsub, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmaddsub_ps&expand=2613) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd +pub unsafe fn _mm512_mask3_fmaddsub_pd(a: __m512d, b: __m512d, c: __m512d, k: __mmask8) -> __m512d { + let fmaddsub = _mm512_fmaddsub_pd(a, b, c).as_f64x8(); + transmute(simd_select_bitmask(k, fmaddsub, c.as_f64x8())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_fmaddsub_pd&expand=2596) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd +pub unsafe fn _mm256_mask_fmaddsub_pd(a: __m256d, k: __mmask8, b: __m256d, c: __m256d) -> __m256d { + let fmaddsub = _mm256_fmaddsub_pd(a, b, c).as_f64x4(); + transmute(simd_select_bitmask(k, fmaddsub, a.as_f64x4())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_fmaddsub_pd&expand=2598) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd +pub unsafe fn _mm256_maskz_fmaddsub_pd(k: __mmask8, a: __m256d, b: __m256d, c: __m256d) -> __m256d { + let fmaddsub = _mm256_fmaddsub_pd(a, b, c).as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, fmaddsub, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask3_fmaddsub_pd&expand=2597) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd +pub unsafe fn _mm256_mask3_fmaddsub_pd(a: __m256d, b: __m256d, c: __m256d, k: __mmask8) -> __m256d { + let fmaddsub = _mm256_fmaddsub_pd(a, b, c).as_f64x4(); + transmute(simd_select_bitmask(k, fmaddsub, c.as_f64x4())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_fmaddsub_pd&expand=2592) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd +pub unsafe fn _mm_mask_fmaddsub_pd(a: __m128d, k: __mmask8, b: __m128d, c: __m128d) -> __m128d { + let fmaddsub = _mm_fmaddsub_pd(a, b, c).as_f64x2(); + transmute(simd_select_bitmask(k, fmaddsub, a.as_f64x2())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_fmaddsub_pd&expand=2594) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd +pub unsafe fn _mm_maskz_fmaddsub_pd(k: __mmask8, a: __m128d, b: __m128d, c: __m128d) -> __m128d { + let fmaddsub = _mm_fmaddsub_pd(a, b, c).as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, fmaddsub, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask3_fmaddsub_pd&expand=2593) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd +pub unsafe fn _mm_mask3_fmaddsub_pd(a: __m128d, b: __m128d, c: __m128d, k: __mmask8) -> __m128d { + let fmaddsub = _mm_fmaddsub_pd(a, b, c).as_f64x2(); + transmute(simd_select_bitmask(k, fmaddsub, c.as_f64x2())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmsubadd_ps&expand=2691) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps +pub unsafe fn _mm512_fmsubadd_ps(a: __m512, b: __m512, c: __m512) -> __m512 { + let zero: f32x16 = mem::zeroed(); + let sub = simd_sub(zero, c.as_f32x16()); + transmute(vfmaddsub213ps( + a.as_f32x16(), + b.as_f32x16(), + sub, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmsubadd_ps&expand=2692) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps +pub unsafe fn _mm512_mask_fmsubadd_ps(a: __m512, k: __mmask16, b: __m512, c: __m512) -> __m512 { + let fmsubadd = _mm512_fmsubadd_ps(a, b, c).as_f32x16(); + transmute(simd_select_bitmask(k, fmsubadd, a.as_f32x16())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmsubadd_ps&expand=2694) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps +pub unsafe fn _mm512_maskz_fmsubadd_ps(k: __mmask16, a: __m512, b: __m512, c: __m512) -> __m512 { + let fmsubadd = _mm512_fmsubadd_ps(a, b, c).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, fmsubadd, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmsubadd_ps&expand=2693) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps +pub unsafe fn _mm512_mask3_fmsubadd_ps(a: __m512, b: __m512, c: __m512, k: __mmask16) -> __m512 { + let fmsubadd = _mm512_fmsubadd_ps(a, b, c).as_f32x16(); + transmute(simd_select_bitmask(k, fmsubadd, c.as_f32x16())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_fmsubadd_ps&expand=2688) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsubadd))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps +pub unsafe fn _mm256_mask_fmsubadd_ps(a: __m256, k: __mmask8, b: __m256, c: __m256) -> __m256 { + let fmsubadd = _mm256_fmsubadd_ps(a, b, c).as_f32x8(); + transmute(simd_select_bitmask(k, fmsubadd, a.as_f32x8())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_fmsubadd_ps&expand=2690) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsubadd))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps +pub unsafe fn _mm256_maskz_fmsubadd_ps(k: __mmask8, a: __m256, b: __m256, c: __m256) -> __m256 { + let fmsubadd = _mm256_fmsubadd_ps(a, b, c).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, fmsubadd, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask3_fmsubadd_ps&expand=2689) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsubadd))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps +pub unsafe fn _mm256_mask3_fmsubadd_ps(a: __m256, b: __m256, c: __m256, k: __mmask8) -> __m256 { + let fmsubadd = _mm256_fmsubadd_ps(a, b, c).as_f32x8(); + transmute(simd_select_bitmask(k, fmsubadd, c.as_f32x8())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_fmsubadd_ps&expand=2684) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsubadd))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps +pub unsafe fn _mm_mask_fmsubadd_ps(a: __m128, k: __mmask8, b: __m128, c: __m128) -> __m128 { + let fmsubadd = _mm_fmsubadd_ps(a, b, c).as_f32x4(); + transmute(simd_select_bitmask(k, fmsubadd, a.as_f32x4())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_fmsubadd_ps&expand=2686) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsubadd))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps +pub unsafe fn _mm_maskz_fmsubadd_ps(k: __mmask8, a: __m128, b: __m128, c: __m128) -> __m128 { + let fmsubadd = _mm_fmsubadd_ps(a, b, c).as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, fmsubadd, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask3_fmsubadd_ps&expand=2685) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsubadd))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps +pub unsafe fn _mm_mask3_fmsubadd_ps(a: __m128, b: __m128, c: __m128, k: __mmask8) -> __m128 { + let fmsubadd = _mm_fmsubadd_ps(a, b, c).as_f32x4(); + transmute(simd_select_bitmask(k, fmsubadd, c.as_f32x4())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmsubadd_pd&expand=2679) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd +pub unsafe fn _mm512_fmsubadd_pd(a: __m512d, b: __m512d, c: __m512d) -> __m512d { + let zero: f64x8 = mem::zeroed(); + let sub = simd_sub(zero, c.as_f64x8()); + transmute(vfmaddsub213pd( + a.as_f64x8(), + b.as_f64x8(), + sub, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmsubadd_pd&expand=2680) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd +pub unsafe fn _mm512_mask_fmsubadd_pd(a: __m512d, k: __mmask8, b: __m512d, c: __m512d) -> __m512d { + let fmsubadd = _mm512_fmsubadd_pd(a, b, c).as_f64x8(); + transmute(simd_select_bitmask(k, fmsubadd, a.as_f64x8())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmsubadd_pd&expand=2682) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd +pub unsafe fn _mm512_maskz_fmsubadd_pd(k: __mmask8, a: __m512d, b: __m512d, c: __m512d) -> __m512d { + let fmsubadd = _mm512_fmsubadd_pd(a, b, c).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, fmsubadd, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmsubadd_pd&expand=2681) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd +pub unsafe fn _mm512_mask3_fmsubadd_pd(a: __m512d, b: __m512d, c: __m512d, k: __mmask8) -> __m512d { + let fmsubadd = _mm512_fmsubadd_pd(a, b, c).as_f64x8(); + transmute(simd_select_bitmask(k, fmsubadd, c.as_f64x8())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_fmsubadd_pd&expand=2676) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsubadd))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd +pub unsafe fn _mm256_mask_fmsubadd_pd(a: __m256d, k: __mmask8, b: __m256d, c: __m256d) -> __m256d { + let fmsubadd = _mm256_fmsubadd_pd(a, b, c).as_f64x4(); + transmute(simd_select_bitmask(k, fmsubadd, a.as_f64x4())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_fmsubadd_pd&expand=2678) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsubadd))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd +pub unsafe fn _mm256_maskz_fmsubadd_pd(k: __mmask8, a: __m256d, b: __m256d, c: __m256d) -> __m256d { + let fmsubadd = _mm256_fmsubadd_pd(a, b, c).as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, fmsubadd, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask3_fmsubadd_pd&expand=2677) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsubadd))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd +pub unsafe fn _mm256_mask3_fmsubadd_pd(a: __m256d, b: __m256d, c: __m256d, k: __mmask8) -> __m256d { + let fmsubadd = _mm256_fmsubadd_pd(a, b, c).as_f64x4(); + transmute(simd_select_bitmask(k, fmsubadd, c.as_f64x4())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_fmsubadd_pd&expand=2672) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsubadd))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd +pub unsafe fn _mm_mask_fmsubadd_pd(a: __m128d, k: __mmask8, b: __m128d, c: __m128d) -> __m128d { + let fmsubadd = _mm_fmsubadd_pd(a, b, c).as_f64x2(); + transmute(simd_select_bitmask(k, fmsubadd, a.as_f64x2())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_fmsubadd_pd&expand=2674) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsubadd))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd +pub unsafe fn _mm_maskz_fmsubadd_pd(k: __mmask8, a: __m128d, b: __m128d, c: __m128d) -> __m128d { + let fmsubadd = _mm_fmsubadd_pd(a, b, c).as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, fmsubadd, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask3_fmsubadd_pd&expand=2673) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfmsubadd))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd +pub unsafe fn _mm_mask3_fmsubadd_pd(a: __m128d, b: __m128d, c: __m128d, k: __mmask8) -> __m128d { + let fmsubadd = _mm_fmsubadd_pd(a, b, c).as_f64x2(); + transmute(simd_select_bitmask(k, fmsubadd, c.as_f64x2())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fnmadd_ps&expand=2723) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps +pub unsafe fn _mm512_fnmadd_ps(a: __m512, b: __m512, c: __m512) -> __m512 { + let zero: f32x16 = mem::zeroed(); + let sub = simd_sub(zero, a.as_f32x16()); + transmute(vfmadd132ps(sub, b.as_f32x16(), c.as_f32x16())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fnmadd_ps&expand=2724) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps +pub unsafe fn _mm512_mask_fnmadd_ps(a: __m512, k: __mmask16, b: __m512, c: __m512) -> __m512 { + let fnmadd = _mm512_fnmadd_ps(a, b, c).as_f32x16(); + transmute(simd_select_bitmask(k, fnmadd, a.as_f32x16())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fnmadd_ps&expand=2726) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps +pub unsafe fn _mm512_maskz_fnmadd_ps(k: __mmask16, a: __m512, b: __m512, c: __m512) -> __m512 { + let fnmadd = _mm512_fnmadd_ps(a, b, c).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, fnmadd, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fnmadd_ps&expand=2725) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps +pub unsafe fn _mm512_mask3_fnmadd_ps(a: __m512, b: __m512, c: __m512, k: __mmask16) -> __m512 { + let fnmadd = _mm512_fnmadd_ps(a, b, c).as_f32x16(); + transmute(simd_select_bitmask(k, fnmadd, c.as_f32x16())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_fnmadd_ps&expand=2720) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmadd))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps +pub unsafe fn _mm256_mask_fnmadd_ps(a: __m256, k: __mmask8, b: __m256, c: __m256) -> __m256 { + let fnmadd = _mm256_fnmadd_ps(a, b, c).as_f32x8(); + transmute(simd_select_bitmask(k, fnmadd, a.as_f32x8())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_fnmadd_ps&expand=2722) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmadd))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps +pub unsafe fn _mm256_maskz_fnmadd_ps(k: __mmask8, a: __m256, b: __m256, c: __m256) -> __m256 { + let fnmadd = _mm256_fnmadd_ps(a, b, c).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, fnmadd, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask3_fnmadd_ps&expand=2721) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmadd))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps +pub unsafe fn _mm256_mask3_fnmadd_ps(a: __m256, b: __m256, c: __m256, k: __mmask8) -> __m256 { + let fnmadd = _mm256_fnmadd_ps(a, b, c).as_f32x8(); + transmute(simd_select_bitmask(k, fnmadd, c.as_f32x8())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_fnmadd_ps&expand=2716) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmadd))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps +pub unsafe fn _mm_mask_fnmadd_ps(a: __m128, k: __mmask8, b: __m128, c: __m128) -> __m128 { + let fnmadd = _mm_fnmadd_ps(a, b, c).as_f32x4(); + transmute(simd_select_bitmask(k, fnmadd, a.as_f32x4())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_fnmadd_ps&expand=2718) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmadd))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps +pub unsafe fn _mm_maskz_fnmadd_ps(k: __mmask8, a: __m128, b: __m128, c: __m128) -> __m128 { + let fnmadd = _mm_fnmadd_ps(a, b, c).as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, fnmadd, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask3_fnmadd_ps&expand=2717) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmadd))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps +pub unsafe fn _mm_mask3_fnmadd_ps(a: __m128, b: __m128, c: __m128, k: __mmask8) -> __m128 { + let fnmadd = _mm_fnmadd_ps(a, b, c).as_f32x4(); + transmute(simd_select_bitmask(k, fnmadd, c.as_f32x4())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fnmadd_pd&expand=2711) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd +pub unsafe fn _mm512_fnmadd_pd(a: __m512d, b: __m512d, c: __m512d) -> __m512d { + let zero: f64x8 = mem::zeroed(); + let sub = simd_sub(zero, a.as_f64x8()); + transmute(vfmadd132pd(sub, b.as_f64x8(), c.as_f64x8())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fnmadd_pd&expand=2712) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd +pub unsafe fn _mm512_mask_fnmadd_pd(a: __m512d, k: __mmask8, b: __m512d, c: __m512d) -> __m512d { + let fnmadd = _mm512_fnmadd_pd(a, b, c).as_f64x8(); + transmute(simd_select_bitmask(k, fnmadd, a.as_f64x8())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fnmadd_pd&expand=2714) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd +pub unsafe fn _mm512_maskz_fnmadd_pd(k: __mmask8, a: __m512d, b: __m512d, c: __m512d) -> __m512d { + let fnmadd = _mm512_fnmadd_pd(a, b, c).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, fnmadd, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fnmadd_pd&expand=2713) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd +pub unsafe fn _mm512_mask3_fnmadd_pd(a: __m512d, b: __m512d, c: __m512d, k: __mmask8) -> __m512d { + let fnmadd = _mm512_fnmadd_pd(a, b, c).as_f64x8(); + transmute(simd_select_bitmask(k, fnmadd, c.as_f64x8())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_fnmadd_pd&expand=2708) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmadd))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd +pub unsafe fn _mm256_mask_fnmadd_pd(a: __m256d, k: __mmask8, b: __m256d, c: __m256d) -> __m256d { + let fnmadd = _mm256_fnmadd_pd(a, b, c).as_f64x4(); + transmute(simd_select_bitmask(k, fnmadd, a.as_f64x4())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_fnmadd_pd&expand=2710) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmadd))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd +pub unsafe fn _mm256_maskz_fnmadd_pd(k: __mmask8, a: __m256d, b: __m256d, c: __m256d) -> __m256d { + let fnmadd = _mm256_fnmadd_pd(a, b, c).as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, fnmadd, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask3_fnmadd_pd&expand=2709) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmadd))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd +pub unsafe fn _mm256_mask3_fnmadd_pd(a: __m256d, b: __m256d, c: __m256d, k: __mmask8) -> __m256d { + let fnmadd = _mm256_fnmadd_pd(a, b, c).as_f64x4(); + transmute(simd_select_bitmask(k, fnmadd, c.as_f64x4())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_fnmadd_pd&expand=2704) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmadd))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd +pub unsafe fn _mm_mask_fnmadd_pd(a: __m128d, k: __mmask8, b: __m128d, c: __m128d) -> __m128d { + let fnmadd = _mm_fnmadd_pd(a, b, c).as_f64x2(); + transmute(simd_select_bitmask(k, fnmadd, a.as_f64x2())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_fnmadd_pd&expand=2706) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmadd))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd +pub unsafe fn _mm_maskz_fnmadd_pd(k: __mmask8, a: __m128d, b: __m128d, c: __m128d) -> __m128d { + let fnmadd = _mm_fnmadd_pd(a, b, c).as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, fnmadd, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask3_fnmadd_pd&expand=2705) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmadd))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd +pub unsafe fn _mm_mask3_fnmadd_pd(a: __m128d, b: __m128d, c: __m128d, k: __mmask8) -> __m128d { + let fnmadd = _mm_fnmadd_pd(a, b, c).as_f64x2(); + transmute(simd_select_bitmask(k, fnmadd, c.as_f64x2())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fnmsub_ps&expand=2771) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps +pub unsafe fn _mm512_fnmsub_ps(a: __m512, b: __m512, c: __m512) -> __m512 { + let zero: f32x16 = mem::zeroed(); + let suba = simd_sub(zero, a.as_f32x16()); + let subc = simd_sub(zero, c.as_f32x16()); + transmute(vfmadd132ps(suba, b.as_f32x16(), subc)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fnmsub_ps&expand=2772) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps +pub unsafe fn _mm512_mask_fnmsub_ps(a: __m512, k: __mmask16, b: __m512, c: __m512) -> __m512 { + let fnmsub = _mm512_fnmsub_ps(a, b, c).as_f32x16(); + transmute(simd_select_bitmask(k, fnmsub, a.as_f32x16())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fnmsub_ps&expand=2774) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps +pub unsafe fn _mm512_maskz_fnmsub_ps(k: __mmask16, a: __m512, b: __m512, c: __m512) -> __m512 { + let fnmsub = _mm512_fnmsub_ps(a, b, c).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, fnmsub, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fnmsub_ps&expand=2773) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps +pub unsafe fn _mm512_mask3_fnmsub_ps(a: __m512, b: __m512, c: __m512, k: __mmask16) -> __m512 { + let fnmsub = _mm512_fnmsub_ps(a, b, c).as_f32x16(); + transmute(simd_select_bitmask(k, fnmsub, c.as_f32x16())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_fnmsub_ps&expand=2768) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmsub))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps +pub unsafe fn _mm256_mask_fnmsub_ps(a: __m256, k: __mmask8, b: __m256, c: __m256) -> __m256 { + let fnmsub = _mm256_fnmsub_ps(a, b, c).as_f32x8(); + transmute(simd_select_bitmask(k, fnmsub, a.as_f32x8())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_fnmsub_ps&expand=2770) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmsub))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps +pub unsafe fn _mm256_maskz_fnmsub_ps(k: __mmask8, a: __m256, b: __m256, c: __m256) -> __m256 { + let fnmsub = _mm256_fnmsub_ps(a, b, c).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, fnmsub, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask3_fnmsub_ps&expand=2769) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmsub))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps +pub unsafe fn _mm256_mask3_fnmsub_ps(a: __m256, b: __m256, c: __m256, k: __mmask8) -> __m256 { + let fnmsub = _mm256_fnmsub_ps(a, b, c).as_f32x8(); + transmute(simd_select_bitmask(k, fnmsub, c.as_f32x8())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_fnmsub_ps&expand=2764) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmsub))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps +pub unsafe fn _mm_mask_fnmsub_ps(a: __m128, k: __mmask8, b: __m128, c: __m128) -> __m128 { + let fnmsub = _mm_fnmsub_ps(a, b, c).as_f32x4(); + transmute(simd_select_bitmask(k, fnmsub, a.as_f32x4())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_fnmsub_ps&expand=2766) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmsub))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps +pub unsafe fn _mm_maskz_fnmsub_ps(k: __mmask8, a: __m128, b: __m128, c: __m128) -> __m128 { + let fnmsub = _mm_fnmsub_ps(a, b, c).as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, fnmsub, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask3_fnmsub_ps&expand=2765) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmsub))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps +pub unsafe fn _mm_mask3_fnmsub_ps(a: __m128, b: __m128, c: __m128, k: __mmask8) -> __m128 { + let fnmsub = _mm_fnmsub_ps(a, b, c).as_f32x4(); + transmute(simd_select_bitmask(k, fnmsub, c.as_f32x4())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fnmsub_pd&expand=2759) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd +pub unsafe fn _mm512_fnmsub_pd(a: __m512d, b: __m512d, c: __m512d) -> __m512d { + let zero: f64x8 = mem::zeroed(); + let suba = simd_sub(zero, a.as_f64x8()); + let subc = simd_sub(zero, c.as_f64x8()); + transmute(vfmadd132pd(suba, b.as_f64x8(), subc)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fnmsub_pd&expand=2760) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd +pub unsafe fn _mm512_mask_fnmsub_pd(a: __m512d, k: __mmask8, b: __m512d, c: __m512d) -> __m512d { + let fnmsub = _mm512_fnmsub_pd(a, b, c).as_f64x8(); + transmute(simd_select_bitmask(k, fnmsub, a.as_f64x8())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fnmsub_pd&expand=2762) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd +pub unsafe fn _mm512_maskz_fnmsub_pd(k: __mmask8, a: __m512d, b: __m512d, c: __m512d) -> __m512d { + let fnmsub = _mm512_fnmsub_pd(a, b, c).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, fnmsub, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fnmsub_pd&expand=2761) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd +pub unsafe fn _mm512_mask3_fnmsub_pd(a: __m512d, b: __m512d, c: __m512d, k: __mmask8) -> __m512d { + let fnmsub = _mm512_fnmsub_pd(a, b, c).as_f64x8(); + transmute(simd_select_bitmask(k, fnmsub, c.as_f64x8())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_fnmsub_pd&expand=2756) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmsub))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd +pub unsafe fn _mm256_mask_fnmsub_pd(a: __m256d, k: __mmask8, b: __m256d, c: __m256d) -> __m256d { + let fnmsub = _mm256_fnmsub_pd(a, b, c).as_f64x4(); + transmute(simd_select_bitmask(k, fnmsub, a.as_f64x4())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_fnmsub_pd&expand=2758) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmsub))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd +pub unsafe fn _mm256_maskz_fnmsub_pd(k: __mmask8, a: __m256d, b: __m256d, c: __m256d) -> __m256d { + let fnmsub = _mm256_fnmsub_pd(a, b, c).as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, fnmsub, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask3_fnmsub_pd&expand=2757) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmsub))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd +pub unsafe fn _mm256_mask3_fnmsub_pd(a: __m256d, b: __m256d, c: __m256d, k: __mmask8) -> __m256d { + let fnmsub = _mm256_fnmsub_pd(a, b, c).as_f64x4(); + transmute(simd_select_bitmask(k, fnmsub, c.as_f64x4())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_fnmsub_pd&expand=2752) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmsub))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd +pub unsafe fn _mm_mask_fnmsub_pd(a: __m128d, k: __mmask8, b: __m128d, c: __m128d) -> __m128d { + let fnmsub = _mm_fnmsub_pd(a, b, c).as_f64x2(); + transmute(simd_select_bitmask(k, fnmsub, a.as_f64x2())) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_fnmsub_pd&expand=2754) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmsub))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd +pub unsafe fn _mm_maskz_fnmsub_pd(k: __mmask8, a: __m128d, b: __m128d, c: __m128d) -> __m128d { + let fnmsub = _mm_fnmsub_pd(a, b, c).as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, fnmsub, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask3_fnmsub_pd&expand=2753) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfnmsub))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd +pub unsafe fn _mm_mask3_fnmsub_pd(a: __m128d, b: __m128d, c: __m128d, k: __mmask8) -> __m128d { + let fnmsub = _mm_fnmsub_pd(a, b, c).as_f64x2(); + transmute(simd_select_bitmask(k, fnmsub, c.as_f64x2())) +} + +/// Compute the approximate reciprocal of packed single-precision (32-bit) floating-point elements in a, and store the results in dst. The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_rcp14_ps&expand=4502) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrcp14ps))] +pub unsafe fn _mm512_rcp14_ps(a: __m512) -> __m512 { + transmute(vrcp14ps( + a.as_f32x16(), + _mm512_setzero_ps().as_f32x16(), + 0b11111111_11111111, + )) +} + +/// Compute the approximate reciprocal of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_rcp14_ps&expand=4500) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrcp14ps))] +pub unsafe fn _mm512_mask_rcp14_ps(src: __m512, k: __mmask16, a: __m512) -> __m512 { + transmute(vrcp14ps(a.as_f32x16(), src.as_f32x16(), k)) +} + +/// Compute the approximate reciprocal of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_rcp14_ps&expand=4501) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrcp14ps))] +pub unsafe fn _mm512_maskz_rcp14_ps(k: __mmask16, a: __m512) -> __m512 { + transmute(vrcp14ps(a.as_f32x16(), _mm512_setzero_ps().as_f32x16(), k)) +} + +/// Compute the approximate reciprocal of packed single-precision (32-bit) floating-point elements in a, and store the results in dst. The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_rcp14_ps&expand=4499) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrcp14ps))] +pub unsafe fn _mm256_rcp14_ps(a: __m256) -> __m256 { + transmute(vrcp14ps256( + a.as_f32x8(), + _mm256_setzero_ps().as_f32x8(), + 0b11111111, + )) +} + +/// Compute the approximate reciprocal of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_rcp14_ps&expand=4497) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrcp14ps))] +pub unsafe fn _mm256_mask_rcp14_ps(src: __m256, k: __mmask8, a: __m256) -> __m256 { + transmute(vrcp14ps256(a.as_f32x8(), src.as_f32x8(), k)) +} + +/// Compute the approximate reciprocal of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_rcp14_ps&expand=4498) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrcp14ps))] +pub unsafe fn _mm256_maskz_rcp14_ps(k: __mmask8, a: __m256) -> __m256 { + transmute(vrcp14ps256(a.as_f32x8(), _mm256_setzero_ps().as_f32x8(), k)) +} + +/// Compute the approximate reciprocal of packed single-precision (32-bit) floating-point elements in a, and store the results in dst. The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_rcp14_ps&expand=4496) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrcp14ps))] +pub unsafe fn _mm_rcp14_ps(a: __m128) -> __m128 { + transmute(vrcp14ps128( + a.as_f32x4(), + _mm_setzero_ps().as_f32x4(), + 0b00001111, + )) +} + +/// Compute the approximate reciprocal of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_rcp14_ps&expand=4494) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrcp14ps))] +pub unsafe fn _mm_mask_rcp14_ps(src: __m128, k: __mmask8, a: __m128) -> __m128 { + transmute(vrcp14ps128(a.as_f32x4(), src.as_f32x4(), k)) +} + +/// Compute the approximate reciprocal of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_rcp14_ps&expand=4495) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrcp14ps))] +pub unsafe fn _mm_maskz_rcp14_ps(k: __mmask8, a: __m128) -> __m128 { + transmute(vrcp14ps128(a.as_f32x4(), _mm_setzero_ps().as_f32x4(), k)) +} + +/// Compute the approximate reciprocal of packed double-precision (64-bit) floating-point elements in a, and store the results in dst. The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_rcp14_pd&expand=4493) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrcp14pd))] +pub unsafe fn _mm512_rcp14_pd(a: __m512d) -> __m512d { + transmute(vrcp14pd( + a.as_f64x8(), + _mm512_setzero_pd().as_f64x8(), + 0b11111111, + )) +} + +/// Compute the approximate reciprocal of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_rcp14_pd&expand=4491) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrcp14pd))] +pub unsafe fn _mm512_mask_rcp14_pd(src: __m512d, k: __mmask8, a: __m512d) -> __m512d { + transmute(vrcp14pd(a.as_f64x8(), src.as_f64x8(), k)) +} + +/// Compute the approximate reciprocal of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_rcp14_pd&expand=4492) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrcp14pd))] +pub unsafe fn _mm512_maskz_rcp14_pd(k: __mmask8, a: __m512d) -> __m512d { + transmute(vrcp14pd(a.as_f64x8(), _mm512_setzero_pd().as_f64x8(), k)) +} + +/// Compute the approximate reciprocal of packed double-precision (64-bit) floating-point elements in a, and store the results in dst. The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_rcp14_pd&expand=4490) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrcp14pd))] +pub unsafe fn _mm256_rcp14_pd(a: __m256d) -> __m256d { + transmute(vrcp14pd256( + a.as_f64x4(), + _mm256_setzero_pd().as_f64x4(), + 0b00001111, + )) +} + +/// Compute the approximate reciprocal of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_rcp14_pd&expand=4488) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrcp14pd))] +pub unsafe fn _mm256_mask_rcp14_pd(src: __m256d, k: __mmask8, a: __m256d) -> __m256d { + transmute(vrcp14pd256(a.as_f64x4(), src.as_f64x4(), k)) +} + +/// Compute the approximate reciprocal of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_rcp14_pd&expand=4489) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrcp14pd))] +pub unsafe fn _mm256_maskz_rcp14_pd(k: __mmask8, a: __m256d) -> __m256d { + transmute(vrcp14pd256(a.as_f64x4(), _mm256_setzero_pd().as_f64x4(), k)) +} + +/// Compute the approximate reciprocal of packed double-precision (64-bit) floating-point elements in a, and store the results in dst. The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_rcp14_pd&expand=4487) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrcp14pd))] +pub unsafe fn _mm_rcp14_pd(a: __m128d) -> __m128d { + transmute(vrcp14pd128( + a.as_f64x2(), + _mm_setzero_pd().as_f64x2(), + 0b00000011, + )) +} + +/// Compute the approximate reciprocal of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_rcp14_pd&expand=4485) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrcp14pd))] +pub unsafe fn _mm_mask_rcp14_pd(src: __m128d, k: __mmask8, a: __m128d) -> __m128d { + transmute(vrcp14pd128(a.as_f64x2(), src.as_f64x2(), k)) +} + +/// Compute the approximate reciprocal of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_rcp14_pd&expand=4486) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrcp14pd))] +pub unsafe fn _mm_maskz_rcp14_pd(k: __mmask8, a: __m128d) -> __m128d { + transmute(vrcp14pd128(a.as_f64x2(), _mm_setzero_pd().as_f64x2(), k)) +} + +/// Compute the approximate reciprocal square root of packed single-precision (32-bit) floating-point elements in a, and store the results in dst. The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_rsqrt14_ps&expand=4819) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrsqrt14ps))] +pub unsafe fn _mm512_rsqrt14_ps(a: __m512) -> __m512 { + transmute(vrsqrt14ps( + a.as_f32x16(), + _mm512_setzero_ps().as_f32x16(), + 0b11111111_11111111, + )) +} + +/// Compute the approximate reciprocal square root of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_rsqrt14_ps&expand=4817) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrsqrt14ps))] +pub unsafe fn _mm512_mask_rsqrt14_ps(src: __m512, k: __mmask16, a: __m512) -> __m512 { + transmute(vrsqrt14ps(a.as_f32x16(), src.as_f32x16(), k)) +} + +/// Compute the approximate reciprocal square root of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_rsqrt14_ps&expand=4818) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrsqrt14ps))] +pub unsafe fn _mm512_maskz_rsqrt14_ps(k: __mmask16, a: __m512) -> __m512 { + transmute(vrsqrt14ps( + a.as_f32x16(), + _mm512_setzero_ps().as_f32x16(), + k, + )) +} + +/// Compute the approximate reciprocal square root of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_rsqrt14_ps&expand=4815) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrsqrt14ps))] +pub unsafe fn _mm256_mask_rsqrt14_ps(src: __m256, k: __mmask8, a: __m256) -> __m256 { + transmute(vrsqrt14ps256(a.as_f32x8(), src.as_f32x8(), k)) +} + +/// Compute the approximate reciprocal square root of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_rsqrt14_ps&expand=4816) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrsqrt14ps))] +pub unsafe fn _mm256_maskz_rsqrt14_ps(k: __mmask8, a: __m256) -> __m256 { + transmute(vrsqrt14ps256( + a.as_f32x8(), + _mm256_setzero_ps().as_f32x8(), + k, + )) +} + +/// Compute the approximate reciprocal square root of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_rsqrt14_ps&expand=4813) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrsqrt14ps))] +pub unsafe fn _mm_mask_rsqrt14_ps(src: __m128, k: __mmask8, a: __m128) -> __m128 { + transmute(vrsqrt14ps128(a.as_f32x4(), src.as_f32x4(), k)) +} + +/// Compute the approximate reciprocal square root of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_rsqrt14_ps&expand=4814) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrsqrt14ps))] +pub unsafe fn _mm_maskz_rsqrt14_ps(k: __mmask8, a: __m128) -> __m128 { + transmute(vrsqrt14ps128(a.as_f32x4(), _mm_setzero_ps().as_f32x4(), k)) +} + +/// Compute the approximate reciprocal square root of packed double-precision (64-bit) floating-point elements in a, and store the results in dst. The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_rsqrt14_pd&expand=4812) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrsqrt14pd))] +pub unsafe fn _mm512_rsqrt14_pd(a: __m512d) -> __m512d { + transmute(vrsqrt14pd( + a.as_f64x8(), + _mm512_setzero_pd().as_f64x8(), + 0b11111111, + )) +} + +/// Compute the approximate reciprocal square root of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_rsqrt14_pd&expand=4810) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrsqrt14pd))] +pub unsafe fn _mm512_mask_rsqrt14_pd(src: __m512d, k: __mmask8, a: __m512d) -> __m512d { + transmute(vrsqrt14pd(a.as_f64x8(), src.as_f64x8(), k)) +} + +/// Compute the approximate reciprocal square root of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_rsqrt14_pd&expand=4811) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrsqrt14pd))] +pub unsafe fn _mm512_maskz_rsqrt14_pd(k: __mmask8, a: __m512d) -> __m512d { + transmute(vrsqrt14pd(a.as_f64x8(), _mm512_setzero_pd().as_f64x8(), k)) +} + +/// Compute the approximate reciprocal square root of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_rsqrt14_pd&expand=4808) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrsqrt14pd))] +pub unsafe fn _mm256_mask_rsqrt14_pd(src: __m256d, k: __mmask8, a: __m256d) -> __m256d { + transmute(vrsqrt14pd256(a.as_f64x4(), src.as_f64x4(), k)) +} + +/// Compute the approximate reciprocal square root of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_rsqrt14_pd&expand=4809) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrsqrt14pd))] +pub unsafe fn _mm256_maskz_rsqrt14_pd(k: __mmask8, a: __m256d) -> __m256d { + transmute(vrsqrt14pd256( + a.as_f64x4(), + _mm256_setzero_pd().as_f64x4(), + k, + )) +} + +/// Compute the approximate reciprocal square root of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_rsqrt14_pd&expand=4806) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrsqrt14pd))] +pub unsafe fn _mm_mask_rsqrt14_pd(src: __m128d, k: __mmask8, a: __m128d) -> __m128d { + transmute(vrsqrt14pd128(a.as_f64x2(), src.as_f64x2(), k)) +} + +/// Compute the approximate reciprocal square root of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_rsqrt14_pd&expand=4807) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrsqrt14pd))] +pub unsafe fn _mm_maskz_rsqrt14_pd(k: __mmask8, a: __m128d) -> __m128d { + transmute(vrsqrt14pd128(a.as_f64x2(), _mm_setzero_pd().as_f64x2(), k)) +} + +/// Convert the exponent of each packed single-precision (32-bit) floating-point element in a to a single-precision (32-bit) floating-point number representing the integer exponent, and store the results in dst. This intrinsic essentially calculates floor(log2(x)) for each element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_getexp_ps&expand=2844) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexpps))] +pub unsafe fn _mm512_getexp_ps(a: __m512) -> __m512 { + transmute(vgetexpps( + a.as_f32x16(), + _mm512_setzero_ps().as_f32x16(), + 0b11111111_11111111, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert the exponent of each packed single-precision (32-bit) floating-point element in a to a single-precision (32-bit) floating-point number representing the integer exponent, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). This intrinsic essentially calculates floor(log2(x)) for each element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_getexp_ps&expand=2845) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexpps))] +pub unsafe fn _mm512_mask_getexp_ps(src: __m512, k: __mmask16, a: __m512) -> __m512 { + transmute(vgetexpps( + a.as_f32x16(), + src.as_f32x16(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert the exponent of each packed single-precision (32-bit) floating-point element in a to a single-precision (32-bit) floating-point number representing the integer exponent, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). This intrinsic essentially calculates floor(log2(x)) for each element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_getexp_ps&expand=2846) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexpps))] +pub unsafe fn _mm512_maskz_getexp_ps(k: __mmask16, a: __m512) -> __m512 { + transmute(vgetexpps( + a.as_f32x16(), + _mm512_setzero_ps().as_f32x16(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert the exponent of each packed single-precision (32-bit) floating-point element in a to a single-precision (32-bit) floating-point number representing the integer exponent, and store the results in dst. This intrinsic essentially calculates floor(log2(x)) for each element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_getexp_ps&expand=2841) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetexpps))] +pub unsafe fn _mm256_getexp_ps(a: __m256) -> __m256 { + transmute(vgetexpps256( + a.as_f32x8(), + _mm256_setzero_ps().as_f32x8(), + 0b11111111, + )) +} + +/// Convert the exponent of each packed single-precision (32-bit) floating-point element in a to a single-precision (32-bit) floating-point number representing the integer exponent, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). This intrinsic essentially calculates floor(log2(x)) for each element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_getexp_ps&expand=2842) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetexpps))] +pub unsafe fn _mm256_mask_getexp_ps(src: __m256, k: __mmask8, a: __m256) -> __m256 { + transmute(vgetexpps256(a.as_f32x8(), src.as_f32x8(), k)) +} + +/// Convert the exponent of each packed single-precision (32-bit) floating-point element in a to a single-precision (32-bit) floating-point number representing the integer exponent, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). This intrinsic essentially calculates floor(log2(x)) for each element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_getexp_ps&expand=2843) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetexpps))] +pub unsafe fn _mm256_maskz_getexp_ps(k: __mmask8, a: __m256) -> __m256 { + transmute(vgetexpps256( + a.as_f32x8(), + _mm256_setzero_ps().as_f32x8(), + k, + )) +} + +/// Convert the exponent of each packed single-precision (32-bit) floating-point element in a to a single-precision (32-bit) floating-point number representing the integer exponent, and store the results in dst. This intrinsic essentially calculates floor(log2(x)) for each element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_getexp_ps&expand=2838) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetexpps))] +pub unsafe fn _mm_getexp_ps(a: __m128) -> __m128 { + transmute(vgetexpps128( + a.as_f32x4(), + _mm_setzero_ps().as_f32x4(), + 0b00001111, + )) +} + +/// Convert the exponent of each packed single-precision (32-bit) floating-point element in a to a single-precision (32-bit) floating-point number representing the integer exponent, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). This intrinsic essentially calculates floor(log2(x)) for each element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_getexp_ps&expand=2839) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetexpps))] +pub unsafe fn _mm_mask_getexp_ps(src: __m128, k: __mmask8, a: __m128) -> __m128 { + transmute(vgetexpps128(a.as_f32x4(), src.as_f32x4(), k)) +} + +/// Convert the exponent of each packed single-precision (32-bit) floating-point element in a to a single-precision (32-bit) floating-point number representing the integer exponent, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). This intrinsic essentially calculates floor(log2(x)) for each element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_getexp_ps&expand=2840) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetexpps))] +pub unsafe fn _mm_maskz_getexp_ps(k: __mmask8, a: __m128) -> __m128 { + transmute(vgetexpps128(a.as_f32x4(), _mm_setzero_ps().as_f32x4(), k)) +} + +/// Convert the exponent of each packed double-precision (64-bit) floating-point element in a to a double-precision (64-bit) floating-point number representing the integer exponent, and store the results in dst. This intrinsic essentially calculates floor(log2(x)) for each element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_getexp_pd&expand=2835) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexppd))] +pub unsafe fn _mm512_getexp_pd(a: __m512d) -> __m512d { + transmute(vgetexppd( + a.as_f64x8(), + _mm512_setzero_pd().as_f64x8(), + 0b11111111, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert the exponent of each packed double-precision (64-bit) floating-point element in a to a double-precision (64-bit) floating-point number representing the integer exponent, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). This intrinsic essentially calculates floor(log2(x)) for each element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_getexp_pd&expand=2836) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexppd))] +pub unsafe fn _mm512_mask_getexp_pd(src: __m512d, k: __mmask8, a: __m512d) -> __m512d { + transmute(vgetexppd( + a.as_f64x8(), + src.as_f64x8(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert the exponent of each packed double-precision (64-bit) floating-point element in a to a double-precision (64-bit) floating-point number representing the integer exponent, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). This intrinsic essentially calculates floor(log2(x)) for each element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_getexp_pd&expand=2837) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexppd))] +pub unsafe fn _mm512_maskz_getexp_pd(k: __mmask8, a: __m512d) -> __m512d { + transmute(vgetexppd( + a.as_f64x8(), + _mm512_setzero_pd().as_f64x8(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert the exponent of each packed double-precision (64-bit) floating-point element in a to a double-precision (64-bit) floating-point number representing the integer exponent, and store the results in dst. This intrinsic essentially calculates floor(log2(x)) for each element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_getexp_pd&expand=2832) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetexppd))] +pub unsafe fn _mm256_getexp_pd(a: __m256d) -> __m256d { + transmute(vgetexppd256( + a.as_f64x4(), + _mm256_setzero_pd().as_f64x4(), + 0b00001111, + )) +} + +/// Convert the exponent of each packed double-precision (64-bit) floating-point element in a to a double-precision (64-bit) floating-point number representing the integer exponent, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). This intrinsic essentially calculates floor(log2(x)) for each element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_getexp_pd&expand=2833) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetexppd))] +pub unsafe fn _mm256_mask_getexp_pd(src: __m256d, k: __mmask8, a: __m256d) -> __m256d { + transmute(vgetexppd256(a.as_f64x4(), src.as_f64x4(), k)) +} + +/// Convert the exponent of each packed double-precision (64-bit) floating-point element in a to a double-precision (64-bit) floating-point number representing the integer exponent, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). This intrinsic essentially calculates floor(log2(x)) for each element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_getexp_pd&expand=2834) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetexppd))] +pub unsafe fn _mm256_maskz_getexp_pd(k: __mmask8, a: __m256d) -> __m256d { + transmute(vgetexppd256( + a.as_f64x4(), + _mm256_setzero_pd().as_f64x4(), + k, + )) +} + +/// Convert the exponent of each packed double-precision (64-bit) floating-point element in a to a double-precision (64-bit) floating-point number representing the integer exponent, and store the results in dst. This intrinsic essentially calculates floor(log2(x)) for each element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_getexp_pd&expand=2829) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetexppd))] +pub unsafe fn _mm_getexp_pd(a: __m128d) -> __m128d { + transmute(vgetexppd128( + a.as_f64x2(), + _mm_setzero_pd().as_f64x2(), + 0b00000011, + )) +} + +/// Convert the exponent of each packed double-precision (64-bit) floating-point element in a to a double-precision (64-bit) floating-point number representing the integer exponent, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). This intrinsic essentially calculates floor(log2(x)) for each element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_getexp_pd&expand=2830) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetexppd))] +pub unsafe fn _mm_mask_getexp_pd(src: __m128d, k: __mmask8, a: __m128d) -> __m128d { + transmute(vgetexppd128(a.as_f64x2(), src.as_f64x2(), k)) +} + +/// Convert the exponent of each packed double-precision (64-bit) floating-point element in a to a double-precision (64-bit) floating-point number representing the integer exponent, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). This intrinsic essentially calculates floor(log2(x)) for each element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_getexp_pd&expand=2831) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetexppd))] +pub unsafe fn _mm_maskz_getexp_pd(k: __mmask8, a: __m128d) -> __m128d { + transmute(vgetexppd128(a.as_f64x2(), _mm_setzero_pd().as_f64x2(), k)) +} + +/// Round packed single-precision (32-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst.\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_roundscale_ps&expand=4784) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_roundscale_ps(a: __m512) -> __m512 { + static_assert_imm8!(IMM8); + let a = a.as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + let r = vrndscaleps(a, IMM8, zero, 0b11111111_11111111, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Round packed single-precision (32-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_roundscale_ps&expand=4782) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_roundscale_ps( + src: __m512, + k: __mmask16, + a: __m512, +) -> __m512 { + static_assert_imm8!(IMM8); + let a = a.as_f32x16(); + let src = src.as_f32x16(); + let r = vrndscaleps(a, IMM8, src, k, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Round packed single-precision (32-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_roundscale_ps&expand=4783) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_roundscale_ps(k: __mmask16, a: __m512) -> __m512 { + static_assert_imm8!(IMM8); + let a = a.as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + let r = vrndscaleps(a, IMM8, zero, k, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Round packed single-precision (32-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst.\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_roundscale_ps&expand=4781) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 250))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_roundscale_ps(a: __m256) -> __m256 { + static_assert_imm8!(IMM8); + let a = a.as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + let r = vrndscaleps256(a, IMM8, zero, 0b11111111); + transmute(r) +} + +/// Round packed single-precision (32-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_roundscale_ps&expand=4779) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_roundscale_ps( + src: __m256, + k: __mmask8, + a: __m256, +) -> __m256 { + static_assert_imm8!(IMM8); + let a = a.as_f32x8(); + let src = src.as_f32x8(); + let r = vrndscaleps256(a, IMM8, src, k); + transmute(r) +} + +/// Round packed single-precision (32-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_roundscale_ps&expand=4780) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_roundscale_ps(k: __mmask8, a: __m256) -> __m256 { + static_assert_imm8!(IMM8); + let a = a.as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + let r = vrndscaleps256(a, IMM8, zero, k); + transmute(r) +} + +/// Round packed single-precision (32-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst.\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_roundscale_ps&expand=4778) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 250))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_roundscale_ps(a: __m128) -> __m128 { + static_assert_imm8!(IMM8); + let a = a.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vrndscaleps128(a, IMM8, zero, 0b00001111); + transmute(r) +} + +/// Round packed single-precision (32-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_roundscale_ps&expand=4776) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_roundscale_ps( + src: __m128, + k: __mmask8, + a: __m128, +) -> __m128 { + static_assert_imm8!(IMM8); + let a = a.as_f32x4(); + let src = src.as_f32x4(); + let r = vrndscaleps128(a, IMM8, src, k); + transmute(r) +} + +/// Round packed single-precision (32-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_roundscale_ps&expand=4777) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_roundscale_ps(k: __mmask8, a: __m128) -> __m128 { + static_assert_imm8!(IMM8); + let a = a.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vrndscaleps128(a, IMM8, zero, k); + transmute(r) +} + +/// Round packed double-precision (64-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst.\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_roundscale_pd&expand=4775) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_roundscale_pd(a: __m512d) -> __m512d { + static_assert_imm8!(IMM8); + let a = a.as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + let r = vrndscalepd(a, IMM8, zero, 0b11111111, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Round packed double-precision (64-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_roundscale_pd&expand=4773) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_roundscale_pd( + src: __m512d, + k: __mmask8, + a: __m512d, +) -> __m512d { + static_assert_imm8!(IMM8); + let a = a.as_f64x8(); + let src = src.as_f64x8(); + let r = vrndscalepd(a, IMM8, src, k, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Round packed double-precision (64-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_roundscale_pd&expand=4774) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_roundscale_pd(k: __mmask8, a: __m512d) -> __m512d { + static_assert_imm8!(IMM8); + let a = a.as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + let r = vrndscalepd(a, IMM8, zero, k, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Round packed double-precision (64-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst.\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_roundscale_pd&expand=4772) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_roundscale_pd(a: __m256d) -> __m256d { + static_assert_imm8!(IMM8); + let a = a.as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + let r = vrndscalepd256(a, IMM8, zero, 0b00001111); + transmute(r) +} + +/// Round packed double-precision (64-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_roundscale_pd&expand=4770) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_roundscale_pd( + src: __m256d, + k: __mmask8, + a: __m256d, +) -> __m256d { + static_assert_imm8!(IMM8); + let a = a.as_f64x4(); + let src = src.as_f64x4(); + let r = vrndscalepd256(a, IMM8, src, k); + transmute(r) +} + +/// Round packed double-precision (64-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_roundscale_pd&expand=4771) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_roundscale_pd(k: __mmask8, a: __m256d) -> __m256d { + static_assert_imm8!(IMM8); + let a = a.as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + let r = vrndscalepd256(a, IMM8, zero, k); + transmute(r) +} + +/// Round packed double-precision (64-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst.\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_roundscale_pd&expand=4769) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_roundscale_pd(a: __m128d) -> __m128d { + static_assert_imm8!(IMM8); + let a = a.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vrndscalepd128(a, IMM8, zero, 0b00000011); + transmute(r) +} + +/// Round packed double-precision (64-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_roundscale_pd&expand=4767) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_roundscale_pd( + src: __m128d, + k: __mmask8, + a: __m128d, +) -> __m128d { + static_assert_imm8!(IMM8); + let a = a.as_f64x2(); + let src = src.as_f64x2(); + let r = vrndscalepd128(a, IMM8, src, k); + transmute(r) +} + +/// Round packed double-precision (64-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_roundscale_pd&expand=4768) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_roundscale_pd(k: __mmask8, a: __m128d) -> __m128d { + static_assert_imm8!(IMM8); + let a = a.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vrndscalepd128(a, IMM8, zero, k); + transmute(r) +} + +/// Scale the packed single-precision (32-bit) floating-point elements in a using values from b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_scalef_ps&expand=4883) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefps))] +pub unsafe fn _mm512_scalef_ps(a: __m512, b: __m512) -> __m512 { + transmute(vscalefps( + a.as_f32x16(), + b.as_f32x16(), + _mm512_setzero_ps().as_f32x16(), + 0b11111111_11111111, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Scale the packed single-precision (32-bit) floating-point elements in a using values from b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_scalef_ps&expand=4881) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefps))] +pub unsafe fn _mm512_mask_scalef_ps(src: __m512, k: __mmask16, a: __m512, b: __m512) -> __m512 { + transmute(vscalefps( + a.as_f32x16(), + b.as_f32x16(), + src.as_f32x16(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Scale the packed single-precision (32-bit) floating-point elements in a using values from b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_scalef_ps&expand=4882) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefps))] +pub unsafe fn _mm512_maskz_scalef_ps(k: __mmask16, a: __m512, b: __m512) -> __m512 { + transmute(vscalefps( + a.as_f32x16(), + b.as_f32x16(), + _mm512_setzero_ps().as_f32x16(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Scale the packed single-precision (32-bit) floating-point elements in a using values from b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_scalef_ps&expand=4880) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vscalefps))] +pub unsafe fn _mm256_scalef_ps(a: __m256, b: __m256) -> __m256 { + transmute(vscalefps256( + a.as_f32x8(), + b.as_f32x8(), + _mm256_setzero_ps().as_f32x8(), + 0b11111111, + )) +} + +/// Scale the packed single-precision (32-bit) floating-point elements in a using values from b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_scalef_ps&expand=4878) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vscalefps))] +pub unsafe fn _mm256_mask_scalef_ps(src: __m256, k: __mmask8, a: __m256, b: __m256) -> __m256 { + transmute(vscalefps256(a.as_f32x8(), b.as_f32x8(), src.as_f32x8(), k)) +} + +/// Scale the packed single-precision (32-bit) floating-point elements in a using values from b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_scalef_ps&expand=4879) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vscalefps))] +pub unsafe fn _mm256_maskz_scalef_ps(k: __mmask8, a: __m256, b: __m256) -> __m256 { + transmute(vscalefps256( + a.as_f32x8(), + b.as_f32x8(), + _mm256_setzero_ps().as_f32x8(), + k, + )) +} + +/// Scale the packed single-precision (32-bit) floating-point elements in a using values from b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_scalef_ps&expand=4877) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vscalefps))] +pub unsafe fn _mm_scalef_ps(a: __m128, b: __m128) -> __m128 { + transmute(vscalefps128( + a.as_f32x4(), + b.as_f32x4(), + _mm_setzero_ps().as_f32x4(), + 0b00001111, + )) +} + +/// Scale the packed single-precision (32-bit) floating-point elements in a using values from b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_scalef_ps&expand=4875) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vscalefps))] +pub unsafe fn _mm_mask_scalef_ps(src: __m128, k: __mmask8, a: __m128, b: __m128) -> __m128 { + transmute(vscalefps128(a.as_f32x4(), b.as_f32x4(), src.as_f32x4(), k)) +} + +/// Scale the packed single-precision (32-bit) floating-point elements in a using values from b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_scalef_ps&expand=4876) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vscalefps))] +pub unsafe fn _mm_maskz_scalef_ps(k: __mmask8, a: __m128, b: __m128) -> __m128 { + transmute(vscalefps128( + a.as_f32x4(), + b.as_f32x4(), + _mm_setzero_ps().as_f32x4(), + k, + )) +} + +/// Scale the packed double-precision (64-bit) floating-point elements in a using values from b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_scalef_pd&expand=4874) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefpd))] +pub unsafe fn _mm512_scalef_pd(a: __m512d, b: __m512d) -> __m512d { + transmute(vscalefpd( + a.as_f64x8(), + b.as_f64x8(), + _mm512_setzero_pd().as_f64x8(), + 0b11111111, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Scale the packed double-precision (64-bit) floating-point elements in a using values from b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_scalef_pd&expand=4872) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefpd))] +pub unsafe fn _mm512_mask_scalef_pd(src: __m512d, k: __mmask8, a: __m512d, b: __m512d) -> __m512d { + transmute(vscalefpd( + a.as_f64x8(), + b.as_f64x8(), + src.as_f64x8(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Scale the packed double-precision (64-bit) floating-point elements in a using values from b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_scalef_pd&expand=4873) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefpd))] +pub unsafe fn _mm512_maskz_scalef_pd(k: __mmask8, a: __m512d, b: __m512d) -> __m512d { + transmute(vscalefpd( + a.as_f64x8(), + b.as_f64x8(), + _mm512_setzero_pd().as_f64x8(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Scale the packed double-precision (64-bit) floating-point elements in a using values from b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_scalef_pd&expand=4871) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vscalefpd))] +pub unsafe fn _mm256_scalef_pd(a: __m256d, b: __m256d) -> __m256d { + transmute(vscalefpd256( + a.as_f64x4(), + b.as_f64x4(), + _mm256_setzero_pd().as_f64x4(), + 0b00001111, + )) +} + +/// Scale the packed double-precision (64-bit) floating-point elements in a using values from b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_scalef_pd&expand=4869) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vscalefpd))] +pub unsafe fn _mm256_mask_scalef_pd(src: __m256d, k: __mmask8, a: __m256d, b: __m256d) -> __m256d { + transmute(vscalefpd256(a.as_f64x4(), b.as_f64x4(), src.as_f64x4(), k)) +} + +/// Scale the packed double-precision (64-bit) floating-point elements in a using values from b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_scalef_pd&expand=4870) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vscalefpd))] +pub unsafe fn _mm256_maskz_scalef_pd(k: __mmask8, a: __m256d, b: __m256d) -> __m256d { + transmute(vscalefpd256( + a.as_f64x4(), + b.as_f64x4(), + _mm256_setzero_pd().as_f64x4(), + k, + )) +} + +/// Scale the packed double-precision (64-bit) floating-point elements in a using values from b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_scalef_pd&expand=4868) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vscalefpd))] +pub unsafe fn _mm_scalef_pd(a: __m128d, b: __m128d) -> __m128d { + transmute(vscalefpd128( + a.as_f64x2(), + b.as_f64x2(), + _mm_setzero_pd().as_f64x2(), + 0b00000011, + )) +} + +/// Scale the packed double-precision (64-bit) floating-point elements in a using values from b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_scalef_pd&expand=4866) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vscalefpd))] +pub unsafe fn _mm_mask_scalef_pd(src: __m128d, k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + transmute(vscalefpd128(a.as_f64x2(), b.as_f64x2(), src.as_f64x2(), k)) +} + +/// Scale the packed double-precision (64-bit) floating-point elements in a using values from b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_scalef_pd&expand=4867) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vscalefpd))] +pub unsafe fn _mm_maskz_scalef_pd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + transmute(vscalefpd128( + a.as_f64x2(), + b.as_f64x2(), + _mm_setzero_pd().as_f64x2(), + k, + )) +} + +/// Fix up packed single-precision (32-bit) floating-point elements in a and b using packed 32-bit integers in c, and store the results in dst. imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fixupimm_ps&expand=2499) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fixupimm_ps(a: __m512, b: __m512, c: __m512i) -> __m512 { + static_assert_imm8!(IMM8); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let c = c.as_i32x16(); + let r = vfixupimmps(a, b, c, IMM8, 0b11111111_11111111, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Fix up packed single-precision (32-bit) floating-point elements in a and b using packed 32-bit integers in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fixupimm_ps&expand=2500) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fixupimm_ps( + a: __m512, + k: __mmask16, + b: __m512, + c: __m512i, +) -> __m512 { + static_assert_imm8!(IMM8); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let c = c.as_i32x16(); + let r = vfixupimmps(a, b, c, IMM8, k, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Fix up packed single-precision (32-bit) floating-point elements in a and b using packed 32-bit integers in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fixupimm_ps&expand=2501) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fixupimm_ps( + k: __mmask16, + a: __m512, + b: __m512, + c: __m512i, +) -> __m512 { + static_assert_imm8!(IMM8); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let c = c.as_i32x16(); + let r = vfixupimmpsz(a, b, c, IMM8, k, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Fix up packed single-precision (32-bit) floating-point elements in a and b using packed 32-bit integers in c, and store the results in dst. imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_fixupimm_ps&expand=2496) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_fixupimm_ps(a: __m256, b: __m256, c: __m256i) -> __m256 { + static_assert_imm8!(IMM8); + let a = a.as_f32x8(); + let b = b.as_f32x8(); + let c = c.as_i32x8(); + let r = vfixupimmps256(a, b, c, IMM8, 0b11111111); + transmute(r) +} + +/// Fix up packed single-precision (32-bit) floating-point elements in a and b using packed 32-bit integers in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_fixupimm_ps&expand=2497) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_fixupimm_ps( + a: __m256, + k: __mmask8, + b: __m256, + c: __m256i, +) -> __m256 { + static_assert_imm8!(IMM8); + let a = a.as_f32x8(); + let b = b.as_f32x8(); + let c = c.as_i32x8(); + let r = vfixupimmps256(a, b, c, IMM8, k); + transmute(r) +} + +/// Fix up packed single-precision (32-bit) floating-point elements in a and b using packed 32-bit integers in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_fixupimm_ps&expand=2498) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_maskz_fixupimm_ps( + k: __mmask8, + a: __m256, + b: __m256, + c: __m256i, +) -> __m256 { + static_assert_imm8!(IMM8); + let a = a.as_f32x8(); + let b = b.as_f32x8(); + let c = c.as_i32x8(); + let r = vfixupimmpsz256(a, b, c, IMM8, k); + transmute(r) +} + +/// Fix up packed single-precision (32-bit) floating-point elements in a and b using packed 32-bit integers in c, and store the results in dst. imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_fixupimm_ps&expand=2493) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fixupimm_ps(a: __m128, b: __m128, c: __m128i) -> __m128 { + static_assert_imm8!(IMM8); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let c = c.as_i32x4(); + let r = vfixupimmps128(a, b, c, IMM8, 0b00001111); + transmute(r) +} + +/// Fix up packed single-precision (32-bit) floating-point elements in a and b using packed 32-bit integers in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_fixupimm_ps&expand=2494) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fixupimm_ps( + a: __m128, + k: __mmask8, + b: __m128, + c: __m128i, +) -> __m128 { + static_assert_imm8!(IMM8); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let c = c.as_i32x4(); + let r = vfixupimmps128(a, b, c, IMM8, k); + transmute(r) +} + +/// Fix up packed single-precision (32-bit) floating-point elements in a and b using packed 32-bit integers in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_fixupimm_ps&expand=2495) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fixupimm_ps( + k: __mmask8, + a: __m128, + b: __m128, + c: __m128i, +) -> __m128 { + static_assert_imm8!(IMM8); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let c = c.as_i32x4(); + let r = vfixupimmpsz128(a, b, c, IMM8, k); + transmute(r) +} + +/// Fix up packed double-precision (64-bit) floating-point elements in a and b using packed 64-bit integers in c, and store the results in dst. imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fixupimm_pd&expand=2490) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fixupimm_pd(a: __m512d, b: __m512d, c: __m512i) -> __m512d { + static_assert_imm8!(IMM8); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let c = c.as_i64x8(); + let r = vfixupimmpd(a, b, c, IMM8, 0b11111111, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Fix up packed double-precision (64-bit) floating-point elements in a and b using packed 64-bit integers in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fixupimm_pd&expand=2491) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fixupimm_pd( + a: __m512d, + k: __mmask8, + b: __m512d, + c: __m512i, +) -> __m512d { + static_assert_imm8!(IMM8); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let c = c.as_i64x8(); + let r = vfixupimmpd(a, b, c, IMM8, k, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Fix up packed double-precision (64-bit) floating-point elements in a and b using packed 64-bit integers in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fixupimm_pd&expand=2492) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fixupimm_pd( + k: __mmask8, + a: __m512d, + b: __m512d, + c: __m512i, +) -> __m512d { + static_assert_imm8!(IMM8); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let c = c.as_i64x8(); + let r = vfixupimmpdz(a, b, c, IMM8, k, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Fix up packed double-precision (64-bit) floating-point elements in a and b using packed 64-bit integers in c, and store the results in dst. imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_fixupimm_pd&expand=2487) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_fixupimm_pd(a: __m256d, b: __m256d, c: __m256i) -> __m256d { + static_assert_imm8!(IMM8); + let a = a.as_f64x4(); + let b = b.as_f64x4(); + let c = c.as_i64x4(); + let r = vfixupimmpd256(a, b, c, IMM8, 0b00001111); + transmute(r) +} + +/// Fix up packed double-precision (64-bit) floating-point elements in a and b using packed 64-bit integers in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_fixupimm_pd&expand=2488) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_fixupimm_pd( + a: __m256d, + k: __mmask8, + b: __m256d, + c: __m256i, +) -> __m256d { + static_assert_imm8!(IMM8); + let a = a.as_f64x4(); + let b = b.as_f64x4(); + let c = c.as_i64x4(); + let r = vfixupimmpd256(a, b, c, IMM8, k); + transmute(r) +} + +/// Fix up packed double-precision (64-bit) floating-point elements in a and b using packed 64-bit integers in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_fixupimm_pd&expand=2489) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_maskz_fixupimm_pd( + k: __mmask8, + a: __m256d, + b: __m256d, + c: __m256i, +) -> __m256d { + static_assert_imm8!(IMM8); + let a = a.as_f64x4(); + let b = b.as_f64x4(); + let c = c.as_i64x4(); + let r = vfixupimmpdz256(a, b, c, IMM8, k); + transmute(r) +} + +/// Fix up packed double-precision (64-bit) floating-point elements in a and b using packed 64-bit integers in c, and store the results in dst. imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_fixupimm_pd&expand=2484) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fixupimm_pd(a: __m128d, b: __m128d, c: __m128i) -> __m128d { + static_assert_imm8!(IMM8); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let c = c.as_i64x2(); + let r = vfixupimmpd128(a, b, c, IMM8, 0b00000011); + transmute(r) +} + +/// Fix up packed double-precision (64-bit) floating-point elements in a and b using packed 64-bit integers in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_fixupimm_pd&expand=2485) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fixupimm_pd( + a: __m128d, + k: __mmask8, + b: __m128d, + c: __m128i, +) -> __m128d { + static_assert_imm8!(IMM8); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let c = c.as_i64x2(); + let r = vfixupimmpd128(a, b, c, IMM8, k); + transmute(r) +} + +/// Fix up packed double-precision (64-bit) floating-point elements in a and b using packed 64-bit integers in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_fixupimm_pd&expand=2486) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fixupimm_pd( + k: __mmask8, + a: __m128d, + b: __m128d, + c: __m128i, +) -> __m128d { + static_assert_imm8!(IMM8); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let c = c.as_i64x2(); + let r = vfixupimmpdz128(a, b, c, IMM8, k); + transmute(r) +} + +/// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 32-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_ternarylogic_epi32&expand=5867) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpternlogd, IMM8 = 114))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_ternarylogic_epi32( + a: __m512i, + b: __m512i, + c: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_i32x16(); + let b = b.as_i32x16(); + let c = c.as_i32x16(); + let r = vpternlogd(a, b, c, IMM8); + transmute(r) +} + +/// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 32-bit integer, the corresponding bit from src, a, and b are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst using writemask k at 32-bit granularity (32-bit elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_ternarylogic_epi32&expand=5865) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpternlogd, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_ternarylogic_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let src = src.as_i32x16(); + let a = a.as_i32x16(); + let b = b.as_i32x16(); + let r = vpternlogd(src, a, b, IMM8); + transmute(simd_select_bitmask(k, r, src)) +} + +/// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 32-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst using zeromask k at 32-bit granularity (32-bit elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_ternarylogic_epi32&expand=5866) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpternlogd, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_ternarylogic_epi32( + k: __mmask16, + a: __m512i, + b: __m512i, + c: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_i32x16(); + let b = b.as_i32x16(); + let c = c.as_i32x16(); + let r = vpternlogd(a, b, c, IMM8); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 32-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_ternarylogic_epi32&expand=5864) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpternlogd, IMM8 = 114))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_ternarylogic_epi32( + a: __m256i, + b: __m256i, + c: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_i32x8(); + let b = b.as_i32x8(); + let c = c.as_i32x8(); + let r = vpternlogd256(a, b, c, IMM8); + transmute(r) +} + +/// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 32-bit integer, the corresponding bit from src, a, and b are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst using writemask k at 32-bit granularity (32-bit elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_ternarylogic_epi32&expand=5862) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpternlogd, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_ternarylogic_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let src = src.as_i32x8(); + let a = a.as_i32x8(); + let b = b.as_i32x8(); + let r = vpternlogd256(src, a, b, IMM8); + transmute(simd_select_bitmask(k, r, src)) +} + +/// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 32-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst using zeromask k at 32-bit granularity (32-bit elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_ternarylogic_epi32&expand=5863) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpternlogd, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_maskz_ternarylogic_epi32( + k: __mmask8, + a: __m256i, + b: __m256i, + c: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_i32x8(); + let b = b.as_i32x8(); + let c = c.as_i32x8(); + let r = vpternlogd256(a, b, c, IMM8); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 32-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_ternarylogic_epi32&expand=5861) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpternlogd, IMM8 = 114))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_ternarylogic_epi32( + a: __m128i, + b: __m128i, + c: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_i32x4(); + let b = b.as_i32x4(); + let c = c.as_i32x4(); + let r = vpternlogd128(a, b, c, IMM8); + transmute(r) +} + +/// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 32-bit integer, the corresponding bit from src, a, and b are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst using writemask k at 32-bit granularity (32-bit elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_ternarylogic_epi32&expand=5859) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpternlogd, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_ternarylogic_epi32( + src: __m128i, + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let src = src.as_i32x4(); + let a = a.as_i32x4(); + let b = b.as_i32x4(); + let r = vpternlogd128(src, a, b, IMM8); + transmute(simd_select_bitmask(k, r, src)) +} + +/// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 32-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst using zeromask k at 32-bit granularity (32-bit elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_ternarylogic_epi32&expand=5860) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpternlogd, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_ternarylogic_epi32( + k: __mmask8, + a: __m128i, + b: __m128i, + c: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_i32x4(); + let b = b.as_i32x4(); + let c = c.as_i32x4(); + let r = vpternlogd128(a, b, c, IMM8); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 64-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_ternarylogic_epi64&expand=5876) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpternlogq, IMM8 = 114))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_ternarylogic_epi64( + a: __m512i, + b: __m512i, + c: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_i64x8(); + let b = b.as_i64x8(); + let c = c.as_i64x8(); + let r = vpternlogq(a, b, c, IMM8); + transmute(r) +} + +/// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 64-bit integer, the corresponding bit from src, a, and b are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst using writemask k at 64-bit granularity (64-bit elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_ternarylogic_epi64&expand=5874) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpternlogq, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_ternarylogic_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let src = src.as_i64x8(); + let a = a.as_i64x8(); + let b = b.as_i64x8(); + let r = vpternlogq(src, a, b, IMM8); + transmute(simd_select_bitmask(k, r, src)) +} + +/// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 64-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst using zeromask k at 64-bit granularity (64-bit elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_ternarylogic_epi64&expand=5875) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpternlogq, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_ternarylogic_epi64( + k: __mmask8, + a: __m512i, + b: __m512i, + c: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_i64x8(); + let b = b.as_i64x8(); + let c = c.as_i64x8(); + let r = vpternlogq(a, b, c, IMM8); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 64-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_ternarylogic_epi64&expand=5873) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpternlogq, IMM8 = 114))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_ternarylogic_epi64( + a: __m256i, + b: __m256i, + c: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_i64x4(); + let b = b.as_i64x4(); + let c = c.as_i64x4(); + let r = vpternlogq256(a, b, c, IMM8); + transmute(r) +} + +/// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 64-bit integer, the corresponding bit from src, a, and b are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst using writemask k at 64-bit granularity (64-bit elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_ternarylogic_epi64&expand=5871) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpternlogq, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_ternarylogic_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let src = src.as_i64x4(); + let a = a.as_i64x4(); + let b = b.as_i64x4(); + let r = vpternlogq256(src, a, b, IMM8); + transmute(simd_select_bitmask(k, r, src)) +} + +/// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 64-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst using zeromask k at 64-bit granularity (64-bit elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_ternarylogic_epi64&expand=5872) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpternlogq, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_maskz_ternarylogic_epi64( + k: __mmask8, + a: __m256i, + b: __m256i, + c: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_i64x4(); + let b = b.as_i64x4(); + let c = c.as_i64x4(); + let r = vpternlogq256(a, b, c, IMM8); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 64-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_ternarylogic_epi64&expand=5870) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpternlogq, IMM8 = 114))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_ternarylogic_epi64( + a: __m128i, + b: __m128i, + c: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_i64x2(); + let b = b.as_i64x2(); + let c = c.as_i64x2(); + let r = vpternlogq128(a, b, c, IMM8); + transmute(r) +} + +/// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 64-bit integer, the corresponding bit from src, a, and b are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst using writemask k at 64-bit granularity (64-bit elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_ternarylogic_epi64&expand=5868) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpternlogq, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_ternarylogic_epi64( + src: __m128i, + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let src = src.as_i64x2(); + let a = a.as_i64x2(); + let b = b.as_i64x2(); + let r = vpternlogq128(src, a, b, IMM8); + transmute(simd_select_bitmask(k, r, src)) +} + +/// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 64-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst using zeromask k at 64-bit granularity (64-bit elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_ternarylogic_epi64&expand=5869) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpternlogq, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_ternarylogic_epi64( + k: __mmask8, + a: __m128i, + b: __m128i, + c: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_i64x2(); + let b = b.as_i64x2(); + let c = c.as_i64x2(); + let r = vpternlogq128(a, b, c, IMM8); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Normalize the mantissas of packed single-precision (32-bit) floating-point elements in a, and store the results in dst. This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign. +/// The mantissa is normalized to the interval specified by interv, which can take the following values: +/// _MM_MANT_NORM_1_2 // interval [1, 2) +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2) +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1) +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5) +/// The sign is determined by sc which can take the following values: +/// _MM_MANT_SIGN_src // sign = sign(src) +/// _MM_MANT_SIGN_zero // sign = 0 +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1 +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_getmant_ps&expand=2880) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(1, 2)] +pub unsafe fn _mm512_getmant_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + a: __m512, +) -> __m512 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + let r = vgetmantps( + a, + SIGN << 2 | NORM, + zero, + 0b11111111_11111111, + _MM_FROUND_CUR_DIRECTION, + ); + transmute(r) +} + +/// Normalize the mantissas of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1 +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_getmant_ps&expand=2881) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm512_mask_getmant_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + src: __m512, + k: __mmask16, + a: __m512, +) -> __m512 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f32x16(); + let src = src.as_f32x16(); + let r = vgetmantps(a, SIGN << 2 | NORM, src, k, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Normalize the mantissas of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1 +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_getmant_ps&expand=2882) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm512_maskz_getmant_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + k: __mmask16, + a: __m512, +) -> __m512 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + let r = vgetmantps(a, SIGN << 2 | NORM, zero, k, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Normalize the mantissas of packed single-precision (32-bit) floating-point elements in a, and store the results in dst. This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign. +/// The mantissa is normalized to the interval specified by interv, which can take the following values: +/// _MM_MANT_NORM_1_2 // interval [1, 2) +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2) +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1) +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5) +/// The sign is determined by sc which can take the following values: +/// _MM_MANT_SIGN_src // sign = sign(src) +/// _MM_MANT_SIGN_zero // sign = 0 +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1 +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_getmant_ps&expand=2877) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(1, 2)] +pub unsafe fn _mm256_getmant_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + a: __m256, +) -> __m256 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + let r = vgetmantps256(a, SIGN << 2 | NORM, zero, 0b11111111); + transmute(r) +} + +/// Normalize the mantissas of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1 +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_getmant_ps&expand=2878) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm256_mask_getmant_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + src: __m256, + k: __mmask8, + a: __m256, +) -> __m256 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f32x8(); + let src = src.as_f32x8(); + let r = vgetmantps256(a, SIGN << 2 | NORM, src, k); + transmute(r) +} + +/// Normalize the mantissas of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1 +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_getmant_ps&expand=2879) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm256_maskz_getmant_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + k: __mmask8, + a: __m256, +) -> __m256 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + let r = vgetmantps256(a, SIGN << 2 | NORM, zero, k); + transmute(r) +} + +/// Normalize the mantissas of packed single-precision (32-bit) floating-point elements in a, and store the results in dst. This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign. +/// The mantissa is normalized to the interval specified by interv, which can take the following values: +/// _MM_MANT_NORM_1_2 // interval [1, 2) +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2) +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1) +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5) +/// The sign is determined by sc which can take the following values: +/// _MM_MANT_SIGN_src // sign = sign(src) +/// _MM_MANT_SIGN_zero // sign = 0 +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1 +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_getmant_ps&expand=2874) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(1, 2)] +pub unsafe fn _mm_getmant_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + a: __m128, +) -> __m128 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vgetmantps128(a, SIGN << 2 | NORM, zero, 0b00001111); + transmute(r) +} + +/// Normalize the mantissas of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1 +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_getmant_ps&expand=2875) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm_mask_getmant_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + src: __m128, + k: __mmask8, + a: __m128, +) -> __m128 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f32x4(); + let src = src.as_f32x4(); + let r = vgetmantps128(a, SIGN << 2 | NORM, src, k); + transmute(r) +} + +/// Normalize the mantissas of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1 +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_getmant_ps&expand=2876) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm_maskz_getmant_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + k: __mmask8, + a: __m128, +) -> __m128 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vgetmantps128(a, SIGN << 2 | NORM, zero, k); + transmute(r) +} + +/// Normalize the mantissas of packed double-precision (64-bit) floating-point elements in a, and store the results in dst. This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1 +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_getmant_pd&expand=2871) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(1, 2)] +pub unsafe fn _mm512_getmant_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + a: __m512d, +) -> __m512d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + let r = vgetmantpd( + a, + SIGN << 2 | NORM, + zero, + 0b11111111, + _MM_FROUND_CUR_DIRECTION, + ); + transmute(r) +} + +/// Normalize the mantissas of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1 +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_getmant_pd&expand=2872) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm512_mask_getmant_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + src: __m512d, + k: __mmask8, + a: __m512d, +) -> __m512d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f64x8(); + let src = src.as_f64x8(); + let r = vgetmantpd(a, SIGN << 2 | NORM, src, k, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Normalize the mantissas of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1 +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_getmant_pd&expand=2873) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm512_maskz_getmant_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + k: __mmask8, + a: __m512d, +) -> __m512d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + let r = vgetmantpd(a, SIGN << 2 | NORM, zero, k, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Normalize the mantissas of packed double-precision (64-bit) floating-point elements in a, and store the results in dst. This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1 +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_getmant_pd&expand=2868) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(1, 2)] +pub unsafe fn _mm256_getmant_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + a: __m256d, +) -> __m256d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + let r = vgetmantpd256(a, SIGN << 2 | NORM, zero, 0b00001111); + transmute(r) +} + +/// Normalize the mantissas of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1 +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_getmant_pd&expand=2869) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm256_mask_getmant_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + src: __m256d, + k: __mmask8, + a: __m256d, +) -> __m256d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f64x4(); + let src = src.as_f64x4(); + let r = vgetmantpd256(a, SIGN << 2 | NORM, src, k); + transmute(r) +} + +/// Normalize the mantissas of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1 +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_getmant_pd&expand=2870) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm256_maskz_getmant_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + k: __mmask8, + a: __m256d, +) -> __m256d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + let r = vgetmantpd256(a, SIGN << 2 | NORM, zero, k); + transmute(r) +} + +/// Normalize the mantissas of packed double-precision (64-bit) floating-point elements in a, and store the results in dst. This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1 +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_getmant_pd&expand=2865) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(1, 2)] +pub unsafe fn _mm_getmant_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + a: __m128d, +) -> __m128d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vgetmantpd128(a, SIGN << 2 | NORM, zero, 0b00000011); + transmute(r) +} + +/// Normalize the mantissas of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1 +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_getmant_pd&expand=2866) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm_mask_getmant_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + src: __m128d, + k: __mmask8, + a: __m128d, +) -> __m128d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f64x2(); + let src = src.as_f64x2(); + let r = vgetmantpd128(a, SIGN << 2 | NORM, src, k); + transmute(r) +} + +/// Normalize the mantissas of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1 +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_getmant_pd&expand=2867) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm_maskz_getmant_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + k: __mmask8, + a: __m128d, +) -> __m128d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vgetmantpd128(a, SIGN << 2 | NORM, zero, k); + transmute(r) +} + +/// Add packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_add_round_ps&expand=145) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddps, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_add_round_ps(a: __m512, b: __m512) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vaddps(a, b, ROUNDING); + transmute(r) +} + +/// Add packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_add_round_ps&expand=146) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddps, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_add_round_ps( + src: __m512, + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vaddps(a, b, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f32x16())) +} + +/// Add packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_add_round_ps&expand=147) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddps, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_add_round_ps( + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vaddps(a, b, ROUNDING); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Add packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_add_round_pd&expand=142) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_add_round_pd(a: __m512d, b: __m512d) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vaddpd(a, b, ROUNDING); + transmute(r) +} + +/// Add packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_add_round_pd&expand=143) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_add_round_pd( + src: __m512d, + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vaddpd(a, b, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f64x8())) +} + +/// Add packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_add_round_pd&expand=144) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_add_round_pd( + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vaddpd(a, b, ROUNDING); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Subtract packed single-precision (32-bit) floating-point elements in b from packed single-precision (32-bit) floating-point elements in a, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sub_round_ps&expand=5739) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubps, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_sub_round_ps(a: __m512, b: __m512) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vsubps(a, b, ROUNDING); + transmute(r) +} + +/// Subtract packed single-precision (32-bit) floating-point elements in b from packed single-precision (32-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sub_round_ps&expand=5737) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubps, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_sub_round_ps( + src: __m512, + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vsubps(a, b, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f32x16())) +} + +/// Subtract packed single-precision (32-bit) floating-point elements in b from packed single-precision (32-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sub_round_ps&expand=5738) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubps, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_sub_round_ps( + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vsubps(a, b, ROUNDING); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Subtract packed double-precision (64-bit) floating-point elements in b from packed double-precision (64-bit) floating-point elements in a, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sub_round_pd&expand=5736) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_sub_round_pd(a: __m512d, b: __m512d) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vsubpd(a, b, ROUNDING); + transmute(r) +} + +/// Subtract packed double-precision (64-bit) floating-point elements in b from packed double-precision (64-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sub_round_pd&expand=5734) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_sub_round_pd( + src: __m512d, + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vsubpd(a, b, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f64x8())) +} + +/// Subtract packed double-precision (64-bit) floating-point elements in b from packed double-precision (64-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sub_round_pd&expand=5735) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_sub_round_pd( + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vsubpd(a, b, ROUNDING); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mul_round_ps&expand=3940) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulps, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_mul_round_ps(a: __m512, b: __m512) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vmulps(a, b, ROUNDING); + transmute(r) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_mul_round_ps&expand=3938) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulps, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_mul_round_ps( + src: __m512, + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vmulps(a, b, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f32x16())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_mul_round_ps&expand=3939) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulps, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_mul_round_ps( + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vmulps(a, b, ROUNDING); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mul_round_pd&expand=3937) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_mul_round_pd(a: __m512d, b: __m512d) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vmulpd(a, b, ROUNDING); + transmute(r) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_mul_round_pd&expand=3935) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_mul_round_pd( + src: __m512d, + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vmulpd(a, b, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f64x8())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_mul_round_ps&expand=3939) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_mul_round_pd( + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vmulpd(a, b, ROUNDING); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Divide packed single-precision (32-bit) floating-point elements in a by packed elements in b, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_div_round_ps&expand=2168) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivps, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_div_round_ps(a: __m512, b: __m512) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vdivps(a, b, ROUNDING); + transmute(r) +} + +/// Divide packed single-precision (32-bit) floating-point elements in a by packed elements in b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_div_round_ps&expand=2169) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivps, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_div_round_ps( + src: __m512, + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vdivps(a, b, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f32x16())) +} + +/// Divide packed single-precision (32-bit) floating-point elements in a by packed elements in b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_div_round_ps&expand=2170) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivps, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_div_round_ps( + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vdivps(a, b, ROUNDING); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Divide packed double-precision (64-bit) floating-point elements in a by packed elements in b, =and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_div_round_pd&expand=2165) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_div_round_pd(a: __m512d, b: __m512d) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vdivpd(a, b, ROUNDING); + transmute(r) +} + +/// Divide packed double-precision (64-bit) floating-point elements in a by packed elements in b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_div_round_pd&expand=2166) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_div_round_pd( + src: __m512d, + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vdivpd(a, b, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f64x8())) +} + +/// Divide packed double-precision (64-bit) floating-point elements in a by packed elements in b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_div_round_pd&expand=2167) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_div_round_pd( + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vdivpd(a, b, ROUNDING); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Compute the square root of packed single-precision (32-bit) floating-point elements in a, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sqrt_round_ps&expand=5377) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtps, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_sqrt_round_ps(a: __m512) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let r = vsqrtps(a, ROUNDING); + transmute(r) +} + +/// Compute the square root of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sqrt_round_ps&expand=5375) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtps, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_sqrt_round_ps( + src: __m512, + k: __mmask16, + a: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let r = vsqrtps(a, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f32x16())) +} + +/// Compute the square root of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sqrt_round_ps&expand=5376) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtps, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_sqrt_round_ps(k: __mmask16, a: __m512) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let r = vsqrtps(a, ROUNDING); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Compute the square root of packed double-precision (64-bit) floating-point elements in a, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sqrt_round_pd&expand=5374) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_sqrt_round_pd(a: __m512d) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let r = vsqrtpd(a, ROUNDING); + transmute(r) +} + +/// Compute the square root of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sqrt_round_pd&expand=5372) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_sqrt_round_pd( + src: __m512d, + k: __mmask8, + a: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let r = vsqrtpd(a, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f64x8())) +} + +/// Compute the square root of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sqrt_round_pd&expand=5373) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_sqrt_round_pd(k: __mmask8, a: __m512d) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let r = vsqrtpd(a, ROUNDING); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmadd_round_ps&expand=2565) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmadd132ps or vfmadd213ps or vfmadd231ps +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fmadd_round_ps( + a: __m512, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let c = c.as_f32x16(); + let r = vfmadd132psround(a, b, c, ROUNDING); + transmute(r) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmadd_round_ps&expand=2566) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmadd132ps or vfmadd213ps or vfmadd231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fmadd_round_ps( + a: __m512, + k: __mmask16, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let c = c.as_f32x16(); + let r = vfmadd132psround(a, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in a using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmadd_round_ps&expand=2568) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmadd132ps or vfmadd213ps or vfmadd231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fmadd_round_ps( + k: __mmask16, + a: __m512, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let c = c.as_f32x16(); + let r = vfmadd132psround(a, b, c, ROUNDING); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmadd_round_ps&expand=2567) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmadd132ps or vfmadd213ps or vfmadd231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fmadd_round_ps( + a: __m512, + b: __m512, + c: __m512, + k: __mmask16, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let c = c.as_f32x16(); + let r = vfmadd132psround(a, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmadd_round_pd&expand=2561) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmadd132pd or vfmadd213pd or vfmadd231pd +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fmadd_round_pd( + a: __m512d, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let c = c.as_f64x8(); + let r = vfmadd132pdround(a, b, c, ROUNDING); + transmute(r) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmadd_round_pd&expand=2562) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmadd132pd or vfmadd213pd or vfmadd231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fmadd_round_pd( + a: __m512d, + k: __mmask8, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let c = c.as_f64x8(); + let r = vfmadd132pdround(a, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmadd_round_pd&expand=2564) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmadd132pd or vfmadd213pd or vfmadd231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fmadd_round_pd( + k: __mmask8, + a: __m512d, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let c = c.as_f64x8(); + let r = vfmadd132pdround(a, b, c, ROUNDING); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmadd_round_pd&expand=2563) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmadd132pd or vfmadd213pd or vfmadd231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fmadd_round_pd( + a: __m512d, + b: __m512d, + c: __m512d, + k: __mmask8, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let c = c.as_f64x8(); + let r = vfmadd132pdround(a, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmsub_round_ps&expand=2651) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generates vfmadd, gcc generates vfmsub +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fmsub_round_ps( + a: __m512, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let zero: f32x16 = mem::zeroed(); + let sub = simd_sub(zero, c.as_f32x16()); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vfmadd132psround(a, b, sub, ROUNDING); + transmute(r) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmsub_round_ps&expand=2652) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generates vfmadd, gcc generates vfmsub +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fmsub_round_ps( + a: __m512, + k: __mmask16, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let zero: f32x16 = mem::zeroed(); + let sub = simd_sub(zero, c.as_f32x16()); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vfmadd132psround(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmsub_round_ps&expand=2654) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generates vfmadd, gcc generates vfmsub +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fmsub_round_ps( + k: __mmask16, + a: __m512, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let zero: f32x16 = mem::zeroed(); + let sub = simd_sub(zero, c.as_f32x16()); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vfmadd132psround(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmsub_round_ps&expand=2653) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generates vfmadd, gcc generates vfmsub +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fmsub_round_ps( + a: __m512, + b: __m512, + c: __m512, + k: __mmask16, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let zero: f32x16 = mem::zeroed(); + let c = c.as_f32x16(); + let sub = simd_sub(zero, c); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vfmadd132psround(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmsub_round_pd&expand=2647) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang generates fmadd, gcc generates fmsub +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fmsub_round_pd( + a: __m512d, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let zero: f64x8 = mem::zeroed(); + let sub = simd_sub(zero, c.as_f64x8()); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vfmadd132pdround(a, b, sub, ROUNDING); + transmute(r) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmsub_round_pd&expand=2648) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang generates fmadd, gcc generates fmsub +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fmsub_round_pd( + a: __m512d, + k: __mmask8, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let zero: f64x8 = mem::zeroed(); + let sub = simd_sub(zero, c.as_f64x8()); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vfmadd132pdround(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmsub_round_pd&expand=2650) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang generates fmadd, gcc generates fmsub +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fmsub_round_pd( + k: __mmask8, + a: __m512d, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let zero: f64x8 = mem::zeroed(); + let sub = simd_sub(zero, c.as_f64x8()); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vfmadd132pdround(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmsub_round_pd&expand=2649) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang generates fmadd, gcc generates fmsub +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fmsub_round_pd( + a: __m512d, + b: __m512d, + c: __m512d, + k: __mmask8, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let zero: f64x8 = mem::zeroed(); + let c = c.as_f64x8(); + let sub = simd_sub(zero, c); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vfmadd132pdround(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmaddsub_round_ps&expand=2619) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fmaddsub_round_ps( + a: __m512, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let c = c.as_f32x16(); + let r = vfmaddsub213ps(a, b, c, ROUNDING); + transmute(r) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmaddsub_round_ps&expand=2620) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fmaddsub_round_ps( + a: __m512, + k: __mmask16, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let c = c.as_f32x16(); + let r = vfmaddsub213ps(a, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmaddsub_round_ps&expand=2622) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fmaddsub_round_ps( + k: __mmask16, + a: __m512, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let c = c.as_f32x16(); + let r = vfmaddsub213ps(a, b, c, ROUNDING); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmaddsub_round_ps&expand=2621) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fmaddsub_round_ps( + a: __m512, + b: __m512, + c: __m512, + k: __mmask16, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let c = c.as_f32x16(); + let r = vfmaddsub213ps(a, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmaddsub_round_pd&expand=2615) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fmaddsub_round_pd( + a: __m512d, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let c = c.as_f64x8(); + let r = vfmaddsub213pd(a, b, c, ROUNDING); + transmute(r) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmaddsub_round_pd&expand=2616) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fmaddsub_round_pd( + a: __m512d, + k: __mmask8, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let c = c.as_f64x8(); + let r = vfmaddsub213pd(a, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmaddsub_round_pd&expand=2618) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fmaddsub_round_pd( + k: __mmask8, + a: __m512d, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let c = c.as_f64x8(); + let r = vfmaddsub213pd(a, b, c, ROUNDING); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmaddsub_round_pd&expand=2617) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fmaddsub_round_pd( + a: __m512d, + b: __m512d, + c: __m512d, + k: __mmask8, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let c = c.as_f64x8(); + let r = vfmaddsub213pd(a, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmsubadd_round_ps&expand=2699) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fmsubadd_round_ps( + a: __m512, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let zero: f32x16 = mem::zeroed(); + let sub = simd_sub(zero, c.as_f32x16()); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vfmaddsub213ps(a, b, sub, ROUNDING); + transmute(r) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmsubadd_round_ps&expand=2700) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fmsubadd_round_ps( + a: __m512, + k: __mmask16, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let zero: f32x16 = mem::zeroed(); + let sub = simd_sub(zero, c.as_f32x16()); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vfmaddsub213ps(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmsubadd_round_ps&expand=2702) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fmsubadd_round_ps( + k: __mmask16, + a: __m512, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let zero: f32x16 = mem::zeroed(); + let sub = simd_sub(zero, c.as_f32x16()); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vfmaddsub213ps(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmsubadd_round_ps&expand=2701) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fmsubadd_round_ps( + a: __m512, + b: __m512, + c: __m512, + k: __mmask16, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let zero: f32x16 = mem::zeroed(); + let c = c.as_f32x16(); + let sub = simd_sub(zero, c); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vfmaddsub213ps(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmsubadd_round_pd&expand=2695) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fmsubadd_round_pd( + a: __m512d, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let zero: f64x8 = mem::zeroed(); + let sub = simd_sub(zero, c.as_f64x8()); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vfmaddsub213pd(a, b, sub, ROUNDING); + transmute(r) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmsubadd_round_pd&expand=2696) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fmsubadd_round_pd( + a: __m512d, + k: __mmask8, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let zero: f64x8 = mem::zeroed(); + let sub = simd_sub(zero, c.as_f64x8()); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vfmaddsub213pd(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmsubadd_round_pd&expand=2698) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fmsubadd_round_pd( + k: __mmask8, + a: __m512d, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let zero: f64x8 = mem::zeroed(); + let sub = simd_sub(zero, c.as_f64x8()); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vfmaddsub213pd(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmsubadd_round_pd&expand=2697) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fmsubadd_round_pd( + a: __m512d, + b: __m512d, + c: __m512d, + k: __mmask8, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let zero: f64x8 = mem::zeroed(); + let c = c.as_f64x8(); + let sub = simd_sub(zero, c); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vfmaddsub213pd(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fnmadd_round_ps&expand=2731) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fnmadd_round_ps( + a: __m512, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let zero: f32x16 = mem::zeroed(); + let sub = simd_sub(zero, a.as_f32x16()); + let b = b.as_f32x16(); + let c = c.as_f32x16(); + let r = vfmadd132psround(sub, b, c, ROUNDING); + transmute(r) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fnmadd_round_ps&expand=2732) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fnmadd_round_ps( + a: __m512, + k: __mmask16, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let zero: f32x16 = mem::zeroed(); + let sub = simd_sub(zero, a.as_f32x16()); + let b = b.as_f32x16(); + let c = c.as_f32x16(); + let r = vfmadd132psround(sub, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, a.as_f32x16())) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fnmadd_round_ps&expand=2734) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fnmadd_round_ps( + k: __mmask16, + a: __m512, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let zero: f32x16 = mem::zeroed(); + let sub = simd_sub(zero, a.as_f32x16()); + let b = b.as_f32x16(); + let c = c.as_f32x16(); + let r = vfmadd132psround(sub, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fnmadd_round_ps&expand=2733) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fnmadd_round_ps( + a: __m512, + b: __m512, + c: __m512, + k: __mmask16, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let zero: f32x16 = mem::zeroed(); + let sub = simd_sub(zero, a.as_f32x16()); + let b = b.as_f32x16(); + let c = c.as_f32x16(); + let r = vfmadd132psround(sub, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fnmadd_pd&expand=2711) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fnmadd_round_pd( + a: __m512d, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let zero: f64x8 = mem::zeroed(); + let sub = simd_sub(zero, a.as_f64x8()); + let b = b.as_f64x8(); + let c = c.as_f64x8(); + let r = vfmadd132pdround(sub, b, c, ROUNDING); + transmute(r) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fnmadd_round_pd&expand=2728) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fnmadd_round_pd( + a: __m512d, + k: __mmask8, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let zero: f64x8 = mem::zeroed(); + let a = a.as_f64x8(); + let sub = simd_sub(zero, a); + let b = b.as_f64x8(); + let c = c.as_f64x8(); + let r = vfmadd132pdround(sub, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fnmadd_round_pd&expand=2730) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fnmadd_round_pd( + k: __mmask8, + a: __m512d, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let zero: f64x8 = mem::zeroed(); + let sub = simd_sub(zero, a.as_f64x8()); + let b = b.as_f64x8(); + let c = c.as_f64x8(); + let r = vfmadd132pdround(sub, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fnmadd_round_pd&expand=2729) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fnmadd_round_pd( + a: __m512d, + b: __m512d, + c: __m512d, + k: __mmask8, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let zero: f64x8 = mem::zeroed(); + let sub = simd_sub(zero, a.as_f64x8()); + let b = b.as_f64x8(); + let c = c.as_f64x8(); + let r = vfmadd132pdround(sub, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fnmsub_round_ps&expand=2779) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fnmsub_round_ps( + a: __m512, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let zero: f32x16 = mem::zeroed(); + let suba = simd_sub(zero, a.as_f32x16()); + let subc = simd_sub(zero, c.as_f32x16()); + let b = b.as_f32x16(); + let r = vfmadd132psround(suba, b, subc, ROUNDING); + transmute(r) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fnmsub_round_ps&expand=2780) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fnmsub_round_ps( + a: __m512, + k: __mmask16, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let zero: f32x16 = mem::zeroed(); + let a = a.as_f32x16(); + let suba = simd_sub(zero, a); + let subc = simd_sub(zero, c.as_f32x16()); + let b = b.as_f32x16(); + let r = vfmadd132psround(suba, b, subc, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fnmsub_round_ps&expand=2782) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fnmsub_round_ps( + k: __mmask16, + a: __m512, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let zero: f32x16 = mem::zeroed(); + let suba = simd_sub(zero, a.as_f32x16()); + let subc = simd_sub(zero, c.as_f32x16()); + let b = b.as_f32x16(); + let r = vfmadd132psround(suba, b, subc, ROUNDING); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fnmsub_round_ps&expand=2781) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fnmsub_round_ps( + a: __m512, + b: __m512, + c: __m512, + k: __mmask16, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let zero: f32x16 = mem::zeroed(); + let suba = simd_sub(zero, a.as_f32x16()); + let c = c.as_f32x16(); + let subc = simd_sub(zero, c); + let b = b.as_f32x16(); + let r = vfmadd132psround(suba, b, subc, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fnmsub_round_pd&expand=2775) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fnmsub_round_pd( + a: __m512d, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let zero: f64x8 = mem::zeroed(); + let suba = simd_sub(zero, a.as_f64x8()); + let subc = simd_sub(zero, c.as_f64x8()); + let b = b.as_f64x8(); + let r = vfmadd132pdround(suba, b, subc, ROUNDING); + transmute(r) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fnmsub_round_pd&expand=2776) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fnmsub_round_pd( + a: __m512d, + k: __mmask8, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let zero: f64x8 = mem::zeroed(); + let a = a.as_f64x8(); + let suba = simd_sub(zero, a); + let subc = simd_sub(zero, c.as_f64x8()); + let b = b.as_f64x8(); + let r = vfmadd132pdround(suba, b, subc, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fnmsub_round_pd&expand=2778) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fnmsub_round_pd( + k: __mmask8, + a: __m512d, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let zero: f64x8 = mem::zeroed(); + let suba = simd_sub(zero, a.as_f64x8()); + let subc = simd_sub(zero, c.as_f64x8()); + let b = b.as_f64x8(); + let r = vfmadd132pdround(suba, b, subc, ROUNDING); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fnmsub_round_pd&expand=2777) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fnmsub_round_pd( + a: __m512d, + b: __m512d, + c: __m512d, + k: __mmask8, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let zero: f64x8 = mem::zeroed(); + let suba = simd_sub(zero, a.as_f64x8()); + let c = c.as_f64x8(); + let subc = simd_sub(zero, c); + let b = b.as_f64x8(); + let r = vfmadd132pdround(suba, b, subc, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed maximum values in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_max_round_ps&expand=3662) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxps, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_max_round_ps(a: __m512, b: __m512) -> __m512 { + static_assert_sae!(SAE); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vmaxps(a, b, SAE); + transmute(r) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_max_round_ps&expand=3660) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxps, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_max_round_ps( + src: __m512, + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_sae!(SAE); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vmaxps(a, b, SAE); + transmute(simd_select_bitmask(k, r, src.as_f32x16())) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_max_round_ps&expand=3661) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxps, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_max_round_ps( + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_sae!(SAE); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vmaxps(a, b, SAE); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed maximum values in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_max_round_pd&expand=3659) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxpd, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_max_round_pd(a: __m512d, b: __m512d) -> __m512d { + static_assert_sae!(SAE); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vmaxpd(a, b, SAE); + transmute(r) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed maximum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_max_round_pd&expand=3657) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxpd, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_max_round_pd( + src: __m512d, + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + static_assert_sae!(SAE); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vmaxpd(a, b, SAE); + transmute(simd_select_bitmask(k, r, src.as_f64x8())) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_max_round_pd&expand=3658) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxpd, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_max_round_pd( + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + static_assert_sae!(SAE); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vmaxpd(a, b, SAE); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed minimum values in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_min_round_ps&expand=3776) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminps, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_min_round_ps(a: __m512, b: __m512) -> __m512 { + static_assert_sae!(SAE); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vminps(a, b, SAE); + transmute(r) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_min_round_ps&expand=3774) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminps, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_min_round_ps( + src: __m512, + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_sae!(SAE); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vminps(a, b, SAE); + transmute(simd_select_bitmask(k, r, src.as_f32x16())) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_min_round_ps&expand=3775) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminps, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_min_round_ps( + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_sae!(SAE); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vminps(a, b, SAE); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed minimum values in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_min_round_pd&expand=3773) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminpd, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_min_round_pd(a: __m512d, b: __m512d) -> __m512d { + static_assert_sae!(SAE); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vminpd(a, b, SAE); + transmute(r) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed minimum values in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_min_round_pd&expand=3771) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminpd, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_min_round_pd( + src: __m512d, + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + static_assert_sae!(SAE); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vminpd(a, b, SAE); + transmute(simd_select_bitmask(k, r, src.as_f64x8())) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_min_round_pd&expand=3772) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminpd, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_min_round_pd( + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + static_assert_sae!(SAE); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vminpd(a, b, SAE); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Convert the exponent of each packed single-precision (32-bit) floating-point element in a to a single-precision (32-bit) floating-point number representing the integer exponent, and store the results in dst. This intrinsic essentially calculates floor(log2(x)) for each element.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_getexp_round_ps&expand=2850) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexpps, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_getexp_round_ps(a: __m512) -> __m512 { + static_assert_sae!(SAE); + let a = a.as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + let r = vgetexpps(a, zero, 0b11111111_11111111, SAE); + transmute(r) +} + +/// Convert the exponent of each packed single-precision (32-bit) floating-point element in a to a single-precision (32-bit) floating-point number representing the integer exponent, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). This intrinsic essentially calculates floor(log2(x)) for each element.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_getexp_round_ps&expand=2851) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexpps, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_getexp_round_ps( + src: __m512, + k: __mmask16, + a: __m512, +) -> __m512 { + static_assert_sae!(SAE); + let a = a.as_f32x16(); + let src = src.as_f32x16(); + let r = vgetexpps(a, src, k, SAE); + transmute(r) +} + +/// Convert the exponent of each packed single-precision (32-bit) floating-point element in a to a single-precision (32-bit) floating-point number representing the integer exponent, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). This intrinsic essentially calculates floor(log2(x)) for each element.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_getexp_round_ps&expand=2852) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexpps, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_getexp_round_ps(k: __mmask16, a: __m512) -> __m512 { + static_assert_sae!(SAE); + let a = a.as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + let r = vgetexpps(a, zero, k, SAE); + transmute(r) +} + +/// Convert the exponent of each packed double-precision (64-bit) floating-point element in a to a double-precision (64-bit) floating-point number representing the integer exponent, and store the results in dst. This intrinsic essentially calculates floor(log2(x)) for each element.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_getexp_round_pd&expand=2847) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexppd, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_getexp_round_pd(a: __m512d) -> __m512d { + static_assert_sae!(SAE); + let a = a.as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + let r = vgetexppd(a, zero, 0b11111111, SAE); + transmute(r) +} + +/// Convert the exponent of each packed double-precision (64-bit) floating-point element in a to a double-precision (64-bit) floating-point number representing the integer exponent, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). This intrinsic essentially calculates floor(log2(x)) for each element.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_getexp_round_pd&expand=2848) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexppd, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_getexp_round_pd( + src: __m512d, + k: __mmask8, + a: __m512d, +) -> __m512d { + static_assert_sae!(SAE); + let a = a.as_f64x8(); + let src = src.as_f64x8(); + let r = vgetexppd(a, src, k, SAE); + transmute(r) +} + +/// Convert the exponent of each packed double-precision (64-bit) floating-point element in a to a double-precision (64-bit) floating-point number representing the integer exponent, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). This intrinsic essentially calculates floor(log2(x)) for each element.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_getexp_round_pd&expand=2849) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexppd, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_getexp_round_pd(k: __mmask8, a: __m512d) -> __m512d { + static_assert_sae!(SAE); + let a = a.as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + let r = vgetexppd(a, zero, k, SAE); + transmute(r) +} + +/// Round packed single-precision (32-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst.\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE\ +/// +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_roundscale_round_ps&expand=4790) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(1, 2)] +pub unsafe fn _mm512_roundscale_round_ps(a: __m512) -> __m512 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + let r = vrndscaleps(a, IMM8, zero, 0b11111111_11111111, SAE); + transmute(r) +} + +/// Round packed single-precision (32-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE\ +/// +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_roundscale_round_ps&expand=4788) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm512_mask_roundscale_round_ps( + src: __m512, + k: __mmask16, + a: __m512, +) -> __m512 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f32x16(); + let src = src.as_f32x16(); + let r = vrndscaleps(a, IMM8, src, k, SAE); + transmute(r) +} + +/// Round packed single-precision (32-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE\ +/// +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_roundscale_round_ps&expand=4789) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm512_maskz_roundscale_round_ps( + k: __mmask16, + a: __m512, +) -> __m512 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + let r = vrndscaleps(a, IMM8, zero, k, SAE); + transmute(r) +} + +/// Round packed double-precision (64-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst.\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE\ +/// +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_roundscale_round_pd&expand=4787) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(1, 2)] +pub unsafe fn _mm512_roundscale_round_pd(a: __m512d) -> __m512d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + let r = vrndscalepd(a, IMM8, zero, 0b11111111, SAE); + transmute(r) +} + +/// Round packed double-precision (64-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE\ +/// +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_roundscale_round_pd&expand=4785) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm512_mask_roundscale_round_pd( + src: __m512d, + k: __mmask8, + a: __m512d, +) -> __m512d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f64x8(); + let src = src.as_f64x8(); + let r = vrndscalepd(a, IMM8, src, k, SAE); + transmute(r) +} + +/// Round packed double-precision (64-bit) floating-point elements in a to the number of fraction bits specified by imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE\ +/// +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_roundscale_round_pd&expand=4786) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm512_maskz_roundscale_round_pd( + k: __mmask8, + a: __m512d, +) -> __m512d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + let r = vrndscalepd(a, IMM8, zero, k, SAE); + transmute(r) +} + +/// Scale the packed single-precision (32-bit) floating-point elements in a using values from b, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_scalef_round_ps&expand=4889) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefps, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_scalef_round_ps(a: __m512, b: __m512) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + let r = vscalefps(a, b, zero, 0b11111111_11111111, ROUNDING); + transmute(r) +} + +/// Scale the packed single-precision (32-bit) floating-point elements in a using values from b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_scalef_round_ps&expand=4887) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefps, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_scalef_round_ps( + src: __m512, + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let src = src.as_f32x16(); + let r = vscalefps(a, b, src, k, ROUNDING); + transmute(r) +} + +/// Scale the packed single-precision (32-bit) floating-point elements in a using values from b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_scalef_round_ps&expand=4888) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefps, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_scalef_round_ps( + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + let r = vscalefps(a, b, zero, k, ROUNDING); + transmute(r) +} + +/// Scale the packed double-precision (64-bit) floating-point elements in a using values from b, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_scalef_round_pd&expand=4886) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_scalef_round_pd(a: __m512d, b: __m512d) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + let r = vscalefpd(a, b, zero, 0b11111111, ROUNDING); + transmute(r) +} + +/// Scale the packed double-precision (64-bit) floating-point elements in a using values from b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_scalef_round_pd&expand=4884) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_scalef_round_pd( + src: __m512d, + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let src = src.as_f64x8(); + let r = vscalefpd(a, b, src, k, ROUNDING); + transmute(r) +} + +/// Scale the packed double-precision (64-bit) floating-point elements in a using values from b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_scalef_round_pd&expand=4885) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_scalef_round_pd( + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + let r = vscalefpd(a, b, zero, k, ROUNDING); + transmute(r) +} + +/// Fix up packed single-precision (32-bit) floating-point elements in a and b using packed 32-bit integers in c, and store the results in dst. imm8 is used to set the required flags reporting.\ +/// +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fixupimm_round_ps&expand=2505) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm512_fixupimm_round_ps( + a: __m512, + b: __m512, + c: __m512i, +) -> __m512 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let c = c.as_i32x16(); + let r = vfixupimmps(a, b, c, IMM8, 0b11111111_11111111, SAE); + transmute(r) +} + +/// Fix up packed single-precision (32-bit) floating-point elements in a and b using packed 32-bit integers in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). imm8 is used to set the required flags reporting.\ +/// +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fixupimm_round_ps&expand=2506) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm512_mask_fixupimm_round_ps( + a: __m512, + k: __mmask16, + b: __m512, + c: __m512i, +) -> __m512 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let c = c.as_i32x16(); + let r = vfixupimmps(a, b, c, IMM8, k, SAE); + transmute(r) +} + +/// Fix up packed single-precision (32-bit) floating-point elements in a and b using packed 32-bit integers in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). imm8 is used to set the required flags reporting.\ +/// +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fixupimm_round_ps&expand=2507) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm512_maskz_fixupimm_round_ps( + k: __mmask16, + a: __m512, + b: __m512, + c: __m512i, +) -> __m512 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let c = c.as_i32x16(); + let r = vfixupimmpsz(a, b, c, IMM8, k, SAE); + transmute(r) +} + +/// Fix up packed double-precision (64-bit) floating-point elements in a and b using packed 64-bit integers in c, and store the results in dst. imm8 is used to set the required flags reporting.\ +/// +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fixupimm_round_pd&expand=2502) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm512_fixupimm_round_pd( + a: __m512d, + b: __m512d, + c: __m512i, +) -> __m512d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let c = c.as_i64x8(); + let r = vfixupimmpd(a, b, c, IMM8, 0b11111111, SAE); + transmute(r) +} + +/// Fix up packed double-precision (64-bit) floating-point elements in a and b using packed 64-bit integers in c, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). imm8 is used to set the required flags reporting.\ +/// +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fixupimm_round_pd&expand=2503) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm512_mask_fixupimm_round_pd( + a: __m512d, + k: __mmask8, + b: __m512d, + c: __m512i, +) -> __m512d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let c = c.as_i64x8(); + let r = vfixupimmpd(a, b, c, IMM8, k, SAE); + transmute(r) +} + +/// Fix up packed double-precision (64-bit) floating-point elements in a and b using packed 64-bit integers in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). imm8 is used to set the required flags reporting.\ +/// +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fixupimm_round_pd&expand=2504) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm512_maskz_fixupimm_round_pd( + k: __mmask8, + a: __m512d, + b: __m512d, + c: __m512i, +) -> __m512d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let c = c.as_i64x8(); + let r = vfixupimmpdz(a, b, c, IMM8, k, SAE); + transmute(r) +} + +/// Normalize the mantissas of packed single-precision (32-bit) floating-point elements in a, and store the results in dst. This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_getmant_round_ps&expand=2886) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(1, 2, 3)] +pub unsafe fn _mm512_getmant_round_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( + a: __m512, +) -> __m512 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); + let a = a.as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + let r = vgetmantps(a, SIGN << 2 | NORM, zero, 0b11111111_11111111, SAE); + transmute(r) +} + +/// Normalize the mantissas of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_getmant_round_ps&expand=2887) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(3, 4, 5)] +pub unsafe fn _mm512_mask_getmant_round_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( + src: __m512, + k: __mmask16, + a: __m512, +) -> __m512 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); + let a = a.as_f32x16(); + let src = src.as_f32x16(); + let r = vgetmantps(a, SIGN << 2 | NORM, src, k, SAE); + transmute(r) +} + +/// Normalize the mantissas of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_getmant_round_ps&expand=2888) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(2, 3, 4)] +pub unsafe fn _mm512_maskz_getmant_round_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( + k: __mmask16, + a: __m512, +) -> __m512 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); + let a = a.as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + let r = vgetmantps(a, SIGN << 2 | NORM, zero, k, SAE); + transmute(r) +} + +/// Normalize the mantissas of packed double-precision (64-bit) floating-point elements in a, and store the results in dst. This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_getmant_round_pd&expand=2883) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(1, 2, 3)] +pub unsafe fn _mm512_getmant_round_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( + a: __m512d, +) -> __m512d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); + let a = a.as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + let r = vgetmantpd(a, SIGN << 2 | NORM, zero, 0b11111111, SAE); + transmute(r) +} + +/// Normalize the mantissas of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_getmant_round_pd&expand=2884) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(3, 4, 5)] +pub unsafe fn _mm512_mask_getmant_round_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( + src: __m512d, + k: __mmask8, + a: __m512d, +) -> __m512d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); + let a = a.as_f64x8(); + let src = src.as_f64x8(); + let r = vgetmantpd(a, SIGN << 2 | NORM, src, k, SAE); + transmute(r) +} + +/// Normalize the mantissas of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_512_maskz_getmant_round_pd&expand=2885) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(2, 3, 4)] +pub unsafe fn _mm512_maskz_getmant_round_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( + k: __mmask8, + a: __m512d, +) -> __m512d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); + let a = a.as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + let r = vgetmantpd(a, SIGN << 2 | NORM, zero, k, SAE); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed 32-bit integers, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtps_epi32&expand=1737) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2dq))] +pub unsafe fn _mm512_cvtps_epi32(a: __m512) -> __m512i { + transmute(vcvtps2dq( + a.as_f32x16(), + _mm512_setzero_si512().as_i32x16(), + 0b11111111_11111111, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtps_epi32&expand=1738) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2dq))] +pub unsafe fn _mm512_mask_cvtps_epi32(src: __m512i, k: __mmask16, a: __m512) -> __m512i { + transmute(vcvtps2dq( + a.as_f32x16(), + src.as_i32x16(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtps_epi32&expand=1739) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2dq))] +pub unsafe fn _mm512_maskz_cvtps_epi32(k: __mmask16, a: __m512) -> __m512i { + transmute(vcvtps2dq( + a.as_f32x16(), + _mm512_setzero_si512().as_i32x16(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtps_epi32&expand=1735) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtps2dq))] +pub unsafe fn _mm256_mask_cvtps_epi32(src: __m256i, k: __mmask8, a: __m256) -> __m256i { + let convert = _mm256_cvtps_epi32(a); + transmute(simd_select_bitmask(k, convert.as_i32x8(), src.as_i32x8())) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtps_epi32&expand=1736) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtps2dq))] +pub unsafe fn _mm256_maskz_cvtps_epi32(k: __mmask8, a: __m256) -> __m256i { + let convert = _mm256_cvtps_epi32(a); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, convert.as_i32x8(), zero)) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtps_epi32&expand=1732) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtps2dq))] +pub unsafe fn _mm_mask_cvtps_epi32(src: __m128i, k: __mmask8, a: __m128) -> __m128i { + let convert = _mm_cvtps_epi32(a); + transmute(simd_select_bitmask(k, convert.as_i32x4(), src.as_i32x4())) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtps_epi32&expand=1733) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtps2dq))] +pub unsafe fn _mm_maskz_cvtps_epi32(k: __mmask8, a: __m128) -> __m128i { + let convert = _mm_cvtps_epi32(a); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, convert.as_i32x4(), zero)) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtps_epu32&expand=1755) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2udq))] +pub unsafe fn _mm512_cvtps_epu32(a: __m512) -> __m512i { + transmute(vcvtps2udq( + a.as_f32x16(), + _mm512_setzero_si512().as_u32x16(), + 0b11111111_11111111, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtps_epu32&expand=1756) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2udq))] +pub unsafe fn _mm512_mask_cvtps_epu32(src: __m512i, k: __mmask16, a: __m512) -> __m512i { + transmute(vcvtps2udq( + a.as_f32x16(), + src.as_u32x16(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvt_roundps_epu32&expand=1343) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2udq))] +pub unsafe fn _mm512_maskz_cvtps_epu32(k: __mmask16, a: __m512) -> __m512i { + transmute(vcvtps2udq( + a.as_f32x16(), + _mm512_setzero_si512().as_u32x16(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvtps_epu32&expand=1752) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtps2udq))] +pub unsafe fn _mm256_cvtps_epu32(a: __m256) -> __m256i { + transmute(vcvtps2udq256( + a.as_f32x8(), + _mm256_setzero_si256().as_u32x8(), + 0b11111111, + )) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtps_epu32&expand=1753) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtps2udq))] +pub unsafe fn _mm256_mask_cvtps_epu32(src: __m256i, k: __mmask8, a: __m256) -> __m256i { + transmute(vcvtps2udq256(a.as_f32x8(), src.as_u32x8(), k)) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtps_epu32&expand=1754) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtps2udq))] +pub unsafe fn _mm256_maskz_cvtps_epu32(k: __mmask8, a: __m256) -> __m256i { + transmute(vcvtps2udq256( + a.as_f32x8(), + _mm256_setzero_si256().as_u32x8(), + k, + )) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtps_epu32&expand=1749) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtps2udq))] +pub unsafe fn _mm_cvtps_epu32(a: __m128) -> __m128i { + transmute(vcvtps2udq128( + a.as_f32x4(), + _mm_setzero_si128().as_u32x4(), + 0b11111111, + )) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtps_epu32&expand=1750) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtps2udq))] +pub unsafe fn _mm_mask_cvtps_epu32(src: __m128i, k: __mmask8, a: __m128) -> __m128i { + transmute(vcvtps2udq128(a.as_f32x4(), src.as_u32x4(), k)) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtps_epu32&expand=1751) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtps2udq))] +pub unsafe fn _mm_maskz_cvtps_epu32(k: __mmask8, a: __m128) -> __m128i { + transmute(vcvtps2udq128( + a.as_f32x4(), + _mm_setzero_si128().as_u32x4(), + k, + )) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed double-precision (64-bit) floating-point elements, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtps_pd&expand=1769) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2pd))] +pub unsafe fn _mm512_cvtps_pd(a: __m256) -> __m512d { + transmute(vcvtps2pd( + a.as_f32x8(), + _mm512_setzero_pd().as_f64x8(), + 0b11111111, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed double-precision (64-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtps_pd&expand=1770) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2pd))] +pub unsafe fn _mm512_mask_cvtps_pd(src: __m512d, k: __mmask8, a: __m256) -> __m512d { + transmute(vcvtps2pd( + a.as_f32x8(), + src.as_f64x8(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed double-precision (64-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtps_pd&expand=1771) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2pd))] +pub unsafe fn _mm512_maskz_cvtps_pd(k: __mmask8, a: __m256) -> __m512d { + transmute(vcvtps2pd( + a.as_f32x8(), + _mm512_setzero_pd().as_f64x8(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Performs element-by-element conversion of the lower half of packed single-precision (32-bit) floating-point elements in v2 to packed double-precision (64-bit) floating-point elements, storing the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtpslo_pd&expand=1784) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2pd))] +pub unsafe fn _mm512_cvtpslo_pd(v2: __m512) -> __m512d { + transmute(vcvtps2pd( + _mm512_castps512_ps256(v2).as_f32x8(), + _mm512_setzero_pd().as_f64x8(), + 0b11111111, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Performs element-by-element conversion of the lower half of packed single-precision (32-bit) floating-point elements in v2 to packed double-precision (64-bit) floating-point elements, storing the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtpslo_pd&expand=1785) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2pd))] +pub unsafe fn _mm512_mask_cvtpslo_pd(src: __m512d, k: __mmask8, v2: __m512) -> __m512d { + transmute(vcvtps2pd( + _mm512_castps512_ps256(v2).as_f32x8(), + src.as_f64x8(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed single-precision (32-bit) floating-point elements, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtpd_ps&expand=1712) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtpd2ps))] +pub unsafe fn _mm512_cvtpd_ps(a: __m512d) -> __m256 { + transmute(vcvtpd2ps( + a.as_f64x8(), + _mm256_setzero_ps().as_f32x8(), + 0b11111111, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtpd_ps&expand=1713) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtpd2ps))] +pub unsafe fn _mm512_mask_cvtpd_ps(src: __m256, k: __mmask8, a: __m512d) -> __m256 { + transmute(vcvtpd2ps( + a.as_f64x8(), + src.as_f32x8(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtpd_ps&expand=1714) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtpd2ps))] +pub unsafe fn _mm512_maskz_cvtpd_ps(k: __mmask8, a: __m512d) -> __m256 { + transmute(vcvtpd2ps( + a.as_f64x8(), + _mm256_setzero_ps().as_f32x8(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtpd_ps&expand=1710) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtpd2ps))] +pub unsafe fn _mm256_mask_cvtpd_ps(src: __m128, k: __mmask8, a: __m256d) -> __m128 { + let convert = _mm256_cvtpd_ps(a); + transmute(simd_select_bitmask(k, convert.as_f32x4(), src.as_f32x4())) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtpd_ps&expand=1711) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtpd2ps))] +pub unsafe fn _mm256_maskz_cvtpd_ps(k: __mmask8, a: __m256d) -> __m128 { + let convert = _mm256_cvtpd_ps(a); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, convert.as_f32x4(), zero)) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtpd_ps&expand=1707) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtpd2ps))] +pub unsafe fn _mm_mask_cvtpd_ps(src: __m128, k: __mmask8, a: __m128d) -> __m128 { + let convert = _mm_cvtpd_ps(a); + transmute(simd_select_bitmask(k, convert.as_f32x4(), src.as_f32x4())) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtpd_ps&expand=1708) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtpd2ps))] +pub unsafe fn _mm_maskz_cvtpd_ps(k: __mmask8, a: __m128d) -> __m128 { + let convert = _mm_cvtpd_ps(a); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, convert.as_f32x4(), zero)) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed 32-bit integers, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtpd_epi32&expand=1675) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtpd2dq))] +pub unsafe fn _mm512_cvtpd_epi32(a: __m512d) -> __m256i { + transmute(vcvtpd2dq( + a.as_f64x8(), + _mm256_setzero_si256().as_i32x8(), + 0b11111111, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtpd_epi32&expand=1676) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtpd2dq))] +pub unsafe fn _mm512_mask_cvtpd_epi32(src: __m256i, k: __mmask8, a: __m512d) -> __m256i { + transmute(vcvtpd2dq( + a.as_f64x8(), + src.as_i32x8(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtpd_epi32&expand=1677) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtpd2dq))] +pub unsafe fn _mm512_maskz_cvtpd_epi32(k: __mmask8, a: __m512d) -> __m256i { + transmute(vcvtpd2dq( + a.as_f64x8(), + _mm256_setzero_si256().as_i32x8(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtpd_epi32&expand=1673) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtpd2dq))] +pub unsafe fn _mm256_mask_cvtpd_epi32(src: __m128i, k: __mmask8, a: __m256d) -> __m128i { + let convert = _mm256_cvtpd_epi32(a); + transmute(simd_select_bitmask(k, convert.as_i32x4(), src.as_i32x4())) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtpd_epi32&expand=1674) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtpd2dq))] +pub unsafe fn _mm256_maskz_cvtpd_epi32(k: __mmask8, a: __m256d) -> __m128i { + let convert = _mm256_cvtpd_epi32(a); + transmute(simd_select_bitmask( + k, + convert.as_i32x4(), + _mm_setzero_si128().as_i32x4(), + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtpd_epi32&expand=1670) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtpd2dq))] +pub unsafe fn _mm_mask_cvtpd_epi32(src: __m128i, k: __mmask8, a: __m128d) -> __m128i { + let convert = _mm_cvtpd_epi32(a); + transmute(simd_select_bitmask(k, convert.as_i32x4(), src.as_i32x4())) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtpd_epi32&expand=1671) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtpd2dq))] +pub unsafe fn _mm_maskz_cvtpd_epi32(k: __mmask8, a: __m128d) -> __m128i { + let convert = _mm_cvtpd_epi32(a); + transmute(simd_select_bitmask( + k, + convert.as_i32x4(), + _mm_setzero_si128().as_i32x4(), + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtpd_epu32&expand=1693) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtpd2udq))] +pub unsafe fn _mm512_cvtpd_epu32(a: __m512d) -> __m256i { + transmute(vcvtpd2udq( + a.as_f64x8(), + _mm256_setzero_si256().as_u32x8(), + 0b11111111, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtpd_epu32&expand=1694) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtpd2udq))] +pub unsafe fn _mm512_mask_cvtpd_epu32(src: __m256i, k: __mmask8, a: __m512d) -> __m256i { + transmute(vcvtpd2udq( + a.as_f64x8(), + src.as_u32x8(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtpd_epu32&expand=1695) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtpd2udq))] +pub unsafe fn _mm512_maskz_cvtpd_epu32(k: __mmask8, a: __m512d) -> __m256i { + transmute(vcvtpd2udq( + a.as_f64x8(), + _mm256_setzero_si256().as_u32x8(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvtpd_epu32&expand=1690) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtpd2udq))] +pub unsafe fn _mm256_cvtpd_epu32(a: __m256d) -> __m128i { + transmute(vcvtpd2udq256( + a.as_f64x4(), + _mm_setzero_si128().as_u32x4(), + 0b11111111, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtpd_epu32&expand=1691) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtpd2udq))] +pub unsafe fn _mm256_mask_cvtpd_epu32(src: __m128i, k: __mmask8, a: __m256d) -> __m128i { + transmute(vcvtpd2udq256(a.as_f64x4(), src.as_u32x4(), k)) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtpd_epu32&expand=1692) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtpd2udq))] +pub unsafe fn _mm256_maskz_cvtpd_epu32(k: __mmask8, a: __m256d) -> __m128i { + transmute(vcvtpd2udq256( + a.as_f64x4(), + _mm_setzero_si128().as_u32x4(), + k, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtpd_epu32&expand=1687) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtpd2udq))] +pub unsafe fn _mm_cvtpd_epu32(a: __m128d) -> __m128i { + transmute(vcvtpd2udq128( + a.as_f64x2(), + _mm_setzero_si128().as_u32x4(), + 0b11111111, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtpd_epu32&expand=1688) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtpd2udq))] +pub unsafe fn _mm_mask_cvtpd_epu32(src: __m128i, k: __mmask8, a: __m128d) -> __m128i { + transmute(vcvtpd2udq128(a.as_f64x2(), src.as_u32x4(), k)) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtpd_epu32&expand=1689) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtpd2udq))] +pub unsafe fn _mm_maskz_cvtpd_epu32(k: __mmask8, a: __m128d) -> __m128i { + transmute(vcvtpd2udq128( + a.as_f64x2(), + _mm_setzero_si128().as_u32x4(), + k, + )) +} + +/// Performs an element-by-element conversion of packed double-precision (64-bit) floating-point elements in v2 to single-precision (32-bit) floating-point elements and stores them in dst. The elements are stored in the lower half of the results vector, while the remaining upper half locations are set to 0. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtpd_pslo&expand=1715) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtpd2ps))] +pub unsafe fn _mm512_cvtpd_pslo(v2: __m512d) -> __m512 { + let r: f32x8 = vcvtpd2ps( + v2.as_f64x8(), + _mm256_setzero_ps().as_f32x8(), + 0b11111111, + _MM_FROUND_CUR_DIRECTION, + ); + simd_shuffle16!( + r, + _mm256_setzero_ps().as_f32x8(), + [0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 8, 8, 8, 8, 8, 8], + ) +} + +/// Performs an element-by-element conversion of packed double-precision (64-bit) floating-point elements in v2 to single-precision (32-bit) floating-point elements and stores them in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). The elements are stored in the lower half of the results vector, while the remaining upper half locations are set to 0. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtpd_pslo&expand=1716) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtpd2ps))] +pub unsafe fn _mm512_mask_cvtpd_pslo(src: __m512, k: __mmask8, v2: __m512d) -> __m512 { + let r: f32x8 = vcvtpd2ps( + v2.as_f64x8(), + _mm512_castps512_ps256(src).as_f32x8(), + k, + _MM_FROUND_CUR_DIRECTION, + ); + simd_shuffle16!( + r, + _mm256_setzero_ps().as_f32x8(), + [0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 8, 8, 8, 8, 8, 8], + ) +} + +/// Sign extend packed 8-bit integers in a to packed 32-bit integers, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepi8_epi32&expand=1535) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsxbd))] +pub unsafe fn _mm512_cvtepi8_epi32(a: __m128i) -> __m512i { + let a = a.as_i8x16(); + transmute::(simd_cast(a)) +} + +/// Sign extend packed 8-bit integers in a to packed 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepi8_epi32&expand=1536) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsxbd))] +pub unsafe fn _mm512_mask_cvtepi8_epi32(src: __m512i, k: __mmask16, a: __m128i) -> __m512i { + let convert = _mm512_cvtepi8_epi32(a).as_i32x16(); + transmute(simd_select_bitmask(k, convert, src.as_i32x16())) +} + +/// Sign extend packed 8-bit integers in a to packed 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepi8_epi32&expand=1537) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsxbd))] +pub unsafe fn _mm512_maskz_cvtepi8_epi32(k: __mmask16, a: __m128i) -> __m512i { + let convert = _mm512_cvtepi8_epi32(a).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Sign extend packed 8-bit integers in a to packed 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepi8_epi32&expand=1533) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxbd))] +pub unsafe fn _mm256_mask_cvtepi8_epi32(src: __m256i, k: __mmask8, a: __m128i) -> __m256i { + let convert = _mm256_cvtepi8_epi32(a).as_i32x8(); + transmute(simd_select_bitmask(k, convert, src.as_i32x8())) +} + +/// Sign extend packed 8-bit integers in a to packed 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtepi8_epi32&expand=1534) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxbd))] +pub unsafe fn _mm256_maskz_cvtepi8_epi32(k: __mmask8, a: __m128i) -> __m256i { + let convert = _mm256_cvtepi8_epi32(a).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Sign extend packed 8-bit integers in a to packed 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepi8_epi32&expand=1530) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxbd))] +pub unsafe fn _mm_mask_cvtepi8_epi32(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepi8_epi32(a).as_i32x4(); + transmute(simd_select_bitmask(k, convert, src.as_i32x4())) +} + +/// Sign extend packed 8-bit integers in a to packed 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtepi8_epi32&expand=1531) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxbd))] +pub unsafe fn _mm_maskz_cvtepi8_epi32(k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepi8_epi32(a).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Sign extend packed 8-bit integers in the low 8 bytes of a to packed 64-bit integers, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepi8_epi64&expand=1544) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsxbq))] +pub unsafe fn _mm512_cvtepi8_epi64(a: __m128i) -> __m512i { + let a = a.as_i8x16(); + let v64: i8x8 = simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); + transmute::(simd_cast(v64)) +} + +/// Sign extend packed 8-bit integers in the low 8 bytes of a to packed 64-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepi8_epi64&expand=1545) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsxbq))] +pub unsafe fn _mm512_mask_cvtepi8_epi64(src: __m512i, k: __mmask8, a: __m128i) -> __m512i { + let convert = _mm512_cvtepi8_epi64(a).as_i64x8(); + transmute(simd_select_bitmask(k, convert, src.as_i64x8())) +} + +/// Sign extend packed 8-bit integers in the low 8 bytes of a to packed 64-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepi8_epi64&expand=1546) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsxbq))] +pub unsafe fn _mm512_maskz_cvtepi8_epi64(k: __mmask8, a: __m128i) -> __m512i { + let convert = _mm512_cvtepi8_epi64(a).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Sign extend packed 8-bit integers in the low 4 bytes of a to packed 64-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepi8_epi64&expand=1542) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxbq))] +pub unsafe fn _mm256_mask_cvtepi8_epi64(src: __m256i, k: __mmask8, a: __m128i) -> __m256i { + let convert = _mm256_cvtepi8_epi64(a).as_i64x4(); + transmute(simd_select_bitmask(k, convert, src.as_i64x4())) +} + +/// Sign extend packed 8-bit integers in the low 4 bytes of a to packed 64-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtepi8_epi64&expand=1543) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxbq))] +pub unsafe fn _mm256_maskz_cvtepi8_epi64(k: __mmask8, a: __m128i) -> __m256i { + let convert = _mm256_cvtepi8_epi64(a).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Sign extend packed 8-bit integers in the low 2 bytes of a to packed 64-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepi8_epi64&expand=1539) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxbq))] +pub unsafe fn _mm_mask_cvtepi8_epi64(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepi8_epi64(a).as_i64x2(); + transmute(simd_select_bitmask(k, convert, src.as_i64x2())) +} + +/// Sign extend packed 8-bit integers in the low 2 bytes of a to packed 64-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtepi8_epi64&expand=1540) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxbq))] +pub unsafe fn _mm_maskz_cvtepi8_epi64(k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepi8_epi64(a).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Zero extend packed unsigned 8-bit integers in a to packed 32-bit integers, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepu8_epi32&expand=1621) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovzxbd))] +pub unsafe fn _mm512_cvtepu8_epi32(a: __m128i) -> __m512i { + let a = a.as_u8x16(); + transmute::(simd_cast(a)) +} + +/// Zero extend packed unsigned 8-bit integers in a to packed 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepu8_epi32&expand=1622) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovzxbd))] +pub unsafe fn _mm512_mask_cvtepu8_epi32(src: __m512i, k: __mmask16, a: __m128i) -> __m512i { + let convert = _mm512_cvtepu8_epi32(a).as_i32x16(); + transmute(simd_select_bitmask(k, convert, src.as_i32x16())) +} + +/// Zero extend packed unsigned 8-bit integers in a to packed 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepu8_epi32&expand=1623) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovzxbd))] +pub unsafe fn _mm512_maskz_cvtepu8_epi32(k: __mmask16, a: __m128i) -> __m512i { + let convert = _mm512_cvtepu8_epi32(a).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Zero extend packed unsigned 8-bit integers in the low 8 bytes of a to packed 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepu8_epi32&expand=1619) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxbd))] +pub unsafe fn _mm256_mask_cvtepu8_epi32(src: __m256i, k: __mmask8, a: __m128i) -> __m256i { + let convert = _mm256_cvtepu8_epi32(a).as_i32x8(); + transmute(simd_select_bitmask(k, convert, src.as_i32x8())) +} + +/// Zero extend packed unsigned 8-bit integers in the low 8 bytes of a to packed 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/IntrinsicsGuide/#text=_mm256_maskz_cvtepu8_epi32&expand=1620) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxbd))] +pub unsafe fn _mm256_maskz_cvtepu8_epi32(k: __mmask8, a: __m128i) -> __m256i { + let convert = _mm256_cvtepu8_epi32(a).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Zero extend packed unsigned 8-bit integers in the low 4 bytes of a to packed 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepu8_epi32&expand=1616) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxbd))] +pub unsafe fn _mm_mask_cvtepu8_epi32(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepu8_epi32(a).as_i32x4(); + transmute(simd_select_bitmask(k, convert, src.as_i32x4())) +} + +/// Zero extend packed unsigned 8-bit integers in th elow 4 bytes of a to packed 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/IntrinsicsGuide/#text=_mm_maskz_cvtepu8_epi32&expand=1617) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxbd))] +pub unsafe fn _mm_maskz_cvtepu8_epi32(k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepu8_epi32(a).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Zero extend packed unsigned 8-bit integers in the low 8 byte sof a to packed 64-bit integers, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepu8_epi64&expand=1630) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovzxbq))] +pub unsafe fn _mm512_cvtepu8_epi64(a: __m128i) -> __m512i { + let a = a.as_u8x16(); + let v64: u8x8 = simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); + transmute::(simd_cast(v64)) +} + +/// Zero extend packed unsigned 8-bit integers in the low 8 bytes of a to packed 64-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepu8_epi64&expand=1631) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovzxbq))] +pub unsafe fn _mm512_mask_cvtepu8_epi64(src: __m512i, k: __mmask8, a: __m128i) -> __m512i { + let convert = _mm512_cvtepu8_epi64(a).as_i64x8(); + transmute(simd_select_bitmask(k, convert, src.as_i64x8())) +} + +/// Zero extend packed unsigned 8-bit integers in the low 8 bytes of a to packed 64-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepu8_epi64&expand=1632) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovzxbq))] +pub unsafe fn _mm512_maskz_cvtepu8_epi64(k: __mmask8, a: __m128i) -> __m512i { + let convert = _mm512_cvtepu8_epi64(a).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Zero extend packed unsigned 8-bit integers in the low 4 bytes of a to packed 64-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepu8_epi64&expand=1628) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxbq))] +pub unsafe fn _mm256_mask_cvtepu8_epi64(src: __m256i, k: __mmask8, a: __m128i) -> __m256i { + let convert = _mm256_cvtepu8_epi64(a).as_i64x4(); + transmute(simd_select_bitmask(k, convert, src.as_i64x4())) +} + +/// Zero extend packed unsigned 8-bit integers in the low 4 bytes of a to packed 64-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtepu8_epi64&expand=1629) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxbq))] +pub unsafe fn _mm256_maskz_cvtepu8_epi64(k: __mmask8, a: __m128i) -> __m256i { + let convert = _mm256_cvtepu8_epi64(a).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Zero extend packed unsigned 8-bit integers in the low 2 bytes of a to packed 64-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepu8_epi64&expand=1625) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxbq))] +pub unsafe fn _mm_mask_cvtepu8_epi64(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepu8_epi64(a).as_i64x2(); + transmute(simd_select_bitmask(k, convert, src.as_i64x2())) +} + +/// Zero extend packed unsigned 8-bit integers in the low 2 bytes of a to packed 64-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtepu8_epi64&expand=1626) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxbq))] +pub unsafe fn _mm_maskz_cvtepu8_epi64(k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepu8_epi64(a).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Sign extend packed 16-bit integers in a to packed 32-bit integers, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepi16_epi32&expand=1389) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsxwd))] +pub unsafe fn _mm512_cvtepi16_epi32(a: __m256i) -> __m512i { + let a = a.as_i16x16(); + transmute::(simd_cast(a)) +} + +/// Sign extend packed 16-bit integers in a to packed 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepi16_epi32&expand=1390) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsxwd))] +pub unsafe fn _mm512_mask_cvtepi16_epi32(src: __m512i, k: __mmask16, a: __m256i) -> __m512i { + let convert = _mm512_cvtepi16_epi32(a).as_i32x16(); + transmute(simd_select_bitmask(k, convert, src.as_i32x16())) +} + +/// Sign extend packed 16-bit integers in a to packed 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepi16_epi32&expand=1391) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsxwd))] +pub unsafe fn _mm512_maskz_cvtepi16_epi32(k: __mmask16, a: __m256i) -> __m512i { + let convert = _mm512_cvtepi16_epi32(a).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Sign extend packed 16-bit integers in a to packed 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepi16_epi32&expand=1387) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxwd))] +pub unsafe fn _mm256_mask_cvtepi16_epi32(src: __m256i, k: __mmask8, a: __m128i) -> __m256i { + let convert = _mm256_cvtepi16_epi32(a).as_i32x8(); + transmute(simd_select_bitmask(k, convert, src.as_i32x8())) +} + +/// Sign extend packed 16-bit integers in a to packed 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtepi16_epi32&expand=1388) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxwd))] +pub unsafe fn _mm256_maskz_cvtepi16_epi32(k: __mmask8, a: __m128i) -> __m256i { + let convert = _mm256_cvtepi16_epi32(a).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Sign extend packed 16-bit integers in a to packed 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepi16_epi32&expand=1384) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxwd))] +pub unsafe fn _mm_mask_cvtepi16_epi32(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepi16_epi32(a).as_i32x4(); + transmute(simd_select_bitmask(k, convert, src.as_i32x4())) +} + +/// Sign extend packed 16-bit integers in a to packed 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtepi16_epi32&expand=1385) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxwd))] +pub unsafe fn _mm_maskz_cvtepi16_epi32(k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepi16_epi32(a).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Sign extend packed 16-bit integers in a to packed 64-bit integers, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepi16_epi64&expand=1398) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsxwq))] +pub unsafe fn _mm512_cvtepi16_epi64(a: __m128i) -> __m512i { + let a = a.as_i16x8(); + transmute::(simd_cast(a)) +} + +/// Sign extend packed 16-bit integers in a to packed 64-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepi16_epi64&expand=1399) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsxwq))] +pub unsafe fn _mm512_mask_cvtepi16_epi64(src: __m512i, k: __mmask8, a: __m128i) -> __m512i { + let convert = _mm512_cvtepi16_epi64(a).as_i64x8(); + transmute(simd_select_bitmask(k, convert, src.as_i64x8())) +} + +/// Sign extend packed 16-bit integers in a to packed 64-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepi16_epi64&expand=1400) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsxwq))] +pub unsafe fn _mm512_maskz_cvtepi16_epi64(k: __mmask8, a: __m128i) -> __m512i { + let convert = _mm512_cvtepi16_epi64(a).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Sign extend packed 16-bit integers in a to packed 64-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepi16_epi64&expand=1396) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxwq))] +pub unsafe fn _mm256_mask_cvtepi16_epi64(src: __m256i, k: __mmask8, a: __m128i) -> __m256i { + let convert = _mm256_cvtepi16_epi64(a).as_i64x4(); + transmute(simd_select_bitmask(k, convert, src.as_i64x4())) +} + +/// Sign extend packed 16-bit integers in a to packed 64-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtepi16_epi64&expand=1397) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxwq))] +pub unsafe fn _mm256_maskz_cvtepi16_epi64(k: __mmask8, a: __m128i) -> __m256i { + let convert = _mm256_cvtepi16_epi64(a).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Sign extend packed 16-bit integers in a to packed 64-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepi16_epi64&expand=1393) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxwq))] +pub unsafe fn _mm_mask_cvtepi16_epi64(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepi16_epi64(a).as_i64x2(); + transmute(simd_select_bitmask(k, convert, src.as_i64x2())) +} + +/// Sign extend packed 16-bit integers in a to packed 64-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtepi16_epi64&expand=1394) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxwq))] +pub unsafe fn _mm_maskz_cvtepi16_epi64(k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepi16_epi64(a).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Zero extend packed unsigned 16-bit integers in a to packed 32-bit integers, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepu16_epi32&expand=1553) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovzxwd))] +pub unsafe fn _mm512_cvtepu16_epi32(a: __m256i) -> __m512i { + let a = a.as_u16x16(); + transmute::(simd_cast(a)) +} + +/// Zero extend packed unsigned 16-bit integers in a to packed 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepu16_epi32&expand=1554) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovzxwd))] +pub unsafe fn _mm512_mask_cvtepu16_epi32(src: __m512i, k: __mmask16, a: __m256i) -> __m512i { + let convert = _mm512_cvtepu16_epi32(a).as_i32x16(); + transmute(simd_select_bitmask(k, convert, src.as_i32x16())) +} + +/// Zero extend packed unsigned 16-bit integers in a to packed 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepu16_epi32&expand=1555) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovzxwd))] +pub unsafe fn _mm512_maskz_cvtepu16_epi32(k: __mmask16, a: __m256i) -> __m512i { + let convert = _mm512_cvtepu16_epi32(a).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Zero extend packed unsigned 16-bit integers in a to packed 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepu16_epi32&expand=1551) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxwd))] +pub unsafe fn _mm256_mask_cvtepu16_epi32(src: __m256i, k: __mmask8, a: __m128i) -> __m256i { + let convert = _mm256_cvtepu16_epi32(a).as_i32x8(); + transmute(simd_select_bitmask(k, convert, src.as_i32x8())) +} + +/// Zero extend packed unsigned 16-bit integers in a to packed 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtepu16_epi32&expand=1552) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxwd))] +pub unsafe fn _mm256_maskz_cvtepu16_epi32(k: __mmask8, a: __m128i) -> __m256i { + let convert = _mm256_cvtepu16_epi32(a).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Zero extend packed unsigned 16-bit integers in a to packed 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepu16_epi32&expand=1548) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxwd))] +pub unsafe fn _mm_mask_cvtepu16_epi32(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepu16_epi32(a).as_i32x4(); + transmute(simd_select_bitmask(k, convert, src.as_i32x4())) +} + +/// Zero extend packed unsigned 16-bit integers in a to packed 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtepu16_epi32&expand=1549) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxwd))] +pub unsafe fn _mm_maskz_cvtepu16_epi32(k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepu16_epi32(a).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Zero extend packed unsigned 16-bit integers in a to packed 64-bit integers, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepu16_epi64&expand=1562) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovzxwq))] +pub unsafe fn _mm512_cvtepu16_epi64(a: __m128i) -> __m512i { + let a = a.as_u16x8(); + transmute::(simd_cast(a)) +} + +/// Zero extend packed unsigned 16-bit integers in a to packed 64-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepu16_epi64&expand=1563) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovzxwq))] +pub unsafe fn _mm512_mask_cvtepu16_epi64(src: __m512i, k: __mmask8, a: __m128i) -> __m512i { + let convert = _mm512_cvtepu16_epi64(a).as_i64x8(); + transmute(simd_select_bitmask(k, convert, src.as_i64x8())) +} + +/// Zero extend packed unsigned 16-bit integers in a to packed 64-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepu16_epi64&expand=1564) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovzxwq))] +pub unsafe fn _mm512_maskz_cvtepu16_epi64(k: __mmask8, a: __m128i) -> __m512i { + let convert = _mm512_cvtepu16_epi64(a).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Zero extend packed unsigned 16-bit integers in the low 8 bytes of a to packed 64-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepu16_epi64&expand=1560) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxwq))] +pub unsafe fn _mm256_mask_cvtepu16_epi64(src: __m256i, k: __mmask8, a: __m128i) -> __m256i { + let convert = _mm256_cvtepu16_epi64(a).as_i64x4(); + transmute(simd_select_bitmask(k, convert, src.as_i64x4())) +} + +/// Zero extend packed unsigned 16-bit integers in the low 8 bytes of a to packed 64-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtepu16_epi64&expand=1561) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxwq))] +pub unsafe fn _mm256_maskz_cvtepu16_epi64(k: __mmask8, a: __m128i) -> __m256i { + let convert = _mm256_cvtepu16_epi64(a).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Zero extend packed unsigned 16-bit integers in the low 4 bytes of a to packed 64-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepu16_epi64&expand=1557) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxwq))] +pub unsafe fn _mm_mask_cvtepu16_epi64(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepu16_epi64(a).as_i64x2(); + transmute(simd_select_bitmask(k, convert, src.as_i64x2())) +} + +/// Zero extend packed unsigned 16-bit integers in the low 4 bytes of a to packed 64-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtepu16_epi64&expand=1558) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxwq))] +pub unsafe fn _mm_maskz_cvtepu16_epi64(k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepu16_epi64(a).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Sign extend packed 32-bit integers in a to packed 64-bit integers, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepi32_epi64&expand=1428) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsxdq))] +pub unsafe fn _mm512_cvtepi32_epi64(a: __m256i) -> __m512i { + let a = a.as_i32x8(); + transmute::(simd_cast(a)) +} + +/// Sign extend packed 32-bit integers in a to packed 64-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepi32_epi64&expand=1429) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsxdq))] +pub unsafe fn _mm512_mask_cvtepi32_epi64(src: __m512i, k: __mmask8, a: __m256i) -> __m512i { + let convert = _mm512_cvtepi32_epi64(a).as_i64x8(); + transmute(simd_select_bitmask(k, convert, src.as_i64x8())) +} + +/// Sign extend packed 32-bit integers in a to packed 64-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepi32_epi64&expand=1430) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsxdq))] +pub unsafe fn _mm512_maskz_cvtepi32_epi64(k: __mmask8, a: __m256i) -> __m512i { + let convert = _mm512_cvtepi32_epi64(a).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Sign extend packed 32-bit integers in a to packed 64-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepi32_epi64&expand=1426) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxdq))] +pub unsafe fn _mm256_mask_cvtepi32_epi64(src: __m256i, k: __mmask8, a: __m128i) -> __m256i { + let convert = _mm256_cvtepi32_epi64(a).as_i64x4(); + transmute(simd_select_bitmask(k, convert, src.as_i64x4())) +} + +/// Sign extend packed 32-bit integers in a to packed 64-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtepi32_epi64&expand=1427) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxdq))] +pub unsafe fn _mm256_maskz_cvtepi32_epi64(k: __mmask8, a: __m128i) -> __m256i { + let convert = _mm256_cvtepi32_epi64(a).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Sign extend packed 32-bit integers in a to packed 64-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepi32_epi64&expand=1423) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxdq))] +pub unsafe fn _mm_mask_cvtepi32_epi64(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepi32_epi64(a).as_i64x2(); + transmute(simd_select_bitmask(k, convert, src.as_i64x2())) +} + +/// Sign extend packed 32-bit integers in a to packed 64-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtepi32_epi64&expand=1424) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsxdq))] +pub unsafe fn _mm_maskz_cvtepi32_epi64(k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepi32_epi64(a).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Zero extend packed unsigned 32-bit integers in a to packed 64-bit integers, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepu32_epi64&expand=1571) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovzxdq))] +pub unsafe fn _mm512_cvtepu32_epi64(a: __m256i) -> __m512i { + let a = a.as_u32x8(); + transmute::(simd_cast(a)) +} + +/// Zero extend packed unsigned 32-bit integers in a to packed 64-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepu32_epi64&expand=1572) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovzxdq))] +pub unsafe fn _mm512_mask_cvtepu32_epi64(src: __m512i, k: __mmask8, a: __m256i) -> __m512i { + let convert = _mm512_cvtepu32_epi64(a).as_i64x8(); + transmute(simd_select_bitmask(k, convert, src.as_i64x8())) +} + +/// Zero extend packed unsigned 32-bit integers in a to packed 64-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepu32_epi64&expand=1573) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovzxdq))] +pub unsafe fn _mm512_maskz_cvtepu32_epi64(k: __mmask8, a: __m256i) -> __m512i { + let convert = _mm512_cvtepu32_epi64(a).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Zero extend packed unsigned 32-bit integers in a to packed 64-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepu32_epi64&expand=1569) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxdq))] +pub unsafe fn _mm256_mask_cvtepu32_epi64(src: __m256i, k: __mmask8, a: __m128i) -> __m256i { + let convert = _mm256_cvtepu32_epi64(a).as_i64x4(); + transmute(simd_select_bitmask(k, convert, src.as_i64x4())) +} + +/// Zero extend packed unsigned 32-bit integers in a to packed 64-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtepu32_epi64&expand=1570) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxdq))] +pub unsafe fn _mm256_maskz_cvtepu32_epi64(k: __mmask8, a: __m128i) -> __m256i { + let convert = _mm256_cvtepu32_epi64(a).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Zero extend packed unsigned 32-bit integers in a to packed 64-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepu32_epi64&expand=1566) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxdq))] +pub unsafe fn _mm_mask_cvtepu32_epi64(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepu32_epi64(a).as_i64x2(); + transmute(simd_select_bitmask(k, convert, src.as_i64x2())) +} + +/// Zero extend packed unsigned 32-bit integers in a to packed 64-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtepu32_epi64&expand=1567) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovzxdq))] +pub unsafe fn _mm_maskz_cvtepu32_epi64(k: __mmask8, a: __m128i) -> __m128i { + let convert = _mm_cvtepu32_epi64(a).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Convert packed signed 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepi32_ps&expand=1455) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtdq2ps))] +pub unsafe fn _mm512_cvtepi32_ps(a: __m512i) -> __m512 { + let a = a.as_i32x16(); + transmute::(simd_cast(a)) +} + +/// Convert packed signed 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepi32_ps&expand=1456) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtdq2ps))] +pub unsafe fn _mm512_mask_cvtepi32_ps(src: __m512, k: __mmask16, a: __m512i) -> __m512 { + let convert = _mm512_cvtepi32_ps(a).as_f32x16(); + transmute(simd_select_bitmask(k, convert, src.as_f32x16())) +} + +/// Convert packed signed 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepi32_ps&expand=1457) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtdq2ps))] +pub unsafe fn _mm512_maskz_cvtepi32_ps(k: __mmask16, a: __m512i) -> __m512 { + let convert = _mm512_cvtepi32_ps(a).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Convert packed signed 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepi32_ps&expand=1453) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtdq2ps))] +pub unsafe fn _mm256_mask_cvtepi32_ps(src: __m256, k: __mmask8, a: __m256i) -> __m256 { + let convert = _mm256_cvtepi32_ps(a).as_f32x8(); + transmute(simd_select_bitmask(k, convert, src.as_f32x8())) +} + +/// Convert packed signed 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtepi32_ps&expand=1454) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtdq2ps))] +pub unsafe fn _mm256_maskz_cvtepi32_ps(k: __mmask8, a: __m256i) -> __m256 { + let convert = _mm256_cvtepi32_ps(a).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Convert packed signed 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepi32_ps&expand=1450) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtdq2ps))] +pub unsafe fn _mm_mask_cvtepi32_ps(src: __m128, k: __mmask8, a: __m128i) -> __m128 { + let convert = _mm_cvtepi32_ps(a).as_f32x4(); + transmute(simd_select_bitmask(k, convert, src.as_f32x4())) +} + +/// Convert packed signed 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtepi32_ps&expand=1451) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtdq2ps))] +pub unsafe fn _mm_maskz_cvtepi32_ps(k: __mmask8, a: __m128i) -> __m128 { + let convert = _mm_cvtepi32_ps(a).as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Convert packed signed 32-bit integers in a to packed double-precision (64-bit) floating-point elements, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepi32_pd&expand=1446) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtdq2pd))] +pub unsafe fn _mm512_cvtepi32_pd(a: __m256i) -> __m512d { + let a = a.as_i32x8(); + transmute::(simd_cast(a)) +} + +/// Convert packed signed 32-bit integers in a to packed double-precision (64-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepi32_pd&expand=1447) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtdq2pd))] +pub unsafe fn _mm512_mask_cvtepi32_pd(src: __m512d, k: __mmask8, a: __m256i) -> __m512d { + let convert = _mm512_cvtepi32_pd(a).as_f64x8(); + transmute(simd_select_bitmask(k, convert, src.as_f64x8())) +} + +/// Convert packed signed 32-bit integers in a to packed double-precision (64-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepi32_pd&expand=1448) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtdq2pd))] +pub unsafe fn _mm512_maskz_cvtepi32_pd(k: __mmask8, a: __m256i) -> __m512d { + let convert = _mm512_cvtepi32_pd(a).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Convert packed signed 32-bit integers in a to packed double-precision (64-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepi32_pd&expand=1444) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtdq2pd))] +pub unsafe fn _mm256_mask_cvtepi32_pd(src: __m256d, k: __mmask8, a: __m128i) -> __m256d { + let convert = _mm256_cvtepi32_pd(a).as_f64x4(); + transmute(simd_select_bitmask(k, convert, src.as_f64x4())) +} + +/// Convert packed signed 32-bit integers in a to packed double-precision (64-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtepi32_pd&expand=1445) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtdq2pd))] +pub unsafe fn _mm256_maskz_cvtepi32_pd(k: __mmask8, a: __m128i) -> __m256d { + let convert = _mm256_cvtepi32_pd(a).as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Convert packed signed 32-bit integers in a to packed double-precision (64-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepi32_pd&expand=1441) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtdq2pd))] +pub unsafe fn _mm_mask_cvtepi32_pd(src: __m128d, k: __mmask8, a: __m128i) -> __m128d { + let convert = _mm_cvtepi32_pd(a).as_f64x2(); + transmute(simd_select_bitmask(k, convert, src.as_f64x2())) +} + +/// Convert packed signed 32-bit integers in a to packed double-precision (64-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtepi32_pd&expand=1442) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtdq2pd))] +pub unsafe fn _mm_maskz_cvtepi32_pd(k: __mmask8, a: __m128i) -> __m128d { + let convert = _mm_cvtepi32_pd(a).as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Convert packed unsigned 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepu32_ps&expand=1583) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtudq2ps))] +pub unsafe fn _mm512_cvtepu32_ps(a: __m512i) -> __m512 { + let a = a.as_u32x16(); + transmute::(simd_cast(a)) +} + +/// Convert packed unsigned 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepu32_ps&expand=1584) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtudq2ps))] +pub unsafe fn _mm512_mask_cvtepu32_ps(src: __m512, k: __mmask16, a: __m512i) -> __m512 { + let convert = _mm512_cvtepu32_ps(a).as_f32x16(); + transmute(simd_select_bitmask(k, convert, src.as_f32x16())) +} + +/// Convert packed unsigned 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepu32_ps&expand=1585) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtudq2ps))] +pub unsafe fn _mm512_maskz_cvtepu32_ps(k: __mmask16, a: __m512i) -> __m512 { + let convert = _mm512_cvtepu32_ps(a).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Convert packed unsigned 32-bit integers in a to packed double-precision (64-bit) floating-point elements, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepu32_pd&expand=1580) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtudq2pd))] +pub unsafe fn _mm512_cvtepu32_pd(a: __m256i) -> __m512d { + let a = a.as_u32x8(); + transmute::(simd_cast(a)) +} + +/// Convert packed unsigned 32-bit integers in a to packed double-precision (64-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepu32_pd&expand=1581) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtudq2pd))] +pub unsafe fn _mm512_mask_cvtepu32_pd(src: __m512d, k: __mmask8, a: __m256i) -> __m512d { + let convert = _mm512_cvtepu32_pd(a).as_f64x8(); + transmute(simd_select_bitmask(k, convert, src.as_f64x8())) +} + +/// Convert packed unsigned 32-bit integers in a to packed double-precision (64-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepu32_pd&expand=1582) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtudq2pd))] +pub unsafe fn _mm512_maskz_cvtepu32_pd(k: __mmask8, a: __m256i) -> __m512d { + let convert = _mm512_cvtepu32_pd(a).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Convert packed unsigned 32-bit integers in a to packed double-precision (64-bit) floating-point elements, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvtepu32_pd&expand=1577) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtudq2pd))] +pub unsafe fn _mm256_cvtepu32_pd(a: __m128i) -> __m256d { + let a = a.as_u32x4(); + transmute::(simd_cast(a)) +} + +/// Convert packed unsigned 32-bit integers in a to packed double-precision (64-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepu32_pd&expand=1578) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtudq2pd))] +pub unsafe fn _mm256_mask_cvtepu32_pd(src: __m256d, k: __mmask8, a: __m128i) -> __m256d { + let convert = _mm256_cvtepu32_pd(a).as_f64x4(); + transmute(simd_select_bitmask(k, convert, src.as_f64x4())) +} + +/// Convert packed unsigned 32-bit integers in a to packed double-precision (64-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtepu32_pd&expand=1579) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtudq2pd))] +pub unsafe fn _mm256_maskz_cvtepu32_pd(k: __mmask8, a: __m128i) -> __m256d { + let convert = _mm256_cvtepu32_pd(a).as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Convert packed unsigned 32-bit integers in a to packed double-precision (64-bit) floating-point elements, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtepu32_pd&expand=1574) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtudq2pd))] +pub unsafe fn _mm_cvtepu32_pd(a: __m128i) -> __m128d { + let a = a.as_u32x4(); + let u64: u32x2 = simd_shuffle2!(a, a, [0, 1]); + transmute::(simd_cast(u64)) +} + +/// Convert packed unsigned 32-bit integers in a to packed double-precision (64-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepu32_pd&expand=1575) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtudq2pd))] +pub unsafe fn _mm_mask_cvtepu32_pd(src: __m128d, k: __mmask8, a: __m128i) -> __m128d { + let convert = _mm_cvtepu32_pd(a).as_f64x2(); + transmute(simd_select_bitmask(k, convert, src.as_f64x2())) +} + +/// Convert packed unsigned 32-bit integers in a to packed double-precision (64-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtepu32_pd&expand=1576) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtudq2pd))] +pub unsafe fn _mm_maskz_cvtepu32_pd(k: __mmask8, a: __m128i) -> __m128d { + let convert = _mm_cvtepu32_pd(a).as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Performs element-by-element conversion of the lower half of packed 32-bit integer elements in v2 to packed double-precision (64-bit) floating-point elements, storing the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepi32lo_pd&expand=1464) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtdq2pd))] +pub unsafe fn _mm512_cvtepi32lo_pd(v2: __m512i) -> __m512d { + let v2 = v2.as_i32x16(); + let v256: i32x8 = simd_shuffle8!(v2, v2, [0, 1, 2, 3, 4, 5, 6, 7]); + transmute::(simd_cast(v256)) +} + +/// Performs element-by-element conversion of the lower half of packed 32-bit integer elements in v2 to packed double-precision (64-bit) floating-point elements, storing the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepi32lo_pd&expand=1465) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtdq2pd))] +pub unsafe fn _mm512_mask_cvtepi32lo_pd(src: __m512d, k: __mmask8, v2: __m512i) -> __m512d { + let convert = _mm512_cvtepi32lo_pd(v2).as_f64x8(); + transmute(simd_select_bitmask(k, convert, src.as_f64x8())) +} + +/// Performs element-by-element conversion of the lower half of packed 32-bit unsigned integer elements in v2 to packed double-precision (64-bit) floating-point elements, storing the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepu32lo_pd&expand=1586) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtudq2pd))] +pub unsafe fn _mm512_cvtepu32lo_pd(v2: __m512i) -> __m512d { + let v2 = v2.as_u32x16(); + let v256: u32x8 = simd_shuffle8!(v2, v2, [0, 1, 2, 3, 4, 5, 6, 7]); + transmute::(simd_cast(v256)) +} + +/// Performs element-by-element conversion of the lower half of 32-bit unsigned integer elements in v2 to packed double-precision (64-bit) floating-point elements, storing the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepu32lo_pd&expand=1587) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtudq2pd))] +pub unsafe fn _mm512_mask_cvtepu32lo_pd(src: __m512d, k: __mmask8, v2: __m512i) -> __m512d { + let convert = _mm512_cvtepu32lo_pd(v2).as_f64x8(); + transmute(simd_select_bitmask(k, convert, src.as_f64x8())) +} + +/// Convert packed 32-bit integers in a to packed 16-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepi32_epi16&expand=1419) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovdw))] +pub unsafe fn _mm512_cvtepi32_epi16(a: __m512i) -> __m256i { + let a = a.as_i32x16(); + transmute::(simd_cast(a)) +} + +/// Convert packed 32-bit integers in a to packed 16-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepi32_epi16&expand=1420) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovdw))] +pub unsafe fn _mm512_mask_cvtepi32_epi16(src: __m256i, k: __mmask16, a: __m512i) -> __m256i { + let convert = _mm512_cvtepi32_epi16(a).as_i16x16(); + transmute(simd_select_bitmask(k, convert, src.as_i16x16())) +} + +/// Convert packed 32-bit integers in a to packed 16-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepi32_epi16&expand=1421) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovdw))] +pub unsafe fn _mm512_maskz_cvtepi32_epi16(k: __mmask16, a: __m512i) -> __m256i { + let convert = _mm512_cvtepi32_epi16(a).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Convert packed 32-bit integers in a to packed 16-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvtepi32_epi16&expand=1416) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovdw))] +pub unsafe fn _mm256_cvtepi32_epi16(a: __m256i) -> __m128i { + let a = a.as_i32x8(); + transmute::(simd_cast(a)) +} + +/// Convert packed 32-bit integers in a to packed 16-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepi32_epi16&expand=1417) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovdw))] +pub unsafe fn _mm256_mask_cvtepi32_epi16(src: __m128i, k: __mmask8, a: __m256i) -> __m128i { + let convert = _mm256_cvtepi32_epi16(a).as_i16x8(); + transmute(simd_select_bitmask(k, convert, src.as_i16x8())) +} + +/// Convert packed 32-bit integers in a to packed 16-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtepi32_epi16&expand=1418) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovdw))] +pub unsafe fn _mm256_maskz_cvtepi32_epi16(k: __mmask8, a: __m256i) -> __m128i { + let convert = _mm256_cvtepi32_epi16(a).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Convert packed 32-bit integers in a to packed 16-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtepi32_epi16&expand=1413) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovdw))] +pub unsafe fn _mm_cvtepi32_epi16(a: __m128i) -> __m128i { + transmute(vpmovdw128( + a.as_i32x4(), + _mm_setzero_si128().as_i16x8(), + 0b11111111, + )) +} + +/// Convert packed 32-bit integers in a to packed 16-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepi32_epi16&expand=1414) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovdw))] +pub unsafe fn _mm_mask_cvtepi32_epi16(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovdw128(a.as_i32x4(), src.as_i16x8(), k)) +} + +/// Convert packed 32-bit integers in a to packed 16-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtepi32_epi16&expand=1415) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovdw))] +pub unsafe fn _mm_maskz_cvtepi32_epi16(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovdw128(a.as_i32x4(), _mm_setzero_si128().as_i16x8(), k)) +} + +/// Convert packed 32-bit integers in a to packed 8-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepi32_epi8&expand=1437) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovdb))] +pub unsafe fn _mm512_cvtepi32_epi8(a: __m512i) -> __m128i { + let a = a.as_i32x16(); + transmute::(simd_cast(a)) +} + +/// Convert packed 32-bit integers in a to packed 8-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepi32_epi8&expand=1438) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovdb))] +pub unsafe fn _mm512_mask_cvtepi32_epi8(src: __m128i, k: __mmask16, a: __m512i) -> __m128i { + let convert = _mm512_cvtepi32_epi8(a).as_i8x16(); + transmute(simd_select_bitmask(k, convert, src.as_i8x16())) +} + +/// Convert packed 32-bit integers in a to packed 8-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepi32_epi8&expand=1439) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovdb))] +pub unsafe fn _mm512_maskz_cvtepi32_epi8(k: __mmask16, a: __m512i) -> __m128i { + let convert = _mm512_cvtepi32_epi8(a).as_i8x16(); + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Convert packed 32-bit integers in a to packed 8-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvtepi32_epi8&expand=1434) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovdb))] +pub unsafe fn _mm256_cvtepi32_epi8(a: __m256i) -> __m128i { + transmute(vpmovdb256( + a.as_i32x8(), + _mm_setzero_si128().as_i8x16(), + 0b11111111, + )) +} + +/// Convert packed 32-bit integers in a to packed 8-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepi32_epi8&expand=1435) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovdb))] +pub unsafe fn _mm256_mask_cvtepi32_epi8(src: __m128i, k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovdb256(a.as_i32x8(), src.as_i8x16(), k)) +} + +/// Convert packed 32-bit integers in a to packed 8-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtepi32_epi8&expand=1436) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovdb))] +pub unsafe fn _mm256_maskz_cvtepi32_epi8(k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovdb256(a.as_i32x8(), _mm_setzero_si128().as_i8x16(), k)) +} + +/// Convert packed 32-bit integers in a to packed 8-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtepi32_epi8&expand=1431) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovdb))] +pub unsafe fn _mm_cvtepi32_epi8(a: __m128i) -> __m128i { + transmute(vpmovdb128( + a.as_i32x4(), + _mm_setzero_si128().as_i8x16(), + 0b11111111, + )) +} + +/// Convert packed 32-bit integers in a to packed 8-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepi32_epi8&expand=1432) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovdb))] +pub unsafe fn _mm_mask_cvtepi32_epi8(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovdb128(a.as_i32x4(), src.as_i8x16(), k)) +} + +/// Convert packed 32-bit integers in a to packed 8-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtepi32_epi8&expand=1433) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovdb))] +pub unsafe fn _mm_maskz_cvtepi32_epi8(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovdb128(a.as_i32x4(), _mm_setzero_si128().as_i8x16(), k)) +} + +/// Convert packed 64-bit integers in a to packed 32-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepi64_epi32&expand=1481) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovqd))] +pub unsafe fn _mm512_cvtepi64_epi32(a: __m512i) -> __m256i { + let a = a.as_i64x8(); + transmute::(simd_cast(a)) +} + +/// Convert packed 64-bit integers in a to packed 32-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepi64_epi32&expand=1482) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovqd))] +pub unsafe fn _mm512_mask_cvtepi64_epi32(src: __m256i, k: __mmask8, a: __m512i) -> __m256i { + let convert = _mm512_cvtepi64_epi32(a).as_i32x8(); + transmute(simd_select_bitmask(k, convert, src.as_i32x8())) +} + +/// Convert packed 64-bit integers in a to packed 32-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepi64_epi32&expand=1483) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovqd))] +pub unsafe fn _mm512_maskz_cvtepi64_epi32(k: __mmask8, a: __m512i) -> __m256i { + let convert = _mm512_cvtepi64_epi32(a).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Convert packed 64-bit integers in a to packed 32-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvtepi64_epi32&expand=1478) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqd))] +pub unsafe fn _mm256_cvtepi64_epi32(a: __m256i) -> __m128i { + let a = a.as_i64x4(); + transmute::(simd_cast(a)) +} + +/// Convert packed 64-bit integers in a to packed 32-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepi64_epi32&expand=1479) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqd))] +pub unsafe fn _mm256_mask_cvtepi64_epi32(src: __m128i, k: __mmask8, a: __m256i) -> __m128i { + let convert = _mm256_cvtepi64_epi32(a).as_i32x4(); + transmute(simd_select_bitmask(k, convert, src.as_i32x4())) +} + +/// Convert packed 64-bit integers in a to packed 32-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtepi64_epi32&expand=1480) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqd))] +pub unsafe fn _mm256_maskz_cvtepi64_epi32(k: __mmask8, a: __m256i) -> __m128i { + let convert = _mm256_cvtepi64_epi32(a).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Convert packed 64-bit integers in a to packed 32-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtepi64_epi32&expand=1475) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqd))] +pub unsafe fn _mm_cvtepi64_epi32(a: __m128i) -> __m128i { + transmute(vpmovqd128( + a.as_i64x2(), + _mm_setzero_si128().as_i32x4(), + 0b11111111, + )) +} + +/// Convert packed 64-bit integers in a to packed 32-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepi64_epi32&expand=1476) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqd))] +pub unsafe fn _mm_mask_cvtepi64_epi32(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovqd128(a.as_i64x2(), src.as_i32x4(), k)) +} + +/// Convert packed 64-bit integers in a to packed 32-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtepi64_epi32&expand=1477) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqd))] +pub unsafe fn _mm_maskz_cvtepi64_epi32(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovqd128(a.as_i64x2(), _mm_setzero_si128().as_i32x4(), k)) +} + +/// Convert packed 64-bit integers in a to packed 16-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepi64_epi16&expand=1472) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovqw))] +pub unsafe fn _mm512_cvtepi64_epi16(a: __m512i) -> __m128i { + let a = a.as_i64x8(); + transmute::(simd_cast(a)) +} + +/// Convert packed 64-bit integers in a to packed 16-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepi64_epi16&expand=1473) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovqw))] +pub unsafe fn _mm512_mask_cvtepi64_epi16(src: __m128i, k: __mmask8, a: __m512i) -> __m128i { + let convert = _mm512_cvtepi64_epi16(a).as_i16x8(); + transmute(simd_select_bitmask(k, convert, src.as_i16x8())) +} + +/// Convert packed 64-bit integers in a to packed 16-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepi64_epi16&expand=1474) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovqw))] +pub unsafe fn _mm512_maskz_cvtepi64_epi16(k: __mmask8, a: __m512i) -> __m128i { + let convert = _mm512_cvtepi64_epi16(a).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, convert, zero)) +} + +/// Convert packed 64-bit integers in a to packed 16-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvtepi64_epi16&expand=1469) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqw))] +pub unsafe fn _mm256_cvtepi64_epi16(a: __m256i) -> __m128i { + transmute(vpmovqw256( + a.as_i64x4(), + _mm_setzero_si128().as_i16x8(), + 0b11111111, + )) +} + +/// Convert packed 64-bit integers in a to packed 16-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepi64_epi16&expand=1470) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqw))] +pub unsafe fn _mm256_mask_cvtepi64_epi16(src: __m128i, k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovqw256(a.as_i64x4(), src.as_i16x8(), k)) +} + +/// Convert packed 64-bit integers in a to packed 16-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtepi64_epi16&expand=1471) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqw))] +pub unsafe fn _mm256_maskz_cvtepi64_epi16(k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovqw256(a.as_i64x4(), _mm_setzero_si128().as_i16x8(), k)) +} + +/// Convert packed 64-bit integers in a to packed 16-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtepi64_epi16&expand=1466) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqw))] +pub unsafe fn _mm_cvtepi64_epi16(a: __m128i) -> __m128i { + transmute(vpmovqw128( + a.as_i64x2(), + _mm_setzero_si128().as_i16x8(), + 0b11111111, + )) +} + +/// Convert packed 64-bit integers in a to packed 16-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepi64_epi16&expand=1467) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqw))] +pub unsafe fn _mm_mask_cvtepi64_epi16(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovqw128(a.as_i64x2(), src.as_i16x8(), k)) +} + +/// Convert packed 64-bit integers in a to packed 16-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtepi64_epi16&expand=1468) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqw))] +pub unsafe fn _mm_maskz_cvtepi64_epi16(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovqw128(a.as_i64x2(), _mm_setzero_si128().as_i16x8(), k)) +} + +/// Convert packed 64-bit integers in a to packed 8-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtepi64_epi8&expand=1490) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovqb))] +pub unsafe fn _mm512_cvtepi64_epi8(a: __m512i) -> __m128i { + transmute(vpmovqb( + a.as_i64x8(), + _mm_setzero_si128().as_i8x16(), + 0b11111111, + )) +} + +/// Convert packed 64-bit integers in a to packed 8-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepi64_epi8&expand=1491) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovqb))] +pub unsafe fn _mm512_mask_cvtepi64_epi8(src: __m128i, k: __mmask8, a: __m512i) -> __m128i { + transmute(vpmovqb(a.as_i64x8(), src.as_i8x16(), k)) +} + +/// Convert packed 64-bit integers in a to packed 8-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtepi64_epi8&expand=1492) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovqb))] +pub unsafe fn _mm512_maskz_cvtepi64_epi8(k: __mmask8, a: __m512i) -> __m128i { + transmute(vpmovqb(a.as_i64x8(), _mm_setzero_si128().as_i8x16(), k)) +} + +/// Convert packed 64-bit integers in a to packed 8-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvtepi64_epi8&expand=1487) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqb))] +pub unsafe fn _mm256_cvtepi64_epi8(a: __m256i) -> __m128i { + transmute(vpmovqb256( + a.as_i64x4(), + _mm_setzero_si128().as_i8x16(), + 0b11111111, + )) +} + +/// Convert packed 64-bit integers in a to packed 8-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepi64_epi8&expand=1488) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqb))] +pub unsafe fn _mm256_mask_cvtepi64_epi8(src: __m128i, k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovqb256(a.as_i64x4(), src.as_i8x16(), k)) +} + +/// Convert packed 64-bit integers in a to packed 8-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtepi64_epi8&expand=1489) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqb))] +pub unsafe fn _mm256_maskz_cvtepi64_epi8(k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovqb256(a.as_i64x4(), _mm_setzero_si128().as_i8x16(), k)) +} + +/// Convert packed 64-bit integers in a to packed 8-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtepi64_epi8&expand=1484) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqb))] +pub unsafe fn _mm_cvtepi64_epi8(a: __m128i) -> __m128i { + transmute(vpmovqb128( + a.as_i64x2(), + _mm_setzero_si128().as_i8x16(), + 0b11111111, + )) +} + +/// Convert packed 64-bit integers in a to packed 8-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepi64_epi8&expand=1485) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqb))] +pub unsafe fn _mm_mask_cvtepi64_epi8(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovqb128(a.as_i64x2(), src.as_i8x16(), k)) +} + +/// Convert packed 64-bit integers in a to packed 8-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtepi64_epi8&expand=1486) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqb))] +pub unsafe fn _mm_maskz_cvtepi64_epi8(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovqb128(a.as_i64x2(), _mm_setzero_si128().as_i8x16(), k)) +} + +/// Convert packed signed 32-bit integers in a to packed 16-bit integers with signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtsepi32_epi16&expand=1819) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsdw))] +pub unsafe fn _mm512_cvtsepi32_epi16(a: __m512i) -> __m256i { + transmute(vpmovsdw( + a.as_i32x16(), + _mm256_setzero_si256().as_i16x16(), + 0b11111111_11111111, + )) +} + +/// Convert packed signed 32-bit integers in a to packed 16-bit integers with signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtsepi32_epi16&expand=1820) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsdw))] +pub unsafe fn _mm512_mask_cvtsepi32_epi16(src: __m256i, k: __mmask16, a: __m512i) -> __m256i { + transmute(vpmovsdw(a.as_i32x16(), src.as_i16x16(), k)) +} + +/// Convert packed signed 32-bit integers in a to packed 16-bit integers with signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtsepi32_epi16&expand=1819) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsdw))] +pub unsafe fn _mm512_maskz_cvtsepi32_epi16(k: __mmask16, a: __m512i) -> __m256i { + transmute(vpmovsdw( + a.as_i32x16(), + _mm256_setzero_si256().as_i16x16(), + k, + )) +} + +/// Convert packed signed 32-bit integers in a to packed 16-bit integers with signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvtsepi32_epi16&expand=1816) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsdw))] +pub unsafe fn _mm256_cvtsepi32_epi16(a: __m256i) -> __m128i { + transmute(vpmovsdw256( + a.as_i32x8(), + _mm_setzero_si128().as_i16x8(), + 0b11111111, + )) +} + +/// Convert packed signed 32-bit integers in a to packed 16-bit integers with signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtsepi32_epi16&expand=1817) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsdw))] +pub unsafe fn _mm256_mask_cvtsepi32_epi16(src: __m128i, k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovsdw256(a.as_i32x8(), src.as_i16x8(), k)) +} + +/// Convert packed signed 32-bit integers in a to packed 16-bit integers with signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtsepi32_epi16&expand=1818) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsdw))] +pub unsafe fn _mm256_maskz_cvtsepi32_epi16(k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovsdw256(a.as_i32x8(), _mm_setzero_si128().as_i16x8(), k)) +} + +/// Convert packed signed 32-bit integers in a to packed 16-bit integers with signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtsepi32_epi16&expand=1813) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsdw))] +pub unsafe fn _mm_cvtsepi32_epi16(a: __m128i) -> __m128i { + transmute(vpmovsdw128( + a.as_i32x4(), + _mm_setzero_si128().as_i16x8(), + 0b11111111, + )) +} + +/// Convert packed signed 32-bit integers in a to packed 16-bit integers with signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtsepi32_epi16&expand=1814) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsdw))] +pub unsafe fn _mm_mask_cvtsepi32_epi16(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovsdw128(a.as_i32x4(), src.as_i16x8(), k)) +} + +/// Convert packed signed 32-bit integers in a to packed 16-bit integers with signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtsepi32_epi16&expand=1815) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsdw))] +pub unsafe fn _mm_maskz_cvtsepi32_epi16(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovsdw128(a.as_i32x4(), _mm_setzero_si128().as_i16x8(), k)) +} + +/// Convert packed signed 32-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtsepi32_epi8&expand=1828) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsdb))] +pub unsafe fn _mm512_cvtsepi32_epi8(a: __m512i) -> __m128i { + transmute(vpmovsdb( + a.as_i32x16(), + _mm_setzero_si128().as_i8x16(), + 0b11111111_11111111, + )) +} + +/// Convert packed signed 32-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtsepi32_epi8&expand=1829) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsdb))] +pub unsafe fn _mm512_mask_cvtsepi32_epi8(src: __m128i, k: __mmask16, a: __m512i) -> __m128i { + transmute(vpmovsdb(a.as_i32x16(), src.as_i8x16(), k)) +} + +/// Convert packed signed 32-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtsepi32_epi8&expand=1830) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsdb))] +pub unsafe fn _mm512_maskz_cvtsepi32_epi8(k: __mmask16, a: __m512i) -> __m128i { + transmute(vpmovsdb(a.as_i32x16(), _mm_setzero_si128().as_i8x16(), k)) +} + +/// Convert packed signed 32-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvtsepi32_epi8&expand=1825) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsdb))] +pub unsafe fn _mm256_cvtsepi32_epi8(a: __m256i) -> __m128i { + transmute(vpmovsdb256( + a.as_i32x8(), + _mm_setzero_si128().as_i8x16(), + 0b11111111, + )) +} + +/// Convert packed signed 32-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtsepi32_epi8&expand=1826) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsdb))] +pub unsafe fn _mm256_mask_cvtsepi32_epi8(src: __m128i, k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovsdb256(a.as_i32x8(), src.as_i8x16(), k)) +} + +/// Convert packed signed 32-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtsepi32_epi8&expand=1827) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsdb))] +pub unsafe fn _mm256_maskz_cvtsepi32_epi8(k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovsdb256(a.as_i32x8(), _mm_setzero_si128().as_i8x16(), k)) +} + +/// Convert packed signed 32-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtsepi32_epi8&expand=1822) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsdb))] +pub unsafe fn _mm_cvtsepi32_epi8(a: __m128i) -> __m128i { + transmute(vpmovsdb128( + a.as_i32x4(), + _mm_setzero_si128().as_i8x16(), + 0b11111111, + )) +} + +/// Convert packed signed 32-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtsepi32_epi8&expand=1823) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsdb))] +pub unsafe fn _mm_mask_cvtsepi32_epi8(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovsdb128(a.as_i32x4(), src.as_i8x16(), k)) +} + +/// Convert packed signed 32-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtsepi32_epi8&expand=1824) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsdb))] +pub unsafe fn _mm_maskz_cvtsepi32_epi8(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovsdb128(a.as_i32x4(), _mm_setzero_si128().as_i8x16(), k)) +} + +/// Convert packed signed 64-bit integers in a to packed 32-bit integers with signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtsepi64_epi32&expand=1852) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsqd))] +pub unsafe fn _mm512_cvtsepi64_epi32(a: __m512i) -> __m256i { + transmute(vpmovsqd( + a.as_i64x8(), + _mm256_setzero_si256().as_i32x8(), + 0b11111111, + )) +} + +/// Convert packed signed 64-bit integers in a to packed 32-bit integers with signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtsepi64_epi32&expand=1853) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsqd))] +pub unsafe fn _mm512_mask_cvtsepi64_epi32(src: __m256i, k: __mmask8, a: __m512i) -> __m256i { + transmute(vpmovsqd(a.as_i64x8(), src.as_i32x8(), k)) +} + +/// Convert packed signed 64-bit integers in a to packed 32-bit integers with signed saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtsepi64_epi32&expand=1854) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsqd))] +pub unsafe fn _mm512_maskz_cvtsepi64_epi32(k: __mmask8, a: __m512i) -> __m256i { + transmute(vpmovsqd(a.as_i64x8(), _mm256_setzero_si256().as_i32x8(), k)) +} + +/// Convert packed signed 64-bit integers in a to packed 32-bit integers with signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvtsepi64_epi32&expand=1849) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqd))] +pub unsafe fn _mm256_cvtsepi64_epi32(a: __m256i) -> __m128i { + transmute(vpmovsqd256( + a.as_i64x4(), + _mm_setzero_si128().as_i32x4(), + 0b11111111, + )) +} + +/// Convert packed signed 64-bit integers in a to packed 32-bit integers with signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtsepi64_epi32&expand=1850) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqd))] +pub unsafe fn _mm256_mask_cvtsepi64_epi32(src: __m128i, k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovsqd256(a.as_i64x4(), src.as_i32x4(), k)) +} + +/// Convert packed signed 64-bit integers in a to packed 32-bit integers with signed saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtsepi64_epi32&expand=1851) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqd))] +pub unsafe fn _mm256_maskz_cvtsepi64_epi32(k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovsqd256(a.as_i64x4(), _mm_setzero_si128().as_i32x4(), k)) +} + +/// Convert packed signed 64-bit integers in a to packed 32-bit integers with signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtsepi64_epi32&expand=1846) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqd))] +pub unsafe fn _mm_cvtsepi64_epi32(a: __m128i) -> __m128i { + transmute(vpmovsqd128( + a.as_i64x2(), + _mm_setzero_si128().as_i32x4(), + 0b11111111, + )) +} + +/// Convert packed signed 64-bit integers in a to packed 32-bit integers with signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtsepi64_epi32&expand=1847) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqd))] +pub unsafe fn _mm_mask_cvtsepi64_epi32(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovsqd128(a.as_i64x2(), src.as_i32x4(), k)) +} + +/// Convert packed signed 64-bit integers in a to packed 32-bit integers with signed saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtsepi64_epi32&expand=1848) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqd))] +pub unsafe fn _mm_maskz_cvtsepi64_epi32(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovsqd128(a.as_i64x2(), _mm_setzero_si128().as_i32x4(), k)) +} + +/// Convert packed signed 64-bit integers in a to packed 16-bit integers with signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtsepi64_epi16&expand=1843) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsqw))] +pub unsafe fn _mm512_cvtsepi64_epi16(a: __m512i) -> __m128i { + transmute(vpmovsqw( + a.as_i64x8(), + _mm_setzero_si128().as_i16x8(), + 0b11111111, + )) +} + +/// Convert packed signed 64-bit integers in a to packed 16-bit integers with signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtsepi64_epi16&expand=1844) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsqw))] +pub unsafe fn _mm512_mask_cvtsepi64_epi16(src: __m128i, k: __mmask8, a: __m512i) -> __m128i { + transmute(vpmovsqw(a.as_i64x8(), src.as_i16x8(), k)) +} + +/// Convert packed signed 64-bit integers in a to packed 16-bit integers with signed saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtsepi64_epi16&expand=1845) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsqw))] +pub unsafe fn _mm512_maskz_cvtsepi64_epi16(k: __mmask8, a: __m512i) -> __m128i { + transmute(vpmovsqw(a.as_i64x8(), _mm_setzero_si128().as_i16x8(), k)) +} + +/// Convert packed signed 64-bit integers in a to packed 16-bit integers with signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvtsepi64_epi16&expand=1840) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqw))] +pub unsafe fn _mm256_cvtsepi64_epi16(a: __m256i) -> __m128i { + transmute(vpmovsqw256( + a.as_i64x4(), + _mm_setzero_si128().as_i16x8(), + 0b11111111, + )) +} + +/// Convert packed signed 64-bit integers in a to packed 16-bit integers with signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtsepi64_epi16&expand=1841) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqw))] +pub unsafe fn _mm256_mask_cvtsepi64_epi16(src: __m128i, k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovsqw256(a.as_i64x4(), src.as_i16x8(), k)) +} + +/// Convert packed signed 64-bit integers in a to packed 16-bit integers with signed saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtsepi64_epi16&expand=1842) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqw))] +pub unsafe fn _mm256_maskz_cvtsepi64_epi16(k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovsqw256(a.as_i64x4(), _mm_setzero_si128().as_i16x8(), k)) +} + +/// Convert packed signed 64-bit integers in a to packed 16-bit integers with signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtsepi64_epi16&expand=1837) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqw))] +pub unsafe fn _mm_cvtsepi64_epi16(a: __m128i) -> __m128i { + transmute(vpmovsqw128( + a.as_i64x2(), + _mm_setzero_si128().as_i16x8(), + 0b11111111, + )) +} + +/// Convert packed signed 64-bit integers in a to packed 16-bit integers with signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtsepi64_epi16&expand=1838) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqw))] +pub unsafe fn _mm_mask_cvtsepi64_epi16(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovsqw128(a.as_i64x2(), src.as_i16x8(), k)) +} + +/// Convert packed signed 64-bit integers in a to packed 16-bit integers with signed saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtsepi64_epi16&expand=1839) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqw))] +pub unsafe fn _mm_maskz_cvtsepi64_epi16(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovsqw128(a.as_i64x2(), _mm_setzero_si128().as_i16x8(), k)) +} + +/// Convert packed signed 64-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtsepi64_epi8&expand=1861) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsqb))] +pub unsafe fn _mm512_cvtsepi64_epi8(a: __m512i) -> __m128i { + transmute(vpmovsqb( + a.as_i64x8(), + _mm_setzero_si128().as_i8x16(), + 0b11111111, + )) +} + +/// Convert packed signed 64-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtsepi64_epi8&expand=1862) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsqb))] +pub unsafe fn _mm512_mask_cvtsepi64_epi8(src: __m128i, k: __mmask8, a: __m512i) -> __m128i { + transmute(vpmovsqb(a.as_i64x8(), src.as_i8x16(), k)) +} + +/// Convert packed signed 64-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtsepi64_epi8&expand=1863) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsqb))] +pub unsafe fn _mm512_maskz_cvtsepi64_epi8(k: __mmask8, a: __m512i) -> __m128i { + transmute(vpmovsqb(a.as_i64x8(), _mm_setzero_si128().as_i8x16(), k)) +} + +/// Convert packed signed 64-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvtsepi64_epi8&expand=1858) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqb))] +pub unsafe fn _mm256_cvtsepi64_epi8(a: __m256i) -> __m128i { + transmute(vpmovsqb256( + a.as_i64x4(), + _mm_setzero_si128().as_i8x16(), + 0b11111111, + )) +} + +/// Convert packed signed 64-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtsepi64_epi8&expand=1859) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqb))] +pub unsafe fn _mm256_mask_cvtsepi64_epi8(src: __m128i, k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovsqb256(a.as_i64x4(), src.as_i8x16(), k)) +} + +/// Convert packed signed 64-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtsepi64_epi8&expand=1860) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqb))] +pub unsafe fn _mm256_maskz_cvtsepi64_epi8(k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovsqb256(a.as_i64x4(), _mm_setzero_si128().as_i8x16(), k)) +} + +/// Convert packed signed 64-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtsepi64_epi8&expand=1855) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqb))] +pub unsafe fn _mm_cvtsepi64_epi8(a: __m128i) -> __m128i { + transmute(vpmovsqb128( + a.as_i64x2(), + _mm_setzero_si128().as_i8x16(), + 0b11111111, + )) +} + +/// Convert packed signed 64-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtsepi64_epi8&expand=1856) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqb))] +pub unsafe fn _mm_mask_cvtsepi64_epi8(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovsqb128(a.as_i64x2(), src.as_i8x16(), k)) +} + +/// Convert packed signed 64-bit integers in a to packed 8-bit integers with signed saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtsepi64_epi8&expand=1857) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqb))] +pub unsafe fn _mm_maskz_cvtsepi64_epi8(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovsqb128(a.as_i64x2(), _mm_setzero_si128().as_i8x16(), k)) +} + +/// Convert packed unsigned 32-bit integers in a to packed unsigned 16-bit integers with unsigned saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtusepi32_epi16&expand=2054) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovusdw))] +pub unsafe fn _mm512_cvtusepi32_epi16(a: __m512i) -> __m256i { + transmute(vpmovusdw( + a.as_u32x16(), + _mm256_setzero_si256().as_u16x16(), + 0b11111111_11111111, + )) +} + +/// Convert packed unsigned 32-bit integers in a to packed unsigned 16-bit integers with unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtusepi32_epi16&expand=2055) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovusdw))] +pub unsafe fn _mm512_mask_cvtusepi32_epi16(src: __m256i, k: __mmask16, a: __m512i) -> __m256i { + transmute(vpmovusdw(a.as_u32x16(), src.as_u16x16(), k)) +} + +/// Convert packed unsigned 32-bit integers in a to packed unsigned 16-bit integers with unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtusepi32_epi16&expand=2056) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovusdw))] +pub unsafe fn _mm512_maskz_cvtusepi32_epi16(k: __mmask16, a: __m512i) -> __m256i { + transmute(vpmovusdw( + a.as_u32x16(), + _mm256_setzero_si256().as_u16x16(), + k, + )) +} + +/// Convert packed unsigned 32-bit integers in a to packed unsigned 16-bit integers with unsigned saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvtusepi32_epi16&expand=2051) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusdw))] +pub unsafe fn _mm256_cvtusepi32_epi16(a: __m256i) -> __m128i { + transmute(vpmovusdw256( + a.as_u32x8(), + _mm_setzero_si128().as_u16x8(), + 0b11111111, + )) +} + +/// Convert packed unsigned 32-bit integers in a to packed unsigned 16-bit integers with unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtusepi32_epi16&expand=2052) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusdw))] +pub unsafe fn _mm256_mask_cvtusepi32_epi16(src: __m128i, k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovusdw256(a.as_u32x8(), src.as_u16x8(), k)) +} + +/// Convert packed unsigned 32-bit integers in a to packed unsigned 16-bit integers with unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtusepi32_epi16&expand=2053) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusdw))] +pub unsafe fn _mm256_maskz_cvtusepi32_epi16(k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovusdw256( + a.as_u32x8(), + _mm_setzero_si128().as_u16x8(), + k, + )) +} + +/// Convert packed unsigned 32-bit integers in a to packed unsigned 16-bit integers with unsigned saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtusepi32_epi16&expand=2048) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusdw))] +pub unsafe fn _mm_cvtusepi32_epi16(a: __m128i) -> __m128i { + transmute(vpmovusdw128( + a.as_u32x4(), + _mm_setzero_si128().as_u16x8(), + 0b11111111, + )) +} + +/// Convert packed unsigned 32-bit integers in a to packed unsigned 16-bit integers with unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtusepi32_epi16&expand=2049) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusdw))] +pub unsafe fn _mm_mask_cvtusepi32_epi16(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovusdw128(a.as_u32x4(), src.as_u16x8(), k)) +} + +/// Convert packed unsigned 32-bit integers in a to packed unsigned 16-bit integers with unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtusepi32_epi16&expand=2050) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusdw))] +pub unsafe fn _mm_maskz_cvtusepi32_epi16(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovusdw128( + a.as_u32x4(), + _mm_setzero_si128().as_u16x8(), + k, + )) +} + +/// Convert packed unsigned 32-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtusepi32_epi8&expand=2063) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovusdb))] +pub unsafe fn _mm512_cvtusepi32_epi8(a: __m512i) -> __m128i { + transmute(vpmovusdb( + a.as_u32x16(), + _mm_setzero_si128().as_u8x16(), + 0b11111111_11111111, + )) +} + +/// Convert packed unsigned 32-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtusepi32_epi8&expand=2064) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovusdb))] +pub unsafe fn _mm512_mask_cvtusepi32_epi8(src: __m128i, k: __mmask16, a: __m512i) -> __m128i { + transmute(vpmovusdb(a.as_u32x16(), src.as_u8x16(), k)) +} + +/// Convert packed unsigned 32-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtusepi32_epi8&expand=2065) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovusdb))] +pub unsafe fn _mm512_maskz_cvtusepi32_epi8(k: __mmask16, a: __m512i) -> __m128i { + transmute(vpmovusdb(a.as_u32x16(), _mm_setzero_si128().as_u8x16(), k)) +} + +/// Convert packed unsigned 32-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvtusepi32_epi8&expand=2060) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusdb))] +pub unsafe fn _mm256_cvtusepi32_epi8(a: __m256i) -> __m128i { + transmute(vpmovusdb256( + a.as_u32x8(), + _mm_setzero_si128().as_u8x16(), + 0b11111111, + )) +} + +/// Convert packed unsigned 32-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtusepi32_epi8&expand=2061) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusdb))] +pub unsafe fn _mm256_mask_cvtusepi32_epi8(src: __m128i, k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovusdb256(a.as_u32x8(), src.as_u8x16(), k)) +} + +/// Convert packed unsigned 32-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtusepi32_epi8&expand=2062) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusdb))] +pub unsafe fn _mm256_maskz_cvtusepi32_epi8(k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovusdb256( + a.as_u32x8(), + _mm_setzero_si128().as_u8x16(), + k, + )) +} + +/// Convert packed unsigned 32-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtusepi32_epi8&expand=2057) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusdb))] +pub unsafe fn _mm_cvtusepi32_epi8(a: __m128i) -> __m128i { + transmute(vpmovusdb128( + a.as_u32x4(), + _mm_setzero_si128().as_u8x16(), + 0b11111111, + )) +} + +/// Convert packed unsigned 32-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtusepi32_epi8&expand=2058) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusdb))] +pub unsafe fn _mm_mask_cvtusepi32_epi8(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovusdb128(a.as_u32x4(), src.as_u8x16(), k)) +} + +/// Convert packed unsigned 32-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtusepi32_epi8&expand=2059) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusdb))] +pub unsafe fn _mm_maskz_cvtusepi32_epi8(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovusdb128( + a.as_u32x4(), + _mm_setzero_si128().as_u8x16(), + k, + )) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 32-bit integers with unsigned saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtusepi64_epi32&expand=2087) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovusqd))] +pub unsafe fn _mm512_cvtusepi64_epi32(a: __m512i) -> __m256i { + transmute(vpmovusqd( + a.as_u64x8(), + _mm256_setzero_si256().as_u32x8(), + 0b11111111, + )) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 32-bit integers with unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtusepi64_epi32&expand=2088) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovusqd))] +pub unsafe fn _mm512_mask_cvtusepi64_epi32(src: __m256i, k: __mmask8, a: __m512i) -> __m256i { + transmute(vpmovusqd(a.as_u64x8(), src.as_u32x8(), k)) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 32-bit integers with unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtusepi64_epi32&expand=2089) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovusqd))] +pub unsafe fn _mm512_maskz_cvtusepi64_epi32(k: __mmask8, a: __m512i) -> __m256i { + transmute(vpmovusqd( + a.as_u64x8(), + _mm256_setzero_si256().as_u32x8(), + k, + )) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 32-bit integers with unsigned saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvtusepi64_epi32&expand=2084) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqd))] +pub unsafe fn _mm256_cvtusepi64_epi32(a: __m256i) -> __m128i { + transmute(vpmovusqd256( + a.as_u64x4(), + _mm_setzero_si128().as_u32x4(), + 0b11111111, + )) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 32-bit integers with unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtusepi64_epi32&expand=2085) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqd))] +pub unsafe fn _mm256_mask_cvtusepi64_epi32(src: __m128i, k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovusqd256(a.as_u64x4(), src.as_u32x4(), k)) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 32-bit integers with unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtusepi64_epi32&expand=2086) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqd))] +pub unsafe fn _mm256_maskz_cvtusepi64_epi32(k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovusqd256( + a.as_u64x4(), + _mm_setzero_si128().as_u32x4(), + k, + )) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 32-bit integers with unsigned saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtusepi64_epi32&expand=2081) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqd))] +pub unsafe fn _mm_cvtusepi64_epi32(a: __m128i) -> __m128i { + transmute(vpmovusqd128( + a.as_u64x2(), + _mm_setzero_si128().as_u32x4(), + 0b11111111, + )) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 32-bit integers with unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtusepi64_epi32&expand=2082) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqd))] +pub unsafe fn _mm_mask_cvtusepi64_epi32(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovusqd128(a.as_u64x2(), src.as_u32x4(), k)) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 32-bit integers with unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtusepi64_epi32&expand=2083) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqd))] +pub unsafe fn _mm_maskz_cvtusepi64_epi32(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovusqd128( + a.as_u64x2(), + _mm_setzero_si128().as_u32x4(), + k, + )) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 16-bit integers with unsigned saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtusepi64_epi16&expand=2078) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovusqw))] +pub unsafe fn _mm512_cvtusepi64_epi16(a: __m512i) -> __m128i { + transmute(vpmovusqw( + a.as_u64x8(), + _mm_setzero_si128().as_u16x8(), + 0b11111111, + )) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 16-bit integers with unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtusepi64_epi16&expand=2079) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovusqw))] +pub unsafe fn _mm512_mask_cvtusepi64_epi16(src: __m128i, k: __mmask8, a: __m512i) -> __m128i { + transmute(vpmovusqw(a.as_u64x8(), src.as_u16x8(), k)) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 16-bit integers with unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtusepi64_epi16&expand=2080) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovusqw))] +pub unsafe fn _mm512_maskz_cvtusepi64_epi16(k: __mmask8, a: __m512i) -> __m128i { + transmute(vpmovusqw(a.as_u64x8(), _mm_setzero_si128().as_u16x8(), k)) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 16-bit integers with unsigned saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvtusepi64_epi16&expand=2075) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqw))] +pub unsafe fn _mm256_cvtusepi64_epi16(a: __m256i) -> __m128i { + transmute(vpmovusqw256( + a.as_u64x4(), + _mm_setzero_si128().as_u16x8(), + 0b11111111, + )) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 16-bit integers with unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtusepi64_epi16&expand=2076) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqw))] +pub unsafe fn _mm256_mask_cvtusepi64_epi16(src: __m128i, k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovusqw256(a.as_u64x4(), src.as_u16x8(), k)) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 16-bit integers with unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtusepi64_epi16&expand=2077) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqw))] +pub unsafe fn _mm256_maskz_cvtusepi64_epi16(k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovusqw256( + a.as_u64x4(), + _mm_setzero_si128().as_u16x8(), + k, + )) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 16-bit integers with unsigned saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtusepi64_epi16&expand=2072) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqw))] +pub unsafe fn _mm_cvtusepi64_epi16(a: __m128i) -> __m128i { + transmute(vpmovusqw128( + a.as_u64x2(), + _mm_setzero_si128().as_u16x8(), + 0b11111111, + )) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 16-bit integers with unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtusepi64_epi16&expand=2073) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqw))] +pub unsafe fn _mm_mask_cvtusepi64_epi16(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovusqw128(a.as_u64x2(), src.as_u16x8(), k)) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 16-bit integers with unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtusepi64_epi16&expand=2074) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqw))] +pub unsafe fn _mm_maskz_cvtusepi64_epi16(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovusqw128( + a.as_u64x2(), + _mm_setzero_si128().as_u16x8(), + k, + )) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtusepi64_epi8&expand=2096) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovusqb))] +pub unsafe fn _mm512_cvtusepi64_epi8(a: __m512i) -> __m128i { + transmute(vpmovusqb( + a.as_u64x8(), + _mm_setzero_si128().as_u8x16(), + 0b11111111, + )) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtusepi64_epi8&expand=2097) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovusqb))] +pub unsafe fn _mm512_mask_cvtusepi64_epi8(src: __m128i, k: __mmask8, a: __m512i) -> __m128i { + transmute(vpmovusqb(a.as_u64x8(), src.as_u8x16(), k)) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtusepi64_epi8&expand=2098) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovusqb))] +pub unsafe fn _mm512_maskz_cvtusepi64_epi8(k: __mmask8, a: __m512i) -> __m128i { + transmute(vpmovusqb(a.as_u64x8(), _mm_setzero_si128().as_u8x16(), k)) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvtusepi64_epi8&expand=2093) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqb))] +pub unsafe fn _mm256_cvtusepi64_epi8(a: __m256i) -> __m128i { + transmute(vpmovusqb256( + a.as_u64x4(), + _mm_setzero_si128().as_u8x16(), + 0b11111111, + )) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtusepi64_epi8&expand=2094) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqb))] +pub unsafe fn _mm256_mask_cvtusepi64_epi8(src: __m128i, k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovusqb256(a.as_u64x4(), src.as_u8x16(), k)) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtusepi64_epi8&expand=2095) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqb))] +pub unsafe fn _mm256_maskz_cvtusepi64_epi8(k: __mmask8, a: __m256i) -> __m128i { + transmute(vpmovusqb256( + a.as_u64x4(), + _mm_setzero_si128().as_u8x16(), + k, + )) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtusepi64_epi8&expand=2090) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqb))] +pub unsafe fn _mm_cvtusepi64_epi8(a: __m128i) -> __m128i { + transmute(vpmovusqb128( + a.as_u64x2(), + _mm_setzero_si128().as_u8x16(), + 0b11111111, + )) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtusepi64_epi8&expand=2091) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqb))] +pub unsafe fn _mm_mask_cvtusepi64_epi8(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovusqb128(a.as_u64x2(), src.as_u8x16(), k)) +} + +/// Convert packed unsigned 64-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtusepi64_epi8&expand=2092) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqb))] +pub unsafe fn _mm_maskz_cvtusepi64_epi8(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpmovusqb128( + a.as_u64x2(), + _mm_setzero_si128().as_u8x16(), + k, + )) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed 32-bit integers, and store the results in dst. +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of: +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvt_roundps_epi32&expand=1335) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2dq, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvt_roundps_epi32(a: __m512) -> __m512i { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + let r = vcvtps2dq(a, zero, 0b11111111_11111111, ROUNDING); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvt_roundps_epi32&expand=1336) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2dq, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvt_roundps_epi32( + src: __m512i, + k: __mmask16, + a: __m512, +) -> __m512i { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let src = src.as_i32x16(); + let r = vcvtps2dq(a, src, k, ROUNDING); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvt_roundps_epi32&expand=1337) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2dq, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvt_roundps_epi32( + k: __mmask16, + a: __m512, +) -> __m512i { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + let r = vcvtps2dq(a, zero, k, ROUNDING); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvt_roundps_epu32&expand=1341) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2udq, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvt_roundps_epu32(a: __m512) -> __m512i { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let zero = _mm512_setzero_si512().as_u32x16(); + let r = vcvtps2udq(a, zero, 0b11111111_11111111, ROUNDING); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvt_roundps_epu32&expand=1342) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2udq, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvt_roundps_epu32( + src: __m512i, + k: __mmask16, + a: __m512, +) -> __m512i { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let src = src.as_u32x16(); + let r = vcvtps2udq(a, src, k, ROUNDING); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvt_roundps_epu32&expand=1343) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2udq, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvt_roundps_epu32( + k: __mmask16, + a: __m512, +) -> __m512i { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x16(); + let zero = _mm512_setzero_si512().as_u32x16(); + let r = vcvtps2udq(a, zero, k, ROUNDING); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed double-precision (64-bit) floating-point elements, and store the results in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvt_roundps_pd&expand=1347) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2pd, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvt_roundps_pd(a: __m256) -> __m512d { + static_assert_sae!(SAE); + let a = a.as_f32x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + let r = vcvtps2pd(a, zero, 0b11111111, SAE); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed double-precision (64-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvt_roundps_epi32&expand=1336) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2pd, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvt_roundps_pd( + src: __m512d, + k: __mmask8, + a: __m256, +) -> __m512d { + static_assert_sae!(SAE); + let a = a.as_f32x8(); + let src = src.as_f64x8(); + let r = vcvtps2pd(a, src, k, SAE); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed double-precision (64-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvt_roundps_epi32&expand=1337) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2pd, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvt_roundps_pd(k: __mmask8, a: __m256) -> __m512d { + static_assert_sae!(SAE); + let a = a.as_f32x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + let r = vcvtps2pd(a, zero, k, SAE); + transmute(r) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed 32-bit integers, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvt_roundpd_epi32&expand=1315) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtpd2dq, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvt_roundpd_epi32(a: __m512d) -> __m256i { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + let r = vcvtpd2dq(a, zero, 0b11111111, ROUNDING); + transmute(r) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvt_roundpd_epi32&expand=1316) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtpd2dq, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvt_roundpd_epi32( + src: __m256i, + k: __mmask8, + a: __m512d, +) -> __m256i { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let src = src.as_i32x8(); + let r = vcvtpd2dq(a, src, k, ROUNDING); + transmute(r) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/IntrinsicsGuide/#text=_mm512_maskz_cvt_roundpd_epi32&expand=1317) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtpd2dq, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvt_roundpd_epi32( + k: __mmask8, + a: __m512d, +) -> __m256i { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + let r = vcvtpd2dq(a, zero, k, ROUNDING); + transmute(r) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvt_roundpd_epu32&expand=1321) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtpd2udq, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvt_roundpd_epu32(a: __m512d) -> __m256i { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let zero = _mm256_setzero_si256().as_u32x8(); + let r = vcvtpd2udq(a, zero, 0b11111111, ROUNDING); + transmute(r) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvt_roundpd_epu32&expand=1322) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtpd2udq, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvt_roundpd_epu32( + src: __m256i, + k: __mmask8, + a: __m512d, +) -> __m256i { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let src = src.as_u32x8(); + let r = vcvtpd2udq(a, src, k, ROUNDING); + transmute(r) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/IntrinsicsGuide/#text=_mm512_maskz_cvt_roundpd_epu32&expand=1323) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtpd2udq, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvt_roundpd_epu32( + k: __mmask8, + a: __m512d, +) -> __m256i { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let zero = _mm256_setzero_si256().as_u32x8(); + let r = vcvtpd2udq(a, zero, k, ROUNDING); + transmute(r) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed single-precision (32-bit) floating-point elements, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvt_roundpd_ps&expand=1327) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtpd2ps, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvt_roundpd_ps(a: __m512d) -> __m256 { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + let r = vcvtpd2ps(a, zero, 0b11111111, ROUNDING); + transmute(r) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvt_roundpd_ps&expand=1328) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtpd2ps, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvt_roundpd_ps( + src: __m256, + k: __mmask8, + a: __m512d, +) -> __m256 { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let src = src.as_f32x8(); + let r = vcvtpd2ps(a, src, k, ROUNDING); + transmute(r) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvt_roundpd_ps&expand=1329) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtpd2ps, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvt_roundpd_ps(k: __mmask8, a: __m512d) -> __m256 { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + let r = vcvtpd2ps(a, zero, k, ROUNDING); + transmute(r) +} + +/// Convert packed signed 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvt_roundepi32_ps&expand=1294) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtdq2ps, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvt_roundepi32_ps(a: __m512i) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_i32x16(); + let r = vcvtdq2ps(a, ROUNDING); + transmute(r) +} + +/// Convert packed signed 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvt_roundepi32_ps&expand=1295) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtdq2ps, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvt_roundepi32_ps( + src: __m512, + k: __mmask16, + a: __m512i, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_i32x16(); + let r = vcvtdq2ps(a, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f32x16())) +} + +/// Convert packed signed 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvt_roundepi32_ps&expand=1296) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtdq2ps, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvt_roundepi32_ps( + k: __mmask16, + a: __m512i, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_i32x16(); + let r = vcvtdq2ps(a, ROUNDING); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Convert packed unsigned 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvt_roundepu32_ps&expand=1303) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtudq2ps, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvt_roundepu32_ps(a: __m512i) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_u32x16(); + let r = vcvtudq2ps(a, ROUNDING); + transmute(r) +} + +/// Convert packed unsigned 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvt_roundepu32_ps&expand=1304) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtudq2ps, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvt_roundepu32_ps( + src: __m512, + k: __mmask16, + a: __m512i, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_u32x16(); + let r = vcvtudq2ps(a, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f32x16())) +} + +/// Convert packed unsigned 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvt_roundepu32_ps&expand=1305) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtudq2ps, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvt_roundepu32_ps( + k: __mmask16, + a: __m512i, +) -> __m512 { + static_assert_rounding!(ROUNDING); + let a = a.as_u32x16(); + let r = vcvtudq2ps(a, ROUNDING); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed half-precision (16-bit) floating-point elements, and store the results in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvt_roundps_ph&expand=1354) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2ph, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvt_roundps_ph(a: __m512) -> __m256i { + static_assert_sae!(SAE); + let a = a.as_f32x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + let r = vcvtps2ph(a, SAE, zero, 0b11111111_11111111); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed half-precision (16-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvt_roundps_ph&expand=1355) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2ph, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvt_roundps_ph( + src: __m256i, + k: __mmask16, + a: __m512, +) -> __m256i { + static_assert_sae!(SAE); + let a = a.as_f32x16(); + let src = src.as_i16x16(); + let r = vcvtps2ph(a, SAE, src, k); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed half-precision (16-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvt_roundps_ph&expand=1356) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2ph, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvt_roundps_ph(k: __mmask16, a: __m512) -> __m256i { + static_assert_sae!(SAE); + let a = a.as_f32x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + let r = vcvtps2ph(a, SAE, zero, k); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed half-precision (16-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of: +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvt_roundps_ph&expand=1352) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtps2ph, IMM8 = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_cvt_roundps_ph( + src: __m128i, + k: __mmask8, + a: __m256, +) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_f32x8(); + let src = src.as_i16x8(); + let r = vcvtps2ph256(a, IMM8, src, k); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed half-precision (16-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvt_roundps_ph&expand=1353) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtps2ph, IMM8 = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_cvt_roundps_ph(k: __mmask8, a: __m256) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_f32x8(); + let zero = _mm_setzero_si128().as_i16x8(); + let r = vcvtps2ph256(a, IMM8, zero, k); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed half-precision (16-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvt_roundps_ph&expand=1350) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtps2ph, IMM8 = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_cvt_roundps_ph( + src: __m128i, + k: __mmask8, + a: __m128, +) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_f32x4(); + let src = src.as_i16x8(); + let r = vcvtps2ph128(a, IMM8, src, k); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed half-precision (16-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvt_roundps_ph&expand=1351) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtps2ph, IMM8 = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_cvt_roundps_ph(k: __mmask8, a: __m128) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_f32x4(); + let zero = _mm_setzero_si128().as_i16x8(); + let r = vcvtps2ph128(a, IMM8, zero, k); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed half-precision (16-bit) floating-point elements, and store the results in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtps_ph&expand=1778) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2ph, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvtps_ph(a: __m512) -> __m256i { + static_assert_sae!(SAE); + let a = a.as_f32x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + let r = vcvtps2ph(a, SAE, zero, 0b11111111_11111111); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed half-precision (16-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtps_ph&expand=1779) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2ph, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvtps_ph( + src: __m256i, + k: __mmask16, + a: __m512, +) -> __m256i { + static_assert_sae!(SAE); + let a = a.as_f32x16(); + let src = src.as_i16x16(); + let r = vcvtps2ph(a, SAE, src, k); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed half-precision (16-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtps_ph&expand=1780) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtps2ph, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvtps_ph(k: __mmask16, a: __m512) -> __m256i { + static_assert_sae!(SAE); + let a = a.as_f32x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + let r = vcvtps2ph(a, SAE, zero, k); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed half-precision (16-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtps_ph&expand=1776) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtps2ph, IMM8 = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_cvtps_ph( + src: __m128i, + k: __mmask8, + a: __m256, +) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_f32x8(); + let src = src.as_i16x8(); + let r = vcvtps2ph256(a, IMM8, src, k); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed half-precision (16-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtps_ph&expand=1777) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtps2ph, IMM8 = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_cvtps_ph(k: __mmask8, a: __m256) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_f32x8(); + let zero = _mm_setzero_si128().as_i16x8(); + let r = vcvtps2ph256(a, IMM8, zero, k); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed half-precision (16-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtps_ph&expand=1773) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtps2ph, IMM8 = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_cvtps_ph(src: __m128i, k: __mmask8, a: __m128) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_f32x4(); + let src = src.as_i16x8(); + let r = vcvtps2ph128(a, IMM8, src, k); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed half-precision (16-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtps_ph&expand=1774) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtps2ph, IMM8 = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_cvtps_ph(k: __mmask8, a: __m128) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_f32x4(); + let zero = _mm_setzero_si128().as_i16x8(); + let r = vcvtps2ph128(a, IMM8, zero, k); + transmute(r) +} + +/// Convert packed half-precision (16-bit) floating-point elements in a to packed single-precision (32-bit) floating-point elements, and store the results in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvt_roundph_ps&expand=1332) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtph2ps, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvt_roundph_ps(a: __m256i) -> __m512 { + static_assert_sae!(SAE); + let a = a.as_i16x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + let r = vcvtph2ps(a, zero, 0b11111111_11111111, SAE); + transmute(r) +} + +/// Convert packed half-precision (16-bit) floating-point elements in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvt_roundph_ps&expand=1333) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtph2ps, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvt_roundph_ps( + src: __m512, + k: __mmask16, + a: __m256i, +) -> __m512 { + static_assert_sae!(SAE); + let a = a.as_i16x16(); + let src = src.as_f32x16(); + let r = vcvtph2ps(a, src, k, SAE); + transmute(r) +} + +/// Convert packed half-precision (16-bit) floating-point elements in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvt_roundph_ps&expand=1334) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtph2ps, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvt_roundph_ps(k: __mmask16, a: __m256i) -> __m512 { + static_assert_sae!(SAE); + let a = a.as_i16x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + let r = vcvtph2ps(a, zero, k, SAE); + transmute(r) +} + +/// Convert packed half-precision (16-bit) floating-point elements in a to packed single-precision (32-bit) floating-point elements, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtph_ps&expand=1723) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtph2ps))] +pub unsafe fn _mm512_cvtph_ps(a: __m256i) -> __m512 { + transmute(vcvtph2ps( + a.as_i16x16(), + _mm512_setzero_ps().as_f32x16(), + 0b11111111_11111111, + _MM_FROUND_NO_EXC, + )) +} + +/// Convert packed half-precision (16-bit) floating-point elements in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtph_ps&expand=1724) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtph2ps))] +pub unsafe fn _mm512_mask_cvtph_ps(src: __m512, k: __mmask16, a: __m256i) -> __m512 { + transmute(vcvtph2ps( + a.as_i16x16(), + src.as_f32x16(), + k, + _MM_FROUND_NO_EXC, + )) +} + +/// Convert packed half-precision (16-bit) floating-point elements in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtph_ps&expand=1725) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtph2ps))] +pub unsafe fn _mm512_maskz_cvtph_ps(k: __mmask16, a: __m256i) -> __m512 { + transmute(vcvtph2ps( + a.as_i16x16(), + _mm512_setzero_ps().as_f32x16(), + k, + _MM_FROUND_NO_EXC, + )) +} + +/// Convert packed half-precision (16-bit) floating-point elements in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtph_ps&expand=1721) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtph2ps))] +pub unsafe fn _mm256_mask_cvtph_ps(src: __m256, k: __mmask8, a: __m128i) -> __m256 { + let convert = _mm256_cvtph_ps(a); + transmute(simd_select_bitmask(k, convert.as_f32x8(), src.as_f32x8())) +} + +/// Convert packed half-precision (16-bit) floating-point elements in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtph_ps&expand=1722) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtph2ps))] +pub unsafe fn _mm256_maskz_cvtph_ps(k: __mmask8, a: __m128i) -> __m256 { + let convert = _mm256_cvtph_ps(a); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, convert.as_f32x8(), zero)) +} + +/// Convert packed half-precision (16-bit) floating-point elements in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtph_ps&expand=1718) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtph2ps))] +pub unsafe fn _mm_mask_cvtph_ps(src: __m128, k: __mmask8, a: __m128i) -> __m128 { + let convert = _mm_cvtph_ps(a); + transmute(simd_select_bitmask(k, convert.as_f32x4(), src.as_f32x4())) +} + +/// Convert packed half-precision (16-bit) floating-point elements in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtph_ps&expand=1719) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvtph2ps))] +pub unsafe fn _mm_maskz_cvtph_ps(k: __mmask8, a: __m128i) -> __m128 { + let convert = _mm_cvtph_ps(a); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, convert.as_f32x4(), zero)) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed 32-bit integers with truncation, and store the results in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtt_roundps_epi32&expand=1916) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttps2dq, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvtt_roundps_epi32(a: __m512) -> __m512i { + static_assert_sae!(SAE); + let a = a.as_f32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + let r = vcvttps2dq(a, zero, 0b11111111_11111111, SAE); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed 32-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtt_roundps_epi32&expand=1917) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttps2dq, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvtt_roundps_epi32( + src: __m512i, + k: __mmask16, + a: __m512, +) -> __m512i { + static_assert_sae!(SAE); + let a = a.as_f32x16(); + let src = src.as_i32x16(); + let r = vcvttps2dq(a, src, k, SAE); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed 32-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtt_roundps_epi32&expand=1918) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttps2dq, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvtt_roundps_epi32(k: __mmask16, a: __m512) -> __m512i { + static_assert_sae!(SAE); + let a = a.as_f32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + let r = vcvttps2dq(a, zero, k, SAE); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtt_roundps_epu32&expand=1922) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttps2udq, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvtt_roundps_epu32(a: __m512) -> __m512i { + static_assert_sae!(SAE); + let a = a.as_f32x16(); + let zero = _mm512_setzero_si512().as_u32x16(); + let r = vcvttps2udq(a, zero, 0b11111111_11111111, SAE); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtt_roundps_epu32&expand=1923) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttps2udq, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvtt_roundps_epu32( + src: __m512i, + k: __mmask16, + a: __m512, +) -> __m512i { + static_assert_sae!(SAE); + let a = a.as_f32x16(); + let src = src.as_u32x16(); + let r = vcvttps2udq(a, src, k, SAE); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtt_roundps_epu32&expand=1924) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttps2udq, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvtt_roundps_epu32(k: __mmask16, a: __m512) -> __m512i { + static_assert_sae!(SAE); + let a = a.as_f32x16(); + let zero = _mm512_setzero_si512().as_u32x16(); + let r = vcvttps2udq(a, zero, k, SAE); + transmute(r) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed 32-bit integers with truncation, and store the results in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtt_roundpd_epi32&expand=1904) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttpd2dq, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvtt_roundpd_epi32(a: __m512d) -> __m256i { + static_assert_sae!(SAE); + let a = a.as_f64x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + let r = vcvttpd2dq(a, zero, 0b11111111, SAE); + transmute(r) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed 32-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtt_roundpd_epi32&expand=1905) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttpd2dq, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvtt_roundpd_epi32( + src: __m256i, + k: __mmask8, + a: __m512d, +) -> __m256i { + static_assert_sae!(SAE); + let a = a.as_f64x8(); + let src = src.as_i32x8(); + let r = vcvttpd2dq(a, src, k, SAE); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed 32-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtt_roundps_epi32&expand=1918) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttpd2dq, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvtt_roundpd_epi32(k: __mmask8, a: __m512d) -> __m256i { + static_assert_sae!(SAE); + let a = a.as_f64x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + let r = vcvttpd2dq(a, zero, k, SAE); + transmute(r) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtt_roundpd_epu32&expand=1910) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttpd2udq, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvtt_roundpd_epu32(a: __m512d) -> __m256i { + static_assert_sae!(SAE); + let a = a.as_f64x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + let r = vcvttpd2udq(a, zero, 0b11111111, SAE); + transmute(r) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtt_roundpd_epu32&expand=1911) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttpd2udq, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvtt_roundpd_epu32( + src: __m256i, + k: __mmask8, + a: __m512d, +) -> __m256i { + static_assert_sae!(SAE); + let a = a.as_f64x8(); + let src = src.as_i32x8(); + let r = vcvttpd2udq(a, src, k, SAE); + transmute(r) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed 32-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvttps_epi32&expand=1984) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttps2dq))] +pub unsafe fn _mm512_cvttps_epi32(a: __m512) -> __m512i { + transmute(vcvttps2dq( + a.as_f32x16(), + _mm512_setzero_si512().as_i32x16(), + 0b11111111_11111111, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed 32-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvttps_epi32&expand=1985) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttps2dq))] +pub unsafe fn _mm512_mask_cvttps_epi32(src: __m512i, k: __mmask16, a: __m512) -> __m512i { + transmute(vcvttps2dq( + a.as_f32x16(), + src.as_i32x16(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed 32-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvttps_epi32&expand=1986) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttps2dq))] +pub unsafe fn _mm512_maskz_cvttps_epi32(k: __mmask16, a: __m512) -> __m512i { + transmute(vcvttps2dq( + a.as_f32x16(), + _mm512_setzero_si512().as_i32x16(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed 32-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvttps_epi32&expand=1982) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvttps2dq))] +pub unsafe fn _mm256_mask_cvttps_epi32(src: __m256i, k: __mmask8, a: __m256) -> __m256i { + transmute(vcvttps2dq256(a.as_f32x8(), src.as_i32x8(), k)) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed 32-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvttps_epi32&expand=1983) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvttps2dq))] +pub unsafe fn _mm256_maskz_cvttps_epi32(k: __mmask8, a: __m256) -> __m256i { + transmute(vcvttps2dq256( + a.as_f32x8(), + _mm256_setzero_si256().as_i32x8(), + k, + )) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed 32-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvttps_epi32&expand=1979) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvttps2dq))] +pub unsafe fn _mm_mask_cvttps_epi32(src: __m128i, k: __mmask8, a: __m128) -> __m128i { + transmute(vcvttps2dq128(a.as_f32x4(), src.as_i32x4(), k)) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed 32-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvttps_epi32&expand=1980) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvttps2dq))] +pub unsafe fn _mm_maskz_cvttps_epi32(k: __mmask8, a: __m128) -> __m128i { + transmute(vcvttps2dq128( + a.as_f32x4(), + _mm_setzero_si128().as_i32x4(), + k, + )) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvttps_epu32&expand=2002) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttps2udq))] +pub unsafe fn _mm512_cvttps_epu32(a: __m512) -> __m512i { + transmute(vcvttps2udq( + a.as_f32x16(), + _mm512_setzero_si512().as_u32x16(), + 0b11111111_11111111, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed double-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvttps_epu32&expand=2003) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttps2udq))] +pub unsafe fn _mm512_mask_cvttps_epu32(src: __m512i, k: __mmask16, a: __m512) -> __m512i { + transmute(vcvttps2udq( + a.as_f32x16(), + src.as_u32x16(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed double-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvttps_epu32&expand=2004) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttps2udq))] +pub unsafe fn _mm512_maskz_cvttps_epu32(k: __mmask16, a: __m512) -> __m512i { + transmute(vcvttps2udq( + a.as_f32x16(), + _mm512_setzero_si512().as_u32x16(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvttps_epu32&expand=1999) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvttps2udq))] +pub unsafe fn _mm256_cvttps_epu32(a: __m256) -> __m256i { + transmute(vcvttps2udq256( + a.as_f32x8(), + _mm256_setzero_si256().as_u32x8(), + 0b11111111, + )) +} + +/// Convert packed double-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvttps_epu32&expand=2000) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvttps2udq))] +pub unsafe fn _mm256_mask_cvttps_epu32(src: __m256i, k: __mmask8, a: __m256) -> __m256i { + transmute(vcvttps2udq256(a.as_f32x8(), src.as_u32x8(), k)) +} + +/// Convert packed double-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvttps_epu32&expand=2001) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvttps2udq))] +pub unsafe fn _mm256_maskz_cvttps_epu32(k: __mmask8, a: __m256) -> __m256i { + transmute(vcvttps2udq256( + a.as_f32x8(), + _mm256_setzero_si256().as_u32x8(), + k, + )) +} + +/// Convert packed single-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttps_epu32&expand=1996) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvttps2udq))] +pub unsafe fn _mm_cvttps_epu32(a: __m128) -> __m128i { + transmute(vcvttps2udq128( + a.as_f32x4(), + _mm_setzero_si128().as_u32x4(), + 0b11111111, + )) +} + +/// Convert packed double-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvttps_epu32&expand=1997) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvttps2udq))] +pub unsafe fn _mm_mask_cvttps_epu32(src: __m128i, k: __mmask8, a: __m128) -> __m128i { + transmute(vcvttps2udq128(a.as_f32x4(), src.as_u32x4(), k)) +} + +/// Convert packed double-precision (32-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvttps_epu32&expand=1998) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvttps2udq))] +pub unsafe fn _mm_maskz_cvttps_epu32(k: __mmask8, a: __m128) -> __m128i { + transmute(vcvttps2udq128( + a.as_f32x4(), + _mm_setzero_si128().as_u32x4(), + k, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtt_roundpd_epu32&expand=1912) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttpd2udq, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvtt_roundpd_epu32(k: __mmask8, a: __m512d) -> __m256i { + static_assert_sae!(SAE); + let a = a.as_f64x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + let r = vcvttpd2udq(a, zero, k, SAE); + transmute(r) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed 32-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvttpd_epi32&expand=1947) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttpd2dq))] +pub unsafe fn _mm512_cvttpd_epi32(a: __m512d) -> __m256i { + transmute(vcvttpd2dq( + a.as_f64x8(), + _mm256_setzero_si256().as_i32x8(), + 0b11111111, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed 32-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvttpd_epi32&expand=1948) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttpd2dq))] +pub unsafe fn _mm512_mask_cvttpd_epi32(src: __m256i, k: __mmask8, a: __m512d) -> __m256i { + transmute(vcvttpd2dq( + a.as_f64x8(), + src.as_i32x8(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed 32-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvttpd_epi32&expand=1949) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttpd2dq))] +pub unsafe fn _mm512_maskz_cvttpd_epi32(k: __mmask8, a: __m512d) -> __m256i { + transmute(vcvttpd2dq( + a.as_f64x8(), + _mm256_setzero_si256().as_i32x8(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed 32-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvttpd_epi32&expand=1945) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvttpd2dq))] +pub unsafe fn _mm256_mask_cvttpd_epi32(src: __m128i, k: __mmask8, a: __m256d) -> __m128i { + transmute(vcvttpd2dq256(a.as_f64x4(), src.as_i32x4(), k)) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed 32-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvttpd_epi32&expand=1946) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvttpd2dq))] +pub unsafe fn _mm256_maskz_cvttpd_epi32(k: __mmask8, a: __m256d) -> __m128i { + transmute(vcvttpd2dq256( + a.as_f64x4(), + _mm_setzero_si128().as_i32x4(), + k, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed 32-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvttpd_epi32&expand=1942) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvttpd2dq))] +pub unsafe fn _mm_mask_cvttpd_epi32(src: __m128i, k: __mmask8, a: __m128d) -> __m128i { + transmute(vcvttpd2dq128(a.as_f64x2(), src.as_i32x4(), k)) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed 32-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvttpd_epi32&expand=1943) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvttpd2dq))] +pub unsafe fn _mm_maskz_cvttpd_epi32(k: __mmask8, a: __m128d) -> __m128i { + transmute(vcvttpd2dq128( + a.as_f64x2(), + _mm_setzero_si128().as_i32x4(), + k, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvttpd_epu32&expand=1965) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttpd2udq))] +pub unsafe fn _mm512_cvttpd_epu32(a: __m512d) -> __m256i { + transmute(vcvttpd2udq( + a.as_f64x8(), + _mm256_setzero_si256().as_i32x8(), + 0b11111111, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvttpd_epu32&expand=1966) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttpd2udq))] +pub unsafe fn _mm512_mask_cvttpd_epu32(src: __m256i, k: __mmask8, a: __m512d) -> __m256i { + transmute(vcvttpd2udq( + a.as_f64x8(), + src.as_i32x8(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvttpd_epu32&expand=1967) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvttpd2udq))] +pub unsafe fn _mm512_maskz_cvttpd_epu32(k: __mmask8, a: __m512d) -> __m256i { + transmute(vcvttpd2udq( + a.as_f64x8(), + _mm256_setzero_si256().as_i32x8(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cvttpd_epu32&expand=1962) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvttpd2udq))] +pub unsafe fn _mm256_cvttpd_epu32(a: __m256d) -> __m128i { + transmute(vcvttpd2udq256( + a.as_f64x4(), + _mm_setzero_si128().as_i32x4(), + 0b11111111, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvttpd_epu32&expand=1963) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvttpd2udq))] +pub unsafe fn _mm256_mask_cvttpd_epu32(src: __m128i, k: __mmask8, a: __m256d) -> __m128i { + transmute(vcvttpd2udq256(a.as_f64x4(), src.as_i32x4(), k)) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvttpd_epu32&expand=1964) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvttpd2udq))] +pub unsafe fn _mm256_maskz_cvttpd_epu32(k: __mmask8, a: __m256d) -> __m128i { + transmute(vcvttpd2udq256( + a.as_f64x4(), + _mm_setzero_si128().as_i32x4(), + k, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttpd_epu32&expand=1959) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvttpd2udq))] +pub unsafe fn _mm_cvttpd_epu32(a: __m128d) -> __m128i { + transmute(vcvttpd2udq128( + a.as_f64x2(), + _mm_setzero_si128().as_i32x4(), + 0b11111111, + )) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvttpd_epu32&expand=1960) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvttpd2udq))] +pub unsafe fn _mm_mask_cvttpd_epu32(src: __m128i, k: __mmask8, a: __m128d) -> __m128i { + transmute(vcvttpd2udq128(a.as_f64x2(), src.as_i32x4(), k)) +} + +/// Convert packed double-precision (64-bit) floating-point elements in a to packed unsigned 32-bit integers with truncation, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvttpd_epu32&expand=1961) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcvttpd2udq))] +pub unsafe fn _mm_maskz_cvttpd_epu32(k: __mmask8, a: __m128d) -> __m128i { + transmute(vcvttpd2udq128( + a.as_f64x2(), + _mm_setzero_si128().as_i32x4(), + k, + )) +} + +/// Returns vector of type `__m512d` with all elements set to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_setzero_pd&expand=5018) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vxorps))] +pub unsafe fn _mm512_setzero_pd() -> __m512d { + // All-0 is a properly initialized __m512d + mem::zeroed() +} + +/// Returns vector of type `__m512d` with all elements set to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_setzero_ps&expand=5021) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vxorps))] +pub unsafe fn _mm512_setzero_ps() -> __m512 { + // All-0 is a properly initialized __m512 + mem::zeroed() +} + +/// Return vector of type __m512 with all elements set to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_setzero&expand=5014) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vxorps))] +pub unsafe fn _mm512_setzero() -> __m512 { + // All-0 is a properly initialized __m512 + mem::zeroed() +} + +/// Returns vector of type `__m512i` with all elements set to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_setzero_si512&expand=5024) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vxorps))] +pub unsafe fn _mm512_setzero_si512() -> __m512i { + // All-0 is a properly initialized __m512i + mem::zeroed() +} + +/// Return vector of type __m512i with all elements set to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_setzero_epi32&expand=5015) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vxorps))] +pub unsafe fn _mm512_setzero_epi32() -> __m512i { + // All-0 is a properly initialized __m512i + mem::zeroed() +} + +/// Sets packed 32-bit integers in `dst` with the supplied values in reverse +/// order. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_setr_epi32&expand=4991) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_setr_epi32( + e15: i32, + e14: i32, + e13: i32, + e12: i32, + e11: i32, + e10: i32, + e9: i32, + e8: i32, + e7: i32, + e6: i32, + e5: i32, + e4: i32, + e3: i32, + e2: i32, + e1: i32, + e0: i32, +) -> __m512i { + let r = i32x16( + e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0, + ); + transmute(r) +} + +/// Set packed 8-bit integers in dst with the supplied values. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_set_epi8&expand=4915) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_set_epi8( + e63: i8, + e62: i8, + e61: i8, + e60: i8, + e59: i8, + e58: i8, + e57: i8, + e56: i8, + e55: i8, + e54: i8, + e53: i8, + e52: i8, + e51: i8, + e50: i8, + e49: i8, + e48: i8, + e47: i8, + e46: i8, + e45: i8, + e44: i8, + e43: i8, + e42: i8, + e41: i8, + e40: i8, + e39: i8, + e38: i8, + e37: i8, + e36: i8, + e35: i8, + e34: i8, + e33: i8, + e32: i8, + e31: i8, + e30: i8, + e29: i8, + e28: i8, + e27: i8, + e26: i8, + e25: i8, + e24: i8, + e23: i8, + e22: i8, + e21: i8, + e20: i8, + e19: i8, + e18: i8, + e17: i8, + e16: i8, + e15: i8, + e14: i8, + e13: i8, + e12: i8, + e11: i8, + e10: i8, + e9: i8, + e8: i8, + e7: i8, + e6: i8, + e5: i8, + e4: i8, + e3: i8, + e2: i8, + e1: i8, + e0: i8, +) -> __m512i { + let r = i8x64( + e0, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13, e14, e15, e16, e17, e18, e19, + e20, e21, e22, e23, e24, e25, e26, e27, e28, e29, e30, e31, e32, e33, e34, e35, e36, e37, + e38, e39, e40, e41, e42, e43, e44, e45, e46, e47, e48, e49, e50, e51, e52, e53, e54, e55, + e56, e57, e58, e59, e60, e61, e62, e63, + ); + transmute(r) +} + +/// Set packed 16-bit integers in dst with the supplied values. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_set_epi16&expand=4905) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_set_epi16( + e31: i16, + e30: i16, + e29: i16, + e28: i16, + e27: i16, + e26: i16, + e25: i16, + e24: i16, + e23: i16, + e22: i16, + e21: i16, + e20: i16, + e19: i16, + e18: i16, + e17: i16, + e16: i16, + e15: i16, + e14: i16, + e13: i16, + e12: i16, + e11: i16, + e10: i16, + e9: i16, + e8: i16, + e7: i16, + e6: i16, + e5: i16, + e4: i16, + e3: i16, + e2: i16, + e1: i16, + e0: i16, +) -> __m512i { + let r = i16x32( + e0, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13, e14, e15, e16, e17, e18, e19, + e20, e21, e22, e23, e24, e25, e26, e27, e28, e29, e30, e31, + ); + transmute(r) +} + +/// Set packed 32-bit integers in dst with the repeated 4 element sequence. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_set4_epi32&expand=4982) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_set4_epi32(d: i32, c: i32, b: i32, a: i32) -> __m512i { + _mm512_set_epi32(d, c, b, a, d, c, b, a, d, c, b, a, d, c, b, a) +} + +/// Set packed single-precision (32-bit) floating-point elements in dst with the repeated 4 element sequence. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_set4_ps&expand=4985) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_set4_ps(d: f32, c: f32, b: f32, a: f32) -> __m512 { + _mm512_set_ps(d, c, b, a, d, c, b, a, d, c, b, a, d, c, b, a) +} + +/// Set packed double-precision (64-bit) floating-point elements in dst with the repeated 4 element sequence. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_set4_pd&expand=4984) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_set4_pd(d: f64, c: f64, b: f64, a: f64) -> __m512d { + _mm512_set_pd(d, c, b, a, d, c, b, a) +} + +/// Set packed 32-bit integers in dst with the repeated 4 element sequence in reverse order. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_setr4_epi32&expand=5009) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_setr4_epi32(d: i32, c: i32, b: i32, a: i32) -> __m512i { + _mm512_set_epi32(a, b, c, d, a, b, c, d, a, b, c, d, a, b, c, d) +} + +/// Set packed single-precision (32-bit) floating-point elements in dst with the repeated 4 element sequence in reverse order. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_setr4_ps&expand=5012) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_setr4_ps(d: f32, c: f32, b: f32, a: f32) -> __m512 { + _mm512_set_ps(a, b, c, d, a, b, c, d, a, b, c, d, a, b, c, d) +} + +/// Set packed double-precision (64-bit) floating-point elements in dst with the repeated 4 element sequence in reverse order. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_setr4_pd&expand=5011) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_setr4_pd(d: f64, c: f64, b: f64, a: f64) -> __m512d { + _mm512_set_pd(a, b, c, d, a, b, c, d) +} + +/// Set packed 64-bit integers in dst with the supplied values. +/// +/// [Intel's documentation]( https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_set_epi64&expand=4910) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_set_epi64( + e0: i64, + e1: i64, + e2: i64, + e3: i64, + e4: i64, + e5: i64, + e6: i64, + e7: i64, +) -> __m512i { + _mm512_setr_epi64(e7, e6, e5, e4, e3, e2, e1, e0) +} + +/// Set packed 64-bit integers in dst with the supplied values in reverse order. +/// +/// [Intel's documentation]( https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_setr_epi64&expand=4993) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_setr_epi64( + e0: i64, + e1: i64, + e2: i64, + e3: i64, + e4: i64, + e5: i64, + e6: i64, + e7: i64, +) -> __m512i { + let r = i64x8::new(e0, e1, e2, e3, e4, e5, e6, e7); + transmute(r) +} + +/// Gather double-precision (64-bit) floating-point elements from memory using 32-bit indices. 64-bit elements are loaded from addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale). Gathered elements are merged into dst. scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i32gather_pd&expand=3002) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgatherdpd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_i32gather_pd(offsets: __m256i, slice: *const u8) -> __m512d { + static_assert_imm8_scale!(SCALE); + let zero = _mm512_setzero_pd().as_f64x8(); + let neg_one = -1; + let slice = slice as *const i8; + let offsets = offsets.as_i32x8(); + let r = vgatherdpd(zero, slice, offsets, neg_one, SCALE); + transmute(r) +} + +/// Gather double-precision (64-bit) floating-point elements from memory using 32-bit indices. 64-bit elements are loaded from addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale). Gathered elements are merged into dst using writemask k (elements are copied from src when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i32gather_pd&expand=3003) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgatherdpd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i32gather_pd( + src: __m512d, + mask: __mmask8, + offsets: __m256i, + slice: *const u8, +) -> __m512d { + static_assert_imm8_scale!(SCALE); + let src = src.as_f64x8(); + let slice = slice as *const i8; + let offsets = offsets.as_i32x8(); + let r = vgatherdpd(src, slice, offsets, mask as i8, SCALE); + transmute(r) +} + +/// Gather double-precision (64-bit) floating-point elements from memory using 64-bit indices. 64-bit elements are loaded from addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale). Gathered elements are merged into dst. scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i64gather_pd&expand=3092) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgatherqpd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_i64gather_pd(offsets: __m512i, slice: *const u8) -> __m512d { + static_assert_imm8_scale!(SCALE); + let zero = _mm512_setzero_pd().as_f64x8(); + let neg_one = -1; + let slice = slice as *const i8; + let offsets = offsets.as_i64x8(); + let r = vgatherqpd(zero, slice, offsets, neg_one, SCALE); + transmute(r) +} + +/// Gather double-precision (64-bit) floating-point elements from memory using 64-bit indices. 64-bit elements are loaded from addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale). Gathered elements are merged into dst using writemask k (elements are copied from src when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i64gather_pd&expand=3093) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgatherqpd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i64gather_pd( + src: __m512d, + mask: __mmask8, + offsets: __m512i, + slice: *const u8, +) -> __m512d { + static_assert_imm8_scale!(SCALE); + let src = src.as_f64x8(); + let slice = slice as *const i8; + let offsets = offsets.as_i64x8(); + let r = vgatherqpd(src, slice, offsets, mask as i8, SCALE); + transmute(r) +} + +/// Gather single-precision (32-bit) floating-point elements from memory using 64-bit indices. 32-bit elements are loaded from addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale). Gathered elements are merged into dst. scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i64gather_ps&expand=3100) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgatherqps, SCALE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_i64gather_ps(offsets: __m512i, slice: *const u8) -> __m256 { + static_assert_imm8_scale!(SCALE); + let zero = _mm256_setzero_ps().as_f32x8(); + let neg_one = -1; + let slice = slice as *const i8; + let offsets = offsets.as_i64x8(); + let r = vgatherqps(zero, slice, offsets, neg_one, SCALE); + transmute(r) +} + +/// Gather single-precision (32-bit) floating-point elements from memory using 64-bit indices. 32-bit elements are loaded from addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale). Gathered elements are merged into dst using writemask k (elements are copied from src when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i64gather_ps&expand=3101) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgatherqps, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i64gather_ps( + src: __m256, + mask: __mmask8, + offsets: __m512i, + slice: *const u8, +) -> __m256 { + static_assert_imm8_scale!(SCALE); + let src = src.as_f32x8(); + let slice = slice as *const i8; + let offsets = offsets.as_i64x8(); + let r = vgatherqps(src, slice, offsets, mask as i8, SCALE); + transmute(r) +} + +/// Gather single-precision (32-bit) floating-point elements from memory using 32-bit indices. 32-bit elements are loaded from addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale). Gathered elements are merged into dst. scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i32gather_ps&expand=3010) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgatherdps, SCALE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_i32gather_ps(offsets: __m512i, slice: *const u8) -> __m512 { + static_assert_imm8_scale!(SCALE); + let zero = _mm512_setzero_ps().as_f32x16(); + let neg_one = -1; + let slice = slice as *const i8; + let offsets = offsets.as_i32x16(); + let r = vgatherdps(zero, slice, offsets, neg_one, SCALE); + transmute(r) +} + +/// Gather single-precision (32-bit) floating-point elements from memory using 32-bit indices. 32-bit elements are loaded from addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale). Gathered elements are merged into dst using writemask k (elements are copied from src when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i32gather_ps&expand=3011) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgatherdps, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i32gather_ps( + src: __m512, + mask: __mmask16, + offsets: __m512i, + slice: *const u8, +) -> __m512 { + static_assert_imm8_scale!(SCALE); + let src = src.as_f32x16(); + let slice = slice as *const i8; + let offsets = offsets.as_i32x16(); + let r = vgatherdps(src, slice, offsets, mask as i16, SCALE); + transmute(r) +} + +/// Gather 32-bit integers from memory using 32-bit indices. 32-bit elements are loaded from addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale). Gathered elements are merged into dst. scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i32gather_epi32&expand=2986) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpgatherdd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_i32gather_epi32( + offsets: __m512i, + slice: *const u8, +) -> __m512i { + static_assert_imm8_scale!(SCALE); + let zero = _mm512_setzero_si512().as_i32x16(); + let neg_one = -1; + let slice = slice as *const i8; + let offsets = offsets.as_i32x16(); + let r = vpgatherdd(zero, slice, offsets, neg_one, SCALE); + transmute(r) +} + +/// Gather 32-bit integers from memory using 32-bit indices. 32-bit elements are loaded from addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale). Gathered elements are merged into dst using writemask k (elements are copied from src when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i32gather_epi32&expand=2987) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpgatherdd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i32gather_epi32( + src: __m512i, + mask: __mmask16, + offsets: __m512i, + slice: *const u8, +) -> __m512i { + static_assert_imm8_scale!(SCALE); + let src = src.as_i32x16(); + let mask = mask as i16; + let slice = slice as *const i8; + let offsets = offsets.as_i32x16(); + let r = vpgatherdd(src, slice, offsets, mask, SCALE); + transmute(r) +} + +/// Gather 64-bit integers from memory using 32-bit indices. 64-bit elements are loaded from addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale). Gathered elements are merged into dst. scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i32gather_epi64&expand=2994) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpgatherdq, SCALE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_i32gather_epi64( + offsets: __m256i, + slice: *const u8, +) -> __m512i { + static_assert_imm8_scale!(SCALE); + let zero = _mm512_setzero_si512().as_i64x8(); + let neg_one = -1; + let slice = slice as *const i8; + let offsets = offsets.as_i32x8(); + let r = vpgatherdq(zero, slice, offsets, neg_one, SCALE); + transmute(r) +} + +/// Gather 64-bit integers from memory using 32-bit indices. 64-bit elements are loaded from addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale). Gathered elements are merged into dst using writemask k (elements are copied from src when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i32gather_epi64&expand=2995) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpgatherdq, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i32gather_epi64( + src: __m512i, + mask: __mmask8, + offsets: __m256i, + slice: *const u8, +) -> __m512i { + static_assert_imm8_scale!(SCALE); + let src = src.as_i64x8(); + let mask = mask as i8; + let slice = slice as *const i8; + let offsets = offsets.as_i32x8(); + let r = vpgatherdq(src, slice, offsets, mask, SCALE); + transmute(r) +} + +/// Gather 64-bit integers from memory using 64-bit indices. 64-bit elements are loaded from addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale). Gathered elements are merged into dst. scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i64gather_epi64&expand=3084) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpgatherqq, SCALE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_i64gather_epi64( + offsets: __m512i, + slice: *const u8, +) -> __m512i { + static_assert_imm8_scale!(SCALE); + let zero = _mm512_setzero_si512().as_i64x8(); + let neg_one = -1; + let slice = slice as *const i8; + let offsets = offsets.as_i64x8(); + let r = vpgatherqq(zero, slice, offsets, neg_one, SCALE); + transmute(r) +} + +/// Gather 64-bit integers from memory using 64-bit indices. 64-bit elements are loaded from addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale). Gathered elements are merged into dst using writemask k (elements are copied from src when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i64gather_epi64&expand=3085) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpgatherqq, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i64gather_epi64( + src: __m512i, + mask: __mmask8, + offsets: __m512i, + slice: *const u8, +) -> __m512i { + static_assert_imm8_scale!(SCALE); + let src = src.as_i64x8(); + let mask = mask as i8; + let slice = slice as *const i8; + let offsets = offsets.as_i64x8(); + let r = vpgatherqq(src, slice, offsets, mask, SCALE); + transmute(r) +} + +/// Gather 32-bit integers from memory using 64-bit indices. 32-bit elements are loaded from addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale). Gathered elements are merged into dst. scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i64gather_epi32&expand=3074) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpgatherqd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_i64gather_epi32( + offsets: __m512i, + slice: *const u8, +) -> __m256i { + static_assert_imm8_scale!(SCALE); + let zeros = _mm256_setzero_si256().as_i32x8(); + let neg_one = -1; + let slice = slice as *const i8; + let offsets = offsets.as_i64x8(); + let r = vpgatherqd(zeros, slice, offsets, neg_one, SCALE); + transmute(r) +} + +/// Gather 32-bit integers from memory using 64-bit indices. 32-bit elements are loaded from addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale). Gathered elements are merged into dst using writemask k (elements are copied from src when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i64gather_epi32&expand=3075) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpgatherqd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i64gather_epi32( + src: __m256i, + mask: __mmask8, + offsets: __m512i, + slice: *const u8, +) -> __m256i { + static_assert_imm8_scale!(SCALE); + let src = src.as_i32x8(); + let mask = mask as i8; + let slice = slice as *const i8; + let offsets = offsets.as_i64x8(); + let r = vpgatherqd(src, slice, offsets, mask, SCALE); + transmute(r) +} + +/// Scatter double-precision (64-bit) floating-point elements from a into memory using 32-bit indices. 64-bit elements are stored at addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i32scatter_pd&expand=3044) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscatterdpd, SCALE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_i32scatter_pd( + slice: *mut u8, + offsets: __m256i, + src: __m512d, +) { + static_assert_imm8_scale!(SCALE); + let src = src.as_f64x8(); + let neg_one = -1; + let slice = slice as *mut i8; + let offsets = offsets.as_i32x8(); + vscatterdpd(slice, neg_one, offsets, src, SCALE); +} + +/// Scatter double-precision (64-bit) floating-point elements from a into memory using 32-bit indices. 64-bit elements are stored at addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale) subject to mask k (elements are not stored when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i32scatter_pd&expand=3045) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscatterdpd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i32scatter_pd( + slice: *mut u8, + mask: __mmask8, + offsets: __m256i, + src: __m512d, +) { + static_assert_imm8_scale!(SCALE); + let src = src.as_f64x8(); + let slice = slice as *mut i8; + let offsets = offsets.as_i32x8(); + vscatterdpd(slice, mask as i8, offsets, src, SCALE); +} + +/// Scatter double-precision (64-bit) floating-point elements from a into memory using 64-bit indices. 64-bit elements are stored at addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i64scatter_pd&expand=3122) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscatterqpd, SCALE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_i64scatter_pd( + slice: *mut u8, + offsets: __m512i, + src: __m512d, +) { + static_assert_imm8_scale!(SCALE); + let src = src.as_f64x8(); + let neg_one = -1; + let slice = slice as *mut i8; + let offsets = offsets.as_i64x8(); + vscatterqpd(slice, neg_one, offsets, src, SCALE); +} + +/// Scatter double-precision (64-bit) floating-point elements from a into memory using 64-bit indices. 64-bit elements are stored at addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale) subject to mask k (elements are not stored when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i64scatter_pd&expand=3123) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscatterqpd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i64scatter_pd( + slice: *mut u8, + mask: __mmask8, + offsets: __m512i, + src: __m512d, +) { + static_assert_imm8_scale!(SCALE); + let src = src.as_f64x8(); + let slice = slice as *mut i8; + let offsets = offsets.as_i64x8(); + vscatterqpd(slice, mask as i8, offsets, src, SCALE); +} + +/// Scatter single-precision (32-bit) floating-point elements from a into memory using 32-bit indices. 32-bit elements are stored at addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i32scatter_ps&expand=3050) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscatterdps, SCALE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_i32scatter_ps( + slice: *mut u8, + offsets: __m512i, + src: __m512, +) { + static_assert_imm8_scale!(SCALE); + let src = src.as_f32x16(); + let neg_one = -1; + let slice = slice as *mut i8; + let offsets = offsets.as_i32x16(); + vscatterdps(slice, neg_one, offsets, src, SCALE); +} + +/// Scatter single-precision (32-bit) floating-point elements from a into memory using 32-bit indices. 32-bit elements are stored at addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale) subject to mask k (elements are not stored when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i32scatter_ps&expand=3051) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscatterdps, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i32scatter_ps( + slice: *mut u8, + mask: __mmask16, + offsets: __m512i, + src: __m512, +) { + static_assert_imm8_scale!(SCALE); + let src = src.as_f32x16(); + let slice = slice as *mut i8; + let offsets = offsets.as_i32x16(); + vscatterdps(slice, mask as i16, offsets, src, SCALE); +} + +/// Scatter single-precision (32-bit) floating-point elements from a into memory using 64-bit indices. 32-bit elements are stored at addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale) subject to mask k (elements are not stored when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i64scatter_ps&expand=3128) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscatterqps, SCALE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_i64scatter_ps( + slice: *mut u8, + offsets: __m512i, + src: __m256, +) { + static_assert_imm8_scale!(SCALE); + let src = src.as_f32x8(); + let neg_one = -1; + let slice = slice as *mut i8; + let offsets = offsets.as_i64x8(); + vscatterqps(slice, neg_one, offsets, src, SCALE); +} + +/// Scatter single-precision (32-bit) floating-point elements from a into memory using 64-bit indices. 32-bit elements are stored at addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale) subject to mask k (elements are not stored when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i64scatter_ps&expand=3129) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscatterqps, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i64scatter_ps( + slice: *mut u8, + mask: __mmask8, + offsets: __m512i, + src: __m256, +) { + static_assert_imm8_scale!(SCALE); + let src = src.as_f32x8(); + let slice = slice as *mut i8; + let offsets = offsets.as_i64x8(); + vscatterqps(slice, mask as i8, offsets, src, SCALE); +} + +/// Scatter 64-bit integers from a into memory using 32-bit indices. 64-bit elements are stored at addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i32scatter_epi64&expand=3038) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpscatterdq, SCALE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_i32scatter_epi64( + slice: *mut u8, + offsets: __m256i, + src: __m512i, +) { + static_assert_imm8_scale!(SCALE); + let src = src.as_i64x8(); + let neg_one = -1; + let slice = slice as *mut i8; + let offsets = offsets.as_i32x8(); + vpscatterdq(slice, neg_one, offsets, src, SCALE); +} + +/// Scatter 64-bit integers from a into memory using 32-bit indices. 64-bit elements are stored at addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale) subject to mask k (elements are not stored when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i32scatter_epi64&expand=3039) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpscatterdq, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i32scatter_epi64( + slice: *mut u8, + mask: __mmask8, + offsets: __m256i, + src: __m512i, +) { + static_assert_imm8_scale!(SCALE); + let src = src.as_i64x8(); + let mask = mask as i8; + let slice = slice as *mut i8; + let offsets = offsets.as_i32x8(); + vpscatterdq(slice, mask, offsets, src, SCALE); +} + +/// Scatter 64-bit integers from a into memory using 64-bit indices. 64-bit elements are stored at addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i64scatter_epi64&expand=3116) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpscatterqq, SCALE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_i64scatter_epi64( + slice: *mut u8, + offsets: __m512i, + src: __m512i, +) { + static_assert_imm8_scale!(SCALE); + let src = src.as_i64x8(); + let neg_one = -1; + let slice = slice as *mut i8; + let offsets = offsets.as_i64x8(); + vpscatterqq(slice, neg_one, offsets, src, SCALE); +} + +/// Scatter 64-bit integers from a into memory using 64-bit indices. 64-bit elements are stored at addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale) subject to mask k (elements are not stored when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i64scatter_epi64&expand=3117) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpscatterqq, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i64scatter_epi64( + slice: *mut u8, + mask: __mmask8, + offsets: __m512i, + src: __m512i, +) { + static_assert_imm8_scale!(SCALE); + let src = src.as_i64x8(); + let mask = mask as i8; + let slice = slice as *mut i8; + let offsets = offsets.as_i64x8(); + vpscatterqq(slice, mask, offsets, src, SCALE); +} + +/// Scatter 32-bit integers from a into memory using 32-bit indices. 32-bit elements are stored at addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i32scatter_epi32&expand=3032) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpscatterdd, SCALE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_i32scatter_epi32( + slice: *mut u8, + offsets: __m512i, + src: __m512i, +) { + static_assert_imm8_scale!(SCALE); + let src = src.as_i32x16(); + let neg_one = -1; + let slice = slice as *mut i8; + let offsets = offsets.as_i32x16(); + vpscatterdd(slice, neg_one, offsets, src, SCALE); +} + +/// Scatter 32-bit integers from a into memory using 32-bit indices. 32-bit elements are stored at addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale) subject to mask k (elements are not stored when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i32scatter_epi32&expand=3033) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpscatterdd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i32scatter_epi32( + slice: *mut u8, + mask: __mmask16, + offsets: __m512i, + src: __m512i, +) { + static_assert_imm8_scale!(SCALE); + let src = src.as_i32x16(); + let mask = mask as i16; + let slice = slice as *mut i8; + let offsets = offsets.as_i32x16(); + vpscatterdd(slice, mask, offsets, src, SCALE); +} + +/// Scatter 32-bit integers from a into memory using 64-bit indices. 32-bit elements are stored at addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i64scatter_epi32&expand=3108) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpscatterqd, SCALE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_i64scatter_epi32( + slice: *mut u8, + offsets: __m512i, + src: __m256i, +) { + static_assert_imm8_scale!(SCALE); + let src = src.as_i32x8(); + let neg_one = -1; + let slice = slice as *mut i8; + let offsets = offsets.as_i64x8(); + vpscatterqd(slice, neg_one, offsets, src, SCALE); +} + +/// Scatter 32-bit integers from a into memory using 64-bit indices. 32-bit elements are stored at addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale) subject to mask k (elements are not stored when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i64scatter_epi32&expand=3109) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpscatterqd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i64scatter_epi32( + slice: *mut u8, + mask: __mmask8, + offsets: __m512i, + src: __m256i, +) { + static_assert_imm8_scale!(SCALE); + let src = src.as_i32x8(); + let mask = mask as i8; + let slice = slice as *mut i8; + let offsets = offsets.as_i64x8(); + vpscatterqd(slice, mask, offsets, src, SCALE); +} + +/// Contiguously store the active 32-bit integers in a (those with their respective bit set in writemask k) to dst, and pass through the remaining elements from src. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_compress_epi32&expand=1198) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcompressd))] +pub unsafe fn _mm512_mask_compress_epi32(src: __m512i, k: __mmask16, a: __m512i) -> __m512i { + transmute(vpcompressd(a.as_i32x16(), src.as_i32x16(), k)) +} + +/// Contiguously store the active 32-bit integers in a (those with their respective bit set in zeromask k) to dst, and set the remaining elements to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_compress_epi32&expand=1199) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcompressd))] +pub unsafe fn _mm512_maskz_compress_epi32(k: __mmask16, a: __m512i) -> __m512i { + transmute(vpcompressd( + a.as_i32x16(), + _mm512_setzero_si512().as_i32x16(), + k, + )) +} + +/// Contiguously store the active 32-bit integers in a (those with their respective bit set in writemask k) to dst, and pass through the remaining elements from src. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_compress_epi32&expand=1196) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressd))] +pub unsafe fn _mm256_mask_compress_epi32(src: __m256i, k: __mmask8, a: __m256i) -> __m256i { + transmute(vpcompressd256(a.as_i32x8(), src.as_i32x8(), k)) +} + +/// Contiguously store the active 32-bit integers in a (those with their respective bit set in zeromask k) to dst, and set the remaining elements to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_compress_epi32&expand=1197) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressd))] +pub unsafe fn _mm256_maskz_compress_epi32(k: __mmask8, a: __m256i) -> __m256i { + transmute(vpcompressd256( + a.as_i32x8(), + _mm256_setzero_si256().as_i32x8(), + k, + )) +} + +/// Contiguously store the active 32-bit integers in a (those with their respective bit set in writemask k) to dst, and pass through the remaining elements from src. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_compress_epi32&expand=1194) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressd))] +pub unsafe fn _mm_mask_compress_epi32(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpcompressd128(a.as_i32x4(), src.as_i32x4(), k)) +} + +/// Contiguously store the active 32-bit integers in a (those with their respective bit set in zeromask k) to dst, and set the remaining elements to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_compress_epi32&expand=1195) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressd))] +pub unsafe fn _mm_maskz_compress_epi32(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpcompressd128( + a.as_i32x4(), + _mm_setzero_si128().as_i32x4(), + k, + )) +} + +/// Contiguously store the active 64-bit integers in a (those with their respective bit set in writemask k) to dst, and pass through the remaining elements from src. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_compress_epi64&expand=1204) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcompressq))] +pub unsafe fn _mm512_mask_compress_epi64(src: __m512i, k: __mmask8, a: __m512i) -> __m512i { + transmute(vpcompressq(a.as_i64x8(), src.as_i64x8(), k)) +} + +/// Contiguously store the active 64-bit integers in a (those with their respective bit set in zeromask k) to dst, and set the remaining elements to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_compress_epi64&expand=1205) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcompressq))] +pub unsafe fn _mm512_maskz_compress_epi64(k: __mmask8, a: __m512i) -> __m512i { + transmute(vpcompressq( + a.as_i64x8(), + _mm512_setzero_si512().as_i64x8(), + k, + )) +} + +/// Contiguously store the active 64-bit integers in a (those with their respective bit set in writemask k) to dst, and pass through the remaining elements from src. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_compress_epi64&expand=1202) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressq))] +pub unsafe fn _mm256_mask_compress_epi64(src: __m256i, k: __mmask8, a: __m256i) -> __m256i { + transmute(vpcompressq256(a.as_i64x4(), src.as_i64x4(), k)) +} + +/// Contiguously store the active 64-bit integers in a (those with their respective bit set in zeromask k) to dst, and set the remaining elements to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_compress_epi64&expand=1203) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressq))] +pub unsafe fn _mm256_maskz_compress_epi64(k: __mmask8, a: __m256i) -> __m256i { + transmute(vpcompressq256( + a.as_i64x4(), + _mm256_setzero_si256().as_i64x4(), + k, + )) +} + +/// Contiguously store the active 64-bit integers in a (those with their respective bit set in writemask k) to dst, and pass through the remaining elements from src. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_compress_epi64&expand=1200) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressq))] +pub unsafe fn _mm_mask_compress_epi64(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpcompressq128(a.as_i64x2(), src.as_i64x2(), k)) +} + +/// Contiguously store the active 64-bit integers in a (those with their respective bit set in zeromask k) to dst, and set the remaining elements to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_compress_epi64&expand=1201) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressq))] +pub unsafe fn _mm_maskz_compress_epi64(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpcompressq128( + a.as_i64x2(), + _mm_setzero_si128().as_i64x2(), + k, + )) +} + +/// Contiguously store the active single-precision (32-bit) floating-point elements in a (those with their respective bit set in writemask k) to dst, and pass through the remaining elements from src. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_compress_ps&expand=1222) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcompressps))] +pub unsafe fn _mm512_mask_compress_ps(src: __m512, k: __mmask16, a: __m512) -> __m512 { + transmute(vcompressps(a.as_f32x16(), src.as_f32x16(), k)) +} + +/// Contiguously store the active single-precision (32-bit) floating-point elements in a (those with their respective bit set in zeromask k) to dst, and set the remaining elements to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_compress_ps&expand=1223) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcompressps))] +pub unsafe fn _mm512_maskz_compress_ps(k: __mmask16, a: __m512) -> __m512 { + transmute(vcompressps( + a.as_f32x16(), + _mm512_setzero_ps().as_f32x16(), + k, + )) +} + +/// Contiguously store the active single-precision (32-bit) floating-point elements in a (those with their respective bit set in writemask k) to dst, and pass through the remaining elements from src. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_compress_ps&expand=1220) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcompressps))] +pub unsafe fn _mm256_mask_compress_ps(src: __m256, k: __mmask8, a: __m256) -> __m256 { + transmute(vcompressps256(a.as_f32x8(), src.as_f32x8(), k)) +} + +/// Contiguously store the active single-precision (32-bit) floating-point elements in a (those with their respective bit set in zeromask k) to dst, and set the remaining elements to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_compress_ps&expand=1221) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcompressps))] +pub unsafe fn _mm256_maskz_compress_ps(k: __mmask8, a: __m256) -> __m256 { + transmute(vcompressps256( + a.as_f32x8(), + _mm256_setzero_ps().as_f32x8(), + k, + )) +} + +/// Contiguously store the active single-precision (32-bit) floating-point elements in a (those with their respective bit set in writemask k) to dst, and pass through the remaining elements from src. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_compress_ps&expand=1218) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcompressps))] +pub unsafe fn _mm_mask_compress_ps(src: __m128, k: __mmask8, a: __m128) -> __m128 { + transmute(vcompressps128(a.as_f32x4(), src.as_f32x4(), k)) +} + +/// Contiguously store the active single-precision (32-bit) floating-point elements in a (those with their respective bit set in zeromask k) to dst, and set the remaining elements to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_compress_ps&expand=1219) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcompressps))] +pub unsafe fn _mm_maskz_compress_ps(k: __mmask8, a: __m128) -> __m128 { + transmute(vcompressps128(a.as_f32x4(), _mm_setzero_ps().as_f32x4(), k)) +} + +/// Contiguously store the active double-precision (64-bit) floating-point elements in a (those with their respective bit set in writemask k) to dst, and pass through the remaining elements from src. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_compress_pd&expand=1216) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcompresspd))] +pub unsafe fn _mm512_mask_compress_pd(src: __m512d, k: __mmask8, a: __m512d) -> __m512d { + transmute(vcompresspd(a.as_f64x8(), src.as_f64x8(), k)) +} + +/// Contiguously store the active double-precision (64-bit) floating-point elements in a (those with their respective bit set in zeromask k) to dst, and set the remaining elements to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_compress_pd&expand=1217) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcompresspd))] +pub unsafe fn _mm512_maskz_compress_pd(k: __mmask8, a: __m512d) -> __m512d { + transmute(vcompresspd(a.as_f64x8(), _mm512_setzero_pd().as_f64x8(), k)) +} + +/// Contiguously store the active double-precision (64-bit) floating-point elements in a (those with their respective bit set in writemask k) to dst, and pass through the remaining elements from src. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_compress_pd&expand=1214) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcompresspd))] +pub unsafe fn _mm256_mask_compress_pd(src: __m256d, k: __mmask8, a: __m256d) -> __m256d { + transmute(vcompresspd256(a.as_f64x4(), src.as_f64x4(), k)) +} + +/// Contiguously store the active double-precision (64-bit) floating-point elements in a (those with their respective bit set in zeromask k) to dst, and set the remaining elements to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_compress_pd&expand=1215) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcompresspd))] +pub unsafe fn _mm256_maskz_compress_pd(k: __mmask8, a: __m256d) -> __m256d { + transmute(vcompresspd256( + a.as_f64x4(), + _mm256_setzero_pd().as_f64x4(), + k, + )) +} + +/// Contiguously store the active double-precision (64-bit) floating-point elements in a (those with their respective bit set in writemask k) to dst, and pass through the remaining elements from src. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_compress_pd&expand=1212) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcompresspd))] +pub unsafe fn _mm_mask_compress_pd(src: __m128d, k: __mmask8, a: __m128d) -> __m128d { + transmute(vcompresspd128(a.as_f64x2(), src.as_f64x2(), k)) +} + +/// Contiguously store the active double-precision (64-bit) floating-point elements in a (those with their respective bit set in zeromask k) to dst, and set the remaining elements to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_compress_pd&expand=1213) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcompresspd))] +pub unsafe fn _mm_maskz_compress_pd(k: __mmask8, a: __m128d) -> __m128d { + transmute(vcompresspd128(a.as_f64x2(), _mm_setzero_pd().as_f64x2(), k)) +} + +/// Contiguously store the active 32-bit integers in a (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_compressstoreu_epi32) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcompressd))] +pub unsafe fn _mm512_mask_compressstoreu_epi32(base_addr: *mut u8, k: __mmask16, a: __m512i) { + vcompressstored(base_addr as *mut _, a.as_i32x16(), k) +} + +/// Contiguously store the active 32-bit integers in a (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_compressstoreu_epi32) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressd))] +pub unsafe fn _mm256_mask_compressstoreu_epi32(base_addr: *mut u8, k: __mmask8, a: __m256i) { + vcompressstored256(base_addr as *mut _, a.as_i32x8(), k) +} + +/// Contiguously store the active 32-bit integers in a (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_compressstoreu_epi32) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressd))] +pub unsafe fn _mm_mask_compressstoreu_epi32(base_addr: *mut u8, k: __mmask8, a: __m128i) { + vcompressstored128(base_addr as *mut _, a.as_i32x4(), k) +} + +/// Contiguously store the active 64-bit integers in a (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_compressstoreu_epi64) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcompressq))] +pub unsafe fn _mm512_mask_compressstoreu_epi64(base_addr: *mut u8, k: __mmask8, a: __m512i) { + vcompressstoreq(base_addr as *mut _, a.as_i64x8(), k) +} + +/// Contiguously store the active 64-bit integers in a (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_compressstoreu_epi64) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressq))] +pub unsafe fn _mm256_mask_compressstoreu_epi64(base_addr: *mut u8, k: __mmask8, a: __m256i) { + vcompressstoreq256(base_addr as *mut _, a.as_i64x4(), k) +} + +/// Contiguously store the active 64-bit integers in a (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_compressstoreu_epi64) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressq))] +pub unsafe fn _mm_mask_compressstoreu_epi64(base_addr: *mut u8, k: __mmask8, a: __m128i) { + vcompressstoreq128(base_addr as *mut _, a.as_i64x2(), k) +} + +/// Contiguously store the active single-precision (32-bit) floating-point elements in a (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_compressstoreu_ps) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcompressps))] +pub unsafe fn _mm512_mask_compressstoreu_ps(base_addr: *mut u8, k: __mmask16, a: __m512) { + vcompressstoreps(base_addr as *mut _, a.as_f32x16(), k) +} + +/// Contiguously store the active single-precision (32-bit) floating-point elements in a (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_compressstoreu_ps) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcompressps))] +pub unsafe fn _mm256_mask_compressstoreu_ps(base_addr: *mut u8, k: __mmask8, a: __m256) { + vcompressstoreps256(base_addr as *mut _, a.as_f32x8(), k) +} + +/// Contiguously store the active single-precision (32-bit) floating-point elements in a (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_compressstoreu_ps) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcompressps))] +pub unsafe fn _mm_mask_compressstoreu_ps(base_addr: *mut u8, k: __mmask8, a: __m128) { + vcompressstoreps128(base_addr as *mut _, a.as_f32x4(), k) +} + +/// Contiguously store the active double-precision (64-bit) floating-point elements in a (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_compressstoreu_pd) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcompresspd))] +pub unsafe fn _mm512_mask_compressstoreu_pd(base_addr: *mut u8, k: __mmask8, a: __m512d) { + vcompressstorepd(base_addr as *mut _, a.as_f64x8(), k) +} + +/// Contiguously store the active double-precision (64-bit) floating-point elements in a (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_compressstoreu_pd) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcompresspd))] +pub unsafe fn _mm256_mask_compressstoreu_pd(base_addr: *mut u8, k: __mmask8, a: __m256d) { + vcompressstorepd256(base_addr as *mut _, a.as_f64x4(), k) +} + +/// Contiguously store the active double-precision (64-bit) floating-point elements in a (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_compressstoreu_pd) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vcompresspd))] +pub unsafe fn _mm_mask_compressstoreu_pd(base_addr: *mut u8, k: __mmask8, a: __m128d) { + vcompressstorepd128(base_addr as *mut _, a.as_f64x2(), k) +} + +/// Load contiguous active 32-bit integers from a (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_expand_epi32&expand=2316) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpexpandd))] +pub unsafe fn _mm512_mask_expand_epi32(src: __m512i, k: __mmask16, a: __m512i) -> __m512i { + transmute(vpexpandd(a.as_i32x16(), src.as_i32x16(), k)) +} + +/// Load contiguous active 32-bit integers from a (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_expand_epi32&expand=2317) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpexpandd))] +pub unsafe fn _mm512_maskz_expand_epi32(k: __mmask16, a: __m512i) -> __m512i { + transmute(vpexpandd( + a.as_i32x16(), + _mm512_setzero_si512().as_i32x16(), + k, + )) +} + +/// Load contiguous active 32-bit integers from a (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_expand_epi32&expand=2314) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpexpandd))] +pub unsafe fn _mm256_mask_expand_epi32(src: __m256i, k: __mmask8, a: __m256i) -> __m256i { + transmute(vpexpandd256(a.as_i32x8(), src.as_i32x8(), k)) +} + +/// Load contiguous active 32-bit integers from a (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_expand_epi32&expand=2315) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpexpandd))] +pub unsafe fn _mm256_maskz_expand_epi32(k: __mmask8, a: __m256i) -> __m256i { + transmute(vpexpandd256( + a.as_i32x8(), + _mm256_setzero_si256().as_i32x8(), + k, + )) +} + +/// Load contiguous active 32-bit integers from a (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_expand_epi32&expand=2312) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpexpandd))] +pub unsafe fn _mm_mask_expand_epi32(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpexpandd128(a.as_i32x4(), src.as_i32x4(), k)) +} + +/// Load contiguous active 32-bit integers from a (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_expand_epi32&expand=2313) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpexpandd))] +pub unsafe fn _mm_maskz_expand_epi32(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpexpandd128( + a.as_i32x4(), + _mm_setzero_si128().as_i32x4(), + k, + )) +} + +/// Load contiguous active 64-bit integers from a (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_expand_epi64&expand=2322) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpexpandq))] +pub unsafe fn _mm512_mask_expand_epi64(src: __m512i, k: __mmask8, a: __m512i) -> __m512i { + transmute(vpexpandq(a.as_i64x8(), src.as_i64x8(), k)) +} + +/// Load contiguous active 64-bit integers from a (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_expand_epi64&expand=2323) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpexpandq))] +pub unsafe fn _mm512_maskz_expand_epi64(k: __mmask8, a: __m512i) -> __m512i { + transmute(vpexpandq( + a.as_i64x8(), + _mm512_setzero_si512().as_i64x8(), + k, + )) +} + +/// Load contiguous active 64-bit integers from a (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_expand_epi64&expand=2320) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpexpandq))] +pub unsafe fn _mm256_mask_expand_epi64(src: __m256i, k: __mmask8, a: __m256i) -> __m256i { + transmute(vpexpandq256(a.as_i64x4(), src.as_i64x4(), k)) +} + +/// Load contiguous active 64-bit integers from a (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_expand_epi64&expand=2321) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpexpandq))] +pub unsafe fn _mm256_maskz_expand_epi64(k: __mmask8, a: __m256i) -> __m256i { + transmute(vpexpandq256( + a.as_i64x4(), + _mm256_setzero_si256().as_i64x4(), + k, + )) +} + +/// Load contiguous active 64-bit integers from a (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_expand_epi64&expand=2318) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpexpandq))] +pub unsafe fn _mm_mask_expand_epi64(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpexpandq128(a.as_i64x2(), src.as_i64x2(), k)) +} + +/// Load contiguous active 64-bit integers from a (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_expand_epi64&expand=2319) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpexpandq))] +pub unsafe fn _mm_maskz_expand_epi64(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpexpandq128( + a.as_i64x2(), + _mm_setzero_si128().as_i64x2(), + k, + )) +} + +/// Load contiguous active single-precision (32-bit) floating-point elements from a (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_expand_ps&expand=2340) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vexpandps))] +pub unsafe fn _mm512_mask_expand_ps(src: __m512, k: __mmask16, a: __m512) -> __m512 { + transmute(vexpandps(a.as_f32x16(), src.as_f32x16(), k)) +} + +/// Load contiguous active single-precision (32-bit) floating-point elements from a (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_expand_ps&expand=2341) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vexpandps))] +pub unsafe fn _mm512_maskz_expand_ps(k: __mmask16, a: __m512) -> __m512 { + transmute(vexpandps(a.as_f32x16(), _mm512_setzero_ps().as_f32x16(), k)) +} + +/// Load contiguous active single-precision (32-bit) floating-point elements from a (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_expand_ps&expand=2338) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vexpandps))] +pub unsafe fn _mm256_mask_expand_ps(src: __m256, k: __mmask8, a: __m256) -> __m256 { + transmute(vexpandps256(a.as_f32x8(), src.as_f32x8(), k)) +} + +/// Load contiguous active single-precision (32-bit) floating-point elements from a (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_expand_ps&expand=2339) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vexpandps))] +pub unsafe fn _mm256_maskz_expand_ps(k: __mmask8, a: __m256) -> __m256 { + transmute(vexpandps256( + a.as_f32x8(), + _mm256_setzero_ps().as_f32x8(), + k, + )) +} + +/// Load contiguous active single-precision (32-bit) floating-point elements from a (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_expand_ps&expand=2336) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vexpandps))] +pub unsafe fn _mm_mask_expand_ps(src: __m128, k: __mmask8, a: __m128) -> __m128 { + transmute(vexpandps128(a.as_f32x4(), src.as_f32x4(), k)) +} + +/// Load contiguous active single-precision (32-bit) floating-point elements from a (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_expand_ps&expand=2337) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vexpandps))] +pub unsafe fn _mm_maskz_expand_ps(k: __mmask8, a: __m128) -> __m128 { + transmute(vexpandps128(a.as_f32x4(), _mm_setzero_ps().as_f32x4(), k)) +} + +/// Load contiguous active double-precision (64-bit) floating-point elements from a (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_expand_pd&expand=2334) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vexpandpd))] +pub unsafe fn _mm512_mask_expand_pd(src: __m512d, k: __mmask8, a: __m512d) -> __m512d { + transmute(vexpandpd(a.as_f64x8(), src.as_f64x8(), k)) +} + +/// Load contiguous active double-precision (64-bit) floating-point elements from a (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_expand_pd&expand=2335) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vexpandpd))] +pub unsafe fn _mm512_maskz_expand_pd(k: __mmask8, a: __m512d) -> __m512d { + transmute(vexpandpd(a.as_f64x8(), _mm512_setzero_pd().as_f64x8(), k)) +} + +/// Load contiguous active double-precision (64-bit) floating-point elements from a (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_expand_pd&expand=2332) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vexpandpd))] +pub unsafe fn _mm256_mask_expand_pd(src: __m256d, k: __mmask8, a: __m256d) -> __m256d { + transmute(vexpandpd256(a.as_f64x4(), src.as_f64x4(), k)) +} + +/// Load contiguous active double-precision (64-bit) floating-point elements from a (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_expand_pd&expand=2333) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vexpandpd))] +pub unsafe fn _mm256_maskz_expand_pd(k: __mmask8, a: __m256d) -> __m256d { + transmute(vexpandpd256( + a.as_f64x4(), + _mm256_setzero_pd().as_f64x4(), + k, + )) +} + +/// Load contiguous active double-precision (64-bit) floating-point elements from a (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_expand_pd&expand=2330) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vexpandpd))] +pub unsafe fn _mm_mask_expand_pd(src: __m128d, k: __mmask8, a: __m128d) -> __m128d { + transmute(vexpandpd128(a.as_f64x2(), src.as_f64x2(), k)) +} + +/// Load contiguous active double-precision (64-bit) floating-point elements from a (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_expand_pd&expand=2331) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vexpandpd))] +pub unsafe fn _mm_maskz_expand_pd(k: __mmask8, a: __m128d) -> __m128d { + transmute(vexpandpd128(a.as_f64x2(), _mm_setzero_pd().as_f64x2(), k)) +} + +/// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_rol_epi32&expand=4685) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_rol_epi32(a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_i32x16(); + let r = vprold(a, IMM8); + transmute(r) +} + +/// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_rol_epi32&expand=4683) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_rol_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_i32x16(); + let r = vprold(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i32x16())) +} + +/// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_rol_epi32&expand=4684) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_rol_epi32(k: __mmask16, a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_i32x16(); + let r = vprold(a, IMM8); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_rol_epi32&expand=4682) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_rol_epi32(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_i32x8(); + let r = vprold256(a, IMM8); + transmute(r) +} + +/// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_rol_epi32&expand=4680) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_rol_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_i32x8(); + let r = vprold256(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i32x8())) +} + +/// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_rol_epi32&expand=4681) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_rol_epi32(k: __mmask8, a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_i32x8(); + let r = vprold256(a, IMM8); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_rol_epi32&expand=4679) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_rol_epi32(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_i32x4(); + let r = vprold128(a, IMM8); + transmute(r) +} + +/// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_rol_epi32&expand=4677) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_rol_epi32( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_i32x4(); + let r = vprold128(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i32x4())) +} + +/// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_rol_epi32&expand=4678) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_rol_epi32(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_i32x4(); + let r = vprold128(a, IMM8); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_ror_epi32&expand=4721) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_ror_epi32(a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_i32x16(); + let r = vprord(a, IMM8); + transmute(r) +} + +/// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_ror_epi32&expand=4719) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprold, IMM8 = 123))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_ror_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_i32x16(); + let r = vprord(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i32x16())) +} + +/// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_ror_epi32&expand=4720) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprold, IMM8 = 123))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_ror_epi32(k: __mmask16, a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_i32x16(); + let r = vprord(a, IMM8); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_ror_epi32&expand=4718) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_ror_epi32(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_i32x8(); + let r = vprord256(a, IMM8); + transmute(r) +} + +/// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_ror_epi32&expand=4716) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprold, IMM8 = 123))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_ror_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_i32x8(); + let r = vprord256(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i32x8())) +} + +/// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_ror_epi32&expand=4717) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprold, IMM8 = 123))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_ror_epi32(k: __mmask8, a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_i32x8(); + let r = vprord256(a, IMM8); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_ror_epi32&expand=4715) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_ror_epi32(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_i32x4(); + let r = vprord128(a, IMM8); + transmute(r) +} + +/// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_ror_epi32&expand=4713) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprold, IMM8 = 123))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_ror_epi32( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_i32x4(); + let r = vprord128(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i32x4())) +} + +/// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_ror_epi32&expand=4714) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprold, IMM8 = 123))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_ror_epi32(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_i32x4(); + let r = vprord128(a, IMM8); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_rol_epi64&expand=4694) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_rol_epi64(a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_i64x8(); + let r = vprolq(a, IMM8); + transmute(r) +} + +/// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_rol_epi64&expand=4692) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_rol_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_i64x8(); + let r = vprolq(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i64x8())) +} + +/// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_rol_epi64&expand=4693) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_rol_epi64(k: __mmask8, a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_i64x8(); + let r = vprolq(a, IMM8); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_rol_epi64&expand=4691) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_rol_epi64(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_i64x4(); + let r = vprolq256(a, IMM8); + transmute(r) +} + +/// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_rol_epi64&expand=4689) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_rol_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_i64x4(); + let r = vprolq256(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i64x4())) +} + +/// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_rol_epi64&expand=4690) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_rol_epi64(k: __mmask8, a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_i64x4(); + let r = vprolq256(a, IMM8); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_rol_epi64&expand=4688) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_rol_epi64(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_i64x2(); + let r = vprolq128(a, IMM8); + transmute(r) +} + +/// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_rol_epi64&expand=4686) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_rol_epi64( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_i64x2(); + let r = vprolq128(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i64x2())) +} + +/// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_rol_epi64&expand=4687) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_rol_epi64(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_i64x2(); + let r = vprolq128(a, IMM8); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_ror_epi64&expand=4730) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 15))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_ror_epi64(a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_i64x8(); + let r = vprorq(a, IMM8); + transmute(r) +} + +/// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_ror_epi64&expand=4728) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 15))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_ror_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_i64x8(); + let r = vprorq(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i64x8())) +} + +/// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_ror_epi64&expand=4729) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 15))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_ror_epi64(k: __mmask8, a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_i64x8(); + let r = vprorq(a, IMM8); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_ror_epi64&expand=4727) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 15))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_ror_epi64(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_i64x4(); + let r = vprorq256(a, IMM8); + transmute(r) +} + +/// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_ror_epi64&expand=4725) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 15))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_ror_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_i64x4(); + let r = vprorq256(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i64x4())) +} + +/// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_ror_epi64&expand=4726) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 15))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_ror_epi64(k: __mmask8, a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_i64x4(); + let r = vprorq256(a, IMM8); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_ror_epi64&expand=4724) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 15))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_ror_epi64(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_i64x2(); + let r = vprorq128(a, IMM8); + transmute(r) +} + +/// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_ror_epi64&expand=4722) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 15))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_ror_epi64( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_i64x2(); + let r = vprorq128(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i64x2())) +} + +/// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_ror_epi64&expand=4723) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 15))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_ror_epi64(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_i64x2(); + let r = vprorq128(a, IMM8); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Shift packed 32-bit integers in a left by imm8 while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_slli_epi32&expand=5310) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpslld, IMM8 = 5))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_slli_epi32(a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i32x16(); + let r = vpsllid(a, IMM8); + transmute(r) +} + +/// Shift packed 32-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_slli_epi32&expand=5308) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpslld, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_slli_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, +) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i32x16(); + let shf = vpsllid(a, IMM8); + transmute(simd_select_bitmask(k, shf, src.as_i32x16())) +} + +/// Shift packed 32-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_slli_epi32&expand=5309) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpslld, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_slli_epi32(k: __mmask16, a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i32x16(); + let shf = vpsllid(a, IMM8); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 32-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_slli_epi32&expand=5305) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpslld, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_slli_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psllid256(a.as_i32x8(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i32x8())) +} + +/// Shift packed 32-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_slli_epi32&expand=5306) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpslld, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_slli_epi32(k: __mmask8, a: __m256i) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psllid256(a.as_i32x8(), imm8); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Shift packed 32-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_slli_epi32&expand=5302) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpslld, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_slli_epi32( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psllid128(a.as_i32x4(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i32x4())) +} + +/// Shift packed 32-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_slli_epi32&expand=5303) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpslld, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_slli_epi32(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psllid128(a.as_i32x4(), imm8); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Shift packed 32-bit integers in a right by imm8 while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_srli_epi32&expand=5522) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrld, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_srli_epi32(a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i32x16(); + let r = vpsrlid(a, IMM8); + transmute(r) +} + +/// Shift packed 32-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_srli_epi32&expand=5520) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrld, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_srli_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, +) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i32x16(); + let shf = vpsrlid(a, IMM8); + transmute(simd_select_bitmask(k, shf, src.as_i32x16())) +} + +/// Shift packed 32-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_srli_epi32&expand=5521) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrld, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_srli_epi32(k: __mmask16, a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i32x16(); + let shf = vpsrlid(a, IMM8); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 32-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_srli_epi32&expand=5517) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrld, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_srli_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psrlid256(a.as_i32x8(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i32x8())) +} + +/// Shift packed 32-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_srli_epi32&expand=5518) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrld, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_srli_epi32(k: __mmask8, a: __m256i) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psrlid256(a.as_i32x8(), imm8); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Shift packed 32-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_srli_epi32&expand=5514) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrld, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_srli_epi32( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psrlid128(a.as_i32x4(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i32x4())) +} + +/// Shift packed 32-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_srli_epi32&expand=5515) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrld, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_srli_epi32(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psrlid128(a.as_i32x4(), imm8); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Shift packed 64-bit integers in a left by imm8 while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_slli_epi64&expand=5319) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsllq, IMM8 = 5))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_slli_epi64(a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i64x8(); + let r = vpslliq(a, IMM8); + transmute(r) +} + +/// Shift packed 64-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_slli_epi64&expand=5317) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsllq, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_slli_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, +) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i64x8(); + let shf = vpslliq(a, IMM8); + transmute(simd_select_bitmask(k, shf, src.as_i64x8())) +} + +/// Shift packed 64-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_slli_epi64&expand=5318) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsllq, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_slli_epi64(k: __mmask8, a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i64x8(); + let shf = vpslliq(a, IMM8); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_slli_epi64&expand=5314) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllq, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_slli_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = pslliq256(a.as_i64x4(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i64x4())) +} + +/// Shift packed 64-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_slli_epi64&expand=5315) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllq, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_slli_epi64(k: __mmask8, a: __m256i) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = pslliq256(a.as_i64x4(), imm8); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Shift packed 64-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_slli_epi64&expand=5311) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllq, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_slli_epi64( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = pslliq128(a.as_i64x2(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i64x2())) +} + +/// Shift packed 64-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_slli_epi64&expand=5312) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllq, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_slli_epi64(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = pslliq128(a.as_i64x2(), imm8); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Shift packed 64-bit integers in a right by imm8 while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_srli_epi64&expand=5531) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrlq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_srli_epi64(a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i64x8(); + let r = vpsrliq(a, IMM8); + transmute(r) +} + +/// Shift packed 64-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_srli_epi64&expand=5529) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrlq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_srli_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, +) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i64x8(); + let shf = vpsrliq(a, IMM8); + transmute(simd_select_bitmask(k, shf, src.as_i64x8())) +} + +/// Shift packed 64-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_srli_epi64&expand=5530) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrlq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_srli_epi64(k: __mmask8, a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i64x8(); + let shf = vpsrliq(a, IMM8); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_srli_epi64&expand=5526) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_srli_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psrliq256(a.as_i64x4(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i64x4())) +} + +/// Shift packed 64-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_srli_epi64&expand=5527) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_srli_epi64(k: __mmask8, a: __m256i) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psrliq256(a.as_i64x4(), imm8); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Shift packed 64-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_srli_epi64&expand=5523) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_srli_epi64( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psrliq128(a.as_i64x2(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i64x2())) +} + +/// Shift packed 64-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_srli_epi64&expand=5524) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_srli_epi64(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psrliq128(a.as_i64x2(), imm8); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Shift packed 32-bit integers in a left by count while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sll_epi32&expand=5280) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpslld))] +pub unsafe fn _mm512_sll_epi32(a: __m512i, count: __m128i) -> __m512i { + transmute(vpslld(a.as_i32x16(), count.as_i32x4())) +} + +/// Shift packed 32-bit integers in a left by count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sll_epi32&expand=5278) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpslld))] +pub unsafe fn _mm512_mask_sll_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, + count: __m128i, +) -> __m512i { + let shf = _mm512_sll_epi32(a, count).as_i32x16(); + transmute(simd_select_bitmask(k, shf, src.as_i32x16())) +} + +/// Shift packed 32-bit integers in a left by count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sll_epi32&expand=5279) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpslld))] +pub unsafe fn _mm512_maskz_sll_epi32(k: __mmask16, a: __m512i, count: __m128i) -> __m512i { + let shf = _mm512_sll_epi32(a, count).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 32-bit integers in a left by count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_sll_epi32&expand=5275) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpslld))] +pub unsafe fn _mm256_mask_sll_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, + count: __m128i, +) -> __m256i { + let shf = _mm256_sll_epi32(a, count).as_i32x8(); + transmute(simd_select_bitmask(k, shf, src.as_i32x8())) +} + +/// Shift packed 32-bit integers in a left by count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_sll_epi32&expand=5276) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpslld))] +pub unsafe fn _mm256_maskz_sll_epi32(k: __mmask8, a: __m256i, count: __m128i) -> __m256i { + let shf = _mm256_sll_epi32(a, count).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 32-bit integers in a left by count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_sll_epi32&expand=5272) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpslld))] +pub unsafe fn _mm_mask_sll_epi32(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_sll_epi32(a, count).as_i32x4(); + transmute(simd_select_bitmask(k, shf, src.as_i32x4())) +} + +/// Shift packed 32-bit integers in a left by count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_sll_epi32&expand=5273) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpslld))] +pub unsafe fn _mm_maskz_sll_epi32(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_sll_epi32(a, count).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 32-bit integers in a right by count while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_srl_epi32&expand=5492) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrld))] +pub unsafe fn _mm512_srl_epi32(a: __m512i, count: __m128i) -> __m512i { + transmute(vpsrld(a.as_i32x16(), count.as_i32x4())) +} + +/// Shift packed 32-bit integers in a right by count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_srl_epi32&expand=5490) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrld))] +pub unsafe fn _mm512_mask_srl_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, + count: __m128i, +) -> __m512i { + let shf = _mm512_srl_epi32(a, count).as_i32x16(); + transmute(simd_select_bitmask(k, shf, src.as_i32x16())) +} + +/// Shift packed 32-bit integers in a right by count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_srl_epi32&expand=5491) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrld))] +pub unsafe fn _mm512_maskz_srl_epi32(k: __mmask16, a: __m512i, count: __m128i) -> __m512i { + let shf = _mm512_srl_epi32(a, count).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 32-bit integers in a right by count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_srl_epi32&expand=5487) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrld))] +pub unsafe fn _mm256_mask_srl_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, + count: __m128i, +) -> __m256i { + let shf = _mm256_srl_epi32(a, count).as_i32x8(); + transmute(simd_select_bitmask(k, shf, src.as_i32x8())) +} + +/// Shift packed 32-bit integers in a right by count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_srl_epi32&expand=5488) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrld))] +pub unsafe fn _mm256_maskz_srl_epi32(k: __mmask8, a: __m256i, count: __m128i) -> __m256i { + let shf = _mm256_srl_epi32(a, count).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 32-bit integers in a right by count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_srl_epi32&expand=5484) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrld))] +pub unsafe fn _mm_mask_srl_epi32(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_srl_epi32(a, count).as_i32x4(); + transmute(simd_select_bitmask(k, shf, src.as_i32x4())) +} + +/// Shift packed 32-bit integers in a right by count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_srl_epi32&expand=5485) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrld))] +pub unsafe fn _mm_maskz_srl_epi32(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_srl_epi32(a, count).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a left by count while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sll_epi64&expand=5289) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsllq))] +pub unsafe fn _mm512_sll_epi64(a: __m512i, count: __m128i) -> __m512i { + transmute(vpsllq(a.as_i64x8(), count.as_i64x2())) +} + +/// Shift packed 64-bit integers in a left by count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sll_epi64&expand=5287) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsllq))] +pub unsafe fn _mm512_mask_sll_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, + count: __m128i, +) -> __m512i { + let shf = _mm512_sll_epi64(a, count).as_i64x8(); + transmute(simd_select_bitmask(k, shf, src.as_i64x8())) +} + +/// Shift packed 64-bit integers in a left by count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sll_epi64&expand=5288) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsllq))] +pub unsafe fn _mm512_maskz_sll_epi64(k: __mmask8, a: __m512i, count: __m128i) -> __m512i { + let shf = _mm512_sll_epi64(a, count).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a left by count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_sll_epi64&expand=5284) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllq))] +pub unsafe fn _mm256_mask_sll_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, + count: __m128i, +) -> __m256i { + let shf = _mm256_sll_epi64(a, count).as_i64x4(); + transmute(simd_select_bitmask(k, shf, src.as_i64x4())) +} + +/// Shift packed 64-bit integers in a left by count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_sll_epi64&expand=5285) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllq))] +pub unsafe fn _mm256_maskz_sll_epi64(k: __mmask8, a: __m256i, count: __m128i) -> __m256i { + let shf = _mm256_sll_epi64(a, count).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a left by count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_sll_epi64&expand=5281) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllq))] +pub unsafe fn _mm_mask_sll_epi64(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_sll_epi64(a, count).as_i64x2(); + transmute(simd_select_bitmask(k, shf, src.as_i64x2())) +} + +/// Shift packed 64-bit integers in a left by count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_sll_epi64&expand=5282) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllq))] +pub unsafe fn _mm_maskz_sll_epi64(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_sll_epi64(a, count).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a right by count while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_srl_epi64&expand=5501) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrlq))] +pub unsafe fn _mm512_srl_epi64(a: __m512i, count: __m128i) -> __m512i { + transmute(vpsrlq(a.as_i64x8(), count.as_i64x2())) +} + +/// Shift packed 64-bit integers in a right by count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_srl_epi64&expand=5499) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrlq))] +pub unsafe fn _mm512_mask_srl_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, + count: __m128i, +) -> __m512i { + let shf = _mm512_srl_epi64(a, count).as_i64x8(); + transmute(simd_select_bitmask(k, shf, src.as_i64x8())) +} + +/// Shift packed 64-bit integers in a right by count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_srl_epi64&expand=5500) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrlq))] +pub unsafe fn _mm512_maskz_srl_epi64(k: __mmask8, a: __m512i, count: __m128i) -> __m512i { + let shf = _mm512_srl_epi64(a, count).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a right by count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_srl_epi64&expand=5496) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlq))] +pub unsafe fn _mm256_mask_srl_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, + count: __m128i, +) -> __m256i { + let shf = _mm256_srl_epi64(a, count).as_i64x4(); + transmute(simd_select_bitmask(k, shf, src.as_i64x4())) +} + +/// Shift packed 64-bit integers in a right by count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_srl_epi64&expand=5497) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlq))] +pub unsafe fn _mm256_maskz_srl_epi64(k: __mmask8, a: __m256i, count: __m128i) -> __m256i { + let shf = _mm256_srl_epi64(a, count).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a right by count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_srl_epi64&expand=5493) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlq))] +pub unsafe fn _mm_mask_srl_epi64(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_srl_epi64(a, count).as_i64x2(); + transmute(simd_select_bitmask(k, shf, src.as_i64x2())) +} + +/// Shift packed 64-bit integers in a right by count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_srl_epi64&expand=5494) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlq))] +pub unsafe fn _mm_maskz_srl_epi64(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_srl_epi64(a, count).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 32-bit integers in a right by count while shifting in sign bits, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sra_epi32&expand=5407) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrad))] +pub unsafe fn _mm512_sra_epi32(a: __m512i, count: __m128i) -> __m512i { + transmute(vpsrad(a.as_i32x16(), count.as_i32x4())) +} + +/// Shift packed 32-bit integers in a right by count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sra_epi32&expand=5405) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrad))] +pub unsafe fn _mm512_mask_sra_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, + count: __m128i, +) -> __m512i { + let shf = _mm512_sra_epi32(a, count).as_i32x16(); + transmute(simd_select_bitmask(k, shf, src.as_i32x16())) +} + +/// Shift packed 32-bit integers in a right by count while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sra_epi32&expand=5406) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrad))] +pub unsafe fn _mm512_maskz_sra_epi32(k: __mmask16, a: __m512i, count: __m128i) -> __m512i { + let shf = _mm512_sra_epi32(a, count).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 32-bit integers in a right by count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_sra_epi32&expand=5402) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrad))] +pub unsafe fn _mm256_mask_sra_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, + count: __m128i, +) -> __m256i { + let shf = _mm256_sra_epi32(a, count).as_i32x8(); + transmute(simd_select_bitmask(k, shf, src.as_i32x8())) +} + +/// Shift packed 32-bit integers in a right by count while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_sra_epi32&expand=5403) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrad))] +pub unsafe fn _mm256_maskz_sra_epi32(k: __mmask8, a: __m256i, count: __m128i) -> __m256i { + let shf = _mm256_sra_epi32(a, count).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 32-bit integers in a right by count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_sra_epi32&expand=5399) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrad))] +pub unsafe fn _mm_mask_sra_epi32(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_sra_epi32(a, count).as_i32x4(); + transmute(simd_select_bitmask(k, shf, src.as_i32x4())) +} + +/// Shift packed 32-bit integers in a right by count while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_sra_epi32&expand=5400) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrad))] +pub unsafe fn _mm_maskz_sra_epi32(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_sra_epi32(a, count).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a right by count while shifting in sign bits, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sra_epi64&expand=5416) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsraq))] +pub unsafe fn _mm512_sra_epi64(a: __m512i, count: __m128i) -> __m512i { + transmute(vpsraq(a.as_i64x8(), count.as_i64x2())) +} + +/// Shift packed 64-bit integers in a right by count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sra_epi64&expand=5414) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsraq))] +pub unsafe fn _mm512_mask_sra_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, + count: __m128i, +) -> __m512i { + let shf = _mm512_sra_epi64(a, count).as_i64x8(); + transmute(simd_select_bitmask(k, shf, src.as_i64x8())) +} + +/// Shift packed 64-bit integers in a right by count while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sra_epi64&expand=5415) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsraq))] +pub unsafe fn _mm512_maskz_sra_epi64(k: __mmask8, a: __m512i, count: __m128i) -> __m512i { + let shf = _mm512_sra_epi64(a, count).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a right by count while shifting in sign bits, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_sra_epi64&expand=5413) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsraq))] +pub unsafe fn _mm256_sra_epi64(a: __m256i, count: __m128i) -> __m256i { + transmute(vpsraq256(a.as_i64x4(), count.as_i64x2())) +} + +/// Shift packed 64-bit integers in a right by count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_sra_epi64&expand=5411) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsraq))] +pub unsafe fn _mm256_mask_sra_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, + count: __m128i, +) -> __m256i { + let shf = _mm256_sra_epi64(a, count).as_i64x4(); + transmute(simd_select_bitmask(k, shf, src.as_i64x4())) +} + +/// Shift packed 64-bit integers in a right by count while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_sra_epi64&expand=5412) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsraq))] +pub unsafe fn _mm256_maskz_sra_epi64(k: __mmask8, a: __m256i, count: __m128i) -> __m256i { + let shf = _mm256_sra_epi64(a, count).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a right by count while shifting in sign bits, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_sra_epi64&expand=5410) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsraq))] +pub unsafe fn _mm_sra_epi64(a: __m128i, count: __m128i) -> __m128i { + transmute(vpsraq128(a.as_i64x2(), count.as_i64x2())) +} + +/// Shift packed 64-bit integers in a right by count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_sra_epi64&expand=5408) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsraq))] +pub unsafe fn _mm_mask_sra_epi64(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_sra_epi64(a, count).as_i64x2(); + transmute(simd_select_bitmask(k, shf, src.as_i64x2())) +} + +/// Shift packed 64-bit integers in a right by count while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_sra_epi64&expand=5409) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsraq))] +pub unsafe fn _mm_maskz_sra_epi64(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_sra_epi64(a, count).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 32-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_srai_epi32&expand=5436) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrad, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_srai_epi32(a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i32x16(); + let r = vpsraid512(a, IMM8); + transmute(r) +} + +/// Shift packed 32-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_srai_epi32&expand=5434) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrad, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_srai_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, +) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i32x16(); + let r = vpsraid512(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i32x16())) +} + +/// Shift packed 32-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_srai_epi32&expand=5435) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrad, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_srai_epi32(k: __mmask16, a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i32x16(); + let r = vpsraid512(a, IMM8); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Shift packed 32-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_srai_epi32&expand=5431) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrad, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_srai_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + let imm8 = IMM8 as i32; + let r = psraid256(a.as_i32x8(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i32x8())) +} + +/// Shift packed 32-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_srai_epi32&expand=5432) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrad, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_srai_epi32(k: __mmask8, a: __m256i) -> __m256i { + let imm8 = IMM8 as i32; + let r = psraid256(a.as_i32x8(), imm8); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Shift packed 32-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_srai_epi32&expand=5428) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrad, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_srai_epi32( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + let imm8 = IMM8 as i32; + let r = psraid128(a.as_i32x4(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i32x4())) +} + +/// Shift packed 32-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_srai_epi32&expand=5429) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrad, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_srai_epi32(k: __mmask8, a: __m128i) -> __m128i { + let imm8 = IMM8 as i32; + let r = psraid128(a.as_i32x4(), imm8); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Shift packed 64-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_srai_epi64&expand=5445) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsraq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_srai_epi64(a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i64x8(); + let r = vpsraiq(a, IMM8); + transmute(r) +} + +/// Shift packed 64-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_srai_epi64&expand=5443) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsraq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_srai_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, +) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i64x8(); + let shf = vpsraiq(a, IMM8); + transmute(simd_select_bitmask(k, shf, src.as_i64x8())) +} + +/// Shift packed 64-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_srai_epi64&expand=5444) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsraq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_srai_epi64(k: __mmask8, a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); + let a = a.as_i64x8(); + let shf = vpsraiq(a, IMM8); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_srai_epi64&expand=5442) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsraq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_srai_epi64(a: __m256i) -> __m256i { + static_assert_imm_u8!(IMM8); + let a = a.as_i64x4(); + let r = vpsraiq256(a, IMM8); + transmute(r) +} + +/// Shift packed 64-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_srai_epi64&expand=5440) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsraq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_srai_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm_u8!(IMM8); + let a = a.as_i64x4(); + let shf = vpsraiq256(a, IMM8); + transmute(simd_select_bitmask(k, shf, src.as_i64x4())) +} + +/// Shift packed 64-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_srai_epi64&expand=5441) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsraq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_srai_epi64(k: __mmask8, a: __m256i) -> __m256i { + static_assert_imm_u8!(IMM8); + let a = a.as_i64x4(); + let shf = vpsraiq256(a, IMM8); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srai_epi64&expand=5439) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsraq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_srai_epi64(a: __m128i) -> __m128i { + static_assert_imm_u8!(IMM8); + let a = a.as_i64x2(); + let r = vpsraiq128(a, IMM8); + transmute(r) +} + +/// Shift packed 64-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_srai_epi64&expand=5437) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsraq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_srai_epi64( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm_u8!(IMM8); + let a = a.as_i64x2(); + let shf = vpsraiq128(a, IMM8); + transmute(simd_select_bitmask(k, shf, src.as_i64x2())) +} + +/// Shift packed 64-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_srai_epi64&expand=5438) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsraq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_srai_epi64(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm_u8!(IMM8); + let a = a.as_i64x2(); + let shf = vpsraiq128(a, IMM8); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 32-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_srav_epi32&expand=5465) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsravd))] +pub unsafe fn _mm512_srav_epi32(a: __m512i, count: __m512i) -> __m512i { + transmute(vpsravd(a.as_i32x16(), count.as_i32x16())) +} + +/// Shift packed 32-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_srav_epi32&expand=5463) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsravd))] +pub unsafe fn _mm512_mask_srav_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, + count: __m512i, +) -> __m512i { + let shf = _mm512_srav_epi32(a, count).as_i32x16(); + transmute(simd_select_bitmask(k, shf, src.as_i32x16())) +} + +/// Shift packed 32-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_srav_epi32&expand=5464) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsravd))] +pub unsafe fn _mm512_maskz_srav_epi32(k: __mmask16, a: __m512i, count: __m512i) -> __m512i { + let shf = _mm512_srav_epi32(a, count).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 32-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_srav_epi32&expand=5460) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsravd))] +pub unsafe fn _mm256_mask_srav_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, + count: __m256i, +) -> __m256i { + let shf = _mm256_srav_epi32(a, count).as_i32x8(); + transmute(simd_select_bitmask(k, shf, src.as_i32x8())) +} + +/// Shift packed 32-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_srav_epi32&expand=5461) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsravd))] +pub unsafe fn _mm256_maskz_srav_epi32(k: __mmask8, a: __m256i, count: __m256i) -> __m256i { + let shf = _mm256_srav_epi32(a, count).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 32-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_srav_epi32&expand=5457) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsravd))] +pub unsafe fn _mm_mask_srav_epi32( + src: __m128i, + k: __mmask8, + a: __m128i, + count: __m128i, +) -> __m128i { + let shf = _mm_srav_epi32(a, count).as_i32x4(); + transmute(simd_select_bitmask(k, shf, src.as_i32x4())) +} + +/// Shift packed 32-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_srav_epi32&expand=5458) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsravd))] +pub unsafe fn _mm_maskz_srav_epi32(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_srav_epi32(a, count).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_srav_epi64&expand=5474) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsravq))] +pub unsafe fn _mm512_srav_epi64(a: __m512i, count: __m512i) -> __m512i { + transmute(vpsravq(a.as_i64x8(), count.as_i64x8())) +} + +/// Shift packed 64-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_srav_epi64&expand=5472) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsravq))] +pub unsafe fn _mm512_mask_srav_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, + count: __m512i, +) -> __m512i { + let shf = _mm512_srav_epi64(a, count).as_i64x8(); + transmute(simd_select_bitmask(k, shf, src.as_i64x8())) +} + +/// Shift packed 64-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_srav_epi64&expand=5473) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsravq))] +pub unsafe fn _mm512_maskz_srav_epi64(k: __mmask8, a: __m512i, count: __m512i) -> __m512i { + let shf = _mm512_srav_epi64(a, count).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_srav_epi64&expand=5471) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsravq))] +pub unsafe fn _mm256_srav_epi64(a: __m256i, count: __m256i) -> __m256i { + transmute(vpsravq256(a.as_i64x4(), count.as_i64x4())) +} + +/// Shift packed 64-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_srav_epi64&expand=5469) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsravq))] +pub unsafe fn _mm256_mask_srav_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, + count: __m256i, +) -> __m256i { + let shf = _mm256_srav_epi64(a, count).as_i64x4(); + transmute(simd_select_bitmask(k, shf, src.as_i64x4())) +} + +/// Shift packed 64-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_srav_epi64&expand=5470) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsravq))] +pub unsafe fn _mm256_maskz_srav_epi64(k: __mmask8, a: __m256i, count: __m256i) -> __m256i { + let shf = _mm256_srav_epi64(a, count).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srav_epi64&expand=5468) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsravq))] +pub unsafe fn _mm_srav_epi64(a: __m128i, count: __m128i) -> __m128i { + transmute(vpsravq128(a.as_i64x2(), count.as_i64x2())) +} + +/// Shift packed 64-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_srav_epi64&expand=5466) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsravq))] +pub unsafe fn _mm_mask_srav_epi64( + src: __m128i, + k: __mmask8, + a: __m128i, + count: __m128i, +) -> __m128i { + let shf = _mm_srav_epi64(a, count).as_i64x2(); + transmute(simd_select_bitmask(k, shf, src.as_i64x2())) +} + +/// Shift packed 64-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_srav_epi64&expand=5467) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsravq))] +pub unsafe fn _mm_maskz_srav_epi64(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_srav_epi64(a, count).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in the corresponding element of b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_rolv_epi32&expand=4703) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprolvd))] +pub unsafe fn _mm512_rolv_epi32(a: __m512i, b: __m512i) -> __m512i { + transmute(vprolvd(a.as_i32x16(), b.as_i32x16())) +} + +/// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in the corresponding element of b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_rolv_epi32&expand=4701) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprolvd))] +pub unsafe fn _mm512_mask_rolv_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + let rol = _mm512_rolv_epi32(a, b).as_i32x16(); + transmute(simd_select_bitmask(k, rol, src.as_i32x16())) +} + +/// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in the corresponding element of b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_rolv_epi32&expand=4702) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprolvd))] +pub unsafe fn _mm512_maskz_rolv_epi32(k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let rol = _mm512_rolv_epi32(a, b).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, rol, zero)) +} + +/// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in the corresponding element of b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_rolv_epi32&expand=4700) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolvd))] +pub unsafe fn _mm256_rolv_epi32(a: __m256i, b: __m256i) -> __m256i { + transmute(vprolvd256(a.as_i32x8(), b.as_i32x8())) +} + +/// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in the corresponding element of b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_rolv_epi3&expand=4698) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolvd))] +pub unsafe fn _mm256_mask_rolv_epi32(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let rol = _mm256_rolv_epi32(a, b).as_i32x8(); + transmute(simd_select_bitmask(k, rol, src.as_i32x8())) +} + +/// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in the corresponding element of b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_rolv_epi32&expand=4699) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolvd))] +pub unsafe fn _mm256_maskz_rolv_epi32(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let rol = _mm256_rolv_epi32(a, b).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, rol, zero)) +} + +/// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in the corresponding element of b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_rolv_epi32&expand=4697) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolvd))] +pub unsafe fn _mm_rolv_epi32(a: __m128i, b: __m128i) -> __m128i { + transmute(vprolvd128(a.as_i32x4(), b.as_i32x4())) +} + +/// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in the corresponding element of b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_rolv_epi32&expand=4695) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolvd))] +pub unsafe fn _mm_mask_rolv_epi32(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let rol = _mm_rolv_epi32(a, b).as_i32x4(); + transmute(simd_select_bitmask(k, rol, src.as_i32x4())) +} + +/// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in the corresponding element of b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_rolv_epi32&expand=4696) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolvd))] +pub unsafe fn _mm_maskz_rolv_epi32(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let rol = _mm_rolv_epi32(a, b).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, rol, zero)) +} + +/// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in the corresponding element of b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_rorv_epi32&expand=4739) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprorvd))] +pub unsafe fn _mm512_rorv_epi32(a: __m512i, b: __m512i) -> __m512i { + transmute(vprorvd(a.as_i32x16(), b.as_i32x16())) +} + +/// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in the corresponding element of b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_rorv_epi32&expand=4737) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprorvd))] +pub unsafe fn _mm512_mask_rorv_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + let ror = _mm512_rorv_epi32(a, b).as_i32x16(); + transmute(simd_select_bitmask(k, ror, src.as_i32x16())) +} + +/// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in the corresponding element of b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_rorv_epi32&expand=4738) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprorvd))] +pub unsafe fn _mm512_maskz_rorv_epi32(k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let ror = _mm512_rorv_epi32(a, b).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, ror, zero)) +} + +/// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in the corresponding element of b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_rorv_epi32&expand=4736) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprorvd))] +pub unsafe fn _mm256_rorv_epi32(a: __m256i, b: __m256i) -> __m256i { + transmute(vprorvd256(a.as_i32x8(), b.as_i32x8())) +} + +/// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in the corresponding element of b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_rorv_epi32&expand=4734) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprorvd))] +pub unsafe fn _mm256_mask_rorv_epi32(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let ror = _mm256_rorv_epi32(a, b).as_i32x8(); + transmute(simd_select_bitmask(k, ror, src.as_i32x8())) +} + +/// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in the corresponding element of b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_rorv_epi32&expand=4735) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprorvd))] +pub unsafe fn _mm256_maskz_rorv_epi32(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let ror = _mm256_rorv_epi32(a, b).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, ror, zero)) +} + +/// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in the corresponding element of b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_rorv_epi32&expand=4733) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprorvd))] +pub unsafe fn _mm_rorv_epi32(a: __m128i, b: __m128i) -> __m128i { + transmute(vprorvd128(a.as_i32x4(), b.as_i32x4())) +} + +/// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in the corresponding element of b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_rorv_epi32&expand=4731) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprorvd))] +pub unsafe fn _mm_mask_rorv_epi32(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let ror = _mm_rorv_epi32(a, b).as_i32x4(); + transmute(simd_select_bitmask(k, ror, src.as_i32x4())) +} + +/// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in the corresponding element of b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_rorv_epi32&expand=4732) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprorvd))] +pub unsafe fn _mm_maskz_rorv_epi32(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let ror = _mm_rorv_epi32(a, b).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, ror, zero)) +} + +/// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in the corresponding element of b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_rolv_epi64&expand=4712) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprolvq))] +pub unsafe fn _mm512_rolv_epi64(a: __m512i, b: __m512i) -> __m512i { + transmute(vprolvq(a.as_i64x8(), b.as_i64x8())) +} + +/// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in the corresponding element of b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_rolv_epi64&expand=4710) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprolvq))] +pub unsafe fn _mm512_mask_rolv_epi64(src: __m512i, k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let rol = _mm512_rolv_epi64(a, b).as_i64x8(); + transmute(simd_select_bitmask(k, rol, src.as_i64x8())) +} + +/// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in the corresponding element of b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_rolv_epi64&expand=4711) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprolvq))] +pub unsafe fn _mm512_maskz_rolv_epi64(k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let rol = _mm512_rolv_epi64(a, b).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, rol, zero)) +} + +/// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in the corresponding element of b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_rolv_epi64&expand=4709) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolvq))] +pub unsafe fn _mm256_rolv_epi64(a: __m256i, b: __m256i) -> __m256i { + transmute(vprolvq256(a.as_i64x4(), b.as_i64x4())) +} + +/// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in the corresponding element of b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_rolv_epi64&expand=4707) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolvq))] +pub unsafe fn _mm256_mask_rolv_epi64(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let rol = _mm256_rolv_epi64(a, b).as_i64x4(); + transmute(simd_select_bitmask(k, rol, src.as_i64x4())) +} + +/// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in the corresponding element of b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_rolv_epi64&expand=4708) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolvq))] +pub unsafe fn _mm256_maskz_rolv_epi64(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let rol = _mm256_rolv_epi64(a, b).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, rol, zero)) +} + +/// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in the corresponding element of b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_rolv_epi64&expand=4706) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolvq))] +pub unsafe fn _mm_rolv_epi64(a: __m128i, b: __m128i) -> __m128i { + transmute(vprolvq128(a.as_i64x2(), b.as_i64x2())) +} + +/// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in the corresponding element of b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_rolv_epi64&expand=4704) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolvq))] +pub unsafe fn _mm_mask_rolv_epi64(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let rol = _mm_rolv_epi64(a, b).as_i64x2(); + transmute(simd_select_bitmask(k, rol, src.as_i64x2())) +} + +/// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in the corresponding element of b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_rolv_epi64&expand=4705) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprolvq))] +pub unsafe fn _mm_maskz_rolv_epi64(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let rol = _mm_rolv_epi64(a, b).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, rol, zero)) +} + +/// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in the corresponding element of b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_rorv_epi64&expand=4748) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprorvq))] +pub unsafe fn _mm512_rorv_epi64(a: __m512i, b: __m512i) -> __m512i { + transmute(vprorvq(a.as_i64x8(), b.as_i64x8())) +} + +/// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in the corresponding element of b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_rorv_epi64&expand=4746) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprorvq))] +pub unsafe fn _mm512_mask_rorv_epi64(src: __m512i, k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let ror = _mm512_rorv_epi64(a, b).as_i64x8(); + transmute(simd_select_bitmask(k, ror, src.as_i64x8())) +} + +/// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in the corresponding element of b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_rorv_epi64&expand=4747) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vprorvq))] +pub unsafe fn _mm512_maskz_rorv_epi64(k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let ror = _mm512_rorv_epi64(a, b).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, ror, zero)) +} + +/// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in the corresponding element of b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_rorv_epi64&expand=4745) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprorvq))] +pub unsafe fn _mm256_rorv_epi64(a: __m256i, b: __m256i) -> __m256i { + transmute(vprorvq256(a.as_i64x4(), b.as_i64x4())) +} + +/// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in the corresponding element of b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_rorv_epi64&expand=4743) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprorvq))] +pub unsafe fn _mm256_mask_rorv_epi64(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let ror = _mm256_rorv_epi64(a, b).as_i64x4(); + transmute(simd_select_bitmask(k, ror, src.as_i64x4())) +} + +/// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in the corresponding element of b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_rorv_epi64&expand=4744) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprorvq))] +pub unsafe fn _mm256_maskz_rorv_epi64(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let ror = _mm256_rorv_epi64(a, b).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, ror, zero)) +} + +/// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in the corresponding element of b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_rorv_epi64&expand=4742) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprorvq))] +pub unsafe fn _mm_rorv_epi64(a: __m128i, b: __m128i) -> __m128i { + transmute(vprorvq128(a.as_i64x2(), b.as_i64x2())) +} + +/// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in the corresponding element of b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_rorv_epi64&expand=4740) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprorvq))] +pub unsafe fn _mm_mask_rorv_epi64(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let ror = _mm_rorv_epi64(a, b).as_i64x2(); + transmute(simd_select_bitmask(k, ror, src.as_i64x2())) +} + +/// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in the corresponding element of b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_rorv_epi64&expand=4741) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vprorvq))] +pub unsafe fn _mm_maskz_rorv_epi64(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let ror = _mm_rorv_epi64(a, b).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, ror, zero)) +} + +/// Shift packed 32-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sllv_epi32&expand=5342) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsllvd))] +pub unsafe fn _mm512_sllv_epi32(a: __m512i, count: __m512i) -> __m512i { + transmute(vpsllvd(a.as_i32x16(), count.as_i32x16())) +} + +/// Shift packed 32-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sllv_epi32&expand=5340) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsllvd))] +pub unsafe fn _mm512_mask_sllv_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, + count: __m512i, +) -> __m512i { + let shf = _mm512_sllv_epi32(a, count).as_i32x16(); + transmute(simd_select_bitmask(k, shf, src.as_i32x16())) +} + +/// Shift packed 32-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sllv_epi32&expand=5341) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsllvd))] +pub unsafe fn _mm512_maskz_sllv_epi32(k: __mmask16, a: __m512i, count: __m512i) -> __m512i { + let shf = _mm512_sllv_epi32(a, count).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 32-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_sllv_epi32&expand=5337) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllvd))] +pub unsafe fn _mm256_mask_sllv_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, + count: __m256i, +) -> __m256i { + let shf = _mm256_sllv_epi32(a, count).as_i32x8(); + transmute(simd_select_bitmask(k, shf, src.as_i32x8())) +} + +/// Shift packed 32-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_sllv_epi32&expand=5338) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllvd))] +pub unsafe fn _mm256_maskz_sllv_epi32(k: __mmask8, a: __m256i, count: __m256i) -> __m256i { + let shf = _mm256_sllv_epi32(a, count).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 32-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_sllv_epi32&expand=5334) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllvd))] +pub unsafe fn _mm_mask_sllv_epi32( + src: __m128i, + k: __mmask8, + a: __m128i, + count: __m128i, +) -> __m128i { + let shf = _mm_sllv_epi32(a, count).as_i32x4(); + transmute(simd_select_bitmask(k, shf, src.as_i32x4())) +} + +/// Shift packed 32-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_sllv_epi32&expand=5335) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllvd))] +pub unsafe fn _mm_maskz_sllv_epi32(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_sllv_epi32(a, count).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 32-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_srlv_epi32&expand=5554) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrlvd))] +pub unsafe fn _mm512_srlv_epi32(a: __m512i, count: __m512i) -> __m512i { + transmute(vpsrlvd(a.as_i32x16(), count.as_i32x16())) +} + +/// Shift packed 32-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_srlv_epi32&expand=5552) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrlvd))] +pub unsafe fn _mm512_mask_srlv_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, + count: __m512i, +) -> __m512i { + let shf = _mm512_srlv_epi32(a, count).as_i32x16(); + transmute(simd_select_bitmask(k, shf, src.as_i32x16())) +} + +/// Shift packed 32-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_srlv_epi32&expand=5553) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrlvd))] +pub unsafe fn _mm512_maskz_srlv_epi32(k: __mmask16, a: __m512i, count: __m512i) -> __m512i { + let shf = _mm512_srlv_epi32(a, count).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 32-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_srlv_epi32&expand=5549) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlvd))] +pub unsafe fn _mm256_mask_srlv_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, + count: __m256i, +) -> __m256i { + let shf = _mm256_srlv_epi32(a, count).as_i32x8(); + transmute(simd_select_bitmask(k, shf, src.as_i32x8())) +} + +/// Shift packed 32-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_srlv_epi32&expand=5550) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlvd))] +pub unsafe fn _mm256_maskz_srlv_epi32(k: __mmask8, a: __m256i, count: __m256i) -> __m256i { + let shf = _mm256_srlv_epi32(a, count).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 32-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_srlv_epi32&expand=5546) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlvd))] +pub unsafe fn _mm_mask_srlv_epi32( + src: __m128i, + k: __mmask8, + a: __m128i, + count: __m128i, +) -> __m128i { + let shf = _mm_srlv_epi32(a, count).as_i32x4(); + transmute(simd_select_bitmask(k, shf, src.as_i32x4())) +} + +/// Shift packed 32-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_srlv_epi32&expand=5547) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlvd))] +pub unsafe fn _mm_maskz_srlv_epi32(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_srlv_epi32(a, count).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sllv_epi64&expand=5351) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsllvq))] +pub unsafe fn _mm512_sllv_epi64(a: __m512i, count: __m512i) -> __m512i { + transmute(vpsllvq(a.as_i64x8(), count.as_i64x8())) +} + +/// Shift packed 64-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sllv_epi64&expand=5349) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsllvq))] +pub unsafe fn _mm512_mask_sllv_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, + count: __m512i, +) -> __m512i { + let shf = _mm512_sllv_epi64(a, count).as_i64x8(); + transmute(simd_select_bitmask(k, shf, src.as_i64x8())) +} + +/// Shift packed 64-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sllv_epi64&expand=5350) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsllvq))] +pub unsafe fn _mm512_maskz_sllv_epi64(k: __mmask8, a: __m512i, count: __m512i) -> __m512i { + let shf = _mm512_sllv_epi64(a, count).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_sllv_epi64&expand=5346) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllvq))] +pub unsafe fn _mm256_mask_sllv_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, + count: __m256i, +) -> __m256i { + let shf = _mm256_sllv_epi64(a, count).as_i64x4(); + transmute(simd_select_bitmask(k, shf, src.as_i64x4())) +} + +/// Shift packed 64-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_sllv_epi64&expand=5347) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllvq))] +pub unsafe fn _mm256_maskz_sllv_epi64(k: __mmask8, a: __m256i, count: __m256i) -> __m256i { + let shf = _mm256_sllv_epi64(a, count).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_sllv_epi64&expand=5343) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllvq))] +pub unsafe fn _mm_mask_sllv_epi64( + src: __m128i, + k: __mmask8, + a: __m128i, + count: __m128i, +) -> __m128i { + let shf = _mm_sllv_epi64(a, count).as_i64x2(); + transmute(simd_select_bitmask(k, shf, src.as_i64x2())) +} + +/// Shift packed 64-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_sllv_epi64&expand=5344) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsllvq))] +pub unsafe fn _mm_maskz_sllv_epi64(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_sllv_epi64(a, count).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_srlv_epi64&expand=5563) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrlvq))] +pub unsafe fn _mm512_srlv_epi64(a: __m512i, count: __m512i) -> __m512i { + transmute(vpsrlvq(a.as_i64x8(), count.as_i64x8())) +} + +/// Shift packed 64-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_srlv_epi64&expand=5561) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrlvq))] +pub unsafe fn _mm512_mask_srlv_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, + count: __m512i, +) -> __m512i { + let shf = _mm512_srlv_epi64(a, count).as_i64x8(); + transmute(simd_select_bitmask(k, shf, src.as_i64x8())) +} + +/// Shift packed 64-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_srlv_epi64&expand=5562) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpsrlvq))] +pub unsafe fn _mm512_maskz_srlv_epi64(k: __mmask8, a: __m512i, count: __m512i) -> __m512i { + let shf = _mm512_srlv_epi64(a, count).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_srlv_epi64&expand=5558) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlvq))] +pub unsafe fn _mm256_mask_srlv_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, + count: __m256i, +) -> __m256i { + let shf = _mm256_srlv_epi64(a, count).as_i64x4(); + transmute(simd_select_bitmask(k, shf, src.as_i64x4())) +} + +/// Shift packed 64-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_srlv_epi64&expand=5559) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlvq))] +pub unsafe fn _mm256_maskz_srlv_epi64(k: __mmask8, a: __m256i, count: __m256i) -> __m256i { + let shf = _mm256_srlv_epi64(a, count).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shift packed 64-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_srlv_epi64&expand=5555) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlvq))] +pub unsafe fn _mm_mask_srlv_epi64( + src: __m128i, + k: __mmask8, + a: __m128i, + count: __m128i, +) -> __m128i { + let shf = _mm_srlv_epi64(a, count).as_i64x2(); + transmute(simd_select_bitmask(k, shf, src.as_i64x2())) +} + +/// Shift packed 64-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_srlv_epi64&expand=5556) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpsrlvq))] +pub unsafe fn _mm_maskz_srlv_epi64(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { + let shf = _mm_srlv_epi64(a, count).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permute_ps&expand=4170) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermilps, MASK = 0b11_00_01_11))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_permute_ps(a: __m512) -> __m512 { + static_assert_imm8!(MASK); + simd_shuffle16!( + a, + a, + [ + MASK as u32 & 0b11, + (MASK as u32 >> 2) & 0b11, + ((MASK as u32 >> 4) & 0b11), + ((MASK as u32 >> 6) & 0b11), + (MASK as u32 & 0b11) + 4, + ((MASK as u32 >> 2) & 0b11) + 4, + ((MASK as u32 >> 4) & 0b11) + 4, + ((MASK as u32 >> 6) & 0b11) + 4, + (MASK as u32 & 0b11) + 8, + ((MASK as u32 >> 2) & 0b11) + 8, + ((MASK as u32 >> 4) & 0b11) + 8, + ((MASK as u32 >> 6) & 0b11) + 8, + (MASK as u32 & 0b11) + 12, + ((MASK as u32 >> 2) & 0b11) + 12, + ((MASK as u32 >> 4) & 0b11) + 12, + ((MASK as u32 >> 6) & 0b11) + 12, + ], + ) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permute_ps&expand=4168) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermilps, MASK = 0b11_00_01_11))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_permute_ps( + src: __m512, + k: __mmask16, + a: __m512, +) -> __m512 { + static_assert_imm8!(MASK); + let r = _mm512_permute_ps::(a); + transmute(simd_select_bitmask(k, r.as_f32x16(), src.as_f32x16())) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permute_ps&expand=4169) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermilps, MASK = 0b11_00_01_11))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_permute_ps(k: __mmask16, a: __m512) -> __m512 { + static_assert_imm8!(MASK); + let r = _mm512_permute_ps::(a); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, r.as_f32x16(), zero)) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_permute_ps&expand=4165) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermilps, MASK = 0b11_00_01_11))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_permute_ps( + src: __m256, + k: __mmask8, + a: __m256, +) -> __m256 { + let r = _mm256_permute_ps::(a); + transmute(simd_select_bitmask(k, r.as_f32x8(), src.as_f32x8())) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permute_ps&expand=4166) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermilps, MASK = 0b11_00_01_11))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_permute_ps(k: __mmask8, a: __m256) -> __m256 { + let r = _mm256_permute_ps::(a); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, r.as_f32x8(), zero)) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_permute_ps&expand=4162) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermilps, MASK = 0b11_00_01_11))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_permute_ps(src: __m128, k: __mmask8, a: __m128) -> __m128 { + let r = _mm_permute_ps::(a); + transmute(simd_select_bitmask(k, r.as_f32x4(), src.as_f32x4())) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_permute_ps&expand=4163) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermilps, MASK = 0b11_00_01_11))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_permute_ps(k: __mmask8, a: __m128) -> __m128 { + let r = _mm_permute_ps::(a); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, r.as_f32x4(), zero)) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permute_pd&expand=4161) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermilpd, MASK = 0b11_01_10_01))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_permute_pd(a: __m512d) -> __m512d { + static_assert_imm8!(MASK); + simd_shuffle8!( + a, + a, + [ + MASK as u32 & 0b1, + ((MASK as u32 >> 1) & 0b1), + ((MASK as u32 >> 2) & 0b1) + 2, + ((MASK as u32 >> 3) & 0b1) + 2, + ((MASK as u32 >> 4) & 0b1) + 4, + ((MASK as u32 >> 5) & 0b1) + 4, + ((MASK as u32 >> 6) & 0b1) + 6, + ((MASK as u32 >> 7) & 0b1) + 6, + ], + ) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permute_pd&expand=4159) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermilpd, MASK = 0b11_01_10_01))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_permute_pd( + src: __m512d, + k: __mmask8, + a: __m512d, +) -> __m512d { + static_assert_imm8!(MASK); + let r = _mm512_permute_pd::(a); + transmute(simd_select_bitmask(k, r.as_f64x8(), src.as_f64x8())) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permute_pd&expand=4160) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermilpd, MASK = 0b11_01_10_01))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_permute_pd(k: __mmask8, a: __m512d) -> __m512d { + static_assert_imm8!(MASK); + let r = _mm512_permute_pd::(a); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, r.as_f64x8(), zero)) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_permute_pd&expand=4156) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermilpd, MASK = 0b11_01))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_permute_pd( + src: __m256d, + k: __mmask8, + a: __m256d, +) -> __m256d { + static_assert_imm4!(MASK); + let r = _mm256_permute_pd::(a); + transmute(simd_select_bitmask(k, r.as_f64x4(), src.as_f64x4())) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permute_pd&expand=4157) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermilpd, MASK = 0b11_01))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_permute_pd(k: __mmask8, a: __m256d) -> __m256d { + static_assert_imm4!(MASK); + let r = _mm256_permute_pd::(a); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, r.as_f64x4(), zero)) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_permute_pd&expand=4153) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermilpd, IMM2 = 0b01))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_permute_pd( + src: __m128d, + k: __mmask8, + a: __m128d, +) -> __m128d { + static_assert_imm2!(IMM2); + let r = _mm_permute_pd::(a); + transmute(simd_select_bitmask(k, r.as_f64x2(), src.as_f64x2())) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_permute_pd&expand=4154) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermilpd, IMM2 = 0b01))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_permute_pd(k: __mmask8, a: __m128d) -> __m128d { + static_assert_imm2!(IMM2); + let r = _mm_permute_pd::(a); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, r.as_f64x2(), zero)) +} + +/// Shuffle 64-bit integers in a within 256-bit lanes using the control in imm8, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permutex_epi64&expand=4208) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //should be vpermq +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_permutex_epi64(a: __m512i) -> __m512i { + static_assert_imm8!(MASK); + simd_shuffle8!( + a, + a, + [ + MASK as u32 & 0b11, + (MASK as u32 >> 2) & 0b11, + ((MASK as u32 >> 4) & 0b11), + ((MASK as u32 >> 6) & 0b11), + (MASK as u32 & 0b11) + 4, + ((MASK as u32 >> 2) & 0b11) + 4, + ((MASK as u32 >> 4) & 0b11) + 4, + ((MASK as u32 >> 6) & 0b11) + 4, + ], + ) +} + +/// Shuffle 64-bit integers in a within 256-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permutex_epi64&expand=4206) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //should be vpermq +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_permutex_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, +) -> __m512i { + static_assert_imm8!(MASK); + let r = _mm512_permutex_epi64::(a); + transmute(simd_select_bitmask(k, r.as_i64x8(), src.as_i64x8())) +} + +/// Shuffle 64-bit integers in a within 256-bit lanes using the control in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permutex_epi64&expand=4207) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //should be vpermq +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_permutex_epi64(k: __mmask8, a: __m512i) -> __m512i { + static_assert_imm8!(MASK); + let r = _mm512_permutex_epi64::(a); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, r.as_i64x8(), zero)) +} + +/// Shuffle 64-bit integers in a within 256-bit lanes using the control in imm8, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permutex_epi64&expand=4205) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //should be vpermq +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_permutex_epi64(a: __m256i) -> __m256i { + static_assert_imm8!(MASK); + simd_shuffle4!( + a, + a, + [ + MASK as u32 & 0b11, + (MASK as u32 >> 2) & 0b11, + ((MASK as u32 >> 4) & 0b11), + ((MASK as u32 >> 6) & 0b11), + ], + ) +} + +/// Shuffle 64-bit integers in a within 256-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_permutex_epi6&expand=4203) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //should be vpermq +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_permutex_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm8!(MASK); + let r = _mm256_permutex_epi64::(a); + transmute(simd_select_bitmask(k, r.as_i64x4(), src.as_i64x4())) +} + +/// Shuffle 64-bit integers in a within 256-bit lanes using the control in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permutex_epi64&expand=4204) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //should be vpermq +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_permutex_epi64(k: __mmask8, a: __m256i) -> __m256i { + static_assert_imm8!(MASK); + let r = _mm256_permutex_epi64::(a); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, r.as_i64x4(), zero)) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a within 256-bit lanes using the control in imm8, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permutex_pd&expand=4214) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //should be vpermpd +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_permutex_pd(a: __m512d) -> __m512d { + static_assert_imm8!(MASK); + simd_shuffle8!( + a, + a, + [ + MASK as u32 & 0b11, + (MASK as u32 >> 2) & 0b11, + ((MASK as u32 >> 4) & 0b11), + ((MASK as u32 >> 6) & 0b11), + (MASK as u32 & 0b11) + 4, + ((MASK as u32 >> 2) & 0b11) + 4, + ((MASK as u32 >> 4) & 0b11) + 4, + ((MASK as u32 >> 6) & 0b11) + 4, + ], + ) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a within 256-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permutex_pd&expand=4212) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //should be vpermpd +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_permutex_pd( + src: __m512d, + k: __mmask8, + a: __m512d, +) -> __m512d { + let r = _mm512_permutex_pd::(a); + transmute(simd_select_bitmask(k, r.as_f64x8(), src.as_f64x8())) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a within 256-bit lanes using the control in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permutex_pd&expand=4213) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //should be vpermpd +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_permutex_pd(k: __mmask8, a: __m512d) -> __m512d { + let r = _mm512_permutex_pd::(a); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, r.as_f64x8(), zero)) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a within 256-bit lanes using the control in imm8, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permutex_pd&expand=4211) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //should be vpermpd +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_permutex_pd(a: __m256d) -> __m256d { + static_assert_imm8!(MASK); + simd_shuffle4!( + a, + a, + [ + MASK as u32 & 0b11, + (MASK as u32 >> 2) & 0b11, + ((MASK as u32 >> 4) & 0b11), + ((MASK as u32 >> 6) & 0b11), + ], + ) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a within 256-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_permutex_pd&expand=4209) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //should be vpermpd +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_permutex_pd( + src: __m256d, + k: __mmask8, + a: __m256d, +) -> __m256d { + static_assert_imm8!(MASK); + let r = _mm256_permutex_pd::(a); + transmute(simd_select_bitmask(k, r.as_f64x4(), src.as_f64x4())) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a within 256-bit lanes using the control in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permutex_pd&expand=4210) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //should be vpermpd +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_permutex_pd(k: __mmask8, a: __m256d) -> __m256d { + static_assert_imm8!(MASK); + let r = _mm256_permutex_pd::(a); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, r.as_f64x4(), zero)) +} + +/// Shuffle 32-bit integers in a across lanes using the corresponding index in idx, and store the results in dst. Note that this intrinsic shuffles across 128-bit lanes, unlike past intrinsics that use the permutevar name. This intrinsic is identical to _mm512_permutexvar_epi32, and it is recommended that you use that intrinsic name. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permutevar_epi32&expand=4182) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vperm))] //should be vpermd +pub unsafe fn _mm512_permutevar_epi32(idx: __m512i, a: __m512i) -> __m512i { + transmute(vpermd(a.as_i32x16(), idx.as_i32x16())) +} + +/// Shuffle 32-bit integers in a across lanes using the corresponding index in idx, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). Note that this intrinsic shuffles across 128-bit lanes, unlike past intrinsics that use the permutevar name. This intrinsic is identical to _mm512_mask_permutexvar_epi32, and it is recommended that you use that intrinsic name. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permutevar_epi32&expand=4181) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermd))] +pub unsafe fn _mm512_mask_permutevar_epi32( + src: __m512i, + k: __mmask16, + idx: __m512i, + a: __m512i, +) -> __m512i { + let permute = _mm512_permutevar_epi32(idx, a).as_i32x16(); + transmute(simd_select_bitmask(k, permute, src.as_i32x16())) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permutevar_ps&expand=4200) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermilps))] +pub unsafe fn _mm512_permutevar_ps(a: __m512, b: __m512i) -> __m512 { + transmute(vpermilps(a.as_f32x16(), b.as_i32x16())) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permutevar_ps&expand=4198) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermilps))] +pub unsafe fn _mm512_mask_permutevar_ps( + src: __m512, + k: __mmask16, + a: __m512, + b: __m512i, +) -> __m512 { + let permute = _mm512_permutevar_ps(a, b).as_f32x16(); + transmute(simd_select_bitmask(k, permute, src.as_f32x16())) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permutevar_ps&expand=4199) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermilps))] +pub unsafe fn _mm512_maskz_permutevar_ps(k: __mmask16, a: __m512, b: __m512i) -> __m512 { + let permute = _mm512_permutevar_ps(a, b).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm256_mask_permutevar_ps&expand=4195) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermilps))] +pub unsafe fn _mm256_mask_permutevar_ps(src: __m256, k: __mmask8, a: __m256, b: __m256i) -> __m256 { + let permute = _mm256_permutevar_ps(a, b).as_f32x8(); + transmute(simd_select_bitmask(k, permute, src.as_f32x8())) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permutevar_ps&expand=4196) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermilps))] +pub unsafe fn _mm256_maskz_permutevar_ps(k: __mmask8, a: __m256, b: __m256i) -> __m256 { + let permute = _mm256_permutevar_ps(a, b).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_permutevar_ps&expand=4192) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermilps))] +pub unsafe fn _mm_mask_permutevar_ps(src: __m128, k: __mmask8, a: __m128, b: __m128i) -> __m128 { + let permute = _mm_permutevar_ps(a, b).as_f32x4(); + transmute(simd_select_bitmask(k, permute, src.as_f32x4())) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_permutevar_ps&expand=4193) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermilps))] +pub unsafe fn _mm_maskz_permutevar_ps(k: __mmask8, a: __m128, b: __m128i) -> __m128 { + let permute = _mm_permutevar_ps(a, b).as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a within 128-bit lanes using the control in b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permutevar_pd&expand=4191) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermilpd))] +pub unsafe fn _mm512_permutevar_pd(a: __m512d, b: __m512i) -> __m512d { + transmute(vpermilpd(a.as_f64x8(), b.as_i64x8())) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a within 128-bit lanes using the control in b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permutevar_pd&expand=4189) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermilpd))] +pub unsafe fn _mm512_mask_permutevar_pd( + src: __m512d, + k: __mmask8, + a: __m512d, + b: __m512i, +) -> __m512d { + let permute = _mm512_permutevar_pd(a, b).as_f64x8(); + transmute(simd_select_bitmask(k, permute, src.as_f64x8())) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a within 128-bit lanes using the control in b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permutevar_pd&expand=4190) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermilpd))] +pub unsafe fn _mm512_maskz_permutevar_pd(k: __mmask8, a: __m512d, b: __m512i) -> __m512d { + let permute = _mm512_permutevar_pd(a, b).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a within 128-bit lanes using the control in b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_permutevar_pd&expand=4186) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermilpd))] +pub unsafe fn _mm256_mask_permutevar_pd( + src: __m256d, + k: __mmask8, + a: __m256d, + b: __m256i, +) -> __m256d { + let permute = _mm256_permutevar_pd(a, b).as_f64x4(); + transmute(simd_select_bitmask(k, permute, src.as_f64x4())) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a within 128-bit lanes using the control in b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permutevar_pd&expand=4187) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermilpd))] +pub unsafe fn _mm256_maskz_permutevar_pd(k: __mmask8, a: __m256d, b: __m256i) -> __m256d { + let permute = _mm256_permutevar_pd(a, b).as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a within 128-bit lanes using the control in b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_permutevar_pd&expand=4183) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermilpd))] +pub unsafe fn _mm_mask_permutevar_pd(src: __m128d, k: __mmask8, a: __m128d, b: __m128i) -> __m128d { + let permute = _mm_permutevar_pd(a, b).as_f64x2(); + transmute(simd_select_bitmask(k, permute, src.as_f64x2())) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a within 128-bit lanes using the control in b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_permutevar_pd&expand=4184) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermilpd))] +pub unsafe fn _mm_maskz_permutevar_pd(k: __mmask8, a: __m128d, b: __m128i) -> __m128d { + let permute = _mm_permutevar_pd(a, b).as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle 32-bit integers in a across lanes using the corresponding index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permutexvar_epi32&expand=4301) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vperm))] //should be vpermd +pub unsafe fn _mm512_permutexvar_epi32(idx: __m512i, a: __m512i) -> __m512i { + transmute(vpermd(a.as_i32x16(), idx.as_i32x16())) +} + +/// Shuffle 32-bit integers in a across lanes using the corresponding index in idx, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permutexvar_epi32&expand=4299) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermd))] +pub unsafe fn _mm512_mask_permutexvar_epi32( + src: __m512i, + k: __mmask16, + idx: __m512i, + a: __m512i, +) -> __m512i { + let permute = _mm512_permutexvar_epi32(idx, a).as_i32x16(); + transmute(simd_select_bitmask(k, permute, src.as_i32x16())) +} + +/// Shuffle 32-bit integers in a across lanes using the corresponding index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permutexvar_epi32&expand=4300) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermd))] +pub unsafe fn _mm512_maskz_permutexvar_epi32(k: __mmask16, idx: __m512i, a: __m512i) -> __m512i { + let permute = _mm512_permutexvar_epi32(idx, a).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle 32-bit integers in a across lanes using the corresponding index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permutexvar_epi32&expand=4298) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //should be vpermd +pub unsafe fn _mm256_permutexvar_epi32(idx: __m256i, a: __m256i) -> __m256i { + transmute(_mm256_permutevar8x32_epi32(a, idx)) // llvm use llvm.x86.avx2.permd +} + +/// Shuffle 32-bit integers in a across lanes using the corresponding index in idx, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_permutexvar_epi32&expand=4296) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermd))] +pub unsafe fn _mm256_mask_permutexvar_epi32( + src: __m256i, + k: __mmask8, + idx: __m256i, + a: __m256i, +) -> __m256i { + let permute = _mm256_permutexvar_epi32(idx, a).as_i32x8(); + transmute(simd_select_bitmask(k, permute, src.as_i32x8())) +} + +/// Shuffle 32-bit integers in a across lanes using the corresponding index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permutexvar_epi32&expand=4297) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermd))] +pub unsafe fn _mm256_maskz_permutexvar_epi32(k: __mmask8, idx: __m256i, a: __m256i) -> __m256i { + let permute = _mm256_permutexvar_epi32(idx, a).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle 64-bit integers in a across lanes using the corresponding index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permutexvar_epi64&expand=4307) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vperm))] //should be vpermq +pub unsafe fn _mm512_permutexvar_epi64(idx: __m512i, a: __m512i) -> __m512i { + transmute(vpermq(a.as_i64x8(), idx.as_i64x8())) +} + +/// Shuffle 64-bit integers in a across lanes using the corresponding index in idx, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permutexvar_epi64&expand=4305) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermq))] +pub unsafe fn _mm512_mask_permutexvar_epi64( + src: __m512i, + k: __mmask8, + idx: __m512i, + a: __m512i, +) -> __m512i { + let permute = _mm512_permutexvar_epi64(idx, a).as_i64x8(); + transmute(simd_select_bitmask(k, permute, src.as_i64x8())) +} + +/// Shuffle 64-bit integers in a across lanes using the corresponding index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permutexvar_epi64&expand=4306) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermq))] +pub unsafe fn _mm512_maskz_permutexvar_epi64(k: __mmask8, idx: __m512i, a: __m512i) -> __m512i { + let permute = _mm512_permutexvar_epi64(idx, a).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle 64-bit integers in a across lanes using the corresponding index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permutexvar_epi64&expand=4304) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //should be vpermq +pub unsafe fn _mm256_permutexvar_epi64(idx: __m256i, a: __m256i) -> __m256i { + transmute(vpermq256(a.as_i64x4(), idx.as_i64x4())) +} + +/// Shuffle 64-bit integers in a across lanes using the corresponding index in idx, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_permutexvar_epi64&expand=4302) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermq))] +pub unsafe fn _mm256_mask_permutexvar_epi64( + src: __m256i, + k: __mmask8, + idx: __m256i, + a: __m256i, +) -> __m256i { + let permute = _mm256_permutexvar_epi64(idx, a).as_i64x4(); + transmute(simd_select_bitmask(k, permute, src.as_i64x4())) +} + +/// Shuffle 64-bit integers in a across lanes using the corresponding index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permutexvar_epi64&expand=4303) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermq))] +pub unsafe fn _mm256_maskz_permutexvar_epi64(k: __mmask8, idx: __m256i, a: __m256i) -> __m256i { + let permute = _mm256_permutexvar_epi64(idx, a).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a across lanes using the corresponding index in idx. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permutevar_ps&expand=4200) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermps))] +pub unsafe fn _mm512_permutexvar_ps(idx: __m512i, a: __m512) -> __m512 { + transmute(vpermps(a.as_f32x16(), idx.as_i32x16())) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a across lanes using the corresponding index in idx, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permutexvar_ps&expand=4326) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermps))] +pub unsafe fn _mm512_mask_permutexvar_ps( + src: __m512, + k: __mmask16, + idx: __m512i, + a: __m512, +) -> __m512 { + let permute = _mm512_permutexvar_ps(idx, a).as_f32x16(); + transmute(simd_select_bitmask(k, permute, src.as_f32x16())) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a across lanes using the corresponding index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permutexvar_ps&expand=4327) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermps))] +pub unsafe fn _mm512_maskz_permutexvar_ps(k: __mmask16, idx: __m512i, a: __m512) -> __m512 { + let permute = _mm512_permutexvar_ps(idx, a).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a across lanes using the corresponding index in idx. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permutexvar_ps&expand=4325) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermps))] +pub unsafe fn _mm256_permutexvar_ps(idx: __m256i, a: __m256) -> __m256 { + transmute(_mm256_permutevar8x32_ps(a, idx)) //llvm.x86.avx2.permps +} + +/// Shuffle single-precision (32-bit) floating-point elements in a across lanes using the corresponding index in idx, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_permutexvar_ps&expand=4323) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermps))] +pub unsafe fn _mm256_mask_permutexvar_ps( + src: __m256, + k: __mmask8, + idx: __m256i, + a: __m256, +) -> __m256 { + let permute = _mm256_permutexvar_ps(idx, a).as_f32x8(); + transmute(simd_select_bitmask(k, permute, src.as_f32x8())) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a across lanes using the corresponding index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permutexvar_ps&expand=4324) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermps))] +pub unsafe fn _mm256_maskz_permutexvar_ps(k: __mmask8, idx: __m256i, a: __m256) -> __m256 { + let permute = _mm256_permutexvar_ps(idx, a).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a across lanes using the corresponding index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permutexvar_pd&expand=4322) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermpd))] +pub unsafe fn _mm512_permutexvar_pd(idx: __m512i, a: __m512d) -> __m512d { + transmute(vpermpd(a.as_f64x8(), idx.as_i64x8())) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a across lanes using the corresponding index in idx, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permutexvar_pd&expand=4320) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermpd))] +pub unsafe fn _mm512_mask_permutexvar_pd( + src: __m512d, + k: __mmask8, + idx: __m512i, + a: __m512d, +) -> __m512d { + let permute = _mm512_permutexvar_pd(idx, a).as_f64x8(); + transmute(simd_select_bitmask(k, permute, src.as_f64x8())) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a across lanes using the corresponding index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permutexvar_pd&expand=4321) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermpd))] +pub unsafe fn _mm512_maskz_permutexvar_pd(k: __mmask8, idx: __m512i, a: __m512d) -> __m512d { + let permute = _mm512_permutexvar_pd(idx, a).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a across lanes using the corresponding index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permutexvar_pd&expand=4319) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermpd))] +pub unsafe fn _mm256_permutexvar_pd(idx: __m256i, a: __m256d) -> __m256d { + transmute(vpermpd256(a.as_f64x4(), idx.as_i64x4())) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a across lanes using the corresponding index in idx, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_permutexvar_pd&expand=4317) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermpd))] +pub unsafe fn _mm256_mask_permutexvar_pd( + src: __m256d, + k: __mmask8, + idx: __m256i, + a: __m256d, +) -> __m256d { + let permute = _mm256_permutexvar_pd(idx, a).as_f64x4(); + transmute(simd_select_bitmask(k, permute, src.as_f64x4())) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a across lanes using the corresponding index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permutexvar_pd&expand=4318) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermpd))] +pub unsafe fn _mm256_maskz_permutexvar_pd(k: __mmask8, idx: __m256i, a: __m256d) -> __m256d { + let permute = _mm256_permutexvar_pd(idx, a).as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle 32-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permutex2var_epi32&expand=4238) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2d or vpermt2d +pub unsafe fn _mm512_permutex2var_epi32(a: __m512i, idx: __m512i, b: __m512i) -> __m512i { + transmute(vpermi2d(a.as_i32x16(), idx.as_i32x16(), b.as_i32x16())) +} + +/// Shuffle 32-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permutex2var_epi32&expand=4235) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermt2d))] +pub unsafe fn _mm512_mask_permutex2var_epi32( + a: __m512i, + k: __mmask16, + idx: __m512i, + b: __m512i, +) -> __m512i { + let permute = _mm512_permutex2var_epi32(a, idx, b).as_i32x16(); + transmute(simd_select_bitmask(k, permute, a.as_i32x16())) +} + +/// Shuffle 32-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permutex2var_epi32&expand=4237) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2d or vpermt2d +pub unsafe fn _mm512_maskz_permutex2var_epi32( + k: __mmask16, + a: __m512i, + idx: __m512i, + b: __m512i, +) -> __m512i { + let permute = _mm512_permutex2var_epi32(a, idx, b).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle 32-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from idx when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask2_permutex2var_epi32&expand=4236) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermi2d))] +pub unsafe fn _mm512_mask2_permutex2var_epi32( + a: __m512i, + idx: __m512i, + k: __mmask16, + b: __m512i, +) -> __m512i { + let permute = _mm512_permutex2var_epi32(a, idx, b).as_i32x16(); + transmute(simd_select_bitmask(k, permute, idx.as_i32x16())) +} + +/// Shuffle 32-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permutex2var_epi32&expand=4234) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2d or vpermt2d +pub unsafe fn _mm256_permutex2var_epi32(a: __m256i, idx: __m256i, b: __m256i) -> __m256i { + transmute(vpermi2d256(a.as_i32x8(), idx.as_i32x8(), b.as_i32x8())) +} + +/// Shuffle 32-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_permutex2var_epi32&expand=4231) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermt2d))] +pub unsafe fn _mm256_mask_permutex2var_epi32( + a: __m256i, + k: __mmask8, + idx: __m256i, + b: __m256i, +) -> __m256i { + let permute = _mm256_permutex2var_epi32(a, idx, b).as_i32x8(); + transmute(simd_select_bitmask(k, permute, a.as_i32x8())) +} + +/// Shuffle 32-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permutex2var_epi32&expand=4233) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2d or vpermt2d +pub unsafe fn _mm256_maskz_permutex2var_epi32( + k: __mmask8, + a: __m256i, + idx: __m256i, + b: __m256i, +) -> __m256i { + let permute = _mm256_permutex2var_epi32(a, idx, b).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle 32-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from idx when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask2_permutex2var_epi32&expand=4232) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermi2d))] +pub unsafe fn _mm256_mask2_permutex2var_epi32( + a: __m256i, + idx: __m256i, + k: __mmask8, + b: __m256i, +) -> __m256i { + let permute = _mm256_permutex2var_epi32(a, idx, b).as_i32x8(); + transmute(simd_select_bitmask(k, permute, idx.as_i32x8())) +} + +/// Shuffle 32-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_permutex2var_epi32&expand=4230) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2d or vpermt2d +pub unsafe fn _mm_permutex2var_epi32(a: __m128i, idx: __m128i, b: __m128i) -> __m128i { + transmute(vpermi2d128(a.as_i32x4(), idx.as_i32x4(), b.as_i32x4())) +} + +/// Shuffle 32-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_permutex2var_epi32&expand=4227) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermt2d))] +pub unsafe fn _mm_mask_permutex2var_epi32( + a: __m128i, + k: __mmask8, + idx: __m128i, + b: __m128i, +) -> __m128i { + let permute = _mm_permutex2var_epi32(a, idx, b).as_i32x4(); + transmute(simd_select_bitmask(k, permute, a.as_i32x4())) +} + +/// Shuffle 32-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_permutex2var_epi32&expand=4229) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2d or vpermt2d +pub unsafe fn _mm_maskz_permutex2var_epi32( + k: __mmask8, + a: __m128i, + idx: __m128i, + b: __m128i, +) -> __m128i { + let permute = _mm_permutex2var_epi32(a, idx, b).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle 32-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from idx when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask2_permutex2var_epi32&expand=4228) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermi2d))] +pub unsafe fn _mm_mask2_permutex2var_epi32( + a: __m128i, + idx: __m128i, + k: __mmask8, + b: __m128i, +) -> __m128i { + let permute = _mm_permutex2var_epi32(a, idx, b).as_i32x4(); + transmute(simd_select_bitmask(k, permute, idx.as_i32x4())) +} + +/// Shuffle 64-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permutex2var_epi64&expand=4250) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2q or vpermt2q +pub unsafe fn _mm512_permutex2var_epi64(a: __m512i, idx: __m512i, b: __m512i) -> __m512i { + transmute(vpermi2q(a.as_i64x8(), idx.as_i64x8(), b.as_i64x8())) +} + +/// Shuffle 64-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permutex2var_epi64&expand=4247) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermt2q))] +pub unsafe fn _mm512_mask_permutex2var_epi64( + a: __m512i, + k: __mmask8, + idx: __m512i, + b: __m512i, +) -> __m512i { + let permute = _mm512_permutex2var_epi64(a, idx, b).as_i64x8(); + transmute(simd_select_bitmask(k, permute, a.as_i64x8())) +} + +/// Shuffle 64-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permutex2var_epi64&expand=4249) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2q or vpermt2q +pub unsafe fn _mm512_maskz_permutex2var_epi64( + k: __mmask8, + a: __m512i, + idx: __m512i, + b: __m512i, +) -> __m512i { + let permute = _mm512_permutex2var_epi64(a, idx, b).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle 64-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from idx when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask2_permutex2var_epi64&expand=4248) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermi2q))] +pub unsafe fn _mm512_mask2_permutex2var_epi64( + a: __m512i, + idx: __m512i, + k: __mmask8, + b: __m512i, +) -> __m512i { + let permute = _mm512_permutex2var_epi64(a, idx, b).as_i64x8(); + transmute(simd_select_bitmask(k, permute, idx.as_i64x8())) +} + +/// Shuffle 64-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permutex2var_epi64&expand=4246) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2q or vpermt2q +pub unsafe fn _mm256_permutex2var_epi64(a: __m256i, idx: __m256i, b: __m256i) -> __m256i { + transmute(vpermi2q256(a.as_i64x4(), idx.as_i64x4(), b.as_i64x4())) +} + +/// Shuffle 64-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_permutex2var_epi64&expand=4243) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermt2q))] +pub unsafe fn _mm256_mask_permutex2var_epi64( + a: __m256i, + k: __mmask8, + idx: __m256i, + b: __m256i, +) -> __m256i { + let permute = _mm256_permutex2var_epi64(a, idx, b).as_i64x4(); + transmute(simd_select_bitmask(k, permute, a.as_i64x4())) +} + +/// Shuffle 64-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permutex2var_epi64&expand=4245) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2q or vpermt2q +pub unsafe fn _mm256_maskz_permutex2var_epi64( + k: __mmask8, + a: __m256i, + idx: __m256i, + b: __m256i, +) -> __m256i { + let permute = _mm256_permutex2var_epi64(a, idx, b).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle 64-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from idx when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask2_permutex2var_epi64&expand=4244) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermi2q))] +pub unsafe fn _mm256_mask2_permutex2var_epi64( + a: __m256i, + idx: __m256i, + k: __mmask8, + b: __m256i, +) -> __m256i { + let permute = _mm256_permutex2var_epi64(a, idx, b).as_i64x4(); + transmute(simd_select_bitmask(k, permute, idx.as_i64x4())) +} + +/// Shuffle 64-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_permutex2var_epi64&expand=4242) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2q or vpermt2q +pub unsafe fn _mm_permutex2var_epi64(a: __m128i, idx: __m128i, b: __m128i) -> __m128i { + transmute(vpermi2q128(a.as_i64x2(), idx.as_i64x2(), b.as_i64x2())) +} + +/// Shuffle 64-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_permutex2var_epi64&expand=4239) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermt2q))] +pub unsafe fn _mm_mask_permutex2var_epi64( + a: __m128i, + k: __mmask8, + idx: __m128i, + b: __m128i, +) -> __m128i { + let permute = _mm_permutex2var_epi64(a, idx, b).as_i64x2(); + transmute(simd_select_bitmask(k, permute, a.as_i64x2())) +} + +/// Shuffle 64-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_permutex2var_epi64&expand=4241) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2q or vpermt2q +pub unsafe fn _mm_maskz_permutex2var_epi64( + k: __mmask8, + a: __m128i, + idx: __m128i, + b: __m128i, +) -> __m128i { + let permute = _mm_permutex2var_epi64(a, idx, b).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle 64-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from idx when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask2_permutex2var_epi64&expand=4240) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermi2q))] +pub unsafe fn _mm_mask2_permutex2var_epi64( + a: __m128i, + idx: __m128i, + k: __mmask8, + b: __m128i, +) -> __m128i { + let permute = _mm_permutex2var_epi64(a, idx, b).as_i64x2(); + transmute(simd_select_bitmask(k, permute, idx.as_i64x2())) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permutex2var_ps&expand=4286) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2ps or vpermt2ps +pub unsafe fn _mm512_permutex2var_ps(a: __m512, idx: __m512i, b: __m512) -> __m512 { + transmute(vpermi2ps(a.as_f32x16(), idx.as_i32x16(), b.as_f32x16())) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permutex2var_ps&expand=4283) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermt2ps))] +pub unsafe fn _mm512_mask_permutex2var_ps( + a: __m512, + k: __mmask16, + idx: __m512i, + b: __m512, +) -> __m512 { + let permute = _mm512_permutex2var_ps(a, idx, b).as_f32x16(); + transmute(simd_select_bitmask(k, permute, a.as_f32x16())) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permutex2var_ps&expand=4285) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2ps or vpermt2ps +pub unsafe fn _mm512_maskz_permutex2var_ps( + k: __mmask16, + a: __m512, + idx: __m512i, + b: __m512, +) -> __m512 { + let permute = _mm512_permutex2var_ps(a, idx, b).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from idx when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask2_permutex2var_ps&expand=4284) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vperm))] //should be vpermi2ps, but it shows vpermt2ps +pub unsafe fn _mm512_mask2_permutex2var_ps( + a: __m512, + idx: __m512i, + k: __mmask16, + b: __m512, +) -> __m512 { + let permute = _mm512_permutex2var_ps(a, idx, b).as_f32x16(); + let idx = _mm512_castsi512_ps(idx).as_f32x16(); + transmute(simd_select_bitmask(k, permute, idx)) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permutex2var_ps&expand=4282) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2ps or vpermt2ps +pub unsafe fn _mm256_permutex2var_ps(a: __m256, idx: __m256i, b: __m256) -> __m256 { + transmute(vpermi2ps256(a.as_f32x8(), idx.as_i32x8(), b.as_f32x8())) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_permutex2var_ps&expand=4279) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermt2ps))] +pub unsafe fn _mm256_mask_permutex2var_ps( + a: __m256, + k: __mmask8, + idx: __m256i, + b: __m256, +) -> __m256 { + let permute = _mm256_permutex2var_ps(a, idx, b).as_f32x8(); + transmute(simd_select_bitmask(k, permute, a.as_f32x8())) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permutex2var_ps&expand=4281) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2ps or vpermt2ps +pub unsafe fn _mm256_maskz_permutex2var_ps( + k: __mmask8, + a: __m256, + idx: __m256i, + b: __m256, +) -> __m256 { + let permute = _mm256_permutex2var_ps(a, idx, b).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from idx when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask2_permutex2var_ps&expand=4280) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //should be vpermi2ps, but it shows vpermt2ps +pub unsafe fn _mm256_mask2_permutex2var_ps( + a: __m256, + idx: __m256i, + k: __mmask8, + b: __m256, +) -> __m256 { + let permute = _mm256_permutex2var_ps(a, idx, b).as_f32x8(); + let idx = _mm256_castsi256_ps(idx).as_f32x8(); + transmute(simd_select_bitmask(k, permute, idx)) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_permutex2var_ps&expand=4278) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2ps or vpermt2ps +pub unsafe fn _mm_permutex2var_ps(a: __m128, idx: __m128i, b: __m128) -> __m128 { + transmute(vpermi2ps128(a.as_f32x4(), idx.as_i32x4(), b.as_f32x4())) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_permutex2var_ps&expand=4275) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermt2ps))] +pub unsafe fn _mm_mask_permutex2var_ps(a: __m128, k: __mmask8, idx: __m128i, b: __m128) -> __m128 { + let permute = _mm_permutex2var_ps(a, idx, b).as_f32x4(); + transmute(simd_select_bitmask(k, permute, a.as_f32x4())) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_permutex2var_ps&expand=4277) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2ps or vpermt2ps +pub unsafe fn _mm_maskz_permutex2var_ps(k: __mmask8, a: __m128, idx: __m128i, b: __m128) -> __m128 { + let permute = _mm_permutex2var_ps(a, idx, b).as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from idx when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask2_permutex2var_ps&expand=4276) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //should be vpermi2ps, but it shows vpermt2ps +pub unsafe fn _mm_mask2_permutex2var_ps(a: __m128, idx: __m128i, k: __mmask8, b: __m128) -> __m128 { + let permute = _mm_permutex2var_ps(a, idx, b).as_f32x4(); + let idx = _mm_castsi128_ps(idx).as_f32x4(); + transmute(simd_select_bitmask(k, permute, idx)) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permutex2var_pd&expand=4274) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2pd or vpermt2pd +pub unsafe fn _mm512_permutex2var_pd(a: __m512d, idx: __m512i, b: __m512d) -> __m512d { + transmute(vpermi2pd(a.as_f64x8(), idx.as_i64x8(), b.as_f64x8())) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permutex2var_pd&expand=4271) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermt2pd))] +pub unsafe fn _mm512_mask_permutex2var_pd( + a: __m512d, + k: __mmask8, + idx: __m512i, + b: __m512d, +) -> __m512d { + let permute = _mm512_permutex2var_pd(a, idx, b).as_f64x8(); + transmute(simd_select_bitmask(k, permute, a.as_f64x8())) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permutex2var_pd&expand=4273) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2pd or vpermt2pd +pub unsafe fn _mm512_maskz_permutex2var_pd( + k: __mmask8, + a: __m512d, + idx: __m512i, + b: __m512d, +) -> __m512d { + let permute = _mm512_permutex2var_pd(a, idx, b).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from idx when the corresponding mask bit is not set) +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask2_permutex2var_pd&expand=4272) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vperm))] //should be vpermi2pd, but it shows vpermt2pd +pub unsafe fn _mm512_mask2_permutex2var_pd( + a: __m512d, + idx: __m512i, + k: __mmask8, + b: __m512d, +) -> __m512d { + let permute = _mm512_permutex2var_pd(a, idx, b).as_f64x8(); + let idx = _mm512_castsi512_pd(idx).as_f64x8(); + transmute(simd_select_bitmask(k, permute, idx)) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permutex2var_pd&expand=4270) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2pd or vpermt2pd +pub unsafe fn _mm256_permutex2var_pd(a: __m256d, idx: __m256i, b: __m256d) -> __m256d { + transmute(vpermi2pd256(a.as_f64x4(), idx.as_i64x4(), b.as_f64x4())) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_permutex2var_pd&expand=4267) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermt2pd))] +pub unsafe fn _mm256_mask_permutex2var_pd( + a: __m256d, + k: __mmask8, + idx: __m256i, + b: __m256d, +) -> __m256d { + let permute = _mm256_permutex2var_pd(a, idx, b).as_f64x4(); + transmute(simd_select_bitmask(k, permute, a.as_f64x4())) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permutex2var_pd&expand=4269) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2pd or vpermt2pd +pub unsafe fn _mm256_maskz_permutex2var_pd( + k: __mmask8, + a: __m256d, + idx: __m256i, + b: __m256d, +) -> __m256d { + let permute = _mm256_permutex2var_pd(a, idx, b).as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from idx when the corresponding mask bit is not set) +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask2_permutex2var_pd&expand=4268) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //should be vpermi2pd, but it shows vpermt2pd +pub unsafe fn _mm256_mask2_permutex2var_pd( + a: __m256d, + idx: __m256i, + k: __mmask8, + b: __m256d, +) -> __m256d { + let permute = _mm256_permutex2var_pd(a, idx, b).as_f64x4(); + let idx = _mm256_castsi256_pd(idx).as_f64x4(); + transmute(simd_select_bitmask(k, permute, idx)) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_permutex2var_pd&expand=4266) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2pd or vpermt2pd +pub unsafe fn _mm_permutex2var_pd(a: __m128d, idx: __m128i, b: __m128d) -> __m128d { + transmute(vpermi2pd128(a.as_f64x2(), idx.as_i64x2(), b.as_f64x2())) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_permutex2var_pd&expand=4263) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpermt2pd))] +pub unsafe fn _mm_mask_permutex2var_pd( + a: __m128d, + k: __mmask8, + idx: __m128i, + b: __m128d, +) -> __m128d { + let permute = _mm_permutex2var_pd(a, idx, b).as_f64x2(); + transmute(simd_select_bitmask(k, permute, a.as_f64x2())) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_permutex2var_pd&expand=4265) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //vpermi2pd or vpermt2pd +pub unsafe fn _mm_maskz_permutex2var_pd( + k: __mmask8, + a: __m128d, + idx: __m128i, + b: __m128d, +) -> __m128d { + let permute = _mm_permutex2var_pd(a, idx, b).as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle double-precision (64-bit) floating-point elements in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from idx when the corresponding mask bit is not set) +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask2_permutex2var_pd&expand=4264) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //should be vpermi2pd, but it shows vpermt2pd +pub unsafe fn _mm_mask2_permutex2var_pd( + a: __m128d, + idx: __m128i, + k: __mmask8, + b: __m128d, +) -> __m128d { + let permute = _mm_permutex2var_pd(a, idx, b).as_f64x2(); + let idx = _mm_castsi128_pd(idx).as_f64x2(); + transmute(simd_select_bitmask(k, permute, idx)) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_shuffle_epi32&expand=5150) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpermilps, MASK = 9))] //should be vpshufd +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_shuffle_epi32(a: __m512i) -> __m512i { + static_assert_imm8!(MASK); + let r: i32x16 = simd_shuffle16!( + a.as_i32x16(), + a.as_i32x16(), + [ + MASK as u32 & 0b11, + (MASK as u32 >> 2) & 0b11, + (MASK as u32 >> 4) & 0b11, + (MASK as u32 >> 6) & 0b11, + (MASK as u32 & 0b11) + 4, + ((MASK as u32 >> 2) & 0b11) + 4, + ((MASK as u32 >> 4) & 0b11) + 4, + ((MASK as u32 >> 6) & 0b11) + 4, + (MASK as u32 & 0b11) + 8, + ((MASK as u32 >> 2) & 0b11) + 8, + ((MASK as u32 >> 4) & 0b11) + 8, + ((MASK as u32 >> 6) & 0b11) + 8, + (MASK as u32 & 0b11) + 12, + ((MASK as u32 >> 2) & 0b11) + 12, + ((MASK as u32 >> 4) & 0b11) + 12, + ((MASK as u32 >> 6) & 0b11) + 12, + ], + ); + transmute(r) +} + +/// Shuffle 32-bit integers in a within 128-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shuffle_epi32&expand=5148) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpshufd, MASK = 9))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_shuffle_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, +) -> __m512i { + static_assert_imm8!(MASK); + let r = _mm512_shuffle_epi32::(a); + transmute(simd_select_bitmask(k, r.as_i32x16(), src.as_i32x16())) +} + +/// Shuffle 32-bit integers in a within 128-bit lanes using the control in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shuffle_epi32&expand=5149) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpshufd, MASK = 9))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_shuffle_epi32( + k: __mmask16, + a: __m512i, +) -> __m512i { + static_assert_imm8!(MASK); + let r = _mm512_shuffle_epi32::(a); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, r.as_i32x16(), zero)) +} + +/// Shuffle 32-bit integers in a within 128-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shuffle_epi32&expand=5145) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpshufd, MASK = 9))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_shuffle_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm8!(MASK); + let r = _mm256_shuffle_epi32::(a); + transmute(simd_select_bitmask(k, r.as_i32x8(), src.as_i32x8())) +} + +/// Shuffle 32-bit integers in a within 128-bit lanes using the control in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shuffle_epi32&expand=5146) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpshufd, MASK = 9))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_shuffle_epi32( + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm8!(MASK); + let r = _mm256_shuffle_epi32::(a); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, r.as_i32x8(), zero)) +} + +/// Shuffle 32-bit integers in a within 128-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shuffle_epi32&expand=5142) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpshufd, MASK = 9))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_shuffle_epi32( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm8!(MASK); + let r = _mm_shuffle_epi32::(a); + transmute(simd_select_bitmask(k, r.as_i32x4(), src.as_i32x4())) +} + +/// Shuffle 32-bit integers in a within 128-bit lanes using the control in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shuffle_epi32&expand=5143) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpshufd, MASK = 9))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_shuffle_epi32( + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm8!(MASK); + let r = _mm_shuffle_epi32::(a); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, r.as_i32x4(), zero)) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shuffle_ps&expand=5203) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vshufps, MASK = 3))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shuffle_ps(a: __m512, b: __m512) -> __m512 { + static_assert_imm8!(MASK); + simd_shuffle16!( + a, + b, + [ + MASK as u32 & 0b11, + (MASK as u32 >> 2) & 0b11, + ((MASK as u32 >> 4) & 0b11) + 16, + ((MASK as u32 >> 6) & 0b11) + 16, + (MASK as u32 & 0b11) + 4, + ((MASK as u32 >> 2) & 0b11) + 4, + ((MASK as u32 >> 4) & 0b11) + 20, + ((MASK as u32 >> 6) & 0b11) + 20, + (MASK as u32 & 0b11) + 8, + ((MASK as u32 >> 2) & 0b11) + 8, + ((MASK as u32 >> 4) & 0b11) + 24, + ((MASK as u32 >> 6) & 0b11) + 24, + (MASK as u32 & 0b11) + 12, + ((MASK as u32 >> 2) & 0b11) + 12, + ((MASK as u32 >> 4) & 0b11) + 28, + ((MASK as u32 >> 6) & 0b11) + 28, + ], + ) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shuffle_ps&expand=5201) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vshufps, MASK = 3))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shuffle_ps( + src: __m512, + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_imm8!(MASK); + let r = _mm512_shuffle_ps::(a, b); + transmute(simd_select_bitmask(k, r.as_f32x16(), src.as_f32x16())) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shuffle_ps&expand=5202) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vshufps, MASK = 3))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shuffle_ps( + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_imm8!(MASK); + let r = _mm512_shuffle_ps::(a, b); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, r.as_f32x16(), zero)) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shuffle_ps&expand=5198) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vshufps, MASK = 3))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shuffle_ps( + src: __m256, + k: __mmask8, + a: __m256, + b: __m256, +) -> __m256 { + static_assert_imm8!(MASK); + let r = _mm256_shuffle_ps::(a, b); + transmute(simd_select_bitmask(k, r.as_f32x8(), src.as_f32x8())) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shuffle_ps&expand=5199) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vshufps, MASK = 3))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shuffle_ps( + k: __mmask8, + a: __m256, + b: __m256, +) -> __m256 { + static_assert_imm8!(MASK); + let r = _mm256_shuffle_ps::(a, b); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, r.as_f32x8(), zero)) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shuffle_ps&expand=5195) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vshufps, MASK = 3))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_shuffle_ps( + src: __m128, + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_imm8!(MASK); + let r = _mm_shuffle_ps::(a, b); + transmute(simd_select_bitmask(k, r.as_f32x4(), src.as_f32x4())) +} + +/// Shuffle single-precision (32-bit) floating-point elements in a using the control in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shuffle_ps&expand=5196) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vshufps, MASK = 3))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_shuffle_ps(k: __mmask8, a: __m128, b: __m128) -> __m128 { + static_assert_imm8!(MASK); + let r = _mm_shuffle_ps::(a, b); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, r.as_f32x4(), zero)) +} + +/// Shuffle double-precision (64-bit) floating-point elements within 128-bit lanes using the control in imm8, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shuffle_pd&expand=5192) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vshufpd, MASK = 3))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shuffle_pd(a: __m512d, b: __m512d) -> __m512d { + static_assert_imm8!(MASK); + simd_shuffle8!( + a, + b, + [ + MASK as u32 & 0b1, + ((MASK as u32 >> 1) & 0b1) + 8, + ((MASK as u32 >> 2) & 0b1) + 2, + ((MASK as u32 >> 3) & 0b1) + 10, + ((MASK as u32 >> 4) & 0b1) + 4, + ((MASK as u32 >> 5) & 0b1) + 12, + ((MASK as u32 >> 6) & 0b1) + 6, + ((MASK as u32 >> 7) & 0b1) + 14, + ], + ) +} + +/// Shuffle double-precision (64-bit) floating-point elements within 128-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shuffle_pd&expand=5190) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vshufpd, MASK = 3))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shuffle_pd( + src: __m512d, + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + static_assert_imm8!(MASK); + let r = _mm512_shuffle_pd::(a, b); + transmute(simd_select_bitmask(k, r.as_f64x8(), src.as_f64x8())) +} + +/// Shuffle double-precision (64-bit) floating-point elements within 128-bit lanes using the control in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shuffle_pd&expand=5191) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vshufpd, MASK = 3))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shuffle_pd( + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + static_assert_imm8!(MASK); + let r = _mm512_shuffle_pd::(a, b); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, r.as_f64x8(), zero)) +} + +/// Shuffle double-precision (64-bit) floating-point elements within 128-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shuffle_pd&expand=5187) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vshufpd, MASK = 3))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shuffle_pd( + src: __m256d, + k: __mmask8, + a: __m256d, + b: __m256d, +) -> __m256d { + static_assert_imm8!(MASK); + let r = _mm256_shuffle_pd::(a, b); + transmute(simd_select_bitmask(k, r.as_f64x4(), src.as_f64x4())) +} + +/// Shuffle double-precision (64-bit) floating-point elements within 128-bit lanes using the control in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shuffle_pd&expand=5188) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vshufpd, MASK = 3))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shuffle_pd( + k: __mmask8, + a: __m256d, + b: __m256d, +) -> __m256d { + static_assert_imm8!(MASK); + let r = _mm256_shuffle_pd::(a, b); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, r.as_f64x4(), zero)) +} + +/// Shuffle double-precision (64-bit) floating-point elements within 128-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shuffle_pd&expand=5184) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vshufpd, MASK = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_shuffle_pd( + src: __m128d, + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_imm8!(MASK); + let r = _mm_shuffle_pd::(a, b); + transmute(simd_select_bitmask(k, r.as_f64x2(), src.as_f64x2())) +} + +/// Shuffle double-precision (64-bit) floating-point elements within 128-bit lanes using the control in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shuffle_pd&expand=5185) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vshufpd, MASK = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_shuffle_pd( + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_imm8!(MASK); + let r = _mm_shuffle_pd::(a, b); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, r.as_f64x2(), zero)) +} + +/// Shuffle 128-bits (composed of 4 32-bit integers) selected by imm8 from a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shuffle_i32&expand=5177) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vshufi64x2, MASK = 0b10_01_01_01))] //should be vshufi32x4 +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shuffle_i32x4(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(MASK); + let a = a.as_i32x16(); + let b = b.as_i32x16(); + let r: i32x16 = simd_shuffle16!( + a, + b, + [ + (MASK as u32 & 0b11) * 4 + 0, + (MASK as u32 & 0b11) * 4 + 1, + (MASK as u32 & 0b11) * 4 + 2, + (MASK as u32 & 0b11) * 4 + 3, + ((MASK as u32 >> 2) & 0b11) * 4 + 0, + ((MASK as u32 >> 2) & 0b11) * 4 + 1, + ((MASK as u32 >> 2) & 0b11) * 4 + 2, + ((MASK as u32 >> 2) & 0b11) * 4 + 3, + ((MASK as u32 >> 4) & 0b11) * 4 + 0 + 16, + ((MASK as u32 >> 4) & 0b11) * 4 + 1 + 16, + ((MASK as u32 >> 4) & 0b11) * 4 + 2 + 16, + ((MASK as u32 >> 4) & 0b11) * 4 + 3 + 16, + ((MASK as u32 >> 6) & 0b11) * 4 + 0 + 16, + ((MASK as u32 >> 6) & 0b11) * 4 + 1 + 16, + ((MASK as u32 >> 6) & 0b11) * 4 + 2 + 16, + ((MASK as u32 >> 6) & 0b11) * 4 + 3 + 16, + ], + ); + transmute(r) +} + +/// Shuffle 128-bits (composed of 4 32-bit integers) selected by imm8 from a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shuffle_i32x&expand=5175) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vshufi32x4, MASK = 0b10_11_01_01))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shuffle_i32x4( + src: __m512i, + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(MASK); + let r = _mm512_shuffle_i32x4::(a, b); + transmute(simd_select_bitmask(k, r.as_i32x16(), src.as_i32x16())) +} + +/// Shuffle 128-bits (composed of 4 32-bit integers) selected by imm8 from a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shuffle_i32&expand=5176) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vshufi32x4, MASK = 0b10_11_01_01))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shuffle_i32x4( + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(MASK); + let r = _mm512_shuffle_i32x4::(a, b); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, r.as_i32x16(), zero)) +} + +/// Shuffle 128-bits (composed of 4 32-bit integers) selected by imm8 from a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shuffle_i32x4&expand=5174) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm, MASK = 0b11))] //should be vshufi32x4 +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_shuffle_i32x4(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(MASK); + let a = a.as_i32x8(); + let b = b.as_i32x8(); + let r: i32x8 = simd_shuffle8!( + a, + b, + [ + (MASK as u32 & 0b1) * 4 + 0, + (MASK as u32 & 0b1) * 4 + 1, + (MASK as u32 & 0b1) * 4 + 2, + (MASK as u32 & 0b1) * 4 + 3, + ((MASK as u32 >> 1) & 0b1) * 4 + 0 + 8, + ((MASK as u32 >> 1) & 0b1) * 4 + 1 + 8, + ((MASK as u32 >> 1) & 0b1) * 4 + 2 + 8, + ((MASK as u32 >> 1) & 0b1) * 4 + 3 + 8, + ], + ); + transmute(r) +} + +/// Shuffle 128-bits (composed of 4 32-bit integers) selected by imm8 from a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shuffle_i32x4&expand=5172) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vshufi32x4, MASK = 0b11))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shuffle_i32x4( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(MASK); + let r = _mm256_shuffle_i32x4::(a, b); + transmute(simd_select_bitmask(k, r.as_i32x8(), src.as_i32x8())) +} + +/// Shuffle 128-bits (composed of 4 32-bit integers) selected by imm8 from a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shuffle_i32x4&expand=5173) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vshufi32x4, MASK = 0b11))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shuffle_i32x4( + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(MASK); + let r = _mm256_shuffle_i32x4::(a, b); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, r.as_i32x8(), zero)) +} + +/// Shuffle 128-bits (composed of 2 64-bit integers) selected by imm8 from a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shuffle_i64x2&expand=5183) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vshufi64x2, MASK = 0b10_11_11_11))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shuffle_i64x2(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(MASK); + let a = a.as_i64x8(); + let b = b.as_i64x8(); + let r: i64x8 = simd_shuffle8!( + a, + b, + [ + (MASK as u32 & 0b11) * 2 + 0, + (MASK as u32 & 0b11) * 2 + 1, + ((MASK as u32 >> 2) & 0b11) * 2 + 0, + ((MASK as u32 >> 2) & 0b11) * 2 + 1, + ((MASK as u32 >> 4) & 0b11) * 2 + 0 + 8, + ((MASK as u32 >> 4) & 0b11) * 2 + 1 + 8, + ((MASK as u32 >> 6) & 0b11) * 2 + 0 + 8, + ((MASK as u32 >> 6) & 0b11) * 2 + 1 + 8, + ], + ); + transmute(r) +} + +/// Shuffle 128-bits (composed of 2 64-bit integers) selected by imm8 from a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shuffle_i64x&expand=5181) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vshufi64x2, MASK = 0b10_11_11_11))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shuffle_i64x2( + src: __m512i, + k: __mmask8, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(MASK); + let r = _mm512_shuffle_i64x2::(a, b); + transmute(simd_select_bitmask(k, r.as_i64x8(), src.as_i64x8())) +} + +/// Shuffle 128-bits (composed of 2 64-bit integers) selected by imm8 from a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shuffle_i64&expand=5182) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vshufi64x2, MASK = 0b10_11_11_11))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shuffle_i64x2( + k: __mmask8, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(MASK); + let r = _mm512_shuffle_i64x2::(a, b); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, r.as_i64x8(), zero)) +} + +/// Shuffle 128-bits (composed of 2 64-bit integers) selected by imm8 from a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shuffle_i64x2&expand=5180) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm, MASK = 0b01))] //should be vshufi64x2 +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_shuffle_i64x2(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(MASK); + let a = a.as_i64x4(); + let b = b.as_i64x4(); + let r: i64x4 = simd_shuffle4!( + a, + b, + [ + (MASK as u32 & 0b1) * 2 + 0, + (MASK as u32 & 0b1) * 2 + 1, + ((MASK as u32 >> 1) & 0b1) * 2 + 0 + 4, + ((MASK as u32 >> 1) & 0b1) * 2 + 1 + 4, + ], + ); + transmute(r) +} + +/// Shuffle 128-bits (composed of 2 64-bit integers) selected by imm8 from a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shuffle_i64x2&expand=5178) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vshufi64x2, MASK = 0b11))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shuffle_i64x2( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(MASK); + let r = _mm256_shuffle_i64x2::(a, b); + transmute(simd_select_bitmask(k, r.as_i64x4(), src.as_i64x4())) +} + +/// Shuffle 128-bits (composed of 2 64-bit integers) selected by imm8 from a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shuffle_i64x2&expand=5179) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vshufi64x2, MASK = 0b11))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shuffle_i64x2( + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(MASK); + let r = _mm256_shuffle_i64x2::(a, b); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, r.as_i64x4(), zero)) +} + +/// Shuffle 128-bits (composed of 4 single-precision (32-bit) floating-point elements) selected by imm8 from a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shuffle_f32x4&expand=5165) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vshuff64x2, MASK = 0b1011))] //should be vshuff32x4, but generate vshuff64x2 +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shuffle_f32x4(a: __m512, b: __m512) -> __m512 { + static_assert_imm8!(MASK); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r: f32x16 = simd_shuffle16!( + a, + b, + [ + (MASK as u32 & 0b11) * 4 + 0, + (MASK as u32 & 0b11) * 4 + 1, + (MASK as u32 & 0b11) * 4 + 2, + (MASK as u32 & 0b11) * 4 + 3, + ((MASK as u32 >> 2) & 0b11) * 4 + 0, + ((MASK as u32 >> 2) & 0b11) * 4 + 1, + ((MASK as u32 >> 2) & 0b11) * 4 + 2, + ((MASK as u32 >> 2) & 0b11) * 4 + 3, + ((MASK as u32 >> 4) & 0b11) * 4 + 0 + 16, + ((MASK as u32 >> 4) & 0b11) * 4 + 1 + 16, + ((MASK as u32 >> 4) & 0b11) * 4 + 2 + 16, + ((MASK as u32 >> 4) & 0b11) * 4 + 3 + 16, + ((MASK as u32 >> 6) & 0b11) * 4 + 0 + 16, + ((MASK as u32 >> 6) & 0b11) * 4 + 1 + 16, + ((MASK as u32 >> 6) & 0b11) * 4 + 2 + 16, + ((MASK as u32 >> 6) & 0b11) * 4 + 3 + 16, + ], + ); + transmute(r) +} + +/// Shuffle 128-bits (composed of 4 single-precision (32-bit) floating-point elements) selected by imm8 from a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shuffle_f32&expand=5163) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vshuff32x4, MASK = 0b1011))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shuffle_f32x4( + src: __m512, + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_imm8!(MASK); + let r = _mm512_shuffle_f32x4::(a, b); + transmute(simd_select_bitmask(k, r.as_f32x16(), src.as_f32x16())) +} + +/// Shuffle 128-bits (composed of 4 single-precision (32-bit) floating-point elements) selected by imm8 from a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shuffle_f32&expand=5164) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vshuff32x4, MASK = 0b1011))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shuffle_f32x4( + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_imm8!(MASK); + let r = _mm512_shuffle_f32x4::(a, b); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, r.as_f32x16(), zero)) +} + +/// Shuffle 128-bits (composed of 4 single-precision (32-bit) floating-point elements) selected by imm8 from a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shuffle_f32x4&expand=5162) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm, MASK = 0b01))] //should be vshuff32x4 +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_shuffle_f32x4(a: __m256, b: __m256) -> __m256 { + static_assert_imm8!(MASK); + let a = a.as_f32x8(); + let b = b.as_f32x8(); + let r: f32x8 = simd_shuffle8!( + a, + b, + [ + (MASK as u32 & 0b1) * 4 + 0, + (MASK as u32 & 0b1) * 4 + 1, + (MASK as u32 & 0b1) * 4 + 2, + (MASK as u32 & 0b1) * 4 + 3, + ((MASK as u32 >> 1) & 0b1) * 4 + 0 + 8, + ((MASK as u32 >> 1) & 0b1) * 4 + 1 + 8, + ((MASK as u32 >> 1) & 0b1) * 4 + 2 + 8, + ((MASK as u32 >> 1) & 0b1) * 4 + 3 + 8, + ], + ); + transmute(r) +} + +/// Shuffle 128-bits (composed of 4 single-precision (32-bit) floating-point elements) selected by imm8 from a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shuffle_f32x4&expand=5160) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vshuff32x4, MASK = 0b11))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shuffle_f32x4( + src: __m256, + k: __mmask8, + a: __m256, + b: __m256, +) -> __m256 { + static_assert_imm8!(MASK); + let r = _mm256_shuffle_f32x4::(a, b); + transmute(simd_select_bitmask(k, r.as_f32x8(), src.as_f32x8())) +} + +/// Shuffle 128-bits (composed of 4 single-precision (32-bit) floating-point elements) selected by imm8 from a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shuffle_f32x4&expand=5161) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vshuff32x4, MASK = 0b11))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shuffle_f32x4( + k: __mmask8, + a: __m256, + b: __m256, +) -> __m256 { + static_assert_imm8!(MASK); + let r = _mm256_shuffle_f32x4::(a, b); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, r.as_f32x8(), zero)) +} + +/// Shuffle 128-bits (composed of 2 double-precision (64-bit) floating-point elements) selected by imm8 from a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shuffle_f64x2&expand=5171) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vshuff64x2, MASK = 0b10_11_11_11))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shuffle_f64x2(a: __m512d, b: __m512d) -> __m512d { + static_assert_imm8!(MASK); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r: f64x8 = simd_shuffle8!( + a, + b, + [ + (MASK as u32 & 0b11) * 2 + 0, + (MASK as u32 & 0b11) * 2 + 1, + ((MASK as u32 >> 2) & 0b11) * 2 + 0, + ((MASK as u32 >> 2) & 0b11) * 2 + 1, + ((MASK as u32 >> 4) & 0b11) * 2 + 0 + 8, + ((MASK as u32 >> 4) & 0b11) * 2 + 1 + 8, + ((MASK as u32 >> 6) & 0b11) * 2 + 0 + 8, + ((MASK as u32 >> 6) & 0b11) * 2 + 1 + 8, + ], + ); + transmute(r) +} + +/// Shuffle 128-bits (composed of 2 double-precision (64-bit) floating-point elements) selected by imm8 from a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shuffle_f64x2&expand=5169) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vshuff64x2, MASK = 0b10_11_11_11))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shuffle_f64x2( + src: __m512d, + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + static_assert_imm8!(MASK); + let r = _mm512_shuffle_f64x2::(a, b); + transmute(simd_select_bitmask(k, r.as_f64x8(), src.as_f64x8())) +} + +/// Shuffle 128-bits (composed of 2 double-precision (64-bit) floating-point elements) selected by imm8 from a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shuffle_f64x2&expand=5170) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vshuff64x2, MASK = 0b10_11_11_11))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shuffle_f64x2( + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + static_assert_imm8!(MASK); + let r = _mm512_shuffle_f64x2::(a, b); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, r.as_f64x8(), zero)) +} + +/// Shuffle 128-bits (composed of 2 double-precision (64-bit) floating-point elements) selected by imm8 from a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shuffle_f64x2&expand=5168) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vperm, MASK = 0b01))] //should be vshuff64x2 +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_shuffle_f64x2(a: __m256d, b: __m256d) -> __m256d { + static_assert_imm8!(MASK); + let a = a.as_f64x4(); + let b = b.as_f64x4(); + let r: f64x4 = simd_shuffle4!( + a, + b, + [ + (MASK as u32 & 0b1) * 2 + 0, + (MASK as u32 & 0b1) * 2 + 1, + ((MASK as u32 >> 1) & 0b1) * 2 + 0 + 4, + ((MASK as u32 >> 1) & 0b1) * 2 + 1 + 4, + ], + ); + transmute(r) +} + +/// Shuffle 128-bits (composed of 2 double-precision (64-bit) floating-point elements) selected by imm8 from a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shuffle_f64x2&expand=5166) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vshuff64x2, MASK = 0b11))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shuffle_f64x2( + src: __m256d, + k: __mmask8, + a: __m256d, + b: __m256d, +) -> __m256d { + static_assert_imm8!(MASK); + let r = _mm256_shuffle_f64x2::(a, b); + transmute(simd_select_bitmask(k, r.as_f64x4(), src.as_f64x4())) +} + +/// Shuffle 128-bits (composed of 2 double-precision (64-bit) floating-point elements) selected by imm8 from a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shuffle_f64x2&expand=5167) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vshuff64x2, MASK = 0b11))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shuffle_f64x2( + k: __mmask8, + a: __m256d, + b: __m256d, +) -> __m256d { + static_assert_imm8!(MASK); + let r = _mm256_shuffle_f64x2::(a, b); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, r.as_f64x4(), zero)) +} + +/// Extract 128 bits (composed of 4 packed single-precision (32-bit) floating-point elements) from a, selected with imm8, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_extractf32x4_ps&expand=2442) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vextractf32x4, IMM8 = 3) +)] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_extractf32x4_ps(a: __m512) -> __m128 { + static_assert_imm2!(IMM8); + match IMM8 & 0x3 { + 0 => simd_shuffle4!(a, _mm512_undefined_ps(), [0, 1, 2, 3]), + 1 => simd_shuffle4!(a, _mm512_undefined_ps(), [4, 5, 6, 7]), + 2 => simd_shuffle4!(a, _mm512_undefined_ps(), [8, 9, 10, 11]), + _ => simd_shuffle4!(a, _mm512_undefined_ps(), [12, 13, 14, 15]), + } +} + +/// Extract 128 bits (composed of 4 packed single-precision (32-bit) floating-point elements) from a, selected with imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_extractf32x4_ps&expand=2443) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vextractf32x4, IMM8 = 3) +)] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_extractf32x4_ps( + src: __m128, + k: __mmask8, + a: __m512, +) -> __m128 { + static_assert_imm2!(IMM8); + let r = _mm512_extractf32x4_ps::(a); + transmute(simd_select_bitmask(k, r.as_f32x4(), src.as_f32x4())) +} + +/// Extract 128 bits (composed of 4 packed single-precision (32-bit) floating-point elements) from a, selected with imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_extractf32x4_ps&expand=2444) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vextractf32x4, IMM8 = 3) +)] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_extractf32x4_ps(k: __mmask8, a: __m512) -> __m128 { + static_assert_imm2!(IMM8); + let r = _mm512_extractf32x4_ps::(a); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, r.as_f32x4(), zero)) +} + +/// Extract 128 bits (composed of 4 packed single-precision (32-bit) floating-point elements) from a, selected with imm8, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_extractf32x4_ps&expand=2439) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vextract, IMM8 = 1) //should be vextractf32x4 +)] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_extractf32x4_ps(a: __m256) -> __m128 { + static_assert_imm1!(IMM8); + match IMM8 & 0x1 { + 0 => simd_shuffle4!(a, _mm256_undefined_ps(), [0, 1, 2, 3]), + _ => simd_shuffle4!(a, _mm256_undefined_ps(), [4, 5, 6, 7]), + } +} + +/// Extract 128 bits (composed of 4 packed single-precision (32-bit) floating-point elements) from a, selected with imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_extractf32x4_ps&expand=2440) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vextractf32x4, IMM8 = 1) +)] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_extractf32x4_ps( + src: __m128, + k: __mmask8, + a: __m256, +) -> __m128 { + static_assert_imm1!(IMM8); + let r = _mm256_extractf32x4_ps::(a); + transmute(simd_select_bitmask(k, r.as_f32x4(), src.as_f32x4())) +} + +/// Extract 128 bits (composed of 4 packed single-precision (32-bit) floating-point elements) from a, selected with imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_extractf32x4_ps&expand=2441) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vextractf32x4, IMM8 = 1) +)] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_extractf32x4_ps(k: __mmask8, a: __m256) -> __m128 { + static_assert_imm1!(IMM8); + let r = _mm256_extractf32x4_ps::(a); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, r.as_f32x4(), zero)) +} + +/// Extract 256 bits (composed of 4 packed 64-bit integers) from a, selected with IMM1, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_extracti64x4_epi64&expand=2473) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vextractf64x4, IMM1 = 1) //should be vextracti64x4 +)] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_extracti64x4_epi64(a: __m512i) -> __m256i { + static_assert_imm1!(IMM1); + match IMM1 { + 0 => simd_shuffle4!(a, _mm512_set1_epi64(0), [0, 1, 2, 3]), + _ => simd_shuffle4!(a, _mm512_set1_epi64(0), [4, 5, 6, 7]), + } +} + +/// Extract 256 bits (composed of 4 packed 64-bit integers) from a, selected with IMM1, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_extracti64x4_epi64&expand=2474) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vextracti64x4, IMM1 = 1) +)] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_extracti64x4_epi64( + src: __m256i, + k: __mmask8, + a: __m512i, +) -> __m256i { + static_assert_imm1!(IMM1); + let r = _mm512_extracti64x4_epi64::(a); + transmute(simd_select_bitmask(k, r.as_i64x4(), src.as_i64x4())) +} + +/// Extract 256 bits (composed of 4 packed 64-bit integers) from a, selected with IMM1, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_extracti64x4_epi64&expand=2475) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vextracti64x4, IMM1 = 1) +)] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_extracti64x4_epi64(k: __mmask8, a: __m512i) -> __m256i { + static_assert_imm1!(IMM1); + let r = _mm512_extracti64x4_epi64::(a); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, r.as_i64x4(), zero)) +} + +/// Extract 256 bits (composed of 4 packed double-precision (64-bit) floating-point elements) from a, selected with imm8, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_extractf64x4_pd&expand=2454) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vextractf64x4, IMM8 = 1) +)] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_extractf64x4_pd(a: __m512d) -> __m256d { + static_assert_imm1!(IMM8); + match IMM8 & 0x1 { + 0 => simd_shuffle4!(a, _mm512_undefined_pd(), [0, 1, 2, 3]), + _ => simd_shuffle4!(a, _mm512_undefined_pd(), [4, 5, 6, 7]), + } +} + +/// Extract 256 bits (composed of 4 packed double-precision (64-bit) floating-point elements) from a, selected with imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_extractf64x4_pd&expand=2455) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vextractf64x4, IMM8 = 1) +)] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_extractf64x4_pd( + src: __m256d, + k: __mmask8, + a: __m512d, +) -> __m256d { + static_assert_imm1!(IMM8); + let r = _mm512_extractf64x4_pd::(a); + transmute(simd_select_bitmask(k, r.as_f64x4(), src.as_f64x4())) +} + +/// Extract 256 bits (composed of 4 packed double-precision (64-bit) floating-point elements) from a, selected with imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_extractf64x4_pd&expand=2456) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vextractf64x4, IMM8 = 1) +)] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_extractf64x4_pd(k: __mmask8, a: __m512d) -> __m256d { + static_assert_imm1!(IMM8); + let r = _mm512_extractf64x4_pd::(a); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, r.as_f64x4(), zero)) +} + +/// Extract 128 bits (composed of 4 packed 32-bit integers) from a, selected with IMM2, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_extracti32x4_epi32&expand=2461) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vextractf32x4, IMM2 = 3) //should be vextracti32x4 +)] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_extracti32x4_epi32(a: __m512i) -> __m128i { + static_assert_imm2!(IMM2); + let a = a.as_i32x16(); + let undefined = _mm512_undefined_epi32().as_i32x16(); + let extract: i32x4 = match IMM2 { + 0 => simd_shuffle4!(a, undefined, [0, 1, 2, 3]), + 1 => simd_shuffle4!(a, undefined, [4, 5, 6, 7]), + 2 => simd_shuffle4!(a, undefined, [8, 9, 10, 11]), + _ => simd_shuffle4!(a, undefined, [12, 13, 14, 15]), + }; + transmute(extract) +} + +/// Extract 128 bits (composed of 4 packed 32-bit integers) from a, selected with IMM2, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_extracti32x4_epi32&expand=2462) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vextracti32x4, IMM2 = 3) +)] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_extracti32x4_epi32( + src: __m128i, + k: __mmask8, + a: __m512i, +) -> __m128i { + static_assert_imm2!(IMM2); + let r = _mm512_extracti32x4_epi32::(a); + transmute(simd_select_bitmask(k, r.as_i32x4(), src.as_i32x4())) +} + +/// Extract 128 bits (composed of 4 packed 32-bit integers) from a, selected with IMM2, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_extracti32x4_epi32&expand=2463) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vextracti32x4, IMM2 = 3) +)] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_extracti32x4_epi32(k: __mmask8, a: __m512i) -> __m128i { + static_assert_imm2!(IMM2); + let r = _mm512_extracti32x4_epi32::(a); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, r.as_i32x4(), zero)) +} + +/// Extract 128 bits (composed of 4 packed 32-bit integers) from a, selected with IMM1, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_extracti32x4_epi32&expand=2458) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vextract, IMM1 = 1) //should be vextracti32x4 +)] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_extracti32x4_epi32(a: __m256i) -> __m128i { + static_assert_imm1!(IMM1); + let a = a.as_i32x8(); + let undefined = _mm256_undefined_si256().as_i32x8(); + let extract: i32x4 = match IMM1 { + 0 => simd_shuffle4!(a, undefined, [0, 1, 2, 3]), + _ => simd_shuffle4!(a, undefined, [4, 5, 6, 7]), + }; + transmute(extract) +} + +/// Extract 128 bits (composed of 4 packed 32-bit integers) from a, selected with IMM1, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_extracti32x4_epi32&expand=2459) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vextracti32x4, IMM1 = 1) +)] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_extracti32x4_epi32( + src: __m128i, + k: __mmask8, + a: __m256i, +) -> __m128i { + static_assert_imm1!(IMM1); + let r = _mm256_extracti32x4_epi32::(a); + transmute(simd_select_bitmask(k, r.as_i32x4(), src.as_i32x4())) +} + +/// Extract 128 bits (composed of 4 packed 32-bit integers) from a, selected with IMM1, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_extracti32x4_epi32&expand=2460) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vextracti32x4, IMM1 = 1) +)] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_extracti32x4_epi32(k: __mmask8, a: __m256i) -> __m128i { + static_assert_imm1!(IMM1); + let r = _mm256_extracti32x4_epi32::(a); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, r.as_i32x4(), zero)) +} + +/// Duplicate even-indexed single-precision (32-bit) floating-point elements from a, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_moveldup_ps&expand=3862) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovsldup))] +pub unsafe fn _mm512_moveldup_ps(a: __m512) -> __m512 { + let r: f32x16 = simd_shuffle16!(a, a, [0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14]); + transmute(r) +} + +/// Duplicate even-indexed single-precision (32-bit) floating-point elements from a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_moveldup_ps&expand=3860) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovsldup))] +pub unsafe fn _mm512_mask_moveldup_ps(src: __m512, k: __mmask16, a: __m512) -> __m512 { + let mov: f32x16 = simd_shuffle16!(a, a, [0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14]); + transmute(simd_select_bitmask(k, mov, src.as_f32x16())) +} + +/// Duplicate even-indexed single-precision (32-bit) floating-point elements from a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_moveldup_ps&expand=3861) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovsldup))] +pub unsafe fn _mm512_maskz_moveldup_ps(k: __mmask16, a: __m512) -> __m512 { + let mov: f32x16 = simd_shuffle16!(a, a, [0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14]); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, mov, zero)) +} + +/// Duplicate even-indexed single-precision (32-bit) floating-point elements from a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_moveldup_ps&expand=3857) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovsldup))] +pub unsafe fn _mm256_mask_moveldup_ps(src: __m256, k: __mmask8, a: __m256) -> __m256 { + let mov = _mm256_moveldup_ps(a); + transmute(simd_select_bitmask(k, mov.as_f32x8(), src.as_f32x8())) +} + +/// Duplicate even-indexed single-precision (32-bit) floating-point elements from a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_moveldup_ps&expand=3858) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovsldup))] +pub unsafe fn _mm256_maskz_moveldup_ps(k: __mmask8, a: __m256) -> __m256 { + let mov = _mm256_moveldup_ps(a); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, mov.as_f32x8(), zero)) +} + +/// Duplicate even-indexed single-precision (32-bit) floating-point elements from a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_moveldup_ps&expand=3854) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovsldup))] +pub unsafe fn _mm_mask_moveldup_ps(src: __m128, k: __mmask8, a: __m128) -> __m128 { + let mov = _mm_moveldup_ps(a); + transmute(simd_select_bitmask(k, mov.as_f32x4(), src.as_f32x4())) +} + +/// Duplicate even-indexed single-precision (32-bit) floating-point elements from a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_moveldup_ps&expand=3855) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovsldup))] +pub unsafe fn _mm_maskz_moveldup_ps(k: __mmask8, a: __m128) -> __m128 { + let mov = _mm_moveldup_ps(a); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, mov.as_f32x4(), zero)) +} + +/// Duplicate odd-indexed single-precision (32-bit) floating-point elements from a, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_movehdup_ps&expand=3852) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovshdup))] +pub unsafe fn _mm512_movehdup_ps(a: __m512) -> __m512 { + let r: f32x16 = simd_shuffle16!(a, a, [1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15]); + transmute(r) +} + +/// Duplicate odd-indexed single-precision (32-bit) floating-point elements from a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_movehdup&expand=3850) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovshdup))] +pub unsafe fn _mm512_mask_movehdup_ps(src: __m512, k: __mmask16, a: __m512) -> __m512 { + let mov: f32x16 = simd_shuffle16!(a, a, [1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15]); + transmute(simd_select_bitmask(k, mov, src.as_f32x16())) +} + +/// Duplicate odd-indexed single-precision (32-bit) floating-point elements from a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_moveh&expand=3851) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovshdup))] +pub unsafe fn _mm512_maskz_movehdup_ps(k: __mmask16, a: __m512) -> __m512 { + let mov: f32x16 = simd_shuffle16!(a, a, [1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15]); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, mov, zero)) +} + +/// Duplicate odd-indexed single-precision (32-bit) floating-point elements from a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_movehdup_ps&expand=3847) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovshdup))] +pub unsafe fn _mm256_mask_movehdup_ps(src: __m256, k: __mmask8, a: __m256) -> __m256 { + let mov = _mm256_movehdup_ps(a); + transmute(simd_select_bitmask(k, mov.as_f32x8(), src.as_f32x8())) +} + +/// Duplicate odd-indexed single-precision (32-bit) floating-point elements from a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_movehdup_ps&expand=3848) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovshdup))] +pub unsafe fn _mm256_maskz_movehdup_ps(k: __mmask8, a: __m256) -> __m256 { + let mov = _mm256_movehdup_ps(a); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, mov.as_f32x8(), zero)) +} + +/// Duplicate odd-indexed single-precision (32-bit) floating-point elements from a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_movehdup_ps&expand=3844) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovshdup))] +pub unsafe fn _mm_mask_movehdup_ps(src: __m128, k: __mmask8, a: __m128) -> __m128 { + let mov = _mm_movehdup_ps(a); + transmute(simd_select_bitmask(k, mov.as_f32x4(), src.as_f32x4())) +} + +/// Duplicate odd-indexed single-precision (32-bit) floating-point elements from a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_movehdup_ps&expand=3845) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovshdup))] +pub unsafe fn _mm_maskz_movehdup_ps(k: __mmask8, a: __m128) -> __m128 { + let mov = _mm_movehdup_ps(a); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, mov.as_f32x4(), zero)) +} + +/// Duplicate even-indexed double-precision (64-bit) floating-point elements from a, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_movedup_pd&expand=3843) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovddup))] +pub unsafe fn _mm512_movedup_pd(a: __m512d) -> __m512d { + let r: f64x8 = simd_shuffle8!(a, a, [0, 0, 2, 2, 4, 4, 6, 6]); + transmute(r) +} + +/// Duplicate even-indexed double-precision (64-bit) floating-point elements from a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_movedup_pd&expand=3841) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovddup))] +pub unsafe fn _mm512_mask_movedup_pd(src: __m512d, k: __mmask8, a: __m512d) -> __m512d { + let mov: f64x8 = simd_shuffle8!(a, a, [0, 0, 2, 2, 4, 4, 6, 6]); + transmute(simd_select_bitmask(k, mov, src.as_f64x8())) +} + +/// Duplicate even-indexed double-precision (64-bit) floating-point elements from a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_movedup_pd&expand=3842) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovddup))] +pub unsafe fn _mm512_maskz_movedup_pd(k: __mmask8, a: __m512d) -> __m512d { + let mov: f64x8 = simd_shuffle8!(a, a, [0, 0, 2, 2, 4, 4, 6, 6]); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, mov, zero)) +} + +/// Duplicate even-indexed double-precision (64-bit) floating-point elements from a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_movedup_pd&expand=3838) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovddup))] +pub unsafe fn _mm256_mask_movedup_pd(src: __m256d, k: __mmask8, a: __m256d) -> __m256d { + let mov = _mm256_movedup_pd(a); + transmute(simd_select_bitmask(k, mov.as_f64x4(), src.as_f64x4())) +} + +/// Duplicate even-indexed double-precision (64-bit) floating-point elements from a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_movedup_pd&expand=3839) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovddup))] +pub unsafe fn _mm256_maskz_movedup_pd(k: __mmask8, a: __m256d) -> __m256d { + let mov = _mm256_movedup_pd(a); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, mov.as_f64x4(), zero)) +} + +/// Duplicate even-indexed double-precision (64-bit) floating-point elements from a, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_movedup_pd&expand=3835) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovddup))] +pub unsafe fn _mm_mask_movedup_pd(src: __m128d, k: __mmask8, a: __m128d) -> __m128d { + let mov = _mm_movedup_pd(a); + transmute(simd_select_bitmask(k, mov.as_f64x2(), src.as_f64x2())) +} + +/// Duplicate even-indexed double-precision (64-bit) floating-point elements from a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_movedup_pd&expand=3836) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovddup))] +pub unsafe fn _mm_maskz_movedup_pd(k: __mmask8, a: __m128d) -> __m128d { + let mov = _mm_movedup_pd(a); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, mov.as_f64x2(), zero)) +} + +/// Copy a to dst, then insert 128 bits (composed of 4 packed 32-bit integers) from b into dst at the location specified by imm8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_inserti32x4&expand=3174) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vinsertf32x4, IMM8 = 2))] //should be vinserti32x4 +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_inserti32x4(a: __m512i, b: __m128i) -> __m512i { + static_assert_imm2!(IMM8); + let a = a.as_i32x16(); + let b = _mm512_castsi128_si512(b).as_i32x16(); + let ret: i32x16 = match IMM8 & 0b11 { + 0 => simd_shuffle16!( + a, + b, + [16, 17, 18, 19, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + ), + 1 => simd_shuffle16!( + a, + b, + [0, 1, 2, 3, 16, 17, 18, 19, 8, 9, 10, 11, 12, 13, 14, 15], + ), + 2 => simd_shuffle16!( + a, + b, + [0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 12, 13, 14, 15], + ), + _ => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16, 17, 18, 19]), + }; + transmute(ret) +} + +/// Copy a to tmp, then insert 128 bits (composed of 4 packed 32-bit integers) from b into tmp at the location specified by imm8. Store tmp to dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_inserti32x4&expand=3175) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vinserti32x4, IMM8 = 2))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_inserti32x4( + src: __m512i, + k: __mmask16, + a: __m512i, + b: __m128i, +) -> __m512i { + static_assert_imm2!(IMM8); + let r = _mm512_inserti32x4::(a, b); + transmute(simd_select_bitmask(k, r.as_i32x16(), src.as_i32x16())) +} + +/// Copy a to tmp, then insert 128 bits (composed of 4 packed 32-bit integers) from b into tmp at the location specified by imm8. Store tmp to dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_inserti32x4&expand=3176) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vinserti32x4, IMM8 = 2))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_inserti32x4( + k: __mmask16, + a: __m512i, + b: __m128i, +) -> __m512i { + static_assert_imm2!(IMM8); + let r = _mm512_inserti32x4::(a, b); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, r.as_i32x16(), zero)) +} + +/// Copy a to dst, then insert 128 bits (composed of 4 packed 32-bit integers) from b into dst at the location specified by imm8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_inserti32x4&expand=3171) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vinsert, IMM8 = 1) //should be vinserti32x4 +)] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_inserti32x4(a: __m256i, b: __m128i) -> __m256i { + static_assert_imm1!(IMM8); + let a = a.as_i32x8(); + let b = _mm256_castsi128_si256(b).as_i32x8(); + let ret: i32x8 = match IMM8 & 0b1 { + 0 => simd_shuffle8!(a, b, [8, 9, 10, 11, 4, 5, 6, 7]), + _ => simd_shuffle8!(a, b, [0, 1, 2, 3, 8, 9, 10, 11]), + }; + transmute(ret) +} + +/// Copy a to tmp, then insert 128 bits (composed of 4 packed 32-bit integers) from b into tmp at the location specified by imm8. Store tmp to dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_inserti32x4&expand=3172) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vinserti32x4, IMM8 = 1) +)] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_inserti32x4( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m128i, +) -> __m256i { + static_assert_imm1!(IMM8); + let r = _mm256_inserti32x4::(a, b); + transmute(simd_select_bitmask(k, r.as_i32x8(), src.as_i32x8())) +} + +/// Copy a to tmp, then insert 128 bits (composed of 4 packed 32-bit integers) from b into tmp at the location specified by imm8. Store tmp to dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_inserti32x4&expand=3173) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vinserti32x4, IMM8 = 1) +)] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_inserti32x4( + k: __mmask8, + a: __m256i, + b: __m128i, +) -> __m256i { + static_assert_imm1!(IMM8); + let r = _mm256_inserti32x4::(a, b); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, r.as_i32x8(), zero)) +} + +/// Copy a to dst, then insert 256 bits (composed of 4 packed 64-bit integers) from b into dst at the location specified by imm8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_inserti64x4&expand=3186) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vinsertf64x4, IMM8 = 1))] //should be vinserti64x4 +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_inserti64x4(a: __m512i, b: __m256i) -> __m512i { + static_assert_imm1!(IMM8); + let b = _mm512_castsi256_si512(b); + match IMM8 & 0b1 { + 0 => simd_shuffle8!(a, b, [8, 9, 10, 11, 4, 5, 6, 7]), + _ => simd_shuffle8!(a, b, [0, 1, 2, 3, 8, 9, 10, 11]), + } +} + +/// Copy a to tmp, then insert 256 bits (composed of 4 packed 64-bit integers) from b into tmp at the location specified by imm8. Store tmp to dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_inserti64x4&expand=3187) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vinserti64x4, IMM8 = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_inserti64x4( + src: __m512i, + k: __mmask8, + a: __m512i, + b: __m256i, +) -> __m512i { + static_assert_imm1!(IMM8); + let r = _mm512_inserti64x4::(a, b); + transmute(simd_select_bitmask(k, r.as_i64x8(), src.as_i64x8())) +} + +/// Copy a to tmp, then insert 256 bits (composed of 4 packed 64-bit integers) from b into tmp at the location specified by imm8. Store tmp to dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_inserti64x4&expand=3188) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vinserti64x4, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_inserti64x4( + k: __mmask8, + a: __m512i, + b: __m256i, +) -> __m512i { + static_assert_imm1!(IMM8); + let r = _mm512_inserti64x4::(a, b); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, r.as_i64x8(), zero)) +} + +/// Copy a to dst, then insert 128 bits (composed of 4 packed single-precision (32-bit) floating-point elements) from b into dst at the location specified by imm8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_insertf32x4&expand=3155) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vinsertf32x4, IMM8 = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_insertf32x4(a: __m512, b: __m128) -> __m512 { + static_assert_imm2!(IMM8); + let b = _mm512_castps128_ps512(b); + match IMM8 & 0b11 { + 0 => simd_shuffle16!( + a, + b, + [16, 17, 18, 19, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + ), + 1 => simd_shuffle16!( + a, + b, + [0, 1, 2, 3, 16, 17, 18, 19, 8, 9, 10, 11, 12, 13, 14, 15], + ), + 2 => simd_shuffle16!( + a, + b, + [0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 12, 13, 14, 15], + ), + _ => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16, 17, 18, 19]), + } +} + +/// Copy a to tmp, then insert 128 bits (composed of 4 packed single-precision (32-bit) floating-point elements) from b into tmp at the location specified by imm8. Store tmp to dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_insertf32x4&expand=3156) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vinsertf32x4, IMM8 = 2))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_insertf32x4( + src: __m512, + k: __mmask16, + a: __m512, + b: __m128, +) -> __m512 { + static_assert_imm2!(IMM8); + let r = _mm512_insertf32x4::(a, b); + transmute(simd_select_bitmask(k, r.as_f32x16(), src.as_f32x16())) +} + +/// Copy a to tmp, then insert 128 bits (composed of 4 packed single-precision (32-bit) floating-point elements) from b into tmp at the location specified by imm8. Store tmp to dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_insertf32x4&expand=3157) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vinsertf32x4, IMM8 = 2))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_insertf32x4( + k: __mmask16, + a: __m512, + b: __m128, +) -> __m512 { + static_assert_imm2!(IMM8); + let r = _mm512_insertf32x4::(a, b); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, r.as_f32x16(), zero)) +} + +/// Copy a to dst, then insert 128 bits (composed of 4 packed single-precision (32-bit) floating-point elements) from b into dst at the location specified by imm8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_insertf32x4&expand=3152) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vinsert, IMM8 = 1) //should be vinsertf32x4 +)] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_insertf32x4(a: __m256, b: __m128) -> __m256 { + static_assert_imm1!(IMM8); + let b = _mm256_castps128_ps256(b); + match IMM8 & 0b1 { + 0 => simd_shuffle8!(a, b, [8, 9, 10, 11, 4, 5, 6, 7]), + _ => simd_shuffle8!(a, b, [0, 1, 2, 3, 8, 9, 10, 11]), + } +} + +/// Copy a to tmp, then insert 128 bits (composed of 4 packed single-precision (32-bit) floating-point elements) from b into tmp at the location specified by imm8. Store tmp to dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_insertf32x4&expand=3153) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vinsertf32x4, IMM8 = 1) +)] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_insertf32x4( + src: __m256, + k: __mmask8, + a: __m256, + b: __m128, +) -> __m256 { + static_assert_imm1!(IMM8); + let r = _mm256_insertf32x4::(a, b); + transmute(simd_select_bitmask(k, r.as_f32x8(), src.as_f32x8())) +} + +/// Copy a to tmp, then insert 128 bits (composed of 4 packed single-precision (32-bit) floating-point elements) from b into tmp at the location specified by imm8. Store tmp to dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_insertf32x4&expand=3154) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr( + all(test, not(target_os = "windows")), + assert_instr(vinsertf32x4, IMM8 = 1) +)] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_insertf32x4( + k: __mmask8, + a: __m256, + b: __m128, +) -> __m256 { + static_assert_imm1!(IMM8); + let r = _mm256_insertf32x4::(a, b); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, r.as_f32x8(), zero)) +} + +/// Copy a to dst, then insert 256 bits (composed of 4 packed double-precision (64-bit) floating-point elements) from b into dst at the location specified by imm8. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_insertf64x4&expand=3167) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vinsertf64x4, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_insertf64x4(a: __m512d, b: __m256d) -> __m512d { + static_assert_imm1!(IMM8); + let b = _mm512_castpd256_pd512(b); + match IMM8 & 0b1 { + 0 => simd_shuffle8!(a, b, [8, 9, 10, 11, 4, 5, 6, 7]), + _ => simd_shuffle8!(a, b, [0, 1, 2, 3, 8, 9, 10, 11]), + } +} + +/// Copy a to tmp, then insert 256 bits (composed of 4 packed double-precision (64-bit) floating-point elements) from b into tmp at the location specified by imm8. Store tmp to dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_insertf64x4&expand=3168) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vinsertf64x4, IMM8 = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_insertf64x4( + src: __m512d, + k: __mmask8, + a: __m512d, + b: __m256d, +) -> __m512d { + static_assert_imm1!(IMM8); + let r = _mm512_insertf64x4::(a, b); + transmute(simd_select_bitmask(k, r.as_f64x8(), src.as_f64x8())) +} + +/// Copy a to tmp, then insert 256 bits (composed of 4 packed double-precision (64-bit) floating-point elements) from b into tmp at the location specified by imm8. Store tmp to dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_insertf64x4&expand=3169) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vinsertf64x4, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_insertf64x4( + k: __mmask8, + a: __m512d, + b: __m256d, +) -> __m512d { + static_assert_imm1!(IMM8); + let r = _mm512_insertf64x4::(a, b); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, r.as_f64x8(), zero)) +} + +/// Unpack and interleave 32-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_unpackhi_epi32&expand=6021) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vunpckhps))] //should be vpunpckhdq +pub unsafe fn _mm512_unpackhi_epi32(a: __m512i, b: __m512i) -> __m512i { + let a = a.as_i32x16(); + let b = b.as_i32x16(); + #[rustfmt::skip] + let r: i32x16 = simd_shuffle16!( + a, b, + [ 2, 18, 3, 19, + 2 + 4, 18 + 4, 3 + 4, 19 + 4, + 2 + 8, 18 + 8, 3 + 8, 19 + 8, + 2 + 12, 18 + 12, 3 + 12, 19 + 12], + ); + transmute(r) +} + +/// Unpack and interleave 32-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_unpackhi_epi32&expand=6019) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpunpckhdq))] +pub unsafe fn _mm512_mask_unpackhi_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + let unpackhi = _mm512_unpackhi_epi32(a, b).as_i32x16(); + transmute(simd_select_bitmask(k, unpackhi, src.as_i32x16())) +} + +/// Unpack and interleave 32-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_unpackhi_epi32&expand=6020) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpunpckhdq))] +pub unsafe fn _mm512_maskz_unpackhi_epi32(k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let unpackhi = _mm512_unpackhi_epi32(a, b).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, unpackhi, zero)) +} + +/// Unpack and interleave 32-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_unpackhi_epi32&expand=6016) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpckhdq))] +pub unsafe fn _mm256_mask_unpackhi_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + let unpackhi = _mm256_unpackhi_epi32(a, b).as_i32x8(); + transmute(simd_select_bitmask(k, unpackhi, src.as_i32x8())) +} + +/// Unpack and interleave 32-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_unpackhi_epi32&expand=6017) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpckhdq))] +pub unsafe fn _mm256_maskz_unpackhi_epi32(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let unpackhi = _mm256_unpackhi_epi32(a, b).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, unpackhi, zero)) +} + +/// Unpack and interleave 32-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_unpackhi_epi32&expand=6013) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpckhdq))] +pub unsafe fn _mm_mask_unpackhi_epi32( + src: __m128i, + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + let unpackhi = _mm_unpackhi_epi32(a, b).as_i32x4(); + transmute(simd_select_bitmask(k, unpackhi, src.as_i32x4())) +} + +/// Unpack and interleave 32-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_unpackhi_epi32&expand=6014) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpckhdq))] +pub unsafe fn _mm_maskz_unpackhi_epi32(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let unpackhi = _mm_unpackhi_epi32(a, b).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, unpackhi, zero)) +} + +/// Unpack and interleave 64-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_unpackhi_epi64&expand=6030) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vunpckhpd))] //should be vpunpckhqdq +pub unsafe fn _mm512_unpackhi_epi64(a: __m512i, b: __m512i) -> __m512i { + simd_shuffle8!(a, b, [1, 9, 1 + 2, 9 + 2, 1 + 4, 9 + 4, 1 + 6, 9 + 6]) +} + +/// Unpack and interleave 64-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_unpackhi_epi64&expand=6028) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpunpckhqdq))] +pub unsafe fn _mm512_mask_unpackhi_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, + b: __m512i, +) -> __m512i { + let unpackhi = _mm512_unpackhi_epi64(a, b).as_i64x8(); + transmute(simd_select_bitmask(k, unpackhi, src.as_i64x8())) +} + +/// Unpack and interleave 64-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_unpackhi_epi64&expand=6029) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpunpckhqdq))] +pub unsafe fn _mm512_maskz_unpackhi_epi64(k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let unpackhi = _mm512_unpackhi_epi64(a, b).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, unpackhi, zero)) +} + +/// Unpack and interleave 64-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_unpackhi_epi64&expand=6025) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpckhqdq))] +pub unsafe fn _mm256_mask_unpackhi_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + let unpackhi = _mm256_unpackhi_epi64(a, b).as_i64x4(); + transmute(simd_select_bitmask(k, unpackhi, src.as_i64x4())) +} + +/// Unpack and interleave 64-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_unpackhi_epi64&expand=6026) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpckhqdq))] +pub unsafe fn _mm256_maskz_unpackhi_epi64(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let unpackhi = _mm256_unpackhi_epi64(a, b).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, unpackhi, zero)) +} + +/// Unpack and interleave 64-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_unpackhi_epi64&expand=6022) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpckhqdq))] +pub unsafe fn _mm_mask_unpackhi_epi64( + src: __m128i, + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + let unpackhi = _mm_unpackhi_epi64(a, b).as_i64x2(); + transmute(simd_select_bitmask(k, unpackhi, src.as_i64x2())) +} + +/// Unpack and interleave 64-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_unpackhi_epi64&expand=6023) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpckhqdq))] +pub unsafe fn _mm_maskz_unpackhi_epi64(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let unpackhi = _mm_unpackhi_epi64(a, b).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, unpackhi, zero)) +} + +/// Unpack and interleave single-precision (32-bit) floating-point elements from the high half of each 128-bit lane in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_unpackhi_ps&expand=6060) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vunpckhps))] +pub unsafe fn _mm512_unpackhi_ps(a: __m512, b: __m512) -> __m512 { + #[rustfmt::skip] + simd_shuffle16!( + a, b, + [ 2, 18, 3, 19, + 2 + 4, 18 + 4, 3 + 4, 19 + 4, + 2 + 8, 18 + 8, 3 + 8, 19 + 8, + 2 + 12, 18 + 12, 3 + 12, 19 + 12], + ) +} + +/// Unpack and interleave single-precision (32-bit) floating-point elements from the high half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_unpackhi_ps&expand=6058) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vunpckhps))] +pub unsafe fn _mm512_mask_unpackhi_ps(src: __m512, k: __mmask16, a: __m512, b: __m512) -> __m512 { + let unpackhi = _mm512_unpackhi_ps(a, b).as_f32x16(); + transmute(simd_select_bitmask(k, unpackhi, src.as_f32x16())) +} + +/// Unpack and interleave single-precision (32-bit) floating-point elements from the high half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_unpackhi_ps&expand=6059) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vunpckhps))] +pub unsafe fn _mm512_maskz_unpackhi_ps(k: __mmask16, a: __m512, b: __m512) -> __m512 { + let unpackhi = _mm512_unpackhi_ps(a, b).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, unpackhi, zero)) +} + +/// Unpack and interleave single-precision (32-bit) floating-point elements from the high half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_unpackhi_ps&expand=6055) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vunpckhps))] +pub unsafe fn _mm256_mask_unpackhi_ps(src: __m256, k: __mmask8, a: __m256, b: __m256) -> __m256 { + let unpackhi = _mm256_unpackhi_ps(a, b).as_f32x8(); + transmute(simd_select_bitmask(k, unpackhi, src.as_f32x8())) +} + +/// Unpack and interleave single-precision (32-bit) floating-point elements from the high half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_unpackhi_ps&expand=6056) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vunpckhps))] +pub unsafe fn _mm256_maskz_unpackhi_ps(k: __mmask8, a: __m256, b: __m256) -> __m256 { + let unpackhi = _mm256_unpackhi_ps(a, b).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, unpackhi, zero)) +} + +/// Unpack and interleave single-precision (32-bit) floating-point elements from the high half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_unpackhi_ps&expand=6052) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vunpckhps))] +pub unsafe fn _mm_mask_unpackhi_ps(src: __m128, k: __mmask8, a: __m128, b: __m128) -> __m128 { + let unpackhi = _mm_unpackhi_ps(a, b).as_f32x4(); + transmute(simd_select_bitmask(k, unpackhi, src.as_f32x4())) +} + +/// Unpack and interleave single-precision (32-bit) floating-point elements from the high half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_unpackhi_ps&expand=6053) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vunpckhps))] +pub unsafe fn _mm_maskz_unpackhi_ps(k: __mmask8, a: __m128, b: __m128) -> __m128 { + let unpackhi = _mm_unpackhi_ps(a, b).as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, unpackhi, zero)) +} + +/// Unpack and interleave double-precision (64-bit) floating-point elements from the high half of each 128-bit lane in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_unpackhi_pd&expand=6048) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vunpckhpd))] +pub unsafe fn _mm512_unpackhi_pd(a: __m512d, b: __m512d) -> __m512d { + simd_shuffle8!(a, b, [1, 9, 1 + 2, 9 + 2, 1 + 4, 9 + 4, 1 + 6, 9 + 6]) +} + +/// Unpack and interleave double-precision (64-bit) floating-point elements from the high half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_unpackhi_pd&expand=6046) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vunpckhpd))] +pub unsafe fn _mm512_mask_unpackhi_pd( + src: __m512d, + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + let unpackhi = _mm512_unpackhi_pd(a, b).as_f64x8(); + transmute(simd_select_bitmask(k, unpackhi, src.as_f64x8())) +} + +/// Unpack and interleave double-precision (64-bit) floating-point elements from the high half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_unpackhi_pd&expand=6047) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vunpckhpd))] +pub unsafe fn _mm512_maskz_unpackhi_pd(k: __mmask8, a: __m512d, b: __m512d) -> __m512d { + let unpackhi = _mm512_unpackhi_pd(a, b).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, unpackhi, zero)) +} + +/// Unpack and interleave double-precision (64-bit) floating-point elements from the high half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_unpackhi_pd&expand=6043) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vunpckhpd))] +pub unsafe fn _mm256_mask_unpackhi_pd( + src: __m256d, + k: __mmask8, + a: __m256d, + b: __m256d, +) -> __m256d { + let unpackhi = _mm256_unpackhi_pd(a, b).as_f64x4(); + transmute(simd_select_bitmask(k, unpackhi, src.as_f64x4())) +} + +/// Unpack and interleave double-precision (64-bit) floating-point elements from the high half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_unpackhi_pd&expand=6044) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vunpckhpd))] +pub unsafe fn _mm256_maskz_unpackhi_pd(k: __mmask8, a: __m256d, b: __m256d) -> __m256d { + let unpackhi = _mm256_unpackhi_pd(a, b).as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, unpackhi, zero)) +} + +/// Unpack and interleave double-precision (64-bit) floating-point elements from the high half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_unpackhi_pd&expand=6040) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vunpckhpd))] +pub unsafe fn _mm_mask_unpackhi_pd(src: __m128d, k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let unpackhi = _mm_unpackhi_pd(a, b).as_f64x2(); + transmute(simd_select_bitmask(k, unpackhi, src.as_f64x2())) +} + +/// Unpack and interleave double-precision (64-bit) floating-point elements from the high half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_unpackhi_pd&expand=6041) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vunpckhpd))] +pub unsafe fn _mm_maskz_unpackhi_pd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let unpackhi = _mm_unpackhi_pd(a, b).as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, unpackhi, zero)) +} + +/// Unpack and interleave 32-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_unpacklo_epi32&expand=6078) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vunpcklps))] //should be vpunpckldq +pub unsafe fn _mm512_unpacklo_epi32(a: __m512i, b: __m512i) -> __m512i { + let a = a.as_i32x16(); + let b = b.as_i32x16(); + #[rustfmt::skip] + let r: i32x16 = simd_shuffle16!( + a, b, + [ 0, 16, 1, 17, + 0 + 4, 16 + 4, 1 + 4, 17 + 4, + 0 + 8, 16 + 8, 1 + 8, 17 + 8, + 0 + 12, 16 + 12, 1 + 12, 17 + 12], + ); + transmute(r) +} + +/// Unpack and interleave 32-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_unpacklo_epi32&expand=6076) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpunpckldq))] +pub unsafe fn _mm512_mask_unpacklo_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + let unpacklo = _mm512_unpacklo_epi32(a, b).as_i32x16(); + transmute(simd_select_bitmask(k, unpacklo, src.as_i32x16())) +} + +/// Unpack and interleave 32-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_unpacklo_epi32&expand=6077) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpunpckldq))] +pub unsafe fn _mm512_maskz_unpacklo_epi32(k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let unpacklo = _mm512_unpacklo_epi32(a, b).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, unpacklo, zero)) +} + +/// Unpack and interleave 32-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_unpacklo_epi32&expand=6073) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpckldq))] +pub unsafe fn _mm256_mask_unpacklo_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + let unpacklo = _mm256_unpacklo_epi32(a, b).as_i32x8(); + transmute(simd_select_bitmask(k, unpacklo, src.as_i32x8())) +} + +/// Unpack and interleave 32-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_unpacklo_epi32&expand=6074) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpckldq))] +pub unsafe fn _mm256_maskz_unpacklo_epi32(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let unpacklo = _mm256_unpacklo_epi32(a, b).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, unpacklo, zero)) +} + +/// Unpack and interleave 32-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_unpacklo_epi32&expand=6070) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpckldq))] +pub unsafe fn _mm_mask_unpacklo_epi32( + src: __m128i, + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + let unpacklo = _mm_unpacklo_epi32(a, b).as_i32x4(); + transmute(simd_select_bitmask(k, unpacklo, src.as_i32x4())) +} + +/// Unpack and interleave 32-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_unpacklo_epi32&expand=6071) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpckldq))] +pub unsafe fn _mm_maskz_unpacklo_epi32(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let unpacklo = _mm_unpacklo_epi32(a, b).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, unpacklo, zero)) +} + +/// Unpack and interleave 64-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_unpacklo_epi64&expand=6087) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vunpcklpd))] //should be vpunpcklqdq +pub unsafe fn _mm512_unpacklo_epi64(a: __m512i, b: __m512i) -> __m512i { + simd_shuffle8!(a, b, [0, 8, 0 + 2, 8 + 2, 0 + 4, 8 + 4, 0 + 6, 8 + 6]) +} + +/// Unpack and interleave 64-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_unpacklo_epi64&expand=6085) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpunpcklqdq))] +pub unsafe fn _mm512_mask_unpacklo_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, + b: __m512i, +) -> __m512i { + let unpacklo = _mm512_unpacklo_epi64(a, b).as_i64x8(); + transmute(simd_select_bitmask(k, unpacklo, src.as_i64x8())) +} + +/// Unpack and interleave 64-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_unpacklo_epi64&expand=6086) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpunpcklqdq))] +pub unsafe fn _mm512_maskz_unpacklo_epi64(k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let unpacklo = _mm512_unpacklo_epi64(a, b).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, unpacklo, zero)) +} + +/// Unpack and interleave 64-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_unpacklo_epi64&expand=6082) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpcklqdq))] +pub unsafe fn _mm256_mask_unpacklo_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + let unpacklo = _mm256_unpacklo_epi64(a, b).as_i64x4(); + transmute(simd_select_bitmask(k, unpacklo, src.as_i64x4())) +} + +/// Unpack and interleave 64-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_unpacklo_epi64&expand=6083) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpcklqdq))] +pub unsafe fn _mm256_maskz_unpacklo_epi64(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let unpacklo = _mm256_unpacklo_epi64(a, b).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, unpacklo, zero)) +} + +/// Unpack and interleave 64-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_unpacklo_epi64&expand=6079) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpcklqdq))] +pub unsafe fn _mm_mask_unpacklo_epi64( + src: __m128i, + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + let unpacklo = _mm_unpacklo_epi64(a, b).as_i64x2(); + transmute(simd_select_bitmask(k, unpacklo, src.as_i64x2())) +} + +/// Unpack and interleave 64-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_unpacklo_epi64&expand=6080) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpunpcklqdq))] +pub unsafe fn _mm_maskz_unpacklo_epi64(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let unpacklo = _mm_unpacklo_epi64(a, b).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, unpacklo, zero)) +} + +/// Unpack and interleave single-precision (32-bit) floating-point elements from the low half of each 128-bit lane in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_unpacklo_ps&expand=6117) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vunpcklps))] +pub unsafe fn _mm512_unpacklo_ps(a: __m512, b: __m512) -> __m512 { + #[rustfmt::skip] + simd_shuffle16!(a, b, + [ 0, 16, 1, 17, + 0 + 4, 16 + 4, 1 + 4, 17 + 4, + 0 + 8, 16 + 8, 1 + 8, 17 + 8, + 0 + 12, 16 + 12, 1 + 12, 17 + 12], + ) +} + +/// Unpack and interleave single-precision (32-bit) floating-point elements from the low half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_unpacklo_ps&expand=6115) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vunpcklps))] +pub unsafe fn _mm512_mask_unpacklo_ps(src: __m512, k: __mmask16, a: __m512, b: __m512) -> __m512 { + let unpacklo = _mm512_unpacklo_ps(a, b).as_f32x16(); + transmute(simd_select_bitmask(k, unpacklo, src.as_f32x16())) +} + +/// Unpack and interleave single-precision (32-bit) floating-point elements from the low half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_unpacklo_ps&expand=6116) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vunpcklps))] +pub unsafe fn _mm512_maskz_unpacklo_ps(k: __mmask16, a: __m512, b: __m512) -> __m512 { + let unpacklo = _mm512_unpacklo_ps(a, b).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, unpacklo, zero)) +} + +/// Unpack and interleave single-precision (32-bit) floating-point elements from the low half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_unpacklo_ps&expand=6112) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vunpcklps))] +pub unsafe fn _mm256_mask_unpacklo_ps(src: __m256, k: __mmask8, a: __m256, b: __m256) -> __m256 { + let unpacklo = _mm256_unpacklo_ps(a, b).as_f32x8(); + transmute(simd_select_bitmask(k, unpacklo, src.as_f32x8())) +} + +/// Unpack and interleave single-precision (32-bit) floating-point elements from the low half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_unpacklo_ps&expand=6113) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vunpcklps))] +pub unsafe fn _mm256_maskz_unpacklo_ps(k: __mmask8, a: __m256, b: __m256) -> __m256 { + let unpacklo = _mm256_unpacklo_ps(a, b).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, unpacklo, zero)) +} + +/// Unpack and interleave single-precision (32-bit) floating-point elements from the low half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_unpacklo_ps&expand=6109) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vunpcklps))] +pub unsafe fn _mm_mask_unpacklo_ps(src: __m128, k: __mmask8, a: __m128, b: __m128) -> __m128 { + let unpacklo = _mm_unpacklo_ps(a, b).as_f32x4(); + transmute(simd_select_bitmask(k, unpacklo, src.as_f32x4())) +} + +/// Unpack and interleave single-precision (32-bit) floating-point elements from the low half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_unpacklo_ps&expand=6110) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vunpcklps))] +pub unsafe fn _mm_maskz_unpacklo_ps(k: __mmask8, a: __m128, b: __m128) -> __m128 { + let unpacklo = _mm_unpacklo_ps(a, b).as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, unpacklo, zero)) +} + +/// Unpack and interleave double-precision (64-bit) floating-point elements from the low half of each 128-bit lane in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_unpacklo_pd&expand=6105) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vunpcklpd))] +pub unsafe fn _mm512_unpacklo_pd(a: __m512d, b: __m512d) -> __m512d { + simd_shuffle8!(a, b, [0, 8, 0 + 2, 8 + 2, 0 + 4, 8 + 4, 0 + 6, 8 + 6]) +} + +/// Unpack and interleave double-precision (64-bit) floating-point elements from the low half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_unpacklo_pd&expand=6103) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vunpcklpd))] +pub unsafe fn _mm512_mask_unpacklo_pd( + src: __m512d, + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + let unpacklo = _mm512_unpacklo_pd(a, b).as_f64x8(); + transmute(simd_select_bitmask(k, unpacklo, src.as_f64x8())) +} + +/// Unpack and interleave double-precision (64-bit) floating-point elements from the low half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_unpacklo_pd&expand=6104) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vunpcklpd))] +pub unsafe fn _mm512_maskz_unpacklo_pd(k: __mmask8, a: __m512d, b: __m512d) -> __m512d { + let unpacklo = _mm512_unpacklo_pd(a, b).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, unpacklo, zero)) +} + +/// Unpack and interleave double-precision (64-bit) floating-point elements from the low half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_unpacklo_pd&expand=6100) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vunpcklpd))] +pub unsafe fn _mm256_mask_unpacklo_pd( + src: __m256d, + k: __mmask8, + a: __m256d, + b: __m256d, +) -> __m256d { + let unpacklo = _mm256_unpacklo_pd(a, b).as_f64x4(); + transmute(simd_select_bitmask(k, unpacklo, src.as_f64x4())) +} + +/// Unpack and interleave double-precision (64-bit) floating-point elements from the low half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_unpacklo_pd&expand=6101) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vunpcklpd))] +pub unsafe fn _mm256_maskz_unpacklo_pd(k: __mmask8, a: __m256d, b: __m256d) -> __m256d { + let unpacklo = _mm256_unpacklo_pd(a, b).as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, unpacklo, zero)) +} + +/// Unpack and interleave double-precision (64-bit) floating-point elements from the low half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_unpacklo_pd&expand=6097) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vunpcklpd))] +pub unsafe fn _mm_mask_unpacklo_pd(src: __m128d, k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let unpacklo = _mm_unpacklo_pd(a, b).as_f64x2(); + transmute(simd_select_bitmask(k, unpacklo, src.as_f64x2())) +} + +/// Unpack and interleave double-precision (64-bit) floating-point elements from the low half of each 128-bit lane in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_unpacklo_pd&expand=6098) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vunpcklpd))] +pub unsafe fn _mm_maskz_unpacklo_pd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let unpacklo = _mm_unpacklo_pd(a, b).as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + transmute(simd_select_bitmask(k, unpacklo, zero)) +} + +/// Cast vector of type __m128 to type __m512; the upper 384 bits of the result are undefined. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_castps128_ps512&expand=621) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_castps128_ps512(a: __m128) -> __m512 { + simd_shuffle16!( + a, + _mm_set1_ps(-1.), + [0, 1, 2, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4], + ) +} + +/// Cast vector of type __m256 to type __m512; the upper 256 bits of the result are undefined. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_castps256_ps512&expand=623) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_castps256_ps512(a: __m256) -> __m512 { + simd_shuffle16!( + a, + _mm256_set1_ps(-1.), + [0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 8, 8, 8, 8, 8, 8], + ) +} + +/// Cast vector of type __m128 to type __m512; the upper 384 bits of the result are zeroed. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_zextps128_ps512&expand=6196) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_zextps128_ps512(a: __m128) -> __m512 { + simd_shuffle16!( + a, + _mm_set1_ps(0.), + [0, 1, 2, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4], + ) +} + +/// Cast vector of type __m256 to type __m512; the upper 256 bits of the result are zeroed. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_zextps256_ps512&expand=6197) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_zextps256_ps512(a: __m256) -> __m512 { + simd_shuffle16!( + a, + _mm256_set1_ps(0.), + [0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 8, 8, 8, 8, 8, 8], + ) +} + +/// Cast vector of type __m512 to type __m128. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_castps512_ps128&expand=624) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_castps512_ps128(a: __m512) -> __m128 { + simd_shuffle4!(a, a, [0, 1, 2, 3]) +} + +/// Cast vector of type __m512 to type __m256. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_castps512_ps256&expand=625) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_castps512_ps256(a: __m512) -> __m256 { + simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Cast vector of type __m512 to type __m512d. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_castps_pd&expand=616) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_castps_pd(a: __m512) -> __m512d { + transmute(a.as_m512()) +} + +/// Cast vector of type __m512 to type __m512i. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_castps_si512&expand=619) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_castps_si512(a: __m512) -> __m512i { + transmute(a.as_m512()) +} + +/// Cast vector of type __m128d to type __m512d; the upper 384 bits of the result are undefined. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_castpd128_pd512&expand=609) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_castpd128_pd512(a: __m128d) -> __m512d { + simd_shuffle8!(a, _mm_set1_pd(-1.), [0, 1, 2, 2, 2, 2, 2, 2]) +} + +/// Cast vector of type __m256d to type __m512d; the upper 256 bits of the result are undefined. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_castpd256_pd512&expand=611) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_castpd256_pd512(a: __m256d) -> __m512d { + simd_shuffle8!(a, _mm256_set1_pd(-1.), [0, 1, 2, 3, 4, 4, 4, 4]) +} + +/// Cast vector of type __m128d to type __m512d; the upper 384 bits of the result are zeroed. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_zextpd128_pd512&expand=6193) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_zextpd128_pd512(a: __m128d) -> __m512d { + simd_shuffle8!(a, _mm_set1_pd(0.), [0, 1, 2, 2, 2, 2, 2, 2]) +} + +/// Cast vector of type __m256d to type __m512d; the upper 256 bits of the result are zeroed. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_zextpd256_pd512&expand=6194) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_zextpd256_pd512(a: __m256d) -> __m512d { + simd_shuffle8!(a, _mm256_set1_pd(0.), [0, 1, 2, 3, 4, 4, 4, 4]) +} + +/// Cast vector of type __m512d to type __m128d. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_castpd512_pd128&expand=612) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_castpd512_pd128(a: __m512d) -> __m128d { + simd_shuffle2!(a, a, [0, 1]) +} + +/// Cast vector of type __m512d to type __m256d. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_castpd512_pd256&expand=613) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_castpd512_pd256(a: __m512d) -> __m256d { + simd_shuffle4!(a, a, [0, 1, 2, 3]) +} + +/// Cast vector of type __m512d to type __m512. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_castpd_ps&expand=604) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_castpd_ps(a: __m512d) -> __m512 { + transmute(a.as_m512d()) +} + +/// Cast vector of type __m512d to type __m512i. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_castpd_si512&expand=607) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_castpd_si512(a: __m512d) -> __m512i { + transmute(a.as_m512d()) +} + +/// Cast vector of type __m128i to type __m512i; the upper 384 bits of the result are undefined. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_castsi128_si512&expand=629) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_castsi128_si512(a: __m128i) -> __m512i { + simd_shuffle8!(a, _mm_set1_epi64x(-1), [0, 1, 2, 2, 2, 2, 2, 2]) +} + +/// Cast vector of type __m256i to type __m512i; the upper 256 bits of the result are undefined. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_castsi256_si512&expand=633) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_castsi256_si512(a: __m256i) -> __m512i { + simd_shuffle8!(a, _mm256_set1_epi64x(-1), [0, 1, 2, 3, 4, 4, 4, 4]) +} + +/// Cast vector of type __m128i to type __m512i; the upper 384 bits of the result are zeroed. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_zextsi128_si512&expand=6199) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_zextsi128_si512(a: __m128i) -> __m512i { + simd_shuffle8!(a, _mm_set1_epi64x(0), [0, 1, 2, 2, 2, 2, 2, 2]) +} + +/// Cast vector of type __m256i to type __m512i; the upper 256 bits of the result are zeroed. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_zextsi256_si512&expand=6200) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_zextsi256_si512(a: __m256i) -> __m512i { + simd_shuffle8!(a, _mm256_set1_epi64x(0), [0, 1, 2, 3, 4, 4, 4, 4]) +} + +/// Cast vector of type __m512i to type __m128i. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_castsi512_si128&expand=636) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_castsi512_si128(a: __m512i) -> __m128i { + simd_shuffle2!(a, a, [0, 1]) +} + +/// Cast vector of type __m512i to type __m256i. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_castsi512_si256&expand=637) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_castsi512_si256(a: __m512i) -> __m256i { + simd_shuffle4!(a, a, [0, 1, 2, 3]) +} + +/// Cast vector of type __m512i to type __m512. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_castsi512_ps&expand=635) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_castsi512_ps(a: __m512i) -> __m512 { + transmute(a) +} + +/// Cast vector of type __m512i to type __m512d. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_castsi512_pd&expand=634) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_castsi512_pd(a: __m512i) -> __m512d { + transmute(a) +} + +/// Copy the lower 32-bit integer in a to dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtsi512_si32&expand=1882) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(all(test, not(target_os = "windows")), assert_instr(vmovd))] +pub unsafe fn _mm512_cvtsi512_si32(a: __m512i) -> i32 { + let extract: i32 = simd_extract(a.as_i32x16(), 0); + transmute(extract) +} + +/// Broadcast the low packed 32-bit integer from a to all elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_broadcastd_epi32&expand=545) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vbroadcast))] //should be vpbroadcastd +pub unsafe fn _mm512_broadcastd_epi32(a: __m128i) -> __m512i { + let a = _mm512_castsi128_si512(a).as_i32x16(); + let ret: i32x16 = simd_shuffle16!(a, a, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + transmute(ret) +} + +/// Broadcast the low packed 32-bit integer from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_broadcastd_epi32&expand=546) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpbroadcast))] //should be vpbroadcastd +pub unsafe fn _mm512_mask_broadcastd_epi32(src: __m512i, k: __mmask16, a: __m128i) -> __m512i { + let broadcast = _mm512_broadcastd_epi32(a).as_i32x16(); + transmute(simd_select_bitmask(k, broadcast, src.as_i32x16())) +} + +/// Broadcast the low packed 32-bit integer from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_broadcastd_epi32&expand=547) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpbroadcast))] //should be vpbroadcastd +pub unsafe fn _mm512_maskz_broadcastd_epi32(k: __mmask16, a: __m128i) -> __m512i { + let broadcast = _mm512_broadcastd_epi32(a).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Broadcast the low packed 32-bit integer from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_broadcastd_epi32&expand=543) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcast))] //should be vpbroadcastd +pub unsafe fn _mm256_mask_broadcastd_epi32(src: __m256i, k: __mmask8, a: __m128i) -> __m256i { + let broadcast = _mm256_broadcastd_epi32(a).as_i32x8(); + transmute(simd_select_bitmask(k, broadcast, src.as_i32x8())) +} + +/// Broadcast the low packed 32-bit integer from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_broadcastd_epi32&expand=544) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcast))] //should be vpbroadcastd +pub unsafe fn _mm256_maskz_broadcastd_epi32(k: __mmask8, a: __m128i) -> __m256i { + let broadcast = _mm256_broadcastd_epi32(a).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Broadcast the low packed 32-bit integer from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_broadcastd_epi32&expand=540) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcast))] //should be vpbroadcastd +pub unsafe fn _mm_mask_broadcastd_epi32(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let broadcast = _mm_broadcastd_epi32(a).as_i32x4(); + transmute(simd_select_bitmask(k, broadcast, src.as_i32x4())) +} + +/// Broadcast the low packed 32-bit integer from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_broadcastd_epi32&expand=541) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcast))] //should be vpbroadcastd +pub unsafe fn _mm_maskz_broadcastd_epi32(k: __mmask8, a: __m128i) -> __m128i { + let broadcast = _mm_broadcastd_epi32(a).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Broadcast the low packed 64-bit integer from a to all elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_broadcastq_epi64&expand=560) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vbroadcas))] //should be vpbroadcastq +pub unsafe fn _mm512_broadcastq_epi64(a: __m128i) -> __m512i { + simd_shuffle8!(a, a, [0, 0, 0, 0, 0, 0, 0, 0]) +} + +/// Broadcast the low packed 64-bit integer from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_broadcastq_epi64&expand=561) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpbroadcast))] //should be vpbroadcastq +pub unsafe fn _mm512_mask_broadcastq_epi64(src: __m512i, k: __mmask8, a: __m128i) -> __m512i { + let broadcast = _mm512_broadcastq_epi64(a).as_i64x8(); + transmute(simd_select_bitmask(k, broadcast, src.as_i64x8())) +} + +/// Broadcast the low packed 64-bit integer from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_broadcastq_epi64&expand=562) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpbroadcast))] //should be vpbroadcastq +pub unsafe fn _mm512_maskz_broadcastq_epi64(k: __mmask8, a: __m128i) -> __m512i { + let broadcast = _mm512_broadcastq_epi64(a).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Broadcast the low packed 64-bit integer from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_broadcastq_epi64&expand=558) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcast))] //should be vpbroadcastq +pub unsafe fn _mm256_mask_broadcastq_epi64(src: __m256i, k: __mmask8, a: __m128i) -> __m256i { + let broadcast = _mm256_broadcastq_epi64(a).as_i64x4(); + transmute(simd_select_bitmask(k, broadcast, src.as_i64x4())) +} + +/// Broadcast the low packed 64-bit integer from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_broadcastq_epi64&expand=559) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcast))] //should be vpbroadcastq +pub unsafe fn _mm256_maskz_broadcastq_epi64(k: __mmask8, a: __m128i) -> __m256i { + let broadcast = _mm256_broadcastq_epi64(a).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Broadcast the low packed 64-bit integer from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_broadcastq_epi64&expand=555) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcast))] //should be vpbroadcastq +pub unsafe fn _mm_mask_broadcastq_epi64(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + let broadcast = _mm_broadcastq_epi64(a).as_i64x2(); + transmute(simd_select_bitmask(k, broadcast, src.as_i64x2())) +} + +/// Broadcast the low packed 64-bit integer from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_broadcastq_epi64&expand=556) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcast))] //should be vpbroadcastq +pub unsafe fn _mm_maskz_broadcastq_epi64(k: __mmask8, a: __m128i) -> __m128i { + let broadcast = _mm_broadcastq_epi64(a).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Broadcast the low single-precision (32-bit) floating-point element from a to all elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_broadcastss_ps&expand=578) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vbroadcastss))] +pub unsafe fn _mm512_broadcastss_ps(a: __m128) -> __m512 { + simd_shuffle16!(a, a, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) +} + +/// Broadcast the low single-precision (32-bit) floating-point element from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_broadcastss_ps&expand=579) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vbroadcastss))] +pub unsafe fn _mm512_mask_broadcastss_ps(src: __m512, k: __mmask16, a: __m128) -> __m512 { + let broadcast = _mm512_broadcastss_ps(a).as_f32x16(); + transmute(simd_select_bitmask(k, broadcast, src.as_f32x16())) +} + +/// Broadcast the low single-precision (32-bit) floating-point element from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_broadcastss_ps&expand=580) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vbroadcastss))] +pub unsafe fn _mm512_maskz_broadcastss_ps(k: __mmask16, a: __m128) -> __m512 { + let broadcast = _mm512_broadcastss_ps(a).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Broadcast the low single-precision (32-bit) floating-point element from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_broadcastss_ps&expand=576) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vbroadcastss))] +pub unsafe fn _mm256_mask_broadcastss_ps(src: __m256, k: __mmask8, a: __m128) -> __m256 { + let broadcast = _mm256_broadcastss_ps(a).as_f32x8(); + transmute(simd_select_bitmask(k, broadcast, src.as_f32x8())) +} + +/// Broadcast the low single-precision (32-bit) floating-point element from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_broadcastss_ps&expand=577) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vbroadcastss))] +pub unsafe fn _mm256_maskz_broadcastss_ps(k: __mmask8, a: __m128) -> __m256 { + let broadcast = _mm256_broadcastss_ps(a).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Broadcast the low single-precision (32-bit) floating-point element from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_broadcastss_ps&expand=573) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vbroadcastss))] +pub unsafe fn _mm_mask_broadcastss_ps(src: __m128, k: __mmask8, a: __m128) -> __m128 { + let broadcast = _mm_broadcastss_ps(a).as_f32x4(); + transmute(simd_select_bitmask(k, broadcast, src.as_f32x4())) +} + +/// Broadcast the low single-precision (32-bit) floating-point element from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_broadcastss_ps&expand=574) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vbroadcastss))] +pub unsafe fn _mm_maskz_broadcastss_ps(k: __mmask8, a: __m128) -> __m128 { + let broadcast = _mm_broadcastss_ps(a).as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Broadcast the low double-precision (64-bit) floating-point element from a to all elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_broadcastsd_pd&expand=567) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vbroadcastsd))] +pub unsafe fn _mm512_broadcastsd_pd(a: __m128d) -> __m512d { + simd_shuffle8!(a, a, [0, 0, 0, 0, 0, 0, 0, 0]) +} + +/// Broadcast the low double-precision (64-bit) floating-point element from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_broadcastsd_pd&expand=568) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vbroadcastsd))] +pub unsafe fn _mm512_mask_broadcastsd_pd(src: __m512d, k: __mmask8, a: __m128d) -> __m512d { + let broadcast = _mm512_broadcastsd_pd(a).as_f64x8(); + transmute(simd_select_bitmask(k, broadcast, src.as_f64x8())) +} + +/// Broadcast the low double-precision (64-bit) floating-point element from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_broadcastsd_pd&expand=569) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vbroadcastsd))] +pub unsafe fn _mm512_maskz_broadcastsd_pd(k: __mmask8, a: __m128d) -> __m512d { + let broadcast = _mm512_broadcastsd_pd(a).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Broadcast the low double-precision (64-bit) floating-point element from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_broadcastsd_pd&expand=565) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vbroadcastsd))] +pub unsafe fn _mm256_mask_broadcastsd_pd(src: __m256d, k: __mmask8, a: __m128d) -> __m256d { + let broadcast = _mm256_broadcastsd_pd(a).as_f64x4(); + transmute(simd_select_bitmask(k, broadcast, src.as_f64x4())) +} + +/// Broadcast the low double-precision (64-bit) floating-point element from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_broadcastsd_pd&expand=566) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vbroadcastsd))] +pub unsafe fn _mm256_maskz_broadcastsd_pd(k: __mmask8, a: __m128d) -> __m256d { + let broadcast = _mm256_broadcastsd_pd(a).as_f64x4(); + let zero = _mm256_setzero_pd().as_f64x4(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Broadcast the 4 packed 32-bit integers from a to all elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_broadcast_i32x4&expand=510) +#[inline] +#[target_feature(enable = "avx512f")] //msvc: vbroadcasti32x4, linux: vshuf +pub unsafe fn _mm512_broadcast_i32x4(a: __m128i) -> __m512i { + let a = a.as_i32x4(); + let ret: i32x16 = simd_shuffle16!(a, a, [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3]); + transmute(ret) +} + +/// Broadcast the 4 packed 32-bit integers from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_broadcast_i32x4&expand=511) +#[inline] +#[target_feature(enable = "avx512f")] //msvc: vbroadcasti32x4, linux: vshuf +pub unsafe fn _mm512_mask_broadcast_i32x4(src: __m512i, k: __mmask16, a: __m128i) -> __m512i { + let broadcast = _mm512_broadcast_i32x4(a).as_i32x16(); + transmute(simd_select_bitmask(k, broadcast, src.as_i32x16())) +} + +/// Broadcast the 4 packed 32-bit integers from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_broadcast_i32x4&expand=512) +#[inline] +#[target_feature(enable = "avx512f")] //msvc: vbroadcasti32x4, linux: vshuf +pub unsafe fn _mm512_maskz_broadcast_i32x4(k: __mmask16, a: __m128i) -> __m512i { + let broadcast = _mm512_broadcast_i32x4(a).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Broadcast the 4 packed 32-bit integers from a to all elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_broadcast_i32x4&expand=507) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] //msvc: vbroadcasti32x4, linux: vshuf +pub unsafe fn _mm256_broadcast_i32x4(a: __m128i) -> __m256i { + let a = a.as_i32x4(); + let ret: i32x8 = simd_shuffle8!(a, a, [0, 1, 2, 3, 0, 1, 2, 3]); + transmute(ret) +} + +/// Broadcast the 4 packed 32-bit integers from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_broadcast_i32x4&expand=508) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] //msvc: vbroadcasti32x4, linux: vshuf +pub unsafe fn _mm256_mask_broadcast_i32x4(src: __m256i, k: __mmask8, a: __m128i) -> __m256i { + let broadcast = _mm256_broadcast_i32x4(a).as_i32x8(); + transmute(simd_select_bitmask(k, broadcast, src.as_i32x8())) +} + +/// Broadcast the 4 packed 32-bit integers from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_broadcast_i32x4&expand=509) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] //msvc: vbroadcasti32x4, linux: vshuf +pub unsafe fn _mm256_maskz_broadcast_i32x4(k: __mmask8, a: __m128i) -> __m256i { + let broadcast = _mm256_broadcast_i32x4(a).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Broadcast the 4 packed 64-bit integers from a to all elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_broadcast_i64x4&expand=522) +#[inline] +#[target_feature(enable = "avx512f")] //msvc: vbroadcasti64x4, linux: vperm +pub unsafe fn _mm512_broadcast_i64x4(a: __m256i) -> __m512i { + simd_shuffle8!(a, a, [0, 1, 2, 3, 0, 1, 2, 3]) +} + +/// Broadcast the 4 packed 64-bit integers from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_mask_broadcast_i64x4&expand=523) +#[inline] +#[target_feature(enable = "avx512f")] //msvc: vbroadcasti64x4, linux: vperm +pub unsafe fn _mm512_mask_broadcast_i64x4(src: __m512i, k: __mmask8, a: __m256i) -> __m512i { + let broadcast = _mm512_broadcast_i64x4(a).as_i64x8(); + transmute(simd_select_bitmask(k, broadcast, src.as_i64x8())) +} + +/// Broadcast the 4 packed 64-bit integers from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_maskz_broadcast_i64x4&expand=524) +#[inline] +#[target_feature(enable = "avx512f")] //msvc: vbroadcasti64x4, linux: vperm +pub unsafe fn _mm512_maskz_broadcast_i64x4(k: __mmask8, a: __m256i) -> __m512i { + let broadcast = _mm512_broadcast_i64x4(a).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Broadcast the 4 packed single-precision (32-bit) floating-point elements from a to all elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_broadcast_f32x4&expand=483) +#[inline] +#[target_feature(enable = "avx512f")] //msvc: vbroadcastf32x4, linux: vshuf +pub unsafe fn _mm512_broadcast_f32x4(a: __m128) -> __m512 { + simd_shuffle16!(a, a, [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3]) +} + +/// Broadcast the 4 packed single-precision (32-bit) floating-point elements from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_broadcast_f32x4&expand=484) +#[inline] +#[target_feature(enable = "avx512f")] //msvc: vbroadcastf32x4, linux: vshu +pub unsafe fn _mm512_mask_broadcast_f32x4(src: __m512, k: __mmask16, a: __m128) -> __m512 { + let broadcast = _mm512_broadcast_f32x4(a).as_f32x16(); + transmute(simd_select_bitmask(k, broadcast, src.as_f32x16())) +} + +/// Broadcast the 4 packed single-precision (32-bit) floating-point elements from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_broadcast_f32x4&expand=485) +#[inline] +#[target_feature(enable = "avx512f")] //msvc: vbroadcastf32x4, linux: vshu +pub unsafe fn _mm512_maskz_broadcast_f32x4(k: __mmask16, a: __m128) -> __m512 { + let broadcast = _mm512_broadcast_f32x4(a).as_f32x16(); + let zero = _mm512_setzero_ps().as_f32x16(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Broadcast the 4 packed single-precision (32-bit) floating-point elements from a to all elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_broadcast_f32x4&expand=480) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] //msvc: vbroadcastf32x4, linux: vshuf +pub unsafe fn _mm256_broadcast_f32x4(a: __m128) -> __m256 { + simd_shuffle8!(a, a, [0, 1, 2, 3, 0, 1, 2, 3]) +} + +/// Broadcast the 4 packed single-precision (32-bit) floating-point elements from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_broadcast_f32x4&expand=481) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] //msvc: vbroadcastf32x4, linux: vshu +pub unsafe fn _mm256_mask_broadcast_f32x4(src: __m256, k: __mmask8, a: __m128) -> __m256 { + let broadcast = _mm256_broadcast_f32x4(a).as_f32x8(); + transmute(simd_select_bitmask(k, broadcast, src.as_f32x8())) +} + +/// Broadcast the 4 packed single-precision (32-bit) floating-point elements from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_broadcast_f32x4&expand=482) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] //msvc: vbroadcastf32x4, linux: vshu +pub unsafe fn _mm256_maskz_broadcast_f32x4(k: __mmask8, a: __m128) -> __m256 { + let broadcast = _mm256_broadcast_f32x4(a).as_f32x8(); + let zero = _mm256_setzero_ps().as_f32x8(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Broadcast the 4 packed double-precision (64-bit) floating-point elements from a to all elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_broadcast_f64x4&expand=495) +#[inline] +#[target_feature(enable = "avx512f")] //msvc: vbroadcastf64x4, linux: vperm +pub unsafe fn _mm512_broadcast_f64x4(a: __m256d) -> __m512d { + simd_shuffle8!(a, a, [0, 1, 2, 3, 0, 1, 2, 3]) +} + +/// Broadcast the 4 packed double-precision (64-bit) floating-point elements from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_mask_broadcast_f64x4&expand=496) +#[inline] +#[target_feature(enable = "avx512f")] //msvc: vbroadcastf64x4, linux: vper +pub unsafe fn _mm512_mask_broadcast_f64x4(src: __m512d, k: __mmask8, a: __m256d) -> __m512d { + let broadcast = _mm512_broadcast_f64x4(a).as_f64x8(); + transmute(simd_select_bitmask(k, broadcast, src.as_f64x8())) +} + +/// Broadcast the 4 packed double-precision (64-bit) floating-point elements from a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_maskz_broadcast_f64x4&expand=497) +#[inline] +#[target_feature(enable = "avx512f")] //msvc: vbroadcastf64x4, linux: vper +pub unsafe fn _mm512_maskz_broadcast_f64x4(k: __mmask8, a: __m256d) -> __m512d { + let broadcast = _mm512_broadcast_f64x4(a).as_f64x8(); + let zero = _mm512_setzero_pd().as_f64x8(); + transmute(simd_select_bitmask(k, broadcast, zero)) +} + +/// Blend packed 32-bit integers from a and b using control mask k, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_blend_epi32&expand=435) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovdqa32))] //should be vpblendmd +pub unsafe fn _mm512_mask_blend_epi32(k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + transmute(simd_select_bitmask(k, b.as_i32x16(), a.as_i32x16())) +} + +/// Blend packed 32-bit integers from a and b using control mask k, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_blend_epi32&expand=434) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqa32))] //should be vpblendmd +pub unsafe fn _mm256_mask_blend_epi32(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + transmute(simd_select_bitmask(k, b.as_i32x8(), a.as_i32x8())) +} + +/// Blend packed 32-bit integers from a and b using control mask k, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_blend_epi32&expand=432) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqa32))] //should be vpblendmd +pub unsafe fn _mm_mask_blend_epi32(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + transmute(simd_select_bitmask(k, b.as_i32x4(), a.as_i32x4())) +} + +/// Blend packed 64-bit integers from a and b using control mask k, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_blend_epi64&expand=438) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovdqa64))] //should be vpblendmq +pub unsafe fn _mm512_mask_blend_epi64(k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + transmute(simd_select_bitmask(k, b.as_i64x8(), a.as_i64x8())) +} + +/// Blend packed 64-bit integers from a and b using control mask k, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_blend_epi64&expand=437) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqa64))] //should be vpblendmq +pub unsafe fn _mm256_mask_blend_epi64(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + transmute(simd_select_bitmask(k, b.as_i64x4(), a.as_i64x4())) +} + +/// Blend packed 64-bit integers from a and b using control mask k, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_blend_epi64&expand=436) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovdqa64))] //should be vpblendmq +pub unsafe fn _mm_mask_blend_epi64(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + transmute(simd_select_bitmask(k, b.as_i64x2(), a.as_i64x2())) +} + +/// Blend packed single-precision (32-bit) floating-point elements from a and b using control mask k, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_blend_ps&expand=451) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovaps))] //should be vpblendmps +pub unsafe fn _mm512_mask_blend_ps(k: __mmask16, a: __m512, b: __m512) -> __m512 { + transmute(simd_select_bitmask(k, b.as_f32x16(), a.as_f32x16())) +} + +/// Blend packed single-precision (32-bit) floating-point elements from a and b using control mask k, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_blend_ps&expand=450) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovaps))] //should be vpblendmps +pub unsafe fn _mm256_mask_blend_ps(k: __mmask8, a: __m256, b: __m256) -> __m256 { + transmute(simd_select_bitmask(k, b.as_f32x8(), a.as_f32x8())) +} + +/// Blend packed single-precision (32-bit) floating-point elements from a and b using control mask k, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_blend_ps&expand=448) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovaps))] //should be vpblendmps +pub unsafe fn _mm_mask_blend_ps(k: __mmask8, a: __m128, b: __m128) -> __m128 { + transmute(simd_select_bitmask(k, b.as_f32x4(), a.as_f32x4())) +} + +/// Blend packed double-precision (64-bit) floating-point elements from a and b using control mask k, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_blend_pd&expand=446) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovapd))] //should be vpblendmpd +pub unsafe fn _mm512_mask_blend_pd(k: __mmask8, a: __m512d, b: __m512d) -> __m512d { + transmute(simd_select_bitmask(k, b.as_f64x8(), a.as_f64x8())) +} + +/// Blend packed double-precision (64-bit) floating-point elements from a and b using control mask k, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_blend_pd&expand=445) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovapd))] //should be vpblendmpd +pub unsafe fn _mm256_mask_blend_pd(k: __mmask8, a: __m256d, b: __m256d) -> __m256d { + transmute(simd_select_bitmask(k, b.as_f64x4(), a.as_f64x4())) +} + +/// Blend packed double-precision (64-bit) floating-point elements from a and b using control mask k, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_blend_pd&expand=443) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovapd))] //should be vpblendmpd +pub unsafe fn _mm_mask_blend_pd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + transmute(simd_select_bitmask(k, b.as_f64x2(), a.as_f64x2())) +} + +/// Concatenate a and b into a 128-byte immediate result, shift the result right by imm8 32-bit elements, and store the low 64 bytes (16 elements) in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_alignr_epi32&expand=245) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(valignd, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_alignr_epi32(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let a = a.as_i32x16(); + let b = b.as_i32x16(); + let imm8: i32 = IMM8 % 16; + let r: i32x16 = match imm8 { + 0 => simd_shuffle16!( + a, + b, + [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,], + ), + 1 => simd_shuffle16!( + a, + b, + [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0,], + ), + 2 => simd_shuffle16!( + a, + b, + [18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1], + ), + 3 => simd_shuffle16!( + a, + b, + [19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2], + ), + 4 => simd_shuffle16!( + a, + b, + [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3], + ), + 5 => simd_shuffle16!( + a, + b, + [21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4], + ), + 6 => simd_shuffle16!( + a, + b, + [22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5], + ), + 7 => simd_shuffle16!( + a, + b, + [23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6], + ), + 8 => simd_shuffle16!( + a, + b, + [24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7], + ), + 9 => simd_shuffle16!( + a, + b, + [25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8], + ), + 10 => simd_shuffle16!(a, b, [26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), + 11 => simd_shuffle16!(a, b, [27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), + 12 => simd_shuffle16!(a, b, [28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]), + 13 => simd_shuffle16!(a, b, [29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]), + 14 => simd_shuffle16!(a, b, [30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]), + _ => simd_shuffle16!(a, b, [31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]), + }; + transmute(r) +} + +/// Concatenate a and b into a 128-byte immediate result, shift the result right by imm8 32-bit elements, and store the low 64 bytes (16 elements) in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_mask_alignr_epi32&expand=246) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(valignd, IMM8 = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_alignr_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let r = _mm512_alignr_epi32::(a, b); + transmute(simd_select_bitmask(k, r.as_i32x16(), src.as_i32x16())) +} + +/// Concatenate a and b into a 128-byte immediate result, shift the result right by imm8 32-bit elements, and stores the low 64 bytes (16 elements) in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_maskz_alignr_epi32&expand=247) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(valignd, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_alignr_epi32( + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let r = _mm512_alignr_epi32::(a, b); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, r.as_i32x16(), zero)) +} + +/// Concatenate a and b into a 64-byte immediate result, shift the result right by imm8 32-bit elements, and store the low 32 bytes (8 elements) in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_alignr_epi32&expand=242) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(valignd, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_alignr_epi32(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_i32x8(); + let b = b.as_i32x8(); + let imm8: i32 = IMM8 % 16; + let r: i32x8 = match imm8 { + 0 => simd_shuffle8!(a, b, [8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle8!(a, b, [9, 10, 11, 12, 13, 14, 15, 0]), + 2 => simd_shuffle8!(a, b, [10, 11, 12, 13, 14, 15, 0, 1]), + 3 => simd_shuffle8!(a, b, [11, 12, 13, 14, 15, 0, 1, 2]), + 4 => simd_shuffle8!(a, b, [12, 13, 14, 15, 0, 1, 2, 3]), + 5 => simd_shuffle8!(a, b, [13, 14, 15, 0, 1, 2, 3, 4]), + 6 => simd_shuffle8!(a, b, [14, 15, 0, 1, 2, 3, 4, 5]), + 7 => simd_shuffle8!(a, b, [15, 0, 1, 2, 3, 4, 5, 6]), + 8 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), + 9 => simd_shuffle8!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), + 10 => simd_shuffle8!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), + 11 => simd_shuffle8!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), + 12 => simd_shuffle8!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), + 13 => simd_shuffle8!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), + 14 => simd_shuffle8!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), + _ => simd_shuffle8!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), + }; + transmute(r) +} + +/// Concatenate a and b into a 64-byte immediate result, shift the result right by imm8 32-bit elements, and store the low 32 bytes (8 elements) in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_alignr_epi32&expand=243) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(valignd, IMM8 = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_alignr_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let r = _mm256_alignr_epi32::(a, b); + transmute(simd_select_bitmask(k, r.as_i32x8(), src.as_i32x8())) +} + +/// Concatenate a and b into a 64-byte immediate result, shift the result right by imm8 32-bit elements, and store the low 32 bytes (8 elements) in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_alignr_epi32&expand=244) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(valignd, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_alignr_epi32( + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let r = _mm256_alignr_epi32::(a, b); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, r.as_i32x8(), zero)) +} + +/// Concatenate a and b into a 32-byte immediate result, shift the result right by imm8 32-bit elements, and store the low 16 bytes (4 elements) in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_alignr_epi32&expand=239) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpalignr, IMM8 = 1))] //should be valignd +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_alignr_epi32(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let a = a.as_i32x4(); + let b = b.as_i32x4(); + let imm8: i32 = IMM8 % 8; + let r: i32x4 = match imm8 { + 0 => simd_shuffle4!(a, b, [4, 5, 6, 7]), + 1 => simd_shuffle4!(a, b, [5, 6, 7, 0]), + 2 => simd_shuffle4!(a, b, [6, 7, 0, 1]), + 3 => simd_shuffle4!(a, b, [7, 0, 1, 2]), + 4 => simd_shuffle4!(a, b, [0, 1, 2, 3]), + 5 => simd_shuffle4!(a, b, [1, 2, 3, 0]), + 6 => simd_shuffle4!(a, b, [2, 3, 0, 1]), + _ => simd_shuffle4!(a, b, [3, 0, 1, 2]), + }; + transmute(r) +} + +/// Concatenate a and b into a 32-byte immediate result, shift the result right by imm8 32-bit elements, and store the low 16 bytes (4 elements) in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_alignr_epi32&expand=240) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(valignd, IMM8 = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_alignr_epi32( + src: __m128i, + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let r = _mm_alignr_epi32::(a, b); + transmute(simd_select_bitmask(k, r.as_i32x4(), src.as_i32x4())) +} + +/// Concatenate a and b into a 32-byte immediate result, shift the result right by imm8 32-bit elements, and store the low 16 bytes (4 elements) in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_alignr_epi32&expand=241) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(valignd, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_alignr_epi32( + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let r = _mm_alignr_epi32::(a, b); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, r.as_i32x4(), zero)) +} + +/// Concatenate a and b into a 128-byte immediate result, shift the result right by imm8 64-bit elements, and store the low 64 bytes (8 elements) in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_alignr_epi64&expand=254) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(valignq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_alignr_epi64(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let imm8: i32 = IMM8 % 8; + let r: i64x8 = match imm8 { + 0 => simd_shuffle8!(a, b, [8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle8!(a, b, [9, 10, 11, 12, 13, 14, 15, 0]), + 2 => simd_shuffle8!(a, b, [10, 11, 12, 13, 14, 15, 0, 1]), + 3 => simd_shuffle8!(a, b, [11, 12, 13, 14, 15, 0, 1, 2]), + 4 => simd_shuffle8!(a, b, [12, 13, 14, 15, 0, 1, 2, 3]), + 5 => simd_shuffle8!(a, b, [13, 14, 15, 0, 1, 2, 3, 4]), + 6 => simd_shuffle8!(a, b, [14, 15, 0, 1, 2, 3, 4, 5]), + _ => simd_shuffle8!(a, b, [15, 0, 1, 2, 3, 4, 5, 6]), + }; + transmute(r) +} + +/// Concatenate a and b into a 128-byte immediate result, shift the result right by imm8 64-bit elements, and store the low 64 bytes (8 elements) in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_mask_alignr_epi64&expand=255) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(valignq, IMM8 = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_alignr_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let r = _mm512_alignr_epi64::(a, b); + transmute(simd_select_bitmask(k, r.as_i64x8(), src.as_i64x8())) +} + +/// Concatenate a and b into a 128-byte immediate result, shift the result right by imm8 64-bit elements, and stores the low 64 bytes (8 elements) in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_maskz_alignr_epi64&expand=256) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(valignq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_alignr_epi64( + k: __mmask8, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let r = _mm512_alignr_epi64::(a, b); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, r.as_i64x8(), zero)) +} + +/// Concatenate a and b into a 64-byte immediate result, shift the result right by imm8 64-bit elements, and store the low 32 bytes (4 elements) in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_alignr_epi64&expand=251) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(valignq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_alignr_epi64(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let imm8: i32 = IMM8 % 8; + let r: i64x4 = match imm8 { + 0 => simd_shuffle4!(a, b, [4, 5, 6, 7]), + 1 => simd_shuffle4!(a, b, [5, 6, 7, 0]), + 2 => simd_shuffle4!(a, b, [6, 7, 0, 1]), + 3 => simd_shuffle4!(a, b, [7, 0, 1, 2]), + 4 => simd_shuffle4!(a, b, [0, 1, 2, 3]), + 5 => simd_shuffle4!(a, b, [1, 2, 3, 4]), + 6 => simd_shuffle4!(a, b, [2, 3, 4, 5]), + _ => simd_shuffle4!(a, b, [3, 4, 5, 6]), + }; + transmute(r) +} + +/// Concatenate a and b into a 64-byte immediate result, shift the result right by imm8 64-bit elements, and store the low 32 bytes (4 elements) in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_alignr_epi64&expand=252) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(valignq, IMM8 = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_alignr_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let r = _mm256_alignr_epi64::(a, b); + transmute(simd_select_bitmask(k, r.as_i64x4(), src.as_i64x4())) +} + +/// Concatenate a and b into a 64-byte immediate result, shift the result right by imm8 64-bit elements, and store the low 32 bytes (4 elements) in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_alignr_epi64&expand=253) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(valignq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_alignr_epi64( + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let r = _mm256_alignr_epi64::(a, b); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, r.as_i64x4(), zero)) +} + +/// Concatenate a and b into a 32-byte immediate result, shift the result right by imm8 64-bit elements, and store the low 16 bytes (2 elements) in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_alignr_epi64&expand=248) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpalignr, IMM8 = 1))] //should be valignq +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_alignr_epi64(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let imm8: i32 = IMM8 % 4; + let r: i64x2 = match imm8 { + 0 => simd_shuffle2!(a, b, [2, 3]), + 1 => simd_shuffle2!(a, b, [3, 0]), + 2 => simd_shuffle2!(a, b, [0, 1]), + _ => simd_shuffle2!(a, b, [1, 2]), + }; + transmute(r) +} + +/// Concatenate a and b into a 32-byte immediate result, shift the result right by imm8 64-bit elements, and store the low 16 bytes (2 elements) in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_alignr_epi64&expand=249) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(valignq, IMM8 = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_alignr_epi64( + src: __m128i, + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let r = _mm_alignr_epi64::(a, b); + transmute(simd_select_bitmask(k, r.as_i64x2(), src.as_i64x2())) +} + +/// Concatenate a and b into a 32-byte immediate result, shift the result right by imm8 64-bit elements, and store the low 16 bytes (2 elements) in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_alignr_epi64&expand=250) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(valignq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_alignr_epi64( + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let r = _mm_alignr_epi64::(a, b); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, r.as_i64x2(), zero)) +} + +/// Compute the bitwise AND of packed 32-bit integers in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_and_epi32&expand=272) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpandq))] //should be vpandd, but generate vpandq +pub unsafe fn _mm512_and_epi32(a: __m512i, b: __m512i) -> __m512i { + transmute(simd_and(a.as_i32x16(), b.as_i32x16())) +} + +/// Performs element-by-element bitwise AND between packed 32-bit integer elements of a and b, storing the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_and_epi32&expand=273) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpandd))] +pub unsafe fn _mm512_mask_and_epi32(src: __m512i, k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let and = _mm512_and_epi32(a, b).as_i32x16(); + transmute(simd_select_bitmask(k, and, src.as_i32x16())) +} + +/// Compute the bitwise AND of packed 32-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_and_epi32&expand=274) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpandd))] +pub unsafe fn _mm512_maskz_and_epi32(k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let and = _mm512_and_epi32(a, b).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, and, zero)) +} + +/// Performs element-by-element bitwise AND between packed 32-bit integer elements of a and b, storing the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_and_epi32&expand=270) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpandd))] +pub unsafe fn _mm256_mask_and_epi32(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let and = simd_and(a.as_i32x8(), b.as_i32x8()); + transmute(simd_select_bitmask(k, and, src.as_i32x8())) +} + +/// Compute the bitwise AND of packed 32-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_and_epi32&expand=271) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpandd))] +pub unsafe fn _mm256_maskz_and_epi32(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let and = simd_and(a.as_i32x8(), b.as_i32x8()); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, and, zero)) +} + +/// Performs element-by-element bitwise AND between packed 32-bit integer elements of a and b, storing the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_and_epi32&expand=268) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpandd))] +pub unsafe fn _mm_mask_and_epi32(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let and = simd_and(a.as_i32x4(), b.as_i32x4()); + transmute(simd_select_bitmask(k, and, src.as_i32x4())) +} + +/// Compute the bitwise AND of packed 32-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_and_epi32&expand=269) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpandd))] +pub unsafe fn _mm_maskz_and_epi32(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let and = simd_and(a.as_i32x4(), b.as_i32x4()); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, and, zero)) +} + +/// Compute the bitwise AND of 512 bits (composed of packed 64-bit integers) in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_and_epi64&expand=279) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpandq))] +pub unsafe fn _mm512_and_epi64(a: __m512i, b: __m512i) -> __m512i { + transmute(simd_and(a.as_i64x8(), b.as_i64x8())) +} + +/// Compute the bitwise AND of packed 64-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_and_epi64&expand=280) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpandq))] +pub unsafe fn _mm512_mask_and_epi64(src: __m512i, k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let and = _mm512_and_epi64(a, b).as_i64x8(); + transmute(simd_select_bitmask(k, and, src.as_i64x8())) +} + +/// Compute the bitwise AND of packed 64-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_and_epi64&expand=281) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpandq))] +pub unsafe fn _mm512_maskz_and_epi64(k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let and = _mm512_and_epi64(a, b).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, and, zero)) +} + +/// Compute the bitwise AND of packed 64-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_and_epi64&expand=277) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpandq))] +pub unsafe fn _mm256_mask_and_epi64(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let and = simd_and(a.as_i64x4(), b.as_i64x4()); + transmute(simd_select_bitmask(k, and, src.as_i64x4())) +} + +/// Compute the bitwise AND of packed 64-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_and_epi64&expand=278) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpandq))] +pub unsafe fn _mm256_maskz_and_epi64(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let and = simd_and(a.as_i64x4(), b.as_i64x4()); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, and, zero)) +} + +/// Compute the bitwise AND of packed 64-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_and_epi64&expand=275) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpandq))] +pub unsafe fn _mm_mask_and_epi64(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let and = simd_and(a.as_i64x2(), b.as_i64x2()); + transmute(simd_select_bitmask(k, and, src.as_i64x2())) +} + +/// Compute the bitwise AND of packed 64-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_and_epi64&expand=276) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpandq))] +pub unsafe fn _mm_maskz_and_epi64(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let and = simd_and(a.as_i64x2(), b.as_i64x2()); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, and, zero)) +} + +/// Compute the bitwise AND of 512 bits (representing integer data) in a and b, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_and_si512&expand=302) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpandq))] +pub unsafe fn _mm512_and_si512(a: __m512i, b: __m512i) -> __m512i { + transmute(simd_and(a.as_i32x16(), b.as_i32x16())) +} + +/// Compute the bitwise OR of packed 32-bit integers in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_or_epi32&expand=4042) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vporq))] +pub unsafe fn _mm512_or_epi32(a: __m512i, b: __m512i) -> __m512i { + transmute(simd_or(a.as_i32x16(), b.as_i32x16())) +} + +/// Compute the bitwise OR of packed 32-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_or_epi32&expand=4040) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpord))] +pub unsafe fn _mm512_mask_or_epi32(src: __m512i, k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let or = _mm512_or_epi32(a, b).as_i32x16(); + transmute(simd_select_bitmask(k, or, src.as_i32x16())) +} + +/// Compute the bitwise OR of packed 32-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_or_epi32&expand=4041) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpord))] +pub unsafe fn _mm512_maskz_or_epi32(k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let or = _mm512_or_epi32(a, b).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, or, zero)) +} + +/// Compute the bitwise OR of packed 32-bit integers in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_or_epi32&expand=4039) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vor))] //should be vpord +pub unsafe fn _mm256_or_epi32(a: __m256i, b: __m256i) -> __m256i { + transmute(simd_or(a.as_i32x8(), b.as_i32x8())) +} + +/// Compute the bitwise OR of packed 32-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_or_epi32&expand=4037) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpord))] +pub unsafe fn _mm256_mask_or_epi32(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let or = _mm256_or_epi32(a, b).as_i32x8(); + transmute(simd_select_bitmask(k, or, src.as_i32x8())) +} + +/// Compute the bitwise OR of packed 32-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_or_epi32&expand=4038) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpord))] +pub unsafe fn _mm256_maskz_or_epi32(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let or = _mm256_or_epi32(a, b).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, or, zero)) +} + +/// Compute the bitwise OR of packed 32-bit integers in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_or_epi32&expand=4036) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vor))] //should be vpord +pub unsafe fn _mm_or_epi32(a: __m128i, b: __m128i) -> __m128i { + transmute(simd_or(a.as_i32x4(), b.as_i32x4())) +} + +/// Compute the bitwise OR of packed 32-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_or_epi32&expand=4034) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpord))] +pub unsafe fn _mm_mask_or_epi32(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let or = _mm_or_epi32(a, b).as_i32x4(); + transmute(simd_select_bitmask(k, or, src.as_i32x4())) +} + +/// Compute the bitwise OR of packed 32-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_or_epi32&expand=4035) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpord))] +pub unsafe fn _mm_maskz_or_epi32(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let or = _mm_or_epi32(a, b).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, or, zero)) +} + +/// Compute the bitwise OR of packed 64-bit integers in a and b, and store the resut in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_or_epi64&expand=4051) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vporq))] +pub unsafe fn _mm512_or_epi64(a: __m512i, b: __m512i) -> __m512i { + transmute(simd_or(a.as_i64x8(), b.as_i64x8())) +} + +/// Compute the bitwise OR of packed 64-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_or_epi64&expand=4049) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vporq))] +pub unsafe fn _mm512_mask_or_epi64(src: __m512i, k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let or = _mm512_or_epi64(a, b).as_i64x8(); + transmute(simd_select_bitmask(k, or, src.as_i64x8())) +} + +/// Compute the bitwise OR of packed 64-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_or_epi64&expand=4050) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vporq))] +pub unsafe fn _mm512_maskz_or_epi64(k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let or = _mm512_or_epi64(a, b).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, or, zero)) +} + +/// Compute the bitwise OR of packed 64-bit integers in a and b, and store the resut in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_or_epi64&expand=4048) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vor))] //should be vporq +pub unsafe fn _mm256_or_epi64(a: __m256i, b: __m256i) -> __m256i { + transmute(simd_or(a.as_i64x4(), b.as_i64x4())) +} + +/// Compute the bitwise OR of packed 64-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_or_epi64&expand=4046) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vporq))] +pub unsafe fn _mm256_mask_or_epi64(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let or = _mm256_or_epi64(a, b).as_i64x4(); + transmute(simd_select_bitmask(k, or, src.as_i64x4())) +} + +/// Compute the bitwise OR of packed 64-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_or_epi64&expand=4047) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vporq))] +pub unsafe fn _mm256_maskz_or_epi64(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let or = _mm256_or_epi64(a, b).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, or, zero)) +} + +/// Compute the bitwise OR of packed 64-bit integers in a and b, and store the resut in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_or_epi64&expand=4045) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vor))] //should be vporq +pub unsafe fn _mm_or_epi64(a: __m128i, b: __m128i) -> __m128i { + transmute(simd_or(a.as_i64x2(), b.as_i64x2())) +} + +/// Compute the bitwise OR of packed 64-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_or_epi64&expand=4043) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vporq))] +pub unsafe fn _mm_mask_or_epi64(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let or = _mm_or_epi64(a, b).as_i64x2(); + transmute(simd_select_bitmask(k, or, src.as_i64x2())) +} + +/// Compute the bitwise OR of packed 64-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_or_epi64&expand=4044) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vporq))] +pub unsafe fn _mm_maskz_or_epi64(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let or = _mm_or_epi64(a, b).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, or, zero)) +} + +/// Compute the bitwise OR of 512 bits (representing integer data) in a and b, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_or_si512&expand=4072) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vporq))] +pub unsafe fn _mm512_or_si512(a: __m512i, b: __m512i) -> __m512i { + transmute(simd_or(a.as_i32x16(), b.as_i32x16())) +} + +/// Compute the bitwise XOR of packed 32-bit integers in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_xor_epi32&expand=6142) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpxorq))] //should be vpxord +pub unsafe fn _mm512_xor_epi32(a: __m512i, b: __m512i) -> __m512i { + transmute(simd_xor(a.as_i32x16(), b.as_i32x16())) +} + +/// Compute the bitwise XOR of packed 32-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_xor_epi32&expand=6140) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpxord))] +pub unsafe fn _mm512_mask_xor_epi32(src: __m512i, k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let xor = _mm512_xor_epi32(a, b).as_i32x16(); + transmute(simd_select_bitmask(k, xor, src.as_i32x16())) +} + +/// Compute the bitwise XOR of packed 32-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_xor_epi32&expand=6141) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpxord))] +pub unsafe fn _mm512_maskz_xor_epi32(k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let xor = _mm512_xor_epi32(a, b).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, xor, zero)) +} + +/// Compute the bitwise XOR of packed 32-bit integers in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_xor_epi32&expand=6139) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vxor))] //should be vpxord +pub unsafe fn _mm256_xor_epi32(a: __m256i, b: __m256i) -> __m256i { + transmute(simd_xor(a.as_i32x8(), b.as_i32x8())) +} + +/// Compute the bitwise XOR of packed 32-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_xor_epi32&expand=6137) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpxord))] +pub unsafe fn _mm256_mask_xor_epi32(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let xor = _mm256_xor_epi32(a, b).as_i32x8(); + transmute(simd_select_bitmask(k, xor, src.as_i32x8())) +} + +/// Compute the bitwise XOR of packed 32-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_xor_epi32&expand=6138) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpxord))] +pub unsafe fn _mm256_maskz_xor_epi32(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let xor = _mm256_xor_epi32(a, b).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, xor, zero)) +} + +/// Compute the bitwise XOR of packed 32-bit integers in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_xor_epi32&expand=6136) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vxor))] //should be vpxord +pub unsafe fn _mm_xor_epi32(a: __m128i, b: __m128i) -> __m128i { + transmute(simd_xor(a.as_i32x4(), b.as_i32x4())) +} + +/// Compute the bitwise XOR of packed 32-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_xor_epi32&expand=6134) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpxord))] +pub unsafe fn _mm_mask_xor_epi32(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let xor = _mm_xor_epi32(a, b).as_i32x4(); + transmute(simd_select_bitmask(k, xor, src.as_i32x4())) +} + +/// Compute the bitwise XOR of packed 32-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_xor_epi32&expand=6135) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpxord))] +pub unsafe fn _mm_maskz_xor_epi32(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let xor = _mm_xor_epi32(a, b).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, xor, zero)) +} + +/// Compute the bitwise XOR of packed 64-bit integers in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_xor_epi64&expand=6151) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpxorq))] +pub unsafe fn _mm512_xor_epi64(a: __m512i, b: __m512i) -> __m512i { + transmute(simd_xor(a.as_i64x8(), b.as_i64x8())) +} + +/// Compute the bitwise XOR of packed 64-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_xor_epi64&expand=6149) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpxorq))] +pub unsafe fn _mm512_mask_xor_epi64(src: __m512i, k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let xor = _mm512_xor_epi64(a, b).as_i64x8(); + transmute(simd_select_bitmask(k, xor, src.as_i64x8())) +} + +/// Compute the bitwise XOR of packed 64-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_xor_epi64&expand=6150) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpxorq))] +pub unsafe fn _mm512_maskz_xor_epi64(k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let xor = _mm512_xor_epi64(a, b).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, xor, zero)) +} + +/// Compute the bitwise XOR of packed 64-bit integers in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_xor_epi64&expand=6148) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vxor))] //should be vpxorq +pub unsafe fn _mm256_xor_epi64(a: __m256i, b: __m256i) -> __m256i { + transmute(simd_xor(a.as_i64x4(), b.as_i64x4())) +} + +/// Compute the bitwise XOR of packed 64-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_xor_epi64&expand=6146) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpxorq))] +pub unsafe fn _mm256_mask_xor_epi64(src: __m256i, k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let xor = _mm256_xor_epi64(a, b).as_i64x4(); + transmute(simd_select_bitmask(k, xor, src.as_i64x4())) +} + +/// Compute the bitwise XOR of packed 64-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_xor_epi64&expand=6147) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpxorq))] +pub unsafe fn _mm256_maskz_xor_epi64(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let xor = _mm256_xor_epi64(a, b).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, xor, zero)) +} + +/// Compute the bitwise XOR of packed 64-bit integers in a and b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_xor_epi64&expand=6145) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vxor))] //should be vpxorq +pub unsafe fn _mm_xor_epi64(a: __m128i, b: __m128i) -> __m128i { + transmute(simd_xor(a.as_i64x2(), b.as_i64x2())) +} + +/// Compute the bitwise XOR of packed 64-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_xor_epi64&expand=6143) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpxorq))] +pub unsafe fn _mm_mask_xor_epi64(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let xor = _mm_xor_epi64(a, b).as_i64x2(); + transmute(simd_select_bitmask(k, xor, src.as_i64x2())) +} + +/// Compute the bitwise XOR of packed 64-bit integers in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_xor_epi64&expand=6144) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpxorq))] +pub unsafe fn _mm_maskz_xor_epi64(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let xor = _mm_xor_epi64(a, b).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, xor, zero)) +} + +/// Compute the bitwise XOR of 512 bits (representing integer data) in a and b, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_xor_si512&expand=6172) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpxorq))] +pub unsafe fn _mm512_xor_si512(a: __m512i, b: __m512i) -> __m512i { + transmute(simd_xor(a.as_i32x16(), b.as_i32x16())) +} + +/// Compute the bitwise NOT of packed 32-bit integers in a and then AND with b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_andnot_epi32&expand=310) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpandnq))] //should be vpandnd +pub unsafe fn _mm512_andnot_epi32(a: __m512i, b: __m512i) -> __m512i { + _mm512_and_epi32(_mm512_xor_epi32(a, _mm512_set1_epi32(u32::MAX as i32)), b) +} + +/// Compute the bitwise NOT of packed 32-bit integers in a and then AND with b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_andnot_epi32&expand=311) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpandnd))] +pub unsafe fn _mm512_mask_andnot_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + let andnot = _mm512_andnot_epi32(a, b).as_i32x16(); + transmute(simd_select_bitmask(k, andnot, src.as_i32x16())) +} + +/// Compute the bitwise NOT of packed 32-bit integers in a and then AND with b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_andnot_epi32&expand=312) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpandnd))] +pub unsafe fn _mm512_maskz_andnot_epi32(k: __mmask16, a: __m512i, b: __m512i) -> __m512i { + let andnot = _mm512_andnot_epi32(a, b).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, andnot, zero)) +} + +/// Compute the bitwise NOT of packed 32-bit integers in a and then AND with b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_andnot_epi32&expand=308) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpandnd))] +pub unsafe fn _mm256_mask_andnot_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + let not = _mm256_xor_epi32(a, _mm256_set1_epi32(u32::MAX as i32)); + let andnot = simd_and(not.as_i32x8(), b.as_i32x8()); + transmute(simd_select_bitmask(k, andnot, src.as_i32x8())) +} + +/// Compute the bitwise NOT of packed 32-bit integers in a and then AND with b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_andnot_epi32&expand=309) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpandnd))] +pub unsafe fn _mm256_maskz_andnot_epi32(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let not = _mm256_xor_epi32(a, _mm256_set1_epi32(u32::MAX as i32)); + let andnot = simd_and(not.as_i32x8(), b.as_i32x8()); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, andnot, zero)) +} + +/// Compute the bitwise NOT of packed 32-bit integers in a and then AND with b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_andnot_epi32&expand=306) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpandnd))] +pub unsafe fn _mm_mask_andnot_epi32(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let not = _mm_xor_epi32(a, _mm_set1_epi32(u32::MAX as i32)); + let andnot = simd_and(not.as_i32x4(), b.as_i32x4()); + transmute(simd_select_bitmask(k, andnot, src.as_i32x4())) +} + +/// Compute the bitwise NOT of packed 32-bit integers in a and then AND with b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_andnot_epi32&expand=307) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpandnd))] +pub unsafe fn _mm_maskz_andnot_epi32(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let not = _mm_xor_epi32(a, _mm_set1_epi32(u32::MAX as i32)); + let andnot = simd_and(not.as_i32x4(), b.as_i32x4()); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, andnot, zero)) +} + +/// Compute the bitwise NOT of 512 bits (composed of packed 64-bit integers) in a and then AND with b, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_andnot_epi64&expand=317) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpandnq))] //should be vpandnd +pub unsafe fn _mm512_andnot_epi64(a: __m512i, b: __m512i) -> __m512i { + _mm512_and_epi64(_mm512_xor_epi64(a, _mm512_set1_epi64(u64::MAX as i64)), b) +} + +/// Compute the bitwise NOT of packed 64-bit integers in a and then AND with b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_andnot_epi64&expand=318) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpandnq))] +pub unsafe fn _mm512_mask_andnot_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, + b: __m512i, +) -> __m512i { + let andnot = _mm512_andnot_epi64(a, b).as_i64x8(); + transmute(simd_select_bitmask(k, andnot, src.as_i64x8())) +} + +/// Compute the bitwise NOT of packed 64-bit integers in a and then AND with b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_andnot_epi64&expand=319) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpandnq))] +pub unsafe fn _mm512_maskz_andnot_epi64(k: __mmask8, a: __m512i, b: __m512i) -> __m512i { + let andnot = _mm512_andnot_epi64(a, b).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, andnot, zero)) +} + +/// Compute the bitwise NOT of packed 64-bit integers in a and then AND with b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_andnot_epi64&expand=315) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpandnq))] +pub unsafe fn _mm256_mask_andnot_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + let not = _mm256_xor_epi64(a, _mm256_set1_epi64x(u64::MAX as i64)); + let andnot = simd_and(not.as_i64x4(), b.as_i64x4()); + transmute(simd_select_bitmask(k, andnot, src.as_i64x4())) +} + +/// Compute the bitwise NOT of packed 64-bit integers in a and then AND with b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_andnot_epi64&expand=316) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpandnq))] +pub unsafe fn _mm256_maskz_andnot_epi64(k: __mmask8, a: __m256i, b: __m256i) -> __m256i { + let not = _mm256_xor_epi64(a, _mm256_set1_epi64x(u64::MAX as i64)); + let andnot = simd_and(not.as_i64x4(), b.as_i64x4()); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, andnot, zero)) +} + +/// Compute the bitwise NOT of packed 64-bit integers in a and then AND with b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_andnot_epi64&expand=313) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpandnq))] +pub unsafe fn _mm_mask_andnot_epi64(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let not = _mm_xor_epi64(a, _mm_set1_epi64x(u64::MAX as i64)); + let andnot = simd_and(not.as_i64x2(), b.as_i64x2()); + transmute(simd_select_bitmask(k, andnot, src.as_i64x2())) +} + +/// Compute the bitwise NOT of packed 64-bit integers in a and then AND with b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_andnot_epi64&expand=314) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpandnq))] +pub unsafe fn _mm_maskz_andnot_epi64(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let not = _mm_xor_epi64(a, _mm_set1_epi64x(u64::MAX as i64)); + let andnot = simd_and(not.as_i64x2(), b.as_i64x2()); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, andnot, zero)) +} + +/// Compute the bitwise NOT of 512 bits (representing integer data) in a and then AND with b, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_andnot_si512&expand=340) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpandnq))] +pub unsafe fn _mm512_andnot_si512(a: __m512i, b: __m512i) -> __m512i { + _mm512_and_epi64(_mm512_xor_epi64(a, _mm512_set1_epi64(u64::MAX as i64)), b) +} + +/// Compute the bitwise AND of 16-bit masks a and b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=kand_mask16&expand=3212) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(and))] // generate normal and code instead of kandw +pub unsafe fn _kand_mask16(a: __mmask16, b: __mmask16) -> __mmask16 { + transmute(a & b) +} + +/// Compute the bitwise AND of 16-bit masks a and b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_kand&expand=3210) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(and))] // generate normal and code instead of kandw +pub unsafe fn _mm512_kand(a: __mmask16, b: __mmask16) -> __mmask16 { + transmute(a & b) +} + +/// Compute the bitwise OR of 16-bit masks a and b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=kor_mask16&expand=3239) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(or))] // generate normal or code instead of korw +pub unsafe fn _kor_mask16(a: __mmask16, b: __mmask16) -> __mmask16 { + transmute(a | b) +} + +/// Compute the bitwise OR of 16-bit masks a and b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_kor&expand=3237) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(or))] // generate normal or code instead of korw +pub unsafe fn _mm512_kor(a: __mmask16, b: __mmask16) -> __mmask16 { + transmute(a | b) +} + +/// Compute the bitwise XOR of 16-bit masks a and b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=kxor_mask16&expand=3291) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(xor))] // generate normal xor code instead of kxorw +pub unsafe fn _kxor_mask16(a: __mmask16, b: __mmask16) -> __mmask16 { + transmute(a ^ b) +} + +/// Compute the bitwise XOR of 16-bit masks a and b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_kxor&expand=3289) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(xor))] // generate normal xor code instead of kxorw +pub unsafe fn _mm512_kxor(a: __mmask16, b: __mmask16) -> __mmask16 { + transmute(a ^ b) +} + +/// Compute the bitwise NOT of 16-bit mask a, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=knot_mask16&expand=3233) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _knot_mask16(a: __mmask16) -> __mmask16 { + transmute(a ^ 0b11111111_11111111) +} + +/// Compute the bitwise NOT of 16-bit mask a, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_knot&expand=3231) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_knot(a: __mmask16) -> __mmask16 { + transmute(a ^ 0b11111111_11111111) +} + +/// Compute the bitwise NOT of 16-bit masks a and then AND with b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=kandn_mask16&expand=3218) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(not))] // generate normal and, not code instead of kandnw +pub unsafe fn _kandn_mask16(a: __mmask16, b: __mmask16) -> __mmask16 { + _mm512_kand(_mm512_knot(a), b) +} + +/// Compute the bitwise NOT of 16-bit masks a and then AND with b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_kandn&expand=3216) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(not))] // generate normal and code instead of kandw +pub unsafe fn _mm512_kandn(a: __mmask16, b: __mmask16) -> __mmask16 { + _mm512_kand(_mm512_knot(a), b) +} + +/// Compute the bitwise XNOR of 16-bit masks a and b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=kxnor_mask16&expand=3285) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(xor))] // generate normal xor, not code instead of kxnorw +pub unsafe fn _kxnor_mask16(a: __mmask16, b: __mmask16) -> __mmask16 { + _mm512_knot(_mm512_kxor(a, b)) +} + +/// Compute the bitwise XNOR of 16-bit masks a and b, and store the result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_kxnor&expand=3283) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(xor))] // generate normal and code instead of kandw +pub unsafe fn _mm512_kxnor(a: __mmask16, b: __mmask16) -> __mmask16 { + _mm512_knot(_mm512_kxor(a, b)) +} + +/// Copy 16-bit mask a to k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm512_kmov&expand=3228) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(mov))] // generate normal and code instead of kmovw +pub unsafe fn _mm512_kmov(a: __mmask16) -> __mmask16 { + let r: u16 = a; + transmute(r) +} + +/// Converts integer mask into bitmask, storing the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_int2mask&expand=3189) +#[inline] +#[target_feature(enable = "avx512f")] // generate normal and code instead of kmovw +pub unsafe fn _mm512_int2mask(mask: i32) -> __mmask16 { + let r: u16 = mask as u16; + transmute(r) +} + +/// Converts bit mask k1 into an integer value, storing the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_mask2int&expand=3544) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(mov))] // generate normal and code instead of kmovw +pub unsafe fn _mm512_mask2int(k1: __mmask16) -> i32 { + let r: i32 = k1 as i32; + transmute(r) +} + +/// Unpack and interleave 8 bits from masks a and b, and store the 16-bit result in k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_kunpackb&expand=3280) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(mov))] // generate normal and code instead of kunpckbw +pub unsafe fn _mm512_kunpackb(a: __mmask16, b: __mmask16) -> __mmask16 { + let a = a & 0b00000000_11111111; + let b = b & 0b11111111_00000000; + transmute(a | b) +} + +/// Performs bitwise OR between k1 and k2, storing the result in dst. CF flag is set if dst consists of all 1's. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_kortestc&expand=3247) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(cmp))] // generate normal and code instead of kortestw +pub unsafe fn _mm512_kortestc(a: __mmask16, b: __mmask16) -> i32 { + let r = a | b; + if r == 0b11111111_11111111 { + 1 + } else { + 0 + } +} + +/// Compute the bitwise AND of packed 32-bit integers in a and b, producing intermediate 32-bit values, and set the corresponding bit in result mask k if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_test_epi32_mask&expand=5890) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vptestmd))] +pub unsafe fn _mm512_test_epi32_mask(a: __m512i, b: __m512i) -> __mmask16 { + let and = _mm512_and_epi32(a, b); + let zero = _mm512_setzero_si512(); + _mm512_cmpneq_epi32_mask(and, zero) +} + +/// Compute the bitwise AND of packed 32-bit integers in a and b, producing intermediate 32-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_test_epi32_mask&expand=5889) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vptestmd))] +pub unsafe fn _mm512_mask_test_epi32_mask(k: __mmask16, a: __m512i, b: __m512i) -> __mmask16 { + let and = _mm512_and_epi32(a, b); + let zero = _mm512_setzero_si512(); + _mm512_mask_cmpneq_epi32_mask(k, and, zero) +} + +/// Compute the bitwise AND of packed 32-bit integers in a and b, producing intermediate 32-bit values, and set the corresponding bit in result mask k if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_test_epi32_mask&expand=5888) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vptestmd))] +pub unsafe fn _mm256_test_epi32_mask(a: __m256i, b: __m256i) -> __mmask8 { + let and = _mm256_and_si256(a, b); + let zero = _mm256_setzero_si256(); + _mm256_cmpneq_epi32_mask(and, zero) +} + +/// Compute the bitwise AND of packed 32-bit integers in a and b, producing intermediate 32-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_test_epi32_mask&expand=5887) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vptestmd))] +pub unsafe fn _mm256_mask_test_epi32_mask(k: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + let and = _mm256_and_si256(a, b); + let zero = _mm256_setzero_si256(); + _mm256_mask_cmpneq_epi32_mask(k, and, zero) +} + +/// Compute the bitwise AND of packed 32-bit integers in a and b, producing intermediate 32-bit values, and set the corresponding bit in result mask k if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_test_epi32_mask&expand=5886) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vptestmd))] +pub unsafe fn _mm_test_epi32_mask(a: __m128i, b: __m128i) -> __mmask8 { + let and = _mm_and_si128(a, b); + let zero = _mm_setzero_si128(); + _mm_cmpneq_epi32_mask(and, zero) +} + +/// Compute the bitwise AND of packed 32-bit integers in a and b, producing intermediate 32-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_test_epi32_mask&expand=5885) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vptestmd))] +pub unsafe fn _mm_mask_test_epi32_mask(k: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + let and = _mm_and_si128(a, b); + let zero = _mm_setzero_si128(); + _mm_mask_cmpneq_epi32_mask(k, and, zero) +} + +/// Compute the bitwise AND of packed 64-bit integers in a and b, producing intermediate 64-bit values, and set the corresponding bit in result mask k if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_test_epi64_mask&expand=5896) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vptestmq))] +pub unsafe fn _mm512_test_epi64_mask(a: __m512i, b: __m512i) -> __mmask8 { + let and = _mm512_and_epi64(a, b); + let zero = _mm512_setzero_si512(); + _mm512_cmpneq_epi64_mask(and, zero) +} + +/// Compute the bitwise AND of packed 64-bit integers in a and b, producing intermediate 64-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_test_epi64_mask&expand=5895) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vptestmq))] +pub unsafe fn _mm512_mask_test_epi64_mask(k: __mmask8, a: __m512i, b: __m512i) -> __mmask8 { + let and = _mm512_and_epi64(a, b); + let zero = _mm512_setzero_si512(); + _mm512_mask_cmpneq_epi64_mask(k, and, zero) +} + +/// Compute the bitwise AND of packed 64-bit integers in a and b, producing intermediate 64-bit values, and set the corresponding bit in result mask k if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_test_epi64_mask&expand=5894) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vptestmq))] +pub unsafe fn _mm256_test_epi64_mask(a: __m256i, b: __m256i) -> __mmask8 { + let and = _mm256_and_si256(a, b); + let zero = _mm256_setzero_si256(); + _mm256_cmpneq_epi64_mask(and, zero) +} + +/// Compute the bitwise AND of packed 64-bit integers in a and b, producing intermediate 64-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_test_epi64_mask&expand=5893) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vptestmq))] +pub unsafe fn _mm256_mask_test_epi64_mask(k: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + let and = _mm256_and_si256(a, b); + let zero = _mm256_setzero_si256(); + _mm256_mask_cmpneq_epi64_mask(k, and, zero) +} + +/// Compute the bitwise AND of packed 64-bit integers in a and b, producing intermediate 64-bit values, and set the corresponding bit in result mask k if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_test_epi64_mask&expand=5892) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vptestmq))] +pub unsafe fn _mm_test_epi64_mask(a: __m128i, b: __m128i) -> __mmask8 { + let and = _mm_and_si128(a, b); + let zero = _mm_setzero_si128(); + _mm_cmpneq_epi64_mask(and, zero) +} + +/// Compute the bitwise AND of packed 64-bit integers in a and b, producing intermediate 64-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is non-zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_test_epi64_mask&expand=5891) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vptestmq))] +pub unsafe fn _mm_mask_test_epi64_mask(k: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + let and = _mm_and_si128(a, b); + let zero = _mm_setzero_si128(); + _mm_mask_cmpneq_epi64_mask(k, and, zero) +} + +/// Compute the bitwise NAND of packed 32-bit integers in a and b, producing intermediate 32-bit values, and set the corresponding bit in result mask k if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_testn_epi32_mask&expand=5921) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vptestnmd))] +pub unsafe fn _mm512_testn_epi32_mask(a: __m512i, b: __m512i) -> __mmask16 { + let and = _mm512_and_epi32(a, b); + let zero = _mm512_setzero_si512(); + _mm512_cmpeq_epi32_mask(and, zero) +} + +/// Compute the bitwise NAND of packed 32-bit integers in a and b, producing intermediate 32-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_testn_epi32_mask&expand=5920) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vptestnmd))] +pub unsafe fn _mm512_mask_testn_epi32_mask(k: __mmask16, a: __m512i, b: __m512i) -> __mmask16 { + let and = _mm512_and_epi32(a, b); + let zero = _mm512_setzero_si512(); + _mm512_mask_cmpeq_epi32_mask(k, and, zero) +} + +/// Compute the bitwise NAND of packed 32-bit integers in a and b, producing intermediate 32-bit values, and set the corresponding bit in result mask k if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_testn_epi32_mask&expand=5919) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vptestnmd))] +pub unsafe fn _mm256_testn_epi32_mask(a: __m256i, b: __m256i) -> __mmask8 { + let and = _mm256_and_si256(a, b); + let zero = _mm256_setzero_si256(); + _mm256_cmpeq_epi32_mask(and, zero) +} + +/// Compute the bitwise NAND of packed 32-bit integers in a and b, producing intermediate 32-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_testn_epi32_mask&expand=5918) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vptestnmd))] +pub unsafe fn _mm256_mask_testn_epi32_mask(k: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + let and = _mm256_and_si256(a, b); + let zero = _mm256_setzero_si256(); + _mm256_mask_cmpeq_epi32_mask(k, and, zero) +} + +/// Compute the bitwise NAND of packed 32-bit integers in a and b, producing intermediate 32-bit values, and set the corresponding bit in result mask k if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_testn_epi32_mask&expand=5917) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vptestnmd))] +pub unsafe fn _mm_testn_epi32_mask(a: __m128i, b: __m128i) -> __mmask8 { + let and = _mm_and_si128(a, b); + let zero = _mm_setzero_si128(); + _mm_cmpeq_epi32_mask(and, zero) +} + +/// Compute the bitwise NAND of packed 32-bit integers in a and b, producing intermediate 32-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_testn_epi32_mask&expand=5916) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vptestnmd))] +pub unsafe fn _mm_mask_testn_epi32_mask(k: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + let and = _mm_and_si128(a, b); + let zero = _mm_setzero_si128(); + _mm_mask_cmpeq_epi32_mask(k, and, zero) +} + +/// Compute the bitwise NAND of packed 64-bit integers in a and b, producing intermediate 64-bit values, and set the corresponding bit in result mask k if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_testn_epi64_mask&expand=5927) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vptestnmq))] +pub unsafe fn _mm512_testn_epi64_mask(a: __m512i, b: __m512i) -> __mmask8 { + let and = _mm512_and_epi64(a, b); + let zero = _mm512_setzero_si512(); + _mm512_cmpeq_epi64_mask(and, zero) +} + +/// Compute the bitwise NAND of packed 64-bit integers in a and b, producing intermediate 64-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_testn_epi64_mask&expand=5926) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vptestnmq))] +pub unsafe fn _mm512_mask_testn_epi64_mask(k: __mmask8, a: __m512i, b: __m512i) -> __mmask8 { + let and = _mm512_and_epi64(a, b); + let zero = _mm512_setzero_si512(); + _mm512_mask_cmpeq_epi64_mask(k, and, zero) +} + +/// Compute the bitwise NAND of packed 64-bit integers in a and b, producing intermediate 64-bit values, and set the corresponding bit in result mask k if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_testn_epi64_mask&expand=5925) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vptestnmq))] +pub unsafe fn _mm256_testn_epi64_mask(a: __m256i, b: __m256i) -> __mmask8 { + let and = _mm256_and_si256(a, b); + let zero = _mm256_setzero_si256(); + _mm256_cmpeq_epi64_mask(and, zero) +} + +/// Compute the bitwise NAND of packed 64-bit integers in a and b, producing intermediate 64-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_testn_epi64_mask&expand=5924) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vptestnmq))] +pub unsafe fn _mm256_mask_testn_epi64_mask(k: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + let and = _mm256_and_si256(a, b); + let zero = _mm256_setzero_si256(); + _mm256_mask_cmpeq_epi64_mask(k, and, zero) +} + +/// Compute the bitwise NAND of packed 64-bit integers in a and b, producing intermediate 64-bit values, and set the corresponding bit in result mask k if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_testn_epi64_mask&expand=5923) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vptestnmq))] +pub unsafe fn _mm_testn_epi64_mask(a: __m128i, b: __m128i) -> __mmask8 { + let and = _mm_and_si128(a, b); + let zero = _mm_setzero_si128(); + _mm_cmpeq_epi64_mask(and, zero) +} + +/// Compute the bitwise NAND of packed 64-bit integers in a and b, producing intermediate 64-bit values, and set the corresponding bit in result mask k (subject to writemask k) if the intermediate value is zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_testn_epi64_mask&expand=5922) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vptestnmq))] +pub unsafe fn _mm_mask_testn_epi64_mask(k: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + let and = _mm_and_si128(a, b); + let zero = _mm_setzero_si128(); + _mm_mask_cmpeq_epi64_mask(k, and, zero) +} + +/// Store 512-bits (composed of 16 packed single-precision (32-bit) floating-point elements) from a into memory using a non-temporal memory hint. mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_stream_ps&expand=5671) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovntps))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn _mm512_stream_ps(mem_addr: *mut f32, a: __m512) { + intrinsics::nontemporal_store(mem_addr as *mut __m512, a); +} + +/// Store 512-bits (composed of 8 packed double-precision (64-bit) floating-point elements) from a into memory using a non-temporal memory hint. mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_stream_pd&expand=5667) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovntps))] //should be vmovntpd +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn _mm512_stream_pd(mem_addr: *mut f64, a: __m512d) { + intrinsics::nontemporal_store(mem_addr as *mut __m512d, a); +} + +/// Store 512-bits of integer data from a into memory using a non-temporal memory hint. mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_stream_si512&expand=5675) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovntps))] //should be vmovntdq +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn _mm512_stream_si512(mem_addr: *mut i64, a: __m512i) { + intrinsics::nontemporal_store(mem_addr as *mut __m512i, a); +} + +/// Sets packed 32-bit integers in `dst` with the supplied values. +/// +/// [Intel's documentation]( https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_set_ps&expand=4931) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_set_ps( + e0: f32, + e1: f32, + e2: f32, + e3: f32, + e4: f32, + e5: f32, + e6: f32, + e7: f32, + e8: f32, + e9: f32, + e10: f32, + e11: f32, + e12: f32, + e13: f32, + e14: f32, + e15: f32, +) -> __m512 { + _mm512_setr_ps( + e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0, + ) +} + +/// Sets packed 32-bit integers in `dst` with the supplied values in +/// reverse order. +/// +/// [Intel's documentation]( https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_setr_ps&expand=5008) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_setr_ps( + e0: f32, + e1: f32, + e2: f32, + e3: f32, + e4: f32, + e5: f32, + e6: f32, + e7: f32, + e8: f32, + e9: f32, + e10: f32, + e11: f32, + e12: f32, + e13: f32, + e14: f32, + e15: f32, +) -> __m512 { + let r = f32x16::new( + e0, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13, e14, e15, + ); + transmute(r) +} + +/// Broadcast 64-bit float `a` to all elements of `dst`. +/// +/// [Intel's documentation]( https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_set1_pd&expand=4975) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_set1_pd(a: f64) -> __m512d { + transmute(f64x8::splat(a)) +} + +/// Broadcast 32-bit float `a` to all elements of `dst`. +/// +/// [Intel's documentation]( https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_set1_ps&expand=4981) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_set1_ps(a: f32) -> __m512 { + transmute(f32x16::splat(a)) +} + +/// Sets packed 32-bit integers in `dst` with the supplied values. +/// +/// [Intel's documentation]( https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_set_epi32&expand=4908) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_set_epi32( + e15: i32, + e14: i32, + e13: i32, + e12: i32, + e11: i32, + e10: i32, + e9: i32, + e8: i32, + e7: i32, + e6: i32, + e5: i32, + e4: i32, + e3: i32, + e2: i32, + e1: i32, + e0: i32, +) -> __m512i { + _mm512_setr_epi32( + e0, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13, e14, e15, + ) +} + +/// Broadcast 8-bit integer a to all elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_set1_epi8&expand=4972) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_set1_epi8(a: i8) -> __m512i { + transmute(i8x64::splat(a)) +} + +/// Broadcast the low packed 16-bit integer from a to all all elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_set1_epi16&expand=4944) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_set1_epi16(a: i16) -> __m512i { + transmute(i16x32::splat(a)) +} + +/// Broadcast 32-bit integer `a` to all elements of `dst`. +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_set1_epi32(a: i32) -> __m512i { + transmute(i32x16::splat(a)) +} + +/// Broadcast 32-bit integer a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_set1_epi32&expand=4951) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpbroadcastd))] +pub unsafe fn _mm512_mask_set1_epi32(src: __m512i, k: __mmask16, a: i32) -> __m512i { + let r = _mm512_set1_epi32(a).as_i32x16(); + transmute(simd_select_bitmask(k, r, src.as_i32x16())) +} + +/// Broadcast 32-bit integer a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_set1_epi32&expand=4952) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpbroadcastd))] +pub unsafe fn _mm512_maskz_set1_epi32(k: __mmask16, a: i32) -> __m512i { + let r = _mm512_set1_epi32(a).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Broadcast 32-bit integer a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_set1_epi32&expand=4948) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastd))] +pub unsafe fn _mm256_mask_set1_epi32(src: __m256i, k: __mmask8, a: i32) -> __m256i { + let r = _mm256_set1_epi32(a).as_i32x8(); + transmute(simd_select_bitmask(k, r, src.as_i32x8())) +} + +/// Broadcast 32-bit integer a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_set1_epi32&expand=4949) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastd))] +pub unsafe fn _mm256_maskz_set1_epi32(k: __mmask8, a: i32) -> __m256i { + let r = _mm256_set1_epi32(a).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Broadcast 32-bit integer a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_set1_epi32&expand=4945) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastd))] +pub unsafe fn _mm_mask_set1_epi32(src: __m128i, k: __mmask8, a: i32) -> __m128i { + let r = _mm_set1_epi32(a).as_i32x4(); + transmute(simd_select_bitmask(k, r, src.as_i32x4())) +} + +/// Broadcast 32-bit integer a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_set1_epi32&expand=4946) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastd))] +pub unsafe fn _mm_maskz_set1_epi32(k: __mmask8, a: i32) -> __m128i { + let r = _mm_set1_epi32(a).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Broadcast 64-bit integer `a` to all elements of `dst`. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_set1_epi64&expand=4961) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_set1_epi64(a: i64) -> __m512i { + transmute(i64x8::splat(a)) +} + +/// Broadcast 64-bit integer a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_set1_epi64&expand=4959) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpbroadcastq))] +pub unsafe fn _mm512_mask_set1_epi64(src: __m512i, k: __mmask8, a: i64) -> __m512i { + let r = _mm512_set1_epi64(a).as_i64x8(); + transmute(simd_select_bitmask(k, r, src.as_i64x8())) +} + +/// Broadcast 64-bit integer a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_set1_epi64&expand=4960) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpbroadcastq))] +pub unsafe fn _mm512_maskz_set1_epi64(k: __mmask8, a: i64) -> __m512i { + let r = _mm512_set1_epi64(a).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Broadcast 64-bit integer a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_set1_epi64&expand=4957) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastq))] +pub unsafe fn _mm256_mask_set1_epi64(src: __m256i, k: __mmask8, a: i64) -> __m256i { + let r = _mm256_set1_epi64x(a).as_i64x4(); + transmute(simd_select_bitmask(k, r, src.as_i64x4())) +} + +/// Broadcast 64-bit integer a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_set1_epi64&expand=4958) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastq))] +pub unsafe fn _mm256_maskz_set1_epi64(k: __mmask8, a: i64) -> __m256i { + let r = _mm256_set1_epi64x(a).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Broadcast 64-bit integer a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_set1_epi64&expand=4954) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastq))] +pub unsafe fn _mm_mask_set1_epi64(src: __m128i, k: __mmask8, a: i64) -> __m128i { + let r = _mm_set1_epi64x(a).as_i64x2(); + transmute(simd_select_bitmask(k, r, src.as_i64x2())) +} + +/// Broadcast 64-bit integer a to all elements of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_set1_epi64&expand=4955) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpbroadcastq))] +pub unsafe fn _mm_maskz_set1_epi64(k: __mmask8, a: i64) -> __m128i { + let r = _mm_set1_epi64x(a).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Set packed 64-bit integers in dst with the repeated 4 element sequence. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_set4_epi64&expand=4983) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_set4_epi64(d: i64, c: i64, b: i64, a: i64) -> __m512i { + let r = i64x8::new(d, c, b, a, d, c, b, a); + transmute(r) +} + +/// Set packed 64-bit integers in dst with the repeated 4 element sequence in reverse order. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_setr4_epi64&expand=5010) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_setr4_epi64(d: i64, c: i64, b: i64, a: i64) -> __m512i { + let r = i64x8::new(a, b, c, d, a, b, c, d); + transmute(r) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmplt_ps_mask&expand=1074) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps +pub unsafe fn _mm512_cmplt_ps_mask(a: __m512, b: __m512) -> __mmask16 { + _mm512_cmp_ps_mask::<_CMP_LT_OS>(a, b) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmplt_ps_mask&expand=1075) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps +pub unsafe fn _mm512_mask_cmplt_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> __mmask16 { + _mm512_mask_cmp_ps_mask::<_CMP_LT_OS>(k1, a, b) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b for not-less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpnlt_ps_mask&expand=1154) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps +pub unsafe fn _mm512_cmpnlt_ps_mask(a: __m512, b: __m512) -> __mmask16 { + _mm512_cmp_ps_mask::<_CMP_NLT_US>(a, b) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b for not-less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpnlt_ps_mask&expand=1155) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps +pub unsafe fn _mm512_mask_cmpnlt_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> __mmask16 { + _mm512_mask_cmp_ps_mask::<_CMP_NLT_US>(k1, a, b) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmple_ps_mask&expand=1013) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps +pub unsafe fn _mm512_cmple_ps_mask(a: __m512, b: __m512) -> __mmask16 { + _mm512_cmp_ps_mask::<_CMP_LE_OS>(a, b) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmple_ps_mask&expand=1014) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps +pub unsafe fn _mm512_mask_cmple_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> __mmask16 { + _mm512_mask_cmp_ps_mask::<_CMP_LE_OS>(k1, a, b) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b for not-less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpnle_ps_mask&expand=1146) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps +pub unsafe fn _mm512_cmpnle_ps_mask(a: __m512, b: __m512) -> __mmask16 { + _mm512_cmp_ps_mask::<_CMP_NLE_US>(a, b) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b for not-less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpnle_ps_mask&expand=1147) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps +pub unsafe fn _mm512_mask_cmpnle_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> __mmask16 { + _mm512_mask_cmp_ps_mask::<_CMP_NLE_US>(k1, a, b) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpeq_ps_mask&expand=828) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps +pub unsafe fn _mm512_cmpeq_ps_mask(a: __m512, b: __m512) -> __mmask16 { + _mm512_cmp_ps_mask::<_CMP_EQ_OQ>(a, b) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpeq_ps_mask&expand=829) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps +pub unsafe fn _mm512_mask_cmpeq_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> __mmask16 { + _mm512_mask_cmp_ps_mask::<_CMP_EQ_OQ>(k1, a, b) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpneq_ps_mask&expand=1130) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps +pub unsafe fn _mm512_cmpneq_ps_mask(a: __m512, b: __m512) -> __mmask16 { + _mm512_cmp_ps_mask::<_CMP_NEQ_UQ>(a, b) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpneq_ps_mask&expand=1131) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps +pub unsafe fn _mm512_mask_cmpneq_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> __mmask16 { + _mm512_mask_cmp_ps_mask::<_CMP_NEQ_UQ>(k1, a, b) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_ps_mask&expand=749) +#[inline] +#[target_feature(enable = "avx512f")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm512_cmp_ps_mask(a: __m512, b: __m512) -> __mmask16 { + static_assert_imm5!(IMM8); + let neg_one = -1; + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vcmpps(a, b, IMM8, neg_one, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_ps_mask&expand=750) +#[inline] +#[target_feature(enable = "avx512f")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm512_mask_cmp_ps_mask( + k1: __mmask16, + a: __m512, + b: __m512, +) -> __mmask16 { + static_assert_imm5!(IMM8); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vcmpps(a, b, IMM8, k1 as i16, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_ps_mask&expand=747) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm256_cmp_ps_mask(a: __m256, b: __m256) -> __mmask8 { + static_assert_imm5!(IMM8); + let neg_one = -1; + let a = a.as_f32x8(); + let b = b.as_f32x8(); + let r = vcmpps256(a, b, IMM8, neg_one); + transmute(r) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmp_ps_mask&expand=748) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm256_mask_cmp_ps_mask( + k1: __mmask8, + a: __m256, + b: __m256, +) -> __mmask8 { + static_assert_imm5!(IMM8); + let a = a.as_f32x8(); + let b = b.as_f32x8(); + let r = vcmpps256(a, b, IMM8, k1 as i8); + transmute(r) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_ps_mask&expand=745) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm_cmp_ps_mask(a: __m128, b: __m128) -> __mmask8 { + static_assert_imm5!(IMM8); + let neg_one = -1; + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let r = vcmpps128(a, b, IMM8, neg_one); + transmute(r) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_ps_mask&expand=746) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm_mask_cmp_ps_mask( + k1: __mmask8, + a: __m128, + b: __m128, +) -> __mmask8 { + static_assert_imm5!(IMM8); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let r = vcmpps128(a, b, IMM8, k1 as i8); + transmute(r) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b based on the comparison operand specified by imm8, and store the results in mask vector k.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_round_ps_mask&expand=753) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp, IMM5 = 0, SAE = 4))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm512_cmp_round_ps_mask( + a: __m512, + b: __m512, +) -> __mmask16 { + static_assert_imm5!(IMM5); + static_assert_mantissas_sae!(SAE); + let neg_one = -1; + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vcmpps(a, b, IMM5, neg_one, SAE); + transmute(r) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_round_ps_mask&expand=754) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp, IMM5 = 0, SAE = 4))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm512_mask_cmp_round_ps_mask( + m: __mmask16, + a: __m512, + b: __m512, +) -> __mmask16 { + static_assert_imm5!(IMM5); + static_assert_mantissas_sae!(SAE); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r = vcmpps(a, b, IMM5, m as i16, SAE); + transmute(r) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b to see if neither is NaN, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpord_ps_mask&expand=1162) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmps +pub unsafe fn _mm512_cmpord_ps_mask(a: __m512, b: __m512) -> __mmask16 { + _mm512_cmp_ps_mask::<_CMP_ORD_Q>(a, b) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b to see if neither is NaN, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpord_ps_mask&expand=1163) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps +pub unsafe fn _mm512_mask_cmpord_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> __mmask16 { + _mm512_mask_cmp_ps_mask::<_CMP_ORD_Q>(k1, a, b) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b to see if either is NaN, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpunord_ps_mask&expand=1170) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps +pub unsafe fn _mm512_cmpunord_ps_mask(a: __m512, b: __m512) -> __mmask16 { + _mm512_cmp_ps_mask::<_CMP_UNORD_Q>(a, b) +} + +/// Compare packed single-precision (32-bit) floating-point elements in a and b to see if either is NaN, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpunord_ps_mask&expand=1171) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps +pub unsafe fn _mm512_mask_cmpunord_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> __mmask16 { + _mm512_mask_cmp_ps_mask::<_CMP_UNORD_Q>(k1, a, b) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmplt_pd_mask&expand=1071) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd +pub unsafe fn _mm512_cmplt_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { + _mm512_cmp_pd_mask::<_CMP_LT_OS>(a, b) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmplt_pd_mask&expand=1072) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd +pub unsafe fn _mm512_mask_cmplt_pd_mask(k1: __mmask8, a: __m512d, b: __m512d) -> __mmask8 { + _mm512_mask_cmp_pd_mask::<_CMP_LT_OS>(k1, a, b) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b for not-less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpnlt_pd_mask&expand=1151) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd +pub unsafe fn _mm512_cmpnlt_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { + _mm512_cmp_pd_mask::<_CMP_NLT_US>(a, b) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b for not-less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpnlt_pd_mask&expand=1152) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd +pub unsafe fn _mm512_mask_cmpnlt_pd_mask(m: __mmask8, a: __m512d, b: __m512d) -> __mmask8 { + _mm512_mask_cmp_pd_mask::<_CMP_NLT_US>(m, a, b) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmple_pd_mask&expand=1010) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd +pub unsafe fn _mm512_cmple_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { + _mm512_cmp_pd_mask::<_CMP_LE_OS>(a, b) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmple_pd_mask&expand=1011) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd +pub unsafe fn _mm512_mask_cmple_pd_mask(k1: __mmask8, a: __m512d, b: __m512d) -> __mmask8 { + _mm512_mask_cmp_pd_mask::<_CMP_LE_OS>(k1, a, b) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b for not-less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpnle_pd_mask&expand=1143) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd +pub unsafe fn _mm512_cmpnle_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { + _mm512_cmp_pd_mask::<_CMP_NLE_US>(a, b) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b for not-less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpnle_pd_mask&expand=1144) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd +pub unsafe fn _mm512_mask_cmpnle_pd_mask(k1: __mmask8, a: __m512d, b: __m512d) -> __mmask8 { + _mm512_mask_cmp_pd_mask::<_CMP_NLE_US>(k1, a, b) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpeq_pd_mask&expand=822) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd +pub unsafe fn _mm512_cmpeq_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { + _mm512_cmp_pd_mask::<_CMP_EQ_OQ>(a, b) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpeq_pd_mask&expand=823) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd +pub unsafe fn _mm512_mask_cmpeq_pd_mask(k1: __mmask8, a: __m512d, b: __m512d) -> __mmask8 { + _mm512_mask_cmp_pd_mask::<_CMP_EQ_OQ>(k1, a, b) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpneq_pd_mask&expand=1127) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd +pub unsafe fn _mm512_cmpneq_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { + _mm512_cmp_pd_mask::<_CMP_NEQ_UQ>(a, b) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpneq_pd_mask&expand=1128) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd +pub unsafe fn _mm512_mask_cmpneq_pd_mask(k1: __mmask8, a: __m512d, b: __m512d) -> __mmask8 { + _mm512_mask_cmp_pd_mask::<_CMP_NEQ_UQ>(k1, a, b) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_pd_mask&expand=741) +#[inline] +#[target_feature(enable = "avx512f")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm512_cmp_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { + static_assert_imm5!(IMM8); + let neg_one = -1; + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vcmppd(a, b, IMM8, neg_one, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_pd_mask&expand=742) +#[inline] +#[target_feature(enable = "avx512f")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm512_mask_cmp_pd_mask( + k1: __mmask8, + a: __m512d, + b: __m512d, +) -> __mmask8 { + static_assert_imm5!(IMM8); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vcmppd(a, b, IMM8, k1 as i8, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_pd_mask&expand=739) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm256_cmp_pd_mask(a: __m256d, b: __m256d) -> __mmask8 { + static_assert_imm5!(IMM8); + let neg_one = -1; + let a = a.as_f64x4(); + let b = b.as_f64x4(); + let r = vcmppd256(a, b, IMM8, neg_one); + transmute(r) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmp_pd_mask&expand=740) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm256_mask_cmp_pd_mask( + k1: __mmask8, + a: __m256d, + b: __m256d, +) -> __mmask8 { + static_assert_imm5!(IMM8); + let a = a.as_f64x4(); + let b = b.as_f64x4(); + let r = vcmppd256(a, b, IMM8, k1 as i8); + transmute(r) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_pd_mask&expand=737) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm_cmp_pd_mask(a: __m128d, b: __m128d) -> __mmask8 { + static_assert_imm5!(IMM8); + let neg_one = -1; + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let r = vcmppd128(a, b, IMM8, neg_one); + transmute(r) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_pd_mask&expand=738) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm_mask_cmp_pd_mask( + k1: __mmask8, + a: __m128d, + b: __m128d, +) -> __mmask8 { + static_assert_imm5!(IMM8); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let r = vcmppd128(a, b, IMM8, k1 as i8); + transmute(r) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b based on the comparison operand specified by imm8, and store the results in mask vector k.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_round_pd_mask&expand=751) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp, IMM5 = 0, SAE = 4))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm512_cmp_round_pd_mask( + a: __m512d, + b: __m512d, +) -> __mmask8 { + static_assert_imm5!(IMM5); + static_assert_mantissas_sae!(SAE); + let neg_one = -1; + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vcmppd(a, b, IMM5, neg_one, SAE); + transmute(r) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_round_pd_mask&expand=752) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp, IMM5 = 0, SAE = 4))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm512_mask_cmp_round_pd_mask( + k1: __mmask8, + a: __m512d, + b: __m512d, +) -> __mmask8 { + static_assert_imm5!(IMM5); + static_assert_mantissas_sae!(SAE); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r = vcmppd(a, b, IMM5, k1 as i8, SAE); + transmute(r) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b to see if neither is NaN, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpord_pd_mask&expand=1159) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd +pub unsafe fn _mm512_cmpord_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { + _mm512_cmp_pd_mask::<_CMP_ORD_Q>(a, b) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b to see if neither is NaN, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpord_pd_mask&expand=1160) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd +pub unsafe fn _mm512_mask_cmpord_pd_mask(k1: __mmask8, a: __m512d, b: __m512d) -> __mmask8 { + _mm512_mask_cmp_pd_mask::<_CMP_ORD_Q>(k1, a, b) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b to see if either is NaN, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpunord_pd_mask&expand=1167) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd +pub unsafe fn _mm512_cmpunord_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { + _mm512_cmp_pd_mask::<_CMP_UNORD_Q>(a, b) +} + +/// Compare packed double-precision (64-bit) floating-point elements in a and b to see if either is NaN, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpunord_pd_mask&expand=1168) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd +pub unsafe fn _mm512_mask_cmpunord_pd_mask(k1: __mmask8, a: __m512d, b: __m512d) -> __mmask8 { + _mm512_mask_cmp_pd_mask::<_CMP_UNORD_Q>(k1, a, b) +} + +/// Compare the lower single-precision (32-bit) floating-point element in a and b based on the comparison operand specified by imm8, and store the result in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_ss_mask&expand=763) +#[inline] +#[target_feature(enable = "avx512f")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm_cmp_ss_mask(a: __m128, b: __m128) -> __mmask8 { + static_assert_imm5!(IMM8); + let neg_one = -1; + let r = vcmpss(a, b, IMM8, neg_one, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Compare the lower single-precision (32-bit) floating-point element in a and b based on the comparison operand specified by imm8, and store the result in mask vector k using zeromask k1 (the element is zeroed out when mask bit 0 is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_ss_mask&expand=764) +#[inline] +#[target_feature(enable = "avx512f")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm_mask_cmp_ss_mask( + k1: __mmask8, + a: __m128, + b: __m128, +) -> __mmask8 { + static_assert_imm5!(IMM8); + let r = vcmpss(a, b, IMM8, k1 as i8, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Compare the lower single-precision (32-bit) floating-point element in a and b based on the comparison operand specified by imm8, and store the result in mask vector k.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_round_ss_mask&expand=757) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp, IMM5 = 0, SAE = 4))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm_cmp_round_ss_mask( + a: __m128, + b: __m128, +) -> __mmask8 { + static_assert_imm5!(IMM5); + static_assert_mantissas_sae!(SAE); + let neg_one = -1; + let r = vcmpss(a, b, IMM5, neg_one, SAE); + transmute(r) +} + +/// Compare the lower single-precision (32-bit) floating-point element in a and b based on the comparison operand specified by imm8, and store the result in mask vector k using zeromask k1 (the element is zeroed out when mask bit 0 is not seti).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_round_ss_mask&expand=758) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp, IMM5 = 0, SAE = 4))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm_mask_cmp_round_ss_mask( + k1: __mmask8, + a: __m128, + b: __m128, +) -> __mmask8 { + static_assert_imm5!(IMM5); + static_assert_mantissas_sae!(SAE); + let r = vcmpss(a, b, IMM5, k1 as i8, SAE); + transmute(r) +} + +/// Compare the lower double-precision (64-bit) floating-point element in a and b based on the comparison operand specified by imm8, and store the result in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_sd_mask&expand=760) +#[inline] +#[target_feature(enable = "avx512f")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm_cmp_sd_mask(a: __m128d, b: __m128d) -> __mmask8 { + static_assert_imm5!(IMM8); + let neg_one = -1; + let r = vcmpsd(a, b, IMM8, neg_one, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Compare the lower double-precision (64-bit) floating-point element in a and b based on the comparison operand specified by imm8, and store the result in mask vector k using zeromask k1 (the element is zeroed out when mask bit 0 is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_sd_mask&expand=761) +#[inline] +#[target_feature(enable = "avx512f")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm_mask_cmp_sd_mask( + k1: __mmask8, + a: __m128d, + b: __m128d, +) -> __mmask8 { + static_assert_imm5!(IMM8); + let r = vcmpsd(a, b, IMM8, k1 as i8, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Compare the lower double-precision (64-bit) floating-point element in a and b based on the comparison operand specified by imm8, and store the result in mask vector k.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_round_sd_mask&expand=755) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp, IMM5 = 0, SAE = 4))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm_cmp_round_sd_mask( + a: __m128d, + b: __m128d, +) -> __mmask8 { + static_assert_imm5!(IMM5); + static_assert_mantissas_sae!(SAE); + let neg_one = -1; + let r = vcmpsd(a, b, IMM5, neg_one, SAE); + transmute(r) +} + +/// Compare the lower double-precision (64-bit) floating-point element in a and b based on the comparison operand specified by imm8, and store the result in mask vector k using zeromask k1 (the element is zeroed out when mask bit 0 is not set).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_round_sd_mask&expand=756) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp, IMM5 = 0, SAE = 4))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm_mask_cmp_round_sd_mask( + k1: __mmask8, + a: __m128d, + b: __m128d, +) -> __mmask8 { + static_assert_imm5!(IMM5); + static_assert_mantissas_sae!(SAE); + let r = vcmpsd(a, b, IMM5, k1 as i8, SAE); + transmute(r) +} + +/// Compare packed unsigned 32-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmplt_epu32_mask&expand=1056) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm512_cmplt_epu32_mask(a: __m512i, b: __m512i) -> __mmask16 { + simd_bitmask::(simd_lt(a.as_u32x16(), b.as_u32x16())) +} + +/// Compare packed unsigned 32-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmplt_epu32_mask&expand=1057) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm512_mask_cmplt_epu32_mask(k1: __mmask16, a: __m512i, b: __m512i) -> __mmask16 { + _mm512_cmplt_epu32_mask(a, b) & k1 +} + +/// Compare packed unsigned 32-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmplt_epu32_mask&expand=1054) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm256_cmplt_epu32_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::(simd_lt(a.as_u32x8(), b.as_u32x8())) +} + +/// Compare packed unsigned 32-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmplt_epu32_mask&expand=1055) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm256_mask_cmplt_epu32_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmplt_epu32_mask(a, b) & k1 +} + +/// Compare packed unsigned 32-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmplt_epu32_mask&expand=1052) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm_cmplt_epu32_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_lt(a.as_u32x4(), b.as_u32x4())) +} + +/// Compare packed unsigned 32-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmplt_epu32_mask&expand=1053) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm_mask_cmplt_epu32_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmplt_epu32_mask(a, b) & k1 +} + +/// Compare packed unsigned 32-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpgt_epu32_mask&expand=933) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm512_cmpgt_epu32_mask(a: __m512i, b: __m512i) -> __mmask16 { + simd_bitmask::(simd_gt(a.as_u32x16(), b.as_u32x16())) +} + +/// Compare packed unsigned 32-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpgt_epu32_mask&expand=934) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm512_mask_cmpgt_epu32_mask(k1: __mmask16, a: __m512i, b: __m512i) -> __mmask16 { + _mm512_cmpgt_epu32_mask(a, b) & k1 +} + +/// Compare packed unsigned 32-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpgt_epu32_mask&expand=931) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm256_cmpgt_epu32_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::(simd_gt(a.as_u32x8(), b.as_u32x8())) +} + +/// Compare packed unsigned 32-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpgt_epu32_mask&expand=932) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm256_mask_cmpgt_epu32_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmpgt_epu32_mask(a, b) & k1 +} + +/// Compare packed unsigned 32-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpgt_epu32_mask&expand=929) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm_cmpgt_epu32_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_gt(a.as_u32x4(), b.as_u32x4())) +} + +/// Compare packed unsigned 32-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpgt_epu32_mask&expand=930) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm_mask_cmpgt_epu32_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpgt_epu32_mask(a, b) & k1 +} + +/// Compare packed unsigned 32-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmple_epu32_mask&expand=995) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm512_cmple_epu32_mask(a: __m512i, b: __m512i) -> __mmask16 { + simd_bitmask::(simd_le(a.as_u32x16(), b.as_u32x16())) +} + +/// Compare packed unsigned 32-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmple_epu32_mask&expand=996) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm512_mask_cmple_epu32_mask(k1: __mmask16, a: __m512i, b: __m512i) -> __mmask16 { + _mm512_cmple_epu32_mask(a, b) & k1 +} + +/// Compare packed unsigned 32-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmple_epu32_mask&expand=993) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm256_cmple_epu32_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::(simd_le(a.as_u32x8(), b.as_u32x8())) +} + +/// Compare packed unsigned 32-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmple_epu32_mask&expand=994) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm256_mask_cmple_epu32_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmple_epu32_mask(a, b) & k1 +} + +/// Compare packed unsigned 32-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmple_epu32_mask&expand=991) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm_cmple_epu32_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_le(a.as_u32x4(), b.as_u32x4())) +} + +/// Compare packed unsigned 32-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmple_epu32_mask&expand=992) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm_mask_cmple_epu32_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmple_epu32_mask(a, b) & k1 +} + +/// Compare packed unsigned 32-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpge_epu32_mask&expand=873) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm512_cmpge_epu32_mask(a: __m512i, b: __m512i) -> __mmask16 { + simd_bitmask::(simd_ge(a.as_u32x16(), b.as_u32x16())) +} + +/// Compare packed unsigned 32-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpge_epu32_mask&expand=874) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm512_mask_cmpge_epu32_mask(k1: __mmask16, a: __m512i, b: __m512i) -> __mmask16 { + _mm512_cmpge_epu32_mask(a, b) & k1 +} + +/// Compare packed unsigned 32-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpge_epu32_mask&expand=871) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm256_cmpge_epu32_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::(simd_ge(a.as_u32x8(), b.as_u32x8())) +} + +/// Compare packed unsigned 32-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpge_epu32_mask&expand=872) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm256_mask_cmpge_epu32_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmpge_epu32_mask(a, b) & k1 +} + +/// Compare packed unsigned 32-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpge_epu32_mask&expand=869) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm_cmpge_epu32_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_ge(a.as_u32x4(), b.as_u32x4())) +} + +/// Compare packed unsigned 32-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpge_epu32_mask&expand=870) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm_mask_cmpge_epu32_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpge_epu32_mask(a, b) & k1 +} + +/// Compare packed unsigned 32-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpeq_epu32_mask&expand=807) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm512_cmpeq_epu32_mask(a: __m512i, b: __m512i) -> __mmask16 { + simd_bitmask::(simd_eq(a.as_u32x16(), b.as_u32x16())) +} + +/// Compare packed unsigned 32-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpeq_epu32_mask&expand=808) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm512_mask_cmpeq_epu32_mask(k1: __mmask16, a: __m512i, b: __m512i) -> __mmask16 { + _mm512_cmpeq_epu32_mask(a, b) & k1 +} + +/// Compare packed unsigned 32-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpeq_epu32_mask&expand=805) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm256_cmpeq_epu32_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::(simd_eq(a.as_u32x8(), b.as_u32x8())) +} + +/// Compare packed unsigned 32-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpeq_epu32_mask&expand=806) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm256_mask_cmpeq_epu32_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmpeq_epu32_mask(a, b) & k1 +} + +/// Compare packed unsigned 32-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpeq_epu32_mask&expand=803) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm_cmpeq_epu32_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_eq(a.as_u32x4(), b.as_u32x4())) +} + +/// Compare packed unsigned 32-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpeq_epu32_mask&expand=804) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm_mask_cmpeq_epu32_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpeq_epu32_mask(a, b) & k1 +} + +/// Compare packed unsigned 32-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpneq_epu32_mask&expand=1112) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm512_cmpneq_epu32_mask(a: __m512i, b: __m512i) -> __mmask16 { + simd_bitmask::(simd_ne(a.as_u32x16(), b.as_u32x16())) +} + +/// Compare packed unsigned 32-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpneq_epu32_mask&expand=1113) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm512_mask_cmpneq_epu32_mask(k1: __mmask16, a: __m512i, b: __m512i) -> __mmask16 { + _mm512_cmpneq_epu32_mask(a, b) & k1 +} + +/// Compare packed unsigned 32-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpneq_epu32_mask&expand=1110) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm256_cmpneq_epu32_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::(simd_ne(a.as_u32x8(), b.as_u32x8())) +} + +/// Compare packed unsigned 32-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpneq_epu32_mask&expand=1111) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm256_mask_cmpneq_epu32_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmpneq_epu32_mask(a, b) & k1 +} + +/// Compare packed unsigned 32-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpneq_epu32_mask&expand=1108) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm_cmpneq_epu32_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_ne(a.as_u32x4(), b.as_u32x4())) +} + +/// Compare packed unsigned 32-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpneq_epu32_mask&expand=1109) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpud +pub unsafe fn _mm_mask_cmpneq_epu32_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpneq_epu32_mask(a, b) & k1 +} + +/// Compare packed unsigned 32-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_epu32_mask&expand=721) +#[inline] +#[target_feature(enable = "avx512f")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm512_cmp_epu32_mask( + a: __m512i, + b: __m512i, +) -> __mmask16 { + static_assert_imm3!(IMM3); + let neg_one = -1; + let a = a.as_i32x16(); + let b = b.as_i32x16(); + let r = vpcmpud(a, b, IMM3, neg_one); + transmute(r) +} + +/// Compare packed unsigned 32-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_epu32_mask&expand=722) +#[inline] +#[target_feature(enable = "avx512f")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm512_mask_cmp_epu32_mask( + k1: __mmask16, + a: __m512i, + b: __m512i, +) -> __mmask16 { + static_assert_imm3!(IMM3); + let a = a.as_i32x16(); + let b = b.as_i32x16(); + let r = vpcmpud(a, b, IMM3, k1 as i16); + transmute(r) +} + +/// Compare packed unsigned 32-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_epu32_mask&expand=719) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm256_cmp_epu32_mask( + a: __m256i, + b: __m256i, +) -> __mmask8 { + static_assert_imm3!(IMM3); + let neg_one = -1; + let a = a.as_i32x8(); + let b = b.as_i32x8(); + let r = vpcmpud256(a, b, IMM3, neg_one); + transmute(r) +} + +/// Compare packed unsigned 32-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmp_epu32_mask&expand=720) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm256_mask_cmp_epu32_mask( + k1: __mmask8, + a: __m256i, + b: __m256i, +) -> __mmask8 { + static_assert_imm3!(IMM3); + let a = a.as_i32x8(); + let b = b.as_i32x8(); + let r = vpcmpud256(a, b, IMM3, k1 as i8); + transmute(r) +} + +/// Compare packed unsigned 32-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_epu32_mask&expand=717) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm_cmp_epu32_mask(a: __m128i, b: __m128i) -> __mmask8 { + static_assert_imm3!(IMM3); + let neg_one = -1; + let a = a.as_i32x4(); + let b = b.as_i32x4(); + let r = vpcmpud128(a, b, IMM3, neg_one); + transmute(r) +} + +/// Compare packed unsigned 32-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_epu32_mask&expand=718) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm_mask_cmp_epu32_mask( + k1: __mmask8, + a: __m128i, + b: __m128i, +) -> __mmask8 { + static_assert_imm3!(IMM3); + let a = a.as_i32x4(); + let b = b.as_i32x4(); + let r = vpcmpud128(a, b, IMM3, k1 as i8); + transmute(r) +} + +/// Compare packed signed 32-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmplt_epi32_mask&expand=1029) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm512_cmplt_epi32_mask(a: __m512i, b: __m512i) -> __mmask16 { + simd_bitmask::(simd_lt(a.as_i32x16(), b.as_i32x16())) +} + +/// Compare packed signed 32-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmplt_epi32_mask&expand=1031) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm512_mask_cmplt_epi32_mask(k1: __mmask16, a: __m512i, b: __m512i) -> __mmask16 { + _mm512_cmplt_epi32_mask(a, b) & k1 +} + +/// Compare packed signed 32-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmplt_epi32_mask&expand=1027) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm256_cmplt_epi32_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::(simd_lt(a.as_i32x8(), b.as_i32x8())) +} + +/// Compare packed signed 32-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmplt_epi32_mask&expand=1028) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm256_mask_cmplt_epi32_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmplt_epi32_mask(a, b) & k1 +} + +/// Compare packed signed 32-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmplt_epi32_mask&expand=1025) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm_cmplt_epi32_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_lt(a.as_i32x4(), b.as_i32x4())) +} + +/// Compare packed signed 32-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmplt_epi32_mask&expand=1026) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm_mask_cmplt_epi32_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmplt_epi32_mask(a, b) & k1 +} + +/// Compare packed signed 32-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpgt_epi32_mask&expand=905) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm512_cmpgt_epi32_mask(a: __m512i, b: __m512i) -> __mmask16 { + simd_bitmask::(simd_gt(a.as_i32x16(), b.as_i32x16())) +} + +/// Compare packed signed 32-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpgt_epi32_mask&expand=906) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm512_mask_cmpgt_epi32_mask(k1: __mmask16, a: __m512i, b: __m512i) -> __mmask16 { + _mm512_cmpgt_epi32_mask(a, b) & k1 +} + +/// Compare packed signed 32-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpgt_epi32_mask&expand=903) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm256_cmpgt_epi32_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::(simd_gt(a.as_i32x8(), b.as_i32x8())) +} + +/// Compare packed signed 32-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpgt_epi32_mask&expand=904) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm256_mask_cmpgt_epi32_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmpgt_epi32_mask(a, b) & k1 +} + +/// Compare packed signed 32-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpgt_epi32_mask&expand=901) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm_cmpgt_epi32_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_gt(a.as_i32x4(), b.as_i32x4())) +} + +/// Compare packed signed 32-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpgt_epi32_mask&expand=902) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm_mask_cmpgt_epi32_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpgt_epi32_mask(a, b) & k1 +} + +/// Compare packed signed 32-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmple_epi32_mask&expand=971) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm512_cmple_epi32_mask(a: __m512i, b: __m512i) -> __mmask16 { + simd_bitmask::(simd_le(a.as_i32x16(), b.as_i32x16())) +} + +/// Compare packed signed 32-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmple_epi32_mask&expand=972) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm512_mask_cmple_epi32_mask(k1: __mmask16, a: __m512i, b: __m512i) -> __mmask16 { + _mm512_cmple_epi32_mask(a, b) & k1 +} + +/// Compare packed signed 32-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmple_epi32_mask&expand=969) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm256_cmple_epi32_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::(simd_le(a.as_i32x8(), b.as_i32x8())) +} + +/// Compare packed signed 32-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmple_epi32_mask&expand=970) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm256_mask_cmple_epi32_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmple_epi32_mask(a, b) & k1 +} + +/// Compare packed signed 32-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmple_epi32_mask&expand=967) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm_cmple_epi32_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_le(a.as_i32x4(), b.as_i32x4())) +} + +/// Compare packed signed 32-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmple_epi32_mask&expand=968) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm_mask_cmple_epi32_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmple_epi32_mask(a, b) & k1 +} + +/// Compare packed signed 32-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpge_epi32_mask&expand=849) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm512_cmpge_epi32_mask(a: __m512i, b: __m512i) -> __mmask16 { + simd_bitmask::(simd_ge(a.as_i32x16(), b.as_i32x16())) +} + +/// Compare packed signed 32-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpge_epi32_mask&expand=850) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm512_mask_cmpge_epi32_mask(k1: __mmask16, a: __m512i, b: __m512i) -> __mmask16 { + _mm512_cmpge_epi32_mask(a, b) & k1 +} + +/// Compare packed signed 32-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpge_epi32_mask&expand=847) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm256_cmpge_epi32_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::(simd_ge(a.as_i32x8(), b.as_i32x8())) +} + +/// Compare packed signed 32-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpge_epi32_mask&expand=848) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm256_mask_cmpge_epi32_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmpge_epi32_mask(a, b) & k1 +} + +/// Compare packed signed 32-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpge_epi32_mask&expand=845) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm_cmpge_epi32_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_ge(a.as_i32x4(), b.as_i32x4())) +} + +/// Compare packed signed 32-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpge_epi32_mask&expand=846) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm_mask_cmpge_epi32_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpge_epi32_mask(a, b) & k1 +} + +/// Compare packed 32-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpeq_epi32_mask&expand=779) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm512_cmpeq_epi32_mask(a: __m512i, b: __m512i) -> __mmask16 { + simd_bitmask::(simd_eq(a.as_i32x16(), b.as_i32x16())) +} + +/// Compare packed 32-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpeq_epi32_mask&expand=780) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm512_mask_cmpeq_epi32_mask(k1: __mmask16, a: __m512i, b: __m512i) -> __mmask16 { + _mm512_cmpeq_epi32_mask(a, b) & k1 +} + +/// Compare packed 32-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpeq_epi32_mask&expand=777) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm256_cmpeq_epi32_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::(simd_eq(a.as_i32x8(), b.as_i32x8())) +} + +/// Compare packed 32-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpeq_epi32_mask&expand=778) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm256_mask_cmpeq_epi32_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmpeq_epi32_mask(a, b) & k1 +} + +/// Compare packed 32-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpeq_epi32_mask&expand=775) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm_cmpeq_epi32_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_eq(a.as_i32x4(), b.as_i32x4())) +} + +/// Compare packed 32-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpeq_epi32_mask&expand=776) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm_mask_cmpeq_epi32_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpeq_epi32_mask(a, b) & k1 +} + +/// Compare packed 32-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpneq_epi32_mask&expand=1088) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm512_cmpneq_epi32_mask(a: __m512i, b: __m512i) -> __mmask16 { + simd_bitmask::(simd_ne(a.as_i32x16(), b.as_i32x16())) +} + +/// Compare packed 32-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpneq_epi32_mask&expand=1089) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm512_mask_cmpneq_epi32_mask(k1: __mmask16, a: __m512i, b: __m512i) -> __mmask16 { + _mm512_cmpneq_epi32_mask(a, b) & k1 +} + +/// Compare packed 32-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpneq_epi32_mask&expand=1086) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm256_cmpneq_epi32_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::(simd_ne(a.as_i32x8(), b.as_i32x8())) +} + +/// Compare packed 32-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpneq_epi32_mask&expand=1087) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm256_mask_cmpneq_epi32_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmpneq_epi32_mask(a, b) & k1 +} + +/// Compare packed 32-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpneq_epi32_mask&expand=1084) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm_cmpneq_epi32_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::(simd_ne(a.as_i32x4(), b.as_i32x4())) +} + +/// Compare packed 32-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpneq_epi32_mask&expand=1085) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpd +pub unsafe fn _mm_mask_cmpneq_epi32_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpneq_epi32_mask(a, b) & k1 +} + +/// Compare packed signed 32-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_epi32_mask&expand=697) +#[inline] +#[target_feature(enable = "avx512f")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm512_cmp_epi32_mask( + a: __m512i, + b: __m512i, +) -> __mmask16 { + static_assert_imm3!(IMM3); + let neg_one = -1; + let a = a.as_i32x16(); + let b = b.as_i32x16(); + let r = vpcmpd(a, b, IMM3, neg_one); + transmute(r) +} + +/// Compare packed signed 32-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_epi32_mask&expand=698) +#[inline] +#[target_feature(enable = "avx512f")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm512_mask_cmp_epi32_mask( + k1: __mmask16, + a: __m512i, + b: __m512i, +) -> __mmask16 { + static_assert_imm3!(IMM3); + let a = a.as_i32x16(); + let b = b.as_i32x16(); + let r = vpcmpd(a, b, IMM3, k1 as i16); + transmute(r) +} + +/// Compare packed signed 32-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=#text=_mm256_cmp_epi32_mask&expand=695) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm256_cmp_epi32_mask( + a: __m256i, + b: __m256i, +) -> __mmask8 { + static_assert_imm3!(IMM3); + let neg_one = -1; + let a = a.as_i32x8(); + let b = b.as_i32x8(); + let r = vpcmpd256(a, b, IMM3, neg_one); + transmute(r) +} + +/// Compare packed signed 32-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmp_epi32_mask&expand=696) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm256_mask_cmp_epi32_mask( + k1: __mmask8, + a: __m256i, + b: __m256i, +) -> __mmask8 { + static_assert_imm3!(IMM3); + let a = a.as_i32x8(); + let b = b.as_i32x8(); + let r = vpcmpd256(a, b, IMM3, k1 as i8); + transmute(r) +} + +/// Compare packed signed 32-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_epi32_mask&expand=693) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm_cmp_epi32_mask(a: __m128i, b: __m128i) -> __mmask8 { + static_assert_imm3!(IMM3); + let neg_one = -1; + let a = a.as_i32x4(); + let b = b.as_i32x4(); + let r = vpcmpd128(a, b, IMM3, neg_one); + transmute(r) +} + +/// Compare packed signed 32-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_epi32_mask&expand=694) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm_mask_cmp_epi32_mask( + k1: __mmask8, + a: __m128i, + b: __m128i, +) -> __mmask8 { + static_assert_imm3!(IMM3); + let a = a.as_i32x4(); + let b = b.as_i32x4(); + let r = vpcmpd128(a, b, IMM3, k1 as i8); + transmute(r) +} + +/// Compare packed unsigned 64-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmplt_epu64_mask&expand=1062) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm512_cmplt_epu64_mask(a: __m512i, b: __m512i) -> __mmask8 { + simd_bitmask::<__m512i, _>(simd_lt(a.as_u64x8(), b.as_u64x8())) +} + +/// Compare packed unsigned 64-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmplt_epu64_mask&expand=1063) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm512_mask_cmplt_epu64_mask(k1: __mmask8, a: __m512i, b: __m512i) -> __mmask8 { + _mm512_cmplt_epu64_mask(a, b) & k1 +} + +/// Compare packed unsigned 64-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmplt_epu64_mask&expand=1060) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm256_cmplt_epu64_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::<__m256i, _>(simd_lt(a.as_u64x4(), b.as_u64x4())) +} + +/// Compare packed unsigned 64-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmplt_epu64_mask&expand=1061) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm256_mask_cmplt_epu64_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmplt_epu64_mask(a, b) & k1 +} + +/// Compare packed unsigned 64-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmplt_epu64_mask&expand=1058) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm_cmplt_epu64_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::<__m128i, _>(simd_lt(a.as_u64x2(), b.as_u64x2())) +} + +/// Compare packed unsigned 64-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmplt_epu64_mask&expand=1059) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm_mask_cmplt_epu64_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmplt_epu64_mask(a, b) & k1 +} + +/// Compare packed unsigned 64-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpgt_epu64_mask&expand=939) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm512_cmpgt_epu64_mask(a: __m512i, b: __m512i) -> __mmask8 { + simd_bitmask::<__m512i, _>(simd_gt(a.as_u64x8(), b.as_u64x8())) +} + +/// Compare packed unsigned 64-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpgt_epu64_mask&expand=940) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm512_mask_cmpgt_epu64_mask(k1: __mmask8, a: __m512i, b: __m512i) -> __mmask8 { + _mm512_cmpgt_epu64_mask(a, b) & k1 +} + +/// Compare packed unsigned 64-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpgt_epu64_mask&expand=937) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm256_cmpgt_epu64_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::<__m256i, _>(simd_gt(a.as_u64x4(), b.as_u64x4())) +} + +/// Compare packed unsigned 64-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpgt_epu64_mask&expand=938) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm256_mask_cmpgt_epu64_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmpgt_epu64_mask(a, b) & k1 +} + +/// Compare packed unsigned 64-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpgt_epu64_mask&expand=935) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm_cmpgt_epu64_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::<__m128i, _>(simd_gt(a.as_u64x2(), b.as_u64x2())) +} + +/// Compare packed unsigned 64-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpgt_epu64_mask&expand=936) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm_mask_cmpgt_epu64_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpgt_epu64_mask(a, b) & k1 +} + +/// Compare packed unsigned 64-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmple_epu64_mask&expand=1001) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm512_cmple_epu64_mask(a: __m512i, b: __m512i) -> __mmask8 { + simd_bitmask::<__m512i, _>(simd_le(a.as_u64x8(), b.as_u64x8())) +} + +/// Compare packed unsigned 64-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmple_epu64_mask&expand=1002) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm512_mask_cmple_epu64_mask(k1: __mmask8, a: __m512i, b: __m512i) -> __mmask8 { + _mm512_cmple_epu64_mask(a, b) & k1 +} + +/// Compare packed unsigned 64-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmple_epu64_mask&expand=999) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm256_cmple_epu64_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::<__m256i, _>(simd_le(a.as_u64x4(), b.as_u64x4())) +} + +/// Compare packed unsigned 64-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmple_epu64_mask&expand=1000) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm256_mask_cmple_epu64_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmple_epu64_mask(a, b) & k1 +} + +/// Compare packed unsigned 64-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmple_epu64_mask&expand=997) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm_cmple_epu64_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::<__m128i, _>(simd_le(a.as_u64x2(), b.as_u64x2())) +} + +/// Compare packed unsigned 64-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmple_epu64_mask&expand=998) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm_mask_cmple_epu64_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmple_epu64_mask(a, b) & k1 +} + +/// Compare packed unsigned 64-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpge_epu64_mask&expand=879) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm512_cmpge_epu64_mask(a: __m512i, b: __m512i) -> __mmask8 { + simd_bitmask::<__m512i, _>(simd_ge(a.as_u64x8(), b.as_u64x8())) +} + +/// Compare packed unsigned 64-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpge_epu64_mask&expand=880) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm512_mask_cmpge_epu64_mask(k1: __mmask8, a: __m512i, b: __m512i) -> __mmask8 { + _mm512_cmpge_epu64_mask(a, b) & k1 +} + +/// Compare packed unsigned 64-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpge_epu64_mask&expand=877) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm256_cmpge_epu64_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::<__m256i, _>(simd_ge(a.as_u64x4(), b.as_u64x4())) +} + +/// Compare packed unsigned 64-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpge_epu64_mask&expand=878) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm256_mask_cmpge_epu64_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmpge_epu64_mask(a, b) & k1 +} + +/// Compare packed unsigned 64-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpge_epu64_mask&expand=875) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm_cmpge_epu64_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::<__m128i, _>(simd_ge(a.as_u64x2(), b.as_u64x2())) +} + +/// Compare packed unsigned 64-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpge_epu64_mask&expand=876) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm_mask_cmpge_epu64_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpge_epu64_mask(a, b) & k1 +} + +/// Compare packed unsigned 64-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpeq_epu64_mask&expand=813) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm512_cmpeq_epu64_mask(a: __m512i, b: __m512i) -> __mmask8 { + simd_bitmask::<__m512i, _>(simd_eq(a.as_u64x8(), b.as_u64x8())) +} + +/// Compare packed unsigned 64-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpeq_epu64_mask&expand=814) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm512_mask_cmpeq_epu64_mask(k1: __mmask8, a: __m512i, b: __m512i) -> __mmask8 { + _mm512_cmpeq_epu64_mask(a, b) & k1 +} + +/// Compare packed unsigned 64-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpeq_epu64_mask&expand=811) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm256_cmpeq_epu64_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::<__m256i, _>(simd_eq(a.as_u64x4(), b.as_u64x4())) +} + +/// Compare packed unsigned 64-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpeq_epu64_mask&expand=812) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm256_mask_cmpeq_epu64_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmpeq_epu64_mask(a, b) & k1 +} + +/// Compare packed unsigned 64-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpeq_epu64_mask&expand=809) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm_cmpeq_epu64_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::<__m128i, _>(simd_eq(a.as_u64x2(), b.as_u64x2())) +} + +/// Compare packed unsigned 64-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpeq_epu64_mask&expand=810) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm_mask_cmpeq_epu64_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpeq_epu64_mask(a, b) & k1 +} + +/// Compare packed unsigned 64-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpneq_epu64_mask&expand=1118) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm512_cmpneq_epu64_mask(a: __m512i, b: __m512i) -> __mmask8 { + simd_bitmask::<__m512i, _>(simd_ne(a.as_u64x8(), b.as_u64x8())) +} + +/// Compare packed unsigned 64-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpneq_epu64_mask&expand=1119) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm512_mask_cmpneq_epu64_mask(k1: __mmask8, a: __m512i, b: __m512i) -> __mmask8 { + _mm512_cmpneq_epu64_mask(a, b) & k1 +} + +/// Compare packed unsigned 64-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpneq_epu64_mask&expand=1116) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm256_cmpneq_epu64_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::<__m256i, _>(simd_ne(a.as_u64x4(), b.as_u64x4())) +} + +/// Compare packed unsigned 64-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpneq_epu64_mask&expand=1117) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm256_mask_cmpneq_epu64_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmpneq_epu64_mask(a, b) & k1 +} + +/// Compare packed unsigned 64-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpneq_epu64_mask&expand=1114) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm_cmpneq_epu64_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::<__m128i, _>(simd_ne(a.as_u64x2(), b.as_u64x2())) +} + +/// Compare packed unsigned 64-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpneq_epu64_mask&expand=1115) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpuq +pub unsafe fn _mm_mask_cmpneq_epu64_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpneq_epu64_mask(a, b) & k1 +} + +/// Compare packed unsigned 64-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_epu64_mask&expand=727) +#[inline] +#[target_feature(enable = "avx512f")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm512_cmp_epu64_mask( + a: __m512i, + b: __m512i, +) -> __mmask8 { + static_assert_imm3!(IMM3); + let neg_one = -1; + let a = a.as_i64x8(); + let b = b.as_i64x8(); + let r = vpcmpuq(a, b, IMM3, neg_one); + transmute(r) +} + +/// Compare packed unsigned 64-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_epu64_mask&expand=728) +#[inline] +#[target_feature(enable = "avx512f")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm512_mask_cmp_epu64_mask( + k1: __mmask8, + a: __m512i, + b: __m512i, +) -> __mmask8 { + static_assert_imm3!(IMM3); + let a = a.as_i64x8(); + let b = b.as_i64x8(); + let r = vpcmpuq(a, b, IMM3, k1 as i8); + transmute(r) +} + +/// Compare packed unsigned 64-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_epu64_mask&expand=725) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm256_cmp_epu64_mask( + a: __m256i, + b: __m256i, +) -> __mmask8 { + static_assert_imm3!(IMM3); + let neg_one = -1; + let a = a.as_i64x4(); + let b = b.as_i64x4(); + let r = vpcmpuq256(a, b, IMM3, neg_one); + transmute(r) +} + +/// Compare packed unsigned 64-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmp_epu64_mask&expand=726) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm256_mask_cmp_epu64_mask( + k1: __mmask8, + a: __m256i, + b: __m256i, +) -> __mmask8 { + static_assert_imm3!(IMM3); + let a = a.as_i64x4(); + let b = b.as_i64x4(); + let r = vpcmpuq256(a, b, IMM3, k1 as i8); + transmute(r) +} + +/// Compare packed unsigned 64-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_epu64_mask&expand=723) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm_cmp_epu64_mask(a: __m128i, b: __m128i) -> __mmask8 { + static_assert_imm3!(IMM3); + let neg_one = -1; + let a = a.as_i64x2(); + let b = b.as_i64x2(); + let r = vpcmpuq128(a, b, IMM3, neg_one); + transmute(r) +} + +/// Compare packed unsigned 64-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_epu64_mask&expand=724) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm_mask_cmp_epu64_mask( + k1: __mmask8, + a: __m128i, + b: __m128i, +) -> __mmask8 { + static_assert_imm3!(IMM3); + let a = a.as_i64x2(); + let b = b.as_i64x2(); + let r = vpcmpuq128(a, b, IMM3, k1 as i8); + transmute(r) +} + +/// Compare packed signed 64-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmplt_epi64_mask&expand=1037) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm512_cmplt_epi64_mask(a: __m512i, b: __m512i) -> __mmask8 { + simd_bitmask::<__m512i, _>(simd_lt(a.as_i64x8(), b.as_i64x8())) +} + +/// Compare packed signed 64-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmplt_epi64_mask&expand=1038) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm512_mask_cmplt_epi64_mask(k1: __mmask8, a: __m512i, b: __m512i) -> __mmask8 { + _mm512_cmplt_epi64_mask(a, b) & k1 +} + +/// Compare packed signed 64-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmplt_epi64_mask&expand=1035) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm256_cmplt_epi64_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::<__m256i, _>(simd_lt(a.as_i64x4(), b.as_i64x4())) +} + +/// Compare packed signed 64-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmplt_epi64_mask&expand=1036) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm256_mask_cmplt_epi64_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmplt_epi64_mask(a, b) & k1 +} + +/// Compare packed signed 64-bit integers in a and b for less-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmplt_epi64_mask&expand=1033) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm_cmplt_epi64_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::<__m128i, _>(simd_lt(a.as_i64x2(), b.as_i64x2())) +} + +/// Compare packed signed 64-bit integers in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmplt_epi64_mask&expand=1034) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm_mask_cmplt_epi64_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmplt_epi64_mask(a, b) & k1 +} + +/// Compare packed signed 64-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpgt_epi64_mask&expand=913) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm512_cmpgt_epi64_mask(a: __m512i, b: __m512i) -> __mmask8 { + simd_bitmask::<__m512i, _>(simd_gt(a.as_i64x8(), b.as_i64x8())) +} + +/// Compare packed signed 64-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpgt_epi64_mask&expand=914) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm512_mask_cmpgt_epi64_mask(k1: __mmask8, a: __m512i, b: __m512i) -> __mmask8 { + _mm512_cmpgt_epi64_mask(a, b) & k1 +} + +/// Compare packed signed 64-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpgt_epi64_mask&expand=911) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm256_cmpgt_epi64_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::<__m256i, _>(simd_gt(a.as_i64x4(), b.as_i64x4())) +} + +/// Compare packed signed 64-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpgt_epi64_mask&expand=912) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm256_mask_cmpgt_epi64_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmpgt_epi64_mask(a, b) & k1 +} + +/// Compare packed signed 64-bit integers in a and b for greater-than, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpgt_epi64_mask&expand=909) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm_cmpgt_epi64_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::<__m128i, _>(simd_gt(a.as_i64x2(), b.as_i64x2())) +} + +/// Compare packed signed 64-bit integers in a and b for greater-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpgt_epi64_mask&expand=910) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm_mask_cmpgt_epi64_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpgt_epi64_mask(a, b) & k1 +} + +/// Compare packed signed 64-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmple_epi64_mask&expand=977) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm512_cmple_epi64_mask(a: __m512i, b: __m512i) -> __mmask8 { + simd_bitmask::<__m512i, _>(simd_le(a.as_i64x8(), b.as_i64x8())) +} + +/// Compare packed signed 64-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmple_epi64_mask&expand=978) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm512_mask_cmple_epi64_mask(k1: __mmask8, a: __m512i, b: __m512i) -> __mmask8 { + _mm512_cmple_epi64_mask(a, b) & k1 +} + +/// Compare packed signed 64-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmple_epi64_mask&expand=975) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm256_cmple_epi64_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::<__m256i, _>(simd_le(a.as_i64x4(), b.as_i64x4())) +} + +/// Compare packed signed 64-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmple_epi64_mask&expand=976) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm256_mask_cmple_epi64_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmple_epi64_mask(a, b) & k1 +} + +/// Compare packed signed 64-bit integers in a and b for less-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmple_epi64_mask&expand=973) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm_cmple_epi64_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::<__m128i, _>(simd_le(a.as_i64x2(), b.as_i64x2())) +} + +/// Compare packed signed 64-bit integers in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmple_epi64_mask&expand=974) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm_mask_cmple_epi64_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmple_epi64_mask(a, b) & k1 +} + +/// Compare packed signed 64-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpge_epi64_mask&expand=855) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm512_cmpge_epi64_mask(a: __m512i, b: __m512i) -> __mmask8 { + simd_bitmask::<__m512i, _>(simd_ge(a.as_i64x8(), b.as_i64x8())) +} + +/// Compare packed signed 64-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpge_epi64_mask&expand=856) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm512_mask_cmpge_epi64_mask(k1: __mmask8, a: __m512i, b: __m512i) -> __mmask8 { + _mm512_cmpge_epi64_mask(a, b) & k1 +} + +/// Compare packed signed 64-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpge_epi64_mask&expand=853) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm256_cmpge_epi64_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::<__m256i, _>(simd_ge(a.as_i64x4(), b.as_i64x4())) +} + +/// Compare packed signed 64-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpge_epi64_mask&expand=854) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm256_mask_cmpge_epi64_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmpge_epi64_mask(a, b) & k1 +} + +/// Compare packed signed 64-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpge_epi64_mask&expand=851) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm_cmpge_epi64_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::<__m128i, _>(simd_ge(a.as_i64x2(), b.as_i64x2())) +} + +/// Compare packed signed 64-bit integers in a and b for greater-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpge_epi64_mask&expand=852) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm_mask_cmpge_epi64_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpge_epi64_mask(a, b) & k1 +} + +/// Compare packed 64-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpeq_epi64_mask&expand=787) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm512_cmpeq_epi64_mask(a: __m512i, b: __m512i) -> __mmask8 { + simd_bitmask::<__m512i, _>(simd_eq(a.as_i64x8(), b.as_i64x8())) +} + +/// Compare packed 64-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpeq_epi64_mask&expand=788) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm512_mask_cmpeq_epi64_mask(k1: __mmask8, a: __m512i, b: __m512i) -> __mmask8 { + _mm512_cmpeq_epi64_mask(a, b) & k1 +} + +/// Compare packed 64-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpeq_epi64_mask&expand=785) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm256_cmpeq_epi64_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::<__m256i, _>(simd_eq(a.as_i64x4(), b.as_i64x4())) +} + +/// Compare packed 64-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpeq_epi64_mask&expand=786) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm256_mask_cmpeq_epi64_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmpeq_epi64_mask(a, b) & k1 +} + +/// Compare packed 64-bit integers in a and b for equality, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpeq_epi64_mask&expand=783) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm_cmpeq_epi64_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::<__m128i, _>(simd_eq(a.as_i64x2(), b.as_i64x2())) +} + +/// Compare packed 64-bit integers in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpeq_epi64_mask&expand=784) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm_mask_cmpeq_epi64_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpeq_epi64_mask(a, b) & k1 +} + +/// Compare packed signed 64-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmpneq_epi64_mask&expand=1094) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm512_cmpneq_epi64_mask(a: __m512i, b: __m512i) -> __mmask8 { + simd_bitmask::<__m512i, _>(simd_ne(a.as_i64x8(), b.as_i64x8())) +} + +/// Compare packed signed 64-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmpneq_epi64_mask&expand=1095) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm512_mask_cmpneq_epi64_mask(k1: __mmask8, a: __m512i, b: __m512i) -> __mmask8 { + _mm512_cmpneq_epi64_mask(a, b) & k1 +} + +/// Compare packed signed 64-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmpneq_epi64_mask&expand=1092) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm256_cmpneq_epi64_mask(a: __m256i, b: __m256i) -> __mmask8 { + simd_bitmask::<__m256i, _>(simd_ne(a.as_i64x4(), b.as_i64x4())) +} + +/// Compare packed signed 64-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmpneq_epi64_mask&expand=1093) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm256_mask_cmpneq_epi64_mask(k1: __mmask8, a: __m256i, b: __m256i) -> __mmask8 { + _mm256_cmpneq_epi64_mask(a, b) & k1 +} + +/// Compare packed signed 64-bit integers in a and b for not-equal, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpneq_epi64_mask&expand=1090) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm_cmpneq_epi64_mask(a: __m128i, b: __m128i) -> __mmask8 { + simd_bitmask::<__m128i, _>(simd_ne(a.as_i64x2(), b.as_i64x2())) +} + +/// Compare packed signed 64-bit integers in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmpneq_epi64_mask&expand=1091) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpcmp))] //should be vpcmpq +pub unsafe fn _mm_mask_cmpneq_epi64_mask(k1: __mmask8, a: __m128i, b: __m128i) -> __mmask8 { + _mm_cmpneq_epi64_mask(a, b) & k1 +} + +/// Compare packed signed 64-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_epi64_mask&expand=703) +#[inline] +#[target_feature(enable = "avx512f")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm512_cmp_epi64_mask( + a: __m512i, + b: __m512i, +) -> __mmask8 { + static_assert_imm3!(IMM3); + let neg_one = -1; + let a = a.as_i64x8(); + let b = b.as_i64x8(); + let r = vpcmpq(a, b, IMM3, neg_one); + transmute(r) +} + +/// Compare packed signed 64-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_epi64_mask&expand=704) +#[inline] +#[target_feature(enable = "avx512f")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm512_mask_cmp_epi64_mask( + k1: __mmask8, + a: __m512i, + b: __m512i, +) -> __mmask8 { + static_assert_imm3!(IMM3); + let a = a.as_i64x8(); + let b = b.as_i64x8(); + let r = vpcmpq(a, b, IMM3, k1 as i8); + transmute(r) +} + +/// Compare packed signed 64-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_epi64_mask&expand=701) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm256_cmp_epi64_mask( + a: __m256i, + b: __m256i, +) -> __mmask8 { + static_assert_imm3!(IMM3); + let neg_one = -1; + let a = a.as_i64x4(); + let b = b.as_i64x4(); + let r = vpcmpq256(a, b, IMM3, neg_one); + transmute(r) +} + +/// Compare packed signed 64-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmp_epi64_mask&expand=702) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm256_mask_cmp_epi64_mask( + k1: __mmask8, + a: __m256i, + b: __m256i, +) -> __mmask8 { + static_assert_imm3!(IMM3); + let a = a.as_i64x4(); + let b = b.as_i64x4(); + let r = vpcmpq256(a, b, IMM3, k1 as i8); + transmute(r) +} + +/// Compare packed signed 64-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_epi64_mask&expand=699) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm_cmp_epi64_mask(a: __m128i, b: __m128i) -> __mmask8 { + static_assert_imm3!(IMM3); + let neg_one = -1; + let a = a.as_i64x2(); + let b = b.as_i64x2(); + let r = vpcmpq128(a, b, IMM3, neg_one); + transmute(r) +} + +/// Compare packed signed 64-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_epi64_mask&expand=700) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm_mask_cmp_epi64_mask( + k1: __mmask8, + a: __m128i, + b: __m128i, +) -> __mmask8 { + static_assert_imm3!(IMM3); + let a = a.as_i64x2(); + let b = b.as_i64x2(); + let r = vpcmpq128(a, b, IMM3, k1 as i8); + transmute(r) +} + +/// Reduce the packed 32-bit integers in a by addition. Returns the sum of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_add_epi32&expand=4556) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_add_epi32(a: __m512i) -> i32 { + simd_reduce_add_unordered(a.as_i32x16()) +} + +/// Reduce the packed 32-bit integers in a by addition using mask k. Returns the sum of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_add_epi32&expand=4555) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_add_epi32(k: __mmask16, a: __m512i) -> i32 { + simd_reduce_add_unordered(simd_select_bitmask( + k, + a.as_i32x16(), + _mm512_setzero_si512().as_i32x16(), + )) +} + +/// Reduce the packed 64-bit integers in a by addition. Returns the sum of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_add_epi64&expand=4558) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_add_epi64(a: __m512i) -> i64 { + simd_reduce_add_unordered(a.as_i64x8()) +} + +/// Reduce the packed 64-bit integers in a by addition using mask k. Returns the sum of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_add_epi64&expand=4557) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_add_epi64(k: __mmask8, a: __m512i) -> i64 { + simd_reduce_add_unordered(simd_select_bitmask( + k, + a.as_i64x8(), + _mm512_setzero_si512().as_i64x8(), + )) +} + +/// Reduce the packed single-precision (32-bit) floating-point elements in a by addition. Returns the sum of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_add_ps&expand=4562) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_add_ps(a: __m512) -> f32 { + simd_reduce_add_unordered(a.as_f32x16()) +} + +/// Reduce the packed single-precision (32-bit) floating-point elements in a by addition using mask k. Returns the sum of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_add_ps&expand=4561) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_add_ps(k: __mmask16, a: __m512) -> f32 { + simd_reduce_add_unordered(simd_select_bitmask( + k, + a.as_f32x16(), + _mm512_setzero_ps().as_f32x16(), + )) +} + +/// Reduce the packed double-precision (64-bit) floating-point elements in a by addition. Returns the sum of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_add_pd&expand=4560) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_add_pd(a: __m512d) -> f64 { + simd_reduce_add_unordered(a.as_f64x8()) +} + +/// Reduce the packed double-precision (64-bit) floating-point elements in a by addition using mask k. Returns the sum of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_add_pd&expand=4559) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_add_pd(k: __mmask8, a: __m512d) -> f64 { + simd_reduce_add_unordered(simd_select_bitmask( + k, + a.as_f64x8(), + _mm512_setzero_pd().as_f64x8(), + )) +} + +/// Reduce the packed 32-bit integers in a by multiplication. Returns the product of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_mul_epi32&expand=4600) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_mul_epi32(a: __m512i) -> i32 { + simd_reduce_mul_unordered(a.as_i32x16()) +} + +/// Reduce the packed 32-bit integers in a by multiplication using mask k. Returns the product of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_mul_epi32&expand=4599) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_mul_epi32(k: __mmask16, a: __m512i) -> i32 { + simd_reduce_mul_unordered(simd_select_bitmask( + k, + a.as_i32x16(), + _mm512_set1_epi32(1).as_i32x16(), + )) +} + +/// Reduce the packed 64-bit integers in a by multiplication. Returns the product of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_mul_epi64&expand=4602) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_mul_epi64(a: __m512i) -> i64 { + simd_reduce_mul_unordered(a.as_i64x8()) +} + +/// Reduce the packed 64-bit integers in a by multiplication using mask k. Returns the product of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_mul_epi64&expand=4601) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_mul_epi64(k: __mmask8, a: __m512i) -> i64 { + simd_reduce_mul_unordered(simd_select_bitmask( + k, + a.as_i64x8(), + _mm512_set1_epi64(1).as_i64x8(), + )) +} + +/// Reduce the packed single-precision (32-bit) floating-point elements in a by multiplication. Returns the product of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_mul_ps&expand=4606) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_mul_ps(a: __m512) -> f32 { + simd_reduce_mul_unordered(a.as_f32x16()) +} + +/// Reduce the packed single-precision (32-bit) floating-point elements in a by multiplication using mask k. Returns the product of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_mul_ps&expand=4605) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_mul_ps(k: __mmask16, a: __m512) -> f32 { + simd_reduce_mul_unordered(simd_select_bitmask( + k, + a.as_f32x16(), + _mm512_set1_ps(1.).as_f32x16(), + )) +} + +/// Reduce the packed double-precision (64-bit) floating-point elements in a by multiplication. Returns the product of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_mul_pd&expand=4604) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_mul_pd(a: __m512d) -> f64 { + simd_reduce_mul_unordered(a.as_f64x8()) +} + +/// Reduce the packed double-precision (64-bit) floating-point elements in a by multiplication using mask k. Returns the product of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_mul_pd&expand=4603) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_mul_pd(k: __mmask8, a: __m512d) -> f64 { + simd_reduce_mul_unordered(simd_select_bitmask( + k, + a.as_f64x8(), + _mm512_set1_pd(1.).as_f64x8(), + )) +} + +/// Reduce the packed signed 32-bit integers in a by maximum. Returns the maximum of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_max_epi32&expand=4576) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_max_epi32(a: __m512i) -> i32 { + simd_reduce_max(a.as_i32x16()) +} + +/// Reduce the packed signed 32-bit integers in a by maximum using mask k. Returns the maximum of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_max_epi32&expand=4575) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_max_epi32(k: __mmask16, a: __m512i) -> i32 { + simd_reduce_max(simd_select_bitmask( + k, + a.as_i32x16(), + _mm512_undefined_epi32().as_i32x16(), + )) +} + +/// Reduce the packed signed 64-bit integers in a by maximum. Returns the maximum of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_max_epi64&expand=4578) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_max_epi64(a: __m512i) -> i64 { + simd_reduce_max(a.as_i64x8()) +} + +/// Reduce the packed signed 64-bit integers in a by maximum using mask k. Returns the maximum of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_max_epi64&expand=4577) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_max_epi64(k: __mmask8, a: __m512i) -> i64 { + simd_reduce_max(simd_select_bitmask( + k, + a.as_i64x8(), + _mm512_set1_epi64(0).as_i64x8(), + )) +} + +/// Reduce the packed unsigned 32-bit integers in a by maximum. Returns the maximum of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_max_epu32&expand=4580) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_max_epu32(a: __m512i) -> u32 { + simd_reduce_max(a.as_u32x16()) +} + +/// Reduce the packed unsigned 32-bit integers in a by maximum using mask k. Returns the maximum of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_max_epu32&expand=4579) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_max_epu32(k: __mmask16, a: __m512i) -> u32 { + simd_reduce_max(simd_select_bitmask( + k, + a.as_u32x16(), + _mm512_undefined_epi32().as_u32x16(), + )) +} + +/// Reduce the packed unsigned 64-bit integers in a by maximum. Returns the maximum of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_max_epu64&expand=4582) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_max_epu64(a: __m512i) -> u64 { + simd_reduce_max(a.as_u64x8()) +} + +/// Reduce the packed unsigned 64-bit integers in a by maximum using mask k. Returns the maximum of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_max_epu64&expand=4581) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_max_epu64(k: __mmask8, a: __m512i) -> u64 { + simd_reduce_max(simd_select_bitmask( + k, + a.as_u64x8(), + _mm512_set1_epi64(0).as_u64x8(), + )) +} + +/// Reduce the packed single-precision (32-bit) floating-point elements in a by maximum. Returns the maximum of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_max_ps&expand=4586) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_max_ps(a: __m512) -> f32 { + simd_reduce_max(a.as_f32x16()) +} + +/// Reduce the packed single-precision (32-bit) floating-point elements in a by maximum using mask k. Returns the maximum of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_max_ps&expand=4585) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_max_ps(k: __mmask16, a: __m512) -> f32 { + simd_reduce_max(simd_select_bitmask( + k, + a.as_f32x16(), + _mm512_undefined_ps().as_f32x16(), + )) +} + +/// Reduce the packed double-precision (64-bit) floating-point elements in a by maximum. Returns the maximum of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_max_pd&expand=4584) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_max_pd(a: __m512d) -> f64 { + simd_reduce_max(a.as_f64x8()) +} + +/// Reduce the packed double-precision (64-bit) floating-point elements in a by maximum using mask k. Returns the maximum of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_max_pd&expand=4583) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_max_pd(k: __mmask8, a: __m512d) -> f64 { + simd_reduce_max(simd_select_bitmask( + k, + a.as_f64x8(), + _mm512_undefined_pd().as_f64x8(), + )) +} + +/// Reduce the packed signed 32-bit integers in a by minimum. Returns the minimum of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_min_epi32&expand=4588) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_min_epi32(a: __m512i) -> i32 { + simd_reduce_min(a.as_i32x16()) +} + +/// Reduce the packed signed 32-bit integers in a by maximum using mask k. Returns the minimum of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_min_epi32&expand=4587) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_min_epi32(k: __mmask16, a: __m512i) -> i32 { + simd_reduce_min(simd_select_bitmask( + k, + a.as_i32x16(), + _mm512_undefined_epi32().as_i32x16(), + )) +} + +/// Reduce the packed signed 64-bit integers in a by minimum. Returns the minimum of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_min_epi64&expand=4590) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_min_epi64(a: __m512i) -> i64 { + simd_reduce_min(a.as_i64x8()) +} + +/// Reduce the packed signed 64-bit integers in a by maximum using mask k. Returns the minimum of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_min_epi64&expand=4589) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_min_epi64(k: __mmask8, a: __m512i) -> i64 { + simd_reduce_min(simd_select_bitmask( + k, + a.as_i64x8(), + _mm512_set1_epi64(0).as_i64x8(), + )) +} + +/// Reduce the packed unsigned 32-bit integers in a by minimum. Returns the minimum of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_min_epu32&expand=4592) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_min_epu32(a: __m512i) -> u32 { + simd_reduce_min(a.as_u32x16()) +} + +/// Reduce the packed unsigned 32-bit integers in a by maximum using mask k. Returns the minimum of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_min_epu32&expand=4591) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_min_epu32(k: __mmask16, a: __m512i) -> u32 { + simd_reduce_min(simd_select_bitmask( + k, + a.as_u32x16(), + _mm512_undefined_epi32().as_u32x16(), + )) +} + +/// Reduce the packed unsigned 64-bit integers in a by minimum. Returns the minimum of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_min_epu64&expand=4594) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_min_epu64(a: __m512i) -> u64 { + simd_reduce_min(a.as_u64x8()) +} + +/// Reduce the packed signed 64-bit integers in a by maximum using mask k. Returns the minimum of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_min_epi64&expand=4589) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_min_epu64(k: __mmask8, a: __m512i) -> u64 { + simd_reduce_min(simd_select_bitmask( + k, + a.as_u64x8(), + _mm512_set1_epi64(0).as_u64x8(), + )) +} + +/// Reduce the packed single-precision (32-bit) floating-point elements in a by minimum. Returns the minimum of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_min_ps&expand=4598) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_min_ps(a: __m512) -> f32 { + simd_reduce_min(a.as_f32x16()) +} + +/// Reduce the packed single-precision (32-bit) floating-point elements in a by maximum using mask k. Returns the minimum of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_min_ps&expand=4597) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_min_ps(k: __mmask16, a: __m512) -> f32 { + simd_reduce_min(simd_select_bitmask( + k, + a.as_f32x16(), + _mm512_undefined_ps().as_f32x16(), + )) +} + +/// Reduce the packed double-precision (64-bit) floating-point elements in a by minimum. Returns the minimum of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_min_pd&expand=4596) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_min_pd(a: __m512d) -> f64 { + simd_reduce_min(a.as_f64x8()) +} + +/// Reduce the packed double-precision (64-bit) floating-point elements in a by maximum using mask k. Returns the minimum of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_min_pd&expand=4595) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_min_pd(k: __mmask8, a: __m512d) -> f64 { + simd_reduce_min(simd_select_bitmask( + k, + a.as_f64x8(), + _mm512_undefined_pd().as_f64x8(), + )) +} + +/// Reduce the packed 32-bit integers in a by bitwise AND. Returns the bitwise AND of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_and_epi32&expand=4564) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_and_epi32(a: __m512i) -> i32 { + simd_reduce_and(a.as_i32x16()) +} + +/// Reduce the packed 32-bit integers in a by bitwise AND using mask k. Returns the bitwise AND of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_and_epi32&expand=4563) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_and_epi32(k: __mmask16, a: __m512i) -> i32 { + simd_reduce_and(simd_select_bitmask( + k, + a.as_i32x16(), + _mm512_set1_epi32(0xFF).as_i32x16(), + )) +} + +/// Reduce the packed 64-bit integers in a by bitwise AND. Returns the bitwise AND of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_and_epi64&expand=4566) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_and_epi64(a: __m512i) -> i64 { + simd_reduce_and(a.as_i64x8()) +} + +/// Reduce the packed 64-bit integers in a by addition using mask k. Returns the sum of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_add_epi64&expand=4557) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_and_epi64(k: __mmask8, a: __m512i) -> i64 { + simd_reduce_and(simd_select_bitmask( + k, + a.as_i64x8(), + _mm512_set1_epi64(1 << 0 | 1 << 1 | 1 << 2 | 1 << 3 | 1 << 4 | 1 << 5 | 1 << 6 | 1 << 7) + .as_i64x8(), + )) +} + +/// Reduce the packed 32-bit integers in a by bitwise OR. Returns the bitwise OR of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_or_epi32&expand=4608) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_or_epi32(a: __m512i) -> i32 { + simd_reduce_or(a.as_i32x16()) +} + +/// Reduce the packed 32-bit integers in a by bitwise OR using mask k. Returns the bitwise OR of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_or_epi32&expand=4607) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_or_epi32(k: __mmask16, a: __m512i) -> i32 { + simd_reduce_or(simd_select_bitmask( + k, + a.as_i32x16(), + _mm512_setzero_si512().as_i32x16(), + )) +} + +/// Reduce the packed 64-bit integers in a by bitwise OR. Returns the bitwise OR of all elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_reduce_or_epi64&expand=4610) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_reduce_or_epi64(a: __m512i) -> i64 { + simd_reduce_or(a.as_i64x8()) +} + +/// Reduce the packed 64-bit integers in a by bitwise OR using mask k. Returns the bitwise OR of all active elements in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_reduce_or_epi64&expand=4609) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_reduce_or_epi64(k: __mmask8, a: __m512i) -> i64 { + simd_reduce_or(simd_select_bitmask( + k, + a.as_i64x8(), + _mm512_setzero_si512().as_i64x8(), + )) +} + +/// Returns vector of type `__m512d` with undefined elements. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_undefined_pd) +#[inline] +#[target_feature(enable = "avx512f")] +// This intrinsic has no corresponding instruction. +pub unsafe fn _mm512_undefined_pd() -> __m512d { + _mm512_set1_pd(0.0) +} + +/// Returns vector of type `__m512` with undefined elements. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_undefined_ps) +#[inline] +#[target_feature(enable = "avx512f")] +// This intrinsic has no corresponding instruction. +pub unsafe fn _mm512_undefined_ps() -> __m512 { + _mm512_set1_ps(0.0) +} + +/// Return vector of type __m512i with undefined elements. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_undefined_epi32&expand=5995) +#[inline] +#[target_feature(enable = "avx512f")] +// This intrinsic has no corresponding instruction. +pub unsafe fn _mm512_undefined_epi32() -> __m512i { + _mm512_set1_epi32(0) +} + +/// Return vector of type __m512 with undefined elements. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_undefined&expand=5994) +#[inline] +#[target_feature(enable = "avx512f")] +// This intrinsic has no corresponding instruction. +pub unsafe fn _mm512_undefined() -> __m512 { + _mm512_set1_ps(0.0) +} + +/// Load 512-bits (composed of 16 packed 32-bit integers) from memory into dst. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_loadu_epi32&expand=3377) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu32 +pub unsafe fn _mm512_loadu_epi32(mem_addr: *const i32) -> __m512i { + ptr::read_unaligned(mem_addr as *const __m512i) +} + +/// Load 256-bits (composed of 8 packed 32-bit integers) from memory into dst. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_loadu_epi32&expand=3374) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu32 +pub unsafe fn _mm256_loadu_epi32(mem_addr: *const i32) -> __m256i { + ptr::read_unaligned(mem_addr as *const __m256i) +} + +/// Load 128-bits (composed of 4 packed 32-bit integers) from memory into dst. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadu_epi32&expand=3371) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu32 +pub unsafe fn _mm_loadu_epi32(mem_addr: *const i32) -> __m128i { + ptr::read_unaligned(mem_addr as *const __m128i) +} + +/// Convert packed 32-bit integers in a to packed 16-bit integers with truncation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepi32_storeu_epi16&expand=1460) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovdw))] +pub unsafe fn _mm512_mask_cvtepi32_storeu_epi16(mem_addr: *mut i8, k: __mmask16, a: __m512i) { + vpmovdwmem(mem_addr as *mut i8, a.as_i32x16(), k); +} + +/// Convert packed 32-bit integers in a to packed 16-bit integers with truncation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepi32_storeu_epi8&expand=1462) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovdw))] +pub unsafe fn _mm256_mask_cvtepi32_storeu_epi16(mem_addr: *mut i8, k: __mmask8, a: __m256i) { + vpmovdwmem256(mem_addr as *mut i8, a.as_i32x8(), k); +} + +/// Convert packed 32-bit integers in a to packed 16-bit integers with truncation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepi32_storeu_epi8&expand=1461) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovdw))] +pub unsafe fn _mm_mask_cvtepi32_storeu_epi16(mem_addr: *mut i8, k: __mmask8, a: __m128i) { + vpmovdwmem128(mem_addr as *mut i8, a.as_i32x4(), k); +} + +/// Convert packed signed 32-bit integers in a to packed 16-bit integers with signed saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtsepi32_storeu_epi16&expand=1833) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsdw))] +pub unsafe fn _mm512_mask_cvtsepi32_storeu_epi16(mem_addr: *mut i8, k: __mmask16, a: __m512i) { + vpmovsdwmem(mem_addr as *mut i8, a.as_i32x16(), k); +} + +/// Convert packed signed 32-bit integers in a to packed 16-bit integers with signed saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtsepi32_storeu_epi16&expand=1832) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsdw))] +pub unsafe fn _mm256_mask_cvtsepi32_storeu_epi16(mem_addr: *mut i8, k: __mmask8, a: __m256i) { + vpmovsdwmem256(mem_addr as *mut i8, a.as_i32x8(), k); +} + +/// Convert packed signed 32-bit integers in a to packed 16-bit integers with signed saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtsepi32_storeu_epi16&expand=1831) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsdw))] +pub unsafe fn _mm_mask_cvtsepi32_storeu_epi16(mem_addr: *mut i8, k: __mmask8, a: __m128i) { + vpmovsdwmem128(mem_addr as *mut i8, a.as_i32x4(), k); +} + +/// Convert packed unsigned 32-bit integers in a to packed 16-bit integers with unsigned saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtusepi32_storeu_epi16&expand=2068) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovusdw))] +pub unsafe fn _mm512_mask_cvtusepi32_storeu_epi16(mem_addr: *mut i8, k: __mmask16, a: __m512i) { + vpmovusdwmem(mem_addr as *mut i8, a.as_i32x16(), k); +} + +/// Convert packed unsigned 32-bit integers in a to packed unsigned 16-bit integers with unsigned saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtusepi32_storeu_epi16&expand=2067) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusdw))] +pub unsafe fn _mm256_mask_cvtusepi32_storeu_epi16(mem_addr: *mut i8, k: __mmask8, a: __m256i) { + vpmovusdwmem256(mem_addr as *mut i8, a.as_i32x8(), k); +} + +/// Convert packed unsigned 32-bit integers in a to packed unsigned 16-bit integers with unsigned saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtusepi32_storeu_epi16&expand=2066) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusdw))] +pub unsafe fn _mm_mask_cvtusepi32_storeu_epi16(mem_addr: *mut i8, k: __mmask8, a: __m128i) { + vpmovusdwmem128(mem_addr as *mut i8, a.as_i32x4(), k); +} + +/// Convert packed 32-bit integers in a to packed 8-bit integers with truncation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepi32_storeu_epi8&expand=1463) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovdb))] +pub unsafe fn _mm512_mask_cvtepi32_storeu_epi8(mem_addr: *mut i8, k: __mmask16, a: __m512i) { + vpmovdbmem(mem_addr as *mut i8, a.as_i32x16(), k); +} + +/// Convert packed 32-bit integers in a to packed 8-bit integers with truncation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepi32_storeu_epi8&expand=1462) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovdb))] +pub unsafe fn _mm256_mask_cvtepi32_storeu_epi8(mem_addr: *mut i8, k: __mmask8, a: __m256i) { + vpmovdbmem256(mem_addr as *mut i8, a.as_i32x8(), k); +} + +/// Convert packed 32-bit integers in a to packed 8-bit integers with truncation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepi32_storeu_epi8&expand=1461) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovdb))] +pub unsafe fn _mm_mask_cvtepi32_storeu_epi8(mem_addr: *mut i8, k: __mmask8, a: __m128i) { + vpmovdbmem128(mem_addr as *mut i8, a.as_i32x4(), k); +} + +/// Convert packed signed 32-bit integers in a to packed 8-bit integers with signed saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtsepi32_storeu_epi8&expand=1836) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsdb))] +pub unsafe fn _mm512_mask_cvtsepi32_storeu_epi8(mem_addr: *mut i8, k: __mmask16, a: __m512i) { + vpmovsdbmem(mem_addr as *mut i8, a.as_i32x16(), k); +} + +/// Convert packed signed 32-bit integers in a to packed 8-bit integers with signed saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtsepi32_storeu_epi8&expand=1835) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsdb))] +pub unsafe fn _mm256_mask_cvtsepi32_storeu_epi8(mem_addr: *mut i8, k: __mmask8, a: __m256i) { + vpmovsdbmem256(mem_addr as *mut i8, a.as_i32x8(), k); +} + +/// Convert packed signed 32-bit integers in a to packed 8-bit integers with signed saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtsepi32_storeu_epi8&expand=1834) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsdb))] +pub unsafe fn _mm_mask_cvtsepi32_storeu_epi8(mem_addr: *mut i8, k: __mmask8, a: __m128i) { + vpmovsdbmem128(mem_addr as *mut i8, a.as_i32x4(), k); +} + +/// Convert packed unsigned 32-bit integers in a to packed 8-bit integers with unsigned saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtusepi32_storeu_epi8&expand=2071) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovusdb))] +pub unsafe fn _mm512_mask_cvtusepi32_storeu_epi8(mem_addr: *mut i8, k: __mmask16, a: __m512i) { + vpmovusdbmem(mem_addr as *mut i8, a.as_i32x16(), k); +} + +/// Convert packed unsigned 32-bit integers in a to packed 8-bit integers with unsigned saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtusepi32_storeu_epi8&expand=2070) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusdb))] +pub unsafe fn _mm256_mask_cvtusepi32_storeu_epi8(mem_addr: *mut i8, k: __mmask8, a: __m256i) { + vpmovusdbmem256(mem_addr as *mut i8, a.as_i32x8(), k); +} + +/// Convert packed unsigned 32-bit integers in a to packed 8-bit integers with unsigned saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtusepi32_storeu_epi8&expand=2069) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusdb))] +pub unsafe fn _mm_mask_cvtusepi32_storeu_epi8(mem_addr: *mut i8, k: __mmask8, a: __m128i) { + vpmovusdbmem128(mem_addr as *mut i8, a.as_i32x4(), k); +} + +/// Convert packed 64-bit integers in a to packed 16-bit integers with truncation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepi64_storeu_epi16&expand=1513) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovqw))] +pub unsafe fn _mm512_mask_cvtepi64_storeu_epi16(mem_addr: *mut i8, k: __mmask8, a: __m512i) { + vpmovqwmem(mem_addr as *mut i8, a.as_i64x8(), k); +} + +/// Convert packed 64-bit integers in a to packed 16-bit integers with truncation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepi64_storeu_epi16&expand=1512) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqw))] +pub unsafe fn _mm256_mask_cvtepi64_storeu_epi16(mem_addr: *mut i8, k: __mmask8, a: __m256i) { + vpmovqwmem256(mem_addr as *mut i8, a.as_i64x4(), k); +} + +/// Convert packed 64-bit integers in a to packed 16-bit integers with truncation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepi64_storeu_epi16&expand=1511) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqw))] +pub unsafe fn _mm_mask_cvtepi64_storeu_epi16(mem_addr: *mut i8, k: __mmask8, a: __m128i) { + vpmovqwmem128(mem_addr as *mut i8, a.as_i64x2(), k); +} + +/// Convert packed signed 64-bit integers in a to packed 16-bit integers with signed saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtsepi64_storeu_epi16&expand=1866) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsqw))] +pub unsafe fn _mm512_mask_cvtsepi64_storeu_epi16(mem_addr: *mut i8, k: __mmask8, a: __m512i) { + vpmovsqwmem(mem_addr as *mut i8, a.as_i64x8(), k); +} + +/// Convert packed signed 64-bit integers in a to packed 16-bit integers with signed saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtsepi64_storeu_epi16&expand=1865) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqw))] +pub unsafe fn _mm256_mask_cvtsepi64_storeu_epi16(mem_addr: *mut i8, k: __mmask8, a: __m256i) { + vpmovsqwmem256(mem_addr as *mut i8, a.as_i64x4(), k); +} + +/// Convert packed signed 64-bit integers in a to packed 16-bit integers with signed saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtsepi64_storeu_epi16&expand=1864) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqw))] +pub unsafe fn _mm_mask_cvtsepi64_storeu_epi16(mem_addr: *mut i8, k: __mmask8, a: __m128i) { + vpmovsqwmem128(mem_addr as *mut i8, a.as_i64x2(), k); +} + +/// Convert packed unsigned 64-bit integers in a to packed 16-bit integers with unsigned saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtusepi64_storeu_epi16&expand=2101) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovusqw))] +pub unsafe fn _mm512_mask_cvtusepi64_storeu_epi16(mem_addr: *mut i8, k: __mmask8, a: __m512i) { + vpmovusqwmem(mem_addr as *mut i8, a.as_i64x8(), k); +} + +/// Convert packed unsigned 64-bit integers in a to packed 16-bit integers with unsigned saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtusepi64_storeu_epi16&expand=2100) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqw))] +pub unsafe fn _mm256_mask_cvtusepi64_storeu_epi16(mem_addr: *mut i8, k: __mmask8, a: __m256i) { + vpmovusqwmem256(mem_addr as *mut i8, a.as_i64x4(), k); +} + +/// Convert packed unsigned 64-bit integers in a to packed 16-bit integers with unsigned saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtusepi64_storeu_epi16&expand=2099) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqw))] +pub unsafe fn _mm_mask_cvtusepi64_storeu_epi16(mem_addr: *mut i8, k: __mmask8, a: __m128i) { + vpmovusqwmem128(mem_addr as *mut i8, a.as_i64x2(), k); +} + +/// Convert packed 64-bit integers in a to packed 8-bit integers with truncation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepi64_storeu_epi8&expand=1519) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovqb))] +pub unsafe fn _mm512_mask_cvtepi64_storeu_epi8(mem_addr: *mut i8, k: __mmask8, a: __m512i) { + vpmovqbmem(mem_addr as *mut i8, a.as_i64x8(), k); +} + +/// Convert packed 64-bit integers in a to packed 8-bit integers with truncation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepi64_storeu_epi8&expand=1518) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqb))] +pub unsafe fn _mm256_mask_cvtepi64_storeu_epi8(mem_addr: *mut i8, k: __mmask8, a: __m256i) { + vpmovqbmem256(mem_addr as *mut i8, a.as_i64x4(), k); +} + +/// Convert packed 64-bit integers in a to packed 8-bit integers with truncation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepi64_storeu_epi8&expand=1517) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqb))] +pub unsafe fn _mm_mask_cvtepi64_storeu_epi8(mem_addr: *mut i8, k: __mmask8, a: __m128i) { + vpmovqbmem128(mem_addr as *mut i8, a.as_i64x2(), k); +} + +/// Convert packed signed 64-bit integers in a to packed 8-bit integers with signed saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtsepi64_storeu_epi8&expand=1872) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsqb))] +pub unsafe fn _mm512_mask_cvtsepi64_storeu_epi8(mem_addr: *mut i8, k: __mmask8, a: __m512i) { + vpmovsqbmem(mem_addr as *mut i8, a.as_i64x8(), k); +} + +/// Convert packed signed 64-bit integers in a to packed 8-bit integers with signed saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtsepi64_storeu_epi8&expand=1871) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqb))] +pub unsafe fn _mm256_mask_cvtsepi64_storeu_epi8(mem_addr: *mut i8, k: __mmask8, a: __m256i) { + vpmovsqbmem256(mem_addr as *mut i8, a.as_i64x4(), k); +} + +/// Convert packed signed 64-bit integers in a to packed 8-bit integers with signed saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtsepi64_storeu_epi8&expand=1870) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqb))] +pub unsafe fn _mm_mask_cvtsepi64_storeu_epi8(mem_addr: *mut i8, k: __mmask8, a: __m128i) { + vpmovsqbmem128(mem_addr as *mut i8, a.as_i64x2(), k); +} + +/// Convert packed unsigned 64-bit integers in a to packed 8-bit integers with unsigned saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtusepi64_storeu_epi8&expand=2107) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovusqb))] +pub unsafe fn _mm512_mask_cvtusepi64_storeu_epi8(mem_addr: *mut i8, k: __mmask8, a: __m512i) { + vpmovusqbmem(mem_addr as *mut i8, a.as_i64x8(), k); +} + +/// Convert packed unsigned 64-bit integers in a to packed 8-bit integers with unsigned saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtusepi64_storeu_epi8&expand=2106) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqb))] +pub unsafe fn _mm256_mask_cvtusepi64_storeu_epi8(mem_addr: *mut i8, k: __mmask8, a: __m256i) { + vpmovusqbmem256(mem_addr as *mut i8, a.as_i64x4(), k); +} + +/// Convert packed unsigned 64-bit integers in a to packed 8-bit integers with unsigned saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtusepi64_storeu_epi8&expand=2105) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqb))] +pub unsafe fn _mm_mask_cvtusepi64_storeu_epi8(mem_addr: *mut i8, k: __mmask8, a: __m128i) { + vpmovusqbmem128(mem_addr as *mut i8, a.as_i64x2(), k); +} + +///Convert packed 64-bit integers in a to packed 32-bit integers with truncation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtepi64_storeu_epi32&expand=1516) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovqd))] +pub unsafe fn _mm512_mask_cvtepi64_storeu_epi32(mem_addr: *mut i8, k: __mmask8, a: __m512i) { + vpmovqdmem(mem_addr as *mut i8, a.as_i64x8(), k); +} + +///Convert packed 64-bit integers in a to packed 32-bit integers with truncation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtepi64_storeu_epi32&expand=1515) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqd))] +pub unsafe fn _mm256_mask_cvtepi64_storeu_epi32(mem_addr: *mut i8, k: __mmask8, a: __m256i) { + vpmovqdmem256(mem_addr as *mut i8, a.as_i64x4(), k); +} + +///Convert packed 64-bit integers in a to packed 32-bit integers with truncation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtepi64_storeu_epi32&expand=1514) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovqd))] +pub unsafe fn _mm_mask_cvtepi64_storeu_epi32(mem_addr: *mut i8, k: __mmask8, a: __m128i) { + vpmovqdmem128(mem_addr as *mut i8, a.as_i64x2(), k); +} + +/// Convert packed signed 64-bit integers in a to packed 32-bit integers with signed saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtsepi64_storeu_epi32&expand=1869) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovsqd))] +pub unsafe fn _mm512_mask_cvtsepi64_storeu_epi32(mem_addr: *mut i8, k: __mmask8, a: __m512i) { + vpmovsqdmem(mem_addr as *mut i8, a.as_i64x8(), k); +} + +/// Convert packed signed 64-bit integers in a to packed 32-bit integers with signed saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtsepi64_storeu_epi32&expand=1868) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqd))] +pub unsafe fn _mm256_mask_cvtsepi64_storeu_epi32(mem_addr: *mut i8, k: __mmask8, a: __m256i) { + vpmovsqdmem256(mem_addr as *mut i8, a.as_i64x4(), k); +} + +/// Convert packed signed 64-bit integers in a to packed 32-bit integers with signed saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtsepi64_storeu_epi32&expand=1867) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovsqd))] +pub unsafe fn _mm_mask_cvtsepi64_storeu_epi32(mem_addr: *mut i8, k: __mmask8, a: __m128i) { + vpmovsqdmem128(mem_addr as *mut i8, a.as_i64x2(), k); +} + +/// Convert packed unsigned 64-bit integers in a to packed 32-bit integers with unsigned saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtusepi64_storeu_epi32&expand=2104) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vpmovusqd))] +pub unsafe fn _mm512_mask_cvtusepi64_storeu_epi32(mem_addr: *mut i8, k: __mmask8, a: __m512i) { + vpmovusqdmem(mem_addr as *mut i8, a.as_i64x8(), k); +} + +/// Convert packed unsigned 64-bit integers in a to packed 32-bit integers with unsigned saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtusepi64_storeu_epi32&expand=2103) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqd))] +pub unsafe fn _mm256_mask_cvtusepi64_storeu_epi32(mem_addr: *mut i8, k: __mmask8, a: __m256i) { + vpmovusqdmem256(mem_addr as *mut i8, a.as_i64x4(), k); +} + +/// Convert packed unsigned 64-bit integers in a to packed 32-bit integers with unsigned saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtusepi64_storeu_epi32&expand=2102) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vpmovusqd))] +pub unsafe fn _mm_mask_cvtusepi64_storeu_epi32(mem_addr: *mut i8, k: __mmask8, a: __m128i) { + vpmovusqdmem128(mem_addr as *mut i8, a.as_i64x2(), k); +} + +/// Store 512-bits (composed of 16 packed 32-bit integers) from a into memory. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_storeu_epi32&expand=5628) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu32 +pub unsafe fn _mm512_storeu_epi32(mem_addr: *mut i32, a: __m512i) { + ptr::write_unaligned(mem_addr as *mut __m512i, a); +} + +/// Store 256-bits (composed of 8 packed 32-bit integers) from a into memory. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_storeu_epi32&expand=5626) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu32 +pub unsafe fn _mm256_storeu_epi32(mem_addr: *mut i32, a: __m256i) { + ptr::write_unaligned(mem_addr as *mut __m256i, a); +} + +/// Store 128-bits (composed of 4 packed 32-bit integers) from a into memory. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_storeu_epi32&expand=5624) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu32 +pub unsafe fn _mm_storeu_epi32(mem_addr: *mut i32, a: __m128i) { + ptr::write_unaligned(mem_addr as *mut __m128i, a); +} + +/// Load 512-bits (composed of 8 packed 64-bit integers) from memory into dst. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_loadu_epi64&expand=3386) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu64 +pub unsafe fn _mm512_loadu_epi64(mem_addr: *const i64) -> __m512i { + ptr::read_unaligned(mem_addr as *const __m512i) +} + +/// Load 256-bits (composed of 4 packed 64-bit integers) from memory into dst. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_loadu_epi64&expand=3383) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu64 +pub unsafe fn _mm256_loadu_epi64(mem_addr: *const i64) -> __m256i { + ptr::read_unaligned(mem_addr as *const __m256i) +} + +/// Load 128-bits (composed of 2 packed 64-bit integers) from memory into dst. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadu_epi64&expand=3380) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu64 +pub unsafe fn _mm_loadu_epi64(mem_addr: *const i64) -> __m128i { + ptr::read_unaligned(mem_addr as *const __m128i) +} + +/// Store 512-bits (composed of 8 packed 64-bit integers) from a into memory. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_storeu_epi64&expand=5634) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu64 +pub unsafe fn _mm512_storeu_epi64(mem_addr: *mut i64, a: __m512i) { + ptr::write_unaligned(mem_addr as *mut __m512i, a); +} + +/// Store 256-bits (composed of 4 packed 64-bit integers) from a into memory. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_storeu_epi64&expand=5632) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu64 +pub unsafe fn _mm256_storeu_epi64(mem_addr: *mut i64, a: __m256i) { + ptr::write_unaligned(mem_addr as *mut __m256i, a); +} + +/// Store 128-bits (composed of 2 packed 64-bit integers) from a into memory. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_storeu_epi64&expand=5630) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu64 +pub unsafe fn _mm_storeu_epi64(mem_addr: *mut i64, a: __m128i) { + ptr::write_unaligned(mem_addr as *mut __m128i, a); +} + +/// Load 512-bits of integer data from memory into dst. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_loadu_si512&expand=3420) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu32 +pub unsafe fn _mm512_loadu_si512(mem_addr: *const i32) -> __m512i { + ptr::read_unaligned(mem_addr as *const __m512i) +} + +/// Store 512-bits of integer data from a into memory. mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_storeu_si512&expand=5657) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovups))] //should be vmovdqu32 +pub unsafe fn _mm512_storeu_si512(mem_addr: *mut i32, a: __m512i) { + ptr::write_unaligned(mem_addr as *mut __m512i, a); +} + +/// Loads 512-bits (composed of 8 packed double-precision (64-bit) +/// floating-point elements) from memory into result. +/// `mem_addr` does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_loadu_pd) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovups))] +pub unsafe fn _mm512_loadu_pd(mem_addr: *const f64) -> __m512d { + ptr::read_unaligned(mem_addr as *const __m512d) +} + +/// Stores 512-bits (composed of 8 packed double-precision (64-bit) +/// floating-point elements) from `a` into memory. +/// `mem_addr` does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_storeu_pd) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovups))] +pub unsafe fn _mm512_storeu_pd(mem_addr: *mut f64, a: __m512d) { + ptr::write_unaligned(mem_addr as *mut __m512d, a); +} + +/// Loads 512-bits (composed of 16 packed single-precision (32-bit) +/// floating-point elements) from memory into result. +/// `mem_addr` does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_loadu_ps) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovups))] +pub unsafe fn _mm512_loadu_ps(mem_addr: *const f32) -> __m512 { + ptr::read_unaligned(mem_addr as *const __m512) +} + +/// Stores 512-bits (composed of 16 packed single-precision (32-bit) +/// floating-point elements) from `a` into memory. +/// `mem_addr` does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_storeu_ps) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovups))] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm512_storeu_ps(mem_addr: *mut f32, a: __m512) { + ptr::write_unaligned(mem_addr as *mut __m512, a); +} + +/// Load 512-bits of integer data from memory into dst. mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_load_si512&expand=3345) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovaps))] //should be vmovdqa32 +pub unsafe fn _mm512_load_si512(mem_addr: *const i32) -> __m512i { + ptr::read(mem_addr as *const __m512i) +} + +/// Store 512-bits of integer data from a into memory. mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_store_si512&expand=5598) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovaps))] //should be vmovdqa32 +pub unsafe fn _mm512_store_si512(mem_addr: *mut i32, a: __m512i) { + ptr::write(mem_addr as *mut __m512i, a); +} + +/// Load 512-bits (composed of 16 packed 32-bit integers) from memory into dst. mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_load_epi32&expand=3304) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovaps))] //should be vmovdqa32 +pub unsafe fn _mm512_load_epi32(mem_addr: *const i32) -> __m512i { + ptr::read(mem_addr as *const __m512i) +} + +/// Load 256-bits (composed of 8 packed 32-bit integers) from memory into dst. mem_addr must be aligned on a 32-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_load_epi32&expand=3301) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovaps))] //should be vmovdqa32 +pub unsafe fn _mm256_load_epi32(mem_addr: *const i32) -> __m256i { + ptr::read(mem_addr as *const __m256i) +} + +/// Load 128-bits (composed of 4 packed 32-bit integers) from memory into dst. mem_addr must be aligned on a 16-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_load_epi32&expand=3298) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovaps))] //should be vmovdqa32 +pub unsafe fn _mm_load_epi32(mem_addr: *const i32) -> __m128i { + ptr::read(mem_addr as *const __m128i) +} + +/// Store 512-bits (composed of 16 packed 32-bit integers) from a into memory. mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_store_epi32&expand=5569) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovaps))] //should be vmovdqa32 +pub unsafe fn _mm512_store_epi32(mem_addr: *mut i32, a: __m512i) { + ptr::write(mem_addr as *mut __m512i, a); +} + +/// Store 256-bits (composed of 8 packed 32-bit integers) from a into memory. mem_addr must be aligned on a 32-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_store_epi32&expand=5567) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovaps))] //should be vmovdqa32 +pub unsafe fn _mm256_store_epi32(mem_addr: *mut i32, a: __m256i) { + ptr::write(mem_addr as *mut __m256i, a); +} + +/// Store 128-bits (composed of 4 packed 32-bit integers) from a into memory. mem_addr must be aligned on a 16-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_store_epi32&expand=5565) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovaps))] //should be vmovdqa32 +pub unsafe fn _mm_store_epi32(mem_addr: *mut i32, a: __m128i) { + ptr::write(mem_addr as *mut __m128i, a); +} + +/// Load 512-bits (composed of 8 packed 64-bit integers) from memory into dst. mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_load_epi64&expand=3313) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovaps))] //should be vmovdqa64 +pub unsafe fn _mm512_load_epi64(mem_addr: *const i64) -> __m512i { + ptr::read(mem_addr as *const __m512i) +} + +/// Load 256-bits (composed of 4 packed 64-bit integers) from memory into dst. mem_addr must be aligned on a 32-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_load_epi64&expand=3310) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovaps))] //should be vmovdqa64 +pub unsafe fn _mm256_load_epi64(mem_addr: *const i64) -> __m256i { + ptr::read(mem_addr as *const __m256i) +} + +/// Load 128-bits (composed of 2 packed 64-bit integers) from memory into dst. mem_addr must be aligned on a 16-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_load_epi64&expand=3307) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovaps))] //should be vmovdqa64 +pub unsafe fn _mm_load_epi64(mem_addr: *const i64) -> __m128i { + ptr::read(mem_addr as *const __m128i) +} + +/// Store 512-bits (composed of 8 packed 64-bit integers) from a into memory. mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_store_epi64&expand=5575) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovaps))] //should be vmovdqa64 +pub unsafe fn _mm512_store_epi64(mem_addr: *mut i64, a: __m512i) { + ptr::write(mem_addr as *mut __m512i, a); +} + +/// Store 256-bits (composed of 4 packed 64-bit integers) from a into memory. mem_addr must be aligned on a 32-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_store_epi64&expand=5573) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovaps))] //should be vmovdqa64 +pub unsafe fn _mm256_store_epi64(mem_addr: *mut i64, a: __m256i) { + ptr::write(mem_addr as *mut __m256i, a); +} + +/// Store 128-bits (composed of 2 packed 64-bit integers) from a into memory. mem_addr must be aligned on a 16-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_store_epi64&expand=5571) +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[cfg_attr(test, assert_instr(vmovaps))] //should be vmovdqa64 +pub unsafe fn _mm_store_epi64(mem_addr: *mut i64, a: __m128i) { + ptr::write(mem_addr as *mut __m128i, a); +} + +/// Load 512-bits (composed of 16 packed single-precision (32-bit) floating-point elements) from memory into dst. mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_load_ps&expand=3336) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovaps))] +pub unsafe fn _mm512_load_ps(mem_addr: *const f32) -> __m512 { + ptr::read(mem_addr as *const __m512) +} + +/// Store 512-bits of integer data from a into memory. mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_store_ps&expand=5592) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovaps))] +pub unsafe fn _mm512_store_ps(mem_addr: *mut f32, a: __m512) { + ptr::write(mem_addr as *mut __m512, a); +} + +/// Load 512-bits (composed of 8 packed double-precision (64-bit) floating-point elements) from memory into dst. mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_load_pd&expand=3326) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovaps))] //should be vmovapd +pub unsafe fn _mm512_load_pd(mem_addr: *const f64) -> __m512d { + ptr::read(mem_addr as *const __m512d) +} + +/// Store 512-bits (composed of 8 packed double-precision (64-bit) floating-point elements) from a into memory. mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_store_pd&expand=5585) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovaps))] //should be vmovapd +pub unsafe fn _mm512_store_pd(mem_addr: *mut f64, a: __m512d) { + ptr::write(mem_addr as *mut __m512d, a); +} + +/// Load packed 32-bit integers from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_loadu_epi32) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_loadu_epi32(src: __m512i, k: __mmask16, mem_addr: *const i32) -> __m512i { + let mut dst: __m512i = src; + asm!( + vpl!("vmovdqu32 {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 32-bit integers from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_loadu_epi32) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_maskz_loadu_epi32(k: __mmask16, mem_addr: *const i32) -> __m512i { + let mut dst: __m512i; + asm!( + vpl!("vmovdqu32 {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 64-bit integers from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_loadu_epi64) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_loadu_epi64(src: __m512i, k: __mmask8, mem_addr: *const i64) -> __m512i { + let mut dst: __m512i = src; + asm!( + vpl!("vmovdqu64 {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 64-bit integers from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_loadu_epi64) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_maskz_loadu_epi64(k: __mmask8, mem_addr: *const i64) -> __m512i { + let mut dst: __m512i; + asm!( + vpl!("vmovdqu64 {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed single-precision (32-bit) floating-point elements from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_loadu_ps) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_loadu_ps(src: __m512, k: __mmask16, mem_addr: *const f32) -> __m512 { + let mut dst: __m512 = src; + asm!( + vpl!("vmovups {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed single-precision (32-bit) floating-point elements from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_loadu_ps) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_maskz_loadu_ps(k: __mmask16, mem_addr: *const f32) -> __m512 { + let mut dst: __m512; + asm!( + vpl!("vmovups {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed double-precision (64-bit) floating-point elements from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_loadu_pd) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_loadu_pd(src: __m512d, k: __mmask8, mem_addr: *const f64) -> __m512d { + let mut dst: __m512d = src; + asm!( + vpl!("vmovupd {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed double-precision (64-bit) floating-point elements from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_loadu_pd) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_maskz_loadu_pd(k: __mmask8, mem_addr: *const f64) -> __m512d { + let mut dst: __m512d; + asm!( + vpl!("vmovupd {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 32-bit integers from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_loadu_epi32) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_mask_loadu_epi32(src: __m256i, k: __mmask8, mem_addr: *const i32) -> __m256i { + let mut dst: __m256i = src; + asm!( + vpl!("vmovdqu32 {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 32-bit integers from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_loadu_epi32) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_maskz_loadu_epi32(k: __mmask8, mem_addr: *const i32) -> __m256i { + let mut dst: __m256i; + asm!( + vpl!("vmovdqu32 {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 64-bit integers from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_loadu_epi64) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_mask_loadu_epi64(src: __m256i, k: __mmask8, mem_addr: *const i64) -> __m256i { + let mut dst: __m256i = src; + asm!( + vpl!("vmovdqu64 {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 64-bit integers from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_loadu_epi64) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_maskz_loadu_epi64(k: __mmask8, mem_addr: *const i64) -> __m256i { + let mut dst: __m256i; + asm!( + vpl!("vmovdqu64 {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed single-precision (32-bit) floating-point elements from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_loadu_ps) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_mask_loadu_ps(src: __m256, k: __mmask8, mem_addr: *const f32) -> __m256 { + let mut dst: __m256 = src; + asm!( + vpl!("vmovups {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed single-precision (32-bit) floating-point elements from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_loadu_ps) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_maskz_loadu_ps(k: __mmask8, mem_addr: *const f32) -> __m256 { + let mut dst: __m256; + asm!( + vpl!("vmovups {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed double-precision (64-bit) floating-point elements from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_loadu_pd) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_mask_loadu_pd(src: __m256d, k: __mmask8, mem_addr: *const f64) -> __m256d { + let mut dst: __m256d = src; + asm!( + vpl!("vmovupd {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed double-precision (64-bit) floating-point elements from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_loadu_pd) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_maskz_loadu_pd(k: __mmask8, mem_addr: *const f64) -> __m256d { + let mut dst: __m256d; + asm!( + vpl!("vmovupd {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 32-bit integers from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_loadu_epi32) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_loadu_epi32(src: __m128i, k: __mmask8, mem_addr: *const i32) -> __m128i { + let mut dst: __m128i = src; + asm!( + vpl!("vmovdqu32 {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 32-bit integers from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_loadu_epi32) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_maskz_loadu_epi32(k: __mmask8, mem_addr: *const i32) -> __m128i { + let mut dst: __m128i; + asm!( + vpl!("vmovdqu32 {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 64-bit integers from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_loadu_epi64) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_loadu_epi64(src: __m128i, k: __mmask8, mem_addr: *const i64) -> __m128i { + let mut dst: __m128i = src; + asm!( + vpl!("vmovdqu64 {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 64-bit integers from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_loadu_epi64) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_maskz_loadu_epi64(k: __mmask8, mem_addr: *const i64) -> __m128i { + let mut dst: __m128i; + asm!( + vpl!("vmovdqu64 {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed single-precision (32-bit) floating-point elements from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_loadu_ps) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_loadu_ps(src: __m128, k: __mmask8, mem_addr: *const f32) -> __m128 { + let mut dst: __m128 = src; + asm!( + vpl!("vmovups {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed single-precision (32-bit) floating-point elements from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_loadu_ps) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_maskz_loadu_ps(k: __mmask8, mem_addr: *const f32) -> __m128 { + let mut dst: __m128; + asm!( + vpl!("vmovups {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed double-precision (64-bit) floating-point elements from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_loadu_pd) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_loadu_pd(src: __m128d, k: __mmask8, mem_addr: *const f64) -> __m128d { + let mut dst: __m128d = src; + asm!( + vpl!("vmovupd {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed double-precision (64-bit) floating-point elements from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_loadu_pd) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_maskz_loadu_pd(k: __mmask8, mem_addr: *const f64) -> __m128d { + let mut dst: __m128d; + asm!( + vpl!("vmovupd {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 32-bit integers from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_load_epi32) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_load_epi32(src: __m512i, k: __mmask16, mem_addr: *const i32) -> __m512i { + let mut dst: __m512i = src; + asm!( + vpl!("vmovdqa32 {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 32-bit integers from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_load_epi32) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_maskz_load_epi32(k: __mmask16, mem_addr: *const i32) -> __m512i { + let mut dst: __m512i; + asm!( + vpl!("vmovdqa32 {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 64-bit integers from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_load_epi64) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_load_epi64(src: __m512i, k: __mmask8, mem_addr: *const i64) -> __m512i { + let mut dst: __m512i = src; + asm!( + vpl!("vmovdqa64 {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 64-bit integers from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_load_epi64) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_maskz_load_epi64(k: __mmask8, mem_addr: *const i64) -> __m512i { + let mut dst: __m512i; + asm!( + vpl!("vmovdqa64 {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed single-precision (32-bit) floating-point elements from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_load_ps) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_load_ps(src: __m512, k: __mmask16, mem_addr: *const f32) -> __m512 { + let mut dst: __m512 = src; + asm!( + vpl!("vmovaps {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed single-precision (32-bit) floating-point elements from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_load_ps) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_maskz_load_ps(k: __mmask16, mem_addr: *const f32) -> __m512 { + let mut dst: __m512; + asm!( + vpl!("vmovaps {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed double-precision (64-bit) floating-point elements from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_load_pd) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_load_pd(src: __m512d, k: __mmask8, mem_addr: *const f64) -> __m512d { + let mut dst: __m512d = src; + asm!( + vpl!("vmovapd {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed double-precision (64-bit) floating-point elements from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_load_pd) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_maskz_load_pd(k: __mmask8, mem_addr: *const f64) -> __m512d { + let mut dst: __m512d; + asm!( + vpl!("vmovapd {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 32-bit integers from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 32-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_load_epi32) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_mask_load_epi32(src: __m256i, k: __mmask8, mem_addr: *const i32) -> __m256i { + let mut dst: __m256i = src; + asm!( + vpl!("vmovdqa32 {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 32-bit integers from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 32-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_load_epi32) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_maskz_load_epi32(k: __mmask8, mem_addr: *const i32) -> __m256i { + let mut dst: __m256i; + asm!( + vpl!("vmovdqa32 {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 64-bit integers from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 32-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_load_epi64) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_mask_load_epi64(src: __m256i, k: __mmask8, mem_addr: *const i64) -> __m256i { + let mut dst: __m256i = src; + asm!( + vpl!("vmovdqa64 {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 64-bit integers from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 32-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_load_epi64) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_maskz_load_epi64(k: __mmask8, mem_addr: *const i64) -> __m256i { + let mut dst: __m256i; + asm!( + vpl!("vmovdqa64 {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed single-precision (32-bit) floating-point elements from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 32-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_load_ps) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_mask_load_ps(src: __m256, k: __mmask8, mem_addr: *const f32) -> __m256 { + let mut dst: __m256 = src; + asm!( + vpl!("vmovaps {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed single-precision (32-bit) floating-point elements from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 32-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_load_ps) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_maskz_load_ps(k: __mmask8, mem_addr: *const f32) -> __m256 { + let mut dst: __m256; + asm!( + vpl!("vmovaps {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed double-precision (64-bit) floating-point elements from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 32-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_load_pd) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_mask_load_pd(src: __m256d, k: __mmask8, mem_addr: *const f64) -> __m256d { + let mut dst: __m256d = src; + asm!( + vpl!("vmovapd {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed double-precision (64-bit) floating-point elements from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 32-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_load_pd) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_maskz_load_pd(k: __mmask8, mem_addr: *const f64) -> __m256d { + let mut dst: __m256d; + asm!( + vpl!("vmovapd {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 32-bit integers from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 16-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_load_epi32) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_load_epi32(src: __m128i, k: __mmask8, mem_addr: *const i32) -> __m128i { + let mut dst: __m128i = src; + asm!( + vpl!("vmovdqa32 {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 32-bit integers from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 16-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_load_epi32) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_maskz_load_epi32(k: __mmask8, mem_addr: *const i32) -> __m128i { + let mut dst: __m128i; + asm!( + vpl!("vmovdqa32 {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 64-bit integers from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 16-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_load_epi64) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_load_epi64(src: __m128i, k: __mmask8, mem_addr: *const i64) -> __m128i { + let mut dst: __m128i = src; + asm!( + vpl!("vmovdqa64 {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed 64-bit integers from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 16-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_load_epi64) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_maskz_load_epi64(k: __mmask8, mem_addr: *const i64) -> __m128i { + let mut dst: __m128i; + asm!( + vpl!("vmovdqa64 {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed single-precision (32-bit) floating-point elements from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 16-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_load_ps) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_load_ps(src: __m128, k: __mmask8, mem_addr: *const f32) -> __m128 { + let mut dst: __m128 = src; + asm!( + vpl!("vmovaps {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed single-precision (32-bit) floating-point elements from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 16-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_load_ps) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_maskz_load_ps(k: __mmask8, mem_addr: *const f32) -> __m128 { + let mut dst: __m128; + asm!( + vpl!("vmovaps {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed double-precision (64-bit) floating-point elements from memory into dst using writemask k +/// (elements are copied from src when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 16-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_load_pd) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_load_pd(src: __m128d, k: __mmask8, mem_addr: *const f64) -> __m128d { + let mut dst: __m128d = src; + asm!( + vpl!("vmovapd {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load packed double-precision (64-bit) floating-point elements from memory into dst using zeromask k +/// (elements are zeroed out when the corresponding mask bit is not set). +/// mem_addr must be aligned on a 16-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_load_pd) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_maskz_load_pd(k: __mmask8, mem_addr: *const f64) -> __m128d { + let mut dst: __m128d; + asm!( + vpl!("vmovapd {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Store packed 32-bit integers from a into memory using writemask k. +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_storeu_epi32) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_storeu_epi32(mem_addr: *mut i32, mask: __mmask16, a: __m512i) { + asm!( + vps!("vmovdqu32", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(zmm_reg) a, + options(nostack) + ); +} + +/// Store packed 64-bit integers from a into memory using writemask k. +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_storeu_epi64) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_storeu_epi64(mem_addr: *mut i64, mask: __mmask8, a: __m512i) { + asm!( + vps!("vmovdqu64", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(zmm_reg) a, + options(nostack) + ); +} + +/// Store packed single-precision (32-bit) floating-point elements from a into memory using writemask k. +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_storeu_ps) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_storeu_ps(mem_addr: *mut f32, mask: __mmask16, a: __m512) { + asm!( + vps!("vmovups", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(zmm_reg) a, + options(nostack) + ); +} + +/// Store packed double-precision (64-bit) floating-point elements from a into memory using writemask k. +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_storeu_pd) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_storeu_pd(mem_addr: *mut f64, mask: __mmask8, a: __m512d) { + asm!( + vps!("vmovupd", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(zmm_reg) a, + options(nostack) + ); +} + +/// Store packed 32-bit integers from a into memory using writemask k. +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_storeu_epi32) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_mask_storeu_epi32(mem_addr: *mut i32, mask: __mmask8, a: __m256i) { + asm!( + vps!("vmovdqu32", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(ymm_reg) a, + options(nostack) + ); +} + +/// Store packed 64-bit integers from a into memory using writemask k. +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_storeu_epi64) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_mask_storeu_epi64(mem_addr: *mut i64, mask: __mmask8, a: __m256i) { + asm!( + vps!("vmovdqu64", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(ymm_reg) a, + options(nostack) + ); +} + +/// Store packed single-precision (32-bit) floating-point elements from a into memory using writemask k. +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_storeu_ps) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_mask_storeu_ps(mem_addr: *mut f32, mask: __mmask8, a: __m256) { + asm!( + vps!("vmovups", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(ymm_reg) a, + options(nostack) + ); +} + +/// Store packed double-precision (64-bit) floating-point elements from a into memory using writemask k. +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_storeu_pd) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_mask_storeu_pd(mem_addr: *mut f64, mask: __mmask8, a: __m256d) { + asm!( + vps!("vmovupd", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(ymm_reg) a, + options(nostack) + ); +} + +/// Store packed 32-bit integers from a into memory using writemask k. +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_storeu_epi32) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_storeu_epi32(mem_addr: *mut i32, mask: __mmask8, a: __m128i) { + asm!( + vps!("vmovdqu32", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(xmm_reg) a, + options(nostack) + ); +} + +/// Store packed 64-bit integers from a into memory using writemask k. +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_storeu_epi64) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_storeu_epi64(mem_addr: *mut i64, mask: __mmask8, a: __m128i) { + asm!( + vps!("vmovdqu64", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(xmm_reg) a, + options(nostack) + ); +} + +/// Store packed single-precision (32-bit) floating-point elements from a into memory using writemask k. +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_storeu_ps) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_storeu_ps(mem_addr: *mut f32, mask: __mmask8, a: __m128) { + asm!( + vps!("vmovups", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(xmm_reg) a, + options(nostack) + ); +} + +/// Store packed double-precision (64-bit) floating-point elements from a into memory using writemask k. +/// mem_addr does not need to be aligned on any particular boundary. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_storeu_pd) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_storeu_pd(mem_addr: *mut f64, mask: __mmask8, a: __m128d) { + asm!( + vps!("vmovupd", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(xmm_reg) a, + options(nostack) + ); +} + +/// Store packed 32-bit integers from a into memory using writemask k. +/// mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_store_epi32) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_store_epi32(mem_addr: *mut i32, mask: __mmask16, a: __m512i) { + asm!( + vps!("vmovdqa32", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(zmm_reg) a, + options(nostack) + ); +} + +/// Store packed 64-bit integers from a into memory using writemask k. +/// mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_store_epi64) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_store_epi64(mem_addr: *mut i64, mask: __mmask8, a: __m512i) { + asm!( + vps!("vmovdqa64", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(zmm_reg) a, + options(nostack) + ); +} + +/// Store packed single-precision (32-bit) floating-point elements from a into memory using writemask k. +/// mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_store_ps) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_store_ps(mem_addr: *mut f32, mask: __mmask16, a: __m512) { + asm!( + vps!("vmovaps", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(zmm_reg) a, + options(nostack) + ); +} + +/// Store packed double-precision (64-bit) floating-point elements from a into memory using writemask k. +/// mem_addr must be aligned on a 64-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_store_pd) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_store_pd(mem_addr: *mut f64, mask: __mmask8, a: __m512d) { + asm!( + vps!("vmovapd", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(zmm_reg) a, + options(nostack) + ); +} + +/// Store packed 32-bit integers from a into memory using writemask k. +/// mem_addr must be aligned on a 32-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_store_epi32) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_mask_store_epi32(mem_addr: *mut i32, mask: __mmask8, a: __m256i) { + asm!( + vps!("vmovdqa32", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(ymm_reg) a, + options(nostack) + ); +} + +/// Store packed 64-bit integers from a into memory using writemask k. +/// mem_addr must be aligned on a 32-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_store_epi64) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_mask_store_epi64(mem_addr: *mut i64, mask: __mmask8, a: __m256i) { + asm!( + vps!("vmovdqa64", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(ymm_reg) a, + options(nostack) + ); +} + +/// Store packed single-precision (32-bit) floating-point elements from a into memory using writemask k. +/// mem_addr must be aligned on a 32-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_store_ps) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_mask_store_ps(mem_addr: *mut f32, mask: __mmask8, a: __m256) { + asm!( + vps!("vmovaps", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(ymm_reg) a, + options(nostack) + ); +} + +/// Store packed double-precision (64-bit) floating-point elements from a into memory using writemask k. +/// mem_addr must be aligned on a 32-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_store_pd) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_mask_store_pd(mem_addr: *mut f64, mask: __mmask8, a: __m256d) { + asm!( + vps!("vmovapd", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(ymm_reg) a, + options(nostack) + ); +} + +/// Store packed 32-bit integers from a into memory using writemask k. +/// mem_addr must be aligned on a 16-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_store_epi32) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_store_epi32(mem_addr: *mut i32, mask: __mmask8, a: __m128i) { + asm!( + vps!("vmovdqa32", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(xmm_reg) a, + options(nostack) + ); +} + +/// Store packed 64-bit integers from a into memory using writemask k. +/// mem_addr must be aligned on a 16-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_store_epi64) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_store_epi64(mem_addr: *mut i64, mask: __mmask8, a: __m128i) { + asm!( + vps!("vmovdqa64", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(xmm_reg) a, + options(nostack) + ); +} + +/// Store packed single-precision (32-bit) floating-point elements from a into memory using writemask k. +/// mem_addr must be aligned on a 16-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_store_ps) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_store_ps(mem_addr: *mut f32, mask: __mmask8, a: __m128) { + asm!( + vps!("vmovaps", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(xmm_reg) a, + options(nostack) + ); +} + +/// Store packed double-precision (64-bit) floating-point elements from a into memory using writemask k. +/// mem_addr must be aligned on a 16-byte boundary or a general-protection exception may be generated. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_store_pd) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_store_pd(mem_addr: *mut f64, mask: __mmask8, a: __m128d) { + asm!( + vps!("vmovapd", "{{{mask}}}, {a}"), + p = in(reg) mem_addr, + mask = in(kreg) mask, + a = in(xmm_reg) a, + options(nostack) + ); +} + +/// Load contiguous active 32-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_expandloadu_epi32) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_expandloadu_epi32( + src: __m512i, + k: __mmask16, + mem_addr: *const i32, +) -> __m512i { + let mut dst: __m512i = src; + asm!( + vpl!("vpexpandd {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 32-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_expandloadu_epi32) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_maskz_expandloadu_epi32(k: __mmask16, mem_addr: *const i32) -> __m512i { + let mut dst: __m512i; + asm!( + vpl!("vpexpandd {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 32-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_expandloadu_epi32) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_mask_expandloadu_epi32( + src: __m256i, + k: __mmask8, + mem_addr: *const i32, +) -> __m256i { + let mut dst: __m256i = src; + asm!( + vpl!("vpexpandd {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 32-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_expandloadu_epi32) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_maskz_expandloadu_epi32(k: __mmask8, mem_addr: *const i32) -> __m256i { + let mut dst: __m256i; + asm!( + vpl!("vpexpandd {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 32-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_expandloadu_epi32) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_expandloadu_epi32( + src: __m128i, + k: __mmask8, + mem_addr: *const i32, +) -> __m128i { + let mut dst: __m128i = src; + asm!( + vpl!("vpexpandd {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 32-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_expandloadu_epi32) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_maskz_expandloadu_epi32(k: __mmask8, mem_addr: *const i32) -> __m128i { + let mut dst: __m128i; + asm!( + vpl!("vpexpandd {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 64-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_expandloadu_epi64) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_expandloadu_epi64( + src: __m512i, + k: __mmask8, + mem_addr: *const i64, +) -> __m512i { + let mut dst: __m512i = src; + asm!( + vpl!("vpexpandq {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 64-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_expandloadu_epi64) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_maskz_expandloadu_epi64(k: __mmask8, mem_addr: *const i64) -> __m512i { + let mut dst: __m512i; + asm!( + vpl!("vpexpandq {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 64-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_expandloadu_epi64) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_mask_expandloadu_epi64( + src: __m256i, + k: __mmask8, + mem_addr: *const i64, +) -> __m256i { + let mut dst: __m256i = src; + asm!( + vpl!("vpexpandq {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 64-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_expandloadu_epi64) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_maskz_expandloadu_epi64(k: __mmask8, mem_addr: *const i64) -> __m256i { + let mut dst: __m256i; + asm!( + vpl!("vpexpandq {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 64-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_expandloadu_epi64) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_expandloadu_epi64( + src: __m128i, + k: __mmask8, + mem_addr: *const i64, +) -> __m128i { + let mut dst: __m128i = src; + asm!( + vpl!("vpexpandq {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 64-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_expandloadu_epi64) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_maskz_expandloadu_epi64(k: __mmask8, mem_addr: *const i64) -> __m128i { + let mut dst: __m128i; + asm!( + vpl!("vpexpandq {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active single-precision (32-bit) floating-point elements from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_expandloadu_ps) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_expandloadu_ps( + src: __m512, + k: __mmask16, + mem_addr: *const f32, +) -> __m512 { + let mut dst: __m512 = src; + asm!( + vpl!("vexpandps {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active single-precision (32-bit) floating-point elements from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_expandloadu_ps) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_maskz_expandloadu_ps(k: __mmask16, mem_addr: *const f32) -> __m512 { + let mut dst: __m512; + asm!( + vpl!("vexpandps {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active single-precision (32-bit) floating-point elements from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_expandloadu_ps) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_mask_expandloadu_ps(src: __m256, k: __mmask8, mem_addr: *const f32) -> __m256 { + let mut dst: __m256 = src; + asm!( + vpl!("vexpandps {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active single-precision (32-bit) floating-point elements from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_expandloadu_ps) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_maskz_expandloadu_ps(k: __mmask8, mem_addr: *const f32) -> __m256 { + let mut dst: __m256; + asm!( + vpl!("vexpandps {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active single-precision (32-bit) floating-point elements from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_expandloadu_ps) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_expandloadu_ps(src: __m128, k: __mmask8, mem_addr: *const f32) -> __m128 { + let mut dst: __m128 = src; + asm!( + vpl!("vexpandps {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active single-precision (32-bit) floating-point elements from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_expandloadu_ps) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_maskz_expandloadu_ps(k: __mmask8, mem_addr: *const f32) -> __m128 { + let mut dst: __m128; + asm!( + vpl!("vexpandps {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active single-precision (64-bit) floating-point elements from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_expandloadu_pd) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_mask_expandloadu_pd( + src: __m512d, + k: __mmask8, + mem_addr: *const f64, +) -> __m512d { + let mut dst: __m512d = src; + asm!( + vpl!("vexpandpd {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active single-precision (64-bit) floating-point elements from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_expandloadu_pd) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_maskz_expandloadu_pd(k: __mmask8, mem_addr: *const f64) -> __m512d { + let mut dst: __m512d; + asm!( + vpl!("vexpandpd {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active single-precision (64-bit) floating-point elements from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_expandloadu_pd) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_mask_expandloadu_pd( + src: __m256d, + k: __mmask8, + mem_addr: *const f64, +) -> __m256d { + let mut dst: __m256d = src; + asm!( + vpl!("vexpandpd {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active single-precision (64-bit) floating-point elements from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_expandloadu_pd) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx")] +pub unsafe fn _mm256_maskz_expandloadu_pd(k: __mmask8, mem_addr: *const f64) -> __m256d { + let mut dst: __m256d; + asm!( + vpl!("vexpandpd {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active single-precision (64-bit) floating-point elements from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_expandloadu_pd) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_expandloadu_pd(src: __m128d, k: __mmask8, mem_addr: *const f64) -> __m128d { + let mut dst: __m128d = src; + asm!( + vpl!("vexpandpd {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active single-precision (64-bit) floating-point elements from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_expandloadu_pd) +#[inline] +#[target_feature(enable = "avx512f,avx512vl,avx,sse")] +pub unsafe fn _mm_maskz_expandloadu_pd(k: __mmask8, mem_addr: *const f64) -> __m128d { + let mut dst: __m128d; + asm!( + vpl!("vexpandpd {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Set packed double-precision (64-bit) floating-point elements in dst with the supplied values in reverse order. +/// +/// [Intel's documentation]( https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_setr_pd&expand=5002) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_setr_pd( + e0: f64, + e1: f64, + e2: f64, + e3: f64, + e4: f64, + e5: f64, + e6: f64, + e7: f64, +) -> __m512d { + let r = f64x8::new(e0, e1, e2, e3, e4, e5, e6, e7); + transmute(r) +} + +/// Set packed double-precision (64-bit) floating-point elements in dst with the supplied values. +/// +/// [Intel's documentation]( https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_set_pd&expand=4924) +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn _mm512_set_pd( + e0: f64, + e1: f64, + e2: f64, + e3: f64, + e4: f64, + e5: f64, + e6: f64, + e7: f64, +) -> __m512d { + _mm512_setr_pd(e7, e6, e5, e4, e3, e2, e1, e0) +} + +/// Move the lower single-precision (32-bit) floating-point element from b to the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_move_ss&expand=3832) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovss))] +pub unsafe fn _mm_mask_move_ss(src: __m128, k: __mmask8, a: __m128, b: __m128) -> __m128 { + let extractsrc: f32 = simd_extract(src, 0); + let mut mov: f32 = extractsrc; + if (k & 0b00000001) != 0 { + mov = simd_extract(b, 0); + } + let r = simd_insert(a, 0, mov); + transmute(r) +} + +/// Move the lower single-precision (32-bit) floating-point element from b to the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_move_ss&expand=3833) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovss))] +pub unsafe fn _mm_maskz_move_ss(k: __mmask8, a: __m128, b: __m128) -> __m128 { + let mut mov: f32 = 0.; + if (k & 0b00000001) != 0 { + mov = simd_extract(b, 0); + } + let r = simd_insert(a, 0, mov); + transmute(r) +} + +/// Move the lower double-precision (64-bit) floating-point element from b to the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_move_sd&expand=3829) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovsd))] +pub unsafe fn _mm_mask_move_sd(src: __m128d, k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let extractsrc: f64 = simd_extract(src, 0); + let mut mov: f64 = extractsrc; + if (k & 0b00000001) != 0 { + mov = simd_extract(b, 0); + } + let r = simd_insert(a, 0, mov); + transmute(r) +} + +/// Move the lower double-precision (64-bit) floating-point element from b to the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_move_sd&expand=3830) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmovsd))] +pub unsafe fn _mm_maskz_move_sd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let mut mov: f64 = 0.; + if (k & 0b00000001) != 0 { + mov = simd_extract(b, 0); + } + let r = simd_insert(a, 0, mov); + transmute(r) +} + +/// Add the lower single-precision (32-bit) floating-point element in a and b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_add_ss&expand=159) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddss))] +pub unsafe fn _mm_mask_add_ss(src: __m128, k: __mmask8, a: __m128, b: __m128) -> __m128 { + let extractsrc: f32 = simd_extract(src, 0); + let mut add: f32 = extractsrc; + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extractb: f32 = simd_extract(b, 0); + add = extracta + extractb; + } + let r = simd_insert(a, 0, add); + transmute(r) +} + +/// Add the lower single-precision (32-bit) floating-point element in a and b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_add_ss&expand=160) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddss))] +pub unsafe fn _mm_maskz_add_ss(k: __mmask8, a: __m128, b: __m128) -> __m128 { + let mut add: f32 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extractb: f32 = simd_extract(b, 0); + add = extracta + extractb; + } + let r = simd_insert(a, 0, add); + transmute(r) +} + +/// Add the lower double-precision (64-bit) floating-point element in a and b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_add_sd&expand=155) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddsd))] +pub unsafe fn _mm_mask_add_sd(src: __m128d, k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let extractsrc: f64 = simd_extract(src, 0); + let mut add: f64 = extractsrc; + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extractb: f64 = simd_extract(b, 0); + add = extracta + extractb; + } + let r = simd_insert(a, 0, add); + transmute(r) +} + +/// Add the lower double-precision (64-bit) floating-point element in a and b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_add_sd&expand=156) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddsd))] +pub unsafe fn _mm_maskz_add_sd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let mut add: f64 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extractb: f64 = simd_extract(b, 0); + add = extracta + extractb; + } + let r = simd_insert(a, 0, add); + transmute(r) +} + +/// Subtract the lower single-precision (32-bit) floating-point element in b from the lower single-precision (32-bit) floating-point element in a, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_sub_ss&expand=5750) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubss))] +pub unsafe fn _mm_mask_sub_ss(src: __m128, k: __mmask8, a: __m128, b: __m128) -> __m128 { + let extractsrc: f32 = simd_extract(src, 0); + let mut add: f32 = extractsrc; + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extractb: f32 = simd_extract(b, 0); + add = extracta - extractb; + } + let r = simd_insert(a, 0, add); + transmute(r) +} + +/// Subtract the lower single-precision (32-bit) floating-point element in b from the lower single-precision (32-bit) floating-point element in a, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_sub_ss&expand=5751) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubss))] +pub unsafe fn _mm_maskz_sub_ss(k: __mmask8, a: __m128, b: __m128) -> __m128 { + let mut add: f32 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extractb: f32 = simd_extract(b, 0); + add = extracta - extractb; + } + let r = simd_insert(a, 0, add); + transmute(r) +} + +/// Subtract the lower double-precision (64-bit) floating-point element in b from the lower double-precision (64-bit) floating-point element in a, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_sub_sd&expand=5746) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubsd))] +pub unsafe fn _mm_mask_sub_sd(src: __m128d, k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let extractsrc: f64 = simd_extract(src, 0); + let mut add: f64 = extractsrc; + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extractb: f64 = simd_extract(b, 0); + add = extracta - extractb; + } + let r = simd_insert(a, 0, add); + transmute(r) +} + +/// Subtract the lower double-precision (64-bit) floating-point element in b from the lower double-precision (64-bit) floating-point element in a, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_sub_sd&expand=5747) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubsd))] +pub unsafe fn _mm_maskz_sub_sd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let mut add: f64 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extractb: f64 = simd_extract(b, 0); + add = extracta - extractb; + } + let r = simd_insert(a, 0, add); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point element in a and b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_mul_ss&expand=3950) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulss))] +pub unsafe fn _mm_mask_mul_ss(src: __m128, k: __mmask8, a: __m128, b: __m128) -> __m128 { + let extractsrc: f32 = simd_extract(src, 0); + let mut add: f32 = extractsrc; + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extractb: f32 = simd_extract(b, 0); + add = extracta * extractb; + } + let r = simd_insert(a, 0, add); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point element in a and b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_mul_ss&expand=3951) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulss))] +pub unsafe fn _mm_maskz_mul_ss(k: __mmask8, a: __m128, b: __m128) -> __m128 { + let mut add: f32 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extractb: f32 = simd_extract(b, 0); + add = extracta * extractb; + } + let r = simd_insert(a, 0, add); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point element in a and b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_mul_sd&expand=3947) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulsd))] +pub unsafe fn _mm_mask_mul_sd(src: __m128d, k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let extractsrc: f64 = simd_extract(src, 0); + let mut add: f64 = extractsrc; + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extractb: f64 = simd_extract(b, 0); + add = extracta * extractb; + } + let r = simd_insert(a, 0, add); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point element in a and b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_mul_sd&expand=3948) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulsd))] +pub unsafe fn _mm_maskz_mul_sd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let mut add: f64 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extractb: f64 = simd_extract(b, 0); + add = extracta * extractb; + } + let r = simd_insert(a, 0, add); + transmute(r) +} + +/// Divide the lower single-precision (32-bit) floating-point element in a by the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_div_ss&expand=2181) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivss))] +pub unsafe fn _mm_mask_div_ss(src: __m128, k: __mmask8, a: __m128, b: __m128) -> __m128 { + let extractsrc: f32 = simd_extract(src, 0); + let mut add: f32 = extractsrc; + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extractb: f32 = simd_extract(b, 0); + add = extracta / extractb; + } + let r = simd_insert(a, 0, add); + transmute(r) +} + +/// Divide the lower single-precision (32-bit) floating-point element in a by the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_div_ss&expand=2182) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivss))] +pub unsafe fn _mm_maskz_div_ss(k: __mmask8, a: __m128, b: __m128) -> __m128 { + let mut add: f32 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extractb: f32 = simd_extract(b, 0); + add = extracta / extractb; + } + let r = simd_insert(a, 0, add); + transmute(r) +} + +/// Divide the lower double-precision (64-bit) floating-point element in a by the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_div_sd&expand=2178) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivsd))] +pub unsafe fn _mm_mask_div_sd(src: __m128d, k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let extractsrc: f64 = simd_extract(src, 0); + let mut add: f64 = extractsrc; + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extractb: f64 = simd_extract(b, 0); + add = extracta / extractb; + } + let r = simd_insert(a, 0, add); + transmute(r) +} + +/// Divide the lower double-precision (64-bit) floating-point element in a by the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_div_sd&expand=2179) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivsd))] +pub unsafe fn _mm_maskz_div_sd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + let mut add: f64 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extractb: f64 = simd_extract(b, 0); + add = extracta / extractb; + } + let r = simd_insert(a, 0, add); + transmute(r) +} + +/// Compare the lower single-precision (32-bit) floating-point elements in a and b, store the maximum value in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_max_ss&expand=3672) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxss))] +pub unsafe fn _mm_mask_max_ss(src: __m128, k: __mmask8, a: __m128, b: __m128) -> __m128 { + transmute(vmaxss( + a.as_f32x4(), + b.as_f32x4(), + src.as_f32x4(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Compare the lower single-precision (32-bit) floating-point elements in a and b, store the maximum value in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_max_ss&expand=3673) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxss))] +pub unsafe fn _mm_maskz_max_ss(k: __mmask8, a: __m128, b: __m128) -> __m128 { + transmute(vmaxss( + a.as_f32x4(), + b.as_f32x4(), + _mm_setzero_ps().as_f32x4(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Compare the lower double-precision (64-bit) floating-point elements in a and b, store the maximum value in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_max_sd&expand=3669) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxsd))] +pub unsafe fn _mm_mask_max_sd(src: __m128d, k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + transmute(vmaxsd( + a.as_f64x2(), + b.as_f64x2(), + src.as_f64x2(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Compare the lower double-precision (64-bit) floating-point elements in a and b, store the maximum value in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_max_sd&expand=3670) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxsd))] +pub unsafe fn _mm_maskz_max_sd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + transmute(vmaxsd( + a.as_f64x2(), + b.as_f64x2(), + _mm_setzero_pd().as_f64x2(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Compare the lower single-precision (32-bit) floating-point elements in a and b, store the minimum value in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_min_ss&expand=3786) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminss))] +pub unsafe fn _mm_mask_min_ss(src: __m128, k: __mmask8, a: __m128, b: __m128) -> __m128 { + transmute(vminss( + a.as_f32x4(), + b.as_f32x4(), + src.as_f32x4(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Compare the lower single-precision (32-bit) floating-point elements in a and b, store the minimum value in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_min_ss&expand=3787) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminss))] +pub unsafe fn _mm_maskz_min_ss(k: __mmask8, a: __m128, b: __m128) -> __m128 { + transmute(vminss( + a.as_f32x4(), + b.as_f32x4(), + _mm_setzero_ps().as_f32x4(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Compare the lower double-precision (64-bit) floating-point elements in a and b, store the minimum value in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_min_sd&expand=3783) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminsd))] +pub unsafe fn _mm_mask_min_sd(src: __m128d, k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + transmute(vminsd( + a.as_f64x2(), + b.as_f64x2(), + src.as_f64x2(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Compare the lower double-precision (64-bit) floating-point elements in a and b, store the minimum value in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_min_sd&expand=3784) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminsd))] +pub unsafe fn _mm_maskz_min_sd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + transmute(vminsd( + a.as_f64x2(), + b.as_f64x2(), + _mm_setzero_pd().as_f64x2(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Compute the square root of the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_sqrt_ss&expand=5387) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtss))] +pub unsafe fn _mm_mask_sqrt_ss(src: __m128, k: __mmask8, a: __m128, b: __m128) -> __m128 { + transmute(vsqrtss( + a.as_f32x4(), + b.as_f32x4(), + src.as_f32x4(), + k, + _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC, + )) +} + +/// Compute the square root of the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_sqrt_ss&expand=5388) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtss))] +pub unsafe fn _mm_maskz_sqrt_ss(k: __mmask8, a: __m128, b: __m128) -> __m128 { + transmute(vsqrtss( + a.as_f32x4(), + b.as_f32x4(), + _mm_setzero_ps().as_f32x4(), + k, + _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC, + )) +} + +/// Compute the square root of the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_sqrt_sd&expand=5384) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtsd))] +pub unsafe fn _mm_mask_sqrt_sd(src: __m128d, k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + transmute(vsqrtsd( + a.as_f64x2(), + b.as_f64x2(), + src.as_f64x2(), + k, + _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC, + )) +} + +/// Compute the square root of the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_sqrt_sd&expand=5385) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtsd))] +pub unsafe fn _mm_maskz_sqrt_sd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + transmute(vsqrtsd( + a.as_f64x2(), + b.as_f64x2(), + _mm_setzero_pd().as_f64x2(), + k, + _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC, + )) +} + +/// Compute the approximate reciprocal square root of the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst. The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_rsqrt14_ss&expand=4825) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrsqrt14ss))] +pub unsafe fn _mm_rsqrt14_ss(a: __m128, b: __m128) -> __m128 { + transmute(vrsqrt14ss( + a.as_f32x4(), + b.as_f32x4(), + _mm_setzero_ps().as_f32x4(), + 0b1, + )) +} + +/// Compute the approximate reciprocal square root of the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_rsqrt14_ss&expand=4823) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrsqrt14ss))] +pub unsafe fn _mm_mask_rsqrt14_ss(src: __m128, k: __mmask8, a: __m128, b: __m128) -> __m128 { + transmute(vrsqrt14ss(a.as_f32x4(), b.as_f32x4(), src.as_f32x4(), k)) +} + +/// Compute the approximate reciprocal square root of the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_rsqrt14_ss&expand=4824) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrsqrt14ss))] +pub unsafe fn _mm_maskz_rsqrt14_ss(k: __mmask8, a: __m128, b: __m128) -> __m128 { + transmute(vrsqrt14ss( + a.as_f32x4(), + b.as_f32x4(), + _mm_setzero_ps().as_f32x4(), + k, + )) +} + +/// Compute the approximate reciprocal square root of the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst. The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_rsqrt14_sd&expand=4822) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrsqrt14sd))] +pub unsafe fn _mm_rsqrt14_sd(a: __m128d, b: __m128d) -> __m128d { + transmute(vrsqrt14sd( + a.as_f64x2(), + b.as_f64x2(), + _mm_setzero_pd().as_f64x2(), + 0b1, + )) +} + +/// Compute the approximate reciprocal square root of the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_rsqrt14_sd&expand=4820) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrsqrt14sd))] +pub unsafe fn _mm_mask_rsqrt14_sd(src: __m128d, k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + transmute(vrsqrt14sd(a.as_f64x2(), b.as_f64x2(), src.as_f64x2(), k)) +} + +/// Compute the approximate reciprocal square root of the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_rsqrt14_sd&expand=4821) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrsqrt14sd))] +pub unsafe fn _mm_maskz_rsqrt14_sd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + transmute(vrsqrt14sd( + a.as_f64x2(), + b.as_f64x2(), + _mm_setzero_pd().as_f64x2(), + k, + )) +} + +/// Compute the approximate reciprocal of the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst. The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_rcp14_ss&expand=4508) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrcp14ss))] +pub unsafe fn _mm_rcp14_ss(a: __m128, b: __m128) -> __m128 { + transmute(vrcp14ss( + a.as_f32x4(), + b.as_f32x4(), + _mm_setzero_ps().as_f32x4(), + 0b1, + )) +} + +/// Compute the approximate reciprocal of the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_rcp14_ss&expand=4506) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrcp14ss))] +pub unsafe fn _mm_mask_rcp14_ss(src: __m128, k: __mmask8, a: __m128, b: __m128) -> __m128 { + transmute(vrcp14ss(a.as_f32x4(), b.as_f32x4(), src.as_f32x4(), k)) +} + +/// Compute the approximate reciprocal of the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_rcp14_ss&expand=4507) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrcp14ss))] +pub unsafe fn _mm_maskz_rcp14_ss(k: __mmask8, a: __m128, b: __m128) -> __m128 { + transmute(vrcp14ss( + a.as_f32x4(), + b.as_f32x4(), + _mm_setzero_ps().as_f32x4(), + k, + )) +} + +/// Compute the approximate reciprocal of the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst. The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_rcp14_sd&expand=4505) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrcp14sd))] +pub unsafe fn _mm_rcp14_sd(a: __m128d, b: __m128d) -> __m128d { + transmute(vrcp14sd( + a.as_f64x2(), + b.as_f64x2(), + _mm_setzero_pd().as_f64x2(), + 0b1, + )) +} + +/// Compute the approximate reciprocal of the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_rcp14_sd&expand=4503) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrcp14sd))] +pub unsafe fn _mm_mask_rcp14_sd(src: __m128d, k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + transmute(vrcp14sd(a.as_f64x2(), b.as_f64x2(), src.as_f64x2(), k)) +} + +/// Compute the approximate reciprocal of the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. The maximum relative error for this approximation is less than 2^-14. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_rcp14_sd&expand=4504) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrcp14sd))] +pub unsafe fn _mm_maskz_rcp14_sd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + transmute(vrcp14sd( + a.as_f64x2(), + b.as_f64x2(), + _mm_setzero_pd().as_f64x2(), + k, + )) +} + +/// Convert the exponent of the lower single-precision (32-bit) floating-point element in b to a single-precision (32-bit) floating-point number representing the integer exponent, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst. This intrinsic essentially calculates floor(log2(x)) for the lower element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_getexp_ss&expand=2862) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexpss))] +pub unsafe fn _mm_getexp_ss(a: __m128, b: __m128) -> __m128 { + transmute(vgetexpss( + a.as_f32x4(), + b.as_f32x4(), + _mm_setzero_ps().as_f32x4(), + 0b1, + _MM_FROUND_NO_EXC, + )) +} + +/// Convert the exponent of the lower single-precision (32-bit) floating-point element in b to a single-precision (32-bit) floating-point number representing the integer exponent, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. This intrinsic essentially calculates floor(log2(x)) for the lower element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_getexp_ss&expand=2863) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexpss))] +pub unsafe fn _mm_mask_getexp_ss(src: __m128, k: __mmask8, a: __m128, b: __m128) -> __m128 { + transmute(vgetexpss( + a.as_f32x4(), + b.as_f32x4(), + src.as_f32x4(), + k, + _MM_FROUND_NO_EXC, + )) +} + +/// Convert the exponent of the lower single-precision (32-bit) floating-point element in b to a single-precision (32-bit) floating-point number representing the integer exponent, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. This intrinsic essentially calculates floor(log2(x)) for the lower element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_getexp_ss&expand=2864) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexpss))] +pub unsafe fn _mm_maskz_getexp_ss(k: __mmask8, a: __m128, b: __m128) -> __m128 { + transmute(vgetexpss( + a.as_f32x4(), + b.as_f32x4(), + _mm_setzero_ps().as_f32x4(), + k, + _MM_FROUND_NO_EXC, + )) +} + +/// Convert the exponent of the lower double-precision (64-bit) floating-point element in b to a double-precision (64-bit) floating-point number representing the integer exponent, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst. This intrinsic essentially calculates floor(log2(x)) for the lower element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_getexp_sd&expand=2859) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexpsd))] +pub unsafe fn _mm_getexp_sd(a: __m128d, b: __m128d) -> __m128d { + transmute(vgetexpsd( + a.as_f64x2(), + b.as_f64x2(), + _mm_setzero_pd().as_f64x2(), + 0b1, + _MM_FROUND_NO_EXC, + )) +} + +/// Convert the exponent of the lower double-precision (64-bit) floating-point element in b to a double-precision (64-bit) floating-point number representing the integer exponent, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. This intrinsic essentially calculates floor(log2(x)) for the lower element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_getexp_sd&expand=2860) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexpsd))] +pub unsafe fn _mm_mask_getexp_sd(src: __m128d, k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + transmute(vgetexpsd( + a.as_f64x2(), + b.as_f64x2(), + src.as_f64x2(), + k, + _MM_FROUND_NO_EXC, + )) +} + +/// Convert the exponent of the lower double-precision (64-bit) floating-point element in b to a double-precision (64-bit) floating-point number representing the integer exponent, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. This intrinsic essentially calculates floor(log2(x)) for the lower element. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_getexp_sd&expand=2861) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexpsd))] +pub unsafe fn _mm_maskz_getexp_sd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + transmute(vgetexpsd( + a.as_f64x2(), + b.as_f64x2(), + _mm_setzero_pd().as_f64x2(), + k, + _MM_FROUND_NO_EXC, + )) +} + +/// Normalize the mantissas of the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst. This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_getmant_ss&expand=2898) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantss, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm_getmant_ss< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + a: __m128, + b: __m128, +) -> __m128 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vgetmantss(a, b, SIGN << 2 | NORM, zero, 0b1, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Normalize the mantissas of the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_getmant_ss&expand=2899) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantss, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm_mask_getmant_ss< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + src: __m128, + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let src = src.as_f32x4(); + let r = vgetmantss(a, b, SIGN << 2 | NORM, src, k, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Normalize the mantissas of the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_getmant_ss&expand=2900) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantss, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm_maskz_getmant_ss< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vgetmantss(a, b, SIGN << 2 | NORM, zero, k, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Normalize the mantissas of the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst. This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_getmant_sd&expand=2895) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantsd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm_getmant_sd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vgetmantsd(a, b, SIGN << 2 | NORM, zero, 0b1, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Normalize the mantissas of the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_getmant_sd&expand=2896) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantsd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm_mask_getmant_sd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + src: __m128d, + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let src = src.as_f64x2(); + let r = vgetmantsd(a, b, SIGN << 2 | NORM, src, k, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Normalize the mantissas of the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_getmant_sd&expand=2897) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantsd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm_maskz_getmant_sd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vgetmantsd(a, b, SIGN << 2 | NORM, zero, k, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Round the lower single-precision (32-bit) floating-point element in b to the number of fraction bits specified by imm8, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_roundscale_ss&expand=4802) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscaless, IMM8 = 255))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_roundscale_ss(a: __m128, b: __m128) -> __m128 { + static_assert_imm8!(IMM8); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vrndscaless(a, b, zero, 0b11111111, IMM8, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Round the lower single-precision (32-bit) floating-point element in b to the number of fraction bits specified by imm8, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_roundscale_ss&expand=4800) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscaless, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_roundscale_ss( + src: __m128, + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_imm8!(IMM8); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let src = src.as_f32x4(); + let r = vrndscaless(a, b, src, k, IMM8, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Round the lower single-precision (32-bit) floating-point element in b to the number of fraction bits specified by imm8, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_roundscale_ss&expand=4801) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscaless, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_roundscale_ss( + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_imm8!(IMM8); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vrndscaless(a, b, zero, k, IMM8, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Round the lower double-precision (64-bit) floating-point element in b to the number of fraction bits specified by imm8, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst.\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_roundscale_sd&expand=4799) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscalesd, IMM8 = 255))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_roundscale_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_imm8!(IMM8); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vrndscalesd(a, b, zero, 0b11111111, IMM8, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Round the lower double-precision (64-bit) floating-point element in b to the number of fraction bits specified by imm8, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_roundscale_sd&expand=4797) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscalesd, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_roundscale_sd( + src: __m128d, + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_imm8!(IMM8); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let src = src.as_f64x2(); + let r = vrndscalesd(a, b, src, k, IMM8, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Round the lower double-precision (64-bit) floating-point element in b to the number of fraction bits specified by imm8, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_roundscale_sd&expand=4798) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscalesd, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_roundscale_sd( + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_imm8!(IMM8); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vrndscalesd(a, b, zero, k, IMM8, _MM_FROUND_CUR_DIRECTION); + transmute(r) +} + +/// Scale the packed single-precision (32-bit) floating-point elements in a using values from b, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_scalef_ss&expand=4901) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefss))] +pub unsafe fn _mm_scalef_ss(a: __m128, b: __m128) -> __m128 { + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + transmute(vscalefss(a, b, zero, 0b11111111, _MM_FROUND_CUR_DIRECTION)) +} + +/// Scale the packed single-precision (32-bit) floating-point elements in a using values from b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_scalef_ss&expand=4899) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefss))] +pub unsafe fn _mm_mask_scalef_ss(src: __m128, k: __mmask8, a: __m128, b: __m128) -> __m128 { + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let src = src.as_f32x4(); + transmute(vscalefss(a, b, src, k, _MM_FROUND_CUR_DIRECTION)) +} + +/// Scale the packed single-precision (32-bit) floating-point elements in a using values from b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_scalef_ss&expand=4900) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefss))] +pub unsafe fn _mm_maskz_scalef_ss(k: __mmask8, a: __m128, b: __m128) -> __m128 { + transmute(vscalefss( + a.as_f32x4(), + b.as_f32x4(), + _mm_setzero_ps().as_f32x4(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Scale the packed double-precision (64-bit) floating-point elements in a using values from b, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_scalef_sd&expand=4898) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefsd))] +pub unsafe fn _mm_scalef_sd(a: __m128d, b: __m128d) -> __m128d { + transmute(vscalefsd( + a.as_f64x2(), + b.as_f64x2(), + _mm_setzero_pd().as_f64x2(), + 0b11111111, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Scale the packed double-precision (64-bit) floating-point elements in a using values from b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_scalef_sd&expand=4896) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefsd))] +pub unsafe fn _mm_mask_scalef_sd(src: __m128d, k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + transmute(vscalefsd( + a.as_f64x2(), + b.as_f64x2(), + src.as_f64x2(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Scale the packed double-precision (64-bit) floating-point elements in a using values from b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_scalef_sd&expand=4897) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefsd))] +pub unsafe fn _mm_maskz_scalef_sd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d { + transmute(vscalefsd( + a.as_f64x2(), + b.as_f64x2(), + _mm_setzero_pd().as_f64x2(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and add the intermediate result to the lower element in c. Store the result in the lower element of dst using writemask k (the element is copied from a when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fmadd_ss&expand=2582) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd213ss))] +pub unsafe fn _mm_mask_fmadd_ss(a: __m128, k: __mmask8, b: __m128, c: __m128) -> __m128 { + let mut fmadd: f32 = simd_extract(a, 0); + if (k & 0b00000001) != 0 { + let extractb: f32 = simd_extract(b, 0); + let extractc: f32 = simd_extract(c, 0); + fmadd = vfmadd132ss(fmadd, extractb, extractc, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(a, 0, fmadd); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and add the intermediate result to the lower element in c. Store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fmadd_ss&expand=2584) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd213ss))] +pub unsafe fn _mm_maskz_fmadd_ss(k: __mmask8, a: __m128, b: __m128, c: __m128) -> __m128 { + let mut fmadd: f32 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extractb: f32 = simd_extract(b, 0); + let extractc: f32 = simd_extract(c, 0); + fmadd = vfmadd132ss(extracta, extractb, extractc, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(a, 0, fmadd); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and add the intermediate result to the lower element in c. Store the result in the lower element of dst using writemask k (the element is copied from c when mask bit 0 is not set), and copy the upper 3 packed elements from c to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fmadd_ss&expand=2583) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd213ss))] +pub unsafe fn _mm_mask3_fmadd_ss(a: __m128, b: __m128, c: __m128, k: __mmask8) -> __m128 { + let mut fmadd: f32 = simd_extract(c, 0); + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extractb: f32 = simd_extract(b, 0); + fmadd = vfmadd132ss(extracta, extractb, fmadd, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(c, 0, fmadd); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and add the intermediate result to the lower element in c. Store the result in the lower element of dst using writemask k (the element is copied from a when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fmadd_sd&expand=2578) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd213sd))] +pub unsafe fn _mm_mask_fmadd_sd(a: __m128d, k: __mmask8, b: __m128d, c: __m128d) -> __m128d { + let mut fmadd: f64 = simd_extract(a, 0); + if (k & 0b00000001) != 0 { + let extractb: f64 = simd_extract(b, 0); + let extractc: f64 = simd_extract(c, 0); + fmadd = vfmadd132sd(fmadd, extractb, extractc, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(a, 0, fmadd); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and add the intermediate result to the lower element in c. Store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fmadd_sd&expand=2580) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd213sd))] +pub unsafe fn _mm_maskz_fmadd_sd(k: __mmask8, a: __m128d, b: __m128d, c: __m128d) -> __m128d { + let mut fmadd: f64 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extractb: f64 = simd_extract(b, 0); + let extractc: f64 = simd_extract(c, 0); + fmadd = vfmadd132sd(extracta, extractb, extractc, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(a, 0, fmadd); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and add the intermediate result to the lower element in c. Store the result in the lower element of dst using writemask k (the element is copied from c when mask bit 0 is not set), and copy the upper element from c to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fmadd_sd&expand=2579) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd213sd))] +pub unsafe fn _mm_mask3_fmadd_sd(a: __m128d, b: __m128d, c: __m128d, k: __mmask8) -> __m128d { + let mut fmadd: f64 = simd_extract(c, 0); + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extractb: f64 = simd_extract(b, 0); + fmadd = vfmadd132sd(extracta, extractb, fmadd, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(c, 0, fmadd); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and subtract the lower element in c from the intermediate result. Store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fmsub_ss&expand=2668) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmsub213ss))] +pub unsafe fn _mm_mask_fmsub_ss(a: __m128, k: __mmask8, b: __m128, c: __m128) -> __m128 { + let mut fmsub: f32 = simd_extract(a, 0); + if (k & 0b00000001) != 0 { + let extractb: f32 = simd_extract(b, 0); + let extractc: f32 = simd_extract(c, 0); + let extractc = -extractc; + fmsub = vfmadd132ss(fmsub, extractb, extractc, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(a, 0, fmsub); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and subtract the lower element in c from the intermediate result. Store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fmsub_ss&expand=2670) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmsub213ss))] +pub unsafe fn _mm_maskz_fmsub_ss(k: __mmask8, a: __m128, b: __m128, c: __m128) -> __m128 { + let mut fmsub: f32 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extractb: f32 = simd_extract(b, 0); + let extractc: f32 = simd_extract(c, 0); + let extractc = -extractc; + fmsub = vfmadd132ss(extracta, extractb, extractc, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(a, 0, fmsub); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and subtract the lower element in c from the intermediate result. Store the result in the lower element of dst using writemask k (the element is copied from c when mask bit 0 is not set), and copy the upper 3 packed elements from c to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fmsub_ss&expand=2669) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmsub213ss))] +pub unsafe fn _mm_mask3_fmsub_ss(a: __m128, b: __m128, c: __m128, k: __mmask8) -> __m128 { + let mut fmsub: f32 = simd_extract(c, 0); + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extractb: f32 = simd_extract(b, 0); + let extractc = -fmsub; + fmsub = vfmadd132ss(extracta, extractb, extractc, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(c, 0, fmsub); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and subtract the lower element in c from the intermediate result. Store the result in the lower element of dst using writemask k (the element is copied from a when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fmsub_sd&expand=2664) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmsub213sd))] +pub unsafe fn _mm_mask_fmsub_sd(a: __m128d, k: __mmask8, b: __m128d, c: __m128d) -> __m128d { + let mut fmsub: f64 = simd_extract(a, 0); + if (k & 0b00000001) != 0 { + let extractb: f64 = simd_extract(b, 0); + let extractc: f64 = simd_extract(c, 0); + let extractc = -extractc; + fmsub = vfmadd132sd(fmsub, extractb, extractc, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(a, 0, fmsub); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and subtract the lower element in c from the intermediate result. Store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fmsub_sd&expand=2666) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmsub213sd))] +pub unsafe fn _mm_maskz_fmsub_sd(k: __mmask8, a: __m128d, b: __m128d, c: __m128d) -> __m128d { + let mut fmsub: f64 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extractb: f64 = simd_extract(b, 0); + let extractc: f64 = simd_extract(c, 0); + let extractc = -extractc; + fmsub = vfmadd132sd(extracta, extractb, extractc, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(a, 0, fmsub); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and subtract the lower element in c from the intermediate result. Store the result in the lower element of dst using writemask k (the element is copied from c when mask bit 0 is not set), and copy the upper element from c to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fmsub_sd&expand=2665) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmsub213sd))] +pub unsafe fn _mm_mask3_fmsub_sd(a: __m128d, b: __m128d, c: __m128d, k: __mmask8) -> __m128d { + let mut fmsub: f64 = simd_extract(c, 0); + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extractb: f64 = simd_extract(b, 0); + let extractc = -fmsub; + fmsub = vfmadd132sd(extracta, extractb, extractc, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(c, 0, fmsub); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and add the negated intermediate result to the lower element in c. Store the result in the lower element of dst using writemask k (the element is copied from a when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fnmadd_ss&expand=2748) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmadd213ss))] +pub unsafe fn _mm_mask_fnmadd_ss(a: __m128, k: __mmask8, b: __m128, c: __m128) -> __m128 { + let mut fnmadd: f32 = simd_extract(a, 0); + if (k & 0b00000001) != 0 { + let extracta = -fnmadd; + let extractb: f32 = simd_extract(b, 0); + let extractc: f32 = simd_extract(c, 0); + fnmadd = vfmadd132ss(extracta, extractb, extractc, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(a, 0, fnmadd); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and add the negated intermediate result to the lower element in c. Store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fnmadd_ss&expand=2750) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmadd213ss))] +pub unsafe fn _mm_maskz_fnmadd_ss(k: __mmask8, a: __m128, b: __m128, c: __m128) -> __m128 { + let mut fnmadd: f32 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extracta = -extracta; + let extractb: f32 = simd_extract(b, 0); + let extractc: f32 = simd_extract(c, 0); + fnmadd = vfmadd132ss(extracta, extractb, extractc, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(a, 0, fnmadd); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and add the negated intermediate result to the lower element in c. Store the result in the lower element of dst using writemask k (the element is copied from c when mask bit 0 is not set), and copy the upper 3 packed elements from c to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fnmadd_ss&expand=2749) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmadd213ss))] +pub unsafe fn _mm_mask3_fnmadd_ss(a: __m128, b: __m128, c: __m128, k: __mmask8) -> __m128 { + let mut fnmadd: f32 = simd_extract(c, 0); + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extracta = -extracta; + let extractb: f32 = simd_extract(b, 0); + fnmadd = vfmadd132ss(extracta, extractb, fnmadd, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(c, 0, fnmadd); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and add the negated intermediate result to the lower element in c. Store the result in the lower element of dst using writemask k (the element is copied from a when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fnmadd_sd&expand=2744) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmadd213sd))] +pub unsafe fn _mm_mask_fnmadd_sd(a: __m128d, k: __mmask8, b: __m128d, c: __m128d) -> __m128d { + let mut fnmadd: f64 = simd_extract(a, 0); + if (k & 0b00000001) != 0 { + let extracta = -fnmadd; + let extractb: f64 = simd_extract(b, 0); + let extractc: f64 = simd_extract(c, 0); + fnmadd = vfmadd132sd(extracta, extractb, extractc, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(a, 0, fnmadd); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and add the negated intermediate result to the lower element in c. Store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fnmadd_sd&expand=2746) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmadd213sd))] +pub unsafe fn _mm_maskz_fnmadd_sd(k: __mmask8, a: __m128d, b: __m128d, c: __m128d) -> __m128d { + let mut fnmadd: f64 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extracta = -extracta; + let extractb: f64 = simd_extract(b, 0); + let extractc: f64 = simd_extract(c, 0); + fnmadd = vfmadd132sd(extracta, extractb, extractc, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(a, 0, fnmadd); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and add the negated intermediate result to the lower element in c. Store the result in the lower element of dst using writemask k (the element is copied from c when mask bit 0 is not set), and copy the upper element from c to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fnmadd_sd&expand=2745) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmadd213sd))] +pub unsafe fn _mm_mask3_fnmadd_sd(a: __m128d, b: __m128d, c: __m128d, k: __mmask8) -> __m128d { + let mut fnmadd: f64 = simd_extract(c, 0); + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extracta = -extracta; + let extractb: f64 = simd_extract(b, 0); + fnmadd = vfmadd132sd(extracta, extractb, fnmadd, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(c, 0, fnmadd); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and subtract the lower element in c from the negated intermediate result. Store the result in the lower element of dst using writemask k (the element is copied from c when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fnmsub_ss&expand=2796) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmsub213ss))] +pub unsafe fn _mm_mask_fnmsub_ss(a: __m128, k: __mmask8, b: __m128, c: __m128) -> __m128 { + let mut fnmsub: f32 = simd_extract(a, 0); + if (k & 0b00000001) != 0 { + let extracta = -fnmsub; + let extractb: f32 = simd_extract(b, 0); + let extractc: f32 = simd_extract(c, 0); + let extractc = -extractc; + fnmsub = vfmadd132ss(extracta, extractb, extractc, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(a, 0, fnmsub); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and subtract the lower element in c from the negated intermediate result. Store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fnmsub_ss&expand=2798) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmsub213ss))] +pub unsafe fn _mm_maskz_fnmsub_ss(k: __mmask8, a: __m128, b: __m128, c: __m128) -> __m128 { + let mut fnmsub: f32 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extracta = -extracta; + let extractb: f32 = simd_extract(b, 0); + let extractc: f32 = simd_extract(c, 0); + let extractc = -extractc; + fnmsub = vfmadd132ss(extracta, extractb, extractc, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(a, 0, fnmsub); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and subtract the lower element in c from the negated intermediate result. Store the result in the lower element of dst using writemask k (the element is copied from c when mask bit 0 is not set), and copy the upper 3 packed elements from c to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fnmsub_ss&expand=2797) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmsub213ss))] +pub unsafe fn _mm_mask3_fnmsub_ss(a: __m128, b: __m128, c: __m128, k: __mmask8) -> __m128 { + let mut fnmsub: f32 = simd_extract(c, 0); + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extracta = -extracta; + let extractb: f32 = simd_extract(b, 0); + let extractc = -fnmsub; + fnmsub = vfmadd132ss(extracta, extractb, extractc, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(c, 0, fnmsub); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and subtract the lower element in c from the negated intermediate result. Store the result in the lower element of dst using writemask k (the element is copied from c when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fnmsub_sd&expand=2792) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmsub213sd))] +pub unsafe fn _mm_mask_fnmsub_sd(a: __m128d, k: __mmask8, b: __m128d, c: __m128d) -> __m128d { + let mut fnmsub: f64 = simd_extract(a, 0); + if (k & 0b00000001) != 0 { + let extracta = -fnmsub; + let extractb: f64 = simd_extract(b, 0); + let extractc: f64 = simd_extract(c, 0); + let extractc = -extractc; + fnmsub = vfmadd132sd(extracta, extractb, extractc, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(a, 0, fnmsub); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and subtract the lower element in c from the negated intermediate result. Store the result in dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fnmsub_sd&expand=2794) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmsub213sd))] +pub unsafe fn _mm_maskz_fnmsub_sd(k: __mmask8, a: __m128d, b: __m128d, c: __m128d) -> __m128d { + let mut fnmsub: f64 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extracta = -extracta; + let extractb: f64 = simd_extract(b, 0); + let extractc: f64 = simd_extract(c, 0); + let extractc = -extractc; + fnmsub = vfmadd132sd(extracta, extractb, extractc, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(a, 0, fnmsub); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and subtract the lower element in c from the negated intermediate result. Store the result in the lower element of dst using writemask k (the element is copied from c when mask bit 0 is not set), and copy the upper element from c to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fnmsub_sd&expand=2793) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmsub213sd))] +pub unsafe fn _mm_mask3_fnmsub_sd(a: __m128d, b: __m128d, c: __m128d, k: __mmask8) -> __m128d { + let mut fnmsub: f64 = simd_extract(c, 0); + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extracta = -extracta; + let extractb: f64 = simd_extract(b, 0); + let extractc = -fnmsub; + fnmsub = vfmadd132sd(extracta, extractb, extractc, _MM_FROUND_CUR_DIRECTION); + } + let r = simd_insert(c, 0, fnmsub); + transmute(r) +} + +/// Add the lower single-precision (32-bit) floating-point element in a and b, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_add_round_ss&expand=151) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_add_round_ss(a: __m128, b: __m128) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vaddss(a, b, zero, 0b1, ROUNDING); + transmute(r) +} + +/// Add the lower single-precision (32-bit) floating-point element in a and b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_add_round_ss&expand=152) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_add_round_ss( + src: __m128, + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let src = src.as_f32x4(); + let r = vaddss(a, b, src, k, ROUNDING); + transmute(r) +} + +/// Add the lower single-precision (32-bit) floating-point element in a and b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_add_round_ss&expand=153) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_add_round_ss( + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vaddss(a, b, zero, k, ROUNDING); + transmute(r) +} + +/// Add the lower double-precision (64-bit) floating-point element in a and b, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_add_round_sd&expand=148) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_add_round_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vaddsd(a, b, zero, 0b1, ROUNDING); + transmute(r) +} + +/// Add the lower double-precision (64-bit) floating-point element in a and b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_add_round_Sd&expand=149) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_add_round_sd( + src: __m128d, + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let src = src.as_f64x2(); + let r = vaddsd(a, b, src, k, ROUNDING); + transmute(r) +} + +/// Add the lower double-precision (64-bit) floating-point element in a and b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_add_round_sd&expand=150) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vaddsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_add_round_sd( + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vaddsd(a, b, zero, k, ROUNDING); + transmute(r) +} + +/// Subtract the lower single-precision (32-bit) floating-point element in b from the lower single-precision (32-bit) floating-point element in a, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_sub_round_ss&expand=5745) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_sub_round_ss(a: __m128, b: __m128) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vsubss(a, b, zero, 0b1, ROUNDING); + transmute(r) +} + +/// Subtract the lower single-precision (32-bit) floating-point element in b from the lower single-precision (32-bit) floating-point element in a, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_sub_round_ss&expand=5743) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_sub_round_ss( + src: __m128, + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let src = src.as_f32x4(); + let r = vsubss(a, b, src, k, ROUNDING); + transmute(r) +} + +/// Subtract the lower single-precision (32-bit) floating-point element in b from the lower single-precision (32-bit) floating-point element in a, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_sub_round_ss&expand=5744) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_sub_round_ss( + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vsubss(a, b, zero, k, ROUNDING); + transmute(r) +} + +/// Subtract the lower double-precision (64-bit) floating-point element in b from the lower double-precision (64-bit) floating-point element in a, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_sub_round_sd&expand=5742) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_sub_round_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vsubsd(a, b, zero, 0b1, ROUNDING); + transmute(r) +} + +/// Subtract the lower double-precision (64-bit) floating-point element in b from the lower double-precision (64-bit) floating-point element in a, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_sub_round_sd&expand=5740) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_sub_round_sd( + src: __m128d, + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let src = src.as_f64x2(); + let r = vsubsd(a, b, src, k, ROUNDING); + transmute(r) +} + +/// Subtract the lower double-precision (64-bit) floating-point element in b from the lower double-precision (64-bit) floating-point element in a, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_sub_round_sd&expand=5741) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsubsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_sub_round_sd( + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vsubsd(a, b, zero, k, ROUNDING); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point element in a and b, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mul_round_ss&expand=3946) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_mul_round_ss(a: __m128, b: __m128) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vmulss(a, b, zero, 0b1, ROUNDING); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point element in a and b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_mul_round_ss&expand=3944) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_mul_round_ss( + src: __m128, + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let src = src.as_f32x4(); + let r = vmulss(a, b, src, k, ROUNDING); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point element in a and b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_mul_round_ss&expand=3945) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_mul_round_ss( + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vmulss(a, b, zero, k, ROUNDING); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point element in a and b, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mul_round_sd&expand=3943) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_mul_round_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vmulsd(a, b, zero, 0b1, ROUNDING); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point element in a and b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_mul_round_sd&expand=3941) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_mul_round_sd( + src: __m128d, + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let src = src.as_f64x2(); + let r = vmulsd(a, b, src, k, ROUNDING); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point element in a and b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_mul_round_sd&expand=3942) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmulsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_mul_round_sd( + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vmulsd(a, b, zero, k, ROUNDING); + transmute(r) +} + +/// Divide the lower single-precision (32-bit) floating-point element in a by the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_div_round_ss&expand=2174) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_div_round_ss(a: __m128, b: __m128) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vdivss(a, b, zero, 0b1, ROUNDING); + transmute(r) +} + +/// Divide the lower single-precision (32-bit) floating-point element in a by the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_div_round_ss&expand=2175) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_div_round_ss( + src: __m128, + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let src = src.as_f32x4(); + let r = vdivss(a, b, src, k, ROUNDING); + transmute(r) +} + +/// Divide the lower single-precision (32-bit) floating-point element in a by the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_div_round_ss&expand=2176) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_div_round_ss( + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vdivss(a, b, zero, k, ROUNDING); + transmute(r) +} + +/// Divide the lower double-precision (64-bit) floating-point element in a by the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_div_round_sd&expand=2171) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_div_round_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vdivsd(a, b, zero, 0b1, ROUNDING); + transmute(r) +} + +/// Divide the lower double-precision (64-bit) floating-point element in a by the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_div_round_sd&expand=2172) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_div_round_sd( + src: __m128d, + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let src = src.as_f64x2(); + let r = vdivsd(a, b, src, k, ROUNDING); + transmute(r) +} + +/// Divide the lower double-precision (64-bit) floating-point element in a by the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_div_round_sd&expand=2173) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vdivsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_div_round_sd( + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vdivsd(a, b, zero, k, ROUNDING); + transmute(r) +} + +/// Compare the lower single-precision (32-bit) floating-point elements in a and b, store the maximum value in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_max_round_ss&expand=3668) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxss, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_max_round_ss(a: __m128, b: __m128) -> __m128 { + static_assert_sae!(SAE); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vmaxss(a, b, zero, 0b1, SAE); + transmute(r) +} + +/// Compare the lower single-precision (32-bit) floating-point elements in a and b, store the maximum value in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_max_ss&expand=3672) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxss, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_max_round_ss( + src: __m128, + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_sae!(SAE); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let src = src.as_f32x4(); + let r = vmaxss(a, b, src, k, SAE); + transmute(r) +} + +/// Compare the lower single-precision (32-bit) floating-point elements in a and b, store the maximum value in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_max_round_ss&expand=3667) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxss, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_max_round_ss(k: __mmask8, a: __m128, b: __m128) -> __m128 { + static_assert_sae!(SAE); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vmaxss(a, b, zero, k, SAE); + transmute(r) +} + +/// Compare the lower double-precision (64-bit) floating-point elements in a and b, store the maximum value in the lower element of dst, and copy the upper element from a to the upper element of dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_max_round_sd&expand=3665) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxsd, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_max_round_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vmaxsd(a, b, zero, 0b1, SAE); + transmute(r) +} + +/// Compare the lower double-precision (64-bit) floating-point elements in a and b, store the maximum value in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_max_round_sd&expand=3663) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxsd, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_max_round_sd( + src: __m128d, + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let src = src.as_f64x2(); + let r = vmaxsd(a, b, src, k, SAE); + transmute(r) +} + +/// Compare the lower double-precision (64-bit) floating-point elements in a and b, store the maximum value in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_max_sd&expand=3670) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vmaxsd, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_max_round_sd( + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vmaxsd(a, b, zero, k, SAE); + transmute(r) +} + +/// Compare the lower single-precision (32-bit) floating-point elements in a and b, store the minimum value in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_min_round_ss&expand=3782) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminss, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_min_round_ss(a: __m128, b: __m128) -> __m128 { + static_assert_sae!(SAE); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vminss(a, b, zero, 0b1, SAE); + transmute(r) +} + +/// Compare the lower single-precision (32-bit) floating-point elements in a and b, store the minimum value in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_min_round_Ss&expand=3780) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminss, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_min_round_ss( + src: __m128, + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_sae!(SAE); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let src = src.as_f32x4(); + let r = vminss(a, b, src, k, SAE); + transmute(r) +} + +/// Compare the lower single-precision (32-bit) floating-point elements in a and b, store the minimum value in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_min_round_ss&expand=3781) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminss, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_min_round_ss(k: __mmask8, a: __m128, b: __m128) -> __m128 { + static_assert_sae!(SAE); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vminss(a, b, zero, k, SAE); + transmute(r) +} + +/// Compare the lower double-precision (64-bit) floating-point elements in a and b, store the minimum value in the lower element of dst , and copy the upper element from a to the upper element of dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_min_round_sd&expand=3779) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminsd, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_min_round_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vminsd(a, b, zero, 0b1, SAE); + transmute(r) +} + +/// Compare the lower double-precision (64-bit) floating-point elements in a and b, store the minimum value in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_min_round_sd&expand=3777) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminsd, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_min_round_sd( + src: __m128d, + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let src = src.as_f64x2(); + let r = vminsd(a, b, src, k, SAE); + transmute(r) +} + +/// Compare the lower double-precision (64-bit) floating-point elements in a and b, store the minimum value in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_min_round_Sd&expand=3778) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vminsd, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_min_round_sd( + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vminsd(a, b, zero, k, SAE); + transmute(r) +} + +/// Compute the square root of the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_sqrt_round_ss&expand=5383) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_sqrt_round_ss(a: __m128, b: __m128) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vsqrtss(a, b, zero, 0b1, ROUNDING); + transmute(r) +} + +/// Compute the square root of the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_sqrt_round_ss&expand=5381) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_sqrt_round_ss( + src: __m128, + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let src = src.as_f32x4(); + let r = vsqrtss(a, b, src, k, ROUNDING); + transmute(r) +} + +/// Compute the square root of the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_sqrt_round_ss&expand=5382) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_sqrt_round_ss( + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vsqrtss(a, b, zero, k, ROUNDING); + transmute(r) +} + +/// Compute the square root of the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_sqrt_round_sd&expand=5380) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_sqrt_round_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vsqrtsd(a, b, zero, 0b1, ROUNDING); + transmute(r) +} + +/// Compute the square root of the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_sqrt_round_sd&expand=5378) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_sqrt_round_sd( + src: __m128d, + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let src = src.as_f64x2(); + let r = vsqrtsd(a, b, src, k, ROUNDING); + transmute(r) +} + +/// Compute the square root of the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_sqrt_round_sd&expand=5379) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vsqrtsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_sqrt_round_sd( + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vsqrtsd(a, b, zero, k, ROUNDING); + transmute(r) +} + +/// Convert the exponent of the lower single-precision (32-bit) floating-point element in b to a single-precision (32-bit) floating-point number representing the integer exponent, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst. This intrinsic essentially calculates floor(log2(x)) for the lower element.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_getexp_round_ss&expand=2856) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexpss, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_getexp_round_ss(a: __m128, b: __m128) -> __m128 { + static_assert_sae!(SAE); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vgetexpss(a, b, zero, 0b1, SAE); + transmute(r) +} + +/// Convert the exponent of the lower single-precision (32-bit) floating-point element in b to a single-precision (32-bit) floating-point number representing the integer exponent, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. This intrinsic essentially calculates floor(log2(x)) for the lower element.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_getexp_round_ss&expand=2857) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexpss, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_getexp_round_ss( + src: __m128, + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_sae!(SAE); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let src = src.as_f32x4(); + let r = vgetexpss(a, b, src, k, SAE); + transmute(r) +} + +/// Convert the exponent of the lower single-precision (32-bit) floating-point element in b to a single-precision (32-bit) floating-point number representing the integer exponent, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. This intrinsic essentially calculates floor(log2(x)) for the lower element.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_getexp_round_ss&expand=2858) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexpss, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_getexp_round_ss( + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_sae!(SAE); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vgetexpss(a, b, zero, k, SAE); + transmute(r) +} + +/// Convert the exponent of the lower double-precision (64-bit) floating-point element in b to a double-precision (64-bit) floating-point number representing the integer exponent, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst. This intrinsic essentially calculates floor(log2(x)) for the lower element.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_getexp_round_sd&expand=2853) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexpsd, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_getexp_round_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vgetexpsd(a, b, zero, 0b1, SAE); + transmute(r) +} + +/// Convert the exponent of the lower double-precision (64-bit) floating-point element in b to a double-precision (64-bit) floating-point number representing the integer exponent, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. This intrinsic essentially calculates floor(log2(x)) for the lower element.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_getexp_round_sd&expand=2854) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexpsd, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_getexp_round_sd( + src: __m128d, + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let src = src.as_f64x2(); + let r = vgetexpsd(a, b, src, k, SAE); + transmute(r) +} + +/// Convert the exponent of the lower double-precision (64-bit) floating-point element in b to a double-precision (64-bit) floating-point number representing the integer exponent, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. This intrinsic essentially calculates floor(log2(x)) for the lower element.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_getexp_round_sd&expand=2855) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetexpsd, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_getexp_round_sd( + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vgetexpsd(a, b, zero, k, SAE); + transmute(r) +} + +/// Normalize the mantissas of the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst. This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_getmant_round_ss&expand=2892) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantss, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(2, 3, 4)] +pub unsafe fn _mm_getmant_round_ss< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( + a: __m128, + b: __m128, +) -> __m128 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vgetmantss(a, b, SIGN << 2 | NORM, zero, 0b1, SAE); + transmute(r) +} + +/// Normalize the mantissas of the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_getmant_round_ss&expand=2893) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantss, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(4, 5, 6)] +pub unsafe fn _mm_mask_getmant_round_ss< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( + src: __m128, + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let src = src.as_f32x4(); + let r = vgetmantss(a, b, SIGN << 2 | NORM, src, k, SAE); + transmute(r) +} + +/// Normalize the mantissas of the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_getmant_round_ss&expand=2894) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantss, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(3, 4, 5)] +pub unsafe fn _mm_maskz_getmant_round_ss< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vgetmantss(a, b, SIGN << 2 | NORM, zero, k, SAE); + transmute(r) +} + +/// Normalize the mantissas of the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst. This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_getmant_round_sd&expand=2889) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantsd, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(2, 3, 4)] +pub unsafe fn _mm_getmant_round_sd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vgetmantsd(a, b, SIGN << 2 | NORM, zero, 0b1, SAE); + transmute(r) +} + +/// Normalize the mantissas of the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_getmant_round_sd&expand=2890) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantsd, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(4, 5, 6)] +pub unsafe fn _mm_mask_getmant_round_sd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( + src: __m128d, + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let src = src.as_f64x2(); + let r = vgetmantsd(a, b, SIGN << 2 | NORM, src, k, SAE); + transmute(r) +} + +/// Normalize the mantissas of the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign.\ +/// The mantissa is normalized to the interval specified by interv, which can take the following values:\ +/// _MM_MANT_NORM_1_2 // interval [1, 2)\ +/// _MM_MANT_NORM_p5_2 // interval [0.5, 2)\ +/// _MM_MANT_NORM_p5_1 // interval [0.5, 1)\ +/// _MM_MANT_NORM_p75_1p5 // interval [0.75, 1.5)\ +/// The sign is determined by sc which can take the following values:\ +/// _MM_MANT_SIGN_src // sign = sign(src)\ +/// _MM_MANT_SIGN_zero // sign = 0\ +/// _MM_MANT_SIGN_nan // dst = NaN if sign(src) = 1\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_getmant_round_sd&expand=2891) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vgetmantsd, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(3, 4, 5)] +pub unsafe fn _mm_maskz_getmant_round_sd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vgetmantsd(a, b, SIGN << 2 | NORM, zero, k, SAE); + transmute(r) +} + +/// Round the lower single-precision (32-bit) floating-point element in b to the number of fraction bits specified by imm8, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE\ +/// +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_roundscale_round_ss&expand=4796) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscaless, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm_roundscale_round_ss( + a: __m128, + b: __m128, +) -> __m128 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vrndscaless(a, b, zero, 0b11111111, IMM8, SAE); + transmute(r) +} + +/// Round the lower single-precision (32-bit) floating-point element in b to the number of fraction bits specified by imm8, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE\ +/// +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_roundscale_round_ss&expand=4794) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscaless, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm_mask_roundscale_round_ss( + src: __m128, + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let src = src.as_f32x4(); + let r = vrndscaless(a, b, src, k, IMM8, SAE); + transmute(r) +} + +/// Round the lower single-precision (32-bit) floating-point element in b to the number of fraction bits specified by imm8, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE\ +/// +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_roundscale_round_ss&expand=4795) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscaless, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm_maskz_roundscale_round_ss( + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vrndscaless(a, b, zero, k, IMM8, SAE); + transmute(r) +} + +/// Round the lower double-precision (64-bit) floating-point element in b to the number of fraction bits specified by imm8, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst.\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE\ +/// +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_roundscale_round_sd&expand=4793) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscalesd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm_roundscale_round_sd( + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vrndscalesd(a, b, zero, 0b11111111, IMM8, SAE); + transmute(r) +} + +/// Round the lower double-precision (64-bit) floating-point element in b to the number of fraction bits specified by imm8, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE\ +/// +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_roundscale_round_sd&expand=4791) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscalesd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm_mask_roundscale_round_sd( + src: __m128d, + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let src = src.as_f64x2(); + let r = vrndscalesd(a, b, src, k, IMM8, SAE); + transmute(r) +} + +/// Round the lower double-precision (64-bit) floating-point element in b to the number of fraction bits specified by imm8, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// Rounding is done according to the imm8\[2:0\] parameter, which can be one of:\ +/// _MM_FROUND_TO_NEAREST_INT // round to nearest\ +/// _MM_FROUND_TO_NEG_INF // round down\ +/// _MM_FROUND_TO_POS_INF // round up\ +/// _MM_FROUND_TO_ZERO // truncate\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE\ +/// +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_roundscale_round_sd&expand=4792) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vrndscalesd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm_maskz_roundscale_round_sd( + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vrndscalesd(a, b, zero, k, IMM8, SAE); + transmute(r) +} + +/// Scale the packed single-precision (32-bit) floating-point elements in a using values from b, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_scalef_round_ss&expand=4895) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_scalef_round_ss(a: __m128, b: __m128) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vscalefss(a, b, zero, 0b11111111, ROUNDING); + transmute(r) +} + +/// Scale the packed single-precision (32-bit) floating-point elements in a using values from b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_scalef_round_ss&expand=4893) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_scalef_round_ss( + src: __m128, + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let src = src.as_f32x4(); + let r = vscalefss(a, b, src, k, ROUNDING); + transmute(r) +} + +/// Scale the packed single-precision (32-bit) floating-point elements in a using values from b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_scalef_round_ss&expand=4894) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_scalef_round_ss( + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vscalefss(a, b, zero, k, ROUNDING); + transmute(r) +} + +/// Scale the packed double-precision (64-bit) floating-point elements in a using values from b, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_scalef_round_sd&expand=4892) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_scalef_round_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vscalefsd(a, b, zero, 0b11111111, ROUNDING); + transmute(r) +} + +/// Scale the packed double-precision (64-bit) floating-point elements in a using values from b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_scalef_round_sd&expand=4890) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_scalef_round_sd( + src: __m128d, + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let src = src.as_f64x2(); + let r = vscalefsd(a, b, src, k, ROUNDING); + transmute(r) +} + +/// Scale the packed double-precision (64-bit) floating-point elements in a using values from b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_scalef_round_sd&expand=4891) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vscalefsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_scalef_round_sd( + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vscalefsd(a, b, zero, k, ROUNDING); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and add the intermediate result to the lower element in c. Store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fmadd_round_ss&expand=2573) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fmadd_round_ss(a: __m128, b: __m128, c: __m128) -> __m128 { + static_assert_rounding!(ROUNDING); + let extracta: f32 = simd_extract(a, 0); + let extractb: f32 = simd_extract(b, 0); + let extractc: f32 = simd_extract(c, 0); + let r = vfmadd132ss(extracta, extractb, extractc, ROUNDING); + let r = simd_insert(a, 0, r); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and add the intermediate result to the lower element in c. Store the result in the lower element of dst using writemask k (the element is copied from a when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fmadd_round_ss&expand=2574) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fmadd_round_ss( + a: __m128, + k: __mmask8, + b: __m128, + c: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let mut fmadd: f32 = simd_extract(a, 0); + if (k & 0b00000001) != 0 { + let extractb: f32 = simd_extract(b, 0); + let extractc: f32 = simd_extract(c, 0); + fmadd = vfmadd132ss(fmadd, extractb, extractc, ROUNDING); + } + let r = simd_insert(a, 0, fmadd); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and add the intermediate result to the lower element in c. Store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fmadd_round_ss&expand=2576) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fmadd_round_ss( + k: __mmask8, + a: __m128, + b: __m128, + c: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let mut fmadd: f32 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extractb: f32 = simd_extract(b, 0); + let extractc: f32 = simd_extract(c, 0); + fmadd = vfmadd132ss(extracta, extractb, extractc, ROUNDING); + } + let r = simd_insert(a, 0, fmadd); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and add the intermediate result to the lower element in c. Store the result in the lower element of dst using writemask k (the element is copied from c when mask bit 0 is not set), and copy the upper 3 packed elements from c to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fmadd_round_ss&expand=2575) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask3_fmadd_round_ss( + a: __m128, + b: __m128, + c: __m128, + k: __mmask8, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let mut fmadd: f32 = simd_extract(c, 0); + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extractb: f32 = simd_extract(b, 0); + fmadd = vfmadd132ss(extracta, extractb, fmadd, ROUNDING); + } + let r = simd_insert(c, 0, fmadd); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and add the intermediate result to the lower element in c. Store the result in the lower element of dst, and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fmadd_round_sd&expand=2569) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fmadd_round_sd( + a: __m128d, + b: __m128d, + c: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let extracta: f64 = simd_extract(a, 0); + let extractb: f64 = simd_extract(b, 0); + let extractc: f64 = simd_extract(c, 0); + let fmadd = vfmadd132sd(extracta, extractb, extractc, ROUNDING); + let r = simd_insert(a, 0, fmadd); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and add the intermediate result to the lower element in c. Store the result in the lower element of dst using writemask k (the element is copied from a when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fmadd_round_sd&expand=2570) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fmadd_round_sd( + a: __m128d, + k: __mmask8, + b: __m128d, + c: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let mut fmadd: f64 = simd_extract(a, 0); + if (k & 0b00000001) != 0 { + let extractb: f64 = simd_extract(b, 0); + let extractc: f64 = simd_extract(c, 0); + fmadd = vfmadd132sd(fmadd, extractb, extractc, ROUNDING); + } + let r = simd_insert(a, 0, fmadd); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and add the intermediate result to the lower element in c. Store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fmadd_round_sd&expand=2572) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fmadd_round_sd( + k: __mmask8, + a: __m128d, + b: __m128d, + c: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let mut fmadd: f64 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extractb: f64 = simd_extract(b, 0); + let extractc: f64 = simd_extract(c, 0); + fmadd = vfmadd132sd(extracta, extractb, extractc, ROUNDING); + } + let r = simd_insert(a, 0, fmadd); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and add the intermediate result to the lower element in c. Store the result in the lower element of dst using writemask k (the element is copied from c when mask bit 0 is not set), and copy the upper element from c to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fmadd_round_Sd&expand=2571) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmadd213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask3_fmadd_round_sd( + a: __m128d, + b: __m128d, + c: __m128d, + k: __mmask8, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let mut fmadd: f64 = simd_extract(c, 0); + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extractb: f64 = simd_extract(b, 0); + fmadd = vfmadd132sd(extracta, extractb, fmadd, ROUNDING); + } + let r = simd_insert(c, 0, fmadd); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and subtract the lower element in c from the intermediate result. Store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fmsub_round_ss&expand=2659) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmsub213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fmsub_round_ss(a: __m128, b: __m128, c: __m128) -> __m128 { + static_assert_rounding!(ROUNDING); + let extracta: f32 = simd_extract(a, 0); + let extractb: f32 = simd_extract(b, 0); + let extractc: f32 = simd_extract(c, 0); + let extractc = -extractc; + let fmsub = vfmadd132ss(extracta, extractb, extractc, ROUNDING); + let r = simd_insert(a, 0, fmsub); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and subtract the lower element in c from the intermediate result. Store the result in the lower element of dst using writemask k (the element is copied from a when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fmsub_round_ss&expand=2660) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmsub213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fmsub_round_ss( + a: __m128, + k: __mmask8, + b: __m128, + c: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let mut fmsub: f32 = simd_extract(a, 0); + if (k & 0b00000001) != 0 { + let extractb: f32 = simd_extract(b, 0); + let extractc: f32 = simd_extract(c, 0); + let extractc = -extractc; + fmsub = vfmadd132ss(fmsub, extractb, extractc, ROUNDING); + } + let r = simd_insert(a, 0, fmsub); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and subtract the lower element in c from the intermediate result. Store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fmsub_round_ss&expand=2662) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmsub213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fmsub_round_ss( + k: __mmask8, + a: __m128, + b: __m128, + c: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let mut fmsub: f32 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extractb: f32 = simd_extract(b, 0); + let extractc: f32 = simd_extract(c, 0); + let extractc = -extractc; + fmsub = vfmadd132ss(extracta, extractb, extractc, ROUNDING); + } + let r = simd_insert(a, 0, fmsub); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and subtract the lower element in c from the intermediate result. Store the result in the lower element of dst using writemask k (the element is copied from c when mask bit 0 is not set), and copy the upper 3 packed elements from c to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fmsub_round_ss&expand=2661) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmsub213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask3_fmsub_round_ss( + a: __m128, + b: __m128, + c: __m128, + k: __mmask8, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let mut fmsub: f32 = simd_extract(c, 0); + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extractb: f32 = simd_extract(b, 0); + let extractc = -fmsub; + fmsub = vfmadd132ss(extracta, extractb, extractc, ROUNDING); + } + let r = simd_insert(c, 0, fmsub); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and subtract the lower element in c from the intermediate result. Store the result in the lower element of dst, and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fmsub_round_sd&expand=2655) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmsub213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fmsub_round_sd( + a: __m128d, + b: __m128d, + c: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let extracta: f64 = simd_extract(a, 0); + let extractb: f64 = simd_extract(b, 0); + let extractc: f64 = simd_extract(c, 0); + let extractc = -extractc; + let fmsub = vfmadd132sd(extracta, extractb, extractc, ROUNDING); + let r = simd_insert(a, 0, fmsub); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and subtract the lower element in c from the intermediate result. Store the result in the lower element of dst using writemask k (the element is copied from a when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fmsub_round_sd&expand=2656) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmsub213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fmsub_round_sd( + a: __m128d, + k: __mmask8, + b: __m128d, + c: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let mut fmsub: f64 = simd_extract(a, 0); + if (k & 0b00000001) != 0 { + let extractb: f64 = simd_extract(b, 0); + let extractc: f64 = simd_extract(c, 0); + let extractc = -extractc; + fmsub = vfmadd132sd(fmsub, extractb, extractc, ROUNDING); + } + let r = simd_insert(a, 0, fmsub); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and subtract the lower element in c from the intermediate result. Store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fmsub_round_sd&expand=2658) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmsub213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fmsub_round_sd( + k: __mmask8, + a: __m128d, + b: __m128d, + c: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let mut fmsub: f64 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extractb: f64 = simd_extract(b, 0); + let extractc: f64 = simd_extract(c, 0); + let extractc = -extractc; + fmsub = vfmadd132sd(extracta, extractb, extractc, ROUNDING); + } + let r = simd_insert(a, 0, fmsub); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and subtract the lower element in c from the intermediate result. Store the result in the lower element of dst using writemask k (the element is copied from c when mask bit 0 is not set), and copy the upper element from c to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fmsub_round_sd&expand=2657) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfmsub213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask3_fmsub_round_sd( + a: __m128d, + b: __m128d, + c: __m128d, + k: __mmask8, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let mut fmsub: f64 = simd_extract(c, 0); + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extractb: f64 = simd_extract(b, 0); + let extractc = -fmsub; + fmsub = vfmadd132sd(extracta, extractb, extractc, ROUNDING); + } + let r = simd_insert(c, 0, fmsub); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and add the negated intermediate result to the lower element in c. Store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fnmadd_round_ss&expand=2739) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmadd213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fnmadd_round_ss(a: __m128, b: __m128, c: __m128) -> __m128 { + static_assert_rounding!(ROUNDING); + let extracta: f32 = simd_extract(a, 0); + let extracta = -extracta; + let extractb: f32 = simd_extract(b, 0); + let extractc: f32 = simd_extract(c, 0); + let fnmadd = vfmadd132ss(extracta, extractb, extractc, ROUNDING); + let r = simd_insert(a, 0, fnmadd); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and add the negated intermediate result to the lower element in c. Store the result in the lower element of dst using writemask k (the element is copied from a when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fnmadd_round_ss&expand=2740) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmadd213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fnmadd_round_ss( + a: __m128, + k: __mmask8, + b: __m128, + c: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let mut fnmadd: f32 = simd_extract(a, 0); + if (k & 0b00000001) != 0 { + let extracta = -fnmadd; + let extractb: f32 = simd_extract(b, 0); + let extractc: f32 = simd_extract(c, 0); + fnmadd = vfmadd132ss(extracta, extractb, extractc, ROUNDING); + } + let r = simd_insert(a, 0, fnmadd); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and add the negated intermediate result to the lower element in c. Store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fnmadd_round_ss&expand=2742) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmadd213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fnmadd_round_ss( + k: __mmask8, + a: __m128, + b: __m128, + c: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let mut fnmadd: f32 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extracta = -extracta; + let extractb: f32 = simd_extract(b, 0); + let extractc: f32 = simd_extract(c, 0); + fnmadd = vfmadd132ss(extracta, extractb, extractc, ROUNDING); + } + let r = simd_insert(a, 0, fnmadd); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and add the negated intermediate result to the lower element in c. Store the result in the lower element of dst using writemask k (the element is copied from c when mask bit 0 is not set), and copy the upper 3 packed elements from c to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fnmadd_round_ss&expand=2741) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmadd213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask3_fnmadd_round_ss( + a: __m128, + b: __m128, + c: __m128, + k: __mmask8, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let mut fnmadd: f32 = simd_extract(c, 0); + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extracta = -extracta; + let extractb: f32 = simd_extract(b, 0); + fnmadd = vfmadd132ss(extracta, extractb, fnmadd, ROUNDING); + } + let r = simd_insert(c, 0, fnmadd); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and add the negated intermediate result to the lower element in c. Store the result in the lower element of dst, and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fnmadd_round_sd&expand=2735) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmadd213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fnmadd_round_sd( + a: __m128d, + b: __m128d, + c: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let extracta: f64 = simd_extract(a, 0); + let extracta = -extracta; + let extractb: f64 = simd_extract(b, 0); + let extractc: f64 = simd_extract(c, 0); + let fnmadd = vfmadd132sd(extracta, extractb, extractc, ROUNDING); + let r = simd_insert(a, 0, fnmadd); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and add the negated intermediate result to the lower element in c. Store the result in the lower element of dst using writemask k (the element is copied from a when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fnmadd_round_sd&expand=2736) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmadd213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fnmadd_round_sd( + a: __m128d, + k: __mmask8, + b: __m128d, + c: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let mut fnmadd: f64 = simd_extract(a, 0); + if (k & 0b00000001) != 0 { + let extracta = -fnmadd; + let extractb: f64 = simd_extract(b, 0); + let extractc: f64 = simd_extract(c, 0); + fnmadd = vfmadd132sd(extracta, extractb, extractc, ROUNDING); + } + let r = simd_insert(a, 0, fnmadd); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and add the negated intermediate result to the lower element in c. Store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fnmadd_round_sd&expand=2738) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmadd213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fnmadd_round_sd( + k: __mmask8, + a: __m128d, + b: __m128d, + c: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let mut fnmadd: f64 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extracta = -extracta; + let extractb: f64 = simd_extract(b, 0); + let extractc: f64 = simd_extract(c, 0); + fnmadd = vfmadd132sd(extracta, extractb, extractc, ROUNDING); + } + let r = simd_insert(a, 0, fnmadd); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and add the negated intermediate result to the lower element in c. Store the result in the lower element of dst using writemask k (the element is copied from c when mask bit 0 is not set), and copy the upper element from c to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fnmadd_round_Sd&expand=2737) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmadd213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask3_fnmadd_round_sd( + a: __m128d, + b: __m128d, + c: __m128d, + k: __mmask8, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let mut fnmadd: f64 = simd_extract(c, 0); + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extracta = -extracta; + let extractb: f64 = simd_extract(b, 0); + fnmadd = vfmadd132sd(extracta, extractb, fnmadd, ROUNDING); + } + let r = simd_insert(c, 0, fnmadd); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, subtract the lower element in c from the negated intermediate result, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fnmsub_round_ss&expand=2787) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmsub213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fnmsub_round_ss(a: __m128, b: __m128, c: __m128) -> __m128 { + static_assert_rounding!(ROUNDING); + let extracta: f32 = simd_extract(a, 0); + let extracta = -extracta; + let extractb: f32 = simd_extract(b, 0); + let extractc: f32 = simd_extract(c, 0); + let extractc = -extractc; + let fnmsub = vfmadd132ss(extracta, extractb, extractc, ROUNDING); + let r = simd_insert(a, 0, fnmsub); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and subtract the lower element in c from the negated intermediate result. Store the result in the lower element of dst using writemask k (the element is copied from c when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fnmsub_round_ss&expand=2788) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmsub213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fnmsub_round_ss( + a: __m128, + k: __mmask8, + b: __m128, + c: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let mut fnmsub: f32 = simd_extract(a, 0); + if (k & 0b00000001) != 0 { + let extracta = -fnmsub; + let extractb: f32 = simd_extract(b, 0); + let extractc: f32 = simd_extract(c, 0); + let extractc = -extractc; + fnmsub = vfmadd132ss(extracta, extractb, extractc, ROUNDING); + } + let r = simd_insert(a, 0, fnmsub); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, and subtract the lower element in c from the negated intermediate result. Store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fnmsub_round_ss&expand=2790) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmsub213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fnmsub_round_ss( + k: __mmask8, + a: __m128, + b: __m128, + c: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let mut fnmsub: f32 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extracta = -extracta; + let extractb: f32 = simd_extract(b, 0); + let extractc: f32 = simd_extract(c, 0); + let extractc = -extractc; + fnmsub = vfmadd132ss(extracta, extractb, extractc, ROUNDING); + } + let r = simd_insert(a, 0, fnmsub); + transmute(r) +} + +/// Multiply the lower single-precision (32-bit) floating-point elements in a and b, subtract the lower element in c from the negated intermediate result. Store the result in the lower element of dst using writemask k (the element is copied from c when mask bit 0 is not set), and copy the upper 3 packed elements from c to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fnmsub_round_ss&expand=2789) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmsub213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask3_fnmsub_round_ss( + a: __m128, + b: __m128, + c: __m128, + k: __mmask8, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let mut fnmsub: f32 = simd_extract(c, 0); + if (k & 0b00000001) != 0 { + let extracta: f32 = simd_extract(a, 0); + let extracta = -extracta; + let extractb: f32 = simd_extract(b, 0); + let extractc = -fnmsub; + fnmsub = vfmadd132ss(extracta, extractb, extractc, ROUNDING); + } + let r = simd_insert(c, 0, fnmsub); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and subtract the lower element in c from the negated intermediate result. Store the result in the lower element of dst, and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fnmsub_round_sd&expand=2783) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmsub213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fnmsub_round_sd( + a: __m128d, + b: __m128d, + c: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let extracta: f64 = simd_extract(a, 0); + let extracta = -extracta; + let extractb: f64 = simd_extract(b, 0); + let extractc: f64 = simd_extract(c, 0); + let extractc = -extractc; + let fnmsub = vfmadd132sd(extracta, extractb, extractc, ROUNDING); + let r = simd_insert(a, 0, fnmsub); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and subtract the lower element in c from the negated intermediate result. Store the result in the lower element of dst using writemask k (the element is copied from c when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fnmsub_round_sd&expand=2784) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmsub213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fnmsub_round_sd( + a: __m128d, + k: __mmask8, + b: __m128d, + c: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let mut fnmsub: f64 = simd_extract(a, 0); + if (k & 0b00000001) != 0 { + let extracta = -fnmsub; + let extractb: f64 = simd_extract(b, 0); + let extractc: f64 = simd_extract(c, 0); + let extractc = -extractc; + fnmsub = vfmadd132sd(extracta, extractb, extractc, ROUNDING); + } + let r = simd_insert(a, 0, fnmsub); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and subtract the lower element in c from the negated intermediate result. Store the result in dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fnmsub_round_sd&expand=2786) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmsub213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fnmsub_round_sd( + k: __mmask8, + a: __m128d, + b: __m128d, + c: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let mut fnmsub: f64 = 0.; + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extracta = -extracta; + let extractb: f64 = simd_extract(b, 0); + let extractc: f64 = simd_extract(c, 0); + let extractc = -extractc; + fnmsub = vfmadd132sd(extracta, extractb, extractc, ROUNDING); + } + let r = simd_insert(a, 0, fnmsub); + transmute(r) +} + +/// Multiply the lower double-precision (64-bit) floating-point elements in a and b, and subtract the lower element in c from the negated intermediate result. Store the result in the lower element of dst using writemask k (the element is copied from c when mask bit 0 is not set), and copy the upper element from c to the upper element of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fnmsub_round_sd&expand=2785) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfnmsub213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask3_fnmsub_round_sd( + a: __m128d, + b: __m128d, + c: __m128d, + k: __mmask8, +) -> __m128d { + static_assert_rounding!(ROUNDING); + let mut fnmsub: f64 = simd_extract(c, 0); + if (k & 0b00000001) != 0 { + let extracta: f64 = simd_extract(a, 0); + let extracta = -extracta; + let extractb: f64 = simd_extract(b, 0); + let extractc = -fnmsub; + fnmsub = vfmadd132sd(extracta, extractb, extractc, ROUNDING); + } + let r = simd_insert(c, 0, fnmsub); + transmute(r) +} + +/// Fix up the lower single-precision (32-bit) floating-point elements in a and b using the lower 32-bit integer in c, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst. imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fixupimm_ss&expand=2517) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmss, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fixupimm_ss(a: __m128, b: __m128, c: __m128i) -> __m128 { + static_assert_imm8!(IMM8); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let c = c.as_i32x4(); + let r = vfixupimmss(a, b, c, IMM8, 0b11111111, _MM_FROUND_CUR_DIRECTION); + let fixupimm: f32 = simd_extract(r, 0); + let r = simd_insert(a, 0, fixupimm); + transmute(r) +} + +/// Fix up the lower single-precision (32-bit) floating-point elements in a and b using the lower 32-bit integer in c, store the result in the lower element of dst using writemask k (the element is copied from a when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fixupimm_ss&expand=2518) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmss, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fixupimm_ss( + a: __m128, + k: __mmask8, + b: __m128, + c: __m128i, +) -> __m128 { + static_assert_imm8!(IMM8); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let c = c.as_i32x4(); + let fixupimm = vfixupimmss(a, b, c, IMM8, k, _MM_FROUND_CUR_DIRECTION); + let fixupimm: f32 = simd_extract(fixupimm, 0); + let r = simd_insert(a, 0, fixupimm); + transmute(r) +} + +/// Fix up the lower single-precision (32-bit) floating-point elements in a and b using the lower 32-bit integer in c, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fixupimm_ss&expand=2519) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmss, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fixupimm_ss( + k: __mmask8, + a: __m128, + b: __m128, + c: __m128i, +) -> __m128 { + static_assert_imm8!(IMM8); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let c = c.as_i32x4(); + let fixupimm = vfixupimmssz(a, b, c, IMM8, k, _MM_FROUND_CUR_DIRECTION); + let fixupimm: f32 = simd_extract(fixupimm, 0); + let r = simd_insert(a, 0, fixupimm); + transmute(r) +} + +/// Fix up the lower double-precision (64-bit) floating-point elements in a and b using the lower 64-bit integer in c, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst. imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fixupimm_sd&expand=2514) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmsd, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fixupimm_sd(a: __m128d, b: __m128d, c: __m128i) -> __m128d { + static_assert_imm8!(IMM8); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let c = c.as_i64x2(); + let fixupimm = vfixupimmsd(a, b, c, IMM8, 0b11111111, _MM_FROUND_CUR_DIRECTION); + let fixupimm: f64 = simd_extract(fixupimm, 0); + let r = simd_insert(a, 0, fixupimm); + transmute(r) +} + +/// Fix up the lower double-precision (64-bit) floating-point elements in a and b using the lower 64-bit integer in c, store the result in the lower element of dst using writemask k (the element is copied from a when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fixupimm_sd&expand=2515) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmsd, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fixupimm_sd( + a: __m128d, + k: __mmask8, + b: __m128d, + c: __m128i, +) -> __m128d { + static_assert_imm8!(IMM8); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let c = c.as_i64x2(); + let fixupimm = vfixupimmsd(a, b, c, IMM8, k, _MM_FROUND_CUR_DIRECTION); + let fixupimm: f64 = simd_extract(fixupimm, 0); + let r = simd_insert(a, 0, fixupimm); + transmute(r) +} + +/// Fix up the lower double-precision (64-bit) floating-point elements in a and b using the lower 64-bit integer in c, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. imm8 is used to set the required flags reporting. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fixupimm_sd&expand=2516) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmsd, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fixupimm_sd( + k: __mmask8, + a: __m128d, + b: __m128d, + c: __m128i, +) -> __m128d { + static_assert_imm8!(IMM8); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let c = c.as_i64x2(); + let fixupimm = vfixupimmsdz(a, b, c, IMM8, k, _MM_FROUND_CUR_DIRECTION); + let fixupimm: f64 = simd_extract(fixupimm, 0); + let r = simd_insert(a, 0, fixupimm); + transmute(r) +} + +/// Fix up the lower single-precision (32-bit) floating-point elements in a and b using the lower 32-bit integer in c, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst. imm8 is used to set the required flags reporting.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fixupimm_round_ss&expand=2511) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmss, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm_fixupimm_round_ss( + a: __m128, + b: __m128, + c: __m128i, +) -> __m128 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let c = c.as_i32x4(); + let r = vfixupimmss(a, b, c, IMM8, 0b11111111, SAE); + let fixupimm: f32 = simd_extract(r, 0); + let r = simd_insert(a, 0, fixupimm); + transmute(r) +} + +/// Fix up the lower single-precision (32-bit) floating-point elements in a and b using the lower 32-bit integer in c, store the result in the lower element of dst using writemask k (the element is copied from a when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. imm8 is used to set the required flags reporting.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fixupimm_round_ss&expand=2512) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmss, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm_mask_fixupimm_round_ss( + a: __m128, + k: __mmask8, + b: __m128, + c: __m128i, +) -> __m128 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let c = c.as_i32x4(); + let r = vfixupimmss(a, b, c, IMM8, k, SAE); + let fixupimm: f32 = simd_extract(r, 0); + let r = simd_insert(a, 0, fixupimm); + transmute(r) +} + +/// Fix up the lower single-precision (32-bit) floating-point elements in a and b using the lower 32-bit integer in c, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. imm8 is used to set the required flags reporting.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fixupimm_round_ss&expand=2513) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmss, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm_maskz_fixupimm_round_ss( + k: __mmask8, + a: __m128, + b: __m128, + c: __m128i, +) -> __m128 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let c = c.as_i32x4(); + let r = vfixupimmssz(a, b, c, IMM8, k, SAE); + let fixupimm: f32 = simd_extract(r, 0); + let r = simd_insert(a, 0, fixupimm); + transmute(r) +} + +/// Fix up the lower double-precision (64-bit) floating-point elements in a and b using the lower 64-bit integer in c, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst. imm8 is used to set the required flags reporting.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fixupimm_round_sd&expand=2508) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmsd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm_fixupimm_round_sd( + a: __m128d, + b: __m128d, + c: __m128i, +) -> __m128d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let c = c.as_i64x2(); + let r = vfixupimmsd(a, b, c, IMM8, 0b11111111, SAE); + let fixupimm: f64 = simd_extract(r, 0); + let r = simd_insert(a, 0, fixupimm); + transmute(r) +} + +/// Fix up the lower double-precision (64-bit) floating-point elements in a and b using the lower 64-bit integer in c, store the result in the lower element of dst using writemask k (the element is copied from a when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. imm8 is used to set the required flags reporting.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fixupimm_round_sd&expand=2509) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmsd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm_mask_fixupimm_round_sd( + a: __m128d, + k: __mmask8, + b: __m128d, + c: __m128i, +) -> __m128d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let c = c.as_i64x2(); + let r = vfixupimmsd(a, b, c, IMM8, k, SAE); + let fixupimm: f64 = simd_extract(r, 0); + let r = simd_insert(a, 0, fixupimm); + transmute(r) +} + +/// Fix up the lower double-precision (64-bit) floating-point elements in a and b using the lower 64-bit integer in c, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. imm8 is used to set the required flags reporting.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fixupimm_round_sd&expand=2510) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vfixupimmsd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm_maskz_fixupimm_round_sd( + k: __mmask8, + a: __m128d, + b: __m128d, + c: __m128i, +) -> __m128d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let c = c.as_i64x2(); + let r = vfixupimmsdz(a, b, c, IMM8, k, SAE); + let fixupimm: f64 = simd_extract(r, 0); + let r = simd_insert(a, 0, fixupimm); + transmute(r) +} + +/// Convert the lower single-precision (32-bit) floating-point element in b to a double-precision (64-bit) floating-point element, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_cvtss_sd&expand=1896) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2sd))] +pub unsafe fn _mm_mask_cvtss_sd(src: __m128d, k: __mmask8, a: __m128d, b: __m128) -> __m128d { + transmute(vcvtss2sd( + a.as_f64x2(), + b.as_f32x4(), + src.as_f64x2(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert the lower single-precision (32-bit) floating-point element in b to a double-precision (64-bit) floating-point element, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_cvtss_sd&expand=1897) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2sd))] +pub unsafe fn _mm_maskz_cvtss_sd(k: __mmask8, a: __m128d, b: __m128) -> __m128d { + transmute(vcvtss2sd( + a.as_f64x2(), + b.as_f32x4(), + _mm_setzero_pd().as_f64x2(), + k, + _MM_FROUND_CUR_DIRECTION, + )) +} + +/// Convert the lower double-precision (64-bit) floating-point element in b to a single-precision (32-bit) floating-point element, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_cvtsd_ss&expand=1797) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2ss))] +pub unsafe fn _mm_mask_cvtsd_ss(src: __m128, k: __mmask8, a: __m128, b: __m128d) -> __m128 { + transmute(vcvtsd2ss( + a.as_f32x4(), + b.as_f64x2(), + src.as_f32x4(), + k, + _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC, + )) +} + +/// Convert the lower double-precision (64-bit) floating-point element in b to a single-precision (32-bit) floating-point element, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_cvtsd_ss&expand=1798) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2ss))] +pub unsafe fn _mm_maskz_cvtsd_ss(k: __mmask8, a: __m128, b: __m128d) -> __m128 { + transmute(vcvtsd2ss( + a.as_f32x4(), + b.as_f64x2(), + _mm_setzero_ps().as_f32x4(), + k, + _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC, + )) +} + +/// Convert the lower single-precision (32-bit) floating-point element in b to a double-precision (64-bit) floating-point element, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvt_roundss_sd&expand=1371) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2sd, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundss_sd(a: __m128d, b: __m128) -> __m128d { + static_assert_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f32x4(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vcvtss2sd(a, b, zero, 0b11111111, SAE); + transmute(r) +} + +/// Convert the lower single-precision (32-bit) floating-point element in b to a double-precision (64-bit) floating-point element, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_cvt_roundss_sd&expand=1372) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2sd, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_cvt_roundss_sd( + src: __m128d, + k: __mmask8, + a: __m128d, + b: __m128, +) -> __m128d { + static_assert_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f32x4(); + let src = src.as_f64x2(); + let r = vcvtss2sd(a, b, src, k, SAE); + transmute(r) +} + +/// Convert the lower single-precision (32-bit) floating-point element in b to a double-precision (64-bit) floating-point element, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_cvt_roundss_sd&expand=1373) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2sd, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_cvt_roundss_sd( + k: __mmask8, + a: __m128d, + b: __m128, +) -> __m128d { + static_assert_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f32x4(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vcvtss2sd(a, b, zero, k, SAE); + transmute(r) +} + +/// Convert the lower double-precision (64-bit) floating-point element in b to a single-precision (32-bit) floating-point element, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvt_roundsd_ss&expand=1361) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundsd_ss(a: __m128, b: __m128d) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let b = b.as_f64x2(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vcvtsd2ss(a, b, zero, 0b11111111, ROUNDING); + transmute(r) +} + +/// Convert the lower double-precision (64-bit) floating-point element in b to a single-precision (32-bit) floating-point element, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_cvt_roundsd_ss&expand=1362) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_cvt_roundsd_ss( + src: __m128, + k: __mmask8, + a: __m128, + b: __m128d, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let b = b.as_f64x2(); + let src = src.as_f32x4(); + let r = vcvtsd2ss(a, b, src, k, ROUNDING); + transmute(r) +} + +/// Convert the lower double-precision (64-bit) floating-point element in b to a single-precision (32-bit) floating-point element, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_cvt_roundsd_ss&expand=1363) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_cvt_roundsd_ss( + k: __mmask8, + a: __m128, + b: __m128d, +) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let b = b.as_f64x2(); + let zero = _mm_setzero_ps().as_f32x4(); + let r = vcvtsd2ss(a, b, zero, k, ROUNDING); + transmute(r) +} + +/// Convert the lower single-precision (32-bit) floating-point element in a to a 32-bit integer, and store the result in dst.\ +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvt_roundss_si32&expand=1374) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2si, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundss_si32(a: __m128) -> i32 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let r = vcvtss2si(a, ROUNDING); + transmute(r) +} + +/// Convert the lower single-precision (32-bit) floating-point element in a to a 32-bit integer, and store the result in dst.\ +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvt_roundss_i32&expand=1369) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2si, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundss_i32(a: __m128) -> i32 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let r = vcvtss2si(a, ROUNDING); + transmute(r) +} + +/// Convert the lower single-precision (32-bit) floating-point element in a to an unsigned 32-bit integer, and store the result in dst.\ +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvt_roundss_u32&expand=1376) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2usi, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundss_u32(a: __m128) -> u32 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let r = vcvtss2usi(a, ROUNDING); + transmute(r) +} + +/// Convert the lower single-precision (32-bit) floating-point element in a to a 32-bit integer, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvtss_i32&expand=1893) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2si))] +pub unsafe fn _mm_cvtss_i32(a: __m128) -> i32 { + transmute(vcvtss2si(a.as_f32x4(), _MM_FROUND_CUR_DIRECTION)) +} + +/// Convert the lower single-precision (32-bit) floating-point element in a to an unsigned 32-bit integer, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvtss_u32&expand=1901) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2usi))] +pub unsafe fn _mm_cvtss_u32(a: __m128) -> u32 { + transmute(vcvtss2usi(a.as_f32x4(), _MM_FROUND_CUR_DIRECTION)) +} + +/// Convert the lower double-precision (64-bit) floating-point element in a to a 32-bit integer, and store the result in dst.\ +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvt_roundsd_si32&expand=1359) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2si, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundsd_si32(a: __m128d) -> i32 { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let r = vcvtsd2si(a, ROUNDING); + transmute(r) +} + +/// Convert the lower single-precision (32-bit) floating-point element in a to a 32-bit integer, and store the result in dst.\ +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvt_roundsd_i32&expand=1357) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2si, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundsd_i32(a: __m128d) -> i32 { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let r = vcvtsd2si(a, ROUNDING); + transmute(r) +} + +/// Convert the lower double-precision (64-bit) floating-point element in a to an unsigned 32-bit integer, and store the result in dst.\ +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=cvt_roundsd_u32&expand=1364) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2usi, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundsd_u32(a: __m128d) -> u32 { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let r = vcvtsd2usi(a, ROUNDING); + transmute(r) +} + +/// Convert the lower double-precision (64-bit) floating-point element in a to a 32-bit integer, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvtsd_i32&expand=1791) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2si))] +pub unsafe fn _mm_cvtsd_i32(a: __m128d) -> i32 { + transmute(vcvtsd2si(a.as_f64x2(), _MM_FROUND_CUR_DIRECTION)) +} + +/// Convert the lower double-precision (64-bit) floating-point element in a to an unsigned 32-bit integer, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvtsd_u32&expand=1799) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2usi))] +pub unsafe fn _mm_cvtsd_u32(a: __m128d) -> u32 { + transmute(vcvtsd2usi(a.as_f64x2(), _MM_FROUND_CUR_DIRECTION)) +} + +/// Convert the signed 32-bit integer b to a single-precision (32-bit) floating-point element, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvt_roundi32_ss&expand=1312) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsi2ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundi32_ss(a: __m128, b: i32) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let r = vcvtsi2ss(a, b, ROUNDING); + transmute(r) +} + +/// Convert the signed 32-bit integer b to a single-precision (32-bit) floating-point element, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvt_roundsi32_ss&expand=1366) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsi2ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundsi32_ss(a: __m128, b: i32) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let r = vcvtsi2ss(a, b, ROUNDING); + transmute(r) +} + +/// Convert the unsigned 32-bit integer b to a single-precision (32-bit) floating-point element, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvt_roundu32_ss&expand=1378) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtusi2ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundu32_ss(a: __m128, b: u32) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let r = vcvtusi2ss(a, b, ROUNDING); + transmute(r) +} + +/// Convert the signed 32-bit integer b to a single-precision (32-bit) floating-point element, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvti32_ss&expand=1643) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsi2ss))] +pub unsafe fn _mm_cvti32_ss(a: __m128, b: i32) -> __m128 { + let b = b as f32; + let r = simd_insert(a, 0, b); + transmute(r) +} + +/// Convert the signed 32-bit integer b to a double-precision (64-bit) floating-point element, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvti32_sd&expand=1642) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsi2sd))] +pub unsafe fn _mm_cvti32_sd(a: __m128d, b: i32) -> __m128d { + let b = b as f64; + let r = simd_insert(a, 0, b); + transmute(r) +} + +/// Convert the lower single-precision (32-bit) floating-point element in a to a 32-bit integer with truncation, and store the result in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvtt_roundss_Si32&expand=1936) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2si, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundss_si32(a: __m128) -> i32 { + static_assert_sae!(SAE); + let a = a.as_f32x4(); + let r = vcvtss2si(a, SAE); + transmute(r) +} + +/// Convert the lower single-precision (32-bit) floating-point element in a to a 32-bit integer with truncation, and store the result in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvtt_roundss_i32&expand=1934) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2si, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundss_i32(a: __m128) -> i32 { + static_assert_sae!(SAE); + let a = a.as_f32x4(); + let r = vcvtss2si(a, SAE); + transmute(r) +} + +/// Convert the lower single-precision (32-bit) floating-point element in a to an unsigned 32-bit integer with truncation, and store the result in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvtt_roundss_u32&expand=1938) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2usi, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundss_u32(a: __m128) -> u32 { + static_assert_sae!(SAE); + let a = a.as_f32x4(); + let r = vcvtss2usi(a, SAE); + transmute(r) +} + +/// Convert the lower single-precision (32-bit) floating-point element in a to a 32-bit integer with truncation, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttss_i32&expand=2022) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2si))] +pub unsafe fn _mm_cvttss_i32(a: __m128) -> i32 { + transmute(vcvtss2si(a.as_f32x4(), _MM_FROUND_CUR_DIRECTION)) +} + +/// Convert the lower single-precision (32-bit) floating-point element in a to an unsigned 32-bit integer with truncation, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttss_u32&expand=2026) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2usi))] +pub unsafe fn _mm_cvttss_u32(a: __m128) -> u32 { + transmute(vcvtss2usi(a.as_f32x4(), _MM_FROUND_CUR_DIRECTION)) +} + +/// Convert the lower double-precision (64-bit) floating-point element in a to a 32-bit integer with truncation, and store the result in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_roundsd_si32&expand=1930) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2si, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundsd_si32(a: __m128d) -> i32 { + static_assert_sae!(SAE); + let a = a.as_f64x2(); + let r = vcvtsd2si(a, SAE); + transmute(r) +} + +/// Convert the lower double-precision (64-bit) floating-point element in a to a 32-bit integer with truncation, and store the result in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_roundsd_i32&expand=1928) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2si, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundsd_i32(a: __m128d) -> i32 { + static_assert_sae!(SAE); + let a = a.as_f64x2(); + let r = vcvtsd2si(a, SAE); + transmute(r) +} + +/// Convert the lower double-precision (64-bit) floating-point element in a to an unsigned 32-bit integer with truncation, and store the result in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvtt_roundsd_u32&expand=1932) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2usi, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundsd_u32(a: __m128d) -> u32 { + static_assert_sae!(SAE); + let a = a.as_f64x2(); + let r = vcvtsd2usi(a, SAE); + transmute(r) +} + +/// Convert the lower double-precision (64-bit) floating-point element in a to a 32-bit integer with truncation, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttsd_i32&expand=2015) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2si))] +pub unsafe fn _mm_cvttsd_i32(a: __m128d) -> i32 { + transmute(vcvtsd2si(a.as_f64x2(), _MM_FROUND_CUR_DIRECTION)) +} + +/// Convert the lower double-precision (64-bit) floating-point element in a to an unsigned 32-bit integer with truncation, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttsd_u32&expand=2020) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2usi))] +pub unsafe fn _mm_cvttsd_u32(a: __m128d) -> u32 { + transmute(vcvtsd2usi(a.as_f64x2(), _MM_FROUND_CUR_DIRECTION)) +} + +/// Convert the unsigned 32-bit integer b to a single-precision (32-bit) floating-point element, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtu32_ss&expand=2032) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtusi2ss))] +pub unsafe fn _mm_cvtu32_ss(a: __m128, b: u32) -> __m128 { + let b = b as f32; + let r = simd_insert(a, 0, b); + transmute(r) +} + +/// Convert the unsigned 32-bit integer b to a double-precision (64-bit) floating-point element, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtu32_sd&expand=2031) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtusi2sd))] +pub unsafe fn _mm_cvtu32_sd(a: __m128d, b: u32) -> __m128d { + let b = b as f64; + let r = simd_insert(a, 0, b); + transmute(r) +} + +/// Compare the lower single-precision (32-bit) floating-point element in a and b based on the comparison operand specified by imm8, and return the boolean result (0 or 1).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_comi_round_ss&expand=1175) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp, IMM5 = 5, SAE = 4))] //should be vcomiss +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm_comi_round_ss(a: __m128, b: __m128) -> i32 { + static_assert_imm5!(IMM5); + static_assert_mantissas_sae!(SAE); + let a = a.as_f32x4(); + let b = b.as_f32x4(); + let r = vcomiss(a, b, IMM5, SAE); + transmute(r) +} + +/// Compare the lower double-precision (64-bit) floating-point element in a and b based on the comparison operand specified by imm8, and return the boolean result (0 or 1).\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_comi_round_sd&expand=1174) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcmp, IMM5 = 5, SAE = 4))] //should be vcomisd +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm_comi_round_sd(a: __m128d, b: __m128d) -> i32 { + static_assert_imm5!(IMM5); + static_assert_mantissas_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f64x2(); + let r = vcomisd(a, b, IMM5, SAE); + transmute(r) +} + +/// Equal +pub const _MM_CMPINT_EQ: _MM_CMPINT_ENUM = 0x00; +/// Less-than +pub const _MM_CMPINT_LT: _MM_CMPINT_ENUM = 0x01; +/// Less-than-or-equal +pub const _MM_CMPINT_LE: _MM_CMPINT_ENUM = 0x02; +/// False +pub const _MM_CMPINT_FALSE: _MM_CMPINT_ENUM = 0x03; +/// Not-equal +pub const _MM_CMPINT_NE: _MM_CMPINT_ENUM = 0x04; +/// Not less-than +pub const _MM_CMPINT_NLT: _MM_CMPINT_ENUM = 0x05; +/// Not less-than-or-equal +pub const _MM_CMPINT_NLE: _MM_CMPINT_ENUM = 0x06; +/// True +pub const _MM_CMPINT_TRUE: _MM_CMPINT_ENUM = 0x07; + +/// interval [1, 2) +pub const _MM_MANT_NORM_1_2: _MM_MANTISSA_NORM_ENUM = 0x00; +/// interval [0.5, 2) +pub const _MM_MANT_NORM_P5_2: _MM_MANTISSA_NORM_ENUM = 0x01; +/// interval [0.5, 1) +pub const _MM_MANT_NORM_P5_1: _MM_MANTISSA_NORM_ENUM = 0x02; +/// interval [0.75, 1.5) +pub const _MM_MANT_NORM_P75_1P5: _MM_MANTISSA_NORM_ENUM = 0x03; + +/// sign = sign(SRC) +pub const _MM_MANT_SIGN_SRC: _MM_MANTISSA_SIGN_ENUM = 0x00; +/// sign = 0 +pub const _MM_MANT_SIGN_ZERO: _MM_MANTISSA_SIGN_ENUM = 0x01; +/// DEST = NaN if sign(SRC) = 1 +pub const _MM_MANT_SIGN_NAN: _MM_MANTISSA_SIGN_ENUM = 0x02; + +pub const _MM_PERM_AAAA: _MM_PERM_ENUM = 0x00; +pub const _MM_PERM_AAAB: _MM_PERM_ENUM = 0x01; +pub const _MM_PERM_AAAC: _MM_PERM_ENUM = 0x02; +pub const _MM_PERM_AAAD: _MM_PERM_ENUM = 0x03; +pub const _MM_PERM_AABA: _MM_PERM_ENUM = 0x04; +pub const _MM_PERM_AABB: _MM_PERM_ENUM = 0x05; +pub const _MM_PERM_AABC: _MM_PERM_ENUM = 0x06; +pub const _MM_PERM_AABD: _MM_PERM_ENUM = 0x07; +pub const _MM_PERM_AACA: _MM_PERM_ENUM = 0x08; +pub const _MM_PERM_AACB: _MM_PERM_ENUM = 0x09; +pub const _MM_PERM_AACC: _MM_PERM_ENUM = 0x0A; +pub const _MM_PERM_AACD: _MM_PERM_ENUM = 0x0B; +pub const _MM_PERM_AADA: _MM_PERM_ENUM = 0x0C; +pub const _MM_PERM_AADB: _MM_PERM_ENUM = 0x0D; +pub const _MM_PERM_AADC: _MM_PERM_ENUM = 0x0E; +pub const _MM_PERM_AADD: _MM_PERM_ENUM = 0x0F; +pub const _MM_PERM_ABAA: _MM_PERM_ENUM = 0x10; +pub const _MM_PERM_ABAB: _MM_PERM_ENUM = 0x11; +pub const _MM_PERM_ABAC: _MM_PERM_ENUM = 0x12; +pub const _MM_PERM_ABAD: _MM_PERM_ENUM = 0x13; +pub const _MM_PERM_ABBA: _MM_PERM_ENUM = 0x14; +pub const _MM_PERM_ABBB: _MM_PERM_ENUM = 0x15; +pub const _MM_PERM_ABBC: _MM_PERM_ENUM = 0x16; +pub const _MM_PERM_ABBD: _MM_PERM_ENUM = 0x17; +pub const _MM_PERM_ABCA: _MM_PERM_ENUM = 0x18; +pub const _MM_PERM_ABCB: _MM_PERM_ENUM = 0x19; +pub const _MM_PERM_ABCC: _MM_PERM_ENUM = 0x1A; +pub const _MM_PERM_ABCD: _MM_PERM_ENUM = 0x1B; +pub const _MM_PERM_ABDA: _MM_PERM_ENUM = 0x1C; +pub const _MM_PERM_ABDB: _MM_PERM_ENUM = 0x1D; +pub const _MM_PERM_ABDC: _MM_PERM_ENUM = 0x1E; +pub const _MM_PERM_ABDD: _MM_PERM_ENUM = 0x1F; +pub const _MM_PERM_ACAA: _MM_PERM_ENUM = 0x20; +pub const _MM_PERM_ACAB: _MM_PERM_ENUM = 0x21; +pub const _MM_PERM_ACAC: _MM_PERM_ENUM = 0x22; +pub const _MM_PERM_ACAD: _MM_PERM_ENUM = 0x23; +pub const _MM_PERM_ACBA: _MM_PERM_ENUM = 0x24; +pub const _MM_PERM_ACBB: _MM_PERM_ENUM = 0x25; +pub const _MM_PERM_ACBC: _MM_PERM_ENUM = 0x26; +pub const _MM_PERM_ACBD: _MM_PERM_ENUM = 0x27; +pub const _MM_PERM_ACCA: _MM_PERM_ENUM = 0x28; +pub const _MM_PERM_ACCB: _MM_PERM_ENUM = 0x29; +pub const _MM_PERM_ACCC: _MM_PERM_ENUM = 0x2A; +pub const _MM_PERM_ACCD: _MM_PERM_ENUM = 0x2B; +pub const _MM_PERM_ACDA: _MM_PERM_ENUM = 0x2C; +pub const _MM_PERM_ACDB: _MM_PERM_ENUM = 0x2D; +pub const _MM_PERM_ACDC: _MM_PERM_ENUM = 0x2E; +pub const _MM_PERM_ACDD: _MM_PERM_ENUM = 0x2F; +pub const _MM_PERM_ADAA: _MM_PERM_ENUM = 0x30; +pub const _MM_PERM_ADAB: _MM_PERM_ENUM = 0x31; +pub const _MM_PERM_ADAC: _MM_PERM_ENUM = 0x32; +pub const _MM_PERM_ADAD: _MM_PERM_ENUM = 0x33; +pub const _MM_PERM_ADBA: _MM_PERM_ENUM = 0x34; +pub const _MM_PERM_ADBB: _MM_PERM_ENUM = 0x35; +pub const _MM_PERM_ADBC: _MM_PERM_ENUM = 0x36; +pub const _MM_PERM_ADBD: _MM_PERM_ENUM = 0x37; +pub const _MM_PERM_ADCA: _MM_PERM_ENUM = 0x38; +pub const _MM_PERM_ADCB: _MM_PERM_ENUM = 0x39; +pub const _MM_PERM_ADCC: _MM_PERM_ENUM = 0x3A; +pub const _MM_PERM_ADCD: _MM_PERM_ENUM = 0x3B; +pub const _MM_PERM_ADDA: _MM_PERM_ENUM = 0x3C; +pub const _MM_PERM_ADDB: _MM_PERM_ENUM = 0x3D; +pub const _MM_PERM_ADDC: _MM_PERM_ENUM = 0x3E; +pub const _MM_PERM_ADDD: _MM_PERM_ENUM = 0x3F; +pub const _MM_PERM_BAAA: _MM_PERM_ENUM = 0x40; +pub const _MM_PERM_BAAB: _MM_PERM_ENUM = 0x41; +pub const _MM_PERM_BAAC: _MM_PERM_ENUM = 0x42; +pub const _MM_PERM_BAAD: _MM_PERM_ENUM = 0x43; +pub const _MM_PERM_BABA: _MM_PERM_ENUM = 0x44; +pub const _MM_PERM_BABB: _MM_PERM_ENUM = 0x45; +pub const _MM_PERM_BABC: _MM_PERM_ENUM = 0x46; +pub const _MM_PERM_BABD: _MM_PERM_ENUM = 0x47; +pub const _MM_PERM_BACA: _MM_PERM_ENUM = 0x48; +pub const _MM_PERM_BACB: _MM_PERM_ENUM = 0x49; +pub const _MM_PERM_BACC: _MM_PERM_ENUM = 0x4A; +pub const _MM_PERM_BACD: _MM_PERM_ENUM = 0x4B; +pub const _MM_PERM_BADA: _MM_PERM_ENUM = 0x4C; +pub const _MM_PERM_BADB: _MM_PERM_ENUM = 0x4D; +pub const _MM_PERM_BADC: _MM_PERM_ENUM = 0x4E; +pub const _MM_PERM_BADD: _MM_PERM_ENUM = 0x4F; +pub const _MM_PERM_BBAA: _MM_PERM_ENUM = 0x50; +pub const _MM_PERM_BBAB: _MM_PERM_ENUM = 0x51; +pub const _MM_PERM_BBAC: _MM_PERM_ENUM = 0x52; +pub const _MM_PERM_BBAD: _MM_PERM_ENUM = 0x53; +pub const _MM_PERM_BBBA: _MM_PERM_ENUM = 0x54; +pub const _MM_PERM_BBBB: _MM_PERM_ENUM = 0x55; +pub const _MM_PERM_BBBC: _MM_PERM_ENUM = 0x56; +pub const _MM_PERM_BBBD: _MM_PERM_ENUM = 0x57; +pub const _MM_PERM_BBCA: _MM_PERM_ENUM = 0x58; +pub const _MM_PERM_BBCB: _MM_PERM_ENUM = 0x59; +pub const _MM_PERM_BBCC: _MM_PERM_ENUM = 0x5A; +pub const _MM_PERM_BBCD: _MM_PERM_ENUM = 0x5B; +pub const _MM_PERM_BBDA: _MM_PERM_ENUM = 0x5C; +pub const _MM_PERM_BBDB: _MM_PERM_ENUM = 0x5D; +pub const _MM_PERM_BBDC: _MM_PERM_ENUM = 0x5E; +pub const _MM_PERM_BBDD: _MM_PERM_ENUM = 0x5F; +pub const _MM_PERM_BCAA: _MM_PERM_ENUM = 0x60; +pub const _MM_PERM_BCAB: _MM_PERM_ENUM = 0x61; +pub const _MM_PERM_BCAC: _MM_PERM_ENUM = 0x62; +pub const _MM_PERM_BCAD: _MM_PERM_ENUM = 0x63; +pub const _MM_PERM_BCBA: _MM_PERM_ENUM = 0x64; +pub const _MM_PERM_BCBB: _MM_PERM_ENUM = 0x65; +pub const _MM_PERM_BCBC: _MM_PERM_ENUM = 0x66; +pub const _MM_PERM_BCBD: _MM_PERM_ENUM = 0x67; +pub const _MM_PERM_BCCA: _MM_PERM_ENUM = 0x68; +pub const _MM_PERM_BCCB: _MM_PERM_ENUM = 0x69; +pub const _MM_PERM_BCCC: _MM_PERM_ENUM = 0x6A; +pub const _MM_PERM_BCCD: _MM_PERM_ENUM = 0x6B; +pub const _MM_PERM_BCDA: _MM_PERM_ENUM = 0x6C; +pub const _MM_PERM_BCDB: _MM_PERM_ENUM = 0x6D; +pub const _MM_PERM_BCDC: _MM_PERM_ENUM = 0x6E; +pub const _MM_PERM_BCDD: _MM_PERM_ENUM = 0x6F; +pub const _MM_PERM_BDAA: _MM_PERM_ENUM = 0x70; +pub const _MM_PERM_BDAB: _MM_PERM_ENUM = 0x71; +pub const _MM_PERM_BDAC: _MM_PERM_ENUM = 0x72; +pub const _MM_PERM_BDAD: _MM_PERM_ENUM = 0x73; +pub const _MM_PERM_BDBA: _MM_PERM_ENUM = 0x74; +pub const _MM_PERM_BDBB: _MM_PERM_ENUM = 0x75; +pub const _MM_PERM_BDBC: _MM_PERM_ENUM = 0x76; +pub const _MM_PERM_BDBD: _MM_PERM_ENUM = 0x77; +pub const _MM_PERM_BDCA: _MM_PERM_ENUM = 0x78; +pub const _MM_PERM_BDCB: _MM_PERM_ENUM = 0x79; +pub const _MM_PERM_BDCC: _MM_PERM_ENUM = 0x7A; +pub const _MM_PERM_BDCD: _MM_PERM_ENUM = 0x7B; +pub const _MM_PERM_BDDA: _MM_PERM_ENUM = 0x7C; +pub const _MM_PERM_BDDB: _MM_PERM_ENUM = 0x7D; +pub const _MM_PERM_BDDC: _MM_PERM_ENUM = 0x7E; +pub const _MM_PERM_BDDD: _MM_PERM_ENUM = 0x7F; +pub const _MM_PERM_CAAA: _MM_PERM_ENUM = 0x80; +pub const _MM_PERM_CAAB: _MM_PERM_ENUM = 0x81; +pub const _MM_PERM_CAAC: _MM_PERM_ENUM = 0x82; +pub const _MM_PERM_CAAD: _MM_PERM_ENUM = 0x83; +pub const _MM_PERM_CABA: _MM_PERM_ENUM = 0x84; +pub const _MM_PERM_CABB: _MM_PERM_ENUM = 0x85; +pub const _MM_PERM_CABC: _MM_PERM_ENUM = 0x86; +pub const _MM_PERM_CABD: _MM_PERM_ENUM = 0x87; +pub const _MM_PERM_CACA: _MM_PERM_ENUM = 0x88; +pub const _MM_PERM_CACB: _MM_PERM_ENUM = 0x89; +pub const _MM_PERM_CACC: _MM_PERM_ENUM = 0x8A; +pub const _MM_PERM_CACD: _MM_PERM_ENUM = 0x8B; +pub const _MM_PERM_CADA: _MM_PERM_ENUM = 0x8C; +pub const _MM_PERM_CADB: _MM_PERM_ENUM = 0x8D; +pub const _MM_PERM_CADC: _MM_PERM_ENUM = 0x8E; +pub const _MM_PERM_CADD: _MM_PERM_ENUM = 0x8F; +pub const _MM_PERM_CBAA: _MM_PERM_ENUM = 0x90; +pub const _MM_PERM_CBAB: _MM_PERM_ENUM = 0x91; +pub const _MM_PERM_CBAC: _MM_PERM_ENUM = 0x92; +pub const _MM_PERM_CBAD: _MM_PERM_ENUM = 0x93; +pub const _MM_PERM_CBBA: _MM_PERM_ENUM = 0x94; +pub const _MM_PERM_CBBB: _MM_PERM_ENUM = 0x95; +pub const _MM_PERM_CBBC: _MM_PERM_ENUM = 0x96; +pub const _MM_PERM_CBBD: _MM_PERM_ENUM = 0x97; +pub const _MM_PERM_CBCA: _MM_PERM_ENUM = 0x98; +pub const _MM_PERM_CBCB: _MM_PERM_ENUM = 0x99; +pub const _MM_PERM_CBCC: _MM_PERM_ENUM = 0x9A; +pub const _MM_PERM_CBCD: _MM_PERM_ENUM = 0x9B; +pub const _MM_PERM_CBDA: _MM_PERM_ENUM = 0x9C; +pub const _MM_PERM_CBDB: _MM_PERM_ENUM = 0x9D; +pub const _MM_PERM_CBDC: _MM_PERM_ENUM = 0x9E; +pub const _MM_PERM_CBDD: _MM_PERM_ENUM = 0x9F; +pub const _MM_PERM_CCAA: _MM_PERM_ENUM = 0xA0; +pub const _MM_PERM_CCAB: _MM_PERM_ENUM = 0xA1; +pub const _MM_PERM_CCAC: _MM_PERM_ENUM = 0xA2; +pub const _MM_PERM_CCAD: _MM_PERM_ENUM = 0xA3; +pub const _MM_PERM_CCBA: _MM_PERM_ENUM = 0xA4; +pub const _MM_PERM_CCBB: _MM_PERM_ENUM = 0xA5; +pub const _MM_PERM_CCBC: _MM_PERM_ENUM = 0xA6; +pub const _MM_PERM_CCBD: _MM_PERM_ENUM = 0xA7; +pub const _MM_PERM_CCCA: _MM_PERM_ENUM = 0xA8; +pub const _MM_PERM_CCCB: _MM_PERM_ENUM = 0xA9; +pub const _MM_PERM_CCCC: _MM_PERM_ENUM = 0xAA; +pub const _MM_PERM_CCCD: _MM_PERM_ENUM = 0xAB; +pub const _MM_PERM_CCDA: _MM_PERM_ENUM = 0xAC; +pub const _MM_PERM_CCDB: _MM_PERM_ENUM = 0xAD; +pub const _MM_PERM_CCDC: _MM_PERM_ENUM = 0xAE; +pub const _MM_PERM_CCDD: _MM_PERM_ENUM = 0xAF; +pub const _MM_PERM_CDAA: _MM_PERM_ENUM = 0xB0; +pub const _MM_PERM_CDAB: _MM_PERM_ENUM = 0xB1; +pub const _MM_PERM_CDAC: _MM_PERM_ENUM = 0xB2; +pub const _MM_PERM_CDAD: _MM_PERM_ENUM = 0xB3; +pub const _MM_PERM_CDBA: _MM_PERM_ENUM = 0xB4; +pub const _MM_PERM_CDBB: _MM_PERM_ENUM = 0xB5; +pub const _MM_PERM_CDBC: _MM_PERM_ENUM = 0xB6; +pub const _MM_PERM_CDBD: _MM_PERM_ENUM = 0xB7; +pub const _MM_PERM_CDCA: _MM_PERM_ENUM = 0xB8; +pub const _MM_PERM_CDCB: _MM_PERM_ENUM = 0xB9; +pub const _MM_PERM_CDCC: _MM_PERM_ENUM = 0xBA; +pub const _MM_PERM_CDCD: _MM_PERM_ENUM = 0xBB; +pub const _MM_PERM_CDDA: _MM_PERM_ENUM = 0xBC; +pub const _MM_PERM_CDDB: _MM_PERM_ENUM = 0xBD; +pub const _MM_PERM_CDDC: _MM_PERM_ENUM = 0xBE; +pub const _MM_PERM_CDDD: _MM_PERM_ENUM = 0xBF; +pub const _MM_PERM_DAAA: _MM_PERM_ENUM = 0xC0; +pub const _MM_PERM_DAAB: _MM_PERM_ENUM = 0xC1; +pub const _MM_PERM_DAAC: _MM_PERM_ENUM = 0xC2; +pub const _MM_PERM_DAAD: _MM_PERM_ENUM = 0xC3; +pub const _MM_PERM_DABA: _MM_PERM_ENUM = 0xC4; +pub const _MM_PERM_DABB: _MM_PERM_ENUM = 0xC5; +pub const _MM_PERM_DABC: _MM_PERM_ENUM = 0xC6; +pub const _MM_PERM_DABD: _MM_PERM_ENUM = 0xC7; +pub const _MM_PERM_DACA: _MM_PERM_ENUM = 0xC8; +pub const _MM_PERM_DACB: _MM_PERM_ENUM = 0xC9; +pub const _MM_PERM_DACC: _MM_PERM_ENUM = 0xCA; +pub const _MM_PERM_DACD: _MM_PERM_ENUM = 0xCB; +pub const _MM_PERM_DADA: _MM_PERM_ENUM = 0xCC; +pub const _MM_PERM_DADB: _MM_PERM_ENUM = 0xCD; +pub const _MM_PERM_DADC: _MM_PERM_ENUM = 0xCE; +pub const _MM_PERM_DADD: _MM_PERM_ENUM = 0xCF; +pub const _MM_PERM_DBAA: _MM_PERM_ENUM = 0xD0; +pub const _MM_PERM_DBAB: _MM_PERM_ENUM = 0xD1; +pub const _MM_PERM_DBAC: _MM_PERM_ENUM = 0xD2; +pub const _MM_PERM_DBAD: _MM_PERM_ENUM = 0xD3; +pub const _MM_PERM_DBBA: _MM_PERM_ENUM = 0xD4; +pub const _MM_PERM_DBBB: _MM_PERM_ENUM = 0xD5; +pub const _MM_PERM_DBBC: _MM_PERM_ENUM = 0xD6; +pub const _MM_PERM_DBBD: _MM_PERM_ENUM = 0xD7; +pub const _MM_PERM_DBCA: _MM_PERM_ENUM = 0xD8; +pub const _MM_PERM_DBCB: _MM_PERM_ENUM = 0xD9; +pub const _MM_PERM_DBCC: _MM_PERM_ENUM = 0xDA; +pub const _MM_PERM_DBCD: _MM_PERM_ENUM = 0xDB; +pub const _MM_PERM_DBDA: _MM_PERM_ENUM = 0xDC; +pub const _MM_PERM_DBDB: _MM_PERM_ENUM = 0xDD; +pub const _MM_PERM_DBDC: _MM_PERM_ENUM = 0xDE; +pub const _MM_PERM_DBDD: _MM_PERM_ENUM = 0xDF; +pub const _MM_PERM_DCAA: _MM_PERM_ENUM = 0xE0; +pub const _MM_PERM_DCAB: _MM_PERM_ENUM = 0xE1; +pub const _MM_PERM_DCAC: _MM_PERM_ENUM = 0xE2; +pub const _MM_PERM_DCAD: _MM_PERM_ENUM = 0xE3; +pub const _MM_PERM_DCBA: _MM_PERM_ENUM = 0xE4; +pub const _MM_PERM_DCBB: _MM_PERM_ENUM = 0xE5; +pub const _MM_PERM_DCBC: _MM_PERM_ENUM = 0xE6; +pub const _MM_PERM_DCBD: _MM_PERM_ENUM = 0xE7; +pub const _MM_PERM_DCCA: _MM_PERM_ENUM = 0xE8; +pub const _MM_PERM_DCCB: _MM_PERM_ENUM = 0xE9; +pub const _MM_PERM_DCCC: _MM_PERM_ENUM = 0xEA; +pub const _MM_PERM_DCCD: _MM_PERM_ENUM = 0xEB; +pub const _MM_PERM_DCDA: _MM_PERM_ENUM = 0xEC; +pub const _MM_PERM_DCDB: _MM_PERM_ENUM = 0xED; +pub const _MM_PERM_DCDC: _MM_PERM_ENUM = 0xEE; +pub const _MM_PERM_DCDD: _MM_PERM_ENUM = 0xEF; +pub const _MM_PERM_DDAA: _MM_PERM_ENUM = 0xF0; +pub const _MM_PERM_DDAB: _MM_PERM_ENUM = 0xF1; +pub const _MM_PERM_DDAC: _MM_PERM_ENUM = 0xF2; +pub const _MM_PERM_DDAD: _MM_PERM_ENUM = 0xF3; +pub const _MM_PERM_DDBA: _MM_PERM_ENUM = 0xF4; +pub const _MM_PERM_DDBB: _MM_PERM_ENUM = 0xF5; +pub const _MM_PERM_DDBC: _MM_PERM_ENUM = 0xF6; +pub const _MM_PERM_DDBD: _MM_PERM_ENUM = 0xF7; +pub const _MM_PERM_DDCA: _MM_PERM_ENUM = 0xF8; +pub const _MM_PERM_DDCB: _MM_PERM_ENUM = 0xF9; +pub const _MM_PERM_DDCC: _MM_PERM_ENUM = 0xFA; +pub const _MM_PERM_DDCD: _MM_PERM_ENUM = 0xFB; +pub const _MM_PERM_DDDA: _MM_PERM_ENUM = 0xFC; +pub const _MM_PERM_DDDB: _MM_PERM_ENUM = 0xFD; +pub const _MM_PERM_DDDC: _MM_PERM_ENUM = 0xFE; +pub const _MM_PERM_DDDD: _MM_PERM_ENUM = 0xFF; + +#[allow(improper_ctypes)] +extern "C" { + #[link_name = "llvm.x86.avx512.pmul.dq.512"] + fn vpmuldq(a: i32x16, b: i32x16) -> i64x8; + #[link_name = "llvm.x86.avx512.pmulu.dq.512"] + fn vpmuludq(a: u32x16, b: u32x16) -> u64x8; + + #[link_name = "llvm.x86.avx512.mask.pmaxs.d.512"] + fn vpmaxsd(a: i32x16, b: i32x16) -> i32x16; + + #[link_name = "llvm.x86.avx512.mask.pmaxs.q.512"] + fn vpmaxsq(a: i64x8, b: i64x8) -> i64x8; + #[link_name = "llvm.x86.avx512.mask.pmaxs.q.256"] + fn vpmaxsq256(a: i64x4, b: i64x4) -> i64x4; + #[link_name = "llvm.x86.avx512.mask.pmaxs.q.128"] + fn vpmaxsq128(a: i64x2, b: i64x2) -> i64x2; + + #[link_name = "llvm.x86.avx512.mask.pmins.d.512"] + fn vpminsd(a: i32x16, b: i32x16) -> i32x16; + + #[link_name = "llvm.x86.avx512.mask.pmins.q.512"] + fn vpminsq(a: i64x8, b: i64x8) -> i64x8; + #[link_name = "llvm.x86.avx512.mask.pmins.q.256"] + fn vpminsq256(a: i64x4, b: i64x4) -> i64x4; + #[link_name = "llvm.x86.avx512.mask.pmins.q.128"] + fn vpminsq128(a: i64x2, b: i64x2) -> i64x2; + + #[link_name = "llvm.x86.avx512.mask.pmaxu.d.512"] + fn vpmaxud(a: u32x16, b: u32x16) -> u32x16; + + #[link_name = "llvm.x86.avx512.mask.pmaxu.q.512"] + fn vpmaxuq(a: u64x8, b: u64x8) -> u64x8; + #[link_name = "llvm.x86.avx512.mask.pmaxu.q.256"] + fn vpmaxuq256(a: u64x4, b: u64x4) -> u64x4; + #[link_name = "llvm.x86.avx512.mask.pmaxu.q.128"] + fn vpmaxuq128(a: u64x2, b: u64x2) -> u64x2; + + #[link_name = "llvm.x86.avx512.mask.pminu.d.512"] + fn vpminud(a: u32x16, b: u32x16) -> u32x16; + + #[link_name = "llvm.x86.avx512.mask.pminu.q.512"] + fn vpminuq(a: u64x8, b: u64x8) -> u64x8; + #[link_name = "llvm.x86.avx512.mask.pminu.q.256"] + fn vpminuq256(a: u64x4, b: u64x4) -> u64x4; + #[link_name = "llvm.x86.avx512.mask.pminu.q.128"] + fn vpminuq128(a: u64x2, b: u64x2) -> u64x2; + + #[link_name = "llvm.x86.avx512.sqrt.ps.512"] + fn vsqrtps(a: f32x16, rounding: i32) -> f32x16; + #[link_name = "llvm.x86.avx512.sqrt.pd.512"] + fn vsqrtpd(a: f64x8, rounding: i32) -> f64x8; + + #[link_name = "llvm.fma.v16f32"] + fn vfmadd132ps(a: f32x16, b: f32x16, c: f32x16) -> f32x16; + #[link_name = "llvm.fma.v8f64"] + fn vfmadd132pd(a: f64x8, b: f64x8, c: f64x8) -> f64x8; + + #[link_name = "llvm.x86.avx512.vfmadd.ps.512"] + fn vfmadd132psround(a: f32x16, b: f32x16, c: f32x16, rounding: i32) -> f32x16; + #[link_name = "llvm.x86.avx512.vfmadd.pd.512"] + fn vfmadd132pdround(a: f64x8, b: f64x8, c: f64x8, rounding: i32) -> f64x8; + + #[link_name = "llvm.x86.avx512.vfmaddsub.ps.512"] + fn vfmaddsub213ps(a: f32x16, b: f32x16, c: f32x16, d: i32) -> f32x16; //from clang + #[link_name = "llvm.x86.avx512.vfmaddsub.pd.512"] + fn vfmaddsub213pd(a: f64x8, b: f64x8, c: f64x8, d: i32) -> f64x8; //from clang + + #[link_name = "llvm.x86.avx512.add.ps.512"] + fn vaddps(a: f32x16, b: f32x16, rounding: i32) -> f32x16; + #[link_name = "llvm.x86.avx512.add.pd.512"] + fn vaddpd(a: f64x8, b: f64x8, rounding: i32) -> f64x8; + #[link_name = "llvm.x86.avx512.sub.ps.512"] + fn vsubps(a: f32x16, b: f32x16, rounding: i32) -> f32x16; + #[link_name = "llvm.x86.avx512.sub.pd.512"] + fn vsubpd(a: f64x8, b: f64x8, rounding: i32) -> f64x8; + #[link_name = "llvm.x86.avx512.mul.ps.512"] + fn vmulps(a: f32x16, b: f32x16, rounding: i32) -> f32x16; + #[link_name = "llvm.x86.avx512.mul.pd.512"] + fn vmulpd(a: f64x8, b: f64x8, rounding: i32) -> f64x8; + #[link_name = "llvm.x86.avx512.div.ps.512"] + fn vdivps(a: f32x16, b: f32x16, rounding: i32) -> f32x16; + #[link_name = "llvm.x86.avx512.div.pd.512"] + fn vdivpd(a: f64x8, b: f64x8, rounding: i32) -> f64x8; + + #[link_name = "llvm.x86.avx512.max.ps.512"] + fn vmaxps(a: f32x16, b: f32x16, sae: i32) -> f32x16; + #[link_name = "llvm.x86.avx512.max.pd.512"] + fn vmaxpd(a: f64x8, b: f64x8, sae: i32) -> f64x8; + #[link_name = "llvm.x86.avx512.min.ps.512"] + fn vminps(a: f32x16, b: f32x16, sae: i32) -> f32x16; + #[link_name = "llvm.x86.avx512.min.pd.512"] + fn vminpd(a: f64x8, b: f64x8, sae: i32) -> f64x8; + + #[link_name = "llvm.x86.avx512.mask.getexp.ps.512"] + fn vgetexpps(a: f32x16, src: f32x16, m: u16, sae: i32) -> f32x16; + + #[link_name = "llvm.x86.avx512.mask.getexp.ps.256"] + fn vgetexpps256(a: f32x8, src: f32x8, m: u8) -> f32x8; + #[link_name = "llvm.x86.avx512.mask.getexp.ps.128"] + fn vgetexpps128(a: f32x4, src: f32x4, m: u8) -> f32x4; + + #[link_name = "llvm.x86.avx512.mask.getexp.pd.512"] + fn vgetexppd(a: f64x8, src: f64x8, m: u8, sae: i32) -> f64x8; + #[link_name = "llvm.x86.avx512.mask.getexp.pd.256"] + fn vgetexppd256(a: f64x4, src: f64x4, m: u8) -> f64x4; + #[link_name = "llvm.x86.avx512.mask.getexp.pd.128"] + fn vgetexppd128(a: f64x2, src: f64x2, m: u8) -> f64x2; + + #[link_name = "llvm.x86.avx512.mask.rndscale.ps.512"] + fn vrndscaleps(a: f32x16, imm8: i32, src: f32x16, mask: u16, sae: i32) -> f32x16; + #[link_name = "llvm.x86.avx512.mask.rndscale.ps.256"] + fn vrndscaleps256(a: f32x8, imm8: i32, src: f32x8, mask: u8) -> f32x8; + #[link_name = "llvm.x86.avx512.mask.rndscale.ps.128"] + fn vrndscaleps128(a: f32x4, imm8: i32, src: f32x4, mask: u8) -> f32x4; + + #[link_name = "llvm.x86.avx512.mask.rndscale.pd.512"] + fn vrndscalepd(a: f64x8, imm8: i32, src: f64x8, mask: u8, sae: i32) -> f64x8; + #[link_name = "llvm.x86.avx512.mask.rndscale.pd.256"] + fn vrndscalepd256(a: f64x4, imm8: i32, src: f64x4, mask: u8) -> f64x4; + #[link_name = "llvm.x86.avx512.mask.rndscale.pd.128"] + fn vrndscalepd128(a: f64x2, imm8: i32, src: f64x2, mask: u8) -> f64x2; + + #[link_name = "llvm.x86.avx512.mask.scalef.ps.512"] + fn vscalefps(a: f32x16, b: f32x16, src: f32x16, mask: u16, rounding: i32) -> f32x16; + #[link_name = "llvm.x86.avx512.mask.scalef.ps.256"] + fn vscalefps256(a: f32x8, b: f32x8, src: f32x8, mask: u8) -> f32x8; + #[link_name = "llvm.x86.avx512.mask.scalef.ps.128"] + fn vscalefps128(a: f32x4, b: f32x4, src: f32x4, mask: u8) -> f32x4; + + #[link_name = "llvm.x86.avx512.mask.scalef.pd.512"] + fn vscalefpd(a: f64x8, b: f64x8, src: f64x8, mask: u8, rounding: i32) -> f64x8; + #[link_name = "llvm.x86.avx512.mask.scalef.pd.256"] + fn vscalefpd256(a: f64x4, b: f64x4, src: f64x4, mask: u8) -> f64x4; + #[link_name = "llvm.x86.avx512.mask.scalef.pd.128"] + fn vscalefpd128(a: f64x2, b: f64x2, src: f64x2, mask: u8) -> f64x2; + + #[link_name = "llvm.x86.avx512.mask.fixupimm.ps.512"] + fn vfixupimmps(a: f32x16, b: f32x16, c: i32x16, imm8: i32, mask: u16, sae: i32) -> f32x16; + #[link_name = "llvm.x86.avx512.mask.fixupimm.ps.256"] + fn vfixupimmps256(a: f32x8, b: f32x8, c: i32x8, imm8: i32, mask: u8) -> f32x8; + #[link_name = "llvm.x86.avx512.mask.fixupimm.ps.128"] + fn vfixupimmps128(a: f32x4, b: f32x4, c: i32x4, imm8: i32, mask: u8) -> f32x4; + + #[link_name = "llvm.x86.avx512.mask.fixupimm.pd.512"] + fn vfixupimmpd(a: f64x8, b: f64x8, c: i64x8, imm8: i32, mask: u8, sae: i32) -> f64x8; + #[link_name = "llvm.x86.avx512.mask.fixupimm.pd.256"] + fn vfixupimmpd256(a: f64x4, b: f64x4, c: i64x4, imm8: i32, mask: u8) -> f64x4; + #[link_name = "llvm.x86.avx512.mask.fixupimm.pd.128"] + fn vfixupimmpd128(a: f64x2, b: f64x2, c: i64x2, imm8: i32, mask: u8) -> f64x2; + + #[link_name = "llvm.x86.avx512.maskz.fixupimm.ps.512"] + fn vfixupimmpsz(a: f32x16, b: f32x16, c: i32x16, imm8: i32, mask: u16, sae: i32) -> f32x16; + #[link_name = "llvm.x86.avx512.maskz.fixupimm.ps.256"] + fn vfixupimmpsz256(a: f32x8, b: f32x8, c: i32x8, imm8: i32, mask: u8) -> f32x8; + #[link_name = "llvm.x86.avx512.maskz.fixupimm.ps.128"] + fn vfixupimmpsz128(a: f32x4, b: f32x4, c: i32x4, imm8: i32, mask: u8) -> f32x4; + + #[link_name = "llvm.x86.avx512.maskz.fixupimm.pd.512"] + fn vfixupimmpdz(a: f64x8, b: f64x8, c: i64x8, imm8: i32, mask: u8, sae: i32) -> f64x8; + #[link_name = "llvm.x86.avx512.maskz.fixupimm.pd.256"] + fn vfixupimmpdz256(a: f64x4, b: f64x4, c: i64x4, imm8: i32, mask: u8) -> f64x4; + #[link_name = "llvm.x86.avx512.maskz.fixupimm.pd.128"] + fn vfixupimmpdz128(a: f64x2, b: f64x2, c: i64x2, imm8: i32, mask: u8) -> f64x2; + + #[link_name = "llvm.x86.avx512.pternlog.d.512"] + fn vpternlogd(a: i32x16, b: i32x16, c: i32x16, imm8: i32) -> i32x16; + #[link_name = "llvm.x86.avx512.pternlog.d.256"] + fn vpternlogd256(a: i32x8, b: i32x8, c: i32x8, imm8: i32) -> i32x8; + #[link_name = "llvm.x86.avx512.pternlog.d.128"] + fn vpternlogd128(a: i32x4, b: i32x4, c: i32x4, imm8: i32) -> i32x4; + + #[link_name = "llvm.x86.avx512.pternlog.q.512"] + fn vpternlogq(a: i64x8, b: i64x8, c: i64x8, imm8: i32) -> i64x8; + #[link_name = "llvm.x86.avx512.pternlog.q.256"] + fn vpternlogq256(a: i64x4, b: i64x4, c: i64x4, imm8: i32) -> i64x4; + #[link_name = "llvm.x86.avx512.pternlog.q.128"] + fn vpternlogq128(a: i64x2, b: i64x2, c: i64x2, imm8: i32) -> i64x2; + + #[link_name = "llvm.x86.avx512.mask.getmant.ps.512"] + fn vgetmantps(a: f32x16, mantissas: i32, src: f32x16, m: u16, sae: i32) -> f32x16; + #[link_name = "llvm.x86.avx512.mask.getmant.ps.256"] + fn vgetmantps256(a: f32x8, mantissas: i32, src: f32x8, m: u8) -> f32x8; + #[link_name = "llvm.x86.avx512.mask.getmant.ps.128"] + fn vgetmantps128(a: f32x4, mantissas: i32, src: f32x4, m: u8) -> f32x4; + + #[link_name = "llvm.x86.avx512.mask.getmant.pd.512"] + fn vgetmantpd(a: f64x8, mantissas: i32, src: f64x8, m: u8, sae: i32) -> f64x8; + #[link_name = "llvm.x86.avx512.mask.getmant.pd.256"] + fn vgetmantpd256(a: f64x4, mantissas: i32, src: f64x4, m: u8) -> f64x4; + #[link_name = "llvm.x86.avx512.mask.getmant.pd.128"] + fn vgetmantpd128(a: f64x2, mantissas: i32, src: f64x2, m: u8) -> f64x2; + + #[link_name = "llvm.x86.avx512.rcp14.ps.512"] + fn vrcp14ps(a: f32x16, src: f32x16, m: u16) -> f32x16; + #[link_name = "llvm.x86.avx512.rcp14.ps.256"] + fn vrcp14ps256(a: f32x8, src: f32x8, m: u8) -> f32x8; + #[link_name = "llvm.x86.avx512.rcp14.ps.128"] + fn vrcp14ps128(a: f32x4, src: f32x4, m: u8) -> f32x4; + + #[link_name = "llvm.x86.avx512.rcp14.pd.512"] + fn vrcp14pd(a: f64x8, src: f64x8, m: u8) -> f64x8; + #[link_name = "llvm.x86.avx512.rcp14.pd.256"] + fn vrcp14pd256(a: f64x4, src: f64x4, m: u8) -> f64x4; + #[link_name = "llvm.x86.avx512.rcp14.pd.128"] + fn vrcp14pd128(a: f64x2, src: f64x2, m: u8) -> f64x2; + + #[link_name = "llvm.x86.avx512.rsqrt14.ps.512"] + fn vrsqrt14ps(a: f32x16, src: f32x16, m: u16) -> f32x16; + #[link_name = "llvm.x86.avx512.rsqrt14.ps.256"] + fn vrsqrt14ps256(a: f32x8, src: f32x8, m: u8) -> f32x8; + #[link_name = "llvm.x86.avx512.rsqrt14.ps.128"] + fn vrsqrt14ps128(a: f32x4, src: f32x4, m: u8) -> f32x4; + + #[link_name = "llvm.x86.avx512.rsqrt14.pd.512"] + fn vrsqrt14pd(a: f64x8, src: f64x8, m: u8) -> f64x8; + #[link_name = "llvm.x86.avx512.rsqrt14.pd.256"] + fn vrsqrt14pd256(a: f64x4, src: f64x4, m: u8) -> f64x4; + #[link_name = "llvm.x86.avx512.rsqrt14.pd.128"] + fn vrsqrt14pd128(a: f64x2, src: f64x2, m: u8) -> f64x2; + + #[link_name = "llvm.x86.avx512.mask.cvtps2dq.512"] + fn vcvtps2dq(a: f32x16, src: i32x16, mask: u16, rounding: i32) -> i32x16; + + #[link_name = "llvm.x86.avx512.mask.cvtps2udq.512"] + fn vcvtps2udq(a: f32x16, src: u32x16, mask: u16, rounding: i32) -> u32x16; + #[link_name = "llvm.x86.avx512.mask.cvtps2udq.256"] + fn vcvtps2udq256(a: f32x8, src: u32x8, mask: u8) -> u32x8; + #[link_name = "llvm.x86.avx512.mask.cvtps2udq.128"] + fn vcvtps2udq128(a: f32x4, src: u32x4, mask: u8) -> u32x4; + + #[link_name = "llvm.x86.avx512.mask.cvtps2pd.512"] + fn vcvtps2pd(a: f32x8, src: f64x8, mask: u8, sae: i32) -> f64x8; + #[link_name = "llvm.x86.avx512.mask.cvtpd2ps.512"] + fn vcvtpd2ps(a: f64x8, src: f32x8, mask: u8, rounding: i32) -> f32x8; + + #[link_name = "llvm.x86.avx512.mask.cvtpd2dq.512"] + fn vcvtpd2dq(a: f64x8, src: i32x8, mask: u8, rounding: i32) -> i32x8; + + #[link_name = "llvm.x86.avx512.mask.cvtpd2udq.512"] + fn vcvtpd2udq(a: f64x8, src: u32x8, mask: u8, rounding: i32) -> u32x8; + #[link_name = "llvm.x86.avx512.mask.cvtpd2udq.256"] + fn vcvtpd2udq256(a: f64x4, src: u32x4, mask: u8) -> u32x4; + #[link_name = "llvm.x86.avx512.mask.cvtpd2udq.128"] + fn vcvtpd2udq128(a: f64x2, src: u32x4, mask: u8) -> u32x4; + + #[link_name = "llvm.x86.avx512.sitofp.round.v16f32.v16i32"] + fn vcvtdq2ps(a: i32x16, rounding: i32) -> f32x16; + #[link_name = "llvm.x86.avx512.uitofp.round.v16f32.v16i32"] + fn vcvtudq2ps(a: u32x16, rounding: i32) -> f32x16; + + #[link_name = "llvm.x86.avx512.mask.vcvtps2ph.512"] + fn vcvtps2ph(a: f32x16, sae: i32, src: i16x16, mask: u16) -> i16x16; + #[link_name = "llvm.x86.avx512.mask.vcvtps2ph.256"] + fn vcvtps2ph256(a: f32x8, sae: i32, src: i16x8, mask: u8) -> i16x8; + #[link_name = "llvm.x86.avx512.mask.vcvtps2ph.128"] + fn vcvtps2ph128(a: f32x4, sae: i32, src: i16x8, mask: u8) -> i16x8; + + #[link_name = "llvm.x86.avx512.mask.vcvtph2ps.512"] + fn vcvtph2ps(a: i16x16, src: f32x16, mask: u16, sae: i32) -> f32x16; + + #[link_name = "llvm.x86.avx512.mask.cvttps2dq.512"] + fn vcvttps2dq(a: f32x16, src: i32x16, mask: u16, rounding: i32) -> i32x16; + #[link_name = "llvm.x86.avx512.mask.cvttps2dq.256"] + fn vcvttps2dq256(a: f32x8, src: i32x8, mask: u8) -> i32x8; + #[link_name = "llvm.x86.avx512.mask.cvttps2dq.128"] + fn vcvttps2dq128(a: f32x4, src: i32x4, mask: u8) -> i32x4; + + #[link_name = "llvm.x86.avx512.mask.cvttps2udq.512"] + fn vcvttps2udq(a: f32x16, src: u32x16, mask: u16, rounding: i32) -> u32x16; + #[link_name = "llvm.x86.avx512.mask.cvttps2udq.256"] + fn vcvttps2udq256(a: f32x8, src: u32x8, mask: u8) -> u32x8; + #[link_name = "llvm.x86.avx512.mask.cvttps2udq.128"] + fn vcvttps2udq128(a: f32x4, src: u32x4, mask: u8) -> u32x4; + + #[link_name = "llvm.x86.avx512.mask.cvttpd2dq.512"] + fn vcvttpd2dq(a: f64x8, src: i32x8, mask: u8, rounding: i32) -> i32x8; + #[link_name = "llvm.x86.avx512.mask.cvttpd2dq.256"] + fn vcvttpd2dq256(a: f64x4, src: i32x4, mask: u8) -> i32x4; + #[link_name = "llvm.x86.avx512.mask.cvttpd2dq.128"] + fn vcvttpd2dq128(a: f64x2, src: i32x4, mask: u8) -> i32x4; + + #[link_name = "llvm.x86.avx512.mask.cvttpd2udq.512"] + fn vcvttpd2udq(a: f64x8, src: i32x8, mask: u8, rounding: i32) -> u32x8; + #[link_name = "llvm.x86.avx512.mask.cvttpd2udq.256"] + fn vcvttpd2udq256(a: f64x4, src: i32x4, mask: u8) -> u32x4; + #[link_name = "llvm.x86.avx512.mask.cvttpd2udq.128"] + fn vcvttpd2udq128(a: f64x2, src: i32x4, mask: u8) -> u32x4; + + #[link_name = "llvm.x86.avx512.mask.pmov.dw.128"] + fn vpmovdw128(a: i32x4, src: i16x8, mask: u8) -> i16x8; + #[link_name = "llvm.x86.avx512.mask.pmov.db.256"] + fn vpmovdb256(a: i32x8, src: i8x16, mask: u8) -> i8x16; + #[link_name = "llvm.x86.avx512.mask.pmov.db.128"] + fn vpmovdb128(a: i32x4, src: i8x16, mask: u8) -> i8x16; + + #[link_name = "llvm.x86.avx512.mask.pmov.qw.256"] + fn vpmovqw256(a: i64x4, src: i16x8, mask: u8) -> i16x8; + #[link_name = "llvm.x86.avx512.mask.pmov.qw.128"] + fn vpmovqw128(a: i64x2, src: i16x8, mask: u8) -> i16x8; + #[link_name = "llvm.x86.avx512.mask.pmov.qb.256"] + fn vpmovqb256(a: i64x4, src: i8x16, mask: u8) -> i8x16; + #[link_name = "llvm.x86.avx512.mask.pmov.qb.128"] + fn vpmovqb128(a: i64x2, src: i8x16, mask: u8) -> i8x16; + #[link_name = "llvm.x86.avx512.mask.pmov.qd.128"] + fn vpmovqd128(a: i64x2, src: i32x4, mask: u8) -> i32x4; + + #[link_name = "llvm.x86.avx512.mask.pmov.dw.mem.512"] + fn vpmovdwmem(mem_addr: *mut i8, a: i32x16, mask: u16); + #[link_name = "llvm.x86.avx512.mask.pmov.dw.mem.256"] + fn vpmovdwmem256(mem_addr: *mut i8, a: i32x8, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmov.dw.mem.128"] + fn vpmovdwmem128(mem_addr: *mut i8, a: i32x4, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.pmovs.dw.mem.512"] + fn vpmovsdwmem(mem_addr: *mut i8, a: i32x16, mask: u16); + #[link_name = "llvm.x86.avx512.mask.pmovs.dw.mem.256"] + fn vpmovsdwmem256(mem_addr: *mut i8, a: i32x8, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmovs.dw.mem.128"] + fn vpmovsdwmem128(mem_addr: *mut i8, a: i32x4, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.pmovus.dw.mem.512"] + fn vpmovusdwmem(mem_addr: *mut i8, a: i32x16, mask: u16); + #[link_name = "llvm.x86.avx512.mask.pmovus.dw.mem.256"] + fn vpmovusdwmem256(mem_addr: *mut i8, a: i32x8, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmovus.dw.mem.128"] + fn vpmovusdwmem128(mem_addr: *mut i8, a: i32x4, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.pmov.db.mem.512"] + fn vpmovdbmem(mem_addr: *mut i8, a: i32x16, mask: u16); + #[link_name = "llvm.x86.avx512.mask.pmov.db.mem.256"] + fn vpmovdbmem256(mem_addr: *mut i8, a: i32x8, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmov.db.mem.128"] + fn vpmovdbmem128(mem_addr: *mut i8, a: i32x4, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.pmovs.db.mem.512"] + fn vpmovsdbmem(mem_addr: *mut i8, a: i32x16, mask: u16); + #[link_name = "llvm.x86.avx512.mask.pmovs.db.mem.256"] + fn vpmovsdbmem256(mem_addr: *mut i8, a: i32x8, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmovs.db.mem.128"] + fn vpmovsdbmem128(mem_addr: *mut i8, a: i32x4, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.pmovus.db.mem.512"] + fn vpmovusdbmem(mem_addr: *mut i8, a: i32x16, mask: u16); + #[link_name = "llvm.x86.avx512.mask.pmovus.db.mem.256"] + fn vpmovusdbmem256(mem_addr: *mut i8, a: i32x8, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmovus.db.mem.128"] + fn vpmovusdbmem128(mem_addr: *mut i8, a: i32x4, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.pmov.qw.mem.512"] + fn vpmovqwmem(mem_addr: *mut i8, a: i64x8, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmov.qw.mem.256"] + fn vpmovqwmem256(mem_addr: *mut i8, a: i64x4, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmov.qw.mem.128"] + fn vpmovqwmem128(mem_addr: *mut i8, a: i64x2, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.pmovs.qw.mem.512"] + fn vpmovsqwmem(mem_addr: *mut i8, a: i64x8, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmovs.qw.mem.256"] + fn vpmovsqwmem256(mem_addr: *mut i8, a: i64x4, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmovs.qw.mem.128"] + fn vpmovsqwmem128(mem_addr: *mut i8, a: i64x2, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.pmovus.qw.mem.512"] + fn vpmovusqwmem(mem_addr: *mut i8, a: i64x8, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmovus.qw.mem.256"] + fn vpmovusqwmem256(mem_addr: *mut i8, a: i64x4, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmovus.qw.mem.128"] + fn vpmovusqwmem128(mem_addr: *mut i8, a: i64x2, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.pmov.qb.mem.512"] + fn vpmovqbmem(mem_addr: *mut i8, a: i64x8, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmov.qb.mem.256"] + fn vpmovqbmem256(mem_addr: *mut i8, a: i64x4, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmov.qb.mem.128"] + fn vpmovqbmem128(mem_addr: *mut i8, a: i64x2, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.pmovs.qb.mem.512"] + fn vpmovsqbmem(mem_addr: *mut i8, a: i64x8, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmovs.qb.mem.256"] + fn vpmovsqbmem256(mem_addr: *mut i8, a: i64x4, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmovs.qb.mem.128"] + fn vpmovsqbmem128(mem_addr: *mut i8, a: i64x2, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.pmovus.qb.mem.512"] + fn vpmovusqbmem(mem_addr: *mut i8, a: i64x8, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmovus.qb.mem.256"] + fn vpmovusqbmem256(mem_addr: *mut i8, a: i64x4, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmovus.qb.mem.128"] + fn vpmovusqbmem128(mem_addr: *mut i8, a: i64x2, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.pmov.qd.mem.512"] + fn vpmovqdmem(mem_addr: *mut i8, a: i64x8, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmov.qd.mem.256"] + fn vpmovqdmem256(mem_addr: *mut i8, a: i64x4, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmov.qd.mem.128"] + fn vpmovqdmem128(mem_addr: *mut i8, a: i64x2, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.pmovs.qd.mem.512"] + fn vpmovsqdmem(mem_addr: *mut i8, a: i64x8, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmovs.qd.mem.256"] + fn vpmovsqdmem256(mem_addr: *mut i8, a: i64x4, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmovs.qd.mem.128"] + fn vpmovsqdmem128(mem_addr: *mut i8, a: i64x2, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.pmovus.qd.mem.512"] + fn vpmovusqdmem(mem_addr: *mut i8, a: i64x8, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmovus.qd.mem.256"] + fn vpmovusqdmem256(mem_addr: *mut i8, a: i64x4, mask: u8); + #[link_name = "llvm.x86.avx512.mask.pmovus.qd.mem.128"] + fn vpmovusqdmem128(mem_addr: *mut i8, a: i64x2, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.pmov.qb.512"] + fn vpmovqb(a: i64x8, src: i8x16, mask: u8) -> i8x16; + + #[link_name = "llvm.x86.avx512.mask.pmovs.dw.512"] + fn vpmovsdw(a: i32x16, src: i16x16, mask: u16) -> i16x16; + #[link_name = "llvm.x86.avx512.mask.pmovs.dw.256"] + fn vpmovsdw256(a: i32x8, src: i16x8, mask: u8) -> i16x8; + #[link_name = "llvm.x86.avx512.mask.pmovs.dw.128"] + fn vpmovsdw128(a: i32x4, src: i16x8, mask: u8) -> i16x8; + + #[link_name = "llvm.x86.avx512.mask.pmovs.db.512"] + fn vpmovsdb(a: i32x16, src: i8x16, mask: u16) -> i8x16; + #[link_name = "llvm.x86.avx512.mask.pmovs.db.256"] + fn vpmovsdb256(a: i32x8, src: i8x16, mask: u8) -> i8x16; + #[link_name = "llvm.x86.avx512.mask.pmovs.db.128"] + fn vpmovsdb128(a: i32x4, src: i8x16, mask: u8) -> i8x16; + + #[link_name = "llvm.x86.avx512.mask.pmovs.qd.512"] + fn vpmovsqd(a: i64x8, src: i32x8, mask: u8) -> i32x8; + #[link_name = "llvm.x86.avx512.mask.pmovs.qd.256"] + fn vpmovsqd256(a: i64x4, src: i32x4, mask: u8) -> i32x4; + #[link_name = "llvm.x86.avx512.mask.pmovs.qd.128"] + fn vpmovsqd128(a: i64x2, src: i32x4, mask: u8) -> i32x4; + + #[link_name = "llvm.x86.avx512.mask.pmovs.qw.512"] + fn vpmovsqw(a: i64x8, src: i16x8, mask: u8) -> i16x8; + #[link_name = "llvm.x86.avx512.mask.pmovs.qw.256"] + fn vpmovsqw256(a: i64x4, src: i16x8, mask: u8) -> i16x8; + #[link_name = "llvm.x86.avx512.mask.pmovs.qw.128"] + fn vpmovsqw128(a: i64x2, src: i16x8, mask: u8) -> i16x8; + + #[link_name = "llvm.x86.avx512.mask.pmovs.qb.512"] + fn vpmovsqb(a: i64x8, src: i8x16, mask: u8) -> i8x16; + #[link_name = "llvm.x86.avx512.mask.pmovs.qb.256"] + fn vpmovsqb256(a: i64x4, src: i8x16, mask: u8) -> i8x16; + #[link_name = "llvm.x86.avx512.mask.pmovs.qb.128"] + fn vpmovsqb128(a: i64x2, src: i8x16, mask: u8) -> i8x16; + + #[link_name = "llvm.x86.avx512.mask.pmovus.dw.512"] + fn vpmovusdw(a: u32x16, src: u16x16, mask: u16) -> u16x16; + #[link_name = "llvm.x86.avx512.mask.pmovus.dw.256"] + fn vpmovusdw256(a: u32x8, src: u16x8, mask: u8) -> u16x8; + #[link_name = "llvm.x86.avx512.mask.pmovus.dw.128"] + fn vpmovusdw128(a: u32x4, src: u16x8, mask: u8) -> u16x8; + + #[link_name = "llvm.x86.avx512.mask.pmovus.db.512"] + fn vpmovusdb(a: u32x16, src: u8x16, mask: u16) -> u8x16; + #[link_name = "llvm.x86.avx512.mask.pmovus.db.256"] + fn vpmovusdb256(a: u32x8, src: u8x16, mask: u8) -> u8x16; + #[link_name = "llvm.x86.avx512.mask.pmovus.db.128"] + fn vpmovusdb128(a: u32x4, src: u8x16, mask: u8) -> u8x16; + + #[link_name = "llvm.x86.avx512.mask.pmovus.qd.512"] + fn vpmovusqd(a: u64x8, src: u32x8, mask: u8) -> u32x8; + #[link_name = "llvm.x86.avx512.mask.pmovus.qd.256"] + fn vpmovusqd256(a: u64x4, src: u32x4, mask: u8) -> u32x4; + #[link_name = "llvm.x86.avx512.mask.pmovus.qd.128"] + fn vpmovusqd128(a: u64x2, src: u32x4, mask: u8) -> u32x4; + + #[link_name = "llvm.x86.avx512.mask.pmovus.qw.512"] + fn vpmovusqw(a: u64x8, src: u16x8, mask: u8) -> u16x8; + #[link_name = "llvm.x86.avx512.mask.pmovus.qw.256"] + fn vpmovusqw256(a: u64x4, src: u16x8, mask: u8) -> u16x8; + #[link_name = "llvm.x86.avx512.mask.pmovus.qw.128"] + fn vpmovusqw128(a: u64x2, src: u16x8, mask: u8) -> u16x8; + + #[link_name = "llvm.x86.avx512.mask.pmovus.qb.512"] + fn vpmovusqb(a: u64x8, src: u8x16, mask: u8) -> u8x16; + #[link_name = "llvm.x86.avx512.mask.pmovus.qb.256"] + fn vpmovusqb256(a: u64x4, src: u8x16, mask: u8) -> u8x16; + #[link_name = "llvm.x86.avx512.mask.pmovus.qb.128"] + fn vpmovusqb128(a: u64x2, src: u8x16, mask: u8) -> u8x16; + + #[link_name = "llvm.x86.avx512.gather.dpd.512"] + fn vgatherdpd(src: f64x8, slice: *const i8, offsets: i32x8, mask: i8, scale: i32) -> f64x8; + #[link_name = "llvm.x86.avx512.gather.dps.512"] + fn vgatherdps(src: f32x16, slice: *const i8, offsets: i32x16, mask: i16, scale: i32) -> f32x16; + #[link_name = "llvm.x86.avx512.gather.qpd.512"] + fn vgatherqpd(src: f64x8, slice: *const i8, offsets: i64x8, mask: i8, scale: i32) -> f64x8; + #[link_name = "llvm.x86.avx512.gather.qps.512"] + fn vgatherqps(src: f32x8, slice: *const i8, offsets: i64x8, mask: i8, scale: i32) -> f32x8; + #[link_name = "llvm.x86.avx512.gather.dpq.512"] + fn vpgatherdq(src: i64x8, slice: *const i8, offsets: i32x8, mask: i8, scale: i32) -> i64x8; + #[link_name = "llvm.x86.avx512.gather.dpi.512"] + fn vpgatherdd(src: i32x16, slice: *const i8, offsets: i32x16, mask: i16, scale: i32) -> i32x16; + #[link_name = "llvm.x86.avx512.gather.qpq.512"] + fn vpgatherqq(src: i64x8, slice: *const i8, offsets: i64x8, mask: i8, scale: i32) -> i64x8; + #[link_name = "llvm.x86.avx512.gather.qpi.512"] + fn vpgatherqd(src: i32x8, slice: *const i8, offsets: i64x8, mask: i8, scale: i32) -> i32x8; + + #[link_name = "llvm.x86.avx512.scatter.dpd.512"] + fn vscatterdpd(slice: *mut i8, mask: i8, offsets: i32x8, src: f64x8, scale: i32); + #[link_name = "llvm.x86.avx512.scatter.dps.512"] + fn vscatterdps(slice: *mut i8, mask: i16, offsets: i32x16, src: f32x16, scale: i32); + #[link_name = "llvm.x86.avx512.scatter.qpd.512"] + fn vscatterqpd(slice: *mut i8, mask: i8, offsets: i64x8, src: f64x8, scale: i32); + #[link_name = "llvm.x86.avx512.scatter.qps.512"] + fn vscatterqps(slice: *mut i8, mask: i8, offsets: i64x8, src: f32x8, scale: i32); + #[link_name = "llvm.x86.avx512.scatter.dpq.512"] + fn vpscatterdq(slice: *mut i8, mask: i8, offsets: i32x8, src: i64x8, scale: i32); + + #[link_name = "llvm.x86.avx512.scatter.dpi.512"] + fn vpscatterdd(slice: *mut i8, mask: i16, offsets: i32x16, src: i32x16, scale: i32); + #[link_name = "llvm.x86.avx512.scatter.qpq.512"] + fn vpscatterqq(slice: *mut i8, mask: i8, offsets: i64x8, src: i64x8, scale: i32); + #[link_name = "llvm.x86.avx512.scatter.qpi.512"] + fn vpscatterqd(slice: *mut i8, mask: i8, offsets: i64x8, src: i32x8, scale: i32); + + #[link_name = "llvm.x86.avx512.mask.cmp.ss"] + fn vcmpss(a: __m128, b: __m128, op: i32, m: i8, sae: i32) -> i8; + #[link_name = "llvm.x86.avx512.mask.cmp.sd"] + fn vcmpsd(a: __m128d, b: __m128d, op: i32, m: i8, sae: i32) -> i8; + + #[link_name = "llvm.x86.avx512.mask.cmp.ps.512"] + fn vcmpps(a: f32x16, b: f32x16, op: i32, m: i16, sae: i32) -> i16; + #[link_name = "llvm.x86.avx512.mask.cmp.ps.256"] + fn vcmpps256(a: f32x8, b: f32x8, op: i32, m: i8) -> i8; + #[link_name = "llvm.x86.avx512.mask.cmp.ps.128"] + fn vcmpps128(a: f32x4, b: f32x4, op: i32, m: i8) -> i8; + + #[link_name = "llvm.x86.avx512.mask.cmp.pd.512"] + fn vcmppd(a: f64x8, b: f64x8, op: i32, m: i8, sae: i32) -> i8; + #[link_name = "llvm.x86.avx512.mask.cmp.pd.256"] + fn vcmppd256(a: f64x4, b: f64x4, op: i32, m: i8) -> i8; + #[link_name = "llvm.x86.avx512.mask.cmp.pd.128"] + fn vcmppd128(a: f64x2, b: f64x2, op: i32, m: i8) -> i8; + + #[link_name = "llvm.x86.avx512.mask.ucmp.q.512"] + fn vpcmpuq(a: i64x8, b: i64x8, op: i32, m: i8) -> i8; + #[link_name = "llvm.x86.avx512.mask.ucmp.q.256"] + fn vpcmpuq256(a: i64x4, b: i64x4, op: i32, m: i8) -> i8; + #[link_name = "llvm.x86.avx512.mask.ucmp.q.128"] + fn vpcmpuq128(a: i64x2, b: i64x2, op: i32, m: i8) -> i8; + + #[link_name = "llvm.x86.avx512.mask.cmp.q.512"] + fn vpcmpq(a: i64x8, b: i64x8, op: i32, m: i8) -> i8; + #[link_name = "llvm.x86.avx512.mask.cmp.q.256"] + fn vpcmpq256(a: i64x4, b: i64x4, op: i32, m: i8) -> i8; + #[link_name = "llvm.x86.avx512.mask.cmp.q.128"] + fn vpcmpq128(a: i64x2, b: i64x2, op: i32, m: i8) -> i8; + + #[link_name = "llvm.x86.avx512.mask.ucmp.d.512"] + fn vpcmpud(a: i32x16, b: i32x16, op: i32, m: i16) -> i16; + #[link_name = "llvm.x86.avx512.mask.ucmp.d.256"] + fn vpcmpud256(a: i32x8, b: i32x8, op: i32, m: i8) -> i8; + #[link_name = "llvm.x86.avx512.mask.ucmp.d.128"] + fn vpcmpud128(a: i32x4, b: i32x4, op: i32, m: i8) -> i8; + + #[link_name = "llvm.x86.avx512.mask.cmp.d.512"] + fn vpcmpd(a: i32x16, b: i32x16, op: i32, m: i16) -> i16; + #[link_name = "llvm.x86.avx512.mask.cmp.d.256"] + fn vpcmpd256(a: i32x8, b: i32x8, op: i32, m: i8) -> i8; + #[link_name = "llvm.x86.avx512.mask.cmp.d.128"] + fn vpcmpd128(a: i32x4, b: i32x4, op: i32, m: i8) -> i8; + + #[link_name = "llvm.x86.avx512.mask.prol.d.512"] + fn vprold(a: i32x16, i8: i32) -> i32x16; + #[link_name = "llvm.x86.avx512.mask.prol.d.256"] + fn vprold256(a: i32x8, i8: i32) -> i32x8; + #[link_name = "llvm.x86.avx512.mask.prol.d.128"] + fn vprold128(a: i32x4, i8: i32) -> i32x4; + + #[link_name = "llvm.x86.avx512.mask.pror.d.512"] + fn vprord(a: i32x16, i8: i32) -> i32x16; + #[link_name = "llvm.x86.avx512.mask.pror.d.256"] + fn vprord256(a: i32x8, i8: i32) -> i32x8; + #[link_name = "llvm.x86.avx512.mask.pror.d.128"] + fn vprord128(a: i32x4, i8: i32) -> i32x4; + + #[link_name = "llvm.x86.avx512.mask.prol.q.512"] + fn vprolq(a: i64x8, i8: i32) -> i64x8; + #[link_name = "llvm.x86.avx512.mask.prol.q.256"] + fn vprolq256(a: i64x4, i8: i32) -> i64x4; + #[link_name = "llvm.x86.avx512.mask.prol.q.128"] + fn vprolq128(a: i64x2, i8: i32) -> i64x2; + + #[link_name = "llvm.x86.avx512.mask.pror.q.512"] + fn vprorq(a: i64x8, i8: i32) -> i64x8; + #[link_name = "llvm.x86.avx512.mask.pror.q.256"] + fn vprorq256(a: i64x4, i8: i32) -> i64x4; + #[link_name = "llvm.x86.avx512.mask.pror.q.128"] + fn vprorq128(a: i64x2, i8: i32) -> i64x2; + + #[link_name = "llvm.x86.avx512.mask.prolv.d.512"] + fn vprolvd(a: i32x16, b: i32x16) -> i32x16; + #[link_name = "llvm.x86.avx512.mask.prolv.d.256"] + fn vprolvd256(a: i32x8, b: i32x8) -> i32x8; + #[link_name = "llvm.x86.avx512.mask.prolv.d.128"] + fn vprolvd128(a: i32x4, b: i32x4) -> i32x4; + + #[link_name = "llvm.x86.avx512.mask.prorv.d.512"] + fn vprorvd(a: i32x16, b: i32x16) -> i32x16; + #[link_name = "llvm.x86.avx512.mask.prorv.d.256"] + fn vprorvd256(a: i32x8, b: i32x8) -> i32x8; + #[link_name = "llvm.x86.avx512.mask.prorv.d.128"] + fn vprorvd128(a: i32x4, b: i32x4) -> i32x4; + + #[link_name = "llvm.x86.avx512.mask.prolv.q.512"] + fn vprolvq(a: i64x8, b: i64x8) -> i64x8; + #[link_name = "llvm.x86.avx512.mask.prolv.q.256"] + fn vprolvq256(a: i64x4, b: i64x4) -> i64x4; + #[link_name = "llvm.x86.avx512.mask.prolv.q.128"] + fn vprolvq128(a: i64x2, b: i64x2) -> i64x2; + + #[link_name = "llvm.x86.avx512.mask.prorv.q.512"] + fn vprorvq(a: i64x8, b: i64x8) -> i64x8; + #[link_name = "llvm.x86.avx512.mask.prorv.q.256"] + fn vprorvq256(a: i64x4, b: i64x4) -> i64x4; + #[link_name = "llvm.x86.avx512.mask.prorv.q.128"] + fn vprorvq128(a: i64x2, b: i64x2) -> i64x2; + + #[link_name = "llvm.x86.avx512.psllv.d.512"] + fn vpsllvd(a: i32x16, b: i32x16) -> i32x16; + #[link_name = "llvm.x86.avx512.psrlv.d.512"] + fn vpsrlvd(a: i32x16, b: i32x16) -> i32x16; + #[link_name = "llvm.x86.avx512.psllv.q.512"] + fn vpsllvq(a: i64x8, b: i64x8) -> i64x8; + #[link_name = "llvm.x86.avx512.psrlv.q.512"] + fn vpsrlvq(a: i64x8, b: i64x8) -> i64x8; + + #[link_name = "llvm.x86.avx512.pslli.d.512"] + fn vpsllid(a: i32x16, imm8: u32) -> i32x16; + + #[link_name = "llvm.x86.avx2.pslli.d"] + fn psllid256(a: i32x8, imm8: i32) -> i32x8; + #[link_name = "llvm.x86.sse2.pslli.d"] + fn psllid128(a: i32x4, imm8: i32) -> i32x4; + + #[link_name = "llvm.x86.avx512.psrli.d.512"] + fn vpsrlid(a: i32x16, imm8: u32) -> i32x16; + + #[link_name = "llvm.x86.avx2.psrli.d"] + fn psrlid256(a: i32x8, imm8: i32) -> i32x8; + #[link_name = "llvm.x86.sse2.psrli.d"] + fn psrlid128(a: i32x4, imm8: i32) -> i32x4; + + #[link_name = "llvm.x86.avx512.pslli.q.512"] + fn vpslliq(a: i64x8, imm8: u32) -> i64x8; + + #[link_name = "llvm.x86.avx2.pslli.q"] + fn pslliq256(a: i64x4, imm8: i32) -> i64x4; + #[link_name = "llvm.x86.sse2.pslli.q"] + fn pslliq128(a: i64x2, imm8: i32) -> i64x2; + + #[link_name = "llvm.x86.avx512.psrli.q.512"] + fn vpsrliq(a: i64x8, imm8: u32) -> i64x8; + + #[link_name = "llvm.x86.avx2.psrli.q"] + fn psrliq256(a: i64x4, imm8: i32) -> i64x4; + #[link_name = "llvm.x86.sse2.psrli.q"] + fn psrliq128(a: i64x2, imm8: i32) -> i64x2; + + #[link_name = "llvm.x86.avx512.psll.d.512"] + fn vpslld(a: i32x16, count: i32x4) -> i32x16; + #[link_name = "llvm.x86.avx512.psrl.d.512"] + fn vpsrld(a: i32x16, count: i32x4) -> i32x16; + #[link_name = "llvm.x86.avx512.psll.q.512"] + fn vpsllq(a: i64x8, count: i64x2) -> i64x8; + #[link_name = "llvm.x86.avx512.psrl.q.512"] + fn vpsrlq(a: i64x8, count: i64x2) -> i64x8; + + #[link_name = "llvm.x86.avx512.psra.d.512"] + fn vpsrad(a: i32x16, count: i32x4) -> i32x16; + + #[link_name = "llvm.x86.avx512.psra.q.512"] + fn vpsraq(a: i64x8, count: i64x2) -> i64x8; + #[link_name = "llvm.x86.avx512.psra.q.256"] + fn vpsraq256(a: i64x4, count: i64x2) -> i64x4; + #[link_name = "llvm.x86.avx512.psra.q.128"] + fn vpsraq128(a: i64x2, count: i64x2) -> i64x2; + + #[link_name = "llvm.x86.avx512.psrai.d.512"] + fn vpsraid512(a: i32x16, imm8: u32) -> i32x16; + #[link_name = "llvm.x86.avx2.psrai.d"] + fn psraid256(a: i32x8, imm8: i32) -> i32x8; + #[link_name = "llvm.x86.sse2.psrai.d"] + fn psraid128(a: i32x4, imm8: i32) -> i32x4; + + #[link_name = "llvm.x86.avx512.psrai.q.512"] + fn vpsraiq(a: i64x8, imm8: u32) -> i64x8; + #[link_name = "llvm.x86.avx512.psrai.q.256"] + fn vpsraiq256(a: i64x4, imm8: u32) -> i64x4; + #[link_name = "llvm.x86.avx512.psrai.q.128"] + fn vpsraiq128(a: i64x2, imm8: u32) -> i64x2; + + #[link_name = "llvm.x86.avx512.psrav.d.512"] + fn vpsravd(a: i32x16, count: i32x16) -> i32x16; + + #[link_name = "llvm.x86.avx512.psrav.q.512"] + fn vpsravq(a: i64x8, count: i64x8) -> i64x8; + #[link_name = "llvm.x86.avx512.psrav.q.256"] + fn vpsravq256(a: i64x4, count: i64x4) -> i64x4; + #[link_name = "llvm.x86.avx512.psrav.q.128"] + fn vpsravq128(a: i64x2, count: i64x2) -> i64x2; + + #[link_name = "llvm.x86.avx512.vpermilvar.ps.512"] + fn vpermilps(a: f32x16, b: i32x16) -> f32x16; + #[link_name = "llvm.x86.avx512.vpermilvar.pd.512"] + fn vpermilpd(a: f64x8, b: i64x8) -> f64x8; + + #[link_name = "llvm.x86.avx512.permvar.si.512"] + fn vpermd(a: i32x16, idx: i32x16) -> i32x16; + + #[link_name = "llvm.x86.avx512.permvar.di.512"] + fn vpermq(a: i64x8, idx: i64x8) -> i64x8; + #[link_name = "llvm.x86.avx512.permvar.di.256"] + fn vpermq256(a: i64x4, idx: i64x4) -> i64x4; + + #[link_name = "llvm.x86.avx512.permvar.sf.512"] + fn vpermps(a: f32x16, idx: i32x16) -> f32x16; + + #[link_name = "llvm.x86.avx512.permvar.df.512"] + fn vpermpd(a: f64x8, idx: i64x8) -> f64x8; + #[link_name = "llvm.x86.avx512.permvar.df.256"] + fn vpermpd256(a: f64x4, idx: i64x4) -> f64x4; + + #[link_name = "llvm.x86.avx512.vpermi2var.d.512"] + fn vpermi2d(a: i32x16, idx: i32x16, b: i32x16) -> i32x16; + #[link_name = "llvm.x86.avx512.vpermi2var.d.256"] + fn vpermi2d256(a: i32x8, idx: i32x8, b: i32x8) -> i32x8; + #[link_name = "llvm.x86.avx512.vpermi2var.d.128"] + fn vpermi2d128(a: i32x4, idx: i32x4, b: i32x4) -> i32x4; + + #[link_name = "llvm.x86.avx512.vpermi2var.q.512"] + fn vpermi2q(a: i64x8, idx: i64x8, b: i64x8) -> i64x8; + #[link_name = "llvm.x86.avx512.vpermi2var.q.256"] + fn vpermi2q256(a: i64x4, idx: i64x4, b: i64x4) -> i64x4; + #[link_name = "llvm.x86.avx512.vpermi2var.q.128"] + fn vpermi2q128(a: i64x2, idx: i64x2, b: i64x2) -> i64x2; + + #[link_name = "llvm.x86.avx512.vpermi2var.ps.512"] + fn vpermi2ps(a: f32x16, idx: i32x16, b: f32x16) -> f32x16; + #[link_name = "llvm.x86.avx512.vpermi2var.ps.256"] + fn vpermi2ps256(a: f32x8, idx: i32x8, b: f32x8) -> f32x8; + #[link_name = "llvm.x86.avx512.vpermi2var.ps.128"] + fn vpermi2ps128(a: f32x4, idx: i32x4, b: f32x4) -> f32x4; + + #[link_name = "llvm.x86.avx512.vpermi2var.pd.512"] + fn vpermi2pd(a: f64x8, idx: i64x8, b: f64x8) -> f64x8; + #[link_name = "llvm.x86.avx512.vpermi2var.pd.256"] + fn vpermi2pd256(a: f64x4, idx: i64x4, b: f64x4) -> f64x4; + #[link_name = "llvm.x86.avx512.vpermi2var.pd.128"] + fn vpermi2pd128(a: f64x2, idx: i64x2, b: f64x2) -> f64x2; + + #[link_name = "llvm.x86.avx512.mask.compress.d.512"] + fn vpcompressd(a: i32x16, src: i32x16, mask: u16) -> i32x16; + #[link_name = "llvm.x86.avx512.mask.compress.d.256"] + fn vpcompressd256(a: i32x8, src: i32x8, mask: u8) -> i32x8; + #[link_name = "llvm.x86.avx512.mask.compress.d.128"] + fn vpcompressd128(a: i32x4, src: i32x4, mask: u8) -> i32x4; + + #[link_name = "llvm.x86.avx512.mask.compress.q.512"] + fn vpcompressq(a: i64x8, src: i64x8, mask: u8) -> i64x8; + #[link_name = "llvm.x86.avx512.mask.compress.q.256"] + fn vpcompressq256(a: i64x4, src: i64x4, mask: u8) -> i64x4; + #[link_name = "llvm.x86.avx512.mask.compress.q.128"] + fn vpcompressq128(a: i64x2, src: i64x2, mask: u8) -> i64x2; + + #[link_name = "llvm.x86.avx512.mask.compress.ps.512"] + fn vcompressps(a: f32x16, src: f32x16, mask: u16) -> f32x16; + #[link_name = "llvm.x86.avx512.mask.compress.ps.256"] + fn vcompressps256(a: f32x8, src: f32x8, mask: u8) -> f32x8; + #[link_name = "llvm.x86.avx512.mask.compress.ps.128"] + fn vcompressps128(a: f32x4, src: f32x4, mask: u8) -> f32x4; + + #[link_name = "llvm.x86.avx512.mask.compress.pd.512"] + fn vcompresspd(a: f64x8, src: f64x8, mask: u8) -> f64x8; + #[link_name = "llvm.x86.avx512.mask.compress.pd.256"] + fn vcompresspd256(a: f64x4, src: f64x4, mask: u8) -> f64x4; + #[link_name = "llvm.x86.avx512.mask.compress.pd.128"] + fn vcompresspd128(a: f64x2, src: f64x2, mask: u8) -> f64x2; + + #[link_name = "llvm.x86.avx512.mask.compress.store.d.512"] + fn vcompressstored(mem: *mut i8, data: i32x16, mask: u16); + #[link_name = "llvm.x86.avx512.mask.compress.store.d.256"] + fn vcompressstored256(mem: *mut i8, data: i32x8, mask: u8); + #[link_name = "llvm.x86.avx512.mask.compress.store.d.128"] + fn vcompressstored128(mem: *mut i8, data: i32x4, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.compress.store.q.512"] + fn vcompressstoreq(mem: *mut i8, data: i64x8, mask: u8); + #[link_name = "llvm.x86.avx512.mask.compress.store.q.256"] + fn vcompressstoreq256(mem: *mut i8, data: i64x4, mask: u8); + #[link_name = "llvm.x86.avx512.mask.compress.store.q.128"] + fn vcompressstoreq128(mem: *mut i8, data: i64x2, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.compress.store.ps.512"] + fn vcompressstoreps(mem: *mut i8, data: f32x16, mask: u16); + #[link_name = "llvm.x86.avx512.mask.compress.store.ps.256"] + fn vcompressstoreps256(mem: *mut i8, data: f32x8, mask: u8); + #[link_name = "llvm.x86.avx512.mask.compress.store.ps.128"] + fn vcompressstoreps128(mem: *mut i8, data: f32x4, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.compress.store.pd.512"] + fn vcompressstorepd(mem: *mut i8, data: f64x8, mask: u8); + #[link_name = "llvm.x86.avx512.mask.compress.store.pd.256"] + fn vcompressstorepd256(mem: *mut i8, data: f64x4, mask: u8); + #[link_name = "llvm.x86.avx512.mask.compress.store.pd.128"] + fn vcompressstorepd128(mem: *mut i8, data: f64x2, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.expand.d.512"] + fn vpexpandd(a: i32x16, src: i32x16, mask: u16) -> i32x16; + #[link_name = "llvm.x86.avx512.mask.expand.d.256"] + fn vpexpandd256(a: i32x8, src: i32x8, mask: u8) -> i32x8; + #[link_name = "llvm.x86.avx512.mask.expand.d.128"] + fn vpexpandd128(a: i32x4, src: i32x4, mask: u8) -> i32x4; + + #[link_name = "llvm.x86.avx512.mask.expand.q.512"] + fn vpexpandq(a: i64x8, src: i64x8, mask: u8) -> i64x8; + #[link_name = "llvm.x86.avx512.mask.expand.q.256"] + fn vpexpandq256(a: i64x4, src: i64x4, mask: u8) -> i64x4; + #[link_name = "llvm.x86.avx512.mask.expand.q.128"] + fn vpexpandq128(a: i64x2, src: i64x2, mask: u8) -> i64x2; + + #[link_name = "llvm.x86.avx512.mask.expand.ps.512"] + fn vexpandps(a: f32x16, src: f32x16, mask: u16) -> f32x16; + #[link_name = "llvm.x86.avx512.mask.expand.ps.256"] + fn vexpandps256(a: f32x8, src: f32x8, mask: u8) -> f32x8; + #[link_name = "llvm.x86.avx512.mask.expand.ps.128"] + fn vexpandps128(a: f32x4, src: f32x4, mask: u8) -> f32x4; + + #[link_name = "llvm.x86.avx512.mask.expand.pd.512"] + fn vexpandpd(a: f64x8, src: f64x8, mask: u8) -> f64x8; + #[link_name = "llvm.x86.avx512.mask.expand.pd.256"] + fn vexpandpd256(a: f64x4, src: f64x4, mask: u8) -> f64x4; + #[link_name = "llvm.x86.avx512.mask.expand.pd.128"] + fn vexpandpd128(a: f64x2, src: f64x2, mask: u8) -> f64x2; + + #[link_name = "llvm.x86.avx512.mask.add.ss.round"] + fn vaddss(a: f32x4, b: f32x4, src: f32x4, mask: u8, rounding: i32) -> f32x4; + #[link_name = "llvm.x86.avx512.mask.add.sd.round"] + fn vaddsd(a: f64x2, b: f64x2, src: f64x2, mask: u8, rounding: i32) -> f64x2; + #[link_name = "llvm.x86.avx512.mask.sub.ss.round"] + fn vsubss(a: f32x4, b: f32x4, src: f32x4, mask: u8, rounding: i32) -> f32x4; + #[link_name = "llvm.x86.avx512.mask.sub.sd.round"] + fn vsubsd(a: f64x2, b: f64x2, src: f64x2, mask: u8, rounding: i32) -> f64x2; + #[link_name = "llvm.x86.avx512.mask.mul.ss.round"] + fn vmulss(a: f32x4, b: f32x4, src: f32x4, mask: u8, rounding: i32) -> f32x4; + #[link_name = "llvm.x86.avx512.mask.mul.sd.round"] + fn vmulsd(a: f64x2, b: f64x2, src: f64x2, mask: u8, rounding: i32) -> f64x2; + #[link_name = "llvm.x86.avx512.mask.div.ss.round"] + fn vdivss(a: f32x4, b: f32x4, src: f32x4, mask: u8, rounding: i32) -> f32x4; + #[link_name = "llvm.x86.avx512.mask.div.sd.round"] + fn vdivsd(a: f64x2, b: f64x2, src: f64x2, mask: u8, rounding: i32) -> f64x2; + #[link_name = "llvm.x86.avx512.mask.max.ss.round"] + fn vmaxss(a: f32x4, b: f32x4, src: f32x4, mask: u8, sae: i32) -> f32x4; + #[link_name = "llvm.x86.avx512.mask.max.sd.round"] + fn vmaxsd(a: f64x2, b: f64x2, src: f64x2, mask: u8, sae: i32) -> f64x2; + #[link_name = "llvm.x86.avx512.mask.min.ss.round"] + fn vminss(a: f32x4, b: f32x4, src: f32x4, mask: u8, sae: i32) -> f32x4; + #[link_name = "llvm.x86.avx512.mask.min.sd.round"] + fn vminsd(a: f64x2, b: f64x2, src: f64x2, mask: u8, sae: i32) -> f64x2; + #[link_name = "llvm.x86.avx512.mask.sqrt.ss"] + fn vsqrtss(a: f32x4, b: f32x4, src: f32x4, mask: u8, rounding: i32) -> f32x4; + #[link_name = "llvm.x86.avx512.mask.sqrt.sd"] + fn vsqrtsd(a: f64x2, b: f64x2, src: f64x2, mask: u8, rounding: i32) -> f64x2; + #[link_name = "llvm.x86.avx512.mask.getexp.ss"] + fn vgetexpss(a: f32x4, b: f32x4, src: f32x4, mask: u8, sae: i32) -> f32x4; + #[link_name = "llvm.x86.avx512.mask.getexp.sd"] + fn vgetexpsd(a: f64x2, b: f64x2, src: f64x2, mask: u8, sae: i32) -> f64x2; + #[link_name = "llvm.x86.avx512.mask.getmant.ss"] + fn vgetmantss(a: f32x4, b: f32x4, mantissas: i32, src: f32x4, m: u8, sae: i32) -> f32x4; + #[link_name = "llvm.x86.avx512.mask.getmant.sd"] + fn vgetmantsd(a: f64x2, b: f64x2, mantissas: i32, src: f64x2, m: u8, sae: i32) -> f64x2; + + #[link_name = "llvm.x86.avx512.rsqrt14.ss"] + fn vrsqrt14ss(a: f32x4, b: f32x4, src: f32x4, mask: u8) -> f32x4; + #[link_name = "llvm.x86.avx512.rsqrt14.sd"] + fn vrsqrt14sd(a: f64x2, b: f64x2, src: f64x2, mask: u8) -> f64x2; + #[link_name = "llvm.x86.avx512.rcp14.ss"] + fn vrcp14ss(a: f32x4, b: f32x4, src: f32x4, mask: u8) -> f32x4; + #[link_name = "llvm.x86.avx512.rcp14.sd"] + fn vrcp14sd(a: f64x2, b: f64x2, src: f64x2, mask: u8) -> f64x2; + + #[link_name = "llvm.x86.avx512.mask.rndscale.ss"] + fn vrndscaless(a: f32x4, b: f32x4, src: f32x4, mask: u8, imm8: i32, sae: i32) -> f32x4; + #[link_name = "llvm.x86.avx512.mask.rndscale.sd"] + fn vrndscalesd(a: f64x2, b: f64x2, src: f64x2, mask: u8, imm8: i32, sae: i32) -> f64x2; + #[link_name = "llvm.x86.avx512.mask.scalef.ss"] + fn vscalefss(a: f32x4, b: f32x4, src: f32x4, mask: u8, rounding: i32) -> f32x4; + #[link_name = "llvm.x86.avx512.mask.scalef.sd"] + fn vscalefsd(a: f64x2, b: f64x2, src: f64x2, mask: u8, rounding: i32) -> f64x2; + + #[link_name = "llvm.x86.avx512.vfmadd.f32"] + fn vfmadd132ss(a: f32, b: f32, c: f32, rounding: i32) -> f32; + #[link_name = "llvm.x86.avx512.vfmadd.f64"] + fn vfmadd132sd(a: f64, b: f64, c: f64, rounding: i32) -> f64; + + #[link_name = "llvm.x86.avx512.mask.fixupimm.ss"] + fn vfixupimmss(a: f32x4, b: f32x4, c: i32x4, imm8: i32, mask: u8, sae: i32) -> f32x4; + #[link_name = "llvm.x86.avx512.mask.fixupimm.sd"] + fn vfixupimmsd(a: f64x2, b: f64x2, c: i64x2, imm8: i32, mask: u8, sae: i32) -> f64x2; + #[link_name = "llvm.x86.avx512.maskz.fixupimm.ss"] + fn vfixupimmssz(a: f32x4, b: f32x4, c: i32x4, imm8: i32, mask: u8, sae: i32) -> f32x4; + #[link_name = "llvm.x86.avx512.maskz.fixupimm.sd"] + fn vfixupimmsdz(a: f64x2, b: f64x2, c: i64x2, imm8: i32, mask: u8, sae: i32) -> f64x2; + + #[link_name = "llvm.x86.avx512.mask.cvtss2sd.round"] + fn vcvtss2sd(a: f64x2, a: f32x4, src: f64x2, mask: u8, sae: i32) -> f64x2; + #[link_name = "llvm.x86.avx512.mask.cvtsd2ss.round"] + fn vcvtsd2ss(a: f32x4, b: f64x2, src: f32x4, mask: u8, rounding: i32) -> f32x4; + + #[link_name = "llvm.x86.avx512.vcvtss2si32"] + fn vcvtss2si(a: f32x4, rounding: i32) -> i32; + #[link_name = "llvm.x86.avx512.vcvtss2usi32"] + fn vcvtss2usi(a: f32x4, rounding: i32) -> u32; + + #[link_name = "llvm.x86.avx512.vcvtsd2si32"] + fn vcvtsd2si(a: f64x2, rounding: i32) -> i32; + #[link_name = "llvm.x86.avx512.vcvtsd2usi32"] + fn vcvtsd2usi(a: f64x2, rounding: i32) -> u32; + + #[link_name = "llvm.x86.avx512.cvtsi2ss32"] + fn vcvtsi2ss(a: f32x4, b: i32, rounding: i32) -> f32x4; + #[link_name = "llvm.x86.avx512.cvtsi2sd64"] + fn vcvtsi2sd(a: f64x2, b: i64, rounding: i32) -> f64x2; + + #[link_name = "llvm.x86.avx512.cvtusi2ss"] + fn vcvtusi2ss(a: f32x4, b: u32, rounding: i32) -> f32x4; + #[link_name = "llvm.x86.avx512.cvtusi642sd"] + fn vcvtusi2sd(a: f64x2, b: u64, rounding: i32) -> f64x2; + + #[link_name = "llvm.x86.avx512.vcomi.ss"] + fn vcomiss(a: f32x4, b: f32x4, imm8: i32, sae: i32) -> i32; + #[link_name = "llvm.x86.avx512.vcomi.sd"] + fn vcomisd(a: f64x2, b: f64x2, imm8: i32, sae: i32) -> i32; +} + +#[cfg(test)] +mod tests { + + use stdarch_test::simd_test; + + use crate::core_arch::x86::*; + use crate::hint::black_box; + use crate::mem::{self}; + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_abs_epi32() { + #[rustfmt::skip] + let a = _mm512_setr_epi32( + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + ); + let r = _mm512_abs_epi32(a); + #[rustfmt::skip] + let e = _mm512_setr_epi32( + 0, 1, 1, i32::MAX, + i32::MAX.wrapping_add(1), 100, 100, 32, + 0, 1, 1, i32::MAX, + i32::MAX.wrapping_add(1), 100, 100, 32, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_abs_epi32() { + #[rustfmt::skip] + let a = _mm512_setr_epi32( + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + ); + let r = _mm512_mask_abs_epi32(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_abs_epi32(a, 0b00000000_11111111, a); + #[rustfmt::skip] + let e = _mm512_setr_epi32( + 0, 1, 1, i32::MAX, + i32::MAX.wrapping_add(1), 100, 100, 32, + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_abs_epi32() { + #[rustfmt::skip] + let a = _mm512_setr_epi32( + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + ); + let r = _mm512_maskz_abs_epi32(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_abs_epi32(0b00000000_11111111, a); + #[rustfmt::skip] + let e = _mm512_setr_epi32( + 0, 1, 1, i32::MAX, + i32::MAX.wrapping_add(1), 100, 100, 32, + 0, 0, 0, 0, + 0, 0, 0, 0, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_abs_epi32() { + #[rustfmt::skip] + let a = _mm256_setr_epi32( + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + ); + let r = _mm256_mask_abs_epi32(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_abs_epi32(a, 0b00001111, a); + #[rustfmt::skip] + let e = _mm256_setr_epi32( + 0, 1, 1, i32::MAX, + i32::MAX.wrapping_add(1), 100, -100, -32, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_abs_epi32() { + #[rustfmt::skip] + let a = _mm256_setr_epi32( + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + ); + let r = _mm256_maskz_abs_epi32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_abs_epi32(0b00001111, a); + #[rustfmt::skip] + let e = _mm256_setr_epi32( + 0, 1, 1, i32::MAX, + 0, 0, 0, 0, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_abs_epi32() { + let a = _mm_setr_epi32(i32::MIN, 100, -100, -32); + let r = _mm_mask_abs_epi32(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_abs_epi32(a, 0b00001111, a); + let e = _mm_setr_epi32(i32::MAX.wrapping_add(1), 100, 100, 32); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_abs_epi32() { + let a = _mm_setr_epi32(i32::MIN, 100, -100, -32); + let r = _mm_maskz_abs_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_abs_epi32(0b00001111, a); + let e = _mm_setr_epi32(i32::MAX.wrapping_add(1), 100, 100, 32); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_abs_ps() { + #[rustfmt::skip] + let a = _mm512_setr_ps( + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + ); + let r = _mm512_abs_ps(a); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 0., 1., 1., f32::MAX, + f32::MAX, 100., 100., 32., + 0., 1., 1., f32::MAX, + f32::MAX, 100., 100., 32., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_abs_ps() { + #[rustfmt::skip] + let a = _mm512_setr_ps( + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + ); + let r = _mm512_mask_abs_ps(a, 0, a); + assert_eq_m512(r, a); + let r = _mm512_mask_abs_ps(a, 0b00000000_11111111, a); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 0., 1., 1., f32::MAX, + f32::MAX, 100., 100., 32., + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_mov_epi32() { + let src = _mm512_set1_epi32(1); + let a = _mm512_set1_epi32(2); + let r = _mm512_mask_mov_epi32(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_mov_epi32(src, 0b11111111_11111111, a); + assert_eq_m512i(r, a); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_mov_epi32() { + let a = _mm512_set1_epi32(2); + let r = _mm512_maskz_mov_epi32(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_mov_epi32(0b11111111_11111111, a); + assert_eq_m512i(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_mov_epi32() { + let src = _mm256_set1_epi32(1); + let a = _mm256_set1_epi32(2); + let r = _mm256_mask_mov_epi32(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_mov_epi32(src, 0b11111111, a); + assert_eq_m256i(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_mov_epi32() { + let a = _mm256_set1_epi32(2); + let r = _mm256_maskz_mov_epi32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_mov_epi32(0b11111111, a); + assert_eq_m256i(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_mov_epi32() { + let src = _mm_set1_epi32(1); + let a = _mm_set1_epi32(2); + let r = _mm_mask_mov_epi32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_mov_epi32(src, 0b00001111, a); + assert_eq_m128i(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_mov_epi32() { + let a = _mm_set1_epi32(2); + let r = _mm_maskz_mov_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_mov_epi32(0b00001111, a); + assert_eq_m128i(r, a); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_mov_ps() { + let src = _mm512_set1_ps(1.); + let a = _mm512_set1_ps(2.); + let r = _mm512_mask_mov_ps(src, 0, a); + assert_eq_m512(r, src); + let r = _mm512_mask_mov_ps(src, 0b11111111_11111111, a); + assert_eq_m512(r, a); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_mov_ps() { + let a = _mm512_set1_ps(2.); + let r = _mm512_maskz_mov_ps(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_mov_ps(0b11111111_11111111, a); + assert_eq_m512(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_mov_ps() { + let src = _mm256_set1_ps(1.); + let a = _mm256_set1_ps(2.); + let r = _mm256_mask_mov_ps(src, 0, a); + assert_eq_m256(r, src); + let r = _mm256_mask_mov_ps(src, 0b11111111, a); + assert_eq_m256(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_mov_ps() { + let a = _mm256_set1_ps(2.); + let r = _mm256_maskz_mov_ps(0, a); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_mov_ps(0b11111111, a); + assert_eq_m256(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_mov_ps() { + let src = _mm_set1_ps(1.); + let a = _mm_set1_ps(2.); + let r = _mm_mask_mov_ps(src, 0, a); + assert_eq_m128(r, src); + let r = _mm_mask_mov_ps(src, 0b00001111, a); + assert_eq_m128(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_mov_ps() { + let a = _mm_set1_ps(2.); + let r = _mm_maskz_mov_ps(0, a); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_mov_ps(0b00001111, a); + assert_eq_m128(r, a); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_add_epi32() { + #[rustfmt::skip] + let a = _mm512_setr_epi32( + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + ); + let b = _mm512_set1_epi32(1); + let r = _mm512_add_epi32(a, b); + #[rustfmt::skip] + let e = _mm512_setr_epi32( + 1, 2, 0, i32::MIN, + i32::MIN + 1, 101, -99, -31, + 1, 2, 0, i32::MIN, + i32::MIN + 1, 101, -99, -31, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_add_epi32() { + #[rustfmt::skip] + let a = _mm512_setr_epi32( + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + ); + let b = _mm512_set1_epi32(1); + let r = _mm512_mask_add_epi32(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_add_epi32(a, 0b00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_setr_epi32( + 1, 2, 0, i32::MIN, + i32::MIN + 1, 101, -99, -31, + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_add_epi32() { + #[rustfmt::skip] + let a = _mm512_setr_epi32( + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + ); + let b = _mm512_set1_epi32(1); + let r = _mm512_maskz_add_epi32(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_add_epi32(0b00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_setr_epi32( + 1, 2, 0, i32::MIN, + i32::MIN + 1, 101, -99, -31, + 0, 0, 0, 0, + 0, 0, 0, 0, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_add_epi32() { + let a = _mm256_set_epi32(0, 1, -1, i32::MAX, i32::MIN, 100, -100, -32); + let b = _mm256_set1_epi32(1); + let r = _mm256_mask_add_epi32(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_add_epi32(a, 0b11111111, a, b); + let e = _mm256_set_epi32(1, 2, 0, i32::MIN, i32::MIN + 1, 101, -99, -31); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_add_epi32() { + let a = _mm256_setr_epi32(0, 1, -1, i32::MAX, i32::MIN, 100, -100, -32); + let b = _mm256_set1_epi32(1); + let r = _mm256_maskz_add_epi32(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_add_epi32(0b11111111, a, b); + let e = _mm256_setr_epi32(1, 2, 0, i32::MIN, i32::MIN + 1, 101, -99, -31); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_add_epi32() { + let a = _mm_set_epi32(1, -1, i32::MAX, i32::MIN); + let b = _mm_set1_epi32(1); + let r = _mm_mask_add_epi32(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_add_epi32(a, 0b00001111, a, b); + let e = _mm_set_epi32(2, 0, i32::MIN, i32::MIN + 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_add_epi32() { + let a = _mm_setr_epi32(1, -1, i32::MAX, i32::MIN); + let b = _mm_set1_epi32(1); + let r = _mm_maskz_add_epi32(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_add_epi32(0b00001111, a, b); + let e = _mm_setr_epi32(2, 0, i32::MIN, i32::MIN + 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_add_ps() { + #[rustfmt::skip] + let a = _mm512_setr_ps( + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + ); + let b = _mm512_set1_ps(1.); + let r = _mm512_add_ps(a, b); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 1., 2., 0., f32::MAX, + f32::MIN + 1., 101., -99., -31., + 1., 2., 0., f32::MAX, + f32::MIN + 1., 101., -99., -31., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_add_ps() { + #[rustfmt::skip] + let a = _mm512_setr_ps( + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + ); + let b = _mm512_set1_ps(1.); + let r = _mm512_mask_add_ps(a, 0, a, b); + assert_eq_m512(r, a); + let r = _mm512_mask_add_ps(a, 0b00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 1., 2., 0., f32::MAX, + f32::MIN + 1., 101., -99., -31., + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_add_ps() { + #[rustfmt::skip] + let a = _mm512_setr_ps( + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + ); + let b = _mm512_set1_ps(1.); + let r = _mm512_maskz_add_ps(0, a, b); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_add_ps(0b00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 1., 2., 0., f32::MAX, + f32::MIN + 1., 101., -99., -31., + 0., 0., 0., 0., + 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_add_ps() { + let a = _mm256_set_ps(0., 1., -1., f32::MAX, f32::MIN, 100., -100., -32.); + let b = _mm256_set1_ps(1.); + let r = _mm256_mask_add_ps(a, 0, a, b); + assert_eq_m256(r, a); + let r = _mm256_mask_add_ps(a, 0b11111111, a, b); + let e = _mm256_set_ps(1., 2., 0., f32::MAX, f32::MIN + 1., 101., -99., -31.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_add_ps() { + let a = _mm256_set_ps(0., 1., -1., f32::MAX, f32::MIN, 100., -100., -32.); + let b = _mm256_set1_ps(1.); + let r = _mm256_maskz_add_ps(0, a, b); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_add_ps(0b11111111, a, b); + let e = _mm256_set_ps(1., 2., 0., f32::MAX, f32::MIN + 1., 101., -99., -31.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_add_ps() { + let a = _mm_set_ps(1., -1., f32::MAX, f32::MIN); + let b = _mm_set1_ps(1.); + let r = _mm_mask_add_ps(a, 0, a, b); + assert_eq_m128(r, a); + let r = _mm_mask_add_ps(a, 0b00001111, a, b); + let e = _mm_set_ps(2., 0., f32::MAX, f32::MIN + 1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_add_ps() { + let a = _mm_set_ps(1., -1., f32::MAX, f32::MIN); + let b = _mm_set1_ps(1.); + let r = _mm_maskz_add_ps(0, a, b); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_add_ps(0b00001111, a, b); + let e = _mm_set_ps(2., 0., f32::MAX, f32::MIN + 1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_sub_epi32() { + #[rustfmt::skip] + let a = _mm512_setr_epi32( + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + ); + let b = _mm512_set1_epi32(1); + let r = _mm512_sub_epi32(a, b); + #[rustfmt::skip] + let e = _mm512_setr_epi32( + -1, 0, -2, i32::MAX - 1, + i32::MAX, 99, -101, -33, + -1, 0, -2, i32::MAX - 1, + i32::MAX, 99, -101, -33, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_sub_epi32() { + #[rustfmt::skip] + let a = _mm512_setr_epi32( + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + ); + let b = _mm512_set1_epi32(1); + let r = _mm512_mask_sub_epi32(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_sub_epi32(a, 0b00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_setr_epi32( + -1, 0, -2, i32::MAX - 1, + i32::MAX, 99, -101, -33, + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_sub_epi32() { + #[rustfmt::skip] + let a = _mm512_setr_epi32( + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + ); + let b = _mm512_set1_epi32(1); + let r = _mm512_maskz_sub_epi32(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_sub_epi32(0b00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_setr_epi32( + -1, 0, -2, i32::MAX - 1, + i32::MAX, 99, -101, -33, + 0, 0, 0, 0, + 0, 0, 0, 0, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_sub_epi32() { + let a = _mm256_set_epi32(0, 1, -1, i32::MAX, i32::MIN, 100, -100, -32); + let b = _mm256_set1_epi32(1); + let r = _mm256_mask_sub_epi32(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_sub_epi32(a, 0b11111111, a, b); + let e = _mm256_set_epi32(-1, 0, -2, i32::MAX - 1, i32::MAX, 99, -101, -33); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_sub_epi32() { + let a = _mm256_set_epi32(0, 1, -1, i32::MAX, i32::MIN, 100, -100, -32); + let b = _mm256_set1_epi32(1); + let r = _mm256_maskz_sub_epi32(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_sub_epi32(0b11111111, a, b); + let e = _mm256_set_epi32(-1, 0, -2, i32::MAX - 1, i32::MAX, 99, -101, -33); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_sub_epi32() { + let a = _mm_set_epi32(1, -1, i32::MAX, i32::MIN); + let b = _mm_set1_epi32(1); + let r = _mm_mask_sub_epi32(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_sub_epi32(a, 0b00001111, a, b); + let e = _mm_set_epi32(0, -2, i32::MAX - 1, i32::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_sub_epi32() { + let a = _mm_set_epi32(1, -1, i32::MAX, i32::MIN); + let b = _mm_set1_epi32(1); + let r = _mm_maskz_sub_epi32(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_sub_epi32(0b00001111, a, b); + let e = _mm_set_epi32(0, -2, i32::MAX - 1, i32::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_sub_ps() { + #[rustfmt::skip] + let a = _mm512_setr_ps( + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + ); + let b = _mm512_set1_ps(1.); + let r = _mm512_sub_ps(a, b); + #[rustfmt::skip] + let e = _mm512_setr_ps( + -1., 0., -2., f32::MAX - 1., + f32::MIN, 99., -101., -33., + -1., 0., -2., f32::MAX - 1., + f32::MIN, 99., -101., -33., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_sub_ps() { + #[rustfmt::skip] + let a = _mm512_setr_ps( + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + ); + let b = _mm512_set1_ps(1.); + let r = _mm512_mask_sub_ps(a, 0, a, b); + assert_eq_m512(r, a); + let r = _mm512_mask_sub_ps(a, 0b00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_setr_ps( + -1., 0., -2., f32::MAX - 1., + f32::MIN, 99., -101., -33., + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_sub_ps() { + #[rustfmt::skip] + let a = _mm512_setr_ps( + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + ); + let b = _mm512_set1_ps(1.); + let r = _mm512_maskz_sub_ps(0, a, b); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_sub_ps(0b00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_setr_ps( + -1., 0., -2., f32::MAX - 1., + f32::MIN, 99., -101., -33., + 0., 0., 0., 0., + 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_sub_ps() { + let a = _mm256_set_ps(0., 1., -1., f32::MAX, f32::MIN, 100., -100., -32.); + let b = _mm256_set1_ps(1.); + let r = _mm256_mask_sub_ps(a, 0, a, b); + assert_eq_m256(r, a); + let r = _mm256_mask_sub_ps(a, 0b11111111, a, b); + let e = _mm256_set_ps(-1., 0., -2., f32::MAX - 1., f32::MIN, 99., -101., -33.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_sub_ps() { + let a = _mm256_set_ps(0., 1., -1., f32::MAX, f32::MIN, 100., -100., -32.); + let b = _mm256_set1_ps(1.); + let r = _mm256_maskz_sub_ps(0, a, b); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_sub_ps(0b11111111, a, b); + let e = _mm256_set_ps(-1., 0., -2., f32::MAX - 1., f32::MIN, 99., -101., -33.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_sub_ps() { + let a = _mm_set_ps(1., -1., f32::MAX, f32::MIN); + let b = _mm_set1_ps(1.); + let r = _mm_mask_sub_ps(a, 0, a, b); + assert_eq_m128(r, a); + let r = _mm_mask_sub_ps(a, 0b00001111, a, b); + let e = _mm_set_ps(0., -2., f32::MAX - 1., f32::MIN); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_sub_ps() { + let a = _mm_set_ps(1., -1., f32::MAX, f32::MIN); + let b = _mm_set1_ps(1.); + let r = _mm_maskz_sub_ps(0, a, b); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_sub_ps(0b00001111, a, b); + let e = _mm_set_ps(0., -2., f32::MAX - 1., f32::MIN); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mullo_epi32() { + #[rustfmt::skip] + let a = _mm512_setr_epi32( + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + ); + let b = _mm512_set1_epi32(2); + let r = _mm512_mullo_epi32(a, b); + let e = _mm512_setr_epi32( + 0, 2, -2, -2, 0, 200, -200, -64, 0, 2, -2, -2, 0, 200, -200, -64, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_mullo_epi32() { + #[rustfmt::skip] + let a = _mm512_setr_epi32( + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + ); + let b = _mm512_set1_epi32(2); + let r = _mm512_mask_mullo_epi32(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_mullo_epi32(a, 0b00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_setr_epi32( + 0, 2, -2, -2, + 0, 200, -200, -64, + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_mullo_epi32() { + #[rustfmt::skip] + let a = _mm512_setr_epi32( + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + ); + let b = _mm512_set1_epi32(2); + let r = _mm512_maskz_mullo_epi32(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_mullo_epi32(0b00000000_11111111, a, b); + let e = _mm512_setr_epi32(0, 2, -2, -2, 0, 200, -200, -64, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_mullo_epi32() { + let a = _mm256_set_epi32(0, 1, -1, i32::MAX, i32::MIN, 100, -100, -32); + let b = _mm256_set1_epi32(2); + let r = _mm256_mask_mullo_epi32(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_mullo_epi32(a, 0b11111111, a, b); + let e = _mm256_set_epi32(0, 2, -2, -2, 0, 200, -200, -64); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_mullo_epi32() { + let a = _mm256_set_epi32(0, 1, -1, i32::MAX, i32::MIN, 100, -100, -32); + let b = _mm256_set1_epi32(2); + let r = _mm256_maskz_mullo_epi32(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_mullo_epi32(0b11111111, a, b); + let e = _mm256_set_epi32(0, 2, -2, -2, 0, 200, -200, -64); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_mullo_epi32() { + let a = _mm_set_epi32(1, -1, i32::MAX, i32::MIN); + let b = _mm_set1_epi32(2); + let r = _mm_mask_mullo_epi32(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_mullo_epi32(a, 0b00001111, a, b); + let e = _mm_set_epi32(2, -2, -2, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_mullo_epi32() { + let a = _mm_set_epi32(1, -1, i32::MAX, i32::MIN); + let b = _mm_set1_epi32(2); + let r = _mm_maskz_mullo_epi32(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_mullo_epi32(0b00001111, a, b); + let e = _mm_set_epi32(2, -2, -2, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mul_ps() { + #[rustfmt::skip] + let a = _mm512_setr_ps( + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + ); + let b = _mm512_set1_ps(2.); + let r = _mm512_mul_ps(a, b); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 0., 2., -2., f32::INFINITY, + f32::NEG_INFINITY, 200., -200., -64., + 0., 2., -2., f32::INFINITY, + f32::NEG_INFINITY, 200., -200., + -64., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_mul_ps() { + #[rustfmt::skip] + let a = _mm512_setr_ps( + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + ); + let b = _mm512_set1_ps(2.); + let r = _mm512_mask_mul_ps(a, 0, a, b); + assert_eq_m512(r, a); + let r = _mm512_mask_mul_ps(a, 0b00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 0., 2., -2., f32::INFINITY, + f32::NEG_INFINITY, 200., -200., -64., + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_mul_ps() { + #[rustfmt::skip] + let a = _mm512_setr_ps( + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + 0., 1., -1., f32::MAX, + f32::MIN, 100., -100., -32., + ); + let b = _mm512_set1_ps(2.); + let r = _mm512_maskz_mul_ps(0, a, b); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_mul_ps(0b00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 0., 2., -2., f32::INFINITY, + f32::NEG_INFINITY, 200., -200., -64., + 0., 0., 0., 0., + 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_mul_ps() { + let a = _mm256_set_ps(0., 1., -1., f32::MAX, f32::MIN, 100., -100., -32.); + let b = _mm256_set1_ps(2.); + let r = _mm256_mask_mul_ps(a, 0, a, b); + assert_eq_m256(r, a); + let r = _mm256_mask_mul_ps(a, 0b11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_ps( + 0., 2., -2., f32::INFINITY, + f32::NEG_INFINITY, 200., -200., -64., + ); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_mul_ps() { + let a = _mm256_set_ps(0., 1., -1., f32::MAX, f32::MIN, 100., -100., -32.); + let b = _mm256_set1_ps(2.); + let r = _mm256_maskz_mul_ps(0, a, b); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_mul_ps(0b11111111, a, b); + #[rustfmt::skip] + let e = _mm256_set_ps( + 0., 2., -2., f32::INFINITY, + f32::NEG_INFINITY, 200., -200., -64., + ); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_mul_ps() { + let a = _mm_set_ps(1., -1., f32::MAX, f32::MIN); + let b = _mm_set1_ps(2.); + let r = _mm_mask_mul_ps(a, 0, a, b); + assert_eq_m128(r, a); + let r = _mm_mask_mul_ps(a, 0b00001111, a, b); + let e = _mm_set_ps(2., -2., f32::INFINITY, f32::NEG_INFINITY); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_mul_ps() { + let a = _mm_set_ps(1., -1., f32::MAX, f32::MIN); + let b = _mm_set1_ps(2.); + let r = _mm_maskz_mul_ps(0, a, b); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_mul_ps(0b00001111, a, b); + let e = _mm_set_ps(2., -2., f32::INFINITY, f32::NEG_INFINITY); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_div_ps() { + let a = _mm512_setr_ps( + 0., 1., -1., -2., 100., 100., -100., -32., 0., 1., -1., 1000., -131., 100., -100., -32., + ); + let b = _mm512_setr_ps( + 2., 2., 2., 2., 2., 0., 2., 2., 2., 2., 2., 2., 0., 2., 2., 2., + ); + let r = _mm512_div_ps(a, b); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 0., 0.5, -0.5, -1., + 50., f32::INFINITY, -50., -16., + 0., 0.5, -0.5, 500., + f32::NEG_INFINITY, 50., -50., -16., + ); + assert_eq_m512(r, e); // 0/0 = NAN + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_div_ps() { + let a = _mm512_setr_ps( + 0., 1., -1., -2., 100., 100., -100., -32., 0., 1., -1., 1000., -131., 100., -100., -32., + ); + let b = _mm512_setr_ps( + 2., 2., 2., 2., 2., 0., 2., 2., 2., 2., 2., 2., 0., 2., 2., 2., + ); + let r = _mm512_mask_div_ps(a, 0, a, b); + assert_eq_m512(r, a); + let r = _mm512_mask_div_ps(a, 0b00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 0., 0.5, -0.5, -1., + 50., f32::INFINITY, -50., -16., + 0., 1., -1., 1000., + -131., 100., -100., -32., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_div_ps() { + let a = _mm512_setr_ps( + 0., 1., -1., -2., 100., 100., -100., -32., 0., 1., -1., 1000., -131., 100., -100., -32., + ); + let b = _mm512_setr_ps( + 2., 2., 2., 2., 2., 0., 2., 2., 2., 2., 2., 2., 0., 2., 2., 2., + ); + let r = _mm512_maskz_div_ps(0, a, b); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_div_ps(0b00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 0., 0.5, -0.5, -1., + 50., f32::INFINITY, -50., -16., + 0., 0., 0., 0., + 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_div_ps() { + let a = _mm256_set_ps(0., 1., -1., -2., 100., 100., -100., -32.); + let b = _mm256_set_ps(2., 2., 2., 2., 2., 0., 2., 2.); + let r = _mm256_mask_div_ps(a, 0, a, b); + assert_eq_m256(r, a); + let r = _mm256_mask_div_ps(a, 0b11111111, a, b); + let e = _mm256_set_ps(0., 0.5, -0.5, -1., 50., f32::INFINITY, -50., -16.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_div_ps() { + let a = _mm256_set_ps(0., 1., -1., -2., 100., 100., -100., -32.); + let b = _mm256_set_ps(2., 2., 2., 2., 2., 0., 2., 2.); + let r = _mm256_maskz_div_ps(0, a, b); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_div_ps(0b11111111, a, b); + let e = _mm256_set_ps(0., 0.5, -0.5, -1., 50., f32::INFINITY, -50., -16.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_div_ps() { + let a = _mm_set_ps(100., 100., -100., -32.); + let b = _mm_set_ps(2., 0., 2., 2.); + let r = _mm_mask_div_ps(a, 0, a, b); + assert_eq_m128(r, a); + let r = _mm_mask_div_ps(a, 0b00001111, a, b); + let e = _mm_set_ps(50., f32::INFINITY, -50., -16.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_div_ps() { + let a = _mm_set_ps(100., 100., -100., -32.); + let b = _mm_set_ps(2., 0., 2., 2.); + let r = _mm_maskz_div_ps(0, a, b); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_div_ps(0b00001111, a, b); + let e = _mm_set_ps(50., f32::INFINITY, -50., -16.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_max_epi32() { + let a = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm512_setr_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_max_epi32(a, b); + let e = _mm512_setr_epi32(15, 14, 13, 12, 11, 10, 9, 8, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_max_epi32() { + let a = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm512_setr_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_mask_max_epi32(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_max_epi32(a, 0b00000000_11111111, a, b); + let e = _mm512_setr_epi32(15, 14, 13, 12, 11, 10, 9, 8, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_max_epi32() { + let a = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm512_setr_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_maskz_max_epi32(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_max_epi32(0b00000000_11111111, a, b); + let e = _mm512_setr_epi32(15, 14, 13, 12, 11, 10, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_max_epi32() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_mask_max_epi32(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_max_epi32(a, 0b11111111, a, b); + let e = _mm256_set_epi32(7, 6, 5, 4, 4, 5, 6, 7); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_max_epi32() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_maskz_max_epi32(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_max_epi32(0b11111111, a, b); + let e = _mm256_set_epi32(7, 6, 5, 4, 4, 5, 6, 7); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_max_epi32() { + let a = _mm_set_epi32(0, 1, 2, 3); + let b = _mm_set_epi32(3, 2, 1, 0); + let r = _mm_mask_max_epi32(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_max_epi32(a, 0b00001111, a, b); + let e = _mm_set_epi32(3, 2, 2, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_max_epi32() { + let a = _mm_set_epi32(0, 1, 2, 3); + let b = _mm_set_epi32(3, 2, 1, 0); + let r = _mm_maskz_max_epi32(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_max_epi32(0b00001111, a, b); + let e = _mm_set_epi32(3, 2, 2, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_max_ps() { + let a = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let b = _mm512_setr_ps( + 15., 14., 13., 12., 11., 10., 9., 8., 7., 6., 5., 4., 3., 2., 1., 0., + ); + let r = _mm512_max_ps(a, b); + let e = _mm512_setr_ps( + 15., 14., 13., 12., 11., 10., 9., 8., 8., 9., 10., 11., 12., 13., 14., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_max_ps() { + let a = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let b = _mm512_setr_ps( + 15., 14., 13., 12., 11., 10., 9., 8., 7., 6., 5., 4., 3., 2., 1., 0., + ); + let r = _mm512_mask_max_ps(a, 0, a, b); + assert_eq_m512(r, a); + let r = _mm512_mask_max_ps(a, 0b00000000_11111111, a, b); + let e = _mm512_setr_ps( + 15., 14., 13., 12., 11., 10., 9., 8., 8., 9., 10., 11., 12., 13., 14., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_max_ps() { + let a = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let b = _mm512_setr_ps( + 15., 14., 13., 12., 11., 10., 9., 8., 7., 6., 5., 4., 3., 2., 1., 0., + ); + let r = _mm512_maskz_max_ps(0, a, b); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_max_ps(0b00000000_11111111, a, b); + let e = _mm512_setr_ps( + 15., 14., 13., 12., 11., 10., 9., 8., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_max_ps() { + let a = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let b = _mm256_set_ps(7., 6., 5., 4., 3., 2., 1., 0.); + let r = _mm256_mask_max_ps(a, 0, a, b); + assert_eq_m256(r, a); + let r = _mm256_mask_max_ps(a, 0b11111111, a, b); + let e = _mm256_set_ps(7., 6., 5., 4., 4., 5., 6., 7.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_max_ps() { + let a = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let b = _mm256_set_ps(7., 6., 5., 4., 3., 2., 1., 0.); + let r = _mm256_maskz_max_ps(0, a, b); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_max_ps(0b11111111, a, b); + let e = _mm256_set_ps(7., 6., 5., 4., 4., 5., 6., 7.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_max_ps() { + let a = _mm_set_ps(0., 1., 2., 3.); + let b = _mm_set_ps(3., 2., 1., 0.); + let r = _mm_mask_max_ps(a, 0, a, b); + assert_eq_m128(r, a); + let r = _mm_mask_max_ps(a, 0b00001111, a, b); + let e = _mm_set_ps(3., 2., 2., 3.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_max_ps() { + let a = _mm_set_ps(0., 1., 2., 3.); + let b = _mm_set_ps(3., 2., 1., 0.); + let r = _mm_maskz_max_ps(0, a, b); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_mask_max_ps(a, 0b00001111, a, b); + let e = _mm_set_ps(3., 2., 2., 3.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_max_epu32() { + let a = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm512_setr_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_max_epu32(a, b); + let e = _mm512_setr_epi32(15, 14, 13, 12, 11, 10, 9, 8, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_max_epu32() { + let a = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm512_setr_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_mask_max_epu32(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_max_epu32(a, 0b00000000_11111111, a, b); + let e = _mm512_setr_epi32(15, 14, 13, 12, 11, 10, 9, 8, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_max_epu32() { + let a = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm512_setr_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_maskz_max_epu32(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_max_epu32(0b00000000_11111111, a, b); + let e = _mm512_setr_epi32(15, 14, 13, 12, 11, 10, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_max_epu32() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_mask_max_epu32(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_max_epu32(a, 0b11111111, a, b); + let e = _mm256_set_epi32(7, 6, 5, 4, 4, 5, 6, 7); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_max_epu32() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_maskz_max_epu32(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_max_epu32(0b11111111, a, b); + let e = _mm256_set_epi32(7, 6, 5, 4, 4, 5, 6, 7); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_max_epu32() { + let a = _mm_set_epi32(0, 1, 2, 3); + let b = _mm_set_epi32(3, 2, 1, 0); + let r = _mm_mask_max_epu32(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_max_epu32(a, 0b00001111, a, b); + let e = _mm_set_epi32(3, 2, 2, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_max_epu32() { + let a = _mm_set_epi32(0, 1, 2, 3); + let b = _mm_set_epi32(3, 2, 1, 0); + let r = _mm_maskz_max_epu32(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_max_epu32(0b00001111, a, b); + let e = _mm_set_epi32(3, 2, 2, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_min_epi32() { + let a = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm512_setr_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_min_epi32(a, b); + let e = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_min_epi32() { + let a = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm512_setr_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_mask_min_epi32(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_min_epi32(a, 0b00000000_11111111, a, b); + let e = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_min_epi32() { + let a = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm512_setr_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_maskz_min_epi32(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_min_epi32(0b00000000_11111111, a, b); + let e = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_min_epi32() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_mask_min_epi32(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_min_epi32(a, 0b11111111, a, b); + let e = _mm256_set_epi32(0, 1, 2, 3, 3, 2, 1, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_min_epi32() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_maskz_min_epi32(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_min_epi32(0b11111111, a, b); + let e = _mm256_set_epi32(0, 1, 2, 3, 3, 2, 1, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_min_epi32() { + let a = _mm_set_epi32(0, 1, 2, 3); + let b = _mm_set_epi32(3, 2, 1, 0); + let r = _mm_mask_min_epi32(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_min_epi32(a, 0b00001111, a, b); + let e = _mm_set_epi32(0, 1, 1, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_min_epi32() { + let a = _mm_set_epi32(0, 1, 2, 3); + let b = _mm_set_epi32(3, 2, 1, 0); + let r = _mm_maskz_min_epi32(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_min_epi32(0b00001111, a, b); + let e = _mm_set_epi32(0, 1, 1, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_min_ps() { + let a = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let b = _mm512_setr_ps( + 15., 14., 13., 12., 11., 10., 9., 8., 7., 6., 5., 4., 3., 2., 1., 0., + ); + let r = _mm512_min_ps(a, b); + let e = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 7., 6., 5., 4., 3., 2., 1., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_min_ps() { + let a = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let b = _mm512_setr_ps( + 15., 14., 13., 12., 11., 10., 9., 8., 7., 6., 5., 4., 3., 2., 1., 0., + ); + let r = _mm512_mask_min_ps(a, 0, a, b); + assert_eq_m512(r, a); + let r = _mm512_mask_min_ps(a, 0b00000000_11111111, a, b); + let e = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_min_ps() { + let a = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let b = _mm512_setr_ps( + 15., 14., 13., 12., 11., 10., 9., 8., 7., 6., 5., 4., 3., 2., 1., 0., + ); + let r = _mm512_maskz_min_ps(0, a, b); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_min_ps(0b00000000_11111111, a, b); + let e = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_min_ps() { + let a = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let b = _mm256_set_ps(7., 6., 5., 4., 3., 2., 1., 0.); + let r = _mm256_mask_min_ps(a, 0, a, b); + assert_eq_m256(r, a); + let r = _mm256_mask_min_ps(a, 0b11111111, a, b); + let e = _mm256_set_ps(0., 1., 2., 3., 3., 2., 1., 0.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_min_ps() { + let a = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let b = _mm256_set_ps(7., 6., 5., 4., 3., 2., 1., 0.); + let r = _mm256_maskz_min_ps(0, a, b); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_min_ps(0b11111111, a, b); + let e = _mm256_set_ps(0., 1., 2., 3., 3., 2., 1., 0.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_min_ps() { + let a = _mm_set_ps(0., 1., 2., 3.); + let b = _mm_set_ps(3., 2., 1., 0.); + let r = _mm_mask_min_ps(a, 0, a, b); + assert_eq_m128(r, a); + let r = _mm_mask_min_ps(a, 0b00001111, a, b); + let e = _mm_set_ps(0., 1., 1., 0.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_min_ps() { + let a = _mm_set_ps(0., 1., 2., 3.); + let b = _mm_set_ps(3., 2., 1., 0.); + let r = _mm_maskz_min_ps(0, a, b); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_min_ps(0b00001111, a, b); + let e = _mm_set_ps(0., 1., 1., 0.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_min_epu32() { + let a = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm512_setr_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_min_epu32(a, b); + let e = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 7, 6, 5, 4, 3, 2, 1, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_min_epu32() { + let a = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm512_setr_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_mask_min_epu32(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_min_epu32(a, 0b00000000_11111111, a, b); + let e = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_min_epu32() { + let a = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = _mm512_setr_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_maskz_min_epu32(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_min_epu32(0b00000000_11111111, a, b); + let e = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_min_epu32() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_mask_min_epu32(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_min_epu32(a, 0b11111111, a, b); + let e = _mm256_set_epi32(0, 1, 2, 3, 3, 2, 1, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_min_epu32() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_maskz_min_epu32(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_min_epu32(0b11111111, a, b); + let e = _mm256_set_epi32(0, 1, 2, 3, 3, 2, 1, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_min_epu32() { + let a = _mm_set_epi32(0, 1, 2, 3); + let b = _mm_set_epi32(3, 2, 1, 0); + let r = _mm_mask_min_epu32(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_min_epu32(a, 0b00001111, a, b); + let e = _mm_set_epi32(0, 1, 1, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_min_epu32() { + let a = _mm_set_epi32(0, 1, 2, 3); + let b = _mm_set_epi32(3, 2, 1, 0); + let r = _mm_maskz_min_epu32(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_min_epu32(0b00001111, a, b); + let e = _mm_set_epi32(0, 1, 1, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_sqrt_ps() { + let a = _mm512_setr_ps( + 0., 1., 4., 9., 16., 25., 36., 49., 64., 81., 100., 121., 144., 169., 196., 225., + ); + let r = _mm512_sqrt_ps(a); + let e = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_sqrt_ps() { + let a = _mm512_setr_ps( + 0., 1., 4., 9., 16., 25., 36., 49., 64., 81., 100., 121., 144., 169., 196., 225., + ); + let r = _mm512_mask_sqrt_ps(a, 0, a); + assert_eq_m512(r, a); + let r = _mm512_mask_sqrt_ps(a, 0b00000000_11111111, a); + let e = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 64., 81., 100., 121., 144., 169., 196., 225., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_sqrt_ps() { + let a = _mm512_setr_ps( + 0., 1., 4., 9., 16., 25., 36., 49., 64., 81., 100., 121., 144., 169., 196., 225., + ); + let r = _mm512_maskz_sqrt_ps(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_sqrt_ps(0b00000000_11111111, a); + let e = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_sqrt_ps() { + let a = _mm256_set_ps(0., 1., 4., 9., 16., 25., 36., 49.); + let r = _mm256_mask_sqrt_ps(a, 0, a); + assert_eq_m256(r, a); + let r = _mm256_mask_sqrt_ps(a, 0b11111111, a); + let e = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_sqrt_ps() { + let a = _mm256_set_ps(0., 1., 4., 9., 16., 25., 36., 49.); + let r = _mm256_maskz_sqrt_ps(0, a); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_sqrt_ps(0b11111111, a); + let e = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_sqrt_ps() { + let a = _mm_set_ps(0., 1., 4., 9.); + let r = _mm_mask_sqrt_ps(a, 0, a); + assert_eq_m128(r, a); + let r = _mm_mask_sqrt_ps(a, 0b00001111, a); + let e = _mm_set_ps(0., 1., 2., 3.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_sqrt_ps() { + let a = _mm_set_ps(0., 1., 4., 9.); + let r = _mm_maskz_sqrt_ps(0, a); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_sqrt_ps(0b00001111, a); + let e = _mm_set_ps(0., 1., 2., 3.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fmadd_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_set1_ps(1.); + let r = _mm512_fmadd_ps(a, b, c); + let e = _mm512_setr_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fmadd_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_set1_ps(1.); + let r = _mm512_mask_fmadd_ps(a, 0, b, c); + assert_eq_m512(r, a); + let r = _mm512_mask_fmadd_ps(a, 0b00000000_11111111, b, c); + let e = _mm512_setr_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 1., 1., 1., 1., 1., 1., 1., 1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fmadd_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_set1_ps(1.); + let r = _mm512_maskz_fmadd_ps(0, a, b, c); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_fmadd_ps(0b00000000_11111111, a, b, c); + let e = _mm512_setr_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fmadd_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_set1_ps(2.); + let r = _mm512_mask3_fmadd_ps(a, b, c, 0); + assert_eq_m512(r, c); + let r = _mm512_mask3_fmadd_ps(a, b, c, 0b00000000_11111111); + let e = _mm512_setr_ps( + 2., 3., 4., 5., 6., 7., 8., 9., 2., 2., 2., 2., 2., 2., 2., 2., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_fmadd_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm256_set1_ps(1.); + let r = _mm256_mask_fmadd_ps(a, 0, b, c); + assert_eq_m256(r, a); + let r = _mm256_mask_fmadd_ps(a, 0b11111111, b, c); + let e = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_fmadd_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm256_set1_ps(1.); + let r = _mm256_maskz_fmadd_ps(0, a, b, c); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_fmadd_ps(0b11111111, a, b, c); + let e = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask3_fmadd_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm256_set1_ps(1.); + let r = _mm256_mask3_fmadd_ps(a, b, c, 0); + assert_eq_m256(r, c); + let r = _mm256_mask3_fmadd_ps(a, b, c, 0b11111111); + let e = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_fmadd_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set_ps(0., 1., 2., 3.); + let c = _mm_set1_ps(1.); + let r = _mm_mask_fmadd_ps(a, 0, b, c); + assert_eq_m128(r, a); + let r = _mm_mask_fmadd_ps(a, 0b00001111, b, c); + let e = _mm_set_ps(1., 2., 3., 4.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_fmadd_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set_ps(0., 1., 2., 3.); + let c = _mm_set1_ps(1.); + let r = _mm_maskz_fmadd_ps(0, a, b, c); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_fmadd_ps(0b00001111, a, b, c); + let e = _mm_set_ps(1., 2., 3., 4.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask3_fmadd_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set_ps(0., 1., 2., 3.); + let c = _mm_set1_ps(1.); + let r = _mm_mask3_fmadd_ps(a, b, c, 0); + assert_eq_m128(r, c); + let r = _mm_mask3_fmadd_ps(a, b, c, 0b00001111); + let e = _mm_set_ps(1., 2., 3., 4.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fmsub_ps() { + let a = _mm512_setr_ps( + 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., + ); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_setr_ps( + 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., + ); + let r = _mm512_fmsub_ps(a, b, c); + let e = _mm512_setr_ps( + -1., 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fmsub_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_set1_ps(1.); + let r = _mm512_mask_fmsub_ps(a, 0, b, c); + assert_eq_m512(r, a); + let r = _mm512_mask_fmsub_ps(a, 0b00000000_11111111, b, c); + let e = _mm512_setr_ps( + -1., 0., 1., 2., 3., 4., 5., 6., 1., 1., 1., 1., 1., 1., 1., 1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fmsub_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_set1_ps(1.); + let r = _mm512_maskz_fmsub_ps(0, a, b, c); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_fmsub_ps(0b00000000_11111111, a, b, c); + let e = _mm512_setr_ps( + -1., 0., 1., 2., 3., 4., 5., 6., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fmsub_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_setr_ps( + 1., 1., 1., 1., 1., 1., 1., 1., 2., 2., 2., 2., 2., 2., 2., 2., + ); + let r = _mm512_mask3_fmsub_ps(a, b, c, 0); + assert_eq_m512(r, c); + let r = _mm512_mask3_fmsub_ps(a, b, c, 0b00000000_11111111); + let e = _mm512_setr_ps( + -1., 0., 1., 2., 3., 4., 5., 6., 2., 2., 2., 2., 2., 2., 2., 2., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_fmsub_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm256_set1_ps(1.); + let r = _mm256_mask_fmsub_ps(a, 0, b, c); + assert_eq_m256(r, a); + let r = _mm256_mask_fmsub_ps(a, 0b11111111, b, c); + let e = _mm256_set_ps(-1., 0., 1., 2., 3., 4., 5., 6.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_fmsub_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm256_set1_ps(1.); + let r = _mm256_maskz_fmsub_ps(0, a, b, c); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_fmsub_ps(0b11111111, a, b, c); + let e = _mm256_set_ps(-1., 0., 1., 2., 3., 4., 5., 6.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask3_fmsub_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm256_set1_ps(1.); + let r = _mm256_mask3_fmsub_ps(a, b, c, 0); + assert_eq_m256(r, c); + let r = _mm256_mask3_fmsub_ps(a, b, c, 0b11111111); + let e = _mm256_set_ps(-1., 0., 1., 2., 3., 4., 5., 6.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_fmsub_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set_ps(0., 1., 2., 3.); + let c = _mm_set1_ps(1.); + let r = _mm_mask_fmsub_ps(a, 0, b, c); + assert_eq_m128(r, a); + let r = _mm_mask_fmsub_ps(a, 0b00001111, b, c); + let e = _mm_set_ps(-1., 0., 1., 2.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_fmsub_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set_ps(0., 1., 2., 3.); + let c = _mm_set1_ps(1.); + let r = _mm_maskz_fmsub_ps(0, a, b, c); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_fmsub_ps(0b00001111, a, b, c); + let e = _mm_set_ps(-1., 0., 1., 2.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask3_fmsub_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set_ps(0., 1., 2., 3.); + let c = _mm_set1_ps(1.); + let r = _mm_mask3_fmsub_ps(a, b, c, 0); + assert_eq_m128(r, c); + let r = _mm_mask3_fmsub_ps(a, b, c, 0b00001111); + let e = _mm_set_ps(-1., 0., 1., 2.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fmaddsub_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_set1_ps(1.); + let r = _mm512_fmaddsub_ps(a, b, c); + let e = _mm512_setr_ps( + -1., 2., 1., 4., 3., 6., 5., 8., 7., 10., 9., 12., 11., 14., 13., 16., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fmaddsub_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_set1_ps(1.); + let r = _mm512_mask_fmaddsub_ps(a, 0, b, c); + assert_eq_m512(r, a); + let r = _mm512_mask_fmaddsub_ps(a, 0b00000000_11111111, b, c); + let e = _mm512_setr_ps( + -1., 2., 1., 4., 3., 6., 5., 8., 1., 1., 1., 1., 1., 1., 1., 1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fmaddsub_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_set1_ps(1.); + let r = _mm512_maskz_fmaddsub_ps(0, a, b, c); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_fmaddsub_ps(0b00000000_11111111, a, b, c); + let e = _mm512_setr_ps( + -1., 2., 1., 4., 3., 6., 5., 8., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fmaddsub_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_setr_ps( + 1., 1., 1., 1., 1., 1., 1., 1., 2., 2., 2., 2., 2., 2., 2., 2., + ); + let r = _mm512_mask3_fmaddsub_ps(a, b, c, 0); + assert_eq_m512(r, c); + let r = _mm512_mask3_fmaddsub_ps(a, b, c, 0b00000000_11111111); + let e = _mm512_setr_ps( + -1., 2., 1., 4., 3., 6., 5., 8., 2., 2., 2., 2., 2., 2., 2., 2., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_fmaddsub_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm256_set1_ps(1.); + let r = _mm256_mask_fmaddsub_ps(a, 0, b, c); + assert_eq_m256(r, a); + let r = _mm256_mask_fmaddsub_ps(a, 0b11111111, b, c); + let e = _mm256_set_ps(1., 0., 3., 2., 5., 4., 7., 6.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_fmaddsub_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm256_set1_ps(1.); + let r = _mm256_maskz_fmaddsub_ps(0, a, b, c); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_fmaddsub_ps(0b11111111, a, b, c); + let e = _mm256_set_ps(1., 0., 3., 2., 5., 4., 7., 6.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask3_fmaddsub_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm256_set1_ps(1.); + let r = _mm256_mask3_fmaddsub_ps(a, b, c, 0); + assert_eq_m256(r, c); + let r = _mm256_mask3_fmaddsub_ps(a, b, c, 0b11111111); + let e = _mm256_set_ps(1., 0., 3., 2., 5., 4., 7., 6.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_fmaddsub_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set_ps(0., 1., 2., 3.); + let c = _mm_set1_ps(1.); + let r = _mm_mask_fmaddsub_ps(a, 0, b, c); + assert_eq_m128(r, a); + let r = _mm_mask_fmaddsub_ps(a, 0b00001111, b, c); + let e = _mm_set_ps(1., 0., 3., 2.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_fmaddsub_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set_ps(0., 1., 2., 3.); + let c = _mm_set1_ps(1.); + let r = _mm_maskz_fmaddsub_ps(0, a, b, c); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_fmaddsub_ps(0b00001111, a, b, c); + let e = _mm_set_ps(1., 0., 3., 2.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask3_fmaddsub_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set_ps(0., 1., 2., 3.); + let c = _mm_set1_ps(1.); + let r = _mm_mask3_fmaddsub_ps(a, b, c, 0); + assert_eq_m128(r, c); + let r = _mm_mask3_fmaddsub_ps(a, b, c, 0b00001111); + let e = _mm_set_ps(1., 0., 3., 2.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fmsubadd_ps() { + let a = _mm512_setr_ps( + 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., + ); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_setr_ps( + 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., + ); + let r = _mm512_fmsubadd_ps(a, b, c); + let e = _mm512_setr_ps( + 1., 0., 3., 2., 5., 4., 7., 6., 9., 8., 11., 10., 13., 12., 15., 14., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fmsubadd_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_set1_ps(1.); + let r = _mm512_mask_fmsubadd_ps(a, 0, b, c); + assert_eq_m512(r, a); + let r = _mm512_mask_fmsubadd_ps(a, 0b00000000_11111111, b, c); + let e = _mm512_setr_ps( + 1., 0., 3., 2., 5., 4., 7., 6., 1., 1., 1., 1., 1., 1., 1., 1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fmsubadd_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_set1_ps(1.); + let r = _mm512_maskz_fmsubadd_ps(0, a, b, c); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_fmsubadd_ps(0b00000000_11111111, a, b, c); + let e = _mm512_setr_ps( + 1., 0., 3., 2., 5., 4., 7., 6., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fmsubadd_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_setr_ps( + 1., 1., 1., 1., 1., 1., 1., 1., 2., 2., 2., 2., 2., 2., 2., 2., + ); + let r = _mm512_mask3_fmsubadd_ps(a, b, c, 0); + assert_eq_m512(r, c); + let r = _mm512_mask3_fmsubadd_ps(a, b, c, 0b00000000_11111111); + let e = _mm512_setr_ps( + 1., 0., 3., 2., 5., 4., 7., 6., 2., 2., 2., 2., 2., 2., 2., 2., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_fmsubadd_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm256_set1_ps(1.); + let r = _mm256_mask_fmsubadd_ps(a, 0, b, c); + assert_eq_m256(r, a); + let r = _mm256_mask_fmsubadd_ps(a, 0b11111111, b, c); + let e = _mm256_set_ps(-1., 2., 1., 4., 3., 6., 5., 8.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_fmsubadd_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm256_set1_ps(1.); + let r = _mm256_maskz_fmsubadd_ps(0, a, b, c); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_fmsubadd_ps(0b11111111, a, b, c); + let e = _mm256_set_ps(-1., 2., 1., 4., 3., 6., 5., 8.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask3_fmsubadd_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm256_set1_ps(1.); + let r = _mm256_mask3_fmsubadd_ps(a, b, c, 0); + assert_eq_m256(r, c); + let r = _mm256_mask3_fmsubadd_ps(a, b, c, 0b11111111); + let e = _mm256_set_ps(-1., 2., 1., 4., 3., 6., 5., 8.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_fmsubadd_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set_ps(0., 1., 2., 3.); + let c = _mm_set1_ps(1.); + let r = _mm_mask_fmsubadd_ps(a, 0, b, c); + assert_eq_m128(r, a); + let r = _mm_mask_fmsubadd_ps(a, 0b00001111, b, c); + let e = _mm_set_ps(-1., 2., 1., 4.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_fmsubadd_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set_ps(0., 1., 2., 3.); + let c = _mm_set1_ps(1.); + let r = _mm_maskz_fmsubadd_ps(0, a, b, c); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_fmsubadd_ps(0b00001111, a, b, c); + let e = _mm_set_ps(-1., 2., 1., 4.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask3_fmsubadd_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set_ps(0., 1., 2., 3.); + let c = _mm_set1_ps(1.); + let r = _mm_mask3_fmsubadd_ps(a, b, c, 0); + assert_eq_m128(r, c); + let r = _mm_mask3_fmsubadd_ps(a, b, c, 0b00001111); + let e = _mm_set_ps(-1., 2., 1., 4.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fnmadd_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_set1_ps(1.); + let r = _mm512_fnmadd_ps(a, b, c); + let e = _mm512_setr_ps( + 1., 0., -1., -2., -3., -4., -5., -6., -7., -8., -9., -10., -11., -12., -13., -14., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fnmadd_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_set1_ps(1.); + let r = _mm512_mask_fnmadd_ps(a, 0, b, c); + assert_eq_m512(r, a); + let r = _mm512_mask_fnmadd_ps(a, 0b00000000_11111111, b, c); + let e = _mm512_setr_ps( + 1., 0., -1., -2., -3., -4., -5., -6., 1., 1., 1., 1., 1., 1., 1., 1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fnmadd_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_set1_ps(1.); + let r = _mm512_maskz_fnmadd_ps(0, a, b, c); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_fnmadd_ps(0b00000000_11111111, a, b, c); + let e = _mm512_setr_ps( + 1., 0., -1., -2., -3., -4., -5., -6., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fnmadd_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_setr_ps( + 1., 1., 1., 1., 1., 1., 1., 1., 2., 2., 2., 2., 2., 2., 2., 2., + ); + let r = _mm512_mask3_fnmadd_ps(a, b, c, 0); + assert_eq_m512(r, c); + let r = _mm512_mask3_fnmadd_ps(a, b, c, 0b00000000_11111111); + let e = _mm512_setr_ps( + 1., 0., -1., -2., -3., -4., -5., -6., 2., 2., 2., 2., 2., 2., 2., 2., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_fnmadd_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm256_set1_ps(1.); + let r = _mm256_mask_fnmadd_ps(a, 0, b, c); + assert_eq_m256(r, a); + let r = _mm256_mask_fnmadd_ps(a, 0b11111111, b, c); + let e = _mm256_set_ps(1., 0., -1., -2., -3., -4., -5., -6.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_fnmadd_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm256_set1_ps(1.); + let r = _mm256_maskz_fnmadd_ps(0, a, b, c); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_fnmadd_ps(0b11111111, a, b, c); + let e = _mm256_set_ps(1., 0., -1., -2., -3., -4., -5., -6.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask3_fnmadd_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm256_set1_ps(1.); + let r = _mm256_mask3_fnmadd_ps(a, b, c, 0); + assert_eq_m256(r, c); + let r = _mm256_mask3_fnmadd_ps(a, b, c, 0b11111111); + let e = _mm256_set_ps(1., 0., -1., -2., -3., -4., -5., -6.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_fnmadd_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set_ps(0., 1., 2., 3.); + let c = _mm_set1_ps(1.); + let r = _mm_mask_fnmadd_ps(a, 0, b, c); + assert_eq_m128(r, a); + let r = _mm_mask_fnmadd_ps(a, 0b00001111, b, c); + let e = _mm_set_ps(1., 0., -1., -2.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_fnmadd_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set_ps(0., 1., 2., 3.); + let c = _mm_set1_ps(1.); + let r = _mm_maskz_fnmadd_ps(0, a, b, c); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_fnmadd_ps(0b00001111, a, b, c); + let e = _mm_set_ps(1., 0., -1., -2.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask3_fnmadd_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set_ps(0., 1., 2., 3.); + let c = _mm_set1_ps(1.); + let r = _mm_mask3_fnmadd_ps(a, b, c, 0); + assert_eq_m128(r, c); + let r = _mm_mask3_fnmadd_ps(a, b, c, 0b00001111); + let e = _mm_set_ps(1., 0., -1., -2.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fnmsub_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_set1_ps(1.); + let r = _mm512_fnmsub_ps(a, b, c); + let e = _mm512_setr_ps( + -1., -2., -3., -4., -5., -6., -7., -8., -9., -10., -11., -12., -13., -14., -15., -16., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fnmsub_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_set1_ps(1.); + let r = _mm512_mask_fnmsub_ps(a, 0, b, c); + assert_eq_m512(r, a); + let r = _mm512_mask_fnmsub_ps(a, 0b00000000_11111111, b, c); + let e = _mm512_setr_ps( + -1., -2., -3., -4., -5., -6., -7., -8., 1., 1., 1., 1., 1., 1., 1., 1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fnmsub_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_set1_ps(1.); + let r = _mm512_maskz_fnmsub_ps(0, a, b, c); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_fnmsub_ps(0b00000000_11111111, a, b, c); + let e = _mm512_setr_ps( + -1., -2., -3., -4., -5., -6., -7., -8., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fnmsub_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let c = _mm512_setr_ps( + 1., 1., 1., 1., 1., 1., 1., 1., 2., 2., 2., 2., 2., 2., 2., 2., + ); + let r = _mm512_mask3_fnmsub_ps(a, b, c, 0); + assert_eq_m512(r, c); + let r = _mm512_mask3_fnmsub_ps(a, b, c, 0b00000000_11111111); + let e = _mm512_setr_ps( + -1., -2., -3., -4., -5., -6., -7., -8., 2., 2., 2., 2., 2., 2., 2., 2., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_fnmsub_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm256_set1_ps(1.); + let r = _mm256_mask_fnmsub_ps(a, 0, b, c); + assert_eq_m256(r, a); + let r = _mm256_mask_fnmsub_ps(a, 0b11111111, b, c); + let e = _mm256_set_ps(-1., -2., -3., -4., -5., -6., -7., -8.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_fnmsub_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm256_set1_ps(1.); + let r = _mm256_maskz_fnmsub_ps(0, a, b, c); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_fnmsub_ps(0b11111111, a, b, c); + let e = _mm256_set_ps(-1., -2., -3., -4., -5., -6., -7., -8.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask3_fnmsub_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm256_set1_ps(1.); + let r = _mm256_mask3_fnmsub_ps(a, b, c, 0); + assert_eq_m256(r, c); + let r = _mm256_mask3_fnmsub_ps(a, b, c, 0b11111111); + let e = _mm256_set_ps(-1., -2., -3., -4., -5., -6., -7., -8.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_fnmsub_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set_ps(0., 1., 2., 3.); + let c = _mm_set1_ps(1.); + let r = _mm_mask_fnmsub_ps(a, 0, b, c); + assert_eq_m128(r, a); + let r = _mm_mask_fnmsub_ps(a, 0b00001111, b, c); + let e = _mm_set_ps(-1., -2., -3., -4.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_fnmsub_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set_ps(0., 1., 2., 3.); + let c = _mm_set1_ps(1.); + let r = _mm_maskz_fnmsub_ps(0, a, b, c); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_fnmsub_ps(0b00001111, a, b, c); + let e = _mm_set_ps(-1., -2., -3., -4.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask3_fnmsub_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set_ps(0., 1., 2., 3.); + let c = _mm_set1_ps(1.); + let r = _mm_mask3_fnmsub_ps(a, b, c, 0); + assert_eq_m128(r, c); + let r = _mm_mask3_fnmsub_ps(a, b, c, 0b00001111); + let e = _mm_set_ps(-1., -2., -3., -4.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_rcp14_ps() { + let a = _mm512_set1_ps(3.); + let r = _mm512_rcp14_ps(a); + let e = _mm512_set1_ps(0.33333206); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_rcp14_ps() { + let a = _mm512_set1_ps(3.); + let r = _mm512_mask_rcp14_ps(a, 0, a); + assert_eq_m512(r, a); + let r = _mm512_mask_rcp14_ps(a, 0b11111111_00000000, a); + let e = _mm512_setr_ps( + 3., 3., 3., 3., 3., 3., 3., 3., 0.33333206, 0.33333206, 0.33333206, 0.33333206, + 0.33333206, 0.33333206, 0.33333206, 0.33333206, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_rcp14_ps() { + let a = _mm512_set1_ps(3.); + let r = _mm512_maskz_rcp14_ps(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_rcp14_ps(0b11111111_00000000, a); + let e = _mm512_setr_ps( + 0., 0., 0., 0., 0., 0., 0., 0., 0.33333206, 0.33333206, 0.33333206, 0.33333206, + 0.33333206, 0.33333206, 0.33333206, 0.33333206, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_rcp14_ps() { + let a = _mm256_set1_ps(3.); + let r = _mm256_rcp14_ps(a); + let e = _mm256_set1_ps(0.33333206); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_rcp14_ps() { + let a = _mm256_set1_ps(3.); + let r = _mm256_mask_rcp14_ps(a, 0, a); + assert_eq_m256(r, a); + let r = _mm256_mask_rcp14_ps(a, 0b11111111, a); + let e = _mm256_set1_ps(0.33333206); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_rcp14_ps() { + let a = _mm256_set1_ps(3.); + let r = _mm256_maskz_rcp14_ps(0, a); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_rcp14_ps(0b11111111, a); + let e = _mm256_set1_ps(0.33333206); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_rcp14_ps() { + let a = _mm_set1_ps(3.); + let r = _mm_rcp14_ps(a); + let e = _mm_set1_ps(0.33333206); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_rcp14_ps() { + let a = _mm_set1_ps(3.); + let r = _mm_mask_rcp14_ps(a, 0, a); + assert_eq_m128(r, a); + let r = _mm_mask_rcp14_ps(a, 0b00001111, a); + let e = _mm_set1_ps(0.33333206); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_rcp14_ps() { + let a = _mm_set1_ps(3.); + let r = _mm_maskz_rcp14_ps(0, a); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_rcp14_ps(0b00001111, a); + let e = _mm_set1_ps(0.33333206); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_rsqrt14_ps() { + let a = _mm512_set1_ps(3.); + let r = _mm512_rsqrt14_ps(a); + let e = _mm512_set1_ps(0.5773392); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_rsqrt14_ps() { + let a = _mm512_set1_ps(3.); + let r = _mm512_mask_rsqrt14_ps(a, 0, a); + assert_eq_m512(r, a); + let r = _mm512_mask_rsqrt14_ps(a, 0b11111111_00000000, a); + let e = _mm512_setr_ps( + 3., 3., 3., 3., 3., 3., 3., 3., 0.5773392, 0.5773392, 0.5773392, 0.5773392, 0.5773392, + 0.5773392, 0.5773392, 0.5773392, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_rsqrt14_ps() { + let a = _mm512_set1_ps(3.); + let r = _mm512_maskz_rsqrt14_ps(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_rsqrt14_ps(0b11111111_00000000, a); + let e = _mm512_setr_ps( + 0., 0., 0., 0., 0., 0., 0., 0., 0.5773392, 0.5773392, 0.5773392, 0.5773392, 0.5773392, + 0.5773392, 0.5773392, 0.5773392, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_rsqrt14_ps() { + let a = _mm256_set1_ps(3.); + let r = _mm256_mask_rsqrt14_ps(a, 0, a); + assert_eq_m256(r, a); + let r = _mm256_mask_rsqrt14_ps(a, 0b11111111, a); + let e = _mm256_set1_ps(0.5773392); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_rsqrt14_ps() { + let a = _mm256_set1_ps(3.); + let r = _mm256_maskz_rsqrt14_ps(0, a); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_rsqrt14_ps(0b11111111, a); + let e = _mm256_set1_ps(0.5773392); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_rsqrt14_ps() { + let a = _mm_set1_ps(3.); + let r = _mm_mask_rsqrt14_ps(a, 0, a); + assert_eq_m128(r, a); + let r = _mm_mask_rsqrt14_ps(a, 0b00001111, a); + let e = _mm_set1_ps(0.5773392); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_rsqrt14_ps() { + let a = _mm_set1_ps(3.); + let r = _mm_maskz_rsqrt14_ps(0, a); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_rsqrt14_ps(0b00001111, a); + let e = _mm_set1_ps(0.5773392); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_getexp_ps() { + let a = _mm512_set1_ps(3.); + let r = _mm512_getexp_ps(a); + let e = _mm512_set1_ps(1.); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_getexp_ps() { + let a = _mm512_set1_ps(3.); + let r = _mm512_mask_getexp_ps(a, 0, a); + assert_eq_m512(r, a); + let r = _mm512_mask_getexp_ps(a, 0b11111111_00000000, a); + let e = _mm512_setr_ps( + 3., 3., 3., 3., 3., 3., 3., 3., 1., 1., 1., 1., 1., 1., 1., 1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_getexp_ps() { + let a = _mm512_set1_ps(3.); + let r = _mm512_maskz_getexp_ps(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_getexp_ps(0b11111111_00000000, a); + let e = _mm512_setr_ps( + 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_getexp_ps() { + let a = _mm256_set1_ps(3.); + let r = _mm256_getexp_ps(a); + let e = _mm256_set1_ps(1.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_getexp_ps() { + let a = _mm256_set1_ps(3.); + let r = _mm256_mask_getexp_ps(a, 0, a); + assert_eq_m256(r, a); + let r = _mm256_mask_getexp_ps(a, 0b11111111, a); + let e = _mm256_set1_ps(1.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_getexp_ps() { + let a = _mm256_set1_ps(3.); + let r = _mm256_maskz_getexp_ps(0, a); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_getexp_ps(0b11111111, a); + let e = _mm256_set1_ps(1.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_getexp_ps() { + let a = _mm_set1_ps(3.); + let r = _mm_getexp_ps(a); + let e = _mm_set1_ps(1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_getexp_ps() { + let a = _mm_set1_ps(3.); + let r = _mm_mask_getexp_ps(a, 0, a); + assert_eq_m128(r, a); + let r = _mm_mask_getexp_ps(a, 0b00001111, a); + let e = _mm_set1_ps(1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_getexp_ps() { + let a = _mm_set1_ps(3.); + let r = _mm_maskz_getexp_ps(0, a); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_getexp_ps(0b00001111, a); + let e = _mm_set1_ps(1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_roundscale_ps() { + let a = _mm512_set1_ps(1.1); + let r = _mm512_roundscale_ps::<0b00_00_00_00>(a); + let e = _mm512_set1_ps(1.0); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_roundscale_ps() { + let a = _mm512_set1_ps(1.1); + let r = _mm512_mask_roundscale_ps::<0b00_00_00_00>(a, 0, a); + let e = _mm512_set1_ps(1.1); + assert_eq_m512(r, e); + let r = _mm512_mask_roundscale_ps::<0b00_00_00_00>(a, 0b11111111_11111111, a); + let e = _mm512_set1_ps(1.0); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_roundscale_ps() { + let a = _mm512_set1_ps(1.1); + let r = _mm512_maskz_roundscale_ps::<0b00_00_00_00>(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_roundscale_ps::<0b00_00_00_00>(0b11111111_11111111, a); + let e = _mm512_set1_ps(1.0); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_roundscale_ps() { + let a = _mm256_set1_ps(1.1); + let r = _mm256_roundscale_ps::<0b00_00_00_00>(a); + let e = _mm256_set1_ps(1.0); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_roundscale_ps() { + let a = _mm256_set1_ps(1.1); + let r = _mm256_mask_roundscale_ps::<0b00_00_00_00>(a, 0, a); + let e = _mm256_set1_ps(1.1); + assert_eq_m256(r, e); + let r = _mm256_mask_roundscale_ps::<0b00_00_00_00>(a, 0b11111111, a); + let e = _mm256_set1_ps(1.0); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_roundscale_ps() { + let a = _mm256_set1_ps(1.1); + let r = _mm256_maskz_roundscale_ps::<0b00_00_00_00>(0, a); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_roundscale_ps::<0b00_00_00_00>(0b11111111, a); + let e = _mm256_set1_ps(1.0); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_roundscale_ps() { + let a = _mm_set1_ps(1.1); + let r = _mm_roundscale_ps::<0b00_00_00_00>(a); + let e = _mm_set1_ps(1.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_roundscale_ps() { + let a = _mm_set1_ps(1.1); + let r = _mm_mask_roundscale_ps::<0b00_00_00_00>(a, 0, a); + let e = _mm_set1_ps(1.1); + assert_eq_m128(r, e); + let r = _mm_mask_roundscale_ps::<0b00_00_00_00>(a, 0b00001111, a); + let e = _mm_set1_ps(1.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_roundscale_ps() { + let a = _mm_set1_ps(1.1); + let r = _mm_maskz_roundscale_ps::<0b00_00_00_00>(0, a); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_roundscale_ps::<0b00_00_00_00>(0b00001111, a); + let e = _mm_set1_ps(1.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_scalef_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_set1_ps(3.); + let r = _mm512_scalef_ps(a, b); + let e = _mm512_set1_ps(8.); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_scalef_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_set1_ps(3.); + let r = _mm512_mask_scalef_ps(a, 0, a, b); + assert_eq_m512(r, a); + let r = _mm512_mask_scalef_ps(a, 0b11111111_00000000, a, b); + let e = _mm512_set_ps( + 8., 8., 8., 8., 8., 8., 8., 8., 1., 1., 1., 1., 1., 1., 1., 1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_scalef_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_set1_ps(3.); + let r = _mm512_maskz_scalef_ps(0, a, b); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_scalef_ps(0b11111111_00000000, a, b); + let e = _mm512_set_ps( + 8., 8., 8., 8., 8., 8., 8., 8., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_scalef_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set1_ps(3.); + let r = _mm256_scalef_ps(a, b); + let e = _mm256_set1_ps(8.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_scalef_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set1_ps(3.); + let r = _mm256_mask_scalef_ps(a, 0, a, b); + assert_eq_m256(r, a); + let r = _mm256_mask_scalef_ps(a, 0b11111111, a, b); + let e = _mm256_set1_ps(8.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_scalef_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set1_ps(3.); + let r = _mm256_maskz_scalef_ps(0, a, b); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_scalef_ps(0b11111111, a, b); + let e = _mm256_set1_ps(8.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_scalef_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(3.); + let r = _mm_scalef_ps(a, b); + let e = _mm_set1_ps(8.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_scalef_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(3.); + let r = _mm_mask_scalef_ps(a, 0, a, b); + assert_eq_m128(r, a); + let r = _mm_mask_scalef_ps(a, 0b00001111, a, b); + let e = _mm_set1_ps(8.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_scalef_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(3.); + let r = _mm_maskz_scalef_ps(0, a, b); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_scalef_ps(0b00001111, a, b); + let e = _mm_set1_ps(8.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fixupimm_ps() { + let a = _mm512_set1_ps(f32::NAN); + let b = _mm512_set1_ps(f32::MAX); + let c = _mm512_set1_epi32(i32::MAX); + //let r = _mm512_fixupimm_ps(a, b, c, 5); + let r = _mm512_fixupimm_ps::<5>(a, b, c); + let e = _mm512_set1_ps(0.0); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fixupimm_ps() { + #[rustfmt::skip] + let a = _mm512_set_ps( + f32::NAN, f32::NAN, f32::NAN, f32::NAN, + f32::NAN, f32::NAN, f32::NAN, f32::NAN, + 1., 1., 1., 1., + 1., 1., 1., 1., + ); + let b = _mm512_set1_ps(f32::MAX); + let c = _mm512_set1_epi32(i32::MAX); + let r = _mm512_mask_fixupimm_ps::<5>(a, 0b11111111_00000000, b, c); + let e = _mm512_set_ps( + 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fixupimm_ps() { + #[rustfmt::skip] + let a = _mm512_set_ps( + f32::NAN, f32::NAN, f32::NAN, f32::NAN, + f32::NAN, f32::NAN, f32::NAN, f32::NAN, + 1., 1., 1., 1., + 1., 1., 1., 1., + ); + let b = _mm512_set1_ps(f32::MAX); + let c = _mm512_set1_epi32(i32::MAX); + let r = _mm512_maskz_fixupimm_ps::<5>(0b11111111_00000000, a, b, c); + let e = _mm512_set_ps( + 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_fixupimm_ps() { + let a = _mm256_set1_ps(f32::NAN); + let b = _mm256_set1_ps(f32::MAX); + let c = _mm256_set1_epi32(i32::MAX); + let r = _mm256_fixupimm_ps::<5>(a, b, c); + let e = _mm256_set1_ps(0.0); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_fixupimm_ps() { + let a = _mm256_set1_ps(f32::NAN); + let b = _mm256_set1_ps(f32::MAX); + let c = _mm256_set1_epi32(i32::MAX); + let r = _mm256_mask_fixupimm_ps::<5>(a, 0b11111111, b, c); + let e = _mm256_set1_ps(0.0); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_fixupimm_ps() { + let a = _mm256_set1_ps(f32::NAN); + let b = _mm256_set1_ps(f32::MAX); + let c = _mm256_set1_epi32(i32::MAX); + let r = _mm256_maskz_fixupimm_ps::<5>(0b11111111, a, b, c); + let e = _mm256_set1_ps(0.0); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_fixupimm_ps() { + let a = _mm_set1_ps(f32::NAN); + let b = _mm_set1_ps(f32::MAX); + let c = _mm_set1_epi32(i32::MAX); + let r = _mm_fixupimm_ps::<5>(a, b, c); + let e = _mm_set1_ps(0.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_fixupimm_ps() { + let a = _mm_set1_ps(f32::NAN); + let b = _mm_set1_ps(f32::MAX); + let c = _mm_set1_epi32(i32::MAX); + let r = _mm_mask_fixupimm_ps::<5>(a, 0b00001111, b, c); + let e = _mm_set1_ps(0.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_fixupimm_ps() { + let a = _mm_set1_ps(f32::NAN); + let b = _mm_set1_ps(f32::MAX); + let c = _mm_set1_epi32(i32::MAX); + let r = _mm_maskz_fixupimm_ps::<5>(0b00001111, a, b, c); + let e = _mm_set1_ps(0.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_ternarylogic_epi32() { + let a = _mm512_set1_epi32(1 << 2); + let b = _mm512_set1_epi32(1 << 1); + let c = _mm512_set1_epi32(1 << 0); + let r = _mm512_ternarylogic_epi32::<8>(a, b, c); + let e = _mm512_set1_epi32(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_ternarylogic_epi32() { + let src = _mm512_set1_epi32(1 << 2); + let a = _mm512_set1_epi32(1 << 1); + let b = _mm512_set1_epi32(1 << 0); + let r = _mm512_mask_ternarylogic_epi32::<8>(src, 0, a, b); + assert_eq_m512i(r, src); + let r = _mm512_mask_ternarylogic_epi32::<8>(src, 0b11111111_11111111, a, b); + let e = _mm512_set1_epi32(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_ternarylogic_epi32() { + let a = _mm512_set1_epi32(1 << 2); + let b = _mm512_set1_epi32(1 << 1); + let c = _mm512_set1_epi32(1 << 0); + let r = _mm512_maskz_ternarylogic_epi32::<9>(0, a, b, c); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_ternarylogic_epi32::<8>(0b11111111_11111111, a, b, c); + let e = _mm512_set1_epi32(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_ternarylogic_epi32() { + let a = _mm256_set1_epi32(1 << 2); + let b = _mm256_set1_epi32(1 << 1); + let c = _mm256_set1_epi32(1 << 0); + let r = _mm256_ternarylogic_epi32::<8>(a, b, c); + let e = _mm256_set1_epi32(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_ternarylogic_epi32() { + let src = _mm256_set1_epi32(1 << 2); + let a = _mm256_set1_epi32(1 << 1); + let b = _mm256_set1_epi32(1 << 0); + let r = _mm256_mask_ternarylogic_epi32::<8>(src, 0, a, b); + assert_eq_m256i(r, src); + let r = _mm256_mask_ternarylogic_epi32::<8>(src, 0b11111111, a, b); + let e = _mm256_set1_epi32(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_ternarylogic_epi32() { + let a = _mm256_set1_epi32(1 << 2); + let b = _mm256_set1_epi32(1 << 1); + let c = _mm256_set1_epi32(1 << 0); + let r = _mm256_maskz_ternarylogic_epi32::<9>(0, a, b, c); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_ternarylogic_epi32::<8>(0b11111111, a, b, c); + let e = _mm256_set1_epi32(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_ternarylogic_epi32() { + let a = _mm_set1_epi32(1 << 2); + let b = _mm_set1_epi32(1 << 1); + let c = _mm_set1_epi32(1 << 0); + let r = _mm_ternarylogic_epi32::<8>(a, b, c); + let e = _mm_set1_epi32(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_ternarylogic_epi32() { + let src = _mm_set1_epi32(1 << 2); + let a = _mm_set1_epi32(1 << 1); + let b = _mm_set1_epi32(1 << 0); + let r = _mm_mask_ternarylogic_epi32::<8>(src, 0, a, b); + assert_eq_m128i(r, src); + let r = _mm_mask_ternarylogic_epi32::<8>(src, 0b00001111, a, b); + let e = _mm_set1_epi32(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_ternarylogic_epi32() { + let a = _mm_set1_epi32(1 << 2); + let b = _mm_set1_epi32(1 << 1); + let c = _mm_set1_epi32(1 << 0); + let r = _mm_maskz_ternarylogic_epi32::<9>(0, a, b, c); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_ternarylogic_epi32::<8>(0b00001111, a, b, c); + let e = _mm_set1_epi32(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_getmant_ps() { + let a = _mm512_set1_ps(10.); + let r = _mm512_getmant_ps::<_MM_MANT_NORM_P75_1P5, _MM_MANT_SIGN_NAN>(a); + let e = _mm512_set1_ps(1.25); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_getmant_ps() { + let a = _mm512_set1_ps(10.); + let r = _mm512_mask_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0, a); + assert_eq_m512(r, a); + let r = _mm512_mask_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>( + a, + 0b11111111_00000000, + a, + ); + let e = _mm512_setr_ps( + 10., 10., 10., 10., 10., 10., 10., 10., 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_getmant_ps() { + let a = _mm512_set1_ps(10.); + let r = _mm512_maskz_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = + _mm512_maskz_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0b11111111_00000000, a); + let e = _mm512_setr_ps( + 0., 0., 0., 0., 0., 0., 0., 0., 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_getmant_ps() { + let a = _mm256_set1_ps(10.); + let r = _mm256_getmant_ps::<_MM_MANT_NORM_P75_1P5, _MM_MANT_SIGN_NAN>(a); + let e = _mm256_set1_ps(1.25); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_getmant_ps() { + let a = _mm256_set1_ps(10.); + let r = _mm256_mask_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0, a); + assert_eq_m256(r, a); + let r = _mm256_mask_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0b11111111, a); + let e = _mm256_set1_ps(1.25); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_getmant_ps() { + let a = _mm256_set1_ps(10.); + let r = _mm256_maskz_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0, a); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0b11111111, a); + let e = _mm256_set1_ps(1.25); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_getmant_ps() { + let a = _mm_set1_ps(10.); + let r = _mm_getmant_ps::<_MM_MANT_NORM_P75_1P5, _MM_MANT_SIGN_NAN>(a); + let e = _mm_set1_ps(1.25); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_getmant_ps() { + let a = _mm_set1_ps(10.); + let r = _mm_mask_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0, a); + assert_eq_m128(r, a); + let r = _mm_mask_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0b00001111, a); + let e = _mm_set1_ps(1.25); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_getmant_ps() { + let a = _mm_set1_ps(10.); + let r = _mm_maskz_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0, a); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0b00001111, a); + let e = _mm_set1_ps(1.25); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_add_round_ps() { + let a = _mm512_setr_ps( + 0., 1.5, 2., 3.5, 4., 5.5, 6., 7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 0.00000007, + ); + let b = _mm512_set1_ps(-1.); + let r = _mm512_add_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); + #[rustfmt::skip] + let e = _mm512_setr_ps( + -1., 0.5, 1., 2.5, + 3., 4.5, 5., 6.5, + 7., 8.5, 9., 10.5, + 11., 12.5, 13., -0.99999994, + ); + assert_eq_m512(r, e); + let r = _mm512_add_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm512_setr_ps( + -1., 0.5, 1., 2.5, 3., 4.5, 5., 6.5, 7., 8.5, 9., 10.5, 11., 12.5, 13., -0.9999999, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_add_round_ps() { + let a = _mm512_setr_ps( + 0., 1.5, 2., 3.5, 4., 5.5, 6., 7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 0.00000007, + ); + let b = _mm512_set1_ps(-1.); + let r = _mm512_mask_add_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, 0, a, b); + assert_eq_m512(r, a); + let r = _mm512_mask_add_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, + 0b11111111_00000000, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 0., 1.5, 2., 3.5, + 4., 5.5, 6., 7.5, + 7., 8.5, 9., 10.5, + 11., 12.5, 13., -0.99999994, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_add_round_ps() { + let a = _mm512_setr_ps( + 0., 1.5, 2., 3.5, 4., 5.5, 6., 7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 0.00000007, + ); + let b = _mm512_set1_ps(-1.); + let r = _mm512_maskz_add_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_add_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111_00000000, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 0., 0., 0., 0., + 0., 0., 0., 0., + 7., 8.5, 9., 10.5, + 11., 12.5, 13., -0.99999994, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_sub_round_ps() { + let a = _mm512_setr_ps( + 0., 1.5, 2., 3.5, 4., 5.5, 6., 7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 0.00000007, + ); + let b = _mm512_set1_ps(1.); + let r = _mm512_sub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); + #[rustfmt::skip] + let e = _mm512_setr_ps( + -1., 0.5, 1., 2.5, + 3., 4.5, 5., 6.5, + 7., 8.5, 9., 10.5, + 11., 12.5, 13., -0.99999994, + ); + assert_eq_m512(r, e); + let r = _mm512_sub_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm512_setr_ps( + -1., 0.5, 1., 2.5, 3., 4.5, 5., 6.5, 7., 8.5, 9., 10.5, 11., 12.5, 13., -0.9999999, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_sub_round_ps() { + let a = _mm512_setr_ps( + 0., 1.5, 2., 3.5, 4., 5.5, 6., 7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 0.00000007, + ); + let b = _mm512_set1_ps(1.); + let r = _mm512_mask_sub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); + assert_eq_m512(r, a); + let r = _mm512_mask_sub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, + 0b11111111_00000000, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 0., 1.5, 2., 3.5, + 4., 5.5, 6., 7.5, + 7., 8.5, 9., 10.5, + 11., 12.5, 13., -0.99999994, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_sub_round_ps() { + let a = _mm512_setr_ps( + 0., 1.5, 2., 3.5, 4., 5.5, 6., 7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 0.00000007, + ); + let b = _mm512_set1_ps(1.); + let r = + _mm512_maskz_sub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a, b); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_sub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111_00000000, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 0., 0., 0., 0., + 0., 0., 0., 0., + 7., 8.5, 9., 10.5, + 11., 12.5, 13., -0.99999994, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mul_round_ps() { + #[rustfmt::skip] + let a = _mm512_setr_ps( + 0., 1.5, 2., 3.5, + 4., 5.5, 6., 7.5, + 8., 9.5, 10., 11.5, + 12., 13.5, 14., 0.00000000000000000000007, + ); + let b = _mm512_set1_ps(0.1); + let r = _mm512_mul_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 0., 0.15, 0.2, 0.35, + 0.4, 0.55, 0.6, 0.75, + 0.8, 0.95, 1.0, 1.15, + 1.2, 1.35, 1.4, 0.000000000000000000000007000001, + ); + assert_eq_m512(r, e); + let r = _mm512_mul_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 0., 0.14999999, 0.2, 0.35, + 0.4, 0.54999995, 0.59999996, 0.75, + 0.8, 0.95, 1.0, 1.15, + 1.1999999, 1.3499999, 1.4, 0.000000000000000000000007, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_mul_round_ps() { + #[rustfmt::skip] + let a = _mm512_setr_ps( + 0., 1.5, 2., 3.5, + 4., 5.5, 6., 7.5, + 8., 9.5, 10., 11.5, + 12., 13.5, 14., 0.00000000000000000000007, + ); + let b = _mm512_set1_ps(0.1); + let r = _mm512_mask_mul_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); + assert_eq_m512(r, a); + let r = _mm512_mask_mul_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, + 0b11111111_00000000, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 0., 1.5, 2., 3.5, + 4., 5.5, 6., 7.5, + 0.8, 0.95, 1.0, 1.15, + 1.2, 1.35, 1.4, 0.000000000000000000000007000001, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_mul_round_ps() { + #[rustfmt::skip] + let a = _mm512_setr_ps( + 0., 1.5, 2., 3.5, + 4., 5.5, 6., 7.5, + 8., 9.5, 10., 11.5, + 12., 13.5, 14., 0.00000000000000000000007, + ); + let b = _mm512_set1_ps(0.1); + let r = + _mm512_maskz_mul_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a, b); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_mul_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111_00000000, + a, + b, + ); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 0., 0., 0., 0., + 0., 0., 0., 0., + 0.8, 0.95, 1.0, 1.15, + 1.2, 1.35, 1.4, 0.000000000000000000000007000001, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_div_round_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_set1_ps(3.); + let r = _mm512_div_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm512_set1_ps(0.33333334); + assert_eq_m512(r, e); + let r = _mm512_div_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm512_set1_ps(0.3333333); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_div_round_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_set1_ps(3.); + let r = _mm512_mask_div_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); + assert_eq_m512(r, a); + let r = _mm512_mask_div_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, + 0b11111111_00000000, + a, + b, + ); + let e = _mm512_setr_ps( + 1., 1., 1., 1., 1., 1., 1., 1., 0.33333334, 0.33333334, 0.33333334, 0.33333334, + 0.33333334, 0.33333334, 0.33333334, 0.33333334, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_div_round_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_set1_ps(3.); + let r = + _mm512_maskz_div_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a, b); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_div_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111_00000000, + a, + b, + ); + let e = _mm512_setr_ps( + 0., 0., 0., 0., 0., 0., 0., 0., 0.33333334, 0.33333334, 0.33333334, 0.33333334, + 0.33333334, 0.33333334, 0.33333334, 0.33333334, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_sqrt_round_ps() { + let a = _mm512_set1_ps(3.); + let r = _mm512_sqrt_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a); + let e = _mm512_set1_ps(1.7320508); + assert_eq_m512(r, e); + let r = _mm512_sqrt_round_ps::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a); + let e = _mm512_set1_ps(1.7320509); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_sqrt_round_ps() { + let a = _mm512_set1_ps(3.); + let r = + _mm512_mask_sqrt_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, 0, a); + assert_eq_m512(r, a); + let r = _mm512_mask_sqrt_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, + 0b11111111_00000000, + a, + ); + let e = _mm512_setr_ps( + 3., 3., 3., 3., 3., 3., 3., 3., 1.7320508, 1.7320508, 1.7320508, 1.7320508, 1.7320508, + 1.7320508, 1.7320508, 1.7320508, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_sqrt_round_ps() { + let a = _mm512_set1_ps(3.); + let r = + _mm512_maskz_sqrt_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_sqrt_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111_00000000, + a, + ); + let e = _mm512_setr_ps( + 0., 0., 0., 0., 0., 0., 0., 0., 1.7320508, 1.7320508, 1.7320508, 1.7320508, 1.7320508, + 1.7320508, 1.7320508, 1.7320508, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fmadd_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(-1.); + let r = _mm512_fmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_set1_ps(-0.99999994); + assert_eq_m512(r, e); + let r = _mm512_fmadd_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_set1_ps(-0.9999999); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fmadd_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(-1.); + let r = _mm512_mask_fmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); + assert_eq_m512(r, a); + let r = _mm512_mask_fmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, + 0b00000000_11111111, + b, + c, + ); + #[rustfmt::skip] + let e = _mm512_setr_ps( + -0.99999994, -0.99999994, -0.99999994, -0.99999994, + -0.99999994, -0.99999994, -0.99999994, -0.99999994, + 0.00000007, 0.00000007, 0.00000007, 0.00000007, + 0.00000007, 0.00000007, 0.00000007, 0.00000007, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fmadd_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(-1.); + let r = _mm512_maskz_fmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); + assert_eq_m512(r, _mm512_setzero_ps()); + #[rustfmt::skip] + let r = _mm512_maskz_fmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00000000_11111111, + a, + b, + c, + ); + #[rustfmt::skip] + let e = _mm512_setr_ps( + -0.99999994, -0.99999994, -0.99999994, -0.99999994, + -0.99999994, -0.99999994, -0.99999994, -0.99999994, + 0., 0., 0., 0., + 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fmadd_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(-1.); + let r = _mm512_mask3_fmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); + assert_eq_m512(r, c); + let r = _mm512_mask3_fmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, + b, + c, + 0b00000000_11111111, + ); + #[rustfmt::skip] + let e = _mm512_setr_ps( + -0.99999994, -0.99999994, -0.99999994, -0.99999994, + -0.99999994, -0.99999994, -0.99999994, -0.99999994, + -1., -1., -1., -1., + -1., -1., -1., -1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fmsub_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(1.); + let r = _mm512_fmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_set1_ps(-0.99999994); + assert_eq_m512(r, e); + let r = _mm512_fmsub_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_set1_ps(-0.9999999); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fmsub_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(1.); + let r = _mm512_mask_fmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); + assert_eq_m512(r, a); + let r = _mm512_mask_fmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, + 0b00000000_11111111, + b, + c, + ); + #[rustfmt::skip] + let e = _mm512_setr_ps( + -0.99999994, -0.99999994, -0.99999994, -0.99999994, + -0.99999994, -0.99999994, -0.99999994, -0.99999994, + 0.00000007, 0.00000007, 0.00000007, 0.00000007, + 0.00000007, 0.00000007, 0.00000007, 0.00000007, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fmsub_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(1.); + let r = _mm512_maskz_fmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_fmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00000000_11111111, + a, + b, + c, + ); + #[rustfmt::skip] + let e = _mm512_setr_ps( + -0.99999994, -0.99999994, -0.99999994, -0.99999994, + -0.99999994, -0.99999994, -0.99999994, -0.99999994, + 0., 0., 0., 0., + 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fmsub_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(1.); + let r = _mm512_mask3_fmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); + assert_eq_m512(r, c); + let r = _mm512_mask3_fmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, + b, + c, + 0b00000000_11111111, + ); + #[rustfmt::skip] + let e = _mm512_setr_ps( + -0.99999994, -0.99999994, -0.99999994, -0.99999994, + -0.99999994, -0.99999994, -0.99999994, -0.99999994, + 1., 1., 1., 1., + 1., 1., 1., 1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fmaddsub_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(-1.); + let r = + _mm512_fmaddsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 1.0000001, -0.99999994, 1.0000001, -0.99999994, + 1.0000001, -0.99999994, 1.0000001, -0.99999994, + 1.0000001, -0.99999994, 1.0000001, -0.99999994, + 1.0000001, -0.99999994, 1.0000001, -0.99999994, + ); + assert_eq_m512(r, e); + let r = _mm512_fmaddsub_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_setr_ps( + 1., -0.9999999, 1., -0.9999999, 1., -0.9999999, 1., -0.9999999, 1., -0.9999999, 1., + -0.9999999, 1., -0.9999999, 1., -0.9999999, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fmaddsub_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(-1.); + let r = _mm512_mask_fmaddsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); + assert_eq_m512(r, a); + let r = _mm512_mask_fmaddsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, + 0b00000000_11111111, + b, + c, + ); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 1.0000001, -0.99999994, 1.0000001, -0.99999994, + 1.0000001, -0.99999994, 1.0000001, -0.99999994, + 0.00000007, 0.00000007, 0.00000007, 0.00000007, + 0.00000007, 0.00000007, 0.00000007, 0.00000007, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fmaddsub_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(-1.); + let r = _mm512_maskz_fmaddsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_fmaddsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00000000_11111111, + a, + b, + c, + ); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 1.0000001, -0.99999994, 1.0000001, -0.99999994, + 1.0000001, -0.99999994, 1.0000001, -0.99999994, + 0., 0., 0., 0., + 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fmaddsub_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(-1.); + let r = _mm512_mask3_fmaddsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); + assert_eq_m512(r, c); + let r = _mm512_mask3_fmaddsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, + b, + c, + 0b00000000_11111111, + ); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 1.0000001, -0.99999994, 1.0000001, -0.99999994, + 1.0000001, -0.99999994, 1.0000001, -0.99999994, + -1., -1., -1., -1., + -1., -1., -1., -1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fmsubadd_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(-1.); + let r = + _mm512_fmsubadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); + #[rustfmt::skip] + let e = _mm512_setr_ps( + -0.99999994, 1.0000001, -0.99999994, 1.0000001, + -0.99999994, 1.0000001, -0.99999994, 1.0000001, + -0.99999994, 1.0000001, -0.99999994, 1.0000001, + -0.99999994, 1.0000001, -0.99999994, 1.0000001, + ); + assert_eq_m512(r, e); + let r = _mm512_fmsubadd_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_setr_ps( + -0.9999999, 1., -0.9999999, 1., -0.9999999, 1., -0.9999999, 1., -0.9999999, 1., + -0.9999999, 1., -0.9999999, 1., -0.9999999, 1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fmsubadd_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(-1.); + let r = _mm512_mask_fmsubadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); + assert_eq_m512(r, a); + let r = _mm512_mask_fmsubadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, + 0b00000000_11111111, + b, + c, + ); + #[rustfmt::skip] + let e = _mm512_setr_ps( + -0.99999994, 1.0000001, -0.99999994, 1.0000001, + -0.99999994, 1.0000001, -0.99999994, 1.0000001, + 0.00000007, 0.00000007, 0.00000007, 0.00000007, + 0.00000007, 0.00000007, 0.00000007, 0.00000007, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fmsubadd_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(-1.); + let r = _mm512_maskz_fmsubadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_fmsubadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00000000_11111111, + a, + b, + c, + ); + #[rustfmt::skip] + let e = _mm512_setr_ps( + -0.99999994, 1.0000001, -0.99999994, 1.0000001, + -0.99999994, 1.0000001, -0.99999994, 1.0000001, + 0., 0., 0., 0., + 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fmsubadd_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(-1.); + let r = _mm512_mask3_fmsubadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); + assert_eq_m512(r, c); + let r = _mm512_mask3_fmsubadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, + b, + c, + 0b00000000_11111111, + ); + #[rustfmt::skip] + let e = _mm512_setr_ps( + -0.99999994, 1.0000001, -0.99999994, 1.0000001, + -0.99999994, 1.0000001, -0.99999994, 1.0000001, + -1., -1., -1., -1., + -1., -1., -1., -1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fnmadd_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(1.); + let r = + _mm512_fnmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_set1_ps(0.99999994); + assert_eq_m512(r, e); + let r = _mm512_fnmadd_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_set1_ps(0.9999999); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fnmadd_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(1.); + let r = _mm512_mask_fnmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); + assert_eq_m512(r, a); + let r = _mm512_mask_fnmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, + 0b00000000_11111111, + b, + c, + ); + let e = _mm512_setr_ps( + 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, + 0.99999994, 0.00000007, 0.00000007, 0.00000007, 0.00000007, 0.00000007, 0.00000007, + 0.00000007, 0.00000007, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fnmadd_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(1.); + let r = _mm512_maskz_fnmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_fnmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00000000_11111111, + a, + b, + c, + ); + let e = _mm512_setr_ps( + 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, + 0.99999994, 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fnmadd_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(1.); + let r = _mm512_mask3_fnmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); + assert_eq_m512(r, c); + let r = _mm512_mask3_fnmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, + b, + c, + 0b00000000_11111111, + ); + let e = _mm512_setr_ps( + 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, + 0.99999994, 1., 1., 1., 1., 1., 1., 1., 1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fnmsub_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(-1.); + let r = + _mm512_fnmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_set1_ps(0.99999994); + assert_eq_m512(r, e); + let r = _mm512_fnmsub_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_set1_ps(0.9999999); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fnmsub_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(-1.); + let r = _mm512_mask_fnmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); + assert_eq_m512(r, a); + let r = _mm512_mask_fnmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, + 0b00000000_11111111, + b, + c, + ); + let e = _mm512_setr_ps( + 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, + 0.99999994, 0.00000007, 0.00000007, 0.00000007, 0.00000007, 0.00000007, 0.00000007, + 0.00000007, 0.00000007, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fnmsub_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(-1.); + let r = _mm512_maskz_fnmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_fnmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00000000_11111111, + a, + b, + c, + ); + let e = _mm512_setr_ps( + 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, + 0.99999994, 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fnmsub_round_ps() { + let a = _mm512_set1_ps(0.00000007); + let b = _mm512_set1_ps(1.); + let c = _mm512_set1_ps(-1.); + let r = _mm512_mask3_fnmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); + assert_eq_m512(r, c); + let r = _mm512_mask3_fnmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, + b, + c, + 0b00000000_11111111, + ); + let e = _mm512_setr_ps( + 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, + 0.99999994, -1., -1., -1., -1., -1., -1., -1., -1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_max_round_ps() { + let a = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let b = _mm512_setr_ps( + 15., 14., 13., 12., 11., 10., 9., 8., 7., 6., 5., 4., 3., 2., 1., 0., + ); + let r = _mm512_max_round_ps::<_MM_FROUND_CUR_DIRECTION>(a, b); + let e = _mm512_setr_ps( + 15., 14., 13., 12., 11., 10., 9., 8., 8., 9., 10., 11., 12., 13., 14., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_max_round_ps() { + let a = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let b = _mm512_setr_ps( + 15., 14., 13., 12., 11., 10., 9., 8., 7., 6., 5., 4., 3., 2., 1., 0., + ); + let r = _mm512_mask_max_round_ps::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); + assert_eq_m512(r, a); + let r = _mm512_mask_max_round_ps::<_MM_FROUND_CUR_DIRECTION>(a, 0b00000000_11111111, a, b); + let e = _mm512_setr_ps( + 15., 14., 13., 12., 11., 10., 9., 8., 8., 9., 10., 11., 12., 13., 14., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_max_round_ps() { + let a = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let b = _mm512_setr_ps( + 15., 14., 13., 12., 11., 10., 9., 8., 7., 6., 5., 4., 3., 2., 1., 0., + ); + let r = _mm512_maskz_max_round_ps::<_MM_FROUND_CUR_DIRECTION>(0, a, b); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_max_round_ps::<_MM_FROUND_CUR_DIRECTION>(0b00000000_11111111, a, b); + let e = _mm512_setr_ps( + 15., 14., 13., 12., 11., 10., 9., 8., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_min_round_ps() { + let a = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let b = _mm512_setr_ps( + 15., 14., 13., 12., 11., 10., 9., 8., 7., 6., 5., 4., 3., 2., 1., 0., + ); + let r = _mm512_min_round_ps::<_MM_FROUND_CUR_DIRECTION>(a, b); + let e = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 7., 6., 5., 4., 3., 2., 1., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_min_round_ps() { + let a = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let b = _mm512_setr_ps( + 15., 14., 13., 12., 11., 10., 9., 8., 7., 6., 5., 4., 3., 2., 1., 0., + ); + let r = _mm512_mask_min_round_ps::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); + assert_eq_m512(r, a); + let r = _mm512_mask_min_round_ps::<_MM_FROUND_CUR_DIRECTION>(a, 0b00000000_11111111, a, b); + let e = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_min_round_ps() { + let a = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let b = _mm512_setr_ps( + 15., 14., 13., 12., 11., 10., 9., 8., 7., 6., 5., 4., 3., 2., 1., 0., + ); + let r = _mm512_maskz_min_round_ps::<_MM_FROUND_CUR_DIRECTION>(0, a, b); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_min_round_ps::<_MM_FROUND_CUR_DIRECTION>(0b00000000_11111111, a, b); + let e = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_getexp_round_ps() { + let a = _mm512_set1_ps(3.); + let r = _mm512_getexp_round_ps::<_MM_FROUND_CUR_DIRECTION>(a); + let e = _mm512_set1_ps(1.); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_getexp_round_ps() { + let a = _mm512_set1_ps(3.); + let r = _mm512_mask_getexp_round_ps::<_MM_FROUND_CUR_DIRECTION>(a, 0, a); + assert_eq_m512(r, a); + let r = _mm512_mask_getexp_round_ps::<_MM_FROUND_CUR_DIRECTION>(a, 0b11111111_00000000, a); + let e = _mm512_setr_ps( + 3., 3., 3., 3., 3., 3., 3., 3., 1., 1., 1., 1., 1., 1., 1., 1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_getexp_round_ps() { + let a = _mm512_set1_ps(3.); + let r = _mm512_maskz_getexp_round_ps::<_MM_FROUND_CUR_DIRECTION>(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_getexp_round_ps::<_MM_FROUND_CUR_DIRECTION>(0b11111111_00000000, a); + let e = _mm512_setr_ps( + 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_roundscale_round_ps() { + let a = _mm512_set1_ps(1.1); + let r = _mm512_roundscale_round_ps::<0, _MM_FROUND_CUR_DIRECTION>(a); + let e = _mm512_set1_ps(1.0); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_roundscale_round_ps() { + let a = _mm512_set1_ps(1.1); + let r = _mm512_mask_roundscale_round_ps::<0, _MM_FROUND_CUR_DIRECTION>(a, 0, a); + let e = _mm512_set1_ps(1.1); + assert_eq_m512(r, e); + let r = _mm512_mask_roundscale_round_ps::<0, _MM_FROUND_CUR_DIRECTION>( + a, + 0b11111111_11111111, + a, + ); + let e = _mm512_set1_ps(1.0); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_roundscale_round_ps() { + let a = _mm512_set1_ps(1.1); + let r = _mm512_maskz_roundscale_round_ps::<0, _MM_FROUND_CUR_DIRECTION>(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = + _mm512_maskz_roundscale_round_ps::<0, _MM_FROUND_CUR_DIRECTION>(0b11111111_11111111, a); + let e = _mm512_set1_ps(1.0); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_scalef_round_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_set1_ps(3.); + let r = _mm512_scalef_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm512_set1_ps(8.); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_scalef_round_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_set1_ps(3.); + let r = _mm512_mask_scalef_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); + assert_eq_m512(r, a); + let r = _mm512_mask_scalef_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, + 0b11111111_00000000, + a, + b, + ); + let e = _mm512_set_ps( + 8., 8., 8., 8., 8., 8., 8., 8., 1., 1., 1., 1., 1., 1., 1., 1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_scalef_round_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_set1_ps(3.); + let r = _mm512_maskz_scalef_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, + ); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_scalef_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111_00000000, + a, + b, + ); + let e = _mm512_set_ps( + 8., 8., 8., 8., 8., 8., 8., 8., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fixupimm_round_ps() { + let a = _mm512_set1_ps(f32::NAN); + let b = _mm512_set1_ps(f32::MAX); + let c = _mm512_set1_epi32(i32::MAX); + let r = _mm512_fixupimm_round_ps::<5, _MM_FROUND_CUR_DIRECTION>(a, b, c); + let e = _mm512_set1_ps(0.0); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fixupimm_round_ps() { + #[rustfmt::skip] + let a = _mm512_set_ps( + f32::NAN, f32::NAN, f32::NAN, f32::NAN, + f32::NAN, f32::NAN, f32::NAN, f32::NAN, + 1., 1., 1., 1., + 1., 1., 1., 1., + ); + let b = _mm512_set1_ps(f32::MAX); + let c = _mm512_set1_epi32(i32::MAX); + let r = _mm512_mask_fixupimm_round_ps::<5, _MM_FROUND_CUR_DIRECTION>( + a, + 0b11111111_00000000, + b, + c, + ); + let e = _mm512_set_ps( + 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fixupimm_round_ps() { + #[rustfmt::skip] + let a = _mm512_set_ps( + f32::NAN, f32::NAN, f32::NAN, f32::NAN, + f32::NAN, f32::NAN, f32::NAN, f32::NAN, + 1., 1., 1., 1., + 1., 1., 1., 1., + ); + let b = _mm512_set1_ps(f32::MAX); + let c = _mm512_set1_epi32(i32::MAX); + let r = _mm512_maskz_fixupimm_round_ps::<5, _MM_FROUND_CUR_DIRECTION>( + 0b11111111_00000000, + a, + b, + c, + ); + let e = _mm512_set_ps( + 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_getmant_round_ps() { + let a = _mm512_set1_ps(10.); + let r = _mm512_getmant_round_ps::< + _MM_MANT_NORM_1_2, + _MM_MANT_SIGN_SRC, + _MM_FROUND_CUR_DIRECTION, + >(a); + let e = _mm512_set1_ps(1.25); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_getmant_round_ps() { + let a = _mm512_set1_ps(10.); + let r = _mm512_mask_getmant_round_ps::< + _MM_MANT_NORM_1_2, + _MM_MANT_SIGN_SRC, + _MM_FROUND_CUR_DIRECTION, + >(a, 0, a); + assert_eq_m512(r, a); + let r = _mm512_mask_getmant_round_ps::< + _MM_MANT_NORM_1_2, + _MM_MANT_SIGN_SRC, + _MM_FROUND_CUR_DIRECTION, + >(a, 0b11111111_00000000, a); + let e = _mm512_setr_ps( + 10., 10., 10., 10., 10., 10., 10., 10., 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_getmant_round_ps() { + let a = _mm512_set1_ps(10.); + let r = _mm512_maskz_getmant_round_ps::< + _MM_MANT_NORM_1_2, + _MM_MANT_SIGN_SRC, + _MM_FROUND_CUR_DIRECTION, + >(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_getmant_round_ps::< + _MM_MANT_NORM_1_2, + _MM_MANT_SIGN_SRC, + _MM_FROUND_CUR_DIRECTION, + >(0b11111111_00000000, a); + let e = _mm512_setr_ps( + 0., 0., 0., 0., 0., 0., 0., 0., 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtps_epi32() { + let a = _mm512_setr_ps( + 0., -1.4, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let r = _mm512_cvtps_epi32(a); + let e = _mm512_setr_epi32(0, -1, 2, -4, 4, -6, 6, -8, 8, 10, 10, 12, 12, 14, 14, 16); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtps_epi32() { + let a = _mm512_setr_ps( + 0., -1.4, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let src = _mm512_set1_epi32(0); + let r = _mm512_mask_cvtps_epi32(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_cvtps_epi32(src, 0b00000000_11111111, a); + let e = _mm512_setr_epi32(0, -1, 2, -4, 4, -6, 6, -8, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtps_epi32() { + let a = _mm512_setr_ps( + 0., -1.4, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let r = _mm512_maskz_cvtps_epi32(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_cvtps_epi32(0b00000000_11111111, a); + let e = _mm512_setr_epi32(0, -1, 2, -4, 4, -6, 6, -8, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtps_epi32() { + let a = _mm256_set_ps(8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5); + let src = _mm256_set1_epi32(0); + let r = _mm256_mask_cvtps_epi32(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_cvtps_epi32(src, 0b11111111, a); + let e = _mm256_set_epi32(8, 10, 10, 12, 12, 14, 14, 16); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtps_epi32() { + let a = _mm256_set_ps(8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5); + let r = _mm256_maskz_cvtps_epi32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_cvtps_epi32(0b11111111, a); + let e = _mm256_set_epi32(8, 10, 10, 12, 12, 14, 14, 16); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtps_epi32() { + let a = _mm_set_ps(12., 13.5, 14., 15.5); + let src = _mm_set1_epi32(0); + let r = _mm_mask_cvtps_epi32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtps_epi32(src, 0b00001111, a); + let e = _mm_set_epi32(12, 14, 14, 16); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtps_epi32() { + let a = _mm_set_ps(12., 13.5, 14., 15.5); + let r = _mm_maskz_cvtps_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtps_epi32(0b00001111, a); + let e = _mm_set_epi32(12, 14, 14, 16); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtps_epu32() { + let a = _mm512_setr_ps( + 0., -1.4, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let r = _mm512_cvtps_epu32(a); + let e = _mm512_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1, 8, 10, 10, 12, 12, 14, 14, 16); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtps_epu32() { + let a = _mm512_setr_ps( + 0., -1.4, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let src = _mm512_set1_epi32(0); + let r = _mm512_mask_cvtps_epu32(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_cvtps_epu32(src, 0b00000000_11111111, a); + let e = _mm512_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtps_epu32() { + let a = _mm512_setr_ps( + 0., -1.4, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let r = _mm512_maskz_cvtps_epu32(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_cvtps_epu32(0b00000000_11111111, a); + let e = _mm512_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cvtps_epu32() { + let a = _mm256_set_ps(8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5); + let r = _mm256_cvtps_epu32(a); + let e = _mm256_set_epi32(8, 10, 10, 12, 12, 14, 14, 16); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtps_epu32() { + let a = _mm256_set_ps(8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5); + let src = _mm256_set1_epi32(0); + let r = _mm256_mask_cvtps_epu32(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_cvtps_epu32(src, 0b11111111, a); + let e = _mm256_set_epi32(8, 10, 10, 12, 12, 14, 14, 16); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtps_epu32() { + let a = _mm256_set_ps(8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5); + let r = _mm256_maskz_cvtps_epu32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_cvtps_epu32(0b11111111, a); + let e = _mm256_set_epi32(8, 10, 10, 12, 12, 14, 14, 16); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cvtps_epu32() { + let a = _mm_set_ps(12., 13.5, 14., 15.5); + let r = _mm_cvtps_epu32(a); + let e = _mm_set_epi32(12, 14, 14, 16); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtps_epu32() { + let a = _mm_set_ps(12., 13.5, 14., 15.5); + let src = _mm_set1_epi32(0); + let r = _mm_mask_cvtps_epu32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtps_epu32(src, 0b00001111, a); + let e = _mm_set_epi32(12, 14, 14, 16); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtps_epu32() { + let a = _mm_set_ps(12., 13.5, 14., 15.5); + let r = _mm_maskz_cvtps_epu32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtps_epu32(0b00001111, a); + let e = _mm_set_epi32(12, 14, 14, 16); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtepi8_epi32() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_cvtepi8_epi32(a); + let e = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepi8_epi32() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm512_set1_epi32(-1); + let r = _mm512_mask_cvtepi8_epi32(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_cvtepi8_epi32(src, 0b00000000_11111111, a); + let e = _mm512_set_epi32(-1, -1, -1, -1, -1, -1, -1, -1, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtepi8_epi32() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_cvtepi8_epi32(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_cvtepi8_epi32(0b00000000_11111111, a); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepi8_epi32() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm256_set1_epi32(-1); + let r = _mm256_mask_cvtepi8_epi32(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_cvtepi8_epi32(src, 0b11111111, a); + let e = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtepi8_epi32() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm256_maskz_cvtepi8_epi32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_cvtepi8_epi32(0b11111111, a); + let e = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepi8_epi32() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm_set1_epi32(-1); + let r = _mm_mask_cvtepi8_epi32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtepi8_epi32(src, 0b00001111, a); + let e = _mm_set_epi32(12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtepi8_epi32() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm_maskz_cvtepi8_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtepi8_epi32(0b00001111, a); + let e = _mm_set_epi32(12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtepu8_epi32() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_cvtepu8_epi32(a); + let e = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepu8_epi32() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm512_set1_epi32(-1); + let r = _mm512_mask_cvtepu8_epi32(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_cvtepu8_epi32(src, 0b00000000_11111111, a); + let e = _mm512_set_epi32(-1, -1, -1, -1, -1, -1, -1, -1, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtepu8_epi32() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_cvtepu8_epi32(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_cvtepu8_epi32(0b00000000_11111111, a); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepu8_epi32() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm256_set1_epi32(-1); + let r = _mm256_mask_cvtepu8_epi32(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_cvtepu8_epi32(src, 0b11111111, a); + let e = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtepu8_epi32() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm256_maskz_cvtepu8_epi32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_cvtepu8_epi32(0b11111111, a); + let e = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepu8_epi32() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm_set1_epi32(-1); + let r = _mm_mask_cvtepu8_epi32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtepu8_epi32(src, 0b00001111, a); + let e = _mm_set_epi32(12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtepu8_epi32() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm_maskz_cvtepu8_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtepu8_epi32(0b00001111, a); + let e = _mm_set_epi32(12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtepi16_epi32() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_cvtepi16_epi32(a); + let e = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepi16_epi32() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm512_set1_epi32(-1); + let r = _mm512_mask_cvtepi16_epi32(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_cvtepi16_epi32(src, 0b00000000_11111111, a); + let e = _mm512_set_epi32(-1, -1, -1, -1, -1, -1, -1, -1, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtepi16_epi32() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_cvtepi16_epi32(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_cvtepi16_epi32(0b00000000_11111111, a); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepi16_epi32() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let src = _mm256_set1_epi32(-1); + let r = _mm256_mask_cvtepi16_epi32(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_cvtepi16_epi32(src, 0b11111111, a); + let e = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtepi16_epi32() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm256_maskz_cvtepi16_epi32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_cvtepi16_epi32(0b11111111, a); + let e = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepi16_epi32() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let src = _mm_set1_epi32(-1); + let r = _mm_mask_cvtepi16_epi32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtepi16_epi32(src, 0b00001111, a); + let e = _mm_set_epi32(4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtepi16_epi32() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm_maskz_cvtepi16_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtepi16_epi32(0b00001111, a); + let e = _mm_set_epi32(4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtepu16_epi32() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_cvtepu16_epi32(a); + let e = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepu16_epi32() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm512_set1_epi32(-1); + let r = _mm512_mask_cvtepu16_epi32(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_cvtepu16_epi32(src, 0b00000000_11111111, a); + let e = _mm512_set_epi32(-1, -1, -1, -1, -1, -1, -1, -1, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtepu16_epi32() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_cvtepu16_epi32(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_cvtepu16_epi32(0b00000000_11111111, a); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepu16_epi32() { + let a = _mm_set_epi16(8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm256_set1_epi32(-1); + let r = _mm256_mask_cvtepu16_epi32(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_cvtepu16_epi32(src, 0b11111111, a); + let e = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtepu16_epi32() { + let a = _mm_set_epi16(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm256_maskz_cvtepu16_epi32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_cvtepu16_epi32(0b11111111, a); + let e = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepu16_epi32() { + let a = _mm_set_epi16(8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm_set1_epi32(-1); + let r = _mm_mask_cvtepu16_epi32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtepu16_epi32(src, 0b00001111, a); + let e = _mm_set_epi32(12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtepu16_epi32() { + let a = _mm_set_epi16(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm_maskz_cvtepu16_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtepu16_epi32(0b00001111, a); + let e = _mm_set_epi32(12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtepi32_ps() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_cvtepi32_ps(a); + let e = _mm512_set_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepi32_ps() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm512_set1_ps(-1.); + let r = _mm512_mask_cvtepi32_ps(src, 0, a); + assert_eq_m512(r, src); + let r = _mm512_mask_cvtepi32_ps(src, 0b00000000_11111111, a); + let e = _mm512_set_ps( + -1., -1., -1., -1., -1., -1., -1., -1., 8., 9., 10., 11., 12., 13., 14., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtepi32_ps() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_cvtepi32_ps(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_cvtepi32_ps(0b00000000_11111111, a); + let e = _mm512_set_ps( + 0., 0., 0., 0., 0., 0., 0., 0., 8., 9., 10., 11., 12., 13., 14., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepi32_ps() { + let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let src = _mm256_set1_ps(-1.); + let r = _mm256_mask_cvtepi32_ps(src, 0, a); + assert_eq_m256(r, src); + let r = _mm256_mask_cvtepi32_ps(src, 0b11111111, a); + let e = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtepi32_ps() { + let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let r = _mm256_maskz_cvtepi32_ps(0, a); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_cvtepi32_ps(0b11111111, a); + let e = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepi32_ps() { + let a = _mm_set_epi32(1, 2, 3, 4); + let src = _mm_set1_ps(-1.); + let r = _mm_mask_cvtepi32_ps(src, 0, a); + assert_eq_m128(r, src); + let r = _mm_mask_cvtepi32_ps(src, 0b00001111, a); + let e = _mm_set_ps(1., 2., 3., 4.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtepi32_ps() { + let a = _mm_set_epi32(1, 2, 3, 4); + let r = _mm_maskz_cvtepi32_ps(0, a); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_cvtepi32_ps(0b00001111, a); + let e = _mm_set_ps(1., 2., 3., 4.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtepu32_ps() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_cvtepu32_ps(a); + let e = _mm512_set_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepu32_ps() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm512_set1_ps(-1.); + let r = _mm512_mask_cvtepu32_ps(src, 0, a); + assert_eq_m512(r, src); + let r = _mm512_mask_cvtepu32_ps(src, 0b00000000_11111111, a); + let e = _mm512_set_ps( + -1., -1., -1., -1., -1., -1., -1., -1., 8., 9., 10., 11., 12., 13., 14., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtepu32_ps() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_cvtepu32_ps(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_cvtepu32_ps(0b00000000_11111111, a); + let e = _mm512_set_ps( + 0., 0., 0., 0., 0., 0., 0., 0., 8., 9., 10., 11., 12., 13., 14., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtepi32_epi16() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_cvtepi32_epi16(a); + let e = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepi32_epi16() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm256_set1_epi16(-1); + let r = _mm512_mask_cvtepi32_epi16(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm512_mask_cvtepi32_epi16(src, 0b00000000_11111111, a); + let e = _mm256_set_epi16(-1, -1, -1, -1, -1, -1, -1, -1, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtepi32_epi16() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_cvtepi32_epi16(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm512_maskz_cvtepi32_epi16(0b00000000_11111111, a); + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cvtepi32_epi16() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm256_cvtepi32_epi16(a); + let e = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepi32_epi16() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let src = _mm_set1_epi16(-1); + let r = _mm256_mask_cvtepi32_epi16(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvtepi32_epi16(src, 0b11111111, a); + let e = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtepi32_epi16() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm256_maskz_cvtepi32_epi16(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvtepi32_epi16(0b11111111, a); + let e = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cvtepi32_epi16() { + let a = _mm_set_epi32(4, 5, 6, 7); + let r = _mm_cvtepi32_epi16(a); + let e = _mm_set_epi16(0, 0, 0, 0, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepi32_epi16() { + let a = _mm_set_epi32(4, 5, 6, 7); + let src = _mm_set1_epi16(0); + let r = _mm_mask_cvtepi32_epi16(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtepi32_epi16(src, 0b00001111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtepi32_epi16() { + let a = _mm_set_epi32(4, 5, 6, 7); + let r = _mm_maskz_cvtepi32_epi16(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtepi32_epi16(0b00001111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtepi32_epi8() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_cvtepi32_epi8(a); + let e = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepi32_epi8() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm_set1_epi8(-1); + let r = _mm512_mask_cvtepi32_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm512_mask_cvtepi32_epi8(src, 0b00000000_11111111, a); + let e = _mm_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtepi32_epi8() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_cvtepi32_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm512_maskz_cvtepi32_epi8(0b00000000_11111111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cvtepi32_epi8() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm256_cvtepi32_epi8(a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepi32_epi8() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let src = _mm_set1_epi8(0); + let r = _mm256_mask_cvtepi32_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvtepi32_epi8(src, 0b11111111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtepi32_epi8() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm256_maskz_cvtepi32_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvtepi32_epi8(0b11111111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cvtepi32_epi8() { + let a = _mm_set_epi32(4, 5, 6, 7); + let r = _mm_cvtepi32_epi8(a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepi32_epi8() { + let a = _mm_set_epi32(4, 5, 6, 7); + let src = _mm_set1_epi8(0); + let r = _mm_mask_cvtepi32_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtepi32_epi8(src, 0b00001111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtepi32_epi8() { + let a = _mm_set_epi32(4, 5, 6, 7); + let r = _mm_maskz_cvtepi32_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtepi32_epi8(0b00001111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtsepi32_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, i32::MIN, i32::MAX, + ); + let r = _mm512_cvtsepi32_epi16(a); + #[rustfmt::skip] + let e = _mm256_set_epi16( + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, i16::MIN, i16::MAX, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtsepi32_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, i32::MIN, i32::MAX, + ); + let src = _mm256_set1_epi16(-1); + let r = _mm512_mask_cvtsepi32_epi16(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm512_mask_cvtsepi32_epi16(src, 0b00000000_11111111, a); + #[rustfmt::skip] + let e = _mm256_set_epi16( + -1, -1, -1, -1, + -1, -1, -1, -1, + 8, 9, 10, 11, + 12, 13, i16::MIN, i16::MAX, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtsepi32_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, i32::MIN, i32::MAX, + ); + let r = _mm512_maskz_cvtsepi32_epi16(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm512_maskz_cvtsepi32_epi16(0b00000000_11111111, a); + #[rustfmt::skip] + let e = _mm256_set_epi16( + 0, 0, 0, 0, + 0, 0, 0, 0, + 8, 9, 10, 11, + 12, 13, i16::MIN, i16::MAX, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cvtsepi32_epi16() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm256_cvtsepi32_epi16(a); + let e = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtsepi32_epi16() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let src = _mm_set1_epi16(-1); + let r = _mm256_mask_cvtsepi32_epi16(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvtsepi32_epi16(src, 0b11111111, a); + let e = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtsepi32_epi16() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm256_maskz_cvtsepi32_epi16(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvtsepi32_epi16(0b11111111, a); + let e = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cvtsepi32_epi16() { + let a = _mm_set_epi32(4, 5, 6, 7); + let r = _mm_cvtsepi32_epi16(a); + let e = _mm_set_epi16(0, 0, 0, 0, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtsepi32_epi16() { + let a = _mm_set_epi32(4, 5, 6, 7); + let src = _mm_set1_epi16(0); + let r = _mm_mask_cvtsepi32_epi16(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtsepi32_epi16(src, 0b11111111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtsepi32_epi16() { + let a = _mm_set_epi32(4, 5, 6, 7); + let r = _mm_maskz_cvtsepi32_epi16(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtsepi32_epi16(0b11111111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtsepi32_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, i32::MIN, i32::MAX, + ); + let r = _mm512_cvtsepi32_epi8(a); + #[rustfmt::skip] + let e = _mm_set_epi8( + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, i8::MIN, i8::MAX, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtsepi32_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, i32::MIN, i32::MAX, + ); + let src = _mm_set1_epi8(-1); + let r = _mm512_mask_cvtsepi32_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm512_mask_cvtsepi32_epi8(src, 0b00000000_11111111, a); + #[rustfmt::skip] + let e = _mm_set_epi8( + -1, -1, -1, -1, + -1, -1, -1, -1, + 8, 9, 10, 11, + 12, 13, i8::MIN, i8::MAX, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtsepi32_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, i32::MIN, i32::MAX, + ); + let r = _mm512_maskz_cvtsepi32_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm512_maskz_cvtsepi32_epi8(0b00000000_11111111, a); + #[rustfmt::skip] + let e = _mm_set_epi8( + 0, 0, 0, 0, + 0, 0, 0, 0, + 8, 9, 10, 11, + 12, 13, i8::MIN, i8::MAX, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cvtsepi32_epi8() { + let a = _mm256_set_epi32(9, 10, 11, 12, 13, 14, 15, 16); + let r = _mm256_cvtsepi32_epi8(a); + #[rustfmt::skip] + let e = _mm_set_epi8( + 0, 0, 0, 0, + 0, 0, 0, 0, + 9, 10, 11, 12, + 13, 14, 15, 16, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtsepi32_epi8() { + let a = _mm256_set_epi32(9, 10, 11, 12, 13, 14, 15, 16); + let src = _mm_set1_epi8(0); + let r = _mm256_mask_cvtsepi32_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvtsepi32_epi8(src, 0b11111111, a); + #[rustfmt::skip] + let e = _mm_set_epi8( + 0, 0, 0, 0, + 0, 0, 0, 0, + 9, 10, 11, 12, + 13, 14, 15, 16, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtsepi32_epi8() { + let a = _mm256_set_epi32(9, 10, 11, 12, 13, 14, 15, 16); + let r = _mm256_maskz_cvtsepi32_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvtsepi32_epi8(0b11111111, a); + #[rustfmt::skip] + let e = _mm_set_epi8( + 0, 0, 0, 0, + 0, 0, 0, 0, + 9, 10, 11, 12, + 13, 14, 15, 16, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cvtsepi32_epi8() { + let a = _mm_set_epi32(13, 14, 15, 16); + let r = _mm_cvtsepi32_epi8(a); + #[rustfmt::skip] + let e = _mm_set_epi8( + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 13, 14, 15, 16, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtsepi32_epi8() { + let a = _mm_set_epi32(13, 14, 15, 16); + let src = _mm_set1_epi8(0); + let r = _mm_mask_cvtsepi32_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtsepi32_epi8(src, 0b00001111, a); + #[rustfmt::skip] + let e = _mm_set_epi8( + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 13, 14, 15, 16, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtsepi32_epi8() { + let a = _mm_set_epi32(13, 14, 15, 16); + let r = _mm_maskz_cvtsepi32_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtsepi32_epi8(0b00001111, a); + #[rustfmt::skip] + let e = _mm_set_epi8( + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 13, 14, 15, 16, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtusepi32_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, i32::MIN, i32::MIN, + ); + let r = _mm512_cvtusepi32_epi16(a); + let e = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, -1, -1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtusepi32_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, i32::MIN, i32::MIN, + ); + let src = _mm256_set1_epi16(-1); + let r = _mm512_mask_cvtusepi32_epi16(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm512_mask_cvtusepi32_epi16(src, 0b00000000_11111111, a); + let e = _mm256_set_epi16(-1, -1, -1, -1, -1, -1, -1, -1, 8, 9, 10, 11, 12, 13, -1, -1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtusepi32_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, i32::MIN, i32::MIN, + ); + let r = _mm512_maskz_cvtusepi32_epi16(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm512_maskz_cvtusepi32_epi16(0b00000000_11111111, a); + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, -1, -1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cvtusepi32_epi16() { + let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let r = _mm256_cvtusepi32_epi16(a); + let e = _mm_set_epi16(1, 2, 3, 4, 5, 6, 7, 8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtusepi32_epi16() { + let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let src = _mm_set1_epi16(0); + let r = _mm256_mask_cvtusepi32_epi16(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvtusepi32_epi16(src, 0b11111111, a); + let e = _mm_set_epi16(1, 2, 3, 4, 5, 6, 7, 8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtusepi32_epi16() { + let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let r = _mm256_maskz_cvtusepi32_epi16(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvtusepi32_epi16(0b11111111, a); + let e = _mm_set_epi16(1, 2, 3, 4, 5, 6, 7, 8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cvtusepi32_epi16() { + let a = _mm_set_epi32(5, 6, 7, 8); + let r = _mm_cvtusepi32_epi16(a); + let e = _mm_set_epi16(0, 0, 0, 0, 5, 6, 7, 8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtusepi32_epi16() { + let a = _mm_set_epi32(5, 6, 7, 8); + let src = _mm_set1_epi16(0); + let r = _mm_mask_cvtusepi32_epi16(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtusepi32_epi16(src, 0b00001111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 5, 6, 7, 8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtusepi32_epi16() { + let a = _mm_set_epi32(5, 6, 7, 8); + let r = _mm_maskz_cvtusepi32_epi16(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtusepi32_epi16(0b00001111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 5, 6, 7, 8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtusepi32_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, i32::MIN, i32::MIN, + ); + let r = _mm512_cvtusepi32_epi8(a); + let e = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, -1, -1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtusepi32_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, i32::MIN, i32::MIN, + ); + let src = _mm_set1_epi8(-1); + let r = _mm512_mask_cvtusepi32_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm512_mask_cvtusepi32_epi8(src, 0b00000000_11111111, a); + let e = _mm_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, 8, 9, 10, 11, 12, 13, -1, -1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtusepi32_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, i32::MIN, i32::MIN, + ); + let r = _mm512_maskz_cvtusepi32_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm512_maskz_cvtusepi32_epi8(0b00000000_11111111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, -1, -1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cvtusepi32_epi8() { + let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, i32::MAX); + let r = _mm256_cvtusepi32_epi8(a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, u8::MAX as i8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtusepi32_epi8() { + let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, i32::MAX); + let src = _mm_set1_epi8(0); + let r = _mm256_mask_cvtusepi32_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvtusepi32_epi8(src, 0b11111111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, u8::MAX as i8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtusepi32_epi8() { + let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, i32::MAX); + let r = _mm256_maskz_cvtusepi32_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvtusepi32_epi8(0b11111111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, u8::MAX as i8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cvtusepi32_epi8() { + let a = _mm_set_epi32(5, 6, 7, i32::MAX); + let r = _mm_cvtusepi32_epi8(a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 7, u8::MAX as i8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtusepi32_epi8() { + let a = _mm_set_epi32(5, 6, 7, i32::MAX); + let src = _mm_set1_epi8(0); + let r = _mm_mask_cvtusepi32_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtusepi32_epi8(src, 0b00001111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 7, u8::MAX as i8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtusepi32_epi8() { + let a = _mm_set_epi32(5, 6, 7, i32::MAX); + let r = _mm_maskz_cvtusepi32_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtusepi32_epi8(0b00001111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 7, u8::MAX as i8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvt_roundps_epi32() { + let a = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let r = _mm512_cvt_roundps_epi32::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a); + let e = _mm512_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8, 8, 10, 10, 12, 12, 14, 14, 16); + assert_eq_m512i(r, e); + let r = _mm512_cvt_roundps_epi32::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a); + let e = _mm512_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvt_roundps_epi32() { + let a = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let src = _mm512_set1_epi32(0); + let r = _mm512_mask_cvt_roundps_epi32::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + src, 0, a, + ); + assert_eq_m512i(r, src); + let r = _mm512_mask_cvt_roundps_epi32::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + src, + 0b00000000_11111111, + a, + ); + let e = _mm512_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvt_roundps_epi32() { + let a = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let r = _mm512_maskz_cvt_roundps_epi32::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, + ); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_cvt_roundps_epi32::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00000000_11111111, + a, + ); + let e = _mm512_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvt_roundps_epu32() { + let a = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let r = _mm512_cvt_roundps_epu32::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a); + let e = _mm512_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1, 8, 10, 10, 12, 12, 14, 14, 16); + assert_eq_m512i(r, e); + let r = _mm512_cvt_roundps_epu32::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a); + let e = _mm512_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvt_roundps_epu32() { + let a = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let src = _mm512_set1_epi32(0); + let r = _mm512_mask_cvt_roundps_epu32::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + src, 0, a, + ); + assert_eq_m512i(r, src); + let r = _mm512_mask_cvt_roundps_epu32::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + src, + 0b00000000_11111111, + a, + ); + let e = _mm512_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvt_roundps_epu32() { + let a = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let r = _mm512_maskz_cvt_roundps_epu32::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, + ); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_cvt_roundps_epu32::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00000000_11111111, + a, + ); + let e = _mm512_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvt_roundepi32_ps() { + let a = _mm512_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8, 8, 10, 10, 12, 12, 14, 14, 16); + let r = _mm512_cvt_roundepi32_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a); + let e = _mm512_setr_ps( + 0., -2., 2., -4., 4., -6., 6., -8., 8., 10., 10., 12., 12., 14., 14., 16., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvt_roundepi32_ps() { + let a = _mm512_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8, 8, 10, 10, 12, 12, 14, 14, 16); + let src = _mm512_set1_ps(0.); + let r = _mm512_mask_cvt_roundepi32_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + src, 0, a, + ); + assert_eq_m512(r, src); + let r = _mm512_mask_cvt_roundepi32_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + src, + 0b00000000_11111111, + a, + ); + let e = _mm512_setr_ps( + 0., -2., 2., -4., 4., -6., 6., -8., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvt_roundepi32_ps() { + let a = _mm512_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8, 8, 10, 10, 12, 12, 14, 14, 16); + let r = _mm512_maskz_cvt_roundepi32_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, + ); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_cvt_roundepi32_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00000000_11111111, + a, + ); + let e = _mm512_setr_ps( + 0., -2., 2., -4., 4., -6., 6., -8., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvt_roundepu32_ps() { + let a = _mm512_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8, 8, 10, 10, 12, 12, 14, 14, 16); + let r = _mm512_cvt_roundepu32_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 0., 4294967300., 2., 4294967300., + 4., 4294967300., 6., 4294967300., + 8., 10., 10., 12., + 12., 14., 14., 16., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvt_roundepu32_ps() { + let a = _mm512_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8, 8, 10, 10, 12, 12, 14, 14, 16); + let src = _mm512_set1_ps(0.); + let r = _mm512_mask_cvt_roundepu32_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + src, 0, a, + ); + assert_eq_m512(r, src); + let r = _mm512_mask_cvt_roundepu32_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + src, + 0b00000000_11111111, + a, + ); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 0., 4294967300., 2., 4294967300., + 4., 4294967300., 6., 4294967300., + 0., 0., 0., 0., + 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvt_roundepu32_ps() { + let a = _mm512_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8, 8, 10, 10, 12, 12, 14, 14, 16); + let r = _mm512_maskz_cvt_roundepu32_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, + ); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_cvt_roundepu32_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00000000_11111111, + a, + ); + #[rustfmt::skip] + let e = _mm512_setr_ps( + 0., 4294967300., 2., 4294967300., + 4., 4294967300., 6., 4294967300., + 0., 0., 0., 0., + 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvt_roundps_ph() { + let a = _mm512_set1_ps(1.); + let r = _mm512_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(a); + let e = _mm256_setr_epi64x( + 4323521613979991040, + 4323521613979991040, + 4323521613979991040, + 4323521613979991040, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvt_roundps_ph() { + let a = _mm512_set1_ps(1.); + let src = _mm256_set1_epi16(0); + let r = _mm512_mask_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm512_mask_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(src, 0b00000000_11111111, a); + let e = _mm256_setr_epi64x(4323521613979991040, 4323521613979991040, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvt_roundps_ph() { + let a = _mm512_set1_ps(1.); + let r = _mm512_maskz_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm512_maskz_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(0b00000000_11111111, a); + let e = _mm256_setr_epi64x(4323521613979991040, 4323521613979991040, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvt_roundps_ph() { + let a = _mm256_set1_ps(1.); + let src = _mm_set1_epi16(0); + let r = _mm256_mask_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(src, 0b11111111, a); + let e = _mm_setr_epi64x(4323521613979991040, 4323521613979991040); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvt_roundps_ph() { + let a = _mm256_set1_ps(1.); + let r = _mm256_maskz_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(0b11111111, a); + let e = _mm_setr_epi64x(4323521613979991040, 4323521613979991040); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvt_roundps_ph() { + let a = _mm_set1_ps(1.); + let src = _mm_set1_epi16(0); + let r = _mm_mask_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(src, 0b00001111, a); + let e = _mm_setr_epi64x(4323521613979991040, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvt_roundps_ph() { + let a = _mm_set1_ps(1.); + let r = _mm_maskz_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(0b00001111, a); + let e = _mm_setr_epi64x(4323521613979991040, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtps_ph() { + let a = _mm512_set1_ps(1.); + let r = _mm512_cvtps_ph::<_MM_FROUND_NO_EXC>(a); + let e = _mm256_setr_epi64x( + 4323521613979991040, + 4323521613979991040, + 4323521613979991040, + 4323521613979991040, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtps_ph() { + let a = _mm512_set1_ps(1.); + let src = _mm256_set1_epi16(0); + let r = _mm512_mask_cvtps_ph::<_MM_FROUND_NO_EXC>(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm512_mask_cvtps_ph::<_MM_FROUND_NO_EXC>(src, 0b00000000_11111111, a); + let e = _mm256_setr_epi64x(4323521613979991040, 4323521613979991040, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtps_ph() { + let a = _mm512_set1_ps(1.); + let r = _mm512_maskz_cvtps_ph::<_MM_FROUND_NO_EXC>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm512_maskz_cvtps_ph::<_MM_FROUND_NO_EXC>(0b00000000_11111111, a); + let e = _mm256_setr_epi64x(4323521613979991040, 4323521613979991040, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtps_ph() { + let a = _mm256_set1_ps(1.); + let src = _mm_set1_epi16(0); + let r = _mm256_mask_cvtps_ph::<_MM_FROUND_NO_EXC>(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvtps_ph::<_MM_FROUND_NO_EXC>(src, 0b11111111, a); + let e = _mm_setr_epi64x(4323521613979991040, 4323521613979991040); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtps_ph() { + let a = _mm256_set1_ps(1.); + let r = _mm256_maskz_cvtps_ph::<_MM_FROUND_NO_EXC>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvtps_ph::<_MM_FROUND_NO_EXC>(0b11111111, a); + let e = _mm_setr_epi64x(4323521613979991040, 4323521613979991040); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtps_ph() { + let a = _mm_set1_ps(1.); + let src = _mm_set1_epi16(0); + let r = _mm_mask_cvtps_ph::<_MM_FROUND_NO_EXC>(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtps_ph::<_MM_FROUND_NO_EXC>(src, 0b00001111, a); + let e = _mm_setr_epi64x(4323521613979991040, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtps_ph() { + let a = _mm_set1_ps(1.); + let r = _mm_maskz_cvtps_ph::<_MM_FROUND_NO_EXC>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtps_ph::<_MM_FROUND_NO_EXC>(0b00001111, a); + let e = _mm_setr_epi64x(4323521613979991040, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvt_roundph_ps() { + let a = _mm256_setr_epi64x( + 4323521613979991040, + 4323521613979991040, + 4323521613979991040, + 4323521613979991040, + ); + let r = _mm512_cvt_roundph_ps::<_MM_FROUND_NO_EXC>(a); + let e = _mm512_set1_ps(1.); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvt_roundph_ps() { + let a = _mm256_setr_epi64x( + 4323521613979991040, + 4323521613979991040, + 4323521613979991040, + 4323521613979991040, + ); + let src = _mm512_set1_ps(0.); + let r = _mm512_mask_cvt_roundph_ps::<_MM_FROUND_NO_EXC>(src, 0, a); + assert_eq_m512(r, src); + let r = _mm512_mask_cvt_roundph_ps::<_MM_FROUND_NO_EXC>(src, 0b00000000_11111111, a); + let e = _mm512_setr_ps( + 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvt_roundph_ps() { + let a = _mm256_setr_epi64x( + 4323521613979991040, + 4323521613979991040, + 4323521613979991040, + 4323521613979991040, + ); + let r = _mm512_maskz_cvt_roundph_ps::<_MM_FROUND_NO_EXC>(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_cvt_roundph_ps::<_MM_FROUND_NO_EXC>(0b00000000_11111111, a); + let e = _mm512_setr_ps( + 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtph_ps() { + let a = _mm256_setr_epi64x( + 4323521613979991040, + 4323521613979991040, + 4323521613979991040, + 4323521613979991040, + ); + let r = _mm512_cvtph_ps(a); + let e = _mm512_set1_ps(1.); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtph_ps() { + let a = _mm256_setr_epi64x( + 4323521613979991040, + 4323521613979991040, + 4323521613979991040, + 4323521613979991040, + ); + let src = _mm512_set1_ps(0.); + let r = _mm512_mask_cvtph_ps(src, 0, a); + assert_eq_m512(r, src); + let r = _mm512_mask_cvtph_ps(src, 0b00000000_11111111, a); + let e = _mm512_setr_ps( + 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtph_ps() { + let a = _mm256_setr_epi64x( + 4323521613979991040, + 4323521613979991040, + 4323521613979991040, + 4323521613979991040, + ); + let r = _mm512_maskz_cvtph_ps(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_cvtph_ps(0b00000000_11111111, a); + let e = _mm512_setr_ps( + 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtph_ps() { + let a = _mm_setr_epi64x(4323521613979991040, 4323521613979991040); + let src = _mm256_set1_ps(0.); + let r = _mm256_mask_cvtph_ps(src, 0, a); + assert_eq_m256(r, src); + let r = _mm256_mask_cvtph_ps(src, 0b11111111, a); + let e = _mm256_setr_ps(1., 1., 1., 1., 1., 1., 1., 1.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtph_ps() { + let a = _mm_setr_epi64x(4323521613979991040, 4323521613979991040); + let r = _mm256_maskz_cvtph_ps(0, a); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_cvtph_ps(0b11111111, a); + let e = _mm256_setr_ps(1., 1., 1., 1., 1., 1., 1., 1.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtph_ps() { + let a = _mm_setr_epi64x(4323521613979991040, 4323521613979991040); + let src = _mm_set1_ps(0.); + let r = _mm_mask_cvtph_ps(src, 0, a); + assert_eq_m128(r, src); + let r = _mm_mask_cvtph_ps(src, 0b00001111, a); + let e = _mm_setr_ps(1., 1., 1., 1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtph_ps() { + let a = _mm_setr_epi64x(4323521613979991040, 4323521613979991040); + let r = _mm_maskz_cvtph_ps(0, a); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_cvtph_ps(0b00001111, a); + let e = _mm_setr_ps(1., 1., 1., 1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtt_roundps_epi32() { + let a = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let r = _mm512_cvtt_roundps_epi32::<_MM_FROUND_NO_EXC>(a); + let e = _mm512_setr_epi32(0, -1, 2, -3, 4, -5, 6, -7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtt_roundps_epi32() { + let a = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let src = _mm512_set1_epi32(0); + let r = _mm512_mask_cvtt_roundps_epi32::<_MM_FROUND_NO_EXC>(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_cvtt_roundps_epi32::<_MM_FROUND_NO_EXC>(src, 0b00000000_11111111, a); + let e = _mm512_setr_epi32(0, -1, 2, -3, 4, -5, 6, -7, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtt_roundps_epi32() { + let a = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let r = _mm512_maskz_cvtt_roundps_epi32::<_MM_FROUND_NO_EXC>(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_cvtt_roundps_epi32::<_MM_FROUND_NO_EXC>(0b00000000_11111111, a); + let e = _mm512_setr_epi32(0, -1, 2, -3, 4, -5, 6, -7, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtt_roundps_epu32() { + let a = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let r = _mm512_cvtt_roundps_epu32::<_MM_FROUND_NO_EXC>(a); + let e = _mm512_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtt_roundps_epu32() { + let a = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let src = _mm512_set1_epi32(0); + let r = _mm512_mask_cvtt_roundps_epu32::<_MM_FROUND_NO_EXC>(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_cvtt_roundps_epu32::<_MM_FROUND_NO_EXC>(src, 0b00000000_11111111, a); + let e = _mm512_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtt_roundps_epu32() { + let a = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let r = _mm512_maskz_cvtt_roundps_epu32::<_MM_FROUND_NO_EXC>(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_cvtt_roundps_epu32::<_MM_FROUND_NO_EXC>(0b00000000_11111111, a); + let e = _mm512_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvttps_epi32() { + let a = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let r = _mm512_cvttps_epi32(a); + let e = _mm512_setr_epi32(0, -1, 2, -3, 4, -5, 6, -7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvttps_epi32() { + let a = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let src = _mm512_set1_epi32(0); + let r = _mm512_mask_cvttps_epi32(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_cvttps_epi32(src, 0b00000000_11111111, a); + let e = _mm512_setr_epi32(0, -1, 2, -3, 4, -5, 6, -7, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvttps_epi32() { + let a = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let r = _mm512_maskz_cvttps_epi32(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_cvttps_epi32(0b00000000_11111111, a); + let e = _mm512_setr_epi32(0, -1, 2, -3, 4, -5, 6, -7, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvttps_epi32() { + let a = _mm256_set_ps(8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5); + let src = _mm256_set1_epi32(0); + let r = _mm256_mask_cvttps_epi32(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_cvttps_epi32(src, 0b11111111, a); + let e = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvttps_epi32() { + let a = _mm256_set_ps(8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5); + let r = _mm256_maskz_cvttps_epi32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_cvttps_epi32(0b11111111, a); + let e = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvttps_epi32() { + let a = _mm_set_ps(12., 13.5, 14., 15.5); + let src = _mm_set1_epi32(0); + let r = _mm_mask_cvttps_epi32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvttps_epi32(src, 0b00001111, a); + let e = _mm_set_epi32(12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvttps_epi32() { + let a = _mm_set_ps(12., 13.5, 14., 15.5); + let r = _mm_maskz_cvttps_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvttps_epi32(0b00001111, a); + let e = _mm_set_epi32(12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvttps_epu32() { + let a = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let r = _mm512_cvttps_epu32(a); + let e = _mm512_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvttps_epu32() { + let a = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let src = _mm512_set1_epi32(0); + let r = _mm512_mask_cvttps_epu32(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_cvttps_epu32(src, 0b00000000_11111111, a); + let e = _mm512_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvttps_epu32() { + let a = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, + ); + let r = _mm512_maskz_cvttps_epu32(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_cvttps_epu32(0b00000000_11111111, a); + let e = _mm512_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cvttps_epu32() { + let a = _mm256_set_ps(8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5); + let r = _mm256_cvttps_epu32(a); + let e = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvttps_epu32() { + let a = _mm256_set_ps(8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5); + let src = _mm256_set1_epi32(0); + let r = _mm256_mask_cvttps_epu32(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_cvttps_epu32(src, 0b11111111, a); + let e = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvttps_epu32() { + let a = _mm256_set_ps(8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5); + let r = _mm256_maskz_cvttps_epu32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_cvttps_epu32(0b11111111, a); + let e = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cvttps_epu32() { + let a = _mm_set_ps(12., 13.5, 14., 15.5); + let r = _mm_cvttps_epu32(a); + let e = _mm_set_epi32(12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvttps_epu32() { + let a = _mm_set_ps(12., 13.5, 14., 15.5); + let src = _mm_set1_epi32(0); + let r = _mm_mask_cvttps_epu32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvttps_epu32(src, 0b00001111, a); + let e = _mm_set_epi32(12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvttps_epu32() { + let a = _mm_set_ps(12., 13.5, 14., 15.5); + let r = _mm_maskz_cvttps_epu32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvttps_epu32(0b00001111, a); + let e = _mm_set_epi32(12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_i32gather_ps() { + let mut arr = [0f32; 256]; + for i in 0..256 { + arr[i] = i as f32; + } + // A multiplier of 4 is word-addressing + #[rustfmt::skip] + let index = _mm512_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112, + 120, 128, 136, 144, 152, 160, 168, 176); + let r = _mm512_i32gather_ps::<4>(index, arr.as_ptr() as *const u8); + #[rustfmt::skip] + assert_eq_m512(r, _mm512_setr_ps(0., 16., 32., 48., 64., 80., 96., 112., + 120., 128., 136., 144., 152., 160., 168., 176.)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_i32gather_ps() { + let mut arr = [0f32; 256]; + for i in 0..256 { + arr[i] = i as f32; + } + let src = _mm512_set1_ps(2.); + let mask = 0b10101010_10101010; + #[rustfmt::skip] + let index = _mm512_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112, + 120, 128, 136, 144, 152, 160, 168, 176); + // A multiplier of 4 is word-addressing + let r = _mm512_mask_i32gather_ps::<4>(src, mask, index, arr.as_ptr() as *const u8); + #[rustfmt::skip] + assert_eq_m512(r, _mm512_setr_ps(2., 16., 2., 48., 2., 80., 2., 112., + 2., 128., 2., 144., 2., 160., 2., 176.)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_i32gather_epi32() { + let mut arr = [0i32; 256]; + for i in 0..256 { + arr[i] = i as i32; + } + // A multiplier of 4 is word-addressing + #[rustfmt::skip] + let index = _mm512_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112, + 120, 128, 136, 144, 152, 160, 168, 176); + let r = _mm512_i32gather_epi32::<4>(index, arr.as_ptr() as *const u8); + #[rustfmt::skip] + assert_eq_m512i(r, _mm512_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112, + 120, 128, 136, 144, 152, 160, 168, 176)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_i32gather_epi32() { + let mut arr = [0i32; 256]; + for i in 0..256 { + arr[i] = i as i32; + } + let src = _mm512_set1_epi32(2); + let mask = 0b10101010_10101010; + let index = _mm512_setr_epi32( + 0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, + ); + // A multiplier of 4 is word-addressing + let r = _mm512_mask_i32gather_epi32::<4>(src, mask, index, arr.as_ptr() as *const u8); + assert_eq_m512i( + r, + _mm512_setr_epi32(2, 16, 2, 48, 2, 80, 2, 112, 2, 144, 2, 176, 2, 208, 2, 240), + ); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_i32scatter_ps() { + let mut arr = [0f32; 256]; + #[rustfmt::skip] + let index = _mm512_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112, + 128, 144, 160, 176, 192, 208, 224, 240); + let src = _mm512_setr_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + // A multiplier of 4 is word-addressing + _mm512_i32scatter_ps::<4>(arr.as_mut_ptr() as *mut u8, index, src); + let mut expected = [0f32; 256]; + for i in 0..16 { + expected[i * 16] = (i + 1) as f32; + } + assert_eq!(&arr[..], &expected[..],); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_i32scatter_ps() { + let mut arr = [0f32; 256]; + let mask = 0b10101010_10101010; + #[rustfmt::skip] + let index = _mm512_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112, + 128, 144, 160, 176, 192, 208, 224, 240); + let src = _mm512_setr_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + // A multiplier of 4 is word-addressing + _mm512_mask_i32scatter_ps::<4>(arr.as_mut_ptr() as *mut u8, mask, index, src); + let mut expected = [0f32; 256]; + for i in 0..8 { + expected[i * 32 + 16] = 2. * (i + 1) as f32; + } + assert_eq!(&arr[..], &expected[..],); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_i32scatter_epi32() { + let mut arr = [0i32; 256]; + #[rustfmt::skip] + + let index = _mm512_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112, + 128, 144, 160, 176, 192, 208, 224, 240); + let src = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + // A multiplier of 4 is word-addressing + _mm512_i32scatter_epi32::<4>(arr.as_mut_ptr() as *mut u8, index, src); + let mut expected = [0i32; 256]; + for i in 0..16 { + expected[i * 16] = (i + 1) as i32; + } + assert_eq!(&arr[..], &expected[..],); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_i32scatter_epi32() { + let mut arr = [0i32; 256]; + let mask = 0b10101010_10101010; + #[rustfmt::skip] + let index = _mm512_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112, + 128, 144, 160, 176, 192, 208, 224, 240); + let src = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + // A multiplier of 4 is word-addressing + _mm512_mask_i32scatter_epi32::<4>(arr.as_mut_ptr() as *mut u8, mask, index, src); + let mut expected = [0i32; 256]; + for i in 0..8 { + expected[i * 32 + 16] = 2 * (i + 1) as i32; + } + assert_eq!(&arr[..], &expected[..],); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmplt_ps_mask() { + #[rustfmt::skip] + let a = _mm512_set_ps(0., 1., -1., f32::MAX, f32::NAN, f32::MIN, 100., -100., + 0., 1., -1., f32::MAX, f32::NAN, f32::MIN, 100., -100.); + let b = _mm512_set1_ps(-1.); + let m = _mm512_cmplt_ps_mask(a, b); + assert_eq!(m, 0b00000101_00000101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmplt_ps_mask() { + #[rustfmt::skip] + let a = _mm512_set_ps(0., 1., -1., f32::MAX, f32::NAN, f32::MIN, 100., -100., + 0., 1., -1., f32::MAX, f32::NAN, f32::MIN, 100., -100.); + let b = _mm512_set1_ps(-1.); + let mask = 0b01100110_01100110; + let r = _mm512_mask_cmplt_ps_mask(mask, a, b); + assert_eq!(r, 0b00000100_00000100); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpnlt_ps_mask() { + #[rustfmt::skip] + let a = _mm512_set_ps(0., 1., -1., f32::MAX, f32::NAN, f32::MIN, 100., -100., + 0., 1., -1., f32::MAX, f32::NAN, f32::MIN, 100., -100.); + let b = _mm512_set1_ps(-1.); + assert_eq!(_mm512_cmpnlt_ps_mask(a, b), !_mm512_cmplt_ps_mask(a, b)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpnlt_ps_mask() { + #[rustfmt::skip] + let a = _mm512_set_ps(0., 1., -1., f32::MAX, f32::NAN, f32::MIN, 100., -100., + 0., 1., -1., f32::MAX, f32::NAN, f32::MIN, 100., -100.); + let b = _mm512_set1_ps(-1.); + let mask = 0b01111010_01111010; + assert_eq!(_mm512_mask_cmpnlt_ps_mask(mask, a, b), 0b01111010_01111010); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpnle_ps_mask() { + #[rustfmt::skip] + let a = _mm512_set_ps(0., 1., -1., f32::MAX, f32::NAN, f32::MIN, 100., -100., + 0., 1., -1., f32::MAX, f32::NAN, f32::MIN, 100., -100.); + let b = _mm512_set1_ps(-1.); + let m = _mm512_cmpnle_ps_mask(b, a); + assert_eq!(m, 0b00001101_00001101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpnle_ps_mask() { + #[rustfmt::skip] + let a = _mm512_set_ps(0., 1., -1., f32::MAX, f32::NAN, f32::MIN, 100., -100., + 0., 1., -1., f32::MAX, f32::NAN, f32::MIN, 100., -100.); + let b = _mm512_set1_ps(-1.); + let mask = 0b01100110_01100110; + let r = _mm512_mask_cmpnle_ps_mask(mask, b, a); + assert_eq!(r, 0b00000100_00000100); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmple_ps_mask() { + #[rustfmt::skip] + let a = _mm512_set_ps(0., 1., -1., f32::MAX, f32::NAN, f32::MIN, 100., -100., + 0., 1., -1., f32::MAX, f32::NAN, f32::MIN, 100., -100.); + let b = _mm512_set1_ps(-1.); + assert_eq!(_mm512_cmple_ps_mask(a, b), 0b00100101_00100101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmple_ps_mask() { + #[rustfmt::skip] + let a = _mm512_set_ps(0., 1., -1., f32::MAX, f32::NAN, f32::MIN, 100., -100., + 0., 1., -1., f32::MAX, f32::NAN, f32::MIN, 100., -100.); + let b = _mm512_set1_ps(-1.); + let mask = 0b01111010_01111010; + assert_eq!(_mm512_mask_cmple_ps_mask(mask, a, b), 0b00100000_00100000); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpeq_ps_mask() { + #[rustfmt::skip] + let a = _mm512_set_ps(0., 1., -1., 13., f32::MAX, f32::MIN, f32::NAN, -100., + 0., 1., -1., 13., f32::MAX, f32::MIN, f32::NAN, -100.); + #[rustfmt::skip] + let b = _mm512_set_ps(0., 1., 13., 42., f32::MAX, f32::MIN, f32::NAN, -100., + 0., 1., 13., 42., f32::MAX, f32::MIN, f32::NAN, -100.); + let m = _mm512_cmpeq_ps_mask(b, a); + assert_eq!(m, 0b11001101_11001101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpeq_ps_mask() { + #[rustfmt::skip] + let a = _mm512_set_ps(0., 1., -1., 13., f32::MAX, f32::MIN, f32::NAN, -100., + 0., 1., -1., 13., f32::MAX, f32::MIN, f32::NAN, -100.); + #[rustfmt::skip] + let b = _mm512_set_ps(0., 1., 13., 42., f32::MAX, f32::MIN, f32::NAN, -100., + 0., 1., 13., 42., f32::MAX, f32::MIN, f32::NAN, -100.); + let mask = 0b01111010_01111010; + let r = _mm512_mask_cmpeq_ps_mask(mask, b, a); + assert_eq!(r, 0b01001000_01001000); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpneq_ps_mask() { + #[rustfmt::skip] + let a = _mm512_set_ps(0., 1., -1., 13., f32::MAX, f32::MIN, f32::NAN, -100., + 0., 1., -1., 13., f32::MAX, f32::MIN, f32::NAN, -100.); + #[rustfmt::skip] + let b = _mm512_set_ps(0., 1., 13., 42., f32::MAX, f32::MIN, f32::NAN, -100., + 0., 1., 13., 42., f32::MAX, f32::MIN, f32::NAN, -100.); + let m = _mm512_cmpneq_ps_mask(b, a); + assert_eq!(m, 0b00110010_00110010); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpneq_ps_mask() { + #[rustfmt::skip] + let a = _mm512_set_ps(0., 1., -1., 13., f32::MAX, f32::MIN, f32::NAN, -100., + 0., 1., -1., 13., f32::MAX, f32::MIN, f32::NAN, -100.); + #[rustfmt::skip] + let b = _mm512_set_ps(0., 1., 13., 42., f32::MAX, f32::MIN, f32::NAN, -100., + 0., 1., 13., 42., f32::MAX, f32::MIN, f32::NAN, -100.); + let mask = 0b01111010_01111010; + let r = _mm512_mask_cmpneq_ps_mask(mask, b, a); + assert_eq!(r, 0b00110010_00110010) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmp_ps_mask() { + #[rustfmt::skip] + let a = _mm512_set_ps(0., 1., -1., 13., f32::MAX, f32::MIN, 100., -100., + 0., 1., -1., 13., f32::MAX, f32::MIN, 100., -100.); + let b = _mm512_set1_ps(-1.); + let m = _mm512_cmp_ps_mask::<_CMP_LT_OQ>(a, b); + assert_eq!(m, 0b00000101_00000101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmp_ps_mask() { + #[rustfmt::skip] + let a = _mm512_set_ps(0., 1., -1., 13., f32::MAX, f32::MIN, 100., -100., + 0., 1., -1., 13., f32::MAX, f32::MIN, 100., -100.); + let b = _mm512_set1_ps(-1.); + let mask = 0b01100110_01100110; + let r = _mm512_mask_cmp_ps_mask::<_CMP_LT_OQ>(mask, a, b); + assert_eq!(r, 0b00000100_00000100); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmp_ps_mask() { + let a = _mm256_set_ps(0., 1., -1., 13., f32::MAX, f32::MIN, 100., -100.); + let b = _mm256_set1_ps(-1.); + let m = _mm256_cmp_ps_mask::<_CMP_LT_OQ>(a, b); + assert_eq!(m, 0b00000101); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmp_ps_mask() { + let a = _mm256_set_ps(0., 1., -1., 13., f32::MAX, f32::MIN, 100., -100.); + let b = _mm256_set1_ps(-1.); + let mask = 0b01100110; + let r = _mm256_mask_cmp_ps_mask::<_CMP_LT_OQ>(mask, a, b); + assert_eq!(r, 0b00000100); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmp_ps_mask() { + let a = _mm_set_ps(0., 1., -1., 13.); + let b = _mm_set1_ps(1.); + let m = _mm_cmp_ps_mask::<_CMP_LT_OQ>(a, b); + assert_eq!(m, 0b00001010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmp_ps_mask() { + let a = _mm_set_ps(0., 1., -1., 13.); + let b = _mm_set1_ps(1.); + let mask = 0b11111111; + let r = _mm_mask_cmp_ps_mask::<_CMP_LT_OQ>(mask, a, b); + assert_eq!(r, 0b00001010); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmp_round_ps_mask() { + #[rustfmt::skip] + let a = _mm512_set_ps(0., 1., -1., 13., f32::MAX, f32::MIN, 100., -100., + 0., 1., -1., 13., f32::MAX, f32::MIN, 100., -100.); + let b = _mm512_set1_ps(-1.); + let m = _mm512_cmp_round_ps_mask::<_CMP_LT_OQ, _MM_FROUND_CUR_DIRECTION>(a, b); + assert_eq!(m, 0b00000101_00000101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmp_round_ps_mask() { + #[rustfmt::skip] + let a = _mm512_set_ps(0., 1., -1., 13., f32::MAX, f32::MIN, 100., -100., + 0., 1., -1., 13., f32::MAX, f32::MIN, 100., -100.); + let b = _mm512_set1_ps(-1.); + let mask = 0b01100110_01100110; + let r = _mm512_mask_cmp_round_ps_mask::<_CMP_LT_OQ, _MM_FROUND_CUR_DIRECTION>(mask, a, b); + assert_eq!(r, 0b00000100_00000100); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpord_ps_mask() { + #[rustfmt::skip] + let a = _mm512_set_ps(f32::NAN, f32::MAX, f32::NAN, f32::MIN, f32::NAN, -1., f32::NAN, 0., + f32::NAN, f32::MAX, f32::NAN, f32::MIN, f32::NAN, 1., f32::NAN, 2.); + #[rustfmt::skip] + let b = _mm512_set_ps(f32::NAN, f32::NAN, f32::NAN, f32::NAN, f32::MIN, f32::MAX, -1., 0., + f32::NAN, f32::NAN, f32::NAN, f32::NAN, f32::MIN, f32::MAX, -1., 2.); + let m = _mm512_cmpord_ps_mask(a, b); + assert_eq!(m, 0b00000101_00000101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpord_ps_mask() { + #[rustfmt::skip] + let a = _mm512_set_ps(f32::NAN, f32::MAX, f32::NAN, f32::MIN, f32::NAN, -1., f32::NAN, 0., + f32::NAN, f32::MAX, f32::NAN, f32::MIN, f32::NAN, 1., f32::NAN, 2.); + #[rustfmt::skip] + let b = _mm512_set_ps(f32::NAN, f32::NAN, f32::NAN, f32::NAN, f32::MIN, f32::MAX, -1., 0., + f32::NAN, f32::NAN, f32::NAN, f32::NAN, f32::MIN, f32::MAX, -1., 2.); + let mask = 0b11000011_11000011; + let m = _mm512_mask_cmpord_ps_mask(mask, a, b); + assert_eq!(m, 0b00000001_00000001); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpunord_ps_mask() { + #[rustfmt::skip] + let a = _mm512_set_ps(f32::NAN, f32::MAX, f32::NAN, f32::MIN, f32::NAN, -1., f32::NAN, 0., + f32::NAN, f32::MAX, f32::NAN, f32::MIN, f32::NAN, 1., f32::NAN, 2.); + #[rustfmt::skip] + let b = _mm512_set_ps(f32::NAN, f32::NAN, f32::NAN, f32::NAN, f32::MIN, f32::MAX, -1., 0., + f32::NAN, f32::NAN, f32::NAN, f32::NAN, f32::MIN, f32::MAX, -1., 2.); + let m = _mm512_cmpunord_ps_mask(a, b); + + assert_eq!(m, 0b11111010_11111010); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpunord_ps_mask() { + #[rustfmt::skip] + let a = _mm512_set_ps(f32::NAN, f32::MAX, f32::NAN, f32::MIN, f32::NAN, -1., f32::NAN, 0., + f32::NAN, f32::MAX, f32::NAN, f32::MIN, f32::NAN, 1., f32::NAN, 2.); + #[rustfmt::skip] + let b = _mm512_set_ps(f32::NAN, f32::NAN, f32::NAN, f32::NAN, f32::MIN, f32::MAX, -1., 0., + f32::NAN, f32::NAN, f32::NAN, f32::NAN, f32::MIN, f32::MAX, -1., 2.); + let mask = 0b00001111_00001111; + let m = _mm512_mask_cmpunord_ps_mask(mask, a, b); + assert_eq!(m, 0b000001010_00001010); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cmp_ss_mask() { + let a = _mm_setr_ps(2., 1., 1., 1.); + let b = _mm_setr_ps(1., 2., 2., 2.); + let m = _mm_cmp_ss_mask::<_CMP_GE_OS>(a, b); + assert_eq!(m, 1); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_cmp_ss_mask() { + let a = _mm_setr_ps(2., 1., 1., 1.); + let b = _mm_setr_ps(1., 2., 2., 2.); + let m = _mm_mask_cmp_ss_mask::<_CMP_GE_OS>(0b10, a, b); + assert_eq!(m, 0); + let m = _mm_mask_cmp_ss_mask::<_CMP_GE_OS>(0b1, a, b); + assert_eq!(m, 1); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cmp_round_ss_mask() { + let a = _mm_setr_ps(2., 1., 1., 1.); + let b = _mm_setr_ps(1., 2., 2., 2.); + let m = _mm_cmp_round_ss_mask::<_CMP_GE_OS, _MM_FROUND_CUR_DIRECTION>(a, b); + assert_eq!(m, 1); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_cmp_round_ss_mask() { + let a = _mm_setr_ps(2., 1., 1., 1.); + let b = _mm_setr_ps(1., 2., 2., 2.); + let m = _mm_mask_cmp_round_ss_mask::<_CMP_GE_OS, _MM_FROUND_CUR_DIRECTION>(0b10, a, b); + assert_eq!(m, 0); + let m = _mm_mask_cmp_round_ss_mask::<_CMP_GE_OS, _MM_FROUND_CUR_DIRECTION>(0b1, a, b); + assert_eq!(m, 1); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cmp_sd_mask() { + let a = _mm_setr_pd(2., 1.); + let b = _mm_setr_pd(1., 2.); + let m = _mm_cmp_sd_mask::<_CMP_GE_OS>(a, b); + assert_eq!(m, 1); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_cmp_sd_mask() { + let a = _mm_setr_pd(2., 1.); + let b = _mm_setr_pd(1., 2.); + let m = _mm_mask_cmp_sd_mask::<_CMP_GE_OS>(0b10, a, b); + assert_eq!(m, 0); + let m = _mm_mask_cmp_sd_mask::<_CMP_GE_OS>(0b1, a, b); + assert_eq!(m, 1); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cmp_round_sd_mask() { + let a = _mm_setr_pd(2., 1.); + let b = _mm_setr_pd(1., 2.); + let m = _mm_cmp_round_sd_mask::<_CMP_GE_OS, _MM_FROUND_CUR_DIRECTION>(a, b); + assert_eq!(m, 1); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_cmp_round_sd_mask() { + let a = _mm_setr_pd(2., 1.); + let b = _mm_setr_pd(1., 2.); + let m = _mm_mask_cmp_round_sd_mask::<_CMP_GE_OS, _MM_FROUND_CUR_DIRECTION>(0b10, a, b); + assert_eq!(m, 0); + let m = _mm_mask_cmp_round_sd_mask::<_CMP_GE_OS, _MM_FROUND_CUR_DIRECTION>(0b1, a, b); + assert_eq!(m, 1); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmplt_epu32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm512_set1_epi32(-1); + let m = _mm512_cmplt_epu32_mask(a, b); + assert_eq!(m, 0b11001111_11001111); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmplt_epu32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm512_set1_epi32(-1); + let mask = 0b01111010_01111010; + let r = _mm512_mask_cmplt_epu32_mask(mask, a, b); + assert_eq!(r, 0b01001010_01001010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmplt_epu32_mask() { + let a = _mm256_set_epi32(0, 1, 2, u32::MAX as i32, i32::MAX, 101, 100, 99); + let b = _mm256_set1_epi32(1); + let r = _mm256_cmplt_epu32_mask(a, b); + assert_eq!(r, 0b10000000); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmplt_epu32_mask() { + let a = _mm256_set_epi32(0, 1, 2, u32::MAX as i32, i32::MAX, 101, 100, 99); + let b = _mm256_set1_epi32(1); + let mask = 0b11111111; + let r = _mm256_mask_cmplt_epu32_mask(mask, a, b); + assert_eq!(r, 0b10000000); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmplt_epu32_mask() { + let a = _mm_set_epi32(0, 1, 2, u32::MAX as i32); + let b = _mm_set1_epi32(1); + let r = _mm_cmplt_epu32_mask(a, b); + assert_eq!(r, 0b00001000); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmplt_epu32_mask() { + let a = _mm_set_epi32(0, 1, 2, u32::MAX as i32); + let b = _mm_set1_epi32(1); + let mask = 0b11111111; + let r = _mm_mask_cmplt_epu32_mask(mask, a, b); + assert_eq!(r, 0b00001000); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpgt_epu32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm512_set1_epi32(-1); + let m = _mm512_cmpgt_epu32_mask(b, a); + assert_eq!(m, 0b11001111_11001111); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpgt_epu32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm512_set1_epi32(-1); + let mask = 0b01111010_01111010; + let r = _mm512_mask_cmpgt_epu32_mask(mask, b, a); + assert_eq!(r, 0b01001010_01001010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmpgt_epu32_mask() { + let a = _mm256_set_epi32(0, 1, 2, u32::MAX as i32, i32::MAX, 99, 100, 101); + let b = _mm256_set1_epi32(1); + let r = _mm256_cmpgt_epu32_mask(a, b); + assert_eq!(r, 0b00111111); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmpgt_epu32_mask() { + let a = _mm256_set_epi32(0, 1, 2, u32::MAX as i32, i32::MAX, 99, 100, 101); + let b = _mm256_set1_epi32(1); + let mask = 0b11111111; + let r = _mm256_mask_cmpgt_epu32_mask(mask, a, b); + assert_eq!(r, 0b00111111); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmpgt_epu32_mask() { + let a = _mm_set_epi32(0, 1, 2, u32::MAX as i32); + let b = _mm_set1_epi32(1); + let r = _mm_cmpgt_epu32_mask(a, b); + assert_eq!(r, 0b00000011); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmpgt_epu32_mask() { + let a = _mm_set_epi32(0, 1, 2, u32::MAX as i32); + let b = _mm_set1_epi32(1); + let mask = 0b11111111; + let r = _mm_mask_cmpgt_epu32_mask(mask, a, b); + assert_eq!(r, 0b00000011); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmple_epu32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm512_set1_epi32(-1); + assert_eq!( + _mm512_cmple_epu32_mask(a, b), + !_mm512_cmpgt_epu32_mask(a, b) + ) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmple_epu32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm512_set1_epi32(-1); + let mask = 0b01111010_01111010; + assert_eq!( + _mm512_mask_cmple_epu32_mask(mask, a, b), + 0b01111010_01111010 + ); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmple_epu32_mask() { + let a = _mm256_set_epi32(0, 1, 2, u32::MAX as i32, i32::MAX, 200, 100, 101); + let b = _mm256_set1_epi32(1); + let r = _mm256_cmple_epu32_mask(a, b); + assert_eq!(r, 0b11000000) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmple_epu32_mask() { + let a = _mm256_set_epi32(0, 1, 2, u32::MAX as i32, i32::MAX, 200, 100, 101); + let b = _mm256_set1_epi32(1); + let mask = 0b11111111; + let r = _mm256_mask_cmple_epu32_mask(mask, a, b); + assert_eq!(r, 0b11000000) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmple_epu32_mask() { + let a = _mm_set_epi32(0, 1, 2, u32::MAX as i32); + let b = _mm_set1_epi32(1); + let r = _mm_cmple_epu32_mask(a, b); + assert_eq!(r, 0b00001100) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmple_epu32_mask() { + let a = _mm_set_epi32(0, 1, 2, u32::MAX as i32); + let b = _mm_set1_epi32(1); + let mask = 0b11111111; + let r = _mm_mask_cmple_epu32_mask(mask, a, b); + assert_eq!(r, 0b00001100) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpge_epu32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm512_set1_epi32(-1); + assert_eq!( + _mm512_cmpge_epu32_mask(a, b), + !_mm512_cmplt_epu32_mask(a, b) + ) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpge_epu32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm512_set1_epi32(-1); + let mask = 0b01111010_01111010; + assert_eq!(_mm512_mask_cmpge_epu32_mask(mask, a, b), 0b01100000_0110000); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmpge_epu32_mask() { + let a = _mm256_set_epi32(0, 1, 2, u32::MAX as i32, i32::MAX, 300, 100, 200); + let b = _mm256_set1_epi32(1); + let r = _mm256_cmpge_epu32_mask(a, b); + assert_eq!(r, 0b01111111) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmpge_epu32_mask() { + let a = _mm256_set_epi32(0, 1, 2, u32::MAX as i32, i32::MAX, 300, 100, 200); + let b = _mm256_set1_epi32(1); + let mask = 0b11111111; + let r = _mm256_mask_cmpge_epu32_mask(mask, a, b); + assert_eq!(r, 0b01111111) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmpge_epu32_mask() { + let a = _mm_set_epi32(0, 1, 2, u32::MAX as i32); + let b = _mm_set1_epi32(1); + let r = _mm_cmpge_epu32_mask(a, b); + assert_eq!(r, 0b00000111) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmpge_epu32_mask() { + let a = _mm_set_epi32(0, 1, 2, u32::MAX as i32); + let b = _mm_set1_epi32(1); + let mask = 0b11111111; + let r = _mm_mask_cmpge_epu32_mask(mask, a, b); + assert_eq!(r, 0b00000111) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpeq_epu32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + #[rustfmt::skip] + let b = _mm512_set_epi32(0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100, + 0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100); + let m = _mm512_cmpeq_epu32_mask(b, a); + assert_eq!(m, 0b11001111_11001111); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpeq_epu32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + #[rustfmt::skip] + let b = _mm512_set_epi32(0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100, + 0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100); + let mask = 0b01111010_01111010; + let r = _mm512_mask_cmpeq_epu32_mask(mask, b, a); + assert_eq!(r, 0b01001010_01001010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmpeq_epu32_mask() { + let a = _mm256_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm256_set_epi32(0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100); + let m = _mm256_cmpeq_epu32_mask(b, a); + assert_eq!(m, 0b11001111); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmpeq_epu32_mask() { + let a = _mm256_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm256_set_epi32(0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100); + let mask = 0b01111010; + let r = _mm256_mask_cmpeq_epu32_mask(mask, b, a); + assert_eq!(r, 0b01001010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmpeq_epu32_mask() { + let a = _mm_set_epi32(0, 1, -1, u32::MAX as i32); + let b = _mm_set_epi32(0, 1, 13, 42); + let m = _mm_cmpeq_epu32_mask(b, a); + assert_eq!(m, 0b00001100); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmpeq_epu32_mask() { + let a = _mm_set_epi32(0, 1, -1, u32::MAX as i32); + let b = _mm_set_epi32(0, 1, 13, 42); + let mask = 0b11111111; + let r = _mm_mask_cmpeq_epu32_mask(mask, b, a); + assert_eq!(r, 0b00001100); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpneq_epu32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + #[rustfmt::skip] + let b = _mm512_set_epi32(0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100, + 0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100); + let m = _mm512_cmpneq_epu32_mask(b, a); + assert_eq!(m, !_mm512_cmpeq_epu32_mask(b, a)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpneq_epu32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, -100, 100, + 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, -100, 100); + #[rustfmt::skip] + let b = _mm512_set_epi32(0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100, + 0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100); + let mask = 0b01111010_01111010; + let r = _mm512_mask_cmpneq_epu32_mask(mask, b, a); + assert_eq!(r, 0b00110010_00110010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmpneq_epu32_mask() { + let a = _mm256_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, -100, 100); + let b = _mm256_set_epi32(0, 1, 13, 42, i32::MAX, i32::MIN, -100, 100); + let r = _mm256_cmpneq_epu32_mask(b, a); + assert_eq!(r, 0b00110000); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmpneq_epu32_mask() { + let a = _mm256_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, -100, 100); + let b = _mm256_set_epi32(0, 1, 13, 42, i32::MAX, i32::MIN, -100, 100); + let mask = 0b11111111; + let r = _mm256_mask_cmpneq_epu32_mask(mask, b, a); + assert_eq!(r, 0b00110000); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmpneq_epu32_mask() { + let a = _mm_set_epi32(0, 1, -1, u32::MAX as i32); + let b = _mm_set_epi32(0, 1, 13, 42); + let r = _mm_cmpneq_epu32_mask(b, a); + assert_eq!(r, 0b00000011); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmpneq_epu32_mask() { + let a = _mm_set_epi32(0, 1, -1, u32::MAX as i32); + let b = _mm_set_epi32(0, 1, 13, 42); + let mask = 0b11111111; + let r = _mm_mask_cmpneq_epu32_mask(mask, b, a); + assert_eq!(r, 0b00000011); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmp_epu32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm512_set1_epi32(-1); + let m = _mm512_cmp_epu32_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b11001111_11001111); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmp_epu32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm512_set1_epi32(-1); + let mask = 0b01111010_01111010; + let r = _mm512_mask_cmp_epu32_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b01001010_01001010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmp_epu32_mask() { + let a = _mm256_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm256_set1_epi32(-1); + let m = _mm256_cmp_epu32_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b11001111); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmp_epu32_mask() { + let a = _mm256_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm256_set1_epi32(-1); + let mask = 0b11111111; + let r = _mm256_mask_cmp_epu32_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b11001111); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmp_epu32_mask() { + let a = _mm_set_epi32(0, 1, -1, i32::MAX); + let b = _mm_set1_epi32(1); + let m = _mm_cmp_epu32_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b00001000); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmp_epu32_mask() { + let a = _mm_set_epi32(0, 1, -1, i32::MAX); + let b = _mm_set1_epi32(1); + let mask = 0b11111111; + let r = _mm_mask_cmp_epu32_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b00001000); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmplt_epi32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm512_set1_epi32(-1); + let m = _mm512_cmplt_epi32_mask(a, b); + assert_eq!(m, 0b00000101_00000101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmplt_epi32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm512_set1_epi32(-1); + let mask = 0b01100110_01100110; + let r = _mm512_mask_cmplt_epi32_mask(mask, a, b); + assert_eq!(r, 0b00000100_00000100); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmplt_epi32_mask() { + let a = _mm256_set_epi32(0, 1, -1, 101, i32::MAX, i32::MIN, 100, -100); + let b = _mm256_set1_epi32(-1); + let r = _mm256_cmplt_epi32_mask(a, b); + assert_eq!(r, 0b00000101); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmplt_epi32_mask() { + let a = _mm256_set_epi32(0, 1, -1, 101, i32::MAX, i32::MIN, 100, -100); + let b = _mm256_set1_epi32(-1); + let mask = 0b11111111; + let r = _mm256_mask_cmplt_epi32_mask(mask, a, b); + assert_eq!(r, 0b00000101); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmplt_epi32_mask() { + let a = _mm_set_epi32(i32::MAX, i32::MIN, 100, -100); + let b = _mm_set1_epi32(-1); + let r = _mm_cmplt_epi32_mask(a, b); + assert_eq!(r, 0b00000101); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmplt_epi32_mask() { + let a = _mm_set_epi32(i32::MAX, i32::MIN, 100, -100); + let b = _mm_set1_epi32(-1); + let mask = 0b11111111; + let r = _mm_mask_cmplt_epi32_mask(mask, a, b); + assert_eq!(r, 0b00000101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpgt_epi32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100); + let b = _mm512_set1_epi32(-1); + let m = _mm512_cmpgt_epi32_mask(b, a); + assert_eq!(m, 0b00000101_00000101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpgt_epi32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100); + let b = _mm512_set1_epi32(-1); + let mask = 0b01100110_01100110; + let r = _mm512_mask_cmpgt_epi32_mask(mask, b, a); + assert_eq!(r, 0b00000100_00000100); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmpgt_epi32_mask() { + let a = _mm256_set_epi32(0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100); + let b = _mm256_set1_epi32(-1); + let r = _mm256_cmpgt_epi32_mask(a, b); + assert_eq!(r, 0b11011010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmpgt_epi32_mask() { + let a = _mm256_set_epi32(0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100); + let b = _mm256_set1_epi32(-1); + let mask = 0b11111111; + let r = _mm256_mask_cmpgt_epi32_mask(mask, a, b); + assert_eq!(r, 0b11011010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmpgt_epi32_mask() { + let a = _mm_set_epi32(0, 1, -1, 13); + let b = _mm_set1_epi32(-1); + let r = _mm_cmpgt_epi32_mask(a, b); + assert_eq!(r, 0b00001101); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmpgt_epi32_mask() { + let a = _mm_set_epi32(0, 1, -1, 13); + let b = _mm_set1_epi32(-1); + let mask = 0b11111111; + let r = _mm_mask_cmpgt_epi32_mask(mask, a, b); + assert_eq!(r, 0b00001101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmple_epi32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm512_set1_epi32(-1); + assert_eq!( + _mm512_cmple_epi32_mask(a, b), + !_mm512_cmpgt_epi32_mask(a, b) + ) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmple_epi32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm512_set1_epi32(-1); + let mask = 0b01111010_01111010; + assert_eq!(_mm512_mask_cmple_epi32_mask(mask, a, b), 0b01100000_0110000); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmple_epi32_mask() { + let a = _mm256_set_epi32(0, 1, -1, 200, i32::MAX, i32::MIN, 100, -100); + let b = _mm256_set1_epi32(-1); + let r = _mm256_cmple_epi32_mask(a, b); + assert_eq!(r, 0b00100101) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmple_epi32_mask() { + let a = _mm256_set_epi32(0, 1, -1, 200, i32::MAX, i32::MIN, 100, -100); + let b = _mm256_set1_epi32(-1); + let mask = 0b11111111; + let r = _mm256_mask_cmple_epi32_mask(mask, a, b); + assert_eq!(r, 0b00100101) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmple_epi32_mask() { + let a = _mm_set_epi32(0, 1, -1, 200); + let b = _mm_set1_epi32(-1); + let r = _mm_cmple_epi32_mask(a, b); + assert_eq!(r, 0b00000010) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmple_epi32_mask() { + let a = _mm_set_epi32(0, 1, -1, 200); + let b = _mm_set1_epi32(-1); + let mask = 0b11111111; + let r = _mm_mask_cmple_epi32_mask(mask, a, b); + assert_eq!(r, 0b00000010) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpge_epi32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm512_set1_epi32(-1); + assert_eq!( + _mm512_cmpge_epi32_mask(a, b), + !_mm512_cmplt_epi32_mask(a, b) + ) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpge_epi32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm512_set1_epi32(-1); + let mask = 0b01111010_01111010; + assert_eq!( + _mm512_mask_cmpge_epi32_mask(mask, a, b), + 0b01111010_01111010 + ); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmpge_epi32_mask() { + let a = _mm256_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm256_set1_epi32(-1); + let r = _mm256_cmpge_epi32_mask(a, b); + assert_eq!(r, 0b11111010) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmpge_epi32_mask() { + let a = _mm256_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); + let b = _mm256_set1_epi32(-1); + let mask = 0b11111111; + let r = _mm256_mask_cmpge_epi32_mask(mask, a, b); + assert_eq!(r, 0b11111010) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmpge_epi32_mask() { + let a = _mm_set_epi32(0, 1, -1, u32::MAX as i32); + let b = _mm_set1_epi32(-1); + let r = _mm_cmpge_epi32_mask(a, b); + assert_eq!(r, 0b00001111) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmpge_epi32_mask() { + let a = _mm_set_epi32(0, 1, -1, u32::MAX as i32); + let b = _mm_set1_epi32(-1); + let mask = 0b11111111; + let r = _mm_mask_cmpge_epi32_mask(mask, a, b); + assert_eq!(r, 0b00001111) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpeq_epi32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100); + #[rustfmt::skip] + let b = _mm512_set_epi32(0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100, + 0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100); + let m = _mm512_cmpeq_epi32_mask(b, a); + assert_eq!(m, 0b11001111_11001111); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpeq_epi32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100); + #[rustfmt::skip] + let b = _mm512_set_epi32(0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100, + 0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100); + let mask = 0b01111010_01111010; + let r = _mm512_mask_cmpeq_epi32_mask(mask, b, a); + assert_eq!(r, 0b01001010_01001010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmpeq_epi32_mask() { + let a = _mm256_set_epi32(0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100); + let b = _mm256_set_epi32(0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100); + let m = _mm256_cmpeq_epi32_mask(b, a); + assert_eq!(m, 0b11001111); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmpeq_epi32_mask() { + let a = _mm256_set_epi32(0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100); + let b = _mm256_set_epi32(0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100); + let mask = 0b01111010; + let r = _mm256_mask_cmpeq_epi32_mask(mask, b, a); + assert_eq!(r, 0b01001010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmpeq_epi32_mask() { + let a = _mm_set_epi32(0, 1, -1, 13); + let b = _mm_set_epi32(0, 1, 13, 42); + let m = _mm_cmpeq_epi32_mask(b, a); + assert_eq!(m, 0b00001100); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmpeq_epi32_mask() { + let a = _mm_set_epi32(0, 1, -1, 13); + let b = _mm_set_epi32(0, 1, 13, 42); + let mask = 0b11111111; + let r = _mm_mask_cmpeq_epi32_mask(mask, b, a); + assert_eq!(r, 0b00001100); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpneq_epi32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100); + #[rustfmt::skip] + let b = _mm512_set_epi32(0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100, + 0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100); + let m = _mm512_cmpneq_epi32_mask(b, a); + assert_eq!(m, !_mm512_cmpeq_epi32_mask(b, a)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpneq_epi32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, 13, i32::MAX, i32::MIN, -100, 100, + 0, 1, -1, 13, i32::MAX, i32::MIN, -100, 100); + #[rustfmt::skip] + let b = _mm512_set_epi32(0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100, + 0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100); + let mask = 0b01111010_01111010; + let r = _mm512_mask_cmpneq_epi32_mask(mask, b, a); + assert_eq!(r, 0b00110010_00110010) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmpneq_epi32_mask() { + let a = _mm256_set_epi32(0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100); + let b = _mm256_set_epi32(0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100); + let m = _mm256_cmpneq_epi32_mask(b, a); + assert_eq!(m, !_mm256_cmpeq_epi32_mask(b, a)); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmpneq_epi32_mask() { + let a = _mm256_set_epi32(0, 1, -1, 13, i32::MAX, i32::MIN, -100, 100); + let b = _mm256_set_epi32(0, 1, 13, 42, i32::MAX, i32::MIN, 100, -100); + let mask = 0b11111111; + let r = _mm256_mask_cmpneq_epi32_mask(mask, b, a); + assert_eq!(r, 0b00110011) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmpneq_epi32_mask() { + let a = _mm_set_epi32(0, 1, -1, 13); + let b = _mm_set_epi32(0, 1, 13, 42); + let r = _mm_cmpneq_epi32_mask(b, a); + assert_eq!(r, 0b00000011) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmpneq_epi32_mask() { + let a = _mm_set_epi32(0, 1, -1, 13); + let b = _mm_set_epi32(0, 1, 13, 42); + let mask = 0b11111111; + let r = _mm_mask_cmpneq_epi32_mask(mask, b, a); + assert_eq!(r, 0b00000011) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmp_epi32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100); + let b = _mm512_set1_epi32(-1); + let m = _mm512_cmp_epi32_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b00000101_00000101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmp_epi32_mask() { + #[rustfmt::skip] + let a = _mm512_set_epi32(0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100, + 0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100); + let b = _mm512_set1_epi32(-1); + let mask = 0b01100110_01100110; + let r = _mm512_mask_cmp_epi32_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b00000100_00000100); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmp_epi32_mask() { + let a = _mm256_set_epi32(0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100); + let b = _mm256_set1_epi32(-1); + let m = _mm256_cmp_epi32_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b00000101); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmp_epi32_mask() { + let a = _mm256_set_epi32(0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100); + let b = _mm256_set1_epi32(-1); + let mask = 0b01100110; + let r = _mm256_mask_cmp_epi32_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b00000100); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmp_epi32_mask() { + let a = _mm_set_epi32(0, 1, -1, 13); + let b = _mm_set1_epi32(1); + let m = _mm_cmp_epi32_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b00001010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmp_epi32_mask() { + let a = _mm_set_epi32(0, 1, -1, 13); + let b = _mm_set1_epi32(1); + let mask = 0b11111111; + let r = _mm_mask_cmp_epi32_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b00001010); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_set_epi8() { + let r = _mm512_set1_epi8(2); + assert_eq_m512i( + r, + _mm512_set_epi8( + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, + ), + ) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_set_epi16() { + let r = _mm512_set1_epi16(2); + assert_eq_m512i( + r, + _mm512_set_epi16( + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, + ), + ) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_set_epi32() { + let r = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i( + r, + _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), + ) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_setr_epi32() { + let r = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i( + r, + _mm512_setr_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), + ) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_set1_epi8() { + let r = _mm512_set_epi8( + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, + ); + assert_eq_m512i(r, _mm512_set1_epi8(2)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_set1_epi16() { + let r = _mm512_set_epi16( + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, + ); + assert_eq_m512i(r, _mm512_set1_epi16(2)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_set1_epi32() { + let r = _mm512_set_epi32(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m512i(r, _mm512_set1_epi32(2)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_setzero_si512() { + assert_eq_m512i(_mm512_set1_epi32(0), _mm512_setzero_si512()); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_setzero_epi32() { + assert_eq_m512i(_mm512_set1_epi32(0), _mm512_setzero_epi32()); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_set_ps() { + let r = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + assert_eq_m512( + r, + _mm512_set_ps( + 15., 14., 13., 12., 11., 10., 9., 8., 7., 6., 5., 4., 3., 2., 1., 0., + ), + ) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_setr_ps() { + let r = _mm512_set_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + assert_eq_m512( + r, + _mm512_setr_ps( + 15., 14., 13., 12., 11., 10., 9., 8., 7., 6., 5., 4., 3., 2., 1., 0., + ), + ) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_set1_ps() { + #[rustfmt::skip] + let expected = _mm512_set_ps(2., 2., 2., 2., 2., 2., 2., 2., + 2., 2., 2., 2., 2., 2., 2., 2.); + assert_eq_m512(expected, _mm512_set1_ps(2.)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_set4_epi32() { + let r = _mm512_set_epi32(4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1); + assert_eq_m512i(r, _mm512_set4_epi32(4, 3, 2, 1)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_set4_ps() { + let r = _mm512_set_ps( + 4., 3., 2., 1., 4., 3., 2., 1., 4., 3., 2., 1., 4., 3., 2., 1., + ); + assert_eq_m512(r, _mm512_set4_ps(4., 3., 2., 1.)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_setr4_epi32() { + let r = _mm512_set_epi32(4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1); + assert_eq_m512i(r, _mm512_setr4_epi32(1, 2, 3, 4)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_setr4_ps() { + let r = _mm512_set_ps( + 4., 3., 2., 1., 4., 3., 2., 1., 4., 3., 2., 1., 4., 3., 2., 1., + ); + assert_eq_m512(r, _mm512_setr4_ps(1., 2., 3., 4.)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_setzero_ps() { + assert_eq_m512(_mm512_setzero_ps(), _mm512_set1_ps(0.)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_setzero() { + assert_eq_m512(_mm512_setzero(), _mm512_set1_ps(0.)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_loadu_pd() { + let a = &[4., 3., 2., 5., 8., 9., 64., 50.]; + let p = a.as_ptr(); + let r = _mm512_loadu_pd(black_box(p)); + let e = _mm512_setr_pd(4., 3., 2., 5., 8., 9., 64., 50.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_storeu_pd() { + let a = _mm512_set1_pd(9.); + let mut r = _mm512_undefined_pd(); + _mm512_storeu_pd(&mut r as *mut _ as *mut f64, a); + assert_eq_m512d(r, a); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_loadu_ps() { + let a = &[ + 4., 3., 2., 5., 8., 9., 64., 50., -4., -3., -2., -5., -8., -9., -64., -50., + ]; + let p = a.as_ptr(); + let r = _mm512_loadu_ps(black_box(p)); + let e = _mm512_setr_ps( + 4., 3., 2., 5., 8., 9., 64., 50., -4., -3., -2., -5., -8., -9., -64., -50., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_storeu_ps() { + let a = _mm512_set1_ps(9.); + let mut r = _mm512_undefined_ps(); + _mm512_storeu_ps(&mut r as *mut _ as *mut f32, a); + assert_eq_m512(r, a); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_loadu_epi32() { + let src = _mm512_set1_epi32(42); + let a = &[1_i32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let p = a.as_ptr(); + let m = 0b11101000_11001010; + let r = _mm512_mask_loadu_epi32(src, m, black_box(p)); + let e = _mm512_setr_epi32(42, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_loadu_epi32() { + let a = &[1_i32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let p = a.as_ptr(); + let m = 0b11101000_11001010; + let r = _mm512_maskz_loadu_epi32(m, black_box(p)); + let e = _mm512_setr_epi32(0, 2, 0, 4, 0, 0, 7, 8, 0, 0, 0, 12, 0, 14, 15, 16); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_load_epi32() { + #[repr(align(64))] + struct Align { + data: [i32; 16], // 64 bytes + } + let src = _mm512_set1_epi32(42); + let a = Align { + data: [1_i32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + }; + let p = a.data.as_ptr(); + let m = 0b11101000_11001010; + let r = _mm512_mask_load_epi32(src, m, black_box(p)); + let e = _mm512_setr_epi32(42, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_load_epi32() { + #[repr(align(64))] + struct Align { + data: [i32; 16], // 64 bytes + } + let a = Align { + data: [1_i32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + }; + let p = a.data.as_ptr(); + let m = 0b11101000_11001010; + let r = _mm512_maskz_load_epi32(m, black_box(p)); + let e = _mm512_setr_epi32(0, 2, 0, 4, 0, 0, 7, 8, 0, 0, 0, 12, 0, 14, 15, 16); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_storeu_epi32() { + let mut r = [42_i32; 16]; + let a = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let m = 0b11101000_11001010; + _mm512_mask_storeu_epi32(r.as_mut_ptr(), m, a); + let e = _mm512_setr_epi32(42, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16); + assert_eq_m512i(_mm512_loadu_epi32(r.as_ptr()), e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_store_epi32() { + #[repr(align(64))] + struct Align { + data: [i32; 16], + } + let mut r = Align { data: [42; 16] }; + let a = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let m = 0b11101000_11001010; + _mm512_mask_store_epi32(r.data.as_mut_ptr(), m, a); + let e = _mm512_setr_epi32(42, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16); + assert_eq_m512i(_mm512_load_epi32(r.data.as_ptr()), e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_loadu_epi64() { + let src = _mm512_set1_epi64(42); + let a = &[1_i64, 2, 3, 4, 5, 6, 7, 8]; + let p = a.as_ptr(); + let m = 0b11001010; + let r = _mm512_mask_loadu_epi64(src, m, black_box(p)); + let e = _mm512_setr_epi64(42, 2, 42, 4, 42, 42, 7, 8); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_loadu_epi64() { + let a = &[1_i64, 2, 3, 4, 5, 6, 7, 8]; + let p = a.as_ptr(); + let m = 0b11001010; + let r = _mm512_maskz_loadu_epi64(m, black_box(p)); + let e = _mm512_setr_epi64(0, 2, 0, 4, 0, 0, 7, 8); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_load_epi64() { + #[repr(align(64))] + struct Align { + data: [i64; 8], // 64 bytes + } + let src = _mm512_set1_epi64(42); + let a = Align { + data: [1_i64, 2, 3, 4, 5, 6, 7, 8], + }; + let p = a.data.as_ptr(); + let m = 0b11001010; + let r = _mm512_mask_load_epi64(src, m, black_box(p)); + let e = _mm512_setr_epi64(42, 2, 42, 4, 42, 42, 7, 8); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_load_epi64() { + #[repr(align(64))] + struct Align { + data: [i64; 8], // 64 bytes + } + let a = Align { + data: [1_i64, 2, 3, 4, 5, 6, 7, 8], + }; + let p = a.data.as_ptr(); + let m = 0b11001010; + let r = _mm512_maskz_load_epi64(m, black_box(p)); + let e = _mm512_setr_epi64(0, 2, 0, 4, 0, 0, 7, 8); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_storeu_epi64() { + let mut r = [42_i64; 8]; + let a = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); + let m = 0b11001010; + _mm512_mask_storeu_epi64(r.as_mut_ptr(), m, a); + let e = _mm512_setr_epi64(42, 2, 42, 4, 42, 42, 7, 8); + assert_eq_m512i(_mm512_loadu_epi64(r.as_ptr()), e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_store_epi64() { + #[repr(align(64))] + struct Align { + data: [i64; 8], + } + let mut r = Align { data: [42; 8] }; + let a = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); + let m = 0b11001010; + let p = r.data.as_mut_ptr(); + _mm512_mask_store_epi64(p, m, a); + let e = _mm512_setr_epi64(42, 2, 42, 4, 42, 42, 7, 8); + assert_eq_m512i(_mm512_load_epi64(r.data.as_ptr()), e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_loadu_ps() { + let src = _mm512_set1_ps(42.0); + let a = &[ + 1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, + 16.0, + ]; + let p = a.as_ptr(); + let m = 0b11101000_11001010; + let r = _mm512_mask_loadu_ps(src, m, black_box(p)); + let e = _mm512_setr_ps( + 42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0, 42.0, 42.0, 42.0, 12.0, 42.0, 14.0, 15.0, + 16.0, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_loadu_ps() { + let a = &[ + 1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, + 16.0, + ]; + let p = a.as_ptr(); + let m = 0b11101000_11001010; + let r = _mm512_maskz_loadu_ps(m, black_box(p)); + let e = _mm512_setr_ps( + 0.0, 2.0, 0.0, 4.0, 0.0, 0.0, 7.0, 8.0, 0.0, 0.0, 0.0, 12.0, 0.0, 14.0, 15.0, 16.0, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_load_ps() { + #[repr(align(64))] + struct Align { + data: [f32; 16], // 64 bytes + } + let src = _mm512_set1_ps(42.0); + let a = Align { + data: [ + 1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, + 15.0, 16.0, + ], + }; + let p = a.data.as_ptr(); + let m = 0b11101000_11001010; + let r = _mm512_mask_load_ps(src, m, black_box(p)); + let e = _mm512_setr_ps( + 42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0, 42.0, 42.0, 42.0, 12.0, 42.0, 14.0, 15.0, + 16.0, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_load_ps() { + #[repr(align(64))] + struct Align { + data: [f32; 16], // 64 bytes + } + let a = Align { + data: [ + 1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, + 15.0, 16.0, + ], + }; + let p = a.data.as_ptr(); + let m = 0b11101000_11001010; + let r = _mm512_maskz_load_ps(m, black_box(p)); + let e = _mm512_setr_ps( + 0.0, 2.0, 0.0, 4.0, 0.0, 0.0, 7.0, 8.0, 0.0, 0.0, 0.0, 12.0, 0.0, 14.0, 15.0, 16.0, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_storeu_ps() { + let mut r = [42_f32; 16]; + let a = _mm512_setr_ps( + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, + ); + let m = 0b11101000_11001010; + _mm512_mask_storeu_ps(r.as_mut_ptr(), m, a); + let e = _mm512_setr_ps( + 42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0, 42.0, 42.0, 42.0, 12.0, 42.0, 14.0, 15.0, + 16.0, + ); + assert_eq_m512(_mm512_loadu_ps(r.as_ptr()), e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_store_ps() { + #[repr(align(64))] + struct Align { + data: [f32; 16], + } + let mut r = Align { data: [42.0; 16] }; + let a = _mm512_setr_ps( + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, + ); + let m = 0b11101000_11001010; + _mm512_mask_store_ps(r.data.as_mut_ptr(), m, a); + let e = _mm512_setr_ps( + 42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0, 42.0, 42.0, 42.0, 12.0, 42.0, 14.0, 15.0, + 16.0, + ); + assert_eq_m512(_mm512_load_ps(r.data.as_ptr()), e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_loadu_pd() { + let src = _mm512_set1_pd(42.0); + let a = &[1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; + let p = a.as_ptr(); + let m = 0b11001010; + let r = _mm512_mask_loadu_pd(src, m, black_box(p)); + let e = _mm512_setr_pd(42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_loadu_pd() { + let a = &[1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; + let p = a.as_ptr(); + let m = 0b11001010; + let r = _mm512_maskz_loadu_pd(m, black_box(p)); + let e = _mm512_setr_pd(0.0, 2.0, 0.0, 4.0, 0.0, 0.0, 7.0, 8.0); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_load_pd() { + #[repr(align(64))] + struct Align { + data: [f64; 8], // 64 bytes + } + let src = _mm512_set1_pd(42.0); + let a = Align { + data: [1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0], + }; + let p = a.data.as_ptr(); + let m = 0b11001010; + let r = _mm512_mask_load_pd(src, m, black_box(p)); + let e = _mm512_setr_pd(42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_load_pd() { + #[repr(align(64))] + struct Align { + data: [f64; 8], // 64 bytes + } + let a = Align { + data: [1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0], + }; + let p = a.data.as_ptr(); + let m = 0b11001010; + let r = _mm512_maskz_load_pd(m, black_box(p)); + let e = _mm512_setr_pd(0.0, 2.0, 0.0, 4.0, 0.0, 0.0, 7.0, 8.0); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_storeu_pd() { + let mut r = [42_f64; 8]; + let a = _mm512_setr_pd(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0); + let m = 0b11001010; + _mm512_mask_storeu_pd(r.as_mut_ptr(), m, a); + let e = _mm512_setr_pd(42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0); + assert_eq_m512d(_mm512_loadu_pd(r.as_ptr()), e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_store_pd() { + #[repr(align(64))] + struct Align { + data: [f64; 8], + } + let mut r = Align { data: [42.0; 8] }; + let a = _mm512_setr_pd(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0); + let m = 0b11001010; + _mm512_mask_store_pd(r.data.as_mut_ptr(), m, a); + let e = _mm512_setr_pd(42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0); + assert_eq_m512d(_mm512_load_pd(r.data.as_ptr()), e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_loadu_epi32() { + let src = _mm256_set1_epi32(42); + let a = &[1_i32, 2, 3, 4, 5, 6, 7, 8]; + let p = a.as_ptr(); + let m = 0b11001010; + let r = _mm256_mask_loadu_epi32(src, m, black_box(p)); + let e = _mm256_setr_epi32(42, 2, 42, 4, 42, 42, 7, 8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_loadu_epi32() { + let a = &[1_i32, 2, 3, 4, 5, 6, 7, 8]; + let p = a.as_ptr(); + let m = 0b11001010; + let r = _mm256_maskz_loadu_epi32(m, black_box(p)); + let e = _mm256_setr_epi32(0, 2, 0, 4, 0, 0, 7, 8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_load_epi32() { + #[repr(align(32))] + struct Align { + data: [i32; 8], // 32 bytes + } + let src = _mm256_set1_epi32(42); + let a = Align { + data: [1_i32, 2, 3, 4, 5, 6, 7, 8], + }; + let p = a.data.as_ptr(); + let m = 0b11001010; + let r = _mm256_mask_load_epi32(src, m, black_box(p)); + let e = _mm256_setr_epi32(42, 2, 42, 4, 42, 42, 7, 8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_load_epi32() { + #[repr(align(32))] + struct Align { + data: [i32; 8], // 32 bytes + } + let a = Align { + data: [1_i32, 2, 3, 4, 5, 6, 7, 8], + }; + let p = a.data.as_ptr(); + let m = 0b11001010; + let r = _mm256_maskz_load_epi32(m, black_box(p)); + let e = _mm256_setr_epi32(0, 2, 0, 4, 0, 0, 7, 8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_storeu_epi32() { + let mut r = [42_i32; 8]; + let a = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let m = 0b11001010; + _mm256_mask_storeu_epi32(r.as_mut_ptr(), m, a); + let e = _mm256_setr_epi32(42, 2, 42, 4, 42, 42, 7, 8); + assert_eq_m256i(_mm256_loadu_epi32(r.as_ptr()), e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_store_epi32() { + #[repr(align(64))] + struct Align { + data: [i32; 8], + } + let mut r = Align { data: [42; 8] }; + let a = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let m = 0b11001010; + _mm256_mask_store_epi32(r.data.as_mut_ptr(), m, a); + let e = _mm256_setr_epi32(42, 2, 42, 4, 42, 42, 7, 8); + assert_eq_m256i(_mm256_load_epi32(r.data.as_ptr()), e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_loadu_epi64() { + let src = _mm256_set1_epi64x(42); + let a = &[1_i64, 2, 3, 4]; + let p = a.as_ptr(); + let m = 0b1010; + let r = _mm256_mask_loadu_epi64(src, m, black_box(p)); + let e = _mm256_setr_epi64x(42, 2, 42, 4); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_loadu_epi64() { + let a = &[1_i64, 2, 3, 4]; + let p = a.as_ptr(); + let m = 0b1010; + let r = _mm256_maskz_loadu_epi64(m, black_box(p)); + let e = _mm256_setr_epi64x(0, 2, 0, 4); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_load_epi64() { + #[repr(align(32))] + struct Align { + data: [i64; 4], // 32 bytes + } + let src = _mm256_set1_epi64x(42); + let a = Align { + data: [1_i64, 2, 3, 4], + }; + let p = a.data.as_ptr(); + let m = 0b1010; + let r = _mm256_mask_load_epi64(src, m, black_box(p)); + let e = _mm256_setr_epi64x(42, 2, 42, 4); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_load_epi64() { + #[repr(align(32))] + struct Align { + data: [i64; 4], // 32 bytes + } + let a = Align { + data: [1_i64, 2, 3, 4], + }; + let p = a.data.as_ptr(); + let m = 0b1010; + let r = _mm256_maskz_load_epi64(m, black_box(p)); + let e = _mm256_setr_epi64x(0, 2, 0, 4); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_storeu_epi64() { + let mut r = [42_i64; 4]; + let a = _mm256_setr_epi64x(1, 2, 3, 4); + let m = 0b1010; + _mm256_mask_storeu_epi64(r.as_mut_ptr(), m, a); + let e = _mm256_setr_epi64x(42, 2, 42, 4); + assert_eq_m256i(_mm256_loadu_epi64(r.as_ptr()), e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_store_epi64() { + #[repr(align(32))] + struct Align { + data: [i64; 4], + } + let mut r = Align { data: [42; 4] }; + let a = _mm256_setr_epi64x(1, 2, 3, 4); + let m = 0b1010; + _mm256_mask_store_epi64(r.data.as_mut_ptr(), m, a); + let e = _mm256_setr_epi64x(42, 2, 42, 4); + assert_eq_m256i(_mm256_load_epi64(r.data.as_ptr()), e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_loadu_ps() { + let src = _mm256_set1_ps(42.0); + let a = &[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; + let p = a.as_ptr(); + let m = 0b11001010; + let r = _mm256_mask_loadu_ps(src, m, black_box(p)); + let e = _mm256_setr_ps(42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_loadu_ps() { + let a = &[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; + let p = a.as_ptr(); + let m = 0b11001010; + let r = _mm256_maskz_loadu_ps(m, black_box(p)); + let e = _mm256_setr_ps(0.0, 2.0, 0.0, 4.0, 0.0, 0.0, 7.0, 8.0); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_load_ps() { + #[repr(align(32))] + struct Align { + data: [f32; 8], // 32 bytes + } + let src = _mm256_set1_ps(42.0); + let a = Align { + data: [1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0], + }; + let p = a.data.as_ptr(); + let m = 0b11001010; + let r = _mm256_mask_load_ps(src, m, black_box(p)); + let e = _mm256_setr_ps(42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_load_ps() { + #[repr(align(32))] + struct Align { + data: [f32; 8], // 32 bytes + } + let a = Align { + data: [1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0], + }; + let p = a.data.as_ptr(); + let m = 0b11001010; + let r = _mm256_maskz_load_ps(m, black_box(p)); + let e = _mm256_setr_ps(0.0, 2.0, 0.0, 4.0, 0.0, 0.0, 7.0, 8.0); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_storeu_ps() { + let mut r = [42_f32; 8]; + let a = _mm256_setr_ps(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0); + let m = 0b11001010; + _mm256_mask_storeu_ps(r.as_mut_ptr(), m, a); + let e = _mm256_setr_ps(42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0); + assert_eq_m256(_mm256_loadu_ps(r.as_ptr()), e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_store_ps() { + #[repr(align(32))] + struct Align { + data: [f32; 8], + } + let mut r = Align { data: [42.0; 8] }; + let a = _mm256_setr_ps(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0); + let m = 0b11001010; + _mm256_mask_store_ps(r.data.as_mut_ptr(), m, a); + let e = _mm256_setr_ps(42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0); + assert_eq_m256(_mm256_load_ps(r.data.as_ptr()), e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_loadu_pd() { + let src = _mm256_set1_pd(42.0); + let a = &[1.0_f64, 2.0, 3.0, 4.0]; + let p = a.as_ptr(); + let m = 0b1010; + let r = _mm256_mask_loadu_pd(src, m, black_box(p)); + let e = _mm256_setr_pd(42.0, 2.0, 42.0, 4.0); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_loadu_pd() { + let a = &[1.0_f64, 2.0, 3.0, 4.0]; + let p = a.as_ptr(); + let m = 0b1010; + let r = _mm256_maskz_loadu_pd(m, black_box(p)); + let e = _mm256_setr_pd(0.0, 2.0, 0.0, 4.0); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_load_pd() { + #[repr(align(32))] + struct Align { + data: [f64; 4], // 32 bytes + } + let src = _mm256_set1_pd(42.0); + let a = Align { + data: [1.0_f64, 2.0, 3.0, 4.0], + }; + let p = a.data.as_ptr(); + let m = 0b1010; + let r = _mm256_mask_load_pd(src, m, black_box(p)); + let e = _mm256_setr_pd(42.0, 2.0, 42.0, 4.0); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_load_pd() { + #[repr(align(32))] + struct Align { + data: [f64; 4], // 32 bytes + } + let a = Align { + data: [1.0_f64, 2.0, 3.0, 4.0], + }; + let p = a.data.as_ptr(); + let m = 0b1010; + let r = _mm256_maskz_load_pd(m, black_box(p)); + let e = _mm256_setr_pd(0.0, 2.0, 0.0, 4.0); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_storeu_pd() { + let mut r = [42_f64; 4]; + let a = _mm256_setr_pd(1.0, 2.0, 3.0, 4.0); + let m = 0b1010; + _mm256_mask_storeu_pd(r.as_mut_ptr(), m, a); + let e = _mm256_setr_pd(42.0, 2.0, 42.0, 4.0); + assert_eq_m256d(_mm256_loadu_pd(r.as_ptr()), e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_store_pd() { + #[repr(align(32))] + struct Align { + data: [f64; 4], + } + let mut r = Align { data: [42.0; 4] }; + let a = _mm256_setr_pd(1.0, 2.0, 3.0, 4.0); + let m = 0b1010; + _mm256_mask_store_pd(r.data.as_mut_ptr(), m, a); + let e = _mm256_setr_pd(42.0, 2.0, 42.0, 4.0); + assert_eq_m256d(_mm256_load_pd(r.data.as_ptr()), e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_loadu_epi32() { + let src = _mm_set1_epi32(42); + let a = &[1_i32, 2, 3, 4]; + let p = a.as_ptr(); + let m = 0b1010; + let r = _mm_mask_loadu_epi32(src, m, black_box(p)); + let e = _mm_setr_epi32(42, 2, 42, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_loadu_epi32() { + let a = &[1_i32, 2, 3, 4]; + let p = a.as_ptr(); + let m = 0b1010; + let r = _mm_maskz_loadu_epi32(m, black_box(p)); + let e = _mm_setr_epi32(0, 2, 0, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_load_epi32() { + #[repr(align(16))] + struct Align { + data: [i32; 4], // 32 bytes + } + let src = _mm_set1_epi32(42); + let a = Align { + data: [1_i32, 2, 3, 4], + }; + let p = a.data.as_ptr(); + let m = 0b1010; + let r = _mm_mask_load_epi32(src, m, black_box(p)); + let e = _mm_setr_epi32(42, 2, 42, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_load_epi32() { + #[repr(align(16))] + struct Align { + data: [i32; 4], // 16 bytes + } + let a = Align { + data: [1_i32, 2, 3, 4], + }; + let p = a.data.as_ptr(); + let m = 0b1010; + let r = _mm_maskz_load_epi32(m, black_box(p)); + let e = _mm_setr_epi32(0, 2, 0, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_storeu_epi32() { + let mut r = [42_i32; 4]; + let a = _mm_setr_epi32(1, 2, 3, 4); + let m = 0b1010; + _mm_mask_storeu_epi32(r.as_mut_ptr(), m, a); + let e = _mm_setr_epi32(42, 2, 42, 4); + assert_eq_m128i(_mm_loadu_epi32(r.as_ptr()), e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_store_epi32() { + #[repr(align(16))] + struct Align { + data: [i32; 4], // 16 bytes + } + let mut r = Align { data: [42; 4] }; + let a = _mm_setr_epi32(1, 2, 3, 4); + let m = 0b1010; + _mm_mask_store_epi32(r.data.as_mut_ptr(), m, a); + let e = _mm_setr_epi32(42, 2, 42, 4); + assert_eq_m128i(_mm_load_epi32(r.data.as_ptr()), e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_loadu_epi64() { + let src = _mm_set1_epi64x(42); + let a = &[1_i64, 2]; + let p = a.as_ptr(); + let m = 0b10; + let r = _mm_mask_loadu_epi64(src, m, black_box(p)); + let e = _mm_setr_epi64x(42, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_loadu_epi64() { + let a = &[1_i64, 2]; + let p = a.as_ptr(); + let m = 0b10; + let r = _mm_maskz_loadu_epi64(m, black_box(p)); + let e = _mm_setr_epi64x(0, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_load_epi64() { + #[repr(align(16))] + struct Align { + data: [i64; 2], // 16 bytes + } + let src = _mm_set1_epi64x(42); + let a = Align { data: [1_i64, 2] }; + let p = a.data.as_ptr(); + let m = 0b10; + let r = _mm_mask_load_epi64(src, m, black_box(p)); + let e = _mm_setr_epi64x(42, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_load_epi64() { + #[repr(align(16))] + struct Align { + data: [i64; 2], // 16 bytes + } + let a = Align { data: [1_i64, 2] }; + let p = a.data.as_ptr(); + let m = 0b10; + let r = _mm_maskz_load_epi64(m, black_box(p)); + let e = _mm_setr_epi64x(0, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_storeu_epi64() { + let mut r = [42_i64; 2]; + let a = _mm_setr_epi64x(1, 2); + let m = 0b10; + _mm_mask_storeu_epi64(r.as_mut_ptr(), m, a); + let e = _mm_setr_epi64x(42, 2); + assert_eq_m128i(_mm_loadu_epi64(r.as_ptr()), e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_store_epi64() { + #[repr(align(16))] + struct Align { + data: [i64; 2], // 16 bytes + } + let mut r = Align { data: [42; 2] }; + let a = _mm_setr_epi64x(1, 2); + let m = 0b10; + _mm_mask_store_epi64(r.data.as_mut_ptr(), m, a); + let e = _mm_setr_epi64x(42, 2); + assert_eq_m128i(_mm_load_epi64(r.data.as_ptr()), e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_loadu_ps() { + let src = _mm_set1_ps(42.0); + let a = &[1.0_f32, 2.0, 3.0, 4.0]; + let p = a.as_ptr(); + let m = 0b1010; + let r = _mm_mask_loadu_ps(src, m, black_box(p)); + let e = _mm_setr_ps(42.0, 2.0, 42.0, 4.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_loadu_ps() { + let a = &[1.0_f32, 2.0, 3.0, 4.0]; + let p = a.as_ptr(); + let m = 0b1010; + let r = _mm_maskz_loadu_ps(m, black_box(p)); + let e = _mm_setr_ps(0.0, 2.0, 0.0, 4.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_load_ps() { + #[repr(align(16))] + struct Align { + data: [f32; 4], // 16 bytes + } + let src = _mm_set1_ps(42.0); + let a = Align { + data: [1.0_f32, 2.0, 3.0, 4.0], + }; + let p = a.data.as_ptr(); + let m = 0b1010; + let r = _mm_mask_load_ps(src, m, black_box(p)); + let e = _mm_setr_ps(42.0, 2.0, 42.0, 4.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_load_ps() { + #[repr(align(16))] + struct Align { + data: [f32; 4], // 16 bytes + } + let a = Align { + data: [1.0_f32, 2.0, 3.0, 4.0], + }; + let p = a.data.as_ptr(); + let m = 0b1010; + let r = _mm_maskz_load_ps(m, black_box(p)); + let e = _mm_setr_ps(0.0, 2.0, 0.0, 4.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_storeu_ps() { + let mut r = [42_f32; 4]; + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let m = 0b1010; + _mm_mask_storeu_ps(r.as_mut_ptr(), m, a); + let e = _mm_setr_ps(42.0, 2.0, 42.0, 4.0); + assert_eq_m128(_mm_loadu_ps(r.as_ptr()), e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_store_ps() { + #[repr(align(16))] + struct Align { + data: [f32; 4], // 16 bytes + } + let mut r = Align { data: [42.0; 4] }; + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let m = 0b1010; + _mm_mask_store_ps(r.data.as_mut_ptr(), m, a); + let e = _mm_setr_ps(42.0, 2.0, 42.0, 4.0); + assert_eq_m128(_mm_load_ps(r.data.as_ptr()), e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_loadu_pd() { + let src = _mm_set1_pd(42.0); + let a = &[1.0_f64, 2.0]; + let p = a.as_ptr(); + let m = 0b10; + let r = _mm_mask_loadu_pd(src, m, black_box(p)); + let e = _mm_setr_pd(42.0, 2.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_loadu_pd() { + let a = &[1.0_f64, 2.0]; + let p = a.as_ptr(); + let m = 0b10; + let r = _mm_maskz_loadu_pd(m, black_box(p)); + let e = _mm_setr_pd(0.0, 2.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_load_pd() { + #[repr(align(16))] + struct Align { + data: [f64; 2], // 16 bytes + } + let src = _mm_set1_pd(42.0); + let a = Align { + data: [1.0_f64, 2.0], + }; + let p = a.data.as_ptr(); + let m = 0b10; + let r = _mm_mask_load_pd(src, m, black_box(p)); + let e = _mm_setr_pd(42.0, 2.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_load_pd() { + #[repr(align(16))] + struct Align { + data: [f64; 2], // 16 bytes + } + let a = Align { + data: [1.0_f64, 2.0], + }; + let p = a.data.as_ptr(); + let m = 0b10; + let r = _mm_maskz_load_pd(m, black_box(p)); + let e = _mm_setr_pd(0.0, 2.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_storeu_pd() { + let mut r = [42_f64; 2]; + let a = _mm_setr_pd(1.0, 2.0); + let m = 0b10; + _mm_mask_storeu_pd(r.as_mut_ptr(), m, a); + let e = _mm_setr_pd(42.0, 2.0); + assert_eq_m128d(_mm_loadu_pd(r.as_ptr()), e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_store_pd() { + #[repr(align(16))] + struct Align { + data: [f64; 2], // 16 bytes + } + let mut r = Align { data: [42.0; 2] }; + let a = _mm_setr_pd(1.0, 2.0); + let m = 0b10; + _mm_mask_store_pd(r.data.as_mut_ptr(), m, a); + let e = _mm_setr_pd(42.0, 2.0); + assert_eq_m128d(_mm_load_pd(r.data.as_ptr()), e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_setr_pd() { + let r = _mm512_set_pd(0., 1., 2., 3., 4., 5., 6., 7.); + assert_eq_m512d(r, _mm512_setr_pd(7., 6., 5., 4., 3., 2., 1., 0.)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_set_pd() { + let r = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + assert_eq_m512d(r, _mm512_set_pd(7., 6., 5., 4., 3., 2., 1., 0.)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_rol_epi32() { + let a = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let r = _mm512_rol_epi32::<1>(a); + let e = _mm512_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_rol_epi32() { + let a = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let r = _mm512_mask_rol_epi32::<1>(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_rol_epi32::<1>(a, 0b11111111_11111111, a); + let e = _mm512_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_rol_epi32() { + let a = _mm512_set_epi32(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 << 31); + let r = _mm512_maskz_rol_epi32::<1>(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_rol_epi32::<1>(0b00000000_11111111, a); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 1 << 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_rol_epi32() { + let a = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); + let r = _mm256_rol_epi32::<1>(a); + let e = _mm256_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_rol_epi32() { + let a = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); + let r = _mm256_mask_rol_epi32::<1>(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_rol_epi32::<1>(a, 0b11111111, a); + let e = _mm256_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_rol_epi32() { + let a = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); + let r = _mm256_maskz_rol_epi32::<1>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_rol_epi32::<1>(0b11111111, a); + let e = _mm256_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_rol_epi32() { + let a = _mm_set_epi32(1 << 31, 1, 1, 1); + let r = _mm_rol_epi32::<1>(a); + let e = _mm_set_epi32(1 << 0, 2, 2, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_rol_epi32() { + let a = _mm_set_epi32(1 << 31, 1, 1, 1); + let r = _mm_mask_rol_epi32::<1>(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_rol_epi32::<1>(a, 0b00001111, a); + let e = _mm_set_epi32(1 << 0, 2, 2, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_rol_epi32() { + let a = _mm_set_epi32(1 << 31, 1, 1, 1); + let r = _mm_maskz_rol_epi32::<1>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_rol_epi32::<1>(0b00001111, a); + let e = _mm_set_epi32(1 << 0, 2, 2, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_ror_epi32() { + let a = _mm512_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let r = _mm512_ror_epi32::<1>(a); + let e = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_ror_epi32() { + let a = _mm512_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let r = _mm512_mask_ror_epi32::<1>(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_ror_epi32::<1>(a, 0b11111111_11111111, a); + let e = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_ror_epi32() { + let a = _mm512_set_epi32(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1 << 0); + let r = _mm512_maskz_ror_epi32::<1>(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_ror_epi32::<1>(0b00000000_11111111, a); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 << 31); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_ror_epi32() { + let a = _mm256_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2); + let r = _mm256_ror_epi32::<1>(a); + let e = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_ror_epi32() { + let a = _mm256_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2); + let r = _mm256_mask_ror_epi32::<1>(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_ror_epi32::<1>(a, 0b11111111, a); + let e = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_ror_epi32() { + let a = _mm256_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2); + let r = _mm256_maskz_ror_epi32::<1>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_ror_epi32::<1>(0b11111111, a); + let e = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_ror_epi32() { + let a = _mm_set_epi32(1 << 0, 2, 2, 2); + let r = _mm_ror_epi32::<1>(a); + let e = _mm_set_epi32(1 << 31, 1, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_ror_epi32() { + let a = _mm_set_epi32(1 << 0, 2, 2, 2); + let r = _mm_mask_ror_epi32::<1>(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_ror_epi32::<1>(a, 0b00001111, a); + let e = _mm_set_epi32(1 << 31, 1, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_ror_epi32() { + let a = _mm_set_epi32(1 << 0, 2, 2, 2); + let r = _mm_maskz_ror_epi32::<1>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_ror_epi32::<1>(0b00001111, a); + let e = _mm_set_epi32(1 << 31, 1, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_slli_epi32() { + let a = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let r = _mm512_slli_epi32::<1>(a); + let e = _mm512_set_epi32(0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_slli_epi32() { + let a = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let r = _mm512_mask_slli_epi32::<1>(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_slli_epi32::<1>(a, 0b11111111_11111111, a); + let e = _mm512_set_epi32(0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_slli_epi32() { + let a = _mm512_set_epi32(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 << 31); + let r = _mm512_maskz_slli_epi32::<1>(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_slli_epi32::<1>(0b00000000_11111111, a); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_slli_epi32() { + let a = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); + let r = _mm256_mask_slli_epi32::<1>(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_slli_epi32::<1>(a, 0b11111111, a); + let e = _mm256_set_epi32(0, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_slli_epi32() { + let a = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); + let r = _mm256_maskz_slli_epi32::<1>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_slli_epi32::<1>(0b11111111, a); + let e = _mm256_set_epi32(0, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_slli_epi32() { + let a = _mm_set_epi32(1 << 31, 1, 1, 1); + let r = _mm_mask_slli_epi32::<1>(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_slli_epi32::<1>(a, 0b00001111, a); + let e = _mm_set_epi32(0, 2, 2, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_slli_epi32() { + let a = _mm_set_epi32(1 << 31, 1, 1, 1); + let r = _mm_maskz_slli_epi32::<1>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_slli_epi32::<1>(0b00001111, a); + let e = _mm_set_epi32(0, 2, 2, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_srli_epi32() { + let a = _mm512_set_epi32(0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let r = _mm512_srli_epi32::<1>(a); + let e = _mm512_set_epi32(0 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_srli_epi32() { + let a = _mm512_set_epi32(0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let r = _mm512_mask_srli_epi32::<1>(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_srli_epi32::<1>(a, 0b11111111_11111111, a); + let e = _mm512_set_epi32(0 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_srli_epi32() { + let a = _mm512_set_epi32(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0); + let r = _mm512_maskz_srli_epi32::<1>(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_srli_epi32::<1>(0b00000000_11111111, a); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0 << 31); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_srli_epi32() { + let a = _mm256_set_epi32(1 << 5, 0, 0, 0, 0, 0, 0, 0); + let r = _mm256_mask_srli_epi32::<1>(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_srli_epi32::<1>(a, 0b11111111, a); + let e = _mm256_set_epi32(1 << 4, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_srli_epi32() { + let a = _mm256_set_epi32(1 << 5, 0, 0, 0, 0, 0, 0, 0); + let r = _mm256_maskz_srli_epi32::<1>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_srli_epi32::<1>(0b11111111, a); + let e = _mm256_set_epi32(1 << 4, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_srli_epi32() { + let a = _mm_set_epi32(1 << 5, 0, 0, 0); + let r = _mm_mask_srli_epi32::<1>(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_srli_epi32::<1>(a, 0b00001111, a); + let e = _mm_set_epi32(1 << 4, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_srli_epi32() { + let a = _mm_set_epi32(1 << 5, 0, 0, 0); + let r = _mm_maskz_srli_epi32::<1>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_srli_epi32::<1>(0b00001111, a); + let e = _mm_set_epi32(1 << 4, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_rolv_epi32() { + let a = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let b = _mm512_set1_epi32(1); + let r = _mm512_rolv_epi32(a, b); + let e = _mm512_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_rolv_epi32() { + let a = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let b = _mm512_set1_epi32(1); + let r = _mm512_mask_rolv_epi32(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_rolv_epi32(a, 0b11111111_11111111, a, b); + let e = _mm512_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_rolv_epi32() { + let a = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 << 31); + let b = _mm512_set1_epi32(1); + let r = _mm512_maskz_rolv_epi32(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_rolv_epi32(0b00000000_11111111, a, b); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 1 << 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_rolv_epi32() { + let a = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); + let b = _mm256_set1_epi32(1); + let r = _mm256_rolv_epi32(a, b); + let e = _mm256_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_rolv_epi32() { + let a = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); + let b = _mm256_set1_epi32(1); + let r = _mm256_mask_rolv_epi32(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_rolv_epi32(a, 0b11111111, a, b); + let e = _mm256_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_rolv_epi32() { + let a = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); + let b = _mm256_set1_epi32(1); + let r = _mm256_maskz_rolv_epi32(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_rolv_epi32(0b11111111, a, b); + let e = _mm256_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_rolv_epi32() { + let a = _mm_set_epi32(1 << 31, 1, 1, 1); + let b = _mm_set1_epi32(1); + let r = _mm_rolv_epi32(a, b); + let e = _mm_set_epi32(1 << 0, 2, 2, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_rolv_epi32() { + let a = _mm_set_epi32(1 << 31, 1, 1, 1); + let b = _mm_set1_epi32(1); + let r = _mm_mask_rolv_epi32(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_rolv_epi32(a, 0b00001111, a, b); + let e = _mm_set_epi32(1 << 0, 2, 2, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_rolv_epi32() { + let a = _mm_set_epi32(1 << 31, 1, 1, 1); + let b = _mm_set1_epi32(1); + let r = _mm_maskz_rolv_epi32(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_rolv_epi32(0b00001111, a, b); + let e = _mm_set_epi32(1 << 0, 2, 2, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_rorv_epi32() { + let a = _mm512_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let b = _mm512_set1_epi32(1); + let r = _mm512_rorv_epi32(a, b); + let e = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_rorv_epi32() { + let a = _mm512_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let b = _mm512_set1_epi32(1); + let r = _mm512_mask_rorv_epi32(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_rorv_epi32(a, 0b11111111_11111111, a, b); + let e = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_rorv_epi32() { + let a = _mm512_set_epi32(3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1 << 0); + let b = _mm512_set1_epi32(1); + let r = _mm512_maskz_rorv_epi32(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_rorv_epi32(0b00000000_11111111, a, b); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 << 31); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_rorv_epi32() { + let a = _mm256_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2); + let b = _mm256_set1_epi32(1); + let r = _mm256_rorv_epi32(a, b); + let e = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_rorv_epi32() { + let a = _mm256_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2); + let b = _mm256_set1_epi32(1); + let r = _mm256_mask_rorv_epi32(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_rorv_epi32(a, 0b11111111, a, b); + let e = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_rorv_epi32() { + let a = _mm256_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2); + let b = _mm256_set1_epi32(1); + let r = _mm256_maskz_rorv_epi32(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_rorv_epi32(0b11111111, a, b); + let e = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_rorv_epi32() { + let a = _mm_set_epi32(1 << 0, 2, 2, 2); + let b = _mm_set1_epi32(1); + let r = _mm_rorv_epi32(a, b); + let e = _mm_set_epi32(1 << 31, 1, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_rorv_epi32() { + let a = _mm_set_epi32(1 << 0, 2, 2, 2); + let b = _mm_set1_epi32(1); + let r = _mm_mask_rorv_epi32(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_rorv_epi32(a, 0b00001111, a, b); + let e = _mm_set_epi32(1 << 31, 1, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_rorv_epi32() { + let a = _mm_set_epi32(1 << 0, 2, 2, 2); + let b = _mm_set1_epi32(1); + let r = _mm_maskz_rorv_epi32(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_rorv_epi32(0b00001111, a, b); + let e = _mm_set_epi32(1 << 31, 1, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_sllv_epi32() { + let a = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let count = _mm512_set1_epi32(1); + let r = _mm512_sllv_epi32(a, count); + let e = _mm512_set_epi32(0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_sllv_epi32() { + let a = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let count = _mm512_set1_epi32(1); + let r = _mm512_mask_sllv_epi32(a, 0, a, count); + assert_eq_m512i(r, a); + let r = _mm512_mask_sllv_epi32(a, 0b11111111_11111111, a, count); + let e = _mm512_set_epi32(0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_sllv_epi32() { + let a = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 << 31); + let count = _mm512_set_epi32(0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let r = _mm512_maskz_sllv_epi32(0, a, count); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_sllv_epi32(0b00000000_11111111, a, count); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_sllv_epi32() { + let a = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); + let count = _mm256_set1_epi32(1); + let r = _mm256_mask_sllv_epi32(a, 0, a, count); + assert_eq_m256i(r, a); + let r = _mm256_mask_sllv_epi32(a, 0b11111111, a, count); + let e = _mm256_set_epi32(0, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_sllv_epi32() { + let a = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); + let count = _mm256_set1_epi32(1); + let r = _mm256_maskz_sllv_epi32(0, a, count); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_sllv_epi32(0b11111111, a, count); + let e = _mm256_set_epi32(0, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_sllv_epi32() { + let a = _mm_set_epi32(1 << 31, 1, 1, 1); + let count = _mm_set1_epi32(1); + let r = _mm_mask_sllv_epi32(a, 0, a, count); + assert_eq_m128i(r, a); + let r = _mm_mask_sllv_epi32(a, 0b00001111, a, count); + let e = _mm_set_epi32(0, 2, 2, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_sllv_epi32() { + let a = _mm_set_epi32(1 << 31, 1, 1, 1); + let count = _mm_set1_epi32(1); + let r = _mm_maskz_sllv_epi32(0, a, count); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_sllv_epi32(0b00001111, a, count); + let e = _mm_set_epi32(0, 2, 2, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_srlv_epi32() { + let a = _mm512_set_epi32(0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let count = _mm512_set1_epi32(1); + let r = _mm512_srlv_epi32(a, count); + let e = _mm512_set_epi32(0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_srlv_epi32() { + let a = _mm512_set_epi32(0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let count = _mm512_set1_epi32(1); + let r = _mm512_mask_srlv_epi32(a, 0, a, count); + assert_eq_m512i(r, a); + let r = _mm512_mask_srlv_epi32(a, 0b11111111_11111111, a, count); + let e = _mm512_set_epi32(0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_srlv_epi32() { + let a = _mm512_set_epi32(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0); + let count = _mm512_set_epi32(0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let r = _mm512_maskz_srlv_epi32(0, a, count); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_srlv_epi32(0b00000000_11111111, a, count); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_srlv_epi32() { + let a = _mm256_set_epi32(1 << 5, 0, 0, 0, 0, 0, 0, 0); + let count = _mm256_set1_epi32(1); + let r = _mm256_mask_srlv_epi32(a, 0, a, count); + assert_eq_m256i(r, a); + let r = _mm256_mask_srlv_epi32(a, 0b11111111, a, count); + let e = _mm256_set_epi32(1 << 4, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_srlv_epi32() { + let a = _mm256_set_epi32(1 << 5, 0, 0, 0, 0, 0, 0, 0); + let count = _mm256_set1_epi32(1); + let r = _mm256_maskz_srlv_epi32(0, a, count); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_srlv_epi32(0b11111111, a, count); + let e = _mm256_set_epi32(1 << 4, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_srlv_epi32() { + let a = _mm_set_epi32(1 << 5, 0, 0, 0); + let count = _mm_set1_epi32(1); + let r = _mm_mask_srlv_epi32(a, 0, a, count); + assert_eq_m128i(r, a); + let r = _mm_mask_srlv_epi32(a, 0b00001111, a, count); + let e = _mm_set_epi32(1 << 4, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_srlv_epi32() { + let a = _mm_set_epi32(1 << 5, 0, 0, 0); + let count = _mm_set1_epi32(1); + let r = _mm_maskz_srlv_epi32(0, a, count); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_srlv_epi32(0b00001111, a, count); + let e = _mm_set_epi32(1 << 4, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_sll_epi32() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 1 << 31, 1 << 0, 1 << 1, 1 << 2, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + ); + let count = _mm_set_epi32(0, 0, 0, 2); + let r = _mm512_sll_epi32(a, count); + #[rustfmt::skip] + let e = _mm512_set_epi32( + 0, 1 << 2, 1 << 3, 1 << 4, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_sll_epi32() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 1 << 31, 1 << 0, 1 << 1, 1 << 2, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + ); + let count = _mm_set_epi32(0, 0, 0, 2); + let r = _mm512_mask_sll_epi32(a, 0, a, count); + assert_eq_m512i(r, a); + let r = _mm512_mask_sll_epi32(a, 0b11111111_11111111, a, count); + #[rustfmt::skip] + let e = _mm512_set_epi32( + 0, 1 << 2, 1 << 3, 1 << 4, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_sll_epi32() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 1 << 31, 1 << 0, 1 << 1, 1 << 2, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 31, + ); + let count = _mm_set_epi32(2, 0, 0, 2); + let r = _mm512_maskz_sll_epi32(0, a, count); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_sll_epi32(0b00000000_11111111, a, count); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_sll_epi32() { + let a = _mm256_set_epi32(1 << 13, 0, 0, 0, 0, 0, 0, 0); + let count = _mm_set_epi32(0, 0, 0, 1); + let r = _mm256_mask_sll_epi32(a, 0, a, count); + assert_eq_m256i(r, a); + let r = _mm256_mask_sll_epi32(a, 0b11111111, a, count); + let e = _mm256_set_epi32(1 << 14, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_sll_epi32() { + let a = _mm256_set_epi32(1 << 13, 0, 0, 0, 0, 0, 0, 0); + let count = _mm_set_epi32(0, 0, 0, 1); + let r = _mm256_maskz_sll_epi32(0, a, count); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_sll_epi32(0b11111111, a, count); + let e = _mm256_set_epi32(1 << 14, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_sll_epi32() { + let a = _mm_set_epi32(1 << 13, 0, 0, 0); + let count = _mm_set_epi32(0, 0, 0, 1); + let r = _mm_mask_sll_epi32(a, 0, a, count); + assert_eq_m128i(r, a); + let r = _mm_mask_sll_epi32(a, 0b00001111, a, count); + let e = _mm_set_epi32(1 << 14, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_sll_epi32() { + let a = _mm_set_epi32(1 << 13, 0, 0, 0); + let count = _mm_set_epi32(0, 0, 0, 1); + let r = _mm_maskz_sll_epi32(0, a, count); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_sll_epi32(0b00001111, a, count); + let e = _mm_set_epi32(1 << 14, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_srl_epi32() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 1 << 31, 1 << 0, 1 << 1, 1 << 2, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + ); + let count = _mm_set_epi32(0, 0, 0, 2); + let r = _mm512_srl_epi32(a, count); + let e = _mm512_set_epi32(1 << 29, 0, 0, 1 << 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_srl_epi32() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 1 << 31, 1 << 0, 1 << 1, 1 << 2, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + ); + let count = _mm_set_epi32(0, 0, 0, 2); + let r = _mm512_mask_srl_epi32(a, 0, a, count); + assert_eq_m512i(r, a); + let r = _mm512_mask_srl_epi32(a, 0b11111111_11111111, a, count); + let e = _mm512_set_epi32(1 << 29, 0, 0, 1 << 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_srl_epi32() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 1 << 31, 1 << 0, 1 << 1, 1 << 2, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 31, + ); + let count = _mm_set_epi32(2, 0, 0, 2); + let r = _mm512_maskz_srl_epi32(0, a, count); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_srl_epi32(0b00000000_11111111, a, count); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 << 29); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_srl_epi32() { + let a = _mm256_set_epi32(1 << 5, 0, 0, 0, 0, 0, 0, 0); + let count = _mm_set_epi32(0, 0, 0, 1); + let r = _mm256_mask_srl_epi32(a, 0, a, count); + assert_eq_m256i(r, a); + let r = _mm256_mask_srl_epi32(a, 0b11111111, a, count); + let e = _mm256_set_epi32(1 << 4, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_srl_epi32() { + let a = _mm256_set_epi32(1 << 5, 0, 0, 0, 0, 0, 0, 0); + let count = _mm_set_epi32(0, 0, 0, 1); + let r = _mm256_maskz_srl_epi32(0, a, count); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_srl_epi32(0b11111111, a, count); + let e = _mm256_set_epi32(1 << 4, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_srl_epi32() { + let a = _mm_set_epi32(1 << 5, 0, 0, 0); + let count = _mm_set_epi32(0, 0, 0, 1); + let r = _mm_mask_srl_epi32(a, 0, a, count); + assert_eq_m128i(r, a); + let r = _mm_mask_srl_epi32(a, 0b00001111, a, count); + let e = _mm_set_epi32(1 << 4, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_srl_epi32() { + let a = _mm_set_epi32(1 << 5, 0, 0, 0); + let count = _mm_set_epi32(0, 0, 0, 1); + let r = _mm_maskz_srl_epi32(0, a, count); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_srl_epi32(0b00001111, a, count); + let e = _mm_set_epi32(1 << 4, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_sra_epi32() { + let a = _mm512_set_epi32(8, -8, 16, -15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1); + let count = _mm_set_epi32(1, 0, 0, 2); + let r = _mm512_sra_epi32(a, count); + let e = _mm512_set_epi32(2, -2, 4, -4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_sra_epi32() { + let a = _mm512_set_epi32(8, -8, 16, -15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16); + let count = _mm_set_epi32(0, 0, 0, 2); + let r = _mm512_mask_sra_epi32(a, 0, a, count); + assert_eq_m512i(r, a); + let r = _mm512_mask_sra_epi32(a, 0b11111111_11111111, a, count); + let e = _mm512_set_epi32(2, -2, 4, -4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_sra_epi32() { + let a = _mm512_set_epi32(8, -8, 16, -15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -15, -14); + let count = _mm_set_epi32(2, 0, 0, 2); + let r = _mm512_maskz_sra_epi32(0, a, count); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_sra_epi32(0b00000000_11111111, a, count); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -4, -4); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_sra_epi32() { + let a = _mm256_set_epi32(1 << 5, 0, 0, 0, 0, 0, 0, 0); + let count = _mm_set_epi32(0, 0, 0, 1); + let r = _mm256_mask_sra_epi32(a, 0, a, count); + assert_eq_m256i(r, a); + let r = _mm256_mask_sra_epi32(a, 0b11111111, a, count); + let e = _mm256_set_epi32(1 << 4, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_sra_epi32() { + let a = _mm256_set_epi32(1 << 5, 0, 0, 0, 0, 0, 0, 0); + let count = _mm_set_epi32(0, 0, 0, 1); + let r = _mm256_maskz_sra_epi32(0, a, count); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_sra_epi32(0b11111111, a, count); + let e = _mm256_set_epi32(1 << 4, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_sra_epi32() { + let a = _mm_set_epi32(1 << 5, 0, 0, 0); + let count = _mm_set_epi32(0, 0, 0, 1); + let r = _mm_mask_sra_epi32(a, 0, a, count); + assert_eq_m128i(r, a); + let r = _mm_mask_sra_epi32(a, 0b00001111, a, count); + let e = _mm_set_epi32(1 << 4, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_sra_epi32() { + let a = _mm_set_epi32(1 << 5, 0, 0, 0); + let count = _mm_set_epi32(0, 0, 0, 1); + let r = _mm_maskz_sra_epi32(0, a, count); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_sra_epi32(0b00001111, a, count); + let e = _mm_set_epi32(1 << 4, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_srav_epi32() { + let a = _mm512_set_epi32(8, -8, 16, -15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1); + let count = _mm512_set_epi32(2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r = _mm512_srav_epi32(a, count); + let e = _mm512_set_epi32(2, -2, 4, -4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_srav_epi32() { + let a = _mm512_set_epi32(8, -8, 16, -15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16); + let count = _mm512_set_epi32(2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1); + let r = _mm512_mask_srav_epi32(a, 0, a, count); + assert_eq_m512i(r, a); + let r = _mm512_mask_srav_epi32(a, 0b11111111_11111111, a, count); + let e = _mm512_set_epi32(2, -2, 4, -4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_srav_epi32() { + let a = _mm512_set_epi32(8, -8, 16, -15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -15, -14); + let count = _mm512_set_epi32(2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2); + let r = _mm512_maskz_srav_epi32(0, a, count); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_srav_epi32(0b00000000_11111111, a, count); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -4, -4); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_srav_epi32() { + let a = _mm256_set_epi32(1 << 5, 0, 0, 0, 0, 0, 0, 0); + let count = _mm256_set1_epi32(1); + let r = _mm256_mask_srav_epi32(a, 0, a, count); + assert_eq_m256i(r, a); + let r = _mm256_mask_srav_epi32(a, 0b11111111, a, count); + let e = _mm256_set_epi32(1 << 4, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_srav_epi32() { + let a = _mm256_set_epi32(1 << 5, 0, 0, 0, 0, 0, 0, 0); + let count = _mm256_set1_epi32(1); + let r = _mm256_maskz_srav_epi32(0, a, count); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_srav_epi32(0b11111111, a, count); + let e = _mm256_set_epi32(1 << 4, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_srav_epi32() { + let a = _mm_set_epi32(1 << 5, 0, 0, 0); + let count = _mm_set1_epi32(1); + let r = _mm_mask_srav_epi32(a, 0, a, count); + assert_eq_m128i(r, a); + let r = _mm_mask_srav_epi32(a, 0b00001111, a, count); + let e = _mm_set_epi32(1 << 4, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_srav_epi32() { + let a = _mm_set_epi32(1 << 5, 0, 0, 0); + let count = _mm_set1_epi32(1); + let r = _mm_maskz_srav_epi32(0, a, count); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_srav_epi32(0b00001111, a, count); + let e = _mm_set_epi32(1 << 4, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_srai_epi32() { + let a = _mm512_set_epi32(8, -8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, -15); + let r = _mm512_srai_epi32::<2>(a); + let e = _mm512_set_epi32(2, -2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, -4); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_srai_epi32() { + let a = _mm512_set_epi32(8, -8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, -15); + let r = _mm512_mask_srai_epi32::<2>(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_srai_epi32::<2>(a, 0b11111111_11111111, a); + let e = _mm512_set_epi32(2, -2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, -4); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_srai_epi32() { + let a = _mm512_set_epi32(8, -8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, -15); + let r = _mm512_maskz_srai_epi32::<2>(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_srai_epi32::<2>(0b00000000_11111111, a); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, -4); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_srai_epi32() { + let a = _mm256_set_epi32(1 << 5, 0, 0, 0, 0, 0, 0, 0); + let r = _mm256_mask_srai_epi32::<1>(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_srai_epi32::<1>(a, 0b11111111, a); + let e = _mm256_set_epi32(1 << 4, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_srai_epi32() { + let a = _mm256_set_epi32(1 << 5, 0, 0, 0, 0, 0, 0, 0); + let r = _mm256_maskz_srai_epi32::<1>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_srai_epi32::<1>(0b11111111, a); + let e = _mm256_set_epi32(1 << 4, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_srai_epi32() { + let a = _mm_set_epi32(1 << 5, 0, 0, 0); + let r = _mm_mask_srai_epi32::<1>(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_srai_epi32::<1>(a, 0b00001111, a); + let e = _mm_set_epi32(1 << 4, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_srai_epi32() { + let a = _mm_set_epi32(1 << 5, 0, 0, 0); + let r = _mm_maskz_srai_epi32::<1>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_srai_epi32::<1>(0b00001111, a); + let e = _mm_set_epi32(1 << 4, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_permute_ps() { + let a = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let r = _mm512_permute_ps::<0b11_11_11_11>(a); + let e = _mm512_setr_ps( + 3., 3., 3., 3., 7., 7., 7., 7., 11., 11., 11., 11., 15., 15., 15., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_permute_ps() { + let a = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let r = _mm512_mask_permute_ps::<0b11_11_11_11>(a, 0, a); + assert_eq_m512(r, a); + let r = _mm512_mask_permute_ps::<0b11_11_11_11>(a, 0b11111111_11111111, a); + let e = _mm512_setr_ps( + 3., 3., 3., 3., 7., 7., 7., 7., 11., 11., 11., 11., 15., 15., 15., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_permute_ps() { + let a = _mm512_setr_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let r = _mm512_maskz_permute_ps::<0b11_11_11_11>(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_permute_ps::<0b11_11_11_11>(0b11111111_11111111, a); + let e = _mm512_setr_ps( + 3., 3., 3., 3., 7., 7., 7., 7., 11., 11., 11., 11., 15., 15., 15., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_permute_ps() { + let a = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm256_mask_permute_ps::<0b11_11_11_11>(a, 0, a); + assert_eq_m256(r, a); + let r = _mm256_mask_permute_ps::<0b11_11_11_11>(a, 0b11111111, a); + let e = _mm256_set_ps(0., 0., 0., 0., 4., 4., 4., 4.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_permute_ps() { + let a = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm256_maskz_permute_ps::<0b11_11_11_11>(0, a); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_permute_ps::<0b11_11_11_11>(0b11111111, a); + let e = _mm256_set_ps(0., 0., 0., 0., 4., 4., 4., 4.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_permute_ps() { + let a = _mm_set_ps(0., 1., 2., 3.); + let r = _mm_mask_permute_ps::<0b11_11_11_11>(a, 0, a); + assert_eq_m128(r, a); + let r = _mm_mask_permute_ps::<0b11_11_11_11>(a, 0b00001111, a); + let e = _mm_set_ps(0., 0., 0., 0.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_permute_ps() { + let a = _mm_set_ps(0., 1., 2., 3.); + let r = _mm_maskz_permute_ps::<0b11_11_11_11>(0, a); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_permute_ps::<0b11_11_11_11>(0b00001111, a); + let e = _mm_set_ps(0., 0., 0., 0.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_permutevar_epi32() { + let idx = _mm512_set1_epi32(1); + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_permutevar_epi32(idx, a); + let e = _mm512_set1_epi32(14); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_permutevar_epi32() { + let idx = _mm512_set1_epi32(1); + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_mask_permutevar_epi32(a, 0, idx, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_permutevar_epi32(a, 0b11111111_11111111, idx, a); + let e = _mm512_set1_epi32(14); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_permutevar_ps() { + let a = _mm512_set_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let b = _mm512_set1_epi32(0b01); + let r = _mm512_permutevar_ps(a, b); + let e = _mm512_set_ps( + 2., 2., 2., 2., 6., 6., 6., 6., 10., 10., 10., 10., 14., 14., 14., 14., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_permutevar_ps() { + let a = _mm512_set_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let b = _mm512_set1_epi32(0b01); + let r = _mm512_mask_permutevar_ps(a, 0, a, b); + assert_eq_m512(r, a); + let r = _mm512_mask_permutevar_ps(a, 0b11111111_11111111, a, b); + let e = _mm512_set_ps( + 2., 2., 2., 2., 6., 6., 6., 6., 10., 10., 10., 10., 14., 14., 14., 14., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_permutevar_ps() { + let a = _mm512_set_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let b = _mm512_set1_epi32(0b01); + let r = _mm512_maskz_permutevar_ps(0, a, b); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_permutevar_ps(0b00000000_11111111, a, b); + let e = _mm512_set_ps( + 0., 0., 0., 0., 0., 0., 0., 0., 10., 10., 10., 10., 14., 14., 14., 14., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_permutevar_ps() { + let a = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let b = _mm256_set1_epi32(0b01); + let r = _mm256_mask_permutevar_ps(a, 0, a, b); + assert_eq_m256(r, a); + let r = _mm256_mask_permutevar_ps(a, 0b11111111, a, b); + let e = _mm256_set_ps(2., 2., 2., 2., 6., 6., 6., 6.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_permutevar_ps() { + let a = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let b = _mm256_set1_epi32(0b01); + let r = _mm256_maskz_permutevar_ps(0, a, b); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_permutevar_ps(0b11111111, a, b); + let e = _mm256_set_ps(2., 2., 2., 2., 6., 6., 6., 6.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_permutevar_ps() { + let a = _mm_set_ps(0., 1., 2., 3.); + let b = _mm_set1_epi32(0b01); + let r = _mm_mask_permutevar_ps(a, 0, a, b); + assert_eq_m128(r, a); + let r = _mm_mask_permutevar_ps(a, 0b00001111, a, b); + let e = _mm_set_ps(2., 2., 2., 2.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_permutevar_ps() { + let a = _mm_set_ps(0., 1., 2., 3.); + let b = _mm_set1_epi32(0b01); + let r = _mm_maskz_permutevar_ps(0, a, b); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_permutevar_ps(0b00001111, a, b); + let e = _mm_set_ps(2., 2., 2., 2.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_permutexvar_epi32() { + let idx = _mm512_set1_epi32(1); + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_permutexvar_epi32(idx, a); + let e = _mm512_set1_epi32(14); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_permutexvar_epi32() { + let idx = _mm512_set1_epi32(1); + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_mask_permutexvar_epi32(a, 0, idx, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_permutexvar_epi32(a, 0b11111111_11111111, idx, a); + let e = _mm512_set1_epi32(14); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_permutexvar_epi32() { + let idx = _mm512_set1_epi32(1); + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_permutexvar_epi32(0, idx, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_permutexvar_epi32(0b00000000_11111111, idx, a); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 14, 14, 14, 14, 14, 14, 14, 14); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_permutexvar_epi32() { + let idx = _mm256_set1_epi32(1); + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm256_permutexvar_epi32(idx, a); + let e = _mm256_set1_epi32(6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_permutexvar_epi32() { + let idx = _mm256_set1_epi32(1); + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm256_mask_permutexvar_epi32(a, 0, idx, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_permutexvar_epi32(a, 0b11111111, idx, a); + let e = _mm256_set1_epi32(6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_permutexvar_epi32() { + let idx = _mm256_set1_epi32(1); + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm256_maskz_permutexvar_epi32(0, idx, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_permutexvar_epi32(0b11111111, idx, a); + let e = _mm256_set1_epi32(6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_permutexvar_ps() { + let idx = _mm512_set1_epi32(1); + let a = _mm512_set_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let r = _mm512_permutexvar_ps(idx, a); + let e = _mm512_set1_ps(14.); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_permutexvar_ps() { + let idx = _mm512_set1_epi32(1); + let a = _mm512_set_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let r = _mm512_mask_permutexvar_ps(a, 0, idx, a); + assert_eq_m512(r, a); + let r = _mm512_mask_permutexvar_ps(a, 0b11111111_11111111, idx, a); + let e = _mm512_set1_ps(14.); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_permutexvar_ps() { + let idx = _mm512_set1_epi32(1); + let a = _mm512_set_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let r = _mm512_maskz_permutexvar_ps(0, idx, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_permutexvar_ps(0b00000000_11111111, idx, a); + let e = _mm512_set_ps( + 0., 0., 0., 0., 0., 0., 0., 0., 14., 14., 14., 14., 14., 14., 14., 14., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_permutexvar_ps() { + let idx = _mm256_set1_epi32(1); + let a = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm256_permutexvar_ps(idx, a); + let e = _mm256_set1_ps(6.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_permutexvar_ps() { + let idx = _mm256_set1_epi32(1); + let a = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm256_mask_permutexvar_ps(a, 0, idx, a); + assert_eq_m256(r, a); + let r = _mm256_mask_permutexvar_ps(a, 0b11111111, idx, a); + let e = _mm256_set1_ps(6.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_permutexvar_ps() { + let idx = _mm256_set1_epi32(1); + let a = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm256_maskz_permutexvar_ps(0, idx, a); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_permutexvar_ps(0b11111111, idx, a); + let e = _mm256_set1_ps(6.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_permutex2var_epi32() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let idx = _mm512_set_epi32( + 1, 1 << 4, 2, 1 << 4, + 3, 1 << 4, 4, 1 << 4, + 5, 1 << 4, 6, 1 << 4, + 7, 1 << 4, 8, 1 << 4, + ); + let b = _mm512_set1_epi32(100); + let r = _mm512_permutex2var_epi32(a, idx, b); + let e = _mm512_set_epi32( + 14, 100, 13, 100, 12, 100, 11, 100, 10, 100, 9, 100, 8, 100, 7, 100, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_permutex2var_epi32() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let idx = _mm512_set_epi32( + 1, 1 << 4, 2, 1 << 4, + 3, 1 << 4, 4, 1 << 4, + 5, 1 << 4, 6, 1 << 4, + 7, 1 << 4, 8, 1 << 4, + ); + let b = _mm512_set1_epi32(100); + let r = _mm512_mask_permutex2var_epi32(a, 0, idx, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_permutex2var_epi32(a, 0b11111111_11111111, idx, b); + let e = _mm512_set_epi32( + 14, 100, 13, 100, 12, 100, 11, 100, 10, 100, 9, 100, 8, 100, 7, 100, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_permutex2var_epi32() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let idx = _mm512_set_epi32( + 1, 1 << 4, 2, 1 << 4, + 3, 1 << 4, 4, 1 << 4, + 5, 1 << 4, 6, 1 << 4, + 7, 1 << 4, 8, 1 << 4, + ); + let b = _mm512_set1_epi32(100); + let r = _mm512_maskz_permutex2var_epi32(0, a, idx, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_permutex2var_epi32(0b00000000_11111111, a, idx, b); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 10, 100, 9, 100, 8, 100, 7, 100); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask2_permutex2var_epi32() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let idx = _mm512_set_epi32( + 1000, 1 << 4, 2000, 1 << 4, + 3000, 1 << 4, 4000, 1 << 4, + 5, 1 << 4, 6, 1 << 4, + 7, 1 << 4, 8, 1 << 4, + ); + let b = _mm512_set1_epi32(100); + let r = _mm512_mask2_permutex2var_epi32(a, idx, 0, b); + assert_eq_m512i(r, idx); + let r = _mm512_mask2_permutex2var_epi32(a, idx, 0b00000000_11111111, b); + #[rustfmt::skip] + let e = _mm512_set_epi32( + 1000, 1 << 4, 2000, 1 << 4, + 3000, 1 << 4, 4000, 1 << 4, + 10, 100, 9, 100, + 8, 100, 7, 100, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_permutex2var_epi32() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let idx = _mm256_set_epi32(1, 1 << 3, 2, 1 << 3, 3, 1 << 3, 4, 1 << 3); + let b = _mm256_set1_epi32(100); + let r = _mm256_permutex2var_epi32(a, idx, b); + let e = _mm256_set_epi32(6, 100, 5, 100, 4, 100, 3, 100); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_permutex2var_epi32() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let idx = _mm256_set_epi32(1, 1 << 3, 2, 1 << 3, 3, 1 << 3, 4, 1 << 3); + let b = _mm256_set1_epi32(100); + let r = _mm256_mask_permutex2var_epi32(a, 0, idx, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_permutex2var_epi32(a, 0b11111111, idx, b); + let e = _mm256_set_epi32(6, 100, 5, 100, 4, 100, 3, 100); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_permutex2var_epi32() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let idx = _mm256_set_epi32(1, 1 << 3, 2, 1 << 3, 3, 1 << 3, 4, 1 << 3); + let b = _mm256_set1_epi32(100); + let r = _mm256_maskz_permutex2var_epi32(0, a, idx, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_permutex2var_epi32(0b11111111, a, idx, b); + let e = _mm256_set_epi32(6, 100, 5, 100, 4, 100, 3, 100); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask2_permutex2var_epi32() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let idx = _mm256_set_epi32(1, 1 << 3, 2, 1 << 3, 3, 1 << 3, 4, 1 << 3); + let b = _mm256_set1_epi32(100); + let r = _mm256_mask2_permutex2var_epi32(a, idx, 0, b); + assert_eq_m256i(r, idx); + let r = _mm256_mask2_permutex2var_epi32(a, idx, 0b11111111, b); + let e = _mm256_set_epi32(6, 100, 5, 100, 4, 100, 3, 100); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_permutex2var_epi32() { + let a = _mm_set_epi32(0, 1, 2, 3); + let idx = _mm_set_epi32(1, 1 << 2, 2, 1 << 2); + let b = _mm_set1_epi32(100); + let r = _mm_permutex2var_epi32(a, idx, b); + let e = _mm_set_epi32(2, 100, 1, 100); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_permutex2var_epi32() { + let a = _mm_set_epi32(0, 1, 2, 3); + let idx = _mm_set_epi32(1, 1 << 2, 2, 1 << 2); + let b = _mm_set1_epi32(100); + let r = _mm_mask_permutex2var_epi32(a, 0, idx, b); + assert_eq_m128i(r, a); + let r = _mm_mask_permutex2var_epi32(a, 0b00001111, idx, b); + let e = _mm_set_epi32(2, 100, 1, 100); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_permutex2var_epi32() { + let a = _mm_set_epi32(0, 1, 2, 3); + let idx = _mm_set_epi32(1, 1 << 2, 2, 1 << 2); + let b = _mm_set1_epi32(100); + let r = _mm_maskz_permutex2var_epi32(0, a, idx, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_permutex2var_epi32(0b00001111, a, idx, b); + let e = _mm_set_epi32(2, 100, 1, 100); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask2_permutex2var_epi32() { + let a = _mm_set_epi32(0, 1, 2, 3); + let idx = _mm_set_epi32(1, 1 << 2, 2, 1 << 2); + let b = _mm_set1_epi32(100); + let r = _mm_mask2_permutex2var_epi32(a, idx, 0, b); + assert_eq_m128i(r, idx); + let r = _mm_mask2_permutex2var_epi32(a, idx, 0b00001111, b); + let e = _mm_set_epi32(2, 100, 1, 100); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_permutex2var_ps() { + let a = _mm512_set_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + #[rustfmt::skip] + let idx = _mm512_set_epi32( + 1, 1 << 4, 2, 1 << 4, + 3, 1 << 4, 4, 1 << 4, + 5, 1 << 4, 6, 1 << 4, + 7, 1 << 4, 8, 1 << 4, + ); + let b = _mm512_set1_ps(100.); + let r = _mm512_permutex2var_ps(a, idx, b); + let e = _mm512_set_ps( + 14., 100., 13., 100., 12., 100., 11., 100., 10., 100., 9., 100., 8., 100., 7., 100., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_permutex2var_ps() { + let a = _mm512_set_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + #[rustfmt::skip] + let idx = _mm512_set_epi32( + 1, 1 << 4, 2, 1 << 4, + 3, 1 << 4, 4, 1 << 4, + 5, 1 << 4, 6, 1 << 4, + 7, 1 << 4, 8, 1 << 4, + ); + let b = _mm512_set1_ps(100.); + let r = _mm512_mask_permutex2var_ps(a, 0, idx, b); + assert_eq_m512(r, a); + let r = _mm512_mask_permutex2var_ps(a, 0b11111111_11111111, idx, b); + let e = _mm512_set_ps( + 14., 100., 13., 100., 12., 100., 11., 100., 10., 100., 9., 100., 8., 100., 7., 100., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_permutex2var_ps() { + let a = _mm512_set_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + #[rustfmt::skip] + let idx = _mm512_set_epi32( + 1, 1 << 4, 2, 1 << 4, + 3, 1 << 4, 4, 1 << 4, + 5, 1 << 4, 6, 1 << 4, + 7, 1 << 4, 8, 1 << 4, + ); + let b = _mm512_set1_ps(100.); + let r = _mm512_maskz_permutex2var_ps(0, a, idx, b); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_permutex2var_ps(0b00000000_11111111, a, idx, b); + let e = _mm512_set_ps( + 0., 0., 0., 0., 0., 0., 0., 0., 10., 100., 9., 100., 8., 100., 7., 100., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask2_permutex2var_ps() { + let a = _mm512_set_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + #[rustfmt::skip] + let idx = _mm512_set_epi32( + 1, 1 << 4, 2, 1 << 4, + 3, 1 << 4, 4, 1 << 4, + 5, 1 << 4, 6, 1 << 4, + 7, 1 << 4, 8, 1 << 4, + ); + let b = _mm512_set1_ps(100.); + let r = _mm512_mask2_permutex2var_ps(a, idx, 0, b); + assert_eq_m512(r, _mm512_castsi512_ps(idx)); + let r = _mm512_mask2_permutex2var_ps(a, idx, 0b11111111_11111111, b); + let e = _mm512_set_ps( + 14., 100., 13., 100., 12., 100., 11., 100., 10., 100., 9., 100., 8., 100., 7., 100., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_permutex2var_ps() { + let a = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let idx = _mm256_set_epi32(1, 1 << 3, 2, 1 << 3, 3, 1 << 3, 4, 1 << 3); + let b = _mm256_set1_ps(100.); + let r = _mm256_permutex2var_ps(a, idx, b); + let e = _mm256_set_ps(6., 100., 5., 100., 4., 100., 3., 100.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_permutex2var_ps() { + let a = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let idx = _mm256_set_epi32(1, 1 << 3, 2, 1 << 3, 3, 1 << 3, 4, 1 << 3); + let b = _mm256_set1_ps(100.); + let r = _mm256_mask_permutex2var_ps(a, 0, idx, b); + assert_eq_m256(r, a); + let r = _mm256_mask_permutex2var_ps(a, 0b11111111, idx, b); + let e = _mm256_set_ps(6., 100., 5., 100., 4., 100., 3., 100.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_permutex2var_ps() { + let a = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let idx = _mm256_set_epi32(1, 1 << 3, 2, 1 << 3, 3, 1 << 3, 4, 1 << 3); + let b = _mm256_set1_ps(100.); + let r = _mm256_maskz_permutex2var_ps(0, a, idx, b); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_permutex2var_ps(0b11111111, a, idx, b); + let e = _mm256_set_ps(6., 100., 5., 100., 4., 100., 3., 100.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask2_permutex2var_ps() { + let a = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let idx = _mm256_set_epi32(1, 1 << 3, 2, 1 << 3, 3, 1 << 3, 4, 1 << 3); + let b = _mm256_set1_ps(100.); + let r = _mm256_mask2_permutex2var_ps(a, idx, 0, b); + assert_eq_m256(r, _mm256_castsi256_ps(idx)); + let r = _mm256_mask2_permutex2var_ps(a, idx, 0b11111111, b); + let e = _mm256_set_ps(6., 100., 5., 100., 4., 100., 3., 100.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_permutex2var_ps() { + let a = _mm_set_ps(0., 1., 2., 3.); + let idx = _mm_set_epi32(1, 1 << 2, 2, 1 << 2); + let b = _mm_set1_ps(100.); + let r = _mm_permutex2var_ps(a, idx, b); + let e = _mm_set_ps(2., 100., 1., 100.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_permutex2var_ps() { + let a = _mm_set_ps(0., 1., 2., 3.); + let idx = _mm_set_epi32(1, 1 << 2, 2, 1 << 2); + let b = _mm_set1_ps(100.); + let r = _mm_mask_permutex2var_ps(a, 0, idx, b); + assert_eq_m128(r, a); + let r = _mm_mask_permutex2var_ps(a, 0b00001111, idx, b); + let e = _mm_set_ps(2., 100., 1., 100.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_permutex2var_ps() { + let a = _mm_set_ps(0., 1., 2., 3.); + let idx = _mm_set_epi32(1, 1 << 2, 2, 1 << 2); + let b = _mm_set1_ps(100.); + let r = _mm_maskz_permutex2var_ps(0, a, idx, b); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_permutex2var_ps(0b00001111, a, idx, b); + let e = _mm_set_ps(2., 100., 1., 100.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask2_permutex2var_ps() { + let a = _mm_set_ps(0., 1., 2., 3.); + let idx = _mm_set_epi32(1, 1 << 2, 2, 1 << 2); + let b = _mm_set1_ps(100.); + let r = _mm_mask2_permutex2var_ps(a, idx, 0, b); + assert_eq_m128(r, _mm_castsi128_ps(idx)); + let r = _mm_mask2_permutex2var_ps(a, idx, 0b00001111, b); + let e = _mm_set_ps(2., 100., 1., 100.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_shuffle_epi32() { + let a = _mm512_setr_epi32(1, 4, 5, 8, 9, 12, 13, 16, 1, 4, 5, 8, 9, 12, 13, 16); + let r = _mm512_shuffle_epi32::<_MM_PERM_AADD>(a); + let e = _mm512_setr_epi32(8, 8, 1, 1, 16, 16, 9, 9, 8, 8, 1, 1, 16, 16, 9, 9); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_shuffle_epi32() { + let a = _mm512_setr_epi32(1, 4, 5, 8, 9, 12, 13, 16, 1, 4, 5, 8, 9, 12, 13, 16); + let r = _mm512_mask_shuffle_epi32::<_MM_PERM_AADD>(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_shuffle_epi32::<_MM_PERM_AADD>(a, 0b11111111_11111111, a); + let e = _mm512_setr_epi32(8, 8, 1, 1, 16, 16, 9, 9, 8, 8, 1, 1, 16, 16, 9, 9); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_shuffle_epi32() { + let a = _mm512_setr_epi32(1, 4, 5, 8, 9, 12, 13, 16, 1, 4, 5, 8, 9, 12, 13, 16); + let r = _mm512_maskz_shuffle_epi32::<_MM_PERM_AADD>(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_shuffle_epi32::<_MM_PERM_AADD>(0b00000000_11111111, a); + let e = _mm512_setr_epi32(8, 8, 1, 1, 16, 16, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_shuffle_epi32() { + let a = _mm256_set_epi32(1, 4, 5, 8, 9, 12, 13, 16); + let r = _mm256_mask_shuffle_epi32::<_MM_PERM_AADD>(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_shuffle_epi32::<_MM_PERM_AADD>(a, 0b11111111, a); + let e = _mm256_set_epi32(8, 8, 1, 1, 16, 16, 9, 9); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_shuffle_epi32() { + let a = _mm256_set_epi32(1, 4, 5, 8, 9, 12, 13, 16); + let r = _mm256_maskz_shuffle_epi32::<_MM_PERM_AADD>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_shuffle_epi32::<_MM_PERM_AADD>(0b11111111, a); + let e = _mm256_set_epi32(8, 8, 1, 1, 16, 16, 9, 9); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_shuffle_epi32() { + let a = _mm_set_epi32(1, 4, 5, 8); + let r = _mm_mask_shuffle_epi32::<_MM_PERM_AADD>(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_shuffle_epi32::<_MM_PERM_AADD>(a, 0b00001111, a); + let e = _mm_set_epi32(8, 8, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_shuffle_epi32() { + let a = _mm_set_epi32(1, 4, 5, 8); + let r = _mm_maskz_shuffle_epi32::<_MM_PERM_AADD>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_shuffle_epi32::<_MM_PERM_AADD>(0b00001111, a); + let e = _mm_set_epi32(8, 8, 1, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_shuffle_ps() { + let a = _mm512_setr_ps( + 1., 4., 5., 8., 9., 12., 13., 16., 1., 4., 5., 8., 9., 12., 13., 16., + ); + let b = _mm512_setr_ps( + 2., 3., 6., 7., 10., 11., 14., 15., 2., 3., 6., 7., 10., 11., 14., 15., + ); + let r = _mm512_shuffle_ps::<0b00_00_11_11>(a, b); + let e = _mm512_setr_ps( + 8., 8., 2., 2., 16., 16., 10., 10., 8., 8., 2., 2., 16., 16., 10., 10., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_shuffle_ps() { + let a = _mm512_setr_ps( + 1., 4., 5., 8., 9., 12., 13., 16., 1., 4., 5., 8., 9., 12., 13., 16., + ); + let b = _mm512_setr_ps( + 2., 3., 6., 7., 10., 11., 14., 15., 2., 3., 6., 7., 10., 11., 14., 15., + ); + let r = _mm512_mask_shuffle_ps::<0b00_00_11_11>(a, 0, a, b); + assert_eq_m512(r, a); + let r = _mm512_mask_shuffle_ps::<0b00_00_11_11>(a, 0b11111111_11111111, a, b); + let e = _mm512_setr_ps( + 8., 8., 2., 2., 16., 16., 10., 10., 8., 8., 2., 2., 16., 16., 10., 10., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_shuffle_ps() { + let a = _mm512_setr_ps( + 1., 4., 5., 8., 9., 12., 13., 16., 1., 4., 5., 8., 9., 12., 13., 16., + ); + let b = _mm512_setr_ps( + 2., 3., 6., 7., 10., 11., 14., 15., 2., 3., 6., 7., 10., 11., 14., 15., + ); + let r = _mm512_maskz_shuffle_ps::<0b00_00_11_11>(0, a, b); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_shuffle_ps::<0b00_00_11_11>(0b00000000_11111111, a, b); + let e = _mm512_setr_ps( + 8., 8., 2., 2., 16., 16., 10., 10., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_shuffle_ps() { + let a = _mm256_set_ps(1., 4., 5., 8., 9., 12., 13., 16.); + let b = _mm256_set_ps(2., 3., 6., 7., 10., 11., 14., 15.); + let r = _mm256_mask_shuffle_ps::<0b11_11_11_11>(a, 0, a, b); + assert_eq_m256(r, a); + let r = _mm256_mask_shuffle_ps::<0b00_00_11_11>(a, 0b11111111, a, b); + let e = _mm256_set_ps(7., 7., 1., 1., 15., 15., 9., 9.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_shuffle_ps() { + let a = _mm256_set_ps(1., 4., 5., 8., 9., 12., 13., 16.); + let b = _mm256_set_ps(2., 3., 6., 7., 10., 11., 14., 15.); + let r = _mm256_maskz_shuffle_ps::<0b11_11_11_11>(0, a, b); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_shuffle_ps::<0b00_00_11_11>(0b11111111, a, b); + let e = _mm256_set_ps(7., 7., 1., 1., 15., 15., 9., 9.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_shuffle_ps() { + let a = _mm_set_ps(1., 4., 5., 8.); + let b = _mm_set_ps(2., 3., 6., 7.); + let r = _mm_mask_shuffle_ps::<0b11_11_11_11>(a, 0, a, b); + assert_eq_m128(r, a); + let r = _mm_mask_shuffle_ps::<0b00_00_11_11>(a, 0b00001111, a, b); + let e = _mm_set_ps(7., 7., 1., 1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_shuffle_ps() { + let a = _mm_set_ps(1., 4., 5., 8.); + let b = _mm_set_ps(2., 3., 6., 7.); + let r = _mm_maskz_shuffle_ps::<0b11_11_11_11>(0, a, b); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_shuffle_ps::<0b00_00_11_11>(0b00001111, a, b); + let e = _mm_set_ps(7., 7., 1., 1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_shuffle_i32x4() { + let a = _mm512_setr_epi32(1, 4, 5, 8, 9, 12, 13, 16, 1, 4, 5, 8, 9, 12, 13, 16); + let b = _mm512_setr_epi32(2, 3, 6, 7, 10, 11, 14, 15, 2, 3, 6, 7, 10, 11, 14, 15); + let r = _mm512_shuffle_i32x4::<0b00_00_00_00>(a, b); + let e = _mm512_setr_epi32(1, 4, 5, 8, 1, 4, 5, 8, 2, 3, 6, 7, 2, 3, 6, 7); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_shuffle_i32x4() { + let a = _mm512_setr_epi32(1, 4, 5, 8, 9, 12, 13, 16, 1, 4, 5, 8, 9, 12, 13, 16); + let b = _mm512_setr_epi32(2, 3, 6, 7, 10, 11, 14, 15, 2, 3, 6, 7, 10, 11, 14, 15); + let r = _mm512_mask_shuffle_i32x4::<0b00_00_00_00>(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_shuffle_i32x4::<0b00_00_00_00>(a, 0b11111111_11111111, a, b); + let e = _mm512_setr_epi32(1, 4, 5, 8, 1, 4, 5, 8, 2, 3, 6, 7, 2, 3, 6, 7); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_shuffle_i32x4() { + let a = _mm512_setr_epi32(1, 4, 5, 8, 9, 12, 13, 16, 1, 4, 5, 8, 9, 12, 13, 16); + let b = _mm512_setr_epi32(2, 3, 6, 7, 10, 11, 14, 15, 2, 3, 6, 7, 10, 11, 14, 15); + let r = _mm512_maskz_shuffle_i32x4::<0b00_00_00_00>(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_shuffle_i32x4::<0b00_00_00_00>(0b00000000_11111111, a, b); + let e = _mm512_setr_epi32(1, 4, 5, 8, 1, 4, 5, 8, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_shuffle_i32x4() { + let a = _mm256_set_epi32(1, 4, 5, 8, 9, 12, 13, 16); + let b = _mm256_set_epi32(2, 3, 6, 7, 10, 11, 14, 15); + let r = _mm256_shuffle_i32x4::<0b00>(a, b); + let e = _mm256_set_epi32(10, 11, 14, 15, 9, 12, 13, 16); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_shuffle_i32x4() { + let a = _mm256_set_epi32(1, 4, 5, 8, 9, 12, 13, 16); + let b = _mm256_set_epi32(2, 3, 6, 7, 10, 11, 14, 15); + let r = _mm256_mask_shuffle_i32x4::<0b00>(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_shuffle_i32x4::<0b00>(a, 0b11111111, a, b); + let e = _mm256_set_epi32(10, 11, 14, 15, 9, 12, 13, 16); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_shuffle_i32x4() { + let a = _mm256_set_epi32(1, 4, 5, 8, 9, 12, 13, 16); + let b = _mm256_set_epi32(2, 3, 6, 7, 10, 11, 14, 15); + let r = _mm256_maskz_shuffle_i32x4::<0b00>(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_shuffle_i32x4::<0b00>(0b11111111, a, b); + let e = _mm256_set_epi32(10, 11, 14, 15, 9, 12, 13, 16); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_shuffle_f32x4() { + let a = _mm512_setr_ps( + 1., 4., 5., 8., 9., 12., 13., 16., 1., 4., 5., 8., 9., 12., 13., 16., + ); + let b = _mm512_setr_ps( + 2., 3., 6., 7., 10., 11., 14., 15., 2., 3., 6., 7., 10., 11., 14., 15., + ); + let r = _mm512_shuffle_f32x4::<0b00_00_00_00>(a, b); + let e = _mm512_setr_ps( + 1., 4., 5., 8., 1., 4., 5., 8., 2., 3., 6., 7., 2., 3., 6., 7., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_shuffle_f32x4() { + let a = _mm512_setr_ps( + 1., 4., 5., 8., 9., 12., 13., 16., 1., 4., 5., 8., 9., 12., 13., 16., + ); + let b = _mm512_setr_ps( + 2., 3., 6., 7., 10., 11., 14., 15., 2., 3., 6., 7., 10., 11., 14., 15., + ); + let r = _mm512_mask_shuffle_f32x4::<0b00_00_00_00>(a, 0, a, b); + assert_eq_m512(r, a); + let r = _mm512_mask_shuffle_f32x4::<0b00_00_00_00>(a, 0b11111111_11111111, a, b); + let e = _mm512_setr_ps( + 1., 4., 5., 8., 1., 4., 5., 8., 2., 3., 6., 7., 2., 3., 6., 7., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_shuffle_f32x4() { + let a = _mm512_setr_ps( + 1., 4., 5., 8., 9., 12., 13., 16., 1., 4., 5., 8., 9., 12., 13., 16., + ); + let b = _mm512_setr_ps( + 2., 3., 6., 7., 10., 11., 14., 15., 2., 3., 6., 7., 10., 11., 14., 15., + ); + let r = _mm512_maskz_shuffle_f32x4::<0b00_00_00_00>(0, a, b); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_shuffle_f32x4::<0b00_00_00_00>(0b00000000_11111111, a, b); + let e = _mm512_setr_ps( + 1., 4., 5., 8., 1., 4., 5., 8., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_shuffle_f32x4() { + let a = _mm256_set_ps(1., 4., 5., 8., 9., 12., 13., 16.); + let b = _mm256_set_ps(2., 3., 6., 7., 10., 11., 14., 15.); + let r = _mm256_shuffle_f32x4::<0b00>(a, b); + let e = _mm256_set_ps(10., 11., 14., 15., 9., 12., 13., 16.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_shuffle_f32x4() { + let a = _mm256_set_ps(1., 4., 5., 8., 9., 12., 13., 16.); + let b = _mm256_set_ps(2., 3., 6., 7., 10., 11., 14., 15.); + let r = _mm256_mask_shuffle_f32x4::<0b00>(a, 0, a, b); + assert_eq_m256(r, a); + let r = _mm256_mask_shuffle_f32x4::<0b00>(a, 0b11111111, a, b); + let e = _mm256_set_ps(10., 11., 14., 15., 9., 12., 13., 16.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_shuffle_f32x4() { + let a = _mm256_set_ps(1., 4., 5., 8., 9., 12., 13., 16.); + let b = _mm256_set_ps(2., 3., 6., 7., 10., 11., 14., 15.); + let r = _mm256_maskz_shuffle_f32x4::<0b00>(0, a, b); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_shuffle_f32x4::<0b00>(0b11111111, a, b); + let e = _mm256_set_ps(10., 11., 14., 15., 9., 12., 13., 16.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_extractf32x4_ps() { + let a = _mm512_setr_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + let r = _mm512_extractf32x4_ps::<1>(a); + let e = _mm_setr_ps(5., 6., 7., 8.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_extractf32x4_ps() { + let a = _mm512_setr_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + let src = _mm_set1_ps(100.); + let r = _mm512_mask_extractf32x4_ps::<1>(src, 0, a); + assert_eq_m128(r, src); + let r = _mm512_mask_extractf32x4_ps::<1>(src, 0b11111111, a); + let e = _mm_setr_ps(5., 6., 7., 8.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_extractf32x4_ps() { + let a = _mm512_setr_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + let r = _mm512_maskz_extractf32x4_ps::<1>(0, a); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm512_maskz_extractf32x4_ps::<1>(0b00000001, a); + let e = _mm_setr_ps(5., 0., 0., 0.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_extractf32x4_ps() { + let a = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); + let r = _mm256_extractf32x4_ps::<1>(a); + let e = _mm_set_ps(1., 2., 3., 4.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_extractf32x4_ps() { + let a = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); + let src = _mm_set1_ps(100.); + let r = _mm256_mask_extractf32x4_ps::<1>(src, 0, a); + assert_eq_m128(r, src); + let r = _mm256_mask_extractf32x4_ps::<1>(src, 0b00001111, a); + let e = _mm_set_ps(1., 2., 3., 4.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_extractf32x4_ps() { + let a = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); + let r = _mm256_maskz_extractf32x4_ps::<1>(0, a); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm256_maskz_extractf32x4_ps::<1>(0b00001111, a); + let e = _mm_set_ps(1., 2., 3., 4.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_extracti32x4_epi32() { + let a = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r = _mm512_extracti32x4_epi32::<1>(a); + let e = _mm_setr_epi32(5, 6, 7, 8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_extracti32x4_epi32() { + let a = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let src = _mm_set1_epi32(100); + let r = _mm512_mask_extracti32x4_epi32::<1>(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm512_mask_extracti32x4_epi32::<1>(src, 0b11111111, a); + let e = _mm_setr_epi32(5, 6, 7, 8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm512_maskz_extracti32x4_epi32() { + let a = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r = _mm512_maskz_extracti32x4_epi32::<1>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm512_maskz_extracti32x4_epi32::<1>(0b00000001, a); + let e = _mm_setr_epi32(5, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_extracti32x4_epi32() { + let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let r = _mm256_extracti32x4_epi32::<1>(a); + let e = _mm_set_epi32(1, 2, 3, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_extracti32x4_epi32() { + let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let src = _mm_set1_epi32(100); + let r = _mm256_mask_extracti32x4_epi32::<1>(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_extracti32x4_epi32::<1>(src, 0b00001111, a); + let e = _mm_set_epi32(1, 2, 3, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_extracti32x4_epi32() { + let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let r = _mm256_maskz_extracti32x4_epi32::<1>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_extracti32x4_epi32::<1>(0b00001111, a); + let e = _mm_set_epi32(1, 2, 3, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_moveldup_ps() { + let a = _mm512_setr_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + let r = _mm512_moveldup_ps(a); + let e = _mm512_setr_ps( + 1., 1., 3., 3., 5., 5., 7., 7., 9., 9., 11., 11., 13., 13., 15., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_moveldup_ps() { + let a = _mm512_setr_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + let r = _mm512_mask_moveldup_ps(a, 0, a); + assert_eq_m512(r, a); + let r = _mm512_mask_moveldup_ps(a, 0b11111111_11111111, a); + let e = _mm512_setr_ps( + 1., 1., 3., 3., 5., 5., 7., 7., 9., 9., 11., 11., 13., 13., 15., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_moveldup_ps() { + let a = _mm512_setr_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + let r = _mm512_maskz_moveldup_ps(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_moveldup_ps(0b00000000_11111111, a); + let e = _mm512_setr_ps( + 1., 1., 3., 3., 5., 5., 7., 7., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_moveldup_ps() { + let a = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); + let r = _mm256_mask_moveldup_ps(a, 0, a); + assert_eq_m256(r, a); + let r = _mm256_mask_moveldup_ps(a, 0b11111111, a); + let e = _mm256_set_ps(2., 2., 4., 4., 6., 6., 8., 8.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_moveldup_ps() { + let a = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); + let r = _mm256_maskz_moveldup_ps(0, a); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_moveldup_ps(0b11111111, a); + let e = _mm256_set_ps(2., 2., 4., 4., 6., 6., 8., 8.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_moveldup_ps() { + let a = _mm_set_ps(1., 2., 3., 4.); + let r = _mm_mask_moveldup_ps(a, 0, a); + assert_eq_m128(r, a); + let r = _mm_mask_moveldup_ps(a, 0b00001111, a); + let e = _mm_set_ps(2., 2., 4., 4.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_moveldup_ps() { + let a = _mm_set_ps(1., 2., 3., 4.); + let r = _mm_maskz_moveldup_ps(0, a); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_moveldup_ps(0b00001111, a); + let e = _mm_set_ps(2., 2., 4., 4.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_movehdup_ps() { + let a = _mm512_setr_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + let r = _mm512_movehdup_ps(a); + let e = _mm512_setr_ps( + 2., 2., 4., 4., 6., 6., 8., 8., 10., 10., 12., 12., 14., 14., 16., 16., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_movehdup_ps() { + let a = _mm512_setr_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + let r = _mm512_mask_movehdup_ps(a, 0, a); + assert_eq_m512(r, a); + let r = _mm512_mask_movehdup_ps(a, 0b11111111_11111111, a); + let e = _mm512_setr_ps( + 2., 2., 4., 4., 6., 6., 8., 8., 10., 10., 12., 12., 14., 14., 16., 16., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_movehdup_ps() { + let a = _mm512_setr_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + let r = _mm512_maskz_movehdup_ps(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_movehdup_ps(0b00000000_11111111, a); + let e = _mm512_setr_ps( + 2., 2., 4., 4., 6., 6., 8., 8., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_movehdup_ps() { + let a = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); + let r = _mm256_mask_movehdup_ps(a, 0, a); + assert_eq_m256(r, a); + let r = _mm256_mask_movehdup_ps(a, 0b11111111, a); + let e = _mm256_set_ps(1., 1., 3., 3., 5., 5., 7., 7.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_movehdup_ps() { + let a = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); + let r = _mm256_maskz_movehdup_ps(0, a); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_movehdup_ps(0b11111111, a); + let e = _mm256_set_ps(1., 1., 3., 3., 5., 5., 7., 7.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_movehdup_ps() { + let a = _mm_set_ps(1., 2., 3., 4.); + let r = _mm_mask_movehdup_ps(a, 0, a); + assert_eq_m128(r, a); + let r = _mm_mask_movehdup_ps(a, 0b00001111, a); + let e = _mm_set_ps(1., 1., 3., 3.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_movehdup_ps() { + let a = _mm_set_ps(1., 2., 3., 4.); + let r = _mm_maskz_movehdup_ps(0, a); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_movehdup_ps(0b00001111, a); + let e = _mm_set_ps(1., 1., 3., 3.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_inserti32x4() { + let a = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b = _mm_setr_epi32(17, 18, 19, 20); + let r = _mm512_inserti32x4::<0>(a, b); + let e = _mm512_setr_epi32(17, 18, 19, 20, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_inserti32x4() { + let a = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b = _mm_setr_epi32(17, 18, 19, 20); + let r = _mm512_mask_inserti32x4::<0>(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_inserti32x4::<0>(a, 0b11111111_11111111, a, b); + let e = _mm512_setr_epi32(17, 18, 19, 20, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_inserti32x4() { + let a = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b = _mm_setr_epi32(17, 18, 19, 20); + let r = _mm512_maskz_inserti32x4::<0>(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_inserti32x4::<0>(0b00000000_11111111, a, b); + let e = _mm512_setr_epi32(17, 18, 19, 20, 5, 6, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_inserti32x4() { + let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm_set_epi32(17, 18, 19, 20); + let r = _mm256_inserti32x4::<1>(a, b); + let e = _mm256_set_epi32(17, 18, 19, 20, 5, 6, 7, 8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_inserti32x4() { + let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm_set_epi32(17, 18, 19, 20); + let r = _mm256_mask_inserti32x4::<0>(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_inserti32x4::<1>(a, 0b11111111, a, b); + let e = _mm256_set_epi32(17, 18, 19, 20, 5, 6, 7, 8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_inserti32x4() { + let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm_set_epi32(17, 18, 19, 20); + let r = _mm256_maskz_inserti32x4::<0>(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_inserti32x4::<1>(0b11111111, a, b); + let e = _mm256_set_epi32(17, 18, 19, 20, 5, 6, 7, 8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_insertf32x4() { + let a = _mm512_setr_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + let b = _mm_setr_ps(17., 18., 19., 20.); + let r = _mm512_insertf32x4::<0>(a, b); + let e = _mm512_setr_ps( + 17., 18., 19., 20., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_insertf32x4() { + let a = _mm512_setr_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + let b = _mm_setr_ps(17., 18., 19., 20.); + let r = _mm512_mask_insertf32x4::<0>(a, 0, a, b); + assert_eq_m512(r, a); + let r = _mm512_mask_insertf32x4::<0>(a, 0b11111111_11111111, a, b); + let e = _mm512_setr_ps( + 17., 18., 19., 20., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_insertf32x4() { + let a = _mm512_setr_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + let b = _mm_setr_ps(17., 18., 19., 20.); + let r = _mm512_maskz_insertf32x4::<0>(0, a, b); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_insertf32x4::<0>(0b00000000_11111111, a, b); + let e = _mm512_setr_ps( + 17., 18., 19., 20., 5., 6., 7., 8., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_insertf32x4() { + let a = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); + let b = _mm_set_ps(17., 18., 19., 20.); + let r = _mm256_insertf32x4::<1>(a, b); + let e = _mm256_set_ps(17., 18., 19., 20., 5., 6., 7., 8.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_insertf32x4() { + let a = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); + let b = _mm_set_ps(17., 18., 19., 20.); + let r = _mm256_mask_insertf32x4::<0>(a, 0, a, b); + assert_eq_m256(r, a); + let r = _mm256_mask_insertf32x4::<1>(a, 0b11111111, a, b); + let e = _mm256_set_ps(17., 18., 19., 20., 5., 6., 7., 8.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_insertf32x4() { + let a = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); + let b = _mm_set_ps(17., 18., 19., 20.); + let r = _mm256_maskz_insertf32x4::<0>(0, a, b); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_insertf32x4::<1>(0b11111111, a, b); + let e = _mm256_set_ps(17., 18., 19., 20., 5., 6., 7., 8.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_castps128_ps512() { + let a = _mm_setr_ps(17., 18., 19., 20.); + let r = _mm512_castps128_ps512(a); + let e = _mm512_setr_ps( + 17., 18., 19., 20., -1., -1., -1., -1., -1., -1., -1., -1., -1., -1., -1., -1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_castps256_ps512() { + let a = _mm256_setr_ps(17., 18., 19., 20., 21., 22., 23., 24.); + let r = _mm512_castps256_ps512(a); + let e = _mm512_setr_ps( + 17., 18., 19., 20., 21., 22., 23., 24., -1., -1., -1., -1., -1., -1., -1., -1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_zextps128_ps512() { + let a = _mm_setr_ps(17., 18., 19., 20.); + let r = _mm512_zextps128_ps512(a); + let e = _mm512_setr_ps( + 17., 18., 19., 20., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_zextps256_ps512() { + let a = _mm256_setr_ps(17., 18., 19., 20., 21., 22., 23., 24.); + let r = _mm512_zextps256_ps512(a); + let e = _mm512_setr_ps( + 17., 18., 19., 20., 21., 22., 23., 24., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_castps512_ps128() { + let a = _mm512_setr_ps( + 17., 18., 19., 20., -1., -1., -1., -1., -1., -1., -1., -1., -1., -1., -1., -1., + ); + let r = _mm512_castps512_ps128(a); + let e = _mm_setr_ps(17., 18., 19., 20.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_castps512_ps256() { + let a = _mm512_setr_ps( + 17., 18., 19., 20., 21., 22., 23., 24., -1., -1., -1., -1., -1., -1., -1., -1., + ); + let r = _mm512_castps512_ps256(a); + let e = _mm256_setr_ps(17., 18., 19., 20., 21., 22., 23., 24.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_castps_pd() { + let a = _mm512_set1_ps(1.); + let r = _mm512_castps_pd(a); + let e = _mm512_set1_pd(0.007812501848093234); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_castps_si512() { + let a = _mm512_set1_ps(1.); + let r = _mm512_castps_si512(a); + let e = _mm512_set1_epi32(1065353216); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_broadcastd_epi32() { + let a = _mm_set_epi32(17, 18, 19, 20); + let r = _mm512_broadcastd_epi32(a); + let e = _mm512_set1_epi32(20); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_broadcastd_epi32() { + let src = _mm512_set1_epi32(20); + let a = _mm_set_epi32(17, 18, 19, 20); + let r = _mm512_mask_broadcastd_epi32(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_broadcastd_epi32(src, 0b11111111_11111111, a); + let e = _mm512_set1_epi32(20); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_broadcastd_epi32() { + let a = _mm_set_epi32(17, 18, 19, 20); + let r = _mm512_maskz_broadcastd_epi32(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_broadcastd_epi32(0b00000000_11111111, a); + let e = _mm512_setr_epi32(20, 20, 20, 20, 20, 20, 20, 20, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_broadcastd_epi32() { + let src = _mm256_set1_epi32(20); + let a = _mm_set_epi32(17, 18, 19, 20); + let r = _mm256_mask_broadcastd_epi32(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_broadcastd_epi32(src, 0b11111111, a); + let e = _mm256_set1_epi32(20); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_broadcastd_epi32() { + let a = _mm_set_epi32(17, 18, 19, 20); + let r = _mm256_maskz_broadcastd_epi32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_broadcastd_epi32(0b11111111, a); + let e = _mm256_set1_epi32(20); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_broadcastd_epi32() { + let src = _mm_set1_epi32(20); + let a = _mm_set_epi32(17, 18, 19, 20); + let r = _mm_mask_broadcastd_epi32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_broadcastd_epi32(src, 0b00001111, a); + let e = _mm_set1_epi32(20); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_broadcastd_epi32() { + let a = _mm_set_epi32(17, 18, 19, 20); + let r = _mm_maskz_broadcastd_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_broadcastd_epi32(0b00001111, a); + let e = _mm_set1_epi32(20); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_broadcastss_ps() { + let a = _mm_set_ps(17., 18., 19., 20.); + let r = _mm512_broadcastss_ps(a); + let e = _mm512_set1_ps(20.); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_broadcastss_ps() { + let src = _mm512_set1_ps(20.); + let a = _mm_set_ps(17., 18., 19., 20.); + let r = _mm512_mask_broadcastss_ps(src, 0, a); + assert_eq_m512(r, src); + let r = _mm512_mask_broadcastss_ps(src, 0b11111111_11111111, a); + let e = _mm512_set1_ps(20.); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_broadcastss_ps() { + let a = _mm_set_ps(17., 18., 19., 20.); + let r = _mm512_maskz_broadcastss_ps(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_broadcastss_ps(0b00000000_11111111, a); + let e = _mm512_setr_ps( + 20., 20., 20., 20., 20., 20., 20., 20., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_broadcastss_ps() { + let src = _mm256_set1_ps(20.); + let a = _mm_set_ps(17., 18., 19., 20.); + let r = _mm256_mask_broadcastss_ps(src, 0, a); + assert_eq_m256(r, src); + let r = _mm256_mask_broadcastss_ps(src, 0b11111111, a); + let e = _mm256_set1_ps(20.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_broadcastss_ps() { + let a = _mm_set_ps(17., 18., 19., 20.); + let r = _mm256_maskz_broadcastss_ps(0, a); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_broadcastss_ps(0b11111111, a); + let e = _mm256_set1_ps(20.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_broadcastss_ps() { + let src = _mm_set1_ps(20.); + let a = _mm_set_ps(17., 18., 19., 20.); + let r = _mm_mask_broadcastss_ps(src, 0, a); + assert_eq_m128(r, src); + let r = _mm_mask_broadcastss_ps(src, 0b00001111, a); + let e = _mm_set1_ps(20.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_broadcastss_ps() { + let a = _mm_set_ps(17., 18., 19., 20.); + let r = _mm_maskz_broadcastss_ps(0, a); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_broadcastss_ps(0b00001111, a); + let e = _mm_set1_ps(20.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_broadcast_i32x4() { + let a = _mm_set_epi32(17, 18, 19, 20); + let r = _mm512_broadcast_i32x4(a); + let e = _mm512_set_epi32( + 17, 18, 19, 20, 17, 18, 19, 20, 17, 18, 19, 20, 17, 18, 19, 20, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_broadcast_i32x4() { + let src = _mm512_set1_epi32(20); + let a = _mm_set_epi32(17, 18, 19, 20); + let r = _mm512_mask_broadcast_i32x4(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_broadcast_i32x4(src, 0b11111111_11111111, a); + let e = _mm512_set_epi32( + 17, 18, 19, 20, 17, 18, 19, 20, 17, 18, 19, 20, 17, 18, 19, 20, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_broadcast_i32x4() { + let a = _mm_set_epi32(17, 18, 19, 20); + let r = _mm512_maskz_broadcast_i32x4(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_broadcast_i32x4(0b00000000_11111111, a); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 17, 18, 19, 20, 17, 18, 19, 20); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_broadcast_i32x4() { + let a = _mm_set_epi32(17, 18, 19, 20); + let r = _mm256_broadcast_i32x4(a); + let e = _mm256_set_epi32(17, 18, 19, 20, 17, 18, 19, 20); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_broadcast_i32x4() { + let src = _mm256_set1_epi32(20); + let a = _mm_set_epi32(17, 18, 19, 20); + let r = _mm256_mask_broadcast_i32x4(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_broadcast_i32x4(src, 0b11111111, a); + let e = _mm256_set_epi32(17, 18, 19, 20, 17, 18, 19, 20); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_broadcast_i32x4() { + let a = _mm_set_epi32(17, 18, 19, 20); + let r = _mm256_maskz_broadcast_i32x4(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_broadcast_i32x4(0b11111111, a); + let e = _mm256_set_epi32(17, 18, 19, 20, 17, 18, 19, 20); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_broadcast_f32x4() { + let a = _mm_set_ps(17., 18., 19., 20.); + let r = _mm512_broadcast_f32x4(a); + let e = _mm512_set_ps( + 17., 18., 19., 20., 17., 18., 19., 20., 17., 18., 19., 20., 17., 18., 19., 20., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_broadcast_f32x4() { + let src = _mm512_set1_ps(20.); + let a = _mm_set_ps(17., 18., 19., 20.); + let r = _mm512_mask_broadcast_f32x4(src, 0, a); + assert_eq_m512(r, src); + let r = _mm512_mask_broadcast_f32x4(src, 0b11111111_11111111, a); + let e = _mm512_set_ps( + 17., 18., 19., 20., 17., 18., 19., 20., 17., 18., 19., 20., 17., 18., 19., 20., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_broadcast_f32x4() { + let a = _mm_set_ps(17., 18., 19., 20.); + let r = _mm512_maskz_broadcast_f32x4(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_broadcast_f32x4(0b00000000_11111111, a); + let e = _mm512_set_ps( + 0., 0., 0., 0., 0., 0., 0., 0., 17., 18., 19., 20., 17., 18., 19., 20., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_broadcast_f32x4() { + let a = _mm_set_ps(17., 18., 19., 20.); + let r = _mm256_broadcast_f32x4(a); + let e = _mm256_set_ps(17., 18., 19., 20., 17., 18., 19., 20.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_broadcast_f32x4() { + let src = _mm256_set1_ps(20.); + let a = _mm_set_ps(17., 18., 19., 20.); + let r = _mm256_mask_broadcast_f32x4(src, 0, a); + assert_eq_m256(r, src); + let r = _mm256_mask_broadcast_f32x4(src, 0b11111111, a); + let e = _mm256_set_ps(17., 18., 19., 20., 17., 18., 19., 20.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_broadcast_f32x4() { + let a = _mm_set_ps(17., 18., 19., 20.); + let r = _mm256_maskz_broadcast_f32x4(0, a); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_broadcast_f32x4(0b11111111, a); + let e = _mm256_set_ps(17., 18., 19., 20., 17., 18., 19., 20.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_blend_epi32() { + let a = _mm512_set1_epi32(1); + let b = _mm512_set1_epi32(2); + let r = _mm512_mask_blend_epi32(0b11111111_00000000, a, b); + let e = _mm512_set_epi32(2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_blend_epi32() { + let a = _mm256_set1_epi32(1); + let b = _mm256_set1_epi32(2); + let r = _mm256_mask_blend_epi32(0b11111111, a, b); + let e = _mm256_set1_epi32(2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_blend_epi32() { + let a = _mm_set1_epi32(1); + let b = _mm_set1_epi32(2); + let r = _mm_mask_blend_epi32(0b00001111, a, b); + let e = _mm_set1_epi32(2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_blend_ps() { + let a = _mm512_set1_ps(1.); + let b = _mm512_set1_ps(2.); + let r = _mm512_mask_blend_ps(0b11111111_00000000, a, b); + let e = _mm512_set_ps( + 2., 2., 2., 2., 2., 2., 2., 2., 1., 1., 1., 1., 1., 1., 1., 1., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_blend_ps() { + let a = _mm256_set1_ps(1.); + let b = _mm256_set1_ps(2.); + let r = _mm256_mask_blend_ps(0b11111111, a, b); + let e = _mm256_set1_ps(2.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_blend_ps() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let r = _mm_mask_blend_ps(0b00001111, a, b); + let e = _mm_set1_ps(2.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_unpackhi_epi32() { + let a = _mm512_set_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b = _mm512_set_epi32( + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ); + let r = _mm512_unpackhi_epi32(a, b); + let e = _mm512_set_epi32(17, 1, 18, 2, 21, 5, 22, 6, 25, 9, 26, 10, 29, 13, 30, 14); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_unpackhi_epi32() { + let a = _mm512_set_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b = _mm512_set_epi32( + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ); + let r = _mm512_mask_unpackhi_epi32(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_unpackhi_epi32(a, 0b11111111_11111111, a, b); + let e = _mm512_set_epi32(17, 1, 18, 2, 21, 5, 22, 6, 25, 9, 26, 10, 29, 13, 30, 14); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_unpackhi_epi32() { + let a = _mm512_set_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b = _mm512_set_epi32( + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ); + let r = _mm512_maskz_unpackhi_epi32(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_unpackhi_epi32(0b00000000_11111111, a, b); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 25, 9, 26, 10, 29, 13, 30, 14); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_unpackhi_epi32() { + let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm256_set_epi32(17, 18, 19, 20, 21, 22, 23, 24); + let r = _mm256_mask_unpackhi_epi32(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_unpackhi_epi32(a, 0b11111111, a, b); + let e = _mm256_set_epi32(17, 1, 18, 2, 21, 5, 22, 6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_unpackhi_epi32() { + let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm256_set_epi32(17, 18, 19, 20, 21, 22, 23, 24); + let r = _mm256_maskz_unpackhi_epi32(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_unpackhi_epi32(0b11111111, a, b); + let e = _mm256_set_epi32(17, 1, 18, 2, 21, 5, 22, 6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_unpackhi_epi32() { + let a = _mm_set_epi32(1, 2, 3, 4); + let b = _mm_set_epi32(17, 18, 19, 20); + let r = _mm_mask_unpackhi_epi32(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_unpackhi_epi32(a, 0b00001111, a, b); + let e = _mm_set_epi32(17, 1, 18, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_unpackhi_epi32() { + let a = _mm_set_epi32(1, 2, 3, 4); + let b = _mm_set_epi32(17, 18, 19, 20); + let r = _mm_maskz_unpackhi_epi32(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_unpackhi_epi32(0b00001111, a, b); + let e = _mm_set_epi32(17, 1, 18, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_unpackhi_ps() { + let a = _mm512_set_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + let b = _mm512_set_ps( + 17., 18., 19., 20., 21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., + ); + let r = _mm512_unpackhi_ps(a, b); + let e = _mm512_set_ps( + 17., 1., 18., 2., 21., 5., 22., 6., 25., 9., 26., 10., 29., 13., 30., 14., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_unpackhi_ps() { + let a = _mm512_set_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + let b = _mm512_set_ps( + 17., 18., 19., 20., 21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., + ); + let r = _mm512_mask_unpackhi_ps(a, 0, a, b); + assert_eq_m512(r, a); + let r = _mm512_mask_unpackhi_ps(a, 0b11111111_11111111, a, b); + let e = _mm512_set_ps( + 17., 1., 18., 2., 21., 5., 22., 6., 25., 9., 26., 10., 29., 13., 30., 14., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_unpackhi_ps() { + let a = _mm512_set_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + let b = _mm512_set_ps( + 17., 18., 19., 20., 21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., + ); + let r = _mm512_maskz_unpackhi_ps(0, a, b); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_unpackhi_ps(0b00000000_11111111, a, b); + let e = _mm512_set_ps( + 0., 0., 0., 0., 0., 0., 0., 0., 25., 9., 26., 10., 29., 13., 30., 14., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_unpackhi_ps() { + let a = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); + let b = _mm256_set_ps(17., 18., 19., 20., 21., 22., 23., 24.); + let r = _mm256_mask_unpackhi_ps(a, 0, a, b); + assert_eq_m256(r, a); + let r = _mm256_mask_unpackhi_ps(a, 0b11111111, a, b); + let e = _mm256_set_ps(17., 1., 18., 2., 21., 5., 22., 6.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_unpackhi_ps() { + let a = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); + let b = _mm256_set_ps(17., 18., 19., 20., 21., 22., 23., 24.); + let r = _mm256_maskz_unpackhi_ps(0, a, b); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_unpackhi_ps(0b11111111, a, b); + let e = _mm256_set_ps(17., 1., 18., 2., 21., 5., 22., 6.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_unpackhi_ps() { + let a = _mm_set_ps(1., 2., 3., 4.); + let b = _mm_set_ps(17., 18., 19., 20.); + let r = _mm_mask_unpackhi_ps(a, 0, a, b); + assert_eq_m128(r, a); + let r = _mm_mask_unpackhi_ps(a, 0b00001111, a, b); + let e = _mm_set_ps(17., 1., 18., 2.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_unpackhi_ps() { + let a = _mm_set_ps(1., 2., 3., 4.); + let b = _mm_set_ps(17., 18., 19., 20.); + let r = _mm_maskz_unpackhi_ps(0, a, b); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_unpackhi_ps(0b00001111, a, b); + let e = _mm_set_ps(17., 1., 18., 2.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_unpacklo_epi32() { + let a = _mm512_set_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b = _mm512_set_epi32( + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ); + let r = _mm512_unpacklo_epi32(a, b); + let e = _mm512_set_epi32(19, 3, 20, 4, 23, 7, 24, 8, 27, 11, 28, 12, 31, 15, 32, 16); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_unpacklo_epi32() { + let a = _mm512_set_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b = _mm512_set_epi32( + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ); + let r = _mm512_mask_unpacklo_epi32(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_unpacklo_epi32(a, 0b11111111_11111111, a, b); + let e = _mm512_set_epi32(19, 3, 20, 4, 23, 7, 24, 8, 27, 11, 28, 12, 31, 15, 32, 16); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_unpacklo_epi32() { + let a = _mm512_set_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b = _mm512_set_epi32( + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ); + let r = _mm512_maskz_unpacklo_epi32(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_unpacklo_epi32(0b00000000_11111111, a, b); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 27, 11, 28, 12, 31, 15, 32, 16); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_unpacklo_epi32() { + let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm256_set_epi32(17, 18, 19, 20, 21, 22, 23, 24); + let r = _mm256_mask_unpacklo_epi32(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_unpacklo_epi32(a, 0b11111111, a, b); + let e = _mm256_set_epi32(19, 3, 20, 4, 23, 7, 24, 8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_unpacklo_epi32() { + let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm256_set_epi32(17, 18, 19, 20, 21, 22, 23, 24); + let r = _mm256_maskz_unpacklo_epi32(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_unpacklo_epi32(0b11111111, a, b); + let e = _mm256_set_epi32(19, 3, 20, 4, 23, 7, 24, 8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_unpacklo_epi32() { + let a = _mm_set_epi32(1, 2, 3, 4); + let b = _mm_set_epi32(17, 18, 19, 20); + let r = _mm_mask_unpacklo_epi32(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_unpacklo_epi32(a, 0b00001111, a, b); + let e = _mm_set_epi32(19, 3, 20, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_unpacklo_epi32() { + let a = _mm_set_epi32(1, 2, 3, 4); + let b = _mm_set_epi32(17, 18, 19, 20); + let r = _mm_maskz_unpacklo_epi32(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_unpacklo_epi32(0b00001111, a, b); + let e = _mm_set_epi32(19, 3, 20, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_unpacklo_ps() { + let a = _mm512_set_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + let b = _mm512_set_ps( + 17., 18., 19., 20., 21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., + ); + let r = _mm512_unpacklo_ps(a, b); + let e = _mm512_set_ps( + 19., 3., 20., 4., 23., 7., 24., 8., 27., 11., 28., 12., 31., 15., 32., 16., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_unpacklo_ps() { + let a = _mm512_set_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + let b = _mm512_set_ps( + 17., 18., 19., 20., 21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., + ); + let r = _mm512_mask_unpacklo_ps(a, 0, a, b); + assert_eq_m512(r, a); + let r = _mm512_mask_unpacklo_ps(a, 0b11111111_11111111, a, b); + let e = _mm512_set_ps( + 19., 3., 20., 4., 23., 7., 24., 8., 27., 11., 28., 12., 31., 15., 32., 16., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_unpacklo_ps() { + let a = _mm512_set_ps( + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ); + let b = _mm512_set_ps( + 17., 18., 19., 20., 21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., + ); + let r = _mm512_maskz_unpacklo_ps(0, a, b); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_unpacklo_ps(0b00000000_11111111, a, b); + let e = _mm512_set_ps( + 0., 0., 0., 0., 0., 0., 0., 0., 27., 11., 28., 12., 31., 15., 32., 16., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_unpacklo_ps() { + let a = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); + let b = _mm256_set_ps(17., 18., 19., 20., 21., 22., 23., 24.); + let r = _mm256_mask_unpacklo_ps(a, 0, a, b); + assert_eq_m256(r, a); + let r = _mm256_mask_unpacklo_ps(a, 0b11111111, a, b); + let e = _mm256_set_ps(19., 3., 20., 4., 23., 7., 24., 8.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_unpacklo_ps() { + let a = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); + let b = _mm256_set_ps(17., 18., 19., 20., 21., 22., 23., 24.); + let r = _mm256_maskz_unpacklo_ps(0, a, b); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_unpacklo_ps(0b11111111, a, b); + let e = _mm256_set_ps(19., 3., 20., 4., 23., 7., 24., 8.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_unpacklo_ps() { + let a = _mm_set_ps(1., 2., 3., 4.); + let b = _mm_set_ps(17., 18., 19., 20.); + let r = _mm_mask_unpacklo_ps(a, 0, a, b); + assert_eq_m128(r, a); + let r = _mm_mask_unpacklo_ps(a, 0b00001111, a, b); + let e = _mm_set_ps(19., 3., 20., 4.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_unpacklo_ps() { + let a = _mm_set_ps(1., 2., 3., 4.); + let b = _mm_set_ps(17., 18., 19., 20.); + let r = _mm_maskz_unpacklo_ps(0, a, b); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_unpacklo_ps(0b00001111, a, b); + let e = _mm_set_ps(19., 3., 20., 4.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_alignr_epi32() { + let a = _mm512_set_epi32(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + let b = _mm512_set_epi32( + 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, + ); + let r = _mm512_alignr_epi32::<0>(a, b); + assert_eq_m512i(r, b); + let r = _mm512_alignr_epi32::<16>(a, b); + assert_eq_m512i(r, b); + let r = _mm512_alignr_epi32::<1>(a, b); + let e = _mm512_set_epi32( + 1, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_alignr_epi32() { + let a = _mm512_set_epi32(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + let b = _mm512_set_epi32( + 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, + ); + let r = _mm512_mask_alignr_epi32::<1>(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_alignr_epi32::<1>(a, 0b11111111_11111111, a, b); + let e = _mm512_set_epi32( + 1, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_alignr_epi32() { + let a = _mm512_set_epi32(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + let b = _mm512_set_epi32( + 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, + ); + let r = _mm512_maskz_alignr_epi32::<1>(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_alignr_epi32::<1>(0b00000000_11111111, a, b); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 25, 24, 23, 22, 21, 20, 19, 18); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_alignr_epi32() { + let a = _mm256_set_epi32(8, 7, 6, 5, 4, 3, 2, 1); + let b = _mm256_set_epi32(16, 15, 14, 13, 12, 11, 10, 9); + let r = _mm256_alignr_epi32::<0>(a, b); + assert_eq_m256i(r, b); + let r = _mm256_alignr_epi32::<1>(a, b); + let e = _mm256_set_epi32(1, 16, 15, 14, 13, 12, 11, 10); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_alignr_epi32() { + let a = _mm256_set_epi32(8, 7, 6, 5, 4, 3, 2, 1); + let b = _mm256_set_epi32(16, 15, 14, 13, 12, 11, 10, 9); + let r = _mm256_mask_alignr_epi32::<1>(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_alignr_epi32::<1>(a, 0b11111111, a, b); + let e = _mm256_set_epi32(1, 16, 15, 14, 13, 12, 11, 10); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_alignr_epi32() { + let a = _mm256_set_epi32(8, 7, 6, 5, 4, 3, 2, 1); + let b = _mm256_set_epi32(16, 15, 14, 13, 12, 11, 10, 9); + let r = _mm256_maskz_alignr_epi32::<1>(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_alignr_epi32::<1>(0b11111111, a, b); + let e = _mm256_set_epi32(1, 16, 15, 14, 13, 12, 11, 10); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_alignr_epi32() { + let a = _mm_set_epi32(4, 3, 2, 1); + let b = _mm_set_epi32(8, 7, 6, 5); + let r = _mm_alignr_epi32::<0>(a, b); + assert_eq_m128i(r, b); + let r = _mm_alignr_epi32::<1>(a, b); + let e = _mm_set_epi32(1, 8, 7, 6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_alignr_epi32() { + let a = _mm_set_epi32(4, 3, 2, 1); + let b = _mm_set_epi32(8, 7, 6, 5); + let r = _mm_mask_alignr_epi32::<1>(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_alignr_epi32::<1>(a, 0b00001111, a, b); + let e = _mm_set_epi32(1, 8, 7, 6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_alignr_epi32() { + let a = _mm_set_epi32(4, 3, 2, 1); + let b = _mm_set_epi32(8, 7, 6, 5); + let r = _mm_maskz_alignr_epi32::<1>(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_alignr_epi32::<1>(0b00001111, a, b); + let e = _mm_set_epi32(1, 8, 7, 6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_and_epi32() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 1 << 1 | 1 << 2, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 3, + ); + #[rustfmt::skip] + let b = _mm512_set_epi32( + 1 << 1, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 3 | 1 << 4, + ); + let r = _mm512_and_epi32(a, b); + let e = _mm512_set_epi32(1 << 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 << 3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_and_epi32() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 1 << 1 | 1 << 2, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 3, + ); + #[rustfmt::skip] + let b = _mm512_set_epi32( + 1 << 1, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 3 | 1 << 4, + ); + let r = _mm512_mask_and_epi32(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_and_epi32(a, 0b01111111_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi32( + 1 << 1 | 1 << 2, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 3, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_and_epi32() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 1 << 1 | 1 << 2, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 3, + ); + #[rustfmt::skip] + let b = _mm512_set_epi32( + 1 << 1, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 3 | 1 << 4, + ); + let r = _mm512_maskz_and_epi32(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_and_epi32(0b00000000_11111111, a, b); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 << 3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_and_epi32() { + let a = _mm256_set1_epi32(1 << 1 | 1 << 2); + let b = _mm256_set1_epi32(1 << 1); + let r = _mm256_mask_and_epi32(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_and_epi32(a, 0b11111111, a, b); + let e = _mm256_set1_epi32(1 << 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_and_epi32() { + let a = _mm256_set1_epi32(1 << 1 | 1 << 2); + let b = _mm256_set1_epi32(1 << 1); + let r = _mm256_maskz_and_epi32(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_and_epi32(0b11111111, a, b); + let e = _mm256_set1_epi32(1 << 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_and_epi32() { + let a = _mm_set1_epi32(1 << 1 | 1 << 2); + let b = _mm_set1_epi32(1 << 1); + let r = _mm_mask_and_epi32(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_and_epi32(a, 0b00001111, a, b); + let e = _mm_set1_epi32(1 << 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_and_epi32() { + let a = _mm_set1_epi32(1 << 1 | 1 << 2); + let b = _mm_set1_epi32(1 << 1); + let r = _mm_maskz_and_epi32(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_and_epi32(0b00001111, a, b); + let e = _mm_set1_epi32(1 << 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_and_si512() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 1 << 1 | 1 << 2, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 3, + ); + #[rustfmt::skip] + let b = _mm512_set_epi32( + 1 << 1, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 3 | 1 << 4, + ); + let r = _mm512_and_epi32(a, b); + let e = _mm512_set_epi32(1 << 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 << 3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_or_epi32() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 1 << 1 | 1 << 2, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 3, + ); + #[rustfmt::skip] + let b = _mm512_set_epi32( + 1 << 1, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 3 | 1 << 4, + ); + let r = _mm512_or_epi32(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi32( + 1 << 1 | 1 << 2, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 3 | 1 << 4, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_or_epi32() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 1 << 1 | 1 << 2, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 3, + ); + #[rustfmt::skip] + let b = _mm512_set_epi32( + 1 << 1, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 3 | 1 << 4, + ); + let r = _mm512_mask_or_epi32(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_or_epi32(a, 0b11111111_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi32( + 1 << 1 | 1 << 2, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 3 | 1 << 4, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_or_epi32() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 1 << 1 | 1 << 2, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 3, + ); + #[rustfmt::skip] + let b = _mm512_set_epi32( + 1 << 1, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 3 | 1 << 4, + ); + let r = _mm512_maskz_or_epi32(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_or_epi32(0b00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi32( + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 3 | 1 << 4, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_or_epi32() { + let a = _mm256_set1_epi32(1 << 1 | 1 << 2); + let b = _mm256_set1_epi32(1 << 1); + let r = _mm256_or_epi32(a, b); + let e = _mm256_set1_epi32(1 << 1 | 1 << 2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_or_epi32() { + let a = _mm256_set1_epi32(1 << 1 | 1 << 2); + let b = _mm256_set1_epi32(1 << 1); + let r = _mm256_mask_or_epi32(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_or_epi32(a, 0b11111111, a, b); + let e = _mm256_set1_epi32(1 << 1 | 1 << 2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_or_epi32() { + let a = _mm256_set1_epi32(1 << 1 | 1 << 2); + let b = _mm256_set1_epi32(1 << 1); + let r = _mm256_maskz_or_epi32(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_or_epi32(0b11111111, a, b); + let e = _mm256_set1_epi32(1 << 1 | 1 << 2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_or_epi32() { + let a = _mm_set1_epi32(1 << 1 | 1 << 2); + let b = _mm_set1_epi32(1 << 1); + let r = _mm_or_epi32(a, b); + let e = _mm_set1_epi32(1 << 1 | 1 << 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_or_epi32() { + let a = _mm_set1_epi32(1 << 1 | 1 << 2); + let b = _mm_set1_epi32(1 << 1); + let r = _mm_mask_or_epi32(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_or_epi32(a, 0b00001111, a, b); + let e = _mm_set1_epi32(1 << 1 | 1 << 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_or_epi32() { + let a = _mm_set1_epi32(1 << 1 | 1 << 2); + let b = _mm_set1_epi32(1 << 1); + let r = _mm_maskz_or_epi32(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_or_epi32(0b00001111, a, b); + let e = _mm_set1_epi32(1 << 1 | 1 << 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_or_si512() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 1 << 1 | 1 << 2, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 3, + ); + #[rustfmt::skip] + let b = _mm512_set_epi32( + 1 << 1, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 3 | 1 << 4, + ); + let r = _mm512_or_epi32(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi32( + 1 << 1 | 1 << 2, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 3 | 1 << 4, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_xor_epi32() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 1 << 1 | 1 << 2, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 3, + ); + #[rustfmt::skip] + let b = _mm512_set_epi32( + 1 << 1, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 3 | 1 << 4, + ); + let r = _mm512_xor_epi32(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi32( + 1 << 2, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 4, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_xor_epi32() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 1 << 1 | 1 << 2, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 3, + ); + #[rustfmt::skip] + let b = _mm512_set_epi32( + 1 << 1, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 3 | 1 << 4, + ); + let r = _mm512_mask_xor_epi32(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_xor_epi32(a, 0b01111111_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi32( + 1 << 1 | 1 << 2, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 4, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_xor_epi32() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 1 << 1 | 1 << 2, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 3, + ); + #[rustfmt::skip] + let b = _mm512_set_epi32( + 1 << 1, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 3 | 1 << 4, + ); + let r = _mm512_maskz_xor_epi32(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_xor_epi32(0b00000000_11111111, a, b); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 4); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_xor_epi32() { + let a = _mm256_set1_epi32(1 << 1 | 1 << 2); + let b = _mm256_set1_epi32(1 << 1); + let r = _mm256_xor_epi32(a, b); + let e = _mm256_set1_epi32(1 << 2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_xor_epi32() { + let a = _mm256_set1_epi32(1 << 1 | 1 << 2); + let b = _mm256_set1_epi32(1 << 1); + let r = _mm256_mask_xor_epi32(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_xor_epi32(a, 0b11111111, a, b); + let e = _mm256_set1_epi32(1 << 2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_xor_epi32() { + let a = _mm256_set1_epi32(1 << 1 | 1 << 2); + let b = _mm256_set1_epi32(1 << 1); + let r = _mm256_maskz_xor_epi32(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_xor_epi32(0b11111111, a, b); + let e = _mm256_set1_epi32(1 << 2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_xor_epi32() { + let a = _mm_set1_epi32(1 << 1 | 1 << 2); + let b = _mm_set1_epi32(1 << 1); + let r = _mm_xor_epi32(a, b); + let e = _mm_set1_epi32(1 << 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_xor_epi32() { + let a = _mm_set1_epi32(1 << 1 | 1 << 2); + let b = _mm_set1_epi32(1 << 1); + let r = _mm_mask_xor_epi32(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_xor_epi32(a, 0b00001111, a, b); + let e = _mm_set1_epi32(1 << 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_xor_epi32() { + let a = _mm_set1_epi32(1 << 1 | 1 << 2); + let b = _mm_set1_epi32(1 << 1); + let r = _mm_maskz_xor_epi32(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_xor_epi32(0b00001111, a, b); + let e = _mm_set1_epi32(1 << 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_xor_si512() { + #[rustfmt::skip] + let a = _mm512_set_epi32( + 1 << 1 | 1 << 2, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 3, + ); + #[rustfmt::skip] + let b = _mm512_set_epi32( + 1 << 1, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 3 | 1 << 4, + ); + let r = _mm512_xor_epi32(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi32( + 1 << 2, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 4, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_andnot_epi32() { + let a = _mm512_set1_epi32(0); + let b = _mm512_set1_epi32(1 << 3 | 1 << 4); + let r = _mm512_andnot_epi32(a, b); + let e = _mm512_set1_epi32(1 << 3 | 1 << 4); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_andnot_epi32() { + let a = _mm512_set1_epi32(1 << 1 | 1 << 2); + let b = _mm512_set1_epi32(1 << 3 | 1 << 4); + let r = _mm512_mask_andnot_epi32(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_andnot_epi32(a, 0b11111111_11111111, a, b); + let e = _mm512_set1_epi32(1 << 3 | 1 << 4); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_andnot_epi32() { + let a = _mm512_set1_epi32(1 << 1 | 1 << 2); + let b = _mm512_set1_epi32(1 << 3 | 1 << 4); + let r = _mm512_maskz_andnot_epi32(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_andnot_epi32(0b00000000_11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi32( + 0, 0, 0, 0, + 0, 0, 0, 0, + 1 << 3 | 1 << 4, 1 << 3 | 1 << 4, 1 << 3 | 1 << 4, 1 << 3 | 1 << 4, + 1 << 3 | 1 << 4, 1 << 3 | 1 << 4, 1 << 3 | 1 << 4, 1 << 3 | 1 << 4, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_andnot_epi32() { + let a = _mm256_set1_epi32(1 << 1 | 1 << 2); + let b = _mm256_set1_epi32(1 << 3 | 1 << 4); + let r = _mm256_mask_andnot_epi32(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_andnot_epi32(a, 0b11111111, a, b); + let e = _mm256_set1_epi32(1 << 3 | 1 << 4); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_andnot_epi32() { + let a = _mm256_set1_epi32(1 << 1 | 1 << 2); + let b = _mm256_set1_epi32(1 << 3 | 1 << 4); + let r = _mm256_maskz_andnot_epi32(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_andnot_epi32(0b11111111, a, b); + let e = _mm256_set1_epi32(1 << 3 | 1 << 4); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_andnot_epi32() { + let a = _mm_set1_epi32(1 << 1 | 1 << 2); + let b = _mm_set1_epi32(1 << 3 | 1 << 4); + let r = _mm_mask_andnot_epi32(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_andnot_epi32(a, 0b00001111, a, b); + let e = _mm_set1_epi32(1 << 3 | 1 << 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_andnot_epi32() { + let a = _mm_set1_epi32(1 << 1 | 1 << 2); + let b = _mm_set1_epi32(1 << 3 | 1 << 4); + let r = _mm_maskz_andnot_epi32(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_andnot_epi32(0b00001111, a, b); + let e = _mm_set1_epi32(1 << 3 | 1 << 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_kand() { + let a: u16 = 0b11001100_00110011; + let b: u16 = 0b11001100_00110011; + let r = _mm512_kand(a, b); + let e: u16 = 0b11001100_00110011; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_kand_mask16() { + let a: u16 = 0b11001100_00110011; + let b: u16 = 0b11001100_00110011; + let r = _kand_mask16(a, b); + let e: u16 = 0b11001100_00110011; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_kor() { + let a: u16 = 0b11001100_00110011; + let b: u16 = 0b00101110_00001011; + let r = _mm512_kor(a, b); + let e: u16 = 0b11101110_00111011; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_kor_mask16() { + let a: u16 = 0b11001100_00110011; + let b: u16 = 0b00101110_00001011; + let r = _kor_mask16(a, b); + let e: u16 = 0b11101110_00111011; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_kxor() { + let a: u16 = 0b11001100_00110011; + let b: u16 = 0b00101110_00001011; + let r = _mm512_kxor(a, b); + let e: u16 = 0b11100010_00111000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_kxor_mask16() { + let a: u16 = 0b11001100_00110011; + let b: u16 = 0b00101110_00001011; + let r = _kxor_mask16(a, b); + let e: u16 = 0b11100010_00111000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_knot() { + let a: u16 = 0b11001100_00110011; + let r = _mm512_knot(a); + let e: u16 = 0b00110011_11001100; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_knot_mask16() { + let a: u16 = 0b11001100_00110011; + let r = _knot_mask16(a); + let e: u16 = 0b00110011_11001100; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_kandn() { + let a: u16 = 0b11001100_00110011; + let b: u16 = 0b00101110_00001011; + let r = _mm512_kandn(a, b); + let e: u16 = 0b00100010_00001000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_kandn_mask16() { + let a: u16 = 0b11001100_00110011; + let b: u16 = 0b00101110_00001011; + let r = _kandn_mask16(a, b); + let e: u16 = 0b00100010_00001000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_kxnor() { + let a: u16 = 0b11001100_00110011; + let b: u16 = 0b00101110_00001011; + let r = _mm512_kxnor(a, b); + let e: u16 = 0b00011101_11000111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_kxnor_mask16() { + let a: u16 = 0b11001100_00110011; + let b: u16 = 0b00101110_00001011; + let r = _kxnor_mask16(a, b); + let e: u16 = 0b00011101_11000111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_kmov() { + let a: u16 = 0b11001100_00110011; + let r = _mm512_kmov(a); + let e: u16 = 0b11001100_00110011; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_int2mask() { + let a: i32 = 0b11001100_00110011; + let r = _mm512_int2mask(a); + let e: u16 = 0b11001100_00110011; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask2int() { + let k1: __mmask16 = 0b11001100_00110011; + let r = _mm512_mask2int(k1); + let e: i32 = 0b11001100_00110011; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_kunpackb() { + let a: u16 = 0b11001100_00110011; + let b: u16 = 0b00101110_00001011; + let r = _mm512_kunpackb(a, b); + let e: u16 = 0b00101110_00110011; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_kortestc() { + let a: u16 = 0b11001100_00110011; + let b: u16 = 0b00101110_00001011; + let r = _mm512_kortestc(a, b); + assert_eq!(r, 0); + let b: u16 = 0b11111111_11111111; + let r = _mm512_kortestc(a, b); + assert_eq!(r, 1); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_test_epi32_mask() { + let a = _mm512_set1_epi32(1 << 0); + let b = _mm512_set1_epi32(1 << 0 | 1 << 1); + let r = _mm512_test_epi32_mask(a, b); + let e: __mmask16 = 0b11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_test_epi32_mask() { + let a = _mm512_set1_epi32(1 << 0); + let b = _mm512_set1_epi32(1 << 0 | 1 << 1); + let r = _mm512_mask_test_epi32_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm512_mask_test_epi32_mask(0b11111111_11111111, a, b); + let e: __mmask16 = 0b11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_test_epi32_mask() { + let a = _mm256_set1_epi32(1 << 0); + let b = _mm256_set1_epi32(1 << 0 | 1 << 1); + let r = _mm256_test_epi32_mask(a, b); + let e: __mmask8 = 0b11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_test_epi32_mask() { + let a = _mm256_set1_epi32(1 << 0); + let b = _mm256_set1_epi32(1 << 0 | 1 << 1); + let r = _mm256_mask_test_epi32_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm256_mask_test_epi32_mask(0b11111111, a, b); + let e: __mmask8 = 0b11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_test_epi32_mask() { + let a = _mm_set1_epi32(1 << 0); + let b = _mm_set1_epi32(1 << 0 | 1 << 1); + let r = _mm_test_epi32_mask(a, b); + let e: __mmask8 = 0b00001111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_test_epi32_mask() { + let a = _mm_set1_epi32(1 << 0); + let b = _mm_set1_epi32(1 << 0 | 1 << 1); + let r = _mm_mask_test_epi32_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm_mask_test_epi32_mask(0b11111111, a, b); + let e: __mmask8 = 0b00001111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_testn_epi32_mask() { + let a = _mm512_set1_epi32(1 << 0); + let b = _mm512_set1_epi32(1 << 0 | 1 << 1); + let r = _mm512_testn_epi32_mask(a, b); + let e: __mmask16 = 0b00000000_00000000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_testn_epi32_mask() { + let a = _mm512_set1_epi32(1 << 0); + let b = _mm512_set1_epi32(1 << 1); + let r = _mm512_mask_test_epi32_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm512_mask_testn_epi32_mask(0b11111111_11111111, a, b); + let e: __mmask16 = 0b11111111_11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_testn_epi32_mask() { + let a = _mm256_set1_epi32(1 << 0); + let b = _mm256_set1_epi32(1 << 1); + let r = _mm256_testn_epi32_mask(a, b); + let e: __mmask8 = 0b11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_testn_epi32_mask() { + let a = _mm256_set1_epi32(1 << 0); + let b = _mm256_set1_epi32(1 << 1); + let r = _mm256_mask_test_epi32_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm256_mask_testn_epi32_mask(0b11111111, a, b); + let e: __mmask8 = 0b11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_testn_epi32_mask() { + let a = _mm_set1_epi32(1 << 0); + let b = _mm_set1_epi32(1 << 1); + let r = _mm_testn_epi32_mask(a, b); + let e: __mmask8 = 0b00001111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_testn_epi32_mask() { + let a = _mm_set1_epi32(1 << 0); + let b = _mm_set1_epi32(1 << 1); + let r = _mm_mask_test_epi32_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm_mask_testn_epi32_mask(0b11111111, a, b); + let e: __mmask8 = 0b00001111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_stream_ps() { + #[repr(align(32))] + struct Memory { + pub data: [f32; 16], + } + let a = _mm512_set1_ps(7.0); + let mut mem = Memory { data: [-1.0; 16] }; + + _mm512_stream_ps(&mut mem.data[0] as *mut f32, a); + for i in 0..16 { + assert_eq!(mem.data[i], get_m512(a, i)); + } + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_add_epi32() { + let a = _mm512_set1_epi32(1); + let e: i32 = _mm512_reduce_add_epi32(a); + assert_eq!(16, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_add_epi32() { + let a = _mm512_set1_epi32(1); + let e: i32 = _mm512_mask_reduce_add_epi32(0b11111111_00000000, a); + assert_eq!(8, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_add_ps() { + let a = _mm512_set1_ps(1.); + let e: f32 = _mm512_reduce_add_ps(a); + assert_eq!(16., e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_add_ps() { + let a = _mm512_set1_ps(1.); + let e: f32 = _mm512_mask_reduce_add_ps(0b11111111_00000000, a); + assert_eq!(8., e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_mul_epi32() { + let a = _mm512_set1_epi32(2); + let e: i32 = _mm512_reduce_mul_epi32(a); + assert_eq!(65536, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_mul_epi32() { + let a = _mm512_set1_epi32(2); + let e: i32 = _mm512_mask_reduce_mul_epi32(0b11111111_00000000, a); + assert_eq!(256, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_mul_ps() { + let a = _mm512_set1_ps(2.); + let e: f32 = _mm512_reduce_mul_ps(a); + assert_eq!(65536., e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_mul_ps() { + let a = _mm512_set1_ps(2.); + let e: f32 = _mm512_mask_reduce_mul_ps(0b11111111_00000000, a); + assert_eq!(256., e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_max_epi32() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: i32 = _mm512_reduce_max_epi32(a); + assert_eq!(15, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_max_epi32() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: i32 = _mm512_mask_reduce_max_epi32(0b11111111_00000000, a); + assert_eq!(7, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_max_epu32() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: u32 = _mm512_reduce_max_epu32(a); + assert_eq!(15, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_max_epu32() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: u32 = _mm512_mask_reduce_max_epu32(0b11111111_00000000, a); + assert_eq!(7, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_max_ps() { + let a = _mm512_set_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let e: f32 = _mm512_reduce_max_ps(a); + assert_eq!(15., e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_max_ps() { + let a = _mm512_set_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let e: f32 = _mm512_mask_reduce_max_ps(0b11111111_00000000, a); + assert_eq!(7., e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_min_epi32() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: i32 = _mm512_reduce_min_epi32(a); + assert_eq!(0, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_min_epi32() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: i32 = _mm512_mask_reduce_min_epi32(0b11111111_00000000, a); + assert_eq!(0, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_min_epu32() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: u32 = _mm512_reduce_min_epu32(a); + assert_eq!(0, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_min_epu32() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: u32 = _mm512_mask_reduce_min_epu32(0b11111111_00000000, a); + assert_eq!(0, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_min_ps() { + let a = _mm512_set_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let e: f32 = _mm512_reduce_min_ps(a); + assert_eq!(0., e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_min_ps() { + let a = _mm512_set_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let e: f32 = _mm512_mask_reduce_min_ps(0b11111111_00000000, a); + assert_eq!(0., e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_and_epi32() { + let a = _mm512_set_epi32(1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2); + let e: i32 = _mm512_reduce_and_epi32(a); + assert_eq!(0, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_and_epi32() { + let a = _mm512_set_epi32(1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2); + let e: i32 = _mm512_mask_reduce_and_epi32(0b11111111_00000000, a); + assert_eq!(1, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_or_epi32() { + let a = _mm512_set_epi32(1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2); + let e: i32 = _mm512_reduce_or_epi32(a); + assert_eq!(3, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_or_epi32() { + let a = _mm512_set_epi32(1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2); + let e: i32 = _mm512_mask_reduce_and_epi32(0b11111111_00000000, a); + assert_eq!(1, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_compress_epi32() { + let src = _mm512_set1_epi32(200); + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_mask_compress_epi32(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_compress_epi32(src, 0b01010101_01010101, a); + let e = _mm512_set_epi32( + 200, 200, 200, 200, 200, 200, 200, 200, 1, 3, 5, 7, 9, 11, 13, 15, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_compress_epi32() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_compress_epi32(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_compress_epi32(0b01010101_01010101, a); + let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 5, 7, 9, 11, 13, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_compress_epi32() { + let src = _mm256_set1_epi32(200); + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm256_mask_compress_epi32(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_compress_epi32(src, 0b01010101, a); + let e = _mm256_set_epi32(200, 200, 200, 200, 1, 3, 5, 7); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_compress_epi32() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm256_maskz_compress_epi32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_compress_epi32(0b01010101, a); + let e = _mm256_set_epi32(0, 0, 0, 0, 1, 3, 5, 7); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_compress_epi32() { + let src = _mm_set1_epi32(200); + let a = _mm_set_epi32(0, 1, 2, 3); + let r = _mm_mask_compress_epi32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_compress_epi32(src, 0b00000101, a); + let e = _mm_set_epi32(200, 200, 1, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_compress_epi32() { + let a = _mm_set_epi32(0, 1, 2, 3); + let r = _mm_maskz_compress_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_compress_epi32(0b00000101, a); + let e = _mm_set_epi32(0, 0, 1, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_compress_ps() { + let src = _mm512_set1_ps(200.); + let a = _mm512_set_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let r = _mm512_mask_compress_ps(src, 0, a); + assert_eq_m512(r, src); + let r = _mm512_mask_compress_ps(src, 0b01010101_01010101, a); + let e = _mm512_set_ps( + 200., 200., 200., 200., 200., 200., 200., 200., 1., 3., 5., 7., 9., 11., 13., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_compress_ps() { + let a = _mm512_set_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let r = _mm512_maskz_compress_ps(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_compress_ps(0b01010101_01010101, a); + let e = _mm512_set_ps( + 0., 0., 0., 0., 0., 0., 0., 0., 1., 3., 5., 7., 9., 11., 13., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_compress_ps() { + let src = _mm256_set1_ps(200.); + let a = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm256_mask_compress_ps(src, 0, a); + assert_eq_m256(r, src); + let r = _mm256_mask_compress_ps(src, 0b01010101, a); + let e = _mm256_set_ps(200., 200., 200., 200., 1., 3., 5., 7.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_compress_ps() { + let a = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm256_maskz_compress_ps(0, a); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_compress_ps(0b01010101, a); + let e = _mm256_set_ps(0., 0., 0., 0., 1., 3., 5., 7.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_compress_ps() { + let src = _mm_set1_ps(200.); + let a = _mm_set_ps(0., 1., 2., 3.); + let r = _mm_mask_compress_ps(src, 0, a); + assert_eq_m128(r, src); + let r = _mm_mask_compress_ps(src, 0b00000101, a); + let e = _mm_set_ps(200., 200., 1., 3.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_compress_ps() { + let a = _mm_set_ps(0., 1., 2., 3.); + let r = _mm_maskz_compress_ps(0, a); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_compress_ps(0b00000101, a); + let e = _mm_set_ps(0., 0., 1., 3.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_compressstoreu_epi32() { + let a = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let mut r = [0_i32; 16]; + _mm512_mask_compressstoreu_epi32(r.as_mut_ptr() as *mut _, 0, a); + assert_eq!(&r, &[0_i32; 16]); + _mm512_mask_compressstoreu_epi32(r.as_mut_ptr() as *mut _, 0b1111000011001010, a); + assert_eq!(&r, &[2, 4, 7, 8, 13, 14, 15, 16, 0, 0, 0, 0, 0, 0, 0, 0]); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_compressstoreu_epi32() { + let a = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let mut r = [0_i32; 8]; + _mm256_mask_compressstoreu_epi32(r.as_mut_ptr() as *mut _, 0, a); + assert_eq!(&r, &[0_i32; 8]); + _mm256_mask_compressstoreu_epi32(r.as_mut_ptr() as *mut _, 0b11001010, a); + assert_eq!(&r, &[2, 4, 7, 8, 0, 0, 0, 0]); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_compressstoreu_epi32() { + let a = _mm_setr_epi32(1, 2, 3, 4); + let mut r = [0_i32; 4]; + _mm_mask_compressstoreu_epi32(r.as_mut_ptr() as *mut _, 0, a); + assert_eq!(&r, &[0_i32; 4]); + _mm_mask_compressstoreu_epi32(r.as_mut_ptr() as *mut _, 0b1011, a); + assert_eq!(&r, &[1, 2, 4, 0]); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_compressstoreu_epi64() { + let a = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); + let mut r = [0_i64; 8]; + _mm512_mask_compressstoreu_epi64(r.as_mut_ptr() as *mut _, 0, a); + assert_eq!(&r, &[0_i64; 8]); + _mm512_mask_compressstoreu_epi64(r.as_mut_ptr() as *mut _, 0b11001010, a); + assert_eq!(&r, &[2, 4, 7, 8, 0, 0, 0, 0]); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_compressstoreu_epi64() { + let a = _mm256_setr_epi64x(1, 2, 3, 4); + let mut r = [0_i64; 4]; + _mm256_mask_compressstoreu_epi64(r.as_mut_ptr() as *mut _, 0, a); + assert_eq!(&r, &[0_i64; 4]); + _mm256_mask_compressstoreu_epi64(r.as_mut_ptr() as *mut _, 0b1011, a); + assert_eq!(&r, &[1, 2, 4, 0]); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_compressstoreu_epi64() { + let a = _mm_setr_epi64x(1, 2); + let mut r = [0_i64; 2]; + _mm_mask_compressstoreu_epi64(r.as_mut_ptr() as *mut _, 0, a); + assert_eq!(&r, &[0_i64; 2]); + _mm_mask_compressstoreu_epi64(r.as_mut_ptr() as *mut _, 0b10, a); + assert_eq!(&r, &[2, 0]); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_compressstoreu_ps() { + let a = _mm512_setr_ps( + 1_f32, 2_f32, 3_f32, 4_f32, 5_f32, 6_f32, 7_f32, 8_f32, 9_f32, 10_f32, 11_f32, 12_f32, + 13_f32, 14_f32, 15_f32, 16_f32, + ); + let mut r = [0_f32; 16]; + _mm512_mask_compressstoreu_ps(r.as_mut_ptr() as *mut _, 0, a); + assert_eq!(&r, &[0_f32; 16]); + _mm512_mask_compressstoreu_ps(r.as_mut_ptr() as *mut _, 0b1111000011001010, a); + assert_eq!( + &r, + &[ + 2_f32, 4_f32, 7_f32, 8_f32, 13_f32, 14_f32, 15_f32, 16_f32, 0_f32, 0_f32, 0_f32, + 0_f32, 0_f32, 0_f32, 0_f32, 0_f32 + ] + ); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_compressstoreu_ps() { + let a = _mm256_setr_ps(1_f32, 2_f32, 3_f32, 4_f32, 5_f32, 6_f32, 7_f32, 8_f32); + let mut r = [0_f32; 8]; + _mm256_mask_compressstoreu_ps(r.as_mut_ptr() as *mut _, 0, a); + assert_eq!(&r, &[0_f32; 8]); + _mm256_mask_compressstoreu_ps(r.as_mut_ptr() as *mut _, 0b11001010, a); + assert_eq!( + &r, + &[2_f32, 4_f32, 7_f32, 8_f32, 0_f32, 0_f32, 0_f32, 0_f32] + ); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_compressstoreu_ps() { + let a = _mm_setr_ps(1_f32, 2_f32, 3_f32, 4_f32); + let mut r = [0.; 4]; + _mm_mask_compressstoreu_ps(r.as_mut_ptr() as *mut _, 0, a); + assert_eq!(&r, &[0.; 4]); + _mm_mask_compressstoreu_ps(r.as_mut_ptr() as *mut _, 0b1011, a); + assert_eq!(&r, &[1_f32, 2_f32, 4_f32, 0_f32]); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_compressstoreu_pd() { + let a = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); + let mut r = [0.; 8]; + _mm512_mask_compressstoreu_pd(r.as_mut_ptr() as *mut _, 0, a); + assert_eq!(&r, &[0.; 8]); + _mm512_mask_compressstoreu_pd(r.as_mut_ptr() as *mut _, 0b11001010, a); + assert_eq!(&r, &[2., 4., 7., 8., 0., 0., 0., 0.]); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_compressstoreu_pd() { + let a = _mm256_setr_pd(1., 2., 3., 4.); + let mut r = [0.; 4]; + _mm256_mask_compressstoreu_pd(r.as_mut_ptr() as *mut _, 0, a); + assert_eq!(&r, &[0.; 4]); + _mm256_mask_compressstoreu_pd(r.as_mut_ptr() as *mut _, 0b1011, a); + assert_eq!(&r, &[1., 2., 4., 0.]); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_compressstoreu_pd() { + let a = _mm_setr_pd(1., 2.); + let mut r = [0.; 2]; + _mm_mask_compressstoreu_pd(r.as_mut_ptr() as *mut _, 0, a); + assert_eq!(&r, &[0.; 2]); + _mm_mask_compressstoreu_pd(r.as_mut_ptr() as *mut _, 0b10, a); + assert_eq!(&r, &[2., 0.]); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_expand_epi32() { + let src = _mm512_set1_epi32(200); + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_mask_expand_epi32(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_expand_epi32(src, 0b01010101_01010101, a); + let e = _mm512_set_epi32( + 200, 8, 200, 9, 200, 10, 200, 11, 200, 12, 200, 13, 200, 14, 200, 15, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_expand_epi32() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_expand_epi32(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_expand_epi32(0b01010101_01010101, a); + let e = _mm512_set_epi32(0, 8, 0, 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_expand_epi32() { + let src = _mm256_set1_epi32(200); + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm256_mask_expand_epi32(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_expand_epi32(src, 0b01010101, a); + let e = _mm256_set_epi32(200, 4, 200, 5, 200, 6, 200, 7); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_expand_epi32() { + let a = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm256_maskz_expand_epi32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_expand_epi32(0b01010101, a); + let e = _mm256_set_epi32(0, 4, 0, 5, 0, 6, 0, 7); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_expand_epi32() { + let src = _mm_set1_epi32(200); + let a = _mm_set_epi32(0, 1, 2, 3); + let r = _mm_mask_expand_epi32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_expand_epi32(src, 0b00000101, a); + let e = _mm_set_epi32(200, 2, 200, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_expand_epi32() { + let a = _mm_set_epi32(0, 1, 2, 3); + let r = _mm_maskz_expand_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_expand_epi32(0b00000101, a); + let e = _mm_set_epi32(0, 2, 0, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_expand_ps() { + let src = _mm512_set1_ps(200.); + let a = _mm512_set_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let r = _mm512_mask_expand_ps(src, 0, a); + assert_eq_m512(r, src); + let r = _mm512_mask_expand_ps(src, 0b01010101_01010101, a); + let e = _mm512_set_ps( + 200., 8., 200., 9., 200., 10., 200., 11., 200., 12., 200., 13., 200., 14., 200., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_expand_ps() { + let a = _mm512_set_ps( + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ); + let r = _mm512_maskz_expand_ps(0, a); + assert_eq_m512(r, _mm512_setzero_ps()); + let r = _mm512_maskz_expand_ps(0b01010101_01010101, a); + let e = _mm512_set_ps( + 0., 8., 0., 9., 0., 10., 0., 11., 0., 12., 0., 13., 0., 14., 0., 15., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_expand_ps() { + let src = _mm256_set1_ps(200.); + let a = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm256_mask_expand_ps(src, 0, a); + assert_eq_m256(r, src); + let r = _mm256_mask_expand_ps(src, 0b01010101, a); + let e = _mm256_set_ps(200., 4., 200., 5., 200., 6., 200., 7.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_expand_ps() { + let a = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm256_maskz_expand_ps(0, a); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm256_maskz_expand_ps(0b01010101, a); + let e = _mm256_set_ps(0., 4., 0., 5., 0., 6., 0., 7.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_expand_ps() { + let src = _mm_set1_ps(200.); + let a = _mm_set_ps(0., 1., 2., 3.); + let r = _mm_mask_expand_ps(src, 0, a); + assert_eq_m128(r, src); + let r = _mm_mask_expand_ps(src, 0b00000101, a); + let e = _mm_set_ps(200., 2., 200., 3.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_expand_ps() { + let a = _mm_set_ps(0., 1., 2., 3.); + let r = _mm_maskz_expand_ps(0, a); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_expand_ps(0b00000101, a); + let e = _mm_set_ps(0., 2., 0., 3.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_loadu_epi32() { + let a = &[4, 3, 2, 5, 8, 9, 64, 50, -4, -3, -2, -5, -8, -9, -64, -50]; + let p = a.as_ptr(); + let r = _mm512_loadu_epi32(black_box(p)); + let e = _mm512_setr_epi32(4, 3, 2, 5, 8, 9, 64, 50, -4, -3, -2, -5, -8, -9, -64, -50); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_loadu_epi32() { + let a = &[4, 3, 2, 5, 8, 9, 64, 50]; + let p = a.as_ptr(); + let r = _mm256_loadu_epi32(black_box(p)); + let e = _mm256_setr_epi32(4, 3, 2, 5, 8, 9, 64, 50); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_loadu_epi32() { + let a = &[4, 3, 2, 5]; + let p = a.as_ptr(); + let r = _mm_loadu_epi32(black_box(p)); + let e = _mm_setr_epi32(4, 3, 2, 5); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepi32_storeu_epi16() { + let a = _mm512_set1_epi32(9); + let mut r = _mm256_undefined_si256(); + _mm512_mask_cvtepi32_storeu_epi16(&mut r as *mut _ as *mut i8, 0b11111111_11111111, a); + let e = _mm256_set1_epi16(9); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepi32_storeu_epi16() { + let a = _mm256_set1_epi32(9); + let mut r = _mm_undefined_si128(); + _mm256_mask_cvtepi32_storeu_epi16(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set1_epi16(9); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepi32_storeu_epi16() { + let a = _mm_set1_epi32(9); + let mut r = _mm_set1_epi8(0); + _mm_mask_cvtepi32_storeu_epi16(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 9, 9, 9, 9); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtsepi32_storeu_epi16() { + let a = _mm512_set1_epi32(i32::MAX); + let mut r = _mm256_undefined_si256(); + _mm512_mask_cvtsepi32_storeu_epi16(&mut r as *mut _ as *mut i8, 0b11111111_11111111, a); + let e = _mm256_set1_epi16(i16::MAX); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtsepi32_storeu_epi16() { + let a = _mm256_set1_epi32(i32::MAX); + let mut r = _mm_undefined_si128(); + _mm256_mask_cvtsepi32_storeu_epi16(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set1_epi16(i16::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtsepi32_storeu_epi16() { + let a = _mm_set1_epi32(i32::MAX); + let mut r = _mm_set1_epi8(0); + _mm_mask_cvtsepi32_storeu_epi16(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set_epi16(0, 0, 0, 0, i16::MAX, i16::MAX, i16::MAX, i16::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtusepi32_storeu_epi16() { + let a = _mm512_set1_epi32(i32::MAX); + let mut r = _mm256_undefined_si256(); + _mm512_mask_cvtusepi32_storeu_epi16(&mut r as *mut _ as *mut i8, 0b11111111_11111111, a); + let e = _mm256_set1_epi16(u16::MAX as i16); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtusepi32_storeu_epi16() { + let a = _mm256_set1_epi32(i32::MAX); + let mut r = _mm_undefined_si128(); + _mm256_mask_cvtusepi32_storeu_epi16(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set1_epi16(u16::MAX as i16); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtusepi32_storeu_epi16() { + let a = _mm_set1_epi32(i32::MAX); + let mut r = _mm_set1_epi8(0); + _mm_mask_cvtusepi32_storeu_epi16(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set_epi16( + 0, + 0, + 0, + 0, + u16::MAX as i16, + u16::MAX as i16, + u16::MAX as i16, + u16::MAX as i16, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepi32_storeu_epi8() { + let a = _mm512_set1_epi32(9); + let mut r = _mm_undefined_si128(); + _mm512_mask_cvtepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111_11111111, a); + let e = _mm_set1_epi8(9); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepi32_storeu_epi8() { + let a = _mm256_set1_epi32(9); + let mut r = _mm_set1_epi8(0); + _mm256_mask_cvtepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepi32_storeu_epi8() { + let a = _mm_set1_epi32(9); + let mut r = _mm_set1_epi8(0); + _mm_mask_cvtepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtsepi32_storeu_epi8() { + let a = _mm512_set1_epi32(i32::MAX); + let mut r = _mm_undefined_si128(); + _mm512_mask_cvtsepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111_11111111, a); + let e = _mm_set1_epi8(i8::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtsepi32_storeu_epi8() { + let a = _mm256_set1_epi32(i32::MAX); + let mut r = _mm_set1_epi8(0); + _mm256_mask_cvtsepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + #[rustfmt::skip] + let e = _mm_set_epi8( + 0, 0, 0, 0, + 0, 0, 0, 0, + i8::MAX, i8::MAX, i8::MAX, i8::MAX, + i8::MAX, i8::MAX, i8::MAX, i8::MAX, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtsepi32_storeu_epi8() { + let a = _mm_set1_epi32(i32::MAX); + let mut r = _mm_set1_epi8(0); + _mm_mask_cvtsepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + #[rustfmt::skip] + let e = _mm_set_epi8( + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + i8::MAX, i8::MAX, i8::MAX, i8::MAX, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtusepi32_storeu_epi8() { + let a = _mm512_set1_epi32(i32::MAX); + let mut r = _mm_undefined_si128(); + _mm512_mask_cvtusepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111_11111111, a); + let e = _mm_set1_epi8(u8::MAX as i8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtusepi32_storeu_epi8() { + let a = _mm256_set1_epi32(i32::MAX); + let mut r = _mm_set1_epi8(0); + _mm256_mask_cvtusepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + #[rustfmt::skip] + let e = _mm_set_epi8( + 0, 0, 0, 0, + 0, 0, 0, 0, + u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, + u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtusepi32_storeu_epi8() { + let a = _mm_set1_epi32(i32::MAX); + let mut r = _mm_set1_epi8(0); + _mm_mask_cvtusepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + #[rustfmt::skip] + let e = _mm_set_epi8( + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_storeu_epi32() { + let a = _mm512_set1_epi32(9); + let mut r = _mm512_undefined_epi32(); + _mm512_storeu_epi32(&mut r as *mut _ as *mut i32, a); + assert_eq_m512i(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_storeu_epi32() { + let a = _mm256_set1_epi32(9); + let mut r = _mm256_undefined_si256(); + _mm256_storeu_epi32(&mut r as *mut _ as *mut i32, a); + assert_eq_m256i(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_storeu_epi32() { + let a = _mm_set1_epi32(9); + let mut r = _mm_undefined_si128(); + _mm_storeu_epi32(&mut r as *mut _ as *mut i32, a); + assert_eq_m128i(r, a); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_loadu_si512() { + let a = &[4, 3, 2, 5, 8, 9, 64, 50, -4, -3, -2, -5, -8, -9, -64, -50]; + let p = a.as_ptr(); + let r = _mm512_loadu_si512(black_box(p)); + let e = _mm512_setr_epi32(4, 3, 2, 5, 8, 9, 64, 50, -4, -3, -2, -5, -8, -9, -64, -50); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_storeu_si512() { + let a = _mm512_set1_epi32(9); + let mut r = _mm512_undefined_epi32(); + _mm512_storeu_si512(&mut r as *mut _ as *mut i32, a); + assert_eq_m512i(r, a); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_load_si512() { + #[repr(align(64))] + struct Align { + data: [i32; 16], // 64 bytes + } + let a = Align { + data: [4, 3, 2, 5, 8, 9, 64, 50, -4, -3, -2, -5, -8, -9, -64, -50], + }; + let p = (a.data).as_ptr(); + let r = _mm512_load_si512(black_box(p)); + let e = _mm512_setr_epi32(4, 3, 2, 5, 8, 9, 64, 50, -4, -3, -2, -5, -8, -9, -64, -50); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_store_si512() { + let a = _mm512_set1_epi32(9); + let mut r = _mm512_undefined_epi32(); + _mm512_store_si512(&mut r as *mut _ as *mut i32, a); + assert_eq_m512i(r, a); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_load_epi32() { + #[repr(align(64))] + struct Align { + data: [i32; 16], // 64 bytes + } + let a = Align { + data: [4, 3, 2, 5, 8, 9, 64, 50, -4, -3, -2, -5, -8, -9, -64, -50], + }; + let p = (a.data).as_ptr(); + let r = _mm512_load_epi32(black_box(p)); + let e = _mm512_setr_epi32(4, 3, 2, 5, 8, 9, 64, 50, -4, -3, -2, -5, -8, -9, -64, -50); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_load_epi32() { + #[repr(align(64))] + struct Align { + data: [i32; 8], + } + let a = Align { + data: [4, 3, 2, 5, 8, 9, 64, 50], + }; + let p = (a.data).as_ptr(); + let r = _mm256_load_epi32(black_box(p)); + let e = _mm256_setr_epi32(4, 3, 2, 5, 8, 9, 64, 50); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_load_epi32() { + #[repr(align(64))] + struct Align { + data: [i32; 4], + } + let a = Align { data: [4, 3, 2, 5] }; + let p = (a.data).as_ptr(); + let r = _mm_load_epi32(black_box(p)); + let e = _mm_setr_epi32(4, 3, 2, 5); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_store_epi32() { + let a = _mm512_set1_epi32(9); + let mut r = _mm512_undefined_epi32(); + _mm512_store_epi32(&mut r as *mut _ as *mut i32, a); + assert_eq_m512i(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_store_epi32() { + let a = _mm256_set1_epi32(9); + let mut r = _mm256_undefined_si256(); + _mm256_store_epi32(&mut r as *mut _ as *mut i32, a); + assert_eq_m256i(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_store_epi32() { + let a = _mm_set1_epi32(9); + let mut r = _mm_undefined_si128(); + _mm_store_epi32(&mut r as *mut _ as *mut i32, a); + assert_eq_m128i(r, a); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_load_ps() { + #[repr(align(64))] + struct Align { + data: [f32; 16], // 64 bytes + } + let a = Align { + data: [ + 4., 3., 2., 5., 8., 9., 64., 50., -4., -3., -2., -5., -8., -9., -64., -50., + ], + }; + let p = (a.data).as_ptr(); + let r = _mm512_load_ps(black_box(p)); + let e = _mm512_setr_ps( + 4., 3., 2., 5., 8., 9., 64., 50., -4., -3., -2., -5., -8., -9., -64., -50., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_store_ps() { + let a = _mm512_set1_ps(9.); + let mut r = _mm512_undefined_ps(); + _mm512_store_ps(&mut r as *mut _ as *mut f32, a); + assert_eq_m512(r, a); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_set1_epi32() { + let src = _mm512_set1_epi32(2); + let a: i32 = 11; + let r = _mm512_mask_set1_epi32(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_set1_epi32(src, 0b11111111_11111111, a); + let e = _mm512_set1_epi32(11); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_set1_epi32() { + let a: i32 = 11; + let r = _mm512_maskz_set1_epi32(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_set1_epi32(0b11111111_11111111, a); + let e = _mm512_set1_epi32(11); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_set1_epi32() { + let src = _mm256_set1_epi32(2); + let a: i32 = 11; + let r = _mm256_mask_set1_epi32(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_set1_epi32(src, 0b11111111, a); + let e = _mm256_set1_epi32(11); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm256_maskz_set1_epi32() { + let a: i32 = 11; + let r = _mm256_maskz_set1_epi32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_set1_epi32(0b11111111, a); + let e = _mm256_set1_epi32(11); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_set1_epi32() { + let src = _mm_set1_epi32(2); + let a: i32 = 11; + let r = _mm_mask_set1_epi32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_set1_epi32(src, 0b00001111, a); + let e = _mm_set1_epi32(11); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_set1_epi32() { + let a: i32 = 11; + let r = _mm_maskz_set1_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_set1_epi32(0b00001111, a); + let e = _mm_set1_epi32(11); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_move_ss() { + let src = _mm_set_ps(10., 11., 100., 110.); + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_mask_move_ss(src, 0, a, b); + let e = _mm_set_ps(1., 2., 10., 110.); + assert_eq_m128(r, e); + let r = _mm_mask_move_ss(src, 0b11111111, a, b); + let e = _mm_set_ps(1., 2., 10., 40.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_move_ss() { + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_maskz_move_ss(0, a, b); + let e = _mm_set_ps(1., 2., 10., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_move_ss(0b11111111, a, b); + let e = _mm_set_ps(1., 2., 10., 40.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_move_sd() { + let src = _mm_set_pd(10., 11.); + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_mask_move_sd(src, 0, a, b); + let e = _mm_set_pd(1., 11.); + assert_eq_m128d(r, e); + let r = _mm_mask_move_sd(src, 0b11111111, a, b); + let e = _mm_set_pd(1., 4.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_move_sd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_maskz_move_sd(0, a, b); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_move_sd(0b11111111, a, b); + let e = _mm_set_pd(1., 4.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_add_ss() { + let src = _mm_set_ps(10., 11., 100., 110.); + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_mask_add_ss(src, 0, a, b); + let e = _mm_set_ps(1., 2., 10., 110.); + assert_eq_m128(r, e); + let r = _mm_mask_add_ss(src, 0b11111111, a, b); + let e = _mm_set_ps(1., 2., 10., 60.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_add_ss() { + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_maskz_add_ss(0, a, b); + let e = _mm_set_ps(1., 2., 10., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_add_ss(0b11111111, a, b); + let e = _mm_set_ps(1., 2., 10., 60.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_add_sd() { + let src = _mm_set_pd(10., 11.); + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_mask_add_sd(src, 0, a, b); + let e = _mm_set_pd(1., 11.); + assert_eq_m128d(r, e); + let r = _mm_mask_add_sd(src, 0b11111111, a, b); + let e = _mm_set_pd(1., 6.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_add_sd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_maskz_add_sd(0, a, b); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_add_sd(0b11111111, a, b); + let e = _mm_set_pd(1., 6.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_sub_ss() { + let src = _mm_set_ps(10., 11., 100., 110.); + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_mask_sub_ss(src, 0, a, b); + let e = _mm_set_ps(1., 2., 10., 110.); + assert_eq_m128(r, e); + let r = _mm_mask_sub_ss(src, 0b11111111, a, b); + let e = _mm_set_ps(1., 2., 10., -20.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_sub_ss() { + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_maskz_sub_ss(0, a, b); + let e = _mm_set_ps(1., 2., 10., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_sub_ss(0b11111111, a, b); + let e = _mm_set_ps(1., 2., 10., -20.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_sub_sd() { + let src = _mm_set_pd(10., 11.); + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_mask_sub_sd(src, 0, a, b); + let e = _mm_set_pd(1., 11.); + assert_eq_m128d(r, e); + let r = _mm_mask_sub_sd(src, 0b11111111, a, b); + let e = _mm_set_pd(1., -2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_sub_sd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_maskz_sub_sd(0, a, b); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_sub_sd(0b11111111, a, b); + let e = _mm_set_pd(1., -2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_mul_ss() { + let src = _mm_set_ps(10., 11., 100., 110.); + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_mask_mul_ss(src, 0, a, b); + let e = _mm_set_ps(1., 2., 10., 110.); + assert_eq_m128(r, e); + let r = _mm_mask_mul_ss(src, 0b11111111, a, b); + let e = _mm_set_ps(1., 2., 10., 800.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_mul_ss() { + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_maskz_mul_ss(0, a, b); + let e = _mm_set_ps(1., 2., 10., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_mul_ss(0b11111111, a, b); + let e = _mm_set_ps(1., 2., 10., 800.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_mul_sd() { + let src = _mm_set_pd(10., 11.); + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_mask_mul_sd(src, 0, a, b); + let e = _mm_set_pd(1., 11.); + assert_eq_m128d(r, e); + let r = _mm_mask_mul_sd(src, 0b11111111, a, b); + let e = _mm_set_pd(1., 8.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_mul_sd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_maskz_mul_sd(0, a, b); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_mul_sd(0b11111111, a, b); + let e = _mm_set_pd(1., 8.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_div_ss() { + let src = _mm_set_ps(10., 11., 100., 110.); + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_mask_div_ss(src, 0, a, b); + let e = _mm_set_ps(1., 2., 10., 110.); + assert_eq_m128(r, e); + let r = _mm_mask_div_ss(src, 0b11111111, a, b); + let e = _mm_set_ps(1., 2., 10., 0.5); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_div_ss() { + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_maskz_div_ss(0, a, b); + let e = _mm_set_ps(1., 2., 10., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_div_ss(0b11111111, a, b); + let e = _mm_set_ps(1., 2., 10., 0.5); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_div_sd() { + let src = _mm_set_pd(10., 11.); + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_mask_div_sd(src, 0, a, b); + let e = _mm_set_pd(1., 11.); + assert_eq_m128d(r, e); + let r = _mm_mask_div_sd(src, 0b11111111, a, b); + let e = _mm_set_pd(1., 0.5); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_div_sd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_maskz_div_sd(0, a, b); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_div_sd(0b11111111, a, b); + let e = _mm_set_pd(1., 0.5); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_max_ss() { + let a = _mm_set_ps(0., 1., 2., 3.); + let b = _mm_set_ps(4., 5., 6., 7.); + let r = _mm_mask_max_ss(a, 0, a, b); + let e = _mm_set_ps(0., 1., 2., 3.); + assert_eq_m128(r, e); + let r = _mm_mask_max_ss(a, 0b11111111, a, b); + let e = _mm_set_ps(0., 1., 2., 7.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_max_ss() { + let a = _mm_set_ps(0., 1., 2., 3.); + let b = _mm_set_ps(4., 5., 6., 7.); + let r = _mm_maskz_max_ss(0, a, b); + let e = _mm_set_ps(0., 1., 2., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_max_ss(0b11111111, a, b); + let e = _mm_set_ps(0., 1., 2., 7.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_max_sd() { + let a = _mm_set_pd(0., 1.); + let b = _mm_set_pd(2., 3.); + let r = _mm_mask_max_sd(a, 0, a, b); + let e = _mm_set_pd(0., 1.); + assert_eq_m128d(r, e); + let r = _mm_mask_max_sd(a, 0b11111111, a, b); + let e = _mm_set_pd(0., 3.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_max_sd() { + let a = _mm_set_pd(0., 1.); + let b = _mm_set_pd(2., 3.); + let r = _mm_maskz_max_sd(0, a, b); + let e = _mm_set_pd(0., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_max_sd(0b11111111, a, b); + let e = _mm_set_pd(0., 3.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_min_ss() { + let a = _mm_set_ps(0., 1., 2., 3.); + let b = _mm_set_ps(4., 5., 6., 7.); + let r = _mm_mask_min_ss(a, 0, a, b); + let e = _mm_set_ps(0., 1., 2., 3.); + assert_eq_m128(r, e); + let r = _mm_mask_min_ss(a, 0b11111111, a, b); + let e = _mm_set_ps(0., 1., 2., 3.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_min_ss() { + let a = _mm_set_ps(0., 1., 2., 3.); + let b = _mm_set_ps(4., 5., 6., 7.); + let r = _mm_maskz_min_ss(0, a, b); + let e = _mm_set_ps(0., 1., 2., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_min_ss(0b11111111, a, b); + let e = _mm_set_ps(0., 1., 2., 3.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_min_sd() { + let a = _mm_set_pd(0., 1.); + let b = _mm_set_pd(2., 3.); + let r = _mm_mask_min_sd(a, 0, a, b); + let e = _mm_set_pd(0., 1.); + assert_eq_m128d(r, e); + let r = _mm_mask_min_sd(a, 0b11111111, a, b); + let e = _mm_set_pd(0., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_min_sd() { + let a = _mm_set_pd(0., 1.); + let b = _mm_set_pd(2., 3.); + let r = _mm_maskz_min_sd(0, a, b); + let e = _mm_set_pd(0., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_min_sd(0b11111111, a, b); + let e = _mm_set_pd(0., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_sqrt_ss() { + let src = _mm_set_ps(10., 11., 100., 110.); + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 4.); + let r = _mm_mask_sqrt_ss(src, 0, a, b); + let e = _mm_set_ps(1., 2., 10., 110.); + assert_eq_m128(r, e); + let r = _mm_mask_sqrt_ss(src, 0b11111111, a, b); + let e = _mm_set_ps(1., 2., 10., 2.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_sqrt_ss() { + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 4.); + let r = _mm_maskz_sqrt_ss(0, a, b); + let e = _mm_set_ps(1., 2., 10., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_sqrt_ss(0b11111111, a, b); + let e = _mm_set_ps(1., 2., 10., 2.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_sqrt_sd() { + let src = _mm_set_pd(10., 11.); + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_mask_sqrt_sd(src, 0, a, b); + let e = _mm_set_pd(1., 11.); + assert_eq_m128d(r, e); + let r = _mm_mask_sqrt_sd(src, 0b11111111, a, b); + let e = _mm_set_pd(1., 2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_sqrt_sd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_maskz_sqrt_sd(0, a, b); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_sqrt_sd(0b11111111, a, b); + let e = _mm_set_pd(1., 2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_rsqrt14_ss() { + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 4.); + let r = _mm_rsqrt14_ss(a, b); + let e = _mm_set_ps(1., 2., 10., 0.5); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_rsqrt14_ss() { + let src = _mm_set_ps(10., 11., 100., 110.); + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 4.); + let r = _mm_mask_rsqrt14_ss(src, 0, a, b); + let e = _mm_set_ps(1., 2., 10., 110.); + assert_eq_m128(r, e); + let r = _mm_mask_rsqrt14_ss(src, 0b11111111, a, b); + let e = _mm_set_ps(1., 2., 10., 0.5); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_rsqrt14_ss() { + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 4.); + let r = _mm_maskz_rsqrt14_ss(0, a, b); + let e = _mm_set_ps(1., 2., 10., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_rsqrt14_ss(0b11111111, a, b); + let e = _mm_set_ps(1., 2., 10., 0.5); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_rsqrt14_sd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_rsqrt14_sd(a, b); + let e = _mm_set_pd(1., 0.5); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_rsqrt14_sd() { + let src = _mm_set_pd(10., 11.); + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_mask_rsqrt14_sd(src, 0, a, b); + let e = _mm_set_pd(1., 11.); + assert_eq_m128d(r, e); + let r = _mm_mask_rsqrt14_sd(src, 0b11111111, a, b); + let e = _mm_set_pd(1., 0.5); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_rsqrt14_sd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_maskz_rsqrt14_sd(0, a, b); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_rsqrt14_sd(0b11111111, a, b); + let e = _mm_set_pd(1., 0.5); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_rcp14_ss() { + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 4.); + let r = _mm_rcp14_ss(a, b); + let e = _mm_set_ps(1., 2., 10., 0.25); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_rcp14_ss() { + let src = _mm_set_ps(10., 11., 100., 110.); + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 4.); + let r = _mm_mask_rcp14_ss(src, 0, a, b); + let e = _mm_set_ps(1., 2., 10., 110.); + assert_eq_m128(r, e); + let r = _mm_mask_rcp14_ss(src, 0b11111111, a, b); + let e = _mm_set_ps(1., 2., 10., 0.25); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_rcp14_ss() { + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 4.); + let r = _mm_maskz_rcp14_ss(0, a, b); + let e = _mm_set_ps(1., 2., 10., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_rcp14_ss(0b11111111, a, b); + let e = _mm_set_ps(1., 2., 10., 0.25); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_rcp14_sd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_rcp14_sd(a, b); + let e = _mm_set_pd(1., 0.25); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_rcp14_sd() { + let src = _mm_set_pd(10., 11.); + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_mask_rcp14_sd(src, 0, a, b); + let e = _mm_set_pd(1., 11.); + assert_eq_m128d(r, e); + let r = _mm_mask_rcp14_sd(src, 0b11111111, a, b); + let e = _mm_set_pd(1., 0.25); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_rcp14_sd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_maskz_rcp14_sd(0, a, b); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_rcp14_sd(0b11111111, a, b); + let e = _mm_set_pd(1., 0.25); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_getexp_ss() { + let a = _mm_set1_ps(2.); + let b = _mm_set1_ps(3.); + let r = _mm_getexp_ss(a, b); + let e = _mm_set_ps(2., 2., 2., 1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_getexp_ss() { + let a = _mm_set1_ps(2.); + let b = _mm_set1_ps(3.); + let r = _mm_mask_getexp_ss(a, 0, a, b); + let e = _mm_set_ps(2., 2., 2., 2.); + assert_eq_m128(r, e); + let r = _mm_mask_getexp_ss(a, 0b11111111, a, b); + let e = _mm_set_ps(2., 2., 2., 1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_getexp_ss() { + let a = _mm_set1_ps(2.); + let b = _mm_set1_ps(3.); + let r = _mm_maskz_getexp_ss(0, a, b); + let e = _mm_set_ps(2., 2., 2., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_getexp_ss(0b11111111, a, b); + let e = _mm_set_ps(2., 2., 2., 1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_getexp_sd() { + let a = _mm_set1_pd(2.); + let b = _mm_set1_pd(3.); + let r = _mm_getexp_sd(a, b); + let e = _mm_set_pd(2., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_getexp_sd() { + let a = _mm_set1_pd(2.); + let b = _mm_set1_pd(3.); + let r = _mm_mask_getexp_sd(a, 0, a, b); + let e = _mm_set_pd(2., 2.); + assert_eq_m128d(r, e); + let r = _mm_mask_getexp_sd(a, 0b11111111, a, b); + let e = _mm_set_pd(2., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_getexp_sd() { + let a = _mm_set1_pd(2.); + let b = _mm_set1_pd(3.); + let r = _mm_maskz_getexp_sd(0, a, b); + let e = _mm_set_pd(2., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_getexp_sd(0b11111111, a, b); + let e = _mm_set_pd(2., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_getmant_ss() { + let a = _mm_set1_ps(20.); + let b = _mm_set1_ps(10.); + let r = _mm_getmant_ss::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, b); + let e = _mm_set_ps(20., 20., 20., 1.25); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_getmant_ss() { + let a = _mm_set1_ps(20.); + let b = _mm_set1_ps(10.); + let r = _mm_mask_getmant_ss::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0, a, b); + let e = _mm_set_ps(20., 20., 20., 20.); + assert_eq_m128(r, e); + let r = _mm_mask_getmant_ss::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0b11111111, a, b); + let e = _mm_set_ps(20., 20., 20., 1.25); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_getmant_ss() { + let a = _mm_set1_ps(20.); + let b = _mm_set1_ps(10.); + let r = _mm_maskz_getmant_ss::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0, a, b); + let e = _mm_set_ps(20., 20., 20., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_getmant_ss::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0b11111111, a, b); + let e = _mm_set_ps(20., 20., 20., 1.25); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_getmant_sd() { + let a = _mm_set1_pd(20.); + let b = _mm_set1_pd(10.); + let r = _mm_getmant_sd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, b); + let e = _mm_set_pd(20., 1.25); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_getmant_sd() { + let a = _mm_set1_pd(20.); + let b = _mm_set1_pd(10.); + let r = _mm_mask_getmant_sd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0, a, b); + let e = _mm_set_pd(20., 20.); + assert_eq_m128d(r, e); + let r = _mm_mask_getmant_sd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0b11111111, a, b); + let e = _mm_set_pd(20., 1.25); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_getmant_sd() { + let a = _mm_set1_pd(20.); + let b = _mm_set1_pd(10.); + let r = _mm_maskz_getmant_sd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0, a, b); + let e = _mm_set_pd(20., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_getmant_sd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0b11111111, a, b); + let e = _mm_set_pd(20., 1.25); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_roundscale_ss() { + let a = _mm_set1_ps(2.2); + let b = _mm_set1_ps(1.1); + let r = _mm_roundscale_ss::<0>(a, b); + let e = _mm_set_ps(2.2, 2.2, 2.2, 1.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_roundscale_ss() { + let a = _mm_set1_ps(2.2); + let b = _mm_set1_ps(1.1); + let r = _mm_mask_roundscale_ss::<0>(a, 0, a, b); + let e = _mm_set_ps(2.2, 2.2, 2.2, 2.2); + assert_eq_m128(r, e); + let r = _mm_mask_roundscale_ss::<0>(a, 0b11111111, a, b); + let e = _mm_set_ps(2.2, 2.2, 2.2, 1.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_roundscale_ss() { + let a = _mm_set1_ps(2.2); + let b = _mm_set1_ps(1.1); + let r = _mm_maskz_roundscale_ss::<0>(0, a, b); + let e = _mm_set_ps(2.2, 2.2, 2.2, 0.0); + assert_eq_m128(r, e); + let r = _mm_maskz_roundscale_ss::<0>(0b11111111, a, b); + let e = _mm_set_ps(2.2, 2.2, 2.2, 1.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_roundscale_sd() { + let a = _mm_set1_pd(2.2); + let b = _mm_set1_pd(1.1); + let r = _mm_roundscale_sd::<0>(a, b); + let e = _mm_set_pd(2.2, 1.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_roundscale_sd() { + let a = _mm_set1_pd(2.2); + let b = _mm_set1_pd(1.1); + let r = _mm_mask_roundscale_sd::<0>(a, 0, a, b); + let e = _mm_set_pd(2.2, 2.2); + assert_eq_m128d(r, e); + let r = _mm_mask_roundscale_sd::<0>(a, 0b11111111, a, b); + let e = _mm_set_pd(2.2, 1.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_roundscale_sd() { + let a = _mm_set1_pd(2.2); + let b = _mm_set1_pd(1.1); + let r = _mm_maskz_roundscale_sd::<0>(0, a, b); + let e = _mm_set_pd(2.2, 0.0); + assert_eq_m128d(r, e); + let r = _mm_maskz_roundscale_sd::<0>(0b11111111, a, b); + let e = _mm_set_pd(2.2, 1.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_scalef_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(3.); + let r = _mm_scalef_ss(a, b); + let e = _mm_set_ps(1., 1., 1., 8.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_scalef_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(3.); + let r = _mm_mask_scalef_ss(a, 0, a, b); + let e = _mm_set_ps(1., 1., 1., 1.); + assert_eq_m128(r, e); + let r = _mm_mask_scalef_ss(a, 0b11111111, a, b); + let e = _mm_set_ps(1., 1., 1., 8.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_scalef_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(3.); + let r = _mm_maskz_scalef_ss(0, a, b); + let e = _mm_set_ps(1., 1., 1., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_scalef_ss(0b11111111, a, b); + let e = _mm_set_ps(1., 1., 1., 8.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_scalef_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(3.); + let r = _mm_scalef_sd(a, b); + let e = _mm_set_pd(1., 8.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_scalef_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(3.); + let r = _mm_mask_scalef_sd(a, 0, a, b); + let e = _mm_set_pd(1., 1.); + assert_eq_m128d(r, e); + let r = _mm_mask_scalef_sd(a, 0b11111111, a, b); + let e = _mm_set_pd(1., 8.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_scalef_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(3.); + let r = _mm_maskz_scalef_sd(0, a, b); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_scalef_sd(0b11111111, a, b); + let e = _mm_set_pd(1., 8.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_fmadd_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_mask_fmadd_ss(a, 0, b, c); + assert_eq_m128(r, a); + let r = _mm_mask_fmadd_ss(a, 0b11111111, b, c); + let e = _mm_set_ps(1., 1., 1., 5.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_fmadd_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_maskz_fmadd_ss(0, a, b, c); + let e = _mm_set_ps(1., 1., 1., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_fmadd_ss(0b11111111, a, b, c); + let e = _mm_set_ps(1., 1., 1., 5.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask3_fmadd_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_mask3_fmadd_ss(a, b, c, 0); + assert_eq_m128(r, c); + let r = _mm_mask3_fmadd_ss(a, b, c, 0b11111111); + let e = _mm_set_ps(3., 3., 3., 5.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_fmadd_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_mask_fmadd_sd(a, 0, b, c); + assert_eq_m128d(r, a); + let r = _mm_mask_fmadd_sd(a, 0b11111111, b, c); + let e = _mm_set_pd(1., 5.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_fmadd_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_maskz_fmadd_sd(0, a, b, c); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_fmadd_sd(0b11111111, a, b, c); + let e = _mm_set_pd(1., 5.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask3_fmadd_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_mask3_fmadd_sd(a, b, c, 0); + assert_eq_m128d(r, c); + let r = _mm_mask3_fmadd_sd(a, b, c, 0b11111111); + let e = _mm_set_pd(3., 5.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_fmsub_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_mask_fmsub_ss(a, 0, b, c); + assert_eq_m128(r, a); + let r = _mm_mask_fmsub_ss(a, 0b11111111, b, c); + let e = _mm_set_ps(1., 1., 1., -1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_fmsub_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_maskz_fmsub_ss(0, a, b, c); + let e = _mm_set_ps(1., 1., 1., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_fmsub_ss(0b11111111, a, b, c); + let e = _mm_set_ps(1., 1., 1., -1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask3_fmsub_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_mask3_fmsub_ss(a, b, c, 0); + assert_eq_m128(r, c); + let r = _mm_mask3_fmsub_ss(a, b, c, 0b11111111); + let e = _mm_set_ps(3., 3., 3., -1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_fmsub_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_mask_fmsub_sd(a, 0, b, c); + assert_eq_m128d(r, a); + let r = _mm_mask_fmsub_sd(a, 0b11111111, b, c); + let e = _mm_set_pd(1., -1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_fmsub_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_maskz_fmsub_sd(0, a, b, c); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_fmsub_sd(0b11111111, a, b, c); + let e = _mm_set_pd(1., -1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask3_fmsub_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_mask3_fmsub_sd(a, b, c, 0); + assert_eq_m128d(r, c); + let r = _mm_mask3_fmsub_sd(a, b, c, 0b11111111); + let e = _mm_set_pd(3., -1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_fnmadd_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_mask_fnmadd_ss(a, 0, b, c); + assert_eq_m128(r, a); + let r = _mm_mask_fnmadd_ss(a, 0b11111111, b, c); + let e = _mm_set_ps(1., 1., 1., 1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_fnmadd_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_maskz_fnmadd_ss(0, a, b, c); + let e = _mm_set_ps(1., 1., 1., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_fnmadd_ss(0b11111111, a, b, c); + let e = _mm_set_ps(1., 1., 1., 1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask3_fnmadd_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_mask3_fnmadd_ss(a, b, c, 0); + assert_eq_m128(r, c); + let r = _mm_mask3_fnmadd_ss(a, b, c, 0b11111111); + let e = _mm_set_ps(3., 3., 3., 1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_fnmadd_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_mask_fnmadd_sd(a, 0, b, c); + assert_eq_m128d(r, a); + let r = _mm_mask_fnmadd_sd(a, 0b11111111, b, c); + let e = _mm_set_pd(1., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_fnmadd_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_maskz_fnmadd_sd(0, a, b, c); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_fnmadd_sd(0b11111111, a, b, c); + let e = _mm_set_pd(1., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask3_fnmadd_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_mask3_fnmadd_sd(a, b, c, 0); + assert_eq_m128d(r, c); + let r = _mm_mask3_fnmadd_sd(a, b, c, 0b11111111); + let e = _mm_set_pd(3., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_fnmsub_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_mask_fnmsub_ss(a, 0, b, c); + assert_eq_m128(r, a); + let r = _mm_mask_fnmsub_ss(a, 0b11111111, b, c); + let e = _mm_set_ps(1., 1., 1., -5.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_fnmsub_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_maskz_fnmsub_ss(0, a, b, c); + let e = _mm_set_ps(1., 1., 1., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_fnmsub_ss(0b11111111, a, b, c); + let e = _mm_set_ps(1., 1., 1., -5.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask3_fnmsub_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_mask3_fnmsub_ss(a, b, c, 0); + assert_eq_m128(r, c); + let r = _mm_mask3_fnmsub_ss(a, b, c, 0b11111111); + let e = _mm_set_ps(3., 3., 3., -5.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_fnmsub_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_mask_fnmsub_sd(a, 0, b, c); + assert_eq_m128d(r, a); + let r = _mm_mask_fnmsub_sd(a, 0b11111111, b, c); + let e = _mm_set_pd(1., -5.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_fnmsub_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_maskz_fnmsub_sd(0, a, b, c); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_fnmsub_sd(0b11111111, a, b, c); + let e = _mm_set_pd(1., -5.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask3_fnmsub_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_mask3_fnmsub_sd(a, b, c, 0); + assert_eq_m128d(r, c); + let r = _mm_mask3_fnmsub_sd(a, b, c, 0b11111111); + let e = _mm_set_pd(3., -5.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_add_round_ss() { + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_add_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_ps(1., 2., 10., 60.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_add_round_ss() { + let src = _mm_set_ps(10., 11., 100., 110.); + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_mask_add_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(src, 0, a, b); + let e = _mm_set_ps(1., 2., 10., 110.); + assert_eq_m128(r, e); + let r = _mm_mask_add_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + src, 0b11111111, a, b, + ); + let e = _mm_set_ps(1., 2., 10., 60.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_add_round_ss() { + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_maskz_add_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); + let e = _mm_set_ps(1., 2., 10., 0.); + assert_eq_m128(r, e); + let r = + _mm_maskz_add_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0b11111111, a, b); + let e = _mm_set_ps(1., 2., 10., 60.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_add_round_sd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_add_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_pd(1., 6.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_add_round_sd() { + let src = _mm_set_pd(10., 11.); + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_mask_add_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(src, 0, a, b); + let e = _mm_set_pd(1., 11.); + assert_eq_m128d(r, e); + let r = _mm_mask_add_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + src, 0b11111111, a, b, + ); + let e = _mm_set_pd(1., 6.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_add_round_sd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_maskz_add_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = + _mm_maskz_add_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0b11111111, a, b); + let e = _mm_set_pd(1., 6.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_sub_round_ss() { + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_sub_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_ps(1., 2., 10., -20.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_sub_round_ss() { + let src = _mm_set_ps(10., 11., 100., 110.); + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_mask_sub_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(src, 0, a, b); + let e = _mm_set_ps(1., 2., 10., 110.); + assert_eq_m128(r, e); + let r = _mm_mask_sub_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + src, 0b11111111, a, b, + ); + let e = _mm_set_ps(1., 2., 10., -20.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_sub_round_ss() { + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_maskz_sub_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); + let e = _mm_set_ps(1., 2., 10., 0.); + assert_eq_m128(r, e); + let r = + _mm_maskz_sub_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0b11111111, a, b); + let e = _mm_set_ps(1., 2., 10., -20.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_sub_round_sd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_sub_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_pd(1., -2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_sub_round_sd() { + let src = _mm_set_pd(10., 11.); + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_mask_sub_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(src, 0, a, b); + let e = _mm_set_pd(1., 11.); + assert_eq_m128d(r, e); + let r = _mm_mask_sub_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + src, 0b11111111, a, b, + ); + let e = _mm_set_pd(1., -2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_sub_round_sd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_maskz_sub_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = + _mm_maskz_sub_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0b11111111, a, b); + let e = _mm_set_pd(1., -2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mul_round_ss() { + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_mul_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_ps(1., 2., 10., 800.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_mul_round_ss() { + let src = _mm_set_ps(10., 11., 100., 110.); + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_mask_mul_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(src, 0, a, b); + let e = _mm_set_ps(1., 2., 10., 110.); + assert_eq_m128(r, e); + let r = _mm_mask_mul_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + src, 0b11111111, a, b, + ); + let e = _mm_set_ps(1., 2., 10., 800.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_mul_round_ss() { + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_maskz_mul_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); + let e = _mm_set_ps(1., 2., 10., 0.); + assert_eq_m128(r, e); + let r = + _mm_maskz_mul_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0b11111111, a, b); + let e = _mm_set_ps(1., 2., 10., 800.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mul_round_sd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_mul_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_pd(1., 8.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_mul_round_sd() { + let src = _mm_set_pd(10., 11.); + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_mask_mul_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(src, 0, a, b); + let e = _mm_set_pd(1., 11.); + assert_eq_m128d(r, e); + let r = _mm_mask_mul_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + src, 0b11111111, a, b, + ); + let e = _mm_set_pd(1., 8.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_mul_round_sd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_maskz_mul_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = + _mm_maskz_mul_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0b11111111, a, b); + let e = _mm_set_pd(1., 8.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_div_round_ss() { + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_div_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_ps(1., 2., 10., 0.5); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_div_round_ss() { + let src = _mm_set_ps(10., 11., 100., 110.); + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_mask_div_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(src, 0, a, b); + let e = _mm_set_ps(1., 2., 10., 110.); + assert_eq_m128(r, e); + let r = _mm_mask_div_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + src, 0b11111111, a, b, + ); + let e = _mm_set_ps(1., 2., 10., 0.5); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_div_round_ss() { + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 40.); + let r = _mm_maskz_div_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); + let e = _mm_set_ps(1., 2., 10., 0.); + assert_eq_m128(r, e); + let r = + _mm_maskz_div_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0b11111111, a, b); + let e = _mm_set_ps(1., 2., 10., 0.5); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_div_round_sd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_div_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_pd(1., 0.5); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_div_round_sd() { + let src = _mm_set_pd(10., 11.); + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_mask_div_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(src, 0, a, b); + let e = _mm_set_pd(1., 11.); + assert_eq_m128d(r, e); + let r = _mm_mask_div_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + src, 0b11111111, a, b, + ); + let e = _mm_set_pd(1., 0.5); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_div_round_sd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_maskz_div_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = + _mm_maskz_div_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0b11111111, a, b); + let e = _mm_set_pd(1., 0.5); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_max_round_ss() { + let a = _mm_set_ps(0., 1., 2., 3.); + let b = _mm_set_ps(4., 5., 6., 7.); + let r = _mm_max_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, b); + let e = _mm_set_ps(0., 1., 2., 7.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_max_round_ss() { + let a = _mm_set_ps(0., 1., 2., 3.); + let b = _mm_set_ps(4., 5., 6., 7.); + let r = _mm_mask_max_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); + let e = _mm_set_ps(0., 1., 2., 3.); + assert_eq_m128(r, e); + let r = _mm_mask_max_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, 0b11111111, a, b); + let e = _mm_set_ps(0., 1., 2., 7.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_max_round_ss() { + let a = _mm_set_ps(0., 1., 2., 3.); + let b = _mm_set_ps(4., 5., 6., 7.); + let r = _mm_maskz_max_round_ss::<_MM_FROUND_CUR_DIRECTION>(0, a, b); + let e = _mm_set_ps(0., 1., 2., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_max_round_ss::<_MM_FROUND_CUR_DIRECTION>(0b11111111, a, b); + let e = _mm_set_ps(0., 1., 2., 7.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_max_round_sd() { + let a = _mm_set_pd(0., 1.); + let b = _mm_set_pd(2., 3.); + let r = _mm_max_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, b); + let e = _mm_set_pd(0., 3.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_max_round_sd() { + let a = _mm_set_pd(0., 1.); + let b = _mm_set_pd(2., 3.); + let r = _mm_mask_max_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); + let e = _mm_set_pd(0., 1.); + assert_eq_m128d(r, e); + let r = _mm_mask_max_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, 0b11111111, a, b); + let e = _mm_set_pd(0., 3.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_max_round_sd() { + let a = _mm_set_pd(0., 1.); + let b = _mm_set_pd(2., 3.); + let r = _mm_maskz_max_round_sd::<_MM_FROUND_CUR_DIRECTION>(0, a, b); + let e = _mm_set_pd(0., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_max_round_sd::<_MM_FROUND_CUR_DIRECTION>(0b11111111, a, b); + let e = _mm_set_pd(0., 3.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_min_round_ss() { + let a = _mm_set_ps(0., 1., 2., 3.); + let b = _mm_set_ps(4., 5., 6., 7.); + let r = _mm_min_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, b); + let e = _mm_set_ps(0., 1., 2., 3.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_min_round_ss() { + let a = _mm_set_ps(0., 1., 2., 3.); + let b = _mm_set_ps(4., 5., 6., 7.); + let r = _mm_mask_min_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); + let e = _mm_set_ps(0., 1., 2., 3.); + assert_eq_m128(r, e); + let r = _mm_mask_min_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, 0b11111111, a, b); + let e = _mm_set_ps(0., 1., 2., 3.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_min_round_ss() { + let a = _mm_set_ps(0., 1., 2., 3.); + let b = _mm_set_ps(4., 5., 6., 7.); + let r = _mm_maskz_min_round_ss::<_MM_FROUND_CUR_DIRECTION>(0, a, b); + let e = _mm_set_ps(0., 1., 2., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_min_round_ss::<_MM_FROUND_CUR_DIRECTION>(0b11111111, a, b); + let e = _mm_set_ps(0., 1., 2., 3.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_min_round_sd() { + let a = _mm_set_pd(0., 1.); + let b = _mm_set_pd(2., 3.); + let r = _mm_min_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, b); + let e = _mm_set_pd(0., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_min_round_sd() { + let a = _mm_set_pd(0., 1.); + let b = _mm_set_pd(2., 3.); + let r = _mm_mask_min_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); + let e = _mm_set_pd(0., 1.); + assert_eq_m128d(r, e); + let r = _mm_mask_min_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, 0b11111111, a, b); + let e = _mm_set_pd(0., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_min_round_sd() { + let a = _mm_set_pd(0., 1.); + let b = _mm_set_pd(2., 3.); + let r = _mm_maskz_min_round_sd::<_MM_FROUND_CUR_DIRECTION>(0, a, b); + let e = _mm_set_pd(0., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_min_round_sd::<_MM_FROUND_CUR_DIRECTION>(0b11111111, a, b); + let e = _mm_set_pd(0., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_sqrt_round_ss() { + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 4.); + let r = _mm_sqrt_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_ps(1., 2., 10., 2.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_sqrt_round_ss() { + let src = _mm_set_ps(10., 11., 100., 110.); + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 4.); + let r = _mm_mask_sqrt_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(src, 0, a, b); + let e = _mm_set_ps(1., 2., 10., 110.); + assert_eq_m128(r, e); + let r = _mm_mask_sqrt_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + src, 0b11111111, a, b, + ); + let e = _mm_set_ps(1., 2., 10., 2.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_sqrt_round_ss() { + let a = _mm_set_ps(1., 2., 10., 20.); + let b = _mm_set_ps(3., 4., 30., 4.); + let r = _mm_maskz_sqrt_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); + let e = _mm_set_ps(1., 2., 10., 0.); + assert_eq_m128(r, e); + let r = + _mm_maskz_sqrt_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0b11111111, a, b); + let e = _mm_set_ps(1., 2., 10., 2.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_sqrt_round_sd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_sqrt_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_pd(1., 2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_sqrt_round_sd() { + let src = _mm_set_pd(10., 11.); + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_mask_sqrt_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(src, 0, a, b); + let e = _mm_set_pd(1., 11.); + assert_eq_m128d(r, e); + let r = _mm_mask_sqrt_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + src, 0b11111111, a, b, + ); + let e = _mm_set_pd(1., 2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_sqrt_round_sd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(3., 4.); + let r = _mm_maskz_sqrt_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = + _mm_maskz_sqrt_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0b11111111, a, b); + let e = _mm_set_pd(1., 2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_getexp_round_ss() { + let a = _mm_set1_ps(2.); + let b = _mm_set1_ps(3.); + let r = _mm_getexp_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, b); + let e = _mm_set_ps(2., 2., 2., 1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_getexp_round_ss() { + let a = _mm_set1_ps(2.); + let b = _mm_set1_ps(3.); + let r = _mm_mask_getexp_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); + let e = _mm_set_ps(2., 2., 2., 2.); + assert_eq_m128(r, e); + let r = _mm_mask_getexp_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, 0b11111111, a, b); + let e = _mm_set_ps(2., 2., 2., 1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_getexp_round_ss() { + let a = _mm_set1_ps(2.); + let b = _mm_set1_ps(3.); + let r = _mm_maskz_getexp_round_ss::<_MM_FROUND_CUR_DIRECTION>(0, a, b); + let e = _mm_set_ps(2., 2., 2., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_getexp_round_ss::<_MM_FROUND_CUR_DIRECTION>(0b11111111, a, b); + let e = _mm_set_ps(2., 2., 2., 1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_getexp_round_sd() { + let a = _mm_set1_pd(2.); + let b = _mm_set1_pd(3.); + let r = _mm_getexp_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, b); + let e = _mm_set_pd(2., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_getexp_round_sd() { + let a = _mm_set1_pd(2.); + let b = _mm_set1_pd(3.); + let r = _mm_mask_getexp_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); + let e = _mm_set_pd(2., 2.); + assert_eq_m128d(r, e); + let r = _mm_mask_getexp_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, 0b11111111, a, b); + let e = _mm_set_pd(2., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_getexp_round_sd() { + let a = _mm_set1_pd(2.); + let b = _mm_set1_pd(3.); + let r = _mm_maskz_getexp_round_sd::<_MM_FROUND_CUR_DIRECTION>(0, a, b); + let e = _mm_set_pd(2., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_getexp_round_sd::<_MM_FROUND_CUR_DIRECTION>(0b11111111, a, b); + let e = _mm_set_pd(2., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_getmant_round_ss() { + let a = _mm_set1_ps(20.); + let b = _mm_set1_ps(10.); + let r = + _mm_getmant_round_ss::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION>( + a, b, + ); + let e = _mm_set_ps(20., 20., 20., 1.25); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_getmant_round_ss() { + let a = _mm_set1_ps(20.); + let b = _mm_set1_ps(10.); + let r = _mm_mask_getmant_round_ss::< + _MM_MANT_NORM_1_2, + _MM_MANT_SIGN_SRC, + _MM_FROUND_CUR_DIRECTION, + >(a, 0, a, b); + let e = _mm_set_ps(20., 20., 20., 20.); + assert_eq_m128(r, e); + let r = _mm_mask_getmant_round_ss::< + _MM_MANT_NORM_1_2, + _MM_MANT_SIGN_SRC, + _MM_FROUND_CUR_DIRECTION, + >(a, 0b11111111, a, b); + let e = _mm_set_ps(20., 20., 20., 1.25); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_getmant_round_ss() { + let a = _mm_set1_ps(20.); + let b = _mm_set1_ps(10.); + let r = _mm_maskz_getmant_round_ss::< + _MM_MANT_NORM_1_2, + _MM_MANT_SIGN_SRC, + _MM_FROUND_CUR_DIRECTION, + >(0, a, b); + let e = _mm_set_ps(20., 20., 20., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_getmant_round_ss::< + _MM_MANT_NORM_1_2, + _MM_MANT_SIGN_SRC, + _MM_FROUND_CUR_DIRECTION, + >(0b11111111, a, b); + let e = _mm_set_ps(20., 20., 20., 1.25); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_getmant_round_sd() { + let a = _mm_set1_pd(20.); + let b = _mm_set1_pd(10.); + let r = + _mm_getmant_round_sd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION>( + a, b, + ); + let e = _mm_set_pd(20., 1.25); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_getmant_round_sd() { + let a = _mm_set1_pd(20.); + let b = _mm_set1_pd(10.); + let r = _mm_mask_getmant_round_sd::< + _MM_MANT_NORM_1_2, + _MM_MANT_SIGN_SRC, + _MM_FROUND_CUR_DIRECTION, + >(a, 0, a, b); + let e = _mm_set_pd(20., 20.); + assert_eq_m128d(r, e); + let r = _mm_mask_getmant_round_sd::< + _MM_MANT_NORM_1_2, + _MM_MANT_SIGN_SRC, + _MM_FROUND_CUR_DIRECTION, + >(a, 0b11111111, a, b); + let e = _mm_set_pd(20., 1.25); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_getmant_round_sd() { + let a = _mm_set1_pd(20.); + let b = _mm_set1_pd(10.); + let r = _mm_maskz_getmant_round_sd::< + _MM_MANT_NORM_1_2, + _MM_MANT_SIGN_SRC, + _MM_FROUND_CUR_DIRECTION, + >(0, a, b); + let e = _mm_set_pd(20., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_getmant_round_sd::< + _MM_MANT_NORM_1_2, + _MM_MANT_SIGN_SRC, + _MM_FROUND_CUR_DIRECTION, + >(0b11111111, a, b); + let e = _mm_set_pd(20., 1.25); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_roundscale_round_ss() { + let a = _mm_set1_ps(2.2); + let b = _mm_set1_ps(1.1); + let r = _mm_roundscale_round_ss::<0, _MM_FROUND_CUR_DIRECTION>(a, b); + let e = _mm_set_ps(2.2, 2.2, 2.2, 1.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_roundscale_round_ss() { + let a = _mm_set1_ps(2.2); + let b = _mm_set1_ps(1.1); + let r = _mm_mask_roundscale_round_ss::<0, _MM_FROUND_CUR_DIRECTION>(a, 0, a, b); + let e = _mm_set_ps(2.2, 2.2, 2.2, 2.2); + assert_eq_m128(r, e); + let r = _mm_mask_roundscale_round_ss::<0, _MM_FROUND_CUR_DIRECTION>(a, 0b11111111, a, b); + let e = _mm_set_ps(2.2, 2.2, 2.2, 1.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_roundscale_round_ss() { + let a = _mm_set1_ps(2.2); + let b = _mm_set1_ps(1.1); + let r = _mm_maskz_roundscale_round_ss::<0, _MM_FROUND_CUR_DIRECTION>(0, a, b); + let e = _mm_set_ps(2.2, 2.2, 2.2, 0.0); + assert_eq_m128(r, e); + let r = _mm_maskz_roundscale_round_ss::<0, _MM_FROUND_CUR_DIRECTION>(0b11111111, a, b); + let e = _mm_set_ps(2.2, 2.2, 2.2, 1.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_roundscale_round_sd() { + let a = _mm_set1_pd(2.2); + let b = _mm_set1_pd(1.1); + let r = _mm_roundscale_round_sd::<0, _MM_FROUND_CUR_DIRECTION>(a, b); + let e = _mm_set_pd(2.2, 1.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_roundscale_round_sd() { + let a = _mm_set1_pd(2.2); + let b = _mm_set1_pd(1.1); + let r = _mm_mask_roundscale_round_sd::<0, _MM_FROUND_CUR_DIRECTION>(a, 0, a, b); + let e = _mm_set_pd(2.2, 2.2); + assert_eq_m128d(r, e); + let r = _mm_mask_roundscale_round_sd::<0, _MM_FROUND_CUR_DIRECTION>(a, 0b11111111, a, b); + let e = _mm_set_pd(2.2, 1.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_roundscale_round_sd() { + let a = _mm_set1_pd(2.2); + let b = _mm_set1_pd(1.1); + let r = _mm_maskz_roundscale_round_sd::<0, _MM_FROUND_CUR_DIRECTION>(0, a, b); + let e = _mm_set_pd(2.2, 0.0); + assert_eq_m128d(r, e); + let r = _mm_maskz_roundscale_round_sd::<0, _MM_FROUND_CUR_DIRECTION>(0b11111111, a, b); + let e = _mm_set_pd(2.2, 1.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_scalef_round_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(3.); + let r = _mm_scalef_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_ps(1., 1., 1., 8.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_scalef_round_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(3.); + let r = _mm_mask_scalef_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); + let e = _mm_set_ps(1., 1., 1., 1.); + assert_eq_m128(r, e); + let r = _mm_mask_scalef_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11111111, a, b, + ); + let e = _mm_set_ps(1., 1., 1., 8.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_scalef_round_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(3.); + let r = + _mm_maskz_scalef_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a, b); + let e = _mm_set_ps(1., 1., 1., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_scalef_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, + ); + let e = _mm_set_ps(1., 1., 1., 8.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_scalef_round_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(3.); + let r = _mm_scalef_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_pd(1., 8.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_scalef_round_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(3.); + let r = _mm_mask_scalef_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); + let e = _mm_set_pd(1., 1.); + assert_eq_m128d(r, e); + let r = _mm_mask_scalef_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11111111, a, b, + ); + let e = _mm_set_pd(1., 8.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_scalef_round_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(3.); + let r = + _mm_maskz_scalef_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a, b); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_scalef_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, + ); + let e = _mm_set_pd(1., 8.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_fmadd_round_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_fmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm_set_ps(1., 1., 1., 5.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_fmadd_round_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_mask_fmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); + assert_eq_m128(r, a); + let r = _mm_mask_fmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11111111, b, c, + ); + let e = _mm_set_ps(1., 1., 1., 5.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_fmadd_round_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_maskz_fmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); + let e = _mm_set_ps(1., 1., 1., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_fmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, c, + ); + let e = _mm_set_ps(1., 1., 1., 5.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask3_fmadd_round_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_mask3_fmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); + assert_eq_m128(r, c); + let r = _mm_mask3_fmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b11111111, + ); + let e = _mm_set_ps(3., 3., 3., 5.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_fmadd_round_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_fmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm_set_pd(1., 5.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_fmadd_round_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_mask_fmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); + assert_eq_m128d(r, a); + let r = _mm_mask_fmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11111111, b, c, + ); + let e = _mm_set_pd(1., 5.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_fmadd_round_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_maskz_fmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_fmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, c, + ); + let e = _mm_set_pd(1., 5.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask3_fmadd_round_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_mask3_fmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); + assert_eq_m128d(r, c); + let r = _mm_mask3_fmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b11111111, + ); + let e = _mm_set_pd(3., 5.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_fmsub_round_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_fmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm_set_ps(1., 1., 1., -1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_fmsub_round_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_mask_fmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); + assert_eq_m128(r, a); + let r = _mm_mask_fmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11111111, b, c, + ); + let e = _mm_set_ps(1., 1., 1., -1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_fmsub_round_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_maskz_fmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); + let e = _mm_set_ps(1., 1., 1., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_fmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, c, + ); + let e = _mm_set_ps(1., 1., 1., -1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask3_fmsub_round_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_mask3_fmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); + assert_eq_m128(r, c); + let r = _mm_mask3_fmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b11111111, + ); + let e = _mm_set_ps(3., 3., 3., -1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_fmsub_round_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_fmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm_set_pd(1., -1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_fmsub_round_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_mask_fmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); + assert_eq_m128d(r, a); + let r = _mm_mask_fmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11111111, b, c, + ); + let e = _mm_set_pd(1., -1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_fmsub_round_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_maskz_fmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_fmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, c, + ); + let e = _mm_set_pd(1., -1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask3_fmsub_round_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_mask3_fmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); + assert_eq_m128d(r, c); + let r = _mm_mask3_fmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b11111111, + ); + let e = _mm_set_pd(3., -1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_fnmadd_round_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_fnmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm_set_ps(1., 1., 1., 1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_fnmadd_round_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_mask_fnmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); + assert_eq_m128(r, a); + let r = _mm_mask_fnmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11111111, b, c, + ); + let e = _mm_set_ps(1., 1., 1., 1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_fnmadd_round_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_maskz_fnmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); + let e = _mm_set_ps(1., 1., 1., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_fnmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, c, + ); + let e = _mm_set_ps(1., 1., 1., 1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask3_fnmadd_round_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_mask3_fnmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); + assert_eq_m128(r, c); + let r = _mm_mask3_fnmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b11111111, + ); + let e = _mm_set_ps(3., 3., 3., 1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_fnmadd_round_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_fnmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm_set_pd(1., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_fnmadd_round_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_mask_fnmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); + assert_eq_m128d(r, a); + let r = _mm_mask_fnmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11111111, b, c, + ); + let e = _mm_set_pd(1., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_fnmadd_round_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_maskz_fnmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_fnmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, c, + ); + let e = _mm_set_pd(1., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask3_fnmadd_round_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_mask3_fnmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); + assert_eq_m128d(r, c); + let r = _mm_mask3_fnmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b11111111, + ); + let e = _mm_set_pd(3., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_fnmsub_round_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_fnmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm_set_ps(1., 1., 1., -5.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_fnmsub_round_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_mask_fnmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); + assert_eq_m128(r, a); + let r = _mm_mask_fnmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11111111, b, c, + ); + let e = _mm_set_ps(1., 1., 1., -5.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_fnmsub_round_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_maskz_fnmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); + let e = _mm_set_ps(1., 1., 1., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_fnmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, c, + ); + let e = _mm_set_ps(1., 1., 1., -5.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask3_fnmsub_round_ss() { + let a = _mm_set1_ps(1.); + let b = _mm_set1_ps(2.); + let c = _mm_set1_ps(3.); + let r = _mm_mask3_fnmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); + assert_eq_m128(r, c); + let r = _mm_mask3_fnmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b11111111, + ); + let e = _mm_set_ps(3., 3., 3., -5.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_fnmsub_round_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_fnmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm_set_pd(1., -5.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_fnmsub_round_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_mask_fnmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); + assert_eq_m128d(r, a); + let r = _mm_mask_fnmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11111111, b, c, + ); + let e = _mm_set_pd(1., -5.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_fnmsub_round_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_maskz_fnmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_fnmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, c, + ); + let e = _mm_set_pd(1., -5.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask3_fnmsub_round_sd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let c = _mm_set1_pd(3.); + let r = _mm_mask3_fnmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); + assert_eq_m128d(r, c); + let r = _mm_mask3_fnmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b11111111, + ); + let e = _mm_set_pd(3., -5.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_fixupimm_ss() { + let a = _mm_set_ps(0., 0., 0., f32::NAN); + let b = _mm_set1_ps(f32::MAX); + let c = _mm_set1_epi32(i32::MAX); + let r = _mm_fixupimm_ss::<5>(a, b, c); + let e = _mm_set_ps(0., 0., 0., -0.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_fixupimm_ss() { + let a = _mm_set_ps(0., 0., 0., f32::NAN); + let b = _mm_set1_ps(f32::MAX); + let c = _mm_set1_epi32(i32::MAX); + let r = _mm_mask_fixupimm_ss::<5>(a, 0b11111111, b, c); + let e = _mm_set_ps(0., 0., 0., -0.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_fixupimm_ss() { + let a = _mm_set_ps(0., 0., 0., f32::NAN); + let b = _mm_set1_ps(f32::MAX); + let c = _mm_set1_epi32(i32::MAX); + let r = _mm_maskz_fixupimm_ss::<5>(0b00000000, a, b, c); + let e = _mm_set_ps(0., 0., 0., 0.0); + assert_eq_m128(r, e); + let r = _mm_maskz_fixupimm_ss::<5>(0b11111111, a, b, c); + let e = _mm_set_ps(0., 0., 0., -0.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_fixupimm_sd() { + let a = _mm_set_pd(0., f64::NAN); + let b = _mm_set1_pd(f64::MAX); + let c = _mm_set1_epi64x(i32::MAX as i64); + let r = _mm_fixupimm_sd::<5>(a, b, c); + let e = _mm_set_pd(0., -0.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_fixupimm_sd() { + let a = _mm_set_pd(0., f64::NAN); + let b = _mm_set1_pd(f64::MAX); + let c = _mm_set1_epi64x(i32::MAX as i64); + let r = _mm_mask_fixupimm_sd::<5>(a, 0b11111111, b, c); + let e = _mm_set_pd(0., -0.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_fixupimm_sd() { + let a = _mm_set_pd(0., f64::NAN); + let b = _mm_set1_pd(f64::MAX); + let c = _mm_set1_epi64x(i32::MAX as i64); + let r = _mm_maskz_fixupimm_sd::<5>(0b00000000, a, b, c); + let e = _mm_set_pd(0., 0.0); + assert_eq_m128d(r, e); + let r = _mm_maskz_fixupimm_sd::<5>(0b11111111, a, b, c); + let e = _mm_set_pd(0., -0.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_fixupimm_round_ss() { + let a = _mm_set_ps(1., 0., 0., f32::NAN); + let b = _mm_set1_ps(f32::MAX); + let c = _mm_set1_epi32(i32::MAX); + let r = _mm_fixupimm_round_ss::<5, _MM_FROUND_CUR_DIRECTION>(a, b, c); + let e = _mm_set_ps(1., 0., 0., -0.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_fixupimm_round_ss() { + let a = _mm_set_ps(0., 0., 0., f32::NAN); + let b = _mm_set1_ps(f32::MAX); + let c = _mm_set1_epi32(i32::MAX); + let r = _mm_mask_fixupimm_round_ss::<5, _MM_FROUND_CUR_DIRECTION>(a, 0b11111111, b, c); + let e = _mm_set_ps(0., 0., 0., -0.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_fixupimm_round_ss() { + let a = _mm_set_ps(0., 0., 0., f32::NAN); + let b = _mm_set1_ps(f32::MAX); + let c = _mm_set1_epi32(i32::MAX); + let r = _mm_maskz_fixupimm_round_ss::<5, _MM_FROUND_CUR_DIRECTION>(0b00000000, a, b, c); + let e = _mm_set_ps(0., 0., 0., 0.0); + assert_eq_m128(r, e); + let r = _mm_maskz_fixupimm_round_ss::<5, _MM_FROUND_CUR_DIRECTION>(0b11111111, a, b, c); + let e = _mm_set_ps(0., 0., 0., -0.0); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_fixupimm_round_sd() { + let a = _mm_set_pd(0., f64::NAN); + let b = _mm_set1_pd(f64::MAX); + let c = _mm_set1_epi64x(i32::MAX as i64); + let r = _mm_fixupimm_round_sd::<5, _MM_FROUND_CUR_DIRECTION>(a, b, c); + let e = _mm_set_pd(0., -0.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_fixupimm_round_sd() { + let a = _mm_set_pd(0., f64::NAN); + let b = _mm_set1_pd(f64::MAX); + let c = _mm_set1_epi64x(i32::MAX as i64); + let r = _mm_mask_fixupimm_round_sd::<5, _MM_FROUND_CUR_DIRECTION>(a, 0b11111111, b, c); + let e = _mm_set_pd(0., -0.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_fixupimm_round_sd() { + let a = _mm_set_pd(0., f64::NAN); + let b = _mm_set1_pd(f64::MAX); + let c = _mm_set1_epi64x(i32::MAX as i64); + let r = _mm_maskz_fixupimm_round_sd::<5, _MM_FROUND_CUR_DIRECTION>(0b00000000, a, b, c); + let e = _mm_set_pd(0., 0.0); + assert_eq_m128d(r, e); + let r = _mm_maskz_fixupimm_round_sd::<5, _MM_FROUND_CUR_DIRECTION>(0b11111111, a, b, c); + let e = _mm_set_pd(0., -0.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_cvtss_sd() { + let a = _mm_set_pd(6., -7.5); + let b = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_mask_cvtss_sd(a, 0, a, b); + assert_eq_m128d(r, a); + let r = _mm_mask_cvtss_sd(a, 0b11111111, a, b); + let e = _mm_set_pd(6., -1.5); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_cvtss_sd() { + let a = _mm_set_pd(6., -7.5); + let b = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_maskz_cvtss_sd(0, a, b); + let e = _mm_set_pd(6., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_cvtss_sd(0b11111111, a, b); + let e = _mm_set_pd(6., -1.5); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_cvtsd_ss() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let b = _mm_set_pd(6., -7.5); + let r = _mm_mask_cvtsd_ss(a, 0, a, b); + assert_eq_m128(r, a); + let r = _mm_mask_cvtsd_ss(a, 0b11111111, a, b); + let e = _mm_set_ps(0., -0.5, 1., -7.5); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_cvtsd_ss() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let b = _mm_set_pd(6., -7.5); + let r = _mm_maskz_cvtsd_ss(0, a, b); + let e = _mm_set_ps(0., -0.5, 1., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_cvtsd_ss(0b11111111, a, b); + let e = _mm_set_ps(0., -0.5, 1., -7.5); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundss_sd() { + let a = _mm_set_pd(6., -7.5); + let b = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_cvt_roundss_sd::<_MM_FROUND_CUR_DIRECTION>(a, b); + let e = _mm_set_pd(6., -1.5); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_cvt_roundss_sd() { + let a = _mm_set_pd(6., -7.5); + let b = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_mask_cvt_roundss_sd::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); + assert_eq_m128d(r, a); + let r = _mm_mask_cvt_roundss_sd::<_MM_FROUND_CUR_DIRECTION>(a, 0b11111111, a, b); + let e = _mm_set_pd(6., -1.5); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_cvt_roundss_sd() { + let a = _mm_set_pd(6., -7.5); + let b = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_maskz_cvt_roundss_sd::<_MM_FROUND_CUR_DIRECTION>(0, a, b); + let e = _mm_set_pd(6., 0.); + assert_eq_m128d(r, e); + let r = _mm_maskz_cvt_roundss_sd::<_MM_FROUND_CUR_DIRECTION>(0b11111111, a, b); + let e = _mm_set_pd(6., -1.5); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundsd_ss() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let b = _mm_set_pd(6., -7.5); + let r = _mm_cvt_roundsd_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_ps(0., -0.5, 1., -7.5); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_mask_cvt_roundsd_ss() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let b = _mm_set_pd(6., -7.5); + let r = _mm_mask_cvt_roundsd_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, 0, a, b); + assert_eq_m128(r, a); + let r = _mm_mask_cvt_roundsd_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + a, 0b11111111, a, b, + ); + let e = _mm_set_ps(0., -0.5, 1., -7.5); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_maskz_cvt_roundsd_ss() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let b = _mm_set_pd(6., -7.5); + let r = _mm_maskz_cvt_roundsd_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); + let e = _mm_set_ps(0., -0.5, 1., 0.); + assert_eq_m128(r, e); + let r = _mm_maskz_cvt_roundsd_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, + ); + let e = _mm_set_ps(0., -0.5, 1., -7.5); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundss_si32() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_cvt_roundss_si32::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); + let e: i32 = -1; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundss_i32() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_cvt_roundss_i32::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); + let e: i32 = -1; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundss_u32() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_cvt_roundss_u32::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); + let e: u32 = u32::MAX; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtss_i32() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_cvtss_i32(a); + let e: i32 = -2; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtss_u32() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_cvtss_u32(a); + let e: u32 = u32::MAX; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundsd_si32() { + let a = _mm_set_pd(1., -1.5); + let r = _mm_cvt_roundsd_si32::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); + let e: i32 = -1; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundsd_i32() { + let a = _mm_set_pd(1., -1.5); + let r = _mm_cvt_roundsd_i32::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); + let e: i32 = -1; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundsd_u32() { + let a = _mm_set_pd(1., -1.5); + let r = _mm_cvt_roundsd_u32::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); + let e: u32 = u32::MAX; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtsd_i32() { + let a = _mm_set_pd(1., -1.5); + let r = _mm_cvtsd_i32(a); + let e: i32 = -2; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtsd_u32() { + let a = _mm_set_pd(1., -1.5); + let r = _mm_cvtsd_u32(a); + let e: u32 = u32::MAX; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundi32_ss() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let b: i32 = 9; + let r = _mm_cvt_roundi32_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_ps(0., -0.5, 1., 9.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundsi32_ss() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let b: i32 = 9; + let r = _mm_cvt_roundsi32_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_ps(0., -0.5, 1., 9.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundu32_ss() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let b: u32 = 9; + let r = _mm_cvt_roundu32_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_ps(0., -0.5, 1., 9.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvti32_ss() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let b: i32 = 9; + let r = _mm_cvti32_ss(a, b); + let e = _mm_set_ps(0., -0.5, 1., 9.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvti32_sd() { + let a = _mm_set_pd(1., -1.5); + let b: i32 = 9; + let r = _mm_cvti32_sd(a, b); + let e = _mm_set_pd(1., 9.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtt_roundss_si32() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_cvtt_roundss_si32::<_MM_FROUND_CUR_DIRECTION>(a); + let e: i32 = -2; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtt_roundss_i32() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_cvtt_roundss_i32::<_MM_FROUND_CUR_DIRECTION>(a); + let e: i32 = -2; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtt_roundss_u32() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_cvtt_roundss_u32::<_MM_FROUND_CUR_DIRECTION>(a); + let e: u32 = u32::MAX; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvttss_i32() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_cvttss_i32(a); + let e: i32 = -2; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvttss_u32() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_cvttss_u32(a); + let e: u32 = u32::MAX; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtt_roundsd_si32() { + let a = _mm_set_pd(1., -1.5); + let r = _mm_cvtt_roundsd_si32::<_MM_FROUND_CUR_DIRECTION>(a); + let e: i32 = -2; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtt_roundsd_i32() { + let a = _mm_set_pd(1., -1.5); + let r = _mm_cvtt_roundsd_i32::<_MM_FROUND_CUR_DIRECTION>(a); + let e: i32 = -2; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtt_roundsd_u32() { + let a = _mm_set_pd(1., -1.5); + let r = _mm_cvtt_roundsd_u32::<_MM_FROUND_CUR_DIRECTION>(a); + let e: u32 = u32::MAX; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvttsd_i32() { + let a = _mm_set_pd(1., -1.5); + let r = _mm_cvttsd_i32(a); + let e: i32 = -2; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvttsd_u32() { + let a = _mm_set_pd(1., -1.5); + let r = _mm_cvttsd_u32(a); + let e: u32 = u32::MAX; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtu32_ss() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let b: u32 = 9; + let r = _mm_cvtu32_ss(a, b); + let e = _mm_set_ps(0., -0.5, 1., 9.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtu32_sd() { + let a = _mm_set_pd(1., -1.5); + let b: u32 = 9; + let r = _mm_cvtu32_sd(a, b); + let e = _mm_set_pd(1., 9.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_comi_round_ss() { + let a = _mm_set1_ps(2.2); + let b = _mm_set1_ps(1.1); + let r = _mm_comi_round_ss::<0, _MM_FROUND_CUR_DIRECTION>(a, b); + let e: i32 = 0; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_comi_round_sd() { + let a = _mm_set1_pd(2.2); + let b = _mm_set1_pd(1.1); + let r = _mm_comi_round_sd::<0, _MM_FROUND_CUR_DIRECTION>(a, b); + let e: i32 = 0; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtsi512_si32() { + let a = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r = _mm512_cvtsi512_si32(a); + let e: i32 = 1; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_shuffle_pd() { + let a = _mm512_setr_pd(1., 4., 5., 8., 1., 4., 5., 8.); + let b = _mm512_setr_pd(2., 3., 6., 7., 2., 3., 6., 7.); + let r = _mm512_shuffle_pd::<0b11_11_11_11>(a, b); + let e = _mm512_setr_pd(4., 3., 8., 7., 4., 3., 8., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_shuffle_pd() { + let a = _mm512_setr_pd(1., 4., 5., 8., 1., 4., 5., 8.); + let b = _mm512_setr_pd(2., 3., 6., 7., 2., 3., 6., 7.); + let r = _mm512_mask_shuffle_pd::<0b11_11_11_11>(a, 0, a, b); + assert_eq_m512d(r, a); + let r = _mm512_mask_shuffle_pd::<0b11_11_11_11>(a, 0b11111111, a, b); + let e = _mm512_setr_pd(4., 3., 8., 7., 4., 3., 8., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_shuffle_pd() { + let a = _mm512_setr_pd(1., 4., 5., 8., 1., 4., 5., 8.); + let b = _mm512_setr_pd(2., 3., 6., 7., 2., 3., 6., 7.); + let r = _mm512_maskz_shuffle_pd::<0b11_11_11_11>(0, a, b); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_shuffle_pd::<0b11_11_11_11>(0b00001111, a, b); + let e = _mm512_setr_pd(4., 3., 8., 7., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_expandloadu_epi32() { + let src = _mm512_set1_epi32(42); + let a = &[1_i32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let p = a.as_ptr(); + let m = 0b11101000_11001010; + let r = _mm512_mask_expandloadu_epi32(src, m, black_box(p)); + let e = _mm512_set_epi32(8, 7, 6, 42, 5, 42, 42, 42, 4, 3, 42, 42, 2, 42, 1, 42); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_expandloadu_epi32() { + let a = &[1_i32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let p = a.as_ptr(); + let m = 0b11101000_11001010; + let r = _mm512_maskz_expandloadu_epi32(m, black_box(p)); + let e = _mm512_set_epi32(8, 7, 6, 0, 5, 0, 0, 0, 4, 3, 0, 0, 2, 0, 1, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_expandloadu_epi32() { + let src = _mm256_set1_epi32(42); + let a = &[1_i32, 2, 3, 4, 5, 6, 7, 8]; + let p = a.as_ptr(); + let m = 0b11101000; + let r = _mm256_mask_expandloadu_epi32(src, m, black_box(p)); + let e = _mm256_set_epi32(4, 3, 2, 42, 1, 42, 42, 42); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_expandloadu_epi32() { + let a = &[1_i32, 2, 3, 4, 5, 6, 7, 8]; + let p = a.as_ptr(); + let m = 0b11101000; + let r = _mm256_maskz_expandloadu_epi32(m, black_box(p)); + let e = _mm256_set_epi32(4, 3, 2, 0, 1, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_expandloadu_epi32() { + let src = _mm_set1_epi32(42); + let a = &[1_i32, 2, 3, 4]; + let p = a.as_ptr(); + let m = 0b11111000; + let r = _mm_mask_expandloadu_epi32(src, m, black_box(p)); + let e = _mm_set_epi32(1, 42, 42, 42); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_expandloadu_epi32() { + let a = &[1_i32, 2, 3, 4]; + let p = a.as_ptr(); + let m = 0b11111000; + let r = _mm_maskz_expandloadu_epi32(m, black_box(p)); + let e = _mm_set_epi32(1, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_expandloadu_epi64() { + let src = _mm512_set1_epi64(42); + let a = &[1_i64, 2, 3, 4, 5, 6, 7, 8]; + let p = a.as_ptr(); + let m = 0b11101000; + let r = _mm512_mask_expandloadu_epi64(src, m, black_box(p)); + let e = _mm512_set_epi64(4, 3, 2, 42, 1, 42, 42, 42); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_expandloadu_epi64() { + let a = &[1_i64, 2, 3, 4, 5, 6, 7, 8]; + let p = a.as_ptr(); + let m = 0b11101000; + let r = _mm512_maskz_expandloadu_epi64(m, black_box(p)); + let e = _mm512_set_epi64(4, 3, 2, 0, 1, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_expandloadu_epi64() { + let src = _mm256_set1_epi64x(42); + let a = &[1_i64, 2, 3, 4]; + let p = a.as_ptr(); + let m = 0b11101000; + let r = _mm256_mask_expandloadu_epi64(src, m, black_box(p)); + let e = _mm256_set_epi64x(1, 42, 42, 42); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_expandloadu_epi64() { + let a = &[1_i64, 2, 3, 4]; + let p = a.as_ptr(); + let m = 0b11101000; + let r = _mm256_maskz_expandloadu_epi64(m, black_box(p)); + let e = _mm256_set_epi64x(1, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_expandloadu_epi64() { + let src = _mm_set1_epi64x(42); + let a = &[1_i64, 2]; + let p = a.as_ptr(); + let m = 0b11101000; + let r = _mm_mask_expandloadu_epi64(src, m, black_box(p)); + let e = _mm_set_epi64x(42, 42); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_expandloadu_epi64() { + let a = &[1_i64, 2]; + let p = a.as_ptr(); + let m = 0b11101000; + let r = _mm_maskz_expandloadu_epi64(m, black_box(p)); + let e = _mm_set_epi64x(0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_expandloadu_ps() { + let src = _mm512_set1_ps(42.); + let a = &[ + 1.0f32, 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ]; + let p = a.as_ptr(); + let m = 0b11101000_11001010; + let r = _mm512_mask_expandloadu_ps(src, m, black_box(p)); + let e = _mm512_set_ps( + 8., 7., 6., 42., 5., 42., 42., 42., 4., 3., 42., 42., 2., 42., 1., 42., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_expandloadu_ps() { + let a = &[ + 1.0f32, 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., + ]; + let p = a.as_ptr(); + let m = 0b11101000_11001010; + let r = _mm512_maskz_expandloadu_ps(m, black_box(p)); + let e = _mm512_set_ps( + 8., 7., 6., 0., 5., 0., 0., 0., 4., 3., 0., 0., 2., 0., 1., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_expandloadu_ps() { + let src = _mm256_set1_ps(42.); + let a = &[1.0f32, 2., 3., 4., 5., 6., 7., 8.]; + let p = a.as_ptr(); + let m = 0b11101000; + let r = _mm256_mask_expandloadu_ps(src, m, black_box(p)); + let e = _mm256_set_ps(4., 3., 2., 42., 1., 42., 42., 42.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_expandloadu_ps() { + let a = &[1.0f32, 2., 3., 4., 5., 6., 7., 8.]; + let p = a.as_ptr(); + let m = 0b11101000; + let r = _mm256_maskz_expandloadu_ps(m, black_box(p)); + let e = _mm256_set_ps(4., 3., 2., 0., 1., 0., 0., 0.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_expandloadu_ps() { + let src = _mm_set1_ps(42.); + let a = &[1.0f32, 2., 3., 4.]; + let p = a.as_ptr(); + let m = 0b11101000; + let r = _mm_mask_expandloadu_ps(src, m, black_box(p)); + let e = _mm_set_ps(1., 42., 42., 42.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_expandloadu_ps() { + let a = &[1.0f32, 2., 3., 4.]; + let p = a.as_ptr(); + let m = 0b11101000; + let r = _mm_maskz_expandloadu_ps(m, black_box(p)); + let e = _mm_set_ps(1., 0., 0., 0.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_expandloadu_pd() { + let src = _mm512_set1_pd(42.); + let a = &[1.0f64, 2., 3., 4., 5., 6., 7., 8.]; + let p = a.as_ptr(); + let m = 0b11101000; + let r = _mm512_mask_expandloadu_pd(src, m, black_box(p)); + let e = _mm512_set_pd(4., 3., 2., 42., 1., 42., 42., 42.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_expandloadu_pd() { + let a = &[1.0f64, 2., 3., 4., 5., 6., 7., 8.]; + let p = a.as_ptr(); + let m = 0b11101000; + let r = _mm512_maskz_expandloadu_pd(m, black_box(p)); + let e = _mm512_set_pd(4., 3., 2., 0., 1., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_expandloadu_pd() { + let src = _mm256_set1_pd(42.); + let a = &[1.0f64, 2., 3., 4.]; + let p = a.as_ptr(); + let m = 0b11101000; + let r = _mm256_mask_expandloadu_pd(src, m, black_box(p)); + let e = _mm256_set_pd(1., 42., 42., 42.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_expandloadu_pd() { + let a = &[1.0f64, 2., 3., 4.]; + let p = a.as_ptr(); + let m = 0b11101000; + let r = _mm256_maskz_expandloadu_pd(m, black_box(p)); + let e = _mm256_set_pd(1., 0., 0., 0.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_expandloadu_pd() { + let src = _mm_set1_pd(42.); + let a = &[1.0f64, 2.]; + let p = a.as_ptr(); + let m = 0b11101000; + let r = _mm_mask_expandloadu_pd(src, m, black_box(p)); + let e = _mm_set_pd(42., 42.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_expandloadu_pd() { + let a = &[1.0f64, 2.]; + let p = a.as_ptr(); + let m = 0b11101000; + let r = _mm_maskz_expandloadu_pd(m, black_box(p)); + let e = _mm_set_pd(0., 0.); + assert_eq_m128d(r, e); } } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512gfni.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512gfni.rs new file mode 100644 index 000000000..66fd1c2e1 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512gfni.rs @@ -0,0 +1,1492 @@ +//! Galois Field New Instructions (GFNI) +//! +//! The intrinsics here correspond to those in the `immintrin.h` C header. +//! +//! The reference is [Intel 64 and IA-32 Architectures Software Developer's +//! Manual Volume 2: Instruction Set Reference, A-Z][intel64_ref]. +//! +//! [intel64_ref]: http://www.intel.de/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf + +use crate::core_arch::simd::i8x16; +use crate::core_arch::simd::i8x32; +use crate::core_arch::simd::i8x64; +use crate::core_arch::simd_llvm::simd_select_bitmask; +use crate::core_arch::x86::__m128i; +use crate::core_arch::x86::__m256i; +use crate::core_arch::x86::__m512i; +use crate::core_arch::x86::__mmask16; +use crate::core_arch::x86::__mmask32; +use crate::core_arch::x86::__mmask64; +use crate::core_arch::x86::_mm256_setzero_si256; +use crate::core_arch::x86::_mm512_setzero_si512; +use crate::core_arch::x86::_mm_setzero_si128; +use crate::core_arch::x86::m128iExt; +use crate::core_arch::x86::m256iExt; +use crate::core_arch::x86::m512iExt; +use crate::mem::transmute; + +#[cfg(test)] +use stdarch_test::assert_instr; + +#[allow(improper_ctypes)] +extern "C" { + #[link_name = "llvm.x86.vgf2p8affineinvqb.512"] + fn vgf2p8affineinvqb_512(x: i8x64, a: i8x64, imm8: u8) -> i8x64; + #[link_name = "llvm.x86.vgf2p8affineinvqb.256"] + fn vgf2p8affineinvqb_256(x: i8x32, a: i8x32, imm8: u8) -> i8x32; + #[link_name = "llvm.x86.vgf2p8affineinvqb.128"] + fn vgf2p8affineinvqb_128(x: i8x16, a: i8x16, imm8: u8) -> i8x16; + #[link_name = "llvm.x86.vgf2p8affineqb.512"] + fn vgf2p8affineqb_512(x: i8x64, a: i8x64, imm8: u8) -> i8x64; + #[link_name = "llvm.x86.vgf2p8affineqb.256"] + fn vgf2p8affineqb_256(x: i8x32, a: i8x32, imm8: u8) -> i8x32; + #[link_name = "llvm.x86.vgf2p8affineqb.128"] + fn vgf2p8affineqb_128(x: i8x16, a: i8x16, imm8: u8) -> i8x16; + #[link_name = "llvm.x86.vgf2p8mulb.512"] + fn vgf2p8mulb_512(a: i8x64, b: i8x64) -> i8x64; + #[link_name = "llvm.x86.vgf2p8mulb.256"] + fn vgf2p8mulb_256(a: i8x32, b: i8x32) -> i8x32; + #[link_name = "llvm.x86.vgf2p8mulb.128"] + fn vgf2p8mulb_128(a: i8x16, b: i8x16) -> i8x16; +} + +// LLVM requires AVX512BW for a lot of these instructions, see +// https://github.com/llvm/llvm-project/blob/release/9.x/clang/include/clang/Basic/BuiltinsX86.def#L457 +// however our tests also require the target feature list to match Intel's +// which *doesn't* require AVX512BW but only AVX512F, so we added the redundant AVX512F +// requirement (for now) +// also see +// https://github.com/llvm/llvm-project/blob/release/9.x/clang/lib/Headers/gfniintrin.h +// for forcing GFNI, BW and optionally VL extension + +/// Performs a multiplication in GF(2^8) on the packed bytes. +/// The field is in polynomial representation with the reduction polynomial +/// x^8 + x^4 + x^3 + x + 1. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_gf2p8mul_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512f")] +#[cfg_attr(test, assert_instr(vgf2p8mulb))] +pub unsafe fn _mm512_gf2p8mul_epi8(a: __m512i, b: __m512i) -> __m512i { + transmute(vgf2p8mulb_512(a.as_i8x64(), b.as_i8x64())) +} + +/// Performs a multiplication in GF(2^8) on the packed bytes. +/// The field is in polynomial representation with the reduction polynomial +/// x^8 + x^4 + x^3 + x + 1. +/// +/// Uses the writemask in k - elements are copied from src if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_gf2p8mul_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512f")] +#[cfg_attr(test, assert_instr(vgf2p8mulb))] +pub unsafe fn _mm512_mask_gf2p8mul_epi8( + src: __m512i, + k: __mmask64, + a: __m512i, + b: __m512i, +) -> __m512i { + transmute(simd_select_bitmask( + k, + vgf2p8mulb_512(a.as_i8x64(), b.as_i8x64()), + src.as_i8x64(), + )) +} + +/// Performs a multiplication in GF(2^8) on the packed bytes. +/// The field is in polynomial representation with the reduction polynomial +/// x^8 + x^4 + x^3 + x + 1. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_gf2p8mul_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512f")] +#[cfg_attr(test, assert_instr(vgf2p8mulb))] +pub unsafe fn _mm512_maskz_gf2p8mul_epi8(k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + let zero = _mm512_setzero_si512().as_i8x64(); + transmute(simd_select_bitmask( + k, + vgf2p8mulb_512(a.as_i8x64(), b.as_i8x64()), + zero, + )) +} + +/// Performs a multiplication in GF(2^8) on the packed bytes. +/// The field is in polynomial representation with the reduction polynomial +/// x^8 + x^4 + x^3 + x + 1. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_gf2p8mul_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vgf2p8mulb))] +pub unsafe fn _mm256_gf2p8mul_epi8(a: __m256i, b: __m256i) -> __m256i { + transmute(vgf2p8mulb_256(a.as_i8x32(), b.as_i8x32())) +} + +/// Performs a multiplication in GF(2^8) on the packed bytes. +/// The field is in polynomial representation with the reduction polynomial +/// x^8 + x^4 + x^3 + x + 1. +/// +/// Uses the writemask in k - elements are copied from src if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_gf2p8mul_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vgf2p8mulb))] +pub unsafe fn _mm256_mask_gf2p8mul_epi8( + src: __m256i, + k: __mmask32, + a: __m256i, + b: __m256i, +) -> __m256i { + transmute(simd_select_bitmask( + k, + vgf2p8mulb_256(a.as_i8x32(), b.as_i8x32()), + src.as_i8x32(), + )) +} + +/// Performs a multiplication in GF(2^8) on the packed bytes. +/// The field is in polynomial representation with the reduction polynomial +/// x^8 + x^4 + x^3 + x + 1. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_gf2p8mul_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vgf2p8mulb))] +pub unsafe fn _mm256_maskz_gf2p8mul_epi8(k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + let zero = _mm256_setzero_si256().as_i8x32(); + transmute(simd_select_bitmask( + k, + vgf2p8mulb_256(a.as_i8x32(), b.as_i8x32()), + zero, + )) +} + +/// Performs a multiplication in GF(2^8) on the packed bytes. +/// The field is in polynomial representation with the reduction polynomial +/// x^8 + x^4 + x^3 + x + 1. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_gf2p8mul_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vgf2p8mulb))] +pub unsafe fn _mm_gf2p8mul_epi8(a: __m128i, b: __m128i) -> __m128i { + transmute(vgf2p8mulb_128(a.as_i8x16(), b.as_i8x16())) +} + +/// Performs a multiplication in GF(2^8) on the packed bytes. +/// The field is in polynomial representation with the reduction polynomial +/// x^8 + x^4 + x^3 + x + 1. +/// +/// Uses the writemask in k - elements are copied from src if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_gf2p8mul_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vgf2p8mulb))] +pub unsafe fn _mm_mask_gf2p8mul_epi8( + src: __m128i, + k: __mmask16, + a: __m128i, + b: __m128i, +) -> __m128i { + transmute(simd_select_bitmask( + k, + vgf2p8mulb_128(a.as_i8x16(), b.as_i8x16()), + src.as_i8x16(), + )) +} + +/// Performs a multiplication in GF(2^8) on the packed bytes. +/// The field is in polynomial representation with the reduction polynomial +/// x^8 + x^4 + x^3 + x + 1. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_gf2p8mul_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vgf2p8mulb))] +pub unsafe fn _mm_maskz_gf2p8mul_epi8(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask( + k, + vgf2p8mulb_128(a.as_i8x16(), b.as_i8x16()), + zero, + )) +} + +/// Performs an affine transformation on the packed bytes in x. +/// That is computes a*x+b over the Galois Field 2^8 for each packed byte with a being a 8x8 bit matrix +/// and b being a constant 8-bit immediate value. +/// Each pack of 8 bytes in x is paired with the 64-bit word at the same position in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_gf2p8affine_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512f")] +#[cfg_attr(test, assert_instr(vgf2p8affineqb, B = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_gf2p8affine_epi64_epi8(x: __m512i, a: __m512i) -> __m512i { + static_assert_imm8!(B); + let b = B as u8; + let x = x.as_i8x64(); + let a = a.as_i8x64(); + let r = vgf2p8affineqb_512(x, a, b); + transmute(r) +} + +/// Performs an affine transformation on the packed bytes in x. +/// That is computes a*x+b over the Galois Field 2^8 for each packed byte with a being a 8x8 bit matrix +/// and b being a constant 8-bit immediate value. +/// Each pack of 8 bytes in x is paired with the 64-bit word at the same position in a. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_gf2p8affine_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512f")] +#[cfg_attr(test, assert_instr(vgf2p8affineqb, B = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_gf2p8affine_epi64_epi8( + k: __mmask64, + x: __m512i, + a: __m512i, +) -> __m512i { + static_assert_imm8!(B); + let b = B as u8; + let zero = _mm512_setzero_si512().as_i8x64(); + let x = x.as_i8x64(); + let a = a.as_i8x64(); + let r = vgf2p8affineqb_512(x, a, b); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Performs an affine transformation on the packed bytes in x. +/// That is computes a*x+b over the Galois Field 2^8 for each packed byte with a being a 8x8 bit matrix +/// and b being a constant 8-bit immediate value. +/// Each pack of 8 bytes in x is paired with the 64-bit word at the same position in a. +/// +/// Uses the writemask in k - elements are copied from src if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_gf2p8affine_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512f")] +#[cfg_attr(test, assert_instr(vgf2p8affineqb, B = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_gf2p8affine_epi64_epi8( + src: __m512i, + k: __mmask64, + x: __m512i, + a: __m512i, +) -> __m512i { + static_assert_imm8!(B); + let b = B as u8; + let x = x.as_i8x64(); + let a = a.as_i8x64(); + let r = vgf2p8affineqb_512(x, a, b); + transmute(simd_select_bitmask(k, r, src.as_i8x64())) +} + +/// Performs an affine transformation on the packed bytes in x. +/// That is computes a*x+b over the Galois Field 2^8 for each packed byte with a being a 8x8 bit matrix +/// and b being a constant 8-bit immediate value. +/// Each pack of 8 bytes in x is paired with the 64-bit word at the same position in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_gf2p8affine_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vgf2p8affineqb, B = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_gf2p8affine_epi64_epi8(x: __m256i, a: __m256i) -> __m256i { + static_assert_imm8!(B); + let b = B as u8; + let x = x.as_i8x32(); + let a = a.as_i8x32(); + let r = vgf2p8affineqb_256(x, a, b); + transmute(r) +} + +/// Performs an affine transformation on the packed bytes in x. +/// That is computes a*x+b over the Galois Field 2^8 for each packed byte with a being a 8x8 bit matrix +/// and b being a constant 8-bit immediate value. +/// Each pack of 8 bytes in x is paired with the 64-bit word at the same position in a. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_gf2p8affine_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vgf2p8affineqb, B = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_gf2p8affine_epi64_epi8( + k: __mmask32, + x: __m256i, + a: __m256i, +) -> __m256i { + static_assert_imm8!(B); + let b = B as u8; + let zero = _mm256_setzero_si256().as_i8x32(); + let x = x.as_i8x32(); + let a = a.as_i8x32(); + let r = vgf2p8affineqb_256(x, a, b); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Performs an affine transformation on the packed bytes in x. +/// That is computes a*x+b over the Galois Field 2^8 for each packed byte with a being a 8x8 bit matrix +/// and b being a constant 8-bit immediate value. +/// Each pack of 8 bytes in x is paired with the 64-bit word at the same position in a. +/// +/// Uses the writemask in k - elements are copied from src if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_gf2p8affine_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vgf2p8affineqb, B = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_gf2p8affine_epi64_epi8( + src: __m256i, + k: __mmask32, + x: __m256i, + a: __m256i, +) -> __m256i { + static_assert_imm8!(B); + let b = B as u8; + let x = x.as_i8x32(); + let a = a.as_i8x32(); + let r = vgf2p8affineqb_256(x, a, b); + transmute(simd_select_bitmask(k, r, src.as_i8x32())) +} + +/// Performs an affine transformation on the packed bytes in x. +/// That is computes a*x+b over the Galois Field 2^8 for each packed byte with a being a 8x8 bit matrix +/// and b being a constant 8-bit immediate value. +/// Each pack of 8 bytes in x is paired with the 64-bit word at the same position in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_gf2p8affine_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vgf2p8affineqb, B = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_gf2p8affine_epi64_epi8(x: __m128i, a: __m128i) -> __m128i { + static_assert_imm8!(B); + let b = B as u8; + let x = x.as_i8x16(); + let a = a.as_i8x16(); + let r = vgf2p8affineqb_128(x, a, b); + transmute(r) +} + +/// Performs an affine transformation on the packed bytes in x. +/// That is computes a*x+b over the Galois Field 2^8 for each packed byte with a being a 8x8 bit matrix +/// and b being a constant 8-bit immediate value. +/// Each pack of 8 bytes in x is paired with the 64-bit word at the same position in a. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_gf2p8affine_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vgf2p8affineqb, B = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_gf2p8affine_epi64_epi8( + k: __mmask16, + x: __m128i, + a: __m128i, +) -> __m128i { + static_assert_imm8!(B); + let b = B as u8; + let zero = _mm_setzero_si128().as_i8x16(); + let x = x.as_i8x16(); + let a = a.as_i8x16(); + let r = vgf2p8affineqb_128(x, a, b); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Performs an affine transformation on the packed bytes in x. +/// That is computes a*x+b over the Galois Field 2^8 for each packed byte with a being a 8x8 bit matrix +/// and b being a constant 8-bit immediate value. +/// Each pack of 8 bytes in x is paired with the 64-bit word at the same position in a. +/// +/// Uses the writemask in k - elements are copied from src if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_gf2p8affine_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vgf2p8affineqb, B = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_gf2p8affine_epi64_epi8( + src: __m128i, + k: __mmask16, + x: __m128i, + a: __m128i, +) -> __m128i { + static_assert_imm8!(B); + let b = B as u8; + let x = x.as_i8x16(); + let a = a.as_i8x16(); + let r = vgf2p8affineqb_128(x, a, b); + transmute(simd_select_bitmask(k, r, src.as_i8x16())) +} + +/// Performs an affine transformation on the inverted packed bytes in x. +/// That is computes a*inv(x)+b over the Galois Field 2^8 for each packed byte with a being a 8x8 bit matrix +/// and b being a constant 8-bit immediate value. +/// The inverse of a byte is defined with respect to the reduction polynomial x^8+x^4+x^3+x+1. +/// The inverse of 0 is 0. +/// Each pack of 8 bytes in x is paired with the 64-bit word at the same position in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_gf2p8affineinv_epi64_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512f")] +#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, B = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_gf2p8affineinv_epi64_epi8(x: __m512i, a: __m512i) -> __m512i { + static_assert_imm8!(B); + let b = B as u8; + let x = x.as_i8x64(); + let a = a.as_i8x64(); + let r = vgf2p8affineinvqb_512(x, a, b); + transmute(r) +} + +/// Performs an affine transformation on the inverted packed bytes in x. +/// That is computes a*inv(x)+b over the Galois Field 2^8 for each packed byte with a being a 8x8 bit matrix +/// and b being a constant 8-bit immediate value. +/// The inverse of a byte is defined with respect to the reduction polynomial x^8+x^4+x^3+x+1. +/// The inverse of 0 is 0. +/// Each pack of 8 bytes in x is paired with the 64-bit word at the same position in a. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_gf2p8affineinv_epi64_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512f")] +#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, B = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_gf2p8affineinv_epi64_epi8( + k: __mmask64, + x: __m512i, + a: __m512i, +) -> __m512i { + static_assert_imm8!(B); + let b = B as u8; + let zero = _mm512_setzero_si512().as_i8x64(); + let x = x.as_i8x64(); + let a = a.as_i8x64(); + let r = vgf2p8affineinvqb_512(x, a, b); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Performs an affine transformation on the inverted packed bytes in x. +/// That is computes a*inv(x)+b over the Galois Field 2^8 for each packed byte with a being a 8x8 bit matrix +/// and b being a constant 8-bit immediate value. +/// The inverse of a byte is defined with respect to the reduction polynomial x^8+x^4+x^3+x+1. +/// The inverse of 0 is 0. +/// Each pack of 8 bytes in x is paired with the 64-bit word at the same position in a. +/// +/// Uses the writemask in k - elements are copied from src if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_gf2p8affineinv_epi64_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512f")] +#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, B = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_gf2p8affineinv_epi64_epi8( + src: __m512i, + k: __mmask64, + x: __m512i, + a: __m512i, +) -> __m512i { + static_assert_imm8!(B); + let b = B as u8; + let x = x.as_i8x64(); + let a = a.as_i8x64(); + let r = vgf2p8affineinvqb_512(x, a, b); + transmute(simd_select_bitmask(k, r, src.as_i8x64())) +} + +/// Performs an affine transformation on the inverted packed bytes in x. +/// That is computes a*inv(x)+b over the Galois Field 2^8 for each packed byte with a being a 8x8 bit matrix +/// and b being a constant 8-bit immediate value. +/// The inverse of a byte is defined with respect to the reduction polynomial x^8+x^4+x^3+x+1. +/// The inverse of 0 is 0. +/// Each pack of 8 bytes in x is paired with the 64-bit word at the same position in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_gf2p8affineinv_epi64_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, B = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_gf2p8affineinv_epi64_epi8(x: __m256i, a: __m256i) -> __m256i { + static_assert_imm8!(B); + let b = B as u8; + let x = x.as_i8x32(); + let a = a.as_i8x32(); + let r = vgf2p8affineinvqb_256(x, a, b); + transmute(r) +} + +/// Performs an affine transformation on the inverted packed bytes in x. +/// That is computes a*inv(x)+b over the Galois Field 2^8 for each packed byte with a being a 8x8 bit matrix +/// and b being a constant 8-bit immediate value. +/// The inverse of a byte is defined with respect to the reduction polynomial x^8+x^4+x^3+x+1. +/// The inverse of 0 is 0. +/// Each pack of 8 bytes in x is paired with the 64-bit word at the same position in a. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_gf2p8affineinv_epi64_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, B = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_gf2p8affineinv_epi64_epi8( + k: __mmask32, + x: __m256i, + a: __m256i, +) -> __m256i { + static_assert_imm8!(B); + let b = B as u8; + let zero = _mm256_setzero_si256().as_i8x32(); + let x = x.as_i8x32(); + let a = a.as_i8x32(); + let r = vgf2p8affineinvqb_256(x, a, b); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Performs an affine transformation on the inverted packed bytes in x. +/// That is computes a*inv(x)+b over the Galois Field 2^8 for each packed byte with a being a 8x8 bit matrix +/// and b being a constant 8-bit immediate value. +/// The inverse of a byte is defined with respect to the reduction polynomial x^8+x^4+x^3+x+1. +/// The inverse of 0 is 0. +/// Each pack of 8 bytes in x is paired with the 64-bit word at the same position in a. +/// +/// Uses the writemask in k - elements are copied from src if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_gf2p8affineinv_epi64_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, B = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_gf2p8affineinv_epi64_epi8( + src: __m256i, + k: __mmask32, + x: __m256i, + a: __m256i, +) -> __m256i { + static_assert_imm8!(B); + let b = B as u8; + let x = x.as_i8x32(); + let a = a.as_i8x32(); + let r = vgf2p8affineinvqb_256(x, a, b); + transmute(simd_select_bitmask(k, r, src.as_i8x32())) +} + +/// Performs an affine transformation on the inverted packed bytes in x. +/// That is computes a*inv(x)+b over the Galois Field 2^8 for each packed byte with a being a 8x8 bit matrix +/// and b being a constant 8-bit immediate value. +/// The inverse of a byte is defined with respect to the reduction polynomial x^8+x^4+x^3+x+1. +/// The inverse of 0 is 0. +/// Each pack of 8 bytes in x is paired with the 64-bit word at the same position in a. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_gf2p8affineinv_epi64_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, B = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_gf2p8affineinv_epi64_epi8(x: __m128i, a: __m128i) -> __m128i { + static_assert_imm8!(B); + let b = B as u8; + let x = x.as_i8x16(); + let a = a.as_i8x16(); + let r = vgf2p8affineinvqb_128(x, a, b); + transmute(r) +} + +/// Performs an affine transformation on the inverted packed bytes in x. +/// That is computes a*inv(x)+b over the Galois Field 2^8 for each packed byte with a being a 8x8 bit matrix +/// and b being a constant 8-bit immediate value. +/// The inverse of a byte is defined with respect to the reduction polynomial x^8+x^4+x^3+x+1. +/// The inverse of 0 is 0. +/// Each pack of 8 bytes in x is paired with the 64-bit word at the same position in a. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_gf2p8affineinv_epi64_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, B = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_gf2p8affineinv_epi64_epi8( + k: __mmask16, + x: __m128i, + a: __m128i, +) -> __m128i { + static_assert_imm8!(B); + let b = B as u8; + let zero = _mm_setzero_si128().as_i8x16(); + let x = x.as_i8x16(); + let a = a.as_i8x16(); + let r = vgf2p8affineinvqb_128(x, a, b); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Performs an affine transformation on the inverted packed bytes in x. +/// That is computes a*inv(x)+b over the Galois Field 2^8 for each packed byte with a being a 8x8 bit matrix +/// and b being a constant 8-bit immediate value. +/// The inverse of a byte is defined with respect to the reduction polynomial x^8+x^4+x^3+x+1. +/// The inverse of 0 is 0. +/// Each pack of 8 bytes in x is paired with the 64-bit word at the same position in a. +/// +/// Uses the writemask in k - elements are copied from src if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_gf2p8affineinv_epi64_epi8) +#[inline] +#[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] +#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, B = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_gf2p8affineinv_epi64_epi8( + src: __m128i, + k: __mmask16, + x: __m128i, + a: __m128i, +) -> __m128i { + static_assert_imm8!(B); + let b = B as u8; + let x = x.as_i8x16(); + let a = a.as_i8x16(); + let r = vgf2p8affineinvqb_128(x, a, b); + transmute(simd_select_bitmask(k, r, src.as_i8x16())) +} + +#[cfg(test)] +mod tests { + // The constants in the tests below are just bit patterns. They should not + // be interpreted as integers; signedness does not make sense for them, but + // __mXXXi happens to be defined in terms of signed integers. + #![allow(overflowing_literals)] + + use core::hint::black_box; + use core::intrinsics::size_of; + use stdarch_test::simd_test; + + use crate::core_arch::x86::*; + + fn mulbyte(left: u8, right: u8) -> u8 { + // this implementation follows the description in + // https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_gf2p8mul_epi8 + const REDUCTION_POLYNOMIAL: u16 = 0x11b; + let left: u16 = left.into(); + let right: u16 = right.into(); + let mut carryless_product: u16 = 0; + + // Carryless multiplication + for i in 0..8 { + if ((left >> i) & 0x01) != 0 { + carryless_product ^= right << i; + } + } + + // reduction, adding in "0" where appropriate to clear out high bits + // note that REDUCTION_POLYNOMIAL is zero in this context + for i in (8..=14).rev() { + if ((carryless_product >> i) & 0x01) != 0 { + carryless_product ^= REDUCTION_POLYNOMIAL << (i - 8); + } + } + + carryless_product as u8 + } + + const NUM_TEST_WORDS_512: usize = 4; + const NUM_TEST_WORDS_256: usize = NUM_TEST_WORDS_512 * 2; + const NUM_TEST_WORDS_128: usize = NUM_TEST_WORDS_256 * 2; + const NUM_TEST_ENTRIES: usize = NUM_TEST_WORDS_512 * 64; + const NUM_TEST_WORDS_64: usize = NUM_TEST_WORDS_128 * 2; + const NUM_BYTES: usize = 256; + const NUM_BYTES_WORDS_128: usize = NUM_BYTES / 16; + const NUM_BYTES_WORDS_256: usize = NUM_BYTES_WORDS_128 / 2; + const NUM_BYTES_WORDS_512: usize = NUM_BYTES_WORDS_256 / 2; + + fn parity(input: u8) -> u8 { + let mut accumulator = 0; + for i in 0..8 { + accumulator ^= (input >> i) & 0x01; + } + accumulator + } + + fn mat_vec_multiply_affine(matrix: u64, x: u8, b: u8) -> u8 { + // this implementation follows the description in + // https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_gf2p8affine_epi64_epi8 + let mut accumulator = 0; + + for bit in 0..8 { + accumulator |= parity(x & matrix.to_le_bytes()[bit]) << (7 - bit); + } + + accumulator ^ b + } + + fn generate_affine_mul_test_data( + immediate: u8, + ) -> ( + [u64; NUM_TEST_WORDS_64], + [u8; NUM_TEST_ENTRIES], + [u8; NUM_TEST_ENTRIES], + ) { + let mut left: [u64; NUM_TEST_WORDS_64] = [0; NUM_TEST_WORDS_64]; + let mut right: [u8; NUM_TEST_ENTRIES] = [0; NUM_TEST_ENTRIES]; + let mut result: [u8; NUM_TEST_ENTRIES] = [0; NUM_TEST_ENTRIES]; + + for i in 0..NUM_TEST_WORDS_64 { + left[i] = (i as u64) * 103 * 101; + for j in 0..8 { + let j64 = j as u64; + right[i * 8 + j] = ((left[i] + j64) % 256) as u8; + result[i * 8 + j] = mat_vec_multiply_affine(left[i], right[i * 8 + j], immediate); + } + } + + (left, right, result) + } + + fn generate_inv_tests_data() -> ([u8; NUM_BYTES], [u8; NUM_BYTES]) { + let mut input: [u8; NUM_BYTES] = [0; NUM_BYTES]; + let mut result: [u8; NUM_BYTES] = [0; NUM_BYTES]; + + for i in 0..NUM_BYTES { + input[i] = (i % 256) as u8; + result[i] = if i == 0 { 0 } else { 1 }; + } + + (input, result) + } + + const AES_S_BOX: [u8; NUM_BYTES] = [ + 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, + 0x76, 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, + 0x72, 0xc0, 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, + 0xd8, 0x31, 0x15, 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, + 0xeb, 0x27, 0xb2, 0x75, 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, + 0xb3, 0x29, 0xe3, 0x2f, 0x84, 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, + 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, + 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, + 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, + 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, + 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, 0xe0, 0x32, 0x3a, 0x0a, 0x49, + 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 0xe7, 0xc8, 0x37, 0x6d, + 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, 0xba, 0x78, 0x25, + 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, 0x70, 0x3e, + 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 0xe1, + 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, + 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, + 0x16, + ]; + + fn generate_byte_mul_test_data() -> ( + [u8; NUM_TEST_ENTRIES], + [u8; NUM_TEST_ENTRIES], + [u8; NUM_TEST_ENTRIES], + ) { + let mut left: [u8; NUM_TEST_ENTRIES] = [0; NUM_TEST_ENTRIES]; + let mut right: [u8; NUM_TEST_ENTRIES] = [0; NUM_TEST_ENTRIES]; + let mut result: [u8; NUM_TEST_ENTRIES] = [0; NUM_TEST_ENTRIES]; + + for i in 0..NUM_TEST_ENTRIES { + left[i] = (i % 256) as u8; + right[i] = left[i].wrapping_mul(101); + result[i] = mulbyte(left[i], right[i]); + } + + (left, right, result) + } + + #[target_feature(enable = "sse2")] + unsafe fn load_m128i_word(data: &[T], word_index: usize) -> __m128i { + let byte_offset = word_index * 16 / size_of::(); + let pointer = data.as_ptr().add(byte_offset) as *const __m128i; + _mm_loadu_si128(black_box(pointer)) + } + + #[target_feature(enable = "avx")] + unsafe fn load_m256i_word(data: &[T], word_index: usize) -> __m256i { + let byte_offset = word_index * 32 / size_of::(); + let pointer = data.as_ptr().add(byte_offset) as *const __m256i; + _mm256_loadu_si256(black_box(pointer)) + } + + #[target_feature(enable = "avx512f")] + unsafe fn load_m512i_word(data: &[T], word_index: usize) -> __m512i { + let byte_offset = word_index * 64 / size_of::(); + let pointer = data.as_ptr().add(byte_offset) as *const i32; + _mm512_loadu_si512(black_box(pointer)) + } + + #[simd_test(enable = "avx512gfni,avx512bw")] + unsafe fn test_mm512_gf2p8mul_epi8() { + let (left, right, expected) = generate_byte_mul_test_data(); + + for i in 0..NUM_TEST_WORDS_512 { + let left = load_m512i_word(&left, i); + let right = load_m512i_word(&right, i); + let expected = load_m512i_word(&expected, i); + let result = _mm512_gf2p8mul_epi8(left, right); + assert_eq_m512i(result, expected); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw")] + unsafe fn test_mm512_maskz_gf2p8mul_epi8() { + let (left, right, _expected) = generate_byte_mul_test_data(); + + for i in 0..NUM_TEST_WORDS_512 { + let left = load_m512i_word(&left, i); + let right = load_m512i_word(&right, i); + let result_zero = _mm512_maskz_gf2p8mul_epi8(0, left, right); + assert_eq_m512i(result_zero, _mm512_setzero_si512()); + let mask_bytes: __mmask64 = 0x0F_0F_0F_0F_FF_FF_00_00; + let mask_words: __mmask16 = 0b01_01_01_01_11_11_00_00; + let expected_result = _mm512_gf2p8mul_epi8(left, right); + let result_masked = _mm512_maskz_gf2p8mul_epi8(mask_bytes, left, right); + let expected_masked = + _mm512_mask_blend_epi32(mask_words, _mm512_setzero_si512(), expected_result); + assert_eq_m512i(result_masked, expected_masked); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw")] + unsafe fn test_mm512_mask_gf2p8mul_epi8() { + let (left, right, _expected) = generate_byte_mul_test_data(); + + for i in 0..NUM_TEST_WORDS_512 { + let left = load_m512i_word(&left, i); + let right = load_m512i_word(&right, i); + let result_left = _mm512_mask_gf2p8mul_epi8(left, 0, left, right); + assert_eq_m512i(result_left, left); + let mask_bytes: __mmask64 = 0x0F_0F_0F_0F_FF_FF_00_00; + let mask_words: __mmask16 = 0b01_01_01_01_11_11_00_00; + let expected_result = _mm512_gf2p8mul_epi8(left, right); + let result_masked = _mm512_mask_gf2p8mul_epi8(left, mask_bytes, left, right); + let expected_masked = _mm512_mask_blend_epi32(mask_words, left, expected_result); + assert_eq_m512i(result_masked, expected_masked); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw,avx512vl")] + unsafe fn test_mm256_gf2p8mul_epi8() { + let (left, right, expected) = generate_byte_mul_test_data(); + + for i in 0..NUM_TEST_WORDS_256 { + let left = load_m256i_word(&left, i); + let right = load_m256i_word(&right, i); + let expected = load_m256i_word(&expected, i); + let result = _mm256_gf2p8mul_epi8(left, right); + assert_eq_m256i(result, expected); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_gf2p8mul_epi8() { + let (left, right, _expected) = generate_byte_mul_test_data(); + + for i in 0..NUM_TEST_WORDS_256 { + let left = load_m256i_word(&left, i); + let right = load_m256i_word(&right, i); + let result_zero = _mm256_maskz_gf2p8mul_epi8(0, left, right); + assert_eq_m256i(result_zero, _mm256_setzero_si256()); + let mask_bytes: __mmask32 = 0x0F_F0_FF_00; + const MASK_WORDS: i32 = 0b01_10_11_00; + let expected_result = _mm256_gf2p8mul_epi8(left, right); + let result_masked = _mm256_maskz_gf2p8mul_epi8(mask_bytes, left, right); + let expected_masked = + _mm256_blend_epi32::(_mm256_setzero_si256(), expected_result); + assert_eq_m256i(result_masked, expected_masked); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw,avx512vl")] + unsafe fn test_mm256_mask_gf2p8mul_epi8() { + let (left, right, _expected) = generate_byte_mul_test_data(); + + for i in 0..NUM_TEST_WORDS_256 { + let left = load_m256i_word(&left, i); + let right = load_m256i_word(&right, i); + let result_left = _mm256_mask_gf2p8mul_epi8(left, 0, left, right); + assert_eq_m256i(result_left, left); + let mask_bytes: __mmask32 = 0x0F_F0_FF_00; + const MASK_WORDS: i32 = 0b01_10_11_00; + let expected_result = _mm256_gf2p8mul_epi8(left, right); + let result_masked = _mm256_mask_gf2p8mul_epi8(left, mask_bytes, left, right); + let expected_masked = _mm256_blend_epi32::(left, expected_result); + assert_eq_m256i(result_masked, expected_masked); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw,avx512vl")] + unsafe fn test_mm_gf2p8mul_epi8() { + let (left, right, expected) = generate_byte_mul_test_data(); + + for i in 0..NUM_TEST_WORDS_128 { + let left = load_m128i_word(&left, i); + let right = load_m128i_word(&right, i); + let expected = load_m128i_word(&expected, i); + let result = _mm_gf2p8mul_epi8(left, right); + assert_eq_m128i(result, expected); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw,avx512vl")] + unsafe fn test_mm_maskz_gf2p8mul_epi8() { + let (left, right, _expected) = generate_byte_mul_test_data(); + + for i in 0..NUM_TEST_WORDS_128 { + let left = load_m128i_word(&left, i); + let right = load_m128i_word(&right, i); + let result_zero = _mm_maskz_gf2p8mul_epi8(0, left, right); + assert_eq_m128i(result_zero, _mm_setzero_si128()); + let mask_bytes: __mmask16 = 0x0F_F0; + const MASK_WORDS: i32 = 0b01_10; + let expected_result = _mm_gf2p8mul_epi8(left, right); + let result_masked = _mm_maskz_gf2p8mul_epi8(mask_bytes, left, right); + let expected_masked = + _mm_blend_epi32::(_mm_setzero_si128(), expected_result); + assert_eq_m128i(result_masked, expected_masked); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw,avx512vl")] + unsafe fn test_mm_mask_gf2p8mul_epi8() { + let (left, right, _expected) = generate_byte_mul_test_data(); + + for i in 0..NUM_TEST_WORDS_128 { + let left = load_m128i_word(&left, i); + let right = load_m128i_word(&right, i); + let result_left = _mm_mask_gf2p8mul_epi8(left, 0, left, right); + assert_eq_m128i(result_left, left); + let mask_bytes: __mmask16 = 0x0F_F0; + const MASK_WORDS: i32 = 0b01_10; + let expected_result = _mm_gf2p8mul_epi8(left, right); + let result_masked = _mm_mask_gf2p8mul_epi8(left, mask_bytes, left, right); + let expected_masked = _mm_blend_epi32::(left, expected_result); + assert_eq_m128i(result_masked, expected_masked); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw")] + unsafe fn test_mm512_gf2p8affine_epi64_epi8() { + let identity: i64 = 0x01_02_04_08_10_20_40_80; + const IDENTITY_BYTE: i32 = 0; + let constant: i64 = 0; + const CONSTANT_BYTE: i32 = 0x63; + let identity = _mm512_set1_epi64(identity); + let constant = _mm512_set1_epi64(constant); + let constant_reference = _mm512_set1_epi8(CONSTANT_BYTE as i8); + + let (bytes, more_bytes, _) = generate_byte_mul_test_data(); + let (matrices, vectors, references) = generate_affine_mul_test_data(IDENTITY_BYTE as u8); + + for i in 0..NUM_TEST_WORDS_512 { + let data = load_m512i_word(&bytes, i); + let result = _mm512_gf2p8affine_epi64_epi8::(data, identity); + assert_eq_m512i(result, data); + let result = _mm512_gf2p8affine_epi64_epi8::(data, constant); + assert_eq_m512i(result, constant_reference); + let data = load_m512i_word(&more_bytes, i); + let result = _mm512_gf2p8affine_epi64_epi8::(data, identity); + assert_eq_m512i(result, data); + let result = _mm512_gf2p8affine_epi64_epi8::(data, constant); + assert_eq_m512i(result, constant_reference); + + let matrix = load_m512i_word(&matrices, i); + let vector = load_m512i_word(&vectors, i); + let reference = load_m512i_word(&references, i); + + let result = _mm512_gf2p8affine_epi64_epi8::(vector, matrix); + assert_eq_m512i(result, reference); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw")] + unsafe fn test_mm512_maskz_gf2p8affine_epi64_epi8() { + const CONSTANT_BYTE: i32 = 0x63; + let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); + + for i in 0..NUM_TEST_WORDS_512 { + let matrix = load_m512i_word(&matrices, i); + let vector = load_m512i_word(&vectors, i); + let result_zero = + _mm512_maskz_gf2p8affine_epi64_epi8::(0, vector, matrix); + assert_eq_m512i(result_zero, _mm512_setzero_si512()); + let mask_bytes: __mmask64 = 0x0F_0F_0F_0F_FF_FF_00_00; + let mask_words: __mmask16 = 0b01_01_01_01_11_11_00_00; + let expected_result = _mm512_gf2p8affine_epi64_epi8::(vector, matrix); + let result_masked = + _mm512_maskz_gf2p8affine_epi64_epi8::(mask_bytes, vector, matrix); + let expected_masked = + _mm512_mask_blend_epi32(mask_words, _mm512_setzero_si512(), expected_result); + assert_eq_m512i(result_masked, expected_masked); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw")] + unsafe fn test_mm512_mask_gf2p8affine_epi64_epi8() { + const CONSTANT_BYTE: i32 = 0x63; + let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); + + for i in 0..NUM_TEST_WORDS_512 { + let left = load_m512i_word(&vectors, i); + let right = load_m512i_word(&matrices, i); + let result_left = + _mm512_mask_gf2p8affine_epi64_epi8::(left, 0, left, right); + assert_eq_m512i(result_left, left); + let mask_bytes: __mmask64 = 0x0F_0F_0F_0F_FF_FF_00_00; + let mask_words: __mmask16 = 0b01_01_01_01_11_11_00_00; + let expected_result = _mm512_gf2p8affine_epi64_epi8::(left, right); + let result_masked = + _mm512_mask_gf2p8affine_epi64_epi8::(left, mask_bytes, left, right); + let expected_masked = _mm512_mask_blend_epi32(mask_words, left, expected_result); + assert_eq_m512i(result_masked, expected_masked); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw,avx512vl")] + unsafe fn test_mm256_gf2p8affine_epi64_epi8() { + let identity: i64 = 0x01_02_04_08_10_20_40_80; + const IDENTITY_BYTE: i32 = 0; + let constant: i64 = 0; + const CONSTANT_BYTE: i32 = 0x63; + let identity = _mm256_set1_epi64x(identity); + let constant = _mm256_set1_epi64x(constant); + let constant_reference = _mm256_set1_epi8(CONSTANT_BYTE as i8); + + let (bytes, more_bytes, _) = generate_byte_mul_test_data(); + let (matrices, vectors, references) = generate_affine_mul_test_data(IDENTITY_BYTE as u8); + + for i in 0..NUM_TEST_WORDS_256 { + let data = load_m256i_word(&bytes, i); + let result = _mm256_gf2p8affine_epi64_epi8::(data, identity); + assert_eq_m256i(result, data); + let result = _mm256_gf2p8affine_epi64_epi8::(data, constant); + assert_eq_m256i(result, constant_reference); + let data = load_m256i_word(&more_bytes, i); + let result = _mm256_gf2p8affine_epi64_epi8::(data, identity); + assert_eq_m256i(result, data); + let result = _mm256_gf2p8affine_epi64_epi8::(data, constant); + assert_eq_m256i(result, constant_reference); + + let matrix = load_m256i_word(&matrices, i); + let vector = load_m256i_word(&vectors, i); + let reference = load_m256i_word(&references, i); + + let result = _mm256_gf2p8affine_epi64_epi8::(vector, matrix); + assert_eq_m256i(result, reference); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_gf2p8affine_epi64_epi8() { + const CONSTANT_BYTE: i32 = 0x63; + let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); + + for i in 0..NUM_TEST_WORDS_256 { + let matrix = load_m256i_word(&matrices, i); + let vector = load_m256i_word(&vectors, i); + let result_zero = + _mm256_maskz_gf2p8affine_epi64_epi8::(0, vector, matrix); + assert_eq_m256i(result_zero, _mm256_setzero_si256()); + let mask_bytes: __mmask32 = 0xFF_0F_F0_00; + const MASK_WORDS: i32 = 0b11_01_10_00; + let expected_result = _mm256_gf2p8affine_epi64_epi8::(vector, matrix); + let result_masked = + _mm256_maskz_gf2p8affine_epi64_epi8::(mask_bytes, vector, matrix); + let expected_masked = + _mm256_blend_epi32::(_mm256_setzero_si256(), expected_result); + assert_eq_m256i(result_masked, expected_masked); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw,avx512vl")] + unsafe fn test_mm256_mask_gf2p8affine_epi64_epi8() { + const CONSTANT_BYTE: i32 = 0x63; + let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); + + for i in 0..NUM_TEST_WORDS_256 { + let left = load_m256i_word(&vectors, i); + let right = load_m256i_word(&matrices, i); + let result_left = + _mm256_mask_gf2p8affine_epi64_epi8::(left, 0, left, right); + assert_eq_m256i(result_left, left); + let mask_bytes: __mmask32 = 0xFF_0F_F0_00; + const MASK_WORDS: i32 = 0b11_01_10_00; + let expected_result = _mm256_gf2p8affine_epi64_epi8::(left, right); + let result_masked = + _mm256_mask_gf2p8affine_epi64_epi8::(left, mask_bytes, left, right); + let expected_masked = _mm256_blend_epi32::(left, expected_result); + assert_eq_m256i(result_masked, expected_masked); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw,avx512vl")] + unsafe fn test_mm_gf2p8affine_epi64_epi8() { + let identity: i64 = 0x01_02_04_08_10_20_40_80; + const IDENTITY_BYTE: i32 = 0; + let constant: i64 = 0; + const CONSTANT_BYTE: i32 = 0x63; + let identity = _mm_set1_epi64x(identity); + let constant = _mm_set1_epi64x(constant); + let constant_reference = _mm_set1_epi8(CONSTANT_BYTE as i8); + + let (bytes, more_bytes, _) = generate_byte_mul_test_data(); + let (matrices, vectors, references) = generate_affine_mul_test_data(IDENTITY_BYTE as u8); + + for i in 0..NUM_TEST_WORDS_128 { + let data = load_m128i_word(&bytes, i); + let result = _mm_gf2p8affine_epi64_epi8::(data, identity); + assert_eq_m128i(result, data); + let result = _mm_gf2p8affine_epi64_epi8::(data, constant); + assert_eq_m128i(result, constant_reference); + let data = load_m128i_word(&more_bytes, i); + let result = _mm_gf2p8affine_epi64_epi8::(data, identity); + assert_eq_m128i(result, data); + let result = _mm_gf2p8affine_epi64_epi8::(data, constant); + assert_eq_m128i(result, constant_reference); + + let matrix = load_m128i_word(&matrices, i); + let vector = load_m128i_word(&vectors, i); + let reference = load_m128i_word(&references, i); + + let result = _mm_gf2p8affine_epi64_epi8::(vector, matrix); + assert_eq_m128i(result, reference); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw,avx512vl")] + unsafe fn test_mm_maskz_gf2p8affine_epi64_epi8() { + const CONSTANT_BYTE: i32 = 0x63; + let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); + + for i in 0..NUM_TEST_WORDS_128 { + let matrix = load_m128i_word(&matrices, i); + let vector = load_m128i_word(&vectors, i); + let result_zero = _mm_maskz_gf2p8affine_epi64_epi8::(0, vector, matrix); + assert_eq_m128i(result_zero, _mm_setzero_si128()); + let mask_bytes: __mmask16 = 0x0F_F0; + const MASK_WORDS: i32 = 0b01_10; + let expected_result = _mm_gf2p8affine_epi64_epi8::(vector, matrix); + let result_masked = + _mm_maskz_gf2p8affine_epi64_epi8::(mask_bytes, vector, matrix); + let expected_masked = + _mm_blend_epi32::(_mm_setzero_si128(), expected_result); + assert_eq_m128i(result_masked, expected_masked); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw,avx512vl")] + unsafe fn test_mm_mask_gf2p8affine_epi64_epi8() { + const CONSTANT_BYTE: i32 = 0x63; + let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); + + for i in 0..NUM_TEST_WORDS_128 { + let left = load_m128i_word(&vectors, i); + let right = load_m128i_word(&matrices, i); + let result_left = + _mm_mask_gf2p8affine_epi64_epi8::(left, 0, left, right); + assert_eq_m128i(result_left, left); + let mask_bytes: __mmask16 = 0x0F_F0; + const MASK_WORDS: i32 = 0b01_10; + let expected_result = _mm_gf2p8affine_epi64_epi8::(left, right); + let result_masked = + _mm_mask_gf2p8affine_epi64_epi8::(left, mask_bytes, left, right); + let expected_masked = _mm_blend_epi32::(left, expected_result); + assert_eq_m128i(result_masked, expected_masked); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw")] + unsafe fn test_mm512_gf2p8affineinv_epi64_epi8() { + let identity: i64 = 0x01_02_04_08_10_20_40_80; + const IDENTITY_BYTE: i32 = 0; + const CONSTANT_BYTE: i32 = 0x63; + let identity = _mm512_set1_epi64(identity); + + // validate inversion + let (inputs, results) = generate_inv_tests_data(); + + for i in 0..NUM_BYTES_WORDS_512 { + let input = load_m512i_word(&inputs, i); + let reference = load_m512i_word(&results, i); + let result = _mm512_gf2p8affineinv_epi64_epi8::(input, identity); + let remultiplied = _mm512_gf2p8mul_epi8(result, input); + assert_eq_m512i(remultiplied, reference); + } + + // validate subsequent affine operation + let (matrices, vectors, _affine_expected) = + generate_affine_mul_test_data(CONSTANT_BYTE as u8); + + for i in 0..NUM_TEST_WORDS_512 { + let vector = load_m512i_word(&vectors, i); + let matrix = load_m512i_word(&matrices, i); + + let inv_vec = _mm512_gf2p8affineinv_epi64_epi8::(vector, identity); + let reference = _mm512_gf2p8affine_epi64_epi8::(inv_vec, matrix); + let result = _mm512_gf2p8affineinv_epi64_epi8::(vector, matrix); + assert_eq_m512i(result, reference); + } + + // validate everything by virtue of checking against the AES SBox + const AES_S_BOX_MATRIX: i64 = 0xF1_E3_C7_8F_1F_3E_7C_F8; + let sbox_matrix = _mm512_set1_epi64(AES_S_BOX_MATRIX); + + for i in 0..NUM_BYTES_WORDS_512 { + let reference = load_m512i_word(&AES_S_BOX, i); + let input = load_m512i_word(&inputs, i); + let result = _mm512_gf2p8affineinv_epi64_epi8::(input, sbox_matrix); + assert_eq_m512i(result, reference); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw")] + unsafe fn test_mm512_maskz_gf2p8affineinv_epi64_epi8() { + const CONSTANT_BYTE: i32 = 0x63; + let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); + + for i in 0..NUM_TEST_WORDS_512 { + let matrix = load_m512i_word(&matrices, i); + let vector = load_m512i_word(&vectors, i); + let result_zero = + _mm512_maskz_gf2p8affineinv_epi64_epi8::(0, vector, matrix); + assert_eq_m512i(result_zero, _mm512_setzero_si512()); + let mask_bytes: __mmask64 = 0x0F_0F_0F_0F_FF_FF_00_00; + let mask_words: __mmask16 = 0b01_01_01_01_11_11_00_00; + let expected_result = _mm512_gf2p8affineinv_epi64_epi8::(vector, matrix); + let result_masked = + _mm512_maskz_gf2p8affineinv_epi64_epi8::(mask_bytes, vector, matrix); + let expected_masked = + _mm512_mask_blend_epi32(mask_words, _mm512_setzero_si512(), expected_result); + assert_eq_m512i(result_masked, expected_masked); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw")] + unsafe fn test_mm512_mask_gf2p8affineinv_epi64_epi8() { + const CONSTANT_BYTE: i32 = 0x63; + let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); + + for i in 0..NUM_TEST_WORDS_512 { + let left = load_m512i_word(&vectors, i); + let right = load_m512i_word(&matrices, i); + let result_left = + _mm512_mask_gf2p8affineinv_epi64_epi8::(left, 0, left, right); + assert_eq_m512i(result_left, left); + let mask_bytes: __mmask64 = 0x0F_0F_0F_0F_FF_FF_00_00; + let mask_words: __mmask16 = 0b01_01_01_01_11_11_00_00; + let expected_result = _mm512_gf2p8affineinv_epi64_epi8::(left, right); + let result_masked = _mm512_mask_gf2p8affineinv_epi64_epi8::( + left, mask_bytes, left, right, + ); + let expected_masked = _mm512_mask_blend_epi32(mask_words, left, expected_result); + assert_eq_m512i(result_masked, expected_masked); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw,avx512vl")] + unsafe fn test_mm256_gf2p8affineinv_epi64_epi8() { + let identity: i64 = 0x01_02_04_08_10_20_40_80; + const IDENTITY_BYTE: i32 = 0; + const CONSTANT_BYTE: i32 = 0x63; + let identity = _mm256_set1_epi64x(identity); + + // validate inversion + let (inputs, results) = generate_inv_tests_data(); + + for i in 0..NUM_BYTES_WORDS_256 { + let input = load_m256i_word(&inputs, i); + let reference = load_m256i_word(&results, i); + let result = _mm256_gf2p8affineinv_epi64_epi8::(input, identity); + let remultiplied = _mm256_gf2p8mul_epi8(result, input); + assert_eq_m256i(remultiplied, reference); + } + + // validate subsequent affine operation + let (matrices, vectors, _affine_expected) = + generate_affine_mul_test_data(CONSTANT_BYTE as u8); + + for i in 0..NUM_TEST_WORDS_256 { + let vector = load_m256i_word(&vectors, i); + let matrix = load_m256i_word(&matrices, i); + + let inv_vec = _mm256_gf2p8affineinv_epi64_epi8::(vector, identity); + let reference = _mm256_gf2p8affine_epi64_epi8::(inv_vec, matrix); + let result = _mm256_gf2p8affineinv_epi64_epi8::(vector, matrix); + assert_eq_m256i(result, reference); + } + + // validate everything by virtue of checking against the AES SBox + const AES_S_BOX_MATRIX: i64 = 0xF1_E3_C7_8F_1F_3E_7C_F8; + let sbox_matrix = _mm256_set1_epi64x(AES_S_BOX_MATRIX); + + for i in 0..NUM_BYTES_WORDS_256 { + let reference = load_m256i_word(&AES_S_BOX, i); + let input = load_m256i_word(&inputs, i); + let result = _mm256_gf2p8affineinv_epi64_epi8::(input, sbox_matrix); + assert_eq_m256i(result, reference); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw,avx512vl")] + unsafe fn test_mm256_maskz_gf2p8affineinv_epi64_epi8() { + const CONSTANT_BYTE: i32 = 0x63; + let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); + + for i in 0..NUM_TEST_WORDS_256 { + let matrix = load_m256i_word(&matrices, i); + let vector = load_m256i_word(&vectors, i); + let result_zero = + _mm256_maskz_gf2p8affineinv_epi64_epi8::(0, vector, matrix); + assert_eq_m256i(result_zero, _mm256_setzero_si256()); + let mask_bytes: __mmask32 = 0xFF_0F_F0_00; + const MASK_WORDS: i32 = 0b11_01_10_00; + let expected_result = _mm256_gf2p8affineinv_epi64_epi8::(vector, matrix); + let result_masked = + _mm256_maskz_gf2p8affineinv_epi64_epi8::(mask_bytes, vector, matrix); + let expected_masked = + _mm256_blend_epi32::(_mm256_setzero_si256(), expected_result); + assert_eq_m256i(result_masked, expected_masked); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw,avx512vl")] + unsafe fn test_mm256_mask_gf2p8affineinv_epi64_epi8() { + const CONSTANT_BYTE: i32 = 0x63; + let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); + + for i in 0..NUM_TEST_WORDS_256 { + let left = load_m256i_word(&vectors, i); + let right = load_m256i_word(&matrices, i); + let result_left = + _mm256_mask_gf2p8affineinv_epi64_epi8::(left, 0, left, right); + assert_eq_m256i(result_left, left); + let mask_bytes: __mmask32 = 0xFF_0F_F0_00; + const MASK_WORDS: i32 = 0b11_01_10_00; + let expected_result = _mm256_gf2p8affineinv_epi64_epi8::(left, right); + let result_masked = _mm256_mask_gf2p8affineinv_epi64_epi8::( + left, mask_bytes, left, right, + ); + let expected_masked = _mm256_blend_epi32::(left, expected_result); + assert_eq_m256i(result_masked, expected_masked); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw,avx512vl")] + unsafe fn test_mm_gf2p8affineinv_epi64_epi8() { + let identity: i64 = 0x01_02_04_08_10_20_40_80; + const IDENTITY_BYTE: i32 = 0; + const CONSTANT_BYTE: i32 = 0x63; + let identity = _mm_set1_epi64x(identity); + + // validate inversion + let (inputs, results) = generate_inv_tests_data(); + + for i in 0..NUM_BYTES_WORDS_128 { + let input = load_m128i_word(&inputs, i); + let reference = load_m128i_word(&results, i); + let result = _mm_gf2p8affineinv_epi64_epi8::(input, identity); + let remultiplied = _mm_gf2p8mul_epi8(result, input); + assert_eq_m128i(remultiplied, reference); + } + + // validate subsequent affine operation + let (matrices, vectors, _affine_expected) = + generate_affine_mul_test_data(CONSTANT_BYTE as u8); + + for i in 0..NUM_TEST_WORDS_128 { + let vector = load_m128i_word(&vectors, i); + let matrix = load_m128i_word(&matrices, i); + + let inv_vec = _mm_gf2p8affineinv_epi64_epi8::(vector, identity); + let reference = _mm_gf2p8affine_epi64_epi8::(inv_vec, matrix); + let result = _mm_gf2p8affineinv_epi64_epi8::(vector, matrix); + assert_eq_m128i(result, reference); + } + + // validate everything by virtue of checking against the AES SBox + const AES_S_BOX_MATRIX: i64 = 0xF1_E3_C7_8F_1F_3E_7C_F8; + let sbox_matrix = _mm_set1_epi64x(AES_S_BOX_MATRIX); + + for i in 0..NUM_BYTES_WORDS_128 { + let reference = load_m128i_word(&AES_S_BOX, i); + let input = load_m128i_word(&inputs, i); + let result = _mm_gf2p8affineinv_epi64_epi8::(input, sbox_matrix); + assert_eq_m128i(result, reference); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw,avx512vl")] + unsafe fn test_mm_maskz_gf2p8affineinv_epi64_epi8() { + const CONSTANT_BYTE: i32 = 0x63; + let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); + + for i in 0..NUM_TEST_WORDS_128 { + let matrix = load_m128i_word(&matrices, i); + let vector = load_m128i_word(&vectors, i); + let result_zero = + _mm_maskz_gf2p8affineinv_epi64_epi8::(0, vector, matrix); + assert_eq_m128i(result_zero, _mm_setzero_si128()); + let mask_bytes: __mmask16 = 0x0F_F0; + const MASK_WORDS: i32 = 0b01_10; + let expected_result = _mm_gf2p8affineinv_epi64_epi8::(vector, matrix); + let result_masked = + _mm_maskz_gf2p8affineinv_epi64_epi8::(mask_bytes, vector, matrix); + let expected_masked = + _mm_blend_epi32::(_mm_setzero_si128(), expected_result); + assert_eq_m128i(result_masked, expected_masked); + } + } + + #[simd_test(enable = "avx512gfni,avx512bw,avx512vl")] + unsafe fn test_mm_mask_gf2p8affineinv_epi64_epi8() { + const CONSTANT_BYTE: i32 = 0x63; + let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); + + for i in 0..NUM_TEST_WORDS_128 { + let left = load_m128i_word(&vectors, i); + let right = load_m128i_word(&matrices, i); + let result_left = + _mm_mask_gf2p8affineinv_epi64_epi8::(left, 0, left, right); + assert_eq_m128i(result_left, left); + let mask_bytes: __mmask16 = 0x0F_F0; + const MASK_WORDS: i32 = 0b01_10; + let expected_result = _mm_gf2p8affineinv_epi64_epi8::(left, right); + let result_masked = + _mm_mask_gf2p8affineinv_epi64_epi8::(left, mask_bytes, left, right); + let expected_masked = _mm_blend_epi32::(left, expected_result); + assert_eq_m128i(result_masked, expected_masked); + } + } +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512ifma.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512ifma.rs index 425d0ff7e..26aa0320f 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512ifma.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512ifma.rs @@ -105,7 +105,7 @@ extern "C" { #[cfg(test)] mod tests { - use std; + use stdarch_test::simd_test; use crate::core_arch::x86::*; diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vaes.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vaes.rs new file mode 100644 index 000000000..676de312b --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vaes.rs @@ -0,0 +1,332 @@ +//! Vectorized AES Instructions (VAES) +//! +//! The intrinsics here correspond to those in the `immintrin.h` C header. +//! +//! The reference is [Intel 64 and IA-32 Architectures Software Developer's +//! Manual Volume 2: Instruction Set Reference, A-Z][intel64_ref]. +//! +//! [intel64_ref]: http://www.intel.de/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf + +use crate::core_arch::x86::__m256i; +use crate::core_arch::x86::__m512i; + +#[cfg(test)] +use stdarch_test::assert_instr; + +#[allow(improper_ctypes)] +extern "C" { + #[link_name = "llvm.x86.aesni.aesenc.256"] + fn aesenc_256(a: __m256i, round_key: __m256i) -> __m256i; + #[link_name = "llvm.x86.aesni.aesenclast.256"] + fn aesenclast_256(a: __m256i, round_key: __m256i) -> __m256i; + #[link_name = "llvm.x86.aesni.aesdec.256"] + fn aesdec_256(a: __m256i, round_key: __m256i) -> __m256i; + #[link_name = "llvm.x86.aesni.aesdeclast.256"] + fn aesdeclast_256(a: __m256i, round_key: __m256i) -> __m256i; + #[link_name = "llvm.x86.aesni.aesenc.512"] + fn aesenc_512(a: __m512i, round_key: __m512i) -> __m512i; + #[link_name = "llvm.x86.aesni.aesenclast.512"] + fn aesenclast_512(a: __m512i, round_key: __m512i) -> __m512i; + #[link_name = "llvm.x86.aesni.aesdec.512"] + fn aesdec_512(a: __m512i, round_key: __m512i) -> __m512i; + #[link_name = "llvm.x86.aesni.aesdeclast.512"] + fn aesdeclast_512(a: __m512i, round_key: __m512i) -> __m512i; +} + +/// Performs one round of an AES encryption flow on each 128-bit word (state) in `a` using +/// the corresponding 128-bit word (key) in `round_key`. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_aesenc_epi128) +#[inline] +#[target_feature(enable = "avx512vaes,avx512vl")] +#[cfg_attr(test, assert_instr(vaesenc))] +pub unsafe fn _mm256_aesenc_epi128(a: __m256i, round_key: __m256i) -> __m256i { + aesenc_256(a, round_key) +} + +/// Performs the last round of an AES encryption flow on each 128-bit word (state) in `a` using +/// the corresponding 128-bit word (key) in `round_key`. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_aesenclast_epi128) +#[inline] +#[target_feature(enable = "avx512vaes,avx512vl")] +#[cfg_attr(test, assert_instr(vaesenclast))] +pub unsafe fn _mm256_aesenclast_epi128(a: __m256i, round_key: __m256i) -> __m256i { + aesenclast_256(a, round_key) +} + +/// Performs one round of an AES decryption flow on each 128-bit word (state) in `a` using +/// the corresponding 128-bit word (key) in `round_key`. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_aesdec_epi128) +#[inline] +#[target_feature(enable = "avx512vaes,avx512vl")] +#[cfg_attr(test, assert_instr(vaesdec))] +pub unsafe fn _mm256_aesdec_epi128(a: __m256i, round_key: __m256i) -> __m256i { + aesdec_256(a, round_key) +} + +/// Performs the last round of an AES decryption flow on each 128-bit word (state) in `a` using +/// the corresponding 128-bit word (key) in `round_key`. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_aesdeclast_epi128) +#[inline] +#[target_feature(enable = "avx512vaes,avx512vl")] +#[cfg_attr(test, assert_instr(vaesdeclast))] +pub unsafe fn _mm256_aesdeclast_epi128(a: __m256i, round_key: __m256i) -> __m256i { + aesdeclast_256(a, round_key) +} + +/// Performs one round of an AES encryption flow on each 128-bit word (state) in `a` using +/// the corresponding 128-bit word (key) in `round_key`. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_aesenc_epi128) +#[inline] +#[target_feature(enable = "avx512vaes,avx512f")] +#[cfg_attr(test, assert_instr(vaesenc))] +pub unsafe fn _mm512_aesenc_epi128(a: __m512i, round_key: __m512i) -> __m512i { + aesenc_512(a, round_key) +} + +/// Performs the last round of an AES encryption flow on each 128-bit word (state) in `a` using +/// the corresponding 128-bit word (key) in `round_key`. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_aesenclast_epi128) +#[inline] +#[target_feature(enable = "avx512vaes,avx512f")] +#[cfg_attr(test, assert_instr(vaesenclast))] +pub unsafe fn _mm512_aesenclast_epi128(a: __m512i, round_key: __m512i) -> __m512i { + aesenclast_512(a, round_key) +} + +/// Performs one round of an AES decryption flow on each 128-bit word (state) in `a` using +/// the corresponding 128-bit word (key) in `round_key`. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_aesdec_epi128) +#[inline] +#[target_feature(enable = "avx512vaes,avx512f")] +#[cfg_attr(test, assert_instr(vaesdec))] +pub unsafe fn _mm512_aesdec_epi128(a: __m512i, round_key: __m512i) -> __m512i { + aesdec_512(a, round_key) +} + +/// Performs the last round of an AES decryption flow on each 128-bit word (state) in `a` using +/// the corresponding 128-bit word (key) in `round_key`. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_aesdeclast_epi128) +#[inline] +#[target_feature(enable = "avx512vaes,avx512f")] +#[cfg_attr(test, assert_instr(vaesdeclast))] +pub unsafe fn _mm512_aesdeclast_epi128(a: __m512i, round_key: __m512i) -> __m512i { + aesdeclast_512(a, round_key) +} + +#[cfg(test)] +mod tests { + // The constants in the tests below are just bit patterns. They should not + // be interpreted as integers; signedness does not make sense for them, but + // __mXXXi happens to be defined in terms of signed integers. + #![allow(overflowing_literals)] + + use stdarch_test::simd_test; + + use crate::core_arch::x86::*; + + // the first parts of these tests are straight ports from the AES-NI tests + // the second parts directly compare the two, for inputs that are different across lanes + // and "more random" than the standard test vectors + // ideally we'd be using quickcheck here instead + + #[target_feature(enable = "avx2")] + unsafe fn helper_for_256_avx512vaes( + linear: unsafe fn(__m128i, __m128i) -> __m128i, + vectorized: unsafe fn(__m256i, __m256i) -> __m256i, + ) { + let a = _mm256_set_epi64x( + 0xDCB4DB3657BF0B7D, + 0x18DB0601068EDD9F, + 0xB76B908233200DC5, + 0xE478235FA8E22D5E, + ); + let k = _mm256_set_epi64x( + 0x672F6F105A94CEA7, + 0x8298B8FFCA5F829C, + 0xA3927047B3FB61D8, + 0x978093862CDE7187, + ); + let mut a_decomp = [_mm_setzero_si128(); 2]; + a_decomp[0] = _mm256_extracti128_si256::<0>(a); + a_decomp[1] = _mm256_extracti128_si256::<1>(a); + let mut k_decomp = [_mm_setzero_si128(); 2]; + k_decomp[0] = _mm256_extracti128_si256::<0>(k); + k_decomp[1] = _mm256_extracti128_si256::<1>(k); + let r = vectorized(a, k); + let mut e_decomp = [_mm_setzero_si128(); 2]; + for i in 0..2 { + e_decomp[i] = linear(a_decomp[i], k_decomp[i]); + } + assert_eq_m128i(_mm256_extracti128_si256::<0>(r), e_decomp[0]); + assert_eq_m128i(_mm256_extracti128_si256::<1>(r), e_decomp[1]); + } + + #[target_feature(enable = "sse2")] + unsafe fn setup_state_key(broadcast: unsafe fn(__m128i) -> T) -> (T, T) { + // Constants taken from https://msdn.microsoft.com/en-us/library/cc664949.aspx. + let a = _mm_set_epi64x(0x0123456789abcdef, 0x8899aabbccddeeff); + let k = _mm_set_epi64x(0x1133557799bbddff, 0x0022446688aaccee); + (broadcast(a), broadcast(k)) + } + + #[target_feature(enable = "avx2")] + unsafe fn setup_state_key_256() -> (__m256i, __m256i) { + setup_state_key(_mm256_broadcastsi128_si256) + } + + #[target_feature(enable = "avx512f")] + unsafe fn setup_state_key_512() -> (__m512i, __m512i) { + setup_state_key(_mm512_broadcast_i32x4) + } + + #[simd_test(enable = "avx512vaes,avx512vl")] + unsafe fn test_mm256_aesdec_epi128() { + // Constants taken from https://msdn.microsoft.com/en-us/library/cc664949.aspx. + let (a, k) = setup_state_key_256(); + let e = _mm_set_epi64x(0x044e4f5176fec48f, 0xb57ecfa381da39ee); + let e = _mm256_broadcastsi128_si256(e); + let r = _mm256_aesdec_epi128(a, k); + assert_eq_m256i(r, e); + + helper_for_256_avx512vaes(_mm_aesdec_si128, _mm256_aesdec_epi128); + } + + #[simd_test(enable = "avx512vaes,avx512vl")] + unsafe fn test_mm256_aesdeclast_epi128() { + // Constants taken from https://msdn.microsoft.com/en-us/library/cc714178.aspx. + let (a, k) = setup_state_key_256(); + let e = _mm_set_epi64x(0x36cad57d9072bf9e, 0xf210dd981fa4a493); + let e = _mm256_broadcastsi128_si256(e); + let r = _mm256_aesdeclast_epi128(a, k); + assert_eq_m256i(r, e); + + helper_for_256_avx512vaes(_mm_aesdeclast_si128, _mm256_aesdeclast_epi128); + } + + #[simd_test(enable = "avx512vaes,avx512vl")] + unsafe fn test_mm256_aesenc_epi128() { + // Constants taken from https://msdn.microsoft.com/en-us/library/cc664810.aspx. + // they are repeated appropriately + let (a, k) = setup_state_key_256(); + let e = _mm_set_epi64x(0x16ab0e57dfc442ed, 0x28e4ee1884504333); + let e = _mm256_broadcastsi128_si256(e); + let r = _mm256_aesenc_epi128(a, k); + assert_eq_m256i(r, e); + + helper_for_256_avx512vaes(_mm_aesenc_si128, _mm256_aesenc_epi128); + } + + #[simd_test(enable = "avx512vaes,avx512vl")] + unsafe fn test_mm256_aesenclast_epi128() { + // Constants taken from https://msdn.microsoft.com/en-us/library/cc714136.aspx. + let (a, k) = setup_state_key_256(); + let e = _mm_set_epi64x(0xb6dd7df25d7ab320, 0x4b04f98cf4c860f8); + let e = _mm256_broadcastsi128_si256(e); + let r = _mm256_aesenclast_epi128(a, k); + assert_eq_m256i(r, e); + + helper_for_256_avx512vaes(_mm_aesenclast_si128, _mm256_aesenclast_epi128); + } + + #[target_feature(enable = "avx512f")] + unsafe fn helper_for_512_avx512vaes( + linear: unsafe fn(__m128i, __m128i) -> __m128i, + vectorized: unsafe fn(__m512i, __m512i) -> __m512i, + ) { + let a = _mm512_set_epi64( + 0xDCB4DB3657BF0B7D, + 0x18DB0601068EDD9F, + 0xB76B908233200DC5, + 0xE478235FA8E22D5E, + 0xAB05CFFA2621154C, + 0x1171B47A186174C9, + 0x8C6B6C0E7595CEC9, + 0xBE3E7D4934E961BD, + ); + let k = _mm512_set_epi64( + 0x672F6F105A94CEA7, + 0x8298B8FFCA5F829C, + 0xA3927047B3FB61D8, + 0x978093862CDE7187, + 0xB1927AB22F31D0EC, + 0xA9A5DA619BE4D7AF, + 0xCA2590F56884FDC6, + 0x19BE9F660038BDB5, + ); + let mut a_decomp = [_mm_setzero_si128(); 4]; + a_decomp[0] = _mm512_extracti32x4_epi32::<0>(a); + a_decomp[1] = _mm512_extracti32x4_epi32::<1>(a); + a_decomp[2] = _mm512_extracti32x4_epi32::<2>(a); + a_decomp[3] = _mm512_extracti32x4_epi32::<3>(a); + let mut k_decomp = [_mm_setzero_si128(); 4]; + k_decomp[0] = _mm512_extracti32x4_epi32::<0>(k); + k_decomp[1] = _mm512_extracti32x4_epi32::<1>(k); + k_decomp[2] = _mm512_extracti32x4_epi32::<2>(k); + k_decomp[3] = _mm512_extracti32x4_epi32::<3>(k); + let r = vectorized(a, k); + let mut e_decomp = [_mm_setzero_si128(); 4]; + for i in 0..4 { + e_decomp[i] = linear(a_decomp[i], k_decomp[i]); + } + assert_eq_m128i(_mm512_extracti32x4_epi32::<0>(r), e_decomp[0]); + assert_eq_m128i(_mm512_extracti32x4_epi32::<1>(r), e_decomp[1]); + assert_eq_m128i(_mm512_extracti32x4_epi32::<2>(r), e_decomp[2]); + assert_eq_m128i(_mm512_extracti32x4_epi32::<3>(r), e_decomp[3]); + } + + #[simd_test(enable = "avx512vaes,avx512f")] + unsafe fn test_mm512_aesdec_epi128() { + // Constants taken from https://msdn.microsoft.com/en-us/library/cc664949.aspx. + let (a, k) = setup_state_key_512(); + let e = _mm_set_epi64x(0x044e4f5176fec48f, 0xb57ecfa381da39ee); + let e = _mm512_broadcast_i32x4(e); + let r = _mm512_aesdec_epi128(a, k); + assert_eq_m512i(r, e); + + helper_for_512_avx512vaes(_mm_aesdec_si128, _mm512_aesdec_epi128); + } + + #[simd_test(enable = "avx512vaes,avx512f")] + unsafe fn test_mm512_aesdeclast_epi128() { + // Constants taken from https://msdn.microsoft.com/en-us/library/cc714178.aspx. + let (a, k) = setup_state_key_512(); + let e = _mm_set_epi64x(0x36cad57d9072bf9e, 0xf210dd981fa4a493); + let e = _mm512_broadcast_i32x4(e); + let r = _mm512_aesdeclast_epi128(a, k); + assert_eq_m512i(r, e); + + helper_for_512_avx512vaes(_mm_aesdeclast_si128, _mm512_aesdeclast_epi128); + } + + #[simd_test(enable = "avx512vaes,avx512f")] + unsafe fn test_mm512_aesenc_epi128() { + // Constants taken from https://msdn.microsoft.com/en-us/library/cc664810.aspx. + let (a, k) = setup_state_key_512(); + let e = _mm_set_epi64x(0x16ab0e57dfc442ed, 0x28e4ee1884504333); + let e = _mm512_broadcast_i32x4(e); + let r = _mm512_aesenc_epi128(a, k); + assert_eq_m512i(r, e); + + helper_for_512_avx512vaes(_mm_aesenc_si128, _mm512_aesenc_epi128); + } + + #[simd_test(enable = "avx512vaes,avx512f")] + unsafe fn test_mm512_aesenclast_epi128() { + // Constants taken from https://msdn.microsoft.com/en-us/library/cc714136.aspx. + let (a, k) = setup_state_key_512(); + let e = _mm_set_epi64x(0xb6dd7df25d7ab320, 0x4b04f98cf4c860f8); + let e = _mm512_broadcast_i32x4(e); + let r = _mm512_aesenclast_epi128(a, k); + assert_eq_m512i(r, e); + + helper_for_512_avx512vaes(_mm_aesenclast_si128, _mm512_aesenclast_epi128); + } +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vbmi.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vbmi.rs new file mode 100644 index 000000000..f0ff75162 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vbmi.rs @@ -0,0 +1,916 @@ +use crate::core_arch::{simd::*, simd_llvm::*, x86::*}; + +#[cfg(test)] +use stdarch_test::assert_instr; + +/// Shuffle 8-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permutex2var_epi8&expand=4262) +#[inline] +#[target_feature(enable = "avx512vbmi")] +#[cfg_attr(test, assert_instr(vperm))] //should be vpermi2b +pub unsafe fn _mm512_permutex2var_epi8(a: __m512i, idx: __m512i, b: __m512i) -> __m512i { + transmute(vpermi2b(a.as_i8x64(), idx.as_i8x64(), b.as_i8x64())) +} + +/// Shuffle 8-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permutex2var_epi8&expand=4259) +#[inline] +#[target_feature(enable = "avx512vbmi")] +#[cfg_attr(test, assert_instr(vpermt2b))] +pub unsafe fn _mm512_mask_permutex2var_epi8( + a: __m512i, + k: __mmask64, + idx: __m512i, + b: __m512i, +) -> __m512i { + let permute = _mm512_permutex2var_epi8(a, idx, b).as_i8x64(); + transmute(simd_select_bitmask(k, permute, a.as_i8x64())) +} + +/// Shuffle 8-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permutex2var_epi8&expand=4261) +#[inline] +#[target_feature(enable = "avx512vbmi")] +#[cfg_attr(test, assert_instr(vperm))] //should be vpermi2b +pub unsafe fn _mm512_maskz_permutex2var_epi8( + k: __mmask64, + a: __m512i, + idx: __m512i, + b: __m512i, +) -> __m512i { + let permute = _mm512_permutex2var_epi8(a, idx, b).as_i8x64(); + let zero = _mm512_setzero_si512().as_i8x64(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle 8-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask2_permutex2var_epi8&expand=4260) +#[inline] +#[target_feature(enable = "avx512vbmi")] +#[cfg_attr(test, assert_instr(vpermi2b))] +pub unsafe fn _mm512_mask2_permutex2var_epi8( + a: __m512i, + idx: __m512i, + k: __mmask64, + b: __m512i, +) -> __m512i { + let permute = _mm512_permutex2var_epi8(a, idx, b).as_i8x64(); + transmute(simd_select_bitmask(k, permute, idx.as_i8x64())) +} + +/// Shuffle 8-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permutex2var_epi8&expand=4258) +#[inline] +#[target_feature(enable = "avx512vbmi,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //should be vpermi2b +pub unsafe fn _mm256_permutex2var_epi8(a: __m256i, idx: __m256i, b: __m256i) -> __m256i { + transmute(vpermi2b256(a.as_i8x32(), idx.as_i8x32(), b.as_i8x32())) +} + +/// Shuffle 8-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_permutex2var_epi8&expand=4255) +#[inline] +#[target_feature(enable = "avx512vbmi,avx512vl")] +#[cfg_attr(test, assert_instr(vpermt2b))] +pub unsafe fn _mm256_mask_permutex2var_epi8( + a: __m256i, + k: __mmask32, + idx: __m256i, + b: __m256i, +) -> __m256i { + let permute = _mm256_permutex2var_epi8(a, idx, b).as_i8x32(); + transmute(simd_select_bitmask(k, permute, a.as_i8x32())) +} + +/// Shuffle 8-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permutex2var_epi8&expand=4257) +#[inline] +#[target_feature(enable = "avx512vbmi,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //should be vpermi2b +pub unsafe fn _mm256_maskz_permutex2var_epi8( + k: __mmask32, + a: __m256i, + idx: __m256i, + b: __m256i, +) -> __m256i { + let permute = _mm256_permutex2var_epi8(a, idx, b).as_i8x32(); + let zero = _mm256_setzero_si256().as_i8x32(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle 8-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask2_permutex2var_epi8&expand=4256) +#[inline] +#[target_feature(enable = "avx512vbmi,avx512vl")] +#[cfg_attr(test, assert_instr(vpermi2b))] +pub unsafe fn _mm256_mask2_permutex2var_epi8( + a: __m256i, + idx: __m256i, + k: __mmask32, + b: __m256i, +) -> __m256i { + let permute = _mm256_permutex2var_epi8(a, idx, b).as_i8x32(); + transmute(simd_select_bitmask(k, permute, idx.as_i8x32())) +} + +/// Shuffle 8-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_permutex2var_epi8&expand=4254) +#[inline] +#[target_feature(enable = "avx512vbmi,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //should be vpermi2b +pub unsafe fn _mm_permutex2var_epi8(a: __m128i, idx: __m128i, b: __m128i) -> __m128i { + transmute(vpermi2b128(a.as_i8x16(), idx.as_i8x16(), b.as_i8x16())) +} + +/// Shuffle 8-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_permutex2var_epi8&expand=4251) +#[inline] +#[target_feature(enable = "avx512vbmi,avx512vl")] +#[cfg_attr(test, assert_instr(vpermt2b))] +pub unsafe fn _mm_mask_permutex2var_epi8( + a: __m128i, + k: __mmask16, + idx: __m128i, + b: __m128i, +) -> __m128i { + let permute = _mm_permutex2var_epi8(a, idx, b).as_i8x16(); + transmute(simd_select_bitmask(k, permute, a.as_i8x16())) +} + +/// Shuffle 8-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_permutex2var_epi8&expand=4253) +#[inline] +#[target_feature(enable = "avx512vbmi,avx512vl")] +#[cfg_attr(test, assert_instr(vperm))] //should be vpermi2b +pub unsafe fn _mm_maskz_permutex2var_epi8( + k: __mmask16, + a: __m128i, + idx: __m128i, + b: __m128i, +) -> __m128i { + let permute = _mm_permutex2var_epi8(a, idx, b).as_i8x16(); + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle 8-bit integers in a and b across lanes using the corresponding selector and index in idx, and store the results in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask2_permutex2var_epi8&expand=4252) +#[inline] +#[target_feature(enable = "avx512vbmi,avx512vl")] +#[cfg_attr(test, assert_instr(vpermi2b))] +pub unsafe fn _mm_mask2_permutex2var_epi8( + a: __m128i, + idx: __m128i, + k: __mmask16, + b: __m128i, +) -> __m128i { + let permute = _mm_permutex2var_epi8(a, idx, b).as_i8x16(); + transmute(simd_select_bitmask(k, permute, idx.as_i8x16())) +} + +/// Shuffle 8-bit integers in a across lanes using the corresponding index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permutexvar_epi8&expand=4316) +#[inline] +#[target_feature(enable = "avx512vbmi")] +#[cfg_attr(test, assert_instr(vpermb))] +pub unsafe fn _mm512_permutexvar_epi8(idx: __m512i, a: __m512i) -> __m512i { + transmute(vpermb(a.as_i8x64(), idx.as_i8x64())) +} + +/// Shuffle 8-bit integers in a across lanes using the corresponding index in idx, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permutexvar_epi8&expand=4314) +#[inline] +#[target_feature(enable = "avx512vbmi")] +#[cfg_attr(test, assert_instr(vpermb))] +pub unsafe fn _mm512_mask_permutexvar_epi8( + src: __m512i, + k: __mmask64, + idx: __m512i, + a: __m512i, +) -> __m512i { + let permute = _mm512_permutexvar_epi8(idx, a).as_i8x64(); + transmute(simd_select_bitmask(k, permute, src.as_i8x64())) +} + +/// Shuffle 8-bit integers in a across lanes using the corresponding index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permutexvar_epi8&expand=4315) +#[inline] +#[target_feature(enable = "avx512vbmi")] +#[cfg_attr(test, assert_instr(vpermb))] +pub unsafe fn _mm512_maskz_permutexvar_epi8(k: __mmask64, idx: __m512i, a: __m512i) -> __m512i { + let permute = _mm512_permutexvar_epi8(idx, a).as_i8x64(); + let zero = _mm512_setzero_si512().as_i8x64(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle 8-bit integers in a across lanes using the corresponding index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permutexvar_epi8&expand=4313) +#[inline] +#[target_feature(enable = "avx512vbmi,avx512vl")] +#[cfg_attr(test, assert_instr(vpermb))] +pub unsafe fn _mm256_permutexvar_epi8(idx: __m256i, a: __m256i) -> __m256i { + transmute(vpermb256(a.as_i8x32(), idx.as_i8x32())) +} + +/// Shuffle 8-bit integers in a across lanes using the corresponding index in idx, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_permutexvar_epi8&expand=4311) +#[inline] +#[target_feature(enable = "avx512vbmi,avx512vl")] +#[cfg_attr(test, assert_instr(vpermb))] +pub unsafe fn _mm256_mask_permutexvar_epi8( + src: __m256i, + k: __mmask32, + idx: __m256i, + a: __m256i, +) -> __m256i { + let permute = _mm256_permutexvar_epi8(idx, a).as_i8x32(); + transmute(simd_select_bitmask(k, permute, src.as_i8x32())) +} + +/// Shuffle 8-bit integers in a across lanes using the corresponding index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permutexvar_epi8&expand=4312) +#[inline] +#[target_feature(enable = "avx512vbmi,avx512vl")] +#[cfg_attr(test, assert_instr(vpermb))] +pub unsafe fn _mm256_maskz_permutexvar_epi8(k: __mmask32, idx: __m256i, a: __m256i) -> __m256i { + let permute = _mm256_permutexvar_epi8(idx, a).as_i8x32(); + let zero = _mm256_setzero_si256().as_i8x32(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// Shuffle 8-bit integers in a across lanes using the corresponding index in idx, and store the results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_permutexvar_epi8&expand=4310) +#[inline] +#[target_feature(enable = "avx512vbmi,avx512vl")] +#[cfg_attr(test, assert_instr(vpermb))] +pub unsafe fn _mm_permutexvar_epi8(idx: __m128i, a: __m128i) -> __m128i { + transmute(vpermb128(a.as_i8x16(), idx.as_i8x16())) +} + +/// Shuffle 8-bit integers in a across lanes using the corresponding index in idx, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_permutexvar_epi8&expand=4308) +#[inline] +#[target_feature(enable = "avx512vbmi,avx512vl")] +#[cfg_attr(test, assert_instr(vpermb))] +pub unsafe fn _mm_mask_permutexvar_epi8( + src: __m128i, + k: __mmask16, + idx: __m128i, + a: __m128i, +) -> __m128i { + let permute = _mm_permutexvar_epi8(idx, a).as_i8x16(); + transmute(simd_select_bitmask(k, permute, src.as_i8x16())) +} + +/// Shuffle 8-bit integers in a across lanes using the corresponding index in idx, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_permutexvar_epi8&expand=4309) +#[inline] +#[target_feature(enable = "avx512vbmi,avx512vl")] +#[cfg_attr(test, assert_instr(vpermb))] +pub unsafe fn _mm_maskz_permutexvar_epi8(k: __mmask16, idx: __m128i, a: __m128i) -> __m128i { + let permute = _mm_permutexvar_epi8(idx, a).as_i8x16(); + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask(k, permute, zero)) +} + +/// For each 64-bit element in b, select 8 unaligned bytes using a byte-granular shift control within the corresponding 64-bit element of a, and store the 8 assembled bytes to the corresponding 64-bit element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_multishift_epi64_epi8&expand=4026) +#[inline] +#[target_feature(enable = "avx512vbmi")] +#[cfg_attr(test, assert_instr(vpmultishiftqb))] +pub unsafe fn _mm512_multishift_epi64_epi8(a: __m512i, b: __m512i) -> __m512i { + transmute(vpmultishiftqb(a.as_i8x64(), b.as_i8x64())) +} + +/// For each 64-bit element in b, select 8 unaligned bytes using a byte-granular shift control within the corresponding 64-bit element of a, and store the 8 assembled bytes to the corresponding 64-bit element of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_multishift_epi64_epi8&expand=4024) +#[inline] +#[target_feature(enable = "avx512vbmi")] +#[cfg_attr(test, assert_instr(vpmultishiftqb))] +pub unsafe fn _mm512_mask_multishift_epi64_epi8( + src: __m512i, + k: __mmask64, + a: __m512i, + b: __m512i, +) -> __m512i { + let multishift = _mm512_multishift_epi64_epi8(a, b).as_i8x64(); + transmute(simd_select_bitmask(k, multishift, src.as_i8x64())) +} + +/// For each 64-bit element in b, select 8 unaligned bytes using a byte-granular shift control within the corresponding 64-bit element of a, and store the 8 assembled bytes to the corresponding 64-bit element of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_multishift_epi64_epi8&expand=4025) +#[inline] +#[target_feature(enable = "avx512vbmi")] +#[cfg_attr(test, assert_instr(vpmultishiftqb))] +pub unsafe fn _mm512_maskz_multishift_epi64_epi8(k: __mmask64, a: __m512i, b: __m512i) -> __m512i { + let multishift = _mm512_multishift_epi64_epi8(a, b).as_i8x64(); + let zero = _mm512_setzero_si512().as_i8x64(); + transmute(simd_select_bitmask(k, multishift, zero)) +} + +/// For each 64-bit element in b, select 8 unaligned bytes using a byte-granular shift control within the corresponding 64-bit element of a, and store the 8 assembled bytes to the corresponding 64-bit element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_multishift_epi64_epi8&expand=4023) +#[inline] +#[target_feature(enable = "avx512vbmi,avx512vl")] +#[cfg_attr(test, assert_instr(vpmultishiftqb))] +pub unsafe fn _mm256_multishift_epi64_epi8(a: __m256i, b: __m256i) -> __m256i { + transmute(vpmultishiftqb256(a.as_i8x32(), b.as_i8x32())) +} + +/// For each 64-bit element in b, select 8 unaligned bytes using a byte-granular shift control within the corresponding 64-bit element of a, and store the 8 assembled bytes to the corresponding 64-bit element of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_multishift_epi64_epi8&expand=4021) +#[inline] +#[target_feature(enable = "avx512vbmi,avx512vl")] +#[cfg_attr(test, assert_instr(vpmultishiftqb))] +pub unsafe fn _mm256_mask_multishift_epi64_epi8( + src: __m256i, + k: __mmask32, + a: __m256i, + b: __m256i, +) -> __m256i { + let multishift = _mm256_multishift_epi64_epi8(a, b).as_i8x32(); + transmute(simd_select_bitmask(k, multishift, src.as_i8x32())) +} + +/// For each 64-bit element in b, select 8 unaligned bytes using a byte-granular shift control within the corresponding 64-bit element of a, and store the 8 assembled bytes to the corresponding 64-bit element of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_multishift_epi64_epi8&expand=4022) +#[inline] +#[target_feature(enable = "avx512vbmi,avx512vl")] +#[cfg_attr(test, assert_instr(vpmultishiftqb))] +pub unsafe fn _mm256_maskz_multishift_epi64_epi8(k: __mmask32, a: __m256i, b: __m256i) -> __m256i { + let multishift = _mm256_multishift_epi64_epi8(a, b).as_i8x32(); + let zero = _mm256_setzero_si256().as_i8x32(); + transmute(simd_select_bitmask(k, multishift, zero)) +} + +/// For each 64-bit element in b, select 8 unaligned bytes using a byte-granular shift control within the corresponding 64-bit element of a, and store the 8 assembled bytes to the corresponding 64-bit element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/IntrinsicsGuide/#text=_mm_multishift_epi64_epi8&expand=4020) +#[inline] +#[target_feature(enable = "avx512vbmi,avx512vl")] +#[cfg_attr(test, assert_instr(vpmultishiftqb))] +pub unsafe fn _mm_multishift_epi64_epi8(a: __m128i, b: __m128i) -> __m128i { + transmute(vpmultishiftqb128(a.as_i8x16(), b.as_i8x16())) +} + +/// For each 64-bit element in b, select 8 unaligned bytes using a byte-granular shift control within the corresponding 64-bit element of a, and store the 8 assembled bytes to the corresponding 64-bit element of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_multishift_epi64_epi8&expand=4018) +#[inline] +#[target_feature(enable = "avx512vbmi,avx512vl")] +#[cfg_attr(test, assert_instr(vpmultishiftqb))] +pub unsafe fn _mm_mask_multishift_epi64_epi8( + src: __m128i, + k: __mmask16, + a: __m128i, + b: __m128i, +) -> __m128i { + let multishift = _mm_multishift_epi64_epi8(a, b).as_i8x16(); + transmute(simd_select_bitmask(k, multishift, src.as_i8x16())) +} + +/// For each 64-bit element in b, select 8 unaligned bytes using a byte-granular shift control within the corresponding 64-bit element of a, and store the 8 assembled bytes to the corresponding 64-bit element of dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_multishift_epi64_epi8&expand=4019) +#[inline] +#[target_feature(enable = "avx512vbmi,avx512vl")] +#[cfg_attr(test, assert_instr(vpmultishiftqb))] +pub unsafe fn _mm_maskz_multishift_epi64_epi8(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { + let multishift = _mm_multishift_epi64_epi8(a, b).as_i8x16(); + let zero = _mm_setzero_si128().as_i8x16(); + transmute(simd_select_bitmask(k, multishift, zero)) +} + +#[allow(improper_ctypes)] +extern "C" { + #[link_name = "llvm.x86.avx512.vpermi2var.qi.512"] + fn vpermi2b(a: i8x64, idx: i8x64, b: i8x64) -> i8x64; + #[link_name = "llvm.x86.avx512.vpermi2var.qi.256"] + fn vpermi2b256(a: i8x32, idx: i8x32, b: i8x32) -> i8x32; + #[link_name = "llvm.x86.avx512.vpermi2var.qi.128"] + fn vpermi2b128(a: i8x16, idx: i8x16, b: i8x16) -> i8x16; + + #[link_name = "llvm.x86.avx512.permvar.qi.512"] + fn vpermb(a: i8x64, idx: i8x64) -> i8x64; + #[link_name = "llvm.x86.avx512.permvar.qi.256"] + fn vpermb256(a: i8x32, idx: i8x32) -> i8x32; + #[link_name = "llvm.x86.avx512.permvar.qi.128"] + fn vpermb128(a: i8x16, idx: i8x16) -> i8x16; + + #[link_name = "llvm.x86.avx512.pmultishift.qb.512"] + fn vpmultishiftqb(a: i8x64, b: i8x64) -> i8x64; + #[link_name = "llvm.x86.avx512.pmultishift.qb.256"] + fn vpmultishiftqb256(a: i8x32, b: i8x32) -> i8x32; + #[link_name = "llvm.x86.avx512.pmultishift.qb.128"] + fn vpmultishiftqb128(a: i8x16, b: i8x16) -> i8x16; +} + +#[cfg(test)] +mod tests { + + use stdarch_test::simd_test; + + use crate::core_arch::x86::*; + + #[simd_test(enable = "avx512vbmi")] + unsafe fn test_mm512_permutex2var_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63); + #[rustfmt::skip] + let idx = _mm512_set_epi8(1, 1<<6, 2, 1<<6, 3, 1<<6, 4, 1<<6, 5, 1<<6, 6, 1<<6, 7, 1<<6, 8, 1<<6, + 9, 1<<6, 10, 1<<6, 11, 1<<6, 12, 1<<6, 13, 1<<6, 14, 1<<6, 15, 1<<6, 16, 1<<6, + 17, 1<<6, 18, 1<<6, 19, 1<<6, 20, 1<<6, 21, 1<<6, 22, 1<<6, 23, 1<<6, 24, 1<<6, + 25, 1<<6, 26, 1<<6, 27, 1<<6, 28, 1<<6, 29, 1<<6, 30, 1<<6, 31, 1<<6, 32, 1<<6); + let b = _mm512_set1_epi8(100); + let r = _mm512_permutex2var_epi8(a, idx, b); + #[rustfmt::skip] + let e = _mm512_set_epi8( + 62, 100, 61, 100, 60, 100, 59, 100, 58, 100, 57, 100, 56, 100, 55, 100, + 54, 100, 53, 100, 52, 100, 51, 100, 50, 100, 49, 100, 48, 100, 47, 100, + 46, 100, 45, 100, 44, 100, 43, 100, 42, 100, 41, 100, 40, 100, 39, 100, + 38, 100, 37, 100, 36, 100, 35, 100, 34, 100, 33, 100, 32, 100, 31, 100, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi")] + unsafe fn test_mm512_mask_permutex2var_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63); + #[rustfmt::skip] + let idx = _mm512_set_epi8(1, 1<<6, 2, 1<<6, 3, 1<<6, 4, 1<<6, 5, 1<<6, 6, 1<<6, 7, 1<<6, 8, 1<<6, + 9, 1<<6, 10, 1<<6, 11, 1<<6, 12, 1<<6, 13, 1<<6, 14, 1<<6, 15, 1<<6, 16, 1<<6, + 17, 1<<6, 18, 1<<6, 19, 1<<6, 20, 1<<6, 21, 1<<6, 22, 1<<6, 23, 1<<6, 24, 1<<6, + 25, 1<<6, 26, 1<<6, 27, 1<<6, 28, 1<<6, 29, 1<<6, 30, 1<<6, 31, 1<<6, 32, 1<<6); + let b = _mm512_set1_epi8(100); + let r = _mm512_mask_permutex2var_epi8(a, 0, idx, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_permutex2var_epi8( + a, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + idx, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8( + 62, 100, 61, 100, 60, 100, 59, 100, 58, 100, 57, 100, 56, 100, 55, 100, + 54, 100, 53, 100, 52, 100, 51, 100, 50, 100, 49, 100, 48, 100, 47, 100, + 46, 100, 45, 100, 44, 100, 43, 100, 42, 100, 41, 100, 40, 100, 39, 100, + 38, 100, 37, 100, 36, 100, 35, 100, 34, 100, 33, 100, 32, 100, 31, 100, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi")] + unsafe fn test_mm512_maskz_permutex2var_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63); + #[rustfmt::skip] + let idx = _mm512_set_epi8(1, 1<<6, 2, 1<<6, 3, 1<<6, 4, 1<<6, 5, 1<<6, 6, 1<<6, 7, 1<<6, 8, 1<<6, + 9, 1<<6, 10, 1<<6, 11, 1<<6, 12, 1<<6, 13, 1<<6, 14, 1<<6, 15, 1<<6, 16, 1<<6, + 17, 1<<6, 18, 1<<6, 19, 1<<6, 20, 1<<6, 21, 1<<6, 22, 1<<6, 23, 1<<6, 24, 1<<6, + 25, 1<<6, 26, 1<<6, 27, 1<<6, 28, 1<<6, 29, 1<<6, 30, 1<<6, 31, 1<<6, 32, 1<<6); + let b = _mm512_set1_epi8(100); + let r = _mm512_maskz_permutex2var_epi8(0, a, idx, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_permutex2var_epi8( + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + a, + idx, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8( + 62, 100, 61, 100, 60, 100, 59, 100, 58, 100, 57, 100, 56, 100, 55, 100, + 54, 100, 53, 100, 52, 100, 51, 100, 50, 100, 49, 100, 48, 100, 47, 100, + 46, 100, 45, 100, 44, 100, 43, 100, 42, 100, 41, 100, 40, 100, 39, 100, + 38, 100, 37, 100, 36, 100, 35, 100, 34, 100, 33, 100, 32, 100, 31, 100, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi")] + unsafe fn test_mm512_mask2_permutex2var_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63); + #[rustfmt::skip] + let idx = _mm512_set_epi8(1, 1<<6, 2, 1<<6, 3, 1<<6, 4, 1<<6, 5, 1<<6, 6, 1<<6, 7, 1<<6, 8, 1<<6, + 9, 1<<6, 10, 1<<6, 11, 1<<6, 12, 1<<6, 13, 1<<6, 14, 1<<6, 15, 1<<6, 16, 1<<6, + 17, 1<<6, 18, 1<<6, 19, 1<<6, 20, 1<<6, 21, 1<<6, 22, 1<<6, 23, 1<<6, 24, 1<<6, + 25, 1<<6, 26, 1<<6, 27, 1<<6, 28, 1<<6, 29, 1<<6, 30, 1<<6, 31, 1<<6, 32, 1<<6); + let b = _mm512_set1_epi8(100); + let r = _mm512_mask2_permutex2var_epi8(a, idx, 0, b); + assert_eq_m512i(r, idx); + let r = _mm512_mask2_permutex2var_epi8( + a, + idx, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + b, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8( + 62, 100, 61, 100, 60, 100, 59, 100, 58, 100, 57, 100, 56, 100, 55, 100, + 54, 100, 53, 100, 52, 100, 51, 100, 50, 100, 49, 100, 48, 100, 47, 100, + 46, 100, 45, 100, 44, 100, 43, 100, 42, 100, 41, 100, 40, 100, 39, 100, + 38, 100, 37, 100, 36, 100, 35, 100, 34, 100, 33, 100, 32, 100, 31, 100, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi,avx512vl")] + unsafe fn test_mm256_permutex2var_epi8() { + #[rustfmt::skip] + let a = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + #[rustfmt::skip] + let idx = _mm256_set_epi8(1, 1<<5, 2, 1<<5, 3, 1<<5, 4, 1<<5, 5, 1<<5, 6, 1<<5, 7, 1<<5, 8, 1<<5, + 9, 1<<5, 10, 1<<5, 11, 1<<5, 12, 1<<5, 13, 1<<5, 14, 1<<5, 15, 1<<5, 16, 1<<5); + let b = _mm256_set1_epi8(100); + let r = _mm256_permutex2var_epi8(a, idx, b); + #[rustfmt::skip] + let e = _mm256_set_epi8( + 30, 100, 29, 100, 28, 100, 27, 100, 26, 100, 25, 100, 24, 100, 23, 100, + 22, 100, 21, 100, 20, 100, 19, 100, 18, 100, 17, 100, 16, 100, 15, 100, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi,avx512vl")] + unsafe fn test_mm256_mask_permutex2var_epi8() { + #[rustfmt::skip] + let a = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + #[rustfmt::skip] + let idx = _mm256_set_epi8(1, 1<<5, 2, 1<<5, 3, 1<<5, 4, 1<<5, 5, 1<<5, 6, 1<<5, 7, 1<<5, 8, 1<<5, + 9, 1<<5, 10, 1<<5, 11, 1<<5, 12, 1<<5, 13, 1<<5, 14, 1<<5, 15, 1<<5, 16, 1<<5); + let b = _mm256_set1_epi8(100); + let r = _mm256_mask_permutex2var_epi8(a, 0, idx, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_permutex2var_epi8(a, 0b11111111_11111111_11111111_11111111, idx, b); + #[rustfmt::skip] + let e = _mm256_set_epi8( + 30, 100, 29, 100, 28, 100, 27, 100, 26, 100, 25, 100, 24, 100, 23, 100, + 22, 100, 21, 100, 20, 100, 19, 100, 18, 100, 17, 100, 16, 100, 15, 100, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi,avx512vl")] + unsafe fn test_mm256_maskz_permutex2var_epi8() { + #[rustfmt::skip] + let a = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + #[rustfmt::skip] + let idx = _mm256_set_epi8(1, 1<<5, 2, 1<<5, 3, 1<<5, 4, 1<<5, 5, 1<<5, 6, 1<<5, 7, 1<<5, 8, 1<<5, + 9, 1<<5, 10, 1<<5, 11, 1<<5, 12, 1<<5, 13, 1<<5, 14, 1<<5, 15, 1<<5, 16, 1<<5); + let b = _mm256_set1_epi8(100); + let r = _mm256_maskz_permutex2var_epi8(0, a, idx, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_permutex2var_epi8(0b11111111_11111111_11111111_11111111, a, idx, b); + #[rustfmt::skip] + let e = _mm256_set_epi8( + 30, 100, 29, 100, 28, 100, 27, 100, 26, 100, 25, 100, 24, 100, 23, 100, + 22, 100, 21, 100, 20, 100, 19, 100, 18, 100, 17, 100, 16, 100, 15, 100, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi,avx512vl")] + unsafe fn test_mm256_mask2_permutex2var_epi8() { + #[rustfmt::skip] + let a = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + #[rustfmt::skip] + let idx = _mm256_set_epi8(1, 1<<5, 2, 1<<5, 3, 1<<5, 4, 1<<5, 5, 1<<5, 6, 1<<5, 7, 1<<5, 8, 1<<5, + 9, 1<<5, 10, 1<<5, 11, 1<<5, 12, 1<<5, 13, 1<<5, 14, 1<<5, 15, 1<<5, 16, 1<<5); + let b = _mm256_set1_epi8(100); + let r = _mm256_mask2_permutex2var_epi8(a, idx, 0, b); + assert_eq_m256i(r, idx); + let r = _mm256_mask2_permutex2var_epi8(a, idx, 0b11111111_11111111_11111111_11111111, b); + #[rustfmt::skip] + let e = _mm256_set_epi8( + 30, 100, 29, 100, 28, 100, 27, 100, 26, 100, 25, 100, 24, 100, 23, 100, + 22, 100, 21, 100, 20, 100, 19, 100, 18, 100, 17, 100, 16, 100, 15, 100, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi,avx512vl")] + unsafe fn test_mm_permutex2var_epi8() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let idx = _mm_set_epi8(1, 1 << 4, 2, 1 << 4, 3, 1 << 4, 4, 1 << 4, 5, 1 << 4, 6, 1 << 4, 7, 1 << 4, 8, 1 << 4); + let b = _mm_set1_epi8(100); + let r = _mm_permutex2var_epi8(a, idx, b); + let e = _mm_set_epi8( + 14, 100, 13, 100, 12, 100, 11, 100, 10, 100, 9, 100, 8, 100, 7, 100, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi,avx512vl")] + unsafe fn test_mm_mask_permutex2var_epi8() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let idx = _mm_set_epi8(1, 1 << 4, 2, 1 << 4, 3, 1 << 4, 4, 1 << 4, 5, 1 << 4, 6, 1 << 4, 7, 1 << 4, 8, 1 << 4); + let b = _mm_set1_epi8(100); + let r = _mm_mask_permutex2var_epi8(a, 0, idx, b); + assert_eq_m128i(r, a); + let r = _mm_mask_permutex2var_epi8(a, 0b11111111_11111111, idx, b); + let e = _mm_set_epi8( + 14, 100, 13, 100, 12, 100, 11, 100, 10, 100, 9, 100, 8, 100, 7, 100, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi,avx512vl")] + unsafe fn test_mm_maskz_permutex2var_epi8() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let idx = _mm_set_epi8(1, 1 << 4, 2, 1 << 4, 3, 1 << 4, 4, 1 << 4, 5, 1 << 4, 6, 1 << 4, 7, 1 << 4, 8, 1 << 4); + let b = _mm_set1_epi8(100); + let r = _mm_maskz_permutex2var_epi8(0, a, idx, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_permutex2var_epi8(0b11111111_11111111, a, idx, b); + let e = _mm_set_epi8( + 14, 100, 13, 100, 12, 100, 11, 100, 10, 100, 9, 100, 8, 100, 7, 100, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi,avx512vl")] + unsafe fn test_mm_mask2_permutex2var_epi8() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let idx = _mm_set_epi8(1, 1 << 4, 2, 1 << 4, 3, 1 << 4, 4, 1 << 4, 5, 1 << 4, 6, 1 << 4, 7, 1 << 4, 8, 1 << 4); + let b = _mm_set1_epi8(100); + let r = _mm_mask2_permutex2var_epi8(a, idx, 0, b); + assert_eq_m128i(r, idx); + let r = _mm_mask2_permutex2var_epi8(a, idx, 0b11111111_11111111, b); + let e = _mm_set_epi8( + 14, 100, 13, 100, 12, 100, 11, 100, 10, 100, 9, 100, 8, 100, 7, 100, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi")] + unsafe fn test_mm512_permutexvar_epi8() { + let idx = _mm512_set1_epi8(1); + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63); + let r = _mm512_permutexvar_epi8(idx, a); + let e = _mm512_set1_epi8(62); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi")] + unsafe fn test_mm512_mask_permutexvar_epi8() { + let idx = _mm512_set1_epi8(1); + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63); + let r = _mm512_mask_permutexvar_epi8(a, 0, idx, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_permutexvar_epi8( + a, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + idx, + a, + ); + let e = _mm512_set1_epi8(62); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi")] + unsafe fn test_mm512_maskz_permutexvar_epi8() { + let idx = _mm512_set1_epi8(1); + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63); + let r = _mm512_maskz_permutexvar_epi8(0, idx, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_permutexvar_epi8( + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + idx, + a, + ); + let e = _mm512_set1_epi8(62); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi,avx512vl")] + unsafe fn test_mm256_permutexvar_epi8() { + let idx = _mm256_set1_epi8(1); + #[rustfmt::skip] + let a = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let r = _mm256_permutexvar_epi8(idx, a); + let e = _mm256_set1_epi8(30); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi,avx512vl")] + unsafe fn test_mm256_mask_permutexvar_epi8() { + let idx = _mm256_set1_epi8(1); + #[rustfmt::skip] + let a = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let r = _mm256_mask_permutexvar_epi8(a, 0, idx, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_permutexvar_epi8(a, 0b11111111_11111111_11111111_11111111, idx, a); + let e = _mm256_set1_epi8(30); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi,avx512vl")] + unsafe fn test_mm256_maskz_permutexvar_epi8() { + let idx = _mm256_set1_epi8(1); + #[rustfmt::skip] + let a = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let r = _mm256_maskz_permutexvar_epi8(0, idx, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_permutexvar_epi8(0b11111111_11111111_11111111_11111111, idx, a); + let e = _mm256_set1_epi8(30); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi,avx512vl")] + unsafe fn test_mm_permutexvar_epi8() { + let idx = _mm_set1_epi8(1); + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm_permutexvar_epi8(idx, a); + let e = _mm_set1_epi8(14); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi,avx512vl")] + unsafe fn test_mm_mask_permutexvar_epi8() { + let idx = _mm_set1_epi8(1); + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm_mask_permutexvar_epi8(a, 0, idx, a); + assert_eq_m128i(r, a); + let r = _mm_mask_permutexvar_epi8(a, 0b11111111_11111111, idx, a); + let e = _mm_set1_epi8(14); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi,avx512vl")] + unsafe fn test_mm_maskz_permutexvar_epi8() { + let idx = _mm_set1_epi8(1); + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm_maskz_permutexvar_epi8(0, idx, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_permutexvar_epi8(0b11111111_11111111, idx, a); + let e = _mm_set1_epi8(14); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi")] + unsafe fn test_mm512_multishift_epi64_epi8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(1); + let r = _mm512_multishift_epi64_epi8(a, b); + let e = _mm512_set1_epi8(1 << 7); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi")] + unsafe fn test_mm512_mask_multishift_epi64_epi8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(1); + let r = _mm512_mask_multishift_epi64_epi8(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_multishift_epi64_epi8( + a, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + a, + b, + ); + let e = _mm512_set1_epi8(1 << 7); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi")] + unsafe fn test_mm512_maskz_multishift_epi64_epi8() { + let a = _mm512_set1_epi8(1); + let b = _mm512_set1_epi8(1); + let r = _mm512_maskz_multishift_epi64_epi8(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_multishift_epi64_epi8( + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + a, + b, + ); + let e = _mm512_set1_epi8(1 << 7); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi,avx512vl")] + unsafe fn test_mm256_multishift_epi64_epi8() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(1); + let r = _mm256_multishift_epi64_epi8(a, b); + let e = _mm256_set1_epi8(1 << 7); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi,avx512vl")] + unsafe fn test_mm256_mask_multishift_epi64_epi8() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(1); + let r = _mm256_mask_multishift_epi64_epi8(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_multishift_epi64_epi8(a, 0b11111111_11111111_11111111_11111111, a, b); + let e = _mm256_set1_epi8(1 << 7); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi,avx512vl")] + unsafe fn test_mm256_maskz_multishift_epi64_epi8() { + let a = _mm256_set1_epi8(1); + let b = _mm256_set1_epi8(1); + let r = _mm256_maskz_multishift_epi64_epi8(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_multishift_epi64_epi8(0b11111111_11111111_11111111_11111111, a, b); + let e = _mm256_set1_epi8(1 << 7); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi,avx512vl")] + unsafe fn test_mm_multishift_epi64_epi8() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(1); + let r = _mm_multishift_epi64_epi8(a, b); + let e = _mm_set1_epi8(1 << 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi,avx512vl")] + unsafe fn test_mm_mask_multishift_epi64_epi8() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(1); + let r = _mm_mask_multishift_epi64_epi8(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_multishift_epi64_epi8(a, 0b11111111_11111111, a, b); + let e = _mm_set1_epi8(1 << 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi,avx512vl")] + unsafe fn test_mm_maskz_multishift_epi64_epi8() { + let a = _mm_set1_epi8(1); + let b = _mm_set1_epi8(1); + let r = _mm_maskz_multishift_epi64_epi8(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_multishift_epi64_epi8(0b11111111_11111111, a, b); + let e = _mm_set1_epi8(1 << 7); + assert_eq_m128i(r, e); + } +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vbmi2.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vbmi2.rs new file mode 100644 index 000000000..1c81840ba --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vbmi2.rs @@ -0,0 +1,4121 @@ +use crate::{ + arch::asm, + core_arch::{simd::*, simd_llvm::*, x86::*}, +}; + +#[cfg(test)] +use stdarch_test::assert_instr; + +/// Load contiguous active 16-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_expandloadu_epi16) +#[inline] +#[target_feature(enable = "avx512f,avx512bw,avx512vbmi2")] +pub unsafe fn _mm512_mask_expandloadu_epi16( + src: __m512i, + k: __mmask32, + mem_addr: *const i16, +) -> __m512i { + let mut dst: __m512i = src; + asm!( + vpl!("vpexpandw {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 16-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_expandloadu_epi16) +#[inline] +#[target_feature(enable = "avx512f,avx512bw,avx512vbmi2")] +pub unsafe fn _mm512_maskz_expandloadu_epi16(k: __mmask32, mem_addr: *const i16) -> __m512i { + let mut dst: __m512i; + asm!( + vpl!("vpexpandw {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 16-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_expandloadu_epi16) +#[inline] +#[target_feature(enable = "avx512f,avx512vbmi2,avx512vl,avx")] +pub unsafe fn _mm256_mask_expandloadu_epi16( + src: __m256i, + k: __mmask16, + mem_addr: *const i16, +) -> __m256i { + let mut dst: __m256i = src; + asm!( + vpl!("vpexpandw {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 16-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_expandloadu_epi16) +#[inline] +#[target_feature(enable = "avx512f,avx512vbmi2,avx512vl,avx")] +pub unsafe fn _mm256_maskz_expandloadu_epi16(k: __mmask16, mem_addr: *const i16) -> __m256i { + let mut dst: __m256i; + asm!( + vpl!("vpexpandw {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 16-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_expandloadu_epi16) +#[inline] +#[target_feature(enable = "avx512f,avx512vbmi2,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_expandloadu_epi16( + src: __m128i, + k: __mmask8, + mem_addr: *const i16, +) -> __m128i { + let mut dst: __m128i = src; + asm!( + vpl!("vpexpandw {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 16-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_expandloadu_epi16) +#[inline] +#[target_feature(enable = "avx512f,avx512vbmi2,avx512vl,avx,sse")] +pub unsafe fn _mm_maskz_expandloadu_epi16(k: __mmask8, mem_addr: *const i16) -> __m128i { + let mut dst: __m128i; + asm!( + vpl!("vpexpandw {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 8-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_expandloadu_epi8) +#[inline] +#[target_feature(enable = "avx512f,avx512bw,avx512vbmi2")] +pub unsafe fn _mm512_mask_expandloadu_epi8( + src: __m512i, + k: __mmask64, + mem_addr: *const i8, +) -> __m512i { + let mut dst: __m512i = src; + asm!( + vpl!("vpexpandb {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 8-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_expandloadu_epi8) +#[inline] +#[target_feature(enable = "avx512f,avx512bw,avx512vbmi2")] +pub unsafe fn _mm512_maskz_expandloadu_epi8(k: __mmask64, mem_addr: *const i8) -> __m512i { + let mut dst: __m512i; + asm!( + vpl!("vpexpandb {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(zmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 8-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_expandloadu_epi8) +#[inline] +#[target_feature(enable = "avx512f,avx512bw,avx512vbmi2,avx512vl,avx")] +pub unsafe fn _mm256_mask_expandloadu_epi8( + src: __m256i, + k: __mmask32, + mem_addr: *const i8, +) -> __m256i { + let mut dst: __m256i = src; + asm!( + vpl!("vpexpandb {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 8-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_expandloadu_epi8) +#[inline] +#[target_feature(enable = "avx512f,avx512bw,avx512vbmi2,avx512vl,avx")] +pub unsafe fn _mm256_maskz_expandloadu_epi8(k: __mmask32, mem_addr: *const i8) -> __m256i { + let mut dst: __m256i; + asm!( + vpl!("vpexpandb {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(ymm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 8-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_expandloadu_epi8) +#[inline] +#[target_feature(enable = "avx512f,avx512vbmi2,avx512vl,avx,sse")] +pub unsafe fn _mm_mask_expandloadu_epi8( + src: __m128i, + k: __mmask16, + mem_addr: *const i8, +) -> __m128i { + let mut dst: __m128i = src; + asm!( + vpl!("vpexpandb {dst}{{{k}}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = inout(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Load contiguous active 8-bit integers from unaligned memory at mem_addr (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_expandloadu_epi8) +#[inline] +#[target_feature(enable = "avx512f,avx512vbmi2,avx512vl,avx,sse")] +pub unsafe fn _mm_maskz_expandloadu_epi8(k: __mmask16, mem_addr: *const i8) -> __m128i { + let mut dst: __m128i; + asm!( + vpl!("vpexpandb {dst}{{{k}}} {{z}}"), + p = in(reg) mem_addr, + k = in(kreg) k, + dst = out(xmm_reg) dst, + options(pure, readonly, nostack) + ); + dst +} + +/// Contiguously store the active 16-bit integers in a (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_compressstoreu_epi16) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpcompressw))] +pub unsafe fn _mm512_mask_compressstoreu_epi16(base_addr: *mut u8, k: __mmask32, a: __m512i) { + vcompressstorew(base_addr as *mut _, a.as_i16x32(), k) +} + +/// Contiguously store the active 16-bit integers in a (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_compressstoreu_epi16) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressw))] +pub unsafe fn _mm256_mask_compressstoreu_epi16(base_addr: *mut u8, k: __mmask16, a: __m256i) { + vcompressstorew256(base_addr as *mut _, a.as_i16x16(), k) +} + +/// Contiguously store the active 16-bit integers in a (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_compressstoreu_epi16) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressw))] +pub unsafe fn _mm_mask_compressstoreu_epi16(base_addr: *mut u8, k: __mmask8, a: __m128i) { + vcompressstorew128(base_addr as *mut _, a.as_i16x8(), k) +} + +/// Contiguously store the active 8-bit integers in a (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_compressstoreu_epi8) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpcompressb))] +pub unsafe fn _mm512_mask_compressstoreu_epi8(base_addr: *mut u8, k: __mmask64, a: __m512i) { + vcompressstoreb(base_addr as *mut _, a.as_i8x64(), k) +} + +/// Contiguously store the active 8-bit integers in a (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_compressstoreu_epi8) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressb))] +pub unsafe fn _mm256_mask_compressstoreu_epi8(base_addr: *mut u8, k: __mmask32, a: __m256i) { + vcompressstoreb256(base_addr as *mut _, a.as_i8x32(), k) +} + +/// Contiguously store the active 8-bit integers in a (those with their respective bit set in writemask k) to unaligned memory at base_addr. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_compressstoreu_epi8) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressb))] +pub unsafe fn _mm_mask_compressstoreu_epi8(base_addr: *mut u8, k: __mmask16, a: __m128i) { + vcompressstoreb128(base_addr as *mut _, a.as_i8x16(), k) +} + +/// Contiguously store the active 16-bit integers in a (those with their respective bit set in writemask k) to dst, and pass through the remaining elements from src. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_compress_epi16&expand=1192) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpcompressw))] +pub unsafe fn _mm512_mask_compress_epi16(src: __m512i, k: __mmask32, a: __m512i) -> __m512i { + transmute(vpcompressw(a.as_i16x32(), src.as_i16x32(), k)) +} + +/// Contiguously store the active 16-bit integers in a (those with their respective bit set in zeromask k) to dst, and set the remaining elements to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_compress_epi16&expand=1193) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpcompressw))] +pub unsafe fn _mm512_maskz_compress_epi16(k: __mmask32, a: __m512i) -> __m512i { + transmute(vpcompressw( + a.as_i16x32(), + _mm512_setzero_si512().as_i16x32(), + k, + )) +} + +/// Contiguously store the active 16-bit integers in a (those with their respective bit set in writemask k) to dst, and pass through the remaining elements from src. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_compress_epi16&expand=1190) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressw))] +pub unsafe fn _mm256_mask_compress_epi16(src: __m256i, k: __mmask16, a: __m256i) -> __m256i { + transmute(vpcompressw256(a.as_i16x16(), src.as_i16x16(), k)) +} + +/// Contiguously store the active 16-bit integers in a (those with their respective bit set in zeromask k) to dst, and set the remaining elements to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_compress_epi16&expand=1191) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressw))] +pub unsafe fn _mm256_maskz_compress_epi16(k: __mmask16, a: __m256i) -> __m256i { + transmute(vpcompressw256( + a.as_i16x16(), + _mm256_setzero_si256().as_i16x16(), + k, + )) +} + +/// Contiguously store the active 16-bit integers in a (those with their respective bit set in writemask k) to dst, and pass through the remaining elements from src. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_compress_epi16&expand=1188) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressw))] +pub unsafe fn _mm_mask_compress_epi16(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpcompressw128(a.as_i16x8(), src.as_i16x8(), k)) +} + +/// Contiguously store the active 16-bit integers in a (those with their respective bit set in zeromask k) to dst, and set the remaining elements to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_compress_epi16&expand=1189) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressw))] +pub unsafe fn _mm_maskz_compress_epi16(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpcompressw128( + a.as_i16x8(), + _mm_setzero_si128().as_i16x8(), + k, + )) +} + +/// Contiguously store the active 8-bit integers in a (those with their respective bit set in writemask k) to dst, and pass through the remaining elements from src. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_compress_epi8&expand=1210) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpcompressb))] +pub unsafe fn _mm512_mask_compress_epi8(src: __m512i, k: __mmask64, a: __m512i) -> __m512i { + transmute(vpcompressb(a.as_i8x64(), src.as_i8x64(), k)) +} + +/// Contiguously store the active 8-bit integers in a (those with their respective bit set in zeromask k) to dst, and set the remaining elements to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_compress_epi8&expand=1211) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpcompressb))] +pub unsafe fn _mm512_maskz_compress_epi8(k: __mmask64, a: __m512i) -> __m512i { + transmute(vpcompressb( + a.as_i8x64(), + _mm512_setzero_si512().as_i8x64(), + k, + )) +} + +/// Contiguously store the active 8-bit integers in a (those with their respective bit set in writemask k) to dst, and pass through the remaining elements from src. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_compress_epi8&expand=1208) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressb))] +pub unsafe fn _mm256_mask_compress_epi8(src: __m256i, k: __mmask32, a: __m256i) -> __m256i { + transmute(vpcompressb256(a.as_i8x32(), src.as_i8x32(), k)) +} + +/// Contiguously store the active 8-bit integers in a (those with their respective bit set in zeromask k) to dst, and set the remaining elements to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_compress_epi8&expand=1209) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressb))] +pub unsafe fn _mm256_maskz_compress_epi8(k: __mmask32, a: __m256i) -> __m256i { + transmute(vpcompressb256( + a.as_i8x32(), + _mm256_setzero_si256().as_i8x32(), + k, + )) +} + +/// Contiguously store the active 8-bit integers in a (those with their respective bit set in writemask k) to dst, and pass through the remaining elements from src. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_compress_epi8&expand=1206) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressb))] +pub unsafe fn _mm_mask_compress_epi8(src: __m128i, k: __mmask16, a: __m128i) -> __m128i { + transmute(vpcompressb128(a.as_i8x16(), src.as_i8x16(), k)) +} + +/// Contiguously store the active 8-bit integers in a (those with their respective bit set in zeromask k) to dst, and set the remaining elements to zero. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_compress_epi8&expand=1207) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpcompressb))] +pub unsafe fn _mm_maskz_compress_epi8(k: __mmask16, a: __m128i) -> __m128i { + transmute(vpcompressb128( + a.as_i8x16(), + _mm_setzero_si128().as_i8x16(), + k, + )) +} + +/// Load contiguous active 16-bit integers from a (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_expand_epi16&expand=2310) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpexpandw))] +pub unsafe fn _mm512_mask_expand_epi16(src: __m512i, k: __mmask32, a: __m512i) -> __m512i { + transmute(vpexpandw(a.as_i16x32(), src.as_i16x32(), k)) +} + +/// Load contiguous active 16-bit integers from a (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_expand_epi16&expand=2311) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpexpandw))] +pub unsafe fn _mm512_maskz_expand_epi16(k: __mmask32, a: __m512i) -> __m512i { + transmute(vpexpandw( + a.as_i16x32(), + _mm512_setzero_si512().as_i16x32(), + k, + )) +} + +/// Load contiguous active 16-bit integers from a (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_expand_epi16&expand=2308) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpexpandw))] +pub unsafe fn _mm256_mask_expand_epi16(src: __m256i, k: __mmask16, a: __m256i) -> __m256i { + transmute(vpexpandw256(a.as_i16x16(), src.as_i16x16(), k)) +} + +/// Load contiguous active 16-bit integers from a (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_expand_epi16&expand=2309) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpexpandw))] +pub unsafe fn _mm256_maskz_expand_epi16(k: __mmask16, a: __m256i) -> __m256i { + transmute(vpexpandw256( + a.as_i16x16(), + _mm256_setzero_si256().as_i16x16(), + k, + )) +} + +/// Load contiguous active 16-bit integers from a (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_expand_epi16&expand=2306) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpexpandw))] +pub unsafe fn _mm_mask_expand_epi16(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(vpexpandw128(a.as_i16x8(), src.as_i16x8(), k)) +} + +/// Load contiguous active 16-bit integers from a (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_expand_epi16&expand=2307) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpexpandw))] +pub unsafe fn _mm_maskz_expand_epi16(k: __mmask8, a: __m128i) -> __m128i { + transmute(vpexpandw128( + a.as_i16x8(), + _mm_setzero_si128().as_i16x8(), + k, + )) +} + +/// Load contiguous active 8-bit integers from a (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_expand_epi8&expand=2328) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpexpandb))] +pub unsafe fn _mm512_mask_expand_epi8(src: __m512i, k: __mmask64, a: __m512i) -> __m512i { + transmute(vpexpandb(a.as_i8x64(), src.as_i8x64(), k)) +} + +/// Load contiguous active 8-bit integers from a (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_expand_epi8&expand=2329) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpexpandb))] +pub unsafe fn _mm512_maskz_expand_epi8(k: __mmask64, a: __m512i) -> __m512i { + transmute(vpexpandb( + a.as_i8x64(), + _mm512_setzero_si512().as_i8x64(), + k, + )) +} + +/// Load contiguous active 8-bit integers from a (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_expand_epi8&expand=2326) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpexpandb))] +pub unsafe fn _mm256_mask_expand_epi8(src: __m256i, k: __mmask32, a: __m256i) -> __m256i { + transmute(vpexpandb256(a.as_i8x32(), src.as_i8x32(), k)) +} + +/// Load contiguous active 8-bit integers from a (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_expand_epi8&expand=2327) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpexpandb))] +pub unsafe fn _mm256_maskz_expand_epi8(k: __mmask32, a: __m256i) -> __m256i { + transmute(vpexpandb256( + a.as_i8x32(), + _mm256_setzero_si256().as_i8x32(), + k, + )) +} + +/// Load contiguous active 8-bit integers from a (those with their respective bit set in mask k), and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_expand_epi8&expand=2324) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpexpandb))] +pub unsafe fn _mm_mask_expand_epi8(src: __m128i, k: __mmask16, a: __m128i) -> __m128i { + transmute(vpexpandb128(a.as_i8x16(), src.as_i8x16(), k)) +} + +/// Load contiguous active 8-bit integers from a (those with their respective bit set in mask k), and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_expand_epi8&expand=2325) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpexpandb))] +pub unsafe fn _mm_maskz_expand_epi8(k: __mmask16, a: __m128i) -> __m128i { + transmute(vpexpandb128( + a.as_i8x16(), + _mm_setzero_si128().as_i8x16(), + k, + )) +} + +/// Concatenate packed 64-bit integers in a and b producing an intermediate 128-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 64-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shldv_epi64&expand=5087) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldvq))] +pub unsafe fn _mm512_shldv_epi64(a: __m512i, b: __m512i, c: __m512i) -> __m512i { + transmute(vpshldvq(a.as_i64x8(), b.as_i64x8(), c.as_i64x8())) +} + +/// Concatenate packed 64-bit integers in a and b producing an intermediate 128-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 64-bits in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shldv_epi64&expand=5085) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldvq))] +pub unsafe fn _mm512_mask_shldv_epi64(a: __m512i, k: __mmask8, b: __m512i, c: __m512i) -> __m512i { + let shf = _mm512_shldv_epi64(a, b, c).as_i64x8(); + transmute(simd_select_bitmask(k, shf, a.as_i64x8())) +} + +/// Concatenate packed 64-bit integers in a and b producing an intermediate 128-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 64-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shldv_epi64&expand=5086) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldvq))] +pub unsafe fn _mm512_maskz_shldv_epi64(k: __mmask8, a: __m512i, b: __m512i, c: __m512i) -> __m512i { + let shf = _mm512_shldv_epi64(a, b, c).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 64-bit integers in a and b producing an intermediate 128-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 64-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shldv_epi64&expand=5084) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldvq))] +pub unsafe fn _mm256_shldv_epi64(a: __m256i, b: __m256i, c: __m256i) -> __m256i { + transmute(vpshldvq256(a.as_i64x4(), b.as_i64x4(), c.as_i64x4())) +} + +/// Concatenate packed 64-bit integers in a and b producing an intermediate 128-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 64-bits in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shldv_epi64&expand=5082) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldvq))] +pub unsafe fn _mm256_mask_shldv_epi64(a: __m256i, k: __mmask8, b: __m256i, c: __m256i) -> __m256i { + let shf = _mm256_shldv_epi64(a, b, c).as_i64x4(); + transmute(simd_select_bitmask(k, shf, a.as_i64x4())) +} + +/// Concatenate packed 64-bit integers in a and b producing an intermediate 128-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 64-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shldv_epi64&expand=5083) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldvq))] +pub unsafe fn _mm256_maskz_shldv_epi64(k: __mmask8, a: __m256i, b: __m256i, c: __m256i) -> __m256i { + let shf = _mm256_shldv_epi64(a, b, c).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 64-bit integers in a and b producing an intermediate 128-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 64-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shldv_epi64&expand=5081) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldvq))] +pub unsafe fn _mm_shldv_epi64(a: __m128i, b: __m128i, c: __m128i) -> __m128i { + transmute(vpshldvq128(a.as_i64x2(), b.as_i64x2(), c.as_i64x2())) +} + +/// Concatenate packed 64-bit integers in a and b producing an intermediate 128-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 64-bits in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shldv_epi64&expand=5079) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldvq))] +pub unsafe fn _mm_mask_shldv_epi64(a: __m128i, k: __mmask8, b: __m128i, c: __m128i) -> __m128i { + let shf = _mm_shldv_epi64(a, b, c).as_i64x2(); + transmute(simd_select_bitmask(k, shf, a.as_i64x2())) +} + +/// Concatenate packed 64-bit integers in a and b producing an intermediate 128-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 64-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shldv_epi64&expand=5080) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldvq))] +pub unsafe fn _mm_maskz_shldv_epi64(k: __mmask8, a: __m128i, b: __m128i, c: __m128i) -> __m128i { + let shf = _mm_shldv_epi64(a, b, c).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 32-bit integers in a and b producing an intermediate 64-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 32-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shldv_epi32&expand=5078) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldvd))] +pub unsafe fn _mm512_shldv_epi32(a: __m512i, b: __m512i, c: __m512i) -> __m512i { + transmute(vpshldvd(a.as_i32x16(), b.as_i32x16(), c.as_i32x16())) +} + +/// Concatenate packed 32-bit integers in a and b producing an intermediate 64-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 32-bits in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shldv_epi32&expand=5076) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldvd))] +pub unsafe fn _mm512_mask_shldv_epi32(a: __m512i, k: __mmask16, b: __m512i, c: __m512i) -> __m512i { + let shf = _mm512_shldv_epi32(a, b, c).as_i32x16(); + transmute(simd_select_bitmask(k, shf, a.as_i32x16())) +} + +/// Concatenate packed 32-bit integers in a and b producing an intermediate 64-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 32-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shldv_epi32&expand=5077) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldvd))] +pub unsafe fn _mm512_maskz_shldv_epi32( + k: __mmask16, + a: __m512i, + b: __m512i, + c: __m512i, +) -> __m512i { + let shf = _mm512_shldv_epi32(a, b, c).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 32-bit integers in a and b producing an intermediate 64-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 32-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shldv_epi32&expand=5075) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldvd))] +pub unsafe fn _mm256_shldv_epi32(a: __m256i, b: __m256i, c: __m256i) -> __m256i { + transmute(vpshldvd256(a.as_i32x8(), b.as_i32x8(), c.as_i32x8())) +} + +/// Concatenate packed 32-bit integers in a and b producing an intermediate 64-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 32-bits in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shldv_epi32&expand=5073) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldvd))] +pub unsafe fn _mm256_mask_shldv_epi32(a: __m256i, k: __mmask8, b: __m256i, c: __m256i) -> __m256i { + let shf = _mm256_shldv_epi32(a, b, c).as_i32x8(); + transmute(simd_select_bitmask(k, shf, a.as_i32x8())) +} + +/// Concatenate packed 32-bit integers in a and b producing an intermediate 64-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 32-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shldv_epi32&expand=5074) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldvd))] +pub unsafe fn _mm256_maskz_shldv_epi32(k: __mmask8, a: __m256i, b: __m256i, c: __m256i) -> __m256i { + let shf = _mm256_shldv_epi32(a, b, c).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 32-bit integers in a and b producing an intermediate 64-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 32-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shldv_epi32&expand=5072) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldvd))] +pub unsafe fn _mm_shldv_epi32(a: __m128i, b: __m128i, c: __m128i) -> __m128i { + transmute(vpshldvd128(a.as_i32x4(), b.as_i32x4(), c.as_i32x4())) +} + +/// Concatenate packed 32-bit integers in a and b producing an intermediate 64-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 32-bits in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shldv_epi32&expand=5070) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldvd))] +pub unsafe fn _mm_mask_shldv_epi32(a: __m128i, k: __mmask8, b: __m128i, c: __m128i) -> __m128i { + let shf = _mm_shldv_epi32(a, b, c).as_i32x4(); + transmute(simd_select_bitmask(k, shf, a.as_i32x4())) +} + +/// Concatenate packed 32-bit integers in a and b producing an intermediate 64-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 32-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shldv_epi32&expand=5071) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldvd))] +pub unsafe fn _mm_maskz_shldv_epi32(k: __mmask8, a: __m128i, b: __m128i, c: __m128i) -> __m128i { + let shf = _mm_shldv_epi32(a, b, c).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 16-bit integers in a and b producing an intermediate 32-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 16-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shldv_epi16&expand=5069) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldvw))] +pub unsafe fn _mm512_shldv_epi16(a: __m512i, b: __m512i, c: __m512i) -> __m512i { + transmute(vpshldvw(a.as_i16x32(), b.as_i16x32(), c.as_i16x32())) +} + +/// Concatenate packed 16-bit integers in a and b producing an intermediate 32-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 16-bits in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shldv_epi16&expand=5067) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldvw))] +pub unsafe fn _mm512_mask_shldv_epi16(a: __m512i, k: __mmask32, b: __m512i, c: __m512i) -> __m512i { + let shf = _mm512_shldv_epi16(a, b, c).as_i16x32(); + transmute(simd_select_bitmask(k, shf, a.as_i16x32())) +} + +/// Concatenate packed 16-bit integers in a and b producing an intermediate 32-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 16-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shldv_epi16&expand=5068) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldvw))] +pub unsafe fn _mm512_maskz_shldv_epi16( + k: __mmask32, + a: __m512i, + b: __m512i, + c: __m512i, +) -> __m512i { + let shf = _mm512_shldv_epi16(a, b, c).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 16-bit integers in a and b producing an intermediate 32-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 16-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shldv_epi16&expand=5066) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldvw))] +pub unsafe fn _mm256_shldv_epi16(a: __m256i, b: __m256i, c: __m256i) -> __m256i { + transmute(vpshldvw256(a.as_i16x16(), b.as_i16x16(), c.as_i16x16())) +} + +/// Concatenate packed 16-bit integers in a and b producing an intermediate 32-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 16-bits in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shldv_epi16&expand=5064) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldvw))] +pub unsafe fn _mm256_mask_shldv_epi16(a: __m256i, k: __mmask16, b: __m256i, c: __m256i) -> __m256i { + let shf = _mm256_shldv_epi16(a, b, c).as_i16x16(); + transmute(simd_select_bitmask(k, shf, a.as_i16x16())) +} + +/// Concatenate packed 16-bit integers in a and b producing an intermediate 32-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 16-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shldv_epi16&expand=5065) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldvw))] +pub unsafe fn _mm256_maskz_shldv_epi16( + k: __mmask16, + a: __m256i, + b: __m256i, + c: __m256i, +) -> __m256i { + let shf = _mm256_shldv_epi16(a, b, c).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 16-bit integers in a and b producing an intermediate 32-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 16-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shldv_epi16&expand=5063) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldvw))] +pub unsafe fn _mm_shldv_epi16(a: __m128i, b: __m128i, c: __m128i) -> __m128i { + transmute(vpshldvw128(a.as_i16x8(), b.as_i16x8(), c.as_i16x8())) +} + +/// Concatenate packed 16-bit integers in a and b producing an intermediate 32-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 16-bits in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shldv_epi16&expand=5061) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldvw))] +pub unsafe fn _mm_mask_shldv_epi16(a: __m128i, k: __mmask8, b: __m128i, c: __m128i) -> __m128i { + let shf = _mm_shldv_epi16(a, b, c).as_i16x8(); + transmute(simd_select_bitmask(k, shf, a.as_i16x8())) +} + +/// Concatenate packed 16-bit integers in a and b producing an intermediate 32-bit result. Shift the result left by the amount specified in the corresponding element of c, and store the upper 16-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shldv_epi16&expand=5062) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldvw))] +pub unsafe fn _mm_maskz_shldv_epi16(k: __mmask8, a: __m128i, b: __m128i, c: __m128i) -> __m128i { + let shf = _mm_shldv_epi16(a, b, c).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 64-bit integers in b and a producing an intermediate 128-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 64-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shrdv_epi64&expand=5141) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshrdvq))] +pub unsafe fn _mm512_shrdv_epi64(a: __m512i, b: __m512i, c: __m512i) -> __m512i { + transmute(vpshrdvq(a.as_i64x8(), b.as_i64x8(), c.as_i64x8())) +} + +/// Concatenate packed 64-bit integers in b and a producing an intermediate 128-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 64-bits in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shrdv_epi64&expand=5139) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshrdvq))] +pub unsafe fn _mm512_mask_shrdv_epi64(a: __m512i, k: __mmask8, b: __m512i, c: __m512i) -> __m512i { + let shf = _mm512_shrdv_epi64(a, b, c).as_i64x8(); + transmute(simd_select_bitmask(k, shf, a.as_i64x8())) +} + +/// Concatenate packed 64-bit integers in b and a producing an intermediate 128-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 64-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shrdv_epi64&expand=5140) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshrdvq))] +pub unsafe fn _mm512_maskz_shrdv_epi64(k: __mmask8, a: __m512i, b: __m512i, c: __m512i) -> __m512i { + let shf = _mm512_shrdv_epi64(a, b, c).as_i64x8(); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 64-bit integers in b and a producing an intermediate 128-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 64-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shrdv_epi64&expand=5138) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshrdvq))] +pub unsafe fn _mm256_shrdv_epi64(a: __m256i, b: __m256i, c: __m256i) -> __m256i { + transmute(vpshrdvq256(a.as_i64x4(), b.as_i64x4(), c.as_i64x4())) +} + +/// Concatenate packed 64-bit integers in b and a producing an intermediate 128-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 64-bits in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shrdv_epi64&expand=5136) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshrdvq))] +pub unsafe fn _mm256_mask_shrdv_epi64(a: __m256i, k: __mmask8, b: __m256i, c: __m256i) -> __m256i { + let shf = _mm256_shrdv_epi64(a, b, c).as_i64x4(); + transmute(simd_select_bitmask(k, shf, a.as_i64x4())) +} + +/// Concatenate packed 64-bit integers in b and a producing an intermediate 128-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 64-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shrdv_epi64&expand=5137) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshrdvq))] +pub unsafe fn _mm256_maskz_shrdv_epi64(k: __mmask8, a: __m256i, b: __m256i, c: __m256i) -> __m256i { + let shf = _mm256_shrdv_epi64(a, b, c).as_i64x4(); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 64-bit integers in b and a producing an intermediate 128-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 64-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shrdv_epi64&expand=5135) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshrdvq))] +pub unsafe fn _mm_shrdv_epi64(a: __m128i, b: __m128i, c: __m128i) -> __m128i { + transmute(vpshrdvq128(a.as_i64x2(), b.as_i64x2(), c.as_i64x2())) +} + +/// Concatenate packed 64-bit integers in b and a producing an intermediate 128-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 64-bits in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shrdv_epi64&expand=5133) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshrdvq))] +pub unsafe fn _mm_mask_shrdv_epi64(a: __m128i, k: __mmask8, b: __m128i, c: __m128i) -> __m128i { + let shf = _mm_shrdv_epi64(a, b, c).as_i64x2(); + transmute(simd_select_bitmask(k, shf, a.as_i64x2())) +} + +/// Concatenate packed 64-bit integers in b and a producing an intermediate 128-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 64-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shrdv_epi64&expand=5134) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshrdvq))] +pub unsafe fn _mm_maskz_shrdv_epi64(k: __mmask8, a: __m128i, b: __m128i, c: __m128i) -> __m128i { + let shf = _mm_shrdv_epi64(a, b, c).as_i64x2(); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 32-bit integers in b and a producing an intermediate 64-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 32-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shrdv_epi32&expand=5132) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshrdvd))] +pub unsafe fn _mm512_shrdv_epi32(a: __m512i, b: __m512i, c: __m512i) -> __m512i { + transmute(vpshrdvd(a.as_i32x16(), b.as_i32x16(), c.as_i32x16())) +} + +/// Concatenate packed 32-bit integers in b and a producing an intermediate 64-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 32-bits in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shrdv_epi32&expand=5130) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshrdvd))] +pub unsafe fn _mm512_mask_shrdv_epi32(a: __m512i, k: __mmask16, b: __m512i, c: __m512i) -> __m512i { + let shf = _mm512_shrdv_epi32(a, b, c).as_i32x16(); + transmute(simd_select_bitmask(k, shf, a.as_i32x16())) +} + +/// Concatenate packed 32-bit integers in b and a producing an intermediate 64-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 32-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shrdv_epi32&expand=5131) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshrdvd))] +pub unsafe fn _mm512_maskz_shrdv_epi32( + k: __mmask16, + a: __m512i, + b: __m512i, + c: __m512i, +) -> __m512i { + let shf = _mm512_shrdv_epi32(a, b, c).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 32-bit integers in b and a producing an intermediate 64-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 32-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shrdv_epi32&expand=5129) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshrdvd))] +pub unsafe fn _mm256_shrdv_epi32(a: __m256i, b: __m256i, c: __m256i) -> __m256i { + transmute(vpshrdvd256(a.as_i32x8(), b.as_i32x8(), c.as_i32x8())) +} + +/// Concatenate packed 32-bit integers in b and a producing an intermediate 64-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 32-bits in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shrdv_epi32&expand=5127) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshrdvd))] +pub unsafe fn _mm256_mask_shrdv_epi32(a: __m256i, k: __mmask8, b: __m256i, c: __m256i) -> __m256i { + let shf = _mm256_shrdv_epi32(a, b, c).as_i32x8(); + transmute(simd_select_bitmask(k, shf, a.as_i32x8())) +} + +/// Concatenate packed 32-bit integers in b and a producing an intermediate 64-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 32-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shrdv_epi32&expand=5128) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshrdvd))] +pub unsafe fn _mm256_maskz_shrdv_epi32(k: __mmask8, a: __m256i, b: __m256i, c: __m256i) -> __m256i { + let shf = _mm256_shrdv_epi32(a, b, c).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 32-bit integers in b and a producing an intermediate 64-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 32-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shrdv_epi32&expand=5126) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshrdvd))] +pub unsafe fn _mm_shrdv_epi32(a: __m128i, b: __m128i, c: __m128i) -> __m128i { + transmute(vpshrdvd128(a.as_i32x4(), b.as_i32x4(), c.as_i32x4())) +} + +/// Concatenate packed 32-bit integers in b and a producing an intermediate 64-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 32-bits in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shrdv_epi32&expand=5124) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshrdvd))] +pub unsafe fn _mm_mask_shrdv_epi32(a: __m128i, k: __mmask8, b: __m128i, c: __m128i) -> __m128i { + let shf = _mm_shrdv_epi32(a, b, c).as_i32x4(); + transmute(simd_select_bitmask(k, shf, a.as_i32x4())) +} + +/// Concatenate packed 32-bit integers in b and a producing an intermediate 64-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 32-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shrdv_epi32&expand=5125) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshrdvd))] +pub unsafe fn _mm_maskz_shrdv_epi32(k: __mmask8, a: __m128i, b: __m128i, c: __m128i) -> __m128i { + let shf = _mm_shrdv_epi32(a, b, c).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 16-bit integers in b and a producing an intermediate 32-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 16-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shrdv_epi16&expand=5123) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshrdvw))] +pub unsafe fn _mm512_shrdv_epi16(a: __m512i, b: __m512i, c: __m512i) -> __m512i { + transmute(vpshrdvw(a.as_i16x32(), b.as_i16x32(), c.as_i16x32())) +} + +/// Concatenate packed 16-bit integers in b and a producing an intermediate 32-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 16-bits in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shrdv_epi16&expand=5121) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshrdvw))] +pub unsafe fn _mm512_mask_shrdv_epi16(a: __m512i, k: __mmask32, b: __m512i, c: __m512i) -> __m512i { + let shf = _mm512_shrdv_epi16(a, b, c).as_i16x32(); + transmute(simd_select_bitmask(k, shf, a.as_i16x32())) +} + +/// Concatenate packed 16-bit integers in b and a producing an intermediate 32-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 16-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shrdv_epi16&expand=5122) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshrdvw))] +pub unsafe fn _mm512_maskz_shrdv_epi16( + k: __mmask32, + a: __m512i, + b: __m512i, + c: __m512i, +) -> __m512i { + let shf = _mm512_shrdv_epi16(a, b, c).as_i16x32(); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 16-bit integers in b and a producing an intermediate 32-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 16-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shrdv_epi16&expand=5120) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshrdvw))] +pub unsafe fn _mm256_shrdv_epi16(a: __m256i, b: __m256i, c: __m256i) -> __m256i { + transmute(vpshrdvw256(a.as_i16x16(), b.as_i16x16(), c.as_i16x16())) +} + +/// Concatenate packed 16-bit integers in b and a producing an intermediate 32-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 16-bits in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shrdv_epi16&expand=5118) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshrdvw))] +pub unsafe fn _mm256_mask_shrdv_epi16(a: __m256i, k: __mmask16, b: __m256i, c: __m256i) -> __m256i { + let shf = _mm256_shrdv_epi16(a, b, c).as_i16x16(); + transmute(simd_select_bitmask(k, shf, a.as_i16x16())) +} + +/// Concatenate packed 16-bit integers in b and a producing an intermediate 32-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 16-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shrdv_epi16&expand=5119) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshrdvw))] +pub unsafe fn _mm256_maskz_shrdv_epi16( + k: __mmask16, + a: __m256i, + b: __m256i, + c: __m256i, +) -> __m256i { + let shf = _mm256_shrdv_epi16(a, b, c).as_i16x16(); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 16-bit integers in b and a producing an intermediate 32-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 16-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shrdv_epi16&expand=5117) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshrdvw))] +pub unsafe fn _mm_shrdv_epi16(a: __m128i, b: __m128i, c: __m128i) -> __m128i { + transmute(vpshrdvw128(a.as_i16x8(), b.as_i16x8(), c.as_i16x8())) +} + +/// Concatenate packed 16-bit integers in b and a producing an intermediate 32-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 16-bits in dst using writemask k (elements are copied from a when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shrdv_epi16&expand=5115) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshrdvw))] +pub unsafe fn _mm_mask_shrdv_epi16(a: __m128i, k: __mmask8, b: __m128i, c: __m128i) -> __m128i { + let shf = _mm_shrdv_epi16(a, b, c).as_i16x8(); + transmute(simd_select_bitmask(k, shf, a.as_i16x8())) +} + +/// Concatenate packed 16-bit integers in b and a producing an intermediate 32-bit result. Shift the result right by the amount specified in the corresponding element of c, and store the lower 16-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shrdv_epi16&expand=5116) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshrdvw))] +pub unsafe fn _mm_maskz_shrdv_epi16(k: __mmask8, a: __m128i, b: __m128i, c: __m128i) -> __m128i { + let shf = _mm_shrdv_epi16(a, b, c).as_i16x8(); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 64-bit integers in a and b producing an intermediate 128-bit result. Shift the result left by imm8 bits, and store the upper 64-bits in dst). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shldi_epi64&expand=5060) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shldi_epi64(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + transmute(vpshldvq( + a.as_i64x8(), + b.as_i64x8(), + _mm512_set1_epi64(imm8).as_i64x8(), + )) +} + +/// Concatenate packed 64-bit integers in a and b producing an intermediate 128-bit result. Shift the result left by imm8 bits, and store the upper 64-bits in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shldi_epi64&expand=5058) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shldi_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + let shf: i64x8 = vpshldvq( + a.as_i64x8(), + b.as_i64x8(), + _mm512_set1_epi64(imm8).as_i64x8(), + ); + transmute(simd_select_bitmask(k, shf, src.as_i64x8())) +} + +/// Concatenate packed 64-bit integers in a and b producing an intermediate 128-bit result. Shift the result left by imm8 bits, and store the upper 64-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shldi_epi64&expand=5059) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shldi_epi64( + k: __mmask8, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + let shf: i64x8 = vpshldvq( + a.as_i64x8(), + b.as_i64x8(), + _mm512_set1_epi64(imm8).as_i64x8(), + ); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 64-bit integers in a and b producing an intermediate 128-bit result. Shift the result left by imm8 bits, and store the upper 64-bits in dst). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shldi_epi64&expand=5057) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_shldi_epi64(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + transmute(vpshldvq256( + a.as_i64x4(), + b.as_i64x4(), + _mm256_set1_epi64x(imm8).as_i64x4(), + )) +} + +/// Concatenate packed 64-bit integers in a and b producing an intermediate 128-bit result. Shift the result left by imm8 bits, and store the upper 64-bits in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shldi_epi64&expand=5055) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shldi_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + let shf: i64x4 = vpshldvq256( + a.as_i64x4(), + b.as_i64x4(), + _mm256_set1_epi64x(imm8).as_i64x4(), + ); + transmute(simd_select_bitmask(k, shf, src.as_i64x4())) +} + +/// Concatenate packed 64-bit integers in a and b producing an intermediate 128-bit result. Shift the result left by imm8 bits, and store the upper 64-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shldi_epi64&expand=5056) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shldi_epi64( + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + let shf: i64x4 = vpshldvq256( + a.as_i64x4(), + b.as_i64x4(), + _mm256_set1_epi64x(imm8).as_i64x4(), + ); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 64-bit integers in a and b producing an intermediate 128-bit result. Shift the result left by imm8 bits, and store the upper 64-bits in dst). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shldi_epi64&expand=5054) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_shldi_epi64(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + transmute(vpshldvq128( + a.as_i64x2(), + b.as_i64x2(), + _mm_set1_epi64x(imm8).as_i64x2(), + )) +} + +/// Concatenate packed 64-bit integers in a and b producing an intermediate 128-bit result. Shift the result left by imm8 bits, and store the upper 64-bits in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shldi_epi64&expand=5052) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_shldi_epi64( + src: __m128i, + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + let shf: i64x2 = vpshldvq128(a.as_i64x2(), b.as_i64x2(), _mm_set1_epi64x(imm8).as_i64x2()); + transmute(simd_select_bitmask(k, shf, src.as_i64x2())) +} + +/// Concatenate packed 64-bit integers in a and b producing an intermediate 128-bit result. Shift the result left by imm8 bits, and store the upper 64-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shldi_epi64&expand=5053) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_shldi_epi64( + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + let shf: i64x2 = vpshldvq128(a.as_i64x2(), b.as_i64x2(), _mm_set1_epi64x(imm8).as_i64x2()); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 32-bit integers in a and b producing an intermediate 64-bit result. Shift the result left by imm8 bits, and store the upper 32-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shldi_epi32&expand=5051) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shldi_epi32(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + transmute(vpshldvd( + a.as_i32x16(), + b.as_i32x16(), + _mm512_set1_epi32(IMM8).as_i32x16(), + )) +} + +/// Concatenate packed 32-bit integers in a and b producing an intermediate 64-bit result. Shift the result left by imm8 bits, and store the upper 32-bits in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shldi_epi32&expand=5049) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shldi_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let shf: i32x16 = vpshldvd( + a.as_i32x16(), + b.as_i32x16(), + _mm512_set1_epi32(IMM8).as_i32x16(), + ); + transmute(simd_select_bitmask(k, shf, src.as_i32x16())) +} + +/// Concatenate packed 32-bit integers in a and b producing an intermediate 64-bit result. Shift the result left by imm8 bits, and store the upper 32-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shldi_epi32&expand=5050) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shldi_epi32( + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let shf: i32x16 = vpshldvd( + a.as_i32x16(), + b.as_i32x16(), + _mm512_set1_epi32(IMM8).as_i32x16(), + ); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 32-bit integers in a and b producing an intermediate 64-bit result. Shift the result left by imm8 bits, and store the upper 32-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shldi_epi32&expand=5048) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_shldi_epi32(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(vpshldvd256( + a.as_i32x8(), + b.as_i32x8(), + _mm256_set1_epi32(IMM8).as_i32x8(), + )) +} + +/// Concatenate packed 32-bit integers in a and b producing an intermediate 64-bit result. Shift the result left by imm8 bits, and store the upper 32-bits in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shldi_epi32&expand=5046) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shldi_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let shf: i32x8 = vpshldvd256( + a.as_i32x8(), + b.as_i32x8(), + _mm256_set1_epi32(IMM8).as_i32x8(), + ); + transmute(simd_select_bitmask(k, shf, src.as_i32x8())) +} + +/// Concatenate packed 32-bit integers in a and b producing an intermediate 64-bit result. Shift the result left by imm8 bits, and store the upper 32-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shldi_epi32&expand=5047) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shldi_epi32( + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let shf: i32x8 = vpshldvd256( + a.as_i32x8(), + b.as_i32x8(), + _mm256_set1_epi32(IMM8).as_i32x8(), + ); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 32-bit integers in a and b producing an intermediate 64-bit result. Shift the result left by imm8 bits, and store the upper 32-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shldi_epi32&expand=5045) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_shldi_epi32(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(vpshldvd128( + a.as_i32x4(), + b.as_i32x4(), + _mm_set1_epi32(IMM8).as_i32x4(), + )) +} + +/// Concatenate packed 32-bit integers in a and b producing an intermediate 64-bit result. Shift the result left by imm8 bits, and store the upper 32-bits in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shldi_epi32&expand=5043) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_shldi_epi32( + src: __m128i, + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let shf: i32x4 = vpshldvd128(a.as_i32x4(), b.as_i32x4(), _mm_set1_epi32(IMM8).as_i32x4()); + transmute(simd_select_bitmask(k, shf, src.as_i32x4())) +} + +/// Concatenate packed 32-bit integers in a and b producing an intermediate 64-bit result. Shift the result left by imm8 bits, and store the upper 32-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shldi_epi32&expand=5044) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_shldi_epi32( + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let shf: i32x4 = vpshldvd128(a.as_i32x4(), b.as_i32x4(), _mm_set1_epi32(IMM8).as_i32x4()); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 16-bit integers in a and b producing an intermediate 32-bit result. Shift the result left by imm8 bits, and store the upper 16-bits in dst). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shldi_epi16&expand=5042) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shldi_epi16(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + transmute(vpshldvw( + a.as_i16x32(), + b.as_i16x32(), + _mm512_set1_epi16(imm8).as_i16x32(), + )) +} + +/// Concatenate packed 16-bit integers in a and b producing an intermediate 32-bit result. Shift the result left by imm8 bits, and store the upper 16-bits in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shldi_epi16&expand=5040) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shldi_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + let shf: i16x32 = vpshldvw( + a.as_i16x32(), + b.as_i16x32(), + _mm512_set1_epi16(imm8).as_i16x32(), + ); + transmute(simd_select_bitmask(k, shf, src.as_i16x32())) +} + +/// Concatenate packed 16-bit integers in a and b producing an intermediate 32-bit result. Shift the result left by imm8 bits, and store the upper 16-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shldi_epi16&expand=5041) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shldi_epi16( + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + let shf: i16x32 = vpshldvw( + a.as_i16x32(), + b.as_i16x32(), + _mm512_set1_epi16(imm8).as_i16x32(), + ); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 16-bit integers in a and b producing an intermediate 32-bit result. Shift the result left by imm8 bits, and store the upper 16-bits in dst). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shldi_epi16&expand=5039) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_shldi_epi16(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + transmute(vpshldvw256( + a.as_i16x16(), + b.as_i16x16(), + _mm256_set1_epi16(imm8).as_i16x16(), + )) +} + +/// Concatenate packed 16-bit integers in a and b producing an intermediate 32-bit result. Shift the result left by imm8 bits, and store the upper 16-bits in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shldi_epi16&expand=5037) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shldi_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + let shf: i16x16 = vpshldvw256( + a.as_i16x16(), + b.as_i16x16(), + _mm256_set1_epi16(imm8).as_i16x16(), + ); + transmute(simd_select_bitmask(k, shf, src.as_i16x16())) +} + +/// Concatenate packed 16-bit integers in a and b producing an intermediate 32-bit result. Shift the result left by imm8 bits, and store the upper 16-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shldi_epi16&expand=5038) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shldi_epi16( + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + let shf: i16x16 = vpshldvw256( + a.as_i16x16(), + b.as_i16x16(), + _mm256_set1_epi16(imm8).as_i16x16(), + ); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 16-bit integers in a and b producing an intermediate 32-bit result. Shift the result left by imm8 bits, and store the upper 16-bits in dst). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shldi_epi16&expand=5036) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_shldi_epi16(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + transmute(vpshldvw128( + a.as_i16x8(), + b.as_i16x8(), + _mm_set1_epi16(imm8).as_i16x8(), + )) +} + +/// Concatenate packed 16-bit integers in a and b producing an intermediate 32-bit result. Shift the result left by imm8 bits, and store the upper 16-bits in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shldi_epi16&expand=5034) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_shldi_epi16( + src: __m128i, + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + let shf: i16x8 = vpshldvw128(a.as_i16x8(), b.as_i16x8(), _mm_set1_epi16(imm8).as_i16x8()); + transmute(simd_select_bitmask(k, shf, src.as_i16x8())) +} + +/// Concatenate packed 16-bit integers in a and b producing an intermediate 32-bit result. Shift the result left by imm8 bits, and store the upper 16-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shldi_epi16&expand=5035) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_shldi_epi16( + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + let shf: i16x8 = vpshldvw128(a.as_i16x8(), b.as_i16x8(), _mm_set1_epi16(imm8).as_i16x8()); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 64-bit integers in b and a producing an intermediate 128-bit result. Shift the result right by imm8 bits, and store the lower 64-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shrdi_epi64&expand=5114) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] //should be vpshrdq +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shrdi_epi64(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + transmute(vpshrdvq( + a.as_i64x8(), + b.as_i64x8(), + _mm512_set1_epi64(imm8).as_i64x8(), + )) +} + +/// Concatenate packed 64-bit integers in b and a producing an intermediate 128-bit result. Shift the result right by imm8 bits, and store the lower 64-bits in dst using writemask k (elements are copied from src" when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shrdi_epi64&expand=5112) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] //should be vpshrdq +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shrdi_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + let shf: i64x8 = vpshrdvq( + a.as_i64x8(), + b.as_i64x8(), + _mm512_set1_epi64(imm8).as_i64x8(), + ); + transmute(simd_select_bitmask(k, shf, src.as_i64x8())) +} + +/// Concatenate packed 64-bit integers in b and a producing an intermediate 128-bit result. Shift the result right by imm8 bits, and store the lower 64-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shrdi_epi64&expand=5113) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 255))] //should be vpshrdq +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shrdi_epi64( + k: __mmask8, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + let shf: i64x8 = vpshrdvq( + a.as_i64x8(), + b.as_i64x8(), + _mm512_set1_epi64(imm8).as_i64x8(), + ); + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 64-bit integers in b and a producing an intermediate 128-bit result. Shift the result right by imm8 bits, and store the lower 64-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shrdi_epi64&expand=5111) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] //should be vpshrdq +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_shrdi_epi64(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + transmute(vpshrdvq256( + a.as_i64x4(), + b.as_i64x4(), + _mm256_set1_epi64x(imm8).as_i64x4(), + )) +} + +/// Concatenate packed 64-bit integers in b and a producing an intermediate 128-bit result. Shift the result right by imm8 bits, and store the lower 64-bits in dst using writemask k (elements are copied from src" when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shrdi_epi64&expand=5109) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] //should be vpshrdq +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shrdi_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + let shf: i64x4 = vpshrdvq256( + a.as_i64x4(), + b.as_i64x4(), + _mm256_set1_epi64x(imm8).as_i64x4(), + ); + transmute(simd_select_bitmask(k, shf, src.as_i64x4())) +} + +/// Concatenate packed 64-bit integers in b and a producing an intermediate 128-bit result. Shift the result right by imm8 bits, and store the lower 64-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shrdi_epi64&expand=5110) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] //should be vpshrdq +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shrdi_epi64( + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + let shf: i64x4 = vpshrdvq256( + a.as_i64x4(), + b.as_i64x4(), + _mm256_set1_epi64x(imm8).as_i64x4(), + ); + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 64-bit integers in b and a producing an intermediate 128-bit result. Shift the result right by imm8 bits, and store the lower 64-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shrdi_epi64&expand=5108) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] //should be vpshrdq +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_shrdi_epi64(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + transmute(vpshrdvq128( + a.as_i64x2(), + b.as_i64x2(), + _mm_set1_epi64x(imm8).as_i64x2(), + )) +} + +/// Concatenate packed 64-bit integers in b and a producing an intermediate 128-bit result. Shift the result right by imm8 bits, and store the lower 64-bits in dst using writemask k (elements are copied from src" when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shrdi_epi64&expand=5106) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] //should be vpshrdq +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_shrdi_epi64( + src: __m128i, + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + let shf: i64x2 = vpshrdvq128(a.as_i64x2(), b.as_i64x2(), _mm_set1_epi64x(imm8).as_i64x2()); + transmute(simd_select_bitmask(k, shf, src.as_i64x2())) +} + +/// Concatenate packed 64-bit integers in b and a producing an intermediate 128-bit result. Shift the result right by imm8 bits, and store the lower 64-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shrdi_epi64&expand=5107) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] //should be vpshrdq +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_shrdi_epi64( + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + let shf: i64x2 = vpshrdvq128(a.as_i64x2(), b.as_i64x2(), _mm_set1_epi64x(imm8).as_i64x2()); + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 32-bit integers in b and a producing an intermediate 64-bit result. Shift the result right by imm8 bits, and store the lower 32-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shrdi_epi32&expand=5105) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] //should be vpshldd +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shrdi_epi32(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + transmute(vpshrdvd( + a.as_i32x16(), + b.as_i32x16(), + _mm512_set1_epi32(IMM8).as_i32x16(), + )) +} + +/// Concatenate packed 32-bit integers in b and a producing an intermediate 64-bit result. Shift the result right by imm8 bits, and store the lower 32-bits in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shrdi_epi32&expand=5103) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] //should be vpshldd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shrdi_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let shf: i32x16 = vpshrdvd( + a.as_i32x16(), + b.as_i32x16(), + _mm512_set1_epi32(IMM8).as_i32x16(), + ); + transmute(simd_select_bitmask(k, shf, src.as_i32x16())) +} + +/// Concatenate packed 32-bit integers in b and a producing an intermediate 64-bit result. Shift the result right by imm8 bits, and store the lower 32-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shrdi_epi32&expand=5104) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] //should be vpshldd +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shrdi_epi32( + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let shf: i32x16 = vpshrdvd( + a.as_i32x16(), + b.as_i32x16(), + _mm512_set1_epi32(IMM8).as_i32x16(), + ); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 32-bit integers in b and a producing an intermediate 64-bit result. Shift the result right by imm8 bits, and store the lower 32-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shrdi_epi32&expand=5102) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] //should be vpshldd +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_shrdi_epi32(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(vpshrdvd256( + a.as_i32x8(), + b.as_i32x8(), + _mm256_set1_epi32(IMM8).as_i32x8(), + )) +} + +/// Concatenate packed 32-bit integers in b and a producing an intermediate 64-bit result. Shift the result right by imm8 bits, and store the lower 32-bits in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shrdi_epi32&expand=5100) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] //should be vpshldd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shrdi_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let shf: i32x8 = vpshrdvd256( + a.as_i32x8(), + b.as_i32x8(), + _mm256_set1_epi32(IMM8).as_i32x8(), + ); + transmute(simd_select_bitmask(k, shf, src.as_i32x8())) +} + +/// Concatenate packed 32-bit integers in b and a producing an intermediate 64-bit result. Shift the result right by imm8 bits, and store the lower 32-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shrdi_epi32&expand=5101) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] //should be vpshldd +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shrdi_epi32( + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let shf: i32x8 = vpshrdvd256( + a.as_i32x8(), + b.as_i32x8(), + _mm256_set1_epi32(IMM8).as_i32x8(), + ); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 32-bit integers in b and a producing an intermediate 64-bit result. Shift the result right by imm8 bits, and store the lower 32-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shrdi_epi32&expand=5099) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] //should be vpshldd +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_shrdi_epi32(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(vpshrdvd128( + a.as_i32x4(), + b.as_i32x4(), + _mm_set1_epi32(IMM8).as_i32x4(), + )) +} + +/// Concatenate packed 32-bit integers in b and a producing an intermediate 64-bit result. Shift the result right by imm8 bits, and store the lower 32-bits in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shrdi_epi32&expand=5097) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] //should be vpshldd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_shrdi_epi32( + src: __m128i, + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let shf: i32x4 = vpshrdvd128(a.as_i32x4(), b.as_i32x4(), _mm_set1_epi32(IMM8).as_i32x4()); + transmute(simd_select_bitmask(k, shf, src.as_i32x4())) +} + +/// Concatenate packed 32-bit integers in b and a producing an intermediate 64-bit result. Shift the result right by imm8 bits, and store the lower 32-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shrdi_epi32&expand=5098) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] //should be vpshldd +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_shrdi_epi32( + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let shf: i32x4 = vpshrdvd128(a.as_i32x4(), b.as_i32x4(), _mm_set1_epi32(IMM8).as_i32x4()); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 16-bit integers in b and a producing an intermediate 32-bit result. Shift the result right by imm8 bits, and store the lower 16-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shrdi_epi16&expand=5096) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] //should be vpshrdw +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shrdi_epi16(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + assert!(imm8 >= 0 && imm8 <= 255); + transmute(vpshrdvw( + a.as_i16x32(), + b.as_i16x32(), + _mm512_set1_epi16(imm8).as_i16x32(), + )) +} + +/// Concatenate packed 16-bit integers in b and a producing an intermediate 32-bit result. Shift the result right by imm8 bits, and store the lower 16-bits in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shrdi_epi16&expand=5094) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] //should be vpshrdw +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shrdi_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + assert!(imm8 >= 0 && imm8 <= 255); + let shf: i16x32 = vpshrdvw( + a.as_i16x32(), + b.as_i16x32(), + _mm512_set1_epi16(imm8).as_i16x32(), + ); + transmute(simd_select_bitmask(k, shf, src.as_i16x32())) +} + +/// Concatenate packed 16-bit integers in b and a producing an intermediate 32-bit result. Shift the result right by imm8 bits, and store the lower 16-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shrdi_epi16&expand=5095) +#[inline] +#[target_feature(enable = "avx512vbmi2")] +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] //should be vpshrdw +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shrdi_epi16( + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + assert!(imm8 >= 0 && imm8 <= 255); + let shf: i16x32 = vpshrdvw( + a.as_i16x32(), + b.as_i16x32(), + _mm512_set1_epi16(imm8).as_i16x32(), + ); + let zero = _mm512_setzero_si512().as_i16x32(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 16-bit integers in b and a producing an intermediate 32-bit result. Shift the result right by imm8 bits, and store the lower 16-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shrdi_epi16&expand=5093) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] //should be vpshrdw +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_shrdi_epi16(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + assert!(imm8 >= 0 && imm8 <= 255); + transmute(vpshrdvw256( + a.as_i16x16(), + b.as_i16x16(), + _mm256_set1_epi16(imm8).as_i16x16(), + )) +} + +/// Concatenate packed 16-bit integers in b and a producing an intermediate 32-bit result. Shift the result right by imm8 bits, and store the lower 16-bits in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shrdi_epi16&expand=5091) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] //should be vpshrdw +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shrdi_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + assert!(imm8 >= 0 && imm8 <= 255); + let shf: i16x16 = vpshrdvw256( + a.as_i16x16(), + b.as_i16x16(), + _mm256_set1_epi16(imm8).as_i16x16(), + ); + transmute(simd_select_bitmask(k, shf, src.as_i16x16())) +} + +/// Concatenate packed 16-bit integers in b and a producing an intermediate 32-bit result. Shift the result right by imm8 bits, and store the lower 16-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shrdi_epi16&expand=5092) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] //should be vpshrdw +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shrdi_epi16( + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + let shf: i16x16 = vpshrdvw256( + a.as_i16x16(), + b.as_i16x16(), + _mm256_set1_epi16(imm8).as_i16x16(), + ); + let zero = _mm256_setzero_si256().as_i16x16(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +/// Concatenate packed 16-bit integers in b and a producing an intermediate 32-bit result. Shift the result right by imm8 bits, and store the lower 16-bits in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shrdi_epi16&expand=5090) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] //should be vpshrdw +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_shrdi_epi16(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + transmute(vpshrdvw128( + a.as_i16x8(), + b.as_i16x8(), + _mm_set1_epi16(imm8).as_i16x8(), + )) +} + +/// Concatenate packed 16-bit integers in b and a producing an intermediate 32-bit result. Shift the result right by imm8 bits, and store the lower 16-bits in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shrdi_epi16&expand=5088) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] //should be vpshrdw +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_shrdi_epi16( + src: __m128i, + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + let shf: i16x8 = vpshrdvw128(a.as_i16x8(), b.as_i16x8(), _mm_set1_epi16(imm8).as_i16x8()); + transmute(simd_select_bitmask(k, shf, src.as_i16x8())) +} + +/// Concatenate packed 16-bit integers in b and a producing an intermediate 32-bit result. Shift the result right by imm8 bits, and store the lower 16-bits in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shrdi_epi16&expand=5089) +#[inline] +#[target_feature(enable = "avx512vbmi2,avx512vl")] +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] //should be vpshrdw +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_shrdi_epi16( + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + let shf: i16x8 = vpshrdvw128(a.as_i16x8(), b.as_i16x8(), _mm_set1_epi16(imm8).as_i16x8()); + let zero = _mm_setzero_si128().as_i16x8(); + transmute(simd_select_bitmask(k, shf, zero)) +} + +#[allow(improper_ctypes)] +extern "C" { + #[link_name = "llvm.x86.avx512.mask.compress.store.w.512"] + fn vcompressstorew(mem: *mut i8, data: i16x32, mask: u32); + #[link_name = "llvm.x86.avx512.mask.compress.store.w.256"] + fn vcompressstorew256(mem: *mut i8, data: i16x16, mask: u16); + #[link_name = "llvm.x86.avx512.mask.compress.store.w.128"] + fn vcompressstorew128(mem: *mut i8, data: i16x8, mask: u8); + + #[link_name = "llvm.x86.avx512.mask.compress.store.b.512"] + fn vcompressstoreb(mem: *mut i8, data: i8x64, mask: u64); + #[link_name = "llvm.x86.avx512.mask.compress.store.b.256"] + fn vcompressstoreb256(mem: *mut i8, data: i8x32, mask: u32); + #[link_name = "llvm.x86.avx512.mask.compress.store.b.128"] + fn vcompressstoreb128(mem: *mut i8, data: i8x16, mask: u16); + + #[link_name = "llvm.x86.avx512.mask.compress.w.512"] + fn vpcompressw(a: i16x32, src: i16x32, mask: u32) -> i16x32; + #[link_name = "llvm.x86.avx512.mask.compress.w.256"] + fn vpcompressw256(a: i16x16, src: i16x16, mask: u16) -> i16x16; + #[link_name = "llvm.x86.avx512.mask.compress.w.128"] + fn vpcompressw128(a: i16x8, src: i16x8, mask: u8) -> i16x8; + + #[link_name = "llvm.x86.avx512.mask.compress.b.512"] + fn vpcompressb(a: i8x64, src: i8x64, mask: u64) -> i8x64; + #[link_name = "llvm.x86.avx512.mask.compress.b.256"] + fn vpcompressb256(a: i8x32, src: i8x32, mask: u32) -> i8x32; + #[link_name = "llvm.x86.avx512.mask.compress.b.128"] + fn vpcompressb128(a: i8x16, src: i8x16, mask: u16) -> i8x16; + + #[link_name = "llvm.x86.avx512.mask.expand.w.512"] + fn vpexpandw(a: i16x32, src: i16x32, mask: u32) -> i16x32; + #[link_name = "llvm.x86.avx512.mask.expand.w.256"] + fn vpexpandw256(a: i16x16, src: i16x16, mask: u16) -> i16x16; + #[link_name = "llvm.x86.avx512.mask.expand.w.128"] + fn vpexpandw128(a: i16x8, src: i16x8, mask: u8) -> i16x8; + + #[link_name = "llvm.x86.avx512.mask.expand.b.512"] + fn vpexpandb(a: i8x64, src: i8x64, mask: u64) -> i8x64; + #[link_name = "llvm.x86.avx512.mask.expand.b.256"] + fn vpexpandb256(a: i8x32, src: i8x32, mask: u32) -> i8x32; + #[link_name = "llvm.x86.avx512.mask.expand.b.128"] + fn vpexpandb128(a: i8x16, src: i8x16, mask: u16) -> i8x16; + + #[link_name = "llvm.fshl.v8i64"] + fn vpshldvq(a: i64x8, b: i64x8, c: i64x8) -> i64x8; + #[link_name = "llvm.fshl.v4i64"] + fn vpshldvq256(a: i64x4, b: i64x4, c: i64x4) -> i64x4; + #[link_name = "llvm.fshl.v2i64"] + fn vpshldvq128(a: i64x2, b: i64x2, c: i64x2) -> i64x2; + #[link_name = "llvm.fshl.v16i32"] + fn vpshldvd(a: i32x16, b: i32x16, c: i32x16) -> i32x16; + #[link_name = "llvm.fshl.v8i32"] + fn vpshldvd256(a: i32x8, b: i32x8, c: i32x8) -> i32x8; + #[link_name = "llvm.fshl.v4i32"] + fn vpshldvd128(a: i32x4, b: i32x4, c: i32x4) -> i32x4; + #[link_name = "llvm.fshl.v32i16"] + fn vpshldvw(a: i16x32, b: i16x32, c: i16x32) -> i16x32; + #[link_name = "llvm.fshl.v16i16"] + fn vpshldvw256(a: i16x16, b: i16x16, c: i16x16) -> i16x16; + #[link_name = "llvm.fshl.v8i16"] + fn vpshldvw128(a: i16x8, b: i16x8, c: i16x8) -> i16x8; + + #[link_name = "llvm.fshr.v8i64"] + fn vpshrdvq(a: i64x8, b: i64x8, c: i64x8) -> i64x8; + #[link_name = "llvm.fshr.v4i64"] + fn vpshrdvq256(a: i64x4, b: i64x4, c: i64x4) -> i64x4; + #[link_name = "llvm.fshr.v2i64"] + fn vpshrdvq128(a: i64x2, b: i64x2, c: i64x2) -> i64x2; + #[link_name = "llvm.fshr.v16i32"] + fn vpshrdvd(a: i32x16, b: i32x16, c: i32x16) -> i32x16; + #[link_name = "llvm.fshr.v8i32"] + fn vpshrdvd256(a: i32x8, b: i32x8, c: i32x8) -> i32x8; + #[link_name = "llvm.fshr.v4i32"] + fn vpshrdvd128(a: i32x4, b: i32x4, c: i32x4) -> i32x4; + #[link_name = "llvm.fshr.v32i16"] + fn vpshrdvw(a: i16x32, b: i16x32, c: i16x32) -> i16x32; + #[link_name = "llvm.fshr.v16i16"] + fn vpshrdvw256(a: i16x16, b: i16x16, c: i16x16) -> i16x16; + #[link_name = "llvm.fshr.v8i16"] + fn vpshrdvw128(a: i16x8, b: i16x8, c: i16x8) -> i16x8; +} + +#[cfg(test)] +mod tests { + + use stdarch_test::simd_test; + + use crate::core_arch::x86::*; + use crate::hint::black_box; + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_mask_compress_epi16() { + let src = _mm512_set1_epi16(200); + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let r = _mm512_mask_compress_epi16(src, 0b01010101_01010101_01010101_01010101, a); + #[rustfmt::skip] + let e = _mm512_set_epi16( + 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_maskz_compress_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let r = _mm512_maskz_compress_epi16(0b01010101_01010101_01010101_01010101, a); + #[rustfmt::skip] + let e = _mm512_set_epi16( + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_mask_compress_epi16() { + let src = _mm256_set1_epi16(200); + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm256_mask_compress_epi16(src, 0b01010101_01010101, a); + let e = _mm256_set_epi16( + 200, 200, 200, 200, 200, 200, 200, 200, 1, 3, 5, 7, 9, 11, 13, 15, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_maskz_compress_epi16() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm256_maskz_compress_epi16(0b01010101_01010101, a); + let e = _mm256_set_epi16(0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 5, 7, 9, 11, 13, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_mask_compress_epi16() { + let src = _mm_set1_epi16(200); + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm_mask_compress_epi16(src, 0b01010101, a); + let e = _mm_set_epi16(200, 200, 200, 200, 1, 3, 5, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_maskz_compress_epi16() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm_maskz_compress_epi16(0b01010101, a); + let e = _mm_set_epi16(0, 0, 0, 0, 1, 3, 5, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_mask_compress_epi8() { + let src = _mm512_set1_epi8(100); + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63); + let r = _mm512_mask_compress_epi8( + src, + 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101, + a, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8( + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, + 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_maskz_compress_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63); + let r = _mm512_maskz_compress_epi8( + 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101, + a, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8( + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, + 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_mask_compress_epi8() { + let src = _mm256_set1_epi8(100); + #[rustfmt::skip] + let a = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let r = _mm256_mask_compress_epi8(src, 0b01010101_01010101_01010101_01010101, a); + #[rustfmt::skip] + let e = _mm256_set_epi8( + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_maskz_compress_epi8() { + #[rustfmt::skip] + let a = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let r = _mm256_maskz_compress_epi8(0b01010101_01010101_01010101_01010101, a); + #[rustfmt::skip] + let e = _mm256_set_epi8( + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_mask_compress_epi8() { + let src = _mm_set1_epi8(100); + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm_mask_compress_epi8(src, 0b01010101_01010101, a); + let e = _mm_set_epi8( + 100, 100, 100, 100, 100, 100, 100, 100, 1, 3, 5, 7, 9, 11, 13, 15, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_maskz_compress_epi8() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm_maskz_compress_epi8(0b01010101_01010101, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 5, 7, 9, 11, 13, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_mask_expand_epi16() { + let src = _mm512_set1_epi16(200); + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let r = _mm512_mask_expand_epi16(src, 0b01010101_01010101_01010101_01010101, a); + #[rustfmt::skip] + let e = _mm512_set_epi16( + 200, 16, 200, 17, 200, 18, 200, 19, 200, 20, 200, 21, 200, 22, 200, 23, + 200, 24, 200, 25, 200, 26, 200, 27, 200, 28, 200, 29, 200, 30, 200, 31, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_maskz_expand_epi16() { + #[rustfmt::skip] + let a = _mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let r = _mm512_maskz_expand_epi16(0b01010101_01010101_01010101_01010101, a); + #[rustfmt::skip] + let e = _mm512_set_epi16(0, 16, 0, 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, + 0, 24, 0, 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_mask_expand_epi16() { + let src = _mm256_set1_epi16(200); + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm256_mask_expand_epi16(src, 0b01010101_01010101, a); + let e = _mm256_set_epi16( + 200, 8, 200, 9, 200, 10, 200, 11, 200, 12, 200, 13, 200, 14, 200, 15, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_maskz_expand_epi16() { + let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm256_maskz_expand_epi16(0b01010101_01010101, a); + let e = _mm256_set_epi16(0, 8, 0, 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_mask_expand_epi16() { + let src = _mm_set1_epi16(200); + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm_mask_expand_epi16(src, 0b01010101, a); + let e = _mm_set_epi16(200, 4, 200, 5, 200, 6, 200, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_maskz_expand_epi16() { + let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm_maskz_expand_epi16(0b01010101, a); + let e = _mm_set_epi16(0, 4, 0, 5, 0, 6, 0, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_mask_expand_epi8() { + let src = _mm512_set1_epi8(100); + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63); + let r = _mm512_mask_expand_epi8( + src, + 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101, + a, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8( + 100, 32, 100, 33, 100, 34, 100, 35, 100, 36, 100, 37, 100, 38, 100, 39, + 100, 40, 100, 41, 100, 42, 100, 43, 100, 44, 100, 45, 100, 46, 100, 47, + 100, 48, 100, 49, 100, 50, 100, 51, 100, 52, 100, 53, 100, 54, 100, 55, + 100, 56, 100, 57, 100, 58, 100, 59, 100, 60, 100, 61, 100, 62, 100, 63, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_maskz_expand_epi8() { + #[rustfmt::skip] + let a = _mm512_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63); + let r = _mm512_maskz_expand_epi8( + 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101, + a, + ); + #[rustfmt::skip] + let e = _mm512_set_epi8( + 0, 32, 0, 33, 0, 34, 0, 35, 0, 36, 0, 37, 0, 38, 0, 39, + 0, 40, 0, 41, 0, 42, 0, 43, 0, 44, 0, 45, 0, 46, 0, 47, + 0, 48, 0, 49, 0, 50, 0, 51, 0, 52, 0, 53, 0, 54, 0, 55, + 0, 56, 0, 57, 0, 58, 0, 59, 0, 60, 0, 61, 0, 62, 0, 63, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_mask_expand_epi8() { + let src = _mm256_set1_epi8(100); + #[rustfmt::skip] + let a = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let r = _mm256_mask_expand_epi8(src, 0b01010101_01010101_01010101_01010101, a); + #[rustfmt::skip] + let e = _mm256_set_epi8( + 100, 16, 100, 17, 100, 18, 100, 19, 100, 20, 100, 21, 100, 22, 100, 23, + 100, 24, 100, 25, 100, 26, 100, 27, 100, 28, 100, 29, 100, 30, 100, 31, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_maskz_expand_epi8() { + #[rustfmt::skip] + let a = _mm256_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let r = _mm256_maskz_expand_epi8(0b01010101_01010101_01010101_01010101, a); + #[rustfmt::skip] + let e = _mm256_set_epi8( + 0, 16, 0, 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, + 0, 24, 0, 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_mask_expand_epi8() { + let src = _mm_set1_epi8(100); + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm_mask_expand_epi8(src, 0b01010101_01010101, a); + let e = _mm_set_epi8( + 100, 8, 100, 9, 100, 10, 100, 11, 100, 12, 100, 13, 100, 14, 100, 15, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_maskz_expand_epi8() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm_maskz_expand_epi8(0b01010101_01010101, a); + let e = _mm_set_epi8(0, 8, 0, 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_shldv_epi64() { + let a = _mm512_set1_epi64(1); + let b = _mm512_set1_epi64(1 << 63); + let c = _mm512_set1_epi64(2); + let r = _mm512_shldv_epi64(a, b, c); + let e = _mm512_set1_epi64(6); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_mask_shldv_epi64() { + let a = _mm512_set1_epi64(1); + let b = _mm512_set1_epi64(1 << 63); + let c = _mm512_set1_epi64(2); + let r = _mm512_mask_shldv_epi64(a, 0, b, c); + assert_eq_m512i(r, a); + let r = _mm512_mask_shldv_epi64(a, 0b11111111, b, c); + let e = _mm512_set1_epi64(6); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_maskz_shldv_epi64() { + let a = _mm512_set1_epi64(1); + let b = _mm512_set1_epi64(1 << 63); + let c = _mm512_set1_epi64(2); + let r = _mm512_maskz_shldv_epi64(0, a, b, c); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_shldv_epi64(0b11111111, a, b, c); + let e = _mm512_set1_epi64(6); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_shldv_epi64() { + let a = _mm256_set1_epi64x(1); + let b = _mm256_set1_epi64x(1 << 63); + let c = _mm256_set1_epi64x(2); + let r = _mm256_shldv_epi64(a, b, c); + let e = _mm256_set1_epi64x(6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_mask_shldv_epi64() { + let a = _mm256_set1_epi64x(1); + let b = _mm256_set1_epi64x(1 << 63); + let c = _mm256_set1_epi64x(2); + let r = _mm256_mask_shldv_epi64(a, 0, b, c); + assert_eq_m256i(r, a); + let r = _mm256_mask_shldv_epi64(a, 0b00001111, b, c); + let e = _mm256_set1_epi64x(6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_maskz_shldv_epi64() { + let a = _mm256_set1_epi64x(1); + let b = _mm256_set1_epi64x(1 << 63); + let c = _mm256_set1_epi64x(2); + let r = _mm256_maskz_shldv_epi64(0, a, b, c); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_shldv_epi64(0b00001111, a, b, c); + let e = _mm256_set1_epi64x(6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_shldv_epi64() { + let a = _mm_set1_epi64x(1); + let b = _mm_set1_epi64x(1 << 63); + let c = _mm_set1_epi64x(2); + let r = _mm_shldv_epi64(a, b, c); + let e = _mm_set1_epi64x(6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_mask_shldv_epi64() { + let a = _mm_set1_epi64x(1); + let b = _mm_set1_epi64x(1 << 63); + let c = _mm_set1_epi64x(2); + let r = _mm_mask_shldv_epi64(a, 0, b, c); + assert_eq_m128i(r, a); + let r = _mm_mask_shldv_epi64(a, 0b00000011, b, c); + let e = _mm_set1_epi64x(6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_maskz_shldv_epi64() { + let a = _mm_set1_epi64x(1); + let b = _mm_set1_epi64x(1 << 63); + let c = _mm_set1_epi64x(2); + let r = _mm_maskz_shldv_epi64(0, a, b, c); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_shldv_epi64(0b00000011, a, b, c); + let e = _mm_set1_epi64x(6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_shldv_epi32() { + let a = _mm512_set1_epi32(1); + let b = _mm512_set1_epi32(1 << 31); + let c = _mm512_set1_epi32(2); + let r = _mm512_shldv_epi32(a, b, c); + let e = _mm512_set1_epi32(6); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_mask_shldv_epi32() { + let a = _mm512_set1_epi32(1); + let b = _mm512_set1_epi32(1 << 31); + let c = _mm512_set1_epi32(2); + let r = _mm512_mask_shldv_epi32(a, 0, b, c); + assert_eq_m512i(r, a); + let r = _mm512_mask_shldv_epi32(a, 0b11111111_11111111, b, c); + let e = _mm512_set1_epi32(6); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_maskz_shldv_epi32() { + let a = _mm512_set1_epi32(1); + let b = _mm512_set1_epi32(1 << 31); + let c = _mm512_set1_epi32(2); + let r = _mm512_maskz_shldv_epi32(0, a, b, c); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_shldv_epi32(0b11111111_11111111, a, b, c); + let e = _mm512_set1_epi32(6); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_shldv_epi32() { + let a = _mm256_set1_epi32(1); + let b = _mm256_set1_epi32(1 << 31); + let c = _mm256_set1_epi32(2); + let r = _mm256_shldv_epi32(a, b, c); + let e = _mm256_set1_epi32(6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_mask_shldv_epi32() { + let a = _mm256_set1_epi32(1); + let b = _mm256_set1_epi32(1 << 31); + let c = _mm256_set1_epi32(2); + let r = _mm256_mask_shldv_epi32(a, 0, b, c); + assert_eq_m256i(r, a); + let r = _mm256_mask_shldv_epi32(a, 0b11111111, b, c); + let e = _mm256_set1_epi32(6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_maskz_shldv_epi32() { + let a = _mm256_set1_epi32(1); + let b = _mm256_set1_epi32(1 << 31); + let c = _mm256_set1_epi32(2); + let r = _mm256_maskz_shldv_epi32(0, a, b, c); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_shldv_epi32(0b11111111, a, b, c); + let e = _mm256_set1_epi32(6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_shldv_epi32() { + let a = _mm_set1_epi32(1); + let b = _mm_set1_epi32(1 << 31); + let c = _mm_set1_epi32(2); + let r = _mm_shldv_epi32(a, b, c); + let e = _mm_set1_epi32(6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_mask_shldv_epi32() { + let a = _mm_set1_epi32(1); + let b = _mm_set1_epi32(1 << 31); + let c = _mm_set1_epi32(2); + let r = _mm_mask_shldv_epi32(a, 0, b, c); + assert_eq_m128i(r, a); + let r = _mm_mask_shldv_epi32(a, 0b00001111, b, c); + let e = _mm_set1_epi32(6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_maskz_shldv_epi32() { + let a = _mm_set1_epi32(1); + let b = _mm_set1_epi32(1 << 31); + let c = _mm_set1_epi32(2); + let r = _mm_maskz_shldv_epi32(0, a, b, c); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_shldv_epi32(0b00001111, a, b, c); + let e = _mm_set1_epi32(6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_shldv_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1 << 15); + let c = _mm512_set1_epi16(2); + let r = _mm512_shldv_epi16(a, b, c); + let e = _mm512_set1_epi16(6); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_mask_shldv_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1 << 15); + let c = _mm512_set1_epi16(2); + let r = _mm512_mask_shldv_epi16(a, 0, b, c); + assert_eq_m512i(r, a); + let r = _mm512_mask_shldv_epi16(a, 0b11111111_11111111_11111111_11111111, b, c); + let e = _mm512_set1_epi16(6); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_maskz_shldv_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1 << 15); + let c = _mm512_set1_epi16(2); + let r = _mm512_maskz_shldv_epi16(0, a, b, c); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_shldv_epi16(0b11111111_11111111_11111111_11111111, a, b, c); + let e = _mm512_set1_epi16(6); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_shldv_epi16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1 << 15); + let c = _mm256_set1_epi16(2); + let r = _mm256_shldv_epi16(a, b, c); + let e = _mm256_set1_epi16(6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_mask_shldv_epi16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1 << 15); + let c = _mm256_set1_epi16(2); + let r = _mm256_mask_shldv_epi16(a, 0, b, c); + assert_eq_m256i(r, a); + let r = _mm256_mask_shldv_epi16(a, 0b11111111_11111111, b, c); + let e = _mm256_set1_epi16(6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_maskz_shldv_epi16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1 << 15); + let c = _mm256_set1_epi16(2); + let r = _mm256_maskz_shldv_epi16(0, a, b, c); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_shldv_epi16(0b11111111_11111111, a, b, c); + let e = _mm256_set1_epi16(6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_shldv_epi16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1 << 15); + let c = _mm_set1_epi16(2); + let r = _mm_shldv_epi16(a, b, c); + let e = _mm_set1_epi16(6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_mask_shldv_epi16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1 << 15); + let c = _mm_set1_epi16(2); + let r = _mm_mask_shldv_epi16(a, 0, b, c); + assert_eq_m128i(r, a); + let r = _mm_mask_shldv_epi16(a, 0b11111111, b, c); + let e = _mm_set1_epi16(6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_maskz_shldv_epi16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1 << 15); + let c = _mm_set1_epi16(2); + let r = _mm_maskz_shldv_epi16(0, a, b, c); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_shldv_epi16(0b11111111, a, b, c); + let e = _mm_set1_epi16(6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_shrdv_epi64() { + let a = _mm512_set1_epi64(8); + let b = _mm512_set1_epi64(2); + let c = _mm512_set1_epi64(1); + let r = _mm512_shrdv_epi64(a, b, c); + let e = _mm512_set1_epi64(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_mask_shrdv_epi64() { + let a = _mm512_set1_epi64(8); + let b = _mm512_set1_epi64(2); + let c = _mm512_set1_epi64(1); + let r = _mm512_mask_shrdv_epi64(a, 0, b, c); + assert_eq_m512i(r, a); + let r = _mm512_mask_shrdv_epi64(a, 0b11111111, b, c); + let e = _mm512_set1_epi64(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_maskz_shrdv_epi64() { + let a = _mm512_set1_epi64(8); + let b = _mm512_set1_epi64(2); + let c = _mm512_set1_epi64(1); + let r = _mm512_maskz_shrdv_epi64(0, a, b, c); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_shrdv_epi64(0b11111111, a, b, c); + let e = _mm512_set1_epi64(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_shrdv_epi64() { + let a = _mm256_set1_epi64x(8); + let b = _mm256_set1_epi64x(2); + let c = _mm256_set1_epi64x(1); + let r = _mm256_shrdv_epi64(a, b, c); + let e = _mm256_set1_epi64x(1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_mask_shrdv_epi64() { + let a = _mm256_set1_epi64x(8); + let b = _mm256_set1_epi64x(2); + let c = _mm256_set1_epi64x(1); + let r = _mm256_mask_shrdv_epi64(a, 0, b, c); + assert_eq_m256i(r, a); + let r = _mm256_mask_shrdv_epi64(a, 0b00001111, b, c); + let e = _mm256_set1_epi64x(1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_maskz_shrdv_epi64() { + let a = _mm256_set1_epi64x(8); + let b = _mm256_set1_epi64x(2); + let c = _mm256_set1_epi64x(1); + let r = _mm256_maskz_shrdv_epi64(0, a, b, c); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_shrdv_epi64(0b00001111, a, b, c); + let e = _mm256_set1_epi64x(1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_shrdv_epi64() { + let a = _mm_set1_epi64x(8); + let b = _mm_set1_epi64x(2); + let c = _mm_set1_epi64x(1); + let r = _mm_shrdv_epi64(a, b, c); + let e = _mm_set1_epi64x(1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_mask_shrdv_epi64() { + let a = _mm_set1_epi64x(8); + let b = _mm_set1_epi64x(2); + let c = _mm_set1_epi64x(1); + let r = _mm_mask_shrdv_epi64(a, 0, b, c); + assert_eq_m128i(r, a); + let r = _mm_mask_shrdv_epi64(a, 0b00000011, b, c); + let e = _mm_set1_epi64x(1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_maskz_shrdv_epi64() { + let a = _mm_set1_epi64x(8); + let b = _mm_set1_epi64x(2); + let c = _mm_set1_epi64x(1); + let r = _mm_maskz_shrdv_epi64(0, a, b, c); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_shrdv_epi64(0b00000011, a, b, c); + let e = _mm_set1_epi64x(1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_shrdv_epi32() { + let a = _mm512_set1_epi32(8); + let b = _mm512_set1_epi32(2); + let c = _mm512_set1_epi32(1); + let r = _mm512_shrdv_epi32(a, b, c); + let e = _mm512_set1_epi32(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_mask_shrdv_epi32() { + let a = _mm512_set1_epi32(8); + let b = _mm512_set1_epi32(2); + let c = _mm512_set1_epi32(1); + let r = _mm512_mask_shrdv_epi32(a, 0, b, c); + assert_eq_m512i(r, a); + let r = _mm512_mask_shrdv_epi32(a, 0b11111111_11111111, b, c); + let e = _mm512_set1_epi32(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_maskz_shrdv_epi32() { + let a = _mm512_set1_epi32(8); + let b = _mm512_set1_epi32(2); + let c = _mm512_set1_epi32(1); + let r = _mm512_maskz_shrdv_epi32(0, a, b, c); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_shrdv_epi32(0b11111111_11111111, a, b, c); + let e = _mm512_set1_epi32(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_shrdv_epi32() { + let a = _mm256_set1_epi32(8); + let b = _mm256_set1_epi32(2); + let c = _mm256_set1_epi32(1); + let r = _mm256_shrdv_epi32(a, b, c); + let e = _mm256_set1_epi32(1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_mask_shrdv_epi32() { + let a = _mm256_set1_epi32(8); + let b = _mm256_set1_epi32(2); + let c = _mm256_set1_epi32(1); + let r = _mm256_mask_shrdv_epi32(a, 0, b, c); + assert_eq_m256i(r, a); + let r = _mm256_mask_shrdv_epi32(a, 0b11111111, b, c); + let e = _mm256_set1_epi32(1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_maskz_shrdv_epi32() { + let a = _mm256_set1_epi32(8); + let b = _mm256_set1_epi32(2); + let c = _mm256_set1_epi32(1); + let r = _mm256_maskz_shrdv_epi32(0, a, b, c); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_shrdv_epi32(0b11111111, a, b, c); + let e = _mm256_set1_epi32(1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_shrdv_epi32() { + let a = _mm_set1_epi32(8); + let b = _mm_set1_epi32(2); + let c = _mm_set1_epi32(1); + let r = _mm_shrdv_epi32(a, b, c); + let e = _mm_set1_epi32(1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_mask_shrdv_epi32() { + let a = _mm_set1_epi32(8); + let b = _mm_set1_epi32(2); + let c = _mm_set1_epi32(1); + let r = _mm_mask_shrdv_epi32(a, 0, b, c); + assert_eq_m128i(r, a); + let r = _mm_mask_shrdv_epi32(a, 0b00001111, b, c); + let e = _mm_set1_epi32(1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_maskz_shrdv_epi32() { + let a = _mm_set1_epi32(8); + let b = _mm_set1_epi32(2); + let c = _mm_set1_epi32(1); + let r = _mm_maskz_shrdv_epi32(0, a, b, c); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_shrdv_epi32(0b00001111, a, b, c); + let e = _mm_set1_epi32(1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_shrdv_epi16() { + let a = _mm512_set1_epi16(8); + let b = _mm512_set1_epi16(2); + let c = _mm512_set1_epi16(1); + let r = _mm512_shrdv_epi16(a, b, c); + let e = _mm512_set1_epi16(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_mask_shrdv_epi16() { + let a = _mm512_set1_epi16(8); + let b = _mm512_set1_epi16(2); + let c = _mm512_set1_epi16(1); + let r = _mm512_mask_shrdv_epi16(a, 0, b, c); + assert_eq_m512i(r, a); + let r = _mm512_mask_shrdv_epi16(a, 0b11111111_11111111_11111111_11111111, b, c); + let e = _mm512_set1_epi16(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_maskz_shrdv_epi16() { + let a = _mm512_set1_epi16(8); + let b = _mm512_set1_epi16(2); + let c = _mm512_set1_epi16(1); + let r = _mm512_maskz_shrdv_epi16(0, a, b, c); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_shrdv_epi16(0b11111111_11111111_11111111_11111111, a, b, c); + let e = _mm512_set1_epi16(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_shrdv_epi16() { + let a = _mm256_set1_epi16(8); + let b = _mm256_set1_epi16(2); + let c = _mm256_set1_epi16(1); + let r = _mm256_shrdv_epi16(a, b, c); + let e = _mm256_set1_epi16(1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_mask_shrdv_epi16() { + let a = _mm256_set1_epi16(8); + let b = _mm256_set1_epi16(2); + let c = _mm256_set1_epi16(1); + let r = _mm256_mask_shrdv_epi16(a, 0, b, c); + assert_eq_m256i(r, a); + let r = _mm256_mask_shrdv_epi16(a, 0b11111111_11111111, b, c); + let e = _mm256_set1_epi16(1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_maskz_shrdv_epi16() { + let a = _mm256_set1_epi16(8); + let b = _mm256_set1_epi16(2); + let c = _mm256_set1_epi16(1); + let r = _mm256_maskz_shrdv_epi16(0, a, b, c); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_shrdv_epi16(0b11111111_11111111, a, b, c); + let e = _mm256_set1_epi16(1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_shrdv_epi16() { + let a = _mm_set1_epi16(8); + let b = _mm_set1_epi16(2); + let c = _mm_set1_epi16(1); + let r = _mm_shrdv_epi16(a, b, c); + let e = _mm_set1_epi16(1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_mask_shrdv_epi16() { + let a = _mm_set1_epi16(8); + let b = _mm_set1_epi16(2); + let c = _mm_set1_epi16(1); + let r = _mm_mask_shrdv_epi16(a, 0, b, c); + assert_eq_m128i(r, a); + let r = _mm_mask_shrdv_epi16(a, 0b11111111, b, c); + let e = _mm_set1_epi16(1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_maskz_shrdv_epi16() { + let a = _mm_set1_epi16(8); + let b = _mm_set1_epi16(2); + let c = _mm_set1_epi16(1); + let r = _mm_maskz_shrdv_epi16(0, a, b, c); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_shrdv_epi16(0b11111111, a, b, c); + let e = _mm_set1_epi16(1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_shldi_epi64() { + let a = _mm512_set1_epi64(1); + let b = _mm512_set1_epi64(1 << 63); + let r = _mm512_shldi_epi64::<2>(a, b); + let e = _mm512_set1_epi64(6); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_mask_shldi_epi64() { + let a = _mm512_set1_epi64(1); + let b = _mm512_set1_epi64(1 << 63); + let r = _mm512_mask_shldi_epi64::<2>(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_shldi_epi64::<2>(a, 0b11111111, a, b); + let e = _mm512_set1_epi64(6); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_maskz_shldi_epi64() { + let a = _mm512_set1_epi64(1); + let b = _mm512_set1_epi64(1 << 63); + let r = _mm512_maskz_shldi_epi64::<2>(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_shldi_epi64::<2>(0b11111111, a, b); + let e = _mm512_set1_epi64(6); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_shldi_epi64() { + let a = _mm256_set1_epi64x(1); + let b = _mm256_set1_epi64x(1 << 63); + let r = _mm256_shldi_epi64::<2>(a, b); + let e = _mm256_set1_epi64x(6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_mask_shldi_epi64() { + let a = _mm256_set1_epi64x(1); + let b = _mm256_set1_epi64x(1 << 63); + let r = _mm256_mask_shldi_epi64::<2>(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_shldi_epi64::<2>(a, 0b00001111, a, b); + let e = _mm256_set1_epi64x(6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_maskz_shldi_epi64() { + let a = _mm256_set1_epi64x(1); + let b = _mm256_set1_epi64x(1 << 63); + let r = _mm256_maskz_shldi_epi64::<2>(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_shldi_epi64::<2>(0b00001111, a, b); + let e = _mm256_set1_epi64x(6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_shldi_epi64() { + let a = _mm_set1_epi64x(1); + let b = _mm_set1_epi64x(1 << 63); + let r = _mm_shldi_epi64::<2>(a, b); + let e = _mm_set1_epi64x(6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_mask_shldi_epi64() { + let a = _mm_set1_epi64x(1); + let b = _mm_set1_epi64x(1 << 63); + let r = _mm_mask_shldi_epi64::<2>(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_shldi_epi64::<2>(a, 0b00000011, a, b); + let e = _mm_set1_epi64x(6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_maskz_shldi_epi64() { + let a = _mm_set1_epi64x(1); + let b = _mm_set1_epi64x(1 << 63); + let r = _mm_maskz_shldi_epi64::<2>(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_shldi_epi64::<2>(0b00000011, a, b); + let e = _mm_set1_epi64x(6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_shldi_epi32() { + let a = _mm512_set1_epi32(1); + let b = _mm512_set1_epi32(1 << 31); + let r = _mm512_shldi_epi32::<2>(a, b); + let e = _mm512_set1_epi32(6); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_mask_shldi_epi32() { + let a = _mm512_set1_epi32(1); + let b = _mm512_set1_epi32(1 << 31); + let r = _mm512_mask_shldi_epi32::<2>(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_shldi_epi32::<2>(a, 0b11111111_11111111, a, b); + let e = _mm512_set1_epi32(6); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_maskz_shldi_epi32() { + let a = _mm512_set1_epi32(1); + let b = _mm512_set1_epi32(1 << 31); + let r = _mm512_maskz_shldi_epi32::<2>(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_shldi_epi32::<2>(0b11111111_11111111, a, b); + let e = _mm512_set1_epi32(6); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_shldi_epi32() { + let a = _mm256_set1_epi32(1); + let b = _mm256_set1_epi32(1 << 31); + let r = _mm256_shldi_epi32::<2>(a, b); + let e = _mm256_set1_epi32(6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_mask_shldi_epi32() { + let a = _mm256_set1_epi32(1); + let b = _mm256_set1_epi32(1 << 31); + let r = _mm256_mask_shldi_epi32::<2>(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_shldi_epi32::<2>(a, 0b11111111, a, b); + let e = _mm256_set1_epi32(6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_maskz_shldi_epi32() { + let a = _mm256_set1_epi32(1); + let b = _mm256_set1_epi32(1 << 31); + let r = _mm256_maskz_shldi_epi32::<2>(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_shldi_epi32::<2>(0b11111111, a, b); + let e = _mm256_set1_epi32(6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_shldi_epi32() { + let a = _mm_set1_epi32(1); + let b = _mm_set1_epi32(1 << 31); + let r = _mm_shldi_epi32::<2>(a, b); + let e = _mm_set1_epi32(6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_mask_shldi_epi32() { + let a = _mm_set1_epi32(1); + let b = _mm_set1_epi32(1 << 31); + let r = _mm_mask_shldi_epi32::<2>(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_shldi_epi32::<2>(a, 0b00001111, a, b); + let e = _mm_set1_epi32(6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_maskz_shldi_epi32() { + let a = _mm_set1_epi32(1); + let b = _mm_set1_epi32(1 << 31); + let r = _mm_maskz_shldi_epi32::<2>(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_shldi_epi32::<2>(0b00001111, a, b); + let e = _mm_set1_epi32(6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_shldi_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1 << 15); + let r = _mm512_shldi_epi16::<2>(a, b); + let e = _mm512_set1_epi16(6); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_mask_shldi_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1 << 15); + let r = _mm512_mask_shldi_epi16::<2>(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_shldi_epi16::<2>(a, 0b11111111_11111111_11111111_11111111, a, b); + let e = _mm512_set1_epi16(6); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_maskz_shldi_epi16() { + let a = _mm512_set1_epi16(1); + let b = _mm512_set1_epi16(1 << 15); + let r = _mm512_maskz_shldi_epi16::<2>(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_shldi_epi16::<2>(0b11111111_11111111_11111111_11111111, a, b); + let e = _mm512_set1_epi16(6); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_shldi_epi16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1 << 15); + let r = _mm256_shldi_epi16::<2>(a, b); + let e = _mm256_set1_epi16(6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_mask_shldi_epi16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1 << 15); + let r = _mm256_mask_shldi_epi16::<2>(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_shldi_epi16::<2>(a, 0b11111111_11111111, a, b); + let e = _mm256_set1_epi16(6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_maskz_shldi_epi16() { + let a = _mm256_set1_epi16(1); + let b = _mm256_set1_epi16(1 << 15); + let r = _mm256_maskz_shldi_epi16::<2>(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_shldi_epi16::<2>(0b11111111_11111111, a, b); + let e = _mm256_set1_epi16(6); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_shldi_epi16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1 << 15); + let r = _mm_shldi_epi16::<2>(a, b); + let e = _mm_set1_epi16(6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_mask_shldi_epi16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1 << 15); + let r = _mm_mask_shldi_epi16::<2>(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_shldi_epi16::<2>(a, 0b11111111, a, b); + let e = _mm_set1_epi16(6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_maskz_shldi_epi16() { + let a = _mm_set1_epi16(1); + let b = _mm_set1_epi16(1 << 15); + let r = _mm_maskz_shldi_epi16::<2>(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_shldi_epi16::<2>(0b11111111, a, b); + let e = _mm_set1_epi16(6); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_shrdi_epi64() { + let a = _mm512_set1_epi64(8); + let b = _mm512_set1_epi64(2); + let r = _mm512_shrdi_epi64::<1>(a, b); + let e = _mm512_set1_epi64(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_mask_shrdi_epi64() { + let a = _mm512_set1_epi64(8); + let b = _mm512_set1_epi64(2); + let r = _mm512_mask_shrdi_epi64::<1>(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_shrdi_epi64::<1>(a, 0b11111111, a, b); + let e = _mm512_set1_epi64(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_maskz_shrdi_epi64() { + let a = _mm512_set1_epi64(8); + let b = _mm512_set1_epi64(2); + let r = _mm512_maskz_shrdi_epi64::<1>(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_shrdi_epi64::<1>(0b11111111, a, b); + let e = _mm512_set1_epi64(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_shrdi_epi64() { + let a = _mm256_set1_epi64x(8); + let b = _mm256_set1_epi64x(2); + let r = _mm256_shrdi_epi64::<1>(a, b); + let e = _mm256_set1_epi64x(1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_mask_shrdi_epi64() { + let a = _mm256_set1_epi64x(8); + let b = _mm256_set1_epi64x(2); + let r = _mm256_mask_shrdi_epi64::<1>(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_shrdi_epi64::<1>(a, 0b00001111, a, b); + let e = _mm256_set1_epi64x(1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_maskz_shrdi_epi64() { + let a = _mm256_set1_epi64x(8); + let b = _mm256_set1_epi64x(2); + let r = _mm256_maskz_shrdi_epi64::<1>(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_shrdi_epi64::<1>(0b00001111, a, b); + let e = _mm256_set1_epi64x(1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_shrdi_epi64() { + let a = _mm_set1_epi64x(8); + let b = _mm_set1_epi64x(2); + let r = _mm_shrdi_epi64::<1>(a, b); + let e = _mm_set1_epi64x(1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_mask_shrdi_epi64() { + let a = _mm_set1_epi64x(8); + let b = _mm_set1_epi64x(2); + let r = _mm_mask_shrdi_epi64::<1>(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_shrdi_epi64::<1>(a, 0b00000011, a, b); + let e = _mm_set1_epi64x(1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_maskz_shrdi_epi64() { + let a = _mm_set1_epi64x(8); + let b = _mm_set1_epi64x(2); + let r = _mm_maskz_shrdi_epi64::<1>(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_shrdi_epi64::<1>(0b00000011, a, b); + let e = _mm_set1_epi64x(1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_shrdi_epi32() { + let a = _mm512_set1_epi32(8); + let b = _mm512_set1_epi32(2); + let r = _mm512_shrdi_epi32::<1>(a, b); + let e = _mm512_set1_epi32(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_mask_shrdi_epi32() { + let a = _mm512_set1_epi32(8); + let b = _mm512_set1_epi32(2); + let r = _mm512_mask_shrdi_epi32::<1>(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_shrdi_epi32::<1>(a, 0b11111111_11111111, a, b); + let e = _mm512_set1_epi32(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_maskz_shrdi_epi32() { + let a = _mm512_set1_epi32(8); + let b = _mm512_set1_epi32(2); + let r = _mm512_maskz_shrdi_epi32::<1>(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_shrdi_epi32::<1>(0b11111111_11111111, a, b); + let e = _mm512_set1_epi32(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_shrdi_epi32() { + let a = _mm256_set1_epi32(8); + let b = _mm256_set1_epi32(2); + let r = _mm256_shrdi_epi32::<1>(a, b); + let e = _mm256_set1_epi32(1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_mask_shrdi_epi32() { + let a = _mm256_set1_epi32(8); + let b = _mm256_set1_epi32(2); + let r = _mm256_mask_shrdi_epi32::<1>(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_shrdi_epi32::<1>(a, 0b11111111, a, b); + let e = _mm256_set1_epi32(1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_maskz_shrdi_epi32() { + let a = _mm256_set1_epi32(8); + let b = _mm256_set1_epi32(2); + let r = _mm256_maskz_shrdi_epi32::<1>(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_shrdi_epi32::<1>(0b11111111, a, b); + let e = _mm256_set1_epi32(1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_shrdi_epi32() { + let a = _mm_set1_epi32(8); + let b = _mm_set1_epi32(2); + let r = _mm_shrdi_epi32::<1>(a, b); + let e = _mm_set1_epi32(1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_mask_shrdi_epi32() { + let a = _mm_set1_epi32(8); + let b = _mm_set1_epi32(2); + let r = _mm_mask_shrdi_epi32::<1>(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_shrdi_epi32::<1>(a, 0b00001111, a, b); + let e = _mm_set1_epi32(1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_maskz_shrdi_epi32() { + let a = _mm_set1_epi32(8); + let b = _mm_set1_epi32(2); + let r = _mm_maskz_shrdi_epi32::<1>(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_shrdi_epi32::<1>(0b00001111, a, b); + let e = _mm_set1_epi32(1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_shrdi_epi16() { + let a = _mm512_set1_epi16(8); + let b = _mm512_set1_epi16(2); + let r = _mm512_shrdi_epi16::<1>(a, b); + let e = _mm512_set1_epi16(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_mask_shrdi_epi16() { + let a = _mm512_set1_epi16(8); + let b = _mm512_set1_epi16(2); + let r = _mm512_mask_shrdi_epi16::<1>(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_shrdi_epi16::<1>(a, 0b11111111_11111111_11111111_11111111, a, b); + let e = _mm512_set1_epi16(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_maskz_shrdi_epi16() { + let a = _mm512_set1_epi16(8); + let b = _mm512_set1_epi16(2); + let r = _mm512_maskz_shrdi_epi16::<1>(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_shrdi_epi16::<1>(0b11111111_11111111_11111111_11111111, a, b); + let e = _mm512_set1_epi16(1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_shrdi_epi16() { + let a = _mm256_set1_epi16(8); + let b = _mm256_set1_epi16(2); + let r = _mm256_shrdi_epi16::<1>(a, b); + let e = _mm256_set1_epi16(1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_mask_shrdi_epi16() { + let a = _mm256_set1_epi16(8); + let b = _mm256_set1_epi16(2); + let r = _mm256_mask_shrdi_epi16::<1>(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_shrdi_epi16::<1>(a, 0b11111111_11111111, a, b); + let e = _mm256_set1_epi16(1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_maskz_shrdi_epi16() { + let a = _mm256_set1_epi16(8); + let b = _mm256_set1_epi16(2); + let r = _mm256_maskz_shrdi_epi16::<1>(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_shrdi_epi16::<1>(0b11111111_11111111, a, b); + let e = _mm256_set1_epi16(1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_shrdi_epi16() { + let a = _mm_set1_epi16(8); + let b = _mm_set1_epi16(2); + let r = _mm_shrdi_epi16::<1>(a, b); + let e = _mm_set1_epi16(1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_mask_shrdi_epi16() { + let a = _mm_set1_epi16(8); + let b = _mm_set1_epi16(2); + let r = _mm_mask_shrdi_epi16::<1>(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_shrdi_epi16::<1>(a, 0b11111111, a, b); + let e = _mm_set1_epi16(1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_maskz_shrdi_epi16() { + let a = _mm_set1_epi16(8); + let b = _mm_set1_epi16(2); + let r = _mm_maskz_shrdi_epi16::<1>(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_shrdi_epi16::<1>(0b11111111, a, b); + let e = _mm_set1_epi16(1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_mask_expandloadu_epi16() { + let src = _mm512_set1_epi16(42); + let a = &[ + 1_i16, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]; + let p = a.as_ptr(); + let m = 0b11101000_11001010_11110000_00001111; + let r = _mm512_mask_expandloadu_epi16(src, m, black_box(p)); + let e = _mm512_set_epi16( + 16, 15, 14, 42, 13, 42, 42, 42, 12, 11, 42, 42, 10, 42, 9, 42, 8, 7, 6, 5, 42, 42, 42, + 42, 42, 42, 42, 42, 4, 3, 2, 1, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_maskz_expandloadu_epi16() { + let a = &[ + 1_i16, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]; + let p = a.as_ptr(); + let m = 0b11101000_11001010_11110000_00001111; + let r = _mm512_maskz_expandloadu_epi16(m, black_box(p)); + let e = _mm512_set_epi16( + 16, 15, 14, 0, 13, 0, 0, 0, 12, 11, 0, 0, 10, 0, 9, 0, 8, 7, 6, 5, 0, 0, 0, 0, 0, 0, 0, + 0, 4, 3, 2, 1, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_mask_expandloadu_epi16() { + let src = _mm256_set1_epi16(42); + let a = &[1_i16, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let p = a.as_ptr(); + let m = 0b11101000_11001010; + let r = _mm256_mask_expandloadu_epi16(src, m, black_box(p)); + let e = _mm256_set_epi16(8, 7, 6, 42, 5, 42, 42, 42, 4, 3, 42, 42, 2, 42, 1, 42); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_maskz_expandloadu_epi16() { + let a = &[1_i16, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let p = a.as_ptr(); + let m = 0b11101000_11001010; + let r = _mm256_maskz_expandloadu_epi16(m, black_box(p)); + let e = _mm256_set_epi16(8, 7, 6, 0, 5, 0, 0, 0, 4, 3, 0, 0, 2, 0, 1, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_mask_expandloadu_epi16() { + let src = _mm_set1_epi16(42); + let a = &[1_i16, 2, 3, 4, 5, 6, 7, 8]; + let p = a.as_ptr(); + let m = 0b11101000; + let r = _mm_mask_expandloadu_epi16(src, m, black_box(p)); + let e = _mm_set_epi16(4, 3, 2, 42, 1, 42, 42, 42); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_maskz_expandloadu_epi16() { + let a = &[1_i16, 2, 3, 4, 5, 6, 7, 8]; + let p = a.as_ptr(); + let m = 0b11101000; + let r = _mm_maskz_expandloadu_epi16(m, black_box(p)); + let e = _mm_set_epi16(4, 3, 2, 0, 1, 0, 0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_mask_expandloadu_epi8() { + let src = _mm512_set1_epi8(42); + let a = &[ + 1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + ]; + let p = a.as_ptr(); + let m = 0b11101000_11001010_11110000_00001111_11111111_00000000_10101010_01010101; + let r = _mm512_mask_expandloadu_epi8(src, m, black_box(p)); + let e = _mm512_set_epi8( + 32, 31, 30, 42, 29, 42, 42, 42, 28, 27, 42, 42, 26, 42, 25, 42, 24, 23, 22, 21, 42, 42, + 42, 42, 42, 42, 42, 42, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 42, 42, 42, 42, + 42, 42, 42, 42, 8, 42, 7, 42, 6, 42, 5, 42, 42, 4, 42, 3, 42, 2, 42, 1, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_maskz_expandloadu_epi8() { + let a = &[ + 1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + ]; + let p = a.as_ptr(); + let m = 0b11101000_11001010_11110000_00001111_11111111_00000000_10101010_01010101; + let r = _mm512_maskz_expandloadu_epi8(m, black_box(p)); + let e = _mm512_set_epi8( + 32, 31, 30, 0, 29, 0, 0, 0, 28, 27, 0, 0, 26, 0, 25, 0, 24, 23, 22, 21, 0, 0, 0, 0, 0, + 0, 0, 0, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, + 7, 0, 6, 0, 5, 0, 0, 4, 0, 3, 0, 2, 0, 1, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_mask_expandloadu_epi8() { + let src = _mm256_set1_epi8(42); + let a = &[ + 1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]; + let p = a.as_ptr(); + let m = 0b11101000_11001010_11110000_00001111; + let r = _mm256_mask_expandloadu_epi8(src, m, black_box(p)); + let e = _mm256_set_epi8( + 16, 15, 14, 42, 13, 42, 42, 42, 12, 11, 42, 42, 10, 42, 9, 42, 8, 7, 6, 5, 42, 42, 42, + 42, 42, 42, 42, 42, 4, 3, 2, 1, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_maskz_expandloadu_epi8() { + let a = &[ + 1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]; + let p = a.as_ptr(); + let m = 0b11101000_11001010_11110000_00001111; + let r = _mm256_maskz_expandloadu_epi8(m, black_box(p)); + let e = _mm256_set_epi8( + 16, 15, 14, 0, 13, 0, 0, 0, 12, 11, 0, 0, 10, 0, 9, 0, 8, 7, 6, 5, 0, 0, 0, 0, 0, 0, 0, + 0, 4, 3, 2, 1, + ); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_mask_expandloadu_epi8() { + let src = _mm_set1_epi8(42); + let a = &[1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let p = a.as_ptr(); + let m = 0b11101000_11001010; + let r = _mm_mask_expandloadu_epi8(src, m, black_box(p)); + let e = _mm_set_epi8(8, 7, 6, 42, 5, 42, 42, 42, 4, 3, 42, 42, 2, 42, 1, 42); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_maskz_expandloadu_epi8() { + let a = &[1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let p = a.as_ptr(); + let m = 0b11101000_11001010; + let r = _mm_maskz_expandloadu_epi8(m, black_box(p)); + let e = _mm_set_epi8(8, 7, 6, 0, 5, 0, 0, 0, 4, 3, 0, 0, 2, 0, 1, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_mask_compressstoreu_epi16() { + let a = _mm512_set_epi16( + 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, + 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, + ); + let mut r = [0_i16; 32]; + _mm512_mask_compressstoreu_epi16(r.as_mut_ptr() as *mut _, 0, a); + assert_eq!(&r, &[0_i16; 32]); + _mm512_mask_compressstoreu_epi16( + r.as_mut_ptr() as *mut _, + 0b11110000_11001010_11111111_00000000, + a, + ); + assert_eq!( + &r, + &[ + 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 23, 24, 29, 30, 31, 32, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0 + ] + ); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_mask_compressstoreu_epi16() { + let a = _mm256_set_epi16(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + let mut r = [0_i16; 16]; + _mm256_mask_compressstoreu_epi16(r.as_mut_ptr() as *mut _, 0, a); + assert_eq!(&r, &[0_i16; 16]); + _mm256_mask_compressstoreu_epi16(r.as_mut_ptr() as *mut _, 0b11110000_11001010, a); + assert_eq!(&r, &[2, 4, 7, 8, 13, 14, 15, 16, 0, 0, 0, 0, 0, 0, 0, 0]); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_mask_compressstoreu_epi16() { + let a = _mm_set_epi16(8, 7, 6, 5, 4, 3, 2, 1); + let mut r = [0_i16; 8]; + _mm_mask_compressstoreu_epi16(r.as_mut_ptr() as *mut _, 0, a); + assert_eq!(&r, &[0_i16; 8]); + _mm_mask_compressstoreu_epi16(r.as_mut_ptr() as *mut _, 0b11110000, a); + assert_eq!(&r, &[5, 6, 7, 8, 0, 0, 0, 0]); + } + + #[simd_test(enable = "avx512vbmi2")] + unsafe fn test_mm512_mask_compressstoreu_epi8() { + let a = _mm512_set_epi8( + 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, + 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, + 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, + ); + let mut r = [0_i8; 64]; + _mm512_mask_compressstoreu_epi8(r.as_mut_ptr() as *mut _, 0, a); + assert_eq!(&r, &[0_i8; 64]); + _mm512_mask_compressstoreu_epi8( + r.as_mut_ptr() as *mut _, + 0b11110000_11001010_11111111_00000000_10101010_01010101_11110000_00001111, + a, + ); + assert_eq!( + &r, + &[ + 1, 2, 3, 4, 13, 14, 15, 16, 17, 19, 21, 23, 26, 28, 30, 32, 41, 42, 43, 44, 45, 46, + 47, 48, 50, 52, 55, 56, 61, 62, 63, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + ] + ); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm256_mask_compressstoreu_epi8() { + let a = _mm256_set_epi8( + 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, + 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, + ); + let mut r = [0_i8; 32]; + _mm256_mask_compressstoreu_epi8(r.as_mut_ptr() as *mut _, 0, a); + assert_eq!(&r, &[0_i8; 32]); + _mm256_mask_compressstoreu_epi8( + r.as_mut_ptr() as *mut _, + 0b11110000_11001010_11111111_00000000, + a, + ); + assert_eq!( + &r, + &[ + 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 23, 24, 29, 30, 31, 32, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0 + ] + ); + } + + #[simd_test(enable = "avx512vbmi2,avx512vl")] + unsafe fn test_mm_mask_compressstoreu_epi8() { + let a = _mm_set_epi8(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + let mut r = [0_i8; 16]; + _mm_mask_compressstoreu_epi8(r.as_mut_ptr() as *mut _, 0, a); + assert_eq!(&r, &[0_i8; 16]); + _mm_mask_compressstoreu_epi8(r.as_mut_ptr() as *mut _, 0b11110000_11001010, a); + assert_eq!(&r, &[2, 4, 7, 8, 13, 14, 15, 16, 0, 0, 0, 0, 0, 0, 0, 0]); + } +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vnni.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vnni.rs new file mode 100644 index 000000000..ff2c773ec --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vnni.rs @@ -0,0 +1,939 @@ +use crate::{ + core_arch::{simd::*, simd_llvm::*, x86::*}, + mem::transmute, +}; + +#[cfg(test)] +use stdarch_test::assert_instr; + +/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in a with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results. Sum these 2 results with the corresponding 32-bit integer in src, and store the packed 32-bit results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_dpwssd_epi32&expand=2219) +#[inline] +#[target_feature(enable = "avx512vnni")] +#[cfg_attr(test, assert_instr(vpdpwssd))] +pub unsafe fn _mm512_dpwssd_epi32(src: __m512i, a: __m512i, b: __m512i) -> __m512i { + transmute(vpdpwssd(src.as_i32x16(), a.as_i32x16(), b.as_i32x16())) +} + +/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in a with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results. Sum these 2 results with the corresponding 32-bit integer in src, and store the packed 32-bit results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_dpwssd_epi32&expand=2220) +#[inline] +#[target_feature(enable = "avx512vnni")] +#[cfg_attr(test, assert_instr(vpdpwssd))] +pub unsafe fn _mm512_mask_dpwssd_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + let r = _mm512_dpwssd_epi32(src, a, b).as_i32x16(); + transmute(simd_select_bitmask(k, r, src.as_i32x16())) +} + +/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in a with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results. Sum these 2 results with the corresponding 32-bit integer in src, and store the packed 32-bit results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_dpwssd_epi32&expand=2221) +#[inline] +#[target_feature(enable = "avx512vnni")] +#[cfg_attr(test, assert_instr(vpdpwssd))] +pub unsafe fn _mm512_maskz_dpwssd_epi32( + k: __mmask16, + src: __m512i, + a: __m512i, + b: __m512i, +) -> __m512i { + let r = _mm512_dpwssd_epi32(src, a, b).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in a with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results. Sum these 2 results with the corresponding 32-bit integer in src, and store the packed 32-bit results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_dpwssd_epi32&expand=2216) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpwssd))] +pub unsafe fn _mm256_dpwssd_epi32(src: __m256i, a: __m256i, b: __m256i) -> __m256i { + transmute(vpdpwssd256(src.as_i32x8(), a.as_i32x8(), b.as_i32x8())) +} + +/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in a with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results. Sum these 2 results with the corresponding 32-bit integer in src, and store the packed 32-bit results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_dpwssd_epi32&expand=2217) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpwssd))] +pub unsafe fn _mm256_mask_dpwssd_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + let r = _mm256_dpwssd_epi32(src, a, b).as_i32x8(); + transmute(simd_select_bitmask(k, r, src.as_i32x8())) +} + +/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in a with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results. Sum these 2 results with the corresponding 32-bit integer in src, and store the packed 32-bit results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_dpwssd_epi32&expand=2218) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpwssd))] +pub unsafe fn _mm256_maskz_dpwssd_epi32( + k: __mmask8, + src: __m256i, + a: __m256i, + b: __m256i, +) -> __m256i { + let r = _mm256_dpwssd_epi32(src, a, b).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in a with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results. Sum these 2 results with the corresponding 32-bit integer in src, and store the packed 32-bit results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_dpwssd_epi32&expand=2213) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpwssd))] +pub unsafe fn _mm_dpwssd_epi32(src: __m128i, a: __m128i, b: __m128i) -> __m128i { + transmute(vpdpwssd128(src.as_i32x4(), a.as_i32x4(), b.as_i32x4())) +} + +/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in a with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results. Sum these 2 results with the corresponding 32-bit integer in src, and store the packed 32-bit results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_dpwssd_epi32&expand=2214) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpwssd))] +pub unsafe fn _mm_mask_dpwssd_epi32(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let r = _mm_dpwssd_epi32(src, a, b).as_i32x4(); + transmute(simd_select_bitmask(k, r, src.as_i32x4())) +} + +/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in a with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results. Sum these 2 results with the corresponding 32-bit integer in src, and store the packed 32-bit results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_dpwssd_epi32&expand=2215) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpwssd))] +pub unsafe fn _mm_maskz_dpwssd_epi32(k: __mmask8, src: __m128i, a: __m128i, b: __m128i) -> __m128i { + let r = _mm_dpwssd_epi32(src, a, b).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in a with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results. Sum these 2 results with the corresponding 32-bit integer in src using signed saturation, and store the packed 32-bit results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_dpwssds_epi32&expand=2228) +#[inline] +#[target_feature(enable = "avx512vnni")] +#[cfg_attr(test, assert_instr(vpdpwssds))] +pub unsafe fn _mm512_dpwssds_epi32(src: __m512i, a: __m512i, b: __m512i) -> __m512i { + transmute(vpdpwssds(src.as_i32x16(), a.as_i32x16(), b.as_i32x16())) +} + +/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in a with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results. Sum these 2 results with the corresponding 32-bit integer in src using signed saturation, and store the packed 32-bit results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_dpwssds_epi32&expand=2229) +#[inline] +#[target_feature(enable = "avx512vnni")] +#[cfg_attr(test, assert_instr(vpdpwssds))] +pub unsafe fn _mm512_mask_dpwssds_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + let r = _mm512_dpwssds_epi32(src, a, b).as_i32x16(); + transmute(simd_select_bitmask(k, r, src.as_i32x16())) +} + +/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in a with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results. Sum these 2 results with the corresponding 32-bit integer in src using signed saturation, and store the packed 32-bit results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_dpwssds_epi32&expand=2230) +#[inline] +#[target_feature(enable = "avx512vnni")] +#[cfg_attr(test, assert_instr(vpdpwssds))] +pub unsafe fn _mm512_maskz_dpwssds_epi32( + k: __mmask16, + src: __m512i, + a: __m512i, + b: __m512i, +) -> __m512i { + let r = _mm512_dpwssds_epi32(src, a, b).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in a with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results. Sum these 2 results with the corresponding 32-bit integer in src using signed saturation, and store the packed 32-bit results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_dpwssds_epi32&expand=2225) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpwssds))] +pub unsafe fn _mm256_dpwssds_epi32(src: __m256i, a: __m256i, b: __m256i) -> __m256i { + transmute(vpdpwssds256(src.as_i32x8(), a.as_i32x8(), b.as_i32x8())) +} + +/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in a with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results. Sum these 2 results with the corresponding 32-bit integer in src using signed saturation, and store the packed 32-bit results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_dpwssds_epi32&expand=2226) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpwssds))] +pub unsafe fn _mm256_mask_dpwssds_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + let r = _mm256_dpwssds_epi32(src, a, b).as_i32x8(); + transmute(simd_select_bitmask(k, r, src.as_i32x8())) +} + +/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in a with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results. Sum these 2 results with the corresponding 32-bit integer in src using signed saturation, and store the packed 32-bit results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_dpwssds_epi32&expand=2227) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpwssds))] +pub unsafe fn _mm256_maskz_dpwssds_epi32( + k: __mmask8, + src: __m256i, + a: __m256i, + b: __m256i, +) -> __m256i { + let r = _mm256_dpwssds_epi32(src, a, b).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in a with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results. Sum these 2 results with the corresponding 32-bit integer in src using signed saturation, and store the packed 32-bit results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_dpwssds_epi32&expand=2222) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpwssds))] +pub unsafe fn _mm_dpwssds_epi32(src: __m128i, a: __m128i, b: __m128i) -> __m128i { + transmute(vpdpwssds128(src.as_i32x4(), a.as_i32x4(), b.as_i32x4())) +} + +/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in a with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results. Sum these 2 results with the corresponding 32-bit integer in src using signed saturation, and store the packed 32-bit results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_dpwssds_epi32&expand=2223) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpwssds))] +pub unsafe fn _mm_mask_dpwssds_epi32(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let r = _mm_dpwssds_epi32(src, a, b).as_i32x4(); + transmute(simd_select_bitmask(k, r, src.as_i32x4())) +} + +/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in a with corresponding 16-bit integers in b, producing 2 intermediate signed 32-bit results. Sum these 2 results with the corresponding 32-bit integer in src using signed saturation, and store the packed 32-bit results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_dpwssds_epi32&expand=2224) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpwssds))] +pub unsafe fn _mm_maskz_dpwssds_epi32( + k: __mmask8, + src: __m128i, + a: __m128i, + b: __m128i, +) -> __m128i { + let r = _mm_dpwssds_epi32(src, a, b).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed 8-bit integers in b, producing 4 intermediate signed 16-bit results. Sum these 4 results with the corresponding 32-bit integer in src, and store the packed 32-bit results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_dpbusd_epi32&expand=2201) +#[inline] +#[target_feature(enable = "avx512vnni")] +#[cfg_attr(test, assert_instr(vpdpbusd))] +pub unsafe fn _mm512_dpbusd_epi32(src: __m512i, a: __m512i, b: __m512i) -> __m512i { + transmute(vpdpbusd(src.as_i32x16(), a.as_i32x16(), b.as_i32x16())) +} + +/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed 8-bit integers in b, producing 4 intermediate signed 16-bit results. Sum these 4 results with the corresponding 32-bit integer in src, and store the packed 32-bit results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_dpbusd_epi32&expand=2202) +#[inline] +#[target_feature(enable = "avx512vnni")] +#[cfg_attr(test, assert_instr(vpdpbusd))] +pub unsafe fn _mm512_mask_dpbusd_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + let r = _mm512_dpbusd_epi32(src, a, b).as_i32x16(); + transmute(simd_select_bitmask(k, r, src.as_i32x16())) +} + +/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed 8-bit integers in b, producing 4 intermediate signed 16-bit results. Sum these 4 results with the corresponding 32-bit integer in src, and store the packed 32-bit results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_dpbusd_epi32&expand=2203) +#[inline] +#[target_feature(enable = "avx512vnni")] +#[cfg_attr(test, assert_instr(vpdpbusd))] +pub unsafe fn _mm512_maskz_dpbusd_epi32( + k: __mmask16, + src: __m512i, + a: __m512i, + b: __m512i, +) -> __m512i { + let r = _mm512_dpbusd_epi32(src, a, b).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed 8-bit integers in b, producing 4 intermediate signed 16-bit results. Sum these 4 results with the corresponding 32-bit integer in src, and store the packed 32-bit results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_dpbusd_epi32&expand=2198) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpbusd))] +pub unsafe fn _mm256_dpbusd_epi32(src: __m256i, a: __m256i, b: __m256i) -> __m256i { + transmute(vpdpbusd256(src.as_i32x8(), a.as_i32x8(), b.as_i32x8())) +} + +/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed 8-bit integers in b, producing 4 intermediate signed 16-bit results. Sum these 4 results with the corresponding 32-bit integer in src, and store the packed 32-bit results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_dpbusd_epi32&expand=2199) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpbusd))] +pub unsafe fn _mm256_mask_dpbusd_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + let r = _mm256_dpbusd_epi32(src, a, b).as_i32x8(); + transmute(simd_select_bitmask(k, r, src.as_i32x8())) +} + +/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed 8-bit integers in b, producing 4 intermediate signed 16-bit results. Sum these 4 results with the corresponding 32-bit integer in src, and store the packed 32-bit results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_dpbusd_epi32&expand=2200) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpbusd))] +pub unsafe fn _mm256_maskz_dpbusd_epi32( + k: __mmask8, + src: __m256i, + a: __m256i, + b: __m256i, +) -> __m256i { + let r = _mm256_dpbusd_epi32(src, a, b).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed 8-bit integers in b, producing 4 intermediate signed 16-bit results. Sum these 4 results with the corresponding 32-bit integer in src, and store the packed 32-bit results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_dpbusd_epi32&expand=2195) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpbusd))] +pub unsafe fn _mm_dpbusd_epi32(src: __m128i, a: __m128i, b: __m128i) -> __m128i { + transmute(vpdpbusd128(src.as_i32x4(), a.as_i32x4(), b.as_i32x4())) +} + +/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed 8-bit integers in b, producing 4 intermediate signed 16-bit results. Sum these 4 results with the corresponding 32-bit integer in src, and store the packed 32-bit results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_dpbusd_epi32&expand=2196) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpbusd))] +pub unsafe fn _mm_mask_dpbusd_epi32(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let r = _mm_dpbusd_epi32(src, a, b).as_i32x4(); + transmute(simd_select_bitmask(k, r, src.as_i32x4())) +} + +/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed 8-bit integers in b, producing 4 intermediate signed 16-bit results. Sum these 4 results with the corresponding 32-bit integer in src, and store the packed 32-bit results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_dpbusd_epi32&expand=2197) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpbusd))] +pub unsafe fn _mm_maskz_dpbusd_epi32(k: __mmask8, src: __m128i, a: __m128i, b: __m128i) -> __m128i { + let r = _mm_dpbusd_epi32(src, a, b).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed 8-bit integers in b, producing 4 intermediate signed 16-bit results. Sum these 4 results with the corresponding 32-bit integer in src using signed saturation, and store the packed 32-bit results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_dpbusds_epi32&expand=2210) +#[inline] +#[target_feature(enable = "avx512vnni")] +#[cfg_attr(test, assert_instr(vpdpbusds))] +pub unsafe fn _mm512_dpbusds_epi32(src: __m512i, a: __m512i, b: __m512i) -> __m512i { + transmute(vpdpbusds(src.as_i32x16(), a.as_i32x16(), b.as_i32x16())) +} + +/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed 8-bit integers in b, producing 4 intermediate signed 16-bit results. Sum these 4 results with the corresponding 32-bit integer in src using signed saturation, and store the packed 32-bit results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_dpbusds_epi32&expand=2211) +#[inline] +#[target_feature(enable = "avx512vnni")] +#[cfg_attr(test, assert_instr(vpdpbusds))] +pub unsafe fn _mm512_mask_dpbusds_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + let r = _mm512_dpbusds_epi32(src, a, b).as_i32x16(); + transmute(simd_select_bitmask(k, r, src.as_i32x16())) +} + +/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed 8-bit integers in b, producing 4 intermediate signed 16-bit results. Sum these 4 results with the corresponding 32-bit integer in src using signed saturation, and store the packed 32-bit results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_dpbusds_epi32&expand=2212) +#[inline] +#[target_feature(enable = "avx512vnni")] +#[cfg_attr(test, assert_instr(vpdpbusds))] +pub unsafe fn _mm512_maskz_dpbusds_epi32( + k: __mmask16, + src: __m512i, + a: __m512i, + b: __m512i, +) -> __m512i { + let r = _mm512_dpbusds_epi32(src, a, b).as_i32x16(); + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed 8-bit integers in b, producing 4 intermediate signed 16-bit results. Sum these 4 results with the corresponding 32-bit integer in src using signed saturation, and store the packed 32-bit results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_dpbusds_epi32&expand=2207) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpbusds))] +pub unsafe fn _mm256_dpbusds_epi32(src: __m256i, a: __m256i, b: __m256i) -> __m256i { + transmute(vpdpbusds256(src.as_i32x8(), a.as_i32x8(), b.as_i32x8())) +} + +/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed 8-bit integers in b, producing 4 intermediate signed 16-bit results. Sum these 4 results with the corresponding 32-bit integer in src using signed saturation, and store the packed 32-bit results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_dpbusds_epi32&expand=2208) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpbusds))] +pub unsafe fn _mm256_mask_dpbusds_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + let r = _mm256_dpbusds_epi32(src, a, b).as_i32x8(); + transmute(simd_select_bitmask(k, r, src.as_i32x8())) +} + +/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed 8-bit integers in b, producing 4 intermediate signed 16-bit results. Sum these 4 results with the corresponding 32-bit integer in src using signed saturation, and store the packed 32-bit results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_dpbusds_epi32&expand=2209) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpbusds))] +pub unsafe fn _mm256_maskz_dpbusds_epi32( + k: __mmask8, + src: __m256i, + a: __m256i, + b: __m256i, +) -> __m256i { + let r = _mm256_dpbusds_epi32(src, a, b).as_i32x8(); + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, r, zero)) +} + +/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed 8-bit integers in b, producing 4 intermediate signed 16-bit results. Sum these 4 results with the corresponding 32-bit integer in src using signed saturation, and store the packed 32-bit results in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_dpbusds_epi32&expand=2204) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpbusds))] +pub unsafe fn _mm_dpbusds_epi32(src: __m128i, a: __m128i, b: __m128i) -> __m128i { + transmute(vpdpbusds128(src.as_i32x4(), a.as_i32x4(), b.as_i32x4())) +} + +/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed 8-bit integers in b, producing 4 intermediate signed 16-bit results. Sum these 4 results with the corresponding 32-bit integer in src using signed saturation, and store the packed 32-bit results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_dpbusds_epi32&expand=2205) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpbusds))] +pub unsafe fn _mm_mask_dpbusds_epi32(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { + let r = _mm_dpbusds_epi32(src, a, b).as_i32x4(); + transmute(simd_select_bitmask(k, r, src.as_i32x4())) +} + +/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding signed 8-bit integers in b, producing 4 intermediate signed 16-bit results. Sum these 4 results with the corresponding 32-bit integer in src using signed saturation, and store the packed 32-bit results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_dpbusds_epi32&expand=2206) +#[inline] +#[target_feature(enable = "avx512vnni,avx512vl")] +#[cfg_attr(test, assert_instr(vpdpbusds))] +pub unsafe fn _mm_maskz_dpbusds_epi32( + k: __mmask8, + src: __m128i, + a: __m128i, + b: __m128i, +) -> __m128i { + let r = _mm_dpbusds_epi32(src, a, b).as_i32x4(); + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, r, zero)) +} + +#[allow(improper_ctypes)] +extern "C" { + #[link_name = "llvm.x86.avx512.vpdpwssd.512"] + fn vpdpwssd(src: i32x16, a: i32x16, b: i32x16) -> i32x16; + #[link_name = "llvm.x86.avx512.vpdpwssd.256"] + fn vpdpwssd256(src: i32x8, a: i32x8, b: i32x8) -> i32x8; + #[link_name = "llvm.x86.avx512.vpdpwssd.128"] + fn vpdpwssd128(src: i32x4, a: i32x4, b: i32x4) -> i32x4; + + #[link_name = "llvm.x86.avx512.vpdpwssds.512"] + fn vpdpwssds(src: i32x16, a: i32x16, b: i32x16) -> i32x16; + #[link_name = "llvm.x86.avx512.vpdpwssds.256"] + fn vpdpwssds256(src: i32x8, a: i32x8, b: i32x8) -> i32x8; + #[link_name = "llvm.x86.avx512.vpdpwssds.128"] + fn vpdpwssds128(src: i32x4, a: i32x4, b: i32x4) -> i32x4; + + #[link_name = "llvm.x86.avx512.vpdpbusd.512"] + fn vpdpbusd(src: i32x16, a: i32x16, b: i32x16) -> i32x16; + #[link_name = "llvm.x86.avx512.vpdpbusd.256"] + fn vpdpbusd256(src: i32x8, a: i32x8, b: i32x8) -> i32x8; + #[link_name = "llvm.x86.avx512.vpdpbusd.128"] + fn vpdpbusd128(src: i32x4, a: i32x4, b: i32x4) -> i32x4; + + #[link_name = "llvm.x86.avx512.vpdpbusds.512"] + fn vpdpbusds(src: i32x16, a: i32x16, b: i32x16) -> i32x16; + #[link_name = "llvm.x86.avx512.vpdpbusds.256"] + fn vpdpbusds256(src: i32x8, a: i32x8, b: i32x8) -> i32x8; + #[link_name = "llvm.x86.avx512.vpdpbusds.128"] + fn vpdpbusds128(src: i32x4, a: i32x4, b: i32x4) -> i32x4; +} + +#[cfg(test)] +mod tests { + + use crate::core_arch::x86::*; + use stdarch_test::simd_test; + + #[simd_test(enable = "avx512vnni")] + unsafe fn test_mm512_dpwssd_epi32() { + let src = _mm512_set1_epi32(1); + let a = _mm512_set1_epi32(1 << 16 | 1 << 0); + let b = _mm512_set1_epi32(1 << 16 | 1 << 0); + let r = _mm512_dpwssd_epi32(src, a, b); + let e = _mm512_set1_epi32(3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vnni")] + unsafe fn test_mm512_mask_dpwssd_epi32() { + let src = _mm512_set1_epi32(1); + let a = _mm512_set1_epi32(1 << 16 | 1 << 0); + let b = _mm512_set1_epi32(1 << 16 | 1 << 0); + let r = _mm512_mask_dpwssd_epi32(src, 0b00000000_00000000, a, b); + assert_eq_m512i(r, src); + let r = _mm512_mask_dpwssd_epi32(src, 0b11111111_11111111, a, b); + let e = _mm512_set1_epi32(3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vnni")] + unsafe fn test_mm512_maskz_dpwssd_epi32() { + let src = _mm512_set1_epi32(1); + let a = _mm512_set1_epi32(1 << 16 | 1 << 0); + let b = _mm512_set1_epi32(1 << 16 | 1 << 0); + let r = _mm512_maskz_dpwssd_epi32(0b00000000_00000000, src, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_dpwssd_epi32(0b11111111_11111111, src, a, b); + let e = _mm512_set1_epi32(3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm256_dpwssd_epi32() { + let src = _mm256_set1_epi32(1); + let a = _mm256_set1_epi32(1 << 16 | 1 << 0); + let b = _mm256_set1_epi32(1 << 16 | 1 << 0); + let r = _mm256_dpwssd_epi32(src, a, b); + let e = _mm256_set1_epi32(3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm256_mask_dpwssd_epi32() { + let src = _mm256_set1_epi32(1); + let a = _mm256_set1_epi32(1 << 16 | 1 << 0); + let b = _mm256_set1_epi32(1 << 16 | 1 << 0); + let r = _mm256_mask_dpwssd_epi32(src, 0b00000000, a, b); + assert_eq_m256i(r, src); + let r = _mm256_mask_dpwssd_epi32(src, 0b11111111, a, b); + let e = _mm256_set1_epi32(3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm256_maskz_dpwssd_epi32() { + let src = _mm256_set1_epi32(1); + let a = _mm256_set1_epi32(1 << 16 | 1 << 0); + let b = _mm256_set1_epi32(1 << 16 | 1 << 0); + let r = _mm256_maskz_dpwssd_epi32(0b00000000, src, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_dpwssd_epi32(0b11111111, src, a, b); + let e = _mm256_set1_epi32(3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm_dpwssd_epi32() { + let src = _mm_set1_epi32(1); + let a = _mm_set1_epi32(1 << 16 | 1 << 0); + let b = _mm_set1_epi32(1 << 16 | 1 << 0); + let r = _mm_dpwssd_epi32(src, a, b); + let e = _mm_set1_epi32(3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm_mask_dpwssd_epi32() { + let src = _mm_set1_epi32(1); + let a = _mm_set1_epi32(1 << 16 | 1 << 0); + let b = _mm_set1_epi32(1 << 16 | 1 << 0); + let r = _mm_mask_dpwssd_epi32(src, 0b00000000, a, b); + assert_eq_m128i(r, src); + let r = _mm_mask_dpwssd_epi32(src, 0b00001111, a, b); + let e = _mm_set1_epi32(3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm_maskz_dpwssd_epi32() { + let src = _mm_set1_epi32(1); + let a = _mm_set1_epi32(1 << 16 | 1 << 0); + let b = _mm_set1_epi32(1 << 16 | 1 << 0); + let r = _mm_maskz_dpwssd_epi32(0b00000000, src, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_dpwssd_epi32(0b00001111, src, a, b); + let e = _mm_set1_epi32(3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vnni")] + unsafe fn test_mm512_dpwssds_epi32() { + let src = _mm512_set1_epi32(1); + let a = _mm512_set1_epi32(1 << 16 | 1 << 0); + let b = _mm512_set1_epi32(1 << 16 | 1 << 0); + let r = _mm512_dpwssds_epi32(src, a, b); + let e = _mm512_set1_epi32(3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vnni")] + unsafe fn test_mm512_mask_dpwssds_epi32() { + let src = _mm512_set1_epi32(1); + let a = _mm512_set1_epi32(1 << 16 | 1 << 0); + let b = _mm512_set1_epi32(1 << 16 | 1 << 0); + let r = _mm512_mask_dpwssds_epi32(src, 0b00000000_00000000, a, b); + assert_eq_m512i(r, src); + let r = _mm512_mask_dpwssds_epi32(src, 0b11111111_11111111, a, b); + let e = _mm512_set1_epi32(3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vnni")] + unsafe fn test_mm512_maskz_dpwssds_epi32() { + let src = _mm512_set1_epi32(1); + let a = _mm512_set1_epi32(1 << 16 | 1 << 0); + let b = _mm512_set1_epi32(1 << 16 | 1 << 0); + let r = _mm512_maskz_dpwssds_epi32(0b00000000_00000000, src, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_dpwssds_epi32(0b11111111_11111111, src, a, b); + let e = _mm512_set1_epi32(3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm256_dpwssds_epi32() { + let src = _mm256_set1_epi32(1); + let a = _mm256_set1_epi32(1 << 16 | 1 << 0); + let b = _mm256_set1_epi32(1 << 16 | 1 << 0); + let r = _mm256_dpwssds_epi32(src, a, b); + let e = _mm256_set1_epi32(3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm256_mask_dpwssds_epi32() { + let src = _mm256_set1_epi32(1); + let a = _mm256_set1_epi32(1 << 16 | 1 << 0); + let b = _mm256_set1_epi32(1 << 16 | 1 << 0); + let r = _mm256_mask_dpwssds_epi32(src, 0b00000000, a, b); + assert_eq_m256i(r, src); + let r = _mm256_mask_dpwssds_epi32(src, 0b11111111, a, b); + let e = _mm256_set1_epi32(3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm256_maskz_dpwssds_epi32() { + let src = _mm256_set1_epi32(1); + let a = _mm256_set1_epi32(1 << 16 | 1 << 0); + let b = _mm256_set1_epi32(1 << 16 | 1 << 0); + let r = _mm256_maskz_dpwssds_epi32(0b00000000, src, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_dpwssds_epi32(0b11111111, src, a, b); + let e = _mm256_set1_epi32(3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm_dpwssds_epi32() { + let src = _mm_set1_epi32(1); + let a = _mm_set1_epi32(1 << 16 | 1 << 0); + let b = _mm_set1_epi32(1 << 16 | 1 << 0); + let r = _mm_dpwssds_epi32(src, a, b); + let e = _mm_set1_epi32(3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm_mask_dpwssds_epi32() { + let src = _mm_set1_epi32(1); + let a = _mm_set1_epi32(1 << 16 | 1 << 0); + let b = _mm_set1_epi32(1 << 16 | 1 << 0); + let r = _mm_mask_dpwssds_epi32(src, 0b00000000, a, b); + assert_eq_m128i(r, src); + let r = _mm_mask_dpwssds_epi32(src, 0b00001111, a, b); + let e = _mm_set1_epi32(3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm_maskz_dpwssds_epi32() { + let src = _mm_set1_epi32(1); + let a = _mm_set1_epi32(1 << 16 | 1 << 0); + let b = _mm_set1_epi32(1 << 16 | 1 << 0); + let r = _mm_maskz_dpwssds_epi32(0b00000000, src, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_dpwssds_epi32(0b00001111, src, a, b); + let e = _mm_set1_epi32(3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vnni")] + unsafe fn test_mm512_dpbusd_epi32() { + let src = _mm512_set1_epi32(1); + let a = _mm512_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let b = _mm512_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let r = _mm512_dpbusd_epi32(src, a, b); + let e = _mm512_set1_epi32(5); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vnni")] + unsafe fn test_mm512_mask_dpbusd_epi32() { + let src = _mm512_set1_epi32(1); + let a = _mm512_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let b = _mm512_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let r = _mm512_mask_dpbusd_epi32(src, 0b00000000_00000000, a, b); + assert_eq_m512i(r, src); + let r = _mm512_mask_dpbusd_epi32(src, 0b11111111_11111111, a, b); + let e = _mm512_set1_epi32(5); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vnni")] + unsafe fn test_mm512_maskz_dpbusd_epi32() { + let src = _mm512_set1_epi32(1); + let a = _mm512_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let b = _mm512_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let r = _mm512_maskz_dpbusd_epi32(0b00000000_00000000, src, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_dpbusd_epi32(0b11111111_11111111, src, a, b); + let e = _mm512_set1_epi32(5); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm256_dpbusd_epi32() { + let src = _mm256_set1_epi32(1); + let a = _mm256_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let b = _mm256_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let r = _mm256_dpbusd_epi32(src, a, b); + let e = _mm256_set1_epi32(5); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm256_mask_dpbusd_epi32() { + let src = _mm256_set1_epi32(1); + let a = _mm256_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let b = _mm256_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let r = _mm256_mask_dpbusd_epi32(src, 0b00000000, a, b); + assert_eq_m256i(r, src); + let r = _mm256_mask_dpbusd_epi32(src, 0b11111111, a, b); + let e = _mm256_set1_epi32(5); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm256_maskz_dpbusd_epi32() { + let src = _mm256_set1_epi32(1); + let a = _mm256_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let b = _mm256_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let r = _mm256_maskz_dpbusd_epi32(0b00000000, src, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_dpbusd_epi32(0b11111111, src, a, b); + let e = _mm256_set1_epi32(5); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm_dpbusd_epi32() { + let src = _mm_set1_epi32(1); + let a = _mm_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let b = _mm_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let r = _mm_dpbusd_epi32(src, a, b); + let e = _mm_set1_epi32(5); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm_mask_dpbusd_epi32() { + let src = _mm_set1_epi32(1); + let a = _mm_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let b = _mm_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let r = _mm_mask_dpbusd_epi32(src, 0b00000000, a, b); + assert_eq_m128i(r, src); + let r = _mm_mask_dpbusd_epi32(src, 0b00001111, a, b); + let e = _mm_set1_epi32(5); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm_maskz_dpbusd_epi32() { + let src = _mm_set1_epi32(1); + let a = _mm_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let b = _mm_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let r = _mm_maskz_dpbusd_epi32(0b00000000, src, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_dpbusd_epi32(0b00001111, src, a, b); + let e = _mm_set1_epi32(5); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vnni")] + unsafe fn test_mm512_dpbusds_epi32() { + let src = _mm512_set1_epi32(1); + let a = _mm512_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let b = _mm512_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let r = _mm512_dpbusds_epi32(src, a, b); + let e = _mm512_set1_epi32(5); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vnni")] + unsafe fn test_mm512_mask_dpbusds_epi32() { + let src = _mm512_set1_epi32(1); + let a = _mm512_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let b = _mm512_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let r = _mm512_mask_dpbusds_epi32(src, 0b00000000_00000000, a, b); + assert_eq_m512i(r, src); + let r = _mm512_mask_dpbusds_epi32(src, 0b11111111_11111111, a, b); + let e = _mm512_set1_epi32(5); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vnni")] + unsafe fn test_mm512_maskz_dpbusds_epi32() { + let src = _mm512_set1_epi32(1); + let a = _mm512_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let b = _mm512_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let r = _mm512_maskz_dpbusds_epi32(0b00000000_00000000, src, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_dpbusds_epi32(0b11111111_11111111, src, a, b); + let e = _mm512_set1_epi32(5); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm256_dpbusds_epi32() { + let src = _mm256_set1_epi32(1); + let a = _mm256_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let b = _mm256_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let r = _mm256_dpbusds_epi32(src, a, b); + let e = _mm256_set1_epi32(5); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm256_mask_dpbusds_epi32() { + let src = _mm256_set1_epi32(1); + let a = _mm256_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let b = _mm256_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let r = _mm256_mask_dpbusds_epi32(src, 0b00000000, a, b); + assert_eq_m256i(r, src); + let r = _mm256_mask_dpbusds_epi32(src, 0b11111111, a, b); + let e = _mm256_set1_epi32(5); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm256_maskz_dpbusds_epi32() { + let src = _mm256_set1_epi32(1); + let a = _mm256_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let b = _mm256_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let r = _mm256_maskz_dpbusds_epi32(0b00000000, src, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_dpbusds_epi32(0b11111111, src, a, b); + let e = _mm256_set1_epi32(5); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm_dpbusds_epi32() { + let src = _mm_set1_epi32(1); + let a = _mm_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let b = _mm_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let r = _mm_dpbusds_epi32(src, a, b); + let e = _mm_set1_epi32(5); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm_mask_dpbusds_epi32() { + let src = _mm_set1_epi32(1); + let a = _mm_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let b = _mm_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let r = _mm_mask_dpbusds_epi32(src, 0b00000000, a, b); + assert_eq_m128i(r, src); + let r = _mm_mask_dpbusds_epi32(src, 0b00001111, a, b); + let e = _mm_set1_epi32(5); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512vnni,avx512vl")] + unsafe fn test_mm_maskz_dpbusds_epi32() { + let src = _mm_set1_epi32(1); + let a = _mm_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let b = _mm_set1_epi32(1 << 24 | 1 << 16 | 1 << 8 | 1 << 0); + let r = _mm_maskz_dpbusds_epi32(0b00000000, src, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_dpbusds_epi32(0b00001111, src, a, b); + let e = _mm_set1_epi32(5); + assert_eq_m128i(r, e); + } +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vpclmulqdq.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vpclmulqdq.rs new file mode 100644 index 000000000..9bfeb903a --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vpclmulqdq.rs @@ -0,0 +1,258 @@ +//! Vectorized Carry-less Multiplication (VCLMUL) +//! +//! The reference is [Intel 64 and IA-32 Architectures Software Developer's +//! Manual Volume 2: Instruction Set Reference, A-Z][intel64_ref] (p. 4-241). +//! +//! [intel64_ref]: http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf + +use crate::core_arch::x86::__m256i; +use crate::core_arch::x86::__m512i; + +#[cfg(test)] +use stdarch_test::assert_instr; + +#[allow(improper_ctypes)] +extern "C" { + #[link_name = "llvm.x86.pclmulqdq.256"] + fn pclmulqdq_256(a: __m256i, round_key: __m256i, imm8: u8) -> __m256i; + #[link_name = "llvm.x86.pclmulqdq.512"] + fn pclmulqdq_512(a: __m512i, round_key: __m512i, imm8: u8) -> __m512i; +} + +// for some odd reason on x86_64 we generate the correct long name instructions +// but on i686 we generate the short name + imm8 +// so we need to special-case on that... + +/// Performs a carry-less multiplication of two 64-bit polynomials over the +/// finite field GF(2^k) - in each of the 4 128-bit lanes. +/// +/// The immediate byte is used for determining which halves of each lane `a` and `b` +/// should be used. Immediate bits other than 0 and 4 are ignored. +/// All lanes share immediate byte. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_clmulepi64_epi128) +#[inline] +#[target_feature(enable = "avx512vpclmulqdq,avx512f")] +// technically according to Intel's documentation we don't need avx512f here, however LLVM gets confused otherwise +#[cfg_attr(test, assert_instr(vpclmul, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_clmulepi64_epi128(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + pclmulqdq_512(a, b, IMM8 as u8) +} + +/// Performs a carry-less multiplication of two 64-bit polynomials over the +/// finite field GF(2^k) - in each of the 2 128-bit lanes. +/// +/// The immediate byte is used for determining which halves of each lane `a` and `b` +/// should be used. Immediate bits other than 0 and 4 are ignored. +/// All lanes share immediate byte. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_clmulepi64_epi128) +#[inline] +#[target_feature(enable = "avx512vpclmulqdq,avx512vl")] +#[cfg_attr(test, assert_instr(vpclmul, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_clmulepi64_epi128(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + pclmulqdq_256(a, b, IMM8 as u8) +} + +#[cfg(test)] +mod tests { + // The constants in the tests below are just bit patterns. They should not + // be interpreted as integers; signedness does not make sense for them, but + // __mXXXi happens to be defined in terms of signed integers. + #![allow(overflowing_literals)] + + use stdarch_test::simd_test; + + use crate::core_arch::x86::*; + + macro_rules! verify_kat_pclmul { + ($broadcast:ident, $clmul:ident, $assert:ident) => { + // Constants taken from https://software.intel.com/sites/default/files/managed/72/cc/clmul-wp-rev-2.02-2014-04-20.pdf + let a = _mm_set_epi64x(0x7b5b546573745665, 0x63746f725d53475d); + let a = $broadcast(a); + let b = _mm_set_epi64x(0x4869285368617929, 0x5b477565726f6e5d); + let b = $broadcast(b); + let r00 = _mm_set_epi64x(0x1d4d84c85c3440c0, 0x929633d5d36f0451); + let r00 = $broadcast(r00); + let r01 = _mm_set_epi64x(0x1bd17c8d556ab5a1, 0x7fa540ac2a281315); + let r01 = $broadcast(r01); + let r10 = _mm_set_epi64x(0x1a2bf6db3a30862f, 0xbabf262df4b7d5c9); + let r10 = $broadcast(r10); + let r11 = _mm_set_epi64x(0x1d1e1f2c592e7c45, 0xd66ee03e410fd4ed); + let r11 = $broadcast(r11); + + $assert($clmul::<0x00>(a, b), r00); + $assert($clmul::<0x10>(a, b), r01); + $assert($clmul::<0x01>(a, b), r10); + $assert($clmul::<0x11>(a, b), r11); + + let a0 = _mm_set_epi64x(0x0000000000000000, 0x8000000000000000); + let a0 = $broadcast(a0); + let r = _mm_set_epi64x(0x4000000000000000, 0x0000000000000000); + let r = $broadcast(r); + $assert($clmul::<0x00>(a0, a0), r); + } + } + + macro_rules! unroll { + ($target:ident[4] = $op:ident::<4>($source:ident);) => { + $target[3] = $op::<3>($source); + $target[2] = $op::<2>($source); + unroll! {$target[2] = $op::<2>($source);} + }; + ($target:ident[2] = $op:ident::<2>($source:ident);) => { + $target[1] = $op::<1>($source); + $target[0] = $op::<0>($source); + }; + (assert_eq_m128i($op:ident::<4>($vec_res:ident),$lin_res:ident[4]);) => { + assert_eq_m128i($op::<3>($vec_res), $lin_res[3]); + assert_eq_m128i($op::<2>($vec_res), $lin_res[2]); + unroll! {assert_eq_m128i($op::<2>($vec_res),$lin_res[2]);} + }; + (assert_eq_m128i($op:ident::<2>($vec_res:ident),$lin_res:ident[2]);) => { + assert_eq_m128i($op::<1>($vec_res), $lin_res[1]); + assert_eq_m128i($op::<0>($vec_res), $lin_res[0]); + }; + } + + // this function tests one of the possible 4 instances + // with different inputs across lanes + #[target_feature(enable = "avx512vpclmulqdq,avx512f")] + unsafe fn verify_512_helper( + linear: unsafe fn(__m128i, __m128i) -> __m128i, + vectorized: unsafe fn(__m512i, __m512i) -> __m512i, + ) { + let a = _mm512_set_epi64( + 0xDCB4DB3657BF0B7D, + 0x18DB0601068EDD9F, + 0xB76B908233200DC5, + 0xE478235FA8E22D5E, + 0xAB05CFFA2621154C, + 0x1171B47A186174C9, + 0x8C6B6C0E7595CEC9, + 0xBE3E7D4934E961BD, + ); + let b = _mm512_set_epi64( + 0x672F6F105A94CEA7, + 0x8298B8FFCA5F829C, + 0xA3927047B3FB61D8, + 0x978093862CDE7187, + 0xB1927AB22F31D0EC, + 0xA9A5DA619BE4D7AF, + 0xCA2590F56884FDC6, + 0x19BE9F660038BDB5, + ); + + let mut a_decomp = [_mm_setzero_si128(); 4]; + unroll! {a_decomp[4] = _mm512_extracti32x4_epi32::<4>(a);} + let mut b_decomp = [_mm_setzero_si128(); 4]; + unroll! {b_decomp[4] = _mm512_extracti32x4_epi32::<4>(b);} + + let r = vectorized(a, b); + let mut e_decomp = [_mm_setzero_si128(); 4]; + for i in 0..4 { + e_decomp[i] = linear(a_decomp[i], b_decomp[i]); + } + unroll! {assert_eq_m128i(_mm512_extracti32x4_epi32::<4>(r),e_decomp[4]);} + } + + // this function tests one of the possible 4 instances + // with different inputs across lanes for the VL version + #[target_feature(enable = "avx512vpclmulqdq,avx512vl")] + unsafe fn verify_256_helper( + linear: unsafe fn(__m128i, __m128i) -> __m128i, + vectorized: unsafe fn(__m256i, __m256i) -> __m256i, + ) { + let a = _mm512_set_epi64( + 0xDCB4DB3657BF0B7D, + 0x18DB0601068EDD9F, + 0xB76B908233200DC5, + 0xE478235FA8E22D5E, + 0xAB05CFFA2621154C, + 0x1171B47A186174C9, + 0x8C6B6C0E7595CEC9, + 0xBE3E7D4934E961BD, + ); + let b = _mm512_set_epi64( + 0x672F6F105A94CEA7, + 0x8298B8FFCA5F829C, + 0xA3927047B3FB61D8, + 0x978093862CDE7187, + 0xB1927AB22F31D0EC, + 0xA9A5DA619BE4D7AF, + 0xCA2590F56884FDC6, + 0x19BE9F660038BDB5, + ); + + let mut a_decomp = [_mm_setzero_si128(); 2]; + unroll! {a_decomp[2] = _mm512_extracti32x4_epi32::<2>(a);} + let mut b_decomp = [_mm_setzero_si128(); 2]; + unroll! {b_decomp[2] = _mm512_extracti32x4_epi32::<2>(b);} + + let r = vectorized( + _mm512_extracti64x4_epi64::<0>(a), + _mm512_extracti64x4_epi64::<0>(b), + ); + let mut e_decomp = [_mm_setzero_si128(); 2]; + for i in 0..2 { + e_decomp[i] = linear(a_decomp[i], b_decomp[i]); + } + unroll! {assert_eq_m128i(_mm256_extracti128_si256::<2>(r),e_decomp[2]);} + } + + #[simd_test(enable = "avx512vpclmulqdq,avx512f")] + unsafe fn test_mm512_clmulepi64_epi128() { + verify_kat_pclmul!( + _mm512_broadcast_i32x4, + _mm512_clmulepi64_epi128, + assert_eq_m512i + ); + + verify_512_helper( + |a, b| _mm_clmulepi64_si128::<0x00>(a, b), + |a, b| _mm512_clmulepi64_epi128::<0x00>(a, b), + ); + verify_512_helper( + |a, b| _mm_clmulepi64_si128::<0x01>(a, b), + |a, b| _mm512_clmulepi64_epi128::<0x01>(a, b), + ); + verify_512_helper( + |a, b| _mm_clmulepi64_si128::<0x10>(a, b), + |a, b| _mm512_clmulepi64_epi128::<0x10>(a, b), + ); + verify_512_helper( + |a, b| _mm_clmulepi64_si128::<0x11>(a, b), + |a, b| _mm512_clmulepi64_epi128::<0x11>(a, b), + ); + } + + #[simd_test(enable = "avx512vpclmulqdq,avx512vl")] + unsafe fn test_mm256_clmulepi64_epi128() { + verify_kat_pclmul!( + _mm256_broadcastsi128_si256, + _mm256_clmulepi64_epi128, + assert_eq_m256i + ); + + verify_256_helper( + |a, b| _mm_clmulepi64_si128::<0x00>(a, b), + |a, b| _mm256_clmulepi64_epi128::<0x00>(a, b), + ); + verify_256_helper( + |a, b| _mm_clmulepi64_si128::<0x01>(a, b), + |a, b| _mm256_clmulepi64_epi128::<0x01>(a, b), + ); + verify_256_helper( + |a, b| _mm_clmulepi64_si128::<0x10>(a, b), + |a, b| _mm256_clmulepi64_epi128::<0x10>(a, b), + ); + verify_256_helper( + |a, b| _mm_clmulepi64_si128::<0x11>(a, b), + |a, b| _mm256_clmulepi64_epi128::<0x11>(a, b), + ); + } +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vpopcntdq.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vpopcntdq.rs new file mode 100644 index 000000000..3b97c4c19 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/avx512vpopcntdq.rs @@ -0,0 +1,541 @@ +//! Vectorized Population Count Instructions for Double- and Quadwords (VPOPCNTDQ) +//! +//! The intrinsics here correspond to those in the `immintrin.h` C header. +//! +//! The reference is [Intel 64 and IA-32 Architectures Software Developer's +//! Manual Volume 2: Instruction Set Reference, A-Z][intel64_ref]. +//! +//! [intel64_ref]: http://www.intel.de/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf + +use crate::core_arch::simd::i32x16; +use crate::core_arch::simd::i32x4; +use crate::core_arch::simd::i32x8; +use crate::core_arch::simd::i64x2; +use crate::core_arch::simd::i64x4; +use crate::core_arch::simd::i64x8; +use crate::core_arch::simd_llvm::simd_select_bitmask; +use crate::core_arch::x86::__m128i; +use crate::core_arch::x86::__m256i; +use crate::core_arch::x86::__m512i; +use crate::core_arch::x86::__mmask16; +use crate::core_arch::x86::__mmask8; +use crate::core_arch::x86::_mm256_setzero_si256; +use crate::core_arch::x86::_mm512_setzero_si512; +use crate::core_arch::x86::_mm_setzero_si128; +use crate::core_arch::x86::m128iExt; +use crate::core_arch::x86::m256iExt; +use crate::core_arch::x86::m512iExt; +use crate::mem::transmute; + +#[cfg(test)] +use stdarch_test::assert_instr; + +#[allow(improper_ctypes)] +extern "C" { + #[link_name = "llvm.ctpop.v16i32"] + fn popcnt_v16i32(x: i32x16) -> i32x16; + #[link_name = "llvm.ctpop.v8i32"] + fn popcnt_v8i32(x: i32x8) -> i32x8; + #[link_name = "llvm.ctpop.v4i32"] + fn popcnt_v4i32(x: i32x4) -> i32x4; + + #[link_name = "llvm.ctpop.v8i64"] + fn popcnt_v8i64(x: i64x8) -> i64x8; + #[link_name = "llvm.ctpop.v4i64"] + fn popcnt_v4i64(x: i64x4) -> i64x4; + #[link_name = "llvm.ctpop.v2i64"] + fn popcnt_v2i64(x: i64x2) -> i64x2; +} + +/// For each packed 32-bit integer maps the value to the number of logical 1 bits. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_popcnt_epi32) +#[inline] +#[target_feature(enable = "avx512vpopcntdq")] +#[cfg_attr(test, assert_instr(vpopcntd))] +pub unsafe fn _mm512_popcnt_epi32(a: __m512i) -> __m512i { + transmute(popcnt_v16i32(a.as_i32x16())) +} + +/// For each packed 32-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_popcnt_epi32) +#[inline] +#[target_feature(enable = "avx512vpopcntdq")] +#[cfg_attr(test, assert_instr(vpopcntd))] +pub unsafe fn _mm512_maskz_popcnt_epi32(k: __mmask16, a: __m512i) -> __m512i { + let zero = _mm512_setzero_si512().as_i32x16(); + transmute(simd_select_bitmask(k, popcnt_v16i32(a.as_i32x16()), zero)) +} + +/// For each packed 32-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are copied from src if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_popcnt_epi32) +#[inline] +#[target_feature(enable = "avx512vpopcntdq")] +#[cfg_attr(test, assert_instr(vpopcntd))] +pub unsafe fn _mm512_mask_popcnt_epi32(src: __m512i, k: __mmask16, a: __m512i) -> __m512i { + transmute(simd_select_bitmask( + k, + popcnt_v16i32(a.as_i32x16()), + src.as_i32x16(), + )) +} + +/// For each packed 32-bit integer maps the value to the number of logical 1 bits. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_popcnt_epi32) +#[inline] +#[target_feature(enable = "avx512vpopcntdq,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntd))] +pub unsafe fn _mm256_popcnt_epi32(a: __m256i) -> __m256i { + transmute(popcnt_v8i32(a.as_i32x8())) +} + +/// For each packed 32-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_popcnt_epi32) +#[inline] +#[target_feature(enable = "avx512vpopcntdq,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntd))] +pub unsafe fn _mm256_maskz_popcnt_epi32(k: __mmask8, a: __m256i) -> __m256i { + let zero = _mm256_setzero_si256().as_i32x8(); + transmute(simd_select_bitmask(k, popcnt_v8i32(a.as_i32x8()), zero)) +} + +/// For each packed 32-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are copied from src if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_popcnt_epi32) +#[inline] +#[target_feature(enable = "avx512vpopcntdq,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntd))] +pub unsafe fn _mm256_mask_popcnt_epi32(src: __m256i, k: __mmask8, a: __m256i) -> __m256i { + transmute(simd_select_bitmask( + k, + popcnt_v8i32(a.as_i32x8()), + src.as_i32x8(), + )) +} + +/// For each packed 32-bit integer maps the value to the number of logical 1 bits. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_popcnt_epi32) +#[inline] +#[target_feature(enable = "avx512vpopcntdq,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntd))] +pub unsafe fn _mm_popcnt_epi32(a: __m128i) -> __m128i { + transmute(popcnt_v4i32(a.as_i32x4())) +} + +/// For each packed 32-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_popcnt_epi32) +#[inline] +#[target_feature(enable = "avx512vpopcntdq,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntd))] +pub unsafe fn _mm_maskz_popcnt_epi32(k: __mmask8, a: __m128i) -> __m128i { + let zero = _mm_setzero_si128().as_i32x4(); + transmute(simd_select_bitmask(k, popcnt_v4i32(a.as_i32x4()), zero)) +} + +/// For each packed 32-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are copied from src if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_popcnt_epi32) +#[inline] +#[target_feature(enable = "avx512vpopcntdq,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntd))] +pub unsafe fn _mm_mask_popcnt_epi32(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(simd_select_bitmask( + k, + popcnt_v4i32(a.as_i32x4()), + src.as_i32x4(), + )) +} + +/// For each packed 64-bit integer maps the value to the number of logical 1 bits. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_popcnt_epi64) +#[inline] +#[target_feature(enable = "avx512vpopcntdq")] +#[cfg_attr(test, assert_instr(vpopcntq))] +pub unsafe fn _mm512_popcnt_epi64(a: __m512i) -> __m512i { + transmute(popcnt_v8i64(a.as_i64x8())) +} + +/// For each packed 64-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_popcnt_epi64) +#[inline] +#[target_feature(enable = "avx512vpopcntdq")] +#[cfg_attr(test, assert_instr(vpopcntq))] +pub unsafe fn _mm512_maskz_popcnt_epi64(k: __mmask8, a: __m512i) -> __m512i { + let zero = _mm512_setzero_si512().as_i64x8(); + transmute(simd_select_bitmask(k, popcnt_v8i64(a.as_i64x8()), zero)) +} + +/// For each packed 64-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are copied from src if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_popcnt_epi64) +#[inline] +#[target_feature(enable = "avx512vpopcntdq")] +#[cfg_attr(test, assert_instr(vpopcntq))] +pub unsafe fn _mm512_mask_popcnt_epi64(src: __m512i, k: __mmask8, a: __m512i) -> __m512i { + transmute(simd_select_bitmask( + k, + popcnt_v8i64(a.as_i64x8()), + src.as_i64x8(), + )) +} + +/// For each packed 64-bit integer maps the value to the number of logical 1 bits. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_popcnt_epi64) +#[inline] +#[target_feature(enable = "avx512vpopcntdq,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntq))] +pub unsafe fn _mm256_popcnt_epi64(a: __m256i) -> __m256i { + transmute(popcnt_v4i64(a.as_i64x4())) +} + +/// For each packed 64-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_popcnt_epi64) +#[inline] +#[target_feature(enable = "avx512vpopcntdq,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntq))] +pub unsafe fn _mm256_maskz_popcnt_epi64(k: __mmask8, a: __m256i) -> __m256i { + let zero = _mm256_setzero_si256().as_i64x4(); + transmute(simd_select_bitmask(k, popcnt_v4i64(a.as_i64x4()), zero)) +} + +/// For each packed 64-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are copied from src if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_popcnt_epi64) +#[inline] +#[target_feature(enable = "avx512vpopcntdq,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntq))] +pub unsafe fn _mm256_mask_popcnt_epi64(src: __m256i, k: __mmask8, a: __m256i) -> __m256i { + transmute(simd_select_bitmask( + k, + popcnt_v4i64(a.as_i64x4()), + src.as_i64x4(), + )) +} + +/// For each packed 64-bit integer maps the value to the number of logical 1 bits. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_popcnt_epi64) +#[inline] +#[target_feature(enable = "avx512vpopcntdq,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntq))] +pub unsafe fn _mm_popcnt_epi64(a: __m128i) -> __m128i { + transmute(popcnt_v2i64(a.as_i64x2())) +} + +/// For each packed 64-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are zeroed in the result if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_popcnt_epi64) +#[inline] +#[target_feature(enable = "avx512vpopcntdq,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntq))] +pub unsafe fn _mm_maskz_popcnt_epi64(k: __mmask8, a: __m128i) -> __m128i { + let zero = _mm_setzero_si128().as_i64x2(); + transmute(simd_select_bitmask(k, popcnt_v2i64(a.as_i64x2()), zero)) +} + +/// For each packed 64-bit integer maps the value to the number of logical 1 bits. +/// +/// Uses the writemask in k - elements are copied from src if the corresponding mask bit is not set. +/// Otherwise the computation result is written into the result. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_popcnt_epi64) +#[inline] +#[target_feature(enable = "avx512vpopcntdq,avx512vl")] +#[cfg_attr(test, assert_instr(vpopcntq))] +pub unsafe fn _mm_mask_popcnt_epi64(src: __m128i, k: __mmask8, a: __m128i) -> __m128i { + transmute(simd_select_bitmask( + k, + popcnt_v2i64(a.as_i64x2()), + src.as_i64x2(), + )) +} + +#[cfg(test)] +mod tests { + use stdarch_test::simd_test; + + use crate::core_arch::x86::*; + + #[simd_test(enable = "avx512vpopcntdq,avx512f")] + unsafe fn test_mm512_popcnt_epi32() { + let test_data = _mm512_set_epi32( + 0, + 1, + -1, + 2, + 7, + 0xFF_FE, + 0x7F_FF_FF_FF, + -100, + 0x40_00_00_00, + 103, + 371, + 552, + 432_948, + 818_826_998, + 255, + 256, + ); + let actual_result = _mm512_popcnt_epi32(test_data); + let reference_result = + _mm512_set_epi32(0, 1, 32, 1, 3, 15, 31, 28, 1, 5, 6, 3, 10, 17, 8, 1); + assert_eq_m512i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512vpopcntdq,avx512f")] + unsafe fn test_mm512_mask_popcnt_epi32() { + let test_data = _mm512_set_epi32( + 0, + 1, + -1, + 2, + 7, + 0xFF_FE, + 0x7F_FF_FF_FF, + -100, + 0x40_00_00_00, + 103, + 371, + 552, + 432_948, + 818_826_998, + 255, + 256, + ); + let mask = 0xFF_00; + let actual_result = _mm512_mask_popcnt_epi32(test_data, mask, test_data); + let reference_result = _mm512_set_epi32( + 0, + 1, + 32, + 1, + 3, + 15, + 31, + 28, + 0x40_00_00_00, + 103, + 371, + 552, + 432_948, + 818_826_998, + 255, + 256, + ); + assert_eq_m512i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512vpopcntdq,avx512f")] + unsafe fn test_mm512_maskz_popcnt_epi32() { + let test_data = _mm512_set_epi32( + 0, + 1, + -1, + 2, + 7, + 0xFF_FE, + 0x7F_FF_FF_FF, + -100, + 0x40_00_00_00, + 103, + 371, + 552, + 432_948, + 818_826_998, + 255, + 256, + ); + let mask = 0xFF_00; + let actual_result = _mm512_maskz_popcnt_epi32(mask, test_data); + let reference_result = _mm512_set_epi32(0, 1, 32, 1, 3, 15, 31, 28, 0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")] + unsafe fn test_mm256_popcnt_epi32() { + let test_data = _mm256_set_epi32(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF, -100); + let actual_result = _mm256_popcnt_epi32(test_data); + let reference_result = _mm256_set_epi32(0, 1, 32, 1, 3, 15, 31, 28); + assert_eq_m256i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")] + unsafe fn test_mm256_mask_popcnt_epi32() { + let test_data = _mm256_set_epi32(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF, -100); + let mask = 0xF0; + let actual_result = _mm256_mask_popcnt_epi32(test_data, mask, test_data); + let reference_result = _mm256_set_epi32(0, 1, 32, 1, 7, 0xFF_FE, 0x7F_FF_FF_FF, -100); + assert_eq_m256i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")] + unsafe fn test_mm256_maskz_popcnt_epi32() { + let test_data = _mm256_set_epi32(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF, -100); + let mask = 0xF0; + let actual_result = _mm256_maskz_popcnt_epi32(mask, test_data); + let reference_result = _mm256_set_epi32(0, 1, 32, 1, 0, 0, 0, 0); + assert_eq_m256i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")] + unsafe fn test_mm_popcnt_epi32() { + let test_data = _mm_set_epi32(0, 1, -1, -100); + let actual_result = _mm_popcnt_epi32(test_data); + let reference_result = _mm_set_epi32(0, 1, 32, 28); + assert_eq_m128i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")] + unsafe fn test_mm_mask_popcnt_epi32() { + let test_data = _mm_set_epi32(0, 1, -1, -100); + let mask = 0xE; + let actual_result = _mm_mask_popcnt_epi32(test_data, mask, test_data); + let reference_result = _mm_set_epi32(0, 1, 32, -100); + assert_eq_m128i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")] + unsafe fn test_mm_maskz_popcnt_epi32() { + let test_data = _mm_set_epi32(0, 1, -1, -100); + let mask = 0xE; + let actual_result = _mm_maskz_popcnt_epi32(mask, test_data); + let reference_result = _mm_set_epi32(0, 1, 32, 0); + assert_eq_m128i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512vpopcntdq,avx512f")] + unsafe fn test_mm512_popcnt_epi64() { + let test_data = _mm512_set_epi64(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF_FF_FF_FF_FF, -100); + let actual_result = _mm512_popcnt_epi64(test_data); + let reference_result = _mm512_set_epi64(0, 1, 64, 1, 3, 15, 63, 60); + assert_eq_m512i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512vpopcntdq,avx512f")] + unsafe fn test_mm512_mask_popcnt_epi64() { + let test_data = _mm512_set_epi64(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF_FF_FF_FF_FF, -100); + let mask = 0xF0; + let actual_result = _mm512_mask_popcnt_epi64(test_data, mask, test_data); + let reference_result = + _mm512_set_epi64(0, 1, 64, 1, 7, 0xFF_FE, 0x7F_FF_FF_FF_FF_FF_FF_FF, -100); + assert_eq_m512i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512vpopcntdq,avx512f")] + unsafe fn test_mm512_maskz_popcnt_epi64() { + let test_data = _mm512_set_epi64(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF_FF_FF_FF_FF, -100); + let mask = 0xF0; + let actual_result = _mm512_maskz_popcnt_epi64(mask, test_data); + let reference_result = _mm512_set_epi64(0, 1, 64, 1, 0, 0, 0, 0); + assert_eq_m512i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512vpopcntdq,avx512vl")] + unsafe fn test_mm256_popcnt_epi64() { + let test_data = _mm256_set_epi64x(0, 1, -1, -100); + let actual_result = _mm256_popcnt_epi64(test_data); + let reference_result = _mm256_set_epi64x(0, 1, 64, 60); + assert_eq_m256i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512vpopcntdq,avx512vl")] + unsafe fn test_mm256_mask_popcnt_epi64() { + let test_data = _mm256_set_epi64x(0, 1, -1, -100); + let mask = 0xE; + let actual_result = _mm256_mask_popcnt_epi64(test_data, mask, test_data); + let reference_result = _mm256_set_epi64x(0, 1, 64, -100); + assert_eq_m256i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512vpopcntdq,avx512vl")] + unsafe fn test_mm256_maskz_popcnt_epi64() { + let test_data = _mm256_set_epi64x(0, 1, -1, -100); + let mask = 0xE; + let actual_result = _mm256_maskz_popcnt_epi64(mask, test_data); + let reference_result = _mm256_set_epi64x(0, 1, 64, 0); + assert_eq_m256i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512vpopcntdq,avx512vl")] + unsafe fn test_mm_popcnt_epi64() { + let test_data = _mm_set_epi64x(0, 1); + let actual_result = _mm_popcnt_epi64(test_data); + let reference_result = _mm_set_epi64x(0, 1); + assert_eq_m128i(actual_result, reference_result); + let test_data = _mm_set_epi64x(-1, -100); + let actual_result = _mm_popcnt_epi64(test_data); + let reference_result = _mm_set_epi64x(64, 60); + assert_eq_m128i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512vpopcntdq,avx512vl")] + unsafe fn test_mm_mask_popcnt_epi64() { + let test_data = _mm_set_epi64x(0, -100); + let mask = 0x2; + let actual_result = _mm_mask_popcnt_epi64(test_data, mask, test_data); + let reference_result = _mm_set_epi64x(0, -100); + assert_eq_m128i(actual_result, reference_result); + let test_data = _mm_set_epi64x(-1, 1); + let mask = 0x2; + let actual_result = _mm_mask_popcnt_epi64(test_data, mask, test_data); + let reference_result = _mm_set_epi64x(64, 1); + assert_eq_m128i(actual_result, reference_result); + } + + #[simd_test(enable = "avx512vpopcntdq,avx512vl")] + unsafe fn test_mm_maskz_popcnt_epi64() { + let test_data = _mm_set_epi64x(0, 1); + let mask = 0x2; + let actual_result = _mm_maskz_popcnt_epi64(mask, test_data); + let reference_result = _mm_set_epi64x(0, 0); + assert_eq_m128i(actual_result, reference_result); + let test_data = _mm_set_epi64x(-1, -100); + let mask = 0x2; + let actual_result = _mm_maskz_popcnt_epi64(mask, test_data); + let reference_result = _mm_set_epi64x(64, 0); + assert_eq_m128i(actual_result, reference_result); + } +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/bt.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/bt.rs index 6e42828dd..338cb97a3 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/bt.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/bt.rs @@ -1,55 +1,88 @@ +use crate::arch::asm; #[cfg(test)] use stdarch_test::assert_instr; +// x32 wants to use a 32-bit address size, but asm! defaults to using the full +// register name (e.g. rax). We have to explicitly override the placeholder to +// use the 32-bit register name in that case. +#[cfg(target_pointer_width = "32")] +macro_rules! bt { + ($inst:expr) => { + concat!($inst, " {b:e}, ({p:e})") + }; +} +#[cfg(target_pointer_width = "64")] +macro_rules! bt { + ($inst:expr) => { + concat!($inst, " {b:e}, ({p})") + }; +} + /// Returns the bit in position `b` of the memory addressed by `p`. #[inline] #[cfg_attr(test, assert_instr(bt))] -#[unstable(feature = "simd_x86_bittest", issue = "59414")] +#[stable(feature = "simd_x86_bittest", since = "1.55.0")] pub unsafe fn _bittest(p: *const i32, b: i32) -> u8 { let r: u8; - asm!("btl $2, $1\n\tsetc ${0:b}" - : "=r"(r) - : "*m"(p), "r"(b) - : "cc", "memory"); + asm!( + bt!("btl"), + "setc {r}", + p = in(reg) p, + b = in(reg) b, + r = out(reg_byte) r, + options(readonly, nostack, pure, att_syntax) + ); r } /// Returns the bit in position `b` of the memory addressed by `p`, then sets the bit to `1`. #[inline] #[cfg_attr(test, assert_instr(bts))] -#[unstable(feature = "simd_x86_bittest", issue = "59414")] +#[stable(feature = "simd_x86_bittest", since = "1.55.0")] pub unsafe fn _bittestandset(p: *mut i32, b: i32) -> u8 { let r: u8; - asm!("btsl $2, $1\n\tsetc ${0:b}" - : "=r"(r), "+*m"(p) - : "r"(b) - : "cc", "memory"); + asm!( + bt!("btsl"), + "setc {r}", + p = in(reg) p, + b = in(reg) b, + r = out(reg_byte) r, + options(nostack, att_syntax) + ); r } /// Returns the bit in position `b` of the memory addressed by `p`, then resets that bit to `0`. #[inline] #[cfg_attr(test, assert_instr(btr))] -#[unstable(feature = "simd_x86_bittest", issue = "59414")] +#[stable(feature = "simd_x86_bittest", since = "1.55.0")] pub unsafe fn _bittestandreset(p: *mut i32, b: i32) -> u8 { let r: u8; - asm!("btrl $2, $1\n\tsetc ${0:b}" - : "=r"(r), "+*m"(p) - : "r"(b) - : "cc", "memory"); + asm!( + bt!("btrl"), + "setc {r}", + p = in(reg) p, + b = in(reg) b, + r = out(reg_byte) r, + options(nostack, att_syntax) + ); r } /// Returns the bit in position `b` of the memory addressed by `p`, then inverts that bit. #[inline] #[cfg_attr(test, assert_instr(btc))] -#[unstable(feature = "simd_x86_bittest", issue = "59414")] +#[stable(feature = "simd_x86_bittest", since = "1.55.0")] pub unsafe fn _bittestandcomplement(p: *mut i32, b: i32) -> u8 { let r: u8; - asm!("btcl $2, $1\n\tsetc ${0:b}" - : "=r"(r), "+*m"(p) - : "r"(b) - : "cc", "memory"); + asm!( + bt!("btcl"), + "setc {r}", + p = in(reg) p, + b = in(reg) b, + r = out(reg_byte) r, + options(nostack, att_syntax) + ); r } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/cpuid.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/cpuid.rs index f313c4281..2624e8bdf 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/cpuid.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/cpuid.rs @@ -1,6 +1,7 @@ //! `cpuid` intrinsics #![allow(clippy::module_name_repetitions)] +use crate::arch::asm; #[cfg(test)] use stdarch_test::assert_instr; @@ -55,20 +56,34 @@ pub unsafe fn __cpuid_count(leaf: u32, sub_leaf: u32) -> CpuidResult { let ebx; let ecx; let edx; + + // LLVM sometimes reserves `ebx` for its internal use, we so we need to use + // a scratch register for it instead. #[cfg(target_arch = "x86")] { - asm!("cpuid" - : "={eax}"(eax), "={ebx}"(ebx), "={ecx}"(ecx), "={edx}"(edx) - : "{eax}"(leaf), "{ecx}"(sub_leaf) - : :); + asm!( + "mov {0}, ebx", + "cpuid", + "xchg {0}, ebx", + out(reg) ebx, + inout("eax") leaf => eax, + inout("ecx") sub_leaf => ecx, + out("edx") edx, + options(nostack, preserves_flags), + ); } #[cfg(target_arch = "x86_64")] { - // x86-64 uses %rbx as the base register, so preserve it. - asm!("cpuid" - : "={eax}"(eax), "={ebx}"(ebx), "={ecx}"(ecx), "={edx}"(edx) - : "{eax}"(leaf), "{ecx}"(sub_leaf) - : "rbx" :); + asm!( + "mov {0:r}, rbx", + "cpuid", + "xchg {0:r}, rbx", + out(reg) ebx, + inout("eax") leaf => eax, + inout("ecx") sub_leaf => ecx, + out("edx") edx, + options(nostack, preserves_flags), + ); } CpuidResult { eax, ebx, ecx, edx } } @@ -112,30 +127,25 @@ pub fn has_cpuid() -> bool { // the 21st bit of the EFLAGS register is modifiable or not. // If it is, then `cpuid` is available. let result: u32; - let _temp: u32; - asm!(r#" - # Read eflags into $0 and copy it into $1: - pushfd - pop $0 - mov $1, $0 - # Flip 21st bit of $0. - xor $0, 0x200000 - # Set eflags to the value of $0 - # - # Bit 21st can only be modified if cpuid is available - push $0 - popfd # A - # Read eflags into $0: - pushfd # B - pop $0 - # xor with the original eflags sets the bits that - # have been modified: - xor $0, $1 - "# - : "=r"(result), "=r"(_temp) - : - : "cc", "memory" - : "intel"); + asm!( + // Read eflags and save a copy of it + "pushfd", + "pop {result}", + "mov {result}, {saved_flags}", + // Flip 21st bit of the flags + "xor $0x200000, {result}", + // Load the modified flags and read them back. + // Bit 21 can only be modified if cpuid is available. + "push {result}", + "popfd", + "pushfd", + "pop {result}", + // Use xor to find out whether bit 21 has changed + "xor {saved_flags}, {result}", + result = out(reg) result, + saved_flags = out(reg) _, + options(nomem, att_syntax), + ); // There is a race between popfd (A) and pushfd (B) // where other bits beyond 21st may have been modified due to // interrupts, a debugger stepping through the asm, etc. diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/eflags.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/eflags.rs index 4d800fc4f..e9ebdf22b 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/eflags.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/eflags.rs @@ -1,19 +1,21 @@ //! `i386` intrinsics +use crate::arch::asm; + /// Reads EFLAGS. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=__readeflags) #[cfg(target_arch = "x86")] #[inline(always)] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_deprecated( +#[deprecated( since = "1.29.0", - reason = "See issue #51810 - use inline assembly instead" + note = "See issue #51810 - use inline assembly instead" )] #[doc(hidden)] pub unsafe fn __readeflags() -> u32 { let eflags: u32; - asm!("pushfd; popl $0" : "=r"(eflags) : : : "volatile"); + asm!("pushfd", "pop {}", out(reg) eflags, options(nomem, att_syntax)); eflags } @@ -23,14 +25,14 @@ pub unsafe fn __readeflags() -> u32 { #[cfg(target_arch = "x86_64")] #[inline(always)] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_deprecated( +#[deprecated( since = "1.29.0", - reason = "See issue #51810 - use inline assembly instead" + note = "See issue #51810 - use inline assembly instead" )] #[doc(hidden)] pub unsafe fn __readeflags() -> u64 { let eflags: u64; - asm!("pushfq; popq $0" : "=r"(eflags) : : : "volatile"); + asm!("pushfq", "pop {}", out(reg) eflags, options(nomem, att_syntax)); eflags } @@ -40,13 +42,13 @@ pub unsafe fn __readeflags() -> u64 { #[cfg(target_arch = "x86")] #[inline(always)] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_deprecated( +#[deprecated( since = "1.29.0", - reason = "See issue #51810 - use inline assembly instead" + note = "See issue #51810 - use inline assembly instead" )] #[doc(hidden)] pub unsafe fn __writeeflags(eflags: u32) { - asm!("pushl $0; popfd" : : "r"(eflags) : "cc", "flags" : "volatile"); + asm!("push {}", "popfd", in(reg) eflags, options(nomem, att_syntax)); } /// Write EFLAGS. @@ -55,13 +57,13 @@ pub unsafe fn __writeeflags(eflags: u32) { #[cfg(target_arch = "x86_64")] #[inline(always)] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_deprecated( +#[deprecated( since = "1.29.0", - reason = "See issue #51810 - use inline assembly instead" + note = "See issue #51810 - use inline assembly instead" )] #[doc(hidden)] pub unsafe fn __writeeflags(eflags: u64) { - asm!("pushq $0; popfq" : : "r"(eflags) : "cc", "flags" : "volatile"); + asm!("push {}", "popfq", in(reg) eflags, options(nomem, att_syntax)); } #[cfg(test)] diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/f16c.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/f16c.rs index 503bd41d2..8b25fd65e 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/f16c.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/f16c.rs @@ -4,7 +4,7 @@ use crate::{ core_arch::{simd::*, x86::*}, - hint::unreachable_unchecked, + // hint::unreachable_unchecked, mem::transmute, }; @@ -42,22 +42,6 @@ pub unsafe fn _mm256_cvtph_ps(a: __m128i) -> __m256 { transmute(llvm_vcvtph2ps_256(transmute(a))) } -macro_rules! dispatch_rounding { - ($rounding:ident, $call:ident) => {{ - match $rounding { - 0 => call!(0), - 1 => call!(1), - 2 => call!(2), - 3 => call!(3), - 4 => call!(4), - 5 => call!(5), - 6 => call!(6), - 7 => call!(7), - _ => unreachable_unchecked(), - } - }}; -} - /// Converts the 4 x 32-bit float values in the 128-bit vector `a` into 4 x /// 16-bit half-precision float values stored in the lowest 64-bit of a 128-bit /// vector. @@ -71,16 +55,13 @@ macro_rules! dispatch_rounding { /// * `_MM_FROUND_CUR_DIRECTION`: use `MXCSR.RC` - see [`_MM_SET_ROUNDING_MODE`]. #[inline] #[target_feature(enable = "f16c")] -#[rustc_args_required_const(1)] -#[cfg_attr(test, assert_instr("vcvtps2ph", imm_rounding = 0))] -pub unsafe fn _mm_cvtps_ph(a: __m128, imm_rounding: i32) -> __m128i { - let a = transmute(a); - macro_rules! call { - ($rounding:expr) => { - llvm_vcvtps2ph_128(a, $rounding) - }; - } - transmute(dispatch_rounding!(imm_rounding, call)) +#[cfg_attr(test, assert_instr("vcvtps2ph", IMM_ROUNDING = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtps_ph(a: __m128) -> __m128i { + static_assert_imm3!(IMM_ROUNDING); + let a = a.as_f32x4(); + let r = llvm_vcvtps2ph_128(a, IMM_ROUNDING); + transmute(r) } /// Converts the 8 x 32-bit float values in the 256-bit vector `a` into 8 x @@ -95,16 +76,13 @@ pub unsafe fn _mm_cvtps_ph(a: __m128, imm_rounding: i32) -> __m128i { /// * `_MM_FROUND_CUR_DIRECTION`: use `MXCSR.RC` - see [`_MM_SET_ROUNDING_MODE`]. #[inline] #[target_feature(enable = "f16c")] -#[rustc_args_required_const(1)] -#[cfg_attr(test, assert_instr("vcvtps2ph", imm_rounding = 0))] -pub unsafe fn _mm256_cvtps_ph(a: __m256, imm_rounding: i32) -> __m128i { - let a = transmute(a); - macro_rules! call { - ($rounding:expr) => { - llvm_vcvtps2ph_256(a, $rounding) - }; - } - transmute(dispatch_rounding!(imm_rounding, call)) +#[cfg_attr(test, assert_instr("vcvtps2ph", IMM_ROUNDING = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_cvtps_ph(a: __m256) -> __m128i { + static_assert_imm3!(IMM_ROUNDING); + let a = a.as_f32x8(); + let r = llvm_vcvtps2ph_256(a, IMM_ROUNDING); + transmute(r) } #[cfg(test)] @@ -116,7 +94,7 @@ mod tests { unsafe fn test_mm_cvtph_ps() { let array = [1_f32, 2_f32, 3_f32, 4_f32]; let float_vec: __m128 = transmute(array); - let halfs: __m128i = _mm_cvtps_ph(float_vec, 0); + let halfs: __m128i = _mm_cvtps_ph::<0>(float_vec); let floats: __m128 = _mm_cvtph_ps(halfs); let result: [f32; 4] = transmute(floats); assert_eq!(result, array); @@ -126,7 +104,7 @@ mod tests { unsafe fn test_mm256_cvtph_ps() { let array = [1_f32, 2_f32, 3_f32, 4_f32, 5_f32, 6_f32, 7_f32, 8_f32]; let float_vec: __m256 = transmute(array); - let halfs: __m128i = _mm256_cvtps_ph(float_vec, 0); + let halfs: __m128i = _mm256_cvtps_ph::<0>(float_vec); let floats: __m256 = _mm256_cvtph_ps(halfs); let result: [f32; 8] = transmute(floats); assert_eq!(result, array); diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/fma.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/fma.rs index f3dda6d52..476f4538c 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/fma.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/fma.rs @@ -500,7 +500,7 @@ extern "C" { #[cfg(test)] mod tests { - use std; + use stdarch_test::simd_test; use crate::core_arch::x86::*; diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/fxsr.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/fxsr.rs index 83d53f477..8ea1bfab7 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/fxsr.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/fxsr.rs @@ -1,4 +1,4 @@ -//! FXSR floating-point context fast save and restor. +//! FXSR floating-point context fast save and restore. #[cfg(test)] use stdarch_test::assert_instr; @@ -6,9 +6,9 @@ use stdarch_test::assert_instr; #[allow(improper_ctypes)] extern "C" { #[link_name = "llvm.x86.fxsave"] - fn fxsave(p: *mut u8) -> (); + fn fxsave(p: *mut u8); #[link_name = "llvm.x86.fxrstor"] - fn fxrstor(p: *const u8) -> (); + fn fxrstor(p: *const u8); } /// Saves the `x87` FPU, `MMX` technology, `XMM`, and `MXCSR` registers to the @@ -87,7 +87,7 @@ mod tests { } impl fmt::Debug for FxsaveArea { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "[")?; for i in 0..self.data.len() { write!(f, "{}", self.data[i])?; diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/macros.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/macros.rs index b8c283f1f..e686e65b3 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/macros.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/macros.rs @@ -1,94 +1,89 @@ //! Utility macros. +//! +// Helper struct used to trigger const eval errors when the const generic immediate value `imm` is +// not a round number. +pub(crate) struct ValidateConstRound; +impl ValidateConstRound { + pub(crate) const VALID: () = { + assert!( + IMM == 4 || IMM == 8 || IMM == 9 || IMM == 10 || IMM == 11, + "Invalid IMM value" + ); + }; +} -macro_rules! constify_imm6 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1_1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - 15 => $expand!(15), - 16 => $expand!(16), - 17 => $expand!(17), - 18 => $expand!(18), - 19 => $expand!(19), - 20 => $expand!(20), - 21 => $expand!(21), - 22 => $expand!(22), - 23 => $expand!(23), - 24 => $expand!(24), - 25 => $expand!(25), - 26 => $expand!(26), - 27 => $expand!(27), - 28 => $expand!(28), - 29 => $expand!(29), - 30 => $expand!(30), - _ => $expand!(31), - } +#[allow(unused)] +macro_rules! static_assert_rounding { + ($imm:ident) => { + let _ = $crate::core_arch::x86::macros::ValidateConstRound::<$imm>::VALID; }; } -macro_rules! constify_imm4 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - _ => $expand!(15), - } +// Helper struct used to trigger const eval errors when the const generic immediate value `imm` is +// not a sae number. +pub(crate) struct ValidateConstSae; +impl ValidateConstSae { + pub(crate) const VALID: () = { + assert!(IMM == 4 || IMM == 8, "Invalid IMM value"); }; } -macro_rules! constify_imm3 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - _ => $expand!(7), - } +#[allow(unused)] +macro_rules! static_assert_sae { + ($imm:ident) => { + let _ = $crate::core_arch::x86::macros::ValidateConstSae::<$imm>::VALID; + }; +} + +// Helper struct used to trigger const eval errors when the const generic immediate value `imm` is +// not a mantissas sae number. +pub(crate) struct ValidateConstMantissasSae; +impl ValidateConstMantissasSae { + pub(crate) const VALID: () = { + assert!(IMM == 4 || IMM == 8 || IMM == 12, "Invalid IMM value"); + }; +} + +#[allow(unused)] +macro_rules! static_assert_mantissas_sae { + ($imm:ident) => { + let _ = $crate::core_arch::x86::macros::ValidateConstMantissasSae::<$imm>::VALID; + }; +} + +// Helper struct used to trigger const eval errors when the unsigned const generic immediate value +// `IMM` is out of `[MIN-MAX]` range. +pub(crate) struct ValidateConstImmU32; +impl ValidateConstImmU32 { + pub(crate) const VALID: () = { + assert!(IMM >= MIN && IMM <= MAX, "IMM value not in expected range"); + }; +} + +#[allow(unused_macros)] +macro_rules! static_assert_imm_u8 { + ($imm:ident) => { + let _ = + $crate::core_arch::x86::macros::ValidateConstImmU32::<$imm, 0, { (1 << 8) - 1 }>::VALID; + }; +} + +// Helper struct used to trigger const eval errors when the const generic immediate value `SCALE` is +// not valid for gather instructions: the only valid scale values are 1, 2, 4 and 8. +pub(crate) struct ValidateConstGatherScale; +impl ValidateConstGatherScale { + pub(crate) const VALID: () = { + assert!( + SCALE == 1 || SCALE == 2 || SCALE == 4 || SCALE == 8, + "Invalid SCALE value" + ); }; } -macro_rules! constify_imm2 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b11 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - _ => $expand!(3), - } +#[allow(unused)] +macro_rules! static_assert_imm8_scale { + ($imm:ident) => { + let _ = $crate::core_arch::x86::macros::ValidateConstGatherScale::<$imm>::VALID; }; } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/mmx.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/mmx.rs deleted file mode 100644 index ff4f8277e..000000000 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/mmx.rs +++ /dev/null @@ -1,795 +0,0 @@ -//! `i586` MMX instruction set. -//! -//! The intrinsics here roughly correspond to those in the `mmintrin.h` C -//! header. -//! -//! The reference is [Intel 64 and IA-32 Architectures Software Developer's -//! Manual Volume 2: Instruction Set Reference, A-Z][intel64_ref]. -//! -//! [intel64_ref]: http://www.intel.de/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf - -use crate::{ - core_arch::{simd::*, x86::*}, - mem::transmute, -}; - -#[cfg(test)] -use stdarch_test::assert_instr; - -/// Constructs a 64-bit integer vector initialized to zero. -#[inline] -#[target_feature(enable = "mmx")] -// FIXME: this produces a movl instead of xorps on x86 -// FIXME: this produces a xor intrinsic instead of xorps on x86_64 -#[cfg_attr(all(test, target_arch = "x86_64"), assert_instr(xor))] -pub unsafe fn _mm_setzero_si64() -> __m64 { - transmute(0_i64) -} - -/// Adds packed 8-bit integers in `a` and `b`. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(paddb))] -pub unsafe fn _mm_add_pi8(a: __m64, b: __m64) -> __m64 { - paddb(a, b) -} - -/// Adds packed 8-bit integers in `a` and `b`. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(paddb))] -pub unsafe fn _m_paddb(a: __m64, b: __m64) -> __m64 { - _mm_add_pi8(a, b) -} - -/// Adds packed 16-bit integers in `a` and `b`. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(paddw))] -pub unsafe fn _mm_add_pi16(a: __m64, b: __m64) -> __m64 { - paddw(a, b) -} - -/// Adds packed 16-bit integers in `a` and `b`. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(paddw))] -pub unsafe fn _m_paddw(a: __m64, b: __m64) -> __m64 { - _mm_add_pi16(a, b) -} - -/// Adds packed 32-bit integers in `a` and `b`. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(paddd))] -pub unsafe fn _mm_add_pi32(a: __m64, b: __m64) -> __m64 { - paddd(a, b) -} - -/// Adds packed 32-bit integers in `a` and `b`. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(paddd))] -pub unsafe fn _m_paddd(a: __m64, b: __m64) -> __m64 { - _mm_add_pi32(a, b) -} - -/// Adds packed 8-bit integers in `a` and `b` using saturation. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(paddsb))] -pub unsafe fn _mm_adds_pi8(a: __m64, b: __m64) -> __m64 { - paddsb(a, b) -} - -/// Adds packed 8-bit integers in `a` and `b` using saturation. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(paddsb))] -pub unsafe fn _m_paddsb(a: __m64, b: __m64) -> __m64 { - _mm_adds_pi8(a, b) -} - -/// Adds packed 16-bit integers in `a` and `b` using saturation. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(paddsw))] -pub unsafe fn _mm_adds_pi16(a: __m64, b: __m64) -> __m64 { - paddsw(a, b) -} - -/// Adds packed 16-bit integers in `a` and `b` using saturation. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(paddsw))] -pub unsafe fn _m_paddsw(a: __m64, b: __m64) -> __m64 { - _mm_adds_pi16(a, b) -} - -/// Adds packed unsigned 8-bit integers in `a` and `b` using saturation. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(paddusb))] -pub unsafe fn _mm_adds_pu8(a: __m64, b: __m64) -> __m64 { - paddusb(a, b) -} - -/// Adds packed unsigned 8-bit integers in `a` and `b` using saturation. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(paddusb))] -pub unsafe fn _m_paddusb(a: __m64, b: __m64) -> __m64 { - _mm_adds_pu8(a, b) -} - -/// Adds packed unsigned 16-bit integers in `a` and `b` using saturation. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(paddusw))] -pub unsafe fn _mm_adds_pu16(a: __m64, b: __m64) -> __m64 { - paddusw(a, b) -} - -/// Adds packed unsigned 16-bit integers in `a` and `b` using saturation. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(paddusw))] -pub unsafe fn _m_paddusw(a: __m64, b: __m64) -> __m64 { - _mm_adds_pu16(a, b) -} - -/// Subtract packed 8-bit integers in `b` from packed 8-bit integers in `a`. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(psubb))] -pub unsafe fn _mm_sub_pi8(a: __m64, b: __m64) -> __m64 { - psubb(a, b) -} - -/// Subtract packed 8-bit integers in `b` from packed 8-bit integers in `a`. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(psubb))] -pub unsafe fn _m_psubb(a: __m64, b: __m64) -> __m64 { - _mm_sub_pi8(a, b) -} - -/// Subtract packed 16-bit integers in `b` from packed 16-bit integers in `a`. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(psubw))] -pub unsafe fn _mm_sub_pi16(a: __m64, b: __m64) -> __m64 { - psubw(a, b) -} - -/// Subtract packed 16-bit integers in `b` from packed 16-bit integers in `a`. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(psubw))] -pub unsafe fn _m_psubw(a: __m64, b: __m64) -> __m64 { - _mm_sub_pi16(a, b) -} - -/// Subtract packed 32-bit integers in `b` from packed 32-bit integers in `a`. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(psubd))] -pub unsafe fn _mm_sub_pi32(a: __m64, b: __m64) -> __m64 { - psubd(a, b) -} - -/// Subtract packed 32-bit integers in `b` from packed 32-bit integers in `a`. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(psubd))] -pub unsafe fn _m_psubd(a: __m64, b: __m64) -> __m64 { - _mm_sub_pi32(a, b) -} - -/// Subtract packed 8-bit integers in `b` from packed 8-bit integers in `a` -/// using saturation. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(psubsb))] -pub unsafe fn _mm_subs_pi8(a: __m64, b: __m64) -> __m64 { - psubsb(a, b) -} - -/// Subtract packed 8-bit integers in `b` from packed 8-bit integers in `a` -/// using saturation. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(psubsb))] -pub unsafe fn _m_psubsb(a: __m64, b: __m64) -> __m64 { - _mm_subs_pi8(a, b) -} - -/// Subtract packed 16-bit integers in `b` from packed 16-bit integers in `a` -/// using saturation. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(psubsw))] -pub unsafe fn _mm_subs_pi16(a: __m64, b: __m64) -> __m64 { - psubsw(a, b) -} - -/// Subtract packed 16-bit integers in `b` from packed 16-bit integers in `a` -/// using saturation. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(psubsw))] -pub unsafe fn _m_psubsw(a: __m64, b: __m64) -> __m64 { - _mm_subs_pi16(a, b) -} - -/// Subtract packed unsigned 8-bit integers in `b` from packed unsigned 8-bit -/// integers in `a` using saturation. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(psubusb))] -pub unsafe fn _mm_subs_pu8(a: __m64, b: __m64) -> __m64 { - psubusb(a, b) -} - -/// Subtract packed unsigned 8-bit integers in `b` from packed unsigned 8-bit -/// integers in `a` using saturation. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(psubusb))] -pub unsafe fn _m_psubusb(a: __m64, b: __m64) -> __m64 { - _mm_subs_pu8(a, b) -} - -/// Subtract packed unsigned 16-bit integers in `b` from packed unsigned -/// 16-bit integers in `a` using saturation. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(psubusw))] -pub unsafe fn _mm_subs_pu16(a: __m64, b: __m64) -> __m64 { - psubusw(a, b) -} - -/// Subtract packed unsigned 16-bit integers in `b` from packed unsigned -/// 16-bit integers in `a` using saturation. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(psubusw))] -pub unsafe fn _m_psubusw(a: __m64, b: __m64) -> __m64 { - _mm_subs_pu16(a, b) -} - -/// Converts packed 16-bit integers from `a` and `b` to packed 8-bit integers -/// using signed saturation. -/// -/// Positive values greater than 0x7F are saturated to 0x7F. Negative values -/// less than 0x80 are saturated to 0x80. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(packsswb))] -pub unsafe fn _mm_packs_pi16(a: __m64, b: __m64) -> __m64 { - packsswb(a, b) -} - -/// Converts packed 32-bit integers from `a` and `b` to packed 16-bit integers -/// using signed saturation. -/// -/// Positive values greater than 0x7F are saturated to 0x7F. Negative values -/// less than 0x80 are saturated to 0x80. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(packssdw))] -pub unsafe fn _mm_packs_pi32(a: __m64, b: __m64) -> __m64 { - packssdw(a, b) -} - -/// Compares whether each element of `a` is greater than the corresponding -/// element of `b` returning `0` for `false` and `-1` for `true`. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(pcmpgtb))] -pub unsafe fn _mm_cmpgt_pi8(a: __m64, b: __m64) -> __m64 { - pcmpgtb(a, b) -} - -/// Compares whether each element of `a` is greater than the corresponding -/// element of `b` returning `0` for `false` and `-1` for `true`. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(pcmpgtw))] -pub unsafe fn _mm_cmpgt_pi16(a: __m64, b: __m64) -> __m64 { - pcmpgtw(a, b) -} - -/// Compares whether each element of `a` is greater than the corresponding -/// element of `b` returning `0` for `false` and `-1` for `true`. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(pcmpgtd))] -pub unsafe fn _mm_cmpgt_pi32(a: __m64, b: __m64) -> __m64 { - pcmpgtd(a, b) -} - -/// Unpacks the upper two elements from two `i16x4` vectors and interleaves -/// them into the result: `[a.2, b.2, a.3, b.3]`. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(punpckhwd))] // FIXME punpcklbw expected -pub unsafe fn _mm_unpackhi_pi16(a: __m64, b: __m64) -> __m64 { - punpckhwd(a, b) -} - -/// Unpacks the upper four elements from two `i8x8` vectors and interleaves -/// them into the result: `[a.4, b.4, a.5, b.5, a.6, b.6, a.7, b.7]`. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(punpckhbw))] -pub unsafe fn _mm_unpackhi_pi8(a: __m64, b: __m64) -> __m64 { - punpckhbw(a, b) -} - -/// Unpacks the lower four elements from two `i8x8` vectors and interleaves -/// them into the result: `[a.0, b.0, a.1, b.1, a.2, b.2, a.3, b.3]`. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(punpcklbw))] -pub unsafe fn _mm_unpacklo_pi8(a: __m64, b: __m64) -> __m64 { - punpcklbw(a, b) -} - -/// Unpacks the lower two elements from two `i16x4` vectors and interleaves -/// them into the result: `[a.0 b.0 a.1 b.1]`. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(punpcklwd))] -pub unsafe fn _mm_unpacklo_pi16(a: __m64, b: __m64) -> __m64 { - punpcklwd(a, b) -} - -/// Unpacks the upper element from two `i32x2` vectors and interleaves them -/// into the result: `[a.1, b.1]`. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(punpckhdq))] -pub unsafe fn _mm_unpackhi_pi32(a: __m64, b: __m64) -> __m64 { - punpckhdq(a, b) -} - -/// Unpacks the lower element from two `i32x2` vectors and interleaves them -/// into the result: `[a.0, b.0]`. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(punpckldq))] -pub unsafe fn _mm_unpacklo_pi32(a: __m64, b: __m64) -> __m64 { - punpckldq(a, b) -} - -/// Sets packed 16-bit integers in dst with the supplied values. -#[inline] -#[target_feature(enable = "mmx")] -pub unsafe fn _mm_set_pi16(e3: i16, e2: i16, e1: i16, e0: i16) -> __m64 { - _mm_setr_pi16(e0, e1, e2, e3) -} - -/// Sets packed 32-bit integers in dst with the supplied values. -#[inline] -#[target_feature(enable = "mmx")] -pub unsafe fn _mm_set_pi32(e1: i32, e0: i32) -> __m64 { - _mm_setr_pi32(e0, e1) -} - -/// Sets packed 8-bit integers in dst with the supplied values. -#[inline] -#[target_feature(enable = "mmx")] -pub unsafe fn _mm_set_pi8(e7: i8, e6: i8, e5: i8, e4: i8, e3: i8, e2: i8, e1: i8, e0: i8) -> __m64 { - _mm_setr_pi8(e0, e1, e2, e3, e4, e5, e6, e7) -} - -/// Broadcasts 16-bit integer a to all all elements of dst. -#[inline] -#[target_feature(enable = "mmx")] -pub unsafe fn _mm_set1_pi16(a: i16) -> __m64 { - _mm_setr_pi16(a, a, a, a) -} - -/// Broadcasts 32-bit integer a to all all elements of dst. -#[inline] -#[target_feature(enable = "mmx")] -pub unsafe fn _mm_set1_pi32(a: i32) -> __m64 { - _mm_setr_pi32(a, a) -} - -/// Broadcasts 8-bit integer a to all all elements of dst. -#[inline] -#[target_feature(enable = "mmx")] -pub unsafe fn _mm_set1_pi8(a: i8) -> __m64 { - _mm_setr_pi8(a, a, a, a, a, a, a, a) -} - -/// Sets packed 16-bit integers in dst with the supplied values in reverse -/// order. -#[inline] -#[target_feature(enable = "mmx")] -pub unsafe fn _mm_setr_pi16(e0: i16, e1: i16, e2: i16, e3: i16) -> __m64 { - transmute(i16x4::new(e0, e1, e2, e3)) -} - -/// Sets packed 32-bit integers in dst with the supplied values in reverse -/// order. -#[inline] -#[target_feature(enable = "mmx")] -pub unsafe fn _mm_setr_pi32(e0: i32, e1: i32) -> __m64 { - transmute(i32x2::new(e0, e1)) -} - -/// Sets packed 8-bit integers in dst with the supplied values in reverse order. -#[inline] -#[target_feature(enable = "mmx")] -pub unsafe fn _mm_setr_pi8( - e0: i8, - e1: i8, - e2: i8, - e3: i8, - e4: i8, - e5: i8, - e6: i8, - e7: i8, -) -> __m64 { - transmute(i8x8::new(e0, e1, e2, e3, e4, e5, e6, e7)) -} - -/// Empty the MMX state, which marks the x87 FPU registers as available for use -/// by x87 instructions. This instruction must be used at the end of all MMX -/// technology procedures. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(emms))] -pub unsafe fn _mm_empty() { - emms() -} - -/// Empty the MMX state, which marks the x87 FPU registers as available for use -/// by x87 instructions. This instruction must be used at the end of all MMX -/// technology procedures. -#[inline] -#[target_feature(enable = "mmx")] -#[cfg_attr(test, assert_instr(emms))] -pub unsafe fn _m_empty() { - emms() -} - -/// Copies 32-bit integer `a` to the lower elements of the return value, and zero -/// the upper element of the return value. -#[inline] -#[target_feature(enable = "mmx")] -pub unsafe fn _mm_cvtsi32_si64(a: i32) -> __m64 { - transmute(i32x2::new(a, 0)) -} - -/// Return the lower 32-bit integer in `a`. -#[inline] -#[target_feature(enable = "mmx")] -pub unsafe fn _mm_cvtsi64_si32(a: __m64) -> i32 { - let r: i32x2 = transmute(a); - r.0 -} - -#[allow(improper_ctypes)] -extern "C" { - #[link_name = "llvm.x86.mmx.padd.b"] - fn paddb(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.padd.w"] - fn paddw(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.padd.d"] - fn paddd(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.padds.b"] - fn paddsb(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.padds.w"] - fn paddsw(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.paddus.b"] - fn paddusb(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.paddus.w"] - fn paddusw(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.psub.b"] - fn psubb(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.psub.w"] - fn psubw(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.psub.d"] - fn psubd(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.psubs.b"] - fn psubsb(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.psubs.w"] - fn psubsw(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.psubus.b"] - fn psubusb(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.psubus.w"] - fn psubusw(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.packsswb"] - fn packsswb(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.packssdw"] - fn packssdw(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.pcmpgt.b"] - fn pcmpgtb(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.pcmpgt.w"] - fn pcmpgtw(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.pcmpgt.d"] - fn pcmpgtd(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.punpckhwd"] - fn punpckhwd(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.punpcklwd"] - fn punpcklwd(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.punpckhbw"] - fn punpckhbw(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.punpcklbw"] - fn punpcklbw(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.punpckhdq"] - fn punpckhdq(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.punpckldq"] - fn punpckldq(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.emms"] - fn emms(); -} - -#[cfg(test)] -mod tests { - use crate::core_arch::x86::*; - use stdarch_test::simd_test; - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_setzero_si64() { - let r: __m64 = transmute(0_i64); - assert_eq_m64(r, _mm_setzero_si64()); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_add_pi8() { - let a = _mm_setr_pi8(-1, -1, 1, 1, -1, 0, 1, 0); - let b = _mm_setr_pi8(-127, 101, 99, 126, 0, -1, 0, 1); - let e = _mm_setr_pi8(-128, 100, 100, 127, -1, -1, 1, 1); - assert_eq_m64(e, _mm_add_pi8(a, b)); - assert_eq_m64(e, _m_paddb(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_add_pi16() { - let a = _mm_setr_pi16(-1, -1, 1, 1); - let b = _mm_setr_pi16(i16::min_value() + 1, 30001, -30001, i16::max_value() - 1); - let e = _mm_setr_pi16(i16::min_value(), 30000, -30000, i16::max_value()); - assert_eq_m64(e, _mm_add_pi16(a, b)); - assert_eq_m64(e, _m_paddw(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_add_pi32() { - let a = _mm_setr_pi32(1, -1); - let b = _mm_setr_pi32(i32::max_value() - 1, i32::min_value() + 1); - let e = _mm_setr_pi32(i32::max_value(), i32::min_value()); - assert_eq_m64(e, _mm_add_pi32(a, b)); - assert_eq_m64(e, _m_paddd(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_adds_pi8() { - let a = _mm_setr_pi8(-100, -1, 1, 100, -1, 0, 1, 0); - let b = _mm_setr_pi8(-100, 1, -1, 100, 0, -1, 0, 1); - let e = _mm_setr_pi8(i8::min_value(), 0, 0, i8::max_value(), -1, -1, 1, 1); - assert_eq_m64(e, _mm_adds_pi8(a, b)); - assert_eq_m64(e, _m_paddsb(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_adds_pi16() { - let a = _mm_setr_pi16(-32000, 32000, 4, 0); - let b = _mm_setr_pi16(-32000, 32000, -5, 1); - let e = _mm_setr_pi16(i16::min_value(), i16::max_value(), -1, 1); - assert_eq_m64(e, _mm_adds_pi16(a, b)); - assert_eq_m64(e, _m_paddsw(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_adds_pu8() { - let a = _mm_setr_pi8(0, 1, 2, 3, 4, 5, 6, 200u8 as i8); - let b = _mm_setr_pi8(0, 10, 20, 30, 40, 50, 60, 200u8 as i8); - let e = _mm_setr_pi8(0, 11, 22, 33, 44, 55, 66, u8::max_value() as i8); - assert_eq_m64(e, _mm_adds_pu8(a, b)); - assert_eq_m64(e, _m_paddusb(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_adds_pu16() { - let a = _mm_setr_pi16(0, 1, 2, 60000u16 as i16); - let b = _mm_setr_pi16(0, 10, 20, 60000u16 as i16); - let e = _mm_setr_pi16(0, 11, 22, u16::max_value() as i16); - assert_eq_m64(e, _mm_adds_pu16(a, b)); - assert_eq_m64(e, _m_paddusw(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_sub_pi8() { - let a = _mm_setr_pi8(0, 0, 1, 1, -1, -1, 0, 0); - let b = _mm_setr_pi8(-1, 1, -2, 2, 100, -100, -127, 127); - let e = _mm_setr_pi8(1, -1, 3, -1, -101, 99, 127, -127); - assert_eq_m64(e, _mm_sub_pi8(a, b)); - assert_eq_m64(e, _m_psubb(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_sub_pi16() { - let a = _mm_setr_pi16(-20000, -20000, 20000, 30000); - let b = _mm_setr_pi16(-10000, 10000, -10000, 30000); - let e = _mm_setr_pi16(-10000, -30000, 30000, 0); - assert_eq_m64(e, _mm_sub_pi16(a, b)); - assert_eq_m64(e, _m_psubw(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_sub_pi32() { - let a = _mm_setr_pi32(500_000, -500_000); - let b = _mm_setr_pi32(500_000, 500_000); - let e = _mm_setr_pi32(0, -1_000_000); - assert_eq_m64(e, _mm_sub_pi32(a, b)); - assert_eq_m64(e, _m_psubd(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_subs_pi8() { - let a = _mm_setr_pi8(-100, 100, 0, 0, 0, 0, -5, 5); - let b = _mm_setr_pi8(100, -100, i8::min_value(), 127, -1, 1, 3, -3); - let e = _mm_setr_pi8( - i8::min_value(), - i8::max_value(), - i8::max_value(), - -127, - 1, - -1, - -8, - 8, - ); - assert_eq_m64(e, _mm_subs_pi8(a, b)); - assert_eq_m64(e, _m_psubsb(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_subs_pi16() { - let a = _mm_setr_pi16(-20000, 20000, 0, 0); - let b = _mm_setr_pi16(20000, -20000, -1, 1); - let e = _mm_setr_pi16(i16::min_value(), i16::max_value(), 1, -1); - assert_eq_m64(e, _mm_subs_pi16(a, b)); - assert_eq_m64(e, _m_psubsw(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_subs_pu8() { - let a = _mm_setr_pi8(50, 10, 20, 30, 40, 60, 70, 80); - let b = _mm_setr_pi8(60, 20, 30, 40, 30, 20, 10, 0); - let e = _mm_setr_pi8(0, 0, 0, 0, 10, 40, 60, 80); - assert_eq_m64(e, _mm_subs_pu8(a, b)); - assert_eq_m64(e, _m_psubusb(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_subs_pu16() { - let a = _mm_setr_pi16(10000, 200, 0, 44444u16 as i16); - let b = _mm_setr_pi16(20000, 300, 1, 11111); - let e = _mm_setr_pi16(0, 0, 0, 33333u16 as i16); - assert_eq_m64(e, _mm_subs_pu16(a, b)); - assert_eq_m64(e, _m_psubusw(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_packs_pi16() { - let a = _mm_setr_pi16(-1, 2, -3, 4); - let b = _mm_setr_pi16(-5, 6, -7, 8); - let r = _mm_setr_pi8(-1, 2, -3, 4, -5, 6, -7, 8); - assert_eq_m64(r, _mm_packs_pi16(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_packs_pi32() { - let a = _mm_setr_pi32(-1, 2); - let b = _mm_setr_pi32(-5, 6); - let r = _mm_setr_pi16(-1, 2, -5, 6); - assert_eq_m64(r, _mm_packs_pi32(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_cmpgt_pi8() { - let a = _mm_setr_pi8(0, 1, 2, 3, 4, 5, 6, 7); - let b = _mm_setr_pi8(8, 7, 6, 5, 4, 3, 2, 1); - let r = _mm_setr_pi8(0, 0, 0, 0, 0, -1, -1, -1); - assert_eq_m64(r, _mm_cmpgt_pi8(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_cmpgt_pi16() { - let a = _mm_setr_pi16(0, 1, 2, 3); - let b = _mm_setr_pi16(4, 3, 2, 1); - let r = _mm_setr_pi16(0, 0, 0, -1); - assert_eq_m64(r, _mm_cmpgt_pi16(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_cmpgt_pi32() { - let a = _mm_setr_pi32(0, 3); - let b = _mm_setr_pi32(1, 2); - let r0 = _mm_setr_pi32(0, -1); - let r1 = _mm_setr_pi32(-1, 0); - - assert_eq_m64(r0, _mm_cmpgt_pi32(a, b)); - assert_eq_m64(r1, _mm_cmpgt_pi32(b, a)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_unpackhi_pi8() { - let a = _mm_setr_pi8(0, 3, 4, 7, 8, 11, 12, 15); - let b = _mm_setr_pi8(1, 2, 5, 6, 9, 10, 13, 14); - let r = _mm_setr_pi8(8, 9, 11, 10, 12, 13, 15, 14); - - assert_eq_m64(r, _mm_unpackhi_pi8(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_unpacklo_pi8() { - let a = _mm_setr_pi8(0, 1, 2, 3, 4, 5, 6, 7); - let b = _mm_setr_pi8(8, 9, 10, 11, 12, 13, 14, 15); - let r = _mm_setr_pi8(0, 8, 1, 9, 2, 10, 3, 11); - assert_eq_m64(r, _mm_unpacklo_pi8(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_unpackhi_pi16() { - let a = _mm_setr_pi16(0, 1, 2, 3); - let b = _mm_setr_pi16(4, 5, 6, 7); - let r = _mm_setr_pi16(2, 6, 3, 7); - assert_eq_m64(r, _mm_unpackhi_pi16(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_unpacklo_pi16() { - let a = _mm_setr_pi16(0, 1, 2, 3); - let b = _mm_setr_pi16(4, 5, 6, 7); - let r = _mm_setr_pi16(0, 4, 1, 5); - assert_eq_m64(r, _mm_unpacklo_pi16(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_unpackhi_pi32() { - let a = _mm_setr_pi32(0, 3); - let b = _mm_setr_pi32(1, 2); - let r = _mm_setr_pi32(3, 2); - - assert_eq_m64(r, _mm_unpackhi_pi32(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_unpacklo_pi32() { - let a = _mm_setr_pi32(0, 3); - let b = _mm_setr_pi32(1, 2); - let r = _mm_setr_pi32(0, 1); - - assert_eq_m64(r, _mm_unpacklo_pi32(a, b)); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_empty() { - _mm_empty(); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_m_empty() { - _m_empty(); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_cvtsi32_si64() { - let a = _mm_cvtsi32_si64(42); - let b = _mm_setr_pi32(42, 0); - assert_eq_m64(a, b); - } - - #[simd_test(enable = "mmx")] - unsafe fn test_mm_cvtsi64_si32() { - let a = _mm_setr_pi32(42, 666); - let b = _mm_cvtsi64_si32(a); - assert_eq!(b, 42); - } -} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/mod.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/mod.rs index 78a3e2317..6b50e95b2 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/mod.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/mod.rs @@ -6,50 +6,6 @@ use crate::{intrinsics, marker::Sized, mem::transmute}; mod macros; types! { - /// 64-bit wide integer vector type, x86-specific - /// - /// This type is the same as the `__m64` type defined by Intel, - /// representing a 64-bit SIMD register. Usage of this type typically - /// corresponds to the `mmx` target feature. - /// - /// Internally this type may be viewed as: - /// - /// * `i8x8` - eight `i8` variables packed together - /// * `i16x4` - four `i16` variables packed together - /// * `i32x2` - two `i32` variables packed together - /// - /// (as well as unsigned versions). Each intrinsic may interpret the - /// internal bits differently, check the documentation of the intrinsic - /// to see how it's being used. - /// - /// Note that this means that an instance of `__m64` typically just means - /// a "bag of bits" which is left up to interpretation at the point of use. - /// - /// Most intrinsics using `__m64` are prefixed with `_mm_` and the - /// integer types tend to correspond to suffixes like "pi8" or "pi32" (not - /// to be confused with "epiXX", used for `__m128i`). - /// - /// # Examples - /// - /// ``` - /// # #![feature(stdsimd, mmx_target_feature)] - /// #[cfg(target_arch = "x86")] - /// use std::arch::x86::*; - /// #[cfg(target_arch = "x86_64")] - /// use std::arch::x86_64::*; - /// - /// # fn main() { - /// # #[target_feature(enable = "mmx")] - /// # unsafe fn foo() { - /// let all_bytes_zero = _mm_setzero_si64(); - /// let all_bytes_one = _mm_set1_pi8(1); - /// let two_i32 = _mm_set_pi32(1, 2); - /// # } - /// # if is_x86_feature_detected!("mmx") { unsafe { foo() } } - /// # } - /// ``` - pub struct __m64(i64); - /// 128-bit wide integer vector type, x86-specific /// /// This type is the same as the `__m128i` type defined by Intel, @@ -340,62 +296,78 @@ types! { /// suffixed with "pd" (or otherwise contain "pd"). Not to be confused with /// "ps" which is used for `__m512`. pub struct __m512d(f64, f64, f64, f64, f64, f64, f64, f64); + + /// 128-bit wide set of eight 'u16' types, x86-specific + /// + /// This type is representing a 128-bit SIMD register which internally is consisted of + /// eight packed `u16` instances. Its purpose is for bf16 related intrinsic + /// implementations. + pub struct __m128bh(u16, u16, u16, u16, u16, u16, u16, u16); + + /// 256-bit wide set of 16 'u16' types, x86-specific + /// + /// This type is the same as the `__m256bh` type defined by Intel, + /// representing a 256-bit SIMD register which internally is consisted of + /// 16 packed `u16` instances. Its purpose is for bf16 related intrinsic + /// implementations. + pub struct __m256bh( + u16, u16, u16, u16, u16, u16, u16, u16, + u16, u16, u16, u16, u16, u16, u16, u16 + ); + + /// 512-bit wide set of 32 'u16' types, x86-specific + /// + /// This type is the same as the `__m512bh` type defined by Intel, + /// representing a 512-bit SIMD register which internally is consisted of + /// 32 packed `u16` instances. Its purpose is for bf16 related intrinsic + /// implementations. + pub struct __m512bh( + u16, u16, u16, u16, u16, u16, u16, u16, + u16, u16, u16, u16, u16, u16, u16, u16, + u16, u16, u16, u16, u16, u16, u16, u16, + u16, u16, u16, u16, u16, u16, u16, u16 + ); } -/// The `__mmask16` type used in AVX-512 intrinsics, a 16-bit integer +/// The `__mmask64` type used in AVX-512 intrinsics, a 64-bit integer #[allow(non_camel_case_types)] -pub type __mmask16 = i16; - -#[cfg(test)] -mod test; -#[cfg(test)] -pub use self::test::*; +pub type __mmask64 = u64; +/// The `__mmask32` type used in AVX-512 intrinsics, a 32-bit integer #[allow(non_camel_case_types)] -#[unstable(feature = "stdimd_internal", issue = "none")] -pub(crate) trait m64Ext: Sized { - fn as_m64(self) -> __m64; +pub type __mmask32 = u32; - #[inline] - fn as_u8x8(self) -> crate::core_arch::simd::u8x8 { - unsafe { transmute(self.as_m64()) } - } +/// The `__mmask16` type used in AVX-512 intrinsics, a 16-bit integer +#[allow(non_camel_case_types)] +pub type __mmask16 = u16; - #[inline] - fn as_u16x4(self) -> crate::core_arch::simd::u16x4 { - unsafe { transmute(self.as_m64()) } - } +/// The `__mmask8` type used in AVX-512 intrinsics, a 8-bit integer +#[allow(non_camel_case_types)] +pub type __mmask8 = u8; - #[inline] - fn as_u32x2(self) -> crate::core_arch::simd::u32x2 { - unsafe { transmute(self.as_m64()) } - } +/// The `_MM_CMPINT_ENUM` type used to specify comparison operations in AVX-512 intrinsics. +#[allow(non_camel_case_types)] +pub type _MM_CMPINT_ENUM = i32; - #[inline] - fn as_i8x8(self) -> crate::core_arch::simd::i8x8 { - unsafe { transmute(self.as_m64()) } - } +/// The `MM_MANTISSA_NORM_ENUM` type used to specify mantissa normalized operations in AVX-512 intrinsics. +#[allow(non_camel_case_types)] +pub type _MM_MANTISSA_NORM_ENUM = i32; - #[inline] - fn as_i16x4(self) -> crate::core_arch::simd::i16x4 { - unsafe { transmute(self.as_m64()) } - } +/// The `MM_MANTISSA_SIGN_ENUM` type used to specify mantissa signed operations in AVX-512 intrinsics. +#[allow(non_camel_case_types)] +pub type _MM_MANTISSA_SIGN_ENUM = i32; - #[inline] - fn as_i32x2(self) -> crate::core_arch::simd::i32x2 { - unsafe { transmute(self.as_m64()) } - } -} +/// The `MM_PERM_ENUM` type used to specify shuffle operations in AVX-512 intrinsics. +#[allow(non_camel_case_types)] +pub type _MM_PERM_ENUM = i32; -impl m64Ext for __m64 { - #[inline] - fn as_m64(self) -> Self { - self - } -} +#[cfg(test)] +mod test; +#[cfg(test)] +pub use self::test::*; #[allow(non_camel_case_types)] -#[unstable(feature = "stdimd_internal", issue = "none")] +#[unstable(feature = "stdsimd_internal", issue = "none")] pub(crate) trait m128iExt: Sized { fn as_m128i(self) -> __m128i; @@ -448,7 +420,7 @@ impl m128iExt for __m128i { } #[allow(non_camel_case_types)] -#[unstable(feature = "stdimd_internal", issue = "none")] +#[unstable(feature = "stdsimd_internal", issue = "none")] pub(crate) trait m256iExt: Sized { fn as_m256i(self) -> __m256i; @@ -501,14 +473,121 @@ impl m256iExt for __m256i { } #[allow(non_camel_case_types)] -#[unstable(feature = "stdimd_internal", issue = "none")] +#[unstable(feature = "stdsimd_internal", issue = "none")] +pub(crate) trait m128Ext: Sized { + fn as_m128(self) -> __m128; + + #[inline] + fn as_f32x4(self) -> crate::core_arch::simd::f32x4 { + unsafe { transmute(self.as_m128()) } + } +} + +impl m128Ext for __m128 { + #[inline] + fn as_m128(self) -> Self { + self + } +} + +#[allow(non_camel_case_types)] +#[unstable(feature = "stdsimd_internal", issue = "none")] +pub(crate) trait m128dExt: Sized { + fn as_m128d(self) -> __m128d; + + #[inline] + fn as_f64x2(self) -> crate::core_arch::simd::f64x2 { + unsafe { transmute(self.as_m128d()) } + } +} + +impl m128dExt for __m128d { + #[inline] + fn as_m128d(self) -> Self { + self + } +} + +#[allow(non_camel_case_types)] +#[unstable(feature = "stdsimd_internal", issue = "none")] +pub(crate) trait m256Ext: Sized { + fn as_m256(self) -> __m256; + + #[inline] + fn as_f32x8(self) -> crate::core_arch::simd::f32x8 { + unsafe { transmute(self.as_m256()) } + } +} + +impl m256Ext for __m256 { + #[inline] + fn as_m256(self) -> Self { + self + } +} + +#[allow(non_camel_case_types)] +#[unstable(feature = "stdsimd_internal", issue = "none")] +pub(crate) trait m256dExt: Sized { + fn as_m256d(self) -> __m256d; + + #[inline] + fn as_f64x4(self) -> crate::core_arch::simd::f64x4 { + unsafe { transmute(self.as_m256d()) } + } +} + +impl m256dExt for __m256d { + #[inline] + fn as_m256d(self) -> Self { + self + } +} + +#[allow(non_camel_case_types)] +#[unstable(feature = "stdsimd_internal", issue = "none")] pub(crate) trait m512iExt: Sized { fn as_m512i(self) -> __m512i; + #[inline] + fn as_u8x64(self) -> crate::core_arch::simd::u8x64 { + unsafe { transmute(self.as_m512i()) } + } + + #[inline] + fn as_i8x64(self) -> crate::core_arch::simd::i8x64 { + unsafe { transmute(self.as_m512i()) } + } + + #[inline] + fn as_u16x32(self) -> crate::core_arch::simd::u16x32 { + unsafe { transmute(self.as_m512i()) } + } + + #[inline] + fn as_i16x32(self) -> crate::core_arch::simd::i16x32 { + unsafe { transmute(self.as_m512i()) } + } + + #[inline] + fn as_u32x16(self) -> crate::core_arch::simd::u32x16 { + unsafe { transmute(self.as_m512i()) } + } + #[inline] fn as_i32x16(self) -> crate::core_arch::simd::i32x16 { unsafe { transmute(self.as_m512i()) } } + + #[inline] + fn as_u64x8(self) -> crate::core_arch::simd::u64x8 { + unsafe { transmute(self.as_m512i()) } + } + + #[inline] + fn as_i64x8(self) -> crate::core_arch::simd::i64x8 { + unsafe { transmute(self.as_m512i()) } + } } impl m512iExt for __m512i { @@ -518,6 +597,141 @@ impl m512iExt for __m512i { } } +#[allow(non_camel_case_types)] +#[unstable(feature = "stdsimd_internal", issue = "none")] +pub(crate) trait m512Ext: Sized { + fn as_m512(self) -> __m512; + + #[inline] + fn as_f32x16(self) -> crate::core_arch::simd::f32x16 { + unsafe { transmute(self.as_m512()) } + } +} + +impl m512Ext for __m512 { + #[inline] + fn as_m512(self) -> Self { + self + } +} + +#[allow(non_camel_case_types)] +#[unstable(feature = "stdsimd_internal", issue = "none")] +pub(crate) trait m512dExt: Sized { + fn as_m512d(self) -> __m512d; + + #[inline] + fn as_f64x8(self) -> crate::core_arch::simd::f64x8 { + unsafe { transmute(self.as_m512d()) } + } +} + +impl m512dExt for __m512d { + #[inline] + fn as_m512d(self) -> Self { + self + } +} + +#[allow(non_camel_case_types)] +#[unstable(feature = "stdsimd_internal", issue = "none")] +pub(crate) trait m128bhExt: Sized { + fn as_m128bh(self) -> __m128bh; + + #[inline] + fn as_u16x8(self) -> crate::core_arch::simd::u16x8 { + unsafe { transmute(self.as_m128bh()) } + } + + #[inline] + fn as_i16x8(self) -> crate::core_arch::simd::i16x8 { + unsafe { transmute(self.as_m128bh()) } + } + + #[inline] + fn as_u32x4(self) -> crate::core_arch::simd::u32x4 { + unsafe { transmute(self.as_m128bh()) } + } + + #[inline] + fn as_i32x4(self) -> crate::core_arch::simd::i32x4 { + unsafe { transmute(self.as_m128bh()) } + } +} + +impl m128bhExt for __m128bh { + #[inline] + fn as_m128bh(self) -> Self { + self + } +} + +#[allow(non_camel_case_types)] +#[unstable(feature = "stdsimd_internal", issue = "none")] +pub(crate) trait m256bhExt: Sized { + fn as_m256bh(self) -> __m256bh; + + #[inline] + fn as_u16x16(self) -> crate::core_arch::simd::u16x16 { + unsafe { transmute(self.as_m256bh()) } + } + + #[inline] + fn as_i16x16(self) -> crate::core_arch::simd::i16x16 { + unsafe { transmute(self.as_m256bh()) } + } + + #[inline] + fn as_u32x8(self) -> crate::core_arch::simd::u32x8 { + unsafe { transmute(self.as_m256bh()) } + } + + #[inline] + fn as_i32x8(self) -> crate::core_arch::simd::i32x8 { + unsafe { transmute(self.as_m256bh()) } + } +} + +impl m256bhExt for __m256bh { + #[inline] + fn as_m256bh(self) -> Self { + self + } +} + +#[allow(non_camel_case_types)] +#[unstable(feature = "stdsimd_internal", issue = "none")] +pub(crate) trait m512bhExt: Sized { + fn as_m512bh(self) -> __m512bh; + + #[inline] + fn as_u16x32(self) -> crate::core_arch::simd::u16x32 { + unsafe { transmute(self.as_m512bh()) } + } + + #[inline] + fn as_i16x32(self) -> crate::core_arch::simd::i16x32 { + unsafe { transmute(self.as_m512bh()) } + } + + #[inline] + fn as_u32x16(self) -> crate::core_arch::simd::u32x16 { + unsafe { transmute(self.as_m512bh()) } + } + + #[inline] + fn as_i32x16(self) -> crate::core_arch::simd::i32x16 { + unsafe { transmute(self.as_m512bh()) } + } +} + +impl m512bhExt for __m512bh { + #[inline] + fn as_m512bh(self) -> Self { + self + } +} + mod eflags; pub use self::eflags::*; @@ -572,9 +786,6 @@ mod tbm; #[cfg(not(stdarch_intel_sde))] pub use self::tbm::*; -mod mmx; -pub use self::mmx::*; - mod pclmulqdq; pub use self::pclmulqdq::*; @@ -603,9 +814,39 @@ pub unsafe fn ud2() -> ! { mod avx512f; pub use self::avx512f::*; +mod avx512bw; +pub use self::avx512bw::*; + +mod avx512cd; +pub use self::avx512cd::*; + mod avx512ifma; pub use self::avx512ifma::*; +mod avx512vbmi; +pub use self::avx512vbmi::*; + +mod avx512vbmi2; +pub use self::avx512vbmi2::*; + +mod avx512vnni; +pub use self::avx512vnni::*; + +mod avx512bitalg; +pub use self::avx512bitalg::*; + +mod avx512gfni; +pub use self::avx512gfni::*; + +mod avx512vpopcntdq; +pub use self::avx512vpopcntdq::*; + +mod avx512vaes; +pub use self::avx512vaes::*; + +mod avx512vpclmulqdq; +pub use self::avx512vpclmulqdq::*; + mod bt; pub use self::bt::*; @@ -614,3 +855,6 @@ pub use self::rtm::*; mod f16c; pub use self::f16c::*; + +mod avx512bf16; +pub use self::avx512bf16::*; diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/pclmulqdq.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/pclmulqdq.rs index 0e1bebae9..a2ebdf9c8 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/pclmulqdq.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/pclmulqdq.rs @@ -8,7 +8,7 @@ use crate::core_arch::x86::__m128i; #[cfg(test)] -use crate::stdarch_test::assert_instr; +use stdarch_test::assert_instr; #[allow(improper_ctypes)] extern "C" { @@ -25,20 +25,16 @@ extern "C" { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_clmulepi64_si128) #[inline] #[target_feature(enable = "pclmulqdq")] -#[cfg_attr(all(test, not(target_os = "linux")), assert_instr(pclmulqdq, imm8 = 0))] -#[cfg_attr(all(test, target_os = "linux"), assert_instr(pclmullqlqdq, imm8 = 0))] -#[cfg_attr(all(test, target_os = "linux"), assert_instr(pclmulhqlqdq, imm8 = 1))] -#[cfg_attr(all(test, target_os = "linux"), assert_instr(pclmullqhqdq, imm8 = 16))] -#[cfg_attr(all(test, target_os = "linux"), assert_instr(pclmulhqhqdq, imm8 = 17))] -#[rustc_args_required_const(2)] +#[cfg_attr(all(test, not(target_os = "linux")), assert_instr(pclmulqdq, IMM8 = 0))] +#[cfg_attr(all(test, target_os = "linux"), assert_instr(pclmullqlqdq, IMM8 = 0))] +#[cfg_attr(all(test, target_os = "linux"), assert_instr(pclmulhqlqdq, IMM8 = 1))] +#[cfg_attr(all(test, target_os = "linux"), assert_instr(pclmullqhqdq, IMM8 = 16))] +#[cfg_attr(all(test, target_os = "linux"), assert_instr(pclmulhqhqdq, IMM8 = 17))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_clmulepi64_si128(a: __m128i, b: __m128i, imm8: i32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - pclmulqdq(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +pub unsafe fn _mm_clmulepi64_si128(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + pclmulqdq(a, b, IMM8 as u8) } #[cfg(test)] @@ -62,13 +58,13 @@ mod tests { let r10 = _mm_set_epi64x(0x1a2bf6db3a30862f, 0xbabf262df4b7d5c9); let r11 = _mm_set_epi64x(0x1d1e1f2c592e7c45, 0xd66ee03e410fd4ed); - assert_eq_m128i(_mm_clmulepi64_si128(a, b, 0x00), r00); - assert_eq_m128i(_mm_clmulepi64_si128(a, b, 0x10), r01); - assert_eq_m128i(_mm_clmulepi64_si128(a, b, 0x01), r10); - assert_eq_m128i(_mm_clmulepi64_si128(a, b, 0x11), r11); + assert_eq_m128i(_mm_clmulepi64_si128::<0x00>(a, b), r00); + assert_eq_m128i(_mm_clmulepi64_si128::<0x10>(a, b), r01); + assert_eq_m128i(_mm_clmulepi64_si128::<0x01>(a, b), r10); + assert_eq_m128i(_mm_clmulepi64_si128::<0x11>(a, b), r11); let a0 = _mm_set_epi64x(0x0000000000000000, 0x8000000000000000); let r = _mm_set_epi64x(0x4000000000000000, 0x0000000000000000); - assert_eq_m128i(_mm_clmulepi64_si128(a0, a0, 0x00), r); + assert_eq_m128i(_mm_clmulepi64_si128::<0x00>(a0, a0), r); } } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/rtm.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/rtm.rs index 1e532b4ef..dab73cde9 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/rtm.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/rtm.rs @@ -20,9 +20,9 @@ extern "C" { #[link_name = "llvm.x86.xbegin"] fn x86_xbegin() -> i32; #[link_name = "llvm.x86.xend"] - fn x86_xend() -> (); + fn x86_xend(); #[link_name = "llvm.x86.xabort"] - fn x86_xabort(imm8: i8) -> (); + fn x86_xabort(imm8: i8); #[link_name = "llvm.x86.xtest"] fn x86_xtest() -> i32; } @@ -76,15 +76,11 @@ pub unsafe fn _xend() { /// [Intel's documentation](https://software.intel.com/en-us/cpp-compiler-developer-guide-and-reference-xabort). #[inline] #[target_feature(enable = "rtm")] -#[cfg_attr(test, assert_instr(xabort, imm8 = 0x0))] -#[rustc_args_required_const(0)] -pub unsafe fn _xabort(imm8: u32) { - macro_rules! call { - ($imm8:expr) => { - x86_xabort($imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(xabort, IMM8 = 0x0))] +#[rustc_legacy_const_generics(0)] +pub unsafe fn _xabort() { + static_assert_imm_u8!(IMM8); + x86_xabort(IMM8 as i8) } /// Queries whether the processor is executing in a transactional region identified by restricted @@ -130,14 +126,14 @@ mod tests { unsafe fn test_xabort() { const ABORT_CODE: u32 = 42; // aborting outside a transactional region does nothing - _xabort(ABORT_CODE); + _xabort::(); for _ in 0..10 { let mut x = 0; let code = rtm::_xbegin(); if code == _XBEGIN_STARTED { x += 1; - rtm::_xabort(ABORT_CODE); + rtm::_xabort::(); } else if code & _XABORT_EXPLICIT != 0 { let test_abort_code = rtm::_xabort_code(code); assert_eq!(test_abort_code, ABORT_CODE); diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/sha.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/sha.rs index 362a97ccd..cfb330cfb 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/sha.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/sha.rs @@ -66,25 +66,18 @@ pub unsafe fn _mm_sha1nexte_epu32(a: __m128i, b: __m128i) -> __m128i { /// Performs four rounds of SHA1 operation using an initial SHA1 state (A,B,C,D) /// from `a` and some pre-computed sum of the next 4 round message values /// (unsigned 32-bit integers), and state variable E from `b`, and return the -/// updated SHA1 state (A,B,C,D). `func` contains the logic functions and round +/// updated SHA1 state (A,B,C,D). `FUNC` contains the logic functions and round /// constants. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_sha1rnds4_epu32) #[inline] #[target_feature(enable = "sha")] -#[cfg_attr(test, assert_instr(sha1rnds4, func = 0))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(sha1rnds4, FUNC = 0))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_sha1rnds4_epu32(a: __m128i, b: __m128i, func: i32) -> __m128i { - let a = a.as_i32x4(); - let b = b.as_i32x4(); - macro_rules! call { - ($imm2:expr) => { - sha1rnds4(a, b, $imm2) - }; - } - let ret = constify_imm2!(func, call); - transmute(ret) +pub unsafe fn _mm_sha1rnds4_epu32(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm2!(FUNC); + transmute(sha1rnds4(a.as_i32x4(), b.as_i32x4(), FUNC as i8)) } /// Performs an intermediate calculation for the next four SHA256 message values @@ -179,19 +172,19 @@ mod tests { let a = _mm_set_epi64x(0xe9b5dba5b5c0fbcf, 0x71374491428a2f98); let b = _mm_set_epi64x(0xab1c5ed5923f82a4, 0x59f111f13956c25b); let expected = _mm_set_epi64x(0x32b13cd8322f5268, 0xc54420862bd9246f); - let r = _mm_sha1rnds4_epu32(a, b, 0); + let r = _mm_sha1rnds4_epu32::<0>(a, b); assert_eq_m128i(r, expected); let expected = _mm_set_epi64x(0x6d4c43e56a3c25d9, 0xa7e00fb775cbd3fe); - let r = _mm_sha1rnds4_epu32(a, b, 1); + let r = _mm_sha1rnds4_epu32::<1>(a, b); assert_eq_m128i(r, expected); let expected = _mm_set_epi64x(0xb304e383c01222f4, 0x66f6b3b1f89d8001); - let r = _mm_sha1rnds4_epu32(a, b, 2); + let r = _mm_sha1rnds4_epu32::<2>(a, b); assert_eq_m128i(r, expected); let expected = _mm_set_epi64x(0x8189b758bfabfa79, 0xdb08f6e78cae098b); - let r = _mm_sha1rnds4_epu32(a, b, 3); + let r = _mm_sha1rnds4_epu32::<3>(a, b); assert_eq_m128i(r, expected); } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse.rs index b07971c50..03c3a14a5 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse.rs @@ -148,8 +148,8 @@ pub unsafe fn _mm_rcp_ps(a: __m128) -> __m128 { rcpps(a) } -/// Returns the approximate reciprocal square root of the fist single-precision -/// (32-bit) floating-point elements in `a`, the other elements are unchanged. +/// Returns the approximate reciprocal square root of the first single-precision +/// (32-bit) floating-point element in `a`, the other elements are unchanged. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_rsqrt_ss) #[inline] @@ -350,7 +350,7 @@ pub unsafe fn _mm_cmple_ss(a: __m128, b: __m128) -> __m128 { #[cfg_attr(test, assert_instr(cmpltss))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cmpgt_ss(a: __m128, b: __m128) -> __m128 { - simd_shuffle4(a, cmpss(b, a, 1), [4, 1, 2, 3]) + simd_shuffle4!(a, cmpss(b, a, 1), [4, 1, 2, 3]) } /// Compares the lowest `f32` of both inputs for greater than or equal. The @@ -364,7 +364,7 @@ pub unsafe fn _mm_cmpgt_ss(a: __m128, b: __m128) -> __m128 { #[cfg_attr(test, assert_instr(cmpless))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cmpge_ss(a: __m128, b: __m128) -> __m128 { - simd_shuffle4(a, cmpss(b, a, 2), [4, 1, 2, 3]) + simd_shuffle4!(a, cmpss(b, a, 2), [4, 1, 2, 3]) } /// Compares the lowest `f32` of both inputs for inequality. The lowest 32 bits @@ -420,7 +420,7 @@ pub unsafe fn _mm_cmpnle_ss(a: __m128, b: __m128) -> __m128 { #[cfg_attr(test, assert_instr(cmpnltss))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cmpngt_ss(a: __m128, b: __m128) -> __m128 { - simd_shuffle4(a, cmpss(b, a, 5), [4, 1, 2, 3]) + simd_shuffle4!(a, cmpss(b, a, 5), [4, 1, 2, 3]) } /// Compares the lowest `f32` of both inputs for not-greater-than-or-equal. The @@ -434,7 +434,7 @@ pub unsafe fn _mm_cmpngt_ss(a: __m128, b: __m128) -> __m128 { #[cfg_attr(test, assert_instr(cmpnless))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cmpnge_ss(a: __m128, b: __m128) -> __m128 { - simd_shuffle4(a, cmpss(b, a, 6), [4, 1, 2, 3]) + simd_shuffle4!(a, cmpss(b, a, 6), [4, 1, 2, 3]) } /// Checks if the lowest `f32` of both inputs are ordered. The lowest 32 bits of @@ -790,7 +790,7 @@ pub unsafe fn _mm_ucomineq_ss(a: __m128, b: __m128) -> i32 { /// /// The result is rounded according to the current rounding mode. If the result /// cannot be represented as a 32 bit integer the result will be `0x8000_0000` -/// (`std::i32::MIN`) or an invalid operation floating point exception if +/// (`i32::MIN`) or an invalid operation floating point exception if /// unmasked (see [`_mm_setcsr`](fn._mm_setcsr.html)). /// /// This corresponds to the `CVTSS2SI` instruction (with 32 bit output). @@ -821,7 +821,7 @@ pub unsafe fn _mm_cvt_ss2si(a: __m128) -> i32 { /// /// The result is rounded always using truncation (round towards zero). If the /// result cannot be represented as a 32 bit integer the result will be -/// `0x8000_0000` (`std::i32::MIN`) or an invalid operation floating point +/// `0x8000_0000` (`i32::MIN`) or an invalid operation floating point /// exception if unmasked (see [`_mm_setcsr`](fn._mm_setcsr.html)). /// /// This corresponds to the `CVTTSS2SI` instruction (with 32 bit output). @@ -957,9 +957,15 @@ pub unsafe fn _mm_set_ps(a: f32, b: f32, c: f32, d: f32) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_setr_ps) #[inline] #[target_feature(enable = "sse")] -#[cfg_attr(all(test, target_arch = "x86_64"), assert_instr(unpcklps))] -// On a 32-bit architecture it just copies the operands from the stack. -#[cfg_attr(all(test, target_arch = "x86"), assert_instr(movaps))] +#[cfg_attr( + all(test, any(target_os = "windows", target_arch = "x86_64")), + assert_instr(unpcklps) +)] +// On a 32-bit architecture on non-Windows it just copies the operands from the stack. +#[cfg_attr( + all(test, all(not(target_os = "windows"), target_arch = "x86")), + assert_instr(movaps) +)] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_setr_ps(a: f32, b: f32, c: f32, d: f32) -> __m128 { __m128(a, b, c, d) @@ -986,61 +992,35 @@ pub const fn _MM_SHUFFLE(z: u32, y: u32, x: u32, w: u32) -> i32 { } /// Shuffles packed single-precision (32-bit) floating-point elements in `a` and -/// `b` using `mask`. +/// `b` using `MASK`. /// /// The lower half of result takes values from `a` and the higher half from /// `b`. Mask is split to 2 control bits each to index the element from inputs. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shuffle_ps) +/// +/// Note that there appears to be a mistake within Intel's Intrinsics Guide. +/// `_mm_shuffle_ps` is supposed to take an `i32` instead of a `u32` +/// as is the case for [other shuffle intrinsics](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shuffle_). +/// Performing an implicit type conversion between an unsigned integer and a signed integer +/// does not cause a problem in C, however Rust's commitment to strong typing does not allow this. #[inline] #[target_feature(enable = "sse")] -#[cfg_attr(test, assert_instr(shufps, mask = 3))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(shufps, MASK = 3))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_shuffle_ps(a: __m128, b: __m128, mask: i32) -> __m128 { - let mask = (mask & 0xFF) as u8; - - macro_rules! shuffle_done { - ($x01:expr, $x23:expr, $x45:expr, $x67:expr) => { - simd_shuffle4(a, b, [$x01, $x23, $x45, $x67]) - }; - } - macro_rules! shuffle_x67 { - ($x01:expr, $x23:expr, $x45:expr) => { - match (mask >> 6) & 0b11 { - 0b00 => shuffle_done!($x01, $x23, $x45, 4), - 0b01 => shuffle_done!($x01, $x23, $x45, 5), - 0b10 => shuffle_done!($x01, $x23, $x45, 6), - _ => shuffle_done!($x01, $x23, $x45, 7), - } - }; - } - macro_rules! shuffle_x45 { - ($x01:expr, $x23:expr) => { - match (mask >> 4) & 0b11 { - 0b00 => shuffle_x67!($x01, $x23, 4), - 0b01 => shuffle_x67!($x01, $x23, 5), - 0b10 => shuffle_x67!($x01, $x23, 6), - _ => shuffle_x67!($x01, $x23, 7), - } - }; - } - macro_rules! shuffle_x23 { - ($x01:expr) => { - match (mask >> 2) & 0b11 { - 0b00 => shuffle_x45!($x01, 0), - 0b01 => shuffle_x45!($x01, 1), - 0b10 => shuffle_x45!($x01, 2), - _ => shuffle_x45!($x01, 3), - } - }; - } - match mask & 0b11 { - 0b00 => shuffle_x23!(0), - 0b01 => shuffle_x23!(1), - 0b10 => shuffle_x23!(2), - _ => shuffle_x23!(3), - } +pub unsafe fn _mm_shuffle_ps(a: __m128, b: __m128) -> __m128 { + static_assert_imm8!(MASK); + simd_shuffle4!( + a, + b, + [ + MASK as u32 & 0b11, + (MASK as u32 >> 2) & 0b11, + ((MASK as u32 >> 4) & 0b11) + 4, + ((MASK as u32 >> 6) & 0b11) + 4, + ], + ) } /// Unpacks and interleave single-precision (32-bit) floating-point elements @@ -1052,7 +1032,7 @@ pub unsafe fn _mm_shuffle_ps(a: __m128, b: __m128, mask: i32) -> __m128 { #[cfg_attr(test, assert_instr(unpckhps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpackhi_ps(a: __m128, b: __m128) -> __m128 { - simd_shuffle4(a, b, [2, 6, 3, 7]) + simd_shuffle4!(a, b, [2, 6, 3, 7]) } /// Unpacks and interleave single-precision (32-bit) floating-point elements @@ -1064,7 +1044,7 @@ pub unsafe fn _mm_unpackhi_ps(a: __m128, b: __m128) -> __m128 { #[cfg_attr(test, assert_instr(unpcklps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpacklo_ps(a: __m128, b: __m128) -> __m128 { - simd_shuffle4(a, b, [0, 4, 1, 5]) + simd_shuffle4!(a, b, [0, 4, 1, 5]) } /// Combine higher half of `a` and `b`. The highwe half of `b` occupies the @@ -1077,7 +1057,7 @@ pub unsafe fn _mm_unpacklo_ps(a: __m128, b: __m128) -> __m128 { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_movehl_ps(a: __m128, b: __m128) -> __m128 { // TODO; figure why this is a different instruction on Windows? - simd_shuffle4(a, b, [6, 7, 2, 3]) + simd_shuffle4!(a, b, [6, 7, 2, 3]) } /// Combine lower half of `a` and `b`. The lower half of `b` occupies the @@ -1089,7 +1069,7 @@ pub unsafe fn _mm_movehl_ps(a: __m128, b: __m128) -> __m128 { #[cfg_attr(all(test, not(target_os = "windows")), assert_instr(movlhps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_movelh_ps(a: __m128, b: __m128) -> __m128 { - simd_shuffle4(a, b, [0, 1, 4, 5]) + simd_shuffle4!(a, b, [0, 1, 4, 5]) } /// Returns a mask of the most significant bit of each element in `a`. @@ -1109,33 +1089,6 @@ pub unsafe fn _mm_movemask_ps(a: __m128) -> i32 { movmskps(a) } -/// Sets the upper two single-precision floating-point values with 64 bits of -/// data loaded from the address `p`; the lower two values are passed through -/// from `a`. -#[inline] -#[target_feature(enable = "sse")] -#[cfg_attr(test, assert_instr(movhps))] -// TODO: this function is actually not limited to floats, but that's what -// what matches the C type most closely: `(__m128, *const __m64) -> __m128`. -pub unsafe fn _mm_loadh_pi(a: __m128, p: *const __m64) -> __m128 { - let q = p as *const f32x2; - let b: f32x2 = *q; - let bb = simd_shuffle4(b, b, [0, 1, 0, 1]); - simd_shuffle4(a, bb, [0, 1, 4, 5]) -} - -/// Loads two floats from `p` into the lower half of a `__m128`. The upper half -/// is copied from the upper half of `a`. -#[inline] -#[target_feature(enable = "sse")] -#[cfg_attr(test, assert_instr(movlps))] -pub unsafe fn _mm_loadl_pi(a: __m128, p: *const __m64) -> __m128 { - let q = p as *const f32x2; - let b: f32x2 = *q; - let bb = simd_shuffle4(b, b, [0, 1, 0, 1]); - simd_shuffle4(a, bb, [4, 5, 2, 3]) -} - /// Construct a `__m128` with the lowest element read from `p` and the other /// elements set to zero. /// @@ -1232,9 +1185,9 @@ pub unsafe fn _mm_loadu_ps(p: *const f32) -> __m128 { /// /// ```text /// let a0 = *p; -/// let a1 = *p.offset(1); -/// let a2 = *p.offset(2); -/// let a3 = *p.offset(3); +/// let a1 = *p.add(1); +/// let a2 = *p.add(2); +/// let a3 = *p.add(3); /// __m128::new(a3, a2, a1, a0) /// ``` /// @@ -1248,73 +1201,19 @@ pub unsafe fn _mm_loadu_ps(p: *const f32) -> __m128 { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_loadr_ps(p: *const f32) -> __m128 { let a = _mm_load_ps(p); - simd_shuffle4(a, a, [3, 2, 1, 0]) + simd_shuffle4!(a, a, [3, 2, 1, 0]) } -/// Stores the upper half of `a` (64 bits) into memory. +/// Loads unaligned 64-bits of integer data from memory into new vector. /// -/// This intrinsic corresponds to the `MOVHPS` instruction. The compiler may -/// choose to generate an equivalent sequence of other instructions. -#[inline] -#[target_feature(enable = "sse")] -// On i686 and up LLVM actually generates MOVHPD instead of MOVHPS, that's -// fine. -// On i586 (no SSE2) it just generates plain MOV instructions. -#[cfg_attr( - all(test, any(target_arch = "x86_64", target_feature = "sse2"), - not(target_os = "windows")), - // assert_instr(movhpd) - assert_instr(movhps) // LLVM7 prefers single-precision instructions -)] -pub unsafe fn _mm_storeh_pi(p: *mut __m64, a: __m128) { - #[cfg(target_arch = "x86")] - { - // If this is a `f64x2` then on i586, LLVM generates fldl & fstpl which - // is just silly - let a64: u64x2 = mem::transmute(a); - let a_hi = a64.extract(1); - *(p as *mut u64) = a_hi; - } - #[cfg(target_arch = "x86_64")] - { - // If this is a `u64x2` LLVM generates a pshufd + movq, but we really - // want a a MOVHPD or MOVHPS here. - let a64: f64x2 = mem::transmute(a); - let a_hi = a64.extract(1); - *p = mem::transmute(a_hi); - } -} - -/// Stores the lower half of `a` (64 bits) into memory. +/// `mem_addr` does not need to be aligned on any particular boundary. /// -/// This intrinsic corresponds to the `MOVQ` instruction. The compiler may -/// choose to generate an equivalent sequence of other instructions. +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadu_si64) #[inline] #[target_feature(enable = "sse")] -// On i586 the codegen just generates plane MOVs. No need to test for that. -#[cfg_attr( - all( - test, - any(target_arch = "x86_64", target_feature = "sse2"), - not(target_os = "windows") - ), - assert_instr(movlps) -)] -pub unsafe fn _mm_storel_pi(p: *mut __m64, a: __m128) { - #[cfg(target_arch = "x86")] - { - // Same as for _mm_storeh_pi: i586 code gen would use floating point - // stack. - let a64: u64x2 = mem::transmute(a); - let a_hi = a64.extract(0); - *(p as *mut u64) = a_hi; - } - #[cfg(target_arch = "x86_64")] - { - let a64: f64x2 = mem::transmute(a); - let a_hi = a64.extract(0); - *p = mem::transmute(a_hi); - } +#[stable(feature = "simd_x86_mm_loadu_si64", since = "1.46.0")] +pub unsafe fn _mm_loadu_si64(mem_addr: *const u8) -> __m128i { + transmute(i64x2(ptr::read_unaligned(mem_addr as *const i64), 0)) } /// Stores the lowest 32 bit float of `a` into memory. @@ -1342,9 +1241,9 @@ pub unsafe fn _mm_store_ss(p: *mut f32, a: __m128) { /// ```text /// let x = a.extract(0); /// *p = x; -/// *p.offset(1) = x; -/// *p.offset(2) = x; -/// *p.offset(3) = x; +/// *p.add(1) = x; +/// *p.add(2) = x; +/// *p.add(3) = x; /// ``` /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_store1_ps) @@ -1354,7 +1253,7 @@ pub unsafe fn _mm_store_ss(p: *mut f32, a: __m128) { #[stable(feature = "simd_x86", since = "1.27.0")] #[allow(clippy::cast_ptr_alignment)] pub unsafe fn _mm_store1_ps(p: *mut f32, a: __m128) { - let b: __m128 = simd_shuffle4(a, a, [0, 0, 0, 0]); + let b: __m128 = simd_shuffle4!(a, a, [0, 0, 0, 0]); *(p as *mut __m128) = b; } @@ -1418,9 +1317,9 @@ pub unsafe fn _mm_storeu_ps(p: *mut f32, a: __m128) { /// /// ```text /// *p = a.extract(3); -/// *p.offset(1) = a.extract(2); -/// *p.offset(2) = a.extract(1); -/// *p.offset(3) = a.extract(0); +/// *p.add(1) = a.extract(2); +/// *p.add(2) = a.extract(1); +/// *p.add(3) = a.extract(0); /// ``` /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_storer_ps) @@ -1430,7 +1329,7 @@ pub unsafe fn _mm_storeu_ps(p: *mut f32, a: __m128) { #[stable(feature = "simd_x86", since = "1.27.0")] #[allow(clippy::cast_ptr_alignment)] pub unsafe fn _mm_storer_ps(p: *mut f32, a: __m128) { - let b: __m128 = simd_shuffle4(a, a, [3, 2, 1, 0]); + let b: __m128 = simd_shuffle4!(a, a, [3, 2, 1, 0]); *(p as *mut __m128) = b; } @@ -1448,7 +1347,7 @@ pub unsafe fn _mm_storer_ps(p: *mut f32, a: __m128) { #[cfg_attr(test, assert_instr(movss))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_move_ss(a: __m128, b: __m128) -> __m128 { - simd_shuffle4(a, b, [4, 1, 2, 3]) + simd_shuffle4!(a, b, [4, 1, 2, 3]) } /// Performs a serializing operation on all store-to-memory instructions that @@ -1513,19 +1412,19 @@ pub unsafe fn _mm_getcsr() -> u32 { /// * `_MM_EXCEPT_DENORM`: An operation attempted to operate on a denormalized /// number. Mainly this can cause loss of precision. /// -/// * `_MM_EXCEPT_DIV_ZERO`: Division by zero occured. +/// * `_MM_EXCEPT_DIV_ZERO`: Division by zero occurred. /// -/// * `_MM_EXCEPT_OVERFLOW`: A numeric overflow exception occured, i.e., a +/// * `_MM_EXCEPT_OVERFLOW`: A numeric overflow exception occurred, i.e., a /// result was too large to be represented (e.g., an `f32` with absolute /// value /// greater than `2^128`). /// -/// * `_MM_EXCEPT_UNDERFLOW`: A numeric underflow exception occured, i.e., a +/// * `_MM_EXCEPT_UNDERFLOW`: A numeric underflow exception occurred, i.e., a /// result was too small to be represented in a normalized way (e.g., an /// `f32` /// with absulte value smaller than `2^-126`.) /// -/// * `_MM_EXCEPT_INEXACT`: An inexact-result exception occured (a.k.a. +/// * `_MM_EXCEPT_INEXACT`: An inexact-result exception occurred (a.k.a. /// precision exception). This means some precision was lost due to rounding. /// For example, the fraction `1/3` cannot be represented accurately in a /// 32 or 64 bit float and computing it would cause this exception to be @@ -1794,9 +1693,17 @@ pub const _MM_HINT_T2: i32 = 1; #[stable(feature = "simd_x86", since = "1.27.0")] pub const _MM_HINT_NTA: i32 = 0; -/// Fetch the cache line that contains address `p` using the given `strategy`. +/// See [`_mm_prefetch`](fn._mm_prefetch.html). +#[stable(feature = "simd_x86", since = "1.27.0")] +pub const _MM_HINT_ET0: i32 = 7; + +/// See [`_mm_prefetch`](fn._mm_prefetch.html). +#[stable(feature = "simd_x86", since = "1.27.0")] +pub const _MM_HINT_ET1: i32 = 6; + +/// Fetch the cache line that contains address `p` using the given `STRATEGY`. /// -/// The `strategy` must be one of: +/// The `STRATEGY` must be one of: /// /// * [`_MM_HINT_T0`](constant._MM_HINT_T0.html): Fetch into all levels of the /// cache hierarchy. @@ -1811,6 +1718,10 @@ pub const _MM_HINT_NTA: i32 = 0; /// but outside of the cache hierarchy. This is used to reduce access latency /// without polluting the cache. /// +/// * [`_MM_HINT_ET0`](constant._MM_HINT_ET0.html) and +/// [`_MM_HINT_ET1`](constant._MM_HINT_ET1.html) are similar to `_MM_HINT_T0` +/// and `_MM_HINT_T1` but indicate an anticipation to write to the address. +/// /// The actual implementation depends on the particular CPU. This instruction /// is considered a hint, so the CPU is also free to simply ignore the request. /// @@ -1834,28 +1745,16 @@ pub const _MM_HINT_NTA: i32 = 0; /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_prefetch) #[inline] #[target_feature(enable = "sse")] -#[cfg_attr(test, assert_instr(prefetcht0, strategy = _MM_HINT_T0))] -#[cfg_attr(test, assert_instr(prefetcht1, strategy = _MM_HINT_T1))] -#[cfg_attr(test, assert_instr(prefetcht2, strategy = _MM_HINT_T2))] -#[cfg_attr(test, assert_instr(prefetchnta, strategy = _MM_HINT_NTA))] -#[rustc_args_required_const(1)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_prefetch(p: *const i8, strategy: i32) { - // The `strategy` must be a compile-time constant, so we use a short form - // of `constify_imm8!` for now. - // We use the `llvm.prefetch` instrinsic with `rw` = 0 (read), and - // `cache type` = 1 (data cache). `locality` is based on our `strategy`. - macro_rules! pref { - ($imm8:expr) => { - match $imm8 { - 0 => prefetch(p, 0, 0, 1), - 1 => prefetch(p, 0, 1, 1), - 2 => prefetch(p, 0, 2, 1), - _ => prefetch(p, 0, 3, 1), - } - }; - } - pref!(strategy) +#[cfg_attr(test, assert_instr(prefetcht0, STRATEGY = _MM_HINT_T0))] +#[cfg_attr(test, assert_instr(prefetcht1, STRATEGY = _MM_HINT_T1))] +#[cfg_attr(test, assert_instr(prefetcht2, STRATEGY = _MM_HINT_T2))] +#[cfg_attr(test, assert_instr(prefetchnta, STRATEGY = _MM_HINT_NTA))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_prefetch(p: *const i8) { + // We use the `llvm.prefetch` intrinsic with `cache type` = 1 (data cache). + // `locality` and `rw` are based on our `STRATEGY`. + prefetch(p, (STRATEGY >> 2) & 1, STRATEGY & 3, 1); } /// Returns vector of type __m128 with undefined elements. @@ -1865,8 +1764,7 @@ pub unsafe fn _mm_prefetch(p: *const i8, strategy: i32) { #[target_feature(enable = "sse")] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_undefined_ps() -> __m128 { - // FIXME: this function should return MaybeUninit<__m128> - mem::MaybeUninit::<__m128>::uninit().assume_init() + _mm_set1_ps(0.0) } /// Transpose the 4x4 matrix formed by 4 rows of __m128 in place. @@ -1967,42 +1865,6 @@ extern "C" { fn prefetch(p: *const i8, rw: i32, loc: i32, ty: i32); #[link_name = "llvm.x86.sse.cmp.ss"] fn cmpss(a: __m128, b: __m128, imm8: i8) -> __m128; - #[link_name = "llvm.x86.mmx.movnt.dq"] - fn movntdq(a: *mut __m64, b: __m64); - #[link_name = "llvm.x86.sse.cvtpi2ps"] - fn cvtpi2ps(a: __m128, b: __m64) -> __m128; - #[link_name = "llvm.x86.mmx.maskmovq"] - fn maskmovq(a: __m64, mask: __m64, mem_addr: *mut i8); - #[link_name = "llvm.x86.mmx.pextr.w"] - fn pextrw(a: __m64, imm8: i32) -> i32; - #[link_name = "llvm.x86.mmx.pinsr.w"] - fn pinsrw(a: __m64, d: i32, imm8: i32) -> __m64; - #[link_name = "llvm.x86.mmx.pmovmskb"] - fn pmovmskb(a: __m64) -> i32; - #[link_name = "llvm.x86.sse.pshuf.w"] - fn pshufw(a: __m64, imm8: i8) -> __m64; - #[link_name = "llvm.x86.mmx.pmaxs.w"] - fn pmaxsw(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.pmaxu.b"] - fn pmaxub(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.pmins.w"] - fn pminsw(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.pminu.b"] - fn pminub(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.pmulhu.w"] - fn pmulhuw(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.pmull.w"] - fn pmullw(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.pavg.b"] - fn pavgb(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.pavg.w"] - fn pavgw(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.psad.bw"] - fn psadbw(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.sse.cvtps2pi"] - fn cvtps2pi(a: __m128) -> __m64; - #[link_name = "llvm.x86.sse.cvttps2pi"] - fn cvttps2pi(a: __m128) -> __m64; } /// Stores `a` into the memory at `mem_addr` using a non-temporal memory hint. @@ -2020,463 +1882,6 @@ pub unsafe fn _mm_stream_ps(mem_addr: *mut f32, a: __m128) { intrinsics::nontemporal_store(mem_addr as *mut __m128, a); } -/// Stores 64-bits of integer data from a into memory using a non-temporal -/// memory hint. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(movntq))] -pub unsafe fn _mm_stream_pi(mem_addr: *mut __m64, a: __m64) { - movntdq(mem_addr, a) -} - -/// Compares the packed 16-bit signed integers of `a` and `b` writing the -/// greatest value into the result. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pmaxsw))] -pub unsafe fn _mm_max_pi16(a: __m64, b: __m64) -> __m64 { - pmaxsw(a, b) -} - -/// Compares the packed 16-bit signed integers of `a` and `b` writing the -/// greatest value into the result. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pmaxsw))] -pub unsafe fn _m_pmaxsw(a: __m64, b: __m64) -> __m64 { - _mm_max_pi16(a, b) -} - -/// Compares the packed 8-bit signed integers of `a` and `b` writing the -/// greatest value into the result. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pmaxub))] -pub unsafe fn _mm_max_pu8(a: __m64, b: __m64) -> __m64 { - pmaxub(a, b) -} - -/// Compares the packed 8-bit signed integers of `a` and `b` writing the -/// greatest value into the result. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pmaxub))] -pub unsafe fn _m_pmaxub(a: __m64, b: __m64) -> __m64 { - _mm_max_pu8(a, b) -} - -/// Compares the packed 16-bit signed integers of `a` and `b` writing the -/// smallest value into the result. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pminsw))] -pub unsafe fn _mm_min_pi16(a: __m64, b: __m64) -> __m64 { - pminsw(a, b) -} - -/// Compares the packed 16-bit signed integers of `a` and `b` writing the -/// smallest value into the result. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pminsw))] -pub unsafe fn _m_pminsw(a: __m64, b: __m64) -> __m64 { - _mm_min_pi16(a, b) -} - -/// Compares the packed 8-bit signed integers of `a` and `b` writing the -/// smallest value into the result. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pminub))] -pub unsafe fn _mm_min_pu8(a: __m64, b: __m64) -> __m64 { - pminub(a, b) -} - -/// Compares the packed 8-bit signed integers of `a` and `b` writing the -/// smallest value into the result. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pminub))] -pub unsafe fn _m_pminub(a: __m64, b: __m64) -> __m64 { - _mm_min_pu8(a, b) -} - -/// Multiplies packed 16-bit unsigned integer values and writes the -/// high-order 16 bits of each 32-bit product to the corresponding bits in -/// the destination. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pmulhuw))] -pub unsafe fn _mm_mulhi_pu16(a: __m64, b: __m64) -> __m64 { - pmulhuw(a, b) -} - -/// Multiplies packed 16-bit integer values and writes the -/// low-order 16 bits of each 32-bit product to the corresponding bits in -/// the destination. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pmullw))] -pub unsafe fn _mm_mullo_pi16(a: __m64, b: __m64) -> __m64 { - pmullw(a, b) -} - -/// Multiplies packed 16-bit unsigned integer values and writes the -/// high-order 16 bits of each 32-bit product to the corresponding bits in -/// the destination. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pmulhuw))] -pub unsafe fn _m_pmulhuw(a: __m64, b: __m64) -> __m64 { - _mm_mulhi_pu16(a, b) -} - -/// Computes the rounded averages of the packed unsigned 8-bit integer -/// values and writes the averages to the corresponding bits in the -/// destination. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pavgb))] -pub unsafe fn _mm_avg_pu8(a: __m64, b: __m64) -> __m64 { - pavgb(a, b) -} - -/// Computes the rounded averages of the packed unsigned 8-bit integer -/// values and writes the averages to the corresponding bits in the -/// destination. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pavgb))] -pub unsafe fn _m_pavgb(a: __m64, b: __m64) -> __m64 { - _mm_avg_pu8(a, b) -} - -/// Computes the rounded averages of the packed unsigned 16-bit integer -/// values and writes the averages to the corresponding bits in the -/// destination. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pavgw))] -pub unsafe fn _mm_avg_pu16(a: __m64, b: __m64) -> __m64 { - pavgw(a, b) -} - -/// Computes the rounded averages of the packed unsigned 16-bit integer -/// values and writes the averages to the corresponding bits in the -/// destination. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pavgw))] -pub unsafe fn _m_pavgw(a: __m64, b: __m64) -> __m64 { - _mm_avg_pu16(a, b) -} - -/// Subtracts the corresponding 8-bit unsigned integer values of the two -/// 64-bit vector operands and computes the absolute value for each of the -/// difference. Then sum of the 8 absolute differences is written to the -/// bits `[15:0]` of the destination; the remaining bits `[63:16]` are cleared. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(psadbw))] -pub unsafe fn _mm_sad_pu8(a: __m64, b: __m64) -> __m64 { - psadbw(a, b) -} - -/// Subtracts the corresponding 8-bit unsigned integer values of the two -/// 64-bit vector operands and computes the absolute value for each of the -/// difference. Then sum of the 8 absolute differences is written to the -/// bits `[15:0]` of the destination; the remaining bits `[63:16]` are cleared. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(psadbw))] -pub unsafe fn _m_psadbw(a: __m64, b: __m64) -> __m64 { - _mm_sad_pu8(a, b) -} - -/// Converts two elements of a 64-bit vector of `[2 x i32]` into two -/// floating point values and writes them to the lower 64-bits of the -/// destination. The remaining higher order elements of the destination are -/// copied from the corresponding elements in the first operand. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(cvtpi2ps))] -pub unsafe fn _mm_cvtpi32_ps(a: __m128, b: __m64) -> __m128 { - cvtpi2ps(a, b) -} - -/// Converts two elements of a 64-bit vector of `[2 x i32]` into two -/// floating point values and writes them to the lower 64-bits of the -/// destination. The remaining higher order elements of the destination are -/// copied from the corresponding elements in the first operand. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(cvtpi2ps))] -pub unsafe fn _mm_cvt_pi2ps(a: __m128, b: __m64) -> __m128 { - _mm_cvtpi32_ps(a, b) -} - -/// Converts the lower 4 8-bit values of `a` into a 128-bit vector of 4 `f32`s. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(cvtpi2ps))] -pub unsafe fn _mm_cvtpi8_ps(a: __m64) -> __m128 { - let b = _mm_setzero_si64(); - let b = _mm_cmpgt_pi8(b, a); - let b = _mm_unpacklo_pi8(a, b); - _mm_cvtpi16_ps(b) -} - -/// Converts the lower 4 8-bit values of `a` into a 128-bit vector of 4 `f32`s. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(cvtpi2ps))] -pub unsafe fn _mm_cvtpu8_ps(a: __m64) -> __m128 { - let b = _mm_setzero_si64(); - let b = _mm_unpacklo_pi8(a, b); - _mm_cvtpi16_ps(b) -} - -/// Converts a 64-bit vector of `i16`s into a 128-bit vector of 4 `f32`s. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(cvtpi2ps))] -pub unsafe fn _mm_cvtpi16_ps(a: __m64) -> __m128 { - let b = _mm_setzero_si64(); - let b = _mm_cmpgt_pi16(b, a); - let c = _mm_unpackhi_pi16(a, b); - let r = _mm_setzero_ps(); - let r = cvtpi2ps(r, c); - let r = _mm_movelh_ps(r, r); - let c = _mm_unpacklo_pi16(a, b); - cvtpi2ps(r, c) -} - -/// Converts a 64-bit vector of `i16`s into a 128-bit vector of 4 `f32`s. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(cvtpi2ps))] -pub unsafe fn _mm_cvtpu16_ps(a: __m64) -> __m128 { - let b = _mm_setzero_si64(); - let c = _mm_unpackhi_pi16(a, b); - let r = _mm_setzero_ps(); - let r = cvtpi2ps(r, c); - let r = _mm_movelh_ps(r, r); - let c = _mm_unpacklo_pi16(a, b); - cvtpi2ps(r, c) -} - -/// Converts the two 32-bit signed integer values from each 64-bit vector -/// operand of `[2 x i32]` into a 128-bit vector of `[4 x float]`. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(cvtpi2ps))] -pub unsafe fn _mm_cvtpi32x2_ps(a: __m64, b: __m64) -> __m128 { - let c = _mm_setzero_ps(); - let c = _mm_cvtpi32_ps(c, b); - let c = _mm_movelh_ps(c, c); - _mm_cvtpi32_ps(c, a) -} - -/// Conditionally copies the values from each 8-bit element in the first -/// 64-bit integer vector operand to the specified memory location, as -/// specified by the most significant bit in the corresponding element in the -/// second 64-bit integer vector operand. -/// -/// To minimize caching, the data is flagged as non-temporal -/// (unlikely to be used again soon). -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(maskmovq))] -pub unsafe fn _mm_maskmove_si64(a: __m64, mask: __m64, mem_addr: *mut i8) { - maskmovq(a, mask, mem_addr) -} - -/// Conditionally copies the values from each 8-bit element in the first -/// 64-bit integer vector operand to the specified memory location, as -/// specified by the most significant bit in the corresponding element in the -/// second 64-bit integer vector operand. -/// -/// To minimize caching, the data is flagged as non-temporal -/// (unlikely to be used again soon). -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(maskmovq))] -pub unsafe fn _m_maskmovq(a: __m64, mask: __m64, mem_addr: *mut i8) { - _mm_maskmove_si64(a, mask, mem_addr) -} - -/// Extracts 16-bit element from a 64-bit vector of `[4 x i16]` and -/// returns it, as specified by the immediate integer operand. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pextrw, imm2 = 0))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_extract_pi16(a: __m64, imm2: i32) -> i32 { - macro_rules! call { - ($imm2:expr) => { - pextrw(a, $imm2) as i32 - }; - } - constify_imm2!(imm2, call) -} - -/// Extracts 16-bit element from a 64-bit vector of `[4 x i16]` and -/// returns it, as specified by the immediate integer operand. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pextrw, imm2 = 0))] -#[rustc_args_required_const(1)] -pub unsafe fn _m_pextrw(a: __m64, imm2: i32) -> i32 { - macro_rules! call { - ($imm2:expr) => { - pextrw(a, $imm2) as i32 - }; - } - constify_imm2!(imm2, call) -} - -/// Copies data from the 64-bit vector of `[4 x i16]` to the destination, -/// and inserts the lower 16-bits of an integer operand at the 16-bit offset -/// specified by the immediate operand `n`. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pinsrw, imm2 = 0))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_insert_pi16(a: __m64, d: i32, imm2: i32) -> __m64 { - macro_rules! call { - ($imm2:expr) => { - pinsrw(a, d, $imm2) - }; - } - constify_imm2!(imm2, call) -} - -/// Copies data from the 64-bit vector of `[4 x i16]` to the destination, -/// and inserts the lower 16-bits of an integer operand at the 16-bit offset -/// specified by the immediate operand `n`. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pinsrw, imm2 = 0))] -#[rustc_args_required_const(2)] -pub unsafe fn _m_pinsrw(a: __m64, d: i32, imm2: i32) -> __m64 { - macro_rules! call { - ($imm2:expr) => { - pinsrw(a, d, $imm2) - }; - } - constify_imm2!(imm2, call) -} - -/// Takes the most significant bit from each 8-bit element in a 64-bit -/// integer vector to create a 16-bit mask value. Zero-extends the value to -/// 32-bit integer and writes it to the destination. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pmovmskb))] -pub unsafe fn _mm_movemask_pi8(a: __m64) -> i32 { - pmovmskb(a) -} - -/// Takes the most significant bit from each 8-bit element in a 64-bit -/// integer vector to create a 16-bit mask value. Zero-extends the value to -/// 32-bit integer and writes it to the destination. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pmovmskb))] -pub unsafe fn _m_pmovmskb(a: __m64) -> i32 { - _mm_movemask_pi8(a) -} - -/// Shuffles the 4 16-bit integers from a 64-bit integer vector to the -/// destination, as specified by the immediate value operand. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pshufw, imm8 = 0))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_shuffle_pi16(a: __m64, imm8: i32) -> __m64 { - macro_rules! call { - ($imm8:expr) => { - pshufw(a, $imm8) - }; - } - constify_imm8!(imm8, call) -} - -/// Shuffles the 4 16-bit integers from a 64-bit integer vector to the -/// destination, as specified by the immediate value operand. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(pshufw, imm8 = 0))] -#[rustc_args_required_const(1)] -pub unsafe fn _m_pshufw(a: __m64, imm8: i32) -> __m64 { - macro_rules! call { - ($imm8:expr) => { - pshufw(a, $imm8) - }; - } - constify_imm8!(imm8, call) -} - -/// Converts the two lower packed single-precision (32-bit) floating-point -/// elements in `a` to packed 32-bit integers with truncation. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(cvttps2pi))] -pub unsafe fn _mm_cvttps_pi32(a: __m128) -> __m64 { - cvttps2pi(a) -} - -/// Converts the two lower packed single-precision (32-bit) floating-point -/// elements in `a` to packed 32-bit integers with truncation. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(cvttps2pi))] -pub unsafe fn _mm_cvtt_ps2pi(a: __m128) -> __m64 { - _mm_cvttps_pi32(a) -} - -/// Converts the two lower packed single-precision (32-bit) floating-point -/// elements in `a` to packed 32-bit integers. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(cvtps2pi))] -pub unsafe fn _mm_cvtps_pi32(a: __m128) -> __m64 { - cvtps2pi(a) -} - -/// Converts the two lower packed single-precision (32-bit) floating-point -/// elements in `a` to packed 32-bit integers. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(cvtps2pi))] -pub unsafe fn _mm_cvt_ps2pi(a: __m128) -> __m64 { - _mm_cvtps_pi32(a) -} - -/// Converts packed single-precision (32-bit) floating-point elements in `a` to -/// packed 16-bit integers. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(cvtps2pi))] -pub unsafe fn _mm_cvtps_pi16(a: __m128) -> __m64 { - let b = _mm_cvtps_pi32(a); - let a = _mm_movehl_ps(a, a); - let c = _mm_cvtps_pi32(a); - _mm_packs_pi32(b, c) -} - -/// Converts packed single-precision (32-bit) floating-point elements in `a` to -/// packed 8-bit integers, and returns theem in the lower 4 elements of the -/// result. -#[inline] -#[target_feature(enable = "sse,mmx")] -#[cfg_attr(test, assert_instr(cvtps2pi))] -pub unsafe fn _mm_cvtps_pi8(a: __m128) -> __m64 { - let b = _mm_cvtps_pi16(a); - let c = _mm_setzero_si64(); - _mm_packs_pi16(b, c) -} - #[cfg(test)] mod tests { use crate::{hint::black_box, mem::transmute}; @@ -3414,7 +2819,7 @@ mod tests { #[simd_test(enable = "sse")] unsafe fn test_mm_cvtss_si32() { let inputs = &[42.0f32, -3.1, 4.0e10, 4.0e-20, NAN, 2147483500.1]; - let result = &[42i32, -3, i32::min_value(), 0, i32::min_value(), 2147483520]; + let result = &[42i32, -3, i32::MIN, 0, i32::MIN, 2147483520]; for i in 0..inputs.len() { let x = _mm_setr_ps(inputs[i], 1.0, 3.0, 4.0); let e = result[i]; @@ -3436,9 +2841,9 @@ mod tests { (-34.5, -34), (10.999, 10), (-5.99, -5), - (4.0e10, i32::min_value()), + (4.0e10, i32::MIN), (4.0e-10, 0), - (NAN, i32::min_value()), + (NAN, i32::MIN), (2147483500.1, 2147483520), ]; for i in 0..inputs.len() { @@ -3539,7 +2944,7 @@ mod tests { unsafe fn test_mm_shuffle_ps() { let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); let b = _mm_setr_ps(5.0, 6.0, 7.0, 8.0); - let r = _mm_shuffle_ps(a, b, 0b00_01_01_11); + let r = _mm_shuffle_ps::<0b00_01_01_11>(a, b); assert_eq_m128(r, _mm_setr_ps(4.0, 2.0, 6.0, 5.0)); } @@ -3575,24 +2980,6 @@ mod tests { assert_eq_m128(r, _mm_setr_ps(1.0, 2.0, 5.0, 6.0)); } - #[simd_test(enable = "sse")] - unsafe fn test_mm_loadh_pi() { - let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - let x: [f32; 4] = [5.0, 6.0, 7.0, 8.0]; - let p = x[..].as_ptr(); - let r = _mm_loadh_pi(a, p as *const _); - assert_eq_m128(r, _mm_setr_ps(1.0, 2.0, 5.0, 6.0)); - } - - #[simd_test(enable = "sse")] - unsafe fn test_mm_loadl_pi() { - let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - let x: [f32; 4] = [5.0, 6.0, 7.0, 8.0]; - let p = x[..].as_ptr(); - let r = _mm_loadl_pi(a, p as *const _); - assert_eq_m128(r, _mm_setr_ps(5.0, 6.0, 3.0, 4.0)); - } - #[simd_test(enable = "sse")] unsafe fn test_mm_load_ss() { let a = 42.0f32; @@ -3619,9 +3006,9 @@ mod tests { let unalignment = (p as usize) & 0xf; if unalignment != 0 { - let delta = ((16 - unalignment) >> 2) as isize; + let delta = (16 - unalignment) >> 2; fixup = delta as f32; - p = p.offset(delta); + p = p.add(delta); } let r = _mm_load_ps(p); @@ -3632,7 +3019,7 @@ mod tests { #[simd_test(enable = "sse")] unsafe fn test_mm_loadu_ps() { let vals = &[1.0f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; - let p = vals.as_ptr().offset(3); + let p = vals.as_ptr().add(3); let r = _mm_loadu_ps(black_box(p)); assert_eq_m128(r, _mm_setr_ps(4.0, 5.0, 6.0, 7.0)); } @@ -3649,9 +3036,9 @@ mod tests { let unalignment = (p as usize) & 0xf; if unalignment != 0 { - let delta = ((16 - unalignment) >> 2) as isize; + let delta = (16 - unalignment) >> 2; fixup = delta as f32; - p = p.offset(delta); + p = p.add(delta); } let r = _mm_loadr_ps(p); @@ -3659,33 +3046,18 @@ mod tests { assert_eq_m128(r, e); } - #[simd_test(enable = "sse")] - unsafe fn test_mm_storeh_pi() { - let mut vals = [0.0f32; 8]; - let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - _mm_storeh_pi(vals.as_mut_ptr() as *mut _, a); - - assert_eq!(vals[0], 3.0); - assert_eq!(vals[1], 4.0); - assert_eq!(vals[2], 0.0); - } - - #[simd_test(enable = "sse")] - unsafe fn test_mm_storel_pi() { - let mut vals = [0.0f32; 8]; - let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - _mm_storel_pi(vals.as_mut_ptr() as *mut _, a); - - assert_eq!(vals[0], 1.0); - assert_eq!(vals[1], 2.0); - assert_eq!(vals[2], 0.0); + #[simd_test(enable = "sse2")] + unsafe fn test_mm_loadu_si64() { + let a = _mm_setr_epi64x(5, 6); + let r = _mm_loadu_si64(&a as *const _ as *const _); + assert_eq_m128i(r, _mm_setr_epi64x(5, 0)); } #[simd_test(enable = "sse")] unsafe fn test_mm_store_ss() { let mut vals = [0.0f32; 8]; let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - _mm_store_ss(vals.as_mut_ptr().offset(1), a); + _mm_store_ss(vals.as_mut_ptr().add(1), a); assert_eq!(vals[0], 0.0); assert_eq!(vals[1], 1.0); @@ -3701,8 +3073,8 @@ mod tests { let mut p = vals.as_mut_ptr(); if (p as usize) & 0xf != 0 { - ofs = (16 - (p as usize) & 0xf) >> 2; - p = p.offset(ofs as isize); + ofs = ((16 - (p as usize)) & 0xf) >> 2; + p = p.add(ofs); } _mm_store1_ps(p, *black_box(&a)); @@ -3727,8 +3099,8 @@ mod tests { // Align p to 16-byte boundary if (p as usize) & 0xf != 0 { - ofs = (16 - (p as usize) & 0xf) >> 2; - p = p.offset(ofs as isize); + ofs = ((16 - (p as usize)) & 0xf) >> 2; + p = p.add(ofs); } _mm_store_ps(p, *black_box(&a)); @@ -3753,8 +3125,8 @@ mod tests { // Align p to 16-byte boundary if (p as usize) & 0xf != 0 { - ofs = (16 - (p as usize) & 0xf) >> 2; - p = p.offset(ofs as isize); + ofs = ((16 - (p as usize)) & 0xf) >> 2; + p = p.add(ofs); } _mm_storer_ps(p, *black_box(&a)); @@ -3780,7 +3152,7 @@ mod tests { // Make sure p is **not** aligned to 16-byte boundary if (p as usize) & 0xf == 0 { ofs = 1; - p = p.offset(1); + p = p.add(1); } _mm_storeu_ps(p, *black_box(&a)); @@ -3901,254 +3273,4 @@ mod tests { assert_eq!(mem.data[i], get_m128(a, i)); } } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_stream_pi() { - let a = transmute(i8x8::new(0, 0, 0, 0, 0, 0, 0, 7)); - let mut mem = boxed::Box::<__m64>::new(transmute(i8x8::splat(1))); - _mm_stream_pi(&mut *mem as *mut _ as *mut _, a); - assert_eq_m64(a, *mem); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_max_pi16() { - let a = _mm_setr_pi16(-1, 6, -3, 8); - let b = _mm_setr_pi16(5, -2, 7, -4); - let r = _mm_setr_pi16(5, 6, 7, 8); - - assert_eq_m64(r, _mm_max_pi16(a, b)); - assert_eq_m64(r, _m_pmaxsw(a, b)); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_max_pu8() { - let a = _mm_setr_pi8(2, 6, 3, 8, 2, 6, 3, 8); - let b = _mm_setr_pi8(5, 2, 7, 4, 5, 2, 7, 4); - let r = _mm_setr_pi8(5, 6, 7, 8, 5, 6, 7, 8); - - assert_eq_m64(r, _mm_max_pu8(a, b)); - assert_eq_m64(r, _m_pmaxub(a, b)); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_min_pi16() { - let a = _mm_setr_pi16(-1, 6, -3, 8); - let b = _mm_setr_pi16(5, -2, 7, -4); - let r = _mm_setr_pi16(-1, -2, -3, -4); - - assert_eq_m64(r, _mm_min_pi16(a, b)); - assert_eq_m64(r, _m_pminsw(a, b)); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_min_pu8() { - let a = _mm_setr_pi8(2, 6, 3, 8, 2, 6, 3, 8); - let b = _mm_setr_pi8(5, 2, 7, 4, 5, 2, 7, 4); - let r = _mm_setr_pi8(2, 2, 3, 4, 2, 2, 3, 4); - - assert_eq_m64(r, _mm_min_pu8(a, b)); - assert_eq_m64(r, _m_pminub(a, b)); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_mulhi_pu16() { - let (a, b) = (_mm_set1_pi16(1000), _mm_set1_pi16(1001)); - let r = _mm_mulhi_pu16(a, b); - assert_eq_m64(r, _mm_set1_pi16(15)); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_mullo_pi16() { - let (a, b) = (_mm_set1_pi16(1000), _mm_set1_pi16(1001)); - let r = _mm_mullo_pi16(a, b); - assert_eq_m64(r, _mm_set1_pi16(17960)); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_m_pmulhuw() { - let (a, b) = (_mm_set1_pi16(1000), _mm_set1_pi16(1001)); - let r = _m_pmulhuw(a, b); - assert_eq_m64(r, _mm_set1_pi16(15)); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_avg_pu8() { - let (a, b) = (_mm_set1_pi8(3), _mm_set1_pi8(9)); - let r = _mm_avg_pu8(a, b); - assert_eq_m64(r, _mm_set1_pi8(6)); - - let r = _m_pavgb(a, b); - assert_eq_m64(r, _mm_set1_pi8(6)); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_avg_pu16() { - let (a, b) = (_mm_set1_pi16(3), _mm_set1_pi16(9)); - let r = _mm_avg_pu16(a, b); - assert_eq_m64(r, _mm_set1_pi16(6)); - - let r = _m_pavgw(a, b); - assert_eq_m64(r, _mm_set1_pi16(6)); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_sad_pu8() { - #[rustfmt::skip] - let a = _mm_setr_pi8( - 255u8 as i8, 254u8 as i8, 253u8 as i8, 252u8 as i8, - 1, 2, 3, 4, - ); - let b = _mm_setr_pi8(0, 0, 0, 0, 2, 1, 2, 1); - let r = _mm_sad_pu8(a, b); - assert_eq_m64(r, _mm_setr_pi16(1020, 0, 0, 0)); - - let r = _m_psadbw(a, b); - assert_eq_m64(r, _mm_setr_pi16(1020, 0, 0, 0)); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_cvtpi32_ps() { - let a = _mm_setr_ps(0., 0., 3., 4.); - let b = _mm_setr_pi32(1, 2); - let expected = _mm_setr_ps(1., 2., 3., 4.); - let r = _mm_cvtpi32_ps(a, b); - assert_eq_m128(r, expected); - - let r = _mm_cvt_pi2ps(a, b); - assert_eq_m128(r, expected); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_cvtpi16_ps() { - let a = _mm_setr_pi16(1, 2, 3, 4); - let expected = _mm_setr_ps(1., 2., 3., 4.); - let r = _mm_cvtpi16_ps(a); - assert_eq_m128(r, expected); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_cvtpu16_ps() { - let a = _mm_setr_pi16(1, 2, 3, 4); - let expected = _mm_setr_ps(1., 2., 3., 4.); - let r = _mm_cvtpu16_ps(a); - assert_eq_m128(r, expected); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_cvtpi8_ps() { - let a = _mm_setr_pi8(1, 2, 3, 4, 5, 6, 7, 8); - let expected = _mm_setr_ps(1., 2., 3., 4.); - let r = _mm_cvtpi8_ps(a); - assert_eq_m128(r, expected); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_cvtpu8_ps() { - let a = _mm_setr_pi8(1, 2, 3, 4, 5, 6, 7, 8); - let expected = _mm_setr_ps(1., 2., 3., 4.); - let r = _mm_cvtpu8_ps(a); - assert_eq_m128(r, expected); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_cvtpi32x2_ps() { - let a = _mm_setr_pi32(1, 2); - let b = _mm_setr_pi32(3, 4); - let expected = _mm_setr_ps(1., 2., 3., 4.); - let r = _mm_cvtpi32x2_ps(a, b); - assert_eq_m128(r, expected); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_maskmove_si64() { - let a = _mm_set1_pi8(9); - let mask = _mm_setr_pi8(0, 0, 0x80u8 as i8, 0, 0, 0, 0, 0); - let mut r = _mm_set1_pi8(0); - _mm_maskmove_si64(a, mask, &mut r as *mut _ as *mut i8); - let e = _mm_setr_pi8(0, 0, 9, 0, 0, 0, 0, 0); - assert_eq_m64(r, e); - - let mut r = _mm_set1_pi8(0); - _m_maskmovq(a, mask, &mut r as *mut _ as *mut i8); - assert_eq_m64(r, e); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_extract_pi16() { - let a = _mm_setr_pi16(1, 2, 3, 4); - let r = _mm_extract_pi16(a, 0); - assert_eq!(r, 1); - let r = _mm_extract_pi16(a, 1); - assert_eq!(r, 2); - - let r = _m_pextrw(a, 1); - assert_eq!(r, 2); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_insert_pi16() { - let a = _mm_setr_pi16(1, 2, 3, 4); - let r = _mm_insert_pi16(a, 0, 0b0); - let expected = _mm_setr_pi16(0, 2, 3, 4); - assert_eq_m64(r, expected); - let r = _mm_insert_pi16(a, 0, 0b10); - let expected = _mm_setr_pi16(1, 2, 0, 4); - assert_eq_m64(r, expected); - - let r = _m_pinsrw(a, 0, 0b10); - assert_eq_m64(r, expected); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_movemask_pi8() { - let a = _mm_setr_pi16(0b1000_0000, 0b0100_0000, 0b1000_0000, 0b0100_0000); - let r = _mm_movemask_pi8(a); - assert_eq!(r, 0b10001); - - let r = _m_pmovmskb(a); - assert_eq!(r, 0b10001); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_shuffle_pi16() { - let a = _mm_setr_pi16(1, 2, 3, 4); - let r = _mm_shuffle_pi16(a, 0b00_01_01_11); - let expected = _mm_setr_pi16(4, 2, 2, 1); - assert_eq_m64(r, expected); - - let r = _m_pshufw(a, 0b00_01_01_11); - assert_eq_m64(r, expected); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_cvtps_pi32() { - let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - let r = _mm_setr_pi32(1, 2); - - assert_eq_m64(r, _mm_cvtps_pi32(a)); - assert_eq_m64(r, _mm_cvt_ps2pi(a)); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_cvttps_pi32() { - let a = _mm_setr_ps(7.0, 2.0, 3.0, 4.0); - let r = _mm_setr_pi32(7, 2); - - assert_eq_m64(r, _mm_cvttps_pi32(a)); - assert_eq_m64(r, _mm_cvtt_ps2pi(a)); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_cvtps_pi16() { - let a = _mm_setr_ps(7.0, 2.0, 3.0, 4.0); - let r = _mm_setr_pi16(7, 2, 3, 4); - assert_eq_m64(r, _mm_cvtps_pi16(a)); - } - - #[simd_test(enable = "sse,mmx")] - unsafe fn test_mm_cvtps_pi8() { - let a = _mm_setr_ps(7.0, 2.0, 3.0, 4.0); - let r = _mm_setr_pi8(7, 2, 3, 4, 0, 0, 0, 0); - assert_eq_m64(r, _mm_cvtps_pi8(a)); - } } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse2.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse2.rs index 3e738e335..3e79b3539 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse2.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse2.rs @@ -406,107 +406,94 @@ pub unsafe fn _mm_subs_epu16(a: __m128i, b: __m128i) -> __m128i { transmute(simd_saturating_sub(a.as_u16x8(), b.as_u16x8())) } -/// Shifts `a` left by `imm8` bytes while shifting in zeros. +/// Shifts `a` left by `IMM8` bytes while shifting in zeros. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_slli_si128) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(pslldq, imm8 = 1))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(pslldq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_slli_si128(a: __m128i, imm8: i32) -> __m128i { - _mm_slli_si128_impl(a, imm8) +pub unsafe fn _mm_slli_si128(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + _mm_slli_si128_impl::(a) } /// Implementation detail: converts the immediate argument of the /// `_mm_slli_si128` intrinsic into a compile-time constant. #[inline] #[target_feature(enable = "sse2")] -unsafe fn _mm_slli_si128_impl(a: __m128i, imm8: i32) -> __m128i { - let (zero, imm8) = (_mm_set1_epi8(0).as_i8x16(), imm8 as u32); - let a = a.as_i8x16(); - macro_rules! shuffle { - ($shift:expr) => { - simd_shuffle16::( - zero, - a, - [ - 16 - $shift, - 17 - $shift, - 18 - $shift, - 19 - $shift, - 20 - $shift, - 21 - $shift, - 22 - $shift, - 23 - $shift, - 24 - $shift, - 25 - $shift, - 26 - $shift, - 27 - $shift, - 28 - $shift, - 29 - $shift, - 30 - $shift, - 31 - $shift, - ], - ) - }; +unsafe fn _mm_slli_si128_impl(a: __m128i) -> __m128i { + const fn mask(shift: i32, i: u32) -> u32 { + let shift = shift as u32 & 0xff; + if shift > 15 { + i + } else { + 16 - shift + i + } } - let x = match imm8 { - 0 => shuffle!(0), - 1 => shuffle!(1), - 2 => shuffle!(2), - 3 => shuffle!(3), - 4 => shuffle!(4), - 5 => shuffle!(5), - 6 => shuffle!(6), - 7 => shuffle!(7), - 8 => shuffle!(8), - 9 => shuffle!(9), - 10 => shuffle!(10), - 11 => shuffle!(11), - 12 => shuffle!(12), - 13 => shuffle!(13), - 14 => shuffle!(14), - 15 => shuffle!(15), - _ => shuffle!(16), - }; - transmute(x) + let zero = _mm_set1_epi8(0).as_i8x16(); + transmute::(simd_shuffle16!( + zero, + a.as_i8x16(), + [ + mask(IMM8, 0), + mask(IMM8, 1), + mask(IMM8, 2), + mask(IMM8, 3), + mask(IMM8, 4), + mask(IMM8, 5), + mask(IMM8, 6), + mask(IMM8, 7), + mask(IMM8, 8), + mask(IMM8, 9), + mask(IMM8, 10), + mask(IMM8, 11), + mask(IMM8, 12), + mask(IMM8, 13), + mask(IMM8, 14), + mask(IMM8, 15), + ], + )) } -/// Shifts `a` left by `imm8` bytes while shifting in zeros. +/// Shifts `a` left by `IMM8` bytes while shifting in zeros. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_bslli_si128) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(pslldq, imm8 = 1))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(pslldq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_bslli_si128(a: __m128i, imm8: i32) -> __m128i { - _mm_slli_si128_impl(a, imm8) +pub unsafe fn _mm_bslli_si128(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + _mm_slli_si128_impl::(a) } -/// Shifts `a` right by `imm8` bytes while shifting in zeros. +/// Shifts `a` right by `IMM8` bytes while shifting in zeros. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_bsrli_si128) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(psrldq, imm8 = 1))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(psrldq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_bsrli_si128(a: __m128i, imm8: i32) -> __m128i { - _mm_srli_si128_impl(a, imm8) +pub unsafe fn _mm_bsrli_si128(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + _mm_srli_si128_impl::(a) } -/// Shifts packed 16-bit integers in `a` left by `imm8` while shifting in zeros. +/// Shifts packed 16-bit integers in `a` left by `IMM8` while shifting in zeros. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_slli_epi16) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(psllw, imm8 = 7))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(psllw, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_slli_epi16(a: __m128i, imm8: i32) -> __m128i { - transmute(pslliw(a.as_i16x8(), imm8)) +pub unsafe fn _mm_slli_epi16(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(pslliw(a.as_i16x8(), IMM8)) } /// Shifts packed 16-bit integers in `a` left by `count` while shifting in @@ -521,16 +508,17 @@ pub unsafe fn _mm_sll_epi16(a: __m128i, count: __m128i) -> __m128i { transmute(psllw(a.as_i16x8(), count.as_i16x8())) } -/// Shifts packed 32-bit integers in `a` left by `imm8` while shifting in zeros. +/// Shifts packed 32-bit integers in `a` left by `IMM8` while shifting in zeros. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_slli_epi32) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(pslld, imm8 = 7))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(pslld, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_slli_epi32(a: __m128i, imm8: i32) -> __m128i { - transmute(psllid(a.as_i32x4(), imm8)) +pub unsafe fn _mm_slli_epi32(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(psllid(a.as_i32x4(), IMM8)) } /// Shifts packed 32-bit integers in `a` left by `count` while shifting in @@ -545,16 +533,17 @@ pub unsafe fn _mm_sll_epi32(a: __m128i, count: __m128i) -> __m128i { transmute(pslld(a.as_i32x4(), count.as_i32x4())) } -/// Shifts packed 64-bit integers in `a` left by `imm8` while shifting in zeros. +/// Shifts packed 64-bit integers in `a` left by `IMM8` while shifting in zeros. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_slli_epi64) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(psllq, imm8 = 7))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(psllq, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_slli_epi64(a: __m128i, imm8: i32) -> __m128i { - transmute(pslliq(a.as_i64x2(), imm8)) +pub unsafe fn _mm_slli_epi64(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(pslliq(a.as_i64x2(), IMM8)) } /// Shifts packed 64-bit integers in `a` left by `count` while shifting in @@ -569,17 +558,18 @@ pub unsafe fn _mm_sll_epi64(a: __m128i, count: __m128i) -> __m128i { transmute(psllq(a.as_i64x2(), count.as_i64x2())) } -/// Shifts packed 16-bit integers in `a` right by `imm8` while shifting in sign +/// Shifts packed 16-bit integers in `a` right by `IMM8` while shifting in sign /// bits. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srai_epi16) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(psraw, imm8 = 1))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(psraw, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_srai_epi16(a: __m128i, imm8: i32) -> __m128i { - transmute(psraiw(a.as_i16x8(), imm8)) +pub unsafe fn _mm_srai_epi16(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(psraiw(a.as_i16x8(), IMM8)) } /// Shifts packed 16-bit integers in `a` right by `count` while shifting in sign @@ -594,17 +584,18 @@ pub unsafe fn _mm_sra_epi16(a: __m128i, count: __m128i) -> __m128i { transmute(psraw(a.as_i16x8(), count.as_i16x8())) } -/// Shifts packed 32-bit integers in `a` right by `imm8` while shifting in sign +/// Shifts packed 32-bit integers in `a` right by `IMM8` while shifting in sign /// bits. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srai_epi32) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(psrad, imm8 = 1))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(psrad, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_srai_epi32(a: __m128i, imm8: i32) -> __m128i { - transmute(psraid(a.as_i32x4(), imm8)) +pub unsafe fn _mm_srai_epi32(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(psraid(a.as_i32x4(), IMM8)) } /// Shifts packed 32-bit integers in `a` right by `count` while shifting in sign @@ -619,84 +610,69 @@ pub unsafe fn _mm_sra_epi32(a: __m128i, count: __m128i) -> __m128i { transmute(psrad(a.as_i32x4(), count.as_i32x4())) } -/// Shifts `a` right by `imm8` bytes while shifting in zeros. +/// Shifts `a` right by `IMM8` bytes while shifting in zeros. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srli_si128) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(psrldq, imm8 = 1))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(psrldq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_srli_si128(a: __m128i, imm8: i32) -> __m128i { - _mm_srli_si128_impl(a, imm8) +pub unsafe fn _mm_srli_si128(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + _mm_srli_si128_impl::(a) } /// Implementation detail: converts the immediate argument of the /// `_mm_srli_si128` intrinsic into a compile-time constant. #[inline] #[target_feature(enable = "sse2")] -unsafe fn _mm_srli_si128_impl(a: __m128i, imm8: i32) -> __m128i { - let (zero, imm8) = (_mm_set1_epi8(0).as_i8x16(), imm8 as u32); - let a = a.as_i8x16(); - macro_rules! shuffle { - ($shift:expr) => { - simd_shuffle16( - a, - zero, - [ - 0 + $shift, - 1 + $shift, - 2 + $shift, - 3 + $shift, - 4 + $shift, - 5 + $shift, - 6 + $shift, - 7 + $shift, - 8 + $shift, - 9 + $shift, - 10 + $shift, - 11 + $shift, - 12 + $shift, - 13 + $shift, - 14 + $shift, - 15 + $shift, - ], - ) - }; +unsafe fn _mm_srli_si128_impl(a: __m128i) -> __m128i { + const fn mask(shift: i32, i: u32) -> u32 { + if (shift as u32) > 15 { + i + 16 + } else { + i + (shift as u32) + } } - let x: i8x16 = match imm8 { - 0 => shuffle!(0), - 1 => shuffle!(1), - 2 => shuffle!(2), - 3 => shuffle!(3), - 4 => shuffle!(4), - 5 => shuffle!(5), - 6 => shuffle!(6), - 7 => shuffle!(7), - 8 => shuffle!(8), - 9 => shuffle!(9), - 10 => shuffle!(10), - 11 => shuffle!(11), - 12 => shuffle!(12), - 13 => shuffle!(13), - 14 => shuffle!(14), - 15 => shuffle!(15), - _ => shuffle!(16), - }; + let zero = _mm_set1_epi8(0).as_i8x16(); + let x: i8x16 = simd_shuffle16!( + a.as_i8x16(), + zero, + [ + mask(IMM8, 0), + mask(IMM8, 1), + mask(IMM8, 2), + mask(IMM8, 3), + mask(IMM8, 4), + mask(IMM8, 5), + mask(IMM8, 6), + mask(IMM8, 7), + mask(IMM8, 8), + mask(IMM8, 9), + mask(IMM8, 10), + mask(IMM8, 11), + mask(IMM8, 12), + mask(IMM8, 13), + mask(IMM8, 14), + mask(IMM8, 15), + ], + ); transmute(x) } -/// Shifts packed 16-bit integers in `a` right by `imm8` while shifting in +/// Shifts packed 16-bit integers in `a` right by `IMM8` while shifting in /// zeros. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srli_epi16) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(psrlw, imm8 = 1))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(psrlw, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_srli_epi16(a: __m128i, imm8: i32) -> __m128i { - transmute(psrliw(a.as_i16x8(), imm8)) +pub unsafe fn _mm_srli_epi16(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(psrliw(a.as_i16x8(), IMM8)) } /// Shifts packed 16-bit integers in `a` right by `count` while shifting in @@ -711,17 +687,18 @@ pub unsafe fn _mm_srl_epi16(a: __m128i, count: __m128i) -> __m128i { transmute(psrlw(a.as_i16x8(), count.as_i16x8())) } -/// Shifts packed 32-bit integers in `a` right by `imm8` while shifting in +/// Shifts packed 32-bit integers in `a` right by `IMM8` while shifting in /// zeros. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srli_epi32) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(psrld, imm8 = 8))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(psrld, IMM8 = 8))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_srli_epi32(a: __m128i, imm8: i32) -> __m128i { - transmute(psrlid(a.as_i32x4(), imm8)) +pub unsafe fn _mm_srli_epi32(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(psrlid(a.as_i32x4(), IMM8)) } /// Shifts packed 32-bit integers in `a` right by `count` while shifting in @@ -736,17 +713,18 @@ pub unsafe fn _mm_srl_epi32(a: __m128i, count: __m128i) -> __m128i { transmute(psrld(a.as_i32x4(), count.as_i32x4())) } -/// Shifts packed 64-bit integers in `a` right by `imm8` while shifting in +/// Shifts packed 64-bit integers in `a` right by `IMM8` while shifting in /// zeros. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srli_epi64) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(psrlq, imm8 = 1))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(psrlq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_srli_epi64(a: __m128i, imm8: i32) -> __m128i { - transmute(psrliq(a.as_i64x2(), imm8)) +pub unsafe fn _mm_srli_epi64(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(psrliq(a.as_i64x2(), IMM8)) } /// Shifts packed 64-bit integers in `a` right by `count` while shifting in @@ -918,7 +896,7 @@ pub unsafe fn _mm_cmplt_epi32(a: __m128i, b: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepi32_pd(a: __m128i) -> __m128d { let a = a.as_i32x4(); - simd_cast::(simd_shuffle2(a, a, [0, 1])) + simd_cast::(simd_shuffle2!(a, a, [0, 1])) } /// Returns `a` with its lower element replaced by `b` after converting it to @@ -1326,7 +1304,7 @@ pub unsafe fn _mm_stream_si32(mem_addr: *mut i32, a: i32) { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_move_epi64(a: __m128i) -> __m128i { let zero = _mm_setzero_si128(); - let r: i64x2 = simd_shuffle2(a.as_i64x2(), zero.as_i64x2(), [0, 2]); + let r: i64x2 = simd_shuffle2!(a.as_i64x2(), zero.as_i64x2(), [0, 2]); transmute(r) } @@ -1371,11 +1349,12 @@ pub unsafe fn _mm_packus_epi16(a: __m128i, b: __m128i) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_extract_epi16) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(pextrw, imm8 = 9))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(pextrw, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_extract_epi16(a: __m128i, imm8: i32) -> i32 { - simd_extract::<_, i16>(a.as_i16x8(), (imm8 & 7) as u32) as i32 +pub unsafe fn _mm_extract_epi16(a: __m128i) -> i32 { + static_assert_imm3!(IMM8); + simd_extract::<_, u16>(a.as_u16x8(), IMM8 as u32) as i32 } /// Returns a new vector where the `imm8` element of `a` is replaced with `i`. @@ -1383,11 +1362,12 @@ pub unsafe fn _mm_extract_epi16(a: __m128i, imm8: i32) -> i32 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_insert_epi16) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(pinsrw, imm8 = 9))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(pinsrw, IMM8 = 7))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_insert_epi16(a: __m128i, i: i32, imm8: i32) -> __m128i { - transmute(simd_insert(a.as_i16x8(), (imm8 & 7) as u32, i as i16)) +pub unsafe fn _mm_insert_epi16(a: __m128i, i: i32) -> __m128i { + static_assert_imm3!(IMM8); + transmute(simd_insert(a.as_i16x8(), IMM8 as u32, i as i16)) } /// Returns a mask of the most significant bit of each element in `a`. @@ -1398,74 +1378,35 @@ pub unsafe fn _mm_insert_epi16(a: __m128i, i: i32, imm8: i32) -> __m128i { #[cfg_attr(test, assert_instr(pmovmskb))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_movemask_epi8(a: __m128i) -> i32 { - pmovmskb(a.as_i8x16()) + simd_bitmask::<_, u16>(a.as_i8x16()) as u32 as i32 } -/// Shuffles 32-bit integers in `a` using the control in `imm8`. +/// Shuffles 32-bit integers in `a` using the control in `IMM8`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shuffle_epi32) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(pshufd, imm8 = 9))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(pshufd, IMM8 = 9))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_shuffle_epi32(a: __m128i, imm8: i32) -> __m128i { - // simd_shuffleX requires that its selector parameter be made up of - // constant values, but we can't enforce that here. In spirit, we need - // to write a `match` on all possible values of a byte, and for each value, - // hard-code the correct `simd_shuffleX` call using only constants. We - // then hope for LLVM to do the rest. - // - // Of course, that's... awful. So we try to use macros to do it for us. - let imm8 = (imm8 & 0xFF) as u8; +pub unsafe fn _mm_shuffle_epi32(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i32x4(); - - macro_rules! shuffle_done { - ($x01:expr, $x23:expr, $x45:expr, $x67:expr) => { - simd_shuffle4(a, a, [$x01, $x23, $x45, $x67]) - }; - } - macro_rules! shuffle_x67 { - ($x01:expr, $x23:expr, $x45:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle_done!($x01, $x23, $x45, 0), - 0b01 => shuffle_done!($x01, $x23, $x45, 1), - 0b10 => shuffle_done!($x01, $x23, $x45, 2), - _ => shuffle_done!($x01, $x23, $x45, 3), - } - }; - } - macro_rules! shuffle_x45 { - ($x01:expr, $x23:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle_x67!($x01, $x23, 0), - 0b01 => shuffle_x67!($x01, $x23, 1), - 0b10 => shuffle_x67!($x01, $x23, 2), - _ => shuffle_x67!($x01, $x23, 3), - } - }; - } - macro_rules! shuffle_x23 { - ($x01:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle_x45!($x01, 0), - 0b01 => shuffle_x45!($x01, 1), - 0b10 => shuffle_x45!($x01, 2), - _ => shuffle_x45!($x01, 3), - } - }; - } - let x: i32x4 = match imm8 & 0b11 { - 0b00 => shuffle_x23!(0), - 0b01 => shuffle_x23!(1), - 0b10 => shuffle_x23!(2), - _ => shuffle_x23!(3), - }; + let x: i32x4 = simd_shuffle4!( + a, + a, + [ + IMM8 as u32 & 0b11, + (IMM8 as u32 >> 2) & 0b11, + (IMM8 as u32 >> 4) & 0b11, + (IMM8 as u32 >> 6) & 0b11, + ], + ); transmute(x) } /// Shuffles 16-bit integers in the high 64 bits of `a` using the control in -/// `imm8`. +/// `IMM8`. /// /// Put the results in the high 64 bits of the returned vector, with the low 64 /// bits being copied from from `a`. @@ -1473,59 +1414,31 @@ pub unsafe fn _mm_shuffle_epi32(a: __m128i, imm8: i32) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shufflehi_epi16) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(pshufhw, imm8 = 9))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(pshufhw, IMM8 = 9))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_shufflehi_epi16(a: __m128i, imm8: i32) -> __m128i { - // See _mm_shuffle_epi32. - let imm8 = (imm8 & 0xFF) as u8; +pub unsafe fn _mm_shufflehi_epi16(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i16x8(); - macro_rules! shuffle_done { - ($x01:expr, $x23:expr, $x45:expr, $x67:expr) => { - simd_shuffle8(a, a, [0, 1, 2, 3, $x01 + 4, $x23 + 4, $x45 + 4, $x67 + 4]) - }; - } - macro_rules! shuffle_x67 { - ($x01:expr, $x23:expr, $x45:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle_done!($x01, $x23, $x45, 0), - 0b01 => shuffle_done!($x01, $x23, $x45, 1), - 0b10 => shuffle_done!($x01, $x23, $x45, 2), - _ => shuffle_done!($x01, $x23, $x45, 3), - } - }; - } - macro_rules! shuffle_x45 { - ($x01:expr, $x23:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle_x67!($x01, $x23, 0), - 0b01 => shuffle_x67!($x01, $x23, 1), - 0b10 => shuffle_x67!($x01, $x23, 2), - _ => shuffle_x67!($x01, $x23, 3), - } - }; - } - macro_rules! shuffle_x23 { - ($x01:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle_x45!($x01, 0), - 0b01 => shuffle_x45!($x01, 1), - 0b10 => shuffle_x45!($x01, 2), - _ => shuffle_x45!($x01, 3), - } - }; - } - let x: i16x8 = match imm8 & 0b11 { - 0b00 => shuffle_x23!(0), - 0b01 => shuffle_x23!(1), - 0b10 => shuffle_x23!(2), - _ => shuffle_x23!(3), - }; + let x: i16x8 = simd_shuffle8!( + a, + a, + [ + 0, + 1, + 2, + 3, + (IMM8 as u32 & 0b11) + 4, + ((IMM8 as u32 >> 2) & 0b11) + 4, + ((IMM8 as u32 >> 4) & 0b11) + 4, + ((IMM8 as u32 >> 6) & 0b11) + 4, + ], + ); transmute(x) } /// Shuffles 16-bit integers in the low 64 bits of `a` using the control in -/// `imm8`. +/// `IMM8`. /// /// Put the results in the low 64 bits of the returned vector, with the high 64 /// bits being copied from from `a`. @@ -1533,55 +1446,26 @@ pub unsafe fn _mm_shufflehi_epi16(a: __m128i, imm8: i32) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shufflelo_epi16) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(pshuflw, imm8 = 9))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(pshuflw, IMM8 = 9))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_shufflelo_epi16(a: __m128i, imm8: i32) -> __m128i { - // See _mm_shuffle_epi32. - let imm8 = (imm8 & 0xFF) as u8; +pub unsafe fn _mm_shufflelo_epi16(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i16x8(); - - macro_rules! shuffle_done { - ($x01:expr, $x23:expr, $x45:expr, $x67:expr) => { - simd_shuffle8(a, a, [$x01, $x23, $x45, $x67, 4, 5, 6, 7]) - }; - } - macro_rules! shuffle_x67 { - ($x01:expr, $x23:expr, $x45:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle_done!($x01, $x23, $x45, 0), - 0b01 => shuffle_done!($x01, $x23, $x45, 1), - 0b10 => shuffle_done!($x01, $x23, $x45, 2), - _ => shuffle_done!($x01, $x23, $x45, 3), - } - }; - } - macro_rules! shuffle_x45 { - ($x01:expr, $x23:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle_x67!($x01, $x23, 0), - 0b01 => shuffle_x67!($x01, $x23, 1), - 0b10 => shuffle_x67!($x01, $x23, 2), - _ => shuffle_x67!($x01, $x23, 3), - } - }; - } - macro_rules! shuffle_x23 { - ($x01:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle_x45!($x01, 0), - 0b01 => shuffle_x45!($x01, 1), - 0b10 => shuffle_x45!($x01, 2), - _ => shuffle_x45!($x01, 3), - } - }; - } - let x: i16x8 = match imm8 & 0b11 { - 0b00 => shuffle_x23!(0), - 0b01 => shuffle_x23!(1), - 0b10 => shuffle_x23!(2), - _ => shuffle_x23!(3), - }; + let x: i16x8 = simd_shuffle8!( + a, + a, + [ + IMM8 as u32 & 0b11, + (IMM8 as u32 >> 2) & 0b11, + (IMM8 as u32 >> 4) & 0b11, + (IMM8 as u32 >> 6) & 0b11, + 4, + 5, + 6, + 7, + ], + ); transmute(x) } @@ -1593,7 +1477,7 @@ pub unsafe fn _mm_shufflelo_epi16(a: __m128i, imm8: i32) -> __m128i { #[cfg_attr(test, assert_instr(punpckhbw))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpackhi_epi8(a: __m128i, b: __m128i) -> __m128i { - transmute::(simd_shuffle16( + transmute::(simd_shuffle16!( a.as_i8x16(), b.as_i8x16(), [8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31], @@ -1608,7 +1492,7 @@ pub unsafe fn _mm_unpackhi_epi8(a: __m128i, b: __m128i) -> __m128i { #[cfg_attr(test, assert_instr(punpckhwd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpackhi_epi16(a: __m128i, b: __m128i) -> __m128i { - let x = simd_shuffle8(a.as_i16x8(), b.as_i16x8(), [4, 12, 5, 13, 6, 14, 7, 15]); + let x = simd_shuffle8!(a.as_i16x8(), b.as_i16x8(), [4, 12, 5, 13, 6, 14, 7, 15]); transmute::(x) } @@ -1620,7 +1504,7 @@ pub unsafe fn _mm_unpackhi_epi16(a: __m128i, b: __m128i) -> __m128i { #[cfg_attr(test, assert_instr(unpckhps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpackhi_epi32(a: __m128i, b: __m128i) -> __m128i { - transmute::(simd_shuffle4(a.as_i32x4(), b.as_i32x4(), [2, 6, 3, 7])) + transmute::(simd_shuffle4!(a.as_i32x4(), b.as_i32x4(), [2, 6, 3, 7])) } /// Unpacks and interleave 64-bit integers from the high half of `a` and `b`. @@ -1631,7 +1515,7 @@ pub unsafe fn _mm_unpackhi_epi32(a: __m128i, b: __m128i) -> __m128i { #[cfg_attr(test, assert_instr(unpckhpd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpackhi_epi64(a: __m128i, b: __m128i) -> __m128i { - transmute::(simd_shuffle2(a.as_i64x2(), b.as_i64x2(), [1, 3])) + transmute::(simd_shuffle2!(a.as_i64x2(), b.as_i64x2(), [1, 3])) } /// Unpacks and interleave 8-bit integers from the low half of `a` and `b`. @@ -1642,7 +1526,7 @@ pub unsafe fn _mm_unpackhi_epi64(a: __m128i, b: __m128i) -> __m128i { #[cfg_attr(test, assert_instr(punpcklbw))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpacklo_epi8(a: __m128i, b: __m128i) -> __m128i { - transmute::(simd_shuffle16( + transmute::(simd_shuffle16!( a.as_i8x16(), b.as_i8x16(), [0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23], @@ -1657,7 +1541,7 @@ pub unsafe fn _mm_unpacklo_epi8(a: __m128i, b: __m128i) -> __m128i { #[cfg_attr(test, assert_instr(punpcklwd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpacklo_epi16(a: __m128i, b: __m128i) -> __m128i { - let x = simd_shuffle8(a.as_i16x8(), b.as_i16x8(), [0, 8, 1, 9, 2, 10, 3, 11]); + let x = simd_shuffle8!(a.as_i16x8(), b.as_i16x8(), [0, 8, 1, 9, 2, 10, 3, 11]); transmute::(x) } @@ -1669,7 +1553,7 @@ pub unsafe fn _mm_unpacklo_epi16(a: __m128i, b: __m128i) -> __m128i { #[cfg_attr(test, assert_instr(unpcklps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpacklo_epi32(a: __m128i, b: __m128i) -> __m128i { - transmute::(simd_shuffle4(a.as_i32x4(), b.as_i32x4(), [0, 4, 1, 5])) + transmute::(simd_shuffle4!(a.as_i32x4(), b.as_i32x4(), [0, 4, 1, 5])) } /// Unpacks and interleave 64-bit integers from the low half of `a` and `b`. @@ -1680,7 +1564,7 @@ pub unsafe fn _mm_unpacklo_epi32(a: __m128i, b: __m128i) -> __m128i { #[cfg_attr(all(test, not(target_os = "windows")), assert_instr(movlhps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpacklo_epi64(a: __m128i, b: __m128i) -> __m128i { - transmute::(simd_shuffle2(a.as_i64x2(), b.as_i64x2(), [0, 2])) + transmute::(simd_shuffle2!(a.as_i64x2(), b.as_i64x2(), [0, 2])) } /// Returns a new vector with the low element of `a` replaced by the sum of the @@ -1890,7 +1774,7 @@ pub unsafe fn _mm_or_pd(a: __m128d, b: __m128d) -> __m128d { transmute(_mm_or_si128(a, b)) } -/// Computes the bitwise OR of `a` and `b`. +/// Computes the bitwise XOR of `a` and `b`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_xor_pd) #[inline] @@ -2636,7 +2520,7 @@ pub unsafe fn _mm_storeu_pd(mem_addr: *mut f64, a: __m128d) { #[stable(feature = "simd_x86", since = "1.27.0")] #[allow(clippy::cast_ptr_alignment)] pub unsafe fn _mm_store1_pd(mem_addr: *mut f64, a: __m128d) { - let b: __m128d = simd_shuffle2(a, a, [0, 0]); + let b: __m128d = simd_shuffle2!(a, a, [0, 0]); *(mem_addr as *mut __m128d) = b; } @@ -2650,7 +2534,7 @@ pub unsafe fn _mm_store1_pd(mem_addr: *mut f64, a: __m128d) { #[stable(feature = "simd_x86", since = "1.27.0")] #[allow(clippy::cast_ptr_alignment)] pub unsafe fn _mm_store_pd1(mem_addr: *mut f64, a: __m128d) { - let b: __m128d = simd_shuffle2(a, a, [0, 0]); + let b: __m128d = simd_shuffle2!(a, a, [0, 0]); *(mem_addr as *mut __m128d) = b; } @@ -2665,7 +2549,7 @@ pub unsafe fn _mm_store_pd1(mem_addr: *mut f64, a: __m128d) { #[stable(feature = "simd_x86", since = "1.27.0")] #[allow(clippy::cast_ptr_alignment)] pub unsafe fn _mm_storer_pd(mem_addr: *mut f64, a: __m128d) { - let b: __m128d = simd_shuffle2(a, a, [1, 0]); + let b: __m128d = simd_shuffle2!(a, a, [1, 0]); *(mem_addr as *mut __m128d) = b; } @@ -2729,7 +2613,7 @@ pub unsafe fn _mm_load_pd1(mem_addr: *const f64) -> __m128d { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_loadr_pd(mem_addr: *const f64) -> __m128d { let a = _mm_load_pd(mem_addr); - simd_shuffle2(a, a, [1, 0]) + simd_shuffle2!(a, a, [1, 0]) } /// Loads 128-bits (composed of 2 packed double-precision (64-bit) @@ -2758,17 +2642,12 @@ pub unsafe fn _mm_loadu_pd(mem_addr: *const f64) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shuffle_pd) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(all(test, not(target_os = "windows")), assert_instr(shufps, imm8 = 1))] -#[cfg_attr(all(test, target_os = "windows"), assert_instr(shufpd, imm8 = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(shufps, MASK = 2))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_shuffle_pd(a: __m128d, b: __m128d, imm8: i32) -> __m128d { - match imm8 & 0b11 { - 0b00 => simd_shuffle2(a, b, [0, 2]), - 0b01 => simd_shuffle2(a, b, [1, 2]), - 0b10 => simd_shuffle2(a, b, [0, 3]), - _ => simd_shuffle2(a, b, [1, 3]), - } +pub unsafe fn _mm_shuffle_pd(a: __m128d, b: __m128d) -> __m128d { + static_assert_imm8!(MASK); + simd_shuffle2!(a, b, [MASK as u32 & 0b1, ((MASK as u32 >> 1) & 0b1) + 2]) } /// Constructs a 128-bit floating-point vector of `[2 x double]`. The lower @@ -2778,8 +2657,7 @@ pub unsafe fn _mm_shuffle_pd(a: __m128d, b: __m128d, imm8: i32) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_move_sd) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(all(test, not(target_os = "windows")), assert_instr(movsd))] -#[cfg_attr(all(test, target_os = "windows"), assert_instr(movlps))] +#[cfg_attr(test, assert_instr(movsd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_move_sd(a: __m128d, b: __m128d) -> __m128d { _mm_setr_pd(simd_extract(b, 0), simd_extract(a, 1)) @@ -2858,8 +2736,7 @@ pub unsafe fn _mm_castsi128_ps(a: __m128i) -> __m128 { #[target_feature(enable = "sse2")] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_undefined_pd() -> __m128d { - // FIXME: this function should return MaybeUninit<__m128d> - mem::MaybeUninit::<__m128d>::uninit().assume_init() + __m128d(0.0, 0.0) } /// Returns vector of type __m128i with undefined elements. @@ -2869,8 +2746,7 @@ pub unsafe fn _mm_undefined_pd() -> __m128d { #[target_feature(enable = "sse2")] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_undefined_si128() -> __m128i { - // FIXME: this function should return MaybeUninit<__m128i> - mem::MaybeUninit::<__m128i>::uninit().assume_init() + __m128i(0, 0) } /// The resulting `__m128d` element is composed by the low-order values of @@ -2886,7 +2762,7 @@ pub unsafe fn _mm_undefined_si128() -> __m128i { #[cfg_attr(test, assert_instr(unpckhpd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpackhi_pd(a: __m128d, b: __m128d) -> __m128d { - simd_shuffle2(a, b, [1, 3]) + simd_shuffle2!(a, b, [1, 3]) } /// The resulting `__m128d` element is composed by the high-order values of @@ -2901,114 +2777,7 @@ pub unsafe fn _mm_unpackhi_pd(a: __m128d, b: __m128d) -> __m128d { #[cfg_attr(all(test, not(target_os = "windows")), assert_instr(movlhps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpacklo_pd(a: __m128d, b: __m128d) -> __m128d { - simd_shuffle2(a, b, [0, 2]) -} - -/// Adds two signed or unsigned 64-bit integer values, returning the -/// lower 64 bits of the sum. -#[inline] -#[target_feature(enable = "sse2,mmx")] -#[cfg_attr(test, assert_instr(paddq))] -pub unsafe fn _mm_add_si64(a: __m64, b: __m64) -> __m64 { - paddq(a, b) -} - -/// Multiplies 32-bit unsigned integer values contained in the lower bits -/// of the two 64-bit integer vectors and returns the 64-bit unsigned -/// product. -#[inline] -#[target_feature(enable = "sse2,mmx")] -#[cfg_attr(test, assert_instr(pmuludq))] -pub unsafe fn _mm_mul_su32(a: __m64, b: __m64) -> __m64 { - pmuludq2(a, b) -} - -/// Subtracts signed or unsigned 64-bit integer values and writes the -/// difference to the corresponding bits in the destination. -#[inline] -#[target_feature(enable = "sse2,mmx")] -#[cfg_attr(test, assert_instr(psubq))] -pub unsafe fn _mm_sub_si64(a: __m64, b: __m64) -> __m64 { - psubq(a, b) -} - -/// Converts the two signed 32-bit integer elements of a 64-bit vector of -/// `[2 x i32]` into two double-precision floating-point values, returned in a -/// 128-bit vector of `[2 x double]`. -#[inline] -#[target_feature(enable = "sse2,mmx")] -#[cfg_attr(test, assert_instr(cvtpi2pd))] -pub unsafe fn _mm_cvtpi32_pd(a: __m64) -> __m128d { - cvtpi2pd(a) -} - -/// Initializes both 64-bit values in a 128-bit vector of `[2 x i64]` with -/// the specified 64-bit integer values. -#[inline] -#[target_feature(enable = "sse2,mmx")] -// no particular instruction to test -pub unsafe fn _mm_set_epi64(e1: __m64, e0: __m64) -> __m128i { - _mm_set_epi64x(transmute(e1), transmute(e0)) -} - -/// Initializes both values in a 128-bit vector of `[2 x i64]` with the -/// specified 64-bit value. -#[inline] -#[target_feature(enable = "sse2,mmx")] -// no particular instruction to test -pub unsafe fn _mm_set1_epi64(a: __m64) -> __m128i { - _mm_set_epi64x(transmute(a), transmute(a)) -} - -/// Constructs a 128-bit integer vector, initialized in reverse order -/// with the specified 64-bit integral values. -#[inline] -#[target_feature(enable = "sse2,mmx")] -// no particular instruction to test -pub unsafe fn _mm_setr_epi64(e1: __m64, e0: __m64) -> __m128i { - _mm_set_epi64x(transmute(e0), transmute(e1)) -} - -/// Returns the lower 64 bits of a 128-bit integer vector as a 64-bit -/// integer. -#[inline] -#[target_feature(enable = "sse2,mmx")] -// #[cfg_attr(test, assert_instr(movdq2q))] // FIXME: llvm codegens wrong -// instr? -pub unsafe fn _mm_movepi64_pi64(a: __m128i) -> __m64 { - transmute(simd_extract::<_, i64>(a.as_i64x2(), 0)) -} - -/// Moves the 64-bit operand to a 128-bit integer vector, zeroing the -/// upper bits. -#[inline] -#[target_feature(enable = "sse2,mmx")] -// #[cfg_attr(test, assert_instr(movq2dq))] // FIXME: llvm codegens wrong -// instr? -pub unsafe fn _mm_movpi64_epi64(a: __m64) -> __m128i { - _mm_set_epi64x(0, transmute(a)) -} - -/// Converts the two double-precision floating-point elements of a -/// 128-bit vector of `[2 x double]` into two signed 32-bit integer values, -/// returned in a 64-bit vector of `[2 x i32]`. -#[inline] -#[target_feature(enable = "sse2,mmx")] -#[cfg_attr(test, assert_instr(cvtpd2pi))] -pub unsafe fn _mm_cvtpd_pi32(a: __m128d) -> __m64 { - cvtpd2pi(a) -} - -/// Converts the two double-precision floating-point elements of a -/// 128-bit vector of `[2 x double]` into two signed 32-bit integer values, -/// returned in a 64-bit vector of `[2 x i32]`. -/// If the result of either conversion is inexact, the result is truncated -/// (rounded towards zero) regardless of the current MXCSR setting. -#[inline] -#[target_feature(enable = "sse2,mmx")] -#[cfg_attr(test, assert_instr(cvttpd2pi))] -pub unsafe fn _mm_cvttpd_pi32(a: __m128d) -> __m64 { - cvttpd2pi(a) + simd_shuffle2!(a, b, [0, 2]) } #[allow(improper_ctypes)] @@ -3087,8 +2856,6 @@ extern "C" { fn packssdw(a: i32x4, b: i32x4) -> i16x8; #[link_name = "llvm.x86.sse2.packuswb.128"] fn packuswb(a: i16x8, b: i16x8) -> u8x16; - #[link_name = "llvm.x86.sse2.pmovmskb.128"] - fn pmovmskb(a: i8x16) -> i32; #[link_name = "llvm.x86.sse2.max.sd"] fn maxsd(a: __m128d, b: __m128d) -> __m128d; #[link_name = "llvm.x86.sse2.max.pd"] @@ -3153,18 +2920,6 @@ extern "C" { fn storeudq(mem_addr: *mut i8, a: __m128i); #[link_name = "llvm.x86.sse2.storeu.pd"] fn storeupd(mem_addr: *mut i8, a: __m128d); - #[link_name = "llvm.x86.mmx.padd.q"] - fn paddq(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.pmulu.dq"] - fn pmuludq2(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.mmx.psub.q"] - fn psubq(a: __m64, b: __m64) -> __m64; - #[link_name = "llvm.x86.sse.cvtpi2pd"] - fn cvtpi2pd(a: __m64) -> __m128d; - #[link_name = "llvm.x86.sse.cvtpd2pi"] - fn cvtpd2pi(a: __m128d) -> __m64; - #[link_name = "llvm.x86.sse.cvttpd2pi"] - fn cvttpd2pi(a: __m128d) -> __m64; } #[cfg(test)] @@ -3558,7 +3313,7 @@ mod tests { let a = _mm_setr_epi8( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, ); - let r = _mm_slli_si128(a, 1); + let r = _mm_slli_si128::<1>(a); let e = _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); assert_eq_m128i(r, e); @@ -3566,7 +3321,7 @@ mod tests { let a = _mm_setr_epi8( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, ); - let r = _mm_slli_si128(a, 15); + let r = _mm_slli_si128::<15>(a); let e = _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1); assert_eq_m128i(r, e); @@ -3574,21 +3329,7 @@ mod tests { let a = _mm_setr_epi8( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, ); - let r = _mm_slli_si128(a, 16); - assert_eq_m128i(r, _mm_set1_epi8(0)); - - #[rustfmt::skip] - let a = _mm_setr_epi8( - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - ); - let r = _mm_slli_si128(a, -1); - assert_eq_m128i(_mm_set1_epi8(0), r); - - #[rustfmt::skip] - let a = _mm_setr_epi8( - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - ); - let r = _mm_slli_si128(a, -0x80000000); + let r = _mm_slli_si128::<16>(a); assert_eq_m128i(r, _mm_set1_epi8(0)); } @@ -3598,7 +3339,7 @@ mod tests { let a = _mm_setr_epi16( 0xFFFF as u16 as i16, 0x0FFF, 0x00FF, 0x000F, 0, 0, 0, 0, ); - let r = _mm_slli_epi16(a, 4); + let r = _mm_slli_epi16::<4>(a); #[rustfmt::skip] let e = _mm_setr_epi16( @@ -3619,7 +3360,7 @@ mod tests { #[simd_test(enable = "sse2")] unsafe fn test_mm_slli_epi32() { - let r = _mm_slli_epi32(_mm_set1_epi32(0xFFFF), 4); + let r = _mm_slli_epi32::<4>(_mm_set1_epi32(0xFFFF)); assert_eq_m128i(r, _mm_set1_epi32(0xFFFF0)); } @@ -3633,7 +3374,7 @@ mod tests { #[simd_test(enable = "sse2")] unsafe fn test_mm_slli_epi64() { - let r = _mm_slli_epi64(_mm_set1_epi64x(0xFFFFFFFF), 4); + let r = _mm_slli_epi64::<4>(_mm_set1_epi64x(0xFFFFFFFF)); assert_eq_m128i(r, _mm_set1_epi64x(0xFFFFFFFF0)); } @@ -3647,7 +3388,7 @@ mod tests { #[simd_test(enable = "sse2")] unsafe fn test_mm_srai_epi16() { - let r = _mm_srai_epi16(_mm_set1_epi16(-1), 1); + let r = _mm_srai_epi16::<1>(_mm_set1_epi16(-1)); assert_eq_m128i(r, _mm_set1_epi16(-1)); } @@ -3661,7 +3402,7 @@ mod tests { #[simd_test(enable = "sse2")] unsafe fn test_mm_srai_epi32() { - let r = _mm_srai_epi32(_mm_set1_epi32(-1), 1); + let r = _mm_srai_epi32::<1>(_mm_set1_epi32(-1)); assert_eq_m128i(r, _mm_set1_epi32(-1)); } @@ -3679,7 +3420,7 @@ mod tests { let a = _mm_setr_epi8( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, ); - let r = _mm_srli_si128(a, 1); + let r = _mm_srli_si128::<1>(a); #[rustfmt::skip] let e = _mm_setr_epi8( 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 0, @@ -3690,7 +3431,7 @@ mod tests { let a = _mm_setr_epi8( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, ); - let r = _mm_srli_si128(a, 15); + let r = _mm_srli_si128::<15>(a); let e = _mm_setr_epi8(16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); assert_eq_m128i(r, e); @@ -3698,21 +3439,7 @@ mod tests { let a = _mm_setr_epi8( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, ); - let r = _mm_srli_si128(a, 16); - assert_eq_m128i(r, _mm_set1_epi8(0)); - - #[rustfmt::skip] - let a = _mm_setr_epi8( - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - ); - let r = _mm_srli_si128(a, -1); - assert_eq_m128i(r, _mm_set1_epi8(0)); - - #[rustfmt::skip] - let a = _mm_setr_epi8( - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - ); - let r = _mm_srli_si128(a, -0x80000000); + let r = _mm_srli_si128::<16>(a); assert_eq_m128i(r, _mm_set1_epi8(0)); } @@ -3722,7 +3449,7 @@ mod tests { let a = _mm_setr_epi16( 0xFFFF as u16 as i16, 0x0FFF, 0x00FF, 0x000F, 0, 0, 0, 0, ); - let r = _mm_srli_epi16(a, 4); + let r = _mm_srli_epi16::<4>(a); #[rustfmt::skip] let e = _mm_setr_epi16( 0xFFF as u16 as i16, 0xFF as u16 as i16, 0xF, 0, 0, 0, 0, 0, @@ -3741,7 +3468,7 @@ mod tests { #[simd_test(enable = "sse2")] unsafe fn test_mm_srli_epi32() { - let r = _mm_srli_epi32(_mm_set1_epi32(0xFFFF), 4); + let r = _mm_srli_epi32::<4>(_mm_set1_epi32(0xFFFF)); assert_eq_m128i(r, _mm_set1_epi32(0xFFF)); } @@ -3755,7 +3482,7 @@ mod tests { #[simd_test(enable = "sse2")] unsafe fn test_mm_srli_epi64() { - let r = _mm_srli_epi64(_mm_set1_epi64x(0xFFFFFFFF), 4); + let r = _mm_srli_epi64::<4>(_mm_set1_epi64x(0xFFFFFFFF)); assert_eq_m128i(r, _mm_set1_epi64x(0xFFFFFFF)); } @@ -4130,16 +3857,16 @@ mod tests { #[simd_test(enable = "sse2")] unsafe fn test_mm_extract_epi16() { let a = _mm_setr_epi16(-1, 1, 2, 3, 4, 5, 6, 7); - let r1 = _mm_extract_epi16(a, 0); - let r2 = _mm_extract_epi16(a, 11); - assert_eq!(r1, -1); + let r1 = _mm_extract_epi16::<0>(a); + let r2 = _mm_extract_epi16::<3>(a); + assert_eq!(r1, 0xFFFF); assert_eq!(r2, 3); } #[simd_test(enable = "sse2")] unsafe fn test_mm_insert_epi16() { let a = _mm_setr_epi16(0, 1, 2, 3, 4, 5, 6, 7); - let r = _mm_insert_epi16(a, 9, 0); + let r = _mm_insert_epi16::<0>(a, 9); let e = _mm_setr_epi16(9, 1, 2, 3, 4, 5, 6, 7); assert_eq_m128i(r, e); } @@ -4160,7 +3887,7 @@ mod tests { #[simd_test(enable = "sse2")] unsafe fn test_mm_shuffle_epi32() { let a = _mm_setr_epi32(5, 10, 15, 20); - let r = _mm_shuffle_epi32(a, 0b00_01_01_11); + let r = _mm_shuffle_epi32::<0b00_01_01_11>(a); let e = _mm_setr_epi32(20, 10, 10, 5); assert_eq_m128i(r, e); } @@ -4168,7 +3895,7 @@ mod tests { #[simd_test(enable = "sse2")] unsafe fn test_mm_shufflehi_epi16() { let a = _mm_setr_epi16(1, 2, 3, 4, 5, 10, 15, 20); - let r = _mm_shufflehi_epi16(a, 0b00_01_01_11); + let r = _mm_shufflehi_epi16::<0b00_01_01_11>(a); let e = _mm_setr_epi16(1, 2, 3, 4, 20, 10, 10, 5); assert_eq_m128i(r, e); } @@ -4176,7 +3903,7 @@ mod tests { #[simd_test(enable = "sse2")] unsafe fn test_mm_shufflelo_epi16() { let a = _mm_setr_epi16(5, 10, 15, 20, 1, 2, 3, 4); - let r = _mm_shufflelo_epi16(a, 0b00_01_01_11); + let r = _mm_shufflelo_epi16::<0b00_01_01_11>(a); let e = _mm_setr_epi16(20, 10, 10, 5, 1, 2, 3, 4); assert_eq_m128i(r, e); } @@ -4777,7 +4504,7 @@ mod tests { assert_eq!(vals[1], 2.0); } - #[simd_test(enable = "sse")] + #[simd_test(enable = "sse2")] unsafe fn test_mm_storeu_pd() { let mut mem = Memory { data: [0.0f64; 4] }; let vals = &mut mem.data; @@ -4789,7 +4516,7 @@ mod tests { // Make sure p is **not** aligned to 16-byte boundary if (p as usize) & 0xf == 0 { ofs = 1; - p = p.offset(1); + p = p.add(1); } _mm_storeu_pd(p, *black_box(&a)); @@ -4877,7 +4604,7 @@ mod tests { let mut offset = 0; if (d as usize) & 0xf == 0 { offset = 1; - d = d.offset(offset as isize); + d = d.add(offset); } let r = _mm_loadu_pd(d); @@ -5094,7 +4821,7 @@ mod tests { let a = _mm_setr_pd(1., 2.); let b = _mm_setr_pd(3., 4.); let expected = _mm_setr_pd(1., 3.); - let r = _mm_shuffle_pd(a, b, 0); + let r = _mm_shuffle_pd::<0b00_00_00_00>(a, b); assert_eq_m128d(r, expected); } @@ -5154,89 +4881,4 @@ mod tests { let r = _mm_castsi128_ps(a); assert_eq_m128(r, expected); } - - #[simd_test(enable = "sse2,mmx")] - unsafe fn test_mm_add_si64() { - let a = 1i64; - let b = 2i64; - let expected = 3i64; - let r = _mm_add_si64(transmute(a), transmute(b)); - assert_eq!(transmute::<__m64, i64>(r), expected); - } - - #[simd_test(enable = "sse2,mmx")] - unsafe fn test_mm_mul_su32() { - let a = _mm_setr_pi32(1, 2); - let b = _mm_setr_pi32(3, 4); - let expected = 3u64; - let r = _mm_mul_su32(a, b); - assert_eq_m64(r, transmute(expected)); - } - - #[simd_test(enable = "sse2,mmx")] - unsafe fn test_mm_sub_si64() { - let a = 1i64; - let b = 2i64; - let expected = -1i64; - let r = _mm_sub_si64(transmute(a), transmute(b)); - assert_eq!(transmute::<__m64, i64>(r), expected); - } - - #[simd_test(enable = "sse2,mmx")] - unsafe fn test_mm_cvtpi32_pd() { - let a = _mm_setr_pi32(1, 2); - let expected = _mm_setr_pd(1., 2.); - let r = _mm_cvtpi32_pd(a); - assert_eq_m128d(r, expected); - } - - #[simd_test(enable = "sse2,mmx")] - unsafe fn test_mm_set_epi64() { - let r = _mm_set_epi64(transmute(1i64), transmute(2i64)); - assert_eq_m128i(r, _mm_setr_epi64x(2, 1)); - } - - #[simd_test(enable = "sse2,mmx")] - unsafe fn test_mm_set1_epi64() { - let r = _mm_set1_epi64(transmute(1i64)); - assert_eq_m128i(r, _mm_setr_epi64x(1, 1)); - } - - #[simd_test(enable = "sse2,mmx")] - unsafe fn test_mm_setr_epi64() { - let r = _mm_setr_epi64(transmute(1i64), transmute(2i64)); - assert_eq_m128i(r, _mm_setr_epi64x(1, 2)); - } - - #[simd_test(enable = "sse2,mmx")] - unsafe fn test_mm_movepi64_pi64() { - let r = _mm_movepi64_pi64(_mm_setr_epi64x(5, 0)); - assert_eq_m64(r, _mm_setr_pi8(5, 0, 0, 0, 0, 0, 0, 0)); - } - - #[simd_test(enable = "sse2,mmx")] - unsafe fn test_mm_movpi64_epi64() { - let r = _mm_movpi64_epi64(_mm_setr_pi8(5, 0, 0, 0, 0, 0, 0, 0)); - assert_eq_m128i(r, _mm_setr_epi64x(5, 0)); - } - - #[simd_test(enable = "sse2,mmx")] - unsafe fn test_mm_cvtpd_pi32() { - let a = _mm_setr_pd(5., 0.); - let r = _mm_cvtpd_pi32(a); - assert_eq_m64(r, _mm_setr_pi32(5, 0)); - } - - #[simd_test(enable = "sse2,mmx")] - unsafe fn test_mm_cvttpd_pi32() { - use std::{f64, i32}; - - let a = _mm_setr_pd(5., 0.); - let r = _mm_cvttpd_pi32(a); - assert_eq_m64(r, _mm_setr_pi32(5, 0)); - - let a = _mm_setr_pd(f64::NEG_INFINITY, f64::NAN); - let r = _mm_cvttpd_pi32(a); - assert_eq_m64(r, _mm_setr_pi32(i32::MIN, i32::MIN)); - } } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse3.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse3.rs index 977de1dc1..61f8a4e78 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse3.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse3.rs @@ -1,11 +1,7 @@ //! Streaming SIMD Extensions 3 (SSE3) use crate::{ - core_arch::{ - simd::*, - simd_llvm::{simd_shuffle2, simd_shuffle4}, - x86::*, - }, + core_arch::{simd::*, simd_llvm::simd_shuffle, x86::*}, mem::transmute, }; @@ -106,7 +102,7 @@ pub unsafe fn _mm_lddqu_si128(mem_addr: *const __m128i) -> __m128i { #[cfg_attr(test, assert_instr(movddup))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_movedup_pd(a: __m128d) -> __m128d { - simd_shuffle2(a, a, [0, 0]) + simd_shuffle2!(a, a, [0, 0]) } /// Loads a double-precision (64-bit) floating-point element from memory @@ -130,7 +126,7 @@ pub unsafe fn _mm_loaddup_pd(mem_addr: *const f64) -> __m128d { #[cfg_attr(test, assert_instr(movshdup))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_movehdup_ps(a: __m128) -> __m128 { - simd_shuffle4(a, a, [1, 1, 3, 3]) + simd_shuffle4!(a, a, [1, 1, 3, 3]) } /// Duplicate even-indexed single-precision (32-bit) floating-point elements @@ -142,7 +138,7 @@ pub unsafe fn _mm_movehdup_ps(a: __m128) -> __m128 { #[cfg_attr(test, assert_instr(movsldup))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_moveldup_ps(a: __m128) -> __m128 { - simd_shuffle4(a, a, [0, 0, 2, 2]) + simd_shuffle4!(a, a, [0, 0, 2, 2]) } #[allow(improper_ctypes)] diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse41.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse41.rs index 19ba1a847..7c59f2702 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse41.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse41.rs @@ -8,7 +8,7 @@ use crate::{ #[cfg(test)] use stdarch_test::assert_instr; -// SSE4 rounding constans +// SSE4 rounding constants /// round to nearest #[stable(feature = "simd_x86", since = "1.27.0")] pub const _MM_FROUND_TO_NEAREST_INT: i32 = 0x00; @@ -65,7 +65,7 @@ pub unsafe fn _mm_blendv_epi8(a: __m128i, b: __m128i, mask: __m128i) -> __m128i transmute(pblendvb(a.as_i8x16(), b.as_i8x16(), mask.as_i8x16())) } -/// Blend packed 16-bit integers from `a` and `b` using the mask `imm8`. +/// Blend packed 16-bit integers from `a` and `b` using the mask `IMM8`. /// /// The mask bits determine the selection. A clear bit selects the /// corresponding element of `a`, and a set bit the corresponding @@ -76,19 +76,13 @@ pub unsafe fn _mm_blendv_epi8(a: __m128i, b: __m128i, mask: __m128i) -> __m128i #[target_feature(enable = "sse4.1")] // Note: LLVM7 prefers the single-precision floating-point domain when possible // see https://bugs.llvm.org/show_bug.cgi?id=38195 -// #[cfg_attr(test, assert_instr(pblendw, imm8 = 0xF0))] -#[cfg_attr(test, assert_instr(blendps, imm8 = 0xF0))] -#[rustc_args_required_const(2)] +// #[cfg_attr(test, assert_instr(pblendw, IMM8 = 0xF0))] +#[cfg_attr(test, assert_instr(blendps, IMM8 = 0xF0))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_blend_epi16(a: __m128i, b: __m128i, imm8: i32) -> __m128i { - let a = a.as_i16x8(); - let b = b.as_i16x8(); - macro_rules! call { - ($imm8:expr) => { - pblendw(a, b, $imm8) - }; - } - transmute(constify_imm8!(imm8, call)) +pub unsafe fn _mm_blend_epi16(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(pblendw(a.as_i16x8(), b.as_i16x8(), IMM8 as u8)) } /// Blend packed double-precision (64-bit) floating-point elements from `a` @@ -116,96 +110,110 @@ pub unsafe fn _mm_blendv_ps(a: __m128, b: __m128, mask: __m128) -> __m128 { } /// Blend packed double-precision (64-bit) floating-point elements from `a` -/// and `b` using control mask `imm2` +/// and `b` using control mask `IMM2` /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_blend_pd) #[inline] #[target_feature(enable = "sse4.1")] // Note: LLVM7 prefers the single-precision floating-point domain when possible // see https://bugs.llvm.org/show_bug.cgi?id=38195 -// #[cfg_attr(test, assert_instr(blendpd, imm2 = 0b10))] -#[cfg_attr(test, assert_instr(blendps, imm2 = 0b10))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_blend_pd(a: __m128d, b: __m128d, imm2: i32) -> __m128d { - macro_rules! call { - ($imm2:expr) => { - blendpd(a, b, $imm2) - }; - } - constify_imm2!(imm2, call) +// #[cfg_attr(test, assert_instr(blendpd, IMM2 = 0b10))] +#[cfg_attr(test, assert_instr(blendps, IMM2 = 0b10))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_blend_pd(a: __m128d, b: __m128d) -> __m128d { + static_assert_imm2!(IMM2); + blendpd(a, b, IMM2 as u8) } /// Blend packed single-precision (32-bit) floating-point elements from `a` -/// and `b` using mask `imm4` +/// and `b` using mask `IMM4` /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_blend_ps) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(blendps, imm4 = 0b0101))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_blend_ps(a: __m128, b: __m128, imm4: i32) -> __m128 { - macro_rules! call { - ($imm4:expr) => { - blendps(a, b, $imm4) - }; - } - constify_imm4!(imm4, call) +#[cfg_attr(test, assert_instr(blendps, IMM4 = 0b0101))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_blend_ps(a: __m128, b: __m128) -> __m128 { + static_assert_imm4!(IMM4); + blendps(a, b, IMM4 as u8) } /// Extracts a single-precision (32-bit) floating-point element from `a`, -/// selected with `imm8` -/// +/// selected with `IMM8`. The returned `i32` stores the float's bit-pattern, +/// and may be converted back to a floating point number via casting. +/// +/// # Example +/// ```rust +/// # #[cfg(target_arch = "x86")] +/// # use std::arch::x86::*; +/// # #[cfg(target_arch = "x86_64")] +/// # use std::arch::x86_64::*; +/// # fn main() { +/// # if is_x86_feature_detected!("sse4.1") { +/// # #[target_feature(enable = "sse4.1")] +/// # unsafe fn worker() { +/// let mut float_store = vec![1.0, 1.0, 2.0, 3.0]; +/// let simd_floats = _mm_set_ps(2.5, 5.0, 7.5, 10.0); +/// let x: i32 = _mm_extract_ps::<2>(simd_floats); +/// float_store.push(f32::from_bits(x as u32)); +/// assert_eq!(float_store, vec![1.0, 1.0, 2.0, 3.0, 5.0]); +/// # } +/// # unsafe { worker() } +/// # } +/// # } +/// ``` /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_extract_ps) #[inline] #[target_feature(enable = "sse4.1")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(extractps, imm8 = 0) + assert_instr(extractps, IMM8 = 0) )] -#[rustc_args_required_const(1)] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_extract_ps(a: __m128, imm8: i32) -> i32 { - transmute(simd_extract::<_, f32>(a, imm8 as u32 & 0b11)) +pub unsafe fn _mm_extract_ps(a: __m128) -> i32 { + static_assert_imm2!(IMM8); + transmute(simd_extract::<_, f32>(a, IMM8 as u32)) } -/// Extracts an 8-bit integer from `a`, selected with `imm8`. Returns a 32-bit +/// Extracts an 8-bit integer from `a`, selected with `IMM8`. Returns a 32-bit /// integer containing the zero-extended integer data. /// -/// See [LLVM commit D20468][https://reviews.llvm.org/D20468]. +/// See [LLVM commit D20468](https://reviews.llvm.org/D20468). /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_extract_epi8) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(pextrb, imm8 = 0))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(pextrb, IMM8 = 0))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_extract_epi8(a: __m128i, imm8: i32) -> i32 { - let imm8 = (imm8 & 15) as u32; - simd_extract::<_, u8>(a.as_u8x16(), imm8) as i32 +pub unsafe fn _mm_extract_epi8(a: __m128i) -> i32 { + static_assert_imm4!(IMM8); + simd_extract::<_, u8>(a.as_u8x16(), IMM8 as u32) as i32 } -/// Extracts an 32-bit integer from `a` selected with `imm8` +/// Extracts an 32-bit integer from `a` selected with `IMM8` /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_extract_epi32) #[inline] #[target_feature(enable = "sse4.1")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(extractps, imm8 = 1) + assert_instr(extractps, IMM8 = 1) )] -#[rustc_args_required_const(1)] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_extract_epi32(a: __m128i, imm8: i32) -> i32 { - let imm8 = (imm8 & 3) as u32; - simd_extract::<_, i32>(a.as_i32x4(), imm8) +pub unsafe fn _mm_extract_epi32(a: __m128i) -> i32 { + static_assert_imm2!(IMM8); + simd_extract::<_, i32>(a.as_i32x4(), IMM8 as u32) } /// Select a single value in `a` to store at some position in `b`, -/// Then zero elements according to `imm8`. +/// Then zero elements according to `IMM8`. /// -/// `imm8` specifies which bits from operand `a` will be copied, which bits in +/// `IMM8` specifies which bits from operand `a` will be copied, which bits in /// the result they will be copied to, and which bits in the result will be /// cleared. The following assignments are made: /// @@ -228,42 +236,40 @@ pub unsafe fn _mm_extract_epi32(a: __m128i, imm8: i32) -> i32 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_insert_ps) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(insertps, imm8 = 0b1010))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_insert_ps(a: __m128, b: __m128, imm8: i32) -> __m128 { - macro_rules! call { - ($imm8:expr) => { - insertps(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(insertps, IMM8 = 0b1010))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_insert_ps(a: __m128, b: __m128) -> __m128 { + static_assert_imm8!(IMM8); + insertps(a, b, IMM8 as u8) } /// Returns a copy of `a` with the 8-bit integer from `i` inserted at a -/// location specified by `imm8`. +/// location specified by `IMM8`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_insert_epi8) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(pinsrb, imm8 = 0))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(pinsrb, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_insert_epi8(a: __m128i, i: i32, imm8: i32) -> __m128i { - transmute(simd_insert(a.as_i8x16(), (imm8 & 0b1111) as u32, i as i8)) +pub unsafe fn _mm_insert_epi8(a: __m128i, i: i32) -> __m128i { + static_assert_imm4!(IMM8); + transmute(simd_insert(a.as_i8x16(), IMM8 as u32, i as i8)) } /// Returns a copy of `a` with the 32-bit integer from `i` inserted at a -/// location specified by `imm8`. +/// location specified by `IMM8`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_insert_epi32) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(pinsrd, imm8 = 0))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(pinsrd, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_insert_epi32(a: __m128i, i: i32, imm8: i32) -> __m128i { - transmute(simd_insert(a.as_i32x4(), (imm8 & 0b11) as u32, i)) +pub unsafe fn _mm_insert_epi32(a: __m128i, i: i32) -> __m128i { + static_assert_imm2!(IMM8); + transmute(simd_insert(a.as_i32x4(), IMM8 as u32, i)) } /// Compares packed 8-bit integers in `a` and `b` and returns packed maximum @@ -394,7 +400,7 @@ pub unsafe fn _mm_cmpeq_epi64(a: __m128i, b: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepi8_epi16(a: __m128i) -> __m128i { let a = a.as_i8x16(); - let a = simd_shuffle8::<_, i8x8>(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); + let a: i8x8 = simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); transmute(simd_cast::<_, i16x8>(a)) } @@ -407,7 +413,7 @@ pub unsafe fn _mm_cvtepi8_epi16(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepi8_epi32(a: __m128i) -> __m128i { let a = a.as_i8x16(); - let a = simd_shuffle4::<_, i8x4>(a, a, [0, 1, 2, 3]); + let a: i8x4 = simd_shuffle4!(a, a, [0, 1, 2, 3]); transmute(simd_cast::<_, i32x4>(a)) } @@ -421,7 +427,7 @@ pub unsafe fn _mm_cvtepi8_epi32(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepi8_epi64(a: __m128i) -> __m128i { let a = a.as_i8x16(); - let a = simd_shuffle2::<_, i8x2>(a, a, [0, 1]); + let a: i8x2 = simd_shuffle2!(a, a, [0, 1]); transmute(simd_cast::<_, i64x2>(a)) } @@ -434,7 +440,7 @@ pub unsafe fn _mm_cvtepi8_epi64(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepi16_epi32(a: __m128i) -> __m128i { let a = a.as_i16x8(); - let a = simd_shuffle4::<_, i16x4>(a, a, [0, 1, 2, 3]); + let a: i16x4 = simd_shuffle4!(a, a, [0, 1, 2, 3]); transmute(simd_cast::<_, i32x4>(a)) } @@ -447,7 +453,7 @@ pub unsafe fn _mm_cvtepi16_epi32(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepi16_epi64(a: __m128i) -> __m128i { let a = a.as_i16x8(); - let a = simd_shuffle2::<_, i16x2>(a, a, [0, 1]); + let a: i16x2 = simd_shuffle2!(a, a, [0, 1]); transmute(simd_cast::<_, i64x2>(a)) } @@ -460,7 +466,7 @@ pub unsafe fn _mm_cvtepi16_epi64(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepi32_epi64(a: __m128i) -> __m128i { let a = a.as_i32x4(); - let a = simd_shuffle2::<_, i32x2>(a, a, [0, 1]); + let a: i32x2 = simd_shuffle2!(a, a, [0, 1]); transmute(simd_cast::<_, i64x2>(a)) } @@ -473,7 +479,7 @@ pub unsafe fn _mm_cvtepi32_epi64(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepu8_epi16(a: __m128i) -> __m128i { let a = a.as_u8x16(); - let a = simd_shuffle8::<_, u8x8>(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); + let a: u8x8 = simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); transmute(simd_cast::<_, i16x8>(a)) } @@ -486,7 +492,7 @@ pub unsafe fn _mm_cvtepu8_epi16(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepu8_epi32(a: __m128i) -> __m128i { let a = a.as_u8x16(); - let a = simd_shuffle4::<_, u8x4>(a, a, [0, 1, 2, 3]); + let a: u8x4 = simd_shuffle4!(a, a, [0, 1, 2, 3]); transmute(simd_cast::<_, i32x4>(a)) } @@ -499,7 +505,7 @@ pub unsafe fn _mm_cvtepu8_epi32(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepu8_epi64(a: __m128i) -> __m128i { let a = a.as_u8x16(); - let a = simd_shuffle2::<_, u8x2>(a, a, [0, 1]); + let a: u8x2 = simd_shuffle2!(a, a, [0, 1]); transmute(simd_cast::<_, i64x2>(a)) } @@ -513,7 +519,7 @@ pub unsafe fn _mm_cvtepu8_epi64(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepu16_epi32(a: __m128i) -> __m128i { let a = a.as_u16x8(); - let a = simd_shuffle4::<_, u16x4>(a, a, [0, 1, 2, 3]); + let a: u16x4 = simd_shuffle4!(a, a, [0, 1, 2, 3]); transmute(simd_cast::<_, i32x4>(a)) } @@ -527,7 +533,7 @@ pub unsafe fn _mm_cvtepu16_epi32(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepu16_epi64(a: __m128i) -> __m128i { let a = a.as_u16x8(); - let a = simd_shuffle2::<_, u16x2>(a, a, [0, 1]); + let a: u16x2 = simd_shuffle2!(a, a, [0, 1]); transmute(simd_cast::<_, i64x2>(a)) } @@ -541,13 +547,13 @@ pub unsafe fn _mm_cvtepu16_epi64(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepu32_epi64(a: __m128i) -> __m128i { let a = a.as_u32x4(); - let a = simd_shuffle2::<_, u32x2>(a, a, [0, 1]); + let a: u32x2 = simd_shuffle2!(a, a, [0, 1]); transmute(simd_cast::<_, i64x2>(a)) } /// Returns the dot product of two __m128d vectors. /// -/// `imm8[1:0]` is the broadcast mask, and `imm8[5:4]` is the condition mask. +/// `IMM8[1:0]` is the broadcast mask, and `IMM8[5:4]` is the condition mask. /// If a condition mask bit is zero, the corresponding multiplication is /// replaced by a value of `0.0`. If a broadcast mask bit is one, the result of /// the dot product will be stored in the return value component. Otherwise if @@ -556,21 +562,17 @@ pub unsafe fn _mm_cvtepu32_epi64(a: __m128i) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_dp_pd) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(dppd, imm8 = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_dp_pd(a: __m128d, b: __m128d, imm8: i32) -> __m128d { - macro_rules! call { - ($imm8:expr) => { - dppd(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(dppd, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_dp_pd(a: __m128d, b: __m128d) -> __m128d { + static_assert_imm8!(IMM8); + dppd(a, b, IMM8 as u8) } /// Returns the dot product of two __m128 vectors. /// -/// `imm8[3:0]` is the broadcast mask, and `imm8[7:4]` is the condition mask. +/// `IMM8[3:0]` is the broadcast mask, and `IMM8[7:4]` is the condition mask. /// If a condition mask bit is zero, the corresponding multiplication is /// replaced by a value of `0.0`. If a broadcast mask bit is one, the result of /// the dot product will be stored in the return value component. Otherwise if @@ -579,16 +581,12 @@ pub unsafe fn _mm_dp_pd(a: __m128d, b: __m128d, imm8: i32) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_dp_ps) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(dpps, imm8 = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_dp_ps(a: __m128, b: __m128, imm8: i32) -> __m128 { - macro_rules! call { - ($imm8:expr) => { - dpps(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(dpps, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_dp_ps(a: __m128, b: __m128) -> __m128 { + static_assert_imm8!(IMM8); + dpps(a, b, IMM8 as u8) } /// Round the packed double-precision (64-bit) floating-point elements in `a` @@ -704,7 +702,7 @@ pub unsafe fn _mm_ceil_ss(a: __m128, b: __m128) -> __m128 { } /// Round the packed double-precision (64-bit) floating-point elements in `a` -/// using the `rounding` parameter, and stores the results as packed +/// using the `ROUNDING` parameter, and stores the results as packed /// double-precision floating-point elements. /// Rounding is done according to the rounding parameter, which can be one of: /// @@ -736,20 +734,16 @@ pub unsafe fn _mm_ceil_ss(a: __m128, b: __m128) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_round_pd) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(roundpd, rounding = 0))] -#[rustc_args_required_const(1)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_round_pd(a: __m128d, rounding: i32) -> __m128d { - macro_rules! call { - ($imm4:expr) => { - roundpd(a, $imm4) - }; - } - constify_imm4!(rounding, call) +#[cfg_attr(test, assert_instr(roundpd, ROUNDING = 0))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_round_pd(a: __m128d) -> __m128d { + static_assert_imm4!(ROUNDING); + roundpd(a, ROUNDING) } /// Round the packed single-precision (32-bit) floating-point elements in `a` -/// using the `rounding` parameter, and stores the results as packed +/// using the `ROUNDING` parameter, and stores the results as packed /// single-precision floating-point elements. /// Rounding is done according to the rounding parameter, which can be one of: /// @@ -781,20 +775,16 @@ pub unsafe fn _mm_round_pd(a: __m128d, rounding: i32) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_round_ps) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(roundps, rounding = 0))] -#[rustc_args_required_const(1)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_round_ps(a: __m128, rounding: i32) -> __m128 { - macro_rules! call { - ($imm4:expr) => { - roundps(a, $imm4) - }; - } - constify_imm4!(rounding, call) +#[cfg_attr(test, assert_instr(roundps, ROUNDING = 0))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_round_ps(a: __m128) -> __m128 { + static_assert_imm4!(ROUNDING); + roundps(a, ROUNDING) } /// Round the lower double-precision (64-bit) floating-point element in `b` -/// using the `rounding` parameter, store the result as a double-precision +/// using the `ROUNDING` parameter, store the result as a double-precision /// floating-point element in the lower element of the intrinsic result, /// and copies the upper element from `a` to the upper element of the intrinsic /// result. @@ -828,23 +818,19 @@ pub unsafe fn _mm_round_ps(a: __m128, rounding: i32) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_round_sd) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(roundsd, rounding = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_round_sd(a: __m128d, b: __m128d, rounding: i32) -> __m128d { - macro_rules! call { - ($imm4:expr) => { - roundsd(a, b, $imm4) - }; - } - constify_imm4!(rounding, call) +#[cfg_attr(test, assert_instr(roundsd, ROUNDING = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_round_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_imm4!(ROUNDING); + roundsd(a, b, ROUNDING) } /// Round the lower single-precision (32-bit) floating-point element in `b` -/// using the `rounding` parameter, store the result as a single-precision +/// using the `ROUNDING` parameter, store the result as a single-precision /// floating-point element in the lower element of the intrinsic result, /// and copies the upper 3 packed elements from `a` to the upper elements -/// of the instrinsic result. +/// of the intrinsic result. /// Rounding is done according to the rounding parameter, which can be one of: /// /// ``` @@ -875,16 +861,12 @@ pub unsafe fn _mm_round_sd(a: __m128d, b: __m128d, rounding: i32) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_round_ss) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(roundss, rounding = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_round_ss(a: __m128, b: __m128, rounding: i32) -> __m128 { - macro_rules! call { - ($imm4:expr) => { - roundss(a, b, $imm4) - }; - } - constify_imm4!(rounding, call) +#[cfg_attr(test, assert_instr(roundss, ROUNDING = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_round_ss(a: __m128, b: __m128) -> __m128 { + static_assert_imm4!(ROUNDING); + roundss(a, b, ROUNDING) } /// Finds the minimum unsigned 16-bit element in the 128-bit __m128i vector, @@ -892,7 +874,7 @@ pub unsafe fn _mm_round_ss(a: __m128, b: __m128, rounding: i32) -> __m128 { /// index /// in its second position; all other elements are set to zero. /// -/// This intrinsic corresponds to the VPHMINPOSUW / PHMINPOSUW +/// This intrinsic corresponds to the `VPHMINPOSUW` / `PHMINPOSUW` /// instruction. /// /// Arguments: @@ -952,8 +934,8 @@ pub unsafe fn _mm_mullo_epi32(a: __m128i, b: __m128i) -> __m128i { /// The following algorithm is performed: /// /// ```ignore -/// i = imm8[2] * 4 -/// j = imm8[1:0] * 4 +/// i = IMM8[2] * 4 +/// j = IMM8[1:0] * 4 /// for k := 0 to 7 /// d0 = abs(a[i + k + 0] - b[j + 0]) /// d1 = abs(a[i + k + 1] - b[j + 1]) @@ -966,7 +948,7 @@ pub unsafe fn _mm_mullo_epi32(a: __m128i, b: __m128i) -> __m128i { /// /// * `a` - A 128-bit vector of type `__m128i`. /// * `b` - A 128-bit vector of type `__m128i`. -/// * `imm8` - An 8-bit immediate operand specifying how the absolute +/// * `IMM8` - An 8-bit immediate operand specifying how the absolute /// differences are to be calculated /// * Bit `[2]` specify the offset for operand `a` /// * Bits `[1:0]` specify the offset for operand `b` @@ -979,18 +961,12 @@ pub unsafe fn _mm_mullo_epi32(a: __m128i, b: __m128i) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mpsadbw_epu8) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(mpsadbw, imm8 = 0))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(mpsadbw, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_mpsadbw_epu8(a: __m128i, b: __m128i, imm8: i32) -> __m128i { - let a = a.as_u8x16(); - let b = b.as_u8x16(); - macro_rules! call { - ($imm8:expr) => { - mpsadbw(a, b, $imm8) - }; - } - transmute(constify_imm3!(imm8, call)) +pub unsafe fn _mm_mpsadbw_epu8(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm3!(IMM8); + transmute(mpsadbw(a.as_u8x16(), b.as_u8x16(), IMM8 as u8)) } /// Tests whether the specified bits in a 128-bit integer vector are all @@ -1243,7 +1219,7 @@ mod tests { unsafe fn test_mm_blend_pd() { let a = _mm_set1_pd(0.0); let b = _mm_set1_pd(1.0); - let r = _mm_blend_pd(a, b, 0b10); + let r = _mm_blend_pd::<0b10>(a, b); let e = _mm_setr_pd(0.0, 1.0); assert_eq_m128d(r, e); } @@ -1252,7 +1228,7 @@ mod tests { unsafe fn test_mm_blend_ps() { let a = _mm_set1_ps(0.0); let b = _mm_set1_ps(1.0); - let r = _mm_blend_ps(a, b, 0b1010); + let r = _mm_blend_ps::<0b1010>(a, b); let e = _mm_setr_ps(0.0, 1.0, 0.0, 1.0); assert_eq_m128(r, e); } @@ -1261,7 +1237,7 @@ mod tests { unsafe fn test_mm_blend_epi16() { let a = _mm_set1_epi16(0); let b = _mm_set1_epi16(1); - let r = _mm_blend_epi16(a, b, 0b1010_1100); + let r = _mm_blend_epi16::<0b1010_1100>(a, b); let e = _mm_setr_epi16(0, 0, 1, 1, 0, 1, 0, 1); assert_eq_m128i(r, e); } @@ -1269,10 +1245,10 @@ mod tests { #[simd_test(enable = "sse4.1")] unsafe fn test_mm_extract_ps() { let a = _mm_setr_ps(0.0, 1.0, 2.0, 3.0); - let r: f32 = transmute(_mm_extract_ps(a, 1)); - assert_eq!(r, 1.0); - let r: f32 = transmute(_mm_extract_ps(a, 5)); + let r: f32 = transmute(_mm_extract_ps::<1>(a)); assert_eq!(r, 1.0); + let r: f32 = transmute(_mm_extract_ps::<3>(a)); + assert_eq!(r, 3.0); } #[simd_test(enable = "sse4.1")] @@ -1282,8 +1258,8 @@ mod tests { -1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ); - let r1 = _mm_extract_epi8(a, 0); - let r2 = _mm_extract_epi8(a, 19); + let r1 = _mm_extract_epi8::<0>(a); + let r2 = _mm_extract_epi8::<3>(a); assert_eq!(r1, 0xFF); assert_eq!(r2, 3); } @@ -1291,17 +1267,17 @@ mod tests { #[simd_test(enable = "sse4.1")] unsafe fn test_mm_extract_epi32() { let a = _mm_setr_epi32(0, 1, 2, 3); - let r = _mm_extract_epi32(a, 1); - assert_eq!(r, 1); - let r = _mm_extract_epi32(a, 5); + let r = _mm_extract_epi32::<1>(a); assert_eq!(r, 1); + let r = _mm_extract_epi32::<3>(a); + assert_eq!(r, 3); } #[simd_test(enable = "sse4.1")] unsafe fn test_mm_insert_ps() { let a = _mm_set1_ps(1.0); let b = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - let r = _mm_insert_ps(a, b, 0b11_00_1100); + let r = _mm_insert_ps::<0b11_00_1100>(a, b); let e = _mm_setr_ps(4.0, 1.0, 0.0, 0.0); assert_eq_m128(r, e); } @@ -1310,9 +1286,10 @@ mod tests { unsafe fn test_mm_insert_epi8() { let a = _mm_set1_epi8(0); let e = _mm_setr_epi8(0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); - let r = _mm_insert_epi8(a, 32, 1); + let r = _mm_insert_epi8::<1>(a, 32); assert_eq_m128i(r, e); - let r = _mm_insert_epi8(a, 32, 17); + let e = _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0); + let r = _mm_insert_epi8::<14>(a, 32); assert_eq_m128i(r, e); } @@ -1320,9 +1297,10 @@ mod tests { unsafe fn test_mm_insert_epi32() { let a = _mm_set1_epi32(0); let e = _mm_setr_epi32(0, 32, 0, 0); - let r = _mm_insert_epi32(a, 32, 1); + let r = _mm_insert_epi32::<1>(a, 32); assert_eq_m128i(r, e); - let r = _mm_insert_epi32(a, 32, 5); + let e = _mm_setr_epi32(0, 0, 0, 32); + let r = _mm_insert_epi32::<3>(a, 32); assert_eq_m128i(r, e); } @@ -1595,7 +1573,7 @@ mod tests { let a = _mm_setr_pd(2.0, 3.0); let b = _mm_setr_pd(1.0, 4.0); let e = _mm_setr_pd(14.0, 0.0); - assert_eq_m128d(_mm_dp_pd(a, b, 0b00110001), e); + assert_eq_m128d(_mm_dp_pd::<0b00110001>(a, b), e); } #[simd_test(enable = "sse4.1")] @@ -1603,7 +1581,7 @@ mod tests { let a = _mm_setr_ps(2.0, 3.0, 1.0, 10.0); let b = _mm_setr_ps(1.0, 4.0, 0.5, 10.0); let e = _mm_setr_ps(14.5, 0.0, 14.5, 0.0); - assert_eq_m128(_mm_dp_ps(a, b, 0b01110101), e); + assert_eq_m128(_mm_dp_ps::<0b01110101>(a, b), e); } #[simd_test(enable = "sse4.1")] @@ -1677,7 +1655,7 @@ mod tests { #[simd_test(enable = "sse4.1")] unsafe fn test_mm_round_pd() { let a = _mm_setr_pd(1.25, 3.75); - let r = _mm_round_pd(a, _MM_FROUND_TO_NEAREST_INT); + let r = _mm_round_pd::<_MM_FROUND_TO_NEAREST_INT>(a); let e = _mm_setr_pd(1.0, 4.0); assert_eq_m128d(r, e); } @@ -1685,7 +1663,7 @@ mod tests { #[simd_test(enable = "sse4.1")] unsafe fn test_mm_round_ps() { let a = _mm_setr_ps(2.25, 4.75, -1.75, -4.25); - let r = _mm_round_ps(a, _MM_FROUND_TO_ZERO); + let r = _mm_round_ps::<_MM_FROUND_TO_ZERO>(a); let e = _mm_setr_ps(2.0, 4.0, -1.0, -4.0); assert_eq_m128(r, e); } @@ -1696,7 +1674,7 @@ mod tests { let b = _mm_setr_pd(-2.5, -4.5); let old_mode = _MM_GET_ROUNDING_MODE(); _MM_SET_ROUNDING_MODE(_MM_ROUND_TOWARD_ZERO); - let r = _mm_round_sd(a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, b); _MM_SET_ROUNDING_MODE(old_mode); let e = _mm_setr_pd(-2.0, 3.5); assert_eq_m128d(r, e); @@ -1708,7 +1686,7 @@ mod tests { let b = _mm_setr_ps(-1.75, -4.5, -8.5, -16.5); let old_mode = _MM_GET_ROUNDING_MODE(); _MM_SET_ROUNDING_MODE(_MM_ROUND_NEAREST); - let r = _mm_round_ss(a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, b); _MM_SET_ROUNDING_MODE(old_mode); let e = _mm_setr_ps(-2.0, 3.5, 7.5, 15.5); assert_eq_m128(r, e); @@ -1788,23 +1766,23 @@ mod tests { 8, 9, 10, 11, 12, 13, 14, 15, ); - let r = _mm_mpsadbw_epu8(a, a, 0b000); + let r = _mm_mpsadbw_epu8::<0b000>(a, a); let e = _mm_setr_epi16(0, 4, 8, 12, 16, 20, 24, 28); assert_eq_m128i(r, e); - let r = _mm_mpsadbw_epu8(a, a, 0b001); + let r = _mm_mpsadbw_epu8::<0b001>(a, a); let e = _mm_setr_epi16(16, 12, 8, 4, 0, 4, 8, 12); assert_eq_m128i(r, e); - let r = _mm_mpsadbw_epu8(a, a, 0b100); + let r = _mm_mpsadbw_epu8::<0b100>(a, a); let e = _mm_setr_epi16(16, 20, 24, 28, 32, 36, 40, 44); assert_eq_m128i(r, e); - let r = _mm_mpsadbw_epu8(a, a, 0b101); + let r = _mm_mpsadbw_epu8::<0b101>(a, a); let e = _mm_setr_epi16(0, 4, 8, 12, 16, 20, 24, 28); assert_eq_m128i(r, e); - let r = _mm_mpsadbw_epu8(a, a, 0b111); + let r = _mm_mpsadbw_epu8::<0b111>(a, a); let e = _mm_setr_epi16(32, 28, 24, 20, 16, 12, 8, 4); assert_eq_m128i(r, e); } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse42.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse42.rs index d4d8aa644..f474b0671 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse42.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse42.rs @@ -65,33 +65,27 @@ pub const _SIDD_BIT_MASK: i32 = 0b0000_0000; pub const _SIDD_UNIT_MASK: i32 = 0b0100_0000; /// Compares packed strings with implicit lengths in `a` and `b` using the -/// control in `imm8`, and return the generated mask. +/// control in `IMM8`, and return the generated mask. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpistrm) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpistrm, imm8 = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpistrm(a: __m128i, b: __m128i, imm8: i32) -> __m128i { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpistrm128(a, b, $imm8) - }; - } - transmute(constify_imm8!(imm8, call)) +#[cfg_attr(test, assert_instr(pcmpistrm, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpistrm(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(pcmpistrm128(a.as_i8x16(), b.as_i8x16(), IMM8 as i8)) } /// Compares packed strings with implicit lengths in `a` and `b` using the -/// control in `imm8` and return the generated index. Similar to +/// control in `IMM8` and return the generated index. Similar to /// [`_mm_cmpestri`] with the exception that [`_mm_cmpestri`] requires the /// lengths of `a` and `b` to be explicitly specified. /// /// # Control modes /// -/// The control specified by `imm8` may be one or more of the following. +/// The control specified by `IMM8` may be one or more of the following. /// /// ## Data size and signedness /// @@ -152,7 +146,7 @@ pub unsafe fn _mm_cmpistrm(a: __m128i, b: __m128i, imm8: i32) -> __m128i { /// # } /// ``` /// -/// The `_mm_cmpistri` intrinsic may also be used to find the existance of +/// The `_mm_cmpistri` intrinsic may also be used to find the existence of /// one or more of a given set of characters in the haystack. /// /// ``` @@ -261,169 +255,113 @@ pub unsafe fn _mm_cmpistrm(a: __m128i, b: __m128i, imm8: i32) -> __m128i { /// # } /// ``` /// -/// [`_SIDD_UBYTE_OPS`]: constant._SIDD_UBYTE_OPS.html -/// [`_SIDD_UWORD_OPS`]: constant._SIDD_UWORD_OPS.html -/// [`_SIDD_SBYTE_OPS`]: constant._SIDD_SBYTE_OPS.html -/// [`_SIDD_SWORD_OPS`]: constant._SIDD_SWORD_OPS.html -/// [`_SIDD_CMP_EQUAL_ANY`]: constant._SIDD_CMP_EQUAL_ANY.html -/// [`_SIDD_CMP_RANGES`]: constant._SIDD_CMP_RANGES.html -/// [`_SIDD_CMP_EQUAL_EACH`]: constant._SIDD_CMP_EQUAL_EACH.html -/// [`_SIDD_CMP_EQUAL_ORDERED`]: constant._SIDD_CMP_EQUAL_ORDERED.html -/// [`_SIDD_POSITIVE_POLARITY`]: constant._SIDD_POSITIVE_POLARITY.html -/// [`_SIDD_NEGATIVE_POLARITY`]: constant._SIDD_NEGATIVE_POLARITY.html -/// [`_SIDD_LEAST_SIGNIFICANT`]: constant._SIDD_LEAST_SIGNIFICANT.html -/// [`_SIDD_MOST_SIGNIFICANT`]: constant._SIDD_MOST_SIGNIFICANT.html -/// [`_mm_cmpestri`]: fn._mm_cmpestri.html -/// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpistri) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpistri, imm8 = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpistri(a: __m128i, b: __m128i, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpistri128(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpistri, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpistri(a: __m128i, b: __m128i) -> i32 { + static_assert_imm8!(IMM8); + pcmpistri128(a.as_i8x16(), b.as_i8x16(), IMM8 as i8) } /// Compares packed strings with implicit lengths in `a` and `b` using the -/// control in `imm8`, and return `1` if any character in `b` was null. +/// control in `IMM8`, and return `1` if any character in `b` was null. /// and `0` otherwise. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpistrz) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpistri, imm8 = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpistrz(a: __m128i, b: __m128i, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpistriz128(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpistri, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpistrz(a: __m128i, b: __m128i) -> i32 { + static_assert_imm8!(IMM8); + pcmpistriz128(a.as_i8x16(), b.as_i8x16(), IMM8 as i8) } /// Compares packed strings with implicit lengths in `a` and `b` using the -/// control in `imm8`, and return `1` if the resulting mask was non-zero, +/// control in `IMM8`, and return `1` if the resulting mask was non-zero, /// and `0` otherwise. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpistrc) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpistri, imm8 = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpistrc(a: __m128i, b: __m128i, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpistric128(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpistri, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpistrc(a: __m128i, b: __m128i) -> i32 { + static_assert_imm8!(IMM8); + pcmpistric128(a.as_i8x16(), b.as_i8x16(), IMM8 as i8) } /// Compares packed strings with implicit lengths in `a` and `b` using the -/// control in `imm8`, and returns `1` if any character in `a` was null, +/// control in `IMM8`, and returns `1` if any character in `a` was null, /// and `0` otherwise. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpistrs) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpistri, imm8 = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpistrs(a: __m128i, b: __m128i, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpistris128(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpistri, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpistrs(a: __m128i, b: __m128i) -> i32 { + static_assert_imm8!(IMM8); + pcmpistris128(a.as_i8x16(), b.as_i8x16(), IMM8 as i8) } /// Compares packed strings with implicit lengths in `a` and `b` using the -/// control in `imm8`, and return bit `0` of the resulting bit mask. +/// control in `IMM8`, and return bit `0` of the resulting bit mask. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpistro) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpistri, imm8 = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpistro(a: __m128i, b: __m128i, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpistrio128(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpistri, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpistro(a: __m128i, b: __m128i) -> i32 { + static_assert_imm8!(IMM8); + pcmpistrio128(a.as_i8x16(), b.as_i8x16(), IMM8 as i8) } /// Compares packed strings with implicit lengths in `a` and `b` using the -/// control in `imm8`, and return `1` if `b` did not contain a null +/// control in `IMM8`, and return `1` if `b` did not contain a null /// character and the resulting mask was zero, and `0` otherwise. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpistra) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpistri, imm8 = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpistra(a: __m128i, b: __m128i, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpistria128(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpistri, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpistra(a: __m128i, b: __m128i) -> i32 { + static_assert_imm8!(IMM8); + pcmpistria128(a.as_i8x16(), b.as_i8x16(), IMM8 as i8) } /// Compares packed strings in `a` and `b` with lengths `la` and `lb` -/// using the control in `imm8`, and return the generated mask. +/// using the control in `IMM8`, and return the generated mask. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpestrm) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpestrm, imm8 = 0))] -#[rustc_args_required_const(4)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpestrm(a: __m128i, la: i32, b: __m128i, lb: i32, imm8: i32) -> __m128i { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpestrm128(a, la, b, lb, $imm8) - }; - } - transmute(constify_imm8!(imm8, call)) +#[cfg_attr(test, assert_instr(pcmpestrm, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpestrm(a: __m128i, la: i32, b: __m128i, lb: i32) -> __m128i { + static_assert_imm8!(IMM8); + transmute(pcmpestrm128(a.as_i8x16(), la, b.as_i8x16(), lb, IMM8 as i8)) } /// Compares packed strings `a` and `b` with lengths `la` and `lb` using the -/// control in `imm8` and return the generated index. Similar to +/// control in `IMM8` and return the generated index. Similar to /// [`_mm_cmpistri`] with the exception that [`_mm_cmpistri`] implicitly /// determines the length of `a` and `b`. /// /// # Control modes /// -/// The control specified by `imm8` may be one or more of the following. +/// The control specified by `IMM8` may be one or more of the following. /// /// ## Data size and signedness /// @@ -497,128 +435,92 @@ pub unsafe fn _mm_cmpestrm(a: __m128i, la: i32, b: __m128i, lb: i32, imm8: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpestri) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpestri, imm8 = 0))] -#[rustc_args_required_const(4)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpestri(a: __m128i, la: i32, b: __m128i, lb: i32, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpestri128(a, la, b, lb, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpestri, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpestri(a: __m128i, la: i32, b: __m128i, lb: i32) -> i32 { + static_assert_imm8!(IMM8); + pcmpestri128(a.as_i8x16(), la, b.as_i8x16(), lb, IMM8 as i8) } /// Compares packed strings in `a` and `b` with lengths `la` and `lb` -/// using the control in `imm8`, and return `1` if any character in +/// using the control in `IMM8`, and return `1` if any character in /// `b` was null, and `0` otherwise. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpestrz) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpestri, imm8 = 0))] -#[rustc_args_required_const(4)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpestrz(a: __m128i, la: i32, b: __m128i, lb: i32, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpestriz128(a, la, b, lb, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpestri, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpestrz(a: __m128i, la: i32, b: __m128i, lb: i32) -> i32 { + static_assert_imm8!(IMM8); + pcmpestriz128(a.as_i8x16(), la, b.as_i8x16(), lb, IMM8 as i8) } /// Compares packed strings in `a` and `b` with lengths `la` and `lb` -/// using the control in `imm8`, and return `1` if the resulting mask +/// using the control in `IMM8`, and return `1` if the resulting mask /// was non-zero, and `0` otherwise. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpestrc) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpestri, imm8 = 0))] -#[rustc_args_required_const(4)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpestrc(a: __m128i, la: i32, b: __m128i, lb: i32, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpestric128(a, la, b, lb, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpestri, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpestrc(a: __m128i, la: i32, b: __m128i, lb: i32) -> i32 { + static_assert_imm8!(IMM8); + pcmpestric128(a.as_i8x16(), la, b.as_i8x16(), lb, IMM8 as i8) } /// Compares packed strings in `a` and `b` with lengths `la` and `lb` -/// using the control in `imm8`, and return `1` if any character in +/// using the control in `IMM8`, and return `1` if any character in /// a was null, and `0` otherwise. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpestrs) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpestri, imm8 = 0))] -#[rustc_args_required_const(4)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpestrs(a: __m128i, la: i32, b: __m128i, lb: i32, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpestris128(a, la, b, lb, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpestri, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpestrs(a: __m128i, la: i32, b: __m128i, lb: i32) -> i32 { + static_assert_imm8!(IMM8); + pcmpestris128(a.as_i8x16(), la, b.as_i8x16(), lb, IMM8 as i8) } /// Compares packed strings in `a` and `b` with lengths `la` and `lb` -/// using the control in `imm8`, and return bit `0` of the resulting +/// using the control in `IMM8`, and return bit `0` of the resulting /// bit mask. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpestro) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpestri, imm8 = 0))] -#[rustc_args_required_const(4)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpestro(a: __m128i, la: i32, b: __m128i, lb: i32, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpestrio128(a, la, b, lb, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpestri, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpestro(a: __m128i, la: i32, b: __m128i, lb: i32) -> i32 { + static_assert_imm8!(IMM8); + pcmpestrio128(a.as_i8x16(), la, b.as_i8x16(), lb, IMM8 as i8) } /// Compares packed strings in `a` and `b` with lengths `la` and `lb` -/// using the control in `imm8`, and return `1` if `b` did not +/// using the control in `IMM8`, and return `1` if `b` did not /// contain a null character and the resulting mask was zero, and `0` /// otherwise. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpestra) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpestri, imm8 = 0))] -#[rustc_args_required_const(4)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpestra(a: __m128i, la: i32, b: __m128i, lb: i32, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpestria128(a, la, b, lb, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpestri, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpestra(a: __m128i, la: i32, b: __m128i, lb: i32) -> i32 { + static_assert_imm8!(IMM8); + pcmpestria128(a.as_i8x16(), la, b.as_i8x16(), lb, IMM8 as i8) } /// Starting with the initial value in `crc`, return the accumulated -/// CRC32 value for unsigned 8-bit integer `v`. +/// CRC32-C value for unsigned 8-bit integer `v`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_crc32_u8) #[inline] @@ -630,7 +532,7 @@ pub unsafe fn _mm_crc32_u8(crc: u32, v: u8) -> u32 { } /// Starting with the initial value in `crc`, return the accumulated -/// CRC32 value for unsigned 16-bit integer `v`. +/// CRC32-C value for unsigned 16-bit integer `v`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_crc32_u16) #[inline] @@ -642,7 +544,7 @@ pub unsafe fn _mm_crc32_u16(crc: u32, v: u16) -> u32 { } /// Starting with the initial value in `crc`, return the accumulated -/// CRC32 value for unsigned 32-bit integer `v`. +/// CRC32-C value for unsigned 32-bit integer `v`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_crc32_u32) #[inline] @@ -733,7 +635,7 @@ mod tests { unsafe fn test_mm_cmpistrm() { let a = str_to_m128i(b"Hello! Good-Bye!"); let b = str_to_m128i(b"hello! good-bye!"); - let i = _mm_cmpistrm(a, b, _SIDD_UNIT_MASK); + let i = _mm_cmpistrm::<_SIDD_UNIT_MASK>(a, b); #[rustfmt::skip] let res = _mm_setr_epi8( 0x00, !0, !0, !0, !0, !0, !0, 0x00, @@ -746,7 +648,7 @@ mod tests { unsafe fn test_mm_cmpistri() { let a = str_to_m128i(b"Hello"); let b = str_to_m128i(b" Hello "); - let i = _mm_cmpistri(a, b, _SIDD_CMP_EQUAL_ORDERED); + let i = _mm_cmpistri::<_SIDD_CMP_EQUAL_ORDERED>(a, b); assert_eq!(3, i); } @@ -754,7 +656,7 @@ mod tests { unsafe fn test_mm_cmpistrz() { let a = str_to_m128i(b""); let b = str_to_m128i(b"Hello"); - let i = _mm_cmpistrz(a, b, _SIDD_CMP_EQUAL_ORDERED); + let i = _mm_cmpistrz::<_SIDD_CMP_EQUAL_ORDERED>(a, b); assert_eq!(1, i); } @@ -762,7 +664,7 @@ mod tests { unsafe fn test_mm_cmpistrc() { let a = str_to_m128i(b" "); let b = str_to_m128i(b" ! "); - let i = _mm_cmpistrc(a, b, _SIDD_UNIT_MASK); + let i = _mm_cmpistrc::<_SIDD_UNIT_MASK>(a, b); assert_eq!(1, i); } @@ -770,7 +672,7 @@ mod tests { unsafe fn test_mm_cmpistrs() { let a = str_to_m128i(b"Hello"); let b = str_to_m128i(b""); - let i = _mm_cmpistrs(a, b, _SIDD_CMP_EQUAL_ORDERED); + let i = _mm_cmpistrs::<_SIDD_CMP_EQUAL_ORDERED>(a, b); assert_eq!(1, i); } @@ -788,7 +690,7 @@ mod tests { ); let a = a_bytes; let b = b_bytes; - let i = _mm_cmpistro(a, b, _SIDD_UWORD_OPS | _SIDD_UNIT_MASK); + let i = _mm_cmpistro::<{ _SIDD_UWORD_OPS | _SIDD_UNIT_MASK }>(a, b); assert_eq!(0, i); } @@ -796,7 +698,7 @@ mod tests { unsafe fn test_mm_cmpistra() { let a = str_to_m128i(b""); let b = str_to_m128i(b"Hello!!!!!!!!!!!"); - let i = _mm_cmpistra(a, b, _SIDD_UNIT_MASK); + let i = _mm_cmpistra::<_SIDD_UNIT_MASK>(a, b); assert_eq!(1, i); } @@ -804,7 +706,7 @@ mod tests { unsafe fn test_mm_cmpestrm() { let a = str_to_m128i(b"Hello!"); let b = str_to_m128i(b"Hello."); - let i = _mm_cmpestrm(a, 5, b, 5, _SIDD_UNIT_MASK); + let i = _mm_cmpestrm::<_SIDD_UNIT_MASK>(a, 5, b, 5); #[rustfmt::skip] let r = _mm_setr_epi8( !0, !0, !0, !0, !0, 0x00, 0x00, 0x00, @@ -817,7 +719,7 @@ mod tests { unsafe fn test_mm_cmpestri() { let a = str_to_m128i(b"bar - garbage"); let b = str_to_m128i(b"foobar"); - let i = _mm_cmpestri(a, 3, b, 6, _SIDD_CMP_EQUAL_ORDERED); + let i = _mm_cmpestri::<_SIDD_CMP_EQUAL_ORDERED>(a, 3, b, 6); assert_eq!(3, i); } @@ -825,7 +727,7 @@ mod tests { unsafe fn test_mm_cmpestrz() { let a = str_to_m128i(b""); let b = str_to_m128i(b"Hello"); - let i = _mm_cmpestrz(a, 16, b, 6, _SIDD_CMP_EQUAL_ORDERED); + let i = _mm_cmpestrz::<_SIDD_CMP_EQUAL_ORDERED>(a, 16, b, 6); assert_eq!(1, i); } @@ -833,7 +735,7 @@ mod tests { unsafe fn test_mm_cmpestrc() { let va = str_to_m128i(b"!!!!!!!!"); let vb = str_to_m128i(b" "); - let i = _mm_cmpestrc(va, 7, vb, 7, _SIDD_UNIT_MASK); + let i = _mm_cmpestrc::<_SIDD_UNIT_MASK>(va, 7, vb, 7); assert_eq!(0, i); } @@ -846,7 +748,7 @@ mod tests { ); let a = a_bytes; let b = _mm_set1_epi8(0x00); - let i = _mm_cmpestrs(a, 8, b, 0, _SIDD_UWORD_OPS); + let i = _mm_cmpestrs::<_SIDD_UWORD_OPS>(a, 8, b, 0); assert_eq!(0, i); } @@ -854,7 +756,7 @@ mod tests { unsafe fn test_mm_cmpestro() { let a = str_to_m128i(b"Hello"); let b = str_to_m128i(b"World"); - let i = _mm_cmpestro(a, 5, b, 5, _SIDD_UBYTE_OPS); + let i = _mm_cmpestro::<_SIDD_UBYTE_OPS>(a, 5, b, 5); assert_eq!(0, i); } @@ -862,7 +764,7 @@ mod tests { unsafe fn test_mm_cmpestra() { let a = str_to_m128i(b"Cannot match a"); let b = str_to_m128i(b"Null after 14"); - let i = _mm_cmpestra(a, 14, b, 16, _SIDD_CMP_EQUAL_EACH | _SIDD_UNIT_MASK); + let i = _mm_cmpestra::<{ _SIDD_CMP_EQUAL_EACH | _SIDD_UNIT_MASK }>(a, 14, b, 16); assert_eq!(1, i); } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse4a.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse4a.rs index e6345d0da..976c907cb 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse4a.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/sse4a.rs @@ -32,7 +32,7 @@ extern "C" { /// If the length is zero, it is interpreted as `64`. If the length and index /// are zero, the lower 64 bits of `x` are extracted. /// -/// If `length == 0 && index > 0` or `lenght + index > 64` the result is +/// If `length == 0 && index > 0` or `length + index > 64` the result is /// undefined. #[inline] #[target_feature(enable = "sse4a")] diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/ssse3.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/ssse3.rs index 6a45603e4..4beb496b6 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/ssse3.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/ssse3.rs @@ -86,71 +86,55 @@ pub unsafe fn _mm_shuffle_epi8(a: __m128i, b: __m128i) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_alignr_epi8) #[inline] #[target_feature(enable = "ssse3")] -#[cfg_attr(test, assert_instr(palignr, n = 15))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(palignr, IMM8 = 15))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_alignr_epi8(a: __m128i, b: __m128i, n: i32) -> __m128i { - let n = n as u32; +pub unsafe fn _mm_alignr_epi8(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); // If palignr is shifting the pair of vectors more than the size of two // lanes, emit zero. - if n > 32 { + if IMM8 > 32 { return _mm_set1_epi8(0); } // If palignr is shifting the pair of input vectors more than one lane, // but less than two lanes, convert to shifting in zeroes. - let (a, b, n) = if n > 16 { - (_mm_set1_epi8(0), a, n - 16) + let (a, b) = if IMM8 > 16 { + (_mm_set1_epi8(0), a) } else { - (a, b, n) - }; - let a = a.as_i8x16(); - let b = b.as_i8x16(); - - macro_rules! shuffle { - ($shift:expr) => { - simd_shuffle16( - b, - a, - [ - 0 + $shift, - 1 + $shift, - 2 + $shift, - 3 + $shift, - 4 + $shift, - 5 + $shift, - 6 + $shift, - 7 + $shift, - 8 + $shift, - 9 + $shift, - 10 + $shift, - 11 + $shift, - 12 + $shift, - 13 + $shift, - 14 + $shift, - 15 + $shift, - ], - ) - }; - } - let r: i8x16 = match n { - 0 => shuffle!(0), - 1 => shuffle!(1), - 2 => shuffle!(2), - 3 => shuffle!(3), - 4 => shuffle!(4), - 5 => shuffle!(5), - 6 => shuffle!(6), - 7 => shuffle!(7), - 8 => shuffle!(8), - 9 => shuffle!(9), - 10 => shuffle!(10), - 11 => shuffle!(11), - 12 => shuffle!(12), - 13 => shuffle!(13), - 14 => shuffle!(14), - 15 => shuffle!(15), - _ => shuffle!(16), + (a, b) }; + const fn mask(shift: u32, i: u32) -> u32 { + if shift > 32 { + // Unused, but needs to be a valid index. + i + } else if shift > 16 { + shift - 16 + i + } else { + shift + i + } + } + let r: i8x16 = simd_shuffle16!( + b.as_i8x16(), + a.as_i8x16(), + [ + mask(IMM8 as u32, 0), + mask(IMM8 as u32, 1), + mask(IMM8 as u32, 2), + mask(IMM8 as u32, 3), + mask(IMM8 as u32, 4), + mask(IMM8 as u32, 5), + mask(IMM8 as u32, 6), + mask(IMM8 as u32, 7), + mask(IMM8 as u32, 8), + mask(IMM8 as u32, 9), + mask(IMM8 as u32, 10), + mask(IMM8 as u32, 11), + mask(IMM8 as u32, 12), + mask(IMM8 as u32, 13), + mask(IMM8 as u32, 14), + mask(IMM8 as u32, 15), + ], + ); transmute(r) } @@ -299,169 +283,6 @@ pub unsafe fn _mm_sign_epi32(a: __m128i, b: __m128i) -> __m128i { transmute(psignd128(a.as_i32x4(), b.as_i32x4())) } -/// Computes the absolute value of packed 8-bit integers in `a` and -/// return the unsigned results. -#[inline] -#[target_feature(enable = "ssse3,mmx")] -#[cfg_attr(test, assert_instr(pabsb))] -pub unsafe fn _mm_abs_pi8(a: __m64) -> __m64 { - pabsb(a) -} - -/// Computes the absolute value of packed 8-bit integers in `a`, and returns the -/// unsigned results. -#[inline] -#[target_feature(enable = "ssse3,mmx")] -#[cfg_attr(test, assert_instr(pabsw))] -pub unsafe fn _mm_abs_pi16(a: __m64) -> __m64 { - pabsw(a) -} - -/// Computes the absolute value of packed 32-bit integers in `a`, and returns the -/// unsigned results. -#[inline] -#[target_feature(enable = "ssse3,mmx")] -#[cfg_attr(test, assert_instr(pabsd))] -pub unsafe fn _mm_abs_pi32(a: __m64) -> __m64 { - pabsd(a) -} - -/// Shuffles packed 8-bit integers in `a` according to shuffle control mask in -/// the corresponding 8-bit element of `b`, and returns the results -#[inline] -#[target_feature(enable = "ssse3,mmx")] -#[cfg_attr(test, assert_instr(pshufb))] -pub unsafe fn _mm_shuffle_pi8(a: __m64, b: __m64) -> __m64 { - pshufb(a, b) -} - -/// Concatenates the two 64-bit integer vector operands, and right-shifts -/// the result by the number of bytes specified in the immediate operand. -#[inline] -#[target_feature(enable = "ssse3,mmx")] -#[cfg_attr(test, assert_instr(palignr, n = 15))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_alignr_pi8(a: __m64, b: __m64, n: i32) -> __m64 { - macro_rules! call { - ($imm8:expr) => { - palignrb(a, b, $imm8) - }; - } - constify_imm8!(n, call) -} - -/// Horizontally adds the adjacent pairs of values contained in 2 packed -/// 64-bit vectors of `[4 x i16]`. -#[inline] -#[target_feature(enable = "ssse3,mmx")] -#[cfg_attr(test, assert_instr(phaddw))] -pub unsafe fn _mm_hadd_pi16(a: __m64, b: __m64) -> __m64 { - phaddw(a, b) -} - -/// Horizontally adds the adjacent pairs of values contained in 2 packed -/// 64-bit vectors of `[2 x i32]`. -#[inline] -#[target_feature(enable = "ssse3,mmx")] -#[cfg_attr(test, assert_instr(phaddd))] -pub unsafe fn _mm_hadd_pi32(a: __m64, b: __m64) -> __m64 { - phaddd(a, b) -} - -/// Horizontally adds the adjacent pairs of values contained in 2 packed -/// 64-bit vectors of `[4 x i16]`. Positive sums greater than 7FFFh are -/// saturated to 7FFFh. Negative sums less than 8000h are saturated to 8000h. -#[inline] -#[target_feature(enable = "ssse3,mmx")] -#[cfg_attr(test, assert_instr(phaddsw))] -pub unsafe fn _mm_hadds_pi16(a: __m64, b: __m64) -> __m64 { - phaddsw(a, b) -} - -/// Horizontally subtracts the adjacent pairs of values contained in 2 -/// packed 64-bit vectors of `[4 x i16]`. -#[inline] -#[target_feature(enable = "ssse3,mmx")] -#[cfg_attr(test, assert_instr(phsubw))] -pub unsafe fn _mm_hsub_pi16(a: __m64, b: __m64) -> __m64 { - phsubw(a, b) -} - -/// Horizontally subtracts the adjacent pairs of values contained in 2 -/// packed 64-bit vectors of `[2 x i32]`. -#[inline] -#[target_feature(enable = "ssse3,mmx")] -#[cfg_attr(test, assert_instr(phsubd))] -pub unsafe fn _mm_hsub_pi32(a: __m64, b: __m64) -> __m64 { - phsubd(a, b) -} - -/// Horizontally subtracts the adjacent pairs of values contained in 2 -/// packed 64-bit vectors of `[4 x i16]`. Positive differences greater than -/// 7FFFh are saturated to 7FFFh. Negative differences less than 8000h are -/// saturated to 8000h. -#[inline] -#[target_feature(enable = "ssse3,mmx")] -#[cfg_attr(test, assert_instr(phsubsw))] -pub unsafe fn _mm_hsubs_pi16(a: __m64, b: __m64) -> __m64 { - phsubsw(a, b) -} - -/// Multiplies corresponding pairs of packed 8-bit unsigned integer -/// values contained in the first source operand and packed 8-bit signed -/// integer values contained in the second source operand, adds pairs of -/// contiguous products with signed saturation, and writes the 16-bit sums to -/// the corresponding bits in the destination. -#[inline] -#[target_feature(enable = "ssse3,mmx")] -#[cfg_attr(test, assert_instr(pmaddubsw))] -pub unsafe fn _mm_maddubs_pi16(a: __m64, b: __m64) -> __m64 { - pmaddubsw(a, b) -} - -/// Multiplies packed 16-bit signed integer values, truncates the 32-bit -/// products to the 18 most significant bits by right-shifting, rounds the -/// truncated value by adding 1, and writes bits `[16:1]` to the destination. -#[inline] -#[target_feature(enable = "ssse3,mmx")] -#[cfg_attr(test, assert_instr(pmulhrsw))] -pub unsafe fn _mm_mulhrs_pi16(a: __m64, b: __m64) -> __m64 { - pmulhrsw(a, b) -} - -/// Negates packed 8-bit integers in `a` when the corresponding signed 8-bit -/// integer in `b` is negative, and returns the results. -/// Element in result are zeroed out when the corresponding element in `b` is -/// zero. -#[inline] -#[target_feature(enable = "ssse3,mmx")] -#[cfg_attr(test, assert_instr(psignb))] -pub unsafe fn _mm_sign_pi8(a: __m64, b: __m64) -> __m64 { - psignb(a, b) -} - -/// Negates packed 16-bit integers in `a` when the corresponding signed 16-bit -/// integer in `b` is negative, and returns the results. -/// Element in result are zeroed out when the corresponding element in `b` is -/// zero. -#[inline] -#[target_feature(enable = "ssse3,mmx")] -#[cfg_attr(test, assert_instr(psignw))] -pub unsafe fn _mm_sign_pi16(a: __m64, b: __m64) -> __m64 { - psignw(a, b) -} - -/// Negates packed 32-bit integers in `a` when the corresponding signed 32-bit -/// integer in `b` is negative, and returns the results. -/// Element in result are zeroed out when the corresponding element in `b` is -/// zero. -#[inline] -#[target_feature(enable = "ssse3,mmx")] -#[cfg_attr(test, assert_instr(psignd))] -pub unsafe fn _mm_sign_pi32(a: __m64, b: __m64) -> __m64 { - psignd(a, b) -} - #[allow(improper_ctypes)] extern "C" { #[link_name = "llvm.x86.ssse3.pabs.b.128"] @@ -508,54 +329,6 @@ extern "C" { #[link_name = "llvm.x86.ssse3.psign.d.128"] fn psignd128(a: i32x4, b: i32x4) -> i32x4; - - #[link_name = "llvm.x86.ssse3.pabs.b"] - fn pabsb(a: __m64) -> __m64; - - #[link_name = "llvm.x86.ssse3.pabs.w"] - fn pabsw(a: __m64) -> __m64; - - #[link_name = "llvm.x86.ssse3.pabs.d"] - fn pabsd(a: __m64) -> __m64; - - #[link_name = "llvm.x86.ssse3.pshuf.b"] - fn pshufb(a: __m64, b: __m64) -> __m64; - - #[link_name = "llvm.x86.mmx.palignr.b"] - fn palignrb(a: __m64, b: __m64, n: u8) -> __m64; - - #[link_name = "llvm.x86.ssse3.phadd.w"] - fn phaddw(a: __m64, b: __m64) -> __m64; - - #[link_name = "llvm.x86.ssse3.phadd.d"] - fn phaddd(a: __m64, b: __m64) -> __m64; - - #[link_name = "llvm.x86.ssse3.phadd.sw"] - fn phaddsw(a: __m64, b: __m64) -> __m64; - - #[link_name = "llvm.x86.ssse3.phsub.w"] - fn phsubw(a: __m64, b: __m64) -> __m64; - - #[link_name = "llvm.x86.ssse3.phsub.d"] - fn phsubd(a: __m64, b: __m64) -> __m64; - - #[link_name = "llvm.x86.ssse3.phsub.sw"] - fn phsubsw(a: __m64, b: __m64) -> __m64; - - #[link_name = "llvm.x86.ssse3.pmadd.ub.sw"] - fn pmaddubsw(a: __m64, b: __m64) -> __m64; - - #[link_name = "llvm.x86.ssse3.pmul.hr.sw"] - fn pmulhrsw(a: __m64, b: __m64) -> __m64; - - #[link_name = "llvm.x86.ssse3.psign.b"] - fn psignb(a: __m64, b: __m64) -> __m64; - - #[link_name = "llvm.x86.ssse3.psign.w"] - fn psignw(a: __m64, b: __m64) -> __m64; - - #[link_name = "llvm.x86.ssse3.psign.d"] - fn psignd(a: __m64, b: __m64) -> __m64; } #[cfg(test)] @@ -615,10 +388,10 @@ mod tests { 12, 5, 5, 10, 4, 1, 8, 0, ); - let r = _mm_alignr_epi8(a, b, 33); + let r = _mm_alignr_epi8::<33>(a, b); assert_eq_m128i(r, _mm_set1_epi8(0)); - let r = _mm_alignr_epi8(a, b, 17); + let r = _mm_alignr_epi8::<17>(a, b); #[rustfmt::skip] let expected = _mm_setr_epi8( 2, 3, 4, 5, 6, 7, 8, 9, @@ -626,10 +399,10 @@ mod tests { ); assert_eq_m128i(r, expected); - let r = _mm_alignr_epi8(a, b, 16); + let r = _mm_alignr_epi8::<16>(a, b); assert_eq_m128i(r, a); - let r = _mm_alignr_epi8(a, b, 15); + let r = _mm_alignr_epi8::<15>(a, b); #[rustfmt::skip] let expected = _mm_setr_epi8( 0, 1, 2, 3, 4, 5, 6, 7, @@ -637,7 +410,7 @@ mod tests { ); assert_eq_m128i(r, expected); - let r = _mm_alignr_epi8(a, b, 0); + let r = _mm_alignr_epi8::<0>(a, b); assert_eq_m128i(r, b); } @@ -761,138 +534,4 @@ mod tests { let r = _mm_sign_epi32(a, b); assert_eq_m128i(r, expected); } - - #[simd_test(enable = "ssse3,mmx")] - unsafe fn test_mm_abs_pi8() { - let r = _mm_abs_pi8(_mm_set1_pi8(-5)); - assert_eq_m64(r, _mm_set1_pi8(5)); - } - - #[simd_test(enable = "ssse3,mmx")] - unsafe fn test_mm_abs_pi16() { - let r = _mm_abs_pi16(_mm_set1_pi16(-5)); - assert_eq_m64(r, _mm_set1_pi16(5)); - } - - #[simd_test(enable = "ssse3,mmx")] - unsafe fn test_mm_abs_pi32() { - let r = _mm_abs_pi32(_mm_set1_pi32(-5)); - assert_eq_m64(r, _mm_set1_pi32(5)); - } - - #[simd_test(enable = "ssse3,mmx")] - unsafe fn test_mm_shuffle_pi8() { - let a = _mm_setr_pi8(1, 2, 3, 4, 5, 6, 7, 8); - let b = _mm_setr_pi8(4, 128u8 as i8, 4, 3, 24, 12, 6, 19); - let expected = _mm_setr_pi8(5, 0, 5, 4, 1, 5, 7, 4); - let r = _mm_shuffle_pi8(a, b); - assert_eq_m64(r, expected); - } - - #[simd_test(enable = "ssse3,mmx")] - unsafe fn test_mm_alignr_pi8() { - let a = _mm_setr_pi32(0x89ABCDEF_u32 as i32, 0x01234567_u32 as i32); - let b = _mm_setr_pi32(0xBBAA9988_u32 as i32, 0xFFDDEECC_u32 as i32); - let r = _mm_alignr_pi8(a, b, 4); - assert_eq_m64(r, transmute(0x89abcdefffddeecc_u64)); - } - - #[simd_test(enable = "ssse3,mmx")] - unsafe fn test_mm_hadd_pi16() { - let a = _mm_setr_pi16(1, 2, 3, 4); - let b = _mm_setr_pi16(4, 128, 4, 3); - let expected = _mm_setr_pi16(3, 7, 132, 7); - let r = _mm_hadd_pi16(a, b); - assert_eq_m64(r, expected); - } - - #[simd_test(enable = "ssse3,mmx")] - unsafe fn test_mm_hadd_pi32() { - let a = _mm_setr_pi32(1, 2); - let b = _mm_setr_pi32(4, 128); - let expected = _mm_setr_pi32(3, 132); - let r = _mm_hadd_pi32(a, b); - assert_eq_m64(r, expected); - } - - #[simd_test(enable = "ssse3,mmx")] - unsafe fn test_mm_hadds_pi16() { - let a = _mm_setr_pi16(1, 2, 3, 4); - let b = _mm_setr_pi16(32767, 1, -32768, -1); - let expected = _mm_setr_pi16(3, 7, 32767, -32768); - let r = _mm_hadds_pi16(a, b); - assert_eq_m64(r, expected); - } - - #[simd_test(enable = "ssse3,mmx")] - unsafe fn test_mm_hsub_pi16() { - let a = _mm_setr_pi16(1, 2, 3, 4); - let b = _mm_setr_pi16(4, 128, 4, 3); - let expected = _mm_setr_pi16(-1, -1, -124, 1); - let r = _mm_hsub_pi16(a, b); - assert_eq_m64(r, expected); - } - - #[simd_test(enable = "ssse3,mmx")] - unsafe fn test_mm_hsub_pi32() { - let a = _mm_setr_pi32(1, 2); - let b = _mm_setr_pi32(4, 128); - let expected = _mm_setr_pi32(-1, -124); - let r = _mm_hsub_pi32(a, b); - assert_eq_m64(r, expected); - } - - #[simd_test(enable = "ssse3,mmx")] - unsafe fn test_mm_hsubs_pi16() { - let a = _mm_setr_pi16(1, 2, 3, 4); - let b = _mm_setr_pi16(4, 128, 4, 3); - let expected = _mm_setr_pi16(-1, -1, -124, 1); - let r = _mm_hsubs_pi16(a, b); - assert_eq_m64(r, expected); - } - - #[simd_test(enable = "ssse3,mmx")] - unsafe fn test_mm_maddubs_pi16() { - let a = _mm_setr_pi8(1, 2, 3, 4, 5, 6, 7, 8); - let b = _mm_setr_pi8(4, 63, 4, 3, 24, 12, 6, 19); - let expected = _mm_setr_pi16(130, 24, 192, 194); - let r = _mm_maddubs_pi16(a, b); - assert_eq_m64(r, expected); - } - - #[simd_test(enable = "ssse3,mmx")] - unsafe fn test_mm_mulhrs_pi16() { - let a = _mm_setr_pi16(1, 2, 3, 4); - let b = _mm_setr_pi16(4, 32767, -1, -32768); - let expected = _mm_setr_pi16(0, 2, 0, -4); - let r = _mm_mulhrs_pi16(a, b); - assert_eq_m64(r, expected); - } - - #[simd_test(enable = "ssse3,mmx")] - unsafe fn test_mm_sign_pi8() { - let a = _mm_setr_pi8(1, 2, 3, 4, -5, -6, 7, 8); - let b = _mm_setr_pi8(4, 64, 0, 3, 1, -1, -2, 1); - let expected = _mm_setr_pi8(1, 2, 0, 4, -5, 6, -7, 8); - let r = _mm_sign_pi8(a, b); - assert_eq_m64(r, expected); - } - - #[simd_test(enable = "ssse3,mmx")] - unsafe fn test_mm_sign_pi16() { - let a = _mm_setr_pi16(-1, 2, 3, 4); - let b = _mm_setr_pi16(1, -1, 1, 0); - let expected = _mm_setr_pi16(-1, -2, 3, 0); - let r = _mm_sign_pi16(a, b); - assert_eq_m64(r, expected); - } - - #[simd_test(enable = "ssse3,mmx")] - unsafe fn test_mm_sign_pi32() { - let a = _mm_setr_pi32(-1, 2); - let b = _mm_setr_pi32(1, 0); - let expected = _mm_setr_pi32(-1, 0); - let r = _mm_sign_pi32(a, b); - assert_eq_m64(r, expected); - } } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/test.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/test.rs index d981a5e0c..bab89e61a 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/test.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/test.rs @@ -1,23 +1,11 @@ //! Utilities used in testing the x86 intrinsics use crate::core_arch::x86::*; - -#[target_feature(enable = "mmx")] -pub unsafe fn assert_eq_m64(a: __m64, b: __m64) { - union A { - a: __m64, - b: u64, - } - assert_eq!(A { a }.b, A { a: b }.b) -} +use std::mem::transmute; #[target_feature(enable = "sse2")] pub unsafe fn assert_eq_m128i(a: __m128i, b: __m128i) { - union A { - a: __m128i, - b: [u64; 2], - } - assert_eq!(A { a }.b, A { a: b }.b) + assert_eq!(transmute::<_, [u64; 2]>(a), transmute::<_, [u64; 2]>(b)) } #[target_feature(enable = "sse2")] @@ -29,11 +17,7 @@ pub unsafe fn assert_eq_m128d(a: __m128d, b: __m128d) { #[target_feature(enable = "sse2")] pub unsafe fn get_m128d(a: __m128d, idx: usize) -> f64 { - union A { - a: __m128d, - b: [f64; 2], - }; - A { a }.b[idx] + transmute::<_, [f64; 2]>(a)[idx] } #[target_feature(enable = "sse")] @@ -46,11 +30,7 @@ pub unsafe fn assert_eq_m128(a: __m128, b: __m128) { #[target_feature(enable = "sse")] pub unsafe fn get_m128(a: __m128, idx: usize) -> f32 { - union A { - a: __m128, - b: [f32; 4], - }; - A { a }.b[idx] + transmute::<_, [f32; 4]>(a)[idx] } // not actually an intrinsic but useful in various tests as we proted from @@ -62,16 +42,12 @@ pub unsafe fn _mm_setr_epi64x(a: i64, b: i64) -> __m128i { #[target_feature(enable = "avx")] pub unsafe fn assert_eq_m256i(a: __m256i, b: __m256i) { - union A { - a: __m256i, - b: [u64; 4], - } - assert_eq!(A { a }.b, A { a: b }.b) + assert_eq!(transmute::<_, [u64; 4]>(a), transmute::<_, [u64; 4]>(b)) } #[target_feature(enable = "avx")] pub unsafe fn assert_eq_m256d(a: __m256d, b: __m256d) { - let cmp = _mm256_cmp_pd(a, b, _CMP_EQ_OQ); + let cmp = _mm256_cmp_pd::<_CMP_EQ_OQ>(a, b); if _mm256_movemask_pd(cmp) != 0b1111 { panic!("{:?} != {:?}", a, b); } @@ -79,16 +55,12 @@ pub unsafe fn assert_eq_m256d(a: __m256d, b: __m256d) { #[target_feature(enable = "avx")] pub unsafe fn get_m256d(a: __m256d, idx: usize) -> f64 { - union A { - a: __m256d, - b: [f64; 4], - }; - A { a }.b[idx] + transmute::<_, [f64; 4]>(a)[idx] } #[target_feature(enable = "avx")] pub unsafe fn assert_eq_m256(a: __m256, b: __m256) { - let cmp = _mm256_cmp_ps(a, b, _CMP_EQ_OQ); + let cmp = _mm256_cmp_ps::<_CMP_EQ_OQ>(a, b); if _mm256_movemask_ps(cmp) != 0b11111111 { panic!("{:?} != {:?}", a, b); } @@ -96,11 +68,22 @@ pub unsafe fn assert_eq_m256(a: __m256, b: __m256) { #[target_feature(enable = "avx")] pub unsafe fn get_m256(a: __m256, idx: usize) -> f32 { - union A { - a: __m256, - b: [f32; 8], - }; - A { a }.b[idx] + transmute::<_, [f32; 8]>(a)[idx] +} + +#[target_feature(enable = "avx512f")] +pub unsafe fn get_m512(a: __m512, idx: usize) -> f32 { + transmute::<_, [f32; 16]>(a)[idx] +} + +#[target_feature(enable = "avx512f")] +pub unsafe fn get_m512d(a: __m512d, idx: usize) -> f64 { + transmute::<_, [f64; 8]>(a)[idx] +} + +#[target_feature(enable = "avx512f")] +pub unsafe fn get_m512i(a: __m512i, idx: usize) -> i64 { + transmute::<_, [i64; 8]>(a)[idx] } // These intrinsics doesn't exist on x86 b/c it requires a 64-bit register, @@ -109,24 +92,30 @@ pub unsafe fn get_m256(a: __m256, idx: usize) -> f32 { mod x86_polyfill { use crate::core_arch::x86::*; - pub unsafe fn _mm_insert_epi64(a: __m128i, val: i64, idx: i32) -> __m128i { + #[rustc_legacy_const_generics(2)] + pub unsafe fn _mm_insert_epi64(a: __m128i, val: i64) -> __m128i { + static_assert_imm1!(INDEX); + #[repr(C)] union A { a: __m128i, b: [i64; 2], - }; + } let mut a = A { a }; - a.b[idx as usize] = val; + a.b[INDEX as usize] = val; a.a } #[target_feature(enable = "avx2")] - pub unsafe fn _mm256_insert_epi64(a: __m256i, val: i64, idx: i32) -> __m256i { + #[rustc_legacy_const_generics(2)] + pub unsafe fn _mm256_insert_epi64(a: __m256i, val: i64) -> __m256i { + static_assert_imm2!(INDEX); + #[repr(C)] union A { a: __m256i, b: [i64; 4], - }; + } let mut a = A { a }; - a.b[idx as usize] = val; + a.b[INDEX as usize] = val; a.a } } @@ -137,9 +126,19 @@ mod x86_polyfill { pub use self::x86_polyfill::*; pub unsafe fn assert_eq_m512i(a: __m512i, b: __m512i) { - union A { - a: __m512i, - b: [i32; 16], + assert_eq!(transmute::<_, [i32; 16]>(a), transmute::<_, [i32; 16]>(b)) +} + +pub unsafe fn assert_eq_m512(a: __m512, b: __m512) { + let cmp = _mm512_cmp_ps_mask::<_CMP_EQ_OQ>(a, b); + if cmp != 0b11111111_11111111 { + panic!("{:?} != {:?}", a, b); + } +} + +pub unsafe fn assert_eq_m512d(a: __m512d, b: __m512d) { + let cmp = _mm512_cmp_pd_mask::<_CMP_EQ_OQ>(a, b); + if cmp != 0b11111111 { + panic!("{:?} != {:?}", a, b); } - assert_eq!(A { a }.b, A { a: b }.b) } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86/xsave.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86/xsave.rs index f3814dd7f..30f807e44 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86/xsave.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86/xsave.rs @@ -7,19 +7,21 @@ use stdarch_test::assert_instr; #[allow(improper_ctypes)] extern "C" { #[link_name = "llvm.x86.xsave"] - fn xsave(p: *mut u8, hi: u32, lo: u32) -> (); + fn xsave(p: *mut u8, hi: u32, lo: u32); #[link_name = "llvm.x86.xrstor"] - fn xrstor(p: *const u8, hi: u32, lo: u32) -> (); + fn xrstor(p: *const u8, hi: u32, lo: u32); #[link_name = "llvm.x86.xsetbv"] - fn xsetbv(v: u32, hi: u32, lo: u32) -> (); + fn xsetbv(v: u32, hi: u32, lo: u32); + #[link_name = "llvm.x86.xgetbv"] + fn xgetbv(v: u32) -> i64; #[link_name = "llvm.x86.xsaveopt"] - fn xsaveopt(p: *mut u8, hi: u32, lo: u32) -> (); + fn xsaveopt(p: *mut u8, hi: u32, lo: u32); #[link_name = "llvm.x86.xsavec"] - fn xsavec(p: *mut u8, hi: u32, lo: u32) -> (); + fn xsavec(p: *mut u8, hi: u32, lo: u32); #[link_name = "llvm.x86.xsaves"] - fn xsaves(p: *mut u8, hi: u32, lo: u32) -> (); + fn xsaves(p: *mut u8, hi: u32, lo: u32); #[link_name = "llvm.x86.xrstors"] - fn xrstors(p: *const u8, hi: u32, lo: u32) -> (); + fn xrstors(p: *const u8, hi: u32, lo: u32); } /// Performs a full or partial save of the enabled processor states to memory at @@ -85,10 +87,7 @@ pub unsafe fn _xsetbv(a: u32, val: u64) { #[cfg_attr(test, assert_instr(xgetbv))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _xgetbv(xcr_no: u32) -> u64 { - let eax: u32; - let edx: u32; - asm!("xgetbv" : "={eax}"(eax), "={edx}"(edx) : "{ecx}"(xcr_no)); - ((edx as u64) << 32) | (eax as u64) + xgetbv(xcr_no) as u64 } /// Performs a full or partial save of the enabled processor states to memory at @@ -197,7 +196,7 @@ mod tests { } impl fmt::Debug for XsaveArea { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "[")?; for i in 0..self.data.len() { write!(f, "{}", self.data[i])?; diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/adx.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/adx.rs index 57efe75dd..a54d71136 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/adx.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/adx.rs @@ -55,7 +55,7 @@ mod tests { #[test] fn test_addcarry_u64() { unsafe { - let a = u64::max_value(); + let a = u64::MAX; let mut out = 0; let r = _addcarry_u64(0, a, 1, &mut out); @@ -86,7 +86,7 @@ mod tests { #[simd_test(enable = "adx")] unsafe fn test_addcarryx_u64() { - let a = u64::max_value(); + let a = u64::MAX; let mut out = 0; let r = _addcarry_u64(0, a, 1, &mut out); @@ -117,7 +117,7 @@ mod tests { #[test] fn test_subborrow_u64() { unsafe { - let a = u64::max_value(); + let a = u64::MAX; let mut out = 0; let r = _subborrow_u64(0, 0, 1, &mut out); diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/avx.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/avx.rs index 5215fd6fb..7ba26371c 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/avx.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/avx.rs @@ -23,12 +23,13 @@ use crate::{ /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_insert_epi64) #[inline] -#[rustc_args_required_const(2)] +#[rustc_legacy_const_generics(2)] #[target_feature(enable = "avx")] // This intrinsic has no corresponding instruction. #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_insert_epi64(a: __m256i, i: i64, index: i32) -> __m256i { - transmute(simd_insert(a.as_i64x4(), (index as u32) & 3, i)) +pub unsafe fn _mm256_insert_epi64(a: __m256i, i: i64) -> __m256i { + static_assert_imm2!(INDEX); + transmute(simd_insert(a.as_i64x4(), INDEX as u32, i)) } #[cfg(test)] @@ -40,7 +41,7 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm256_insert_epi64() { let a = _mm256_setr_epi64x(1, 2, 3, 4); - let r = _mm256_insert_epi64(a, 0, 3); + let r = _mm256_insert_epi64::<3>(a, 0); let e = _mm256_setr_epi64x(1, 2, 3, 0); assert_eq_m256i(r, e); } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/avx2.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/avx2.rs index 7cc3fb6ef..14447a137 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/avx2.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/avx2.rs @@ -20,17 +20,17 @@ use crate::core_arch::{simd_llvm::*, x86::*}; -/// Extracts a 64-bit integer from `a`, selected with `imm8`. +/// Extracts a 64-bit integer from `a`, selected with `INDEX`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_extract_epi64) #[inline] #[target_feature(enable = "avx2")] -#[rustc_args_required_const(1)] +#[rustc_legacy_const_generics(1)] // This intrinsic has no corresponding instruction. #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_extract_epi64(a: __m256i, imm8: i32) -> i64 { - let imm8 = (imm8 & 3) as u32; - simd_extract(a.as_i64x4(), imm8) +pub unsafe fn _mm256_extract_epi64(a: __m256i) -> i64 { + static_assert_imm2!(INDEX); + simd_extract(a.as_i64x4(), INDEX as u32) } #[cfg(test)] @@ -41,7 +41,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_extract_epi64() { let a = _mm256_setr_epi64x(0, 1, 2, 3); - let r = _mm256_extract_epi64(a, 3); + let r = _mm256_extract_epi64::<3>(a); assert_eq!(r, 3); } } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/avx512f.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/avx512f.rs new file mode 100644 index 000000000..5eed0502c --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/avx512f.rs @@ -0,0 +1,12346 @@ +use crate::{ + core_arch::{simd::*, simd_llvm::*, x86::*, x86_64::*}, + mem::transmute, +}; + +#[cfg(test)] +use stdarch_test::assert_instr; + +/// Convert the lower double-precision (64-bit) floating-point element in a to a 64-bit integer, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtsd_i64&expand=1792) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2si))] +pub unsafe fn _mm_cvtsd_i64(a: __m128d) -> i64 { + _mm_cvtsd_si64(a) +} + +/// Convert the lower single-precision (32-bit) floating-point element in a to a 64-bit integer, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtss_i64&expand=1894) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2si))] +pub unsafe fn _mm_cvtss_i64(a: __m128) -> i64 { + _mm_cvtss_si64(a) +} + +/// Convert the lower single-precision (32-bit) floating-point element in a to an unsigned 64-bit integer, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtss_u64&expand=1902) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2usi))] +pub unsafe fn _mm_cvtss_u64(a: __m128) -> u64 { + transmute(vcvtss2usi64(a.as_f32x4(), _MM_FROUND_CUR_DIRECTION)) +} + +/// Convert the lower double-precision (64-bit) floating-point element in a to an unsigned 64-bit integer, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtsd_u64&expand=1800) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2usi))] +pub unsafe fn _mm_cvtsd_u64(a: __m128d) -> u64 { + transmute(vcvtsd2usi64(a.as_f64x2(), _MM_FROUND_CUR_DIRECTION)) +} + +/// Convert the signed 64-bit integer b to a single-precision (32-bit) floating-point element, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvti32_ss&expand=1643) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsi2ss))] +pub unsafe fn _mm_cvti64_ss(a: __m128, b: i64) -> __m128 { + let b = b as f32; + let r = simd_insert(a, 0, b); + transmute(r) +} + +/// Convert the signed 64-bit integer b to a double-precision (64-bit) floating-point element, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvti64_sd&expand=1644) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsi2sd))] +pub unsafe fn _mm_cvti64_sd(a: __m128d, b: i64) -> __m128d { + let b = b as f64; + let r = simd_insert(a, 0, b); + transmute(r) +} + +/// Convert the unsigned 64-bit integer b to a single-precision (32-bit) floating-point element, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtu64_ss&expand=2035) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtusi2ss))] +pub unsafe fn _mm_cvtu64_ss(a: __m128, b: u64) -> __m128 { + let b = b as f32; + let r = simd_insert(a, 0, b); + transmute(r) +} + +/// Convert the unsigned 64-bit integer b to a double-precision (64-bit) floating-point element, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtu64_sd&expand=2034) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtusi2sd))] +pub unsafe fn _mm_cvtu64_sd(a: __m128d, b: u64) -> __m128d { + let b = b as f64; + let r = simd_insert(a, 0, b); + transmute(r) +} + +/// Convert the lower double-precision (64-bit) floating-point element in a to a 64-bit integer with truncation, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttsd_i64&expand=2016) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2si))] +pub unsafe fn _mm_cvttsd_i64(a: __m128d) -> i64 { + transmute(vcvtsd2si64(a.as_f64x2(), _MM_FROUND_CUR_DIRECTION)) +} + +/// Convert the lower double-precision (64-bit) floating-point element in a to an unsigned 64-bit integer with truncation, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttsd_u64&expand=2021) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2usi))] +pub unsafe fn _mm_cvttsd_u64(a: __m128d) -> u64 { + transmute(vcvtsd2usi64(a.as_f64x2(), _MM_FROUND_CUR_DIRECTION)) +} + +/// Convert the lower single-precision (32-bit) floating-point element in a to a 64-bit integer with truncation, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=#text=_mm_cvttss_i64&expand=2023) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2si))] +pub unsafe fn _mm_cvttss_i64(a: __m128) -> i64 { + transmute(vcvtss2si64(a.as_f32x4(), _MM_FROUND_CUR_DIRECTION)) +} + +/// Convert the lower single-precision (32-bit) floating-point element in a to an unsigned 64-bit integer with truncation, and store the result in dst. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttss_u64&expand=2027) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2usi))] +pub unsafe fn _mm_cvttss_u64(a: __m128) -> u64 { + transmute(vcvtss2usi64(a.as_f32x4(), _MM_FROUND_CUR_DIRECTION)) +} + +/// Convert the signed 64-bit integer b to a double-precision (64-bit) floating-point element, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst. +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundi64_sd&expand=1313) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsi2sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundi64_sd(a: __m128d, b: i64) -> __m128d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let r = vcvtsi2sd64(a, b, ROUNDING); + transmute(r) +} + +/// Convert the signed 64-bit integer b to a double-precision (64-bit) floating-point element, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst. +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundsi64_sd&expand=1367) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsi2sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundsi64_sd(a: __m128d, b: i64) -> __m128d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let r = vcvtsi2sd64(a, b, ROUNDING); + transmute(r) +} + +/// Convert the signed 64-bit integer b to a single-precision (32-bit) floating-point element, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst. +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundi64_ss&expand=1314) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsi2ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundi64_ss(a: __m128, b: i64) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let r = vcvtsi2ss64(a, b, ROUNDING); + transmute(r) +} + +/// Convert the unsigned 64-bit integer b to a double-precision (64-bit) floating-point element, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst.\ +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundu64_sd&expand=1379) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtusi2sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundu64_sd(a: __m128d, b: u64) -> __m128d { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let r = vcvtusi2sd64(a, b, ROUNDING); + transmute(r) +} + +/// Convert the signed 64-bit integer b to a single-precision (32-bit) floating-point element, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst. +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundsi64_ss&expand=1368) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsi2ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundsi64_ss(a: __m128, b: i64) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let r = vcvtsi2ss64(a, b, ROUNDING); + transmute(r) +} + +/// Convert the unsigned 64-bit integer b to a single-precision (32-bit) floating-point element, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundu64_ss&expand=1380) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtusi2ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundu64_ss(a: __m128, b: u64) -> __m128 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let r = vcvtusi2ss64(a, b, ROUNDING); + transmute(r) +} + +/// Convert the lower double-precision (64-bit) floating-point element in a to a 64-bit integer, and store the result in dst.\ +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundsd_si64&expand=1360) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2si, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundsd_si64(a: __m128d) -> i64 { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let r = vcvtsd2si64(a, ROUNDING); + transmute(r) +} + +/// Convert the lower double-precision (64-bit) floating-point element in a to a 64-bit integer, and store the result in dst.\ +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundsd_i64&expand=1358) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2si, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundsd_i64(a: __m128d) -> i64 { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let r = vcvtsd2si64(a, ROUNDING); + transmute(r) +} + +/// Convert the lower double-precision (64-bit) floating-point element in a to an unsigned 64-bit integer, and store the result in dst.\ +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundsd_u64&expand=1365) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2usi, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundsd_u64(a: __m128d) -> u64 { + static_assert_rounding!(ROUNDING); + let a = a.as_f64x2(); + let r = vcvtsd2usi64(a, ROUNDING); + transmute(r) +} + +/// Convert the lower single-precision (32-bit) floating-point element in a to a 64-bit integer, and store the result in dst.\ +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundss_si64&expand=1375) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2si, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundss_si64(a: __m128) -> i64 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let r = vcvtss2si64(a, ROUNDING); + transmute(r) +} + +/// Convert the lower single-precision (32-bit) floating-point element in a to a 64-bit integer, and store the result in dst.\ +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundss_i64&expand=1370) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2si, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundss_i64(a: __m128) -> i64 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let r = vcvtss2si64(a, ROUNDING); + transmute(r) +} + +/// Convert the lower single-precision (32-bit) floating-point element in a to an unsigned 64-bit integer, and store the result in dst.\ +/// Rounding is done according to the rounding\[3:0\] parameter, which can be one of:\ +/// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and suppress exceptions\ +/// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and suppress exceptions\ +/// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress exceptions\ +/// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress exceptions\ +/// _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see _MM_SET_ROUNDING_MODE +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundss_u64&expand=1377) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2usi, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundss_u64(a: __m128) -> u64 { + static_assert_rounding!(ROUNDING); + let a = a.as_f32x4(); + let r = vcvtss2usi64(a, ROUNDING); + transmute(r) +} + +/// Convert the lower double-precision (64-bit) floating-point element in a to a 64-bit integer with truncation, and store the result in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_roundsd_si64&expand=1931) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2si, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundsd_si64(a: __m128d) -> i64 { + static_assert_sae!(SAE); + let a = a.as_f64x2(); + let r = vcvtsd2si64(a, SAE); + transmute(r) +} + +/// Convert the lower double-precision (64-bit) floating-point element in a to a 64-bit integer with truncation, and store the result in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_roundsd_i64&expand=1929) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2si, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundsd_i64(a: __m128d) -> i64 { + static_assert_sae!(SAE); + let a = a.as_f64x2(); + let r = vcvtsd2si64(a, SAE); + transmute(r) +} + +/// Convert the lower double-precision (64-bit) floating-point element in a to an unsigned 64-bit integer with truncation, and store the result in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_roundsd_u64&expand=1933) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtsd2usi, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundsd_u64(a: __m128d) -> u64 { + static_assert_sae!(SAE); + let a = a.as_f64x2(); + let r = vcvtsd2usi64(a, SAE); + transmute(r) +} + +/// Convert the lower single-precision (32-bit) floating-point element in a to a 64-bit integer with truncation, and store the result in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_roundss_i64&expand=1935) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2si, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundss_i64(a: __m128) -> i64 { + static_assert_sae!(SAE); + let a = a.as_f32x4(); + let r = vcvtss2si64(a, SAE); + transmute(r) +} + +/// Convert the lower single-precision (32-bit) floating-point element in a to a 64-bit integer with truncation, and store the result in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_roundss_si64&expand=1937) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2si, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundss_si64(a: __m128) -> i64 { + static_assert_sae!(SAE); + let a = a.as_f32x4(); + let r = vcvtss2si64(a, SAE); + transmute(r) +} + +/// Convert the lower single-precision (32-bit) floating-point element in a to an unsigned 64-bit integer with truncation, and store the result in dst.\ +/// Exceptions can be suppressed by passing _MM_FROUND_NO_EXC in the sae parameter. +/// +/// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_roundss_u64&expand=1939) +#[inline] +#[target_feature(enable = "avx512f")] +#[cfg_attr(test, assert_instr(vcvtss2usi, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundss_u64(a: __m128) -> u64 { + static_assert_sae!(SAE); + let a = a.as_f32x4(); + let r = vcvtss2usi64(a, SAE); + transmute(r) +} + +#[allow(improper_ctypes)] +extern "C" { + #[link_name = "llvm.x86.avx512.vcvtss2si64"] + fn vcvtss2si64(a: f32x4, rounding: i32) -> i64; + #[link_name = "llvm.x86.avx512.vcvtss2usi64"] + fn vcvtss2usi64(a: f32x4, rounding: i32) -> u64; + #[link_name = "llvm.x86.avx512.vcvtsd2si64"] + fn vcvtsd2si64(a: f64x2, rounding: i32) -> i64; + #[link_name = "llvm.x86.avx512.vcvtsd2usi64"] + fn vcvtsd2usi64(a: f64x2, rounding: i32) -> u64; + + #[link_name = "llvm.x86.avx512.cvtsi2ss64"] + fn vcvtsi2ss64(a: f32x4, b: i64, rounding: i32) -> f32x4; + #[link_name = "llvm.x86.avx512.cvtsi2sd64"] + fn vcvtsi2sd64(a: f64x2, b: i64, rounding: i32) -> f64x2; + #[link_name = "llvm.x86.avx512.cvtusi642ss"] + fn vcvtusi2ss64(a: f32x4, b: u64, rounding: i32) -> f32x4; + #[link_name = "llvm.x86.avx512.cvtusi642sd"] + fn vcvtusi2sd64(a: f64x2, b: u64, rounding: i32) -> f64x2; +} + +#[cfg(test)] +mod tests { + + use stdarch_test::simd_test; + + use crate::core_arch::x86::*; + use crate::core_arch::x86_64::*; + use crate::hint::black_box; + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_abs_epi64() { + let a = _mm512_set_epi64(0, 1, -1, i64::MAX, i64::MIN, 100, -100, -32); + let r = _mm512_abs_epi64(a); + let e = _mm512_set_epi64(0, 1, 1, i64::MAX, i64::MAX.wrapping_add(1), 100, 100, 32); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_abs_epi64() { + let a = _mm512_set_epi64(0, 1, -1, i64::MAX, i64::MIN, 100, -100, -32); + let r = _mm512_mask_abs_epi64(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_abs_epi64(a, 0b11111111, a); + let e = _mm512_set_epi64(0, 1, 1, i64::MAX, i64::MIN, 100, 100, 32); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_abs_epi64() { + let a = _mm512_set_epi64(0, 1, -1, i64::MAX, i64::MIN, 100, -100, -32); + let r = _mm512_maskz_abs_epi64(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_abs_epi64(0b11111111, a); + let e = _mm512_set_epi64(0, 1, 1, i64::MAX, i64::MIN, 100, 100, 32); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_abs_epi64() { + let a = _mm256_set_epi64x(i64::MAX, i64::MIN, 100, -100); + let r = _mm256_abs_epi64(a); + let e = _mm256_set_epi64x(i64::MAX, i64::MAX.wrapping_add(1), 100, 100); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_abs_epi64() { + let a = _mm256_set_epi64x(i64::MAX, i64::MIN, 100, -100); + let r = _mm256_mask_abs_epi64(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_abs_epi64(a, 0b00001111, a); + let e = _mm256_set_epi64x(i64::MAX, i64::MAX.wrapping_add(1), 100, 100); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_abs_epi64() { + let a = _mm256_set_epi64x(i64::MAX, i64::MIN, 100, -100); + let r = _mm256_maskz_abs_epi64(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_abs_epi64(0b00001111, a); + let e = _mm256_set_epi64x(i64::MAX, i64::MAX.wrapping_add(1), 100, 100); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_abs_pd() { + let a = _mm512_setr_pd(0., 1., -1., f64::MAX, f64::MIN, 100., -100., -32.); + let r = _mm512_abs_pd(a); + let e = _mm512_setr_pd(0., 1., 1., f64::MAX, f64::MAX, 100., 100., 32.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_abs_pd() { + let a = _mm512_setr_pd(0., 1., -1., f64::MAX, f64::MIN, 100., -100., -32.); + let r = _mm512_mask_abs_pd(a, 0, a); + assert_eq_m512d(r, a); + let r = _mm512_mask_abs_pd(a, 0b00001111, a); + let e = _mm512_setr_pd(0., 1., 1., f64::MAX, f64::MIN, 100., -100., -32.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_mov_epi64() { + let src = _mm512_set1_epi64(1); + let a = _mm512_set1_epi64(2); + let r = _mm512_mask_mov_epi64(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_mov_epi64(src, 0b11111111, a); + assert_eq_m512i(r, a); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_mov_epi64() { + let a = _mm512_set1_epi64(2); + let r = _mm512_maskz_mov_epi64(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_mov_epi64(0b11111111, a); + assert_eq_m512i(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_mov_epi64() { + let src = _mm256_set1_epi64x(1); + let a = _mm256_set1_epi64x(2); + let r = _mm256_mask_mov_epi64(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_mov_epi64(src, 0b00001111, a); + assert_eq_m256i(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_mov_epi64() { + let a = _mm256_set1_epi64x(2); + let r = _mm256_maskz_mov_epi64(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_mov_epi64(0b00001111, a); + assert_eq_m256i(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_mov_epi64() { + let src = _mm_set1_epi64x(1); + let a = _mm_set1_epi64x(2); + let r = _mm_mask_mov_epi64(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_mov_epi64(src, 0b00000011, a); + assert_eq_m128i(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_mov_epi64() { + let a = _mm_set1_epi64x(2); + let r = _mm_maskz_mov_epi64(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_mov_epi64(0b00000011, a); + assert_eq_m128i(r, a); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_mov_pd() { + let src = _mm512_set1_pd(1.); + let a = _mm512_set1_pd(2.); + let r = _mm512_mask_mov_pd(src, 0, a); + assert_eq_m512d(r, src); + let r = _mm512_mask_mov_pd(src, 0b11111111, a); + assert_eq_m512d(r, a); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_mov_pd() { + let a = _mm512_set1_pd(2.); + let r = _mm512_maskz_mov_pd(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_mov_pd(0b11111111, a); + assert_eq_m512d(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_mov_pd() { + let src = _mm256_set1_pd(1.); + let a = _mm256_set1_pd(2.); + let r = _mm256_mask_mov_pd(src, 0, a); + assert_eq_m256d(r, src); + let r = _mm256_mask_mov_pd(src, 0b00001111, a); + assert_eq_m256d(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_mov_pd() { + let a = _mm256_set1_pd(2.); + let r = _mm256_maskz_mov_pd(0, a); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_mov_pd(0b00001111, a); + assert_eq_m256d(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_mov_pd() { + let src = _mm_set1_pd(1.); + let a = _mm_set1_pd(2.); + let r = _mm_mask_mov_pd(src, 0, a); + assert_eq_m128d(r, src); + let r = _mm_mask_mov_pd(src, 0b00000011, a); + assert_eq_m128d(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_mov_pd() { + let a = _mm_set1_pd(2.); + let r = _mm_maskz_mov_pd(0, a); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_mov_pd(0b00000011, a); + assert_eq_m128d(r, a); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_add_epi64() { + let a = _mm512_setr_epi64(0, 1, -1, i64::MAX, i64::MIN, 100, -100, -32); + let b = _mm512_set1_epi64(1); + let r = _mm512_add_epi64(a, b); + let e = _mm512_setr_epi64(1, 2, 0, i64::MIN, i64::MIN + 1, 101, -99, -31); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_add_epi64() { + let a = _mm512_setr_epi64(0, 1, -1, i64::MAX, i64::MIN, 100, -100, -32); + let b = _mm512_set1_epi64(1); + let r = _mm512_mask_add_epi64(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_add_epi64(a, 0b00001111, a, b); + let e = _mm512_setr_epi64(1, 2, 0, i64::MIN, i64::MIN, 100, -100, -32); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_add_epi64() { + let a = _mm512_setr_epi64(0, 1, -1, i64::MAX, i64::MIN, 100, -100, -32); + let b = _mm512_set1_epi64(1); + let r = _mm512_maskz_add_epi64(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_add_epi64(0b00001111, a, b); + let e = _mm512_setr_epi64(1, 2, 0, i64::MIN, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_add_epi64() { + let a = _mm256_set_epi64x(1, -1, i64::MAX, i64::MIN); + let b = _mm256_set1_epi64x(1); + let r = _mm256_mask_add_epi64(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_add_epi64(a, 0b00001111, a, b); + let e = _mm256_set_epi64x(2, 0, i64::MIN, i64::MIN + 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_add_epi64() { + let a = _mm256_set_epi64x(1, -1, i64::MAX, i64::MIN); + let b = _mm256_set1_epi64x(1); + let r = _mm256_maskz_add_epi64(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_add_epi64(0b00001111, a, b); + let e = _mm256_set_epi64x(2, 0, i64::MIN, i64::MIN + 1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_add_epi64() { + let a = _mm_set_epi64x(i64::MAX, i64::MIN); + let b = _mm_set1_epi64x(1); + let r = _mm_mask_add_epi64(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_add_epi64(a, 0b00000011, a, b); + let e = _mm_set_epi64x(i64::MIN, i64::MIN + 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_add_epi64() { + let a = _mm_set_epi64x(i64::MAX, i64::MIN); + let b = _mm_set1_epi64x(1); + let r = _mm_maskz_add_epi64(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_add_epi64(0b00000011, a, b); + let e = _mm_set_epi64x(i64::MIN, i64::MIN + 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_add_pd() { + let a = _mm512_setr_pd(0., 1., -1., f64::MAX, f64::MIN, 100., -100., -32.); + let b = _mm512_set1_pd(1.); + let r = _mm512_add_pd(a, b); + let e = _mm512_setr_pd(1., 2., 0., f64::MAX, f64::MIN + 1., 101., -99., -31.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_add_pd() { + let a = _mm512_setr_pd(0., 1., -1., f64::MAX, f64::MIN, 100., -100., -32.); + let b = _mm512_set1_pd(1.); + let r = _mm512_mask_add_pd(a, 0, a, b); + assert_eq_m512d(r, a); + let r = _mm512_mask_add_pd(a, 0b00001111, a, b); + let e = _mm512_setr_pd(1., 2., 0., f64::MAX, f64::MIN, 100., -100., -32.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_add_pd() { + let a = _mm512_setr_pd(0., 1., -1., f64::MAX, f64::MIN, 100., -100., -32.); + let b = _mm512_set1_pd(1.); + let r = _mm512_maskz_add_pd(0, a, b); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_add_pd(0b00001111, a, b); + let e = _mm512_setr_pd(1., 2., 0., f64::MAX, 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_add_pd() { + let a = _mm256_set_pd(1., -1., f64::MAX, f64::MIN); + let b = _mm256_set1_pd(1.); + let r = _mm256_mask_add_pd(a, 0, a, b); + assert_eq_m256d(r, a); + let r = _mm256_mask_add_pd(a, 0b00001111, a, b); + let e = _mm256_set_pd(2., 0., f64::MAX, f64::MIN + 1.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_add_pd() { + let a = _mm256_set_pd(1., -1., f64::MAX, f64::MIN); + let b = _mm256_set1_pd(1.); + let r = _mm256_maskz_add_pd(0, a, b); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_add_pd(0b00001111, a, b); + let e = _mm256_set_pd(2., 0., f64::MAX, f64::MIN + 1.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_add_pd() { + let a = _mm_set_pd(f64::MAX, f64::MIN); + let b = _mm_set1_pd(1.); + let r = _mm_mask_add_pd(a, 0, a, b); + assert_eq_m128d(r, a); + let r = _mm_mask_add_pd(a, 0b00000011, a, b); + let e = _mm_set_pd(f64::MAX, f64::MIN + 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_add_pd() { + let a = _mm_set_pd(f64::MAX, f64::MIN); + let b = _mm_set1_pd(1.); + let r = _mm_maskz_add_pd(0, a, b); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_add_pd(0b00000011, a, b); + let e = _mm_set_pd(f64::MAX, f64::MIN + 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_sub_epi64() { + let a = _mm512_setr_epi64(0, 1, -1, i64::MAX, i64::MIN, 100, -100, -32); + let b = _mm512_set1_epi64(1); + let r = _mm512_sub_epi64(a, b); + let e = _mm512_setr_epi64(-1, 0, -2, i64::MAX - 1, i64::MAX, 99, -101, -33); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_sub_epi64() { + let a = _mm512_setr_epi64(0, 1, -1, i64::MAX, i64::MIN, 100, -100, -32); + let b = _mm512_set1_epi64(1); + let r = _mm512_mask_sub_epi64(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_sub_epi64(a, 0b00001111, a, b); + let e = _mm512_setr_epi64(-1, 0, -2, i64::MAX - 1, i64::MIN, 100, -100, -32); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_sub_epi64() { + let a = _mm512_setr_epi64(0, 1, -1, i64::MAX, i64::MIN, 100, -100, -32); + let b = _mm512_set1_epi64(1); + let r = _mm512_maskz_sub_epi64(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_sub_epi64(0b00001111, a, b); + let e = _mm512_setr_epi64(-1, 0, -2, i64::MAX - 1, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_sub_epi64() { + let a = _mm256_set_epi64x(1, -1, i64::MAX, i64::MIN); + let b = _mm256_set1_epi64x(1); + let r = _mm256_mask_sub_epi64(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_sub_epi64(a, 0b00001111, a, b); + let e = _mm256_set_epi64x(0, -2, i64::MAX - 1, i64::MAX); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_sub_epi64() { + let a = _mm256_set_epi64x(1, -1, i64::MAX, i64::MIN); + let b = _mm256_set1_epi64x(1); + let r = _mm256_maskz_sub_epi64(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_sub_epi64(0b00001111, a, b); + let e = _mm256_set_epi64x(0, -2, i64::MAX - 1, i64::MAX); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_sub_epi64() { + let a = _mm_set_epi64x(i64::MAX, i64::MIN); + let b = _mm_set1_epi64x(1); + let r = _mm_mask_sub_epi64(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_sub_epi64(a, 0b00000011, a, b); + let e = _mm_set_epi64x(i64::MAX - 1, i64::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_sub_epi64() { + let a = _mm_set_epi64x(i64::MAX, i64::MIN); + let b = _mm_set1_epi64x(1); + let r = _mm_maskz_sub_epi64(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_sub_epi64(0b00000011, a, b); + let e = _mm_set_epi64x(i64::MAX - 1, i64::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_sub_pd() { + let a = _mm512_setr_pd(0., 1., -1., f64::MAX, f64::MIN, 100., -100., -32.); + let b = _mm512_set1_pd(1.); + let r = _mm512_sub_pd(a, b); + let e = _mm512_setr_pd(-1., 0., -2., f64::MAX - 1., f64::MIN, 99., -101., -33.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_sub_pd() { + let a = _mm512_setr_pd(0., 1., -1., f64::MAX, f64::MIN, 100., -100., -32.); + let b = _mm512_set1_pd(1.); + let r = _mm512_mask_sub_pd(a, 0, a, b); + assert_eq_m512d(r, a); + let r = _mm512_mask_sub_pd(a, 0b00001111, a, b); + let e = _mm512_setr_pd(-1., 0., -2., f64::MAX - 1., f64::MIN, 100., -100., -32.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_sub_pd() { + let a = _mm512_setr_pd(0., 1., -1., f64::MAX, f64::MIN, 100., -100., -32.); + let b = _mm512_set1_pd(1.); + let r = _mm512_maskz_sub_pd(0, a, b); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_sub_pd(0b00001111, a, b); + let e = _mm512_setr_pd(-1., 0., -2., f64::MAX - 1., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_sub_pd() { + let a = _mm256_set_pd(1., -1., f64::MAX, f64::MIN); + let b = _mm256_set1_pd(1.); + let r = _mm256_mask_sub_pd(a, 0, a, b); + assert_eq_m256d(r, a); + let r = _mm256_mask_sub_pd(a, 0b00001111, a, b); + let e = _mm256_set_pd(0., -2., f64::MAX - 1., f64::MIN); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_sub_pd() { + let a = _mm256_set_pd(1., -1., f64::MAX, f64::MIN); + let b = _mm256_set1_pd(1.); + let r = _mm256_maskz_sub_pd(0, a, b); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_sub_pd(0b00001111, a, b); + let e = _mm256_set_pd(0., -2., f64::MAX - 1., f64::MIN); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_sub_pd() { + let a = _mm_set_pd(f64::MAX, f64::MIN); + let b = _mm_set1_pd(1.); + let r = _mm_mask_sub_pd(a, 0, a, b); + assert_eq_m128d(r, a); + let r = _mm_mask_sub_pd(a, 0b00000011, a, b); + let e = _mm_set_pd(f64::MAX - 1., f64::MIN); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_sub_pd() { + let a = _mm_set_pd(f64::MAX, f64::MIN); + let b = _mm_set1_pd(1.); + let r = _mm_maskz_sub_pd(0, a, b); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_sub_pd(0b00000011, a, b); + let e = _mm_set_pd(f64::MAX - 1., f64::MIN); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mul_epi32() { + let a = _mm512_set1_epi32(1); + let b = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r = _mm512_mul_epi32(a, b); + let e = _mm512_set_epi64(15, 13, 11, 9, 7, 5, 3, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_mul_epi32() { + let a = _mm512_set1_epi32(1); + let b = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r = _mm512_mask_mul_epi32(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_mul_epi32(a, 0b00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 1 | 1 << 32, 1 | 1 << 32, 1 | 1 << 32, 1 | 1 << 32, + 7, 5, 3, 1, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_mul_epi32() { + let a = _mm512_set1_epi32(1); + let b = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r = _mm512_maskz_mul_epi32(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_mul_epi32(0b00001111, a, b); + let e = _mm512_set_epi64(0, 0, 0, 0, 7, 5, 3, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_mul_epi32() { + let a = _mm256_set1_epi32(1); + let b = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let r = _mm256_mask_mul_epi32(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_mul_epi32(a, 0b00001111, a, b); + let e = _mm256_set_epi64x(2, 4, 6, 8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_mul_epi32() { + let a = _mm256_set1_epi32(1); + let b = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let r = _mm256_maskz_mul_epi32(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_mul_epi32(0b00001111, a, b); + let e = _mm256_set_epi64x(2, 4, 6, 8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_mul_epi32() { + let a = _mm_set1_epi32(1); + let b = _mm_set_epi32(1, 2, 3, 4); + let r = _mm_mask_mul_epi32(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_mul_epi32(a, 0b00000011, a, b); + let e = _mm_set_epi64x(2, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_mul_epi32() { + let a = _mm_set1_epi32(1); + let b = _mm_set_epi32(1, 2, 3, 4); + let r = _mm_maskz_mul_epi32(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_mul_epi32(0b00000011, a, b); + let e = _mm_set_epi64x(2, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mul_epu32() { + let a = _mm512_set1_epi32(1); + let b = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r = _mm512_mul_epu32(a, b); + let e = _mm512_set_epi64(15, 13, 11, 9, 7, 5, 3, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_mul_epu32() { + let a = _mm512_set1_epi32(1); + let b = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r = _mm512_mask_mul_epu32(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_mul_epu32(a, 0b00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 1 | 1 << 32, 1 | 1 << 32, 1 | 1 << 32, 1 | 1 << 32, + 7, 5, 3, 1, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_mul_epu32() { + let a = _mm512_set1_epi32(1); + let b = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r = _mm512_maskz_mul_epu32(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_mul_epu32(0b00001111, a, b); + let e = _mm512_set_epi64(0, 0, 0, 0, 7, 5, 3, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_mul_epu32() { + let a = _mm256_set1_epi32(1); + let b = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let r = _mm256_mask_mul_epu32(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_mul_epu32(a, 0b00001111, a, b); + let e = _mm256_set_epi64x(2, 4, 6, 8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_mul_epu32() { + let a = _mm256_set1_epi32(1); + let b = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let r = _mm256_maskz_mul_epu32(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_mul_epu32(0b00001111, a, b); + let e = _mm256_set_epi64x(2, 4, 6, 8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_mul_epu32() { + let a = _mm_set1_epi32(1); + let b = _mm_set_epi32(1, 2, 3, 4); + let r = _mm_mask_mul_epu32(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_mul_epu32(a, 0b00000011, a, b); + let e = _mm_set_epi64x(2, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_mul_epu32() { + let a = _mm_set1_epi32(1); + let b = _mm_set_epi32(1, 2, 3, 4); + let r = _mm_maskz_mul_epu32(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_mul_epu32(0b00000011, a, b); + let e = _mm_set_epi64x(2, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mullox_epi64() { + let a = _mm512_setr_epi64(0, 1, i64::MAX, i64::MIN, i64::MAX, 100, -100, -32); + let b = _mm512_set1_epi64(2); + let r = _mm512_mullox_epi64(a, b); + let e = _mm512_setr_epi64(0, 2, -2, 0, -2, 200, -200, -64); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_mullox_epi64() { + let a = _mm512_setr_epi64(0, 1, i64::MAX, i64::MIN, i64::MAX, 100, -100, -32); + let b = _mm512_set1_epi64(2); + let r = _mm512_mask_mullox_epi64(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_mullox_epi64(a, 0b00001111, a, b); + let e = _mm512_setr_epi64(0, 2, -2, 0, i64::MAX, 100, -100, -32); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mul_pd() { + let a = _mm512_setr_pd(0., 1., f64::MAX, f64::MIN, f64::MAX, f64::MIN, -100., -32.); + let b = _mm512_set1_pd(2.); + let r = _mm512_mul_pd(a, b); + #[rustfmt::skip] + let e = _mm512_setr_pd( + 0., 2., f64::INFINITY, f64::NEG_INFINITY, + f64::INFINITY, f64::NEG_INFINITY, -200., -64., + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_mul_pd() { + let a = _mm512_setr_pd(0., 1., f64::MAX, f64::MIN, f64::MAX, f64::MIN, -100., -32.); + let b = _mm512_set1_pd(2.); + let r = _mm512_mask_mul_pd(a, 0, a, b); + assert_eq_m512d(r, a); + let r = _mm512_mask_mul_pd(a, 0b00001111, a, b); + #[rustfmt::skip] + let e = _mm512_setr_pd( + 0., 2., f64::INFINITY, f64::NEG_INFINITY, + f64::MAX, f64::MIN, -100., -32., + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_mul_pd() { + let a = _mm512_setr_pd(0., 1., f64::MAX, f64::MIN, f64::MAX, f64::MIN, -100., -32.); + let b = _mm512_set1_pd(2.); + let r = _mm512_maskz_mul_pd(0, a, b); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_mul_pd(0b00001111, a, b); + let e = _mm512_setr_pd(0., 2., f64::INFINITY, f64::NEG_INFINITY, 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_mul_pd() { + let a = _mm256_set_pd(0., 1., f64::MAX, f64::MIN); + let b = _mm256_set1_pd(2.); + let r = _mm256_mask_mul_pd(a, 0, a, b); + assert_eq_m256d(r, a); + let r = _mm256_mask_mul_pd(a, 0b00001111, a, b); + let e = _mm256_set_pd(0., 2., f64::INFINITY, f64::NEG_INFINITY); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_mul_pd() { + let a = _mm256_set_pd(0., 1., f64::MAX, f64::MIN); + let b = _mm256_set1_pd(2.); + let r = _mm256_maskz_mul_pd(0, a, b); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_mul_pd(0b00001111, a, b); + let e = _mm256_set_pd(0., 2., f64::INFINITY, f64::NEG_INFINITY); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_mul_pd() { + let a = _mm_set_pd(f64::MAX, f64::MIN); + let b = _mm_set1_pd(2.); + let r = _mm_mask_mul_pd(a, 0, a, b); + assert_eq_m128d(r, a); + let r = _mm_mask_mul_pd(a, 0b00000011, a, b); + let e = _mm_set_pd(f64::INFINITY, f64::NEG_INFINITY); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_mul_pd() { + let a = _mm_set_pd(f64::MAX, f64::MIN); + let b = _mm_set1_pd(2.); + let r = _mm_maskz_mul_pd(0, a, b); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_mul_pd(0b00000011, a, b); + let e = _mm_set_pd(f64::INFINITY, f64::NEG_INFINITY); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_div_pd() { + let a = _mm512_setr_pd(0., 1., f64::MAX, f64::MIN, f64::MAX, f64::MIN, -100., -32.); + let b = _mm512_setr_pd(2., 2., 0., 0., 0., 0., 2., 2.); + let r = _mm512_div_pd(a, b); + #[rustfmt::skip] + let e = _mm512_setr_pd( + 0., 0.5, f64::INFINITY, f64::NEG_INFINITY, + f64::INFINITY, f64::NEG_INFINITY, -50., -16., + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_div_pd() { + let a = _mm512_setr_pd(0., 1., f64::MAX, f64::MIN, f64::MAX, f64::MIN, -100., -32.); + let b = _mm512_setr_pd(2., 2., 0., 0., 0., 0., 2., 2.); + let r = _mm512_mask_div_pd(a, 0, a, b); + assert_eq_m512d(r, a); + let r = _mm512_mask_div_pd(a, 0b00001111, a, b); + #[rustfmt::skip] + let e = _mm512_setr_pd( + 0., 0.5, f64::INFINITY, f64::NEG_INFINITY, + f64::MAX, f64::MIN, -100., -32., + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_div_pd() { + let a = _mm512_setr_pd(0., 1., f64::MAX, f64::MIN, f64::MAX, f64::MIN, -100., -32.); + let b = _mm512_setr_pd(2., 2., 0., 0., 0., 0., 2., 2.); + let r = _mm512_maskz_div_pd(0, a, b); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_div_pd(0b00001111, a, b); + let e = _mm512_setr_pd(0., 0.5, f64::INFINITY, f64::NEG_INFINITY, 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_div_pd() { + let a = _mm256_set_pd(0., 1., f64::MAX, f64::MIN); + let b = _mm256_set_pd(2., 2., 0., 0.); + let r = _mm256_mask_div_pd(a, 0, a, b); + assert_eq_m256d(r, a); + let r = _mm256_mask_div_pd(a, 0b00001111, a, b); + let e = _mm256_set_pd(0., 0.5, f64::INFINITY, f64::NEG_INFINITY); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_div_pd() { + let a = _mm256_set_pd(0., 1., f64::MAX, f64::MIN); + let b = _mm256_set_pd(2., 2., 0., 0.); + let r = _mm256_maskz_div_pd(0, a, b); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_div_pd(0b00001111, a, b); + let e = _mm256_set_pd(0., 0.5, f64::INFINITY, f64::NEG_INFINITY); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_div_pd() { + let a = _mm_set_pd(f64::MAX, f64::MIN); + let b = _mm_set_pd(0., 0.); + let r = _mm_mask_div_pd(a, 0, a, b); + assert_eq_m128d(r, a); + let r = _mm_mask_div_pd(a, 0b00000011, a, b); + let e = _mm_set_pd(f64::INFINITY, f64::NEG_INFINITY); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_div_pd() { + let a = _mm_set_pd(f64::MAX, f64::MIN); + let b = _mm_set_pd(0., 0.); + let r = _mm_maskz_div_pd(0, a, b); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_div_pd(0b00000011, a, b); + let e = _mm_set_pd(f64::INFINITY, f64::NEG_INFINITY); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_max_epi64() { + let a = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm512_setr_epi64(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_max_epi64(a, b); + let e = _mm512_setr_epi64(7, 6, 5, 4, 4, 5, 6, 7); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_max_epi64() { + let a = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm512_setr_epi64(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_mask_max_epi64(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_max_epi64(a, 0b00001111, a, b); + let e = _mm512_setr_epi64(7, 6, 5, 4, 4, 5, 6, 7); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_max_epi64() { + let a = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm512_setr_epi64(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_maskz_max_epi64(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_max_epi64(0b00001111, a, b); + let e = _mm512_setr_epi64(7, 6, 5, 4, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_max_epi64() { + let a = _mm256_set_epi64x(0, 1, 2, 3); + let b = _mm256_set_epi64x(3, 2, 1, 0); + let r = _mm256_max_epi64(a, b); + let e = _mm256_set_epi64x(3, 2, 2, 3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_max_epi64() { + let a = _mm256_set_epi64x(0, 1, 2, 3); + let b = _mm256_set_epi64x(3, 2, 1, 0); + let r = _mm256_mask_max_epi64(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_max_epi64(a, 0b00001111, a, b); + let e = _mm256_set_epi64x(3, 2, 2, 3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_max_epi64() { + let a = _mm256_set_epi64x(0, 1, 2, 3); + let b = _mm256_set_epi64x(3, 2, 1, 0); + let r = _mm256_maskz_max_epi64(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_max_epi64(0b00001111, a, b); + let e = _mm256_set_epi64x(3, 2, 2, 3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_max_epi64() { + let a = _mm_set_epi64x(2, 3); + let b = _mm_set_epi64x(3, 2); + let r = _mm_max_epi64(a, b); + let e = _mm_set_epi64x(3, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_max_epi64() { + let a = _mm_set_epi64x(2, 3); + let b = _mm_set_epi64x(3, 2); + let r = _mm_mask_max_epi64(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_max_epi64(a, 0b00000011, a, b); + let e = _mm_set_epi64x(3, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_max_epi64() { + let a = _mm_set_epi64x(2, 3); + let b = _mm_set_epi64x(3, 2); + let r = _mm_maskz_max_epi64(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_max_epi64(0b00000011, a, b); + let e = _mm_set_epi64x(3, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_max_pd() { + let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let b = _mm512_setr_pd(7., 6., 5., 4., 3., 2., 1., 0.); + let r = _mm512_max_pd(a, b); + let e = _mm512_setr_pd(7., 6., 5., 4., 4., 5., 6., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_max_pd() { + let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let b = _mm512_setr_pd(7., 6., 5., 4., 3., 2., 1., 0.); + let r = _mm512_mask_max_pd(a, 0, a, b); + assert_eq_m512d(r, a); + let r = _mm512_mask_max_pd(a, 0b00001111, a, b); + let e = _mm512_setr_pd(7., 6., 5., 4., 4., 5., 6., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_max_pd() { + let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let b = _mm512_setr_pd(7., 6., 5., 4., 3., 2., 1., 0.); + let r = _mm512_maskz_max_pd(0, a, b); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_max_pd(0b00001111, a, b); + let e = _mm512_setr_pd(7., 6., 5., 4., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_max_pd() { + let a = _mm256_set_pd(0., 1., 2., 3.); + let b = _mm256_set_pd(3., 2., 1., 0.); + let r = _mm256_mask_max_pd(a, 0, a, b); + assert_eq_m256d(r, a); + let r = _mm256_mask_max_pd(a, 0b00001111, a, b); + let e = _mm256_set_pd(3., 2., 2., 3.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_max_pd() { + let a = _mm256_set_pd(0., 1., 2., 3.); + let b = _mm256_set_pd(3., 2., 1., 0.); + let r = _mm256_maskz_max_pd(0, a, b); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_max_pd(0b00001111, a, b); + let e = _mm256_set_pd(3., 2., 2., 3.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_max_pd() { + let a = _mm_set_pd(2., 3.); + let b = _mm_set_pd(3., 2.); + let r = _mm_mask_max_pd(a, 0, a, b); + assert_eq_m128d(r, a); + let r = _mm_mask_max_pd(a, 0b00000011, a, b); + let e = _mm_set_pd(3., 3.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_max_pd() { + let a = _mm_set_pd(2., 3.); + let b = _mm_set_pd(3., 2.); + let r = _mm_maskz_max_pd(0, a, b); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_max_pd(0b00000011, a, b); + let e = _mm_set_pd(3., 3.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_max_epu64() { + let a = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm512_setr_epi64(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_max_epu64(a, b); + let e = _mm512_setr_epi64(7, 6, 5, 4, 4, 5, 6, 7); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_max_epu64() { + let a = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm512_setr_epi64(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_mask_max_epu64(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_max_epu64(a, 0b00001111, a, b); + let e = _mm512_setr_epi64(7, 6, 5, 4, 4, 5, 6, 7); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_max_epu64() { + let a = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm512_setr_epi64(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_maskz_max_epu64(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_max_epu64(0b00001111, a, b); + let e = _mm512_setr_epi64(7, 6, 5, 4, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_max_epu64() { + let a = _mm256_set_epi64x(0, 1, 2, 3); + let b = _mm256_set_epi64x(3, 2, 1, 0); + let r = _mm256_max_epu64(a, b); + let e = _mm256_set_epi64x(3, 2, 2, 3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_max_epu64() { + let a = _mm256_set_epi64x(0, 1, 2, 3); + let b = _mm256_set_epi64x(3, 2, 1, 0); + let r = _mm256_mask_max_epu64(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_max_epu64(a, 0b00001111, a, b); + let e = _mm256_set_epi64x(3, 2, 2, 3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_max_epu64() { + let a = _mm256_set_epi64x(0, 1, 2, 3); + let b = _mm256_set_epi64x(3, 2, 1, 0); + let r = _mm256_maskz_max_epu64(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_max_epu64(0b00001111, a, b); + let e = _mm256_set_epi64x(3, 2, 2, 3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_max_epu64() { + let a = _mm_set_epi64x(2, 3); + let b = _mm_set_epi64x(3, 2); + let r = _mm_max_epu64(a, b); + let e = _mm_set_epi64x(3, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_max_epu64() { + let a = _mm_set_epi64x(2, 3); + let b = _mm_set_epi64x(3, 2); + let r = _mm_mask_max_epu64(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_max_epu64(a, 0b00000011, a, b); + let e = _mm_set_epi64x(3, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_max_epu64() { + let a = _mm_set_epi64x(2, 3); + let b = _mm_set_epi64x(3, 2); + let r = _mm_maskz_max_epu64(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_max_epu64(0b00000011, a, b); + let e = _mm_set_epi64x(3, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_min_epi64() { + let a = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm512_setr_epi64(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_min_epi64(a, b); + let e = _mm512_setr_epi64(0, 1, 2, 3, 3, 2, 1, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_min_epi64() { + let a = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm512_setr_epi64(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_mask_min_epi64(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_min_epi64(a, 0b00001111, a, b); + let e = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_min_epi64() { + let a = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm512_setr_epi64(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_maskz_min_epi64(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_min_epi64(0b00001111, a, b); + let e = _mm512_setr_epi64(0, 1, 2, 3, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_min_epi64() { + let a = _mm256_set_epi64x(0, 1, 2, 3); + let b = _mm256_set_epi64x(3, 2, 1, 0); + let r = _mm256_min_epi64(a, b); + let e = _mm256_set_epi64x(0, 1, 1, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_min_epi64() { + let a = _mm256_set_epi64x(0, 1, 2, 3); + let b = _mm256_set_epi64x(3, 2, 1, 0); + let r = _mm256_mask_min_epi64(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_min_epi64(a, 0b00001111, a, b); + let e = _mm256_set_epi64x(0, 1, 1, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_min_epi64() { + let a = _mm256_set_epi64x(0, 1, 2, 3); + let b = _mm256_set_epi64x(3, 2, 1, 0); + let r = _mm256_maskz_min_epi64(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_min_epi64(0b00001111, a, b); + let e = _mm256_set_epi64x(0, 1, 1, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_min_pd() { + let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let b = _mm512_setr_pd(7., 6., 5., 4., 3., 2., 1., 0.); + let r = _mm512_min_pd(a, b); + let e = _mm512_setr_pd(0., 1., 2., 3., 3., 2., 1., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_min_pd() { + let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let b = _mm512_setr_pd(7., 6., 5., 4., 3., 2., 1., 0.); + let r = _mm512_mask_min_pd(a, 0, a, b); + assert_eq_m512d(r, a); + let r = _mm512_mask_min_pd(a, 0b00001111, a, b); + let e = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_min_pd() { + let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let b = _mm512_setr_pd(7., 6., 5., 4., 3., 2., 1., 0.); + let r = _mm512_maskz_min_pd(0, a, b); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_min_pd(0b00001111, a, b); + let e = _mm512_setr_pd(0., 1., 2., 3., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_min_pd() { + let a = _mm256_set_pd(0., 1., 2., 3.); + let b = _mm256_set_pd(3., 2., 1., 0.); + let r = _mm256_mask_min_pd(a, 0, a, b); + assert_eq_m256d(r, a); + let r = _mm256_mask_min_pd(a, 0b00001111, a, b); + let e = _mm256_set_pd(0., 1., 1., 0.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_min_pd() { + let a = _mm256_set_pd(0., 1., 2., 3.); + let b = _mm256_set_pd(3., 2., 1., 0.); + let r = _mm256_maskz_min_pd(0, a, b); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_min_pd(0b00001111, a, b); + let e = _mm256_set_pd(0., 1., 1., 0.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_min_pd() { + let a = _mm_set_pd(0., 1.); + let b = _mm_set_pd(1., 0.); + let r = _mm_mask_min_pd(a, 0, a, b); + assert_eq_m128d(r, a); + let r = _mm_mask_min_pd(a, 0b00000011, a, b); + let e = _mm_set_pd(0., 0.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_min_pd() { + let a = _mm_set_pd(0., 1.); + let b = _mm_set_pd(1., 0.); + let r = _mm_maskz_min_pd(0, a, b); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_min_pd(0b00000011, a, b); + let e = _mm_set_pd(0., 0.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_min_epu64() { + let a = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm512_setr_epi64(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_min_epu64(a, b); + let e = _mm512_setr_epi64(0, 1, 2, 3, 3, 2, 1, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_min_epu64() { + let a = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm512_setr_epi64(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_mask_min_epu64(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_min_epu64(a, 0b00001111, a, b); + let e = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_min_epu64() { + let a = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let b = _mm512_setr_epi64(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm512_maskz_min_epu64(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_min_epu64(0b00001111, a, b); + let e = _mm512_setr_epi64(0, 1, 2, 3, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_min_epu64() { + let a = _mm256_set_epi64x(0, 1, 2, 3); + let b = _mm256_set_epi64x(3, 2, 1, 0); + let r = _mm256_min_epu64(a, b); + let e = _mm256_set_epi64x(0, 1, 1, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_min_epu64() { + let a = _mm256_set_epi64x(0, 1, 2, 3); + let b = _mm256_set_epi64x(3, 2, 1, 0); + let r = _mm256_mask_min_epu64(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_min_epu64(a, 0b00001111, a, b); + let e = _mm256_set_epi64x(0, 1, 1, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_min_epu64() { + let a = _mm256_set_epi64x(0, 1, 2, 3); + let b = _mm256_set_epi64x(3, 2, 1, 0); + let r = _mm256_maskz_min_epu64(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_min_epu64(0b00001111, a, b); + let e = _mm256_set_epi64x(0, 1, 1, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_min_epu64() { + let a = _mm_set_epi64x(0, 1); + let b = _mm_set_epi64x(1, 0); + let r = _mm_min_epu64(a, b); + let e = _mm_set_epi64x(0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_min_epu64() { + let a = _mm_set_epi64x(0, 1); + let b = _mm_set_epi64x(1, 0); + let r = _mm_mask_min_epu64(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_min_epu64(a, 0b00000011, a, b); + let e = _mm_set_epi64x(0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_min_epu64() { + let a = _mm_set_epi64x(0, 1); + let b = _mm_set_epi64x(1, 0); + let r = _mm_maskz_min_epu64(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_min_epu64(0b00000011, a, b); + let e = _mm_set_epi64x(0, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_sqrt_pd() { + let a = _mm512_setr_pd(0., 1., 4., 9., 16., 25., 36., 49.); + let r = _mm512_sqrt_pd(a); + let e = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_sqrt_pd() { + let a = _mm512_setr_pd(0., 1., 4., 9., 16., 25., 36., 49.); + let r = _mm512_mask_sqrt_pd(a, 0, a); + assert_eq_m512d(r, a); + let r = _mm512_mask_sqrt_pd(a, 0b00001111, a); + let e = _mm512_setr_pd(0., 1., 2., 3., 16., 25., 36., 49.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_sqrt_pd() { + let a = _mm512_setr_pd(0., 1., 4., 9., 16., 25., 36., 49.); + let r = _mm512_maskz_sqrt_pd(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_sqrt_pd(0b00001111, a); + let e = _mm512_setr_pd(0., 1., 2., 3., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_sqrt_pd() { + let a = _mm256_set_pd(0., 1., 4., 9.); + let r = _mm256_mask_sqrt_pd(a, 0, a); + assert_eq_m256d(r, a); + let r = _mm256_mask_sqrt_pd(a, 0b00001111, a); + let e = _mm256_set_pd(0., 1., 2., 3.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_sqrt_pd() { + let a = _mm256_set_pd(0., 1., 4., 9.); + let r = _mm256_maskz_sqrt_pd(0, a); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_sqrt_pd(0b00001111, a); + let e = _mm256_set_pd(0., 1., 2., 3.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_sqrt_pd() { + let a = _mm_set_pd(0., 1.); + let r = _mm_mask_sqrt_pd(a, 0, a); + assert_eq_m128d(r, a); + let r = _mm_mask_sqrt_pd(a, 0b00000011, a); + let e = _mm_set_pd(0., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_sqrt_pd() { + let a = _mm_set_pd(0., 1.); + let r = _mm_maskz_sqrt_pd(0, a); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_sqrt_pd(0b00000011, a); + let e = _mm_set_pd(0., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fmadd_pd() { + let a = _mm512_setr_pd(1., 1., 1., 1., 1., 1., 1., 1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_setr_pd(1., 1., 1., 1., 1., 1., 1., 1.); + let r = _mm512_fmadd_pd(a, b, c); + let e = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fmadd_pd() { + let a = _mm512_setr_pd(1., 1., 1., 1., 1., 1., 1., 1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_setr_pd(1., 1., 1., 1., 1., 1., 1., 1.); + let r = _mm512_mask_fmadd_pd(a, 0, b, c); + assert_eq_m512d(r, a); + let r = _mm512_mask_fmadd_pd(a, 0b00001111, b, c); + let e = _mm512_setr_pd(1., 2., 3., 4., 1., 1., 1., 1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fmadd_pd() { + let a = _mm512_setr_pd(1., 1., 1., 1., 1., 1., 1., 1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_setr_pd(1., 1., 1., 1., 1., 1., 1., 1.); + let r = _mm512_maskz_fmadd_pd(0, a, b, c); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_fmadd_pd(0b00001111, a, b, c); + let e = _mm512_setr_pd(1., 2., 3., 4., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fmadd_pd() { + let a = _mm512_setr_pd(1., 1., 1., 1., 1., 1., 1., 1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_setr_pd(1., 1., 1., 1., 2., 2., 2., 2.); + let r = _mm512_mask3_fmadd_pd(a, b, c, 0); + assert_eq_m512d(r, c); + let r = _mm512_mask3_fmadd_pd(a, b, c, 0b00001111); + let e = _mm512_setr_pd(1., 2., 3., 4., 2., 2., 2., 2.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_fmadd_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set_pd(0., 1., 2., 3.); + let c = _mm256_set1_pd(1.); + let r = _mm256_mask_fmadd_pd(a, 0, b, c); + assert_eq_m256d(r, a); + let r = _mm256_mask_fmadd_pd(a, 0b00001111, b, c); + let e = _mm256_set_pd(1., 2., 3., 4.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_fmadd_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set_pd(0., 1., 2., 3.); + let c = _mm256_set1_pd(1.); + let r = _mm256_maskz_fmadd_pd(0, a, b, c); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_fmadd_pd(0b00001111, a, b, c); + let e = _mm256_set_pd(1., 2., 3., 4.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask3_fmadd_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set_pd(0., 1., 2., 3.); + let c = _mm256_set1_pd(1.); + let r = _mm256_mask3_fmadd_pd(a, b, c, 0); + assert_eq_m256d(r, c); + let r = _mm256_mask3_fmadd_pd(a, b, c, 0b00001111); + let e = _mm256_set_pd(1., 2., 3., 4.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_fmadd_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set_pd(0., 1.); + let c = _mm_set1_pd(1.); + let r = _mm_mask_fmadd_pd(a, 0, b, c); + assert_eq_m128d(r, a); + let r = _mm_mask_fmadd_pd(a, 0b00000011, b, c); + let e = _mm_set_pd(1., 2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_fmadd_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set_pd(0., 1.); + let c = _mm_set1_pd(1.); + let r = _mm_maskz_fmadd_pd(0, a, b, c); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_fmadd_pd(0b00000011, a, b, c); + let e = _mm_set_pd(1., 2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask3_fmadd_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set_pd(0., 1.); + let c = _mm_set1_pd(1.); + let r = _mm_mask3_fmadd_pd(a, b, c, 0); + assert_eq_m128d(r, c); + let r = _mm_mask3_fmadd_pd(a, b, c, 0b00000011); + let e = _mm_set_pd(1., 2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fmsub_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_set1_pd(1.); + let r = _mm512_fmsub_pd(a, b, c); + let e = _mm512_setr_pd(-1., 0., 1., 2., 3., 4., 5., 6.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fmsub_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_set1_pd(1.); + let r = _mm512_mask_fmsub_pd(a, 0, b, c); + assert_eq_m512d(r, a); + let r = _mm512_mask_fmsub_pd(a, 0b00001111, b, c); + let e = _mm512_setr_pd(-1., 0., 1., 2., 1., 1., 1., 1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fmsub_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_set1_pd(1.); + let r = _mm512_maskz_fmsub_pd(0, a, b, c); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_fmsub_pd(0b00001111, a, b, c); + let e = _mm512_setr_pd(-1., 0., 1., 2., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fmsub_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_setr_pd(1., 1., 1., 1., 2., 2., 2., 2.); + let r = _mm512_mask3_fmsub_pd(a, b, c, 0); + assert_eq_m512d(r, c); + let r = _mm512_mask3_fmsub_pd(a, b, c, 0b00001111); + let e = _mm512_setr_pd(-1., 0., 1., 2., 2., 2., 2., 2.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_fmsub_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set_pd(0., 1., 2., 3.); + let c = _mm256_set1_pd(1.); + let r = _mm256_mask_fmsub_pd(a, 0, b, c); + assert_eq_m256d(r, a); + let r = _mm256_mask_fmsub_pd(a, 0b00001111, b, c); + let e = _mm256_set_pd(-1., 0., 1., 2.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_fmsub_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set_pd(0., 1., 2., 3.); + let c = _mm256_set1_pd(1.); + let r = _mm256_maskz_fmsub_pd(0, a, b, c); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_fmsub_pd(0b00001111, a, b, c); + let e = _mm256_set_pd(-1., 0., 1., 2.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask3_fmsub_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set_pd(0., 1., 2., 3.); + let c = _mm256_set1_pd(1.); + let r = _mm256_mask3_fmsub_pd(a, b, c, 0); + assert_eq_m256d(r, c); + let r = _mm256_mask3_fmsub_pd(a, b, c, 0b00001111); + let e = _mm256_set_pd(-1., 0., 1., 2.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_fmsub_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set_pd(0., 1.); + let c = _mm_set1_pd(1.); + let r = _mm_mask_fmsub_pd(a, 0, b, c); + assert_eq_m128d(r, a); + let r = _mm_mask_fmsub_pd(a, 0b00000011, b, c); + let e = _mm_set_pd(-1., 0.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_fmsub_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set_pd(0., 1.); + let c = _mm_set1_pd(1.); + let r = _mm_maskz_fmsub_pd(0, a, b, c); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_fmsub_pd(0b00000011, a, b, c); + let e = _mm_set_pd(-1., 0.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask3_fmsub_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set_pd(0., 1.); + let c = _mm_set1_pd(1.); + let r = _mm_mask3_fmsub_pd(a, b, c, 0); + assert_eq_m128d(r, c); + let r = _mm_mask3_fmsub_pd(a, b, c, 0b00000011); + let e = _mm_set_pd(-1., 0.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fmaddsub_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_set1_pd(1.); + let r = _mm512_fmaddsub_pd(a, b, c); + let e = _mm512_setr_pd(-1., 2., 1., 4., 3., 6., 5., 8.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fmaddsub_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_set1_pd(1.); + let r = _mm512_mask_fmaddsub_pd(a, 0, b, c); + assert_eq_m512d(r, a); + let r = _mm512_mask_fmaddsub_pd(a, 0b00001111, b, c); + let e = _mm512_setr_pd(-1., 2., 1., 4., 1., 1., 1., 1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fmaddsub_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_set1_pd(1.); + let r = _mm512_maskz_fmaddsub_pd(0, a, b, c); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_fmaddsub_pd(0b00001111, a, b, c); + let e = _mm512_setr_pd(-1., 2., 1., 4., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fmaddsub_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_setr_pd(1., 1., 1., 1., 2., 2., 2., 2.); + let r = _mm512_mask3_fmaddsub_pd(a, b, c, 0); + assert_eq_m512d(r, c); + let r = _mm512_mask3_fmaddsub_pd(a, b, c, 0b00001111); + let e = _mm512_setr_pd(-1., 2., 1., 4., 2., 2., 2., 2.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_fmaddsub_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set_pd(0., 1., 2., 3.); + let c = _mm256_set1_pd(1.); + let r = _mm256_mask_fmaddsub_pd(a, 0, b, c); + assert_eq_m256d(r, a); + let r = _mm256_mask_fmaddsub_pd(a, 0b00001111, b, c); + let e = _mm256_set_pd(1., 0., 3., 2.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_fmaddsub_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set_pd(0., 1., 2., 3.); + let c = _mm256_set1_pd(1.); + let r = _mm256_maskz_fmaddsub_pd(0, a, b, c); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_fmaddsub_pd(0b00001111, a, b, c); + let e = _mm256_set_pd(1., 0., 3., 2.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask3_fmaddsub_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set_pd(0., 1., 2., 3.); + let c = _mm256_set1_pd(1.); + let r = _mm256_mask3_fmaddsub_pd(a, b, c, 0); + assert_eq_m256d(r, c); + let r = _mm256_mask3_fmaddsub_pd(a, b, c, 0b00001111); + let e = _mm256_set_pd(1., 0., 3., 2.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_fmaddsub_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set_pd(0., 1.); + let c = _mm_set1_pd(1.); + let r = _mm_mask_fmaddsub_pd(a, 0, b, c); + assert_eq_m128d(r, a); + let r = _mm_mask_fmaddsub_pd(a, 0b00000011, b, c); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_fmaddsub_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set_pd(0., 1.); + let c = _mm_set1_pd(1.); + let r = _mm_maskz_fmaddsub_pd(0, a, b, c); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_fmaddsub_pd(0b00000011, a, b, c); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask3_fmaddsub_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set_pd(0., 1.); + let c = _mm_set1_pd(1.); + let r = _mm_mask3_fmaddsub_pd(a, b, c, 0); + assert_eq_m128d(r, c); + let r = _mm_mask3_fmaddsub_pd(a, b, c, 0b00000011); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fmsubadd_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_set1_pd(1.); + let r = _mm512_fmsubadd_pd(a, b, c); + let e = _mm512_setr_pd(1., 0., 3., 2., 5., 4., 7., 6.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fmsubadd_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_set1_pd(1.); + let r = _mm512_mask_fmsubadd_pd(a, 0, b, c); + assert_eq_m512d(r, a); + let r = _mm512_mask_fmsubadd_pd(a, 0b00001111, b, c); + let e = _mm512_setr_pd(1., 0., 3., 2., 1., 1., 1., 1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fmsubadd_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_set1_pd(1.); + let r = _mm512_maskz_fmsubadd_pd(0, a, b, c); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_fmsubadd_pd(0b00001111, a, b, c); + let e = _mm512_setr_pd(1., 0., 3., 2., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fmsubadd_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_setr_pd(1., 1., 1., 1., 2., 2., 2., 2.); + let r = _mm512_mask3_fmsubadd_pd(a, b, c, 0); + assert_eq_m512d(r, c); + let r = _mm512_mask3_fmsubadd_pd(a, b, c, 0b00001111); + let e = _mm512_setr_pd(1., 0., 3., 2., 2., 2., 2., 2.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_fmsubadd_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set_pd(0., 1., 2., 3.); + let c = _mm256_set1_pd(1.); + let r = _mm256_mask_fmsubadd_pd(a, 0, b, c); + assert_eq_m256d(r, a); + let r = _mm256_mask_fmsubadd_pd(a, 0b00001111, b, c); + let e = _mm256_set_pd(-1., 2., 1., 4.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_fmsubadd_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set_pd(0., 1., 2., 3.); + let c = _mm256_set1_pd(1.); + let r = _mm256_maskz_fmsubadd_pd(0, a, b, c); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_fmsubadd_pd(0b00001111, a, b, c); + let e = _mm256_set_pd(-1., 2., 1., 4.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask3_fmsubadd_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set_pd(0., 1., 2., 3.); + let c = _mm256_set1_pd(1.); + let r = _mm256_mask3_fmsubadd_pd(a, b, c, 0); + assert_eq_m256d(r, c); + let r = _mm256_mask3_fmsubadd_pd(a, b, c, 0b00001111); + let e = _mm256_set_pd(-1., 2., 1., 4.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_fmsubadd_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set_pd(0., 1.); + let c = _mm_set1_pd(1.); + let r = _mm_mask_fmsubadd_pd(a, 0, b, c); + assert_eq_m128d(r, a); + let r = _mm_mask_fmsubadd_pd(a, 0b00000011, b, c); + let e = _mm_set_pd(-1., 2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_fmsubadd_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set_pd(0., 1.); + let c = _mm_set1_pd(1.); + let r = _mm_maskz_fmsubadd_pd(0, a, b, c); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_fmsubadd_pd(0b00000011, a, b, c); + let e = _mm_set_pd(-1., 2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask3_fmsubadd_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set_pd(0., 1.); + let c = _mm_set1_pd(1.); + let r = _mm_mask3_fmsubadd_pd(a, b, c, 0); + assert_eq_m128d(r, c); + let r = _mm_mask3_fmsubadd_pd(a, b, c, 0b00000011); + let e = _mm_set_pd(-1., 2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fnmadd_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_set1_pd(1.); + let r = _mm512_fnmadd_pd(a, b, c); + let e = _mm512_setr_pd(1., 0., -1., -2., -3., -4., -5., -6.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fnmadd_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_set1_pd(1.); + let r = _mm512_mask_fnmadd_pd(a, 0, b, c); + assert_eq_m512d(r, a); + let r = _mm512_mask_fnmadd_pd(a, 0b00001111, b, c); + let e = _mm512_setr_pd(1., 0., -1., -2., 1., 1., 1., 1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fnmadd_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_set1_pd(1.); + let r = _mm512_maskz_fnmadd_pd(0, a, b, c); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_fnmadd_pd(0b00001111, a, b, c); + let e = _mm512_setr_pd(1., 0., -1., -2., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fnmadd_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_setr_pd(1., 1., 1., 1., 2., 2., 2., 2.); + let r = _mm512_mask3_fnmadd_pd(a, b, c, 0); + assert_eq_m512d(r, c); + let r = _mm512_mask3_fnmadd_pd(a, b, c, 0b00001111); + let e = _mm512_setr_pd(1., 0., -1., -2., 2., 2., 2., 2.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_fnmadd_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set_pd(0., 1., 2., 3.); + let c = _mm256_set1_pd(1.); + let r = _mm256_mask_fnmadd_pd(a, 0, b, c); + assert_eq_m256d(r, a); + let r = _mm256_mask_fnmadd_pd(a, 0b00001111, b, c); + let e = _mm256_set_pd(1., 0., -1., -2.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_fnmadd_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set_pd(0., 1., 2., 3.); + let c = _mm256_set1_pd(1.); + let r = _mm256_maskz_fnmadd_pd(0, a, b, c); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_fnmadd_pd(0b00001111, a, b, c); + let e = _mm256_set_pd(1., 0., -1., -2.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask3_fnmadd_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set_pd(0., 1., 2., 3.); + let c = _mm256_set1_pd(1.); + let r = _mm256_mask3_fnmadd_pd(a, b, c, 0); + assert_eq_m256d(r, c); + let r = _mm256_mask3_fnmadd_pd(a, b, c, 0b00001111); + let e = _mm256_set_pd(1., 0., -1., -2.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_fnmadd_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set_pd(0., 1.); + let c = _mm_set1_pd(1.); + let r = _mm_mask_fnmadd_pd(a, 0, b, c); + assert_eq_m128d(r, a); + let r = _mm_mask_fnmadd_pd(a, 0b00000011, b, c); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_fnmadd_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set_pd(0., 1.); + let c = _mm_set1_pd(1.); + let r = _mm_maskz_fnmadd_pd(0, a, b, c); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_fnmadd_pd(0b00000011, a, b, c); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask3_fnmadd_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set_pd(0., 1.); + let c = _mm_set1_pd(1.); + let r = _mm_mask3_fnmadd_pd(a, b, c, 0); + assert_eq_m128d(r, c); + let r = _mm_mask3_fnmadd_pd(a, b, c, 0b00000011); + let e = _mm_set_pd(1., 0.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fnmsub_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_set1_pd(1.); + let r = _mm512_fnmsub_pd(a, b, c); + let e = _mm512_setr_pd(-1., -2., -3., -4., -5., -6., -7., -8.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fnmsub_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_set1_pd(1.); + let r = _mm512_mask_fnmsub_pd(a, 0, b, c); + assert_eq_m512d(r, a); + let r = _mm512_mask_fnmsub_pd(a, 0b00001111, b, c); + let e = _mm512_setr_pd(-1., -2., -3., -4., 1., 1., 1., 1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fnmsub_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_set1_pd(1.); + let r = _mm512_maskz_fnmsub_pd(0, a, b, c); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_fnmsub_pd(0b00001111, a, b, c); + let e = _mm512_setr_pd(-1., -2., -3., -4., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fnmsub_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let c = _mm512_setr_pd(1., 1., 1., 1., 2., 2., 2., 2.); + let r = _mm512_mask3_fnmsub_pd(a, b, c, 0); + assert_eq_m512d(r, c); + let r = _mm512_mask3_fnmsub_pd(a, b, c, 0b00001111); + let e = _mm512_setr_pd(-1., -2., -3., -4., 2., 2., 2., 2.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_fnmsub_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set_pd(0., 1., 2., 3.); + let c = _mm256_set1_pd(1.); + let r = _mm256_mask_fnmsub_pd(a, 0, b, c); + assert_eq_m256d(r, a); + let r = _mm256_mask_fnmsub_pd(a, 0b00001111, b, c); + let e = _mm256_set_pd(-1., -2., -3., -4.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_fnmsub_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set_pd(0., 1., 2., 3.); + let c = _mm256_set1_pd(1.); + let r = _mm256_maskz_fnmsub_pd(0, a, b, c); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_fnmsub_pd(0b00001111, a, b, c); + let e = _mm256_set_pd(-1., -2., -3., -4.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask3_fnmsub_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set_pd(0., 1., 2., 3.); + let c = _mm256_set1_pd(1.); + let r = _mm256_mask3_fnmsub_pd(a, b, c, 0); + assert_eq_m256d(r, c); + let r = _mm256_mask3_fnmsub_pd(a, b, c, 0b00001111); + let e = _mm256_set_pd(-1., -2., -3., -4.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_fnmsub_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set_pd(0., 1.); + let c = _mm_set1_pd(1.); + let r = _mm_mask_fnmsub_pd(a, 0, b, c); + assert_eq_m128d(r, a); + let r = _mm_mask_fnmsub_pd(a, 0b00000011, b, c); + let e = _mm_set_pd(-1., -2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_fnmsub_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set_pd(0., 1.); + let c = _mm_set1_pd(1.); + let r = _mm_maskz_fnmsub_pd(0, a, b, c); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_fnmsub_pd(0b00000011, a, b, c); + let e = _mm_set_pd(-1., -2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask3_fnmsub_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set_pd(0., 1.); + let c = _mm_set1_pd(1.); + let r = _mm_mask3_fnmsub_pd(a, b, c, 0); + assert_eq_m128d(r, c); + let r = _mm_mask3_fnmsub_pd(a, b, c, 0b00000011); + let e = _mm_set_pd(-1., -2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_rcp14_pd() { + let a = _mm512_set1_pd(3.); + let r = _mm512_rcp14_pd(a); + let e = _mm512_set1_pd(0.3333320617675781); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_rcp14_pd() { + let a = _mm512_set1_pd(3.); + let r = _mm512_mask_rcp14_pd(a, 0, a); + assert_eq_m512d(r, a); + let r = _mm512_mask_rcp14_pd(a, 0b11110000, a); + #[rustfmt::skip] + let e = _mm512_setr_pd( + 3., 3., 3., 3., + 0.3333320617675781, 0.3333320617675781, 0.3333320617675781, 0.3333320617675781, + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_rcp14_pd() { + let a = _mm512_set1_pd(3.); + let r = _mm512_maskz_rcp14_pd(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_rcp14_pd(0b11110000, a); + #[rustfmt::skip] + let e = _mm512_setr_pd( + 0., 0., 0., 0., + 0.3333320617675781, 0.3333320617675781, 0.3333320617675781, 0.3333320617675781, + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_rcp14_pd() { + let a = _mm256_set1_pd(3.); + let r = _mm256_rcp14_pd(a); + let e = _mm256_set1_pd(0.3333320617675781); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_rcp14_pd() { + let a = _mm256_set1_pd(3.); + let r = _mm256_mask_rcp14_pd(a, 0, a); + assert_eq_m256d(r, a); + let r = _mm256_mask_rcp14_pd(a, 0b00001111, a); + let e = _mm256_set1_pd(0.3333320617675781); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_rcp14_pd() { + let a = _mm256_set1_pd(3.); + let r = _mm256_maskz_rcp14_pd(0, a); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_rcp14_pd(0b00001111, a); + let e = _mm256_set1_pd(0.3333320617675781); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_rcp14_pd() { + let a = _mm_set1_pd(3.); + let r = _mm_rcp14_pd(a); + let e = _mm_set1_pd(0.3333320617675781); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_rcp14_pd() { + let a = _mm_set1_pd(3.); + let r = _mm_mask_rcp14_pd(a, 0, a); + assert_eq_m128d(r, a); + let r = _mm_mask_rcp14_pd(a, 0b00000011, a); + let e = _mm_set1_pd(0.3333320617675781); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_rcp14_pd() { + let a = _mm_set1_pd(3.); + let r = _mm_maskz_rcp14_pd(0, a); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_rcp14_pd(0b00000011, a); + let e = _mm_set1_pd(0.3333320617675781); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_rsqrt14_pd() { + let a = _mm512_set1_pd(3.); + let r = _mm512_rsqrt14_pd(a); + let e = _mm512_set1_pd(0.5773391723632813); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_rsqrt14_pd() { + let a = _mm512_set1_pd(3.); + let r = _mm512_mask_rsqrt14_pd(a, 0, a); + assert_eq_m512d(r, a); + let r = _mm512_mask_rsqrt14_pd(a, 0b11110000, a); + #[rustfmt::skip] + let e = _mm512_setr_pd( + 3., 3., 3., 3., + 0.5773391723632813, 0.5773391723632813, 0.5773391723632813, 0.5773391723632813, + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_rsqrt14_pd() { + let a = _mm512_set1_pd(3.); + let r = _mm512_maskz_rsqrt14_pd(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_rsqrt14_pd(0b11110000, a); + #[rustfmt::skip] + let e = _mm512_setr_pd( + 0., 0., 0., 0., + 0.5773391723632813, 0.5773391723632813, 0.5773391723632813, 0.5773391723632813, + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_rsqrt14_pd() { + let a = _mm256_set1_pd(3.); + let r = _mm256_mask_rsqrt14_pd(a, 0, a); + assert_eq_m256d(r, a); + let r = _mm256_mask_rsqrt14_pd(a, 0b00001111, a); + let e = _mm256_set1_pd(0.5773391723632813); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_rsqrt14_pd() { + let a = _mm256_set1_pd(3.); + let r = _mm256_maskz_rsqrt14_pd(0, a); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_rsqrt14_pd(0b00001111, a); + let e = _mm256_set1_pd(0.5773391723632813); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_rsqrt14_pd() { + let a = _mm_set1_pd(3.); + let r = _mm_mask_rsqrt14_pd(a, 0, a); + assert_eq_m128d(r, a); + let r = _mm_mask_rsqrt14_pd(a, 0b00000011, a); + let e = _mm_set1_pd(0.5773391723632813); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_rsqrt14_pd() { + let a = _mm_set1_pd(3.); + let r = _mm_maskz_rsqrt14_pd(0, a); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_rsqrt14_pd(0b00000011, a); + let e = _mm_set1_pd(0.5773391723632813); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_getexp_pd() { + let a = _mm512_set1_pd(3.); + let r = _mm512_getexp_pd(a); + let e = _mm512_set1_pd(1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_getexp_pd() { + let a = _mm512_set1_pd(3.); + let r = _mm512_mask_getexp_pd(a, 0, a); + assert_eq_m512d(r, a); + let r = _mm512_mask_getexp_pd(a, 0b11110000, a); + let e = _mm512_setr_pd(3., 3., 3., 3., 1., 1., 1., 1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_getexp_pd() { + let a = _mm512_set1_pd(3.); + let r = _mm512_maskz_getexp_pd(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_getexp_pd(0b11110000, a); + let e = _mm512_setr_pd(0., 0., 0., 0., 1., 1., 1., 1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_getexp_pd() { + let a = _mm256_set1_pd(3.); + let r = _mm256_getexp_pd(a); + let e = _mm256_set1_pd(1.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_getexp_pd() { + let a = _mm256_set1_pd(3.); + let r = _mm256_mask_getexp_pd(a, 0, a); + assert_eq_m256d(r, a); + let r = _mm256_mask_getexp_pd(a, 0b00001111, a); + let e = _mm256_set1_pd(1.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_getexp_pd() { + let a = _mm256_set1_pd(3.); + let r = _mm256_maskz_getexp_pd(0, a); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_getexp_pd(0b00001111, a); + let e = _mm256_set1_pd(1.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_getexp_pd() { + let a = _mm_set1_pd(3.); + let r = _mm_getexp_pd(a); + let e = _mm_set1_pd(1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_getexp_pd() { + let a = _mm_set1_pd(3.); + let r = _mm_mask_getexp_pd(a, 0, a); + assert_eq_m128d(r, a); + let r = _mm_mask_getexp_pd(a, 0b00000011, a); + let e = _mm_set1_pd(1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_getexp_pd() { + let a = _mm_set1_pd(3.); + let r = _mm_maskz_getexp_pd(0, a); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_getexp_pd(0b00000011, a); + let e = _mm_set1_pd(1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_roundscale_pd() { + let a = _mm512_set1_pd(1.1); + let r = _mm512_roundscale_pd::<0b00_00_00_00>(a); + let e = _mm512_set1_pd(1.0); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_roundscale_pd() { + let a = _mm512_set1_pd(1.1); + let r = _mm512_mask_roundscale_pd::<0b00_00_00_00>(a, 0, a); + let e = _mm512_set1_pd(1.1); + assert_eq_m512d(r, e); + let r = _mm512_mask_roundscale_pd::<0b00_00_00_00>(a, 0b11111111, a); + let e = _mm512_set1_pd(1.0); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_roundscale_pd() { + let a = _mm512_set1_pd(1.1); + let r = _mm512_maskz_roundscale_pd::<0b00_00_00_00>(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_roundscale_pd::<0b00_00_00_00>(0b11111111, a); + let e = _mm512_set1_pd(1.0); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_roundscale_pd() { + let a = _mm256_set1_pd(1.1); + let r = _mm256_roundscale_pd::<0b00_00_00_00>(a); + let e = _mm256_set1_pd(1.0); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_roundscale_pd() { + let a = _mm256_set1_pd(1.1); + let r = _mm256_mask_roundscale_pd::<0b00_00_00_00>(a, 0, a); + assert_eq_m256d(r, a); + let r = _mm256_mask_roundscale_pd::<0b00_00_00_00>(a, 0b00001111, a); + let e = _mm256_set1_pd(1.0); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_roundscale_pd() { + let a = _mm256_set1_pd(1.1); + let r = _mm256_maskz_roundscale_pd::<0b00_00_00_00>(0, a); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_roundscale_pd::<0b00_00_00_00>(0b00001111, a); + let e = _mm256_set1_pd(1.0); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_roundscale_pd() { + let a = _mm_set1_pd(1.1); + let r = _mm_roundscale_pd::<0b00_00_00_00>(a); + let e = _mm_set1_pd(1.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_roundscale_pd() { + let a = _mm_set1_pd(1.1); + let r = _mm_mask_roundscale_pd::<0b00_00_00_00>(a, 0, a); + let e = _mm_set1_pd(1.1); + assert_eq_m128d(r, e); + let r = _mm_mask_roundscale_pd::<0b00_00_00_00>(a, 0b00000011, a); + let e = _mm_set1_pd(1.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_roundscale_pd() { + let a = _mm_set1_pd(1.1); + let r = _mm_maskz_roundscale_pd::<0b00_00_00_00>(0, a); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_roundscale_pd::<0b00_00_00_00>(0b00000011, a); + let e = _mm_set1_pd(1.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_scalef_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_set1_pd(3.); + let r = _mm512_scalef_pd(a, b); + let e = _mm512_set1_pd(8.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_scalef_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_set1_pd(3.); + let r = _mm512_mask_scalef_pd(a, 0, a, b); + assert_eq_m512d(r, a); + let r = _mm512_mask_scalef_pd(a, 0b11110000, a, b); + let e = _mm512_set_pd(8., 8., 8., 8., 1., 1., 1., 1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_scalef_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_set1_pd(3.); + let r = _mm512_maskz_scalef_pd(0, a, b); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_scalef_pd(0b11110000, a, b); + let e = _mm512_set_pd(8., 8., 8., 8., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_scalef_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set1_pd(3.); + let r = _mm256_scalef_pd(a, b); + let e = _mm256_set1_pd(8.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_scalef_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set1_pd(3.); + let r = _mm256_mask_scalef_pd(a, 0, a, b); + assert_eq_m256d(r, a); + let r = _mm256_mask_scalef_pd(a, 0b00001111, a, b); + let e = _mm256_set1_pd(8.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_scalef_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set1_pd(3.); + let r = _mm256_maskz_scalef_pd(0, a, b); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_scalef_pd(0b00001111, a, b); + let e = _mm256_set1_pd(8.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_scalef_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(3.); + let r = _mm_scalef_pd(a, b); + let e = _mm_set1_pd(8.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_scalef_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(3.); + let r = _mm_mask_scalef_pd(a, 0, a, b); + assert_eq_m128d(r, a); + let r = _mm_mask_scalef_pd(a, 0b00000011, a, b); + let e = _mm_set1_pd(8.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_scalef_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(3.); + let r = _mm_maskz_scalef_pd(0, a, b); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_scalef_pd(0b00000011, a, b); + let e = _mm_set1_pd(8.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fixupimm_pd() { + let a = _mm512_set1_pd(f64::NAN); + let b = _mm512_set1_pd(f64::MAX); + let c = _mm512_set1_epi64(i32::MAX as i64); + let r = _mm512_fixupimm_pd::<5>(a, b, c); + let e = _mm512_set1_pd(0.0); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fixupimm_pd() { + let a = _mm512_set_pd(f64::NAN, f64::NAN, f64::NAN, f64::NAN, 1., 1., 1., 1.); + let b = _mm512_set1_pd(f64::MAX); + let c = _mm512_set1_epi64(i32::MAX as i64); + let r = _mm512_mask_fixupimm_pd::<5>(a, 0b11110000, b, c); + let e = _mm512_set_pd(0., 0., 0., 0., 1., 1., 1., 1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fixupimm_pd() { + let a = _mm512_set_pd(f64::NAN, f64::NAN, f64::NAN, f64::NAN, 1., 1., 1., 1.); + let b = _mm512_set1_pd(f64::MAX); + let c = _mm512_set1_epi64(i32::MAX as i64); + let r = _mm512_maskz_fixupimm_pd::<5>(0b11110000, a, b, c); + let e = _mm512_set_pd(0., 0., 0., 0., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_fixupimm_pd() { + let a = _mm256_set1_pd(f64::NAN); + let b = _mm256_set1_pd(f64::MAX); + let c = _mm256_set1_epi64x(i32::MAX as i64); + let r = _mm256_fixupimm_pd::<5>(a, b, c); + let e = _mm256_set1_pd(0.0); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_fixupimm_pd() { + let a = _mm256_set1_pd(f64::NAN); + let b = _mm256_set1_pd(f64::MAX); + let c = _mm256_set1_epi64x(i32::MAX as i64); + let r = _mm256_mask_fixupimm_pd::<5>(a, 0b00001111, b, c); + let e = _mm256_set1_pd(0.0); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_fixupimm_pd() { + let a = _mm256_set1_pd(f64::NAN); + let b = _mm256_set1_pd(f64::MAX); + let c = _mm256_set1_epi64x(i32::MAX as i64); + let r = _mm256_maskz_fixupimm_pd::<5>(0b00001111, a, b, c); + let e = _mm256_set1_pd(0.0); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_fixupimm_pd() { + let a = _mm_set1_pd(f64::NAN); + let b = _mm_set1_pd(f64::MAX); + let c = _mm_set1_epi64x(i32::MAX as i64); + let r = _mm_fixupimm_pd::<5>(a, b, c); + let e = _mm_set1_pd(0.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_fixupimm_pd() { + let a = _mm_set1_pd(f64::NAN); + let b = _mm_set1_pd(f64::MAX); + let c = _mm_set1_epi64x(i32::MAX as i64); + let r = _mm_mask_fixupimm_pd::<5>(a, 0b00000011, b, c); + let e = _mm_set1_pd(0.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_fixupimm_pd() { + let a = _mm_set1_pd(f64::NAN); + let b = _mm_set1_pd(f64::MAX); + let c = _mm_set1_epi64x(i32::MAX as i64); + let r = _mm_maskz_fixupimm_pd::<5>(0b00000011, a, b, c); + let e = _mm_set1_pd(0.0); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_ternarylogic_epi64() { + let a = _mm512_set1_epi64(1 << 2); + let b = _mm512_set1_epi64(1 << 1); + let c = _mm512_set1_epi64(1 << 0); + let r = _mm512_ternarylogic_epi64::<8>(a, b, c); + let e = _mm512_set1_epi64(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_ternarylogic_epi64() { + let src = _mm512_set1_epi64(1 << 2); + let a = _mm512_set1_epi64(1 << 1); + let b = _mm512_set1_epi64(1 << 0); + let r = _mm512_mask_ternarylogic_epi64::<8>(src, 0, a, b); + assert_eq_m512i(r, src); + let r = _mm512_mask_ternarylogic_epi64::<8>(src, 0b11111111, a, b); + let e = _mm512_set1_epi64(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_ternarylogic_epi64() { + let a = _mm512_set1_epi64(1 << 2); + let b = _mm512_set1_epi64(1 << 1); + let c = _mm512_set1_epi64(1 << 0); + let r = _mm512_maskz_ternarylogic_epi64::<8>(0, a, b, c); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_ternarylogic_epi64::<8>(0b11111111, a, b, c); + let e = _mm512_set1_epi64(0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_ternarylogic_epi64() { + let a = _mm256_set1_epi64x(1 << 2); + let b = _mm256_set1_epi64x(1 << 1); + let c = _mm256_set1_epi64x(1 << 0); + let r = _mm256_ternarylogic_epi64::<8>(a, b, c); + let e = _mm256_set1_epi64x(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_ternarylogic_epi64() { + let src = _mm256_set1_epi64x(1 << 2); + let a = _mm256_set1_epi64x(1 << 1); + let b = _mm256_set1_epi64x(1 << 0); + let r = _mm256_mask_ternarylogic_epi64::<8>(src, 0, a, b); + assert_eq_m256i(r, src); + let r = _mm256_mask_ternarylogic_epi64::<8>(src, 0b00001111, a, b); + let e = _mm256_set1_epi64x(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_ternarylogic_epi64() { + let a = _mm256_set1_epi64x(1 << 2); + let b = _mm256_set1_epi64x(1 << 1); + let c = _mm256_set1_epi64x(1 << 0); + let r = _mm256_maskz_ternarylogic_epi64::<9>(0, a, b, c); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_ternarylogic_epi64::<8>(0b00001111, a, b, c); + let e = _mm256_set1_epi64x(0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_ternarylogic_epi64() { + let a = _mm_set1_epi64x(1 << 2); + let b = _mm_set1_epi64x(1 << 1); + let c = _mm_set1_epi64x(1 << 0); + let r = _mm_ternarylogic_epi64::<8>(a, b, c); + let e = _mm_set1_epi64x(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_ternarylogic_epi64() { + let src = _mm_set1_epi64x(1 << 2); + let a = _mm_set1_epi64x(1 << 1); + let b = _mm_set1_epi64x(1 << 0); + let r = _mm_mask_ternarylogic_epi64::<8>(src, 0, a, b); + assert_eq_m128i(r, src); + let r = _mm_mask_ternarylogic_epi64::<8>(src, 0b00000011, a, b); + let e = _mm_set1_epi64x(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_ternarylogic_epi64() { + let a = _mm_set1_epi64x(1 << 2); + let b = _mm_set1_epi64x(1 << 1); + let c = _mm_set1_epi64x(1 << 0); + let r = _mm_maskz_ternarylogic_epi64::<9>(0, a, b, c); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_ternarylogic_epi64::<8>(0b00000011, a, b, c); + let e = _mm_set1_epi64x(0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_getmant_pd() { + let a = _mm512_set1_pd(10.); + let r = _mm512_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a); + let e = _mm512_set1_pd(1.25); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_getmant_pd() { + let a = _mm512_set1_pd(10.); + let r = _mm512_mask_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0, a); + assert_eq_m512d(r, a); + let r = _mm512_mask_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0b11110000, a); + let e = _mm512_setr_pd(10., 10., 10., 10., 1.25, 1.25, 1.25, 1.25); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_getmant_pd() { + let a = _mm512_set1_pd(10.); + let r = _mm512_maskz_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0b11110000, a); + let e = _mm512_setr_pd(0., 0., 0., 0., 1.25, 1.25, 1.25, 1.25); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_getmant_pd() { + let a = _mm256_set1_pd(10.); + let r = _mm256_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a); + let e = _mm256_set1_pd(1.25); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_getmant_pd() { + let a = _mm256_set1_pd(10.); + let r = _mm256_mask_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0, a); + assert_eq_m256d(r, a); + let r = _mm256_mask_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0b00001111, a); + let e = _mm256_set1_pd(1.25); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_getmant_pd() { + let a = _mm256_set1_pd(10.); + let r = _mm256_maskz_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0, a); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0b00001111, a); + let e = _mm256_set1_pd(1.25); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_getmant_pd() { + let a = _mm_set1_pd(10.); + let r = _mm_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a); + let e = _mm_set1_pd(1.25); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_getmant_pd() { + let a = _mm_set1_pd(10.); + let r = _mm_mask_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0, a); + assert_eq_m128d(r, a); + let r = _mm_mask_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0b00000011, a); + let e = _mm_set1_pd(1.25); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_getmant_pd() { + let a = _mm_set1_pd(10.); + let r = _mm_maskz_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0, a); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0b00000011, a); + let e = _mm_set1_pd(1.25); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtps_pd() { + let a = _mm256_setr_ps(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_cvtps_pd(a); + let e = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtps_pd() { + let a = _mm256_setr_ps(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let src = _mm512_set1_pd(0.); + let r = _mm512_mask_cvtps_pd(src, 0, a); + assert_eq_m512d(r, src); + let r = _mm512_mask_cvtps_pd(src, 0b00001111, a); + let e = _mm512_setr_pd(0., -1.5, 2., -3.5, 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtps_pd() { + let a = _mm256_setr_ps(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_maskz_cvtps_pd(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_cvtps_pd(0b00001111, a); + let e = _mm512_setr_pd(0., -1.5, 2., -3.5, 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtpslo_pd() { + let v2 = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 100., 100., 100., 100., 100., 100., 100., 100., + ); + let r = _mm512_cvtpslo_pd(v2); + let e = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtpslo_pd() { + let v2 = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 100., 100., 100., 100., 100., 100., 100., 100., + ); + let src = _mm512_set1_pd(0.); + let r = _mm512_mask_cvtpslo_pd(src, 0, v2); + assert_eq_m512d(r, src); + let r = _mm512_mask_cvtpslo_pd(src, 0b00001111, v2); + let e = _mm512_setr_pd(0., -1.5, 2., -3.5, 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtpd_ps() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_cvtpd_ps(a); + let e = _mm256_setr_ps(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtpd_ps() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let src = _mm256_set1_ps(0.); + let r = _mm512_mask_cvtpd_ps(src, 0, a); + assert_eq_m256(r, src); + let r = _mm512_mask_cvtpd_ps(src, 0b00001111, a); + let e = _mm256_setr_ps(0., -1.5, 2., -3.5, 0., 0., 0., 0.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtpd_ps() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_maskz_cvtpd_ps(0, a); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm512_maskz_cvtpd_ps(0b00001111, a); + let e = _mm256_setr_ps(0., -1.5, 2., -3.5, 0., 0., 0., 0.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtpd_ps() { + let a = _mm256_set_pd(4., -5.5, 6., -7.5); + let src = _mm_set1_ps(0.); + let r = _mm256_mask_cvtpd_ps(src, 0, a); + assert_eq_m128(r, src); + let r = _mm256_mask_cvtpd_ps(src, 0b00001111, a); + let e = _mm_set_ps(4., -5.5, 6., -7.5); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtpd_ps() { + let a = _mm256_set_pd(4., -5.5, 6., -7.5); + let r = _mm256_maskz_cvtpd_ps(0, a); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm256_maskz_cvtpd_ps(0b00001111, a); + let e = _mm_set_ps(4., -5.5, 6., -7.5); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtpd_ps() { + let a = _mm_set_pd(6., -7.5); + let src = _mm_set1_ps(0.); + let r = _mm_mask_cvtpd_ps(src, 0, a); + assert_eq_m128(r, src); + let r = _mm_mask_cvtpd_ps(src, 0b00000011, a); + let e = _mm_set_ps(0., 0., 6., -7.5); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtpd_ps() { + let a = _mm_set_pd(6., -7.5); + let r = _mm_maskz_cvtpd_ps(0, a); + assert_eq_m128(r, _mm_setzero_ps()); + let r = _mm_maskz_cvtpd_ps(0b00000011, a); + let e = _mm_set_ps(0., 0., 6., -7.5); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtpd_epi32() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_cvtpd_epi32(a); + let e = _mm256_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtpd_epi32() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let src = _mm256_set1_epi32(0); + let r = _mm512_mask_cvtpd_epi32(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm512_mask_cvtpd_epi32(src, 0b11111111, a); + let e = _mm256_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtpd_epi32() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_maskz_cvtpd_epi32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm512_maskz_cvtpd_epi32(0b11111111, a); + let e = _mm256_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtpd_epi32() { + let a = _mm256_set_pd(4., -5.5, 6., -7.5); + let src = _mm_set1_epi32(0); + let r = _mm256_mask_cvtpd_epi32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvtpd_epi32(src, 0b00001111, a); + let e = _mm_set_epi32(4, -6, 6, -8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtpd_epi32() { + let a = _mm256_set_pd(4., -5.5, 6., -7.5); + let r = _mm256_maskz_cvtpd_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvtpd_epi32(0b00001111, a); + let e = _mm_set_epi32(4, -6, 6, -8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtpd_epi32() { + let a = _mm_set_pd(6., -7.5); + let src = _mm_set1_epi32(0); + let r = _mm_mask_cvtpd_epi32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtpd_epi32(src, 0b00000011, a); + let e = _mm_set_epi32(0, 0, 6, -8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtpd_epi32() { + let a = _mm_set_pd(6., -7.5); + let r = _mm_maskz_cvtpd_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtpd_epi32(0b00000011, a); + let e = _mm_set_epi32(0, 0, 6, -8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtpd_epu32() { + let a = _mm512_setr_pd(0., 1.5, 2., 3.5, 4., 5.5, 6., 7.5); + let r = _mm512_cvtpd_epu32(a); + let e = _mm256_setr_epi32(0, 2, 2, 4, 4, 6, 6, 8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtpd_epu32() { + let a = _mm512_setr_pd(0., 1.5, 2., 3.5, 4., 5.5, 6., 7.5); + let src = _mm256_set1_epi32(0); + let r = _mm512_mask_cvtpd_epu32(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm512_mask_cvtpd_epu32(src, 0b11111111, a); + let e = _mm256_setr_epi32(0, 2, 2, 4, 4, 6, 6, 8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtpd_epu32() { + let a = _mm512_setr_pd(0., 1.5, 2., 3.5, 4., 5.5, 6., 7.5); + let r = _mm512_maskz_cvtpd_epu32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm512_maskz_cvtpd_epu32(0b11111111, a); + let e = _mm256_setr_epi32(0, 2, 2, 4, 4, 6, 6, 8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cvtpd_epu32() { + let a = _mm256_set_pd(4., 5.5, 6., 7.5); + let r = _mm256_cvtpd_epu32(a); + let e = _mm_set_epi32(4, 6, 6, 8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtpd_epu32() { + let a = _mm256_set_pd(4., 5.5, 6., 7.5); + let src = _mm_set1_epi32(0); + let r = _mm256_mask_cvtpd_epu32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvtpd_epu32(src, 0b00001111, a); + let e = _mm_set_epi32(4, 6, 6, 8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtpd_epu32() { + let a = _mm256_set_pd(4., 5.5, 6., 7.5); + let r = _mm256_maskz_cvtpd_epu32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvtpd_epu32(0b00001111, a); + let e = _mm_set_epi32(4, 6, 6, 8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cvtpd_epu32() { + let a = _mm_set_pd(6., 7.5); + let r = _mm_cvtpd_epu32(a); + let e = _mm_set_epi32(0, 0, 6, 8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtpd_epu32() { + let a = _mm_set_pd(6., 7.5); + let src = _mm_set1_epi32(0); + let r = _mm_mask_cvtpd_epu32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtpd_epu32(src, 0b00000011, a); + let e = _mm_set_epi32(0, 0, 6, 8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtpd_epu32() { + let a = _mm_set_pd(6., 7.5); + let r = _mm_maskz_cvtpd_epu32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtpd_epu32(0b00000011, a); + let e = _mm_set_epi32(0, 0, 6, 8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtpd_pslo() { + let v2 = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_cvtpd_pslo(v2); + let e = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtpd_pslo() { + let v2 = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let src = _mm512_set1_ps(0.); + let r = _mm512_mask_cvtpd_pslo(src, 0, v2); + assert_eq_m512(r, src); + let r = _mm512_mask_cvtpd_pslo(src, 0b00001111, v2); + let e = _mm512_setr_ps( + 0., -1.5, 2., -3.5, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtepi8_epi64() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_cvtepi8_epi64(a); + let e = _mm512_set_epi64(8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepi8_epi64() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm512_set1_epi64(-1); + let r = _mm512_mask_cvtepi8_epi64(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_cvtepi8_epi64(src, 0b00001111, a); + let e = _mm512_set_epi64(-1, -1, -1, -1, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtepi8_epi64() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_cvtepi8_epi64(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_cvtepi8_epi64(0b00001111, a); + let e = _mm512_set_epi64(0, 0, 0, 0, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepi8_epi64() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm256_set1_epi64x(-1); + let r = _mm256_mask_cvtepi8_epi64(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_cvtepi8_epi64(src, 0b00001111, a); + let e = _mm256_set_epi64x(12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtepi8_epi64() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm256_maskz_cvtepi8_epi64(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_cvtepi8_epi64(0b00001111, a); + let e = _mm256_set_epi64x(12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepi8_epi64() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm_set1_epi64x(-1); + let r = _mm_mask_cvtepi8_epi64(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtepi8_epi64(src, 0b00000011, a); + let e = _mm_set_epi64x(14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtepi8_epi64() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm_maskz_cvtepi8_epi64(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtepi8_epi64(0b00000011, a); + let e = _mm_set_epi64x(14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtepu8_epi64() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_cvtepu8_epi64(a); + let e = _mm512_set_epi64(8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepu8_epi64() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm512_set1_epi64(-1); + let r = _mm512_mask_cvtepu8_epi64(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_cvtepu8_epi64(src, 0b00001111, a); + let e = _mm512_set_epi64(-1, -1, -1, -1, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtepu8_epi64() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_cvtepu8_epi64(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_cvtepu8_epi64(0b00001111, a); + let e = _mm512_set_epi64(0, 0, 0, 0, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepu8_epi64() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm256_set1_epi64x(-1); + let r = _mm256_mask_cvtepu8_epi64(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_cvtepu8_epi64(src, 0b00001111, a); + let e = _mm256_set_epi64x(12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtepu8_epi64() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm256_maskz_cvtepu8_epi64(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_cvtepu8_epi64(0b00001111, a); + let e = _mm256_set_epi64x(12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepu8_epi64() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm_set1_epi64x(-1); + let r = _mm_mask_cvtepu8_epi64(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtepu8_epi64(src, 0b00000011, a); + let e = _mm_set_epi64x(14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtepu8_epi64() { + let a = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm_maskz_cvtepu8_epi64(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtepu8_epi64(0b00000011, a); + let e = _mm_set_epi64x(14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtepi16_epi64() { + let a = _mm_set_epi16(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_cvtepi16_epi64(a); + let e = _mm512_set_epi64(8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepi16_epi64() { + let a = _mm_set_epi16(8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm512_set1_epi64(-1); + let r = _mm512_mask_cvtepi16_epi64(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_cvtepi16_epi64(src, 0b00001111, a); + let e = _mm512_set_epi64(-1, -1, -1, -1, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtepi16_epi64() { + let a = _mm_set_epi16(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_cvtepi16_epi64(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_cvtepi16_epi64(0b00001111, a); + let e = _mm512_set_epi64(0, 0, 0, 0, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepi16_epi64() { + let a = _mm_set_epi16(8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm256_set1_epi64x(-1); + let r = _mm256_mask_cvtepi16_epi64(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_cvtepi16_epi64(src, 0b00001111, a); + let e = _mm256_set_epi64x(12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtepi16_epi64() { + let a = _mm_set_epi16(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm256_maskz_cvtepi16_epi64(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_cvtepi16_epi64(0b00001111, a); + let e = _mm256_set_epi64x(12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepi16_epi64() { + let a = _mm_set_epi16(8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm_set1_epi64x(-1); + let r = _mm_mask_cvtepi16_epi64(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtepi16_epi64(src, 0b00000011, a); + let e = _mm_set_epi64x(14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtepi16_epi64() { + let a = _mm_set_epi16(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm_maskz_cvtepi16_epi64(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtepi16_epi64(0b00000011, a); + let e = _mm_set_epi64x(14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtepu16_epi64() { + let a = _mm_set_epi16(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_cvtepu16_epi64(a); + let e = _mm512_set_epi64(8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepu16_epi64() { + let a = _mm_set_epi16(8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm512_set1_epi64(-1); + let r = _mm512_mask_cvtepu16_epi64(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_cvtepu16_epi64(src, 0b00001111, a); + let e = _mm512_set_epi64(-1, -1, -1, -1, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtepu16_epi64() { + let a = _mm_set_epi16(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_cvtepu16_epi64(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_cvtepu16_epi64(0b00001111, a); + let e = _mm512_set_epi64(0, 0, 0, 0, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepu16_epi64() { + let a = _mm_set_epi16(8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm256_set1_epi64x(-1); + let r = _mm256_mask_cvtepu16_epi64(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_cvtepu16_epi64(src, 0b00001111, a); + let e = _mm256_set_epi64x(12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtepu16_epi64() { + let a = _mm_set_epi16(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm256_maskz_cvtepu16_epi64(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_cvtepu16_epi64(0b00001111, a); + let e = _mm256_set_epi64x(12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepu16_epi64() { + let a = _mm_set_epi16(8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm_set1_epi64x(-1); + let r = _mm_mask_cvtepu16_epi64(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtepu16_epi64(src, 0b00000011, a); + let e = _mm_set_epi64x(14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtepu16_epi64() { + let a = _mm_set_epi16(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm_maskz_cvtepu16_epi64(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtepu16_epi64(0b00000011, a); + let e = _mm_set_epi64x(14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtepi32_epi64() { + let a = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_cvtepi32_epi64(a); + let e = _mm512_set_epi64(8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepi32_epi64() { + let a = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm512_set1_epi64(-1); + let r = _mm512_mask_cvtepi32_epi64(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_cvtepi32_epi64(src, 0b00001111, a); + let e = _mm512_set_epi64(-1, -1, -1, -1, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtepi32_epi64() { + let a = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_cvtepi32_epi64(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_cvtepi32_epi64(0b00001111, a); + let e = _mm512_set_epi64(0, 0, 0, 0, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepi32_epi64() { + let a = _mm_set_epi32(8, 9, 10, 11); + let src = _mm256_set1_epi64x(-1); + let r = _mm256_mask_cvtepi32_epi64(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_cvtepi32_epi64(src, 0b00001111, a); + let e = _mm256_set_epi64x(8, 9, 10, 11); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtepi32_epi64() { + let a = _mm_set_epi32(8, 9, 10, 11); + let r = _mm256_maskz_cvtepi32_epi64(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_cvtepi32_epi64(0b00001111, a); + let e = _mm256_set_epi64x(8, 9, 10, 11); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepi32_epi64() { + let a = _mm_set_epi32(8, 9, 10, 11); + let src = _mm_set1_epi64x(0); + let r = _mm_mask_cvtepi32_epi64(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtepi32_epi64(src, 0b00000011, a); + let e = _mm_set_epi64x(10, 11); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtepi32_epi64() { + let a = _mm_set_epi32(8, 9, 10, 11); + let r = _mm_maskz_cvtepi32_epi64(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtepi32_epi64(0b00000011, a); + let e = _mm_set_epi64x(10, 11); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtepu32_epi64() { + let a = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_cvtepu32_epi64(a); + let e = _mm512_set_epi64(8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepu32_epi64() { + let a = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm512_set1_epi64(-1); + let r = _mm512_mask_cvtepu32_epi64(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_cvtepu32_epi64(src, 0b00001111, a); + let e = _mm512_set_epi64(-1, -1, -1, -1, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtepu32_epi64() { + let a = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_cvtepu32_epi64(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_cvtepu32_epi64(0b00001111, a); + let e = _mm512_set_epi64(0, 0, 0, 0, 12, 13, 14, 15); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepu32_epi64() { + let a = _mm_set_epi32(12, 13, 14, 15); + let src = _mm256_set1_epi64x(-1); + let r = _mm256_mask_cvtepu32_epi64(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_cvtepu32_epi64(src, 0b00001111, a); + let e = _mm256_set_epi64x(12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtepu32_epi64() { + let a = _mm_set_epi32(12, 13, 14, 15); + let r = _mm256_maskz_cvtepu32_epi64(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_cvtepu32_epi64(0b00001111, a); + let e = _mm256_set_epi64x(12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepu32_epi64() { + let a = _mm_set_epi32(12, 13, 14, 15); + let src = _mm_set1_epi64x(-1); + let r = _mm_mask_cvtepu32_epi64(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtepu32_epi64(src, 0b00000011, a); + let e = _mm_set_epi64x(14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtepu32_epi64() { + let a = _mm_set_epi32(12, 13, 14, 15); + let r = _mm_maskz_cvtepu32_epi64(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtepu32_epi64(0b00000011, a); + let e = _mm_set_epi64x(14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtepi32_pd() { + let a = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_cvtepi32_pd(a); + let e = _mm512_set_pd(8., 9., 10., 11., 12., 13., 14., 15.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepi32_pd() { + let a = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm512_set1_pd(-1.); + let r = _mm512_mask_cvtepi32_pd(src, 0, a); + assert_eq_m512d(r, src); + let r = _mm512_mask_cvtepi32_pd(src, 0b00001111, a); + let e = _mm512_set_pd(-1., -1., -1., -1., 12., 13., 14., 15.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtepi32_pd() { + let a = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_cvtepi32_pd(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_cvtepi32_pd(0b00001111, a); + let e = _mm512_set_pd(0., 0., 0., 0., 12., 13., 14., 15.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepi32_pd() { + let a = _mm_set_epi32(12, 13, 14, 15); + let src = _mm256_set1_pd(-1.); + let r = _mm256_mask_cvtepi32_pd(src, 0, a); + assert_eq_m256d(r, src); + let r = _mm256_mask_cvtepi32_pd(src, 0b00001111, a); + let e = _mm256_set_pd(12., 13., 14., 15.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtepi32_pd() { + let a = _mm_set_epi32(12, 13, 14, 15); + let r = _mm256_maskz_cvtepi32_pd(0, a); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_cvtepi32_pd(0b00001111, a); + let e = _mm256_set_pd(12., 13., 14., 15.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepi32_pd() { + let a = _mm_set_epi32(12, 13, 14, 15); + let src = _mm_set1_pd(-1.); + let r = _mm_mask_cvtepi32_pd(src, 0, a); + assert_eq_m128d(r, src); + let r = _mm_mask_cvtepi32_pd(src, 0b00000011, a); + let e = _mm_set_pd(14., 15.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtepi32_pd() { + let a = _mm_set_epi32(12, 13, 14, 15); + let r = _mm_maskz_cvtepi32_pd(0, a); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_cvtepi32_pd(0b00000011, a); + let e = _mm_set_pd(14., 15.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtepu32_pd() { + let a = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_cvtepu32_pd(a); + let e = _mm512_set_pd(8., 9., 10., 11., 12., 13., 14., 15.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepu32_pd() { + let a = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm512_set1_pd(-1.); + let r = _mm512_mask_cvtepu32_pd(src, 0, a); + assert_eq_m512d(r, src); + let r = _mm512_mask_cvtepu32_pd(src, 0b00001111, a); + let e = _mm512_set_pd(-1., -1., -1., -1., 12., 13., 14., 15.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtepu32_pd() { + let a = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_cvtepu32_pd(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_cvtepu32_pd(0b00001111, a); + let e = _mm512_set_pd(0., 0., 0., 0., 12., 13., 14., 15.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cvtepu32_pd() { + let a = _mm_set_epi32(12, 13, 14, 15); + let r = _mm256_cvtepu32_pd(a); + let e = _mm256_set_pd(12., 13., 14., 15.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepu32_pd() { + let a = _mm_set_epi32(12, 13, 14, 15); + let src = _mm256_set1_pd(-1.); + let r = _mm256_mask_cvtepu32_pd(src, 0, a); + assert_eq_m256d(r, src); + let r = _mm256_mask_cvtepu32_pd(src, 0b00001111, a); + let e = _mm256_set_pd(12., 13., 14., 15.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtepu32_pd() { + let a = _mm_set_epi32(12, 13, 14, 15); + let r = _mm256_maskz_cvtepu32_pd(0, a); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_cvtepu32_pd(0b00001111, a); + let e = _mm256_set_pd(12., 13., 14., 15.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cvtepu32_pd() { + let a = _mm_set_epi32(12, 13, 14, 15); + let r = _mm_cvtepu32_pd(a); + let e = _mm_set_pd(14., 15.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepu32_pd() { + let a = _mm_set_epi32(12, 13, 14, 15); + let src = _mm_set1_pd(-1.); + let r = _mm_mask_cvtepu32_pd(src, 0, a); + assert_eq_m128d(r, src); + let r = _mm_mask_cvtepu32_pd(src, 0b00000011, a); + let e = _mm_set_pd(14., 15.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtepu32_pd() { + let a = _mm_set_epi32(12, 13, 14, 15); + let r = _mm_maskz_cvtepu32_pd(0, a); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_cvtepu32_pd(0b00000011, a); + let e = _mm_set_pd(14., 15.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtepi32lo_pd() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_cvtepi32lo_pd(a); + let e = _mm512_set_pd(8., 9., 10., 11., 12., 13., 14., 15.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepi32lo_pd() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm512_set1_pd(-1.); + let r = _mm512_mask_cvtepi32lo_pd(src, 0, a); + assert_eq_m512d(r, src); + let r = _mm512_mask_cvtepi32lo_pd(src, 0b00001111, a); + let e = _mm512_set_pd(-1., -1., -1., -1., 12., 13., 14., 15.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtepu32lo_pd() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_cvtepu32lo_pd(a); + let e = _mm512_set_pd(8., 9., 10., 11., 12., 13., 14., 15.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepu32lo_pd() { + let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm512_set1_pd(-1.); + let r = _mm512_mask_cvtepu32lo_pd(src, 0, a); + assert_eq_m512d(r, src); + let r = _mm512_mask_cvtepu32lo_pd(src, 0b00001111, a); + let e = _mm512_set_pd(-1., -1., -1., -1., 12., 13., 14., 15.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtepi64_epi32() { + let a = _mm512_set_epi64(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_cvtepi64_epi32(a); + let e = _mm256_set_epi32(8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepi64_epi32() { + let a = _mm512_set_epi64(8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm256_set1_epi32(-1); + let r = _mm512_mask_cvtepi64_epi32(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm512_mask_cvtepi64_epi32(src, 0b00001111, a); + let e = _mm256_set_epi32(-1, -1, -1, -1, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtepi64_epi32() { + let a = _mm512_set_epi64(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_cvtepi64_epi32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm512_maskz_cvtepi64_epi32(0b00001111, a); + let e = _mm256_set_epi32(0, 0, 0, 0, 12, 13, 14, 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cvtepi64_epi32() { + let a = _mm256_set_epi64x(1, 2, 3, 4); + let r = _mm256_cvtepi64_epi32(a); + let e = _mm_set_epi32(1, 2, 3, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepi64_epi32() { + let a = _mm256_set_epi64x(1, 2, 3, 4); + let src = _mm_set1_epi32(0); + let r = _mm256_mask_cvtepi64_epi32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvtepi64_epi32(src, 0b00001111, a); + let e = _mm_set_epi32(1, 2, 3, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtepi64_epi32() { + let a = _mm256_set_epi64x(1, 2, 3, 4); + let r = _mm256_maskz_cvtepi64_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvtepi64_epi32(0b00001111, a); + let e = _mm_set_epi32(1, 2, 3, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cvtepi64_epi32() { + let a = _mm_set_epi64x(3, 4); + let r = _mm_cvtepi64_epi32(a); + let e = _mm_set_epi32(0, 0, 3, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepi64_epi32() { + let a = _mm_set_epi64x(3, 4); + let src = _mm_set1_epi32(0); + let r = _mm_mask_cvtepi64_epi32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtepi64_epi32(src, 0b00000011, a); + let e = _mm_set_epi32(0, 0, 3, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtepi64_epi32() { + let a = _mm_set_epi64x(3, 4); + let r = _mm_maskz_cvtepi64_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtepi64_epi32(0b00000011, a); + let e = _mm_set_epi32(0, 0, 3, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtepi64_epi16() { + let a = _mm512_set_epi64(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_cvtepi64_epi16(a); + let e = _mm_set_epi16(8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepi64_epi16() { + let a = _mm512_set_epi64(8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm_set1_epi16(-1); + let r = _mm512_mask_cvtepi64_epi16(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm512_mask_cvtepi64_epi16(src, 0b00001111, a); + let e = _mm_set_epi16(-1, -1, -1, -1, 12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtepi64_epi16() { + let a = _mm512_set_epi64(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_cvtepi64_epi16(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm512_maskz_cvtepi64_epi16(0b00001111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cvtepi64_epi16() { + let a = _mm256_set_epi64x(12, 13, 14, 15); + let r = _mm256_cvtepi64_epi16(a); + let e = _mm_set_epi16(0, 0, 0, 0, 12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepi64_epi16() { + let a = _mm256_set_epi64x(12, 13, 14, 15); + let src = _mm_set1_epi16(0); + let r = _mm256_mask_cvtepi64_epi16(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvtepi64_epi16(src, 0b11111111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtepi64_epi16() { + let a = _mm256_set_epi64x(12, 13, 14, 15); + let r = _mm256_maskz_cvtepi64_epi16(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvtepi64_epi16(0b11111111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cvtepi64_epi16() { + let a = _mm_set_epi64x(14, 15); + let r = _mm_cvtepi64_epi16(a); + let e = _mm_set_epi16(0, 0, 0, 0, 0, 0, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepi64_epi16() { + let a = _mm_set_epi64x(14, 15); + let src = _mm_set1_epi16(0); + let r = _mm_mask_cvtepi64_epi16(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtepi64_epi16(src, 0b11111111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 0, 0, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtepi64_epi16() { + let a = _mm_set_epi64x(14, 15); + let r = _mm_maskz_cvtepi64_epi16(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtepi64_epi16(0b11111111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 0, 0, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtepi64_epi8() { + let a = _mm512_set_epi64(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_cvtepi64_epi8(a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepi64_epi8() { + let a = _mm512_set_epi64(8, 9, 10, 11, 12, 13, 14, 15); + let src = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1); + let r = _mm512_mask_cvtepi64_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm512_mask_cvtepi64_epi8(src, 0b00001111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, 12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtepi64_epi8() { + let a = _mm512_set_epi64(8, 9, 10, 11, 12, 13, 14, 15); + let r = _mm512_maskz_cvtepi64_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm512_maskz_cvtepi64_epi8(0b00001111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cvtepi64_epi8() { + let a = _mm256_set_epi64x(12, 13, 14, 15); + let r = _mm256_cvtepi64_epi8(a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepi64_epi8() { + let a = _mm256_set_epi64x(12, 13, 14, 15); + let src = _mm_set1_epi8(0); + let r = _mm256_mask_cvtepi64_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvtepi64_epi8(src, 0b00001111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtepi64_epi8() { + let a = _mm256_set_epi64x(12, 13, 14, 15); + let r = _mm256_maskz_cvtepi64_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvtepi64_epi8(0b00001111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cvtepi64_epi8() { + let a = _mm_set_epi64x(14, 15); + let r = _mm_cvtepi64_epi8(a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepi64_epi8() { + let a = _mm_set_epi64x(14, 15); + let src = _mm_set1_epi8(0); + let r = _mm_mask_cvtepi64_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtepi64_epi8(src, 0b00000011, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtepi64_epi8() { + let a = _mm_set_epi64x(14, 15); + let r = _mm_maskz_cvtepi64_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtepi64_epi8(0b00000011, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtsepi64_epi32() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, i64::MIN, i64::MAX); + let r = _mm512_cvtsepi64_epi32(a); + let e = _mm256_set_epi32(0, 1, 2, 3, 4, 5, i32::MIN, i32::MAX); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtsepi64_epi32() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, i64::MIN, i64::MAX); + let src = _mm256_set1_epi32(-1); + let r = _mm512_mask_cvtsepi64_epi32(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm512_mask_cvtsepi64_epi32(src, 0b00001111, a); + let e = _mm256_set_epi32(-1, -1, -1, -1, 4, 5, i32::MIN, i32::MAX); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtsepi64_epi32() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, i64::MIN, i64::MAX); + let r = _mm512_maskz_cvtsepi64_epi32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm512_maskz_cvtsepi64_epi32(0b00001111, a); + let e = _mm256_set_epi32(0, 0, 0, 0, 4, 5, i32::MIN, i32::MAX); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cvtsepi64_epi32() { + let a = _mm256_set_epi64x(4, 5, i64::MIN, i64::MAX); + let r = _mm256_cvtsepi64_epi32(a); + let e = _mm_set_epi32(4, 5, i32::MIN, i32::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtsepi64_epi32() { + let a = _mm256_set_epi64x(4, 5, i64::MIN, i64::MAX); + let src = _mm_set1_epi32(-1); + let r = _mm256_mask_cvtsepi64_epi32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvtsepi64_epi32(src, 0b00001111, a); + let e = _mm_set_epi32(4, 5, i32::MIN, i32::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtsepi64_epi32() { + let a = _mm256_set_epi64x(4, 5, i64::MIN, i64::MAX); + let r = _mm256_maskz_cvtsepi64_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvtsepi64_epi32(0b00001111, a); + let e = _mm_set_epi32(4, 5, i32::MIN, i32::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cvtsepi64_epi32() { + let a = _mm_set_epi64x(i64::MIN, i64::MAX); + let r = _mm_cvtsepi64_epi32(a); + let e = _mm_set_epi32(0, 0, i32::MIN, i32::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtsepi64_epi32() { + let a = _mm_set_epi64x(i64::MIN, i64::MAX); + let src = _mm_set1_epi32(0); + let r = _mm_mask_cvtsepi64_epi32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtsepi64_epi32(src, 0b00000011, a); + let e = _mm_set_epi32(0, 0, i32::MIN, i32::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtsepi64_epi32() { + let a = _mm_set_epi64x(i64::MIN, i64::MAX); + let r = _mm_maskz_cvtsepi64_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtsepi64_epi32(0b00000011, a); + let e = _mm_set_epi32(0, 0, i32::MIN, i32::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtsepi64_epi16() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, i64::MIN, i64::MAX); + let r = _mm512_cvtsepi64_epi16(a); + let e = _mm_set_epi16(0, 1, 2, 3, 4, 5, i16::MIN, i16::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtsepi64_epi16() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, i64::MIN, i64::MAX); + let src = _mm_set1_epi16(-1); + let r = _mm512_mask_cvtsepi64_epi16(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm512_mask_cvtsepi64_epi16(src, 0b00001111, a); + let e = _mm_set_epi16(-1, -1, -1, -1, 4, 5, i16::MIN, i16::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtsepi64_epi16() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, i64::MIN, i64::MAX); + let r = _mm512_maskz_cvtsepi64_epi16(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm512_maskz_cvtsepi64_epi16(0b00001111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 4, 5, i16::MIN, i16::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cvtsepi64_epi16() { + let a = _mm256_set_epi64x(4, 5, i64::MIN, i64::MAX); + let r = _mm256_cvtsepi64_epi16(a); + let e = _mm_set_epi16(0, 0, 0, 0, 4, 5, i16::MIN, i16::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtsepi64_epi16() { + let a = _mm256_set_epi64x(4, 5, i64::MIN, i64::MAX); + let src = _mm_set1_epi16(0); + let r = _mm256_mask_cvtsepi64_epi16(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvtsepi64_epi16(src, 0b00001111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 4, 5, i16::MIN, i16::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtsepi64_epi16() { + let a = _mm256_set_epi64x(4, 5, i64::MIN, i64::MAX); + let r = _mm256_maskz_cvtsepi64_epi16(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvtsepi64_epi16(0b00001111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 4, 5, i16::MIN, i16::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cvtsepi64_epi16() { + let a = _mm_set_epi64x(i64::MIN, i64::MAX); + let r = _mm_cvtsepi64_epi16(a); + let e = _mm_set_epi16(0, 0, 0, 0, 0, 0, i16::MIN, i16::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtsepi64_epi16() { + let a = _mm_set_epi64x(i64::MIN, i64::MAX); + let src = _mm_set1_epi16(0); + let r = _mm_mask_cvtsepi64_epi16(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtsepi64_epi16(src, 0b00000011, a); + let e = _mm_set_epi16(0, 0, 0, 0, 0, 0, i16::MIN, i16::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtsepi64_epi16() { + let a = _mm_set_epi64x(i64::MIN, i64::MAX); + let r = _mm_maskz_cvtsepi64_epi16(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtsepi64_epi16(0b00000011, a); + let e = _mm_set_epi16(0, 0, 0, 0, 0, 0, i16::MIN, i16::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtsepi64_epi8() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, i64::MIN, i64::MAX); + let r = _mm512_cvtsepi64_epi8(a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, i8::MIN, i8::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtsepi64_epi8() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, i64::MIN, i64::MAX); + let src = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1); + let r = _mm512_mask_cvtsepi64_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm512_mask_cvtsepi64_epi8(src, 0b00001111, a); + #[rustfmt::skip] + let e = _mm_set_epi8( + 0, 0, 0, 0, + 0, 0, 0, 0, + -1, -1, -1, -1, + 4, 5, i8::MIN, i8::MAX, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtsepi64_epi8() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, i64::MIN, i64::MAX); + let r = _mm512_maskz_cvtsepi64_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm512_maskz_cvtsepi64_epi8(0b00001111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, i8::MIN, i8::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cvtsepi64_epi8() { + let a = _mm256_set_epi64x(4, 5, i64::MIN, i64::MAX); + let r = _mm256_cvtsepi64_epi8(a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, i8::MIN, i8::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtsepi64_epi8() { + let a = _mm256_set_epi64x(4, 5, i64::MIN, i64::MAX); + let src = _mm_set1_epi8(0); + let r = _mm256_mask_cvtsepi64_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvtsepi64_epi8(src, 0b00001111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, i8::MIN, i8::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtsepi64_epi8() { + let a = _mm256_set_epi64x(4, 5, i64::MIN, i64::MAX); + let r = _mm256_maskz_cvtsepi64_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvtsepi64_epi8(0b00001111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, i8::MIN, i8::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cvtsepi64_epi8() { + let a = _mm_set_epi64x(i64::MIN, i64::MAX); + let r = _mm_cvtsepi64_epi8(a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, i8::MIN, i8::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtsepi64_epi8() { + let a = _mm_set_epi64x(i64::MIN, i64::MAX); + let src = _mm_set1_epi8(0); + let r = _mm_mask_cvtsepi64_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtsepi64_epi8(src, 0b00000011, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, i8::MIN, i8::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtsepi64_epi8() { + let a = _mm_set_epi64x(i64::MIN, i64::MAX); + let r = _mm_maskz_cvtsepi64_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtsepi64_epi8(0b00000011, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, i8::MIN, i8::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtusepi64_epi32() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, i64::MIN, i64::MIN); + let r = _mm512_cvtusepi64_epi32(a); + let e = _mm256_set_epi32(0, 1, 2, 3, 4, 5, -1, -1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtusepi64_epi32() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, i64::MIN, i64::MIN); + let src = _mm256_set1_epi32(-1); + let r = _mm512_mask_cvtusepi64_epi32(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm512_mask_cvtusepi64_epi32(src, 0b00001111, a); + let e = _mm256_set_epi32(-1, -1, -1, -1, 4, 5, -1, -1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtusepi64_epi32() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, i64::MIN, i64::MIN); + let r = _mm512_maskz_cvtusepi64_epi32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm512_maskz_cvtusepi64_epi32(0b00001111, a); + let e = _mm256_set_epi32(0, 0, 0, 0, 4, 5, -1, -1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cvtusepi64_epi32() { + let a = _mm256_set_epi64x(4, 5, 6, i64::MAX); + let r = _mm256_cvtusepi64_epi32(a); + let e = _mm_set_epi32(4, 5, 6, u32::MAX as i32); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtusepi64_epi32() { + let a = _mm256_set_epi64x(4, 5, 6, i64::MAX); + let src = _mm_set1_epi32(0); + let r = _mm256_mask_cvtusepi64_epi32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvtusepi64_epi32(src, 0b00001111, a); + let e = _mm_set_epi32(4, 5, 6, u32::MAX as i32); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtusepi64_epi32() { + let a = _mm256_set_epi64x(4, 5, 6, i64::MAX); + let r = _mm256_maskz_cvtusepi64_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvtusepi64_epi32(0b00001111, a); + let e = _mm_set_epi32(4, 5, 6, u32::MAX as i32); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cvtusepi64_epi32() { + let a = _mm_set_epi64x(6, i64::MAX); + let r = _mm_cvtusepi64_epi32(a); + let e = _mm_set_epi32(0, 0, 6, u32::MAX as i32); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtusepi64_epi32() { + let a = _mm_set_epi64x(6, i64::MAX); + let src = _mm_set1_epi32(0); + let r = _mm_mask_cvtusepi64_epi32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtusepi64_epi32(src, 0b00000011, a); + let e = _mm_set_epi32(0, 0, 6, u32::MAX as i32); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtusepi64_epi32() { + let a = _mm_set_epi64x(6, i64::MAX); + let r = _mm_maskz_cvtusepi64_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtusepi64_epi32(0b00000011, a); + let e = _mm_set_epi32(0, 0, 6, u32::MAX as i32); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtusepi64_epi16() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, i64::MIN, i64::MIN); + let r = _mm512_cvtusepi64_epi16(a); + let e = _mm_set_epi16(0, 1, 2, 3, 4, 5, -1, -1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtusepi64_epi16() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, i64::MIN, i64::MIN); + let src = _mm_set1_epi16(-1); + let r = _mm512_mask_cvtusepi64_epi16(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm512_mask_cvtusepi64_epi16(src, 0b00001111, a); + let e = _mm_set_epi16(-1, -1, -1, -1, 4, 5, -1, -1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtusepi64_epi16() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, i64::MIN, i64::MIN); + let r = _mm512_maskz_cvtusepi64_epi16(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm512_maskz_cvtusepi64_epi16(0b00001111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 4, 5, -1, -1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cvtusepi64_epi16() { + let a = _mm256_set_epi64x(4, 5, 6, i64::MAX); + let r = _mm256_cvtusepi64_epi16(a); + let e = _mm_set_epi16(0, 0, 0, 0, 4, 5, 6, u16::MAX as i16); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtusepi64_epi16() { + let a = _mm256_set_epi64x(4, 5, 6, i64::MAX); + let src = _mm_set1_epi16(0); + let r = _mm256_mask_cvtusepi64_epi16(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvtusepi64_epi16(src, 0b00001111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 4, 5, 6, u16::MAX as i16); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtusepi64_epi16() { + let a = _mm256_set_epi64x(4, 5, 6, i64::MAX); + let r = _mm256_maskz_cvtusepi64_epi16(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvtusepi64_epi16(0b00001111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 4, 5, 6, u16::MAX as i16); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cvtusepi64_epi16() { + let a = _mm_set_epi64x(6, i64::MAX); + let r = _mm_cvtusepi64_epi16(a); + let e = _mm_set_epi16(0, 0, 0, 0, 0, 0, 6, u16::MAX as i16); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtusepi64_epi16() { + let a = _mm_set_epi64x(6, i64::MAX); + let src = _mm_set1_epi16(0); + let r = _mm_mask_cvtusepi64_epi16(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtusepi64_epi16(src, 0b00000011, a); + let e = _mm_set_epi16(0, 0, 0, 0, 0, 0, 6, u16::MAX as i16); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtusepi64_epi16() { + let a = _mm_set_epi64x(6, i64::MAX); + let r = _mm_maskz_cvtusepi64_epi16(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtusepi64_epi16(0b00000011, a); + let e = _mm_set_epi16(0, 0, 0, 0, 0, 0, 6, u16::MAX as i16); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtusepi64_epi8() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, i64::MIN, i64::MIN); + let r = _mm512_cvtusepi64_epi8(a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, -1, -1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtusepi64_epi8() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, i64::MIN, i64::MIN); + let src = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1); + let r = _mm512_mask_cvtusepi64_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm512_mask_cvtusepi64_epi8(src, 0b00001111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, 4, 5, -1, -1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtusepi64_epi8() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, i64::MIN, i64::MIN); + let r = _mm512_maskz_cvtusepi64_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm512_maskz_cvtusepi64_epi8(0b00001111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, -1, -1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cvtusepi64_epi8() { + let a = _mm256_set_epi64x(4, 5, 6, i64::MAX); + let r = _mm256_cvtusepi64_epi8(a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 6, u8::MAX as i8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtusepi64_epi8() { + let a = _mm256_set_epi64x(4, 5, 6, i64::MAX); + let src = _mm_set1_epi8(0); + let r = _mm256_mask_cvtusepi64_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvtusepi64_epi8(src, 0b00001111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 6, u8::MAX as i8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvtusepi64_epi8() { + let a = _mm256_set_epi64x(4, 5, 6, i64::MAX); + let r = _mm256_maskz_cvtusepi64_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvtusepi64_epi8(0b00001111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 6, u8::MAX as i8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cvtusepi64_epi8() { + let a = _mm_set_epi64x(6, i64::MAX); + let r = _mm_cvtusepi64_epi8(a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, u8::MAX as i8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtusepi64_epi8() { + let a = _mm_set_epi64x(6, i64::MAX); + let src = _mm_set1_epi8(0); + let r = _mm_mask_cvtusepi64_epi8(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvtusepi64_epi8(src, 0b00000011, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, u8::MAX as i8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvtusepi64_epi8() { + let a = _mm_set_epi64x(6, i64::MAX); + let r = _mm_maskz_cvtusepi64_epi8(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvtusepi64_epi8(0b00000011, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, u8::MAX as i8); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtt_roundpd_epi32() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_cvtt_roundpd_epi32::<_MM_FROUND_NO_EXC>(a); + let e = _mm256_setr_epi32(0, -1, 2, -3, 4, -5, 6, -7); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtt_roundpd_epi32() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let src = _mm256_set1_epi32(0); + let r = _mm512_mask_cvtt_roundpd_epi32::<_MM_FROUND_NO_EXC>(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm512_mask_cvtt_roundpd_epi32::<_MM_FROUND_NO_EXC>(src, 0b00001111, a); + let e = _mm256_setr_epi32(0, -1, 2, -3, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtt_roundpd_epi32() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_maskz_cvtt_roundpd_epi32::<_MM_FROUND_NO_EXC>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm512_maskz_cvtt_roundpd_epi32::<_MM_FROUND_NO_EXC>(0b00001111, a); + let e = _mm256_setr_epi32(0, -1, 2, -3, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvtt_roundpd_epu32() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_cvtt_roundpd_epu32::<_MM_FROUND_NO_EXC>(a); + let e = _mm256_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtt_roundpd_epu32() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let src = _mm256_set1_epi32(0); + let r = _mm512_mask_cvtt_roundpd_epu32::<_MM_FROUND_NO_EXC>(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm512_mask_cvtt_roundpd_epu32::<_MM_FROUND_NO_EXC>(src, 0b00001111, a); + let e = _mm256_setr_epi32(0, -1, 2, -1, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvtt_roundpd_epu32() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_maskz_cvtt_roundpd_epu32::<_MM_FROUND_NO_EXC>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm512_maskz_cvtt_roundpd_epu32::<_MM_FROUND_NO_EXC>(0b00001111, a); + let e = _mm256_setr_epi32(0, -1, 2, -1, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvttpd_epi32() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_cvttpd_epi32(a); + let e = _mm256_setr_epi32(0, -1, 2, -3, 4, -5, 6, -7); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvttpd_epi32() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let src = _mm256_set1_epi32(0); + let r = _mm512_mask_cvttpd_epi32(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm512_mask_cvttpd_epi32(src, 0b00001111, a); + let e = _mm256_setr_epi32(0, -1, 2, -3, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvttpd_epi32() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_maskz_cvttpd_epi32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm512_maskz_cvttpd_epi32(0b00001111, a); + let e = _mm256_setr_epi32(0, -1, 2, -3, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvttpd_epi32() { + let a = _mm256_setr_pd(4., -5.5, 6., -7.5); + let src = _mm_set1_epi32(0); + let r = _mm256_mask_cvttpd_epi32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvttpd_epi32(src, 0b00001111, a); + let e = _mm_setr_epi32(4, -5, 6, -7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvttpd_epi32() { + let a = _mm256_setr_pd(4., -5.5, 6., -7.5); + let r = _mm256_maskz_cvttpd_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvttpd_epi32(0b00001111, a); + let e = _mm_setr_epi32(4, -5, 6, -7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvttpd_epi32() { + let a = _mm_set_pd(6., -7.5); + let src = _mm_set1_epi32(0); + let r = _mm_mask_cvttpd_epi32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvttpd_epi32(src, 0b00000011, a); + let e = _mm_set_epi32(0, 0, 6, -7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvttpd_epi32() { + let a = _mm_set_pd(6., -7.5); + let r = _mm_maskz_cvttpd_epi32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvttpd_epi32(0b00000011, a); + let e = _mm_set_epi32(0, 0, 6, -7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvttpd_epu32() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_cvttpd_epu32(a); + let e = _mm256_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvttpd_epu32() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let src = _mm256_set1_epi32(0); + let r = _mm512_mask_cvttpd_epu32(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm512_mask_cvttpd_epu32(src, 0b00001111, a); + let e = _mm256_setr_epi32(0, -1, 2, -1, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvttpd_epu32() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_maskz_cvttpd_epu32(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm512_maskz_cvttpd_epu32(0b00001111, a); + let e = _mm256_setr_epi32(0, -1, 2, -1, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cvttpd_epu32() { + let a = _mm256_set_pd(4., 5.5, 6., 7.5); + let r = _mm256_cvttpd_epu32(a); + let e = _mm_set_epi32(4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvttpd_epu32() { + let a = _mm256_set_pd(4., 5.5, 6., 7.5); + let src = _mm_set1_epi32(0); + let r = _mm256_mask_cvttpd_epu32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm256_mask_cvttpd_epu32(src, 0b00001111, a); + let e = _mm_set_epi32(4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_cvttpd_epu32() { + let a = _mm256_set_pd(4., 5.5, 6., 7.5); + let r = _mm256_maskz_cvttpd_epu32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm256_maskz_cvttpd_epu32(0b00001111, a); + let e = _mm_set_epi32(4, 5, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cvttpd_epu32() { + let a = _mm_set_pd(6., 7.5); + let r = _mm_cvttpd_epu32(a); + let e = _mm_set_epi32(0, 0, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvttpd_epu32() { + let a = _mm_set_pd(6., 7.5); + let src = _mm_set1_epi32(0); + let r = _mm_mask_cvttpd_epu32(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_cvttpd_epu32(src, 0b00000011, a); + let e = _mm_set_epi32(0, 0, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_cvttpd_epu32() { + let a = _mm_set_pd(6., 7.5); + let r = _mm_maskz_cvttpd_epu32(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_cvttpd_epu32(0b00000011, a); + let e = _mm_set_epi32(0, 0, 6, 7); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_add_round_pd() { + let a = _mm512_setr_pd(8., 9.5, 10., 11.5, 12., 13.5, 14., 0.000000000000000007); + let b = _mm512_set1_pd(-1.); + let r = _mm512_add_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm512_setr_pd(7., 8.5, 9., 10.5, 11., 12.5, 13., -1.0); + assert_eq_m512d(r, e); + let r = _mm512_add_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm512_setr_pd(7., 8.5, 9., 10.5, 11., 12.5, 13., -0.9999999999999999); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_add_round_pd() { + let a = _mm512_setr_pd(8., 9.5, 10., 11.5, 12., 13.5, 14., 0.000000000000000007); + let b = _mm512_set1_pd(-1.); + let r = _mm512_mask_add_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); + assert_eq_m512d(r, a); + let r = _mm512_mask_add_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11110000, a, b, + ); + let e = _mm512_setr_pd(8., 9.5, 10., 11.5, 11., 12.5, 13., -1.0); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_add_round_pd() { + let a = _mm512_setr_pd(8., 9.5, 10., 11.5, 12., 13.5, 14., 0.000000000000000007); + let b = _mm512_set1_pd(-1.); + let r = + _mm512_maskz_add_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a, b); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_add_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11110000, a, b, + ); + let e = _mm512_setr_pd(0., 0., 0., 0., 11., 12.5, 13., -1.0); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_sub_round_pd() { + let a = _mm512_setr_pd(8., 9.5, 10., 11.5, 12., 13.5, 14., 0.000000000000000007); + let b = _mm512_set1_pd(1.); + let r = _mm512_sub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm512_setr_pd(7., 8.5, 9., 10.5, 11., 12.5, 13., -1.0); + assert_eq_m512d(r, e); + let r = _mm512_sub_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm512_setr_pd(7., 8.5, 9., 10.5, 11., 12.5, 13., -0.9999999999999999); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_sub_round_pd() { + let a = _mm512_setr_pd(8., 9.5, 10., 11.5, 12., 13.5, 14., 0.000000000000000007); + let b = _mm512_set1_pd(1.); + let r = _mm512_mask_sub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); + assert_eq_m512d(r, a); + let r = _mm512_mask_sub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11110000, a, b, + ); + let e = _mm512_setr_pd(8., 9.5, 10., 11.5, 11., 12.5, 13., -1.0); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_sub_round_pd() { + let a = _mm512_setr_pd(8., 9.5, 10., 11.5, 12., 13.5, 14., 0.000000000000000007); + let b = _mm512_set1_pd(1.); + let r = + _mm512_maskz_sub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a, b); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_sub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11110000, a, b, + ); + let e = _mm512_setr_pd(0., 0., 0., 0., 11., 12.5, 13., -1.0); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mul_round_pd() { + let a = _mm512_setr_pd(8., 9.5, 10., 11.5, 12., 13.5, 14., 0.); + let b = _mm512_set1_pd(0.1); + let r = _mm512_mul_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm512_setr_pd( + 0.8, + 0.9500000000000001, + 1., + 1.1500000000000001, + 1.2000000000000002, + 1.35, + 1.4000000000000001, + 0., + ); + assert_eq_m512d(r, e); + let r = _mm512_mul_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm512_setr_pd(0.8, 0.95, 1.0, 1.15, 1.2, 1.3499999999999999, 1.4, 0.0); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_mul_round_pd() { + let a = _mm512_setr_pd(8., 9.5, 10., 11.5, 12., 13.5, 14., 0.); + let b = _mm512_set1_pd(0.1); + let r = _mm512_mask_mul_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); + assert_eq_m512d(r, a); + let r = _mm512_mask_mul_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11110000, a, b, + ); + let e = _mm512_setr_pd( + 8., + 9.5, + 10., + 11.5, + 1.2000000000000002, + 1.35, + 1.4000000000000001, + 0., + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_mul_round_pd() { + let a = _mm512_setr_pd(8., 9.5, 10., 11.5, 12., 13.5, 14., 0.); + let b = _mm512_set1_pd(0.1); + let r = + _mm512_maskz_mul_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a, b); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_mul_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11110000, a, b, + ); + let e = _mm512_setr_pd( + 0., + 0., + 0., + 0., + 1.2000000000000002, + 1.35, + 1.4000000000000001, + 0., + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_div_round_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_set1_pd(3.); + let r = _mm512_div_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm512_set1_pd(0.3333333333333333); + assert_eq_m512d(r, e); + let r = _mm512_div_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm512_set1_pd(0.3333333333333333); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_div_round_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_set1_pd(3.); + let r = _mm512_mask_div_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); + assert_eq_m512d(r, a); + let r = _mm512_mask_div_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11110000, a, b, + ); + let e = _mm512_setr_pd( + 1., + 1., + 1., + 1., + 0.3333333333333333, + 0.3333333333333333, + 0.3333333333333333, + 0.3333333333333333, + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_div_round_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_set1_pd(3.); + let r = + _mm512_maskz_div_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a, b); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_div_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11110000, a, b, + ); + let e = _mm512_setr_pd( + 0., + 0., + 0., + 0., + 0.3333333333333333, + 0.3333333333333333, + 0.3333333333333333, + 0.3333333333333333, + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_sqrt_round_pd() { + let a = _mm512_set1_pd(3.); + let r = _mm512_sqrt_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a); + let e = _mm512_set1_pd(1.7320508075688772); + assert_eq_m512d(r, e); + let r = _mm512_sqrt_round_pd::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a); + let e = _mm512_set1_pd(1.7320508075688774); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_sqrt_round_pd() { + let a = _mm512_set1_pd(3.); + let r = + _mm512_mask_sqrt_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, 0, a); + assert_eq_m512d(r, a); + let r = _mm512_mask_sqrt_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11110000, a, + ); + let e = _mm512_setr_pd( + 3., + 3., + 3., + 3., + 1.7320508075688772, + 1.7320508075688772, + 1.7320508075688772, + 1.7320508075688772, + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_sqrt_round_pd() { + let a = _mm512_set1_pd(3.); + let r = + _mm512_maskz_sqrt_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_sqrt_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11110000, a, + ); + let e = _mm512_setr_pd( + 0., + 0., + 0., + 0., + 1.7320508075688772, + 1.7320508075688772, + 1.7320508075688772, + 1.7320508075688772, + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fmadd_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(-1.); + let r = _mm512_fmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_set1_pd(-1.); + assert_eq_m512d(r, e); + let r = _mm512_fmadd_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_set1_pd(-0.9999999999999999); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fmadd_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(-1.); + let r = _mm512_mask_fmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); + assert_eq_m512d(r, a); + let r = _mm512_mask_fmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b00001111, b, c, + ); + let e = _mm512_setr_pd( + -1., + -1., + -1., + -1., + 0.000000000000000007, + 0.000000000000000007, + 0.000000000000000007, + 0.000000000000000007, + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fmadd_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(-1.); + let r = _mm512_maskz_fmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_fmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00001111, a, b, c, + ); + let e = _mm512_setr_pd(-1., -1., -1., -1., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fmadd_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(-1.); + let r = _mm512_mask3_fmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); + assert_eq_m512d(r, c); + let r = _mm512_mask3_fmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b00001111, + ); + let e = _mm512_setr_pd(-1., -1., -1., -1., -1., -1., -1., -1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fmsub_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(1.); + let r = _mm512_fmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_set1_pd(-1.); + assert_eq_m512d(r, e); + let r = _mm512_fmsub_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_set1_pd(-0.9999999999999999); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fmsub_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(1.); + let r = _mm512_mask_fmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); + assert_eq_m512d(r, a); + let r = _mm512_mask_fmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b00001111, b, c, + ); + let e = _mm512_setr_pd( + -1., + -1., + -1., + -1., + 0.000000000000000007, + 0.000000000000000007, + 0.000000000000000007, + 0.000000000000000007, + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fmsub_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(1.); + let r = _mm512_maskz_fmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_fmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00001111, a, b, c, + ); + let e = _mm512_setr_pd(-1., -1., -1., -1., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fmsub_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(1.); + let r = _mm512_mask3_fmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); + assert_eq_m512d(r, c); + let r = _mm512_mask3_fmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b00001111, + ); + let e = _mm512_setr_pd(-1., -1., -1., -1., 1., 1., 1., 1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fmaddsub_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(-1.); + let r = + _mm512_fmaddsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_setr_pd(1., -1., 1., -1., 1., -1., 1., -1.); + assert_eq_m512d(r, e); + let r = _mm512_fmaddsub_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_setr_pd( + 1., + -0.9999999999999999, + 1., + -0.9999999999999999, + 1., + -0.9999999999999999, + 1., + -0.9999999999999999, + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fmaddsub_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(-1.); + let r = _mm512_mask_fmaddsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); + assert_eq_m512d(r, a); + let r = _mm512_mask_fmaddsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b00001111, b, c, + ); + let e = _mm512_setr_pd( + 1., + -1., + 1., + -1., + 0.000000000000000007, + 0.000000000000000007, + 0.000000000000000007, + 0.000000000000000007, + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fmaddsub_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(-1.); + let r = _mm512_maskz_fmaddsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_fmaddsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00001111, a, b, c, + ); + let e = _mm512_setr_pd(1., -1., 1., -1., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fmaddsub_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(-1.); + let r = _mm512_mask3_fmaddsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); + assert_eq_m512d(r, c); + let r = _mm512_mask3_fmaddsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b00001111, + ); + let e = _mm512_setr_pd(1., -1., 1., -1., -1., -1., -1., -1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fmsubadd_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(-1.); + let r = + _mm512_fmsubadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_setr_pd(-1., 1., -1., 1., -1., 1., -1., 1.); + assert_eq_m512d(r, e); + let r = _mm512_fmsubadd_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_setr_pd( + -0.9999999999999999, + 1., + -0.9999999999999999, + 1., + -0.9999999999999999, + 1., + -0.9999999999999999, + 1., + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fmsubadd_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(-1.); + let r = _mm512_mask_fmsubadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); + assert_eq_m512d(r, a); + let r = _mm512_mask_fmsubadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b00001111, b, c, + ); + let e = _mm512_setr_pd( + -1., + 1., + -1., + 1., + 0.000000000000000007, + 0.000000000000000007, + 0.000000000000000007, + 0.000000000000000007, + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fmsubadd_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(-1.); + let r = _mm512_maskz_fmsubadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_fmsubadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00001111, a, b, c, + ); + let e = _mm512_setr_pd(-1., 1., -1., 1., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fmsubadd_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(-1.); + let r = _mm512_mask3_fmsubadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); + assert_eq_m512d(r, c); + let r = _mm512_mask3_fmsubadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b00001111, + ); + let e = _mm512_setr_pd(-1., 1., -1., 1., -1., -1., -1., -1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fnmadd_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(1.); + let r = + _mm512_fnmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_set1_pd(1.); + assert_eq_m512d(r, e); + let r = _mm512_fnmadd_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_set1_pd(0.9999999999999999); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fnmadd_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(1.); + let r = _mm512_mask_fnmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); + assert_eq_m512d(r, a); + let r = _mm512_mask_fnmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b00001111, b, c, + ); + let e = _mm512_setr_pd( + 1., + 1., + 1., + 1., + 0.000000000000000007, + 0.000000000000000007, + 0.000000000000000007, + 0.000000000000000007, + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fnmadd_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(1.); + let r = _mm512_maskz_fnmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_fnmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00001111, a, b, c, + ); + let e = _mm512_setr_pd(1., 1., 1., 1., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fnmadd_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(1.); + let r = _mm512_mask3_fnmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); + assert_eq_m512d(r, c); + let r = _mm512_mask3_fnmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b00001111, + ); + let e = _mm512_setr_pd(1., 1., 1., 1., 1., 1., 1., 1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fnmsub_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(-1.); + let r = + _mm512_fnmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_set1_pd(1.); + assert_eq_m512d(r, e); + let r = _mm512_fnmsub_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); + let e = _mm512_set1_pd(0.9999999999999999); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fnmsub_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(-1.); + let r = _mm512_mask_fnmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); + assert_eq_m512d(r, a); + let r = _mm512_mask_fnmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b00001111, b, c, + ); + let e = _mm512_setr_pd( + 1., + 1., + 1., + 1., + 0.000000000000000007, + 0.000000000000000007, + 0.000000000000000007, + 0.000000000000000007, + ); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fnmsub_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(-1.); + let r = _mm512_maskz_fnmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_fnmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00001111, a, b, c, + ); + let e = _mm512_setr_pd(1., 1., 1., 1., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask3_fnmsub_round_pd() { + let a = _mm512_set1_pd(0.000000000000000007); + let b = _mm512_set1_pd(1.); + let c = _mm512_set1_pd(-1.); + let r = _mm512_mask3_fnmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); + assert_eq_m512d(r, c); + let r = _mm512_mask3_fnmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b00001111, + ); + let e = _mm512_setr_pd(1., 1., 1., 1., -1., -1., -1., -1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_max_round_pd() { + let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let b = _mm512_setr_pd(7., 6., 5., 4., 3., 2., 1., 0.); + let r = _mm512_max_round_pd::<_MM_FROUND_CUR_DIRECTION>(a, b); + let e = _mm512_setr_pd(7., 6., 5., 4., 4., 5., 6., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_max_round_pd() { + let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let b = _mm512_setr_pd(7., 6., 5., 4., 3., 2., 1., 0.); + let r = _mm512_mask_max_round_pd::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); + assert_eq_m512d(r, a); + let r = _mm512_mask_max_round_pd::<_MM_FROUND_CUR_DIRECTION>(a, 0b00001111, a, b); + let e = _mm512_setr_pd(7., 6., 5., 4., 4., 5., 6., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_max_round_pd() { + let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let b = _mm512_setr_pd(7., 6., 5., 4., 3., 2., 1., 0.); + let r = _mm512_maskz_max_round_pd::<_MM_FROUND_CUR_DIRECTION>(0, a, b); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_max_round_pd::<_MM_FROUND_CUR_DIRECTION>(0b00001111, a, b); + let e = _mm512_setr_pd(7., 6., 5., 4., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_min_round_pd() { + let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let b = _mm512_setr_pd(7., 6., 5., 4., 3., 2., 1., 0.); + let r = _mm512_min_round_pd::<_MM_FROUND_CUR_DIRECTION>(a, b); + let e = _mm512_setr_pd(0., 1., 2., 3., 3., 2., 1., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_min_round_pd() { + let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let b = _mm512_setr_pd(7., 6., 5., 4., 3., 2., 1., 0.); + let r = _mm512_mask_min_round_pd::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); + assert_eq_m512d(r, a); + let r = _mm512_mask_min_round_pd::<_MM_FROUND_CUR_DIRECTION>(a, 0b00001111, a, b); + let e = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_min_round_pd() { + let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let b = _mm512_setr_pd(7., 6., 5., 4., 3., 2., 1., 0.); + let r = _mm512_maskz_min_round_pd::<_MM_FROUND_CUR_DIRECTION>(0, a, b); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_min_round_pd::<_MM_FROUND_CUR_DIRECTION>(0b00001111, a, b); + let e = _mm512_setr_pd(0., 1., 2., 3., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_getexp_round_pd() { + let a = _mm512_set1_pd(3.); + let r = _mm512_getexp_round_pd::<_MM_FROUND_CUR_DIRECTION>(a); + let e = _mm512_set1_pd(1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_getexp_round_pd() { + let a = _mm512_set1_pd(3.); + let r = _mm512_mask_getexp_round_pd::<_MM_FROUND_CUR_DIRECTION>(a, 0, a); + assert_eq_m512d(r, a); + let r = _mm512_mask_getexp_round_pd::<_MM_FROUND_CUR_DIRECTION>(a, 0b11110000, a); + let e = _mm512_setr_pd(3., 3., 3., 3., 1., 1., 1., 1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_getexp_round_pd() { + let a = _mm512_set1_pd(3.); + let r = _mm512_maskz_getexp_round_pd::<_MM_FROUND_CUR_DIRECTION>(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_getexp_round_pd::<_MM_FROUND_CUR_DIRECTION>(0b11110000, a); + let e = _mm512_setr_pd(0., 0., 0., 0., 1., 1., 1., 1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_roundscale_round_pd() { + let a = _mm512_set1_pd(1.1); + let r = _mm512_roundscale_round_pd::<0, _MM_FROUND_CUR_DIRECTION>(a); + let e = _mm512_set1_pd(1.0); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_roundscale_round_pd() { + let a = _mm512_set1_pd(1.1); + let r = _mm512_mask_roundscale_round_pd::<0, _MM_FROUND_CUR_DIRECTION>(a, 0, a); + let e = _mm512_set1_pd(1.1); + assert_eq_m512d(r, e); + let r = _mm512_mask_roundscale_round_pd::<0, _MM_FROUND_CUR_DIRECTION>(a, 0b11111111, a); + let e = _mm512_set1_pd(1.0); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_roundscale_round_pd() { + let a = _mm512_set1_pd(1.1); + let r = _mm512_maskz_roundscale_round_pd::<0, _MM_FROUND_CUR_DIRECTION>(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_roundscale_round_pd::<0, _MM_FROUND_CUR_DIRECTION>(0b11111111, a); + let e = _mm512_set1_pd(1.0); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_scalef_round_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_set1_pd(3.); + let r = _mm512_scalef_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm512_set1_pd(8.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_scalef_round_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_set1_pd(3.); + let r = _mm512_mask_scalef_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); + assert_eq_m512d(r, a); + let r = _mm512_mask_scalef_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11110000, a, b, + ); + let e = _mm512_set_pd(8., 8., 8., 8., 1., 1., 1., 1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_scalef_round_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_set1_pd(3.); + let r = _mm512_maskz_scalef_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, + ); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_scalef_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11110000, a, b, + ); + let e = _mm512_set_pd(8., 8., 8., 8., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_fixupimm_round_pd() { + let a = _mm512_set1_pd(f64::NAN); + let b = _mm512_set1_pd(f64::MAX); + let c = _mm512_set1_epi64(i32::MAX as i64); + let r = _mm512_fixupimm_round_pd::<5, _MM_FROUND_CUR_DIRECTION>(a, b, c); + let e = _mm512_set1_pd(0.0); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_fixupimm_round_pd() { + let a = _mm512_set_pd(f64::NAN, f64::NAN, f64::NAN, f64::NAN, 1., 1., 1., 1.); + let b = _mm512_set1_pd(f64::MAX); + let c = _mm512_set1_epi64(i32::MAX as i64); + let r = _mm512_mask_fixupimm_round_pd::<5, _MM_FROUND_CUR_DIRECTION>(a, 0b11110000, b, c); + let e = _mm512_set_pd(0., 0., 0., 0., 1., 1., 1., 1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_fixupimm_round_pd() { + let a = _mm512_set_pd(f64::NAN, f64::NAN, f64::NAN, f64::NAN, 1., 1., 1., 1.); + let b = _mm512_set1_pd(f64::MAX); + let c = _mm512_set1_epi64(i32::MAX as i64); + let r = _mm512_maskz_fixupimm_round_pd::<5, _MM_FROUND_CUR_DIRECTION>(0b11110000, a, b, c); + let e = _mm512_set_pd(0., 0., 0., 0., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_getmant_round_pd() { + let a = _mm512_set1_pd(10.); + let r = _mm512_getmant_round_pd::< + _MM_MANT_NORM_1_2, + _MM_MANT_SIGN_SRC, + _MM_FROUND_CUR_DIRECTION, + >(a); + let e = _mm512_set1_pd(1.25); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_getmant_round_pd() { + let a = _mm512_set1_pd(10.); + let r = _mm512_mask_getmant_round_pd::< + _MM_MANT_NORM_1_2, + _MM_MANT_SIGN_SRC, + _MM_FROUND_CUR_DIRECTION, + >(a, 0, a); + assert_eq_m512d(r, a); + let r = _mm512_mask_getmant_round_pd::< + _MM_MANT_NORM_1_2, + _MM_MANT_SIGN_SRC, + _MM_FROUND_CUR_DIRECTION, + >(a, 0b11110000, a); + let e = _mm512_setr_pd(10., 10., 10., 10., 1.25, 1.25, 1.25, 1.25); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_getmant_round_pd() { + let a = _mm512_set1_pd(10.); + let r = _mm512_maskz_getmant_round_pd::< + _MM_MANT_NORM_1_2, + _MM_MANT_SIGN_SRC, + _MM_FROUND_CUR_DIRECTION, + >(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_getmant_round_pd::< + _MM_MANT_NORM_1_2, + _MM_MANT_SIGN_SRC, + _MM_FROUND_CUR_DIRECTION, + >(0b11110000, a); + let e = _mm512_setr_pd(0., 0., 0., 0., 1.25, 1.25, 1.25, 1.25); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvt_roundps_pd() { + let a = _mm256_setr_ps(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_cvt_roundps_pd::<_MM_FROUND_CUR_DIRECTION>(a); + let e = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvt_roundps_pd() { + let a = _mm256_setr_ps(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let src = _mm512_set1_pd(0.); + let r = _mm512_mask_cvt_roundps_pd::<_MM_FROUND_CUR_DIRECTION>(src, 0, a); + assert_eq_m512d(r, src); + let r = _mm512_mask_cvt_roundps_pd::<_MM_FROUND_CUR_DIRECTION>(src, 0b00001111, a); + let e = _mm512_setr_pd(0., -1.5, 2., -3.5, 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvt_roundps_pd() { + let a = _mm256_setr_ps(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_maskz_cvt_roundps_pd::<_MM_FROUND_CUR_DIRECTION>(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_cvt_roundps_pd::<_MM_FROUND_CUR_DIRECTION>(0b00001111, a); + let e = _mm512_setr_pd(0., -1.5, 2., -3.5, 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvt_roundpd_ps() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_cvt_roundpd_ps::<_MM_FROUND_CUR_DIRECTION>(a); + let e = _mm256_setr_ps(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvt_roundpd_ps() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let src = _mm256_set1_ps(0.); + let r = _mm512_mask_cvt_roundpd_ps::<_MM_FROUND_CUR_DIRECTION>(src, 0, a); + assert_eq_m256(r, src); + let r = _mm512_mask_cvt_roundpd_ps::<_MM_FROUND_CUR_DIRECTION>(src, 0b00001111, a); + let e = _mm256_setr_ps(0., -1.5, 2., -3.5, 0., 0., 0., 0.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvt_roundpd_ps() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_maskz_cvt_roundpd_ps::<_MM_FROUND_CUR_DIRECTION>(0, a); + assert_eq_m256(r, _mm256_setzero_ps()); + let r = _mm512_maskz_cvt_roundpd_ps::<_MM_FROUND_CUR_DIRECTION>(0b00001111, a); + let e = _mm256_setr_ps(0., -1.5, 2., -3.5, 0., 0., 0., 0.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvt_roundpd_epi32() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_cvt_roundpd_epi32::<_MM_FROUND_CUR_DIRECTION>(a); + let e = _mm256_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvt_roundpd_epi32() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let src = _mm256_set1_epi32(0); + let r = _mm512_mask_cvt_roundpd_epi32::<_MM_FROUND_CUR_DIRECTION>(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm512_mask_cvt_roundpd_epi32::<_MM_FROUND_CUR_DIRECTION>(src, 0b00001111, a); + let e = _mm256_setr_epi32(0, -2, 2, -4, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvt_roundpd_epi32() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_maskz_cvt_roundpd_epi32::<_MM_FROUND_CUR_DIRECTION>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm512_maskz_cvt_roundpd_epi32::<_MM_FROUND_CUR_DIRECTION>(0b00001111, a); + let e = _mm256_setr_epi32(0, -2, 2, -4, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cvt_roundpd_epu32() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_cvt_roundpd_epu32::<_MM_FROUND_CUR_DIRECTION>(a); + let e = _mm256_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvt_roundpd_epu32() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let src = _mm256_set1_epi32(0); + let r = _mm512_mask_cvt_roundpd_epu32::<_MM_FROUND_CUR_DIRECTION>(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm512_mask_cvt_roundpd_epu32::<_MM_FROUND_CUR_DIRECTION>(src, 0b00001111, a); + let e = _mm256_setr_epi32(0, -1, 2, -1, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_cvt_roundpd_epu32() { + let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); + let r = _mm512_maskz_cvt_roundpd_epu32::<_MM_FROUND_CUR_DIRECTION>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm512_maskz_cvt_roundpd_epu32::<_MM_FROUND_CUR_DIRECTION>(0b00001111, a); + let e = _mm256_setr_epi32(0, -1, 2, -1, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_setzero_pd() { + assert_eq_m512d(_mm512_setzero_pd(), _mm512_set1_pd(0.)); + } + + unsafe fn test_mm512_set1_epi64() { + let r = _mm512_set_epi64(2, 2, 2, 2, 2, 2, 2, 2); + assert_eq_m512i(r, _mm512_set1_epi64(2)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_set1_pd() { + let expected = _mm512_set_pd(2., 2., 2., 2., 2., 2., 2., 2.); + assert_eq_m512d(expected, _mm512_set1_pd(2.)); + } + + unsafe fn test_mm512_set4_epi64() { + let r = _mm512_set_epi64(4, 3, 2, 1, 4, 3, 2, 1); + assert_eq_m512i(r, _mm512_set4_epi64(4, 3, 2, 1)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_set4_pd() { + let r = _mm512_set_pd(4., 3., 2., 1., 4., 3., 2., 1.); + assert_eq_m512d(r, _mm512_set4_pd(4., 3., 2., 1.)); + } + + unsafe fn test_mm512_setr4_epi64() { + let r = _mm512_set_epi64(4, 3, 2, 1, 4, 3, 2, 1); + assert_eq_m512i(r, _mm512_setr4_epi64(1, 2, 3, 4)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_setr4_pd() { + let r = _mm512_set_pd(4., 3., 2., 1., 4., 3., 2., 1.); + assert_eq_m512d(r, _mm512_setr4_pd(1., 2., 3., 4.)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmplt_pd_mask() { + #[rustfmt::skip] + let a = _mm512_set_pd(0., 1., -1., f64::MAX, f64::NAN, f64::MIN, 100., -100.); + let b = _mm512_set1_pd(-1.); + let m = _mm512_cmplt_pd_mask(a, b); + assert_eq!(m, 0b00000101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmplt_pd_mask() { + #[rustfmt::skip] + let a = _mm512_set_pd(0., 1., -1., f64::MAX, f64::NAN, f64::MIN, 100., -100.); + let b = _mm512_set1_pd(-1.); + let mask = 0b01100110; + let r = _mm512_mask_cmplt_pd_mask(mask, a, b); + assert_eq!(r, 0b00000100); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpnlt_pd_mask() { + #[rustfmt::skip] + let a = _mm512_set_pd(0., 1., -1., f64::MAX, f64::NAN, f64::MIN, 100., -100.); + let b = _mm512_set1_pd(-1.); + assert_eq!(_mm512_cmpnlt_pd_mask(a, b), !_mm512_cmplt_pd_mask(a, b)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpnlt_pd_mask() { + #[rustfmt::skip] + let a = _mm512_set_pd(0., 1., -1., f64::MAX, f64::NAN, f64::MIN, 100., -100.); + let b = _mm512_set1_pd(-1.); + let mask = 0b01111010; + assert_eq!(_mm512_mask_cmpnlt_pd_mask(mask, a, b), 0b01111010); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmple_pd_mask() { + #[rustfmt::skip] + let a = _mm512_set_pd(0., 1., -1., f64::MAX, f64::NAN, f64::MIN, 100., -100.); + let b = _mm512_set1_pd(-1.); + assert_eq!(_mm512_cmple_pd_mask(a, b), 0b00100101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmple_pd_mask() { + #[rustfmt::skip] + let a = _mm512_set_pd(0., 1., -1., f64::MAX, f64::NAN, f64::MIN, 100., -100.); + let b = _mm512_set1_pd(-1.); + let mask = 0b01111010; + assert_eq!(_mm512_mask_cmple_pd_mask(mask, a, b), 0b00100000); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpnle_pd_mask() { + #[rustfmt::skip] + let a = _mm512_set_pd(0., 1., -1., f64::MAX, f64::NAN, f64::MIN, 100., -100.); + let b = _mm512_set1_pd(-1.); + let m = _mm512_cmpnle_pd_mask(b, a); + assert_eq!(m, 0b00001101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpnle_pd_mask() { + #[rustfmt::skip] + let a = _mm512_set_pd(0., 1., -1., f64::MAX, f64::NAN, f64::MIN, 100., -100.); + let b = _mm512_set1_pd(-1.); + let mask = 0b01100110; + let r = _mm512_mask_cmpnle_pd_mask(mask, b, a); + assert_eq!(r, 0b00000100); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpeq_pd_mask() { + let a = _mm512_set_pd(0., 1., -1., 13., f64::MAX, f64::MIN, f64::NAN, -100.); + let b = _mm512_set_pd(0., 1., 13., 42., f64::MAX, f64::MIN, f64::NAN, -100.); + let m = _mm512_cmpeq_pd_mask(b, a); + assert_eq!(m, 0b11001101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpeq_pd_mask() { + let a = _mm512_set_pd(0., 1., -1., 13., f64::MAX, f64::MIN, f64::NAN, -100.); + let b = _mm512_set_pd(0., 1., 13., 42., f64::MAX, f64::MIN, f64::NAN, -100.); + let mask = 0b01111010; + let r = _mm512_mask_cmpeq_pd_mask(mask, b, a); + assert_eq!(r, 0b01001000); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpneq_pd_mask() { + let a = _mm512_set_pd(0., 1., -1., 13., f64::MAX, f64::MIN, f64::NAN, -100.); + let b = _mm512_set_pd(0., 1., 13., 42., f64::MAX, f64::MIN, f64::NAN, -100.); + let m = _mm512_cmpneq_pd_mask(b, a); + assert_eq!(m, 0b00110010); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpneq_pd_mask() { + let a = _mm512_set_pd(0., 1., -1., 13., f64::MAX, f64::MIN, f64::NAN, -100.); + let b = _mm512_set_pd(0., 1., 13., 42., f64::MAX, f64::MIN, f64::NAN, -100.); + let mask = 0b01111010; + let r = _mm512_mask_cmpneq_pd_mask(mask, b, a); + assert_eq!(r, 0b00110010) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmp_pd_mask() { + let a = _mm512_set_pd(0., 1., -1., 13., f64::MAX, f64::MIN, 100., -100.); + let b = _mm512_set1_pd(-1.); + let m = _mm512_cmp_pd_mask::<_CMP_LT_OQ>(a, b); + assert_eq!(m, 0b00000101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmp_pd_mask() { + let a = _mm512_set_pd(0., 1., -1., 13., f64::MAX, f64::MIN, 100., -100.); + let b = _mm512_set1_pd(-1.); + let mask = 0b01100110; + let r = _mm512_mask_cmp_pd_mask::<_CMP_LT_OQ>(mask, a, b); + assert_eq!(r, 0b00000100); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmp_pd_mask() { + let a = _mm256_set_pd(0., 1., -1., 13.); + let b = _mm256_set1_pd(1.); + let m = _mm256_cmp_pd_mask::<_CMP_LT_OQ>(a, b); + assert_eq!(m, 0b00001010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmp_pd_mask() { + let a = _mm256_set_pd(0., 1., -1., 13.); + let b = _mm256_set1_pd(1.); + let mask = 0b11111111; + let r = _mm256_mask_cmp_pd_mask::<_CMP_LT_OQ>(mask, a, b); + assert_eq!(r, 0b00001010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmp_pd_mask() { + let a = _mm_set_pd(0., 1.); + let b = _mm_set1_pd(1.); + let m = _mm_cmp_pd_mask::<_CMP_LT_OQ>(a, b); + assert_eq!(m, 0b00000010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmp_pd_mask() { + let a = _mm_set_pd(0., 1.); + let b = _mm_set1_pd(1.); + let mask = 0b11111111; + let r = _mm_mask_cmp_pd_mask::<_CMP_LT_OQ>(mask, a, b); + assert_eq!(r, 0b00000010); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmp_round_pd_mask() { + #[rustfmt::skip] + let a = _mm512_set_pd(0., 1., -1., 13., f64::MAX, f64::MIN, 100., -100.); + let b = _mm512_set1_pd(-1.); + let m = _mm512_cmp_round_pd_mask::<_CMP_LT_OQ, _MM_FROUND_CUR_DIRECTION>(a, b); + assert_eq!(m, 0b00000101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmp_round_pd_mask() { + #[rustfmt::skip] + let a = _mm512_set_pd(0., 1., -1., 13., f64::MAX, f64::MIN, 100., -100.); + let b = _mm512_set1_pd(-1.); + let mask = 0b01100110; + let r = _mm512_mask_cmp_round_pd_mask::<_CMP_LT_OQ, _MM_FROUND_CUR_DIRECTION>(mask, a, b); + assert_eq!(r, 0b00000100); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpord_pd_mask() { + #[rustfmt::skip] + let a = _mm512_set_pd(f64::NAN, f64::MAX, f64::NAN, f64::MIN, f64::NAN, -1., f64::NAN, 0.); + #[rustfmt::skip] + let b = _mm512_set_pd(f64::NAN, f64::NAN, f64::NAN, f64::NAN, f64::MIN, f64::MAX, -1., 0.); + let m = _mm512_cmpord_pd_mask(a, b); + assert_eq!(m, 0b00000101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpord_pd_mask() { + #[rustfmt::skip] + let a = _mm512_set_pd(f64::NAN, f64::MAX, f64::NAN, f64::MIN, f64::NAN, -1., f64::NAN, 0.); + #[rustfmt::skip] + let b = _mm512_set_pd(f64::NAN, f64::NAN, f64::NAN, f64::NAN, f64::MIN, f64::MAX, -1., 0.); + let mask = 0b11000011; + let m = _mm512_mask_cmpord_pd_mask(mask, a, b); + assert_eq!(m, 0b00000001); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpunord_pd_mask() { + #[rustfmt::skip] + let a = _mm512_set_pd(f64::NAN, f64::MAX, f64::NAN, f64::MIN, f64::NAN, -1., f64::NAN, 0.); + #[rustfmt::skip] + let b = _mm512_set_pd(f64::NAN, f64::NAN, f64::NAN, f64::NAN, f64::MIN, f64::MAX, -1., 0.); + let m = _mm512_cmpunord_pd_mask(a, b); + + assert_eq!(m, 0b11111010); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpunord_pd_mask() { + #[rustfmt::skip] + let a = _mm512_set_pd(f64::NAN, f64::MAX, f64::NAN, f64::MIN, f64::NAN, -1., f64::NAN, 0.); + #[rustfmt::skip] + let b = _mm512_set_pd(f64::NAN, f64::NAN, f64::NAN, f64::NAN, f64::MIN, f64::MAX, -1., 0.); + let mask = 0b00001111; + let m = _mm512_mask_cmpunord_pd_mask(mask, a, b); + assert_eq!(m, 0b000001010); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmplt_epu64_mask() { + let a = _mm512_set_epi64(0, 1, -1, u64::MAX as i64, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set1_epi64(-1); + let m = _mm512_cmplt_epu64_mask(a, b); + assert_eq!(m, 0b11001111); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmplt_epu64_mask() { + let a = _mm512_set_epi64(0, 1, -1, u64::MAX as i64, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set1_epi64(-1); + let mask = 0b01111010; + let r = _mm512_mask_cmplt_epu64_mask(mask, a, b); + assert_eq!(r, 0b01001010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmplt_epu64_mask() { + let a = _mm256_set_epi64x(0, 1, 2, 100); + let b = _mm256_set1_epi64x(2); + let r = _mm256_cmplt_epu64_mask(a, b); + assert_eq!(r, 0b00001100); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmplt_epu64_mask() { + let a = _mm256_set_epi64x(0, 1, 2, 100); + let b = _mm256_set1_epi64x(2); + let mask = 0b11111111; + let r = _mm256_mask_cmplt_epu64_mask(mask, a, b); + assert_eq!(r, 0b00001100); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmplt_epu64_mask() { + let a = _mm_set_epi64x(0, 1); + let b = _mm_set1_epi64x(2); + let r = _mm_cmplt_epu64_mask(a, b); + assert_eq!(r, 0b00000011); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmplt_epu64_mask() { + let a = _mm_set_epi64x(0, 1); + let b = _mm_set1_epi64x(2); + let mask = 0b11111111; + let r = _mm_mask_cmplt_epu64_mask(mask, a, b); + assert_eq!(r, 0b00000011); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpgt_epu64_mask() { + let a = _mm512_set_epi64(0, 1, -1, u64::MAX as i64, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set1_epi64(-1); + let m = _mm512_cmpgt_epu64_mask(b, a); + assert_eq!(m, 0b11001111); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpgt_epu64_mask() { + let a = _mm512_set_epi64(0, 1, -1, u64::MAX as i64, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set1_epi64(-1); + let mask = 0b01111010; + let r = _mm512_mask_cmpgt_epu64_mask(mask, b, a); + assert_eq!(r, 0b01001010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmpgt_epu64_mask() { + let a = _mm256_set_epi64x(0, 1, 2, 3); + let b = _mm256_set1_epi64x(1); + let r = _mm256_cmpgt_epu64_mask(a, b); + assert_eq!(r, 0b00000011); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmpgt_epu64_mask() { + let a = _mm256_set_epi64x(0, 1, 2, 3); + let b = _mm256_set1_epi64x(1); + let mask = 0b11111111; + let r = _mm256_mask_cmpgt_epu64_mask(mask, a, b); + assert_eq!(r, 0b00000011); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmpgt_epu64_mask() { + let a = _mm_set_epi64x(1, 2); + let b = _mm_set1_epi64x(1); + let r = _mm_cmpgt_epu64_mask(a, b); + assert_eq!(r, 0b00000001); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmpgt_epu64_mask() { + let a = _mm_set_epi64x(1, 2); + let b = _mm_set1_epi64x(1); + let mask = 0b11111111; + let r = _mm_mask_cmpgt_epu64_mask(mask, a, b); + assert_eq!(r, 0b00000001); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmple_epu64_mask() { + let a = _mm512_set_epi64(0, 1, -1, u64::MAX as i64, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set1_epi64(-1); + assert_eq!( + _mm512_cmple_epu64_mask(a, b), + !_mm512_cmpgt_epu64_mask(a, b) + ) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmple_epu64_mask() { + let a = _mm512_set_epi64(0, 1, -1, u64::MAX as i64, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set1_epi64(-1); + let mask = 0b01111010; + assert_eq!(_mm512_mask_cmple_epu64_mask(mask, a, b), 0b01111010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmple_epu64_mask() { + let a = _mm256_set_epi64x(0, 1, 2, 1); + let b = _mm256_set1_epi64x(1); + let r = _mm256_cmple_epu64_mask(a, b); + assert_eq!(r, 0b00001101) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmple_epu64_mask() { + let a = _mm256_set_epi64x(0, 1, 2, 1); + let b = _mm256_set1_epi64x(1); + let mask = 0b11111111; + let r = _mm256_mask_cmple_epu64_mask(mask, a, b); + assert_eq!(r, 0b00001101) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmple_epu64_mask() { + let a = _mm_set_epi64x(0, 1); + let b = _mm_set1_epi64x(1); + let r = _mm_cmple_epu64_mask(a, b); + assert_eq!(r, 0b00000011) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmple_epu64_mask() { + let a = _mm_set_epi64x(0, 1); + let b = _mm_set1_epi64x(1); + let mask = 0b11111111; + let r = _mm_mask_cmple_epu64_mask(mask, a, b); + assert_eq!(r, 0b00000011) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpge_epu64_mask() { + let a = _mm512_set_epi64(0, 1, -1, u64::MAX as i64, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set1_epi64(-1); + assert_eq!( + _mm512_cmpge_epu64_mask(a, b), + !_mm512_cmplt_epu64_mask(a, b) + ); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpge_epu64_mask() { + let a = _mm512_set_epi64(0, 1, -1, u64::MAX as i64, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set1_epi64(-1); + let mask = 0b11111111; + let r = _mm512_mask_cmpge_epu64_mask(mask, a, b); + assert_eq!(r, 0b00110000); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmpge_epu64_mask() { + let a = _mm256_set_epi64x(0, 1, 2, u64::MAX as i64); + let b = _mm256_set1_epi64x(1); + let r = _mm256_cmpge_epu64_mask(a, b); + assert_eq!(r, 0b00000111); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmpge_epu64_mask() { + let a = _mm256_set_epi64x(0, 1, 2, u64::MAX as i64); + let b = _mm256_set1_epi64x(1); + let mask = 0b11111111; + let r = _mm256_mask_cmpge_epu64_mask(mask, a, b); + assert_eq!(r, 0b00000111); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmpge_epu64_mask() { + let a = _mm_set_epi64x(0, 1); + let b = _mm_set1_epi64x(1); + let r = _mm_cmpge_epu64_mask(a, b); + assert_eq!(r, 0b00000001); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmpge_epu64_mask() { + let a = _mm_set_epi64x(0, 1); + let b = _mm_set1_epi64x(1); + let mask = 0b11111111; + let r = _mm_mask_cmpge_epu64_mask(mask, a, b); + assert_eq!(r, 0b00000001); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpeq_epu64_mask() { + let a = _mm512_set_epi64(0, 1, -1, u64::MAX as i64, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set_epi64(0, 1, 13, 42, i64::MAX, i64::MIN, 100, -100); + let m = _mm512_cmpeq_epu64_mask(b, a); + assert_eq!(m, 0b11001111); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpeq_epu64_mask() { + let a = _mm512_set_epi64(0, 1, -1, u64::MAX as i64, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set_epi64(0, 1, 13, 42, i64::MAX, i64::MIN, 100, -100); + let mask = 0b01111010; + let r = _mm512_mask_cmpeq_epu64_mask(mask, b, a); + assert_eq!(r, 0b01001010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmpeq_epu64_mask() { + let a = _mm256_set_epi64x(0, 1, -1, u64::MAX as i64); + let b = _mm256_set_epi64x(0, 1, 13, 42); + let m = _mm256_cmpeq_epu64_mask(b, a); + assert_eq!(m, 0b00001100); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmpeq_epu64_mask() { + let a = _mm256_set_epi64x(0, 1, -1, u64::MAX as i64); + let b = _mm256_set_epi64x(0, 1, 13, 42); + let mask = 0b11111111; + let r = _mm256_mask_cmpeq_epu64_mask(mask, b, a); + assert_eq!(r, 0b00001100); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmpeq_epu64_mask() { + let a = _mm_set_epi64x(0, 1); + let b = _mm_set_epi64x(0, 1); + let m = _mm_cmpeq_epu64_mask(b, a); + assert_eq!(m, 0b00000011); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmpeq_epu64_mask() { + let a = _mm_set_epi64x(0, 1); + let b = _mm_set_epi64x(0, 1); + let mask = 0b11111111; + let r = _mm_mask_cmpeq_epu64_mask(mask, b, a); + assert_eq!(r, 0b00000011); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpneq_epu64_mask() { + let a = _mm512_set_epi64(0, 1, -1, u64::MAX as i64, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set_epi64(0, 1, 13, 42, i64::MAX, i64::MIN, 100, -100); + let m = _mm512_cmpneq_epu64_mask(b, a); + assert_eq!(m, !_mm512_cmpeq_epu64_mask(b, a)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpneq_epu64_mask() { + let a = _mm512_set_epi64(0, 1, -1, u64::MAX as i64, i64::MAX, i64::MIN, -100, 100); + let b = _mm512_set_epi64(0, 1, 13, 42, i64::MAX, i64::MIN, 100, -100); + let mask = 0b01111010; + let r = _mm512_mask_cmpneq_epu64_mask(mask, b, a); + assert_eq!(r, 0b00110010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmpneq_epu64_mask() { + let a = _mm256_set_epi64x(0, 1, -1, u64::MAX as i64); + let b = _mm256_set_epi64x(0, 1, 13, 42); + let r = _mm256_cmpneq_epu64_mask(b, a); + assert_eq!(r, 0b00000011); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmpneq_epu64_mask() { + let a = _mm256_set_epi64x(0, 1, -1, u64::MAX as i64); + let b = _mm256_set_epi64x(0, 1, 13, 42); + let mask = 0b11111111; + let r = _mm256_mask_cmpneq_epu64_mask(mask, b, a); + assert_eq!(r, 0b00000011); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmpneq_epu64_mask() { + let a = _mm_set_epi64x(-1, u64::MAX as i64); + let b = _mm_set_epi64x(13, 42); + let r = _mm_cmpneq_epu64_mask(b, a); + assert_eq!(r, 0b00000011); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmpneq_epu64_mask() { + let a = _mm_set_epi64x(-1, u64::MAX as i64); + let b = _mm_set_epi64x(13, 42); + let mask = 0b11111111; + let r = _mm_mask_cmpneq_epu64_mask(mask, b, a); + assert_eq!(r, 0b00000011); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmp_epu64_mask() { + let a = _mm512_set_epi64(0, 1, -1, u64::MAX as i64, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set1_epi64(-1); + let m = _mm512_cmp_epu64_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b11001111); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmp_epu64_mask() { + let a = _mm512_set_epi64(0, 1, -1, u64::MAX as i64, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set1_epi64(-1); + let mask = 0b01111010; + let r = _mm512_mask_cmp_epu64_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b01001010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmp_epu64_mask() { + let a = _mm256_set_epi64x(0, 1, -1, 100); + let b = _mm256_set1_epi64x(1); + let m = _mm256_cmp_epu64_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b00001000); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmp_epu64_mask() { + let a = _mm256_set_epi64x(0, 1, -1, 100); + let b = _mm256_set1_epi64x(1); + let mask = 0b11111111; + let r = _mm256_mask_cmp_epu64_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b00001000); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmp_epu64_mask() { + let a = _mm_set_epi64x(0, 1); + let b = _mm_set1_epi64x(1); + let m = _mm_cmp_epu64_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b00000010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmp_epu64_mask() { + let a = _mm_set_epi64x(0, 1); + let b = _mm_set1_epi64x(1); + let mask = 0b11111111; + let r = _mm_mask_cmp_epu64_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b00000010); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmplt_epi64_mask() { + let a = _mm512_set_epi64(0, 1, -1, 13, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set1_epi64(-1); + let m = _mm512_cmplt_epi64_mask(a, b); + assert_eq!(m, 0b00000101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmplt_epi64_mask() { + let a = _mm512_set_epi64(0, 1, -1, 13, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set1_epi64(-1); + let mask = 0b01100110; + let r = _mm512_mask_cmplt_epi64_mask(mask, a, b); + assert_eq!(r, 0b00000100); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmplt_epi64_mask() { + let a = _mm256_set_epi64x(0, 1, -1, -13); + let b = _mm256_set1_epi64x(-1); + let r = _mm256_cmplt_epi64_mask(a, b); + assert_eq!(r, 0b00000001); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmplt_epi64_mask() { + let a = _mm256_set_epi64x(0, 1, -1, -13); + let b = _mm256_set1_epi64x(-1); + let mask = 0b11111111; + let r = _mm256_mask_cmplt_epi64_mask(mask, a, b); + assert_eq!(r, 0b00000001); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmplt_epi64_mask() { + let a = _mm_set_epi64x(-1, -13); + let b = _mm_set1_epi64x(-1); + let r = _mm_cmplt_epi64_mask(a, b); + assert_eq!(r, 0b00000001); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmplt_epi64_mask() { + let a = _mm_set_epi64x(-1, -13); + let b = _mm_set1_epi64x(-1); + let mask = 0b11111111; + let r = _mm_mask_cmplt_epi64_mask(mask, a, b); + assert_eq!(r, 0b00000001); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpgt_epi64_mask() { + let a = _mm512_set_epi64(0, 1, -1, 13, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set1_epi64(-1); + let m = _mm512_cmpgt_epi64_mask(b, a); + assert_eq!(m, 0b00000101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpgt_epi64_mask() { + let a = _mm512_set_epi64(0, 1, -1, 13, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set1_epi64(-1); + let mask = 0b01100110; + let r = _mm512_mask_cmpgt_epi64_mask(mask, b, a); + assert_eq!(r, 0b00000100); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmpgt_epi64_mask() { + let a = _mm256_set_epi64x(0, 1, -1, 13); + let b = _mm256_set1_epi64x(-1); + let r = _mm256_cmpgt_epi64_mask(a, b); + assert_eq!(r, 0b00001101); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmpgt_epi64_mask() { + let a = _mm256_set_epi64x(0, 1, -1, 13); + let b = _mm256_set1_epi64x(-1); + let mask = 0b11111111; + let r = _mm256_mask_cmpgt_epi64_mask(mask, a, b); + assert_eq!(r, 0b00001101); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmpgt_epi64_mask() { + let a = _mm_set_epi64x(0, -1); + let b = _mm_set1_epi64x(-1); + let r = _mm_cmpgt_epi64_mask(a, b); + assert_eq!(r, 0b00000010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmpgt_epi64_mask() { + let a = _mm_set_epi64x(0, -1); + let b = _mm_set1_epi64x(-1); + let mask = 0b11111111; + let r = _mm_mask_cmpgt_epi64_mask(mask, a, b); + assert_eq!(r, 0b00000010); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmple_epi64_mask() { + let a = _mm512_set_epi64(0, 1, -1, u64::MAX as i64, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set1_epi64(-1); + assert_eq!( + _mm512_cmple_epi64_mask(a, b), + !_mm512_cmpgt_epi64_mask(a, b) + ) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmple_epi64_mask() { + let a = _mm512_set_epi64(0, 1, -1, u64::MAX as i64, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set1_epi64(-1); + let mask = 0b01111010; + assert_eq!(_mm512_mask_cmple_epi64_mask(mask, a, b), 0b00110000); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmple_epi64_mask() { + let a = _mm256_set_epi64x(0, 1, -1, i64::MAX); + let b = _mm256_set1_epi64x(-1); + let r = _mm256_cmple_epi64_mask(a, b); + assert_eq!(r, 0b00000010) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmple_epi64_mask() { + let a = _mm256_set_epi64x(0, 1, -1, i64::MAX); + let b = _mm256_set1_epi64x(-1); + let mask = 0b11111111; + let r = _mm256_mask_cmple_epi64_mask(mask, a, b); + assert_eq!(r, 0b00000010) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmple_epi64_mask() { + let a = _mm_set_epi64x(0, 1); + let b = _mm_set1_epi64x(1); + let r = _mm_cmple_epi64_mask(a, b); + assert_eq!(r, 0b00000011) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmple_epi64_mask() { + let a = _mm_set_epi64x(0, 1); + let b = _mm_set1_epi64x(1); + let mask = 0b11111111; + let r = _mm_mask_cmple_epi64_mask(mask, a, b); + assert_eq!(r, 0b00000011) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpge_epi64_mask() { + let a = _mm512_set_epi64(0, 1, -1, u64::MAX as i64, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set1_epi64(-1); + assert_eq!( + _mm512_cmpge_epi64_mask(a, b), + !_mm512_cmplt_epi64_mask(a, b) + ) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpge_epi64_mask() { + let a = _mm512_set_epi64(0, 1, -1, u64::MAX as i64, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set1_epi64(-1); + let mask = 0b11111111; + let r = _mm512_mask_cmpge_epi64_mask(mask, a, b); + assert_eq!(r, 0b11111010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmpge_epi64_mask() { + let a = _mm256_set_epi64x(0, 1, -1, i64::MAX); + let b = _mm256_set1_epi64x(-1); + let r = _mm256_cmpge_epi64_mask(a, b); + assert_eq!(r, 0b00001111); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmpge_epi64_mask() { + let a = _mm256_set_epi64x(0, 1, -1, i64::MAX); + let b = _mm256_set1_epi64x(-1); + let mask = 0b11111111; + let r = _mm256_mask_cmpge_epi64_mask(mask, a, b); + assert_eq!(r, 0b00001111); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmpge_epi64_mask() { + let a = _mm_set_epi64x(0, 1); + let b = _mm_set1_epi64x(-1); + let r = _mm_cmpge_epi64_mask(a, b); + assert_eq!(r, 0b00000011); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmpge_epi64_mask() { + let a = _mm_set_epi64x(0, 1); + let b = _mm_set1_epi64x(-1); + let mask = 0b11111111; + let r = _mm_mask_cmpge_epi64_mask(mask, a, b); + assert_eq!(r, 0b00000011); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmpeq_epi64_mask() { + let a = _mm512_set_epi64(0, 1, -1, 13, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set_epi64(0, 1, 13, 42, i64::MAX, i64::MIN, 100, -100); + let m = _mm512_cmpeq_epi64_mask(b, a); + assert_eq!(m, 0b11001111); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpeq_epi64_mask() { + let a = _mm512_set_epi64(0, 1, -1, 13, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set_epi64(0, 1, 13, 42, i64::MAX, i64::MIN, 100, -100); + let mask = 0b01111010; + let r = _mm512_mask_cmpeq_epi64_mask(mask, b, a); + assert_eq!(r, 0b01001010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmpeq_epi64_mask() { + let a = _mm256_set_epi64x(0, 1, -1, 13); + let b = _mm256_set_epi64x(0, 1, 13, 42); + let m = _mm256_cmpeq_epi64_mask(b, a); + assert_eq!(m, 0b00001100); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmpeq_epi64_mask() { + let a = _mm256_set_epi64x(0, 1, -1, 13); + let b = _mm256_set_epi64x(0, 1, 13, 42); + let mask = 0b11111111; + let r = _mm256_mask_cmpeq_epi64_mask(mask, b, a); + assert_eq!(r, 0b00001100); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmpeq_epi64_mask() { + let a = _mm_set_epi64x(0, 1); + let b = _mm_set_epi64x(0, 1); + let m = _mm_cmpeq_epi64_mask(b, a); + assert_eq!(m, 0b00000011); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmpeq_epi64_mask() { + let a = _mm_set_epi64x(0, 1); + let b = _mm_set_epi64x(0, 1); + let mask = 0b11111111; + let r = _mm_mask_cmpeq_epi64_mask(mask, b, a); + assert_eq!(r, 0b00000011); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_set_epi64() { + let r = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); + assert_eq_m512i(r, _mm512_set_epi64(7, 6, 5, 4, 3, 2, 1, 0)) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_setr_epi64() { + let r = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + assert_eq_m512i(r, _mm512_setr_epi64(7, 6, 5, 4, 3, 2, 1, 0)) + } + + unsafe fn test_mm512_cmpneq_epi64_mask() { + let a = _mm512_set_epi64(0, 1, -1, 13, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set_epi64(0, 1, 13, 42, i64::MAX, i64::MIN, 100, -100); + let m = _mm512_cmpneq_epi64_mask(b, a); + assert_eq!(m, !_mm512_cmpeq_epi64_mask(b, a)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmpneq_epi64_mask() { + let a = _mm512_set_epi64(0, 1, -1, 13, i64::MAX, i64::MIN, -100, 100); + let b = _mm512_set_epi64(0, 1, 13, 42, i64::MAX, i64::MIN, 100, -100); + let mask = 0b01111010; + let r = _mm512_mask_cmpneq_epi64_mask(mask, b, a); + assert_eq!(r, 0b00110010) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmpneq_epi64_mask() { + let a = _mm256_set_epi64x(0, 1, -1, 13); + let b = _mm256_set_epi64x(0, 1, 13, 42); + let r = _mm256_cmpneq_epi64_mask(b, a); + assert_eq!(r, 0b00000011) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmpneq_epi64_mask() { + let a = _mm256_set_epi64x(0, 1, -1, 13); + let b = _mm256_set_epi64x(0, 1, 13, 42); + let mask = 0b11111111; + let r = _mm256_mask_cmpneq_epi64_mask(mask, b, a); + assert_eq!(r, 0b00000011) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmpneq_epi64_mask() { + let a = _mm_set_epi64x(-1, 13); + let b = _mm_set_epi64x(13, 42); + let r = _mm_cmpneq_epi64_mask(b, a); + assert_eq!(r, 0b00000011) + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmpneq_epi64_mask() { + let a = _mm_set_epi64x(-1, 13); + let b = _mm_set_epi64x(13, 42); + let mask = 0b11111111; + let r = _mm_mask_cmpneq_epi64_mask(mask, b, a); + assert_eq!(r, 0b00000011) + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_cmp_epi64_mask() { + let a = _mm512_set_epi64(0, 1, -1, 13, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set1_epi64(-1); + let m = _mm512_cmp_epi64_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b00000101); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cmp_epi64_mask() { + let a = _mm512_set_epi64(0, 1, -1, 13, i64::MAX, i64::MIN, 100, -100); + let b = _mm512_set1_epi64(-1); + let mask = 0b01100110; + let r = _mm512_mask_cmp_epi64_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b00000100); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_cmp_epi64_mask() { + let a = _mm256_set_epi64x(0, 1, -1, 13); + let b = _mm256_set1_epi64x(1); + let m = _mm256_cmp_epi64_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b00001010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cmp_epi64_mask() { + let a = _mm256_set_epi64x(0, 1, -1, 13); + let b = _mm256_set1_epi64x(1); + let mask = 0b11111111; + let r = _mm256_mask_cmp_epi64_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b00001010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_cmp_epi64_mask() { + let a = _mm_set_epi64x(0, 1); + let b = _mm_set1_epi64x(1); + let m = _mm_cmp_epi64_mask::<_MM_CMPINT_LT>(a, b); + assert_eq!(m, 0b00000010); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cmp_epi64_mask() { + let a = _mm_set_epi64x(0, 1); + let b = _mm_set1_epi64x(1); + let mask = 0b11111111; + let r = _mm_mask_cmp_epi64_mask::<_MM_CMPINT_LT>(mask, a, b); + assert_eq!(r, 0b00000010); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_i32gather_pd() { + let mut arr = [0f64; 128]; + for i in 0..128 { + arr[i] = i as f64; + } + // A multiplier of 8 is word-addressing + let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); + let r = _mm512_i32gather_pd::<8>(index, arr.as_ptr() as *const u8); + assert_eq_m512d(r, _mm512_setr_pd(0., 16., 32., 48., 64., 80., 96., 112.)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_i32gather_pd() { + let mut arr = [0f64; 128]; + for i in 0..128 { + arr[i] = i as f64; + } + let src = _mm512_set1_pd(2.); + let mask = 0b10101010; + let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); + // A multiplier of 8 is word-addressing + let r = _mm512_mask_i32gather_pd::<8>(src, mask, index, arr.as_ptr() as *const u8); + assert_eq_m512d(r, _mm512_setr_pd(2., 16., 2., 48., 2., 80., 2., 112.)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_i64gather_pd() { + let mut arr = [0f64; 128]; + for i in 0..128 { + arr[i] = i as f64; + } + // A multiplier of 8 is word-addressing + let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); + let r = _mm512_i64gather_pd::<8>(index, arr.as_ptr() as *const u8); + assert_eq_m512d(r, _mm512_setr_pd(0., 16., 32., 48., 64., 80., 96., 112.)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_i64gather_pd() { + let mut arr = [0f64; 128]; + for i in 0..128 { + arr[i] = i as f64; + } + let src = _mm512_set1_pd(2.); + let mask = 0b10101010; + let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); + // A multiplier of 8 is word-addressing + let r = _mm512_mask_i64gather_pd::<8>(src, mask, index, arr.as_ptr() as *const u8); + assert_eq_m512d(r, _mm512_setr_pd(2., 16., 2., 48., 2., 80., 2., 112.)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_i64gather_ps() { + let mut arr = [0f32; 128]; + for i in 0..128 { + arr[i] = i as f32; + } + // A multiplier of 4 is word-addressing + #[rustfmt::skip] + let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); + let r = _mm512_i64gather_ps::<4>(index, arr.as_ptr() as *const u8); + assert_eq_m256(r, _mm256_setr_ps(0., 16., 32., 48., 64., 80., 96., 112.)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_i64gather_ps() { + let mut arr = [0f32; 128]; + for i in 0..128 { + arr[i] = i as f32; + } + let src = _mm256_set1_ps(2.); + let mask = 0b10101010; + #[rustfmt::skip] + let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); + // A multiplier of 4 is word-addressing + let r = _mm512_mask_i64gather_ps::<4>(src, mask, index, arr.as_ptr() as *const u8); + assert_eq_m256(r, _mm256_setr_ps(2., 16., 2., 48., 2., 80., 2., 112.)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_i32gather_epi64() { + let mut arr = [0i64; 128]; + for i in 0..128i64 { + arr[i as usize] = i; + } + // A multiplier of 8 is word-addressing + let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); + let r = _mm512_i32gather_epi64::<8>(index, arr.as_ptr() as *const u8); + assert_eq_m512i(r, _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_i32gather_epi64() { + let mut arr = [0i64; 128]; + for i in 0..128i64 { + arr[i as usize] = i; + } + let src = _mm512_set1_epi64(2); + let mask = 0b10101010; + let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); + // A multiplier of 8 is word-addressing + let r = _mm512_mask_i32gather_epi64::<8>(src, mask, index, arr.as_ptr() as *const u8); + assert_eq_m512i(r, _mm512_setr_epi64(2, 16, 2, 48, 2, 80, 2, 112)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_i64gather_epi64() { + let mut arr = [0i64; 128]; + for i in 0..128i64 { + arr[i as usize] = i; + } + // A multiplier of 8 is word-addressing + let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); + let r = _mm512_i64gather_epi64::<8>(index, arr.as_ptr() as *const u8); + assert_eq_m512i(r, _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_i64gather_epi64() { + let mut arr = [0i64; 128]; + for i in 0..128i64 { + arr[i as usize] = i; + } + let src = _mm512_set1_epi64(2); + let mask = 0b10101010; + let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); + // A multiplier of 8 is word-addressing + let r = _mm512_mask_i64gather_epi64::<8>(src, mask, index, arr.as_ptr() as *const u8); + assert_eq_m512i(r, _mm512_setr_epi64(2, 16, 2, 48, 2, 80, 2, 112)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_i64gather_epi32() { + let mut arr = [0i64; 128]; + for i in 0..128i64 { + arr[i as usize] = i; + } + // A multiplier of 8 is word-addressing + let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); + let r = _mm512_i64gather_epi32::<8>(index, arr.as_ptr() as *const u8); + assert_eq_m256i(r, _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_i64gather_epi32() { + let mut arr = [0i64; 128]; + for i in 0..128i64 { + arr[i as usize] = i; + } + let src = _mm256_set1_epi32(2); + let mask = 0b10101010; + let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); + // A multiplier of 8 is word-addressing + let r = _mm512_mask_i64gather_epi32::<8>(src, mask, index, arr.as_ptr() as *const u8); + assert_eq_m256i(r, _mm256_setr_epi32(2, 16, 2, 48, 2, 80, 2, 112)); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_i32scatter_pd() { + let mut arr = [0f64; 128]; + let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); + let src = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); + // A multiplier of 8 is word-addressing + _mm512_i32scatter_pd::<8>(arr.as_mut_ptr() as *mut u8, index, src); + let mut expected = [0f64; 128]; + for i in 0..8 { + expected[i * 16] = (i + 1) as f64; + } + assert_eq!(&arr[..], &expected[..],); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_i32scatter_pd() { + let mut arr = [0f64; 128]; + let mask = 0b10101010; + let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); + let src = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); + // A multiplier of 8 is word-addressing + _mm512_mask_i32scatter_pd::<8>(arr.as_mut_ptr() as *mut u8, mask, index, src); + let mut expected = [0f64; 128]; + for i in 0..4 { + expected[i * 32 + 16] = 2. * (i + 1) as f64; + } + assert_eq!(&arr[..], &expected[..],); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_i64scatter_pd() { + let mut arr = [0f64; 128]; + let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); + let src = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); + // A multiplier of 8 is word-addressing + _mm512_i64scatter_pd::<8>(arr.as_mut_ptr() as *mut u8, index, src); + let mut expected = [0f64; 128]; + for i in 0..8 { + expected[i * 16] = (i + 1) as f64; + } + assert_eq!(&arr[..], &expected[..],); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_i64scatter_pd() { + let mut arr = [0f64; 128]; + let mask = 0b10101010; + let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); + let src = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); + // A multiplier of 8 is word-addressing + _mm512_mask_i64scatter_pd::<8>(arr.as_mut_ptr() as *mut u8, mask, index, src); + let mut expected = [0f64; 128]; + for i in 0..4 { + expected[i * 32 + 16] = 2. * (i + 1) as f64; + } + assert_eq!(&arr[..], &expected[..],); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_i64scatter_ps() { + let mut arr = [0f32; 128]; + let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); + let src = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); + // A multiplier of 4 is word-addressing + _mm512_i64scatter_ps::<4>(arr.as_mut_ptr() as *mut u8, index, src); + let mut expected = [0f32; 128]; + for i in 0..8 { + expected[i * 16] = (i + 1) as f32; + } + assert_eq!(&arr[..], &expected[..],); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_i64scatter_ps() { + let mut arr = [0f32; 128]; + let mask = 0b10101010; + let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); + let src = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); + // A multiplier of 4 is word-addressing + _mm512_mask_i64scatter_ps::<4>(arr.as_mut_ptr() as *mut u8, mask, index, src); + let mut expected = [0f32; 128]; + for i in 0..4 { + expected[i * 32 + 16] = 2. * (i + 1) as f32; + } + assert_eq!(&arr[..], &expected[..],); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_i32scatter_epi64() { + let mut arr = [0i64; 128]; + let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); + let src = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); + // A multiplier of 8 is word-addressing + _mm512_i32scatter_epi64::<8>(arr.as_mut_ptr() as *mut u8, index, src); + let mut expected = [0i64; 128]; + for i in 0..8 { + expected[i * 16] = (i + 1) as i64; + } + assert_eq!(&arr[..], &expected[..],); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_i32scatter_epi64() { + let mut arr = [0i64; 128]; + let mask = 0b10101010; + let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); + let src = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); + // A multiplier of 8 is word-addressing + _mm512_mask_i32scatter_epi64::<8>(arr.as_mut_ptr() as *mut u8, mask, index, src); + let mut expected = [0i64; 128]; + for i in 0..4 { + expected[i * 32 + 16] = 2 * (i + 1) as i64; + } + assert_eq!(&arr[..], &expected[..],); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_i64scatter_epi64() { + let mut arr = [0i64; 128]; + let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); + let src = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); + // A multiplier of 8 is word-addressing + _mm512_i64scatter_epi64::<8>(arr.as_mut_ptr() as *mut u8, index, src); + let mut expected = [0i64; 128]; + for i in 0..8 { + expected[i * 16] = (i + 1) as i64; + } + assert_eq!(&arr[..], &expected[..],); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_i64scatter_epi64() { + let mut arr = [0i64; 128]; + let mask = 0b10101010; + let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); + let src = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); + // A multiplier of 8 is word-addressing + _mm512_mask_i64scatter_epi64::<8>(arr.as_mut_ptr() as *mut u8, mask, index, src); + let mut expected = [0i64; 128]; + for i in 0..4 { + expected[i * 32 + 16] = 2 * (i + 1) as i64; + } + assert_eq!(&arr[..], &expected[..],); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_i64scatter_epi32() { + let mut arr = [0i32; 128]; + let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); + let src = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); + // A multiplier of 4 is word-addressing + _mm512_i64scatter_epi32::<4>(arr.as_mut_ptr() as *mut u8, index, src); + let mut expected = [0i32; 128]; + for i in 0..8 { + expected[i * 16] = (i + 1) as i32; + } + assert_eq!(&arr[..], &expected[..],); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_i64scatter_epi32() { + let mut arr = [0i32; 128]; + let mask = 0b10101010; + let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); + let src = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); + // A multiplier of 4 is word-addressing + _mm512_mask_i64scatter_epi32::<4>(arr.as_mut_ptr() as *mut u8, mask, index, src); + let mut expected = [0i32; 128]; + for i in 0..4 { + expected[i * 32 + 16] = 2 * (i + 1) as i32; + } + assert_eq!(&arr[..], &expected[..],); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_rol_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 63, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + ); + let r = _mm512_rol_epi64::<1>(a); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 1 << 0, 1 << 33, 1 << 33, 1 << 33, + 1 << 33, 1 << 33, 1 << 33, 1 << 33, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_rol_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 63, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + ); + let r = _mm512_mask_rol_epi64::<1>(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_rol_epi64::<1>(a, 0b11111111, a); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 1 << 0, 1 << 33, 1 << 33, 1 << 33, + 1 << 33, 1 << 33, 1 << 33, 1 << 33, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_rol_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 63, + ); + let r = _mm512_maskz_rol_epi64::<1>(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_rol_epi64::<1>(0b00001111, a); + let e = _mm512_set_epi64(0, 0, 0, 0, 1 << 33, 1 << 33, 1 << 33, 1 << 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_rol_epi64() { + let a = _mm256_set_epi64x(1 << 63, 1 << 32, 1 << 32, 1 << 32); + let r = _mm256_rol_epi64::<1>(a); + let e = _mm256_set_epi64x(1 << 0, 1 << 33, 1 << 33, 1 << 33); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_rol_epi64() { + let a = _mm256_set_epi64x(1 << 63, 1 << 32, 1 << 32, 1 << 32); + let r = _mm256_mask_rol_epi64::<1>(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_rol_epi64::<1>(a, 0b00001111, a); + let e = _mm256_set_epi64x(1 << 0, 1 << 33, 1 << 33, 1 << 33); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_rol_epi64() { + let a = _mm256_set_epi64x(1 << 63, 1 << 32, 1 << 32, 1 << 32); + let r = _mm256_maskz_rol_epi64::<1>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_rol_epi64::<1>(0b00001111, a); + let e = _mm256_set_epi64x(1 << 0, 1 << 33, 1 << 33, 1 << 33); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_rol_epi64() { + let a = _mm_set_epi64x(1 << 63, 1 << 32); + let r = _mm_rol_epi64::<1>(a); + let e = _mm_set_epi64x(1 << 0, 1 << 33); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_rol_epi64() { + let a = _mm_set_epi64x(1 << 63, 1 << 32); + let r = _mm_mask_rol_epi64::<1>(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_rol_epi64::<1>(a, 0b00000011, a); + let e = _mm_set_epi64x(1 << 0, 1 << 33); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_rol_epi64() { + let a = _mm_set_epi64x(1 << 63, 1 << 32); + let r = _mm_maskz_rol_epi64::<1>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_rol_epi64::<1>(0b00000011, a); + let e = _mm_set_epi64x(1 << 0, 1 << 33); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_ror_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 0, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + ); + let r = _mm512_ror_epi64::<1>(a); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 1 << 63, 1 << 31, 1 << 31, 1 << 31, + 1 << 31, 1 << 31, 1 << 31, 1 << 31, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_ror_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 0, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + ); + let r = _mm512_mask_ror_epi64::<1>(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_ror_epi64::<1>(a, 0b11111111, a); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 1 << 63, 1 << 31, 1 << 31, 1 << 31, + 1 << 31, 1 << 31, 1 << 31, 1 << 31, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_ror_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 0, + ); + let r = _mm512_maskz_ror_epi64::<1>(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_ror_epi64::<1>(0b00001111, a); + let e = _mm512_set_epi64(0, 0, 0, 0, 1 << 31, 1 << 31, 1 << 31, 1 << 63); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_ror_epi64() { + let a = _mm256_set_epi64x(1 << 0, 1 << 32, 1 << 32, 1 << 32); + let r = _mm256_ror_epi64::<1>(a); + let e = _mm256_set_epi64x(1 << 63, 1 << 31, 1 << 31, 1 << 31); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_ror_epi64() { + let a = _mm256_set_epi64x(1 << 0, 1 << 32, 1 << 32, 1 << 32); + let r = _mm256_mask_ror_epi64::<1>(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_ror_epi64::<1>(a, 0b00001111, a); + let e = _mm256_set_epi64x(1 << 63, 1 << 31, 1 << 31, 1 << 31); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_ror_epi64() { + let a = _mm256_set_epi64x(1 << 0, 1 << 32, 1 << 32, 1 << 32); + let r = _mm256_maskz_ror_epi64::<1>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_ror_epi64::<1>(0b00001111, a); + let e = _mm256_set_epi64x(1 << 63, 1 << 31, 1 << 31, 1 << 31); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_ror_epi64() { + let a = _mm_set_epi64x(1 << 0, 1 << 32); + let r = _mm_ror_epi64::<1>(a); + let e = _mm_set_epi64x(1 << 63, 1 << 31); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_ror_epi64() { + let a = _mm_set_epi64x(1 << 0, 1 << 32); + let r = _mm_mask_ror_epi64::<1>(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_ror_epi64::<1>(a, 0b00000011, a); + let e = _mm_set_epi64x(1 << 63, 1 << 31); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_ror_epi64() { + let a = _mm_set_epi64x(1 << 0, 1 << 32); + let r = _mm_maskz_ror_epi64::<1>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_ror_epi64::<1>(0b00000011, a); + let e = _mm_set_epi64x(1 << 63, 1 << 31); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_slli_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 63, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + ); + let r = _mm512_slli_epi64::<1>(a); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 0, 1 << 33, 1 << 33, 1 << 33, + 1 << 33, 1 << 33, 1 << 33, 1 << 33, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_slli_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 63, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + ); + let r = _mm512_mask_slli_epi64::<1>(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_slli_epi64::<1>(a, 0b11111111, a); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 0, 1 << 33, 1 << 33, 1 << 33, + 1 << 33, 1 << 33, 1 << 33, 1 << 33, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_slli_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 63, + ); + let r = _mm512_maskz_slli_epi64::<1>(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_slli_epi64::<1>(0b00001111, a); + let e = _mm512_set_epi64(0, 0, 0, 0, 1 << 33, 1 << 33, 1 << 33, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_slli_epi64() { + let a = _mm256_set_epi64x(1 << 63, 1 << 32, 1 << 32, 1 << 32); + let r = _mm256_mask_slli_epi64::<1>(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_slli_epi64::<1>(a, 0b00001111, a); + let e = _mm256_set_epi64x(0, 1 << 33, 1 << 33, 1 << 33); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_slli_epi64() { + let a = _mm256_set_epi64x(1 << 63, 1 << 32, 1 << 32, 1 << 32); + let r = _mm256_maskz_slli_epi64::<1>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_slli_epi64::<1>(0b00001111, a); + let e = _mm256_set_epi64x(0, 1 << 33, 1 << 33, 1 << 33); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_slli_epi64() { + let a = _mm_set_epi64x(1 << 63, 1 << 32); + let r = _mm_mask_slli_epi64::<1>(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_slli_epi64::<1>(a, 0b00000011, a); + let e = _mm_set_epi64x(0, 1 << 33); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_slli_epi64() { + let a = _mm_set_epi64x(1 << 63, 1 << 32); + let r = _mm_maskz_slli_epi64::<1>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_slli_epi64::<1>(0b00000011, a); + let e = _mm_set_epi64x(0, 1 << 33); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_srli_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 0, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + ); + let r = _mm512_srli_epi64::<1>(a); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 0, 1 << 31, 1 << 31, 1 << 31, + 1 << 31, 1 << 31, 1 << 31, 1 << 31, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_srli_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 0, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + ); + let r = _mm512_mask_srli_epi64::<1>(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_srli_epi64::<1>(a, 0b11111111, a); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 0, 1 << 31, 1 << 31, 1 << 31, + 1 << 31, 1 << 31, 1 << 31, 1 << 31, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_srli_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 0, + ); + let r = _mm512_maskz_srli_epi64::<1>(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_srli_epi64::<1>(0b00001111, a); + let e = _mm512_set_epi64(0, 0, 0, 0, 1 << 31, 1 << 31, 1 << 31, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_srli_epi64() { + let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); + let r = _mm256_mask_srli_epi64::<1>(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_srli_epi64::<1>(a, 0b00001111, a); + let e = _mm256_set_epi64x(1 << 4, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_srli_epi64() { + let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); + let r = _mm256_maskz_srli_epi64::<1>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_srli_epi64::<1>(0b00001111, a); + let e = _mm256_set_epi64x(1 << 4, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_srli_epi64() { + let a = _mm_set_epi64x(1 << 5, 0); + let r = _mm_mask_srli_epi64::<1>(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_srli_epi64::<1>(a, 0b00000011, a); + let e = _mm_set_epi64x(1 << 4, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_srli_epi64() { + let a = _mm_set_epi64x(1 << 5, 0); + let r = _mm_maskz_srli_epi64::<1>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_srli_epi64::<1>(0b00000011, a); + let e = _mm_set_epi64x(1 << 4, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_rolv_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 32, 1 << 63, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + ); + let b = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm512_rolv_epi64(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 1 << 32, 1 << 0, 1 << 34, 1 << 35, + 1 << 36, 1 << 37, 1 << 38, 1 << 39, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_rolv_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 32, 1 << 63, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + ); + let b = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm512_mask_rolv_epi64(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_rolv_epi64(a, 0b11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 1 << 32, 1 << 0, 1 << 34, 1 << 35, + 1 << 36, 1 << 37, 1 << 38, 1 << 39, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_rolv_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 62, + ); + let b = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 2); + let r = _mm512_maskz_rolv_epi64(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_rolv_epi64(0b00001111, a, b); + let e = _mm512_set_epi64(0, 0, 0, 0, 1 << 36, 1 << 37, 1 << 38, 1 << 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_rolv_epi64() { + let a = _mm256_set_epi64x(1 << 32, 1 << 63, 1 << 32, 1 << 32); + let b = _mm256_set_epi64x(0, 1, 2, 3); + let r = _mm256_rolv_epi64(a, b); + let e = _mm256_set_epi64x(1 << 32, 1 << 0, 1 << 34, 1 << 35); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_rolv_epi64() { + let a = _mm256_set_epi64x(1 << 32, 1 << 63, 1 << 32, 1 << 32); + let b = _mm256_set_epi64x(0, 1, 2, 3); + let r = _mm256_mask_rolv_epi64(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_rolv_epi64(a, 0b00001111, a, b); + let e = _mm256_set_epi64x(1 << 32, 1 << 0, 1 << 34, 1 << 35); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_rolv_epi64() { + let a = _mm256_set_epi64x(1 << 32, 1 << 63, 1 << 32, 1 << 32); + let b = _mm256_set_epi64x(0, 1, 2, 3); + let r = _mm256_maskz_rolv_epi64(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_rolv_epi64(0b00001111, a, b); + let e = _mm256_set_epi64x(1 << 32, 1 << 0, 1 << 34, 1 << 35); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_rolv_epi64() { + let a = _mm_set_epi64x(1 << 32, 1 << 63); + let b = _mm_set_epi64x(0, 1); + let r = _mm_rolv_epi64(a, b); + let e = _mm_set_epi64x(1 << 32, 1 << 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_rolv_epi64() { + let a = _mm_set_epi64x(1 << 32, 1 << 63); + let b = _mm_set_epi64x(0, 1); + let r = _mm_mask_rolv_epi64(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_rolv_epi64(a, 0b00000011, a, b); + let e = _mm_set_epi64x(1 << 32, 1 << 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_rolv_epi64() { + let a = _mm_set_epi64x(1 << 32, 1 << 63); + let b = _mm_set_epi64x(0, 1); + let r = _mm_maskz_rolv_epi64(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_rolv_epi64(0b00000011, a, b); + let e = _mm_set_epi64x(1 << 32, 1 << 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_rorv_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 32, 1 << 0, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + ); + let b = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm512_rorv_epi64(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 1 << 32, 1 << 63, 1 << 30, 1 << 29, + 1 << 28, 1 << 27, 1 << 26, 1 << 25, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_rorv_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 32, 1 << 0, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + ); + let b = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm512_mask_rorv_epi64(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_rorv_epi64(a, 0b11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 1 << 32, 1 << 63, 1 << 30, 1 << 29, + 1 << 28, 1 << 27, 1 << 26, 1 << 25, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_rorv_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 0, + ); + let b = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 2); + let r = _mm512_maskz_rorv_epi64(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_rorv_epi64(0b00001111, a, b); + let e = _mm512_set_epi64(0, 0, 0, 0, 1 << 28, 1 << 27, 1 << 26, 1 << 62); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_rorv_epi64() { + let a = _mm256_set_epi64x(1 << 32, 1 << 0, 1 << 32, 1 << 32); + let b = _mm256_set_epi64x(0, 1, 2, 3); + let r = _mm256_rorv_epi64(a, b); + let e = _mm256_set_epi64x(1 << 32, 1 << 63, 1 << 30, 1 << 29); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_rorv_epi64() { + let a = _mm256_set_epi64x(1 << 32, 1 << 0, 1 << 32, 1 << 32); + let b = _mm256_set_epi64x(0, 1, 2, 3); + let r = _mm256_mask_rorv_epi64(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_rorv_epi64(a, 0b00001111, a, b); + let e = _mm256_set_epi64x(1 << 32, 1 << 63, 1 << 30, 1 << 29); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_rorv_epi64() { + let a = _mm256_set_epi64x(1 << 32, 1 << 0, 1 << 32, 1 << 32); + let b = _mm256_set_epi64x(0, 1, 2, 3); + let r = _mm256_maskz_rorv_epi64(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_rorv_epi64(0b00001111, a, b); + let e = _mm256_set_epi64x(1 << 32, 1 << 63, 1 << 30, 1 << 29); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_rorv_epi64() { + let a = _mm_set_epi64x(1 << 32, 1 << 0); + let b = _mm_set_epi64x(0, 1); + let r = _mm_rorv_epi64(a, b); + let e = _mm_set_epi64x(1 << 32, 1 << 63); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_rorv_epi64() { + let a = _mm_set_epi64x(1 << 32, 1 << 0); + let b = _mm_set_epi64x(0, 1); + let r = _mm_mask_rorv_epi64(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_rorv_epi64(a, 0b00000011, a, b); + let e = _mm_set_epi64x(1 << 32, 1 << 63); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_rorv_epi64() { + let a = _mm_set_epi64x(1 << 32, 1 << 0); + let b = _mm_set_epi64x(0, 1); + let r = _mm_maskz_rorv_epi64(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_rorv_epi64(0b00000011, a, b); + let e = _mm_set_epi64x(1 << 32, 1 << 63); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_sllv_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 32, 1 << 63, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + ); + let count = _mm512_set_epi64(0, 2, 2, 3, 4, 5, 6, 7); + let r = _mm512_sllv_epi64(a, count); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 1 << 32, 0, 1 << 34, 1 << 35, + 1 << 36, 1 << 37, 1 << 38, 1 << 39, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_sllv_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 32, 1 << 32, 1 << 63, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + ); + let count = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm512_mask_sllv_epi64(a, 0, a, count); + assert_eq_m512i(r, a); + let r = _mm512_mask_sllv_epi64(a, 0b11111111, a, count); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 1 << 32, 1 << 33, 0, 1 << 35, + 1 << 36, 1 << 37, 1 << 38, 1 << 39, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_sllv_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 63, + ); + let count = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 1); + let r = _mm512_maskz_sllv_epi64(0, a, count); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_sllv_epi64(0b00001111, a, count); + let e = _mm512_set_epi64(0, 0, 0, 0, 1 << 36, 1 << 37, 1 << 38, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_sllv_epi64() { + let a = _mm256_set_epi64x(1 << 32, 1 << 32, 1 << 63, 1 << 32); + let count = _mm256_set_epi64x(0, 1, 2, 3); + let r = _mm256_mask_sllv_epi64(a, 0, a, count); + assert_eq_m256i(r, a); + let r = _mm256_mask_sllv_epi64(a, 0b00001111, a, count); + let e = _mm256_set_epi64x(1 << 32, 1 << 33, 0, 1 << 35); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_sllv_epi64() { + let a = _mm256_set_epi64x(1 << 32, 1 << 32, 1 << 63, 1 << 32); + let count = _mm256_set_epi64x(0, 1, 2, 3); + let r = _mm256_maskz_sllv_epi64(0, a, count); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_sllv_epi64(0b00001111, a, count); + let e = _mm256_set_epi64x(1 << 32, 1 << 33, 0, 1 << 35); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_sllv_epi64() { + let a = _mm_set_epi64x(1 << 63, 1 << 32); + let count = _mm_set_epi64x(2, 3); + let r = _mm_mask_sllv_epi64(a, 0, a, count); + assert_eq_m128i(r, a); + let r = _mm_mask_sllv_epi64(a, 0b00000011, a, count); + let e = _mm_set_epi64x(0, 1 << 35); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_sllv_epi64() { + let a = _mm_set_epi64x(1 << 63, 1 << 32); + let count = _mm_set_epi64x(2, 3); + let r = _mm_maskz_sllv_epi64(0, a, count); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_sllv_epi64(0b00000011, a, count); + let e = _mm_set_epi64x(0, 1 << 35); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_srlv_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 32, 1 << 0, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + ); + let count = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm512_srlv_epi64(a, count); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 1 << 32, 0, 1 << 30, 1 << 29, + 1 << 28, 1 << 27, 1 << 26, 1 << 25, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_srlv_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 32, 1 << 0, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + ); + let count = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm512_mask_srlv_epi64(a, 0, a, count); + assert_eq_m512i(r, a); + let r = _mm512_mask_srlv_epi64(a, 0b11111111, a, count); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 1 << 32, 0, 1 << 30, 1 << 29, + 1 << 28, 1 << 27, 1 << 26, 1 << 25, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_srlv_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 0, + ); + let count = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm512_maskz_srlv_epi64(0, a, count); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_srlv_epi64(0b00001111, a, count); + let e = _mm512_set_epi64(0, 0, 0, 0, 1 << 28, 1 << 27, 1 << 26, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_srlv_epi64() { + let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); + let count = _mm256_set1_epi64x(1); + let r = _mm256_mask_srlv_epi64(a, 0, a, count); + assert_eq_m256i(r, a); + let r = _mm256_mask_srlv_epi64(a, 0b00001111, a, count); + let e = _mm256_set_epi64x(1 << 4, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_srlv_epi64() { + let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); + let count = _mm256_set1_epi64x(1); + let r = _mm256_maskz_srlv_epi64(0, a, count); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_srlv_epi64(0b00001111, a, count); + let e = _mm256_set_epi64x(1 << 4, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_srlv_epi64() { + let a = _mm_set_epi64x(1 << 5, 0); + let count = _mm_set1_epi64x(1); + let r = _mm_mask_srlv_epi64(a, 0, a, count); + assert_eq_m128i(r, a); + let r = _mm_mask_srlv_epi64(a, 0b00000011, a, count); + let e = _mm_set_epi64x(1 << 4, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_srlv_epi64() { + let a = _mm_set_epi64x(1 << 5, 0); + let count = _mm_set1_epi64x(1); + let r = _mm_maskz_srlv_epi64(0, a, count); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_srlv_epi64(0b00000011, a, count); + let e = _mm_set_epi64x(1 << 4, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_sll_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 63, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + ); + let count = _mm_set_epi64x(0, 1); + let r = _mm512_sll_epi64(a, count); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 0, 1 << 33, 1 << 33, 1 << 33, + 1 << 33, 1 << 33, 1 << 33, 1 << 33, + ); + assert_eq_m512i(r, e); + let count = _mm_set_epi64x(1, 0); + let r = _mm512_sll_epi64(a, count); + assert_eq_m512i(r, a); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_sll_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 63, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + ); + let count = _mm_set_epi64x(0, 1); + let r = _mm512_mask_sll_epi64(a, 0, a, count); + assert_eq_m512i(r, a); + let r = _mm512_mask_sll_epi64(a, 0b11111111, a, count); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 0, 1 << 33, 1 << 33, 1 << 33, + 1 << 33, 1 << 33, 1 << 33, 1 << 33, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_sll_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 63, + ); + let count = _mm_set_epi64x(0, 1); + let r = _mm512_maskz_sll_epi64(0, a, count); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_sll_epi64(0b00001111, a, count); + let e = _mm512_set_epi64(0, 0, 0, 0, 1 << 33, 1 << 33, 1 << 33, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_sll_epi64() { + let a = _mm256_set_epi64x(1 << 63, 1 << 32, 1 << 32, 1 << 32); + let count = _mm_set_epi64x(0, 1); + let r = _mm256_mask_sll_epi64(a, 0, a, count); + assert_eq_m256i(r, a); + let r = _mm256_mask_sll_epi64(a, 0b00001111, a, count); + let e = _mm256_set_epi64x(0, 1 << 33, 1 << 33, 1 << 33); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_sll_epi64() { + let a = _mm256_set_epi64x(1 << 63, 1 << 32, 1 << 32, 1 << 32); + let count = _mm_set_epi64x(0, 1); + let r = _mm256_maskz_sll_epi64(0, a, count); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_sll_epi64(0b00001111, a, count); + let e = _mm256_set_epi64x(0, 1 << 33, 1 << 33, 1 << 33); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_sll_epi64() { + let a = _mm_set_epi64x(1 << 63, 1 << 32); + let count = _mm_set_epi64x(0, 1); + let r = _mm_mask_sll_epi64(a, 0, a, count); + assert_eq_m128i(r, a); + let r = _mm_mask_sll_epi64(a, 0b00000011, a, count); + let e = _mm_set_epi64x(0, 1 << 33); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_sll_epi64() { + let a = _mm_set_epi64x(1 << 63, 1 << 32); + let count = _mm_set_epi64x(0, 1); + let r = _mm_maskz_sll_epi64(0, a, count); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_sll_epi64(0b00000011, a, count); + let e = _mm_set_epi64x(0, 1 << 33); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_srl_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 0, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + ); + let count = _mm_set_epi64x(0, 1); + let r = _mm512_srl_epi64(a, count); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 0, 1 << 31, 1 << 31, 1 << 31, + 1 << 31, 1 << 31, 1 << 31, 1 << 31, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_srl_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 0, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + ); + let count = _mm_set_epi64x(0, 1); + let r = _mm512_mask_srl_epi64(a, 0, a, count); + assert_eq_m512i(r, a); + let r = _mm512_mask_srl_epi64(a, 0b11111111, a, count); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 0, 1 << 31, 1 << 31, 1 << 31, + 1 << 31, 1 << 31, 1 << 31, 1 << 31, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_srl_epi64() { + #[rustfmt::skip] + let a = _mm512_set_epi64( + 1 << 32, 1 << 32, 1 << 32, 1 << 32, + 1 << 32, 1 << 32, 1 << 32, 1 << 0, + ); + let count = _mm_set_epi64x(0, 1); + let r = _mm512_maskz_srl_epi64(0, a, count); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_srl_epi64(0b00001111, a, count); + let e = _mm512_set_epi64(0, 0, 0, 0, 1 << 31, 1 << 31, 1 << 31, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_srl_epi64() { + let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); + let count = _mm_set_epi64x(0, 1); + let r = _mm256_mask_srl_epi64(a, 0, a, count); + assert_eq_m256i(r, a); + let r = _mm256_mask_srl_epi64(a, 0b00001111, a, count); + let e = _mm256_set_epi64x(1 << 4, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_srl_epi64() { + let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); + let count = _mm_set_epi64x(0, 1); + let r = _mm256_maskz_srl_epi64(0, a, count); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_srl_epi64(0b00001111, a, count); + let e = _mm256_set_epi64x(1 << 4, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_srl_epi64() { + let a = _mm_set_epi64x(1 << 5, 0); + let count = _mm_set_epi64x(0, 1); + let r = _mm_mask_srl_epi64(a, 0, a, count); + assert_eq_m128i(r, a); + let r = _mm_mask_srl_epi64(a, 0b00000011, a, count); + let e = _mm_set_epi64x(1 << 4, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_srl_epi64() { + let a = _mm_set_epi64x(1 << 5, 0); + let count = _mm_set_epi64x(0, 1); + let r = _mm_maskz_srl_epi64(0, a, count); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_srl_epi64(0b00000011, a, count); + let e = _mm_set_epi64x(1 << 4, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_sra_epi64() { + let a = _mm512_set_epi64(1, -8, 0, 0, 0, 0, 15, -16); + let count = _mm_set_epi64x(0, 2); + let r = _mm512_sra_epi64(a, count); + let e = _mm512_set_epi64(0, -2, 0, 0, 0, 0, 3, -4); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_sra_epi64() { + let a = _mm512_set_epi64(1, -8, 0, 0, 0, 0, 15, -16); + let count = _mm_set_epi64x(0, 2); + let r = _mm512_mask_sra_epi64(a, 0, a, count); + assert_eq_m512i(r, a); + let r = _mm512_mask_sra_epi64(a, 0b11111111, a, count); + let e = _mm512_set_epi64(0, -2, 0, 0, 0, 0, 3, -4); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_sra_epi64() { + let a = _mm512_set_epi64(1, -8, 0, 0, 0, 0, 15, -16); + let count = _mm_set_epi64x(0, 2); + let r = _mm512_maskz_sra_epi64(0, a, count); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_sra_epi64(0b00001111, a, count); + let e = _mm512_set_epi64(0, 0, 0, 0, 0, 0, 3, -4); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_sra_epi64() { + let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); + let count = _mm_set_epi64x(0, 1); + let r = _mm256_sra_epi64(a, count); + let e = _mm256_set_epi64x(1 << 4, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_sra_epi64() { + let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); + let count = _mm_set_epi64x(0, 1); + let r = _mm256_mask_sra_epi64(a, 0, a, count); + assert_eq_m256i(r, a); + let r = _mm256_mask_sra_epi64(a, 0b00001111, a, count); + let e = _mm256_set_epi64x(1 << 4, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_sra_epi64() { + let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); + let count = _mm_set_epi64x(0, 1); + let r = _mm256_maskz_sra_epi64(0, a, count); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_sra_epi64(0b00001111, a, count); + let e = _mm256_set_epi64x(1 << 4, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_sra_epi64() { + let a = _mm_set_epi64x(1 << 5, 0); + let count = _mm_set_epi64x(0, 1); + let r = _mm_sra_epi64(a, count); + let e = _mm_set_epi64x(1 << 4, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_sra_epi64() { + let a = _mm_set_epi64x(1 << 5, 0); + let count = _mm_set_epi64x(0, 1); + let r = _mm_mask_sra_epi64(a, 0, a, count); + assert_eq_m128i(r, a); + let r = _mm_mask_sra_epi64(a, 0b00000011, a, count); + let e = _mm_set_epi64x(1 << 4, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_sra_epi64() { + let a = _mm_set_epi64x(1 << 5, 0); + let count = _mm_set_epi64x(0, 1); + let r = _mm_maskz_sra_epi64(0, a, count); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_sra_epi64(0b00000011, a, count); + let e = _mm_set_epi64x(1 << 4, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_srav_epi64() { + let a = _mm512_set_epi64(1, -8, 0, 0, 0, 0, 15, -16); + let count = _mm512_set_epi64(2, 2, 0, 0, 0, 0, 2, 1); + let r = _mm512_srav_epi64(a, count); + let e = _mm512_set_epi64(0, -2, 0, 0, 0, 0, 3, -8); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_srav_epi64() { + let a = _mm512_set_epi64(1, -8, 0, 0, 0, 0, 15, -16); + let count = _mm512_set_epi64(2, 2, 0, 0, 0, 0, 2, 1); + let r = _mm512_mask_srav_epi64(a, 0, a, count); + assert_eq_m512i(r, a); + let r = _mm512_mask_srav_epi64(a, 0b11111111, a, count); + let e = _mm512_set_epi64(0, -2, 0, 0, 0, 0, 3, -8); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_srav_epi64() { + let a = _mm512_set_epi64(1, -8, 0, 0, 0, 0, 15, -16); + let count = _mm512_set_epi64(2, 2, 0, 0, 0, 0, 2, 1); + let r = _mm512_maskz_srav_epi64(0, a, count); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_srav_epi64(0b00001111, a, count); + let e = _mm512_set_epi64(0, 0, 0, 0, 0, 0, 3, -8); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_srav_epi64() { + let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); + let count = _mm256_set1_epi64x(1); + let r = _mm256_srav_epi64(a, count); + let e = _mm256_set_epi64x(1 << 4, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_srav_epi64() { + let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); + let count = _mm256_set1_epi64x(1); + let r = _mm256_mask_srav_epi64(a, 0, a, count); + assert_eq_m256i(r, a); + let r = _mm256_mask_srav_epi64(a, 0b00001111, a, count); + let e = _mm256_set_epi64x(1 << 4, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_srav_epi64() { + let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); + let count = _mm256_set1_epi64x(1); + let r = _mm256_maskz_srav_epi64(0, a, count); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_srav_epi64(0b00001111, a, count); + let e = _mm256_set_epi64x(1 << 4, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_srav_epi64() { + let a = _mm_set_epi64x(1 << 5, 0); + let count = _mm_set1_epi64x(1); + let r = _mm_srav_epi64(a, count); + let e = _mm_set_epi64x(1 << 4, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_srav_epi64() { + let a = _mm_set_epi64x(1 << 5, 0); + let count = _mm_set1_epi64x(1); + let r = _mm_mask_srav_epi64(a, 0, a, count); + assert_eq_m128i(r, a); + let r = _mm_mask_srav_epi64(a, 0b00000011, a, count); + let e = _mm_set_epi64x(1 << 4, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_srav_epi64() { + let a = _mm_set_epi64x(1 << 5, 0); + let count = _mm_set1_epi64x(1); + let r = _mm_maskz_srav_epi64(0, a, count); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_srav_epi64(0b00000011, a, count); + let e = _mm_set_epi64x(1 << 4, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_srai_epi64() { + let a = _mm512_set_epi64(1, -4, 15, 0, 0, 0, 0, -16); + let r = _mm512_srai_epi64::<2>(a); + let e = _mm512_set_epi64(0, -1, 3, 0, 0, 0, 0, -4); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_srai_epi64() { + let a = _mm512_set_epi64(1, -4, 15, 0, 0, 0, 0, -16); + let r = _mm512_mask_srai_epi64::<2>(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_srai_epi64::<2>(a, 0b11111111, a); + let e = _mm512_set_epi64(0, -1, 3, 0, 0, 0, 0, -4); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_srai_epi64() { + let a = _mm512_set_epi64(1, -4, 15, 0, 0, 0, 0, -16); + let r = _mm512_maskz_srai_epi64::<2>(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_srai_epi64::<2>(0b00001111, a); + let e = _mm512_set_epi64(0, 0, 0, 0, 0, 0, 0, -4); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_srai_epi64() { + let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); + let r = _mm256_srai_epi64::<1>(a); + let e = _mm256_set_epi64x(1 << 4, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_srai_epi64() { + let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); + let r = _mm256_mask_srai_epi64::<1>(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_srai_epi64::<1>(a, 0b00001111, a); + let e = _mm256_set_epi64x(1 << 4, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_srai_epi64() { + let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); + let r = _mm256_maskz_srai_epi64::<1>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_srai_epi64::<1>(0b00001111, a); + let e = _mm256_set_epi64x(1 << 4, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_srai_epi64() { + let a = _mm_set_epi64x(1 << 5, 0); + let r = _mm_srai_epi64::<1>(a); + let e = _mm_set_epi64x(1 << 4, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_srai_epi64() { + let a = _mm_set_epi64x(1 << 5, 0); + let r = _mm_mask_srai_epi64::<1>(a, 0, a); + assert_eq_m128i(r, a); + let r = _mm_mask_srai_epi64::<1>(a, 0b00000011, a); + let e = _mm_set_epi64x(1 << 4, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_srai_epi64() { + let a = _mm_set_epi64x(1 << 5, 0); + let r = _mm_maskz_srai_epi64::<1>(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_srai_epi64::<1>(0b00000011, a); + let e = _mm_set_epi64x(1 << 4, 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_permute_pd() { + let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm512_permute_pd::<0b11_11_11_11>(a); + let e = _mm512_setr_pd(1., 1., 3., 3., 5., 5., 7., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_permute_pd() { + let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm512_mask_permute_pd::<0b11_11_11_11>(a, 0, a); + assert_eq_m512d(r, a); + let r = _mm512_mask_permute_pd::<0b11_11_11_11>(a, 0b11111111, a); + let e = _mm512_setr_pd(1., 1., 3., 3., 5., 5., 7., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_permute_pd() { + let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm512_maskz_permute_pd::<0b11_11_11_11>(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_permute_pd::<0b11_11_11_11>(0b11111111, a); + let e = _mm512_setr_pd(1., 1., 3., 3., 5., 5., 7., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_permute_pd() { + let a = _mm256_set_pd(3., 2., 1., 0.); + let r = _mm256_mask_permute_pd::<0b11_11>(a, 0, a); + assert_eq_m256d(r, a); + let r = _mm256_mask_permute_pd::<0b11_11>(a, 0b00001111, a); + let e = _mm256_set_pd(3., 3., 1., 1.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_permute_pd() { + let a = _mm256_set_pd(3., 2., 1., 0.); + let r = _mm256_maskz_permute_pd::<0b11_11>(0, a); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_permute_pd::<0b11_11>(0b00001111, a); + let e = _mm256_set_pd(3., 3., 1., 1.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_permute_pd() { + let a = _mm_set_pd(1., 0.); + let r = _mm_mask_permute_pd::<0b11>(a, 0, a); + assert_eq_m128d(r, a); + let r = _mm_mask_permute_pd::<0b11>(a, 0b00000011, a); + let e = _mm_set_pd(1., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_permute_pd() { + let a = _mm_set_pd(1., 0.); + let r = _mm_maskz_permute_pd::<0b11>(0, a); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_permute_pd::<0b11>(0b00000011, a); + let e = _mm_set_pd(1., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_permutex_epi64() { + let a = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm512_permutex_epi64::<0b11_11_11_11>(a); + let e = _mm512_setr_epi64(3, 3, 3, 3, 7, 7, 7, 7); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_permutex_epi64() { + let a = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm512_mask_permutex_epi64::<0b11_11_11_11>(a, 0, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_permutex_epi64::<0b11_11_11_11>(a, 0b11111111, a); + let e = _mm512_setr_epi64(3, 3, 3, 3, 7, 7, 7, 7); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_permutex_epi64() { + let a = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm512_maskz_permutex_epi64::<0b11_11_11_11>(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_permutex_epi64::<0b11_11_11_11>(0b11111111, a); + let e = _mm512_setr_epi64(3, 3, 3, 3, 7, 7, 7, 7); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_permutex_epi64() { + let a = _mm256_set_epi64x(3, 2, 1, 0); + let r = _mm256_permutex_epi64::<0b11_11_11_11>(a); + let e = _mm256_set_epi64x(3, 3, 3, 3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_permutex_epi64() { + let a = _mm256_set_epi64x(3, 2, 1, 0); + let r = _mm256_mask_permutex_epi64::<0b11_11_11_11>(a, 0, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_permutex_epi64::<0b11_11_11_11>(a, 0b00001111, a); + let e = _mm256_set_epi64x(3, 3, 3, 3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm256_maskz_permutex_epi64() { + let a = _mm256_set_epi64x(3, 2, 1, 0); + let r = _mm256_maskz_permutex_epi64::<0b11_11_11_11>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_permutex_epi64::<0b11_11_11_11>(0b00001111, a); + let e = _mm256_set_epi64x(3, 3, 3, 3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_permutex_pd() { + let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm512_permutex_pd::<0b11_11_11_11>(a); + let e = _mm512_setr_pd(3., 3., 3., 3., 7., 7., 7., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_permutex_pd() { + let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm512_mask_permutex_pd::<0b11_11_11_11>(a, 0, a); + assert_eq_m512d(r, a); + let r = _mm512_mask_permutex_pd::<0b11_11_11_11>(a, 0b11111111, a); + let e = _mm512_setr_pd(3., 3., 3., 3., 7., 7., 7., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_permutex_pd() { + let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm512_maskz_permutex_pd::<0b11_11_11_11>(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_permutex_pd::<0b11_11_11_11>(0b11111111, a); + let e = _mm512_setr_pd(3., 3., 3., 3., 7., 7., 7., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_permutex_pd() { + let a = _mm256_set_pd(0., 1., 2., 3.); + let r = _mm256_permutex_pd::<0b11_11_11_11>(a); + let e = _mm256_set_pd(0., 0., 0., 0.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_permutex_pd() { + let a = _mm256_set_pd(0., 1., 2., 3.); + let r = _mm256_mask_permutex_pd::<0b11_11_11_11>(a, 0, a); + assert_eq_m256d(r, a); + let r = _mm256_mask_permutex_pd::<0b11_11_11_11>(a, 0b00001111, a); + let e = _mm256_set_pd(0., 0., 0., 0.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_permutex_pd() { + let a = _mm256_set_pd(0., 1., 2., 3.); + let r = _mm256_maskz_permutex_pd::<0b11_11_11_11>(0, a); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_permutex_pd::<0b11_11_11_11>(0b00001111, a); + let e = _mm256_set_pd(0., 0., 0., 0.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_permutevar_pd() { + let a = _mm512_set_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let b = _mm512_set1_epi64(0b1); + let r = _mm512_permutevar_pd(a, b); + let e = _mm512_set_pd(1., 1., 3., 3., 5., 5., 7., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_permutevar_pd() { + let a = _mm512_set_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let b = _mm512_set1_epi64(0b1); + let r = _mm512_mask_permutevar_pd(a, 0, a, b); + assert_eq_m512d(r, a); + let r = _mm512_mask_permutevar_pd(a, 0b11111111, a, b); + let e = _mm512_set_pd(1., 1., 3., 3., 5., 5., 7., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_permutevar_pd() { + let a = _mm512_set_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let b = _mm512_set1_epi64(0b1); + let r = _mm512_maskz_permutevar_pd(0, a, b); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_permutevar_pd(0b00001111, a, b); + let e = _mm512_set_pd(0., 0., 0., 0., 5., 5., 7., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_permutevar_pd() { + let a = _mm256_set_pd(0., 1., 2., 3.); + let b = _mm256_set1_epi64x(0b1); + let r = _mm256_mask_permutevar_pd(a, 0, a, b); + assert_eq_m256d(r, a); + let r = _mm256_mask_permutevar_pd(a, 0b00001111, a, b); + let e = _mm256_set_pd(1., 1., 3., 3.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_permutevar_pd() { + let a = _mm256_set_pd(0., 1., 2., 3.); + let b = _mm256_set1_epi64x(0b1); + let r = _mm256_maskz_permutevar_pd(0, a, b); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_permutevar_pd(0b00001111, a, b); + let e = _mm256_set_pd(1., 1., 3., 3.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_permutevar_pd() { + let a = _mm_set_pd(0., 1.); + let b = _mm_set1_epi64x(0b1); + let r = _mm_mask_permutevar_pd(a, 0, a, b); + assert_eq_m128d(r, a); + let r = _mm_mask_permutevar_pd(a, 0b00000011, a, b); + let e = _mm_set_pd(1., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_permutevar_pd() { + let a = _mm_set_pd(0., 1.); + let b = _mm_set1_epi64x(0b1); + let r = _mm_maskz_permutevar_pd(0, a, b); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_permutevar_pd(0b00000011, a, b); + let e = _mm_set_pd(1., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_permutexvar_epi64() { + let idx = _mm512_set1_epi64(1); + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm512_permutexvar_epi64(idx, a); + let e = _mm512_set1_epi64(6); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_permutexvar_epi64() { + let idx = _mm512_set1_epi64(1); + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm512_mask_permutexvar_epi64(a, 0, idx, a); + assert_eq_m512i(r, a); + let r = _mm512_mask_permutexvar_epi64(a, 0b11111111, idx, a); + let e = _mm512_set1_epi64(6); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_permutexvar_epi64() { + let idx = _mm512_set1_epi64(1); + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm512_maskz_permutexvar_epi64(0, idx, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_permutexvar_epi64(0b00001111, idx, a); + let e = _mm512_set_epi64(0, 0, 0, 0, 6, 6, 6, 6); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_permutexvar_epi64() { + let idx = _mm256_set1_epi64x(1); + let a = _mm256_set_epi64x(0, 1, 2, 3); + let r = _mm256_permutexvar_epi64(idx, a); + let e = _mm256_set1_epi64x(2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_permutexvar_epi64() { + let idx = _mm256_set1_epi64x(1); + let a = _mm256_set_epi64x(0, 1, 2, 3); + let r = _mm256_mask_permutexvar_epi64(a, 0, idx, a); + assert_eq_m256i(r, a); + let r = _mm256_mask_permutexvar_epi64(a, 0b00001111, idx, a); + let e = _mm256_set1_epi64x(2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_permutexvar_epi64() { + let idx = _mm256_set1_epi64x(1); + let a = _mm256_set_epi64x(0, 1, 2, 3); + let r = _mm256_maskz_permutexvar_epi64(0, idx, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_permutexvar_epi64(0b00001111, idx, a); + let e = _mm256_set1_epi64x(2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_permutexvar_pd() { + let idx = _mm512_set1_epi64(1); + let a = _mm512_set_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm512_permutexvar_pd(idx, a); + let e = _mm512_set1_pd(6.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_permutexvar_pd() { + let idx = _mm512_set1_epi64(1); + let a = _mm512_set_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm512_mask_permutexvar_pd(a, 0, idx, a); + assert_eq_m512d(r, a); + let r = _mm512_mask_permutexvar_pd(a, 0b11111111, idx, a); + let e = _mm512_set1_pd(6.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_permutexvar_pd() { + let idx = _mm512_set1_epi64(1); + let a = _mm512_set_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm512_maskz_permutexvar_pd(0, idx, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_permutexvar_pd(0b00001111, idx, a); + let e = _mm512_set_pd(0., 0., 0., 0., 6., 6., 6., 6.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_permutexvar_pd() { + let idx = _mm256_set1_epi64x(1); + let a = _mm256_set_pd(0., 1., 2., 3.); + let r = _mm256_permutexvar_pd(idx, a); + let e = _mm256_set1_pd(2.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_permutexvar_pd() { + let idx = _mm256_set1_epi64x(1); + let a = _mm256_set_pd(0., 1., 2., 3.); + let r = _mm256_mask_permutexvar_pd(a, 0, idx, a); + assert_eq_m256d(r, a); + let r = _mm256_mask_permutexvar_pd(a, 0b00001111, idx, a); + let e = _mm256_set1_pd(2.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_permutexvar_pd() { + let idx = _mm256_set1_epi64x(1); + let a = _mm256_set_pd(0., 1., 2., 3.); + let r = _mm256_maskz_permutexvar_pd(0, idx, a); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_permutexvar_pd(0b00001111, idx, a); + let e = _mm256_set1_pd(2.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_permutex2var_epi64() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let idx = _mm512_set_epi64(1, 1 << 3, 2, 1 << 3, 3, 1 << 3, 4, 1 << 3); + let b = _mm512_set1_epi64(100); + let r = _mm512_permutex2var_epi64(a, idx, b); + let e = _mm512_set_epi64(6, 100, 5, 100, 4, 100, 3, 100); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_permutex2var_epi64() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let idx = _mm512_set_epi64(1, 1 << 3, 2, 1 << 3, 3, 1 << 3, 4, 1 << 3); + let b = _mm512_set1_epi64(100); + let r = _mm512_mask_permutex2var_epi64(a, 0, idx, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_permutex2var_epi64(a, 0b11111111, idx, b); + let e = _mm512_set_epi64(6, 100, 5, 100, 4, 100, 3, 100); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_permutex2var_epi64() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let idx = _mm512_set_epi64(1, 1 << 3, 2, 1 << 3, 3, 1 << 3, 4, 1 << 3); + let b = _mm512_set1_epi64(100); + let r = _mm512_maskz_permutex2var_epi64(0, a, idx, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_permutex2var_epi64(0b00001111, a, idx, b); + let e = _mm512_set_epi64(0, 0, 0, 0, 4, 100, 3, 100); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask2_permutex2var_epi64() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let idx = _mm512_set_epi64(1000, 1 << 3, 2000, 1 << 3, 3, 1 << 3, 4, 1 << 3); + let b = _mm512_set1_epi64(100); + let r = _mm512_mask2_permutex2var_epi64(a, idx, 0, b); + assert_eq_m512i(r, idx); + let r = _mm512_mask2_permutex2var_epi64(a, idx, 0b00001111, b); + let e = _mm512_set_epi64(1000, 1 << 3, 2000, 1 << 3, 4, 100, 3, 100); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_permutex2var_epi64() { + let a = _mm256_set_epi64x(0, 1, 2, 3); + let idx = _mm256_set_epi64x(1, 1 << 2, 2, 1 << 2); + let b = _mm256_set1_epi64x(100); + let r = _mm256_permutex2var_epi64(a, idx, b); + let e = _mm256_set_epi64x(2, 100, 1, 100); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_permutex2var_epi64() { + let a = _mm256_set_epi64x(0, 1, 2, 3); + let idx = _mm256_set_epi64x(1, 1 << 2, 2, 1 << 2); + let b = _mm256_set1_epi64x(100); + let r = _mm256_mask_permutex2var_epi64(a, 0, idx, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_permutex2var_epi64(a, 0b00001111, idx, b); + let e = _mm256_set_epi64x(2, 100, 1, 100); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_permutex2var_epi64() { + let a = _mm256_set_epi64x(0, 1, 2, 3); + let idx = _mm256_set_epi64x(1, 1 << 2, 2, 1 << 2); + let b = _mm256_set1_epi64x(100); + let r = _mm256_maskz_permutex2var_epi64(0, a, idx, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_permutex2var_epi64(0b00001111, a, idx, b); + let e = _mm256_set_epi64x(2, 100, 1, 100); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask2_permutex2var_epi64() { + let a = _mm256_set_epi64x(0, 1, 2, 3); + let idx = _mm256_set_epi64x(1, 1 << 2, 2, 1 << 2); + let b = _mm256_set1_epi64x(100); + let r = _mm256_mask2_permutex2var_epi64(a, idx, 0, b); + assert_eq_m256i(r, idx); + let r = _mm256_mask2_permutex2var_epi64(a, idx, 0b00001111, b); + let e = _mm256_set_epi64x(2, 100, 1, 100); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_permutex2var_epi64() { + let a = _mm_set_epi64x(0, 1); + let idx = _mm_set_epi64x(1, 1 << 1); + let b = _mm_set1_epi64x(100); + let r = _mm_permutex2var_epi64(a, idx, b); + let e = _mm_set_epi64x(0, 100); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_permutex2var_epi64() { + let a = _mm_set_epi64x(0, 1); + let idx = _mm_set_epi64x(1, 1 << 1); + let b = _mm_set1_epi64x(100); + let r = _mm_mask_permutex2var_epi64(a, 0, idx, b); + assert_eq_m128i(r, a); + let r = _mm_mask_permutex2var_epi64(a, 0b00000011, idx, b); + let e = _mm_set_epi64x(0, 100); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_permutex2var_epi64() { + let a = _mm_set_epi64x(0, 1); + let idx = _mm_set_epi64x(1, 1 << 1); + let b = _mm_set1_epi64x(100); + let r = _mm_maskz_permutex2var_epi64(0, a, idx, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_permutex2var_epi64(0b00000011, a, idx, b); + let e = _mm_set_epi64x(0, 100); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask2_permutex2var_epi64() { + let a = _mm_set_epi64x(0, 1); + let idx = _mm_set_epi64x(1, 1 << 1); + let b = _mm_set1_epi64x(100); + let r = _mm_mask2_permutex2var_epi64(a, idx, 0, b); + assert_eq_m128i(r, idx); + let r = _mm_mask2_permutex2var_epi64(a, idx, 0b00000011, b); + let e = _mm_set_epi64x(0, 100); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_permutex2var_pd() { + let a = _mm512_set_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let idx = _mm512_set_epi64(1, 1 << 3, 2, 1 << 3, 3, 1 << 3, 4, 1 << 3); + let b = _mm512_set1_pd(100.); + let r = _mm512_permutex2var_pd(a, idx, b); + let e = _mm512_set_pd(6., 100., 5., 100., 4., 100., 3., 100.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_permutex2var_pd() { + let a = _mm512_set_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let idx = _mm512_set_epi64(1, 1 << 3, 2, 1 << 3, 3, 1 << 3, 4, 1 << 3); + let b = _mm512_set1_pd(100.); + let r = _mm512_mask_permutex2var_pd(a, 0, idx, b); + assert_eq_m512d(r, a); + let r = _mm512_mask_permutex2var_pd(a, 0b11111111, idx, b); + let e = _mm512_set_pd(6., 100., 5., 100., 4., 100., 3., 100.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_permutex2var_pd() { + let a = _mm512_set_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let idx = _mm512_set_epi64(1, 1 << 3, 2, 1 << 3, 3, 1 << 3, 4, 1 << 3); + let b = _mm512_set1_pd(100.); + let r = _mm512_maskz_permutex2var_pd(0, a, idx, b); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_permutex2var_pd(0b00001111, a, idx, b); + let e = _mm512_set_pd(0., 0., 0., 0., 4., 100., 3., 100.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask2_permutex2var_pd() { + let a = _mm512_set_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let idx = _mm512_set_epi64(1, 1 << 3, 2, 1 << 3, 3, 1 << 3, 4, 1 << 3); + let b = _mm512_set1_pd(100.); + let r = _mm512_mask2_permutex2var_pd(a, idx, 0, b); + assert_eq_m512d(r, _mm512_castsi512_pd(idx)); + let r = _mm512_mask2_permutex2var_pd(a, idx, 0b11111111, b); + let e = _mm512_set_pd(6., 100., 5., 100., 4., 100., 3., 100.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_permutex2var_pd() { + let a = _mm256_set_pd(0., 1., 2., 3.); + let idx = _mm256_set_epi64x(1, 1 << 2, 2, 1 << 2); + let b = _mm256_set1_pd(100.); + let r = _mm256_permutex2var_pd(a, idx, b); + let e = _mm256_set_pd(2., 100., 1., 100.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_permutex2var_pd() { + let a = _mm256_set_pd(0., 1., 2., 3.); + let idx = _mm256_set_epi64x(1, 1 << 2, 2, 1 << 2); + let b = _mm256_set1_pd(100.); + let r = _mm256_mask_permutex2var_pd(a, 0, idx, b); + assert_eq_m256d(r, a); + let r = _mm256_mask_permutex2var_pd(a, 0b00001111, idx, b); + let e = _mm256_set_pd(2., 100., 1., 100.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_permutex2var_pd() { + let a = _mm256_set_pd(0., 1., 2., 3.); + let idx = _mm256_set_epi64x(1, 1 << 2, 2, 1 << 2); + let b = _mm256_set1_pd(100.); + let r = _mm256_maskz_permutex2var_pd(0, a, idx, b); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_permutex2var_pd(0b00001111, a, idx, b); + let e = _mm256_set_pd(2., 100., 1., 100.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask2_permutex2var_pd() { + let a = _mm256_set_pd(0., 1., 2., 3.); + let idx = _mm256_set_epi64x(1, 1 << 2, 2, 1 << 2); + let b = _mm256_set1_pd(100.); + let r = _mm256_mask2_permutex2var_pd(a, idx, 0, b); + assert_eq_m256d(r, _mm256_castsi256_pd(idx)); + let r = _mm256_mask2_permutex2var_pd(a, idx, 0b00001111, b); + let e = _mm256_set_pd(2., 100., 1., 100.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_permutex2var_pd() { + let a = _mm_set_pd(0., 1.); + let idx = _mm_set_epi64x(1, 1 << 1); + let b = _mm_set1_pd(100.); + let r = _mm_permutex2var_pd(a, idx, b); + let e = _mm_set_pd(0., 100.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_permutex2var_pd() { + let a = _mm_set_pd(0., 1.); + let idx = _mm_set_epi64x(1, 1 << 1); + let b = _mm_set1_pd(100.); + let r = _mm_mask_permutex2var_pd(a, 0, idx, b); + assert_eq_m128d(r, a); + let r = _mm_mask_permutex2var_pd(a, 0b00000011, idx, b); + let e = _mm_set_pd(0., 100.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_permutex2var_pd() { + let a = _mm_set_pd(0., 1.); + let idx = _mm_set_epi64x(1, 1 << 1); + let b = _mm_set1_pd(100.); + let r = _mm_maskz_permutex2var_pd(0, a, idx, b); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_permutex2var_pd(0b00000011, a, idx, b); + let e = _mm_set_pd(0., 100.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask2_permutex2var_pd() { + let a = _mm_set_pd(0., 1.); + let idx = _mm_set_epi64x(1, 1 << 1); + let b = _mm_set1_pd(100.); + let r = _mm_mask2_permutex2var_pd(a, idx, 0, b); + assert_eq_m128d(r, _mm_castsi128_pd(idx)); + let r = _mm_mask2_permutex2var_pd(a, idx, 0b00000011, b); + let e = _mm_set_pd(0., 100.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_shuffle_pd() { + let a = _mm256_set_pd(1., 4., 5., 8.); + let b = _mm256_set_pd(2., 3., 6., 7.); + let r = _mm256_mask_shuffle_pd::<0b11_11_11_11>(a, 0, a, b); + assert_eq_m256d(r, a); + let r = _mm256_mask_shuffle_pd::<0b11_11_11_11>(a, 0b00001111, a, b); + let e = _mm256_set_pd(2., 1., 6., 5.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_shuffle_pd() { + let a = _mm256_set_pd(1., 4., 5., 8.); + let b = _mm256_set_pd(2., 3., 6., 7.); + let r = _mm256_maskz_shuffle_pd::<0b11_11_11_11>(0, a, b); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_shuffle_pd::<0b11_11_11_11>(0b00001111, a, b); + let e = _mm256_set_pd(2., 1., 6., 5.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_shuffle_pd() { + let a = _mm_set_pd(1., 4.); + let b = _mm_set_pd(2., 3.); + let r = _mm_mask_shuffle_pd::<0b11_11_11_11>(a, 0, a, b); + assert_eq_m128d(r, a); + let r = _mm_mask_shuffle_pd::<0b11_11_11_11>(a, 0b00000011, a, b); + let e = _mm_set_pd(2., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_shuffle_pd() { + let a = _mm_set_pd(1., 4.); + let b = _mm_set_pd(2., 3.); + let r = _mm_maskz_shuffle_pd::<0b11_11_11_11>(0, a, b); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_shuffle_pd::<0b11_11_11_11>(0b00000011, a, b); + let e = _mm_set_pd(2., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_shuffle_i64x2() { + let a = _mm512_setr_epi64(1, 4, 5, 8, 9, 12, 13, 16); + let b = _mm512_setr_epi64(2, 3, 6, 7, 10, 11, 14, 15); + let r = _mm512_shuffle_i64x2::<0b00_00_00_00>(a, b); + let e = _mm512_setr_epi64(1, 4, 1, 4, 2, 3, 2, 3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_shuffle_i64x2() { + let a = _mm512_setr_epi64(1, 4, 5, 8, 9, 12, 13, 16); + let b = _mm512_setr_epi64(2, 3, 6, 7, 10, 11, 14, 15); + let r = _mm512_mask_shuffle_i64x2::<0b00_00_00_00>(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_shuffle_i64x2::<0b00_00_00_00>(a, 0b11111111, a, b); + let e = _mm512_setr_epi64(1, 4, 1, 4, 2, 3, 2, 3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_shuffle_i64x2() { + let a = _mm512_setr_epi64(1, 4, 5, 8, 9, 12, 13, 16); + let b = _mm512_setr_epi64(2, 3, 6, 7, 10, 11, 14, 15); + let r = _mm512_maskz_shuffle_i64x2::<0b00_00_00_00>(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_shuffle_i64x2::<0b00_00_00_00>(0b00001111, a, b); + let e = _mm512_setr_epi64(1, 4, 1, 4, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_shuffle_i64x2() { + let a = _mm256_set_epi64x(1, 4, 5, 8); + let b = _mm256_set_epi64x(2, 3, 6, 7); + let r = _mm256_shuffle_i64x2::<0b00>(a, b); + let e = _mm256_set_epi64x(6, 7, 5, 8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_shuffle_i64x2() { + let a = _mm256_set_epi64x(1, 4, 5, 8); + let b = _mm256_set_epi64x(2, 3, 6, 7); + let r = _mm256_mask_shuffle_i64x2::<0b00>(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_shuffle_i64x2::<0b00>(a, 0b00001111, a, b); + let e = _mm256_set_epi64x(6, 7, 5, 8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_shuffle_i64x2() { + let a = _mm256_set_epi64x(1, 4, 5, 8); + let b = _mm256_set_epi64x(2, 3, 6, 7); + let r = _mm256_maskz_shuffle_i64x2::<0b00>(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_shuffle_i64x2::<0b00>(0b00001111, a, b); + let e = _mm256_set_epi64x(6, 7, 5, 8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_shuffle_f64x2() { + let a = _mm512_setr_pd(1., 4., 5., 8., 9., 12., 13., 16.); + let b = _mm512_setr_pd(2., 3., 6., 7., 10., 11., 14., 15.); + let r = _mm512_shuffle_f64x2::<0b00_00_00_00>(a, b); + let e = _mm512_setr_pd(1., 4., 1., 4., 2., 3., 2., 3.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_shuffle_f64x2() { + let a = _mm512_setr_pd(1., 4., 5., 8., 9., 12., 13., 16.); + let b = _mm512_setr_pd(2., 3., 6., 7., 10., 11., 14., 15.); + let r = _mm512_mask_shuffle_f64x2::<0b00_00_00_00>(a, 0, a, b); + assert_eq_m512d(r, a); + let r = _mm512_mask_shuffle_f64x2::<0b00_00_00_00>(a, 0b11111111, a, b); + let e = _mm512_setr_pd(1., 4., 1., 4., 2., 3., 2., 3.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_shuffle_f64x2() { + let a = _mm512_setr_pd(1., 4., 5., 8., 9., 12., 13., 16.); + let b = _mm512_setr_pd(2., 3., 6., 7., 10., 11., 14., 15.); + let r = _mm512_maskz_shuffle_f64x2::<0b00_00_00_00>(0, a, b); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_shuffle_f64x2::<0b00_00_00_00>(0b00001111, a, b); + let e = _mm512_setr_pd(1., 4., 1., 4., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_shuffle_f64x2() { + let a = _mm256_set_pd(1., 4., 5., 8.); + let b = _mm256_set_pd(2., 3., 6., 7.); + let r = _mm256_shuffle_f64x2::<0b00>(a, b); + let e = _mm256_set_pd(6., 7., 5., 8.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_shuffle_f64x2() { + let a = _mm256_set_pd(1., 4., 5., 8.); + let b = _mm256_set_pd(2., 3., 6., 7.); + let r = _mm256_mask_shuffle_f64x2::<0b00>(a, 0, a, b); + assert_eq_m256d(r, a); + let r = _mm256_mask_shuffle_f64x2::<0b00>(a, 0b00001111, a, b); + let e = _mm256_set_pd(6., 7., 5., 8.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_shuffle_f64x2() { + let a = _mm256_set_pd(1., 4., 5., 8.); + let b = _mm256_set_pd(2., 3., 6., 7.); + let r = _mm256_maskz_shuffle_f64x2::<0b00>(0, a, b); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_shuffle_f64x2::<0b00>(0b00001111, a, b); + let e = _mm256_set_pd(6., 7., 5., 8.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_movedup_pd() { + let a = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); + let r = _mm512_movedup_pd(a); + let e = _mm512_setr_pd(1., 1., 3., 3., 5., 5., 7., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_movedup_pd() { + let a = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); + let r = _mm512_mask_movedup_pd(a, 0, a); + assert_eq_m512d(r, a); + let r = _mm512_mask_movedup_pd(a, 0b11111111, a); + let e = _mm512_setr_pd(1., 1., 3., 3., 5., 5., 7., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_movedup_pd() { + let a = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); + let r = _mm512_maskz_movedup_pd(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_movedup_pd(0b00001111, a); + let e = _mm512_setr_pd(1., 1., 3., 3., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_movedup_pd() { + let a = _mm256_set_pd(1., 2., 3., 4.); + let r = _mm256_mask_movedup_pd(a, 0, a); + assert_eq_m256d(r, a); + let r = _mm256_mask_movedup_pd(a, 0b00001111, a); + let e = _mm256_set_pd(2., 2., 4., 4.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_movedup_pd() { + let a = _mm256_set_pd(1., 2., 3., 4.); + let r = _mm256_maskz_movedup_pd(0, a); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_movedup_pd(0b00001111, a); + let e = _mm256_set_pd(2., 2., 4., 4.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_movedup_pd() { + let a = _mm_set_pd(1., 2.); + let r = _mm_mask_movedup_pd(a, 0, a); + assert_eq_m128d(r, a); + let r = _mm_mask_movedup_pd(a, 0b00000011, a); + let e = _mm_set_pd(2., 2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_movedup_pd() { + let a = _mm_set_pd(1., 2.); + let r = _mm_maskz_movedup_pd(0, a); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_movedup_pd(0b00000011, a); + let e = _mm_set_pd(2., 2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_inserti64x4() { + let a = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm256_setr_epi64x(17, 18, 19, 20); + let r = _mm512_inserti64x4::<1>(a, b); + let e = _mm512_setr_epi64(1, 2, 3, 4, 17, 18, 19, 20); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_inserti64x4() { + let a = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm256_setr_epi64x(17, 18, 19, 20); + let r = _mm512_mask_inserti64x4::<1>(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_inserti64x4::<1>(a, 0b11111111, a, b); + let e = _mm512_setr_epi64(1, 2, 3, 4, 17, 18, 19, 20); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_inserti64x4() { + let a = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm256_setr_epi64x(17, 18, 19, 20); + let r = _mm512_maskz_inserti64x4::<1>(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_inserti64x4::<1>(0b00001111, a, b); + let e = _mm512_setr_epi64(1, 2, 3, 4, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_insertf64x4() { + let a = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); + let b = _mm256_setr_pd(17., 18., 19., 20.); + let r = _mm512_insertf64x4::<1>(a, b); + let e = _mm512_setr_pd(1., 2., 3., 4., 17., 18., 19., 20.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_insertf64x4() { + let a = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); + let b = _mm256_setr_pd(17., 18., 19., 20.); + let r = _mm512_mask_insertf64x4::<1>(a, 0, a, b); + assert_eq_m512d(r, a); + let r = _mm512_mask_insertf64x4::<1>(a, 0b11111111, a, b); + let e = _mm512_setr_pd(1., 2., 3., 4., 17., 18., 19., 20.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_insertf64x4() { + let a = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); + let b = _mm256_setr_pd(17., 18., 19., 20.); + let r = _mm512_maskz_insertf64x4::<1>(0, a, b); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_insertf64x4::<1>(0b00001111, a, b); + let e = _mm512_setr_pd(1., 2., 3., 4., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_castpd128_pd512() { + let a = _mm_setr_pd(17., 18.); + let r = _mm512_castpd128_pd512(a); + let e = _mm512_setr_pd(17., 18., -1., -1., -1., -1., -1., -1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_castpd256_pd512() { + let a = _mm256_setr_pd(17., 18., 19., 20.); + let r = _mm512_castpd256_pd512(a); + let e = _mm512_setr_pd(17., 18., 19., 20., -1., -1., -1., -1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_zextpd128_pd512() { + let a = _mm_setr_pd(17., 18.); + let r = _mm512_zextpd128_pd512(a); + let e = _mm512_setr_pd(17., 18., 0., 0., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_zextpd256_pd512() { + let a = _mm256_setr_pd(17., 18., 19., 20.); + let r = _mm512_zextpd256_pd512(a); + let e = _mm512_setr_pd(17., 18., 19., 20., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_castpd512_pd128() { + let a = _mm512_setr_pd(17., 18., -1., -1., -1., -1., -1., -1.); + let r = _mm512_castpd512_pd128(a); + let e = _mm_setr_pd(17., 18.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_castpd512_pd256() { + let a = _mm512_setr_pd(17., 18., 19., 20., -1., -1., -1., -1.); + let r = _mm512_castpd512_pd256(a); + let e = _mm256_setr_pd(17., 18., 19., 20.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_castpd_ps() { + let a = _mm512_set1_pd(1.); + let r = _mm512_castpd_ps(a); + let e = _mm512_set_ps( + 1.875, 0.0, 1.875, 0.0, 1.875, 0.0, 1.875, 0.0, 1.875, 0.0, 1.875, 0.0, 1.875, 0.0, + 1.875, 0.0, + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_castpd_si512() { + let a = _mm512_set1_pd(1.); + let r = _mm512_castpd_si512(a); + let e = _mm512_set_epi32( + 1072693248, 0, 1072693248, 0, 1072693248, 0, 1072693248, 0, 1072693248, 0, 1072693248, + 0, 1072693248, 0, 1072693248, 0, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_castsi128_si512() { + let a = _mm_setr_epi64x(17, 18); + let r = _mm512_castsi128_si512(a); + let e = _mm512_setr_epi64(17, 18, -1, -1, -1, -1, -1, -1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_castsi256_si512() { + let a = _mm256_setr_epi64x(17, 18, 19, 20); + let r = _mm512_castsi256_si512(a); + let e = _mm512_setr_epi64(17, 18, 19, 20, -1, -1, -1, -1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_zextsi128_si512() { + let a = _mm_setr_epi64x(17, 18); + let r = _mm512_zextsi128_si512(a); + let e = _mm512_setr_epi64(17, 18, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_zextsi256_si512() { + let a = _mm256_setr_epi64x(17, 18, 19, 20); + let r = _mm512_zextsi256_si512(a); + let e = _mm512_setr_epi64(17, 18, 19, 20, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_castsi512_si128() { + let a = _mm512_setr_epi64(17, 18, -1, -1, -1, -1, -1, -1); + let r = _mm512_castsi512_si128(a); + let e = _mm_setr_epi64x(17, 18); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_castsi512_si256() { + let a = _mm512_setr_epi64(17, 18, 19, 20, -1, -1, -1, -1); + let r = _mm512_castsi512_si256(a); + let e = _mm256_setr_epi64x(17, 18, 19, 20); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_castsi512_ps() { + let a = _mm512_set1_epi64(1 << 62); + let r = _mm512_castsi512_ps(a); + let e = _mm512_set_ps( + 2., 0., 2., 0., 2., 0., 2., 0., 2., 0., 2., 0., 2., 0., 2., 0., + ); + assert_eq_m512(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_castsi512_pd() { + let a = _mm512_set1_epi64(1 << 62); + let r = _mm512_castsi512_pd(a); + let e = _mm512_set_pd(2., 2., 2., 2., 2., 2., 2., 2.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_broadcastq_epi64() { + let a = _mm_setr_epi64x(17, 18); + let r = _mm512_broadcastq_epi64(a); + let e = _mm512_set1_epi64(17); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_broadcastq_epi64() { + let src = _mm512_set1_epi64(18); + let a = _mm_setr_epi64x(17, 18); + let r = _mm512_mask_broadcastq_epi64(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_broadcastq_epi64(src, 0b11111111, a); + let e = _mm512_set1_epi64(17); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_broadcastq_epi64() { + let a = _mm_setr_epi64x(17, 18); + let r = _mm512_maskz_broadcastq_epi64(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_broadcastq_epi64(0b00001111, a); + let e = _mm512_set_epi64(0, 0, 0, 0, 17, 17, 17, 17); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_broadcastq_epi64() { + let src = _mm256_set1_epi64x(18); + let a = _mm_set_epi64x(17, 18); + let r = _mm256_mask_broadcastq_epi64(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_broadcastq_epi64(src, 0b00001111, a); + let e = _mm256_set1_epi64x(18); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_broadcastq_epi64() { + let a = _mm_set_epi64x(17, 18); + let r = _mm256_maskz_broadcastq_epi64(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_broadcastq_epi64(0b00001111, a); + let e = _mm256_set1_epi64x(18); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_broadcastq_epi64() { + let src = _mm_set1_epi64x(18); + let a = _mm_set_epi64x(17, 18); + let r = _mm_mask_broadcastq_epi64(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_broadcastq_epi64(src, 0b00000011, a); + let e = _mm_set1_epi64x(18); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_broadcastq_epi64() { + let a = _mm_set_epi64x(17, 18); + let r = _mm_maskz_broadcastq_epi64(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_broadcastq_epi64(0b00000011, a); + let e = _mm_set1_epi64x(18); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_broadcastsd_pd() { + let a = _mm_set_pd(17., 18.); + let r = _mm512_broadcastsd_pd(a); + let e = _mm512_set1_pd(18.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_broadcastsd_pd() { + let src = _mm512_set1_pd(18.); + let a = _mm_set_pd(17., 18.); + let r = _mm512_mask_broadcastsd_pd(src, 0, a); + assert_eq_m512d(r, src); + let r = _mm512_mask_broadcastsd_pd(src, 0b11111111, a); + let e = _mm512_set1_pd(18.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_broadcastsd_pd() { + let a = _mm_set_pd(17., 18.); + let r = _mm512_maskz_broadcastsd_pd(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_broadcastsd_pd(0b00001111, a); + let e = _mm512_set_pd(0., 0., 0., 0., 18., 18., 18., 18.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_broadcastsd_pd() { + let src = _mm256_set1_pd(18.); + let a = _mm_set_pd(17., 18.); + let r = _mm256_mask_broadcastsd_pd(src, 0, a); + assert_eq_m256d(r, src); + let r = _mm256_mask_broadcastsd_pd(src, 0b00001111, a); + let e = _mm256_set1_pd(18.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_broadcastsd_pd() { + let a = _mm_set_pd(17., 18.); + let r = _mm256_maskz_broadcastsd_pd(0, a); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_broadcastsd_pd(0b00001111, a); + let e = _mm256_set1_pd(18.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_broadcast_i64x4() { + let a = _mm256_set_epi64x(17, 18, 19, 20); + let r = _mm512_broadcast_i64x4(a); + let e = _mm512_set_epi64(17, 18, 19, 20, 17, 18, 19, 20); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_broadcast_i64x4() { + let src = _mm512_set1_epi64(18); + let a = _mm256_set_epi64x(17, 18, 19, 20); + let r = _mm512_mask_broadcast_i64x4(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_broadcast_i64x4(src, 0b11111111, a); + let e = _mm512_set_epi64(17, 18, 19, 20, 17, 18, 19, 20); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_broadcast_i64x4() { + let a = _mm256_set_epi64x(17, 18, 19, 20); + let r = _mm512_maskz_broadcast_i64x4(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_broadcast_i64x4(0b00001111, a); + let e = _mm512_set_epi64(0, 0, 0, 0, 17, 18, 19, 20); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_broadcast_f64x4() { + let a = _mm256_set_pd(17., 18., 19., 20.); + let r = _mm512_broadcast_f64x4(a); + let e = _mm512_set_pd(17., 18., 19., 20., 17., 18., 19., 20.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_broadcast_f64x4() { + let src = _mm512_set1_pd(18.); + let a = _mm256_set_pd(17., 18., 19., 20.); + let r = _mm512_mask_broadcast_f64x4(src, 0, a); + assert_eq_m512d(r, src); + let r = _mm512_mask_broadcast_f64x4(src, 0b11111111, a); + let e = _mm512_set_pd(17., 18., 19., 20., 17., 18., 19., 20.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_broadcast_f64x4() { + let a = _mm256_set_pd(17., 18., 19., 20.); + let r = _mm512_maskz_broadcast_f64x4(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_broadcast_f64x4(0b00001111, a); + let e = _mm512_set_pd(0., 0., 0., 0., 17., 18., 19., 20.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_blend_epi64() { + let a = _mm512_set1_epi64(1); + let b = _mm512_set1_epi64(2); + let r = _mm512_mask_blend_epi64(0b11110000, a, b); + let e = _mm512_set_epi64(2, 2, 2, 2, 1, 1, 1, 1); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_blend_epi64() { + let a = _mm256_set1_epi64x(1); + let b = _mm256_set1_epi64x(2); + let r = _mm256_mask_blend_epi64(0b00001111, a, b); + let e = _mm256_set1_epi64x(2); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_blend_epi64() { + let a = _mm_set1_epi64x(1); + let b = _mm_set1_epi64x(2); + let r = _mm_mask_blend_epi64(0b00000011, a, b); + let e = _mm_set1_epi64x(2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_blend_pd() { + let a = _mm512_set1_pd(1.); + let b = _mm512_set1_pd(2.); + let r = _mm512_mask_blend_pd(0b11110000, a, b); + let e = _mm512_set_pd(2., 2., 2., 2., 1., 1., 1., 1.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_blend_pd() { + let a = _mm256_set1_pd(1.); + let b = _mm256_set1_pd(2.); + let r = _mm256_mask_blend_pd(0b00001111, a, b); + let e = _mm256_set1_pd(2.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_blend_pd() { + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(2.); + let r = _mm_mask_blend_pd(0b00000011, a, b); + let e = _mm_set1_pd(2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_unpackhi_epi64() { + let a = _mm512_set_epi64(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm512_set_epi64(17, 18, 19, 20, 21, 22, 23, 24); + let r = _mm512_unpackhi_epi64(a, b); + let e = _mm512_set_epi64(17, 1, 19, 3, 21, 5, 23, 7); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_unpackhi_epi64() { + let a = _mm512_set_epi64(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm512_set_epi64(17, 18, 19, 20, 21, 22, 23, 24); + let r = _mm512_mask_unpackhi_epi64(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_unpackhi_epi64(a, 0b11111111, a, b); + let e = _mm512_set_epi64(17, 1, 19, 3, 21, 5, 23, 7); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_unpackhi_epi64() { + let a = _mm512_set_epi64(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm512_set_epi64(17, 18, 19, 20, 21, 22, 23, 24); + let r = _mm512_maskz_unpackhi_epi64(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_unpackhi_epi64(0b00001111, a, b); + let e = _mm512_set_epi64(0, 0, 0, 0, 21, 5, 23, 7); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_unpackhi_epi64() { + let a = _mm256_set_epi64x(1, 2, 3, 4); + let b = _mm256_set_epi64x(17, 18, 19, 20); + let r = _mm256_mask_unpackhi_epi64(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_unpackhi_epi64(a, 0b00001111, a, b); + let e = _mm256_set_epi64x(17, 1, 19, 3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_unpackhi_epi64() { + let a = _mm256_set_epi64x(1, 2, 3, 4); + let b = _mm256_set_epi64x(17, 18, 19, 20); + let r = _mm256_maskz_unpackhi_epi64(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_unpackhi_epi64(0b00001111, a, b); + let e = _mm256_set_epi64x(17, 1, 19, 3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_unpackhi_epi64() { + let a = _mm_set_epi64x(1, 2); + let b = _mm_set_epi64x(17, 18); + let r = _mm_mask_unpackhi_epi64(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_unpackhi_epi64(a, 0b00000011, a, b); + let e = _mm_set_epi64x(17, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_unpackhi_epi64() { + let a = _mm_set_epi64x(1, 2); + let b = _mm_set_epi64x(17, 18); + let r = _mm_maskz_unpackhi_epi64(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_unpackhi_epi64(0b00000011, a, b); + let e = _mm_set_epi64x(17, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_unpackhi_pd() { + let a = _mm512_set_pd(1., 2., 3., 4., 5., 6., 7., 8.); + let b = _mm512_set_pd(17., 18., 19., 20., 21., 22., 23., 24.); + let r = _mm512_unpackhi_pd(a, b); + let e = _mm512_set_pd(17., 1., 19., 3., 21., 5., 23., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_unpackhi_pd() { + let a = _mm512_set_pd(1., 2., 3., 4., 5., 6., 7., 8.); + let b = _mm512_set_pd(17., 18., 19., 20., 21., 22., 23., 24.); + let r = _mm512_mask_unpackhi_pd(a, 0, a, b); + assert_eq_m512d(r, a); + let r = _mm512_mask_unpackhi_pd(a, 0b11111111, a, b); + let e = _mm512_set_pd(17., 1., 19., 3., 21., 5., 23., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_unpackhi_pd() { + let a = _mm512_set_pd(1., 2., 3., 4., 5., 6., 7., 8.); + let b = _mm512_set_pd(17., 18., 19., 20., 21., 22., 23., 24.); + let r = _mm512_maskz_unpackhi_pd(0, a, b); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_unpackhi_pd(0b00001111, a, b); + let e = _mm512_set_pd(0., 0., 0., 0., 21., 5., 23., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_unpackhi_pd() { + let a = _mm256_set_pd(1., 2., 3., 4.); + let b = _mm256_set_pd(17., 18., 19., 20.); + let r = _mm256_mask_unpackhi_pd(a, 0, a, b); + assert_eq_m256d(r, a); + let r = _mm256_mask_unpackhi_pd(a, 0b00001111, a, b); + let e = _mm256_set_pd(17., 1., 19., 3.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_unpackhi_pd() { + let a = _mm256_set_pd(1., 2., 3., 4.); + let b = _mm256_set_pd(17., 18., 19., 20.); + let r = _mm256_maskz_unpackhi_pd(0, a, b); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_unpackhi_pd(0b00001111, a, b); + let e = _mm256_set_pd(17., 1., 19., 3.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_unpackhi_pd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(17., 18.); + let r = _mm_mask_unpackhi_pd(a, 0, a, b); + assert_eq_m128d(r, a); + let r = _mm_mask_unpackhi_pd(a, 0b00000011, a, b); + let e = _mm_set_pd(17., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_unpackhi_pd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(17., 18.); + let r = _mm_maskz_unpackhi_pd(0, a, b); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_unpackhi_pd(0b00000011, a, b); + let e = _mm_set_pd(17., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_unpacklo_epi64() { + let a = _mm512_set_epi64(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm512_set_epi64(17, 18, 19, 20, 21, 22, 23, 24); + let r = _mm512_unpacklo_epi64(a, b); + let e = _mm512_set_epi64(18, 2, 20, 4, 22, 6, 24, 8); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_unpacklo_epi64() { + let a = _mm512_set_epi64(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm512_set_epi64(17, 18, 19, 20, 21, 22, 23, 24); + let r = _mm512_mask_unpacklo_epi64(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_unpacklo_epi64(a, 0b11111111, a, b); + let e = _mm512_set_epi64(18, 2, 20, 4, 22, 6, 24, 8); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_unpacklo_epi64() { + let a = _mm512_set_epi64(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm512_set_epi64(17, 18, 19, 20, 21, 22, 23, 24); + let r = _mm512_maskz_unpacklo_epi64(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_unpacklo_epi64(0b00001111, a, b); + let e = _mm512_set_epi64(0, 0, 0, 0, 22, 6, 24, 8); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_unpacklo_epi64() { + let a = _mm256_set_epi64x(1, 2, 3, 4); + let b = _mm256_set_epi64x(17, 18, 19, 20); + let r = _mm256_mask_unpacklo_epi64(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_unpacklo_epi64(a, 0b00001111, a, b); + let e = _mm256_set_epi64x(18, 2, 20, 4); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_unpacklo_epi64() { + let a = _mm256_set_epi64x(1, 2, 3, 4); + let b = _mm256_set_epi64x(17, 18, 19, 20); + let r = _mm256_maskz_unpacklo_epi64(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_unpacklo_epi64(0b00001111, a, b); + let e = _mm256_set_epi64x(18, 2, 20, 4); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_unpacklo_epi64() { + let a = _mm_set_epi64x(1, 2); + let b = _mm_set_epi64x(17, 18); + let r = _mm_mask_unpacklo_epi64(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_unpacklo_epi64(a, 0b00000011, a, b); + let e = _mm_set_epi64x(18, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_unpacklo_epi64() { + let a = _mm_set_epi64x(1, 2); + let b = _mm_set_epi64x(17, 18); + let r = _mm_maskz_unpacklo_epi64(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_unpacklo_epi64(0b00000011, a, b); + let e = _mm_set_epi64x(18, 2); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_unpacklo_pd() { + let a = _mm512_set_pd(1., 2., 3., 4., 5., 6., 7., 8.); + let b = _mm512_set_pd(17., 18., 19., 20., 21., 22., 23., 24.); + let r = _mm512_unpacklo_pd(a, b); + let e = _mm512_set_pd(18., 2., 20., 4., 22., 6., 24., 8.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_unpacklo_pd() { + let a = _mm512_set_pd(1., 2., 3., 4., 5., 6., 7., 8.); + let b = _mm512_set_pd(17., 18., 19., 20., 21., 22., 23., 24.); + let r = _mm512_mask_unpacklo_pd(a, 0, a, b); + assert_eq_m512d(r, a); + let r = _mm512_mask_unpacklo_pd(a, 0b11111111, a, b); + let e = _mm512_set_pd(18., 2., 20., 4., 22., 6., 24., 8.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_unpacklo_pd() { + let a = _mm512_set_pd(1., 2., 3., 4., 5., 6., 7., 8.); + let b = _mm512_set_pd(17., 18., 19., 20., 21., 22., 23., 24.); + let r = _mm512_maskz_unpacklo_pd(0, a, b); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_unpacklo_pd(0b00001111, a, b); + let e = _mm512_set_pd(0., 0., 0., 0., 22., 6., 24., 8.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_unpacklo_pd() { + let a = _mm256_set_pd(1., 2., 3., 4.); + let b = _mm256_set_pd(17., 18., 19., 20.); + let r = _mm256_mask_unpacklo_pd(a, 0, a, b); + assert_eq_m256d(r, a); + let r = _mm256_mask_unpacklo_pd(a, 0b00001111, a, b); + let e = _mm256_set_pd(18., 2., 20., 4.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_unpacklo_pd() { + let a = _mm256_set_pd(1., 2., 3., 4.); + let b = _mm256_set_pd(17., 18., 19., 20.); + let r = _mm256_maskz_unpacklo_pd(0, a, b); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_unpacklo_pd(0b00001111, a, b); + let e = _mm256_set_pd(18., 2., 20., 4.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_unpacklo_pd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(17., 18.); + let r = _mm_mask_unpacklo_pd(a, 0, a, b); + assert_eq_m128d(r, a); + let r = _mm_mask_unpacklo_pd(a, 0b00000011, a, b); + let e = _mm_set_pd(18., 2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_unpacklo_pd() { + let a = _mm_set_pd(1., 2.); + let b = _mm_set_pd(17., 18.); + let r = _mm_maskz_unpacklo_pd(0, a, b); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_unpacklo_pd(0b00000011, a, b); + let e = _mm_set_pd(18., 2.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_alignr_epi64() { + let a = _mm512_set_epi64(8, 7, 6, 5, 4, 3, 2, 1); + let b = _mm512_set_epi64(16, 15, 14, 13, 12, 11, 10, 9); + let r = _mm512_alignr_epi64::<0>(a, b); + assert_eq_m512i(r, b); + let r = _mm512_alignr_epi64::<8>(a, b); + assert_eq_m512i(r, b); + let r = _mm512_alignr_epi64::<1>(a, b); + let e = _mm512_set_epi64(1, 16, 15, 14, 13, 12, 11, 10); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_alignr_epi64() { + let a = _mm512_set_epi64(8, 7, 6, 5, 4, 3, 2, 1); + let b = _mm512_set_epi64(16, 15, 14, 13, 12, 11, 10, 9); + let r = _mm512_mask_alignr_epi64::<1>(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_alignr_epi64::<1>(a, 0b11111111, a, b); + let e = _mm512_set_epi64(1, 16, 15, 14, 13, 12, 11, 10); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_alignr_epi64() { + let a = _mm512_set_epi64(8, 7, 6, 5, 4, 3, 2, 1); + let b = _mm512_set_epi64(16, 15, 14, 13, 12, 11, 10, 9); + let r = _mm512_maskz_alignr_epi64::<1>(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_alignr_epi64::<1>(0b00001111, a, b); + let e = _mm512_set_epi64(0, 0, 0, 0, 13, 12, 11, 10); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_alignr_epi64() { + let a = _mm256_set_epi64x(4, 3, 2, 1); + let b = _mm256_set_epi64x(8, 7, 6, 5); + let r = _mm256_alignr_epi64::<0>(a, b); + let e = _mm256_set_epi64x(8, 7, 6, 5); + assert_eq_m256i(r, e); + let r = _mm256_alignr_epi64::<6>(a, b); + let e = _mm256_set_epi64x(6, 5, 4, 3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_alignr_epi64() { + let a = _mm256_set_epi64x(4, 3, 2, 1); + let b = _mm256_set_epi64x(8, 7, 6, 5); + let r = _mm256_mask_alignr_epi64::<1>(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_alignr_epi64::<0>(a, 0b00001111, a, b); + let e = _mm256_set_epi64x(8, 7, 6, 5); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_alignr_epi64() { + let a = _mm256_set_epi64x(4, 3, 2, 1); + let b = _mm256_set_epi64x(8, 7, 6, 5); + let r = _mm256_maskz_alignr_epi64::<1>(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_alignr_epi64::<0>(0b00001111, a, b); + let e = _mm256_set_epi64x(8, 7, 6, 5); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_alignr_epi64() { + let a = _mm_set_epi64x(2, 1); + let b = _mm_set_epi64x(4, 3); + let r = _mm_alignr_epi64::<0>(a, b); + let e = _mm_set_epi64x(4, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_alignr_epi64() { + let a = _mm_set_epi64x(2, 1); + let b = _mm_set_epi64x(4, 3); + let r = _mm_mask_alignr_epi64::<1>(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_alignr_epi64::<0>(a, 0b00000011, a, b); + let e = _mm_set_epi64x(4, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_alignr_epi64() { + let a = _mm_set_epi64x(2, 1); + let b = _mm_set_epi64x(4, 3); + let r = _mm_maskz_alignr_epi64::<1>(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_alignr_epi64::<0>(0b00000011, a, b); + let e = _mm_set_epi64x(4, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_and_epi64() { + let a = _mm512_set_epi64(1 << 0 | 1 << 15, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let b = _mm512_set_epi64(1 << 13, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let r = _mm512_and_epi64(a, b); + let e = _mm512_set_epi64(0, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_and_epi64() { + let a = _mm512_set_epi64(1 << 0 | 1 << 15, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let b = _mm512_set_epi64(1 << 13, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let r = _mm512_mask_and_epi64(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_and_epi64(a, 0b01111111, a, b); + let e = _mm512_set_epi64(1 << 0 | 1 << 15, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_and_epi64() { + let a = _mm512_set_epi64(1 << 0 | 1 << 15, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let b = _mm512_set_epi64(1 << 13, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let r = _mm512_maskz_and_epi64(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_and_epi64(0b00001111, a, b); + let e = _mm512_set_epi64(0, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_and_epi64() { + let a = _mm256_set1_epi64x(1 << 0 | 1 << 15); + let b = _mm256_set1_epi64x(1 << 0); + let r = _mm256_mask_and_epi64(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_and_epi64(a, 0b00001111, a, b); + let e = _mm256_set1_epi64x(1 << 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_and_epi64() { + let a = _mm256_set1_epi64x(1 << 0 | 1 << 15); + let b = _mm256_set1_epi64x(1 << 0); + let r = _mm256_maskz_and_epi64(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_and_epi64(0b00001111, a, b); + let e = _mm256_set1_epi64x(1 << 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_and_epi64() { + let a = _mm_set1_epi64x(1 << 0 | 1 << 15); + let b = _mm_set1_epi64x(1 << 0); + let r = _mm_mask_and_epi64(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_and_epi64(a, 0b00000011, a, b); + let e = _mm_set1_epi64x(1 << 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_and_epi64() { + let a = _mm_set1_epi64x(1 << 0 | 1 << 15); + let b = _mm_set1_epi64x(1 << 0); + let r = _mm_maskz_and_epi64(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_and_epi64(0b00000011, a, b); + let e = _mm_set1_epi64x(1 << 0); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_and_si512() { + let a = _mm512_set_epi64(1 << 0 | 1 << 15, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let b = _mm512_set_epi64(1 << 13, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let r = _mm512_and_epi64(a, b); + let e = _mm512_set_epi64(0, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_or_epi64() { + let a = _mm512_set_epi64(1 << 0 | 1 << 15, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let b = _mm512_set_epi64(1 << 13, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let r = _mm512_or_epi64(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 1 << 0 | 1 << 13 | 1 << 15, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_or_epi64() { + let a = _mm512_set_epi64(1 << 0 | 1 << 15, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let b = _mm512_set_epi64(1 << 13, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let r = _mm512_mask_or_epi64(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_or_epi64(a, 0b11111111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 1 << 0 | 1 << 13 | 1 << 15, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_or_epi64() { + let a = _mm512_set_epi64(1 << 0 | 1 << 15, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let b = _mm512_set_epi64(1 << 13, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let r = _mm512_maskz_or_epi64(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_or_epi64(0b00001111, a, b); + let e = _mm512_set_epi64(0, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_or_epi64() { + let a = _mm256_set1_epi64x(1 << 0 | 1 << 15); + let b = _mm256_set1_epi64x(1 << 13); + let r = _mm256_or_epi64(a, b); + let e = _mm256_set1_epi64x(1 << 0 | 1 << 13 | 1 << 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_or_epi64() { + let a = _mm256_set1_epi64x(1 << 0 | 1 << 15); + let b = _mm256_set1_epi64x(1 << 13); + let r = _mm256_mask_or_epi64(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_or_epi64(a, 0b00001111, a, b); + let e = _mm256_set1_epi64x(1 << 0 | 1 << 13 | 1 << 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_or_epi64() { + let a = _mm256_set1_epi64x(1 << 0 | 1 << 15); + let b = _mm256_set1_epi64x(1 << 13); + let r = _mm256_maskz_or_epi64(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_or_epi64(0b00001111, a, b); + let e = _mm256_set1_epi64x(1 << 0 | 1 << 13 | 1 << 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_or_epi64() { + let a = _mm_set1_epi64x(1 << 0 | 1 << 15); + let b = _mm_set1_epi64x(1 << 13); + let r = _mm_or_epi64(a, b); + let e = _mm_set1_epi64x(1 << 0 | 1 << 13 | 1 << 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_or_epi64() { + let a = _mm_set1_epi64x(1 << 0 | 1 << 15); + let b = _mm_set1_epi64x(1 << 13); + let r = _mm_mask_or_epi64(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_or_epi64(a, 0b00000011, a, b); + let e = _mm_set1_epi64x(1 << 0 | 1 << 13 | 1 << 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_or_epi64() { + let a = _mm_set1_epi64x(1 << 0 | 1 << 15); + let b = _mm_set1_epi64x(1 << 13); + let r = _mm_maskz_or_epi64(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_or_epi64(0b00000011, a, b); + let e = _mm_set1_epi64x(1 << 0 | 1 << 13 | 1 << 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_or_si512() { + let a = _mm512_set_epi64(1 << 0 | 1 << 15, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let b = _mm512_set_epi64(1 << 13, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let r = _mm512_or_epi64(a, b); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 1 << 0 | 1 << 13 | 1 << 15, 0, 0, 0, + 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_xor_epi64() { + let a = _mm512_set_epi64(1 << 0 | 1 << 15, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let b = _mm512_set_epi64(1 << 13, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let r = _mm512_xor_epi64(a, b); + let e = _mm512_set_epi64(1 << 0 | 1 << 13 | 1 << 15, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_xor_epi64() { + let a = _mm512_set_epi64(1 << 0 | 1 << 15, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let b = _mm512_set_epi64(1 << 13, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let r = _mm512_mask_xor_epi64(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_xor_epi64(a, 0b11111111, a, b); + let e = _mm512_set_epi64(1 << 0 | 1 << 13 | 1 << 15, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_xor_epi64() { + let a = _mm512_set_epi64(1 << 0 | 1 << 15, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let b = _mm512_set_epi64(1 << 13, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let r = _mm512_maskz_xor_epi64(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_xor_epi64(0b00001111, a, b); + let e = _mm512_set_epi64(0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_xor_epi64() { + let a = _mm256_set1_epi64x(1 << 0 | 1 << 15); + let b = _mm256_set1_epi64x(1 << 13); + let r = _mm256_xor_epi64(a, b); + let e = _mm256_set1_epi64x(1 << 0 | 1 << 13 | 1 << 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_xor_epi64() { + let a = _mm256_set1_epi64x(1 << 0 | 1 << 15); + let b = _mm256_set1_epi64x(1 << 13); + let r = _mm256_mask_xor_epi64(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_xor_epi64(a, 0b00001111, a, b); + let e = _mm256_set1_epi64x(1 << 0 | 1 << 13 | 1 << 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_xor_epi64() { + let a = _mm256_set1_epi64x(1 << 0 | 1 << 15); + let b = _mm256_set1_epi64x(1 << 13); + let r = _mm256_maskz_xor_epi64(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_xor_epi64(0b00001111, a, b); + let e = _mm256_set1_epi64x(1 << 0 | 1 << 13 | 1 << 15); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_xor_epi64() { + let a = _mm_set1_epi64x(1 << 0 | 1 << 15); + let b = _mm_set1_epi64x(1 << 13); + let r = _mm_xor_epi64(a, b); + let e = _mm_set1_epi64x(1 << 0 | 1 << 13 | 1 << 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_xor_epi64() { + let a = _mm_set1_epi64x(1 << 0 | 1 << 15); + let b = _mm_set1_epi64x(1 << 13); + let r = _mm_mask_xor_epi64(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_xor_epi64(a, 0b00000011, a, b); + let e = _mm_set1_epi64x(1 << 0 | 1 << 13 | 1 << 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_xor_epi64() { + let a = _mm_set1_epi64x(1 << 0 | 1 << 15); + let b = _mm_set1_epi64x(1 << 13); + let r = _mm_maskz_xor_epi64(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_xor_epi64(0b00000011, a, b); + let e = _mm_set1_epi64x(1 << 0 | 1 << 13 | 1 << 15); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_xor_si512() { + let a = _mm512_set_epi64(1 << 0 | 1 << 15, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let b = _mm512_set_epi64(1 << 13, 0, 0, 0, 0, 0, 0, 1 << 1 | 1 << 2 | 1 << 3); + let r = _mm512_xor_epi64(a, b); + let e = _mm512_set_epi64(1 << 0 | 1 << 13 | 1 << 15, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_andnot_epi64() { + let a = _mm512_set1_epi64(0); + let b = _mm512_set1_epi64(1 << 3 | 1 << 4); + let r = _mm512_andnot_epi64(a, b); + let e = _mm512_set1_epi64(1 << 3 | 1 << 4); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_andnot_epi64() { + let a = _mm512_set1_epi64(1 << 1 | 1 << 2); + let b = _mm512_set1_epi64(1 << 3 | 1 << 4); + let r = _mm512_mask_andnot_epi64(a, 0, a, b); + assert_eq_m512i(r, a); + let r = _mm512_mask_andnot_epi64(a, 0b11111111, a, b); + let e = _mm512_set1_epi64(1 << 3 | 1 << 4); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_andnot_epi64() { + let a = _mm512_set1_epi64(1 << 1 | 1 << 2); + let b = _mm512_set1_epi64(1 << 3 | 1 << 4); + let r = _mm512_maskz_andnot_epi64(0, a, b); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_andnot_epi64(0b00001111, a, b); + #[rustfmt::skip] + let e = _mm512_set_epi64( + 0, 0, 0, 0, + 1 << 3 | 1 << 4, 1 << 3 | 1 << 4, 1 << 3 | 1 << 4, 1 << 3 | 1 << 4, + ); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_andnot_epi64() { + let a = _mm256_set1_epi64x(1 << 1 | 1 << 2); + let b = _mm256_set1_epi64x(1 << 3 | 1 << 4); + let r = _mm256_mask_andnot_epi64(a, 0, a, b); + assert_eq_m256i(r, a); + let r = _mm256_mask_andnot_epi64(a, 0b00001111, a, b); + let e = _mm256_set1_epi64x(1 << 3 | 1 << 4); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_andnot_epi64() { + let a = _mm256_set1_epi64x(1 << 1 | 1 << 2); + let b = _mm256_set1_epi64x(1 << 3 | 1 << 4); + let r = _mm256_maskz_andnot_epi64(0, a, b); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_andnot_epi64(0b00001111, a, b); + let e = _mm256_set1_epi64x(1 << 3 | 1 << 4); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_andnot_epi64() { + let a = _mm_set1_epi64x(1 << 1 | 1 << 2); + let b = _mm_set1_epi64x(1 << 3 | 1 << 4); + let r = _mm_mask_andnot_epi64(a, 0, a, b); + assert_eq_m128i(r, a); + let r = _mm_mask_andnot_epi64(a, 0b00000011, a, b); + let e = _mm_set1_epi64x(1 << 3 | 1 << 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_andnot_epi64() { + let a = _mm_set1_epi64x(1 << 1 | 1 << 2); + let b = _mm_set1_epi64x(1 << 3 | 1 << 4); + let r = _mm_maskz_andnot_epi64(0, a, b); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_andnot_epi64(0b00000011, a, b); + let e = _mm_set1_epi64x(1 << 3 | 1 << 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_andnot_si512() { + let a = _mm512_set1_epi64(0); + let b = _mm512_set1_epi64(1 << 3 | 1 << 4); + let r = _mm512_andnot_si512(a, b); + let e = _mm512_set1_epi64(1 << 3 | 1 << 4); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_add_epi64() { + let a = _mm512_set1_epi64(1); + let e: i64 = _mm512_reduce_add_epi64(a); + assert_eq!(8, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_add_epi64() { + let a = _mm512_set1_epi64(1); + let e: i64 = _mm512_mask_reduce_add_epi64(0b11110000, a); + assert_eq!(4, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_add_pd() { + let a = _mm512_set1_pd(1.); + let e: f64 = _mm512_reduce_add_pd(a); + assert_eq!(8., e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_add_pd() { + let a = _mm512_set1_pd(1.); + let e: f64 = _mm512_mask_reduce_add_pd(0b11110000, a); + assert_eq!(4., e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_mul_epi64() { + let a = _mm512_set1_epi64(2); + let e: i64 = _mm512_reduce_mul_epi64(a); + assert_eq!(256, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_mul_epi64() { + let a = _mm512_set1_epi64(2); + let e: i64 = _mm512_mask_reduce_mul_epi64(0b11110000, a); + assert_eq!(16, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_mul_pd() { + let a = _mm512_set1_pd(2.); + let e: f64 = _mm512_reduce_mul_pd(a); + assert_eq!(256., e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_mul_pd() { + let a = _mm512_set1_pd(2.); + let e: f64 = _mm512_mask_reduce_mul_pd(0b11110000, a); + assert_eq!(16., e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_max_epi64() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let e: i64 = _mm512_reduce_max_epi64(a); + assert_eq!(7, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_max_epi64() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let e: i64 = _mm512_mask_reduce_max_epi64(0b11110000, a); + assert_eq!(3, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_max_epu64() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let e: u64 = _mm512_reduce_max_epu64(a); + assert_eq!(7, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_max_epu64() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let e: u64 = _mm512_mask_reduce_max_epu64(0b11110000, a); + assert_eq!(3, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_max_pd() { + let a = _mm512_set_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let e: f64 = _mm512_reduce_max_pd(a); + assert_eq!(7., e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_max_pd() { + let a = _mm512_set_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let e: f64 = _mm512_mask_reduce_max_pd(0b11110000, a); + assert_eq!(3., e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_min_epi64() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let e: i64 = _mm512_reduce_min_epi64(a); + assert_eq!(0, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_min_epi64() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let e: i64 = _mm512_mask_reduce_min_epi64(0b11110000, a); + assert_eq!(0, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_min_epu64() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let e: u64 = _mm512_reduce_min_epu64(a); + assert_eq!(0, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_min_epu64() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let e: u64 = _mm512_mask_reduce_min_epu64(0b11110000, a); + assert_eq!(0, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_min_pd() { + let a = _mm512_set_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let e: f64 = _mm512_reduce_min_pd(a); + assert_eq!(0., e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_min_pd() { + let a = _mm512_set_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let e: f64 = _mm512_mask_reduce_min_pd(0b11110000, a); + assert_eq!(0., e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_and_epi64() { + let a = _mm512_set_epi64(1, 1, 1, 1, 2, 2, 2, 2); + let e: i64 = _mm512_reduce_and_epi64(a); + assert_eq!(0, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_and_epi64() { + let a = _mm512_set_epi64(1, 1, 1, 1, 2, 2, 2, 2); + let e: i64 = _mm512_mask_reduce_and_epi64(0b11110000, a); + assert_eq!(1, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_reduce_or_epi64() { + let a = _mm512_set_epi64(1, 1, 1, 1, 2, 2, 2, 2); + let e: i64 = _mm512_reduce_or_epi64(a); + assert_eq!(3, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_reduce_or_epi64() { + let a = _mm512_set_epi64(1, 1, 1, 1, 2, 2, 2, 2); + let e: i64 = _mm512_mask_reduce_or_epi64(0b11110000, a); + assert_eq!(1, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_extractf64x4_pd() { + let a = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); + let r = _mm512_extractf64x4_pd::<1>(a); + let e = _mm256_setr_pd(5., 6., 7., 8.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_extractf64x4_pd() { + let a = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); + let src = _mm256_set1_pd(100.); + let r = _mm512_mask_extractf64x4_pd::<1>(src, 0, a); + assert_eq_m256d(r, src); + let r = _mm512_mask_extractf64x4_pd::<1>(src, 0b11111111, a); + let e = _mm256_setr_pd(5., 6., 7., 8.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_extractf64x4_pd() { + let a = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); + let r = _mm512_maskz_extractf64x4_pd::<1>(0, a); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm512_maskz_extractf64x4_pd::<1>(0b00000001, a); + let e = _mm256_setr_pd(5., 0., 0., 0.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_extracti64x4_epi64() { + let a = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); + let r = _mm512_extracti64x4_epi64::<0x1>(a); + let e = _mm256_setr_epi64x(5, 6, 7, 8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_extracti64x4_epi64() { + let a = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); + let src = _mm256_set1_epi64x(100); + let r = _mm512_mask_extracti64x4_epi64::<0x1>(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm512_mask_extracti64x4_epi64::<0x1>(src, 0b11111111, a); + let e = _mm256_setr_epi64x(5, 6, 7, 8); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_extracti64x4_epi64() { + let a = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); + let r = _mm512_maskz_extracti64x4_epi64::<0x1>(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm512_maskz_extracti64x4_epi64::<0x1>(0b00000001, a); + let e = _mm256_setr_epi64x(5, 0, 0, 0); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_compress_epi64() { + let src = _mm512_set1_epi64(200); + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm512_mask_compress_epi64(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_compress_epi64(src, 0b01010101, a); + let e = _mm512_set_epi64(200, 200, 200, 200, 1, 3, 5, 7); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_compress_epi64() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm512_maskz_compress_epi64(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_compress_epi64(0b01010101, a); + let e = _mm512_set_epi64(0, 0, 0, 0, 1, 3, 5, 7); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_compress_epi64() { + let src = _mm256_set1_epi64x(200); + let a = _mm256_set_epi64x(0, 1, 2, 3); + let r = _mm256_mask_compress_epi64(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_compress_epi64(src, 0b00000101, a); + let e = _mm256_set_epi64x(200, 200, 1, 3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_compress_epi64() { + let a = _mm256_set_epi64x(0, 1, 2, 3); + let r = _mm256_maskz_compress_epi64(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_compress_epi64(0b00000101, a); + let e = _mm256_set_epi64x(0, 0, 1, 3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_compress_epi64() { + let src = _mm_set1_epi64x(200); + let a = _mm_set_epi64x(0, 1); + let r = _mm_mask_compress_epi64(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_compress_epi64(src, 0b00000001, a); + let e = _mm_set_epi64x(200, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_compress_epi64() { + let a = _mm_set_epi64x(0, 1); + let r = _mm_maskz_compress_epi64(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_compress_epi64(0b00000001, a); + let e = _mm_set_epi64x(0, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_compress_pd() { + let src = _mm512_set1_pd(200.); + let a = _mm512_set_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm512_mask_compress_pd(src, 0, a); + assert_eq_m512d(r, src); + let r = _mm512_mask_compress_pd(src, 0b01010101, a); + let e = _mm512_set_pd(200., 200., 200., 200., 1., 3., 5., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_compress_pd() { + let a = _mm512_set_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm512_maskz_compress_pd(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_compress_pd(0b01010101, a); + let e = _mm512_set_pd(0., 0., 0., 0., 1., 3., 5., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_compress_pd() { + let src = _mm256_set1_pd(200.); + let a = _mm256_set_pd(0., 1., 2., 3.); + let r = _mm256_mask_compress_pd(src, 0, a); + assert_eq_m256d(r, src); + let r = _mm256_mask_compress_pd(src, 0b00000101, a); + let e = _mm256_set_pd(200., 200., 1., 3.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_compress_pd() { + let a = _mm256_set_pd(0., 1., 2., 3.); + let r = _mm256_maskz_compress_pd(0, a); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_compress_pd(0b00000101, a); + let e = _mm256_set_pd(0., 0., 1., 3.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_compress_pd() { + let src = _mm_set1_pd(200.); + let a = _mm_set_pd(0., 1.); + let r = _mm_mask_compress_pd(src, 0, a); + assert_eq_m128d(r, src); + let r = _mm_mask_compress_pd(src, 0b00000001, a); + let e = _mm_set_pd(200., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_compress_pd() { + let a = _mm_set_pd(0., 1.); + let r = _mm_maskz_compress_pd(0, a); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_compress_pd(0b00000001, a); + let e = _mm_set_pd(0., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_expand_epi64() { + let src = _mm512_set1_epi64(200); + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm512_mask_expand_epi64(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_expand_epi64(src, 0b01010101, a); + let e = _mm512_set_epi64(200, 4, 200, 5, 200, 6, 200, 7); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_expand_epi64() { + let a = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm512_maskz_expand_epi64(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_expand_epi64(0b01010101, a); + let e = _mm512_set_epi64(0, 4, 0, 5, 0, 6, 0, 7); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_expand_epi64() { + let src = _mm256_set1_epi64x(200); + let a = _mm256_set_epi64x(0, 1, 2, 3); + let r = _mm256_mask_expand_epi64(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_expand_epi64(src, 0b00000101, a); + let e = _mm256_set_epi64x(200, 2, 200, 3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_expand_epi64() { + let a = _mm256_set_epi64x(0, 1, 2, 3); + let r = _mm256_maskz_expand_epi64(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_expand_epi64(0b00000101, a); + let e = _mm256_set_epi64x(0, 2, 0, 3); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_expand_epi64() { + let src = _mm_set1_epi64x(200); + let a = _mm_set_epi64x(0, 1); + let r = _mm_mask_expand_epi64(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_expand_epi64(src, 0b00000001, a); + let e = _mm_set_epi64x(200, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_expand_epi64() { + let a = _mm_set_epi64x(0, 1); + let r = _mm_maskz_expand_epi64(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_expand_epi64(0b00000001, a); + let e = _mm_set_epi64x(0, 1); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_expand_pd() { + let src = _mm512_set1_pd(200.); + let a = _mm512_set_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm512_mask_expand_pd(src, 0, a); + assert_eq_m512d(r, src); + let r = _mm512_mask_expand_pd(src, 0b01010101, a); + let e = _mm512_set_pd(200., 4., 200., 5., 200., 6., 200., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_expand_pd() { + let a = _mm512_set_pd(0., 1., 2., 3., 4., 5., 6., 7.); + let r = _mm512_maskz_expand_pd(0, a); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_expand_pd(0b01010101, a); + let e = _mm512_set_pd(0., 4., 0., 5., 0., 6., 0., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_expand_pd() { + let src = _mm256_set1_pd(200.); + let a = _mm256_set_pd(0., 1., 2., 3.); + let r = _mm256_mask_expand_pd(src, 0, a); + assert_eq_m256d(r, src); + let r = _mm256_mask_expand_pd(src, 0b00000101, a); + let e = _mm256_set_pd(200., 2., 200., 3.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_expand_pd() { + let a = _mm256_set_pd(0., 1., 2., 3.); + let r = _mm256_maskz_expand_pd(0, a); + assert_eq_m256d(r, _mm256_setzero_pd()); + let r = _mm256_maskz_expand_pd(0b00000101, a); + let e = _mm256_set_pd(0., 2., 0., 3.); + assert_eq_m256d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_expand_pd() { + let src = _mm_set1_pd(200.); + let a = _mm_set_pd(0., 1.); + let r = _mm_mask_expand_pd(src, 0, a); + assert_eq_m128d(r, src); + let r = _mm_mask_expand_pd(src, 0b00000001, a); + let e = _mm_set_pd(200., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_expand_pd() { + let a = _mm_set_pd(0., 1.); + let r = _mm_maskz_expand_pd(0, a); + assert_eq_m128d(r, _mm_setzero_pd()); + let r = _mm_maskz_expand_pd(0b00000001, a); + let e = _mm_set_pd(0., 1.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_loadu_epi64() { + let a = &[4, 3, 2, 5, -8, -9, -64, -50]; + let p = a.as_ptr(); + let r = _mm512_loadu_epi64(black_box(p)); + let e = _mm512_setr_epi64(4, 3, 2, 5, -8, -9, -64, -50); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_loadu_epi64() { + let a = &[4, 3, 2, 5]; + let p = a.as_ptr(); + let r = _mm256_loadu_epi64(black_box(p)); + let e = _mm256_setr_epi64x(4, 3, 2, 5); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_loadu_epi64() { + let a = &[4, 3]; + let p = a.as_ptr(); + let r = _mm_loadu_epi64(black_box(p)); + let e = _mm_setr_epi64x(4, 3); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepi64_storeu_epi16() { + let a = _mm512_set1_epi64(9); + let mut r = _mm_undefined_si128(); + _mm512_mask_cvtepi64_storeu_epi16(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set1_epi16(9); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepi64_storeu_epi16() { + let a = _mm256_set1_epi64x(9); + let mut r = _mm_set1_epi16(0); + _mm256_mask_cvtepi64_storeu_epi16(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 9, 9, 9, 9); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepi64_storeu_epi16() { + let a = _mm_set1_epi64x(9); + let mut r = _mm_set1_epi16(0); + _mm_mask_cvtepi64_storeu_epi16(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 0, 0, 9, 9); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtsepi64_storeu_epi16() { + let a = _mm512_set1_epi64(i64::MAX); + let mut r = _mm_undefined_si128(); + _mm512_mask_cvtsepi64_storeu_epi16(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set1_epi16(i16::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtsepi64_storeu_epi16() { + let a = _mm256_set1_epi64x(i64::MAX); + let mut r = _mm_set1_epi16(0); + _mm256_mask_cvtsepi64_storeu_epi16(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set_epi16(0, 0, 0, 0, i16::MAX, i16::MAX, i16::MAX, i16::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtsepi64_storeu_epi16() { + let a = _mm_set1_epi64x(i64::MAX); + let mut r = _mm_set1_epi16(0); + _mm_mask_cvtsepi64_storeu_epi16(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 0, 0, i16::MAX, i16::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtusepi64_storeu_epi16() { + let a = _mm512_set1_epi64(i64::MAX); + let mut r = _mm_undefined_si128(); + _mm512_mask_cvtusepi64_storeu_epi16(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set1_epi16(u16::MAX as i16); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtusepi64_storeu_epi16() { + let a = _mm256_set1_epi64x(i64::MAX); + let mut r = _mm_set1_epi16(0); + _mm256_mask_cvtusepi64_storeu_epi16(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set_epi16( + 0, + 0, + 0, + 0, + u16::MAX as i16, + u16::MAX as i16, + u16::MAX as i16, + u16::MAX as i16, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtusepi64_storeu_epi16() { + let a = _mm_set1_epi64x(i64::MAX); + let mut r = _mm_set1_epi16(0); + _mm_mask_cvtusepi64_storeu_epi16(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set_epi16(0, 0, 0, 0, 0, 0, u16::MAX as i16, u16::MAX as i16); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepi64_storeu_epi8() { + let a = _mm512_set1_epi64(9); + let mut r = _mm_set1_epi8(0); + _mm512_mask_cvtepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepi64_storeu_epi8() { + let a = _mm256_set1_epi64x(9); + let mut r = _mm_set1_epi8(0); + _mm256_mask_cvtepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepi64_storeu_epi8() { + let a = _mm_set1_epi64x(9); + let mut r = _mm_set1_epi8(0); + _mm_mask_cvtepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtsepi64_storeu_epi8() { + let a = _mm512_set1_epi64(i64::MAX); + let mut r = _mm_set1_epi8(0); + _mm512_mask_cvtsepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + #[rustfmt::skip] + let e = _mm_set_epi8( + 0, 0, 0, 0, + 0, 0, 0, 0, + i8::MAX, i8::MAX, i8::MAX, i8::MAX, + i8::MAX, i8::MAX, i8::MAX, i8::MAX, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtsepi64_storeu_epi8() { + let a = _mm256_set1_epi64x(i64::MAX); + let mut r = _mm_set1_epi8(0); + _mm256_mask_cvtsepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + #[rustfmt::skip] + let e = _mm_set_epi8( + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + i8::MAX, i8::MAX, i8::MAX, i8::MAX, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtsepi64_storeu_epi8() { + let a = _mm_set1_epi64x(i64::MAX); + let mut r = _mm_set1_epi8(0); + _mm_mask_cvtsepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, i8::MAX, i8::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtusepi64_storeu_epi8() { + let a = _mm512_set1_epi64(i64::MAX); + let mut r = _mm_set1_epi8(0); + _mm512_mask_cvtusepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + #[rustfmt::skip] + let e = _mm_set_epi8( + 0, 0, 0, 0, + 0, 0, 0, 0, + u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, + u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtusepi64_storeu_epi8() { + let a = _mm256_set1_epi64x(i64::MAX); + let mut r = _mm_set1_epi8(0); + _mm256_mask_cvtusepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + #[rustfmt::skip] + let e = _mm_set_epi8( + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, u8::MAX as i8, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtusepi64_storeu_epi8() { + let a = _mm_set1_epi64x(i64::MAX); + let mut r = _mm_set1_epi8(0); + _mm_mask_cvtusepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + #[rustfmt::skip] + let e = _mm_set_epi8( + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, u8::MAX as i8, u8::MAX as i8, + ); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtepi64_storeu_epi32() { + let a = _mm512_set1_epi64(9); + let mut r = _mm256_undefined_si256(); + _mm512_mask_cvtepi64_storeu_epi32(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm256_set1_epi32(9); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtepi64_storeu_epi32() { + let a = _mm256_set1_epi64x(9); + let mut r = _mm_set1_epi32(0); + _mm256_mask_cvtepi64_storeu_epi32(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set_epi32(9, 9, 9, 9); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtepi64_storeu_epi32() { + let a = _mm_set1_epi64x(9); + let mut r = _mm_set1_epi16(0); + _mm_mask_cvtepi64_storeu_epi32(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm_set_epi32(0, 0, 9, 9); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtsepi64_storeu_epi32() { + let a = _mm512_set1_epi64(i64::MAX); + let mut r = _mm256_undefined_si256(); + _mm512_mask_cvtsepi64_storeu_epi32(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm256_set1_epi32(i32::MAX); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtsepi64_storeu_epi32() { + let a = _mm256_set1_epi64x(i64::MAX); + let mut r = _mm_set1_epi32(0); + _mm256_mask_cvtsepi64_storeu_epi32(&mut r as *mut _ as *mut i8, 0b00001111, a); + let e = _mm_set1_epi32(i32::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtsepi64_storeu_epi32() { + let a = _mm_set1_epi64x(i64::MAX); + let mut r = _mm_set1_epi16(0); + _mm_mask_cvtsepi64_storeu_epi32(&mut r as *mut _ as *mut i8, 0b00000011, a); + let e = _mm_set_epi32(0, 0, i32::MAX, i32::MAX); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_cvtusepi64_storeu_epi32() { + let a = _mm512_set1_epi64(i64::MAX); + let mut r = _mm256_undefined_si256(); + _mm512_mask_cvtusepi64_storeu_epi32(&mut r as *mut _ as *mut i8, 0b11111111, a); + let e = _mm256_set1_epi32(u32::MAX as i32); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_cvtusepi64_storeu_epi32() { + let a = _mm256_set1_epi64x(i64::MAX); + let mut r = _mm_set1_epi32(0); + _mm256_mask_cvtusepi64_storeu_epi32(&mut r as *mut _ as *mut i8, 0b00001111, a); + let e = _mm_set1_epi32(u32::MAX as i32); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_cvtusepi64_storeu_epi32() { + let a = _mm_set1_epi64x(i64::MAX); + let mut r = _mm_set1_epi16(0); + _mm_mask_cvtusepi64_storeu_epi32(&mut r as *mut _ as *mut i8, 0b00000011, a); + let e = _mm_set_epi32(0, 0, u32::MAX as i32, u32::MAX as i32); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_storeu_epi64() { + let a = _mm512_set1_epi64(9); + let mut r = _mm512_set1_epi64(0); + _mm512_storeu_epi64(&mut r as *mut _ as *mut i64, a); + assert_eq_m512i(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_storeu_epi64() { + let a = _mm256_set1_epi64x(9); + let mut r = _mm256_set1_epi64x(0); + _mm256_storeu_epi64(&mut r as *mut _ as *mut i64, a); + assert_eq_m256i(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_storeu_epi64() { + let a = _mm_set1_epi64x(9); + let mut r = _mm_set1_epi64x(0); + _mm_storeu_epi64(&mut r as *mut _ as *mut i64, a); + assert_eq_m128i(r, a); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_load_epi64() { + #[repr(align(64))] + struct Align { + data: [i64; 8], // 64 bytes + } + let a = Align { + data: [4, 3, 2, 5, -8, -9, -64, -50], + }; + let p = (a.data).as_ptr(); + let r = _mm512_load_epi64(black_box(p)); + let e = _mm512_setr_epi64(4, 3, 2, 5, -8, -9, -64, -50); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_load_epi64() { + #[repr(align(64))] + struct Align { + data: [i64; 4], + } + let a = Align { data: [4, 3, 2, 5] }; + let p = (a.data).as_ptr(); + let r = _mm256_load_epi64(black_box(p)); + let e = _mm256_set_epi64x(5, 2, 3, 4); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_load_epi64() { + #[repr(align(64))] + struct Align { + data: [i64; 2], + } + let a = Align { data: [4, 3] }; + let p = (a.data).as_ptr(); + let r = _mm_load_epi64(black_box(p)); + let e = _mm_set_epi64x(3, 4); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_store_epi64() { + let a = _mm512_set1_epi64(9); + let mut r = _mm512_set1_epi64(0); + _mm512_store_epi64(&mut r as *mut _ as *mut i64, a); + assert_eq_m512i(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_store_epi64() { + let a = _mm256_set1_epi64x(9); + let mut r = _mm256_set1_epi64x(0); + _mm256_store_epi64(&mut r as *mut _ as *mut i64, a); + assert_eq_m256i(r, a); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_store_epi64() { + let a = _mm_set1_epi64x(9); + let mut r = _mm_set1_epi64x(0); + _mm_store_epi64(&mut r as *mut _ as *mut i64, a); + assert_eq_m128i(r, a); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_load_pd() { + #[repr(align(64))] + struct Align { + data: [f64; 8], // 64 bytes + } + let a = Align { + data: [4., 3., 2., 5., -8., -9., -64., -50.], + }; + let p = (a.data).as_ptr(); + let r = _mm512_load_pd(black_box(p)); + let e = _mm512_setr_pd(4., 3., 2., 5., -8., -9., -64., -50.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_store_pd() { + let a = _mm512_set1_pd(9.); + let mut r = _mm512_undefined_pd(); + _mm512_store_pd(&mut r as *mut _ as *mut f64, a); + assert_eq_m512d(r, a); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_test_epi64_mask() { + let a = _mm512_set1_epi64(1 << 0); + let b = _mm512_set1_epi64(1 << 0 | 1 << 1); + let r = _mm512_test_epi64_mask(a, b); + let e: __mmask8 = 0b11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_test_epi64_mask() { + let a = _mm512_set1_epi64(1 << 0); + let b = _mm512_set1_epi64(1 << 0 | 1 << 1); + let r = _mm512_mask_test_epi64_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm512_mask_test_epi64_mask(0b11111111, a, b); + let e: __mmask8 = 0b11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_test_epi64_mask() { + let a = _mm256_set1_epi64x(1 << 0); + let b = _mm256_set1_epi64x(1 << 0 | 1 << 1); + let r = _mm256_test_epi64_mask(a, b); + let e: __mmask8 = 0b00001111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_test_epi64_mask() { + let a = _mm256_set1_epi64x(1 << 0); + let b = _mm256_set1_epi64x(1 << 0 | 1 << 1); + let r = _mm256_mask_test_epi64_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm256_mask_test_epi64_mask(0b00001111, a, b); + let e: __mmask8 = 0b00001111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_test_epi64_mask() { + let a = _mm_set1_epi64x(1 << 0); + let b = _mm_set1_epi64x(1 << 0 | 1 << 1); + let r = _mm_test_epi64_mask(a, b); + let e: __mmask8 = 0b00000011; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_test_epi64_mask() { + let a = _mm_set1_epi64x(1 << 0); + let b = _mm_set1_epi64x(1 << 0 | 1 << 1); + let r = _mm_mask_test_epi64_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm_mask_test_epi64_mask(0b00000011, a, b); + let e: __mmask8 = 0b00000011; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_testn_epi64_mask() { + let a = _mm512_set1_epi64(1 << 0); + let b = _mm512_set1_epi64(1 << 0 | 1 << 1); + let r = _mm512_testn_epi64_mask(a, b); + let e: __mmask8 = 0b00000000; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_testn_epi64_mask() { + let a = _mm512_set1_epi64(1 << 0); + let b = _mm512_set1_epi64(1 << 1); + let r = _mm512_mask_testn_epi64_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm512_mask_testn_epi64_mask(0b11111111, a, b); + let e: __mmask8 = 0b11111111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_testn_epi64_mask() { + let a = _mm256_set1_epi64x(1 << 0); + let b = _mm256_set1_epi64x(1 << 1); + let r = _mm256_testn_epi64_mask(a, b); + let e: __mmask8 = 0b00001111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_testn_epi64_mask() { + let a = _mm256_set1_epi64x(1 << 0); + let b = _mm256_set1_epi64x(1 << 1); + let r = _mm256_mask_testn_epi64_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm256_mask_testn_epi64_mask(0b11111111, a, b); + let e: __mmask8 = 0b00001111; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_testn_epi64_mask() { + let a = _mm_set1_epi64x(1 << 0); + let b = _mm_set1_epi64x(1 << 1); + let r = _mm_testn_epi64_mask(a, b); + let e: __mmask8 = 0b00000011; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_testn_epi64_mask() { + let a = _mm_set1_epi64x(1 << 0); + let b = _mm_set1_epi64x(1 << 1); + let r = _mm_mask_testn_epi64_mask(0, a, b); + assert_eq!(r, 0); + let r = _mm_mask_testn_epi64_mask(0b11111111, a, b); + let e: __mmask8 = 0b00000011; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_stream_pd() { + #[repr(align(64))] + struct Memory { + pub data: [f64; 8], + } + let a = _mm512_set1_pd(7.0); + let mut mem = Memory { data: [-1.0; 8] }; + + _mm512_stream_pd(&mut mem.data[0] as *mut f64, a); + for i in 0..8 { + assert_eq!(mem.data[i], get_m512d(a, i)); + } + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_stream_si512() { + #[repr(align(64))] + struct Memory { + pub data: [i64; 8], + } + let a = _mm512_set1_epi64(7); + let mut mem = Memory { data: [-1; 8] }; + + _mm512_stream_si512(&mut mem.data[0] as *mut i64, a); + for i in 0..8 { + assert_eq!(mem.data[i], get_m512i(a, i)); + } + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_set1_epi64() { + let src = _mm512_set1_epi64(2); + let a: i64 = 11; + let r = _mm512_mask_set1_epi64(src, 0, a); + assert_eq_m512i(r, src); + let r = _mm512_mask_set1_epi64(src, 0b11111111, a); + let e = _mm512_set1_epi64(11); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_set1_epi64() { + let a: i64 = 11; + let r = _mm512_maskz_set1_epi64(0, a); + assert_eq_m512i(r, _mm512_setzero_si512()); + let r = _mm512_maskz_set1_epi64(0b11111111, a); + let e = _mm512_set1_epi64(11); + assert_eq_m512i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_mask_set1_epi64() { + let src = _mm256_set1_epi64x(2); + let a: i64 = 11; + let r = _mm256_mask_set1_epi64(src, 0, a); + assert_eq_m256i(r, src); + let r = _mm256_mask_set1_epi64(src, 0b00001111, a); + let e = _mm256_set1_epi64x(11); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm256_maskz_set1_epi64() { + let a: i64 = 11; + let r = _mm256_maskz_set1_epi64(0, a); + assert_eq_m256i(r, _mm256_setzero_si256()); + let r = _mm256_maskz_set1_epi64(0b00001111, a); + let e = _mm256_set1_epi64x(11); + assert_eq_m256i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_mask_set1_epi64() { + let src = _mm_set1_epi64x(2); + let a: i64 = 11; + let r = _mm_mask_set1_epi64(src, 0, a); + assert_eq_m128i(r, src); + let r = _mm_mask_set1_epi64(src, 0b00000011, a); + let e = _mm_set1_epi64x(11); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + unsafe fn test_mm_maskz_set1_epi64() { + let a: i64 = 11; + let r = _mm_maskz_set1_epi64(0, a); + assert_eq_m128i(r, _mm_setzero_si128()); + let r = _mm_maskz_set1_epi64(0b00000011, a); + let e = _mm_set1_epi64x(11); + assert_eq_m128i(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtsd_i64() { + let a = _mm_set_pd(1., -1.5); + let r = _mm_cvtsd_i64(a); + let e: i64 = -2; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtss_i64() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_cvtss_i64(a); + let e: i64 = -2; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundi64_ss() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let b: i64 = 9; + let r = _mm_cvt_roundi64_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_ps(0., -0.5, 1., 9.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundsi64_ss() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let b: i64 = 9; + let r = _mm_cvt_roundsi64_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_ps(0., -0.5, 1., 9.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvti64_ss() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let b: i64 = 9; + let r = _mm_cvti64_ss(a, b); + let e = _mm_set_ps(0., -0.5, 1., 9.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvti64_sd() { + let a = _mm_set_pd(1., -1.5); + let b: i64 = 9; + let r = _mm_cvti64_sd(a, b); + let e = _mm_set_pd(1., 9.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundsd_si64() { + let a = _mm_set_pd(1., -1.5); + let r = _mm_cvt_roundsd_si64::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); + let e: i64 = -1; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundsd_i64() { + let a = _mm_set_pd(1., -1.5); + let r = _mm_cvt_roundsd_i64::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); + let e: i64 = -1; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundsd_u64() { + let a = _mm_set_pd(1., f64::MAX); + let r = _mm_cvt_roundsd_u64::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); + let e: u64 = u64::MAX; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtsd_u64() { + let a = _mm_set_pd(1., -1.5); + let r = _mm_cvtsd_u64(a); + let e: u64 = u64::MAX; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundss_i64() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_cvt_roundss_i64::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); + let e: i64 = -1; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundss_si64() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_cvt_roundss_si64::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); + let e: i64 = -1; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundss_u64() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_cvt_roundss_u64::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); + let e: u64 = u64::MAX; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtss_u64() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_cvtss_u64(a); + let e: u64 = u64::MAX; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvttsd_i64() { + let a = _mm_set_pd(1., -1.5); + let r = _mm_cvttsd_i64(a); + let e: i64 = -2; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtt_roundsd_i64() { + let a = _mm_set_pd(1., -1.5); + let r = _mm_cvtt_roundsd_i64::<_MM_FROUND_CUR_DIRECTION>(a); + let e: i64 = -2; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtt_roundsd_si64() { + let a = _mm_set_pd(1., -1.5); + let r = _mm_cvtt_roundsd_si64::<_MM_FROUND_CUR_DIRECTION>(a); + let e: i64 = -2; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtt_roundsd_u64() { + let a = _mm_set_pd(1., -1.5); + let r = _mm_cvtt_roundsd_u64::<_MM_FROUND_CUR_DIRECTION>(a); + let e: u64 = u64::MAX; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvttsd_u64() { + let a = _mm_set_pd(1., -1.5); + let r = _mm_cvttsd_u64(a); + let e: u64 = u64::MAX; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvttss_i64() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_cvttss_i64(a); + let e: i64 = -2; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtt_roundss_i64() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_cvtt_roundss_i64::<_MM_FROUND_CUR_DIRECTION>(a); + let e: i64 = -2; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtt_roundss_si64() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_cvtt_roundss_si64::<_MM_FROUND_CUR_DIRECTION>(a); + let e: i64 = -2; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtt_roundss_u64() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_cvtt_roundss_u64::<_MM_FROUND_CUR_DIRECTION>(a); + let e: u64 = u64::MAX; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvttss_u64() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let r = _mm_cvttss_u64(a); + let e: u64 = u64::MAX; + assert_eq!(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtu64_ss() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let b: u64 = 9; + let r = _mm_cvtu64_ss(a, b); + let e = _mm_set_ps(0., -0.5, 1., 9.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvtu64_sd() { + let a = _mm_set_pd(1., -1.5); + let b: u64 = 9; + let r = _mm_cvtu64_sd(a, b); + let e = _mm_set_pd(1., 9.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundu64_ss() { + let a = _mm_set_ps(0., -0.5, 1., -1.5); + let b: u64 = 9; + let r = _mm_cvt_roundu64_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_ps(0., -0.5, 1., 9.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundu64_sd() { + let a = _mm_set_pd(1., -1.5); + let b: u64 = 9; + let r = _mm_cvt_roundu64_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_pd(1., 9.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundi64_sd() { + let a = _mm_set_pd(1., -1.5); + let b: i64 = 9; + let r = _mm_cvt_roundi64_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_pd(1., 9.); + assert_eq_m128d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm_cvt_roundsi64_sd() { + let a = _mm_set_pd(1., -1.5); + let b: i64 = 9; + let r = _mm_cvt_roundsi64_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); + let e = _mm_set_pd(1., 9.); + assert_eq_m128d(r, e); + } +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/bt.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/bt.rs index 9c6dcf7b6..53da9d02f 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/bt.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/bt.rs @@ -1,55 +1,88 @@ +use crate::arch::asm; #[cfg(test)] use stdarch_test::assert_instr; +// x32 wants to use a 32-bit address size, but asm! defaults to using the full +// register name (e.g. rax). We have to explicitly override the placeholder to +// use the 32-bit register name in that case. +#[cfg(target_pointer_width = "32")] +macro_rules! bt { + ($inst:expr) => { + concat!($inst, " {b}, ({p:e})") + }; +} +#[cfg(target_pointer_width = "64")] +macro_rules! bt { + ($inst:expr) => { + concat!($inst, " {b}, ({p})") + }; +} + /// Returns the bit in position `b` of the memory addressed by `p`. #[inline] #[cfg_attr(test, assert_instr(bt))] -#[unstable(feature = "simd_x86_bittest", issue = "59414")] +#[stable(feature = "simd_x86_bittest", since = "1.55.0")] pub unsafe fn _bittest64(p: *const i64, b: i64) -> u8 { let r: u8; - asm!("btq $2, $1\n\tsetc ${0:b}" - : "=r"(r) - : "*m"(p), "r"(b) - : "cc", "memory"); + asm!( + bt!("btq"), + "setc {r}", + p = in(reg) p, + b = in(reg) b, + r = out(reg_byte) r, + options(readonly, nostack, pure, att_syntax) + ); r } /// Returns the bit in position `b` of the memory addressed by `p`, then sets the bit to `1`. #[inline] #[cfg_attr(test, assert_instr(bts))] -#[unstable(feature = "simd_x86_bittest", issue = "59414")] +#[stable(feature = "simd_x86_bittest", since = "1.55.0")] pub unsafe fn _bittestandset64(p: *mut i64, b: i64) -> u8 { let r: u8; - asm!("btsq $2, $1\n\tsetc ${0:b}" - : "=r"(r), "+*m"(p) - : "r"(b) - : "cc", "memory"); + asm!( + bt!("btsq"), + "setc {r}", + p = in(reg) p, + b = in(reg) b, + r = out(reg_byte) r, + options(nostack, att_syntax) + ); r } /// Returns the bit in position `b` of the memory addressed by `p`, then resets that bit to `0`. #[inline] #[cfg_attr(test, assert_instr(btr))] -#[unstable(feature = "simd_x86_bittest", issue = "59414")] +#[stable(feature = "simd_x86_bittest", since = "1.55.0")] pub unsafe fn _bittestandreset64(p: *mut i64, b: i64) -> u8 { let r: u8; - asm!("btrq $2, $1\n\tsetc ${0:b}" - : "=r"(r), "+*m"(p) - : "r"(b) - : "cc", "memory"); + asm!( + bt!("btrq"), + "setc {r}", + p = in(reg) p, + b = in(reg) b, + r = out(reg_byte) r, + options(nostack, att_syntax) + ); r } /// Returns the bit in position `b` of the memory addressed by `p`, then inverts that bit. #[inline] #[cfg_attr(test, assert_instr(btc))] -#[unstable(feature = "simd_x86_bittest", issue = "59414")] +#[stable(feature = "simd_x86_bittest", since = "1.55.0")] pub unsafe fn _bittestandcomplement64(p: *mut i64, b: i64) -> u8 { let r: u8; - asm!("btcq $2, $1\n\tsetc ${0:b}" - : "=r"(r), "+*m"(p) - : "r"(b) - : "cc", "memory"); + asm!( + bt!("btcq"), + "setc {r}", + p = in(reg) p, + b = in(reg) b, + r = out(reg_byte) r, + options(nostack, att_syntax) + ); r } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/cmpxchg16b.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/cmpxchg16b.rs index 391daed20..a262932af 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/cmpxchg16b.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/cmpxchg16b.rs @@ -34,11 +34,11 @@ use stdarch_test::assert_instr; /// support `cmpxchg16b` and the program enters an execution path that /// eventually would reach this function the behavior is undefined. /// -/// The `success` ordering must also be stronger or equal to `failure`, or this -/// function call is undefined. See the `Atomic*` documentation's -/// `compare_exchange` function for more information. When `compare_exchange` -/// panics, this is undefined behavior. Currently this function aborts the -/// process with an undefined instruction. +/// The failure ordering must be [`Ordering::SeqCst`], [`Ordering::Acquire`] or +/// [`Ordering::Relaxed`], or this function call is undefined. See the `Atomic*` +/// documentation's `compare_exchange` function for more information. When +/// `compare_exchange` panics, this is undefined behavior. Currently this +/// function aborts the process with an undefined instruction. #[inline] #[cfg_attr(test, assert_instr(cmpxchg16b, success = Ordering::SeqCst, failure = Ordering::SeqCst))] #[target_feature(enable = "cmpxchg16b")] @@ -54,15 +54,21 @@ pub unsafe fn cmpxchg16b( debug_assert!(dst as usize % 16 == 0); let (val, _ok) = match (success, failure) { - (Acquire, Acquire) => intrinsics::atomic_cxchg_acq(dst, old, new), - (Release, Relaxed) => intrinsics::atomic_cxchg_rel(dst, old, new), - (AcqRel, Acquire) => intrinsics::atomic_cxchg_acqrel(dst, old, new), - (Relaxed, Relaxed) => intrinsics::atomic_cxchg_relaxed(dst, old, new), - (SeqCst, SeqCst) => intrinsics::atomic_cxchg(dst, old, new), - (Acquire, Relaxed) => intrinsics::atomic_cxchg_acq_failrelaxed(dst, old, new), - (AcqRel, Relaxed) => intrinsics::atomic_cxchg_acqrel_failrelaxed(dst, old, new), - (SeqCst, Relaxed) => intrinsics::atomic_cxchg_failrelaxed(dst, old, new), - (SeqCst, Acquire) => intrinsics::atomic_cxchg_failacq(dst, old, new), + (Relaxed, Relaxed) => intrinsics::atomic_cxchg_relaxed_relaxed(dst, old, new), + (Relaxed, Acquire) => intrinsics::atomic_cxchg_relaxed_acquire(dst, old, new), + (Relaxed, SeqCst) => intrinsics::atomic_cxchg_relaxed_seqcst(dst, old, new), + (Acquire, Relaxed) => intrinsics::atomic_cxchg_acquire_relaxed(dst, old, new), + (Acquire, Acquire) => intrinsics::atomic_cxchg_acquire_acquire(dst, old, new), + (Acquire, SeqCst) => intrinsics::atomic_cxchg_acquire_seqcst(dst, old, new), + (Release, Relaxed) => intrinsics::atomic_cxchg_release_relaxed(dst, old, new), + (Release, Acquire) => intrinsics::atomic_cxchg_release_acquire(dst, old, new), + (Release, SeqCst) => intrinsics::atomic_cxchg_release_seqcst(dst, old, new), + (AcqRel, Relaxed) => intrinsics::atomic_cxchg_acqrel_relaxed(dst, old, new), + (AcqRel, Acquire) => intrinsics::atomic_cxchg_acqrel_acquire(dst, old, new), + (AcqRel, SeqCst) => intrinsics::atomic_cxchg_acqrel_seqcst(dst, old, new), + (SeqCst, Relaxed) => intrinsics::atomic_cxchg_seqcst_relaxed(dst, old, new), + (SeqCst, Acquire) => intrinsics::atomic_cxchg_seqcst_acquire(dst, old, new), + (SeqCst, SeqCst) => intrinsics::atomic_cxchg_seqcst_seqcst(dst, old, new), // The above block is all copied from libcore, and this statement is // also copied from libcore except that it's a panic in libcore and we diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/fxsr.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/fxsr.rs index 0b26fb6d0..d02702046 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/fxsr.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/fxsr.rs @@ -1,4 +1,4 @@ -//! FXSR floating-point context fast save and restor. +//! FXSR floating-point context fast save and restore. #[cfg(test)] use stdarch_test::assert_instr; @@ -6,9 +6,9 @@ use stdarch_test::assert_instr; #[allow(improper_ctypes)] extern "C" { #[link_name = "llvm.x86.fxsave64"] - fn fxsave64(p: *mut u8) -> (); + fn fxsave64(p: *mut u8); #[link_name = "llvm.x86.fxrstor64"] - fn fxrstor64(p: *const u8) -> (); + fn fxrstor64(p: *const u8); } /// Saves the `x87` FPU, `MMX` technology, `XMM`, and `MXCSR` registers to the @@ -87,7 +87,7 @@ mod tests { } impl fmt::Debug for FxsaveArea { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "[")?; for i in 0..self.data.len() { write!(f, "{}", self.data[i])?; diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/macros.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/macros.rs new file mode 100644 index 000000000..a3ea0e821 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/macros.rs @@ -0,0 +1,36 @@ +//! Utility macros. + +// Helper struct used to trigger const eval errors when the const generic immediate value `imm` is +// not a round number. +pub(crate) struct ValidateConstRound; +impl ValidateConstRound { + pub(crate) const VALID: () = { + assert!( + IMM == 4 || IMM == 8 || IMM == 9 || IMM == 10 || IMM == 11, + "Invalid IMM value" + ); + }; +} + +#[allow(unused)] +macro_rules! static_assert_rounding { + ($imm:ident) => { + let _ = $crate::core_arch::x86_64::macros::ValidateConstRound::<$imm>::VALID; + }; +} + +// Helper struct used to trigger const eval errors when the const generic immediate value `imm` is +// not a sae number. +pub(crate) struct ValidateConstSae; +impl ValidateConstSae { + pub(crate) const VALID: () = { + assert!(IMM == 4 || IMM == 8, "Invalid IMM value"); + }; +} + +#[allow(unused)] +macro_rules! static_assert_sae { + ($imm:ident) => { + let _ = $crate::core_arch::x86_64::macros::ValidateConstSae::<$imm>::VALID; + }; +} diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/mod.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/mod.rs index 038f6478c..461874ece 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/mod.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/mod.rs @@ -1,5 +1,8 @@ //! `x86_64` intrinsics +#[macro_use] +mod macros; + mod fxsr; pub use self::fxsr::*; @@ -33,6 +36,9 @@ pub use self::bmi2::*; mod avx2; pub use self::avx2::*; +mod avx512f; +pub use self::avx512f::*; + mod bswap; pub use self::bswap::*; diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/sse.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/sse.rs index ec0928246..ca6799c90 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/sse.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/sse.rs @@ -19,7 +19,7 @@ extern "C" { /// /// The result is rounded according to the current rounding mode. If the result /// cannot be represented as a 64 bit integer the result will be -/// `0x8000_0000_0000_0000` (`std::i64::MIN`) or trigger an invalid operation +/// `0x8000_0000_0000_0000` (`i64::MIN`) or trigger an invalid operation /// floating point exception if unmasked (see /// [`_mm_setcsr`](fn._mm_setcsr.html)). /// @@ -39,7 +39,7 @@ pub unsafe fn _mm_cvtss_si64(a: __m128) -> i64 { /// /// The result is rounded always using truncation (round towards zero). If the /// result cannot be represented as a 64 bit integer the result will be -/// `0x8000_0000_0000_0000` (`std::i64::MIN`) or an invalid operation floating +/// `0x8000_0000_0000_0000` (`i64::MIN`) or an invalid operation floating /// point exception if unmasked (see [`_mm_setcsr`](fn._mm_setcsr.html)). /// /// This corresponds to the `CVTTSS2SI` instruction (with 64 bit output). @@ -71,7 +71,6 @@ pub unsafe fn _mm_cvtsi64_ss(a: __m128, b: i64) -> __m128 { #[cfg(test)] mod tests { use crate::core_arch::arch::x86_64::*; - use std::{f32::NAN, i64::MIN}; use stdarch_test::simd_test; #[simd_test(enable = "sse")] @@ -83,7 +82,7 @@ mod tests { (-34.5, -34), (4.0e10, 40_000_000_000), (4.0e-10, 0), - (NAN, MIN), + (f32::NAN, i64::MIN), (2147483500.1, 2147483520), (9.223371e18, 9223370937343148032), ]; @@ -110,10 +109,10 @@ mod tests { (-5.99, -5), (4.0e10, 40_000_000_000), (4.0e-10, 0), - (NAN, MIN), + (f32::NAN, i64::MIN), (2147483500.1, 2147483520), (9.223371e18, 9223370937343148032), - (9.223372e18, MIN), + (9.223372e18, i64::MIN), ]; for i in 0..inputs.len() { let (xi, e) = inputs[i]; diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/sse2.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/sse2.rs index 94a919d4f..f487a067f 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/sse2.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/sse2.rs @@ -148,7 +148,7 @@ pub unsafe fn _mm_cvtsi64x_sd(a: __m128d, b: i64) -> __m128d { #[cfg(test)] mod tests { use crate::core_arch::arch::x86_64::*; - use std::{boxed, f64, i64}; + use std::boxed; use stdarch_test::simd_test; #[simd_test(enable = "sse2")] diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/sse41.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/sse41.rs index 18c315c90..3d1ea0cf6 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/sse41.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/sse41.rs @@ -8,30 +8,31 @@ use crate::{ #[cfg(test)] use stdarch_test::assert_instr; -/// Extracts an 64-bit integer from `a` selected with `imm8` +/// Extracts an 64-bit integer from `a` selected with `IMM1` /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_extract_epi64) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(all(test, not(target_os = "windows")), assert_instr(pextrq, imm8 = 1))] -#[rustc_args_required_const(1)] +#[cfg_attr(all(test, not(target_os = "windows")), assert_instr(pextrq, IMM1 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_extract_epi64(a: __m128i, imm8: i32) -> i64 { - let imm8 = (imm8 & 1) as u32; - simd_extract(a.as_i64x2(), imm8) +pub unsafe fn _mm_extract_epi64(a: __m128i) -> i64 { + static_assert_imm1!(IMM1); + simd_extract(a.as_i64x2(), IMM1 as u32) } /// Returns a copy of `a` with the 64-bit integer from `i` inserted at a -/// location specified by `imm8`. +/// location specified by `IMM1`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_insert_epi64) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(pinsrq, imm8 = 0))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(pinsrq, IMM1 = 0))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_insert_epi64(a: __m128i, i: i64, imm8: i32) -> __m128i { - transmute(simd_insert(a.as_i64x2(), (imm8 & 1) as u32, i)) +pub unsafe fn _mm_insert_epi64(a: __m128i, i: i64) -> __m128i { + static_assert_imm1!(IMM1); + transmute(simd_insert(a.as_i64x2(), IMM1 as u32, i)) } #[cfg(test)] @@ -42,19 +43,20 @@ mod tests { #[simd_test(enable = "sse4.1")] unsafe fn test_mm_extract_epi64() { let a = _mm_setr_epi64x(0, 1); - let r = _mm_extract_epi64(a, 1); - assert_eq!(r, 1); - let r = _mm_extract_epi64(a, 3); + let r = _mm_extract_epi64::<1>(a); assert_eq!(r, 1); + let r = _mm_extract_epi64::<0>(a); + assert_eq!(r, 0); } #[simd_test(enable = "sse4.1")] unsafe fn test_mm_insert_epi64() { let a = _mm_set1_epi64x(0); let e = _mm_setr_epi64x(0, 32); - let r = _mm_insert_epi64(a, 32, 1); + let r = _mm_insert_epi64::<1>(a, 32); assert_eq_m128i(r, e); - let r = _mm_insert_epi64(a, 32, 3); + let e = _mm_setr_epi64x(32, 0); + let r = _mm_insert_epi64::<0>(a, 32); assert_eq_m128i(r, e); } } diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/sse42.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/sse42.rs index 405073261..6b5d087c1 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/sse42.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/sse42.rs @@ -10,7 +10,7 @@ extern "C" { } /// Starting with the initial value in `crc`, return the accumulated -/// CRC32 value for unsigned 64-bit integer `v`. +/// CRC32-C value for unsigned 64-bit integer `v`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_crc32_u64) #[inline] diff --git a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/xsave.rs b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/xsave.rs index 829669505..2afd3e433 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/xsave.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/src/x86_64/xsave.rs @@ -8,17 +8,17 @@ use stdarch_test::assert_instr; #[allow(improper_ctypes)] extern "C" { #[link_name = "llvm.x86.xsave64"] - fn xsave64(p: *mut u8, hi: u32, lo: u32) -> (); + fn xsave64(p: *mut u8, hi: u32, lo: u32); #[link_name = "llvm.x86.xrstor64"] - fn xrstor64(p: *const u8, hi: u32, lo: u32) -> (); + fn xrstor64(p: *const u8, hi: u32, lo: u32); #[link_name = "llvm.x86.xsaveopt64"] - fn xsaveopt64(p: *mut u8, hi: u32, lo: u32) -> (); + fn xsaveopt64(p: *mut u8, hi: u32, lo: u32); #[link_name = "llvm.x86.xsavec64"] - fn xsavec64(p: *mut u8, hi: u32, lo: u32) -> (); + fn xsavec64(p: *mut u8, hi: u32, lo: u32); #[link_name = "llvm.x86.xsaves64"] - fn xsaves64(p: *mut u8, hi: u32, lo: u32) -> (); + fn xsaves64(p: *mut u8, hi: u32, lo: u32); #[link_name = "llvm.x86.xrstors64"] - fn xrstors64(p: *const u8, hi: u32, lo: u32) -> (); + fn xrstors64(p: *const u8, hi: u32, lo: u32); } /// Performs a full or partial save of the enabled processor states to memory at @@ -164,7 +164,7 @@ mod tests { } impl fmt::Debug for XsaveArea { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "[")?; for i in 0..self.data.len() { write!(f, "{}", self.data[i])?; diff --git a/crux-mir/lib/stdarch/crates/core_arch/tests/cpu-detection.rs b/crux-mir/lib/stdarch/crates/core_arch/tests/cpu-detection.rs index e17050d8c..61f5f0905 100644 --- a/crux-mir/lib/stdarch/crates/core_arch/tests/cpu-detection.rs +++ b/crux-mir/lib/stdarch/crates/core_arch/tests/cpu-detection.rs @@ -1,5 +1,5 @@ #![feature(stdsimd)] -#![allow(clippy::option_unwrap_used, clippy::print_stdout, clippy::use_debug)] +#![allow(clippy::unwrap_used, clippy::print_stdout, clippy::use_debug)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] #[macro_use] diff --git a/crux-mir/lib/stdarch/crates/intrinsic-test/Cargo.toml b/crux-mir/lib/stdarch/crates/intrinsic-test/Cargo.toml new file mode 100644 index 000000000..5fde23c9e --- /dev/null +++ b/crux-mir/lib/stdarch/crates/intrinsic-test/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "intrinsic-test" +version = "0.1.0" +authors = ["Jamie Cunliffe "] +edition = "2018" + +[dependencies] +lazy_static = "1.4.0" +serde = { version = "1", features = ["derive"] } +csv = "1.1" +clap = "2.33.3" +regex = "1.4.2" +log = "0.4.11" +pretty_env_logger = "0.4.0" +rayon = "1.5.0" +diff = "0.1.12" +itertools = "0.10.1" \ No newline at end of file diff --git a/crux-mir/lib/stdarch/crates/intrinsic-test/README.md b/crux-mir/lib/stdarch/crates/intrinsic-test/README.md new file mode 100644 index 000000000..8a8ddab40 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/intrinsic-test/README.md @@ -0,0 +1,24 @@ +Generate and run programs using equivalent C and Rust intrinsics, checking that +each produces the same result from random inputs. + +# Usage +``` +USAGE: + intrinsic-test [OPTIONS] + +FLAGS: + -h, --help Prints help information + -V, --version Prints version information + +OPTIONS: + --cppcompiler The C++ compiler to use for compiling the c++ code [default: clang++] + --runner Run the C programs under emulation with this command + --toolchain The rust toolchain to use for building the rust code + +ARGS: + The input file containing the intrinsics +``` + +The intrinsic.csv is the arm neon tracking google sheet (https://docs.google.com/spreadsheets/d/1MqW1g8c7tlhdRWQixgdWvR4uJHNZzCYAf4V0oHjZkwA/edit#gid=0) +that contains the intrinsic list. The done percentage column should be renamed to "enabled". + diff --git a/crux-mir/lib/stdarch/crates/intrinsic-test/missing_aarch64.txt b/crux-mir/lib/stdarch/crates/intrinsic-test/missing_aarch64.txt new file mode 100644 index 000000000..93fc126e5 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/intrinsic-test/missing_aarch64.txt @@ -0,0 +1,96 @@ +# Not implemented in stdarch yet +vbfdot_f32 +vbfdot_lane_f32 +vbfdot_laneq_f32 +vbfdotq_f32 +vbfdotq_lane_f32 +vbfdotq_laneq_f32 +vbfmlalbq_f32 +vbfmlalbq_lane_f32 +vbfmlalbq_laneq_f32 +vbfmlaltq_f32 +vbfmlaltq_lane_f32 +vbfmlaltq_laneq_f32 +vbfmmlaq_f32 +vsudot_laneq_s32 +vsudot_lane_s32 +vsudotq_laneq_s32 +vsudotq_lane_s32 +vusdot_laneq_s32 +vusdot_lane_s32 +vusdotq_laneq_s32 +vusdotq_lane_s32 +vusdotq_s32 +vusdot_s32 + +# Implemented in Clang but missing from CSV +vcmla_f64 +vcmla_lane_f64 +vcmla_laneq_f64 +vcmlaq_lane_f64 +vcmlaq_laneq_f64 +vcmlaq_rot180_lane_f64 +vcmlaq_rot180_laneq_f64 +vcmlaq_rot270_lane_f64 +vcmlaq_rot270_laneq_f64 +vcmlaq_rot90_lane_f64 +vcmlaq_rot90_laneq_f64 +vcmla_rot180_f64 +vcmla_rot180_lane_f64 +vcmla_rot180_laneq_f64 +vcmla_rot270_f64 +vcmla_rot270_lane_f64 +vcmla_rot270_laneq_f64 +vcmla_rot90_f64 +vcmla_rot90_lane_f64 +vcmla_rot90_laneq_f64 + +# Implemented in Clang and stdarch but missing from CSV +vmov_n_p64 +vmovq_n_p64 +vreinterpret_f32_p64 +vreinterpret_p64_s64 +vreinterpretq_f32_p128 +vreinterpretq_f32_p64 +vreinterpretq_p128_p64 +vreinterpretq_p64_p128 +vtst_p16 +vtstq_p16 + +# Missing from both Clang and stdarch +vrnd32x_f64 +vrnd32xq_f64 +vrnd32z_f64 +vrnd32zq_f64 +vrnd64x_f64 +vrnd64xq_f64 +vrnd64z_f64 +vrnd64zq_f64 + +# QEMU 6.0 doesn't support these instructions +vmmlaq_s32 +vmmlaq_u32 +vsm3partw1q_u32 +vsm3partw2q_u32 +vsm3ss1q_u32 +vsm3tt1aq_u32 +vsm3tt1bq_u32 +vsm3tt2aq_u32 +vsm3tt2bq_u32 +vsm4ekeyq_u32 +vsm4eq_u32 +vusmmlaq_s32 + +# LLVM select error in debug builds +vqshlu_n_s16 +vqshlu_n_s32 +vqshlu_n_s64 +vqshlu_n_s8 +vqshlub_n_s8 +vqshlud_n_s64 +vqshluh_n_s16 +vqshluq_n_s16 +vqshluq_n_s32 +vqshluq_n_s64 +vqshluq_n_s8 +vqshlus_n_s32 diff --git a/crux-mir/lib/stdarch/crates/intrinsic-test/missing_arm.txt b/crux-mir/lib/stdarch/crates/intrinsic-test/missing_arm.txt new file mode 100644 index 000000000..bbc8de584 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/intrinsic-test/missing_arm.txt @@ -0,0 +1,334 @@ +# Not implemented in stdarch yet +vbfdot_f32 +vbfdot_lane_f32 +vbfdot_laneq_f32 +vbfdotq_f32 +vbfdotq_lane_f32 +vbfdotq_laneq_f32 +vbfmlalbq_f32 +vbfmlalbq_lane_f32 +vbfmlalbq_laneq_f32 +vbfmlaltq_f32 +vbfmlaltq_lane_f32 +vbfmlaltq_laneq_f32 +vbfmmlaq_f32 +vsudot_laneq_s32 +vsudot_lane_s32 +vsudotq_laneq_s32 +vsudotq_lane_s32 +vusdot_laneq_s32 +vusdot_lane_s32 +vusdotq_laneq_s32 +vusdotq_lane_s32 +vusdotq_s32 +vusdot_s32 + +# Implemented in Clang and stdarch but missing from CSV +vtst_p16 +vtstq_p16 + +# QEMU 6.0 doesn't support these instructions +vmmlaq_s32 +vmmlaq_u32 +vusmmlaq_s32 + +# Implemented in Clang and stdarch for A64 only even though CSV claims A32 support +__crc32d +__crc32cd +vaddq_p64 +vbsl_p64 +vbslq_p64 +vceq_p64 +vceqq_p64 +vceqz_p64 +vceqzq_p64 +vcombine_p64 +vcopy_lane_p64 +vcopy_laneq_p64 +vcopyq_lane_p64 +vcopyq_laneq_p64 +vcreate_p64 +vdup_lane_p64 +vdup_n_p64 +vdupq_lane_p64 +vdupq_n_p64 +vext_p64 +vextq_p64 +vget_high_p64 +vget_lane_p64 +vget_low_p64 +vgetq_lane_p64 +vmovn_high_s16 +vmovn_high_s32 +vmovn_high_s64 +vmovn_high_u16 +vmovn_high_u32 +vmovn_high_u64 +vmull_high_p64 +vmull_p64 +vreinterpret_p16_p64 +vreinterpret_p64_f32 +vreinterpret_p64_p16 +vreinterpret_p64_p8 +vreinterpret_p64_s16 +vreinterpret_p64_s32 +vreinterpret_p64_s8 +vreinterpret_p64_u16 +vreinterpret_p64_u32 +vreinterpret_p64_u64 +vreinterpret_p64_u8 +vreinterpret_p8_p64 +vreinterpretq_f64_u64 +vreinterpretq_p128_f32 +vreinterpretq_p128_p16 +vreinterpretq_p128_p8 +vreinterpretq_p128_s16 +vreinterpretq_p128_s32 +vreinterpretq_p128_s64 +vreinterpretq_p128_s8 +vreinterpretq_p128_u16 +vreinterpretq_p128_u32 +vreinterpretq_p128_u64 +vreinterpretq_p128_u8 +vreinterpretq_p16_p64 +vreinterpretq_p64_f32 +vreinterpretq_p64_p16 +vreinterpretq_p64_p8 +vreinterpretq_p64_s16 +vreinterpretq_p64_s32 +vreinterpretq_p64_s64 +vreinterpretq_p64_s8 +vreinterpretq_p64_u16 +vreinterpretq_p64_u32 +vreinterpretq_p64_u64 +vreinterpretq_p64_u8 +vreinterpretq_p8_p64 +vreinterpretq_s16_p64 +vreinterpretq_s32_p64 +vreinterpretq_s64_p64 +vreinterpretq_s8_p64 +vreinterpretq_u16_p64 +vreinterpretq_u32_p64 +vreinterpretq_u64_p64 +vreinterpretq_u8_p64 +vreinterpret_s16_p64 +vreinterpret_s32_p64 +vreinterpret_s64_p64 +vreinterpret_s8_p64 +vreinterpret_u16_p64 +vreinterpret_u32_p64 +vreinterpret_u64_p64 +vreinterpret_u8_p64 +vrndn_f64 +vrndnq_f64 +vset_lane_p64 +vsetq_lane_p64 +vsli_n_p64 +vsliq_n_p64 +vsri_n_p64 +vsriq_n_p64 +vtst_p64 +vtstq_p64 + +# Present in Clang header but triggers an ICE due to lack of backend support. +vcmla_f32 +vcmla_lane_f32 +vcmla_laneq_f32 +vcmla_rot180_f32 +vcmla_rot180_lane_f32 +vcmla_rot180_laneq_f32 +vcmla_rot270_f32 +vcmla_rot270_lane_f32 +vcmla_rot270_laneq_f32 +vcmla_rot90_f32 +vcmla_rot90_lane_f32 +vcmla_rot90_laneq_f32 +vcmlaq_f32 +vcmlaq_lane_f32 +vcmlaq_laneq_f32 +vcmlaq_rot180_f32 +vcmlaq_rot180_lane_f32 +vcmlaq_rot180_laneq_f32 +vcmlaq_rot270_f32 +vcmlaq_rot270_lane_f32 +vcmlaq_rot270_laneq_f32 +vcmlaq_rot90_f32 +vcmlaq_rot90_lane_f32 +vcmlaq_rot90_laneq_f32 + +# Implemented in stdarch for A64 only, Clang support both A32/A64 +vadd_s64 +vadd_u64 +vcaddq_rot270_f32 +vcaddq_rot90_f32 +vcadd_rot270_f32 +vcadd_rot90_f32 +vcombine_f32 +vcombine_p16 +vcombine_p8 +vcombine_s16 +vcombine_s32 +vcombine_s64 +vcombine_s8 +vcombine_u16 +vcombine_u32 +vcombine_u64 +vcombine_u8 +vcvtaq_s32_f32 +vcvtaq_u32_f32 +vcvta_s32_f32 +vcvta_u32_f32 +vcvtmq_s32_f32 +vcvtmq_u32_f32 +vcvtm_s32_f32 +vcvtm_u32_f32 +vcvtnq_s32_f32 +vcvtnq_u32_f32 +vcvtn_s32_f32 +vcvtn_u32_f32 +vcvtpq_s32_f32 +vcvtpq_u32_f32 +vcvtp_s32_f32 +vcvtp_u32_f32 +vdot_lane_s32 +vdot_lane_u32 +vdotq_lane_s32 +vdotq_lane_u32 +vdotq_s32 +vdotq_u32 +vdot_s32 +vdot_u32 +vqdmulh_lane_s16 +vqdmulh_lane_s32 +vqdmulhq_lane_s16 +vqdmulhq_lane_s32 +vrnda_f32 +vrnda_f32 +vrndaq_f32 +vrndaq_f32 +vrnd_f32 +vrnd_f32 +vrndi_f32 +vrndi_f32 +vrndiq_f32 +vrndiq_f32 +vrndm_f32 +vrndm_f32 +vrndmq_f32 +vrndmq_f32 +vrndns_f32 +vrndp_f32 +vrndpq_f32 +vrndq_f32 +vrndq_f32 +vrndx_f32 +vrndxq_f32 + +# LLVM select error in debug builds +vqrshrn_n_s16 +vqrshrn_n_s32 +vqrshrn_n_s64 +vqrshrn_n_u16 +vqrshrn_n_u32 +vqrshrn_n_u64 +vqrshrun_n_s16 +vqrshrun_n_s32 +vqrshrun_n_s64 +vqshrn_n_s16 +vqshrn_n_s32 +vqshrn_n_s64 +vqshrn_n_u16 +vqshrn_n_u32 +vqshrn_n_u64 +vqshrun_n_s16 +vqshrun_n_s32 +vqshrun_n_s64 +vrshrn_n_s16 +vrshrn_n_s32 +vrshrn_n_s64 +vrshrn_n_u16 +vrshrn_n_u32 +vrshrn_n_u64 +vshrq_n_u64 +vshr_n_u64 + +# Failing tests: stdarch has incorrect results compared to Clang +vqshlu_n_s16 +vqshlu_n_s32 +vqshlu_n_s64 +vqshlu_n_s8 +vqshluq_n_s16 +vqshluq_n_s32 +vqshluq_n_s64 +vqshluq_n_s8 +vsli_n_p16 +vsli_n_p8 +vsli_n_s16 +vsli_n_s32 +vsli_n_s64 +vsli_n_s8 +vsli_n_u16 +vsli_n_u32 +vsli_n_u64 +vsli_n_u8 +vsliq_n_p16 +vsliq_n_p8 +vsliq_n_s16 +vsliq_n_s32 +vsliq_n_s64 +vsliq_n_s8 +vsliq_n_u16 +vsliq_n_u32 +vsliq_n_u64 +vsliq_n_u8 +vsri_n_p16 +vsri_n_p8 +vsri_n_s16 +vsri_n_s32 +vsri_n_s64 +vsri_n_s8 +vsri_n_u16 +vsri_n_u32 +vsri_n_u64 +vsri_n_u8 +vsriq_n_p16 +vsriq_n_p8 +vsriq_n_s16 +vsriq_n_s32 +vsriq_n_s64 +vsriq_n_s8 +vsriq_n_u16 +vsriq_n_u32 +vsriq_n_u64 +vsriq_n_u8 + +# These produce a different result on Clang depending on the optimization level. +# This is definitely a bug in LLVM. +vadd_f32 +vaddq_f32 +vcvt_s32_f32 +vcvt_u32_f32 +vcvtq_s32_f32 +vcvtq_u32_f32 +vfma_f32 +vfma_n_f32 +vfmaq_f32 +vfmaq_n_f32 +vfms_f32 +vfmsq_f32 +vmla_f32 +vmla_lane_f32 +vmla_n_f32 +vmlaq_f32 +vmlaq_lane_f32 +vmlaq_n_f32 +vmls_f32 +vmls_lane_f32 +vmls_n_f32 +vmlsq_f32 +vmlsq_lane_f32 +vmlsq_n_f32 +vmul_lane_f32 +vmul_n_f32 +vmulq_lane_f32 +vmulq_n_f32 diff --git a/crux-mir/lib/stdarch/crates/intrinsic-test/src/acle_csv_parser.rs b/crux-mir/lib/stdarch/crates/intrinsic-test/src/acle_csv_parser.rs new file mode 100644 index 000000000..d7b066485 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/intrinsic-test/src/acle_csv_parser.rs @@ -0,0 +1,319 @@ +use itertools::Itertools; +use regex::Regex; +use serde::Deserialize; + +use crate::argument::{Argument, ArgumentList, Constraint}; +use crate::intrinsic::Intrinsic; +use crate::types::{IntrinsicType, TypeKind}; + +pub fn get_acle_intrinsics(filename: &str) -> Vec { + let data = std::fs::read_to_string(filename).expect("Failed to open ACLE intrinsics file"); + + let data = data + .lines() + .filter_map(|l| { + (!(l.starts_with("") || l.is_empty() || l.starts_with("
"))) + .then(|| l.replace("
\t", "")) + }) + .join("\n"); + + let mut csv_reader = csv::ReaderBuilder::new() + .delimiter(b'\t') + .from_reader(data.as_bytes()); + + let mut intrinsics: Vec = csv_reader + .deserialize() + .filter_map(|x: Result| x.ok().map(|i| i.into())) + .collect::>(); + + // Intrinsics such as vshll_n_s8 exist twice in the ACLE with different constraints. + intrinsics.sort_by(|a, b| a.name.cmp(&b.name)); + let (intrinsics, duplicates) = intrinsics.partition_dedup_by(|a, b| a.name == b.name); + for duplicate in duplicates { + let name = &duplicate.name; + let constraints = duplicate + .arguments + .args + .drain(..) + .filter(|a| a.has_constraint()); + let intrinsic = intrinsics.iter_mut().find(|i| &i.name == name).unwrap(); + + for mut constraint in constraints { + let real_constraint = intrinsic + .arguments + .args + .iter_mut() + .find(|a| a.name == constraint.name) + .unwrap(); + real_constraint + .constraints + .push(constraint.constraints.pop().unwrap()); + } + } + + intrinsics.to_vec() +} + +impl Into for ACLEIntrinsicLine { + fn into(self) -> Intrinsic { + let signature = self.intrinsic; + let (ret_ty, remaining) = signature.split_once(' ').unwrap(); + + let results = type_from_c(ret_ty) + .unwrap_or_else(|_| panic!("Failed to parse return type: {}", ret_ty)); + + let (name, args) = remaining.split_once('(').unwrap(); + let args = args.trim_end_matches(')'); + + // Typo in ACLE data + let args = args.replace("int16x8q_t", "int16x8_t"); + + let arg_prep = self.argument_preparation.as_str(); + let args = args + .split(',') + .enumerate() + .map(move |(idx, arg)| { + let arg = arg.trim(); + if arg.starts_with("__builtin_constant_p") { + handle_constraint(idx, arg, arg_prep) + } else { + from_c(idx, arg) + } + }) + .collect(); + let arguments = ArgumentList { args }; + let a64_only = match &*self.supported_architectures { + "A64" => true, + "v7/A32/A64" | "A32/A64" => false, + _ => panic!("Invalid supported architectures"), + }; + + Intrinsic { + name: name.to_string(), + arguments, + results, + a64_only, + } + } +} + +fn handle_constraint(idx: usize, arg: &str, prep: &str) -> Argument { + let prep = prep.replace(' ', ""); + + let name = arg + .trim_start_matches("__builtin_constant_p") + .trim_start_matches(|ref c| c == &' ' || c == &'(') + .trim_end_matches(')') + .to_string(); + + let ty = IntrinsicType::Type { + constant: true, + kind: TypeKind::Int, + bit_len: Some(32), + simd_len: None, + vec_len: None, + }; + + let constraints = prep + .split(';') + .find_map(|p| handle_range_constraint(&name, p).or_else(|| handle_eq_constraint(&name, p))) + .map(|c| vec![c]) + .unwrap_or_default(); + + Argument { + pos: idx, + name, + ty, + constraints, + } +} + +fn handle_range_constraint(name: &str, data: &str) -> Option { + lazy_static! { + static ref RANGE_CONSTRAINT: Regex = + Regex::new(r#"([0-9]+)<=([[:alnum:]]+)<=([0-9]+)"#).unwrap(); + } + + let captures = RANGE_CONSTRAINT.captures(data)?; + if captures.get(2).map(|c| c.as_str() == name).unwrap_or(false) { + match (captures.get(1), captures.get(3)) { + (Some(start), Some(end)) => { + let start = start.as_str().parse::().unwrap(); + let end = end.as_str().parse::().unwrap() + 1; + Some(Constraint::Range(start..end)) + } + _ => panic!("Invalid constraint"), + } + } else { + None + } +} + +fn handle_eq_constraint(name: &str, data: &str) -> Option { + lazy_static! { + static ref EQ_CONSTRAINT: Regex = Regex::new(r#"([[:alnum:]]+)==([0-9]+)"#).unwrap(); + } + let captures = EQ_CONSTRAINT.captures(data)?; + if captures.get(1).map(|c| c.as_str() == name).unwrap_or(false) { + captures + .get(2) + .map(|c| Constraint::Equal(c.as_str().parse::().unwrap())) + } else { + None + } +} + +fn from_c(pos: usize, s: &str) -> Argument { + let name_index = s + .chars() + .rev() + .take_while(|c| c != &'*' && c != &' ') + .count(); + + let name_start = s.len() - name_index; + let name = s[name_start..].to_string(); + let s = s[..name_start].trim(); + + Argument { + pos, + name, + ty: type_from_c(s).unwrap_or_else(|_| panic!("Failed to parse type: {}", s)), + constraints: vec![], + } +} + +fn type_from_c(s: &str) -> Result { + const CONST_STR: &str = "const "; + + if let Some(s) = s.strip_suffix('*') { + let (s, constant) = if s.ends_with(CONST_STR) { + (&s[..s.len() - (CONST_STR.len() + 1)], true) + } else { + (s, false) + }; + + let s = s.trim_end(); + + Ok(IntrinsicType::Ptr { + constant, + child: Box::new(type_from_c(s)?), + }) + } else { + // [const ]TYPE[{bitlen}[x{simdlen}[x{vec_len}]]][_t] + + let (mut s, constant) = if let Some(s) = s.strip_prefix(CONST_STR) { + (s, true) + } else { + (s, false) + }; + s = s.strip_suffix("_t").unwrap_or(s); + + let mut parts = s.split('x'); // [[{bitlen}], [{simdlen}], [{vec_len}] ] + + let start = parts.next().ok_or("Impossible to parse type")?; + + if let Some(digit_start) = start.find(|c: char| c.is_ascii_digit()) { + let (arg_kind, bit_len) = start.split_at(digit_start); + + let arg_kind = arg_kind.parse::()?; + let bit_len = bit_len.parse::().map_err(|err| err.to_string())?; + + let simd_len = parts.next().map(|part| part.parse::().ok()).flatten(); + let vec_len = parts.next().map(|part| part.parse::().ok()).flatten(); + + Ok(IntrinsicType::Type { + constant, + kind: arg_kind, + bit_len: Some(bit_len), + simd_len, + vec_len, + }) + } else { + Ok(IntrinsicType::Type { + constant, + kind: start.parse::()?, + bit_len: None, + simd_len: None, + vec_len: None, + }) + } + } +} + +#[derive(Deserialize, Debug, PartialEq, Clone)] +struct ACLEIntrinsicLine { + #[serde(rename = "Intrinsic")] + intrinsic: String, + #[serde(rename = "Argument preparation")] + argument_preparation: String, + #[serde(rename = "AArch64 Instruction")] + aarch64_instruction: String, + #[serde(rename = "Result")] + result: String, + #[serde(rename = "Supported architectures")] + supported_architectures: String, +} + +#[cfg(test)] +mod test { + use super::*; + use crate::argument::Argument; + use crate::types::{IntrinsicType, TypeKind}; + + #[test] + fn parse_simd() { + let expected = Argument { + pos: 0, + name: "a".into(), + ty: IntrinsicType::Type { + constant: false, + kind: TypeKind::Int, + bit_len: Some(32), + simd_len: Some(4), + vec_len: None, + }, + constraints: vec![], + }; + let actual = from_c(0, "int32x4_t a"); + assert_eq!(expected, actual); + } + + #[test] + fn parse_simd_with_vec() { + let expected = Argument { + pos: 0, + name: "a".into(), + ty: IntrinsicType::Type { + constant: false, + kind: TypeKind::Int, + bit_len: Some(32), + simd_len: Some(4), + vec_len: Some(2), + }, + constraints: vec![], + }; + let actual = from_c(0, "int32x4x2_t a"); + assert_eq!(expected, actual); + } + + #[test] + fn test_ptr() { + let expected = Argument { + pos: 0, + name: "ptr".into(), + ty: crate::types::IntrinsicType::Ptr { + constant: true, + child: Box::new(IntrinsicType::Type { + constant: false, + kind: TypeKind::Int, + bit_len: Some(8), + simd_len: None, + vec_len: None, + }), + }, + constraints: vec![], + }; + let actual = from_c(0, "int8_t const *ptr"); + assert_eq!(expected, actual); + } +} diff --git a/crux-mir/lib/stdarch/crates/intrinsic-test/src/argument.rs b/crux-mir/lib/stdarch/crates/intrinsic-test/src/argument.rs new file mode 100644 index 000000000..798854c03 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/intrinsic-test/src/argument.rs @@ -0,0 +1,198 @@ +use std::ops::Range; + +use crate::types::{IntrinsicType, TypeKind}; +use crate::Language; + +/// An argument for the intrinsic. +#[derive(Debug, PartialEq, Clone)] +pub struct Argument { + /// The argument's index in the intrinsic function call. + pub pos: usize, + /// The argument name. + pub name: String, + /// The type of the argument. + pub ty: IntrinsicType, + /// Any constraints that are on this argument + pub constraints: Vec, +} + +#[derive(Debug, PartialEq, Clone)] +pub enum Constraint { + Equal(i64), + Range(Range), +} + +impl Constraint { + pub fn to_range(&self) -> Range { + match self { + Constraint::Equal(eq) => *eq..*eq + 1, + Constraint::Range(range) => range.clone(), + } + } +} + +impl Argument { + fn to_c_type(&self) -> String { + self.ty.c_type() + } + + fn is_simd(&self) -> bool { + self.ty.is_simd() + } + + pub fn is_ptr(&self) -> bool { + self.ty.is_ptr() + } + + pub fn has_constraint(&self) -> bool { + !self.constraints.is_empty() + } +} + +#[derive(Debug, PartialEq, Clone)] +pub struct ArgumentList { + pub args: Vec, +} + +impl ArgumentList { + /// Converts the argument list into the call parameters for a C function call. + /// e.g. this would generate something like `a, &b, c` + pub fn as_call_param_c(&self) -> String { + self.args + .iter() + .map(|arg| match arg.ty { + IntrinsicType::Ptr { .. } => { + format!("&{}", arg.name) + } + IntrinsicType::Type { .. } => arg.name.clone(), + }) + .collect::>() + .join(", ") + } + + /// Converts the argument list into the call parameters for a Rust function. + /// e.g. this would generate something like `a, b, c` + pub fn as_call_param_rust(&self) -> String { + self.args + .iter() + .filter(|a| !a.has_constraint()) + .map(|arg| arg.name.clone()) + .collect::>() + .join(", ") + } + + pub fn as_constraint_parameters_rust(&self) -> String { + self.args + .iter() + .filter(|a| a.has_constraint()) + .map(|arg| arg.name.clone()) + .collect::>() + .join(", ") + } + + /// Creates a line for each argument that initializes an array for C from which `loads` argument + /// values can be loaded as a sliding window. + /// e.g `const int32x2_t a_vals = {0x3effffff, 0x3effffff, 0x3f7fffff}`, if loads=2. + pub fn gen_arglists_c(&self, loads: u32) -> String { + self.iter() + .filter_map(|arg| { + (!arg.has_constraint()).then(|| { + format!( + "const {ty} {name}_vals[] = {{ {values} }};", + ty = arg.ty.c_scalar_type(), + name = arg.name, + values = arg.ty.populate_random(loads, &Language::C) + ) + }) + }) + .collect::>() + .join("\n") + } + + /// Creates a line for each argument that initializes an array for Rust from which `loads` argument + /// values can be loaded as a sliding window, e.g `const A_VALS: [u32; 20] = [...];` + pub fn gen_arglists_rust(&self, loads: u32) -> String { + self.iter() + .filter_map(|arg| { + (!arg.has_constraint()).then(|| { + format!( + "const {upper_name}_VALS: [{ty}; {load_size}] = unsafe{{ [{values}] }};", + upper_name = arg.name.to_uppercase(), + ty = arg.ty.rust_scalar_type(), + load_size = arg.ty.num_lanes() * arg.ty.num_vectors() + loads - 1, + values = arg.ty.populate_random(loads, &Language::Rust) + ) + }) + }) + .collect::>() + .join("\n") + } + + /// Creates a line for each argument that initalizes the argument from an array [arg]_vals at + /// an offset i using a load intrinsic, in C. + /// e.g `uint8x8_t a = vld1_u8(&a_vals[i]);` + pub fn load_values_c(&self, p64_armv7_workaround: bool) -> String { + self.iter() + .filter_map(|arg| { + // The ACLE doesn't support 64-bit polynomial loads on Armv7 + // This and the cast are a workaround for this + let armv7_p64 = if let TypeKind::Poly = arg.ty.kind() { + p64_armv7_workaround + } else { + false + }; + + (!arg.has_constraint()).then(|| { + format!( + "{ty} {name} = {open_cast}{load}(&{name}_vals[i]){close_cast};", + ty = arg.to_c_type(), + name = arg.name, + load = if arg.is_simd() { + arg.ty.get_load_function(p64_armv7_workaround) + } else { + "*".to_string() + }, + open_cast = if armv7_p64 { + format!("cast<{}>(", arg.to_c_type()) + } else { + "".to_string() + }, + close_cast = if armv7_p64 { + ")".to_string() + } else { + "".to_string() + } + ) + }) + }) + .collect::>() + .join("\n ") + } + + /// Creates a line for each argument that initalizes the argument from array [ARG]_VALS at + /// an offset i using a load intrinsic, in Rust. + /// e.g `let a = vld1_u8(A_VALS.as_ptr().offset(i));` + pub fn load_values_rust(&self) -> String { + self.iter() + .filter_map(|arg| { + (!arg.has_constraint()).then(|| { + format!( + "let {name} = {load}({upper_name}_VALS.as_ptr().offset(i));", + name = arg.name, + upper_name = arg.name.to_uppercase(), + load = if arg.is_simd() { + arg.ty.get_load_function(false) + } else { + "*".to_string() + }, + ) + }) + }) + .collect::>() + .join("\n ") + } + + pub fn iter(&self) -> std::slice::Iter<'_, Argument> { + self.args.iter() + } +} diff --git a/crux-mir/lib/stdarch/crates/intrinsic-test/src/intrinsic.rs b/crux-mir/lib/stdarch/crates/intrinsic-test/src/intrinsic.rs new file mode 100644 index 000000000..e0645a36b --- /dev/null +++ b/crux-mir/lib/stdarch/crates/intrinsic-test/src/intrinsic.rs @@ -0,0 +1,134 @@ +use crate::types::{IntrinsicType, TypeKind}; + +use super::argument::ArgumentList; + +/// An intrinsic +#[derive(Debug, PartialEq, Clone)] +pub struct Intrinsic { + /// The function name of this intrinsic. + pub name: String, + + /// Any arguments for this intrinsic. + pub arguments: ArgumentList, + + /// The return type of this intrinsic. + pub results: IntrinsicType, + + /// Whether this intrinsic is only available on A64. + pub a64_only: bool, +} + +impl Intrinsic { + /// Generates a std::cout for the intrinsics results that will match the + /// rust debug output format for the return type. The generated line assumes + /// there is an int i in scope which is the current pass number. + pub fn print_result_c(&self, additional: &str) -> String { + let lanes = if self.results.num_vectors() > 1 { + (0..self.results.num_vectors()) + .map(|vector| { + format!( + r#""{ty}(" << {lanes} << ")""#, + ty = self.results.c_single_vector_type(), + lanes = (0..self.results.num_lanes()) + .map(move |idx| -> std::string::String { + format!( + "{cast}{lane_fn}(__return_value.val[{vector}], {lane})", + cast = self.results.c_promotion(), + lane_fn = self.results.get_lane_function(), + lane = idx, + vector = vector, + ) + }) + .collect::>() + .join(r#" << ", " << "#) + ) + }) + .collect::>() + .join(r#" << ", " << "#) + } else if self.results.num_lanes() > 1 { + (0..self.results.num_lanes()) + .map(|idx| -> std::string::String { + format!( + "{cast}{lane_fn}(__return_value, {lane})", + cast = self.results.c_promotion(), + lane_fn = self.results.get_lane_function(), + lane = idx + ) + }) + .collect::>() + .join(r#" << ", " << "#) + } else { + format!( + "{promote}cast<{cast}>(__return_value)", + cast = match self.results.kind() { + TypeKind::Float if self.results.inner_size() == 32 => "float".to_string(), + TypeKind::Float if self.results.inner_size() == 64 => "double".to_string(), + TypeKind::Int => format!("int{}_t", self.results.inner_size()), + TypeKind::UInt => format!("uint{}_t", self.results.inner_size()), + TypeKind::Poly => format!("poly{}_t", self.results.inner_size()), + ty => todo!("print_result_c - Unknown type: {:#?}", ty), + }, + promote = self.results.c_promotion(), + ) + }; + + format!( + r#"std::cout << "Result {additional}-" << i+1 << ": {ty}" << std::fixed << std::setprecision(150) << {lanes} << "{close}" << std::endl;"#, + ty = if self.results.is_simd() { + format!("{}(", self.results.c_type()) + } else { + String::from("") + }, + close = if self.results.is_simd() { ")" } else { "" }, + lanes = lanes, + additional = additional, + ) + } + + pub fn generate_loop_c( + &self, + additional: &str, + passes: u32, + p64_armv7_workaround: bool, + ) -> String { + format!( + r#" {{ + for (int i=0; i<{passes}; i++) {{ + {loaded_args} + auto __return_value = {intrinsic_call}({args}); + {print_result} + }} + }}"#, + loaded_args = self.arguments.load_values_c(p64_armv7_workaround), + intrinsic_call = self.name, + args = self.arguments.as_call_param_c(), + print_result = self.print_result_c(additional) + ) + } + + pub fn generate_loop_rust(&self, additional: &str, passes: u32) -> String { + let constraints = self.arguments.as_constraint_parameters_rust(); + let constraints = if !constraints.is_empty() { + format!("::<{}>", constraints) + } else { + constraints + }; + + format!( + r#" {{ + for i in 0..{passes} {{ + unsafe {{ + {loaded_args} + let __return_value = {intrinsic_call}{const}({args}); + println!("Result {additional}-{{}}: {{:.150?}}", i+1, __return_value); + }} + }} + }}"#, + loaded_args = self.arguments.load_values_rust(), + intrinsic_call = self.name, + const = constraints, + args = self.arguments.as_call_param_rust(), + additional = additional, + ) + } +} diff --git a/crux-mir/lib/stdarch/crates/intrinsic-test/src/main.rs b/crux-mir/lib/stdarch/crates/intrinsic-test/src/main.rs new file mode 100644 index 000000000..43f2df08b --- /dev/null +++ b/crux-mir/lib/stdarch/crates/intrinsic-test/src/main.rs @@ -0,0 +1,501 @@ +#![feature(slice_partition_dedup)] +#[macro_use] +extern crate lazy_static; +#[macro_use] +extern crate log; + +use std::fs::File; +use std::io::Write; +use std::process::Command; + +use clap::{App, Arg}; +use intrinsic::Intrinsic; +use itertools::Itertools; +use rayon::prelude::*; +use types::TypeKind; + +use crate::acle_csv_parser::get_acle_intrinsics; +use crate::argument::Argument; + +mod acle_csv_parser; +mod argument; +mod intrinsic; +mod types; +mod values; + +// The number of times each intrinsic will be called. +const PASSES: u32 = 20; + +#[derive(Debug, PartialEq)] +pub enum Language { + Rust, + C, +} + +fn gen_code_c( + intrinsic: &Intrinsic, + constraints: &[&Argument], + name: String, + p64_armv7_workaround: bool, +) -> String { + if let Some((current, constraints)) = constraints.split_last() { + let range = current + .constraints + .iter() + .map(|c| c.to_range()) + .flat_map(|r| r.into_iter()); + + range + .map(|i| { + format!( + r#" {{ + {ty} {name} = {val}; +{pass} + }}"#, + name = current.name, + ty = current.ty.c_type(), + val = i, + pass = gen_code_c( + intrinsic, + constraints, + format!("{}-{}", name, i), + p64_armv7_workaround + ) + ) + }) + .collect() + } else { + intrinsic.generate_loop_c(&name, PASSES, p64_armv7_workaround) + } +} + +fn generate_c_program( + header_files: &[&str], + intrinsic: &Intrinsic, + p64_armv7_workaround: bool, +) -> String { + let constraints = intrinsic + .arguments + .iter() + .filter(|i| i.has_constraint()) + .collect_vec(); + + format!( + r#"{header_files} +#include +#include +#include +#include + +template T1 cast(T2 x) {{ + static_assert(sizeof(T1) == sizeof(T2), "sizeof T1 and T2 must be the same"); + T1 ret{{}}; + memcpy(&ret, &x, sizeof(T1)); + return ret; +}} + +#ifdef __aarch64__ +std::ostream& operator<<(std::ostream& os, poly128_t value) {{ + std::stringstream temp; + do {{ + int n = value % 10; + value /= 10; + temp << n; + }} while (value != 0); + std::string tempstr(temp.str()); + std::string res(tempstr.rbegin(), tempstr.rend()); + os << res; + return os; +}} +#endif + +{arglists} + +int main(int argc, char **argv) {{ +{passes} + return 0; +}}"#, + header_files = header_files + .iter() + .map(|header| format!("#include <{}>", header)) + .collect::>() + .join("\n"), + arglists = intrinsic.arguments.gen_arglists_c(PASSES), + passes = gen_code_c( + intrinsic, + constraints.as_slice(), + Default::default(), + p64_armv7_workaround + ), + ) +} + +fn gen_code_rust(intrinsic: &Intrinsic, constraints: &[&Argument], name: String) -> String { + if let Some((current, constraints)) = constraints.split_last() { + let range = current + .constraints + .iter() + .map(|c| c.to_range()) + .flat_map(|r| r.into_iter()); + + range + .map(|i| { + format!( + r#" {{ + const {name}: {ty} = {val}; +{pass} + }}"#, + name = current.name, + ty = current.ty.rust_type(), + val = i, + pass = gen_code_rust(intrinsic, constraints, format!("{}-{}", name, i)) + ) + }) + .collect() + } else { + intrinsic.generate_loop_rust(&name, PASSES) + } +} + +fn generate_rust_program(intrinsic: &Intrinsic, a32: bool) -> String { + let constraints = intrinsic + .arguments + .iter() + .filter(|i| i.has_constraint()) + .collect_vec(); + + format!( + r#"#![feature(simd_ffi)] +#![feature(link_llvm_intrinsics)] +#![feature(stdsimd)] +#![allow(overflowing_literals)] +#![allow(non_upper_case_globals)] +use core_arch::arch::{target_arch}::*; + +{arglists} + +fn main() {{ +{passes} +}} +"#, + target_arch = if a32 { "arm" } else { "aarch64" }, + arglists = intrinsic.arguments.gen_arglists_rust(PASSES), + passes = gen_code_rust(intrinsic, &constraints, Default::default()) + ) +} + +fn compile_c(c_filename: &str, intrinsic: &Intrinsic, compiler: &str, a32: bool) -> bool { + let flags = std::env::var("CPPFLAGS").unwrap_or("".into()); + + let output = Command::new("sh") + .arg("-c") + .arg(format!( + "{cpp} {cppflags} {arch_flags} -Wno-narrowing -O2 -target {target} -o c_programs/{intrinsic} {filename}", + target = if a32 { "armv7-unknown-linux-gnueabihf" } else { "aarch64-unknown-linux-gnu" }, + arch_flags = if a32 { "-march=armv8.6-a+crypto+crc+dotprod" } else { "-march=armv8.6-a+crypto+sha3+crc+dotprod" }, + filename = c_filename, + intrinsic = intrinsic.name, + cpp = compiler, + cppflags = flags, + )) + .output(); + if let Ok(output) = output { + if output.status.success() { + true + } else { + error!( + "Failed to compile code for intrinsic: {}\n\nstdout:\n{}\n\nstderr:\n{}", + intrinsic.name, + std::str::from_utf8(&output.stdout).unwrap_or(""), + std::str::from_utf8(&output.stderr).unwrap_or("") + ); + false + } + } else { + error!("Command failed: {:#?}", output); + false + } +} + +fn build_c(intrinsics: &Vec, compiler: &str, a32: bool) -> bool { + let _ = std::fs::create_dir("c_programs"); + intrinsics + .par_iter() + .map(|i| { + let c_filename = format!(r#"c_programs/{}.cpp"#, i.name); + let mut file = File::create(&c_filename).unwrap(); + + let c_code = generate_c_program(&["arm_neon.h", "arm_acle.h"], &i, a32); + file.write_all(c_code.into_bytes().as_slice()).unwrap(); + compile_c(&c_filename, &i, compiler, a32) + }) + .find_any(|x| !x) + .is_none() +} + +fn build_rust(intrinsics: &Vec, toolchain: &str, a32: bool) -> bool { + intrinsics.iter().for_each(|i| { + let rust_dir = format!(r#"rust_programs/{}"#, i.name); + let _ = std::fs::create_dir_all(&rust_dir); + let rust_filename = format!(r#"{}/main.rs"#, rust_dir); + let mut file = File::create(&rust_filename).unwrap(); + + let c_code = generate_rust_program(&i, a32); + file.write_all(c_code.into_bytes().as_slice()).unwrap(); + }); + + let mut cargo = File::create("rust_programs/Cargo.toml").unwrap(); + cargo + .write_all( + format!( + r#"[package] +name = "intrinsic-test" +version = "{version}" +authors = ["{authors}"] +edition = "2018" +[workspace] +[dependencies] +core_arch = {{ path = "../crates/core_arch" }} +{binaries}"#, + version = env!("CARGO_PKG_VERSION"), + authors = env!("CARGO_PKG_AUTHORS"), + binaries = intrinsics + .iter() + .map(|i| { + format!( + r#"[[bin]] +name = "{intrinsic}" +path = "{intrinsic}/main.rs""#, + intrinsic = i.name + ) + }) + .collect::>() + .join("\n") + ) + .into_bytes() + .as_slice(), + ) + .unwrap(); + + let output = Command::new("sh") + .current_dir("rust_programs") + .arg("-c") + .arg(format!( + "cargo {toolchain} build --target {target} --release", + toolchain = toolchain, + target = if a32 { + "armv7-unknown-linux-gnueabihf" + } else { + "aarch64-unknown-linux-gnu" + }, + )) + .env("RUSTFLAGS", "-Cdebuginfo=0") + .output(); + if let Ok(output) = output { + if output.status.success() { + true + } else { + error!( + "Failed to compile code for intrinsics\n\nstdout:\n{}\n\nstderr:\n{}", + std::str::from_utf8(&output.stdout).unwrap_or(""), + std::str::from_utf8(&output.stderr).unwrap_or("") + ); + false + } + } else { + error!("Command failed: {:#?}", output); + false + } +} + +fn main() { + pretty_env_logger::init(); + + let matches = App::new("Intrinsic test tool") + .about("Generates Rust and C programs for intrinsics and compares the output") + .arg( + Arg::with_name("INPUT") + .help("The input file containing the intrinsics") + .required(true) + .index(1), + ) + .arg( + Arg::with_name("TOOLCHAIN") + .takes_value(true) + .long("toolchain") + .help("The rust toolchain to use for building the rust code"), + ) + .arg( + Arg::with_name("CPPCOMPILER") + .takes_value(true) + .default_value("clang++") + .long("cppcompiler") + .help("The C++ compiler to use for compiling the c++ code"), + ) + .arg( + Arg::with_name("RUNNER") + .takes_value(true) + .long("runner") + .help("Run the C programs under emulation with this command"), + ) + .arg( + Arg::with_name("SKIP") + .takes_value(true) + .long("skip") + .help("Filename for a list of intrinsics to skip (one per line)"), + ) + .arg( + Arg::with_name("A32") + .takes_value(false) + .long("a32") + .help("Run tests for A32 instrinsics instead of A64"), + ) + .get_matches(); + + let filename = matches.value_of("INPUT").unwrap(); + let toolchain = matches + .value_of("TOOLCHAIN") + .map_or("".into(), |t| format!("+{}", t)); + + let cpp_compiler = matches.value_of("CPPCOMPILER").unwrap(); + let c_runner = matches.value_of("RUNNER").unwrap_or(""); + let skip = if let Some(filename) = matches.value_of("SKIP") { + let data = std::fs::read_to_string(&filename).expect("Failed to open file"); + data.lines() + .map(str::trim) + .filter(|s| !s.contains('#')) + .map(String::from) + .collect_vec() + } else { + Default::default() + }; + let a32 = matches.is_present("A32"); + + let intrinsics = get_acle_intrinsics(filename); + + let mut intrinsics = intrinsics + .into_iter() + // Not sure how we would compare intrinsic that returns void. + .filter(|i| i.results.kind() != TypeKind::Void) + .filter(|i| i.results.kind() != TypeKind::BFloat) + .filter(|i| !(i.results.kind() == TypeKind::Float && i.results.inner_size() == 16)) + .filter(|i| !i.arguments.iter().any(|a| a.ty.kind() == TypeKind::BFloat)) + .filter(|i| { + !i.arguments + .iter() + .any(|a| a.ty.kind() == TypeKind::Float && a.ty.inner_size() == 16) + }) + // Skip pointers for now, we would probably need to look at the return + // type to work out how many elements we need to point to. + .filter(|i| !i.arguments.iter().any(|a| a.is_ptr())) + .filter(|i| !i.arguments.iter().any(|a| a.ty.inner_size() == 128)) + .filter(|i| !skip.contains(&i.name)) + .filter(|i| !(a32 && i.a64_only)) + .collect::>(); + intrinsics.dedup(); + + if !build_c(&intrinsics, cpp_compiler, a32) { + std::process::exit(2); + } + + if !build_rust(&intrinsics, &toolchain, a32) { + std::process::exit(3); + } + + if !compare_outputs(&intrinsics, &toolchain, &c_runner, a32) { + std::process::exit(1) + } +} + +enum FailureReason { + RunC(String), + RunRust(String), + Difference(String, String, String), +} + +fn compare_outputs(intrinsics: &Vec, toolchain: &str, runner: &str, a32: bool) -> bool { + let intrinsics = intrinsics + .par_iter() + .filter_map(|intrinsic| { + let c = Command::new("sh") + .arg("-c") + .arg(format!( + "{runner} ./c_programs/{intrinsic}", + runner = runner, + intrinsic = intrinsic.name, + )) + .output(); + let rust = Command::new("sh") + .current_dir("rust_programs") + .arg("-c") + .arg(format!( + "cargo {toolchain} run --target {target} --bin {intrinsic} --release", + intrinsic = intrinsic.name, + toolchain = toolchain, + target = if a32 { + "armv7-unknown-linux-gnueabihf" + } else { + "aarch64-unknown-linux-gnu" + }, + )) + .env("RUSTFLAGS", "-Cdebuginfo=0") + .output(); + + let (c, rust) = match (c, rust) { + (Ok(c), Ok(rust)) => (c, rust), + a => panic!("{:#?}", a), + }; + + if !c.status.success() { + error!("Failed to run C program for intrinsic {}", intrinsic.name); + return Some(FailureReason::RunC(intrinsic.name.clone())); + } + + if !rust.status.success() { + error!( + "Failed to run rust program for intrinsic {}", + intrinsic.name + ); + return Some(FailureReason::RunRust(intrinsic.name.clone())); + } + + info!("Comparing intrinsic: {}", intrinsic.name); + + let c = std::str::from_utf8(&c.stdout) + .unwrap() + .to_lowercase() + .replace("-nan", "nan"); + let rust = std::str::from_utf8(&rust.stdout) + .unwrap() + .to_lowercase() + .replace("-nan", "nan"); + + if c == rust { + None + } else { + Some(FailureReason::Difference(intrinsic.name.clone(), c, rust)) + } + }) + .collect::>(); + + intrinsics.iter().for_each(|reason| match reason { + FailureReason::Difference(intrinsic, c, rust) => { + println!("Difference for intrinsic: {}", intrinsic); + let diff = diff::lines(c, rust); + diff.iter().for_each(|diff| match diff { + diff::Result::Left(c) => println!("C: {}", c), + diff::Result::Right(rust) => println!("Rust: {}", rust), + diff::Result::Both(_, _) => (), + }); + println!("****************************************************************"); + } + FailureReason::RunC(intrinsic) => { + println!("Failed to run C program for intrinsic {}", intrinsic) + } + FailureReason::RunRust(intrinsic) => { + println!("Failed to run rust program for intrinsic {}", intrinsic) + } + }); + println!("{} differences found", intrinsics.len()); + intrinsics.is_empty() +} diff --git a/crux-mir/lib/stdarch/crates/intrinsic-test/src/types.rs b/crux-mir/lib/stdarch/crates/intrinsic-test/src/types.rs new file mode 100644 index 000000000..dd23586e7 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/intrinsic-test/src/types.rs @@ -0,0 +1,436 @@ +use std::fmt; +use std::str::FromStr; + +use crate::values::value_for_array; +use crate::Language; + +#[derive(Debug, PartialEq, Copy, Clone)] +pub enum TypeKind { + BFloat, + Float, + Int, + UInt, + Poly, + Void, +} + +impl FromStr for TypeKind { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "bfloat" => Ok(Self::BFloat), + "float" => Ok(Self::Float), + "int" => Ok(Self::Int), + "poly" => Ok(Self::Poly), + "uint" | "unsigned" => Ok(Self::UInt), + "void" => Ok(Self::Void), + _ => Err(format!("Impossible to parse argument kind {}", s)), + } + } +} + +impl fmt::Display for TypeKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}", + match self { + Self::BFloat => "bfloat", + Self::Float => "float", + Self::Int => "int", + Self::UInt => "uint", + Self::Poly => "poly", + Self::Void => "void", + } + ) + } +} + +impl TypeKind { + /// Gets the type part of a c typedef for a type that's in the form of {type}{size}_t. + pub fn c_prefix(&self) -> &str { + match self { + Self::Float => "float", + Self::Int => "int", + Self::UInt => "uint", + Self::Poly => "poly", + _ => unreachable!("Not used: {:#?}", self), + } + } + + /// Gets the rust prefix for the type kind i.e. i, u, f. + pub fn rust_prefix(&self) -> &str { + match self { + Self::Float => "f", + Self::Int => "i", + Self::UInt => "u", + Self::Poly => "u", + _ => unreachable!("Unused type kind: {:#?}", self), + } + } +} + +#[derive(Debug, PartialEq, Clone)] +pub enum IntrinsicType { + Ptr { + constant: bool, + child: Box, + }, + Type { + constant: bool, + kind: TypeKind, + /// The bit length of this type (e.g. 32 for u32). + bit_len: Option, + + /// Length of the SIMD vector (i.e. 4 for uint32x4_t), A value of `None` + /// means this is not a simd type. A `None` can be assumed to be 1, + /// although in some places a distinction is needed between `u64` and + /// `uint64x1_t` this signals that. + simd_len: Option, + + /// The number of rows for SIMD matrices (i.e. 2 for uint8x8x2_t). + /// A value of `None` represents a type that does not contain any + /// rows encoded in the type (e.g. uint8x8_t). + /// A value of `None` can be assumed to be 1 though. + vec_len: Option, + }, +} + +impl IntrinsicType { + /// Get the TypeKind for this type, recursing into pointers. + pub fn kind(&self) -> TypeKind { + match *self { + IntrinsicType::Ptr { ref child, .. } => child.kind(), + IntrinsicType::Type { kind, .. } => kind, + } + } + + /// Get the size of a single element inside this type, recursing into + /// pointers, i.e. a pointer to a u16 would be 16 rather than the size + /// of a pointer. + pub fn inner_size(&self) -> u32 { + match *self { + IntrinsicType::Ptr { ref child, .. } => child.inner_size(), + IntrinsicType::Type { + bit_len: Some(bl), .. + } => bl, + _ => unreachable!(""), + } + } + + pub fn num_lanes(&self) -> u32 { + match *self { + IntrinsicType::Ptr { ref child, .. } => child.num_lanes(), + IntrinsicType::Type { + simd_len: Some(sl), .. + } => sl, + _ => 1, + } + } + + pub fn num_vectors(&self) -> u32 { + match *self { + IntrinsicType::Ptr { ref child, .. } => child.num_vectors(), + IntrinsicType::Type { + vec_len: Some(vl), .. + } => vl, + _ => 1, + } + } + + /// Determine if the type is a simd type, this will treat a type such as + /// `uint64x1` as simd. + pub fn is_simd(&self) -> bool { + match *self { + IntrinsicType::Ptr { ref child, .. } => child.is_simd(), + IntrinsicType::Type { + simd_len: None, + vec_len: None, + .. + } => false, + _ => true, + } + } + + pub fn is_ptr(&self) -> bool { + match *self { + IntrinsicType::Ptr { .. } => true, + IntrinsicType::Type { .. } => false, + } + } + + pub fn c_scalar_type(&self) -> String { + format!( + "{prefix}{bits}_t", + prefix = self.kind().c_prefix(), + bits = self.inner_size() + ) + } + + pub fn rust_scalar_type(&self) -> String { + format!( + "{prefix}{bits}", + prefix = self.kind().rust_prefix(), + bits = self.inner_size() + ) + } + + /// Gets a string containing the typename for this type in C format. + pub fn c_type(&self) -> String { + match self { + IntrinsicType::Ptr { child, .. } => child.c_type(), + IntrinsicType::Type { + constant, + kind, + bit_len: Some(bit_len), + simd_len: None, + vec_len: None, + .. + } => format!( + "{}{}{}_t", + if *constant { "const " } else { "" }, + kind.c_prefix(), + bit_len + ), + IntrinsicType::Type { + kind, + bit_len: Some(bit_len), + simd_len: Some(simd_len), + vec_len: None, + .. + } => format!("{}{}x{}_t", kind.c_prefix(), bit_len, simd_len), + IntrinsicType::Type { + kind, + bit_len: Some(bit_len), + simd_len: Some(simd_len), + vec_len: Some(vec_len), + .. + } => format!("{}{}x{}x{}_t", kind.c_prefix(), bit_len, simd_len, vec_len), + _ => todo!("{:#?}", self), + } + } + + pub fn c_single_vector_type(&self) -> String { + match self { + IntrinsicType::Ptr { child, .. } => child.c_single_vector_type(), + IntrinsicType::Type { + kind, + bit_len: Some(bit_len), + simd_len: Some(simd_len), + vec_len: Some(_), + .. + } => format!("{}{}x{}_t", kind.c_prefix(), bit_len, simd_len), + _ => unreachable!("Shouldn't be called on this type"), + } + } + + pub fn rust_type(&self) -> String { + match self { + IntrinsicType::Ptr { child, .. } => child.c_type(), + IntrinsicType::Type { + kind, + bit_len: Some(bit_len), + simd_len: None, + vec_len: None, + .. + } => format!("{}{}", kind.rust_prefix(), bit_len), + IntrinsicType::Type { + kind, + bit_len: Some(bit_len), + simd_len: Some(simd_len), + vec_len: None, + .. + } => format!("{}{}x{}_t", kind.c_prefix(), bit_len, simd_len), + IntrinsicType::Type { + kind, + bit_len: Some(bit_len), + simd_len: Some(simd_len), + vec_len: Some(vec_len), + .. + } => format!("{}{}x{}x{}_t", kind.c_prefix(), bit_len, simd_len, vec_len), + _ => todo!("{:#?}", self), + } + } + + /// Gets a cast for this type if needs promotion. + /// This is required for 8 bit types due to printing as the 8 bit types use + /// a char and when using that in `std::cout` it will print as a character, + /// which means value of 0 will be printed as a null byte. + /// + /// This is also needed for polynomial types because we want them to be + /// printed as unsigned integers to match Rust's `Debug` impl. + pub fn c_promotion(&self) -> &str { + match *self { + IntrinsicType::Type { + kind, + bit_len: Some(bit_len), + .. + } if bit_len == 8 => match kind { + TypeKind::Int => "(int)", + TypeKind::UInt => "(unsigned int)", + TypeKind::Poly => "(unsigned int)(uint8_t)", + _ => "", + }, + IntrinsicType::Type { + kind: TypeKind::Poly, + bit_len: Some(bit_len), + .. + } => match bit_len { + 8 => unreachable!("handled above"), + 16 => "(uint16_t)", + 32 => "(uint32_t)", + 64 => "(uint64_t)", + 128 => "", + _ => panic!("invalid bit_len"), + }, + _ => "", + } + } + + /// Generates a comma list of values that can be used to initialize the array that + /// an argument for the intrinsic call is loaded from. + /// This is determistic based on the pass number. + /// + /// * `loads`: The number of values that need to be loaded from the argument array + /// * e.g for argument type uint32x2, loads=2 results in a string representing 4 32-bit values + /// + /// Returns a string such as + /// * `0x1, 0x7F, 0xFF` if `language` is `Language::C` + /// * `0x1 as _, 0x7F as _, 0xFF as _` if `language` is `Language::Rust` + pub fn populate_random(&self, loads: u32, language: &Language) -> String { + match self { + IntrinsicType::Ptr { child, .. } => child.populate_random(loads, language), + IntrinsicType::Type { + bit_len: Some(bit_len), + kind, + simd_len, + vec_len, + .. + } if kind == &TypeKind::Int || kind == &TypeKind::UInt || kind == &TypeKind::Poly => (0 + ..(simd_len.unwrap_or(1) * vec_len.unwrap_or(1) + loads - 1)) + .map(|i| { + format!( + "{}{}", + value_for_array(*bit_len, i), + match language { + &Language::Rust => format!(" as {ty} ", ty = self.rust_scalar_type()), + &Language::C => String::from(""), + } + ) + }) + .collect::>() + .join(","), + IntrinsicType::Type { + kind: TypeKind::Float, + bit_len: Some(32), + simd_len, + vec_len, + .. + } => (0..(simd_len.unwrap_or(1) * vec_len.unwrap_or(1) + loads - 1)) + .map(|i| { + format!( + "{}({})", + match language { + &Language::Rust => "std::mem::transmute", + &Language::C => "cast", + }, + value_for_array(32, i), + ) + }) + .collect::>() + .join(","), + IntrinsicType::Type { + kind: TypeKind::Float, + bit_len: Some(64), + simd_len, + vec_len, + .. + } => (0..(simd_len.unwrap_or(1) * vec_len.unwrap_or(1) + loads - 1)) + .map(|i| { + format!( + "{}({}{})", + match language { + &Language::Rust => "std::mem::transmute", + &Language::C => "cast", + }, + value_for_array(64, i), + match language { + &Language::Rust => " as u64", + &Language::C => "", + } + ) + }) + .collect::>() + .join(","), + _ => unreachable!("populate random: {:#?}", self), + } + } + + /// Determines the load function for this type. + pub fn get_load_function(&self, armv7_p64_workaround: bool) -> String { + match self { + IntrinsicType::Ptr { child, .. } => child.get_load_function(armv7_p64_workaround), + IntrinsicType::Type { + kind: k, + bit_len: Some(bl), + simd_len, + vec_len, + .. + } => { + let quad = if simd_len.unwrap_or(1) * bl > 64 { + "q" + } else { + "" + }; + format!( + "vld{len}{quad}_{type}{size}", + type = match k { + TypeKind::UInt => "u", + TypeKind::Int => "s", + TypeKind::Float => "f", + // The ACLE doesn't support 64-bit polynomial loads on Armv7 + TypeKind::Poly => if armv7_p64_workaround && *bl == 64 {"s"} else {"p"}, + x => todo!("get_load_function TypeKind: {:#?}", x), + }, + size = bl, + quad = quad, + len = vec_len.unwrap_or(1), + ) + } + _ => todo!("get_load_function IntrinsicType: {:#?}", self), + } + } + + /// Determines the get lane function for this type. + pub fn get_lane_function(&self) -> String { + match self { + IntrinsicType::Ptr { child, .. } => child.get_lane_function(), + IntrinsicType::Type { + kind: k, + bit_len: Some(bl), + simd_len, + .. + } => { + let quad = if (simd_len.unwrap_or(1) * bl) > 64 { + "q" + } else { + "" + }; + format!( + "vget{quad}_lane_{type}{size}", + type = match k { + TypeKind::UInt => "u", + TypeKind::Int => "s", + TypeKind::Float => "f", + TypeKind::Poly => "p", + x => todo!("get_load_function TypeKind: {:#?}", x), + }, + size = bl, + quad = quad, + ) + } + _ => todo!("get_lane_function IntrinsicType: {:#?}", self), + } + } +} diff --git a/crux-mir/lib/stdarch/crates/intrinsic-test/src/values.rs b/crux-mir/lib/stdarch/crates/intrinsic-test/src/values.rs new file mode 100644 index 000000000..64b4d9fc9 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/intrinsic-test/src/values.rs @@ -0,0 +1,125 @@ +/// Gets a hex constant value for a single value in the argument values array in a determistic way +/// * `bits`: The number of bits for the type, only 8, 16, 32, 64 are valid values +/// * `index`: The position in the array we are generating for +pub fn value_for_array(bits: u32, index: u32) -> String { + let index = index as usize; + + if bits == 8 { + format!("{:#X}", VALUES_8[index % VALUES_8.len()]) + } else if bits == 16 { + format!("{:#X}", VALUES_16[index % VALUES_16.len()]) + } else if bits == 32 { + format!("{:#X}", VALUES_32[index % VALUES_32.len()]) + } else if bits == 64 { + format!("{:#X}", VALUES_64[index % VALUES_64.len()]) + } else { + panic!("Unknown size: {}", bits); + } +} + +pub const VALUES_8: &[u8] = &[ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0xf0, 0x80, 0x3b, 0xff, +]; + +pub const VALUES_16: &[u16] = &[ + 0x0000, // 0.0 + 0x0400, // The smallest normal value. + 0x37ff, // The value just below 0.5. + 0x3800, // 0.5 + 0x3801, // The value just above 0.5. + 0x3bff, // The value just below 1.0. + 0x3c00, // 1.0 + 0x3c01, // The value just above 1.0. + 0x3e00, // 1.5 + 0x4900, // 10 + 0x7bff, // The largest finite value. + 0x7c00, // Infinity. + // NaNs. + // - Quiet NaNs + 0x7f23, 0x7e00, // - Signalling NaNs + 0x7d23, 0x7c01, // Subnormals. + // - A recognisable bit pattern. + 0x0012, // - The largest subnormal value. + 0x03ff, // - The smallest subnormal value. + 0x0001, // The same values again, but negated. + 0x8000, 0x8400, 0xb7ff, 0xb800, 0xb801, 0xbbff, 0xbc00, 0xbc01, 0xbe00, 0xc900, 0xfbff, 0xfc00, + 0xff23, 0xfe00, 0xfd23, 0xfc01, 0x8012, 0x83ff, 0x8001, +]; + +pub const VALUES_32: &[u32] = &[ + // Simple values. + 0x00000000, // 0.0 + 0x00800000, // The smallest normal value. + 0x3effffff, // The value just below 0.5. + 0x3f000000, // 0.5 + 0x3f000001, // The value just above 0.5. + 0x3f7fffff, // The value just below 1.0. + 0x3f800000, // 1.0 + 0x3f800001, // The value just above 1.0. + 0x3fc00000, // 1.5 + 0x41200000, // 10 + 0x7f8fffff, // The largest finite value. + 0x7f800000, // Infinity. + // NaNs. + // - Quiet NaNs + 0x7fd23456, 0x7fc00000, // - Signalling NaNs + 0x7f923456, 0x7f800001, // Subnormals. + // - A recognisable bit pattern. + 0x00123456, // - The largest subnormal value. + 0x007fffff, // - The smallest subnormal value. + 0x00000001, // The same values again, but negated. + 0x80000000, 0x80800000, 0xbeffffff, 0xbf000000, 0xbf000001, 0xbf7fffff, 0xbf800000, 0xbf800001, + 0xbfc00000, 0xc1200000, 0xff8fffff, 0xff800000, 0xffd23456, 0xffc00000, 0xff923456, 0xff800001, + 0x80123456, 0x807fffff, 0x80000001, +]; + +pub const VALUES_64: &[u64] = &[ + // Simple values. + 0x0000000000000000, // 0.0 + 0x0010000000000000, // The smallest normal value. + 0x3fdfffffffffffff, // The value just below 0.5. + 0x3fe0000000000000, // 0.5 + 0x3fe0000000000001, // The value just above 0.5. + 0x3fefffffffffffff, // The value just below 1.0. + 0x3ff0000000000000, // 1.0 + 0x3ff0000000000001, // The value just above 1.0. + 0x3ff8000000000000, // 1.5 + 0x4024000000000000, // 10 + 0x7fefffffffffffff, // The largest finite value. + 0x7ff0000000000000, // Infinity. + // NaNs. + // - Quiet NaNs + 0x7ff923456789abcd, + 0x7ff8000000000000, + // - Signalling NaNs + 0x7ff123456789abcd, + 0x7ff0000000000000, + // Subnormals. + // - A recognisable bit pattern. + 0x000123456789abcd, + // - The largest subnormal value. + 0x000fffffffffffff, + // - The smallest subnormal value. + 0x0000000000000001, + // The same values again, but negated. + 0x8000000000000000, + 0x8010000000000000, + 0xbfdfffffffffffff, + 0xbfe0000000000000, + 0xbfe0000000000001, + 0xbfefffffffffffff, + 0xbff0000000000000, + 0xbff0000000000001, + 0xbff8000000000000, + 0xc024000000000000, + 0xffefffffffffffff, + 0xfff0000000000000, + 0xfff923456789abcd, + 0xfff8000000000000, + 0xfff123456789abcd, + 0xfff0000000000000, + 0x800123456789abcd, + 0x800fffffffffffff, + 0x8000000000000001, +]; diff --git a/crux-mir/lib/stdarch/crates/simd-test-macro/Cargo.toml b/crux-mir/lib/stdarch/crates/simd-test-macro/Cargo.toml index 039fe8c76..c3ecf981e 100644 --- a/crux-mir/lib/stdarch/crates/simd-test-macro/Cargo.toml +++ b/crux-mir/lib/stdarch/crates/simd-test-macro/Cargo.toml @@ -2,6 +2,7 @@ name = "simd-test-macro" version = "0.1.0" authors = ["Alex Crichton "] +edition = "2018" [lib] proc-macro = true diff --git a/crux-mir/lib/stdarch/crates/simd-test-macro/src/lib.rs b/crux-mir/lib/stdarch/crates/simd-test-macro/src/lib.rs index 4d1170cc7..9d81a4c5e 100644 --- a/crux-mir/lib/stdarch/crates/simd-test-macro/src/lib.rs +++ b/crux-mir/lib/stdarch/crates/simd-test-macro/src/lib.rs @@ -2,13 +2,12 @@ //! //! This macro expands to a `#[test]` function which tests the local machine //! for the appropriate cfg before calling the inner test function. +#![deny(rust_2018_idioms)] -extern crate proc_macro; -extern crate proc_macro2; #[macro_use] extern crate quote; -use proc_macro2::{Ident, Literal, Span, TokenStream, TokenTree}; +use proc_macro2::{Delimiter, Ident, Literal, Span, TokenStream, TokenTree}; use quote::ToTokens; use std::env; @@ -44,8 +43,6 @@ pub fn simd_test( .map(String::from) .collect(); - let mmx = target_features.iter().any(|s| s.starts_with("mmx")); - let enable_feature = string(enable_feature); let item = TokenStream::from(item); let name = find_name(item.clone()); @@ -67,6 +64,7 @@ pub fn simd_test( "i686" | "x86_64" | "i586" => "is_x86_feature_detected", "arm" | "armv7" => "is_arm_feature_detected", "aarch64" => "is_aarch64_feature_detected", + maybe_riscv if maybe_riscv.starts_with("riscv") => "is_riscv_feature_detected", "powerpc" | "powerpcle" => "is_powerpc_feature_detected", "powerpc64" | "powerpc64le" => "is_powerpc64_feature_detected", "mips" | "mipsel" | "mipsisa32r6" | "mipsisa32r6el" => { @@ -106,15 +104,6 @@ pub fn simd_test( TokenStream::new() }; - let emms = if mmx { - // note: if the test requires MMX we need to clear the FPU - // registers once the test finishes before interfacing with - // other x87 code: - quote! { unsafe { super::_mm_empty() }; } - } else { - TokenStream::new() - }; - let ret: TokenStream = quote_spanned! { proc_macro2::Span::call_site() => #[allow(non_snake_case)] @@ -123,7 +112,6 @@ pub fn simd_test( fn #name() { if #force_test | (#cfg_target_features) { let v = unsafe { #name() }; - #emms return v; } else { ::stdarch_test::assert_skip_test_ok(stringify!(#name)); @@ -146,8 +134,18 @@ fn find_name(item: TokenStream) -> Ident { } } - match tokens.next() { - Some(TokenTree::Ident(word)) => word, - _ => panic!("failed to find function name"), + fn get_ident(tt: TokenTree) -> Option { + match tt { + TokenTree::Ident(i) => Some(i), + TokenTree::Group(g) if g.delimiter() == Delimiter::None => { + get_ident(g.stream().into_iter().next()?) + } + _ => None, + } } + + tokens + .next() + .and_then(get_ident) + .expect("failed to find function name") } diff --git a/crux-mir/lib/stdarch/crates/std_detect/Cargo.toml b/crux-mir/lib/stdarch/crates/std_detect/Cargo.toml index 252e42c56..3a482564e 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/Cargo.toml +++ b/crux-mir/lib/stdarch/crates/std_detect/Cargo.toml @@ -7,13 +7,12 @@ authors = [ "Gonzalo Brito Gadeschi ", ] description = "`std::detect` - Rust's standard library run-time CPU feature detection." -documentation = "https://docs.rs/std_detect" homepage = "https://github.com/rust-lang/stdarch" repository = "https://github.com/rust-lang/stdarch" readme = "README.md" keywords = ["std", "run-time", "feature", "detection"] categories = ["hardware-support"] -license = "MIT/Apache-2.0" +license = "MIT OR Apache-2.0" edition = "2018" [badges] @@ -23,7 +22,12 @@ maintenance = { status = "experimental" } [dependencies] libc = { version = "0.2", optional = true, default-features = false } -cfg-if = "0.1.10" +cfg-if = "1.0.0" + +# When built as part of libstd +core = { version = "1.0.0", optional = true, package = "rustc-std-workspace-core" } +compiler_builtins = { version = "0.1.2", optional = true } +alloc = { version = "1.0.0", optional = true, package = "rustc-std-workspace-alloc" } [dev-dependencies] auxv = "0.3.3" @@ -31,6 +35,11 @@ cupid = "0.6.0" [features] default = [ "std_detect_dlsym_getauxval", "std_detect_file_io" ] -std_detect_file_io = [] +std_detect_file_io = [ "libc" ] std_detect_dlsym_getauxval = [ "libc" ] -std_detect_env_override = [] +std_detect_env_override = [ "libc" ] +rustc-dep-of-std = [ + "core", + "compiler_builtins", + "alloc", +] diff --git a/crux-mir/lib/stdarch/crates/std_detect/README.md b/crux-mir/lib/stdarch/crates/std_detect/README.md index bb26c3471..bea7d941a 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/README.md +++ b/crux-mir/lib/stdarch/crates/std_detect/README.md @@ -1,9 +1,6 @@ `std::detect` - Rust's standard library run-time CPU feature detection ======= -[![std_detect_crate_badge]][std_detect_crate_link] [![std_detect_docs_badge]][std_detect_docs_link] - - The private `std::detect` module implements run-time feature detection in Rust's standard library. This allows detecting whether the CPU the binary runs on supports certain features, like SIMD instructions. @@ -74,8 +71,3 @@ at your option. Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in `std_detect` by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. - -[std_detect_crate_badge]: https://img.shields.io/crates/v/std_detect.svg -[std_detect_crate_link]: https://crates.io/crates/std_detect -[std_detect_docs_badge]: https://docs.rs/std_detect/badge.svg -[std_detect_docs_link]: https://docs.rs/std_detect/ diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/aarch64.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/aarch64.rs index 154207e5a..5f46c7696 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/aarch64.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/aarch64.rs @@ -2,35 +2,152 @@ features! { @TARGET: aarch64; + @CFG: target_arch = "aarch64"; @MACRO_NAME: is_aarch64_feature_detected; @MACRO_ATTRS: - /// Checks if `aarch64` feature is enabled. - #[unstable(feature = "stdsimd", issue = "27731")] + /// This macro tests, at runtime, whether an `aarch64` feature is enabled on aarch64 platforms. + /// Currently most features are only supported on linux-based platforms. + /// + /// This macro takes one argument which is a string literal of the feature being tested for. + /// The feature names are mostly taken from their FEAT_* definitions in the [ARM Architecture + /// Reference Manual][docs]. + /// + /// ## Supported arguments + /// + /// * `"asimd"` or "neon" - FEAT_AdvSIMD + /// * `"pmull"` - FEAT_PMULL + /// * `"fp"` - FEAT_FP + /// * `"fp16"` - FEAT_FP16 + /// * `"sve"` - FEAT_SVE + /// * `"crc"` - FEAT_CRC + /// * `"lse"` - FEAT_LSE + /// * `"lse2"` - FEAT_LSE2 + /// * `"rdm"` - FEAT_RDM + /// * `"rcpc"` - FEAT_LRCPC + /// * `"rcpc2"` - FEAT_LRCPC2 + /// * `"dotprod"` - FEAT_DotProd + /// * `"tme"` - FEAT_TME + /// * `"fhm"` - FEAT_FHM + /// * `"dit"` - FEAT_DIT + /// * `"flagm"` - FEAT_FLAGM + /// * `"ssbs"` - FEAT_SSBS + /// * `"sb"` - FEAT_SB + /// * `"paca"` - FEAT_PAuth (address authentication) + /// * `"pacg"` - FEAT_Pauth (generic authentication) + /// * `"dpb"` - FEAT_DPB + /// * `"dpb2"` - FEAT_DPB2 + /// * `"sve2"` - FEAT_SVE2 + /// * `"sve2-aes"` - FEAT_SVE2_AES + /// * `"sve2-sm4"` - FEAT_SVE2_SM4 + /// * `"sve2-sha3"` - FEAT_SVE2_SHA3 + /// * `"sve2-bitperm"` - FEAT_SVE2_BitPerm + /// * `"frintts"` - FEAT_FRINTTS + /// * `"i8mm"` - FEAT_I8MM + /// * `"f32mm"` - FEAT_F32MM + /// * `"f64mm"` - FEAT_F64MM + /// * `"bf16"` - FEAT_BF16 + /// * `"rand"` - FEAT_RNG + /// * `"bti"` - FEAT_BTI + /// * `"mte"` - FEAT_MTE + /// * `"jsconv"` - FEAT_JSCVT + /// * `"fcma"` - FEAT_FCMA + /// * `"aes"` - FEAT_AES + /// * `"sha2"` - FEAT_SHA1 & FEAT_SHA256 + /// * `"sha3"` - FEAT_SHA512 & FEAT_SHA3 + /// * `"sm4"` - FEAT_SM3 & FEAT_SM4 + /// + /// [docs]: https://developer.arm.com/documentation/ddi0487/latest + #[stable(feature = "simd_aarch64", since = "1.60.0")] @BIND_FEATURE_NAME: "asimd"; "neon"; @NO_RUNTIME_DETECTION: "ras"; @NO_RUNTIME_DETECTION: "v8.1a"; @NO_RUNTIME_DETECTION: "v8.2a"; @NO_RUNTIME_DETECTION: "v8.3a"; - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] asimd: "neon"; - /// ARM Advanced SIMD (ASIMD) - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] pmull: "pmull"; - /// Polynomial Multiply - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] fp: "fp"; - /// Floating point support - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] fp16: "fp16"; - /// Half-float support. - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] sve: "sve"; - /// Scalable Vector Extension (SVE) - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] crc: "crc"; - /// CRC32 (Cyclic Redundancy Check) - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] crypto: "crypto"; - /// Crypto: AES + PMULL + SHA1 + SHA2 - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] lse: "lse"; - /// Atomics (Large System Extension) - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] rdm: "rdm"; - /// Rounding Double Multiply (ASIMDRDM) - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] rcpc: "rcpc"; - /// Release consistent Processor consistent (RcPc) - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] dotprod: "dotprod"; - /// Vector Dot-Product (ASIMDDP) + @NO_RUNTIME_DETECTION: "v8.4a"; + @NO_RUNTIME_DETECTION: "v8.5a"; + @NO_RUNTIME_DETECTION: "v8.6a"; + @NO_RUNTIME_DETECTION: "v8.7a"; + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] asimd: "neon"; + /// FEAT_AdvSIMD (Advanced SIMD/NEON) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] pmull: "pmull"; + /// FEAT_PMULL (Polynomial Multiply) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] fp: "fp"; + implied by target_features: ["neon"]; + /// FEAT_FP (Floating point support) - Implied by `neon` target_feature + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] fp16: "fp16"; + /// FEAT_FP16 (Half-float support) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] sve: "sve"; + /// FEAT_SVE (Scalable Vector Extension) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] crc: "crc"; + /// FEAT_CRC32 (Cyclic Redundancy Check) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] lse: "lse"; + /// FEAT_LSE (Large System Extension - atomics) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] lse2: "lse2"; + /// FEAT_LSE2 (unaligned and register-pair atomics) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] rdm: "rdm"; + /// FEAT_RDM (Rounding Doubling Multiply - ASIMDRDM) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] rcpc: "rcpc"; + /// FEAT_LRCPC (Release consistent Processor consistent) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] rcpc2: "rcpc2"; + /// FEAT_LRCPC2 (RCPC with immediate offsets) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] dotprod: "dotprod"; + /// FEAT_DotProd (Vector Dot-Product - ASIMDDP) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] tme: "tme"; + /// FEAT_TME (Transactional Memory Extensions) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] fhm: "fhm"; + /// FEAT_FHM (fp16 multiplication instructions) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] dit: "dit"; + /// FEAT_DIT (Data Independent Timing instructions) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] flagm: "flagm"; + /// FEAT_FLAGM (flag manipulation instructions) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] ssbs: "ssbs"; + /// FEAT_SSBS (speculative store bypass safe) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] sb: "sb"; + /// FEAT_SB (speculation barrier) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] paca: "paca"; + /// FEAT_PAuth (address authentication) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] pacg: "pacg"; + /// FEAT_PAuth (generic authentication) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] dpb: "dpb"; + /// FEAT_DPB (aka dcpop - data cache clean to point of persistence) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] dpb2: "dpb2"; + /// FEAT_DPB2 (aka dcpodp - data cache clean to point of deep persistence) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] sve2: "sve2"; + /// FEAT_SVE2 (Scalable Vector Extension 2) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] sve2_aes: "sve2-aes"; + /// FEAT_SVE_AES (SVE2 AES crypto) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] sve2_sm4: "sve2-sm4"; + /// FEAT_SVE_SM4 (SVE2 SM4 crypto) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] sve2_sha3: "sve2-sha3"; + /// FEAT_SVE_SHA3 (SVE2 SHA3 crypto) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] sve2_bitperm: "sve2-bitperm"; + /// FEAT_SVE_BitPerm (SVE2 bit permutation instructions) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] frintts: "frintts"; + /// FEAT_FRINTTS (float to integer rounding instructions) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] i8mm: "i8mm"; + /// FEAT_I8MM (integer matrix multiplication, plus ASIMD support) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] f32mm: "f32mm"; + /// FEAT_F32MM (single-precision matrix multiplication) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] f64mm: "f64mm"; + /// FEAT_F64MM (double-precision matrix multiplication) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] bf16: "bf16"; + /// FEAT_BF16 (BFloat16 type, plus MM instructions, plus ASIMD support) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] rand: "rand"; + /// FEAT_RNG (Random Number Generator) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] bti: "bti"; + /// FEAT_BTI (Branch Target Identification) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] mte: "mte"; + /// FEAT_MTE (Memory Tagging Extension) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] jsconv: "jsconv"; + /// FEAT_JSCVT (JavaScript float conversion instructions) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] fcma: "fcma"; + /// FEAT_FCMA (float complex number operations) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] aes: "aes"; + /// FEAT_AES (AES instructions) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] sha2: "sha2"; + /// FEAT_SHA1 & FEAT_SHA256 (SHA1 & SHA2-256 instructions) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] sha3: "sha3"; + /// FEAT_SHA512 & FEAT_SHA3 (SHA2-512 & SHA3 instructions) + @FEATURE: #[stable(feature = "simd_aarch64", since = "1.60.0")] sm4: "sm4"; + /// FEAT_SM3 & FEAT_SM4 (SM3 & SM4 instructions) } diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/arm.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/arm.rs index 96978b700..897dc314c 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/arm.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/arm.rs @@ -2,6 +2,7 @@ features! { @TARGET: arm; + @CFG: target_arch = "arm"; @MACRO_NAME: is_arm_feature_detected; @MACRO_ATTRS: /// Checks if `arm` feature is enabled. @@ -16,4 +17,12 @@ features! { /// Polynomial Multiply @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] crc: "crc"; /// CRC32 (Cyclic Redundancy Check) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] crypto: "crypto"; + /// Crypto: AES + PMULL + SHA1 + SHA256. Prefer using the individual features where possible. + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] aes: "aes"; + /// FEAT_AES (AES instructions) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] sha2: "sha2"; + /// FEAT_SHA1 & FEAT_SHA256 (SHA1 & SHA2-256 instructions) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] i8mm: "i8mm"; + /// FEAT_I8MM } diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/mips.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/mips.rs index ada81b83e..ae27d0093 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/mips.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/mips.rs @@ -2,6 +2,7 @@ features! { @TARGET: mips; + @CFG: target_arch = "mips"; @MACRO_NAME: is_mips_feature_detected; @MACRO_ATTRS: /// Checks if `mips` feature is enabled. diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/mips64.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/mips64.rs index 6a0bb159b..7182ec2da 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/mips64.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/mips64.rs @@ -2,6 +2,7 @@ features! { @TARGET: mips64; + @CFG: target_arch = "mips64"; @MACRO_NAME: is_mips64_feature_detected; @MACRO_ATTRS: /// Checks if `mips64` feature is enabled. diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/mod.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/mod.rs new file mode 100644 index 000000000..81a1f23e8 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/mod.rs @@ -0,0 +1,56 @@ +#![allow(dead_code)] + +use cfg_if::cfg_if; + +// Export the macros for all supported architectures. +#[macro_use] +mod x86; +#[macro_use] +mod arm; +#[macro_use] +mod aarch64; +#[macro_use] +mod riscv; +#[macro_use] +mod powerpc; +#[macro_use] +mod powerpc64; +#[macro_use] +mod mips; +#[macro_use] +mod mips64; + +cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + pub use x86::*; + } else if #[cfg(target_arch = "arm")] { + pub use arm::*; + } else if #[cfg(target_arch = "aarch64")] { + pub use aarch64::*; + } else if #[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] { + pub use riscv::*; + } else if #[cfg(target_arch = "powerpc")] { + pub use powerpc::*; + } else if #[cfg(target_arch = "powerpc64")] { + pub use powerpc64::*; + } else if #[cfg(target_arch = "mips")] { + pub use mips::*; + } else if #[cfg(target_arch = "mips64")] { + pub use mips64::*; + } else { + // Unimplemented architecture: + #[doc(hidden)] + pub(crate) enum Feature { + Null + } + #[doc(hidden)] + pub mod __is_feature_detected {} + + impl Feature { + #[doc(hidden)] + pub(crate) fn from_str(_s: &str) -> Result { Err(()) } + #[doc(hidden)] + pub(crate) fn to_str(self) -> &'static str { "" } + } + } +} diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/powerpc.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/powerpc.rs index 44bd7f337..d135cd95d 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/powerpc.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/powerpc.rs @@ -2,6 +2,7 @@ features! { @TARGET: powerpc; + @CFG: target_arch = "powerpc"; @MACRO_NAME: is_powerpc_feature_detected; @MACRO_ATTRS: /// Checks if `powerpc` feature is enabled. diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/powerpc64.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/powerpc64.rs index 17e4d958b..773afd6ce 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/powerpc64.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/powerpc64.rs @@ -2,6 +2,7 @@ features! { @TARGET: powerpc64; + @CFG: target_arch = "powerpc64"; @MACRO_NAME: is_powerpc64_feature_detected; @MACRO_ATTRS: /// Checks if `powerpc` feature is enabled. diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/riscv.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/riscv.rs new file mode 100644 index 000000000..5ea36e7c1 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/riscv.rs @@ -0,0 +1,206 @@ +//! Run-time feature detection on RISC-V. + +features! { + @TARGET: riscv; + @CFG: any(target_arch = "riscv32", target_arch = "riscv64"); + @MACRO_NAME: is_riscv_feature_detected; + @MACRO_ATTRS: + /// A macro to test at *runtime* whether instruction sets are available on + /// RISC-V platforms. + /// + /// RISC-V standard defined the base sets and the extension sets. + /// The base sets are RV32I, RV64I, RV32E or RV128I. Any RISC-V platform + /// must support one base set and/or multiple extension sets. + /// + /// Any RISC-V standard instruction sets can be in state of either ratified, + /// frozen or draft. The version and status of current standard instruction + /// sets can be checked out from preface section of the [ISA manual]. + /// + /// Platform may define and support their own custom instruction sets with + /// ISA prefix X. These sets are highly platform specific and should be + /// detected with their own platform support crates. + /// + /// # Unprivileged Specification + /// + /// The supported ratified RISC-V instruction sets are as follows: + /// + /// * RV32I: `"rv32i"` + /// * Zifencei: `"zifencei"` + /// * Zihintpause: `"zihintpause"` + /// * RV64I: `"rv64i"` + /// * M: `"m"` + /// * A: `"a"` + /// * Zicsr: `"zicsr"` + /// * Zicntr: `"zicntr"` + /// * Zihpm: `"zihpm"` + /// * F: `"f"` + /// * D: `"d"` + /// * Q: `"q"` + /// * C: `"c"` + /// + /// There's also bases and extensions marked as standard instruction set, + /// but they are in frozen or draft state. These instruction sets are also + /// reserved by this macro and can be detected in the future platforms. + /// + /// Frozen RISC-V instruction sets: + /// + /// * Zfinx: `"zfinx"` + /// * Zdinx: `"zdinx"` + /// * Zhinx: `"zhinx"` + /// * Zhinxmin: `"zhinxmin"` + /// * Ztso: `"ztso"` + /// + /// Draft RISC-V instruction sets: + /// + /// * RV32E: `"rv32e"` + /// * RV128I: `"rv128i"` + /// * Zfh: `"zfh"` + /// * Zfhmin: `"zfhmin"` + /// * B: `"b"` + /// * J: `"j"` + /// * P: `"p"` + /// * V: `"v"` + /// * Zam: `"zam"` + /// + /// Defined by Privileged Specification: + /// + /// * Supervisor: `"s"` + /// * Svnapot: `"svnapot"` + /// * Svpbmt: `"svpbmt"` + /// * Svinval: `"svinval"` + /// * Hypervisor: `"h"` + /// + /// # RISC-V Bit-Manipulation ISA-extensions + /// + /// This document defined the following extensions: + /// + /// * Zba: `"zba"` + /// * Zbb: `"zbb"` + /// * Zbc: `"zbc"` + /// * Zbs: `"zbs"` + /// + /// # RISC-V Cryptography Extensions + /// + /// These extensions are defined in Volume I, Scalar & Entropy Source + /// Instructions: + /// + /// * Zbkb: `"zbkb"` + /// * Zbkc: `"zbkc"` + /// * Zbkx: `"zbkx"` + /// * Zknd: `"zknd"` + /// * Zkne: `"zkne"` + /// * Zknh: `"zknh"` + /// * Zksed: `"zksed"` + /// * Zksh: `"zksh"` + /// * Zkr: `"zkr"` + /// * Zkn: `"zkn"` + /// * Zks: `"zks"` + /// * Zk: `"zk"` + /// * Zkt: `"zkt"` + /// + /// [ISA manual]: https://github.com/riscv/riscv-isa-manual/ + #[unstable(feature = "stdsimd", issue = "27731")] + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] rv32i: "rv32i"; + /// RV32I Base Integer Instruction Set + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zifencei: "zifencei"; + /// "Zifencei" Instruction-Fetch Fence + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zihintpause: "zihintpause"; + /// "Zihintpause" Pause Hint + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] rv64i: "rv64i"; + /// RV64I Base Integer Instruction Set + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] m: "m"; + /// "M" Standard Extension for Integer Multiplication and Division + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] a: "a"; + /// "A" Standard Extension for Atomic Instructions + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zicsr: "zicsr"; + /// "Zicsr", Control and Status Register (CSR) Instructions + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zicntr: "zicntr"; + /// "Zicntr", Standard Extension for Base Counters and Timers + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zihpm: "zihpm"; + /// "Zihpm", Standard Extension for Hardware Performance Counters + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] f: "f"; + /// "F" Standard Extension for Single-Precision Floating-Point + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] d: "d"; + /// "D" Standard Extension for Double-Precision Floating-Point + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] q: "q"; + /// "Q" Standard Extension for Quad-Precision Floating-Point + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] c: "c"; + /// "C" Standard Extension for Compressed Instructions + + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zfinx: "zfinx"; + /// "Zfinx" Standard Extension for Single-Precision Floating-Point in Integer Registers + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zdinx: "zdinx"; + /// "Zdinx" Standard Extension for Double-Precision Floating-Point in Integer Registers + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zhinx: "zhinx"; + /// "Zhinx" Standard Extension for Half-Precision Floating-Point in Integer Registers + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zhinxmin: "zhinxmin"; + /// "Zhinxmin" Standard Extension for Minimal Half-Precision Floating-Point in Integer Registers + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] ztso: "ztso"; + /// "Ztso" Standard Extension for Total Store Ordering + + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] rv32e: "rv32e"; + /// RV32E Base Integer Instruction Set + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] rv128i: "rv128i"; + /// RV128I Base Integer Instruction Set + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zfh: "zfh"; + /// "Zfh" Standard Extension for 16-Bit Half-Precision Floating-Point + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zfhmin: "zfhmin"; + /// "Zfhmin" Standard Extension for Minimal Half-Precision Floating-Point Support + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] b: "b"; + /// "B" Standard Extension for Bit Manipulation + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] j: "j"; + /// "J" Standard Extension for Dynamically Translated Languages + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] p: "p"; + /// "P" Standard Extension for Packed-SIMD Instructions + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] v: "v"; + /// "V" Standard Extension for Vector Operations + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zam: "zam"; + /// "Zam" Standard Extension for Misaligned Atomics + + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] s: "s"; + /// Supervisor-Level ISA + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] svnapot: "svnapot"; + /// "Svnapot" Standard Extension for NAPOT Translation Contiguity + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] svpbmt: "svpbmt"; + /// "Svpbmt" Standard Extension for Page-Based Memory Types + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] svinval: "svinval"; + /// "Svinval" Standard Extension for Fine-Grained Address-Translation Cache Invalidation + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] h: "h"; + /// Hypervisor Extension + + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zba: "zba"; + /// "Zba" Standard Extension for Address Generation Instructions + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zbb: "zbb"; + /// "Zbb" Standard Extension for Basic Bit-Manipulation + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zbc: "zbc"; + /// "Zbc" Standard Extension for Carry-less Multiplication + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zbs: "zbs"; + /// "Zbs" Standard Extension for Single-Bit instructions + + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zbkb: "zbkb"; + /// "Zbkb" Standard Extension for Bitmanip instructions for Cryptography + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zbkc: "zbkc"; + /// "Zbkc" Standard Extension for Carry-less multiply instructions + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zbkx: "zbkx"; + /// "Zbkx" Standard Extension for Crossbar permutation instructions + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zknd: "zknd"; + /// "Zknd" Standard Extension for NIST Suite: AES Decryption + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zkne: "zkne"; + /// "Zkne" Standard Extension for NIST Suite: AES Encryption + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zknh: "zknh"; + /// "Zknh" Standard Extension for NIST Suite: Hash Function Instructions + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zksed: "zksed"; + /// "Zksed" Standard Extension for ShangMi Suite: SM4 Block Cipher Instructions + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zksh: "zksh"; + /// "Zksh" Standard Extension for ShangMi Suite: SM3 Hash Function Instructions + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zkr: "zkr"; + /// "Zkr" Standard Extension for Entropy Source Extension + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zkn: "zkn"; + /// "Zkn" Standard Extension for NIST Algorithm Suite + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zks: "zks"; + /// "Zks" Standard Extension for ShangMi Algorithm Suite + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zk: "zk"; + /// "Zk" Standard Extension for Standard scalar cryptography extension + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] zkt: "zkt"; + /// "Zkt" Standard Extension for Data Independent Execution Latency +} diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/x86.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/x86.rs index 578910054..893e1a887 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/x86.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/arch/x86.rs @@ -2,8 +2,8 @@ //! //! The features are detected using the `detect_features` function below. //! This function uses the CPUID instruction to read the feature flags from the -//! CPU and encodes them in an `usize` where each bit position represents -//! whether a feature is available (bit is set) or unavaiable (bit is cleared). +//! CPU and encodes them in a `usize` where each bit position represents +//! whether a feature is available (bit is set) or unavailable (bit is cleared). //! //! The enum `Feature` is used to map bit positions to feature names, and the //! the `__crate::detect::check_for!` macro is used to map string literals (e.g., @@ -17,6 +17,7 @@ features! { @TARGET: x86; + @CFG: any(target_arch = "x86", target_arch = "x86_64"); @MACRO_NAME: is_x86_feature_detected; @MACRO_ATTRS: /// A macro to test at *runtime* whether a CPU feature is available on @@ -66,6 +67,15 @@ features! { /// * `"avx512ifma"` /// * `"avx512vbmi"` /// * `"avx512vpopcntdq"` + /// * `"avx512vbmi2"` + /// * `"avx512gfni"` + /// * `"avx512vaes"` + /// * `"avx512vpclmulqdq"` + /// * `"avx512vnni"` + /// * `"avx512bitalg"` + /// * `"avx512bf16"` + /// * `"avx512vp2intersect"` + /// * `"f16c"` /// * `"fma"` /// * `"bmi1"` /// * `"bmi2"` @@ -78,6 +88,9 @@ features! { /// * `"xsaveopt"` /// * `"xsaves"` /// * `"xsavec"` + /// * `"cmpxchg16b"` + /// * `"adx"` + /// * `"rtm"` /// /// [docs]: https://software.intel.com/sites/landingpage/IntrinsicsGuide #[stable(feature = "simd_x86", since = "1.27.0")] @@ -86,7 +99,7 @@ features! { /// AES (Advanced Encryption Standard New Instructions AES-NI) @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] pclmulqdq: "pclmulqdq"; /// CLMUL (Carry-less Multiplication) - @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] rdrand: "rdrand"; + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] rdrand: "rdrand"; /// RDRAND @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] rdseed: "rdseed"; /// RDSEED @@ -114,10 +127,6 @@ features! { /// AVX (Advanced Vector Extensions) @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx2: "avx2"; /// AVX2 (Advanced Vector Extensions 2) - // Detection for the AVX-512 features below was accidentally stabilized in - // Rust 1.27.0, even though the corresponding intrinsics are still unstable - // or unimplemeted. There are stable callers who rely on detection support, - // e.g. to call AVX-512 C code via FFI. @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512f: "avx512f" ; /// AVX-512 F (Foundation) @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512cd: "avx512cd" ; @@ -132,30 +141,30 @@ features! { /// AVX-512 DQ (Doubleword and Quadword) @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512vl: "avx512vl"; /// AVX-512 VL (Vector Length Extensions) - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] avx512ifma: "avx512ifma"; + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512ifma: "avx512ifma"; /// AVX-512 IFMA (Integer Fused Multiply Add) - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] avx512vbmi: "avx512vbmi"; + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512vbmi: "avx512vbmi"; /// AVX-512 VBMI (Vector Byte Manipulation Instructions) - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] avx512vpopcntdq: "avx512vpopcntdq"; + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512vpopcntdq: "avx512vpopcntdq"; /// AVX-512 VPOPCNTDQ (Vector Population Count Doubleword and /// Quadword) - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] avx512vbmi2: "avx512vbmi2"; + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512vbmi2: "avx512vbmi2"; /// AVX-512 VBMI2 (Additional byte, word, dword and qword capabilities) - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] avx512gfni: "avx512gfni"; + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512gfni: "avx512gfni"; /// AVX-512 GFNI (Galois Field New Instruction) - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] avx512vaes: "avx512vaes"; + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512vaes: "avx512vaes"; /// AVX-512 VAES (Vector AES instruction) - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] avx512vpclmulqdq: "avx512vpclmulqdq"; + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512vpclmulqdq: "avx512vpclmulqdq"; /// AVX-512 VPCLMULQDQ (Vector PCLMULQDQ instructions) - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] avx512vnni: "avx512vnni"; + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512vnni: "avx512vnni"; /// AVX-512 VNNI (Vector Neural Network Instructions) - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] avx512bitalg: "avx512bitalg"; - /// AVX-512 BITALG (Support for VPOPCNT[B,W] and VPSHUFBITQMB) - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] avx512bf16: "avx512bf16"; + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512bitalg: "avx512bitalg"; + /// AVX-512 BITALG (Support for VPOPCNT\[B,W\] and VPSHUFBITQMB) + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512bf16: "avx512bf16"; /// AVX-512 BF16 (BFLOAT16 instructions) - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] avx512vp2intersect: "avx512vp2intersect"; + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] avx512vp2intersect: "avx512vp2intersect"; /// AVX-512 P2INTERSECT - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] f16c: "f16c"; + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] f16c: "f16c"; /// F16C (Conversions between IEEE-754 `binary16` and `binary32` formats) @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] fma: "fma"; /// FMA (Fused Multiply Add) @@ -170,7 +179,7 @@ features! { @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] popcnt: "popcnt"; /// POPCNT (Population Count) @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] fxsr: "fxsr"; - /// FXSR (Floating-point context fast save and restor) + /// FXSR (Floating-point context fast save and restore) @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] xsave: "xsave"; /// XSAVE (Save Processor Extended States) @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] xsaveopt: "xsaveopt"; @@ -179,10 +188,10 @@ features! { /// XSAVES (Save Processor Extended States Supervisor) @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] xsavec: "xsavec"; /// XSAVEC (Save Processor Extended States Compacted) - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] cmpxchg16b: "cmpxchg16b"; + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] cmpxchg16b: "cmpxchg16b"; /// CMPXCH16B (16-byte compare-and-swap instruction) @FEATURE: #[stable(feature = "simd_x86_adx", since = "1.33.0")] adx: "adx"; /// ADX, Intel ADX (Multi-Precision Add-Carry Instruction Extensions) - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] rtm: "rtm"; + @FEATURE: #[stable(feature = "simd_x86", since = "1.27.0")] rtm: "rtm"; /// RTM, Intel (Restricted Transactional Memory) } diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/bit.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/bit.rs index 578f0b16b..6f06c5523 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/bit.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/bit.rs @@ -4,6 +4,6 @@ #[allow(dead_code)] #[inline] pub(crate) fn test(x: usize, bit: u32) -> bool { - debug_assert!(bit < 32, "bit index out-of-bounds"); + debug_assert!(bit < usize::BITS, "bit index out-of-bounds"); x & (1 << bit) != 0 } diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/cache.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/cache.rs index d421037f6..d01a5ea24 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/cache.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/cache.rs @@ -3,9 +3,9 @@ #![allow(dead_code)] // not used on all platforms -use crate::sync::atomic::Ordering; +use core::sync::atomic::Ordering; -use crate::sync::atomic::AtomicUsize; +use core::sync::atomic::AtomicUsize; /// Sets the `bit` of `x`. #[inline] @@ -43,7 +43,6 @@ impl Default for Initializer { // the one fitting our cache. impl Initializer { /// Tests the `bit` of the cache. - #[allow(dead_code)] #[inline] pub(crate) fn test(self, bit: u32) -> bool { debug_assert!( @@ -80,60 +79,72 @@ impl Initializer { // Note: on x64, we only use the first slot static CACHE: [Cache; 2] = [Cache::uninitialized(), Cache::uninitialized()]; -/// Feature cache with capacity for `usize::max_value() - 1` features. +/// Feature cache with capacity for `size_of::() * 8 - 1` features. /// -/// Note: the last feature bit is used to represent an -/// uninitialized cache. +/// Note: 0 is used to represent an uninitialized cache, and (at least) the most +/// significant bit is set on any cache which has been initialized. /// -/// Note: we can use `Relaxed` atomic operations, because we are only interested -/// in the effects of operations on a single memory location. That is, we only -/// need "modification order", and not the full-blown "happens before". However, -/// we use `SeqCst` just to be on the safe side. +/// Note: we use `Relaxed` atomic operations, because we are only interested in +/// the effects of operations on a single memory location. That is, we only need +/// "modification order", and not the full-blown "happens before". struct Cache(AtomicUsize); impl Cache { const CAPACITY: u32 = (core::mem::size_of::() * 8 - 1) as u32; const MASK: usize = (1 << Cache::CAPACITY) - 1; + const INITIALIZED_BIT: usize = 1usize << Cache::CAPACITY; /// Creates an uninitialized cache. #[allow(clippy::declare_interior_mutable_const)] const fn uninitialized() -> Self { - Cache(AtomicUsize::new(usize::max_value())) - } - /// Is the cache uninitialized? - #[inline] - pub(crate) fn is_uninitialized(&self) -> bool { - self.0.load(Ordering::SeqCst) == usize::max_value() + Cache(AtomicUsize::new(0)) } - /// Is the `bit` in the cache set? + /// Is the `bit` in the cache set? Returns `None` if the cache has not been initialized. #[inline] - pub(crate) fn test(&self, bit: u32) -> bool { - test_bit(self.0.load(Ordering::SeqCst) as u64, bit) + pub(crate) fn test(&self, bit: u32) -> Option { + let cached = self.0.load(Ordering::Relaxed); + if cached == 0 { + None + } else { + Some(test_bit(cached as u64, bit)) + } } /// Initializes the cache. #[inline] - fn initialize(&self, value: usize) { - self.0.store(value, Ordering::SeqCst); + fn initialize(&self, value: usize) -> usize { + debug_assert_eq!((value & !Cache::MASK), 0); + self.0 + .store(value | Cache::INITIALIZED_BIT, Ordering::Relaxed); + value } } cfg_if::cfg_if! { if #[cfg(feature = "std_detect_env_override")] { - #[inline(never)] - fn initialize(mut value: Initializer) { - if let Ok(disable) = crate::env::var("RUST_STD_DETECT_UNSTABLE") { - for v in disable.split(" ") { - let _ = super::Feature::from_str(v).map(|v| value.unset(v as u32)); + #[inline] + fn initialize(mut value: Initializer) -> Initializer { + let env = unsafe { + libc::getenv(b"RUST_STD_DETECT_UNSTABLE\0".as_ptr() as *const libc::c_char) + }; + if !env.is_null() { + let len = unsafe { libc::strlen(env) }; + let env = unsafe { core::slice::from_raw_parts(env as *const u8, len) }; + if let Ok(disable) = core::str::from_utf8(env) { + for v in disable.split(" ") { + let _ = super::Feature::from_str(v).map(|v| value.unset(v as u32)); + } } } do_initialize(value); + value } } else { #[inline] - fn initialize(value: Initializer) { + fn initialize(value: Initializer) -> Initializer { do_initialize(value); + value } } } @@ -144,8 +155,22 @@ fn do_initialize(value: Initializer) { CACHE[1].initialize((value.0 >> Cache::CAPACITY) as usize & Cache::MASK); } +// We only have to detect features once, and it's fairly costly, so hint to LLVM +// that it should assume that cache hits are more common than misses (which is +// the point of caching). It's possibly unfortunate that this function needs to +// reach across modules like this to call `os::detect_features`, but it produces +// the best code out of several attempted variants. +// +// The `Initializer` that the cache was initialized with is returned, so that +// the caller can call `test()` on it without having to load the value from the +// cache again. +#[cold] +fn detect_and_initialize() -> Initializer { + initialize(super::os::detect_features()) +} + /// Tests the `bit` of the storage. If the storage has not been initialized, -/// initializes it with the result of `f()`. +/// initializes it with the result of `os::detect_features()`. /// /// On its first invocation, it detects the CPU features and caches them in the /// `CACHE` global variable as an `AtomicU64`. @@ -157,18 +182,13 @@ fn do_initialize(value: Initializer) { /// variable `RUST_STD_DETECT_UNSTABLE` and uses its its content to disable /// Features that would had been otherwise detected. #[inline] -pub(crate) fn test(bit: u32, f: F) -> bool -where - F: FnOnce() -> Initializer, -{ - let (bit, idx) = if bit < Cache::CAPACITY { +pub(crate) fn test(bit: u32) -> bool { + let (relative_bit, idx) = if bit < Cache::CAPACITY { (bit, 0) } else { (bit - Cache::CAPACITY, 1) }; - - if CACHE[idx].is_uninitialized() { - initialize(f()) - } - CACHE[idx].test(bit) + CACHE[idx] + .test(relative_bit) + .unwrap_or_else(|| detect_and_initialize().test(bit)) } diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/error_macros.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/error_macros.rs deleted file mode 100644 index 6769757ed..000000000 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/error_macros.rs +++ /dev/null @@ -1,150 +0,0 @@ -//! The `is_{target_arch}_feature_detected!` macro are only available on their -//! architecture. These macros provide a better error messages when the user -//! attempts to call them in a different architecture. - -/// Prevents compilation if `is_x86_feature_detected` is used somewhere -/// else than `x86` and `x86_64` targets. -#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] -#[macro_export] -#[unstable(feature = "stdsimd", issue = "27731")] -macro_rules! is_x86_feature_detected { - ($t: tt) => { - compile_error!( - r#" - is_x86_feature_detected can only be used on x86 and x86_64 targets. - You can prevent it from being used in other architectures by - guarding it behind a cfg(target_arch) as follows: - - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { - if is_x86_feature_detected(...) { ... } - } - "# - ) - }; -} - -/// Prevents compilation if `is_arm_feature_detected` is used somewhere else -/// than `ARM` targets. -#[cfg(not(target_arch = "arm"))] -#[macro_export] -#[unstable(feature = "stdsimd", issue = "27731")] -macro_rules! is_arm_feature_detected { - ($t:tt) => { - compile_error!( - r#" - is_arm_feature_detected can only be used on ARM targets. - You can prevent it from being used in other architectures by - guarding it behind a cfg(target_arch) as follows: - - #[cfg(target_arch = "arm")] { - if is_arm_feature_detected(...) { ... } - } - "# - ) - }; -} - -/// Prevents compilation if `is_aarch64_feature_detected` is used somewhere else -/// than `aarch64` targets. -#[cfg(not(target_arch = "aarch64"))] -#[macro_export] -#[unstable(feature = "stdsimd", issue = "27731")] -macro_rules! is_aarch64_feature_detected { - ($t: tt) => { - compile_error!( - r#" - is_aarch64_feature_detected can only be used on AArch64 targets. - You can prevent it from being used in other architectures by - guarding it behind a cfg(target_arch) as follows: - - #[cfg(target_arch = "aarch64")] { - if is_aarch64_feature_detected(...) { ... } - } - "# - ) - }; -} - -/// Prevents compilation if `is_powerpc_feature_detected` is used somewhere else -/// than `PowerPC` targets. -#[cfg(not(target_arch = "powerpc"))] -#[macro_export] -#[unstable(feature = "stdsimd", issue = "27731")] -macro_rules! is_powerpc_feature_detected { - ($t:tt) => { - compile_error!( - r#" -is_powerpc_feature_detected can only be used on PowerPC targets. -You can prevent it from being used in other architectures by -guarding it behind a cfg(target_arch) as follows: - - #[cfg(target_arch = "powerpc")] { - if is_powerpc_feature_detected(...) { ... } - } -"# - ) - }; -} - -/// Prevents compilation if `is_powerpc64_feature_detected` is used somewhere -/// else than `PowerPC64` targets. -#[cfg(not(target_arch = "powerpc64"))] -#[macro_export] -#[unstable(feature = "stdsimd", issue = "27731")] -macro_rules! is_powerpc64_feature_detected { - ($t:tt) => { - compile_error!( - r#" -is_powerpc64_feature_detected can only be used on PowerPC64 targets. -You can prevent it from being used in other architectures by -guarding it behind a cfg(target_arch) as follows: - - #[cfg(target_arch = "powerpc64")] { - if is_powerpc64_feature_detected(...) { ... } - } -"# - ) - }; -} - -/// Prevents compilation if `is_mips_feature_detected` is used somewhere else -/// than `MIPS` targets. -#[cfg(not(target_arch = "mips"))] -#[macro_export] -#[unstable(feature = "stdsimd", issue = "27731")] -macro_rules! is_mips_feature_detected { - ($t:tt) => { - compile_error!( - r#" - is_mips_feature_detected can only be used on MIPS targets. - You can prevent it from being used in other architectures by - guarding it behind a cfg(target_arch) as follows: - - #[cfg(target_arch = "mips")] { - if is_mips_feature_detected(...) { ... } - } - "# - ) - }; -} - -/// Prevents compilation if `is_mips64_feature_detected` is used somewhere else -/// than `MIPS64` targets. -#[cfg(not(target_arch = "mips64"))] -#[macro_export] -#[unstable(feature = "stdsimd", issue = "27731")] -macro_rules! is_mips64_feature_detected { - ($t:tt) => { - compile_error!( - r#" - is_mips64_feature_detected can only be used on MIPS64 targets. - You can prevent it from being used in other architectures by - guarding it behind a cfg(target_arch) as follows: - - #[cfg(target_arch = "mips64")] { - if is_mips64_feature_detected(...) { ... } - } - "# - ) - }; -} diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/macros.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/macros.rs index b9dbf9184..a467f9db6 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/macros.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/macros.rs @@ -1,37 +1,98 @@ +#[macro_export] +#[allow_internal_unstable(stdsimd)] +macro_rules! detect_feature { + ($feature:tt, $feature_lit:tt) => { + $crate::detect_feature!($feature, $feature_lit : $feature_lit) + }; + ($feature:tt, $feature_lit:tt : $($target_feature_lit:tt),*) => { + $(cfg!(target_feature = $target_feature_lit) ||)* + $crate::detect::__is_feature_detected::$feature() + }; +} + #[allow(unused)] macro_rules! features { ( @TARGET: $target:ident; + @CFG: $cfg:meta; @MACRO_NAME: $macro_name:ident; @MACRO_ATTRS: $(#[$macro_attrs:meta])* $(@BIND_FEATURE_NAME: $bind_feature:tt; $feature_impl:tt; )* $(@NO_RUNTIME_DETECTION: $nort_feature:tt; )* - $(@FEATURE: #[$stability_attr:meta] $feature:ident: $feature_lit:tt; $(#[$feature_comment:meta])*)* + $(@FEATURE: #[$stability_attr:meta] $feature:ident: $feature_lit:tt; + $(implied by target_features: [$($target_feature_lit:tt),*];)? + $(#[$feature_comment:meta])*)* ) => { #[macro_export] $(#[$macro_attrs])* - #[allow_internal_unstable(stdsimd_internal)] + #[allow_internal_unstable(stdsimd_internal, stdsimd)] + #[cfg($cfg)] + #[doc(cfg($cfg))] + macro_rules! $macro_name { + $( + ($feature_lit) => { + $crate::detect_feature!($feature, $feature_lit $(: $($target_feature_lit),*)?) + }; + )* + $( + ($bind_feature) => { $crate::$macro_name!($feature_impl) }; + )* + $( + ($nort_feature) => { + compile_error!( + concat!( + stringify!($nort_feature), + " feature cannot be detected at run-time" + ) + ) + }; + )* + ($t:tt,) => { + $crate::$macro_name!($t); + }; + ($t:tt) => { + compile_error!( + concat!( + concat!("unknown ", stringify!($target)), + concat!(" target feature: ", $t) + ) + ) + }; + } + + $(#[$macro_attrs])* + #[macro_export] + #[cfg(not($cfg))] + #[doc(cfg($cfg))] macro_rules! $macro_name { $( ($feature_lit) => { - $crate::detect::__is_feature_detected::$feature() + compile_error!( + concat!( + r#"This macro cannot be used on the current target. + You can prevent it from being used in other architectures by + guarding it behind a cfg("#, + stringify!($cfg), + ")." + ) + ) }; )* $( - ($bind_feature) => { $macro_name!($feature_impl); }; + ($bind_feature) => { $crate::$macro_name!($feature_impl) }; )* $( ($nort_feature) => { compile_error!( concat!( - stringify!(nort_feature), + stringify!($nort_feature), " feature cannot be detected at run-time" ) ) }; )* ($t:tt,) => { - $macro_name!($t); + $crate::$macro_name!($t); }; ($t:tt) => { compile_error!( @@ -52,6 +113,7 @@ macro_rules! features { #[derive(Copy, Clone)] #[repr(u8)] #[unstable(feature = "stdsimd_internal", issue = "none")] + #[cfg($cfg)] pub(crate) enum Feature { $( $(#[$feature_comment])* @@ -62,6 +124,7 @@ macro_rules! features { _last } + #[cfg($cfg)] impl Feature { pub(crate) fn to_str(self) -> &'static str { match self { @@ -85,6 +148,7 @@ macro_rules! features { /// PLEASE: do not use this, it is an implementation detail subject /// to change. #[doc(hidden)] + #[cfg($cfg)] pub mod __is_feature_detected { $( @@ -94,8 +158,7 @@ macro_rules! features { #[doc(hidden)] #[$stability_attr] pub fn $feature() -> bool { - cfg!(target_feature = $feature_lit) || - $crate::detect::check_for($crate::detect::Feature::$feature) + $crate::detect::check_for($crate::detect::Feature::$feature) } )* } diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/mod.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/mod.rs index e8cce6e5d..2bca84ca1 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/mod.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/mod.rs @@ -19,60 +19,10 @@ use cfg_if::cfg_if; -#[macro_use] -mod error_macros; - #[macro_use] mod macros; -cfg_if! { - if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { - #[path = "arch/x86.rs"] - #[macro_use] - mod arch; - } else if #[cfg(target_arch = "arm")] { - #[path = "arch/arm.rs"] - #[macro_use] - mod arch; - } else if #[cfg(target_arch = "aarch64")] { - #[path = "arch/aarch64.rs"] - #[macro_use] - mod arch; - } else if #[cfg(target_arch = "powerpc")] { - #[path = "arch/powerpc.rs"] - #[macro_use] - mod arch; - } else if #[cfg(target_arch = "powerpc64")] { - #[path = "arch/powerpc64.rs"] - #[macro_use] - mod arch; - } else if #[cfg(target_arch = "mips")] { - #[path = "arch/mips.rs"] - #[macro_use] - mod arch; - } else if #[cfg(target_arch = "mips64")] { - #[path = "arch/mips64.rs"] - #[macro_use] - mod arch; - } else { - // Unimplemented architecture: - mod arch { - #[doc(hidden)] - pub(crate) enum Feature { - Null - } - #[doc(hidden)] - pub mod __is_feature_detected {} - - impl Feature { - #[doc(hidden)] - pub(crate) fn from_str(_s: &str) -> Result { Err(()) } - #[doc(hidden)] - pub(crate) fn to_str(self) -> &'static str { "" } - } - } - } -} +mod arch; // This module needs to be public because the `is_{arch}_feature_detected!` // macros expand calls to items within it in user crates. @@ -97,10 +47,10 @@ cfg_if! { // On x86/x86_64 no OS specific functionality is required. #[path = "os/x86.rs"] mod os; - } else if #[cfg(all(target_os = "linux", feature = "use_std"))] { + } else if #[cfg(all(target_os = "linux", feature = "libc"))] { #[path = "os/linux/mod.rs"] mod os; - } else if #[cfg(target_os = "freebsd")] { + } else if #[cfg(all(target_os = "freebsd", feature = "libc"))] { #[cfg(target_arch = "aarch64")] #[path = "os/aarch64.rs"] mod aarch64; @@ -117,8 +67,9 @@ cfg_if! { /// Performs run-time feature detection. #[inline] +#[allow(dead_code)] fn check_for(x: Feature) -> bool { - cache::test(x as u32, self::os::detect_features) + cache::test(x as u32) } /// Returns an `Iterator` where @@ -132,13 +83,16 @@ pub fn features() -> impl Iterator { target_arch = "x86_64", target_arch = "arm", target_arch = "aarch64", + target_arch = "riscv32", + target_arch = "riscv64", target_arch = "powerpc", target_arch = "powerpc64", target_arch = "mips", target_arch = "mips64", ))] { (0_u8..Feature::_last as u8).map(|discriminant: u8| { - let f: Feature = unsafe { crate::mem::transmute(discriminant) }; + #[allow(bindings_with_variant_name)] // RISC-V has Feature::f + let f: Feature = unsafe { core::mem::transmute(discriminant) }; let name: &'static str = f.to_str(); let enabled: bool = check_for(f); (name, enabled) diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/aarch64.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/aarch64.rs index 9adc938a2..e0e62ee33 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/aarch64.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/aarch64.rs @@ -17,6 +17,7 @@ //! - [Linux documentation](https://www.kernel.org/doc/Documentation/arm64/cpu-feature-registers.txt) use crate::detect::{cache, Feature}; +use core::arch::asm; /// Try to read the features from the system registers. /// @@ -34,23 +35,26 @@ pub(crate) fn detect_features() -> cache::Initializer { // ID_AA64ISAR0_EL1 - Instruction Set Attribute Register 0 let aa64isar0: u64; unsafe { - asm!("mrs $0, ID_AA64ISAR0_EL1" : "=r"(aa64isar0)); + asm!( + "mrs {}, ID_AA64ISAR0_EL1", + out(reg) aa64isar0, + options(pure, nomem, preserves_flags, nostack) + ); } - let aes = bits_shift(aa64isar0, 7, 4) >= 1; - let pmull = bits_shift(aa64isar0, 7, 4) >= 2; - let sha1 = bits_shift(aa64isar0, 11, 8) >= 1; - let sha2 = bits_shift(aa64isar0, 15, 12) >= 1; - enable_feature(Feature::pmull, pmull); - // Crypto is specified as AES + PMULL + SHA1 + SHA2 per LLVM/hosts.cpp - enable_feature(Feature::crypto, aes && pmull && sha1 && sha2); + enable_feature(Feature::pmull, bits_shift(aa64isar0, 7, 4) >= 2); + enable_feature(Feature::tme, bits_shift(aa64isar0, 27, 24) == 1); enable_feature(Feature::lse, bits_shift(aa64isar0, 23, 20) >= 1); enable_feature(Feature::crc, bits_shift(aa64isar0, 19, 16) >= 1); // ID_AA64PFR0_EL1 - Processor Feature Register 0 let aa64pfr0: u64; unsafe { - asm!("mrs $0, ID_AA64PFR0_EL1" : "=r"(aa64pfr0)); + asm!( + "mrs {}, ID_AA64PFR0_EL1", + out(reg) aa64pfr0, + options(pure, nomem, preserves_flags, nostack) + ); } let fp = bits_shift(aa64pfr0, 19, 16) < 0xF; @@ -63,6 +67,10 @@ pub(crate) fn detect_features() -> cache::Initializer { // supported, it also requires half-float support: enable_feature(Feature::asimd, fp && asimd && (!fphp | asimdhp)); // SIMD extensions require SIMD support: + enable_feature(Feature::aes, asimd && bits_shift(aa64isar0, 7, 4) >= 1); + let sha1 = bits_shift(aa64isar0, 11, 8) >= 1; + let sha2 = bits_shift(aa64isar0, 15, 12) >= 1; + enable_feature(Feature::sha2, asimd && sha1 && sha2); enable_feature(Feature::rdm, asimd && bits_shift(aa64isar0, 31, 28) >= 1); enable_feature( Feature::dotprod, @@ -73,10 +81,18 @@ pub(crate) fn detect_features() -> cache::Initializer { // ID_AA64ISAR1_EL1 - Instruction Set Attribute Register 1 let aa64isar1: u64; unsafe { - asm!("mrs $0, ID_AA64ISAR1_EL1" : "=r"(aa64isar1)); + asm!( + "mrs {}, ID_AA64ISAR1_EL1", + out(reg) aa64isar1, + options(pure, nomem, preserves_flags, nostack) + ); } + // Check for either APA or API field + enable_feature(Feature::paca, bits_shift(aa64isar1, 11, 4) >= 1); enable_feature(Feature::rcpc, bits_shift(aa64isar1, 23, 20) >= 1); + // Check for either GPA or GPI field + enable_feature(Feature::pacg, bits_shift(aa64isar1, 31, 24) >= 1); } value diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/freebsd/aarch64.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/freebsd/aarch64.rs index e5df9ba4c..7d972b373 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/freebsd/aarch64.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/freebsd/aarch64.rs @@ -1,6 +1,6 @@ //! Run-time feature detection for Aarch64 on FreeBSD. -pub use super::super::aarch64::detect_features; +pub(crate) use super::super::aarch64::detect_features; #[cfg(test)] mod tests { @@ -12,10 +12,10 @@ mod tests { println!("fp16: {:?}", is_aarch64_feature_detected!("fp16")); println!("sve: {:?}", is_aarch64_feature_detected!("sve")); println!("crc: {:?}", is_aarch64_feature_detected!("crc")); - println!("crypto: {:?}", is_aarch64_feature_detected!("crypto")); println!("lse: {:?}", is_aarch64_feature_detected!("lse")); println!("rdm: {:?}", is_aarch64_feature_detected!("rdm")); println!("rcpc: {:?}", is_aarch64_feature_detected!("rcpc")); println!("dotprod: {:?}", is_aarch64_feature_detected!("dotprod")); + println!("tme: {:?}", is_aarch64_feature_detected!("tme")); } } diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/freebsd/auxvec.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/freebsd/auxvec.rs index c595ec459..29fcc8cb0 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/freebsd/auxvec.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/freebsd/auxvec.rs @@ -1,5 +1,13 @@ //! Parses ELF auxiliary vectors. -#![cfg_attr(any(target_arch = "arm", target_arch = "powerpc64"), allow(dead_code))] +#![cfg_attr( + any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc64", + target_arch = "riscv64" + ), + allow(dead_code) +)] /// Key to access the CPU Hardware capabilities bitfield. pub(crate) const AT_HWCAP: usize = 25; @@ -42,7 +50,7 @@ pub(crate) fn auxv() -> Result { /// Tries to read the `key` from the auxiliary vector. fn archauxv(key: usize) -> Result { - use crate::mem; + use core::mem; #[derive(Copy, Clone)] #[repr(C)] diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/freebsd/mod.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/freebsd/mod.rs index 4321bce74..ade7fb626 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/freebsd/mod.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/freebsd/mod.rs @@ -5,18 +5,18 @@ mod auxvec; cfg_if::cfg_if! { if #[cfg(target_arch = "aarch64")] { mod aarch64; - pub use self::aarch64::check_for; + pub(crate) use self::aarch64::detect_features; } else if #[cfg(target_arch = "arm")] { mod arm; - pub use self::arm::check_for; + pub(crate) use self::arm::detect_features; } else if #[cfg(target_arch = "powerpc64")] { mod powerpc; - pub use self::powerpc::check_for; + pub(crate) use self::powerpc::detect_features; } else { - use crate::arch::detect::Feature; + use crate::detect::cache; /// Performs run-time feature detection. - pub fn check_for(_x: Feature) -> bool { - false + pub(crate) fn detect_features() -> cache::Initializer { + cache::Initializer::default() } } } diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/aarch64.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/aarch64.rs index b1b68f763..6c79ba86d 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/aarch64.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/aarch64.rs @@ -1,6 +1,6 @@ //! Run-time feature detection for Aarch64 on Linux. -use super::{auxvec, cpuinfo}; +use super::auxvec; use crate::detect::{bit, cache, Feature}; /// Try to read the features from the auxiliary vector, and if that fails, try @@ -10,7 +10,8 @@ pub(crate) fn detect_features() -> cache::Initializer { let hwcap: AtHwcap = auxv.into(); return hwcap.cache(); } - if let Ok(c) = cpuinfo::CpuInfo::new() { + #[cfg(feature = "std_detect_file_io")] + if let Ok(c) = super::cpuinfo::CpuInfo::new() { let hwcap: AtHwcap = c.into(); return hwcap.cache(); } @@ -19,31 +20,65 @@ pub(crate) fn detect_features() -> cache::Initializer { /// These values are part of the platform-specific [asm/hwcap.h][hwcap] . /// +/// The names match those used for cpuinfo. +/// /// [hwcap]: https://github.com/torvalds/linux/blob/master/arch/arm64/include/uapi/asm/hwcap.h +#[derive(Debug, Default, PartialEq)] struct AtHwcap { - fp: bool, // 0 - asimd: bool, // 1 - // evtstrm: bool, // 2 - aes: bool, // 3 - pmull: bool, // 4 - sha1: bool, // 5 - sha2: bool, // 6 - crc32: bool, // 7 - atomics: bool, // 8 - fphp: bool, // 9 - asimdhp: bool, // 10 - // cpuid: bool, // 11 - asimdrdm: bool, // 12 - // jscvt: bool, // 13 - // fcma: bool, // 14 - lrcpc: bool, // 15 - // dcpop: bool, // 16 - // sha3: bool, // 17 - // sm3: bool, // 18 - // sm4: bool, // 19 - asimddp: bool, // 20 - // sha512: bool, // 21 - sve: bool, // 22 + // AT_HWCAP + fp: bool, + asimd: bool, + // evtstrm: No LLVM support. + aes: bool, + pmull: bool, + sha1: bool, + sha2: bool, + crc32: bool, + atomics: bool, + fphp: bool, + asimdhp: bool, + // cpuid: No LLVM support. + asimdrdm: bool, + jscvt: bool, + fcma: bool, + lrcpc: bool, + dcpop: bool, + sha3: bool, + sm3: bool, + sm4: bool, + asimddp: bool, + sha512: bool, + sve: bool, + fhm: bool, + dit: bool, + uscat: bool, + ilrcpc: bool, + flagm: bool, + ssbs: bool, + sb: bool, + paca: bool, + pacg: bool, + + // AT_HWCAP2 + dcpodp: bool, + sve2: bool, + sveaes: bool, + // svepmull: No LLVM support. + svebitperm: bool, + svesha3: bool, + svesm4: bool, + // flagm2: No LLVM support. + frint: bool, + // svei8mm: See i8mm feature. + svef32mm: bool, + svef64mm: bool, + // svebf16: See bf16 feature. + i8mm: bool, + bf16: bool, + // dgh: No LLVM support. + rng: bool, + bti: bool, + mte: bool, } impl From for AtHwcap { @@ -63,23 +98,52 @@ impl From for AtHwcap { asimdhp: bit::test(auxv.hwcap, 10), // cpuid: bit::test(auxv.hwcap, 11), asimdrdm: bit::test(auxv.hwcap, 12), - // jscvt: bit::test(auxv.hwcap, 13), - // fcma: bit::test(auxv.hwcap, 14), + jscvt: bit::test(auxv.hwcap, 13), + fcma: bit::test(auxv.hwcap, 14), lrcpc: bit::test(auxv.hwcap, 15), - // dcpop: bit::test(auxv.hwcap, 16), - // sha3: bit::test(auxv.hwcap, 17), - // sm3: bit::test(auxv.hwcap, 18), - // sm4: bit::test(auxv.hwcap, 19), + dcpop: bit::test(auxv.hwcap, 16), + sha3: bit::test(auxv.hwcap, 17), + sm3: bit::test(auxv.hwcap, 18), + sm4: bit::test(auxv.hwcap, 19), asimddp: bit::test(auxv.hwcap, 20), - // sha512: bit::test(auxv.hwcap, 21), + sha512: bit::test(auxv.hwcap, 21), sve: bit::test(auxv.hwcap, 22), + fhm: bit::test(auxv.hwcap, 23), + dit: bit::test(auxv.hwcap, 24), + uscat: bit::test(auxv.hwcap, 25), + ilrcpc: bit::test(auxv.hwcap, 26), + flagm: bit::test(auxv.hwcap, 27), + ssbs: bit::test(auxv.hwcap, 28), + sb: bit::test(auxv.hwcap, 29), + paca: bit::test(auxv.hwcap, 30), + pacg: bit::test(auxv.hwcap, 31), + dcpodp: bit::test(auxv.hwcap2, 0), + sve2: bit::test(auxv.hwcap2, 1), + sveaes: bit::test(auxv.hwcap2, 2), + // svepmull: bit::test(auxv.hwcap2, 3), + svebitperm: bit::test(auxv.hwcap2, 4), + svesha3: bit::test(auxv.hwcap2, 5), + svesm4: bit::test(auxv.hwcap2, 6), + // flagm2: bit::test(auxv.hwcap2, 7), + frint: bit::test(auxv.hwcap2, 8), + // svei8mm: bit::test(auxv.hwcap2, 9), + svef32mm: bit::test(auxv.hwcap2, 10), + svef64mm: bit::test(auxv.hwcap2, 11), + // svebf16: bit::test(auxv.hwcap2, 12), + i8mm: bit::test(auxv.hwcap2, 13), + bf16: bit::test(auxv.hwcap2, 14), + // dgh: bit::test(auxv.hwcap2, 15), + rng: bit::test(auxv.hwcap2, 16), + bti: bit::test(auxv.hwcap2, 17), + mte: bit::test(auxv.hwcap2, 18), } } } -impl From for AtHwcap { +#[cfg(feature = "std_detect_file_io")] +impl From for AtHwcap { /// Reads AtHwcap from /proc/cpuinfo . - fn from(c: cpuinfo::CpuInfo) -> Self { + fn from(c: super::cpuinfo::CpuInfo) -> Self { let f = &c.field("Features"); AtHwcap { // 64-bit names. FIXME: In 32-bit compatibility mode /proc/cpuinfo will @@ -98,16 +162,44 @@ impl From for AtHwcap { asimdhp: f.has("asimdhp"), // cpuid: f.has("cpuid"), asimdrdm: f.has("asimdrdm"), - // jscvt: f.has("jscvt"), - // fcma: f.has("fcma"), + jscvt: f.has("jscvt"), + fcma: f.has("fcma"), lrcpc: f.has("lrcpc"), - // dcpop: f.has("dcpop"), - // sha3: f.has("sha3"), - // sm3: f.has("sm3"), - // sm4: f.has("sm4"), + dcpop: f.has("dcpop"), + sha3: f.has("sha3"), + sm3: f.has("sm3"), + sm4: f.has("sm4"), asimddp: f.has("asimddp"), - // sha512: f.has("sha512"), + sha512: f.has("sha512"), sve: f.has("sve"), + fhm: f.has("asimdfhm"), + dit: f.has("dit"), + uscat: f.has("uscat"), + ilrcpc: f.has("ilrcpc"), + flagm: f.has("flagm"), + ssbs: f.has("ssbs"), + sb: f.has("sb"), + paca: f.has("paca"), + pacg: f.has("pacg"), + dcpodp: f.has("dcpodp"), + sve2: f.has("sve2"), + sveaes: f.has("sveaes"), + // svepmull: f.has("svepmull"), + svebitperm: f.has("svebitperm"), + svesha3: f.has("svesha3"), + svesm4: f.has("svesm4"), + // flagm2: f.has("flagm2"), + frint: f.has("frint"), + // svei8mm: f.has("svei8mm"), + svef32mm: f.has("svef32mm"), + svef64mm: f.has("svef64mm"), + // svebf16: f.has("svebf16"), + i8mm: f.has("i8mm"), + bf16: f.has("bf16"), + // dgh: f.has("dgh"), + rng: f.has("rng"), + bti: f.has("bti"), + mte: f.has("mte"), } } } @@ -115,8 +207,8 @@ impl From for AtHwcap { impl AtHwcap { /// Initializes the cache from the feature -bits. /// - /// The features are enabled approximately like in LLVM host feature detection: - /// https://github.com/llvm-mirror/llvm/blob/master/lib/Support/Host.cpp#L1273 + /// The feature dependencies here come directly from LLVM's feature definintions: + /// https://github.com/llvm/llvm-project/blob/main/llvm/lib/Target/AArch64/AArch64.td fn cache(self) -> cache::Initializer { let mut value = cache::Initializer::default(); { @@ -129,26 +221,157 @@ impl AtHwcap { enable_feature(Feature::fp, self.fp); // Half-float support requires float support enable_feature(Feature::fp16, self.fp && self.fphp); + // FHM (fp16fml in LLVM) requires half float support + enable_feature(Feature::fhm, self.fphp && self.fhm); enable_feature(Feature::pmull, self.pmull); enable_feature(Feature::crc, self.crc32); enable_feature(Feature::lse, self.atomics); + enable_feature(Feature::lse2, self.uscat); enable_feature(Feature::rcpc, self.lrcpc); + // RCPC2 (rcpc-immo in LLVM) requires RCPC support + enable_feature(Feature::rcpc2, self.ilrcpc && self.lrcpc); + enable_feature(Feature::dit, self.dit); + enable_feature(Feature::flagm, self.flagm); + enable_feature(Feature::ssbs, self.ssbs); + enable_feature(Feature::sb, self.sb); + enable_feature(Feature::paca, self.paca); + enable_feature(Feature::pacg, self.pacg); + enable_feature(Feature::dpb, self.dcpop); + enable_feature(Feature::dpb2, self.dcpodp); + enable_feature(Feature::rand, self.rng); + enable_feature(Feature::bti, self.bti); + enable_feature(Feature::mte, self.mte); + // jsconv requires float support + enable_feature(Feature::jsconv, self.jscvt && self.fp); + enable_feature(Feature::rdm, self.asimdrdm); + enable_feature(Feature::dotprod, self.asimddp); + enable_feature(Feature::frintts, self.frint); + + // FEAT_I8MM & FEAT_BF16 also include optional SVE components which linux exposes + // separately. We ignore that distinction here. + enable_feature(Feature::i8mm, self.i8mm); + enable_feature(Feature::bf16, self.bf16); - // SIMD support requires float support - if half-floats are + // ASIMD support requires float support - if half-floats are // supported, it also requires half-float support: let asimd = self.fp && self.asimd && (!self.fphp | self.asimdhp); enable_feature(Feature::asimd, asimd); - // SIMD extensions require SIMD support: - enable_feature(Feature::rdm, self.asimdrdm && asimd); - enable_feature(Feature::dotprod, self.asimddp && asimd); + // ASIMD extensions require ASIMD support: + enable_feature(Feature::fcma, self.fcma && asimd); enable_feature(Feature::sve, self.sve && asimd); - // Crypto is specified as AES + PMULL + SHA1 + SHA2 per LLVM/hosts.cpp + // SVE extensions require SVE & ASIMD + enable_feature(Feature::f32mm, self.svef32mm && self.sve && asimd); + enable_feature(Feature::f64mm, self.svef64mm && self.sve && asimd); + + // Cryptographic extensions require ASIMD + enable_feature(Feature::aes, self.aes && asimd); + enable_feature(Feature::sha2, self.sha1 && self.sha2 && asimd); + // SHA512/SHA3 require SHA1 & SHA256 + enable_feature( + Feature::sha3, + self.sha512 && self.sha3 && self.sha1 && self.sha2 && asimd, + ); + enable_feature(Feature::sm4, self.sm3 && self.sm4 && asimd); + + // SVE2 requires SVE + let sve2 = self.sve2 && self.sve && asimd; + enable_feature(Feature::sve2, sve2); + // SVE2 extensions require SVE2 and crypto features + enable_feature(Feature::sve2_aes, self.sveaes && sve2 && self.aes); + enable_feature( + Feature::sve2_sm4, + self.svesm4 && sve2 && self.sm3 && self.sm4, + ); enable_feature( - Feature::crypto, - self.aes && self.pmull && self.sha1 && self.sha2, + Feature::sve2_sha3, + self.svesha3 && sve2 && self.sha512 && self.sha3 && self.sha1 && self.sha2, ); + enable_feature(Feature::sve2_bitperm, self.svebitperm && self.sve2); } value } } + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(feature = "std_detect_file_io")] + mod auxv_from_file { + use super::auxvec::auxv_from_file; + use super::*; + // The baseline hwcaps used in the (artificial) auxv test files. + fn baseline_hwcaps() -> AtHwcap { + AtHwcap { + fp: true, + asimd: true, + aes: true, + pmull: true, + sha1: true, + sha2: true, + crc32: true, + atomics: true, + fphp: true, + asimdhp: true, + asimdrdm: true, + lrcpc: true, + dcpop: true, + asimddp: true, + ssbs: true, + ..AtHwcap::default() + } + } + + #[test] + fn linux_empty_hwcap2_aarch64() { + let file = concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/detect/test_data/linux-empty-hwcap2-aarch64.auxv" + ); + println!("file: {}", file); + let v = auxv_from_file(file).unwrap(); + println!("HWCAP : 0x{:0x}", v.hwcap); + println!("HWCAP2: 0x{:0x}", v.hwcap2); + assert_eq!(AtHwcap::from(v), baseline_hwcaps()); + } + #[test] + fn linux_no_hwcap2_aarch64() { + let file = concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/detect/test_data/linux-no-hwcap2-aarch64.auxv" + ); + println!("file: {}", file); + let v = auxv_from_file(file).unwrap(); + println!("HWCAP : 0x{:0x}", v.hwcap); + println!("HWCAP2: 0x{:0x}", v.hwcap2); + assert_eq!(AtHwcap::from(v), baseline_hwcaps()); + } + #[test] + fn linux_hwcap2_aarch64() { + let file = concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/detect/test_data/linux-hwcap2-aarch64.auxv" + ); + println!("file: {}", file); + let v = auxv_from_file(file).unwrap(); + println!("HWCAP : 0x{:0x}", v.hwcap); + println!("HWCAP2: 0x{:0x}", v.hwcap2); + assert_eq!( + AtHwcap::from(v), + AtHwcap { + // Some other HWCAP bits. + paca: true, + pacg: true, + // HWCAP2-only bits. + dcpodp: true, + frint: true, + rng: true, + bti: true, + mte: true, + ..baseline_hwcaps() + } + ); + } + } +} diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/arm.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/arm.rs index f55bc30b8..7383e487f 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/arm.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/arm.rs @@ -1,6 +1,6 @@ //! Run-time feature detection for ARM on Linux. -use super::{auxvec, cpuinfo}; +use super::auxvec; use crate::detect::{bit, cache, Feature}; /// Try to read the features from the auxiliary vector, and if that fails, try @@ -20,10 +20,26 @@ pub(crate) fn detect_features() -> cache::Initializer { enable_feature(&mut value, Feature::neon, bit::test(auxv.hwcap, 12)); enable_feature(&mut value, Feature::pmull, bit::test(auxv.hwcap2, 1)); enable_feature(&mut value, Feature::crc, bit::test(auxv.hwcap2, 4)); + enable_feature( + &mut value, + Feature::crypto, + bit::test(auxv.hwcap2, 0) + && bit::test(auxv.hwcap2, 1) + && bit::test(auxv.hwcap2, 2) + && bit::test(auxv.hwcap2, 3), + ); + enable_feature(&mut value, Feature::aes, bit::test(auxv.hwcap2, 0)); + // SHA2 requires SHA1 & SHA2 features + enable_feature( + &mut value, + Feature::sha2, + bit::test(auxv.hwcap2, 2) && bit::test(auxv.hwcap2, 3), + ); return value; } - if let Ok(c) = cpuinfo::CpuInfo::new() { + #[cfg(feature = "std_detect_file_io")] + if let Ok(c) = super::cpuinfo::CpuInfo::new() { enable_feature( &mut value, Feature::neon, @@ -31,6 +47,20 @@ pub(crate) fn detect_features() -> cache::Initializer { ); enable_feature(&mut value, Feature::pmull, c.field("Features").has("pmull")); enable_feature(&mut value, Feature::crc, c.field("Features").has("crc32")); + enable_feature( + &mut value, + Feature::crypto, + c.field("Features").has("aes") + && c.field("Features").has("pmull") + && c.field("Features").has("sha1") + && c.field("Features").has("sha2"), + ); + enable_feature(&mut value, Feature::aes, c.field("Features").has("aes")); + enable_feature( + &mut value, + Feature::sha2, + c.field("Features").has("sha1") && c.field("Features").has("sha2"), + ); return value; } value @@ -39,7 +69,8 @@ pub(crate) fn detect_features() -> cache::Initializer { /// Is the CPU known to have a broken NEON unit? /// /// See https://crbug.com/341598. -fn has_broken_neon(cpuinfo: &cpuinfo::CpuInfo) -> bool { +#[cfg(feature = "std_detect_file_io")] +fn has_broken_neon(cpuinfo: &super::cpuinfo::CpuInfo) -> bool { cpuinfo.field("CPU implementer") == "0x51" && cpuinfo.field("CPU architecture") == "7" && cpuinfo.field("CPU variant") == "0x1" diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/auxvec.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/auxvec.rs index 6ebae67fb..c903903bd 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/auxvec.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/auxvec.rs @@ -1,13 +1,17 @@ //! Parses ELF auxiliary vectors. -#![cfg_attr(not(target_arch = "aarch64"), allow(dead_code))] +#![allow(dead_code)] -#[cfg(feature = "std_detect_file_io")] -use crate::{fs::File, io::Read}; +pub(crate) const AT_NULL: usize = 0; /// Key to access the CPU Hardware capabilities bitfield. pub(crate) const AT_HWCAP: usize = 16; /// Key to access the CPU Hardware capabilities 2 bitfield. -#[cfg(any(target_arch = "arm", target_arch = "powerpc64"))] +#[cfg(any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc", + target_arch = "powerpc64" +))] pub(crate) const AT_HWCAP2: usize = 26; /// Cache HWCAP bitfields of the ELF Auxiliary Vector. @@ -17,7 +21,12 @@ pub(crate) const AT_HWCAP2: usize = 26; #[derive(Debug, Copy, Clone)] pub(crate) struct AuxVec { pub hwcap: usize, - #[cfg(any(target_arch = "arm", target_arch = "powerpc64"))] + #[cfg(any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc", + target_arch = "powerpc64" + ))] pub hwcap2: usize, } @@ -56,57 +65,90 @@ pub(crate) fn auxv() -> Result { // Try to call a dynamically-linked getauxval function. if let Ok(hwcap) = getauxval(AT_HWCAP) { // Targets with only AT_HWCAP: - #[cfg(any(target_arch = "aarch64", target_arch = "mips", target_arch = "mips64"))] + #[cfg(any( + target_arch = "riscv32", + target_arch = "riscv64", + target_arch = "mips", + target_arch = "mips64" + ))] { + // Zero could indicate that no features were detected, but it's also used to + // indicate an error. In either case, try the fallback. if hwcap != 0 { return Ok(AuxVec { hwcap }); } } // Targets with AT_HWCAP and AT_HWCAP2: - #[cfg(any(target_arch = "arm", target_arch = "powerpc64"))] + #[cfg(any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc", + target_arch = "powerpc64" + ))] { if let Ok(hwcap2) = getauxval(AT_HWCAP2) { - if hwcap != 0 && hwcap2 != 0 { + // Zero could indicate that no features were detected, but it's also used to + // indicate an error. In particular, on many platforms AT_HWCAP2 will be + // legitimately zero, since it contains the most recent feature flags. Use the + // fallback only if no features were detected at all. + if hwcap != 0 || hwcap2 != 0 { return Ok(AuxVec { hwcap, hwcap2 }); } } } drop(hwcap); } - #[cfg(feature = "std_detect_file_io")] - { - // If calling getauxval fails, try to read the auxiliary vector from - // its file: - auxv_from_file("/proc/self/auxv") - } - #[cfg(not(feature = "std_detect_file_io"))] - { - Err(()) - } } #[cfg(not(feature = "std_detect_dlsym_getauxval"))] { - let hwcap = unsafe { ffi_getauxval(AT_HWCAP) }; - // Targets with only AT_HWCAP: - #[cfg(any(target_arch = "aarch64", target_arch = "mips", target_arch = "mips64"))] + #[cfg(any( + target_arch = "riscv32", + target_arch = "riscv64", + target_arch = "mips", + target_arch = "mips64" + ))] { + let hwcap = unsafe { libc::getauxval(AT_HWCAP as libc::c_ulong) as usize }; + // Zero could indicate that no features were detected, but it's also used to indicate + // an error. In either case, try the fallback. if hwcap != 0 { return Ok(AuxVec { hwcap }); } } // Targets with AT_HWCAP and AT_HWCAP2: - #[cfg(any(target_arch = "arm", target_arch = "powerpc64"))] + #[cfg(any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc", + target_arch = "powerpc64" + ))] { - let hwcap2 = unsafe { ffi_getauxval(AT_HWCAP2) }; - if hwcap != 0 && hwcap2 != 0 { + let hwcap = unsafe { libc::getauxval(AT_HWCAP as libc::c_ulong) as usize }; + let hwcap2 = unsafe { libc::getauxval(AT_HWCAP2 as libc::c_ulong) as usize }; + // Zero could indicate that no features were detected, but it's also used to indicate + // an error. In particular, on many platforms AT_HWCAP2 will be legitimately zero, + // since it contains the most recent feature flags. Use the fallback only if no + // features were detected at all. + if hwcap != 0 || hwcap2 != 0 { return Ok(AuxVec { hwcap, hwcap2 }); } } } + + #[cfg(feature = "std_detect_file_io")] + { + // If calling getauxval fails, try to read the auxiliary vector from + // its file: + auxv_from_file("/proc/self/auxv") + } + #[cfg(not(feature = "std_detect_file_io"))] + { + Err(()) + } } /// Tries to read the `key` from the auxiliary vector by calling the @@ -122,7 +164,7 @@ fn getauxval(key: usize) -> Result { return Err(()); } - let ffi_getauxval: F = mem::transmute(ptr); + let ffi_getauxval: F = core::mem::transmute(ptr); Ok(ffi_getauxval(key)) } } @@ -130,8 +172,8 @@ fn getauxval(key: usize) -> Result { /// Tries to read the auxiliary vector from the `file`. If this fails, this /// function returns `Err`. #[cfg(feature = "std_detect_file_io")] -fn auxv_from_file(file: &str) -> Result { - let mut file = File::open(file).map_err(|_| ())?; +pub(super) fn auxv_from_file(file: &str) -> Result { + let file = super::read_file(file)?; // See . // @@ -139,10 +181,11 @@ fn auxv_from_file(file: &str) -> Result { // `AT_EXECFN = 31` to `AT_NULL = 0`. That is, a buffer of // 2*32 `usize` elements is enough to read the whole vector. let mut buf = [0_usize; 64]; - { - let raw: &mut [u8; 64 * mem::size_of::()] = unsafe { mem::transmute(&mut buf) }; - file.read(raw).map_err(|_| ())?; + let len = core::mem::size_of_val(&buf).max(file.len()); + unsafe { + core::ptr::copy_nonoverlapping(file.as_ptr(), buf.as_mut_ptr() as *mut u8, len); } + auxv_from_buf(&buf) } @@ -151,29 +194,42 @@ fn auxv_from_file(file: &str) -> Result { #[cfg(feature = "std_detect_file_io")] fn auxv_from_buf(buf: &[usize; 64]) -> Result { // Targets with only AT_HWCAP: - #[cfg(any(target_arch = "aarch64", target_arch = "mips", target_arch = "mips64"))] + #[cfg(any( + target_arch = "riscv32", + target_arch = "riscv64", + target_arch = "mips", + target_arch = "mips64", + ))] { for el in buf.chunks(2) { match el[0] { + AT_NULL => break, AT_HWCAP => return Ok(AuxVec { hwcap: el[1] }), _ => (), } } } // Targets with AT_HWCAP and AT_HWCAP2: - #[cfg(any(target_arch = "arm", target_arch = "powerpc64"))] + #[cfg(any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc", + target_arch = "powerpc64" + ))] { let mut hwcap = None; - let mut hwcap2 = None; + // For some platforms, AT_HWCAP2 was added recently, so let it default to zero. + let mut hwcap2 = 0; for el in buf.chunks(2) { match el[0] { + AT_NULL => break, AT_HWCAP => hwcap = Some(el[1]), - AT_HWCAP2 => hwcap2 = Some(el[1]), + AT_HWCAP2 => hwcap2 = el[1], _ => (), } } - if let (Some(hwcap), Some(hwcap2)) = (hwcap, hwcap2) { + if let Some(hwcap) = hwcap { return Ok(AuxVec { hwcap, hwcap2 }); } } @@ -214,7 +270,11 @@ mod tests { // FIXME: on mips/mips64 getauxval returns 0, and /proc/self/auxv // does not always contain the AT_HWCAP key under qemu. - #[cfg(not(any(target_arch = "mips", target_arch = "mips64", target_arch = "powerpc")))] + #[cfg(any( + target_arch = "arm", + target_arch = "powerpc", + target_arch = "powerpc64" + ))] #[test] fn auxv_crate() { let v = auxv(); @@ -224,7 +284,12 @@ mod tests { } // Targets with AT_HWCAP and AT_HWCAP2: - #[cfg(any(target_arch = "arm", target_arch = "powerpc64"))] + #[cfg(any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc", + target_arch = "powerpc64" + ))] { if let Some(hwcap2) = auxv_crate_getauxval(AT_HWCAP2) { let rt_hwcap2 = v.expect("failed to find hwcap2 key").hwcap2; @@ -243,7 +308,7 @@ mod tests { } #[cfg(feature = "std_detect_file_io")] - cfg_if! { + cfg_if::cfg_if! { if #[cfg(target_arch = "arm")] { #[test] fn linux_rpi3() { @@ -255,23 +320,31 @@ mod tests { } #[test] - #[should_panic] fn linux_macos_vb() { let file = concat!(env!("CARGO_MANIFEST_DIR"), "/src/detect/test_data/macos-virtualbox-linux-x86-4850HQ.auxv"); println!("file: {}", file); + // The file contains HWCAP but not HWCAP2. In that case, we treat HWCAP2 as zero. let v = auxv_from_file(file).unwrap(); - // this file is incomplete (contains hwcap but not hwcap2), we - // want to fall back to /proc/cpuinfo in this case, so - // reading should fail. assert_eq!(v.hwcap, 126614527); - // assert_eq!(v.hwcap2, 0); + assert_eq!(v.hwcap, 126614527); + assert_eq!(v.hwcap2, 0); } } else if #[cfg(target_arch = "aarch64")] { #[test] - fn linux_x64() { - let file = concat!(env!("CARGO_MANIFEST_DIR"), "/src/detect/test_data/linux-x64-i7-6850k.auxv"); + fn linux_artificial_aarch64() { + let file = concat!(env!("CARGO_MANIFEST_DIR"), "/src/detect/test_data/linux-artificial-aarch64.auxv"); println!("file: {}", file); let v = auxv_from_file(file).unwrap(); - assert_eq!(v.hwcap, 3219913727); + assert_eq!(v.hwcap, 0x0123456789abcdef); + assert_eq!(v.hwcap2, 0x02468ace13579bdf); + } + #[test] + fn linux_no_hwcap2_aarch64() { + let file = concat!(env!("CARGO_MANIFEST_DIR"), "/src/detect/test_data/linux-no-hwcap2-aarch64.auxv"); + println!("file: {}", file); + let v = auxv_from_file(file).unwrap(); + // An absent HWCAP2 is treated as zero, and does not prevent acceptance of HWCAP. + assert_ne!(v.hwcap, 0); + assert_eq!(v.hwcap2, 0); } } } @@ -286,7 +359,14 @@ mod tests { } } + #[cfg(any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc", + target_arch = "powerpc64" + ))] #[test] + #[cfg(feature = "std_detect_file_io")] fn auxv_crate_procfs() { let v = auxv(); if let Some(hwcap) = auxv_crate_getprocfs(AT_HWCAP) { @@ -294,7 +374,12 @@ mod tests { } // Targets with AT_HWCAP and AT_HWCAP2: - #[cfg(any(target_arch = "arm", target_arch = "powerpc64"))] + #[cfg(any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc", + target_arch = "powerpc64" + ))] { if let Some(hwcap2) = auxv_crate_getprocfs(AT_HWCAP2) { assert_eq!(v.unwrap().hwcap2, hwcap2); diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/cpuinfo.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/cpuinfo.rs index f76c48a4b..48a5c9728 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/cpuinfo.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/cpuinfo.rs @@ -1,8 +1,7 @@ //! Parses /proc/cpuinfo #![cfg_attr(not(target_arch = "arm"), allow(dead_code))] -extern crate std; -use self::std::{fs::File, io, io::Read, prelude::v1::*}; +use alloc::string::String; /// cpuinfo pub(crate) struct CpuInfo { @@ -11,14 +10,14 @@ pub(crate) struct CpuInfo { impl CpuInfo { /// Reads /proc/cpuinfo into CpuInfo. - pub(crate) fn new() -> Result { - let mut file = File::open("/proc/cpuinfo")?; - let mut cpui = Self { raw: String::new() }; - file.read_to_string(&mut cpui.raw)?; - Ok(cpui) + pub(crate) fn new() -> Result { + let raw = super::read_file("/proc/cpuinfo")?; + Ok(Self { + raw: String::from_utf8(raw).map_err(|_| ())?, + }) } /// Returns the value of the cpuinfo `field`. - pub(crate) fn field(&self, field: &str) -> CpuInfoField { + pub(crate) fn field(&self, field: &str) -> CpuInfoField<'_> { for l in self.raw.lines() { if l.trim().starts_with(field) { return CpuInfoField::new(l.split(": ").nth(1)); @@ -34,7 +33,7 @@ impl CpuInfo { } #[cfg(test)] - fn from_str(other: &str) -> Result { + fn from_str(other: &str) -> Result { Ok(Self { raw: String::from(other), }) @@ -213,6 +212,38 @@ CPU revision : 1"; assert!(cpuinfo.field("Features").has("asimd")); } + const RISCV_RV64GC: &str = r"processor : 0 +hart : 3 +isa : rv64imafdc +mmu : sv39 +uarch : sifive,u74-mc + +processor : 1 +hart : 1 +isa : rv64imafdc +mmu : sv39 +uarch : sifive,u74-mc + +processor : 2 +hart : 2 +isa : rv64imafdc +mmu : sv39 +uarch : sifive,u74-mc + +processor : 3 +hart : 4 +isa : rv64imafdc +mmu : sv39 +uarch : sifive,u74-mc"; + + #[test] + fn riscv_rv64gc() { + let cpuinfo = CpuInfo::from_str(RISCV_RV64GC).unwrap(); + assert_eq!(cpuinfo.field("isa"), "rv64imafdc"); + assert_eq!(cpuinfo.field("mmu"), "sv39"); + assert_eq!(cpuinfo.field("uarch"), "sifive,u74-mc"); + } + const POWER8E_POWERKVM: &str = r"processor : 0 cpu : POWER8E (raw), altivec supported clock : 3425.000000MHz diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/mod.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/mod.rs index e02d5e6dc..a49a72783 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/mod.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/mod.rs @@ -1,28 +1,64 @@ //! Run-time feature detection on Linux +//! +#[cfg(feature = "std_detect_file_io")] +use alloc::vec::Vec; mod auxvec; #[cfg(feature = "std_detect_file_io")] mod cpuinfo; -cfg_if! { +#[cfg(feature = "std_detect_file_io")] +fn read_file(path: &str) -> Result, ()> { + let mut path = Vec::from(path.as_bytes()); + path.push(0); + + unsafe { + let file = libc::open(path.as_ptr() as *const libc::c_char, libc::O_RDONLY); + if file == -1 { + return Err(()); + } + + let mut data = Vec::new(); + loop { + data.reserve(4096); + let spare = data.spare_capacity_mut(); + match libc::read(file, spare.as_mut_ptr() as *mut _, spare.len()) { + -1 => { + libc::close(file); + return Err(()); + } + 0 => break, + n => data.set_len(data.len() + n as usize), + } + } + + libc::close(file); + Ok(data) + } +} + +cfg_if::cfg_if! { if #[cfg(target_arch = "aarch64")] { mod aarch64; - pub use self::aarch64::check_for; + pub(crate) use self::aarch64::detect_features; } else if #[cfg(target_arch = "arm")] { mod arm; - pub use self::arm::check_for; - } else if #[cfg(any(target_arch = "mips", target_arch = "mips64"))] { + pub(crate) use self::arm::detect_features; + } else if #[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] { + mod riscv; + pub(crate) use self::riscv::detect_features; + } else if #[cfg(any(target_arch = "mips", target_arch = "mips64"))] { mod mips; - pub use self::mips::check_for; + pub(crate) use self::mips::detect_features; } else if #[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] { mod powerpc; - pub use self::powerpc::check_for; + pub(crate) use self::powerpc::detect_features; } else { - use crate::detect::Feature; + use crate::detect::cache; /// Performs run-time feature detection. - pub fn check_for(_x: Feature) -> bool { - false + pub(crate) fn detect_features() -> cache::Initializer { + cache::Initializer::default() } } } diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/powerpc.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/powerpc.rs index 97afe49fe..c3308e815 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/powerpc.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/powerpc.rs @@ -1,6 +1,6 @@ //! Run-time feature detection for PowerPC on Linux. -use super::{auxvec, cpuinfo}; +use super::auxvec; use crate::detect::{cache, Feature}; /// Try to read the features from the auxiliary vector, and if that fails, try @@ -27,7 +27,8 @@ pub(crate) fn detect_features() -> cache::Initializer { // PowerPC's /proc/cpuinfo lacks a proper Feature field, // but `altivec` support is indicated in the `cpu` field. - if let Ok(c) = cpuinfo::CpuInfo::new() { + #[cfg(feature = "std_detect_file_io")] + if let Ok(c) = super::cpuinfo::CpuInfo::new() { enable_feature(&mut value, Feature::altivec, c.field("cpu").has("altivec")); return value; } diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/riscv.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/riscv.rs new file mode 100644 index 000000000..1ec06959a --- /dev/null +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/linux/riscv.rs @@ -0,0 +1,73 @@ +//! Run-time feature detection for RISC-V on Linux. + +use super::auxvec; +use crate::detect::{bit, cache, Feature}; + +/// Read list of supported features from the auxiliary vector. +pub(crate) fn detect_features() -> cache::Initializer { + let mut value = cache::Initializer::default(); + let enable_feature = |value: &mut cache::Initializer, feature, enable| { + if enable { + value.set(feature as u32); + } + }; + let enable_features = |value: &mut cache::Initializer, feature_slice: &[Feature], enable| { + if enable { + for feature in feature_slice { + value.set(*feature as u32); + } + } + }; + + // The values are part of the platform-specific [asm/hwcap.h][hwcap] + // + // [hwcap]: https://github.com/torvalds/linux/blob/master/arch/riscv/include/asm/hwcap.h + let auxv = auxvec::auxv().expect("read auxvec"); // should not fail on RISC-V platform + enable_feature( + &mut value, + Feature::a, + bit::test(auxv.hwcap, (b'a' - b'a').into()), + ); + enable_feature( + &mut value, + Feature::c, + bit::test(auxv.hwcap, (b'c' - b'a').into()), + ); + enable_features( + &mut value, + &[Feature::d, Feature::f, Feature::zicsr], + bit::test(auxv.hwcap, (b'd' - b'a').into()), + ); + enable_features( + &mut value, + &[Feature::f, Feature::zicsr], + bit::test(auxv.hwcap, (b'f' - b'a').into()), + ); + let has_i = bit::test(auxv.hwcap, (b'i' - b'a').into()); + // If future RV128I is supported, implement with `enable_feature` here + #[cfg(target_pointer_width = "64")] + enable_feature(&mut value, Feature::rv64i, has_i); + #[cfg(target_pointer_width = "32")] + enable_feature(&mut value, Feature::rv32i, has_i); + #[cfg(target_pointer_width = "32")] + enable_feature( + &mut value, + Feature::rv32e, + bit::test(auxv.hwcap, (b'e' - b'a').into()), + ); + enable_feature( + &mut value, + Feature::h, + bit::test(auxv.hwcap, (b'h' - b'a').into()), + ); + enable_feature( + &mut value, + Feature::m, + bit::test(auxv.hwcap, (b'm' - b'a').into()), + ); + // FIXME: Auxvec does not show supervisor feature support, but this mode may be useful + // to detect when Rust is used to write Linux kernel modules. + // These should be more than Auxvec way to detect supervisor features. + + value +} diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/other.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/other.rs index bf7be87f0..091fafc4e 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/other.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/other.rs @@ -2,6 +2,7 @@ use crate::detect::cache; +#[allow(dead_code)] pub(crate) fn detect_features() -> cache::Initializer { cache::Initializer::default() } diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/windows/aarch64.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/windows/aarch64.rs index 3c5edfb44..051ad6d1b 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/windows/aarch64.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/windows/aarch64.rs @@ -42,13 +42,17 @@ pub(crate) fn detect_features() -> cache::Initializer { // PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE means aes, sha1, sha2 and // pmull support enable_feature( - Feature::crypto, + Feature::aes, IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) != FALSE, ); enable_feature( Feature::pmull, IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) != FALSE, ); + enable_feature( + Feature::sha2, + IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) != FALSE, + ); } } value diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/x86.rs b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/x86.rs index 436fb00f0..ea5f595ec 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/x86.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/src/detect/os/x86.rs @@ -1,11 +1,11 @@ //! x86 run-time feature detection is OS independent. #[cfg(target_arch = "x86")] -use crate::arch::x86::*; +use core::arch::x86::*; #[cfg(target_arch = "x86_64")] -use crate::arch::x86_64::*; +use core::arch::x86_64::*; -use crate::mem; +use core::mem; use crate::detect::{bit, cache, Feature}; @@ -249,5 +249,25 @@ pub(crate) fn detect_features() -> cache::Initializer { } } + // Unfortunately, some Skylake chips erroneously report support for BMI1 and + // BMI2 without actual support. These chips don't support AVX, and it seems + // that all Intel chips with non-erroneous support BMI do (I didn't check + // other vendors), so we can disable these flags for chips that don't also + // report support for AVX. + // + // It's possible this will pessimize future chips that do support BMI and + // not AVX, but this seems minor compared to a hard crash you get when + // executing an unsupported instruction (to put it another way, it's safe + // for us to under-report CPU features, but not to over-report them). Still, + // to limit any impact this may have in the future, we only do this for + // Intel chips, as it's a bug only present in their chips. + // + // This bug is documented as `SKL052` in the errata section of this document: + // http://www.intel.com/content/dam/www/public/us/en/documents/specification-updates/desktop-6th-gen-core-family-spec-update.pdf + if vendor_id == *b"GenuineIntel" && !value.test(Feature::avx as u32) { + value.unset(Feature::bmi1 as u32); + value.unset(Feature::bmi2 as u32); + } + value } diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/test_data/linux-artificial-aarch64.auxv b/crux-mir/lib/stdarch/crates/std_detect/src/detect/test_data/linux-artificial-aarch64.auxv new file mode 100644 index 0000000000000000000000000000000000000000..ec826afcf38179904c8b4312842dc474687d254c GIT binary patch literal 336 zcmY#nKn2Dyt`vkJ0Oh|wySg*oRhf|uCJ&Vmgz}-v8JJ=6P?`me&x*!pN8@v#@j21> r+)zHuJ{~kaFB+c@jV})6!zf7@15Mri+2O+Hy4;xL(d7Bj_~={!cy$wI|)_SNs gSJrR?cW@66@F@G<1kdmSC)u&PV9(#hMY+P{19AETwEzGB literal 0 HcmV?d00001 diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/test_data/linux-hwcap2-aarch64.auxv b/crux-mir/lib/stdarch/crates/std_detect/src/detect/test_data/linux-hwcap2-aarch64.auxv new file mode 100644 index 0000000000000000000000000000000000000000..1d87264b221901c140c379792e7e14c7e74cbf5c GIT binary patch literal 336 zcmZ9`OA3H6420nn-}vmR=tew)2lh(dNRdt!{@r9gCeR%Af2VWHVUeFJ-e0U`QxRpY kTj9NC30H6fw{Qpdvi~>0BRs)L_OW`vUI_WJ9sLZGFHf=regFUf literal 0 HcmV?d00001 diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/test_data/linux-no-hwcap2-aarch64.auxv b/crux-mir/lib/stdarch/crates/std_detect/src/detect/test_data/linux-no-hwcap2-aarch64.auxv new file mode 100644 index 0000000000000000000000000000000000000000..35f01cc767c507ed0d28a1d349fa84ba33dc7899 GIT binary patch literal 320 zcmZ9_OA3H66hqO1fBbe-?4-B`7wl3MG#Pk1Ne+3?BmCcK=C5q>eZ{x7-Zey7=b`eh ctlYO+n*aa+ literal 0 HcmV?d00001 diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/detect/test_data/linux-x64-i7-6850k.auxv b/crux-mir/lib/stdarch/crates/std_detect/src/detect/test_data/linux-x64-i7-6850k.auxv deleted file mode 100644 index 6afe1b3b46a8d00235789b68fec1bc94280fde71..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 304 zcmY#nfC7ctpa0Y|FbF`o|9`*U4`Q-G`7jAVC_e>CGec +#[cfg_attr(feature = "std_detect_file_io", allow(unused_extern_crates))] +#[cfg(feature = "std_detect_file_io")] +extern crate alloc; #[doc(hidden)] #[unstable(feature = "stdsimd", issue = "27731")] diff --git a/crux-mir/lib/stdarch/crates/std_detect/src/mod.rs b/crux-mir/lib/stdarch/crates/std_detect/src/mod.rs deleted file mode 100644 index b630e7ff3..000000000 --- a/crux-mir/lib/stdarch/crates/std_detect/src/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -//! `std_detect` - -#[doc(hidden)] // unstable implementation detail -#[unstable(feature = "stdsimd", issue = "27731")] -pub mod detect; diff --git a/crux-mir/lib/stdarch/crates/std_detect/tests/cpu-detection.rs b/crux-mir/lib/stdarch/crates/std_detect/tests/cpu-detection.rs index 9be40542c..ca8bf28f4 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/tests/cpu-detection.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/tests/cpu-detection.rs @@ -1,5 +1,5 @@ #![feature(stdsimd)] -#![allow(clippy::option_unwrap_used, clippy::use_debug, clippy::print_stdout)] +#![allow(clippy::unwrap_used, clippy::use_debug, clippy::print_stdout)] #![cfg(any( target_arch = "arm", target_arch = "aarch64", @@ -25,6 +25,9 @@ fn arm_linux() { println!("neon: {}", is_arm_feature_detected!("neon")); println!("pmull: {}", is_arm_feature_detected!("pmull")); println!("crc: {}", is_arm_feature_detected!("crc")); + println!("crypto: {}", is_arm_feature_detected!("crypto")); + println!("aes: {}", is_arm_feature_detected!("aes")); + println!("sha2: {}", is_arm_feature_detected!("sha2")); } #[test] @@ -33,17 +36,51 @@ fn arm_linux() { any(target_os = "linux", target_os = "android") ))] fn aarch64_linux() { + println!("asimd: {}", is_aarch64_feature_detected!("asimd")); + println!("neon: {}", is_aarch64_feature_detected!("neon")); + println!("pmull: {}", is_aarch64_feature_detected!("pmull")); println!("fp: {}", is_aarch64_feature_detected!("fp")); println!("fp16: {}", is_aarch64_feature_detected!("fp16")); - println!("neon: {}", is_aarch64_feature_detected!("neon")); - println!("asimd: {}", is_aarch64_feature_detected!("asimd")); println!("sve: {}", is_aarch64_feature_detected!("sve")); println!("crc: {}", is_aarch64_feature_detected!("crc")); - println!("crypto: {}", is_aarch64_feature_detected!("crypto")); println!("lse: {}", is_aarch64_feature_detected!("lse")); + println!("lse2: {}", is_aarch64_feature_detected!("lse2")); println!("rdm: {}", is_aarch64_feature_detected!("rdm")); println!("rcpc: {}", is_aarch64_feature_detected!("rcpc")); + println!("rcpc2: {}", is_aarch64_feature_detected!("rcpc2")); println!("dotprod: {}", is_aarch64_feature_detected!("dotprod")); + println!("tme: {}", is_aarch64_feature_detected!("tme")); + println!("fhm: {}", is_aarch64_feature_detected!("fhm")); + println!("dit: {}", is_aarch64_feature_detected!("dit")); + println!("flagm: {}", is_aarch64_feature_detected!("flagm")); + println!("ssbs: {}", is_aarch64_feature_detected!("ssbs")); + println!("sb: {}", is_aarch64_feature_detected!("sb")); + println!("paca: {}", is_aarch64_feature_detected!("paca")); + println!("pacg: {}", is_aarch64_feature_detected!("pacg")); + println!("dpb: {}", is_aarch64_feature_detected!("dpb")); + println!("dpb2: {}", is_aarch64_feature_detected!("dpb2")); + println!("sve2: {}", is_aarch64_feature_detected!("sve2")); + println!("sve2-aes: {}", is_aarch64_feature_detected!("sve2-aes")); + println!("sve2-sm4: {}", is_aarch64_feature_detected!("sve2-sm4")); + println!("sve2-sha3: {}", is_aarch64_feature_detected!("sve2-sha3")); + println!( + "sve2-bitperm: {}", + is_aarch64_feature_detected!("sve2-bitperm") + ); + println!("frintts: {}", is_aarch64_feature_detected!("frintts")); + println!("i8mm: {}", is_aarch64_feature_detected!("i8mm")); + println!("f32mm: {}", is_aarch64_feature_detected!("f32mm")); + println!("f64mm: {}", is_aarch64_feature_detected!("f64mm")); + println!("bf16: {}", is_aarch64_feature_detected!("bf16")); + println!("rand: {}", is_aarch64_feature_detected!("rand")); + println!("bti: {}", is_aarch64_feature_detected!("bti")); + println!("mte: {}", is_aarch64_feature_detected!("mte")); + println!("jsconv: {}", is_aarch64_feature_detected!("jsconv")); + println!("fcma: {}", is_aarch64_feature_detected!("fcma")); + println!("aes: {}", is_aarch64_feature_detected!("aes")); + println!("sha2: {}", is_aarch64_feature_detected!("sha2")); + println!("sha3: {}", is_aarch64_feature_detected!("sha3")); + println!("sm4: {}", is_aarch64_feature_detected!("sm4")); } #[test] diff --git a/crux-mir/lib/stdarch/crates/std_detect/tests/macro_trailing_commas.rs b/crux-mir/lib/stdarch/crates/std_detect/tests/macro_trailing_commas.rs index e950523d0..cd597af73 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/tests/macro_trailing_commas.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/tests/macro_trailing_commas.rs @@ -1,5 +1,5 @@ #![feature(stdsimd)] -#![allow(clippy::option_unwrap_used, clippy::use_debug, clippy::print_stdout)] +#![allow(clippy::unwrap_used, clippy::use_debug, clippy::print_stdout)] #[cfg(any( target_arch = "arm", diff --git a/crux-mir/lib/stdarch/crates/std_detect/tests/x86-specific.rs b/crux-mir/lib/stdarch/crates/std_detect/tests/x86-specific.rs index d4182644b..59e9a62fd 100644 --- a/crux-mir/lib/stdarch/crates/std_detect/tests/x86-specific.rs +++ b/crux-mir/lib/stdarch/crates/std_detect/tests/x86-specific.rs @@ -30,7 +30,7 @@ fn dump() { println!("avx512dq {:?}", is_x86_feature_detected!("avx512dq")); println!("avx512vl {:?}", is_x86_feature_detected!("avx512vl")); println!("avx512_ifma {:?}", is_x86_feature_detected!("avx512ifma")); - println!("avx512_vbmi {:?}", is_x86_feature_detected!("avx512vbmi")); + println!("avx512vbmi {:?}", is_x86_feature_detected!("avx512vbmi")); println!( "avx512_vpopcntdq {:?}", is_x86_feature_detected!("avx512vpopcntdq") diff --git a/crux-mir/lib/stdarch/crates/stdarch-gen/Cargo.toml b/crux-mir/lib/stdarch/crates/stdarch-gen/Cargo.toml new file mode 100644 index 000000000..b339672f4 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/stdarch-gen/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "stdarch-gen" +version = "0.1.0" +authors = ["Heinz Gies "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/crux-mir/lib/stdarch/crates/stdarch-gen/README.md b/crux-mir/lib/stdarch/crates/stdarch-gen/README.md new file mode 100644 index 000000000..54b602cdd --- /dev/null +++ b/crux-mir/lib/stdarch/crates/stdarch-gen/README.md @@ -0,0 +1,11 @@ +# Neon intrinsic code generator + +A small tool that allows to quickly generate intrinsics for the NEON architecture. + +The specification for the intrinsics can be found in `neon.spec`. + +To run and re-generate the code run the following from the root of the `stdarch` crate. + +``` +OUT_DIR=`pwd`/crates/core_arch cargo run -p stdarch-gen -- crates/stdarch-gen/neon.spec +``` \ No newline at end of file diff --git a/crux-mir/lib/stdarch/crates/stdarch-gen/neon.spec b/crux-mir/lib/stdarch/crates/stdarch-gen/neon.spec new file mode 100644 index 000000000..95fbc354c --- /dev/null +++ b/crux-mir/lib/stdarch/crates/stdarch-gen/neon.spec @@ -0,0 +1,7560 @@ +// ARM Neon intrinsic specification. +// +// This file contains the specification for a number of +// intrinsics that allows us to generate them along with +// their test cases. +// +// To the syntax of the file - it's not very intelligently parsed! +// +// # Comments +// start with AT LEAST two, or four or more slashes so // is a +// comment /////// is too. +// +// # Sections +// Sections start with EXACTLY three slashes followed +// by AT LEAST one space. Sections are used for two things: +// +// 1) they serve as the doc comment for the given intrinics. +// 2) they reset all variables (name, fn, etc.) +// +// # Variables +// +// name - The prefix of the function, suffixes are auto +// generated by the type they get passed. +// +// fn - The function to call in rust-land. +// +// aarch64 - The intrinsic to check on aarch64 architecture. +// If this is given but no arm intrinsic is provided, +// the function will exclusively be generated for +// aarch64. +// This is used to generate both aarch64 specific and +// shared intrinics by first only specifying th aarch64 +// variant then the arm variant. +// +// arm - The arm v7 intrinics used to checked for arm code +// generation. All neon functions available in arm are +// also available in aarch64. If no aarch64 intrinic was +// set they are assumed to be the same. +// Intrinics ending with a `.` will have a size suffixes +// added (such as `i8` or `i64`) that is not sign specific +// Intrinics ending with a `.s` will have a size suffixes +// added (such as `s8` or `u64`) that is sign specific +// +// a - First input for tests, it gets scaled to the size of +// the type. +// +// b - Second input for tests, it gets scaled to the size of +// the type. +// +// # special values +// +// TRUE - 'true' all bits are set to 1 +// FALSE - 'false' all bits are set to 0 +// FF - same as 'true' +// MIN - minimal value (either 0 or the lowest negative number) +// MAX - maximal value proper to overflow +// +// # validate +// Validates a and b against the expected result of the test. +// The special values 'TRUE' and 'FALSE' can be used to +// represent the correct NEON representation of true or +// false values. It too gets scaled to the type. +// +// Validate needs to be called before generate as it sets +// up the rules for validation that get generated for each +// type. +// # generate +// The generate command generates the intrinsics, it uses the +// Variables set and can be called multiple times while overwriting +// some of the variables. + +/// Vector bitwise and +name = vand +fn = simd_and +arm = vand +aarch64 = and +a = 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x00 +b = 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F +validate 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x00 +b = 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +validate 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +generate int*_t, uint*_t, int64x*_t, uint64x*_t + +/// Vector bitwise or (immediate, inclusive) +name = vorr +fn = simd_or +arm = vorr +aarch64 = orr +a = 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F +b = 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +validate 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F +generate int*_t, uint*_t, int64x*_t, uint64x*_t + + +/// Vector bitwise exclusive or (vector) +name = veor +fn = simd_xor +arm = veor +aarch64 = eor +a = 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F +b = 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +validate 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F +generate int*_t, uint*_t, int64x*_t, uint64x*_t + +/// Three-way exclusive OR +name = veor3 +a = 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F +b = 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +c = 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +validate 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F +target = sha3 + +aarch64 = eor3 +link-aarch64 = llvm.aarch64.crypto.eor3s._EXT_ +generate int8x16_t, int16x8_t, int32x4_t, int64x2_t +link-aarch64 = llvm.aarch64.crypto.eor3u._EXT_ +generate uint8x16_t, uint16x8_t, uint32x4_t, uint64x2_t + +//////////////////// +// Absolute difference between the arguments +//////////////////// + +/// Absolute difference between the arguments +name = vabd +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 +validate 15, 13, 11, 9, 7, 5, 3, 1, 1, 3, 5, 7, 9, 11, 13, 15 + +arm = vabd.s +aarch64 = sabd +link-arm = vabds._EXT_ +link-aarch64 = sabd._EXT_ +generate int*_t + +arm = vabd.s +aarch64 = uabd +link-arm = vabdu._EXT_ +link-aarch64 = uabd._EXT_ +generate uint*_t + +/// Absolute difference between the arguments of Floating +name = vabd +a = 1.0, 2.0, 5.0, -4.0 +b = 9.0, 3.0, 2.0, 8.0 +validate 8.0, 1.0, 3.0, 12.0 + +aarch64 = fabd +link-aarch64 = fabd._EXT_ +generate float64x*_t + +arm = vabd.s +aarch64 = fabd +link-arm = vabds._EXT_ +link-aarch64 = fabd._EXT_ +generate float*_t + +/// Floating-point absolute difference +name = vabd +multi_fn = simd_extract, {vabd-in_ntt-noext, {vdup_n-in_ntt-noext, a}, {vdup_n-in_ntt-noext, b}}, 0 +a = 1.0 +b = 9.0 +validate 8.0 + +aarch64 = fabd +generate f32, f64 + +//////////////////// +// Absolute difference Long +//////////////////// + +/// Unsigned Absolute difference Long +name = vabdl +multi_fn = simd_cast, {vabd-unsigned-noext, a, b} +a = 1, 2, 3, 4, 4, 3, 2, 1 +b = 10, 10, 10, 10, 10, 10, 10, 10 +validate 9, 8, 7, 6, 6, 7, 8, 9 + +arm = vabdl.s +aarch64 = uabdl +generate uint8x8_t:uint8x8_t:uint16x8_t, uint16x4_t:uint16x4_t:uint32x4_t, uint32x2_t:uint32x2_t:uint64x2_t + +/// Signed Absolute difference Long +name = vabdl +multi_fn = simd_cast, c:uint8x8_t, {vabd-signed-noext, a, b} +multi_fn = simd_cast, c +a = 1, 2, 3, 4, 4, 3, 2, 1 +b = 10, 10, 10, 10, 10, 10, 10, 10 +validate 9, 8, 7, 6, 6, 7, 8, 9 + +arm = vabdl.s +aarch64 = sabdl +generate int8x8_t:int8x8_t:int16x8_t + +/// Signed Absolute difference Long +name = vabdl +multi_fn = simd_cast, c:uint16x4_t, {vabd-signed-noext, a, b} +multi_fn = simd_cast, c +a = 1, 2, 11, 12 +b = 10, 10, 10, 10 +validate 9, 8, 1, 2 + +arm = vabdl.s +aarch64 = sabdl +generate int16x4_t:int16x4_t:int32x4_t + +/// Signed Absolute difference Long +name = vabdl +multi_fn = simd_cast, c:uint32x2_t, {vabd-signed-noext, a, b} +multi_fn = simd_cast, c +a = 1, 11 +b = 10, 10 +validate 9, 1 + +arm = vabdl.s +aarch64 = sabdl +generate int32x2_t:int32x2_t:int64x2_t + +/// Unsigned Absolute difference Long +name = vabdl_high +no-q +multi_fn = simd_shuffle8!, c:uint8x8_t, a, a, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_shuffle8!, d:uint8x8_t, b, b, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_cast, {vabd_u8, c, d} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10 +validate 1, 0, 1, 2, 3, 4, 5, 6 + +aarch64 = uabdl +generate uint8x16_t:uint8x16_t:uint16x8_t + +/// Unsigned Absolute difference Long +name = vabdl_high +no-q +multi_fn = simd_shuffle4!, c:uint16x4_t, a, a, [4, 5, 6, 7] +multi_fn = simd_shuffle4!, d:uint16x4_t, b, b, [4, 5, 6, 7] +multi_fn = simd_cast, {vabd_u16, c, d} +a = 1, 2, 3, 4, 8, 9, 11, 12 +b = 10, 10, 10, 10, 10, 10, 10, 10 +validate 2, 1, 1, 2 + +aarch64 = uabdl +generate uint16x8_t:uint16x8_t:uint32x4_t + +/// Unsigned Absolute difference Long +name = vabdl_high +no-q +multi_fn = simd_shuffle2!, c:uint32x2_t, a, a, [2, 3] +multi_fn = simd_shuffle2!, d:uint32x2_t, b, b, [2, 3] +multi_fn = simd_cast, {vabd_u32, c, d} +a = 1, 2, 3, 4 +b = 10, 10, 10, 10 +validate 7, 6 + +aarch64 = uabdl +generate uint32x4_t:uint32x4_t:uint64x2_t + +/// Signed Absolute difference Long +name = vabdl_high +no-q +multi_fn = simd_shuffle8!, c:int8x8_t, a, a, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_shuffle8!, d:int8x8_t, b, b, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_cast, e:uint8x8_t, {vabd_s8, c, d} +multi_fn = simd_cast, e +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10 +validate 1, 0, 1, 2, 3, 4, 5, 6 + +aarch64 = sabdl +generate int8x16_t:int8x16_t:int16x8_t + +/// Signed Absolute difference Long +name = vabdl_high +no-q +multi_fn = simd_shuffle4!, c:int16x4_t, a, a, [4, 5, 6, 7] +multi_fn = simd_shuffle4!, d:int16x4_t, b, b, [4, 5, 6, 7] +multi_fn = simd_cast, e:uint16x4_t, {vabd_s16, c, d} +multi_fn = simd_cast, e +a = 1, 2, 3, 4, 9, 10, 11, 12 +b = 10, 10, 10, 10, 10, 10, 10, 10 +validate 1, 0, 1, 2 + +aarch64 = sabdl +generate int16x8_t:int16x8_t:int32x4_t + +/// Signed Absolute difference Long +name = vabdl_high +no-q +multi_fn = simd_shuffle2!, c:int32x2_t, a, a, [2, 3] +multi_fn = simd_shuffle2!, d:int32x2_t, b, b, [2, 3] +multi_fn = simd_cast, e:uint32x2_t, {vabd_s32, c, d} +multi_fn = simd_cast, e +a = 1, 2, 3, 4 +b = 10, 10, 10, 10 +validate 7, 6 + +aarch64 = sabdl +generate int32x4_t:int32x4_t:int64x2_t + +//////////////////// +// equality +//////////////////// + +/// Compare bitwise Equal (vector) +name = vceq +fn = simd_eq +a = MIN, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, MAX +b = MIN, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, MAX +validate TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE +a = MIN, MIN, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0xCC, 0x0D, 0xEE, MAX +b = MIN, MAX, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08, 0x08, 0x00, 0x0A, 0x0A, 0xCC, 0xD0, 0xEE, MIN +validate TRUE, FALSE, TRUE, FALSE, TRUE, FALSE, TRUE, FALSE, TRUE, FALSE, TRUE, FALSE, TRUE, FALSE, TRUE, FALSE + +aarch64 = cmeq +generate uint64x*_t, int64x1_t:uint64x1_t, int64x2_t:uint64x2_t, poly64x1_t:uint64x1_t, poly64x2_t:uint64x2_t + +arm = vceq. +generate uint*_t, int8x8_t:uint8x8_t, int8x16_t:uint8x16_t, int16x4_t:uint16x4_t, int16x8_t:uint16x8_t, int32x2_t:uint32x2_t, int32x4_t:uint32x4_t, poly8x8_t:uint8x8_t, poly8x16_t:uint8x16_t + +/// Floating-point compare equal +name = vceq +fn = simd_eq +a = 1.2, 3.4, 5.6, 7.8 +b = 1.2, 3.4, 5.6, 7.8 +validate TRUE, TRUE, TRUE, TRUE, TRUE, TRUE + +aarch64 = fcmeq +generate float64x1_t:uint64x1_t, float64x2_t:uint64x2_t + +arm = vceq. +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t + +/// Compare bitwise equal +name = vceq +multi_fn = transmute, {vceq-in_ntt-noext, {transmute, a}, {transmute, b}} +a = 1 +b = 2 +validate 0 + +aarch64 = cmp +generate i64:u64, u64 + +/// Floating-point compare equal +name = vceq +multi_fn = simd_extract, {vceq-in_ntt-noext, {vdup_n-in_ntt-noext, a}, {vdup_n-in_ntt-noext, b}}, 0 +a = 1. +b = 2. +validate 0 + +aarch64 = fcmp +generate f32:u32, f64:u64 + +/// Signed compare bitwise equal to zero +name = vceqz +fn = simd_eq +a = MIN, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, MAX +fixed = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +validate FALSE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE + +aarch64 = cmeq +generate int8x8_t:uint8x8_t, int8x16_t:uint8x16_t, int16x4_t:uint16x4_t, int16x8_t:uint16x8_t, int32x2_t:uint32x2_t, int32x4_t:uint32x4_t, int64x1_t:uint64x1_t, int64x2_t:uint64x2_t, poly8x8_t:uint8x8_t, poly8x16_t:uint8x16_t, poly64x1_t:uint64x1_t, poly64x2_t:uint64x2_t + +/// Unsigned compare bitwise equal to zero +name = vceqz +fn = simd_eq +a = MIN, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, MAX +fixed = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +validate TRUE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE + +aarch64 = cmeq +generate uint*_t, uint64x*_t + +/// Floating-point compare bitwise equal to zero +name = vceqz +fn = simd_eq +a = 0.0, 1.2, 3.4, 5.6 +fixed = 0.0, 0.0, 0.0, 0.0 +validate TRUE, FALSE, FALSE, FALSE + +aarch64 = fcmeq +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t, float64x1_t:uint64x1_t, float64x2_t:uint64x2_t + +/// Compare bitwise equal to zero +name = vceqz +multi_fn = transmute, {vceqz-in_ntt-noext, {transmute, a}} +a = 1 +validate 0 + +aarch64 = cmp +generate i64:u64, u64 + +/// Floating-point compare bitwise equal to zero +name = vceqz +multi_fn = simd_extract, {vceqz-in_ntt-noext, {vdup_n-in_ntt-noext, a}}, 0 +a = 1. +validate 0 + +aarch64 = fcmp +generate f32:u32, f64:u64 + +/// Signed compare bitwise Test bits nonzero +name = vtst +multi_fn = simd_and, c:in_t, a, b +multi_fn = fixed, d:in_t +multi_fn = simd_ne, c, transmute(d) +a = MIN, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, MAX +b = MIN, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, MAX +fixed = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +validate TRUE, FALSE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE + +aarch64 = cmtst +generate int64x1_t:uint64x1_t, int64x2_t:uint64x2_t, poly64x1_t:uint64x1_t, poly64x2_t:uint64x2_t + +arm = vtst +generate int8x8_t:uint8x8_t, int8x16_t:uint8x16_t, int16x4_t:uint16x4_t, int16x8_t:uint16x8_t, int32x2_t:uint32x2_t, int32x4_t:uint32x4_t, poly8x8_t:uint8x8_t, poly8x16_t:uint8x16_t, poly16x4_t:uint16x4_t, poly16x8_t:uint16x8_t + +/// Unsigned compare bitwise Test bits nonzero +name = vtst +multi_fn = simd_and, c:in_t, a, b +multi_fn = fixed, d:in_t +multi_fn = simd_ne, c, transmute(d) +a = MIN, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, MAX +b = MIN, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, MAX +fixed = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +validate FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE + +aarch64 = cmtst +generate uint64x*_t + +arm = vtst +generate uint*_t + +/// Compare bitwise test bits nonzero +name = vtst +multi_fn = transmute, {vtst-in_ntt-noext, {transmute, a}, {transmute, b}} +a = 0 +b = 0 +validate 0 + +aarch64 = tst +generate i64:i64:u64, u64 + +/// Signed saturating accumulate of unsigned value +name = vuqadd +out-suffix +a = 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4 +b = 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4 +validate 2, 4, 6, 8, 2, 4, 6, 8, 2, 4, 6, 8, 2, 4, 6, 8 + +aarch64 = suqadd +link-aarch64 = suqadd._EXT_ +generate i32:u32:i32, i64:u64:i64 + +/// Signed saturating accumulate of unsigned value +name = vuqadd +out-suffix +multi_fn = simd_extract, {vuqadd-out_ntt-noext, {vdup_n-out_ntt-noext, a}, {vdup_n-in_ntt-noext, b}}, 0 +a = 1 +b = 2 +validate 3 + +aarch64 = suqadd +generate i8:u8:i8, i16:u16:i16 + +//////////////////// +// Floating-point absolute value +//////////////////// + +/// Floating-point absolute value +name = vabs +fn = simd_fabs +a = -0.1, -2.2, -3.3, -6.6 +validate 0.1, 2.2, 3.3, 6.6 +aarch64 = fabs +generate float64x1_t:float64x1_t, float64x2_t:float64x2_t + +arm = vabs +generate float32x2_t:float32x2_t, float32x4_t:float32x4_t + +//////////////////// +// greater then +//////////////////// + +/// Compare signed greater than +name = vcgt +fn = simd_gt +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE +aarch64 = cmgt +generate int64x1_t:uint64x1_t, int64x2_t:uint64x2_t + +arm = vcgt.s +generate int8x8_t:uint8x8_t, int8x16_t:uint8x16_t, int16x4_t:uint16x4_t, int16x8_t:uint16x8_t, int32x2_t:uint32x2_t, int32x4_t:uint32x4_t + +/// Compare unsigned highe +name = vcgt +fn = simd_gt +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE + +aarch64 = cmhi +generate uint64x*_t + +arm = vcgt.s +generate uint*_t + +/// Floating-point compare greater than +name = vcgt +fn = simd_gt +a = 1.2, 2.3, 3.4, 4.5, 5.6, 6.7, 7.8, 8.9 +b = 0.1, 1.2, 2.3, 3.4, 4.5, 5.6, 6.7, 7.8 +validate TRUE, TRUE, TRUE, TRUE, TRUE, TRUE + +aarch64 = fcmgt +generate float64x1_t:uint64x1_t, float64x2_t:uint64x2_t + +arm = vcgt.s +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t + +/// Compare greater than +name = vcgt +multi_fn = transmute, {vcgt-in_ntt-noext, {transmute, a}, {transmute, b}} +a = 1 +b = 2 +validate 0 + +aarch64 = cmp +generate i64:u64, u64 + +/// Floating-point compare greater than +name = vcgt +multi_fn = simd_extract, {vcgt-in_ntt-noext, {vdup_n-in_ntt-noext, a}, {vdup_n-in_ntt-noext, b}}, 0 +a = 1. +b = 2. +validate 0 + +aarch64 = fcmp +generate f32:u32, f64:u64 + +//////////////////// +// lesser then +//////////////////// + +/// Compare signed less than +name = vclt +fn = simd_lt +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +validate TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE +aarch64 = cmgt +generate int64x1_t:uint64x1_t, int64x2_t:uint64x2_t + +arm = vcgt.s +generate int8x8_t:uint8x8_t, int8x16_t:uint8x16_t, int16x4_t:uint16x4_t, int16x8_t:uint16x8_t, int32x2_t:uint32x2_t, int32x4_t:uint32x4_t + +/// Compare unsigned less than +name = vclt +fn = simd_lt +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +validate TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE + +aarch64 = cmhi +generate uint64x*_t + +arm = vcgt.s +generate uint*_t + +/// Floating-point compare less than +name = vclt +fn = simd_lt +a = 0.1, 1.2, 2.3, 3.4, 4.5, 5.6, 6.7, 7.8 +b = 1.2, 2.3, 3.4, 4.5, 5.6, 6.7, 7.8, 8.9 +validate TRUE, TRUE, TRUE, TRUE, TRUE, TRUE + +aarch64 = fcmgt +generate float64x1_t:uint64x1_t, float64x2_t:uint64x2_t + +arm = vcgt.s +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t + +/// Compare less than +name = vclt +multi_fn = transmute, {vclt-in_ntt-noext, {transmute, a}, {transmute, b}} +a = 2 +b = 1 +validate 0 + +aarch64 = cmp +generate i64:u64, u64 + +/// Floating-point compare less than +name = vclt +multi_fn = simd_extract, {vclt-in_ntt-noext, {vdup_n-in_ntt-noext, a}, {vdup_n-in_ntt-noext, b}}, 0 +a = 2. +b = 1. +validate 0 + +aarch64 = fcmp +generate f32:u32, f64:u64 + +//////////////////// +// lesser then equals +//////////////////// + +/// Compare signed less than or equal +name = vcle +fn = simd_le +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +validate TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE + +aarch64 = cmge +generate int64x1_t:uint64x1_t, int64x2_t:uint64x2_t + +arm = vcge.s +generate int8x8_t:uint8x8_t, int8x16_t:uint8x16_t, int16x4_t:uint16x4_t, int16x8_t:uint16x8_t, int32x2_t:uint32x2_t, int32x4_t:uint32x4_t + +/// Compare greater than or equal +name = vcge +multi_fn = transmute, {vcge-in_ntt-noext, {transmute, a}, {transmute, b}} +a = 1 +b = 2 +validate 0 + +aarch64 = cmp +generate i64:u64, u64 + +/// Floating-point compare greater than or equal +name = vcge +multi_fn = simd_extract, {vcge-in_ntt-noext, {vdup_n-in_ntt-noext, a}, {vdup_n-in_ntt-noext, b}}, 0 +a = 1. +b = 2. +validate 0 + +aarch64 = fcmp +generate f32:u32, f64:u64 + +/// Compare unsigned less than or equal +name = vcle +fn = simd_le +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +validate TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE + +aarch64 = cmhs +generate uint64x*_t + +arm = vcge.s +generate uint*_t + +/// Floating-point compare less than or equal +name = vcle +fn = simd_le +a = 0.1, 1.2, 2.3, 3.4, 4.5, 5.6, 6.7, 7.8 +b = 1.2, 2.3, 3.4, 4.5, 5.6, 6.7, 7.8, 8.9 +validate TRUE, TRUE, TRUE, TRUE, TRUE, TRUE +aarch64 = fcmge +generate float64x1_t:uint64x1_t, float64x2_t:uint64x2_t + +arm = vcge.s +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t + +/// Compare less than or equal +name = vcle +multi_fn = transmute, {vcle-in_ntt-noext, {transmute, a}, {transmute, b}} +a = 2 +b = 1 +validate 0 + +aarch64 = cmp +generate i64:u64, u64 + +/// Floating-point compare less than or equal +name = vcle +multi_fn = simd_extract, {vcle-in_ntt-noext, {vdup_n-in_ntt-noext, a}, {vdup_n-in_ntt-noext, b}}, 0 +a = 2. +b = 1. +validate 0 + +aarch64 = fcmp +generate f32:u32, f64:u64 + +//////////////////// +// greater then equals +//////////////////// + +/// Compare signed greater than or equal +name = vcge +fn = simd_ge +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE + +aarch64 = cmge +generate int64x1_t:uint64x1_t, int64x2_t:uint64x2_t + +arm = vcge.s +generate int8x8_t:uint8x8_t, int8x16_t:uint8x16_t, int16x4_t:uint16x4_t, int16x8_t:uint16x8_t, int32x2_t:uint32x2_t, int32x4_t:uint32x4_t + +/// Compare unsigned greater than or equal +name = vcge +fn = simd_ge +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE + +aarch64 = cmhs +generate uint64x*_t + +arm = vcge.s +generate uint*_t + +/// Floating-point compare greater than or equal +name = vcge +fn = simd_ge +a = 1.2, 2.3, 3.4, 4.5, 5.6, 6.7, 7.8, 8.9 +b = 0.1, 1.2, 2.3, 3.4, 4.5, 5.6, 6.7, 7.8 +validate TRUE, TRUE, TRUE, TRUE, TRUE, TRUE + +aarch64 = fcmge +generate float64x1_t:uint64x1_t, float64x2_t:uint64x2_t + +arm = vcge.s +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t + +/// Compare signed greater than or equal to zero +name = vcgez +fn = simd_ge +a = MIN, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, MAX +fixed = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +validate FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE + +aarch64 = cmge +generate int8x8_t:uint8x8_t, int8x16_t:uint8x16_t, int16x4_t:uint16x4_t, int16x8_t:uint16x8_t, int32x2_t:uint32x2_t, int32x4_t:uint32x4_t, int64x1_t:uint64x1_t, int64x2_t:uint64x2_t + +/// Floating-point compare greater than or equal to zero +name = vcgez +fn = simd_ge +a = -1.2, 0.0, 1.2, 2.3, 3.4, 4.5, 5.6, 6.7 +fixed = 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 +validate FALSE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE + +aarch64 = fcmge +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t, float64x1_t:uint64x1_t, float64x2_t:uint64x2_t + +/// Compare signed greater than or equal to zero +name = vcgez +multi_fn = transmute, {vcgez-in_ntt-noext, {transmute, a}} +a = -1 +validate 0 + +aarch64 = eor +generate i64:u64 + +/// Floating-point compare greater than or equal to zero +name = vcgez +multi_fn = simd_extract, {vcgez-in_ntt-noext, {vdup_n-in_ntt-noext, a}}, 0 +a = -1. +validate 0 + +aarch64 = fcmp +generate f32:u32, f64:u64 + +/// Compare signed greater than zero +name = vcgtz +fn = simd_gt +a = MIN, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, MAX +fixed = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +validate FALSE, FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE + +aarch64 = cmgt +generate int8x8_t:uint8x8_t, int8x16_t:uint8x16_t, int16x4_t:uint16x4_t, int16x8_t:uint16x8_t, int32x2_t:uint32x2_t, int32x4_t:uint32x4_t, int64x1_t:uint64x1_t, int64x2_t:uint64x2_t + +/// Floating-point compare greater than zero +name = vcgtz +fn = simd_gt +a = -1.2, 0.0, 1.2, 2.3, 3.4, 4.5, 5.6, 6.7 +fixed = 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 +validate FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE + +aarch64 = fcmgt +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t, float64x1_t:uint64x1_t, float64x2_t:uint64x2_t + +/// Compare signed greater than zero +name = vcgtz +multi_fn = transmute, {vcgtz-in_ntt-noext, {transmute, a}} +a = -1 +validate 0 + +aarch64 = cmp +generate i64:u64 + +/// Floating-point compare greater than zero +name = vcgtz +multi_fn = simd_extract, {vcgtz-in_ntt-noext, {vdup_n-in_ntt-noext, a}}, 0 +a = -1. +validate 0 + +aarch64 = fcmp +generate f32:u32, f64:u64 + +/// Compare signed less than or equal to zero +name = vclez +fn = simd_le +a = MIN, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, MAX +fixed = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +validate TRUE, TRUE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE + +aarch64 = cmgt +generate int8x8_t:uint8x8_t, int8x16_t:uint8x16_t, int16x4_t:uint16x4_t, int16x8_t:uint16x8_t, int32x2_t:uint32x2_t, int32x4_t:uint32x4_t, int64x1_t:uint64x1_t, int64x2_t:uint64x2_t + +/// Floating-point compare less than or equal to zero +name = vclez +fn = simd_le +a = -1.2, 0.0, 1.2, 2.3, 3.4, 4.5, 5.6, 6.7 +fixed = 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 +validate TRUE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE + +aarch64 = fcmle +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t, float64x1_t:uint64x1_t, float64x2_t:uint64x2_t + +/// Compare less than or equal to zero +name = vclez +multi_fn = transmute, {vclez-in_ntt-noext, {transmute, a}} +a = 2 +validate 0 + +aarch64 = cmp +generate i64:u64 + +/// Floating-point compare less than or equal to zero +name = vclez +multi_fn = simd_extract, {vclez-in_ntt-noext, {vdup_n-in_ntt-noext, a}}, 0 +a = 2. +validate 0 + +aarch64 = fcmp +generate f32:u32, f64:u64 + +/// Compare signed less than zero +name = vcltz +fn = simd_lt +a = MIN, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, MAX +fixed = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +validate TRUE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE + +aarch64 = cmlt +generate int8x8_t:uint8x8_t, int8x16_t:uint8x16_t, int16x4_t:uint16x4_t, int16x8_t:uint16x8_t, int32x2_t:uint32x2_t, int32x4_t:uint32x4_t, int64x1_t:uint64x1_t, int64x2_t:uint64x2_t + +/// Floating-point compare less than zero +name = vcltz +fn = simd_lt +a = -1.2, 0.0, 1.2, 2.3, 3.4, 4.5, 5.6, 6.7 +fixed = 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 +validate TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE + +aarch64 = fcmlt +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t, float64x1_t:uint64x1_t, float64x2_t:uint64x2_t + +/// Compare less than zero +name = vcltz +multi_fn = transmute, {vcltz-in_ntt-noext, {transmute, a}} +a = 2 +validate 0 + +aarch64 = asr +generate i64:u64 + +/// Floating-point compare less than zero +name = vcltz +multi_fn = simd_extract, {vcltz-in_ntt-noext, {vdup_n-in_ntt-noext, a}}, 0 +a = 2. +validate 0 + +aarch64 = fcmp +generate f32:u32, f64:u64 + +/// Count leading sign bits +name = vcls +a = MIN, -1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, MAX +validate 0, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, 0 + +arm = vcls.s +aarch64 = cls +link-arm = vcls._EXT_ +link-aarch64 = cls._EXT_ +generate int*_t + +/// Count leading sign bits +name = vcls +multi_fn = transmute, {vcls-signed-noext, {transmute, a}} +a = MIN, MAX, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, MAX +validate BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1 + +arm = vcls +aarch64 = cls +generate uint8x8_t:int8x8_t, uint8x16_t:int8x16_t, uint16x4_t:int16x4_t, uint16x8_t:int16x8_t, uint32x2_t:int32x2_t, uint32x4_t:int32x4_t + +/// Count leading zero bits +name = vclz +multi_fn = self-signed-ext, a +a = MIN, -1, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, MAX +validate 0, 0, BITS, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, 1 + +arm = vclz. +aarch64 = clz +generate int*_t + +/// Count leading zero bits +name = vclz +multi_fn = transmute, {self-signed-ext, transmute(a)} +a = MIN, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, MAX +validate BITS, BITS, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, 0 + +arm = vclz. +aarch64 = clz +generate uint*_t + +/// Floating-point absolute compare greater than +name = vcagt +a = -1.2, 0.0, 1.2, 2.3, 3.4, 4.5, 5.6, 6.7 +b = -1.1, 0.0, 1.1, 2.4, 3.3, 4.6, 5.5, 6.8 +validate !0, FALSE, TRUE, FALSE, TRUE, FALSE, TRUE, FALSE + +aarch64 = facgt +link-aarch64 = facgt._EXT2_._EXT_ +generate float64x1_t:uint64x1_t, float64x2_t:uint64x2_t, f32:u32, f64:u64 + +arm = vacgt.s +link-arm = vacgt._EXT2_._EXT_ +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t + +/// Floating-point absolute compare greater than or equal +name = vcage +a = -1.2, 0.0, 1.2, 2.3, 3.4, 4.5, 5.6, 6.7 +b = -1.1, 0.0, 1.1, 2.4, 3.3, 4.6, 5.5, 6.8 +validate !0, TRUE, TRUE, FALSE, TRUE, FALSE, TRUE, FALSE + +aarch64 = facge +link-aarch64 = facge._EXT2_._EXT_ +generate float64x1_t:uint64x1_t, float64x2_t:uint64x2_t, f32:u32, f64:u64 + +arm = vacge.s +link-arm = vacge._EXT2_._EXT_ +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t + +/// Floating-point absolute compare less than +name = vcalt +multi_fn = vcagt-self-noext, b, a +a = -1.2, 0.0, 1.2, 2.3, 3.4, 4.5, 5.6, 6.7 +b = -1.1, 0.0, 1.1, 2.4, 3.3, 4.6, 5.5, 6.8 +validate 0, FALSE, FALSE, TRUE, FALSE, TRUE, FALSE, TRUE + +aarch64 = facgt +generate float64x1_t:uint64x1_t, float64x2_t:uint64x2_t, f32:u32, f64:u64 + +arm = vacgt.s +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t + +/// Floating-point absolute compare less than or equal +name = vcale +multi_fn = vcage-self-noext , b, a +a = -1.2, 0.0, 1.2, 2.3, 3.4, 4.5, 5.6, 6.7 +b = -1.1, 0.0, 1.1, 2.4, 3.3, 4.6, 5.5, 6.8 +validate 0, TRUE, FALSE, TRUE, FALSE, TRUE, FALSE, TRUE + +aarch64 = facge +generate float64x1_t:uint64x1_t, float64x2_t:uint64x2_t, f32:u32, f64:u64 + +arm = vacge.s +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t + +/// Insert vector element from another vector element +name = vcopy +lane-suffixes +constn = LANE1:LANE2 +multi_fn = static_assert_imm-in0_exp_len-LANE1 +multi_fn = static_assert_imm-in_exp_len-LANE2 +multi_fn = matchn-in0_exp_len-LANE1, simd_shuffle-out_len-!, a, b, {ins-in0_len-in0_len-LANE2} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 0, MAX, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 0:1 +validate MAX, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +aarch64 = mov +generate int8x8_t, int8x16_t, int16x4_t, int16x8_t, int32x2_t, int32x4_t, int64x2_t +generate uint8x8_t, uint8x16_t, uint16x4_t, uint16x8_t, uint32x2_t, uint32x4_t, uint64x2_t +generate poly8x8_t, poly8x16_t, poly16x4_t, poly16x8_t, poly64x2_t + +/// Insert vector element from another vector element +name = vcopy +lane-suffixes +constn = LANE1:LANE2 +multi_fn = static_assert_imm-in0_exp_len-LANE1 +multi_fn = static_assert_imm-in_exp_len-LANE2 +multi_fn = matchn-in0_exp_len-LANE1, simd_shuffle-out_len-!, a, b, {ins-in0_len-in0_len-LANE2} +a = 1., 2., 3., 4. +b = 0., 0.5, 0., 0. +n = 0:1 +validate 0.5, 2., 3., 4. + +aarch64 = mov +generate float32x2_t, float32x4_t, float64x2_t + +/// Insert vector element from another vector element +name = vcopy +lane-suffixes +constn = LANE1:LANE2 +multi_fn = static_assert_imm-in0_exp_len-LANE1 +multi_fn = static_assert_imm-in_exp_len-LANE2 +multi_fn = simd_shuffle-in_len-!, a:in_t, a, a, {asc-0-in_len} +multi_fn = matchn-in0_exp_len-LANE1, simd_shuffle-out_len-!, a, b, {ins-in0_len-in_len-LANE2} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 0, MAX, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 0:1 +validate MAX, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +aarch64 = mov +generate int8x8_t:int8x16_t:int8x8_t, int16x4_t:int16x8_t:int16x4_t, int32x2_t:int32x4_t:int32x2_t +generate uint8x8_t:uint8x16_t:uint8x8_t, uint16x4_t:uint16x8_t:uint16x4_t, uint32x2_t:uint32x4_t:uint32x2_t +generate poly8x8_t:poly8x16_t:poly8x8_t, poly16x4_t:poly16x8_t:poly16x4_t + +/// Insert vector element from another vector element +name = vcopy +lane-suffixes +constn = LANE1:LANE2 +multi_fn = static_assert_imm-in0_exp_len-LANE1 +multi_fn = static_assert_imm-in_exp_len-LANE2 +multi_fn = simd_shuffle-in_len-!, a:in_t, a, a, {asc-0-in_len} +multi_fn = matchn-in0_exp_len-LANE1, simd_shuffle-out_len-!, a, b, {ins-in0_len-in_len-LANE2} +a = 1., 2., 3., 4. +b = 0., 0.5, 0., 0. +n = 0:1 +validate 0.5, 2., 3., 4. + +aarch64 = mov +generate float32x2_t:float32x4_t:float32x2_t + +/// Insert vector element from another vector element +name = vcopy +lane-suffixes +constn = LANE1:LANE2 +multi_fn = static_assert_imm-in0_exp_len-LANE1 +multi_fn = static_assert_imm-in_exp_len-LANE2 +multi_fn = simd_shuffle-in0_len-!, b:in_t0, b, b, {asc-0-in0_len} +multi_fn = matchn-in0_exp_len-LANE1, simd_shuffle-out_len-!, a, b, {ins-in0_len-in0_len-LANE2} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 0, MAX, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 0:1 +validate MAX, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +aarch64 = mov +generate int8x16_t:int8x8_t:int8x16_t, int16x8_t:int16x4_t:int16x8_t, int32x4_t:int32x2_t:int32x4_t +generate uint8x16_t:uint8x8_t:uint8x16_t, uint16x8_t:uint16x4_t:uint16x8_t, uint32x4_t:uint32x2_t:uint32x4_t +generate poly8x16_t:poly8x8_t:poly8x16_t, poly16x8_t:poly16x4_t:poly16x8_t + +/// Insert vector element from another vector element +name = vcopy +lane-suffixes +constn = LANE1:LANE2 +multi_fn = static_assert_imm-in0_exp_len-LANE1 +multi_fn = static_assert_imm-in_exp_len-LANE2 +multi_fn = simd_shuffle-in0_len-!, b:in_t0, b, b, {asc-0-in0_len} +multi_fn = matchn-in0_exp_len-LANE1, simd_shuffle-out_len-!, a, b, {ins-in0_len-in0_len-LANE2} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = MAX, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 1:0 +validate 1, MAX + +aarch64 = mov +generate int64x2_t:int64x1_t:int64x2_t, uint64x2_t:uint64x1_t:uint64x2_t, poly64x2_t:poly64x1_t:poly64x2_t + +/// Insert vector element from another vector element +name = vcopy +lane-suffixes +constn = LANE1:LANE2 +multi_fn = static_assert_imm-in0_exp_len-LANE1 +multi_fn = static_assert_imm-in_exp_len-LANE2 +multi_fn = simd_shuffle-in0_len-!, b:in_t0, b, b, {asc-0-in0_len} +multi_fn = matchn-in0_exp_len-LANE1, simd_shuffle-out_len-!, a, b, {ins-in0_len-in0_len-LANE2} +a = 1., 2., 3., 4. +b = 0.5, 0., 0., 0. +n = 1:0 +validate 1., 0.5, 3., 4. + +aarch64 = mov +generate float32x4_t:float32x2_t:float32x4_t +aarch64 = mov +generate float64x2_t:float64x1_t:float64x2_t + +/// Insert vector element from another vector element +name = vcreate +out-suffix +multi_fn = transmute, a +a = 1 +validate 1, 0, 0, 0, 0, 0, 0, 0 + +aarch64 = nop +arm = nop +generate u64:int8x8_t, u64:int16x4_t, u64:int32x2_t, u64:int64x1_t +generate u64:uint8x8_t, u64:uint16x4_t, u64:uint32x2_t, u64:uint64x1_t +generate u64:poly8x8_t, u64:poly16x4_t +target = aes +generate u64:poly64x1_t + +/// Insert vector element from another vector element +name = vcreate +out-suffix +multi_fn = transmute, a +a = 0 +validate 0., 0. + +aarch64 = nop +generate u64:float64x1_t +arm = nop +generate u64:float32x2_t + +/// Fixed-point convert to floating-point +name = vcvt +double-suffixes +fn = simd_cast +a = 1, 2, 3, 4 +validate 1., 2., 3., 4. + +aarch64 = scvtf +generate int64x1_t:float64x1_t, int64x2_t:float64x2_t +aarch64 = ucvtf +generate uint64x1_t:float64x1_t, uint64x2_t:float64x2_t + +arm = vcvt +aarch64 = scvtf +generate int32x2_t:float32x2_t, int32x4_t:float32x4_t +aarch64 = ucvtf +generate uint32x2_t:float32x2_t, uint32x4_t:float32x4_t + +/// Floating-point convert to higher precision long +name = vcvt +double-suffixes +fn = simd_cast +a = -1.2, 1.2 +validate -1.2f32 as f64, 1.2f32 as f64 + +aarch64 = fcvtl +generate float32x2_t:float64x2_t + +/// Floating-point convert to higher precision long +name = vcvt_high +noq-double-suffixes +multi_fn = simd_shuffle2!, b:float32x2_t, a, a, [2, 3] +multi_fn = simd_cast, b +a = -1.2, 1.2, 2.3, 3.4 +validate 2.3f32 as f64, 3.4f32 as f64 + +aarch64 = fcvtl +generate float32x4_t:float64x2_t + +/// Floating-point convert to lower precision narrow +name = vcvt +double-suffixes +fn = simd_cast +a = -1.2, 1.2 +validate -1.2f64 as f32, 1.2f64 as f32 + +aarch64 = fcvtn +generate float64x2_t:float32x2_t + +/// Floating-point convert to lower precision narrow +name = vcvt_high +noq-double-suffixes +multi_fn = simd_shuffle4!, a, {simd_cast, b}, [0, 1, 2, 3] +a = -1.2, 1.2 +b = -2.3, 3.4 +validate -1.2, 1.2, -2.3f64 as f32, 3.4f64 as f32 + +aarch64 = fcvtn +generate float32x2_t:float64x2_t:float32x4_t + +/// Floating-point convert to lower precision narrow, rounding to odd +name = vcvtx +double-suffixes +a = -1.0, 2.0 +validate -1.0, 2.0 + +aarch64 = fcvtxn +link-aarch64 = fcvtxn._EXT2_._EXT_ +generate float64x2_t:float32x2_t + +/// Floating-point convert to lower precision narrow, rounding to odd +name = vcvtx +double-suffixes +multi_fn = simd_extract, {vcvtx-_f32_f64-noext, {vdupq_n-in_ntt-noext, a}}, 0 +a = -1.0 +validate -1.0 + +aarch64 = fcvtxn +generate f64:f32 + +/// Floating-point convert to lower precision narrow, rounding to odd +name = vcvtx_high +noq-double-suffixes +multi_fn = simd_shuffle4!, a, {vcvtx-noq_doubleself-noext, b}, [0, 1, 2, 3] +a = -1.0, 2.0 +b = -3.0, 4.0 +validate -1.0, 2.0, -3.0, 4.0 + +aarch64 = fcvtxn +generate float32x2_t:float64x2_t:float32x4_t + +/// Fixed-point convert to floating-point +name = vcvt +double-n-suffixes +constn = N +multi_fn = static_assert-N-1-bits +a = 1, 2, 3, 4 +n = 2 +validate 0.25, 0.5, 0.75, 1. +arm-aarch64-separate + +aarch64 = scvtf +link-aarch64 = vcvtfxs2fp._EXT2_._EXT_ +const-aarch64 = N +generate int64x1_t:float64x1_t, int64x2_t:float64x2_t, i32:f32, i64:f64 + +aarch64 = ucvtf +link-aarch64 = vcvtfxu2fp._EXT2_._EXT_ +const-aarch64 = N +generate uint64x1_t:float64x1_t, uint64x2_t:float64x2_t, u32:f32, u64:f64 + +aarch64 = scvtf +link-aarch64 = vcvtfxs2fp._EXT2_._EXT_ +arm = vcvt +link-arm = vcvtfxs2fp._EXT2_._EXT_ +const-arm = N:i32 + +generate int32x2_t:float32x2_t, int32x4_t:float32x4_t + +aarch64 = ucvtf +link-aarch64 = vcvtfxu2fp._EXT2_._EXT_ +arm = vcvt +link-arm = vcvtfxu2fp._EXT2_._EXT_ +const-arm = N:i32 +generate uint32x2_t:float32x2_t, uint32x4_t:float32x4_t + +/// Floating-point convert to fixed-point, rounding toward zero +name = vcvt +double-n-suffixes +constn = N +multi_fn = static_assert-N-1-bits +a = 0.25, 0.5, 0.75, 1. +n = 2 +validate 1, 2, 3, 4 +arm-aarch64-separate + +aarch64 = fcvtzs +link-aarch64 = vcvtfp2fxs._EXT2_._EXT_ +const-aarch64 = N +generate float64x1_t:int64x1_t, float64x2_t:int64x2_t, f32:i32, f64:i64 + +aarch64 = fcvtzu +link-aarch64 = vcvtfp2fxu._EXT2_._EXT_ +const-aarch64 = N +generate float64x1_t:uint64x1_t, float64x2_t:uint64x2_t, f32:u32, f64:u64 + +aarch64 = fcvtzs +link-aarch64 = vcvtfp2fxs._EXT2_._EXT_ +arm = vcvt +link-arm = vcvtfp2fxs._EXT2_._EXT_ +const-arm = N:i32 +generate float32x2_t:int32x2_t, float32x4_t:int32x4_t + +aarch64 = fcvtzu +link-aarch64 = vcvtfp2fxu._EXT2_._EXT_ +arm = vcvt +link-arm = vcvtfp2fxu._EXT2_._EXT_ +const-arm = N:i32 +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t + +/// Fixed-point convert to floating-point +name = vcvt +double-suffixes +multi_fn = a as out_t +a = 1 +validate 1. + +aarch64 = scvtf +generate i32:f32, i64:f64 +aarch64 = ucvtf +generate u32:f32, u64:f64 + +/// Fixed-point convert to floating-point +name = vcvt +double-suffixes +multi_fn = a as out_t +a = 1. +validate 1 + +aarch64 = fcvtzs +generate f32:i32, f64:i64 +aarch64 = fcvtzu +generate f32:u32, f64:u64 + +/// Floating-point convert to signed fixed-point, rounding toward zero +name = vcvt +double-suffixes +link-aarch64 = llvm.fptosi.sat._EXT2_._EXT_ +a = -1.1, 2.1, -2.9, 3.9 +validate -1, 2, -2, 3 + +aarch64 = fcvtzs +generate float64x1_t:int64x1_t, float64x2_t:int64x2_t + +link-arm = llvm.fptosi.sat._EXT2_._EXT_ +arm = vcvt +generate float32x2_t:int32x2_t, float32x4_t:int32x4_t + +/// Floating-point convert to unsigned fixed-point, rounding toward zero +name = vcvt +double-suffixes +link-aarch64 = llvm.fptoui.sat._EXT2_._EXT_ +a = 1.1, 2.1, 2.9, 3.9 +validate 1, 2, 2, 3 + +aarch64 = fcvtzu +generate float64x1_t:uint64x1_t, float64x2_t:uint64x2_t + +link-arm = llvm.fptoui.sat._EXT2_._EXT_ +arm = vcvt +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t + +/// Floating-point convert to signed integer, rounding to nearest with ties to away +name = vcvta +double-suffixes +a = -1.1, 2.1, -2.9, 3.9 +validate -1, 2, -3, 4 + +aarch64 = fcvtas +link-aarch64 = fcvtas._EXT2_._EXT_ +generate float32x2_t:int32x2_t, float32x4_t:int32x4_t, float64x1_t:int64x1_t, float64x2_t:int64x2_t + +/// Floating-point convert to integer, rounding to nearest with ties to away +name = vcvta +double-suffixes +a = 2.9 +validate 3 + +aarch64 = fcvtas +link-aarch64 = fcvtas._EXT2_._EXT_ +generate f32:i32, f64:i64 + +aarch64 = fcvtau +link-aarch64 = fcvtau._EXT2_._EXT_ +generate f32:u32, f64:u64 + +/// Floating-point convert to signed integer, rounding to nearest with ties to even +name = vcvtn +double-suffixes +a = -1.5, 2.1, -2.9, 3.9 +validate -2, 2, -3, 4 + +aarch64 = fcvtns +link-aarch64 = fcvtns._EXT2_._EXT_ +generate float32x2_t:int32x2_t, float32x4_t:int32x4_t, float64x1_t:int64x1_t, float64x2_t:int64x2_t, f32:i32, f64:i64 + +/// Floating-point convert to signed integer, rounding toward minus infinity +name = vcvtm +double-suffixes +a = -1.1, 2.1, -2.9, 3.9 +validate -2, 2, -3, 3 + +aarch64 = fcvtms +link-aarch64 = fcvtms._EXT2_._EXT_ +generate float32x2_t:int32x2_t, float32x4_t:int32x4_t, float64x1_t:int64x1_t, float64x2_t:int64x2_t, f32:i32, f64:i64 + +/// Floating-point convert to signed integer, rounding toward plus infinity +name = vcvtp +double-suffixes +a = -1.1, 2.1, -2.9, 3.9 +validate -1, 3, -2, 4 + +aarch64 = fcvtps +link-aarch64 = fcvtps._EXT2_._EXT_ +generate float32x2_t:int32x2_t, float32x4_t:int32x4_t, float64x1_t:int64x1_t, float64x2_t:int64x2_t, f32:i32, f64:i64 + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to away +name = vcvta +double-suffixes +a = 1.1, 2.1, 2.9, 3.9 +validate 1, 2, 3, 4 + +aarch64 = fcvtau +link-aarch64 = fcvtau._EXT2_._EXT_ +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t, float64x1_t:uint64x1_t, float64x2_t:uint64x2_t + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to even +name = vcvtn +double-suffixes +a = 1.5, 2.1, 2.9, 3.9 +validate 2, 2, 3, 4 + +aarch64 = fcvtnu +link-aarch64 = fcvtnu._EXT2_._EXT_ +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t, float64x1_t:uint64x1_t, float64x2_t:uint64x2_t, f32:u32, f64:u64 + +/// Floating-point convert to unsigned integer, rounding toward minus infinity +name = vcvtm +double-suffixes +a = 1.1, 2.1, 2.9, 3.9 +validate 1, 2, 2, 3 + +aarch64 = fcvtmu +link-aarch64 = fcvtmu._EXT2_._EXT_ +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t, float64x1_t:uint64x1_t, float64x2_t:uint64x2_t, f32:u32, f64:u64 + +/// Floating-point convert to unsigned integer, rounding toward plus infinity +name = vcvtp +double-suffixes +a = 1.1, 2.1, 2.9, 3.9 +validate 2, 3, 3, 4 + +aarch64 = fcvtpu +link-aarch64 = fcvtpu._EXT2_._EXT_ +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t, float64x1_t:uint64x1_t, float64x2_t:uint64x2_t, f32:u32, f64:u64 + +/// Set all vector lanes to the same value +name = vdup +lane-suffixes +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_shuffle-out_len-!, a, a, {dup-out_len-N as u32} +a = 1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16 +n = HFLEN +validate 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + +aarch64 = dup +generate poly64x2_t, poly64x1_t:poly64x2_t + +arm = vdup.l +generate int*_t +generate int8x16_t:int8x8_t, int16x8_t:int16x4_t, int32x4_t:int32x2_t +generate int8x8_t:int8x16_t, int16x4_t:int16x8_t, int32x2_t:int32x4_t + +generate uint*_t +generate uint8x16_t:uint8x8_t, uint16x8_t:uint16x4_t, uint32x4_t:uint32x2_t +generate uint8x8_t:uint8x16_t, uint16x4_t:uint16x8_t, uint32x2_t:uint32x4_t + +generate poly8x8_t, poly8x16_t, poly16x4_t, poly16x8_t +generate poly8x16_t:poly8x8_t, poly16x8_t:poly16x4_t +generate poly8x8_t:poly8x16_t, poly16x4_t:poly16x8_t + +/// Set all vector lanes to the same value +name = vdup +lane-suffixes +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_shuffle-out_len-!, a, a, {dup-out_len-N as u32} +a = 1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16 +n = HFLEN +validate 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + +aarch64 = dup +arm = vmov +generate int64x2_t, int64x1_t:int64x2_t, uint64x2_t, uint64x1_t:uint64x2_t + +/// Set all vector lanes to the same value +name = vdup +lane-suffixes +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_shuffle-out_len-!, a, a, {dup-out_len-N as u32} +a = 1., 1., 1., 4. +n = HFLEN +validate 1., 1., 1., 1. + +aarch64 = dup +generate float64x2_t, float64x1_t:float64x2_t + +arm = vdup.l +generate float*_t, float32x4_t:float32x2_t, float32x2_t:float32x4_t + +/// Set all vector lanes to the same value +name = vdup +lane-suffixes +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = a +a = 0 +n = HFLEN +validate 0 + +aarch64 = nop +generate poly64x1_t + +arm = nop +generate int64x1_t, uint64x1_t + +/// Set all vector lanes to the same value +name = vdup +lane-suffixes +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = a +a = 0. +n = HFLEN +validate 0. + +aarch64 = nop +generate float64x1_t + +/// Set all vector lanes to the same value +name = vdup +lane-suffixes +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = transmute--, {simd_extract, a, N as u32} +a = 0, 1 +n = HFLEN +validate 1 + +aarch64 = nop +generate poly64x2_t:poly64x1_t + +arm = vmov +generate int64x2_t:int64x1_t, uint64x2_t:uint64x1_t + +/// Set all vector lanes to the same value +name = vdup +lane-suffixes +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = transmute--, {simd_extract, a, N as u32} +a = 0., 1. +n = HFLEN +validate 1. + +aarch64 = nop +generate float64x2_t:float64x1_t + +/// Set all vector lanes to the same value +name = vdup +lane-suffixes +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_extract, a, N as u32 +a = 1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16 +n = HFLEN +validate 1 + +aarch64 = nop +generate int8x8_t:i8, int8x16_t:i8, int16x4_t:i16, int16x8_t:i16, int32x2_t:i32, int32x4_t:i32, int64x1_t:i64, int64x2_t:i64 +generate uint8x8_t:u8, uint8x16_t:u8, uint16x4_t:u16, uint16x8_t:u16, uint32x2_t:u32, uint32x4_t:u32, uint64x1_t:u64, uint64x2_t:u64 +generate poly8x8_t:p8, poly8x16_t:p8, poly16x4_t:p16, poly16x8_t:p16 + +/// Set all vector lanes to the same value +name = vdup +lane-suffixes +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_extract, a, N as u32 +a = 1., 1., 1., 4. +n = HFLEN +validate 1. + +aarch64 = nop +generate float32x2_t:f32, float32x4_t:f32, float64x1_t:f64, float64x2_t:f64 + +/// Extract vector from pair of vectors +name = vext +constn = N +multi_fn = static_assert_imm-out_exp_len-N +multi_fn = matchn-out_exp_len-N, simd_shuffle-out_len-!, a, b, {asc-n-out_len} +a = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +n = LEN_M1 +validate 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 + +arm = "vext.8" +aarch64 = ext +generate int*_t, uint*_t, poly8x8_t, poly8x16_t, poly16x4_t, poly16x8_t + +/// Extract vector from pair of vectors +name = vext +constn = N +multi_fn = static_assert_imm-out_exp_len-N +multi_fn = matchn-out_exp_len-N, simd_shuffle-out_len-!, a, b, {asc-n-out_len} +a = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +n = LEN_M1 +validate 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 + +aarch64 = ext +generate poly64x2_t + +arm = vmov +generate int64x2_t, uint64x2_t + +/// Extract vector from pair of vectors +name = vext +constn = N +multi_fn = static_assert_imm-out_exp_len-N +multi_fn = matchn-out_exp_len-N, simd_shuffle-out_len-!, a, b, {asc-n-out_len} +a = 1., 1., 1., 1. +b = 2., 2., 2., 2., +n = LEN_M1 +validate 1., 2., 2., 2. + +aarch64 = ext +generate float64x2_t + +arm = "vext.8" +generate float*_t + +/// Multiply-add to accumulator +name = vmla +multi_fn = simd_add, a, {simd_mul, b, c} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 +validate 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 + +arm = vmla. +aarch64 = mla +generate int*_t, uint*_t + +/// Floating-point multiply-add to accumulator +name = vmla +multi_fn = simd_add, a, {simd_mul, b, c} +a = 0., 1., 2., 3. +b = 2., 2., 2., 2. +c = 3., 3., 3., 3. +validate 6., 7., 8., 9. + +aarch64 = fmul +generate float64x*_t + +arm = vmla. +generate float*_t + +/// Vector multiply accumulate with scalar +name = vmla +n-suffix +multi_fn = vmla-self-noext, a, b, {vdup-nself-noext, c} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3 +validate 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 + +aarch64 = mla +arm = vmla. +generate int16x4_t:int16x4_t:i16:int16x4_t, int16x8_t:int16x8_t:i16:int16x8_t, int32x2_t:int32x2_t:i32:int32x2_t, int32x4_t:int32x4_t:i32:int32x4_t +generate uint16x4_t:uint16x4_t:u16:uint16x4_t, uint16x8_t:uint16x8_t:u16:uint16x8_t, uint32x2_t:uint32x2_t:u32:uint32x2_t, uint32x4_t:uint32x4_t:u32:uint32x4_t + +/// Vector multiply accumulate with scalar +name = vmla +n-suffix +multi_fn = vmla-self-noext, a, b, {vdup-nself-noext, c} +a = 0., 1., 2., 3. +b = 2., 2., 2., 2. +c = 3. +validate 6., 7., 8., 9. + +aarch64 = fmul +arm = vmla. +generate float32x2_t:float32x2_t:f32:float32x2_t, float32x4_t:float32x4_t:f32:float32x4_t + +/// Vector multiply accumulate with scalar +name = vmla +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vmla-self-noext, a, b, {simd_shuffle-in_len-!, c, c, {dup-in_len-LANE as u32}} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 1 +validate 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 + +aarch64 = mla +arm = vmla. +generate int16x4_t, int16x4_t:int16x4_t:int16x8_t:int16x4_t, int16x8_t:int16x8_t:int16x4_t:int16x8_t, int16x8_t +generate int32x2_t, int32x2_t:int32x2_t:int32x4_t:int32x2_t, int32x4_t:int32x4_t:int32x2_t:int32x4_t, int32x4_t +generate uint16x4_t, uint16x4_t:uint16x4_t:uint16x8_t:uint16x4_t, uint16x8_t:uint16x8_t:uint16x4_t:uint16x8_t, uint16x8_t +generate uint32x2_t, uint32x2_t:uint32x2_t:uint32x4_t:uint32x2_t, uint32x4_t:uint32x4_t:uint32x2_t:uint32x4_t, uint32x4_t + +/// Vector multiply accumulate with scalar +name = vmla +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vmla-self-noext, a, b, {simd_shuffle-in_len-!, c, c, {dup-in_len-LANE as u32}} +a = 0., 1., 2., 3. +b = 2., 2., 2., 2. +c = 0., 3., 0., 0. +n = 1 +validate 6., 7., 8., 9. + +aarch64 = fmul +arm = vmla. +generate float32x2_t, float32x2_t:float32x2_t:float32x4_t:float32x2_t, float32x4_t:float32x4_t:float32x2_t:float32x4_t, float32x4_t + +/// Signed multiply-add long +name = vmlal +multi_fn = simd_add, a, {vmull-self-noext, b, c} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 +validate 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 + +arm = vmlal.s +aarch64 = smlal +generate int16x8_t:int8x8_t:int8x8_t:int16x8_t, int32x4_t:int16x4_t:int16x4_t:int32x4_t, int64x2_t:int32x2_t:int32x2_t:int64x2_t + +/// Unsigned multiply-add long +name = vmlal +multi_fn = simd_add, a, {vmull-self-noext, b, c} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 +validate 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 + +arm = vmlal.s +aarch64 = umlal +generate uint16x8_t:uint8x8_t:uint8x8_t:uint16x8_t, uint32x4_t:uint16x4_t:uint16x4_t:uint32x4_t, uint64x2_t:uint32x2_t:uint32x2_t:uint64x2_t + +/// Vector widening multiply accumulate with scalar +name = vmlal +n-suffix +multi_fn = vmlal-self-noext, a, b, {vdup-nself-noext, c} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3 +validate 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 + +arm = vmlal.s +aarch64 = smlal +generate int32x4_t:int16x4_t:i16:int32x4_t, int64x2_t:int32x2_t:i32:int64x2_t +aarch64 = umlal +generate uint32x4_t:uint16x4_t:u16:uint32x4_t, uint64x2_t:uint32x2_t:u32:uint64x2_t + +/// Vector widening multiply accumulate with scalar +name = vmlal_lane +in2-suffix +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vmlal-self-noext, a, b, {simd_shuffle-in_len-!, c, c, {dup-in_len-LANE as u32}} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 1 +validate 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 + +arm = vmlal.s +aarch64 = smlal +generate int32x4_t:int16x4_t:int16x4_t:int32x4_t, int32x4_t:int16x4_t:int16x8_t:int32x4_t +generate int64x2_t:int32x2_t:int32x2_t:int64x2_t, int64x2_t:int32x2_t:int32x4_t:int64x2_t +aarch64 = umlal +generate uint32x4_t:uint16x4_t:uint16x4_t:uint32x4_t, uint32x4_t:uint16x4_t:uint16x8_t:uint32x4_t +generate uint64x2_t:uint32x2_t:uint32x2_t:uint64x2_t, uint64x2_t:uint32x2_t:uint32x4_t:uint64x2_t + +/// Signed multiply-add long +name = vmlal_high +no-q +multi_fn = simd_shuffle-out_len-!, b:half, b, b, {fixed-half-right} +multi_fn = simd_shuffle-out_len-!, c:half, c, c, {fixed-half-right} +multi_fn = vmlal-noqself-noext, a, b, c +a = 8, 7, 6, 5, 4, 3, 2, 1 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7 +fixed = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = smlal2 +generate int16x8_t:int8x16_t:int8x16_t:int16x8_t, int32x4_t:int16x8_t:int16x8_t:int32x4_t, int64x2_t:int32x4_t:int32x4_t:int64x2_t + +/// Unsigned multiply-add long +name = vmlal_high +no-q +multi_fn = simd_shuffle-out_len-!, b:half, b, b, {fixed-half-right} +multi_fn = simd_shuffle-out_len-!, c:half, c, c, {fixed-half-right} +multi_fn = vmlal-noqself-noext, a, b, c +a = 8, 7, 6, 5, 4, 3, 2, 1 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7 +fixed = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = umlal2 +generate uint16x8_t:uint8x16_t:uint8x16_t:uint16x8_t, uint32x4_t:uint16x8_t:uint16x8_t:uint32x4_t, uint64x2_t:uint32x4_t:uint32x4_t:uint64x2_t + +/// Multiply-add long +name = vmlal_high_n +no-q +multi_fn = vmlal_high-noqself-noext, a, b, {vdupq_n-noqself-noext, c} +a = 8, 7, 6, 5, 4, 3, 2, 1 +b = 3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7 +c = 2 +validate 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = smlal2 +generate int32x4_t:int16x8_t:i16:int32x4_t, int64x2_t:int32x4_t:i32:int64x2_t +aarch64 = umlal2 +generate uint32x4_t:uint16x8_t:u16:uint32x4_t, uint64x2_t:uint32x4_t:u32:uint64x2_t + +/// Multiply-add long +name = vmlal_high_lane +in2-suffix +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vmlal_high-noqself-noext, a, b, {simd_shuffle-in_len-!, c, c, {dup-in_len-LANE as u32}} +a = 8, 7, 6, 5, 4, 3, 2, 1 +b = 3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7 +c = 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 1 +validate 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = smlal2 +generate int32x4_t:int16x8_t:int16x4_t:int32x4_t, int32x4_t:int16x8_t:int16x8_t:int32x4_t +generate int64x2_t:int32x4_t:int32x2_t:int64x2_t, int64x2_t:int32x4_t:int32x4_t:int64x2_t +aarch64 = umlal2 +generate uint32x4_t:uint16x8_t:uint16x4_t:uint32x4_t, uint32x4_t:uint16x8_t:uint16x8_t:uint32x4_t +generate uint64x2_t:uint32x4_t:uint32x2_t:uint64x2_t, uint64x2_t:uint32x4_t:uint32x4_t:uint64x2_t + +/// Multiply-subtract from accumulator +name = vmls +multi_fn = simd_sub, a, {simd_mul, b, c} +a = 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 +validate 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + +arm = vmls. +aarch64 = mls +generate int*_t, uint*_t + +/// Floating-point multiply-subtract from accumulator +name = vmls +multi_fn = simd_sub, a, {simd_mul, b, c} +a = 6., 7., 8., 9. +b = 2., 2., 2., 2. +c = 3., 3., 3., 3. +validate 0., 1., 2., 3. + +aarch64 = fmul +generate float64x*_t + +arm = vmls. +generate float*_t + +/// Vector multiply subtract with scalar +name = vmls +n-suffix +multi_fn = vmls-self-noext, a, b, {vdup-nself-noext, c} +a = 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3 +validate 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = mls +arm = vmls. +generate int16x4_t:int16x4_t:i16:int16x4_t, int16x8_t:int16x8_t:i16:int16x8_t, int32x2_t:int32x2_t:i32:int32x2_t, int32x4_t:int32x4_t:i32:int32x4_t +generate uint16x4_t:uint16x4_t:u16:uint16x4_t, uint16x8_t:uint16x8_t:u16:uint16x8_t, uint32x2_t:uint32x2_t:u32:uint32x2_t, uint32x4_t:uint32x4_t:u32:uint32x4_t + +/// Vector multiply subtract with scalar +name = vmls +n-suffix +multi_fn = vmls-self-noext, a, b, {vdup-nself-noext, c} +a = 6., 7., 8., 9. +b = 2., 2., 2., 2. +c = 3. +validate 0., 1., 2., 3. + +aarch64 = fmul +arm = vmls. +generate float32x2_t:float32x2_t:f32:float32x2_t, float32x4_t:float32x4_t:f32:float32x4_t + +/// Vector multiply subtract with scalar +name = vmls +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vmls-self-noext, a, b, {simd_shuffle-in_len-!, c, c, {dup-in_len-LANE as u32}} +a = 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 1 +validate 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = mls +arm = vmls. +generate int16x4_t, int16x4_t:int16x4_t:int16x8_t:int16x4_t, int16x8_t:int16x8_t:int16x4_t:int16x8_t, int16x8_t +generate int32x2_t, int32x2_t:int32x2_t:int32x4_t:int32x2_t, int32x4_t:int32x4_t:int32x2_t:int32x4_t, int32x4_t +generate uint16x4_t, uint16x4_t:uint16x4_t:uint16x8_t:uint16x4_t, uint16x8_t:uint16x8_t:uint16x4_t:uint16x8_t, uint16x8_t +generate uint32x2_t, uint32x2_t:uint32x2_t:uint32x4_t:uint32x2_t, uint32x4_t:uint32x4_t:uint32x2_t:uint32x4_t, uint32x4_t + +/// Vector multiply subtract with scalar +name = vmls +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vmls-self-noext, a, b, {simd_shuffle-in_len-!, c, c, {dup-in_len-LANE as u32}} +a = 6., 7., 8., 9. +b = 2., 2., 2., 2. +c = 0., 3., 0., 0. +n = 1 +validate 0., 1., 2., 3. + +aarch64 = fmul +arm = vmls. +generate float32x2_t, float32x2_t:float32x2_t:float32x4_t:float32x2_t, float32x4_t:float32x4_t:float32x2_t:float32x4_t, float32x4_t + +/// Signed multiply-subtract long +name = vmlsl +multi_fn = simd_sub, a, {vmull-self-noext, b, c} +a = 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 +validate 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + +arm = vmlsl.s +aarch64 = smlsl +generate int16x8_t:int8x8_t:int8x8_t:int16x8_t, int32x4_t:int16x4_t:int16x4_t:int32x4_t, int64x2_t:int32x2_t:int32x2_t:int64x2_t + +/// Unsigned multiply-subtract long +name = vmlsl +multi_fn = simd_sub, a, {vmull-self-noext, b, c} +a = 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 +validate 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + +arm = vmlsl.s +aarch64 = umlsl +generate uint16x8_t:uint8x8_t:uint8x8_t:uint16x8_t, uint32x4_t:uint16x4_t:uint16x4_t:uint32x4_t, uint64x2_t:uint32x2_t:uint32x2_t:uint64x2_t + +/// Vector widening multiply subtract with scalar +name = vmlsl +n-suffix +multi_fn = vmlsl-self-noext, a, b, {vdup-nself-noext, c} +a = 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3 +validate 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + +arm = vmlsl.s +aarch64 = smlsl +generate int32x4_t:int16x4_t:i16:int32x4_t, int64x2_t:int32x2_t:i32:int64x2_t +aarch64 = umlsl +generate uint32x4_t:uint16x4_t:u16:uint32x4_t, uint64x2_t:uint32x2_t:u32:uint64x2_t + +/// Vector widening multiply subtract with scalar +name = vmlsl_lane +in2-suffix +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vmlsl-self-noext, a, b, {simd_shuffle-in_len-!, c, c, {dup-in_len-LANE as u32}} +a = 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 1 +validate 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + +arm = vmlsl.s +aarch64 = smlsl +generate int32x4_t:int16x4_t:int16x4_t:int32x4_t, int32x4_t:int16x4_t:int16x8_t:int32x4_t +generate int64x2_t:int32x2_t:int32x2_t:int64x2_t, int64x2_t:int32x2_t:int32x4_t:int64x2_t +aarch64 = umlsl +generate uint32x4_t:uint16x4_t:uint16x4_t:uint32x4_t, uint32x4_t:uint16x4_t:uint16x8_t:uint32x4_t +generate uint64x2_t:uint32x2_t:uint32x2_t:uint64x2_t, uint64x2_t:uint32x2_t:uint32x4_t:uint64x2_t + +/// Signed multiply-subtract long +name = vmlsl_high +no-q +multi_fn = simd_shuffle-out_len-!, b:half, b, b, {fixed-half-right} +multi_fn = simd_shuffle-out_len-!, c:half, c, c, {fixed-half-right} +multi_fn = vmlsl-noqself-noext, a, b, c +a = 14, 15, 16, 17, 18, 19, 20, 21 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7 +fixed = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 14, 13, 12, 11, 10, 9, 8, 7 + +aarch64 = smlsl2 +generate int16x8_t:int8x16_t:int8x16_t:int16x8_t, int32x4_t:int16x8_t:int16x8_t:int32x4_t, int64x2_t:int32x4_t:int32x4_t:int64x2_t + +/// Unsigned multiply-subtract long +name = vmlsl_high +no-q +multi_fn = simd_shuffle-out_len-!, b:half, b, b, {fixed-half-right} +multi_fn = simd_shuffle-out_len-!, c:half, c, c, {fixed-half-right} +multi_fn = vmlsl-noqself-noext, a, b, c +a = 14, 15, 16, 17, 18, 19, 20, 21 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7 +fixed = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 14, 13, 12, 11, 10, 9, 8, 7 + +aarch64 = umlsl2 +generate uint16x8_t:uint8x16_t:uint8x16_t:uint16x8_t, uint32x4_t:uint16x8_t:uint16x8_t:uint32x4_t, uint64x2_t:uint32x4_t:uint32x4_t:uint64x2_t + +/// Multiply-subtract long +name = vmlsl_high_n +no-q +multi_fn = vmlsl_high-noqself-noext, a, b, {vdupq_n-noqself-noext, c} +a = 14, 15, 16, 17, 18, 19, 20, 21 +b = 3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7 +c = 2 +validate 14, 13, 12, 11, 10, 9, 8, 7 + +aarch64 = smlsl2 +generate int32x4_t:int16x8_t:i16:int32x4_t, int64x2_t:int32x4_t:i32:int64x2_t +aarch64 = umlsl2 +generate uint32x4_t:uint16x8_t:u16:uint32x4_t, uint64x2_t:uint32x4_t:u32:uint64x2_t + +/// Multiply-subtract long +name = vmlsl_high_lane +in2-suffix +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vmlsl_high-noqself-noext, a, b, {simd_shuffle-in_len-!, c, c, {dup-in_len-LANE as u32}} +a = 14, 15, 16, 17, 18, 19, 20, 21 +b = 3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7 +c = 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 1 +validate 14, 13, 12, 11, 10, 9, 8, 7 + +aarch64 = smlsl2 +generate int32x4_t:int16x8_t:int16x4_t:int32x4_t, int32x4_t:int16x8_t:int16x8_t:int32x4_t +generate int64x2_t:int32x4_t:int32x2_t:int64x2_t, int64x2_t:int32x4_t:int32x4_t:int64x2_t +aarch64 = umlsl2 +generate uint32x4_t:uint16x8_t:uint16x4_t:uint32x4_t, uint32x4_t:uint16x8_t:uint16x8_t:uint32x4_t +generate uint64x2_t:uint32x4_t:uint32x2_t:uint64x2_t, uint64x2_t:uint32x4_t:uint32x4_t:uint64x2_t + +/// Extract narrow +name = vmovn_high +no-q +multi_fn = simd_cast, c:in_t0, b +multi_fn = simd_shuffle-out_len-!, a, c, {asc-0-out_len} +a = 0, 1, 2, 3, 2, 3, 4, 5 +b = 2, 3, 4, 5, 12, 13, 14, 15 +validate 0, 1, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 12, 13, 14, 15 + +aarch64 = xtn2 +generate int8x8_t:int16x8_t:int8x16_t, int16x4_t:int32x4_t:int16x8_t, int32x2_t:int64x2_t:int32x4_t +generate uint8x8_t:uint16x8_t:uint8x16_t, uint16x4_t:uint32x4_t:uint16x8_t, uint32x2_t:uint64x2_t:uint32x4_t + +/// Negate +name = vneg +fn = simd_neg +a = 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 7, -7, 8 +validate 0, -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7, -8 + +aarch64 = neg +generate int64x*_t + +arm = vneg.s +generate int*_t + +/// Negate +name = vneg +multi_fn = a.wrapping_neg() +a = 1 +validate -1 + +aarch64 = neg +generate i64 + +/// Negate +name = vneg +fn = simd_neg +a = 0., 1., -1., 2., -2., 3., -3., 4. +validate 0., -1., 1., -2., 2., -3., 3., -4. + +aarch64 = fneg +generate float64x*_t + +arm = vneg.s +generate float*_t + +/// Signed saturating negate +name = vqneg +a = MIN, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 7, -7 +validate MAX, 0, -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7 +link-arm = vqneg._EXT_ +link-aarch64 = sqneg._EXT_ + +aarch64 = sqneg +generate int64x*_t + +arm = vqneg.s +generate int*_t + +/// Signed saturating negate +name = vqneg +multi_fn = simd_extract, {vqneg-in_ntt-noext, {vdup_n-in_ntt-noext, a}}, 0 +a = 1 +validate -1 + +aarch64 = sqneg +generate i8, i16, i32, i64 + +/// Saturating subtract +name = vqsub +a = 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +validate 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26 + +arm = vqsub.s +aarch64 = uqsub +link-arm = llvm.usub.sat._EXT_ +link-aarch64 = uqsub._EXT_ +generate uint*_t, uint64x*_t + +arm = vqsub.s +aarch64 = sqsub +link-arm = llvm.ssub.sat._EXT_ +link-aarch64 = sqsub._EXT_ +generate int*_t, int64x*_t + +/// Saturating subtract +name = vqsub +multi_fn = vdup_n-in_ntt-noext, a:in_ntt, a +multi_fn = vdup_n-in_ntt-noext, b:in_ntt, b +multi_fn = simd_extract, {vqsub-in_ntt-noext, a, b}, 0 +a = 42 +b = 1 +validate 41 + +aarch64 = sqsub +generate i8, i16 +aarch64 = uqsub +generate u8, u16 + +/// Saturating subtract +name = vqsub +a = 42 +b = 1 +validate 41 + +aarch64 = uqsub +link-aarch64 = uqsub._EXT_ +generate u32, u64 + +aarch64 = sqsub +link-aarch64 = sqsub._EXT_ +generate i32, i64 + +/// Halving add +name = vhadd +a = 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +validate 21, 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29 + +arm = vhadd.s +aarch64 = uhadd +link-aarch64 = uhadd._EXT_ +link-arm = vhaddu._EXT_ +generate uint*_t + +arm = vhadd.s +aarch64 = shadd +link-aarch64 = shadd._EXT_ +link-arm = vhadds._EXT_ +generate int*_t + +/// Reverse bit order +name = vrbit +a = 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 +validate 0, 64, 32, 96, 16, 80, 48, 112, 8, 72, 40, 104, 24, 88, 56, 120 + +aarch64 = rbit +link-aarch64 = rbit._EXT_ + +generate int8x8_t, int8x16_t + +/// Reverse bit order +name = vrbit +multi_fn = transmute, {vrbit-signed-noext, transmute(a)} +a = 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 +validate 0, 64, 32, 96, 16, 80, 48, 112, 8, 72, 40, 104, 24, 88, 56, 120 + +aarch64 = rbit + +generate uint8x8_t, uint8x16_t, poly8x8_t, poly8x16_t + +/// Rounding halving add +name = vrhadd +a = 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +validate 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29, 29 + +arm = vrhadd.s +aarch64 = urhadd +link-arm = vrhaddu._EXT_ +link-aarch64 = urhadd._EXT_ +generate uint*_t + +arm = vrhadd.s +aarch64 = srhadd +link-arm = vrhadds._EXT_ +link-aarch64 = srhadd._EXT_ +generate int*_t + +/// Floating-point round to integral exact, using current rounding mode +name = vrndx +a = -1.5, 0.5, 1.5, 2.5 +validate -2.0, 0.0, 2.0, 2.0 + +aarch64 = frintx +link-aarch64 = llvm.rint._EXT_ +generate float*_t, float64x*_t + +/// Floating-point round to integral, to nearest with ties to away +name = vrnda +a = -1.5, 0.5, 1.5, 2.5 +validate -2.0, 1.0, 2.0, 3.0 + +aarch64 = frinta +link-aarch64 = llvm.round._EXT_ +generate float*_t, float64x*_t + +/// Floating-point round to integral, to nearest with ties to even +name = vrndn +a = -1.5, 0.5, 1.5, 2.5 +validate -2.0, 0.0, 2.0, 2.0 + +link-aarch64 = frintn._EXT_ +aarch64 = frintn +generate float64x*_t + +target = fp-armv8 +arm = vrintn +link-arm = vrintn._EXT_ +generate float*_t + +/// Floating-point round to integral, to nearest with ties to even +name = vrndn +a = -1.5 +validate -2.0 + +aarch64 = frintn +link-aarch64 = llvm.roundeven._EXT_ +generate f32 + +/// Floating-point round to integral, toward minus infinity +name = vrndm +a = -1.5, 0.5, 1.5, 2.5 +validate -2.0, 0.0, 1.0, 2.0 + +aarch64 = frintm +link-aarch64 = llvm.floor._EXT_ +generate float*_t, float64x*_t + +/// Floating-point round to integral, toward plus infinity +name = vrndp +a = -1.5, 0.5, 1.5, 2.5 +validate -1.0, 1.0, 2.0, 3.0 + +aarch64 = frintp +link-aarch64 = llvm.ceil._EXT_ +generate float*_t, float64x*_t + +/// Floating-point round to integral, toward zero +name = vrnd +a = -1.5, 0.5, 1.5, 2.5 +validate -1.0, 0.0, 1.0, 2.0 + +aarch64 = frintz +link-aarch64 = llvm.trunc._EXT_ +generate float*_t, float64x*_t + +/// Floating-point round to integral, using current rounding mode +name = vrndi +a = -1.5, 0.5, 1.5, 2.5 +validate -2.0, 0.0, 2.0, 2.0 + +aarch64 = frinti +link-aarch64 = llvm.nearbyint._EXT_ +generate float*_t, float64x*_t + +/// Saturating add +name = vqadd +a = 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +validate 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58 + +arm = vqadd.s +aarch64 = uqadd +link-arm = llvm.uadd.sat._EXT_ +link-aarch64 = uqadd._EXT_ +generate uint*_t, uint64x*_t + +arm = vqadd.s +aarch64 = sqadd +link-arm = llvm.sadd.sat._EXT_ +link-aarch64 = sqadd._EXT_ +generate int*_t, int64x*_t + +/// Saturating add +name = vqadd +multi_fn = vdup_n-in_ntt-noext, a:in_ntt, a +multi_fn = vdup_n-in_ntt-noext, b:in_ntt, b +multi_fn = simd_extract, {vqadd-in_ntt-noext, a, b}, 0 +a = 42 +b = 1 +validate 43 + +aarch64 = sqadd +generate i8, i16 +aarch64 = uqadd +generate u8, u16 + +/// Saturating add +name = vqadd +a = 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +validate 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58 + +aarch64 = uqadd +link-aarch64 = uqadd._EXT_ +generate u32, u64 + +aarch64 = sqadd +link-aarch64 = sqadd._EXT_ +generate i32, i64 + +/// Load multiple single-element structures to one, two, three, or four registers +name = vld1 +out-suffix +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32 +load_fn + +aarch64 = ld1 +link-aarch64 = ld1x2._EXT2_ +arm = vld1 +link-arm = vld1x2._EXT2_ +generate *const i8:int8x8x2_t, *const i16:int16x4x2_t, *const i32:int32x2x2_t, *const i64:int64x1x2_t +generate *const i8:int8x16x2_t, *const i16:int16x8x2_t, *const i32:int32x4x2_t, *const i64:int64x2x2_t + +link-aarch64 = ld1x3._EXT2_ +link-arm = vld1x3._EXT2_ +generate *const i8:int8x8x3_t, *const i16:int16x4x3_t, *const i32:int32x2x3_t, *const i64:int64x1x3_t +generate *const i8:int8x16x3_t, *const i16:int16x8x3_t, *const i32:int32x4x3_t, *const i64:int64x2x3_t + +link-aarch64 = ld1x4._EXT2_ +link-arm = vld1x4._EXT2_ +generate *const i8:int8x8x4_t, *const i16:int16x4x4_t, *const i32:int32x2x4_t, *const i64:int64x1x4_t +generate *const i8:int8x16x4_t, *const i16:int16x8x4_t, *const i32:int32x4x4_t, *const i64:int64x2x4_t + +/// Load multiple single-element structures to one, two, three, or four registers +name = vld1 +out-suffix +multi_fn = transmute, {vld1-outsigned-noext, transmute(a)} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32 + +load_fn +aarch64 = ld1 +arm = vld1 +generate *const u8:uint8x8x2_t, *const u16:uint16x4x2_t, *const u32:uint32x2x2_t, *const u64:uint64x1x2_t +generate *const u8:uint8x16x2_t, *const u16:uint16x8x2_t, *const u32:uint32x4x2_t, *const u64:uint64x2x2_t +generate *const u8:uint8x8x3_t, *const u16:uint16x4x3_t, *const u32:uint32x2x3_t, *const u64:uint64x1x3_t +generate *const u8:uint8x16x3_t, *const u16:uint16x8x3_t, *const u32:uint32x4x3_t, *const u64:uint64x2x3_t +generate *const u8:uint8x8x4_t, *const u16:uint16x4x4_t, *const u32:uint32x2x4_t, *const u64:uint64x1x4_t +generate *const u8:uint8x16x4_t, *const u16:uint16x8x4_t, *const u32:uint32x4x4_t, *const u64:uint64x2x4_t +generate *const p8:poly8x8x2_t, *const p8:poly8x8x3_t, *const p8:poly8x8x4_t +generate *const p8:poly8x16x2_t, *const p8:poly8x16x3_t, *const p8:poly8x16x4_t +generate *const p16:poly16x4x2_t, *const p16:poly16x4x3_t, *const p16:poly16x4x4_t +generate *const p16:poly16x8x2_t, *const p16:poly16x8x3_t, *const p16:poly16x8x4_t +target = aes +generate *const p64:poly64x1x2_t +arm = nop +generate *const p64:poly64x1x3_t, *const p64:poly64x1x4_t +generate *const p64:poly64x2x2_t, *const p64:poly64x2x3_t, *const p64:poly64x2x4_t +/// Load multiple single-element structures to one, two, three, or four registers +name = vld1 +out-suffix +a = 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16. +validate 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16. +load_fn + +aarch64 = ld1 +link-aarch64 = ld1x2._EXT2_ +generate *const f64:float64x1x2_t, *const f64:float64x2x2_t + +link-aarch64 = ld1x3._EXT2_ +generate *const f64:float64x1x3_t, *const f64:float64x2x3_t + +link-aarch64 = ld1x4._EXT2_ +generate *const f64:float64x1x4_t, *const f64:float64x2x4_t + +arm = vld1 +link-aarch64 = ld1x2._EXT2_ +link-arm = vld1x2._EXT2_ +generate *const f32:float32x2x2_t, *const f32:float32x4x2_t + +link-aarch64 = ld1x3._EXT2_ +link-arm = vld1x3._EXT2_ +generate *const f32:float32x2x3_t, *const f32:float32x4x3_t + +link-aarch64 = ld1x4._EXT2_ +link-arm = vld1x4._EXT2_ +generate *const f32:float32x2x4_t, *const f32:float32x4x4_t + +/// Load multiple 2-element structures to two registers +name = vld2 +out-nox +a = 0, 1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15, 8, 16, 9, 17 +validate 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 +load_fn +arm-aarch64-separate + +aarch64 = ld2 +link-aarch64 = ld2._EXTv2_ +generate *const i64:int64x2x2_t + +arm = vld2 +link-arm = vld2._EXTpi82_ +generate *const i8:int8x8x2_t, *const i16:int16x4x2_t, *const i32:int32x2x2_t +generate *const i8:int8x16x2_t, *const i16:int16x8x2_t, *const i32:int32x4x2_t +arm = nop +aarch64 = nop +generate *const i64:int64x1x2_t + +/// Load multiple 2-element structures to two registers +name = vld2 +out-nox +multi_fn = transmute, {vld2-outsignednox-noext, transmute(a)} +a = 0, 1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15, 8, 16, 9, 17 +validate 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 +load_fn + +aarch64 = ld2 +generate *const u64:uint64x2x2_t +target = aes +generate *const p64:poly64x2x2_t + +target = default +arm = vld2 +generate *const u8:uint8x8x2_t, *const u16:uint16x4x2_t, *const u32:uint32x2x2_t +generate *const u8:uint8x16x2_t, *const u16:uint16x8x2_t, *const u32:uint32x4x2_t +generate *const p8:poly8x8x2_t, *const p16:poly16x4x2_t, *const p8:poly8x16x2_t, *const p16:poly16x8x2_t +arm = nop +aarch64 = nop +generate *const u64:uint64x1x2_t +target = aes +generate *const p64:poly64x1x2_t + + +/// Load multiple 2-element structures to two registers +name = vld2 +out-nox +a = 0., 1., 2., 2., 3., 2., 4., 3., 5., 2., 6., 3., 7., 4., 8., 5., 9. +validate 1., 2., 2., 3., 2., 3., 4., 5., 2., 3., 4., 5., 6., 7., 8., 9. +load_fn +arm-aarch64-separate + +aarch64 = nop +link-aarch64 = ld2._EXTv2_ +generate *const f64:float64x1x2_t +aarch64 = ld2 +generate *const f64:float64x2x2_t + +arm = vld2 +link-arm = vld2._EXTpi82_ +generate *const f32:float32x2x2_t, *const f32:float32x4x2_t + +/// Load single 2-element structure and replicate to all lanes of two registers +name = vld2 +out-dup-nox +a = 0, 1, 1, 2, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15, 8, 16, 9, 17 +validate 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +load_fn +arm-aarch64-separate + +aarch64 = ld2r +link-aarch64 = ld2r._EXT2_ +generate *const i64:int64x2x2_t + +arm = vld2 +link-arm = vld2dup._EXTpi82_ +generate *const i8:int8x8x2_t, *const i16:int16x4x2_t, *const i32:int32x2x2_t +generate *const i8:int8x16x2_t, *const i16:int16x8x2_t, *const i32:int32x4x2_t +arm = nop +generate *const i64:int64x1x2_t + +/// Load single 2-element structure and replicate to all lanes of two registers +name = vld2 +out-dup-nox +multi_fn = transmute, {vld2-outsigneddupnox-noext, transmute(a)} +a = 0, 1, 1, 2, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15, 8, 16, 9, 17 +validate 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +load_fn + +aarch64 = ld2r +generate *const u64:uint64x2x2_t +target = aes +generate *const p64:poly64x2x2_t + +target = default +arm = vld2 +generate *const u8:uint8x8x2_t, *const u16:uint16x4x2_t, *const u32:uint32x2x2_t +generate *const u8:uint8x16x2_t, *const u16:uint16x8x2_t, *const u32:uint32x4x2_t +generate *const p8:poly8x8x2_t, *const p16:poly16x4x2_t, *const p8:poly8x16x2_t, *const p16:poly16x8x2_t +arm = nop +generate *const u64:uint64x1x2_t +target = aes +generate *const p64:poly64x1x2_t + +/// Load single 2-element structure and replicate to all lanes of two registers +name = vld2 +out-dup-nox +a = 0., 1., 1., 2., 3., 1., 4., 3., 5. +validate 1., 1., 1., 1., 1., 1., 1., 1. +load_fn +arm-aarch64-separate + +aarch64 = ld2r +link-aarch64 = ld2r._EXT2_ +generate *const f64:float64x1x2_t, *const f64:float64x2x2_t + +arm = vld2 +link-arm = vld2dup._EXTpi82_ +generate *const f32:float32x2x2_t, *const f32:float32x4x2_t + +/// Load multiple 2-element structures to two registers +name = vld2 +out-lane-nox +multi_fn = static_assert_imm-in_exp_len-LANE +constn = LANE +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 +b = 0, 2, 2, 14, 2, 16, 17, 18, 2, 20, 21, 22, 23, 24, 25, 26, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26 +n = 0 +validate 1, 2, 2, 14, 2, 16, 17, 18, 2, 20, 21, 22, 23, 24, 25, 26, 2, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26 +load_fn +arm-aarch64-separate + +aarch64 = ld2 +const-aarch64 = LANE +link-aarch64 = ld2lane._EXTpi82_ +generate *const i8:int8x16x2_t:int8x16x2_t, *const i64:int64x1x2_t:int64x1x2_t, *const i64:int64x2x2_t:int64x2x2_t + +arm = vld2 +const-arm = LANE +link-arm = vld2lane._EXTpi82_ +generate *const i8:int8x8x2_t:int8x8x2_t, *const i16:int16x4x2_t:int16x4x2_t, *const i32:int32x2x2_t:int32x2x2_t +generate *const i16:int16x8x2_t:int16x8x2_t, *const i32:int32x4x2_t:int32x4x2_t + +/// Load multiple 2-element structures to two registers +name = vld2 +out-lane-nox +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = transmute, {vld2-outsignedlanenox-::, transmute(a), transmute(b)} +constn = LANE +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 +b = 0, 2, 2, 14, 2, 16, 17, 18, 2, 20, 21, 22, 23, 24, 25, 26, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26 +n = 0 +validate 1, 2, 2, 14, 2, 16, 17, 18, 2, 20, 21, 22, 23, 24, 25, 26, 2, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26 +load_fn + +aarch64 = ld2 +const-aarch64 = LANE + +target = aes +generate *const p64:poly64x1x2_t:poly64x1x2_t, *const p64:poly64x2x2_t:poly64x2x2_t + +target = default +generate *const u8:uint8x16x2_t:uint8x16x2_t, *const u64:uint64x1x2_t:uint64x1x2_t, *const u64:uint64x2x2_t:uint64x2x2_t +generate *const p8:poly8x16x2_t:poly8x16x2_t + +arm = vld2 +const-arm = LANE +generate *const u8:uint8x8x2_t:uint8x8x2_t, *const u16:uint16x4x2_t:uint16x4x2_t, *const u32:uint32x2x2_t:uint32x2x2_t +generate *const u16:uint16x8x2_t:uint16x8x2_t, *const u32:uint32x4x2_t:uint32x4x2_t +generate *const p8:poly8x8x2_t:poly8x8x2_t, *const p16:poly16x4x2_t:poly16x4x2_t +generate *const p16:poly16x8x2_t:poly16x8x2_t + +/// Load multiple 2-element structures to two registers +name = vld2 +out-lane-nox +multi_fn = static_assert_imm-in_exp_len-LANE +constn = LANE +a = 0., 1., 2., 3., 4., 5., 6., 7., 8. +b = 0., 2., 2., 14., 2., 16., 17., 18. +n = 0 +validate 1., 2., 2., 14., 2., 16., 17., 18. +load_fn +arm-aarch64-separate + +aarch64 = ld2 +const-aarch64 = LANE +link-aarch64 = ld2lane._EXTpi82_ +generate *const f64:float64x1x2_t:float64x1x2_t, *const f64:float64x2x2_t:float64x2x2_t + +arm = vld2 +const-arm = LANE +link-arm = vld2lane._EXTpi82_ +generate *const f32:float32x2x2_t:float32x2x2_t, *const f32:float32x4x2_t:float32x4x2_t + +/// Load multiple 3-element structures to three registers +name = vld3 +out-nox +a = 0, 1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16, 2, 25, 41, 4, 26, 42, 7, 27, 43, 8, 28, 44, 13, 29, 45, 14, 30, 46, 15, 31, 47, 16, 32, 48 +validate 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16, 25, 26, 27, 28, 29, 30, 31, 32, 2, 4, 7, 8, 13, 14, 15, 16, 41, 42, 43, 44, 45, 46, 47, 48 +load_fn +arm-aarch64-separate + +aarch64 = ld3 +link-aarch64 = ld3._EXTv2_ +generate *const i64:int64x2x3_t + +arm = vld3 +link-arm = vld3._EXTpi82_ +generate *const i8:int8x8x3_t, *const i16:int16x4x3_t, *const i32:int32x2x3_t +generate *const i8:int8x16x3_t, *const i16:int16x8x3_t, *const i32:int32x4x3_t +arm = nop +aarch64 = nop +generate *const i64:int64x1x3_t + +/// Load multiple 3-element structures to three registers +name = vld3 +out-nox +multi_fn = transmute, {vld3-outsignednox-noext, transmute(a)} +a = 0, 1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16, 2, 25, 41, 4, 26, 42, 7, 27, 43, 8, 28, 44, 13, 29, 45, 14, 30, 46, 15, 31, 47, 16, 32, 48 +validate 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16, 25, 26, 27, 28, 29, 30, 31, 32, 2, 4, 7, 8, 13, 14, 15, 16, 41, 42, 43, 44, 45, 46, 47, 48 +load_fn + +aarch64 = ld3 +generate *const u64:uint64x2x3_t +target = aes +generate *const p64:poly64x2x3_t + +target = default +arm = vld3 +generate *const u8:uint8x8x3_t, *const u16:uint16x4x3_t, *const u32:uint32x2x3_t +generate *const u8:uint8x16x3_t, *const u16:uint16x8x3_t, *const u32:uint32x4x3_t +generate *const p8:poly8x8x3_t, *const p16:poly16x4x3_t, *const p8:poly8x16x3_t, *const p16:poly16x8x3_t +arm = nop +aarch64 = nop +generate *const u64:uint64x1x3_t +target = aes +generate *const p64:poly64x1x3_t + +/// Load multiple 3-element structures to three registers +name = vld3 +out-nox +a = 0., 1., 2., 2., 2., 4., 4., 2., 7., 7., 4., 8., 8. +validate 1., 2., 2., 4., 2., 4., 7., 8., 2., 4., 7., 8. +load_fn +arm-aarch64-separate + +aarch64 = nop +link-aarch64 = ld3._EXTv2_ +generate *const f64:float64x1x3_t +aarch64 = ld3 +generate *const f64:float64x2x3_t + +arm = vld3 +link-arm = vld3._EXTpi82_ +generate *const f32:float32x2x3_t, *const f32:float32x4x3_t + +/// Load single 3-element structure and replicate to all lanes of three registers +name = vld3 +out-dup-nox +a = 0, 1, 1, 1, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15, 8, 16, 9, 17, 6, 14, 7, 15, 8, 16, 9, 17, 6, 14, 7, 15, 8, 16, 9, 17 +validate 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +load_fn +arm-aarch64-separate + +aarch64 = ld3r +link-aarch64 = ld3r._EXT2_ +generate *const i64:int64x2x3_t + +arm = vld3 +link-arm = vld3dup._EXTpi82_ +generate *const i8:int8x8x3_t, *const i16:int16x4x3_t, *const i32:int32x2x3_t +generate *const i8:int8x16x3_t, *const i16:int16x8x3_t, *const i32:int32x4x3_t +arm = nop +generate *const i64:int64x1x3_t + +/// Load single 3-element structure and replicate to all lanes of three registers +name = vld3 +out-dup-nox +multi_fn = transmute, {vld3-outsigneddupnox-noext, transmute(a)} +a = 0, 1, 1, 1, 3, 1, 4, 3, 5, 1, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15, 8, 16, 9, 17, 6, 14, 7, 15, 8, 16, 9, 17, 6, 14, 7, 15, 8, 16, 9, 17 +validate 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +load_fn + +aarch64 = ld3r +generate *const u64:uint64x2x3_t +target = aes +generate *const p64:poly64x2x3_t + +target = default +arm = vld3 +generate *const u8:uint8x8x3_t, *const u16:uint16x4x3_t, *const u32:uint32x2x3_t +generate *const u8:uint8x16x3_t, *const u16:uint16x8x3_t, *const u32:uint32x4x3_t +generate *const p8:poly8x8x3_t, *const p16:poly16x4x3_t, *const p8:poly8x16x3_t, *const p16:poly16x8x3_t +arm = nop +generate *const u64:uint64x1x3_t +target = aes +generate *const p64:poly64x1x3_t + +/// Load single 3-element structure and replicate to all lanes of three registers +name = vld3 +out-dup-nox +a = 0., 1., 1., 1., 3., 1., 4., 3., 5., 1., 4., 3., 5. +validate 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1. +load_fn +arm-aarch64-separate + +aarch64 = ld3r +link-aarch64 = ld3r._EXT2_ +generate *const f64:float64x1x3_t, *const f64:float64x2x3_t + +arm = vld3 +link-arm = vld3dup._EXTpi82_ +generate *const f32:float32x2x3_t, *const f32:float32x4x3_t + +/// Load multiple 3-element structures to two registers +name = vld3 +out-lane-nox +multi_fn = static_assert_imm-in_exp_len-LANE +constn = LANE +a = 0, 1, 2, 2, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 +b = 0, 2, 2, 14, 2, 16, 17, 18, 2, 20, 21, 22, 23, 24, 25, 26, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 +n = 0 +validate 1, 2, 2, 14, 2, 16, 17, 18, 2, 20, 21, 22, 23, 24, 25, 26, 2, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 2, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 +load_fn +arm-aarch64-separate + +aarch64 = ld3 +const-aarch64 = LANE +link-aarch64 = ld3lane._EXTpi82_ +generate *const i8:int8x16x3_t:int8x16x3_t, *const i64:int64x1x3_t:int64x1x3_t, *const i64:int64x2x3_t:int64x2x3_t + +arm = vld3 +const-arm = LANE +link-arm = vld3lane._EXTpi82_ +generate *const i8:int8x8x3_t:int8x8x3_t, *const i16:int16x4x3_t:int16x4x3_t, *const i32:int32x2x3_t:int32x2x3_t +generate *const i16:int16x8x3_t:int16x8x3_t, *const i32:int32x4x3_t:int32x4x3_t + +/// Load multiple 3-element structures to three registers +name = vld3 +out-lane-nox +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = transmute, {vld3-outsignedlanenox-::, transmute(a), transmute(b)} +constn = LANE +a = 0, 1, 2, 2, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 +b = 0, 2, 2, 14, 2, 16, 17, 18, 2, 20, 21, 22, 23, 24, 25, 26, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 +n = 0 +validate 1, 2, 2, 14, 2, 16, 17, 18, 2, 20, 21, 22, 23, 24, 25, 26, 2, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 2, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 +load_fn + +aarch64 = ld3 +const-aarch64 = LANE +target = aes +generate *const p64:poly64x1x3_t:poly64x1x3_t, *const p64:poly64x2x3_t:poly64x2x3_t +target = default +generate *const p8:poly8x16x3_t:poly8x16x3_t, *const u8:uint8x16x3_t:uint8x16x3_t, *const u64:uint64x1x3_t:uint64x1x3_t, *const u64:uint64x2x3_t:uint64x2x3_t + +arm = vld3 +const-arm = LANE +generate *const u8:uint8x8x3_t:uint8x8x3_t, *const u16:uint16x4x3_t:uint16x4x3_t, *const u32:uint32x2x3_t:uint32x2x3_t +generate *const u16:uint16x8x3_t:uint16x8x3_t, *const u32:uint32x4x3_t:uint32x4x3_t +generate *const p8:poly8x8x3_t:poly8x8x3_t, *const p16:poly16x4x3_t:poly16x4x3_t +generate *const p16:poly16x8x3_t:poly16x8x3_t + +/// Load multiple 3-element structures to three registers +name = vld3 +out-lane-nox +multi_fn = static_assert_imm-in_exp_len-LANE +constn = LANE +a = 0., 1., 2., 2., 4., 5., 6., 7., 8., 5., 6., 7., 8. +b = 0., 2., 2., 14., 9., 16., 17., 18., 5., 6., 7., 8. +n = 0 +validate 1., 2., 2., 14., 2., 16., 17., 18., 2., 6., 7., 8. +load_fn +arm-aarch64-separate + +aarch64 = ld3 +const-aarch64 = LANE +link-aarch64 = ld3lane._EXTpi82_ +generate *const f64:float64x1x3_t:float64x1x3_t, *const f64:float64x2x3_t:float64x2x3_t + +arm = vld3 +const-arm = LANE +link-arm = vld3lane._EXTpi82_ +generate *const f32:float32x2x3_t:float32x2x3_t, *const f32:float32x4x3_t:float32x4x3_t + +/// Load multiple 4-element structures to four registers +name = vld4 +out-nox +a = 0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64 +validate 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64 +load_fn +arm-aarch64-separate + +aarch64 = ld4 +link-aarch64 = ld4._EXTv2_ +generate *const i64:int64x2x4_t + +arm = vld4 +link-arm = vld4._EXTpi82_ +generate *const i8:int8x8x4_t, *const i16:int16x4x4_t, *const i32:int32x2x4_t +generate *const i8:int8x16x4_t, *const i16:int16x8x4_t, *const i32:int32x4x4_t +aarch64 = nop +arm = nop +generate *const i64:int64x1x4_t + +/// Load multiple 4-element structures to four registers +name = vld4 +out-nox +multi_fn = transmute, {vld4-outsignednox-noext, transmute(a)} +a = 0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64 +validate 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64 +load_fn + +aarch64 = ld4 +generate *const u64:uint64x2x4_t +target = aes +generate *const p64:poly64x2x4_t + +target = default +arm = vld4 +generate *const u8:uint8x8x4_t, *const u16:uint16x4x4_t, *const u32:uint32x2x4_t +generate *const u8:uint8x16x4_t, *const u16:uint16x8x4_t, *const u32:uint32x4x4_t +generate *const p8:poly8x8x4_t, *const p16:poly16x4x4_t, *const p8:poly8x16x4_t, *const p16:poly16x8x4_t +aarch64 = nop +arm = nop +generate *const u64:uint64x1x4_t +target = aes +generate *const p64:poly64x1x4_t + +/// Load multiple 4-element structures to four registers +name = vld4 +out-nox +a = 0., 1., 2., 2., 6., 2., 6., 6., 8., 2., 6., 6., 8., 6., 8., 15., 16. +validate 1., 2., 2., 6., 2., 6., 6., 8., 2., 6., 6., 15., 6., 8., 8., 16. +load_fn +arm-aarch64-separate + +aarch64 = nop +link-aarch64 = ld4._EXTv2_ +generate *const f64:float64x1x4_t +aarch64 = ld4 +generate *const f64:float64x2x4_t + +arm = vld4 +link-arm = vld4._EXTpi82_ +generate *const f32:float32x2x4_t, *const f32:float32x4x4_t + +/// Load single 4-element structure and replicate to all lanes of four registers +name = vld4 +out-dup-nox +a = 0, 1, 1, 1, 1, 2, 4, 3, 5, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9 +validate 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +load_fn +arm-aarch64-separate + +aarch64 = ld4r +link-aarch64 = ld4r._EXT2_ +generate *const i64:int64x2x4_t + +arm = vld4 +link-arm = vld4dup._EXTpi82_ +generate *const i8:int8x8x4_t, *const i16:int16x4x4_t, *const i32:int32x2x4_t +generate *const i8:int8x16x4_t, *const i16:int16x8x4_t, *const i32:int32x4x4_t +arm = nop +generate *const i64:int64x1x4_t + +/// Load single 4-element structure and replicate to all lanes of four registers +name = vld4 +out-dup-nox +multi_fn = transmute, {vld4-outsigneddupnox-noext, transmute(a)} +a = 0, 1, 1, 1, 1, 2, 4, 3, 5, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9, 8, 6, 3, 7, 4, 8, 5, 9 +validate 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +load_fn + +aarch64 = ld4r +generate *const u64:uint64x2x4_t +target = aes +generate *const p64:poly64x2x4_t + +target = default +arm = vld4 +generate *const u8:uint8x8x4_t, *const u16:uint16x4x4_t, *const u32:uint32x2x4_t +generate *const u8:uint8x16x4_t, *const u16:uint16x8x4_t, *const u32:uint32x4x4_t +generate *const p8:poly8x8x4_t, *const p16:poly16x4x4_t, *const p8:poly8x16x4_t, *const p16:poly16x8x4_t +arm = nop +generate *const u64:uint64x1x4_t +target = aes +generate *const p64:poly64x1x4_t + +/// Load single 4-element structure and replicate to all lanes of four registers +name = vld4 +out-dup-nox +a = 0., 1., 1., 1., 1., 6., 4., 3., 5., 7., 4., 3., 5., 8., 4., 3., 5., 9., 4., 3., 5. +validate 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1. +load_fn +arm-aarch64-separate + +aarch64 = ld4r +link-aarch64 = ld4r._EXT2_ +generate *const f64:float64x1x4_t, *const f64:float64x2x4_t + +arm = vld4 +link-arm = vld4dup._EXTpi82_ +generate *const f32:float32x2x4_t, *const f32:float32x4x4_t + +/// Load multiple 4-element structures to four registers +name = vld4 +out-lane-nox +multi_fn = static_assert_imm-in_exp_len-LANE +constn = LANE +a = 0, 1, 2, 2, 2, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16 +b = 0, 2, 2, 2, 2, 16, 2, 18, 2, 20, 21, 22, 2, 24, 25, 26, 11, 12, 13, 14, 15, 16, 2, 18, 2, 20, 21, 22, 23, 24, 25, 26, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16 +n = 0 +validate 1, 2, 2, 2, 2, 16, 2, 18, 2, 20, 21, 22, 2, 24, 25, 26, 2, 12, 13, 14, 15, 16, 2, 18, 2, 20, 21, 22, 23, 24, 25, 26, 2, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 2, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16 +load_fn +arm-aarch64-separate + +aarch64 = ld4 +const-aarch64 = LANE +link-aarch64 = ld4lane._EXTpi82_ +generate *const i8:int8x16x4_t:int8x16x4_t, *const i64:int64x1x4_t:int64x1x4_t, *const i64:int64x2x4_t:int64x2x4_t + +arm = vld4 +const-arm = LANE +link-arm = vld4lane._EXTpi82_ +generate *const i8:int8x8x4_t:int8x8x4_t, *const i16:int16x4x4_t:int16x4x4_t, *const i32:int32x2x4_t:int32x2x4_t +generate *const i16:int16x8x4_t:int16x8x4_t, *const i32:int32x4x4_t:int32x4x4_t + +/// Load multiple 4-element structures to four registers +name = vld4 +out-lane-nox +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = transmute, {vld4-outsignedlanenox-::, transmute(a), transmute(b)} +constn = LANE +a = 0, 1, 2, 2, 2, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16 +b = 0, 2, 2, 2, 2, 16, 2, 18, 2, 20, 21, 22, 2, 24, 25, 26, 11, 12, 13, 14, 15, 16, 2, 18, 2, 20, 21, 22, 23, 24, 25, 26, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16 +n = 0 +validate 1, 2, 2, 2, 2, 16, 2, 18, 2, 20, 21, 22, 2, 24, 25, 26, 2, 12, 13, 14, 15, 16, 2, 18, 2, 20, 21, 22, 23, 24, 25, 26, 2, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 2, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16 +load_fn + +aarch64 = ld4 +const-aarch64 = LANE +target = aes +generate *const p64:poly64x1x4_t:poly64x1x4_t, *const p64:poly64x2x4_t:poly64x2x4_t +target = default +generate *const p8:poly8x16x4_t:poly8x16x4_t, *const u8:uint8x16x4_t:uint8x16x4_t, *const u64:uint64x1x4_t:uint64x1x4_t, *const u64:uint64x2x4_t:uint64x2x4_t + +arm = vld4 +const-arm = LANE +generate *const u8:uint8x8x4_t:uint8x8x4_t, *const u16:uint16x4x4_t:uint16x4x4_t, *const u32:uint32x2x4_t:uint32x2x4_t +generate *const u16:uint16x8x4_t:uint16x8x4_t, *const u32:uint32x4x4_t:uint32x4x4_t +generate *const p8:poly8x8x4_t:poly8x8x4_t, *const p16:poly16x4x4_t:poly16x4x4_t +generate *const p16:poly16x8x4_t:poly16x8x4_t + +/// Load multiple 4-element structures to four registers +name = vld4 +out-lane-nox +multi_fn = static_assert_imm-in_exp_len-LANE +constn = LANE +a = 0., 1., 2., 2., 2., 5., 6., 7., 8., 5., 6., 7., 8., 1., 4., 3., 5. +b = 0., 2., 2., 2., 2., 16., 2., 18., 5., 6., 7., 8., 1., 4., 3., 5. +n = 0 +validate 1., 2., 2., 2., 2., 16., 2., 18., 2., 6., 7., 8., 2., 4., 3., 5. +load_fn +arm-aarch64-separate + +aarch64 = ld4 +const-aarch64 = LANE +link-aarch64 = ld4lane._EXTpi82_ +generate *const f64:float64x1x4_t:float64x1x4_t, *const f64:float64x2x4_t:float64x2x4_t + +arm = vld4 +const-arm = LANE +link-arm = vld4lane._EXTpi82_ +generate *const f32:float32x2x4_t:float32x2x4_t, *const f32:float32x4x4_t:float32x4x4_t + +/// Store multiple single-element structures from one, two, three, or four registers +name = vst1 +in1-lane-nox +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = *a, {simd_extract, b, LANE as u32} +constn = LANE +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +n = 0 +validate 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +store_fn + +aarch64 = nop +arm = nop +generate *mut i8:int8x8_t:void, *mut i16:int16x4_t:void, *mut i32:int32x2_t:void, *mut i64:int64x1_t:void +generate *mut i8:int8x16_t:void, *mut i16:int16x8_t:void, *mut i32:int32x4_t:void, *mut i64:int64x2_t:void +generate *mut u8:uint8x8_t:void, *mut u16:uint16x4_t:void, *mut u32:uint32x2_t:void, *mut u64:uint64x1_t:void +generate *mut u8:uint8x16_t:void, *mut u16:uint16x8_t:void, *mut u32:uint32x4_t:void, *mut u64:uint64x2_t:void +generate *mut p8:poly8x8_t:void, *mut p16:poly16x4_t:void, *mut p8:poly8x16_t:void, *mut p16:poly16x8_t:void +target = aes +generate *mut p64:poly64x1_t:void, *mut p64:poly64x2_t:void + +/// Store multiple single-element structures from one, two, three, or four registers +name = vst1 +in1-lane-nox +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = *a, {simd_extract, b, LANE as u32} +constn = LANE +a = 0., 1., 2., 3., 4., 5., 6., 7., 8. +n = 0 +validate 1., 0., 0., 0., 0., 0., 0., 0. +store_fn + +aarch64 = nop +generate *mut f64:float64x1_t:void, *mut f64:float64x2_t:void + +arm = nop +generate *mut f32:float32x2_t:void, *mut f32:float32x4_t:void + +/// Store multiple single-element structures from one, two, three, or four registers +name = vst1 +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32 +store_fn +arm-aarch64-separate + +aarch64 = st1 +link-aarch64 = st1x2._EXT3_ +arm = vst1 +link-arm = vst1x2._EXTr3_ +generate *mut i8:int8x8x2_t:void, *mut i16:int16x4x2_t:void, *mut i32:int32x2x2_t:void, *mut i64:int64x1x2_t:void +generate *mut i8:int8x16x2_t:void, *mut i16:int16x8x2_t:void, *mut i32:int32x4x2_t:void, *mut i64:int64x2x2_t:void + +link-aarch64 = st1x3._EXT3_ +link-arm = vst1x3._EXTr3_ +generate *mut i8:int8x8x3_t:void, *mut i16:int16x4x3_t:void, *mut i32:int32x2x3_t:void, *mut i64:int64x1x3_t:void +generate *mut i8:int8x16x3_t:void, *mut i16:int16x8x3_t:void, *mut i32:int32x4x3_t:void, *mut i64:int64x2x3_t:void + +link-aarch64 = st1x4._EXT3_ +link-arm = vst1x4._EXTr3_ +generate *mut i8:int8x8x4_t:void, *mut i16:int16x4x4_t:void, *mut i32:int32x2x4_t:void, *mut i64:int64x1x4_t:void +generate *mut i8:int8x16x4_t:void, *mut i16:int16x8x4_t:void, *mut i32:int32x4x4_t:void, *mut i64:int64x2x4_t:void + +/// Store multiple single-element structures to one, two, three, or four registers +name = vst1 +multi_fn = vst1-signed-noext, transmute(a), transmute(b) +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32 + +store_fn +aarch64 = st1 +arm = vst1 +generate *mut u8:uint8x8x2_t:void, *mut u16:uint16x4x2_t:void, *mut u32:uint32x2x2_t:void, *mut u64:uint64x1x2_t:void +generate *mut u8:uint8x16x2_t:void, *mut u16:uint16x8x2_t:void, *mut u32:uint32x4x2_t:void, *mut u64:uint64x2x2_t:void +generate *mut u8:uint8x8x3_t:void, *mut u16:uint16x4x3_t:void, *mut u32:uint32x2x3_t:void, *mut u64:uint64x1x3_t:void +generate *mut u8:uint8x16x3_t:void, *mut u16:uint16x8x3_t:void, *mut u32:uint32x4x3_t:void, *mut u64:uint64x2x3_t:void +generate *mut u8:uint8x8x4_t:void, *mut u16:uint16x4x4_t:void, *mut u32:uint32x2x4_t:void, *mut u64:uint64x1x4_t:void +generate *mut u8:uint8x16x4_t:void, *mut u16:uint16x8x4_t:void, *mut u32:uint32x4x4_t:void, *mut u64:uint64x2x4_t:void +generate *mut p8:poly8x8x2_t:void, *mut p8:poly8x8x3_t:void, *mut p8:poly8x8x4_t:void +generate *mut p8:poly8x16x2_t:void, *mut p8:poly8x16x3_t:void, *mut p8:poly8x16x4_t:void +generate *mut p16:poly16x4x2_t:void, *mut p16:poly16x4x3_t:void, *mut p16:poly16x4x4_t:void +generate *mut p16:poly16x8x2_t:void, *mut p16:poly16x8x3_t:void, *mut p16:poly16x8x4_t:void +target = aes +generate *mut p64:poly64x1x2_t:void +arm = nop +generate *mut p64:poly64x1x3_t:void, *mut p64:poly64x1x4_t:void +generate *mut p64:poly64x2x2_t:void, *mut p64:poly64x2x3_t:void, *mut p64:poly64x2x4_t:void + +/// Store multiple single-element structures to one, two, three, or four registers +name = vst1 +a = 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16. +validate 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16. +store_fn +arm-aarch64-separate + +aarch64 = st1 +link-aarch64 = st1x2._EXT3_ +generate *mut f64:float64x1x2_t:void, *mut f64:float64x2x2_t:void + +link-aarch64 = st1x3._EXT3_ +generate *mut f64:float64x1x3_t:void, *mut f64:float64x2x3_t:void + +link-aarch64 = st1x4._EXT3_ +generate *mut f64:float64x1x4_t:void, *mut f64:float64x2x4_t:void + +arm = vst1 +link-aarch64 = st1x2._EXT3_ +link-arm = vst1x2._EXTr3_ +generate *mut f32:float32x2x2_t:void, *mut f32:float32x4x2_t:void + +link-aarch64 = st1x3._EXT3_ +link-arm = vst1x3._EXTr3_ +generate *mut f32:float32x2x3_t:void, *mut f32:float32x4x3_t:void + +link-aarch64 = st1x4._EXT3_ +link-arm = vst1x4._EXTr3_ +generate *mut f32:float32x2x4_t:void, *mut f32:float32x4x4_t:void + +/// Store multiple 2-element structures from two registers +name = vst2 +in1-nox +a = 0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 +validate 1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15, 8, 16, 9, 17 +store_fn +arm-aarch64-separate + +aarch64 = st2 +link-aarch64 = st2._EXTpi8_ +generate *mut i64:int64x2x2_t:void + +arm = vst2 +link-arm = vst2._EXTpi8r_ +generate *mut i8:int8x8x2_t:void, *mut i16:int16x4x2_t:void, *mut i32:int32x2x2_t:void +generate *mut i8:int8x16x2_t:void, *mut i16:int16x8x2_t:void, *mut i32:int32x4x2_t:void +arm = nop +aarch64 = nop +generate *mut i64:int64x1x2_t:void + +/// Store multiple 2-element structures from two registers +name = vst2 +multi_fn = transmute, {vst2-in1signednox-noext, transmute(a), transmute(b)} +in1-nox +a = 0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 +validate 1, 2, 2, 3, 2, 4, 3, 5, 2, 6, 3, 7, 4, 8, 5, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15, 8, 16, 9, 17 +store_fn + +aarch64 = st2 +generate *mut u64:uint64x2x2_t:void +target = aes +generate *mut p64:poly64x2x2_t:void + +target = default +arm = vst2 +generate *mut u8:uint8x8x2_t:void, *mut u16:uint16x4x2_t:void, *mut u32:uint32x2x2_t:void +generate *mut u8:uint8x16x2_t:void, *mut u16:uint16x8x2_t:void, *mut u32:uint32x4x2_t:void +generate *mut p8:poly8x8x2_t:void, *mut p16:poly16x4x2_t:void, *mut p8:poly8x16x2_t:void, *mut p16:poly16x8x2_t:void +arm = nop +aarch64 = nop +generate *mut u64:uint64x1x2_t:void +target = aes +generate *mut p64:poly64x1x2_t:void + +/// Store multiple 2-element structures from two registers +name = vst2 +in1-nox +a = 0., 1., 2., 2., 3., 2., 3., 4., 5., 2., 3., 4., 5., 6., 7., 8., 9. +validate 1., 2., 2., 3., 2., 4., 3., 5., 2., 6., 3., 7., 4., 8., 5., 9. +store_fn +arm-aarch64-separate + +aarch64 = st1 +link-aarch64 = st2._EXTpi8_ +generate *mut f64:float64x1x2_t:void +aarch64 = st2 +generate *mut f64:float64x2x2_t:void + +arm = vst2 +link-arm = vst2._EXTpi8r_ +generate *mut f32:float32x2x2_t:void, *mut f32:float32x4x2_t:void + +/// Store multiple 2-element structures from two registers +name = vst2 +in1-lane-nox +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +a = 0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 +n = 0 +validate 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +store_fn +arm-aarch64-separate + +aarch64 = st2 +link-aarch64 = st2lane._EXTpi8_ +const-aarch64 = LANE +generate *mut i8:int8x16x2_t:void, *mut i64:int64x1x2_t:void, *mut i64:int64x2x2_t:void + +arm = vst2 +link-arm = vst2lane._EXTpi8r_ +const-arm = LANE +generate *mut i8:int8x8x2_t:void, *mut i16:int16x4x2_t:void, *mut i32:int32x2x2_t:void +generate *mut i16:int16x8x2_t:void, *mut i32:int32x4x2_t:void + +/// Store multiple 2-element structures from two registers +name = vst2 +in1-lane-nox +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = transmute, {vst2-in1signedlanenox-::, transmute(a), transmute(b)} +a = 0, 1, 2, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 6, 7, 8, 9, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 +n = 0 +validate 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +store_fn + +aarch64 = st2 +generate *mut u8:uint8x16x2_t:void, *mut u64:uint64x1x2_t:void, *mut u64:uint64x2x2_t:void, *mut p8:poly8x16x2_t:void +target = aes +generate *mut p64:poly64x1x2_t:void, *mut p64:poly64x2x2_t:void + +target = default +arm = vst2 +generate *mut u8:uint8x8x2_t:void, *mut u16:uint16x4x2_t:void, *mut u32:uint32x2x2_t:void +generate *mut u16:uint16x8x2_t:void, *mut u32:uint32x4x2_t:void +generate *mut p8:poly8x8x2_t:void, *mut p16:poly16x4x2_t:void, *mut p16:poly16x8x2_t:void + +/// Store multiple 2-element structures from two registers +name = vst2 +in1-lane-nox +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +a = 0., 1., 2., 2., 3., 2., 3., 4., 5., 2., 3., 4., 5., 6., 7., 8., 9. +n = 0 +validate 1., 2., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. +store_fn +arm-aarch64-separate + +aarch64 = st2 +link-aarch64 = st2lane._EXTpi8_ +const-aarch64 = LANE +generate *mut f64:float64x1x2_t:void, *mut f64:float64x2x2_t:void + +arm = vst2 +link-arm = vst2lane._EXTpi8r_ +const-arm = LANE +generate *mut f32:float32x2x2_t:void, *mut f32:float32x4x2_t:void + +/// Store multiple 3-element structures from three registers +name = vst3 +in1-nox +a = 0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16, 25, 26, 27, 28, 29, 30, 31, 32, 2, 4, 7, 8, 13, 14, 15, 16, 41, 42, 43, 44, 45, 46, 47, 48 +validate 1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16, 2, 25, 41, 4, 26, 42, 7, 27, 43, 8, 28, 44, 13, 29, 45, 14, 30, 46, 15, 31, 47, 16, 32, 48 +store_fn +arm-aarch64-separate + +aarch64 = st3 +link-aarch64 = st3._EXTpi8_ +generate *mut i64:int64x2x3_t:void + +arm = vst3 +link-arm = vst3._EXTpi8r_ +generate *mut i8:int8x8x3_t:void, *mut i16:int16x4x3_t:void, *mut i32:int32x2x3_t:void +generate *mut i8:int8x16x3_t:void, *mut i16:int16x8x3_t:void, *mut i32:int32x4x3_t:void +arm = nop +aarch64 = nop +generate *mut i64:int64x1x3_t:void + +/// Store multiple 3-element structures from three registers +name = vst3 +multi_fn = transmute, {vst3-in1signednox-noext, transmute(a), transmute(b)} +in1-nox +a = 0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16, 25, 26, 27, 28, 29, 30, 31, 32, 2, 4, 7, 8, 13, 14, 15, 16, 41, 42, 43, 44, 45, 46, 47, 48 +validate 1, 2, 2, 2, 4, 4, 2, 7, 7, 4, 8, 8, 2, 13, 13, 4, 14, 14, 7, 15, 15, 8, 16, 16, 2, 25, 41, 4, 26, 42, 7, 27, 43, 8, 28, 44, 13, 29, 45, 14, 30, 46, 15, 31, 47, 16, 32, 48 +store_fn + +aarch64 = st3 +generate *mut u64:uint64x2x3_t:void +target = aes +generate *mut p64:poly64x2x3_t:void + +target = default +arm = vst3 +generate *mut u8:uint8x8x3_t:void, *mut u16:uint16x4x3_t:void, *mut u32:uint32x2x3_t:void +generate *mut u8:uint8x16x3_t:void, *mut u16:uint16x8x3_t:void, *mut u32:uint32x4x3_t:void +generate *mut p8:poly8x8x3_t:void, *mut p16:poly16x4x3_t:void, *mut p8:poly8x16x3_t:void, *mut p16:poly16x8x3_t:void +arm = nop +aarch64 = nop +generate *mut u64:uint64x1x3_t:void +target = aes +generate *mut p64:poly64x1x3_t:void + +/// Store multiple 3-element structures from three registers +name = vst3 +in1-nox +a = 0., 1., 2., 2., 4., 2., 4., 7., 8., 2., 4., 7., 8., 13., 14., 15., 16 +validate 1., 2., 2., 2., 4., 4., 2., 7., 7., 4., 8., 8., 2., 13., 13., 4. +store_fn +arm-aarch64-separate + +aarch64 = nop +link-aarch64 = st3._EXTpi8_ +generate *mut f64:float64x1x3_t:void +aarch64 = st3 +generate *mut f64:float64x2x3_t:void + +arm = vst3 +link-arm = vst3._EXTpi8r_ +generate *mut f32:float32x2x3_t:void, *mut f32:float32x4x3_t:void + +/// Store multiple 3-element structures from three registers +name = vst3 +in1-lane-nox +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +a = 0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16, 25, 26, 27, 28, 29, 30, 31, 32, 2, 4, 7, 8, 13, 14, 15, 16, 41, 42, 43, 44, 45, 46, 47, 48 +n = 0 +validate 1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +store_fn +arm-aarch64-separate + +aarch64 = st3 +link-aarch64 = st3lane._EXTpi8_ +const-aarch64 = LANE +generate *mut i8:int8x16x3_t:void, *mut i64:int64x1x3_t:void, *mut i64:int64x2x3_t:void + +arm = vst3 +link-arm = vst3lane._EXTpi8r_ +const-arm = LANE +generate *mut i8:int8x8x3_t:void, *mut i16:int16x4x3_t:void, *mut i32:int32x2x3_t:void +generate *mut i16:int16x8x3_t:void, *mut i32:int32x4x3_t:void + +/// Store multiple 3-element structures from three registers +name = vst3 +in1-lane-nox +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = transmute, {vst3-in1signedlanenox-::, transmute(a), transmute(b)} +a = 0, 1, 2, 2, 4, 2, 4, 7, 8, 2, 4, 7, 8, 13, 14, 15, 16, 2, 4, 7, 8, 13, 14, 15, 16, 25, 26, 27, 28, 29, 30, 31, 32, 2, 4, 7, 8, 13, 14, 15, 16, 41, 42, 43, 44, 45, 46, 47, 48 +n = 0 +validate 1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +store_fn + +aarch64 = st3 +generate *mut u8:uint8x16x3_t:void, *mut u64:uint64x1x3_t:void, *mut u64:uint64x2x3_t:void, *mut p8:poly8x16x3_t:void +target = aes +generate *mut p64:poly64x1x3_t:void, *mut p64:poly64x2x3_t:void + +target = default +arm = vst3 +generate *mut u8:uint8x8x3_t:void, *mut u16:uint16x4x3_t:void, *mut u32:uint32x2x3_t:void +generate *mut u16:uint16x8x3_t:void, *mut u32:uint32x4x3_t:void +generate *mut p8:poly8x8x3_t:void, *mut p16:poly16x4x3_t:void, *mut p16:poly16x8x3_t:void + +/// Store multiple 3-element structures from three registers +name = vst3 +in1-lane-nox +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +a = 0., 1., 2., 2., 3., 2., 3., 4., 5., 2., 3., 4., 5., 6., 7., 8., 9. +n = 0 +validate 1., 2., 2., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. +store_fn +arm-aarch64-separate + +aarch64 = st3 +link-aarch64 = st3lane._EXTpi8_ +const-aarch64 = LANE +generate *mut f64:float64x1x3_t:void, *mut f64:float64x2x3_t:void + +arm = vst3 +link-arm = vst3lane._EXTpi8r_ +const-arm = LANE +generate *mut f32:float32x2x3_t:void, *mut f32:float32x4x3_t:void + +/// Store multiple 4-element structures from four registers +name = vst4 +in1-nox +a = 0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64 +validate 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64 +store_fn +arm-aarch64-separate + +aarch64 = st4 +link-aarch64 = st4._EXTpi8_ +generate *mut i64:int64x2x4_t:void + +arm = vst4 +link-arm = vst4._EXTpi8r_ +generate *mut i8:int8x8x4_t:void, *mut i16:int16x4x4_t:void, *mut i32:int32x2x4_t:void +generate *mut i8:int8x16x4_t:void, *mut i16:int16x8x4_t:void, *mut i32:int32x4x4_t:void +arm = nop +aarch64 = nop +generate *mut i64:int64x1x4_t:void + +/// Store multiple 4-element structures from four registers +name = vst4 +multi_fn = transmute, {vst4-in1signednox-noext, transmute(a), transmute(b)} +in1-nox +a = 0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64 +validate 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64 +store_fn + +aarch64 = st4 +generate *mut u64:uint64x2x4_t:void +target = aes +generate *mut p64:poly64x2x4_t:void + +target = default +arm = vst4 +generate *mut u8:uint8x8x4_t:void, *mut u16:uint16x4x4_t:void, *mut u32:uint32x2x4_t:void +generate *mut u8:uint8x16x4_t:void, *mut u16:uint16x8x4_t:void, *mut u32:uint32x4x4_t:void +generate *mut p8:poly8x8x4_t:void, *mut p16:poly16x4x4_t:void, *mut p8:poly8x16x4_t:void, *mut p16:poly16x8x4_t:void +arm = nop +aarch64 = nop +generate *mut u64:uint64x1x4_t:void +target = aes +generate *mut p64:poly64x1x4_t:void + +/// Store multiple 4-element structures from four registers +name = vst4 +in1-nox +a = 0., 1., 2., 2., 6., 2., 6., 6., 8., 2., 6., 6., 8., 6., 8., 8., 16. +validate 1., 2., 2., 6., 2., 6., 6., 8., 2., 6., 6., 8., 6., 8., 8., 16. +store_fn +arm-aarch64-separate + +aarch64 = nop +link-aarch64 = st4._EXTpi8_ +generate *mut f64:float64x1x4_t:void +aarch64 = st4 +generate *mut f64:float64x2x4_t:void + +arm = vst4 +link-arm = vst4._EXTpi8r_ +generate *mut f32:float32x2x4_t:void, *mut f32:float32x4x4_t:void + +/// Store multiple 4-element structures from four registers +name = vst4 +in1-lane-nox +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +a = 0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64 +n = 0 +validate 1, 2, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +store_fn +arm-aarch64-separate + +aarch64 = st4 +link-aarch64 = st4lane._EXTpi8_ +const-aarch64 = LANE +generate *mut i8:int8x16x4_t:void, *mut i64:int64x1x4_t:void, *mut i64:int64x2x4_t:void + +arm = vst4 +link-arm = vst4lane._EXTpi8r_ +const-arm = LANE +generate *mut i8:int8x8x4_t:void, *mut i16:int16x4x4_t:void, *mut i32:int32x2x4_t:void +generate *mut i16:int16x8x4_t:void, *mut i32:int32x4x4_t:void + +/// Store multiple 4-element structures from four registers +name = vst4 +in1-lane-nox +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = transmute, {vst4-in1signedlanenox-::, transmute(a), transmute(b)} +a = 0, 1, 2, 2, 6, 2, 6, 6, 8, 2, 6, 6, 8, 6, 8, 8, 16, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 8, 16, 8, 16, 16, 32, 2, 6, 6, 8, 6, 8, 8, 16, 6, 8, 43, 44, 8, 16, 44, 48, 6, 8, 8, 16, 8, 16, 16, 32, 8, 16, 44, 48, 16, 32, 48, 64 +n = 0 +validate 1, 2, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +store_fn + +aarch64 = st4 +generate *mut u8:uint8x16x4_t:void, *mut u64:uint64x1x4_t:void, *mut u64:uint64x2x4_t:void, *mut p8:poly8x16x4_t:void +target = aes +generate *mut p64:poly64x1x4_t:void, *mut p64:poly64x2x4_t:void + +target = default +arm = vst4 +generate *mut u8:uint8x8x4_t:void, *mut u16:uint16x4x4_t:void, *mut u32:uint32x2x4_t:void +generate *mut u16:uint16x8x4_t:void, *mut u32:uint32x4x4_t:void +generate *mut p8:poly8x8x4_t:void, *mut p16:poly16x4x4_t:void, *mut p16:poly16x8x4_t:void + +/// Store multiple 4-element structures from four registers +name = vst4 +in1-lane-nox +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +a = 0., 1., 2., 2., 6., 2., 6., 6., 8., 2., 6., 6., 8., 6., 8., 8., 16. +n = 0 +validate 1., 2., 2., 6., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. +store_fn +arm-aarch64-separate + +aarch64 = st4 +link-aarch64 = st4lane._EXTpi8_ +const-aarch64 = LANE +generate *mut f64:float64x1x4_t:void, *mut f64:float64x2x4_t:void + +arm = vst4 +link-arm = vst4lane._EXTpi8r_ +const-arm = LANE +generate *mut f32:float32x2x4_t:void, *mut f32:float32x4x4_t:void + +/// Dot product index form with signed and unsigned integers +name = vsudot +out-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_dot-LANE +multi_fn = simd_shuffle-in_len-!, c:unsigned, c, c, {base-4-LANE} +multi_fn = vsudot-outlane-_, a, b, c +a = 1, 2, 1, 2 +b = 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 +c = 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 +n = 0 +validate 31, 72, 31, 72 +target = dotprod + +aarch64 = sudot +link-aarch64 = usdot._EXT2_._EXT4_:int32x2_t:int8x8_t:uint8x8_t:int32x2_t +// LLVM ERROR: Cannot select: intrinsic %llvm.aarch64.neon.usdot +//generate int32x2_t:int8x8_t:uint8x8_t:int32x2_t, int32x2_t:int8x8_t:uint8x16_t:int32x2_t +link-aarch64 = usdot._EXT2_._EXT4_:int32x4_t:int8x16_t:uint8x16_t:int32x4_t +// LLVM ERROR: Cannot select: intrinsic %llvm.aarch64.neon.usdot +//generate int32x4_t:int8x16_t:uint8x8_t:int32x4_t, int32x4_t:int8x16_t:uint8x16_t:int32x4_t + +/// Multiply +name = vmul +a = 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +validate 1, 4, 3, 8, 5, 12, 7, 16, 9, 20, 11, 24, 13, 28, 15, 32 +arm = vmul. +aarch64 = mul +fn = simd_mul +generate int*_t, uint*_t + +/// Polynomial multiply +name = vmul +a = 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +validate 1, 6, 3, 12, 5, 10, 7, 24, 9, 30, 11, 20, 13, 18, 15, 48 + +aarch64 = pmul +link-aarch64 = pmul._EXT_ +arm = vmul +link-arm = vmulp._EXT_ +generate poly8x8_t, poly8x16_t + +/// Multiply +name = vmul +fn = simd_mul +a = 1.0, 2.0, 1.0, 2.0 +b = 2.0, 3.0, 4.0, 5.0 +validate 2.0, 6.0, 4.0, 10.0 + +aarch64 = fmul +generate float64x*_t + +arm = vmul. +generate float*_t + +/// Vector multiply by scalar +name = vmul +out-n-suffix +multi_fn = simd_mul, a, {vdup-nout-noext, b} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 2 +validate 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32 + +arm = vmul +aarch64 = mul +generate int16x4_t:i16:int16x4_t, int16x8_t:i16:int16x8_t, int32x2_t:i32:int32x2_t, int32x4_t:i32:int32x4_t +generate uint16x4_t:u16:uint16x4_t, uint16x8_t:u16:uint16x8_t, uint32x2_t:u32:uint32x2_t, uint32x4_t:u32:uint32x4_t + +/// Vector multiply by scalar +name = vmul +out-n-suffix +multi_fn = simd_mul, a, {vdup-nout-noext, b} +a = 1., 2., 3., 4. +b = 2. +validate 2., 4., 6., 8. + +aarch64 = fmul +generate float64x1_t:f64:float64x1_t, float64x2_t:f64:float64x2_t + +arm = vmul +generate float32x2_t:f32:float32x2_t, float32x4_t:f32:float32x4_t + +/// Multiply +name = vmul +lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = simd_mul, a, {simd_shuffle-out_len-!, b, b, {dup-out_len-LANE as u32}} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 1 +validate 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32 + +aarch64 = mul +arm = vmul +generate int16x4_t, int16x4_t:int16x8_t:int16x4_t, int16x8_t:int16x4_t:int16x8_t, int16x8_t +generate int32x2_t, int32x2_t:int32x4_t:int32x2_t, int32x4_t:int32x2_t:int32x4_t, int32x4_t +generate uint16x4_t, uint16x4_t:uint16x8_t:uint16x4_t, uint16x8_t:uint16x4_t:uint16x8_t, uint16x8_t +generate uint32x2_t, uint32x2_t:uint32x4_t:uint32x2_t, uint32x4_t:uint32x2_t:uint32x4_t, uint32x4_t + +/// Floating-point multiply +name = vmul +lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = simd_mul, a, {transmute--, {simd_extract, b, LANE as u32}} +a = 1., 2., 3., 4. +b = 2., 0., 0., 0. +n = 0 +validate 2., 4., 6., 8. + +aarch64 = fmul +generate float64x1_t, float64x1_t:float64x2_t:float64x1_t + +/// Floating-point multiply +name = vmul +lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = simd_mul, a, {simd_shuffle-out_len-!, b, b, {dup-out_len-LANE as u32}} +a = 1., 2., 3., 4. +b = 2., 0., 0., 0. +n = 0 +validate 2., 4., 6., 8. + +aarch64 = fmul +generate float64x2_t:float64x1_t:float64x2_t, float64x2_t + +arm = vmul +generate float32x2_t, float32x2_t:float32x4_t:float32x2_t, float32x4_t:float32x2_t:float32x4_t, float32x4_t + +/// Floating-point multiply +name = vmuls_lane +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = simd_extract, b:f32, b, LANE as u32 +multi_fn = a * b +a = 1. +b = 2., 0., 0., 0. +n = 0 +validate 2. +aarch64 = fmul +generate f32:float32x2_t:f32, f32:float32x4_t:f32 + +/// Floating-point multiply +name = vmuld_lane +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = simd_extract, b:f64, b, LANE as u32 +multi_fn = a * b +a = 1. +b = 2., 0. +n = 0 +validate 2. +aarch64 = fmul +generate f64:float64x1_t:f64, f64:float64x2_t:f64 + +/// Signed multiply long +name = vmull +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2 +validate 1, 4, 3, 8, 5, 12, 7, 16, 9, 20, 11, 24, 13, 28, 15, 32 + +arm = vmull.s +aarch64 = smull +link-arm = vmulls._EXT_ +link-aarch64 = smull._EXT_ +generate int8x8_t:int8x8_t:int16x8_t, int16x4_t:int16x4_t:int32x4_t, int32x2_t:int32x2_t:int64x2_t + +/// Signed multiply long +name = vmull_high +no-q +multi_fn = simd_shuffle-out_len-!, a:half, a, a, {fixed-half-right} +multi_fn = simd_shuffle-out_len-!, b:half, b, b, {fixed-half-right} +multi_fn = vmull-noqself-noext, a, b +a = 1, 2, 9, 10, 9, 10, 11, 12, 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2 +fixed = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 9, 20, 11, 24, 13, 28, 15, 32 + +aarch64 = smull2 +generate int8x16_t:int8x16_t:int16x8_t, int16x8_t:int16x8_t:int32x4_t, int32x4_t:int32x4_t:int64x2_t + +/// Unsigned multiply long +name = vmull +a = 1, 2, 3, 4, 5, 6, 7, 8 +b = 1, 2, 1, 2, 1, 2, 1, 2 +validate 1, 4, 3, 8, 5, 12, 7, 16 + +arm = vmull.s +aarch64 = umull +link-arm = vmullu._EXT_ +link-aarch64 = umull._EXT_ +generate uint8x8_t:uint8x8_t:uint16x8_t, uint16x4_t:uint16x4_t:uint32x4_t, uint32x2_t:uint32x2_t:uint64x2_t + +/// Unsigned multiply long +name = vmull_high +no-q +multi_fn = simd_shuffle-out_len-!, a:half, a, a, {fixed-half-right} +multi_fn = simd_shuffle-out_len-!, b:half, b, b, {fixed-half-right} +multi_fn = vmull-noqself-noext, a, b +a = 1, 2, 9, 10, 9, 10, 11, 12, 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2 +fixed = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 9, 20, 11, 24, 13, 28, 15, 32 + +aarch64 = umull2 +generate uint8x16_t:uint8x16_t:uint16x8_t, uint16x8_t:uint16x8_t:uint32x4_t, uint32x4_t:uint32x4_t:uint64x2_t + +/// Polynomial multiply long +name = vmull +a = 1, 2, 3, 4, 5, 6, 7, 8 +b = 1, 3, 1, 3, 1, 3, 1, 3 +validate 1, 6, 3, 12, 5, 10, 7, 24 + +arm = vmull.s +aarch64 = pmull +link-arm = vmullp._EXT_ +link-aarch64 = pmull._EXT_ +generate poly8x8_t:poly8x8_t:poly16x8_t + +/// Polynomial multiply long +name = vmull +no-q +a = 15 +b = 3 +validate 17 +target = aes + +aarch64 = pmull +link-aarch64 = pmull64:p64:p64:p64:int8x16_t +// Because of the support status of llvm, vmull_p64 is currently only available on arm +// arm = vmull +// link-arm = vmullp.v2i64:int64x1_t:int64x1_t:int64x1_t:int64x2_t +generate p64:p64:p128 + + +/// Polynomial multiply long +name = vmull_high +no-q +multi_fn = simd_shuffle-out_len-!, a:half, a, a, {fixed-half-right} +multi_fn = simd_shuffle-out_len-!, b:half, b, b, {fixed-half-right} +multi_fn = vmull-noqself-noext, a, b +a = 1, 2, 9, 10, 9, 10, 11, 12, 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3 +fixed = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 9, 30, 11, 20, 13, 18, 15, 48 + +aarch64 = pmull +generate poly8x16_t:poly8x16_t:poly16x8_t + +/// Polynomial multiply long +name = vmull_high +no-q +multi_fn = vmull-noqself-noext, {simd_extract, a, 1}, {simd_extract, b, 1} +a = 1, 15 +b = 1, 3 +validate 17 +target = aes + +aarch64 = pmull +generate poly64x2_t:poly64x2_t:p128 + +/// Vector long multiply with scalar +name = vmull_n +no-q +multi_fn = vmull-in0-noext, a, {vdup-nin0-noext, b} +a = 1, 2, 3, 4, 5, 6, 7, 8 +b = 2 +validate 2, 4, 6, 8, 10, 12, 14, 16 + +arm = vmull +aarch64 = smull +generate int16x4_t:i16:int32x4_t, int32x2_t:i32:int64x2_t +aarch64 = umull +generate uint16x4_t:u16:uint32x4_t, uint32x2_t:u32:uint64x2_t + +/// Vector long multiply by scalar +name = vmull_lane +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = vmull-in0-noext, a, {simd_shuffle-in0_len-!, b, b, {dup-in0_len-LANE as u32}} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 1 +validate 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32 + +arm = vmull +aarch64 = smull +generate int16x4_t:int16x4_t:int32x4_t, int16x4_t:int16x8_t:int32x4_t +generate int32x2_t:int32x2_t:int64x2_t, int32x2_t:int32x4_t:int64x2_t +aarch64 = umull +generate uint16x4_t:uint16x4_t:uint32x4_t, uint16x4_t:uint16x8_t:uint32x4_t +generate uint32x2_t:uint32x2_t:uint64x2_t, uint32x2_t:uint32x4_t:uint64x2_t + +/// Multiply long +name = vmull_high_n +no-q +multi_fn = vmull_high-noqself-noext, a, {vdup-nin0-noext, b} +a = 1, 2, 9, 10, 9, 10, 11, 12, 9, 10, 11, 12, 13, 14, 15, 16 +b = 2 +validate 18, 20, 22, 24, 26, 28, 30, 32 + +aarch64 = smull2 +generate int16x8_t:i16:int32x4_t, int32x4_t:i32:int64x2_t +aarch64 = umull2 +generate uint16x8_t:u16:uint32x4_t, uint32x4_t:u32:uint64x2_t + +/// Multiply long +name = vmull_high_lane +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = vmull_high-noqself-noext, a, {simd_shuffle-in0_len-!, b, b, {dup-in0_len-LANE as u32}} +a = 1, 2, 9, 10, 9, 10, 11, 12, 9, 10, 11, 12, 13, 14, 15, 16 +b = 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 1 +validate 18, 20, 22, 24, 26, 28, 30, 32 + +aarch64 = smull2 +generate int16x8_t:int16x4_t:int32x4_t, int16x8_t:int16x8_t:int32x4_t +generate int32x4_t:int32x2_t:int64x2_t, int32x4_t:int32x4_t:int64x2_t +aarch64 = umull2 +generate uint16x8_t:uint16x4_t:uint32x4_t, uint16x8_t:uint16x8_t:uint32x4_t +generate uint32x4_t:uint32x2_t:uint64x2_t, uint32x4_t:uint32x4_t:uint64x2_t + +/// Floating-point multiply extended +name = vmulx +a = 1., 2., 3., 4. +b = 2., 2., 2., 2. +validate 2., 4., 6., 8. + +aarch64 = fmulx +link-aarch64 = fmulx._EXT_ +generate float*_t, float64x*_t + +/// Floating-point multiply extended +name = vmulx +lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = vmulx-in0-noext, a, {transmute--, {simd_extract, b, LANE as u32}} +a = 1. +b = 2., 0. +n = 0 +validate 2. + +aarch64 = fmulx +generate float64x1_t, float64x1_t:float64x2_t:float64x1_t + +/// Floating-point multiply extended +name = vmulx +lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = vmulx-in0-noext, a, {simd_shuffle-in0_len-!, b, b, {dup-in0_len-LANE as u32}} +a = 1., 2., 3., 4. +b = 2., 0., 0., 0. +n = 0 +validate 2., 4., 6., 8. + +aarch64 = fmulx +generate float32x2_t, float32x2_t:float32x4_t:float32x2_t, float32x4_t:float32x2_t:float32x4_t, float32x4_t +generate float64x2_t:float64x1_t:float64x2_t, float64x2_t + +/// Floating-point multiply extended +name = vmulx +a = 2. +b = 3. +validate 6. + +aarch64 = fmulx +link-aarch64 = fmulx._EXT_ +generate f32, f64 + +/// Floating-point multiply extended +name = vmulx +lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = vmulx-out-noext, a, {simd_extract, b, LANE as u32} + +a = 2. +b = 3., 0., 0., 0. +n = 0 +validate 6. + +aarch64 = fmulx +generate f32:float32x2_t:f32, f32:float32x4_t:f32, f64:float64x1_t:f64, f64:float64x2_t:f64 + +/// Floating-point fused Multiply-Add to accumulator(vector) +name = vfma +multi_fn = vfma-self-_, b, c, a +a = 8.0, 18.0, 12.0, 10.0 +b = 6.0, 4.0, 7.0, 8.0 +c = 2.0, 3.0, 4.0, 5.0 +validate 20.0, 30.0, 40.0, 50.0 + +link-aarch64 = llvm.fma._EXT_ +aarch64 = fmadd +generate float64x1_t +aarch64 = fmla +generate float64x2_t + +target = vfp4 +arm = vfma +link-arm = llvm.fma._EXT_ +generate float*_t + +/// Floating-point fused Multiply-Add to accumulator(vector) +name = vfma +n-suffix +multi_fn = vfma-self-noext, a, b, {vdup-nselfvfp4-noext, c} +a = 2.0, 3.0, 4.0, 5.0 +b = 6.0, 4.0, 7.0, 8.0 +c = 8.0 +validate 50.0, 35.0, 60.0, 69.0 + +aarch64 = fmadd +generate float64x1_t:float64x1_t:f64:float64x1_t +aarch64 = fmla +generate float64x2_t:float64x2_t:f64:float64x2_t + +target = vfp4 +arm = vfma +generate float32x2_t:float32x2_t:f32:float32x2_t, float32x4_t:float32x4_t:f32:float32x4_t + +/// Floating-point fused multiply-add to accumulator +name = vfma +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vfma-out-noext, a, b, {vdup-nout-noext, {simd_extract, c, LANE as u32}} +a = 2., 3., 4., 5. +b = 6., 4., 7., 8. +c = 2., 0., 0., 0. +n = 0 +validate 14., 11., 18., 21. + +aarch64 = fmla +generate float32x2_t, float32x2_t:float32x2_t:float32x4_t:float32x2_t, float32x4_t:float32x4_t:float32x2_t:float32x4_t, float32x4_t +aarch64 = fmadd +generate float64x1_t +aarch64 = fmla +generate float64x1_t:float64x1_t:float64x2_t:float64x1_t, float64x2_t:float64x2_t:float64x1_t:float64x2_t, float64x2_t + +/// Floating-point fused multiply-add to accumulator +name = vfma +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = simd_extract, c:out_t, c, LANE as u32 +multi_fn = vfma-in2lane-_, b, c, a +a = 2. +b = 6. +c = 3., 0., 0., 0. +n = 0 +validate 20. + +aarch64 = fmla +link-aarch64 = llvm.fma._EXT_:f32:f32:f32:f32 +generate f32:f32:float32x2_t:f32, f32:f32:float32x4_t:f32 +link-aarch64 = llvm.fma._EXT_:f64:f64:f64:f64 +aarch64 = fmadd +generate f64:f64:float64x1_t:f64 +aarch64 = fmla +generate f64:f64:float64x2_t:f64 + +/// Floating-point fused multiply-subtract from accumulator +name = vfms +multi_fn = simd_neg, b:in_t, b +multi_fn = vfma-self-noext, a, b, c +a = 20.0, 30.0, 40.0, 50.0 +b = 6.0, 4.0, 7.0, 8.0 +c = 2.0, 3.0, 4.0, 5.0 +validate 8.0, 18.0, 12.0, 10.0 + +aarch64 = fmsub +generate float64x1_t +aarch64 = fmls +generate float64x2_t + +target = vfp4 +arm = vfms +generate float*_t + +/// Floating-point fused Multiply-subtract to accumulator(vector) +name = vfms +n-suffix +multi_fn = vfms-self-noext, a, b, {vdup-nselfvfp4-noext, c} +a = 50.0, 35.0, 60.0, 69.0 +b = 6.0, 4.0, 7.0, 8.0 +c = 8.0 +validate 2.0, 3.0, 4.0, 5.0 + +aarch64 = fmsub +generate float64x1_t:float64x1_t:f64:float64x1_t +aarch64 = fmls +generate float64x2_t:float64x2_t:f64:float64x2_t + +target = vfp4 +arm = vfms +generate float32x2_t:float32x2_t:f32:float32x2_t, float32x4_t:float32x4_t:f32:float32x4_t + +/// Floating-point fused multiply-subtract to accumulator +name = vfms +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vfms-out-noext, a, b, {vdup-nout-noext, {simd_extract, c, LANE as u32}} +a = 14., 11., 18., 21. +b = 6., 4., 7., 8. +c = 2., 0., 0., 0. +n = 0 +validate 2., 3., 4., 5. + +aarch64 = fmls +generate float32x2_t, float32x2_t:float32x2_t:float32x4_t:float32x2_t, float32x4_t:float32x4_t:float32x2_t:float32x4_t, float32x4_t +aarch64 = fmsub +generate float64x1_t +aarch64 = fmls +generate float64x1_t:float64x1_t:float64x2_t:float64x1_t, float64x2_t:float64x2_t:float64x1_t:float64x2_t, float64x2_t + +/// Floating-point fused multiply-subtract to accumulator +name = vfms +in2-lane-suffixes +constn = LANE +multi_fn = vfma-in2lane-::, a, -b, c +a = 14. +b = 6. +c = 2., 0., 0., 0. +n = 0 +validate 2. + +aarch64 = fmls +generate f32:f32:float32x2_t:f32, f32:f32:float32x4_t:f32 +aarch64 = fmsub +generate f64:f64:float64x1_t:f64 +aarch64 = fmls +generate f64:f64:float64x2_t:f64 + +/// Divide +name = vdiv +fn = simd_div +a = 2.0, 6.0, 4.0, 10.0 +b = 1.0, 2.0, 1.0, 2.0 +validate 2.0, 3.0, 4.0, 5.0 + +aarch64 = fdiv +generate float*_t, float64x*_t + +/// Subtract +name = vsub +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2 +validate 0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14 +arm = vsub. +aarch64 = sub +fn = simd_sub +generate int*_t, uint*_t, int64x*_t, uint64x*_t + +/// Subtract +name = vsub +fn = simd_sub +a = 1.0, 4.0, 3.0, 8.0 +b = 1.0, 2.0, 3.0, 4.0 +validate 0.0, 2.0, 0.0, 4.0 + +aarch64 = fsub +generate float64x*_t + +arm = vsub. +generate float*_t + +/// Subtract +name = vsub +multi_fn = a.wrapping_sub(b) +a = 3 +b = 2 +validate 1 + +aarch64 = nop +generate i64, u64 + +/// Add +name = vadd +multi_fn = a.wrapping_add(b) +a = 1 +b = 2 +validate 3 + +aarch64 = nop +generate i64, u64 + +/// Bitwise exclusive OR +name = vadd +multi_fn = simd_xor, a, b +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +validate 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, 17 + +aarch64 = nop +arm = nop +generate poly8x8_t, poly16x4_t, poly8x16_t, poly16x8_t, poly64x1_t, poly64x2_t + +/// Bitwise exclusive OR +name = vaddq +no-q +multi_fn = a ^ b +a = 16 +b = 1 +validate 17 + +aarch64 = nop +arm = nop +generate p128 + +/// Floating-point add across vector +name = vaddv +a = 1., 2., 0., 0. +validate 3. + +aarch64 = faddp +link-aarch64 = faddv._EXT2_._EXT_ +generate float32x2_t:f32, float32x4_t:f32, float64x2_t:f64 + +/// Signed Add Long across Vector +name = vaddlv +a = 1, 2, 3, 4 +validate 10 + +aarch64 = saddlv +link-aarch64 = llvm.aarch64.neon.saddlv.i32._EXT_ +generate int16x4_t:i32 + +/// Signed Add Long across Vector +name = vaddlv +a = 1, 2, 3, 4, 5, 6, 7, 8 +validate 36 + +aarch64 = saddlv +link-aarch64 = llvm.aarch64.neon.saddlv.i32._EXT_ +generate int16x8_t:i32 + +/// Signed Add Long across Vector +name = vaddlv +a = 1, 2 +validate 3 + +aarch64 = saddlp +link-aarch64 = llvm.aarch64.neon.saddlv.i64._EXT_ +generate int32x2_t:i64 + +/// Signed Add Long across Vector +name = vaddlv +a = 1, 2, 3, 4 +validate 10 + +aarch64 = saddlv +link-aarch64 = llvm.aarch64.neon.saddlv.i64._EXT_ +generate int32x4_t:i64 + +/// Unsigned Add Long across Vector +name = vaddlv +a = 1, 2, 3, 4 +validate 10 + +aarch64 = uaddlv +link-aarch64 = llvm.aarch64.neon.uaddlv.i32._EXT_ +generate uint16x4_t:u32 + +/// Unsigned Add Long across Vector +name = vaddlv +a = 1, 2, 3, 4, 5, 6, 7, 8 +validate 36 + +aarch64 = uaddlv +link-aarch64 = llvm.aarch64.neon.uaddlv.i32._EXT_ +generate uint16x8_t:u32 + +/// Unsigned Add Long across Vector +name = vaddlv +a = 1, 2 +validate 3 + +aarch64 = uaddlp +link-aarch64 = llvm.aarch64.neon.uaddlv.i64._EXT_ +generate uint32x2_t:u64 + +/// Unsigned Add Long across Vector +name = vaddlv +a = 1, 2, 3, 4 +validate 10 + +aarch64 = uaddlv +link-aarch64 = llvm.aarch64.neon.uaddlv.i64._EXT_ +generate uint32x4_t:u64 + +/// Subtract returning high narrow +name = vsubhn +no-q +multi_fn = fixed, c:in_t +multi_fn = simd_cast, {simd_shr, {simd_sub, a, b}, transmute(c)} +a = MAX, MIN, 1, 1, MAX, MIN, 1, 1 +b = 1, 0, 0, 0, 1, 0, 0, 0 +fixed = HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS +validate MAX, MIN, 0, 0, MAX, MIN, 0, 0 + +arm = vsubhn +aarch64 = subhn +generate int16x8_t:int8x8_t, int32x4_t:int16x4_t, int64x2_t:int32x2_t +generate uint16x8_t:uint8x8_t, uint32x4_t:uint16x4_t, uint64x2_t:uint32x2_t + +/// Subtract returning high narrow +name = vsubhn_high +no-q +multi_fn = vsubhn-noqself-noext, d:in_t0, b, c +multi_fn = simd_shuffle-out_len-!, a, d, {asc-0-out_len} +a = MAX, 0, MAX, 0, MAX, 0, MAX, 0 +b = MAX, 1, MAX, 1, MAX, 1, MAX, 1 +c = 1, 0, 1, 0, 1, 0, 1, 0 +validate MAX, 0, MAX, 0, MAX, 0, MAX, 0, MAX, 0, MAX, 0, MAX, 0, MAX, 0 + +arm = vsubhn +aarch64 = subhn2 +generate int8x8_t:int16x8_t:int16x8_t:int8x16_t, int16x4_t:int32x4_t:int32x4_t:int16x8_t, int32x2_t:int64x2_t:int64x2_t:int32x4_t +generate uint8x8_t:uint16x8_t:uint16x8_t:uint8x16_t, uint16x4_t:uint32x4_t:uint32x4_t:uint16x8_t, uint32x2_t:uint64x2_t:uint64x2_t:uint32x4_t + +/// Signed halving subtract +name = vhsub +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2 +validate 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7 + +arm = vhsub.s +aarch64 = uhsub +link-arm = vhsubu._EXT_ +link-aarch64 = uhsub._EXT_ +generate uint*_t + +arm = vhsub.s +aarch64 = shsub +link-arm = vhsubs._EXT_ +link-aarch64 = shsub._EXT_ +generate int*_t + +/// Signed Subtract Wide +name = vsubw +no-q +multi_fn = simd_sub, a, {simd_cast, b} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16 +b = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16 +validate 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + +arm = vsubw +aarch64 = ssubw +generate int16x8_t:int8x8_t:int16x8_t, int32x4_t:int16x4_t:int32x4_t, int64x2_t:int32x2_t:int64x2_t + +/// Unsigned Subtract Wide +name = vsubw +no-q +multi_fn = simd_sub, a, {simd_cast, b} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16 +b = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16 +validate 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + +arm = vsubw +aarch64 = usubw +generate uint16x8_t:uint8x8_t:uint16x8_t, uint32x4_t:uint16x4_t:uint32x4_t, uint64x2_t:uint32x2_t:uint64x2_t + +/// Signed Subtract Wide +name = vsubw_high +no-q +multi_fn = simd_shuffle8!, c:int8x8_t, b, b, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_sub, a, {simd_cast, c} +a = 8, 9, 10, 12, 13, 14, 15, 16 +b = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16 +validate 0, 0, 0, 0, 0, 0, 0, 0 + +aarch64 = ssubw +generate int16x8_t:int8x16_t:int16x8_t + +/// Signed Subtract Wide +name = vsubw_high +no-q +multi_fn = simd_shuffle4!, c:int16x4_t, b, b, [4, 5, 6, 7] +multi_fn = simd_sub, a, {simd_cast, c} +a = 8, 9, 10, 11 +b = 0, 1, 2, 3, 8, 9, 10, 11 +validate 0, 0, 0, 0 + +aarch64 = ssubw +generate int32x4_t:int16x8_t:int32x4_t + +/// Signed Subtract Wide +name = vsubw_high +no-q +multi_fn = simd_shuffle2!, c:int32x2_t, b, b, [2, 3] +multi_fn = simd_sub, a, {simd_cast, c} +a = 8, 9 +b = 6, 7, 8, 9 +validate 0, 0 + +aarch64 = ssubw +generate int64x2_t:int32x4_t:int64x2_t + +/// Unsigned Subtract Wide +name = vsubw_high +no-q +multi_fn = simd_shuffle8!, c:uint8x8_t, b, b, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_sub, a, {simd_cast, c} +a = 8, 9, 10, 11, 12, 13, 14, 15 +b = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 0, 0, 0, 0, 0, 0, 0, 0 + +aarch64 = usubw +generate uint16x8_t:uint8x16_t:uint16x8_t + +/// Unsigned Subtract Wide +name = vsubw_high +no-q +multi_fn = simd_shuffle4!, c:uint16x4_t, b, b, [4, 5, 6, 7] +multi_fn = simd_sub, a, {simd_cast, c} +a = 8, 9, 10, 11 +b = 0, 1, 2, 3, 8, 9, 10, 11 +validate 0, 0, 0, 0 + +aarch64 = usubw +generate uint32x4_t:uint16x8_t:uint32x4_t + +/// Unsigned Subtract Wide +name = vsubw_high +no-q +multi_fn = simd_shuffle2!, c:uint32x2_t, b, b, [2, 3] +multi_fn = simd_sub, a, {simd_cast, c} +a = 8, 9 +b = 6, 7, 8, 9 +validate 0, 0 + +aarch64 = usubw +generate uint64x2_t:uint32x4_t:uint64x2_t + +/// Signed Subtract Long +name = vsubl +no-q +multi_fn = simd_cast, c:out_t, a +multi_fn = simd_cast, d:out_t, b +multi_fn = simd_sub, c, d + +a = MAX, MIN, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = MAX, MIN, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + +arm = vsubl +aarch64 = ssubl +generate int8x8_t:int8x8_t:int16x8_t, int16x4_t:int16x4_t:int32x4_t, int32x2_t:int32x2_t:int64x2_t + +/// Unsigned Subtract Long +name = vsubl +no-q +multi_fn = simd_cast, c:out_t, a +multi_fn = simd_cast, d:out_t, b +multi_fn = simd_sub, c, d + +a = MAX, MIN, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = MAX, MIN, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + +arm = vsubl +aarch64 = usubl +generate uint8x8_t:uint8x8_t:uint16x8_t, uint16x4_t:uint16x4_t:uint32x4_t, uint32x2_t:uint32x2_t:uint64x2_t + +/// Signed Subtract Long +name = vsubl_high +no-q +multi_fn = simd_shuffle8!, c:int8x8_t, a, a, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_cast, d:out_t, c +multi_fn = simd_shuffle8!, e:int8x8_t, b, b, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_cast, f:out_t, e +multi_fn = simd_sub, d, f + +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2 +validate 6, 7, 8, 9, 10, 11, 12, 13 + +aarch64 = ssubl +generate int8x16_t:int8x16_t:int16x8_t + +/// Signed Subtract Long +name = vsubl_high +no-q +multi_fn = simd_shuffle4!, c:int16x4_t, a, a, [4, 5, 6, 7] +multi_fn = simd_cast, d:out_t, c +multi_fn = simd_shuffle4!, e:int16x4_t, b, b, [4, 5, 6, 7] +multi_fn = simd_cast, f:out_t, e +multi_fn = simd_sub, d, f + +a = 8, 9, 10, 11, 12, 13, 14, 15 +b = 6, 6, 6, 6, 8, 8, 8, 8 +validate 4, 5, 6, 7 + +aarch64 = ssubl +generate int16x8_t:int16x8_t:int32x4_t + +/// Signed Subtract Long +name = vsubl_high +no-q +multi_fn = simd_shuffle2!, c:int32x2_t, a, a, [2, 3] +multi_fn = simd_cast, d:out_t, c +multi_fn = simd_shuffle2!, e:int32x2_t, b, b, [2, 3] +multi_fn = simd_cast, f:out_t, e +multi_fn = simd_sub, d, f + +a = 12, 13, 14, 15 +b = 6, 6, 8, 8 +validate 6, 7 + +aarch64 = ssubl +generate int32x4_t:int32x4_t:int64x2_t + +/// Unsigned Subtract Long +name = vsubl_high +no-q +multi_fn = simd_shuffle8!, c:uint8x8_t, a, a, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_cast, d:out_t, c +multi_fn = simd_shuffle8!, e:uint8x8_t, b, b, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_cast, f:out_t, e +multi_fn = simd_sub, d, f + +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2 +validate 6, 7, 8, 9, 10, 11, 12, 13 + +aarch64 = usubl +generate uint8x16_t:uint8x16_t:uint16x8_t + +/// Unsigned Subtract Long +name = vsubl_high +no-q +multi_fn = simd_shuffle4!, c:uint16x4_t, a, a, [4, 5, 6, 7] +multi_fn = simd_cast, d:out_t, c +multi_fn = simd_shuffle4!, e:uint16x4_t, b, b, [4, 5, 6, 7] +multi_fn = simd_cast, f:out_t, e +multi_fn = simd_sub, d, f + +a = 8, 9, 10, 11, 12, 13, 14, 15 +b = 6, 6, 6, 6, 8, 8, 8, 8 +validate 4, 5, 6, 7 + +aarch64 = usubl +generate uint16x8_t:uint16x8_t:uint32x4_t + +/// Unsigned Subtract Long +name = vsubl_high +no-q +multi_fn = simd_shuffle2!, c:uint32x2_t, a, a, [2, 3] +multi_fn = simd_cast, d:out_t, c +multi_fn = simd_shuffle2!, e:uint32x2_t, b, b, [2, 3] +multi_fn = simd_cast, f:out_t, e +multi_fn = simd_sub, d, f + +a = 12, 13, 14, 15 +b = 6, 6, 8, 8 +validate 6, 7 + +aarch64 = usubl +generate uint32x4_t:uint32x4_t:uint64x2_t + +/// Bit clear and exclusive OR +name = vbcax +a = 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0 +b = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +c = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +validate 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14 +target = sha3 + +aarch64 = bcax +link-aarch64 = llvm.aarch64.crypto.bcaxs._EXT_ +generate int8x16_t, int16x8_t, int32x4_t, int64x2_t +link-aarch64 = llvm.aarch64.crypto.bcaxu._EXT_ +generate uint8x16_t, uint16x8_t, uint32x4_t, uint64x2_t + +/// Floating-point complex add +name = vcadd_rot270 +no-q +a = 1., -1., 1., -1. +b = -1., 1., -1., 1. +validate 2., 0., 2., 0. +target = fcma + +aarch64 = fcadd +link-aarch64 = vcadd.rot270._EXT_ +generate float32x2_t +name = vcaddq_rot270 +generate float32x4_t, float64x2_t + +/// Floating-point complex add +name = vcadd_rot90 +no-q +a = 1., -1., 1., -1. +b = -1., 1., -1., 1. +validate 0., -2., 0., -2. +target = fcma + +aarch64 = fcadd +link-aarch64 = vcadd.rot90._EXT_ +generate float32x2_t +name = vcaddq_rot90 +generate float32x4_t, float64x2_t + +/// Floating-point complex multiply accumulate +name = vcmla +a = 1., -1., 1., -1. +b = -1., 1., -1., 1. +c = 1., 1., -1., -1. +validate 0., -2., 2., 0. +target = fcma + +aarch64 = fcmla +link-aarch64 = vcmla.rot0._EXT_ +generate float32x2_t, float32x4_t, float64x2_t + +/// Floating-point complex multiply accumulate +name = vcmla_rot90 +rot-suffix +a = 1., 1., 1., 1. +b = 1., -1., 1., -1. +c = 1., 1., 1., 1. +validate 2., 0., 2., 0. +target = fcma + +aarch64 = fcmla +link-aarch64 = vcmla.rot90._EXT_ +generate float32x2_t, float32x4_t, float64x2_t + +/// Floating-point complex multiply accumulate +name = vcmla_rot180 +rot-suffix +a = 1., 1., 1., 1. +b = 1., -1., 1., -1. +c = 1., 1., 1., 1. +validate 0., 0., 0., 0. +target = fcma + +aarch64 = fcmla +link-aarch64 = vcmla.rot180._EXT_ +generate float32x2_t, float32x4_t, float64x2_t + +/// Floating-point complex multiply accumulate +name = vcmla_rot270 +rot-suffix +a = 1., 1., 1., 1. +b = 1., -1., 1., -1. +c = 1., 1., 1., 1. +validate 0., 2., 0., 2. +target = fcma + +aarch64 = fcmla +link-aarch64 = vcmla.rot270._EXT_ +generate float32x2_t, float32x4_t, float64x2_t + +/// Floating-point complex multiply accumulate +name = vcmla +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_rot-LANE +multi_fn = simd_shuffle-out_len-!, c:out_t, c, c, {base-2-LANE} +multi_fn = vcmla-self-noext, a, b, c +a = 1., -1., 1., -1. +b = -1., 1., -1., 1. +c = 1., 1., -1., -1. +n = 0 +validate 0., -2., 0., -2. +target = fcma + +aarch64 = fcmla +generate float32x2_t, float32x2_t:float32x2_t:float32x4_t:float32x2_t +generate float32x4_t:float32x4_t:float32x2_t:float32x4_t, float32x4_t + +/// Floating-point complex multiply accumulate +name = vcmla_rot90 +rot-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_rot-LANE +multi_fn = simd_shuffle-out_len-!, c:out_t, c, c, {base-2-LANE} +multi_fn = vcmla_rot90-rot-noext, a, b, c +a = 1., -1., 1., -1. +b = -1., 1., -1., 1. +c = 1., 1., -1., -1. +n = 0 +validate 0., 0., 0., 0. +target = fcma + +aarch64 = fcmla +generate float32x2_t, float32x2_t:float32x2_t:float32x4_t:float32x2_t +generate float32x4_t:float32x4_t:float32x2_t:float32x4_t, float32x4_t + +/// Floating-point complex multiply accumulate +name = vcmla_rot180 +rot-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_rot-LANE +multi_fn = simd_shuffle-out_len-!, c:out_t, c, c, {base-2-LANE} +multi_fn = vcmla_rot180-rot-noext, a, b, c +a = 1., -1., 1., -1. +b = -1., 1., -1., 1. +c = 1., 1., -1., -1. +n = 0 +validate 2., 0., 2., 0. +target = fcma + +aarch64 = fcmla +generate float32x2_t, float32x2_t:float32x2_t:float32x4_t:float32x2_t +generate float32x4_t:float32x4_t:float32x2_t:float32x4_t, float32x4_t + +/// Floating-point complex multiply accumulate +name = vcmla_rot270 +rot-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_rot-LANE +multi_fn = simd_shuffle-out_len-!, c:out_t, c, c, {base-2-LANE} +multi_fn = vcmla_rot270-rot-noext, a, b, c +a = 1., -1., 1., -1. +b = -1., 1., -1., 1. +c = 1., 1., -1., -1. +n = 0 +validate 2., -2., 2., -2. +target = fcma + +aarch64 = fcmla +generate float32x2_t, float32x2_t:float32x2_t:float32x4_t:float32x2_t +generate float32x4_t:float32x4_t:float32x2_t:float32x4_t, float32x4_t + +/// Dot product arithmetic +name = vdot +out-suffix +a = 1, 2, 1, 2 +b = 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 +c = 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 +validate 31, 176, 31, 176 +target = dotprod + +aarch64 = sdot +link-aarch64 = sdot._EXT_._EXT3_ +generate int32x2_t:int8x8_t:int8x8_t:int32x2_t, int32x4_t:int8x16_t:int8x16_t:int32x4_t + +aarch64 = udot +link-aarch64 = udot._EXT_._EXT3_ +generate uint32x2_t:uint8x8_t:uint8x8_t:uint32x2_t, uint32x4_t:uint8x16_t:uint8x16_t:uint32x4_t + +/// Dot product arithmetic +name = vdot +out-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_dot-LANE +multi_fn = simd_shuffle-in_len-!, c:in_t, c, c, {base-4-LANE} +multi_fn = vdot-out-noext, a, b, c +a = 1, 2, 1, 2 +b = 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 +c = 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 +n = 0 +validate 31, 72, 31, 72 +target = dotprod + +aarch64 = sdot +generate int32x2_t:int8x8_t:int8x8_t:int32x2_t, int32x2_t:int8x8_t:int8x16_t:int32x2_t +generate int32x4_t:int8x16_t:int8x8_t:int32x4_t, int32x4_t:int8x16_t:int8x16_t:int32x4_t + +aarch64 = udot +generate uint32x2_t:uint8x8_t:uint8x8_t:uint32x2_t, uint32x2_t:uint8x8_t:uint8x16_t:uint32x2_t +generate uint32x4_t:uint8x16_t:uint8x8_t:uint32x4_t, uint32x4_t:uint8x16_t:uint8x16_t:uint32x4_t + +/// Maximum (vector) +name = vmax +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 +validate 16, 15, 14, 13, 12, 11, 10, 9, 9, 10, 11, 12, 13, 14, 15, 16 + +arm = vmax +aarch64 = smax +link-arm = vmaxs._EXT_ +link-aarch64 = smax._EXT_ +generate int*_t + +arm = vmax +aarch64 = umax +link-arm = vmaxu._EXT_ +link-aarch64 = umax._EXT_ +generate uint*_t + +/// Maximum (vector) +name = vmax +a = 1.0, -2.0, 3.0, -4.0 +b = 0.0, 3.0, 2.0, 8.0 +validate 1.0, 3.0, 3.0, 8.0 + +aarch64 = fmax +link-aarch64 = fmax._EXT_ +generate float64x*_t + +arm = vmax +aarch64 = fmax +link-arm = vmaxs._EXT_ +link-aarch64 = fmax._EXT_ +generate float*_t + +/// Floating-point Maximum Number (vector) +name = vmaxnm +a = 1.0, 2.0, 3.0, -4.0 +b = 8.0, 16.0, -1.0, 6.0 +validate 8.0, 16.0, 3.0, 6.0 + +aarch64 = fmaxnm +link-aarch64 = fmaxnm._EXT_ +generate float64x*_t + +target = fp-armv8 +arm = vmaxnm +aarch64 = fmaxnm +link-arm = vmaxnm._EXT_ +link-aarch64 = fmaxnm._EXT_ +generate float*_t + +/// Floating-point maximum number across vector +name = vmaxnmv +a = 1., 2., 0., 1. +validate 2. + +aarch64 = fmaxnmp +link-aarch64 = fmaxnmv._EXT2_._EXT_ +generate float32x2_t:f32, float64x2_t:f64 +aarch64 = fmaxnmv +generate float32x4_t:f32 + +/// Floating-point Maximum Number Pairwise (vector). +name = vpmaxnm +a = 1.0, 2.0 +b = 6.0, -3.0 +validate 2.0, 6.0 +aarch64 = fmaxnmp +link-aarch64 = fmaxnmp._EXT_ +generate float32x2_t:float32x2_t:float32x2_t, float64x2_t:float64x2_t:float64x2_t + +/// Floating-point Maximum Number Pairwise (vector). +name = vpmaxnm +a = 1.0, 2.0, 3.0, -4.0 +b = 8.0, 16.0, -1.0, 6.0 +validate 2.0, 3.0, 16.0, 6.0 +aarch64 = fmaxnmp +link-aarch64 = fmaxnmp._EXT_ +generate float32x4_t:float32x4_t:float32x4_t + +/// Floating-point maximum number pairwise +name = vpmaxnm +out-suffix +a = 1., 2. +validate 2. + +aarch64 = fmaxnmp +link-aarch64 = fmaxnmv._EXT2_._EXT_ +generate float32x2_t:f32 +name = vpmaxnmq +generate float64x2_t:f64 + +/// Floating-point maximum pairwise +name = vpmax +out-suffix +a = 1., 2. +validate 2. + +aarch64 = fmaxp +link-aarch64 = fmaxv._EXT2_._EXT_ +generate float32x2_t:f32 +name = vpmaxq +generate float64x2_t:f64 + +/// Minimum (vector) +name = vmin +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 +validate 1, 2, 3, 4, 5, 6, 7, 8, 8, 7, 6, 5, 4, 3, 2, 1 + +arm = vmin +aarch64 = smin +link-arm = vmins._EXT_ +link-aarch64 = smin._EXT_ +generate int*_t + +arm = vmin +aarch64 = umin +link-arm = vminu._EXT_ +link-aarch64 = umin._EXT_ +generate uint*_t + +/// Minimum (vector) +name = vmin +a = 1.0, -2.0, 3.0, -4.0 +b = 0.0, 3.0, 2.0, 8.0 +validate 0.0, -2.0, 2.0, -4.0 + +aarch64 = fmin +link-aarch64 = fmin._EXT_ +generate float64x*_t + +arm = vmin +aarch64 = fmin +link-arm = vmins._EXT_ +link-aarch64 = fmin._EXT_ +generate float*_t + +/// Floating-point Minimum Number (vector) +name = vminnm +a = 1.0, 2.0, 3.0, -4.0 +b = 8.0, 16.0, -1.0, 6.0 +validate 1.0, 2.0, -1.0, -4.0 + +aarch64 = fminnm +link-aarch64 = fminnm._EXT_ +generate float64x*_t + +target = fp-armv8 +arm = vminnm +aarch64 = fminnm +link-arm = vminnm._EXT_ +link-aarch64 = fminnm._EXT_ +generate float*_t + +/// Floating-point minimum number across vector +name = vminnmv +a = 1., 0., 2., 3. +validate 0. + +aarch64 = fminnmp +link-aarch64 = fminnmv._EXT2_._EXT_ +generate float32x2_t:f32, float64x2_t:f64 +aarch64 = fminnmv +generate float32x4_t:f32 + +/// Vector move +name = vmovl_high +no-q +multi_fn = simd_shuffle-out_len-!, a:half, a, a, {asc-halflen-halflen} +multi_fn = vmovl-noqself-noext, a +a = 1, 2, 3, 4, 3, 4, 5, 6, 3, 4, 5, 6, 7, 8, 9, 10 +validate 3, 4, 5, 6, 7, 8, 9, 10 + +aarch64 = sxtl2 +generate int8x16_t:int16x8_t, int16x8_t:int32x4_t, int32x4_t:int64x2_t + +aarch64 = uxtl2 +generate uint8x16_t:uint16x8_t, uint16x8_t:uint32x4_t, uint32x4_t:uint64x2_t + +/// Floating-point add pairwise +name = vpadd +a = 1., 2., 3., 4. +b = 3., 4., 5., 6. +validate 3., 7., 7., 11. + +aarch64 = faddp +link-aarch64 = faddp._EXT_ +generate float32x4_t, float64x2_t + +arm = vpadd +link-arm = vpadd._EXT_ +generate float32x2_t + +/// Floating-point add pairwise +name = vpadd +out-suffix +multi_fn = simd_extract, a1:out_t, a, 0 +multi_fn = simd_extract, a2:out_t, a, 1 +multi_fn = a1 + a2 +a = 1., 2. +validate 3. + +aarch64 = nop +generate float32x2_t:f32, float64x2_t:f64 + +/// Floating-point Minimum Number Pairwise (vector). +name = vpminnm +a = 1.0, 2.0 +b = 6.0, -3.0 +validate 1.0, -3.0 + +aarch64 = fminnmp +link-aarch64 = fminnmp._EXT_ +generate float32x2_t:float32x2_t:float32x2_t, float64x2_t:float64x2_t:float64x2_t + +/// Floating-point Minimum Number Pairwise (vector). +name = vpminnm +a = 1.0, 2.0, 3.0, -4.0 +b = 8.0, 16.0, -1.0, 6.0 +validate 1.0, -4.0, 8.0, -1.0 +aarch64 = fminnmp +link-aarch64 = fminnmp._EXT_ +generate float32x4_t:float32x4_t:float32x4_t + +/// Floating-point minimum number pairwise +name = vpminnm +out-suffix +a = 1., 2. +validate 1. + +aarch64 = fminnmp +link-aarch64 = fminnmv._EXT2_._EXT_ +generate float32x2_t:f32 +name = vpminnmq +generate float64x2_t:f64 + +/// Floating-point minimum pairwise +name = vpmin +out-suffix +a = 1., 2. +validate 1. + +aarch64 = fminp +link-aarch64 = fminv._EXT2_._EXT_ +generate float32x2_t:f32 +name = vpminq +generate float64x2_t:f64 + +/// Signed saturating doubling multiply long +name = vqdmull +a = 0, 1, 2, 3, 4, 5, 6, 7 +b = 1, 2, 3, 4, 5, 6, 7, 8 +validate 0, 4, 12, 24, 40, 60, 84, 108 + +aarch64 = sqdmull +link-aarch64 = sqdmull._EXT2_ +arm = vqdmull +link-arm = vqdmull._EXT2_ +generate int16x4_t:int16x4_t:int32x4_t, int32x2_t:int32x2_t:int64x2_t + +/// Signed saturating doubling multiply long +name = vqdmull +multi_fn = vdup_n-in_ntt-noext, a:in_ntt, a +multi_fn = vdup_n-in_ntt-noext, b:in_ntt, b +multi_fn = simd_extract, {vqdmull-in_ntt-noext, a, b}, 0 +a = 2 +b = 3 +validate 12 + +aarch64 = sqdmull +generate i16:i16:i32 + +/// Signed saturating doubling multiply long +name = vqdmull +a = 2 +b = 3 +validate 12 + +aarch64 = sqdmull +link-aarch64 = sqdmulls.scalar +generate i32:i32:i64 + +/// Vector saturating doubling long multiply with scalar +name = vqdmull_n +no-q +multi_fn = vqdmull-in_ntt-noext, a, {vdup_n-in_ntt-noext, b} +a = 2, 4, 6, 8 +b = 2 +validate 8, 16, 24, 32 + +aarch64 = sqdmull +arm = vqdmull +generate int16x4_t:i16:int32x4_t, int32x2_t:i32:int64x2_t + +/// Signed saturating doubling multiply long +name = vqdmull_high +no-q +multi_fn = simd_shuffle-out_len-!, a:half, a, a, {asc-halflen-halflen} +multi_fn = simd_shuffle-out_len-!, b:half, b, b, {asc-halflen-halflen} +multi_fn = vqdmull-noqself-noext, a, b +a = 0, 1, 4, 5, 4, 5, 6, 7 +b = 1, 2, 5, 6, 5, 6, 7, 8 +validate 40, 60, 84, 112 + +aarch64 = sqdmull2 +generate int16x8_t:int16x8_t:int32x4_t, int32x4_t:int32x4_t:int64x2_t + +/// Signed saturating doubling multiply long +name = vqdmull_high_n +no-q +multi_fn = simd_shuffle-out_len-!, a:in_ntt, a, a, {asc-out_len-out_len} +multi_fn = vdup_n-in_ntt-noext, b:in_ntt, b +multi_fn = vqdmull-in_ntt-noext, a, b +a = 0, 2, 8, 10, 8, 10, 12, 14 +b = 2 +validate 32, 40, 48, 56 + +aarch64 = sqdmull2 +generate int16x8_t:i16:int32x4_t, int32x4_t:i32:int64x2_t + +/// Vector saturating doubling long multiply by scalar +name = vqdmull_lane +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_shuffle-out_len-!, b:in_t0, b, b, {dup-out_len-N as u32} +multi_fn = vqdmull-noqself-noext, a, b +a = 1, 2, 3, 4 +b = 0, 2, 2, 0, 2, 0, 0, 0 +n = HFLEN +validate 4, 8, 12, 16 + +aarch64 = sqdmull +generate int16x4_t:int16x8_t:int32x4_t, int32x2_t:int32x4_t:int64x2_t + +arm = vqdmull +generate int16x4_t:int16x4_t:int32x4_t, int32x2_t:int32x2_t:int64x2_t + +/// Signed saturating doubling multiply long +name = vqdmullh_lane +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_extract, b:in_t0, b, N as u32 +multi_fn = vqdmullh-noqself-noext, a, b +a = 2 +b = 0, 2, 2, 0, 2, 0, 0, 0 +n = HFLEN +validate 8 + +aarch64 = sqdmull +generate i16:int16x4_t:i32, i16:int16x8_t:i32 + +/// Signed saturating doubling multiply long +name = vqdmulls_lane +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_extract, b:in_t0, b, N as u32 +multi_fn = vqdmulls-noqself-noext, a, b +a = 2 +b = 0, 2, 2, 0, 2, 0, 0, 0 +n = HFLEN +validate 8 + +aarch64 = sqdmull +generate i32:int32x2_t:i64, i32:int32x4_t:i64 + +/// Signed saturating doubling multiply long +name = vqdmull_high_lane +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_shuffle-out_len-!, a:in_t, a, a, {asc-out_len-out_len} +multi_fn = simd_shuffle-out_len-!, b:in_t, b, b, {dup-out_len-N as u32} +multi_fn = vqdmull-self-noext, a, b +a = 0, 1, 4, 5, 4, 5, 6, 7 +b = 0, 2, 2, 0, 2, 0, 0, 0 +n = HFLEN +validate 16, 20, 24, 28 + +aarch64 = sqdmull2 +generate int16x8_t:int16x4_t:int32x4_t, int32x4_t:int32x2_t:int64x2_t + +/// Signed saturating doubling multiply long +name = vqdmull_high_lane +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_shuffle-out_len-!, a:half, a, a, {asc-out_len-out_len} +multi_fn = simd_shuffle-out_len-!, b:half, b, b, {dup-out_len-N as u32} +multi_fn = vqdmull-noqself-noext, a, b +a = 0, 1, 4, 5, 4, 5, 6, 7 +b = 0, 2, 2, 0, 2, 0, 0, 0 +n = HFLEN +validate 16, 20, 24, 28 + +aarch64 = sqdmull2 +generate int16x8_t:int16x8_t:int32x4_t, int32x4_t:int32x4_t:int64x2_t + +/// Signed saturating doubling multiply-add long +name = vqdmlal +multi_fn = vqadd-out-noext, a, {vqdmull-self-noext, b, c} +a = 1, 1, 1, 1 +b = 1, 2, 3, 4 +c = 2, 2, 2, 2 +validate 5, 9, 13, 17 + +aarch64 = sqdmlal +arm = vqdmlal +generate int32x4_t:int16x4_t:int16x4_t:int32x4_t, int64x2_t:int32x2_t:int32x2_t:int64x2_t + +/// Vector widening saturating doubling multiply accumulate with scalar +name = vqdmlal +n-suffix +multi_fn = vqadd-out-noext, a, {vqdmull_n-self-noext, b, c} +a = 1, 1, 1, 1 +b = 1, 2, 3, 4 +c = 2 +validate 5, 9, 13, 17 + +aarch64 = sqdmlal +arm = vqdmlal +generate int32x4_t:int16x4_t:i16:int32x4_t, int64x2_t:int32x2_t:i32:int64x2_t + +/// Signed saturating doubling multiply-add long +name = vqdmlal_high +no-q +multi_fn = vqadd-out-noext, a, {vqdmull_high-noqself-noext, b, c} +a = 1, 2, 3, 4 +b = 0, 1, 4, 5, 4, 5, 6, 7 +c = 1, 2, 5, 6, 5, 6, 7, 8 +validate 41, 62, 87, 116 + +aarch64 = sqdmlal2 +generate int32x4_t:int16x8_t:int16x8_t:int32x4_t, int64x2_t:int32x4_t:int32x4_t:int64x2_t + +/// Signed saturating doubling multiply-add long +name = vqdmlal_high_n +no-q +multi_fn = vqadd-out-noext, a, {vqdmull_high_n-noqself-noext, b, c} +a = 1, 2, 3, 4 +b = 0, 2, 8, 10, 8, 10, 12, 14 +c = 2 +validate 33, 42, 51, 60 + +aarch64 = sqdmlal2 +generate int32x4_t:int16x8_t:i16:int32x4_t, int64x2_t:int32x4_t:i32:int64x2_t + +/// Vector widening saturating doubling multiply accumulate with scalar +name = vqdmlal_lane +in2-suffix +constn = N +multi_fn = static_assert_imm-in2_exp_len-N +multi_fn = vqadd-out-noext, a, {vqdmull_lane-in2-::, b, c} +a = 1, 2, 3, 4 +b = 1, 2, 3, 4 +c = 0, 2, 2, 0, 2, 0, 0, 0 +n = HFLEN +validate 5, 10, 15, 20 + +aarch64 = sqdmlal +generate int32x4_t:int16x4_t:int16x8_t:int32x4_t, int64x2_t:int32x2_t:int32x4_t:int64x2_t + +arm = vqdmlal +generate int32x4_t:int16x4_t:int16x4_t:int32x4_t, int64x2_t:int32x2_t:int32x2_t:int64x2_t + +/// Signed saturating doubling multiply-add long +name = vqdmlal_high_lane +in2-suffix +constn = N +multi_fn = static_assert_imm-in2_exp_len-N +multi_fn = vqadd-out-noext, a, {vqdmull_high_lane-in2-::, b, c} +a = 1, 2, 3, 4 +b = 0, 1, 4, 5, 4, 5, 6, 7 +c = 0, 2, 0, 0, 0, 0, 0, 0 +n = 1 +validate 17, 22, 27, 32 + +aarch64 = sqdmlal2 +generate int32x4_t:int16x8_t:int16x4_t:int32x4_t, int32x4_t:int16x8_t:int16x8_t:int32x4_t, int64x2_t: int32x4_t:int32x2_t:int64x2_t, int64x2_t:int32x4_t:int32x4_t:int64x2_t + +/// Signed saturating doubling multiply-add long +name = vqdmlal +multi_fn = vqdmull-in_ntt-noext, x:out_long_ntt, {vdup_n-in_ntt-noext, b}, {vdup_n-in_ntt-noext, c} +multi_fn = vqadd-out-noext, a, {simd_extract, x, 0} +a = 1 +b = 1 +c = 2 +validate 5 + +aarch64 = sqdmull +generate i32:i16:i16:i32, i64:i32:i32:i64 + +/// Signed saturating doubling multiply-add long +name = vqdmlalh_lane +in2-suffix +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vqdmlal-self-noext, a, b, {simd_extract, c, LANE as u32} +a = 1 +b = 1 +c = 2, 1, 1, 1, 1, 1, 1, 1 +n = 0 +validate 5 + +aarch64 = sqdmlal +generate i32:i16:int16x4_t:i32, i32:i16:int16x8_t:i32 +name = vqdmlals_lane +aarch64 = sqdmull +generate i64:i32:int32x2_t:i64, i64:i32:int32x4_t:i64 + +/// Signed saturating doubling multiply-subtract long +name = vqdmlsl +multi_fn = vqsub-out-noext, a, {vqdmull-self-noext, b, c} +a = 3, 7, 11, 15 +b = 1, 2, 3, 4 +c = 2, 2, 2, 2 +validate -1, -1, -1, -1 + +aarch64 = sqdmlsl +arm = vqdmlsl +generate int32x4_t:int16x4_t:int16x4_t:int32x4_t, int64x2_t:int32x2_t:int32x2_t:int64x2_t + +/// Vector widening saturating doubling multiply subtract with scalar +name = vqdmlsl +n-suffix +multi_fn = vqsub-out-noext, a, {vqdmull_n-self-noext, b, c} +a = 3, 7, 11, 15 +b = 1, 2, 3, 4 +c = 2 +validate -1, -1, -1, -1 + +aarch64 = sqdmlsl +arm = vqdmlsl +generate int32x4_t:int16x4_t:i16:int32x4_t, int64x2_t:int32x2_t:i32:int64x2_t + +/// Signed saturating doubling multiply-subtract long +name = vqdmlsl_high +no-q +multi_fn = vqsub-out-noext, a, {vqdmull_high-noqself-noext, b, c} +a = 39, 58, 81, 108 +b = 0, 1, 4, 5, 4, 5, 6, 7 +c = 1, 2, 5, 6, 5, 6, 7, 8 +validate -1, -2, -3, -4 + +aarch64 = sqdmlsl2 +generate int32x4_t:int16x8_t:int16x8_t:int32x4_t, int64x2_t:int32x4_t:int32x4_t:int64x2_t + +/// Signed saturating doubling multiply-subtract long +name = vqdmlsl_high_n +no-q +multi_fn = vqsub-out-noext, a, {vqdmull_high_n-noqself-noext, b, c} +a = 31, 38, 45, 52 +b = 0, 2, 8, 10, 8, 10, 12, 14 +c = 2 +validate -1, -2, -3, -4 + +aarch64 = sqdmlsl2 +generate int32x4_t:int16x8_t:i16:int32x4_t, int64x2_t:int32x4_t:i32:int64x2_t + +/// Vector widening saturating doubling multiply subtract with scalar +name = vqdmlsl_lane +in2-suffix +constn = N +multi_fn = static_assert_imm-in2_exp_len-N +multi_fn = vqsub-out-noext, a, {vqdmull_lane-in2-::, b, c} +a = 3, 6, 9, 12 +b = 1, 2, 3, 4 +c = 0, 2, 2, 0, 2, 0, 0, 0 +n = HFLEN +validate -1, -2, -3, -4 + +aarch64 = sqdmlsl +generate int32x4_t:int16x4_t:int16x8_t:int32x4_t, int64x2_t:int32x2_t:int32x4_t:int64x2_t + +arm = vqdmlsl +generate int32x4_t:int16x4_t:int16x4_t:int32x4_t, int64x2_t:int32x2_t:int32x2_t:int64x2_t + +/// Signed saturating doubling multiply-subtract long +name = vqdmlsl_high_lane +in2-suffix +constn = N +multi_fn = static_assert_imm-in2_exp_len-N +multi_fn = vqsub-out-noext, a, {vqdmull_high_lane-in2-::, b, c} +a = 15, 18, 21, 24 +b = 0, 1, 4, 5, 4, 5, 6, 7 +c = 0, 2, 0, 0, 0, 0, 0, 0 +n = 1 +validate -1, -2, -3, -4 + +aarch64 = sqdmlsl2 +generate int32x4_t:int16x8_t:int16x4_t:int32x4_t, int32x4_t:int16x8_t:int16x8_t:int32x4_t, int64x2_t: int32x4_t:int32x2_t:int64x2_t, int64x2_t:int32x4_t:int32x4_t:int64x2_t + +/// Signed saturating doubling multiply-subtract long +name = vqdmlsl +multi_fn = vqdmull-in_ntt-noext, x:out_long_ntt, {vdup_n-in_ntt-noext, b}, {vdup_n-in_ntt-noext, c} +multi_fn = vqsub-out-noext, a, {simd_extract, x, 0} +a = 10 +b = 1 +c = 2 +validate 6 + +aarch64 = sqdmull +generate i32:i16:i16:i32, i64:i32:i32:i64 + +/// Signed saturating doubling multiply-subtract long +name = vqdmlslh_lane +in2-suffix +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vqdmlsl-self-noext, a, b, {simd_extract, c, LANE as u32} +a = 10 +b = 1 +c = 2, 1, 1, 1, 1, 1, 1, 1 +n = 0 +validate 6 + +aarch64 = sqdmlsl +generate i32:i16:int16x4_t:i32, i32:i16:int16x8_t:i32 +name = vqdmlsls_lane +aarch64 = sqdmull +generate i64:i32:int32x2_t:i64, i64:i32:int32x4_t:i64 + +/// Signed saturating doubling multiply returning high half +name = vqdmulh +a = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +b = 2, 2, 2, 2, 2, 2, 2, 2 +validate 1, 1, 1, 1, 1, 1, 1, 1 + +aarch64 = sqdmulh +link-aarch64 = sqdmulh._EXT_ +arm = vqdmulh +link-arm = vqdmulh._EXT_ +generate int16x4_t, int16x8_t, int32x2_t, int32x4_t + +/// Signed saturating doubling multiply returning high half +name = vqdmulh +multi_fn = vdup_n-in_ntt-noext, a:in_ntt, a +multi_fn = vdup_n-in_ntt-noext, b:in_ntt, b +multi_fn = simd_extract, {vqdmulh-in_ntt-noext, a, b}, 0 +a = 1 +b = 2 +validate 0 + +aarch64 = sqdmulh +generate i16, i32 + +/// Vector saturating doubling multiply high with scalar +name = vqdmulh_n +out-suffix +multi_fn = vdup_n-in_ntt-noext, b:in_ntt, b +multi_fn = vqdmulh-out-noext, a, b +a = MAX, MAX, MAX, MAX +b = 2 +validate 1, 1, 1, 1 + +aarch64 = sqdmulh +arm = vqdmulh +generate int16x4_t:i16:int16x4_t, int32x2_t:i32:int32x2_t + +/// Vector saturating doubling multiply high with scalar +name = vqdmulhq_n +no-q +multi_fn = vdupq_n-in_ntt-noext, b:out_t, b +multi_fn = vqdmulh-out-noext, a, b +a = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +b = 2 +validate 1, 1, 1, 1, 1, 1, 1, 1 + +aarch64 = sqdmulh +arm = vqdmulh +generate int16x8_t:i16:int16x8_t, int32x4_t:i32:int32x4_t + +/// Signed saturating doubling multiply returning high half +name = vqdmulhh_lane +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_extract, b:in_t0, b, N as u32 +multi_fn = vqdmulhh-out_ntt-noext, a, b +a = 2 +b = 0, 0, MAX, 0, 0, 0, 0, 0 +n = 2 +validate 1 + +aarch64 = sqdmulh +generate i16:int16x4_t:i16, i16:int16x8_t:i16 + +/// Signed saturating doubling multiply returning high half +name = vqdmulhs_lane +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_extract, b:in_t0, b, N as u32 +multi_fn = vqdmulhs-out_ntt-noext, a, b +a = 2 +b = 0, MAX, 0, 0 +n = 1 +validate 1 + +aarch64 = sqdmulh +generate i32:int32x2_t:i32, i32:int32x4_t:i32 + +/// Vector saturating doubling multiply high by scalar +name = vqdmulh +lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vqdmulh-out-noext, a, {vdup-nout-noext, {simd_extract, b, LANE as u32}} +a = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +b = 2, 1, 1, 1, 1, 1, 1, 1 +n = 0 +validate 1, 1, 1, 1, 1, 1, 1, 1 + +aarch64 = sqdmulh +generate int16x4_t, int16x8_t:int16x4_t:int16x8_t +generate int32x2_t, int32x4_t:int32x2_t:int32x4_t +arm = vqdmulh +generate int16x8_t, int16x4_t:int16x8_t:int16x4_t +generate int32x4_t, int32x2_t:int32x4_t:int32x2_t + +/// Signed saturating extract narrow +name = vqmovn +no-q +a = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +validate MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX + +aarch64 = sqxtn +link-aarch64 = sqxtn._EXT2_ +arm = vqmovn +link-arm = vqmovns._EXT2_ +generate int16x8_t:int8x8_t, int32x4_t:int16x4_t, int64x2_t:int32x2_t + +/// Unsigned saturating extract narrow +name = vqmovn +no-q +a = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +validate MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX + +aarch64 = uqxtn +link-aarch64 = uqxtn._EXT2_ +arm = vqmovn +link-arm = vqmovnu._EXT2_ +generate uint16x8_t:uint8x8_t, uint32x4_t:uint16x4_t, uint64x2_t:uint32x2_t + +/// Saturating extract narrow +name = vqmovn +multi_fn = simd_extract, {vqmovn-in_ntt-noext, {vdupq_n-in_ntt-noext, a}}, 0 +a = 1 +validate 1 + +aarch64 = sqxtn +generate i16:i8, i32:i16 +aarch64 = uqxtn +generate u16:u8, u32:u16 + +/// Saturating extract narrow +name = vqmovn +a = 1 +validate 1 + +aarch64 = sqxtn +link-aarch64 = scalar.sqxtn._EXT2_._EXT_ +generate i64:i32 + +aarch64 = uqxtn +link-aarch64 = scalar.uqxtn._EXT2_._EXT_ +generate u64:u32 + +/// Signed saturating extract narrow +name = vqmovn_high +no-q +multi_fn = simd_shuffle-out_len-!, a, {vqmovn-noqself-noext, b}, {asc-0-out_len} +a = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +b = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +validate MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX + +aarch64 = sqxtn2 +generate int8x8_t:int16x8_t:int8x16_t, int16x4_t:int32x4_t:int16x8_t, int32x2_t:int64x2_t:int32x4_t +aarch64 = uqxtn2 +generate uint8x8_t:uint16x8_t:uint8x16_t, uint16x4_t:uint32x4_t:uint16x8_t, uint32x2_t:uint64x2_t:uint32x4_t + +/// Signed saturating extract unsigned narrow +name = vqmovun +no-q +a = -1, -1, -1, -1, -1, -1, -1, -1 +validate 0, 0, 0, 0, 0, 0, 0, 0 + +aarch64 = sqxtun +link-aarch64 = sqxtun._EXT2_ +arm = vqmovun +link-arm = vqmovnsu._EXT2_ +generate int16x8_t:uint8x8_t, int32x4_t:uint16x4_t, int64x2_t:uint32x2_t + +/// Signed saturating extract unsigned narrow +name = vqmovun +multi_fn = simd_extract, {vqmovun-in_ntt-noext, {vdupq_n-in_ntt-noext, a}}, 0 +a = 1 +validate 1 + +aarch64 = sqxtun +generate i16:u8, i32:u16, i64:u32 + +/// Signed saturating extract unsigned narrow +name = vqmovun_high +no-q +multi_fn = simd_shuffle-out_len-!, a, {vqmovun-noqself-noext, b}, {asc-0-out_len} +a = 0, 0, 0, 0, 0, 0, 0, 0 +b = -1, -1, -1, -1, -1, -1, -1, -1 +validate 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + +aarch64 = sqxtun2 +generate uint8x8_t:int16x8_t:uint8x16_t, uint16x4_t:int32x4_t:uint16x8_t, uint32x2_t:int64x2_t:uint32x4_t + +/// Signed saturating rounding doubling multiply returning high half +name = vqrdmulh +a = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +b = 2, 2, 2, 2, 2, 2, 2, 2 +validate 2, 2, 2, 2, 2, 2, 2, 2 + +aarch64 = sqrdmulh +link-aarch64 = sqrdmulh._EXT_ +arm = vqrdmulh +link-arm = vqrdmulh._EXT_ +generate int16x4_t, int16x8_t, int32x2_t, int32x4_t + +/// Signed saturating rounding doubling multiply returning high half +name = vqrdmulh +multi_fn = simd_extract, {vqrdmulh-in_ntt-noext, {vdup_n-in_ntt-noext, a}, {vdup_n-in_ntt-noext, b}}, 0 +a = 1 +b = 2 +validate 0 + +aarch64 = sqrdmulh +generate i16, i32 + +/// Vector saturating rounding doubling multiply high with scalar +name = vqrdmulh +out-n-suffix +multi_fn = vqrdmulh-out-noext, a, {vdup-nout-noext, b} +a = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +b = 2 +validate 2, 2, 2, 2, 2, 2, 2, 2 + +aarch64 = sqrdmulh +arm = vqrdmulh +generate int16x4_t:i16:int16x4_t, int16x8_t:i16:int16x8_t, int32x2_t:i32:int32x2_t, int32x4_t:i32:int32x4_t + +/// Vector rounding saturating doubling multiply high by scalar +name = vqrdmulh +lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = simd_shuffle-out_len-!, b:out_t, b, b, {dup-out_len-LANE as u32} +multi_fn = vqrdmulh-out-noext, a, b +a = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +b = 0, 2, 0, 0, 0, 0, 0, 0, +n = 1 +validate 2, 2, 2, 2, 2, 2, 2, 2 + +aarch64 = sqrdmulh +arm = vqrdmulh +generate int16x4_t, int16x4_t:int16x8_t:int16x4_t, int16x8_t:int16x4_t:int16x8_t, int16x8_t +generate int32x2_t, int32x2_t:int32x4_t:int32x2_t, int32x4_t:int32x2_t:int32x4_t, int32x4_t + +/// Signed saturating rounding doubling multiply returning high half +name = vqrdmulh +lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = vqrdmulh-out-noext, a, {simd_extract, b, LANE as u32} +a = 1 +b = 0, 2, 0, 0, 0, 0, 0, 0, +n = 1 +validate 0 + +aarch64 = sqrdmulh +generate i16:int16x4_t:i16, i16:int16x8_t:i16, i32:int32x2_t:i32, i32:int32x4_t:i32 + +/// Signed saturating rounding doubling multiply accumulate returning high half +name = vqrdmlah +a = 1, 1, 1, 1, 1, 1, 1, 1 +b = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +c = 2, 2, 2, 2, 2, 2, 2, 2 +validate 3, 3, 3, 3, 3, 3, 3, 3 + +aarch64 = sqrdmlah +link-aarch64 = sqrdmlah._EXT_ +target = rdm +generate int16x4_t, int16x8_t, int32x2_t, int32x4_t + +/// Signed saturating rounding doubling multiply accumulate returning high half +name = vqrdmlah +multi_fn = vdup_n-in_ntt-noext, a:in_ntt, a +multi_fn = vdup_n-in_ntt-noext, b:in_ntt, b +multi_fn = vdup_n-in_ntt-noext, c:in_ntt, c +multi_fn = simd_extract, {vqrdmlah-in_ntt-noext, a, b, c}, 0 +a = 1 +b = 1 +c = 2 +validate 1 + +aarch64 = sqrdmlah +target = rdm +generate i16, i32 + +/// Signed saturating rounding doubling multiply accumulate returning high half +name = vqrdmlah +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = simd_shuffle-out_len-!, c:out_t, c, c, {dup-out_len-LANE as u32} +multi_fn = vqrdmlah-out-noext, a, b, c +a = 1, 1, 1, 1, 1, 1, 1, 1 +b = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +c = 0, 2, 0, 0, 0, 0, 0, 0 +n = 1 +validate 3, 3, 3, 3, 3, 3, 3, 3 + +aarch64 = sqrdmlah +target = rdm +generate int16x4_t, int16x4_t:int16x4_t:int16x8_t:int16x4_t, int16x8_t:int16x8_t:int16x4_t:int16x8_t, int16x8_t +generate int32x2_t, int32x2_t:int32x2_t:int32x4_t:int32x2_t, int32x4_t:int32x4_t:int32x2_t:int32x4_t, int32x4_t + +/// Signed saturating rounding doubling multiply accumulate returning high half +name = vqrdmlah +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vqrdmlah-self-noext, a, b, {simd_extract, c, LANE as u32} +a = 1 +b = 1 +c = 0, 2, 0, 0, 0, 0, 0, 0 +n = 1 +validate 1 + +aarch64 = sqrdmlah +target = rdm +generate i16:i16:int16x4_t:i16, i16:i16:int16x8_t:i16, i32:i32:int32x2_t:i32, i32:i32:int32x4_t:i32 + +/// Signed saturating rounding doubling multiply subtract returning high half +name = vqrdmlsh +link-aarch64 = sqrdmlsh._EXT_ +a = 1, 1, 1, 1, 1, 1, 1, 1 +b = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +c = 2, 2, 2, 2, 2, 2, 2, 2 +validate -1, -1, -1, -1, -1, -1, -1, -1 + +aarch64 = sqrdmlsh +target = rdm +generate int16x4_t, int16x8_t, int32x2_t, int32x4_t + +/// Signed saturating rounding doubling multiply subtract returning high half +name = vqrdmlsh +multi_fn = vdup_n-in_ntt-noext, a:in_ntt, a +multi_fn = vdup_n-in_ntt-noext, b:in_ntt, b +multi_fn = vdup_n-in_ntt-noext, c:in_ntt, c +multi_fn = simd_extract, {vqrdmlsh-in_ntt-noext, a, b, c}, 0 +a = 1 +b = 1 +c = 2 +validate 1 + +aarch64 = sqrdmlsh +target = rdm +generate i16, i32 + +/// Signed saturating rounding doubling multiply subtract returning high half +name = vqrdmlsh +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = simd_shuffle-out_len-!, c:out_t, c, c, {dup-out_len-LANE as u32} +multi_fn = vqrdmlsh-out-noext, a, b, c +a = 1, 1, 1, 1, 1, 1, 1, 1 +b = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +c = 0, 2, 0, 0, 0, 0, 0, 0 +n = 1 +validate -1, -1, -1, -1, -1, -1, -1, -1 + +aarch64 = sqrdmlsh +target = rdm +generate int16x4_t, int16x4_t:int16x4_t:int16x8_t:int16x4_t, int16x8_t:int16x8_t:int16x4_t:int16x8_t, int16x8_t +generate int32x2_t, int32x2_t:int32x2_t:int32x4_t:int32x2_t, int32x4_t:int32x4_t:int32x2_t:int32x4_t, int32x4_t + +/// Signed saturating rounding doubling multiply subtract returning high half +name = vqrdmlsh +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vqrdmlsh-self-noext, a, b, {simd_extract, c, LANE as u32} +a = 1 +b = 1 +c = 0, 2, 0, 0, 0, 0, 0, 0 +n = 1 +validate 1 + +aarch64 = sqrdmlsh +target = rdm +generate i16:i16:int16x4_t:i16, i16:i16:int16x8_t:i16, i32:i32:int32x2_t:i32, i32:i32:int32x4_t:i32 + +/// Signed saturating rounding shift left +name = vqrshl +a = 2, MIN, MAX, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +validate 8, MIN, MAX, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 + +aarch64 = sqrshl +link-aarch64 = sqrshl._EXT_ +generate i32, i64 + +arm = vqrshl +link-arm = vqrshifts._EXT_ +generate int*_t, int64x*_t + +/// Signed saturating rounding shift left +name = vqrshl +multi_fn = vdup_n-in_ntt-noext, a:in_ntt, a +multi_fn = vdup_n-in_ntt-noext, b:in_ntt, b +multi_fn = simd_extract, {vqrshl-in_ntt-noext, a, b}, 0 +a = 1 +b = 2 +validate 4 + +aarch64 = sqrshl +generate i8, i16 + +/// Unsigned signed saturating rounding shift left +name = vqrshl +out-suffix +a = 2, MIN, MAX, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +validate 8, 0, MAX, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 + +aarch64 = uqrshl +link-aarch64 = uqrshl._EXT_ +generate u32:i32:u32, u64:i64:u64 + +arm = vqrshl +link-arm = vqrshiftu._EXT_ +generate uint8x8_t:int8x8_t:uint8x8_t, uint8x16_t:int8x16_t:uint8x16_t, uint16x4_t:int16x4_t:uint16x4_t, uint16x8_t:int16x8_t:uint16x8_t +generate uint32x2_t:int32x2_t:uint32x2_t, uint32x4_t:int32x4_t:uint32x4_t, uint64x1_t:int64x1_t:uint64x1_t, uint64x2_t:int64x2_t:uint64x2_t + +/// Unsigned signed saturating rounding shift left +name = vqrshl +out-suffix +multi_fn = vdup_n-out_ntt-noext, a:out_ntt, a +multi_fn = vdup_n-in_ntt-noext, b:in_ntt, b +multi_fn = simd_extract, {vqrshl-out_ntt-noext, a, b}, 0 +a = 1 +b = 2 +validate 4 + +aarch64 = uqrshl +generate u8:i8:u8, u16:i16:u16 + +/// Signed saturating rounded shift right narrow +name = vqrshrn +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +a = MIN, 4, 8, 12, 16, 20, 24, 28 +n = 2 +validate MIN, 1, 2, 3, 4, 5, 6, 7 + +aarch64 = sqrshrn +link-aarch64 = sqrshrn._EXT2_ +const-aarch64 = N + +arm = vqrshrn +link-arm = vqrshiftns._EXT2_ +const-arm = -N as ttn +arm-aarch64-separate +generate int16x8_t:int8x8_t, int32x4_t:int16x4_t, int64x2_t:int32x2_t + +/// Signed saturating rounded shift right narrow +name = vqrshrn +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = vdupq_n-in_ntt-noext, a:in_long_ntt, a +multi_fn = simd_extract, {vqrshrn_n-in_ntt-::, a}, 0 +a = 4 +n = 2 +validate 1 + +aarch64 = sqrshrn +generate i16:i8, i32:i16, i64:i32 + +/// Signed saturating rounded shift right narrow +name = vqrshrn_high +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_shuffle-out_len-!, a, {vqrshrn_n-noqself-::, b}, {asc-0-out_len} +a = 0, 1, 2, 3, 2, 3, 6, 7 +b = 8, 12, 24, 28, 48, 52, 56, 60 +n = 2 +validate 0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 12, 13, 14, 15 + +aarch64 = sqrshrn2 +generate int8x8_t:int16x8_t:int8x16_t, int16x4_t:int32x4_t:int16x8_t, int32x2_t:int64x2_t:int32x4_t + +/// Unsigned signed saturating rounded shift right narrow +name = vqrshrn +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +a = MIN, 4, 8, 12, 16, 20, 24, 28 +n = 2 +validate 0, 1, 2, 3, 4, 5, 6, 7 + +aarch64 = uqrshrn +link-aarch64 = uqrshrn._EXT2_ +const-aarch64 = N + +arm = vqrshrn +link-arm = vqrshiftnu._EXT2_ +const-arm = -N as ttn +arm-aarch64-separate +generate uint16x8_t:uint8x8_t, uint32x4_t:uint16x4_t, uint64x2_t:uint32x2_t + +/// Unsigned saturating rounded shift right narrow +name = vqrshrn +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = vdupq_n-in_ntt-noext, a:in_long_ntt, a +multi_fn = simd_extract, {vqrshrn_n-in_ntt-::, a}, 0 +a = 4 +n = 2 +validate 1 + +aarch64 = uqrshrn +generate u16:u8, u32:u16, u64:u32 + +/// Unsigned saturating rounded shift right narrow +name = vqrshrn_high +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_shuffle-out_len-!, a, {vqrshrn_n-noqself-::, b}, {asc-0-out_len} +a = 0, 1, 2, 3, 2, 3, 6, 7 +b = 8, 12, 24, 28, 48, 52, 56, 60 +n = 2 +validate 0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 12, 13, 14, 15 + +aarch64 = uqrshrn2 +generate uint8x8_t:uint16x8_t:uint8x16_t, uint16x4_t:uint32x4_t:uint16x8_t, uint32x2_t:uint64x2_t:uint32x4_t + +/// Signed saturating rounded shift right unsigned narrow +name = vqrshrun +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +a = 0, 4, 8, 12, 16, 20, 24, 28 +n = 2 +validate 0, 1, 2, 3, 4, 5, 6, 7 + +aarch64 = sqrshrun +link-aarch64 = sqrshrun._EXT2_ +const-aarch64 = N + +arm = vqrshrun +link-arm = vqrshiftnsu._EXT2_ +const-arm = -N as ttn +arm-aarch64-separate +generate int16x8_t:uint8x8_t, int32x4_t:uint16x4_t, int64x2_t:uint32x2_t + +/// Signed saturating rounded shift right unsigned narrow +name = vqrshrun +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = vdupq_n-in_ntt-noext, a:in_long_ntt, a +multi_fn = simd_extract, {vqrshrun_n-in_ntt-::, a}, 0 +a = 4 +n = 2 +validate 1 + +aarch64 = sqrshrun +generate i16:u8, i32:u16, i64:u32 + +/// Signed saturating rounded shift right unsigned narrow +name = vqrshrun_high +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_shuffle-out_len-!, a, {vqrshrun_n-noqself-::, b}, {asc-0-out_len} +a = 0, 1, 2, 3, 2, 3, 6, 7 +b = 8, 12, 24, 28, 48, 52, 56, 60 +n = 2 +validate 0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 12, 13, 14, 15 + +aarch64 = sqrshrun2 +generate uint8x8_t:int16x8_t:uint8x16_t, uint16x4_t:int32x4_t:uint16x8_t, uint32x2_t:int64x2_t:uint32x4_t + +/// Signed saturating shift left +name = vqshl +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +validate 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 + +aarch64 = sqshl +link-aarch64 = sqshl._EXT_ +generate i64 + +arm = vqshl +link-arm = vqshifts._EXT_ +generate int*_t, int64x*_t + +/// Signed saturating shift left +name = vqshl +multi_fn = vqshl-in_ntt-noext, c:in_ntt, {vdup_n-in_ntt-noext, a}, {vdup_n-in_ntt-noext, b} +multi_fn = simd_extract, c, 0 +a = 1 +b = 2 +validate 4 + +aarch64 = sqshl +generate i8, i16, i32 + +/// Unsigned saturating shift left +name = vqshl +out-suffix +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +validate 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 + +aarch64 = uqshl +link-aarch64 = uqshl._EXT_ +generate u64:i64:u64 + +arm = vqshl +link-arm = vqshiftu._EXT_ +generate uint8x8_t:int8x8_t:uint8x8_t, uint8x16_t:int8x16_t:uint8x16_t, uint16x4_t:int16x4_t:uint16x4_t, uint16x8_t:int16x8_t:uint16x8_t +generate uint32x2_t:int32x2_t:uint32x2_t, uint32x4_t:int32x4_t:uint32x4_t, uint64x1_t:int64x1_t:uint64x1_t, uint64x2_t:int64x2_t:uint64x2_t + +/// Unsigned saturating shift left +name = vqshl +out-suffix +multi_fn = vqshl-out_ntt-noext, c:out_ntt, {vdup_n-out_ntt-noext, a}, {vdup_n-in_ntt-noext, b} +multi_fn = simd_extract, c, 0 +a = 1 +b = 2 +validate 4 + +aarch64 = uqshl +generate u8:i8:u8, u16:i16:u16, u32:i32:u32 + +/// Signed saturating shift left +name = vqshl +n-suffix +constn = N +multi_fn = static_assert_imm-out_bits_exp_len-N +multi_fn = vqshl-self-noext, a, {vdup-nself-noext, N as _} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +n = 2 +validate 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 + +aarch64 = sqshl +arm = vqshl +generate int*_t, int64x*_t + +/// Signed saturating shift left +name = vqshl +n-suffix +constn = N +multi_fn = static_assert_imm-out_bits_exp_len-N +multi_fn = simd_extract, {vqshl_n-in_ntt-::, {vdup_n-in_ntt-noext, a}}, 0 +a = 1 +n = 2 +validate 4 + +aarch64 = sqshl +generate i8, i16, i32, i64 + +/// Unsigned saturating shift left +name = vqshl +n-suffix +constn = N +multi_fn = static_assert_imm-out_bits_exp_len-N +multi_fn = vqshl-self-noext, a, {vdup-nsigned-noext, N as _} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +n = 2 +validate 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 + +aarch64 = uqshl +arm = vqshl +generate uint*_t, uint64x*_t + +/// Unsigned saturating shift left +name = vqshl +n-suffix +constn = N +multi_fn = static_assert_imm-out_bits_exp_len-N +multi_fn = simd_extract, {vqshl_n-in_ntt-::, {vdup_n-in_ntt-noext, a}}, 0 +a = 1 +n = 2 +validate 4 + +aarch64 = uqshl +generate u8, u16, u32, u64 + +/// Signed saturating shift left unsigned +name = vqshlu +n-suffix +constn = N +multi_fn = static_assert_imm-out_bits_exp_len-N +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +n = 2 +validate 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 +arm-aarch64-separate + +aarch64 = sqshlu +link-aarch64 = sqshlu._EXT_ +const-aarch64 = {dup-in_len-N as ttn} +arm = vqshlu +link-arm = vqshiftsu._EXT_ +const-arm = N as ttn +generate int8x8_t:uint8x8_t, int16x4_t:uint16x4_t, int32x2_t:uint32x2_t, int64x1_t:uint64x1_t +generate int8x16_t:uint8x16_t, int16x8_t:uint16x8_t, int32x4_t:uint32x4_t, int64x2_t:uint64x2_t + +/// Signed saturating shift left unsigned +name = vqshlu +n-suffix +constn = N +multi_fn = static_assert_imm-out_bits_exp_len-N +multi_fn = simd_extract, {vqshlu_n-in_ntt-::, {vdup_n-in_ntt-noext, a}}, 0 +a = 1 +n = 2 +validate 4 + +aarch64 = sqshlu +generate i8:u8, i16:u16, i32:u32, i64:u64 + +/// Signed saturating shift right narrow +name = vqshrn +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +a = 0, 4, 8, 12, 16, 20, 24, 28 +n = 2 +validate 0, 1, 2, 3, 4, 5, 6, 7 +arm-aarch64-separate + +aarch64 = sqshrn +link-aarch64 = sqshrn._EXT2_ +const-aarch64 = N +generate i64:i32 + +arm = vqshrn +link-arm = vqshiftns._EXT2_ +const-arm = -N as ttn +generate int16x8_t:int8x8_t, int32x4_t:int16x4_t, int64x2_t:int32x2_t + +/// Signed saturating shift right narrow +name = vqshrn +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_extract, {vqshrn_n-in_ntt-::, {vdupq_n-in_ntt-noext, a}}, 0 +a = 4 +n = 2 +validate 1 + +aarch64 = sqshrn +generate i16:i8, i32:i16 + +/// Signed saturating shift right narrow +name = vqshrn_high +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_shuffle-out_len-!, a, {vqshrn_n-noqself-::, b}, {asc-0-out_len} +a = 0, 1, 8, 9, 8, 9, 10, 11 +b = 32, 36, 40, 44, 48, 52, 56, 60 +n = 2 +validate 0, 1, 8, 9, 8, 9, 10, 11, 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = sqshrn2 +generate int8x8_t:int16x8_t:int8x16_t, int16x4_t:int32x4_t:int16x8_t, int32x2_t:int64x2_t:int32x4_t + +/// Unsigned saturating shift right narrow +name = vqshrn +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +a = 0, 4, 8, 12, 16, 20, 24, 28 +n = 2 +validate 0, 1, 2, 3, 4, 5, 6, 7 +arm-aarch64-separate + +aarch64 = uqshrn +link-aarch64 = uqshrn._EXT2_ +const-aarch64 = N +generate u64:u32 + +arm = vqshrn +link-arm = vqshiftnu._EXT2_ +const-arm = -N as ttn +generate uint16x8_t:uint8x8_t, uint32x4_t:uint16x4_t, uint64x2_t:uint32x2_t + +/// Unsigned saturating shift right narrow +name = vqshrn +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_extract, {vqshrn_n-in_ntt-::, {vdupq_n-in_ntt-noext, a}}, 0 +a = 4 +n = 2 +validate 1 + +aarch64 = uqshrn +generate u16:u8, u32:u16 + +/// Unsigned saturating shift right narrow +name = vqshrn_high +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_shuffle-out_len-!, a, {vqshrn_n-noqself-::, b}, {asc-0-out_len} +a = 0, 1, 8, 9, 8, 9, 10, 11 +b = 32, 36, 40, 44, 48, 52, 56, 60 +n = 2 +validate 0, 1, 8, 9, 8, 9, 10, 11, 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = uqshrn2 +generate uint8x8_t:uint16x8_t:uint8x16_t, uint16x4_t:uint32x4_t:uint16x8_t, uint32x2_t:uint64x2_t:uint32x4_t + +/// Signed saturating shift right unsigned narrow +name = vqshrun +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +a = 0, 4, 8, 12, 16, 20, 24, 28 +n = 2 +validate 0, 1, 2, 3, 4, 5, 6, 7 +arm-aarch64-separate + +aarch64 = sqshrun +link-aarch64 = sqshrun._EXT2_ +const-aarch64 = N + +arm = vqshrun +link-arm = vqshiftnsu._EXT2_ +const-arm = -N as ttn +generate int16x8_t:uint8x8_t, int32x4_t:uint16x4_t, int64x2_t:uint32x2_t + +/// Signed saturating shift right unsigned narrow +name = vqshrun +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_extract, {vqshrun_n-in_ntt-::, {vdupq_n-in_ntt-noext, a}}, 0 +a = 4 +n = 2 +validate 1 + +aarch64 = sqshrun +generate i16:u8, i32:u16, i64:u32 + +/// Signed saturating shift right unsigned narrow +name = vqshrun_high +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_shuffle-out_len-!, a, {vqshrun_n-noqself-::, b}, {asc-0-out_len} +a = 0, 1, 8, 9, 8, 9, 10, 11 +b = 32, 36, 40, 44, 48, 52, 56, 60 +n = 2 +validate 0, 1, 8, 9, 8, 9, 10, 11, 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = sqshrun2 +generate uint8x8_t:int16x8_t:uint8x16_t, uint16x4_t:int32x4_t:uint16x8_t, uint32x2_t:int64x2_t:uint32x4_t + +/// Unsigned saturating accumulate of signed value +name = vsqadd +out-suffix +multi_fn = simd_extract, {vsqadd-out_ntt-noext, {vdup_n-out_ntt-noext, a}, {vdup_n-in_ntt-noext, b}}, 0 +a = 2 +b = 2 +validate 4 + +aarch64 = usqadd +generate u8:i8:u8, u16:i16:u16 + +/// Unsigned saturating accumulate of signed value +name = vsqadd +out-suffix +a = 2 +b = 2 +validate 4 + +aarch64 = usqadd +link-aarch64 = usqadd._EXT_ +generate u32:i32:u32, u64:i64:u64 + +/// Calculates the square root of each lane. +name = vsqrt +fn = simd_fsqrt +a = 4.0, 9.0, 16.0, 25.0 +validate 2.0, 3.0, 4.0, 5.0 + +aarch64 = fsqrt +generate float*_t, float64x*_t + +/// Reciprocal square-root estimate. +name = vrsqrte +a = 1.0, 2.0, 3.0, 4.0 +validate 0.998046875, 0.705078125, 0.576171875, 0.4990234375 + +aarch64 = frsqrte +link-aarch64 = frsqrte._EXT_ +generate float64x*_t, f32, f64 + +arm = vrsqrte +link-arm = vrsqrte._EXT_ +generate float*_t + +/// Unsigned reciprocal square root estimate +name = vrsqrte +a = 1, 2, 3, 4 +validate 4294967295, 4294967295, 4294967295, 4294967295 + +aarch64 = ursqrte +link-aarch64 = ursqrte._EXT_ +arm = vrsqrte +link-arm = vrsqrte._EXT_ +generate uint32x2_t, uint32x4_t + +/// Floating-point reciprocal square root step +name = vrsqrts +a = 1.0, 2.0, 3.0, 4.0 +b = 1.0, 2.0, 3.0, 4.0 +validate 1., -0.5, -3.0, -6.5 + +aarch64 = frsqrts +link-aarch64 = frsqrts._EXT_ +generate float64x*_t, f32, f64 + +arm = vrsqrts +link-arm = vrsqrts._EXT_ +generate float*_t + +/// Reciprocal estimate. +name = vrecpe +a = 4.0, 3.0, 2.0, 1.0 +validate 0.24951171875, 0.3330078125, 0.4990234375, 0.998046875 + +aarch64 = frecpe +link-aarch64 = frecpe._EXT_ +generate float64x*_t, f32, f64 + +arm = vrecpe +link-arm = vrecpe._EXT_ +generate float*_t + +/// Unsigned reciprocal estimate +name = vrecpe +a = 4, 3, 2, 1 +validate 4294967295, 4294967295, 4294967295, 4294967295 + +aarch64 = urecpe +link-aarch64 = urecpe._EXT_ +arm = vrecpe +link-arm = vrecpe._EXT_ +generate uint32x2_t, uint32x4_t + +/// Floating-point reciprocal step +name = vrecps +a = 4.0, 3.0, 2.0, 1.0 +b = 4.0, 3.0, 2.0, 1.0 +validate -14., -7., -2., 1. + +aarch64 = frecps +link-aarch64 = frecps._EXT_ +generate float64x*_t, f32, f64 + +arm = vrecps +link-arm = vrecps._EXT_ +generate float*_t + +/// Floating-point reciprocal exponent +name = vrecpx +a = 4.0 +validate 0.5 + +aarch64 = frecpx +link-aarch64 = frecpx._EXT_ +generate f32, f64 + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = nop +generate poly64x1_t:int64x1_t, poly64x1_t:uint64x1_t, int64x1_t:poly64x1_t, uint64x1_t:poly64x1_t +generate poly64x2_t:int64x2_t, poly64x2_t:uint64x2_t, int64x2_t:poly64x2_t, uint64x2_t:poly64x2_t + +arm = nop +generate uint8x8_t:int8x8_t, poly8x8_t:int8x8_t, poly16x4_t:int16x4_t, uint16x4_t:int16x4_t, uint32x2_t:int32x2_t, uint64x1_t:int64x1_t +generate uint8x16_t:int8x16_t, poly8x16_t:int8x16_t, poly16x8_t:int16x8_t, uint16x8_t:int16x8_t, uint32x4_t:int32x4_t, uint64x2_t:int64x2_t +generate poly8x8_t:uint8x8_t, int8x8_t:uint8x8_t, poly16x4_t:uint16x4_t, int16x4_t:uint16x4_t, int32x2_t:uint32x2_t, int64x1_t:uint64x1_t +generate poly8x16_t:uint8x16_t, int8x16_t:uint8x16_t, poly16x8_t:uint16x8_t, int16x8_t:uint16x8_t, int32x4_t:uint32x4_t, int64x2_t:uint64x2_t +generate int8x8_t:poly8x8_t, uint8x8_t:poly8x8_t, int16x4_t:poly16x4_t, uint16x4_t:poly16x4_t +generate int8x16_t:poly8x16_t, uint8x16_t:poly8x16_t, int16x8_t:poly16x8_t, uint16x8_t:poly16x8_t + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 0, 1, 2, 3, 4, 5, 6, 7 +validate 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0 + +aarch64 = nop +arm = nop +generate int16x4_t:int8x8_t, uint16x4_t:int8x8_t, poly16x4_t:int8x8_t, int32x2_t:int16x4_t, uint32x2_t:int16x4_t, int64x1_t:int32x2_t, uint64x1_t:int32x2_t +generate int16x8_t:int8x16_t, uint16x8_t:int8x16_t, poly16x8_t:int8x16_t, int32x4_t:int16x8_t, uint32x4_t:int16x8_t, int64x2_t:int32x4_t, uint64x2_t:int32x4_t +generate poly16x4_t:uint8x8_t, int16x4_t:uint8x8_t, uint16x4_t:uint8x8_t, int32x2_t:uint16x4_t, uint32x2_t:uint16x4_t, int64x1_t:uint32x2_t, uint64x1_t:uint32x2_t +generate poly16x8_t:uint8x16_t, int16x8_t:uint8x16_t, uint16x8_t:uint8x16_t, int32x4_t:uint16x8_t, uint32x4_t:uint16x8_t, int64x2_t:uint32x4_t, uint64x2_t:uint32x4_t +generate poly16x4_t:poly8x8_t, int16x4_t:poly8x8_t, uint16x4_t:poly8x8_t, int32x2_t:poly16x4_t, uint32x2_t:poly16x4_t +generate poly16x8_t:poly8x16_t, int16x8_t:poly8x16_t, uint16x8_t:poly8x16_t, int32x4_t:poly16x8_t, uint32x4_t:poly16x8_t +target = aes +generate poly64x1_t:int32x2_t, poly64x1_t:uint32x2_t +generate poly64x2_t:int32x4_t, poly64x2_t:uint32x4_t +generate p128:int64x2_t, p128:uint64x2_t, p128:poly64x2_t + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0 +validate 0, 1, 2, 3, 4, 5, 6, 7 + +aarch64 = nop +arm = nop +generate poly8x8_t:int16x4_t, int8x8_t:int16x4_t, uint8x8_t:int16x4_t, poly16x4_t:int32x2_t, int16x4_t:int32x2_t, uint16x4_t:int32x2_t, int32x2_t:int64x1_t, uint32x2_t:int64x1_t +generate poly8x16_t:int16x8_t, int8x16_t:int16x8_t, uint8x16_t:int16x8_t, poly16x8_t:int32x4_t, int16x8_t:int32x4_t, uint16x8_t:int32x4_t, int32x4_t:int64x2_t, uint32x4_t:int64x2_t +generate poly8x8_t:uint16x4_t, int8x8_t:uint16x4_t, uint8x8_t:uint16x4_t, poly16x4_t:uint32x2_t, int16x4_t:uint32x2_t, uint16x4_t:uint32x2_t, int32x2_t:uint64x1_t, uint32x2_t:uint64x1_t +generate poly8x16_t:uint16x8_t, int8x16_t:uint16x8_t, uint8x16_t:uint16x8_t, poly16x8_t:uint32x4_t, int16x8_t:uint32x4_t, uint16x8_t:uint32x4_t, int32x4_t:uint64x2_t, uint32x4_t:uint64x2_t +generate poly8x8_t:poly16x4_t, int8x8_t:poly16x4_t, uint8x8_t:poly16x4_t +generate poly8x16_t:poly16x8_t, int8x16_t:poly16x8_t, uint8x16_t:poly16x8_t +target = aes +generate int32x2_t:poly64x1_t, uint32x2_t:poly64x1_t +generate int32x4_t:poly64x2_t, uint32x4_t:poly64x2_t +generate int64x2_t:p128, uint64x2_t:p128, poly64x2_t:p128 + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 0, 1, 2, 3 +validate 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0 + +aarch64 = nop +arm = nop +generate int32x2_t:int8x8_t, uint32x2_t:int8x8_t, int64x1_t:int16x4_t, uint64x1_t:int16x4_t +generate int32x4_t:int8x16_t, uint32x4_t:int8x16_t, int64x2_t:int16x8_t, uint64x2_t:int16x8_t +generate int32x2_t:uint8x8_t, uint32x2_t:uint8x8_t, int64x1_t:uint16x4_t, uint64x1_t:uint16x4_t +generate int32x4_t:uint8x16_t, uint32x4_t:uint8x16_t, int64x2_t:uint16x8_t, uint64x2_t:uint16x8_t +generate int32x2_t:poly8x8_t, uint32x2_t:poly8x8_t, int64x1_t:poly16x4_t, uint64x1_t:poly16x4_t +generate int32x4_t:poly8x16_t, uint32x4_t:poly8x16_t, int64x2_t:poly16x8_t, uint64x2_t:poly16x8_t +target = aes +generate poly64x1_t:int16x4_t, poly64x1_t:uint16x4_t, poly64x1_t:poly16x4_t +generate poly64x2_t:int16x8_t, poly64x2_t:uint16x8_t, poly64x2_t:poly16x8_t +generate p128:int32x4_t, p128:uint32x4_t + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0 +validate 0, 1, 2, 3 + +aarch64 = nop +arm = nop +generate poly8x8_t:int32x2_t, int8x8_t:int32x2_t, uint8x8_t:int32x2_t, poly16x4_t:int64x1_t, int16x4_t:int64x1_t, uint16x4_t:int64x1_t +generate poly8x16_t:int32x4_t, int8x16_t:int32x4_t, uint8x16_t:int32x4_t, poly16x8_t:int64x2_t, int16x8_t:int64x2_t, uint16x8_t:int64x2_t +generate poly8x8_t:uint32x2_t, int8x8_t:uint32x2_t, uint8x8_t:uint32x2_t, poly16x4_t:uint64x1_t, int16x4_t:uint64x1_t, uint16x4_t:uint64x1_t +generate poly8x16_t:uint32x4_t, int8x16_t:uint32x4_t, uint8x16_t:uint32x4_t, poly16x8_t:uint64x2_t, int16x8_t:uint64x2_t, uint16x8_t:uint64x2_t +target = aes +generate poly16x4_t:poly64x1_t, int16x4_t:poly64x1_t, uint16x4_t:poly64x1_t +generate poly16x8_t:poly64x2_t, int16x8_t:poly64x2_t, uint16x8_t:poly64x2_t +generate int32x4_t:p128, uint32x4_t:p128 + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 0, 1 +validate 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 + +aarch64 = nop +arm = nop +generate int64x1_t:int8x8_t, uint64x1_t:int8x8_t, int64x1_t:uint8x8_t, uint64x1_t:uint8x8_t, int64x1_t:poly8x8_t, uint64x1_t:poly8x8_t +generate int64x2_t:int8x16_t, uint64x2_t:int8x16_t, int64x2_t:uint8x16_t, uint64x2_t:uint8x16_t, int64x2_t:poly8x16_t, uint64x2_t:poly8x16_t +target = aes +generate poly64x1_t:int8x8_t, poly64x1_t:uint8x8_t, poly64x1_t:poly8x8_t +generate poly64x2_t:int8x16_t, poly64x2_t:uint8x16_t, poly64x2_t:poly8x16_t +generate p128:int16x8_t, p128:uint16x8_t, p128:poly16x8_t + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 +validate 0, 1 + +aarch64 = nop +arm = nop +generate poly8x8_t:int64x1_t, int8x8_t:int64x1_t, uint8x8_t:int64x1_t, poly8x8_t:uint64x1_t, int8x8_t:uint64x1_t, uint8x8_t:uint64x1_t +generate poly8x16_t:int64x2_t, int8x16_t:int64x2_t, uint8x16_t:int64x2_t, poly8x16_t:uint64x2_t, int8x16_t:uint64x2_t, uint8x16_t:uint64x2_t +target = aes +generate poly8x8_t:poly64x1_t, int8x8_t:poly64x1_t, uint8x8_t:poly64x1_t +generate poly8x16_t:poly64x2_t, int8x16_t:poly64x2_t, uint8x16_t:poly64x2_t +generate int16x8_t:p128, uint16x8_t:p128, poly16x8_t:p128 + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +validate 1 +target = aes + +aarch64 = nop +arm = nop +generate int8x16_t:p128, uint8x16_t:p128, poly8x16_t:p128 + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 1 +validate 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +target = aes + +aarch64 = nop +arm = nop +generate p128:int8x16_t, p128:uint8x16_t, p128:poly8x16_t + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 0., 0., 0., 0., 0., 0., 0., 0. +validate 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + +aarch64 = nop +generate float64x1_t:int8x8_t, float64x1_t:int16x4_t, float64x1_t:int32x2_t, float64x1_t:int64x1_t +generate float64x2_t:int8x16_t, float64x2_t:int16x8_t, float64x2_t:int32x4_t, float64x2_t:int64x2_t +generate float64x1_t:uint8x8_t, float64x1_t:uint16x4_t, float64x1_t:uint32x2_t, float64x1_t:uint64x1_t +generate float64x2_t:uint8x16_t, float64x2_t:uint16x8_t, float64x2_t:uint32x4_t, float64x2_t:uint64x2_t +generate float64x1_t:poly8x8_t, float64x1_t:poly16x4_t, float32x2_t:poly64x1_t, float64x1_t:poly64x1_t +generate float64x2_t:poly8x16_t, float64x2_t:poly16x8_t, float32x4_t:poly64x2_t, float64x2_t:poly64x2_t +generate float64x2_t:p128 + +arm = nop +generate float32x2_t:int8x8_t, float32x2_t:int16x4_t, float32x2_t:int32x2_t, float32x2_t:int64x1_t +generate float32x4_t:int8x16_t, float32x4_t:int16x8_t, float32x4_t:int32x4_t, float32x4_t:int64x2_t +generate float32x2_t:uint8x8_t, float32x2_t:uint16x4_t, float32x2_t:uint32x2_t, float32x2_t:uint64x1_t +generate float32x4_t:uint8x16_t, float32x4_t:uint16x8_t, float32x4_t:uint32x4_t, float32x4_t:uint64x2_t +generate float32x2_t:poly8x8_t, float32x2_t:poly16x4_t +generate float32x4_t:poly8x16_t, float32x4_t:poly16x8_t +generate float32x4_t:p128 + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +validate 0., 0., 0., 0., 0., 0., 0., 0. + +aarch64 = nop +generate int8x8_t:float64x1_t, int16x4_t:float64x1_t, int32x2_t:float64x1_t, int64x1_t:float64x1_t +generate int8x16_t:float64x2_t, int16x8_t:float64x2_t, int32x4_t:float64x2_t, int64x2_t:float64x2_t +generate poly8x8_t:float64x1_t, uint16x4_t:float64x1_t, uint32x2_t:float64x1_t, uint64x1_t:float64x1_t +generate poly8x16_t:float64x2_t, uint16x8_t:float64x2_t, uint32x4_t:float64x2_t, uint64x2_t:float64x2_t +generate uint8x8_t:float64x1_t, poly16x4_t:float64x1_t, poly64x1_t:float64x1_t, poly64x1_t:float32x2_t +generate uint8x16_t:float64x2_t, poly16x8_t:float64x2_t, poly64x2_t:float64x2_t, poly64x2_t:float32x4_t +generate p128:float64x2_t + +arm = nop +generate int8x8_t:float32x2_t, int16x4_t:float32x2_t, int32x2_t:float32x2_t, int64x1_t:float32x2_t +generate int8x16_t:float32x4_t, int16x8_t:float32x4_t, int32x4_t:float32x4_t, int64x2_t:float32x4_t +generate uint8x8_t:float32x2_t, uint16x4_t:float32x2_t, uint32x2_t:float32x2_t, uint64x1_t:float32x2_t +generate uint8x16_t:float32x4_t, uint16x8_t:float32x4_t, uint32x4_t:float32x4_t, uint64x2_t:float32x4_t +generate poly8x8_t:float32x2_t, poly16x4_t:float32x2_t +generate poly8x16_t:float32x4_t, poly16x8_t:float32x4_t +generate p128:float32x4_t + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 0., 0., 0., 0., 0., 0., 0., 0. +validate 0., 0., 0., 0., 0., 0., 0., 0. + +aarch64 = nop +generate float32x2_t:float64x1_t, float64x1_t:float32x2_t +generate float32x4_t:float64x2_t, float64x2_t:float32x4_t + +/// Signed rounding shift left +name = vrshl +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +validate 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 + +aarch64 = srshl +link-aarch64 = srshl._EXT_ +generate i64 + +arm = vrshl +link-arm = vrshifts._EXT_ +generate int*_t, int64x*_t + +/// Unsigned rounding shift left +name = vrshl +out-suffix +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +validate 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 + +aarch64 = urshl +link-aarch64 = urshl._EXT_ +generate u64:i64:u64 + +arm = vrshl +link-arm = vrshiftu._EXT_ +generate uint8x8_t:int8x8_t:uint8x8_t, uint8x16_t:int8x16_t:uint8x16_t, uint16x4_t:int16x4_t:uint16x4_t, uint16x8_t:int16x8_t:uint16x8_t +generate uint32x2_t:int32x2_t:uint32x2_t, uint32x4_t:int32x4_t:uint32x4_t, uint64x1_t:int64x1_t:uint64x1_t, uint64x2_t:int64x2_t:uint64x2_t + +/// Signed rounding shift right +name = vrshr +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = vrshl-self-noext, a, {vdup-nself-noext, (-N) as _} +a = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +aarch64 = srshr +arm = vrshr +generate int*_t, int64x*_t + +/// Signed rounding shift right +name = vrshr +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = vrshl-self-noext, a, -N as i64 +a = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +aarch64 = srshr +generate i64 + +/// Unsigned rounding shift right +name = vrshr +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = vrshl-self-noext, a, {vdup-nsigned-noext, (-N) as _} +a = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +aarch64 = urshr +arm = vrshr +generate uint*_t, uint64x*_t + +/// Unsigned rounding shift right +name = vrshr +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = vrshl-self-noext, a, -N as i64 +a = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +aarch64 = urshr +generate u64 + +/// Rounding shift right narrow +name = vrshrn +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +a = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +arm-aarch64-separate + +aarch64 = rshrn +link-aarch64 = rshrn._EXT2_ +const-aarch64 = N + +arm = vrshrn +link-arm = vrshiftn._EXT2_ +const-arm = -N as ttn +generate int16x8_t:int8x8_t, int32x4_t:int16x4_t, int64x2_t:int32x2_t + +/// Rounding shift right narrow +name = vrshrn +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = transmute, {vrshrn_n-noqsigned-::, transmute(a)} +a = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +aarch64 = rshrn +arm = vrshrn +generate uint16x8_t:uint8x8_t, uint32x4_t:uint16x4_t, uint64x2_t:uint32x2_t + +/// Rounding shift right narrow +name = vrshrn_high +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_shuffle-out_len-!, a, {vrshrn_n-noqself-::, b}, {asc-0-out_len} +a = 0, 1, 8, 9, 8, 9, 10, 11 +b = 32, 36, 40, 44, 48, 52, 56, 60 +n = 2 +validate 0, 1, 8, 9, 8, 9, 10, 11, 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = rshrn2 +generate int8x8_t:int16x8_t:int8x16_t, int16x4_t:int32x4_t:int16x8_t, int32x2_t:int64x2_t:int32x4_t +generate uint8x8_t:uint16x8_t:uint8x16_t, uint16x4_t:uint32x4_t:uint16x8_t, uint32x2_t:uint64x2_t:uint32x4_t + +/// Signed rounding shift right and accumulate +name = vrsra +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = simd_add, a, {vrshr-nself-::, b} +a = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +b = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + +aarch64 = srsra +arm = vrsra +generate int*_t, int64x*_t + +/// Unsigned rounding shift right and accumulate +name = vrsra +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = simd_add, a, {vrshr-nself-::, b} +a = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +b = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + +aarch64 = ursra +arm = vrsra +generate uint*_t, uint64x*_t + +/// Signed rounding shift right and accumulate. +name = vrsra +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = vrshr-nself-::, b:in_t, b +multi_fn = a.wrapping_add(b) +a = 1 +b = 4 +n = 2 +validate 2 + +aarch64 = srsra +generate i64 + +/// Ungisned rounding shift right and accumulate. +name = vrsra +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = vrshr-nself-::, b:in_t, b +multi_fn = a.wrapping_add(b) +a = 1 +b = 4 +n = 2 +validate 2 + +aarch64 = ursra +generate u64 + +/// Rounding subtract returning high narrow +name = vrsubhn +no-q +a = MAX, MIN, 0, 4, 5, 6, 7, 8 +b = 1, 2, 3, 4, 5, 6, 7, 8 +validate MIN, MIN, 0, 0, 0, 0, 0, 0 + +aarch64 = rsubhn +link-aarch64 = rsubhn._EXT2_ +arm = vrsubhn +link-arm = vrsubhn._EXT2_ +generate int16x8_t:int16x8_t:int8x8_t, int32x4_t:int32x4_t:int16x4_t, int64x2_t:int64x2_t:int32x2_t + +/// Rounding subtract returning high narrow +name = vrsubhn +no-q +multi_fn = transmute, {vrsubhn-noqsigned-noext, {transmute, a}, {transmute, b}} +a = MAX, MIN, 3, 4, 5, 6, 7, 8 +b = 1, 2, 3, 4, 5, 6, 7, 8 +validate 0, 0, 0, 0, 0, 0, 0, 0 + +aarch64 = rsubhn +arm = vrsubhn +generate uint16x8_t:uint16x8_t:uint8x8_t, uint32x4_t:uint32x4_t:uint16x4_t, uint64x2_t:uint64x2_t:uint32x2_t + +/// Rounding subtract returning high narrow +name = vrsubhn_high +no-q +multi_fn = vrsubhn-noqself-noext, x:in_t0, b, c +multi_fn = simd_shuffle-out_len-!, a, x, {asc-0-out_len} +a = 1, 2, 0, 0, 0, 0, 0, 0 +b = 1, 2, 3, 4, 5, 6, 7, 8 +c = 1, 2, 3, 4, 5, 6, 7, 8 +validate 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + +aarch64 = rsubhn2 +generate int8x8_t:int16x8_t:int16x8_t:int8x16_t, int16x4_t:int32x4_t:int32x4_t:int16x8_t, int32x2_t:int64x2_t:int64x2_t:int32x4_t +generate uint8x8_t:uint16x8_t:uint16x8_t:uint8x16_t, uint16x4_t:uint32x4_t:uint32x4_t:uint16x8_t, uint32x2_t:uint64x2_t:uint64x2_t:uint32x4_t + +/// Insert vector element from another vector element +name = vset_lane +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = simd_insert, b, LANE as u32, a +a = 1 +b = 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +n = 0 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +aarch64 = nop +arm = nop +generate i8:int8x8_t:int8x8_t, i16:int16x4_t:int16x4_t +generate i32:int32x2_t:int32x2_t, i64:int64x1_t:int64x1_t +generate u8:uint8x8_t:uint8x8_t, u16:uint16x4_t:uint16x4_t +generate u32:uint32x2_t:uint32x2_t, u64:uint64x1_t:uint64x1_t +generate p8:poly8x8_t:poly8x8_t, p16:poly16x4_t:poly16x4_t + +target = aes +generate p64:poly64x1_t:poly64x1_t + +/// Insert vector element from another vector element +name = vsetq_lane +no-q +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = simd_insert, b, LANE as u32, a +a = 1 +b = 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +n = 0 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +aarch64 = nop +arm = nop +generate i8:int8x16_t:int8x16_t, i16:int16x8_t:int16x8_t +generate i32:int32x4_t:int32x4_t, i64:int64x2_t:int64x2_t +generate u8:uint8x16_t:uint8x16_t, u16:uint16x8_t:uint16x8_t +generate u32:uint32x4_t:uint32x4_t, u64:uint64x2_t:uint64x2_t +generate p8:poly8x16_t:poly8x16_t, p16:poly16x8_t:poly16x8_t + +target = aes +generate p64:poly64x2_t:poly64x2_t + +/// Insert vector element from another vector element +name = vset_lane +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = simd_insert, b, LANE as u32, a +a = 1. +b = 0., 2., 3., 4. +n = 0 +validate 1., 2., 3., 4. + +aarch64 = nop +generate f64:float64x1_t:float64x1_t + +arm = nop +generate f32:float32x2_t:float32x2_t + +/// Insert vector element from another vector element +name = vsetq_lane +no-q +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = simd_insert, b, LANE as u32, a +a = 1. +b = 0., 2., 3., 4. +n = 0 +validate 1., 2., 3., 4. + +aarch64 = nop +generate f64:float64x2_t:float64x2_t + +arm = nop +generate f32:float32x4_t:float32x4_t + +/// Signed Shift left +name = vshl +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +validate 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 + +aarch64 = sshl +link-aarch64 = sshl._EXT_ +arm = vshl +link-arm = vshifts._EXT_ +generate int*_t, int64x*_t + +/// Signed Shift left +name = vshl +multi_fn = transmute, {vshl-in_ntt-noext, transmute(a), transmute(b)} +a = 1 +b = 2 +validate 4 + +aarch64 = sshl +generate i64 + +/// Unsigned Shift left +name = vshl +out-suffix +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +validate 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 + +aarch64 = ushl +link-aarch64 = ushl._EXT_ +arm = vshl +link-arm = vshiftu._EXT_ +generate uint8x8_t:int8x8_t:uint8x8_t, uint8x16_t:int8x16_t:uint8x16_t, uint16x4_t:int16x4_t:uint16x4_t, uint16x8_t:int16x8_t:uint16x8_t +generate uint32x2_t:int32x2_t:uint32x2_t, uint32x4_t:int32x4_t:uint32x4_t, uint64x1_t:int64x1_t:uint64x1_t, uint64x2_t:int64x2_t:uint64x2_t + +/// Unsigned Shift left +out-suffix +name = vshl +multi_fn = transmute, {vshl-out_ntt-noext, transmute(a), transmute(b)} +a = 1 +b = 2 +validate 4 + +aarch64 = ushl +generate u64:i64:u64 + +/// Shift left +name = vshl +n-suffix +constn = N +multi_fn = static_assert_imm-out_bits_exp_len-N +multi_fn = simd_shl, a, {vdup-nself-noext, N as _} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +n = 2 +validate 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 + +arm = vshl +aarch64 = shl +generate int*_t, uint*_t, int64x*_t, uint64x*_t + +/// Signed shift left long +name = vshll +n-suffix +constn = N +multi_fn = static_assert-N-0-bits +multi_fn = simd_shl, {simd_cast, a}, {vdup-nout-noext, N as _} +a = 1, 2, 3, 4, 5, 6, 7, 8 +n = 2 +validate 4, 8, 12, 16, 20, 24, 28, 32 + +arm = vshll.s +aarch64 = sshll +generate int8x8_t:int16x8_t, int16x4_t:int32x4_t, int32x2_t:int64x2_t +aarch64 = ushll +generate uint8x8_t:uint16x8_t, uint16x4_t:uint32x4_t, uint32x2_t:uint64x2_t + +/// Signed shift left long +name = vshll_high_n +no-q +constn = N +multi_fn = static_assert-N-0-bits +multi_fn = simd_shuffle-out_len-!, b:half, a, a, {asc-halflen-halflen} +multi_fn = vshll_n-noqself-::, b +a = 0, 0, 1, 2, 1, 2, 3, 4, 1, 2, 3, 4, 5, 6, 7, 8 +n = 2 +validate 4, 8, 12, 16, 20, 24, 28, 32 + +aarch64 = sshll2 +generate int8x16_t:int16x8_t, int16x8_t:int32x4_t, int32x4_t:int64x2_t +aarch64 = ushll2 +generate uint8x16_t:uint16x8_t, uint16x8_t:uint32x4_t, uint32x4_t:uint64x2_t + +/// Shift right +name = vshr +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = fix_right_shift_imm-N-bits +multi_fn = simd_shr, a, {vdup-nself-noext, n as _} +a = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +arm = vshr.s +aarch64 = sshr +generate int*_t, int64x*_t +aarch64 = ushr +generate uint*_t, uint64x*_t + +/// Shift right narrow +name = vshrn_n +no-q +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_cast, {simd_shr, a, {vdup-nself-noext, N as _}} +a = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +arm = vshrn. +aarch64 = shrn +generate int16x8_t:int8x8_t, int32x4_t:int16x4_t, int64x2_t:int32x2_t +generate uint16x8_t:uint8x8_t, uint32x4_t:uint16x4_t, uint64x2_t:uint32x2_t + +/// Shift right narrow +name = vshrn_high_n +no-q +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_shuffle-out_len-!, a, {vshrn_n-noqself-::, b}, {asc-0-out_len} +a = 1, 2, 5, 6, 5, 6, 7, 8 +b = 20, 24, 28, 32, 52, 56, 60, 64 +n = 2 +validate 1, 2, 5, 6, 5, 6, 7, 8, 5, 6, 7, 8, 13, 14, 15, 16 + +aarch64 = shrn2 +generate int8x8_t:int16x8_t:int8x16_t, int16x4_t:int32x4_t:int16x8_t, int32x2_t:int64x2_t:int32x4_t +generate uint8x8_t:uint16x8_t:uint8x16_t, uint16x4_t:uint32x4_t:uint16x8_t, uint32x2_t:uint64x2_t:uint32x4_t + +/// Signed shift right and accumulate +name = vsra +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = simd_add, a, {vshr-nself-::, b} +a = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +b = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + +aarch64 = ssra +arm = vsra +generate int*_t, int64x*_t + +/// Unsigned shift right and accumulate +name = vsra +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = simd_add, a, {vshr-nself-::, b} +a = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +b = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + +aarch64 = usra +arm = vsra +generate uint*_t, uint64x*_t + +/// SM3PARTW1 +name = vsm3partw1 +a = 1, 2, 3, 4 +b = 1, 2, 3, 4 +c = 1, 2, 3, 4 +validate 2147549312, 3221323968, 131329, 2684362752 +target = sm4 + +aarch64 = sm3partw1 +link-aarch64 = llvm.aarch64.crypto.sm3partw1 +generate uint32x4_t + +/// SM3PARTW2 +name = vsm3partw2 +a = 1, 2, 3, 4 +b = 1, 2, 3, 4 +c = 1, 2, 3, 4 +validate 128, 256, 384, 1077977696 +target = sm4 + +aarch64 = sm3partw2 +link-aarch64 = llvm.aarch64.crypto.sm3partw2 +generate uint32x4_t + +/// SM3SS1 +name = vsm3ss1 +a = 1, 2, 3, 4 +b = 1, 2, 3, 4 +c = 1, 2, 3, 4 +validate 0, 0, 0, 2098176 +target = sm4 + +aarch64 = sm3ss1 +link-aarch64 = llvm.aarch64.crypto.sm3ss1 +generate uint32x4_t + +/// SM4 key +name = vsm4ekey +a = 1, 2, 3, 4 +b = 1, 2, 3, 4 +validate 1784948604, 136020997, 2940231695, 3789947679 +target = sm4 + +aarch64 = sm4ekey +link-aarch64 = llvm.aarch64.crypto.sm4ekey +generate uint32x4_t + +/// SM4 encode +name = vsm4e +a = 1, 2, 3, 4 +b = 1, 2, 3, 4 +validate 1093874472, 3616769504, 3878330411, 2765298765 +target = sm4 + +aarch64 = sm4e +link-aarch64 = llvm.aarch64.crypto.sm4e +generate uint32x4_t + +/// Rotate and exclusive OR +name = vrax1 +a = 1, 2 +b = 3, 4 +validate 7, 10 +target = sha3 + +aarch64 = rax1 +link-aarch64 = llvm.aarch64.crypto.rax1 +generate uint64x2_t + +/// SHA512 hash update part 1 +name = vsha512h +a = 1, 2 +b = 3, 4 +c = 5, 6 +validate 11189044327219203, 7177611956453380 +target = sha3 + +aarch64 = sha512h +link-aarch64 = llvm.aarch64.crypto.sha512h +generate uint64x2_t + +/// SHA512 hash update part 2 +name = vsha512h2 +a = 1, 2 +b = 3, 4 +c = 5, 6 +validate 5770237651009406214, 349133864969 +target = sha3 + +aarch64 = sha512h2 +link-aarch64 = llvm.aarch64.crypto.sha512h2 +generate uint64x2_t + +/// SHA512 schedule update 0 +name = vsha512su0 +a = 1, 2 +b = 3, 4 +validate 144115188075855874, 9439544818968559619 +target = sha3 + +aarch64 = sha512su0 +link-aarch64 = llvm.aarch64.crypto.sha512su0 +generate uint64x2_t + +/// SHA512 schedule update 1 +name = vsha512su1 +a = 1, 2 +b = 3, 4 +c = 5, 6 +validate 105553116266526, 140737488355368 +target = sha3 + +aarch64 = sha512su1 +link-aarch64 = llvm.aarch64.crypto.sha512su1 +generate uint64x2_t + +/// Floating-point round to 32-bit integer, using current rounding mode +name = vrnd32x +a = 1.1, 1.9, -1.7, -2.3 +validate 1.0, 2.0, -2.0, -2.0 +target = frintts + +aarch64 = frint32x +link-aarch64 = frint32x._EXT_ +generate float32x2_t, float32x4_t + +/// Floating-point round to 32-bit integer toward zero +name = vrnd32z +a = 1.1, 1.9, -1.7, -2.3 +validate 1.0, 1.0, -1.0, -2.0 +target = frintts + +aarch64 = frint32z +link-aarch64 = frint32z._EXT_ +generate float32x2_t, float32x4_t + +/// Floating-point round to 64-bit integer, using current rounding mode +name = vrnd64x +a = 1.1, 1.9, -1.7, -2.3 +validate 1.0, 2.0, -2.0, -2.0 +target = frintts + +aarch64 = frint64x +link-aarch64 = frint64x._EXT_ +generate float32x2_t, float32x4_t + +/// Floating-point round to 64-bit integer toward zero +name = vrnd64z +a = 1.1, 1.9, -1.7, -2.3 +validate 1.0, 1.0, -1.0, -2.0 +target = frintts + +aarch64 = frint64z +link-aarch64 = frint64z._EXT_ +generate float32x2_t, float32x4_t + +/// Transpose elements +name = vtrn +multi_fn = simd_shuffle-in_len-!, a1:in_t, a, b, {transpose-1-in_len} +multi_fn = simd_shuffle-in_len-!, b1:in_t, a, b, {transpose-2-in_len} +multi_fn = transmute, (a1, b1) +a = 0, 2, 2, 6, 2, 10, 6, 14, 2, 18, 6, 22, 10, 26, 14, 30 +b = 1, 3, 3, 7, 3, 1, 7, 15, 3, 19, 7, 23, 1, 27, 15, 31 +validate 0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 10, 1, 14, 15, 2, 3, 6, 7, 10, 1, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 + +aarch64 = trn +arm = vtrn +generate int8x8_t:int8x8_t:int8x8x2_t, int16x4_t:int16x4_t:int16x4x2_t, int8x16_t:int8x16_t:int8x16x2_t, int16x8_t:int16x8_t:int16x8x2_t, int32x4_t:int32x4_t:int32x4x2_t +generate uint8x8_t:uint8x8_t:uint8x8x2_t, uint16x4_t:uint16x4_t:uint16x4x2_t, uint8x16_t:uint8x16_t:uint8x16x2_t, uint16x8_t:uint16x8_t:uint16x8x2_t, uint32x4_t:uint32x4_t:uint32x4x2_t +generate poly8x8_t:poly8x8_t:poly8x8x2_t, poly16x4_t:poly16x4_t:poly16x4x2_t, poly8x16_t:poly8x16_t:poly8x16x2_t, poly16x8_t:poly16x8_t:poly16x8x2_t +aarch64 = zip +generate int32x2_t:int32x2_t:int32x2x2_t, uint32x2_t:uint32x2_t:uint32x2x2_t + +/// Transpose elements +name = vtrn +multi_fn = simd_shuffle-in_len-!, a1:in_t, a, b, {transpose-1-in_len} +multi_fn = simd_shuffle-in_len-!, b1:in_t, a, b, {transpose-2-in_len} +multi_fn = transmute, (a1, b1) +a = 0., 2., 2., 6. +b = 1., 3., 3., 7. +validate 0., 1., 2., 3., 2., 3., 6., 7. + +aarch64 = zip +arm = vtrn +generate float32x2_t:float32x2_t:float32x2x2_t +aarch64 = trn +generate float32x4_t:float32x4_t:float32x4x2_t + +/// Transpose vectors +name = vtrn1 +multi_fn = simd_shuffle-in_len-!, a, b, {transpose-1-in_len} +a = 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 +b = 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 +validate 0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29 + +aarch64 = trn1 +generate int8x8_t, int8x16_t, int16x4_t, int16x8_t, int32x4_t, uint8x8_t, uint8x16_t, uint16x4_t, uint16x8_t, uint32x4_t, poly8x8_t, poly8x16_t, poly16x4_t, poly16x8_t + +aarch64 = zip1 +generate int32x2_t, int64x2_t, uint32x2_t, uint64x2_t, poly64x2_t + +/// Transpose vectors +name = vtrn1 +multi_fn = simd_shuffle-in_len-!, a, b, {transpose-1-in_len} +a = 0., 2., 4., 6., 8., 10., 12., 14. +b = 1., 3., 5., 7., 9., 11., 13., 15. +validate 0., 1., 4., 5., 8., 9., 12., 13. + +aarch64 = trn1 +generate float32x4_t + +aarch64 = zip1 +generate float32x2_t, float64x2_t + +/// Transpose vectors +name = vtrn2 +multi_fn = simd_shuffle-in_len-!, a, b, {transpose-2-in_len} +a = 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 +b = 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 +validate 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 + +aarch64 = trn2 +generate int8x8_t, int8x16_t, int16x4_t, int16x8_t, int32x4_t, uint8x8_t, uint8x16_t, uint16x4_t, uint16x8_t, uint32x4_t, poly8x8_t, poly8x16_t, poly16x4_t, poly16x8_t + +aarch64 = zip2 +generate int32x2_t, int64x2_t, uint32x2_t, uint64x2_t, poly64x2_t + +/// Transpose vectors +name = vtrn2 +multi_fn = simd_shuffle-in_len-!, a, b, {transpose-2-in_len} +a = 0., 2., 4., 6., 8., 10., 12., 14. +b = 1., 3., 5., 7., 9., 11., 13., 15. +validate 2., 3., 6., 7., 10., 11., 14., 15. + +aarch64 = trn2 +generate float32x4_t + +aarch64 = zip2 +generate float32x2_t, float64x2_t + +/// Zip vectors +name = vzip +multi_fn = simd_shuffle-in_len-!, a0:in_t, a, b, {zip-1-in_len} +multi_fn = simd_shuffle-in_len-!, b0:in_t, a, b, {zip-2-in_len} +multi_fn = transmute, (a0, b0) +a = 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 +b = 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 +validate 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + +aarch64 = zip +arm = vzip +generate int8x8_t:int8x8_t:int8x8x2_t, int16x4_t:int16x4_t:int16x4x2_t +generate uint8x8_t:uint8x8_t:uint8x8x2_t, uint16x4_t:uint16x4_t:uint16x4x2_t +generate poly8x8_t:poly8x8_t:poly8x8x2_t, poly16x4_t:poly16x4_t:poly16x4x2_t +arm = vtrn +generate int32x2_t:int32x2_t:int32x2x2_t, uint32x2_t:uint32x2_t:uint32x2x2_t +aarch64 = ext +arm = vorr +generate int8x16_t:int8x16_t:int8x16x2_t, int16x8_t:int16x8_t:int16x8x2_t, int32x4_t:int32x4_t:int32x4x2_t +generate uint8x16_t:uint8x16_t:uint8x16x2_t, uint16x8_t:uint16x8_t:uint16x8x2_t, uint32x4_t:uint32x4_t:uint32x4x2_t +generate poly8x16_t:poly8x16_t:poly8x16x2_t, poly16x8_t:poly16x8_t:poly16x8x2_t + +/// Zip vectors +name = vzip +multi_fn = simd_shuffle-in_len-!, a0:in_t, a, b, {zip-1-in_len} +multi_fn = simd_shuffle-in_len-!, b0:in_t, a, b, {zip-2-in_len} +multi_fn = transmute, (a0, b0) +a = 1., 2., 3., 4. +b = 5., 6., 7., 8. +validate 1., 5., 2., 6., 3., 7., 4., 8. + +aarch64 = zip +arm = vtrn +generate float32x2_t:float32x2_t:float32x2x2_t +aarch64 = ext +arm = vorr +generate float32x4_t:float32x4_t:float32x4x2_t + +/// Zip vectors +name = vzip1 +multi_fn = simd_shuffle-in_len-!, a, b, {zip-1-in_len} +a = 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 +b = 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 +validate 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = zip1 +generate int*_t, int64x2_t, uint*_t, uint64x2_t, poly8x8_t, poly8x16_t, poly16x4_t, poly16x8_t, poly64x2_t + +/// Zip vectors +name = vzip1 +multi_fn = simd_shuffle-in_len-!, a, b, {zip-1-in_len} +a = 0., 2., 4., 6., 8., 10., 12., 14. +b = 1., 3., 5., 7., 9., 11., 13., 15. +validate 0., 1., 2., 3., 4., 5., 6., 7. + +aarch64 = zip1 +generate float32x2_t, float32x4_t, float64x2_t + +/// Zip vectors +name = vzip2 +multi_fn = simd_shuffle-in_len-!, a, b, {zip-2-in_len} +a = 0, 16, 16, 18, 16, 18, 20, 22, 16, 18, 20, 22, 24, 26, 28, 30 +b = 1, 17, 17, 19, 17, 19, 21, 23, 17, 19, 21, 23, 25, 27, 29, 31 +validate 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + +aarch64 = zip2 +generate int*_t, int64x2_t, uint*_t, uint64x2_t, poly8x8_t, poly8x16_t, poly16x4_t, poly16x8_t, poly64x2_t + +/// Zip vectors +name = vzip2 +multi_fn = simd_shuffle-in_len-!, a, b, {zip-2-in_len} +a = 0., 8., 8., 10., 8., 10., 12., 14. +b = 1., 9., 9., 11., 9., 11., 13., 15. +validate 8., 9., 10., 11., 12., 13., 14., 15. + +aarch64 = zip2 +generate float32x2_t, float32x4_t, float64x2_t + +/// Unzip vectors +name = vuzp +multi_fn = simd_shuffle-in_len-!, a0:in_t, a, b, {unzip-1-in_len} +multi_fn = simd_shuffle-in_len-!, b0:in_t, a, b, {unzip-2-in_len} +multi_fn = transmute, (a0, b0) +a = 1, 2, 2, 3, 2, 3, 3, 8, 2, 3, 3, 8, 3, 15, 8, 16 +b = 2, 3, 3, 8, 3, 15, 8, 16, 3, 29, 8, 30, 15, 31, 16, 32 +validate 1, 2, 2, 3, 2, 3, 3, 8, 2, 3, 3, 8, 3, 8, 15, 16, 2, 3, 3, 8, 3, 8, 15, 16, 3, 8, 15, 16, 29, 30, 31, 32 + +aarch64 = uzp +arm = vuzp +generate int8x8_t:int8x8_t:int8x8x2_t, int16x4_t:int16x4_t:int16x4x2_t, int8x16_t:int8x16_t:int8x16x2_t, int16x8_t:int16x8_t:int16x8x2_t, int32x4_t:int32x4_t:int32x4x2_t +generate uint8x8_t:uint8x8_t:uint8x8x2_t, uint16x4_t:uint16x4_t:uint16x4x2_t, uint8x16_t:uint8x16_t:uint8x16x2_t, uint16x8_t:uint16x8_t:uint16x8x2_t, uint32x4_t:uint32x4_t:uint32x4x2_t +generate poly8x8_t:poly8x8_t:poly8x8x2_t, poly16x4_t:poly16x4_t:poly16x4x2_t, poly8x16_t:poly8x16_t:poly8x16x2_t, poly16x8_t:poly16x8_t:poly16x8x2_t +aarch64 = zip +arm = vtrn +generate int32x2_t:int32x2_t:int32x2x2_t, uint32x2_t:uint32x2_t:uint32x2x2_t + +/// Unzip vectors +name = vuzp +multi_fn = simd_shuffle-in_len-!, a0:in_t, a, b, {unzip-1-in_len} +multi_fn = simd_shuffle-in_len-!, b0:in_t, a, b, {unzip-2-in_len} +multi_fn = transmute, (a0, b0) +a = 1., 2., 2., 4. +b = 2., 6., 6., 8. +validate 1., 2., 2., 6., 2., 4., 6., 8. + +aarch64 = zip +arm = vtrn +generate float32x2_t:float32x2_t:float32x2x2_t +aarch64 = uzp +arm = vuzp +generate float32x4_t:float32x4_t:float32x4x2_t + +/// Unzip vectors +name = vuzp1 +multi_fn = simd_shuffle-in_len-!, a, b, {unzip-1-in_len} +a = 1, 0, 2, 0, 2, 0, 3, 0, 2, 0, 3, 0, 7, 0, 8, 0 +b = 2, 0, 3, 0, 7, 0, 8, 0, 13, 0, 14, 0, 15, 0, 16, 0 +validate 1, 2, 2, 3, 2, 3, 7, 8, 2, 3, 7, 8, 13, 14, 15, 16 + +aarch64 = uzp1 +generate int8x8_t, int8x16_t, int16x4_t, int16x8_t, int32x4_t, uint8x8_t, uint8x16_t, uint16x4_t, uint16x8_t, uint32x4_t, poly8x8_t, poly8x16_t, poly16x4_t, poly16x8_t + +aarch64 = zip1 +generate int32x2_t, int64x2_t, uint32x2_t, uint64x2_t, poly64x2_t + +/// Unzip vectors +name = vuzp1 +multi_fn = simd_shuffle-in_len-!, a, b, {unzip-1-in_len} +a = 0., 8., 1., 9., 4., 12., 5., 13. +b = 1., 10., 3., 11., 6., 14., 7., 15. +validate 0., 1., 1., 3., 4., 5., 6., 7. + +aarch64 = uzp1 +generate float32x4_t + +aarch64 = zip1 +generate float32x2_t, float64x2_t + +/// Unzip vectors +name = vuzp2 +multi_fn = simd_shuffle-in_len-!, a, b, {unzip-2-in_len} +a = 0, 17, 0, 18, 0, 18, 0, 19, 0, 18, 0, 19, 0, 23, 0, 24 +b = 0, 18, 0, 19, 0, 23, 0, 24, 0, 29, 0, 30, 0, 31, 0, 32 +validate 17, 18, 18, 19, 18, 19, 23, 24, 18, 19, 23, 24, 29, 30, 31, 32 + +aarch64 = uzp2 +generate int8x8_t, int8x16_t, int16x4_t, int16x8_t, int32x4_t, uint8x8_t, uint8x16_t, uint16x4_t, uint16x8_t, uint32x4_t, poly8x8_t, poly8x16_t, poly16x4_t, poly16x8_t + +aarch64 = zip2 +generate int32x2_t, int64x2_t, uint32x2_t, uint64x2_t, poly64x2_t + +/// Unzip vectors +name = vuzp2 +multi_fn = simd_shuffle-in_len-!, a, b, {unzip-2-in_len} +a = 0., 8., 1., 9., 4., 12., 5., 13. +b = 2., 9., 3., 11., 6., 14., 7., 15. +validate 8., 9., 9., 11., 12., 13., 14., 15. + +aarch64 = uzp2 +generate float32x4_t + +aarch64 = zip2 +generate float32x2_t, float64x2_t + +//////////////////// +// Unsigned Absolute difference and Accumulate Long +//////////////////// + +/// Unsigned Absolute difference and Accumulate Long +name = vabal +multi_fn = vabd-unsigned-noext, b, c, d:in_t +multi_fn = simd_add, a, {simd_cast, d} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +c = 10, 10, 10, 10, 10, 10, 10, 10, 20, 0, 2, 4, 6, 8, 10, 12 +validate 10, 10, 10, 10, 10, 10, 10, 10, 20, 20, 20, 20, 20, 20, 20, 20 + +arm = vabal.s +aarch64 = uabal +generate uint16x8_t:uint8x8_t:uint8x8_t:uint16x8_t, uint32x4_t:uint16x4_t:uint16x4_t:uint32x4_t, uint64x2_t:uint32x2_t:uint32x2_t:uint64x2_t + +/// Unsigned Absolute difference and Accumulate Long +name = vabal_high +no-q +multi_fn = simd_shuffle8!, d:uint8x8_t, b, b, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_shuffle8!, e:uint8x8_t, c, c, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = vabd_u8, d, e, f:uint8x8_t +multi_fn = simd_add, a, {simd_cast, f} +a = 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +c = 10, 10, 10, 10, 10, 10, 10, 10, 20, 0, 2, 4, 6, 8, 10, 12 +validate 20, 20, 20, 20, 20, 20, 20, 20 + +aarch64 = uabal +generate uint16x8_t:uint8x16_t:uint8x16_t:uint16x8_t + +/// Unsigned Absolute difference and Accumulate Long +name = vabal_high +no-q +multi_fn = simd_shuffle4!, d:uint16x4_t, b, b, [4, 5, 6, 7] +multi_fn = simd_shuffle4!, e:uint16x4_t, c, c, [4, 5, 6, 7] +multi_fn = vabd_u16, d, e, f:uint16x4_t +multi_fn = simd_add, a, {simd_cast, f} +a = 9, 10, 11, 12 +b = 1, 2, 3, 4, 9, 10, 11, 12 +c = 10, 10, 10, 10, 20, 0, 2, 4 +validate 20, 20, 20, 20 + +aarch64 = uabal +generate uint32x4_t:uint16x8_t:uint16x8_t:uint32x4_t + +/// Unsigned Absolute difference and Accumulate Long +name = vabal_high +no-q +multi_fn = simd_shuffle2!, d:uint32x2_t, b, b, [2, 3] +multi_fn = simd_shuffle2!, e:uint32x2_t, c, c, [2, 3] +multi_fn = vabd_u32, d, e, f:uint32x2_t +multi_fn = simd_add, a, {simd_cast, f} +a = 15, 16 +b = 1, 2, 15, 16 +c = 10, 10, 10, 12 +validate 20, 20 + +aarch64 = uabal +generate uint64x2_t:uint32x4_t:uint32x4_t:uint64x2_t + +//////////////////// +// Signed Absolute difference and Accumulate Long +//////////////////// + +/// Signed Absolute difference and Accumulate Long +name = vabal +multi_fn = vabd-signed-noext, b, c, d:int8x8_t +multi_fn = simd_cast, e:uint8x8_t, d +multi_fn = simd_add, a, {simd_cast, e} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +c = 10, 10, 10, 10, 10, 10, 10, 10, 20, 0, 2, 4, 6, 8, 10, 12 +validate 10, 10, 10, 10, 10, 10, 10, 10, 20, 20, 20, 20, 20, 20, 20, 20 + +arm = vabal.s +aarch64 = sabal +generate int16x8_t:int8x8_t:int8x8_t:int16x8_t + +/// Signed Absolute difference and Accumulate Long +name = vabal +multi_fn = vabd-signed-noext, b, c, d:int16x4_t +multi_fn = simd_cast, e:uint16x4_t, d +multi_fn = simd_add, a, {simd_cast, e} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +c = 10, 10, 10, 10, 10, 10, 10, 10, 20, 0, 2, 4, 6, 8, 10, 12 +validate 10, 10, 10, 10, 10, 10, 10, 10, 20, 20, 20, 20, 20, 20, 20, 20 + +arm = vabal.s +aarch64 = sabal +generate int32x4_t:int16x4_t:int16x4_t:int32x4_t + +/// Signed Absolute difference and Accumulate Long +name = vabal +multi_fn = vabd-signed-noext, b, c, d:int32x2_t +multi_fn = simd_cast, e:uint32x2_t, d +multi_fn = simd_add, a, {simd_cast, e} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +c = 10, 10, 10, 10, 10, 10, 10, 10, 20, 0, 2, 4, 6, 8, 10, 12 +validate 10, 10, 10, 10, 10, 10, 10, 10, 20, 20, 20, 20, 20, 20, 20, 20 + +arm = vabal.s +aarch64 = sabal +generate int64x2_t:int32x2_t:int32x2_t:int64x2_t + +/// Signed Absolute difference and Accumulate Long +name = vabal_high +no-q +multi_fn = simd_shuffle8!, d:int8x8_t, b, b, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_shuffle8!, e:int8x8_t, c, c, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = vabd_s8, d, e, f:int8x8_t +multi_fn = simd_cast, f:uint8x8_t, f +multi_fn = simd_add, a, {simd_cast, f} +a = 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +c = 10, 10, 10, 10, 10, 10, 10, 10, 20, 0, 2, 4, 6, 8, 10, 12 +validate 20, 20, 20, 20, 20, 20, 20, 20 + +aarch64 = sabal +generate int16x8_t:int8x16_t:int8x16_t:int16x8_t + +/// Signed Absolute difference and Accumulate Long +name = vabal_high +no-q +multi_fn = simd_shuffle4!, d:int16x4_t, b, b, [4, 5, 6, 7] +multi_fn = simd_shuffle4!, e:int16x4_t, c, c, [4, 5, 6, 7] +multi_fn = vabd_s16, d, e, f:int16x4_t +multi_fn = simd_cast, f:uint16x4_t, f +multi_fn = simd_add, a, {simd_cast, f} +a = 9, 10, 11, 12 +b = 1, 2, 3, 4, 9, 10, 11, 12 +c = 10, 10, 10, 10, 20, 0, 2, 4 +validate 20, 20, 20, 20 + +aarch64 = sabal +generate int32x4_t:int16x8_t:int16x8_t:int32x4_t + +/// Signed Absolute difference and Accumulate Long +name = vabal_high +no-q +multi_fn = simd_shuffle2!, d:int32x2_t, b, b, [2, 3] +multi_fn = simd_shuffle2!, e:int32x2_t, c, c, [2, 3] +multi_fn = vabd_s32, d, e, f:int32x2_t +multi_fn = simd_cast, f:uint32x2_t, f +multi_fn = simd_add, a, {simd_cast, f} +a = 15, 16 +b = 1, 2, 15, 16 +c = 10, 10, 10, 12 +validate 20, 20 + +aarch64 = sabal +generate int64x2_t:int32x4_t:int32x4_t:int64x2_t + +//////////////////// +// Singned saturating Absolute value +//////////////////// + +/// Singned saturating Absolute value +name = vqabs +a = MIN, MAX, -6, -5, -4, -3, -2, -1, 0, -127, 127, 1, 2, 3, 4, 5 +validate MAX, MAX, 6, 5, 4, 3, 2, 1, 0, 127, 127, 1, 2, 3, 4, 5 + +arm = vqabs.s +aarch64 = sqabs +link-arm = vqabs._EXT_ +link-aarch64 = sqabs._EXT_ +generate int*_t + +/// Singned saturating Absolute value +name = vqabs +a = MIN, -7 +validate MAX, 7 + +aarch64 = sqabs +link-aarch64 = sqabs._EXT_ +generate int64x*_t + +/// Signed saturating absolute value +name = vqabs +multi_fn = simd_extract, {vqabs-in_ntt-noext, {vdup_n-in_ntt-noext, a}}, 0 +a = -7 +validate 7 + +aarch64 = sqabs +generate i8:i8, i16:i16 + +/// Signed saturating absolute value +name = vqabs +a = -7 +validate 7 + +aarch64 = sqabs +link-aarch64 = sqabs._EXT_ +generate i32:i32, i64:i64 + +/// Shift left and insert +name = vsli +n-suffix +constn = N +multi_fn = static_assert-N-0-63 +multi_fn = transmute, {vsli_n-in_ntt-::, transmute(a), transmute(b)} +a = 333 +b = 2042 +n = 2 +validate 8169 + +aarch64 = sli +generate i64, u64 + +/// Shift right and insert +name = vsri +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = transmute, {vsri_n-in_ntt-::, transmute(a), transmute(b)} +a = 333 +b = 2042 +n = 2 +validate 510 + +aarch64 = sri +generate i64, u64 diff --git a/crux-mir/lib/stdarch/crates/stdarch-gen/src/main.rs b/crux-mir/lib/stdarch/crates/stdarch-gen/src/main.rs new file mode 100644 index 000000000..d2f865753 --- /dev/null +++ b/crux-mir/lib/stdarch/crates/stdarch-gen/src/main.rs @@ -0,0 +1,3438 @@ +use self::Suffix::*; +use self::TargetFeature::*; +use std::env; +use std::fs::File; +use std::io::prelude::*; +use std::io::{self, BufReader}; +use std::path::PathBuf; + +const IN: &str = "neon.spec"; +const ARM_OUT: &str = "generated.rs"; +const AARCH64_OUT: &str = "generated.rs"; + +const UINT_TYPES: [&str; 6] = [ + "uint8x8_t", + "uint8x16_t", + "uint16x4_t", + "uint16x8_t", + "uint32x2_t", + "uint32x4_t", +]; + +const UINT_TYPES_64: [&str; 2] = ["uint64x1_t", "uint64x2_t"]; + +const INT_TYPES: [&str; 6] = [ + "int8x8_t", + "int8x16_t", + "int16x4_t", + "int16x8_t", + "int32x2_t", + "int32x4_t", +]; + +const INT_TYPES_64: [&str; 2] = ["int64x1_t", "int64x2_t"]; + +const FLOAT_TYPES: [&str; 2] = [ + //"float8x8_t", not supported by rust + //"float8x16_t", not supported by rust + //"float16x4_t", not supported by rust + //"float16x8_t", not supported by rust + "float32x2_t", + "float32x4_t", +]; + +const FLOAT_TYPES_64: [&str; 2] = [ + //"float8x8_t", not supported by rust + //"float8x16_t", not supported by rust + //"float16x4_t", not supported by rust + //"float16x8_t", not supported by rust + "float64x1_t", + "float64x2_t", +]; + +fn type_len(t: &str) -> usize { + let s: Vec<_> = t.split("x").collect(); + if s.len() == 2 { + match &s[1][0..2] { + "1_" => 1, + "2_" => 2, + "4_" => 4, + "8_" => 8, + "16" => 16, + _ => panic!("unknown type: {}", t), + } + } else if s.len() == 3 { + s[1].parse::().unwrap() * type_sub_len(t) + } else { + 1 + } +} + +fn type_sub_len(t: &str) -> usize { + let s: Vec<_> = t.split('x').collect(); + if s.len() != 3 { + 1 + } else { + match s[2] { + "2_t" => 2, + "3_t" => 3, + "4_t" => 4, + _ => panic!("unknown type len: {}", t), + } + } +} + +fn type_bits(t: &str) -> usize { + match t { + "int8x8_t" | "int8x16_t" | "uint8x8_t" | "uint8x16_t" | "poly8x8_t" | "poly8x16_t" + | "i8" | "u8" => 8, + "int16x4_t" | "int16x8_t" | "uint16x4_t" | "uint16x8_t" | "poly16x4_t" | "poly16x8_t" + | "i16" | "u16" => 16, + "int32x2_t" | "int32x4_t" | "uint32x2_t" | "uint32x4_t" | "i32" | "u32" | "float32x2_t" + | "float32x4_t" | "f32" => 32, + "int64x1_t" | "int64x2_t" | "uint64x1_t" | "uint64x2_t" | "poly64x1_t" | "poly64x2_t" + | "i64" | "u64" | "float64x1_t" | "float64x2_t" | "f64" => 64, + _ => panic!("unknown type: {}", t), + } +} + +fn type_exp_len(t: &str, base_len: usize) -> usize { + let t = type_to_sub_type(t); + let len = type_len(&t) / base_len; + match len { + 1 => 0, + 2 => 1, + 4 => 2, + 8 => 3, + 16 => 4, + _ => panic!("unknown type: {}", t), + } +} + +fn type_bits_exp_len(t: &str) -> usize { + match t { + "int8x8_t" | "int8x16_t" | "uint8x8_t" | "uint8x16_t" | "poly8x8_t" | "poly8x16_t" + | "i8" | "u8" => 3, + "int16x4_t" | "int16x8_t" | "uint16x4_t" | "uint16x8_t" | "poly16x4_t" | "poly16x8_t" + | "i16" | "u16" => 4, + "int32x2_t" | "int32x4_t" | "uint32x2_t" | "uint32x4_t" | "i32" | "u32" => 5, + "int64x1_t" | "int64x2_t" | "uint64x1_t" | "uint64x2_t" | "poly64x1_t" | "poly64x2_t" + | "i64" | "u64" => 6, + _ => panic!("unknown type: {}", t), + } +} + +fn type_to_suffix(t: &str) -> &str { + match t { + "int8x8_t" => "_s8", + "int8x16_t" => "q_s8", + "int16x4_t" => "_s16", + "int16x8_t" => "q_s16", + "int32x2_t" => "_s32", + "int32x4_t" => "q_s32", + "int64x1_t" => "_s64", + "int64x2_t" => "q_s64", + "uint8x8_t" => "_u8", + "uint8x16_t" => "q_u8", + "uint16x4_t" => "_u16", + "uint16x8_t" => "q_u16", + "uint32x2_t" => "_u32", + "uint32x4_t" => "q_u32", + "uint64x1_t" => "_u64", + "uint64x2_t" => "q_u64", + "float16x4_t" => "_f16", + "float16x8_t" => "q_f16", + "float32x2_t" => "_f32", + "float32x4_t" => "q_f32", + "float64x1_t" => "_f64", + "float64x2_t" => "q_f64", + "poly8x8_t" => "_p8", + "poly8x16_t" => "q_p8", + "poly16x4_t" => "_p16", + "poly16x8_t" => "q_p16", + "poly64x1_t" => "_p64", + "poly64x2_t" => "q_p64", + "int8x8x2_t" => "_s8_x2", + "int8x8x3_t" => "_s8_x3", + "int8x8x4_t" => "_s8_x4", + "int16x4x2_t" => "_s16_x2", + "int16x4x3_t" => "_s16_x3", + "int16x4x4_t" => "_s16_x4", + "int32x2x2_t" => "_s32_x2", + "int32x2x3_t" => "_s32_x3", + "int32x2x4_t" => "_s32_x4", + "int64x1x2_t" => "_s64_x2", + "int64x1x3_t" => "_s64_x3", + "int64x1x4_t" => "_s64_x4", + "uint8x8x2_t" => "_u8_x2", + "uint8x8x3_t" => "_u8_x3", + "uint8x8x4_t" => "_u8_x4", + "uint16x4x2_t" => "_u16_x2", + "uint16x4x3_t" => "_u16_x3", + "uint16x4x4_t" => "_u16_x4", + "uint32x2x2_t" => "_u32_x2", + "uint32x2x3_t" => "_u32_x3", + "uint32x2x4_t" => "_u32_x4", + "uint64x1x2_t" => "_u64_x2", + "uint64x1x3_t" => "_u64_x3", + "uint64x1x4_t" => "_u64_x4", + "poly8x8x2_t" => "_p8_x2", + "poly8x8x3_t" => "_p8_x3", + "poly8x8x4_t" => "_p8_x4", + "poly16x4x2_t" => "_p16_x2", + "poly16x4x3_t" => "_p16_x3", + "poly16x4x4_t" => "_p16_x4", + "poly64x1x2_t" => "_p64_x2", + "poly64x1x3_t" => "_p64_x3", + "poly64x1x4_t" => "_p64_x4", + "float32x2x2_t" => "_f32_x2", + "float32x2x3_t" => "_f32_x3", + "float32x2x4_t" => "_f32_x4", + "float64x1x2_t" => "_f64_x2", + "float64x1x3_t" => "_f64_x3", + "float64x1x4_t" => "_f64_x4", + "int8x16x2_t" => "q_s8_x2", + "int8x16x3_t" => "q_s8_x3", + "int8x16x4_t" => "q_s8_x4", + "int16x8x2_t" => "q_s16_x2", + "int16x8x3_t" => "q_s16_x3", + "int16x8x4_t" => "q_s16_x4", + "int32x4x2_t" => "q_s32_x2", + "int32x4x3_t" => "q_s32_x3", + "int32x4x4_t" => "q_s32_x4", + "int64x2x2_t" => "q_s64_x2", + "int64x2x3_t" => "q_s64_x3", + "int64x2x4_t" => "q_s64_x4", + "uint8x16x2_t" => "q_u8_x2", + "uint8x16x3_t" => "q_u8_x3", + "uint8x16x4_t" => "q_u8_x4", + "uint16x8x2_t" => "q_u16_x2", + "uint16x8x3_t" => "q_u16_x3", + "uint16x8x4_t" => "q_u16_x4", + "uint32x4x2_t" => "q_u32_x2", + "uint32x4x3_t" => "q_u32_x3", + "uint32x4x4_t" => "q_u32_x4", + "uint64x2x2_t" => "q_u64_x2", + "uint64x2x3_t" => "q_u64_x3", + "uint64x2x4_t" => "q_u64_x4", + "poly8x16x2_t" => "q_p8_x2", + "poly8x16x3_t" => "q_p8_x3", + "poly8x16x4_t" => "q_p8_x4", + "poly16x8x2_t" => "q_p16_x2", + "poly16x8x3_t" => "q_p16_x3", + "poly16x8x4_t" => "q_p16_x4", + "poly64x2x2_t" => "q_p64_x2", + "poly64x2x3_t" => "q_p64_x3", + "poly64x2x4_t" => "q_p64_x4", + "float32x4x2_t" => "q_f32_x2", + "float32x4x3_t" => "q_f32_x3", + "float32x4x4_t" => "q_f32_x4", + "float64x2x2_t" => "q_f64_x2", + "float64x2x3_t" => "q_f64_x3", + "float64x2x4_t" => "q_f64_x4", + "i8" => "b_s8", + "i16" => "h_s16", + "i32" => "s_s32", + "i64" => "d_s64", + "u8" => "b_u8", + "u16" => "h_u16", + "u32" => "s_u32", + "u64" => "d_u64", + "f32" => "s_f32", + "f64" => "d_f64", + "p8" => "b_p8", + "p16" => "h_p16", + "p128" => "q_p128", + _ => panic!("unknown type: {}", t), + } +} + +fn type_to_dup_suffix(t: &str) -> String { + let s: Vec<_> = type_to_suffix(t).split('_').collect(); + assert_eq!(s.len(), 2); + format!("{}_dup_{}", s[0], s[1]) +} + +fn type_to_lane_suffix(t: &str) -> String { + let s: Vec<_> = type_to_suffix(t).split('_').collect(); + assert_eq!(s.len(), 2); + format!("{}_lane_{}", s[0], s[1]) +} + +fn type_to_n_suffix(t: &str) -> &str { + match t { + "int8x8_t" => "_n_s8", + "int8x16_t" => "q_n_s8", + "int16x4_t" => "_n_s16", + "int16x8_t" => "q_n_s16", + "int32x2_t" => "_n_s32", + "int32x4_t" => "q_n_s32", + "int64x1_t" => "_n_s64", + "int64x2_t" => "q_n_s64", + "uint8x8_t" => "_n_u8", + "uint8x16_t" => "q_n_u8", + "uint16x4_t" => "_n_u16", + "uint16x8_t" => "q_n_u16", + "uint32x2_t" => "_n_u32", + "uint32x4_t" => "q_n_u32", + "uint64x1_t" => "_n_u64", + "uint64x2_t" => "q_n_u64", + "float16x4_t" => "_n_f16", + "float16x8_t" => "q_n_f16", + "float32x2_t" => "_n_f32", + "float32x4_t" => "q_n_f32", + "float64x1_t" => "_n_f64", + "float64x2_t" => "q_n_f64", + "poly8x8_t" => "_n_p8", + "poly8x16_t" => "q_n_p8", + "poly16x4_t" => "_n_p16", + "poly16x8_t" => "q_n_p16", + "poly64x1_t" => "_n_p64", + "poly64x2_t" => "q_n_p64", + "i8" => "b_n_s8", + "i16" => "h_n_s16", + "i32" => "s_n_s32", + "i64" => "d_n_s64", + "u8" => "b_n_u8", + "u16" => "h_n_u16", + "u32" => "s_n_u32", + "u64" => "d_n_u64", + _ => panic!("unknown type: {}", t), + } +} + +fn type_to_noq_n_suffix(t: &str) -> &str { + match t { + "int8x8_t" | "int8x16_t" => "_n_s8", + "int16x4_t" | "int16x8_t" => "_n_s16", + "int32x2_t" | "int32x4_t" => "_n_s32", + "int64x1_t" | "int64x2_t" => "_n_s64", + "uint8x8_t" | "uint8x16_t" => "_n_u8", + "uint16x4_t" | "uint16x8_t" => "_n_u16", + "uint32x2_t" | "uint32x4_t" => "_n_u32", + "uint64x1_t" | "uint64x2_t" => "_n_u64", + "float16x4_t" | "float16x8_t" => "_n_f16", + "float32x2_t" | "float32x4_t" => "_n_f32", + "float64x1_t" | "float64x2_t" => "_n_f64", + "poly8x8_t" | "poly8x16_t" => "_n_p8", + "poly16x4_t" | "poly16x8_t" => "_n_p16", + "poly64x1_t" | "poly64x2_t" => "_n_p64", + "i8" => "b_n_s8", + "i16" => "h_n_s16", + "i32" => "s_n_s32", + "i64" => "d_n_s64", + "u8" => "b_n_u8", + "u16" => "h_n_u16", + "u32" => "s_n_u32", + "u64" => "d_n_u64", + _ => panic!("unknown type: {}", t), + } +} + +fn type_to_lane_suffixes<'a>(out_t: &'a str, in_t: &'a str, re_to_out: bool) -> String { + let mut str = String::new(); + let suf = type_to_suffix(out_t); + if !suf.starts_with("_") { + str.push_str(&suf[0..1]); + } + str.push_str("_lane"); + if !re_to_out { + str.push_str(type_to_suffix(in_t)); + } else { + if type_to_suffix(in_t).starts_with("q") { + str.push_str("q"); + }; + let suf2 = type_to_noq_suffix(out_t); + str.push_str(suf2); + } + str +} + +fn type_to_rot_suffix(c_name: &str, suf: &str) -> String { + let ns: Vec<_> = c_name.split('_').collect(); + assert_eq!(ns.len(), 2); + if suf.starts_with("q") { + format!("{}q_{}{}", ns[0], ns[1], &suf[1..]) + } else { + format!("{}{}", c_name, suf) + } +} + +fn type_to_signed(t: &str) -> String { + let s = t.replace("uint", "int"); + let s = s.replace("poly", "int"); + s +} + +fn type_to_unsigned(t: &str) -> String { + if t.contains("uint") { + return t.to_string(); + } + let s = t.replace("int", "uint"); + let s = s.replace("poly", "uint"); + s +} + +fn type_to_double_suffixes<'a>(out_t: &'a str, in_t: &'a str) -> String { + let mut str = String::new(); + let suf = type_to_suffix(in_t); + if suf.starts_with("q") && type_to_suffix(out_t).starts_with("q") { + str.push_str("q"); + } + if !suf.starts_with("_") && !suf.starts_with("q") { + str.push_str(&suf[0..1]); + } + str.push_str(type_to_noq_suffix(out_t)); + str.push_str(type_to_noq_suffix(in_t)); + str +} + +fn type_to_double_n_suffixes<'a>(out_t: &'a str, in_t: &'a str) -> String { + let mut str = String::new(); + let suf = type_to_suffix(in_t); + if suf.starts_with("q") && type_to_suffix(out_t).starts_with("q") { + str.push_str("q"); + } + if !suf.starts_with("_") && !suf.starts_with("q") { + str.push_str(&suf[0..1]); + } + str.push_str("_n"); + str.push_str(type_to_noq_suffix(out_t)); + str.push_str(type_to_noq_suffix(in_t)); + str +} + +fn type_to_noq_double_suffixes<'a>(out_t: &'a str, in_t: &'a str) -> String { + let mut str = String::new(); + str.push_str(type_to_noq_suffix(out_t)); + str.push_str(type_to_noq_suffix(in_t)); + str +} + +fn type_to_noq_suffix(t: &str) -> &str { + match t { + "int8x8_t" | "int8x16_t" | "i8" => "_s8", + "int16x4_t" | "int16x8_t" | "i16" => "_s16", + "int32x2_t" | "int32x4_t" | "i32" => "_s32", + "int64x1_t" | "int64x2_t" | "i64" => "_s64", + "uint8x8_t" | "uint8x16_t" | "u8" => "_u8", + "uint16x4_t" | "uint16x8_t" | "u16" => "_u16", + "uint32x2_t" | "uint32x4_t" | "u32" => "_u32", + "uint64x1_t" | "uint64x2_t" | "u64" => "_u64", + "float16x4_t" | "float16x8_t" => "_f16", + "float32x2_t" | "float32x4_t" | "f32" => "_f32", + "float64x1_t" | "float64x2_t" | "f64" => "_f64", + "poly8x8_t" | "poly8x16_t" => "_p8", + "poly16x4_t" | "poly16x8_t" => "_p16", + "poly64x1_t" | "poly64x2_t" | "p64" => "_p64", + "p128" => "_p128", + _ => panic!("unknown type: {}", t), + } +} + +#[derive(Clone, Copy)] +enum Suffix { + Normal, + Double, + NoQ, + NoQDouble, + NSuffix, + DoubleN, + NoQNSuffix, + OutSuffix, + OutNSuffix, + OutNox, + In1Nox, + OutDupNox, + OutLaneNox, + In1LaneNox, + Lane, + In2, + In2Lane, + OutLane, + Rot, + RotLane, +} + +#[derive(Clone, Copy)] +enum TargetFeature { + Default, + ArmV7, + Vfp4, + FPArmV8, + AES, + FCMA, + Dotprod, + I8MM, + SHA3, + RDM, + SM4, + FTTS, +} + +#[derive(Clone, Copy)] +enum Fntype { + Normal, + Load, + Store, +} + +fn type_to_global_type(t: &str) -> &str { + match t { + "int8x8_t" | "int8x8x2_t" | "int8x8x3_t" | "int8x8x4_t" => "i8x8", + "int8x16_t" | "int8x16x2_t" | "int8x16x3_t" | "int8x16x4_t" => "i8x16", + "int16x4_t" | "int16x4x2_t" | "int16x4x3_t" | "int16x4x4_t" => "i16x4", + "int16x8_t" | "int16x8x2_t" | "int16x8x3_t" | "int16x8x4_t" => "i16x8", + "int32x2_t" | "int32x2x2_t" | "int32x2x3_t" | "int32x2x4_t" => "i32x2", + "int32x4_t" | "int32x4x2_t" | "int32x4x3_t" | "int32x4x4_t" => "i32x4", + "int64x1_t" | "int64x1x2_t" | "int64x1x3_t" | "int64x1x4_t" => "i64x1", + "int64x2_t" | "int64x2x2_t" | "int64x2x3_t" | "int64x2x4_t" => "i64x2", + "uint8x8_t" | "uint8x8x2_t" | "uint8x8x3_t" | "uint8x8x4_t" => "u8x8", + "uint8x16_t" | "uint8x16x2_t" | "uint8x16x3_t" | "uint8x16x4_t" => "u8x16", + "uint16x4_t" | "uint16x4x2_t" | "uint16x4x3_t" | "uint16x4x4_t" => "u16x4", + "uint16x8_t" | "uint16x8x2_t" | "uint16x8x3_t" | "uint16x8x4_t" => "u16x8", + "uint32x2_t" | "uint32x2x2_t" | "uint32x2x3_t" | "uint32x2x4_t" => "u32x2", + "uint32x4_t" | "uint32x4x2_t" | "uint32x4x3_t" | "uint32x4x4_t" => "u32x4", + "uint64x1_t" | "uint64x1x2_t" | "uint64x1x3_t" | "uint64x1x4_t" => "u64x1", + "uint64x2_t" | "uint64x2x2_t" | "uint64x2x3_t" | "uint64x2x4_t" => "u64x2", + "float16x4_t" => "f16x4", + "float16x8_t" => "f16x8", + "float32x2_t" | "float32x2x2_t" | "float32x2x3_t" | "float32x2x4_t" => "f32x2", + "float32x4_t" | "float32x4x2_t" | "float32x4x3_t" | "float32x4x4_t" => "f32x4", + "float64x1_t" | "float64x1x2_t" | "float64x1x3_t" | "float64x1x4_t" => "f64", + "float64x2_t" | "float64x2x2_t" | "float64x2x3_t" | "float64x2x4_t" => "f64x2", + "poly8x8_t" | "poly8x8x2_t" | "poly8x8x3_t" | "poly8x8x4_t" => "i8x8", + "poly8x16_t" | "poly8x16x2_t" | "poly8x16x3_t" | "poly8x16x4_t" => "i8x16", + "poly16x4_t" | "poly16x4x2_t" | "poly16x4x3_t" | "poly16x4x4_t" => "i16x4", + "poly16x8_t" | "poly16x8x2_t" | "poly16x8x3_t" | "poly16x8x4_t" => "i16x8", + "poly64x1_t" | "poly64x1x2_t" | "poly64x1x3_t" | "poly64x1x4_t" => "i64x1", + "poly64x2_t" | "poly64x2x2_t" | "poly64x2x3_t" | "poly64x2x4_t" => "i64x2", + "i8" => "i8", + "i16" => "i16", + "i32" => "i32", + "i64" => "i64", + "u8" => "u8", + "u16" => "u16", + "u32" => "u32", + "u64" => "u64", + "f32" => "f32", + "f64" => "f64", + "p8" => "p8", + "p16" => "p16", + "p64" => "p64", + "p128" => "p128", + _ => panic!("unknown type: {}", t), + } +} + +fn type_to_sub_type(t: &str) -> String { + let s: Vec<_> = t.split('x').collect(); + match s.len() { + 2 => String::from(t), + 3 => format!("{}x{}_t", s[0], s[1]), + _ => panic!("unknown type: {}", t), + } +} + +fn type_to_native_type(t: &str) -> String { + let s: Vec<_> = t.split('x').collect(); + match s.len() { + 1 => { + assert!(t.contains("*const") || t.contains("*mut")); + let sub: Vec<_> = t.split(' ').collect(); + String::from(sub[1]) + } + 2 | 3 => match &s[0][0..3] { + "int" => format!("i{}", &s[0][3..]), + "uin" => format!("u{}", &s[0][4..]), + "flo" => format!("f{}", &s[0][5..]), + "pol" => format!("u{}", &s[0][4..]), + _ => panic!("unknown type: {}", t), + }, + _ => panic!("unknown type: {}", t), + } +} + +fn native_type_to_type(t: &str) -> &str { + match t { + "i8" => "int8x8_t", + "i16" => "int16x4_t", + "i32" => "int32x2_t", + "i64" => "int64x1_t", + "u8" => "uint8x8_t", + "u16" => "uint16x4_t", + "u32" => "uint32x2_t", + "u64" => "uint64x1_t", + "f16" => "float16x4_t", + "f32" => "float32x2_t", + "f64" => "float64x1_t", + _ => panic!("unknown type: {}", t), + } +} + +fn native_type_to_long_type(t: &str) -> &str { + match t { + "i8" => "int8x16_t", + "i16" => "int16x8_t", + "i32" => "int32x4_t", + "i64" => "int64x2_t", + "u8" => "uint8x16_t", + "u16" => "uint16x8_t", + "u32" => "uint32x4_t", + "u64" => "uint64x2_t", + "f16" => "float16x8_t", + "f32" => "float32x4_t", + "f64" => "float64x2_t", + _ => panic!("unknown type: {}", t), + } +} + +fn type_to_half(t: &str) -> &str { + match t { + "int8x16_t" => "int8x8_t", + "int16x8_t" => "int16x4_t", + "int32x4_t" => "int32x2_t", + "int64x2_t" => "int64x1_t", + "uint8x16_t" => "uint8x8_t", + "uint16x8_t" => "uint16x4_t", + "uint32x4_t" => "uint32x2_t", + "uint64x2_t" => "uint64x1_t", + "poly8x16_t" => "poly8x8_t", + "poly16x8_t" => "poly16x4_t", + "float32x4_t" => "float32x2_t", + "float64x2_t" => "float64x1_t", + _ => panic!("unknown half type for {}", t), + } +} + +fn asc(start: i32, len: usize) -> String { + let mut s = String::from("["); + for i in 0..len { + if i != 0 { + s.push_str(", "); + } + let n = start + i as i32; + s.push_str(&n.to_string()); + } + s.push_str("]"); + s +} + +fn transpose1(x: usize) -> &'static str { + match x { + 2 => "[0, 2]", + 4 => "[0, 4, 2, 6]", + 8 => "[0, 8, 2, 10, 4, 12, 6, 14]", + 16 => "[0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]", + _ => panic!("unknown transpose order of len {}", x), + } +} + +fn transpose2(x: usize) -> &'static str { + match x { + 2 => "[1, 3]", + 4 => "[1, 5, 3, 7]", + 8 => "[1, 9, 3, 11, 5, 13, 7, 15]", + 16 => "[1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]", + _ => panic!("unknown transpose order of len {}", x), + } +} + +fn zip1(x: usize) -> &'static str { + match x { + 2 => "[0, 2]", + 4 => "[0, 4, 1, 5]", + 8 => "[0, 8, 1, 9, 2, 10, 3, 11]", + 16 => "[0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]", + _ => panic!("unknown zip order of len {}", x), + } +} + +fn zip2(x: usize) -> &'static str { + match x { + 2 => "[1, 3]", + 4 => "[2, 6, 3, 7]", + 8 => "[4, 12, 5, 13, 6, 14, 7, 15]", + 16 => "[8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]", + _ => panic!("unknown zip order of len {}", x), + } +} + +fn unzip1(x: usize) -> &'static str { + match x { + 2 => "[0, 2]", + 4 => "[0, 2, 4, 6]", + 8 => "[0, 2, 4, 6, 8, 10, 12, 14]", + 16 => "[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]", + _ => panic!("unknown unzip order of len {}", x), + } +} + +fn unzip2(x: usize) -> &'static str { + match x { + 2 => "[1, 3]", + 4 => "[1, 3, 5, 7]", + 8 => "[1, 3, 5, 7, 9, 11, 13, 15]", + 16 => "[1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]", + _ => panic!("unknown unzip order of len {}", x), + } +} + +fn values(t: &str, vs: &[String]) -> String { + if vs.len() == 1 && !t.contains('x') { + format!(": {} = {}", t, vs[0]) + } else if vs.len() == 1 && type_to_global_type(t) == "f64" { + format!(": {} = {}", type_to_global_type(t), vs[0]) + } else { + let s: Vec<_> = t.split('x').collect(); + if s.len() == 3 { + format!( + ": [{}; {}] = [{}]", + type_to_native_type(t), + type_len(t), + vs.iter() + .map(|v| map_val(type_to_global_type(t), v)) + //.map(|v| format!("{}{}", v, type_to_native_type(t))) + .collect::>() + .join(", ") + ) + } else { + format!( + ": {} = {}::new({})", + type_to_global_type(t), + type_to_global_type(t), + vs.iter() + .map(|v| map_val(type_to_global_type(t), v)) + //.map(|v| format!("{}{}", v, type_to_native_type(t))) + .collect::>() + .join(", ") + ) + } + } +} + +fn max_val(t: &str) -> &'static str { + match &t[..3] { + "u8x" => "0xFF", + "u16" => "0xFF_FF", + "u32" => "0xFF_FF_FF_FF", + "u64" => "0xFF_FF_FF_FF_FF_FF_FF_FF", + "i8x" => "0x7F", + "i16" => "0x7F_FF", + "i32" => "0x7F_FF_FF_FF", + "i64" => "0x7F_FF_FF_FF_FF_FF_FF_FF", + "f32" => "3.40282347e+38", + "f64" => "1.7976931348623157e+308", + _ => panic!("No TRUE for type {}", t), + } +} + +fn min_val(t: &str) -> &'static str { + match &t[..3] { + "u8x" => "0", + "u16" => "0", + "u32" => "0", + "u64" => "0", + "i8x" => "-128", + "i16" => "-32768", + "i32" => "-2147483648", + "i64" => "-9223372036854775808", + "f32" => "-3.40282347e+38", + "f64" => "-1.7976931348623157e+308", + _ => panic!("No TRUE for type {}", t), + } +} + +fn true_val(t: &str) -> &'static str { + match &t[..3] { + "u8x" => "0xFF", + "u16" => "0xFF_FF", + "u32" => "0xFF_FF_FF_FF", + "u64" => "0xFF_FF_FF_FF_FF_FF_FF_FF", + _ => panic!("No TRUE for type {}", t), + } +} + +fn ff_val(t: &str) -> &'static str { + match &t[..3] { + "u8x" => "0xFF", + "u16" => "0xFF_FF", + "u32" => "0xFF_FF_FF_FF", + "u64" => "0xFF_FF_FF_FF_FF_FF_FF_FF", + "i8x" => "0xFF", + "i16" => "0xFF_FF", + "i32" => "0xFF_FF_FF_FF", + "i64" => "0xFF_FF_FF_FF_FF_FF_FF_FF", + _ => panic!("No TRUE for type {}", t), + } +} + +fn false_val(_t: &str) -> &'static str { + "0" +} + +fn bits(t: &str) -> &'static str { + match &t[..3] { + "u8x" => "8", + "u16" => "16", + "u32" => "32", + "u64" => "64", + "i8x" => "8", + "i16" => "16", + "i32" => "32", + "i64" => "64", + "p8x" => "8", + "p16" => "16", + "p64" => "64", + _ => panic!("Unknown bits for type {}", t), + } +} + +fn bits_minus_one(t: &str) -> &'static str { + match &t[..3] { + "u8x" => "7", + "u16" => "15", + "u32" => "31", + "u64" => "63", + "i8x" => "7", + "i16" => "15", + "i32" => "31", + "i64" => "63", + "p8x" => "7", + "p16" => "15", + "p64" => "63", + _ => panic!("Unknown bits for type {}", t), + } +} + +fn half_bits(t: &str) -> &'static str { + match &t[..3] { + "u8x" => "4", + "u16" => "8", + "u32" => "16", + "u64" => "32", + "i8x" => "4", + "i16" => "8", + "i32" => "16", + "i64" => "32", + "p8x" => "4", + "p16" => "8", + "p64" => "32", + _ => panic!("Unknown bits for type {}", t), + } +} + +fn type_len_str(t: &str) -> &'static str { + match t { + "int8x8_t" => "8", + "int8x16_t" => "16", + "int16x4_t" => "4", + "int16x8_t" => "8", + "int32x2_t" => "2", + "int32x4_t" => "4", + "int64x1_t" => "1", + "int64x2_t" => "2", + "uint8x8_t" => "8", + "uint8x16_t" => "16", + "uint16x4_t" => "4", + "uint16x8_t" => "8", + "uint32x2_t" => "2", + "uint32x4_t" => "4", + "uint64x1_t" => "1", + "uint64x2_t" => "2", + "float16x4_t" => "4", + "float16x8_t" => "8", + "float32x2_t" => "2", + "float32x4_t" => "4", + "float64x1_t" => "1", + "float64x2_t" => "2", + "poly8x8_t" => "8", + "poly8x16_t" => "16", + "poly16x4_t" => "4", + "poly16x8_t" => "8", + "poly64x1_t" => "1", + "poly64x2_t" => "2", + _ => panic!("unknown type: {}", t), + } +} + +fn type_len_minus_one_str(t: &str) -> &'static str { + match t { + "int8x8_t" => "7", + "int8x16_t" => "15", + "int16x4_t" => "3", + "int16x8_t" => "7", + "int32x2_t" => "1", + "int32x4_t" => "3", + "int64x1_t" => "0", + "int64x2_t" => "1", + "uint8x8_t" => "7", + "uint8x16_t" => "15", + "uint16x4_t" => "3", + "uint16x8_t" => "7", + "uint32x2_t" => "1", + "uint32x4_t" => "3", + "uint64x1_t" => "0", + "uint64x2_t" => "1", + "float16x4_t" => "3", + "float16x8_t" => "7", + "float32x2_t" => "1", + "float32x4_t" => "3", + "float64x1_t" => "0", + "float64x2_t" => "1", + "poly8x8_t" => "7", + "poly8x16_t" => "15", + "poly16x4_t" => "3", + "poly16x8_t" => "7", + "poly64x1_t" => "0", + "poly64x2_t" => "1", + _ => panic!("unknown type: {}", t), + } +} + +fn type_half_len_str(t: &str) -> &'static str { + match t { + "int8x8_t" => "4", + "int8x16_t" => "8", + "int16x4_t" => "2", + "int16x8_t" => "4", + "int32x2_t" => "1", + "int32x4_t" => "2", + "int64x1_t" => "0", + "int64x2_t" => "1", + "uint8x8_t" => "4", + "uint8x16_t" => "8", + "uint16x4_t" => "2", + "uint16x8_t" => "4", + "uint32x2_t" => "1", + "uint32x4_t" => "2", + "uint64x1_t" => "0", + "uint64x2_t" => "1", + "float16x4_t" => "2", + "float16x8_t" => "4", + "float32x2_t" => "1", + "float32x4_t" => "2", + "float64x1_t" => "0", + "float64x2_t" => "1", + "poly8x8_t" => "4", + "poly8x16_t" => "8", + "poly16x4_t" => "2", + "poly16x8_t" => "4", + "poly64x1_t" => "0", + "poly64x2_t" => "1", + _ => panic!("unknown type: {}", t), + } +} + +fn map_val<'v>(t: &str, v: &'v str) -> &'v str { + match v { + "FALSE" => false_val(t), + "TRUE" => true_val(t), + "MAX" => max_val(t), + "MIN" => min_val(t), + "FF" => ff_val(t), + "BITS" => bits(t), + "BITS_M1" => bits_minus_one(t), + "HFBITS" => half_bits(t), + "LEN" => type_len_str(t), + "LEN_M1" => type_len_minus_one_str(t), + "HFLEN" => type_half_len_str(t), + o => o, + } +} + +fn type_to_ext(t: &str, v: bool, r: bool, pi8: bool) -> String { + if !t.contains('x') { + return t.replace("u", "i"); + } + let native = type_to_native_type(t); + let sub_ext = match type_sub_len(t) { + 1 => String::new(), + _ if v => format!( + ".p0v{}{}", + &type_len(&type_to_sub_type(t)).to_string(), + native + ), + _ if pi8 => format!(".p0i8"), + _ => format!(".p0{}", native), + }; + let sub_type = match &native[0..1] { + "i" | "f" => native, + "u" => native.replace("u", "i"), + _ => panic!("unknown type: {}", t), + }; + let ext = format!( + "v{}{}{}", + &type_len(&type_to_sub_type(t)).to_string(), + sub_type, + sub_ext + ); + if r { + let ss: Vec<_> = ext.split('.').collect(); + if ss.len() != 2 { + ext + } else { + format!("{}.{}", ss[1], ss[0]) + } + } else { + ext + } +} + +fn ext(s: &str, in_t: &[&str; 3], out_t: &str) -> String { + s.replace("_EXT_", &type_to_ext(in_t[0], false, false, false)) + .replace("_EXT2_", &type_to_ext(out_t, false, false, false)) + .replace("_EXT3_", &type_to_ext(in_t[1], false, false, false)) + .replace("_EXT4_", &type_to_ext(in_t[2], false, false, false)) + .replace("_EXTr3_", &type_to_ext(in_t[1], false, true, false)) + .replace("_EXTv2_", &type_to_ext(out_t, true, false, false)) + .replace("_EXTpi8_", &type_to_ext(in_t[1], false, false, true)) + .replace("_EXTpi82_", &type_to_ext(out_t, false, false, true)) + .replace("_EXTpi8r_", &type_to_ext(in_t[1], false, true, true)) +} + +fn is_vldx(name: &str) -> bool { + let s: Vec<_> = name.split('_').collect(); + &name[0..3] == "vld" + && name[3..4].parse::().unwrap() > 1 + && (s.last().unwrap().starts_with("s") || s.last().unwrap().starts_with("f")) +} + +fn is_vstx(name: &str) -> bool { + let s: Vec<_> = name.split('_').collect(); + s.len() == 2 + && &name[0..3] == "vst" + && name[3..4].parse::().unwrap() > 1 + && (s[1].starts_with("s") || s[1].starts_with("f")) +} + +fn create_doc_string(comment_string: &str, fn_name: &str) -> String { + format!( + r#"{} +/// +/// [Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/{})"#, + comment_string, fn_name + ) +} + +#[allow(clippy::too_many_arguments)] +fn gen_aarch64( + current_comment: &str, + current_fn: &Option, + current_name: &str, + current_aarch64: &Option, + link_aarch64: &Option, + const_aarch64: &Option, + constn: &Option, + in_t: &[&str; 3], + out_t: &str, + current_tests: &[( + Vec, + Vec, + Vec, + Option, + Vec, + )], + suffix: Suffix, + para_num: i32, + target: TargetFeature, + fixed: &Vec, + multi_fn: &Vec, + fn_type: Fntype, +) -> (String, String) { + let name = match suffix { + Normal => format!("{}{}", current_name, type_to_suffix(in_t[1])), + NoQ => format!("{}{}", current_name, type_to_noq_suffix(in_t[1])), + Double => format!( + "{}{}", + current_name, + type_to_double_suffixes(out_t, in_t[1]) + ), + NoQDouble => format!( + "{}{}", + current_name, + type_to_noq_double_suffixes(out_t, in_t[1]) + ), + NSuffix => format!("{}{}", current_name, type_to_n_suffix(in_t[1])), + DoubleN => format!( + "{}{}", + current_name, + type_to_double_n_suffixes(out_t, in_t[1]) + ), + NoQNSuffix => format!("{}{}", current_name, type_to_noq_n_suffix(in_t[1])), + OutSuffix => format!("{}{}", current_name, type_to_suffix(out_t)), + OutNSuffix => format!("{}{}", current_name, type_to_n_suffix(out_t)), + OutNox => format!( + "{}{}", + current_name, + type_to_suffix(&type_to_sub_type(out_t)) + ), + In1Nox => format!( + "{}{}", + current_name, + type_to_suffix(&type_to_sub_type(in_t[1])) + ), + OutDupNox => format!( + "{}{}", + current_name, + type_to_dup_suffix(&type_to_sub_type(out_t)) + ), + OutLaneNox => format!( + "{}{}", + current_name, + type_to_lane_suffix(&type_to_sub_type(out_t)) + ), + In1LaneNox => format!( + "{}{}", + current_name, + type_to_lane_suffix(&type_to_sub_type(in_t[1])) + ), + Lane => format!( + "{}{}", + current_name, + type_to_lane_suffixes(out_t, in_t[1], false) + ), + In2 => format!("{}{}", current_name, type_to_suffix(in_t[2])), + In2Lane => format!( + "{}{}", + current_name, + type_to_lane_suffixes(out_t, in_t[2], false) + ), + OutLane => format!( + "{}{}", + current_name, + type_to_lane_suffixes(out_t, in_t[2], true) + ), + Rot => type_to_rot_suffix(current_name, type_to_suffix(out_t)), + RotLane => type_to_rot_suffix(current_name, &type_to_lane_suffixes(out_t, in_t[2], false)), + }; + let current_target = match target { + Default => "neon", + ArmV7 => "neon", + Vfp4 => "neon", + FPArmV8 => "neon", + AES => "neon,aes", + FCMA => "neon,fcma", + Dotprod => "neon,dotprod", + I8MM => "neon,i8mm", + SHA3 => "neon,sha3", + RDM => "rdm", + SM4 => "neon,sm4", + FTTS => "neon,frintts", + }; + let current_fn = if let Some(current_fn) = current_fn.clone() { + if link_aarch64.is_some() { + panic!("[{}] Can't specify link and fn at the same time.", name) + } + current_fn + } else if link_aarch64.is_some() { + format!("{}_", name) + } else { + if multi_fn.is_empty() { + panic!( + "[{}] Either (multi) fn or link-aarch have to be specified.", + name + ) + } + String::new() + }; + let current_aarch64 = current_aarch64.clone().unwrap(); + let mut link_t: Vec = vec![ + in_t[0].to_string(), + in_t[1].to_string(), + in_t[2].to_string(), + out_t.to_string(), + ]; + let mut ext_c = String::new(); + if let Some(mut link_aarch64) = link_aarch64.clone() { + if link_aarch64.contains(":") { + let links: Vec<_> = link_aarch64.split(':').map(|v| v.to_string()).collect(); + assert_eq!(links.len(), 5); + link_aarch64 = links[0].to_string(); + link_t = vec![ + links[1].clone(), + links[2].clone(), + links[3].clone(), + links[4].clone(), + ]; + } + let link_aarch64 = if link_aarch64.starts_with("llvm") { + ext(&link_aarch64, in_t, out_t) + } else { + let mut link = String::from("llvm.aarch64.neon."); + link.push_str(&link_aarch64); + ext(&link, in_t, out_t) + }; + let (ext_inputs, ext_output) = { + if const_aarch64.is_some() { + if !matches!(fn_type, Fntype::Normal) { + let ptr_type = match fn_type { + Fntype::Load => "*const i8", + Fntype::Store => "*mut i8", + _ => panic!("unsupported fn type"), + }; + let sub = type_to_sub_type(in_t[1]); + ( + match type_sub_len(in_t[1]) { + 1 => format!("a: {}, n: i64, ptr: {}", sub, ptr_type), + 2 => format!("a: {}, b: {}, n: i64, ptr: {}", sub, sub, ptr_type), + 3 => format!( + "a: {}, b: {}, c: {}, n: i64, ptr: {}", + sub, sub, sub, ptr_type + ), + 4 => format!( + "a: {}, b: {}, c: {}, d: {}, n: i64, ptr: {}", + sub, sub, sub, sub, ptr_type + ), + _ => panic!("unsupported type: {}", in_t[1]), + }, + if out_t != "void" { + format!(" -> {}", out_t) + } else { + String::new() + }, + ) + } else { + ( + match para_num { + 1 => format!("a: {}, n: i32", in_t[0]), + 2 => format!("a: {}, b: {}, n: i32", in_t[0], in_t[1]), + 3 => format!("a: {}, b: {}, c: {}, n: i32", in_t[0], in_t[1], in_t[2]), + _ => unimplemented!("unknown para_num"), + }, + format!(" -> {}", out_t), + ) + } + } else if matches!(fn_type, Fntype::Store) { + let sub = type_to_sub_type(in_t[1]); + let ptr_type = if is_vstx(&name) { + "i8".to_string() + } else { + type_to_native_type(in_t[1]) + }; + let subs = match type_sub_len(in_t[1]) { + 1 => format!("a: {}", sub), + 2 => format!("a: {}, b: {}", sub, sub), + 3 => format!("a: {}, b: {}, c: {}", sub, sub, sub), + 4 => format!("a: {}, b: {}, c: {}, d: {}", sub, sub, sub, sub), + _ => panic!("unsupported type: {}", in_t[1]), + }; + (format!("{}, ptr: *mut {}", subs, ptr_type), String::new()) + } else if is_vldx(&name) { + let ptr_type = if name.contains("dup") { + type_to_native_type(out_t) + } else { + type_to_sub_type(out_t) + }; + ( + format!("ptr: *const {}", ptr_type), + format!(" -> {}", out_t), + ) + } else { + ( + match para_num { + 1 => format!("a: {}", link_t[0]), + 2 => format!("a: {}, b: {}", link_t[0], link_t[1]), + 3 => format!("a: {}, b: {}, c: {}", link_t[0], link_t[1], link_t[2]), + _ => unimplemented!("unknown para_num"), + }, + format!(" -> {}", link_t[3]), + ) + } + }; + ext_c = format!( + r#"#[allow(improper_ctypes)] + extern "unadjusted" {{ + #[cfg_attr(target_arch = "aarch64", link_name = "{}")] + fn {}({}){}; + }} + "#, + link_aarch64, current_fn, ext_inputs, ext_output, + ); + }; + let const_declare = if let Some(constn) = constn { + if constn.contains(":") { + let constns: Vec<_> = constn.split(':').map(|v| v.to_string()).collect(); + assert_eq!(constns.len(), 2); + format!(r#""#, constns[0], constns[1]) + } else { + format!(r#""#, constn) + } + } else { + String::new() + }; + let multi_calls = if !multi_fn.is_empty() { + let mut calls = String::new(); + for i in 0..multi_fn.len() { + if i > 0 { + calls.push_str("\n "); + } + calls.push_str(&get_call( + &multi_fn[i], + current_name, + &const_declare, + in_t, + out_t, + fixed, + None, + true, + )); + } + calls + } else { + String::new() + }; + let const_assert = if let Some(constn) = constn { + if constn.contains(":") { + let constns: Vec<_> = constn.split(':').map(|v| v.to_string()).collect(); + let const_test = current_tests[0].3.as_ref().unwrap(); + let const_tests: Vec<_> = const_test.split(':').map(|v| v.to_string()).collect(); + assert_eq!(constns.len(), 2); + assert_eq!(const_tests.len(), 2); + format!( + r#", {} = {}, {} = {}"#, + constns[0], + map_val(in_t[1], &const_tests[0]), + constns[1], + map_val(in_t[1], &const_tests[1]), + ) + } else { + format!( + r#", {} = {}"#, + constn, + map_val(in_t[1], current_tests[0].3.as_ref().unwrap()) + ) + } + } else { + String::new() + }; + let const_legacy = if let Some(constn) = constn { + if constn.contains(":") { + format!( + "\n#[rustc_legacy_const_generics({}, {})]", + para_num - 1, + para_num + 1 + ) + } else { + format!("\n#[rustc_legacy_const_generics({})]", para_num) + } + } else { + String::new() + }; + let fn_decl = { + let fn_output = if out_t == "void" { + String::new() + } else { + format!("-> {} ", out_t) + }; + let fn_inputs = match para_num { + 1 => format!("(a: {})", in_t[0]), + 2 => format!("(a: {}, b: {})", in_t[0], in_t[1]), + 3 => format!("(a: {}, b: {}, c: {})", in_t[0], in_t[1], in_t[2]), + _ => panic!("unsupported parameter number"), + }; + format!( + "pub unsafe fn {}{}{} {}", + name, const_declare, fn_inputs, fn_output + ) + }; + let call_params = { + if let (Some(const_aarch64), Some(_)) = (const_aarch64, link_aarch64) { + if !matches!(fn_type, Fntype::Normal) { + let subs = match type_sub_len(in_t[1]) { + 1 => "b", + 2 => "b.0, b.1", + 3 => "b.0, b.1, b.2", + 4 => "b.0, b.1, b.2, b.3", + _ => panic!("unsupported type: {}", in_t[1]), + }; + format!( + r#"{} + {}{}({}, {} as i64, a as _)"#, + multi_calls, + ext_c, + current_fn, + subs, + constn.as_deref().unwrap() + ) + } else { + match para_num { + 1 => format!( + r#"{} + {}{}(a, {})"#, + multi_calls, ext_c, current_fn, const_aarch64 + ), + 2 => format!( + r#"{} + {}{}(a, b, {})"#, + multi_calls, ext_c, current_fn, const_aarch64 + ), + _ => String::new(), + } + } + } else if link_aarch64.is_some() && matches!(fn_type, Fntype::Store) { + let cast = if is_vstx(&name) { " as _" } else { "" }; + match type_sub_len(in_t[1]) { + 1 => format!(r#"{}{}(b, a{})"#, ext_c, current_fn, cast), + 2 => format!(r#"{}{}(b.0, b.1, a{})"#, ext_c, current_fn, cast), + 3 => format!(r#"{}{}(b.0, b.1, b.2, a{})"#, ext_c, current_fn, cast), + 4 => format!(r#"{}{}(b.0, b.1, b.2, b.3, a{})"#, ext_c, current_fn, cast), + _ => panic!("unsupported type: {}", in_t[1]), + } + } else if link_aarch64.is_some() && is_vldx(&name) { + format!(r#"{}{}(a as _)"#, ext_c, current_fn,) + } else { + let trans: [&str; 2] = if link_t[3] != out_t { + ["transmute(", ")"] + } else { + ["", ""] + }; + match (multi_calls.len(), para_num, fixed.len()) { + (0, 1, 0) => format!(r#"{}{}{}(a){}"#, ext_c, trans[0], current_fn, trans[1]), + (0, 1, _) => { + let fixed: Vec = + fixed.iter().take(type_len(in_t[0])).cloned().collect(); + format!( + r#"let b{}; + {}{}{}(a, transmute(b)){}"#, + values(in_t[0], &fixed), + ext_c, + trans[0], + current_fn, + trans[1], + ) + } + (0, 2, _) => format!(r#"{}{}{}(a, b){}"#, ext_c, trans[0], current_fn, trans[1],), + (0, 3, _) => format!(r#"{}{}(a, b, c)"#, ext_c, current_fn,), + (_, 1, _) => format!(r#"{}{}"#, ext_c, multi_calls,), + (_, 2, _) => format!(r#"{}{}"#, ext_c, multi_calls,), + (_, 3, _) => format!(r#"{}{}"#, ext_c, multi_calls,), + (_, _, _) => String::new(), + } + } + }; + let stable = match target { + Default | ArmV7 | Vfp4 | FPArmV8 | AES => { + String::from("\n#[stable(feature = \"neon_intrinsics\", since = \"1.59.0\")]") + } + RDM => String::from("\n#[stable(feature = \"rdm_intrinsics\", since = \"1.62.0\")]"), + _ => String::new(), + }; + let function_doc = create_doc_string(current_comment, &name); + let function = format!( + r#" +{} +#[inline] +#[target_feature(enable = "{}")] +#[cfg_attr(test, assert_instr({}{}))]{}{} +{}{{ + {} +}} +"#, + function_doc, + current_target, + current_aarch64, + const_assert, + const_legacy, + stable, + fn_decl, + call_params + ); + let test_target = match target { + I8MM => "neon,i8mm", + SM4 => "neon,sm4", + SHA3 => "neon,sha3", + FTTS => "neon,frintts", + _ => "neon", + }; + let test = match fn_type { + Fntype::Normal => gen_test( + &name, + in_t, + &out_t, + current_tests, + [type_len(in_t[0]), type_len(in_t[1]), type_len(in_t[2])], + type_len(out_t), + para_num, + test_target, + ), + Fntype::Load => gen_load_test(&name, in_t, &out_t, current_tests, type_len(out_t)), + Fntype::Store => gen_store_test(&name, in_t, &out_t, current_tests, type_len(in_t[1])), + }; + (function, test) +} + +fn gen_load_test( + name: &str, + in_t: &[&str; 3], + out_t: &str, + current_tests: &[( + Vec, + Vec, + Vec, + Option, + Vec, + )], + type_len: usize, +) -> String { + let mut test = format!( + r#" + #[simd_test(enable = "neon")] + unsafe fn test_{}() {{"#, + name, + ); + for (a, b, _, n, e) in current_tests { + let a: Vec = a.iter().take(type_len + 1).cloned().collect(); + let e: Vec = e.iter().take(type_len).cloned().collect(); + let has_b = b.len() > 0; + let has_n = n.is_some(); + let mut input = String::from("["); + for i in 0..type_len + 1 { + if i != 0 { + input.push_str(", "); + } + input.push_str(&a[i]) + } + input.push_str("]"); + let output = |v: &Vec| { + let mut output = String::from("["); + for i in 0..type_sub_len(out_t) { + if i != 0 { + output.push_str(", "); + } + let sub_len = type_len / type_sub_len(out_t); + if type_to_global_type(out_t) != "f64" { + let mut sub_output = format!("{}::new(", type_to_global_type(out_t)); + for j in 0..sub_len { + if j != 0 { + sub_output.push_str(", "); + } + sub_output.push_str(&v[i * sub_len + j]); + } + sub_output.push_str(")"); + output.push_str(&sub_output); + } else { + output.push_str(&v[i]); + } + } + output.push_str("]"); + output + }; + let input_b = if has_b { + let b: Vec = b.iter().take(type_len).cloned().collect(); + format!( + r#" + let b: [{}; {}] = {};"#, + type_to_global_type(in_t[1]), + type_sub_len(in_t[1]), + output(&b), + ) + } else { + String::new() + }; + let t = format!( + r#" + let a: [{}; {}] = {};{} + let e: [{}; {}] = {}; + let r: [{}; {}] = transmute({}{}(a[1..].as_ptr(){})); + assert_eq!(r, e); +"#, + type_to_native_type(out_t), + type_len + 1, + input, + input_b, + type_to_global_type(out_t), + type_sub_len(out_t), + output(&e), + type_to_global_type(out_t), + type_sub_len(out_t), + name, + if has_n { + format!("::<{}>", n.as_deref().unwrap()) + } else { + String::new() + }, + if has_b { ", transmute(b)" } else { "" }, + ); + test.push_str(&t); + } + test.push_str(" }\n"); + test +} + +fn gen_store_test( + name: &str, + in_t: &[&str; 3], + _out_t: &str, + current_tests: &[( + Vec, + Vec, + Vec, + Option, + Vec, + )], + type_len: usize, +) -> String { + let mut test = format!( + r#" + #[simd_test(enable = "neon")] + unsafe fn test_{}() {{"#, + name, + ); + for (a, _, _, constn, e) in current_tests { + let a: Vec = a.iter().take(type_len + 1).cloned().collect(); + let e: Vec = e.iter().take(type_len).cloned().collect(); + let mut input = String::from("["); + for i in 0..type_len + 1 { + if i != 0 { + input.push_str(", "); + } + input.push_str(&a[i]) + } + input.push_str("]"); + let mut output = String::from("["); + for i in 0..type_len { + if i != 0 { + output.push_str(", "); + } + output.push_str(&e[i]) + } + output.push_str("]"); + let const_n = constn + .as_deref() + .map_or(String::new(), |n| format!("::<{}>", n.to_string())); + let t = format!( + r#" + let a: [{}; {}] = {}; + let e: [{}; {}] = {}; + let mut r: [{}; {}] = [0{}; {}]; + {}{}(r.as_mut_ptr(), core::ptr::read_unaligned(a[1..].as_ptr() as _)); + assert_eq!(r, e); +"#, + type_to_native_type(in_t[1]), + type_len + 1, + input, + type_to_native_type(in_t[1]), + type_len, + output, + type_to_native_type(in_t[1]), + type_len, + type_to_native_type(in_t[1]), + type_len, + name, + const_n, + ); + test.push_str(&t); + } + test.push_str(" }\n"); + test +} + +fn gen_test( + name: &str, + in_t: &[&str; 3], + out_t: &str, + current_tests: &[( + Vec, + Vec, + Vec, + Option, + Vec, + )], + len_in: [usize; 3], + len_out: usize, + para_num: i32, + target: &str, +) -> String { + let mut test = format!( + r#" + #[simd_test(enable = "{}")] + unsafe fn test_{}() {{"#, + target, name, + ); + for (a, b, c, n, e) in current_tests { + let a: Vec = a.iter().take(len_in[0]).cloned().collect(); + let b: Vec = b.iter().take(len_in[1]).cloned().collect(); + let c: Vec = c.iter().take(len_in[2]).cloned().collect(); + let e: Vec = e.iter().take(len_out).cloned().collect(); + let const_value = if let Some(constn) = n { + if constn.contains(":") { + let constns: Vec<_> = constn.split(':').map(|v| v.to_string()).collect(); + format!( + r#"::<{}, {}>"#, + map_val(in_t[1], &constns[0]), + map_val(in_t[1], &constns[1]) + ) + } else { + format!(r#"::<{}>"#, map_val(in_t[1], constn)) + } + } else { + String::new() + }; + let r_type = match type_sub_len(out_t) { + 1 => type_to_global_type(out_t).to_string(), + _ => format!("[{}; {}]", type_to_native_type(out_t), type_len(out_t)), + }; + let t = { + match para_num { + 1 => { + format!( + r#" + let a{}; + let e{}; + let r: {} = transmute({}{}(transmute(a))); + assert_eq!(r, e); +"#, + values(in_t[0], &a), + values(out_t, &e), + r_type, + name, + const_value + ) + } + 2 => { + format!( + r#" + let a{}; + let b{}; + let e{}; + let r: {} = transmute({}{}(transmute(a), transmute(b))); + assert_eq!(r, e); +"#, + values(in_t[0], &a), + values(in_t[1], &b), + values(out_t, &e), + r_type, + name, + const_value + ) + } + 3 => { + format!( + r#" + let a{}; + let b{}; + let c{}; + let e{}; + let r: {} = transmute({}{}(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); +"#, + values(in_t[0], &a), + values(in_t[1], &b), + values(in_t[2], &c), + values(out_t, &e), + r_type, + name, + const_value + ) + } + _ => { + panic!("no support para_num:{}", para_num.to_string()) + } + } + }; + + test.push_str(&t); + } + test.push_str(" }\n"); + test +} + +#[allow(clippy::too_many_arguments)] +fn gen_arm( + current_comment: &str, + current_fn: &Option, + current_name: &str, + current_arm: &str, + link_arm: &Option, + current_aarch64: &Option, + link_aarch64: &Option, + const_arm: &Option, + const_aarch64: &Option, + constn: &Option, + in_t: &[&str; 3], + out_t: &str, + current_tests: &[( + Vec, + Vec, + Vec, + Option, + Vec, + )], + suffix: Suffix, + para_num: i32, + target: TargetFeature, + fixed: &Vec, + multi_fn: &Vec, + fn_type: Fntype, + separate: bool, +) -> (String, String) { + let name = match suffix { + Normal => format!("{}{}", current_name, type_to_suffix(in_t[1])), + NoQ => format!("{}{}", current_name, type_to_noq_suffix(in_t[1])), + Double => format!( + "{}{}", + current_name, + type_to_double_suffixes(out_t, in_t[1]) + ), + NoQDouble => format!( + "{}{}", + current_name, + type_to_noq_double_suffixes(out_t, in_t[1]) + ), + NSuffix => format!("{}{}", current_name, type_to_n_suffix(in_t[1])), + DoubleN => format!( + "{}{}", + current_name, + type_to_double_n_suffixes(out_t, in_t[1]) + ), + NoQNSuffix => format!("{}{}", current_name, type_to_noq_n_suffix(in_t[1])), + OutSuffix => format!("{}{}", current_name, type_to_suffix(out_t)), + OutNSuffix => format!("{}{}", current_name, type_to_n_suffix(out_t)), + OutNox => format!( + "{}{}", + current_name, + type_to_suffix(&type_to_sub_type(out_t)) + ), + In1Nox => format!( + "{}{}", + current_name, + type_to_suffix(&type_to_sub_type(in_t[1])) + ), + OutDupNox => format!( + "{}{}", + current_name, + type_to_dup_suffix(&type_to_sub_type(out_t)) + ), + OutLaneNox => format!( + "{}{}", + current_name, + type_to_lane_suffix(&type_to_sub_type(out_t)) + ), + In1LaneNox => format!( + "{}{}", + current_name, + type_to_lane_suffix(&type_to_sub_type(in_t[1])) + ), + Lane => format!( + "{}{}", + current_name, + type_to_lane_suffixes(out_t, in_t[1], false) + ), + In2 => format!("{}{}", current_name, type_to_suffix(in_t[2])), + In2Lane => format!( + "{}{}", + current_name, + type_to_lane_suffixes(out_t, in_t[2], false) + ), + OutLane => format!( + "{}{}", + current_name, + type_to_lane_suffixes(out_t, in_t[2], true) + ), + Rot => type_to_rot_suffix(current_name, type_to_suffix(out_t)), + RotLane => type_to_rot_suffix(current_name, &type_to_lane_suffixes(out_t, in_t[2], false)), + }; + let current_aarch64 = current_aarch64 + .clone() + .unwrap_or_else(|| current_arm.to_string()); + let current_target_aarch64 = match target { + Default => "neon", + ArmV7 => "neon", + Vfp4 => "neon", + FPArmV8 => "neon", + AES => "neon,aes", + FCMA => "neon,fcma", + Dotprod => "neon,dotprod", + I8MM => "neon,i8mm", + SHA3 => "neon,sha3", + RDM => "rdm", + SM4 => "neon,sm4", + FTTS => "neon,frintts", + }; + let current_target_arm = match target { + Default => "v7", + ArmV7 => "v7", + Vfp4 => "vfp4", + FPArmV8 => "fp-armv8,v8", + AES => "aes,v8", + FCMA => "v8", // v8.3a + Dotprod => "v8", // v8.2a + I8MM => "v8,i8mm", + RDM => unreachable!(), + SM4 => unreachable!(), + SHA3 => unreachable!(), + FTTS => unreachable!(), + }; + let current_fn = if let Some(current_fn) = current_fn.clone() { + if link_aarch64.is_some() || link_arm.is_some() { + panic!( + "[{}] Can't specify link and function at the same time. {} / {:?} / {:?}", + name, current_fn, link_aarch64, link_arm + ) + } + current_fn + } else if link_aarch64.is_some() || link_arm.is_some() { + format!("{}_", name) + } else { + if multi_fn.is_empty() { + panic!( + "[{}] Either fn or link-arm and link-aarch have to be specified.", + name + ) + } + String::new() + }; + let mut ext_c = String::new(); + let mut ext_c_arm = if multi_fn.is_empty() || link_arm.is_none() { + String::new() + } else { + String::from( + r#" + "#, + ) + }; + let mut ext_c_aarch64 = if multi_fn.is_empty() || link_aarch64.is_none() { + String::new() + } else { + String::from( + r#" + "#, + ) + }; + let mut link_arm_t: Vec = vec![ + in_t[0].to_string(), + in_t[1].to_string(), + in_t[2].to_string(), + out_t.to_string(), + ]; + let mut link_aarch64_t: Vec = vec![ + in_t[0].to_string(), + in_t[1].to_string(), + in_t[2].to_string(), + out_t.to_string(), + ]; + if let (Some(mut link_arm), Some(mut link_aarch64)) = (link_arm.clone(), link_aarch64.clone()) { + if link_arm.contains(":") { + let links: Vec<_> = link_arm.split(':').map(|v| v.to_string()).collect(); + assert_eq!(links.len(), 5); + link_arm = links[0].to_string(); + link_arm_t = vec![ + links[1].clone(), + links[2].clone(), + links[3].clone(), + links[4].clone(), + ]; + } + if link_aarch64.contains(":") { + let links: Vec<_> = link_aarch64.split(':').map(|v| v.to_string()).collect(); + assert_eq!(links.len(), 5); + link_aarch64 = links[0].to_string(); + link_aarch64_t = vec![ + links[1].clone(), + links[2].clone(), + links[3].clone(), + links[4].clone(), + ]; + } + let link_arm = if link_arm.starts_with("llvm") { + ext(&link_arm, in_t, out_t) + } else { + let mut link = String::from("llvm.arm.neon."); + link.push_str(&link_arm); + ext(&link, in_t, out_t) + }; + let link_aarch64 = if link_aarch64.starts_with("llvm") { + ext(&link_aarch64, in_t, out_t) + } else { + let mut link = String::from("llvm.aarch64.neon."); + link.push_str(&link_aarch64); + ext(&link, in_t, out_t) + }; + if out_t == link_arm_t[3] && out_t == link_aarch64_t[3] { + ext_c = format!( + r#"#[allow(improper_ctypes)] + extern "unadjusted" {{ + #[cfg_attr(target_arch = "arm", link_name = "{}")] + #[cfg_attr(target_arch = "aarch64", link_name = "{}")] + fn {}({}) -> {}; + }} +"#, + link_arm, + link_aarch64, + current_fn, + match para_num { + 1 => format!("a: {}", in_t[0]), + 2 => format!("a: {}, b: {}", in_t[0], in_t[1]), + 3 => format!("a: {}, b: {}, c: {}", in_t[0], in_t[1], in_t[2]), + _ => unimplemented!("unknown para_num"), + }, + out_t + ); + }; + let (arm_ext_inputs, arm_ext_output) = { + if let Some(const_arm) = const_arm { + if !matches!(fn_type, Fntype::Normal) { + let ptr_type = match fn_type { + Fntype::Load => "*const i8", + Fntype::Store => "*mut i8", + _ => panic!("unsupported fn type"), + }; + let sub_type = type_to_sub_type(in_t[1]); + let inputs = match type_sub_len(in_t[1]) { + 1 => format!("a: {}", sub_type), + 2 => format!("a: {}, b: {}", sub_type, sub_type,), + 3 => format!("a: {}, b: {}, c: {}", sub_type, sub_type, sub_type,), + 4 => format!( + "a: {}, b: {}, c: {}, d: {}", + sub_type, sub_type, sub_type, sub_type, + ), + _ => panic!("unknown type: {}", in_t[1]), + }; + let out = if out_t == "void" { + String::new() + } else { + format!(" -> {}", out_t) + }; + ( + format!("ptr: {}, {}, n: i32, size: i32", ptr_type, inputs), + out, + ) + } else { + let (_, const_type) = if const_arm.contains(":") { + let consts: Vec<_> = + const_arm.split(':').map(|v| v.trim().to_string()).collect(); + (consts[0].clone(), consts[1].clone()) + } else { + ( + const_arm.to_string(), + in_t[para_num as usize - 1].to_string(), + ) + }; + ( + match para_num { + 1 => format!("a: {}, n: {}", in_t[0], const_type), + 2 => format!("a: {}, b: {}, n: {}", in_t[0], in_t[1], const_type), + 3 => format!( + "a: {}, b: {}, c: {}, n: {}", + in_t[0], in_t[1], in_t[2], const_type + ), + _ => unimplemented!("unknown para_num"), + }, + format!(" -> {}", out_t), + ) + } + } else if out_t != link_arm_t[3] { + ( + match para_num { + 1 => format!("a: {}", link_arm_t[0]), + 2 => format!("a: {}, b: {}", link_arm_t[0], link_arm_t[1]), + 3 => format!( + "a: {}, b: {}, c: {}", + link_arm_t[0], link_arm_t[1], link_arm_t[2] + ), + _ => unimplemented!("unknown para_num"), + }, + format!(" -> {}", link_arm_t[3]), + ) + } else if matches!(fn_type, Fntype::Store) { + let sub_type = type_to_sub_type(in_t[1]); + let inputs = match type_sub_len(in_t[1]) { + 1 => format!("a: {}", sub_type), + 2 => format!("a: {}, b: {}", sub_type, sub_type,), + 3 => format!("a: {}, b: {}, c: {}", sub_type, sub_type, sub_type,), + 4 => format!( + "a: {}, b: {}, c: {}, d: {}", + sub_type, sub_type, sub_type, sub_type, + ), + _ => panic!("unknown type: {}", in_t[1]), + }; + let (ptr_type, size) = if is_vstx(&name) { + ("i8".to_string(), ", size: i32") + } else { + (type_to_native_type(in_t[1]), "") + }; + ( + format!("ptr: *mut {}, {}{}", ptr_type, inputs, size), + String::new(), + ) + } else if is_vldx(&name) { + ( + format!("ptr: *const i8, size: i32"), + format!(" -> {}", out_t), + ) + } else { + (String::new(), String::new()) + } + }; + ext_c_arm.push_str(&format!( + r#"#[allow(improper_ctypes)] + extern "unadjusted" {{ + #[cfg_attr(target_arch = "arm", link_name = "{}")] + fn {}({}){}; + }} +"#, + link_arm, current_fn, arm_ext_inputs, arm_ext_output, + )); + let (aarch64_ext_inputs, aarch64_ext_output) = { + if let Some(const_aarch64) = const_aarch64 { + if !matches!(fn_type, Fntype::Normal) { + let ptr_type = match fn_type { + Fntype::Load => "*const i8", + Fntype::Store => "*mut i8", + _ => panic!("unsupported fn type"), + }; + let sub_type = type_to_sub_type(in_t[1]); + let mut inputs = match type_sub_len(in_t[1]) { + 1 => format!("a: {}", sub_type,), + 2 => format!("a: {}, b: {}", sub_type, sub_type,), + 3 => format!("a: {}, b: {}, c: {}", sub_type, sub_type, sub_type,), + 4 => format!( + "a: {}, b: {}, c: {}, d: {}", + sub_type, sub_type, sub_type, sub_type, + ), + _ => panic!("unknown type: {}", in_t[1]), + }; + inputs.push_str(&format!(", n: i64, ptr: {}", ptr_type)); + let out = if out_t == "void" { + String::new() + } else { + format!(" -> {}", out_t) + }; + (inputs, out) + } else if const_aarch64.contains("dup-in_len-N as ttn") { + ( + match para_num { + 1 => format!("a: {}, n: {}", in_t[0], in_t[0]), + 2 => format!("a: {}, b: {}, n: {}", in_t[0], in_t[1], in_t[1]), + 3 => format!( + "a: {}, b: {}, c: {}, n: {}", + in_t[0], in_t[1], in_t[2], in_t[1] + ), + _ => unimplemented!("unknown para_num"), + }, + format!(" -> {}", out_t), + ) + } else { + ( + match para_num { + 1 => format!("a: {}, n: i32", in_t[0]), + 2 => format!("a: {}, b: {}, n: i32", in_t[0], in_t[1]), + 3 => format!("a: {}, b: {}, c: {}, n: i32", in_t[0], in_t[1], in_t[2]), + _ => unimplemented!("unknown para_num"), + }, + format!(" -> {}", out_t), + ) + } + } else if out_t != link_aarch64_t[3] { + ( + match para_num { + 1 => format!("a: {}", link_aarch64_t[0]), + 2 => format!("a: {}, b: {}", link_aarch64_t[0], link_aarch64_t[1]), + 3 => format!( + "a: {}, b: {}, c: {}", + link_aarch64_t[0], link_aarch64_t[1], link_aarch64_t[2] + ), + _ => unimplemented!("unknown para_num"), + }, + format!(" -> {}", link_aarch64_t[3]), + ) + } else if matches!(fn_type, Fntype::Store) { + let sub_type = type_to_sub_type(in_t[1]); + let mut inputs = match type_sub_len(in_t[1]) { + 1 => format!("a: {}", sub_type,), + 2 => format!("a: {}, b: {}", sub_type, sub_type,), + 3 => format!("a: {}, b: {}, c: {}", sub_type, sub_type, sub_type,), + 4 => format!( + "a: {}, b: {}, c: {}, d: {}", + sub_type, sub_type, sub_type, sub_type, + ), + _ => panic!("unknown type: {}", in_t[1]), + }; + let ptr_type = if is_vstx(&name) { + "i8".to_string() + } else { + type_to_native_type(in_t[1]) + }; + inputs.push_str(&format!(", ptr: *mut {}", ptr_type)); + (inputs, String::new()) + } else if is_vldx(&name) { + let ptr_type = if name.contains("dup") { + type_to_native_type(out_t) + } else { + type_to_sub_type(out_t) + }; + ( + format!("ptr: *const {}", ptr_type), + format!(" -> {}", out_t), + ) + } else { + (String::new(), String::new()) + } + }; + ext_c_aarch64.push_str(&format!( + r#"#[allow(improper_ctypes)] + extern "unadjusted" {{ + #[cfg_attr(target_arch = "aarch64", link_name = "{}")] + fn {}({}){}; + }} +"#, + link_aarch64, current_fn, aarch64_ext_inputs, aarch64_ext_output, + )); + }; + let const_declare = if let Some(constn) = constn { + format!(r#""#, constn) + } else { + String::new() + }; + let multi_calls = if !multi_fn.is_empty() { + let mut calls = String::new(); + for i in 0..multi_fn.len() { + if i > 0 { + calls.push_str("\n "); + } + calls.push_str(&get_call( + &multi_fn[i], + current_name, + &const_declare, + in_t, + out_t, + fixed, + None, + false, + )); + } + calls + } else { + String::new() + }; + let const_assert = if let Some(constn) = constn { + format!( + r#", {} = {}"#, + constn, + map_val(in_t[1], current_tests[0].3.as_ref().unwrap()) + ) + } else { + String::new() + }; + let const_legacy = if constn.is_some() { + format!("\n#[rustc_legacy_const_generics({})]", para_num) + } else { + String::new() + }; + let fn_decl = { + let fn_output = if out_t == "void" { + String::new() + } else { + format!("-> {} ", out_t) + }; + let fn_inputs = match para_num { + 1 => format!("(a: {})", in_t[0]), + 2 => format!("(a: {}, b: {})", in_t[0], in_t[1]), + 3 => format!("(a: {}, b: {}, c: {})", in_t[0], in_t[1], in_t[2]), + _ => panic!("unsupported parameter number"), + }; + format!( + "pub unsafe fn {}{}{} {}", + name, const_declare, fn_inputs, fn_output + ) + }; + let function = if separate { + let call_arm = { + let arm_params = if let (Some(const_arm), Some(_)) = (const_arm, link_arm) { + if !matches!(fn_type, Fntype::Normal) { + let subs = match type_sub_len(in_t[1]) { + 1 => "b", + 2 => "b.0, b.1", + 3 => "b.0, b.1, b.2", + 4 => "b.0, b.1, b.2, b.3", + _ => "", + }; + format!( + "{}(a as _, {}, {}, {})", + current_fn, + subs, + constn.as_deref().unwrap(), + type_bits(&type_to_sub_type(in_t[1])) / 8, + ) + } else { + let cnt = if const_arm.contains(':') { + let consts: Vec<_> = + const_arm.split(':').map(|v| v.trim().to_string()).collect(); + consts[0].clone() + } else { + let const_arm = const_arm.replace("ttn", &type_to_native_type(in_t[1])); + let mut cnt = String::from(in_t[1]); + cnt.push_str("("); + for i in 0..type_len(in_t[1]) { + if i != 0 { + cnt.push_str(", "); + } + cnt.push_str(&const_arm); + } + cnt.push_str(")"); + cnt + }; + match para_num { + 1 => format!("{}(a, {})", current_fn, cnt), + 2 => format!("{}(a, b, {})", current_fn, cnt), + _ => String::new(), + } + } + } else if out_t != link_arm_t[3] { + match para_num { + 1 => format!("transmute({}(a))", current_fn,), + 2 => format!("transmute({}(transmute(a), transmute(b)))", current_fn,), + _ => String::new(), + } + } else if matches!(fn_type, Fntype::Store) { + let (cast, size) = if is_vstx(&name) { + ( + " as _", + format!(", {}", type_bits(&type_to_sub_type(in_t[1])) / 8), + ) + } else { + ("", String::new()) + }; + match type_sub_len(in_t[1]) { + 1 => format!("{}(a{}, b{})", current_fn, cast, size), + 2 => format!("{}(a{}, b.0, b.1{})", current_fn, cast, size), + 3 => format!("{}(a{}, b.0, b.1, b.2{})", current_fn, cast, size), + 4 => format!("{}(a{}, b.0, b.1, b.2, b.3{})", current_fn, cast, size), + _ => String::new(), + } + } else if link_arm.is_some() && is_vldx(&name) { + format!( + "{}(a as *const i8, {})", + current_fn, + type_bits(&type_to_sub_type(out_t)) / 8 + ) + } else { + String::new() + }; + format!( + r#"{}{{ + {}{}{} +}}"#, + fn_decl, multi_calls, ext_c_arm, arm_params + ) + }; + let call_aarch64 = { + let aarch64_params = + if let (Some(const_aarch64), Some(_)) = (const_aarch64, link_aarch64) { + if !matches!(fn_type, Fntype::Normal) { + let subs = match type_sub_len(in_t[1]) { + 1 => "b", + 2 => "b.0, b.1", + 3 => "b.0, b.1, b.2", + 4 => "b.0, b.1, b.2, b.3", + _ => "", + }; + format!( + "{}({}, {} as i64, a as _)", + current_fn, + subs, + constn.as_deref().unwrap() + ) + } else if const_aarch64.contains("dup-in_len-N as ttn") { + let const_aarch64 = format!("N as {}", type_to_native_type(in_t[1])); + let mut cnt = String::from(in_t[1]); + cnt.push_str("("); + for i in 0..type_len(in_t[1]) { + if i != 0 { + cnt.push_str(", "); + } + cnt.push_str(&const_aarch64); + } + cnt.push_str(")"); + format!("{}(a, {})", current_fn, cnt) + } else { + match para_num { + 1 => format!("{}(a, {})", current_fn, const_aarch64), + 2 => format!("{}(a, b, {})", current_fn, const_aarch64), + _ => String::new(), + } + } + } else if out_t != link_aarch64_t[3] { + match para_num { + 1 => format!("transmute({}(a))", current_fn,), + 2 => format!("transmute({}(a, b))", current_fn,), + _ => String::new(), + } + } else if matches!(fn_type, Fntype::Store) { + let cast = if is_vstx(&name) { " as _" } else { "" }; + match type_sub_len(in_t[1]) { + 1 => format!("{}(b, a{})", current_fn, cast), + 2 => format!("{}(b.0, b.1, a{})", current_fn, cast), + 3 => format!("{}(b.0, b.1, b.2, a{})", current_fn, cast), + 4 => format!("{}(b.0, b.1, b.2, b.3, a{})", current_fn, cast), + _ => String::new(), + } + } else if link_aarch64.is_some() && is_vldx(&name) { + format!("{}(a as _)", current_fn) + } else { + String::new() + }; + format!( + r#"{}{{ + {}{}{} +}}"#, + fn_decl, multi_calls, ext_c_aarch64, aarch64_params + ) + }; + let stable_aarch64 = match target { + Default | ArmV7 | Vfp4 | FPArmV8 | AES => { + String::from("\n#[stable(feature = \"neon_intrinsics\", since = \"1.59.0\")]") + } + RDM => String::from("\n#[stable(feature = \"rdm_intrinsics\", since = \"1.62.0\")]"), + _ => String::new(), + }; + let function_doc = create_doc_string(current_comment, &name); + format!( + r#" +{} +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,{}")] +#[cfg_attr(test, assert_instr({}{}))]{} +{} + +{} +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "{}")] +#[cfg_attr(test, assert_instr({}{}))]{}{} +{} +"#, + function_doc, + current_target_arm, + expand_intrinsic(¤t_arm, in_t[1]), + const_assert, + const_legacy, + call_arm, + function_doc, + current_target_aarch64, + expand_intrinsic(¤t_aarch64, in_t[1]), + const_assert, + const_legacy, + stable_aarch64, + call_aarch64, + ) + } else { + let call = { + let stmts = match (multi_calls.len(), para_num, fixed.len()) { + (0, 1, 0) => format!(r#"{}{}(a)"#, ext_c, current_fn,), + (0, 1, _) => { + let fixed: Vec = + fixed.iter().take(type_len(in_t[0])).cloned().collect(); + format!( + r#"let b{}; + {}{}(a, transmute(b))"#, + values(in_t[0], &fixed), + ext_c, + current_fn, + ) + } + (0, 2, _) => format!(r#"{}{}(a, b)"#, ext_c, current_fn,), + (0, 3, _) => format!(r#"{}{}(a, b, c)"#, ext_c, current_fn,), + (_, 1, _) => format!(r#"{}{}"#, ext_c, multi_calls,), + (_, 2, _) => format!(r#"{}{}"#, ext_c, multi_calls,), + (_, 3, _) => format!(r#"{}{}"#, ext_c, multi_calls,), + (_, _, _) => String::new(), + }; + if stmts != String::new() { + format!( + r#"{}{{ + {} +}}"#, + fn_decl, stmts + ) + } else { + String::new() + } + }; + let stable_aarch64 = match target { + Default | ArmV7 | Vfp4 | FPArmV8 | AES => String::from("\n#[cfg_attr(target_arch = \"aarch64\", stable(feature = \"neon_intrinsics\", since = \"1.59.0\"))]"), + RDM => String::from("\n#[cfg_attr(target_arch = \"aarch64\", stable(feature = \"rdm_intrinsics\", since = \"1.62.0\"))]"), + _ => String::new(), + }; + let function_doc = create_doc_string(current_comment, &name); + format!( + r#" +{} +#[inline] +#[target_feature(enable = "{}")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "{}"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr({}{}))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr({}{}))]{}{} +{} +"#, + function_doc, + current_target_aarch64, + current_target_arm, + expand_intrinsic(¤t_arm, in_t[1]), + const_assert, + expand_intrinsic(¤t_aarch64, in_t[1]), + const_assert, + const_legacy, + stable_aarch64, + call, + ) + }; + let test_target = match target { + I8MM => "neon,i8mm", + SM4 => "neon,sm4", + SHA3 => "neon,sha3", + FTTS => "neon,frintts", + _ => "neon", + }; + let test = match fn_type { + Fntype::Normal => gen_test( + &name, + in_t, + &out_t, + current_tests, + [type_len(in_t[0]), type_len(in_t[1]), type_len(in_t[2])], + type_len(out_t), + para_num, + test_target, + ), + Fntype::Load => gen_load_test(&name, in_t, &out_t, current_tests, type_len(out_t)), + Fntype::Store => gen_store_test(&name, in_t, &out_t, current_tests, type_len(in_t[1])), + }; + (function, test) +} + +fn expand_intrinsic(intr: &str, t: &str) -> String { + if intr.ends_with('.') { + let ext = match t { + "int8x8_t" => "i8", + "int8x16_t" => "i8", + "int16x4_t" => "i16", + "int16x8_t" => "i16", + "int32x2_t" => "i32", + "int32x4_t" => "i32", + "int64x1_t" => "i64", + "int64x2_t" => "i64", + "uint8x8_t" => "i8", + "uint8x16_t" => "i8", + "uint16x4_t" => "i16", + "uint16x8_t" => "i16", + "uint32x2_t" => "i32", + "uint32x4_t" => "i32", + "uint64x1_t" => "i64", + "uint64x2_t" => "i64", + "float16x4_t" => "f16", + "float16x8_t" => "f16", + "float32x2_t" => "f32", + "float32x4_t" => "f32", + "float64x1_t" => "f64", + "float64x2_t" => "f64", + "poly8x8_t" => "i8", + "poly8x16_t" => "i8", + "poly16x4_t" => "i16", + "poly16x8_t" => "i16", + /* + "poly64x1_t" => "i64x1", + "poly64x2_t" => "i64x2", + */ + _ => panic!("unknown type for extension: {}", t), + }; + format!(r#""{}{}""#, intr, ext) + } else if intr.ends_with(".s") { + let ext = match t { + "int8x8_t" => "s8", + "int8x16_t" => "s8", + "int16x4_t" => "s16", + "int16x8_t" => "s16", + "int32x2_t" => "s32", + "int32x4_t" => "s32", + "int64x1_t" => "s64", + "int64x2_t" => "s64", + "uint8x8_t" => "u8", + "uint8x16_t" => "u8", + "uint16x4_t" => "u16", + "uint16x8_t" => "u16", + "uint32x2_t" => "u32", + "uint32x4_t" => "u32", + "uint64x1_t" => "u64", + "uint64x2_t" => "u64", + "poly8x8_t" => "p8", + "poly8x16_t" => "p8", + "poly16x4_t" => "p16", + "poly16x8_t" => "p16", + "float16x4_t" => "f16", + "float16x8_t" => "f16", + "float32x2_t" => "f32", + "float32x4_t" => "f32", + "float64x1_t" => "f64", + "float64x2_t" => "f64", + /* + "poly64x1_t" => "i64x1", + "poly64x2_t" => "i64x2", + */ + _ => panic!("unknown type for extension: {}", t), + }; + format!(r#""{}{}""#, &intr[..intr.len() - 1], ext) + } else if intr.ends_with(".l") { + let ext = match t { + "int8x8_t" => "8", + "int8x16_t" => "8", + "int16x4_t" => "16", + "int16x8_t" => "16", + "int32x2_t" => "32", + "int32x4_t" => "32", + "int64x1_t" => "64", + "int64x2_t" => "64", + "uint8x8_t" => "8", + "uint8x16_t" => "8", + "uint16x4_t" => "16", + "uint16x8_t" => "16", + "uint32x2_t" => "32", + "uint32x4_t" => "32", + "uint64x1_t" => "64", + "uint64x2_t" => "64", + "poly8x8_t" => "8", + "poly8x16_t" => "8", + "poly16x4_t" => "16", + "poly16x8_t" => "16", + "float16x4_t" => "16", + "float16x8_t" => "16", + "float32x2_t" => "32", + "float32x4_t" => "32", + "float64x1_t" => "64", + "float64x2_t" => "64", + "poly64x1_t" => "64", + "poly64x2_t" => "64", + _ => panic!("unknown type for extension: {}", t), + }; + format!(r#""{}{}""#, &intr[..intr.len() - 1], ext) + } else { + intr.to_string() + } +} + +fn get_call( + in_str: &str, + current_name: &str, + const_declare: &str, + in_t: &[&str; 3], + out_t: &str, + fixed: &Vec, + n: Option, + aarch64: bool, +) -> String { + let params: Vec<_> = in_str.split(',').map(|v| v.trim().to_string()).collect(); + assert!(params.len() > 0); + let mut fn_name = params[0].clone(); + if fn_name == "a" { + return String::from("a"); + } + if fn_name == "transpose-1-in_len" { + return transpose1(type_len(in_t[1])).to_string(); + } + if fn_name == "transpose-2-in_len" { + return transpose2(type_len(in_t[1])).to_string(); + } + if fn_name == "zip-1-in_len" { + return zip1(type_len(in_t[1])).to_string(); + } + if fn_name == "zip-2-in_len" { + return zip2(type_len(in_t[1])).to_string(); + } + if fn_name == "unzip-1-in_len" { + return unzip1(type_len(in_t[1])).to_string(); + } + if fn_name == "unzip-2-in_len" { + return unzip2(type_len(in_t[1])).to_string(); + } + if fn_name.starts_with("dup") { + let fn_format: Vec<_> = fn_name.split('-').map(|v| v.to_string()).collect(); + let len = match &*fn_format[1] { + "out_len" => type_len(out_t), + "in_len" => type_len(in_t[1]), + "in0_len" => type_len(in_t[0]), + "halflen" => type_len(in_t[1]) / 2, + _ => 0, + }; + let mut s = format!("{} [", const_declare); + for i in 0..len { + if i != 0 { + s.push_str(", "); + } + s.push_str(&fn_format[2]); + } + s.push_str("]"); + return s; + } + if fn_name.starts_with("asc") { + let fn_format: Vec<_> = fn_name.split('-').map(|v| v.to_string()).collect(); + let start = match &*fn_format[1] { + "0" => 0, + "n" => n.unwrap(), + "out_len" => type_len(out_t) as i32, + "halflen" => (type_len(in_t[1]) / 2) as i32, + s => s.parse::().unwrap(), + }; + let len = match &*fn_format[2] { + "out_len" => type_len(out_t), + "in_len" => type_len(in_t[1]), + "in0_len" => type_len(in_t[0]), + "halflen" => type_len(in_t[1]) / 2, + _ => 0, + }; + return asc(start, len); + } + if fn_name.starts_with("base") { + let fn_format: Vec<_> = fn_name.split('-').map(|v| v.to_string()).collect(); + assert_eq!(fn_format.len(), 3); + let mut s = format!(" [", &fn_format[2]); + let base_len = fn_format[1].parse::().unwrap(); + for i in 0..type_len(in_t[1]) / base_len { + for j in 0..base_len { + if i != 0 || j != 0 { + s.push_str(", "); + } + s.push_str(&format!("{} * {} as u32", base_len, &fn_format[2])); + if j != 0 { + s.push_str(&format!(" + {}", j)); + } + } + } + s.push_str("]"); + return s; + } + if fn_name.starts_with("as") { + let fn_format: Vec<_> = fn_name.split('-').map(|v| v.to_string()).collect(); + assert_eq!(fn_format.len(), 3); + let t = match &*fn_format[2] { + "in_ttn" => type_to_native_type(in_t[1]), + _ => String::new(), + }; + return format!("{} as {}", &fn_format[1], t); + } + if fn_name.starts_with("ins") { + let fn_format: Vec<_> = fn_name.split('-').map(|v| v.to_string()).collect(); + let n = n.unwrap(); + let len = match &*fn_format[1] { + "out_len" => type_len(out_t), + "in_len" => type_len(in_t[1]), + "in0_len" => type_len(in_t[0]), + _ => 0, + }; + let offset = match &*fn_format[2] { + "out_len" => type_len(out_t), + "in_len" => type_len(in_t[1]), + "in0_len" => type_len(in_t[0]), + _ => 0, + }; + let mut s = format!("{} [", const_declare); + for i in 0..len { + if i != 0 { + s.push_str(", "); + } + if i == n as usize { + s.push_str(&format!("{} + {} as u32", offset.to_string(), fn_format[3])); + } else { + s.push_str(&i.to_string()); + } + } + s.push_str("]"); + return s; + } + if fn_name.starts_with("static_assert_imm") { + let fn_format: Vec<_> = fn_name.split('-').map(|v| v.to_string()).collect(); + let len = match &*fn_format[1] { + "out_exp_len" => type_exp_len(out_t, 1), + "out_bits_exp_len" => type_bits_exp_len(out_t), + "in_exp_len" => type_exp_len(in_t[1], 1), + "in_bits_exp_len" => type_bits_exp_len(in_t[1]), + "in0_exp_len" => type_exp_len(in_t[0], 1), + "in1_exp_len" => type_exp_len(in_t[1], 1), + "in2_exp_len" => type_exp_len(in_t[2], 1), + "in2_rot" => type_exp_len(in_t[2], 2), + "in2_dot" => type_exp_len(in_t[2], 4), + _ => 0, + }; + if len == 0 { + return format!( + r#"static_assert!({} : i32 where {} == 0);"#, + fn_format[2], fn_format[2] + ); + } else { + return format!(r#"static_assert_imm{}!({});"#, len, fn_format[2]); + } + } + if fn_name.starts_with("static_assert") { + let fn_format: Vec<_> = fn_name.split('-').map(|v| v.to_string()).collect(); + let lim1 = if fn_format[2] == "bits" { + type_bits(in_t[1]).to_string() + } else if fn_format[2] == "halfbits" { + (type_bits(in_t[1]) / 2).to_string() + } else { + fn_format[2].clone() + }; + let lim2 = if fn_format[3] == "bits" { + type_bits(in_t[1]).to_string() + } else if fn_format[3] == "halfbits" { + (type_bits(in_t[1]) / 2).to_string() + } else { + fn_format[3].clone() + }; + if lim1 == lim2 { + return format!( + r#"static_assert!({} : i32 where {} == {});"#, + fn_format[1], fn_format[1], lim1 + ); + } else { + return format!( + r#"static_assert!({} : i32 where {} >= {} && {} <= {});"#, + fn_format[1], fn_format[1], lim1, fn_format[1], lim2 + ); + } + } + if fn_name.starts_with("fix_right_shift_imm") { + let fn_format: Vec<_> = fn_name.split('-').map(|v| v.to_string()).collect(); + let lim = if fn_format[2] == "bits" { + type_bits(in_t[1]).to_string() + } else { + fn_format[2].clone() + }; + let fixed = if in_t[1].starts_with('u') { + format!("return vdup{nself}(0);", nself = type_to_n_suffix(in_t[1])) + } else { + (lim.parse::().unwrap() - 1).to_string() + }; + + return format!( + r#"let {name}: i32 = if {const_name} == {upper} {{ {fixed} }} else {{ N }};"#, + name = fn_format[1].to_lowercase(), + const_name = fn_format[1], + upper = lim, + fixed = fixed, + ); + } + + if fn_name.starts_with("matchn") { + let fn_format: Vec<_> = fn_name.split('-').map(|v| v.to_string()).collect(); + let len = match &*fn_format[1] { + "out_exp_len" => type_exp_len(out_t, 1), + "in_exp_len" => type_exp_len(in_t[1], 1), + "in0_exp_len" => type_exp_len(in_t[0], 1), + _ => 0, + }; + let mut call = format!("match {} & 0b{} {{\n", &fn_format[2], "1".repeat(len)); + let mut sub_call = String::new(); + for p in 1..params.len() { + if !sub_call.is_empty() { + sub_call.push_str(", "); + } + sub_call.push_str(¶ms[p]); + } + for i in 0..(2u32.pow(len as u32) as usize) { + let sub_match = format!( + " {} => {},\n", + i, + get_call( + &sub_call, + current_name, + const_declare, + in_t, + out_t, + fixed, + Some(i as i32), + aarch64 + ) + ); + call.push_str(&sub_match); + } + call.push_str(" _ => unreachable_unchecked(),\n }"); + return call; + } + let mut re: Option<(String, String)> = None; + let mut param_str = String::new(); + let mut i = 1; + while i < params.len() { + let s = ¶ms[i]; + if s.starts_with('{') { + let mut sub_fn = String::new(); + let mut parentheses = 0; + while i < params.len() { + if !sub_fn.is_empty() { + sub_fn.push_str(", "); + } + sub_fn.push_str(¶ms[i]); + let l = params[i].len(); + for j in 0..l { + if ¶ms[i][j..j + 1] == "{" { + parentheses += 1; + } else { + break; + } + } + for j in 0..l { + if ¶ms[i][l - j - 1..l - j] == "}" { + parentheses -= 1; + } else { + break; + } + } + if parentheses == 0 { + break; + } + i += 1; + } + let sub_call = get_call( + &sub_fn[1..sub_fn.len() - 1], + current_name, + const_declare, + in_t, + out_t, + fixed, + n.clone(), + aarch64, + ); + if !param_str.is_empty() { + param_str.push_str(", "); + } + param_str.push_str(&sub_call); + } else if s.contains(':') { + let re_params: Vec<_> = s.split(':').map(|v| v.to_string()).collect(); + if re_params[1] == "" { + re = Some((re_params[0].clone(), in_t[1].to_string())); + } else if re_params[1] == "in_t" { + re = Some((re_params[0].clone(), in_t[1].to_string())); + } else if re_params[1] == "signed" { + re = Some((re_params[0].clone(), type_to_signed(in_t[1]))); + } else if re_params[1] == "unsigned" { + re = Some((re_params[0].clone(), type_to_unsigned(in_t[1]))); + } else if re_params[1] == "in_t0" { + re = Some((re_params[0].clone(), in_t[0].to_string())); + } else if re_params[1] == "in_t1" { + re = Some((re_params[0].clone(), in_t[1].to_string())); + } else if re_params[1] == "out_t" { + re = Some((re_params[0].clone(), out_t.to_string())); + } else if re_params[1] == "half" { + re = Some((re_params[0].clone(), type_to_half(in_t[1]).to_string())); + } else if re_params[1] == "in_ntt" { + re = Some(( + re_params[0].clone(), + native_type_to_type(in_t[1]).to_string(), + )); + } else if re_params[1] == "in_long_ntt" { + re = Some(( + re_params[0].clone(), + native_type_to_long_type(in_t[1]).to_string(), + )); + } else if re_params[1] == "out_ntt" { + re = Some((re_params[0].clone(), native_type_to_type(out_t).to_string())); + } else if re_params[1] == "out_long_ntt" { + re = Some(( + re_params[0].clone(), + native_type_to_long_type(out_t).to_string(), + )); + } else { + re = Some((re_params[0].clone(), re_params[1].clone())); + } + } else { + if !param_str.is_empty() { + param_str.push_str(", "); + } + param_str.push_str(s); + } + i += 1; + } + if fn_name == "fixed" { + let (re_name, re_type) = re.unwrap(); + let fixed: Vec = fixed.iter().take(type_len(in_t[1])).cloned().collect(); + return format!(r#"let {}{};"#, re_name, values(&re_type, &fixed)); + } + if fn_name == "fixed-half-right" { + let fixed: Vec = fixed.iter().take(type_len(in_t[1])).cloned().collect(); + let half = fixed[type_len(in_t[1]) / 2..] + .iter() + .fold(String::new(), |mut s, fix| { + s.push_str(fix); + s.push_str(", "); + s + }); + return format!(r#"[{}]"#, &half[..half.len() - 2]); + } + if fn_name == "a - b" { + return fn_name; + } + if fn_name == "-a" { + return fn_name; + } + if fn_name.contains('-') { + let fn_format: Vec<_> = fn_name.split('-').map(|v| v.to_string()).collect(); + assert_eq!(fn_format.len(), 3); + fn_name = if fn_format[0] == "self" { + current_name.to_string() + } else { + fn_format[0].clone() + }; + if fn_format[1] == "self" { + fn_name.push_str(type_to_suffix(in_t[1])); + } else if fn_format[1] == "nself" { + fn_name.push_str(type_to_n_suffix(in_t[1])); + } else if fn_format[1] == "nselfvfp4" { + fn_name.push_str(type_to_n_suffix(in_t[1])); + if !aarch64 { + fn_name.push_str("_vfp4"); + } + } else if fn_format[1] == "out" { + fn_name.push_str(type_to_suffix(out_t)); + } else if fn_format[1] == "in0" { + fn_name.push_str(type_to_suffix(in_t[0])); + } else if fn_format[1] == "in2" { + fn_name.push_str(type_to_suffix(in_t[2])); + } else if fn_format[1] == "in2lane" { + fn_name.push_str(&type_to_lane_suffixes(out_t, in_t[2], false)); + } else if fn_format[1] == "outlane" { + fn_name.push_str(&type_to_lane_suffixes(out_t, in_t[2], true)); + } else if fn_format[1] == "signed" { + fn_name.push_str(type_to_suffix(&type_to_signed(&String::from(in_t[1])))); + } else if fn_format[1] == "outsigned" { + fn_name.push_str(type_to_suffix(&type_to_signed(&String::from(out_t)))); + } else if fn_format[1] == "outsignednox" { + fn_name.push_str(&type_to_suffix(&type_to_sub_type(&type_to_signed( + &String::from(out_t), + )))); + } else if fn_format[1] == "in1signednox" { + fn_name.push_str(&type_to_suffix(&type_to_sub_type(&type_to_signed( + &String::from(in_t[1]), + )))); + } else if fn_format[1] == "outsigneddupnox" { + fn_name.push_str(&type_to_dup_suffix(&type_to_sub_type(&type_to_signed( + &String::from(out_t), + )))); + } else if fn_format[1] == "outsignedlanenox" { + fn_name.push_str(&type_to_lane_suffix(&type_to_sub_type(&type_to_signed( + &String::from(out_t), + )))); + } else if fn_format[1] == "in1signedlanenox" { + fn_name.push_str(&type_to_lane_suffix(&type_to_sub_type(&type_to_signed( + &String::from(in_t[1]), + )))); + } else if fn_format[1] == "unsigned" { + fn_name.push_str(type_to_suffix(&type_to_unsigned(in_t[1]))); + } else if fn_format[1] == "doubleself" { + fn_name.push_str(&type_to_double_suffixes(out_t, in_t[1])); + } else if fn_format[1] == "noq_doubleself" { + fn_name.push_str(&type_to_noq_double_suffixes(out_t, in_t[1])); + } else if fn_format[1] == "noqself" { + fn_name.push_str(type_to_noq_suffix(in_t[1])); + } else if fn_format[1] == "noqsigned" { + fn_name.push_str(type_to_noq_suffix(&type_to_signed(&String::from(in_t[1])))); + } else if fn_format[1] == "nosuffix" { + } else if fn_format[1] == "in_len" { + fn_name.push_str(&type_len(in_t[1]).to_string()); + } else if fn_format[1] == "in0_len" { + fn_name.push_str(&type_len(in_t[0]).to_string()); + } else if fn_format[1] == "out_len" { + fn_name.push_str(&type_len(out_t).to_string()); + } else if fn_format[1] == "halflen" { + fn_name.push_str(&(type_len(in_t[1]) / 2).to_string()); + } else if fn_format[1] == "nout" { + fn_name.push_str(type_to_n_suffix(out_t)); + } else if fn_format[1] == "nin0" { + fn_name.push_str(type_to_n_suffix(in_t[0])); + } else if fn_format[1] == "nsigned" { + fn_name.push_str(type_to_n_suffix(&type_to_signed(&String::from(in_t[1])))); + } else if fn_format[1] == "in_ntt" { + fn_name.push_str(type_to_suffix(native_type_to_type(in_t[1]))); + } else if fn_format[1] == "out_ntt" { + fn_name.push_str(type_to_suffix(native_type_to_type(out_t))); + } else if fn_format[1] == "rot" { + fn_name = type_to_rot_suffix(&fn_name, type_to_suffix(out_t)); + } else { + fn_name.push_str(&fn_format[1]); + }; + if fn_format[2] == "ext" { + fn_name.push_str("_"); + } else if fn_format[2] == "noext" { + } else if fn_format[2].starts_with("<") { + assert!(fn_format[2].ends_with(">")); + let types: Vec<_> = fn_format[2][1..fn_format[2].len() - 1] + .split(' ') + .map(|v| v.to_string()) + .collect(); + assert_eq!(types.len(), 2); + let type1 = if types[0] == "element_t" { + type_to_native_type(in_t[1]) + } else { + String::from(&types[0]) + }; + let type2 = if types[1] == "element_t" { + type_to_native_type(in_t[1]) + } else { + String::from(&types[1]) + }; + fn_name.push_str(&format!("::<{}, {}>", &type1, &type2)); + } else { + fn_name.push_str(&fn_format[2]); + } + } + if param_str.is_empty() { + return fn_name.replace("out_t", out_t); + } + let fn_str = if let Some((re_name, re_type)) = re.clone() { + format!( + r#"let {}: {} = {}({});"#, + re_name, re_type, fn_name, param_str + ) + } else if fn_name.starts_with("*") { + format!(r#"{} = {};"#, fn_name, param_str) + } else { + format!(r#"{}({})"#, fn_name, param_str) + }; + return fn_str; +} + +fn main() -> io::Result<()> { + let args: Vec = env::args().collect(); + let in_file = args.get(1).cloned().unwrap_or_else(|| IN.to_string()); + + let f = File::open(in_file).expect("Failed to open neon.spec"); + let f = BufReader::new(f); + + let mut current_comment = String::new(); + let mut current_name: Option = None; + let mut current_fn: Option = None; + let mut current_arm: Option = None; + let mut current_aarch64: Option = None; + let mut link_arm: Option = None; + let mut link_aarch64: Option = None; + let mut const_arm: Option = None; + let mut const_aarch64: Option = None; + let mut constn: Option = None; + let mut para_num = 2; + let mut suffix: Suffix = Normal; + let mut a: Vec = Vec::new(); + let mut b: Vec = Vec::new(); + let mut c: Vec = Vec::new(); + let mut n: Option = None; + let mut fixed: Vec = Vec::new(); + let mut current_tests: Vec<( + Vec, + Vec, + Vec, + Option, + Vec, + )> = Vec::new(); + let mut multi_fn: Vec = Vec::new(); + let mut target: TargetFeature = Default; + let mut fn_type: Fntype = Fntype::Normal; + let mut separate = false; + + // + // THIS FILE IS GENERATED FORM neon.spec DO NOT CHANGE IT MANUALLY + // + let mut out_arm = String::from( + r#"// This code is automatically generated. DO NOT MODIFY. +// +// Instead, modify `crates/stdarch-gen/neon.spec` and run the following command to re-generate this file: +// +// ``` +// OUT_DIR=`pwd`/crates/core_arch cargo run -p stdarch-gen -- crates/stdarch-gen/neon.spec +// ``` +use super::*; +#[cfg(test)] +use stdarch_test::assert_instr; +"#, + ); + let mut tests_arm = String::from( + r#" +#[cfg(test)] +#[allow(overflowing_literals)] +mod test { + use super::*; + use crate::core_arch::simd::*; + use std::mem::transmute; + use stdarch_test::simd_test; +"#, + ); + // + // THIS FILE IS GENERATED FORM neon.spec DO NOT CHANGE IT MANUALLY + // + let mut out_aarch64 = String::from( + r#"// This code is automatically generated. DO NOT MODIFY. +// +// Instead, modify `crates/stdarch-gen/neon.spec` and run the following command to re-generate this file: +// +// ``` +// OUT_DIR=`pwd`/crates/core_arch cargo run -p stdarch-gen -- crates/stdarch-gen/neon.spec +// ``` +use super::*; +#[cfg(test)] +use stdarch_test::assert_instr; +"#, + ); + let mut tests_aarch64 = String::from( + r#" +#[cfg(test)] +mod test { + use super::*; + use crate::core_arch::simd::*; + use std::mem::transmute; + use stdarch_test::simd_test; +"#, + ); + + for line in f.lines() { + let line = line.unwrap(); + if line.is_empty() { + continue; + } + if line.starts_with("/// ") { + current_comment = line; + current_name = None; + current_fn = None; + current_arm = None; + current_aarch64 = None; + link_aarch64 = None; + link_arm = None; + const_aarch64 = None; + const_arm = None; + current_tests = Vec::new(); + constn = None; + para_num = 2; + suffix = Normal; + a = Vec::new(); + b = Vec::new(); + c = Vec::new(); + fixed = Vec::new(); + n = None; + multi_fn = Vec::new(); + target = Default; + fn_type = Fntype::Normal; + separate = false; + } else if line.starts_with("//") { + } else if line.starts_with("name = ") { + current_name = Some(String::from(&line[7..])); + } else if line.starts_with("fn = ") { + current_fn = Some(String::from(&line[5..])); + } else if line.starts_with("multi_fn = ") { + multi_fn.push(String::from(&line[11..])); + } else if line.starts_with("constn = ") { + constn = Some(String::from(&line[9..])); + } else if line.starts_with("arm = ") { + current_arm = Some(String::from(&line[6..])); + } else if line.starts_with("aarch64 = ") { + current_aarch64 = Some(String::from(&line[10..])); + } else if line.starts_with("double-suffixes") { + suffix = Double; + } else if line.starts_with("no-q") { + suffix = NoQ; + } else if line.starts_with("noq-double-suffixes") { + suffix = NoQDouble; + } else if line.starts_with("n-suffix") { + suffix = NSuffix; + } else if line.starts_with("double-n-suffixes") { + suffix = DoubleN; + } else if line.starts_with("out-n-suffix") { + suffix = OutNSuffix; + } else if line.starts_with("noq-n-suffix") { + suffix = NoQNSuffix; + } else if line.starts_with("out-suffix") { + suffix = OutSuffix; + } else if line.starts_with("out-nox") { + suffix = OutNox; + } else if line.starts_with("in1-nox") { + suffix = In1Nox; + } else if line.starts_with("out-dup-nox") { + suffix = OutDupNox; + } else if line.starts_with("out-lane-nox") { + suffix = OutLaneNox; + } else if line.starts_with("in1-lane-nox") { + suffix = In1LaneNox; + } else if line.starts_with("lane-suffixes") { + suffix = Lane; + } else if line.starts_with("in2-suffix") { + suffix = In2; + } else if line.starts_with("in2-lane-suffixes") { + suffix = In2Lane; + } else if line.starts_with("out-lane-suffixes") { + suffix = OutLane; + } else if line.starts_with("rot-suffix") { + suffix = Rot; + } else if line.starts_with("rot-lane-suffixes") { + suffix = RotLane; + } else if line.starts_with("a = ") { + a = line[4..].split(',').map(|v| v.trim().to_string()).collect(); + } else if line.starts_with("b = ") { + b = line[4..].split(',').map(|v| v.trim().to_string()).collect(); + } else if line.starts_with("c = ") { + c = line[4..].split(',').map(|v| v.trim().to_string()).collect(); + } else if line.starts_with("n = ") { + n = Some(String::from(&line[4..])); + } else if line.starts_with("fixed = ") { + fixed = line[8..].split(',').map(|v| v.trim().to_string()).collect(); + } else if line.starts_with("validate ") { + let e = line[9..].split(',').map(|v| v.trim().to_string()).collect(); + current_tests.push((a.clone(), b.clone(), c.clone(), n.clone(), e)); + } else if line.starts_with("link-aarch64 = ") { + link_aarch64 = Some(String::from(&line[15..])); + } else if line.starts_with("const-aarch64 = ") { + const_aarch64 = Some(String::from(&line[16..])); + } else if line.starts_with("link-arm = ") { + link_arm = Some(String::from(&line[11..])); + } else if line.starts_with("const-arm = ") { + const_arm = Some(String::from(&line[12..])); + } else if line.starts_with("load_fn") { + fn_type = Fntype::Load; + } else if line.starts_with("store_fn") { + fn_type = Fntype::Store; + } else if line.starts_with("arm-aarch64-separate") { + separate = true; + } else if line.starts_with("target = ") { + target = match Some(String::from(&line[9..])) { + Some(input) => match input.as_str() { + "v7" => ArmV7, + "vfp4" => Vfp4, + "fp-armv8" => FPArmV8, + "aes" => AES, + "fcma" => FCMA, + "dotprod" => Dotprod, + "i8mm" => I8MM, + "sha3" => SHA3, + "rdm" => RDM, + "sm4" => SM4, + "frintts" => FTTS, + _ => Default, + }, + _ => Default, + } + } else if line.starts_with("generate ") { + let line = &line[9..]; + let types: Vec = line + .split(',') + .map(|v| v.trim().to_string()) + .flat_map(|v| match v.as_str() { + "uint*_t" => UINT_TYPES.iter().map(|v| v.to_string()).collect(), + "uint64x*_t" => UINT_TYPES_64.iter().map(|v| v.to_string()).collect(), + "int*_t" => INT_TYPES.iter().map(|v| v.to_string()).collect(), + "int64x*_t" => INT_TYPES_64.iter().map(|v| v.to_string()).collect(), + "float*_t" => FLOAT_TYPES.iter().map(|v| v.to_string()).collect(), + "float64x*_t" => FLOAT_TYPES_64.iter().map(|v| v.to_string()).collect(), + _ => vec![v], + }) + .collect(); + + for line in types { + let spec: Vec<&str> = line.split(':').map(|e| e.trim()).collect(); + let in_t: [&str; 3]; + let out_t; + if spec.len() == 1 { + in_t = [spec[0], spec[0], spec[0]]; + out_t = spec[0]; + } else if spec.len() == 2 { + in_t = [spec[0], spec[0], spec[0]]; + out_t = spec[1]; + } else if spec.len() == 3 { + in_t = [spec[0], spec[1], spec[1]]; + out_t = spec[2]; + } else if spec.len() == 4 { + in_t = [spec[0], spec[1], spec[2]]; + out_t = spec[3]; + } else { + panic!("Bad spec: {}", line) + } + if b.len() == 0 { + if matches!(fn_type, Fntype::Store) { + para_num = 2; + } else { + para_num = 1; + } + } else if c.len() != 0 { + para_num = 3; + } + let current_name = current_name.clone().unwrap(); + if let Some(current_arm) = current_arm.clone() { + let (function, test) = gen_arm( + ¤t_comment, + ¤t_fn, + ¤t_name, + ¤t_arm, + &link_arm, + ¤t_aarch64, + &link_aarch64, + &const_arm, + &const_aarch64, + &constn, + &in_t, + &out_t, + ¤t_tests, + suffix, + para_num, + target, + &fixed, + &multi_fn, + fn_type, + separate, + ); + out_arm.push_str(&function); + tests_arm.push_str(&test); + } else { + let (function, test) = gen_aarch64( + ¤t_comment, + ¤t_fn, + ¤t_name, + ¤t_aarch64, + &link_aarch64, + &const_aarch64, + &constn, + &in_t, + &out_t, + ¤t_tests, + suffix, + para_num, + target, + &fixed, + &multi_fn, + fn_type, + ); + out_aarch64.push_str(&function); + tests_aarch64.push_str(&test); + } + } + } + } + tests_arm.push('}'); + tests_arm.push('\n'); + tests_aarch64.push('}'); + tests_aarch64.push('\n'); + + let arm_out_path: PathBuf = + PathBuf::from(env::var("OUT_DIR").unwrap_or("crates/core_arch".to_string())) + .join("src") + .join("arm_shared") + .join("neon"); + std::fs::create_dir_all(&arm_out_path)?; + + let mut file_arm = File::create(arm_out_path.join(ARM_OUT))?; + file_arm.write_all(out_arm.as_bytes())?; + file_arm.write_all(tests_arm.as_bytes())?; + + let aarch64_out_path: PathBuf = + PathBuf::from(env::var("OUT_DIR").unwrap_or("crates/core_arch".to_string())) + .join("src") + .join("aarch64") + .join("neon"); + std::fs::create_dir_all(&aarch64_out_path)?; + + let mut file_aarch = File::create(aarch64_out_path.join(AARCH64_OUT))?; + file_aarch.write_all(out_aarch64.as_bytes())?; + file_aarch.write_all(tests_aarch64.as_bytes())?; + /* + if let Err(e) = Command::new("rustfmt") + .arg(&arm_out_path) + .arg(&aarch64_out_path) + .status() { + eprintln!("Could not format `{}`: {}", arm_out_path.to_str().unwrap(), e); + eprintln!("Could not format `{}`: {}", aarch64_out_path.to_str().unwrap(), e); + }; + */ + Ok(()) +} diff --git a/crux-mir/lib/stdarch/crates/stdarch-test/Cargo.toml b/crux-mir/lib/stdarch/crates/stdarch-test/Cargo.toml index 2b445f8dc..012b4e959 100644 --- a/crux-mir/lib/stdarch/crates/stdarch-test/Cargo.toml +++ b/crux-mir/lib/stdarch/crates/stdarch-test/Cargo.toml @@ -2,6 +2,7 @@ name = "stdarch-test" version = "0.1.0" authors = ["Alex Crichton "] +edition = "2018" [dependencies] assert-instr-macro = { path = "../assert-instr-macro" } @@ -9,12 +10,15 @@ simd-test-macro = { path = "../simd-test-macro" } cc = "1.0" lazy_static = "1.0" rustc-demangle = "0.1.8" -cfg-if = "0.1" +cfg-if = "1.0" -[target.wasm32-unknown-unknown.dependencies] -wasm-bindgen = "0.2.47" -js-sys = "0.3" -console_error_panic_hook = "0.1" +# We use a crates.io dependency to disassemble wasm binaries to look for +# instructions for `#[assert_instr]`. Note that we use an `=` dependency here +# instead of a floating dependency because the text format for wasm changes over +# time, and we want to make updates to this explicit rather than automatically +# picking up updates which might break CI with new instruction names. +[target.'cfg(target_arch = "wasm32")'.dependencies] +wasmprinter = "=0.2.24" [features] default = [] diff --git a/crux-mir/lib/stdarch/crates/stdarch-test/src/disassembly.rs b/crux-mir/lib/stdarch/crates/stdarch-test/src/disassembly.rs index d82b07d0a..3ace6b20e 100644 --- a/crux-mir/lib/stdarch/crates/stdarch-test/src/disassembly.rs +++ b/crux-mir/lib/stdarch/crates/stdarch-test/src/disassembly.rs @@ -32,62 +32,65 @@ fn normalize(mut symbol: &str) -> String { while symbol.starts_with('_') { symbol.remove(0); } + // Windows/x86 has a suffix such as @@4. + if let Some(idx) = symbol.find("@@") { + symbol = (&symbol[..idx]).to_string(); + } symbol } pub(crate) fn disassemble_myself() -> HashSet { let me = env::current_exe().expect("failed to get current exe"); - let disassembly = - if cfg!(target_arch = "x86_64") && cfg!(target_os = "windows") && cfg!(target_env = "msvc") - { - let mut cmd = cc::windows_registry::find("x86_64-pc-windows-msvc", "dumpbin.exe") - .expect("failed to find `dumpbin` tool"); - let output = cmd - .arg("/DISASM") - .arg(&me) - .output() - .expect("failed to execute dumpbin"); - println!( - "{}\n{}", - output.status, - String::from_utf8_lossy(&output.stderr) - ); - assert!(output.status.success()); - // Windows does not return valid UTF-8 output: - String::from_utf8_lossy(Vec::leak(output.stdout)) - } else if cfg!(target_os = "windows") { + let disassembly = if cfg!(target_os = "windows") && cfg!(target_env = "msvc") { + let target = if cfg!(target_arch = "x86_64") { + "x86_64-pc-windows-msvc" + } else if cfg!(target_arch = "x86") { + "i686-pc-windows-msvc" + } else { panic!("disassembly unimplemented") - } else if cfg!(target_os = "macos") { - let output = Command::new("otool") - .arg("-vt") - .arg(&me) - .output() - .expect("failed to execute otool"); - println!( - "{}\n{}", - output.status, - String::from_utf8_lossy(&output.stderr) - ); - assert!(output.status.success()); - - String::from_utf8_lossy(Vec::leak(output.stdout)) + }; + let mut cmd = cc::windows_registry::find(target, "dumpbin.exe") + .expect("failed to find `dumpbin` tool"); + let output = cmd + .arg("/DISASM") + .arg(&me) + .output() + .expect("failed to execute dumpbin"); + println!( + "{}\n{}", + output.status, + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.status.success()); + // Windows does not return valid UTF-8 output: + String::from_utf8_lossy(Vec::leak(output.stdout)) + } else if cfg!(target_os = "windows") { + panic!("disassembly unimplemented") + } else { + let objdump = env::var("OBJDUMP").unwrap_or_else(|_| "objdump".to_string()); + let add_args = if cfg!(target_os = "macos") && cfg!(target_arch = "aarch64") { + // Target features need to be enabled for LLVM objdump on Macos ARM64 + vec!["--mattr=+v8.6a,+crypto,+tme"] } else { - let objdump = env::var("OBJDUMP").unwrap_or_else(|_| "objdump".to_string()); - let output = Command::new(objdump.clone()) - .arg("--disassemble") - .arg(&me) - .output() - .unwrap_or_else(|_| panic!("failed to execute objdump. OBJDUMP={}", objdump)); - println!( - "{}\n{}", - output.status, - String::from_utf8_lossy(&output.stderr) - ); - assert!(output.status.success()); - - String::from_utf8_lossy(Vec::leak(output.stdout)) + vec![] }; + let output = Command::new(objdump.clone()) + .arg("--disassemble") + .arg("--no-show-raw-insn") + .args(add_args) + .arg(&me) + .output() + .unwrap_or_else(|_| panic!("failed to execute objdump. OBJDUMP={}", objdump)); + println!( + "{}\n{}", + output.status, + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.status.success()); + + String::from_utf8_lossy(Vec::leak(output.stdout)) + }; parse(&disassembly) } @@ -122,16 +125,7 @@ fn parse(output: &str) -> HashSet { cached_header = None; break; } - let parts = if cfg!(target_os = "macos") { - // Each line of instructions should look like: - // - // $addr $instruction... - instruction - .split_whitespace() - .skip(1) - .map(std::string::ToString::to_string) - .collect::>() - } else if cfg!(target_env = "msvc") { + let mut parts = if cfg!(target_env = "msvc") { // Each line looks like: // // > $addr: ab cd ef $instr.. @@ -147,24 +141,40 @@ fn parse(output: &str) -> HashSet { .skip_while(|s| *s == "lock") // skip x86-specific prefix .collect::>() } else { - // objdump + // objdump with --no-show-raw-insn // Each line of instructions should look like: // - // $rel_offset: ab cd ef 00 $instruction... - let expected_len = if cfg!(target_arch = "arm") || cfg!(target_arch = "aarch64") { - 8 - } else { - 2 - }; - + // $rel_offset: $instruction... instruction .split_whitespace() .skip(1) - .skip_while(|s| s.len() == expected_len && usize::from_str_radix(s, 16).is_ok()) .skip_while(|s| *s == "lock") // skip x86-specific prefix .map(std::string::ToString::to_string) .collect::>() }; + + if cfg!(target_arch = "aarch64") { + // Normalize [us]shll.* ..., #0 instructions to the preferred form: [us]xtl.* ... + // as LLVM objdump does not do that. + // See https://developer.arm.com/documentation/ddi0602/latest/SIMD-FP-Instructions/UXTL--UXTL2--Unsigned-extend-Long--an-alias-of-USHLL--USHLL2- + // and https://developer.arm.com/documentation/ddi0602/latest/SIMD-FP-Instructions/SXTL--SXTL2--Signed-extend-Long--an-alias-of-SSHLL--SSHLL2- + // for details. + match (parts.first(), parts.last()) { + (Some(instr), Some(last_arg)) + if (instr.starts_with("ushll.") || instr.starts_with("sshll.")) + && last_arg == "#0" => + { + assert_eq!(parts.len(), 4); + let mut new_parts = Vec::with_capacity(3); + let new_instr = format!("{}{}{}", &instr[..1], "xtl", &instr[5..]); + new_parts.push(new_instr); + new_parts.push(parts[1].clone()); + new_parts.push(parts[2][0..parts[2].len() - 1].to_owned()); // strip trailing comma + parts = new_parts; + } + _ => {} + }; + } instructions.push(parts.join(" ")); } let function = Function { diff --git a/crux-mir/lib/stdarch/crates/stdarch-test/src/lib.rs b/crux-mir/lib/stdarch/crates/stdarch-test/src/lib.rs index 4e25d2a02..eba17771c 100644 --- a/crux-mir/lib/stdarch/crates/stdarch-test/src/lib.rs +++ b/crux-mir/lib/stdarch/crates/stdarch-test/src/lib.rs @@ -3,41 +3,25 @@ //! This basically just disassembles the current executable and then parses the //! output once globally and then provides the `assert` function which makes //! assertions about the disassembly of a function. -#![feature(const_transmute)] -#![feature(vec_leak)] +#![deny(rust_2018_idioms)] #![allow(clippy::missing_docs_in_private_items, clippy::print_stdout)] -extern crate assert_instr_macro; -extern crate cc; #[macro_use] extern crate lazy_static; -extern crate rustc_demangle; -extern crate simd_test_macro; #[macro_use] extern crate cfg_if; pub use assert_instr_macro::*; pub use simd_test_macro::*; -use std::{cmp, collections::HashSet, env, hash, str, sync::atomic::AtomicPtr}; - -// `println!` doesn't work on wasm32 right now, so shadow the compiler's `println!` -// macro with our own shim that redirects to `console.log`. -#[allow(unused)] -#[cfg(target_arch = "wasm32")] -#[macro_export] -macro_rules! println { - ($($args:tt)*) => (crate::wasm::js_console_log(&format!($($args)*))) -} +use std::{cmp, collections::HashSet, env, hash, hint::black_box, str}; cfg_if! { if #[cfg(target_arch = "wasm32")] { - extern crate wasm_bindgen; - extern crate console_error_panic_hook; pub mod wasm; use wasm::disassemble_myself; } else { mod disassembly; - use disassembly::disassemble_myself; + use crate::disassembly::disassemble_myself; } } @@ -76,7 +60,10 @@ impl hash::Hash for Function { /// /// This asserts that the function at `fnptr` contains the instruction /// `expected` provided. -pub fn assert(_fnptr: usize, fnname: &str, expected: &str) { +pub fn assert(shim_addr: usize, fnname: &str, expected: &str) { + // Make sure that the shim is not removed + black_box(shim_addr); + //eprintln!("shim name: {}", fnname); let function = &DISASSEMBLY .get(&Function::new(fnname)) @@ -90,19 +77,34 @@ pub fn assert(_fnptr: usize, fnname: &str, expected: &str) { // Look for `expected` as the first part of any instruction in this // function, e.g., tzcntl in tzcntl %rax,%rax. - let found = instrs.iter().any(|s| s.starts_with(expected)); - - // Look for `call` instructions in the disassembly to detect whether - // inlining failed: all intrinsics are `#[inline(always)]`, so - // calling one intrinsic from another should not generate `call` - // instructions. - let inlining_failed = instrs.windows(2).any(|s| { - // On 32-bit x86 position independent code will call itself and be - // immediately followed by a `pop` to learn about the current address. - // Let's not take that into account when considering whether a function - // failed inlining something. - s[0].contains("call") && (!cfg!(target_arch = "x86") || s[1].contains("pop")) - }); + // + // There are two cases when the expected instruction is nop: + // 1. The expected intrinsic is compiled away so we can't + // check for it - aka the intrinsic is not generating any code. + // 2. It is a mark, indicating that the instruction will be + // compiled into other instructions - mainly because of llvm + // optimization. + let found = expected == "nop" || instrs.iter().any(|s| s.starts_with(expected)); + + // Look for subroutine call instructions in the disassembly to detect whether + // inlining failed: all intrinsics are `#[inline(always)]`, so calling one + // intrinsic from another should not generate subroutine call instructions. + let inlining_failed = if cfg!(target_arch = "x86_64") || cfg!(target_arch = "wasm32") { + instrs.iter().any(|s| s.starts_with("call ")) + } else if cfg!(target_arch = "x86") { + instrs.windows(2).any(|s| { + // On 32-bit x86 position independent code will call itself and be + // immediately followed by a `pop` to learn about the current address. + // Let's not take that into account when considering whether a function + // failed inlining something. + s[0].starts_with("call ") && s[1].starts_with("pop") // FIXME: original logic but does not match comment + }) + } else if cfg!(target_arch = "aarch64") { + instrs.iter().any(|s| s.starts_with("bl ")) + } else { + // FIXME: Add detection for other archs + false + }; let instruction_limit = std::env::var("STDARCH_ASSERT_INSTR_LIMIT") .ok() @@ -122,10 +124,31 @@ pub fn assert(_fnptr: usize, fnname: &str, expected: &str) { // Intrinsics using `cvtpi2ps` are typically "composites" and // in some cases exceed the limit. "cvtpi2ps" => 25, - - // core_arch/src/acle/simd32 - "usad8" => 27, + // core_arch/src/arm_shared/simd32 + // vfmaq_n_f32_vfma : #instructions = 26 >= 22 (limit) + "usad8" | "vfma" | "vfms" => 27, "qadd8" | "qsub8" | "sadd8" | "sel" | "shadd8" | "shsub8" | "usub8" | "ssub8" => 29, + // core_arch/src/arm_shared/simd32 + // vst1q_s64_x4_vst1 : #instructions = 22 >= 22 (limit) + "vld3" => 23, + // core_arch/src/arm_shared/simd32 + // vld4q_lane_u32_vld4 : #instructions = 31 >= 22 (limit) + "vld4" => 32, + // core_arch/src/arm_shared/simd32 + // vst1q_s64_x4_vst1 : #instructions = 40 >= 22 (limit) + "vst1" => 41, + // core_arch/src/arm_shared/simd32 + // vst4q_u32_vst4 : #instructions = 26 >= 22 (limit) + "vst4" => 27, + + // Temporary, currently the fptosi.sat and fptoui.sat LLVM + // intrinsics emit unnecessary code on arm. This can be + // removed once it has been addressed in LLVM. + "fcvtzu" | "fcvtzs" | "vcvt" => 64, + + // core_arch/src/arm_shared/simd32 + // vst1q_p64_x4_nop : #instructions = 33 >= 22 (limit) + "nop" if fnname.contains("vst1q_p64") => 34, // Original limit was 20 instructions, but ARM DSP Intrinsics // are exactly 20 instructions long. So, bump the limit to 22 @@ -161,8 +184,8 @@ pub fn assert(_fnptr: usize, fnname: &str, expected: &str) { ); } else if inlining_failed { panic!( - "instruction found, but the disassembly contains `call` \ - instructions, which hint that inlining failed" + "instruction found, but the disassembly contains subroutine \ + call instructions, which hint that inlining failed" ); } } @@ -175,4 +198,4 @@ pub fn assert_skip_test_ok(name: &str) { } // See comment in `assert-instr-macro` crate for why this exists -pub static _DONT_DEDUP: AtomicPtr = AtomicPtr::new(b"".as_ptr() as *mut _); +pub static mut _DONT_DEDUP: *const u8 = std::ptr::null(); diff --git a/crux-mir/lib/stdarch/crates/stdarch-test/src/wasm.rs b/crux-mir/lib/stdarch/crates/stdarch-test/src/wasm.rs index fc4ced628..bf411c121 100644 --- a/crux-mir/lib/stdarch/crates/stdarch-test/src/wasm.rs +++ b/crux-mir/lib/stdarch/crates/stdarch-test/src/wasm.rs @@ -1,49 +1,17 @@ //! Disassembly calling function for `wasm32` targets. -use wasm_bindgen::prelude::*; use crate::Function; use std::collections::HashSet; -#[wasm_bindgen(module = "child_process")] -extern "C" { - #[wasm_bindgen(js_name = execFileSync)] - fn exec_file_sync(cmd: &str, args: &js_sys::Array, opts: &js_sys::Object) -> Buffer; -} - -#[wasm_bindgen(module = "buffer")] -extern "C" { - type Buffer; - #[wasm_bindgen(method, js_name = toString)] - fn to_string(this: &Buffer) -> String; -} - -#[wasm_bindgen] -extern "C" { - #[wasm_bindgen(js_namespace = require)] - fn resolve(module: &str) -> String; - #[wasm_bindgen(js_namespace = console, js_name = log)] - pub fn js_console_log(s: &str); -} - pub(crate) fn disassemble_myself() -> HashSet { - use std::path::Path; - ::console_error_panic_hook::set_once(); - // Our wasm module in the wasm-bindgen test harness is called - // "wasm-bindgen-test_bg". When running in node this is actually a shim JS - // file. Ask node where that JS file is, and then we use that with a wasm - // extension to find the wasm file itself. - let js_shim = resolve("wasm-bindgen-test_bg"); - let js_shim = Path::new(&js_shim).with_extension("wasm"); - - // Execute `wasm2wat` synchronously, waiting for and capturing all of its - // output. Note that we pass in a custom `maxBuffer` parameter because we're - // generating a ton of output that needs to be buffered. - let args = js_sys::Array::new(); - args.push(&js_shim.display().to_string().into()); - args.push(&"--enable-simd".into()); - let opts = js_sys::Object::new(); - js_sys::Reflect::set(&opts, &"maxBuffer".into(), &(200 * 1024 * 1024).into()).unwrap(); - let output = exec_file_sync("wasm2wat", &args, &opts).to_string(); + // Use `std::env::args` to find the path to our executable. Assume the + // environment is configured such that we can read that file. Read it and + // use the `wasmprinter` crate to transform the binary to text, then search + // the text for appropriately named functions. + let me = std::env::args() + .next() + .expect("failed to find current wasm file"); + let output = wasmprinter::print_file(&me).unwrap(); let mut ret: HashSet = HashSet::new(); let mut lines = output.lines().map(|s| s.trim()); diff --git a/crux-mir/lib/stdarch/crates/stdarch-verify/.gitattributes b/crux-mir/lib/stdarch/crates/stdarch-verify/.gitattributes deleted file mode 100644 index 621fdea6f..000000000 --- a/crux-mir/lib/stdarch/crates/stdarch-verify/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -*.xml binary diff --git a/crux-mir/lib/stdarch/crates/stdarch-verify/src/lib.rs b/crux-mir/lib/stdarch/crates/stdarch-verify/src/lib.rs index d71623c7f..22108d26a 100644 --- a/crux-mir/lib/stdarch/crates/stdarch-verify/src/lib.rs +++ b/crux-mir/lib/stdarch/crates/stdarch-verify/src/lib.rs @@ -1,5 +1,4 @@ -extern crate proc_macro; -extern crate proc_macro2; +#![deny(rust_2018_idioms)] #[macro_use] extern crate quote; #[macro_use] @@ -16,7 +15,14 @@ pub fn x86_functions(input: TokenStream) -> TokenStream { #[proc_macro] pub fn arm_functions(input: TokenStream) -> TokenStream { - functions(input, &["core_arch/src/arm", "core_arch/src/aarch64"]) + functions( + input, + &[ + "core_arch/src/arm", + "core_arch/src/aarch64", + "core_arch/src/arm_shared/neon", + ], + ) } #[proc_macro] @@ -81,6 +87,7 @@ fn functions(input: TokenStream, dirs: &[&str]) -> TokenStream { let name = &f.sig.ident; // println!("{}", name); let mut arguments = Vec::new(); + let mut const_arguments = Vec::new(); for input in f.sig.inputs.iter() { let ty = match *input { syn::FnArg::Typed(ref c) => &c.ty, @@ -88,6 +95,13 @@ fn functions(input: TokenStream, dirs: &[&str]) -> TokenStream { }; arguments.push(to_type(ty)); } + for generic in f.sig.generics.params.iter() { + let ty = match *generic { + syn::GenericParam::Const(ref c) => &c.ty, + _ => panic!("invalid generic argument on {}", name), + }; + const_arguments.push(to_type(ty)); + } let ret = match f.sig.output { syn::ReturnType::Default => quote! { None }, syn::ReturnType::Type(_, ref t) => { @@ -101,7 +115,32 @@ fn functions(input: TokenStream, dirs: &[&str]) -> TokenStream { } else { quote! { None } }; - let required_const = find_required_const(&f.attrs); + + let required_const = find_required_const("rustc_args_required_const", &f.attrs); + let mut legacy_const_generics = + find_required_const("rustc_legacy_const_generics", &f.attrs); + if !required_const.is_empty() && !legacy_const_generics.is_empty() { + panic!( + "Can't have both #[rustc_args_required_const] and \ + #[rustc_legacy_const_generics]" + ); + } + + // The list of required consts, used to verify the arguments, comes from either the + // `rustc_args_required_const` or the `rustc_legacy_const_generics` attribute. + let required_const = if required_const.is_empty() { + legacy_const_generics.clone() + } else { + required_const + }; + + legacy_const_generics.sort(); + for (idx, ty) in legacy_const_generics + .into_iter() + .zip(const_arguments.into_iter()) + { + arguments.insert(idx, ty); + } // strip leading underscore from fn name when building a test // _mm_foo -> mm_foo such that the test name is test_mm_foo. @@ -137,15 +176,25 @@ fn to_type(t: &syn::Type) -> proc_macro2::TokenStream { syn::Type::Path(ref p) => match extract_path_ident(&p.path).to_string().as_ref() { // x86 ... "__m128" => quote! { &M128 }, + "__m128bh" => quote! { &M128BH }, "__m128d" => quote! { &M128D }, "__m128i" => quote! { &M128I }, "__m256" => quote! { &M256 }, + "__m256bh" => quote! { &M256BH }, "__m256d" => quote! { &M256D }, "__m256i" => quote! { &M256I }, "__m512" => quote! { &M512 }, + "__m512bh" => quote! { &M512BH }, "__m512d" => quote! { &M512D }, "__m512i" => quote! { &M512I }, + "__mmask8" => quote! { &MMASK8 }, "__mmask16" => quote! { &MMASK16 }, + "__mmask32" => quote! { &MMASK32 }, + "__mmask64" => quote! { &MMASK64 }, + "_MM_CMPINT_ENUM" => quote! { &MM_CMPINT_ENUM }, + "_MM_MANTISSA_NORM_ENUM" => quote! { &MM_MANTISSA_NORM_ENUM }, + "_MM_MANTISSA_SIGN_ENUM" => quote! { &MM_MANTISSA_SIGN_ENUM }, + "_MM_PERM_ENUM" => quote! { &MM_PERM_ENUM }, "__m64" => quote! { &M64 }, "bool" => quote! { &BOOL }, "f32" => quote! { &F32 }, @@ -159,6 +208,8 @@ fn to_type(t: &syn::Type) -> proc_macro2::TokenStream { "u64" => quote! { &U64 }, "u128" => quote! { &U128 }, "u8" => quote! { &U8 }, + "p8" => quote! { &P8 }, + "p16" => quote! { &P16 }, "Ordering" => quote! { &ORDERING }, "CpuidResult" => quote! { &CPUID }, @@ -174,12 +225,31 @@ fn to_type(t: &syn::Type) -> proc_macro2::TokenStream { "int8x16_t" => quote! { &I8X16 }, "int16x2_t" => quote! { &I16X2 }, "int16x4_t" => quote! { &I16X4 }, + "int16x4x2_t" => quote! { &I16X4X2 }, + "int16x4x3_t" => quote! { &I16X4X3 }, + "int16x4x4_t" => quote! { &I16X4X4 }, "int16x8_t" => quote! { &I16X8 }, + "int16x8x2_t" => quote! { &I16X8X2 }, + "int16x8x3_t" => quote! { &I16X8X3 }, + "int16x8x4_t" => quote! { &I16X8X4 }, "int32x2_t" => quote! { &I32X2 }, + "int32x2x2_t" => quote! { &I32X2X2 }, + "int32x2x3_t" => quote! { &I32X2X3 }, + "int32x2x4_t" => quote! { &I32X2X4 }, "int32x4_t" => quote! { &I32X4 }, + "int32x4x2_t" => quote! { &I32X4X2 }, + "int32x4x3_t" => quote! { &I32X4X3 }, + "int32x4x4_t" => quote! { &I32X4X4 }, "int64x1_t" => quote! { &I64X1 }, + "int64x1x2_t" => quote! { &I64X1X2 }, + "int64x1x3_t" => quote! { &I64X1X3 }, + "int64x1x4_t" => quote! { &I64X1X4 }, "int64x2_t" => quote! { &I64X2 }, + "int64x2x2_t" => quote! { &I64X2X2 }, + "int64x2x3_t" => quote! { &I64X2X3 }, + "int64x2x4_t" => quote! { &I64X2X4 }, "uint8x8_t" => quote! { &U8X8 }, + "uint8x4_t" => quote! { &U8X4 }, "uint8x8x2_t" => quote! { &U8X8X2 }, "uint8x16x2_t" => quote! { &U8X16X2 }, "uint8x16x3_t" => quote! { &U8X16X3 }, @@ -188,15 +258,45 @@ fn to_type(t: &syn::Type) -> proc_macro2::TokenStream { "uint8x8x4_t" => quote! { &U8X8X4 }, "uint8x16_t" => quote! { &U8X16 }, "uint16x4_t" => quote! { &U16X4 }, + "uint16x4x2_t" => quote! { &U16X4X2 }, + "uint16x4x3_t" => quote! { &U16X4X3 }, + "uint16x4x4_t" => quote! { &U16X4X4 }, "uint16x8_t" => quote! { &U16X8 }, + "uint16x8x2_t" => quote! { &U16X8X2 }, + "uint16x8x3_t" => quote! { &U16X8X3 }, + "uint16x8x4_t" => quote! { &U16X8X4 }, "uint32x2_t" => quote! { &U32X2 }, + "uint32x2x2_t" => quote! { &U32X2X2 }, + "uint32x2x3_t" => quote! { &U32X2X3 }, + "uint32x2x4_t" => quote! { &U32X2X4 }, "uint32x4_t" => quote! { &U32X4 }, + "uint32x4x2_t" => quote! { &U32X4X2 }, + "uint32x4x3_t" => quote! { &U32X4X3 }, + "uint32x4x4_t" => quote! { &U32X4X4 }, "uint64x1_t" => quote! { &U64X1 }, + "uint64x1x2_t" => quote! { &U64X1X2 }, + "uint64x1x3_t" => quote! { &U64X1X3 }, + "uint64x1x4_t" => quote! { &U64X1X4 }, "uint64x2_t" => quote! { &U64X2 }, + "uint64x2x2_t" => quote! { &U64X2X2 }, + "uint64x2x3_t" => quote! { &U64X2X3 }, + "uint64x2x4_t" => quote! { &U64X2X4 }, "float32x2_t" => quote! { &F32X2 }, + "float32x2x2_t" => quote! { &F32X2X2 }, + "float32x2x3_t" => quote! { &F32X2X3 }, + "float32x2x4_t" => quote! { &F32X2X4 }, "float32x4_t" => quote! { &F32X4 }, + "float32x4x2_t" => quote! { &F32X4X2 }, + "float32x4x3_t" => quote! { &F32X4X3 }, + "float32x4x4_t" => quote! { &F32X4X4 }, "float64x1_t" => quote! { &F64X1 }, + "float64x1x2_t" => quote! { &F64X1X2 }, + "float64x1x3_t" => quote! { &F64X1X3 }, + "float64x1x4_t" => quote! { &F64X1X4 }, "float64x2_t" => quote! { &F64X2 }, + "float64x2x2_t" => quote! { &F64X2X2 }, + "float64x2x3_t" => quote! { &F64X2X3 }, + "float64x2x4_t" => quote! { &F64X2X4 }, "poly8x8_t" => quote! { &POLY8X8 }, "poly8x8x2_t" => quote! { &POLY8X8X2 }, "poly8x8x3_t" => quote! { &POLY8X8X3 }, @@ -204,11 +304,25 @@ fn to_type(t: &syn::Type) -> proc_macro2::TokenStream { "poly8x16x2_t" => quote! { &POLY8X16X2 }, "poly8x16x3_t" => quote! { &POLY8X16X3 }, "poly8x16x4_t" => quote! { &POLY8X16X4 }, + "p64" => quote! { &P64 }, "poly64x1_t" => quote! { &POLY64X1 }, "poly64x2_t" => quote! { &POLY64X2 }, "poly8x16_t" => quote! { &POLY8X16 }, "poly16x4_t" => quote! { &POLY16X4 }, + "poly16x4x2_t" => quote! { &P16X4X2 }, + "poly16x4x3_t" => quote! { &P16X4X3 }, + "poly16x4x4_t" => quote! { &P16X4X4 }, "poly16x8_t" => quote! { &POLY16X8 }, + "poly16x8x2_t" => quote! { &P16X8X2 }, + "poly16x8x3_t" => quote! { &P16X8X3 }, + "poly16x8x4_t" => quote! { &P16X8X4 }, + "poly64x1x2_t" => quote! { &P64X1X2 }, + "poly64x1x3_t" => quote! { &P64X1X3 }, + "poly64x1x4_t" => quote! { &P64X1X4 }, + "poly64x2x2_t" => quote! { &P64X2X2 }, + "poly64x2x3_t" => quote! { &P64X2X3 }, + "poly64x2x4_t" => quote! { &P64X2X4 }, + "p128" => quote! { &P128 }, "v16i8" => quote! { &v16i8 }, "v8i16" => quote! { &v8i16 }, @@ -222,7 +336,7 @@ fn to_type(t: &syn::Type) -> proc_macro2::TokenStream { "v4f32" => quote! { &v4f32 }, "v2f64" => quote! { &v2f64 }, - s => panic!("unspported type: \"{}\"", s), + s => panic!("unsupported type: \"{}\"", s), }, syn::Type::Ptr(syn::TypePtr { ref elem, @@ -310,7 +424,7 @@ fn find_instrs(attrs: &[syn::Attribute]) -> Vec { // TODO: should probably just reuse `Invoc` from the `assert-instr-macro` // crate. impl syn::parse::Parse for AssertInstr { - fn parse(content: syn::parse::ParseStream) -> syn::Result { + fn parse(content: syn::parse::ParseStream<'_>) -> syn::Result { let input; parenthesized!(input in content); let _ = input.parse::()?; @@ -329,7 +443,7 @@ fn find_instrs(attrs: &[syn::Attribute]) -> Vec { } else if let Ok(ident) = instrs.call(syn::Ident::parse_any) { instr.push_str(&ident.to_string()); } else if instrs.parse::().is_ok() { - instr.push_str("."); + instr.push('.'); } else if instrs.parse::().is_ok() { // consume everything remaining drop(instrs.parse::()); @@ -376,11 +490,11 @@ fn find_target_feature(attrs: &[syn::Attribute]) -> Option { }) } -fn find_required_const(attrs: &[syn::Attribute]) -> Vec { +fn find_required_const(name: &str, attrs: &[syn::Attribute]) -> Vec { attrs .iter() .flat_map(|a| { - if a.path.segments[0].ident == "rustc_args_required_const" { + if a.path.segments[0].ident == name { syn::parse::(a.tokens.clone().into()) .unwrap() .args @@ -396,7 +510,7 @@ struct RustcArgsRequiredConst { } impl syn::parse::Parse for RustcArgsRequiredConst { - fn parse(input: syn::parse::ParseStream) -> syn::Result { + fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { let content; parenthesized!(content in input); let list = diff --git a/crux-mir/lib/stdarch/crates/stdarch-verify/tests/arm.rs b/crux-mir/lib/stdarch/crates/stdarch-verify/tests/arm.rs index 83ba480e0..6ce5ce05f 100644 --- a/crux-mir/lib/stdarch/crates/stdarch-verify/tests/arm.rs +++ b/crux-mir/lib/stdarch/crates/stdarch-verify/tests/arm.rs @@ -149,6 +149,7 @@ static U8X16X2: Type = Type::U(8, 16, 2); static U8X16X3: Type = Type::U(8, 16, 3); static U8X16X4: Type = Type::U(8, 16, 4); static U8X8: Type = Type::U(8, 8, 1); +static U8X4: Type = Type::U(8, 4, 1); static U8X8X2: Type = Type::U(8, 8, 2); static U8X8X3: Type = Type::U(8, 8, 3); static U8X8X4: Type = Type::U(8, 8, 4); @@ -330,6 +331,98 @@ fn verify_all_signatures() { "_rbit_u64", "_cls_u32", "_cls_u64", + "_prefetch", + "vsli_n_s8", + "vsliq_n_s8", + "vsli_n_s16", + "vsliq_n_s16", + "vsli_n_s32", + "vsliq_n_s32", + "vsli_n_s64", + "vsliq_n_s64", + "vsli_n_u8", + "vsliq_n_u8", + "vsli_n_u16", + "vsliq_n_u16", + "vsli_n_u32", + "vsliq_n_u32", + "vsli_n_u64", + "vsliq_n_u64", + "vsli_n_p8", + "vsliq_n_p8", + "vsli_n_p16", + "vsliq_n_p16", + "vsli_n_p64", + "vsliq_n_p64", + "vsri_n_s8", + "vsriq_n_s8", + "vsri_n_s16", + "vsriq_n_s16", + "vsri_n_s32", + "vsriq_n_s32", + "vsri_n_s64", + "vsriq_n_s64", + "vsri_n_u8", + "vsriq_n_u8", + "vsri_n_u16", + "vsriq_n_u16", + "vsri_n_u32", + "vsriq_n_u32", + "vsri_n_u64", + "vsriq_n_u64", + "vsri_n_p8", + "vsriq_n_p8", + "vsri_n_p16", + "vsriq_n_p16", + "vsri_n_p64", + "vsriq_n_p64", + "__smulbb", + "__smultb", + "__smulbt", + "__smultt", + "__smulwb", + "__smulwt", + "__qadd", + "__qsub", + "__qdbl", + "__smlabb", + "__smlabt", + "__smlatb", + "__smlatt", + "__smlawb", + "__smlawt", + "__qadd8", + "__qsub8", + "__qsub16", + "__qadd16", + "__qasx", + "__qsax", + "__sadd16", + "__sadd8", + "__smlad", + "__smlsd", + "__sasx", + "__sel", + "__shadd8", + "__shadd16", + "__shsub8", + "__usub8", + "__ssub8", + "__shsub16", + "__smuad", + "__smuadx", + "__smusd", + "__smusdx", + "__usad8", + "__usada8", + "__ldrex", + "__strex", + "__ldrexb", + "__strexb", + "__ldrexh", + "__strexh", + "__clrex", + "__dbg", ]; if !skip.contains(&rust.name) { println!( @@ -350,9 +443,161 @@ fn verify_all_signatures() { // Skip some intrinsics that aren't NEON and are located in different // places than the whitelists below. match rust.name { - "brk" | "__breakpoint" | "udf" => continue, + "brk" | "__breakpoint" | "udf" | "_prefetch" => continue, _ => {} } + // Skip some intrinsics that are present in GCC and Clang but + // are missing from the official documentation. + let skip_intrinsic_verify = [ + "vmov_n_p64", + "vmovq_n_p64", + "vreinterpret_p64_s64", + "vreinterpret_f32_p64", + "vreinterpretq_f32_p64", + "vreinterpretq_p64_p128", + "vreinterpretq_p128_p64", + "vreinterpretq_f32_p128", + "vqrdmlahh_s16", + "vqrdmlahs_s32", + "vqrdmlahh_lane_s16", + "vqrdmlahh_laneq_s16", + "vqrdmlahs_lane_s32", + "vqrdmlahs_laneq_s32", + "vqrdmlah_s16", + "vqrdmlah_s32", + "vqrdmlahq_s16", + "vqrdmlahq_s32", + "vqrdmlah_lane_s16", + "vqrdmlah_laneq_s16", + "vqrdmlahq_lane_s16", + "vqrdmlahq_laneq_s16", + "vqrdmlah_lane_s32", + "vqrdmlah_laneq_s32", + "vqrdmlahq_lane_s32", + "vqrdmlahq_laneq_s32", + "vqrdmlshh_s16", + "vqrdmlshs_s32", + "vqrdmlshh_lane_s16", + "vqrdmlshh_laneq_s16", + "vqrdmlshs_lane_s32", + "vqrdmlshs_laneq_s32", + "vqrdmlsh_s16", + "vqrdmlshq_s16", + "vqrdmlsh_s32", + "vqrdmlshq_s32", + "vqrdmlsh_lane_s16", + "vqrdmlsh_laneq_s16", + "vqrdmlshq_lane_s16", + "vqrdmlshq_laneq_s16", + "vqrdmlsh_lane_s32", + "vqrdmlsh_laneq_s32", + "vqrdmlshq_lane_s32", + "vqrdmlshq_laneq_s32", + "vcadd_rot270_f32", + "vcadd_rot90_f32", + "vcaddq_rot270_f32", + "vcaddq_rot270_f64", + "vcaddq_rot90_f32", + "vcaddq_rot90_f64", + "vcmla_f32", + "vcmlaq_f32", + "vcmlaq_f64", + "vcmla_rot90_f32", + "vcmlaq_rot90_f32", + "vcmlaq_rot90_f64", + "vcmla_rot180_f32", + "vcmlaq_rot180_f32", + "vcmlaq_rot180_f64", + "vcmla_rot270_f32", + "vcmlaq_rot270_f32", + "vcmlaq_rot270_f64", + "vcmla_lane_f32", + "vcmla_laneq_f32", + "vcmlaq_lane_f32", + "vcmlaq_laneq_f32", + "vcmla_rot90_lane_f32", + "vcmla_rot90_laneq_f32", + "vcmlaq_rot90_lane_f32", + "vcmlaq_rot90_laneq_f32", + "vcmla_rot180_lane_f32", + "vcmla_rot180_laneq_f32", + "vcmlaq_rot180_lane_f32", + "vcmlaq_rot180_laneq_f32", + "vcmla_rot270_lane_f32", + "vcmla_rot270_laneq_f32", + "vcmlaq_rot270_lane_f32", + "vcmlaq_rot270_laneq_f32", + "vdot_s32", + "vdot_u32", + "vdotq_s32", + "vdotq_u32", + "vdot_lane_s32", + "vdot_laneq_s32", + "vdotq_lane_s32", + "vdotq_laneq_s32", + "vdot_lane_u32", + "vdot_laneq_u32", + "vdotq_lane_u32", + "vdotq_laneq_u32", + "vbcaxq_s8", + "vbcaxq_s16", + "vbcaxq_s32", + "vbcaxq_s64", + "vbcaxq_u8", + "vbcaxq_u16", + "vbcaxq_u32", + "vbcaxq_u64", + "veor3q_s8", + "veor3q_s16", + "veor3q_s32", + "veor3q_s64", + "veor3q_u8", + "veor3q_u16", + "veor3q_u32", + "veor3q_u64", + "vadd_p8", + "vadd_p16", + "vadd_p64", + "vaddq_p8", + "vaddq_p16", + "vaddq_p64", + "vaddq_p128", + "vsm4ekeyq_u32", + "vsm4eq_u32", + "vmmlaq_s32", + "vmmlaq_u32", + "vusmmlaq_s32", + "vsm3partw1q_u32", + "vsm3partw2q_u32", + "vsm3ss1q_u32", + "vsm3tt1aq_u32", + "vsm3tt1bq_u32", + "vsm3tt2aq_u32", + "vsm3tt2bq_u32", + "vrax1q_u64", + "vxarq_u64", + "vsha512hq_u64", + "vsha512h2q_u64", + "vsha512su0q_u64", + "vsha512su1q_u64", + "vrnd32x_f32", + "vrnd32xq_f32", + "vrnd32z_f32", + "vrnd32zq_f32", + "vrnd64x_f32", + "vrnd64xq_f32", + "vrnd64z_f32", + "vrnd64zq_f32", + "vcls_u8", + "vcls_u16", + "vcls_u32", + "vclsq_u8", + "vclsq_u16", + "vclsq_u32", + "vtst_p16", + "vtstq_p16", + "__dbg", + ]; let arm = match map.get(rust.name) { Some(i) => i, None => { @@ -362,10 +607,14 @@ fn verify_all_signatures() { // TODO: we still need to verify these intrinsics or find a // reference for them, need to figure out where though! if !rust.file.ends_with("dsp.rs\"") + && !rust.file.ends_with("simd32.rs\"") && !rust.file.ends_with("cmsis.rs\"") && !rust.file.ends_with("v6.rs\"") && !rust.file.ends_with("v7.rs\"") && !rust.file.ends_with("v8.rs\"") + && !rust.file.ends_with("tme.rs\"") + && !rust.file.ends_with("ex.rs\"") + && !skip_intrinsic_verify.contains(&rust.name) { println!( "missing arm definition for {:?} in {}", @@ -426,7 +675,7 @@ fn matches(rust: &Function, arm: &Intrinsic) -> Result<(), String> { } // sometimes arm says `foo` and disassemblers say `vfoo`, or // sometimes disassemblers say `vfoo` and arm says `sfoo` or `ffoo` - if instr.starts_with("v") + if instr.starts_with('v') && (arm.instruction.starts_with(&instr[1..]) || arm.instruction[1..].starts_with(&instr[1..])) { @@ -449,10 +698,10 @@ fn matches(rust: &Function, arm: &Intrinsic) -> Result<(), String> { fn find_accordion(node: &Rc) -> Option> { if let NodeData::Element { attrs, .. } = &node.data { for attr in attrs.borrow().iter() { - if attr.name.local.eq_str_ignore_ascii_case("class") { - if attr.value.to_string() == "intrinsic-accordion" { - return Some(node.clone()); - } + if attr.name.local.eq_str_ignore_ascii_case("class") + && attr.value.to_string() == "intrinsic-accordion" + { + return Some(node.clone()); } } } @@ -480,7 +729,7 @@ fn parse_intrinsics(node: &Rc) -> HashMap { ret.insert(f.name.clone(), f); } } - return ret; + ret } fn parse_intrinsic(node: &Rc) -> Intrinsic { @@ -493,10 +742,9 @@ fn parse_intrinsic(node: &Rc) -> Intrinsic { // ... let children = node.children.borrow(); - let mut children = children.iter().filter(|node| match node.data { - NodeData::Element { .. } => true, - _ => false, - }); + let mut children = children + .iter() + .filter(|node| matches!(node.data, NodeData::Element { .. })); let _input = children.next().expect("no "); let label = children.next().expect("no
"); @@ -516,10 +764,9 @@ fn parse_intrinsic(node: &Rc) -> Intrinsic { // Find contents of inner `
` in `
in { - + type Output : ?Sized; - - fn findex(&self, i:usize, j:A, k:B) -> &Self::Output; - + + fn findex(&self, i:usize, j:A, k:B) -> &Self::Output; + } impl FIndex for [u8] { @@ -39,4 +39,4 @@ const ARG: u8 = 23; pub fn main() { println!("{:?}", f(ARG)); } -#[cfg(not(with_main))] #[cfg_attr(crux, crux_test)] fn crux_test() -> u8 { f(ARG) } +#[cfg(not(with_main))] #[cfg_attr(crux, crux::test)] fn crux_test() -> u8 { f(ARG) } diff --git a/crux-mir/test/conc_eval/traits/tyfam3.rs b/crux-mir/test/conc_eval/traits/tyfam3.rs index 7d0613007..caa7fcb76 100644 --- a/crux-mir/test/conc_eval/traits/tyfam3.rs +++ b/crux-mir/test/conc_eval/traits/tyfam3.rs @@ -1,10 +1,10 @@ #![cfg_attr(not(with_main), no_std)] trait F : Sized { - + type Output : Sized; - fn ff(self) -> Self::Output; - + fn ff(self) -> Self::Output; + } trait G { @@ -39,4 +39,4 @@ const ARG: u8 = 23; pub fn main() { println!("{:?}", f(ARG)); } -#[cfg(not(with_main))] #[cfg_attr(crux, crux_test)] fn crux_test() -> u8 { f(ARG) } +#[cfg(not(with_main))] #[cfg_attr(crux, crux::test)] fn crux_test() -> u8 { f(ARG) } diff --git a/crux-mir/test/conc_eval/traits/tyfam4.rs b/crux-mir/test/conc_eval/traits/tyfam4.rs index ceff8aaf0..850a33936 100644 --- a/crux-mir/test/conc_eval/traits/tyfam4.rs +++ b/crux-mir/test/conc_eval/traits/tyfam4.rs @@ -1,9 +1,9 @@ #![cfg_attr(not(with_main), no_std)] trait F : Sized { - + type Output : Sized; - fn ff(self) -> Self::Output; - + fn ff(self) -> Self::Output; + } impl F for [u8;5] { @@ -34,4 +34,4 @@ const ARG: u8 = 23; pub fn main() { println!("{:?}", f(ARG)); } -#[cfg(not(with_main))] #[cfg_attr(crux, crux_test)] fn crux_test() -> u8 { f(ARG) } +#[cfg(not(with_main))] #[cfg_attr(crux, crux::test)] fn crux_test() -> u8 { f(ARG) } diff --git a/crux-mir/test/conc_eval/traits/tyfam5.rs b/crux-mir/test/conc_eval/traits/tyfam5.rs index f1f689610..338b1cc82 100644 --- a/crux-mir/test/conc_eval/traits/tyfam5.rs +++ b/crux-mir/test/conc_eval/traits/tyfam5.rs @@ -7,7 +7,7 @@ pub enum Opt { } use Opt::*; - + pub trait Ir { type Item; fn dummy(x:Self::Item) -> Self::Item; @@ -34,4 +34,4 @@ fn f(arg: i32) { pub fn main() { println!("{:?}", f(ARG)); } -#[cfg(not(with_main))] #[cfg_attr(crux, crux_test)] fn crux_test() -> () { f(ARG) } +#[cfg(not(with_main))] #[cfg_attr(crux, crux::test)] fn crux_test() -> () { f(ARG) } diff --git a/crux-mir/test/conc_eval/tuple/clone.rs b/crux-mir/test/conc_eval/tuple/clone.rs index a3d743f3f..d9e445ac9 100644 --- a/crux-mir/test/conc_eval/tuple/clone.rs +++ b/crux-mir/test/conc_eval/tuple/clone.rs @@ -1,5 +1,5 @@ -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let x = (1, 2); let y = x.clone(); diff --git a/crux-mir/test/conc_eval/tuple/clone_from.rs b/crux-mir/test/conc_eval/tuple/clone_from.rs index 34f95a789..f1dacd246 100644 --- a/crux-mir/test/conc_eval/tuple/clone_from.rs +++ b/crux-mir/test/conc_eval/tuple/clone_from.rs @@ -1,4 +1,4 @@ -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let x = (1, 2); let mut y = (0, 0); diff --git a/crux-mir/test/conc_eval/tuple/clone_rec.rs b/crux-mir/test/conc_eval/tuple/clone_rec.rs index eb5eec2ae..e7565e8dc 100644 --- a/crux-mir/test/conc_eval/tuple/clone_rec.rs +++ b/crux-mir/test/conc_eval/tuple/clone_rec.rs @@ -1,5 +1,5 @@ -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let x = (1, (2, (3, 4))); let y = x.clone(); diff --git a/crux-mir/test/conc_eval/tuple/clone_struct.rs b/crux-mir/test/conc_eval/tuple/clone_struct.rs index be6277442..65caa59bb 100644 --- a/crux-mir/test/conc_eval/tuple/clone_struct.rs +++ b/crux-mir/test/conc_eval/tuple/clone_struct.rs @@ -2,7 +2,7 @@ #[derive(Clone, PartialEq, Eq)] struct S; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let x = (S, S); let y = x.clone(); diff --git a/crux-mir/test/conc_eval/vec/collect.rs b/crux-mir/test/conc_eval/vec/collect.rs index 92f63867b..75ca13d87 100644 --- a/crux-mir/test/conc_eval/vec/collect.rs +++ b/crux-mir/test/conc_eval/vec/collect.rs @@ -1,4 +1,4 @@ -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { let xs = (0..10).collect::>(); let mut sum = 0; diff --git a/crux-mir/test/conc_eval/vec/drop.rs b/crux-mir/test/conc_eval/vec/drop.rs index 6195f5bac..4f6c1fa11 100644 --- a/crux-mir/test/conc_eval/vec/drop.rs +++ b/crux-mir/test/conc_eval/vec/drop.rs @@ -9,7 +9,7 @@ impl Drop for S { } } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() { let v = vec![S(1), S(2), S(3)]; drop(v); diff --git a/crux-mir/test/conc_eval/vec/extend.rs b/crux-mir/test/conc_eval/vec/extend.rs index d581956fe..b06bb097a 100644 --- a/crux-mir/test/conc_eval/vec/extend.rs +++ b/crux-mir/test/conc_eval/vec/extend.rs @@ -1,6 +1,6 @@ use std::ptr; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> (i32, i32) { let mut v = Vec::new(); v.push(1); diff --git a/crux-mir/test/conc_eval/vec/extend_trusted_len.rs b/crux-mir/test/conc_eval/vec/extend_trusted_len.rs index 78f937c7d..c6cbd5fac 100644 --- a/crux-mir/test/conc_eval/vec/extend_trusted_len.rs +++ b/crux-mir/test/conc_eval/vec/extend_trusted_len.rs @@ -1,6 +1,6 @@ use std::ptr; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> (i32, i32) { let mut v = Vec::new(); v.push(1); diff --git a/crux-mir/test/conc_eval/vec/from_elem_zero.rs b/crux-mir/test/conc_eval/vec/from_elem_zero.rs index ebd1c90bf..217cb93e8 100644 --- a/crux-mir/test/conc_eval/vec/from_elem_zero.rs +++ b/crux-mir/test/conc_eval/vec/from_elem_zero.rs @@ -1,4 +1,4 @@ -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { // `vec::from_elem` (which backs the `vec![x; N]` macro syntax) special-cases `x == 0` to use // `alloc_zeroed`. It expects the allocator to initialize the vector to all zeros, and doesn't diff --git a/crux-mir/test/conc_eval/vec/push.rs b/crux-mir/test/conc_eval/vec/push.rs index b167219bd..ce4de3a25 100644 --- a/crux-mir/test/conc_eval/vec/push.rs +++ b/crux-mir/test/conc_eval/vec/push.rs @@ -1,4 +1,4 @@ -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> (i32, i32) { let mut v = Vec::new(); v.push(1); diff --git a/crux-mir/test/conc_eval/vec/set_len.rs b/crux-mir/test/conc_eval/vec/set_len.rs index f5fcf3327..1b49ca5bf 100644 --- a/crux-mir/test/conc_eval/vec/set_len.rs +++ b/crux-mir/test/conc_eval/vec/set_len.rs @@ -1,4 +1,4 @@ -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> usize { let mut v = vec![1, 2, 3]; unsafe { v.set_len(2); } diff --git a/crux-mir/test/conc_eval/vec_deque/iter_clone.rs b/crux-mir/test/conc_eval/vec_deque/iter_clone.rs index 52be77f73..ecf4018ad 100644 --- a/crux-mir/test/conc_eval/vec_deque/iter_clone.rs +++ b/crux-mir/test/conc_eval/vec_deque/iter_clone.rs @@ -1,6 +1,6 @@ use std::collections::VecDeque; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> [i32; 5] { let mut v: VecDeque<_> = vec![1, 2, 3, 4, 5].into(); let mut it = v.iter(); diff --git a/crux-mir/test/conc_eval/vec_deque/pop.rs b/crux-mir/test/conc_eval/vec_deque/pop.rs index 9b97cfc1b..229b8df51 100644 --- a/crux-mir/test/conc_eval/vec_deque/pop.rs +++ b/crux-mir/test/conc_eval/vec_deque/pop.rs @@ -1,6 +1,6 @@ use std::collections::VecDeque; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> [i32; 5] { let mut v: VecDeque<_> = vec![1, 2, 3, 4, 5].into(); [ diff --git a/crux-mir/test/conc_eval/vec_deque/push.rs b/crux-mir/test/conc_eval/vec_deque/push.rs index c390ff0d7..404854acd 100644 --- a/crux-mir/test/conc_eval/vec_deque/push.rs +++ b/crux-mir/test/conc_eval/vec_deque/push.rs @@ -1,6 +1,6 @@ use std::collections::VecDeque; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> [i32; 5] { let mut v = VecDeque::new(); v.push_back(1); diff --git a/crux-mir/test/conc_eval/vec_deque/retain.rs b/crux-mir/test/conc_eval/vec_deque/retain.rs index 8c4ecf5ed..896096396 100644 --- a/crux-mir/test/conc_eval/vec_deque/retain.rs +++ b/crux-mir/test/conc_eval/vec_deque/retain.rs @@ -1,6 +1,6 @@ use std::collections::VecDeque; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> [i32; 2] { let mut v: VecDeque<_> = vec![1, 2, 3, 4, 5].into(); v.retain(|&x| x % 3 == 1); diff --git a/crux-mir/test/conc_eval/vec_deque/rotate_left.rs b/crux-mir/test/conc_eval/vec_deque/rotate_left.rs index 60df96979..fbefa0c57 100644 --- a/crux-mir/test/conc_eval/vec_deque/rotate_left.rs +++ b/crux-mir/test/conc_eval/vec_deque/rotate_left.rs @@ -1,6 +1,6 @@ use std::collections::VecDeque; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> [i32; 5] { let mut v: VecDeque<_> = vec![1, 2, 3, 4, 5].into(); v.rotate_left(2); diff --git a/crux-mir/test/conc_eval/vec_deque/rotate_right.rs b/crux-mir/test/conc_eval/vec_deque/rotate_right.rs index 877dab516..39d1e047b 100644 --- a/crux-mir/test/conc_eval/vec_deque/rotate_right.rs +++ b/crux-mir/test/conc_eval/vec_deque/rotate_right.rs @@ -1,6 +1,6 @@ use std::collections::VecDeque; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> [i32; 5] { let mut v: VecDeque<_> = vec![1, 2, 3, 4, 5].into(); v.rotate_right(2); diff --git a/crux-mir/test/concurrency/atomic/atomic.rs b/crux-mir/test/concurrency/atomic/atomic.rs index 285aed0cd..91555dcaa 100644 --- a/crux-mir/test/concurrency/atomic/atomic.rs +++ b/crux-mir/test/concurrency/atomic/atomic.rs @@ -4,7 +4,7 @@ use std::thread; use std::sync::{Arc,atomic}; #[cfg(not(with_main))] -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test_nofail() { let N = 3; diff --git a/crux-mir/test/concurrency/atomic/atomic_fetch.rs b/crux-mir/test/concurrency/atomic/atomic_fetch.rs index d58fefb07..317239538 100644 --- a/crux-mir/test/concurrency/atomic/atomic_fetch.rs +++ b/crux-mir/test/concurrency/atomic/atomic_fetch.rs @@ -4,7 +4,7 @@ use std::thread; use std::sync::{Arc,atomic}; #[cfg(not(with_main))] -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test_nofail() { let N = 3; diff --git a/crux-mir/test/concurrency/mutex/mutex.rs b/crux-mir/test/concurrency/mutex/mutex.rs index 8bfaa2301..9d262da15 100644 --- a/crux-mir/test/concurrency/mutex/mutex.rs +++ b/crux-mir/test/concurrency/mutex/mutex.rs @@ -4,7 +4,7 @@ use std::thread; use std::sync::{Arc,Mutex}; #[cfg(not(with_main))] -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test_nofail() { let data = Arc::new(Mutex::new(0 as u32)); let N = 3; @@ -37,7 +37,7 @@ fn crux_test_nofail() { data.crucible_TEMP_unlock(); } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test_fail() { let data = Arc::new(Mutex::new(0 as u32)); let N = 2; diff --git a/crux-mir/test/concurrency/mutex/mutex_atomic.rs b/crux-mir/test/concurrency/mutex/mutex_atomic.rs index b1a763f39..f72ce0f73 100644 --- a/crux-mir/test/concurrency/mutex/mutex_atomic.rs +++ b/crux-mir/test/concurrency/mutex/mutex_atomic.rs @@ -6,7 +6,7 @@ use std::sync::atomic::AtomicU16; use std::sync::atomic::Ordering::SeqCst; #[cfg(not(with_main))] -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test_nofail() { let data = Arc::new(Mutex::new(())); let ctr = Arc::new(AtomicU16::new(0)); diff --git a/crux-mir/test/concurrency/mutex/thread_retval.rs b/crux-mir/test/concurrency/mutex/thread_retval.rs index d517a0d12..548f029a8 100644 --- a/crux-mir/test/concurrency/mutex/thread_retval.rs +++ b/crux-mir/test/concurrency/mutex/thread_retval.rs @@ -4,7 +4,7 @@ use std::thread; use std::sync::{Arc,Mutex}; #[cfg(not(with_main))] -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() { let N = 2; let V = 3; diff --git a/crux-mir/test/concurrency/mutex/thread_retval_mutex.rs b/crux-mir/test/concurrency/mutex/thread_retval_mutex.rs index 12bb062cf..87463402c 100644 --- a/crux-mir/test/concurrency/mutex/thread_retval_mutex.rs +++ b/crux-mir/test/concurrency/mutex/thread_retval_mutex.rs @@ -4,7 +4,7 @@ use std::thread; use std::sync::{Arc,Mutex}; #[cfg(not(with_main))] -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() { let data = Arc::new(Mutex::new(0 as u32)); let N = 3; diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread-C-DAC/pthread-finding-k-matches.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread-C-DAC/pthread-finding-k-matches.rs index 619cef065..85ec50b50 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread-C-DAC/pthread-finding-k-matches.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread-C-DAC/pthread-finding-k-matches.rs @@ -37,7 +37,7 @@ fn find_entries(tid:usize, count.crucible_TEMP_unlock(); } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn pthread_finding_k_matches() { let mut vals = vec![]; diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/bigshot.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/bigshot.rs index b8d16570b..91406e4cd 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/bigshot.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/bigshot.rs @@ -5,7 +5,7 @@ use std::thread; use std::sync::{Arc,Mutex}; #[cfg(not(with_main))] -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn bigshot_p() { let v = Arc::new(Mutex::new(None)); @@ -35,7 +35,7 @@ fn bigshot_p() { } #[cfg(not(with_main))] -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn bigshot_s() { let v = Arc::new(Mutex::new(None)); @@ -65,7 +65,7 @@ fn bigshot_s() { } #[cfg(not(with_main))] -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn bigshot_s2() { let v = Arc::new(Mutex::new(None)); diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/fib_bench.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/fib_bench.rs index 5758f61a4..e83212809 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/fib_bench.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/fib_bench.rs @@ -35,7 +35,7 @@ fn calc_fib() -> i32 { j } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test_fail() { let data = Arc::new(Mutex::new((1 as i32, 1 as i32))); @@ -59,7 +59,7 @@ fn crux_test_fail() { data.crucible_TEMP_unlock(); } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() { let data = Arc::new(Mutex::new((1 as i32, 1 as i32))); diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/indexer.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/indexer.rs index 808e154a9..22a24e724 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/indexer.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/indexer.rs @@ -24,7 +24,7 @@ fn cas(tab:&Arc>>, h:usize, val:i32, new_val:i32) -> i32 } #[cfg(not(with_main))] -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn indexer() { let mut tab = vec![]; let mut ts = vec![]; diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/queue.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/queue.rs index e1f681da5..b41bac011 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/queue.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/queue.rs @@ -125,7 +125,7 @@ fn t2_bad(m : Arc>) { } } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test_fails() { let mut q = Queue::new(); q.enqueue_flag = true; @@ -149,7 +149,7 @@ fn crux_test_fails() { h2.join(); } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test_succeeds() { let mut q = Queue::new(); q.enqueue_flag = true; diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/reorder.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/reorder.rs index ed6d70b9c..99aa4b51e 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/reorder.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/reorder.rs @@ -39,12 +39,12 @@ fn run_test(numSet: usize, numCheck: usize) { } } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn reorder2() { run_test(2,2); } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn reorder5() { run_test(4,1); } diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/singleton.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/singleton.rs index 26f3146cb..c321111f0 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/singleton.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/singleton.rs @@ -46,7 +46,7 @@ fn thread0(v: Arc>>) { } } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn singleton() { let v = Arc::new(Mutex::new(None)); let vv = v.clone(); diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/stack.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/stack.rs index 6a0075583..75e7b9b2a 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/stack.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/stack.rs @@ -85,7 +85,7 @@ fn t2(stack: Arc>, with_flag: bool) } } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn stack_1() { let stk = Arc::new(Mutex::new(Stack::new())); @@ -100,7 +100,7 @@ fn stack_1() h2.join(); } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn stack_2() { let stk = Arc::new(Mutex::new(Stack::new())); diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/stateful.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/stateful.rs index e50eaa738..2578b5acb 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/stateful.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/stateful.rs @@ -45,14 +45,14 @@ fn run_threads() -> (i32, i32) return r } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn stateful01_1() { let (d1, d2) = run_threads(); crucible_assert!( !(d1 == 16 && d2 == 5) ); } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn stateful01_2() { let (d1, d2) = run_threads(); diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/triangular.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/triangular.rs index f0753927e..8687d2802 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/triangular.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/triangular.rs @@ -31,7 +31,7 @@ fn t2(i:Arc, j:Arc, m:Arc>) } } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn triangular_1() { let i = Arc::new(AtomicI32::new(3)); @@ -58,7 +58,7 @@ fn triangular_1() crucible_assert!(! (condI || condJ) ); } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn triangular_2() { let i = Arc::new(AtomicI32::new(3)); diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/triangular_static.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/triangular_static.rs index b3b3c8243..dfc6b0536 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/triangular_static.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/triangular_static.rs @@ -34,7 +34,7 @@ fn t2(m:Arc>) } } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn triangular_1() { let m = Arc::new(Mutex::new(())); @@ -54,7 +54,7 @@ fn triangular_1() crucible_assert!(! (condI || condJ) ); } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn triangular_2() { let m = Arc::new(Mutex::new(())); diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/twostage.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/twostage.rs index ab0826dfa..a0627ce4b 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/twostage.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/twostage.rs @@ -59,7 +59,7 @@ fn twostage_test(t_threads: usize, r_threads: usize) { } } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn twostage_3() { twostage_test(2,1); } diff --git a/crux-mir/test/coverage/coverage.rs b/crux-mir/test/coverage/coverage.rs index 458168fd1..c188ce2db 100644 --- a/crux-mir/test/coverage/coverage.rs +++ b/crux-mir/test/coverage/coverage.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { let x = u8::symbolic("x"); crucible_assume!(x < 2); diff --git a/crux-mir/test/coverage/coverage_cond.rs b/crux-mir/test/coverage/coverage_cond.rs index feb65a54c..8b866db05 100644 --- a/crux-mir/test/coverage/coverage_cond.rs +++ b/crux-mir/test/coverage/coverage_cond.rs @@ -10,7 +10,7 @@ fn g(cond: bool) -> u8 { if !cond { 10 } else { 20 } } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> u8 { f(true) + g(true) } diff --git a/crux-mir/test/coverage/coverage_dual.rs b/crux-mir/test/coverage/coverage_dual.rs index 9c67151ec..b41add186 100644 --- a/crux-mir/test/coverage/coverage_dual.rs +++ b/crux-mir/test/coverage/coverage_dual.rs @@ -6,12 +6,12 @@ fn f(c: bool, t: i32, e: i32) -> i32 { if c { t } else { e } } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test1() -> i32 { f(false, 1, 2) } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test2() -> i32 { f(true, 10, 20) } diff --git a/crux-mir/test/coverage/coverage_loop.rs b/crux-mir/test/coverage/coverage_loop.rs index 682d48563..b7e9114b1 100644 --- a/crux-mir/test/coverage/coverage_loop.rs +++ b/crux-mir/test/coverage/coverage_loop.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { let mut y = 1; for i in 0..10 { diff --git a/crux-mir/test/coverage/coverage_macro.rs b/crux-mir/test/coverage/coverage_macro.rs index fad0660ff..ea30eef3e 100644 --- a/crux-mir/test/coverage/coverage_macro.rs +++ b/crux-mir/test/coverage/coverage_macro.rs @@ -19,7 +19,7 @@ fn f() -> Option { Some(1) } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> u8 { f().unwrap_or(0) } diff --git a/crux-mir/test/coverage/coverage_match.rs b/crux-mir/test/coverage/coverage_match.rs index 327ef327d..c7e7e8fe3 100644 --- a/crux-mir/test/coverage/coverage_match.rs +++ b/crux-mir/test/coverage/coverage_match.rs @@ -7,7 +7,7 @@ enum E { C = 30, } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { let x = u8::symbolic("x"); crucible_assume!(x != 1); diff --git a/crux-mir/test/coverage/coverage_mono.rs b/crux-mir/test/coverage/coverage_mono.rs index 10dd52be1..78563eb1d 100644 --- a/crux-mir/test/coverage/coverage_mono.rs +++ b/crux-mir/test/coverage/coverage_mono.rs @@ -6,7 +6,7 @@ fn f(c: bool, t: T, e: T) -> T { if c { t } else { e } } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { f::(false, 1_u8, 2_u8) as i32 + f::(true, 10, 20) } diff --git a/crux-mir/test/coverage/coverage_shortcircuit.rs b/crux-mir/test/coverage/coverage_shortcircuit.rs index e317ece27..2b9330f15 100644 --- a/crux-mir/test/coverage/coverage_shortcircuit.rs +++ b/crux-mir/test/coverage/coverage_shortcircuit.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { let x = u8::symbolic("x"); crucible_assume!(x < 1); diff --git a/crux-mir/test/coverage/coverage_try.rs b/crux-mir/test/coverage/coverage_try.rs index de87c14ef..68a85ea52 100644 --- a/crux-mir/test/coverage/coverage_try.rs +++ b/crux-mir/test/coverage/coverage_try.rs @@ -11,7 +11,7 @@ fn f() -> Option { x.checked_add(y) } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> u8 { f().unwrap_or(0) } diff --git a/crux-mir/test/symb_eval/alloc/out_of_bounds.rs b/crux-mir/test/symb_eval/alloc/out_of_bounds.rs index f628ea9d6..1c82780ae 100644 --- a/crux-mir/test/symb_eval/alloc/out_of_bounds.rs +++ b/crux-mir/test/symb_eval/alloc/out_of_bounds.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::*; use crucible::alloc::allocate; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { unsafe { let ptr = allocate::(10); diff --git a/crux-mir/test/symb_eval/alloc/uninit_read.rs b/crux-mir/test/symb_eval/alloc/uninit_read.rs index 76904167d..b024c2c82 100644 --- a/crux-mir/test/symb_eval/alloc/uninit_read.rs +++ b/crux-mir/test/symb_eval/alloc/uninit_read.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::*; use crucible::alloc::allocate; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { unsafe { let ptr = allocate::(10); diff --git a/crux-mir/test/symb_eval/alloc/valid_read.rs b/crux-mir/test/symb_eval/alloc/valid_read.rs index 348a979fb..8f2def682 100644 --- a/crux-mir/test/symb_eval/alloc/valid_read.rs +++ b/crux-mir/test/symb_eval/alloc/valid_read.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::*; use crucible::alloc::allocate; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { unsafe { let ptr = allocate::(10); diff --git a/crux-mir/test/symb_eval/alloc/zero_length.rs b/crux-mir/test/symb_eval/alloc/zero_length.rs index c83bedf95..d9b8ec20c 100644 --- a/crux-mir/test/symb_eval/alloc/zero_length.rs +++ b/crux-mir/test/symb_eval/alloc/zero_length.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::*; use crucible::alloc::allocate; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() { unsafe { // Make sure the CustomOp succeeds in creating a pointer to the first element of an empty diff --git a/crux-mir/test/symb_eval/any/downcast.rs b/crux-mir/test/symb_eval/any/downcast.rs index bc622a133..a467d2809 100644 --- a/crux-mir/test/symb_eval/any/downcast.rs +++ b/crux-mir/test/symb_eval/any/downcast.rs @@ -3,7 +3,7 @@ extern crate crucible; use crucible::any::Any; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { let x: i32 = 1; let a = Any::new(x); diff --git a/crux-mir/test/symb_eval/any/downcast_fail.rs b/crux-mir/test/symb_eval/any/downcast_fail.rs index 192632c6e..cfc5705cc 100644 --- a/crux-mir/test/symb_eval/any/downcast_fail.rs +++ b/crux-mir/test/symb_eval/any/downcast_fail.rs @@ -3,7 +3,7 @@ extern crate crucible; use crucible::any::Any; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { let x: () = (); let a = Any::new(x); diff --git a/crux-mir/test/symb_eval/array/basic.rs b/crux-mir/test/symb_eval/array/basic.rs index 542f164ea..37ad426a4 100644 --- a/crux-mir/test/symb_eval/array/basic.rs +++ b/crux-mir/test/symb_eval/array/basic.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::array::Array; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { let arr = Array::::zeroed(); let arr = arr.update(0, 0); diff --git a/crux-mir/test/symb_eval/array/mux_slice.rs b/crux-mir/test/symb_eval/array/mux_slice.rs index 1caa054db..b9fe40042 100644 --- a/crux-mir/test/symb_eval/array/mux_slice.rs +++ b/crux-mir/test/symb_eval/array/mux_slice.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::array::Array; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { let mut arr1 = Array::::zeroed(); let mut arr2 = Array::::zeroed(); diff --git a/crux-mir/test/symb_eval/array/slice.rs b/crux-mir/test/symb_eval/array/slice.rs index e90fccbe8..be1e9d260 100644 --- a/crux-mir/test/symb_eval/array/slice.rs +++ b/crux-mir/test/symb_eval/array/slice.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::array::Array; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { let mut arr = Array::::zeroed(); for i in 0..10 { diff --git a/crux-mir/test/symb_eval/array/slice_mut.rs b/crux-mir/test/symb_eval/array/slice_mut.rs index 9b1c86514..35d9b0363 100644 --- a/crux-mir/test/symb_eval/array/slice_mut.rs +++ b/crux-mir/test/symb_eval/array/slice_mut.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::array::Array; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { let mut arr = Array::::zeroed(); let xs = arr.as_mut_slice(0, 10); diff --git a/crux-mir/test/symb_eval/bitvector/arith.rs b/crux-mir/test/symb_eval/bitvector/arith.rs index 89f34cb48..06ddbc14b 100644 --- a/crux-mir/test/symb_eval/bitvector/arith.rs +++ b/crux-mir/test/symb_eval/bitvector/arith.rs @@ -30,7 +30,7 @@ fn test_shift_op(f: impl FnOnce(u64, usize) -> u64, f_bv: impl FnOnce(Bv256, usi crucible_assert!(u64::from(x) == f(a, b)); } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() { test_binop(u64::wrapping_add, Add::add); test_binop(u64::wrapping_sub, Sub::sub); diff --git a/crux-mir/test/symb_eval/bitvector/cmp.rs b/crux-mir/test/symb_eval/bitvector/cmp.rs index eab3569e3..75626bfc2 100644 --- a/crux-mir/test/symb_eval/bitvector/cmp.rs +++ b/crux-mir/test/symb_eval/bitvector/cmp.rs @@ -3,7 +3,7 @@ use crucible::bitvector::Bv256; use crucible::crucible_assert; use crucible::Symbolic; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() { { let (a, b) = <(u64, u64)>::symbolic("ab"); diff --git a/crux-mir/test/symb_eval/bitvector/from_to.rs b/crux-mir/test/symb_eval/bitvector/from_to.rs index 4810fd94e..7227ab363 100644 --- a/crux-mir/test/symb_eval/bitvector/from_to.rs +++ b/crux-mir/test/symb_eval/bitvector/from_to.rs @@ -8,7 +8,7 @@ fn test_one(i: u64) { crucible_assert!(i == j); } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() { test_one(0); test_one(1); diff --git a/crux-mir/test/symb_eval/bitvector/leading_zeros.rs b/crux-mir/test/symb_eval/bitvector/leading_zeros.rs index b411e8b1a..e46c7fde4 100644 --- a/crux-mir/test/symb_eval/bitvector/leading_zeros.rs +++ b/crux-mir/test/symb_eval/bitvector/leading_zeros.rs @@ -5,7 +5,7 @@ use crucible::Symbolic; use std::ops::{Not, Neg, Add, Sub, Mul, Div, Rem, BitAnd, BitOr, BitXor, Shl, Shr}; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() { { let a_64 = u64::symbolic("a"); diff --git a/crux-mir/test/symb_eval/bitvector/literals.rs b/crux-mir/test/symb_eval/bitvector/literals.rs index 479424138..b9b80e9cc 100644 --- a/crux-mir/test/symb_eval/bitvector/literals.rs +++ b/crux-mir/test/symb_eval/bitvector/literals.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::bitvector::Bv256; use crucible::crucible_assert; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() { crucible_assert!(u64::from(Bv256::ZERO) == 0); crucible_assert!(u64::from(Bv256::ONE) == 1); diff --git a/crux-mir/test/symb_eval/bitvector/overflowing_sub.rs b/crux-mir/test/symb_eval/bitvector/overflowing_sub.rs index 385761449..e5e28acee 100644 --- a/crux-mir/test/symb_eval/bitvector/overflowing_sub.rs +++ b/crux-mir/test/symb_eval/bitvector/overflowing_sub.rs @@ -3,7 +3,7 @@ use crucible::bitvector::Bv256; use crucible::crucible_assert; use crucible::Symbolic; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() { let a = u64::symbolic("a"); let b = u64::symbolic("b"); diff --git a/crux-mir/test/symb_eval/bitvector/symbolic.rs b/crux-mir/test/symb_eval/bitvector/symbolic.rs index 2210e92b5..76362dd71 100644 --- a/crux-mir/test/symb_eval/bitvector/symbolic.rs +++ b/crux-mir/test/symb_eval/bitvector/symbolic.rs @@ -3,7 +3,7 @@ use crucible::bitvector::Bv256; use crucible::crucible_assert; use crucible::Symbolic; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() { { let a = Bv256::symbolic("a"); diff --git a/crux-mir/test/symb_eval/byteorder/read.rs b/crux-mir/test/symb_eval/byteorder/read.rs index 15fbcd5d0..c39a82e08 100644 --- a/crux-mir/test/symb_eval/byteorder/read.rs +++ b/crux-mir/test/symb_eval/byteorder/read.rs @@ -23,4 +23,4 @@ pub fn f(x: u8) -> u8 { pub static ARG: u8 = 1; #[cfg(with_main)] pub fn main() { println!("{:?}", f(ARG)); } -#[cfg(not(with_main))] #[cfg_attr(crux, crux_test)] fn crux_test() -> u8 { f(ARG) } +#[cfg(not(with_main))] #[cfg_attr(crux, crux::test)] fn crux_test() -> u8 { f(ARG) } diff --git a/crux-mir/test/symb_eval/byteorder/write.rs b/crux-mir/test/symb_eval/byteorder/write.rs index 03c38571a..6a4cc4e4d 100644 --- a/crux-mir/test/symb_eval/byteorder/write.rs +++ b/crux-mir/test/symb_eval/byteorder/write.rs @@ -24,4 +24,4 @@ pub fn f(x: u8) -> u8 { pub static ARG: u8 = 1; #[cfg(with_main)] pub fn main() { println!("{:?}", f(ARG)); } -#[cfg(not(with_main))] #[cfg_attr(crux, crux_test)] fn crux_test() -> u8 { f(ARG) } +#[cfg(not(with_main))] #[cfg_attr(crux, crux::test)] fn crux_test() -> u8 { f(ARG) } diff --git a/crux-mir/test/symb_eval/bytes/extend_bytes.rs b/crux-mir/test/symb_eval/bytes/extend_bytes.rs index 62160176e..8e33bf7d4 100644 --- a/crux-mir/test/symb_eval/bytes/extend_bytes.rs +++ b/crux-mir/test/symb_eval/bytes/extend_bytes.rs @@ -3,7 +3,7 @@ extern crate bytes; use bytes::{Bytes, BytesMut, Buf, BufMut}; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let mut bm = BytesMut::with_capacity(10); let b = Bytes::from(b"12345" as &[_]); diff --git a/crux-mir/test/symb_eval/bytes/new.rs b/crux-mir/test/symb_eval/bytes/new.rs index 161499290..e440ac8f8 100644 --- a/crux-mir/test/symb_eval/bytes/new.rs +++ b/crux-mir/test/symb_eval/bytes/new.rs @@ -3,7 +3,7 @@ extern crate bytes; use bytes::{Bytes, BytesMut, Buf, BufMut}; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let mut b = BytesMut::new(); assert!(b.len() == 0); diff --git a/crux-mir/test/symb_eval/bytes/put.rs b/crux-mir/test/symb_eval/bytes/put.rs index a4a00c8c1..05cfc6ec4 100644 --- a/crux-mir/test/symb_eval/bytes/put.rs +++ b/crux-mir/test/symb_eval/bytes/put.rs @@ -3,7 +3,7 @@ extern crate bytes; use bytes::{Bytes, BytesMut, Buf, BufMut}; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let mut b = BytesMut::with_capacity(10); b.put_u8(1); diff --git a/crux-mir/test/symb_eval/bytes/put_overflow.rs b/crux-mir/test/symb_eval/bytes/put_overflow.rs index 252113d12..b0946e2b6 100644 --- a/crux-mir/test/symb_eval/bytes/put_overflow.rs +++ b/crux-mir/test/symb_eval/bytes/put_overflow.rs @@ -3,7 +3,7 @@ extern crate bytes; use bytes::{Bytes, BytesMut, Buf, BufMut}; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let mut b = BytesMut::with_capacity(0); b.put_slice(b"1234567-1234567-1234567-1234567-1234567-1234567-"); diff --git a/crux-mir/test/symb_eval/bytes/split_off.rs b/crux-mir/test/symb_eval/bytes/split_off.rs index 7132cf1ff..35096f039 100644 --- a/crux-mir/test/symb_eval/bytes/split_off.rs +++ b/crux-mir/test/symb_eval/bytes/split_off.rs @@ -3,7 +3,7 @@ extern crate bytes; use bytes::{Bytes, BytesMut, Buf, BufMut}; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let mut b = BytesMut::with_capacity(10); b.put_u32_be(0x01020304); diff --git a/crux-mir/test/symb_eval/bytes/split_to.rs b/crux-mir/test/symb_eval/bytes/split_to.rs index d928acfe2..3ee83891d 100644 --- a/crux-mir/test/symb_eval/bytes/split_to.rs +++ b/crux-mir/test/symb_eval/bytes/split_to.rs @@ -3,7 +3,7 @@ extern crate bytes; use bytes::{Bytes, BytesMut, Buf, BufMut}; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let mut b = BytesMut::with_capacity(10); b.put_u32_be(0x01020304); diff --git a/crux-mir/test/symb_eval/bytes/sym_len.rs b/crux-mir/test/symb_eval/bytes/sym_len.rs index ebf57c5a8..e2e100305 100644 --- a/crux-mir/test/symb_eval/bytes/sym_len.rs +++ b/crux-mir/test/symb_eval/bytes/sym_len.rs @@ -5,7 +5,7 @@ extern crate bytes; use bytes::{Bytes, BytesMut, Buf, BufMut}; use core::ops::Deref; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let mut b = BytesMut::with_capacity(10); b.put_u8(1); diff --git a/crux-mir/test/symb_eval/concretize/array.rs b/crux-mir/test/symb_eval/concretize/array.rs index d83841cd2..61b0fc4a1 100644 --- a/crux-mir/test/symb_eval/concretize/array.rs +++ b/crux-mir/test/symb_eval/concretize/array.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::*; use crucible::array::Array; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> (i32, i32, i32) { let arr = Array::symbolic("arr"); let s = arr.as_slice(0, 3); diff --git a/crux-mir/test/symb_eval/concretize/assert.rs b/crux-mir/test/symb_eval/concretize/assert.rs index 88c451ada..3523abdf9 100644 --- a/crux-mir/test/symb_eval/concretize/assert.rs +++ b/crux-mir/test/symb_eval/concretize/assert.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> u8 { let mut x = u8::symbolic("x"); let mut y = u8::symbolic("y"); diff --git a/crux-mir/test/symb_eval/concretize/assert_ok.rs b/crux-mir/test/symb_eval/concretize/assert_ok.rs index 804aa0331..e3298a572 100644 --- a/crux-mir/test/symb_eval/concretize/assert_ok.rs +++ b/crux-mir/test/symb_eval/concretize/assert_ok.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> u8 { let mut x = u8::symbolic("x"); let mut y = u8::symbolic("y"); diff --git a/crux-mir/test/symb_eval/concretize/conc.rs b/crux-mir/test/symb_eval/concretize/conc.rs index 797c43f53..7ecc5423b 100644 --- a/crux-mir/test/symb_eval/concretize/conc.rs +++ b/crux-mir/test/symb_eval/concretize/conc.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> (u8, u8) { let mut x = u8::symbolic("x"); let mut y = u8::symbolic("y"); diff --git a/crux-mir/test/symb_eval/concretize/no_conc.rs b/crux-mir/test/symb_eval/concretize/no_conc.rs index 31fc438b6..80a911533 100644 --- a/crux-mir/test/symb_eval/concretize/no_conc.rs +++ b/crux-mir/test/symb_eval/concretize/no_conc.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> (u8, u8) { let mut x = u8::symbolic("x"); let mut y = u8::symbolic("y"); diff --git a/crux-mir/test/symb_eval/crux/early_fail.rs b/crux-mir/test/symb_eval/crux/early_fail.rs index 972a52cc4..9efd17cee 100644 --- a/crux-mir/test/symb_eval/crux/early_fail.rs +++ b/crux-mir/test/symb_eval/crux/early_fail.rs @@ -6,12 +6,12 @@ use crucible::*; // because `fail1` panics (terminating execution) on all branches and `fail2` never runs. The // desired behavior is to run both `fail1` and `fail2` and report their errors independently. -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn fail1() { panic!("bad 1"); } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn fail2() { let x = u8::symbolic("x"); crucible_assert!(x == 0); diff --git a/crux-mir/test/symb_eval/crux/fail_return.rs b/crux-mir/test/symb_eval/crux/fail_return.rs index ae610f120..e8c13060b 100644 --- a/crux-mir/test/symb_eval/crux/fail_return.rs +++ b/crux-mir/test/symb_eval/crux/fail_return.rs @@ -2,14 +2,14 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn fail1() -> u8 { let x = u8::symbolic("x"); crucible_assert!(x + 1 > x); x } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn fail2() -> u8 { let x = u8::symbolic("x"); crucible_assert!(x + 1 > x); diff --git a/crux-mir/test/symb_eval/crux/mixed_fail.rs b/crux-mir/test/symb_eval/crux/mixed_fail.rs index 94bd2dec5..2259c4ff6 100644 --- a/crux-mir/test/symb_eval/crux/mixed_fail.rs +++ b/crux-mir/test/symb_eval/crux/mixed_fail.rs @@ -2,25 +2,25 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn fail1() { let x = u8::symbolic("x"); crucible_assert!(x + 1 > x); } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn fail2() { let x = u8::symbolic("x"); crucible_assert!(x + 2 > x); } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn pass1() { let x = u8::symbolic("x"); crucible_assert!(x > 10 || x <= 10); } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn pass2() { let x = u8::symbolic("x"); crucible_assert!(x > 20 || x <= 20); diff --git a/crux-mir/test/symb_eval/crux/multi.rs b/crux-mir/test/symb_eval/crux/multi.rs index f6a35344c..63204952d 100644 --- a/crux-mir/test/symb_eval/crux/multi.rs +++ b/crux-mir/test/symb_eval/crux/multi.rs @@ -2,13 +2,13 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn fail1() { let x = u8::symbolic("x"); crucible_assert!(x + 1 > x); } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn fail2() { let x = u8::symbolic("x"); if x == 0 { @@ -20,7 +20,7 @@ fn assert_zero(x: u8) { crucible_assert!(x == 0); } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn fail3() { let x = u8::symbolic("x"); assert_zero(x); diff --git a/crux-mir/test/symb_eval/crypto/bytes.rs b/crux-mir/test/symb_eval/crypto/bytes.rs index d084c17b3..f0ee5b8ea 100644 --- a/crux-mir/test/symb_eval/crypto/bytes.rs +++ b/crux-mir/test/symb_eval/crypto/bytes.rs @@ -16,7 +16,7 @@ pub fn from_bytes(bytes: &[u8; 32]) -> [u64; 5] { words[i] |= (bytes[(i * 8) + j] as u64) << (j * 8); } } - + let mask = (1u64 << 52) - 1; let top_mask = (1u64 << 48) - 1; let mut s = zero(); @@ -26,14 +26,14 @@ pub fn from_bytes(bytes: &[u8; 32]) -> [u64; 5] { s[ 2] = ((words[1] >> 40) | (words[2] << 24)) & mask; s[ 3] = ((words[2] >> 28) | (words[3] << 36)) & mask; s[ 4] = (words[3] >> 16) & top_mask; - + s } /// Pack the limbs of this `Scalar64` into 32 bytes pub fn to_bytes(x :&[u64;5]) -> [u8; 32] { let mut s = [0u8; 32]; - + s[0] = (x[ 0] >> 0) as u8; s[1] = (x[ 0] >> 8) as u8; s[2] = (x[ 0] >> 16) as u8; @@ -66,22 +66,22 @@ pub fn to_bytes(x :&[u64;5]) -> [u8; 32] { s[29] = (x[ 4] >> 24) as u8; s[30] = (x[ 4] >> 32) as u8; s[31] = (x[ 4] >> 40) as u8; - + s } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let a0 = crucible_u64("a0"); let a1 = crucible_u64("a1"); let a2 = crucible_u64("a2"); let a3 = crucible_u64("a3"); - let a4 = crucible_u64("a4"); - + let a4 = crucible_u64("a4"); + let a = [a0, a1, a2, a3, a4]; let b = from_bytes(&to_bytes(&a)); for i in 0..5 { crucible_assert!(a[i] == b[i]); - } + } } diff --git a/crux-mir/test/symb_eval/crypto/bytes2.rs b/crux-mir/test/symb_eval/crypto/bytes2.rs index 3f4090b7d..4fcc9c1f2 100644 --- a/crux-mir/test/symb_eval/crypto/bytes2.rs +++ b/crux-mir/test/symb_eval/crypto/bytes2.rs @@ -15,7 +15,7 @@ pub fn from_bytes(bytes: &[u8; 32]) -> [u64; 5] { words[i] |= (bytes[(i * 8) + j] as u64) << (j * 8); } } - + let mask = (1u64 << 52) - 1; let top_mask = (1u64 << 48) - 1; let mut s = zero(); @@ -25,14 +25,14 @@ pub fn from_bytes(bytes: &[u8; 32]) -> [u64; 5] { s[ 2] = ((words[1] >> 40) | (words[2] << 24)) & mask; s[ 3] = ((words[2] >> 28) | (words[3] << 36)) & mask; s[ 4] = (words[3] >> 16) & top_mask; - + s } /// Pack the limbs of this `Scalar64` into 32 bytes pub fn to_bytes(x :&[u64;5]) -> [u8; 32] { let mut s = [0u8; 32]; - + s[0] = (x[ 0] >> 0) as u8; s[1] = (x[ 0] >> 8) as u8; s[2] = (x[ 0] >> 16) as u8; @@ -65,11 +65,11 @@ pub fn to_bytes(x :&[u64;5]) -> [u8; 32] { s[29] = (x[ 4] >> 24) as u8; s[30] = (x[ 4] >> 32) as u8; s[31] = (x[ 4] >> 40) as u8; - + s } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let mut s = [0u8; 32]; @@ -83,7 +83,7 @@ pub fn f() { let a7 = crucible_u8("a7"); let a8 = crucible_u8("a8"); let a9 = crucible_u8("a9"); - let a10 = crucible_u8("a10"); + let a10 = crucible_u8("a10"); s[0] = a0; s[1] = a1; @@ -95,11 +95,11 @@ pub fn f() { s[7] = a7; s[8] = a8; s[9] = a9; - s[10] = a10; - + s[10] = a10; + let b = to_bytes(&from_bytes(&s)); for i in 0..11 { crucible_assert!(s[i] == b[i]); - } + } } diff --git a/crux-mir/test/symb_eval/crypto/double.rs b/crux-mir/test/symb_eval/crypto/double.rs index 7fbc5770a..148601a8e 100644 --- a/crux-mir/test/symb_eval/crypto/double.rs +++ b/crux-mir/test/symb_eval/crypto/double.rs @@ -14,7 +14,7 @@ fn double_imp(x : u32) -> u32 { } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f () { let a0 = crucible_u32("a0"); crucible_assert!(double_ref(a0) == double_imp(a0)); diff --git a/crux-mir/test/symb_eval/crypto/ffs.rs b/crux-mir/test/symb_eval/crypto/ffs.rs index daf922c0c..c450510c6 100644 --- a/crux-mir/test/symb_eval/crypto/ffs.rs +++ b/crux-mir/test/symb_eval/crypto/ffs.rs @@ -28,7 +28,7 @@ fn ffs_imp(j : u32) -> u32 { } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f () { let a0 = crucible_u32("a0"); crucible_assert!(ffs_ref(a0) == ffs_imp(a0)); diff --git a/crux-mir/test/symb_eval/enum/mux.rs b/crux-mir/test/symb_eval/enum/mux.rs index 4490d8dd4..2cdc80cb4 100644 --- a/crux-mir/test/symb_eval/enum/mux.rs +++ b/crux-mir/test/symb_eval/enum/mux.rs @@ -8,7 +8,7 @@ fn f(x: bool) -> Option { if x { Some(S { val: 1 }) } else { None } } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn test() { let x = crucible::crucible_u8("x") != 0; let y = f(x); diff --git a/crux-mir/test/symb_eval/fnptr/mux.rs b/crux-mir/test/symb_eval/fnptr/mux.rs index d1608d12e..4dd7044e8 100644 --- a/crux-mir/test/symb_eval/fnptr/mux.rs +++ b/crux-mir/test/symb_eval/fnptr/mux.rs @@ -9,7 +9,7 @@ fn g(x: i32) -> i32 { x - 1 } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { let b = bool::symbolic("cond"); let p1: fn(i32) -> i32 = if b { f } else { g }; diff --git a/crux-mir/test/symb_eval/io/vec_cursor_read.rs b/crux-mir/test/symb_eval/io/vec_cursor_read.rs index 91ddf89e4..33e821129 100644 --- a/crux-mir/test/symb_eval/io/vec_cursor_read.rs +++ b/crux-mir/test/symb_eval/io/vec_cursor_read.rs @@ -4,7 +4,7 @@ extern crate std; use std::io::{Read, Write, Cursor}; use std::vec::Vec; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let mut buf = Vec::new(); buf.write(&[1, 2, 3]); diff --git a/crux-mir/test/symb_eval/io/vec_write.rs b/crux-mir/test/symb_eval/io/vec_write.rs index 133c1d536..2adb15b47 100644 --- a/crux-mir/test/symb_eval/io/vec_write.rs +++ b/crux-mir/test/symb_eval/io/vec_write.rs @@ -4,7 +4,7 @@ extern crate std; use std::io::Write; use std::vec::Vec; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let mut buf = Vec::new(); buf.write(&[1, 2, 3]); diff --git a/crux-mir/test/symb_eval/mux/array.rs b/crux-mir/test/symb_eval/mux/array.rs index bd34ad9ed..0bca292b9 100644 --- a/crux-mir/test/symb_eval/mux/array.rs +++ b/crux-mir/test/symb_eval/mux/array.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::array::Array; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { let arr1 = [1]; let arr2 = [2, 2]; diff --git a/crux-mir/test/symb_eval/mux/array_mut.rs b/crux-mir/test/symb_eval/mux/array_mut.rs index 0ce460bee..fd4f63546 100644 --- a/crux-mir/test/symb_eval/mux/array_mut.rs +++ b/crux-mir/test/symb_eval/mux/array_mut.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::array::Array; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { let mut arr1 = [1]; let mut arr2 = [2, 2]; diff --git a/crux-mir/test/symb_eval/num/checked_add.rs b/crux-mir/test/symb_eval/num/checked_add.rs index fe180ccc5..75a1643f5 100644 --- a/crux-mir/test/symb_eval/num/checked_add.rs +++ b/crux-mir/test/symb_eval/num/checked_add.rs @@ -1,5 +1,5 @@ -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> u8 { let x = 200; 100 + x diff --git a/crux-mir/test/symb_eval/num/checked_div.rs b/crux-mir/test/symb_eval/num/checked_div.rs index 63514534f..751b82ca1 100644 --- a/crux-mir/test/symb_eval/num/checked_div.rs +++ b/crux-mir/test/symb_eval/num/checked_div.rs @@ -8,7 +8,7 @@ fn div_unsigned(x: u8, y: u8) -> u8 { x / y } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> u8 { let a: i8 = div_signed(1, 1); let b: i8 = div_signed(-128, -1); // Should fail diff --git a/crux-mir/test/symb_eval/num/checked_mul.rs b/crux-mir/test/symb_eval/num/checked_mul.rs index d3d423693..7bad23ec6 100644 --- a/crux-mir/test/symb_eval/num/checked_mul.rs +++ b/crux-mir/test/symb_eval/num/checked_mul.rs @@ -1,4 +1,4 @@ -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> u8 { let x = 3; 100 * x diff --git a/crux-mir/test/symb_eval/num/checked_mul_signed.rs b/crux-mir/test/symb_eval/num/checked_mul_signed.rs index cd8e10c52..67aa95093 100644 --- a/crux-mir/test/symb_eval/num/checked_mul_signed.rs +++ b/crux-mir/test/symb_eval/num/checked_mul_signed.rs @@ -1,4 +1,4 @@ -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i8 { let x = -3; -100 * x diff --git a/crux-mir/test/symb_eval/overrides/bad_symb1.rs b/crux-mir/test/symb_eval/overrides/bad_symb1.rs index 9fd19370b..fdb4c9218 100644 --- a/crux-mir/test/symb_eval/overrides/bad_symb1.rs +++ b/crux-mir/test/symb_eval/overrides/bad_symb1.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> u8 { let s = if bool::symbolic("cond") { "a" } else { "b" }; let x = u8::symbolic(s); diff --git a/crux-mir/test/symb_eval/overrides/bad_symb2.rs b/crux-mir/test/symb_eval/overrides/bad_symb2.rs index 581afa6b8..00251ef08 100644 --- a/crux-mir/test/symb_eval/overrides/bad_symb2.rs +++ b/crux-mir/test/symb_eval/overrides/bad_symb2.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> u8 { let x = u8::symbolic("\0:., /"); x diff --git a/crux-mir/test/symb_eval/overrides/override1.rs b/crux-mir/test/symb_eval/overrides/override1.rs index 314af9895..a635cdc14 100644 --- a/crux-mir/test/symb_eval/overrides/override1.rs +++ b/crux-mir/test/symb_eval/overrides/override1.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() -> u8 { let x: u8 = 1; // This call should be replaced by the test override diff --git a/crux-mir/test/symb_eval/overrides/override2.rs b/crux-mir/test/symb_eval/overrides/override2.rs index 718e912d7..78bedd7d7 100644 --- a/crux-mir/test/symb_eval/overrides/override2.rs +++ b/crux-mir/test/symb_eval/overrides/override2.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { // This call should be replaced by the test override let foo = crucible_i8("foo"); diff --git a/crux-mir/test/symb_eval/overrides/override3.rs b/crux-mir/test/symb_eval/overrides/override3.rs index 37343964c..465e8cc42 100644 --- a/crux-mir/test/symb_eval/overrides/override3.rs +++ b/crux-mir/test/symb_eval/overrides/override3.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { // This call should be replaced by the test override let foo = crucible_i8("x"); diff --git a/crux-mir/test/symb_eval/overrides/override4.rs b/crux-mir/test/symb_eval/overrides/override4.rs index fd24068d0..b5443abfd 100644 --- a/crux-mir/test/symb_eval/overrides/override4.rs +++ b/crux-mir/test/symb_eval/overrides/override4.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { // This call should be replaced by the test override let foo = crucible_i8("x"); diff --git a/crux-mir/test/symb_eval/overrides/override5.rs b/crux-mir/test/symb_eval/overrides/override5.rs index 20b665b81..7e971c683 100644 --- a/crux-mir/test/symb_eval/overrides/override5.rs +++ b/crux-mir/test/symb_eval/overrides/override5.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { // This call should be replaced by the test override let foo = crucible_u64("foo"); diff --git a/crux-mir/test/symb_eval/overrides/override_rust.rs b/crux-mir/test/symb_eval/overrides/override_rust.rs index 9b7db10fb..a238736b3 100644 --- a/crux-mir/test/symb_eval/overrides/override_rust.rs +++ b/crux-mir/test/symb_eval/overrides/override_rust.rs @@ -4,7 +4,7 @@ use crucible::*; fn f() -> i32 { 1 } fn g() -> i32 { 2 } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() { crucible_assert!(f() == 1); crucible::override_(f, g); diff --git a/crux-mir/test/symb_eval/refs/mux_init_imm.rs b/crux-mir/test/symb_eval/refs/mux_init_imm.rs index cb8641130..9e087b733 100644 --- a/crux-mir/test/symb_eval/refs/mux_init_imm.rs +++ b/crux-mir/test/symb_eval/refs/mux_init_imm.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { let mut result = 0; let one = 1; diff --git a/crux-mir/test/symb_eval/refs/mux_init_mut.rs b/crux-mir/test/symb_eval/refs/mux_init_mut.rs index a6f981224..d10e81404 100644 --- a/crux-mir/test/symb_eval/refs/mux_init_mut.rs +++ b/crux-mir/test/symb_eval/refs/mux_init_mut.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { let mut result = 0; if bool::symbolic("cond") { diff --git a/crux-mir/test/symb_eval/scalar/test1.rs b/crux-mir/test/symb_eval/scalar/test1.rs index 52fdcd672..42f59ab40 100644 --- a/crux-mir/test/symb_eval/scalar/test1.rs +++ b/crux-mir/test/symb_eval/scalar/test1.rs @@ -76,7 +76,7 @@ macro_rules! crucible_debug_integer { -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { // Int512 -> Scalar64 -> Int512 conversion is the identity function. { @@ -167,7 +167,7 @@ pub fn f() { mod constants { use super::Scalar64; - + /// `L` is the order of base point, i.e. 2^252 + 27742317777372353535851937790883648493 pub(crate) const L: Scalar64 = Scalar64([ 0x0002631a5cf5d3ed, 0x000dea2f79cd6581, 0x000000000014def9, 0x0000000000000000, 0x0000100000000000 ]); diff --git a/crux-mir/test/symb_eval/sym_bytes/construct.rs b/crux-mir/test/symb_eval/sym_bytes/construct.rs index 8a227b5f5..eb5576aee 100644 --- a/crux-mir/test/symb_eval/sym_bytes/construct.rs +++ b/crux-mir/test/symb_eval/sym_bytes/construct.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::*; use crucible::sym_bytes::SymBytes; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> u8 { let sym1 = SymBytes::zeroed(5); // Should succeed - sym1[0] is zero. In fact, this should get simplified away and not produce diff --git a/crux-mir/test/symb_eval/sym_bytes/deserialize.rs b/crux-mir/test/symb_eval/sym_bytes/deserialize.rs index a56742971..31f85fe27 100644 --- a/crux-mir/test/symb_eval/sym_bytes/deserialize.rs +++ b/crux-mir/test/symb_eval/sym_bytes/deserialize.rs @@ -23,7 +23,7 @@ fn deserialize(b: &[u8]) -> (i16, i16) { } } -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { let sym = SymBytes::symbolic("sym", 5); crucible_assume!(sym[0] <= 2); diff --git a/crux-mir/test/symb_eval/vec/clone.rs b/crux-mir/test/symb_eval/vec/clone.rs index 6c602166e..6b3b2ef29 100644 --- a/crux-mir/test/symb_eval/vec/clone.rs +++ b/crux-mir/test/symb_eval/vec/clone.rs @@ -1,5 +1,5 @@ -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let v = vec![1, 2, 3]; let w = v.clone(); diff --git a/crux-mir/test/symb_eval/vec/into_iter.rs b/crux-mir/test/symb_eval/vec/into_iter.rs index af14a100b..24e72b852 100644 --- a/crux-mir/test/symb_eval/vec/into_iter.rs +++ b/crux-mir/test/symb_eval/vec/into_iter.rs @@ -1,4 +1,4 @@ -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let v = vec![1, 2, 3]; let mut it = v.into_iter(); diff --git a/crux-mir/test/symb_eval/vec/macro.rs b/crux-mir/test/symb_eval/vec/macro.rs index 75126487f..396376828 100644 --- a/crux-mir/test/symb_eval/vec/macro.rs +++ b/crux-mir/test/symb_eval/vec/macro.rs @@ -1,5 +1,5 @@ -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let v = vec![1, 2, 3]; assert!(v.len() == 3); diff --git a/crux-mir/test/symb_eval/vec/sort_by_key.rs b/crux-mir/test/symb_eval/vec/sort_by_key.rs index 8bb081a87..a8b839862 100644 --- a/crux-mir/test/symb_eval/vec/sort_by_key.rs +++ b/crux-mir/test/symb_eval/vec/sort_by_key.rs @@ -1,7 +1,7 @@ #[macro_use] extern crate crucible; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let mut v = vec![1_i32, 3, 2]; v.sort_by_key(|&x| -x); diff --git a/crux-mir/test/symb_eval/vector/as_mut_slice.rs b/crux-mir/test/symb_eval/vector/as_mut_slice.rs index bff7725f1..01fc4e63b 100644 --- a/crux-mir/test/symb_eval/vector/as_mut_slice.rs +++ b/crux-mir/test/symb_eval/vector/as_mut_slice.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::vector::Vector; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let mut v = Vector::::new().push(12).push(34); crucible_assert!(v.len() == 2); diff --git a/crux-mir/test/symb_eval/vector/as_slice.rs b/crux-mir/test/symb_eval/vector/as_slice.rs index bdfe8c741..ec7c385c4 100644 --- a/crux-mir/test/symb_eval/vector/as_slice.rs +++ b/crux-mir/test/symb_eval/vector/as_slice.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::vector::Vector; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let v = Vector::::new().push(12).push(34); crucible_assert!(v.len() == 2); diff --git a/crux-mir/test/symb_eval/vector/concat.rs b/crux-mir/test/symb_eval/vector/concat.rs index f919b588f..1d7a6f8ed 100644 --- a/crux-mir/test/symb_eval/vector/concat.rs +++ b/crux-mir/test/symb_eval/vector/concat.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::vector::Vector; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let v1 = Vector::::new().push(1).push(2); let v2 = Vector::::new().push(3).push(4); diff --git a/crux-mir/test/symb_eval/vector/copy_from_slice.rs b/crux-mir/test/symb_eval/vector/copy_from_slice.rs index 74942bf52..945bc9763 100644 --- a/crux-mir/test/symb_eval/vector/copy_from_slice.rs +++ b/crux-mir/test/symb_eval/vector/copy_from_slice.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::vector::Vector; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let v = Vector::::copy_from_slice(&[1, 2, 3, 4]); crucible_assert!(v.len() == 4); diff --git a/crux-mir/test/symb_eval/vector/mut.rs b/crux-mir/test/symb_eval/vector/mut.rs index 7e5c3339a..4e3a92c56 100644 --- a/crux-mir/test/symb_eval/vector/mut.rs +++ b/crux-mir/test/symb_eval/vector/mut.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::vector::Vector; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let mut v = Vector::::new().push(12).push(34); { diff --git a/crux-mir/test/symb_eval/vector/new.rs b/crux-mir/test/symb_eval/vector/new.rs index e39757c90..175c53be6 100644 --- a/crux-mir/test/symb_eval/vector/new.rs +++ b/crux-mir/test/symb_eval/vector/new.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::vector::Vector; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let v = Vector::::new(); crucible_assert!(v.len() == 0); diff --git a/crux-mir/test/symb_eval/vector/pop.rs b/crux-mir/test/symb_eval/vector/pop.rs index 9253d9850..65df4feea 100644 --- a/crux-mir/test/symb_eval/vector/pop.rs +++ b/crux-mir/test/symb_eval/vector/pop.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::vector::Vector; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let v = Vector::::new().push(12).push(34); crucible_assert!(v.len() == 2); diff --git a/crux-mir/test/symb_eval/vector/push.rs b/crux-mir/test/symb_eval/vector/push.rs index 711dce3aa..9bb3be117 100644 --- a/crux-mir/test/symb_eval/vector/push.rs +++ b/crux-mir/test/symb_eval/vector/push.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::vector::Vector; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let v = Vector::::new(); crucible_assert!(v.len() == 0); diff --git a/crux-mir/test/symb_eval/vector/replicate.rs b/crux-mir/test/symb_eval/vector/replicate.rs index 137e04b44..f45e53dbb 100644 --- a/crux-mir/test/symb_eval/vector/replicate.rs +++ b/crux-mir/test/symb_eval/vector/replicate.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::vector::Vector; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let v = Vector::::replicate(123, 4); crucible_assert!(v.len() == 4); diff --git a/crux-mir/test/symb_eval/vector/split_at.rs b/crux-mir/test/symb_eval/vector/split_at.rs index 2b6ea8976..b1c7868a1 100644 --- a/crux-mir/test/symb_eval/vector/split_at.rs +++ b/crux-mir/test/symb_eval/vector/split_at.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::vector::Vector; -#[cfg_attr(crux, crux_test)] +#[cfg_attr(crux, crux::test)] pub fn f() { let v = Vector::::copy_from_slice(&[1, 2, 3, 4]); let (v1, v2) = v.split_at(2); From b4f6235aaefdc3dcf98745484497027736a55a7b Mon Sep 17 00:00:00 2001 From: James LaMar Date: Thu, 16 Mar 2023 13:00:05 -0700 Subject: [PATCH 003/114] [WIP] more bugfixes --- crucible-mir/src/Mir/Generator.hs | 18 ++++++++++++++--- crucible-mir/src/Mir/GenericOps.hs | 8 ++++---- crucible-mir/src/Mir/JSON.hs | 3 ++- crucible-mir/src/Mir/Mir.hs | 8 +++++++- crucible-mir/src/Mir/PP.hs | 4 ++-- crucible-mir/src/Mir/Pass/AllocateEnum.hs | 3 +-- crucible-mir/src/Mir/Trans.hs | 24 ++++++++++++++++------- crucible-mir/src/Mir/TransCustom.hs | 24 ++++++++++++++++++++++- crucible-mir/src/Mir/TransTy.hs | 12 ++++++++---- 9 files changed, 79 insertions(+), 25 deletions(-) diff --git a/crucible-mir/src/Mir/Generator.hs b/crucible-mir/src/Mir/Generator.hs index 74ecab575..4b1af9f47 100644 --- a/crucible-mir/src/Mir/Generator.hs +++ b/crucible-mir/src/Mir/Generator.hs @@ -75,6 +75,7 @@ import Data.Set (Set) import qualified Data.Set as Set import Data.Text (Text) import qualified Data.Text as Text +import Data.Char(isDigit) import Data.Functor.Identity import GHC.Generics (Generic) @@ -110,6 +111,7 @@ import Mir.PP import Unsafe.Coerce(unsafeCoerce) import Debug.Trace import GHC.Stack +import Control.Applicative ((<|>)) -------------------------------------------------------------------------------------- @@ -475,14 +477,24 @@ resolveCustom instDefId = do show (intr ^. intrInst . inDefId) _ -> do let origDefId = intr ^. intrInst . inDefId - let origSubsts = intr ^. intrInst . inSubsts - let edid = idKey origDefId + origSubsts = intr ^. intrInst . inSubsts + edid = idKey origDefId + -- remove section numbers (if any) + removeSectionNumber w = + Maybe.fromMaybe w (Text.dropWhile isDigit <$> Text.stripPrefix "#" w) + stripSectionNumbers w = + let (part1, part2) = Text.breakOn "#" w + in part1 <> removeSectionNumber part2 + + edidSimpl = stripSectionNumbers <$> edid optOp <- use $ customOps . opDefs . at edid - case optOp of + optOpSimpl <- use $ customOps . opDefs . at edidSimpl + case optOp <|> optOpSimpl of Nothing -> return Nothing Just f -> do return $ f origSubsts + --------------------------------------------------------------------------------------------------- -- ** Adding new temporaries to the VarMap diff --git a/crucible-mir/src/Mir/GenericOps.hs b/crucible-mir/src/Mir/GenericOps.hs index c03cf8972..9786dd4f5 100644 --- a/crucible-mir/src/Mir/GenericOps.hs +++ b/crucible-mir/src/Mir/GenericOps.hs @@ -72,9 +72,9 @@ adtIndices (Adt _aname _kind vars _ _ _ _) col = go 0 vars lastExplicit' = if isExplicit v then discr else lastExplicit in discr : go lastExplicit' vs - getDiscr _ (Variant _ _ _ _ (Just i)) = i + getDiscr _ (Variant _ _ _ _ (Just i) _) = i - getDiscr _ (Variant name (Explicit did) _fields _knd _) = case Map.lookup did (_functions col) of + getDiscr _ (Variant name (Explicit did) _fields _knd _ _) = case Map.lookup did (_functions col) of Just fn -> case fn^.fbody.mblocks of ( BasicBlock _info (BasicBlockData [Assign _lhs (Use (OpConstant (Constant _ty (ConstInt i)))) _loc] _term) ):_ -> fromIntegerLit i @@ -83,10 +83,10 @@ adtIndices (Adt _aname _kind vars _ _ _ _) col = go 0 vars Nothing -> error $ "cannot find discriminant constant " ++ show did ++ " for variant " ++ show name - getDiscr lastExplicit (Variant _vname (Relative i) _fields _kind _) = + getDiscr lastExplicit (Variant _vname (Relative i) _fields _kind _ _) = lastExplicit + toInteger i - isExplicit (Variant _ (Explicit _) _ _ _) = True + isExplicit (Variant _ (Explicit _) _ _ _ _) = True isExplicit _ = False -------------------------------------------------------------------------------------- diff --git a/crucible-mir/src/Mir/JSON.hs b/crucible-mir/src/Mir/JSON.hs index c835e2570..fdffcda59 100644 --- a/crucible-mir/src/Mir/JSON.hs +++ b/crucible-mir/src/Mir/JSON.hs @@ -175,6 +175,7 @@ instance FromJSON Variant where <*> v .: "ctor_kind" <*> do val <- v .:? "discr_value" convertIntegerText `traverse` val + <*> v .: "inhabited" instance FromJSON Field where parseJSON = withObject "Field" $ \v -> Field <$> v .: "name" <*> v .: "ty" @@ -525,7 +526,7 @@ instance FromJSON RustcRenderedConst where Just (String "tuple") -> do elems <- map (\(RustcRenderedConst val) -> val) <$> v .: "elements" - return $ ConstArray elems + return $ ConstTuple elems o -> do fail $ "parseJSON - bad rendered constant kind: " ++ show o diff --git a/crucible-mir/src/Mir/Mir.hs b/crucible-mir/src/Mir/Mir.hs index 5219875d7..3b8b67a68 100644 --- a/crucible-mir/src/Mir/Mir.hs +++ b/crucible-mir/src/Mir/Mir.hs @@ -199,7 +199,13 @@ data CtorKind deriving (Eq, Ord, Show, Generic) -data Variant = Variant {_vname :: DefId, _vdiscr :: VariantDiscr, _vfields :: [Field], _vctorkind :: Maybe CtorKind, _discrval :: Maybe Integer } +data Variant = Variant { + _vname :: DefId, + _vdiscr :: VariantDiscr, + _vfields :: [Field], + _vctorkind :: Maybe CtorKind, + _discrval :: Maybe Integer, + _vinhabited :: Bool } deriving (Eq, Ord,Show, Generic) diff --git a/crucible-mir/src/Mir/PP.hs b/crucible-mir/src/Mir/PP.hs index a308b104c..258cf0806 100644 --- a/crucible-mir/src/Mir/PP.hs +++ b/crucible-mir/src/Mir/PP.hs @@ -108,9 +108,9 @@ instance Pretty CtorKind where pretty = viaShow instance Pretty Variant where - pretty (Variant nm dscr flds knd mbVal) = + pretty (Variant nm dscr flds knd mbVal inh) = pretty "Variant" <> - tupled [pretty nm, pretty dscr, pretty flds, pretty knd, pretty mbVal] + tupled [pretty nm, pretty dscr, pretty flds, pretty knd, pretty mbVal, pretty inh] instance Pretty Field where pretty (Field nm ty) = pretty_fn2 "Field" nm ty diff --git a/crucible-mir/src/Mir/Pass/AllocateEnum.hs b/crucible-mir/src/Mir/Pass/AllocateEnum.hs index bd14be432..4a154d409 100644 --- a/crucible-mir/src/Mir/Pass/AllocateEnum.hs +++ b/crucible-mir/src/Mir/Pass/AllocateEnum.hs @@ -82,11 +82,10 @@ makeAggregate updates (lv, k, adt) = (Assign lv (RAdtAg (AdtAg adt (toInteger k) ops ty)) pos) where adt_did = _adtname adt ty = typeOf lv - ops = map rhs $ sortOn fieldNum updates pos = case updates of u:_ -> upos u [] -> "internal" - + ops = map rhs $ sortOn fieldNum updates findAllocEnum :: (?col :: Collection) => [Statement] -> Maybe ( Statement, [Statement] ) findAllocEnum ss = f ss [] where diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index f49ea71a3..111a035ff 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -181,6 +181,8 @@ transConstVal _ty (Some (C.BVRepr w)) (M.ConstChar c) = return $ MirExp (C.BVRepr w) (S.app $ eBVLit w i) transConstVal _ty (Some C.UnitRepr) (M.ConstFunction _did) = return $ MirExp C.UnitRepr $ S.app E.EmptyApp +transConstVal _ty (Some C.UnitRepr) (M.ConstTuple []) = + return $ MirExp C.UnitRepr $ S.app E.EmptyApp transConstVal _ty (Some (C.RealValRepr)) (M.ConstFloat (M.FloatLit _ str)) = case reads str of @@ -1630,14 +1632,20 @@ initialValue (M.TyAdt nm _ _) = do let var = M.onlyVariant adt fldExps <- mapM initField (var^.M.vfields) Just <$> buildStruct' adt fldExps - Enum -> case adt ^. adtvariants of - -- Uninhabited enums can't be initialized. - [] -> return Nothing - -- Inhabited enums get initialized to their first variant. - (var : _) -> do - fldExps <- mapM initField (var^.M.vfields) - Just <$> buildEnum' adt 0 fldExps + Enum -> do + case inhabited (adt ^. adtvariants) of + -- Uninhabited enums can't be initialized. + [] -> return Nothing + -- Inhabited enums get initialized to their first inhabited variant. + vs@(var : _) -> do + fldExps <- mapM initField (var^.M.vfields) + Just <$> buildEnum' adt 0 fldExps Union -> return Nothing + where + inhabited vars = filter _vinhabited vars + + + initialValue (M.TyFnPtr _) = return $ Nothing initialValue (M.TyFnDef _) = return $ Just $ MirExp C.UnitRepr $ R.App E.EmptyApp initialValue (M.TyDynamic _) = return $ Nothing @@ -1648,6 +1656,8 @@ initialValue _ = return Nothing initField :: Field -> MirGenerator h s ret (Maybe (MirExp s)) initField (Field _name ty) = initialValue ty + + -- | Allocate RefCells for all locals and populate `varMap`. Locals are -- default-initialized when possible using the result of `initialValue`. initLocals :: [M.Var] -> Set.Set Text.Text -> MirGenerator h s ret () diff --git a/crucible-mir/src/Mir/TransCustom.hs b/crucible-mir/src/Mir/TransCustom.hs index 23acce31c..b89230b36 100644 --- a/crucible-mir/src/Mir/TransCustom.hs +++ b/crucible-mir/src/Mir/TransCustom.hs @@ -103,6 +103,7 @@ customOpDefs = Map.fromList $ [ , min_align_of , intrinsics_assume , assert_inhabited + , unlikely , mem_transmute , mem_crucible_identity_transmute @@ -679,7 +680,7 @@ makeArithWithOverflow name isSignedOverride bop = add_with_overflow :: (ExplodedDefId, CustomRHS) add_with_overflow = - ( ["core","intrinsics", "", "add_with_overflow"] + ( ["core","intrinsics", "{extern}", "add_with_overflow"] , makeArithWithOverflow "add_with_overflow" Nothing Add ) @@ -1525,6 +1526,15 @@ atomic_funcs = "acq_failrelaxed", "acqrel_failrelaxed", "failrelaxed", "failacq"] fenceVariants = ["acq", "rel", "acqrel"] +-------------------------------------------------------------------------------------------------------------------------- + +unlikely :: (ExplodedDefId, CustomRHS) +unlikely = (name, rhs) + where + name = ["core", "intrinsics", "{extern}", "unlikely"] + rhs substs = Just $ CustomOp $ \_ [op] -> pure op + + -------------------------------------------------------------------------------------------------------------------------- -- MaybeUninit @@ -1541,6 +1551,17 @@ maybe_uninit_uninit = (["core", "mem", "maybe_uninit", "{{impl}}", "uninit"], _ -> Nothing) +-------------------------------------------------------------------------------------------------------------------------- +intrinsicName :: String -> [String] +intrinsicName name = ["core", "intrinsics", "", name] + +-- ctpop :: (ExplodedDefId, CustomRHS) +-- ctpop = (intrinsicName "ctpop", rhs) +-- where +-- rhs [_sz] = +-- Just $ CustomOp $ + + -------------------------------------------------------------------------------------------------------------------------- -- Implementation for `IkFnPtrShim`. Function pointer shims are auto-generated -- `Fn`/`FnMut`/`FnOnce` methods for `TyFnDef` and `TyFnPtr`, allowing ordinary @@ -1610,6 +1631,7 @@ cloneFromShimDef :: Ty -> [M.DefId] -> CustomOp cloneFromShimDef ty parts = CustomOp $ \_ _ -> mirFail $ "cloneFromShimDef not implemented for " ++ show ty + -------------------------------------------------------------------------------------------------------------------------- unwrapMirExp :: C.TypeRepr tp -> MirExp s -> MirGenerator h s ret (R.Expr MIR s tp) diff --git a/crucible-mir/src/Mir/TransTy.hs b/crucible-mir/src/Mir/TransTy.hs index 2284abda9..f64f565a4 100644 --- a/crucible-mir/src/Mir/TransTy.hs +++ b/crucible-mir/src/Mir/TransTy.hs @@ -49,6 +49,7 @@ import qualified Lang.Crucible.Syntax as S import qualified Mir.DefId as M import qualified Mir.Mir as M +import qualified Debug.Trace as Debug import Mir.Generator ( MirExp(..), MirPlace(..), PtrMetadata(..), MirGenerator, mirFail @@ -325,7 +326,7 @@ reprTransparentFieldTy col adt = do variantFields :: TransTyConstraint => M.Collection -> M.Variant -> Some C.CtxRepr -variantFields col (M.Variant _vn _vd vfs _vct _mbVal) = +variantFields col (M.Variant _vn _vd vfs _vct _mbVal _inh) = tyReprListToCtx (map (mapSome fieldType . tyToFieldRepr col . (^. M.fty)) vfs) (\repr -> Some repr) @@ -360,7 +361,7 @@ tyToFieldRepr col ty | otherwise = viewSome (\tpr -> Some $ FieldRepr $ FkMaybe tpr) (tyToRepr col ty) variantFields' :: TransTyConstraint => M.Collection -> M.Variant -> Some FieldCtxRepr -variantFields' col (M.Variant _vn _vd vfs _vct _mbVal) = +variantFields' col (M.Variant _vn _vd vfs _vct _mbVal _inh) = fieldReprListToCtx (map (tyToFieldRepr col . (^. M.fty)) vfs) (\x -> Some x) @@ -842,7 +843,9 @@ buildStructAssign' ctx es = go ctx $ reverse es go ctx (optExp : rest) = case Ctx.viewAssign ctx of Ctx.AssignExtend ctx' fldr -> case (fldr, optExp) of (FieldRepr (FkInit tpr), Nothing) -> - Left $ "got Nothing for mandatory field " ++ show (length rest) + case tpr of + C.UnitRepr -> continue ctx' rest tpr (R.App $ E.NothingValue tpr) + _ -> Left $ "got Nothing for mandatory field " ++ show (length rest) ++ " type:" ++ show tpr (FieldRepr (FkInit tpr), Just (Some e)) -> continue ctx' rest tpr e (FieldRepr (FkMaybe tpr), Nothing) -> @@ -902,7 +905,8 @@ buildEnum' adt i es = do Some fctx' <- variantFieldsM' var asn <- case buildStructAssign' fctx' $ map (fmap (\(MirExp _ e) -> Some e)) es of - Left err -> mirFail $ "error building variant " ++ show (var^.M.vname) ++ ": " ++ err + Left err -> + mirFail $ "error building variant " ++ show (var^.M.vname) ++ ": " ++ err ++ " -- " ++ show es Right x -> return x Refl <- testEqualityOrFail (fieldCtxType fctx') ctx' $ "got wrong fields for " ++ show (adt ^. M.adtname, i) ++ "?" From 82c3cd0b7cd8ba4698714e82dd9c82910793fd06 Mon Sep 17 00:00:00 2001 From: James LaMar Date: Thu, 16 Mar 2023 13:04:57 -0700 Subject: [PATCH 004/114] Update `mir-json` submodule --- dependencies/mir-json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies/mir-json b/dependencies/mir-json index 281fcb31d..5d697d5f3 160000 --- a/dependencies/mir-json +++ b/dependencies/mir-json @@ -1 +1 @@ -Subproject commit 281fcb31dca2de7cf7e6518562b11acdaeb7ede6 +Subproject commit 5d697d5f37e8a041c7010e47776b6cb02be940d9 From 26d1dc520398311d85dc3a5b57d99a0972887ab4 Mon Sep 17 00:00:00 2001 From: James LaMar Date: Wed, 22 Mar 2023 09:51:10 -0700 Subject: [PATCH 005/114] re-add missing libs to linker and libs dir --- crux-mir/build.sh | 17 +- crux-mir/lib/byteorder/LICENSE-MIT | 21 + crux-mir/lib/byteorder/io.rs | 1605 ++++++++++++++ crux-mir/lib/byteorder/lib.rs | 3204 ++++++++++++++++++++++++++++ crux-mir/lib/bytes.rs | 449 ++++ crux-mir/lib/int512.rs | 135 ++ crux-mir/src/Mir/Generate.hs | 37 +- 7 files changed, 5451 insertions(+), 17 deletions(-) create mode 100644 crux-mir/lib/byteorder/LICENSE-MIT create mode 100644 crux-mir/lib/byteorder/io.rs create mode 100644 crux-mir/lib/byteorder/lib.rs create mode 100644 crux-mir/lib/bytes.rs create mode 100644 crux-mir/lib/int512.rs diff --git a/crux-mir/build.sh b/crux-mir/build.sh index deda4ca91..8c465d81e 100644 --- a/crux-mir/build.sh +++ b/crux-mir/build.sh @@ -11,10 +11,6 @@ mir-json lib/libc/src/lib.rs --crate-name libc -L rlibs --out-dir rlibs --crate echo 'Building compiler_builtins...' mir-json lib/compiler_builtins/src/lib.rs --crate-name compiler_builtins -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="compiler-builtins"' --cfg 'feature="core"' --cfg 'feature="default"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="unstable"' --cfg 'feature="mem-unaligned"`' --extern core=rlibs/libcore.rlib -#added crucible manually here -echo "Building crucible..." -mir-json lib/crucible/lib.rs --crate-name crucible --edition=2021 -L rlibs --out-dir rlibs --crate-type rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib - echo 'Building alloc...' mir-json lib/alloc/src/lib.rs --edition=2021 --crate-name alloc -L rlibs --out-dir rlibs --crate-type rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib @@ -66,3 +62,16 @@ mir-json lib/std/src/lib.rs --edition=2021 --crate-name std -L rlibs --out-dir r echo 'Building proc_macro...' mir-json lib/proc_macro/src/lib.rs --edition=2021 --crate-name proc_macro -L rlibs --out-dir rlibs --crate-type rlib --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib +# extra libs (added manually) +echo "Building crucible..." +mir-json lib/crucible/lib.rs --crate-name crucible --edition=2021 -L rlibs --out-dir rlibs --crate-type rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib + +echo 'Building int512...' +mir-json lib/int512.rs --crate-name int512 -L rlibs --out-dir rlibs --crate-type rlib --extern core=rlibs/libcore.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib + +echo 'Building bytes...' +mir-json lib/bytes.rs --edition=2021 --crate-name bytes -L rlibs --out-dir rlibs --crate-type rlib --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern crucible=rlibs/libcrucible.rlib + +echo 'Building byteorder...' +mir-json lib/byteorder/lib.rs --edition=2021 --crate-name byteorder -L rlibs --out-dir rlibs --crate-type rlib --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib + diff --git a/crux-mir/lib/byteorder/LICENSE-MIT b/crux-mir/lib/byteorder/LICENSE-MIT new file mode 100644 index 000000000..3b0a5dc09 --- /dev/null +++ b/crux-mir/lib/byteorder/LICENSE-MIT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 Andrew Gallant + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/crux-mir/lib/byteorder/io.rs b/crux-mir/lib/byteorder/io.rs new file mode 100644 index 000000000..ed6a84845 --- /dev/null +++ b/crux-mir/lib/byteorder/io.rs @@ -0,0 +1,1605 @@ +use std::io::{self, Result}; +use std::slice; + +use ByteOrder; + +/// Extends [`Read`] with methods for reading numbers. (For `std::io`.) +/// +/// Most of the methods defined here have an unconstrained type parameter that +/// must be explicitly instantiated. Typically, it is instantiated with either +/// the [`BigEndian`] or [`LittleEndian`] types defined in this crate. +/// +/// # Examples +/// +/// Read unsigned 16 bit big-endian integers from a [`Read`]: +/// +/// ```rust +/// use std::io::Cursor; +/// use byteorder::{BigEndian, ReadBytesExt}; +/// +/// let mut rdr = Cursor::new(vec![2, 5, 3, 0]); +/// assert_eq!(517, rdr.read_u16::().unwrap()); +/// assert_eq!(768, rdr.read_u16::().unwrap()); +/// ``` +/// +/// [`BigEndian`]: enum.BigEndian.html +/// [`LittleEndian`]: enum.LittleEndian.html +/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html +pub trait ReadBytesExt: io::Read { + /// Reads an unsigned 8 bit integer from the underlying reader. + /// + /// Note that since this reads a single byte, no byte order conversions + /// are used. It is included for completeness. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read unsigned 8 bit integers from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::ReadBytesExt; + /// + /// let mut rdr = Cursor::new(vec![2, 5]); + /// assert_eq!(2, rdr.read_u8().unwrap()); + /// assert_eq!(5, rdr.read_u8().unwrap()); + /// ``` + #[inline] + fn read_u8(&mut self) -> Result { + let mut buf = [0; 1]; + try!(self.read_exact(&mut buf)); + Ok(buf[0]) + } + + /// Reads a signed 8 bit integer from the underlying reader. + /// + /// Note that since this reads a single byte, no byte order conversions + /// are used. It is included for completeness. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read signed 8 bit integers from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::ReadBytesExt; + /// + /// let mut rdr = Cursor::new(vec![0x02, 0xfb]); + /// assert_eq!(2, rdr.read_i8().unwrap()); + /// assert_eq!(-5, rdr.read_i8().unwrap()); + /// ``` + #[inline] + fn read_i8(&mut self) -> Result { + let mut buf = [0; 1]; + try!(self.read_exact(&mut buf)); + Ok(buf[0] as i8) + } + + /// Reads an unsigned 16 bit integer from the underlying reader. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read unsigned 16 bit big-endian integers from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![2, 5, 3, 0]); + /// assert_eq!(517, rdr.read_u16::().unwrap()); + /// assert_eq!(768, rdr.read_u16::().unwrap()); + /// ``` + #[inline] + fn read_u16(&mut self) -> Result { + let mut buf = [0; 2]; + try!(self.read_exact(&mut buf)); + Ok(T::read_u16(&buf)) + } + + /// Reads a signed 16 bit integer from the underlying reader. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read signed 16 bit big-endian integers from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![0x00, 0xc1, 0xff, 0x7c]); + /// assert_eq!(193, rdr.read_i16::().unwrap()); + /// assert_eq!(-132, rdr.read_i16::().unwrap()); + /// ``` + #[inline] + fn read_i16(&mut self) -> Result { + let mut buf = [0; 2]; + try!(self.read_exact(&mut buf)); + Ok(T::read_i16(&buf)) + } + + /// Reads an unsigned 24 bit integer from the underlying reader. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read unsigned 24 bit big-endian integers from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![0x00, 0x01, 0x0b]); + /// assert_eq!(267, rdr.read_u24::().unwrap()); + /// ``` + #[inline] + fn read_u24(&mut self) -> Result { + let mut buf = [0; 3]; + try!(self.read_exact(&mut buf)); + Ok(T::read_u24(&buf)) + } + + /// Reads a signed 24 bit integer from the underlying reader. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read signed 24 bit big-endian integers from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![0xff, 0x7a, 0x33]); + /// assert_eq!(-34253, rdr.read_i24::().unwrap()); + /// ``` + #[inline] + fn read_i24(&mut self) -> Result { + let mut buf = [0; 3]; + try!(self.read_exact(&mut buf)); + Ok(T::read_i24(&buf)) + } + + /// Reads an unsigned 32 bit integer from the underlying reader. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read unsigned 32 bit big-endian integers from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![0x00, 0x00, 0x01, 0x0b]); + /// assert_eq!(267, rdr.read_u32::().unwrap()); + /// ``` + #[inline] + fn read_u32(&mut self) -> Result { + let mut buf = [0; 4]; + try!(self.read_exact(&mut buf)); + Ok(T::read_u32(&buf)) + } + + /// Reads a signed 32 bit integer from the underlying reader. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read signed 32 bit big-endian integers from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![0xff, 0xff, 0x7a, 0x33]); + /// assert_eq!(-34253, rdr.read_i32::().unwrap()); + /// ``` + #[inline] + fn read_i32(&mut self) -> Result { + let mut buf = [0; 4]; + try!(self.read_exact(&mut buf)); + Ok(T::read_i32(&buf)) + } + + /// Reads an unsigned 48 bit integer from the underlying reader. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read unsigned 48 bit big-endian integers from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![0xb6, 0x71, 0x6b, 0xdc, 0x2b, 0x31]); + /// assert_eq!(200598257150769, rdr.read_u48::().unwrap()); + /// ``` + #[inline] + fn read_u48(&mut self) -> Result { + let mut buf = [0; 6]; + try!(self.read_exact(&mut buf)); + Ok(T::read_u48(&buf)) + } + + /// Reads a signed 48 bit integer from the underlying reader. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read signed 48 bit big-endian integers from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![0x9d, 0x71, 0xab, 0xe7, 0x97, 0x8f]); + /// assert_eq!(-108363435763825, rdr.read_i48::().unwrap()); + /// ``` + #[inline] + fn read_i48(&mut self) -> Result { + let mut buf = [0; 6]; + try!(self.read_exact(&mut buf)); + Ok(T::read_i48(&buf)) + } + + /// Reads an unsigned 64 bit integer from the underlying reader. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read an unsigned 64 bit big-endian integer from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![0x00, 0x03, 0x43, 0x95, 0x4d, 0x60, 0x86, 0x83]); + /// assert_eq!(918733457491587, rdr.read_u64::().unwrap()); + /// ``` + #[inline] + fn read_u64(&mut self) -> Result { + let mut buf = [0; 8]; + try!(self.read_exact(&mut buf)); + Ok(T::read_u64(&buf)) + } + + /// Reads a signed 64 bit integer from the underlying reader. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read a signed 64 bit big-endian integer from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![0x80, 0, 0, 0, 0, 0, 0, 0]); + /// assert_eq!(i64::min_value(), rdr.read_i64::().unwrap()); + /// ``` + #[inline] + fn read_i64(&mut self) -> Result { + let mut buf = [0; 8]; + try!(self.read_exact(&mut buf)); + Ok(T::read_i64(&buf)) + } + + /// Reads an unsigned 128 bit integer from the underlying reader. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read an unsigned 128 bit big-endian integer from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![ + /// 0x00, 0x03, 0x43, 0x95, 0x4d, 0x60, 0x86, 0x83, + /// 0x00, 0x03, 0x43, 0x95, 0x4d, 0x60, 0x86, 0x83 + /// ]); + /// assert_eq!(16947640962301618749969007319746179, rdr.read_u128::().unwrap()); + /// ``` + #[cfg(byteorder_i128)] + #[inline] + fn read_u128(&mut self) -> Result { + let mut buf = [0; 16]; + try!(self.read_exact(&mut buf)); + Ok(T::read_u128(&buf)) + } + + /// Reads a signed 128 bit integer from the underlying reader. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read a signed 128 bit big-endian integer from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + /// assert_eq!(i128::min_value(), rdr.read_i128::().unwrap()); + /// ``` + #[cfg(byteorder_i128)] + #[inline] + fn read_i128(&mut self) -> Result { + let mut buf = [0; 16]; + try!(self.read_exact(&mut buf)); + Ok(T::read_i128(&buf)) + } + + /// Reads an unsigned n-bytes integer from the underlying reader. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read an unsigned n-byte big-endian integer from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![0x80, 0x74, 0xfa]); + /// assert_eq!(8418554, rdr.read_uint::(3).unwrap()); + #[inline] + fn read_uint(&mut self, nbytes: usize) -> Result { + let mut buf = [0; 8]; + try!(self.read_exact(&mut buf[..nbytes])); + Ok(T::read_uint(&buf[..nbytes], nbytes)) + } + + /// Reads a signed n-bytes integer from the underlying reader. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read an unsigned n-byte big-endian integer from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![0xc1, 0xff, 0x7c]); + /// assert_eq!(-4063364, rdr.read_int::(3).unwrap()); + #[inline] + fn read_int(&mut self, nbytes: usize) -> Result { + let mut buf = [0; 8]; + try!(self.read_exact(&mut buf[..nbytes])); + Ok(T::read_int(&buf[..nbytes], nbytes)) + } + + /// Reads an unsigned n-bytes integer from the underlying reader. + #[cfg(byteorder_i128)] + #[inline] + fn read_uint128(&mut self, nbytes: usize) -> Result { + let mut buf = [0; 16]; + try!(self.read_exact(&mut buf[..nbytes])); + Ok(T::read_uint128(&buf[..nbytes], nbytes)) + } + + /// Reads a signed n-bytes integer from the underlying reader. + #[cfg(byteorder_i128)] + #[inline] + fn read_int128(&mut self, nbytes: usize) -> Result { + let mut buf = [0; 16]; + try!(self.read_exact(&mut buf[..nbytes])); + Ok(T::read_int128(&buf[..nbytes], nbytes)) + } + + /// Reads a IEEE754 single-precision (4 bytes) floating point number from + /// the underlying reader. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read a big-endian single-precision floating point number from a `Read`: + /// + /// ```rust + /// use std::f32; + /// use std::io::Cursor; + /// + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![ + /// 0x40, 0x49, 0x0f, 0xdb, + /// ]); + /// assert_eq!(f32::consts::PI, rdr.read_f32::().unwrap()); + /// ``` + #[inline] + fn read_f32(&mut self) -> Result { + let mut buf = [0; 4]; + try!(self.read_exact(&mut buf)); + Ok(T::read_f32(&buf)) + } + + /// Reads a IEEE754 double-precision (8 bytes) floating point number from + /// the underlying reader. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read a big-endian double-precision floating point number from a `Read`: + /// + /// ```rust + /// use std::f64; + /// use std::io::Cursor; + /// + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![ + /// 0x40, 0x09, 0x21, 0xfb, 0x54, 0x44, 0x2d, 0x18, + /// ]); + /// assert_eq!(f64::consts::PI, rdr.read_f64::().unwrap()); + /// ``` + #[inline] + fn read_f64(&mut self) -> Result { + let mut buf = [0; 8]; + try!(self.read_exact(&mut buf)); + Ok(T::read_f64(&buf)) + } + + /// Reads a sequence of unsigned 16 bit integers from the underlying + /// reader. + /// + /// The given buffer is either filled completely or an error is returned. + /// If an error is returned, the contents of `dst` are unspecified. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read a sequence of unsigned 16 bit big-endian integers from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![2, 5, 3, 0]); + /// let mut dst = [0; 2]; + /// rdr.read_u16_into::(&mut dst).unwrap(); + /// assert_eq!([517, 768], dst); + /// ``` + #[inline] + fn read_u16_into(&mut self, dst: &mut [u16]) -> Result<()> { + { + let buf = unsafe { slice_to_u8_mut(dst) }; + try!(self.read_exact(buf)); + } + T::from_slice_u16(dst); + Ok(()) + } + + /// Reads a sequence of unsigned 32 bit integers from the underlying + /// reader. + /// + /// The given buffer is either filled completely or an error is returned. + /// If an error is returned, the contents of `dst` are unspecified. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read a sequence of unsigned 32 bit big-endian integers from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![0, 0, 2, 5, 0, 0, 3, 0]); + /// let mut dst = [0; 2]; + /// rdr.read_u32_into::(&mut dst).unwrap(); + /// assert_eq!([517, 768], dst); + /// ``` + #[inline] + fn read_u32_into(&mut self, dst: &mut [u32]) -> Result<()> { + { + let buf = unsafe { slice_to_u8_mut(dst) }; + try!(self.read_exact(buf)); + } + T::from_slice_u32(dst); + Ok(()) + } + + /// Reads a sequence of unsigned 64 bit integers from the underlying + /// reader. + /// + /// The given buffer is either filled completely or an error is returned. + /// If an error is returned, the contents of `dst` are unspecified. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read a sequence of unsigned 64 bit big-endian integers from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![ + /// 0, 0, 0, 0, 0, 0, 2, 5, + /// 0, 0, 0, 0, 0, 0, 3, 0, + /// ]); + /// let mut dst = [0; 2]; + /// rdr.read_u64_into::(&mut dst).unwrap(); + /// assert_eq!([517, 768], dst); + /// ``` + #[inline] + fn read_u64_into(&mut self, dst: &mut [u64]) -> Result<()> { + { + let buf = unsafe { slice_to_u8_mut(dst) }; + try!(self.read_exact(buf)); + } + T::from_slice_u64(dst); + Ok(()) + } + + /// Reads a sequence of unsigned 128 bit integers from the underlying + /// reader. + /// + /// The given buffer is either filled completely or an error is returned. + /// If an error is returned, the contents of `dst` are unspecified. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read a sequence of unsigned 128 bit big-endian integers from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![ + /// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, + /// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, + /// ]); + /// let mut dst = [0; 2]; + /// rdr.read_u128_into::(&mut dst).unwrap(); + /// assert_eq!([517, 768], dst); + /// ``` + #[cfg(byteorder_i128)] + #[inline] + fn read_u128_into( + &mut self, + dst: &mut [u128], + ) -> Result<()> { + { + let buf = unsafe { slice_to_u8_mut(dst) }; + try!(self.read_exact(buf)); + } + T::from_slice_u128(dst); + Ok(()) + } + + /// Reads a sequence of signed 8 bit integers from the underlying reader. + /// + /// The given buffer is either filled completely or an error is returned. + /// If an error is returned, the contents of `dst` are unspecified. + /// + /// Note that since each `i8` is a single byte, no byte order conversions + /// are used. This method is included because it provides a safe, simple + /// way for the caller to read into a `&mut [i8]` buffer. (Without this + /// method, the caller would have to either use `unsafe` code or convert + /// each byte to `i8` individually.) + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read a sequence of signed 8 bit integers from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![2, 251, 3]); + /// let mut dst = [0; 3]; + /// rdr.read_i8_into(&mut dst).unwrap(); + /// assert_eq!([2, -5, 3], dst); + /// ``` + #[inline] + fn read_i8_into(&mut self, dst: &mut [i8]) -> Result<()> { + let buf = unsafe { slice_to_u8_mut(dst) }; + self.read_exact(buf) + } + + /// Reads a sequence of signed 16 bit integers from the underlying + /// reader. + /// + /// The given buffer is either filled completely or an error is returned. + /// If an error is returned, the contents of `dst` are unspecified. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read a sequence of signed 16 bit big-endian integers from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![2, 5, 3, 0]); + /// let mut dst = [0; 2]; + /// rdr.read_i16_into::(&mut dst).unwrap(); + /// assert_eq!([517, 768], dst); + /// ``` + #[inline] + fn read_i16_into(&mut self, dst: &mut [i16]) -> Result<()> { + { + let buf = unsafe { slice_to_u8_mut(dst) }; + try!(self.read_exact(buf)); + } + T::from_slice_i16(dst); + Ok(()) + } + + /// Reads a sequence of signed 32 bit integers from the underlying + /// reader. + /// + /// The given buffer is either filled completely or an error is returned. + /// If an error is returned, the contents of `dst` are unspecified. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read a sequence of signed 32 bit big-endian integers from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![0, 0, 2, 5, 0, 0, 3, 0]); + /// let mut dst = [0; 2]; + /// rdr.read_i32_into::(&mut dst).unwrap(); + /// assert_eq!([517, 768], dst); + /// ``` + #[inline] + fn read_i32_into(&mut self, dst: &mut [i32]) -> Result<()> { + { + let buf = unsafe { slice_to_u8_mut(dst) }; + try!(self.read_exact(buf)); + } + T::from_slice_i32(dst); + Ok(()) + } + + /// Reads a sequence of signed 64 bit integers from the underlying + /// reader. + /// + /// The given buffer is either filled completely or an error is returned. + /// If an error is returned, the contents of `dst` are unspecified. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read a sequence of signed 64 bit big-endian integers from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![ + /// 0, 0, 0, 0, 0, 0, 2, 5, + /// 0, 0, 0, 0, 0, 0, 3, 0, + /// ]); + /// let mut dst = [0; 2]; + /// rdr.read_i64_into::(&mut dst).unwrap(); + /// assert_eq!([517, 768], dst); + /// ``` + #[inline] + fn read_i64_into(&mut self, dst: &mut [i64]) -> Result<()> { + { + let buf = unsafe { slice_to_u8_mut(dst) }; + try!(self.read_exact(buf)); + } + T::from_slice_i64(dst); + Ok(()) + } + + /// Reads a sequence of signed 128 bit integers from the underlying + /// reader. + /// + /// The given buffer is either filled completely or an error is returned. + /// If an error is returned, the contents of `dst` are unspecified. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read a sequence of signed 128 bit big-endian integers from a `Read`: + /// + /// ```rust + /// use std::io::Cursor; + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![ + /// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, + /// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, + /// ]); + /// let mut dst = [0; 2]; + /// rdr.read_i128_into::(&mut dst).unwrap(); + /// assert_eq!([517, 768], dst); + /// ``` + #[cfg(byteorder_i128)] + #[inline] + fn read_i128_into( + &mut self, + dst: &mut [i128], + ) -> Result<()> { + { + let buf = unsafe { slice_to_u8_mut(dst) }; + try!(self.read_exact(buf)); + } + T::from_slice_i128(dst); + Ok(()) + } + + /// Reads a sequence of IEEE754 single-precision (4 bytes) floating + /// point numbers from the underlying reader. + /// + /// The given buffer is either filled completely or an error is returned. + /// If an error is returned, the contents of `dst` are unspecified. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read a sequence of big-endian single-precision floating point number + /// from a `Read`: + /// + /// ```rust + /// use std::f32; + /// use std::io::Cursor; + /// + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![ + /// 0x40, 0x49, 0x0f, 0xdb, + /// 0x3f, 0x80, 0x00, 0x00, + /// ]); + /// let mut dst = [0.0; 2]; + /// rdr.read_f32_into::(&mut dst).unwrap(); + /// assert_eq!([f32::consts::PI, 1.0], dst); + /// ``` + #[inline] + fn read_f32_into( + &mut self, + dst: &mut [f32], + ) -> Result<()> { + { + let buf = unsafe { slice_to_u8_mut(dst) }; + try!(self.read_exact(buf)); + } + T::from_slice_f32(dst); + Ok(()) + } + + /// **DEPRECATED**. + /// + /// This method is deprecated. Use `read_f32_into` instead. + /// + /// Reads a sequence of IEEE754 single-precision (4 bytes) floating + /// point numbers from the underlying reader. + /// + /// The given buffer is either filled completely or an error is returned. + /// If an error is returned, the contents of `dst` are unspecified. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read a sequence of big-endian single-precision floating point number + /// from a `Read`: + /// + /// ```rust + /// use std::f32; + /// use std::io::Cursor; + /// + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![ + /// 0x40, 0x49, 0x0f, 0xdb, + /// 0x3f, 0x80, 0x00, 0x00, + /// ]); + /// let mut dst = [0.0; 2]; + /// rdr.read_f32_into_unchecked::(&mut dst).unwrap(); + /// assert_eq!([f32::consts::PI, 1.0], dst); + /// ``` + #[inline] + #[deprecated(since="1.2.0", note="please use `read_f32_into` instead")] + fn read_f32_into_unchecked( + &mut self, + dst: &mut [f32], + ) -> Result<()> { + self.read_f32_into::(dst) + } + + /// Reads a sequence of IEEE754 double-precision (8 bytes) floating + /// point numbers from the underlying reader. + /// + /// The given buffer is either filled completely or an error is returned. + /// If an error is returned, the contents of `dst` are unspecified. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read a sequence of big-endian single-precision floating point number + /// from a `Read`: + /// + /// ```rust + /// use std::f64; + /// use std::io::Cursor; + /// + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![ + /// 0x40, 0x09, 0x21, 0xfb, 0x54, 0x44, 0x2d, 0x18, + /// 0x3f, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /// ]); + /// let mut dst = [0.0; 2]; + /// rdr.read_f64_into::(&mut dst).unwrap(); + /// assert_eq!([f64::consts::PI, 1.0], dst); + /// ``` + #[inline] + fn read_f64_into( + &mut self, + dst: &mut [f64], + ) -> Result<()> { + { + let buf = unsafe { slice_to_u8_mut(dst) }; + try!(self.read_exact(buf)); + } + T::from_slice_f64(dst); + Ok(()) + } + + /// **DEPRECATED**. + /// + /// This method is deprecated. Use `read_f64_into` instead. + /// + /// Reads a sequence of IEEE754 double-precision (8 bytes) floating + /// point numbers from the underlying reader. + /// + /// The given buffer is either filled completely or an error is returned. + /// If an error is returned, the contents of `dst` are unspecified. + /// + /// # Safety + /// + /// This method is unsafe because there are no guarantees made about the + /// floating point values. In particular, this method does not check for + /// signaling NaNs, which may result in undefined behavior. + /// + /// # Errors + /// + /// This method returns the same errors as [`Read::read_exact`]. + /// + /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact + /// + /// # Examples + /// + /// Read a sequence of big-endian single-precision floating point number + /// from a `Read`: + /// + /// ```rust + /// use std::f64; + /// use std::io::Cursor; + /// + /// use byteorder::{BigEndian, ReadBytesExt}; + /// + /// let mut rdr = Cursor::new(vec![ + /// 0x40, 0x09, 0x21, 0xfb, 0x54, 0x44, 0x2d, 0x18, + /// 0x3f, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /// ]); + /// let mut dst = [0.0; 2]; + /// rdr.read_f64_into_unchecked::(&mut dst).unwrap(); + /// assert_eq!([f64::consts::PI, 1.0], dst); + /// ``` + #[inline] + #[deprecated(since="1.2.0", note="please use `read_f64_into` instead")] + fn read_f64_into_unchecked( + &mut self, + dst: &mut [f64], + ) -> Result<()> { + self.read_f64_into::(dst) + } +} + +/// All types that implement `Read` get methods defined in `ReadBytesExt` +/// for free. +impl ReadBytesExt for R {} + +/// Extends [`Write`] with methods for writing numbers. (For `std::io`.) +/// +/// Most of the methods defined here have an unconstrained type parameter that +/// must be explicitly instantiated. Typically, it is instantiated with either +/// the [`BigEndian`] or [`LittleEndian`] types defined in this crate. +/// +/// # Examples +/// +/// Write unsigned 16 bit big-endian integers to a [`Write`]: +/// +/// ```rust +/// use byteorder::{BigEndian, WriteBytesExt}; +/// +/// let mut wtr = vec![]; +/// wtr.write_u16::(517).unwrap(); +/// wtr.write_u16::(768).unwrap(); +/// assert_eq!(wtr, vec![2, 5, 3, 0]); +/// ``` +/// +/// [`BigEndian`]: enum.BigEndian.html +/// [`LittleEndian`]: enum.LittleEndian.html +/// [`Write`]: https://doc.rust-lang.org/std/io/trait.Write.html +pub trait WriteBytesExt: io::Write { + /// Writes an unsigned 8 bit integer to the underlying writer. + /// + /// Note that since this writes a single byte, no byte order conversions + /// are used. It is included for completeness. + /// + /// # Errors + /// + /// This method returns the same errors as [`Write::write_all`]. + /// + /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all + /// + /// # Examples + /// + /// Write unsigned 8 bit integers to a `Write`: + /// + /// ```rust + /// use byteorder::WriteBytesExt; + /// + /// let mut wtr = Vec::new(); + /// wtr.write_u8(2).unwrap(); + /// wtr.write_u8(5).unwrap(); + /// assert_eq!(wtr, b"\x02\x05"); + /// ``` + #[inline] + fn write_u8(&mut self, n: u8) -> Result<()> { + self.write_all(&[n]) + } + + /// Writes a signed 8 bit integer to the underlying writer. + /// + /// Note that since this writes a single byte, no byte order conversions + /// are used. It is included for completeness. + /// + /// # Errors + /// + /// This method returns the same errors as [`Write::write_all`]. + /// + /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all + /// + /// # Examples + /// + /// Write signed 8 bit integers to a `Write`: + /// + /// ```rust + /// use byteorder::WriteBytesExt; + /// + /// let mut wtr = Vec::new(); + /// wtr.write_i8(2).unwrap(); + /// wtr.write_i8(-5).unwrap(); + /// assert_eq!(wtr, b"\x02\xfb"); + /// ``` + #[inline] + fn write_i8(&mut self, n: i8) -> Result<()> { + self.write_all(&[n as u8]) + } + + /// Writes an unsigned 16 bit integer to the underlying writer. + /// + /// # Errors + /// + /// This method returns the same errors as [`Write::write_all`]. + /// + /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all + /// + /// # Examples + /// + /// Write unsigned 16 bit big-endian integers to a `Write`: + /// + /// ```rust + /// use byteorder::{BigEndian, WriteBytesExt}; + /// + /// let mut wtr = Vec::new(); + /// wtr.write_u16::(517).unwrap(); + /// wtr.write_u16::(768).unwrap(); + /// assert_eq!(wtr, b"\x02\x05\x03\x00"); + /// ``` + #[inline] + fn write_u16(&mut self, n: u16) -> Result<()> { + let mut buf = [0; 2]; + T::write_u16(&mut buf, n); + self.write_all(&buf) + } + + /// Writes a signed 16 bit integer to the underlying writer. + /// + /// # Errors + /// + /// This method returns the same errors as [`Write::write_all`]. + /// + /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all + /// + /// # Examples + /// + /// Write signed 16 bit big-endian integers to a `Write`: + /// + /// ```rust + /// use byteorder::{BigEndian, WriteBytesExt}; + /// + /// let mut wtr = Vec::new(); + /// wtr.write_i16::(193).unwrap(); + /// wtr.write_i16::(-132).unwrap(); + /// assert_eq!(wtr, b"\x00\xc1\xff\x7c"); + /// ``` + #[inline] + fn write_i16(&mut self, n: i16) -> Result<()> { + let mut buf = [0; 2]; + T::write_i16(&mut buf, n); + self.write_all(&buf) + } + + /// Writes an unsigned 24 bit integer to the underlying writer. + /// + /// # Errors + /// + /// This method returns the same errors as [`Write::write_all`]. + /// + /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all + /// + /// # Examples + /// + /// Write unsigned 24 bit big-endian integers to a `Write`: + /// + /// ```rust + /// use byteorder::{BigEndian, WriteBytesExt}; + /// + /// let mut wtr = Vec::new(); + /// wtr.write_u24::(267).unwrap(); + /// wtr.write_u24::(120111).unwrap(); + /// assert_eq!(wtr, b"\x00\x01\x0b\x01\xd5\x2f"); + /// ``` + #[inline] + fn write_u24(&mut self, n: u32) -> Result<()> { + let mut buf = [0; 3]; + T::write_u24(&mut buf, n); + self.write_all(&buf) + } + + /// Writes a signed 24 bit integer to the underlying writer. + /// + /// # Errors + /// + /// This method returns the same errors as [`Write::write_all`]. + /// + /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all + /// + /// # Examples + /// + /// Write signed 24 bit big-endian integers to a `Write`: + /// + /// ```rust + /// use byteorder::{BigEndian, WriteBytesExt}; + /// + /// let mut wtr = Vec::new(); + /// wtr.write_i24::(-34253).unwrap(); + /// wtr.write_i24::(120111).unwrap(); + /// assert_eq!(wtr, b"\xff\x7a\x33\x01\xd5\x2f"); + /// ``` + #[inline] + fn write_i24(&mut self, n: i32) -> Result<()> { + let mut buf = [0; 3]; + T::write_i24(&mut buf, n); + self.write_all(&buf) + } + + /// Writes an unsigned 32 bit integer to the underlying writer. + /// + /// # Errors + /// + /// This method returns the same errors as [`Write::write_all`]. + /// + /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all + /// + /// # Examples + /// + /// Write unsigned 32 bit big-endian integers to a `Write`: + /// + /// ```rust + /// use byteorder::{BigEndian, WriteBytesExt}; + /// + /// let mut wtr = Vec::new(); + /// wtr.write_u32::(267).unwrap(); + /// wtr.write_u32::(1205419366).unwrap(); + /// assert_eq!(wtr, b"\x00\x00\x01\x0b\x47\xd9\x3d\x66"); + /// ``` + #[inline] + fn write_u32(&mut self, n: u32) -> Result<()> { + let mut buf = [0; 4]; + T::write_u32(&mut buf, n); + self.write_all(&buf) + } + + /// Writes a signed 32 bit integer to the underlying writer. + /// + /// # Errors + /// + /// This method returns the same errors as [`Write::write_all`]. + /// + /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all + /// + /// # Examples + /// + /// Write signed 32 bit big-endian integers to a `Write`: + /// + /// ```rust + /// use byteorder::{BigEndian, WriteBytesExt}; + /// + /// let mut wtr = Vec::new(); + /// wtr.write_i32::(-34253).unwrap(); + /// wtr.write_i32::(1205419366).unwrap(); + /// assert_eq!(wtr, b"\xff\xff\x7a\x33\x47\xd9\x3d\x66"); + /// ``` + #[inline] + fn write_i32(&mut self, n: i32) -> Result<()> { + let mut buf = [0; 4]; + T::write_i32(&mut buf, n); + self.write_all(&buf) + } + + /// Writes an unsigned 48 bit integer to the underlying writer. + /// + /// # Errors + /// + /// This method returns the same errors as [`Write::write_all`]. + /// + /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all + /// + /// # Examples + /// + /// Write unsigned 48 bit big-endian integers to a `Write`: + /// + /// ```rust + /// use byteorder::{BigEndian, WriteBytesExt}; + /// + /// let mut wtr = Vec::new(); + /// wtr.write_u48::(52360336390828).unwrap(); + /// wtr.write_u48::(541).unwrap(); + /// assert_eq!(wtr, b"\x2f\x9f\x17\x40\x3a\xac\x00\x00\x00\x00\x02\x1d"); + /// ``` + #[inline] + fn write_u48(&mut self, n: u64) -> Result<()> { + let mut buf = [0; 6]; + T::write_u48(&mut buf, n); + self.write_all(&buf) + } + + /// Writes a signed 48 bit integer to the underlying writer. + /// + /// # Errors + /// + /// This method returns the same errors as [`Write::write_all`]. + /// + /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all + /// + /// # Examples + /// + /// Write signed 48 bit big-endian integers to a `Write`: + /// + /// ```rust + /// use byteorder::{BigEndian, WriteBytesExt}; + /// + /// let mut wtr = Vec::new(); + /// wtr.write_i48::(-108363435763825).unwrap(); + /// wtr.write_i48::(77).unwrap(); + /// assert_eq!(wtr, b"\x9d\x71\xab\xe7\x97\x8f\x00\x00\x00\x00\x00\x4d"); + /// ``` + #[inline] + fn write_i48(&mut self, n: i64) -> Result<()> { + let mut buf = [0; 6]; + T::write_i48(&mut buf, n); + self.write_all(&buf) + } + + /// Writes an unsigned 64 bit integer to the underlying writer. + /// + /// # Errors + /// + /// This method returns the same errors as [`Write::write_all`]. + /// + /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all + /// + /// # Examples + /// + /// Write unsigned 64 bit big-endian integers to a `Write`: + /// + /// ```rust + /// use byteorder::{BigEndian, WriteBytesExt}; + /// + /// let mut wtr = Vec::new(); + /// wtr.write_u64::(918733457491587).unwrap(); + /// wtr.write_u64::(143).unwrap(); + /// assert_eq!(wtr, b"\x00\x03\x43\x95\x4d\x60\x86\x83\x00\x00\x00\x00\x00\x00\x00\x8f"); + /// ``` + #[inline] + fn write_u64(&mut self, n: u64) -> Result<()> { + let mut buf = [0; 8]; + T::write_u64(&mut buf, n); + self.write_all(&buf) + } + + /// Writes a signed 64 bit integer to the underlying writer. + /// + /// # Errors + /// + /// This method returns the same errors as [`Write::write_all`]. + /// + /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all + /// + /// # Examples + /// + /// Write signed 64 bit big-endian integers to a `Write`: + /// + /// ```rust + /// use byteorder::{BigEndian, WriteBytesExt}; + /// + /// let mut wtr = Vec::new(); + /// wtr.write_i64::(i64::min_value()).unwrap(); + /// wtr.write_i64::(i64::max_value()).unwrap(); + /// assert_eq!(wtr, b"\x80\x00\x00\x00\x00\x00\x00\x00\x7f\xff\xff\xff\xff\xff\xff\xff"); + /// ``` + #[inline] + fn write_i64(&mut self, n: i64) -> Result<()> { + let mut buf = [0; 8]; + T::write_i64(&mut buf, n); + self.write_all(&buf) + } + + /// Writes an unsigned 128 bit integer to the underlying writer. + #[cfg(byteorder_i128)] + #[inline] + fn write_u128(&mut self, n: u128) -> Result<()> { + let mut buf = [0; 16]; + T::write_u128(&mut buf, n); + self.write_all(&buf) + } + + /// Writes a signed 128 bit integer to the underlying writer. + #[cfg(byteorder_i128)] + #[inline] + fn write_i128(&mut self, n: i128) -> Result<()> { + let mut buf = [0; 16]; + T::write_i128(&mut buf, n); + self.write_all(&buf) + } + + /// Writes an unsigned n-bytes integer to the underlying writer. + /// + /// # Errors + /// + /// This method returns the same errors as [`Write::write_all`]. + /// + /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all + /// + /// # Panics + /// + /// If the given integer is not representable in the given number of bytes, + /// this method panics. If `nbytes > 8`, this method panics. + /// + /// # Examples + /// + /// Write unsigned 40 bit big-endian integers to a `Write`: + /// + /// ```rust + /// use byteorder::{BigEndian, WriteBytesExt}; + /// + /// let mut wtr = Vec::new(); + /// wtr.write_uint::(312550384361, 5).unwrap(); + /// wtr.write_uint::(43, 5).unwrap(); + /// assert_eq!(wtr, b"\x48\xc5\x74\x62\xe9\x00\x00\x00\x00\x2b"); + /// ``` + #[inline] + fn write_uint( + &mut self, + n: u64, + nbytes: usize, + ) -> Result<()> { + let mut buf = [0; 8]; + T::write_uint(&mut buf, n, nbytes); + self.write_all(&buf[0..nbytes]) + } + + /// Writes a signed n-bytes integer to the underlying writer. + /// + /// # Errors + /// + /// This method returns the same errors as [`Write::write_all`]. + /// + /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all + /// + /// # Panics + /// + /// If the given integer is not representable in the given number of bytes, + /// this method panics. If `nbytes > 8`, this method panics. + /// + /// # Examples + /// + /// Write signed 56 bit big-endian integers to a `Write`: + /// + /// ```rust + /// use byteorder::{BigEndian, WriteBytesExt}; + /// + /// let mut wtr = Vec::new(); + /// wtr.write_int::(-3548172039376767, 7).unwrap(); + /// wtr.write_int::(43, 7).unwrap(); + /// assert_eq!(wtr, b"\xf3\x64\xf4\xd1\xfd\xb0\x81\x00\x00\x00\x00\x00\x00\x2b"); + /// ``` + #[inline] + fn write_int( + &mut self, + n: i64, + nbytes: usize, + ) -> Result<()> { + let mut buf = [0; 8]; + T::write_int(&mut buf, n, nbytes); + self.write_all(&buf[0..nbytes]) + } + + /// Writes an unsigned n-bytes integer to the underlying writer. + /// + /// If the given integer is not representable in the given number of bytes, + /// this method panics. If `nbytes > 16`, this method panics. + #[cfg(byteorder_i128)] + #[inline] + fn write_uint128( + &mut self, + n: u128, + nbytes: usize, + ) -> Result<()> { + let mut buf = [0; 16]; + T::write_uint128(&mut buf, n, nbytes); + self.write_all(&buf[0..nbytes]) + } + + /// Writes a signed n-bytes integer to the underlying writer. + /// + /// If the given integer is not representable in the given number of bytes, + /// this method panics. If `nbytes > 16`, this method panics. + #[cfg(byteorder_i128)] + #[inline] + fn write_int128( + &mut self, + n: i128, + nbytes: usize, + ) -> Result<()> { + let mut buf = [0; 16]; + T::write_int128(&mut buf, n, nbytes); + self.write_all(&buf[0..nbytes]) + } + + /// Writes a IEEE754 single-precision (4 bytes) floating point number to + /// the underlying writer. + /// + /// # Errors + /// + /// This method returns the same errors as [`Write::write_all`]. + /// + /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all + /// + /// # Examples + /// + /// Write a big-endian single-precision floating point number to a `Write`: + /// + /// ```rust + /// use std::f32; + /// + /// use byteorder::{BigEndian, WriteBytesExt}; + /// + /// let mut wtr = Vec::new(); + /// wtr.write_f32::(f32::consts::PI).unwrap(); + /// assert_eq!(wtr, b"\x40\x49\x0f\xdb"); + /// ``` + #[inline] + fn write_f32(&mut self, n: f32) -> Result<()> { + let mut buf = [0; 4]; + T::write_f32(&mut buf, n); + self.write_all(&buf) + } + + /// Writes a IEEE754 double-precision (8 bytes) floating point number to + /// the underlying writer. + /// + /// # Errors + /// + /// This method returns the same errors as [`Write::write_all`]. + /// + /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all + /// + /// # Examples + /// + /// Write a big-endian double-precision floating point number to a `Write`: + /// + /// ```rust + /// use std::f64; + /// + /// use byteorder::{BigEndian, WriteBytesExt}; + /// + /// let mut wtr = Vec::new(); + /// wtr.write_f64::(f64::consts::PI).unwrap(); + /// assert_eq!(wtr, b"\x40\x09\x21\xfb\x54\x44\x2d\x18"); + /// ``` + #[inline] + fn write_f64(&mut self, n: f64) -> Result<()> { + let mut buf = [0; 8]; + T::write_f64(&mut buf, n); + self.write_all(&buf) + } +} + +/// All types that implement `Write` get methods defined in `WriteBytesExt` +/// for free. +impl WriteBytesExt for W {} + +/// Convert a slice of T (where T is plain old data) to its mutable binary +/// representation. +/// +/// This function is wildly unsafe because it permits arbitrary modification of +/// the binary representation of any `Copy` type. Use with care. +unsafe fn slice_to_u8_mut(slice: &mut [T]) -> &mut [u8] { + use std::mem::size_of; + + let len = size_of::() * slice.len(); + slice::from_raw_parts_mut(slice.as_mut_ptr() as *mut u8, len) +} diff --git a/crux-mir/lib/byteorder/lib.rs b/crux-mir/lib/byteorder/lib.rs new file mode 100644 index 000000000..0909a3b95 --- /dev/null +++ b/crux-mir/lib/byteorder/lib.rs @@ -0,0 +1,3204 @@ +/*! +This crate provides convenience methods for encoding and decoding numbers in +either [big-endian or little-endian order]. + +The organization of the crate is pretty simple. A trait, [`ByteOrder`], specifies +byte conversion methods for each type of number in Rust (sans numbers that have +a platform dependent size like `usize` and `isize`). Two types, [`BigEndian`] +and [`LittleEndian`] implement these methods. Finally, [`ReadBytesExt`] and +[`WriteBytesExt`] provide convenience methods available to all types that +implement [`Read`] and [`Write`]. + +An alias, [`NetworkEndian`], for [`BigEndian`] is provided to help improve +code clarity. + +An additional alias, [`NativeEndian`], is provided for the endianness of the +local platform. This is convenient when serializing data for use and +conversions are not desired. + +# Examples + +Read unsigned 16 bit big-endian integers from a [`Read`] type: + +```rust +use std::io::Cursor; +use byteorder::{BigEndian, ReadBytesExt}; + +let mut rdr = Cursor::new(vec![2, 5, 3, 0]); +// Note that we use type parameters to indicate which kind of byte order +// we want! +assert_eq!(517, rdr.read_u16::().unwrap()); +assert_eq!(768, rdr.read_u16::().unwrap()); +``` + +Write unsigned 16 bit little-endian integers to a [`Write`] type: + +```rust +use byteorder::{LittleEndian, WriteBytesExt}; + +let mut wtr = vec![]; +wtr.write_u16::(517).unwrap(); +wtr.write_u16::(768).unwrap(); +assert_eq!(wtr, vec![5, 2, 0, 3]); +``` + +# Optional Features + +This crate optionally provides support for 128 bit values (`i128` and `u128`) +when built with the `i128` feature enabled. + +This crate can also be used without the standard library. + +# Alternatives + +Note that as of Rust 1.32, the standard numeric types provide built-in methods +like `to_le_bytes` and `from_le_bytes`, which support some of the same use +cases. + +[big-endian or little-endian order]: https://en.wikipedia.org/wiki/Endianness +[`ByteOrder`]: trait.ByteOrder.html +[`BigEndian`]: enum.BigEndian.html +[`LittleEndian`]: enum.LittleEndian.html +[`ReadBytesExt`]: trait.ReadBytesExt.html +[`WriteBytesExt`]: trait.WriteBytesExt.html +[`NetworkEndian`]: type.NetworkEndian.html +[`NativeEndian`]: type.NativeEndian.html +[`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html +[`Write`]: https://doc.rust-lang.org/std/io/trait.Write.html +*/ + +#![cfg_attr(crux, feature(crucible_intrinsics))] +#![deny(missing_docs)] +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg(feature = "std")] +extern crate core; + +#[cfg(test)] +#[macro_use] +extern crate doc_comment; + +#[cfg(test)] +doctest!("../README.md"); + +use core::fmt::Debug; +use core::hash::Hash; +use core::mem; +use core::ptr::copy_nonoverlapping; +use core::slice; + +#[cfg(feature = "std")] +pub use io::{ReadBytesExt, WriteBytesExt}; + +#[cfg(feature = "std")] +mod io; + +#[cfg(crux)] use core::mem::crucible_identity_transmute as identity_transmute; +#[cfg(not(crux))] use core::mem::transmute as identity_transmute; + +#[inline] +fn extend_sign(val: u64, nbytes: usize) -> i64 { + let shift = (8 - nbytes) * 8; + (val << shift) as i64 >> shift +} + +#[cfg(byteorder_i128)] +#[inline] +fn extend_sign128(val: u128, nbytes: usize) -> i128 { + let shift = (16 - nbytes) * 8; + (val << shift) as i128 >> shift +} + +#[inline] +fn unextend_sign(val: i64, nbytes: usize) -> u64 { + let shift = (8 - nbytes) * 8; + (val << shift) as u64 >> shift +} + +#[cfg(byteorder_i128)] +#[inline] +fn unextend_sign128(val: i128, nbytes: usize) -> u128 { + let shift = (16 - nbytes) * 8; + (val << shift) as u128 >> shift +} + +#[inline] +fn pack_size(n: u64) -> usize { + if n < 1 << 8 { + 1 + } else if n < 1 << 16 { + 2 + } else if n < 1 << 24 { + 3 + } else if n < 1 << 32 { + 4 + } else if n < 1 << 40 { + 5 + } else if n < 1 << 48 { + 6 + } else if n < 1 << 56 { + 7 + } else { + 8 + } +} + +#[cfg(byteorder_i128)] +#[inline] +fn pack_size128(n: u128) -> usize { + if n < 1 << 8 { + 1 + } else if n < 1 << 16 { + 2 + } else if n < 1 << 24 { + 3 + } else if n < 1 << 32 { + 4 + } else if n < 1 << 40 { + 5 + } else if n < 1 << 48 { + 6 + } else if n < 1 << 56 { + 7 + } else if n < 1 << 64 { + 8 + } else if n < 1 << 72 { + 9 + } else if n < 1 << 80 { + 10 + } else if n < 1 << 88 { + 11 + } else if n < 1 << 96 { + 12 + } else if n < 1 << 104 { + 13 + } else if n < 1 << 112 { + 14 + } else if n < 1 << 120 { + 15 + } else { + 16 + } +} + +mod private { + /// Sealed stops crates other than byteorder from implementing any traits + /// that use it. + pub trait Sealed{} + impl Sealed for super::LittleEndian {} + impl Sealed for super::BigEndian {} +} + +/// `ByteOrder` describes types that can serialize integers as bytes. +/// +/// Note that `Self` does not appear anywhere in this trait's definition! +/// Therefore, in order to use it, you'll need to use syntax like +/// `T::read_u16(&[0, 1])` where `T` implements `ByteOrder`. +/// +/// This crate provides two types that implement `ByteOrder`: [`BigEndian`] +/// and [`LittleEndian`]. +/// This trait is sealed and cannot be implemented for callers to avoid +/// breaking backwards compatibility when adding new derived traits. +/// +/// # Examples +/// +/// Write and read `u32` numbers in little endian order: +/// +/// ```rust +/// use byteorder::{ByteOrder, LittleEndian}; +/// +/// let mut buf = [0; 4]; +/// LittleEndian::write_u32(&mut buf, 1_000_000); +/// assert_eq!(1_000_000, LittleEndian::read_u32(&buf)); +/// ``` +/// +/// Write and read `i16` numbers in big endian order: +/// +/// ```rust +/// use byteorder::{ByteOrder, BigEndian}; +/// +/// let mut buf = [0; 2]; +/// BigEndian::write_i16(&mut buf, -5_000); +/// assert_eq!(-5_000, BigEndian::read_i16(&buf)); +/// ``` +/// +/// [`BigEndian`]: enum.BigEndian.html +/// [`LittleEndian`]: enum.LittleEndian.html +pub trait ByteOrder + : Clone + Copy + Debug + Default + Eq + Hash + Ord + PartialEq + PartialOrd + + private::Sealed +{ + /// Reads an unsigned 16 bit integer from `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 2`. + fn read_u16(buf: &[u8]) -> u16; + + /// Reads an unsigned 24 bit integer from `buf`, stored in u32. + /// + /// # Panics + /// + /// Panics when `buf.len() < 3`. + /// + /// # Examples + /// + /// Write and read 24 bit `u32` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 3]; + /// LittleEndian::write_u24(&mut buf, 1_000_000); + /// assert_eq!(1_000_000, LittleEndian::read_u24(&buf)); + /// ``` + fn read_u24(buf: &[u8]) -> u32 { + Self::read_uint(buf, 3) as u32 + } + + /// Reads an unsigned 32 bit integer from `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 4`. + /// + /// # Examples + /// + /// Write and read `u32` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 4]; + /// LittleEndian::write_u32(&mut buf, 1_000_000); + /// assert_eq!(1_000_000, LittleEndian::read_u32(&buf)); + /// ``` + fn read_u32(buf: &[u8]) -> u32; + + /// Reads an unsigned 48 bit integer from `buf`, stored in u64. + /// + /// # Panics + /// + /// Panics when `buf.len() < 6`. + /// + /// # Examples + /// + /// Write and read 48 bit `u64` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 6]; + /// LittleEndian::write_u48(&mut buf, 1_000_000_000_000); + /// assert_eq!(1_000_000_000_000, LittleEndian::read_u48(&buf)); + /// ``` + fn read_u48(buf: &[u8]) -> u64 { + Self::read_uint(buf, 6) as u64 + } + + /// Reads an unsigned 64 bit integer from `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 8`. + /// + /// # Examples + /// + /// Write and read `u64` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 8]; + /// LittleEndian::write_u64(&mut buf, 1_000_000); + /// assert_eq!(1_000_000, LittleEndian::read_u64(&buf)); + /// ``` + fn read_u64(buf: &[u8]) -> u64; + + /// Reads an unsigned 128 bit integer from `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 16`. + /// + /// # Examples + /// + /// Write and read `u128` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 16]; + /// LittleEndian::write_u128(&mut buf, 1_000_000); + /// assert_eq!(1_000_000, LittleEndian::read_u128(&buf)); + /// ``` + #[cfg(byteorder_i128)] + fn read_u128(buf: &[u8]) -> u128; + + /// Reads an unsigned n-bytes integer from `buf`. + /// + /// # Panics + /// + /// Panics when `nbytes < 1` or `nbytes > 8` or + /// `buf.len() < nbytes` + /// + /// # Examples + /// + /// Write and read an n-byte number in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 3]; + /// LittleEndian::write_uint(&mut buf, 1_000_000, 3); + /// assert_eq!(1_000_000, LittleEndian::read_uint(&buf, 3)); + /// ``` + fn read_uint(buf: &[u8], nbytes: usize) -> u64; + + /// Reads an unsigned n-bytes integer from `buf`. + /// + /// # Panics + /// + /// Panics when `nbytes < 1` or `nbytes > 16` or + /// `buf.len() < nbytes` + /// + /// # Examples + /// + /// Write and read an n-byte number in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 3]; + /// LittleEndian::write_uint128(&mut buf, 1_000_000, 3); + /// assert_eq!(1_000_000, LittleEndian::read_uint128(&buf, 3)); + /// ``` + #[cfg(byteorder_i128)] + fn read_uint128(buf: &[u8], nbytes: usize) -> u128; + + /// Writes an unsigned 16 bit integer `n` to `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 2`. + /// + /// # Examples + /// + /// Write and read `u16` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 2]; + /// LittleEndian::write_u16(&mut buf, 1_000); + /// assert_eq!(1_000, LittleEndian::read_u16(&buf)); + /// ``` + fn write_u16(buf: &mut [u8], n: u16); + + /// Writes an unsigned 24 bit integer `n` to `buf`, stored in u32. + /// + /// # Panics + /// + /// Panics when `buf.len() < 3`. + /// + /// # Examples + /// + /// Write and read 24 bit `u32` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 3]; + /// LittleEndian::write_u24(&mut buf, 1_000_000); + /// assert_eq!(1_000_000, LittleEndian::read_u24(&buf)); + /// ``` + fn write_u24(buf: &mut [u8], n: u32) { + Self::write_uint(buf, n as u64, 3) + } + + /// Writes an unsigned 32 bit integer `n` to `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 4`. + /// + /// # Examples + /// + /// Write and read `u32` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 4]; + /// LittleEndian::write_u32(&mut buf, 1_000_000); + /// assert_eq!(1_000_000, LittleEndian::read_u32(&buf)); + /// ``` + fn write_u32(buf: &mut [u8], n: u32); + + /// Writes an unsigned 48 bit integer `n` to `buf`, stored in u64. + /// + /// # Panics + /// + /// Panics when `buf.len() < 6`. + /// + /// # Examples + /// + /// Write and read 48 bit `u64` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 6]; + /// LittleEndian::write_u48(&mut buf, 1_000_000_000_000); + /// assert_eq!(1_000_000_000_000, LittleEndian::read_u48(&buf)); + /// ``` + fn write_u48(buf: &mut [u8], n: u64) { + Self::write_uint(buf, n as u64, 6) + } + + /// Writes an unsigned 64 bit integer `n` to `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 8`. + /// + /// # Examples + /// + /// Write and read `u64` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 8]; + /// LittleEndian::write_u64(&mut buf, 1_000_000); + /// assert_eq!(1_000_000, LittleEndian::read_u64(&buf)); + /// ``` + fn write_u64(buf: &mut [u8], n: u64); + + /// Writes an unsigned 128 bit integer `n` to `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 16`. + /// + /// # Examples + /// + /// Write and read `u128` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 16]; + /// LittleEndian::write_u128(&mut buf, 1_000_000); + /// assert_eq!(1_000_000, LittleEndian::read_u128(&buf)); + /// ``` + #[cfg(byteorder_i128)] + fn write_u128(buf: &mut [u8], n: u128); + + /// Writes an unsigned integer `n` to `buf` using only `nbytes`. + /// + /// # Panics + /// + /// If `n` is not representable in `nbytes`, or if `nbytes` is `> 8`, then + /// this method panics. + /// + /// # Examples + /// + /// Write and read an n-byte number in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 3]; + /// LittleEndian::write_uint(&mut buf, 1_000_000, 3); + /// assert_eq!(1_000_000, LittleEndian::read_uint(&buf, 3)); + /// ``` + fn write_uint(buf: &mut [u8], n: u64, nbytes: usize); + + /// Writes an unsigned integer `n` to `buf` using only `nbytes`. + /// + /// # Panics + /// + /// If `n` is not representable in `nbytes`, or if `nbytes` is `> 16`, then + /// this method panics. + /// + /// # Examples + /// + /// Write and read an n-byte number in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 3]; + /// LittleEndian::write_uint128(&mut buf, 1_000_000, 3); + /// assert_eq!(1_000_000, LittleEndian::read_uint128(&buf, 3)); + /// ``` + #[cfg(byteorder_i128)] + fn write_uint128(buf: &mut [u8], n: u128, nbytes: usize); + + /// Reads a signed 16 bit integer from `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 2`. + /// + /// # Examples + /// + /// Write and read `i16` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 2]; + /// LittleEndian::write_i16(&mut buf, -1_000); + /// assert_eq!(-1_000, LittleEndian::read_i16(&buf)); + /// ``` + #[inline] + fn read_i16(buf: &[u8]) -> i16 { + Self::read_u16(buf) as i16 + } + + /// Reads a signed 24 bit integer from `buf`, stored in i32. + /// + /// # Panics + /// + /// Panics when `buf.len() < 3`. + /// + /// # Examples + /// + /// Write and read 24 bit `i32` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 3]; + /// LittleEndian::write_i24(&mut buf, -1_000_000); + /// assert_eq!(-1_000_000, LittleEndian::read_i24(&buf)); + /// ``` + #[inline] + fn read_i24(buf: &[u8]) -> i32 { + Self::read_int(buf, 3) as i32 + } + + /// Reads a signed 32 bit integer from `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 4`. + /// + /// # Examples + /// + /// Write and read `i32` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 4]; + /// LittleEndian::write_i32(&mut buf, -1_000_000); + /// assert_eq!(-1_000_000, LittleEndian::read_i32(&buf)); + /// ``` + #[inline] + fn read_i32(buf: &[u8]) -> i32 { + Self::read_u32(buf) as i32 + } + + /// Reads a signed 48 bit integer from `buf`, stored in i64. + /// + /// # Panics + /// + /// Panics when `buf.len() < 6`. + /// + /// # Examples + /// + /// Write and read 48 bit `i64` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 6]; + /// LittleEndian::write_i48(&mut buf, -1_000_000_000_000); + /// assert_eq!(-1_000_000_000_000, LittleEndian::read_i48(&buf)); + /// ``` + #[inline] + fn read_i48(buf: &[u8]) -> i64 { + Self::read_int(buf, 6) as i64 + } + + /// Reads a signed 64 bit integer from `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 8`. + /// + /// # Examples + /// + /// Write and read `i64` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 8]; + /// LittleEndian::write_i64(&mut buf, -1_000_000_000); + /// assert_eq!(-1_000_000_000, LittleEndian::read_i64(&buf)); + /// ``` + #[inline] + fn read_i64(buf: &[u8]) -> i64 { + Self::read_u64(buf) as i64 + } + + /// Reads a signed 128 bit integer from `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 16`. + /// + /// # Examples + /// + /// Write and read `i128` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 16]; + /// LittleEndian::write_i128(&mut buf, -1_000_000_000); + /// assert_eq!(-1_000_000_000, LittleEndian::read_i128(&buf)); + /// ``` + #[cfg(byteorder_i128)] + #[inline] + fn read_i128(buf: &[u8]) -> i128 { + Self::read_u128(buf) as i128 + } + + /// Reads a signed n-bytes integer from `buf`. + /// + /// # Panics + /// + /// Panics when `nbytes < 1` or `nbytes > 8` or + /// `buf.len() < nbytes` + /// + /// # Examples + /// + /// Write and read n-length signed numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 3]; + /// LittleEndian::write_int(&mut buf, -1_000, 3); + /// assert_eq!(-1_000, LittleEndian::read_int(&buf, 3)); + /// ``` + #[inline] + fn read_int(buf: &[u8], nbytes: usize) -> i64 { + extend_sign(Self::read_uint(buf, nbytes), nbytes) + } + + /// Reads a signed n-bytes integer from `buf`. + /// + /// # Panics + /// + /// Panics when `nbytes < 1` or `nbytes > 16` or + /// `buf.len() < nbytes` + /// + /// # Examples + /// + /// Write and read n-length signed numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 3]; + /// LittleEndian::write_int128(&mut buf, -1_000, 3); + /// assert_eq!(-1_000, LittleEndian::read_int128(&buf, 3)); + /// ``` + #[cfg(byteorder_i128)] + #[inline] + fn read_int128(buf: &[u8], nbytes: usize) -> i128 { + extend_sign128(Self::read_uint128(buf, nbytes), nbytes) + } + + /// Reads a IEEE754 single-precision (4 bytes) floating point number. + /// + /// # Panics + /// + /// Panics when `buf.len() < 4`. + /// + /// # Examples + /// + /// Write and read `f32` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let e = 2.71828; + /// let mut buf = [0; 4]; + /// LittleEndian::write_f32(&mut buf, e); + /// assert_eq!(e, LittleEndian::read_f32(&buf)); + /// ``` + #[inline] + fn read_f32(buf: &[u8]) -> f32 { + unimplemented!() + } + + /// Reads a IEEE754 double-precision (8 bytes) floating point number. + /// + /// # Panics + /// + /// Panics when `buf.len() < 8`. + /// + /// # Examples + /// + /// Write and read `f64` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let phi = 1.6180339887; + /// let mut buf = [0; 8]; + /// LittleEndian::write_f64(&mut buf, phi); + /// assert_eq!(phi, LittleEndian::read_f64(&buf)); + /// ``` + #[inline] + fn read_f64(buf: &[u8]) -> f64 { + unimplemented!() + } + + /// Writes a signed 16 bit integer `n` to `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 2`. + /// + /// # Examples + /// + /// Write and read `i16` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 2]; + /// LittleEndian::write_i16(&mut buf, -1_000); + /// assert_eq!(-1_000, LittleEndian::read_i16(&buf)); + /// ``` + #[inline] + fn write_i16(buf: &mut [u8], n: i16) { + Self::write_u16(buf, n as u16) + } + + /// Writes a signed 24 bit integer `n` to `buf`, stored in i32. + /// + /// # Panics + /// + /// Panics when `buf.len() < 3`. + /// + /// # Examples + /// + /// Write and read 24 bit `i32` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 3]; + /// LittleEndian::write_i24(&mut buf, -1_000_000); + /// assert_eq!(-1_000_000, LittleEndian::read_i24(&buf)); + /// ``` + #[inline] + fn write_i24(buf: &mut [u8], n: i32) { + Self::write_int(buf, n as i64, 3) + } + + /// Writes a signed 32 bit integer `n` to `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 4`. + /// + /// # Examples + /// + /// Write and read `i32` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 4]; + /// LittleEndian::write_i32(&mut buf, -1_000_000); + /// assert_eq!(-1_000_000, LittleEndian::read_i32(&buf)); + /// ``` + #[inline] + fn write_i32(buf: &mut [u8], n: i32) { + Self::write_u32(buf, n as u32) + } + + /// Writes a signed 48 bit integer `n` to `buf`, stored in i64. + /// + /// # Panics + /// + /// Panics when `buf.len() < 6`. + /// + /// # Examples + /// + /// Write and read 48 bit `i64` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 6]; + /// LittleEndian::write_i48(&mut buf, -1_000_000_000_000); + /// assert_eq!(-1_000_000_000_000, LittleEndian::read_i48(&buf)); + /// ``` + #[inline] + fn write_i48(buf: &mut [u8], n: i64) { + Self::write_int(buf, n as i64, 6) + } + + /// Writes a signed 64 bit integer `n` to `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 8`. + /// + /// # Examples + /// + /// Write and read `i64` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 8]; + /// LittleEndian::write_i64(&mut buf, -1_000_000_000); + /// assert_eq!(-1_000_000_000, LittleEndian::read_i64(&buf)); + /// ``` + #[inline] + fn write_i64(buf: &mut [u8], n: i64) { + Self::write_u64(buf, n as u64) + } + + /// Writes a signed 128 bit integer `n` to `buf`. + /// + /// # Panics + /// + /// Panics when `buf.len() < 16`. + /// + /// # Examples + /// + /// Write and read n-byte `i128` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 16]; + /// LittleEndian::write_i128(&mut buf, -1_000_000_000); + /// assert_eq!(-1_000_000_000, LittleEndian::read_i128(&buf)); + /// ``` + #[cfg(byteorder_i128)] + #[inline] + fn write_i128(buf: &mut [u8], n: i128) { + Self::write_u128(buf, n as u128) + } + + /// Writes a signed integer `n` to `buf` using only `nbytes`. + /// + /// # Panics + /// + /// If `n` is not representable in `nbytes`, or if `nbytes` is `> 8`, then + /// this method panics. + /// + /// # Examples + /// + /// Write and read an n-byte number in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 3]; + /// LittleEndian::write_int(&mut buf, -1_000, 3); + /// assert_eq!(-1_000, LittleEndian::read_int(&buf, 3)); + /// ``` + #[inline] + fn write_int(buf: &mut [u8], n: i64, nbytes: usize) { + Self::write_uint(buf, unextend_sign(n, nbytes), nbytes) + } + + /// Writes a signed integer `n` to `buf` using only `nbytes`. + /// + /// # Panics + /// + /// If `n` is not representable in `nbytes`, or if `nbytes` is `> 16`, then + /// this method panics. + /// + /// # Examples + /// + /// Write and read n-length signed numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut buf = [0; 3]; + /// LittleEndian::write_int128(&mut buf, -1_000, 3); + /// assert_eq!(-1_000, LittleEndian::read_int128(&buf, 3)); + /// ``` + #[cfg(byteorder_i128)] + #[inline] + fn write_int128(buf: &mut [u8], n: i128, nbytes: usize) { + Self::write_uint128(buf, unextend_sign128(n, nbytes), nbytes) + } + + /// Writes a IEEE754 single-precision (4 bytes) floating point number. + /// + /// # Panics + /// + /// Panics when `buf.len() < 4`. + /// + /// # Examples + /// + /// Write and read `f32` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let e = 2.71828; + /// let mut buf = [0; 4]; + /// LittleEndian::write_f32(&mut buf, e); + /// assert_eq!(e, LittleEndian::read_f32(&buf)); + /// ``` + #[inline] + fn write_f32(buf: &mut [u8], n: f32) { + unimplemented!() + } + + /// Writes a IEEE754 double-precision (8 bytes) floating point number. + /// + /// # Panics + /// + /// Panics when `buf.len() < 8`. + /// + /// # Examples + /// + /// Write and read `f64` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let phi = 1.6180339887; + /// let mut buf = [0; 8]; + /// LittleEndian::write_f64(&mut buf, phi); + /// assert_eq!(phi, LittleEndian::read_f64(&buf)); + /// ``` + #[inline] + fn write_f64(buf: &mut [u8], n: f64) { + unimplemented!() + } + + /// Reads unsigned 16 bit integers from `src` into `dst`. + /// + /// # Panics + /// + /// Panics when `src.len() != 2*dst.len()`. + /// + /// # Examples + /// + /// Write and read `u16` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 8]; + /// let numbers_given = [1, 2, 0xf00f, 0xffee]; + /// LittleEndian::write_u16_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0; 4]; + /// LittleEndian::read_u16_into(&bytes, &mut numbers_got); + /// assert_eq!(numbers_given, numbers_got); + /// ``` + fn read_u16_into(src: &[u8], dst: &mut [u16]); + + /// Reads unsigned 32 bit integers from `src` into `dst`. + /// + /// # Panics + /// + /// Panics when `src.len() != 4*dst.len()`. + /// + /// # Examples + /// + /// Write and read `u32` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 16]; + /// let numbers_given = [1, 2, 0xf00f, 0xffee]; + /// LittleEndian::write_u32_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0; 4]; + /// LittleEndian::read_u32_into(&bytes, &mut numbers_got); + /// assert_eq!(numbers_given, numbers_got); + /// ``` + fn read_u32_into(src: &[u8], dst: &mut [u32]); + + /// Reads unsigned 64 bit integers from `src` into `dst`. + /// + /// # Panics + /// + /// Panics when `src.len() != 8*dst.len()`. + /// + /// # Examples + /// + /// Write and read `u64` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 32]; + /// let numbers_given = [1, 2, 0xf00f, 0xffee]; + /// LittleEndian::write_u64_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0; 4]; + /// LittleEndian::read_u64_into(&bytes, &mut numbers_got); + /// assert_eq!(numbers_given, numbers_got); + /// ``` + fn read_u64_into(src: &[u8], dst: &mut [u64]); + + /// Reads unsigned 128 bit integers from `src` into `dst`. + /// + /// # Panics + /// + /// Panics when `src.len() != 16*dst.len()`. + /// + /// # Examples + /// + /// Write and read `u128` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 64]; + /// let numbers_given = [1, 2, 0xf00f, 0xffee]; + /// LittleEndian::write_u128_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0; 4]; + /// LittleEndian::read_u128_into(&bytes, &mut numbers_got); + /// assert_eq!(numbers_given, numbers_got); + /// ``` + #[cfg(byteorder_i128)] + fn read_u128_into(src: &[u8], dst: &mut [u128]); + + /// Reads signed 16 bit integers from `src` to `dst`. + /// + /// # Panics + /// + /// Panics when `buf.len() != 2*dst.len()`. + /// + /// # Examples + /// + /// Write and read `i16` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 8]; + /// let numbers_given = [1, 2, 0x0f, 0xee]; + /// LittleEndian::write_i16_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0; 4]; + /// LittleEndian::read_i16_into(&bytes, &mut numbers_got); + /// assert_eq!(numbers_given, numbers_got); + /// ``` + #[inline] + fn read_i16_into(src: &[u8], dst: &mut [i16]) { + let dst = unsafe { + identity_transmute::<&mut [i16], &mut [u16]>(dst) + }; + Self::read_u16_into(src, dst) + } + + /// Reads signed 32 bit integers from `src` into `dst`. + /// + /// # Panics + /// + /// Panics when `src.len() != 4*dst.len()`. + /// + /// # Examples + /// + /// Write and read `i32` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 16]; + /// let numbers_given = [1, 2, 0xf00f, 0xffee]; + /// LittleEndian::write_i32_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0; 4]; + /// LittleEndian::read_i32_into(&bytes, &mut numbers_got); + /// assert_eq!(numbers_given, numbers_got); + /// ``` + #[inline] + fn read_i32_into(src: &[u8], dst: &mut [i32]) { + let dst = unsafe { + identity_transmute::<&mut [i32], &mut [u32]>(dst) + }; + Self::read_u32_into(src, dst); + } + + /// Reads signed 64 bit integers from `src` into `dst`. + /// + /// # Panics + /// + /// Panics when `src.len() != 8*dst.len()`. + /// + /// # Examples + /// + /// Write and read `i64` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 32]; + /// let numbers_given = [1, 2, 0xf00f, 0xffee]; + /// LittleEndian::write_i64_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0; 4]; + /// LittleEndian::read_i64_into(&bytes, &mut numbers_got); + /// assert_eq!(numbers_given, numbers_got); + /// ``` + #[inline] + fn read_i64_into(src: &[u8], dst: &mut [i64]) { + let dst = unsafe { + identity_transmute::<&mut [i64], &mut [u64]>(dst) + }; + Self::read_u64_into(src, dst); + } + + /// Reads signed 128 bit integers from `src` into `dst`. + /// + /// # Panics + /// + /// Panics when `src.len() != 16*dst.len()`. + /// + /// # Examples + /// + /// Write and read `i128` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 64]; + /// let numbers_given = [1, 2, 0xf00f, 0xffee]; + /// LittleEndian::write_i128_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0; 4]; + /// LittleEndian::read_i128_into(&bytes, &mut numbers_got); + /// assert_eq!(numbers_given, numbers_got); + /// ``` + #[cfg(byteorder_i128)] + #[inline] + fn read_i128_into(src: &[u8], dst: &mut [i128]) { + let dst = unsafe { + identity_transmute::<&mut [i128], &mut [u128]>(dst) + }; + Self::read_u128_into(src, dst); + } + + /// Reads IEEE754 single-precision (4 bytes) floating point numbers from + /// `src` into `dst`. + /// + /// # Panics + /// + /// Panics when `src.len() != 4*dst.len()`. + /// + /// # Examples + /// + /// Write and read `f32` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 16]; + /// let numbers_given = [1.0, 2.0, 31.312e31, -11.32e19]; + /// LittleEndian::write_f32_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0.0; 4]; + /// LittleEndian::read_f32_into(&bytes, &mut numbers_got); + /// assert_eq!(numbers_given, numbers_got); + /// ``` + #[inline] + fn read_f32_into(src: &[u8], dst: &mut [f32]) { + unimplemented!() + } + + /// **DEPRECATED**. + /// + /// This method is deprecated. Use `read_f32_into` instead. + /// Reads IEEE754 single-precision (4 bytes) floating point numbers from + /// `src` into `dst`. + /// + /// # Panics + /// + /// Panics when `src.len() != 4*dst.len()`. + /// + /// # Examples + /// + /// Write and read `f32` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 16]; + /// let numbers_given = [1.0, 2.0, 31.312e31, -11.32e19]; + /// LittleEndian::write_f32_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0.0; 4]; + /// LittleEndian::read_f32_into_unchecked(&bytes, &mut numbers_got); + /// assert_eq!(numbers_given, numbers_got); + /// ``` + #[inline] + #[deprecated(since="1.3.0", note="please use `read_f32_into` instead")] + fn read_f32_into_unchecked(src: &[u8], dst: &mut [f32]) { + Self::read_f32_into(src, dst); + } + + /// Reads IEEE754 single-precision (4 bytes) floating point numbers from + /// `src` into `dst`. + /// + /// # Panics + /// + /// Panics when `src.len() != 8*dst.len()`. + /// + /// # Examples + /// + /// Write and read `f64` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 32]; + /// let numbers_given = [1.0, 2.0, 31.312e211, -11.32e91]; + /// LittleEndian::write_f64_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0.0; 4]; + /// LittleEndian::read_f64_into(&bytes, &mut numbers_got); + /// assert_eq!(numbers_given, numbers_got); + /// ``` + #[inline] + fn read_f64_into(src: &[u8], dst: &mut [f64]) { + unimplemented!() + } + + /// **DEPRECATED**. + /// + /// This method is deprecated. Use `read_f64_into` instead. + /// + /// Reads IEEE754 single-precision (4 bytes) floating point numbers from + /// `src` into `dst`. + /// + /// # Panics + /// + /// Panics when `src.len() != 8*dst.len()`. + /// + /// # Examples + /// + /// Write and read `f64` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 32]; + /// let numbers_given = [1.0, 2.0, 31.312e211, -11.32e91]; + /// LittleEndian::write_f64_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0.0; 4]; + /// LittleEndian::read_f64_into_unchecked(&bytes, &mut numbers_got); + /// assert_eq!(numbers_given, numbers_got); + /// ``` + #[inline] + #[deprecated(since="1.3.0", note="please use `read_f64_into` instead")] + fn read_f64_into_unchecked(src: &[u8], dst: &mut [f64]) { + Self::read_f64_into(src, dst); + } + + /// Writes unsigned 16 bit integers from `src` into `dst`. + /// + /// # Panics + /// + /// Panics when `dst.len() != 2*src.len()`. + /// + /// # Examples + /// + /// Write and read `u16` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 8]; + /// let numbers_given = [1, 2, 0xf00f, 0xffee]; + /// LittleEndian::write_u16_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0; 4]; + /// LittleEndian::read_u16_into(&bytes, &mut numbers_got); + /// assert_eq!(numbers_given, numbers_got); + /// ``` + fn write_u16_into(src: &[u16], dst: &mut [u8]); + + /// Writes unsigned 32 bit integers from `src` into `dst`. + /// + /// # Panics + /// + /// Panics when `dst.len() != 4*src.len()`. + /// + /// # Examples + /// + /// Write and read `u32` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 16]; + /// let numbers_given = [1, 2, 0xf00f, 0xffee]; + /// LittleEndian::write_u32_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0; 4]; + /// LittleEndian::read_u32_into(&bytes, &mut numbers_got); + /// assert_eq!(numbers_given, numbers_got); + /// ``` + fn write_u32_into(src: &[u32], dst: &mut [u8]); + + /// Writes unsigned 64 bit integers from `src` into `dst`. + /// + /// # Panics + /// + /// Panics when `dst.len() != 8*src.len()`. + /// + /// # Examples + /// + /// Write and read `u64` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 32]; + /// let numbers_given = [1, 2, 0xf00f, 0xffee]; + /// LittleEndian::write_u64_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0; 4]; + /// LittleEndian::read_u64_into(&bytes, &mut numbers_got); + /// assert_eq!(numbers_given, numbers_got); + /// ``` + fn write_u64_into(src: &[u64], dst: &mut [u8]); + + /// Writes unsigned 128 bit integers from `src` into `dst`. + /// + /// # Panics + /// + /// Panics when `dst.len() != 16*src.len()`. + /// + /// # Examples + /// + /// Write and read `u128` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 64]; + /// let numbers_given = [1, 2, 0xf00f, 0xffee]; + /// LittleEndian::write_u128_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0; 4]; + /// LittleEndian::read_u128_into(&bytes, &mut numbers_got); + /// assert_eq!(numbers_given, numbers_got); + /// ``` + #[cfg(byteorder_i128)] + fn write_u128_into(src: &[u128], dst: &mut [u8]); + + /// Writes signed 16 bit integers from `src` into `dst`. + /// + /// # Panics + /// + /// Panics when `buf.len() != 2*src.len()`. + /// + /// # Examples + /// + /// Write and read `i16` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 8]; + /// let numbers_given = [1, 2, 0x0f, 0xee]; + /// LittleEndian::write_i16_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0; 4]; + /// LittleEndian::read_i16_into(&bytes, &mut numbers_got); + /// assert_eq!(numbers_given, numbers_got); + /// ``` + fn write_i16_into(src: &[i16], dst: &mut [u8]) { + let src = unsafe { + identity_transmute::<&[i16], &[u16]>(src) + }; + Self::write_u16_into(src, dst); + } + + /// Writes signed 32 bit integers from `src` into `dst`. + /// + /// # Panics + /// + /// Panics when `dst.len() != 4*src.len()`. + /// + /// # Examples + /// + /// Write and read `i32` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 16]; + /// let numbers_given = [1, 2, 0xf00f, 0xffee]; + /// LittleEndian::write_i32_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0; 4]; + /// LittleEndian::read_i32_into(&bytes, &mut numbers_got); + /// assert_eq!(numbers_given, numbers_got); + /// ``` + fn write_i32_into(src: &[i32], dst: &mut [u8]) { + let src = unsafe { + identity_transmute::<&[i32], &[u32]>(src) + }; + Self::write_u32_into(src, dst); + } + + /// Writes signed 64 bit integers from `src` into `dst`. + /// + /// # Panics + /// + /// Panics when `dst.len() != 8*src.len()`. + /// + /// # Examples + /// + /// Write and read `i64` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 32]; + /// let numbers_given = [1, 2, 0xf00f, 0xffee]; + /// LittleEndian::write_i64_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0; 4]; + /// LittleEndian::read_i64_into(&bytes, &mut numbers_got); + /// assert_eq!(numbers_given, numbers_got); + /// ``` + fn write_i64_into(src: &[i64], dst: &mut [u8]) { + let src = unsafe { + identity_transmute::<&[i64], &[u64]>(src) + }; + Self::write_u64_into(src, dst); + } + + /// Writes signed 128 bit integers from `src` into `dst`. + /// + /// # Panics + /// + /// Panics when `dst.len() != 16*src.len()`. + /// + /// # Examples + /// + /// Write and read `i128` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 64]; + /// let numbers_given = [1, 2, 0xf00f, 0xffee]; + /// LittleEndian::write_i128_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0; 4]; + /// LittleEndian::read_i128_into(&bytes, &mut numbers_got); + /// assert_eq!(numbers_given, numbers_got); + /// ``` + #[cfg(byteorder_i128)] + fn write_i128_into(src: &[i128], dst: &mut [u8]) { + let src = unsafe { + identity_transmute::<&[i128], &[u128]>(src) + }; + Self::write_u128_into(src, dst); + } + + /// Writes IEEE754 single-precision (4 bytes) floating point numbers from + /// `src` into `dst`. + /// + /// # Panics + /// + /// Panics when `src.len() != 4*dst.len()`. + /// + /// # Examples + /// + /// Write and read `f32` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 16]; + /// let numbers_given = [1.0, 2.0, 31.312e31, -11.32e19]; + /// LittleEndian::write_f32_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0.0; 4]; + /// unsafe { + /// LittleEndian::read_f32_into(&bytes, &mut numbers_got); + /// } + /// assert_eq!(numbers_given, numbers_got); + /// ``` + fn write_f32_into(src: &[f32], dst: &mut [u8]) { + unimplemented!() + } + + /// Writes IEEE754 double-precision (8 bytes) floating point numbers from + /// `src` into `dst`. + /// + /// # Panics + /// + /// Panics when `src.len() != 8*dst.len()`. + /// + /// # Examples + /// + /// Write and read `f64` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian}; + /// + /// let mut bytes = [0; 32]; + /// let numbers_given = [1.0, 2.0, 31.312e211, -11.32e91]; + /// LittleEndian::write_f64_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0.0; 4]; + /// unsafe { + /// LittleEndian::read_f64_into(&bytes, &mut numbers_got); + /// } + /// assert_eq!(numbers_given, numbers_got); + /// ``` + fn write_f64_into(src: &[f64], dst: &mut [u8]) { + unimplemented!() + } + + /// Converts the given slice of unsigned 16 bit integers to a particular + /// endianness. + /// + /// If the endianness matches the endianness of the host platform, then + /// this is a no-op. + /// + /// # Examples + /// + /// Convert the host platform's endianness to big-endian: + /// + /// ```rust + /// use byteorder::{ByteOrder, BigEndian}; + /// + /// let mut numbers = [5, 65000]; + /// BigEndian::from_slice_u16(&mut numbers); + /// assert_eq!(numbers, [5u16.to_be(), 65000u16.to_be()]); + /// ``` + fn from_slice_u16(numbers: &mut [u16]); + + /// Converts the given slice of unsigned 32 bit integers to a particular + /// endianness. + /// + /// If the endianness matches the endianness of the host platform, then + /// this is a no-op. + /// + /// # Examples + /// + /// Convert the host platform's endianness to big-endian: + /// + /// ```rust + /// use byteorder::{ByteOrder, BigEndian}; + /// + /// let mut numbers = [5, 65000]; + /// BigEndian::from_slice_u32(&mut numbers); + /// assert_eq!(numbers, [5u32.to_be(), 65000u32.to_be()]); + /// ``` + fn from_slice_u32(numbers: &mut [u32]); + + /// Converts the given slice of unsigned 64 bit integers to a particular + /// endianness. + /// + /// If the endianness matches the endianness of the host platform, then + /// this is a no-op. + /// + /// # Examples + /// + /// Convert the host platform's endianness to big-endian: + /// + /// ```rust + /// use byteorder::{ByteOrder, BigEndian}; + /// + /// let mut numbers = [5, 65000]; + /// BigEndian::from_slice_u64(&mut numbers); + /// assert_eq!(numbers, [5u64.to_be(), 65000u64.to_be()]); + /// ``` + fn from_slice_u64(numbers: &mut [u64]); + + /// Converts the given slice of unsigned 128 bit integers to a particular + /// endianness. + /// + /// If the endianness matches the endianness of the host platform, then + /// this is a no-op. + /// + /// # Examples + /// + /// Convert the host platform's endianness to big-endian: + /// + /// ```rust + /// use byteorder::{ByteOrder, BigEndian}; + /// + /// let mut numbers = [5, 65000]; + /// BigEndian::from_slice_u128(&mut numbers); + /// assert_eq!(numbers, [5u128.to_be(), 65000u128.to_be()]); + /// ``` + #[cfg(byteorder_i128)] + fn from_slice_u128(numbers: &mut [u128]); + + /// Converts the given slice of signed 16 bit integers to a particular + /// endianness. + /// + /// If the endianness matches the endianness of the host platform, then + /// this is a no-op. + /// + /// # Examples + /// + /// Convert the host platform's endianness to big-endian: + /// + /// ```rust + /// use byteorder::{ByteOrder, BigEndian}; + /// + /// let mut numbers = [5, 6500]; + /// BigEndian::from_slice_i16(&mut numbers); + /// assert_eq!(numbers, [5i16.to_be(), 6500i16.to_be()]); + /// ``` + #[inline] + fn from_slice_i16(src: &mut [i16]) { + let src = unsafe { + identity_transmute::<&mut [i16], &mut [u16]>(src) + }; + Self::from_slice_u16(src); + } + + /// Converts the given slice of signed 32 bit integers to a particular + /// endianness. + /// + /// If the endianness matches the endianness of the host platform, then + /// this is a no-op. + /// + /// # Examples + /// + /// Convert the host platform's endianness to big-endian: + /// + /// ```rust + /// use byteorder::{ByteOrder, BigEndian}; + /// + /// let mut numbers = [5, 65000]; + /// BigEndian::from_slice_i32(&mut numbers); + /// assert_eq!(numbers, [5i32.to_be(), 65000i32.to_be()]); + /// ``` + #[inline] + fn from_slice_i32(src: &mut [i32]) { + let src = unsafe { + identity_transmute::<&mut [i32], &mut [u32]>(src) + }; + Self::from_slice_u32(src); + } + + /// Converts the given slice of signed 64 bit integers to a particular + /// endianness. + /// + /// If the endianness matches the endianness of the host platform, then + /// this is a no-op. + /// + /// # Examples + /// + /// Convert the host platform's endianness to big-endian: + /// + /// ```rust + /// use byteorder::{ByteOrder, BigEndian}; + /// + /// let mut numbers = [5, 65000]; + /// BigEndian::from_slice_i64(&mut numbers); + /// assert_eq!(numbers, [5i64.to_be(), 65000i64.to_be()]); + /// ``` + #[inline] + fn from_slice_i64(src: &mut [i64]) { + let src = unsafe { + identity_transmute::<&mut [i64], &mut [u64]>(src) + }; + Self::from_slice_u64(src); + } + + /// Converts the given slice of signed 128 bit integers to a particular + /// endianness. + /// + /// If the endianness matches the endianness of the host platform, then + /// this is a no-op. + /// + /// # Examples + /// + /// Convert the host platform's endianness to big-endian: + /// + /// ```rust + /// use byteorder::{ByteOrder, BigEndian}; + /// + /// let mut numbers = [5, 65000]; + /// BigEndian::from_slice_i128(&mut numbers); + /// assert_eq!(numbers, [5i128.to_be(), 65000i128.to_be()]); + /// ``` + #[cfg(byteorder_i128)] + #[inline] + fn from_slice_i128(src: &mut [i128]) { + let src = unsafe { + identity_transmute::<&mut [i128], &mut [u128]>(src) + }; + Self::from_slice_u128(src); + } + + /// Converts the given slice of IEEE754 single-precision (4 bytes) floating + /// point numbers to a particular endianness. + /// + /// If the endianness matches the endianness of the host platform, then + /// this is a no-op. + fn from_slice_f32(numbers: &mut [f32]); + + /// Converts the given slice of IEEE754 double-precision (8 bytes) floating + /// point numbers to a particular endianness. + /// + /// If the endianness matches the endianness of the host platform, then + /// this is a no-op. + fn from_slice_f64(numbers: &mut [f64]); +} + +/// Defines big-endian serialization. +/// +/// Note that this type has no value constructor. It is used purely at the +/// type level. +/// +/// # Examples +/// +/// Write and read `u32` numbers in big endian order: +/// +/// ```rust +/// use byteorder::{ByteOrder, BigEndian}; +/// +/// let mut buf = [0; 4]; +/// BigEndian::write_u32(&mut buf, 1_000_000); +/// assert_eq!(1_000_000, BigEndian::read_u32(&buf)); +/// ``` +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub enum BigEndian {} + +impl Default for BigEndian { + fn default() -> BigEndian { + panic!("BigEndian default") + } +} + +/// A type alias for [`BigEndian`]. +/// +/// [`BigEndian`]: enum.BigEndian.html +pub type BE = BigEndian; + +/// Defines little-endian serialization. +/// +/// Note that this type has no value constructor. It is used purely at the +/// type level. +/// +/// # Examples +/// +/// Write and read `u32` numbers in little endian order: +/// +/// ```rust +/// use byteorder::{ByteOrder, LittleEndian}; +/// +/// let mut buf = [0; 4]; +/// LittleEndian::write_u32(&mut buf, 1_000_000); +/// assert_eq!(1_000_000, LittleEndian::read_u32(&buf)); +/// ``` +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub enum LittleEndian {} + +impl Default for LittleEndian { + fn default() -> LittleEndian { + panic!("LittleEndian default") + } +} + +/// A type alias for [`LittleEndian`]. +/// +/// [`LittleEndian`]: enum.LittleEndian.html +pub type LE = LittleEndian; + +/// Defines network byte order serialization. +/// +/// Network byte order is defined by [RFC 1700][1] to be big-endian, and is +/// referred to in several protocol specifications. This type is an alias of +/// [`BigEndian`]. +/// +/// [1]: https://tools.ietf.org/html/rfc1700 +/// +/// Note that this type has no value constructor. It is used purely at the +/// type level. +/// +/// # Examples +/// +/// Write and read `i16` numbers in big endian order: +/// +/// ```rust +/// use byteorder::{ByteOrder, NetworkEndian, BigEndian}; +/// +/// let mut buf = [0; 2]; +/// BigEndian::write_i16(&mut buf, -5_000); +/// assert_eq!(-5_000, NetworkEndian::read_i16(&buf)); +/// ``` +/// +/// [`BigEndian`]: enum.BigEndian.html +pub type NetworkEndian = BigEndian; + +/// Defines system native-endian serialization. +/// +/// Note that this type has no value constructor. It is used purely at the +/// type level. +/// +/// On this platform, this is an alias for [`LittleEndian`]. +/// +/// [`LittleEndian`]: enum.LittleEndian.html +#[cfg(target_endian = "little")] +pub type NativeEndian = LittleEndian; + +/// Defines system native-endian serialization. +/// +/// Note that this type has no value constructor. It is used purely at the +/// type level. +/// +/// On this platform, this is an alias for [`BigEndian`]. +/// +/// [`BigEndian`]: enum.BigEndian.html +#[cfg(target_endian = "big")] +pub type NativeEndian = BigEndian; + +macro_rules! safe_read_bytes { + ($src:expr, $ty:ty, $range:expr) => ({ + let mut result: $ty = 0; + for i in $range { + result = (result << 8) | ($src[i] as $ty); + } + result + }); +} + +macro_rules! safe_write_bytes { + ($val:expr, $dst:expr, $range:expr) => ({ + let mut j = 0; + for i in ($range).rev() { + $dst[i] = ($val >> (j * 8)) as u8; + j += 1; + } + }); +} + +macro_rules! safe_read_bytes_rev { + ($src:expr, $ty:ty, $range:expr) => (safe_read_bytes!($src, $ty, $range.rev())); +} + +macro_rules! safe_write_bytes_rev { + ($val:expr, $dst:expr, $range:expr) => (safe_write_bytes!($val, $dst, $range.rev())); +} + +macro_rules! safe_read_slice { + ($src:expr, $dst:expr, $read_one:ident, $step:expr) => ({ + assert!($src.len() == $dst.len() * $step); + let mut base = 0; + for one_dst in $dst { + let one_src = &$src[base .. base + $step]; + *one_dst = Self::$read_one(one_src); + base += $step; + } + }); +} + +macro_rules! safe_write_slice { + ($src:expr, $dst:expr, $write_one:ident, $step:expr) => ({ + assert!($dst.len() == $src.len() * $step); + let mut base = 0; + for one_src in $src { + let one_dst = &mut $dst[base .. base + $step]; + Self::$write_one(one_dst, *one_src); + base += $step; + } + }); +} + +impl ByteOrder for BigEndian { + #[inline] + fn read_u16(buf: &[u8]) -> u16 { + safe_read_bytes!(buf, u16, 0..2) + } + + #[inline] + fn read_u32(buf: &[u8]) -> u32 { + safe_read_bytes!(buf, u32, 0..4) + } + + #[inline] + fn read_u64(buf: &[u8]) -> u64 { + safe_read_bytes!(buf, u64, 0..8) + } + + #[cfg(byteorder_i128)] + #[inline] + fn read_u128(buf: &[u8]) -> u128 { + safe_read_bytes!(buf, u128, 0..16) + } + + #[inline] + fn read_uint(buf: &[u8], nbytes: usize) -> u64 { + assert!(1 <= nbytes && nbytes <= 8 && nbytes <= buf.len()); + safe_read_bytes!(buf, u64, 0..nbytes) + } + + #[cfg(byteorder_i128)] + #[inline] + fn read_uint128(buf: &[u8], nbytes: usize) -> u128 { + assert!(1 <= nbytes && nbytes <= 16 && nbytes <= buf.len()); + safe_read_bytes!(buf, u128, 0..nbytes) + } + + #[inline] + fn write_u16(buf: &mut [u8], n: u16) { + safe_write_bytes!(n, buf, 0..2) + } + + #[inline] + fn write_u32(buf: &mut [u8], n: u32) { + safe_write_bytes!(n, buf, 0..4) + } + + #[inline] + fn write_u64(buf: &mut [u8], n: u64) { + safe_write_bytes!(n, buf, 0..8) + } + + #[cfg(byteorder_i128)] + #[inline] + fn write_u128(buf: &mut [u8], n: u128) { + safe_write_bytes!(n, buf, 0..16) + } + + #[inline] + fn write_uint(buf: &mut [u8], n: u64, nbytes: usize) { + assert!(pack_size(n) <= nbytes && nbytes <= 8); + assert!(nbytes <= buf.len()); + safe_write_bytes!(n, buf, 0..nbytes) + } + + #[cfg(byteorder_i128)] + #[inline] + fn write_uint128(buf: &mut [u8], n: u128, nbytes: usize) { + assert!(pack_size128(n) <= nbytes && nbytes <= 16); + assert!(nbytes <= buf.len()); + safe_write_bytes!(n, buf, 0..nbytes) + } + + #[inline] + fn read_u16_into(src: &[u8], dst: &mut [u16]) { + safe_read_slice!(src, dst, read_u16, 2); + } + + #[inline] + fn read_u32_into(src: &[u8], dst: &mut [u32]) { + safe_read_slice!(src, dst, read_u32, 4); + } + + #[inline] + fn read_u64_into(src: &[u8], dst: &mut [u64]) { + safe_read_slice!(src, dst, read_u64, 8); + } + + #[cfg(byteorder_i128)] + #[inline] + fn read_u128_into(src: &[u8], dst: &mut [u128]) { + safe_read_slice!(src, dst, read_u128, 16); + } + + #[inline] + fn write_u16_into(src: &[u16], dst: &mut [u8]) { + safe_write_slice!(src, dst, write_u16, 2); + } + + #[inline] + fn write_u32_into(src: &[u32], dst: &mut [u8]) { + safe_write_slice!(src, dst, write_u32, 4); + } + + #[inline] + fn write_u64_into(src: &[u64], dst: &mut [u8]) { + safe_write_slice!(src, dst, write_u64, 8); + } + + #[cfg(byteorder_i128)] + #[inline] + fn write_u128_into(src: &[u128], dst: &mut [u8]) { + safe_write_slice!(src, dst, write_u128, 16); + } + + #[inline] + fn from_slice_u16(numbers: &mut [u16]) { + if cfg!(target_endian = "little") { + for n in numbers { + *n = n.to_be(); + } + } + } + + #[inline] + fn from_slice_u32(numbers: &mut [u32]) { + if cfg!(target_endian = "little") { + for n in numbers { + *n = n.to_be(); + } + } + } + + #[inline] + fn from_slice_u64(numbers: &mut [u64]) { + if cfg!(target_endian = "little") { + for n in numbers { + *n = n.to_be(); + } + } + } + + #[cfg(byteorder_i128)] + #[inline] + fn from_slice_u128(numbers: &mut [u128]) { + if cfg!(target_endian = "little") { + for n in numbers { + *n = n.to_be(); + } + } + } + + #[inline] + fn from_slice_f32(numbers: &mut [f32]) { + unimplemented!() + } + + #[inline] + fn from_slice_f64(numbers: &mut [f64]) { + unimplemented!() + } +} + +impl ByteOrder for LittleEndian { + #[inline] + fn read_u16(buf: &[u8]) -> u16 { + safe_read_bytes_rev!(buf, u16, 0..2) + } + + #[inline] + fn read_u32(buf: &[u8]) -> u32 { + safe_read_bytes_rev!(buf, u32, 0..4) + } + + #[inline] + fn read_u64(buf: &[u8]) -> u64 { + safe_read_bytes_rev!(buf, u64, 0..8) + } + + #[cfg(byteorder_i128)] + #[inline] + fn read_u128(buf: &[u8]) -> u128 { + safe_read_bytes_rev!(buf, u128, 0..16) + } + + #[inline] + fn read_uint(buf: &[u8], nbytes: usize) -> u64 { + assert!(1 <= nbytes && nbytes <= 8 && nbytes <= buf.len()); + safe_read_bytes_rev!(buf, u64, 0..nbytes) + } + + #[cfg(byteorder_i128)] + #[inline] + fn read_uint128(buf: &[u8], nbytes: usize) -> u128 { + assert!(1 <= nbytes && nbytes <= 16 && nbytes <= buf.len()); + safe_read_bytes_rev!(buf, u128, 0..nbytes) + } + + #[inline] + fn write_u16(buf: &mut [u8], n: u16) { + safe_write_bytes_rev!(n, buf, 0..2) + } + + #[inline] + fn write_u32(buf: &mut [u8], n: u32) { + safe_write_bytes_rev!(n, buf, 0..4) + } + + #[inline] + fn write_u64(buf: &mut [u8], n: u64) { + safe_write_bytes_rev!(n, buf, 0..8) + } + + #[cfg(byteorder_i128)] + #[inline] + fn write_u128(buf: &mut [u8], n: u128) { + safe_write_bytes_rev!(n, buf, 0..16) + } + + #[inline] + fn write_uint(buf: &mut [u8], n: u64, nbytes: usize) { + assert!(pack_size(n) <= nbytes && nbytes <= 8); + assert!(nbytes <= buf.len()); + safe_write_bytes_rev!(n, buf, 0..nbytes) + } + + #[cfg(byteorder_i128)] + #[inline] + fn write_uint128(buf: &mut [u8], n: u128, nbytes: usize) { + assert!(pack_size128(n) <= nbytes && nbytes <= 16); + assert!(nbytes <= buf.len()); + safe_write_bytes_rev!(n, buf, 0..nbytes) + } + + #[inline] + fn read_u16_into(src: &[u8], dst: &mut [u16]) { + safe_read_slice!(src, dst, read_u16, 2); + } + + #[inline] + fn read_u32_into(src: &[u8], dst: &mut [u32]) { + safe_read_slice!(src, dst, read_u32, 4); + } + + #[inline] + fn read_u64_into(src: &[u8], dst: &mut [u64]) { + safe_read_slice!(src, dst, read_u64, 8); + } + + #[cfg(byteorder_i128)] + #[inline] + fn read_u128_into(src: &[u8], dst: &mut [u128]) { + safe_read_slice!(src, dst, read_u128, 16); + } + + #[inline] + fn write_u16_into(src: &[u16], dst: &mut [u8]) { + safe_write_slice!(src, dst, write_u16, 2); + } + + #[inline] + fn write_u32_into(src: &[u32], dst: &mut [u8]) { + safe_write_slice!(src, dst, write_u32, 4); + } + + #[inline] + fn write_u64_into(src: &[u64], dst: &mut [u8]) { + safe_write_slice!(src, dst, write_u64, 8); + } + + #[cfg(byteorder_i128)] + #[inline] + fn write_u128_into(src: &[u128], dst: &mut [u8]) { + safe_write_slice!(src, dst, write_u128, 16); + } + + #[inline] + fn from_slice_u16(numbers: &mut [u16]) { + if cfg!(target_endian = "big") { + for n in numbers { + *n = n.to_le(); + } + } + } + + #[inline] + fn from_slice_u32(numbers: &mut [u32]) { + if cfg!(target_endian = "big") { + for n in numbers { + *n = n.to_le(); + } + } + } + + #[inline] + fn from_slice_u64(numbers: &mut [u64]) { + if cfg!(target_endian = "big") { + for n in numbers { + *n = n.to_le(); + } + } + } + + #[cfg(byteorder_i128)] + #[inline] + fn from_slice_u128(numbers: &mut [u128]) { + if cfg!(target_endian = "big") { + for n in numbers { + *n = n.to_le(); + } + } + } + + #[inline] + fn from_slice_f32(numbers: &mut [f32]) { + unimplemented!() + } + + #[inline] + fn from_slice_f64(numbers: &mut [f64]) { + unimplemented!() + } +} + +#[cfg(test)] +mod test { + extern crate quickcheck; + extern crate rand; + + use self::quickcheck::{QuickCheck, StdGen, Testable}; + use self::rand::thread_rng; + #[cfg(byteorder_i128)] + use self::rand::Rng; + #[cfg(byteorder_i128)] + use self::quickcheck::{Arbitrary, Gen}; + + pub const U24_MAX: u32 = 16_777_215; + pub const I24_MAX: i32 = 8_388_607; + pub const U48_MAX: u64 = 281_474_976_710_655; + pub const I48_MAX: i64 = 140_737_488_355_327; + + pub const U64_MAX: u64 = ::core::u64::MAX; + pub const I64_MAX: u64 = ::core::i64::MAX as u64; + + macro_rules! calc_max { + ($max:expr, $bytes:expr) => { calc_max!($max, $bytes, 8) }; + ($max:expr, $bytes:expr, $maxbytes:expr) => { + ($max - 1) >> (8 * ($maxbytes - $bytes)) + }; + } + + #[derive(Clone, Debug)] + pub struct Wi128(pub T); + + #[cfg(byteorder_i128)] + impl Wi128 { + pub fn clone(&self) -> T { + self.0.clone() + } + } + + impl PartialEq for Wi128 { + fn eq(&self, other: &T) -> bool { + self.0.eq(other) + } + } + + #[cfg(byteorder_i128)] + impl Arbitrary for Wi128 { + fn arbitrary(gen: &mut G) -> Wi128 { + let max = calc_max!(::core::u128::MAX, gen.size(), 16); + let output = + (gen.gen::() as u128) | + ((gen.gen::() as u128) << 64); + Wi128(output & (max - 1)) + } + } + + #[cfg(byteorder_i128)] + impl Arbitrary for Wi128 { + fn arbitrary(gen: &mut G) -> Wi128 { + let max = calc_max!(::core::i128::MAX, gen.size(), 16); + let output = + (gen.gen::() as i128) | + ((gen.gen::() as i128) << 64); + Wi128(output & (max - 1)) + } + } + + pub fn qc_sized(f: A, size: u64) { + QuickCheck::new() + .gen(StdGen::new(thread_rng(), size as usize)) + .tests(1_00) + .max_tests(10_000) + .quickcheck(f); + } + + macro_rules! qc_byte_order { + ($name:ident, $ty_int:ty, $max:expr, + $bytes:expr, $read:ident, $write:ident) => ( + mod $name { + use {BigEndian, ByteOrder, NativeEndian, LittleEndian}; + #[allow(unused_imports)] use super::{ qc_sized, Wi128 }; + + #[test] + fn big_endian() { + fn prop(n: $ty_int) -> bool { + let mut buf = [0; 16]; + BigEndian::$write(&mut buf, n.clone(), $bytes); + n == BigEndian::$read(&mut buf[..$bytes], $bytes) + } + qc_sized(prop as fn($ty_int) -> bool, $max); + } + + #[test] + fn little_endian() { + fn prop(n: $ty_int) -> bool { + let mut buf = [0; 16]; + LittleEndian::$write(&mut buf, n.clone(), $bytes); + n == LittleEndian::$read(&mut buf[..$bytes], $bytes) + } + qc_sized(prop as fn($ty_int) -> bool, $max); + } + + #[test] + fn native_endian() { + fn prop(n: $ty_int) -> bool { + let mut buf = [0; 16]; + NativeEndian::$write(&mut buf, n.clone(), $bytes); + n == NativeEndian::$read(&mut buf[..$bytes], $bytes) + } + qc_sized(prop as fn($ty_int) -> bool, $max); + } + } + ); + ($name:ident, $ty_int:ty, $max:expr, + $read:ident, $write:ident) => ( + mod $name { + use core::mem::size_of; + use {BigEndian, ByteOrder, NativeEndian, LittleEndian}; + #[allow(unused_imports)] use super::{ qc_sized, Wi128 }; + + #[test] + fn big_endian() { + fn prop(n: $ty_int) -> bool { + let bytes = size_of::<$ty_int>(); + let mut buf = [0; 16]; + BigEndian::$write(&mut buf[16 - bytes..], n.clone()); + n == BigEndian::$read(&mut buf[16 - bytes..]) + } + qc_sized(prop as fn($ty_int) -> bool, $max - 1); + } + + #[test] + fn little_endian() { + fn prop(n: $ty_int) -> bool { + let bytes = size_of::<$ty_int>(); + let mut buf = [0; 16]; + LittleEndian::$write(&mut buf[..bytes], n.clone()); + n == LittleEndian::$read(&mut buf[..bytes]) + } + qc_sized(prop as fn($ty_int) -> bool, $max - 1); + } + + #[test] + fn native_endian() { + fn prop(n: $ty_int) -> bool { + let bytes = size_of::<$ty_int>(); + let mut buf = [0; 16]; + NativeEndian::$write(&mut buf[..bytes], n.clone()); + n == NativeEndian::$read(&mut buf[..bytes]) + } + qc_sized(prop as fn($ty_int) -> bool, $max - 1); + } + } + ); + } + + qc_byte_order!(prop_u16, u16, ::core::u16::MAX as u64, read_u16, write_u16); + qc_byte_order!(prop_i16, i16, ::core::i16::MAX as u64, read_i16, write_i16); + qc_byte_order!(prop_u24, u32, ::test::U24_MAX as u64, read_u24, write_u24); + qc_byte_order!(prop_i24, i32, ::test::I24_MAX as u64, read_i24, write_i24); + qc_byte_order!(prop_u32, u32, ::core::u32::MAX as u64, read_u32, write_u32); + qc_byte_order!(prop_i32, i32, ::core::i32::MAX as u64, read_i32, write_i32); + qc_byte_order!(prop_u48, u64, ::test::U48_MAX as u64, read_u48, write_u48); + qc_byte_order!(prop_i48, i64, ::test::I48_MAX as u64, read_i48, write_i48); + qc_byte_order!(prop_u64, u64, ::core::u64::MAX as u64, read_u64, write_u64); + qc_byte_order!(prop_i64, i64, ::core::i64::MAX as u64, read_i64, write_i64); + //qc_byte_order!(prop_f32, f32, ::core::u64::MAX as u64, read_f32, write_f32); + //qc_byte_order!(prop_f64, f64, ::core::i64::MAX as u64, read_f64, write_f64); + + #[cfg(byteorder_i128)] + qc_byte_order!(prop_u128, Wi128, 16 + 1, read_u128, write_u128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_i128, Wi128, 16 + 1, read_i128, write_i128); + + qc_byte_order!(prop_uint_1, + u64, calc_max!(super::U64_MAX, 1), 1, read_uint, write_uint); + qc_byte_order!(prop_uint_2, + u64, calc_max!(super::U64_MAX, 2), 2, read_uint, write_uint); + qc_byte_order!(prop_uint_3, + u64, calc_max!(super::U64_MAX, 3), 3, read_uint, write_uint); + qc_byte_order!(prop_uint_4, + u64, calc_max!(super::U64_MAX, 4), 4, read_uint, write_uint); + qc_byte_order!(prop_uint_5, + u64, calc_max!(super::U64_MAX, 5), 5, read_uint, write_uint); + qc_byte_order!(prop_uint_6, + u64, calc_max!(super::U64_MAX, 6), 6, read_uint, write_uint); + qc_byte_order!(prop_uint_7, + u64, calc_max!(super::U64_MAX, 7), 7, read_uint, write_uint); + qc_byte_order!(prop_uint_8, + u64, calc_max!(super::U64_MAX, 8), 8, read_uint, write_uint); + + #[cfg(byteorder_i128)] + qc_byte_order!(prop_uint128_1, + Wi128, 1, 1, read_uint128, write_uint128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_uint128_2, + Wi128, 2, 2, read_uint128, write_uint128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_uint128_3, + Wi128, 3, 3, read_uint128, write_uint128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_uint128_4, + Wi128, 4, 4, read_uint128, write_uint128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_uint128_5, + Wi128, 5, 5, read_uint128, write_uint128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_uint128_6, + Wi128, 6, 6, read_uint128, write_uint128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_uint128_7, + Wi128, 7, 7, read_uint128, write_uint128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_uint128_8, + Wi128, 8, 8, read_uint128, write_uint128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_uint128_9, + Wi128, 9, 9, read_uint128, write_uint128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_uint128_10, + Wi128, 10, 10, read_uint128, write_uint128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_uint128_11, + Wi128, 11, 11, read_uint128, write_uint128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_uint128_12, + Wi128, 12, 12, read_uint128, write_uint128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_uint128_13, + Wi128, 13, 13, read_uint128, write_uint128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_uint128_14, + Wi128, 14, 14, read_uint128, write_uint128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_uint128_15, + Wi128, 15, 15, read_uint128, write_uint128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_uint128_16, + Wi128, 16, 16, read_uint128, write_uint128); + + qc_byte_order!(prop_int_1, + i64, calc_max!(super::I64_MAX, 1), 1, read_int, write_int); + qc_byte_order!(prop_int_2, + i64, calc_max!(super::I64_MAX, 2), 2, read_int, write_int); + qc_byte_order!(prop_int_3, + i64, calc_max!(super::I64_MAX, 3), 3, read_int, write_int); + qc_byte_order!(prop_int_4, + i64, calc_max!(super::I64_MAX, 4), 4, read_int, write_int); + qc_byte_order!(prop_int_5, + i64, calc_max!(super::I64_MAX, 5), 5, read_int, write_int); + qc_byte_order!(prop_int_6, + i64, calc_max!(super::I64_MAX, 6), 6, read_int, write_int); + qc_byte_order!(prop_int_7, + i64, calc_max!(super::I64_MAX, 7), 7, read_int, write_int); + qc_byte_order!(prop_int_8, + i64, calc_max!(super::I64_MAX, 8), 8, read_int, write_int); + + #[cfg(byteorder_i128)] + qc_byte_order!(prop_int128_1, + Wi128, 1, 1, read_int128, write_int128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_int128_2, + Wi128, 2, 2, read_int128, write_int128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_int128_3, + Wi128, 3, 3, read_int128, write_int128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_int128_4, + Wi128, 4, 4, read_int128, write_int128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_int128_5, + Wi128, 5, 5, read_int128, write_int128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_int128_6, + Wi128, 6, 6, read_int128, write_int128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_int128_7, + Wi128, 7, 7, read_int128, write_int128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_int128_8, + Wi128, 8, 8, read_int128, write_int128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_int128_9, + Wi128, 9, 9, read_int128, write_int128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_int128_10, + Wi128, 10, 10, read_int128, write_int128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_int128_11, + Wi128, 11, 11, read_int128, write_int128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_int128_12, + Wi128, 12, 12, read_int128, write_int128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_int128_13, + Wi128, 13, 13, read_int128, write_int128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_int128_14, + Wi128, 14, 14, read_int128, write_int128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_int128_15, + Wi128, 15, 15, read_int128, write_int128); + #[cfg(byteorder_i128)] + qc_byte_order!(prop_int128_16, + Wi128, 16, 16, read_int128, write_int128); + + + // Test that all of the byte conversion functions panic when given a + // buffer that is too small. + // + // These tests are critical to ensure safety, otherwise we might end up + // with a buffer overflow. + macro_rules! too_small { + ($name:ident, $maximally_small:expr, $zero:expr, + $read:ident, $write:ident) => ( + mod $name { + use {BigEndian, ByteOrder, NativeEndian, LittleEndian}; + + #[test] + #[should_panic] + fn read_big_endian() { + let buf = [0; $maximally_small]; + BigEndian::$read(&buf); + } + + #[test] + #[should_panic] + fn read_little_endian() { + let buf = [0; $maximally_small]; + LittleEndian::$read(&buf); + } + + #[test] + #[should_panic] + fn read_native_endian() { + let buf = [0; $maximally_small]; + NativeEndian::$read(&buf); + } + + #[test] + #[should_panic] + fn write_big_endian() { + let mut buf = [0; $maximally_small]; + BigEndian::$write(&mut buf, $zero); + } + + #[test] + #[should_panic] + fn write_little_endian() { + let mut buf = [0; $maximally_small]; + LittleEndian::$write(&mut buf, $zero); + } + + #[test] + #[should_panic] + fn write_native_endian() { + let mut buf = [0; $maximally_small]; + NativeEndian::$write(&mut buf, $zero); + } + } + ); + ($name:ident, $maximally_small:expr, $read:ident) => ( + mod $name { + use {BigEndian, ByteOrder, NativeEndian, LittleEndian}; + + #[test] + #[should_panic] + fn read_big_endian() { + let buf = [0; $maximally_small]; + BigEndian::$read(&buf, $maximally_small + 1); + } + + #[test] + #[should_panic] + fn read_little_endian() { + let buf = [0; $maximally_small]; + LittleEndian::$read(&buf, $maximally_small + 1); + } + + #[test] + #[should_panic] + fn read_native_endian() { + let buf = [0; $maximally_small]; + NativeEndian::$read(&buf, $maximally_small + 1); + } + } + ); + } + + too_small!(small_u16, 1, 0, read_u16, write_u16); + too_small!(small_i16, 1, 0, read_i16, write_i16); + too_small!(small_u32, 3, 0, read_u32, write_u32); + too_small!(small_i32, 3, 0, read_i32, write_i32); + too_small!(small_u64, 7, 0, read_u64, write_u64); + too_small!(small_i64, 7, 0, read_i64, write_i64); + //too_small!(small_f32, 3, 0.0, read_f32, write_f32); + //too_small!(small_f64, 7, 0.0, read_f64, write_f64); + #[cfg(byteorder_i128)] + too_small!(small_u128, 15, 0, read_u128, write_u128); + #[cfg(byteorder_i128)] + too_small!(small_i128, 15, 0, read_i128, write_i128); + + too_small!(small_uint_1, 1, read_uint); + too_small!(small_uint_2, 2, read_uint); + too_small!(small_uint_3, 3, read_uint); + too_small!(small_uint_4, 4, read_uint); + too_small!(small_uint_5, 5, read_uint); + too_small!(small_uint_6, 6, read_uint); + too_small!(small_uint_7, 7, read_uint); + + #[cfg(byteorder_i128)] + too_small!(small_uint128_1, 1, read_uint128); + #[cfg(byteorder_i128)] + too_small!(small_uint128_2, 2, read_uint128); + #[cfg(byteorder_i128)] + too_small!(small_uint128_3, 3, read_uint128); + #[cfg(byteorder_i128)] + too_small!(small_uint128_4, 4, read_uint128); + #[cfg(byteorder_i128)] + too_small!(small_uint128_5, 5, read_uint128); + #[cfg(byteorder_i128)] + too_small!(small_uint128_6, 6, read_uint128); + #[cfg(byteorder_i128)] + too_small!(small_uint128_7, 7, read_uint128); + #[cfg(byteorder_i128)] + too_small!(small_uint128_8, 8, read_uint128); + #[cfg(byteorder_i128)] + too_small!(small_uint128_9, 9, read_uint128); + #[cfg(byteorder_i128)] + too_small!(small_uint128_10, 10, read_uint128); + #[cfg(byteorder_i128)] + too_small!(small_uint128_11, 11, read_uint128); + #[cfg(byteorder_i128)] + too_small!(small_uint128_12, 12, read_uint128); + #[cfg(byteorder_i128)] + too_small!(small_uint128_13, 13, read_uint128); + #[cfg(byteorder_i128)] + too_small!(small_uint128_14, 14, read_uint128); + #[cfg(byteorder_i128)] + too_small!(small_uint128_15, 15, read_uint128); + + too_small!(small_int_1, 1, read_int); + too_small!(small_int_2, 2, read_int); + too_small!(small_int_3, 3, read_int); + too_small!(small_int_4, 4, read_int); + too_small!(small_int_5, 5, read_int); + too_small!(small_int_6, 6, read_int); + too_small!(small_int_7, 7, read_int); + + #[cfg(byteorder_i128)] + too_small!(small_int128_1, 1, read_int128); + #[cfg(byteorder_i128)] + too_small!(small_int128_2, 2, read_int128); + #[cfg(byteorder_i128)] + too_small!(small_int128_3, 3, read_int128); + #[cfg(byteorder_i128)] + too_small!(small_int128_4, 4, read_int128); + #[cfg(byteorder_i128)] + too_small!(small_int128_5, 5, read_int128); + #[cfg(byteorder_i128)] + too_small!(small_int128_6, 6, read_int128); + #[cfg(byteorder_i128)] + too_small!(small_int128_7, 7, read_int128); + #[cfg(byteorder_i128)] + too_small!(small_int128_8, 8, read_int128); + #[cfg(byteorder_i128)] + too_small!(small_int128_9, 9, read_int128); + #[cfg(byteorder_i128)] + too_small!(small_int128_10, 10, read_int128); + #[cfg(byteorder_i128)] + too_small!(small_int128_11, 11, read_int128); + #[cfg(byteorder_i128)] + too_small!(small_int128_12, 12, read_int128); + #[cfg(byteorder_i128)] + too_small!(small_int128_13, 13, read_int128); + #[cfg(byteorder_i128)] + too_small!(small_int128_14, 14, read_int128); + #[cfg(byteorder_i128)] + too_small!(small_int128_15, 15, read_int128); + + // Test that reading/writing slices enforces the correct lengths. + macro_rules! slice_lengths { + ($name:ident, $read:ident, $write:ident, + $num_bytes:expr, $numbers:expr) => { + mod $name { + use {ByteOrder, BigEndian, NativeEndian, LittleEndian}; + + #[test] + #[should_panic] + fn read_big_endian() { + let bytes = [0; $num_bytes]; + let mut numbers = $numbers; + BigEndian::$read(&bytes, &mut numbers); + } + + #[test] + #[should_panic] + fn read_little_endian() { + let bytes = [0; $num_bytes]; + let mut numbers = $numbers; + LittleEndian::$read(&bytes, &mut numbers); + } + + #[test] + #[should_panic] + fn read_native_endian() { + let bytes = [0; $num_bytes]; + let mut numbers = $numbers; + NativeEndian::$read(&bytes, &mut numbers); + } + + #[test] + #[should_panic] + fn write_big_endian() { + let mut bytes = [0; $num_bytes]; + let numbers = $numbers; + BigEndian::$write(&numbers, &mut bytes); + } + + #[test] + #[should_panic] + fn write_little_endian() { + let mut bytes = [0; $num_bytes]; + let numbers = $numbers; + LittleEndian::$write(&numbers, &mut bytes); + } + + #[test] + #[should_panic] + fn write_native_endian() { + let mut bytes = [0; $num_bytes]; + let numbers = $numbers; + NativeEndian::$write(&numbers, &mut bytes); + } + } + } + } + + slice_lengths!( + slice_len_too_small_u16, read_u16_into, write_u16_into, 3, [0, 0]); + slice_lengths!( + slice_len_too_big_u16, read_u16_into, write_u16_into, 5, [0, 0]); + slice_lengths!( + slice_len_too_small_i16, read_i16_into, write_i16_into, 3, [0, 0]); + slice_lengths!( + slice_len_too_big_i16, read_i16_into, write_i16_into, 5, [0, 0]); + + slice_lengths!( + slice_len_too_small_u32, read_u32_into, write_u32_into, 7, [0, 0]); + slice_lengths!( + slice_len_too_big_u32, read_u32_into, write_u32_into, 9, [0, 0]); + slice_lengths!( + slice_len_too_small_i32, read_i32_into, write_i32_into, 7, [0, 0]); + slice_lengths!( + slice_len_too_big_i32, read_i32_into, write_i32_into, 9, [0, 0]); + + slice_lengths!( + slice_len_too_small_u64, read_u64_into, write_u64_into, 15, [0, 0]); + slice_lengths!( + slice_len_too_big_u64, read_u64_into, write_u64_into, 17, [0, 0]); + slice_lengths!( + slice_len_too_small_i64, read_i64_into, write_i64_into, 15, [0, 0]); + slice_lengths!( + slice_len_too_big_i64, read_i64_into, write_i64_into, 17, [0, 0]); + + #[cfg(byteorder_i128)] + slice_lengths!( + slice_len_too_small_u128, read_u128_into, write_u128_into, 31, [0, 0]); + #[cfg(byteorder_i128)] + slice_lengths!( + slice_len_too_big_u128, read_u128_into, write_u128_into, 33, [0, 0]); + #[cfg(byteorder_i128)] + slice_lengths!( + slice_len_too_small_i128, read_i128_into, write_i128_into, 31, [0, 0]); + #[cfg(byteorder_i128)] + slice_lengths!( + slice_len_too_big_i128, read_i128_into, write_i128_into, 33, [0, 0]); + + #[test] + fn uint_bigger_buffer() { + use {ByteOrder, LittleEndian}; + let n = LittleEndian::read_uint(&[1, 2, 3, 4, 5, 6, 7, 8], 5); + assert_eq!(n, 0x0504030201); + } +} + +#[cfg(test)] +#[cfg(feature = "std")] +mod stdtests { + extern crate quickcheck; + extern crate rand; + + use self::quickcheck::{QuickCheck, StdGen, Testable}; + use self::rand::thread_rng; + + fn qc_unsized(f: A) { + + QuickCheck::new() + .gen(StdGen::new(thread_rng(), 16)) + .tests(1_00) + .max_tests(10_000) + .quickcheck(f); + } + + macro_rules! calc_max { + ($max:expr, $bytes:expr) => { ($max - 1) >> (8 * (8 - $bytes)) }; + } + + macro_rules! qc_bytes_ext { + ($name:ident, $ty_int:ty, $max:expr, + $bytes:expr, $read:ident, $write:ident) => ( + mod $name { + use std::io::Cursor; + use { + ReadBytesExt, WriteBytesExt, + BigEndian, NativeEndian, LittleEndian, + }; + #[allow(unused_imports)] use test::{qc_sized, Wi128}; + + #[test] + fn big_endian() { + fn prop(n: $ty_int) -> bool { + let mut wtr = vec![]; + wtr.$write::(n.clone()).unwrap(); + let offset = wtr.len() - $bytes; + let mut rdr = Cursor::new(&mut wtr[offset..]); + n == rdr.$read::($bytes).unwrap() + } + qc_sized(prop as fn($ty_int) -> bool, $max); + } + + #[test] + fn little_endian() { + fn prop(n: $ty_int) -> bool { + let mut wtr = vec![]; + wtr.$write::(n.clone()).unwrap(); + let mut rdr = Cursor::new(wtr); + n == rdr.$read::($bytes).unwrap() + } + qc_sized(prop as fn($ty_int) -> bool, $max); + } + + #[test] + fn native_endian() { + fn prop(n: $ty_int) -> bool { + let mut wtr = vec![]; + wtr.$write::(n.clone()).unwrap(); + let offset = if cfg!(target_endian = "big") { + wtr.len() - $bytes + } else { + 0 + }; + let mut rdr = Cursor::new(&mut wtr[offset..]); + n == rdr.$read::($bytes).unwrap() + } + qc_sized(prop as fn($ty_int) -> bool, $max); + } + } + ); + ($name:ident, $ty_int:ty, $max:expr, $read:ident, $write:ident) => ( + mod $name { + use std::io::Cursor; + use { + ReadBytesExt, WriteBytesExt, + BigEndian, NativeEndian, LittleEndian, + }; + #[allow(unused_imports)] use test::{qc_sized, Wi128}; + + #[test] + fn big_endian() { + fn prop(n: $ty_int) -> bool { + let mut wtr = vec![]; + wtr.$write::(n.clone()).unwrap(); + let mut rdr = Cursor::new(wtr); + n == rdr.$read::().unwrap() + } + qc_sized(prop as fn($ty_int) -> bool, $max - 1); + } + + #[test] + fn little_endian() { + fn prop(n: $ty_int) -> bool { + let mut wtr = vec![]; + wtr.$write::(n.clone()).unwrap(); + let mut rdr = Cursor::new(wtr); + n == rdr.$read::().unwrap() + } + qc_sized(prop as fn($ty_int) -> bool, $max - 1); + } + + #[test] + fn native_endian() { + fn prop(n: $ty_int) -> bool { + let mut wtr = vec![]; + wtr.$write::(n.clone()).unwrap(); + let mut rdr = Cursor::new(wtr); + n == rdr.$read::().unwrap() + } + qc_sized(prop as fn($ty_int) -> bool, $max - 1); + } + } + ); + } + + qc_bytes_ext!(prop_ext_u16, + u16, ::std::u16::MAX as u64, read_u16, write_u16); + qc_bytes_ext!(prop_ext_i16, + i16, ::std::i16::MAX as u64, read_i16, write_i16); + qc_bytes_ext!(prop_ext_u32, + u32, ::std::u32::MAX as u64, read_u32, write_u32); + qc_bytes_ext!(prop_ext_i32, + i32, ::std::i32::MAX as u64, read_i32, write_i32); + qc_bytes_ext!(prop_ext_u64, + u64, ::std::u64::MAX as u64, read_u64, write_u64); + qc_bytes_ext!(prop_ext_i64, + i64, ::std::i64::MAX as u64, read_i64, write_i64); + //qc_bytes_ext!(prop_ext_f32, + // f32, ::std::u64::MAX as u64, read_f32, write_f32); + //qc_bytes_ext!(prop_ext_f64, + // f64, ::std::i64::MAX as u64, read_f64, write_f64); + + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_u128, Wi128, 16 + 1, read_u128, write_u128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_i128, Wi128, 16 + 1, read_i128, write_i128); + + qc_bytes_ext!(prop_ext_uint_1, + u64, calc_max!(::test::U64_MAX, 1), 1, read_uint, write_u64); + qc_bytes_ext!(prop_ext_uint_2, + u64, calc_max!(::test::U64_MAX, 2), 2, read_uint, write_u64); + qc_bytes_ext!(prop_ext_uint_3, + u64, calc_max!(::test::U64_MAX, 3), 3, read_uint, write_u64); + qc_bytes_ext!(prop_ext_uint_4, + u64, calc_max!(::test::U64_MAX, 4), 4, read_uint, write_u64); + qc_bytes_ext!(prop_ext_uint_5, + u64, calc_max!(::test::U64_MAX, 5), 5, read_uint, write_u64); + qc_bytes_ext!(prop_ext_uint_6, + u64, calc_max!(::test::U64_MAX, 6), 6, read_uint, write_u64); + qc_bytes_ext!(prop_ext_uint_7, + u64, calc_max!(::test::U64_MAX, 7), 7, read_uint, write_u64); + qc_bytes_ext!(prop_ext_uint_8, + u64, calc_max!(::test::U64_MAX, 8), 8, read_uint, write_u64); + + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_uint128_1, + Wi128, 1, 1, read_uint128, write_u128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_uint128_2, + Wi128, 2, 2, read_uint128, write_u128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_uint128_3, + Wi128, 3, 3, read_uint128, write_u128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_uint128_4, + Wi128, 4, 4, read_uint128, write_u128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_uint128_5, + Wi128, 5, 5, read_uint128, write_u128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_uint128_6, + Wi128, 6, 6, read_uint128, write_u128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_uint128_7, + Wi128, 7, 7, read_uint128, write_u128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_uint128_8, + Wi128, 8, 8, read_uint128, write_u128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_uint128_9, + Wi128, 9, 9, read_uint128, write_u128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_uint128_10, + Wi128, 10, 10, read_uint128, write_u128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_uint128_11, + Wi128, 11, 11, read_uint128, write_u128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_uint128_12, + Wi128, 12, 12, read_uint128, write_u128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_uint128_13, + Wi128, 13, 13, read_uint128, write_u128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_uint128_14, + Wi128, 14, 14, read_uint128, write_u128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_uint128_15, + Wi128, 15, 15, read_uint128, write_u128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_uint128_16, + Wi128, 16, 16, read_uint128, write_u128); + + qc_bytes_ext!(prop_ext_int_1, + i64, calc_max!(::test::I64_MAX, 1), 1, read_int, write_i64); + qc_bytes_ext!(prop_ext_int_2, + i64, calc_max!(::test::I64_MAX, 2), 2, read_int, write_i64); + qc_bytes_ext!(prop_ext_int_3, + i64, calc_max!(::test::I64_MAX, 3), 3, read_int, write_i64); + qc_bytes_ext!(prop_ext_int_4, + i64, calc_max!(::test::I64_MAX, 4), 4, read_int, write_i64); + qc_bytes_ext!(prop_ext_int_5, + i64, calc_max!(::test::I64_MAX, 5), 5, read_int, write_i64); + qc_bytes_ext!(prop_ext_int_6, + i64, calc_max!(::test::I64_MAX, 6), 6, read_int, write_i64); + qc_bytes_ext!(prop_ext_int_7, + i64, calc_max!(::test::I64_MAX, 1), 7, read_int, write_i64); + qc_bytes_ext!(prop_ext_int_8, + i64, calc_max!(::test::I64_MAX, 8), 8, read_int, write_i64); + + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_int128_1, + Wi128, 1, 1, read_int128, write_i128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_int128_2, + Wi128, 2, 2, read_int128, write_i128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_int128_3, + Wi128, 3, 3, read_int128, write_i128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_int128_4, + Wi128, 4, 4, read_int128, write_i128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_int128_5, + Wi128, 5, 5, read_int128, write_i128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_int128_6, + Wi128, 6, 6, read_int128, write_i128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_int128_7, + Wi128, 7, 7, read_int128, write_i128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_int128_8, + Wi128, 8, 8, read_int128, write_i128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_int128_9, + Wi128, 9, 9, read_int128, write_i128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_int128_10, + Wi128, 10, 10, read_int128, write_i128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_int128_11, + Wi128, 11, 11, read_int128, write_i128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_int128_12, + Wi128, 12, 12, read_int128, write_i128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_int128_13, + Wi128, 13, 13, read_int128, write_i128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_int128_14, + Wi128, 14, 14, read_int128, write_i128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_int128_15, + Wi128, 15, 15, read_int128, write_i128); + #[cfg(byteorder_i128)] + qc_bytes_ext!(prop_ext_int128_16, + Wi128, 16, 16, read_int128, write_i128); + + // Test slice serialization/deserialization. + macro_rules! qc_slice { + ($name:ident, $ty_int:ty, $read:ident, $write:ident, $zero:expr) => { + mod $name { + use core::mem::size_of; + use {ByteOrder, BigEndian, NativeEndian, LittleEndian}; + use super::qc_unsized; + #[allow(unused_imports)] + use test::Wi128; + + #[test] + fn big_endian() { + #[allow(unused_unsafe)] + fn prop(numbers: Vec<$ty_int>) -> bool { + let numbers: Vec<_> = numbers + .into_iter() + .map(|x| x.clone()) + .collect(); + let num_bytes = size_of::<$ty_int>() * numbers.len(); + let mut bytes = vec![0; num_bytes]; + + BigEndian::$write(&numbers, &mut bytes); + + let mut got = vec![$zero; numbers.len()]; + unsafe { BigEndian::$read(&bytes, &mut got); } + + numbers == got + } + qc_unsized(prop as fn(_) -> bool); + } + + #[test] + fn little_endian() { + #[allow(unused_unsafe)] + fn prop(numbers: Vec<$ty_int>) -> bool { + let numbers: Vec<_> = numbers + .into_iter() + .map(|x| x.clone()) + .collect(); + let num_bytes = size_of::<$ty_int>() * numbers.len(); + let mut bytes = vec![0; num_bytes]; + + LittleEndian::$write(&numbers, &mut bytes); + + let mut got = vec![$zero; numbers.len()]; + unsafe { LittleEndian::$read(&bytes, &mut got); } + + numbers == got + } + qc_unsized(prop as fn(_) -> bool); + } + + #[test] + fn native_endian() { + #[allow(unused_unsafe)] + fn prop(numbers: Vec<$ty_int>) -> bool { + let numbers: Vec<_> = numbers + .into_iter() + .map(|x| x.clone()) + .collect(); + let num_bytes = size_of::<$ty_int>() * numbers.len(); + let mut bytes = vec![0; num_bytes]; + + NativeEndian::$write(&numbers, &mut bytes); + + let mut got = vec![$zero; numbers.len()]; + unsafe { NativeEndian::$read(&bytes, &mut got); } + + numbers == got + } + qc_unsized(prop as fn(_) -> bool); + } + } + } + } + + qc_slice!(prop_slice_u16, u16, read_u16_into, write_u16_into, 0); + qc_slice!(prop_slice_i16, i16, read_i16_into, write_i16_into, 0); + qc_slice!(prop_slice_u32, u32, read_u32_into, write_u32_into, 0); + qc_slice!(prop_slice_i32, i32, read_i32_into, write_i32_into, 0); + qc_slice!(prop_slice_u64, u64, read_u64_into, write_u64_into, 0); + qc_slice!(prop_slice_i64, i64, read_i64_into, write_i64_into, 0); + #[cfg(byteorder_i128)] + qc_slice!( + prop_slice_u128, Wi128, read_u128_into, write_u128_into, 0); + #[cfg(byteorder_i128)] + qc_slice!( + prop_slice_i128, Wi128, read_i128_into, write_i128_into, 0); + + //qc_slice!( + // prop_slice_f32, f32, read_f32_into, write_f32_into, 0.0); + //qc_slice!( + // prop_slice_f64, f64, read_f64_into, write_f64_into, 0.0); +} diff --git a/crux-mir/lib/bytes.rs b/crux-mir/lib/bytes.rs new file mode 100644 index 000000000..fee6ae1c5 --- /dev/null +++ b/crux-mir/lib/bytes.rs @@ -0,0 +1,449 @@ +//! Verifier-friendly implementations of the `Bytes` and `BytesMut` types from the `bytes` crate. +//! The implementation plays some tricks to provide some support for buffers of symbolic length, +//! even though the underlying `Vector` type requires the length to be concrete. See the doc +//! comments on `Bytes`, `BytesMut`, `reserve`, and `crux_set_fixed` for details. + +extern crate crucible; + +use std::cmp::{self, Ordering}; +use std::fmt; +use std::hash::{Hash, Hasher}; +use std::io; +use std::iter::Extend; +use std::marker::PhantomData; +use std::mem; +use std::ops::{Deref, DerefMut}; + +use crucible::vector::Vector; + + +/// A view into a byte array. The underlying data is stored in a `Vector` of concrete size, +/// but the start and end can be symbolic. +#[derive(Clone)] +pub struct Bytes { + data: Vector, + start: usize, + end: usize, +} + +/// An owned, mutable byte array. The capacity, which is the size of the underlying `Vector`, +/// is always concrete, but the `len` can be symbolic. Only the first `len` bytes of `data` are +/// considered to be initialized. +#[derive(Clone)] +pub struct BytesMut { + data: Vector, + len: usize, + /// When `true`, the capacity cannot actually grow. Useful when calling code that may try to + /// `reserve` a symbolic amount of capacity. + crux_fixed: bool, +} + + +impl Bytes { + pub fn len(&self) -> usize { + self.end - self.start + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn split_off(&mut self, at: usize) -> Bytes { + let mid = self.start + at; + let end = self.end; + self.end = mid; + Bytes { data: self.data, start: mid, end } + } + + pub fn split_to(&mut self, at: usize) -> Bytes { + let start = self.start; + let mid = self.start + at; + self.start = mid; + Bytes { data: self.data, start, end: mid } + } +} + +impl BytesMut { + pub fn new() -> BytesMut { + Self::with_capacity(0) + } + + pub fn with_capacity(cap: usize) -> BytesMut { + BytesMut { + data: Vector::replicate(0, cap), + len: 0, + crux_fixed: false, + } + } + + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Freeze the contents of this buffer, producing a read-only view. + pub fn freeze(self) -> Bytes { + Bytes { data: self.data, start: 0, end: self.len } + } + + pub fn reserve(&mut self, amt: usize) { + if !self.crux_fixed { + let excess = self.data.len() - self.len; + if amt > excess { + let more = amt - excess; + self.data = self.data.concat(Vector::replicate(0, more)); + } + } + } + + pub fn truncate(&mut self, len: usize) { + self.len = cmp::min(len, self.len); + } + + pub fn crux_set_fixed(&mut self, fixed: bool) { + self.crux_fixed = fixed; + } +} + + +pub trait Buf { +} + +pub trait BufMut { + fn put_slice(&mut self, xs: &[u8]); + + fn put_u8(&mut self, x: u8); + + fn put_u16_be(&mut self, x: u16) { + self.put_slice(&x.to_be_bytes()) + } + + fn put_u32_be(&mut self, x: u32) { + self.put_slice(&x.to_be_bytes()) + } + + fn put_u64_be(&mut self, x: u64) { + self.put_slice(&x.to_be_bytes()) + } + + fn put_u128_be(&mut self, x: u128) { + self.put_slice(&x.to_be_bytes()) + } + + fn put_u16_le(&mut self, x: u16) { + self.put_slice(&x.to_le_bytes()) + } + + fn put_u32_le(&mut self, x: u32) { + self.put_slice(&x.to_le_bytes()) + } + + fn put_u64_le(&mut self, x: u64) { + self.put_slice(&x.to_le_bytes()) + } + + fn put_u128_le(&mut self, x: u128) { + self.put_slice(&x.to_le_bytes()) + } + + fn put_i8(&mut self, x: i8) { + self.put_u8(x as u8) + } + + fn put_i16_be(&mut self, x: i16) { + self.put_u16_be(x as u16) + } + + fn put_i32_be(&mut self, x: i32) { + self.put_u32_be(x as u32) + } + + fn put_i64_be(&mut self, x: i64) { + self.put_u64_be(x as u64) + } + + fn put_i128_be(&mut self, x: i128) { + self.put_u128_be(x as u128) + } + + fn put_i16_le(&mut self, x: i16) { + self.put_u16_le(x as u16) + } + + fn put_i32_le(&mut self, x: i32) { + self.put_u32_le(x as u32) + } + + fn put_i64_le(&mut self, x: i64) { + self.put_u64_le(x as u64) + } + + fn put_i128_le(&mut self, x: i128) { + self.put_u128_le(x as u128) + } + + fn writer(self) -> Writer + where Self: Sized; +} + + +impl Buf for Bytes { +} + +impl Buf for BytesMut { +} + +impl BufMut for BytesMut { + fn put_slice(&mut self, xs: &[u8]) { + assert!(xs.len() <= self.data.len() - self.len, + "not enough capacity for put_slice"); + self.data.as_mut_slice()[self.len .. self.len + xs.len()].copy_from_slice(xs); + self.len += xs.len(); + } + + fn put_u8(&mut self, x: u8) { + assert!(1 <= self.data.len() - self.len, + "not enough capacity for put_u8"); + self.data.as_mut_slice()[self.len] = x; + self.len += 1; + } + + fn writer(self) -> Writer + where Self: Sized { + Writer { + inner: self, + } + } +} + +impl BufMut for &mut T { + fn put_slice(&mut self, xs: &[u8]) { + ::put_slice(self, xs) + } + + fn put_u8(&mut self, x: u8) { + ::put_u8(self, x) + } + + fn writer(self) -> Writer + where Self: Sized { + Writer { + inner: self, + } + } +} + + +impl PartialEq for Bytes { + fn eq(&self, other: &Bytes) -> bool { + <[u8] as PartialEq>::eq(self, other) + } + fn ne(&self, other: &Bytes) -> bool { + <[u8] as PartialEq>::ne(self, other) + } +} + +impl PartialOrd for Bytes { + fn partial_cmp(&self, other: &Bytes) -> Option { + <[u8] as PartialOrd>::partial_cmp(self, other) + } + fn lt(&self, other: &Bytes) -> bool { + <[u8] as PartialOrd>::lt(self, other) + } + fn le(&self, other: &Bytes) -> bool { + <[u8] as PartialOrd>::le(self, other) + } + fn gt(&self, other: &Bytes) -> bool { + <[u8] as PartialOrd>::gt(self, other) + } + fn ge(&self, other: &Bytes) -> bool { + <[u8] as PartialOrd>::ge(self, other) + } +} + +impl Eq for Bytes {} + +impl Ord for Bytes { + fn cmp(&self, other: &Bytes) -> Ordering { + <[u8] as Ord>::cmp(self, other) + } +} + +impl fmt::Debug for Bytes { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + <[u8] as fmt::Debug>::fmt(self, fmt) + } +} + +impl Hash for Bytes { + fn hash(&self, state: &mut H) { + <[u8] as Hash>::hash(self, state) + } +} + +impl Deref for Bytes { + type Target = [u8]; + fn deref(&self) -> &[u8] { + &self.data.as_slice()[self.start .. self.end] + } +} + + +impl PartialEq for BytesMut { + fn eq(&self, other: &BytesMut) -> bool { + <[u8] as PartialEq>::eq(self, other) + } + fn ne(&self, other: &BytesMut) -> bool { + <[u8] as PartialEq>::ne(self, other) + } +} + +impl PartialOrd for BytesMut { + fn partial_cmp(&self, other: &BytesMut) -> Option { + <[u8] as PartialOrd>::partial_cmp(self, other) + } + fn lt(&self, other: &BytesMut) -> bool { + <[u8] as PartialOrd>::lt(self, other) + } + fn le(&self, other: &BytesMut) -> bool { + <[u8] as PartialOrd>::le(self, other) + } + fn gt(&self, other: &BytesMut) -> bool { + <[u8] as PartialOrd>::gt(self, other) + } + fn ge(&self, other: &BytesMut) -> bool { + <[u8] as PartialOrd>::ge(self, other) + } +} + +impl Eq for BytesMut {} + +impl Ord for BytesMut { + fn cmp(&self, other: &BytesMut) -> Ordering { + <[u8] as Ord>::cmp(self, other) + } +} + +impl fmt::Debug for BytesMut { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + <[u8] as fmt::Debug>::fmt(self, fmt) + } +} + +impl Hash for BytesMut { + fn hash(&self, state: &mut H) { + <[u8] as Hash>::hash(self, state) + } +} + +impl Deref for BytesMut { + type Target = [u8]; + fn deref(&self) -> &[u8] { + &self.data.as_slice()[.. self.len] + } +} + +impl DerefMut for BytesMut { + fn deref_mut(&mut self) -> &mut [u8] { + &mut self.data.as_mut_slice()[.. self.len] + } +} + + +impl From<&[u8]> for Bytes { + fn from(x: &[u8]) -> Bytes { + Bytes { + data: Vector::copy_from_slice(x), + start: 0, + end: x.len(), + } + } +} + +impl From<&[u8]> for BytesMut { + fn from(x: &[u8]) -> BytesMut { + BytesMut { + data: Vector::copy_from_slice(x), + len: x.len(), + crux_fixed: false, + } + } +} + +impl Extend for BytesMut { + fn extend>(&mut self, iter: I) { + for x in iter { + self.put_u8(x); + } + } +} + +impl<'a> Extend<&'a u8> for BytesMut { + fn extend>(&mut self, iter: I) { + for &x in iter { + self.put_u8(x); + } + } +} + +pub struct Iter { + data: Vector, + idx: usize, + end: usize, +} + +impl IntoIterator for Bytes { + type Item = u8; + type IntoIter = Iter; + fn into_iter(self) -> Iter { + Iter { + data: self.data, + idx: self.start, + end: self.end, + } + } +} + +impl IntoIterator for &Bytes { + type Item = u8; + type IntoIter = Iter; + fn into_iter(self) -> Iter { + Iter { + data: self.data, + idx: self.start, + end: self.end, + } + } +} + +impl Iterator for Iter { + type Item = u8; + fn next(&mut self) -> Option { + if self.idx < self.end { + let val = self.data.as_slice()[self.idx]; + self.idx += 1; + Some(val) + } else { + None + } + } +} + + +pub struct Writer { + inner: T, +} + +impl io::Write for Writer { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.inner.put_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} diff --git a/crux-mir/lib/int512.rs b/crux-mir/lib/int512.rs new file mode 100644 index 000000000..fedc7f119 --- /dev/null +++ b/crux-mir/lib/int512.rs @@ -0,0 +1,135 @@ +#![no_std] + +// extern crate core; +use core::ops::{Add, Sub, Mul, Div, Rem, BitAnd, BitOr, BitXor, Shl, Shr}; +use core::cmp::{Ord, PartialOrd, Ordering}; + +#[derive(Copy)] +pub struct Int512 { _dummy: u8 } + +pub fn clone(_i: &Int512) -> Int512 { unimplemented!() } +impl Clone for Int512 { + fn clone(&self) -> Int512 { clone(self) } +} + +pub fn symbolic(_x: &'static str) -> Int512 { unimplemented!() } +impl Int512 { + pub fn symbolic(x: &'static str) -> Int512 { symbolic(x) } +} + +macro_rules! binop { + ($Op:ident, $op:ident) => { + pub fn $op(_x: Int512, _y: Int512) -> Int512 { unimplemented!() } + + impl Int512 { + pub fn $op(self, other: Int512) -> Int512 { $op(self, other) } + } + + /* + impl $Op for Int512 { + type Output = Int512; + fn $op(self, other: Int512) -> Int512 { $op(self, other) } + } + */ + }; +} +binop!(Add, add); +binop!(Sub, sub); +binop!(Mul, mul); +binop!(Div, div); +binop!(Rem, rem); +binop!(BitAnd, bitand); +binop!(BitOr, bitor); +binop!(BitXor, bitxor); + +macro_rules! shift_op { + ($Op:ident, $op:ident) => { + pub fn $op(_x: Int512, _bits: u32) -> Int512 { unimplemented!() } + + impl Int512 { + pub fn $op(self, bits: u32) -> Int512 { $op(self, bits) } + } + + /* + impl $Op for Int512 { + type Output = Int512; + fn $op(self, bits: u32) -> Int512 { $op(self, bits) } + } + */ + }; +} +shift_op!(Shl, shl); +shift_op!(Shr, shr); + +pub fn eq(_x: Int512, _y: Int512) -> bool { unimplemented!() } +impl PartialEq for Int512 { + fn eq(&self, other: &Int512) -> bool { eq(*self, *other) } + fn ne(&self, other: &Int512) -> bool { !eq(*self, *other) } +} +impl Eq for Int512 {} + +pub fn lt(_x: Int512, _y: Int512) -> bool { unimplemented!() } +impl PartialOrd for Int512 { + fn partial_cmp(&self, other: &Int512) -> Option { + Some(self.cmp(other)) + } +} +impl Ord for Int512 { + fn cmp(&self, other: &Int512) -> Ordering { + if eq(*self, *other) { Ordering::Equal } + else if lt(*self, *other) { Ordering::Less } + else { Ordering::Greater } + } +} + +macro_rules! prim_cast { + ($Prim:ident) => { + pub mod $Prim { + use super::Int512; + pub fn from_prim(_x: $Prim) -> Int512 { unimplemented!() } + pub fn as_prim(_x: Int512) -> $Prim { unimplemented!() } + } + + impl From<$Prim> for Int512 { + fn from(x: $Prim) -> Int512 { self::$Prim::from_prim(x) } + } + + impl From for $Prim { + fn from(x: Int512) -> $Prim { self::$Prim::as_prim(x) } + } + }; +} +prim_cast!(u8); +prim_cast!(u16); +prim_cast!(u32); +prim_cast!(u64); +prim_cast!(u128); +prim_cast!(usize); +prim_cast!(i8); +prim_cast!(i16); +prim_cast!(i32); +prim_cast!(i64); +prim_cast!(i128); +prim_cast!(isize); + + +impl From<[u8; 32]> for Int512 { + fn from(x: [u8; 32]) -> Int512 { + let mut acc = Int512::from(0_i32); + for i in 0..5 { + acc = acc.bitor(Int512::from(x[i]).shl(8 * i as u32)); + } + acc + } +} + +impl From for [u8; 32] { + fn from(x: Int512) -> [u8; 32] { + let mut acc = [0; 32]; + let mask: Int512 = Int512::from((1_u64 << 8) - 1); + for i in 0..32 { + acc[i] = u8::from(x.shr(8 * i as u32).bitand(mask)); + } + acc + } +} diff --git a/crux-mir/src/Mir/Generate.hs b/crux-mir/src/Mir/Generate.hs index 8e659309a..694018fc8 100644 --- a/crux-mir/src/Mir/Generate.hs +++ b/crux-mir/src/Mir/Generate.hs @@ -38,6 +38,7 @@ import Mir.PP() import Debug.Trace + getModificationTimeIfExists :: FilePath -> IO (Maybe UTCTime) getModificationTimeIfExists path = doesFileExist path >>= \case False -> return Nothing @@ -121,23 +122,33 @@ maybeLinkJson jsonFiles cacheFile = do libJsonFiles :: [FilePath] libJsonFiles = + -- std and its dependencies [ "libcore.mir" + , "librustc_std_workspace_core.mir" + , "liblibc.mir" , "libcompiler_builtins.mir" + , "liballoc.mir" + , "libcfg_if.mir" + , "libmemchr.mir" + , "libadler.mir" + , "librustc_demangle.mir" + , "libunwind.mir" + , "libpanic_unwind.mir" + , "librustc_std_workspace_alloc.mir" + , "libpanic_abort.mir" + , "libgimli.mir" + , "libstd_detect.mir" + , "libobject.mir" + , "libminiz_oxide.mir" + , "libhashbrown.mir" + , "libaddr2line.mir" , "libstd.mir" + -- additional libs + , "libcrucible.mir" + , "libint512.mir" + , "libbyteorder.mir" + , "libbytes.mir" ] - -- , "libint512.mir" - -- , "libcrucible.mir" - - -- , "liballoc.mir" - -- , "libstd.mir" - -- , "libunwind.mir" - -- , "libcfg_if.mir" - -- , "libhashbrown.mir" - -- , "liblibc.mir" - - -- , "libbyteorder.mir" - -- , "libbytes.mir" - -- ] -- | Run mir-json on the input, generating lib file on disk From e4e3a65e2c04a75bcd7b7732bb557a0935af797a Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 22 Mar 2023 13:15:16 -0400 Subject: [PATCH 006/114] build.sh: Set STD_ENV_ARCH --- crux-mir/build.sh | 3 +++ 1 file changed, 3 insertions(+) mode change 100644 => 100755 crux-mir/build.sh diff --git a/crux-mir/build.sh b/crux-mir/build.sh old mode 100644 new mode 100755 index 8c465d81e..35cea2e5f --- a/crux-mir/build.sh +++ b/crux-mir/build.sh @@ -1,4 +1,7 @@ set -e + +export STD_ENV_ARCH=$(uname -m) + echo 'Building core...' mir-json lib/core/src/lib.rs --edition=2021 --crate-name core -L rlibs --out-dir rlibs --crate-type rlib From 1cc32baf7daa06adc611eeef781f741f1f73b08f Mon Sep 17 00:00:00 2001 From: James LaMar Date: Wed, 22 Mar 2023 10:28:03 -0700 Subject: [PATCH 007/114] Add `test_output.log` --- crux-mir/test_output.log | 4181 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 4181 insertions(+) create mode 100644 crux-mir/test_output.log diff --git a/crux-mir/test_output.log b/crux-mir/test_output.log new file mode 100644 index 000000000..24df242dd --- /dev/null +++ b/crux-mir/test_output.log @@ -0,0 +1,4181 @@ +Up to date +crux-mir + crux concrete + + vec + set_len: FAIL (0.24s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (0.10s) + Crux output: + failures: + + ---- set_len/e7047f4e::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/set_len/' to rerun this test only. + from_elem_zero: thread 'main' panicked at 'called `Result::unwrap()` on an `Err` value: ErrorImpl { code: Io(Custom { kind: Other, error: "numeric field was not a number: when getting cksum for crate.json" }), offset: 0 }', src/bin/mir-json-dce.rs:38:44 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAIL (0.17s) + Compiling and running oracle program (0.15s) + Oracle output: 0 (0.02s) + user error (Error 101 while running mir-json on ["test/conc_eval/vec/from_elem_zero.mir","rlibs/libcore.mir","rlibs/librustc_std_workspace_core.mir","rlibs/liblibc.mir","rlibs/libcompiler_builtins.mir","rlibs/liballoc.mir","rlibs/libcfg_if.mir","rlibs/libmemchr.mir","rlibs/libadler.mir","rlibs/librustc_demangle.mir","rlibs/libunwind.mir","rlibs/libpanic_unwind.mir","rlibs/librustc_std_workspace_alloc.mir","rlibs/libpanic_abort.mir","rlibs/libgimli.mir","rlibs/libstd_detect.mir","rlibs/libobject.mir","rlibs/libminiz_oxide.mir","rlibs/libhashbrown.mir","rlibs/libaddr2line.mir","rlibs/libstd.mir","rlibs/libcrucible.mir","rlibs/libint512.mir","rlibs/libbyteorder.mir","rlibs/libbytes.mir"]) + + Use -p '/from_elem_zero/' to rerun this test only. + extend: FAIL (0.74s) + Compiling and running oracle program (0.16s) + Oracle output: (1, 10) (0.57s) + Crux output: + failures: + + ---- extend/94e78a5b::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '$0=="crux-mir.crux concrete..vec.extend"' to rerun this test only. + extend_trusted_len: FAIL (0.85s) + Compiling and running oracle program (0.15s) + Oracle output: (1, 10) (0.70s) + Crux output: + failures: + + ---- extend_trusted_len/c2c419e5::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/extend_trusted_len/' to rerun this test only. + collect: FAIL (0.83s) + Compiling and running oracle program (0.16s) + Oracle output: 45 (0.68s) + Crux output: + failures: + + ---- collect/569f54db::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/collect/' to rerun this test only. + push: FAIL (0.76s) + Compiling and running oracle program (0.15s) + Oracle output: (1, 2) (0.62s) + Crux output: + failures: + + ---- push/c96f3e94::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/vec.push/' to rerun this test only. + drop: FAIL (0.25s) + Compiling and running oracle program (0.16s) + Oracle output: () (0.09s) + Crux output: + failures: + + ---- drop/c82c6b8e::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/drop/' to rerun this test only. + dyn + trait_param: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 100 + Crux output: 100 + inherit: FAIL (0.14s) + Compiling and running oracle program (0.13s) + Oracle output: 4 (0.01s) + Crux output: + failures: + + ---- inherit/a89b9890::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/dyn/inherit.rs:20:13: 20:16: error: in inherit/a89b9890::crux_test[0] + [Crux] Translation error in inherit/a89b9890::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/inherit/' to rerun this test only. + plain_trait: OK (0.13s) + Compiling and running oracle program (0.13s) + Oracle output: 100 + Crux output: 100 + assoc_ty: OK (0.13s) + Compiling and running oracle program (0.13s) + Oracle output: 100 + Crux output: 100 + tuple + clone_from: FAIL (0.15s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.02s) + Crux output: + failures: + + ---- clone_from/9cd626ed::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/clone.rs:136:17: 136:31: error: in core/516ae8d9::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0] + [Crux] Translation error in core/516ae8d9::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0]: don't know how to generate CloneShim for unknown method core/516ae8d9::clone[0]::Clone[0]::clone[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/clone_from/' to rerun this test only. + clone_struct: FAIL (0.16s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.02s) + Crux output: + failures: + + ---- clone_struct/9d8703d1::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/tuple/clone_struct.rs:8:13: 8:22: error: in clone_struct/9d8703d1::f[0] + [Crux] Translation error in clone_struct/9d8703d1::f[0]: don't know how to generate CloneShim for unknown method core/516ae8d9::clone[0]::Clone[0]::clone[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/clone_struct/' to rerun this test only. + clone: OK (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.01s) + Crux output: () + clone_rec: FAIL (0.16s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.02s) + Crux output: + failures: + + ---- clone_rec/e47383a3::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/tuple/clone_rec.rs:4:17: 4:28: error: in clone_rec/e47383a3::f[0] + [Crux] Translation error in clone_rec/e47383a3::f[0]: fail or unimp constant: TyTuple [TyInt B32,TyInt B32] (StructRepr [MaybeRepr (BVRepr 32), MaybeRepr (BVRepr 32)]) ConstTuple [ConstInt (I32 3),ConstInt (I32 4)] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/clone_rec/' to rerun this test only. + cell + ref_cell2: FAIL (0.60s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (0.46s) + Crux output: + failures: + + ---- ref_cell2/39dbeea6::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/ref_cell2/' to rerun this test only. + ref_cell: FAIL (0.64s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (0.49s) + Crux output: + failures: + + ---- ref_cell/b2662862::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '$0=="crux-mir.crux concrete..cell.ref_cell"' to rerun this test only. + cell: OK (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (0.02s) + Crux output: 2 + str + format: FAIL (0.76s) + Compiling and running oracle program (0.14s) + Oracle output: true (0.62s) + Crux output: + failures: + + ---- format/703e8c38::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '$0=="crux-mir.crux concrete..str.format"' to rerun this test only. + format_hex: FAIL (0.80s) + Compiling and running oracle program (0.15s) + Oracle output: true (0.65s) + Crux output: + failures: + + ---- format_hex/a2ffc47f::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/format_hex/' to rerun this test only. + to_owned: FAIL (0.24s) + Compiling and running oracle program (0.14s) + Oracle output: true (0.10s) + Crux output: + failures: + + ---- to_owned/7a8109f8::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/516ae8d9::str[0]::{impl#0}[0]::as_bytes[0] + [Crux] Translation error in core/516ae8d9::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/to_owned/' to rerun this test only. + format_struct: FAIL (0.91s) + Compiling and running oracle program (0.15s) + Oracle output: true (0.76s) + Crux output: + failures: + + ---- format_struct/782b6488::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/format_struct/' to rerun this test only. + format_int: FAIL (0.79s) + Compiling and running oracle program (0.14s) + Oracle output: true (0.65s) + Crux output: + failures: + + ---- format_int/dcc1f157::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/format_int/' to rerun this test only. + format_array: FAIL (0.91s) + Compiling and running oracle program (0.15s) + Oracle output: true (0.75s) + Crux output: + failures: + + ---- format_array/82d51d12::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/format_array/' to rerun this test only. + string_push: FAIL (0.71s) + Compiling and running oracle program (0.15s) + Oracle output: true (0.56s) + Crux output: + failures: + + ---- string_push/3e757ad8::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/string_push/' to rerun this test only. + iter + cloned: FAIL (0.17s) + Compiling and running oracle program (0.14s) + Oracle output: 6 (0.03s) + Crux output: + failures: + + ---- cloned/3fb92071::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0] + [Crux] Translation error in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyUint USize) Immut + [Crux] as: TyRawPtr (TyUint B8) Immut + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/cloned/' to rerun this test only. + from_fn: OK (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.01s) + Crux output: () + sum: FAIL (0.18s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.04s) + Crux output: + failures: + + ---- sum/1ed45972::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0] + [Crux] Translation error in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyUint USize) Immut + [Crux] as: TyRawPtr (TyUint B8) Immut + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/sum/' to rerun this test only. + loop: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 2 + Crux output: 2 + zip: FAIL (0.19s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.06s) + Crux output: + failures: + + ---- zip/599ebd70::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyInt B32) Immut + [Crux] as: TyRawPtr (TyUint B8) Immut + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/zip/' to rerun this test only. + for: FAIL (0.17s) + Compiling and running oracle program (0.14s) + Oracle output: 47 (0.03s) + Crux output: + failures: + + ---- for/90feb64a::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:470:54: 470:57 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:921:5: 922:101: error: in core/516ae8d9::num[0]::{impl#9}[0]::unchecked_add[0] + [Crux] Translation error in core/516ae8d9::num[0]::{impl#9}[0]::unchecked_add[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_instc5e93708b8ca6e2a[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/iter.for/' to rerun this test only. + peek: FAIL (0.20s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (0.05s) + Crux output: + failures: + + ---- peek/1909e1c9::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0] + [Crux] Translation error in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyUint USize) Immut + [Crux] as: TyRawPtr (TyUint B8) Immut + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/peek/' to rerun this test only. + filter_chain: OK (0.24s) + Compiling and running oracle program (0.16s) + Oracle output: 0 (0.08s) + Crux output: 0 + hash_map + insert_remove: FAIL (0.99s) + Compiling and running oracle program (0.17s) + Oracle output: 12 (0.81s) + vtable signature mismatch for vtable core/516ae8d9::ops[0]::function[0]::FnMut[0]::_vtblc2e637e13369cbd5[0] of trait core/516ae8d9::ops[0]::function[0]::FnMut[0]::_trait35b29a8a4ae9e473[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, BVRepr 32] BoolRepr] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr] + CallStack (from HasCallStack): + error, called at src/Mir/Trans.hs:904:20 in crux-mir-0.6.0.99-inplace:Mir.Trans + mkTraitObject, called at src/Mir/Trans.hs:747:11 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate + translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language + + Use -p '/insert_remove/' to rerun this test only. + insert_iter: FAIL (0.97s) + Compiling and running oracle program (0.18s) + Oracle output: [11, 12] (0.79s) + vtable signature mismatch for vtable core/516ae8d9::ops[0]::function[0]::FnMut[0]::_vtblc2e637e13369cbd5[0] of trait core/516ae8d9::ops[0]::function[0]::FnMut[0]::_trait35b29a8a4ae9e473[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, BVRepr 32] BoolRepr] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr] + CallStack (from HasCallStack): + error, called at src/Mir/Trans.hs:904:20 in crux-mir-0.6.0.99-inplace:Mir.Trans + mkTraitObject, called at src/Mir/Trans.hs:747:11 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate + translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language + + Use -p '/insert_iter/' to rerun this test only. + insert_multi: FAIL (0.98s) + Compiling and running oracle program (0.17s) + Oracle output: 100 (0.80s) + vtable signature mismatch for vtable core/516ae8d9::ops[0]::function[0]::FnMut[0]::_vtblc2e637e13369cbd5[0] of trait core/516ae8d9::ops[0]::function[0]::FnMut[0]::_trait35b29a8a4ae9e473[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, BVRepr 32] BoolRepr] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr] + CallStack (from HasCallStack): + error, called at src/Mir/Trans.hs:904:20 in crux-mir-0.6.0.99-inplace:Mir.Trans + mkTraitObject, called at src/Mir/Trans.hs:747:11 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate + translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language + + Use -p '/insert_multi/' to rerun this test only. + insert_get: FAIL (0.92s) + Compiling and running oracle program (0.17s) + Oracle output: (11, 12) (0.75s) + vtable signature mismatch for vtable core/516ae8d9::ops[0]::function[0]::FnMut[0]::_vtblc2e637e13369cbd5[0] of trait core/516ae8d9::ops[0]::function[0]::FnMut[0]::_trait35b29a8a4ae9e473[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, BVRepr 32] BoolRepr] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr] + CallStack (from HasCallStack): + error, called at src/Mir/Trans.hs:904:20 in crux-mir-0.6.0.99-inplace:Mir.Trans + mkTraitObject, called at src/Mir/Trans.hs:747:11 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate + translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language + + Use -p '/insert_get/' to rerun this test only. + enum + cmp: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: -23 + Crux output: -23 + eq: FAIL (0.16s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.02s) + Crux output: + failures: + + ---- eq/41d79a98::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/enum/eq.rs:9:17: 9:21: error: in eq/41d79a98::f[0] + [Crux] Translation error in eq/41d79a98::f[0]: cannot find static variable alloc$0/3a1fbbbh + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/enum.eq/' to rerun this test only. + arg2: rustc compilation failed for arg2 +error output: +error[E0518]: attribute should be applied to function or closure + --> test/conc_eval/enum/arg2.rs:3:5 + | +3 | #[inline(never)] + | ^^^^^^^^^^^^^^^^ +4 | A(u8), + | ----- not a function or closure + +error[E0518]: attribute should be applied to function or closure + --> test/conc_eval/enum/arg2.rs:5:5 + | +5 | #[inline(never)] + | ^^^^^^^^^^^^^^^^ +6 | B(i32,i32), + | ---------- not a function or closure + +error: aborting due to 2 previous errors + +For more information about this error, try `rustc --explain E0518`. + +FAIL (0.05s) + Compiling and running oracle program (0.05s) + test/Test.hs:109: + failed to compile and run + + Use -p '/arg2/' to rerun this test only. + cow: FAIL (0.24s) + Compiling and running oracle program (0.14s) + Oracle output: 200 (0.10s) + Crux output: + failures: + + ---- cow/57074db5::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/516ae8d9::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/516ae8d9::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/cow/' to rerun this test only. + field_order: OK (0.14s) + Compiling and running oracle program (0.13s) + Oracle output: () + Crux output: () + arg: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 0 + Crux output: 0 + ret: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: (A(42), B { x: 42 }, C) + Crux output: (A(42), B { x: 42 }, C) + match: OK (0.13s) + Compiling and running oracle program (0.13s) + Oracle output: 42 + Crux output: 42 + inner: OK (0.13s) + Compiling and running oracle program (0.13s) + Oracle output: 4 + Crux output: 4 + mixed_discrs: rustc compilation failed for mixed_discrs +error output: +error[E0554]: `#![feature]` may not be used on the stable release channel + --> test/conc_eval/enum/mixed_discrs.rs:2:12 + | +2 | #![feature(core_intrinsics)] + | ^^^^^^^^^^^^^^^ + +error: aborting due to previous error + +For more information about this error, try `rustc --explain E0554`. + +FAIL (0.06s) + Compiling and running oracle program (0.06s) + test/Test.hs:109: + failed to compile and run + + Use -p '/mixed_discrs/' to rerun this test only. + array + wick2: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: true + Crux output: true + from_slice: FAIL (0.71s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.58s) + Crux output: + failures: + + ---- from_slice/e616df27::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/array.from_slice/' to rerun this test only. + clone: FAIL (0.16s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.02s) + Crux output: + failures: + + ---- clone/12c743af::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:61:9: 61:18: error: in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0] + [Crux] Translation error in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyInt B32) Immut + [Crux] as: TyRawPtr (TyArray (TyInt B32) 3) Immut + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/array.clone/' to rerun this test only. + mut_index: FAIL (0.55s) + Compiling and running oracle program (0.15s) + Oracle output: 7 (0.40s) + Crux output: + failures: + + ---- mut_index/9c08c2ae::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/mut_index/' to rerun this test only. + wick1: FAIL (expected: needs Vec data structure from stdlib) (0.19s) + Compiling and running oracle program (0.15s) + Oracle output: true (0.04s) + user error (Error 101 while running mir-json on test/conc_eval/array/wick1.rs) + (expected failure) + arg: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 2 + Crux output: 2 + iter: FAIL (0.18s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (0.04s) + Crux output: + failures: + + ---- iter/d416c50d::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyInt B32) Immut + [Crux] as: TyRawPtr (TyUint B8) Immut + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/array.iter/' to rerun this test only. + wick3: FAIL (expected: needs Vec data structure from stdlib) (0.19s) + Compiling and running oracle program (0.15s) + Oracle output: true (0.04s) + user error (Error 101 while running mir-json on test/conc_eval/array/wick3.rs) + (expected failure) + mk_and_proj: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 42 + Crux output: 42 + mut_arg: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 42 + Crux output: 42 + const: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 1 + Crux output: 1 + const_impl: OK (0.14s) + Compiling and running oracle program (0.13s) + Oracle output: 5 + Crux output: 5 + stdlib + option: OK (0.14s) + Compiling and running oracle program (0.13s) + Oracle output: true + Crux output: true + range: OK (0.13s) + Compiling and running oracle program (0.13s) + Oracle output: 10 + Crux output: 10 + default_impl: OK (0.13s) + Compiling and running oracle program (0.13s) + Oracle output: () + Crux output: () + teq: OK (0.13s) + Compiling and running oracle program (0.13s) + Oracle output: false + Crux output: false + option3: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 27 + Crux output: 27 + result: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 27 + Crux output: 27 + poly: OK (0.14s) + Compiling and running oracle program (0.13s) + Oracle output: 1 + Crux output: 1 + default: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: true + Crux output: true + cvt: OK (0.13s) + Compiling and running oracle program (0.13s) + Oracle output: 0 + Crux output: 0 + result_interior: OK (0.14s) + Compiling and running oracle program (0.13s) + Oracle output: 3 + Crux output: 3 + option2: OK (0.13s) + Compiling and running oracle program (0.13s) + Oracle output: 0 + Crux output: 0 + intTest + test0039: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 7 + Crux output: 7 + test0038: FAIL (0.19s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.04s) + Crux output: + failures: + + ---- test0038/f7eb1122::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/516ae8d9::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst5dab88813bb37c92[0] + [Crux] Translation error in core/516ae8d9::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst5dab88813bb37c92[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyArray (TyInt B32) 4) Mut + [Crux] as: TyRawPtr (TyUint B8) Mut + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/test0038/' to rerun this test only. + box + new: FAIL (0.20s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.06s) + Crux output: + failures: + + ---- new/2d77246e::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/box.new/' to rerun this test only. + struct: rustc compilation failed for struct +error output: +error[E0554]: `#![feature]` may not be used on the stable release channel + --> test/conc_eval/box/struct.rs:1:1 + | +1 | #![feature(box_syntax)] + | ^^^^^^^^^^^^^^^^^^^^^^^ + +error: aborting due to previous error + +For more information about this error, try `rustc --explain E0554`. + +FAIL (0.05s) + Compiling and running oracle program (0.05s) + test/Test.hs:109: + failed to compile and run + + Use -p '/box.struct/' to rerun this test only. + unsize: FAIL (0.21s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (0.06s) + Crux output: + failures: + + ---- unsize/090d8d2d::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/box.unsize/' to rerun this test only. + mut_ref: FAIL (0.20s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.06s) + Crux output: + failures: + + ---- mut_ref/e4e7f393::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/box.mut_ref/' to rerun this test only. + mut: FAIL (0.20s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.06s) + Crux output: + failures: + + ---- mut/86109469::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '$0=="crux-mir.crux concrete..box.mut"' to rerun this test only. + sync + rwlock: FAIL (0.95s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (0.80s) + standalone use of `dyn` is not supported: TyDynamic core/516ae8d9::error[0]::Error[0]::_trait56353a7b840403b4[0] + CallStack (from HasCallStack): + error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy + tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate + translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language + + Use -p '$0=="crux-mir.crux concrete..sync.rwlock"' to rerun this test only. + atomic_cxchg: FAIL (0.19s) + Compiling and running oracle program (0.15s) + Oracle output: 6 (0.04s) + Crux output: + failures: + + ---- atomic_cxchg/b55893dc::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/sync/atomic_cxchg.rs:8:22: 8:66: error: in atomic_cxchg/b55893dc::crux_test[0] + [Crux] Translation error in atomic_cxchg/b55893dc::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/atomic_cxchg/' to rerun this test only. + atomic_fence: FAIL (0.16s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (0.02s) + Crux output: + failures: + + ---- atomic_fence/f75b8912::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/sync/atomic.rs:3010:60: 3010:63: error: in core/516ae8d9::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/516ae8d9::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::atomic_store_seqcst[0]::_inst1e2825177cd3b608[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/atomic_fence/' to rerun this test only. + atomic_add: FAIL (0.18s) + Compiling and running oracle program (0.14s) + Oracle output: 6 (0.04s) + Crux output: + failures: + + ---- atomic_add/f2b8808e::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/sync/atomic_add.rs:8:22: 8:66: error: in atomic_add/f2b8808e::crux_test[0] + [Crux] Translation error in atomic_add/f2b8808e::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/atomic_add/' to rerun this test only. + arc: FAIL (0.21s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (0.07s) + Crux output: + failures: + + ---- arc/2c43be88::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '$0=="crux-mir.crux concrete..sync.arc"' to rerun this test only. + rwlock_multi: FAIL (1.05s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (0.90s) + standalone use of `dyn` is not supported: TyDynamic core/516ae8d9::error[0]::Error[0]::_trait56353a7b840403b4[0] + CallStack (from HasCallStack): + error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy + tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate + translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language + + Use -p '/rwlock_multi/' to rerun this test only. + arc_clone: FAIL (0.23s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (0.08s) + Crux output: + failures: + + ---- arc_clone/e8f267c7::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/arc_clone/' to rerun this test only. + mutex: FAIL (0.93s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (0.78s) + standalone use of `dyn` is not supported: TyDynamic core/516ae8d9::error[0]::Error[0]::_trait56353a7b840403b4[0] + CallStack (from HasCallStack): + error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy + tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate + translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language + + Use -p '$0=="crux-mir.crux concrete..sync.mutex"' to rerun this test only. + arc_cell: FAIL (0.23s) + Compiling and running oracle program (0.15s) + Oracle output: 4 (0.08s) + Crux output: + failures: + + ---- arc_cell/ae087adc::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/arc_cell/' to rerun this test only. + mutex_multi: FAIL (0.97s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (0.82s) + standalone use of `dyn` is not supported: TyDynamic core/516ae8d9::error[0]::Error[0]::_trait56353a7b840403b4[0] + CallStack (from HasCallStack): + error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy + tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate + translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language + + Use -p '/mutex_multi/' to rerun this test only. + atomic_swap: FAIL (0.16s) + Compiling and running oracle program (0.14s) + Oracle output: (2, 1) (0.02s) + Crux output: + failures: + + ---- atomic_swap/bed7fbdb::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/sync/atomic.rs:3010:60: 3010:63: error: in core/516ae8d9::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/516ae8d9::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::atomic_store_seqcst[0]::_inst1e2825177cd3b608[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/atomic_swap/' to rerun this test only. + slice + range_len: rustc compilation failed for range_len +error output: +warning: unused import: `core::slice::SliceIndex` + --> test/conc_eval/slice/range_len.rs:5:5 + | +5 | use core::slice::SliceIndex; + | ^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: `#[warn(unused_imports)]` on by default + +warning: unused import: `core::ops::Range` + --> test/conc_eval/slice/range_len.rs:6:5 + | +6 | use core::ops::Range; + | ^^^^^^^^^^^^^^^^ + +error[E0554]: `#![feature]` may not be used on the stable release channel + --> test/conc_eval/slice/range_len.rs:2:12 + | +2 | #![feature(slice_index_methods)] + | ^^^^^^^^^^^^^^^^^^^ + +error: aborting due to previous error; 2 warnings emitted + +For more information about this error, try `rustc --explain E0554`. + +FAIL (0.06s) + Compiling and running oracle program (0.06s) + test/Test.hs:109: + failed to compile and run + + Use -p '$0=="crux-mir.crux concrete..slice.range_len"' to rerun this test only. + iter_mut: FAIL (0.17s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.03s) + Crux output: + failures: + + ---- iter_mut/984cc64c::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/516ae8d9::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/516ae8d9::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyInt B32) Mut + [Crux] as: TyRawPtr (TyUint B8) Mut + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/iter_mut/' to rerun this test only. + eq: FAIL (0.15s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.02s) + Crux output: + failures: + + ---- eq/9b499933::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/mem/mod.rs:338:38: 338:41: error: in core/516ae8d9::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/516ae8d9::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::size_of_val[0]::_inst55af81a2cef2701c[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/slice.eq/' to rerun this test only. + swap: OK (0.16s) + Compiling and running oracle program (0.14s) + Oracle output: 2001 (0.02s) + Crux output: 2001 + last: OK (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (0.01s) + Crux output: 3 + mut_range: FAIL (0.62s) + Compiling and running oracle program (0.14s) + Oracle output: 86 (0.48s) + Crux output: + failures: + + ---- mut_range/b300ab62::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/mut_range/' to rerun this test only. + mk_and_proj: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 42 + Crux output: 42 + get: rustc compilation failed for get +error output: +error[E0554]: `#![feature]` may not be used on the stable release channel + --> test/conc_eval/slice/get.rs:2:1 + | +2 | #![feature(never_type)] + | ^^^^^^^^^^^^^^^^^^^^^^^ + +error: aborting due to previous error + +For more information about this error, try `rustc --explain E0554`. + +FAIL (0.05s) + Compiling and running oracle program (0.05s) + test/Test.hs:109: + failed to compile and run + + Use -p '/slice.get/' to rerun this test only. + len: OK (0.13s) + Compiling and running oracle program (0.13s) + Oracle output: 5 + Crux output: 5 + range_len_mut: rustc compilation failed for range_len_mut +error output: +warning: unused import: `core::slice::SliceIndex` + --> test/conc_eval/slice/range_len_mut.rs:5:5 + | +5 | use core::slice::SliceIndex; + | ^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: `#[warn(unused_imports)]` on by default + +warning: unused import: `core::ops::Range` + --> test/conc_eval/slice/range_len_mut.rs:6:5 + | +6 | use core::ops::Range; + | ^^^^^^^^^^^^^^^^ + +error[E0554]: `#![feature]` may not be used on the stable release channel + --> test/conc_eval/slice/range_len_mut.rs:2:12 + | +2 | #![feature(slice_index_methods)] + | ^^^^^^^^^^^^^^^^^^^ + +error: aborting due to previous error; 2 warnings emitted + +For more information about this error, try `rustc --explain E0554`. + +FAIL (0.05s) + Compiling and running oracle program (0.05s) + test/Test.hs:109: + failed to compile and run + + Use -p '/range_len_mut/' to rerun this test only. + mut: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 42 + Crux output: 42 + clos + fnptr_fn: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 4 + Crux output: 4 + fnonce1: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 3 + Crux output: 3 + conv_fnonce_fn: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 2 + Crux output: 2 + fo: rustc compilation failed for fo +error output: +error[E0554]: `#![feature]` may not be used on the stable release channel + --> test/conc_eval/clos/fo.rs:3:1 + | +3 | #![feature(type_ascription)] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unnecessary parentheses around function argument + --> test/conc_eval/clos/fo.rs:11:7 + | +11 | g((1 :i32)) + | ^ ^ + | + = note: `#[warn(unused_parens)]` on by default +help: remove these parentheses + | +11 - g((1 :i32)) +11 + g(1 :i32) + | + +error: aborting due to previous error; 1 warning emitted + +For more information about this error, try `rustc --explain E0554`. + +FAIL (0.05s) + Compiling and running oracle program (0.05s) + test/Test.hs:109: + failed to compile and run + + Use -p '/clos.fo/' to rerun this test only. + direct_fn: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 2 + Crux output: 2 + unique_borrow: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 3 + Crux output: 3 + fn_static: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 3 + Crux output: 3 + fnptr_fnonce: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 4 + Crux output: 4 + direct_fnonce: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 2 + Crux output: 2 + dispatch_fnmut: FAIL (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: 7 (0.02s) + Crux output: + failures: + + ---- dispatch_fnmut/3d7553f9::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/clos/dispatch_fnmut.rs:7:7: 7:9: error: in dispatch_fnmut/3d7553f9::call_it2[0]::_inst15715dbe1d0521ca[0] + [Crux] Translation error in dispatch_fnmut/3d7553f9::call_it2[0]::_inst15715dbe1d0521ca[0]: cannot find static variable alloc$0/3a1fbbbh + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/dispatch_fnmut/' to rerun this test only. + conv_fnmut_fn: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: 2 + Crux output: 2 + fnptr_fnmut: OK (0.16s) + Compiling and running oracle program (0.15s) + Oracle output: 4 + Crux output: 4 + poly: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 3 + Crux output: 3 + conv_fnonce_fnmut: OK (0.17s) + Compiling and running oracle program (0.17s) + Oracle output: 2 + Crux output: 2 + fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (0.16s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.02s) + user error (Error 101 while running mir-json on test/conc_eval/clos/fn_dyn.rs) + (expected failure) + ref_fnmut: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 7 + Crux output: 7 + as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (0.01s) + Crux output: + failures: + + ---- as_fn_ptr/989685c7::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/989685c7::crux_test[0] + [Crux] Translation error in as_fn_ptr/989685c7::crux_test[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [], _fsreturn_ty = TyInt B32, _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + (expected failure) + fnonce: OK (0.13s) + Compiling and running oracle program (0.13s) + Oracle output: false + Crux output: false + promoted: FAIL (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: 33 (0.01s) + Crux output: + failures: + + ---- promoted/1f26acf5::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/clos/promoted.rs:15:9: 15:11: error: in promoted/1f26acf5::f[0] + [Crux] Translation error in promoted/1f26acf5::f[0]: cannot find static variable alloc$0/3a1fbbbh + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/clos.promoted/' to rerun this test only. + fnptr_closure: FAIL (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: 7 (0.02s) + Crux output: + failures: + + ---- fnptr_closure/e8a8c208::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/clos/fnptr_closure.rs:7:7: 7:9: error: in fnptr_closure/e8a8c208::call_it2[0]::_inste366449525d6d5d6[0] + [Crux] Translation error in fnptr_closure/e8a8c208::call_it2[0]::_inste366449525d6d5d6[0]: cannot find static variable alloc$0/3a1fbbbh + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/fnptr_closure/' to rerun this test only. + direct_fnmut2: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 7 + Crux output: 7 + direct_fnmut: OK (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: 7 + Crux output: 7 + fn_static_poly: OK (0.14s) + Compiling and running oracle program (0.13s) + Oracle output: 3 + Crux output: 3 + prim + ffs: FAIL (0.18s) + Compiling and running oracle program (0.14s) + Oracle output: true (0.03s) + Crux output: + failures: + + ---- ffs/2effae63::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/int_macros.rs:460:54: 460:57 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:251:5: 253:45: error: in core/516ae8d9::num[0]::{impl#3}[0]::unchecked_add[0] + [Crux] Translation error in core/516ae8d9::num[0]::{impl#3}[0]::unchecked_add[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_inst1e2825177cd3b608[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/prim.ffs/' to rerun this test only. + add1: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 2 + Crux output: 2 + litstring: FAIL (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: true (0.01s) + Crux output: + failures: + + ---- litstring/07909131::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/516ae8d9::str[0]::{impl#0}[0]::as_bytes[0] + [Crux] Translation error in core/516ae8d9::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/litstring/' to rerun this test only. + wrapping_sub: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: true + Crux output: true + shift2: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 2 + Crux output: 2 + shift3: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: -9223372036854775808 + Crux output: -9223372036854775808 + char_from_u32: FAIL (0.16s) + Compiling and running oracle program (0.14s) + Oracle output: 'A' (0.02s) + Crux output: + failures: + + ---- char_from_u32/3fda601b::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/char/convert.rs:215:31: 215:32: error: in core/516ae8d9::char[0]::convert[0]::char_try_from_u32[0] + [Crux] Translation error in core/516ae8d9::char[0]::convert[0]::char_try_from_u32[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::transmute[0]::_instdc3291f36c45d6de[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/char_from_u32/' to rerun this test only. + div: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 4 + Crux output: 4 + shift_exceeding: rustc compilation failed for shift_exceeding +error output: +error[E0425]: cannot find function `catch_unwind` in module `panic` + --> test/conc_eval/prim/shift_exceeding.rs:14:25 + | +14 | let result = panic::catch_unwind(|| { + | ^^^^^^^^^^^^ not found in `panic` + | +help: consider importing this function + | +3 | use std::panic::catch_unwind; + | +help: if you import `catch_unwind`, refer to it directly + | +14 - let result = panic::catch_unwind(|| { +14 + let result = catch_unwind(|| { + | + +warning: lint `exceeding_bitshifts` has been renamed to `arithmetic_overflow` + --> test/conc_eval/prim/shift_exceeding.rs:4:9 + | +4 | #[allow(exceeding_bitshifts)] + | ^^^^^^^^^^^^^^^^^^^ help: use the new name: `arithmetic_overflow` + | + = note: `#[warn(renamed_and_removed_lints)]` on by default + +error: aborting due to previous error; 1 warning emitted + +For more information about this error, try `rustc --explain E0425`. + +FAIL (expected: Should panic, but doesn't) (0.06s) + Compiling and running oracle program (0.06s) + test/Test.hs:109: + failed to compile and run + (expected failure) + bool: OK (0.13s) + Compiling and running oracle program (0.13s) + Oracle output: false + Crux output: false + ge: OK (0.13s) + Compiling and running oracle program (0.13s) + Oracle output: true + Crux output: true + shift1: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 2 + Crux output: 2 + lit: FAIL (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: false (0.01s) + Crux output: + failures: + + ---- lit/85c762e8::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in lit/85c762e8::crux_test[0] + [Crux] Translation error in lit/85c762e8::crux_test[0]: fail or unimp constant: TyTuple [TyBool,TyBool] (StructRepr [MaybeRepr BoolRepr, MaybeRepr BoolRepr]) ConstTuple [ConstBool True,ConstBool True] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '$0=="crux-mir.crux concrete..prim.lit"' to rerun this test only. + litbstring: OK (0.15s) + Compiling and running oracle program (0.15s) + Oracle output: true + Crux output: true + mut_arg: OK (0.14s) + Compiling and running oracle program (0.13s) + Oracle output: 1 + Crux output: 1 + shift4: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: -9223372036854775808 + Crux output: -9223372036854775808 + mut: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 14 + Crux output: 14 + fnptr + make: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 0 + Crux output: 0 + call: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 2 + Crux output: 2 + custom: rustc compilation failed for custom +error output: +error[E0432]: unresolved import `core` + --> test/conc_eval/fnptr/custom.rs:2:5 + | +2 | use core::mem; + | ^^^^ maybe a missing crate `core`? + | + = help: consider adding `extern crate core` to use the `core` crate + +error: aborting due to previous error + +For more information about this error, try `rustc --explain E0432`. + +FAIL (expected: taking address of an overridden function) (0.05s) + Compiling and running oracle program (0.05s) + test/Test.hs:109: + failed to compile and run + (expected failure) + field: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 2 + Crux output: 2 + mem + maybe_uninit_array_cast: FAIL (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: [1, 2] (0.01s) + Crux output: + failures: + + ---- maybe_uninit_array_cast/efcbcf14::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:487:43: 487:48: error: in core/516ae8d9::ptr[0]::mut_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/516ae8d9::ptr[0]::mut_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::offset[0]::_inst1e2825177cd3b608[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/maybe_uninit_array_cast/' to rerun this test only. + maybe_uninit: FAIL (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (0.01s) + Crux output: + failures: + + ---- maybe_uninit/cceee461::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in core/516ae8d9::mem[0]::maybe_uninit[0]::{impl#2}[0]::assume_init[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/516ae8d9::mem[0]::maybe_uninit[0]::{impl#2}[0]::assume_init[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::assert_inhabited[0]::_inst1e2825177cd3b608[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '$0=="crux-mir.crux concrete..mem.maybe_uninit"' to rerun this test only. + consts + local_key: FAIL (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (0.01s) + Crux output: + failures: + + ---- local_key/c72de59a::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/consts/local_key.rs:22:78: 22:93: error: in local_key/c72de59a::crux_test[0] + [Crux] Translation error in local_key/c72de59a::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/local_key/' to rerun this test only. + struct_val: OK (0.14s) + Compiling and running oracle program (0.13s) + Oracle output: Foo { x: true } + Crux output: Foo { x: true } + fn_def: FAIL (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 1 + user error (JSON Decoding of test/conc_eval/consts/fn_def.all.mir failed: Error in $.fns[0].body.blocks[0].block.terminator.func.data.rendered: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) + + Use -p '/fn_def/' to rerun this test only. + enum_val: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: None + Crux output: None + struct_unit: FAIL (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: Err(Foo { x: () }) (0.01s) + Crux output: + failures: + + ---- struct_unit/52e506ef::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in struct_unit/52e506ef::crux_test[0] + [Crux] Translation error in struct_unit/52e506ef::crux_test[0]: error building variant core/516ae8d9::result[0]::Result[0]::Err[0]: not enough expressions -- [] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/struct_unit/' to rerun this test only. + refs + imm_raw: OK (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: 123 + Crux output: 123 + never_mut: rustc compilation failed for never_mut +error output: +error[E0554]: `#![feature]` may not be used on the stable release channel + --> test/conc_eval/refs/never_mut.rs:1:1 + | +1 | #![feature(never_type)] + | ^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unreachable expression + --> test/conc_eval/refs/never_mut.rs:9:21 + | +9 | let r = &mut e; + | ^^^^^- + | | | + | | any code following this expression is unreachable + | unreachable expression + | + = note: `#[warn(unreachable_code)]` on by default + +error: aborting due to previous error; 1 warning emitted + +For more information about this error, try `rustc --explain E0554`. + +FAIL (0.05s) + Compiling and running oracle program (0.05s) + test/Test.hs:109: + failed to compile and run + + Use -p '/never_mut/' to rerun this test only. + promoted_mut: FAIL (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (0.01s) + Crux output: + failures: + + ---- promoted_mut/4c6f527f::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/refs/promoted_mut.rs:4:27: 4:34: error: in promoted_mut/4c6f527f::crux_test[0] + [Crux] Translation error in promoted_mut/4c6f527f::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/promoted_mut/' to rerun this test only. + mut_nested: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: () + Crux output: () + static_mut: OK (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: 2 + Crux output: 2 + never: rustc compilation failed for never +error output: +error[E0554]: `#![feature]` may not be used on the stable release channel + --> test/conc_eval/refs/never.rs:1:1 + | +1 | #![feature(never_type)] + | ^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unreachable expression + --> test/conc_eval/refs/never.rs:9:21 + | +9 | let r = &e; + | ^- + | || + | |any code following this expression is unreachable + | unreachable expression + | + = note: `#[warn(unreachable_code)]` on by default + +error: aborting due to previous error; 1 warning emitted + +For more information about this error, try `rustc --explain E0554`. + +FAIL (0.05s) + Compiling and running oracle program (0.05s) + test/Test.hs:109: + failed to compile and run + + Use -p '$0=="crux-mir.crux concrete..refs.never"' to rerun this test only. + temp: FAIL (0.15s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.01s) + Crux output: + failures: + + ---- temp/d2b20cfd::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/refs/temp.rs:4:13: 4:16: error: in temp/d2b20cfd::crux_test[0] + [Crux] Translation error in temp/d2b20cfd::crux_test[0]: cannot find static variable alloc$1/3a1fbbbh + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/temp/' to rerun this test only. + fn_ptr: rustc compilation failed for fn_ptr +error output: +error[E0554]: `#![feature]` may not be used on the stable release channel + --> test/conc_eval/refs/fn_ptr.rs:1:1 + | +1 | #![feature(never_type)] + | ^^^^^^^^^^^^^^^^^^^^^^^ + +error: aborting due to previous error + +For more information about this error, try `rustc --explain E0554`. + +FAIL (0.05s) + Compiling and running oracle program (0.05s) + test/Test.hs:109: + failed to compile and run + + Use -p '$0=="crux-mir.crux concrete..refs.fn_ptr"' to rerun this test only. + fn_ptr_mut: rustc compilation failed for fn_ptr_mut +error output: +error[E0554]: `#![feature]` may not be used on the stable release channel + --> test/conc_eval/refs/fn_ptr_mut.rs:1:1 + | +1 | #![feature(never_type)] + | ^^^^^^^^^^^^^^^^^^^^^^^ + +error: aborting due to previous error + +For more information about this error, try `rustc --explain E0554`. + +FAIL (0.05s) + Compiling and running oracle program (0.05s) + test/Test.hs:109: + failed to compile and run + + Use -p '/fn_ptr_mut/' to rerun this test only. + promoted_imm: OK (0.15s) + Compiling and running oracle program (0.15s) + Oracle output: 0 + Crux output: 0 + imm_arg: OK (0.15s) + Compiling and running oracle program (0.15s) + Oracle output: 1 + Crux output: 1 + mut_tuple_field: FAIL (0.17s) + Compiling and running oracle program (0.15s) + Oracle output: () (0.02s) + Crux output: + failures: + + ---- mut_tuple_field/78fb59fb::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/refs/mut_tuple_field.rs:10:19: 10:25: error: in mut_tuple_field/78fb59fb::f[0] + [Crux] Translation error in mut_tuple_field/78fb59fb::f[0]: cannot find static variable alloc$0/3a1fbbbh + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/mut_tuple_field/' to rerun this test only. + mut_raw: OK (0.15s) + Compiling and running oracle program (0.15s) + Oracle output: 123 + Crux output: 123 + imm_ref: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: 123 + Crux output: 123 + mut_ref: OK (0.15s) + Compiling and running oracle program (0.15s) + Oracle output: 1 + Crux output: 1 + mut_arg: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 1 + Crux output: 1 + time + instant: FAIL (0.91s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.77s) + standalone use of `dyn` is not supported: TyDynamic core/516ae8d9::error[0]::Error[0]::_trait56353a7b840403b4[0] + CallStack (from HasCallStack): + error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy + tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate + translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language + + Use -p '/instant/' to rerun this test only. + impl + simple: OK (0.15s) + Compiling and running oracle program (0.15s) + Oracle output: 42 + Crux output: 42 + self_mut: OK (0.15s) + Compiling and running oracle program (0.15s) + Oracle output: 42 + Crux output: 42 + self: OK (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: 42 + Crux output: 42 + vec_deque + iter_clone: FAIL (0.79s) + Compiling and running oracle program (0.17s) + Oracle output: [1, 2, 3, 2, 3] (0.62s) + Crux output: + failures: + + ---- iter_clone/93806b96::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/iter_clone/' to rerun this test only. + pop: FAIL (0.70s) + Compiling and running oracle program (0.16s) + Oracle output: [5, 4, 3, 1, 2] (0.54s) + Crux output: + failures: + + ---- pop/d98f5c82::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/vec_deque.pop/' to rerun this test only. + push: FAIL (0.76s) + Compiling and running oracle program (0.15s) + Oracle output: [4, 1, 2, 3, 5] (0.61s) + Crux output: + failures: + + ---- push/82000b70::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/vec_deque.push/' to rerun this test only. + retain: FAIL (0.77s) + Compiling and running oracle program (0.16s) + Oracle output: [1, 4] (0.60s) + Crux output: + failures: + + ---- retain/37c5dc3c::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/retain/' to rerun this test only. + rotate_left: FAIL (0.74s) + Compiling and running oracle program (0.16s) + Oracle output: [3, 4, 5, 1, 2] (0.59s) + Crux output: + failures: + + ---- rotate_left/a036e026::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/rotate_left/' to rerun this test only. + rotate_right: FAIL (0.81s) + Compiling and running oracle program (0.16s) + Oracle output: [4, 5, 1, 2, 3] (0.65s) + Crux output: + failures: + + ---- rotate_right/605564b5::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/rotate_right/' to rerun this test only. + ops + deref3: OK (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.01s) + Crux output: () + index3: OK (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: () + Crux output: () + deref2: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: () + Crux output: () + deref1: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: () + Crux output: () + index2: OK (0.14s) + Compiling and running oracle program (0.13s) + Oracle output: () + Crux output: () + arith1: OK (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.01s) + Crux output: () + index1: OK (0.13s) + Compiling and running oracle program (0.13s) + Oracle output: () + Crux output: () + num + saturate: FAIL (0.61s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.47s) + Crux output: + failures: + + ---- saturate/46c7dca9::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/saturate/' to rerun this test only. + overflow: FAIL (0.17s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.02s) + Crux output: + failures: + + ---- overflow/d97d6804::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/num/overflow.rs:5:40: 5:50: error: in overflow/d97d6804::crux_test[0] + [Crux] Translation error in overflow/d97d6804::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/num.overflow/' to rerun this test only. + from_bytes: FAIL (0.15s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.02s) + Crux output: + failures: + + ---- from_bytes/c358d064::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:2417:37: 2417:42 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:890:5: 891:49: error: in core/516ae8d9::num[0]::{impl#8}[0]::from_ne_bytes[0] + [Crux] Translation error in core/516ae8d9::num[0]::{impl#8}[0]::from_ne_bytes[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::transmute[0]::_inst850c595be4adc72f[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/from_bytes/' to rerun this test only. + crypto + add: FAIL (0.19s) + Compiling and running oracle program (0.15s) + Oracle output: true (0.04s) + Crux output: + failures: + + ---- add/5176c56c::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/num/uint_macros.rs:470:54: 470:57 !lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::unchecked_add[0] + [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::unchecked_add[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_instaffa7a8b1157c078[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '$0=="crux-mir.crux concrete..crypto.add"' to rerun this test only. + add_noL: FAIL (0.18s) + Compiling and running oracle program (0.15s) + Oracle output: false (0.03s) + Crux output: + failures: + + ---- add_noL/60d8b3f2::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/num/uint_macros.rs:470:54: 470:57 !lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::unchecked_add[0] + [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::unchecked_add[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_instaffa7a8b1157c078[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/add_noL/' to rerun this test only. + traits + bounds2: FAIL (0.16s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.02s) + Crux output: + failures: + + ---- bounds2/1e0bbf28::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in bounds2/1e0bbf28::f[0] + [Crux] Translation error in bounds2/1e0bbf28::f[0]: error building variant core/516ae8d9::option[0]::Option[0]::Some[0]: not enough expressions -- [] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/bounds2/' to rerun this test only. + tyfam: OK (0.15s) + Compiling and running oracle program (0.15s) + Oracle output: 23 + Crux output: 23 + dict_poly: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 1 + Crux output: 1 + dynamic_simple: OK (0.14s) + Compiling and running oracle program (0.13s) + Oracle output: 1 + Crux output: 1 + assoc1: OK (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: () + Crux output: () + static_two: OK (0.13s) + Compiling and running oracle program (0.13s) + Oracle output: 1 + Crux output: 1 + static_self: OK (0.13s) + Compiling and running oracle program (0.13s) + Oracle output: 42 + Crux output: 42 + generic2: FAIL (0.16s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.02s) + Crux output: + failures: + + ---- generic2/02c07c57::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in generic2/02c07c57::f[0] + [Crux] Translation error in generic2/02c07c57::f[0]: error building variant core/516ae8d9::option[0]::Option[0]::Some[0]: not enough expressions -- [] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/generic2/' to rerun this test only. + bounds1: FAIL (0.17s) + Compiling and running oracle program (0.15s) + Oracle output: () (0.02s) + Crux output: + failures: + + ---- bounds1/fab02eaf::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in bounds1/fab02eaf::f[0] + [Crux] Translation error in bounds1/fab02eaf::f[0]: error building variant core/516ae8d9::option[0]::Option[0]::Some[0]: not enough expressions -- [] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/bounds1/' to rerun this test only. + gen_trait_poly: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 12 + Crux output: 12 + dynamic_poly: FAIL (0.17s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (0.03s) + user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_poly.rs) + + Use -p '/dynamic_poly/' to rerun this test only. + static: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 42 + Crux output: 42 + tyfam4: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 0 + Crux output: 0 + assoc2: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: () + Crux output: () + basics1: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: () + Crux output: () + gen_trait: OK (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: -69 + Crux output: -69 + assoc3: OK (0.16s) + Compiling and running oracle program (0.15s) + Oracle output: () + Crux output: () + default: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 12 + Crux output: 12 + bounds4: FAIL (0.16s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.02s) + Crux output: + failures: + + ---- bounds4/7f80af9e::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in bounds4/7f80af9e::f[0] + [Crux] Translation error in bounds4/7f80af9e::f[0]: error building variant core/516ae8d9::option[0]::Option[0]::Some[0]: not enough expressions -- [] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/bounds4/' to rerun this test only. + intoiter: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 0 + Crux output: 0 + tyfam5: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: () + Crux output: () + dict_polymem: OK (0.13s) + Compiling and running oracle program (0.13s) + Oracle output: 4 + Crux output: 4 + dynamic_branch: FAIL (0.16s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.02s) + user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_branch.rs) + + Use -p '/dynamic_branch/' to rerun this test only. + subtrait: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 73 + Crux output: 73 + params: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 25 + Crux output: 25 + conv: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 1 + Crux output: 1 + bounds5: FAIL (0.17s) + Compiling and running oracle program (0.15s) + Oracle output: () (0.02s) + Crux output: + failures: + + ---- bounds5/3b3a5fb2::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in bounds5/3b3a5fb2::f[0] + [Crux] Translation error in bounds5/3b3a5fb2::f[0]: error building variant core/516ae8d9::option[0]::Option[0]::Some[0]: not enough expressions -- [] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/bounds5/' to rerun this test only. + dict_simple: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 32 + Crux output: 32 + subtrait2: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 64 + Crux output: 64 + generic3: FAIL (0.16s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.02s) + Crux output: + failures: + + ---- generic3/256bb722::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in generic3/256bb722::f[0] + [Crux] Translation error in generic3/256bb722::f[0]: error building variant core/516ae8d9::option[0]::Option[0]::Some[0]: not enough expressions -- [] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/generic3/' to rerun this test only. + bounds3: FAIL (0.16s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.02s) + Crux output: + failures: + + ---- bounds3/448ffaab::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in bounds3/448ffaab::f[0] + [Crux] Translation error in bounds3/448ffaab::f[0]: error building variant core/516ae8d9::option[0]::Option[0]::Some[0]: not enough expressions -- [] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/bounds3/' to rerun this test only. + static_eq: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: true + Crux output: true + static_three: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 2 + Crux output: 2 + tyfam2: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 23 + Crux output: 23 + dict_med: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 42 + Crux output: 42 + tyfam3: OK (0.15s) + Compiling and running oracle program (0.15s) + Oracle output: 23 + Crux output: 23 + generic1: FAIL (0.16s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.02s) + Crux output: + failures: + + ---- generic1/d2b80067::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in generic1/d2b80067::f[0] + [Crux] Translation error in generic1/d2b80067::f[0]: error building variant core/516ae8d9::option[0]::Option[0]::Some[0]: not enough expressions -- [] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/generic1/' to rerun this test only. + dynamic_med: FAIL (0.16s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (0.02s) + user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_med.rs) + + Use -p '/dynamic_med/' to rerun this test only. + dynamic_two: FAIL (0.16s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (0.02s) + user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_two.rs) + + Use -p '/dynamic_two/' to rerun this test only. + io + cursor_read: FAIL (0.60s) + Compiling and running oracle program (0.16s) + Oracle output: 0 (0.44s) + standalone use of `dyn` is not supported: TyDynamic core/516ae8d9::error[0]::Error[0]::_trait56353a7b840403b4[0] + CallStack (from HasCallStack): + error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy + tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate + translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language + + Use -p '/io.cursor_read/' to rerun this test only. + cursor_write2: FAIL (0.92s) + Compiling and running oracle program (0.17s) + Oracle output: 0 (0.76s) + standalone use of `dyn` is not supported: TyDynamic core/516ae8d9::error[0]::Error[0]::_trait56353a7b840403b4[0] + CallStack (from HasCallStack): + error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy + tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate + translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language + + Use -p '/cursor_write2/' to rerun this test only. + cursor_write: FAIL (0.60s) + Compiling and running oracle program (0.16s) + Oracle output: 0 (0.43s) + standalone use of `dyn` is not supported: TyDynamic core/516ae8d9::error[0]::Error[0]::_trait56353a7b840403b4[0] + CallStack (from HasCallStack): + error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy + tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate + translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language + + Use -p '$0=="crux-mir.crux concrete..io.cursor_write"' to rerun this test only. + statics + promoted_static: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 1 + Crux output: 1 + promoted_fn: FAIL (0.16s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (0.01s) + Crux output: + failures: + + ---- promoted_fn/e00ec2eb::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/statics/promoted_fn.rs:4:13: 4:16: error: in promoted_fn/e00ec2eb::crux_test[0] + [Crux] Translation error in promoted_fn/e00ec2eb::crux_test[0]: cannot find static variable alloc$1/3a1fbbbh + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/promoted_fn/' to rerun this test only. + struct + repr_transparent: FAIL (0.17s) + Compiling and running oracle program (0.14s) + Oracle output: 6 (0.02s) + Crux output: + failures: + + ---- repr_transparent/444f9e7e::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:61:9: 61:18: error: in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0] + [Crux] Translation error in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyInt B32) Immut + [Crux] as: TyRawPtr (TyArray (TyInt B32) 2) Immut + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '$0=="crux-mir.crux concrete..struct.repr_transparent"' to rerun this test only. + repr_transparent_const: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 124 + Crux output: 124 + field_order: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: () + Crux output: () + arg: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 42 + Crux output: 42 + proj: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: 42 + Crux output: 42 + ret: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: S { x: 42, y: 120 } + Crux output: S { x: 42, y: 120 } + tup: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: () + Crux output: () + ptr + valid_eq: OK (0.14s) + Compiling and running oracle program (0.14s) + Oracle output: () + Crux output: () + cast_eq: OK (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: () + Crux output: () + read_write: OK (0.16s) + Compiling and running oracle program (0.14s) + Oracle output: [1, 3, 1] (0.02s) + Crux output: [1, 3, 1] + coerce_unsized: rustc compilation failed for coerce_unsized +error output: +warning: unused import: `std::ptr` + --> test/conc_eval/ptr/coerce_unsized.rs:5:5 + | +5 | use std::ptr; + | ^^^^^^^^ + | + = note: `#[warn(unused_imports)]` on by default + +error[E0554]: `#![feature]` may not be used on the stable release channel + --> test/conc_eval/ptr/coerce_unsized.rs:1:12 + | +1 | #![feature(unsize)] + | ^^^^^^ + +error[E0554]: `#![feature]` may not be used on the stable release channel + --> test/conc_eval/ptr/coerce_unsized.rs:2:12 + | +2 | #![feature(coerce_unsized)] + | ^^^^^^^^^^^^^^ + +error: aborting due to 2 previous errors; 1 warning emitted + +For more information about this error, try `rustc --explain E0554`. + +FAIL (0.05s) + Compiling and running oracle program (0.05s) + test/Test.hs:109: + failed to compile and run + + Use -p '/coerce_unsized/' to rerun this test only. + null_eq: FAIL (0.15s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.02s) + Crux output: + failures: + + ---- null_eq/fe6c1a4e::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mod.rs:568:29: 568:33: error: in core/516ae8d9::ptr[0]::invalid[0]::_instf59dc84daa91a404[0] + [Crux] Translation error in core/516ae8d9::ptr[0]::invalid[0]::_instf59dc84daa91a404[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::transmute[0]::_insta5d0a99e3b08f3bd[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/null_eq/' to rerun this test only. + offset_from: rustc compilation failed for offset_from +error output: +warning: unused import: `std::ptr` + --> test/conc_eval/ptr/offset_from.rs:2:5 + | +2 | use std::ptr; + | ^^^^^^^^ + | + = note: `#[warn(unused_imports)]` on by default + +error[E0554]: `#![feature]` may not be used on the stable release channel + --> test/conc_eval/ptr/offset_from.rs:1:12 + | +1 | #![feature(ptr_offset_from)] + | ^^^^^^^^^^^^^^^ + +warning: the feature `ptr_offset_from` has been stable since 1.47.0 and no longer requires an attribute to enable + --> test/conc_eval/ptr/offset_from.rs:1:12 + | +1 | #![feature(ptr_offset_from)] + | ^^^^^^^^^^^^^^^ + | + = note: `#[warn(stable_features)]` on by default + +error: aborting due to previous error; 2 warnings emitted + +For more information about this error, try `rustc --explain E0554`. + +FAIL (0.05s) + Compiling and running oracle program (0.05s) + test/Test.hs:109: + failed to compile and run + + Use -p '/offset_from/' to rerun this test only. + is_null: FAIL (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: (false, true) (0.01s) + Crux output: + failures: + + ---- is_null/dda8ee76::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/ptr/is_null.rs:5:13: 5:15: error: in is_null/dda8ee76::crux_test[0] + [Crux] Translation error in is_null/dda8ee76::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '$0=="crux-mir.crux concrete..ptr.is_null"' to rerun this test only. + dangling_eq: FAIL (0.16s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.03s) + Crux output: + failures: + + ---- dangling_eq/7f6bbb93::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mod.rs:599:29: 599:33: error: in core/516ae8d9::ptr[0]::invalid_mut[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/516ae8d9::ptr[0]::invalid_mut[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::transmute[0]::_inst102627303ebf2823[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/dangling_eq/' to rerun this test only. + offset: FAIL (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (0.01s) + Crux output: + failures: + + ---- offset/e803b869::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:473:43: 473:48: error: in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::offset[0]::_inst1e2825177cd3b608[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '$0=="crux-mir.crux concrete..ptr.offset"' to rerun this test only. + offset_mut: FAIL (0.17s) + Compiling and running oracle program (0.16s) + Oracle output: 3 (0.01s) + Crux output: + failures: + + ---- offset_mut/3d54e912::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:487:43: 487:48: error: in core/516ae8d9::ptr[0]::mut_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/516ae8d9::ptr[0]::mut_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::offset[0]::_inst1e2825177cd3b608[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/offset_mut/' to rerun this test only. + unsize_slice: FAIL (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: (1, 2) (0.01s) + Crux output: + failures: + + ---- unsize_slice/2118c582::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:473:43: 473:48: error: in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::offset[0]::_inst1e2825177cd3b608[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + + Use -p '/unsize_slice/' to rerun this test only. + struct_eq: OK (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: () + Crux output: () + copy: OK (0.17s) + Compiling and running oracle program (0.14s) + Oracle output: [1, 2, 3, 1, 2, 3] (0.02s) + Crux output: [1, 2, 3, 1, 2, 3] + is_null_slice: FAIL (expected: can't unsize null pointers) (0.15s) + Compiling and running oracle program (0.14s) + Oracle output: (false, true) (0.01s) + Crux output: + failures: + + ---- is_null_slice/42710a6d::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/ptr/is_null_slice.rs:6:13: 6:20: error: in is_null_slice/42710a6d::crux_test[0] + [Crux] Translation error in is_null_slice/42710a6d::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + + [Crux] Overall status: Invalid. + + test/Test.hs:126: + crux doesn't match oracle + (expected failure) + crux symbolic + Output testing + sort_by_key: FAIL (0.72s) + files test/symb_eval/vec/sort_by_key.good and test/symb_eval/vec/sort_by_key.out differ; test/symb_eval/vec/sort_by_key.out contains: + test sort_by_key/1875b66b::f[0]: FAILED + + failures: + + ---- sort_by_key/1875b66b::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/sort_by_key/' to rerun this test only. + clone: FAIL (0.14s) + files test/symb_eval/vec/clone.good and test/symb_eval/vec/clone.out differ; test/symb_eval/vec/clone.out contains: + test clone/6b1f0898::f[0]: FAILED + + failures: + + ---- clone/6b1f0898::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + + [Crux] Overall status: Invalid. + + Use -p '/Output testing.clone/' to rerun this test only. + into_iter: FAIL (0.71s) + files test/symb_eval/vec/into_iter.good and test/symb_eval/vec/into_iter.out differ; test/symb_eval/vec/into_iter.out contains: + test into_iter/65317545::f[0]: FAILED + + failures: + + ---- into_iter/65317545::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/into_iter/' to rerun this test only. + macro: FAIL (0.11s) + files test/symb_eval/vec/macro.good and test/symb_eval/vec/macro.out differ; test/symb_eval/vec/macro.out contains: + test macro/2d0c4c9f::f[0]: FAILED + + failures: + + ---- macro/2d0c4c9f::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + + [Crux] Overall status: Invalid. + + Use -p '/Output testing.macro/' to rerun this test only. + override4: FAIL (0.10s) + files test/symb_eval/overrides/override4.good and test/symb_eval/overrides/override4.out differ; test/symb_eval/overrides/override4.out contains: + test override4/6b6c1ae4::f[0]: FAILED + + failures: + + ---- override4/6b6c1ae4::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/override4/' to rerun this test only. + bad_symb2: FAIL (0.10s) + files test/symb_eval/overrides/bad_symb2.good and test/symb_eval/overrides/bad_symb2.out differ; test/symb_eval/overrides/bad_symb2.out contains: + test bad_symb2/41a155d5::crux_test[0]: FAILED + + failures: + + ---- bad_symb2/41a155d5::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/bad_symb2/' to rerun this test only. + override3: FAIL (0.10s) + files test/symb_eval/overrides/override3.good and test/symb_eval/overrides/override3.out differ; test/symb_eval/overrides/override3.out contains: + test override3/2b4100d1::f[0]: FAILED + + failures: + + ---- override3/2b4100d1::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/override3/' to rerun this test only. + override2: FAIL (0.10s) + files test/symb_eval/overrides/override2.good and test/symb_eval/overrides/override2.out differ; test/symb_eval/overrides/override2.out contains: + test override2/4c9e2a25::f[0]: FAILED + + failures: + + ---- override2/4c9e2a25::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/override2/' to rerun this test only. + bad_symb1: FAIL (0.10s) + files test/symb_eval/overrides/bad_symb1.good and test/symb_eval/overrides/bad_symb1.out differ; test/symb_eval/overrides/bad_symb1.out contains: + test bad_symb1/8fcb2fc3::crux_test[0]: FAILED + + failures: + + ---- bad_symb1/8fcb2fc3::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/bad_symb1/' to rerun this test only. + override5: FAIL (0.11s) + files test/symb_eval/overrides/override5.good and test/symb_eval/overrides/override5.out differ; test/symb_eval/overrides/override5.out contains: + test override5/3995a1d6::f[0]: FAILED + + failures: + + ---- override5/3995a1d6::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/override5/' to rerun this test only. + override1: FAIL (0.02s) + files test/symb_eval/overrides/override1.good and test/symb_eval/overrides/override1.out differ; test/symb_eval/overrides/override1.out contains: + test override1/dff68baf::f[0]: FAILED + + failures: + + ---- override1/dff68baf::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/897c723b::one[0] + [Crux] panicking::panic, called from crucible/897c723b::one[0] + + [Crux] Overall status: Invalid. + + Use -p '/override1/' to rerun this test only. + override_rust: FAIL (0.09s) + files test/symb_eval/overrides/override_rust.good and test/symb_eval/overrides/override_rust.out differ; test/symb_eval/overrides/override_rust.out contains: + test override_rust/2fce5978::crux_test[0]: FAILED + + failures: + + ---- override_rust/2fce5978::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/override_rust/' to rerun this test only. + read: FAIL (0.05s) + test/symb_eval/byteorder/read.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/read.rs) + Use -p '/Output testing.read/' to rerun this test only. + write: FAIL + test/symb_eval/byteorder/write.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/write.rs) + Use -p '/Output testing.write/' to rerun this test only. + mux: FAIL (0.10s) + files test/symb_eval/enum/mux.good and test/symb_eval/enum/mux.out differ; test/symb_eval/enum/mux.out contains: + test mux/cef3218c::test[0]: FAILED + + failures: + + ---- mux/cef3218c::test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.mux"' to rerun this test only. + split_off: FAIL (0.51s) + files test/symb_eval/bytes/split_off.good and test/symb_eval/bytes/split_off.out differ; test/symb_eval/bytes/split_off.out contains: + test split_off/6d8078c7::f[0]: FAILED + + failures: + + ---- split_off/6d8078c7::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/split_off/' to rerun this test only. + new: FAIL (0.08s) + files test/symb_eval/bytes/new.good and test/symb_eval/bytes/new.out differ; test/symb_eval/bytes/new.out contains: + test new/197434bb::f[0]: FAILED + + failures: + + ---- new/197434bb::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.new"' to rerun this test only. + split_to: FAIL (0.51s) + files test/symb_eval/bytes/split_to.good and test/symb_eval/bytes/split_to.out differ; test/symb_eval/bytes/split_to.out contains: + test split_to/d69c0d48::f[0]: FAILED + + failures: + + ---- split_to/d69c0d48::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/split_to/' to rerun this test only. + put_overflow: FAIL (0.47s) + files test/symb_eval/bytes/put_overflow.good and test/symb_eval/bytes/put_overflow.out differ; test/symb_eval/bytes/put_overflow.out contains: + test put_overflow/b706e629::f[0]: FAILED + + failures: + + ---- put_overflow/b706e629::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/put_overflow/' to rerun this test only. + sym_len: FAIL (0.48s) + files test/symb_eval/bytes/sym_len.good and test/symb_eval/bytes/sym_len.out differ; test/symb_eval/bytes/sym_len.out contains: + test sym_len/4ff12171::f[0]: FAILED + + failures: + + ---- sym_len/4ff12171::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/sym_len/' to rerun this test only. + put: FAIL (0.48s) + files test/symb_eval/bytes/put.good and test/symb_eval/bytes/put.out differ; test/symb_eval/bytes/put.out contains: + test put/7b1c2d40::f[0]: FAILED + + failures: + + ---- put/7b1c2d40::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.put"' to rerun this test only. + extend_bytes: FAIL (0.11s) + files test/symb_eval/bytes/extend_bytes.good and test/symb_eval/bytes/extend_bytes.out differ; test/symb_eval/bytes/extend_bytes.out contains: + test extend_bytes/c2da7e0b::f[0]: FAILED + + failures: + + ---- extend_bytes/c2da7e0b::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/extend_bytes/' to rerun this test only. + basic: FAIL (0.11s) + files test/symb_eval/array/basic.good and test/symb_eval/array/basic.out differ; test/symb_eval/array/basic.out contains: + test basic/35006054::crux_test[0]: FAILED + + failures: + + ---- basic/35006054::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/Output testing.basic/' to rerun this test only. + slice_mut: FAIL (0.10s) + files test/symb_eval/array/slice_mut.good and test/symb_eval/array/slice_mut.out differ; test/symb_eval/array/slice_mut.out contains: + test slice_mut/cd20ad7b::crux_test[0]: FAILED + + failures: + + ---- slice_mut/cd20ad7b::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/slice_mut/' to rerun this test only. + mux_slice: FAIL (0.13s) + files test/symb_eval/array/mux_slice.good and test/symb_eval/array/mux_slice.out differ; test/symb_eval/array/mux_slice.out contains: + test mux_slice/0ab1dfe8::crux_test[0]: FAILED + + failures: + + ---- mux_slice/0ab1dfe8::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/mux_slice/' to rerun this test only. + slice: FAIL (0.13s) + files test/symb_eval/array/slice.good and test/symb_eval/array/slice.out differ; test/symb_eval/array/slice.out contains: + test slice/031b990c::crux_test[0]: FAILED + + failures: + + ---- slice/031b990c::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.slice"' to rerun this test only. + arith: FAIL (0.13s) + files test/symb_eval/bitvector/arith.good and test/symb_eval/bitvector/arith.out differ; test/symb_eval/bitvector/arith.out contains: + test arith/ccf3c161::crux_test[0]: FAILED + + failures: + + ---- arith/ccf3c161::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/Output testing.arith/' to rerun this test only. + literals: FAIL (0.02s) + files test/symb_eval/bitvector/literals.good and test/symb_eval/bitvector/literals.out differ; test/symb_eval/bitvector/literals.out contains: + test literals/abfb591a::crux_test[0]: FAILED + + failures: + + ---- literals/abfb591a::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/bitvector.rs:91:29: 91:30 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/bitvector.rs:125:1: 125:53: error: in crucible/897c723b::bitvector[0]::{impl#27}[0]::from[0]::_inst9ea2f935eda83dab[0] + [Crux] Translation error in crucible/897c723b::bitvector[0]::{impl#27}[0]::from[0]::_inst9ea2f935eda83dab[0]: BUG: invalid arguments to bv_convert: [$31: AnyRepr] + + [Crux] Overall status: Invalid. + + Use -p '/literals/' to rerun this test only. + cmp: FAIL (0.11s) + files test/symb_eval/bitvector/cmp.good and test/symb_eval/bitvector/cmp.out differ; test/symb_eval/bitvector/cmp.out contains: + test cmp/a1f7a6f5::crux_test[0]: FAILED + + failures: + + ---- cmp/a1f7a6f5::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/Output testing.cmp/' to rerun this test only. + overflowing_sub: FAIL (0.08s) + files test/symb_eval/bitvector/overflowing_sub.good and test/symb_eval/bitvector/overflowing_sub.out differ; test/symb_eval/bitvector/overflowing_sub.out contains: + test overflowing_sub/9173f58f::crux_test[0]: FAILED + + failures: + + ---- overflowing_sub/9173f58f::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/overflowing_sub/' to rerun this test only. + from_to: FAIL (0.02s) + files test/symb_eval/bitvector/from_to.good and test/symb_eval/bitvector/from_to.out differ; test/symb_eval/bitvector/from_to.out contains: + test from_to/caac82ba::crux_test[0]: FAILED + + failures: + + ---- from_to/caac82ba::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/bitvector.rs:103:29: 103:30 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/bitvector.rs:125:1: 125:53: error: in crucible/897c723b::bitvector[0]::{impl#21}[0]::from[0]::_inst9ea2f935eda83dab[0] + [Crux] Translation error in crucible/897c723b::bitvector[0]::{impl#21}[0]::from[0]::_inst9ea2f935eda83dab[0]: BUG: invalid arguments to bv_convert: [$27: BVRepr 64] + + [Crux] Overall status: Invalid. + + Use -p '/from_to/' to rerun this test only. + symbolic: FAIL (0.03s) + files test/symb_eval/bitvector/symbolic.good and test/symb_eval/bitvector/symbolic.out differ; test/symb_eval/bitvector/symbolic.out contains: + test symbolic/57c5e3e8::crux_test[0]: FAILED + + failures: + + ---- symbolic/57c5e3e8::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/897c723b::bitvector[0]::make_symbolic_256[0] + [Crux] panicking::panic, called from crucible/897c723b::bitvector[0]::make_symbolic_256[0] + + [Crux] Overall status: Invalid. + + Use -p '/Output testing.symbolic/' to rerun this test only. + leading_zeros: FAIL (0.12s) + files test/symb_eval/bitvector/leading_zeros.good and test/symb_eval/bitvector/leading_zeros.out differ; test/symb_eval/bitvector/leading_zeros.out contains: + test leading_zeros/8f264035::crux_test[0]: FAILED + + failures: + + ---- leading_zeros/8f264035::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/leading_zeros/' to rerun this test only. + zero_length: FAIL (0.10s) + files test/symb_eval/alloc/zero_length.good and test/symb_eval/alloc/zero_length.out differ; test/symb_eval/alloc/zero_length.out contains: + test zero_length/5fa7d318::crux_test[0]: FAILED + + failures: + + ---- zero_length/5fa7d318::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/zero_length/' to rerun this test only. + out_of_bounds: FAIL (0.11s) + files test/symb_eval/alloc/out_of_bounds.good and test/symb_eval/alloc/out_of_bounds.out differ; test/symb_eval/alloc/out_of_bounds.out contains: + test out_of_bounds/fba84cc3::crux_test[0]: FAILED + + failures: + + ---- out_of_bounds/fba84cc3::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/out_of_bounds/' to rerun this test only. + valid_read: FAIL (0.11s) + files test/symb_eval/alloc/valid_read.good and test/symb_eval/alloc/valid_read.out differ; test/symb_eval/alloc/valid_read.out contains: + test valid_read/111c217c::crux_test[0]: FAILED + + failures: + + ---- valid_read/111c217c::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/valid_read/' to rerun this test only. + uninit_read: FAIL (0.10s) + files test/symb_eval/alloc/uninit_read.good and test/symb_eval/alloc/uninit_read.out differ; test/symb_eval/alloc/uninit_read.out contains: + test uninit_read/a73b11f2::crux_test[0]: FAILED + + failures: + + ---- uninit_read/a73b11f2::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/uninit_read/' to rerun this test only. + assert_ok: FAIL (0.68s) + files test/symb_eval/concretize/assert_ok.good and test/symb_eval/concretize/assert_ok.out differ; test/symb_eval/concretize/assert_ok.out contains: + test assert_ok/d97cacde::crux_test[0]: FAILED + + failures: + + ---- assert_ok/d97cacde::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/assert_ok/' to rerun this test only. + array: FAIL (0.12s) + files test/symb_eval/concretize/array.good and test/symb_eval/concretize/array.out differ; test/symb_eval/concretize/array.out contains: + test array/08515081::crux_test[0]: FAILED + + failures: + + ---- array/08515081::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.array"' to rerun this test only. + assert: FAIL (0.66s) + files test/symb_eval/concretize/assert.good and test/symb_eval/concretize/assert.out differ; test/symb_eval/concretize/assert.out contains: + test assert/30ec92d4::crux_test[0]: FAILED + + failures: + + ---- assert/30ec92d4::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.assert"' to rerun this test only. + no_conc: FAIL (0.09s) + files test/symb_eval/concretize/no_conc.good and test/symb_eval/concretize/no_conc.out differ; test/symb_eval/concretize/no_conc.out contains: + test no_conc/73a9c3e5::crux_test[0]: FAILED + + failures: + + ---- no_conc/73a9c3e5::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/no_conc/' to rerun this test only. + conc: FAIL (0.09s) + files test/symb_eval/concretize/conc.good and test/symb_eval/concretize/conc.out differ; test/symb_eval/concretize/conc.out contains: + test conc/052a57bb::crux_test[0]: FAILED + + failures: + + ---- conc/052a57bb::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.conc"' to rerun this test only. + conditional: FAIL + test/symb_eval/any/conditional.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/any/conditional.rs) + Use -p '/conditional/' to rerun this test only. + downcast_fail: FAIL (0.09s) + files test/symb_eval/any/downcast_fail.good and test/symb_eval/any/downcast_fail.out differ; test/symb_eval/any/downcast_fail.out contains: + test downcast_fail/c51bb09a::crux_test[0]: FAILED + + failures: + + ---- downcast_fail/c51bb09a::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/downcast_fail/' to rerun this test only. + downcast: FAIL (0.09s) + files test/symb_eval/any/downcast.good and test/symb_eval/any/downcast.out differ; test/symb_eval/any/downcast.out contains: + test downcast/e8294f5a::crux_test[0]: FAILED + + failures: + + ---- downcast/e8294f5a::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.downcast"' to rerun this test only. + mux: FAIL (0.10s) + files test/symb_eval/fnptr/mux.good and test/symb_eval/fnptr/mux.out differ; test/symb_eval/fnptr/mux.out contains: + test mux/756a6435::crux_test[0]: FAILED + + failures: + + ---- mux/756a6435::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.mux"' to rerun this test only. + mux_init_mut: FAIL (0.10s) + files test/symb_eval/refs/mux_init_mut.good and test/symb_eval/refs/mux_init_mut.out differ; test/symb_eval/refs/mux_init_mut.out contains: + test mux_init_mut/4b29e475::crux_test[0]: FAILED + + failures: + + ---- mux_init_mut/4b29e475::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/mux_init_mut/' to rerun this test only. + mux_init_imm: FAIL (0.10s) + files test/symb_eval/refs/mux_init_imm.good and test/symb_eval/refs/mux_init_imm.out differ; test/symb_eval/refs/mux_init_imm.out contains: + test mux_init_imm/a8d4900d::crux_test[0]: FAILED + + failures: + + ---- mux_init_imm/a8d4900d::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/mux_init_imm/' to rerun this test only. + as_mut_slice: FAIL (0.10s) + files test/symb_eval/vector/as_mut_slice.good and test/symb_eval/vector/as_mut_slice.out differ; test/symb_eval/vector/as_mut_slice.out contains: + test as_mut_slice/393f23fa::f[0]: FAILED + + failures: + + ---- as_mut_slice/393f23fa::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/as_mut_slice/' to rerun this test only. + new: FAIL (0.09s) + files test/symb_eval/vector/new.good and test/symb_eval/vector/new.out differ; test/symb_eval/vector/new.out contains: + test new/d901c99d::f[0]: FAILED + + failures: + + ---- new/d901c99d::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.new"' to rerun this test only. + concat: FAIL (0.10s) + files test/symb_eval/vector/concat.good and test/symb_eval/vector/concat.out differ; test/symb_eval/vector/concat.out contains: + test concat/b4c814d1::f[0]: FAILED + + failures: + + ---- concat/b4c814d1::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/concat/' to rerun this test only. + split_at: FAIL (0.09s) + files test/symb_eval/vector/split_at.good and test/symb_eval/vector/split_at.out differ; test/symb_eval/vector/split_at.out contains: + test split_at/aa8f5aa9::f[0]: FAILED + + failures: + + ---- split_at/aa8f5aa9::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/split_at/' to rerun this test only. + as_slice: FAIL (0.10s) + files test/symb_eval/vector/as_slice.good and test/symb_eval/vector/as_slice.out differ; test/symb_eval/vector/as_slice.out contains: + test as_slice/125ec1c7::f[0]: FAILED + + failures: + + ---- as_slice/125ec1c7::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/as_slice/' to rerun this test only. + copy_from_slice: FAIL (0.11s) + files test/symb_eval/vector/copy_from_slice.good and test/symb_eval/vector/copy_from_slice.out differ; test/symb_eval/vector/copy_from_slice.out contains: + test copy_from_slice/57fc6028::f[0]: FAILED + + failures: + + ---- copy_from_slice/57fc6028::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/copy_from_slice/' to rerun this test only. + pop: FAIL (0.11s) + files test/symb_eval/vector/pop.good and test/symb_eval/vector/pop.out differ; test/symb_eval/vector/pop.out contains: + test pop/ebc7234e::f[0]: FAILED + + failures: + + ---- pop/ebc7234e::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/Output testing.pop/' to rerun this test only. + push: FAIL (0.10s) + files test/symb_eval/vector/push.good and test/symb_eval/vector/push.out differ; test/symb_eval/vector/push.out contains: + test push/0bce6253::f[0]: FAILED + + failures: + + ---- push/0bce6253::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/Output testing.push/' to rerun this test only. + mut: FAIL (0.10s) + files test/symb_eval/vector/mut.good and test/symb_eval/vector/mut.out differ; test/symb_eval/vector/mut.out contains: + test mut/10d04785::f[0]: FAILED + + failures: + + ---- mut/10d04785::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/Output testing.mut/' to rerun this test only. + replicate: FAIL (0.11s) + files test/symb_eval/vector/replicate.good and test/symb_eval/vector/replicate.out differ; test/symb_eval/vector/replicate.out contains: + test replicate/bf8f8227::f[0]: FAILED + + failures: + + ---- replicate/bf8f8227::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/replicate/' to rerun this test only. + multi: FAIL (0.12s) + files test/symb_eval/crux/multi.good and test/symb_eval/crux/multi.out differ; test/symb_eval/crux/multi.out contains: + test multi/05702274::fail1[0]: FAILED + test multi/05702274::fail2[0]: FAILED + test multi/05702274::fail3[0]: FAILED + + failures: + + ---- multi/05702274::fail1[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + ---- multi/05702274::fail2[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + ---- multi/05702274::fail3[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/Output testing.multi/' to rerun this test only. + early_fail: FAIL (0.08s) + files test/symb_eval/crux/early_fail.good and test/symb_eval/crux/early_fail.out differ; test/symb_eval/crux/early_fail.out contains: + test early_fail/c90fbdca::fail1[0]: FAILED + test early_fail/c90fbdca::fail2[0]: FAILED + + failures: + + ---- early_fail/c90fbdca::fail1[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + ---- early_fail/c90fbdca::fail2[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/early_fail/' to rerun this test only. + mixed_fail: FAIL (0.13s) + files test/symb_eval/crux/mixed_fail.good and test/symb_eval/crux/mixed_fail.out differ; test/symb_eval/crux/mixed_fail.out contains: + test mixed_fail/2843d7dd::fail1[0]: FAILED + test mixed_fail/2843d7dd::fail2[0]: FAILED + test mixed_fail/2843d7dd::pass1[0]: FAILED + test mixed_fail/2843d7dd::pass2[0]: FAILED + + failures: + + ---- mixed_fail/2843d7dd::fail1[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + ---- mixed_fail/2843d7dd::fail2[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + ---- mixed_fail/2843d7dd::pass1[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + ---- mixed_fail/2843d7dd::pass2[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/mixed_fail/' to rerun this test only. + fail_return: FAIL (0.09s) + files test/symb_eval/crux/fail_return.good and test/symb_eval/crux/fail_return.out differ; test/symb_eval/crux/fail_return.out contains: + test fail_return/384998af::fail1[0]: FAILED + test fail_return/384998af::fail2[0]: FAILED + + failures: + + ---- fail_return/384998af::fail1[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + ---- fail_return/384998af::fail2[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/fail_return/' to rerun this test only. + array: FAIL (0.11s) + files test/symb_eval/mux/array.good and test/symb_eval/mux/array.out differ; test/symb_eval/mux/array.out contains: + test array/e2517ba7::crux_test[0]: FAILED + + failures: + + ---- array/e2517ba7::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.array"' to rerun this test only. + array_mut: FAIL (0.10s) + files test/symb_eval/mux/array_mut.good and test/symb_eval/mux/array_mut.out differ; test/symb_eval/mux/array_mut.out contains: + test array_mut/f23954b5::crux_test[0]: FAILED + + failures: + + ---- array_mut/f23954b5::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/array_mut/' to rerun this test only. + checked_add: FAIL (0.02s) + files test/symb_eval/num/checked_add.good and test/symb_eval/num/checked_add.out differ; test/symb_eval/num/checked_add.out contains: + test checked_add/a1f7d9d2::crux_test[0]: returned 44, FAILED + + failures: + + ---- checked_add/a1f7d9d2::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/symb_eval/num/checked_add.rs:5:5: 5:12: error: in checked_add/a1f7d9d2::crux_test[0] + [Crux] attempt to compute `const 100_u8 + move _2`, which would overflow + + [Crux] Overall status: Invalid. + + Use -p '/checked_add/' to rerun this test only. + checked_div: FAIL (0.02s) + files test/symb_eval/num/checked_div.good and test/symb_eval/num/checked_div.out differ; test/symb_eval/num/checked_div.out contains: + test checked_div/cb81bc69::crux_test[0]: returned 65, FAILED + + failures: + + ---- checked_div/cb81bc69::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/cb81bc69::div_signed[0] + [Crux] attempt to compute `_3 / _4`, which would overflow + [Crux] Found counterexample for verification goal + [Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/cb81bc69::div_signed[0] + [Crux] attempt to divide `_3` by zero + [Crux] Found counterexample for verification goal + [Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/cb81bc69::div_unsigned[0] + [Crux] attempt to divide `_3` by zero + + [Crux] Overall status: Invalid. + + Use -p '/checked_div/' to rerun this test only. + checked_mul: FAIL (0.01s) + files test/symb_eval/num/checked_mul.good and test/symb_eval/num/checked_mul.out differ; test/symb_eval/num/checked_mul.out contains: + test checked_mul/98f17c38::crux_test[0]: returned 44, FAILED + + failures: + + ---- checked_mul/98f17c38::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/symb_eval/num/checked_mul.rs:4:5: 4:12: error: in checked_mul/98f17c38::crux_test[0] + [Crux] attempt to compute `const 100_u8 * move _2`, which would overflow + + [Crux] Overall status: Invalid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.checked_mul"' to rerun this test only. + checked_mul_signed: FAIL (0.02s) + files test/symb_eval/num/checked_mul_signed.good and test/symb_eval/num/checked_mul_signed.out differ; test/symb_eval/num/checked_mul_signed.out contains: + test checked_mul_signed/2b522f9c::crux_test[0]: returned 44, FAILED + + failures: + + ---- checked_mul_signed/2b522f9c::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/symb_eval/num/checked_mul_signed.rs:4:5: 4:13: error: in checked_mul_signed/2b522f9c::crux_test[0] + [Crux] attempt to compute `const -100_i8 * move _2`, which would overflow + + [Crux] Overall status: Invalid. + + Use -p '/checked_mul_signed/' to rerun this test only. + ffs: FAIL (0.13s) + files test/symb_eval/crypto/ffs.good and test/symb_eval/crypto/ffs.out differ; test/symb_eval/crypto/ffs.out contains: + test ffs/310e17b4::f[0]: FAILED + + failures: + + ---- ffs/310e17b4::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/Output testing.ffs/' to rerun this test only. + bytes2: FAIL (0.15s) + files test/symb_eval/crypto/bytes2.good and test/symb_eval/crypto/bytes2.out differ; test/symb_eval/crypto/bytes2.out contains: + test bytes2/315034bf::f[0]: FAILED + + failures: + + ---- bytes2/315034bf::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/bytes2/' to rerun this test only. + bytes: FAIL (0.14s) + files test/symb_eval/crypto/bytes.good and test/symb_eval/crypto/bytes.out differ; test/symb_eval/crypto/bytes.out contains: + test bytes/c88d09fa::f[0]: FAILED + + failures: + + ---- bytes/c88d09fa::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.bytes"' to rerun this test only. + double: FAIL (0.07s) + files test/symb_eval/crypto/double.good and test/symb_eval/crypto/double.out differ; test/symb_eval/crypto/double.out contains: + test double/15fd1104::f[0]: FAILED + + failures: + + ---- double/15fd1104::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/double/' to rerun this test only. + test1: FAIL (0.22s) + files test/symb_eval/scalar/test1.good and test/symb_eval/scalar/test1.out differ; test/symb_eval/scalar/test1.out contains: + test test1/c6233792::f[0]: FAILED + + failures: + + ---- test1/c6233792::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/test1/' to rerun this test only. + deserialize: FAIL (0.10s) + files test/symb_eval/sym_bytes/deserialize.good and test/symb_eval/sym_bytes/deserialize.out differ; test/symb_eval/sym_bytes/deserialize.out contains: + test deserialize/5d52ecde::crux_test[0]: FAILED + + failures: + + ---- deserialize/5d52ecde::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/deserialize/' to rerun this test only. + construct: FAIL (0.09s) + files test/symb_eval/sym_bytes/construct.good and test/symb_eval/sym_bytes/construct.out differ; test/symb_eval/sym_bytes/construct.out contains: + test construct/caa9861a::crux_test[0]: FAILED + + failures: + + ---- construct/caa9861a::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] + [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + Use -p '/construct/' to rerun this test only. + vec_cursor_read: FAIL (0.79s) + standalone use of `dyn` is not supported: TyDynamic core/516ae8d9::error[0]::Error[0]::_trait56353a7b840403b4[0] + CallStack (from HasCallStack): + error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy + tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate + translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language + Use -p '/vec_cursor_read/' to rerun this test only. + vec_write: FAIL (0.46s) + standalone use of `dyn` is not supported: TyDynamic core/516ae8d9::error[0]::Error[0]::_trait56353a7b840403b4[0] + CallStack (from HasCallStack): + error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy + tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans + transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate + translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language + Use -p '/vec_write/' to rerun this test only. + crux coverage + Output testing + coverage_loop: warning: trailing semicolon in macro used in expression position + --> src/main.rs:45:46 + | +45 | return Err(format!($($args)*).into()); + | ^ +... +309 | _ => die!("unknown tag {:?} for branch", tag), + | ---------------------------------------- in this macro invocation + | + = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default + = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! + = note: for more information, see issue #79813 + = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) + +FAIL (0.09s) + files test/coverage/coverage_loop.good and test/coverage/coverage_loop.out differ; test/coverage/coverage_loop.out contains: + warning: branch condition never has value false + ┌─ /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/iter/range.rs:621:12 + │ + 621 │ if self.start < self.end { + │ ^^^^^^^^^^^^^^^^^^^^^ + + + Use -p '/coverage_loop/' to rerun this test only. + coverage: warning: trailing semicolon in macro used in expression position + --> src/main.rs:45:46 + | +45 | return Err(format!($($args)*).into()); + | ^ +... +309 | _ => die!("unknown tag {:?} for branch", tag), + | ---------------------------------------- in this macro invocation + | + = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default + = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! + = note: for more information, see issue #79813 + = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) + +FAIL (0.16s) + files test/coverage/coverage.good and test/coverage/coverage.out differ; test/coverage/coverage.out contains: + + Use -p '$0=="crux-mir.crux coverage.Output testing.coverage"' to rerun this test only. + coverage_macro: warning: trailing semicolon in macro used in expression position + --> src/main.rs:45:46 + | +45 | return Err(format!($($args)*).into()); + | ^ +... +309 | _ => die!("unknown tag {:?} for branch", tag), + | ---------------------------------------- in this macro invocation + | + = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default + = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! + = note: for more information, see issue #79813 + = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) + +FAIL (0.16s) + files test/coverage/coverage_macro.good and test/coverage/coverage_macro.out differ; test/coverage/coverage_macro.out contains: + + Use -p '/coverage_macro/' to rerun this test only. + coverage_try: warning: trailing semicolon in macro used in expression position + --> src/main.rs:45:46 + | +45 | return Err(format!($($args)*).into()); + | ^ +... +309 | _ => die!("unknown tag {:?} for branch", tag), + | ---------------------------------------- in this macro invocation + | + = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default + = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! + = note: for more information, see issue #79813 + = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) + +FAIL (0.14s) + files test/coverage/coverage_try.good and test/coverage/coverage_try.out differ; test/coverage/coverage_try.out contains: + + Use -p '/coverage_try/' to rerun this test only. + coverage_mono: warning: trailing semicolon in macro used in expression position + --> src/main.rs:45:46 + | +45 | return Err(format!($($args)*).into()); + | ^ +... +309 | _ => die!("unknown tag {:?} for branch", tag), + | ---------------------------------------- in this macro invocation + | + = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default + = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! + = note: for more information, see issue #79813 + = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) + +OK (0.09s) + coverage_shortcircuit: warning: trailing semicolon in macro used in expression position + --> src/main.rs:45:46 + | +45 | return Err(format!($($args)*).into()); + | ^ +... +309 | _ => die!("unknown tag {:?} for branch", tag), + | ---------------------------------------- in this macro invocation + | + = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default + = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! + = note: for more information, see issue #79813 + = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) + +FAIL (0.14s) + files test/coverage/coverage_shortcircuit.good and test/coverage/coverage_shortcircuit.out differ; test/coverage/coverage_shortcircuit.out contains: + + Use -p '/coverage_shortcircuit/' to rerun this test only. + coverage_dual: warning: trailing semicolon in macro used in expression position + --> src/main.rs:45:46 + | +45 | return Err(format!($($args)*).into()); + | ^ +... +309 | _ => die!("unknown tag {:?} for branch", tag), + | ---------------------------------------- in this macro invocation + | + = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default + = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! + = note: for more information, see issue #79813 + = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) + +OK (0.09s) + coverage_cond: warning: trailing semicolon in macro used in expression position + --> src/main.rs:45:46 + | +45 | return Err(format!($($args)*).into()); + | ^ +... +309 | _ => die!("unknown tag {:?} for branch", tag), + | ---------------------------------------- in this macro invocation + | + = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default + = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! + = note: for more information, see issue #79813 + = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) + +OK (0.09s) + coverage_match: warning: trailing semicolon in macro used in expression position + --> src/main.rs:45:46 + | +45 | return Err(format!($($args)*).into()); + | ^ +... +309 | _ => die!("unknown tag {:?} for branch", tag), + | ---------------------------------------- in this macro invocation + | + = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default + = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! + = note: for more information, see issue #79813 + = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) + +FAIL (0.16s) + files test/coverage/coverage_match.good and test/coverage/coverage_match.out differ; test/coverage/coverage_match.out contains: + + Use -p '/coverage_match/' to rerun this test only. + +200 out of 339 tests failed (73.36s) From e208e98321f0f59d207f95f6b885fa7a0ab64cf6 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Mon, 3 Apr 2023 16:42:24 -0400 Subject: [PATCH 008/114] Remove unneeded RustcRenderedConst newtype --- crucible-mir/src/Mir/JSON.hs | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/crucible-mir/src/Mir/JSON.hs b/crucible-mir/src/Mir/JSON.hs index fdffcda59..53ccc370e 100644 --- a/crucible-mir/src/Mir/JSON.hs +++ b/crucible-mir/src/Mir/JSON.hs @@ -420,7 +420,7 @@ instance FromJSON Constant where rend <- v .:? "rendered" init <- v .:? "initializer" case (rend, init) of - (Just (RustcRenderedConst rend), _) -> + (Just rend, _) -> pure $ Constant ty rend (Nothing, Just (RustcConstInitializer defid)) -> pure $ Constant ty $ ConstInitializer defid @@ -434,11 +434,9 @@ instance FromJSON RustcConstInitializer where parseJSON = withObject "Initializer" $ \v -> RustcConstInitializer <$> v .: "def_id" -newtype RustcRenderedConst = RustcRenderedConst ConstVal - -instance FromJSON RustcRenderedConst where - parseJSON = withObject "RenderedConst" $ \v -> - RustcRenderedConst <$> case lookupKM "kind" v of +instance FromJSON ConstVal where + parseJSON = withObject "ConstVal" $ \v -> + case lookupKM "kind" v of Just (String "isize") -> do val <- convertIntegerText =<< v .: "val" pure $ ConstInt $ Isize val @@ -505,12 +503,11 @@ instance FromJSON RustcRenderedConst where bytes <- mapM (withScientific "byte" f) val return $ ConstArray $ V.toList bytes - Just (String "struct") -> do - fields <- map (\(RustcRenderedConst val) -> val) <$> v .: "fields" - return $ ConstStruct fields + Just (String "struct") -> + ConstStruct <$> v .: "fields" Just (String "enum") -> do variant <- v .: "variant" - fields <- map (\(RustcRenderedConst val) -> val) <$> v .: "fields" + fields <- v .: "fields" return $ ConstEnum variant fields Just (String "fndef") -> ConstFunction <$> v .: "def_id" @@ -520,13 +517,11 @@ instance FromJSON RustcRenderedConst where val <- convertIntegerText =<< v .: "val" return $ ConstRawPtr val - Just (String "array") -> do - elems <- map (\(RustcRenderedConst val) -> val) <$> v .: "elements" - return $ ConstArray elems + Just (String "array") -> + ConstArray <$> v .: "elements" - Just (String "tuple") -> do - elems <- map (\(RustcRenderedConst val) -> val) <$> v .: "elements" - return $ ConstTuple elems + Just (String "tuple") -> + ConstTuple <$> v .: "elements" o -> do fail $ "parseJSON - bad rendered constant kind: " ++ show o From b3e74d1b6579d924c396502368735fea200adf16 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Mon, 3 Apr 2023 16:44:05 -0400 Subject: [PATCH 009/114] initFnState: Remove unused argument --- crucible-mir/src/Mir/Trans.hs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index 111a035ff..e9a8c49d6 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -1722,9 +1722,8 @@ buildLabel (M.BasicBlock bi _) = do initFnState :: (?debug::Int,?customOps::CustomOpMap,?assertFalseOnError::Bool) => CollectionState -> Fn - -> FH.FnHandle args ret -> FnState s -initFnState colState fn handle = +initFnState colState fn = FnState { _varMap = Map.empty, _currentFn = fn, _debugLevel = ?debug, @@ -1831,7 +1830,7 @@ transDefine colState fn@(M.Fn fname fargs fsig _) = let rettype = FH.handleReturnType handle let def :: G.FunctionDef MIR FnState args ret (ST h) def inputs = (s,f) where - s = initFnState colState fn handle + s = initFnState colState fn f = do lbl <- genFn fn rettype inputs fti <- use transInfo From 0df06602903996565036da3924efb4c9ad3229dd Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Tue, 4 Apr 2023 08:22:14 -0400 Subject: [PATCH 010/114] genFn/transDefine: Replace unused patterns with wildcards --- crucible-mir/src/Mir/Trans.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index e9a8c49d6..abaa63b8f 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -1763,7 +1763,7 @@ genFn :: HasCallStack => C.TypeRepr ret -> Ctx.Assignment (R.Atom s) args -> MirGenerator h s ret (G.Label s) -genFn (M.Fn fname argvars sig body@(MirBody localvars blocks _)) rettype inputs = do +genFn (M.Fn fname argvars _ body@(MirBody localvars blocks _)) rettype inputs = do lm <- buildLabelMap body labelMap .= lm @@ -1822,7 +1822,7 @@ transDefine :: forall h. => CollectionState -> M.Fn -> ST h (Text, Core.AnyCFG MIR, FnTransInfo) -transDefine colState fn@(M.Fn fname fargs fsig _) = +transDefine colState fn@(M.Fn fname _ _ _) = case (Map.lookup fname (colState^.handleMap)) of Nothing -> error "bad handle!!" Just (MirHandle _hname _hsig (handle :: FH.FnHandle args ret)) -> do From 14ce2da5a295eaf70749607c74c4dbd500d65878 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Tue, 4 Apr 2023 15:33:32 -0400 Subject: [PATCH 011/114] Replace currentFn with a more general FnTransContext This is in preparation for permitting various translation functions to be called from `transStatics`. --- crucible-mir/src/Mir/Generator.hs | 31 +++++++++++++++++++++++++---- crucible-mir/src/Mir/Trans.hs | 14 +++++++------ crucible-mir/src/Mir/TransCustom.hs | 12 +++++------ 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/crucible-mir/src/Mir/Generator.hs b/crucible-mir/src/Mir/Generator.hs index 4b1af9f47..a5205804d 100644 --- a/crucible-mir/src/Mir/Generator.hs +++ b/crucible-mir/src/Mir/Generator.hs @@ -168,13 +168,20 @@ data FnState (s :: Type) = FnState { _varMap :: !(VarMap s), _labelMap :: !(LabelMap s), _debugLevel :: !Int, - _currentFn :: !Fn, + _transContext :: !FnTransContext, _cs :: !CollectionState, _customOps :: !CustomOpMap, _assertFalseOnError :: !Bool, _transInfo :: !FnTransInfo } +-- | The current translation context +data FnTransContext + = FnContext Fn + -- ^ We are translating a function definition. + | StaticContext + -- ^ We are translating the initializer for static values. + -- | State about the entire collection used for the translation data CollectionState = CollectionState { @@ -392,6 +399,19 @@ instance Pretty MirHandle where viaShow nm <> colon <> pretty sig +instance Pretty FnTransContext where + pretty (FnContext f) = pretty f + pretty StaticContext = "the static initializer" + +expectFnContext :: MirGenerator h s ret Fn +expectFnContext = do + transCtxt <- use transContext + case transCtxt of + FnContext f -> pure f + StaticContext -> + mirFail "expected function when translating static initializer" + + varInfoRepr :: VarInfo s tp -> C.TypeRepr tp varInfoRepr (VarRegister reg0) = R.typeOfReg reg0 varInfoRepr (VarReference reg0) = @@ -430,13 +450,16 @@ mirFail :: String -> MirGenerator h s ret a mirFail str = do b <- use assertFalseOnError db <- use debugLevel - f <- use currentFn - let msg = "Translation error in " ++ show (f^.fname) ++ ": " ++ str + transCtxt <- use transContext + let loc = case transCtxt of + FnContext f -> show (f^.fname) + StaticContext -> "the static initializer" + msg = "Translation error in " ++ loc ++ ": " ++ str if b then do when (db > 1) $ do traceM ("Translation failure: " ++ str) when (db > 2) $ do - traceM (fmt f) + traceM (fmt transCtxt) G.reportError (S.litExpr (Text.pack msg)) else error msg diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index abaa63b8f..3f92a1321 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -1250,7 +1250,8 @@ transSwitch pos exp vals blks = do -- and false cases, even when the false case is empty. switchIsDropFlagCheck :: [Integer] -> [M.BasicBlockInfo] -> MirGenerator h s ret Bool switchIsDropFlagCheck [0] [f, t] = do - optTrueBb <- use $ currentFn . fbody . mblockmap . at t + fn <- expectFnContext + let optTrueBb = fn ^. fbody . mblockmap . at t trueBb <- case optTrueBb of Just x -> return $ x Nothing -> mirFail $ "bad switch target " ++ show t @@ -1284,7 +1285,8 @@ doReturn tr = do -- -- To detect if the current function is a static initializer, we check if -- there's an entry in `statics` matching the current `fname`. - curName <- use $ currentFn . fname + fn <- expectFnContext + let curName = fn^.fname isStatic <- use $ cs . collection . statics . to (Map.member curName) when (not isStatic) cleanupLocals @@ -1721,11 +1723,11 @@ buildLabel (M.BasicBlock bi _) = do -- | Build the initial state for translation of functions initFnState :: (?debug::Int,?customOps::CustomOpMap,?assertFalseOnError::Bool) => CollectionState - -> Fn + -> FnTransContext -> FnState s -initFnState colState fn = +initFnState colState transCtxt = FnState { _varMap = Map.empty, - _currentFn = fn, + _transContext = transCtxt, _debugLevel = ?debug, _cs = colState, _labelMap = Map.empty, @@ -1830,7 +1832,7 @@ transDefine colState fn@(M.Fn fname _ _ _) = let rettype = FH.handleReturnType handle let def :: G.FunctionDef MIR FnState args ret (ST h) def inputs = (s,f) where - s = initFnState colState fn + s = initFnState colState (FnContext fn) f = do lbl <- genFn fn rettype inputs fti <- use transInfo diff --git a/crucible-mir/src/Mir/TransCustom.hs b/crucible-mir/src/Mir/TransCustom.hs index b89230b36..de9063096 100644 --- a/crucible-mir/src/Mir/TransCustom.hs +++ b/crucible-mir/src/Mir/TransCustom.hs @@ -192,20 +192,20 @@ abort = (["core", "intrinsics", "", "abort"], \s -> panicking_begin_panic :: (ExplodedDefId, CustomRHS) panicking_begin_panic = (["std", "panicking", "begin_panic"], \s -> Just $ CustomOpExit $ \ops -> do - name <- use $ currentFn . fname - return $ "panicking::begin_panic, called from " <> M.idText name + fn <- expectFnContext + return $ "panicking::begin_panic, called from " <> M.idText (fn^.fname) ) panicking_panic :: (ExplodedDefId, CustomRHS) panicking_panic = (["core", "panicking", "panic"], \s -> Just $ CustomOpExit $ \ops -> do - name <- use $ currentFn . fname - return $ "panicking::panic, called from " <> M.idText name + fn <- expectFnContext + return $ "panicking::panic, called from " <> M.idText (fn^.fname) ) panicking_panic_fmt :: (ExplodedDefId, CustomRHS) panicking_panic_fmt = (["core", "panicking", "panic_fmt"], \s -> Just $ CustomOpExit $ \ops -> do - name <- use $ currentFn . fname - return $ "panicking::panic_fmt, called from " <> M.idText name + fn <- expectFnContext + return $ "panicking::panic_fmt, called from " <> M.idText (fn^.fname) ) panicking_panicking :: (ExplodedDefId, CustomRHS) From 6f97467a2b21b7221da14b272c8fd4284a8296d3 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Tue, 4 Apr 2023 15:58:20 -0400 Subject: [PATCH 012/114] Translate statics with constant initializers properly If a static is initialized with a constant value, we now detect and handle this in `translateStatics`. --- crucible-mir/src/Mir/JSON.hs | 1 + crucible-mir/src/Mir/Mir.hs | 2 ++ crucible-mir/src/Mir/PP.hs | 5 ++-- crucible-mir/src/Mir/Trans.hs | 52 +++++++++++++++++++++++------------ crux-mir/src/Mir/Language.hs | 2 ++ 5 files changed, 42 insertions(+), 20 deletions(-) diff --git a/crucible-mir/src/Mir/JSON.hs b/crucible-mir/src/Mir/JSON.hs index 53ccc370e..35c4edde2 100644 --- a/crucible-mir/src/Mir/JSON.hs +++ b/crucible-mir/src/Mir/JSON.hs @@ -577,6 +577,7 @@ instance FromJSON Static where Static <$> v .: "name" <*> v .: "ty" <*> v .: "mutable" + <*> v .:? "rendered" -- LocalWords: initializer supertraits deserialization impls diff --git a/crucible-mir/src/Mir/Mir.hs b/crucible-mir/src/Mir/Mir.hs index 3b8b67a68..c11dcd3d8 100644 --- a/crucible-mir/src/Mir/Mir.hs +++ b/crucible-mir/src/Mir/Mir.hs @@ -516,6 +516,8 @@ data Static = Static { _sName :: DefId -- ^ name of fn that initializes this static , _sTy :: Ty , _sMutable :: Bool -- ^ true for "static mut" + , _sConstVal :: Maybe ConstVal -- ^ 'Just' if this static is initialized + -- with a constant value. 'Nothing' otherwise. } deriving (Show, Eq, Ord, Generic) diff --git a/crucible-mir/src/Mir/PP.hs b/crucible-mir/src/Mir/PP.hs index 258cf0806..8dca8f05b 100644 --- a/crucible-mir/src/Mir/PP.hs +++ b/crucible-mir/src/Mir/PP.hs @@ -356,8 +356,9 @@ instance Pretty Trait where rbrace] instance Pretty Static where - pretty (Static nm ty mut) = - pretty mut <+> pretty nm <+> pretty ":" <+> pretty ty + pretty (Static nm ty mut mbConst) = + pretty mut <+> pretty nm <+> pretty ":" <+> pretty ty <+> + maybe mempty (\c -> pretty "=" <+> pretty c) mbConst instance Pretty Intrinsic where pretty (Intrinsic name inst) = pretty name <+> pretty "=" <+> pretty inst diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index 3f92a1321..549c0e225 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -3,6 +3,7 @@ {-# LANGUAGE GADTs #-} {-# LANGUAGE ImplicitParams #-} {-# LANGUAGE KindSignatures #-} +{-# LANGUAGE MultiWayIf #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE PatternGuards #-} {-# LANGUAGE PatternSynonyms #-} @@ -2287,32 +2288,47 @@ transCollection col halloc = do -- | Produce a crucible CFG that initializes the global variables for the static -- part of the crate -transStatics :: CollectionState -> FH.HandleAllocator -> IO (Core.AnyCFG MIR) +transStatics :: + (?debug::Int,?customOps::CustomOpMap,?assertFalseOnError::Bool) => + CollectionState -> FH.HandleAllocator -> IO (Core.AnyCFG MIR) transStatics colState halloc = do let sm = colState^.staticMap let hmap = colState^.handleMap - let initializeStatic :: forall h s r . Static -> G.Generator MIR s (Const ()) r (ST h) () - initializeStatic static = do - case Map.lookup (static^.sName) sm of - Just (StaticVar g) -> do - let repr = G.globalType g - case Map.lookup (static^.sName) hmap of - Just (MirHandle _ _ (handle :: FH.FnHandle init ret)) - | Just Refl <- testEquality repr (FH.handleReturnType handle) - , Just Refl <- testEquality (Ctx.empty) (FH.handleArgTypes handle) - -> do val <- G.call (G.App $ E.HandleLit handle) Ctx.empty - G.writeGlobal g val - Just (MirHandle _ _ handle) -> - error $ "BUG: invalid type for initializer " ++ fmt (static^.sName) - Nothing -> error $ "BUG: cannot find handle for static " ++ fmt (static^.sName) - Nothing -> error $ "BUG: cannot find global for " ++ fmt (static^.sName) + let initializeStatic :: forall h s r . Static -> MirGenerator h s r () + initializeStatic static = + let staticName = static^.sName in + case Map.lookup staticName sm of + Just (StaticVar g) -> + let repr = G.globalType g in + if | Just constval <- static^.sConstVal + -> do let constty = static^.sTy + Some tpr <- tyToReprM constty + MirExp constty' constval' <- transConstVal constty (Some tpr) constval + case testEquality repr constty' of + Just Refl -> G.writeGlobal g constval' + Nothing -> error $ "BUG: invalid type for constant initializer " ++ fmt staticName + + | Just (MirHandle _ _ (handle :: FH.FnHandle init ret)) + <- Map.lookup staticName hmap + -> case ( testEquality repr (FH.handleReturnType handle) + , testEquality (Ctx.empty) (FH.handleArgTypes handle) + ) of + (Just Refl, Just Refl) -> do + val <- G.call (G.App $ E.HandleLit handle) Ctx.empty + G.writeGlobal g val + + _ -> error $ "BUG: invalid type for initializer function " ++ fmt staticName + + | otherwise + -> error $ "BUG: cannot find initializer for static " ++ fmt staticName + Nothing -> error $ "BUG: cannot find global for " ++ fmt staticName -- TODO: make the name of the static initialization function configurable let initName = FN.functionNameFromText "static_initializer" initHandle <- FH.mkHandle' halloc initName Ctx.empty C.UnitRepr - let def :: G.FunctionDef MIR (Const ()) Ctx.EmptyCtx C.UnitType (ST w) + let def :: G.FunctionDef MIR FnState Ctx.EmptyCtx C.UnitType (ST w) def inputs = (s, f) where - s = Const () + s = initFnState colState StaticContext f = do mapM_ initializeStatic (colState^.collection.statics) return (R.App $ E.EmptyApp) init_cfg <- stToIO $ do diff --git a/crux-mir/src/Mir/Language.hs b/crux-mir/src/Mir/Language.hs index 394166364..4a760b07c 100644 --- a/crux-mir/src/Mir/Language.hs +++ b/crux-mir/src/Mir/Language.hs @@ -110,6 +110,7 @@ import Mir.Generate (generateMIR) import qualified Mir.Log as Log import Mir.ParseTranslate (translateMIR) import Mir.Trans (transStatics) +import qualified Mir.TransCustom as TransCustom import Mir.TransTy import Mir.Concurrency import Paths_crux_mir (version) @@ -210,6 +211,7 @@ runTestsWithExtraOverrides bindExtra (cruxOpts, mirOpts) = do let ?assertFalseOnError = True let ?printCrucible = printCrucible mirOpts let ?defaultRlibsDir = defaultRlibsDir mirOpts + let ?customOps = TransCustom.customOps let (filename, nameFilter) = case cargoTestFile mirOpts of -- This case is terrible a hack. The goal is to mimic the behavior From cc8d3f24fb0ab862233df7094369806a3828fe0b Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 5 Apr 2023 14:43:43 -0400 Subject: [PATCH 013/114] Add custom op for ctpop --- crucible-mir/src/Mir/TransCustom.hs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/crucible-mir/src/Mir/TransCustom.hs b/crucible-mir/src/Mir/TransCustom.hs index de9063096..b11af8b09 100644 --- a/crucible-mir/src/Mir/TransCustom.hs +++ b/crucible-mir/src/Mir/TransCustom.hs @@ -161,6 +161,8 @@ customOpDefs = Map.fromList $ [ , maybe_uninit_uninit + , ctpop + , integer_from_u8 , integer_from_i32 , integer_from_u64 @@ -1552,14 +1554,15 @@ maybe_uninit_uninit = (["core", "mem", "maybe_uninit", "{{impl}}", "uninit"], -------------------------------------------------------------------------------------------------------------------------- -intrinsicName :: String -> [String] -intrinsicName name = ["core", "intrinsics", "", name] - --- ctpop :: (ExplodedDefId, CustomRHS) --- ctpop = (intrinsicName "ctpop", rhs) --- where --- rhs [_sz] = --- Just $ CustomOp $ + +ctpop :: (ExplodedDefId, CustomRHS) +ctpop = (["core", "intrinsics", "{extern}", "ctpop"], + \(Substs substs) -> case substs of + [_] -> Just $ CustomOp $ \_ ops -> case ops of + [MirExp tpr@(C.BVRepr w) val] -> + return $ MirExp tpr $ R.App $ E.BVPopcount w val + _ -> mirFail $ "bad arguments to intrinsics::ctpop: " ++ show ops + _ -> Nothing) -------------------------------------------------------------------------------------------------------------------------- From 0eacf40ce8e7ee420079732893acd7a21d097fd8 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Fri, 14 Apr 2023 15:25:16 -0400 Subject: [PATCH 014/114] lib: Replace RawVec's allocator Adapted from the original patch in 03743105d0d5730449c3614cc2780b36d765be26 --- crux-mir/build.sh | 9 +-- crux-mir/lib/Patches.md | 21 ++++++ crux-mir/lib/alloc/src/raw_vec.rs | 115 ++++++++---------------------- 3 files changed, 56 insertions(+), 89 deletions(-) create mode 100644 crux-mir/lib/Patches.md diff --git a/crux-mir/build.sh b/crux-mir/build.sh index 35cea2e5f..3b30a684a 100755 --- a/crux-mir/build.sh +++ b/crux-mir/build.sh @@ -14,8 +14,12 @@ mir-json lib/libc/src/lib.rs --crate-name libc -L rlibs --out-dir rlibs --crate echo 'Building compiler_builtins...' mir-json lib/compiler_builtins/src/lib.rs --crate-name compiler_builtins -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="compiler-builtins"' --cfg 'feature="core"' --cfg 'feature="default"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="unstable"' --cfg 'feature="mem-unaligned"`' --extern core=rlibs/libcore.rlib +# extra lib (added manually) +echo "Building crucible..." +mir-json lib/crucible/lib.rs --crate-name crucible --edition=2021 -L rlibs --out-dir rlibs --crate-type rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib + echo 'Building alloc...' -mir-json lib/alloc/src/lib.rs --edition=2021 --crate-name alloc -L rlibs --out-dir rlibs --crate-type rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib +mir-json lib/alloc/src/lib.rs --edition=2021 --crate-name alloc -L rlibs --out-dir rlibs --crate-type rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern crucible=rlibs/libcrucible.rlib echo 'Building cfg_if...' mir-json lib/cfg_if/src/lib.rs --edition=2018 --crate-name cfg_if -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib @@ -66,9 +70,6 @@ echo 'Building proc_macro...' mir-json lib/proc_macro/src/lib.rs --edition=2021 --crate-name proc_macro -L rlibs --out-dir rlibs --crate-type rlib --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib # extra libs (added manually) -echo "Building crucible..." -mir-json lib/crucible/lib.rs --crate-name crucible --edition=2021 -L rlibs --out-dir rlibs --crate-type rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib - echo 'Building int512...' mir-json lib/int512.rs --crate-name int512 -L rlibs --out-dir rlibs --crate-type rlib --extern core=rlibs/libcore.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md new file mode 100644 index 000000000..767e0e9e1 --- /dev/null +++ b/crux-mir/lib/Patches.md @@ -0,0 +1,21 @@ +# Patching the Rust standard library + +This directory bundles a copy of the Rust standard library with various patches +applied in certain key places to make the resulting code easier for Crucible to +handle. These patches must be applied every time that the bundled Rust standard +library is updated. Moreover, this code often looks quite different each time +between Rust versions, so applying the patches is rarely as straightforward as +running `git apply`. + +As a compromise, this document contains high-level descriptions of each type of +patch that we apply, along with rationale for why the patch is necessary. The +intent is that this document can be used in conjunction with `git blame` to +identify all of the code that was changed in each patch. + +* Use Crucible's allocator in `alloc/src/raw_vec.rs` (last applied: April 14, 2023) + + The `Allocator` for `RawVec`s is quite involved and is beyond Crucible's + ability to reason about. We replace the `Allocator` with the corresponding + built-in Crucible allocation functions (e.g., `crucible::alloc::allocate`). + We also make sure to avoid the `Layout::array` function, which has a + particularly tricky use of `transmute` that we do not currently support. diff --git a/crux-mir/lib/alloc/src/raw_vec.rs b/crux-mir/lib/alloc/src/raw_vec.rs index 5a10121bb..6b23f8fa3 100644 --- a/crux-mir/lib/alloc/src/raw_vec.rs +++ b/crux-mir/lib/alloc/src/raw_vec.rs @@ -7,6 +7,7 @@ use core::mem::{self, ManuallyDrop, MaybeUninit, SizedTypeProperties}; use core::ops::Drop; use core::ptr::{self, NonNull, Unique}; use core::slice; +use crucible; #[cfg(not(no_global_oom_handling))] use crate::alloc::handle_alloc_error; @@ -171,23 +172,10 @@ impl RawVec { if T::IS_ZST || capacity == 0 { Self::new_in(alloc) } else { - // We avoid `unwrap_or_else` here because it bloats the amount of - // LLVM IR generated. - let layout = match Layout::array::(capacity) { - Ok(layout) => layout, - Err(_) => capacity_overflow(), - }; - match alloc_guard(layout.size()) { - Ok(_) => {} - Err(_) => capacity_overflow(), - } - let result = match init { - AllocInit::Uninitialized => alloc.allocate(layout), - AllocInit::Zeroed => alloc.allocate_zeroed(layout), - }; - let ptr = match result { - Ok(ptr) => ptr, - Err(_) => handle_alloc_error(layout), + let ptr = { + // NB: This ignores both the choice of allocator and the `init` argument + let ptr = crucible::alloc::allocate::(capacity); + unsafe { NonNull::new_unchecked(ptr) } }; // Allocators currently return a `NonNull<[u8]>` whose length @@ -237,19 +225,6 @@ impl RawVec { &self.alloc } - fn current_memory(&self) -> Option<(NonNull, Layout)> { - if T::IS_ZST || self.cap == 0 { - None - } else { - // We have an allocated chunk of memory, so we can bypass runtime - // checks to get our current layout. - unsafe { - let layout = Layout::array::(self.cap).unwrap_unchecked(); - Some((self.ptr.cast().into(), layout)) - } - } - } - /// Ensures that the buffer contains at least enough space to hold `len + /// additional` elements. If it doesn't already have enough capacity, will /// reallocate enough space plus comfortable slack space to get amortized @@ -394,11 +369,14 @@ impl RawVec { let cap = cmp::max(self.cap * 2, required_cap); let cap = cmp::max(Self::MIN_NON_ZERO_CAP, cap); - let new_layout = Layout::array::(cap); - - // `finish_grow` is non-generic over `T`. - let ptr = finish_grow(new_layout, self.current_memory(), &mut self.alloc)?; - self.set_ptr_and_cap(ptr, cap); + if self.cap == 0 { + unsafe { + self.ptr = NonNull::new_unchecked(crucible::alloc::allocate::(cap)).into(); + } + } else { + crucible::alloc::reallocate::(self.ptr.as_ptr(), cap); + self.cap = cap; + } Ok(()) } @@ -413,11 +391,15 @@ impl RawVec { } let cap = len.checked_add(additional).ok_or(CapacityOverflow)?; - let new_layout = Layout::array::(cap); - // `finish_grow` is non-generic over `T`. - let ptr = finish_grow(new_layout, self.current_memory(), &mut self.alloc)?; - self.set_ptr_and_cap(ptr, cap); + if self.cap == 0 { + unsafe { + self.ptr = NonNull::new_unchecked(crucible::alloc::allocate::(cap)).into(); + } + } else { + crucible::alloc::reallocate::(self.ptr.as_ptr(), cap); + self.cap = cap; + } Ok(()) } @@ -425,59 +407,22 @@ impl RawVec { fn shrink(&mut self, cap: usize) -> Result<(), TryReserveError> { assert!(cap <= self.capacity(), "Tried to shrink to a larger capacity"); - let (ptr, layout) = if let Some(mem) = self.current_memory() { mem } else { return Ok(()) }; - - let ptr = unsafe { - // `Layout::array` cannot overflow here because it would have - // overflowed earlier when capacity was larger. - let new_layout = Layout::array::(cap).unwrap_unchecked(); - self.alloc - .shrink(ptr, layout, new_layout) - .map_err(|_| AllocError { layout: new_layout, non_exhaustive: () })? - }; - self.set_ptr_and_cap(ptr, cap); + if self.cap == 0 { + unsafe { + self.ptr = NonNull::new_unchecked(crucible::alloc::allocate::(cap)).into(); + } + } else { + crucible::alloc::reallocate::(self.ptr.as_ptr(), cap); + self.cap = cap; + } Ok(()) } } -// This function is outside `RawVec` to minimize compile times. See the comment -// above `RawVec::grow_amortized` for details. (The `A` parameter isn't -// significant, because the number of different `A` types seen in practice is -// much smaller than the number of `T` types.) -#[inline(never)] -fn finish_grow( - new_layout: Result, - current_memory: Option<(NonNull, Layout)>, - alloc: &mut A, -) -> Result, TryReserveError> -where - A: Allocator, -{ - // Check for the error here to minimize the size of `RawVec::grow_*`. - let new_layout = new_layout.map_err(|_| CapacityOverflow)?; - - alloc_guard(new_layout.size())?; - - let memory = if let Some((ptr, old_layout)) = current_memory { - debug_assert_eq!(old_layout.align(), new_layout.align()); - unsafe { - // The allocator checks for alignment equality - intrinsics::assume(old_layout.align() == new_layout.align()); - alloc.grow(ptr, old_layout, new_layout) - } - } else { - alloc.allocate(new_layout) - }; - - memory.map_err(|_| AllocError { layout: new_layout, non_exhaustive: () }.into()) -} - unsafe impl<#[may_dangle] T, A: Allocator> Drop for RawVec { /// Frees the memory owned by the `RawVec` *without* trying to drop its contents. fn drop(&mut self) { - if let Some((ptr, layout)) = self.current_memory() { - unsafe { self.alloc.deallocate(ptr, layout) } - } + // Crucible currently doesn't provide a deallocate operation. } } From fa7292a94832da7c5c58d2da0567823603302c04 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Tue, 2 May 2023 09:15:20 -0400 Subject: [PATCH 015/114] Translate more types of ConstArrays Previously, only constant arrays of bytes were translatable. This patch lifts that restriction so that constant arrays of arbitrary types can be translated, which is necessary to support some constant arrays used in a newer Rust nightly. --- crucible-mir/src/Mir/Trans.hs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index 549c0e225..1fa0efd5c 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -64,6 +64,7 @@ import qualified Data.Set as Set import Data.STRef import Data.Text (Text) import qualified Data.Text as Text +import qualified Data.Traversable as Trav import qualified Data.Vector as V import Data.String (fromString) import Numeric @@ -131,9 +132,6 @@ eBVLit w i = E.BVLit w (BV.mkBV w i) -------------------------------------------------------------------------------------- -- ** Expressions -u8ToBV8 :: ConstVal -> R.Expr MIR s (C.BVType 8) -u8ToBV8 (ConstInt (U8 c)) = R.App (eBVLit knownRepr c) -u8ToBV8 _ = error $ "BUG: array literals should only contain bytes (u8)" -- Expressions: variables and constants -- @@ -173,10 +171,15 @@ transConstVal _ty (Some (MirSliceRepr (C.BVRepr w))) (M.ConstStr bs) knownRepr (Ctx.Empty Ctx.:> ref Ctx.:> len) return $ MirExp (MirSliceRepr u8Repr) struct -transConstVal _ty (Some (MirVectorRepr w)) (M.ConstArray arr) - | Just Refl <- testEquality w (C.BVRepr (knownRepr :: NatRepr 8)) - = do let bytes = V.fromList (map u8ToBV8 arr) - MirExp (MirVectorRepr w) <$> mirVector_fromVector w (R.App $ E.VectorLit w bytes) +transConstVal (M.TyArray ty _sz) (Some (MirVectorRepr tpr)) (M.ConstArray arr) = do + arr' <- Trav.for arr $ \e -> do + MirExp tpr' e' <- transConstVal ty (Some tpr) e + Refl <- testEqualityOrFail tpr tpr' $ + "transConstVal (ConstArray): returned wrong type: expected " ++ + show tpr ++ ", got " ++ show tpr' + pure e' + vec <- mirVector_fromVector tpr $ R.App $ E.VectorLit tpr $ V.fromList arr' + return $ MirExp (MirVectorRepr tpr) vec transConstVal _ty (Some (C.BVRepr w)) (M.ConstChar c) = do let i = toInteger (Char.ord c) return $ MirExp (C.BVRepr w) (S.app $ eBVLit w i) From 370caccccff271438c98d0a9acbc208a9eefd470 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Tue, 2 May 2023 12:54:18 -0400 Subject: [PATCH 016/114] Update mir-json submodule --- dependencies/mir-json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies/mir-json b/dependencies/mir-json index 5d697d5f3..a4d0ebefb 160000 --- a/dependencies/mir-json +++ b/dependencies/mir-json @@ -1 +1 @@ -Subproject commit 5d697d5f37e8a041c7010e47776b6cb02be940d9 +Subproject commit a4d0ebefb5665857a916f74d86a681e8b4895dac From 6b4370f1bef998ffd6aa24521b09e79bca46da53 Mon Sep 17 00:00:00 2001 From: James LaMar Date: Tue, 2 May 2023 10:10:13 -0700 Subject: [PATCH 017/114] patch `core::fmt` --- crux-mir/lib/core/src/fmt/mod.rs | 50 +- crux-mir/lib/core/src/fmt/num.rs | 87 +- crux-mir/test_output.log | 3256 ++++++++++-------------------- 3 files changed, 1100 insertions(+), 2293 deletions(-) diff --git a/crux-mir/lib/core/src/fmt/mod.rs b/crux-mir/lib/core/src/fmt/mod.rs index 2a7ec544f..90670c467 100644 --- a/crux-mir/lib/core/src/fmt/mod.rs +++ b/crux-mir/lib/core/src/fmt/mod.rs @@ -11,6 +11,7 @@ use crate::num::fmt as numfmt; use crate::ops::Deref; use crate::result; use crate::str; +use crate::crucible::any::Any; mod builders; #[cfg(not(no_fp_fmt_parse))] @@ -272,8 +273,18 @@ extern "C" { #[unstable(feature = "fmt_internals", reason = "internal to format_args!", issue = "none")] #[doc(hidden)] pub struct ArgumentV1<'a> { - value: &'a Opaque, - formatter: fn(&Opaque, &mut Formatter<'_>) -> Result, + value: Any, + formatter: Any, + dispatch: fn(Any, Any, &mut Formatter<'_>) -> Result, + _marker: PhantomData<&'a Opaque>, +} + +fn dispatch(value: Any, formatter: Any, fmt: &mut Formatter<'_>) -> Result { + unsafe { + let value = value.downcast::<&T>(); + let formatter = formatter.downcast::) -> Result>(); + formatter(value, fmt) + } } /// This struct represents the unsafety of constructing an `Arguments`. @@ -336,15 +347,12 @@ impl<'a> ArgumentV1<'a> { #[unstable(feature = "fmt_internals", reason = "internal to format_args!", issue = "none")] #[inline] pub fn new<'b, T>(x: &'b T, f: fn(&T, &mut Formatter<'_>) -> Result) -> ArgumentV1<'b> { - // SAFETY: `mem::transmute(x)` is safe because - // 1. `&'b T` keeps the lifetime it originated with `'b` - // (so as to not have an unbounded lifetime) - // 2. `&'b T` and `&'b Opaque` have the same memory layout - // (when `T` is `Sized`, as it is here) - // `mem::transmute(f)` is safe since `fn(&T, &mut Formatter<'_>) -> Result` - // and `fn(&Opaque, &mut Formatter<'_>) -> Result` have the same ABI - // (as long as `T` is `Sized`) - unsafe { ArgumentV1 { formatter: mem::transmute(f), value: mem::transmute(x) } } + ArgumentV1 { + value: Any::new(x), + formatter: Any::new(f), + dispatch: dispatch::, + _marker: PhantomData, + } } arg_new!(new_display, Display); @@ -360,21 +368,13 @@ impl<'a> ArgumentV1<'a> { #[doc(hidden)] #[unstable(feature = "fmt_internals", reason = "internal to format_args!", issue = "none")] pub fn from_usize(x: &usize) -> ArgumentV1<'_> { - ArgumentV1::new(x, USIZE_MARKER) + Self::new(x, Display::fmt) } fn as_usize(&self) -> Option { - // We are type punning a bit here: USIZE_MARKER only takes an &usize but - // formatter takes an &Opaque. Rust understandably doesn't think we should compare - // the function pointers if they don't have the same signature, so we cast to - // usizes to tell it that we just want to compare addresses. - if self.formatter as usize == USIZE_MARKER as usize { - // SAFETY: The `formatter` field is only set to USIZE_MARKER if - // the value is a usize, so this is safe - Some(unsafe { *(self.value as *const _ as *const usize) }) - } else { - None - } + // According to the comment on `USIZE_MARKER`, this method is only ever called on arguments + // that have been statically checked to be `usize`. + Some(unsafe { *self.value.downcast::<&usize>() }) } } @@ -1210,7 +1210,7 @@ pub fn write(output: &mut dyn Write, args: Arguments<'_>) -> Result { if !piece.is_empty() { formatter.buf.write_str(*piece)?; } - (arg.formatter)(arg.value, &mut formatter)?; + (arg.dispatch)(arg.value, arg.formatter, &mut formatter)?; idx += 1; } } @@ -1258,7 +1258,7 @@ unsafe fn run(fmt: &mut Formatter<'_>, arg: &rt::v1::Argument, args: &[ArgumentV let value = unsafe { args.get_unchecked(arg.position) }; // Then actually do some printing - (value.formatter)(value.value, fmt) + (value.dispatch)(value.value, value.formatter, fmt) } unsafe fn getcount(args: &[ArgumentV1<'_>], cnt: &rt::v1::Count) -> Option { diff --git a/crux-mir/lib/core/src/fmt/num.rs b/crux-mir/lib/core/src/fmt/num.rs index d8365ae9b..46f6b0114 100644 --- a/crux-mir/lib/core/src/fmt/num.rs +++ b/crux-mir/lib/core/src/fmt/num.rs @@ -69,7 +69,7 @@ trait GenericRadix: Sized { // characters for a base 2 number. let zero = T::zero(); let is_nonnegative = x >= zero; - let mut buf = [MaybeUninit::::uninit(); 128]; + let mut buf = [0; 128]; let mut curr = buf.len(); let base = T::from_u8(Self::BASE); if is_nonnegative { @@ -78,7 +78,7 @@ trait GenericRadix: Sized { for byte in buf.iter_mut().rev() { let n = x % base; // Get the current place value. x = x / base; // Deaccumulate the number. - byte.write(Self::digit(n.to_u8())); // Store the digit in the buffer. + *byte = Self::digit(n.to_u8()); // Store the digit in the buffer. curr -= 1; if x == zero { // No more digits left to accumulate. @@ -90,7 +90,7 @@ trait GenericRadix: Sized { for byte in buf.iter_mut().rev() { let n = zero - (x % base); // Get the current place value. x = x / base; // Deaccumulate the number. - byte.write(Self::digit(n.to_u8())); // Store the digit in the buffer. + *byte = Self::digit(n.to_u8()); // Store the digit in the buffer. curr -= 1; if x == zero { // No more digits left to accumulate. @@ -102,10 +102,7 @@ trait GenericRadix: Sized { // SAFETY: The only chars in `buf` are created by `Self::digit` which are assumed to be // valid UTF-8 let buf = unsafe { - str::from_utf8_unchecked(slice::from_raw_parts( - MaybeUninit::slice_as_ptr(buf), - buf.len(), - )) + str::from_utf8_unchecked(buf) }; f.pad_integral(is_nonnegative, Self::PREFIX, buf) } @@ -209,67 +206,29 @@ static DEC_DIGITS_LUT: &[u8; 200] = b"0001020304050607080910111213141516171819\ macro_rules! impl_Display { ($($t:ident),* as $u:ident via $conv_fn:ident named $name:ident) => { fn $name(mut n: $u, is_nonnegative: bool, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // 2^128 is about 3*10^38, so 39 gives an extra byte of space - let mut buf = [MaybeUninit::::uninit(); 39]; - let mut curr = buf.len(); - let buf_ptr = MaybeUninit::slice_as_mut_ptr(&mut buf); - let lut_ptr = DEC_DIGITS_LUT.as_ptr(); - - // SAFETY: Since `d1` and `d2` are always less than or equal to `198`, we - // can copy from `lut_ptr[d1..d1 + 1]` and `lut_ptr[d2..d2 + 1]`. To show - // that it's OK to copy into `buf_ptr`, notice that at the beginning - // `curr == buf.len() == 39 > log(n)` since `n < 2^128 < 10^39`, and at - // each step this is kept the same as `n` is divided. Since `n` is always - // non-negative, this means that `curr > 0` so `buf_ptr[curr..curr + 1]` - // is safe to access. - unsafe { - // need at least 16 bits for the 4-characters-at-a-time to work. - assert!(crate::mem::size_of::<$u>() >= 2); + let mut buf = [0; 39]; + let mut curr = buf.len() as usize; + let lut = DEC_DIGITS_LUT; - // eagerly decode 4 characters at a time - while n >= 10000 { - let rem = (n % 10000) as usize; - n /= 10000; - - let d1 = (rem / 100) << 1; - let d2 = (rem % 100) << 1; - curr -= 4; - - // We are allowed to copy to `buf_ptr[curr..curr + 3]` here since - // otherwise `curr < 0`. But then `n` was originally at least `10000^10` - // which is `10^40 > 2^128 > n`. - ptr::copy_nonoverlapping(lut_ptr.add(d1), buf_ptr.add(curr), 2); - ptr::copy_nonoverlapping(lut_ptr.add(d2), buf_ptr.add(curr + 2), 2); - } - - // if we reach here numbers are <= 9999, so at most 4 chars long - let mut n = n as usize; // possibly reduce 64bit math - - // decode 2 more chars, if > 2 chars - if n >= 100 { - let d1 = (n % 100) << 1; - n /= 100; - curr -= 2; - ptr::copy_nonoverlapping(lut_ptr.add(d1), buf_ptr.add(curr), 2); - } + // decode 2 more chars, if > 2 chars + while n >= 100 { + let d1 = ((n % 100) as usize) << 1; + n /= 100; + curr -= 2; + buf[curr .. curr + 2].copy_from_slice(&lut[d1 .. d1 + 2]); + } - // decode last 1 or 2 chars - if n < 10 { - curr -= 1; - *buf_ptr.add(curr) = (n as u8) + b'0'; - } else { - let d1 = n << 1; - curr -= 2; - ptr::copy_nonoverlapping(lut_ptr.add(d1), buf_ptr.add(curr), 2); - } + // decode last 1 or 2 chars + if n < 10 { + curr -= 1; + buf[curr] = (n as u8) + b'0'; + } else { + let d1 = (n as usize) << 1; + curr -= 2; + buf[curr .. curr + 2].copy_from_slice(&lut[d1 .. d1 + 2]); } - // SAFETY: `curr` > 0 (since we made `buf` large enough), and all the chars are valid - // UTF-8 since `DEC_DIGITS_LUT` is - let buf_slice = unsafe { - str::from_utf8_unchecked( - slice::from_raw_parts(buf_ptr.add(curr), buf.len() - curr)) - }; + let buf_slice = unsafe { str::from_utf8_unchecked(&buf[curr..]) }; f.pad_integral(is_nonnegative, "", buf_slice) } diff --git a/crux-mir/test_output.log b/crux-mir/test_output.log index 24df242dd..5728a8c47 100644 --- a/crux-mir/test_output.log +++ b/crux-mir/test_output.log @@ -1,18 +1,19 @@ +Running scope as unit: run-rabc937d071ac43b59a95d19eca862da1.scope Up to date crux-mir crux concrete vec - set_len: FAIL (0.24s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (0.10s) + set_len: FAIL (3.44s) + Compiling and running oracle program (0.91s) + Oracle output: 2 (2.53s) Crux output: failures: - ---- set_len/e7047f4e::crux_test[0] counterexamples ---- + ---- set_len/19a6ca9c::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. @@ -20,26 +21,22 @@ crux-mir crux doesn't match oracle Use -p '/set_len/' to rerun this test only. - from_elem_zero: thread 'main' panicked at 'called `Result::unwrap()` on an `Err` value: ErrorImpl { code: Io(Custom { kind: Other, error: "numeric field was not a number: when getting cksum for crate.json" }), offset: 0 }', src/bin/mir-json-dce.rs:38:44 -note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (0.17s) - Compiling and running oracle program (0.15s) - Oracle output: 0 (0.02s) - user error (Error 101 while running mir-json on ["test/conc_eval/vec/from_elem_zero.mir","rlibs/libcore.mir","rlibs/librustc_std_workspace_core.mir","rlibs/liblibc.mir","rlibs/libcompiler_builtins.mir","rlibs/liballoc.mir","rlibs/libcfg_if.mir","rlibs/libmemchr.mir","rlibs/libadler.mir","rlibs/librustc_demangle.mir","rlibs/libunwind.mir","rlibs/libpanic_unwind.mir","rlibs/librustc_std_workspace_alloc.mir","rlibs/libpanic_abort.mir","rlibs/libgimli.mir","rlibs/libstd_detect.mir","rlibs/libobject.mir","rlibs/libminiz_oxide.mir","rlibs/libhashbrown.mir","rlibs/libaddr2line.mir","rlibs/libstd.mir","rlibs/libcrucible.mir","rlibs/libint512.mir","rlibs/libbyteorder.mir","rlibs/libbytes.mir"]) + from_elem_zero: FAIL (0.35s) + Compiling and running oracle program (0.16s) + Oracle output: 0 (0.18s) + user error (Error 101 while running mir-json on test/conc_eval/vec/from_elem_zero.rs) Use -p '/from_elem_zero/' to rerun this test only. - extend: FAIL (0.74s) - Compiling and running oracle program (0.16s) - Oracle output: (1, 10) (0.57s) + extend: FAIL (1.77s) + Compiling and running oracle program (0.20s) + Oracle output: (1, 10) (1.57s) Crux output: failures: - ---- extend/94e78a5b::crux_test[0] counterexamples ---- + ---- extend/d078d8f4::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] lib/core/src/cmp.rs:1300:11: 1300:28: error: in core/7ed35d8a::cmp[0]::max_by[0]::_insta10905add4aa20a5[0] + [Crux] Translation error in core/7ed35d8a::cmp[0]::max_by[0]::_insta10905add4aa20a5[0]: illegal or unimplemented coercion from TyInt USize (concretely BVRepr 32) to TyInt B8 [Crux] Overall status: Invalid. @@ -47,18 +44,16 @@ FAIL (0.17s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..vec.extend"' to rerun this test only. - extend_trusted_len: FAIL (0.85s) - Compiling and running oracle program (0.15s) - Oracle output: (1, 10) (0.70s) + extend_trusted_len: FAIL (1.82s) + Compiling and running oracle program (0.16s) + Oracle output: (1, 10) (1.65s) Crux output: failures: - ---- extend_trusted_len/c2c419e5::crux_test[0] counterexamples ---- + ---- extend_trusted_len/e53a641f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] lib/core/src/cmp.rs:1300:11: 1300:28: error: in core/7ed35d8a::cmp[0]::max_by[0]::_insta10905add4aa20a5[0] + [Crux] Translation error in core/7ed35d8a::cmp[0]::max_by[0]::_insta10905add4aa20a5[0]: illegal or unimplemented coercion from TyInt USize (concretely BVRepr 32) to TyInt B8 [Crux] Overall status: Invalid. @@ -66,18 +61,16 @@ FAIL (0.17s) crux doesn't match oracle Use -p '/extend_trusted_len/' to rerun this test only. - collect: FAIL (0.83s) - Compiling and running oracle program (0.16s) - Oracle output: 45 (0.68s) + collect: FAIL (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: 45 (1.75s) Crux output: failures: - ---- collect/569f54db::crux_test[0] counterexamples ---- + ---- collect/4f2b6e34::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. @@ -85,18 +78,16 @@ FAIL (0.17s) crux doesn't match oracle Use -p '/collect/' to rerun this test only. - push: FAIL (0.76s) - Compiling and running oracle program (0.15s) - Oracle output: (1, 2) (0.62s) + push: FAIL (1.57s) + Compiling and running oracle program (0.14s) + Oracle output: (1, 2) (1.43s) Crux output: failures: - ---- push/c96f3e94::crux_test[0] counterexamples ---- + ---- push/0bc1f3e7::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] lib/core/src/cmp.rs:1300:11: 1300:28: error: in core/7ed35d8a::cmp[0]::max_by[0]::_insta10905add4aa20a5[0] + [Crux] Translation error in core/7ed35d8a::cmp[0]::max_by[0]::_insta10905add4aa20a5[0]: illegal or unimplemented coercion from TyInt USize (concretely BVRepr 32) to TyInt B8 [Crux] Overall status: Invalid. @@ -104,16 +95,16 @@ FAIL (0.17s) crux doesn't match oracle Use -p '/vec.push/' to rerun this test only. - drop: FAIL (0.25s) - Compiling and running oracle program (0.16s) - Oracle output: () (0.09s) + drop: FAIL (1.17s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.02s) Crux output: failures: - ---- drop/c82c6b8e::crux_test[0] counterexamples ---- + ---- drop/034551a4::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. @@ -122,20 +113,20 @@ FAIL (0.17s) Use -p '/drop/' to rerun this test only. dyn - trait_param: OK (0.14s) + trait_param: OK (1.03s) Compiling and running oracle program (0.14s) - Oracle output: 100 + Oracle output: 100 (0.89s) Crux output: 100 - inherit: FAIL (0.14s) - Compiling and running oracle program (0.13s) - Oracle output: 4 (0.01s) + inherit: FAIL (1.04s) + Compiling and running oracle program (0.14s) + Oracle output: 4 (0.90s) Crux output: failures: - ---- inherit/a89b9890::crux_test[0] counterexamples ---- + ---- inherit/79188ef3::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/dyn/inherit.rs:20:13: 20:16: error: in inherit/a89b9890::crux_test[0] - [Crux] Translation error in inherit/a89b9890::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] test/conc_eval/dyn/inherit.rs:20:13: 20:16: error: in inherit/79188ef3::crux_test[0] + [Crux] Translation error in inherit/79188ef3::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. @@ -143,25 +134,25 @@ FAIL (0.17s) crux doesn't match oracle Use -p '/inherit/' to rerun this test only. - plain_trait: OK (0.13s) + plain_trait: OK (1.03s) Compiling and running oracle program (0.13s) - Oracle output: 100 + Oracle output: 100 (0.90s) Crux output: 100 - assoc_ty: OK (0.13s) - Compiling and running oracle program (0.13s) - Oracle output: 100 + assoc_ty: OK (1.07s) + Compiling and running oracle program (0.14s) + Oracle output: 100 (0.93s) Crux output: 100 tuple - clone_from: FAIL (0.15s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.02s) + clone_from: FAIL (1.08s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.95s) Crux output: failures: - ---- clone_from/9cd626ed::f[0] counterexamples ---- + ---- clone_from/c0b06a97::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/clone.rs:136:17: 136:31: error: in core/516ae8d9::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0] - [Crux] Translation error in core/516ae8d9::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0]: don't know how to generate CloneShim for unknown method core/516ae8d9::clone[0]::Clone[0]::clone[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/clone.rs:136:17: 136:31: error: in core/7ed35d8a::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0] + [Crux] Translation error in core/7ed35d8a::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0]: don't know how to generate CloneShim for unknown method core/7ed35d8a::clone[0]::Clone[0]::clone[0] [Crux] Overall status: Invalid. @@ -169,16 +160,16 @@ FAIL (0.17s) crux doesn't match oracle Use -p '/clone_from/' to rerun this test only. - clone_struct: FAIL (0.16s) + clone_struct: FAIL (1.03s) Compiling and running oracle program (0.14s) - Oracle output: () (0.02s) + Oracle output: () (0.90s) Crux output: failures: - ---- clone_struct/9d8703d1::f[0] counterexamples ---- + ---- clone_struct/4ae579ad::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/tuple/clone_struct.rs:8:13: 8:22: error: in clone_struct/9d8703d1::f[0] - [Crux] Translation error in clone_struct/9d8703d1::f[0]: don't know how to generate CloneShim for unknown method core/516ae8d9::clone[0]::Clone[0]::clone[0] + [Crux] test/conc_eval/tuple/clone_struct.rs:8:13: 8:22: error: in clone_struct/4ae579ad::f[0] + [Crux] Translation error in clone_struct/4ae579ad::f[0]: don't know how to generate CloneShim for unknown method core/7ed35d8a::clone[0]::Clone[0]::clone[0] [Crux] Overall status: Invalid. @@ -186,20 +177,20 @@ FAIL (0.17s) crux doesn't match oracle Use -p '/clone_struct/' to rerun this test only. - clone: OK (0.15s) + clone: OK (1.04s) Compiling and running oracle program (0.14s) - Oracle output: () (0.01s) + Oracle output: () (0.90s) Crux output: () - clone_rec: FAIL (0.16s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.02s) + clone_rec: FAIL (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.88s) Crux output: failures: - ---- clone_rec/e47383a3::f[0] counterexamples ---- + ---- clone_rec/9305eb7c::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/tuple/clone_rec.rs:4:17: 4:28: error: in clone_rec/e47383a3::f[0] - [Crux] Translation error in clone_rec/e47383a3::f[0]: fail or unimp constant: TyTuple [TyInt B32,TyInt B32] (StructRepr [MaybeRepr (BVRepr 32), MaybeRepr (BVRepr 32)]) ConstTuple [ConstInt (I32 3),ConstInt (I32 4)] + [Crux] test/conc_eval/tuple/clone_rec.rs:4:17: 4:28: error: in clone_rec/9305eb7c::f[0] + [Crux] Translation error in clone_rec/9305eb7c::f[0]: fail or unimp constant: TyTuple [TyInt B32,TyInt B32] (StructRepr [MaybeRepr (BVRepr 32), MaybeRepr (BVRepr 32)]) ConstTuple [ConstInt (I32 3),ConstInt (I32 4)] [Crux] Overall status: Invalid. @@ -208,18 +199,18 @@ FAIL (0.17s) Use -p '/clone_rec/' to rerun this test only. cell - ref_cell2: FAIL (0.60s) + ref_cell2: FAIL (1.53s) Compiling and running oracle program (0.14s) - Oracle output: 2 (0.46s) + Oracle output: 2 (1.39s) Crux output: failures: - ---- ref_cell2/39dbeea6::crux_test[0] counterexamples ---- + ---- ref_cell2/963774ba::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyInt B32) Mut + [Crux] as: TyRawPtr (TyUint B8) Mut [Crux] Overall status: Invalid. @@ -227,18 +218,18 @@ FAIL (0.17s) crux doesn't match oracle Use -p '/ref_cell2/' to rerun this test only. - ref_cell: FAIL (0.64s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (0.49s) + ref_cell: FAIL (1.52s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.39s) Crux output: failures: - ---- ref_cell/b2662862::crux_test[0] counterexamples ---- + ---- ref_cell/e82ccec1::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyInt B32) Mut + [Crux] as: TyRawPtr (TyUint B8) Mut [Crux] Overall status: Invalid. @@ -246,23 +237,21 @@ FAIL (0.17s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..cell.ref_cell"' to rerun this test only. - cell: OK (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (0.02s) + cell: OK (1.08s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (0.93s) Crux output: 2 str - format: FAIL (0.76s) + format: FAIL (1.71s) Compiling and running oracle program (0.14s) - Oracle output: true (0.62s) + Oracle output: true (1.57s) Crux output: failures: - ---- format/703e8c38::crux_test[0] counterexamples ---- + ---- format/15303e9f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] test/conc_eval/str/format.rs:4:21: 4:27: error: in format/15303e9f::crux_test[0] + [Crux] Translation error in format/15303e9f::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. @@ -270,18 +259,16 @@ FAIL (0.17s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..str.format"' to rerun this test only. - format_hex: FAIL (0.80s) - Compiling and running oracle program (0.15s) - Oracle output: true (0.65s) + format_hex: FAIL (1.68s) + Compiling and running oracle program (0.13s) + Oracle output: true (1.55s) Crux output: failures: - ---- format_hex/a2ffc47f::crux_test[0] counterexamples ---- + ---- format_hex/0a1bfb4d::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] test/conc_eval/str/format_hex.rs:4:21: 4:29: error: in format_hex/0a1bfb4d::crux_test[0] + [Crux] Translation error in format_hex/0a1bfb4d::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. @@ -289,16 +276,16 @@ FAIL (0.17s) crux doesn't match oracle Use -p '/format_hex/' to rerun this test only. - to_owned: FAIL (0.24s) - Compiling and running oracle program (0.14s) - Oracle output: true (0.10s) + to_owned: FAIL (1.15s) + Compiling and running oracle program (0.13s) + Oracle output: true (1.01s) Crux output: failures: - ---- to_owned/7a8109f8::crux_test[0] counterexamples ---- + ---- to_owned/026b6a7a::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/516ae8d9::str[0]::{impl#0}[0]::as_bytes[0] - [Crux] Translation error in core/516ae8d9::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/7ed35d8a::str[0]::{impl#0}[0]::as_bytes[0] + [Crux] Translation error in core/7ed35d8a::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] [Crux] Overall status: Invalid. @@ -306,18 +293,16 @@ FAIL (0.17s) crux doesn't match oracle Use -p '/to_owned/' to rerun this test only. - format_struct: FAIL (0.91s) + format_struct: FAIL (1.78s) Compiling and running oracle program (0.15s) - Oracle output: true (0.76s) + Oracle output: true (1.63s) Crux output: failures: - ---- format_struct/782b6488::crux_test[0] counterexamples ---- + ---- format_struct/b7c0d82c::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] test/conc_eval/str/format_struct.rs:10:21: 10:27: error: in format_struct/b7c0d82c::crux_test[0] + [Crux] Translation error in format_struct/b7c0d82c::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. @@ -325,18 +310,16 @@ FAIL (0.17s) crux doesn't match oracle Use -p '/format_struct/' to rerun this test only. - format_int: FAIL (0.79s) - Compiling and running oracle program (0.14s) - Oracle output: true (0.65s) + format_int: FAIL (1.72s) + Compiling and running oracle program (0.13s) + Oracle output: true (1.58s) Crux output: failures: - ---- format_int/dcc1f157::crux_test[0] counterexamples ---- + ---- format_int/56e6b565::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] test/conc_eval/str/format_int.rs:4:21: 4:27: error: in format_int/56e6b565::crux_test[0] + [Crux] Translation error in format_int/56e6b565::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. @@ -344,18 +327,16 @@ FAIL (0.17s) crux doesn't match oracle Use -p '/format_int/' to rerun this test only. - format_array: FAIL (0.91s) - Compiling and running oracle program (0.15s) - Oracle output: true (0.75s) + format_array: FAIL (1.82s) + Compiling and running oracle program (0.14s) + Oracle output: true (1.68s) Crux output: failures: - ---- format_array/82d51d12::crux_test[0] counterexamples ---- + ---- format_array/3bd38fe7::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] test/conc_eval/str/format_array.rs:4:21: 4:27: error: in format_array/3bd38fe7::crux_test[0] + [Crux] Translation error in format_array/3bd38fe7::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. @@ -363,18 +344,16 @@ FAIL (0.17s) crux doesn't match oracle Use -p '/format_array/' to rerun this test only. - string_push: FAIL (0.71s) - Compiling and running oracle program (0.15s) - Oracle output: true (0.56s) + string_push: FAIL (1.66s) + Compiling and running oracle program (0.14s) + Oracle output: true (1.52s) Crux output: failures: - ---- string_push/3e757ad8::crux_test[0] counterexamples ---- + ---- string_push/82fdec64::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] lib/core/src/cmp.rs:1300:11: 1300:28: error: in core/7ed35d8a::cmp[0]::max_by[0]::_insta10905add4aa20a5[0] + [Crux] Translation error in core/7ed35d8a::cmp[0]::max_by[0]::_insta10905add4aa20a5[0]: illegal or unimplemented coercion from TyInt USize (concretely BVRepr 32) to TyInt B8 [Crux] Overall status: Invalid. @@ -383,16 +362,16 @@ FAIL (0.17s) Use -p '/string_push/' to rerun this test only. iter - cloned: FAIL (0.17s) + cloned: FAIL (1.08s) Compiling and running oracle program (0.14s) - Oracle output: 6 (0.03s) + Oracle output: 6 (0.95s) Crux output: failures: - ---- cloned/3fb92071::crux_test[0] counterexamples ---- + ---- cloned/37b565a2::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0] - [Crux] Translation error in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0]: unimplemented cast: Misc + [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0] + [Crux] Translation error in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyUint USize) Immut [Crux] as: TyRawPtr (TyUint B8) Immut @@ -402,20 +381,20 @@ FAIL (0.17s) crux doesn't match oracle Use -p '/cloned/' to rerun this test only. - from_fn: OK (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.01s) + from_fn: OK (1.02s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.89s) Crux output: () - sum: FAIL (0.18s) + sum: FAIL (1.08s) Compiling and running oracle program (0.14s) - Oracle output: () (0.04s) + Oracle output: () (0.94s) Crux output: failures: - ---- sum/1ed45972::f[0] counterexamples ---- + ---- sum/0909eaa2::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0] - [Crux] Translation error in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0]: unimplemented cast: Misc + [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0] + [Crux] Translation error in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyUint USize) Immut [Crux] as: TyRawPtr (TyUint B8) Immut @@ -425,20 +404,20 @@ FAIL (0.17s) crux doesn't match oracle Use -p '/sum/' to rerun this test only. - loop: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 2 + loop: OK (1.04s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.91s) Crux output: 2 - zip: FAIL (0.19s) + zip: FAIL (1.09s) Compiling and running oracle program (0.14s) - Oracle output: () (0.06s) + Oracle output: () (0.96s) Crux output: failures: - ---- zip/599ebd70::f[0] counterexamples ---- + ---- zip/4609a9d6::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyInt B32) Immut [Crux] as: TyRawPtr (TyUint B8) Immut @@ -448,16 +427,16 @@ FAIL (0.17s) crux doesn't match oracle Use -p '/zip/' to rerun this test only. - for: FAIL (0.17s) - Compiling and running oracle program (0.14s) - Oracle output: 47 (0.03s) + for: FAIL (1.07s) + Compiling and running oracle program (0.13s) + Oracle output: 47 (0.94s) Crux output: failures: - ---- for/90feb64a::crux_test[0] counterexamples ---- + ---- for/3551e9e4::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:470:54: 470:57 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:921:5: 922:101: error: in core/516ae8d9::num[0]::{impl#9}[0]::unchecked_add[0] - [Crux] Translation error in core/516ae8d9::num[0]::{impl#9}[0]::unchecked_add[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_instc5e93708b8ca6e2a[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:470:54: 470:57 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:921:5: 922:101: error: in core/7ed35d8a::num[0]::{impl#9}[0]::unchecked_add[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#9}[0]::unchecked_add[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_instc5e93708b8ca6e2a[0] [Crux] Overall status: Invalid. @@ -465,16 +444,16 @@ FAIL (0.17s) crux doesn't match oracle Use -p '/iter.for/' to rerun this test only. - peek: FAIL (0.20s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (0.05s) + peek: FAIL (1.11s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.98s) Crux output: failures: - ---- peek/1909e1c9::crux_test[0] counterexamples ---- + ---- peek/550fdb93::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0] - [Crux] Translation error in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0]: unimplemented cast: Misc + [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0] + [Crux] Translation error in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyUint USize) Immut [Crux] as: TyRawPtr (TyUint B8) Immut @@ -484,15 +463,15 @@ FAIL (0.17s) crux doesn't match oracle Use -p '/peek/' to rerun this test only. - filter_chain: OK (0.24s) - Compiling and running oracle program (0.16s) - Oracle output: 0 (0.08s) + filter_chain: OK (1.18s) + Compiling and running oracle program (0.15s) + Oracle output: 0 (1.03s) Crux output: 0 hash_map - insert_remove: FAIL (0.99s) + insert_remove: FAIL (2.10s) Compiling and running oracle program (0.17s) - Oracle output: 12 (0.81s) - vtable signature mismatch for vtable core/516ae8d9::ops[0]::function[0]::FnMut[0]::_vtblc2e637e13369cbd5[0] of trait core/516ae8d9::ops[0]::function[0]::FnMut[0]::_trait35b29a8a4ae9e473[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, BVRepr 32] BoolRepr] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr] + Oracle output: 12 (1.94s) + vtable signature mismatch for vtable core/7ed35d8a::ops[0]::function[0]::FnMut[0]::_vtblc2e637e13369cbd5[0] of trait core/7ed35d8a::ops[0]::function[0]::FnMut[0]::_trait35b29a8a4ae9e473[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, BVRepr 32] BoolRepr] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr] CallStack (from HasCallStack): error, called at src/Mir/Trans.hs:904:20 in crux-mir-0.6.0.99-inplace:Mir.Trans mkTraitObject, called at src/Mir/Trans.hs:747:11 in crux-mir-0.6.0.99-inplace:Mir.Trans @@ -508,10 +487,10 @@ FAIL (0.17s) translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '/insert_remove/' to rerun this test only. - insert_iter: FAIL (0.97s) - Compiling and running oracle program (0.18s) - Oracle output: [11, 12] (0.79s) - vtable signature mismatch for vtable core/516ae8d9::ops[0]::function[0]::FnMut[0]::_vtblc2e637e13369cbd5[0] of trait core/516ae8d9::ops[0]::function[0]::FnMut[0]::_trait35b29a8a4ae9e473[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, BVRepr 32] BoolRepr] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr] + insert_iter: FAIL (2.06s) + Compiling and running oracle program (0.17s) + Oracle output: [11, 12] (1.89s) + vtable signature mismatch for vtable core/7ed35d8a::ops[0]::function[0]::FnMut[0]::_vtblc2e637e13369cbd5[0] of trait core/7ed35d8a::ops[0]::function[0]::FnMut[0]::_trait35b29a8a4ae9e473[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, BVRepr 32] BoolRepr] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr] CallStack (from HasCallStack): error, called at src/Mir/Trans.hs:904:20 in crux-mir-0.6.0.99-inplace:Mir.Trans mkTraitObject, called at src/Mir/Trans.hs:747:11 in crux-mir-0.6.0.99-inplace:Mir.Trans @@ -527,10 +506,10 @@ FAIL (0.17s) translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '/insert_iter/' to rerun this test only. - insert_multi: FAIL (0.98s) + insert_multi: FAIL (2.04s) Compiling and running oracle program (0.17s) - Oracle output: 100 (0.80s) - vtable signature mismatch for vtable core/516ae8d9::ops[0]::function[0]::FnMut[0]::_vtblc2e637e13369cbd5[0] of trait core/516ae8d9::ops[0]::function[0]::FnMut[0]::_trait35b29a8a4ae9e473[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, BVRepr 32] BoolRepr] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr] + Oracle output: 100 (1.87s) + vtable signature mismatch for vtable core/7ed35d8a::ops[0]::function[0]::FnMut[0]::_vtblc2e637e13369cbd5[0] of trait core/7ed35d8a::ops[0]::function[0]::FnMut[0]::_trait35b29a8a4ae9e473[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, BVRepr 32] BoolRepr] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr] CallStack (from HasCallStack): error, called at src/Mir/Trans.hs:904:20 in crux-mir-0.6.0.99-inplace:Mir.Trans mkTraitObject, called at src/Mir/Trans.hs:747:11 in crux-mir-0.6.0.99-inplace:Mir.Trans @@ -546,10 +525,10 @@ FAIL (0.17s) translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '/insert_multi/' to rerun this test only. - insert_get: FAIL (0.92s) - Compiling and running oracle program (0.17s) - Oracle output: (11, 12) (0.75s) - vtable signature mismatch for vtable core/516ae8d9::ops[0]::function[0]::FnMut[0]::_vtblc2e637e13369cbd5[0] of trait core/516ae8d9::ops[0]::function[0]::FnMut[0]::_trait35b29a8a4ae9e473[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, BVRepr 32] BoolRepr] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr] + insert_get: FAIL (2.02s) + Compiling and running oracle program (0.16s) + Oracle output: (11, 12) (1.86s) + vtable signature mismatch for vtable core/7ed35d8a::ops[0]::function[0]::FnMut[0]::_vtblc2e637e13369cbd5[0] of trait core/7ed35d8a::ops[0]::function[0]::FnMut[0]::_trait35b29a8a4ae9e473[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, BVRepr 32] BoolRepr] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr] CallStack (from HasCallStack): error, called at src/Mir/Trans.hs:904:20 in crux-mir-0.6.0.99-inplace:Mir.Trans mkTraitObject, called at src/Mir/Trans.hs:747:11 in crux-mir-0.6.0.99-inplace:Mir.Trans @@ -566,20 +545,20 @@ FAIL (0.17s) Use -p '/insert_get/' to rerun this test only. enum - cmp: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: -23 + cmp: OK (1.08s) + Compiling and running oracle program (0.13s) + Oracle output: -23 (0.95s) Crux output: -23 - eq: FAIL (0.16s) + eq: FAIL (1.05s) Compiling and running oracle program (0.14s) - Oracle output: () (0.02s) + Oracle output: () (0.91s) Crux output: failures: - ---- eq/41d79a98::crux_test[0] counterexamples ---- + ---- eq/0bcad3d7::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/enum/eq.rs:9:17: 9:21: error: in eq/41d79a98::f[0] - [Crux] Translation error in eq/41d79a98::f[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] test/conc_eval/enum/eq.rs:9:17: 9:21: error: in eq/0bcad3d7::f[0] + [Crux] Translation error in eq/0bcad3d7::f[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. @@ -609,22 +588,22 @@ error: aborting due to 2 previous errors For more information about this error, try `rustc --explain E0518`. -FAIL (0.05s) - Compiling and running oracle program (0.05s) +FAIL (0.04s) + Compiling and running oracle program (0.04s) test/Test.hs:109: failed to compile and run Use -p '/arg2/' to rerun this test only. - cow: FAIL (0.24s) + cow: FAIL (1.16s) Compiling and running oracle program (0.14s) - Oracle output: 200 (0.10s) + Oracle output: 200 (1.02s) Crux output: failures: - ---- cow/57074db5::crux_test[0] counterexamples ---- + ---- cow/5077b6e9::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/516ae8d9::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/516ae8d9::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/7ed35d8a::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/7ed35d8a::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -632,25 +611,25 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/cow/' to rerun this test only. - field_order: OK (0.14s) + field_order: OK (1.01s) Compiling and running oracle program (0.13s) - Oracle output: () + Oracle output: () (0.88s) Crux output: () - arg: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 0 + arg: OK (1.02s) + Compiling and running oracle program (0.13s) + Oracle output: 0 (0.88s) Crux output: 0 - ret: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: (A(42), B { x: 42 }, C) + ret: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: (A(42), B { x: 42 }, C) (0.87s) Crux output: (A(42), B { x: 42 }, C) - match: OK (0.13s) - Compiling and running oracle program (0.13s) - Oracle output: 42 + match: OK (1.02s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (0.89s) Crux output: 42 - inner: OK (0.13s) + inner: OK (1.01s) Compiling and running oracle program (0.13s) - Oracle output: 4 + Oracle output: 4 (0.88s) Crux output: 4 mixed_discrs: rustc compilation failed for mixed_discrs error output: @@ -664,29 +643,27 @@ error: aborting due to previous error For more information about this error, try `rustc --explain E0554`. -FAIL (0.06s) - Compiling and running oracle program (0.06s) +FAIL (0.05s) + Compiling and running oracle program (0.05s) test/Test.hs:109: failed to compile and run Use -p '/mixed_discrs/' to rerun this test only. array - wick2: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: true + wick2: OK (1.04s) + Compiling and running oracle program (0.13s) + Oracle output: true (0.91s) Crux output: true - from_slice: FAIL (0.71s) + from_slice: FAIL (1.58s) Compiling and running oracle program (0.13s) - Oracle output: () (0.58s) + Oracle output: () (1.45s) Crux output: failures: - ---- from_slice/e616df27::f[0] counterexamples ---- + ---- from_slice/cc2b9543::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] internal: error: in core/7ed35d8a::array[0]::{impl#10}[0]::try_from[0]::_instb3c76ad3dd4c5256[0] + [Crux] Translation error in core/7ed35d8a::array[0]::{impl#10}[0]::try_from[0]::_instb3c76ad3dd4c5256[0]: error building variant core/7ed35d8a::result[0]::Result[0]::Err[0]: not enough expressions -- [] [Crux] Overall status: Invalid. @@ -694,16 +671,16 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/array.from_slice/' to rerun this test only. - clone: FAIL (0.16s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.02s) + clone: FAIL (1.08s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.95s) Crux output: failures: - ---- clone/12c743af::f[0] counterexamples ---- + ---- clone/fccb4b75::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:61:9: 61:18: error: in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0] - [Crux] Translation error in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0]: unimplemented cast: Misc + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:61:9: 61:18: error: in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0] + [Crux] Translation error in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyInt B32) Immut [Crux] as: TyRawPtr (TyArray (TyInt B32) 3) Immut @@ -713,18 +690,16 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/array.clone/' to rerun this test only. - mut_index: FAIL (0.55s) - Compiling and running oracle program (0.15s) - Oracle output: 7 (0.40s) + mut_index: FAIL (1.46s) + Compiling and running oracle program (0.13s) + Oracle output: 7 (1.32s) Crux output: failures: - ---- mut_index/9c08c2ae::crux_test[0] counterexamples ---- + ---- mut_index/db5c06f4::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/7ed35d8a::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/7ed35d8a::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -732,25 +707,25 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/mut_index/' to rerun this test only. - wick1: FAIL (expected: needs Vec data structure from stdlib) (0.19s) - Compiling and running oracle program (0.15s) - Oracle output: true (0.04s) + wick1: FAIL (expected: needs Vec data structure from stdlib) (0.25s) + Compiling and running oracle program (0.14s) + Oracle output: true (0.11s) user error (Error 101 while running mir-json on test/conc_eval/array/wick1.rs) (expected failure) - arg: OK (0.14s) + arg: OK (1.04s) Compiling and running oracle program (0.14s) - Oracle output: 2 + Oracle output: 2 (0.90s) Crux output: 2 - iter: FAIL (0.18s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (0.04s) + iter: FAIL (1.06s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.93s) Crux output: failures: - ---- iter/d416c50d::crux_test[0] counterexamples ---- + ---- iter/f5646cc2::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyInt B32) Immut [Crux] as: TyRawPtr (TyUint B8) Immut @@ -760,87 +735,87 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/array.iter/' to rerun this test only. - wick3: FAIL (expected: needs Vec data structure from stdlib) (0.19s) - Compiling and running oracle program (0.15s) + wick3: FAIL (expected: needs Vec data structure from stdlib) (0.18s) + Compiling and running oracle program (0.14s) Oracle output: true (0.04s) user error (Error 101 while running mir-json on test/conc_eval/array/wick3.rs) (expected failure) - mk_and_proj: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 42 + mk_and_proj: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.85s) Crux output: 42 - mut_arg: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 42 + mut_arg: OK (1.05s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.92s) Crux output: 42 - const: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 1 + const: OK (1.02s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.89s) Crux output: 1 - const_impl: OK (0.14s) + const_impl: OK (1.02s) Compiling and running oracle program (0.13s) - Oracle output: 5 + Oracle output: 5 (0.89s) Crux output: 5 stdlib - option: OK (0.14s) + option: OK (1.01s) Compiling and running oracle program (0.13s) - Oracle output: true + Oracle output: true (0.88s) Crux output: true - range: OK (0.13s) + range: OK (1.04s) Compiling and running oracle program (0.13s) - Oracle output: 10 + Oracle output: 10 (0.91s) Crux output: 10 - default_impl: OK (0.13s) + default_impl: OK (1.02s) Compiling and running oracle program (0.13s) - Oracle output: () + Oracle output: () (0.89s) Crux output: () - teq: OK (0.13s) - Compiling and running oracle program (0.13s) - Oracle output: false + teq: OK (1.01s) + Compiling and running oracle program (0.15s) + Oracle output: false (0.86s) Crux output: false - option3: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 27 + option3: OK (1.03s) + Compiling and running oracle program (0.13s) + Oracle output: 27 (0.90s) Crux output: 27 - result: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 27 + result: OK (1.03s) + Compiling and running oracle program (0.13s) + Oracle output: 27 (0.90s) Crux output: 27 - poly: OK (0.14s) + poly: OK (1.04s) Compiling and running oracle program (0.13s) - Oracle output: 1 + Oracle output: 1 (0.91s) Crux output: 1 - default: OK (0.14s) + default: OK (1.02s) Compiling and running oracle program (0.14s) - Oracle output: true + Oracle output: true (0.87s) Crux output: true - cvt: OK (0.13s) + cvt: OK (1.06s) Compiling and running oracle program (0.13s) - Oracle output: 0 + Oracle output: 0 (0.93s) Crux output: 0 - result_interior: OK (0.14s) - Compiling and running oracle program (0.13s) - Oracle output: 3 + result_interior: OK (1.04s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (0.90s) Crux output: 3 - option2: OK (0.13s) - Compiling and running oracle program (0.13s) - Oracle output: 0 + option2: OK (1.04s) + Compiling and running oracle program (0.15s) + Oracle output: 0 (0.89s) Crux output: 0 intTest - test0039: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 7 + test0039: OK (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: 7 (0.89s) Crux output: 7 - test0038: FAIL (0.19s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.04s) + test0038: FAIL (1.09s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.96s) Crux output: failures: - ---- test0038/f7eb1122::crux_test[0] counterexamples ---- + ---- test0038/ee52fa9c::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/516ae8d9::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst5dab88813bb37c92[0] - [Crux] Translation error in core/516ae8d9::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst5dab88813bb37c92[0]: unimplemented cast: Misc + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst5dab88813bb37c92[0] + [Crux] Translation error in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst5dab88813bb37c92[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyArray (TyInt B32) 4) Mut [Crux] as: TyRawPtr (TyUint B8) Mut @@ -851,16 +826,16 @@ FAIL (0.06s) Use -p '/test0038/' to rerun this test only. box - new: FAIL (0.20s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.06s) + new: FAIL (1.15s) + Compiling and running oracle program (0.13s) + Oracle output: () (1.02s) Crux output: failures: - ---- new/2d77246e::f[0] counterexamples ---- + ---- new/825ce802::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. @@ -880,22 +855,22 @@ error: aborting due to previous error For more information about this error, try `rustc --explain E0554`. -FAIL (0.05s) - Compiling and running oracle program (0.05s) +FAIL (0.04s) + Compiling and running oracle program (0.04s) test/Test.hs:109: failed to compile and run Use -p '/box.struct/' to rerun this test only. - unsize: FAIL (0.21s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (0.06s) + unsize: FAIL (1.09s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.96s) Crux output: failures: - ---- unsize/090d8d2d::f[0] counterexamples ---- + ---- unsize/ad9f3518::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. @@ -903,16 +878,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/box.unsize/' to rerun this test only. - mut_ref: FAIL (0.20s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.06s) + mut_ref: FAIL (1.14s) + Compiling and running oracle program (0.13s) + Oracle output: () (1.01s) Crux output: failures: - ---- mut_ref/e4e7f393::f[0] counterexamples ---- + ---- mut_ref/b1bd8ce3::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. @@ -920,16 +895,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/box.mut_ref/' to rerun this test only. - mut: FAIL (0.20s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.06s) + mut: FAIL (1.17s) + Compiling and running oracle program (0.13s) + Oracle output: () (1.05s) Crux output: failures: - ---- mut/86109469::f[0] counterexamples ---- + ---- mut/38683331::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. @@ -938,10 +913,10 @@ FAIL (0.05s) Use -p '$0=="crux-mir.crux concrete..box.mut"' to rerun this test only. sync - rwlock: FAIL (0.95s) + rwlock: FAIL (1.90s) Compiling and running oracle program (0.14s) - Oracle output: 2 (0.80s) - standalone use of `dyn` is not supported: TyDynamic core/516ae8d9::error[0]::Error[0]::_trait56353a7b840403b4[0] + Oracle output: 2 (1.76s) + standalone use of `dyn` is not supported: TyDynamic core/7ed35d8a::error[0]::Error[0]::_trait56353a7b840403b4[0] CallStack (from HasCallStack): error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans @@ -957,16 +932,16 @@ FAIL (0.05s) translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '$0=="crux-mir.crux concrete..sync.rwlock"' to rerun this test only. - atomic_cxchg: FAIL (0.19s) + atomic_cxchg: FAIL (1.14s) Compiling and running oracle program (0.15s) - Oracle output: 6 (0.04s) + Oracle output: 6 (0.99s) Crux output: failures: - ---- atomic_cxchg/b55893dc::crux_test[0] counterexamples ---- + ---- atomic_cxchg/4bcd370f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/sync/atomic_cxchg.rs:8:22: 8:66: error: in atomic_cxchg/b55893dc::crux_test[0] - [Crux] Translation error in atomic_cxchg/b55893dc::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] test/conc_eval/sync/atomic_cxchg.rs:8:22: 8:66: error: in atomic_cxchg/4bcd370f::crux_test[0] + [Crux] Translation error in atomic_cxchg/4bcd370f::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. @@ -974,16 +949,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/atomic_cxchg/' to rerun this test only. - atomic_fence: FAIL (0.16s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (0.02s) + atomic_fence: FAIL (1.08s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.95s) Crux output: failures: - ---- atomic_fence/f75b8912::crux_test[0] counterexamples ---- + ---- atomic_fence/a8477cc6::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/sync/atomic.rs:3010:60: 3010:63: error: in core/516ae8d9::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/516ae8d9::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::atomic_store_seqcst[0]::_inst1e2825177cd3b608[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/sync/atomic.rs:3010:60: 3010:63: error: in core/7ed35d8a::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/7ed35d8a::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::atomic_store_seqcst[0]::_inst1e2825177cd3b608[0] [Crux] Overall status: Invalid. @@ -991,16 +966,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/atomic_fence/' to rerun this test only. - atomic_add: FAIL (0.18s) + atomic_add: FAIL (1.13s) Compiling and running oracle program (0.14s) - Oracle output: 6 (0.04s) + Oracle output: 6 (0.99s) Crux output: failures: - ---- atomic_add/f2b8808e::crux_test[0] counterexamples ---- + ---- atomic_add/4b66494a::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/sync/atomic_add.rs:8:22: 8:66: error: in atomic_add/f2b8808e::crux_test[0] - [Crux] Translation error in atomic_add/f2b8808e::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] test/conc_eval/sync/atomic_add.rs:8:22: 8:66: error: in atomic_add/4b66494a::crux_test[0] + [Crux] Translation error in atomic_add/4b66494a::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. @@ -1008,16 +983,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/atomic_add/' to rerun this test only. - arc: FAIL (0.21s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (0.07s) + arc: FAIL (1.17s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (1.04s) Crux output: failures: - ---- arc/2c43be88::crux_test[0] counterexamples ---- + ---- arc/b5204711::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. @@ -1025,10 +1000,10 @@ FAIL (0.05s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..sync.arc"' to rerun this test only. - rwlock_multi: FAIL (1.05s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (0.90s) - standalone use of `dyn` is not supported: TyDynamic core/516ae8d9::error[0]::Error[0]::_trait56353a7b840403b4[0] + rwlock_multi: FAIL (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.74s) + standalone use of `dyn` is not supported: TyDynamic core/7ed35d8a::error[0]::Error[0]::_trait56353a7b840403b4[0] CallStack (from HasCallStack): error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans @@ -1044,16 +1019,16 @@ FAIL (0.05s) translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '/rwlock_multi/' to rerun this test only. - arc_clone: FAIL (0.23s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (0.08s) + arc_clone: FAIL (1.13s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (1.00s) Crux output: failures: - ---- arc_clone/e8f267c7::crux_test[0] counterexamples ---- + ---- arc_clone/b9b1f82e::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. @@ -1061,10 +1036,10 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/arc_clone/' to rerun this test only. - mutex: FAIL (0.93s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (0.78s) - standalone use of `dyn` is not supported: TyDynamic core/516ae8d9::error[0]::Error[0]::_trait56353a7b840403b4[0] + mutex: FAIL (1.92s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (1.79s) + standalone use of `dyn` is not supported: TyDynamic core/7ed35d8a::error[0]::Error[0]::_trait56353a7b840403b4[0] CallStack (from HasCallStack): error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans @@ -1080,16 +1055,16 @@ FAIL (0.05s) translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '$0=="crux-mir.crux concrete..sync.mutex"' to rerun this test only. - arc_cell: FAIL (0.23s) - Compiling and running oracle program (0.15s) - Oracle output: 4 (0.08s) + arc_cell: FAIL (1.17s) + Compiling and running oracle program (0.14s) + Oracle output: 4 (1.03s) Crux output: failures: - ---- arc_cell/ae087adc::crux_test[0] counterexamples ---- + ---- arc_cell/9fa1118b::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. @@ -1097,10 +1072,10 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/arc_cell/' to rerun this test only. - mutex_multi: FAIL (0.97s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (0.82s) - standalone use of `dyn` is not supported: TyDynamic core/516ae8d9::error[0]::Error[0]::_trait56353a7b840403b4[0] + mutex_multi: FAIL (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.76s) + standalone use of `dyn` is not supported: TyDynamic core/7ed35d8a::error[0]::Error[0]::_trait56353a7b840403b4[0] CallStack (from HasCallStack): error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans @@ -1116,16 +1091,16 @@ FAIL (0.05s) translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '/mutex_multi/' to rerun this test only. - atomic_swap: FAIL (0.16s) - Compiling and running oracle program (0.14s) - Oracle output: (2, 1) (0.02s) + atomic_swap: FAIL (1.08s) + Compiling and running oracle program (0.13s) + Oracle output: (2, 1) (0.95s) Crux output: failures: - ---- atomic_swap/bed7fbdb::crux_test[0] counterexamples ---- + ---- atomic_swap/6d7e3f7b::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/sync/atomic.rs:3010:60: 3010:63: error: in core/516ae8d9::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/516ae8d9::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::atomic_store_seqcst[0]::_inst1e2825177cd3b608[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/sync/atomic.rs:3010:60: 3010:63: error: in core/7ed35d8a::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/7ed35d8a::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::atomic_store_seqcst[0]::_inst1e2825177cd3b608[0] [Crux] Overall status: Invalid. @@ -1160,22 +1135,22 @@ error: aborting due to previous error; 2 warnings emitted For more information about this error, try `rustc --explain E0554`. -FAIL (0.06s) - Compiling and running oracle program (0.06s) +FAIL (0.04s) + Compiling and running oracle program (0.04s) test/Test.hs:109: failed to compile and run Use -p '$0=="crux-mir.crux concrete..slice.range_len"' to rerun this test only. - iter_mut: FAIL (0.17s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.03s) + iter_mut: FAIL (1.14s) + Compiling and running oracle program (0.13s) + Oracle output: () (1.01s) Crux output: failures: - ---- iter_mut/984cc64c::f[0] counterexamples ---- + ---- iter_mut/51d61b89::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/516ae8d9::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/516ae8d9::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyInt B32) Mut [Crux] as: TyRawPtr (TyUint B8) Mut @@ -1185,16 +1160,16 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/iter_mut/' to rerun this test only. - eq: FAIL (0.15s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.02s) + eq: FAIL (1.09s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.95s) Crux output: failures: - ---- eq/9b499933::f[0] counterexamples ---- + ---- eq/2c663db8::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/mem/mod.rs:338:38: 338:41: error: in core/516ae8d9::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/516ae8d9::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::size_of_val[0]::_inst55af81a2cef2701c[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/mem/mod.rs:338:38: 338:41: error: in core/7ed35d8a::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/7ed35d8a::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::size_of_val[0]::_inst55af81a2cef2701c[0] [Crux] Overall status: Invalid. @@ -1202,26 +1177,24 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/slice.eq/' to rerun this test only. - swap: OK (0.16s) + swap: OK (1.07s) Compiling and running oracle program (0.14s) - Oracle output: 2001 (0.02s) + Oracle output: 2001 (0.93s) Crux output: 2001 - last: OK (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (0.01s) + last: OK (1.08s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.95s) Crux output: 3 - mut_range: FAIL (0.62s) - Compiling and running oracle program (0.14s) - Oracle output: 86 (0.48s) + mut_range: FAIL (1.51s) + Compiling and running oracle program (0.13s) + Oracle output: 86 (1.38s) Crux output: failures: - ---- mut_range/b300ab62::crux_test[0] counterexamples ---- + ---- mut_range/3c8d2622::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/7ed35d8a::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/7ed35d8a::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1229,9 +1202,9 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/mut_range/' to rerun this test only. - mk_and_proj: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 42 + mk_and_proj: OK (1.05s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.92s) Crux output: 42 get: rustc compilation failed for get error output: @@ -1245,15 +1218,15 @@ error: aborting due to previous error For more information about this error, try `rustc --explain E0554`. -FAIL (0.05s) - Compiling and running oracle program (0.05s) +FAIL (0.04s) + Compiling and running oracle program (0.04s) test/Test.hs:109: failed to compile and run Use -p '/slice.get/' to rerun this test only. - len: OK (0.13s) + len: OK (1.01s) Compiling and running oracle program (0.13s) - Oracle output: 5 + Oracle output: 5 (0.88s) Crux output: 5 range_len_mut: rustc compilation failed for range_len_mut error output: @@ -1281,28 +1254,28 @@ error: aborting due to previous error; 2 warnings emitted For more information about this error, try `rustc --explain E0554`. -FAIL (0.05s) - Compiling and running oracle program (0.05s) +FAIL (0.04s) + Compiling and running oracle program (0.04s) test/Test.hs:109: failed to compile and run Use -p '/range_len_mut/' to rerun this test only. - mut: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 42 + mut: OK (1.06s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.93s) Crux output: 42 clos - fnptr_fn: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 4 + fnptr_fn: OK (1.07s) + Compiling and running oracle program (0.13s) + Oracle output: 4 (0.94s) Crux output: 4 - fnonce1: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 3 + fnonce1: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.87s) Crux output: 3 - conv_fnonce_fn: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 2 + conv_fnonce_fn: OK (1.03s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.90s) Crux output: 2 fo: rustc compilation failed for fo error output: @@ -1329,42 +1302,42 @@ error: aborting due to previous error; 1 warning emitted For more information about this error, try `rustc --explain E0554`. -FAIL (0.05s) - Compiling and running oracle program (0.05s) +FAIL (0.04s) + Compiling and running oracle program (0.04s) test/Test.hs:109: failed to compile and run Use -p '/clos.fo/' to rerun this test only. - direct_fn: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 2 + direct_fn: OK (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.88s) Crux output: 2 - unique_borrow: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 3 + unique_borrow: OK (1.09s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.96s) Crux output: 3 - fn_static: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 3 + fn_static: OK (1.07s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.93s) Crux output: 3 - fnptr_fnonce: OK (0.14s) + fnptr_fnonce: OK (1.07s) Compiling and running oracle program (0.14s) - Oracle output: 4 + Oracle output: 4 (0.93s) Crux output: 4 - direct_fnonce: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 2 + direct_fnonce: OK (1.06s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.93s) Crux output: 2 - dispatch_fnmut: FAIL (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: 7 (0.02s) + dispatch_fnmut: FAIL (1.10s) + Compiling and running oracle program (0.13s) + Oracle output: 7 (0.97s) Crux output: failures: - ---- dispatch_fnmut/3d7553f9::crux_test[0] counterexamples ---- + ---- dispatch_fnmut/3f7bd57d::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/clos/dispatch_fnmut.rs:7:7: 7:9: error: in dispatch_fnmut/3d7553f9::call_it2[0]::_inst15715dbe1d0521ca[0] - [Crux] Translation error in dispatch_fnmut/3d7553f9::call_it2[0]::_inst15715dbe1d0521ca[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] test/conc_eval/clos/dispatch_fnmut.rs:7:7: 7:9: error: in dispatch_fnmut/3f7bd57d::call_it2[0]::_inst15715dbe1d0521ca[0] + [Crux] Translation error in dispatch_fnmut/3f7bd57d::call_it2[0]::_inst15715dbe1d0521ca[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. @@ -1372,41 +1345,41 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/dispatch_fnmut/' to rerun this test only. - conv_fnmut_fn: OK (0.16s) - Compiling and running oracle program (0.16s) - Oracle output: 2 + conv_fnmut_fn: OK (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.88s) Crux output: 2 - fnptr_fnmut: OK (0.16s) - Compiling and running oracle program (0.15s) - Oracle output: 4 - Crux output: 4 - poly: OK (0.14s) + fnptr_fnmut: OK (1.08s) Compiling and running oracle program (0.14s) - Oracle output: 3 + Oracle output: 4 (0.94s) + Crux output: 4 + poly: OK (1.02s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.89s) Crux output: 3 - conv_fnonce_fnmut: OK (0.17s) - Compiling and running oracle program (0.17s) - Oracle output: 2 + conv_fnonce_fnmut: OK (1.07s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.94s) Crux output: 2 fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (0.16s) Compiling and running oracle program (0.13s) - Oracle output: 3 (0.02s) + Oracle output: 3 (0.03s) user error (Error 101 while running mir-json on test/conc_eval/clos/fn_dyn.rs) (expected failure) - ref_fnmut: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 7 + ref_fnmut: OK (1.07s) + Compiling and running oracle program (0.13s) + Oracle output: 7 (0.94s) Crux output: 7 - as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (0.01s) + as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (1.01s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (0.89s) Crux output: failures: - ---- as_fn_ptr/989685c7::crux_test[0] counterexamples ---- + ---- as_fn_ptr/d4076aaa::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/989685c7::crux_test[0] - [Crux] Translation error in as_fn_ptr/989685c7::crux_test[0]: unimplemented cast: ClosureFnPointer + [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/d4076aaa::crux_test[0] + [Crux] Translation error in as_fn_ptr/d4076aaa::crux_test[0]: unimplemented cast: ClosureFnPointer [Crux] ty: TyClosure [] [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [], _fsreturn_ty = TyInt B32, _fsabi = RustAbi, _fsspreadarg = Nothing}) @@ -1415,20 +1388,20 @@ FAIL (0.05s) test/Test.hs:126: crux doesn't match oracle (expected failure) - fnonce: OK (0.13s) - Compiling and running oracle program (0.13s) - Oracle output: false + fnonce: OK (1.04s) + Compiling and running oracle program (0.12s) + Oracle output: false (0.91s) Crux output: false - promoted: FAIL (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: 33 (0.01s) + promoted: FAIL (1.08s) + Compiling and running oracle program (0.12s) + Oracle output: 33 (0.96s) Crux output: failures: - ---- promoted/1f26acf5::crux_test[0] counterexamples ---- + ---- promoted/599dd748::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/clos/promoted.rs:15:9: 15:11: error: in promoted/1f26acf5::f[0] - [Crux] Translation error in promoted/1f26acf5::f[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] test/conc_eval/clos/promoted.rs:15:9: 15:11: error: in promoted/599dd748::f[0] + [Crux] Translation error in promoted/599dd748::f[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. @@ -1436,16 +1409,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/clos.promoted/' to rerun this test only. - fnptr_closure: FAIL (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: 7 (0.02s) + fnptr_closure: FAIL (1.06s) + Compiling and running oracle program (0.13s) + Oracle output: 7 (0.92s) Crux output: failures: - ---- fnptr_closure/e8a8c208::crux_test[0] counterexamples ---- + ---- fnptr_closure/af787938::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/clos/fnptr_closure.rs:7:7: 7:9: error: in fnptr_closure/e8a8c208::call_it2[0]::_inste366449525d6d5d6[0] - [Crux] Translation error in fnptr_closure/e8a8c208::call_it2[0]::_inste366449525d6d5d6[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] test/conc_eval/clos/fnptr_closure.rs:7:7: 7:9: error: in fnptr_closure/af787938::call_it2[0]::_inste366449525d6d5d6[0] + [Crux] Translation error in fnptr_closure/af787938::call_it2[0]::_inste366449525d6d5d6[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. @@ -1453,29 +1426,29 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/fnptr_closure/' to rerun this test only. - direct_fnmut2: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 7 - Crux output: 7 - direct_fnmut: OK (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: 7 + direct_fnmut2: OK (1.07s) + Compiling and running oracle program (0.13s) + Oracle output: 7 (0.94s) Crux output: 7 - fn_static_poly: OK (0.14s) + direct_fnmut: OK (1.06s) Compiling and running oracle program (0.13s) - Oracle output: 3 + Oracle output: 7 (0.94s) + Crux output: 7 + fn_static_poly: OK (1.07s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (0.93s) Crux output: 3 prim - ffs: FAIL (0.18s) - Compiling and running oracle program (0.14s) - Oracle output: true (0.03s) + ffs: FAIL (1.17s) + Compiling and running oracle program (0.15s) + Oracle output: true (1.02s) Crux output: failures: - ---- ffs/2effae63::crux_test[0] counterexamples ---- + ---- ffs/61eaece5::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/int_macros.rs:460:54: 460:57 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:251:5: 253:45: error: in core/516ae8d9::num[0]::{impl#3}[0]::unchecked_add[0] - [Crux] Translation error in core/516ae8d9::num[0]::{impl#3}[0]::unchecked_add[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_inst1e2825177cd3b608[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/int_macros.rs:460:54: 460:57 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:251:5: 253:45: error: in core/7ed35d8a::num[0]::{impl#3}[0]::unchecked_add[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#3}[0]::unchecked_add[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_inst1e2825177cd3b608[0] [Crux] Overall status: Invalid. @@ -1483,20 +1456,20 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/prim.ffs/' to rerun this test only. - add1: OK (0.14s) + add1: OK (0.99s) Compiling and running oracle program (0.14s) - Oracle output: 2 + Oracle output: 2 (0.86s) Crux output: 2 - litstring: FAIL (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: true (0.01s) + litstring: FAIL (1.02s) + Compiling and running oracle program (0.13s) + Oracle output: true (0.90s) Crux output: failures: - ---- litstring/07909131::crux_test[0] counterexamples ---- + ---- litstring/aac23d18::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/516ae8d9::str[0]::{impl#0}[0]::as_bytes[0] - [Crux] Translation error in core/516ae8d9::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/7ed35d8a::str[0]::{impl#0}[0]::as_bytes[0] + [Crux] Translation error in core/7ed35d8a::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] [Crux] Overall status: Invalid. @@ -1504,28 +1477,28 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/litstring/' to rerun this test only. - wrapping_sub: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: true + wrapping_sub: OK (1.05s) + Compiling and running oracle program (0.13s) + Oracle output: true (0.92s) Crux output: true - shift2: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 2 + shift2: OK (1.03s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.90s) Crux output: 2 - shift3: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: -9223372036854775808 + shift3: OK (1.00s) + Compiling and running oracle program (0.12s) + Oracle output: -9223372036854775808 (0.87s) Crux output: -9223372036854775808 - char_from_u32: FAIL (0.16s) - Compiling and running oracle program (0.14s) - Oracle output: 'A' (0.02s) + char_from_u32: FAIL (1.07s) + Compiling and running oracle program (0.12s) + Oracle output: 'A' (0.95s) Crux output: failures: - ---- char_from_u32/3fda601b::crux_test[0] counterexamples ---- + ---- char_from_u32/2f7600db::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/char/convert.rs:215:31: 215:32: error: in core/516ae8d9::char[0]::convert[0]::char_try_from_u32[0] - [Crux] Translation error in core/516ae8d9::char[0]::convert[0]::char_try_from_u32[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::transmute[0]::_instdc3291f36c45d6de[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/char/convert.rs:215:31: 215:32: error: in core/7ed35d8a::char[0]::convert[0]::char_try_from_u32[0] + [Crux] Translation error in core/7ed35d8a::char[0]::convert[0]::char_try_from_u32[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::transmute[0]::_instdc3291f36c45d6de[0] [Crux] Overall status: Invalid. @@ -1533,9 +1506,9 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/char_from_u32/' to rerun this test only. - div: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 4 + div: OK (1.03s) + Compiling and running oracle program (0.13s) + Oracle output: 4 (0.90s) Crux output: 4 shift_exceeding: rustc compilation failed for shift_exceeding error output: @@ -1567,33 +1540,33 @@ error: aborting due to previous error; 1 warning emitted For more information about this error, try `rustc --explain E0425`. -FAIL (expected: Should panic, but doesn't) (0.06s) - Compiling and running oracle program (0.06s) +FAIL (expected: Should panic, but doesn't) (0.08s) + Compiling and running oracle program (0.08s) test/Test.hs:109: failed to compile and run (expected failure) - bool: OK (0.13s) - Compiling and running oracle program (0.13s) - Oracle output: false + bool: OK (1.02s) + Compiling and running oracle program (0.12s) + Oracle output: false (0.89s) Crux output: false - ge: OK (0.13s) - Compiling and running oracle program (0.13s) - Oracle output: true + ge: OK (1.01s) + Compiling and running oracle program (0.12s) + Oracle output: true (0.88s) Crux output: true - shift1: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 2 + shift1: OK (1.02s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.89s) Crux output: 2 - lit: FAIL (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: false (0.01s) + lit: FAIL (1.04s) + Compiling and running oracle program (0.13s) + Oracle output: false (0.92s) Crux output: failures: - ---- lit/85c762e8::crux_test[0] counterexamples ---- + ---- lit/c6ed0473::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in lit/85c762e8::crux_test[0] - [Crux] Translation error in lit/85c762e8::crux_test[0]: fail or unimp constant: TyTuple [TyBool,TyBool] (StructRepr [MaybeRepr BoolRepr, MaybeRepr BoolRepr]) ConstTuple [ConstBool True,ConstBool True] + [Crux] internal: error: in lit/c6ed0473::crux_test[0] + [Crux] Translation error in lit/c6ed0473::crux_test[0]: fail or unimp constant: TyTuple [TyBool,TyBool] (StructRepr [MaybeRepr BoolRepr, MaybeRepr BoolRepr]) ConstTuple [ConstBool True,ConstBool True] [Crux] Overall status: Invalid. @@ -1601,30 +1574,30 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..prim.lit"' to rerun this test only. - litbstring: OK (0.15s) - Compiling and running oracle program (0.15s) - Oracle output: true + litbstring: OK (0.99s) + Compiling and running oracle program (0.12s) + Oracle output: true (0.87s) Crux output: true - mut_arg: OK (0.14s) - Compiling and running oracle program (0.13s) - Oracle output: 1 + mut_arg: OK (1.01s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (0.89s) Crux output: 1 - shift4: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: -9223372036854775808 + shift4: OK (1.04s) + Compiling and running oracle program (0.12s) + Oracle output: -9223372036854775808 (0.92s) Crux output: -9223372036854775808 - mut: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 14 + mut: OK (1.03s) + Compiling and running oracle program (0.13s) + Oracle output: 14 (0.90s) Crux output: 14 fnptr - make: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 0 + make: OK (1.04s) + Compiling and running oracle program (0.13s) + Oracle output: 0 (0.91s) Crux output: 0 - call: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 2 + call: OK (1.04s) + Compiling and running oracle program (0.12s) + Oracle output: 2 (0.92s) Crux output: 2 custom: rustc compilation failed for custom error output: @@ -1640,26 +1613,26 @@ error: aborting due to previous error For more information about this error, try `rustc --explain E0432`. -FAIL (expected: taking address of an overridden function) (0.05s) +FAIL (expected: taking address of an overridden function) (0.04s) Compiling and running oracle program (0.05s) test/Test.hs:109: failed to compile and run (expected failure) - field: OK (0.14s) + field: OK (1.02s) Compiling and running oracle program (0.14s) - Oracle output: 2 + Oracle output: 2 (0.88s) Crux output: 2 mem - maybe_uninit_array_cast: FAIL (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: [1, 2] (0.01s) + maybe_uninit_array_cast: FAIL (1.05s) + Compiling and running oracle program (0.13s) + Oracle output: [1, 2] (0.92s) Crux output: failures: - ---- maybe_uninit_array_cast/efcbcf14::crux_test[0] counterexamples ---- + ---- maybe_uninit_array_cast/841e0bf8::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:487:43: 487:48: error: in core/516ae8d9::ptr[0]::mut_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/516ae8d9::ptr[0]::mut_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::offset[0]::_inst1e2825177cd3b608[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:487:43: 487:48: error: in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::offset[0]::_inst1e2825177cd3b608[0] [Crux] Overall status: Invalid. @@ -1667,16 +1640,16 @@ FAIL (expected: taking address of an overridden function) (0.05s) crux doesn't match oracle Use -p '/maybe_uninit_array_cast/' to rerun this test only. - maybe_uninit: FAIL (0.15s) + maybe_uninit: FAIL (1.02s) Compiling and running oracle program (0.14s) - Oracle output: 1 (0.01s) + Oracle output: 1 (0.88s) Crux output: failures: - ---- maybe_uninit/cceee461::crux_test[0] counterexamples ---- + ---- maybe_uninit/b2a646dd::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in core/516ae8d9::mem[0]::maybe_uninit[0]::{impl#2}[0]::assume_init[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/516ae8d9::mem[0]::maybe_uninit[0]::{impl#2}[0]::assume_init[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::assert_inhabited[0]::_inst1e2825177cd3b608[0] + [Crux] internal: error: in core/7ed35d8a::mem[0]::maybe_uninit[0]::{impl#2}[0]::assume_init[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/7ed35d8a::mem[0]::maybe_uninit[0]::{impl#2}[0]::assume_init[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::assert_inhabited[0]::_inst1e2825177cd3b608[0] [Crux] Overall status: Invalid. @@ -1685,16 +1658,16 @@ FAIL (expected: taking address of an overridden function) (0.05s) Use -p '$0=="crux-mir.crux concrete..mem.maybe_uninit"' to rerun this test only. consts - local_key: FAIL (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (0.01s) + local_key: FAIL (1.05s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.92s) Crux output: failures: - ---- local_key/c72de59a::crux_test[0] counterexamples ---- + ---- local_key/cc4d5e1d::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/consts/local_key.rs:22:78: 22:93: error: in local_key/c72de59a::crux_test[0] - [Crux] Translation error in local_key/c72de59a::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] test/conc_eval/consts/local_key.rs:22:78: 22:93: error: in local_key/cc4d5e1d::crux_test[0] + [Crux] Translation error in local_key/cc4d5e1d::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. @@ -1702,30 +1675,30 @@ FAIL (expected: taking address of an overridden function) (0.05s) crux doesn't match oracle Use -p '/local_key/' to rerun this test only. - struct_val: OK (0.14s) - Compiling and running oracle program (0.13s) - Oracle output: Foo { x: true } - Crux output: Foo { x: true } - fn_def: FAIL (0.14s) + struct_val: OK (1.06s) Compiling and running oracle program (0.14s) - Oracle output: 1 + Oracle output: Foo { x: true } (0.92s) + Crux output: Foo { x: true } + fn_def: FAIL (1.04s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.91s) user error (JSON Decoding of test/conc_eval/consts/fn_def.all.mir failed: Error in $.fns[0].body.blocks[0].block.terminator.func.data.rendered: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) Use -p '/fn_def/' to rerun this test only. - enum_val: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: None + enum_val: OK (1.09s) + Compiling and running oracle program (0.13s) + Oracle output: None (0.96s) Crux output: None - struct_unit: FAIL (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: Err(Foo { x: () }) (0.01s) + struct_unit: FAIL (1.10s) + Compiling and running oracle program (0.13s) + Oracle output: Err(Foo { x: () }) (0.97s) Crux output: failures: - ---- struct_unit/52e506ef::crux_test[0] counterexamples ---- + ---- struct_unit/b3f925bc::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in struct_unit/52e506ef::crux_test[0] - [Crux] Translation error in struct_unit/52e506ef::crux_test[0]: error building variant core/516ae8d9::result[0]::Result[0]::Err[0]: not enough expressions -- [] + [Crux] internal: error: in struct_unit/b3f925bc::crux_test[0] + [Crux] Translation error in struct_unit/b3f925bc::crux_test[0]: error building variant core/7ed35d8a::result[0]::Result[0]::Err[0]: not enough expressions -- [] [Crux] Overall status: Invalid. @@ -1734,9 +1707,9 @@ FAIL (expected: taking address of an overridden function) (0.05s) Use -p '/struct_unit/' to rerun this test only. refs - imm_raw: OK (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: 123 + imm_raw: OK (1.04s) + Compiling and running oracle program (0.13s) + Oracle output: 123 (0.90s) Crux output: 123 never_mut: rustc compilation failed for never_mut error output: @@ -1761,22 +1734,22 @@ error: aborting due to previous error; 1 warning emitted For more information about this error, try `rustc --explain E0554`. -FAIL (0.05s) - Compiling and running oracle program (0.05s) +FAIL (0.04s) + Compiling and running oracle program (0.04s) test/Test.hs:109: failed to compile and run Use -p '/never_mut/' to rerun this test only. - promoted_mut: FAIL (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (0.01s) + promoted_mut: FAIL (1.06s) + Compiling and running oracle program (0.13s) + Oracle output: 0 (0.93s) Crux output: failures: - ---- promoted_mut/4c6f527f::crux_test[0] counterexamples ---- + ---- promoted_mut/0c05e950::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/refs/promoted_mut.rs:4:27: 4:34: error: in promoted_mut/4c6f527f::crux_test[0] - [Crux] Translation error in promoted_mut/4c6f527f::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] test/conc_eval/refs/promoted_mut.rs:4:27: 4:34: error: in promoted_mut/0c05e950::crux_test[0] + [Crux] Translation error in promoted_mut/0c05e950::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. @@ -1784,13 +1757,13 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/promoted_mut/' to rerun this test only. - mut_nested: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: () + mut_nested: OK (1.06s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.92s) Crux output: () - static_mut: OK (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: 2 + static_mut: OK (1.04s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.91s) Crux output: 2 never: rustc compilation failed for never error output: @@ -1815,22 +1788,22 @@ error: aborting due to previous error; 1 warning emitted For more information about this error, try `rustc --explain E0554`. -FAIL (0.05s) - Compiling and running oracle program (0.05s) +FAIL (0.04s) + Compiling and running oracle program (0.04s) test/Test.hs:109: failed to compile and run Use -p '$0=="crux-mir.crux concrete..refs.never"' to rerun this test only. - temp: FAIL (0.15s) + temp: FAIL (1.05s) Compiling and running oracle program (0.13s) - Oracle output: 1 (0.01s) + Oracle output: 1 (0.92s) Crux output: failures: - ---- temp/d2b20cfd::crux_test[0] counterexamples ---- + ---- temp/8e197f82::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/refs/temp.rs:4:13: 4:16: error: in temp/d2b20cfd::crux_test[0] - [Crux] Translation error in temp/d2b20cfd::crux_test[0]: cannot find static variable alloc$1/3a1fbbbh + [Crux] test/conc_eval/refs/temp.rs:4:13: 4:16: error: in temp/8e197f82::crux_test[0] + [Crux] Translation error in temp/8e197f82::crux_test[0]: cannot find static variable alloc$1/3a1fbbbh [Crux] Overall status: Invalid. @@ -1850,8 +1823,8 @@ error: aborting due to previous error For more information about this error, try `rustc --explain E0554`. -FAIL (0.05s) - Compiling and running oracle program (0.05s) +FAIL (0.04s) + Compiling and running oracle program (0.04s) test/Test.hs:109: failed to compile and run @@ -1868,30 +1841,30 @@ error: aborting due to previous error For more information about this error, try `rustc --explain E0554`. -FAIL (0.05s) - Compiling and running oracle program (0.05s) +FAIL (0.04s) + Compiling and running oracle program (0.04s) test/Test.hs:109: failed to compile and run Use -p '/fn_ptr_mut/' to rerun this test only. - promoted_imm: OK (0.15s) - Compiling and running oracle program (0.15s) - Oracle output: 0 + promoted_imm: OK (1.08s) + Compiling and running oracle program (0.13s) + Oracle output: 0 (0.95s) Crux output: 0 - imm_arg: OK (0.15s) - Compiling and running oracle program (0.15s) - Oracle output: 1 + imm_arg: OK (1.08s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.95s) Crux output: 1 - mut_tuple_field: FAIL (0.17s) - Compiling and running oracle program (0.15s) - Oracle output: () (0.02s) + mut_tuple_field: FAIL (1.05s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.92s) Crux output: failures: - ---- mut_tuple_field/78fb59fb::f[0] counterexamples ---- + ---- mut_tuple_field/c0cb8da5::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/refs/mut_tuple_field.rs:10:19: 10:25: error: in mut_tuple_field/78fb59fb::f[0] - [Crux] Translation error in mut_tuple_field/78fb59fb::f[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] test/conc_eval/refs/mut_tuple_field.rs:10:19: 10:25: error: in mut_tuple_field/c0cb8da5::f[0] + [Crux] Translation error in mut_tuple_field/c0cb8da5::f[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. @@ -1899,27 +1872,27 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/mut_tuple_field/' to rerun this test only. - mut_raw: OK (0.15s) - Compiling and running oracle program (0.15s) - Oracle output: 123 + mut_raw: OK (1.03s) + Compiling and running oracle program (0.13s) + Oracle output: 123 (0.90s) Crux output: 123 - imm_ref: OK (0.16s) - Compiling and running oracle program (0.16s) - Oracle output: 123 + imm_ref: OK (1.04s) + Compiling and running oracle program (0.13s) + Oracle output: 123 (0.92s) Crux output: 123 - mut_ref: OK (0.15s) - Compiling and running oracle program (0.15s) - Oracle output: 1 + mut_ref: OK (1.03s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (0.89s) Crux output: 1 - mut_arg: OK (0.14s) + mut_arg: OK (1.01s) Compiling and running oracle program (0.14s) - Oracle output: 1 + Oracle output: 1 (0.88s) Crux output: 1 time - instant: FAIL (0.91s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.77s) - standalone use of `dyn` is not supported: TyDynamic core/516ae8d9::error[0]::Error[0]::_trait56353a7b840403b4[0] + instant: FAIL (1.80s) + Compiling and running oracle program (0.13s) + Oracle output: () (1.67s) + standalone use of `dyn` is not supported: TyDynamic core/7ed35d8a::error[0]::Error[0]::_trait56353a7b840403b4[0] CallStack (from HasCallStack): error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans @@ -1936,31 +1909,29 @@ FAIL (0.05s) Use -p '/instant/' to rerun this test only. impl - simple: OK (0.15s) - Compiling and running oracle program (0.15s) - Oracle output: 42 + simple: OK (1.03s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.90s) Crux output: 42 - self_mut: OK (0.15s) - Compiling and running oracle program (0.15s) - Oracle output: 42 + self_mut: OK (1.02s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.89s) Crux output: 42 - self: OK (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: 42 + self: OK (1.03s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (0.88s) Crux output: 42 vec_deque - iter_clone: FAIL (0.79s) - Compiling and running oracle program (0.17s) - Oracle output: [1, 2, 3, 2, 3] (0.62s) + iter_clone: FAIL (1.77s) + Compiling and running oracle program (0.15s) + Oracle output: [1, 2, 3, 2, 3] (1.62s) Crux output: failures: - ---- iter_clone/93806b96::crux_test[0] counterexamples ---- + ---- iter_clone/a40d79d5::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. @@ -1968,18 +1939,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/iter_clone/' to rerun this test only. - pop: FAIL (0.70s) - Compiling and running oracle program (0.16s) - Oracle output: [5, 4, 3, 1, 2] (0.54s) + pop: FAIL (1.61s) + Compiling and running oracle program (0.14s) + Oracle output: [5, 4, 3, 1, 2] (1.47s) Crux output: failures: - ---- pop/d98f5c82::crux_test[0] counterexamples ---- + ---- pop/358c93ef::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. @@ -1987,18 +1956,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/vec_deque.pop/' to rerun this test only. - push: FAIL (0.76s) + push: FAIL (1.72s) Compiling and running oracle program (0.15s) - Oracle output: [4, 1, 2, 3, 5] (0.61s) + Oracle output: [4, 1, 2, 3, 5] (1.57s) Crux output: failures: - ---- push/82000b70::crux_test[0] counterexamples ---- + ---- push/f2efcdfd::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] lib/core/src/cmp.rs:1300:11: 1300:28: error: in core/7ed35d8a::cmp[0]::max_by[0]::_insta10905add4aa20a5[0] + [Crux] Translation error in core/7ed35d8a::cmp[0]::max_by[0]::_insta10905add4aa20a5[0]: illegal or unimplemented coercion from TyInt USize (concretely BVRepr 32) to TyInt B8 [Crux] Overall status: Invalid. @@ -2006,18 +1973,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/vec_deque.push/' to rerun this test only. - retain: FAIL (0.77s) - Compiling and running oracle program (0.16s) - Oracle output: [1, 4] (0.60s) + retain: FAIL (1.66s) + Compiling and running oracle program (0.15s) + Oracle output: [1, 4] (1.52s) Crux output: failures: - ---- retain/37c5dc3c::crux_test[0] counterexamples ---- + ---- retain/405ed612::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. @@ -2025,18 +1990,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/retain/' to rerun this test only. - rotate_left: FAIL (0.74s) - Compiling and running oracle program (0.16s) - Oracle output: [3, 4, 5, 1, 2] (0.59s) + rotate_left: FAIL (1.77s) + Compiling and running oracle program (0.15s) + Oracle output: [3, 4, 5, 1, 2] (1.62s) Crux output: failures: - ---- rotate_left/a036e026::crux_test[0] counterexamples ---- + ---- rotate_left/86784989::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. @@ -2044,18 +2007,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/rotate_left/' to rerun this test only. - rotate_right: FAIL (0.81s) - Compiling and running oracle program (0.16s) - Oracle output: [4, 5, 1, 2, 3] (0.65s) + rotate_right: FAIL (1.72s) + Compiling and running oracle program (0.14s) + Oracle output: [4, 5, 1, 2, 3] (1.58s) Crux output: failures: - ---- rotate_right/605564b5::crux_test[0] counterexamples ---- + ---- rotate_right/0cfc3b17::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. @@ -2064,47 +2025,45 @@ FAIL (0.05s) Use -p '/rotate_right/' to rerun this test only. ops - deref3: OK (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.01s) - Crux output: () - index3: OK (0.15s) + deref3: OK (1.04s) Compiling and running oracle program (0.14s) - Oracle output: () + Oracle output: () (0.90s) Crux output: () - deref2: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: () + index3: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.87s) Crux output: () - deref1: OK (0.14s) + deref2: OK (1.00s) Compiling and running oracle program (0.14s) - Oracle output: () + Oracle output: () (0.86s) Crux output: () - index2: OK (0.14s) + deref1: OK (1.01s) Compiling and running oracle program (0.13s) - Oracle output: () + Oracle output: () (0.88s) Crux output: () - arith1: OK (0.15s) + index2: OK (1.04s) Compiling and running oracle program (0.14s) - Oracle output: () (0.01s) + Oracle output: () (0.91s) Crux output: () - index1: OK (0.13s) + arith1: OK (1.05s) Compiling and running oracle program (0.13s) - Oracle output: () + Oracle output: () (0.92s) + Crux output: () + index1: OK (1.03s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.89s) Crux output: () num - saturate: FAIL (0.61s) + saturate: FAIL (1.49s) Compiling and running oracle program (0.14s) - Oracle output: () (0.47s) + Oracle output: () (1.36s) Crux output: failures: - ---- saturate/46c7dca9::crux_test[0] counterexamples ---- + ---- saturate/6a87c133::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:1022:46: 1022:49 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:305:5: 306:27: error: in core/7ed35d8a::num[0]::{impl#7}[0]::saturating_add[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#7}[0]::saturating_add[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::saturating_add[0]::_instaddce72e1232152c[0] [Crux] Overall status: Invalid. @@ -2112,16 +2071,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/saturate/' to rerun this test only. - overflow: FAIL (0.17s) + overflow: FAIL (1.08s) Compiling and running oracle program (0.14s) - Oracle output: () (0.02s) + Oracle output: () (0.94s) Crux output: failures: - ---- overflow/d97d6804::crux_test[0] counterexamples ---- + ---- overflow/bd0e6831::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/num/overflow.rs:5:40: 5:50: error: in overflow/d97d6804::crux_test[0] - [Crux] Translation error in overflow/d97d6804::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] test/conc_eval/num/overflow.rs:5:40: 5:50: error: in overflow/bd0e6831::crux_test[0] + [Crux] Translation error in overflow/bd0e6831::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. @@ -2129,16 +2088,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/num.overflow/' to rerun this test only. - from_bytes: FAIL (0.15s) + from_bytes: FAIL (1.05s) Compiling and running oracle program (0.13s) - Oracle output: () (0.02s) + Oracle output: () (0.92s) Crux output: failures: - ---- from_bytes/c358d064::f[0] counterexamples ---- + ---- from_bytes/4a8462ce::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:2417:37: 2417:42 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:890:5: 891:49: error: in core/516ae8d9::num[0]::{impl#8}[0]::from_ne_bytes[0] - [Crux] Translation error in core/516ae8d9::num[0]::{impl#8}[0]::from_ne_bytes[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::transmute[0]::_inst850c595be4adc72f[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:2417:37: 2417:42 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:890:5: 891:49: error: in core/7ed35d8a::num[0]::{impl#8}[0]::from_ne_bytes[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#8}[0]::from_ne_bytes[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::transmute[0]::_inst850c595be4adc72f[0] [Crux] Overall status: Invalid. @@ -2147,16 +2106,16 @@ FAIL (0.05s) Use -p '/from_bytes/' to rerun this test only. crypto - add: FAIL (0.19s) - Compiling and running oracle program (0.15s) - Oracle output: true (0.04s) + add: FAIL (1.09s) + Compiling and running oracle program (0.14s) + Oracle output: true (0.95s) Crux output: failures: - ---- add/5176c56c::crux_test[0] counterexamples ---- + ---- add/182a6d35::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/num/uint_macros.rs:470:54: 470:57 !lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::unchecked_add[0] - [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::unchecked_add[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_instaffa7a8b1157c078[0] + [Crux] lib/core/src/num/uint_macros.rs:470:54: 470:57 !lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::unchecked_add[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::unchecked_add[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_instaffa7a8b1157c078[0] [Crux] Overall status: Invalid. @@ -2164,16 +2123,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..crypto.add"' to rerun this test only. - add_noL: FAIL (0.18s) - Compiling and running oracle program (0.15s) - Oracle output: false (0.03s) + add_noL: FAIL (1.06s) + Compiling and running oracle program (0.14s) + Oracle output: false (0.93s) Crux output: failures: - ---- add_noL/60d8b3f2::crux_test[0] counterexamples ---- + ---- add_noL/81992a91::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/num/uint_macros.rs:470:54: 470:57 !lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::unchecked_add[0] - [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::unchecked_add[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_instaffa7a8b1157c078[0] + [Crux] lib/core/src/num/uint_macros.rs:470:54: 470:57 !lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::unchecked_add[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::unchecked_add[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_instaffa7a8b1157c078[0] [Crux] Overall status: Invalid. @@ -2182,16 +2141,16 @@ FAIL (0.05s) Use -p '/add_noL/' to rerun this test only. traits - bounds2: FAIL (0.16s) + bounds2: FAIL (1.05s) Compiling and running oracle program (0.14s) - Oracle output: () (0.02s) + Oracle output: () (0.91s) Crux output: failures: - ---- bounds2/1e0bbf28::crux_test[0] counterexamples ---- + ---- bounds2/5d337868::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in bounds2/1e0bbf28::f[0] - [Crux] Translation error in bounds2/1e0bbf28::f[0]: error building variant core/516ae8d9::option[0]::Option[0]::Some[0]: not enough expressions -- [] + [Crux] internal: error: in bounds2/5d337868::f[0] + [Crux] Translation error in bounds2/5d337868::f[0]: error building variant core/7ed35d8a::option[0]::Option[0]::Some[0]: not enough expressions -- [] [Crux] Overall status: Invalid. @@ -2199,40 +2158,40 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/bounds2/' to rerun this test only. - tyfam: OK (0.15s) - Compiling and running oracle program (0.15s) - Oracle output: 23 + tyfam: OK (1.02s) + Compiling and running oracle program (0.13s) + Oracle output: 23 (0.89s) Crux output: 23 - dict_poly: OK (0.14s) + dict_poly: OK (1.03s) Compiling and running oracle program (0.14s) - Oracle output: 1 + Oracle output: 1 (0.89s) Crux output: 1 - dynamic_simple: OK (0.14s) + dynamic_simple: OK (0.98s) Compiling and running oracle program (0.13s) - Oracle output: 1 + Oracle output: 1 (0.85s) Crux output: 1 - assoc1: OK (0.15s) + assoc1: OK (1.03s) Compiling and running oracle program (0.14s) - Oracle output: () + Oracle output: () (0.89s) Crux output: () - static_two: OK (0.13s) + static_two: OK (1.04s) Compiling and running oracle program (0.13s) - Oracle output: 1 + Oracle output: 1 (0.91s) Crux output: 1 - static_self: OK (0.13s) - Compiling and running oracle program (0.13s) - Oracle output: 42 + static_self: OK (1.02s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (0.88s) Crux output: 42 - generic2: FAIL (0.16s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.02s) + generic2: FAIL (1.05s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.92s) Crux output: failures: - ---- generic2/02c07c57::crux_test[0] counterexamples ---- + ---- generic2/21c9978c::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in generic2/02c07c57::f[0] - [Crux] Translation error in generic2/02c07c57::f[0]: error building variant core/516ae8d9::option[0]::Option[0]::Some[0]: not enough expressions -- [] + [Crux] internal: error: in generic2/21c9978c::f[0] + [Crux] Translation error in generic2/21c9978c::f[0]: error building variant core/7ed35d8a::option[0]::Option[0]::Some[0]: not enough expressions -- [] [Crux] Overall status: Invalid. @@ -2240,16 +2199,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/generic2/' to rerun this test only. - bounds1: FAIL (0.17s) - Compiling and running oracle program (0.15s) - Oracle output: () (0.02s) + bounds1: FAIL (1.08s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.94s) Crux output: failures: - ---- bounds1/fab02eaf::crux_test[0] counterexamples ---- + ---- bounds1/324a6555::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in bounds1/fab02eaf::f[0] - [Crux] Translation error in bounds1/fab02eaf::f[0]: error building variant core/516ae8d9::option[0]::Option[0]::Some[0]: not enough expressions -- [] + [Crux] internal: error: in bounds1/324a6555::f[0] + [Crux] Translation error in bounds1/324a6555::f[0]: error building variant core/7ed35d8a::option[0]::Option[0]::Some[0]: not enough expressions -- [] [Crux] Overall status: Invalid. @@ -2257,9 +2216,9 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/bounds1/' to rerun this test only. - gen_trait_poly: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 12 + gen_trait_poly: OK (1.02s) + Compiling and running oracle program (0.13s) + Oracle output: 12 (0.88s) Crux output: 12 dynamic_poly: FAIL (0.17s) Compiling and running oracle program (0.14s) @@ -2267,44 +2226,44 @@ FAIL (0.05s) user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_poly.rs) Use -p '/dynamic_poly/' to rerun this test only. - static: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 42 + static: OK (1.00s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (0.85s) Crux output: 42 - tyfam4: OK (0.14s) + tyfam4: OK (1.02s) Compiling and running oracle program (0.14s) - Oracle output: 0 + Oracle output: 0 (0.88s) Crux output: 0 - assoc2: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: () + assoc2: OK (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.88s) Crux output: () - basics1: OK (0.14s) + basics1: OK (1.06s) Compiling and running oracle program (0.14s) - Oracle output: () + Oracle output: () (0.92s) Crux output: () - gen_trait: OK (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: -69 + gen_trait: OK (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: -69 (0.88s) Crux output: -69 - assoc3: OK (0.16s) - Compiling and running oracle program (0.15s) - Oracle output: () - Crux output: () - default: OK (0.14s) + assoc3: OK (1.02s) Compiling and running oracle program (0.14s) - Oracle output: 12 + Oracle output: () (0.88s) + Crux output: () + default: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: 12 (0.87s) Crux output: 12 - bounds4: FAIL (0.16s) + bounds4: FAIL (1.05s) Compiling and running oracle program (0.14s) - Oracle output: () (0.02s) + Oracle output: () (0.90s) Crux output: failures: - ---- bounds4/7f80af9e::crux_test[0] counterexamples ---- + ---- bounds4/a5df7d79::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in bounds4/7f80af9e::f[0] - [Crux] Translation error in bounds4/7f80af9e::f[0]: error building variant core/516ae8d9::option[0]::Option[0]::Some[0]: not enough expressions -- [] + [Crux] internal: error: in bounds4/a5df7d79::f[0] + [Crux] Translation error in bounds4/a5df7d79::f[0]: error building variant core/7ed35d8a::option[0]::Option[0]::Some[0]: not enough expressions -- [] [Crux] Overall status: Invalid. @@ -2312,46 +2271,46 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/bounds4/' to rerun this test only. - intoiter: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 0 + intoiter: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 0 (0.85s) Crux output: 0 - tyfam5: OK (0.14s) + tyfam5: OK (1.02s) Compiling and running oracle program (0.14s) - Oracle output: () + Oracle output: () (0.87s) Crux output: () - dict_polymem: OK (0.13s) + dict_polymem: OK (1.04s) Compiling and running oracle program (0.13s) - Oracle output: 4 + Oracle output: 4 (0.91s) Crux output: 4 - dynamic_branch: FAIL (0.16s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.02s) + dynamic_branch: FAIL (0.19s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (0.03s) user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_branch.rs) Use -p '/dynamic_branch/' to rerun this test only. - subtrait: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 73 + subtrait: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: 73 (0.86s) Crux output: 73 - params: OK (0.14s) + params: OK (1.00s) Compiling and running oracle program (0.14s) - Oracle output: 25 + Oracle output: 25 (0.86s) Crux output: 25 - conv: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 1 + conv: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.87s) Crux output: 1 - bounds5: FAIL (0.17s) - Compiling and running oracle program (0.15s) - Oracle output: () (0.02s) + bounds5: FAIL (1.07s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.93s) Crux output: failures: - ---- bounds5/3b3a5fb2::crux_test[0] counterexamples ---- + ---- bounds5/5b4757b0::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in bounds5/3b3a5fb2::f[0] - [Crux] Translation error in bounds5/3b3a5fb2::f[0]: error building variant core/516ae8d9::option[0]::Option[0]::Some[0]: not enough expressions -- [] + [Crux] internal: error: in bounds5/5b4757b0::f[0] + [Crux] Translation error in bounds5/5b4757b0::f[0]: error building variant core/7ed35d8a::option[0]::Option[0]::Some[0]: not enough expressions -- [] [Crux] Overall status: Invalid. @@ -2359,24 +2318,24 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/bounds5/' to rerun this test only. - dict_simple: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 32 + dict_simple: OK (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: 32 (0.88s) Crux output: 32 - subtrait2: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 64 + subtrait2: OK (1.05s) + Compiling and running oracle program (0.13s) + Oracle output: 64 (0.92s) Crux output: 64 - generic3: FAIL (0.16s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.02s) + generic3: FAIL (1.05s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.92s) Crux output: failures: - ---- generic3/256bb722::crux_test[0] counterexamples ---- + ---- generic3/a3e60e58::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in generic3/256bb722::f[0] - [Crux] Translation error in generic3/256bb722::f[0]: error building variant core/516ae8d9::option[0]::Option[0]::Some[0]: not enough expressions -- [] + [Crux] internal: error: in generic3/a3e60e58::f[0] + [Crux] Translation error in generic3/a3e60e58::f[0]: error building variant core/7ed35d8a::option[0]::Option[0]::Some[0]: not enough expressions -- [] [Crux] Overall status: Invalid. @@ -2384,16 +2343,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/generic3/' to rerun this test only. - bounds3: FAIL (0.16s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.02s) + bounds3: FAIL (1.02s) + Compiling and running oracle program (0.16s) + Oracle output: () (0.87s) Crux output: failures: - ---- bounds3/448ffaab::crux_test[0] counterexamples ---- + ---- bounds3/ad390bcf::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in bounds3/448ffaab::f[0] - [Crux] Translation error in bounds3/448ffaab::f[0]: error building variant core/516ae8d9::option[0]::Option[0]::Some[0]: not enough expressions -- [] + [Crux] internal: error: in bounds3/ad390bcf::f[0] + [Crux] Translation error in bounds3/ad390bcf::f[0]: error building variant core/7ed35d8a::option[0]::Option[0]::Some[0]: not enough expressions -- [] [Crux] Overall status: Invalid. @@ -2401,36 +2360,36 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/bounds3/' to rerun this test only. - static_eq: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: true + static_eq: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: true (0.85s) Crux output: true - static_three: OK (0.14s) + static_three: OK (1.05s) Compiling and running oracle program (0.14s) - Oracle output: 2 + Oracle output: 2 (0.92s) Crux output: 2 - tyfam2: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 23 + tyfam2: OK (1.04s) + Compiling and running oracle program (0.13s) + Oracle output: 23 (0.91s) Crux output: 23 - dict_med: OK (0.14s) + dict_med: OK (1.03s) Compiling and running oracle program (0.14s) - Oracle output: 42 + Oracle output: 42 (0.89s) Crux output: 42 - tyfam3: OK (0.15s) + tyfam3: OK (1.00s) Compiling and running oracle program (0.15s) - Oracle output: 23 + Oracle output: 23 (0.86s) Crux output: 23 - generic1: FAIL (0.16s) + generic1: FAIL (1.07s) Compiling and running oracle program (0.14s) - Oracle output: () (0.02s) + Oracle output: () (0.93s) Crux output: failures: - ---- generic1/d2b80067::crux_test[0] counterexamples ---- + ---- generic1/c742374a::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in generic1/d2b80067::f[0] - [Crux] Translation error in generic1/d2b80067::f[0]: error building variant core/516ae8d9::option[0]::Option[0]::Some[0]: not enough expressions -- [] + [Crux] internal: error: in generic1/c742374a::f[0] + [Crux] Translation error in generic1/c742374a::f[0]: error building variant core/7ed35d8a::option[0]::Option[0]::Some[0]: not enough expressions -- [] [Crux] Overall status: Invalid. @@ -2439,22 +2398,22 @@ FAIL (0.05s) Use -p '/generic1/' to rerun this test only. dynamic_med: FAIL (0.16s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (0.02s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.03s) user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_med.rs) Use -p '/dynamic_med/' to rerun this test only. - dynamic_two: FAIL (0.16s) + dynamic_two: FAIL (0.17s) Compiling and running oracle program (0.14s) - Oracle output: 1 (0.02s) + Oracle output: 1 (0.03s) user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_two.rs) Use -p '/dynamic_two/' to rerun this test only. io - cursor_read: FAIL (0.60s) - Compiling and running oracle program (0.16s) - Oracle output: 0 (0.44s) - standalone use of `dyn` is not supported: TyDynamic core/516ae8d9::error[0]::Error[0]::_trait56353a7b840403b4[0] + cursor_read: FAIL (1.50s) + Compiling and running oracle program (0.15s) + Oracle output: 0 (1.34s) + standalone use of `dyn` is not supported: TyDynamic core/7ed35d8a::error[0]::Error[0]::_trait56353a7b840403b4[0] CallStack (from HasCallStack): error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans @@ -2470,10 +2429,10 @@ FAIL (0.05s) translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '/io.cursor_read/' to rerun this test only. - cursor_write2: FAIL (0.92s) - Compiling and running oracle program (0.17s) - Oracle output: 0 (0.76s) - standalone use of `dyn` is not supported: TyDynamic core/516ae8d9::error[0]::Error[0]::_trait56353a7b840403b4[0] + cursor_write2: FAIL (1.87s) + Compiling and running oracle program (0.15s) + Oracle output: 0 (1.72s) + standalone use of `dyn` is not supported: TyDynamic core/7ed35d8a::error[0]::Error[0]::_trait56353a7b840403b4[0] CallStack (from HasCallStack): error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans @@ -2489,10 +2448,10 @@ FAIL (0.05s) translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '/cursor_write2/' to rerun this test only. - cursor_write: FAIL (0.60s) - Compiling and running oracle program (0.16s) - Oracle output: 0 (0.43s) - standalone use of `dyn` is not supported: TyDynamic core/516ae8d9::error[0]::Error[0]::_trait56353a7b840403b4[0] + cursor_write: FAIL (1.48s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.33s) + standalone use of `dyn` is not supported: TyDynamic core/7ed35d8a::error[0]::Error[0]::_trait56353a7b840403b4[0] CallStack (from HasCallStack): error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans @@ -2509,20 +2468,20 @@ FAIL (0.05s) Use -p '$0=="crux-mir.crux concrete..io.cursor_write"' to rerun this test only. statics - promoted_static: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 1 + promoted_static: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.88s) Crux output: 1 - promoted_fn: FAIL (0.16s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (0.01s) + promoted_fn: FAIL (1.04s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (0.90s) Crux output: failures: - ---- promoted_fn/e00ec2eb::crux_test[0] counterexamples ---- + ---- promoted_fn/8e484c2a::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/statics/promoted_fn.rs:4:13: 4:16: error: in promoted_fn/e00ec2eb::crux_test[0] - [Crux] Translation error in promoted_fn/e00ec2eb::crux_test[0]: cannot find static variable alloc$1/3a1fbbbh + [Crux] test/conc_eval/statics/promoted_fn.rs:4:13: 4:16: error: in promoted_fn/8e484c2a::crux_test[0] + [Crux] Translation error in promoted_fn/8e484c2a::crux_test[0]: cannot find static variable alloc$1/3a1fbbbh [Crux] Overall status: Invalid. @@ -2531,16 +2490,16 @@ FAIL (0.05s) Use -p '/promoted_fn/' to rerun this test only. struct - repr_transparent: FAIL (0.17s) + repr_transparent: FAIL (1.07s) Compiling and running oracle program (0.14s) - Oracle output: 6 (0.02s) + Oracle output: 6 (0.93s) Crux output: failures: - ---- repr_transparent/444f9e7e::crux_test[0] counterexamples ---- + ---- repr_transparent/5ed63a8b::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:61:9: 61:18: error: in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0] - [Crux] Translation error in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0]: unimplemented cast: Misc + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:61:9: 61:18: error: in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0] + [Crux] Translation error in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyInt B32) Immut [Crux] as: TyRawPtr (TyArray (TyInt B32) 2) Immut @@ -2550,42 +2509,42 @@ FAIL (0.05s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..struct.repr_transparent"' to rerun this test only. - repr_transparent_const: OK (0.14s) + repr_transparent_const: OK (1.00s) Compiling and running oracle program (0.14s) - Oracle output: 124 + Oracle output: 124 (0.86s) Crux output: 124 - field_order: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: () + field_order: OK (1.04s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.91s) Crux output: () - arg: OK (0.14s) + arg: OK (0.99s) Compiling and running oracle program (0.14s) - Oracle output: 42 + Oracle output: 42 (0.85s) Crux output: 42 - proj: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: 42 + proj: OK (1.02s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.88s) Crux output: 42 - ret: OK (0.14s) + ret: OK (1.05s) Compiling and running oracle program (0.14s) - Oracle output: S { x: 42, y: 120 } + Oracle output: S { x: 42, y: 120 } (0.91s) Crux output: S { x: 42, y: 120 } - tup: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: () + tup: OK (1.06s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.92s) Crux output: () ptr - valid_eq: OK (0.14s) - Compiling and running oracle program (0.14s) - Oracle output: () + valid_eq: OK (1.06s) + Compiling and running oracle program (0.15s) + Oracle output: () (0.91s) Crux output: () - cast_eq: OK (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: () + cast_eq: OK (1.03s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.90s) Crux output: () - read_write: OK (0.16s) + read_write: OK (1.10s) Compiling and running oracle program (0.14s) - Oracle output: [1, 3, 1] (0.02s) + Oracle output: [1, 3, 1] (0.96s) Crux output: [1, 3, 1] coerce_unsized: rustc compilation failed for coerce_unsized error output: @@ -2613,22 +2572,22 @@ error: aborting due to 2 previous errors; 1 warning emitted For more information about this error, try `rustc --explain E0554`. -FAIL (0.05s) - Compiling and running oracle program (0.05s) +FAIL (0.04s) + Compiling and running oracle program (0.04s) test/Test.hs:109: failed to compile and run Use -p '/coerce_unsized/' to rerun this test only. - null_eq: FAIL (0.15s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.02s) + null_eq: FAIL (1.06s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (0.93s) Crux output: failures: - ---- null_eq/fe6c1a4e::crux_test[0] counterexamples ---- + ---- null_eq/399f1a30::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mod.rs:568:29: 568:33: error: in core/516ae8d9::ptr[0]::invalid[0]::_instf59dc84daa91a404[0] - [Crux] Translation error in core/516ae8d9::ptr[0]::invalid[0]::_instf59dc84daa91a404[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::transmute[0]::_insta5d0a99e3b08f3bd[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mod.rs:568:29: 568:33: error: in core/7ed35d8a::ptr[0]::invalid[0]::_instf59dc84daa91a404[0] + [Crux] Translation error in core/7ed35d8a::ptr[0]::invalid[0]::_instf59dc84daa91a404[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::transmute[0]::_insta5d0a99e3b08f3bd[0] [Crux] Overall status: Invalid. @@ -2664,22 +2623,22 @@ error: aborting due to previous error; 2 warnings emitted For more information about this error, try `rustc --explain E0554`. -FAIL (0.05s) - Compiling and running oracle program (0.05s) +FAIL (0.06s) + Compiling and running oracle program (0.06s) test/Test.hs:109: failed to compile and run Use -p '/offset_from/' to rerun this test only. - is_null: FAIL (0.15s) + is_null: FAIL (1.06s) Compiling and running oracle program (0.14s) - Oracle output: (false, true) (0.01s) + Oracle output: (false, true) (0.91s) Crux output: failures: - ---- is_null/dda8ee76::crux_test[0] counterexamples ---- + ---- is_null/42d14c9d::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/ptr/is_null.rs:5:13: 5:15: error: in is_null/dda8ee76::crux_test[0] - [Crux] Translation error in is_null/dda8ee76::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] test/conc_eval/ptr/is_null.rs:5:13: 5:15: error: in is_null/42d14c9d::crux_test[0] + [Crux] Translation error in is_null/42d14c9d::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. @@ -2687,16 +2646,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..ptr.is_null"' to rerun this test only. - dangling_eq: FAIL (0.16s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.03s) + dangling_eq: FAIL (1.03s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.90s) Crux output: failures: - ---- dangling_eq/7f6bbb93::crux_test[0] counterexamples ---- + ---- dangling_eq/f33166fd::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mod.rs:599:29: 599:33: error: in core/516ae8d9::ptr[0]::invalid_mut[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/516ae8d9::ptr[0]::invalid_mut[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::transmute[0]::_inst102627303ebf2823[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mod.rs:599:29: 599:33: error: in core/7ed35d8a::ptr[0]::invalid_mut[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/7ed35d8a::ptr[0]::invalid_mut[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::transmute[0]::_inst102627303ebf2823[0] [Crux] Overall status: Invalid. @@ -2704,16 +2663,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/dangling_eq/' to rerun this test only. - offset: FAIL (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (0.01s) + offset: FAIL (1.06s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (0.92s) Crux output: failures: - ---- offset/e803b869::crux_test[0] counterexamples ---- + ---- offset/dd961ed4::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:473:43: 473:48: error: in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::offset[0]::_inst1e2825177cd3b608[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:473:43: 473:48: error: in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::offset[0]::_inst1e2825177cd3b608[0] [Crux] Overall status: Invalid. @@ -2721,16 +2680,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..ptr.offset"' to rerun this test only. - offset_mut: FAIL (0.17s) - Compiling and running oracle program (0.16s) - Oracle output: 3 (0.01s) + offset_mut: FAIL (1.07s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.93s) Crux output: failures: - ---- offset_mut/3d54e912::crux_test[0] counterexamples ---- + ---- offset_mut/b36ef04b::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:487:43: 487:48: error: in core/516ae8d9::ptr[0]::mut_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/516ae8d9::ptr[0]::mut_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::offset[0]::_inst1e2825177cd3b608[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:487:43: 487:48: error: in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::offset[0]::_inst1e2825177cd3b608[0] [Crux] Overall status: Invalid. @@ -2738,16 +2697,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/offset_mut/' to rerun this test only. - unsize_slice: FAIL (0.15s) + unsize_slice: FAIL (1.06s) Compiling and running oracle program (0.14s) - Oracle output: (1, 2) (0.01s) + Oracle output: (1, 2) (0.91s) Crux output: failures: - ---- unsize_slice/2118c582::crux_test[0] counterexamples ---- + ---- unsize_slice/0d5b391d::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:473:43: 473:48: error: in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/516ae8d9::ptr[0]::const_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::offset[0]::_inst1e2825177cd3b608[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:473:43: 473:48: error: in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::offset[0]::_inst1e2825177cd3b608[0] [Crux] Overall status: Invalid. @@ -2755,24 +2714,24 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/unsize_slice/' to rerun this test only. - struct_eq: OK (0.15s) - Compiling and running oracle program (0.14s) - Oracle output: () + struct_eq: OK (1.03s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.90s) Crux output: () - copy: OK (0.17s) - Compiling and running oracle program (0.14s) - Oracle output: [1, 2, 3, 1, 2, 3] (0.02s) + copy: OK (1.08s) + Compiling and running oracle program (0.15s) + Oracle output: [1, 2, 3, 1, 2, 3] (0.93s) Crux output: [1, 2, 3, 1, 2, 3] - is_null_slice: FAIL (expected: can't unsize null pointers) (0.15s) + is_null_slice: FAIL (expected: can't unsize null pointers) (1.05s) Compiling and running oracle program (0.14s) - Oracle output: (false, true) (0.01s) + Oracle output: (false, true) (0.91s) Crux output: failures: - ---- is_null_slice/42710a6d::crux_test[0] counterexamples ---- + ---- is_null_slice/767ccbda::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/ptr/is_null_slice.rs:6:13: 6:20: error: in is_null_slice/42710a6d::crux_test[0] - [Crux] Translation error in is_null_slice/42710a6d::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] test/conc_eval/ptr/is_null_slice.rs:6:13: 6:20: error: in is_null_slice/767ccbda::crux_test[0] + [Crux] Translation error in is_null_slice/767ccbda::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. @@ -2781,1401 +2740,290 @@ FAIL (0.05s) (expected failure) crux symbolic Output testing - sort_by_key: FAIL (0.72s) + sort_by_key: FAIL (1.57s) files test/symb_eval/vec/sort_by_key.good and test/symb_eval/vec/sort_by_key.out differ; test/symb_eval/vec/sort_by_key.out contains: - test sort_by_key/1875b66b::f[0]: FAILED + test sort_by_key/43dcef21::f[0]: FAILED failures: - ---- sort_by_key/1875b66b::f[0] counterexamples ---- + ---- sort_by_key/43dcef21::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. Use -p '/sort_by_key/' to rerun this test only. - clone: FAIL (0.14s) + clone: FAIL (1.10s) files test/symb_eval/vec/clone.good and test/symb_eval/vec/clone.out differ; test/symb_eval/vec/clone.out contains: - test clone/6b1f0898::f[0]: FAILED + test clone/b5c63e0e::f[0]: FAILED failures: - ---- clone/6b1f0898::f[0] counterexamples ---- + ---- clone/b5c63e0e::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. Use -p '/Output testing.clone/' to rerun this test only. - into_iter: FAIL (0.71s) + into_iter: FAIL (1.67s) files test/symb_eval/vec/into_iter.good and test/symb_eval/vec/into_iter.out differ; test/symb_eval/vec/into_iter.out contains: - test into_iter/65317545::f[0]: FAILED + test into_iter/9e3ff4b9::f[0]: FAILED failures: - ---- into_iter/65317545::f[0] counterexamples ---- + ---- into_iter/9e3ff4b9::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. Use -p '/into_iter/' to rerun this test only. - macro: FAIL (0.11s) + macro: FAIL (1.05s) files test/symb_eval/vec/macro.good and test/symb_eval/vec/macro.out differ; test/symb_eval/vec/macro.out contains: - test macro/2d0c4c9f::f[0]: FAILED + test macro/7e21692f::f[0]: FAILED failures: - ---- macro/2d0c4c9f::f[0] counterexamples ---- + ---- macro/7e21692f::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/516ae8d9::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/516ae8d9::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] + [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. Use -p '/Output testing.macro/' to rerun this test only. - override4: FAIL (0.10s) + override4: FAIL (0.96s) files test/symb_eval/overrides/override4.good and test/symb_eval/overrides/override4.out differ; test/symb_eval/overrides/override4.out contains: - test override4/6b6c1ae4::f[0]: FAILED + test override4/5f4a247f::f[0]: FAILED failures: - ---- override4/6b6c1ae4::f[0] counterexamples ---- + ---- override4/5f4a247f::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/5888ca02::symbolic[0]::symbolic_u8[0] + [Crux] Translation error in crucible/5888ca02::symbolic[0]::symbolic_u8[0]: cannot find static variable alloc$8/3a1fbbbh [Crux] Overall status: Invalid. Use -p '/override4/' to rerun this test only. - bad_symb2: FAIL (0.10s) + bad_symb2: FAIL (0.95s) files test/symb_eval/overrides/bad_symb2.good and test/symb_eval/overrides/bad_symb2.out differ; test/symb_eval/overrides/bad_symb2.out contains: - test bad_symb2/41a155d5::crux_test[0]: FAILED + test bad_symb2/db5a10b0::crux_test[0]: FAILED failures: - ---- bad_symb2/41a155d5::crux_test[0] counterexamples ---- + ---- bad_symb2/db5a10b0::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/5888ca02::symbolic[0]::symbolic_u8[0] + [Crux] Translation error in crucible/5888ca02::symbolic[0]::symbolic_u8[0]: cannot find static variable alloc$8/3a1fbbbh [Crux] Overall status: Invalid. Use -p '/bad_symb2/' to rerun this test only. - override3: FAIL (0.10s) + override3: FAIL (0.96s) files test/symb_eval/overrides/override3.good and test/symb_eval/overrides/override3.out differ; test/symb_eval/overrides/override3.out contains: - test override3/2b4100d1::f[0]: FAILED + test override3/5788e0ec::f[0]: FAILED failures: - ---- override3/2b4100d1::f[0] counterexamples ---- + ---- override3/5788e0ec::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/5888ca02::symbolic[0]::symbolic_u8[0] + [Crux] Translation error in crucible/5888ca02::symbolic[0]::symbolic_u8[0]: cannot find static variable alloc$8/3a1fbbbh [Crux] Overall status: Invalid. Use -p '/override3/' to rerun this test only. - override2: FAIL (0.10s) + override2: FAIL (0.95s) files test/symb_eval/overrides/override2.good and test/symb_eval/overrides/override2.out differ; test/symb_eval/overrides/override2.out contains: - test override2/4c9e2a25::f[0]: FAILED + test override2/463e2c44::f[0]: FAILED failures: - ---- override2/4c9e2a25::f[0] counterexamples ---- + ---- override2/463e2c44::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/5888ca02::symbolic[0]::symbolic_u8[0] + [Crux] Translation error in crucible/5888ca02::symbolic[0]::symbolic_u8[0]: cannot find static variable alloc$8/3a1fbbbh [Crux] Overall status: Invalid. Use -p '/override2/' to rerun this test only. - bad_symb1: FAIL (0.10s) + bad_symb1: FAIL (1.00s) files test/symb_eval/overrides/bad_symb1.good and test/symb_eval/overrides/bad_symb1.out differ; test/symb_eval/overrides/bad_symb1.out contains: - test bad_symb1/8fcb2fc3::crux_test[0]: FAILED + test bad_symb1/69aa9e9b::crux_test[0]: FAILED failures: - ---- bad_symb1/8fcb2fc3::crux_test[0] counterexamples ---- + ---- bad_symb1/69aa9e9b::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/5888ca02::symbolic[0]::symbolic_u8[0] + [Crux] Translation error in crucible/5888ca02::symbolic[0]::symbolic_u8[0]: cannot find static variable alloc$8/3a1fbbbh [Crux] Overall status: Invalid. Use -p '/bad_symb1/' to rerun this test only. - override5: FAIL (0.11s) + override5: FAIL (0.99s) files test/symb_eval/overrides/override5.good and test/symb_eval/overrides/override5.out differ; test/symb_eval/overrides/override5.out contains: - test override5/3995a1d6::f[0]: FAILED + test override5/ae1ba196::f[0]: FAILED failures: - ---- override5/3995a1d6::f[0] counterexamples ---- + ---- override5/ae1ba196::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/5888ca02::symbolic[0]::symbolic_u64[0] + [Crux] Translation error in crucible/5888ca02::symbolic[0]::symbolic_u64[0]: cannot find static variable alloc$6/3a1fbbbh [Crux] Overall status: Invalid. Use -p '/override5/' to rerun this test only. - override1: FAIL (0.02s) + override1: FAIL (0.91s) files test/symb_eval/overrides/override1.good and test/symb_eval/overrides/override1.out differ; test/symb_eval/overrides/override1.out contains: - test override1/dff68baf::f[0]: FAILED + test override1/c3978c77::f[0]: FAILED failures: - ---- override1/dff68baf::f[0] counterexamples ---- + ---- override1/c3978c77::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/897c723b::one[0] - [Crux] panicking::panic, called from crucible/897c723b::one[0] + [Crux] internal: error: in crucible/5888ca02::one[0] + [Crux] panicking::panic, called from crucible/5888ca02::one[0] [Crux] Overall status: Invalid. Use -p '/override1/' to rerun this test only. - override_rust: FAIL (0.09s) + override_rust: FAIL (0.95s) files test/symb_eval/overrides/override_rust.good and test/symb_eval/overrides/override_rust.out differ; test/symb_eval/overrides/override_rust.out contains: - test override_rust/2fce5978::crux_test[0]: FAILED + test override_rust/59b18491::crux_test[0]: FAILED failures: - ---- override_rust/2fce5978::crux_test[0] counterexamples ---- + ---- override_rust/59b18491::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/lib.rs:129:5: 129:42: error: in crucible/5888ca02::override_[0]::_inst28abcb7c380bf102[0] + [Crux] Translation error in crucible/5888ca02::override_[0]::_inst28abcb7c380bf102[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. Use -p '/override_rust/' to rerun this test only. - read: FAIL (0.05s) + read: FAIL (0.09s) test/symb_eval/byteorder/read.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/read.rs) Use -p '/Output testing.read/' to rerun this test only. write: FAIL test/symb_eval/byteorder/write.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/write.rs) Use -p '/Output testing.write/' to rerun this test only. - mux: FAIL (0.10s) + mux: FAIL (0.96s) files test/symb_eval/enum/mux.good and test/symb_eval/enum/mux.out differ; test/symb_eval/enum/mux.out contains: - test mux/cef3218c::test[0]: FAILED + test mux/aa4fe2e6::test[0]: FAILED failures: - ---- mux/cef3218c::test[0] counterexamples ---- + ---- mux/aa4fe2e6::test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/5888ca02::symbolic[0]::symbolic_u8[0] + [Crux] Translation error in crucible/5888ca02::symbolic[0]::symbolic_u8[0]: cannot find static variable alloc$8/3a1fbbbh [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.mux"' to rerun this test only. - split_off: FAIL (0.51s) + split_off: FAIL (1.42s) files test/symb_eval/bytes/split_off.good and test/symb_eval/bytes/split_off.out differ; test/symb_eval/bytes/split_off.out contains: - test split_off/6d8078c7::f[0]: FAILED + test split_off/4ecfcbdd::f[0]: FAILED failures: - ---- split_off/6d8078c7::f[0] counterexamples ---- + ---- split_off/4ecfcbdd::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/vector.rs:81:9: 81:44: error: in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0] + [Crux] Translation error in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. Use -p '/split_off/' to rerun this test only. - new: FAIL (0.08s) + new: FAIL (1.00s) files test/symb_eval/bytes/new.good and test/symb_eval/bytes/new.out differ; test/symb_eval/bytes/new.out contains: - test new/197434bb::f[0]: FAILED + test new/b6888249::f[0]: FAILED failures: - ---- new/197434bb::f[0] counterexamples ---- + ---- new/b6888249::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/vector.rs:81:9: 81:44: error: in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0] + [Crux] Translation error in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.new"' to rerun this test only. - split_to: FAIL (0.51s) + split_to: FAIL (1.35s) files test/symb_eval/bytes/split_to.good and test/symb_eval/bytes/split_to.out differ; test/symb_eval/bytes/split_to.out contains: - test split_to/d69c0d48::f[0]: FAILED + test split_to/437fab7e::f[0]: FAILED failures: - ---- split_to/d69c0d48::f[0] counterexamples ---- + ---- split_to/437fab7e::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/vector.rs:81:9: 81:44: error: in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0] + [Crux] Translation error in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. Use -p '/split_to/' to rerun this test only. - put_overflow: FAIL (0.47s) + put_overflow: FAIL (1.40s) files test/symb_eval/bytes/put_overflow.good and test/symb_eval/bytes/put_overflow.out differ; test/symb_eval/bytes/put_overflow.out contains: - test put_overflow/b706e629::f[0]: FAILED + test put_overflow/bfb2f26d::f[0]: FAILED failures: - ---- put_overflow/b706e629::f[0] counterexamples ---- + ---- put_overflow/bfb2f26d::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/vector.rs:81:9: 81:44: error: in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0] + [Crux] Translation error in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. Use -p '/put_overflow/' to rerun this test only. - sym_len: FAIL (0.48s) + sym_len: FAIL (1.37s) files test/symb_eval/bytes/sym_len.good and test/symb_eval/bytes/sym_len.out differ; test/symb_eval/bytes/sym_len.out contains: - test sym_len/4ff12171::f[0]: FAILED + test sym_len/d901b6ca::f[0]: FAILED failures: - ---- sym_len/4ff12171::f[0] counterexamples ---- + ---- sym_len/d901b6ca::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/vector.rs:81:9: 81:44: error: in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0] + [Crux] Translation error in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. Use -p '/sym_len/' to rerun this test only. - put: FAIL (0.48s) + put: FAIL (1.33s) files test/symb_eval/bytes/put.good and test/symb_eval/bytes/put.out differ; test/symb_eval/bytes/put.out contains: - test put/7b1c2d40::f[0]: FAILED + test put/5170e38f::f[0]: FAILED failures: - ---- put/7b1c2d40::f[0] counterexamples ---- + ---- put/5170e38f::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/vector.rs:81:9: 81:44: error: in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0] + [Crux] Translation error in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.put"' to rerun this test only. - extend_bytes: FAIL (0.11s) + extend_bytes: FAIL (0.94s) files test/symb_eval/bytes/extend_bytes.good and test/symb_eval/bytes/extend_bytes.out differ; test/symb_eval/bytes/extend_bytes.out contains: - test extend_bytes/c2da7e0b::f[0]: FAILED + test extend_bytes/d25848e0::f[0]: FAILED failures: - ---- extend_bytes/c2da7e0b::f[0] counterexamples ---- + ---- extend_bytes/d25848e0::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/vector.rs:81:9: 81:44: error: in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0] + [Crux] Translation error in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0]: cannot find static variable alloc$0/3a1fbbbh [Crux] Overall status: Invalid. Use -p '/extend_bytes/' to rerun this test only. - basic: FAIL (0.11s) - files test/symb_eval/array/basic.good and test/symb_eval/array/basic.out differ; test/symb_eval/array/basic.out contains: - test basic/35006054::crux_test[0]: FAILED - - failures: - - ---- basic/35006054::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/Output testing.basic/' to rerun this test only. - slice_mut: FAIL (0.10s) - files test/symb_eval/array/slice_mut.good and test/symb_eval/array/slice_mut.out differ; test/symb_eval/array/slice_mut.out contains: - test slice_mut/cd20ad7b::crux_test[0]: FAILED - - failures: - - ---- slice_mut/cd20ad7b::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/slice_mut/' to rerun this test only. - mux_slice: FAIL (0.13s) - files test/symb_eval/array/mux_slice.good and test/symb_eval/array/mux_slice.out differ; test/symb_eval/array/mux_slice.out contains: - test mux_slice/0ab1dfe8::crux_test[0]: FAILED - - failures: - - ---- mux_slice/0ab1dfe8::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/mux_slice/' to rerun this test only. - slice: FAIL (0.13s) - files test/symb_eval/array/slice.good and test/symb_eval/array/slice.out differ; test/symb_eval/array/slice.out contains: - test slice/031b990c::crux_test[0]: FAILED - - failures: - - ---- slice/031b990c::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.slice"' to rerun this test only. - arith: FAIL (0.13s) - files test/symb_eval/bitvector/arith.good and test/symb_eval/bitvector/arith.out differ; test/symb_eval/bitvector/arith.out contains: - test arith/ccf3c161::crux_test[0]: FAILED - - failures: - - ---- arith/ccf3c161::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/Output testing.arith/' to rerun this test only. - literals: FAIL (0.02s) - files test/symb_eval/bitvector/literals.good and test/symb_eval/bitvector/literals.out differ; test/symb_eval/bitvector/literals.out contains: - test literals/abfb591a::crux_test[0]: FAILED - - failures: - - ---- literals/abfb591a::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/bitvector.rs:91:29: 91:30 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/bitvector.rs:125:1: 125:53: error: in crucible/897c723b::bitvector[0]::{impl#27}[0]::from[0]::_inst9ea2f935eda83dab[0] - [Crux] Translation error in crucible/897c723b::bitvector[0]::{impl#27}[0]::from[0]::_inst9ea2f935eda83dab[0]: BUG: invalid arguments to bv_convert: [$31: AnyRepr] - - [Crux] Overall status: Invalid. - - Use -p '/literals/' to rerun this test only. - cmp: FAIL (0.11s) - files test/symb_eval/bitvector/cmp.good and test/symb_eval/bitvector/cmp.out differ; test/symb_eval/bitvector/cmp.out contains: - test cmp/a1f7a6f5::crux_test[0]: FAILED - - failures: - - ---- cmp/a1f7a6f5::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/Output testing.cmp/' to rerun this test only. - overflowing_sub: FAIL (0.08s) - files test/symb_eval/bitvector/overflowing_sub.good and test/symb_eval/bitvector/overflowing_sub.out differ; test/symb_eval/bitvector/overflowing_sub.out contains: - test overflowing_sub/9173f58f::crux_test[0]: FAILED - - failures: - - ---- overflowing_sub/9173f58f::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/overflowing_sub/' to rerun this test only. - from_to: FAIL (0.02s) - files test/symb_eval/bitvector/from_to.good and test/symb_eval/bitvector/from_to.out differ; test/symb_eval/bitvector/from_to.out contains: - test from_to/caac82ba::crux_test[0]: FAILED - - failures: - - ---- from_to/caac82ba::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/bitvector.rs:103:29: 103:30 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/bitvector.rs:125:1: 125:53: error: in crucible/897c723b::bitvector[0]::{impl#21}[0]::from[0]::_inst9ea2f935eda83dab[0] - [Crux] Translation error in crucible/897c723b::bitvector[0]::{impl#21}[0]::from[0]::_inst9ea2f935eda83dab[0]: BUG: invalid arguments to bv_convert: [$27: BVRepr 64] - - [Crux] Overall status: Invalid. - - Use -p '/from_to/' to rerun this test only. - symbolic: FAIL (0.03s) - files test/symb_eval/bitvector/symbolic.good and test/symb_eval/bitvector/symbolic.out differ; test/symb_eval/bitvector/symbolic.out contains: - test symbolic/57c5e3e8::crux_test[0]: FAILED - - failures: - - ---- symbolic/57c5e3e8::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/897c723b::bitvector[0]::make_symbolic_256[0] - [Crux] panicking::panic, called from crucible/897c723b::bitvector[0]::make_symbolic_256[0] - - [Crux] Overall status: Invalid. - - Use -p '/Output testing.symbolic/' to rerun this test only. - leading_zeros: FAIL (0.12s) - files test/symb_eval/bitvector/leading_zeros.good and test/symb_eval/bitvector/leading_zeros.out differ; test/symb_eval/bitvector/leading_zeros.out contains: - test leading_zeros/8f264035::crux_test[0]: FAILED - - failures: - - ---- leading_zeros/8f264035::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/leading_zeros/' to rerun this test only. - zero_length: FAIL (0.10s) - files test/symb_eval/alloc/zero_length.good and test/symb_eval/alloc/zero_length.out differ; test/symb_eval/alloc/zero_length.out contains: - test zero_length/5fa7d318::crux_test[0]: FAILED - - failures: - - ---- zero_length/5fa7d318::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/zero_length/' to rerun this test only. - out_of_bounds: FAIL (0.11s) - files test/symb_eval/alloc/out_of_bounds.good and test/symb_eval/alloc/out_of_bounds.out differ; test/symb_eval/alloc/out_of_bounds.out contains: - test out_of_bounds/fba84cc3::crux_test[0]: FAILED - - failures: - - ---- out_of_bounds/fba84cc3::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/out_of_bounds/' to rerun this test only. - valid_read: FAIL (0.11s) - files test/symb_eval/alloc/valid_read.good and test/symb_eval/alloc/valid_read.out differ; test/symb_eval/alloc/valid_read.out contains: - test valid_read/111c217c::crux_test[0]: FAILED - - failures: - - ---- valid_read/111c217c::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/valid_read/' to rerun this test only. - uninit_read: FAIL (0.10s) - files test/symb_eval/alloc/uninit_read.good and test/symb_eval/alloc/uninit_read.out differ; test/symb_eval/alloc/uninit_read.out contains: - test uninit_read/a73b11f2::crux_test[0]: FAILED - - failures: - - ---- uninit_read/a73b11f2::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/uninit_read/' to rerun this test only. - assert_ok: FAIL (0.68s) - files test/symb_eval/concretize/assert_ok.good and test/symb_eval/concretize/assert_ok.out differ; test/symb_eval/concretize/assert_ok.out contains: - test assert_ok/d97cacde::crux_test[0]: FAILED - - failures: - - ---- assert_ok/d97cacde::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/assert_ok/' to rerun this test only. - array: FAIL (0.12s) - files test/symb_eval/concretize/array.good and test/symb_eval/concretize/array.out differ; test/symb_eval/concretize/array.out contains: - test array/08515081::crux_test[0]: FAILED - - failures: - - ---- array/08515081::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.array"' to rerun this test only. - assert: FAIL (0.66s) - files test/symb_eval/concretize/assert.good and test/symb_eval/concretize/assert.out differ; test/symb_eval/concretize/assert.out contains: - test assert/30ec92d4::crux_test[0]: FAILED - - failures: - - ---- assert/30ec92d4::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.assert"' to rerun this test only. - no_conc: FAIL (0.09s) - files test/symb_eval/concretize/no_conc.good and test/symb_eval/concretize/no_conc.out differ; test/symb_eval/concretize/no_conc.out contains: - test no_conc/73a9c3e5::crux_test[0]: FAILED - - failures: - - ---- no_conc/73a9c3e5::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/no_conc/' to rerun this test only. - conc: FAIL (0.09s) - files test/symb_eval/concretize/conc.good and test/symb_eval/concretize/conc.out differ; test/symb_eval/concretize/conc.out contains: - test conc/052a57bb::crux_test[0]: FAILED - - failures: - - ---- conc/052a57bb::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.conc"' to rerun this test only. - conditional: FAIL - test/symb_eval/any/conditional.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/any/conditional.rs) - Use -p '/conditional/' to rerun this test only. - downcast_fail: FAIL (0.09s) - files test/symb_eval/any/downcast_fail.good and test/symb_eval/any/downcast_fail.out differ; test/symb_eval/any/downcast_fail.out contains: - test downcast_fail/c51bb09a::crux_test[0]: FAILED - - failures: - - ---- downcast_fail/c51bb09a::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/downcast_fail/' to rerun this test only. - downcast: FAIL (0.09s) - files test/symb_eval/any/downcast.good and test/symb_eval/any/downcast.out differ; test/symb_eval/any/downcast.out contains: - test downcast/e8294f5a::crux_test[0]: FAILED - - failures: - - ---- downcast/e8294f5a::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.downcast"' to rerun this test only. - mux: FAIL (0.10s) - files test/symb_eval/fnptr/mux.good and test/symb_eval/fnptr/mux.out differ; test/symb_eval/fnptr/mux.out contains: - test mux/756a6435::crux_test[0]: FAILED - - failures: - - ---- mux/756a6435::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.mux"' to rerun this test only. - mux_init_mut: FAIL (0.10s) - files test/symb_eval/refs/mux_init_mut.good and test/symb_eval/refs/mux_init_mut.out differ; test/symb_eval/refs/mux_init_mut.out contains: - test mux_init_mut/4b29e475::crux_test[0]: FAILED - - failures: - - ---- mux_init_mut/4b29e475::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/mux_init_mut/' to rerun this test only. - mux_init_imm: FAIL (0.10s) - files test/symb_eval/refs/mux_init_imm.good and test/symb_eval/refs/mux_init_imm.out differ; test/symb_eval/refs/mux_init_imm.out contains: - test mux_init_imm/a8d4900d::crux_test[0]: FAILED - - failures: - - ---- mux_init_imm/a8d4900d::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/mux_init_imm/' to rerun this test only. - as_mut_slice: FAIL (0.10s) - files test/symb_eval/vector/as_mut_slice.good and test/symb_eval/vector/as_mut_slice.out differ; test/symb_eval/vector/as_mut_slice.out contains: - test as_mut_slice/393f23fa::f[0]: FAILED - - failures: - - ---- as_mut_slice/393f23fa::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/as_mut_slice/' to rerun this test only. - new: FAIL (0.09s) - files test/symb_eval/vector/new.good and test/symb_eval/vector/new.out differ; test/symb_eval/vector/new.out contains: - test new/d901c99d::f[0]: FAILED - - failures: - - ---- new/d901c99d::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.new"' to rerun this test only. - concat: FAIL (0.10s) - files test/symb_eval/vector/concat.good and test/symb_eval/vector/concat.out differ; test/symb_eval/vector/concat.out contains: - test concat/b4c814d1::f[0]: FAILED - - failures: - - ---- concat/b4c814d1::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/concat/' to rerun this test only. - split_at: FAIL (0.09s) - files test/symb_eval/vector/split_at.good and test/symb_eval/vector/split_at.out differ; test/symb_eval/vector/split_at.out contains: - test split_at/aa8f5aa9::f[0]: FAILED - - failures: - - ---- split_at/aa8f5aa9::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/split_at/' to rerun this test only. - as_slice: FAIL (0.10s) - files test/symb_eval/vector/as_slice.good and test/symb_eval/vector/as_slice.out differ; test/symb_eval/vector/as_slice.out contains: - test as_slice/125ec1c7::f[0]: FAILED - - failures: - - ---- as_slice/125ec1c7::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/as_slice/' to rerun this test only. - copy_from_slice: FAIL (0.11s) - files test/symb_eval/vector/copy_from_slice.good and test/symb_eval/vector/copy_from_slice.out differ; test/symb_eval/vector/copy_from_slice.out contains: - test copy_from_slice/57fc6028::f[0]: FAILED - - failures: - - ---- copy_from_slice/57fc6028::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/copy_from_slice/' to rerun this test only. - pop: FAIL (0.11s) - files test/symb_eval/vector/pop.good and test/symb_eval/vector/pop.out differ; test/symb_eval/vector/pop.out contains: - test pop/ebc7234e::f[0]: FAILED - - failures: - - ---- pop/ebc7234e::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/Output testing.pop/' to rerun this test only. - push: FAIL (0.10s) - files test/symb_eval/vector/push.good and test/symb_eval/vector/push.out differ; test/symb_eval/vector/push.out contains: - test push/0bce6253::f[0]: FAILED - - failures: - - ---- push/0bce6253::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/Output testing.push/' to rerun this test only. - mut: FAIL (0.10s) - files test/symb_eval/vector/mut.good and test/symb_eval/vector/mut.out differ; test/symb_eval/vector/mut.out contains: - test mut/10d04785::f[0]: FAILED - - failures: - - ---- mut/10d04785::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/Output testing.mut/' to rerun this test only. - replicate: FAIL (0.11s) - files test/symb_eval/vector/replicate.good and test/symb_eval/vector/replicate.out differ; test/symb_eval/vector/replicate.out contains: - test replicate/bf8f8227::f[0]: FAILED - - failures: - - ---- replicate/bf8f8227::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/replicate/' to rerun this test only. - multi: FAIL (0.12s) - files test/symb_eval/crux/multi.good and test/symb_eval/crux/multi.out differ; test/symb_eval/crux/multi.out contains: - test multi/05702274::fail1[0]: FAILED - test multi/05702274::fail2[0]: FAILED - test multi/05702274::fail3[0]: FAILED - - failures: - - ---- multi/05702274::fail1[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - ---- multi/05702274::fail2[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - ---- multi/05702274::fail3[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/Output testing.multi/' to rerun this test only. - early_fail: FAIL (0.08s) - files test/symb_eval/crux/early_fail.good and test/symb_eval/crux/early_fail.out differ; test/symb_eval/crux/early_fail.out contains: - test early_fail/c90fbdca::fail1[0]: FAILED - test early_fail/c90fbdca::fail2[0]: FAILED - - failures: - - ---- early_fail/c90fbdca::fail1[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - ---- early_fail/c90fbdca::fail2[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/early_fail/' to rerun this test only. - mixed_fail: FAIL (0.13s) - files test/symb_eval/crux/mixed_fail.good and test/symb_eval/crux/mixed_fail.out differ; test/symb_eval/crux/mixed_fail.out contains: - test mixed_fail/2843d7dd::fail1[0]: FAILED - test mixed_fail/2843d7dd::fail2[0]: FAILED - test mixed_fail/2843d7dd::pass1[0]: FAILED - test mixed_fail/2843d7dd::pass2[0]: FAILED - - failures: - - ---- mixed_fail/2843d7dd::fail1[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - ---- mixed_fail/2843d7dd::fail2[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - ---- mixed_fail/2843d7dd::pass1[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - ---- mixed_fail/2843d7dd::pass2[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/mixed_fail/' to rerun this test only. - fail_return: FAIL (0.09s) - files test/symb_eval/crux/fail_return.good and test/symb_eval/crux/fail_return.out differ; test/symb_eval/crux/fail_return.out contains: - test fail_return/384998af::fail1[0]: FAILED - test fail_return/384998af::fail2[0]: FAILED - - failures: - - ---- fail_return/384998af::fail1[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - ---- fail_return/384998af::fail2[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/fail_return/' to rerun this test only. - array: FAIL (0.11s) - files test/symb_eval/mux/array.good and test/symb_eval/mux/array.out differ; test/symb_eval/mux/array.out contains: - test array/e2517ba7::crux_test[0]: FAILED - - failures: - - ---- array/e2517ba7::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.array"' to rerun this test only. - array_mut: FAIL (0.10s) - files test/symb_eval/mux/array_mut.good and test/symb_eval/mux/array_mut.out differ; test/symb_eval/mux/array_mut.out contains: - test array_mut/f23954b5::crux_test[0]: FAILED - - failures: - - ---- array_mut/f23954b5::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/array_mut/' to rerun this test only. - checked_add: FAIL (0.02s) - files test/symb_eval/num/checked_add.good and test/symb_eval/num/checked_add.out differ; test/symb_eval/num/checked_add.out contains: - test checked_add/a1f7d9d2::crux_test[0]: returned 44, FAILED - - failures: - - ---- checked_add/a1f7d9d2::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/num/checked_add.rs:5:5: 5:12: error: in checked_add/a1f7d9d2::crux_test[0] - [Crux] attempt to compute `const 100_u8 + move _2`, which would overflow - - [Crux] Overall status: Invalid. - - Use -p '/checked_add/' to rerun this test only. - checked_div: FAIL (0.02s) - files test/symb_eval/num/checked_div.good and test/symb_eval/num/checked_div.out differ; test/symb_eval/num/checked_div.out contains: - test checked_div/cb81bc69::crux_test[0]: returned 65, FAILED - - failures: - - ---- checked_div/cb81bc69::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/cb81bc69::div_signed[0] - [Crux] attempt to compute `_3 / _4`, which would overflow - [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/cb81bc69::div_signed[0] - [Crux] attempt to divide `_3` by zero - [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/cb81bc69::div_unsigned[0] - [Crux] attempt to divide `_3` by zero - - [Crux] Overall status: Invalid. - - Use -p '/checked_div/' to rerun this test only. - checked_mul: FAIL (0.01s) - files test/symb_eval/num/checked_mul.good and test/symb_eval/num/checked_mul.out differ; test/symb_eval/num/checked_mul.out contains: - test checked_mul/98f17c38::crux_test[0]: returned 44, FAILED - - failures: - - ---- checked_mul/98f17c38::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/num/checked_mul.rs:4:5: 4:12: error: in checked_mul/98f17c38::crux_test[0] - [Crux] attempt to compute `const 100_u8 * move _2`, which would overflow - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.checked_mul"' to rerun this test only. - checked_mul_signed: FAIL (0.02s) - files test/symb_eval/num/checked_mul_signed.good and test/symb_eval/num/checked_mul_signed.out differ; test/symb_eval/num/checked_mul_signed.out contains: - test checked_mul_signed/2b522f9c::crux_test[0]: returned 44, FAILED - - failures: - - ---- checked_mul_signed/2b522f9c::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/num/checked_mul_signed.rs:4:5: 4:13: error: in checked_mul_signed/2b522f9c::crux_test[0] - [Crux] attempt to compute `const -100_i8 * move _2`, which would overflow - - [Crux] Overall status: Invalid. - - Use -p '/checked_mul_signed/' to rerun this test only. - ffs: FAIL (0.13s) - files test/symb_eval/crypto/ffs.good and test/symb_eval/crypto/ffs.out differ; test/symb_eval/crypto/ffs.out contains: - test ffs/310e17b4::f[0]: FAILED - - failures: - - ---- ffs/310e17b4::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/Output testing.ffs/' to rerun this test only. - bytes2: FAIL (0.15s) - files test/symb_eval/crypto/bytes2.good and test/symb_eval/crypto/bytes2.out differ; test/symb_eval/crypto/bytes2.out contains: - test bytes2/315034bf::f[0]: FAILED - - failures: - - ---- bytes2/315034bf::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/bytes2/' to rerun this test only. - bytes: FAIL (0.14s) - files test/symb_eval/crypto/bytes.good and test/symb_eval/crypto/bytes.out differ; test/symb_eval/crypto/bytes.out contains: - test bytes/c88d09fa::f[0]: FAILED - - failures: - - ---- bytes/c88d09fa::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.bytes"' to rerun this test only. - double: FAIL (0.07s) - files test/symb_eval/crypto/double.good and test/symb_eval/crypto/double.out differ; test/symb_eval/crypto/double.out contains: - test double/15fd1104::f[0]: FAILED - - failures: - - ---- double/15fd1104::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/double/' to rerun this test only. - test1: FAIL (0.22s) - files test/symb_eval/scalar/test1.good and test/symb_eval/scalar/test1.out differ; test/symb_eval/scalar/test1.out contains: - test test1/c6233792::f[0]: FAILED - - failures: - - ---- test1/c6233792::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/test1/' to rerun this test only. - deserialize: FAIL (0.10s) - files test/symb_eval/sym_bytes/deserialize.good and test/symb_eval/sym_bytes/deserialize.out differ; test/symb_eval/sym_bytes/deserialize.out contains: - test deserialize/5d52ecde::crux_test[0]: FAILED - - failures: - - ---- deserialize/5d52ecde::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/deserialize/' to rerun this test only. - construct: FAIL (0.09s) - files test/symb_eval/sym_bytes/construct.good and test/symb_eval/sym_bytes/construct.out differ; test/symb_eval/sym_bytes/construct.out contains: - test construct/caa9861a::crux_test[0]: FAILED - - failures: - - ---- construct/caa9861a::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:316:65: 320:2: error: in core/516ae8d9::fmt[0]::USIZE_MARKER[0] - [Crux] Translation error in core/516ae8d9::fmt[0]::USIZE_MARKER[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [TyRef (TyUint USize) Immut,TyRef (TyAdt core/516ae8d9::fmt[0]::Formatter[0]::_adtbd21306cbe4f0b9b[0] core/516ae8d9::fmt[0]::Formatter[0] (Substs [TyLifetime])) Mut], _fsreturn_ty = TyAdt core/516ae8d9::result[0]::Result[0]::_adtb3f4e538b15dbdda[0] core/516ae8d9::result[0]::Result[0] (Substs [TyTuple [],TyAdt core/516ae8d9::fmt[0]::Error[0]::_adtb7803c2264daf0ec[0] core/516ae8d9::fmt[0]::Error[0] (Substs [])]), _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - Use -p '/construct/' to rerun this test only. - vec_cursor_read: FAIL (0.79s) - standalone use of `dyn` is not supported: TyDynamic core/516ae8d9::error[0]::Error[0]::_trait56353a7b840403b4[0] - CallStack (from HasCallStack): - error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy - tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate - translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language - Use -p '/vec_cursor_read/' to rerun this test only. - vec_write: FAIL (0.46s) - standalone use of `dyn` is not supported: TyDynamic core/516ae8d9::error[0]::Error[0]::_trait56353a7b840403b4[0] - CallStack (from HasCallStack): - error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy - tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate - translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language - Use -p '/vec_write/' to rerun this test only. - crux coverage - Output testing - coverage_loop: warning: trailing semicolon in macro used in expression position - --> src/main.rs:45:46 - | -45 | return Err(format!($($args)*).into()); - | ^ -... -309 | _ => die!("unknown tag {:?} for branch", tag), - | ---------------------------------------- in this macro invocation - | - = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default - = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! - = note: for more information, see issue #79813 - = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) - -FAIL (0.09s) - files test/coverage/coverage_loop.good and test/coverage/coverage_loop.out differ; test/coverage/coverage_loop.out contains: - warning: branch condition never has value false - ┌─ /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/iter/range.rs:621:12 - │ - 621 │ if self.start < self.end { - │ ^^^^^^^^^^^^^^^^^^^^^ - - - Use -p '/coverage_loop/' to rerun this test only. - coverage: warning: trailing semicolon in macro used in expression position - --> src/main.rs:45:46 - | -45 | return Err(format!($($args)*).into()); - | ^ -... -309 | _ => die!("unknown tag {:?} for branch", tag), - | ---------------------------------------- in this macro invocation - | - = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default - = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! - = note: for more information, see issue #79813 - = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) - -FAIL (0.16s) - files test/coverage/coverage.good and test/coverage/coverage.out differ; test/coverage/coverage.out contains: - - Use -p '$0=="crux-mir.crux coverage.Output testing.coverage"' to rerun this test only. - coverage_macro: warning: trailing semicolon in macro used in expression position - --> src/main.rs:45:46 - | -45 | return Err(format!($($args)*).into()); - | ^ -... -309 | _ => die!("unknown tag {:?} for branch", tag), - | ---------------------------------------- in this macro invocation - | - = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default - = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! - = note: for more information, see issue #79813 - = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) - -FAIL (0.16s) - files test/coverage/coverage_macro.good and test/coverage/coverage_macro.out differ; test/coverage/coverage_macro.out contains: - - Use -p '/coverage_macro/' to rerun this test only. - coverage_try: warning: trailing semicolon in macro used in expression position - --> src/main.rs:45:46 - | -45 | return Err(format!($($args)*).into()); - | ^ -... -309 | _ => die!("unknown tag {:?} for branch", tag), - | ---------------------------------------- in this macro invocation - | - = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default - = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! - = note: for more information, see issue #79813 - = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) - -FAIL (0.14s) - files test/coverage/coverage_try.good and test/coverage/coverage_try.out differ; test/coverage/coverage_try.out contains: - - Use -p '/coverage_try/' to rerun this test only. - coverage_mono: warning: trailing semicolon in macro used in expression position - --> src/main.rs:45:46 - | -45 | return Err(format!($($args)*).into()); - | ^ -... -309 | _ => die!("unknown tag {:?} for branch", tag), - | ---------------------------------------- in this macro invocation - | - = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default - = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! - = note: for more information, see issue #79813 - = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) - -OK (0.09s) - coverage_shortcircuit: warning: trailing semicolon in macro used in expression position - --> src/main.rs:45:46 - | -45 | return Err(format!($($args)*).into()); - | ^ -... -309 | _ => die!("unknown tag {:?} for branch", tag), - | ---------------------------------------- in this macro invocation - | - = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default - = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! - = note: for more information, see issue #79813 - = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) - -FAIL (0.14s) - files test/coverage/coverage_shortcircuit.good and test/coverage/coverage_shortcircuit.out differ; test/coverage/coverage_shortcircuit.out contains: - - Use -p '/coverage_shortcircuit/' to rerun this test only. - coverage_dual: warning: trailing semicolon in macro used in expression position - --> src/main.rs:45:46 - | -45 | return Err(format!($($args)*).into()); - | ^ -... -309 | _ => die!("unknown tag {:?} for branch", tag), - | ---------------------------------------- in this macro invocation - | - = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default - = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! - = note: for more information, see issue #79813 - = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) - -OK (0.09s) - coverage_cond: warning: trailing semicolon in macro used in expression position - --> src/main.rs:45:46 - | -45 | return Err(format!($($args)*).into()); - | ^ -... -309 | _ => die!("unknown tag {:?} for branch", tag), - | ---------------------------------------- in this macro invocation - | - = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default - = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! - = note: for more information, see issue #79813 - = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) - -OK (0.09s) - coverage_match: warning: trailing semicolon in macro used in expression position - --> src/main.rs:45:46 - | -45 | return Err(format!($($args)*).into()); - | ^ -... -309 | _ => die!("unknown tag {:?} for branch", tag), - | ---------------------------------------- in this macro invocation - | - = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default - = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! - = note: for more information, see issue #79813 - = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) - -FAIL (0.16s) - files test/coverage/coverage_match.good and test/coverage/coverage_match.out differ; test/coverage/coverage_match.out contains: - - Use -p '/coverage_match/' to rerun this test only. - -200 out of 339 tests failed (73.36s) + basic: \ No newline at end of file From 8c590e48d263c59a7da76aeb1db19e64ad3857ab Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Tue, 2 May 2023 13:14:38 -0400 Subject: [PATCH 018/114] crux-mir: Note fmt reimplementation in lib/Patches.md Adapted from the original commit in 52f6f57f42ec7d35b5f19caa844611b6dbd9e82d --- crux-mir/lib/Patches.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index 767e0e9e1..bcd0e3f18 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -19,3 +19,7 @@ identify all of the code that was changed in each patch. built-in Crucible allocation functions (e.g., `crucible::alloc::allocate`). We also make sure to avoid the `Layout::array` function, which has a particularly tricky use of `transmute` that we do not currently support. + +* Reimplement `core::fmt` using `crucible::any::Any` (last applied: May 2, 2023) + + TODO: Describe why this is necessary From 8a680c4fd18f21651e5f2ebbe8c420238c9bfc8a Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Tue, 2 May 2023 14:30:21 -0400 Subject: [PATCH 019/114] Mir.TransCustom: Use {impl} consistently --- crucible-mir/src/Mir/TransCustom.hs | 78 ++++++++++++++--------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/crucible-mir/src/Mir/TransCustom.hs b/crucible-mir/src/Mir/TransCustom.hs index b11af8b09..a0ae5ece1 100644 --- a/crucible-mir/src/Mir/TransCustom.hs +++ b/crucible-mir/src/Mir/TransCustom.hs @@ -221,14 +221,14 @@ panicking_panicking = (["std", "panicking", "panicking"], \_ -> Just $ CustomOp -- Methods for crucible::vector::Vector (which has custom representation) vector_new :: (ExplodedDefId, CustomRHS) -vector_new = ( ["crucible","vector","{{impl}}", "new"], ) $ \substs -> case substs of +vector_new = ( ["crucible","vector","{impl}", "new"], ) $ \substs -> case substs of Substs [t] -> Just $ CustomOp $ \_ _ -> do Some tpr <- tyToReprM t return $ MirExp (C.VectorRepr tpr) (R.App $ E.VectorLit tpr V.empty) _ -> Nothing vector_replicate :: (ExplodedDefId, CustomRHS) -vector_replicate = ( ["crucible","vector","{{impl}}", "replicate"], ) $ \substs -> case substs of +vector_replicate = ( ["crucible","vector","{impl}", "replicate"], ) $ \substs -> case substs of Substs [t] -> Just $ CustomOp $ \_ ops -> case ops of [MirExp tpr eVal, MirExp UsizeRepr eLen] -> do let eLenNat = R.App $ usizeToNat eLen @@ -237,7 +237,7 @@ vector_replicate = ( ["crucible","vector","{{impl}}", "replicate"], ) $ \substs _ -> Nothing vector_len :: (ExplodedDefId, CustomRHS) -vector_len = ( ["crucible","vector","{{impl}}", "len"], ) $ \substs -> case substs of +vector_len = ( ["crucible","vector","{impl}", "len"], ) $ \substs -> case substs of Substs [t] -> Just $ CustomOp $ \_ ops -> case ops of [MirExp (MirReferenceRepr (C.VectorRepr tpr)) eRef] -> do e <- readMirRef (C.VectorRepr tpr) eRef @@ -246,7 +246,7 @@ vector_len = ( ["crucible","vector","{{impl}}", "len"], ) $ \substs -> case subs _ -> Nothing vector_push :: (ExplodedDefId, CustomRHS) -vector_push = ( ["crucible","vector","{{impl}}", "push"], ) $ \substs -> case substs of +vector_push = ( ["crucible","vector","{impl}", "push"], ) $ \substs -> case substs of Substs [t] -> Just $ CustomOp $ \_ ops -> case ops of [MirExp (C.VectorRepr tpr) eVec, MirExp tpr' eItem] | Just Refl <- testEquality tpr tpr' -> do @@ -256,7 +256,7 @@ vector_push = ( ["crucible","vector","{{impl}}", "push"], ) $ \substs -> case su _ -> Nothing vector_push_front :: (ExplodedDefId, CustomRHS) -vector_push_front = ( ["crucible","vector","{{impl}}", "push_front"], ) $ \substs -> case substs of +vector_push_front = ( ["crucible","vector","{impl}", "push_front"], ) $ \substs -> case substs of Substs [t] -> Just $ CustomOp $ \_ ops -> case ops of [MirExp (C.VectorRepr tpr) eVec, MirExp tpr' eItem] | Just Refl <- testEquality tpr tpr' -> do @@ -266,7 +266,7 @@ vector_push_front = ( ["crucible","vector","{{impl}}", "push_front"], ) $ \subst _ -> Nothing vector_pop :: (ExplodedDefId, CustomRHS) -vector_pop = ( ["crucible","vector","{{impl}}", "pop"], ) $ \substs -> case substs of +vector_pop = ( ["crucible","vector","{impl}", "pop"], ) $ \substs -> case substs of Substs [t] -> Just $ CustomOp $ \_ ops -> case ops of [MirExp (C.VectorRepr tpr) eVec] -> do meInit <- MirExp (C.VectorRepr tpr) <$> vectorInit tpr eVec @@ -277,7 +277,7 @@ vector_pop = ( ["crucible","vector","{{impl}}", "pop"], ) $ \substs -> case subs _ -> Nothing vector_pop_front :: (ExplodedDefId, CustomRHS) -vector_pop_front = ( ["crucible","vector","{{impl}}", "pop_front"], ) $ \substs -> case substs of +vector_pop_front = ( ["crucible","vector","{impl}", "pop_front"], ) $ \substs -> case substs of Substs [t] -> Just $ CustomOp $ \_ ops -> case ops of [MirExp (C.VectorRepr tpr) eVec] -> do -- `Option` must exist because it appears in the return type. @@ -306,13 +306,13 @@ vector_as_slice_impl _ = Nothing -- `&Vector` and `&[T]` use the same representations as `&mut Vector` and -- `&mut [T]`, so we can use the same implementation. vector_as_slice :: (ExplodedDefId, CustomRHS) -vector_as_slice = ( ["crucible","vector","{{impl}}", "as_slice"], vector_as_slice_impl ) +vector_as_slice = ( ["crucible","vector","{impl}", "as_slice"], vector_as_slice_impl ) vector_as_mut_slice :: (ExplodedDefId, CustomRHS) -vector_as_mut_slice = ( ["crucible","vector","{{impl}}", "as_mut_slice"], vector_as_slice_impl ) +vector_as_mut_slice = ( ["crucible","vector","{impl}", "as_mut_slice"], vector_as_slice_impl ) vector_concat :: (ExplodedDefId, CustomRHS) -vector_concat = ( ["crucible","vector","{{impl}}", "concat"], ) $ \substs -> case substs of +vector_concat = ( ["crucible","vector","{impl}", "concat"], ) $ \substs -> case substs of Substs [t] -> Just $ CustomOp $ \_ ops -> case ops of [MirExp (C.VectorRepr tpr1) e1, MirExp (C.VectorRepr tpr2) e2] | Just Refl <- testEquality tpr1 tpr2 -> do @@ -321,7 +321,7 @@ vector_concat = ( ["crucible","vector","{{impl}}", "concat"], ) $ \substs -> cas _ -> Nothing vector_split_at :: (ExplodedDefId, CustomRHS) -vector_split_at = ( ["crucible","vector","{{impl}}", "split_at"], ) $ \substs -> case substs of +vector_split_at = ( ["crucible","vector","{impl}", "split_at"], ) $ \substs -> case substs of Substs [t] -> Just $ CustomOp $ \_ ops -> case ops of [MirExp (C.VectorRepr tpr) eVec, MirExp UsizeRepr eIdx] -> do let eIdxNat = R.App $ usizeToNat eIdx @@ -332,7 +332,7 @@ vector_split_at = ( ["crucible","vector","{{impl}}", "split_at"], ) $ \substs -> _ -> Nothing vector_copy_from_slice :: (ExplodedDefId, CustomRHS) -vector_copy_from_slice = ( ["crucible","vector","{{impl}}", "copy_from_slice"], ) $ \substs -> case substs of +vector_copy_from_slice = ( ["crucible","vector","{impl}", "copy_from_slice"], ) $ \substs -> case substs of Substs [t] -> Just $ CustomOp $ \_ ops -> case ops of [MirExp (MirSliceRepr tpr) e] -> do let ptr = getSlicePtr e @@ -349,7 +349,7 @@ vector_copy_from_slice = ( ["crucible","vector","{{impl}}", "copy_from_slice"], -- Methods for crucible::array::Array (which has custom representation) array_zeroed :: (ExplodedDefId, CustomRHS) -array_zeroed = ( ["crucible","array","{{impl}}", "zeroed"], ) $ \substs -> case substs of +array_zeroed = ( ["crucible","array","{impl}", "zeroed"], ) $ \substs -> case substs of Substs [t] -> Just $ CustomOp $ \_ _ -> tyToReprM t >>= \(Some tpr) -> case tpr of C.BVRepr w -> do let idxs = Ctx.Empty Ctx.:> BaseUsizeRepr @@ -359,7 +359,7 @@ array_zeroed = ( ["crucible","array","{{impl}}", "zeroed"], ) $ \substs -> case _ -> Nothing array_lookup :: (ExplodedDefId, CustomRHS) -array_lookup = ( ["crucible","array","{{impl}}", "lookup"], ) $ \substs -> case substs of +array_lookup = ( ["crucible","array","{impl}", "lookup"], ) $ \substs -> case substs of Substs [t] -> Just $ CustomOp $ \_ ops -> case ops of [ MirExp (UsizeArrayRepr btr) eArr, MirExp UsizeRepr eIdx ] -> do @@ -370,7 +370,7 @@ array_lookup = ( ["crucible","array","{{impl}}", "lookup"], ) $ \substs -> case _ -> Nothing array_update :: (ExplodedDefId, CustomRHS) -array_update = ( ["crucible","array","{{impl}}", "update"], ) $ \substs -> case substs of +array_update = ( ["crucible","array","{impl}", "update"], ) $ \substs -> case substs of Substs [t] -> Just $ CustomOp $ \_ ops -> case ops of [ MirExp arrTpr@(UsizeArrayRepr btr) eArr, MirExp UsizeRepr eIdx, @@ -396,10 +396,10 @@ array_as_slice_impl (Substs [t]) = array_as_slice_impl _ = Nothing array_as_slice :: (ExplodedDefId, CustomRHS) -array_as_slice = ( ["crucible","array","{{impl}}", "as_slice"], array_as_slice_impl ) +array_as_slice = ( ["crucible","array","{impl}", "as_slice"], array_as_slice_impl ) array_as_mut_slice :: (ExplodedDefId, CustomRHS) -array_as_mut_slice = ( ["crucible","array","{{impl}}", "as_mut_slice"], array_as_slice_impl ) +array_as_mut_slice = ( ["crucible","array","{impl}", "as_mut_slice"], array_as_slice_impl ) @@ -409,7 +409,7 @@ array_as_mut_slice = ( ["crucible","array","{{impl}}", "as_mut_slice"], array_as -- Methods for crucible::any::Any (which has custom representation) any_new :: (ExplodedDefId, CustomRHS) -any_new = ( ["core", "crucible", "any", "{{impl}}", "new"], \substs -> case substs of +any_new = ( ["core", "crucible", "any", "{impl}", "new"], \substs -> case substs of Substs [_] -> Just $ CustomOp $ \_ ops -> case ops of [MirExp tpr e] -> do return $ MirExp C.AnyRepr $ R.App $ E.PackAny tpr e @@ -418,7 +418,7 @@ any_new = ( ["core", "crucible", "any", "{{impl}}", "new"], \substs -> case subs ) any_downcast :: (ExplodedDefId, CustomRHS) -any_downcast = ( ["core", "crucible", "any", "{{impl}}", "downcast"], \substs -> case substs of +any_downcast = ( ["core", "crucible", "any", "{impl}", "downcast"], \substs -> case substs of Substs [t] -> Just $ CustomOp $ \_ ops -> case ops of [MirExp C.AnyRepr e] -> do Some tpr <- tyToReprM t @@ -444,9 +444,9 @@ ptr_offset_impl = \substs -> case substs of _ -> Nothing ptr_offset :: (ExplodedDefId, CustomRHS) -ptr_offset = (["core", "ptr", "const_ptr", "{{impl}}", "offset"], ptr_offset_impl) +ptr_offset = (["core", "ptr", "const_ptr", "{impl}", "offset"], ptr_offset_impl) ptr_offset_mut :: (ExplodedDefId, CustomRHS) -ptr_offset_mut = (["core", "ptr", "mut_ptr", "{{impl}}", "offset"], ptr_offset_impl) +ptr_offset_mut = (["core", "ptr", "mut_ptr", "{impl}", "offset"], ptr_offset_impl) ptr_wrapping_offset_impl :: CustomRHS ptr_wrapping_offset_impl = \substs -> case substs of @@ -458,10 +458,10 @@ ptr_wrapping_offset_impl = \substs -> case substs of ptr_wrapping_offset :: (ExplodedDefId, CustomRHS) ptr_wrapping_offset = - (["core", "ptr", "const_ptr", "{{impl}}", "wrapping_offset"], ptr_wrapping_offset_impl) + (["core", "ptr", "const_ptr", "{impl}", "wrapping_offset"], ptr_wrapping_offset_impl) ptr_wrapping_offset_mut :: (ExplodedDefId, CustomRHS) ptr_wrapping_offset_mut = - (["core", "ptr", "mut_ptr", "{{impl}}", "wrapping_offset"], ptr_wrapping_offset_impl) + (["core", "ptr", "mut_ptr", "{impl}", "wrapping_offset"], ptr_wrapping_offset_impl) ptr_offset_from_impl :: CustomRHS ptr_offset_from_impl = \substs -> case substs of @@ -477,9 +477,9 @@ ptr_offset_from_impl = \substs -> case substs of _ -> Nothing ptr_offset_from :: (ExplodedDefId, CustomRHS) -ptr_offset_from = (["core", "ptr", "const_ptr", "{{impl}}", "offset_from"], ptr_offset_from_impl) +ptr_offset_from = (["core", "ptr", "const_ptr", "{impl}", "offset_from"], ptr_offset_from_impl) ptr_offset_from_mut :: (ExplodedDefId, CustomRHS) -ptr_offset_from_mut = (["core", "ptr", "mut_ptr", "{{impl}}", "offset_from"], ptr_offset_from_impl) +ptr_offset_from_mut = (["core", "ptr", "mut_ptr", "{impl}", "offset_from"], ptr_offset_from_impl) ptr_compare_usize :: (ExplodedDefId, CustomRHS) ptr_compare_usize = (["core", "crucible", "ptr", "compare_usize"], @@ -1002,7 +1002,7 @@ assert_inhabited = (["core", "intrinsics", "", "assert_inhabited"], \_substs -> Just $ CustomOp $ \_ _ -> return $ MirExp C.UnitRepr $ R.App E.EmptyApp) array_from_slice :: (ExplodedDefId, CustomRHS) -array_from_slice = (["core","array", "{{impl}}", "try_from", "crucible_array_from_slice_hook"], +array_from_slice = (["core","array", "{impl}", "try_from", "crucible_array_from_slice_hook"], \substs -> Just $ CustomOpNamed $ \fnName ops -> do fn <- findFn fnName case (fn ^. fsig . fsreturn_ty, ops) of @@ -1042,7 +1042,7 @@ array_from_slice = (["core","array", "{{impl}}", "try_from", "crucible_array_fro slice_len :: (ExplodedDefId, CustomRHS) slice_len = - (["core","slice","{{impl}}","len", "crucible_slice_len_hook"] + (["core","slice","{impl}","len", "crucible_slice_len_hook"] , \(Substs [_]) -> Just $ CustomOp $ \ _optys ops -> case ops of [MirExp (MirSliceRepr _) e] -> do @@ -1068,12 +1068,12 @@ slice_index_usize_get_unchecked_impl _ = Nothing slice_index_usize_get_unchecked :: (ExplodedDefId, CustomRHS) slice_index_usize_get_unchecked = - ( ["core","slice","{{impl}}","get_unchecked", "crucible_hook_usize"] + ( ["core","slice","{impl}","get_unchecked", "crucible_hook_usize"] , slice_index_usize_get_unchecked_impl ) slice_index_usize_get_unchecked_mut :: (ExplodedDefId, CustomRHS) slice_index_usize_get_unchecked_mut = - ( ["core","slice","{{impl}}","get_unchecked_mut", "crucible_hook_usize"] + ( ["core","slice","{impl}","get_unchecked_mut", "crucible_hook_usize"] , slice_index_usize_get_unchecked_impl ) slice_index_range_get_unchecked_impl :: CustomRHS @@ -1094,12 +1094,12 @@ slice_index_range_get_unchecked_impl _ = Nothing slice_index_range_get_unchecked :: (ExplodedDefId, CustomRHS) slice_index_range_get_unchecked = - ( ["core","slice","{{impl}}","get_unchecked", "crucible_hook_range"] + ( ["core","slice","{impl}","get_unchecked", "crucible_hook_range"] , slice_index_range_get_unchecked_impl ) slice_index_range_get_unchecked_mut :: (ExplodedDefId, CustomRHS) slice_index_range_get_unchecked_mut = - ( ["core","slice","{{impl}}","get_unchecked_mut", "crucible_hook_range"] + ( ["core","slice","{impl}","get_unchecked_mut", "crucible_hook_range"] , slice_index_range_get_unchecked_impl ) -------------------------------------------------------------------------------------------------------------------------- @@ -1286,7 +1286,7 @@ type BVUnOp = forall ext f w. (1 <= w) -> E.App ext f (C.BVType w) bv_unop :: Text -> BVUnOp -> (ExplodedDefId, CustomRHS) -bv_unop name op = (["crucible", "bitvector", "{{impl}}", name], \(Substs [_sz]) -> +bv_unop name op = (["crucible", "bitvector", "{impl}", name], \(Substs [_sz]) -> Just $ CustomOp $ \_optys ops -> case ops of [MirExp (C.BVRepr w1) v1] -> return $ MirExp (C.BVRepr w1) (S.app $ op w1 v1) @@ -1300,7 +1300,7 @@ type BVBinOp = forall ext f w. (1 <= w) -> E.App ext f (C.BVType w) bv_binop :: Text -> BVBinOp -> (ExplodedDefId, CustomRHS) -bv_binop name op = (["crucible", "bitvector", "{{impl}}", name], bv_binop_impl name op) +bv_binop name op = (["crucible", "bitvector", "{impl}", name], bv_binop_impl name op) bv_binop_impl :: Text -> BVBinOp -> CustomRHS bv_binop_impl name op (Substs [_sz]) = Just $ CustomOp $ \_optys ops -> case ops of @@ -1314,12 +1314,12 @@ bv_shift_op name op = (["crucible", "bitvector", name], bv_binop_impl name op) bv_overflowing_binop :: Text -> BinOp -> (ExplodedDefId, CustomRHS) bv_overflowing_binop name bop = - ( ["crucible", "bitvector", "{{impl}}", "overflowing_" <> name] + ( ["crucible", "bitvector", "{impl}", "overflowing_" <> name] , makeArithWithOverflow ("bv_overflowing_" ++ Text.unpack name) (Just False) bop ) bv_eq :: (ExplodedDefId, CustomRHS) -bv_eq = (["crucible", "bitvector", "{{impl}}", "eq"], \(Substs [_sz]) -> +bv_eq = (["crucible", "bitvector", "{impl}", "eq"], \(Substs [_sz]) -> Just $ CustomOp $ \_optys ops -> case ops of [MirExp (MirReferenceRepr (C.BVRepr w1)) r1, MirExp (MirReferenceRepr (C.BVRepr w2)) r2] | Just Refl <- testEquality w1 w2 -> do @@ -1329,7 +1329,7 @@ bv_eq = (["crucible", "bitvector", "{{impl}}", "eq"], \(Substs [_sz]) -> _ -> mirFail $ "BUG: invalid arguments to bv_eq: " ++ show ops) bv_lt :: (ExplodedDefId, CustomRHS) -bv_lt = (["crucible", "bitvector", "{{impl}}", "lt"], \(Substs [_sz]) -> +bv_lt = (["crucible", "bitvector", "{impl}", "lt"], \(Substs [_sz]) -> Just $ CustomOp $ \_optys ops -> case ops of [MirExp (MirReferenceRepr (C.BVRepr w1)) r1, MirExp (MirReferenceRepr (C.BVRepr w2)) r2] | Just Refl <- testEquality w1 w2 -> do @@ -1342,7 +1342,7 @@ type BVMakeLiteral = forall ext f w. (1 <= w) => NatRepr w -> E.App ext f (C.BVType w) bv_literal :: Text -> BVMakeLiteral -> (ExplodedDefId, CustomRHS) -bv_literal name op = (["crucible", "bitvector", "{{impl}}", name], \(Substs [sz]) -> +bv_literal name op = (["crucible", "bitvector", "{impl}", name], \(Substs [sz]) -> Just $ CustomOp $ \_optys _ops -> tyToReprM (CTyBv sz) >>= \(Some tpr) -> case tpr of C.BVRepr w -> return $ MirExp (C.BVRepr w) $ S.app $ op w @@ -1351,7 +1351,7 @@ bv_literal name op = (["crucible", "bitvector", "{{impl}}", name], \(Substs [sz] bv_leading_zeros :: (ExplodedDefId, CustomRHS) bv_leading_zeros = - ( ["crucible", "bitvector", "{{impl}}", "leading_zeros"] + ( ["crucible", "bitvector", "{impl}", "leading_zeros"] , ctlz_impl "bv_leading_zeros" (Just $ Some $ knownNat @32) ) @@ -1542,7 +1542,7 @@ unlikely = (name, rhs) -- MaybeUninit maybe_uninit_uninit :: (ExplodedDefId, CustomRHS) -maybe_uninit_uninit = (["core", "mem", "maybe_uninit", "{{impl}}", "uninit"], +maybe_uninit_uninit = (["core", "mem", "maybe_uninit", "{impl}", "uninit"], \substs -> case substs of Substs [t] -> Just $ CustomOp $ \_ _ -> do adt <- findAdtInst (M.textId "core::mem::maybe_uninit::MaybeUninit") (Substs [t]) From 0653a31ef33162683c258ea7536a3bfd9399629e Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Tue, 2 May 2023 14:32:23 -0400 Subject: [PATCH 020/114] lib/crucible: Don't loop infinitely in zeroed() This took us a while to debug before we realized what was going on. Let's make the failure obvious. --- crux-mir/lib/crucible/array.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crux-mir/lib/crucible/array.rs b/crux-mir/lib/crucible/array.rs index a3b32d27f..cdb4df233 100644 --- a/crux-mir/lib/crucible/array.rs +++ b/crux-mir/lib/crucible/array.rs @@ -13,6 +13,9 @@ impl Clone for Array { } } +// NB: Most of these functions are unimplemented, as crucible-mir overrides +// their behavior with custom operations. + impl Array { /// Construct a new array, filled with zeros. /// @@ -20,7 +23,10 @@ impl Array { /// Crucible representation. All primitive integer types, as well as the wider bitvectors in /// `crucible::bitvector`, satisfy this requirement. pub const fn zeroed() -> Array { - Self::zeroed() + // NB: Unlike the other custom operations below, we can't use + // unimplemented! here, as that is forbidden in constant contexts. + // panic! is perhaps overkill, but it gets the job done. + panic!("Array::zeroed") } } From 592b2a128cde3b3fc25ad216d4079eada966e93f Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Tue, 2 May 2023 14:48:34 -0400 Subject: [PATCH 021/114] crux-mir: Regenerate test_output.log --- crux-mir/test_output.log | 4856 +++++++++++++++++++++----------------- 1 file changed, 2683 insertions(+), 2173 deletions(-) diff --git a/crux-mir/test_output.log b/crux-mir/test_output.log index 5728a8c47..9c39227b4 100644 --- a/crux-mir/test_output.log +++ b/crux-mir/test_output.log @@ -1,3029 +1,3539 @@ -Running scope as unit: run-rabc937d071ac43b59a95d19eca862da1.scope -Up to date +Build profile: -w ghc-9.2.7 -O1 +In order, the following will be built (use -v for more details): + - crux-mir-0.6.0.99 (test:test) (file /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/dist-newstyle/build/x86_64-linux/ghc-9.2.7/crux-mir-0.6.0.99/cache/build changed) +Preprocessing test suite 'test' for crux-mir-0.6.0.99.. +Building test suite 'test' for crux-mir-0.6.0.99.. +Linking /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/dist-newstyle/build/x86_64-linux/ghc-9.2.7/crux-mir-0.6.0.99/t/test/build/test/test ... crux-mir crux concrete - vec - set_len: FAIL (3.44s) - Compiling and running oracle program (0.91s) - Oracle output: 2 (2.53s) + refs + promoted_imm: OK (2.19s) + Compiling and running oracle program (0.19s) + Oracle output: 0 (2.00s) + Crux output: 0 + mut_raw: OK (2.15s) + Compiling and running oracle program (0.17s) + Oracle output: 123 (1.98s) + Crux output: 123 + mut_nested: OK (2.17s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.00s) + Crux output: () + fn_ptr_mut: OK (2.19s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (2.03s) + Crux output: 1 + never: OK (2.17s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (2.00s) + Crux output: 1 + mut_ref: OK (2.16s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (2.00s) + Crux output: 1 + temp: OK (2.22s) + Compiling and running oracle program (0.18s) + Oracle output: 1 (2.04s) + Crux output: 1 + mut_arg: OK (2.15s) + Compiling and running oracle program (0.18s) + Oracle output: 1 (1.97s) + Crux output: 1 + fn_ptr: OK (2.22s) + Compiling and running oracle program (0.21s) + Oracle output: 1 (2.01s) + Crux output: 1 + promoted_mut: OK (2.18s) + Compiling and running oracle program (0.19s) + Oracle output: 0 (2.00s) + Crux output: 0 + never_mut: OK (2.15s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (1.98s) + Crux output: 1 + mut_tuple_field: FAIL (2.22s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.05s) Crux output: failures: - ---- set_len/19a6ca9c::crux_test[0] counterexamples ---- + ---- mut_tuple_field/ef64915e::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] internal: error: in static_initializer + [Crux] Translation error in the static initializer: fail or unimp constant: TyTuple [TyInt B32,TyInt B32] (StructRepr [MaybeRepr (BVRepr 32), MaybeRepr (BVRepr 32)]) ConstTuple [ConstInt (I32 3),ConstInt (I32 4)] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/set_len/' to rerun this test only. - from_elem_zero: FAIL (0.35s) - Compiling and running oracle program (0.16s) - Oracle output: 0 (0.18s) - user error (Error 101 while running mir-json on test/conc_eval/vec/from_elem_zero.rs) + Use -p '/mut_tuple_field/' to rerun this test only. + imm_raw: OK (2.16s) + Compiling and running oracle program (0.18s) + Oracle output: 123 (1.98s) + Crux output: 123 + static_mut: OK (2.17s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (2.00s) + Crux output: 2 + imm_ref: OK (2.13s) + Compiling and running oracle program (0.18s) + Oracle output: 123 (1.95s) + Crux output: 123 + imm_arg: OK (2.14s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (1.97s) + Crux output: 1 + hash_map + insert_multi: FAIL (3.17s) + Compiling and running oracle program (0.22s) + Oracle output: 100 (2.95s) + user error (JSON Decoding of test/conc_eval/hash_map/insert_multi.all.mir failed: Error in $.fns[1463].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) - Use -p '/from_elem_zero/' to rerun this test only. - extend: FAIL (1.77s) - Compiling and running oracle program (0.20s) - Oracle output: (1, 10) (1.57s) + Use -p '/insert_multi/' to rerun this test only. + insert_remove: FAIL (3.26s) + Compiling and running oracle program (0.22s) + Oracle output: 12 (3.04s) + user error (JSON Decoding of test/conc_eval/hash_map/insert_remove.all.mir failed: Error in $.fns[1489].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) + + Use -p '/insert_remove/' to rerun this test only. + insert_iter: FAIL (3.21s) + Compiling and running oracle program (0.23s) + Oracle output: [11, 12] (2.99s) + user error (JSON Decoding of test/conc_eval/hash_map/insert_iter.all.mir failed: Error in $.fns[1464].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) + + Use -p '/insert_iter/' to rerun this test only. + insert_get: FAIL (3.17s) + Compiling and running oracle program (0.21s) + Oracle output: (11, 12) (2.95s) + user error (JSON Decoding of test/conc_eval/hash_map/insert_get.all.mir failed: Error in $.fns[1465].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) + + Use -p '/insert_get/' to rerun this test only. + traits + params: OK (2.16s) + Compiling and running oracle program (0.18s) + Oracle output: 25 (1.98s) + Crux output: 25 + generic3: FAIL (2.26s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.09s) Crux output: failures: - ---- extend/d078d8f4::crux_test[0] counterexamples ---- + ---- generic3/b76a1aa5::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/cmp.rs:1300:11: 1300:28: error: in core/7ed35d8a::cmp[0]::max_by[0]::_insta10905add4aa20a5[0] - [Crux] Translation error in core/7ed35d8a::cmp[0]::max_by[0]::_insta10905add4aa20a5[0]: illegal or unimplemented coercion from TyInt USize (concretely BVRepr 32) to TyInt B8 + [Crux] internal: error: in generic3/b76a1aa5::f[0] + [Crux] Translation error in generic3/b76a1aa5::f[0]: error building variant core/5a1f6960::option[0]::Option[0]::Some[0]: not enough expressions -- [] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '$0=="crux-mir.crux concrete..vec.extend"' to rerun this test only. - extend_trusted_len: FAIL (1.82s) - Compiling and running oracle program (0.16s) - Oracle output: (1, 10) (1.65s) + Use -p '/generic3/' to rerun this test only. + static_three: OK (2.18s) + Compiling and running oracle program (0.18s) + Oracle output: 2 (2.00s) + Crux output: 2 + generic1: FAIL (2.20s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.02s) Crux output: failures: - ---- extend_trusted_len/e53a641f::crux_test[0] counterexamples ---- + ---- generic1/af42c30b::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/cmp.rs:1300:11: 1300:28: error: in core/7ed35d8a::cmp[0]::max_by[0]::_insta10905add4aa20a5[0] - [Crux] Translation error in core/7ed35d8a::cmp[0]::max_by[0]::_insta10905add4aa20a5[0]: illegal or unimplemented coercion from TyInt USize (concretely BVRepr 32) to TyInt B8 + [Crux] internal: error: in generic1/af42c30b::f[0] + [Crux] Translation error in generic1/af42c30b::f[0]: error building variant core/5a1f6960::option[0]::Option[0]::Some[0]: not enough expressions -- [] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/extend_trusted_len/' to rerun this test only. - collect: FAIL (1.91s) - Compiling and running oracle program (0.15s) - Oracle output: 45 (1.75s) + Use -p '/generic1/' to rerun this test only. + subtrait: OK (2.16s) + Compiling and running oracle program (0.17s) + Oracle output: 73 (1.99s) + Crux output: 73 + gen_trait: OK (2.17s) + Compiling and running oracle program (0.17s) + Oracle output: -69 (2.00s) + Crux output: -69 + dynamic_branch: FAIL (0.20s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (0.03s) + user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_branch.rs) + + Use -p '/dynamic_branch/' to rerun this test only. + static_two: OK (2.15s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (1.98s) + Crux output: 1 + tyfam5: OK (2.15s) + Compiling and running oracle program (0.17s) + Oracle output: () (1.98s) + Crux output: () + generic2: FAIL (2.22s) + Compiling and running oracle program (0.21s) + Oracle output: () (2.01s) Crux output: failures: - ---- collect/4f2b6e34::crux_test[0] counterexamples ---- + ---- generic2/adc7abb7::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] internal: error: in generic2/adc7abb7::f[0] + [Crux] Translation error in generic2/adc7abb7::f[0]: error building variant core/5a1f6960::option[0]::Option[0]::Some[0]: not enough expressions -- [] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/collect/' to rerun this test only. - push: FAIL (1.57s) - Compiling and running oracle program (0.14s) - Oracle output: (1, 2) (1.43s) + Use -p '/generic2/' to rerun this test only. + assoc3: OK (2.20s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.02s) + Crux output: () + tyfam4: OK (2.19s) + Compiling and running oracle program (0.17s) + Oracle output: 0 (2.01s) + Crux output: 0 + static_self: OK (2.12s) + Compiling and running oracle program (0.17s) + Oracle output: 42 (1.96s) + Crux output: 42 + conv: OK (2.17s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (2.01s) + Crux output: 1 + dynamic_poly: FAIL (0.22s) + Compiling and running oracle program (0.19s) + Oracle output: 3 (0.04s) + user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_poly.rs) + + Use -p '/dynamic_poly/' to rerun this test only. + dynamic_simple: OK (2.13s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (1.97s) + Crux output: 1 + gen_trait_poly: OK (2.18s) + Compiling and running oracle program (0.17s) + Oracle output: 12 (2.01s) + Crux output: 12 + default: OK (2.14s) + Compiling and running oracle program (0.17s) + Oracle output: 12 (1.97s) + Crux output: 12 + assoc1: OK (2.18s) + Compiling and running oracle program (0.18s) + Oracle output: () (1.99s) + Crux output: () + subtrait2: OK (2.13s) + Compiling and running oracle program (0.18s) + Oracle output: 64 (1.95s) + Crux output: 64 + bounds1: FAIL (2.16s) + Compiling and running oracle program (0.17s) + Oracle output: () (1.99s) Crux output: failures: - ---- push/0bc1f3e7::crux_test[0] counterexamples ---- + ---- bounds1/d7ac532f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/cmp.rs:1300:11: 1300:28: error: in core/7ed35d8a::cmp[0]::max_by[0]::_insta10905add4aa20a5[0] - [Crux] Translation error in core/7ed35d8a::cmp[0]::max_by[0]::_insta10905add4aa20a5[0]: illegal or unimplemented coercion from TyInt USize (concretely BVRepr 32) to TyInt B8 + [Crux] internal: error: in bounds1/d7ac532f::f[0] + [Crux] Translation error in bounds1/d7ac532f::f[0]: error building variant core/5a1f6960::option[0]::Option[0]::Some[0]: not enough expressions -- [] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/vec.push/' to rerun this test only. - drop: FAIL (1.17s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.02s) + Use -p '/bounds1/' to rerun this test only. + bounds3: FAIL (2.20s) + Compiling and running oracle program (0.20s) + Oracle output: () (2.00s) Crux output: failures: - ---- drop/034551a4::crux_test[0] counterexamples ---- + ---- bounds3/6f12d636::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] internal: error: in bounds3/6f12d636::f[0] + [Crux] Translation error in bounds3/6f12d636::f[0]: error building variant core/5a1f6960::option[0]::Option[0]::Some[0]: not enough expressions -- [] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/drop/' to rerun this test only. - dyn - trait_param: OK (1.03s) - Compiling and running oracle program (0.14s) - Oracle output: 100 (0.89s) - Crux output: 100 - inherit: FAIL (1.04s) - Compiling and running oracle program (0.14s) - Oracle output: 4 (0.90s) + Use -p '/bounds3/' to rerun this test only. + static_eq: OK (2.18s) + Compiling and running oracle program (0.16s) + Oracle output: true (2.01s) + Crux output: true + tyfam3: OK (2.17s) + Compiling and running oracle program (0.17s) + Oracle output: 23 (1.99s) + Crux output: 23 + dict_med: OK (2.21s) + Compiling and running oracle program (0.16s) + Oracle output: 42 (2.05s) + Crux output: 42 + basics1: OK (2.24s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.06s) + Crux output: () + assoc2: OK (2.22s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.04s) + Crux output: () + dict_polymem: OK (2.19s) + Compiling and running oracle program (0.18s) + Oracle output: 4 (2.02s) + Crux output: 4 + bounds2: FAIL (2.18s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.01s) Crux output: failures: - ---- inherit/79188ef3::crux_test[0] counterexamples ---- + ---- bounds2/4feb6822::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/dyn/inherit.rs:20:13: 20:16: error: in inherit/79188ef3::crux_test[0] - [Crux] Translation error in inherit/79188ef3::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] internal: error: in bounds2/4feb6822::f[0] + [Crux] Translation error in bounds2/4feb6822::f[0]: error building variant core/5a1f6960::option[0]::Option[0]::Some[0]: not enough expressions -- [] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/inherit/' to rerun this test only. - plain_trait: OK (1.03s) - Compiling and running oracle program (0.13s) - Oracle output: 100 (0.90s) - Crux output: 100 - assoc_ty: OK (1.07s) - Compiling and running oracle program (0.14s) - Oracle output: 100 (0.93s) - Crux output: 100 - tuple - clone_from: FAIL (1.08s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.95s) + Use -p '/bounds2/' to rerun this test only. + intoiter: OK (2.17s) + Compiling and running oracle program (0.19s) + Oracle output: 0 (1.98s) + Crux output: 0 + static: OK (2.15s) + Compiling and running oracle program (0.18s) + Oracle output: 42 (1.97s) + Crux output: 42 + dynamic_two: FAIL (0.21s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (0.04s) + user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_two.rs) + + Use -p '/dynamic_two/' to rerun this test only. + bounds4: FAIL (2.29s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.11s) Crux output: failures: - ---- clone_from/c0b06a97::f[0] counterexamples ---- + ---- bounds4/33198a89::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/clone.rs:136:17: 136:31: error: in core/7ed35d8a::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0] - [Crux] Translation error in core/7ed35d8a::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0]: don't know how to generate CloneShim for unknown method core/7ed35d8a::clone[0]::Clone[0]::clone[0] + [Crux] internal: error: in bounds4/33198a89::f[0] + [Crux] Translation error in bounds4/33198a89::f[0]: error building variant core/5a1f6960::option[0]::Option[0]::Some[0]: not enough expressions -- [] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/clone_from/' to rerun this test only. - clone_struct: FAIL (1.03s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.90s) + Use -p '/bounds4/' to rerun this test only. + dict_poly: OK (2.16s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (1.99s) + Crux output: 1 + dict_simple: OK (2.16s) + Compiling and running oracle program (0.19s) + Oracle output: 32 (1.97s) + Crux output: 32 + tyfam: OK (2.13s) + Compiling and running oracle program (0.16s) + Oracle output: 23 (1.97s) + Crux output: 23 + bounds5: FAIL (2.17s) + Compiling and running oracle program (0.18s) + Oracle output: () (1.99s) Crux output: failures: - ---- clone_struct/4ae579ad::f[0] counterexamples ---- + ---- bounds5/ee058fbf::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/tuple/clone_struct.rs:8:13: 8:22: error: in clone_struct/4ae579ad::f[0] - [Crux] Translation error in clone_struct/4ae579ad::f[0]: don't know how to generate CloneShim for unknown method core/7ed35d8a::clone[0]::Clone[0]::clone[0] + [Crux] internal: error: in bounds5/ee058fbf::f[0] + [Crux] Translation error in bounds5/ee058fbf::f[0]: error building variant core/5a1f6960::option[0]::Option[0]::Some[0]: not enough expressions -- [] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/clone_struct/' to rerun this test only. - clone: OK (1.04s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.90s) - Crux output: () - clone_rec: FAIL (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.88s) + Use -p '/bounds5/' to rerun this test only. + tyfam2: OK (2.17s) + Compiling and running oracle program (0.17s) + Oracle output: 23 (1.99s) + Crux output: 23 + dynamic_med: FAIL (0.20s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (0.03s) + user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_med.rs) + + Use -p '/dynamic_med/' to rerun this test only. + intTest + test0039: OK (2.28s) + Compiling and running oracle program (0.17s) + Oracle output: 7 (2.11s) + Crux output: 7 + test0038: FAIL (2.26s) + Compiling and running oracle program (0.19s) + Oracle output: () (2.07s) Crux output: failures: - ---- clone_rec/9305eb7c::f[0] counterexamples ---- + ---- test0038/8a6cadf1::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/tuple/clone_rec.rs:4:17: 4:28: error: in clone_rec/9305eb7c::f[0] - [Crux] Translation error in clone_rec/9305eb7c::f[0]: fail or unimp constant: TyTuple [TyInt B32,TyInt B32] (StructRepr [MaybeRepr (BVRepr 32), MaybeRepr (BVRepr 32)]) ConstTuple [ConstInt (I32 3),ConstInt (I32 4)] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/5a1f6960::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst5dab88813bb37c92[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst5dab88813bb37c92[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyArray (TyInt B32) 4) Mut + [Crux] as: TyRawPtr (TyUint B8) Mut [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/clone_rec/' to rerun this test only. - cell - ref_cell2: FAIL (1.53s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.39s) + Use -p '/test0038/' to rerun this test only. + tuple + clone_rec: FAIL (2.21s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.04s) Crux output: failures: - ---- ref_cell2/963774ba::crux_test[0] counterexamples ---- + ---- clone_rec/777a7011::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyInt B32) Mut - [Crux] as: TyRawPtr (TyUint B8) Mut + [Crux] test/conc_eval/tuple/clone_rec.rs:4:17: 4:28: error: in clone_rec/777a7011::f[0] + [Crux] Translation error in clone_rec/777a7011::f[0]: fail or unimp constant: TyTuple [TyInt B32,TyInt B32] (StructRepr [MaybeRepr (BVRepr 32), MaybeRepr (BVRepr 32)]) ConstTuple [ConstInt (I32 3),ConstInt (I32 4)] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/ref_cell2/' to rerun this test only. - ref_cell: FAIL (1.52s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.39s) + Use -p '/clone_rec/' to rerun this test only. + clone: OK (2.21s) + Compiling and running oracle program (0.19s) + Oracle output: () (2.02s) + Crux output: () + clone_from: FAIL (2.19s) + Compiling and running oracle program (0.19s) + Oracle output: () (2.00s) Crux output: failures: - ---- ref_cell/e82ccec1::crux_test[0] counterexamples ---- + ---- clone_from/9fe2d3a6::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyInt B32) Mut - [Crux] as: TyRawPtr (TyUint B8) Mut + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/clone.rs:136:17: 136:31: error: in core/5a1f6960::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0] + [Crux] Translation error in core/5a1f6960::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0]: don't know how to generate CloneShim for unknown method core/5a1f6960::clone[0]::Clone[0]::clone[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '$0=="crux-mir.crux concrete..cell.ref_cell"' to rerun this test only. - cell: OK (1.08s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (0.93s) - Crux output: 2 - str - format: FAIL (1.71s) - Compiling and running oracle program (0.14s) - Oracle output: true (1.57s) + Use -p '/clone_from/' to rerun this test only. + clone_struct: FAIL (2.19s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.01s) Crux output: failures: - ---- format/15303e9f::crux_test[0] counterexamples ---- + ---- clone_struct/6843d741::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/str/format.rs:4:21: 4:27: error: in format/15303e9f::crux_test[0] - [Crux] Translation error in format/15303e9f::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] test/conc_eval/tuple/clone_struct.rs:8:13: 8:22: error: in clone_struct/6843d741::f[0] + [Crux] Translation error in clone_struct/6843d741::f[0]: don't know how to generate CloneShim for unknown method core/5a1f6960::clone[0]::Clone[0]::clone[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '$0=="crux-mir.crux concrete..str.format"' to rerun this test only. - format_hex: FAIL (1.68s) - Compiling and running oracle program (0.13s) - Oracle output: true (1.55s) + Use -p '/clone_struct/' to rerun this test only. + iter + for: FAIL (2.25s) + Compiling and running oracle program (0.17s) + Oracle output: 47 (2.08s) Crux output: failures: - ---- format_hex/0a1bfb4d::crux_test[0] counterexamples ---- + ---- for/318ca416::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/str/format_hex.rs:4:21: 4:29: error: in format_hex/0a1bfb4d::crux_test[0] - [Crux] Translation error in format_hex/0a1bfb4d::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/uint_macros.rs:470:54: 470:57 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:921:5: 922:101: error: in core/5a1f6960::num[0]::{impl#9}[0]::unchecked_add[0] + [Crux] Translation error in core/5a1f6960::num[0]::{impl#9}[0]::unchecked_add[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_instc5e93708b8ca6e2a[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/format_hex/' to rerun this test only. - to_owned: FAIL (1.15s) - Compiling and running oracle program (0.13s) - Oracle output: true (1.01s) + Use -p '/iter.for/' to rerun this test only. + sum: FAIL (2.22s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.05s) Crux output: failures: - ---- to_owned/026b6a7a::crux_test[0] counterexamples ---- + ---- sum/32c2d811::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/7ed35d8a::str[0]::{impl#0}[0]::as_bytes[0] - [Crux] Translation error in core/7ed35d8a::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] + [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyUint USize) Immut + [Crux] as: TyRawPtr (TyUint B8) Immut [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/to_owned/' to rerun this test only. - format_struct: FAIL (1.78s) - Compiling and running oracle program (0.15s) - Oracle output: true (1.63s) + Use -p '/sum/' to rerun this test only. + filter_chain: OK (2.36s) + Compiling and running oracle program (0.20s) + Oracle output: 0 (2.16s) + Crux output: 0 + from_fn: OK (2.20s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.03s) + Crux output: () + peek: FAIL (2.33s) + Compiling and running oracle program (0.20s) + Oracle output: 3 (2.12s) Crux output: failures: - ---- format_struct/b7c0d82c::crux_test[0] counterexamples ---- + ---- peek/dee0a907::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/str/format_struct.rs:10:21: 10:27: error: in format_struct/b7c0d82c::crux_test[0] - [Crux] Translation error in format_struct/b7c0d82c::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyUint USize) Immut + [Crux] as: TyRawPtr (TyUint B8) Immut [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/format_struct/' to rerun this test only. - format_int: FAIL (1.72s) - Compiling and running oracle program (0.13s) - Oracle output: true (1.58s) + Use -p '/peek/' to rerun this test only. + zip: FAIL (2.29s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.10s) Crux output: failures: - ---- format_int/56e6b565::crux_test[0] counterexamples ---- + ---- zip/b55065e9::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/str/format_int.rs:4:21: 4:27: error: in format_int/56e6b565::crux_test[0] - [Crux] Translation error in format_int/56e6b565::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyInt B32) Immut + [Crux] as: TyRawPtr (TyUint B8) Immut [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/format_int/' to rerun this test only. - format_array: FAIL (1.82s) - Compiling and running oracle program (0.14s) - Oracle output: true (1.68s) + Use -p '/zip/' to rerun this test only. + loop: OK (2.14s) + Compiling and running oracle program (0.18s) + Oracle output: 2 (1.97s) + Crux output: 2 + cloned: FAIL (2.24s) + Compiling and running oracle program (0.18s) + Oracle output: 6 (2.06s) Crux output: failures: - ---- format_array/3bd38fe7::crux_test[0] counterexamples ---- + ---- cloned/7cb6e236::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/str/format_array.rs:4:21: 4:27: error: in format_array/3bd38fe7::crux_test[0] - [Crux] Translation error in format_array/3bd38fe7::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyUint USize) Immut + [Crux] as: TyRawPtr (TyUint B8) Immut [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/format_array/' to rerun this test only. - string_push: FAIL (1.66s) - Compiling and running oracle program (0.14s) - Oracle output: true (1.52s) + Use -p '/cloned/' to rerun this test only. + str + format_hex: FAIL (3.00s) + Compiling and running oracle program (0.17s) + Oracle output: true (2.83s) Crux output: failures: - ---- string_push/82fdec64::crux_test[0] counterexamples ---- + ---- format_hex/b2ce67ae::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/cmp.rs:1300:11: 1300:28: error: in core/7ed35d8a::cmp[0]::max_by[0]::_insta10905add4aa20a5[0] - [Crux] Translation error in core/7ed35d8a::cmp[0]::max_by[0]::_insta10905add4aa20a5[0]: illegal or unimplemented coercion from TyInt USize (concretely BVRepr 32) to TyInt B8 + [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst69034e259c08d50e[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst69034e259c08d50e[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyRef TyStr Immut) Immut + [Crux] as: TyRawPtr (TyUint B8) Immut [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/string_push/' to rerun this test only. - iter - cloned: FAIL (1.08s) - Compiling and running oracle program (0.14s) - Oracle output: 6 (0.95s) + Use -p '/format_hex/' to rerun this test only. + format_int: FAIL (3.02s) + Compiling and running oracle program (0.18s) + Oracle output: true (2.84s) Crux output: failures: - ---- cloned/37b565a2::crux_test[0] counterexamples ---- + ---- format_int/4ab30ab2::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0] - [Crux] Translation error in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyUint USize) Immut + [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst69034e259c08d50e[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst69034e259c08d50e[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyRef TyStr Immut) Immut [Crux] as: TyRawPtr (TyUint B8) Immut [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/cloned/' to rerun this test only. - from_fn: OK (1.02s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.89s) - Crux output: () - sum: FAIL (1.08s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.94s) + Use -p '/format_int/' to rerun this test only. + format_struct: FAIL (3.11s) + Compiling and running oracle program (0.18s) + Oracle output: true (2.93s) Crux output: failures: - ---- sum/0909eaa2::f[0] counterexamples ---- + ---- format_struct/3a3e646a::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0] - [Crux] Translation error in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyUint USize) Immut + [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst69034e259c08d50e[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst69034e259c08d50e[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyRef TyStr Immut) Immut [Crux] as: TyRawPtr (TyUint B8) Immut [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/sum/' to rerun this test only. - loop: OK (1.04s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.91s) - Crux output: 2 - zip: FAIL (1.09s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.96s) + Use -p '/format_struct/' to rerun this test only. + format: FAIL (3.23s) + Compiling and running oracle program (0.18s) + Oracle output: true (3.05s) Crux output: failures: - ---- zip/4609a9d6::f[0] counterexamples ---- + ---- format/c1518bdc::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyInt B32) Immut + [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst69034e259c08d50e[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst69034e259c08d50e[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyRef TyStr Immut) Immut [Crux] as: TyRawPtr (TyUint B8) Immut [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/zip/' to rerun this test only. - for: FAIL (1.07s) - Compiling and running oracle program (0.13s) - Oracle output: 47 (0.94s) + Use -p '$0=="crux-mir.crux concrete..str.format"' to rerun this test only. + to_owned: FAIL (2.37s) + Compiling and running oracle program (0.18s) + Oracle output: true (2.19s) Crux output: failures: - ---- for/3551e9e4::crux_test[0] counterexamples ---- + ---- to_owned/41c1673f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:470:54: 470:57 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:921:5: 922:101: error: in core/7ed35d8a::num[0]::{impl#9}[0]::unchecked_add[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#9}[0]::unchecked_add[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_instc5e93708b8ca6e2a[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/5a1f6960::str[0]::{impl#0}[0]::as_bytes[0] + [Crux] Translation error in core/5a1f6960::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/iter.for/' to rerun this test only. - peek: FAIL (1.11s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.98s) + Use -p '/to_owned/' to rerun this test only. + string_push: FAIL (3.02s) + Compiling and running oracle program (0.18s) + Oracle output: true (2.84s) Crux output: failures: - ---- peek/550fdb93::crux_test[0] counterexamples ---- + ---- string_push/cf3b0ae9::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0] - [Crux] Translation error in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyUint USize) Immut - [Crux] as: TyRawPtr (TyUint B8) Immut + [Crux] lib/core/src/cmp.rs:1300:11: 1300:28: error: in core/5a1f6960::cmp[0]::max_by[0]::_insta10905add4aa20a5[0] + [Crux] Translation error in core/5a1f6960::cmp[0]::max_by[0]::_insta10905add4aa20a5[0]: illegal or unimplemented coercion from TyInt USize (concretely BVRepr 32) to TyInt B8 [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/peek/' to rerun this test only. - filter_chain: OK (1.18s) - Compiling and running oracle program (0.15s) - Oracle output: 0 (1.03s) - Crux output: 0 - hash_map - insert_remove: FAIL (2.10s) - Compiling and running oracle program (0.17s) - Oracle output: 12 (1.94s) - vtable signature mismatch for vtable core/7ed35d8a::ops[0]::function[0]::FnMut[0]::_vtblc2e637e13369cbd5[0] of trait core/7ed35d8a::ops[0]::function[0]::FnMut[0]::_trait35b29a8a4ae9e473[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, BVRepr 32] BoolRepr] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr] - CallStack (from HasCallStack): - error, called at src/Mir/Trans.hs:904:20 in crux-mir-0.6.0.99-inplace:Mir.Trans - mkTraitObject, called at src/Mir/Trans.hs:747:11 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate - translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language - - Use -p '/insert_remove/' to rerun this test only. - insert_iter: FAIL (2.06s) - Compiling and running oracle program (0.17s) - Oracle output: [11, 12] (1.89s) - vtable signature mismatch for vtable core/7ed35d8a::ops[0]::function[0]::FnMut[0]::_vtblc2e637e13369cbd5[0] of trait core/7ed35d8a::ops[0]::function[0]::FnMut[0]::_trait35b29a8a4ae9e473[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, BVRepr 32] BoolRepr] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr] - CallStack (from HasCallStack): - error, called at src/Mir/Trans.hs:904:20 in crux-mir-0.6.0.99-inplace:Mir.Trans - mkTraitObject, called at src/Mir/Trans.hs:747:11 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate - translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language - - Use -p '/insert_iter/' to rerun this test only. - insert_multi: FAIL (2.04s) - Compiling and running oracle program (0.17s) - Oracle output: 100 (1.87s) - vtable signature mismatch for vtable core/7ed35d8a::ops[0]::function[0]::FnMut[0]::_vtblc2e637e13369cbd5[0] of trait core/7ed35d8a::ops[0]::function[0]::FnMut[0]::_trait35b29a8a4ae9e473[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, BVRepr 32] BoolRepr] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr] - CallStack (from HasCallStack): - error, called at src/Mir/Trans.hs:904:20 in crux-mir-0.6.0.99-inplace:Mir.Trans - mkTraitObject, called at src/Mir/Trans.hs:747:11 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate - translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language - - Use -p '/insert_multi/' to rerun this test only. - insert_get: FAIL (2.02s) - Compiling and running oracle program (0.16s) - Oracle output: (11, 12) (1.86s) - vtable signature mismatch for vtable core/7ed35d8a::ops[0]::function[0]::FnMut[0]::_vtblc2e637e13369cbd5[0] of trait core/7ed35d8a::ops[0]::function[0]::FnMut[0]::_trait35b29a8a4ae9e473[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, BVRepr 32] BoolRepr] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr, FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] BoolRepr] - CallStack (from HasCallStack): - error, called at src/Mir/Trans.hs:904:20 in crux-mir-0.6.0.99-inplace:Mir.Trans - mkTraitObject, called at src/Mir/Trans.hs:747:11 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate - translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language - - Use -p '/insert_get/' to rerun this test only. - enum - cmp: OK (1.08s) - Compiling and running oracle program (0.13s) - Oracle output: -23 (0.95s) - Crux output: -23 - eq: FAIL (1.05s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.91s) + Use -p '/string_push/' to rerun this test only. + format_array: FAIL (3.11s) + Compiling and running oracle program (0.18s) + Oracle output: true (2.92s) Crux output: failures: - ---- eq/0bcad3d7::crux_test[0] counterexamples ---- + ---- format_array/8a564567::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/enum/eq.rs:9:17: 9:21: error: in eq/0bcad3d7::f[0] - [Crux] Translation error in eq/0bcad3d7::f[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst69034e259c08d50e[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst69034e259c08d50e[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyRef TyStr Immut) Immut + [Crux] as: TyRawPtr (TyUint B8) Immut [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/enum.eq/' to rerun this test only. - arg2: rustc compilation failed for arg2 -error output: -error[E0518]: attribute should be applied to function or closure - --> test/conc_eval/enum/arg2.rs:3:5 - | -3 | #[inline(never)] - | ^^^^^^^^^^^^^^^^ -4 | A(u8), - | ----- not a function or closure - -error[E0518]: attribute should be applied to function or closure - --> test/conc_eval/enum/arg2.rs:5:5 - | -5 | #[inline(never)] - | ^^^^^^^^^^^^^^^^ -6 | B(i32,i32), - | ---------- not a function or closure - -error: aborting due to 2 previous errors - -For more information about this error, try `rustc --explain E0518`. - -FAIL (0.04s) - Compiling and running oracle program (0.04s) - test/Test.hs:109: - failed to compile and run + Use -p '/format_array/' to rerun this test only. + sync + mutex: FAIL (2.80s) + Compiling and running oracle program (0.19s) + Oracle output: 1 (2.61s) + user error (JSON Decoding of test/conc_eval/sync/mutex.all.mir failed: Error in $.fns[1069].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) - Use -p '/arg2/' to rerun this test only. - cow: FAIL (1.16s) - Compiling and running oracle program (0.14s) - Oracle output: 200 (1.02s) + Use -p '$0=="crux-mir.crux concrete..sync.mutex"' to rerun this test only. + arc_cell: FAIL (2.41s) + Compiling and running oracle program (0.19s) + Oracle output: 4 (2.23s) Crux output: failures: - ---- cow/5077b6e9::crux_test[0] counterexamples ---- + ---- arc_cell/685cfc00::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/7ed35d8a::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/7ed35d8a::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/cow/' to rerun this test only. - field_order: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.88s) - Crux output: () - arg: OK (1.02s) - Compiling and running oracle program (0.13s) - Oracle output: 0 (0.88s) - Crux output: 0 - ret: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: (A(42), B { x: 42 }, C) (0.87s) - Crux output: (A(42), B { x: 42 }, C) - match: OK (1.02s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (0.89s) - Crux output: 42 - inner: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: 4 (0.88s) - Crux output: 4 - mixed_discrs: rustc compilation failed for mixed_discrs -error output: -error[E0554]: `#![feature]` may not be used on the stable release channel - --> test/conc_eval/enum/mixed_discrs.rs:2:12 - | -2 | #![feature(core_intrinsics)] - | ^^^^^^^^^^^^^^^ - -error: aborting due to previous error - -For more information about this error, try `rustc --explain E0554`. - -FAIL (0.05s) - Compiling and running oracle program (0.05s) - test/Test.hs:109: - failed to compile and run - - Use -p '/mixed_discrs/' to rerun this test only. - array - wick2: OK (1.04s) - Compiling and running oracle program (0.13s) - Oracle output: true (0.91s) - Crux output: true - from_slice: FAIL (1.58s) - Compiling and running oracle program (0.13s) - Oracle output: () (1.45s) + Use -p '/arc_cell/' to rerun this test only. + arc: FAIL (2.46s) + Compiling and running oracle program (0.18s) + Oracle output: 1 (2.28s) Crux output: failures: - ---- from_slice/cc2b9543::f[0] counterexamples ---- + ---- arc/346ee371::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in core/7ed35d8a::array[0]::{impl#10}[0]::try_from[0]::_instb3c76ad3dd4c5256[0] - [Crux] Translation error in core/7ed35d8a::array[0]::{impl#10}[0]::try_from[0]::_instb3c76ad3dd4c5256[0]: error building variant core/7ed35d8a::result[0]::Result[0]::Err[0]: not enough expressions -- [] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/array.from_slice/' to rerun this test only. - clone: FAIL (1.08s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.95s) + Use -p '$0=="crux-mir.crux concrete..sync.arc"' to rerun this test only. + atomic_add: FAIL (2.22s) + Compiling and running oracle program (0.17s) + Oracle output: 6 (2.05s) Crux output: failures: - ---- clone/fccb4b75::f[0] counterexamples ---- + ---- atomic_add/65495182::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:61:9: 61:18: error: in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0] - [Crux] Translation error in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyInt B32) Immut - [Crux] as: TyRawPtr (TyArray (TyInt B32) 3) Immut + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst8c861ae30ba93f05[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst8c861ae30ba93f05[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyAdt core/5a1f6960::sync[0]::atomic[0]::Ordering[0]::_adtb7803c2264daf0ec[0] core/5a1f6960::sync[0]::atomic[0]::Ordering[0] (Substs [])) Immut + [Crux] as: TyRawPtr (TyUint B8) Immut [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/array.clone/' to rerun this test only. - mut_index: FAIL (1.46s) - Compiling and running oracle program (0.13s) - Oracle output: 7 (1.32s) + Use -p '/atomic_add/' to rerun this test only. + rwlock: FAIL (2.87s) + Compiling and running oracle program (0.18s) + Oracle output: 2 (2.69s) + user error (JSON Decoding of test/conc_eval/sync/rwlock.all.mir failed: Error in $.fns[1097].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) + + Use -p '$0=="crux-mir.crux concrete..sync.rwlock"' to rerun this test only. + atomic_cxchg: FAIL (2.40s) + Compiling and running oracle program (0.19s) + Oracle output: 6 (2.22s) Crux output: failures: - ---- mut_index/db5c06f4::crux_test[0] counterexamples ---- + ---- atomic_cxchg/eb5b5103::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/7ed35d8a::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/7ed35d8a::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst8c861ae30ba93f05[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst8c861ae30ba93f05[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyAdt core/5a1f6960::sync[0]::atomic[0]::Ordering[0]::_adtb7803c2264daf0ec[0] core/5a1f6960::sync[0]::atomic[0]::Ordering[0] (Substs [])) Immut + [Crux] as: TyRawPtr (TyUint B8) Immut [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/mut_index/' to rerun this test only. - wick1: FAIL (expected: needs Vec data structure from stdlib) (0.25s) - Compiling and running oracle program (0.14s) - Oracle output: true (0.11s) - user error (Error 101 while running mir-json on test/conc_eval/array/wick1.rs) - (expected failure) - arg: OK (1.04s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (0.90s) - Crux output: 2 - iter: FAIL (1.06s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.93s) + Use -p '/atomic_cxchg/' to rerun this test only. + arc_clone: FAIL (2.32s) + Compiling and running oracle program (0.18s) + Oracle output: 2 (2.13s) Crux output: failures: - ---- iter/f5646cc2::crux_test[0] counterexamples ---- + ---- arc_clone/760f187f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyInt B32) Immut - [Crux] as: TyRawPtr (TyUint B8) Immut + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/array.iter/' to rerun this test only. - wick3: FAIL (expected: needs Vec data structure from stdlib) (0.18s) - Compiling and running oracle program (0.14s) - Oracle output: true (0.04s) - user error (Error 101 while running mir-json on test/conc_eval/array/wick3.rs) - (expected failure) - mk_and_proj: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.85s) - Crux output: 42 - mut_arg: OK (1.05s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.92s) - Crux output: 42 - const: OK (1.02s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.89s) - Crux output: 1 - const_impl: OK (1.02s) - Compiling and running oracle program (0.13s) - Oracle output: 5 (0.89s) - Crux output: 5 - stdlib - option: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: true (0.88s) - Crux output: true - range: OK (1.04s) - Compiling and running oracle program (0.13s) - Oracle output: 10 (0.91s) - Crux output: 10 - default_impl: OK (1.02s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.89s) - Crux output: () - teq: OK (1.01s) - Compiling and running oracle program (0.15s) - Oracle output: false (0.86s) - Crux output: false - option3: OK (1.03s) - Compiling and running oracle program (0.13s) - Oracle output: 27 (0.90s) - Crux output: 27 - result: OK (1.03s) - Compiling and running oracle program (0.13s) - Oracle output: 27 (0.90s) - Crux output: 27 - poly: OK (1.04s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.91s) - Crux output: 1 - default: OK (1.02s) - Compiling and running oracle program (0.14s) - Oracle output: true (0.87s) - Crux output: true - cvt: OK (1.06s) - Compiling and running oracle program (0.13s) - Oracle output: 0 (0.93s) - Crux output: 0 - result_interior: OK (1.04s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (0.90s) - Crux output: 3 - option2: OK (1.04s) - Compiling and running oracle program (0.15s) - Oracle output: 0 (0.89s) - Crux output: 0 - intTest - test0039: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: 7 (0.89s) - Crux output: 7 - test0038: FAIL (1.09s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.96s) + Use -p '/arc_clone/' to rerun this test only. + atomic_swap: FAIL (2.21s) + Compiling and running oracle program (0.18s) + Oracle output: (2, 1) (2.03s) Crux output: failures: - ---- test0038/ee52fa9c::crux_test[0] counterexamples ---- + ---- atomic_swap/25d5ab9a::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst5dab88813bb37c92[0] - [Crux] Translation error in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst5dab88813bb37c92[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyArray (TyInt B32) 4) Mut + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/sync/atomic.rs:3010:60: 3010:63: error: in core/5a1f6960::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/5a1f6960::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::atomic_store_seqcst[0]::_inst1e2825177cd3b608[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:124: + crux doesn't match oracle + + Use -p '/atomic_swap/' to rerun this test only. + rwlock_multi: FAIL (2.87s) + Compiling and running oracle program (0.18s) + Oracle output: 3 (2.69s) + user error (JSON Decoding of test/conc_eval/sync/rwlock_multi.all.mir failed: Error in $.fns[1123].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) + + Use -p '/rwlock_multi/' to rerun this test only. + atomic_fence: FAIL (2.32s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (2.16s) + Crux output: + failures: + + ---- atomic_fence/89629395::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/sync/atomic.rs:3010:60: 3010:63: error: in core/5a1f6960::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/5a1f6960::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::atomic_store_seqcst[0]::_inst1e2825177cd3b608[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:124: + crux doesn't match oracle + + Use -p '/atomic_fence/' to rerun this test only. + mutex_multi: FAIL (2.71s) + Compiling and running oracle program (0.18s) + Oracle output: 3 (2.53s) + user error (JSON Decoding of test/conc_eval/sync/mutex_multi.all.mir failed: Error in $.fns[1079].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) + + Use -p '/mutex_multi/' to rerun this test only. + consts + struct_val: OK (2.21s) + Compiling and running oracle program (0.17s) + Oracle output: Foo { x: true } (2.03s) + Crux output: Foo { x: true } + struct_unit: FAIL (2.17s) + Compiling and running oracle program (0.17s) + Oracle output: Err(Foo { x: () }) (2.00s) + Crux output: + failures: + + ---- struct_unit/c2581b77::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in struct_unit/c2581b77::crux_test[0] + [Crux] Translation error in struct_unit/c2581b77::crux_test[0]: error building variant core/5a1f6960::result[0]::Result[0]::Err[0]: not enough expressions -- [] + + [Crux] Overall status: Invalid. + + test/Test.hs:124: + crux doesn't match oracle + + Use -p '/struct_unit/' to rerun this test only. + local_key: FAIL (2.21s) + Compiling and running oracle program (0.18s) + Oracle output: 1 (2.04s) + user error (JSON Decoding of test/conc_eval/consts/local_key.all.mir failed: Error in $.statics[0].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) + + Use -p '/local_key/' to rerun this test only. + fn_def: FAIL (2.19s) + Compiling and running oracle program (0.21s) + Oracle output: 1 (1.99s) + user error (JSON Decoding of test/conc_eval/consts/fn_def.all.mir failed: Error in $.fns[0].body.blocks[0].block.terminator.func.data.rendered: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) + + Use -p '/fn_def/' to rerun this test only. + enum_val: OK (2.14s) + Compiling and running oracle program (0.17s) + Oracle output: None (1.97s) + Crux output: None + crypto + add: FAIL (2.29s) + Compiling and running oracle program (0.18s) + Oracle output: true (2.11s) + Crux output: + failures: + + ---- add/f9cfa7f7::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/num/uint_macros.rs:470:54: 470:57 !lib/core/src/num/mod.rs:965:5: 970:27: error: in core/5a1f6960::num[0]::{impl#12}[0]::unchecked_add[0] + [Crux] Translation error in core/5a1f6960::num[0]::{impl#12}[0]::unchecked_add[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_instaffa7a8b1157c078[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:124: + crux doesn't match oracle + + Use -p '$0=="crux-mir.crux concrete..crypto.add"' to rerun this test only. + add_noL: FAIL (2.22s) + Compiling and running oracle program (0.19s) + Oracle output: false (2.03s) + Crux output: + failures: + + ---- add_noL/d822cec7::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/num/uint_macros.rs:470:54: 470:57 !lib/core/src/num/mod.rs:965:5: 970:27: error: in core/5a1f6960::num[0]::{impl#12}[0]::unchecked_add[0] + [Crux] Translation error in core/5a1f6960::num[0]::{impl#12}[0]::unchecked_add[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_instaffa7a8b1157c078[0] + + [Crux] Overall status: Invalid. + + test/Test.hs:124: + crux doesn't match oracle + + Use -p '/add_noL/' to rerun this test only. + cell + cell: OK (2.24s) + Compiling and running oracle program (0.18s) + Oracle output: 2 (2.05s) + Crux output: 2 + ref_cell: FAIL (2.95s) + Compiling and running oracle program (0.18s) + Oracle output: 2 (2.77s) + Crux output: + failures: + + ---- ref_cell/67706a18::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/5a1f6960::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyInt B32) Mut [Crux] as: TyRawPtr (TyUint B8) Mut [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/test0038/' to rerun this test only. - box - new: FAIL (1.15s) - Compiling and running oracle program (0.13s) - Oracle output: () (1.02s) + Use -p '$0=="crux-mir.crux concrete..cell.ref_cell"' to rerun this test only. + ref_cell2: FAIL (2.94s) + Compiling and running oracle program (0.19s) + Oracle output: 2 (2.76s) Crux output: failures: - ---- new/825ce802::f[0] counterexamples ---- + ---- ref_cell2/d5dc162d::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/5a1f6960::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyInt B32) Mut + [Crux] as: TyRawPtr (TyUint B8) Mut [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/box.new/' to rerun this test only. - struct: rustc compilation failed for struct + Use -p '/ref_cell2/' to rerun this test only. + fnptr + call: OK (2.32s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (2.15s) + Crux output: 2 + field: OK (2.22s) + Compiling and running oracle program (0.18s) + Oracle output: 2 (2.04s) + Crux output: 2 + make: OK (2.18s) + Compiling and running oracle program (0.18s) + Oracle output: 0 (1.99s) + Crux output: 0 + custom: rustc compilation failed for custom error output: -error[E0554]: `#![feature]` may not be used on the stable release channel - --> test/conc_eval/box/struct.rs:1:1 +error[E0432]: unresolved import `core` + --> test/conc_eval/fnptr/custom.rs:2:5 + | +2 | use core::mem; + | ^^^^ maybe a missing crate `core`? | -1 | #![feature(box_syntax)] - | ^^^^^^^^^^^^^^^^^^^^^^^ + = help: consider adding `extern crate core` to use the `core` crate error: aborting due to previous error -For more information about this error, try `rustc --explain E0554`. +For more information about this error, try `rustc --explain E0432`. -FAIL (0.04s) - Compiling and running oracle program (0.04s) - test/Test.hs:109: +FAIL (expected: taking address of an overridden function) (0.06s) + Compiling and running oracle program (0.06s) + test/Test.hs:107: failed to compile and run - - Use -p '/box.struct/' to rerun this test only. - unsize: FAIL (1.09s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.96s) + (expected failure) + num + overflow: FAIL (2.22s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.05s) Crux output: failures: - ---- unsize/ad9f3518::f[0] counterexamples ---- + ---- overflow/0ff004e4::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] internal: error: in static_initializer + [Crux] Translation error in the static initializer: fail or unimp constant: TyTuple [TyUint B8,TyBool] (StructRepr [MaybeRepr (BVRepr 8), MaybeRepr BoolRepr]) ConstTuple [ConstInt (U8 3),ConstBool False] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/box.unsize/' to rerun this test only. - mut_ref: FAIL (1.14s) - Compiling and running oracle program (0.13s) - Oracle output: () (1.01s) + Use -p '/num.overflow/' to rerun this test only. + from_bytes: FAIL (2.17s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.01s) Crux output: failures: - ---- mut_ref/b1bd8ce3::f[0] counterexamples ---- + ---- from_bytes/84ee4e7f::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/uint_macros.rs:2417:37: 2417:42 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:890:5: 891:49: error: in core/5a1f6960::num[0]::{impl#8}[0]::from_ne_bytes[0] + [Crux] Translation error in core/5a1f6960::num[0]::{impl#8}[0]::from_ne_bytes[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_inst850c595be4adc72f[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/box.mut_ref/' to rerun this test only. - mut: FAIL (1.17s) - Compiling and running oracle program (0.13s) - Oracle output: () (1.05s) + Use -p '/from_bytes/' to rerun this test only. + saturate: FAIL (2.86s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.69s) Crux output: failures: - ---- mut/38683331::f[0] counterexamples ---- + ---- saturate/e630dede::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/uint_macros.rs:1022:46: 1022:49 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:305:5: 306:27: error: in core/5a1f6960::num[0]::{impl#7}[0]::saturating_add[0] + [Crux] Translation error in core/5a1f6960::num[0]::{impl#7}[0]::saturating_add[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::saturating_add[0]::_instaddce72e1232152c[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '$0=="crux-mir.crux concrete..box.mut"' to rerun this test only. - sync - rwlock: FAIL (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.76s) - standalone use of `dyn` is not supported: TyDynamic core/7ed35d8a::error[0]::Error[0]::_trait56353a7b840403b4[0] - CallStack (from HasCallStack): - error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy - tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate - translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language - - Use -p '$0=="crux-mir.crux concrete..sync.rwlock"' to rerun this test only. - atomic_cxchg: FAIL (1.14s) - Compiling and running oracle program (0.15s) - Oracle output: 6 (0.99s) + Use -p '/saturate/' to rerun this test only. + prim + bool: OK (2.13s) + Compiling and running oracle program (0.17s) + Oracle output: false (1.97s) + Crux output: false + shift3: OK (2.27s) + Compiling and running oracle program (0.17s) + Oracle output: -9223372036854775808 (2.10s) + Crux output: -9223372036854775808 + div: OK (2.15s) + Compiling and running oracle program (0.16s) + Oracle output: 4 (1.98s) + Crux output: 4 + ge: OK (2.16s) + Compiling and running oracle program (0.16s) + Oracle output: true (2.00s) + Crux output: true + add1: OK (2.19s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (2.03s) + Crux output: 2 + shift_exceeding: rustc compilation failed for shift_exceeding +error output: +error[E0425]: cannot find function `catch_unwind` in module `panic` + --> test/conc_eval/prim/shift_exceeding.rs:14:25 + | +14 | let result = panic::catch_unwind(|| { + | ^^^^^^^^^^^^ not found in `panic` + | +help: consider importing this function + | +3 | use std::panic::catch_unwind; + | +help: if you import `catch_unwind`, refer to it directly + | +14 - let result = panic::catch_unwind(|| { +14 + let result = catch_unwind(|| { + | + +warning: lint `exceeding_bitshifts` has been renamed to `arithmetic_overflow` + --> test/conc_eval/prim/shift_exceeding.rs:4:9 + | +4 | #[allow(exceeding_bitshifts)] + | ^^^^^^^^^^^^^^^^^^^ help: use the new name: `arithmetic_overflow` + | + = note: `#[warn(renamed_and_removed_lints)]` on by default + +error: aborting due to previous error; 1 warning emitted + +For more information about this error, try `rustc --explain E0425`. + +FAIL (expected: Should panic, but doesn't) (0.06s) + Compiling and running oracle program (0.06s) + test/Test.hs:107: + failed to compile and run + (expected failure) + char_from_u32: FAIL (2.21s) + Compiling and running oracle program (0.17s) + Oracle output: 'A' (2.04s) Crux output: failures: - ---- atomic_cxchg/4bcd370f::crux_test[0] counterexamples ---- + ---- char_from_u32/2f2f276c::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/sync/atomic_cxchg.rs:8:22: 8:66: error: in atomic_cxchg/4bcd370f::crux_test[0] - [Crux] Translation error in atomic_cxchg/4bcd370f::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/char/convert.rs:215:31: 215:32: error: in core/5a1f6960::char[0]::convert[0]::char_try_from_u32[0] + [Crux] Translation error in core/5a1f6960::char[0]::convert[0]::char_try_from_u32[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instdc3291f36c45d6de[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/atomic_cxchg/' to rerun this test only. - atomic_fence: FAIL (1.08s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.95s) + Use -p '/char_from_u32/' to rerun this test only. + ffs: FAIL (2.22s) + Compiling and running oracle program (0.17s) + Oracle output: true (2.04s) Crux output: failures: - ---- atomic_fence/a8477cc6::crux_test[0] counterexamples ---- + ---- ffs/5879a819::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/sync/atomic.rs:3010:60: 3010:63: error: in core/7ed35d8a::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/7ed35d8a::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::atomic_store_seqcst[0]::_inst1e2825177cd3b608[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/int_macros.rs:460:54: 460:57 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:251:5: 253:45: error: in core/5a1f6960::num[0]::{impl#3}[0]::unchecked_add[0] + [Crux] Translation error in core/5a1f6960::num[0]::{impl#3}[0]::unchecked_add[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_inst1e2825177cd3b608[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/atomic_fence/' to rerun this test only. - atomic_add: FAIL (1.13s) - Compiling and running oracle program (0.14s) - Oracle output: 6 (0.99s) + Use -p '/prim.ffs/' to rerun this test only. + mut_arg: OK (2.26s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (2.09s) + Crux output: 1 + litstring: FAIL (2.17s) + Compiling and running oracle program (0.16s) + Oracle output: true (2.00s) Crux output: failures: - ---- atomic_add/4b66494a::crux_test[0] counterexamples ---- + ---- litstring/dd559cbf::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/sync/atomic_add.rs:8:22: 8:66: error: in atomic_add/4b66494a::crux_test[0] - [Crux] Translation error in atomic_add/4b66494a::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/5a1f6960::str[0]::{impl#0}[0]::as_bytes[0] + [Crux] Translation error in core/5a1f6960::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/atomic_add/' to rerun this test only. - arc: FAIL (1.17s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (1.04s) + Use -p '/litstring/' to rerun this test only. + lit: FAIL (2.15s) + Compiling and running oracle program (0.17s) + Oracle output: false (1.97s) Crux output: failures: - ---- arc/b5204711::crux_test[0] counterexamples ---- + ---- lit/7ce419c4::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] internal: error: in lit/7ce419c4::crux_test[0] + [Crux] Translation error in lit/7ce419c4::crux_test[0]: fail or unimp constant: TyTuple [TyBool,TyBool] (StructRepr [MaybeRepr BoolRepr, MaybeRepr BoolRepr]) ConstTuple [ConstBool True,ConstBool True] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '$0=="crux-mir.crux concrete..sync.arc"' to rerun this test only. - rwlock_multi: FAIL (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.74s) - standalone use of `dyn` is not supported: TyDynamic core/7ed35d8a::error[0]::Error[0]::_trait56353a7b840403b4[0] - CallStack (from HasCallStack): - error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy - tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate - translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language - - Use -p '/rwlock_multi/' to rerun this test only. - arc_clone: FAIL (1.13s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (1.00s) + Use -p '$0=="crux-mir.crux concrete..prim.lit"' to rerun this test only. + litbstring: OK (2.12s) + Compiling and running oracle program (0.16s) + Oracle output: true (1.96s) + Crux output: true + shift1: OK (2.17s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (2.01s) + Crux output: 2 + shift2: OK (2.16s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (1.99s) + Crux output: 2 + shift4: OK (2.15s) + Compiling and running oracle program (0.18s) + Oracle output: -9223372036854775808 (1.97s) + Crux output: -9223372036854775808 + mut: OK (2.14s) + Compiling and running oracle program (0.18s) + Oracle output: 14 (1.96s) + Crux output: 14 + wrapping_sub: OK (2.23s) + Compiling and running oracle program (0.21s) + Oracle output: true (2.02s) + Crux output: true + impl + self: OK (2.22s) + Compiling and running oracle program (0.17s) + Oracle output: 42 (2.04s) + Crux output: 42 + simple: OK (2.17s) + Compiling and running oracle program (0.18s) + Oracle output: 42 (2.00s) + Crux output: 42 + self_mut: OK (2.17s) + Compiling and running oracle program (0.17s) + Oracle output: 42 (2.00s) + Crux output: 42 + box + new: FAIL (2.22s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.05s) Crux output: failures: - ---- arc_clone/b9b1f82e::crux_test[0] counterexamples ---- + ---- new/f7f026f5::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/arc_clone/' to rerun this test only. - mutex: FAIL (1.92s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (1.79s) - standalone use of `dyn` is not supported: TyDynamic core/7ed35d8a::error[0]::Error[0]::_trait56353a7b840403b4[0] - CallStack (from HasCallStack): - error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy - tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate - translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language - - Use -p '$0=="crux-mir.crux concrete..sync.mutex"' to rerun this test only. - arc_cell: FAIL (1.17s) - Compiling and running oracle program (0.14s) - Oracle output: 4 (1.03s) + Use -p '/box.new/' to rerun this test only. + mut_ref: FAIL (2.33s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.16s) Crux output: failures: - ---- arc_cell/9fa1118b::crux_test[0] counterexamples ---- + ---- mut_ref/975c3f15::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/arc_cell/' to rerun this test only. - mutex_multi: FAIL (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.76s) - standalone use of `dyn` is not supported: TyDynamic core/7ed35d8a::error[0]::Error[0]::_trait56353a7b840403b4[0] - CallStack (from HasCallStack): - error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy - tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate - translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language - - Use -p '/mutex_multi/' to rerun this test only. - atomic_swap: FAIL (1.08s) - Compiling and running oracle program (0.13s) - Oracle output: (2, 1) (0.95s) + Use -p '/box.mut_ref/' to rerun this test only. + mut: FAIL (2.30s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.13s) Crux output: failures: - ---- atomic_swap/6d7e3f7b::crux_test[0] counterexamples ---- + ---- mut/5b3bcc1f::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/sync/atomic.rs:3010:60: 3010:63: error: in core/7ed35d8a::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/7ed35d8a::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::atomic_store_seqcst[0]::_inst1e2825177cd3b608[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/atomic_swap/' to rerun this test only. - slice - range_len: rustc compilation failed for range_len -error output: -warning: unused import: `core::slice::SliceIndex` - --> test/conc_eval/slice/range_len.rs:5:5 - | -5 | use core::slice::SliceIndex; - | ^^^^^^^^^^^^^^^^^^^^^^^ - | - = note: `#[warn(unused_imports)]` on by default - -warning: unused import: `core::ops::Range` - --> test/conc_eval/slice/range_len.rs:6:5 - | -6 | use core::ops::Range; - | ^^^^^^^^^^^^^^^^ - -error[E0554]: `#![feature]` may not be used on the stable release channel - --> test/conc_eval/slice/range_len.rs:2:12 - | -2 | #![feature(slice_index_methods)] - | ^^^^^^^^^^^^^^^^^^^ - -error: aborting due to previous error; 2 warnings emitted - -For more information about this error, try `rustc --explain E0554`. - -FAIL (0.04s) - Compiling and running oracle program (0.04s) - test/Test.hs:109: - failed to compile and run - - Use -p '$0=="crux-mir.crux concrete..slice.range_len"' to rerun this test only. - iter_mut: FAIL (1.14s) - Compiling and running oracle program (0.13s) - Oracle output: () (1.01s) + Use -p '$0=="crux-mir.crux concrete..box.mut"' to rerun this test only. + unsize: FAIL (2.28s) + Compiling and running oracle program (0.18s) + Oracle output: 2 (2.11s) Crux output: failures: - ---- iter_mut/51d61b89::f[0] counterexamples ---- + ---- unsize/18882248::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyInt B32) Mut - [Crux] as: TyRawPtr (TyUint B8) Mut + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/iter_mut/' to rerun this test only. - eq: FAIL (1.09s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.95s) + Use -p '/box.unsize/' to rerun this test only. + struct: FAIL (2.26s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (2.09s) Crux output: failures: - ---- eq/2c663db8::f[0] counterexamples ---- + ---- struct/c7e3a7f5::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/mem/mod.rs:338:38: 338:41: error: in core/7ed35d8a::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/7ed35d8a::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::size_of_val[0]::_inst55af81a2cef2701c[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/slice.eq/' to rerun this test only. - swap: OK (1.07s) - Compiling and running oracle program (0.14s) - Oracle output: 2001 (0.93s) - Crux output: 2001 - last: OK (1.08s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.95s) - Crux output: 3 - mut_range: FAIL (1.51s) - Compiling and running oracle program (0.13s) - Oracle output: 86 (1.38s) + Use -p '/box.struct/' to rerun this test only. + ptr + is_null_slice: FAIL (expected: can't unsize null pointers) (2.20s) + Compiling and running oracle program (0.17s) + Oracle output: (false, true) (2.03s) Crux output: failures: - ---- mut_range/3c8d2622::crux_test[0] counterexamples ---- + ---- is_null_slice/994d2033::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/ptr/is_null_slice.rs:7:13: 7:33: error: in is_null_slice/994d2033::crux_test[0] + [Crux] attempted subindex on the result of an integer-to-pointer cast [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/7ed35d8a::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/7ed35d8a::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst55af81a2cef2701c[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TySlice (TyInt B32)) Immut + [Crux] as: TyRawPtr (TyUint B8) Immut [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - - Use -p '/mut_range/' to rerun this test only. - mk_and_proj: OK (1.05s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.92s) - Crux output: 42 - get: rustc compilation failed for get -error output: -error[E0554]: `#![feature]` may not be used on the stable release channel - --> test/conc_eval/slice/get.rs:2:1 - | -2 | #![feature(never_type)] - | ^^^^^^^^^^^^^^^^^^^^^^^ - -error: aborting due to previous error - -For more information about this error, try `rustc --explain E0554`. - -FAIL (0.04s) - Compiling and running oracle program (0.04s) - test/Test.hs:109: - failed to compile and run - - Use -p '/slice.get/' to rerun this test only. - len: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: 5 (0.88s) - Crux output: 5 - range_len_mut: rustc compilation failed for range_len_mut -error output: -warning: unused import: `core::slice::SliceIndex` - --> test/conc_eval/slice/range_len_mut.rs:5:5 - | -5 | use core::slice::SliceIndex; - | ^^^^^^^^^^^^^^^^^^^^^^^ - | - = note: `#[warn(unused_imports)]` on by default - -warning: unused import: `core::ops::Range` - --> test/conc_eval/slice/range_len_mut.rs:6:5 - | -6 | use core::ops::Range; - | ^^^^^^^^^^^^^^^^ - -error[E0554]: `#![feature]` may not be used on the stable release channel - --> test/conc_eval/slice/range_len_mut.rs:2:12 - | -2 | #![feature(slice_index_methods)] - | ^^^^^^^^^^^^^^^^^^^ - -error: aborting due to previous error; 2 warnings emitted - -For more information about this error, try `rustc --explain E0554`. - -FAIL (0.04s) - Compiling and running oracle program (0.04s) - test/Test.hs:109: - failed to compile and run - - Use -p '/range_len_mut/' to rerun this test only. - mut: OK (1.06s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.93s) - Crux output: 42 - clos - fnptr_fn: OK (1.07s) - Compiling and running oracle program (0.13s) - Oracle output: 4 (0.94s) - Crux output: 4 - fnonce1: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.87s) - Crux output: 3 - conv_fnonce_fn: OK (1.03s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.90s) - Crux output: 2 - fo: rustc compilation failed for fo -error output: -error[E0554]: `#![feature]` may not be used on the stable release channel - --> test/conc_eval/clos/fo.rs:3:1 - | -3 | #![feature(type_ascription)] - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unnecessary parentheses around function argument - --> test/conc_eval/clos/fo.rs:11:7 - | -11 | g((1 :i32)) - | ^ ^ - | - = note: `#[warn(unused_parens)]` on by default -help: remove these parentheses - | -11 - g((1 :i32)) -11 + g(1 :i32) - | - -error: aborting due to previous error; 1 warning emitted - -For more information about this error, try `rustc --explain E0554`. - -FAIL (0.04s) - Compiling and running oracle program (0.04s) - test/Test.hs:109: - failed to compile and run - - Use -p '/clos.fo/' to rerun this test only. - direct_fn: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.88s) - Crux output: 2 - unique_borrow: OK (1.09s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.96s) - Crux output: 3 - fn_static: OK (1.07s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.93s) + (expected failure) + struct_eq: OK (2.20s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.03s) + Crux output: () + read_write: OK (2.20s) + Compiling and running oracle program (0.17s) + Oracle output: [1, 3, 1] (2.03s) + Crux output: [1, 3, 1] + offset_mut: OK (2.17s) + Compiling and running oracle program (0.17s) + Oracle output: 3 (2.00s) Crux output: 3 - fnptr_fnonce: OK (1.07s) - Compiling and running oracle program (0.14s) - Oracle output: 4 (0.93s) - Crux output: 4 - direct_fnonce: OK (1.06s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.93s) - Crux output: 2 - dispatch_fnmut: FAIL (1.10s) - Compiling and running oracle program (0.13s) - Oracle output: 7 (0.97s) + copy: OK (2.22s) + Compiling and running oracle program (0.17s) + Oracle output: [1, 2, 3, 1, 2, 3] (2.05s) + Crux output: [1, 2, 3, 1, 2, 3] + unsize_slice: OK (2.22s) + Compiling and running oracle program (0.19s) + Oracle output: (1, 2) (2.02s) + Crux output: (1, 2) + dangling_eq: FAIL (2.21s) + Compiling and running oracle program (0.16s) + Oracle output: () (2.04s) Crux output: failures: - ---- dispatch_fnmut/3f7bd57d::crux_test[0] counterexamples ---- + ---- dangling_eq/a15fc55d::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/clos/dispatch_fnmut.rs:7:7: 7:9: error: in dispatch_fnmut/3f7bd57d::call_it2[0]::_inst15715dbe1d0521ca[0] - [Crux] Translation error in dispatch_fnmut/3f7bd57d::call_it2[0]::_inst15715dbe1d0521ca[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/mod.rs:599:29: 599:33: error: in core/5a1f6960::ptr[0]::invalid_mut[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::invalid_mut[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_inst102627303ebf2823[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/dispatch_fnmut/' to rerun this test only. - conv_fnmut_fn: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.88s) - Crux output: 2 - fnptr_fnmut: OK (1.08s) - Compiling and running oracle program (0.14s) - Oracle output: 4 (0.94s) - Crux output: 4 - poly: OK (1.02s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.89s) - Crux output: 3 - conv_fnonce_fnmut: OK (1.07s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.94s) - Crux output: 2 - fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (0.16s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.03s) - user error (Error 101 while running mir-json on test/conc_eval/clos/fn_dyn.rs) - (expected failure) - ref_fnmut: OK (1.07s) - Compiling and running oracle program (0.13s) - Oracle output: 7 (0.94s) - Crux output: 7 - as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (1.01s) - Compiling and running oracle program (0.12s) - Oracle output: 1 (0.89s) + Use -p '/dangling_eq/' to rerun this test only. + cast_eq: OK (2.17s) + Compiling and running oracle program (0.18s) + Oracle output: () (1.99s) + Crux output: () + offset_from: OK (2.21s) + Compiling and running oracle program (0.17s) + Oracle output: -2 (2.03s) + Crux output: -2 + null_eq: FAIL (2.19s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (2.02s) Crux output: failures: - ---- as_fn_ptr/d4076aaa::crux_test[0] counterexamples ---- + ---- null_eq/9be1a456::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/d4076aaa::crux_test[0] - [Crux] Translation error in as_fn_ptr/d4076aaa::crux_test[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [], _fsreturn_ty = TyInt B32, _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/mod.rs:568:29: 568:33: error: in core/5a1f6960::ptr[0]::invalid[0]::_instf59dc84daa91a404[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::invalid[0]::_instf59dc84daa91a404[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_insta5d0a99e3b08f3bd[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - (expected failure) - fnonce: OK (1.04s) - Compiling and running oracle program (0.12s) - Oracle output: false (0.91s) - Crux output: false - promoted: FAIL (1.08s) - Compiling and running oracle program (0.12s) - Oracle output: 33 (0.96s) + + Use -p '/null_eq/' to rerun this test only. + coerce_unsized: OK (2.23s) + Compiling and running oracle program (0.17s) + Oracle output: (1, 2) (2.06s) + Crux output: (1, 2) + offset: OK (2.18s) + Compiling and running oracle program (0.18s) + Oracle output: 3 (2.00s) + Crux output: 3 + is_null: FAIL (2.15s) + Compiling and running oracle program (0.17s) + Oracle output: (false, true) (1.97s) Crux output: failures: - ---- promoted/599dd748::crux_test[0] counterexamples ---- + ---- is_null/7b991371::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/clos/promoted.rs:15:9: 15:11: error: in promoted/599dd748::f[0] - [Crux] Translation error in promoted/599dd748::f[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyInt B32) Immut + [Crux] as: TyRawPtr (TyUint B8) Immut [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/clos.promoted/' to rerun this test only. - fnptr_closure: FAIL (1.06s) - Compiling and running oracle program (0.13s) - Oracle output: 7 (0.92s) - Crux output: - failures: - - ---- fnptr_closure/af787938::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/clos/fnptr_closure.rs:7:7: 7:9: error: in fnptr_closure/af787938::call_it2[0]::_inste366449525d6d5d6[0] - [Crux] Translation error in fnptr_closure/af787938::call_it2[0]::_inste366449525d6d5d6[0]: cannot find static variable alloc$0/3a1fbbbh + Use -p '$0=="crux-mir.crux concrete..ptr.is_null"' to rerun this test only. + valid_eq: OK (2.21s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.05s) + Crux output: () + ops + deref2: OK (2.20s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.02s) + Crux output: () + index2: OK (2.18s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.01s) + Crux output: () + arith1: OK (2.16s) + Compiling and running oracle program (0.18s) + Oracle output: () (1.99s) + Crux output: () + index3: OK (2.19s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.01s) + Crux output: () + deref1: OK (2.19s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.03s) + Crux output: () + index1: OK (2.17s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.00s) + Crux output: () + deref3: OK (2.16s) + Compiling and running oracle program (0.17s) + Oracle output: () (1.99s) + Crux output: () + statics + promoted_fn: OK (2.19s) + Compiling and running oracle program (0.19s) + Oracle output: 1 (1.99s) + Crux output: 1 + promoted_static: OK (2.14s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (1.97s) + Crux output: 1 + io + cursor_write2: FAIL (2.70s) + Compiling and running oracle program (0.18s) + Oracle output: 0 (2.52s) + user error (JSON Decoding of test/conc_eval/io/cursor_write2.all.mir failed: Error in $.statics[61].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) - [Crux] Overall status: Invalid. + Use -p '/cursor_write2/' to rerun this test only. + cursor_write: FAIL (2.80s) + Compiling and running oracle program (0.20s) + Oracle output: 0 (2.60s) + standalone use of `dyn` is not supported: TyDynamic core/5a1f6960::error[0]::Error[0]::_trait56353a7b840403b4[0] + CallStack (from HasCallStack): + error, called at src/Mir/TransTy.hs:225:25 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/Trans.hs:815:12 in crucible-mir-0.1-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:883:5 in crucible-mir-0.1-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:935:30 in crucible-mir-0.1-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1147:11 in crucible-mir-0.1-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1746:68 in crucible-mir-0.1-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1755:28 in crucible-mir-0.1-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1795:10 in crucible-mir-0.1-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1841:24 in crucible-mir-0.1-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2275:30 in crucible-mir-0.1-inplace:Mir.Trans + transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate + translateMIR, called at src/Mir/Language.hs:255:16 in crux-mir-0.6.0.99-inplace:Mir.Language - test/Test.hs:126: - crux doesn't match oracle + Use -p '$0=="crux-mir.crux concrete..io.cursor_write"' to rerun this test only. + cursor_read: FAIL (2.77s) + Compiling and running oracle program (0.19s) + Oracle output: 0 (2.58s) + standalone use of `dyn` is not supported: TyDynamic core/5a1f6960::error[0]::Error[0]::_trait56353a7b840403b4[0] + CallStack (from HasCallStack): + error, called at src/Mir/TransTy.hs:225:25 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/Trans.hs:815:12 in crucible-mir-0.1-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:883:5 in crucible-mir-0.1-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:935:30 in crucible-mir-0.1-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1147:11 in crucible-mir-0.1-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1746:68 in crucible-mir-0.1-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1755:28 in crucible-mir-0.1-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1795:10 in crucible-mir-0.1-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1841:24 in crucible-mir-0.1-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2275:30 in crucible-mir-0.1-inplace:Mir.Trans + transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate + translateMIR, called at src/Mir/Language.hs:255:16 in crux-mir-0.6.0.99-inplace:Mir.Language - Use -p '/fnptr_closure/' to rerun this test only. - direct_fnmut2: OK (1.07s) - Compiling and running oracle program (0.13s) - Oracle output: 7 (0.94s) - Crux output: 7 - direct_fnmut: OK (1.06s) - Compiling and running oracle program (0.13s) - Oracle output: 7 (0.94s) - Crux output: 7 - fn_static_poly: OK (1.07s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (0.93s) - Crux output: 3 - prim - ffs: FAIL (1.17s) - Compiling and running oracle program (0.15s) - Oracle output: true (1.02s) + Use -p '/io.cursor_read/' to rerun this test only. + mem + maybe_uninit: FAIL (2.20s) + Compiling and running oracle program (0.18s) + Oracle output: 1 (2.02s) Crux output: failures: - ---- ffs/61eaece5::crux_test[0] counterexamples ---- + ---- maybe_uninit/419c4a2a::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/int_macros.rs:460:54: 460:57 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:251:5: 253:45: error: in core/7ed35d8a::num[0]::{impl#3}[0]::unchecked_add[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#3}[0]::unchecked_add[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_inst1e2825177cd3b608[0] + [Crux] internal: error: in maybe_uninit/419c4a2a::crux_test[0] + [Crux] Translation error in maybe_uninit/419c4a2a::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyInt B32]) [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/prim.ffs/' to rerun this test only. - add1: OK (0.99s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (0.86s) - Crux output: 2 - litstring: FAIL (1.02s) - Compiling and running oracle program (0.13s) - Oracle output: true (0.90s) + Use -p '$0=="crux-mir.crux concrete..mem.maybe_uninit"' to rerun this test only. + maybe_uninit_array_cast: FAIL (2.20s) + Compiling and running oracle program (0.18s) + Oracle output: [1, 2] (2.02s) Crux output: failures: - ---- litstring/aac23d18::crux_test[0] counterexamples ---- + ---- maybe_uninit_array_cast/c4460e75::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/7ed35d8a::str[0]::{impl#0}[0]::as_bytes[0] - [Crux] Translation error in core/7ed35d8a::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] + [Crux] internal: error: in maybe_uninit_array_cast/c4460e75::crux_test[0] + [Crux] Translation error in maybe_uninit_array_cast/c4460e75::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyArray (TyInt B32) 2]) [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/litstring/' to rerun this test only. - wrapping_sub: OK (1.05s) - Compiling and running oracle program (0.13s) - Oracle output: true (0.92s) - Crux output: true - shift2: OK (1.03s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.90s) - Crux output: 2 - shift3: OK (1.00s) - Compiling and running oracle program (0.12s) - Oracle output: -9223372036854775808 (0.87s) - Crux output: -9223372036854775808 - char_from_u32: FAIL (1.07s) - Compiling and running oracle program (0.12s) - Oracle output: 'A' (0.95s) + Use -p '/maybe_uninit_array_cast/' to rerun this test only. + time + instant: FAIL (2.81s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.64s) + user error (JSON Decoding of test/conc_eval/time/instant.all.mir failed: Error in $.statics[61].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) + + Use -p '/instant/' to rerun this test only. + struct + field_order: OK (2.20s) + Compiling and running oracle program (0.19s) + Oracle output: () (2.01s) + Crux output: () + arg: OK (2.16s) + Compiling and running oracle program (0.18s) + Oracle output: 42 (1.98s) + Crux output: 42 + tup: OK (2.18s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.00s) + Crux output: () + proj: OK (2.14s) + Compiling and running oracle program (0.17s) + Oracle output: 42 (1.98s) + Crux output: 42 + ret: OK (2.16s) + Compiling and running oracle program (0.18s) + Oracle output: S { x: 42, y: 120 } (1.98s) + Crux output: S { x: 42, y: 120 } + repr_transparent: FAIL (2.24s) + Compiling and running oracle program (0.19s) + Oracle output: 6 (2.05s) Crux output: failures: - ---- char_from_u32/2f7600db::crux_test[0] counterexamples ---- + ---- repr_transparent/020bf42b::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/char/convert.rs:215:31: 215:32: error: in core/7ed35d8a::char[0]::convert[0]::char_try_from_u32[0] - [Crux] Translation error in core/7ed35d8a::char[0]::convert[0]::char_try_from_u32[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::transmute[0]::_instdc3291f36c45d6de[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:61:9: 61:18: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyInt B32) Immut + [Crux] as: TyRawPtr (TyArray (TyInt B32) 2) Immut [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/char_from_u32/' to rerun this test only. - div: OK (1.03s) - Compiling and running oracle program (0.13s) - Oracle output: 4 (0.90s) + Use -p '$0=="crux-mir.crux concrete..struct.repr_transparent"' to rerun this test only. + repr_transparent_const: OK (2.16s) + Compiling and running oracle program (0.17s) + Oracle output: 124 (1.99s) + Crux output: 124 + dyn + assoc_ty: OK (2.17s) + Compiling and running oracle program (0.19s) + Oracle output: 100 (1.98s) + Crux output: 100 + trait_param: OK (2.20s) + Compiling and running oracle program (0.18s) + Oracle output: 100 (2.02s) + Crux output: 100 + plain_trait: OK (2.17s) + Compiling and running oracle program (0.17s) + Oracle output: 100 (2.00s) + Crux output: 100 + inherit: OK (2.24s) + Compiling and running oracle program (0.17s) + Oracle output: 4 (2.07s) Crux output: 4 - shift_exceeding: rustc compilation failed for shift_exceeding -error output: -error[E0425]: cannot find function `catch_unwind` in module `panic` - --> test/conc_eval/prim/shift_exceeding.rs:14:25 - | -14 | let result = panic::catch_unwind(|| { - | ^^^^^^^^^^^^ not found in `panic` - | -help: consider importing this function - | -3 | use std::panic::catch_unwind; - | -help: if you import `catch_unwind`, refer to it directly - | -14 - let result = panic::catch_unwind(|| { -14 + let result = catch_unwind(|| { - | - -warning: lint `exceeding_bitshifts` has been renamed to `arithmetic_overflow` - --> test/conc_eval/prim/shift_exceeding.rs:4:9 - | -4 | #[allow(exceeding_bitshifts)] - | ^^^^^^^^^^^^^^^^^^^ help: use the new name: `arithmetic_overflow` - | - = note: `#[warn(renamed_and_removed_lints)]` on by default - -error: aborting due to previous error; 1 warning emitted - -For more information about this error, try `rustc --explain E0425`. - -FAIL (expected: Should panic, but doesn't) (0.08s) - Compiling and running oracle program (0.08s) - test/Test.hs:109: - failed to compile and run - (expected failure) - bool: OK (1.02s) - Compiling and running oracle program (0.12s) - Oracle output: false (0.89s) - Crux output: false - ge: OK (1.01s) - Compiling and running oracle program (0.12s) - Oracle output: true (0.88s) + stdlib + option: OK (2.18s) + Compiling and running oracle program (0.18s) + Oracle output: true (2.00s) Crux output: true - shift1: OK (1.02s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.89s) - Crux output: 2 - lit: FAIL (1.04s) - Compiling and running oracle program (0.13s) - Oracle output: false (0.92s) - Crux output: - failures: - - ---- lit/c6ed0473::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in lit/c6ed0473::crux_test[0] - [Crux] Translation error in lit/c6ed0473::crux_test[0]: fail or unimp constant: TyTuple [TyBool,TyBool] (StructRepr [MaybeRepr BoolRepr, MaybeRepr BoolRepr]) ConstTuple [ConstBool True,ConstBool True] - - [Crux] Overall status: Invalid. - - test/Test.hs:126: - crux doesn't match oracle - - Use -p '$0=="crux-mir.crux concrete..prim.lit"' to rerun this test only. - litbstring: OK (0.99s) - Compiling and running oracle program (0.12s) - Oracle output: true (0.87s) + option2: OK (2.15s) + Compiling and running oracle program (0.17s) + Oracle output: 0 (1.99s) + Crux output: 0 + result: OK (2.13s) + Compiling and running oracle program (0.16s) + Oracle output: 27 (1.97s) + Crux output: 27 + default: OK (2.18s) + Compiling and running oracle program (0.18s) + Oracle output: true (2.00s) Crux output: true - mut_arg: OK (1.01s) - Compiling and running oracle program (0.12s) - Oracle output: 1 (0.89s) - Crux output: 1 - shift4: OK (1.04s) - Compiling and running oracle program (0.12s) - Oracle output: -9223372036854775808 (0.92s) - Crux output: -9223372036854775808 - mut: OK (1.03s) - Compiling and running oracle program (0.13s) - Oracle output: 14 (0.90s) - Crux output: 14 - fnptr - make: OK (1.04s) - Compiling and running oracle program (0.13s) - Oracle output: 0 (0.91s) + cvt: OK (2.14s) + Compiling and running oracle program (0.18s) + Oracle output: 0 (1.97s) Crux output: 0 - call: OK (1.04s) - Compiling and running oracle program (0.12s) - Oracle output: 2 (0.92s) - Crux output: 2 - custom: rustc compilation failed for custom -error output: -error[E0432]: unresolved import `core` - --> test/conc_eval/fnptr/custom.rs:2:5 - | -2 | use core::mem; - | ^^^^ maybe a missing crate `core`? - | - = help: consider adding `extern crate core` to use the `core` crate - -error: aborting due to previous error - -For more information about this error, try `rustc --explain E0432`. - -FAIL (expected: taking address of an overridden function) (0.04s) - Compiling and running oracle program (0.05s) - test/Test.hs:109: - failed to compile and run - (expected failure) - field: OK (1.02s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (0.88s) + range: OK (2.21s) + Compiling and running oracle program (0.19s) + Oracle output: 10 (2.02s) + Crux output: 10 + poly: OK (2.18s) + Compiling and running oracle program (0.18s) + Oracle output: 1 (2.00s) + Crux output: 1 + result_interior: OK (2.15s) + Compiling and running oracle program (0.16s) + Oracle output: 3 (1.98s) + Crux output: 3 + default_impl: OK (2.17s) + Compiling and running oracle program (0.19s) + Oracle output: () (1.98s) + Crux output: () + option3: OK (2.16s) + Compiling and running oracle program (0.17s) + Oracle output: 27 (2.00s) + Crux output: 27 + teq: OK (2.14s) + Compiling and running oracle program (0.17s) + Oracle output: false (1.98s) + Crux output: false + array + wick2: OK (2.19s) + Compiling and running oracle program (0.18s) + Oracle output: true (2.01s) + Crux output: true + arg: OK (2.18s) + Compiling and running oracle program (0.19s) + Oracle output: 2 (1.99s) Crux output: 2 - mem - maybe_uninit_array_cast: FAIL (1.05s) - Compiling and running oracle program (0.13s) - Oracle output: [1, 2] (0.92s) + iter: FAIL (2.26s) + Compiling and running oracle program (0.20s) + Oracle output: 3 (2.06s) Crux output: failures: - ---- maybe_uninit_array_cast/841e0bf8::crux_test[0] counterexamples ---- + ---- iter/d4aa120f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:487:43: 487:48: error: in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::offset[0]::_inst1e2825177cd3b608[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyInt B32) Immut + [Crux] as: TyRawPtr (TyUint B8) Immut [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/maybe_uninit_array_cast/' to rerun this test only. - maybe_uninit: FAIL (1.02s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (0.88s) + Use -p '/array.iter/' to rerun this test only. + mk_and_proj: OK (2.17s) + Compiling and running oracle program (0.17s) + Oracle output: 42 (2.01s) + Crux output: 42 + wick3: FAIL (expected: needs Vec data structure from stdlib) (0.23s) + Compiling and running oracle program (0.19s) + Oracle output: true (0.05s) + user error (Error 101 while running mir-json on test/conc_eval/array/wick3.rs) + (expected failure) + clone: FAIL (2.19s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.01s) Crux output: failures: - ---- maybe_uninit/b2a646dd::crux_test[0] counterexamples ---- + ---- clone/12e7688c::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in core/7ed35d8a::mem[0]::maybe_uninit[0]::{impl#2}[0]::assume_init[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/7ed35d8a::mem[0]::maybe_uninit[0]::{impl#2}[0]::assume_init[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::assert_inhabited[0]::_inst1e2825177cd3b608[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:61:9: 61:18: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyInt B32) Immut + [Crux] as: TyRawPtr (TyArray (TyInt B32) 3) Immut [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '$0=="crux-mir.crux concrete..mem.maybe_uninit"' to rerun this test only. - consts - local_key: FAIL (1.05s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.92s) + Use -p '/array.clone/' to rerun this test only. + wick1: FAIL (expected: needs Vec data structure from stdlib) (0.24s) + Compiling and running oracle program (0.19s) + Oracle output: true (0.05s) + user error (Error 101 while running mir-json on test/conc_eval/array/wick1.rs) + (expected failure) + mut_index: FAIL (2.69s) + Compiling and running oracle program (0.17s) + Oracle output: 7 (2.52s) Crux output: failures: - ---- local_key/cc4d5e1d::crux_test[0] counterexamples ---- + ---- mut_index/00778a49::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/consts/local_key.rs:22:78: 22:93: error: in local_key/cc4d5e1d::crux_test[0] - [Crux] Translation error in local_key/cc4d5e1d::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/local_key/' to rerun this test only. - struct_val: OK (1.06s) - Compiling and running oracle program (0.14s) - Oracle output: Foo { x: true } (0.92s) - Crux output: Foo { x: true } - fn_def: FAIL (1.04s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.91s) - user error (JSON Decoding of test/conc_eval/consts/fn_def.all.mir failed: Error in $.fns[0].body.blocks[0].block.terminator.func.data.rendered: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) - - Use -p '/fn_def/' to rerun this test only. - enum_val: OK (1.09s) - Compiling and running oracle program (0.13s) - Oracle output: None (0.96s) - Crux output: None - struct_unit: FAIL (1.10s) - Compiling and running oracle program (0.13s) - Oracle output: Err(Foo { x: () }) (0.97s) + Use -p '/mut_index/' to rerun this test only. + mut_arg: OK (2.18s) + Compiling and running oracle program (0.17s) + Oracle output: 42 (2.01s) + Crux output: 42 + const: OK (2.14s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (1.97s) + Crux output: 1 + from_slice: FAIL (3.15s) + Compiling and running oracle program (0.19s) + Oracle output: () (2.96s) Crux output: failures: - ---- struct_unit/b3f925bc::crux_test[0] counterexamples ---- + ---- from_slice/f9c79bee::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in struct_unit/b3f925bc::crux_test[0] - [Crux] Translation error in struct_unit/b3f925bc::crux_test[0]: error building variant core/7ed35d8a::result[0]::Result[0]::Err[0]: not enough expressions -- [] + [Crux] internal: error: in core/5a1f6960::array[0]::{impl#10}[0]::try_from[0]::_instb3c76ad3dd4c5256[0] + [Crux] Translation error in core/5a1f6960::array[0]::{impl#10}[0]::try_from[0]::_instb3c76ad3dd4c5256[0]: error building variant core/5a1f6960::result[0]::Result[0]::Err[0]: not enough expressions -- [] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/struct_unit/' to rerun this test only. - refs - imm_raw: OK (1.04s) - Compiling and running oracle program (0.13s) - Oracle output: 123 (0.90s) - Crux output: 123 - never_mut: rustc compilation failed for never_mut + Use -p '/array.from_slice/' to rerun this test only. + const_impl: OK (2.20s) + Compiling and running oracle program (0.18s) + Oracle output: 5 (2.02s) + Crux output: 5 + enum + match: OK (2.15s) + Compiling and running oracle program (0.17s) + Oracle output: 42 (1.98s) + Crux output: 42 + field_order: OK (2.24s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.05s) + Crux output: () + mixed_discrs: OK (2.18s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.00s) + Crux output: () + arg: OK (2.18s) + Compiling and running oracle program (0.18s) + Oracle output: 0 (2.00s) + Crux output: 0 + arg2: rustc compilation failed for arg2 error output: -error[E0554]: `#![feature]` may not be used on the stable release channel - --> test/conc_eval/refs/never_mut.rs:1:1 +error[E0518]: attribute should be applied to function or closure + --> test/conc_eval/enum/arg2.rs:3:5 | -1 | #![feature(never_type)] - | ^^^^^^^^^^^^^^^^^^^^^^^ +3 | #[inline(never)] + | ^^^^^^^^^^^^^^^^ +4 | A(u8), + | ----- not a function or closure -warning: unreachable expression - --> test/conc_eval/refs/never_mut.rs:9:21 - | -9 | let r = &mut e; - | ^^^^^- - | | | - | | any code following this expression is unreachable - | unreachable expression +error[E0518]: attribute should be applied to function or closure + --> test/conc_eval/enum/arg2.rs:5:5 | - = note: `#[warn(unreachable_code)]` on by default +5 | #[inline(never)] + | ^^^^^^^^^^^^^^^^ +6 | B(i32,i32), + | ---------- not a function or closure -error: aborting due to previous error; 1 warning emitted +error: aborting due to 2 previous errors -For more information about this error, try `rustc --explain E0554`. +For more information about this error, try `rustc --explain E0518`. -FAIL (0.04s) - Compiling and running oracle program (0.04s) - test/Test.hs:109: +FAIL (0.06s) + Compiling and running oracle program (0.06s) + test/Test.hs:107: failed to compile and run - Use -p '/never_mut/' to rerun this test only. - promoted_mut: FAIL (1.06s) - Compiling and running oracle program (0.13s) - Oracle output: 0 (0.93s) + Use -p '/arg2/' to rerun this test only. + cow: FAIL (2.35s) + Compiling and running oracle program (0.18s) + Oracle output: 200 (2.17s) Crux output: failures: - ---- promoted_mut/0c05e950::crux_test[0] counterexamples ---- + ---- cow/95160fea::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/refs/promoted_mut.rs:4:27: 4:34: error: in promoted_mut/0c05e950::crux_test[0] - [Crux] Translation error in promoted_mut/0c05e950::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/promoted_mut/' to rerun this test only. - mut_nested: OK (1.06s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.92s) + Use -p '/cow/' to rerun this test only. + ret: OK (2.15s) + Compiling and running oracle program (0.19s) + Oracle output: (A(42), B { x: 42 }, C) (1.96s) + Crux output: (A(42), B { x: 42 }, C) + eq: OK (2.18s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.01s) Crux output: () - static_mut: OK (1.04s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.91s) - Crux output: 2 - never: rustc compilation failed for never -error output: -error[E0554]: `#![feature]` may not be used on the stable release channel - --> test/conc_eval/refs/never.rs:1:1 - | -1 | #![feature(never_type)] - | ^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unreachable expression - --> test/conc_eval/refs/never.rs:9:21 - | -9 | let r = &e; - | ^- - | || - | |any code following this expression is unreachable - | unreachable expression - | - = note: `#[warn(unreachable_code)]` on by default - -error: aborting due to previous error; 1 warning emitted - -For more information about this error, try `rustc --explain E0554`. - -FAIL (0.04s) - Compiling and running oracle program (0.04s) - test/Test.hs:109: - failed to compile and run - - Use -p '$0=="crux-mir.crux concrete..refs.never"' to rerun this test only. - temp: FAIL (1.05s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.92s) + cmp: OK (2.13s) + Compiling and running oracle program (0.17s) + Oracle output: -23 (1.96s) + Crux output: -23 + inner: OK (2.19s) + Compiling and running oracle program (0.17s) + Oracle output: 4 (2.02s) + Crux output: 4 + vec + drop: FAIL (2.33s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.15s) Crux output: failures: - ---- temp/8e197f82::crux_test[0] counterexamples ---- + ---- drop/af39379f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/refs/temp.rs:4:13: 4:16: error: in temp/8e197f82::crux_test[0] - [Crux] Translation error in temp/8e197f82::crux_test[0]: cannot find static variable alloc$1/3a1fbbbh + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/temp/' to rerun this test only. - fn_ptr: rustc compilation failed for fn_ptr -error output: -error[E0554]: `#![feature]` may not be used on the stable release channel - --> test/conc_eval/refs/fn_ptr.rs:1:1 - | -1 | #![feature(never_type)] - | ^^^^^^^^^^^^^^^^^^^^^^^ - -error: aborting due to previous error - -For more information about this error, try `rustc --explain E0554`. - -FAIL (0.04s) - Compiling and running oracle program (0.04s) - test/Test.hs:109: - failed to compile and run - - Use -p '$0=="crux-mir.crux concrete..refs.fn_ptr"' to rerun this test only. - fn_ptr_mut: rustc compilation failed for fn_ptr_mut -error output: -error[E0554]: `#![feature]` may not be used on the stable release channel - --> test/conc_eval/refs/fn_ptr_mut.rs:1:1 - | -1 | #![feature(never_type)] - | ^^^^^^^^^^^^^^^^^^^^^^^ - -error: aborting due to previous error - -For more information about this error, try `rustc --explain E0554`. - -FAIL (0.04s) - Compiling and running oracle program (0.04s) - test/Test.hs:109: - failed to compile and run - - Use -p '/fn_ptr_mut/' to rerun this test only. - promoted_imm: OK (1.08s) - Compiling and running oracle program (0.13s) - Oracle output: 0 (0.95s) - Crux output: 0 - imm_arg: OK (1.08s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.95s) - Crux output: 1 - mut_tuple_field: FAIL (1.05s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.92s) + Use -p '/drop/' to rerun this test only. + set_len: FAIL (2.33s) + Compiling and running oracle program (0.18s) + Oracle output: 2 (2.15s) Crux output: failures: - ---- mut_tuple_field/c0cb8da5::f[0] counterexamples ---- + ---- set_len/ce70a406::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/refs/mut_tuple_field.rs:10:19: 10:25: error: in mut_tuple_field/c0cb8da5::f[0] - [Crux] Translation error in mut_tuple_field/c0cb8da5::f[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/mut_tuple_field/' to rerun this test only. - mut_raw: OK (1.03s) - Compiling and running oracle program (0.13s) - Oracle output: 123 (0.90s) - Crux output: 123 - imm_ref: OK (1.04s) - Compiling and running oracle program (0.13s) - Oracle output: 123 (0.92s) - Crux output: 123 - mut_ref: OK (1.03s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (0.89s) - Crux output: 1 - mut_arg: OK (1.01s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (0.88s) - Crux output: 1 - time - instant: FAIL (1.80s) - Compiling and running oracle program (0.13s) - Oracle output: () (1.67s) - standalone use of `dyn` is not supported: TyDynamic core/7ed35d8a::error[0]::Error[0]::_trait56353a7b840403b4[0] - CallStack (from HasCallStack): - error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy - tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate - translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language - - Use -p '/instant/' to rerun this test only. - impl - simple: OK (1.03s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.90s) - Crux output: 42 - self_mut: OK (1.02s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.89s) - Crux output: 42 - self: OK (1.03s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (0.88s) - Crux output: 42 - vec_deque - iter_clone: FAIL (1.77s) - Compiling and running oracle program (0.15s) - Oracle output: [1, 2, 3, 2, 3] (1.62s) + Use -p '/set_len/' to rerun this test only. + push: FAIL (2.47s) + Compiling and running oracle program (0.19s) + Oracle output: (1, 2) (2.28s) Crux output: failures: - ---- iter_clone/a40d79d5::crux_test[0] counterexamples ---- + ---- push/1ea71252::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] lib/core/src/cmp.rs:1300:11: 1300:28: error: in core/5a1f6960::cmp[0]::max_by[0]::_insta10905add4aa20a5[0] + [Crux] Translation error in core/5a1f6960::cmp[0]::max_by[0]::_insta10905add4aa20a5[0]: illegal or unimplemented coercion from TyInt USize (concretely BVRepr 32) to TyInt B8 [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/iter_clone/' to rerun this test only. - pop: FAIL (1.61s) - Compiling and running oracle program (0.14s) - Oracle output: [5, 4, 3, 1, 2] (1.47s) + Use -p '/vec.push/' to rerun this test only. + extend: FAIL (2.58s) + Compiling and running oracle program (0.19s) + Oracle output: (1, 10) (2.39s) Crux output: failures: - ---- pop/358c93ef::crux_test[0] counterexamples ---- + ---- extend/842dc1c4::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] lib/core/src/cmp.rs:1300:11: 1300:28: error: in core/5a1f6960::cmp[0]::max_by[0]::_insta10905add4aa20a5[0] + [Crux] Translation error in core/5a1f6960::cmp[0]::max_by[0]::_insta10905add4aa20a5[0]: illegal or unimplemented coercion from TyInt USize (concretely BVRepr 32) to TyInt B8 [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/vec_deque.pop/' to rerun this test only. - push: FAIL (1.72s) - Compiling and running oracle program (0.15s) - Oracle output: [4, 1, 2, 3, 5] (1.57s) + Use -p '$0=="crux-mir.crux concrete..vec.extend"' to rerun this test only. + collect: FAIL (3.30s) + Compiling and running oracle program (0.19s) + Oracle output: 45 (3.11s) Crux output: failures: - ---- push/f2efcdfd::crux_test[0] counterexamples ---- + ---- collect/f7c87f57::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/cmp.rs:1300:11: 1300:28: error: in core/7ed35d8a::cmp[0]::max_by[0]::_insta10905add4aa20a5[0] - [Crux] Translation error in core/7ed35d8a::cmp[0]::max_by[0]::_insta10905add4aa20a5[0]: illegal or unimplemented coercion from TyInt USize (concretely BVRepr 32) to TyInt B8 + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/5a1f6960::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyInt B32) Mut + [Crux] as: TyRawPtr (TyUint B8) Mut [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/vec_deque.push/' to rerun this test only. - retain: FAIL (1.66s) - Compiling and running oracle program (0.15s) - Oracle output: [1, 4] (1.52s) + Use -p '/collect/' to rerun this test only. + extend_trusted_len: FAIL (3.18s) + Compiling and running oracle program (0.19s) + Oracle output: (1, 10) (2.99s) Crux output: failures: - ---- retain/405ed612::crux_test[0] counterexamples ---- + ---- extend_trusted_len/87428b97::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] lib/core/src/cmp.rs:1300:11: 1300:28: error: in core/5a1f6960::cmp[0]::max_by[0]::_insta10905add4aa20a5[0] + [Crux] Translation error in core/5a1f6960::cmp[0]::max_by[0]::_insta10905add4aa20a5[0]: illegal or unimplemented coercion from TyInt USize (concretely BVRepr 32) to TyInt B8 [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/retain/' to rerun this test only. - rotate_left: FAIL (1.77s) - Compiling and running oracle program (0.15s) - Oracle output: [3, 4, 5, 1, 2] (1.62s) - Crux output: - failures: - - ---- rotate_left/86784989::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:126: - crux doesn't match oracle + Use -p '/extend_trusted_len/' to rerun this test only. + from_elem_zero: FAIL (0.24s) + Compiling and running oracle program (0.18s) + Oracle output: 0 (0.06s) + user error (Error 101 while running mir-json on test/conc_eval/vec/from_elem_zero.rs) - Use -p '/rotate_left/' to rerun this test only. - rotate_right: FAIL (1.72s) - Compiling and running oracle program (0.14s) - Oracle output: [4, 5, 1, 2, 3] (1.58s) + Use -p '/from_elem_zero/' to rerun this test only. + clos + promoted: OK (2.16s) + Compiling and running oracle program (0.17s) + Oracle output: 33 (1.98s) + Crux output: 33 + as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (2.20s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (2.03s) Crux output: failures: - ---- rotate_right/0cfc3b17::crux_test[0] counterexamples ---- + ---- as_fn_ptr/a835ceec::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/a835ceec::crux_test[0] + [Crux] Translation error in as_fn_ptr/a835ceec::crux_test[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [], _fsreturn_ty = TyInt B32, _fsabi = RustAbi, _fsspreadarg = Nothing}) [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - - Use -p '/rotate_right/' to rerun this test only. - ops - deref3: OK (1.04s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.90s) - Crux output: () - index3: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.87s) - Crux output: () - deref2: OK (1.00s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.86s) - Crux output: () - deref1: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.88s) - Crux output: () - index2: OK (1.04s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.91s) - Crux output: () - arith1: OK (1.05s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.92s) - Crux output: () - index1: OK (1.03s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.89s) - Crux output: () - num - saturate: FAIL (1.49s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.36s) + (expected failure) + fnptr_fnmut: OK (2.14s) + Compiling and running oracle program (0.17s) + Oracle output: 4 (1.97s) + Crux output: 4 + direct_fnonce: OK (2.18s) + Compiling and running oracle program (0.18s) + Oracle output: 2 (2.00s) + Crux output: 2 + conv_fnonce_fnmut: OK (2.14s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (1.97s) + Crux output: 2 + fn_static_poly: OK (2.13s) + Compiling and running oracle program (0.17s) + Oracle output: 3 (1.96s) + Crux output: 3 + fnptr_fnonce: OK (2.24s) + Compiling and running oracle program (0.17s) + Oracle output: 4 (2.07s) + Crux output: 4 + fn_static: OK (2.29s) + Compiling and running oracle program (0.17s) + Oracle output: 3 (2.12s) + Crux output: 3 + fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (0.22s) + Compiling and running oracle program (0.19s) + Oracle output: 3 (0.03s) + user error (Error 101 while running mir-json on test/conc_eval/clos/fn_dyn.rs) + (expected failure) + fnonce: OK (2.15s) + Compiling and running oracle program (0.17s) + Oracle output: false (1.98s) + Crux output: false + conv_fnmut_fn: OK (2.16s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (1.99s) + Crux output: 2 + direct_fnmut2: OK (2.19s) + Compiling and running oracle program (0.19s) + Oracle output: 7 (2.00s) + Crux output: 7 + fo: OK (2.22s) + Compiling and running oracle program (0.17s) + Oracle output: 29 (2.06s) + Crux output: 29 + unique_borrow: OK (2.19s) + Compiling and running oracle program (0.17s) + Oracle output: 3 (2.02s) + Crux output: 3 + conv_fnonce_fn: OK (2.16s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (1.99s) + Crux output: 2 + fnptr_fn: OK (2.12s) + Compiling and running oracle program (0.17s) + Oracle output: 4 (1.96s) + Crux output: 4 + direct_fnmut: OK (2.14s) + Compiling and running oracle program (0.17s) + Oracle output: 7 (1.98s) + Crux output: 7 + dispatch_fnmut: OK (2.19s) + Compiling and running oracle program (0.17s) + Oracle output: 7 (2.02s) + Crux output: 7 + poly: OK (2.19s) + Compiling and running oracle program (0.17s) + Oracle output: 3 (2.03s) + Crux output: 3 + direct_fn: OK (2.15s) + Compiling and running oracle program (0.18s) + Oracle output: 2 (1.98s) + Crux output: 2 + fnptr_closure: OK (2.23s) + Compiling and running oracle program (0.17s) + Oracle output: 7 (2.06s) + Crux output: 7 + fnonce1: OK (2.13s) + Compiling and running oracle program (0.17s) + Oracle output: 3 (1.97s) + Crux output: 3 + ref_fnmut: OK (2.15s) + Compiling and running oracle program (0.17s) + Oracle output: 7 (1.99s) + Crux output: 7 + vec_deque + pop: FAIL (3.06s) + Compiling and running oracle program (0.19s) + Oracle output: [5, 4, 3, 1, 2] (2.87s) Crux output: failures: - ---- saturate/6a87c133::crux_test[0] counterexamples ---- + ---- pop/becc159b::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:1022:46: 1022:49 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:305:5: 306:27: error: in core/7ed35d8a::num[0]::{impl#7}[0]::saturating_add[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#7}[0]::saturating_add[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::saturating_add[0]::_instaddce72e1232152c[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/saturate/' to rerun this test only. - overflow: FAIL (1.08s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.94s) + Use -p '/vec_deque.pop/' to rerun this test only. + iter_clone: FAIL (3.07s) + Compiling and running oracle program (0.19s) + Oracle output: [1, 2, 3, 2, 3] (2.88s) Crux output: failures: - ---- overflow/bd0e6831::crux_test[0] counterexamples ---- + ---- iter_clone/07a0e881::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/num/overflow.rs:5:40: 5:50: error: in overflow/bd0e6831::crux_test[0] - [Crux] Translation error in overflow/bd0e6831::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/num.overflow/' to rerun this test only. - from_bytes: FAIL (1.05s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.92s) + Use -p '/iter_clone/' to rerun this test only. + push: FAIL (3.02s) + Compiling and running oracle program (0.20s) + Oracle output: [4, 1, 2, 3, 5] (2.82s) Crux output: failures: - ---- from_bytes/4a8462ce::f[0] counterexamples ---- + ---- push/2bbd650b::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:2417:37: 2417:42 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:890:5: 891:49: error: in core/7ed35d8a::num[0]::{impl#8}[0]::from_ne_bytes[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#8}[0]::from_ne_bytes[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::transmute[0]::_inst850c595be4adc72f[0] + [Crux] lib/core/src/cmp.rs:1300:11: 1300:28: error: in core/5a1f6960::cmp[0]::max_by[0]::_insta10905add4aa20a5[0] + [Crux] Translation error in core/5a1f6960::cmp[0]::max_by[0]::_insta10905add4aa20a5[0]: illegal or unimplemented coercion from TyInt USize (concretely BVRepr 32) to TyInt B8 [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/from_bytes/' to rerun this test only. - crypto - add: FAIL (1.09s) - Compiling and running oracle program (0.14s) - Oracle output: true (0.95s) + Use -p '/vec_deque.push/' to rerun this test only. + retain: FAIL (3.03s) + Compiling and running oracle program (0.18s) + Oracle output: [1, 4] (2.85s) Crux output: failures: - ---- add/182a6d35::crux_test[0] counterexamples ---- + ---- retain/71809604::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/num/uint_macros.rs:470:54: 470:57 !lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::unchecked_add[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::unchecked_add[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_instaffa7a8b1157c078[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '$0=="crux-mir.crux concrete..crypto.add"' to rerun this test only. - add_noL: FAIL (1.06s) - Compiling and running oracle program (0.14s) - Oracle output: false (0.93s) + Use -p '/retain/' to rerun this test only. + rotate_left: FAIL (3.04s) + Compiling and running oracle program (0.18s) + Oracle output: [3, 4, 5, 1, 2] (2.86s) Crux output: failures: - ---- add_noL/81992a91::crux_test[0] counterexamples ---- + ---- rotate_left/c536d376::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/num/uint_macros.rs:470:54: 470:57 !lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::unchecked_add[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::unchecked_add[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_instaffa7a8b1157c078[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/add_noL/' to rerun this test only. - traits - bounds2: FAIL (1.05s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.91s) + Use -p '/rotate_left/' to rerun this test only. + rotate_right: FAIL (3.14s) + Compiling and running oracle program (0.20s) + Oracle output: [4, 5, 1, 2, 3] (2.93s) Crux output: failures: - ---- bounds2/5d337868::crux_test[0] counterexamples ---- + ---- rotate_right/7b40a8a9::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in bounds2/5d337868::f[0] - [Crux] Translation error in bounds2/5d337868::f[0]: error building variant core/7ed35d8a::option[0]::Option[0]::Some[0]: not enough expressions -- [] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/bounds2/' to rerun this test only. - tyfam: OK (1.02s) - Compiling and running oracle program (0.13s) - Oracle output: 23 (0.89s) - Crux output: 23 - dict_poly: OK (1.03s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (0.89s) - Crux output: 1 - dynamic_simple: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.85s) - Crux output: 1 - assoc1: OK (1.03s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.89s) - Crux output: () - static_two: OK (1.04s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.91s) - Crux output: 1 - static_self: OK (1.02s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (0.88s) - Crux output: 42 - generic2: FAIL (1.05s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.92s) + Use -p '/rotate_right/' to rerun this test only. + slice + len: OK (2.21s) + Compiling and running oracle program (0.17s) + Oracle output: 5 (2.04s) + Crux output: 5 + mut_range: FAIL (2.77s) + Compiling and running oracle program (0.19s) + Oracle output: 86 (2.58s) Crux output: failures: - ---- generic2/21c9978c::crux_test[0] counterexamples ---- + ---- mut_range/037e427f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in generic2/21c9978c::f[0] - [Crux] Translation error in generic2/21c9978c::f[0]: error building variant core/7ed35d8a::option[0]::Option[0]::Some[0]: not enough expressions -- [] + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/generic2/' to rerun this test only. - bounds1: FAIL (1.08s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.94s) + Use -p '/mut_range/' to rerun this test only. + range_len: FAIL (2.74s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (2.57s) Crux output: failures: - ---- bounds1/324a6555::crux_test[0] counterexamples ---- + ---- range_len/d549cefb::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in bounds1/324a6555::f[0] - [Crux] Translation error in bounds1/324a6555::f[0]: error building variant core/7ed35d8a::option[0]::Option[0]::Some[0]: not enough expressions -- [] + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/bounds1/' to rerun this test only. - gen_trait_poly: OK (1.02s) - Compiling and running oracle program (0.13s) - Oracle output: 12 (0.88s) - Crux output: 12 - dynamic_poly: FAIL (0.17s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (0.03s) - user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_poly.rs) - - Use -p '/dynamic_poly/' to rerun this test only. - static: OK (1.00s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (0.85s) - Crux output: 42 - tyfam4: OK (1.02s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (0.88s) - Crux output: 0 - assoc2: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.88s) - Crux output: () - basics1: OK (1.06s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.92s) - Crux output: () - gen_trait: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: -69 (0.88s) - Crux output: -69 - assoc3: OK (1.02s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.88s) - Crux output: () - default: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: 12 (0.87s) - Crux output: 12 - bounds4: FAIL (1.05s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.90s) + Use -p '$0=="crux-mir.crux concrete..slice.range_len"' to rerun this test only. + range_len_mut: FAIL (2.82s) + Compiling and running oracle program (0.18s) + Oracle output: 1 (2.63s) Crux output: failures: - ---- bounds4/a5df7d79::crux_test[0] counterexamples ---- + ---- range_len_mut/efebb8fd::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in bounds4/a5df7d79::f[0] - [Crux] Translation error in bounds4/a5df7d79::f[0]: error building variant core/7ed35d8a::option[0]::Option[0]::Some[0]: not enough expressions -- [] + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/bounds4/' to rerun this test only. - intoiter: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 0 (0.85s) - Crux output: 0 - tyfam5: OK (1.02s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.87s) - Crux output: () - dict_polymem: OK (1.04s) - Compiling and running oracle program (0.13s) - Oracle output: 4 (0.91s) - Crux output: 4 - dynamic_branch: FAIL (0.19s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (0.03s) - user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_branch.rs) - - Use -p '/dynamic_branch/' to rerun this test only. - subtrait: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: 73 (0.86s) - Crux output: 73 - params: OK (1.00s) - Compiling and running oracle program (0.14s) - Oracle output: 25 (0.86s) - Crux output: 25 - conv: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.87s) - Crux output: 1 - bounds5: FAIL (1.07s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.93s) + Use -p '/range_len_mut/' to rerun this test only. + mk_and_proj: OK (2.15s) + Compiling and running oracle program (0.17s) + Oracle output: 42 (1.98s) + Crux output: 42 + swap: OK (2.20s) + Compiling and running oracle program (0.18s) + Oracle output: 2001 (2.02s) + Crux output: 2001 + eq: FAIL (2.20s) + Compiling and running oracle program (0.16s) + Oracle output: () (2.04s) Crux output: failures: - ---- bounds5/5b4757b0::crux_test[0] counterexamples ---- + ---- eq/13e22240::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in bounds5/5b4757b0::f[0] - [Crux] Translation error in bounds5/5b4757b0::f[0]: error building variant core/7ed35d8a::option[0]::Option[0]::Some[0]: not enough expressions -- [] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/mem/mod.rs:338:38: 338:41: error: in core/5a1f6960::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/5a1f6960::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::size_of_val[0]::_inst55af81a2cef2701c[0] [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/bounds5/' to rerun this test only. - dict_simple: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: 32 (0.88s) - Crux output: 32 - subtrait2: OK (1.05s) - Compiling and running oracle program (0.13s) - Oracle output: 64 (0.92s) - Crux output: 64 - generic3: FAIL (1.05s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.92s) + Use -p '/slice.eq/' to rerun this test only. + iter_mut: FAIL (2.28s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.10s) Crux output: failures: - ---- generic3/a3e60e58::crux_test[0] counterexamples ---- + ---- iter_mut/af588650::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in generic3/a3e60e58::f[0] - [Crux] Translation error in generic3/a3e60e58::f[0]: error building variant core/7ed35d8a::option[0]::Option[0]::Some[0]: not enough expressions -- [] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/5a1f6960::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyInt B32) Mut + [Crux] as: TyRawPtr (TyUint B8) Mut [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/generic3/' to rerun this test only. - bounds3: FAIL (1.02s) - Compiling and running oracle program (0.16s) - Oracle output: () (0.87s) + Use -p '/iter_mut/' to rerun this test only. + get: FAIL (2.36s) + Compiling and running oracle program (0.17s) + Oracle output: true (2.19s) Crux output: failures: - ---- bounds3/ad390bcf::crux_test[0] counterexamples ---- + ---- get/8d7d266a::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in bounds3/ad390bcf::f[0] - [Crux] Translation error in bounds3/ad390bcf::f[0]: error building variant core/7ed35d8a::option[0]::Option[0]::Some[0]: not enough expressions -- [] + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. - test/Test.hs:126: + test/Test.hs:124: crux doesn't match oracle - Use -p '/bounds3/' to rerun this test only. - static_eq: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: true (0.85s) - Crux output: true - static_three: OK (1.05s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (0.92s) - Crux output: 2 - tyfam2: OK (1.04s) - Compiling and running oracle program (0.13s) - Oracle output: 23 (0.91s) - Crux output: 23 - dict_med: OK (1.03s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (0.89s) + Use -p '/slice.get/' to rerun this test only. + mut: OK (2.30s) + Compiling and running oracle program (0.17s) + Oracle output: 42 (2.13s) Crux output: 42 - tyfam3: OK (1.00s) - Compiling and running oracle program (0.15s) - Oracle output: 23 (0.86s) - Crux output: 23 - generic1: FAIL (1.07s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.93s) - Crux output: - failures: - - ---- generic1/c742374a::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in generic1/c742374a::f[0] - [Crux] Translation error in generic1/c742374a::f[0]: error building variant core/7ed35d8a::option[0]::Option[0]::Some[0]: not enough expressions -- [] - - [Crux] Overall status: Invalid. - - test/Test.hs:126: - crux doesn't match oracle - - Use -p '/generic1/' to rerun this test only. - dynamic_med: FAIL (0.16s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.03s) - user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_med.rs) - - Use -p '/dynamic_med/' to rerun this test only. - dynamic_two: FAIL (0.17s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (0.03s) - user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_two.rs) - - Use -p '/dynamic_two/' to rerun this test only. - io - cursor_read: FAIL (1.50s) - Compiling and running oracle program (0.15s) - Oracle output: 0 (1.34s) - standalone use of `dyn` is not supported: TyDynamic core/7ed35d8a::error[0]::Error[0]::_trait56353a7b840403b4[0] - CallStack (from HasCallStack): - error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy - tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate - translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language - - Use -p '/io.cursor_read/' to rerun this test only. - cursor_write2: FAIL (1.87s) - Compiling and running oracle program (0.15s) - Oracle output: 0 (1.72s) - standalone use of `dyn` is not supported: TyDynamic core/7ed35d8a::error[0]::Error[0]::_trait56353a7b840403b4[0] - CallStack (from HasCallStack): - error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy - tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate - translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language - - Use -p '/cursor_write2/' to rerun this test only. - cursor_write: FAIL (1.48s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.33s) - standalone use of `dyn` is not supported: TyDynamic core/7ed35d8a::error[0]::Error[0]::_trait56353a7b840403b4[0] - CallStack (from HasCallStack): - error, called at src/Mir/TransTy.hs:230:25 in crux-mir-0.6.0.99-inplace:Mir.TransTy - tyToRepr, called at src/Mir/Trans.hs:810:12 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:878:5 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:930:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1142:11 in crux-mir-0.6.0.99-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1740:68 in crux-mir-0.6.0.99-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1749:28 in crux-mir-0.6.0.99-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1789:10 in crux-mir-0.6.0.99-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1835:24 in crux-mir-0.6.0.99-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2272:30 in crux-mir-0.6.0.99-inplace:Mir.Trans - transCollection, called at src/Mir/Generate.hs:236:26 in crux-mir-0.6.0.99-inplace:Mir.Generate - translateMIR, called at src/Mir/Language.hs:252:16 in crux-mir-0.6.0.99-inplace:Mir.Language - - Use -p '$0=="crux-mir.crux concrete..io.cursor_write"' to rerun this test only. - statics - promoted_static: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.88s) - Crux output: 1 - promoted_fn: FAIL (1.04s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (0.90s) - Crux output: - failures: - - ---- promoted_fn/8e484c2a::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/statics/promoted_fn.rs:4:13: 4:16: error: in promoted_fn/8e484c2a::crux_test[0] - [Crux] Translation error in promoted_fn/8e484c2a::crux_test[0]: cannot find static variable alloc$1/3a1fbbbh - - [Crux] Overall status: Invalid. - - test/Test.hs:126: - crux doesn't match oracle - - Use -p '/promoted_fn/' to rerun this test only. - struct - repr_transparent: FAIL (1.07s) - Compiling and running oracle program (0.14s) - Oracle output: 6 (0.93s) - Crux output: - failures: - - ---- repr_transparent/5ed63a8b::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:61:9: 61:18: error: in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0] - [Crux] Translation error in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyInt B32) Immut - [Crux] as: TyRawPtr (TyArray (TyInt B32) 2) Immut - - [Crux] Overall status: Invalid. - - test/Test.hs:126: - crux doesn't match oracle - - Use -p '$0=="crux-mir.crux concrete..struct.repr_transparent"' to rerun this test only. - repr_transparent_const: OK (1.00s) - Compiling and running oracle program (0.14s) - Oracle output: 124 (0.86s) - Crux output: 124 - field_order: OK (1.04s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.91s) - Crux output: () - arg: OK (0.99s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (0.85s) - Crux output: 42 - proj: OK (1.02s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.88s) - Crux output: 42 - ret: OK (1.05s) - Compiling and running oracle program (0.14s) - Oracle output: S { x: 42, y: 120 } (0.91s) - Crux output: S { x: 42, y: 120 } - tup: OK (1.06s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.92s) - Crux output: () - ptr - valid_eq: OK (1.06s) - Compiling and running oracle program (0.15s) - Oracle output: () (0.91s) - Crux output: () - cast_eq: OK (1.03s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.90s) - Crux output: () - read_write: OK (1.10s) - Compiling and running oracle program (0.14s) - Oracle output: [1, 3, 1] (0.96s) - Crux output: [1, 3, 1] - coerce_unsized: rustc compilation failed for coerce_unsized -error output: -warning: unused import: `std::ptr` - --> test/conc_eval/ptr/coerce_unsized.rs:5:5 - | -5 | use std::ptr; - | ^^^^^^^^ - | - = note: `#[warn(unused_imports)]` on by default - -error[E0554]: `#![feature]` may not be used on the stable release channel - --> test/conc_eval/ptr/coerce_unsized.rs:1:12 - | -1 | #![feature(unsize)] - | ^^^^^^ - -error[E0554]: `#![feature]` may not be used on the stable release channel - --> test/conc_eval/ptr/coerce_unsized.rs:2:12 - | -2 | #![feature(coerce_unsized)] - | ^^^^^^^^^^^^^^ - -error: aborting due to 2 previous errors; 1 warning emitted - -For more information about this error, try `rustc --explain E0554`. - -FAIL (0.04s) - Compiling and running oracle program (0.04s) - test/Test.hs:109: - failed to compile and run - - Use -p '/coerce_unsized/' to rerun this test only. - null_eq: FAIL (1.06s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (0.93s) - Crux output: - failures: - - ---- null_eq/399f1a30::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mod.rs:568:29: 568:33: error: in core/7ed35d8a::ptr[0]::invalid[0]::_instf59dc84daa91a404[0] - [Crux] Translation error in core/7ed35d8a::ptr[0]::invalid[0]::_instf59dc84daa91a404[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::transmute[0]::_insta5d0a99e3b08f3bd[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:126: - crux doesn't match oracle - - Use -p '/null_eq/' to rerun this test only. - offset_from: rustc compilation failed for offset_from -error output: -warning: unused import: `std::ptr` - --> test/conc_eval/ptr/offset_from.rs:2:5 - | -2 | use std::ptr; - | ^^^^^^^^ - | - = note: `#[warn(unused_imports)]` on by default - -error[E0554]: `#![feature]` may not be used on the stable release channel - --> test/conc_eval/ptr/offset_from.rs:1:12 - | -1 | #![feature(ptr_offset_from)] - | ^^^^^^^^^^^^^^^ - -warning: the feature `ptr_offset_from` has been stable since 1.47.0 and no longer requires an attribute to enable - --> test/conc_eval/ptr/offset_from.rs:1:12 - | -1 | #![feature(ptr_offset_from)] - | ^^^^^^^^^^^^^^^ - | - = note: `#[warn(stable_features)]` on by default - -error: aborting due to previous error; 2 warnings emitted - -For more information about this error, try `rustc --explain E0554`. - -FAIL (0.06s) - Compiling and running oracle program (0.06s) - test/Test.hs:109: - failed to compile and run - - Use -p '/offset_from/' to rerun this test only. - is_null: FAIL (1.06s) - Compiling and running oracle program (0.14s) - Oracle output: (false, true) (0.91s) - Crux output: - failures: - - ---- is_null/42d14c9d::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/ptr/is_null.rs:5:13: 5:15: error: in is_null/42d14c9d::crux_test[0] - [Crux] Translation error in is_null/42d14c9d::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh - - [Crux] Overall status: Invalid. - - test/Test.hs:126: - crux doesn't match oracle - - Use -p '$0=="crux-mir.crux concrete..ptr.is_null"' to rerun this test only. - dangling_eq: FAIL (1.03s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.90s) - Crux output: - failures: - - ---- dangling_eq/f33166fd::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mod.rs:599:29: 599:33: error: in core/7ed35d8a::ptr[0]::invalid_mut[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/7ed35d8a::ptr[0]::invalid_mut[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::transmute[0]::_inst102627303ebf2823[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:126: - crux doesn't match oracle - - Use -p '/dangling_eq/' to rerun this test only. - offset: FAIL (1.06s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (0.92s) - Crux output: - failures: - - ---- offset/dd961ed4::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:473:43: 473:48: error: in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::offset[0]::_inst1e2825177cd3b608[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:126: - crux doesn't match oracle - - Use -p '$0=="crux-mir.crux concrete..ptr.offset"' to rerun this test only. - offset_mut: FAIL (1.07s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.93s) - Crux output: - failures: - - ---- offset_mut/b36ef04b::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:487:43: 487:48: error: in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/7ed35d8a::ptr[0]::mut_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::offset[0]::_inst1e2825177cd3b608[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:126: - crux doesn't match oracle - - Use -p '/offset_mut/' to rerun this test only. - unsize_slice: FAIL (1.06s) - Compiling and running oracle program (0.14s) - Oracle output: (1, 2) (0.91s) - Crux output: - failures: - - ---- unsize_slice/0d5b391d::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:473:43: 473:48: error: in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/7ed35d8a::ptr[0]::const_ptr[0]::{impl#0}[0]::offset[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::offset[0]::_inst1e2825177cd3b608[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:126: - crux doesn't match oracle - - Use -p '/unsize_slice/' to rerun this test only. - struct_eq: OK (1.03s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.90s) - Crux output: () - copy: OK (1.08s) - Compiling and running oracle program (0.15s) - Oracle output: [1, 2, 3, 1, 2, 3] (0.93s) - Crux output: [1, 2, 3, 1, 2, 3] - is_null_slice: FAIL (expected: can't unsize null pointers) (1.05s) - Compiling and running oracle program (0.14s) - Oracle output: (false, true) (0.91s) - Crux output: - failures: - - ---- is_null_slice/767ccbda::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/ptr/is_null_slice.rs:6:13: 6:20: error: in is_null_slice/767ccbda::crux_test[0] - [Crux] Translation error in is_null_slice/767ccbda::crux_test[0]: cannot find static variable alloc$0/3a1fbbbh - - [Crux] Overall status: Invalid. - - test/Test.hs:126: - crux doesn't match oracle - (expected failure) + last: OK (2.24s) + Compiling and running oracle program (0.17s) + Oracle output: 3 (2.07s) + Crux output: 3 crux symbolic Output testing - sort_by_key: FAIL (1.57s) - files test/symb_eval/vec/sort_by_key.good and test/symb_eval/vec/sort_by_key.out differ; test/symb_eval/vec/sort_by_key.out contains: - test sort_by_key/43dcef21::f[0]: FAILED + mux_init_mut: FAIL (2.10s) + files test/symb_eval/refs/mux_init_mut.good and test/symb_eval/refs/mux_init_mut.out differ; test/symb_eval/refs/mux_init_mut.out contains: + test mux_init_mut/ba0f083a::crux_test[0]: FAILED + + failures: + + ---- mux_init_mut/ba0f083a::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + [Crux] Overall status: Invalid. + + Use -p '/mux_init_mut/' to rerun this test only. + mux_init_imm: FAIL (2.17s) + files test/symb_eval/refs/mux_init_imm.good and test/symb_eval/refs/mux_init_imm.out differ; test/symb_eval/refs/mux_init_imm.out contains: + test mux_init_imm/f397f768::crux_test[0]: FAILED + + failures: + + ---- mux_init_imm/f397f768::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + [Crux] Overall status: Invalid. + + Use -p '/mux_init_imm/' to rerun this test only. + early_fail: FAIL (2.14s) + files test/symb_eval/crux/early_fail.good and test/symb_eval/crux/early_fail.out differ; test/symb_eval/crux/early_fail.out contains: + test early_fail/aaff3446::fail1[0]: FAILED + test early_fail/aaff3446::fail2[0]: FAILED + + failures: + + ---- early_fail/aaff3446::fail1[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in early_fail/aaff3446::fail1[0] + [Crux] panicking::panic_fmt, called from early_fail/aaff3446::fail1[0] + + ---- early_fail/aaff3446::fail2[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + [Crux] Overall status: Invalid. + + Use -p '/early_fail/' to rerun this test only. + mixed_fail: FAIL (2.16s) + files test/symb_eval/crux/mixed_fail.good and test/symb_eval/crux/mixed_fail.out differ; test/symb_eval/crux/mixed_fail.out contains: + test mixed_fail/927be8d8::fail1[0]: FAILED + test mixed_fail/927be8d8::fail2[0]: FAILED + test mixed_fail/927be8d8::pass1[0]: FAILED + test mixed_fail/927be8d8::pass2[0]: FAILED + + failures: + + ---- mixed_fail/927be8d8::fail1[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + ---- mixed_fail/927be8d8::fail2[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + ---- mixed_fail/927be8d8::pass1[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + ---- mixed_fail/927be8d8::pass2[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + [Crux] Overall status: Invalid. + + Use -p '/mixed_fail/' to rerun this test only. + multi: FAIL (2.11s) + files test/symb_eval/crux/multi.good and test/symb_eval/crux/multi.out differ; test/symb_eval/crux/multi.out contains: + test multi/6273896c::fail1[0]: FAILED + test multi/6273896c::fail2[0]: FAILED + test multi/6273896c::fail3[0]: FAILED + + failures: + + ---- multi/6273896c::fail1[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + ---- multi/6273896c::fail2[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + ---- multi/6273896c::fail3[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + [Crux] Overall status: Invalid. + + Use -p '/Output testing.multi/' to rerun this test only. + fail_return: FAIL (2.16s) + files test/symb_eval/crux/fail_return.good and test/symb_eval/crux/fail_return.out differ; test/symb_eval/crux/fail_return.out contains: + test fail_return/bbd9602c::fail1[0]: FAILED + test fail_return/bbd9602c::fail2[0]: FAILED + + failures: + + ---- fail_return/bbd9602c::fail1[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + ---- fail_return/bbd9602c::fail2[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + [Crux] Overall status: Invalid. + + Use -p '/fail_return/' to rerun this test only. + no_conc: FAIL (2.17s) + files test/symb_eval/concretize/no_conc.good and test/symb_eval/concretize/no_conc.out differ; test/symb_eval/concretize/no_conc.out contains: + test no_conc/5b356412::crux_test[0]: FAILED + + failures: + + ---- no_conc/5b356412::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + [Crux] Overall status: Invalid. + + Use -p '/no_conc/' to rerun this test only. + conc: FAIL (2.10s) + files test/symb_eval/concretize/conc.good and test/symb_eval/concretize/conc.out differ; test/symb_eval/concretize/conc.out contains: + test conc/b85a9e47::crux_test[0]: FAILED + + failures: + + ---- conc/b85a9e47::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + [Crux] Overall status: Invalid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.conc"' to rerun this test only. + array: FAIL (2.14s) + files test/symb_eval/concretize/array.good and test/symb_eval/concretize/array.out differ; test/symb_eval/concretize/array.out contains: + test array/b294edd1::crux_test[0]: FAILED + + failures: + + ---- array/b294edd1::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::array[0]::symbolic[0]::_inst1e2825177cd3b608[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::array[0]::symbolic[0]::_inst1e2825177cd3b608[0] + + [Crux] Overall status: Invalid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.array"' to rerun this test only. + assert_ok: FAIL (2.85s) + files test/symb_eval/concretize/assert_ok.good and test/symb_eval/concretize/assert_ok.out differ; test/symb_eval/concretize/assert_ok.out contains: + test assert_ok/223e01b8::crux_test[0]: FAILED + + failures: + + ---- assert_ok/223e01b8::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + [Crux] Overall status: Invalid. + + Use -p '/assert_ok/' to rerun this test only. + assert: FAIL (2.88s) + files test/symb_eval/concretize/assert.good and test/symb_eval/concretize/assert.out differ; test/symb_eval/concretize/assert.out contains: + test assert/4d446bb2::crux_test[0]: FAILED + + failures: + + ---- assert/4d446bb2::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + [Crux] Overall status: Invalid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.assert"' to rerun this test only. + bytes2: FAIL (2.18s) + files test/symb_eval/crypto/bytes2.good and test/symb_eval/crypto/bytes2.out differ; test/symb_eval/crypto/bytes2.out contains: + test bytes2/cec6fb92::f[0]: FAILED + + failures: + + ---- bytes2/cec6fb92::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + [Crux] Overall status: Invalid. + + Use -p '/bytes2/' to rerun this test only. + double: FAIL (2.15s) + files test/symb_eval/crypto/double.good and test/symb_eval/crypto/double.out differ; test/symb_eval/crypto/double.out contains: + test double/da43fdac::f[0]: FAILED + + failures: + + ---- double/da43fdac::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u32[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u32[0] + + [Crux] Overall status: Invalid. + + Use -p '/double/' to rerun this test only. + ffs: FAIL (2.09s) + files test/symb_eval/crypto/ffs.good and test/symb_eval/crypto/ffs.out differ; test/symb_eval/crypto/ffs.out contains: + test ffs/be5003b6::f[0]: FAILED + + failures: + + ---- ffs/be5003b6::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u32[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u32[0] + + [Crux] Overall status: Invalid. + + Use -p '/Output testing.ffs/' to rerun this test only. + bytes: FAIL (2.19s) + files test/symb_eval/crypto/bytes.good and test/symb_eval/crypto/bytes.out differ; test/symb_eval/crypto/bytes.out contains: + test bytes/39b68e7d::f[0]: FAILED + + failures: + + ---- bytes/39b68e7d::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u64[0] + + [Crux] Overall status: Invalid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.bytes"' to rerun this test only. + bad_symb1: FAIL (2.16s) + files test/symb_eval/overrides/bad_symb1.good and test/symb_eval/overrides/bad_symb1.out differ; test/symb_eval/overrides/bad_symb1.out contains: + test bad_symb1/6eace5d9::crux_test[0]: FAILED + + failures: + + ---- bad_symb1/6eace5d9::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + [Crux] Overall status: Invalid. + + Use -p '/bad_symb1/' to rerun this test only. + bad_symb2: FAIL (2.10s) + files test/symb_eval/overrides/bad_symb2.good and test/symb_eval/overrides/bad_symb2.out differ; test/symb_eval/overrides/bad_symb2.out contains: + test bad_symb2/0e44e20a::crux_test[0]: FAILED + + failures: + + ---- bad_symb2/0e44e20a::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + [Crux] Overall status: Invalid. + + Use -p '/bad_symb2/' to rerun this test only. + override3: FAIL (2.07s) + files test/symb_eval/overrides/override3.good and test/symb_eval/overrides/override3.out differ; test/symb_eval/overrides/override3.out contains: + test override3/406699f3::f[0]: FAILED + + failures: + + ---- override3/406699f3::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + [Crux] Overall status: Invalid. + + Use -p '/override3/' to rerun this test only. + override1: FAIL (2.03s) + files test/symb_eval/overrides/override1.good and test/symb_eval/overrides/override1.out differ; test/symb_eval/overrides/override1.out contains: + test override1/84f90b58::f[0]: FAILED + + failures: + + ---- override1/84f90b58::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::one[0] + [Crux] panicking::panic, called from crucible/6eda2caa::one[0] + + [Crux] Overall status: Invalid. + + Use -p '/override1/' to rerun this test only. + override4: FAIL (2.13s) + files test/symb_eval/overrides/override4.good and test/symb_eval/overrides/override4.out differ; test/symb_eval/overrides/override4.out contains: + test override4/2a387b8d::f[0]: FAILED + + failures: + + ---- override4/2a387b8d::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + [Crux] Overall status: Invalid. + + Use -p '/override4/' to rerun this test only. + override_rust: FAIL (2.15s) + files test/symb_eval/overrides/override_rust.good and test/symb_eval/overrides/override_rust.out differ; test/symb_eval/overrides/override_rust.out contains: + test override_rust/81c3ff88::crux_test[0]: FAILED + + failures: + + ---- override_rust/81c3ff88::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::override_[0]::_inst28abcb7c380bf102[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::override_[0]::_inst28abcb7c380bf102[0] + + [Crux] Overall status: Invalid. + + Use -p '/override_rust/' to rerun this test only. + override2: FAIL (2.12s) + files test/symb_eval/overrides/override2.good and test/symb_eval/overrides/override2.out differ; test/symb_eval/overrides/override2.out contains: + test override2/7910119c::f[0]: FAILED + + failures: + + ---- override2/7910119c::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + [Crux] Overall status: Invalid. + + Use -p '/override2/' to rerun this test only. + override5: FAIL (2.05s) + files test/symb_eval/overrides/override5.good and test/symb_eval/overrides/override5.out differ; test/symb_eval/overrides/override5.out contains: + test override5/5f8d6467::f[0]: FAILED + + failures: + + ---- override5/5f8d6467::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u64[0] + + [Crux] Overall status: Invalid. + + Use -p '/override5/' to rerun this test only. + mux: FAIL (2.10s) + files test/symb_eval/fnptr/mux.good and test/symb_eval/fnptr/mux.out differ; test/symb_eval/fnptr/mux.out contains: + test mux/3918b96f::crux_test[0]: FAILED failures: - ---- sort_by_key/43dcef21::f[0] counterexamples ---- + ---- mux/3918b96f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. - Use -p '/sort_by_key/' to rerun this test only. - clone: FAIL (1.10s) - files test/symb_eval/vec/clone.good and test/symb_eval/vec/clone.out differ; test/symb_eval/vec/clone.out contains: - test clone/b5c63e0e::f[0]: FAILED + Use -p '$0=="crux-mir.crux symbolic.Output testing.mux"' to rerun this test only. + checked_mul_signed: FAIL + test/symb_eval/num/checked_mul_signed.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/num/checked_mul_signed.rs) + Use -p '/checked_mul_signed/' to rerun this test only. + checked_mul: FAIL (0.05s) + test/symb_eval/num/checked_mul.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/num/checked_mul.rs) + Use -p '$0=="crux-mir.crux symbolic.Output testing.checked_mul"' to rerun this test only. + checked_div: FAIL (2.06s) + files test/symb_eval/num/checked_div.good and test/symb_eval/num/checked_div.out differ; test/symb_eval/num/checked_div.out contains: + test checked_div/258a4803::crux_test[0]: returned 65, FAILED failures: - ---- clone/b5c63e0e::f[0] counterexamples ---- + ---- checked_div/258a4803::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/258a4803::div_signed[0] + [Crux] attempt to compute `_3 / _4`, which would overflow + [Crux] Found counterexample for verification goal + [Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/258a4803::div_signed[0] + [Crux] attempt to divide `_3` by zero + [Crux] Found counterexample for verification goal + [Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/258a4803::div_unsigned[0] + [Crux] attempt to divide `_3` by zero [Crux] Overall status: Invalid. - Use -p '/Output testing.clone/' to rerun this test only. - into_iter: FAIL (1.67s) - files test/symb_eval/vec/into_iter.good and test/symb_eval/vec/into_iter.out differ; test/symb_eval/vec/into_iter.out contains: - test into_iter/9e3ff4b9::f[0]: FAILED + Use -p '/checked_div/' to rerun this test only. + checked_add: FAIL (0.05s) + test/symb_eval/num/checked_add.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/num/checked_add.rs) + Use -p '/checked_add/' to rerun this test only. + test1: FAIL (2.39s) + files test/symb_eval/scalar/test1.good and test/symb_eval/scalar/test1.out differ; test/symb_eval/scalar/test1.out contains: + test test1/b6abe038::f[0]: FAILED failures: - ---- into_iter/9e3ff4b9::f[0] counterexamples ---- + ---- test1/b6abe038::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] internal: error: in int512/11aae4f6::symbolic[0] + [Crux] panicking::panic, called from int512/11aae4f6::symbolic[0] [Crux] Overall status: Invalid. - Use -p '/into_iter/' to rerun this test only. - macro: FAIL (1.05s) - files test/symb_eval/vec/macro.good and test/symb_eval/vec/macro.out differ; test/symb_eval/vec/macro.out contains: - test macro/7e21692f::f[0]: FAILED + Use -p '/test1/' to rerun this test only. + pop: FAIL (2.19s) + files test/symb_eval/vector/pop.good and test/symb_eval/vector/pop.out differ; test/symb_eval/vector/pop.out contains: + test pop/34bcb08d::f[0]: FAILED failures: - ---- macro/7e21692f::f[0] counterexamples ---- + ---- pop/34bcb08d::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/uint_macros.rs:89:31: 89:47 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/num/mod.rs:965:5: 970:27: error: in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0] - [Crux] Translation error in core/7ed35d8a::num[0]::{impl#12}[0]::count_ones[0]: callExp: Don't know how to call core/7ed35d8a::intrinsics[0]::{extern#0}[0]::ctpop[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] internal: error: in pop/34bcb08d::f[0] + [Crux] Translation error in pop/34bcb08d::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. - Use -p '/Output testing.macro/' to rerun this test only. - override4: FAIL (0.96s) - files test/symb_eval/overrides/override4.good and test/symb_eval/overrides/override4.out differ; test/symb_eval/overrides/override4.out contains: - test override4/5f4a247f::f[0]: FAILED + Use -p '/Output testing.pop/' to rerun this test only. + as_mut_slice: FAIL (2.16s) + files test/symb_eval/vector/as_mut_slice.good and test/symb_eval/vector/as_mut_slice.out differ; test/symb_eval/vector/as_mut_slice.out contains: + test as_mut_slice/7ce532be::f[0]: FAILED failures: - ---- override4/5f4a247f::f[0] counterexamples ---- + ---- as_mut_slice/7ce532be::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/5888ca02::symbolic[0]::symbolic_u8[0] - [Crux] Translation error in crucible/5888ca02::symbolic[0]::symbolic_u8[0]: cannot find static variable alloc$8/3a1fbbbh + [Crux] internal: error: in as_mut_slice/7ce532be::f[0] + [Crux] Translation error in as_mut_slice/7ce532be::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. - Use -p '/override4/' to rerun this test only. - bad_symb2: FAIL (0.95s) - files test/symb_eval/overrides/bad_symb2.good and test/symb_eval/overrides/bad_symb2.out differ; test/symb_eval/overrides/bad_symb2.out contains: - test bad_symb2/db5a10b0::crux_test[0]: FAILED + Use -p '/as_mut_slice/' to rerun this test only. + push: FAIL (2.14s) + files test/symb_eval/vector/push.good and test/symb_eval/vector/push.out differ; test/symb_eval/vector/push.out contains: + test push/9484b802::f[0]: FAILED failures: - ---- bad_symb2/db5a10b0::crux_test[0] counterexamples ---- + ---- push/9484b802::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/5888ca02::symbolic[0]::symbolic_u8[0] - [Crux] Translation error in crucible/5888ca02::symbolic[0]::symbolic_u8[0]: cannot find static variable alloc$8/3a1fbbbh + [Crux] internal: error: in push/9484b802::f[0] + [Crux] Translation error in push/9484b802::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. - Use -p '/bad_symb2/' to rerun this test only. - override3: FAIL (0.96s) - files test/symb_eval/overrides/override3.good and test/symb_eval/overrides/override3.out differ; test/symb_eval/overrides/override3.out contains: - test override3/5788e0ec::f[0]: FAILED + Use -p '/Output testing.push/' to rerun this test only. + copy_from_slice: FAIL (2.24s) + files test/symb_eval/vector/copy_from_slice.good and test/symb_eval/vector/copy_from_slice.out differ; test/symb_eval/vector/copy_from_slice.out contains: + test copy_from_slice/891db15e::f[0]: FAILED failures: - ---- override3/5788e0ec::f[0] counterexamples ---- + ---- copy_from_slice/891db15e::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/5888ca02::symbolic[0]::symbolic_u8[0] - [Crux] Translation error in crucible/5888ca02::symbolic[0]::symbolic_u8[0]: cannot find static variable alloc$8/3a1fbbbh + [Crux] test/symb_eval/vector/copy_from_slice.rs:7:43: 7:56: error: in copy_from_slice/891db15e::f[0] + [Crux] Translation error in copy_from_slice/891db15e::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. - Use -p '/override3/' to rerun this test only. - override2: FAIL (0.95s) - files test/symb_eval/overrides/override2.good and test/symb_eval/overrides/override2.out differ; test/symb_eval/overrides/override2.out contains: - test override2/463e2c44::f[0]: FAILED + Use -p '/copy_from_slice/' to rerun this test only. + new: FAIL (2.18s) + files test/symb_eval/vector/new.good and test/symb_eval/vector/new.out differ; test/symb_eval/vector/new.out contains: + test new/7c232b74::f[0]: FAILED failures: - ---- override2/463e2c44::f[0] counterexamples ---- + ---- new/7c232b74::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/5888ca02::symbolic[0]::symbolic_u8[0] - [Crux] Translation error in crucible/5888ca02::symbolic[0]::symbolic_u8[0]: cannot find static variable alloc$8/3a1fbbbh + [Crux] internal: error: in new/7c232b74::f[0] + [Crux] Translation error in new/7c232b74::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. - Use -p '/override2/' to rerun this test only. - bad_symb1: FAIL (1.00s) - files test/symb_eval/overrides/bad_symb1.good and test/symb_eval/overrides/bad_symb1.out differ; test/symb_eval/overrides/bad_symb1.out contains: - test bad_symb1/69aa9e9b::crux_test[0]: FAILED + Use -p '$0=="crux-mir.crux symbolic.Output testing.new"' to rerun this test only. + mut: FAIL (2.06s) + files test/symb_eval/vector/mut.good and test/symb_eval/vector/mut.out differ; test/symb_eval/vector/mut.out contains: + test mut/6449c4fb::f[0]: FAILED failures: - ---- bad_symb1/69aa9e9b::crux_test[0] counterexamples ---- + ---- mut/6449c4fb::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/5888ca02::symbolic[0]::symbolic_u8[0] - [Crux] Translation error in crucible/5888ca02::symbolic[0]::symbolic_u8[0]: cannot find static variable alloc$8/3a1fbbbh + [Crux] internal: error: in mut/6449c4fb::f[0] + [Crux] Translation error in mut/6449c4fb::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. - Use -p '/bad_symb1/' to rerun this test only. - override5: FAIL (0.99s) - files test/symb_eval/overrides/override5.good and test/symb_eval/overrides/override5.out differ; test/symb_eval/overrides/override5.out contains: - test override5/ae1ba196::f[0]: FAILED + Use -p '/Output testing.mut/' to rerun this test only. + concat: FAIL (2.13s) + files test/symb_eval/vector/concat.good and test/symb_eval/vector/concat.out differ; test/symb_eval/vector/concat.out contains: + test concat/8ea0a338::f[0]: FAILED failures: - ---- override5/ae1ba196::f[0] counterexamples ---- + ---- concat/8ea0a338::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/5888ca02::symbolic[0]::symbolic_u64[0] - [Crux] Translation error in crucible/5888ca02::symbolic[0]::symbolic_u64[0]: cannot find static variable alloc$6/3a1fbbbh + [Crux] internal: error: in concat/8ea0a338::f[0] + [Crux] Translation error in concat/8ea0a338::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. - Use -p '/override5/' to rerun this test only. - override1: FAIL (0.91s) - files test/symb_eval/overrides/override1.good and test/symb_eval/overrides/override1.out differ; test/symb_eval/overrides/override1.out contains: - test override1/c3978c77::f[0]: FAILED + Use -p '/concat/' to rerun this test only. + split_at: FAIL (2.08s) + files test/symb_eval/vector/split_at.good and test/symb_eval/vector/split_at.out differ; test/symb_eval/vector/split_at.out contains: + test split_at/096c0611::f[0]: FAILED failures: - ---- override1/c3978c77::f[0] counterexamples ---- + ---- split_at/096c0611::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/5888ca02::one[0] - [Crux] panicking::panic, called from crucible/5888ca02::one[0] + [Crux] test/symb_eval/vector/split_at.rs:7:43: 7:56: error: in split_at/096c0611::f[0] + [Crux] Translation error in split_at/096c0611::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. - Use -p '/override1/' to rerun this test only. - override_rust: FAIL (0.95s) - files test/symb_eval/overrides/override_rust.good and test/symb_eval/overrides/override_rust.out differ; test/symb_eval/overrides/override_rust.out contains: - test override_rust/59b18491::crux_test[0]: FAILED + Use -p '/split_at/' to rerun this test only. + replicate: FAIL (2.08s) + files test/symb_eval/vector/replicate.good and test/symb_eval/vector/replicate.out differ; test/symb_eval/vector/replicate.out contains: + test replicate/776d13b1::f[0]: FAILED failures: - ---- override_rust/59b18491::crux_test[0] counterexamples ---- + ---- replicate/776d13b1::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/lib.rs:129:5: 129:42: error: in crucible/5888ca02::override_[0]::_inst28abcb7c380bf102[0] - [Crux] Translation error in crucible/5888ca02::override_[0]::_inst28abcb7c380bf102[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] internal: error: in replicate/776d13b1::f[0] + [Crux] Translation error in replicate/776d13b1::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. - Use -p '/override_rust/' to rerun this test only. - read: FAIL (0.09s) - test/symb_eval/byteorder/read.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/read.rs) - Use -p '/Output testing.read/' to rerun this test only. + Use -p '/replicate/' to rerun this test only. + as_slice: FAIL (2.11s) + files test/symb_eval/vector/as_slice.good and test/symb_eval/vector/as_slice.out differ; test/symb_eval/vector/as_slice.out contains: + test as_slice/a27e71ac::f[0]: FAILED + + failures: + + ---- as_slice/a27e71ac::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in as_slice/a27e71ac::f[0] + [Crux] Translation error in as_slice/a27e71ac::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + + [Crux] Overall status: Invalid. + + Use -p '/as_slice/' to rerun this test only. + array_mut: FAIL (2.06s) + files test/symb_eval/mux/array_mut.good and test/symb_eval/mux/array_mut.out differ; test/symb_eval/mux/array_mut.out contains: + test array_mut/57ba5703::crux_test[0]: FAILED + + failures: + + ---- array_mut/57ba5703::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + [Crux] Overall status: Invalid. + + Use -p '/array_mut/' to rerun this test only. + array: FAIL (2.27s) + files test/symb_eval/mux/array.good and test/symb_eval/mux/array.out differ; test/symb_eval/mux/array.out contains: + test array/0ee8d2fc::crux_test[0]: FAILED + + failures: + + ---- array/0ee8d2fc::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + + [Crux] Overall status: Invalid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.array"' to rerun this test only. + extend_bytes: FAIL (2.17s) + files test/symb_eval/bytes/extend_bytes.good and test/symb_eval/bytes/extend_bytes.out differ; test/symb_eval/bytes/extend_bytes.out contains: + test extend_bytes/3e4d1139::f[0]: FAILED + + failures: + + ---- extend_bytes/3e4d1139::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312a3ac0::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/312a3ac0::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + + [Crux] Overall status: Invalid. + + Use -p '/extend_bytes/' to rerun this test only. + put: FAIL (2.58s) + files test/symb_eval/bytes/put.good and test/symb_eval/bytes/put.out differ; test/symb_eval/bytes/put.out contains: + test put/da785c5a::f[0]: FAILED + + failures: + + ---- put/da785c5a::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312a3ac0::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/312a3ac0::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + + [Crux] Overall status: Invalid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.put"' to rerun this test only. + split_to: FAIL (2.58s) + files test/symb_eval/bytes/split_to.good and test/symb_eval/bytes/split_to.out differ; test/symb_eval/bytes/split_to.out contains: + test split_to/99272bb6::f[0]: FAILED + + failures: + + ---- split_to/99272bb6::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312a3ac0::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/312a3ac0::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + + [Crux] Overall status: Invalid. + + Use -p '/split_to/' to rerun this test only. + new: FAIL (2.16s) + files test/symb_eval/bytes/new.good and test/symb_eval/bytes/new.out differ; test/symb_eval/bytes/new.out contains: + test new/6f028dc4::f[0]: FAILED + + failures: + + ---- new/6f028dc4::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312a3ac0::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/312a3ac0::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + + [Crux] Overall status: Invalid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.new"' to rerun this test only. + split_off: FAIL (2.59s) + files test/symb_eval/bytes/split_off.good and test/symb_eval/bytes/split_off.out differ; test/symb_eval/bytes/split_off.out contains: + test split_off/4bbd6335::f[0]: FAILED + + failures: + + ---- split_off/4bbd6335::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312a3ac0::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/312a3ac0::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + + [Crux] Overall status: Invalid. + + Use -p '/split_off/' to rerun this test only. + sym_len: FAIL (2.72s) + files test/symb_eval/bytes/sym_len.good and test/symb_eval/bytes/sym_len.out differ; test/symb_eval/bytes/sym_len.out contains: + test sym_len/c70884d6::f[0]: FAILED + + failures: + + ---- sym_len/c70884d6::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312a3ac0::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/312a3ac0::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + + [Crux] Overall status: Invalid. + + Use -p '/sym_len/' to rerun this test only. + put_overflow: FAIL (2.56s) + files test/symb_eval/bytes/put_overflow.good and test/symb_eval/bytes/put_overflow.out differ; test/symb_eval/bytes/put_overflow.out contains: + test put_overflow/26c5f05d::f[0]: FAILED + + failures: + + ---- put_overflow/26c5f05d::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312a3ac0::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/312a3ac0::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + + [Crux] Overall status: Invalid. + + Use -p '/put_overflow/' to rerun this test only. + vec_cursor_read: FAIL (2.52s) + test/symb_eval/io/vec_cursor_read.out: withFile: user error (JSON Decoding of test/symb_eval/io/vec_cursor_read.all.mir failed: Error in $.statics[60].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) + Use -p '/vec_cursor_read/' to rerun this test only. + vec_write: FAIL (2.54s) + standalone use of `dyn` is not supported: TyDynamic core/5a1f6960::error[0]::Error[0]::_trait56353a7b840403b4[0] + CallStack (from HasCallStack): + error, called at src/Mir/TransTy.hs:225:25 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/Trans.hs:815:12 in crucible-mir-0.1-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:883:5 in crucible-mir-0.1-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:935:30 in crucible-mir-0.1-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1147:11 in crucible-mir-0.1-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1746:68 in crucible-mir-0.1-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1755:28 in crucible-mir-0.1-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1795:10 in crucible-mir-0.1-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1841:24 in crucible-mir-0.1-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2275:30 in crucible-mir-0.1-inplace:Mir.Trans + transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate + translateMIR, called at src/Mir/Language.hs:255:16 in crux-mir-0.6.0.99-inplace:Mir.Language + Use -p '/vec_write/' to rerun this test only. write: FAIL test/symb_eval/byteorder/write.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/write.rs) Use -p '/Output testing.write/' to rerun this test only. - mux: FAIL (0.96s) + read: FAIL + test/symb_eval/byteorder/read.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/read.rs) + Use -p '/Output testing.read/' to rerun this test only. + slice_mut: FAIL (2.12s) + files test/symb_eval/array/slice_mut.good and test/symb_eval/array/slice_mut.out differ; test/symb_eval/array/slice_mut.out contains: + test slice_mut/9abd2b5d::crux_test[0]: FAILED + + failures: + + ---- slice_mut/9abd2b5d::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in slice_mut/9abd2b5d::crux_test[0] + [Crux] Translation error in slice_mut/9abd2b5d::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/6eda2caa::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/6eda2caa::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/6eda2caa::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) + + [Crux] Overall status: Invalid. + + Use -p '/slice_mut/' to rerun this test only. + basic: FAIL (2.10s) + files test/symb_eval/array/basic.good and test/symb_eval/array/basic.out differ; test/symb_eval/array/basic.out contains: + test basic/79cad869::crux_test[0]: FAILED + + failures: + + ---- basic/79cad869::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in basic/79cad869::crux_test[0] + [Crux] Translation error in basic/79cad869::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/6eda2caa::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/6eda2caa::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/6eda2caa::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/6eda2caa::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) + + [Crux] Overall status: Invalid. + + Use -p '/Output testing.basic/' to rerun this test only. + slice: FAIL (2.20s) + files test/symb_eval/array/slice.good and test/symb_eval/array/slice.out differ; test/symb_eval/array/slice.out contains: + test slice/0e7bff2f::crux_test[0]: FAILED + + failures: + + ---- slice/0e7bff2f::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in slice/0e7bff2f::crux_test[0] + [Crux] Translation error in slice/0e7bff2f::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/6eda2caa::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/6eda2caa::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/6eda2caa::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) + + [Crux] Overall status: Invalid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.slice"' to rerun this test only. + mux_slice: FAIL (2.16s) + files test/symb_eval/array/mux_slice.good and test/symb_eval/array/mux_slice.out differ; test/symb_eval/array/mux_slice.out contains: + test mux_slice/508aacc9::crux_test[0]: FAILED + + failures: + + ---- mux_slice/508aacc9::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in mux_slice/508aacc9::crux_test[0] + [Crux] Translation error in mux_slice/508aacc9::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/6eda2caa::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/6eda2caa::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/6eda2caa::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) + + [Crux] Overall status: Invalid. + + Use -p '/mux_slice/' to rerun this test only. + mux: FAIL (2.12s) files test/symb_eval/enum/mux.good and test/symb_eval/enum/mux.out differ; test/symb_eval/enum/mux.out contains: - test mux/aa4fe2e6::test[0]: FAILED + test mux/2bd00921::test[0]: FAILED failures: - ---- mux/aa4fe2e6::test[0] counterexamples ---- + ---- mux/2bd00921::test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/5888ca02::symbolic[0]::symbolic_u8[0] - [Crux] Translation error in crucible/5888ca02::symbolic[0]::symbolic_u8[0]: cannot find static variable alloc$8/3a1fbbbh + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.mux"' to rerun this test only. - split_off: FAIL (1.42s) - files test/symb_eval/bytes/split_off.good and test/symb_eval/bytes/split_off.out differ; test/symb_eval/bytes/split_off.out contains: - test split_off/4ecfcbdd::f[0]: FAILED + uninit_read: FAIL (2.23s) + files test/symb_eval/alloc/uninit_read.good and test/symb_eval/alloc/uninit_read.out differ; test/symb_eval/alloc/uninit_read.out contains: + test uninit_read/e9c1be4c::crux_test[0]: FAILED failures: - ---- split_off/4ecfcbdd::f[0] counterexamples ---- + ---- uninit_read/e9c1be4c::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/e9c1be4c::crux_test[0] + [Crux] Attempted to read uninitialized vector index [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/vector.rs:81:9: 81:44: error: in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0] - [Crux] Translation error in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/e9c1be4c::crux_test[0] + [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. - Use -p '/split_off/' to rerun this test only. - new: FAIL (1.00s) - files test/symb_eval/bytes/new.good and test/symb_eval/bytes/new.out differ; test/symb_eval/bytes/new.out contains: - test new/b6888249::f[0]: FAILED + Use -p '/uninit_read/' to rerun this test only. + valid_read: FAIL (2.19s) + files test/symb_eval/alloc/valid_read.good and test/symb_eval/alloc/valid_read.out differ; test/symb_eval/alloc/valid_read.out contains: + test valid_read/b81b29bc::crux_test[0]: FAILED failures: - ---- new/b6888249::f[0] counterexamples ---- + ---- valid_read/b81b29bc::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/vector.rs:81:9: 81:44: error: in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0] - [Crux] Translation error in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/int_macros.rs:460:54: 460:57 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:293:5: 298:27: error: in core/5a1f6960::num[0]::{impl#6}[0]::unchecked_add[0] + [Crux] Translation error in core/5a1f6960::num[0]::{impl#6}[0]::unchecked_add[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_inste14122fababf7d68[0] [Crux] Overall status: Invalid. - Use -p '$0=="crux-mir.crux symbolic.Output testing.new"' to rerun this test only. - split_to: FAIL (1.35s) - files test/symb_eval/bytes/split_to.good and test/symb_eval/bytes/split_to.out differ; test/symb_eval/bytes/split_to.out contains: - test split_to/437fab7e::f[0]: FAILED + Use -p '/valid_read/' to rerun this test only. + out_of_bounds: FAIL (2.17s) + files test/symb_eval/alloc/out_of_bounds.good and test/symb_eval/alloc/out_of_bounds.out differ; test/symb_eval/alloc/out_of_bounds.out contains: + test out_of_bounds/a4641c13::crux_test[0]: FAILED failures: - ---- split_to/437fab7e::f[0] counterexamples ---- + ---- out_of_bounds/a4641c13::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/vector.rs:81:9: 81:44: error: in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0] - [Crux] Translation error in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/int_macros.rs:460:54: 460:57 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:293:5: 298:27: error: in core/5a1f6960::num[0]::{impl#6}[0]::unchecked_add[0] + [Crux] Translation error in core/5a1f6960::num[0]::{impl#6}[0]::unchecked_add[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_inste14122fababf7d68[0] [Crux] Overall status: Invalid. - Use -p '/split_to/' to rerun this test only. - put_overflow: FAIL (1.40s) - files test/symb_eval/bytes/put_overflow.good and test/symb_eval/bytes/put_overflow.out differ; test/symb_eval/bytes/put_overflow.out contains: - test put_overflow/bfb2f26d::f[0]: FAILED + Use -p '/out_of_bounds/' to rerun this test only. + zero_length: FAIL (2.05s) + files test/symb_eval/alloc/zero_length.good and test/symb_eval/alloc/zero_length.out differ; test/symb_eval/alloc/zero_length.out contains: + test zero_length/a53db669::crux_test[0]: ok + + [Crux] Overall status: Valid. + + Use -p '/zero_length/' to rerun this test only. + downcast: FAIL (2.06s) + files test/symb_eval/any/downcast.good and test/symb_eval/any/downcast.out differ; test/symb_eval/any/downcast.out contains: + test downcast/cc755b1e::crux_test[0]: returned 1, ok + + [Crux] Overall status: Valid. + + Use -p '$0=="crux-mir.crux symbolic.Output testing.downcast"' to rerun this test only. + downcast_fail: FAIL (2.11s) + files test/symb_eval/any/downcast_fail.good and test/symb_eval/any/downcast_fail.out differ; test/symb_eval/any/downcast_fail.out contains: + test downcast_fail/adccf677::crux_test[0]: FAILED failures: - ---- put_overflow/bfb2f26d::f[0] counterexamples ---- + ---- downcast_fail/adccf677::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/vector.rs:81:9: 81:44: error: in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0] - [Crux] Translation error in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] test/symb_eval/any/downcast_fail.rs:10:14: 10:15: error: in downcast_fail/adccf677::crux_test[0] + [Crux] failed to downcast Any as BVRepr 32 [Crux] Overall status: Invalid. - Use -p '/put_overflow/' to rerun this test only. - sym_len: FAIL (1.37s) - files test/symb_eval/bytes/sym_len.good and test/symb_eval/bytes/sym_len.out differ; test/symb_eval/bytes/sym_len.out contains: - test sym_len/d901b6ca::f[0]: FAILED + Use -p '/downcast_fail/' to rerun this test only. + conditional: FAIL (0.05s) + test/symb_eval/any/conditional.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/any/conditional.rs) + Use -p '/conditional/' to rerun this test only. + literals: FAIL (2.10s) + files test/symb_eval/bitvector/literals.good and test/symb_eval/bitvector/literals.out differ; test/symb_eval/bitvector/literals.out contains: + test literals/bc9c2ca9::crux_test[0]: FAILED failures: - ---- sym_len/d901b6ca::f[0] counterexamples ---- + ---- literals/bc9c2ca9::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/vector.rs:81:9: 81:44: error: in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0] - [Crux] Translation error in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:91:29: 91:30 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:125:1: 125:53: error: in crucible/6eda2caa::bitvector[0]::{impl#27}[0]::from[0]::_inst9ea2f935eda83dab[0] + [Crux] Translation error in crucible/6eda2caa::bitvector[0]::{impl#27}[0]::from[0]::_inst9ea2f935eda83dab[0]: BUG: invalid arguments to bv_convert: [$38: AnyRepr] [Crux] Overall status: Invalid. - Use -p '/sym_len/' to rerun this test only. - put: FAIL (1.33s) - files test/symb_eval/bytes/put.good and test/symb_eval/bytes/put.out differ; test/symb_eval/bytes/put.out contains: - test put/5170e38f::f[0]: FAILED + Use -p '/literals/' to rerun this test only. + arith: FAIL (2.16s) + files test/symb_eval/bitvector/arith.good and test/symb_eval/bitvector/arith.out differ; test/symb_eval/bitvector/arith.out contains: + test arith/58d00074::crux_test[0]: FAILED failures: - ---- put/5170e38f::f[0] counterexamples ---- + ---- arith/58d00074::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/vector.rs:81:9: 81:44: error: in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0] - [Crux] Translation error in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u64[0] [Crux] Overall status: Invalid. - Use -p '$0=="crux-mir.crux symbolic.Output testing.put"' to rerun this test only. - extend_bytes: FAIL (0.94s) - files test/symb_eval/bytes/extend_bytes.good and test/symb_eval/bytes/extend_bytes.out differ; test/symb_eval/bytes/extend_bytes.out contains: - test extend_bytes/d25848e0::f[0]: FAILED + Use -p '/Output testing.arith/' to rerun this test only. + leading_zeros: FAIL (2.16s) + files test/symb_eval/bitvector/leading_zeros.good and test/symb_eval/bitvector/leading_zeros.out differ; test/symb_eval/bitvector/leading_zeros.out contains: + test leading_zeros/5d113916::crux_test[0]: FAILED failures: - ---- extend_bytes/d25848e0::f[0] counterexamples ---- + ---- leading_zeros/5d113916::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/jlamar/prj/chainsaw/crucible/crux-mir/lib/core/src/macros/mod.rs:709:24: 709:45 !/home/jlamar/prj/chainsaw/crucible/crux-mir/lib/crucible/vector.rs:81:9: 81:44: error: in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0] - [Crux] Translation error in crucible/5888ca02::vector[0]::{impl#1}[0]::replicate[0]::_instaddce72e1232152c[0]: cannot find static variable alloc$0/3a1fbbbh + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u64[0] [Crux] Overall status: Invalid. - Use -p '/extend_bytes/' to rerun this test only. - basic: \ No newline at end of file + Use -p '/leading_zeros/' to rerun this test only. + overflowing_sub: FAIL (2.07s) + files test/symb_eval/bitvector/overflowing_sub.good and test/symb_eval/bitvector/overflowing_sub.out differ; test/symb_eval/bitvector/overflowing_sub.out contains: + test overflowing_sub/0e7bfc8b::crux_test[0]: FAILED + + failures: + + ---- overflowing_sub/0e7bfc8b::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u64[0] + + [Crux] Overall status: Invalid. + + Use -p '/overflowing_sub/' to rerun this test only. + cmp: FAIL (2.14s) + files test/symb_eval/bitvector/cmp.good and test/symb_eval/bitvector/cmp.out differ; test/symb_eval/bitvector/cmp.out contains: + test cmp/de4d4043::crux_test[0]: FAILED + + failures: + + ---- cmp/de4d4043::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u64[0] + + [Crux] Overall status: Invalid. + + Use -p '/Output testing.cmp/' to rerun this test only. + symbolic: FAIL (2.00s) + files test/symb_eval/bitvector/symbolic.good and test/symb_eval/bitvector/symbolic.out differ; test/symb_eval/bitvector/symbolic.out contains: + test symbolic/6a0ffa6e::crux_test[0]: FAILED + + failures: + + ---- symbolic/6a0ffa6e::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::bitvector[0]::make_symbolic_256[0] + [Crux] panicking::panic, called from crucible/6eda2caa::bitvector[0]::make_symbolic_256[0] + + [Crux] Overall status: Invalid. + + Use -p '/Output testing.symbolic/' to rerun this test only. + from_to: FAIL (2.04s) + files test/symb_eval/bitvector/from_to.good and test/symb_eval/bitvector/from_to.out differ; test/symb_eval/bitvector/from_to.out contains: + test from_to/dcb07b82::crux_test[0]: FAILED + + failures: + + ---- from_to/dcb07b82::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:103:29: 103:30 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:125:1: 125:53: error: in crucible/6eda2caa::bitvector[0]::{impl#21}[0]::from[0]::_inst9ea2f935eda83dab[0] + [Crux] Translation error in crucible/6eda2caa::bitvector[0]::{impl#21}[0]::from[0]::_inst9ea2f935eda83dab[0]: BUG: invalid arguments to bv_convert: [$34: BVRepr 64] + + [Crux] Overall status: Invalid. + + Use -p '/from_to/' to rerun this test only. + clone: FAIL (2.23s) + files test/symb_eval/vec/clone.good and test/symb_eval/vec/clone.out differ; test/symb_eval/vec/clone.out contains: + test clone/dfc182b4::f[0]: FAILED + + failures: + + ---- clone/dfc182b4::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + + [Crux] Overall status: Invalid. + + Use -p '/Output testing.clone/' to rerun this test only. + sort_by_key: FAIL (3.08s) + files test/symb_eval/vec/sort_by_key.good and test/symb_eval/vec/sort_by_key.out differ; test/symb_eval/vec/sort_by_key.out contains: + test sort_by_key/ea977c8a::f[0]: FAILED + + failures: + + ---- sort_by_key/ea977c8a::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + + [Crux] Overall status: Invalid. + + Use -p '/sort_by_key/' to rerun this test only. + into_iter: FAIL (2.90s) + files test/symb_eval/vec/into_iter.good and test/symb_eval/vec/into_iter.out differ; test/symb_eval/vec/into_iter.out contains: + test into_iter/e3aefb85::f[0]: FAILED + + failures: + + ---- into_iter/e3aefb85::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + + [Crux] Overall status: Invalid. + + Use -p '/into_iter/' to rerun this test only. + macro: FAIL (2.30s) + files test/symb_eval/vec/macro.good and test/symb_eval/vec/macro.out differ; test/symb_eval/vec/macro.out contains: + test macro/e7284038::f[0]: FAILED + + failures: + + ---- macro/e7284038::f[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + + [Crux] Overall status: Invalid. + + Use -p '/Output testing.macro/' to rerun this test only. + construct: FAIL (2.17s) + files test/symb_eval/sym_bytes/construct.good and test/symb_eval/sym_bytes/construct.out differ; test/symb_eval/sym_bytes/construct.out contains: + test construct/63aa86a0::crux_test[0]: FAILED + + failures: + + ---- construct/63aa86a0::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::sym_bytes[0]::{impl#0}[0]::zeroed[0] + [Crux] Translation error in crucible/6eda2caa::sym_bytes[0]::{impl#0}[0]::zeroed[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::array[0]::Array[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::array[0]::Array[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::array[0]::Array[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::array[0]::Array[0] (Substs [TyUint B8]), _varIsZST = False}) + + [Crux] Overall status: Invalid. + + Use -p '/construct/' to rerun this test only. + deserialize: FAIL (2.23s) + files test/symb_eval/sym_bytes/deserialize.good and test/symb_eval/sym_bytes/deserialize.out differ; test/symb_eval/sym_bytes/deserialize.out contains: + test deserialize/40308d9e::crux_test[0]: FAILED + + failures: + + ---- deserialize/40308d9e::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] internal: error: in crucible/6eda2caa::array[0]::symbolic[0]::_instaddce72e1232152c[0] + [Crux] panicking::panic_fmt, called from crucible/6eda2caa::array[0]::symbolic[0]::_instaddce72e1232152c[0] + + [Crux] Overall status: Invalid. + + Use -p '/deserialize/' to rerun this test only. + crux coverage + Output testing + coverage_dual: warning: trailing semicolon in macro used in expression position + --> src/main.rs:45:46 + | +45 | return Err(format!($($args)*).into()); + | ^ +... +309 | _ => die!("unknown tag {:?} for branch", tag), + | ---------------------------------------- in this macro invocation + | + = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! + = note: for more information, see issue #79813 + = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default + = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) + +OK (8.12s) + coverage: warning: trailing semicolon in macro used in expression position + --> src/main.rs:45:46 + | +45 | return Err(format!($($args)*).into()); + | ^ +... +309 | _ => die!("unknown tag {:?} for branch", tag), + | ---------------------------------------- in this macro invocation + | + = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! + = note: for more information, see issue #79813 + = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default + = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) + +FAIL (2.28s) + files test/coverage/coverage.good and test/coverage/coverage.out differ; test/coverage/coverage.out contains: + warning: branch condition never has value true + ┌─ lib/core/src/fmt/mod.rs:400:12 + │ + 400 │ if pieces.len() < args.len() || pieces.len() > args.len() + 1 { + │ ^^^^^^^^^^^^^^^^^^^^^^^^^ + + warning: branch condition never has value true + ┌─ lib/core/src/fmt/mod.rs:400:12 + │ + 400 │ if pieces.len() < args.len() || pieces.len() > args.len() + 1 { + │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + + Use -p '$0=="crux-mir.crux coverage.Output testing.coverage"' to rerun this test only. + coverage_try: warning: trailing semicolon in macro used in expression position + --> src/main.rs:45:46 + | +45 | return Err(format!($($args)*).into()); + | ^ +... +309 | _ => die!("unknown tag {:?} for branch", tag), + | ---------------------------------------- in this macro invocation + | + = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! + = note: for more information, see issue #79813 + = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default + = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) + +FAIL (2.32s) + files test/coverage/coverage_try.good and test/coverage/coverage_try.out differ; test/coverage/coverage_try.out contains: + warning: branch condition never has value true + ┌─ lib/core/src/fmt/mod.rs:400:12 + │ + 400 │ if pieces.len() < args.len() || pieces.len() > args.len() + 1 { + │ ^^^^^^^^^^^^^^^^^^^^^^^^^ + + warning: branch condition never has value true + ┌─ lib/core/src/fmt/mod.rs:400:12 + │ + 400 │ if pieces.len() < args.len() || pieces.len() > args.len() + 1 { + │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + + Use -p '/coverage_try/' to rerun this test only. + coverage_cond: warning: trailing semicolon in macro used in expression position + --> src/main.rs:45:46 + | +45 | return Err(format!($($args)*).into()); + | ^ +... +309 | _ => die!("unknown tag {:?} for branch", tag), + | ---------------------------------------- in this macro invocation + | + = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! + = note: for more information, see issue #79813 + = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default + = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) + +OK (2.04s) + coverage_loop: warning: trailing semicolon in macro used in expression position + --> src/main.rs:45:46 + | +45 | return Err(format!($($args)*).into()); + | ^ +... +309 | _ => die!("unknown tag {:?} for branch", tag), + | ---------------------------------------- in this macro invocation + | + = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! + = note: for more information, see issue #79813 + = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default + = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) + +FAIL (2.19s) + files test/coverage/coverage_loop.good and test/coverage/coverage_loop.out differ; test/coverage/coverage_loop.out contains: + warning: branch condition never has value false + ┌─ /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/iter/range.rs:621:12 + │ + 621 │ if self.start < self.end { + │ ^^^^^^^^^^^^^^^^^^^^^ + + + Use -p '/coverage_loop/' to rerun this test only. + coverage_shortcircuit: warning: trailing semicolon in macro used in expression position + --> src/main.rs:45:46 + | +45 | return Err(format!($($args)*).into()); + | ^ +... +309 | _ => die!("unknown tag {:?} for branch", tag), + | ---------------------------------------- in this macro invocation + | + = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! + = note: for more information, see issue #79813 + = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default + = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) + +FAIL (2.21s) + files test/coverage/coverage_shortcircuit.good and test/coverage/coverage_shortcircuit.out differ; test/coverage/coverage_shortcircuit.out contains: + warning: branch condition never has value true + ┌─ lib/core/src/fmt/mod.rs:400:12 + │ + 400 │ if pieces.len() < args.len() || pieces.len() > args.len() + 1 { + │ ^^^^^^^^^^^^^^^^^^^^^^^^^ + + warning: branch condition never has value true + ┌─ lib/core/src/fmt/mod.rs:400:12 + │ + 400 │ if pieces.len() < args.len() || pieces.len() > args.len() + 1 { + │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + + Use -p '/coverage_shortcircuit/' to rerun this test only. + coverage_mono: warning: trailing semicolon in macro used in expression position + --> src/main.rs:45:46 + | +45 | return Err(format!($($args)*).into()); + | ^ +... +309 | _ => die!("unknown tag {:?} for branch", tag), + | ---------------------------------------- in this macro invocation + | + = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! + = note: for more information, see issue #79813 + = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default + = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) + +OK (1.99s) + coverage_match: warning: trailing semicolon in macro used in expression position + --> src/main.rs:45:46 + | +45 | return Err(format!($($args)*).into()); + | ^ +... +309 | _ => die!("unknown tag {:?} for branch", tag), + | ---------------------------------------- in this macro invocation + | + = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! + = note: for more information, see issue #79813 + = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default + = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) + +FAIL (2.13s) + files test/coverage/coverage_match.good and test/coverage/coverage_match.out differ; test/coverage/coverage_match.out contains: + warning: branch condition never has value true + ┌─ lib/core/src/fmt/mod.rs:400:12 + │ + 400 │ if pieces.len() < args.len() || pieces.len() > args.len() + 1 { + │ ^^^^^^^^^^^^^^^^^^^^^^^^^ + + warning: branch condition never has value true + ┌─ lib/core/src/fmt/mod.rs:400:12 + │ + 400 │ if pieces.len() < args.len() || pieces.len() > args.len() + 1 { + │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + + Use -p '/coverage_match/' to rerun this test only. + coverage_macro: warning: trailing semicolon in macro used in expression position + --> src/main.rs:45:46 + | +45 | return Err(format!($($args)*).into()); + | ^ +... +309 | _ => die!("unknown tag {:?} for branch", tag), + | ---------------------------------------- in this macro invocation + | + = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! + = note: for more information, see issue #79813 + = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default + = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) + +FAIL (2.13s) + files test/coverage/coverage_macro.good and test/coverage/coverage_macro.out differ; test/coverage/coverage_macro.out contains: + warning: branch condition never has value true + ┌─ lib/core/src/fmt/mod.rs:400:12 + │ + 400 │ if pieces.len() < args.len() || pieces.len() > args.len() + 1 { + │ ^^^^^^^^^^^^^^^^^^^^^^^^^ + + warning: branch condition never has value true + ┌─ lib/core/src/fmt/mod.rs:400:12 + │ + 400 │ if pieces.len() < args.len() || pieces.len() > args.len() + 1 { + │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + + Use -p '/coverage_macro/' to rerun this test only. + +181 out of 339 tests failed (747.81s) From c1f8060e74e7f70b0e53157d1531f4ce79f6d6e4 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Tue, 2 May 2023 16:13:36 -0400 Subject: [PATCH 022/114] crucible-mir: Translate all ConstTuples, not just unit tuples --- crucible-mir/src/Mir/Trans.hs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index 1fa0efd5c..a481f4a7c 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -187,6 +187,11 @@ transConstVal _ty (Some C.UnitRepr) (M.ConstFunction _did) = return $ MirExp C.UnitRepr $ S.app E.EmptyApp transConstVal _ty (Some C.UnitRepr) (M.ConstTuple []) = return $ MirExp C.UnitRepr $ S.app E.EmptyApp +transConstVal (M.TyTuple tys) (Some (C.StructRepr tprs)) (M.ConstTuple vals) = do + vals' <- zipWith3M + (\ty tpr val -> transConstVal ty tpr val) + tys (toListFC Some tprs) vals + return $ buildTuple vals' transConstVal _ty (Some (C.RealValRepr)) (M.ConstFloat (M.FloatLit _ str)) = case reads str of @@ -240,6 +245,13 @@ transConstVal ty (Some (MirReferenceRepr tpr)) init = do transConstVal ty tp cv = mirFail $ "fail or unimp constant: " ++ show ty ++ " (" ++ show tp ++ ") " ++ show cv +-- Taken from GHC's source code, which is BSD-3 licensed. +zipWith3M :: Monad m => (a -> b -> c -> m d) -> [a] -> [b] -> [c] -> m [d] +{-# INLINE zipWith3M #-} +-- Inline so that fusion with 'zipWith3' and 'sequenceA' has a chance to fire. +-- See Note [Inline @zipWithNM@ functions] in GHC.Utils.Monad in the +-- GHC source code. +zipWith3M f xs ys zs = sequenceA (zipWith3 f xs ys zs) typedVarInfo :: HasCallStack => Text -> C.TypeRepr tp -> MirGenerator h s ret (VarInfo s tp) typedVarInfo name tpr = do From 0bba133b2262c713581ca613d96cae73ce5f32da Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 3 May 2023 13:15:33 -0400 Subject: [PATCH 023/114] crucible-mir: Translate constant tuples --- crucible-mir/src/Mir/Trans.hs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index a481f4a7c..09e8fc3e0 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -188,10 +188,16 @@ transConstVal _ty (Some C.UnitRepr) (M.ConstFunction _did) = transConstVal _ty (Some C.UnitRepr) (M.ConstTuple []) = return $ MirExp C.UnitRepr $ S.app E.EmptyApp transConstVal (M.TyTuple tys) (Some (C.StructRepr tprs)) (M.ConstTuple vals) = do + col <- use $ cs . collection vals' <- zipWith3M - (\ty tpr val -> transConstVal ty tpr val) + (\ty (Some tpr) val -> + case tpr of + C.MaybeRepr valTpr -> do + transConstVal ty (Some valTpr) val + _ -> + mirFail $ "transConstVal (ConstTuple): expected tuple field to have MaybeType, but got " ++ show tpr) tys (toListFC Some tprs) vals - return $ buildTuple vals' + return $ buildTupleMaybe col tys $ map Just vals' transConstVal _ty (Some (C.RealValRepr)) (M.ConstFloat (M.FloatLit _ str)) = case reads str of @@ -2322,6 +2328,7 @@ transStatics colState halloc = do case testEquality repr constty' of Just Refl -> G.writeGlobal g constval' Nothing -> error $ "BUG: invalid type for constant initializer " ++ fmt staticName + ++ ", expected " ++ show repr ++ ", got " ++ show constty' | Just (MirHandle _ _ (handle :: FH.FnHandle init ret)) <- Map.lookup staticName hmap From b6397902b957d026edea9bf5285dc147ed81189d Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 3 May 2023 13:23:11 -0400 Subject: [PATCH 024/114] crucible-mir: Add seqcst variants of some atomic intrinsics --- crucible-mir/src/Mir/TransCustom.hs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crucible-mir/src/Mir/TransCustom.hs b/crucible-mir/src/Mir/TransCustom.hs index a0ae5ece1..63d5ebe0a 100644 --- a/crucible-mir/src/Mir/TransCustom.hs +++ b/crucible-mir/src/Mir/TransCustom.hs @@ -1433,7 +1433,7 @@ reallocate = (["crucible", "alloc", "reallocate"], \substs -> case substs of -- etc., all with the same `rhs`. makeAtomicIntrinsics :: Text -> [Text] -> CustomRHS -> [(ExplodedDefId, CustomRHS)] makeAtomicIntrinsics name variants rhs = - [(["core", "intrinsics", "", "atomic_" <> name <> suffix], rhs) + [(["core", "intrinsics", "{extern}", "atomic_" <> name <> suffix], rhs) | suffix <- "" : map ("_" <>) variants] atomic_store_impl :: CustomRHS @@ -1499,12 +1499,12 @@ makeAtomicRMW :: MirGenerator h s ret (R.Expr MIR s (C.BVType w))) -> [(ExplodedDefId, CustomRHS)] makeAtomicRMW name rmw = - makeAtomicIntrinsics (Text.pack name) ["acq", "rel", "acqrel", "relaxed"] $ + makeAtomicIntrinsics (Text.pack name) ["acq", "rel", "acqrel", "relaxed", "seqcst"] $ atomic_rmw_impl name rmw atomic_funcs = - makeAtomicIntrinsics "store" ["rel", "relaxed"] atomic_store_impl ++ - makeAtomicIntrinsics "load" ["acq", "relaxed"] atomic_load_impl ++ + makeAtomicIntrinsics "store" ["rel", "relaxed", "seqcst"] atomic_store_impl ++ + makeAtomicIntrinsics "load" ["acq", "relaxed", "seqcst"] atomic_load_impl ++ makeAtomicIntrinsics "cxchg" compareExchangeVariants atomic_cxchg_impl ++ makeAtomicIntrinsics "cxchgweak" compareExchangeVariants atomic_cxchg_impl ++ makeAtomicIntrinsics "fence" fenceVariants atomic_fence_impl ++ @@ -1526,7 +1526,7 @@ atomic_funcs = where compareExchangeVariants = ["acq", "rel", "acqrel", "relaxed", "acq_failrelaxed", "acqrel_failrelaxed", "failrelaxed", "failacq"] - fenceVariants = ["acq", "rel", "acqrel"] + fenceVariants = ["acq", "rel", "acqrel", "seqcst"] -------------------------------------------------------------------------------------------------------------------------- From 07add62ab68cb30412a69340a04ea8168effd0db Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 3 May 2023 13:31:20 -0400 Subject: [PATCH 025/114] crucible-mir: Add some more {extern}s to custom ops --- crucible-mir/src/Mir/TransCustom.hs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/crucible-mir/src/Mir/TransCustom.hs b/crucible-mir/src/Mir/TransCustom.hs index 63d5ebe0a..0151a00c0 100644 --- a/crucible-mir/src/Mir/TransCustom.hs +++ b/crucible-mir/src/Mir/TransCustom.hs @@ -756,13 +756,13 @@ makeSaturatingArith name bop = saturating_add :: (ExplodedDefId, CustomRHS) saturating_add = - ( ["core","intrinsics", "", "saturating_add"] + ( ["core","intrinsics", "{extern}", "saturating_add"] , makeSaturatingArith "saturating_add" Add ) saturating_sub :: (ExplodedDefId, CustomRHS) saturating_sub = - ( ["core","intrinsics", "", "saturating_sub"] + ( ["core","intrinsics", "{extern}", "saturating_sub"] , makeSaturatingArith "saturating_sub" Sub ) @@ -787,43 +787,43 @@ makeUncheckedArith name bop = unchecked_add :: (ExplodedDefId, CustomRHS) unchecked_add = - ( ["core","intrinsics", "", "unchecked_add"] + ( ["core","intrinsics", "{extern}", "unchecked_add"] , makeUncheckedArith "unchecked_add" Add ) unchecked_sub :: (ExplodedDefId, CustomRHS) unchecked_sub = - ( ["core","intrinsics", "", "unchecked_sub"] + ( ["core","intrinsics", "{extern}", "unchecked_sub"] , makeUncheckedArith "unchecked_sub" Sub ) unchecked_mul :: (ExplodedDefId, CustomRHS) unchecked_mul = - ( ["core","intrinsics", "", "unchecked_mul"] + ( ["core","intrinsics", "{extern}", "unchecked_mul"] , makeUncheckedArith "unchecked_mul" Mul ) unchecked_div :: (ExplodedDefId, CustomRHS) unchecked_div = - ( ["core","intrinsics", "", "unchecked_div"] + ( ["core","intrinsics", "{extern}", "unchecked_div"] , makeUncheckedArith "unchecked_div" Div ) unchecked_rem :: (ExplodedDefId, CustomRHS) unchecked_rem = - ( ["core","intrinsics", "", "unchecked_rem"] + ( ["core","intrinsics", "{extern}", "unchecked_rem"] , makeUncheckedArith "unchecked_rem" Rem ) unchecked_shl :: (ExplodedDefId, CustomRHS) unchecked_shl = - ( ["core","intrinsics", "", "unchecked_shl"] + ( ["core","intrinsics", "{extern}", "unchecked_shl"] , makeUncheckedArith "unchecked_shl" Shl ) unchecked_shr :: (ExplodedDefId, CustomRHS) unchecked_shr = - ( ["core","intrinsics", "", "unchecked_shr"] + ( ["core","intrinsics", "{extern}", "unchecked_shr"] , makeUncheckedArith "unchecked_shr" Shr ) From b57531cbaac8a9546d339b1c9c77fa61d57f0717 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 4 May 2023 13:04:11 -0400 Subject: [PATCH 026/114] Regenerate test_output.log --- crux-mir/test_output.log | 1791 ++++++++++++++++++-------------------- 1 file changed, 843 insertions(+), 948 deletions(-) diff --git a/crux-mir/test_output.log b/crux-mir/test_output.log index 9c39227b4..2bb7467ce 100644 --- a/crux-mir/test_output.log +++ b/crux-mir/test_output.log @@ -1,123 +1,104 @@ -Build profile: -w ghc-9.2.7 -O1 -In order, the following will be built (use -v for more details): - - crux-mir-0.6.0.99 (test:test) (file /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/dist-newstyle/build/x86_64-linux/ghc-9.2.7/crux-mir-0.6.0.99/cache/build changed) -Preprocessing test suite 'test' for crux-mir-0.6.0.99.. -Building test suite 'test' for crux-mir-0.6.0.99.. -Linking /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/dist-newstyle/build/x86_64-linux/ghc-9.2.7/crux-mir-0.6.0.99/t/test/build/test/test ... crux-mir crux concrete refs - promoted_imm: OK (2.19s) - Compiling and running oracle program (0.19s) - Oracle output: 0 (2.00s) + promoted_imm: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: 0 Crux output: 0 - mut_raw: OK (2.15s) + mut_raw: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 123 (1.98s) + Oracle output: 123 Crux output: 123 - mut_nested: OK (2.17s) + mut_nested: OK (0.18s) Compiling and running oracle program (0.17s) - Oracle output: () (2.00s) + Oracle output: () (0.01s) Crux output: () - fn_ptr_mut: OK (2.19s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (2.03s) + fn_ptr_mut: OK (0.18s) + Compiling and running oracle program (0.18s) + Oracle output: 1 Crux output: 1 - never: OK (2.17s) + never: OK (0.16s) Compiling and running oracle program (0.16s) - Oracle output: 1 (2.00s) + Oracle output: 1 Crux output: 1 - mut_ref: OK (2.16s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (2.00s) + mut_ref: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: 1 Crux output: 1 - temp: OK (2.22s) + temp: OK (0.18s) Compiling and running oracle program (0.18s) - Oracle output: 1 (2.04s) + Oracle output: 1 Crux output: 1 - mut_arg: OK (2.15s) + mut_arg: OK (0.18s) Compiling and running oracle program (0.18s) - Oracle output: 1 (1.97s) + Oracle output: 1 Crux output: 1 - fn_ptr: OK (2.22s) - Compiling and running oracle program (0.21s) - Oracle output: 1 (2.01s) + fn_ptr: OK (0.17s) + Compiling and running oracle program (0.17s) + Oracle output: 1 Crux output: 1 - promoted_mut: OK (2.18s) - Compiling and running oracle program (0.19s) - Oracle output: 0 (2.00s) + promoted_mut: OK (0.17s) + Compiling and running oracle program (0.17s) + Oracle output: 0 Crux output: 0 - never_mut: OK (2.15s) + never_mut: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 1 (1.98s) + Oracle output: 1 Crux output: 1 - mut_tuple_field: FAIL (2.22s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.05s) - Crux output: - failures: - - ---- mut_tuple_field/ef64915e::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in static_initializer - [Crux] Translation error in the static initializer: fail or unimp constant: TyTuple [TyInt B32,TyInt B32] (StructRepr [MaybeRepr (BVRepr 32), MaybeRepr (BVRepr 32)]) ConstTuple [ConstInt (I32 3),ConstInt (I32 4)] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/mut_tuple_field/' to rerun this test only. - imm_raw: OK (2.16s) - Compiling and running oracle program (0.18s) - Oracle output: 123 (1.98s) + mut_tuple_field: OK (0.19s) + Compiling and running oracle program (0.17s) + Oracle output: () (0.02s) + Crux output: () + imm_raw: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: 123 Crux output: 123 - static_mut: OK (2.17s) + static_mut: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 2 (2.00s) + Oracle output: 2 Crux output: 2 - imm_ref: OK (2.13s) - Compiling and running oracle program (0.18s) - Oracle output: 123 (1.95s) - Crux output: 123 - imm_arg: OK (2.14s) + imm_ref: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 1 (1.97s) + Oracle output: 123 + Crux output: 123 + imm_arg: OK (0.18s) + Compiling and running oracle program (0.18s) + Oracle output: 1 Crux output: 1 hash_map - insert_multi: FAIL (3.17s) + insert_multi: FAIL (0.98s) Compiling and running oracle program (0.22s) - Oracle output: 100 (2.95s) + Oracle output: 100 (0.77s) user error (JSON Decoding of test/conc_eval/hash_map/insert_multi.all.mir failed: Error in $.fns[1463].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) Use -p '/insert_multi/' to rerun this test only. - insert_remove: FAIL (3.26s) - Compiling and running oracle program (0.22s) - Oracle output: 12 (3.04s) + insert_remove: FAIL (0.98s) + Compiling and running oracle program (0.24s) + Oracle output: 12 (0.74s) user error (JSON Decoding of test/conc_eval/hash_map/insert_remove.all.mir failed: Error in $.fns[1489].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) Use -p '/insert_remove/' to rerun this test only. - insert_iter: FAIL (3.21s) - Compiling and running oracle program (0.23s) - Oracle output: [11, 12] (2.99s) + insert_iter: FAIL (0.75s) + Compiling and running oracle program (0.21s) + Oracle output: [11, 12] (0.54s) user error (JSON Decoding of test/conc_eval/hash_map/insert_iter.all.mir failed: Error in $.fns[1464].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) Use -p '/insert_iter/' to rerun this test only. - insert_get: FAIL (3.17s) - Compiling and running oracle program (0.21s) - Oracle output: (11, 12) (2.95s) + insert_get: FAIL (1.23s) + Compiling and running oracle program (0.23s) + Oracle output: (11, 12) (1.00s) user error (JSON Decoding of test/conc_eval/hash_map/insert_get.all.mir failed: Error in $.fns[1465].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) Use -p '/insert_get/' to rerun this test only. traits - params: OK (2.16s) - Compiling and running oracle program (0.18s) - Oracle output: 25 (1.98s) + params: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: 25 Crux output: 25 - generic3: FAIL (2.26s) + generic3: FAIL (0.19s) Compiling and running oracle program (0.17s) - Oracle output: () (2.09s) + Oracle output: () (0.03s) Crux output: failures: @@ -132,13 +113,13 @@ crux-mir crux doesn't match oracle Use -p '/generic3/' to rerun this test only. - static_three: OK (2.18s) - Compiling and running oracle program (0.18s) - Oracle output: 2 (2.00s) + static_three: OK (0.17s) + Compiling and running oracle program (0.17s) + Oracle output: 2 Crux output: 2 - generic1: FAIL (2.20s) + generic1: FAIL (0.19s) Compiling and running oracle program (0.17s) - Oracle output: () (2.02s) + Oracle output: () (0.02s) Crux output: failures: @@ -153,31 +134,31 @@ crux-mir crux doesn't match oracle Use -p '/generic1/' to rerun this test only. - subtrait: OK (2.16s) + subtrait: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 73 (1.99s) + Oracle output: 73 Crux output: 73 - gen_trait: OK (2.17s) + gen_trait: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: -69 (2.00s) + Oracle output: -69 Crux output: -69 - dynamic_branch: FAIL (0.20s) + dynamic_branch: FAIL (0.21s) Compiling and running oracle program (0.17s) Oracle output: 1 (0.03s) user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_branch.rs) Use -p '/dynamic_branch/' to rerun this test only. - static_two: OK (2.15s) + static_two: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 1 (1.98s) + Oracle output: 1 Crux output: 1 - tyfam5: OK (2.15s) + tyfam5: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: () (1.98s) + Oracle output: () Crux output: () - generic2: FAIL (2.22s) - Compiling and running oracle program (0.21s) - Oracle output: () (2.01s) + generic2: FAIL (0.20s) + Compiling and running oracle program (0.17s) + Oracle output: () (0.03s) Crux output: failures: @@ -192,51 +173,51 @@ crux-mir crux doesn't match oracle Use -p '/generic2/' to rerun this test only. - assoc3: OK (2.20s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.02s) + assoc3: OK (0.17s) + Compiling and running oracle program (0.17s) + Oracle output: () Crux output: () - tyfam4: OK (2.19s) + tyfam4: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 0 (2.01s) + Oracle output: 0 Crux output: 0 - static_self: OK (2.12s) - Compiling and running oracle program (0.17s) - Oracle output: 42 (1.96s) + static_self: OK (0.19s) + Compiling and running oracle program (0.19s) + Oracle output: 42 Crux output: 42 - conv: OK (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (2.01s) + conv: OK (0.19s) + Compiling and running oracle program (0.18s) + Oracle output: 1 Crux output: 1 - dynamic_poly: FAIL (0.22s) - Compiling and running oracle program (0.19s) - Oracle output: 3 (0.04s) + dynamic_poly: FAIL (0.21s) + Compiling and running oracle program (0.17s) + Oracle output: 3 (0.03s) user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_poly.rs) Use -p '/dynamic_poly/' to rerun this test only. - dynamic_simple: OK (2.13s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.97s) - Crux output: 1 - gen_trait_poly: OK (2.18s) + dynamic_simple: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 12 (2.01s) - Crux output: 12 - default: OK (2.14s) + Oracle output: 1 + Crux output: 1 + gen_trait_poly: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 12 (1.97s) + Oracle output: 12 Crux output: 12 - assoc1: OK (2.18s) + default: OK (0.18s) Compiling and running oracle program (0.18s) - Oracle output: () (1.99s) + Oracle output: 12 + Crux output: 12 + assoc1: OK (0.18s) + Compiling and running oracle program (0.17s) + Oracle output: () (0.01s) Crux output: () - subtrait2: OK (2.13s) - Compiling and running oracle program (0.18s) - Oracle output: 64 (1.95s) + subtrait2: OK (0.17s) + Compiling and running oracle program (0.17s) + Oracle output: 64 Crux output: 64 - bounds1: FAIL (2.16s) + bounds1: FAIL (0.20s) Compiling and running oracle program (0.17s) - Oracle output: () (1.99s) + Oracle output: () (0.03s) Crux output: failures: @@ -251,9 +232,9 @@ crux-mir crux doesn't match oracle Use -p '/bounds1/' to rerun this test only. - bounds3: FAIL (2.20s) - Compiling and running oracle program (0.20s) - Oracle output: () (2.00s) + bounds3: FAIL (0.20s) + Compiling and running oracle program (0.17s) + Oracle output: () (0.03s) Crux output: failures: @@ -268,33 +249,33 @@ crux-mir crux doesn't match oracle Use -p '/bounds3/' to rerun this test only. - static_eq: OK (2.18s) - Compiling and running oracle program (0.16s) - Oracle output: true (2.01s) + static_eq: OK (0.18s) + Compiling and running oracle program (0.18s) + Oracle output: true Crux output: true - tyfam3: OK (2.17s) + tyfam3: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 23 (1.99s) + Oracle output: 23 Crux output: 23 - dict_med: OK (2.21s) + dict_med: OK (0.16s) Compiling and running oracle program (0.16s) - Oracle output: 42 (2.05s) + Oracle output: 42 Crux output: 42 - basics1: OK (2.24s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.06s) + basics1: OK (0.18s) + Compiling and running oracle program (0.17s) + Oracle output: () (0.01s) Crux output: () - assoc2: OK (2.22s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.04s) + assoc2: OK (0.17s) + Compiling and running oracle program (0.17s) + Oracle output: () Crux output: () - dict_polymem: OK (2.19s) + dict_polymem: OK (0.18s) Compiling and running oracle program (0.18s) - Oracle output: 4 (2.02s) + Oracle output: 4 Crux output: 4 - bounds2: FAIL (2.18s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.01s) + bounds2: FAIL (0.19s) + Compiling and running oracle program (0.17s) + Oracle output: () (0.03s) Crux output: failures: @@ -309,23 +290,23 @@ crux-mir crux doesn't match oracle Use -p '/bounds2/' to rerun this test only. - intoiter: OK (2.17s) - Compiling and running oracle program (0.19s) - Oracle output: 0 (1.98s) + intoiter: OK (0.17s) + Compiling and running oracle program (0.16s) + Oracle output: 0 Crux output: 0 - static: OK (2.15s) - Compiling and running oracle program (0.18s) - Oracle output: 42 (1.97s) - Crux output: 42 - dynamic_two: FAIL (0.21s) + static: OK (0.17s) Compiling and running oracle program (0.17s) + Oracle output: 42 + Crux output: 42 + dynamic_two: FAIL (0.20s) + Compiling and running oracle program (0.16s) Oracle output: 1 (0.04s) user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_two.rs) Use -p '/dynamic_two/' to rerun this test only. - bounds4: FAIL (2.29s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.11s) + bounds4: FAIL (0.20s) + Compiling and running oracle program (0.17s) + Oracle output: () (0.03s) Crux output: failures: @@ -340,21 +321,21 @@ crux-mir crux doesn't match oracle Use -p '/bounds4/' to rerun this test only. - dict_poly: OK (2.16s) + dict_poly: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 1 (1.99s) + Oracle output: 1 Crux output: 1 - dict_simple: OK (2.16s) - Compiling and running oracle program (0.19s) - Oracle output: 32 (1.97s) + dict_simple: OK (0.17s) + Compiling and running oracle program (0.17s) + Oracle output: 32 Crux output: 32 - tyfam: OK (2.13s) - Compiling and running oracle program (0.16s) - Oracle output: 23 (1.97s) + tyfam: OK (0.18s) + Compiling and running oracle program (0.17s) + Oracle output: 23 Crux output: 23 - bounds5: FAIL (2.17s) + bounds5: FAIL (0.20s) Compiling and running oracle program (0.18s) - Oracle output: () (1.99s) + Oracle output: () (0.02s) Crux output: failures: @@ -369,24 +350,24 @@ crux-mir crux doesn't match oracle Use -p '/bounds5/' to rerun this test only. - tyfam2: OK (2.17s) + tyfam2: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 23 (1.99s) + Oracle output: 23 Crux output: 23 - dynamic_med: FAIL (0.20s) - Compiling and running oracle program (0.17s) + dynamic_med: FAIL (0.19s) + Compiling and running oracle program (0.16s) Oracle output: 1 (0.03s) user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_med.rs) Use -p '/dynamic_med/' to rerun this test only. intTest - test0039: OK (2.28s) + test0039: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 7 (2.11s) + Oracle output: 7 Crux output: 7 - test0038: FAIL (2.26s) + test0038: FAIL (0.24s) Compiling and running oracle program (0.19s) - Oracle output: () (2.07s) + Oracle output: () (0.05s) Crux output: failures: @@ -404,30 +385,17 @@ crux-mir Use -p '/test0038/' to rerun this test only. tuple - clone_rec: FAIL (2.21s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.04s) - Crux output: - failures: - - ---- clone_rec/777a7011::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/tuple/clone_rec.rs:4:17: 4:28: error: in clone_rec/777a7011::f[0] - [Crux] Translation error in clone_rec/777a7011::f[0]: fail or unimp constant: TyTuple [TyInt B32,TyInt B32] (StructRepr [MaybeRepr (BVRepr 32), MaybeRepr (BVRepr 32)]) ConstTuple [ConstInt (I32 3),ConstInt (I32 4)] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/clone_rec/' to rerun this test only. - clone: OK (2.21s) - Compiling and running oracle program (0.19s) - Oracle output: () (2.02s) + clone_rec: OK (0.18s) + Compiling and running oracle program (0.17s) + Oracle output: () (0.01s) Crux output: () - clone_from: FAIL (2.19s) - Compiling and running oracle program (0.19s) - Oracle output: () (2.00s) + clone: OK (0.19s) + Compiling and running oracle program (0.17s) + Oracle output: () (0.02s) + Crux output: () + clone_from: FAIL (0.20s) + Compiling and running oracle program (0.17s) + Oracle output: () (0.03s) Crux output: failures: @@ -442,9 +410,9 @@ crux-mir crux doesn't match oracle Use -p '/clone_from/' to rerun this test only. - clone_struct: FAIL (2.19s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.01s) + clone_struct: FAIL (0.22s) + Compiling and running oracle program (0.20s) + Oracle output: () (0.03s) Crux output: failures: @@ -460,26 +428,13 @@ crux-mir Use -p '/clone_struct/' to rerun this test only. iter - for: FAIL (2.25s) - Compiling and running oracle program (0.17s) - Oracle output: 47 (2.08s) - Crux output: - failures: - - ---- for/318ca416::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/uint_macros.rs:470:54: 470:57 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:921:5: 922:101: error: in core/5a1f6960::num[0]::{impl#9}[0]::unchecked_add[0] - [Crux] Translation error in core/5a1f6960::num[0]::{impl#9}[0]::unchecked_add[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_instc5e93708b8ca6e2a[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/iter.for/' to rerun this test only. - sum: FAIL (2.22s) + for: OK (0.20s) + Compiling and running oracle program (0.17s) + Oracle output: 47 (0.03s) + Crux output: 47 + sum: FAIL (0.22s) Compiling and running oracle program (0.17s) - Oracle output: () (2.05s) + Oracle output: () (0.05s) Crux output: failures: @@ -496,17 +451,17 @@ crux-mir crux doesn't match oracle Use -p '/sum/' to rerun this test only. - filter_chain: OK (2.36s) - Compiling and running oracle program (0.20s) - Oracle output: 0 (2.16s) + filter_chain: OK (0.30s) + Compiling and running oracle program (0.21s) + Oracle output: 0 (0.10s) Crux output: 0 - from_fn: OK (2.20s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.03s) + from_fn: OK (0.20s) + Compiling and running oracle program (0.18s) + Oracle output: () (0.02s) Crux output: () - peek: FAIL (2.33s) - Compiling and running oracle program (0.20s) - Oracle output: 3 (2.12s) + peek: FAIL (0.24s) + Compiling and running oracle program (0.18s) + Oracle output: 3 (0.06s) Crux output: failures: @@ -523,9 +478,9 @@ crux-mir crux doesn't match oracle Use -p '/peek/' to rerun this test only. - zip: FAIL (2.29s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.10s) + zip: FAIL (0.26s) + Compiling and running oracle program (0.19s) + Oracle output: () (0.08s) Crux output: failures: @@ -542,13 +497,13 @@ crux-mir crux doesn't match oracle Use -p '/zip/' to rerun this test only. - loop: OK (2.14s) - Compiling and running oracle program (0.18s) - Oracle output: 2 (1.97s) + loop: OK (0.17s) + Compiling and running oracle program (0.17s) + Oracle output: 2 Crux output: 2 - cloned: FAIL (2.24s) + cloned: FAIL (0.22s) Compiling and running oracle program (0.18s) - Oracle output: 6 (2.06s) + Oracle output: 6 (0.04s) Crux output: failures: @@ -566,9 +521,9 @@ crux-mir Use -p '/cloned/' to rerun this test only. str - format_hex: FAIL (3.00s) - Compiling and running oracle program (0.17s) - Oracle output: true (2.83s) + format_hex: FAIL (1.03s) + Compiling and running oracle program (0.20s) + Oracle output: true (0.83s) Crux output: failures: @@ -585,9 +540,9 @@ crux-mir crux doesn't match oracle Use -p '/format_hex/' to rerun this test only. - format_int: FAIL (3.02s) - Compiling and running oracle program (0.18s) - Oracle output: true (2.84s) + format_int: FAIL (1.07s) + Compiling and running oracle program (0.21s) + Oracle output: true (0.86s) Crux output: failures: @@ -604,9 +559,9 @@ crux-mir crux doesn't match oracle Use -p '/format_int/' to rerun this test only. - format_struct: FAIL (3.11s) + format_struct: FAIL (1.11s) Compiling and running oracle program (0.18s) - Oracle output: true (2.93s) + Oracle output: true (0.93s) Crux output: failures: @@ -623,9 +578,9 @@ crux-mir crux doesn't match oracle Use -p '/format_struct/' to rerun this test only. - format: FAIL (3.23s) - Compiling and running oracle program (0.18s) - Oracle output: true (3.05s) + format: FAIL (0.94s) + Compiling and running oracle program (0.19s) + Oracle output: true (0.75s) Crux output: failures: @@ -642,9 +597,9 @@ crux-mir crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..str.format"' to rerun this test only. - to_owned: FAIL (2.37s) - Compiling and running oracle program (0.18s) - Oracle output: true (2.19s) + to_owned: FAIL (0.35s) + Compiling and running oracle program (0.19s) + Oracle output: true (0.16s) Crux output: failures: @@ -659,9 +614,9 @@ crux-mir crux doesn't match oracle Use -p '/to_owned/' to rerun this test only. - string_push: FAIL (3.02s) - Compiling and running oracle program (0.18s) - Oracle output: true (2.84s) + string_push: FAIL (0.86s) + Compiling and running oracle program (0.17s) + Oracle output: true (0.69s) Crux output: failures: @@ -676,9 +631,9 @@ crux-mir crux doesn't match oracle Use -p '/string_push/' to rerun this test only. - format_array: FAIL (3.11s) - Compiling and running oracle program (0.18s) - Oracle output: true (2.92s) + format_array: FAIL (1.10s) + Compiling and running oracle program (0.19s) + Oracle output: true (0.91s) Crux output: failures: @@ -696,15 +651,15 @@ crux-mir Use -p '/format_array/' to rerun this test only. sync - mutex: FAIL (2.80s) - Compiling and running oracle program (0.19s) - Oracle output: 1 (2.61s) + mutex: FAIL (0.67s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (0.50s) user error (JSON Decoding of test/conc_eval/sync/mutex.all.mir failed: Error in $.fns[1069].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) Use -p '$0=="crux-mir.crux concrete..sync.mutex"' to rerun this test only. - arc_cell: FAIL (2.41s) - Compiling and running oracle program (0.19s) - Oracle output: 4 (2.23s) + arc_cell: FAIL (0.30s) + Compiling and running oracle program (0.18s) + Oracle output: 4 (0.12s) Crux output: failures: @@ -719,9 +674,9 @@ crux-mir crux doesn't match oracle Use -p '/arc_cell/' to rerun this test only. - arc: FAIL (2.46s) - Compiling and running oracle program (0.18s) - Oracle output: 1 (2.28s) + arc: FAIL (0.27s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (0.10s) Crux output: failures: @@ -736,9 +691,9 @@ crux-mir crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..sync.arc"' to rerun this test only. - atomic_add: FAIL (2.22s) + atomic_add: FAIL (0.23s) Compiling and running oracle program (0.17s) - Oracle output: 6 (2.05s) + Oracle output: 6 (0.06s) Crux output: failures: @@ -755,15 +710,15 @@ crux-mir crux doesn't match oracle Use -p '/atomic_add/' to rerun this test only. - rwlock: FAIL (2.87s) - Compiling and running oracle program (0.18s) - Oracle output: 2 (2.69s) + rwlock: FAIL (0.71s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (0.53s) user error (JSON Decoding of test/conc_eval/sync/rwlock.all.mir failed: Error in $.fns[1097].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) Use -p '$0=="crux-mir.crux concrete..sync.rwlock"' to rerun this test only. - atomic_cxchg: FAIL (2.40s) - Compiling and running oracle program (0.19s) - Oracle output: 6 (2.22s) + atomic_cxchg: FAIL (0.24s) + Compiling and running oracle program (0.18s) + Oracle output: 6 (0.06s) Crux output: failures: @@ -780,9 +735,9 @@ crux-mir crux doesn't match oracle Use -p '/atomic_cxchg/' to rerun this test only. - arc_clone: FAIL (2.32s) - Compiling and running oracle program (0.18s) - Oracle output: 2 (2.13s) + arc_clone: FAIL (0.27s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (0.10s) Crux output: failures: @@ -797,60 +752,34 @@ crux-mir crux doesn't match oracle Use -p '/arc_clone/' to rerun this test only. - atomic_swap: FAIL (2.21s) - Compiling and running oracle program (0.18s) - Oracle output: (2, 1) (2.03s) - Crux output: - failures: - - ---- atomic_swap/25d5ab9a::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/sync/atomic.rs:3010:60: 3010:63: error: in core/5a1f6960::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/5a1f6960::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::atomic_store_seqcst[0]::_inst1e2825177cd3b608[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/atomic_swap/' to rerun this test only. - rwlock_multi: FAIL (2.87s) + atomic_swap: OK (0.19s) + Compiling and running oracle program (0.17s) + Oracle output: (2, 1) (0.02s) + Crux output: (2, 1) + rwlock_multi: FAIL (0.83s) Compiling and running oracle program (0.18s) - Oracle output: 3 (2.69s) + Oracle output: 3 (0.65s) user error (JSON Decoding of test/conc_eval/sync/rwlock_multi.all.mir failed: Error in $.fns[1123].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) Use -p '/rwlock_multi/' to rerun this test only. - atomic_fence: FAIL (2.32s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (2.16s) - Crux output: - failures: - - ---- atomic_fence/89629395::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/sync/atomic.rs:3010:60: 3010:63: error: in core/5a1f6960::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/5a1f6960::sync[0]::atomic[0]::atomic_store[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::atomic_store_seqcst[0]::_inst1e2825177cd3b608[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/atomic_fence/' to rerun this test only. - mutex_multi: FAIL (2.71s) + atomic_fence: OK (0.18s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (0.02s) + Crux output: 2 + mutex_multi: FAIL (0.76s) Compiling and running oracle program (0.18s) - Oracle output: 3 (2.53s) + Oracle output: 3 (0.59s) user error (JSON Decoding of test/conc_eval/sync/mutex_multi.all.mir failed: Error in $.fns[1079].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) Use -p '/mutex_multi/' to rerun this test only. consts - struct_val: OK (2.21s) + struct_val: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: Foo { x: true } (2.03s) + Oracle output: Foo { x: true } Crux output: Foo { x: true } - struct_unit: FAIL (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: Err(Foo { x: () }) (2.00s) + struct_unit: FAIL (0.20s) + Compiling and running oracle program (0.18s) + Oracle output: Err(Foo { x: () }) (0.02s) Crux output: failures: @@ -865,65 +794,39 @@ crux-mir crux doesn't match oracle Use -p '/struct_unit/' to rerun this test only. - local_key: FAIL (2.21s) - Compiling and running oracle program (0.18s) - Oracle output: 1 (2.04s) + local_key: FAIL (0.17s) + Compiling and running oracle program (0.17s) + Oracle output: 1 user error (JSON Decoding of test/conc_eval/consts/local_key.all.mir failed: Error in $.statics[0].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) Use -p '/local_key/' to rerun this test only. - fn_def: FAIL (2.19s) - Compiling and running oracle program (0.21s) - Oracle output: 1 (1.99s) + fn_def: FAIL (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: 1 user error (JSON Decoding of test/conc_eval/consts/fn_def.all.mir failed: Error in $.fns[0].body.blocks[0].block.terminator.func.data.rendered: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) Use -p '/fn_def/' to rerun this test only. - enum_val: OK (2.14s) + enum_val: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: None (1.97s) + Oracle output: None Crux output: None crypto - add: FAIL (2.29s) - Compiling and running oracle program (0.18s) - Oracle output: true (2.11s) - Crux output: - failures: - - ---- add/f9cfa7f7::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/num/uint_macros.rs:470:54: 470:57 !lib/core/src/num/mod.rs:965:5: 970:27: error: in core/5a1f6960::num[0]::{impl#12}[0]::unchecked_add[0] - [Crux] Translation error in core/5a1f6960::num[0]::{impl#12}[0]::unchecked_add[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_instaffa7a8b1157c078[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '$0=="crux-mir.crux concrete..crypto.add"' to rerun this test only. - add_noL: FAIL (2.22s) - Compiling and running oracle program (0.19s) - Oracle output: false (2.03s) - Crux output: - failures: - - ---- add_noL/d822cec7::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/num/uint_macros.rs:470:54: 470:57 !lib/core/src/num/mod.rs:965:5: 970:27: error: in core/5a1f6960::num[0]::{impl#12}[0]::unchecked_add[0] - [Crux] Translation error in core/5a1f6960::num[0]::{impl#12}[0]::unchecked_add[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_instaffa7a8b1157c078[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/add_noL/' to rerun this test only. + add: OK (0.24s) + Compiling and running oracle program (0.17s) + Oracle output: true (0.06s) + Crux output: true + add_noL: OK (0.21s) + Compiling and running oracle program (0.17s) + Oracle output: false (0.04s) + Crux output: false cell - cell: OK (2.24s) - Compiling and running oracle program (0.18s) - Oracle output: 2 (2.05s) + cell: OK (0.19s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (0.03s) Crux output: 2 - ref_cell: FAIL (2.95s) - Compiling and running oracle program (0.18s) - Oracle output: 2 (2.77s) + ref_cell: FAIL (0.83s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (0.65s) Crux output: failures: @@ -940,9 +843,9 @@ crux-mir crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..cell.ref_cell"' to rerun this test only. - ref_cell2: FAIL (2.94s) - Compiling and running oracle program (0.19s) - Oracle output: 2 (2.76s) + ref_cell2: FAIL (0.78s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (0.61s) Crux output: failures: @@ -960,17 +863,17 @@ crux-mir Use -p '/ref_cell2/' to rerun this test only. fnptr - call: OK (2.32s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (2.15s) + call: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: 2 Crux output: 2 - field: OK (2.22s) - Compiling and running oracle program (0.18s) - Oracle output: 2 (2.04s) + field: OK (0.17s) + Compiling and running oracle program (0.17s) + Oracle output: 2 Crux output: 2 - make: OK (2.18s) - Compiling and running oracle program (0.18s) - Oracle output: 0 (1.99s) + make: OK (0.17s) + Compiling and running oracle program (0.17s) + Oracle output: 0 Crux output: 0 custom: rustc compilation failed for custom error output: @@ -992,16 +895,16 @@ FAIL (expected: taking address of an overridden function) (0.06s) failed to compile and run (expected failure) num - overflow: FAIL (2.22s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.05s) + overflow: FAIL (0.20s) + Compiling and running oracle program (0.17s) + Oracle output: () (0.03s) Crux output: failures: ---- overflow/0ff004e4::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in static_initializer - [Crux] Translation error in the static initializer: fail or unimp constant: TyTuple [TyUint B8,TyBool] (StructRepr [MaybeRepr (BVRepr 8), MaybeRepr BoolRepr]) ConstTuple [ConstInt (U8 3),ConstBool False] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/uint_macros.rs:1578:74: 1578:77 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:305:5: 306:27: error: in core/5a1f6960::num[0]::{impl#7}[0]::overflowing_sub[0] + [Crux] Translation error in core/5a1f6960::num[0]::{impl#7}[0]::overflowing_sub[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::sub_with_overflow[0]::_instaddce72e1232152c[0] [Crux] Overall status: Invalid. @@ -1009,9 +912,9 @@ FAIL (expected: taking address of an overridden function) (0.06s) crux doesn't match oracle Use -p '/num.overflow/' to rerun this test only. - from_bytes: FAIL (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.01s) + from_bytes: FAIL (0.19s) + Compiling and running oracle program (0.16s) + Oracle output: () (0.03s) Crux output: failures: @@ -1026,43 +929,30 @@ FAIL (expected: taking address of an overridden function) (0.06s) crux doesn't match oracle Use -p '/from_bytes/' to rerun this test only. - saturate: FAIL (2.86s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.69s) - Crux output: - failures: - - ---- saturate/e630dede::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/uint_macros.rs:1022:46: 1022:49 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:305:5: 306:27: error: in core/5a1f6960::num[0]::{impl#7}[0]::saturating_add[0] - [Crux] Translation error in core/5a1f6960::num[0]::{impl#7}[0]::saturating_add[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::saturating_add[0]::_instaddce72e1232152c[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/saturate/' to rerun this test only. + saturate: OK (0.76s) + Compiling and running oracle program (0.19s) + Oracle output: () (0.57s) + Crux output: () prim - bool: OK (2.13s) + bool: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: false (1.97s) + Oracle output: false Crux output: false - shift3: OK (2.27s) - Compiling and running oracle program (0.17s) - Oracle output: -9223372036854775808 (2.10s) + shift3: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: -9223372036854775808 Crux output: -9223372036854775808 - div: OK (2.15s) + div: OK (0.16s) Compiling and running oracle program (0.16s) - Oracle output: 4 (1.98s) + Oracle output: 4 Crux output: 4 - ge: OK (2.16s) + ge: OK (0.16s) Compiling and running oracle program (0.16s) - Oracle output: true (2.00s) + Oracle output: true Crux output: true - add1: OK (2.19s) + add1: OK (0.16s) Compiling and running oracle program (0.16s) - Oracle output: 2 (2.03s) + Oracle output: 2 Crux output: 2 shift_exceeding: rustc compilation failed for shift_exceeding error output: @@ -1099,9 +989,9 @@ FAIL (expected: Should panic, but doesn't) (0.06s) test/Test.hs:107: failed to compile and run (expected failure) - char_from_u32: FAIL (2.21s) - Compiling and running oracle program (0.17s) - Oracle output: 'A' (2.04s) + char_from_u32: FAIL (0.19s) + Compiling and running oracle program (0.16s) + Oracle output: 'A' (0.03s) Crux output: failures: @@ -1116,30 +1006,17 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/char_from_u32/' to rerun this test only. - ffs: FAIL (2.22s) - Compiling and running oracle program (0.17s) - Oracle output: true (2.04s) - Crux output: - failures: - - ---- ffs/5879a819::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/int_macros.rs:460:54: 460:57 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:251:5: 253:45: error: in core/5a1f6960::num[0]::{impl#3}[0]::unchecked_add[0] - [Crux] Translation error in core/5a1f6960::num[0]::{impl#3}[0]::unchecked_add[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_inst1e2825177cd3b608[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/prim.ffs/' to rerun this test only. - mut_arg: OK (2.26s) + ffs: OK (0.20s) + Compiling and running oracle program (0.17s) + Oracle output: true (0.03s) + Crux output: true + mut_arg: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 1 (2.09s) + Oracle output: 1 Crux output: 1 - litstring: FAIL (2.17s) - Compiling and running oracle program (0.16s) - Oracle output: true (2.00s) + litstring: FAIL (0.19s) + Compiling and running oracle program (0.17s) + Oracle output: true (0.02s) Crux output: failures: @@ -1154,64 +1031,51 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/litstring/' to rerun this test only. - lit: FAIL (2.15s) - Compiling and running oracle program (0.17s) - Oracle output: false (1.97s) - Crux output: - failures: - - ---- lit/7ce419c4::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in lit/7ce419c4::crux_test[0] - [Crux] Translation error in lit/7ce419c4::crux_test[0]: fail or unimp constant: TyTuple [TyBool,TyBool] (StructRepr [MaybeRepr BoolRepr, MaybeRepr BoolRepr]) ConstTuple [ConstBool True,ConstBool True] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '$0=="crux-mir.crux concrete..prim.lit"' to rerun this test only. - litbstring: OK (2.12s) + lit: OK (0.17s) Compiling and running oracle program (0.16s) - Oracle output: true (1.96s) + Oracle output: false + Crux output: false + litbstring: OK (0.17s) + Compiling and running oracle program (0.17s) + Oracle output: true Crux output: true - shift1: OK (2.17s) + shift1: OK (0.16s) Compiling and running oracle program (0.16s) - Oracle output: 2 (2.01s) + Oracle output: 2 Crux output: 2 - shift2: OK (2.16s) + shift2: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 2 (1.99s) + Oracle output: 2 Crux output: 2 - shift4: OK (2.15s) - Compiling and running oracle program (0.18s) - Oracle output: -9223372036854775808 (1.97s) + shift4: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: -9223372036854775808 Crux output: -9223372036854775808 - mut: OK (2.14s) - Compiling and running oracle program (0.18s) - Oracle output: 14 (1.96s) + mut: OK (0.17s) + Compiling and running oracle program (0.17s) + Oracle output: 14 Crux output: 14 - wrapping_sub: OK (2.23s) - Compiling and running oracle program (0.21s) - Oracle output: true (2.02s) + wrapping_sub: OK (0.18s) + Compiling and running oracle program (0.17s) + Oracle output: true Crux output: true impl - self: OK (2.22s) - Compiling and running oracle program (0.17s) - Oracle output: 42 (2.04s) + self: OK (0.17s) + Compiling and running oracle program (0.16s) + Oracle output: 42 Crux output: 42 - simple: OK (2.17s) - Compiling and running oracle program (0.18s) - Oracle output: 42 (2.00s) + simple: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: 42 Crux output: 42 - self_mut: OK (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: 42 (2.00s) + self_mut: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: 42 Crux output: 42 box - new: FAIL (2.22s) + new: FAIL (0.25s) Compiling and running oracle program (0.17s) - Oracle output: () (2.05s) + Oracle output: () (0.08s) Crux output: failures: @@ -1226,9 +1090,9 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/box.new/' to rerun this test only. - mut_ref: FAIL (2.33s) + mut_ref: FAIL (0.25s) Compiling and running oracle program (0.17s) - Oracle output: () (2.16s) + Oracle output: () (0.09s) Crux output: failures: @@ -1243,9 +1107,9 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/box.mut_ref/' to rerun this test only. - mut: FAIL (2.30s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.13s) + mut: FAIL (0.27s) + Compiling and running oracle program (0.19s) + Oracle output: () (0.08s) Crux output: failures: @@ -1260,9 +1124,9 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..box.mut"' to rerun this test only. - unsize: FAIL (2.28s) - Compiling and running oracle program (0.18s) - Oracle output: 2 (2.11s) + unsize: FAIL (0.25s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (0.09s) Crux output: failures: @@ -1277,9 +1141,9 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/box.unsize/' to rerun this test only. - struct: FAIL (2.26s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (2.09s) + struct: FAIL (0.25s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (0.08s) Crux output: failures: @@ -1295,9 +1159,9 @@ FAIL (expected: Should panic, but doesn't) (0.06s) Use -p '/box.struct/' to rerun this test only. ptr - is_null_slice: FAIL (expected: can't unsize null pointers) (2.20s) - Compiling and running oracle program (0.17s) - Oracle output: (false, true) (2.03s) + is_null_slice: FAIL (expected: can't unsize null pointers) (0.18s) + Compiling and running oracle program (0.16s) + Oracle output: (false, true) (0.02s) Crux output: failures: @@ -1316,29 +1180,29 @@ FAIL (expected: Should panic, but doesn't) (0.06s) test/Test.hs:124: crux doesn't match oracle (expected failure) - struct_eq: OK (2.20s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.03s) + struct_eq: OK (0.17s) + Compiling and running oracle program (0.16s) + Oracle output: () (0.01s) Crux output: () - read_write: OK (2.20s) - Compiling and running oracle program (0.17s) - Oracle output: [1, 3, 1] (2.03s) + read_write: OK (0.20s) + Compiling and running oracle program (0.16s) + Oracle output: [1, 3, 1] (0.04s) Crux output: [1, 3, 1] - offset_mut: OK (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: 3 (2.00s) + offset_mut: OK (0.19s) + Compiling and running oracle program (0.19s) + Oracle output: 3 Crux output: 3 - copy: OK (2.22s) + copy: OK (0.20s) Compiling and running oracle program (0.17s) - Oracle output: [1, 2, 3, 1, 2, 3] (2.05s) + Oracle output: [1, 2, 3, 1, 2, 3] (0.03s) Crux output: [1, 2, 3, 1, 2, 3] - unsize_slice: OK (2.22s) - Compiling and running oracle program (0.19s) - Oracle output: (1, 2) (2.02s) + unsize_slice: OK (0.17s) + Compiling and running oracle program (0.16s) + Oracle output: (1, 2) Crux output: (1, 2) - dangling_eq: FAIL (2.21s) - Compiling and running oracle program (0.16s) - Oracle output: () (2.04s) + dangling_eq: FAIL (0.20s) + Compiling and running oracle program (0.17s) + Oracle output: () (0.03s) Crux output: failures: @@ -1353,17 +1217,17 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/dangling_eq/' to rerun this test only. - cast_eq: OK (2.17s) - Compiling and running oracle program (0.18s) - Oracle output: () (1.99s) - Crux output: () - offset_from: OK (2.21s) + cast_eq: OK (0.18s) Compiling and running oracle program (0.17s) - Oracle output: -2 (2.03s) + Oracle output: () + Crux output: () + offset_from: OK (0.21s) + Compiling and running oracle program (0.20s) + Oracle output: -2 (0.01s) Crux output: -2 - null_eq: FAIL (2.19s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (2.02s) + null_eq: FAIL (0.21s) + Compiling and running oracle program (0.19s) + Oracle output: 1 (0.03s) Crux output: failures: @@ -1378,17 +1242,17 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/null_eq/' to rerun this test only. - coerce_unsized: OK (2.23s) - Compiling and running oracle program (0.17s) - Oracle output: (1, 2) (2.06s) + coerce_unsized: OK (0.17s) + Compiling and running oracle program (0.16s) + Oracle output: (1, 2) Crux output: (1, 2) - offset: OK (2.18s) - Compiling and running oracle program (0.18s) - Oracle output: 3 (2.00s) + offset: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: 3 Crux output: 3 - is_null: FAIL (2.15s) - Compiling and running oracle program (0.17s) - Oracle output: (false, true) (1.97s) + is_null: FAIL (0.18s) + Compiling and running oracle program (0.16s) + Oracle output: (false, true) (0.02s) Crux output: failures: @@ -1405,97 +1269,97 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..ptr.is_null"' to rerun this test only. - valid_eq: OK (2.21s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.05s) + valid_eq: OK (0.17s) + Compiling and running oracle program (0.16s) + Oracle output: () (0.02s) Crux output: () ops - deref2: OK (2.20s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.02s) + deref2: OK (0.18s) + Compiling and running oracle program (0.16s) + Oracle output: () (0.02s) Crux output: () - index2: OK (2.18s) + index2: OK (0.19s) Compiling and running oracle program (0.17s) - Oracle output: () (2.01s) + Oracle output: () (0.01s) Crux output: () - arith1: OK (2.16s) - Compiling and running oracle program (0.18s) - Oracle output: () (1.99s) - Crux output: () - index3: OK (2.19s) + arith1: OK (0.19s) Compiling and running oracle program (0.17s) - Oracle output: () (2.01s) + Oracle output: () (0.02s) Crux output: () - deref1: OK (2.19s) + index3: OK (0.18s) Compiling and running oracle program (0.17s) - Oracle output: () (2.03s) + Oracle output: () (0.01s) Crux output: () - index1: OK (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.00s) + deref1: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: () Crux output: () - deref3: OK (2.16s) + index1: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: () + Crux output: () + deref3: OK (0.18s) Compiling and running oracle program (0.17s) - Oracle output: () (1.99s) + Oracle output: () (0.01s) Crux output: () statics - promoted_fn: OK (2.19s) - Compiling and running oracle program (0.19s) - Oracle output: 1 (1.99s) + promoted_fn: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: 1 Crux output: 1 - promoted_static: OK (2.14s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (1.97s) + promoted_static: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: 1 Crux output: 1 io - cursor_write2: FAIL (2.70s) + cursor_write2: FAIL (0.72s) Compiling and running oracle program (0.18s) - Oracle output: 0 (2.52s) + Oracle output: 0 (0.54s) user error (JSON Decoding of test/conc_eval/io/cursor_write2.all.mir failed: Error in $.statics[61].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) Use -p '/cursor_write2/' to rerun this test only. - cursor_write: FAIL (2.80s) - Compiling and running oracle program (0.20s) - Oracle output: 0 (2.60s) + cursor_write: FAIL (0.66s) + Compiling and running oracle program (0.19s) + Oracle output: 0 (0.47s) standalone use of `dyn` is not supported: TyDynamic core/5a1f6960::error[0]::Error[0]::_trait56353a7b840403b4[0] CallStack (from HasCallStack): error, called at src/Mir/TransTy.hs:225:25 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/Trans.hs:815:12 in crucible-mir-0.1-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:883:5 in crucible-mir-0.1-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:935:30 in crucible-mir-0.1-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1147:11 in crucible-mir-0.1-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1746:68 in crucible-mir-0.1-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1755:28 in crucible-mir-0.1-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1795:10 in crucible-mir-0.1-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1841:24 in crucible-mir-0.1-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2275:30 in crucible-mir-0.1-inplace:Mir.Trans + tyToRepr, called at src/Mir/Trans.hs:833:12 in crucible-mir-0.1-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:901:5 in crucible-mir-0.1-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:953:30 in crucible-mir-0.1-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1165:11 in crucible-mir-0.1-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1764:68 in crucible-mir-0.1-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1773:28 in crucible-mir-0.1-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1813:10 in crucible-mir-0.1-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1859:24 in crucible-mir-0.1-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2293:30 in crucible-mir-0.1-inplace:Mir.Trans transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate translateMIR, called at src/Mir/Language.hs:255:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '$0=="crux-mir.crux concrete..io.cursor_write"' to rerun this test only. - cursor_read: FAIL (2.77s) - Compiling and running oracle program (0.19s) - Oracle output: 0 (2.58s) + cursor_read: FAIL (0.74s) + Compiling and running oracle program (0.20s) + Oracle output: 0 (0.55s) standalone use of `dyn` is not supported: TyDynamic core/5a1f6960::error[0]::Error[0]::_trait56353a7b840403b4[0] CallStack (from HasCallStack): error, called at src/Mir/TransTy.hs:225:25 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/Trans.hs:815:12 in crucible-mir-0.1-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:883:5 in crucible-mir-0.1-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:935:30 in crucible-mir-0.1-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1147:11 in crucible-mir-0.1-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1746:68 in crucible-mir-0.1-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1755:28 in crucible-mir-0.1-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1795:10 in crucible-mir-0.1-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1841:24 in crucible-mir-0.1-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2275:30 in crucible-mir-0.1-inplace:Mir.Trans + tyToRepr, called at src/Mir/Trans.hs:833:12 in crucible-mir-0.1-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:901:5 in crucible-mir-0.1-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:953:30 in crucible-mir-0.1-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1165:11 in crucible-mir-0.1-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1764:68 in crucible-mir-0.1-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1773:28 in crucible-mir-0.1-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1813:10 in crucible-mir-0.1-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1859:24 in crucible-mir-0.1-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2293:30 in crucible-mir-0.1-inplace:Mir.Trans transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate translateMIR, called at src/Mir/Language.hs:255:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '/io.cursor_read/' to rerun this test only. mem - maybe_uninit: FAIL (2.20s) + maybe_uninit: FAIL (0.19s) Compiling and running oracle program (0.18s) - Oracle output: 1 (2.02s) + Oracle output: 1 (0.02s) Crux output: failures: @@ -1510,9 +1374,9 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..mem.maybe_uninit"' to rerun this test only. - maybe_uninit_array_cast: FAIL (2.20s) - Compiling and running oracle program (0.18s) - Oracle output: [1, 2] (2.02s) + maybe_uninit_array_cast: FAIL (0.19s) + Compiling and running oracle program (0.17s) + Oracle output: [1, 2] (0.02s) Crux output: failures: @@ -1528,36 +1392,36 @@ FAIL (expected: Should panic, but doesn't) (0.06s) Use -p '/maybe_uninit_array_cast/' to rerun this test only. time - instant: FAIL (2.81s) + instant: FAIL (0.64s) Compiling and running oracle program (0.17s) - Oracle output: () (2.64s) + Oracle output: () (0.47s) user error (JSON Decoding of test/conc_eval/time/instant.all.mir failed: Error in $.statics[61].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) Use -p '/instant/' to rerun this test only. struct - field_order: OK (2.20s) - Compiling and running oracle program (0.19s) - Oracle output: () (2.01s) + field_order: OK (0.18s) + Compiling and running oracle program (0.17s) + Oracle output: () Crux output: () - arg: OK (2.16s) - Compiling and running oracle program (0.18s) - Oracle output: 42 (1.98s) + arg: OK (0.17s) + Compiling and running oracle program (0.17s) + Oracle output: 42 Crux output: 42 - tup: OK (2.18s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.00s) + tup: OK (0.17s) + Compiling and running oracle program (0.17s) + Oracle output: () Crux output: () - proj: OK (2.14s) + proj: OK (0.18s) Compiling and running oracle program (0.17s) - Oracle output: 42 (1.98s) + Oracle output: 42 Crux output: 42 - ret: OK (2.16s) - Compiling and running oracle program (0.18s) - Oracle output: S { x: 42, y: 120 } (1.98s) + ret: OK (0.19s) + Compiling and running oracle program (0.19s) + Oracle output: S { x: 42, y: 120 } Crux output: S { x: 42, y: 120 } - repr_transparent: FAIL (2.24s) - Compiling and running oracle program (0.19s) - Oracle output: 6 (2.05s) + repr_transparent: FAIL (0.21s) + Compiling and running oracle program (0.18s) + Oracle output: 6 (0.03s) Crux output: failures: @@ -1574,84 +1438,84 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..struct.repr_transparent"' to rerun this test only. - repr_transparent_const: OK (2.16s) - Compiling and running oracle program (0.17s) - Oracle output: 124 (1.99s) + repr_transparent_const: OK (0.18s) + Compiling and running oracle program (0.18s) + Oracle output: 124 Crux output: 124 dyn - assoc_ty: OK (2.17s) + assoc_ty: OK (0.19s) Compiling and running oracle program (0.19s) - Oracle output: 100 (1.98s) - Crux output: 100 - trait_param: OK (2.20s) - Compiling and running oracle program (0.18s) - Oracle output: 100 (2.02s) + Oracle output: 100 Crux output: 100 - plain_trait: OK (2.17s) + trait_param: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 100 (2.00s) + Oracle output: 100 Crux output: 100 - inherit: OK (2.24s) + plain_trait: OK (0.17s) + Compiling and running oracle program (0.16s) + Oracle output: 100 + Crux output: 100 + inherit: OK (0.18s) Compiling and running oracle program (0.17s) - Oracle output: 4 (2.07s) + Oracle output: 4 Crux output: 4 stdlib - option: OK (2.18s) - Compiling and running oracle program (0.18s) - Oracle output: true (2.00s) - Crux output: true - option2: OK (2.15s) - Compiling and running oracle program (0.17s) - Oracle output: 0 (1.99s) - Crux output: 0 - result: OK (2.13s) + option: OK (0.16s) Compiling and running oracle program (0.16s) - Oracle output: 27 (1.97s) - Crux output: 27 - default: OK (2.18s) - Compiling and running oracle program (0.18s) - Oracle output: true (2.00s) + Oracle output: true Crux output: true - cvt: OK (2.14s) + option2: OK (0.18s) Compiling and running oracle program (0.18s) - Oracle output: 0 (1.97s) + Oracle output: 0 Crux output: 0 - range: OK (2.21s) + result: OK (0.19s) Compiling and running oracle program (0.19s) - Oracle output: 10 (2.02s) + Oracle output: 27 + Crux output: 27 + default: OK (0.17s) + Compiling and running oracle program (0.17s) + Oracle output: true + Crux output: true + cvt: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: 0 + Crux output: 0 + range: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: 10 Crux output: 10 - poly: OK (2.18s) - Compiling and running oracle program (0.18s) - Oracle output: 1 (2.00s) + poly: OK (0.17s) + Compiling and running oracle program (0.17s) + Oracle output: 1 Crux output: 1 - result_interior: OK (2.15s) - Compiling and running oracle program (0.16s) - Oracle output: 3 (1.98s) + result_interior: OK (0.18s) + Compiling and running oracle program (0.17s) + Oracle output: 3 Crux output: 3 - default_impl: OK (2.17s) - Compiling and running oracle program (0.19s) - Oracle output: () (1.98s) + default_impl: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: () Crux output: () - option3: OK (2.16s) - Compiling and running oracle program (0.17s) - Oracle output: 27 (2.00s) + option3: OK (0.19s) + Compiling and running oracle program (0.19s) + Oracle output: 27 Crux output: 27 - teq: OK (2.14s) - Compiling and running oracle program (0.17s) - Oracle output: false (1.98s) + teq: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: false Crux output: false array - wick2: OK (2.19s) - Compiling and running oracle program (0.18s) - Oracle output: true (2.01s) + wick2: OK (0.17s) + Compiling and running oracle program (0.16s) + Oracle output: true Crux output: true - arg: OK (2.18s) - Compiling and running oracle program (0.19s) - Oracle output: 2 (1.99s) + arg: OK (0.17s) + Compiling and running oracle program (0.16s) + Oracle output: 2 Crux output: 2 - iter: FAIL (2.26s) - Compiling and running oracle program (0.20s) - Oracle output: 3 (2.06s) + iter: FAIL (0.22s) + Compiling and running oracle program (0.17s) + Oracle output: 3 (0.05s) Crux output: failures: @@ -1668,18 +1532,18 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/array.iter/' to rerun this test only. - mk_and_proj: OK (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: 42 (2.01s) + mk_and_proj: OK (0.17s) + Compiling and running oracle program (0.16s) + Oracle output: 42 Crux output: 42 - wick3: FAIL (expected: needs Vec data structure from stdlib) (0.23s) + wick3: FAIL (expected: needs Vec data structure from stdlib) (0.24s) Compiling and running oracle program (0.19s) Oracle output: true (0.05s) user error (Error 101 while running mir-json on test/conc_eval/array/wick3.rs) (expected failure) - clone: FAIL (2.19s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.01s) + clone: FAIL (0.20s) + Compiling and running oracle program (0.17s) + Oracle output: () (0.03s) Crux output: failures: @@ -1696,14 +1560,14 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/array.clone/' to rerun this test only. - wick1: FAIL (expected: needs Vec data structure from stdlib) (0.24s) - Compiling and running oracle program (0.19s) + wick1: FAIL (expected: needs Vec data structure from stdlib) (0.23s) + Compiling and running oracle program (0.18s) Oracle output: true (0.05s) user error (Error 101 while running mir-json on test/conc_eval/array/wick1.rs) (expected failure) - mut_index: FAIL (2.69s) + mut_index: FAIL (0.76s) Compiling and running oracle program (0.17s) - Oracle output: 7 (2.52s) + Oracle output: 7 (0.60s) Crux output: failures: @@ -1718,17 +1582,17 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/mut_index/' to rerun this test only. - mut_arg: OK (2.18s) - Compiling and running oracle program (0.17s) - Oracle output: 42 (2.01s) + mut_arg: OK (0.18s) + Compiling and running oracle program (0.18s) + Oracle output: 42 Crux output: 42 - const: OK (2.14s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (1.97s) + const: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: 1 Crux output: 1 - from_slice: FAIL (3.15s) - Compiling and running oracle program (0.19s) - Oracle output: () (2.96s) + from_slice: FAIL (0.87s) + Compiling and running oracle program (0.16s) + Oracle output: () (0.71s) Crux output: failures: @@ -1743,26 +1607,26 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/array.from_slice/' to rerun this test only. - const_impl: OK (2.20s) - Compiling and running oracle program (0.18s) - Oracle output: 5 (2.02s) + const_impl: OK (0.17s) + Compiling and running oracle program (0.17s) + Oracle output: 5 Crux output: 5 enum - match: OK (2.15s) + match: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 42 (1.98s) + Oracle output: 42 Crux output: 42 - field_order: OK (2.24s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.05s) + field_order: OK (0.18s) + Compiling and running oracle program (0.17s) + Oracle output: () Crux output: () - mixed_discrs: OK (2.18s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.00s) + mixed_discrs: OK (0.17s) + Compiling and running oracle program (0.17s) + Oracle output: () Crux output: () - arg: OK (2.18s) - Compiling and running oracle program (0.18s) - Oracle output: 0 (2.00s) + arg: OK (0.17s) + Compiling and running oracle program (0.16s) + Oracle output: 0 Crux output: 0 arg2: rustc compilation failed for arg2 error output: @@ -1786,15 +1650,15 @@ error: aborting due to 2 previous errors For more information about this error, try `rustc --explain E0518`. -FAIL (0.06s) - Compiling and running oracle program (0.06s) +FAIL (0.05s) + Compiling and running oracle program (0.05s) test/Test.hs:107: failed to compile and run Use -p '/arg2/' to rerun this test only. - cow: FAIL (2.35s) + cow: FAIL (0.32s) Compiling and running oracle program (0.18s) - Oracle output: 200 (2.17s) + Oracle output: 200 (0.15s) Crux output: failures: @@ -1809,26 +1673,26 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/cow/' to rerun this test only. - ret: OK (2.15s) - Compiling and running oracle program (0.19s) - Oracle output: (A(42), B { x: 42 }, C) (1.96s) + ret: OK (0.17s) + Compiling and running oracle program (0.17s) + Oracle output: (A(42), B { x: 42 }, C) Crux output: (A(42), B { x: 42 }, C) - eq: OK (2.18s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.01s) + eq: OK (0.19s) + Compiling and running oracle program (0.17s) + Oracle output: () (0.01s) Crux output: () - cmp: OK (2.13s) + cmp: OK (0.18s) Compiling and running oracle program (0.17s) - Oracle output: -23 (1.96s) + Oracle output: -23 Crux output: -23 - inner: OK (2.19s) + inner: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 4 (2.02s) + Oracle output: 4 Crux output: 4 vec - drop: FAIL (2.33s) + drop: FAIL (0.29s) Compiling and running oracle program (0.18s) - Oracle output: () (2.15s) + Oracle output: () (0.11s) Crux output: failures: @@ -1843,9 +1707,9 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/drop/' to rerun this test only. - set_len: FAIL (2.33s) + set_len: FAIL (0.30s) Compiling and running oracle program (0.18s) - Oracle output: 2 (2.15s) + Oracle output: 2 (0.12s) Crux output: failures: @@ -1860,9 +1724,9 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/set_len/' to rerun this test only. - push: FAIL (2.47s) - Compiling and running oracle program (0.19s) - Oracle output: (1, 2) (2.28s) + push: FAIL (0.38s) + Compiling and running oracle program (0.18s) + Oracle output: (1, 2) (0.20s) Crux output: failures: @@ -1877,9 +1741,9 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/vec.push/' to rerun this test only. - extend: FAIL (2.58s) + extend: FAIL (0.41s) Compiling and running oracle program (0.19s) - Oracle output: (1, 10) (2.39s) + Oracle output: (1, 10) (0.23s) Crux output: failures: @@ -1894,9 +1758,9 @@ FAIL (0.06s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..vec.extend"' to rerun this test only. - collect: FAIL (3.30s) - Compiling and running oracle program (0.19s) - Oracle output: 45 (3.11s) + collect: FAIL (1.07s) + Compiling and running oracle program (0.18s) + Oracle output: 45 (0.89s) Crux output: failures: @@ -1913,9 +1777,9 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/collect/' to rerun this test only. - extend_trusted_len: FAIL (3.18s) + extend_trusted_len: FAIL (1.00s) Compiling and running oracle program (0.19s) - Oracle output: (1, 10) (2.99s) + Oracle output: (1, 10) (0.81s) Crux output: failures: @@ -1930,20 +1794,22 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/extend_trusted_len/' to rerun this test only. - from_elem_zero: FAIL (0.24s) - Compiling and running oracle program (0.18s) - Oracle output: 0 (0.06s) - user error (Error 101 while running mir-json on test/conc_eval/vec/from_elem_zero.rs) + from_elem_zero: thread 'main' panicked at 'called `Result::unwrap()` on an `Err` value: ErrorImpl { code: Io(Custom { kind: Other, error: "numeric field was not a number: when getting cksum for crate.json" }), offset: 0 }', src/bin/mir-json-dce.rs:38:44 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAIL (0.22s) + Compiling and running oracle program (0.19s) + Oracle output: 0 (0.03s) + user error (Error 101 while running mir-json on ["test/conc_eval/vec/from_elem_zero.mir","rlibs/libcore.mir","rlibs/librustc_std_workspace_core.mir","rlibs/liblibc.mir","rlibs/libcompiler_builtins.mir","rlibs/liballoc.mir","rlibs/libcfg_if.mir","rlibs/libmemchr.mir","rlibs/libadler.mir","rlibs/librustc_demangle.mir","rlibs/libunwind.mir","rlibs/libpanic_unwind.mir","rlibs/librustc_std_workspace_alloc.mir","rlibs/libpanic_abort.mir","rlibs/libgimli.mir","rlibs/libstd_detect.mir","rlibs/libobject.mir","rlibs/libminiz_oxide.mir","rlibs/libhashbrown.mir","rlibs/libaddr2line.mir","rlibs/libstd.mir","rlibs/libcrucible.mir","rlibs/libint512.mir","rlibs/libbyteorder.mir","rlibs/libbytes.mir"]) Use -p '/from_elem_zero/' to rerun this test only. clos - promoted: OK (2.16s) - Compiling and running oracle program (0.17s) - Oracle output: 33 (1.98s) + promoted: OK (0.18s) + Compiling and running oracle program (0.18s) + Oracle output: 33 Crux output: 33 - as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (2.20s) + as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (0.18s) Compiling and running oracle program (0.17s) - Oracle output: 1 (2.03s) + Oracle output: 1 (0.02s) Crux output: failures: @@ -1959,95 +1825,95 @@ FAIL (0.06s) test/Test.hs:124: crux doesn't match oracle (expected failure) - fnptr_fnmut: OK (2.14s) - Compiling and running oracle program (0.17s) - Oracle output: 4 (1.97s) - Crux output: 4 - direct_fnonce: OK (2.18s) + fnptr_fnmut: OK (0.18s) Compiling and running oracle program (0.18s) - Oracle output: 2 (2.00s) + Oracle output: 4 + Crux output: 4 + direct_fnonce: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: 2 Crux output: 2 - conv_fnonce_fnmut: OK (2.14s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (1.97s) + conv_fnonce_fnmut: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: 2 Crux output: 2 - fn_static_poly: OK (2.13s) - Compiling and running oracle program (0.17s) - Oracle output: 3 (1.96s) + fn_static_poly: OK (0.19s) + Compiling and running oracle program (0.19s) + Oracle output: 3 Crux output: 3 - fnptr_fnonce: OK (2.24s) + fnptr_fnonce: OK (0.18s) Compiling and running oracle program (0.17s) - Oracle output: 4 (2.07s) + Oracle output: 4 Crux output: 4 - fn_static: OK (2.29s) - Compiling and running oracle program (0.17s) - Oracle output: 3 (2.12s) + fn_static: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: 3 Crux output: 3 - fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (0.22s) - Compiling and running oracle program (0.19s) + fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (0.20s) + Compiling and running oracle program (0.17s) Oracle output: 3 (0.03s) user error (Error 101 while running mir-json on test/conc_eval/clos/fn_dyn.rs) (expected failure) - fnonce: OK (2.15s) - Compiling and running oracle program (0.17s) - Oracle output: false (1.98s) + fnonce: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: false Crux output: false - conv_fnmut_fn: OK (2.16s) + conv_fnmut_fn: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 2 (1.99s) + Oracle output: 2 Crux output: 2 - direct_fnmut2: OK (2.19s) - Compiling and running oracle program (0.19s) - Oracle output: 7 (2.00s) - Crux output: 7 - fo: OK (2.22s) + direct_fnmut2: OK (0.18s) Compiling and running oracle program (0.17s) - Oracle output: 29 (2.06s) + Oracle output: 7 + Crux output: 7 + fo: OK (0.18s) + Compiling and running oracle program (0.18s) + Oracle output: 29 Crux output: 29 - unique_borrow: OK (2.19s) + unique_borrow: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 3 (2.02s) + Oracle output: 3 Crux output: 3 - conv_fnonce_fn: OK (2.16s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (1.99s) + conv_fnonce_fn: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: 2 Crux output: 2 - fnptr_fn: OK (2.12s) + fnptr_fn: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 4 (1.96s) + Oracle output: 4 Crux output: 4 - direct_fnmut: OK (2.14s) - Compiling and running oracle program (0.17s) - Oracle output: 7 (1.98s) + direct_fnmut: OK (0.19s) + Compiling and running oracle program (0.18s) + Oracle output: 7 Crux output: 7 - dispatch_fnmut: OK (2.19s) + dispatch_fnmut: OK (0.18s) Compiling and running oracle program (0.17s) - Oracle output: 7 (2.02s) + Oracle output: 7 (0.01s) Crux output: 7 - poly: OK (2.19s) - Compiling and running oracle program (0.17s) - Oracle output: 3 (2.03s) + poly: OK (0.16s) + Compiling and running oracle program (0.16s) + Oracle output: 3 Crux output: 3 - direct_fn: OK (2.15s) - Compiling and running oracle program (0.18s) - Oracle output: 2 (1.98s) + direct_fn: OK (0.17s) + Compiling and running oracle program (0.16s) + Oracle output: 2 Crux output: 2 - fnptr_closure: OK (2.23s) - Compiling and running oracle program (0.17s) - Oracle output: 7 (2.06s) + fnptr_closure: OK (0.20s) + Compiling and running oracle program (0.20s) + Oracle output: 7 Crux output: 7 - fnonce1: OK (2.13s) + fnonce1: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 3 (1.97s) + Oracle output: 3 Crux output: 3 - ref_fnmut: OK (2.15s) - Compiling and running oracle program (0.17s) - Oracle output: 7 (1.99s) + ref_fnmut: OK (0.17s) + Compiling and running oracle program (0.16s) + Oracle output: 7 Crux output: 7 vec_deque - pop: FAIL (3.06s) - Compiling and running oracle program (0.19s) - Oracle output: [5, 4, 3, 1, 2] (2.87s) + pop: FAIL (0.92s) + Compiling and running oracle program (0.18s) + Oracle output: [5, 4, 3, 1, 2] (0.73s) Crux output: failures: @@ -2062,9 +1928,9 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/vec_deque.pop/' to rerun this test only. - iter_clone: FAIL (3.07s) - Compiling and running oracle program (0.19s) - Oracle output: [1, 2, 3, 2, 3] (2.88s) + iter_clone: FAIL (0.97s) + Compiling and running oracle program (0.21s) + Oracle output: [1, 2, 3, 2, 3] (0.76s) Crux output: failures: @@ -2079,9 +1945,9 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/iter_clone/' to rerun this test only. - push: FAIL (3.02s) - Compiling and running oracle program (0.20s) - Oracle output: [4, 1, 2, 3, 5] (2.82s) + push: FAIL (0.89s) + Compiling and running oracle program (0.18s) + Oracle output: [4, 1, 2, 3, 5] (0.72s) Crux output: failures: @@ -2096,9 +1962,9 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/vec_deque.push/' to rerun this test only. - retain: FAIL (3.03s) - Compiling and running oracle program (0.18s) - Oracle output: [1, 4] (2.85s) + retain: FAIL (0.93s) + Compiling and running oracle program (0.17s) + Oracle output: [1, 4] (0.75s) Crux output: failures: @@ -2113,9 +1979,9 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/retain/' to rerun this test only. - rotate_left: FAIL (3.04s) - Compiling and running oracle program (0.18s) - Oracle output: [3, 4, 5, 1, 2] (2.86s) + rotate_left: FAIL (0.97s) + Compiling and running oracle program (0.20s) + Oracle output: [3, 4, 5, 1, 2] (0.77s) Crux output: failures: @@ -2130,9 +1996,9 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/rotate_left/' to rerun this test only. - rotate_right: FAIL (3.14s) - Compiling and running oracle program (0.20s) - Oracle output: [4, 5, 1, 2, 3] (2.93s) + rotate_right: FAIL (0.91s) + Compiling and running oracle program (0.18s) + Oracle output: [4, 5, 1, 2, 3] (0.73s) Crux output: failures: @@ -2148,13 +2014,13 @@ FAIL (0.06s) Use -p '/rotate_right/' to rerun this test only. slice - len: OK (2.21s) + len: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 5 (2.04s) + Oracle output: 5 Crux output: 5 - mut_range: FAIL (2.77s) - Compiling and running oracle program (0.19s) - Oracle output: 86 (2.58s) + mut_range: FAIL (0.80s) + Compiling and running oracle program (0.17s) + Oracle output: 86 (0.63s) Crux output: failures: @@ -2169,9 +2035,9 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/mut_range/' to rerun this test only. - range_len: FAIL (2.74s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (2.57s) + range_len: FAIL (0.72s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (0.55s) Crux output: failures: @@ -2186,9 +2052,9 @@ FAIL (0.06s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..slice.range_len"' to rerun this test only. - range_len_mut: FAIL (2.82s) + range_len_mut: FAIL (0.76s) Compiling and running oracle program (0.18s) - Oracle output: 1 (2.63s) + Oracle output: 1 (0.59s) Crux output: failures: @@ -2203,17 +2069,17 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/range_len_mut/' to rerun this test only. - mk_and_proj: OK (2.15s) + mk_and_proj: OK (0.17s) Compiling and running oracle program (0.17s) - Oracle output: 42 (1.98s) + Oracle output: 42 Crux output: 42 - swap: OK (2.20s) - Compiling and running oracle program (0.18s) - Oracle output: 2001 (2.02s) + swap: OK (0.20s) + Compiling and running oracle program (0.17s) + Oracle output: 2001 (0.03s) Crux output: 2001 - eq: FAIL (2.20s) + eq: FAIL (0.19s) Compiling and running oracle program (0.16s) - Oracle output: () (2.04s) + Oracle output: () (0.03s) Crux output: failures: @@ -2228,9 +2094,9 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/slice.eq/' to rerun this test only. - iter_mut: FAIL (2.28s) + iter_mut: FAIL (0.21s) Compiling and running oracle program (0.17s) - Oracle output: () (2.10s) + Oracle output: () (0.05s) Crux output: failures: @@ -2247,9 +2113,9 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/iter_mut/' to rerun this test only. - get: FAIL (2.36s) - Compiling and running oracle program (0.17s) - Oracle output: true (2.19s) + get: FAIL (0.20s) + Compiling and running oracle program (0.16s) + Oracle output: true (0.04s) Crux output: failures: @@ -2264,17 +2130,17 @@ FAIL (0.06s) crux doesn't match oracle Use -p '/slice.get/' to rerun this test only. - mut: OK (2.30s) + mut: OK (0.18s) Compiling and running oracle program (0.17s) - Oracle output: 42 (2.13s) + Oracle output: 42 Crux output: 42 - last: OK (2.24s) - Compiling and running oracle program (0.17s) - Oracle output: 3 (2.07s) + last: OK (0.17s) + Compiling and running oracle program (0.16s) + Oracle output: 3 Crux output: 3 crux symbolic Output testing - mux_init_mut: FAIL (2.10s) + mux_init_mut: FAIL (0.10s) files test/symb_eval/refs/mux_init_mut.good and test/symb_eval/refs/mux_init_mut.out differ; test/symb_eval/refs/mux_init_mut.out contains: test mux_init_mut/ba0f083a::crux_test[0]: FAILED @@ -2288,7 +2154,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/mux_init_mut/' to rerun this test only. - mux_init_imm: FAIL (2.17s) + mux_init_imm: FAIL (0.11s) files test/symb_eval/refs/mux_init_imm.good and test/symb_eval/refs/mux_init_imm.out differ; test/symb_eval/refs/mux_init_imm.out contains: test mux_init_imm/f397f768::crux_test[0]: FAILED @@ -2302,7 +2168,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/mux_init_imm/' to rerun this test only. - early_fail: FAIL (2.14s) + early_fail: FAIL (0.13s) files test/symb_eval/crux/early_fail.good and test/symb_eval/crux/early_fail.out differ; test/symb_eval/crux/early_fail.out contains: test early_fail/aaff3446::fail1[0]: FAILED test early_fail/aaff3446::fail2[0]: FAILED @@ -2322,7 +2188,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/early_fail/' to rerun this test only. - mixed_fail: FAIL (2.16s) + mixed_fail: FAIL (0.17s) files test/symb_eval/crux/mixed_fail.good and test/symb_eval/crux/mixed_fail.out differ; test/symb_eval/crux/mixed_fail.out contains: test mixed_fail/927be8d8::fail1[0]: FAILED test mixed_fail/927be8d8::fail2[0]: FAILED @@ -2354,7 +2220,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/mixed_fail/' to rerun this test only. - multi: FAIL (2.11s) + multi: FAIL (0.16s) files test/symb_eval/crux/multi.good and test/symb_eval/crux/multi.out differ; test/symb_eval/crux/multi.out contains: test multi/6273896c::fail1[0]: FAILED test multi/6273896c::fail2[0]: FAILED @@ -2380,7 +2246,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/Output testing.multi/' to rerun this test only. - fail_return: FAIL (2.16s) + fail_return: FAIL (0.14s) files test/symb_eval/crux/fail_return.good and test/symb_eval/crux/fail_return.out differ; test/symb_eval/crux/fail_return.out contains: test fail_return/bbd9602c::fail1[0]: FAILED test fail_return/bbd9602c::fail2[0]: FAILED @@ -2400,7 +2266,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/fail_return/' to rerun this test only. - no_conc: FAIL (2.17s) + no_conc: FAIL (0.13s) files test/symb_eval/concretize/no_conc.good and test/symb_eval/concretize/no_conc.out differ; test/symb_eval/concretize/no_conc.out contains: test no_conc/5b356412::crux_test[0]: FAILED @@ -2414,7 +2280,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/no_conc/' to rerun this test only. - conc: FAIL (2.10s) + conc: FAIL (0.13s) files test/symb_eval/concretize/conc.good and test/symb_eval/concretize/conc.out differ; test/symb_eval/concretize/conc.out contains: test conc/b85a9e47::crux_test[0]: FAILED @@ -2428,7 +2294,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.conc"' to rerun this test only. - array: FAIL (2.14s) + array: FAIL (0.13s) files test/symb_eval/concretize/array.good and test/symb_eval/concretize/array.out differ; test/symb_eval/concretize/array.out contains: test array/b294edd1::crux_test[0]: FAILED @@ -2442,7 +2308,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.array"' to rerun this test only. - assert_ok: FAIL (2.85s) + assert_ok: FAIL (0.78s) files test/symb_eval/concretize/assert_ok.good and test/symb_eval/concretize/assert_ok.out differ; test/symb_eval/concretize/assert_ok.out contains: test assert_ok/223e01b8::crux_test[0]: FAILED @@ -2456,7 +2322,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/assert_ok/' to rerun this test only. - assert: FAIL (2.88s) + assert: FAIL (0.73s) files test/symb_eval/concretize/assert.good and test/symb_eval/concretize/assert.out differ; test/symb_eval/concretize/assert.out contains: test assert/4d446bb2::crux_test[0]: FAILED @@ -2470,7 +2336,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.assert"' to rerun this test only. - bytes2: FAIL (2.18s) + bytes2: FAIL (0.17s) files test/symb_eval/crypto/bytes2.good and test/symb_eval/crypto/bytes2.out differ; test/symb_eval/crypto/bytes2.out contains: test bytes2/cec6fb92::f[0]: FAILED @@ -2484,7 +2350,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/bytes2/' to rerun this test only. - double: FAIL (2.15s) + double: FAIL (0.11s) files test/symb_eval/crypto/double.good and test/symb_eval/crypto/double.out differ; test/symb_eval/crypto/double.out contains: test double/da43fdac::f[0]: FAILED @@ -2498,7 +2364,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/double/' to rerun this test only. - ffs: FAIL (2.09s) + ffs: FAIL (0.13s) files test/symb_eval/crypto/ffs.good and test/symb_eval/crypto/ffs.out differ; test/symb_eval/crypto/ffs.out contains: test ffs/be5003b6::f[0]: FAILED @@ -2512,7 +2378,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/Output testing.ffs/' to rerun this test only. - bytes: FAIL (2.19s) + bytes: FAIL (0.17s) files test/symb_eval/crypto/bytes.good and test/symb_eval/crypto/bytes.out differ; test/symb_eval/crypto/bytes.out contains: test bytes/39b68e7d::f[0]: FAILED @@ -2526,7 +2392,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.bytes"' to rerun this test only. - bad_symb1: FAIL (2.16s) + bad_symb1: FAIL (0.14s) files test/symb_eval/overrides/bad_symb1.good and test/symb_eval/overrides/bad_symb1.out differ; test/symb_eval/overrides/bad_symb1.out contains: test bad_symb1/6eace5d9::crux_test[0]: FAILED @@ -2540,7 +2406,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/bad_symb1/' to rerun this test only. - bad_symb2: FAIL (2.10s) + bad_symb2: FAIL (0.13s) files test/symb_eval/overrides/bad_symb2.good and test/symb_eval/overrides/bad_symb2.out differ; test/symb_eval/overrides/bad_symb2.out contains: test bad_symb2/0e44e20a::crux_test[0]: FAILED @@ -2554,7 +2420,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/bad_symb2/' to rerun this test only. - override3: FAIL (2.07s) + override3: FAIL (0.11s) files test/symb_eval/overrides/override3.good and test/symb_eval/overrides/override3.out differ; test/symb_eval/overrides/override3.out contains: test override3/406699f3::f[0]: FAILED @@ -2568,7 +2434,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/override3/' to rerun this test only. - override1: FAIL (2.03s) + override1: FAIL (0.02s) files test/symb_eval/overrides/override1.good and test/symb_eval/overrides/override1.out differ; test/symb_eval/overrides/override1.out contains: test override1/84f90b58::f[0]: FAILED @@ -2582,7 +2448,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/override1/' to rerun this test only. - override4: FAIL (2.13s) + override4: FAIL (0.13s) files test/symb_eval/overrides/override4.good and test/symb_eval/overrides/override4.out differ; test/symb_eval/overrides/override4.out contains: test override4/2a387b8d::f[0]: FAILED @@ -2596,7 +2462,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/override4/' to rerun this test only. - override_rust: FAIL (2.15s) + override_rust: FAIL (0.11s) files test/symb_eval/overrides/override_rust.good and test/symb_eval/overrides/override_rust.out differ; test/symb_eval/overrides/override_rust.out contains: test override_rust/81c3ff88::crux_test[0]: FAILED @@ -2610,7 +2476,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/override_rust/' to rerun this test only. - override2: FAIL (2.12s) + override2: FAIL (0.11s) files test/symb_eval/overrides/override2.good and test/symb_eval/overrides/override2.out differ; test/symb_eval/overrides/override2.out contains: test override2/7910119c::f[0]: FAILED @@ -2624,7 +2490,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/override2/' to rerun this test only. - override5: FAIL (2.05s) + override5: FAIL (0.11s) files test/symb_eval/overrides/override5.good and test/symb_eval/overrides/override5.out differ; test/symb_eval/overrides/override5.out contains: test override5/5f8d6467::f[0]: FAILED @@ -2638,7 +2504,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/override5/' to rerun this test only. - mux: FAIL (2.10s) + mux: FAIL (0.11s) files test/symb_eval/fnptr/mux.good and test/symb_eval/fnptr/mux.out differ; test/symb_eval/fnptr/mux.out contains: test mux/3918b96f::crux_test[0]: FAILED @@ -2652,13 +2518,35 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.mux"' to rerun this test only. - checked_mul_signed: FAIL - test/symb_eval/num/checked_mul_signed.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/num/checked_mul_signed.rs) + checked_mul_signed: FAIL (1.93s) + files test/symb_eval/num/checked_mul_signed.good and test/symb_eval/num/checked_mul_signed.out differ; test/symb_eval/num/checked_mul_signed.out contains: + test checked_mul_signed/aea9eef4::crux_test[0]: returned 44, FAILED + + failures: + + ---- checked_mul_signed/aea9eef4::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/symb_eval/num/checked_mul_signed.rs:4:5: 4:13: error: in checked_mul_signed/aea9eef4::crux_test[0] + [Crux] attempt to compute `const -100_i8 * move _2`, which would overflow + + [Crux] Overall status: Invalid. + Use -p '/checked_mul_signed/' to rerun this test only. - checked_mul: FAIL (0.05s) - test/symb_eval/num/checked_mul.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/num/checked_mul.rs) + checked_mul: FAIL (2.02s) + files test/symb_eval/num/checked_mul.good and test/symb_eval/num/checked_mul.out differ; test/symb_eval/num/checked_mul.out contains: + test checked_mul/840048c9::crux_test[0]: returned 44, FAILED + + failures: + + ---- checked_mul/840048c9::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/symb_eval/num/checked_mul.rs:4:5: 4:12: error: in checked_mul/840048c9::crux_test[0] + [Crux] attempt to compute `const 100_u8 * move _2`, which would overflow + + [Crux] Overall status: Invalid. + Use -p '$0=="crux-mir.crux symbolic.Output testing.checked_mul"' to rerun this test only. - checked_div: FAIL (2.06s) + checked_div: FAIL (0.02s) files test/symb_eval/num/checked_div.good and test/symb_eval/num/checked_div.out differ; test/symb_eval/num/checked_div.out contains: test checked_div/258a4803::crux_test[0]: returned 65, FAILED @@ -2678,10 +2566,21 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/checked_div/' to rerun this test only. - checked_add: FAIL (0.05s) - test/symb_eval/num/checked_add.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/num/checked_add.rs) + checked_add: FAIL (1.94s) + files test/symb_eval/num/checked_add.good and test/symb_eval/num/checked_add.out differ; test/symb_eval/num/checked_add.out contains: + test checked_add/b2604487::crux_test[0]: returned 44, FAILED + + failures: + + ---- checked_add/b2604487::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/symb_eval/num/checked_add.rs:5:5: 5:12: error: in checked_add/b2604487::crux_test[0] + [Crux] attempt to compute `const 100_u8 + move _2`, which would overflow + + [Crux] Overall status: Invalid. + Use -p '/checked_add/' to rerun this test only. - test1: FAIL (2.39s) + test1: FAIL (0.23s) files test/symb_eval/scalar/test1.good and test/symb_eval/scalar/test1.out differ; test/symb_eval/scalar/test1.out contains: test test1/b6abe038::f[0]: FAILED @@ -2695,7 +2594,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/test1/' to rerun this test only. - pop: FAIL (2.19s) + pop: FAIL (0.13s) files test/symb_eval/vector/pop.good and test/symb_eval/vector/pop.out differ; test/symb_eval/vector/pop.out contains: test pop/34bcb08d::f[0]: FAILED @@ -2709,7 +2608,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/Output testing.pop/' to rerun this test only. - as_mut_slice: FAIL (2.16s) + as_mut_slice: FAIL (0.13s) files test/symb_eval/vector/as_mut_slice.good and test/symb_eval/vector/as_mut_slice.out differ; test/symb_eval/vector/as_mut_slice.out contains: test as_mut_slice/7ce532be::f[0]: FAILED @@ -2723,7 +2622,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/as_mut_slice/' to rerun this test only. - push: FAIL (2.14s) + push: FAIL (0.12s) files test/symb_eval/vector/push.good and test/symb_eval/vector/push.out differ; test/symb_eval/vector/push.out contains: test push/9484b802::f[0]: FAILED @@ -2737,7 +2636,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/Output testing.push/' to rerun this test only. - copy_from_slice: FAIL (2.24s) + copy_from_slice: FAIL (0.13s) files test/symb_eval/vector/copy_from_slice.good and test/symb_eval/vector/copy_from_slice.out differ; test/symb_eval/vector/copy_from_slice.out contains: test copy_from_slice/891db15e::f[0]: FAILED @@ -2751,7 +2650,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/copy_from_slice/' to rerun this test only. - new: FAIL (2.18s) + new: FAIL (0.12s) files test/symb_eval/vector/new.good and test/symb_eval/vector/new.out differ; test/symb_eval/vector/new.out contains: test new/7c232b74::f[0]: FAILED @@ -2765,7 +2664,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.new"' to rerun this test only. - mut: FAIL (2.06s) + mut: FAIL (0.12s) files test/symb_eval/vector/mut.good and test/symb_eval/vector/mut.out differ; test/symb_eval/vector/mut.out contains: test mut/6449c4fb::f[0]: FAILED @@ -2779,7 +2678,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/Output testing.mut/' to rerun this test only. - concat: FAIL (2.13s) + concat: FAIL (0.12s) files test/symb_eval/vector/concat.good and test/symb_eval/vector/concat.out differ; test/symb_eval/vector/concat.out contains: test concat/8ea0a338::f[0]: FAILED @@ -2793,7 +2692,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/concat/' to rerun this test only. - split_at: FAIL (2.08s) + split_at: FAIL (0.13s) files test/symb_eval/vector/split_at.good and test/symb_eval/vector/split_at.out differ; test/symb_eval/vector/split_at.out contains: test split_at/096c0611::f[0]: FAILED @@ -2807,7 +2706,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/split_at/' to rerun this test only. - replicate: FAIL (2.08s) + replicate: FAIL (0.13s) files test/symb_eval/vector/replicate.good and test/symb_eval/vector/replicate.out differ; test/symb_eval/vector/replicate.out contains: test replicate/776d13b1::f[0]: FAILED @@ -2821,7 +2720,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/replicate/' to rerun this test only. - as_slice: FAIL (2.11s) + as_slice: FAIL (0.13s) files test/symb_eval/vector/as_slice.good and test/symb_eval/vector/as_slice.out differ; test/symb_eval/vector/as_slice.out contains: test as_slice/a27e71ac::f[0]: FAILED @@ -2835,7 +2734,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/as_slice/' to rerun this test only. - array_mut: FAIL (2.06s) + array_mut: FAIL (0.11s) files test/symb_eval/mux/array_mut.good and test/symb_eval/mux/array_mut.out differ; test/symb_eval/mux/array_mut.out contains: test array_mut/57ba5703::crux_test[0]: FAILED @@ -2849,7 +2748,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/array_mut/' to rerun this test only. - array: FAIL (2.27s) + array: FAIL (0.11s) files test/symb_eval/mux/array.good and test/symb_eval/mux/array.out differ; test/symb_eval/mux/array.out contains: test array/0ee8d2fc::crux_test[0]: FAILED @@ -2863,7 +2762,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.array"' to rerun this test only. - extend_bytes: FAIL (2.17s) + extend_bytes: FAIL (0.13s) files test/symb_eval/bytes/extend_bytes.good and test/symb_eval/bytes/extend_bytes.out differ; test/symb_eval/bytes/extend_bytes.out contains: test extend_bytes/3e4d1139::f[0]: FAILED @@ -2877,7 +2776,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/extend_bytes/' to rerun this test only. - put: FAIL (2.58s) + put: FAIL (0.58s) files test/symb_eval/bytes/put.good and test/symb_eval/bytes/put.out differ; test/symb_eval/bytes/put.out contains: test put/da785c5a::f[0]: FAILED @@ -2891,7 +2790,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.put"' to rerun this test only. - split_to: FAIL (2.58s) + split_to: FAIL (0.60s) files test/symb_eval/bytes/split_to.good and test/symb_eval/bytes/split_to.out differ; test/symb_eval/bytes/split_to.out contains: test split_to/99272bb6::f[0]: FAILED @@ -2905,7 +2804,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/split_to/' to rerun this test only. - new: FAIL (2.16s) + new: FAIL (0.10s) files test/symb_eval/bytes/new.good and test/symb_eval/bytes/new.out differ; test/symb_eval/bytes/new.out contains: test new/6f028dc4::f[0]: FAILED @@ -2919,7 +2818,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.new"' to rerun this test only. - split_off: FAIL (2.59s) + split_off: FAIL (0.57s) files test/symb_eval/bytes/split_off.good and test/symb_eval/bytes/split_off.out differ; test/symb_eval/bytes/split_off.out contains: test split_off/4bbd6335::f[0]: FAILED @@ -2933,7 +2832,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/split_off/' to rerun this test only. - sym_len: FAIL (2.72s) + sym_len: FAIL (0.62s) files test/symb_eval/bytes/sym_len.good and test/symb_eval/bytes/sym_len.out differ; test/symb_eval/bytes/sym_len.out contains: test sym_len/c70884d6::f[0]: FAILED @@ -2947,7 +2846,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/sym_len/' to rerun this test only. - put_overflow: FAIL (2.56s) + put_overflow: FAIL (0.63s) files test/symb_eval/bytes/put_overflow.good and test/symb_eval/bytes/put_overflow.out differ; test/symb_eval/bytes/put_overflow.out contains: test put_overflow/26c5f05d::f[0]: FAILED @@ -2961,22 +2860,22 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/put_overflow/' to rerun this test only. - vec_cursor_read: FAIL (2.52s) + vec_cursor_read: FAIL (0.43s) test/symb_eval/io/vec_cursor_read.out: withFile: user error (JSON Decoding of test/symb_eval/io/vec_cursor_read.all.mir failed: Error in $.statics[60].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) Use -p '/vec_cursor_read/' to rerun this test only. - vec_write: FAIL (2.54s) + vec_write: FAIL (0.48s) standalone use of `dyn` is not supported: TyDynamic core/5a1f6960::error[0]::Error[0]::_trait56353a7b840403b4[0] CallStack (from HasCallStack): error, called at src/Mir/TransTy.hs:225:25 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/Trans.hs:815:12 in crucible-mir-0.1-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:883:5 in crucible-mir-0.1-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:935:30 in crucible-mir-0.1-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1147:11 in crucible-mir-0.1-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1746:68 in crucible-mir-0.1-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1755:28 in crucible-mir-0.1-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1795:10 in crucible-mir-0.1-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1841:24 in crucible-mir-0.1-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2275:30 in crucible-mir-0.1-inplace:Mir.Trans + tyToRepr, called at src/Mir/Trans.hs:833:12 in crucible-mir-0.1-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:901:5 in crucible-mir-0.1-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:953:30 in crucible-mir-0.1-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1165:11 in crucible-mir-0.1-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1764:68 in crucible-mir-0.1-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1773:28 in crucible-mir-0.1-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1813:10 in crucible-mir-0.1-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1859:24 in crucible-mir-0.1-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2293:30 in crucible-mir-0.1-inplace:Mir.Trans transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate translateMIR, called at src/Mir/Language.hs:255:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '/vec_write/' to rerun this test only. @@ -2986,7 +2885,7 @@ FAIL (0.06s) read: FAIL test/symb_eval/byteorder/read.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/read.rs) Use -p '/Output testing.read/' to rerun this test only. - slice_mut: FAIL (2.12s) + slice_mut: FAIL (0.12s) files test/symb_eval/array/slice_mut.good and test/symb_eval/array/slice_mut.out differ; test/symb_eval/array/slice_mut.out contains: test slice_mut/9abd2b5d::crux_test[0]: FAILED @@ -3000,7 +2899,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/slice_mut/' to rerun this test only. - basic: FAIL (2.10s) + basic: FAIL (0.13s) files test/symb_eval/array/basic.good and test/symb_eval/array/basic.out differ; test/symb_eval/array/basic.out contains: test basic/79cad869::crux_test[0]: FAILED @@ -3014,7 +2913,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/Output testing.basic/' to rerun this test only. - slice: FAIL (2.20s) + slice: FAIL (0.15s) files test/symb_eval/array/slice.good and test/symb_eval/array/slice.out differ; test/symb_eval/array/slice.out contains: test slice/0e7bff2f::crux_test[0]: FAILED @@ -3028,7 +2927,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.slice"' to rerun this test only. - mux_slice: FAIL (2.16s) + mux_slice: FAIL (0.15s) files test/symb_eval/array/mux_slice.good and test/symb_eval/array/mux_slice.out differ; test/symb_eval/array/mux_slice.out contains: test mux_slice/508aacc9::crux_test[0]: FAILED @@ -3042,7 +2941,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/mux_slice/' to rerun this test only. - mux: FAIL (2.12s) + mux: FAIL (0.11s) files test/symb_eval/enum/mux.good and test/symb_eval/enum/mux.out differ; test/symb_eval/enum/mux.out contains: test mux/2bd00921::test[0]: FAILED @@ -3056,7 +2955,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.mux"' to rerun this test only. - uninit_read: FAIL (2.23s) + uninit_read: FAIL (0.12s) files test/symb_eval/alloc/uninit_read.good and test/symb_eval/alloc/uninit_read.out differ; test/symb_eval/alloc/uninit_read.out contains: test uninit_read/e9c1be4c::crux_test[0]: FAILED @@ -3073,21 +2972,14 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/uninit_read/' to rerun this test only. - valid_read: FAIL (2.19s) + valid_read: FAIL (0.14s) files test/symb_eval/alloc/valid_read.good and test/symb_eval/alloc/valid_read.out differ; test/symb_eval/alloc/valid_read.out contains: - test valid_read/b81b29bc::crux_test[0]: FAILED - - failures: + test valid_read/b81b29bc::crux_test[0]: returned 45, ok - ---- valid_read/b81b29bc::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/int_macros.rs:460:54: 460:57 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:293:5: 298:27: error: in core/5a1f6960::num[0]::{impl#6}[0]::unchecked_add[0] - [Crux] Translation error in core/5a1f6960::num[0]::{impl#6}[0]::unchecked_add[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_inste14122fababf7d68[0] - - [Crux] Overall status: Invalid. + [Crux] Overall status: Valid. Use -p '/valid_read/' to rerun this test only. - out_of_bounds: FAIL (2.17s) + out_of_bounds: FAIL (0.14s) files test/symb_eval/alloc/out_of_bounds.good and test/symb_eval/alloc/out_of_bounds.out differ; test/symb_eval/alloc/out_of_bounds.out contains: test out_of_bounds/a4641c13::crux_test[0]: FAILED @@ -3095,27 +2987,30 @@ FAIL (0.06s) ---- out_of_bounds/a4641c13::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/int_macros.rs:460:54: 460:57 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:293:5: 298:27: error: in core/5a1f6960::num[0]::{impl#6}[0]::unchecked_add[0] - [Crux] Translation error in core/5a1f6960::num[0]::{impl#6}[0]::unchecked_add[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::unchecked_add[0]::_inste14122fababf7d68[0] + [Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/a4641c13::crux_test[0] + [Crux] vector index out of range: the length is 10 but the index is BV 12 + [Crux] Found counterexample for verification goal + [Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/a4641c13::crux_test[0] + [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. Use -p '/out_of_bounds/' to rerun this test only. - zero_length: FAIL (2.05s) + zero_length: FAIL (0.11s) files test/symb_eval/alloc/zero_length.good and test/symb_eval/alloc/zero_length.out differ; test/symb_eval/alloc/zero_length.out contains: test zero_length/a53db669::crux_test[0]: ok [Crux] Overall status: Valid. Use -p '/zero_length/' to rerun this test only. - downcast: FAIL (2.06s) + downcast: FAIL (0.10s) files test/symb_eval/any/downcast.good and test/symb_eval/any/downcast.out differ; test/symb_eval/any/downcast.out contains: test downcast/cc755b1e::crux_test[0]: returned 1, ok [Crux] Overall status: Valid. Use -p '$0=="crux-mir.crux symbolic.Output testing.downcast"' to rerun this test only. - downcast_fail: FAIL (2.11s) + downcast_fail: FAIL (0.10s) files test/symb_eval/any/downcast_fail.good and test/symb_eval/any/downcast_fail.out differ; test/symb_eval/any/downcast_fail.out contains: test downcast_fail/adccf677::crux_test[0]: FAILED @@ -3132,7 +3027,7 @@ FAIL (0.06s) conditional: FAIL (0.05s) test/symb_eval/any/conditional.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/any/conditional.rs) Use -p '/conditional/' to rerun this test only. - literals: FAIL (2.10s) + literals: FAIL (0.02s) files test/symb_eval/bitvector/literals.good and test/symb_eval/bitvector/literals.out differ; test/symb_eval/bitvector/literals.out contains: test literals/bc9c2ca9::crux_test[0]: FAILED @@ -3146,7 +3041,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/literals/' to rerun this test only. - arith: FAIL (2.16s) + arith: FAIL (0.17s) files test/symb_eval/bitvector/arith.good and test/symb_eval/bitvector/arith.out differ; test/symb_eval/bitvector/arith.out contains: test arith/58d00074::crux_test[0]: FAILED @@ -3160,7 +3055,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/Output testing.arith/' to rerun this test only. - leading_zeros: FAIL (2.16s) + leading_zeros: FAIL (0.16s) files test/symb_eval/bitvector/leading_zeros.good and test/symb_eval/bitvector/leading_zeros.out differ; test/symb_eval/bitvector/leading_zeros.out contains: test leading_zeros/5d113916::crux_test[0]: FAILED @@ -3174,7 +3069,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/leading_zeros/' to rerun this test only. - overflowing_sub: FAIL (2.07s) + overflowing_sub: FAIL (0.14s) files test/symb_eval/bitvector/overflowing_sub.good and test/symb_eval/bitvector/overflowing_sub.out differ; test/symb_eval/bitvector/overflowing_sub.out contains: test overflowing_sub/0e7bfc8b::crux_test[0]: FAILED @@ -3188,7 +3083,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/overflowing_sub/' to rerun this test only. - cmp: FAIL (2.14s) + cmp: FAIL (0.13s) files test/symb_eval/bitvector/cmp.good and test/symb_eval/bitvector/cmp.out differ; test/symb_eval/bitvector/cmp.out contains: test cmp/de4d4043::crux_test[0]: FAILED @@ -3202,7 +3097,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/Output testing.cmp/' to rerun this test only. - symbolic: FAIL (2.00s) + symbolic: FAIL (0.04s) files test/symb_eval/bitvector/symbolic.good and test/symb_eval/bitvector/symbolic.out differ; test/symb_eval/bitvector/symbolic.out contains: test symbolic/6a0ffa6e::crux_test[0]: FAILED @@ -3216,7 +3111,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/Output testing.symbolic/' to rerun this test only. - from_to: FAIL (2.04s) + from_to: FAIL (0.02s) files test/symb_eval/bitvector/from_to.good and test/symb_eval/bitvector/from_to.out differ; test/symb_eval/bitvector/from_to.out contains: test from_to/dcb07b82::crux_test[0]: FAILED @@ -3230,7 +3125,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/from_to/' to rerun this test only. - clone: FAIL (2.23s) + clone: FAIL (0.24s) files test/symb_eval/vec/clone.good and test/symb_eval/vec/clone.out differ; test/symb_eval/vec/clone.out contains: test clone/dfc182b4::f[0]: FAILED @@ -3244,7 +3139,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/Output testing.clone/' to rerun this test only. - sort_by_key: FAIL (3.08s) + sort_by_key: FAIL (0.97s) files test/symb_eval/vec/sort_by_key.good and test/symb_eval/vec/sort_by_key.out differ; test/symb_eval/vec/sort_by_key.out contains: test sort_by_key/ea977c8a::f[0]: FAILED @@ -3258,7 +3153,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/sort_by_key/' to rerun this test only. - into_iter: FAIL (2.90s) + into_iter: FAIL (0.82s) files test/symb_eval/vec/into_iter.good and test/symb_eval/vec/into_iter.out differ; test/symb_eval/vec/into_iter.out contains: test into_iter/e3aefb85::f[0]: FAILED @@ -3272,7 +3167,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/into_iter/' to rerun this test only. - macro: FAIL (2.30s) + macro: FAIL (0.11s) files test/symb_eval/vec/macro.good and test/symb_eval/vec/macro.out differ; test/symb_eval/vec/macro.out contains: test macro/e7284038::f[0]: FAILED @@ -3286,7 +3181,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/Output testing.macro/' to rerun this test only. - construct: FAIL (2.17s) + construct: FAIL (0.12s) files test/symb_eval/sym_bytes/construct.good and test/symb_eval/sym_bytes/construct.out differ; test/symb_eval/sym_bytes/construct.out contains: test construct/63aa86a0::crux_test[0]: FAILED @@ -3300,7 +3195,7 @@ FAIL (0.06s) [Crux] Overall status: Invalid. Use -p '/construct/' to rerun this test only. - deserialize: FAIL (2.23s) + deserialize: FAIL (0.14s) files test/symb_eval/sym_bytes/deserialize.good and test/symb_eval/sym_bytes/deserialize.out differ; test/symb_eval/sym_bytes/deserialize.out contains: test deserialize/40308d9e::crux_test[0]: FAILED @@ -3330,7 +3225,7 @@ FAIL (0.06s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (8.12s) +OK (0.08s) coverage: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | @@ -3345,7 +3240,7 @@ FAIL (0.06s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (2.28s) +FAIL (0.14s) files test/coverage/coverage.good and test/coverage/coverage.out differ; test/coverage/coverage.out contains: warning: branch condition never has value true ┌─ lib/core/src/fmt/mod.rs:400:12 @@ -3375,7 +3270,7 @@ FAIL (0.06s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (2.32s) +FAIL (0.20s) files test/coverage/coverage_try.good and test/coverage/coverage_try.out differ; test/coverage/coverage_try.out contains: warning: branch condition never has value true ┌─ lib/core/src/fmt/mod.rs:400:12 @@ -3405,7 +3300,7 @@ FAIL (0.06s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (2.04s) +OK (0.08s) coverage_loop: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | @@ -3420,13 +3315,13 @@ FAIL (0.06s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (2.19s) +FAIL (0.10s) files test/coverage/coverage_loop.good and test/coverage/coverage_loop.out differ; test/coverage/coverage_loop.out contains: - warning: branch condition never has value false - ┌─ /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/iter/range.rs:621:12 - │ - 621 │ if self.start < self.end { - │ ^^^^^^^^^^^^^^^^^^^^^ + warning: branch condition never has value true + ┌─ test/coverage/coverage_loop.rs:12:19 + │ + 12 │ } else if i > 99 { + │ ^^^^^^ Use -p '/coverage_loop/' to rerun this test only. @@ -3444,7 +3339,7 @@ FAIL (0.06s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (2.21s) +FAIL (0.16s) files test/coverage/coverage_shortcircuit.good and test/coverage/coverage_shortcircuit.out differ; test/coverage/coverage_shortcircuit.out contains: warning: branch condition never has value true ┌─ lib/core/src/fmt/mod.rs:400:12 @@ -3474,7 +3369,7 @@ FAIL (0.06s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (1.99s) +OK (0.07s) coverage_match: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | @@ -3489,7 +3384,7 @@ FAIL (0.06s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (2.13s) +FAIL (0.18s) files test/coverage/coverage_match.good and test/coverage/coverage_match.out differ; test/coverage/coverage_match.out contains: warning: branch condition never has value true ┌─ lib/core/src/fmt/mod.rs:400:12 @@ -3519,7 +3414,7 @@ FAIL (0.06s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (2.13s) +FAIL (0.17s) files test/coverage/coverage_macro.good and test/coverage/coverage_macro.out differ; test/coverage/coverage_macro.out contains: warning: branch condition never has value true ┌─ lib/core/src/fmt/mod.rs:400:12 @@ -3536,4 +3431,4 @@ FAIL (0.06s) Use -p '/coverage_macro/' to rerun this test only. -181 out of 339 tests failed (747.81s) +171 out of 339 tests failed (93.19s) From 6aebf2d2d1b2ff2f0f4f1aa51059ccc226936f0a Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 4 May 2023 13:58:33 -0400 Subject: [PATCH 027/114] Reapply "implement HashMap in terms of Vec" Originally from commit 42424ae2a5e6c6d02f2212d9b973ba0e27451ee0 --- crux-mir/lib/Patches.md | 4 + .../std/src/collections/hash/crucible_map.rs | 408 ++++++++++ crux-mir/lib/std/src/collections/hash/map.rs | 725 +----------------- crux-mir/lib/std/src/collections/hash/mod.rs | 2 + crux-mir/lib/std/src/collections/hash/set.rs | 104 +-- crux-mir/lib/std/src/lib.rs | 2 + 6 files changed, 435 insertions(+), 810 deletions(-) create mode 100644 crux-mir/lib/std/src/collections/hash/crucible_map.rs diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index bcd0e3f18..51759b96b 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -23,3 +23,7 @@ identify all of the code that was changed in each patch. * Reimplement `core::fmt` using `crucible::any::Any` (last applied: May 2, 2023) TODO: Describe why this is necessary + +* Implement `HashMap` in terms of `Vec` (last applied: May 4, 2023) + + TODO: Describe why this is necessary diff --git a/crux-mir/lib/std/src/collections/hash/crucible_map.rs b/crux-mir/lib/std/src/collections/hash/crucible_map.rs new file mode 100644 index 000000000..4dd909bb4 --- /dev/null +++ b/crux-mir/lib/std/src/collections/hash/crucible_map.rs @@ -0,0 +1,408 @@ +use crate::borrow::Borrow; +use crate::collections::TryReserveError; +use crate::hash::Hash; +use crate::marker::PhantomData; +use crate::mem; +use crate::slice; +use crate::vec; + +/// A simple Vec-based key/value map. Presents the same API as `hashbrown::HashMap`, so it can be +/// subbed in for `base::HashMap` in the implementation of `std::collections::HashMap`. +#[derive(Clone)] +pub struct HashMap { + items: Vec<(K, V)>, + hasher: S, + _marker: PhantomData, +} + +impl HashMap { + pub fn with_hasher(hash_builder: S) -> HashMap { + Self::with_capacity_and_hasher(0, hash_builder) + } + + pub fn with_capacity_and_hasher(cap: usize, hash_builder: S) -> HashMap { + HashMap { + items: Vec::with_capacity(cap), + hasher: hash_builder, + _marker: PhantomData, + } + } + + pub fn capacity(&self) -> usize { + self.items.capacity() + } + + pub fn iter(&self) -> Iter<'_, K, V> { + Iter { it: self.items.iter() } + } + + pub fn iter_mut(&mut self) -> IterMut<'_, K, V> { + IterMut { it: self.items.iter_mut() } + } + + pub fn len(&self) -> usize { + self.items.len() + } + + pub fn is_empty(&self) -> bool { + self.items.is_empty() + } + + pub fn drain(&mut self) -> Drain<'_, K, V> { + Drain { it: self.items.drain(..) } + } + + pub fn clear(&mut self) { + self.items.clear(); + } + + pub fn hasher(&self) -> &S { + &self.hasher + } + + pub fn retain(&mut self, mut f: F) + where + F: FnMut(&K, &mut V) -> bool, + { + for i in (0 .. self.items.len()).rev() { + let (ref k, ref mut v) = self.items[i]; + if !f(k, v) { + self.items.swap_remove(i); + } + } + } +} + +impl HashMap { + pub fn reserve(&mut self, additional: usize) { + self.items.reserve(additional) + } + + pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.items.try_reserve(additional) + } + + pub fn shrink_to_fit(&mut self) { + self.items.shrink_to_fit(); + } + + pub fn shrink_to(&mut self, min_capacity: usize) { + self.items.shrink_to(min_capacity); + } + + pub fn rustc_entry(&mut self, k: K) -> RustcEntry<'_, K, V> { + match self.items.iter().position(|&(ref k2, _)| k2 == &k) { + Some(idx) => RustcEntry::Occupied(RustcOccupiedEntry { + items: &mut self.items, + idx, + k: Some(k), + }), + None => RustcEntry::Vacant(RustcVacantEntry { + items: &mut self.items, + k, + }), + } + } + + pub fn get(&self, k: &Q) -> Option<&V> + where + K: Borrow, + Q: Hash + Eq, + { + self.get_key_value(k).map(|(_, v)| v) + } + + pub fn get_key_value(&self, k: &Q) -> Option<(&K, &V)> + where + K: Borrow, + Q: Hash + Eq, + { + self.items.iter() + .find(|&(ref k2, _)| k2.borrow() == k) + .map(|&(ref k, ref v)| (k, v)) + } + + pub fn contains_key(&self, k: &Q) -> bool + where + K: Borrow, + Q: Hash + Eq, + { + self.get_key_value(k).is_some() + } + + pub fn get_mut(&mut self, k: &Q) -> Option<&mut V> + where + K: Borrow, + Q: Hash + Eq, + { + self.items.iter_mut() + .find(|&(ref k2, _)| k2.borrow() == k) + .map(|&mut (_, ref mut v)| v) + } + + pub fn insert(&mut self, k: K, v: V) -> Option { + match self.rustc_entry(k) { + RustcEntry::Occupied(mut e) => { Some(e.insert(v)) }, + RustcEntry::Vacant(e) => { + e.insert(v); + None + }, + } + } + + pub fn remove(&mut self, k: &Q) -> Option + where + K: Borrow, + Q: Hash + Eq, + { + self.remove_entry(k).map(|(_, v)| v) + } + + pub fn remove_entry(&mut self, k: &Q) -> Option<(K, V)> + where + K: Borrow, + Q: Hash + Eq, + { + let idx = self.items.iter().position(|&(ref k2, _)| k2.borrow() == k)?; + Some(self.items.swap_remove(idx)) + } +} + + +pub struct Iter<'a, K: 'a, V: 'a> { + it: slice::Iter<'a, (K, V)>, +} + +impl<'a, K, V> Clone for Iter<'a, K, V> { + fn clone(&self) -> Self { + Iter { it: self.it.clone() } + } +} + +impl<'a, K, V> Iterator for Iter<'a, K, V> { + type Item = (&'a K, &'a V); + + fn next(&mut self) -> Option<(&'a K, &'a V)> { + let &(ref k, ref v) = self.it.next()?; + Some((k, v)) + } + + fn size_hint(&self) -> (usize, Option) { + self.it.size_hint() + } +} + +impl<'a, K, V> ExactSizeIterator for Iter<'a, K, V> { + fn len(&self) -> usize { + self.it.len() + } +} + + +pub struct IterMut<'a, K: 'a, V: 'a> { + it: slice::IterMut<'a, (K, V)>, +} + +impl<'a, K, V> IterMut<'a, K, V> { + pub fn rustc_iter(&self) -> Iter<'_, K, V> { + Iter { + it: self.it.as_slice().iter(), + } + } +} + +impl<'a, K, V> Iterator for IterMut<'a, K, V> { + type Item = (&'a K, &'a mut V); + + fn next(&mut self) -> Option<(&'a K, &'a mut V)> { + let &mut (ref k, ref mut v) = self.it.next()?; + Some((k, v)) + } + + fn size_hint(&self) -> (usize, Option) { + self.it.size_hint() + } +} + +impl<'a, K, V> ExactSizeIterator for IterMut<'a, K, V> { + fn len(&self) -> usize { + self.it.len() + } +} + + +impl IntoIterator for HashMap { + type Item = (K, V); + type IntoIter = IntoIter; + + #[inline] + fn into_iter(self) -> IntoIter { + IntoIter { + it: self.items.into_iter(), + } + } +} + +pub struct IntoIter { + it: vec::IntoIter<(K, V)>, +} + +impl IntoIter { + pub fn rustc_iter(&self) -> Iter<'_, K, V> { + Iter { + it: self.it.as_slice().iter(), + } + } +} + +impl Iterator for IntoIter { + type Item = (K, V); + + fn next(&mut self) -> Option<(K, V)> { + self.it.next() + } + + fn size_hint(&self) -> (usize, Option) { + self.it.size_hint() + } +} + +impl ExactSizeIterator for IntoIter { + fn len(&self) -> usize { + self.it.len() + } +} + + +pub struct Drain<'a, K, V> { + it: vec::Drain<'a, (K, V)>, +} + +impl<'a, K, V> Drain<'a, K, V> { + pub fn rustc_iter(&self) -> Iter<'_, K, V> { + Iter { + it: self.it.as_slice().iter(), + } + } +} + +impl<'a, K, V> Iterator for Drain<'a, K, V> { + type Item = (K, V); + + fn next(&mut self) -> Option<(K, V)> { + self.it.next() + } + + fn size_hint(&self) -> (usize, Option) { + self.it.size_hint() + } +} + +impl<'a, K, V> ExactSizeIterator for Drain<'a, K, V> { + fn len(&self) -> usize { + self.it.len() + } +} + + +impl Extend<(K, V)> for HashMap +where + K: Eq + Hash, +{ + fn extend>(&mut self, iter: T) { + for (k, v) in iter { + self.insert(k, v); + } + } +} + +impl<'a, K, V, S> Extend<(&'a K, &'a V)> for HashMap +where + K: Eq + Hash + Copy, + V: Copy, +{ + #[inline] + fn extend>(&mut self, iter: T) { + self.extend(iter.into_iter().map(|(&key, &value)| (key, value))); + } +} + + +pub enum RustcEntry<'a, K: 'a, V: 'a> { + Occupied(RustcOccupiedEntry<'a, K, V>), + Vacant(RustcVacantEntry<'a, K, V>), +} + +pub struct RustcOccupiedEntry<'a, K, V> { + items: &'a mut Vec<(K, V)>, + idx: usize, + k: Option, +} + +pub struct RustcVacantEntry<'a, K, V> { + items: &'a mut Vec<(K, V)>, + k: K, +} + +impl<'a, K, V> RustcOccupiedEntry<'a, K, V> { + pub fn key(&self) -> &K { + &self.items[self.idx].0 + } + + pub fn remove_entry(self) -> (K, V) { + self.items.swap_remove(self.idx) + } + + pub fn get(&self) -> &V { + &self.items[self.idx].1 + } + + pub fn get_mut(&mut self) -> &mut V { + &mut self.items[self.idx].1 + } + + pub fn into_mut(self) -> &'a mut V { + &mut self.items[self.idx].1 + } + + pub fn insert(&mut self, v: V) -> V { + mem::replace(self.get_mut(), v) + } + + pub fn remove(self) -> V { + self.remove_entry().1 + } + + pub fn replace_entry(self, v: V) -> (K, V) { + mem::replace(&mut self.items[self.idx], (self.k.unwrap(), v)) + } + + pub fn replace_key(self) -> K { + mem::replace(&mut self.items[self.idx].0, self.k.unwrap()) + } +} + +impl<'a, K, V> RustcVacantEntry<'a, K, V> { + pub fn key(&self) -> &K { + &self.k + } + + pub fn into_key(self) -> K { + self.k + } + + pub fn insert(self, v: V) -> &'a mut V { + self.items.push((self.k, v)); + &mut self.items.last_mut().unwrap().1 + } + + pub fn insert_entry(self, v: V) -> RustcOccupiedEntry<'a, K, V> { + let idx = self.items.len(); + self.items.push((self.k, v)); + RustcOccupiedEntry { + items: self.items, + idx, + k: None, + } + } +} diff --git a/crux-mir/lib/std/src/collections/hash/map.rs b/crux-mir/lib/std/src/collections/hash/map.rs index df4903588..0c574aa76 100644 --- a/crux-mir/lib/std/src/collections/hash/map.rs +++ b/crux-mir/lib/std/src/collections/hash/map.rs @@ -3,19 +3,16 @@ mod tests; use self::Entry::*; -use hashbrown::hash_map as base; +use super::crucible_map as base; use crate::borrow::Borrow; -use crate::cell::Cell; use crate::collections::TryReserveError; -use crate::collections::TryReserveErrorKind; use crate::error::Error; use crate::fmt::{self, Debug}; #[allow(deprecated)] use crate::hash::{BuildHasher, Hash, Hasher, SipHasher13}; use crate::iter::FusedIterator; use crate::ops::Index; -use crate::sys; /// A [hash map] implemented with quadratic probing and SIMD lookup. /// @@ -280,8 +277,7 @@ impl HashMap { /// ``` #[inline] #[stable(feature = "hashmap_build_hasher", since = "1.7.0")] - #[rustc_const_unstable(feature = "const_collections_with_hasher", issue = "102575")] - pub const fn with_hasher(hash_builder: S) -> HashMap { + pub fn with_hasher(hash_builder: S) -> HashMap { HashMap { base: base::HashMap::with_hasher(hash_builder) } } @@ -617,51 +613,6 @@ impl HashMap { Drain { base: self.base.drain() } } - /// Creates an iterator which uses a closure to determine if an element should be removed. - /// - /// If the closure returns true, the element is removed from the map and yielded. - /// If the closure returns false, or panics, the element remains in the map and will not be - /// yielded. - /// - /// Note that `drain_filter` lets you mutate every value in the filter closure, regardless of - /// whether you choose to keep or remove it. - /// - /// If the iterator is only partially consumed or not consumed at all, each of the remaining - /// elements will still be subjected to the closure and removed and dropped if it returns true. - /// - /// It is unspecified how many more elements will be subjected to the closure - /// if a panic occurs in the closure, or a panic occurs while dropping an element, - /// or if the `DrainFilter` value is leaked. - /// - /// # Examples - /// - /// Splitting a map into even and odd keys, reusing the original map: - /// - /// ``` - /// #![feature(hash_drain_filter)] - /// use std::collections::HashMap; - /// - /// let mut map: HashMap = (0..8).map(|x| (x, x)).collect(); - /// let drained: HashMap = map.drain_filter(|k, _v| k % 2 == 0).collect(); - /// - /// let mut evens = drained.keys().copied().collect::>(); - /// let mut odds = map.keys().copied().collect::>(); - /// evens.sort(); - /// odds.sort(); - /// - /// assert_eq!(evens, vec![0, 2, 4, 6]); - /// assert_eq!(odds, vec![1, 3, 5, 7]); - /// ``` - #[inline] - #[rustc_lint_query_instability] - #[unstable(feature = "hash_drain_filter", issue = "59618")] - pub fn drain_filter(&mut self, pred: F) -> DrainFilter<'_, K, V, F> - where - F: FnMut(&K, &mut V) -> bool, - { - DrainFilter { base: self.base.drain_filter(pred) } - } - /// Retains only the elements specified by the predicate. /// /// In other words, remove all pairs `(k, v)` for which `f(&k, &mut v)` returns `false`. @@ -907,119 +858,6 @@ where self.base.get_key_value(k) } - /// Attempts to get mutable references to `N` values in the map at once. - /// - /// Returns an array of length `N` with the results of each query. For soundness, at most one - /// mutable reference will be returned to any value. `None` will be returned if any of the - /// keys are duplicates or missing. - /// - /// # Examples - /// - /// ``` - /// #![feature(map_many_mut)] - /// use std::collections::HashMap; - /// - /// let mut libraries = HashMap::new(); - /// libraries.insert("Bodleian Library".to_string(), 1602); - /// libraries.insert("Athenæum".to_string(), 1807); - /// libraries.insert("Herzogin-Anna-Amalia-Bibliothek".to_string(), 1691); - /// libraries.insert("Library of Congress".to_string(), 1800); - /// - /// let got = libraries.get_many_mut([ - /// "Athenæum", - /// "Library of Congress", - /// ]); - /// assert_eq!( - /// got, - /// Some([ - /// &mut 1807, - /// &mut 1800, - /// ]), - /// ); - /// - /// // Missing keys result in None - /// let got = libraries.get_many_mut([ - /// "Athenæum", - /// "New York Public Library", - /// ]); - /// assert_eq!(got, None); - /// - /// // Duplicate keys result in None - /// let got = libraries.get_many_mut([ - /// "Athenæum", - /// "Athenæum", - /// ]); - /// assert_eq!(got, None); - /// ``` - #[inline] - #[unstable(feature = "map_many_mut", issue = "97601")] - pub fn get_many_mut(&mut self, ks: [&Q; N]) -> Option<[&'_ mut V; N]> - where - K: Borrow, - Q: Hash + Eq, - { - self.base.get_many_mut(ks) - } - - /// Attempts to get mutable references to `N` values in the map at once, without validating that - /// the values are unique. - /// - /// Returns an array of length `N` with the results of each query. `None` will be returned if - /// any of the keys are missing. - /// - /// For a safe alternative see [`get_many_mut`](Self::get_many_mut). - /// - /// # Safety - /// - /// Calling this method with overlapping keys is *[undefined behavior]* even if the resulting - /// references are not used. - /// - /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html - /// - /// # Examples - /// - /// ``` - /// #![feature(map_many_mut)] - /// use std::collections::HashMap; - /// - /// let mut libraries = HashMap::new(); - /// libraries.insert("Bodleian Library".to_string(), 1602); - /// libraries.insert("Athenæum".to_string(), 1807); - /// libraries.insert("Herzogin-Anna-Amalia-Bibliothek".to_string(), 1691); - /// libraries.insert("Library of Congress".to_string(), 1800); - /// - /// let got = libraries.get_many_mut([ - /// "Athenæum", - /// "Library of Congress", - /// ]); - /// assert_eq!( - /// got, - /// Some([ - /// &mut 1807, - /// &mut 1800, - /// ]), - /// ); - /// - /// // Missing keys result in None - /// let got = libraries.get_many_mut([ - /// "Athenæum", - /// "New York Public Library", - /// ]); - /// assert_eq!(got, None); - /// ``` - #[inline] - #[unstable(feature = "map_many_mut", issue = "97601")] - pub unsafe fn get_many_unchecked_mut( - &mut self, - ks: [&Q; N], - ) -> Option<[&'_ mut V; N]> - where - K: Borrow, - Q: Hash + Eq, - { - self.base.get_many_unchecked_mut(ks) - } - /// Returns `true` if the map contains a value for the specified key. /// /// The key may be any borrowed form of the map's key type, but @@ -1192,69 +1030,6 @@ where } } -impl HashMap -where - S: BuildHasher, -{ - /// Creates a raw entry builder for the HashMap. - /// - /// Raw entries provide the lowest level of control for searching and - /// manipulating a map. They must be manually initialized with a hash and - /// then manually searched. After this, insertions into a vacant entry - /// still require an owned key to be provided. - /// - /// Raw entries are useful for such exotic situations as: - /// - /// * Hash memoization - /// * Deferring the creation of an owned key until it is known to be required - /// * Using a search key that doesn't work with the Borrow trait - /// * Using custom comparison logic without newtype wrappers - /// - /// Because raw entries provide much more low-level control, it's much easier - /// to put the HashMap into an inconsistent state which, while memory-safe, - /// will cause the map to produce seemingly random results. Higher-level and - /// more foolproof APIs like `entry` should be preferred when possible. - /// - /// In particular, the hash used to initialized the raw entry must still be - /// consistent with the hash of the key that is ultimately stored in the entry. - /// This is because implementations of HashMap may need to recompute hashes - /// when resizing, at which point only the keys are available. - /// - /// Raw entries give mutable access to the keys. This must not be used - /// to modify how the key would compare or hash, as the map will not re-evaluate - /// where the key should go, meaning the keys may become "lost" if their - /// location does not reflect their state. For instance, if you change a key - /// so that the map now contains keys which compare equal, search may start - /// acting erratically, with two keys randomly masking each other. Implementations - /// are free to assume this doesn't happen (within the limits of memory-safety). - #[inline] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn raw_entry_mut(&mut self) -> RawEntryBuilderMut<'_, K, V, S> { - RawEntryBuilderMut { map: self } - } - - /// Creates a raw immutable entry builder for the HashMap. - /// - /// Raw entries provide the lowest level of control for searching and - /// manipulating a map. They must be manually initialized with a hash and - /// then manually searched. - /// - /// This is useful for - /// * Hash memoization - /// * Using a search key that doesn't work with the Borrow trait - /// * Using custom comparison logic without newtype wrappers - /// - /// Unless you are in such a situation, higher-level and more foolproof APIs like - /// `get` should be preferred. - /// - /// Immutable raw entries have very limited use; you might instead want `raw_entry_mut`. - #[inline] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn raw_entry(&self) -> RawEntryBuilder<'_, K, V, S> { - RawEntryBuilder { map: self } - } -} - #[stable(feature = "rust1", since = "1.0.0")] impl Clone for HashMap where @@ -1577,32 +1352,6 @@ impl<'a, K, V> Drain<'a, K, V> { } } -/// A draining, filtering iterator over the entries of a `HashMap`. -/// -/// This `struct` is created by the [`drain_filter`] method on [`HashMap`]. -/// -/// [`drain_filter`]: HashMap::drain_filter -/// -/// # Example -/// -/// ``` -/// #![feature(hash_drain_filter)] -/// -/// use std::collections::HashMap; -/// -/// let mut map = HashMap::from([ -/// ("a", 1), -/// ]); -/// let iter = map.drain_filter(|_k, v| *v % 2 == 0); -/// ``` -#[unstable(feature = "hash_drain_filter", issue = "59618")] -pub struct DrainFilter<'a, K, V, F> -where - F: FnMut(&K, &mut V) -> bool, -{ - base: base::DrainFilter<'a, K, V, F>, -} - /// A mutable iterator over the values of a `HashMap`. /// /// This `struct` is created by the [`values_mut`] method on [`HashMap`]. See its @@ -1669,404 +1418,6 @@ pub struct IntoValues { inner: IntoIter, } -/// A builder for computing where in a HashMap a key-value pair would be stored. -/// -/// See the [`HashMap::raw_entry_mut`] docs for usage examples. -#[unstable(feature = "hash_raw_entry", issue = "56167")] -pub struct RawEntryBuilderMut<'a, K: 'a, V: 'a, S: 'a> { - map: &'a mut HashMap, -} - -/// A view into a single entry in a map, which may either be vacant or occupied. -/// -/// This is a lower-level version of [`Entry`]. -/// -/// This `enum` is constructed through the [`raw_entry_mut`] method on [`HashMap`], -/// then calling one of the methods of that [`RawEntryBuilderMut`]. -/// -/// [`raw_entry_mut`]: HashMap::raw_entry_mut -#[unstable(feature = "hash_raw_entry", issue = "56167")] -pub enum RawEntryMut<'a, K: 'a, V: 'a, S: 'a> { - /// An occupied entry. - Occupied(RawOccupiedEntryMut<'a, K, V, S>), - /// A vacant entry. - Vacant(RawVacantEntryMut<'a, K, V, S>), -} - -/// A view into an occupied entry in a `HashMap`. -/// It is part of the [`RawEntryMut`] enum. -#[unstable(feature = "hash_raw_entry", issue = "56167")] -pub struct RawOccupiedEntryMut<'a, K: 'a, V: 'a, S: 'a> { - base: base::RawOccupiedEntryMut<'a, K, V, S>, -} - -/// A view into a vacant entry in a `HashMap`. -/// It is part of the [`RawEntryMut`] enum. -#[unstable(feature = "hash_raw_entry", issue = "56167")] -pub struct RawVacantEntryMut<'a, K: 'a, V: 'a, S: 'a> { - base: base::RawVacantEntryMut<'a, K, V, S>, -} - -/// A builder for computing where in a HashMap a key-value pair would be stored. -/// -/// See the [`HashMap::raw_entry`] docs for usage examples. -#[unstable(feature = "hash_raw_entry", issue = "56167")] -pub struct RawEntryBuilder<'a, K: 'a, V: 'a, S: 'a> { - map: &'a HashMap, -} - -impl<'a, K, V, S> RawEntryBuilderMut<'a, K, V, S> -where - S: BuildHasher, -{ - /// Creates a `RawEntryMut` from the given key. - #[inline] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn from_key(self, k: &Q) -> RawEntryMut<'a, K, V, S> - where - K: Borrow, - Q: Hash + Eq, - { - map_raw_entry(self.map.base.raw_entry_mut().from_key(k)) - } - - /// Creates a `RawEntryMut` from the given key and its hash. - #[inline] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn from_key_hashed_nocheck(self, hash: u64, k: &Q) -> RawEntryMut<'a, K, V, S> - where - K: Borrow, - Q: Eq, - { - map_raw_entry(self.map.base.raw_entry_mut().from_key_hashed_nocheck(hash, k)) - } - - /// Creates a `RawEntryMut` from the given hash. - #[inline] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn from_hash(self, hash: u64, is_match: F) -> RawEntryMut<'a, K, V, S> - where - for<'b> F: FnMut(&'b K) -> bool, - { - map_raw_entry(self.map.base.raw_entry_mut().from_hash(hash, is_match)) - } -} - -impl<'a, K, V, S> RawEntryBuilder<'a, K, V, S> -where - S: BuildHasher, -{ - /// Access an entry by key. - #[inline] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn from_key(self, k: &Q) -> Option<(&'a K, &'a V)> - where - K: Borrow, - Q: Hash + Eq, - { - self.map.base.raw_entry().from_key(k) - } - - /// Access an entry by a key and its hash. - #[inline] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn from_key_hashed_nocheck(self, hash: u64, k: &Q) -> Option<(&'a K, &'a V)> - where - K: Borrow, - Q: Hash + Eq, - { - self.map.base.raw_entry().from_key_hashed_nocheck(hash, k) - } - - /// Access an entry by hash. - #[inline] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn from_hash(self, hash: u64, is_match: F) -> Option<(&'a K, &'a V)> - where - F: FnMut(&K) -> bool, - { - self.map.base.raw_entry().from_hash(hash, is_match) - } -} - -impl<'a, K, V, S> RawEntryMut<'a, K, V, S> { - /// Ensures a value is in the entry by inserting the default if empty, and returns - /// mutable references to the key and value in the entry. - /// - /// # Examples - /// - /// ``` - /// #![feature(hash_raw_entry)] - /// use std::collections::HashMap; - /// - /// let mut map: HashMap<&str, u32> = HashMap::new(); - /// - /// map.raw_entry_mut().from_key("poneyland").or_insert("poneyland", 3); - /// assert_eq!(map["poneyland"], 3); - /// - /// *map.raw_entry_mut().from_key("poneyland").or_insert("poneyland", 10).1 *= 2; - /// assert_eq!(map["poneyland"], 6); - /// ``` - #[inline] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn or_insert(self, default_key: K, default_val: V) -> (&'a mut K, &'a mut V) - where - K: Hash, - S: BuildHasher, - { - match self { - RawEntryMut::Occupied(entry) => entry.into_key_value(), - RawEntryMut::Vacant(entry) => entry.insert(default_key, default_val), - } - } - - /// Ensures a value is in the entry by inserting the result of the default function if empty, - /// and returns mutable references to the key and value in the entry. - /// - /// # Examples - /// - /// ``` - /// #![feature(hash_raw_entry)] - /// use std::collections::HashMap; - /// - /// let mut map: HashMap<&str, String> = HashMap::new(); - /// - /// map.raw_entry_mut().from_key("poneyland").or_insert_with(|| { - /// ("poneyland", "hoho".to_string()) - /// }); - /// - /// assert_eq!(map["poneyland"], "hoho".to_string()); - /// ``` - #[inline] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn or_insert_with(self, default: F) -> (&'a mut K, &'a mut V) - where - F: FnOnce() -> (K, V), - K: Hash, - S: BuildHasher, - { - match self { - RawEntryMut::Occupied(entry) => entry.into_key_value(), - RawEntryMut::Vacant(entry) => { - let (k, v) = default(); - entry.insert(k, v) - } - } - } - - /// Provides in-place mutable access to an occupied entry before any - /// potential inserts into the map. - /// - /// # Examples - /// - /// ``` - /// #![feature(hash_raw_entry)] - /// use std::collections::HashMap; - /// - /// let mut map: HashMap<&str, u32> = HashMap::new(); - /// - /// map.raw_entry_mut() - /// .from_key("poneyland") - /// .and_modify(|_k, v| { *v += 1 }) - /// .or_insert("poneyland", 42); - /// assert_eq!(map["poneyland"], 42); - /// - /// map.raw_entry_mut() - /// .from_key("poneyland") - /// .and_modify(|_k, v| { *v += 1 }) - /// .or_insert("poneyland", 0); - /// assert_eq!(map["poneyland"], 43); - /// ``` - #[inline] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn and_modify(self, f: F) -> Self - where - F: FnOnce(&mut K, &mut V), - { - match self { - RawEntryMut::Occupied(mut entry) => { - { - let (k, v) = entry.get_key_value_mut(); - f(k, v); - } - RawEntryMut::Occupied(entry) - } - RawEntryMut::Vacant(entry) => RawEntryMut::Vacant(entry), - } - } -} - -impl<'a, K, V, S> RawOccupiedEntryMut<'a, K, V, S> { - /// Gets a reference to the key in the entry. - #[inline] - #[must_use] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn key(&self) -> &K { - self.base.key() - } - - /// Gets a mutable reference to the key in the entry. - #[inline] - #[must_use] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn key_mut(&mut self) -> &mut K { - self.base.key_mut() - } - - /// Converts the entry into a mutable reference to the key in the entry - /// with a lifetime bound to the map itself. - #[inline] - #[must_use = "`self` will be dropped if the result is not used"] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn into_key(self) -> &'a mut K { - self.base.into_key() - } - - /// Gets a reference to the value in the entry. - #[inline] - #[must_use] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn get(&self) -> &V { - self.base.get() - } - - /// Converts the `OccupiedEntry` into a mutable reference to the value in the entry - /// with a lifetime bound to the map itself. - #[inline] - #[must_use = "`self` will be dropped if the result is not used"] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn into_mut(self) -> &'a mut V { - self.base.into_mut() - } - - /// Gets a mutable reference to the value in the entry. - #[inline] - #[must_use] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn get_mut(&mut self) -> &mut V { - self.base.get_mut() - } - - /// Gets a reference to the key and value in the entry. - #[inline] - #[must_use] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn get_key_value(&mut self) -> (&K, &V) { - self.base.get_key_value() - } - - /// Gets a mutable reference to the key and value in the entry. - #[inline] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn get_key_value_mut(&mut self) -> (&mut K, &mut V) { - self.base.get_key_value_mut() - } - - /// Converts the `OccupiedEntry` into a mutable reference to the key and value in the entry - /// with a lifetime bound to the map itself. - #[inline] - #[must_use = "`self` will be dropped if the result is not used"] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn into_key_value(self) -> (&'a mut K, &'a mut V) { - self.base.into_key_value() - } - - /// Sets the value of the entry, and returns the entry's old value. - #[inline] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn insert(&mut self, value: V) -> V { - self.base.insert(value) - } - - /// Sets the value of the entry, and returns the entry's old value. - #[inline] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn insert_key(&mut self, key: K) -> K { - self.base.insert_key(key) - } - - /// Takes the value out of the entry, and returns it. - #[inline] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn remove(self) -> V { - self.base.remove() - } - - /// Take the ownership of the key and value from the map. - #[inline] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn remove_entry(self) -> (K, V) { - self.base.remove_entry() - } -} - -impl<'a, K, V, S> RawVacantEntryMut<'a, K, V, S> { - /// Sets the value of the entry with the `VacantEntry`'s key, - /// and returns a mutable reference to it. - #[inline] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn insert(self, key: K, value: V) -> (&'a mut K, &'a mut V) - where - K: Hash, - S: BuildHasher, - { - self.base.insert(key, value) - } - - /// Sets the value of the entry with the VacantEntry's key, - /// and returns a mutable reference to it. - #[inline] - #[unstable(feature = "hash_raw_entry", issue = "56167")] - pub fn insert_hashed_nocheck(self, hash: u64, key: K, value: V) -> (&'a mut K, &'a mut V) - where - K: Hash, - S: BuildHasher, - { - self.base.insert_hashed_nocheck(hash, key, value) - } -} - -#[unstable(feature = "hash_raw_entry", issue = "56167")] -impl Debug for RawEntryBuilderMut<'_, K, V, S> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("RawEntryBuilder").finish_non_exhaustive() - } -} - -#[unstable(feature = "hash_raw_entry", issue = "56167")] -impl Debug for RawEntryMut<'_, K, V, S> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match *self { - RawEntryMut::Vacant(ref v) => f.debug_tuple("RawEntry").field(v).finish(), - RawEntryMut::Occupied(ref o) => f.debug_tuple("RawEntry").field(o).finish(), - } - } -} - -#[unstable(feature = "hash_raw_entry", issue = "56167")] -impl Debug for RawOccupiedEntryMut<'_, K, V, S> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("RawOccupiedEntryMut") - .field("key", self.key()) - .field("value", self.get()) - .finish_non_exhaustive() - } -} - -#[unstable(feature = "hash_raw_entry", issue = "56167")] -impl Debug for RawVacantEntryMut<'_, K, V, S> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("RawVacantEntryMut").finish_non_exhaustive() - } -} - -#[unstable(feature = "hash_raw_entry", issue = "56167")] -impl Debug for RawEntryBuilder<'_, K, V, S> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("RawEntryBuilder").finish_non_exhaustive() - } -} - /// A view into a single entry in a map, which may either be vacant or occupied. /// /// This `enum` is constructed from the [`entry`] method on [`HashMap`]. @@ -2480,36 +1831,6 @@ where } } -#[unstable(feature = "hash_drain_filter", issue = "59618")] -impl Iterator for DrainFilter<'_, K, V, F> -where - F: FnMut(&K, &mut V) -> bool, -{ - type Item = (K, V); - - #[inline] - fn next(&mut self) -> Option<(K, V)> { - self.base.next() - } - #[inline] - fn size_hint(&self) -> (usize, Option) { - self.base.size_hint() - } -} - -#[unstable(feature = "hash_drain_filter", issue = "59618")] -impl FusedIterator for DrainFilter<'_, K, V, F> where F: FnMut(&K, &mut V) -> bool {} - -#[unstable(feature = "hash_drain_filter", issue = "59618")] -impl<'a, K, V, F> fmt::Debug for DrainFilter<'a, K, V, F> -where - F: FnMut(&K, &mut V) -> bool, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("DrainFilter").finish_non_exhaustive() - } -} - impl<'a, K, V> Entry<'a, K, V> { /// Ensures a value is in the entry by inserting the default if empty, and returns /// a mutable reference to the value in the entry. @@ -3110,26 +2431,7 @@ impl RandomState { #[must_use] #[stable(feature = "hashmap_build_hasher", since = "1.7.0")] pub fn new() -> RandomState { - // Historically this function did not cache keys from the OS and instead - // simply always called `rand::thread_rng().gen()` twice. In #31356 it - // was discovered, however, that because we re-seed the thread-local RNG - // from the OS periodically that this can cause excessive slowdown when - // many hash maps are created on a thread. To solve this performance - // trap we cache the first set of randomly generated keys per-thread. - // - // Later in #36481 it was discovered that exposing a deterministic - // iteration order allows a form of DOS attack. To counter that we - // increment one of the seeds on every RandomState creation, giving - // every corresponding HashMap a different iteration order. - thread_local!(static KEYS: Cell<(u64, u64)> = { - Cell::new(sys::hashmap_random_keys()) - }); - - KEYS.with(|keys| { - let (k0, k1) = keys.get(); - keys.set((k0.wrapping_add(1), k1)); - RandomState { k0, k1 } - }) + RandomState { k0: 1, k1: 2 } } } @@ -3228,25 +2530,8 @@ fn map_entry<'a, K: 'a, V: 'a>(raw: base::RustcEntry<'a, K, V>) -> Entry<'a, K, } #[inline] -pub(super) fn map_try_reserve_error(err: hashbrown::TryReserveError) -> TryReserveError { - match err { - hashbrown::TryReserveError::CapacityOverflow => { - TryReserveErrorKind::CapacityOverflow.into() - } - hashbrown::TryReserveError::AllocError { layout } => { - TryReserveErrorKind::AllocError { layout, non_exhaustive: () }.into() - } - } -} - -#[inline] -fn map_raw_entry<'a, K: 'a, V: 'a, S: 'a>( - raw: base::RawEntryMut<'a, K, V, S>, -) -> RawEntryMut<'a, K, V, S> { - match raw { - base::RawEntryMut::Occupied(base) => RawEntryMut::Occupied(RawOccupiedEntryMut { base }), - base::RawEntryMut::Vacant(base) => RawEntryMut::Vacant(RawVacantEntryMut { base }), - } +fn map_try_reserve_error(err: TryReserveError) -> TryReserveError { + err } #[allow(dead_code)] diff --git a/crux-mir/lib/std/src/collections/hash/mod.rs b/crux-mir/lib/std/src/collections/hash/mod.rs index 348820af5..7d37d0f72 100644 --- a/crux-mir/lib/std/src/collections/hash/mod.rs +++ b/crux-mir/lib/std/src/collections/hash/mod.rs @@ -2,3 +2,5 @@ pub mod map; pub mod set; + +mod crucible_map; diff --git a/crux-mir/lib/std/src/collections/hash/set.rs b/crux-mir/lib/std/src/collections/hash/set.rs index b59f89d32..224f38f81 100644 --- a/crux-mir/lib/std/src/collections/hash/set.rs +++ b/crux-mir/lib/std/src/collections/hash/set.rs @@ -5,12 +5,13 @@ use hashbrown::hash_set as base; use crate::borrow::Borrow; use crate::collections::TryReserveError; +use crate::collections::TryReserveErrorKind; use crate::fmt; use crate::hash::{BuildHasher, Hash}; use crate::iter::{Chain, FusedIterator}; use crate::ops::{BitAnd, BitOr, BitXor, Sub}; -use super::map::{map_try_reserve_error, RandomState}; +use super::map::{RandomState}; // Future Optimization (FIXME!) // ============================ @@ -707,95 +708,6 @@ where self.base.get(value) } - /// Inserts the given `value` into the set if it is not present, then - /// returns a reference to the value in the set. - /// - /// # Examples - /// - /// ``` - /// #![feature(hash_set_entry)] - /// - /// use std::collections::HashSet; - /// - /// let mut set = HashSet::from([1, 2, 3]); - /// assert_eq!(set.len(), 3); - /// assert_eq!(set.get_or_insert(2), &2); - /// assert_eq!(set.get_or_insert(100), &100); - /// assert_eq!(set.len(), 4); // 100 was inserted - /// ``` - #[inline] - #[unstable(feature = "hash_set_entry", issue = "60896")] - pub fn get_or_insert(&mut self, value: T) -> &T { - // Although the raw entry gives us `&mut T`, we only return `&T` to be consistent with - // `get`. Key mutation is "raw" because you're not supposed to affect `Eq` or `Hash`. - self.base.get_or_insert(value) - } - - /// Inserts an owned copy of the given `value` into the set if it is not - /// present, then returns a reference to the value in the set. - /// - /// # Examples - /// - /// ``` - /// #![feature(hash_set_entry)] - /// - /// use std::collections::HashSet; - /// - /// let mut set: HashSet = ["cat", "dog", "horse"] - /// .iter().map(|&pet| pet.to_owned()).collect(); - /// - /// assert_eq!(set.len(), 3); - /// for &pet in &["cat", "dog", "fish"] { - /// let value = set.get_or_insert_owned(pet); - /// assert_eq!(value, pet); - /// } - /// assert_eq!(set.len(), 4); // a new "fish" was inserted - /// ``` - #[inline] - #[unstable(feature = "hash_set_entry", issue = "60896")] - pub fn get_or_insert_owned(&mut self, value: &Q) -> &T - where - T: Borrow, - Q: Hash + Eq + ToOwned, - { - // Although the raw entry gives us `&mut T`, we only return `&T` to be consistent with - // `get`. Key mutation is "raw" because you're not supposed to affect `Eq` or `Hash`. - self.base.get_or_insert_owned(value) - } - - /// Inserts a value computed from `f` into the set if the given `value` is - /// not present, then returns a reference to the value in the set. - /// - /// # Examples - /// - /// ``` - /// #![feature(hash_set_entry)] - /// - /// use std::collections::HashSet; - /// - /// let mut set: HashSet = ["cat", "dog", "horse"] - /// .iter().map(|&pet| pet.to_owned()).collect(); - /// - /// assert_eq!(set.len(), 3); - /// for &pet in &["cat", "dog", "fish"] { - /// let value = set.get_or_insert_with(pet, str::to_owned); - /// assert_eq!(value, pet); - /// } - /// assert_eq!(set.len(), 4); // a new "fish" was inserted - /// ``` - #[inline] - #[unstable(feature = "hash_set_entry", issue = "60896")] - pub fn get_or_insert_with(&mut self, value: &Q, f: F) -> &T - where - T: Borrow, - Q: Hash + Eq, - F: FnOnce(&Q) -> T, - { - // Although the raw entry gives us `&mut T`, we only return `&T` to be consistent with - // `get`. Key mutation is "raw" because you're not supposed to affect `Eq` or `Hash`. - self.base.get_or_insert_with(value, f) - } - /// Returns `true` if `self` has no elements in common with `other`. /// This is equivalent to checking for an empty intersection. /// @@ -1843,3 +1755,15 @@ fn assert_covariance() { d } } + +#[inline] +fn map_try_reserve_error(err: hashbrown::TryReserveError) -> TryReserveError { + match err { + hashbrown::TryReserveError::CapacityOverflow => { + TryReserveErrorKind::CapacityOverflow.into() + } + hashbrown::TryReserveError::AllocError { layout } => { + TryReserveErrorKind::AllocError { layout, non_exhaustive: () }.into() + } + } +} diff --git a/crux-mir/lib/std/src/lib.rs b/crux-mir/lib/std/src/lib.rs index a7e13f5b8..99a258cfe 100644 --- a/crux-mir/lib/std/src/lib.rs +++ b/crux-mir/lib/std/src/lib.rs @@ -308,6 +308,7 @@ #![feature(ptr_as_uninit)] #![feature(raw_os_nonzero)] #![feature(slice_internals)] +#![feature(slice_iter_mut_as_slice)] #![feature(slice_ptr_get)] #![feature(std_internals)] #![feature(str_internals)] @@ -324,6 +325,7 @@ #![feature(new_uninit)] #![feature(thin_box)] #![feature(try_reserve_kind)] +#![feature(vec_drain_as_slice)] #![feature(vec_into_raw_parts)] #![feature(slice_concat_trait)] // From 70e1387210988b51bab09bfe9f2a5843d50cd2d9 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Tue, 9 May 2023 09:37:39 -0400 Subject: [PATCH 028/114] Implement ptr::invalid/invalid_mut in terms of casts --- crux-mir/lib/Patches.md | 8 ++++++++ crux-mir/lib/core/src/ptr/mod.rs | 16 ++++------------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index 51759b96b..b287cdf00 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -27,3 +27,11 @@ identify all of the code that was changed in each patch. * Implement `HashMap` in terms of `Vec` (last applied: May 4, 2023) TODO: Describe why this is necessary + +* Implement `ptr::invalid` and `ptr::invalid_mut` in terms of casts instead of + `transmute` (last applied: May 9, 2023) + + The uses of `transmute` in these two functions are particularly nasty, so we + choose a simpler implementation in terms of casts instead. These uses of + `transmute` are meant to support strict provenance for Rust pointers, but we + ignore this in `crucible-mir`. diff --git a/crux-mir/lib/core/src/ptr/mod.rs b/crux-mir/lib/core/src/ptr/mod.rs index 1ad9af154..a2c72e670 100644 --- a/crux-mir/lib/core/src/ptr/mod.rs +++ b/crux-mir/lib/core/src/ptr/mod.rs @@ -559,13 +559,9 @@ pub const fn null_mut() -> *mut T { #[must_use] #[rustc_const_stable(feature = "stable_things_using_strict_provenance", since = "1.61.0")] #[unstable(feature = "strict_provenance", issue = "95228")] +#[allow(fuzzy_provenance_casts)] pub const fn invalid(addr: usize) -> *const T { - // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. - // We use transmute rather than a cast so tools like Miri can tell that this - // is *not* the same as from_exposed_addr. - // SAFETY: every valid integer is also a valid pointer (as long as you don't dereference that - // pointer). - unsafe { mem::transmute(addr) } + addr as *const T } /// Creates an invalid mutable pointer with the given address. @@ -590,13 +586,9 @@ pub const fn invalid(addr: usize) -> *const T { #[must_use] #[rustc_const_stable(feature = "stable_things_using_strict_provenance", since = "1.61.0")] #[unstable(feature = "strict_provenance", issue = "95228")] +#[allow(fuzzy_provenance_casts)] pub const fn invalid_mut(addr: usize) -> *mut T { - // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. - // We use transmute rather than a cast so tools like Miri can tell that this - // is *not* the same as from_exposed_addr. - // SAFETY: every valid integer is also a valid pointer (as long as you don't dereference that - // pointer). - unsafe { mem::transmute(addr) } + addr as *mut T } /// Convert an address back to a pointer, picking up a previously 'exposed' provenance. From 93c803177ceb96a3af2240df4340b0b08b6d702e Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Tue, 9 May 2023 13:52:43 -0400 Subject: [PATCH 029/114] Reapply "Avoid pointer arithmetic in slice iterators" Originally from commit 69e92cb7c9db05610b4660b31edd7ebfad1d19e8 --- crux-mir/lib/Patches.md | 16 ++++++++++++++++ crux-mir/lib/alloc/src/lib.rs | 1 + crux-mir/lib/alloc/src/rc.rs | 2 +- crux-mir/lib/core/src/crucible/ptr.rs | 4 ++-- crux-mir/lib/core/src/ptr/const_ptr.rs | 19 ++----------------- crux-mir/lib/core/src/ptr/mut_ptr.rs | 19 ++----------------- crux-mir/lib/core/src/slice/iter.rs | 18 +++++++++++++++--- crux-mir/lib/core/src/slice/iter/macros.rs | 18 ++++++------------ 8 files changed, 45 insertions(+), 52 deletions(-) diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index b287cdf00..c004af389 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -35,3 +35,19 @@ identify all of the code that was changed in each patch. choose a simpler implementation in terms of casts instead. These uses of `transmute` are meant to support strict provenance for Rust pointers, but we ignore this in `crucible-mir`. + +* Avoid pointer arithmetic in slice iterators (last applied: May 9, 2023) + + Upstream slice::Iter has `ptr` and `end` fields, and computes the length + by subtracting the two. This is difficult for us to support at the + moment. We can compute the difference between two pointers into the + same array, but we don't have a good way to decide whether two + `Const_RefRoot`s are "the same object" or not. (We could assign them a + `Nonce` for identity, but since the point of `Const_RefRoot` is to + support muxing, we'd have to mux the nonces, which makes things much + more complicated. And we can't simply declare that all `Const_RefRoot`s + containing the same value are the same object, because we don't have a + generic equality test that covers all crucible types.) This commit adds + a simple workaround: include an explicit `len` field, which is updated + in sync with `ptr` and `end`, and avoids the need for pointer + arithmetic. diff --git a/crux-mir/lib/alloc/src/lib.rs b/crux-mir/lib/alloc/src/lib.rs index ca75c3895..9f5d17363 100644 --- a/crux-mir/lib/alloc/src/lib.rs +++ b/crux-mir/lib/alloc/src/lib.rs @@ -115,6 +115,7 @@ #![feature(const_eval_select)] #![feature(const_pin)] #![feature(const_waker)] +#![feature(crucible_intrinsics)] #![feature(cstr_from_bytes_until_nul)] #![feature(dispatch_from_dyn)] #![feature(error_generic_member_access)] diff --git a/crux-mir/lib/alloc/src/rc.rs b/crux-mir/lib/alloc/src/rc.rs index c9aa23fc4..c772324df 100644 --- a/crux-mir/lib/alloc/src/rc.rs +++ b/crux-mir/lib/alloc/src/rc.rs @@ -2219,7 +2219,7 @@ impl Weak { } pub(crate) fn is_dangling(ptr: *mut T) -> bool { - (ptr as *mut ()).addr() == usize::MAX + crucible::ptr::compare_usize(ptr, usize::MAX) } /// Helper type to allow accessing the reference counts without diff --git a/crux-mir/lib/core/src/crucible/ptr.rs b/crux-mir/lib/core/src/crucible/ptr.rs index 2d2866afe..abc14cd2a 100644 --- a/crux-mir/lib/core/src/crucible/ptr.rs +++ b/crux-mir/lib/core/src/crucible/ptr.rs @@ -3,6 +3,6 @@ /// Pointer-to-usize comparison. Unlike `ptr as usize == val`, this works on fat pointers and /// valid pointers too (pointers not created via integer-to-pointer casts). #[unstable(feature = "crucible_intrinsics", issue = "none")] -pub fn compare_usize(ptr: *const T, val: usize) -> bool { - unimplemented!("ptr::compare_usize") +pub const fn compare_usize(ptr: *const T, val: usize) -> bool { + panic!("ptr::compare_usize") } diff --git a/crux-mir/lib/core/src/ptr/const_ptr.rs b/crux-mir/lib/core/src/ptr/const_ptr.rs index 7b1cb5488..935605b79 100644 --- a/crux-mir/lib/core/src/ptr/const_ptr.rs +++ b/crux-mir/lib/core/src/ptr/const_ptr.rs @@ -1,5 +1,6 @@ use super::*; use crate::cmp::Ordering::{self, Equal, Greater, Less}; +use crate::crucible; use crate::intrinsics::{self, const_eval_select}; use crate::mem; use crate::slice::{self, SliceIndex}; @@ -34,23 +35,7 @@ impl *const T { #[rustc_const_unstable(feature = "const_ptr_is_null", issue = "74939")] #[inline] pub const fn is_null(self) -> bool { - #[inline] - fn runtime_impl(ptr: *const u8) -> bool { - ptr.addr() == 0 - } - - #[inline] - const fn const_impl(ptr: *const u8) -> bool { - // Compare via a cast to a thin pointer, so fat pointers are only - // considering their "data" part for null-ness. - match (ptr).guaranteed_eq(null_mut()) { - None => false, - Some(res) => res, - } - } - - // SAFETY: The two versions are equivalent at runtime. - unsafe { const_eval_select((self as *const u8,), const_impl, runtime_impl) } + crucible::ptr::compare_usize(self, 0) } /// Casts to a pointer of another type. diff --git a/crux-mir/lib/core/src/ptr/mut_ptr.rs b/crux-mir/lib/core/src/ptr/mut_ptr.rs index ed1e3bd48..6a4fa75c7 100644 --- a/crux-mir/lib/core/src/ptr/mut_ptr.rs +++ b/crux-mir/lib/core/src/ptr/mut_ptr.rs @@ -1,5 +1,6 @@ use super::*; use crate::cmp::Ordering::{self, Equal, Greater, Less}; +use crate::crucible; use crate::intrinsics::{self, const_eval_select}; use crate::slice::{self, SliceIndex}; @@ -33,23 +34,7 @@ impl *mut T { #[rustc_const_unstable(feature = "const_ptr_is_null", issue = "74939")] #[inline] pub const fn is_null(self) -> bool { - #[inline] - fn runtime_impl(ptr: *mut u8) -> bool { - ptr.addr() == 0 - } - - #[inline] - const fn const_impl(ptr: *mut u8) -> bool { - // Compare via a cast to a thin pointer, so fat pointers are only - // considering their "data" part for null-ness. - match (ptr).guaranteed_eq(null_mut()) { - None => false, - Some(res) => res, - } - } - - // SAFETY: The two versions are equivalent at runtime. - unsafe { const_eval_select((self as *mut u8,), const_impl, runtime_impl) } + crucible::ptr::compare_usize(self, 0) } /// Casts to a pointer of another type. diff --git a/crux-mir/lib/core/src/slice/iter.rs b/crux-mir/lib/core/src/slice/iter.rs index 90ab43d12..fa35df7b7 100644 --- a/crux-mir/lib/core/src/slice/iter.rs +++ b/crux-mir/lib/core/src/slice/iter.rs @@ -62,6 +62,7 @@ pub struct Iter<'a, T: 'a> { end: *const T, // If T is a ZST, this is actually ptr+len. This encoding is picked so that // ptr == end is a quick test for the Iterator being empty, that works // for both ZST and non-ZST. + len: usize, _marker: PhantomData<&'a T>, } @@ -88,7 +89,12 @@ impl<'a, T> Iter<'a, T> { let end = if T::IS_ZST { ptr.wrapping_byte_add(slice.len()) } else { ptr.add(slice.len()) }; - Self { ptr: NonNull::new_unchecked(ptr as *mut T), end, _marker: PhantomData } + Self { + ptr: NonNull::new_unchecked(ptr as *mut T), + end, + len: slice.len(), + _marker: PhantomData, + } } } @@ -140,7 +146,7 @@ iterator! {struct Iter -> *const T, &'a T, const, {/* no mut */}, { impl Clone for Iter<'_, T> { #[inline] fn clone(&self) -> Self { - Iter { ptr: self.ptr, end: self.end, _marker: self._marker } + Iter { ptr: self.ptr, end: self.end, len: self.len, _marker: self._marker } } } @@ -183,6 +189,7 @@ pub struct IterMut<'a, T: 'a> { end: *mut T, // If T is a ZST, this is actually ptr+len. This encoding is picked so that // ptr == end is a quick test for the Iterator being empty, that works // for both ZST and non-ZST. + len: usize, _marker: PhantomData<&'a mut T>, } @@ -224,7 +231,12 @@ impl<'a, T> IterMut<'a, T> { let end = if T::IS_ZST { ptr.wrapping_byte_add(slice.len()) } else { ptr.add(slice.len()) }; - Self { ptr: NonNull::new_unchecked(ptr), end, _marker: PhantomData } + Self { + ptr: NonNull::new_unchecked(ptr), + end, + len: slice.len(), + _marker: PhantomData, + } } } diff --git a/crux-mir/lib/core/src/slice/iter/macros.rs b/crux-mir/lib/core/src/slice/iter/macros.rs index 0fd57b197..95eeb39ec 100644 --- a/crux-mir/lib/core/src/slice/iter/macros.rs +++ b/crux-mir/lib/core/src/slice/iter/macros.rs @@ -5,7 +5,7 @@ macro_rules! is_empty { // The way we encode the length of a ZST iterator, this works both for ZST // and non-ZST. ($self: ident) => { - $self.ptr.as_ptr() as *const T == $self.end + $self.len == 0 }; } @@ -13,17 +13,7 @@ macro_rules! len { ($self: ident) => {{ #![allow(unused_unsafe)] // we're sometimes used within an unsafe block - let start = $self.ptr; - if T::IS_ZST { - // This _cannot_ use `ptr_sub` because we depend on wrapping - // to represent the length of long ZST slice iterators. - $self.end.addr().wrapping_sub(start.as_ptr().addr()) - } else { - // To get rid of some bounds checks (see `position`), we use ptr_sub instead of - // offset_from (Tested by `codegen/slice-position-bounds-check`.) - // SAFETY: by the type invariant pointers are aligned and `start <= end` - unsafe { $self.end.sub_ptr(start.as_ptr()) } - } + $self.len }}; } @@ -73,6 +63,7 @@ macro_rules! iterator { // Unsafe because the offset must not exceed `self.len()`. #[inline(always)] unsafe fn post_inc_start(&mut self, offset: usize) -> * $raw_mut T { + self.len -= offset; if mem::size_of::() == 0 { zst_shrink!(self, offset); self.ptr.as_ptr() @@ -90,6 +81,7 @@ macro_rules! iterator { // Unsafe because the offset must not exceed `self.len()`. #[inline(always)] unsafe fn pre_dec_end(&mut self, offset: usize) -> * $raw_mut T { + self.len -= offset; if T::IS_ZST { zst_shrink!(self, offset); self.ptr.as_ptr() @@ -156,6 +148,7 @@ macro_rules! iterator { fn nth(&mut self, n: usize) -> Option<$elem> { if n >= len!(self) { // This iterator is now empty. + self.len = 0; if T::IS_ZST { // We have to do it this way as `ptr` may never be 0, but `end` // could be (due to wrapping). @@ -360,6 +353,7 @@ macro_rules! iterator { fn nth_back(&mut self, n: usize) -> Option<$elem> { if n >= len!(self) { // This iterator is now empty. + self.len = 0; self.end = self.ptr.as_ptr(); return None; } From e7415d0a67d872580fc54eea0a98c535cc7b08be Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 4 May 2023 13:59:08 -0400 Subject: [PATCH 030/114] Stub out ThreadLocalRef --- crucible-mir/src/Mir/JSON.hs | 1 + crucible-mir/src/Mir/Mir.hs | 3 +++ crucible-mir/src/Mir/Trans.hs | 1 + 3 files changed, 5 insertions(+) diff --git a/crucible-mir/src/Mir/JSON.hs b/crucible-mir/src/Mir/JSON.hs index 35c4edde2..5645856d8 100644 --- a/crucible-mir/src/Mir/JSON.hs +++ b/crucible-mir/src/Mir/JSON.hs @@ -311,6 +311,7 @@ instance FromJSON Rvalue where Just (String "Aggregate") -> Aggregate <$> v .: "akind" <*> v .: "ops" Just (String "ShallowInitBox") -> ShallowInitBox <$> v .: "ptr" <*> v .: "ty" Just (String "CopyForDeref") -> CopyForDeref <$> v .: "place" + Just (String "ThreadLocalRef") -> ThreadLocalRef <$> v .: "def_id" k -> fail $ "unsupported RValue " ++ show k instance FromJSON Terminator where diff --git a/crucible-mir/src/Mir/Mir.hs b/crucible-mir/src/Mir/Mir.hs index c11dcd3d8..b029a739c 100644 --- a/crucible-mir/src/Mir/Mir.hs +++ b/crucible-mir/src/Mir/Mir.hs @@ -335,6 +335,8 @@ data Rvalue = | RAdtAg AdtAg | ShallowInitBox { _sibptr :: Operand, _sibty :: Ty } | CopyForDeref Lvalue + | ThreadLocalRef DefId + -- ^ TODO RGS: Is this right? deriving (Show,Eq, Ord, Generic) data AdtAg = AdtAg { _agadt :: Adt, _avgariant :: Integer, _aops :: [Operand], _adtagty :: Ty } @@ -702,6 +704,7 @@ instance TypeOf Rvalue where typeOf (RAdtAg (AdtAg _ _ _ ty)) = ty typeOf (ShallowInitBox _ ty) = ty typeOf (CopyForDeref lv) = typeOf lv + typeOf (ThreadLocalRef _) = error "TODO RGS typeOf ThreadLocalRef" instance TypeOf Operand where typeOf (Move lv) = typeOf lv diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index 09e8fc3e0..88b3e3749 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -1000,6 +1000,7 @@ evalRval rv@(M.RAdtAg (M.AdtAg adt agv ops ty)) = do -- TODO: are these correct? evalRval rv@(M.ShallowInitBox op ty) = evalOperand op evalRval rv@(M.CopyForDeref lv) = evalLvalue lv +evalRval (M.ThreadLocalRef _) = error "TODO RGS evalRval ThreadLocalRef" evalLvalue :: HasCallStack => M.Lvalue -> MirGenerator h s ret (MirExp s) evalLvalue lv = evalPlace lv >>= readPlace From c5b8f254c762eb32864135e3a8f7bf7e8e87c346 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Tue, 9 May 2023 15:50:55 -0400 Subject: [PATCH 031/114] Permit non-isize-typed enum discriminants --- crucible-mir/src/Mir/Generator.hs | 9 +-- crucible-mir/src/Mir/Intrinsics.hs | 92 ++++++++++++++------------ crucible-mir/src/Mir/JSON.hs | 12 ++-- crucible-mir/src/Mir/Mir.hs | 24 +++++-- crucible-mir/src/Mir/PP.hs | 2 +- crucible-mir/src/Mir/Trans.hs | 17 ++--- crucible-mir/src/Mir/TransTy.hs | 101 +++++++++++++++++------------ crux-mir/src/Mir/Concurrency.hs | 2 +- crux-mir/src/Mir/Language.hs | 9 +-- crux-mir/src/Mir/Overrides.hs | 4 +- 10 files changed, 159 insertions(+), 113 deletions(-) diff --git a/crucible-mir/src/Mir/Generator.hs b/crucible-mir/src/Mir/Generator.hs index a5205804d..0895ffa11 100644 --- a/crucible-mir/src/Mir/Generator.hs +++ b/crucible-mir/src/Mir/Generator.hs @@ -613,11 +613,12 @@ subfieldRef :: subfieldRef ctx ref idx = G.extensionStmt (MirSubfieldRef ctx ref idx) subvariantRef :: - C.CtxRepr ctx -> - R.Expr MIR s (MirReferenceType (RustEnumType ctx)) -> - Index ctx tp -> + C.TypeRepr discrTp -> + C.CtxRepr variantsCtx -> + R.Expr MIR s (MirReferenceType (RustEnumType discrTp variantsCtx)) -> + Index variantsCtx tp -> MirGenerator h s ret (R.Expr MIR s (MirReferenceType tp)) -subvariantRef ctx ref idx = G.extensionStmt (MirSubvariantRef ctx ref idx) +subvariantRef tp ctx ref idx = G.extensionStmt (MirSubvariantRef tp ctx ref idx) subindexRef :: C.TypeRepr tp -> diff --git a/crucible-mir/src/Mir/Intrinsics.hs b/crucible-mir/src/Mir/Intrinsics.hs index fd5194718..8a76ab714 100644 --- a/crucible-mir/src/Mir/Intrinsics.hs +++ b/crucible-mir/src/Mir/Intrinsics.hs @@ -121,22 +121,28 @@ import Unsafe.Coerce -- Rust enum representation -- A Rust enum, whose variants have the types listed in `ctx`. -type RustEnumType ctx = StructType (RustEnumFields ctx) -type RustEnumFields ctx = EmptyCtx ::> IsizeType ::> VariantType ctx +type RustEnumType discrTp variantsCtx = StructType (RustEnumFields discrTp variantsCtx) +type RustEnumFields discrTp variantsCtx = EmptyCtx ::> discrTp ::> VariantType variantsCtx -pattern RustEnumFieldsRepr :: () => ctx' ~ RustEnumFields ctx => CtxRepr ctx -> CtxRepr ctx' -pattern RustEnumFieldsRepr ctx = Empty :> IsizeRepr :> VariantRepr ctx -pattern RustEnumRepr :: () => tp ~ RustEnumType ctx => CtxRepr ctx -> TypeRepr tp -pattern RustEnumRepr ctx = StructRepr (RustEnumFieldsRepr ctx) +data SomeRustEnumRepr where + SomeRustEnumRepr :: + !(TypeRepr discrTp) -> + !(CtxRepr variantsCtx) -> + SomeRustEnumRepr -mkRustEnum :: CtxRepr ctx -> f IsizeType -> f (VariantType ctx) -> App ext f (RustEnumType ctx) -mkRustEnum ctx discr variant = MkStruct (RustEnumFieldsRepr ctx) (Empty :> discr :> variant) +pattern RustEnumFieldsRepr :: () => ctx ~ RustEnumFields discrTp variantsCtx => TypeRepr discrTp -> CtxRepr variantsCtx -> CtxRepr ctx +pattern RustEnumFieldsRepr discrTp variantsCtx = Empty :> discrTp :> VariantRepr variantsCtx +pattern RustEnumRepr :: () => tp ~ RustEnumType discrTp variantsCtx => TypeRepr discrTp -> CtxRepr variantsCtx -> TypeRepr tp +pattern RustEnumRepr discrTp variantsCtx = StructRepr (RustEnumFieldsRepr discrTp variantsCtx) -rustEnumDiscriminant :: f (RustEnumType ctx) -> App ext f IsizeType -rustEnumDiscriminant e = GetStruct e i1of2 IsizeRepr +mkRustEnum :: TypeRepr discrTp -> CtxRepr variantsCtx -> f discrTp -> f (VariantType variantsCtx) -> App ext f (RustEnumType discrTp variantsCtx) +mkRustEnum discrTp variantsCtx discr variant = MkStruct (RustEnumFieldsRepr discrTp variantsCtx) (Empty :> discr :> variant) -rustEnumVariant :: CtxRepr ctx -> f (RustEnumType ctx) -> App ext f (VariantType ctx) -rustEnumVariant ctx e = GetStruct e i2of2 (VariantRepr ctx) +rustEnumDiscriminant :: TypeRepr discrTp -> f (RustEnumType discrTp variantsCtx) -> App ext f discrTp +rustEnumDiscriminant discrTp e = GetStruct e i1of2 discrTp + +rustEnumVariant :: CtxRepr variantsCtx -> f (RustEnumType discrTp variantsCtx) -> App ext f (VariantType variantsCtx) +rustEnumVariant variantsCtx e = GetStruct e i2of2 (VariantRepr variantsCtx) -- Rust usize/isize representation @@ -381,9 +387,10 @@ data MirReferencePath sym :: CrucibleType -> CrucibleType -> Type where !(Index ctx tp) -> MirReferencePath sym tp_base tp Variant_RefPath :: - !(CtxRepr ctx) -> - !(MirReferencePath sym tp_base (RustEnumType ctx)) -> - !(Index ctx tp) -> + !(TypeRepr discrTp) -> + !(CtxRepr variantsCtx) -> + !(MirReferencePath sym tp_base (RustEnumType discrTp variantsCtx)) -> + !(Index variantsCtx tp) -> MirReferencePath sym tp_base tp Index_RefPath :: !(TypeRepr tp) -> @@ -429,7 +436,7 @@ instance IsSymInterface sym => Show (MirReferencePath sym tp tp') where show Empty_RefPath = "Empty_RefPath" show (Any_RefPath tpr p) = "(Any_RefPath " ++ show tpr ++ " " ++ show p ++ ")" show (Field_RefPath ctx p idx) = "(Field_RefPath " ++ show ctx ++ " " ++ show p ++ " " ++ show idx ++ ")" - show (Variant_RefPath ctx p idx) = "(Variant_RefPath " ++ show ctx ++ " " ++ show p ++ " " ++ show idx ++ ")" + show (Variant_RefPath tp ctx p idx) = "(Variant_RefPath " ++ show tp ++ " " ++ show ctx ++ " " ++ show p ++ " " ++ show idx ++ ")" show (Index_RefPath tpr p idx) = "(Index_RefPath " ++ show tpr ++ " " ++ show p ++ " " ++ show (printSymExpr idx) ++ ")" show (Just_RefPath tpr p) = "(Just_RefPath " ++ show tpr ++ " " ++ show p ++ ")" show (VectorAsMirVector_RefPath tpr p) = "(VectorAsMirVector_RefPath " ++ show tpr ++ " " ++ show p ++ ")" @@ -471,10 +478,10 @@ instance OrdSkel (MirReference sym tp) where compareSkelF2 ctx1 idx1 ctx2 idx2 <> cmpPath p1 p2 cmpPath (Field_RefPath _ _ _) _ = LT cmpPath _ (Field_RefPath _ _ _) = GT - cmpPath (Variant_RefPath ctx1 p1 idx1) (Variant_RefPath ctx2 p2 idx2) = + cmpPath (Variant_RefPath _ ctx1 p1 idx1) (Variant_RefPath _ ctx2 p2 idx2) = compareSkelF2 ctx1 idx1 ctx2 idx2 <> cmpPath p1 p2 - cmpPath (Variant_RefPath _ _ _) _ = LT - cmpPath _ (Variant_RefPath _ _ _) = GT + cmpPath (Variant_RefPath _ _ _ _) _ = LT + cmpPath _ (Variant_RefPath _ _ _ _) = GT cmpPath (Index_RefPath tpr1 p1 _) (Index_RefPath tpr2 p2 _) = compareSkelF tpr1 tpr2 <> cmpPath p1 p2 cmpPath (Index_RefPath _ _ _) _ = LT @@ -531,11 +538,12 @@ muxRefPath sym c path1 path2 = case (path1,path2) of , Just Refl <- testEquality f1 f2 -> do p' <- muxRefPath sym c p1 p2 return (Field_RefPath ctx1 p' f1) - (Variant_RefPath ctx1 p1 f1, Variant_RefPath ctx2 p2 f2) - | Just Refl <- testEquality ctx1 ctx2 + (Variant_RefPath tp1 ctx1 p1 f1, Variant_RefPath tp2 ctx2 p2 f2) + | Just Refl <- testEquality tp1 tp2 + , Just Refl <- testEquality ctx1 ctx2 , Just Refl <- testEquality f1 f2 -> do p' <- muxRefPath sym c p1 p2 - return (Variant_RefPath ctx1 p' f1) + return (Variant_RefPath tp1 ctx1 p' f1) (Index_RefPath tp p1 i1, Index_RefPath _ p2 i2) -> do p' <- muxRefPath sym c p1 p2 i' <- lift $ bvIte sym c i1 i2 @@ -871,9 +879,10 @@ data MirStmt :: (CrucibleType -> Type) -> CrucibleType -> Type where !(Index ctx tp) -> MirStmt f (MirReferenceType tp) MirSubvariantRef :: - !(CtxRepr ctx) -> - !(f (MirReferenceType (RustEnumType ctx))) -> - !(Index ctx tp) -> + !(TypeRepr discrTp) -> + !(CtxRepr variantsCtx) -> + !(f (MirReferenceType (RustEnumType discrTp variantsCtx))) -> + !(Index variantsCtx tp) -> MirStmt f (MirReferenceType tp) MirSubindexRef :: !(TypeRepr tp) -> @@ -1033,7 +1042,7 @@ instance TypeApp MirStmt where MirDropRef _ -> UnitRepr MirSubanyRef tp _ -> MirReferenceRepr tp MirSubfieldRef ctx _ idx -> MirReferenceRepr (ctx ! idx) - MirSubvariantRef ctx _ idx -> MirReferenceRepr (ctx ! idx) + MirSubvariantRef _ ctx _ idx -> MirReferenceRepr (ctx ! idx) MirSubindexRef tp _ _ -> MirReferenceRepr tp MirSubjustRef tp _ -> MirReferenceRepr tp MirRef_VectorAsMirVector tp _ -> MirReferenceRepr (MirVectorRepr tp) @@ -1068,7 +1077,7 @@ instance PrettyApp MirStmt where MirDropRef x -> "dropMirRef" <+> pp x MirSubanyRef tpr x -> "subanyRef" <+> pretty tpr <+> pp x MirSubfieldRef _ x idx -> "subfieldRef" <+> pp x <+> viaShow idx - MirSubvariantRef _ x idx -> "subvariantRef" <+> pp x <+> viaShow idx + MirSubvariantRef _ _ x idx -> "subvariantRef" <+> pp x <+> viaShow idx MirSubindexRef _ x idx -> "subindexRef" <+> pp x <+> pp idx MirSubjustRef _ x -> "subjustRef" <+> pp x MirRef_VectorAsMirVector _ v -> "mirRef_vectorAsMirVector" <+> pp v @@ -1247,13 +1256,14 @@ subfieldMirRefLeaf _ (MirReference_Integer _ _) _ = "attempted subfield on the result of an integer-to-pointer cast" subvariantMirRefLeaf :: - CtxRepr ctx -> - MirReference sym (RustEnumType ctx) -> - Index ctx tp -> + TypeRepr discrTp -> + CtxRepr variantsCtx -> + MirReference sym (RustEnumType discrTp variantsCtx) -> + Index variantsCtx tp -> MuxLeafT sym IO (MirReference sym tp) -subvariantMirRefLeaf ctx (MirReference root path) idx = - return $ MirReference root (Variant_RefPath ctx path idx) -subvariantMirRefLeaf _ (MirReference_Integer _ _) _ = +subvariantMirRefLeaf tp ctx (MirReference root path) idx = + return $ MirReference root (Variant_RefPath tp ctx path idx) +subvariantMirRefLeaf _ _ (MirReference_Integer _ _) _ = leafAbort $ GenericSimError $ "attempted subvariant on the result of an integer-to-pointer cast" @@ -1325,7 +1335,7 @@ refPathEq sym (Any_RefPath tpr1 p1) (Any_RefPath tpr2 p2) refPathEq sym (Field_RefPath ctx1 p1 idx1) (Field_RefPath ctx2 p2 idx2) | Just Refl <- testEquality ctx1 ctx2 , Just Refl <- testEquality idx1 idx2 = refPathEq sym p1 p2 -refPathEq sym (Variant_RefPath ctx1 p1 idx1) (Variant_RefPath ctx2 p2 idx2) +refPathEq sym (Variant_RefPath _ ctx1 p1 idx1) (Variant_RefPath _ ctx2 p2 idx2) | Just Refl <- testEquality ctx1 ctx2 , Just Refl <- testEquality idx1 idx2 = refPathEq sym p1 p2 refPathEq sym (Index_RefPath tpr1 p1 idx1) (Index_RefPath tpr2 p2 idx2) @@ -1394,8 +1404,8 @@ reverseRefPath rp = go RrpNil rp go acc Empty_RefPath = acc go acc (Field_RefPath ctx rp idx) = go (Field_RefPath ctx Empty_RefPath idx `RrpCons` acc) rp - go acc (Variant_RefPath ctx rp idx) = - go (Variant_RefPath ctx Empty_RefPath idx `RrpCons` acc) rp + go acc (Variant_RefPath tp ctx rp idx) = + go (Variant_RefPath tp ctx Empty_RefPath idx `RrpCons` acc) rp go acc (Index_RefPath tpr rp idx) = go (Index_RefPath tpr Empty_RefPath idx `RrpCons` acc) rp go acc (Just_RefPath tpr rp) = @@ -1458,7 +1468,7 @@ refPathOverlaps sym path1 path2 = do go (Field_RefPath ctx1 _ idx1 `RrpCons` rrp1) (Field_RefPath ctx2 _ idx2 `RrpCons` rrp2) | Just Refl <- testEquality ctx1 ctx2 , Just Refl <- testEquality idx1 idx2 = go rrp1 rrp2 - go (Variant_RefPath ctx1 _ idx1 `RrpCons` rrp1) (Variant_RefPath ctx2 _ idx2 `RrpCons` rrp2) + go (Variant_RefPath _ ctx1 _ idx1 `RrpCons` rrp1) (Variant_RefPath _ ctx2 _ idx2 `RrpCons` rrp2) | Just Refl <- testEquality ctx1 ctx2 , Just Refl <- testEquality idx1 idx2 = go rrp1 rrp2 go (Index_RefPath tpr1 _ idx1 `RrpCons` rrp1) (Index_RefPath tpr2 _ idx2 `RrpCons` rrp2) @@ -1685,8 +1695,8 @@ execMirStmt stmt s = withBackend ctx $ \bak -> readOnly s $ modifyRefMux bak (subanyMirRefLeaf tp) ref MirSubfieldRef ctx0 (regValue -> ref) idx -> readOnly s $ modifyRefMux bak (\ref' -> subfieldMirRefLeaf ctx0 ref' idx) ref - MirSubvariantRef ctx0 (regValue -> ref) idx -> - readOnly s $ modifyRefMux bak (\ref' -> subvariantMirRefLeaf ctx0 ref' idx) ref + MirSubvariantRef tp0 ctx0 (regValue -> ref) idx -> + readOnly s $ modifyRefMux bak (\ref' -> subvariantMirRefLeaf tp0 ctx0 ref' idx) ref MirSubindexRef tpr (regValue -> ref) (regValue -> idx) -> readOnly s $ modifyRefMux bak (\ref' -> subindexMirRefLeaf tpr ref' idx) ref MirSubjustRef tpr (regValue -> ref) -> @@ -1937,7 +1947,7 @@ adjustRefPath bak iTypes v path0 adj = case path0 of Field_RefPath _ctx path fld -> adjustRefPath bak iTypes v path (\x -> adjustM (\x' -> RV <$> adj (unRV x')) fld x) - Variant_RefPath _ctx path fld -> + Variant_RefPath _ _ctx path fld -> -- TODO: report an error if variant `fld` is not selected adjustRefPath bak iTypes v path (field @1 (\(RV x) -> RV <$> adjustM (\x' -> VB <$> mapM adj (unVB x')) fld x)) @@ -1984,7 +1994,7 @@ readRefPath bak iTypes v = \case Field_RefPath _ctx path fld -> do flds <- readRefPath bak iTypes v path return $ unRV $ flds ! fld - Variant_RefPath ctx path fld -> + Variant_RefPath _ ctx path fld -> do (Empty :> _discr :> RV variant) <- readRefPath bak iTypes v path let msg = GenericSimError $ "attempted to read from wrong variant (" ++ show fld ++ " of " ++ show ctx ++ ")" diff --git a/crucible-mir/src/Mir/JSON.hs b/crucible-mir/src/Mir/JSON.hs index 5645856d8..9e590f598 100644 --- a/crucible-mir/src/Mir/JSON.hs +++ b/crucible-mir/src/Mir/JSON.hs @@ -150,11 +150,11 @@ instance FromJSON Adt where <*> v .: "orig_substs" instance FromJSON AdtKind where - parseJSON x = case x of - String "Struct" -> pure Struct - String "Enum" -> pure Enum - String "Union" -> pure Union - _ -> fail $ "unsupported adt kind " ++ show x + parseJSON = withObject "AdtKind" $ \v -> case lookupKM "kind" v of + Just (String "Struct") -> pure Struct + Just (String "Enum") -> Enum <$> v .: "discr_ty" + Just (String "Union") -> pure Union + mbKind -> fail $ "unsupported adt kind " ++ show mbKind instance FromJSON VariantDiscr where parseJSON = withObject "VariantDiscr" $ \v -> case lookupKM "kind" v of @@ -307,7 +307,7 @@ instance FromJSON Rvalue where Just (String "CheckedBinaryOp") -> CheckedBinaryOp <$> v .: "op" <*> v .: "L" <*> v .: "R" Just (String "NullaryOp") -> NullaryOp <$> v .: "op" <*> v .: "ty" Just (String "UnaryOp") -> UnaryOp <$> v .: "uop" <*> v .: "op" - Just (String "Discriminant") -> Discriminant <$> v .: "val" + Just (String "Discriminant") -> Discriminant <$> v .: "val" <*> v .: "ty" Just (String "Aggregate") -> Aggregate <$> v .: "akind" <*> v .: "ops" Just (String "ShallowInitBox") -> ShallowInitBox <$> v .: "ptr" <*> v .: "ty" Just (String "CopyForDeref") -> CopyForDeref <$> v .: "place" diff --git a/crucible-mir/src/Mir/Mir.hs b/crucible-mir/src/Mir/Mir.hs index b029a739c..40228c91f 100644 --- a/crucible-mir/src/Mir/Mir.hs +++ b/crucible-mir/src/Mir/Mir.hs @@ -35,7 +35,7 @@ import qualified Data.ByteString as B import Data.Map.Strict (Map) import Data.Text (Text) -import Control.Lens (makeLenses, makeWrapped) +import Control.Lens (makeLenses, makePrisms, makeWrapped) import GHC.Generics @@ -184,7 +184,13 @@ data Adt = Adt } deriving (Eq, Ord, Show, Generic) -data AdtKind = Struct | Enum | Union +data AdtKind + = Struct + | Enum { -- | The type of the discriminant used to pick which variant of + -- the enum to use. This type can be of varying sizes depending + -- on how many variants the enum has. + _enumDiscrTy :: Ty } + | Union deriving (Eq, Ord, Show, Generic) data VariantDiscr @@ -330,7 +336,16 @@ data Rvalue = | CheckedBinaryOp { _cbop :: BinOp, _cbop1 :: Operand, _cbop2 :: Operand } | NullaryOp { _nuop :: NullOp, _nty :: Ty } | UnaryOp { _unop :: UnOp, _unoperand :: Operand} - | Discriminant { _dvar :: Lvalue } + | Discriminant { _dvar :: Lvalue, + -- | The type of the discriminant. That is, /not/ the + -- type of the enum itself, but rather the type of the + -- value used to choose which variant of the enum to use. + -- + -- Although this type is also stored in the 'Lvalue', it + -- can only be retrieved in a monadic context. We cache + -- the type here for the benefit of `Rvalue`'s 'TypeOf' + -- instance, which requires a pure context. + _dty :: Ty } | Aggregate { _ak :: AggregateKind, _ops :: [Operand] } | RAdtAg AdtAg | ShallowInitBox { _sibptr :: Operand, _sibty :: Ty } @@ -553,6 +568,7 @@ makeLenses ''MirBody makeLenses ''BasicBlock makeLenses ''BasicBlockData makeLenses ''Adt +makePrisms ''AdtKind makeLenses ''AdtAg makeLenses ''Trait makeLenses ''Static @@ -697,7 +713,7 @@ instance TypeOf Rvalue where in case op of Not -> ty Neg -> ty - typeOf (Discriminant _lv) = TyInt USize + typeOf (Discriminant _lv ty) = ty typeOf (Aggregate (AKArray ty) ops) = TyArray ty (length ops) typeOf (Aggregate AKTuple ops) = TyTuple $ map typeOf ops typeOf (Aggregate AKClosure ops) = TyClosure $ map typeOf ops diff --git a/crucible-mir/src/Mir/PP.hs b/crucible-mir/src/Mir/PP.hs index 8dca8f05b..d331b7d84 100644 --- a/crucible-mir/src/Mir/PP.hs +++ b/crucible-mir/src/Mir/PP.hs @@ -202,7 +202,7 @@ instance Pretty Rvalue where pretty (CheckedBinaryOp a b c) = pretty b <+> pretty a <+> pretty c pretty (NullaryOp a _b) = pretty a pretty (UnaryOp a b) = pretty a <+> pretty b - pretty (Discriminant a) = pretty_fn1 "Discriminant" a + pretty (Discriminant a b) = pretty_fn2 "Discriminant" a b pretty (Aggregate a b) = pretty_fn2 "Aggregate" a b pretty (RAdtAg a) = pretty a pretty (ShallowInitBox ptr ty) = pretty_fn2 "ShallowInitBox" ptr ty diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index 88b3e3749..6a3982640 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -47,6 +47,7 @@ import Control.Monad.ST import Control.Monad.Trans.Class import Control.Lens hiding (op,(|>)) +import qualified Control.Lens.Extras as Lens (is) import Data.Foldable import Data.Bits (shift, shiftL) @@ -955,14 +956,14 @@ evalRval (M.BinaryOp binop op1 op2) = transBinOp binop op1 op2 evalRval (M.CheckedBinaryOp binop op1 op2) = transCheckedBinOp binop op1 op2 evalRval (M.NullaryOp nop nty) = transNullaryOp nop nty evalRval (M.UnaryOp uop op) = transUnaryOp uop op -evalRval (M.Discriminant lv) = do +evalRval (M.Discriminant lv discrTy) = do e <- evalLvalue lv - let ty = typeOf lv - case ty of + let enumTy = typeOf lv + case enumTy of TyAdt aname _ _ -> do adt <- findAdt aname enumDiscriminant adt e - _ -> mirFail $ "tried to access discriminant of non-enum type " ++ show ty + _ -> mirFail $ "tried to access discriminant of non-enum type " ++ show enumTy evalRval (M.Aggregate ak ops) = case ak of M.AKTuple -> do @@ -992,7 +993,7 @@ evalRval rv@(M.RAdtAg (M.AdtAg adt agv ops ty)) = do es <- mapM evalOperand ops case adt^.adtkind of M.Struct -> buildStruct adt es - M.Enum -> buildEnum adt (fromInteger agv) es + M.Enum _ -> buildEnum adt (fromInteger agv) es M.Union -> do mirFail $ "evalRval: Union types are unsupported, for " ++ show (adt ^. adtname) _ -> mirFail $ "evalRval: unsupported type for AdtAg: " ++ show ty @@ -1061,7 +1062,7 @@ evalPlaceProj ty pl@(MirPlace tpr ref NoMeta) (M.PField idx _mirTy) = do adt <- findAdt nm case adt^.adtkind of Struct -> structFieldRef adt idx tpr ref - Enum -> mirFail $ "tried to access field of non-downcast " ++ show ty + Enum _ -> mirFail $ "tried to access field of non-downcast " ++ show ty Union -> mirFail $ "evalPlace (PField, Union) NYI" M.TyDowncast (M.TyAdt nm _ _) i -> do @@ -1657,7 +1658,7 @@ initialValue (M.TyAdt nm _ _) = do let var = M.onlyVariant adt fldExps <- mapM initField (var^.M.vfields) Just <$> buildStruct' adt fldExps - Enum -> do + Enum _ -> do case inhabited (adt ^. adtvariants) of -- Uninhabited enums can't be initialized. [] -> return Nothing @@ -2245,7 +2246,7 @@ checkEq a b kNe kEq mkDiscrMap :: M.Collection -> Map M.AdtName [Integer] mkDiscrMap col = mconcat [ Map.singleton (adt^.M.adtname) (adtIndices adt col) - | adt <- Map.elems $ col^.M.adts, adt^.M.adtkind == Enum ] + | adt <- Map.elems $ col^.M.adts, Lens.is _Enum (adt^.M.adtkind) ] diff --git a/crucible-mir/src/Mir/TransTy.hs b/crucible-mir/src/Mir/TransTy.hs index f64f565a4..a3fa283e8 100644 --- a/crucible-mir/src/Mir/TransTy.hs +++ b/crucible-mir/src/Mir/TransTy.hs @@ -26,6 +26,7 @@ module Mir.TransTy where import Control.Monad import Control.Lens +import qualified Data.BitVector.Sized as BV import Data.List (findIndices) import Data.String (fromString) import qualified Data.Vector as V @@ -61,7 +62,8 @@ import Mir.Intrinsics , pattern MirVectorRepr , SizeBits, pattern UsizeRepr, pattern IsizeRepr , isizeLit - , RustEnumType, pattern RustEnumRepr, mkRustEnum, rustEnumVariant, rustEnumDiscriminant + , RustEnumType, pattern RustEnumRepr, SomeRustEnumRepr(..) + , mkRustEnum, rustEnumVariant, rustEnumDiscriminant , pattern MethodSpecRepr, pattern MethodSpecBuilderRepr , DynRefType) @@ -371,17 +373,21 @@ variantFieldsM' v = do col <- use $ cs . collection return $ variantFields' col v -enumVariants :: TransTyConstraint => M.Collection -> M.Adt -> Some C.CtxRepr -enumVariants col (M.Adt name kind vs _ _ _ _) - | kind /= M.Enum = error $ "expected " ++ show name ++ " to have kind Enum" - | otherwise = reprsToCtx variantReprs $ \repr -> Some repr +enumVariants :: TransTyConstraint => M.Collection -> M.Adt -> SomeRustEnumRepr +enumVariants col (M.Adt name kind vs _ _ _ _) = + case kind of + M.Enum discrTy + | Some discrTpr <- tyToRepr col discrTy + -> reprsToCtx variantReprs $ \variantsCxt -> + SomeRustEnumRepr discrTpr variantsCxt + _ -> error $ "expected " ++ show name ++ " to have kind Enum" where variantReprs :: [Some C.TypeRepr] variantReprs = map (\v -> viewSome (\ctx -> Some $ C.StructRepr ctx) $ variantFields col v) vs -enumVariantsM :: TransTyConstraint => M.Adt -> MirGenerator h s ret (Some C.CtxRepr) +enumVariantsM :: TransTyConstraint => M.Adt -> MirGenerator h s ret SomeRustEnumRepr enumVariantsM adt = do col <- use $ cs . collection return $ enumVariants col adt @@ -551,28 +557,28 @@ adjustAnyE tpr f me = do buildAnyE tpr y -readEnumVariant :: C.CtxRepr ctx -> Ctx.Index ctx tp -> - R.Expr MIR s (RustEnumType ctx) -> MirGenerator h s ret (R.Expr MIR s tp) -readEnumVariant ctx idx e = do +readEnumVariant :: C.TypeRepr discrTp -> C.CtxRepr variantsCtx -> Ctx.Index variantsCtx tp -> + R.Expr MIR s (RustEnumType discrTp variantsCtx) -> MirGenerator h s ret (R.Expr MIR s tp) +readEnumVariant tp ctx idx e = do let tpr = ctx Ctx.! idx let optVal = R.App $ E.ProjectVariant ctx idx $ R.App $ rustEnumVariant ctx e readJust' tpr optVal $ "readEnumVariant: wrong variant; expected " ++ show idx -buildEnumVariant :: C.CtxRepr ctx -> Ctx.Index ctx tp -> - R.Expr MIR s tp -> MirGenerator h s ret (R.Expr MIR s (RustEnumType ctx)) -buildEnumVariant ctx idx e = do - let discr = R.App $ isizeLit $ fromIntegral $ Ctx.indexVal idx +buildEnumVariant :: C.TypeRepr discrTp -> C.CtxRepr variantsCtx -> Ctx.Index variantsCtx tp -> + R.Expr MIR s tp -> MirGenerator h s ret (R.Expr MIR s (RustEnumType discrTp variantsCtx)) +buildEnumVariant tp ctx idx e = do + discr <- enumDiscrLit tp $ fromIntegral $ Ctx.indexVal idx let var = R.App $ E.InjectVariant ctx idx e - return $ R.App $ mkRustEnum ctx discr var + return $ R.App $ mkRustEnum tp ctx (R.App discr) var -adjustEnumVariant :: C.CtxRepr ctx -> Ctx.Index ctx tp -> +adjustEnumVariant :: C.TypeRepr discrTp -> C.CtxRepr variantsCtx -> Ctx.Index variantsCtx tp -> (R.Expr MIR s tp -> MirGenerator h s ret (R.Expr MIR s tp)) -> - R.Expr MIR s (RustEnumType ctx) -> MirGenerator h s ret (R.Expr MIR s (RustEnumType ctx)) -adjustEnumVariant ctx idx f e = do - x <- readEnumVariant ctx idx e + R.Expr MIR s (RustEnumType discrTp variantsCtx) -> MirGenerator h s ret (R.Expr MIR s (RustEnumType discrTp variantsCtx)) +adjustEnumVariant tp ctx idx f e = do + x <- readEnumVariant tp ctx idx e y <- f x - buildEnumVariant ctx idx y + buildEnumVariant tp ctx idx y readStructField :: C.CtxRepr ctx -> Ctx.Index ctx tp -> @@ -751,7 +757,8 @@ mapStructField adt i f me = do f' me -data EnumInfo = forall ctx ctx' tp tp'. EnumInfo +data EnumInfo = forall discrTp ctx ctx' tp tp'. EnumInfo + (C.TypeRepr discrTp) (C.CtxRepr ctx) (Ctx.Index ctx (C.StructType ctx')) (C.CtxRepr ctx') @@ -767,7 +774,11 @@ checkStructType _ = Nothing enumInfo :: M.Adt -> Int -> Int -> MirGenerator h s ret EnumInfo enumInfo adt i j = do - when (adt ^. M.adtkind /= M.Enum) $ mirFail $ + Some discrTp <- case adt ^. M.adtkind of + M.Enum discrTy -> tyToReprM discrTy + _ -> mirFail $ "expected enum, but got adt " ++ show (adt ^. M.adtname) + + when (isn't M._Enum (adt ^. M.adtkind)) $ mirFail $ "expected enum, but got adt " ++ show (adt ^. M.adtname) var <- case adt ^? M.adtvariants . ix i of @@ -779,7 +790,7 @@ enumInfo adt i j = do Nothing -> mirFail $ "field index " ++ show j ++ " is out of range for enum " ++ show (adt ^. M.adtname) ++ " variant " ++ show i - Some ctx <- enumVariantsM adt + SomeRustEnumRepr _ ctx <- enumVariantsM adt Some idx <- case Ctx.intIndex (fromIntegral i) (Ctx.size ctx) of Just x -> return x Nothing -> mirFail $ "variant index " ++ show i ++ " is out of range for enum " ++ @@ -799,13 +810,13 @@ enumInfo adt i j = do kind <- checkFieldKind (not $ canInitialize col fldTy) tpr tpr' $ "field " ++ show j ++ " of enum " ++ show (adt ^. M.adtname) ++ " variant " ++ show i - return $ EnumInfo ctx idx ctx' idx' kind + return $ EnumInfo discrTp ctx idx ctx' idx' kind getEnumField :: M.Adt -> Int -> Int -> MirExp s -> MirGenerator h s ret (MirExp s) getEnumField adt i j me = do - EnumInfo ctx idx ctx' idx' fld <- enumInfo adt i j - e <- readAnyE (RustEnumRepr ctx) me - e <- readEnumVariant ctx idx e + EnumInfo discrTp ctx idx ctx' idx' fld <- enumInfo adt i j + e <- readAnyE (RustEnumRepr discrTp ctx) me + e <- readEnumVariant discrTp ctx idx e e <- readStructField ctx' idx' e e <- readFieldData' fld errFieldUninit e return $ MirExp (R.exprType e) e @@ -817,11 +828,11 @@ getEnumField adt i j me = do setEnumField :: M.Adt -> Int -> Int -> MirExp s -> MirExp s -> MirGenerator h s ret (MirExp s) setEnumField adt i j me (MirExp tpr e') = do - EnumInfo ctx idx ctx' idx' fld <- enumInfo adt i j + EnumInfo discrTp ctx idx ctx' idx' fld <- enumInfo adt i j Refl <- testEqualityOrFail tpr (fieldDataType fld) (errFieldType fld) e' <- buildFieldData fld e' - let f' = adjustAnyE (RustEnumRepr ctx) $ - adjustEnumVariant ctx idx $ + let f' = adjustAnyE (RustEnumRepr discrTp ctx) $ + adjustEnumVariant discrTp ctx idx $ \e -> writeStructField ctx' idx' e e' f' me where @@ -885,15 +896,16 @@ buildStruct adt es = buildEnum' :: HasCallStack => M.Adt -> Int -> [Maybe (MirExp s)] -> MirGenerator h s ret (MirExp s) buildEnum' adt i es = do - when (adt ^. M.adtkind /= M.Enum) $ mirFail $ - "expected enum, but got adt " ++ show (adt ^. M.adtname) + Some discrTp <- case adt ^. M.adtkind of + M.Enum discrTy -> tyToReprM discrTy + _ -> mirFail $ "expected enum, but got adt " ++ show (adt ^. M.adtname) var <- case adt ^? M.adtvariants . ix i of Just var -> return var Nothing -> mirFail $ "variant index " ++ show i ++ " is out of range for enum " ++ show (adt ^. M.adtname) - Some ctx <- enumVariantsM adt + SomeRustEnumRepr _ ctx <- enumVariantsM adt Some idx <- case Ctx.intIndex (fromIntegral i) (Ctx.size ctx) of Just x -> return x Nothing -> mirFail $ "variant index " ++ show i ++ " is out of range for enum " ++ @@ -913,11 +925,11 @@ buildEnum' adt i es = do discrs <- use $ cs . discrMap . ix (adt ^. M.adtname) discr <- case discrs ^? ix i of - Just x -> return x + Just x -> enumDiscrLit discrTp x Nothing -> mirFail $ "can't find discr for variant " ++ show (adt ^. M.adtname, i) - buildAnyE (RustEnumRepr ctx) $ - R.App $ mkRustEnum ctx (R.App $ isizeLit discr) $ + buildAnyE (RustEnumRepr discrTp ctx) $ + R.App $ mkRustEnum discrTp ctx (R.App discr) $ R.App $ E.InjectVariant ctx idx $ R.App $ E.MkStruct ctx' asn @@ -926,7 +938,12 @@ buildEnum :: HasCallStack => M.Adt -> Int -> [MirExp s] -> buildEnum adt i es = buildEnum' adt i (map Just es) - +enumDiscrLit :: C.TypeRepr tp -> Integer + -> MirGenerator h s ret (E.App ext f tp) +enumDiscrLit tp discr = + case tp of + C.BVRepr w -> pure $ E.BVLit w $ BV.mkBV w discr + _ -> mirFail $ "Unknown enum discriminant type: " ++ show tp fieldDataRef :: FieldKind tp tp' -> @@ -954,11 +971,11 @@ enumFieldRef :: C.TypeRepr tp -> R.Expr MIR s (MirReferenceType tp) -> MirGenerator h s ret (MirPlace s) enumFieldRef adt i j tpr ref = do - EnumInfo ctx idx ctx' idx' fld <- enumInfo adt i j + EnumInfo discrTp ctx idx ctx' idx' fld <- enumInfo adt i j Refl <- testEqualityOrFail tpr C.AnyRepr $ "enumFieldRef: bad referent type: expected Any, but got " ++ show tpr - ref <- subanyRef (RustEnumRepr ctx) ref - ref <- subvariantRef ctx ref idx + ref <- subanyRef (RustEnumRepr discrTp ctx) ref + ref <- subvariantRef discrTp ctx ref idx ref <- subfieldRef ctx' ref idx' ref <- fieldDataRef fld ref -- TODO: for custom DSTs, we'll need to propagate enum metadata to fields @@ -968,9 +985,9 @@ enumFieldRef adt i j tpr ref = do enumDiscriminant :: M.Adt -> MirExp s -> MirGenerator h s ret (MirExp s) enumDiscriminant adt e = do - Some ctx <- enumVariantsM adt - let v = unpackAnyC (RustEnumRepr ctx) e - return $ MirExp IsizeRepr $ R.App $ rustEnumDiscriminant v + SomeRustEnumRepr discrTpr variantsCtx <- enumVariantsM adt + let v = unpackAnyC (RustEnumRepr discrTpr variantsCtx) e + return $ MirExp discrTpr $ R.App $ rustEnumDiscriminant discrTpr v tupleFieldRef :: [M.Ty] -> Int -> diff --git a/crux-mir/src/Mir/Concurrency.hs b/crux-mir/src/Mir/Concurrency.hs index 2bce1869c..412672506 100644 --- a/crux-mir/src/Mir/Concurrency.hs +++ b/crux-mir/src/Mir/Concurrency.hs @@ -141,7 +141,7 @@ mirPathName p = Empty_RefPath -> "" Any_RefPath _ p -> mirPathName p Field_RefPath _ p idx -> mirPathName p ++ "." ++ show (indexVal idx) - Variant_RefPath _ p idx -> mirPathName p ++ "." ++ show (indexVal idx) + Variant_RefPath _ _ p idx -> mirPathName p ++ "." ++ show (indexVal idx) Index_RefPath _ p idx -> mirPathName p Just_RefPath _ p -> mirPathName p VectorAsMirVector_RefPath _ p -> mirPathName p diff --git a/crux-mir/src/Mir/Language.hs b/crux-mir/src/Mir/Language.hs index 4a760b07c..1b0eeb45f 100644 --- a/crux-mir/src/Mir/Language.hs +++ b/crux-mir/src/Mir/Language.hs @@ -104,7 +104,8 @@ import Mir.DefId import Mir.PP () import Mir.Overrides import Mir.Intrinsics (MIR, mirExtImpl, mirIntrinsicTypes, - pattern RustEnumRepr, pattern MirVectorRepr, MirVector(..)) + pattern RustEnumRepr, SomeRustEnumRepr(..), + pattern MirVectorRepr, MirVector(..)) import Mir.Generator import Mir.Generate (generateMIR) import qualified Mir.Log as Log @@ -617,9 +618,9 @@ showRegEntry col mty (C.RegEntry tp rv) = let ctx = fieldCtxType fctx let fields = unpackAnyValue rv (C.StructRepr ctx) return $ Right (var, readFields fctx fields) - Enum -> do - C.Some vctx <- return $ enumVariants col adt - let enumVal = unpackAnyValue rv (RustEnumRepr vctx) + Enum _ -> do + SomeRustEnumRepr discrTp vctx <- return $ enumVariants col adt + let enumVal = unpackAnyValue rv (RustEnumRepr discrTp vctx) -- Note we don't look at the discriminant here, because mapping -- a discriminant value to a variant index is somewhat complex. -- Instead we just find the first PartExpr that's initialized. diff --git a/crux-mir/src/Mir/Overrides.hs b/crux-mir/src/Mir/Overrides.hs index cdccb05e9..1ca09c170 100644 --- a/crux-mir/src/Mir/Overrides.hs +++ b/crux-mir/src/Mir/Overrides.hs @@ -349,8 +349,8 @@ regEval bak baseEval tpr v = go tpr v Any_RefPath tpr <$> goMirReferencePath p goMirReferencePath (Field_RefPath ctx p idx) = Field_RefPath ctx <$> goMirReferencePath p <*> pure idx - goMirReferencePath (Variant_RefPath ctx p idx) = - Variant_RefPath ctx <$> goMirReferencePath p <*> pure idx + goMirReferencePath (Variant_RefPath discrTp ctx p idx) = + Variant_RefPath discrTp ctx <$> goMirReferencePath p <*> pure idx goMirReferencePath (Index_RefPath tpr p idx) = Index_RefPath tpr <$> goMirReferencePath p <*> go UsizeRepr idx goMirReferencePath (Just_RefPath tpr p) = From a84dc4799d9fef88e7df760aff0b04617eb78c9f Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 10 May 2023 15:22:02 -0400 Subject: [PATCH 032/114] crucible-mir: Infer zero-sized fields in enum variants --- crucible-mir/src/Mir/Trans.hs | 164 -- crucible-mir/src/Mir/TransTy.hs | 197 +- crux-mir/test_output.log | 2960 ++++++++++++++----------------- 3 files changed, 1487 insertions(+), 1834 deletions(-) diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index 6a3982640..7a289f21e 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -38,8 +38,6 @@ module Mir.Trans(transCollection,transStatics,RustModule(..) , evalRval , callExp , derefExp, readPlace, addrOfPlace - , initialValue - , eBVLit ) where import Control.Monad @@ -51,7 +49,6 @@ import qualified Control.Lens.Extras as Lens (is) import Data.Foldable import Data.Bits (shift, shiftL) -import qualified Data.BitVector.Sized as BV import qualified Data.ByteString as BS import qualified Data.Char as Char import qualified Data.List as List @@ -123,13 +120,6 @@ parsePosition posText = setPosition :: Text.Text -> MirGenerator h s ret () setPosition = G.setPosition . parsePosition -eBVLit :: - (1 <= w) => - NatRepr w -> - Integer -> - E.App ext f (C.BVType w) -eBVLit w i = E.BVLit w (BV.mkBV w i) - -------------------------------------------------------------------------------------- -- ** Expressions @@ -1530,160 +1520,6 @@ transTerminator t _tr = --- translation of toplevel glue --- ----- "Allocation" --- --- --- MIR initializes compound structures by initializing their --- components. It does not include a general allocation. Here we add --- general code to initialize the structures for local variables where --- we can. In general, we only need to produce a value of the correct --- type with a structure that is compatible for further --- initialization. --- --- With this code, it is possible for crux-mir to miss --- uninitialized values. So we should revisit this. --- -initialValue :: HasCallStack => M.Ty -> MirGenerator h s ret (Maybe (MirExp s)) -initialValue (CTyInt512) = - let w = knownNat :: NatRepr 512 in - return $ Just $ MirExp (C.BVRepr w) (S.app (eBVLit w 0)) -initialValue (CTyVector t) = do - Some tr <- tyToReprM t - return $ Just (MirExp (C.VectorRepr tr) (S.app $ E.VectorLit tr V.empty)) -initialValue (CTyArray t) = tyToReprM t >>= \(Some tpr) -> case tpr of - C.BVRepr w -> do - let idxs = Ctx.Empty Ctx.:> BaseUsizeRepr - v <- arrayZeroed idxs w - return $ Just $ MirExp (C.SymbolicArrayRepr idxs (C.BaseBVRepr w)) v - _ -> error $ "can't initialize array of " ++ show t ++ " (expected BVRepr)" -initialValue ty@(CTyBv _sz) = tyToReprM ty >>= \(Some tpr) -> case tpr of - C.BVRepr w -> return $ Just $ MirExp (C.BVRepr w) $ S.app $ eBVLit w 0 - _ -> mirFail $ "Bv type " ++ show ty ++ " does not have BVRepr" --- `Any` values have no reasonable default. Any default we provide might get --- muxed with actual non-default values, which will fail (unless the concrete --- type happens to match exactly). -initialValue CTyAny = return Nothing -initialValue CTyMethodSpec = return Nothing -initialValue CTyMethodSpecBuilder = return Nothing - -initialValue M.TyBool = return $ Just $ MirExp C.BoolRepr (S.false) -initialValue (M.TyTuple []) = return $ Just $ MirExp C.UnitRepr (R.App E.EmptyApp) -initialValue (M.TyTuple tys) = do - mexps <- mapM initialValue tys - col <- use $ cs . collection - return $ Just $ buildTupleMaybe col tys mexps -initialValue (M.TyInt M.USize) = return $ Just $ MirExp IsizeRepr (R.App $ isizeLit 0) -initialValue (M.TyInt sz) = baseSizeToNatCont sz $ \w -> - return $ Just $ MirExp (C.BVRepr w) (S.app (eBVLit w 0)) -initialValue (M.TyUint M.USize) = return $ Just $ MirExp UsizeRepr (R.App $ usizeLit 0) -initialValue (M.TyUint sz) = baseSizeToNatCont sz $ \w -> - return $ Just $ MirExp (C.BVRepr w) (S.app (eBVLit w 0)) -initialValue (M.TyArray t size) = do - Some tpr <- tyToReprM t - mv <- mirVector_uninit tpr $ S.app $ eBVLit knownNat (fromIntegral size) - return $ Just $ MirExp (MirVectorRepr tpr) mv --- TODO: disabled to workaround for a bug with muxing null and non-null refs --- The problem is with --- if (*) { --- let x = &...; --- } --- `x` gets default-initialized at the start of the function, which (with these --- cases uncommented) sets it to null (`MirReference_Integer 0`). Then, if the --- branch is taken, it's set to a valid `MirReference` value instead. At the --- end of the `if`, we try to mux together `MirReference_Integer` with a normal --- `MirReference`, which currently fails. --- --- * The short-term fix is to disable initialization of refs, so they never --- get set to `null` in the first place. --- * The medium-term fix is to support muxing the two MirReference variants, --- using something like VariantType. --- * The long-term fix is to remove default-initialization entirely, either by --- writing an AdtAg pass for structs and tuples like we have for enums, or --- by converting all locals to untyped allocations (allow writing each field --- value independently, then materialize a fully-initialized struct the --- first time it's read at struct type). --- --- NB: When re-enabling this, also re-enable the TyRef case of `canInitialize` -{- -initialValue (M.TyRef (M.TySlice t) M.Immut) = do - tyToReprCont t $ \ tr -> do - let vec = R.App $ E.VectorLit tr V.empty - vec' <- MirExp (MirVectorRepr tr) <$> mirVector_fromVector tr vec - let i = MirExp UsizeRepr (R.App $ usizeLit 0) - return $ Just $ buildTuple [vec', i, i] -initialValue (M.TyRef (M.TySlice t) M.Mut) = do - tyToReprCont t $ \ tr -> do - ref <- newMirRef (MirVectorRepr tr) - let i = MirExp UsizeRepr (R.App $ usizeLit 0) - return $ Just $ buildTuple [(MirExp (MirReferenceRepr (MirVectorRepr tr)) ref), i, i] - -- fail ("don't know how to initialize slices for " ++ show t) -initialValue (M.TyRef M.TyStr M.Immut) = do - let tr = C.BVRepr $ knownNat @8 - let vec = R.App $ E.VectorLit tr V.empty - vec' <- MirExp (MirVectorRepr tr) <$> mirVector_fromVector tr vec - let i = MirExp UsizeRepr (R.App $ usizeLit 0) - return $ Just $ buildTuple [vec', i, i] -initialValue (M.TyRef M.TyStr M.Mut) = do - let tr = C.BVRepr $ knownNat @8 - ref <- newMirRef (MirVectorRepr tr) - let i = MirExp UsizeRepr (R.App $ usizeLit 0) - return $ Just $ buildTuple [(MirExp (MirReferenceRepr (MirVectorRepr tr)) ref), i, i] -initialValue (M.TyRef (M.TyDynamic _) _) = do - let x = R.App $ E.PackAny knownRepr $ R.App $ E.EmptyApp - return $ Just $ MirExp knownRepr $ R.App $ E.MkStruct knownRepr $ - Ctx.Empty Ctx.:> x Ctx.:> x -initialValue (M.TyRawPtr (M.TyDynamic _) _) = do - let x = R.App $ E.PackAny knownRepr $ R.App $ E.EmptyApp - return $ Just $ MirExp knownRepr $ R.App $ E.MkStruct knownRepr $ - Ctx.Empty Ctx.:> x Ctx.:> x -initialValue (M.TyRef t M.Immut) = initialValue t -initialValue (M.TyRef t M.Mut) - | Some tpr <- tyToRepr t = do - r <- integerToMirRef tpr $ R.App $ usizeLit 0 - return $ Just $ MirExp (MirReferenceRepr tpr) r --} -initialValue M.TyChar = do - let w = (knownNat :: NatRepr 32) - return $ Just $ MirExp (C.BVRepr w) (S.app (eBVLit w 0)) -initialValue (M.TyClosure tys) = do - mexps <- mapM initialValue tys - col <- use $ cs . collection - return $ Just $ buildTupleMaybe col tys mexps -initialValue (M.TyAdt nm _ _) = do - adt <- findAdt nm - col <- use $ cs . collection - case adt ^. adtkind of - _ | Just ty <- reprTransparentFieldTy col adt -> initialValue ty - Struct -> do - let var = M.onlyVariant adt - fldExps <- mapM initField (var^.M.vfields) - Just <$> buildStruct' adt fldExps - Enum _ -> do - case inhabited (adt ^. adtvariants) of - -- Uninhabited enums can't be initialized. - [] -> return Nothing - -- Inhabited enums get initialized to their first inhabited variant. - vs@(var : _) -> do - fldExps <- mapM initField (var^.M.vfields) - Just <$> buildEnum' adt 0 fldExps - Union -> return Nothing - where - inhabited vars = filter _vinhabited vars - - - -initialValue (M.TyFnPtr _) = return $ Nothing -initialValue (M.TyFnDef _) = return $ Just $ MirExp C.UnitRepr $ R.App E.EmptyApp -initialValue (M.TyDynamic _) = return $ Nothing -initialValue M.TyNever = return $ Just $ MirExp knownRepr $ - R.App $ E.PackAny knownRepr $ R.App $ E.EmptyApp -initialValue _ = return Nothing - -initField :: Field -> MirGenerator h s ret (Maybe (MirExp s)) -initField (Field _name ty) = initialValue ty - - - -- | Allocate RefCells for all locals and populate `varMap`. Locals are -- default-initialized when possible using the result of `initialValue`. initLocals :: [M.Var] -> Set.Set Text.Text -> MirGenerator h s ret () diff --git a/crucible-mir/src/Mir/TransTy.hs b/crucible-mir/src/Mir/TransTy.hs index a3fa283e8..66f2c6f35 100644 --- a/crucible-mir/src/Mir/TransTy.hs +++ b/crucible-mir/src/Mir/TransTy.hs @@ -38,6 +38,7 @@ import qualified Data.Parameterized.Context as Ctx import Data.Parameterized.Classes import Data.Parameterized.NatRepr import Data.Parameterized.Some +import Data.Parameterized.TraversableFC -- crucible @@ -56,7 +57,7 @@ import Mir.Generator ( MirExp(..), MirPlace(..), PtrMetadata(..), MirGenerator, mirFail , subanyRef, subfieldRef, subvariantRef, subjustRef , mirVector_fromVector - , cs, collection, discrMap ) + , cs, collection, discrMap, findAdt, mirVector_uninit, arrayZeroed ) import Mir.Intrinsics ( MIR, pattern MirSliceRepr, pattern MirReferenceRepr, MirReferenceType , pattern MirVectorRepr @@ -65,7 +66,7 @@ import Mir.Intrinsics , RustEnumType, pattern RustEnumRepr, SomeRustEnumRepr(..) , mkRustEnum, rustEnumVariant, rustEnumDiscriminant , pattern MethodSpecRepr, pattern MethodSpecBuilderRepr - , DynRefType) + , DynRefType, usizeLit , pattern BaseUsizeRepr ) ----------------------------------------------------------------------- @@ -377,7 +378,7 @@ enumVariants :: TransTyConstraint => M.Collection -> M.Adt -> SomeRustEnumRepr enumVariants col (M.Adt name kind vs _ _ _ _) = case kind of M.Enum discrTy - | Some discrTpr <- tyToRepr col discrTy + | Some discrTpr <- tyToRepr col discrTy -> reprsToCtx variantReprs $ \variantsCxt -> SomeRustEnumRepr discrTpr variantsCxt _ -> error $ "expected " ++ show name ++ " to have kind Enum" @@ -916,7 +917,9 @@ buildEnum' adt i es = do show (adt ^. M.adtname) ++ " is not a struct?" Some fctx' <- variantFieldsM' var - asn <- case buildStructAssign' fctx' $ map (fmap (\(MirExp _ e) -> Some e)) es of + let ftys = map (^. M.fty) (var ^. M.vfields) + es' <- inferElidedVariantFields ftys es + asn <- case buildStructAssign' fctx' $ map (fmap (\(MirExp _ e) -> Some e)) es' of Left err -> mirFail $ "error building variant " ++ show (var^.M.vname) ++ ": " ++ err ++ " -- " ++ show es Right x -> return x @@ -933,6 +936,33 @@ buildEnum' adt i es = do R.App $ E.InjectVariant ctx idx $ R.App $ E.MkStruct ctx' asn +-- | TODO RGS: Add lots of documentation +inferElidedVariantFields :: [M.Ty] -> [Maybe (MirExp s)] + -> MirGenerator h s ret [Maybe (MirExp s)] +inferElidedVariantFields ftys fes + | length ftys == length fes -- TODO RGS: This is fishy. Consider an alternative approach. + = pure fes + | otherwise + = go ftys fes + where + go [] [] = pure [] + go [] (_:_) = mirFail $ unlines [ "inferElidedVariantFields: more expressions than types" + , "types: " ++ show ftys + , "expressions: " ++ show fes + ] + go (ty:tys) exps = do + col <- use $ cs . collection + if isZeroSized col ty + then do val <- initialValue ty + exps' <- go tys exps + pure $ val : exps' + else + case exps of + e:es -> do + es' <- go tys es + pure $ e : es' + [] -> mirFail "inferElidedVariantFields: not enough expressions" + buildEnum :: HasCallStack => M.Adt -> Int -> [MirExp s] -> MirGenerator h s ret (MirExp s) buildEnum adt i es = @@ -1040,3 +1070,162 @@ eraseSigReceiver :: M.FnSig -> M.FnSig eraseSigReceiver sig = sig & M.fsarg_tys %~ \xs -> case xs of [] -> error $ unwords ["dynamic trait method has no receiver", show sig] (_ : tys) -> M.TyErased : tys + +---- "Allocation" +-- +-- +-- MIR initializes compound structures by initializing their +-- components. It does not include a general allocation. Here we add +-- general code to initialize the structures for local variables where +-- we can. In general, we only need to produce a value of the correct +-- type with a structure that is compatible for further +-- initialization. +-- +-- With this code, it is possible for crux-mir to miss +-- uninitialized values. So we should revisit this. +-- +initialValue :: HasCallStack => M.Ty -> MirGenerator h s ret (Maybe (MirExp s)) +initialValue (CTyInt512) = + let w = knownNat :: NatRepr 512 in + return $ Just $ MirExp (C.BVRepr w) (S.app (eBVLit w 0)) +initialValue (CTyVector t) = do + Some tr <- tyToReprM t + return $ Just (MirExp (C.VectorRepr tr) (S.app $ E.VectorLit tr V.empty)) +initialValue (CTyArray t) = tyToReprM t >>= \(Some tpr) -> case tpr of + C.BVRepr w -> do + let idxs = Ctx.Empty Ctx.:> BaseUsizeRepr + v <- arrayZeroed idxs w + return $ Just $ MirExp (C.SymbolicArrayRepr idxs (C.BaseBVRepr w)) v + _ -> error $ "can't initialize array of " ++ show t ++ " (expected BVRepr)" +initialValue ty@(CTyBv _sz) = tyToReprM ty >>= \(Some tpr) -> case tpr of + C.BVRepr w -> return $ Just $ MirExp (C.BVRepr w) $ S.app $ eBVLit w 0 + _ -> mirFail $ "Bv type " ++ show ty ++ " does not have BVRepr" +-- `Any` values have no reasonable default. Any default we provide might get +-- muxed with actual non-default values, which will fail (unless the concrete +-- type happens to match exactly). +initialValue CTyAny = return Nothing +initialValue CTyMethodSpec = return Nothing +initialValue CTyMethodSpecBuilder = return Nothing + +initialValue M.TyBool = return $ Just $ MirExp C.BoolRepr (S.false) +initialValue (M.TyTuple []) = return $ Just $ MirExp C.UnitRepr (R.App E.EmptyApp) +initialValue (M.TyTuple tys) = do + mexps <- mapM initialValue tys + col <- use $ cs . collection + return $ Just $ buildTupleMaybe col tys mexps +initialValue (M.TyInt M.USize) = return $ Just $ MirExp IsizeRepr (R.App $ isizeLit 0) +initialValue (M.TyInt sz) = baseSizeToNatCont sz $ \w -> + return $ Just $ MirExp (C.BVRepr w) (S.app (eBVLit w 0)) +initialValue (M.TyUint M.USize) = return $ Just $ MirExp UsizeRepr (R.App $ usizeLit 0) +initialValue (M.TyUint sz) = baseSizeToNatCont sz $ \w -> + return $ Just $ MirExp (C.BVRepr w) (S.app (eBVLit w 0)) +initialValue (M.TyArray t size) = do + Some tpr <- tyToReprM t + mv <- mirVector_uninit tpr $ S.app $ eBVLit knownNat (fromIntegral size) + return $ Just $ MirExp (MirVectorRepr tpr) mv +-- TODO: disabled to workaround for a bug with muxing null and non-null refs +-- The problem is with +-- if (*) { +-- let x = &...; +-- } +-- `x` gets default-initialized at the start of the function, which (with these +-- cases uncommented) sets it to null (`MirReference_Integer 0`). Then, if the +-- branch is taken, it's set to a valid `MirReference` value instead. At the +-- end of the `if`, we try to mux together `MirReference_Integer` with a normal +-- `MirReference`, which currently fails. +-- +-- * The short-term fix is to disable initialization of refs, so they never +-- get set to `null` in the first place. +-- * The medium-term fix is to support muxing the two MirReference variants, +-- using something like VariantType. +-- * The long-term fix is to remove default-initialization entirely, either by +-- writing an AdtAg pass for structs and tuples like we have for enums, or +-- by converting all locals to untyped allocations (allow writing each field +-- value independently, then materialize a fully-initialized struct the +-- first time it's read at struct type). +-- +-- NB: When re-enabling this, also re-enable the TyRef case of `canInitialize` +{- +initialValue (M.TyRef (M.TySlice t) M.Immut) = do + tyToReprCont t $ \ tr -> do + let vec = R.App $ E.VectorLit tr V.empty + vec' <- MirExp (MirVectorRepr tr) <$> mirVector_fromVector tr vec + let i = MirExp UsizeRepr (R.App $ usizeLit 0) + return $ Just $ buildTuple [vec', i, i] +initialValue (M.TyRef (M.TySlice t) M.Mut) = do + tyToReprCont t $ \ tr -> do + ref <- newMirRef (MirVectorRepr tr) + let i = MirExp UsizeRepr (R.App $ usizeLit 0) + return $ Just $ buildTuple [(MirExp (MirReferenceRepr (MirVectorRepr tr)) ref), i, i] + -- fail ("don't know how to initialize slices for " ++ show t) +initialValue (M.TyRef M.TyStr M.Immut) = do + let tr = C.BVRepr $ knownNat @8 + let vec = R.App $ E.VectorLit tr V.empty + vec' <- MirExp (MirVectorRepr tr) <$> mirVector_fromVector tr vec + let i = MirExp UsizeRepr (R.App $ usizeLit 0) + return $ Just $ buildTuple [vec', i, i] +initialValue (M.TyRef M.TyStr M.Mut) = do + let tr = C.BVRepr $ knownNat @8 + ref <- newMirRef (MirVectorRepr tr) + let i = MirExp UsizeRepr (R.App $ usizeLit 0) + return $ Just $ buildTuple [(MirExp (MirReferenceRepr (MirVectorRepr tr)) ref), i, i] +initialValue (M.TyRef (M.TyDynamic _) _) = do + let x = R.App $ E.PackAny knownRepr $ R.App $ E.EmptyApp + return $ Just $ MirExp knownRepr $ R.App $ E.MkStruct knownRepr $ + Ctx.Empty Ctx.:> x Ctx.:> x +initialValue (M.TyRawPtr (M.TyDynamic _) _) = do + let x = R.App $ E.PackAny knownRepr $ R.App $ E.EmptyApp + return $ Just $ MirExp knownRepr $ R.App $ E.MkStruct knownRepr $ + Ctx.Empty Ctx.:> x Ctx.:> x +initialValue (M.TyRef t M.Immut) = initialValue t +initialValue (M.TyRef t M.Mut) + | Some tpr <- tyToRepr t = do + r <- integerToMirRef tpr $ R.App $ usizeLit 0 + return $ Just $ MirExp (MirReferenceRepr tpr) r +-} +initialValue M.TyChar = do + let w = (knownNat :: NatRepr 32) + return $ Just $ MirExp (C.BVRepr w) (S.app (eBVLit w 0)) +initialValue (M.TyClosure tys) = do + mexps <- mapM initialValue tys + col <- use $ cs . collection + return $ Just $ buildTupleMaybe col tys mexps +initialValue (M.TyAdt nm _ _) = do + adt <- findAdt nm + col <- use $ cs . collection + case adt ^. M.adtkind of + _ | Just ty <- reprTransparentFieldTy col adt -> initialValue ty + M.Struct -> do + let var = M.onlyVariant adt + fldExps <- mapM initField (var^.M.vfields) + Just <$> buildStruct' adt fldExps + M.Enum _ -> do + case inhabited (adt ^. M.adtvariants) of + -- Uninhabited enums can't be initialized. + [] -> return Nothing + -- Inhabited enums get initialized to their first inhabited variant. + vs@(var : _) -> do + fldExps <- mapM initField (var^.M.vfields) + Just <$> buildEnum' adt 0 fldExps + M.Union -> return Nothing + where + inhabited vars = filter M._vinhabited vars + + + +initialValue (M.TyFnPtr _) = return $ Nothing +initialValue (M.TyFnDef _) = return $ Just $ MirExp C.UnitRepr $ R.App E.EmptyApp +initialValue (M.TyDynamic _) = return $ Nothing +initialValue M.TyNever = return $ Just $ MirExp knownRepr $ + R.App $ E.PackAny knownRepr $ R.App $ E.EmptyApp +initialValue _ = return Nothing + +initField :: M.Field -> MirGenerator h s ret (Maybe (MirExp s)) +initField (M.Field _name ty) = initialValue ty + +eBVLit :: + (1 <= w) => + NatRepr w -> + Integer -> + E.App ext f (C.BVType w) +eBVLit w i = E.BVLit w (BV.mkBV w i) diff --git a/crux-mir/test_output.log b/crux-mir/test_output.log index 2bb7467ce..3fd7819b8 100644 --- a/crux-mir/test_output.log +++ b/crux-mir/test_output.log @@ -2,192 +2,158 @@ crux-mir crux concrete refs - promoted_imm: OK (0.16s) + promoted_imm: OK (2.17s) Compiling and running oracle program (0.16s) - Oracle output: 0 + Oracle output: 0 (2.01s) Crux output: 0 - mut_raw: OK (0.17s) - Compiling and running oracle program (0.17s) - Oracle output: 123 + mut_raw: OK (2.14s) + Compiling and running oracle program (0.18s) + Oracle output: 123 (1.96s) Crux output: 123 - mut_nested: OK (0.18s) + mut_nested: OK (2.20s) Compiling and running oracle program (0.17s) - Oracle output: () (0.01s) + Oracle output: () (2.03s) Crux output: () - fn_ptr_mut: OK (0.18s) - Compiling and running oracle program (0.18s) - Oracle output: 1 - Crux output: 1 - never: OK (0.16s) - Compiling and running oracle program (0.16s) - Oracle output: 1 + fn_ptr_mut: OK (2.23s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (2.07s) Crux output: 1 - mut_ref: OK (0.16s) - Compiling and running oracle program (0.16s) - Oracle output: 1 + never: OK (2.17s) + Compiling and running oracle program (0.19s) + Oracle output: 1 (1.98s) Crux output: 1 - temp: OK (0.18s) - Compiling and running oracle program (0.18s) - Oracle output: 1 + mut_ref: OK (2.14s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (1.97s) Crux output: 1 - mut_arg: OK (0.18s) - Compiling and running oracle program (0.18s) - Oracle output: 1 + temp: OK (2.16s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (2.00s) Crux output: 1 - fn_ptr: OK (0.17s) + mut_arg: OK (2.14s) Compiling and running oracle program (0.17s) - Oracle output: 1 + Oracle output: 1 (1.96s) + Crux output: 1 + fn_ptr: OK (2.14s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (1.98s) Crux output: 1 - promoted_mut: OK (0.17s) + promoted_mut: OK (2.15s) Compiling and running oracle program (0.17s) - Oracle output: 0 + Oracle output: 0 (1.98s) Crux output: 0 - never_mut: OK (0.17s) + never_mut: OK (2.12s) Compiling and running oracle program (0.17s) - Oracle output: 1 + Oracle output: 1 (1.96s) Crux output: 1 - mut_tuple_field: OK (0.19s) - Compiling and running oracle program (0.17s) - Oracle output: () (0.02s) + mut_tuple_field: OK (2.19s) + Compiling and running oracle program (0.20s) + Oracle output: () (1.99s) Crux output: () - imm_raw: OK (0.16s) - Compiling and running oracle program (0.16s) - Oracle output: 123 + imm_raw: OK (2.20s) + Compiling and running oracle program (0.18s) + Oracle output: 123 (2.02s) Crux output: 123 - static_mut: OK (0.17s) + static_mut: OK (2.16s) Compiling and running oracle program (0.17s) - Oracle output: 2 + Oracle output: 2 (2.00s) Crux output: 2 - imm_ref: OK (0.17s) + imm_ref: OK (2.17s) Compiling and running oracle program (0.17s) - Oracle output: 123 + Oracle output: 123 (2.00s) Crux output: 123 - imm_arg: OK (0.18s) - Compiling and running oracle program (0.18s) - Oracle output: 1 + imm_arg: OK (2.15s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (1.98s) Crux output: 1 hash_map - insert_multi: FAIL (0.98s) - Compiling and running oracle program (0.22s) - Oracle output: 100 (0.77s) - user error (JSON Decoding of test/conc_eval/hash_map/insert_multi.all.mir failed: Error in $.fns[1463].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) - - Use -p '/insert_multi/' to rerun this test only. - insert_remove: FAIL (0.98s) - Compiling and running oracle program (0.24s) - Oracle output: 12 (0.74s) - user error (JSON Decoding of test/conc_eval/hash_map/insert_remove.all.mir failed: Error in $.fns[1489].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) - - Use -p '/insert_remove/' to rerun this test only. - insert_iter: FAIL (0.75s) - Compiling and running oracle program (0.21s) - Oracle output: [11, 12] (0.54s) - user error (JSON Decoding of test/conc_eval/hash_map/insert_iter.all.mir failed: Error in $.fns[1464].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) - - Use -p '/insert_iter/' to rerun this test only. - insert_get: FAIL (1.23s) - Compiling and running oracle program (0.23s) - Oracle output: (11, 12) (1.00s) - user error (JSON Decoding of test/conc_eval/hash_map/insert_get.all.mir failed: Error in $.fns[1465].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) - - Use -p '/insert_get/' to rerun this test only. - traits - params: OK (0.16s) - Compiling and running oracle program (0.16s) - Oracle output: 25 - Crux output: 25 - generic3: FAIL (0.19s) - Compiling and running oracle program (0.17s) - Oracle output: () (0.03s) + insert_multi: OK (2.62s) + Compiling and running oracle program (0.20s) + Oracle output: 100 (2.43s) + Crux output: 100 + insert_remove: FAIL (3.32s) + Compiling and running oracle program (0.23s) + Oracle output: 12 (3.10s) Crux output: failures: - ---- generic3/b76a1aa5::crux_test[0] counterexamples ---- + ---- insert_remove/faf037d0::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in generic3/b76a1aa5::f[0] - [Crux] Translation error in generic3/b76a1aa5::f[0]: error building variant core/5a1f6960::option[0]::Option[0]::Some[0]: not enough expressions -- [] + [Crux] internal: error: in alloc/8ccd0c40::vec[0]::{impl#1}[0]::set_len[0]::_inst1b10a593fa289fda[0] + [Crux] panicking::panic, called from alloc/8ccd0c40::vec[0]::{impl#1}[0]::set_len[0]::_inst1b10a593fa289fda[0] [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/generic3/' to rerun this test only. - static_three: OK (0.17s) + Use -p '/insert_remove/' to rerun this test only. + insert_iter: OK (2.56s) + Compiling and running oracle program (0.21s) + Oracle output: [11, 12] (2.35s) + Crux output: [11, 12] + insert_get: OK (2.55s) + Compiling and running oracle program (0.21s) + Oracle output: (11, 12) (2.34s) + Crux output: (11, 12) + traits + params: OK (2.15s) + Compiling and running oracle program (0.16s) + Oracle output: 25 (1.98s) + Crux output: 25 + generic3: OK (2.14s) + Compiling and running oracle program (0.17s) + Oracle output: () (1.97s) + Crux output: () + static_three: OK (2.15s) Compiling and running oracle program (0.17s) - Oracle output: 2 + Oracle output: 2 (1.98s) Crux output: 2 - generic1: FAIL (0.19s) - Compiling and running oracle program (0.17s) - Oracle output: () (0.02s) - Crux output: - failures: - - ---- generic1/af42c30b::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in generic1/af42c30b::f[0] - [Crux] Translation error in generic1/af42c30b::f[0]: error building variant core/5a1f6960::option[0]::Option[0]::Some[0]: not enough expressions -- [] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/generic1/' to rerun this test only. - subtrait: OK (0.17s) + generic1: OK (2.20s) + Compiling and running oracle program (0.19s) + Oracle output: () (2.01s) + Crux output: () + subtrait: OK (2.16s) Compiling and running oracle program (0.17s) - Oracle output: 73 + Oracle output: 73 (1.99s) Crux output: 73 - gen_trait: OK (0.17s) + gen_trait: OK (2.14s) Compiling and running oracle program (0.17s) - Oracle output: -69 + Oracle output: -69 (1.97s) Crux output: -69 - dynamic_branch: FAIL (0.21s) - Compiling and running oracle program (0.17s) + dynamic_branch: FAIL (0.20s) + Compiling and running oracle program (0.16s) Oracle output: 1 (0.03s) user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_branch.rs) Use -p '/dynamic_branch/' to rerun this test only. - static_two: OK (0.17s) + static_two: OK (2.15s) Compiling and running oracle program (0.17s) - Oracle output: 1 + Oracle output: 1 (1.98s) Crux output: 1 - tyfam5: OK (0.17s) - Compiling and running oracle program (0.17s) - Oracle output: () + tyfam5: OK (2.18s) + Compiling and running oracle program (0.16s) + Oracle output: () (2.01s) Crux output: () - generic2: FAIL (0.20s) - Compiling and running oracle program (0.17s) - Oracle output: () (0.03s) - Crux output: - failures: - - ---- generic2/adc7abb7::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in generic2/adc7abb7::f[0] - [Crux] Translation error in generic2/adc7abb7::f[0]: error building variant core/5a1f6960::option[0]::Option[0]::Some[0]: not enough expressions -- [] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/generic2/' to rerun this test only. - assoc3: OK (0.17s) - Compiling and running oracle program (0.17s) - Oracle output: () + generic2: OK (2.20s) + Compiling and running oracle program (0.19s) + Oracle output: () (2.01s) Crux output: () - tyfam4: OK (0.17s) - Compiling and running oracle program (0.17s) - Oracle output: 0 - Crux output: 0 - static_self: OK (0.19s) + assoc3: OK (2.16s) Compiling and running oracle program (0.19s) - Oracle output: 42 + Oracle output: () (1.96s) + Crux output: () + tyfam4: OK (2.27s) + Compiling and running oracle program (0.16s) + Oracle output: 0 (2.11s) + Crux output: 0 + static_self: OK (2.15s) + Compiling and running oracle program (0.16s) + Oracle output: 42 (1.99s) Crux output: 42 - conv: OK (0.19s) - Compiling and running oracle program (0.18s) - Oracle output: 1 + conv: OK (2.20s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (2.03s) Crux output: 1 dynamic_poly: FAIL (0.21s) Compiling and running oracle program (0.17s) @@ -195,108 +161,69 @@ crux-mir user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_poly.rs) Use -p '/dynamic_poly/' to rerun this test only. - dynamic_simple: OK (0.17s) + dynamic_simple: OK (2.29s) Compiling and running oracle program (0.17s) - Oracle output: 1 + Oracle output: 1 (2.11s) Crux output: 1 - gen_trait_poly: OK (0.17s) - Compiling and running oracle program (0.17s) - Oracle output: 12 + gen_trait_poly: OK (2.19s) + Compiling and running oracle program (0.20s) + Oracle output: 12 (2.00s) Crux output: 12 - default: OK (0.18s) - Compiling and running oracle program (0.18s) - Oracle output: 12 + default: OK (2.18s) + Compiling and running oracle program (0.16s) + Oracle output: 12 (2.01s) Crux output: 12 - assoc1: OK (0.18s) + assoc1: OK (2.22s) Compiling and running oracle program (0.17s) - Oracle output: () (0.01s) + Oracle output: () (2.04s) Crux output: () - subtrait2: OK (0.17s) - Compiling and running oracle program (0.17s) - Oracle output: 64 + subtrait2: OK (2.28s) + Compiling and running oracle program (0.20s) + Oracle output: 64 (2.08s) Crux output: 64 - bounds1: FAIL (0.20s) - Compiling and running oracle program (0.17s) - Oracle output: () (0.03s) - Crux output: - failures: - - ---- bounds1/d7ac532f::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in bounds1/d7ac532f::f[0] - [Crux] Translation error in bounds1/d7ac532f::f[0]: error building variant core/5a1f6960::option[0]::Option[0]::Some[0]: not enough expressions -- [] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/bounds1/' to rerun this test only. - bounds3: FAIL (0.20s) - Compiling and running oracle program (0.17s) - Oracle output: () (0.03s) - Crux output: - failures: - - ---- bounds3/6f12d636::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in bounds3/6f12d636::f[0] - [Crux] Translation error in bounds3/6f12d636::f[0]: error building variant core/5a1f6960::option[0]::Option[0]::Some[0]: not enough expressions -- [] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/bounds3/' to rerun this test only. - static_eq: OK (0.18s) + bounds1: OK (2.22s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.05s) + Crux output: () + bounds3: OK (2.30s) Compiling and running oracle program (0.18s) - Oracle output: true + Oracle output: () (2.12s) + Crux output: () + static_eq: OK (2.22s) + Compiling and running oracle program (0.18s) + Oracle output: true (2.05s) Crux output: true - tyfam3: OK (0.17s) - Compiling and running oracle program (0.17s) - Oracle output: 23 + tyfam3: OK (2.32s) + Compiling and running oracle program (0.18s) + Oracle output: 23 (2.14s) Crux output: 23 - dict_med: OK (0.16s) + dict_med: OK (2.13s) Compiling and running oracle program (0.16s) - Oracle output: 42 + Oracle output: 42 (1.97s) Crux output: 42 - basics1: OK (0.18s) + basics1: OK (2.28s) Compiling and running oracle program (0.17s) - Oracle output: () (0.01s) + Oracle output: () (2.11s) Crux output: () - assoc2: OK (0.17s) + assoc2: OK (2.20s) Compiling and running oracle program (0.17s) - Oracle output: () + Oracle output: () (2.03s) Crux output: () - dict_polymem: OK (0.18s) - Compiling and running oracle program (0.18s) - Oracle output: 4 + dict_polymem: OK (2.17s) + Compiling and running oracle program (0.17s) + Oracle output: 4 (2.00s) Crux output: 4 - bounds2: FAIL (0.19s) - Compiling and running oracle program (0.17s) - Oracle output: () (0.03s) - Crux output: - failures: - - ---- bounds2/4feb6822::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in bounds2/4feb6822::f[0] - [Crux] Translation error in bounds2/4feb6822::f[0]: error building variant core/5a1f6960::option[0]::Option[0]::Some[0]: not enough expressions -- [] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/bounds2/' to rerun this test only. - intoiter: OK (0.17s) - Compiling and running oracle program (0.16s) - Oracle output: 0 - Crux output: 0 - static: OK (0.17s) + bounds2: OK (2.20s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.03s) + Crux output: () + intoiter: OK (2.16s) Compiling and running oracle program (0.17s) - Oracle output: 42 + Oracle output: 0 (1.99s) + Crux output: 0 + static: OK (2.17s) + Compiling and running oracle program (0.18s) + Oracle output: 42 (1.99s) Crux output: 42 dynamic_two: FAIL (0.20s) Compiling and running oracle program (0.16s) @@ -304,105 +231,64 @@ crux-mir user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_two.rs) Use -p '/dynamic_two/' to rerun this test only. - bounds4: FAIL (0.20s) - Compiling and running oracle program (0.17s) - Oracle output: () (0.03s) - Crux output: - failures: - - ---- bounds4/33198a89::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in bounds4/33198a89::f[0] - [Crux] Translation error in bounds4/33198a89::f[0]: error building variant core/5a1f6960::option[0]::Option[0]::Some[0]: not enough expressions -- [] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/bounds4/' to rerun this test only. - dict_poly: OK (0.17s) - Compiling and running oracle program (0.17s) - Oracle output: 1 + bounds4: OK (2.32s) + Compiling and running oracle program (0.16s) + Oracle output: () (2.15s) + Crux output: () + dict_poly: OK (2.18s) + Compiling and running oracle program (0.19s) + Oracle output: 1 (1.99s) Crux output: 1 - dict_simple: OK (0.17s) - Compiling and running oracle program (0.17s) - Oracle output: 32 + dict_simple: OK (2.18s) + Compiling and running oracle program (0.19s) + Oracle output: 32 (1.99s) Crux output: 32 - tyfam: OK (0.18s) - Compiling and running oracle program (0.17s) - Oracle output: 23 + tyfam: OK (2.19s) + Compiling and running oracle program (0.18s) + Oracle output: 23 (2.02s) Crux output: 23 - bounds5: FAIL (0.20s) - Compiling and running oracle program (0.18s) - Oracle output: () (0.02s) - Crux output: - failures: - - ---- bounds5/ee058fbf::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in bounds5/ee058fbf::f[0] - [Crux] Translation error in bounds5/ee058fbf::f[0]: error building variant core/5a1f6960::option[0]::Option[0]::Some[0]: not enough expressions -- [] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/bounds5/' to rerun this test only. - tyfam2: OK (0.17s) + bounds5: OK (2.21s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.03s) + Crux output: () + tyfam2: OK (2.19s) Compiling and running oracle program (0.17s) - Oracle output: 23 + Oracle output: 23 (2.02s) Crux output: 23 - dynamic_med: FAIL (0.19s) - Compiling and running oracle program (0.16s) + dynamic_med: FAIL (0.20s) + Compiling and running oracle program (0.17s) Oracle output: 1 (0.03s) user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_med.rs) Use -p '/dynamic_med/' to rerun this test only. intTest - test0039: OK (0.17s) + test0039: OK (2.17s) Compiling and running oracle program (0.17s) - Oracle output: 7 + Oracle output: 7 (2.00s) Crux output: 7 - test0038: FAIL (0.24s) - Compiling and running oracle program (0.19s) - Oracle output: () (0.05s) - Crux output: - failures: - - ---- test0038/8a6cadf1::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/5a1f6960::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst5dab88813bb37c92[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst5dab88813bb37c92[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyArray (TyInt B32) 4) Mut - [Crux] as: TyRawPtr (TyUint B8) Mut - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/test0038/' to rerun this test only. + test0038: OK (2.36s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.19s) + Crux output: () tuple - clone_rec: OK (0.18s) - Compiling and running oracle program (0.17s) - Oracle output: () (0.01s) + clone_rec: OK (2.17s) + Compiling and running oracle program (0.16s) + Oracle output: () (2.01s) Crux output: () - clone: OK (0.19s) - Compiling and running oracle program (0.17s) - Oracle output: () (0.02s) + clone: OK (2.19s) + Compiling and running oracle program (0.19s) + Oracle output: () (1.99s) Crux output: () - clone_from: FAIL (0.20s) + clone_from: FAIL (2.19s) Compiling and running oracle program (0.17s) - Oracle output: () (0.03s) + Oracle output: () (2.02s) Crux output: failures: - ---- clone_from/9fe2d3a6::f[0] counterexamples ---- + ---- clone_from/e1b836b1::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/clone.rs:136:17: 136:31: error: in core/5a1f6960::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0] - [Crux] Translation error in core/5a1f6960::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0]: don't know how to generate CloneShim for unknown method core/5a1f6960::clone[0]::Clone[0]::clone[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/clone.rs:136:17: 136:31: error: in core/1c79c753::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0] + [Crux] Translation error in core/1c79c753::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0]: don't know how to generate CloneShim for unknown method core/1c79c753::clone[0]::Clone[0]::clone[0] [Crux] Overall status: Invalid. @@ -410,16 +296,16 @@ crux-mir crux doesn't match oracle Use -p '/clone_from/' to rerun this test only. - clone_struct: FAIL (0.22s) - Compiling and running oracle program (0.20s) - Oracle output: () (0.03s) + clone_struct: FAIL (2.23s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.05s) Crux output: failures: - ---- clone_struct/6843d741::f[0] counterexamples ---- + ---- clone_struct/96042f83::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/tuple/clone_struct.rs:8:13: 8:22: error: in clone_struct/6843d741::f[0] - [Crux] Translation error in clone_struct/6843d741::f[0]: don't know how to generate CloneShim for unknown method core/5a1f6960::clone[0]::Clone[0]::clone[0] + [Crux] test/conc_eval/tuple/clone_struct.rs:8:13: 8:22: error: in clone_struct/96042f83::f[0] + [Crux] Translation error in clone_struct/96042f83::f[0]: don't know how to generate CloneShim for unknown method core/1c79c753::clone[0]::Clone[0]::clone[0] [Crux] Overall status: Invalid. @@ -428,111 +314,49 @@ crux-mir Use -p '/clone_struct/' to rerun this test only. iter - for: OK (0.20s) + for: OK (2.20s) Compiling and running oracle program (0.17s) - Oracle output: 47 (0.03s) + Oracle output: 47 (2.03s) Crux output: 47 - sum: FAIL (0.22s) - Compiling and running oracle program (0.17s) - Oracle output: () (0.05s) - Crux output: - failures: - - ---- sum/32c2d811::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyUint USize) Immut - [Crux] as: TyRawPtr (TyUint B8) Immut - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/sum/' to rerun this test only. - filter_chain: OK (0.30s) - Compiling and running oracle program (0.21s) - Oracle output: 0 (0.10s) - Crux output: 0 - from_fn: OK (0.20s) + sum: OK (2.28s) Compiling and running oracle program (0.18s) - Oracle output: () (0.02s) + Oracle output: () (2.09s) Crux output: () - peek: FAIL (0.24s) - Compiling and running oracle program (0.18s) - Oracle output: 3 (0.06s) - Crux output: - failures: - - ---- peek/dee0a907::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyUint USize) Immut - [Crux] as: TyRawPtr (TyUint B8) Immut - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/peek/' to rerun this test only. - zip: FAIL (0.26s) - Compiling and running oracle program (0.19s) - Oracle output: () (0.08s) - Crux output: - failures: - - ---- zip/b55065e9::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyInt B32) Immut - [Crux] as: TyRawPtr (TyUint B8) Immut - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/zip/' to rerun this test only. - loop: OK (0.17s) + filter_chain: OK (2.36s) + Compiling and running oracle program (0.19s) + Oracle output: 0 (2.16s) + Crux output: 0 + from_fn: OK (2.16s) + Compiling and running oracle program (0.17s) + Oracle output: () (1.99s) + Crux output: () + peek: OK (2.29s) + Compiling and running oracle program (0.18s) + Oracle output: 3 (2.11s) + Crux output: 3 + zip: OK (2.39s) Compiling and running oracle program (0.17s) - Oracle output: 2 + Oracle output: () (2.22s) + Crux output: () + loop: OK (2.11s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (1.95s) Crux output: 2 - cloned: FAIL (0.22s) - Compiling and running oracle program (0.18s) - Oracle output: 6 (0.04s) - Crux output: - failures: - - ---- cloned/7cb6e236::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_instaffa7a8b1157c078[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyUint USize) Immut - [Crux] as: TyRawPtr (TyUint B8) Immut - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/cloned/' to rerun this test only. + cloned: OK (2.25s) + Compiling and running oracle program (0.17s) + Oracle output: 6 (2.08s) + Crux output: 6 str - format_hex: FAIL (1.03s) - Compiling and running oracle program (0.20s) - Oracle output: true (0.83s) + format_hex: FAIL (3.17s) + Compiling and running oracle program (0.17s) + Oracle output: true (3.00s) Crux output: failures: - ---- format_hex/b2ce67ae::crux_test[0] counterexamples ---- + ---- format_hex/f0662c51::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst69034e259c08d50e[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst69034e259c08d50e[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyRef TyStr Immut) Immut - [Crux] as: TyRawPtr (TyUint B8) Immut + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0] + [Crux] Translation error in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] [Crux] Overall status: Invalid. @@ -540,18 +364,16 @@ crux-mir crux doesn't match oracle Use -p '/format_hex/' to rerun this test only. - format_int: FAIL (1.07s) - Compiling and running oracle program (0.21s) - Oracle output: true (0.86s) + format_int: FAIL (2.97s) + Compiling and running oracle program (0.17s) + Oracle output: true (2.79s) Crux output: failures: - ---- format_int/4ab30ab2::crux_test[0] counterexamples ---- + ---- format_int/b92db7e2::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst69034e259c08d50e[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst69034e259c08d50e[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyRef TyStr Immut) Immut - [Crux] as: TyRawPtr (TyUint B8) Immut + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0] + [Crux] Translation error in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] [Crux] Overall status: Invalid. @@ -559,18 +381,16 @@ crux-mir crux doesn't match oracle Use -p '/format_int/' to rerun this test only. - format_struct: FAIL (1.11s) - Compiling and running oracle program (0.18s) - Oracle output: true (0.93s) + format_struct: FAIL (3.18s) + Compiling and running oracle program (0.21s) + Oracle output: true (2.97s) Crux output: failures: - ---- format_struct/3a3e646a::crux_test[0] counterexamples ---- + ---- format_struct/62367108::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst69034e259c08d50e[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst69034e259c08d50e[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyRef TyStr Immut) Immut - [Crux] as: TyRawPtr (TyUint B8) Immut + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0] + [Crux] Translation error in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] [Crux] Overall status: Invalid. @@ -578,18 +398,16 @@ crux-mir crux doesn't match oracle Use -p '/format_struct/' to rerun this test only. - format: FAIL (0.94s) - Compiling and running oracle program (0.19s) - Oracle output: true (0.75s) + format: FAIL (2.98s) + Compiling and running oracle program (0.18s) + Oracle output: true (2.80s) Crux output: failures: - ---- format/c1518bdc::crux_test[0] counterexamples ---- + ---- format/ab353d66::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst69034e259c08d50e[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst69034e259c08d50e[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyRef TyStr Immut) Immut - [Crux] as: TyRawPtr (TyUint B8) Immut + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0] + [Crux] Translation error in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] [Crux] Overall status: Invalid. @@ -597,16 +415,16 @@ crux-mir crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..str.format"' to rerun this test only. - to_owned: FAIL (0.35s) - Compiling and running oracle program (0.19s) - Oracle output: true (0.16s) + to_owned: FAIL (2.42s) + Compiling and running oracle program (0.17s) + Oracle output: true (2.25s) Crux output: failures: - ---- to_owned/41c1673f::crux_test[0] counterexamples ---- + ---- to_owned/5974b99d::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/5a1f6960::str[0]::{impl#0}[0]::as_bytes[0] - [Crux] Translation error in core/5a1f6960::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0] + [Crux] Translation error in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] [Crux] Overall status: Invalid. @@ -614,16 +432,16 @@ crux-mir crux doesn't match oracle Use -p '/to_owned/' to rerun this test only. - string_push: FAIL (0.86s) - Compiling and running oracle program (0.17s) - Oracle output: true (0.69s) + string_push: FAIL (2.97s) + Compiling and running oracle program (0.18s) + Oracle output: true (2.79s) Crux output: failures: - ---- string_push/cf3b0ae9::crux_test[0] counterexamples ---- + ---- string_push/c3ceb3d7::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/cmp.rs:1300:11: 1300:28: error: in core/5a1f6960::cmp[0]::max_by[0]::_insta10905add4aa20a5[0] - [Crux] Translation error in core/5a1f6960::cmp[0]::max_by[0]::_insta10905add4aa20a5[0]: illegal or unimplemented coercion from TyInt USize (concretely BVRepr 32) to TyInt B8 + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -631,18 +449,16 @@ crux-mir crux doesn't match oracle Use -p '/string_push/' to rerun this test only. - format_array: FAIL (1.10s) - Compiling and running oracle program (0.19s) - Oracle output: true (0.91s) + format_array: FAIL (3.41s) + Compiling and running oracle program (0.18s) + Oracle output: true (3.23s) Crux output: failures: - ---- format_array/8a564567::crux_test[0] counterexamples ---- + ---- format_array/cd770658::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst69034e259c08d50e[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst69034e259c08d50e[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyRef TyStr Immut) Immut - [Crux] as: TyRawPtr (TyUint B8) Immut + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0] + [Crux] Translation error in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] [Crux] Overall status: Invalid. @@ -651,22 +467,22 @@ crux-mir Use -p '/format_array/' to rerun this test only. sync - mutex: FAIL (0.67s) + mutex: FAIL (2.77s) Compiling and running oracle program (0.17s) - Oracle output: 1 (0.50s) - user error (JSON Decoding of test/conc_eval/sync/mutex.all.mir failed: Error in $.fns[1069].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) + Oracle output: 1 (2.59s) + user error (JSON Decoding of test/conc_eval/sync/mutex.all.mir failed: Error in $.statics[12].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) Use -p '$0=="crux-mir.crux concrete..sync.mutex"' to rerun this test only. - arc_cell: FAIL (0.30s) - Compiling and running oracle program (0.18s) - Oracle output: 4 (0.12s) + arc_cell: FAIL (2.34s) + Compiling and running oracle program (0.17s) + Oracle output: 4 (2.16s) Crux output: failures: - ---- arc_cell/685cfc00::crux_test[0] counterexamples ---- + ---- arc_cell/34dc4cdc::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -674,16 +490,16 @@ crux-mir crux doesn't match oracle Use -p '/arc_cell/' to rerun this test only. - arc: FAIL (0.27s) + arc: FAIL (2.33s) Compiling and running oracle program (0.17s) - Oracle output: 1 (0.10s) + Oracle output: 1 (2.16s) Crux output: failures: - ---- arc/346ee371::crux_test[0] counterexamples ---- + ---- arc/cc226314::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -691,18 +507,16 @@ crux-mir crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..sync.arc"' to rerun this test only. - atomic_add: FAIL (0.23s) - Compiling and running oracle program (0.17s) - Oracle output: 6 (0.06s) + atomic_add: FAIL (2.24s) + Compiling and running oracle program (0.18s) + Oracle output: 6 (2.06s) Crux output: failures: - ---- atomic_add/65495182::crux_test[0] counterexamples ---- + ---- atomic_add/4834fad5::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst8c861ae30ba93f05[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst8c861ae30ba93f05[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyAdt core/5a1f6960::sync[0]::atomic[0]::Ordering[0]::_adtb7803c2264daf0ec[0] core/5a1f6960::sync[0]::atomic[0]::Ordering[0] (Substs [])) Immut - [Crux] as: TyRawPtr (TyUint B8) Immut + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/sync/atomic.rs:3058:61: 3058:64: error: in core/1c79c753::sync[0]::atomic[0]::atomic_add[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/1c79c753::sync[0]::atomic[0]::atomic_add[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::atomic_xadd_release[0]::_inst1e2825177cd3b608[0] [Crux] Overall status: Invalid. @@ -710,24 +524,22 @@ crux-mir crux doesn't match oracle Use -p '/atomic_add/' to rerun this test only. - rwlock: FAIL (0.71s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (0.53s) - user error (JSON Decoding of test/conc_eval/sync/rwlock.all.mir failed: Error in $.fns[1097].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) + rwlock: FAIL (2.99s) + Compiling and running oracle program (0.19s) + Oracle output: 2 (2.80s) + user error (JSON Decoding of test/conc_eval/sync/rwlock.all.mir failed: Error in $.statics[12].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) Use -p '$0=="crux-mir.crux concrete..sync.rwlock"' to rerun this test only. - atomic_cxchg: FAIL (0.24s) - Compiling and running oracle program (0.18s) - Oracle output: 6 (0.06s) + atomic_cxchg: FAIL (2.31s) + Compiling and running oracle program (0.19s) + Oracle output: 6 (2.12s) Crux output: failures: - ---- atomic_cxchg/eb5b5103::crux_test[0] counterexamples ---- + ---- atomic_cxchg/ac838f7f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst8c861ae30ba93f05[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst8c861ae30ba93f05[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyAdt core/5a1f6960::sync[0]::atomic[0]::Ordering[0]::_adtb7803c2264daf0ec[0] core/5a1f6960::sync[0]::atomic[0]::Ordering[0] (Substs [])) Immut - [Crux] as: TyRawPtr (TyUint B8) Immut + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/sync/atomic.rs:3095:86: 3095:89: error: in core/1c79c753::sync[0]::atomic[0]::atomic_compare_exchange[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/1c79c753::sync[0]::atomic[0]::atomic_compare_exchange[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::atomic_cxchg_relaxed_relaxed[0]::_inst1e2825177cd3b608[0] [Crux] Overall status: Invalid. @@ -735,16 +547,16 @@ crux-mir crux doesn't match oracle Use -p '/atomic_cxchg/' to rerun this test only. - arc_clone: FAIL (0.27s) + arc_clone: FAIL (2.36s) Compiling and running oracle program (0.17s) - Oracle output: 2 (0.10s) + Oracle output: 2 (2.19s) Crux output: failures: - ---- arc_clone/760f187f::crux_test[0] counterexamples ---- + ---- arc_clone/0035f3b8::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -752,128 +564,85 @@ crux-mir crux doesn't match oracle Use -p '/arc_clone/' to rerun this test only. - atomic_swap: OK (0.19s) - Compiling and running oracle program (0.17s) - Oracle output: (2, 1) (0.02s) + atomic_swap: OK (2.22s) + Compiling and running oracle program (0.16s) + Oracle output: (2, 1) (2.06s) Crux output: (2, 1) - rwlock_multi: FAIL (0.83s) - Compiling and running oracle program (0.18s) - Oracle output: 3 (0.65s) - user error (JSON Decoding of test/conc_eval/sync/rwlock_multi.all.mir failed: Error in $.fns[1123].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) + rwlock_multi: FAIL (2.86s) + Compiling and running oracle program (0.19s) + Oracle output: 3 (2.67s) + user error (JSON Decoding of test/conc_eval/sync/rwlock_multi.all.mir failed: Error in $.statics[12].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) Use -p '/rwlock_multi/' to rerun this test only. - atomic_fence: OK (0.18s) + atomic_fence: OK (2.21s) Compiling and running oracle program (0.16s) - Oracle output: 2 (0.02s) + Oracle output: 2 (2.05s) Crux output: 2 - mutex_multi: FAIL (0.76s) + mutex_multi: FAIL (2.88s) Compiling and running oracle program (0.18s) - Oracle output: 3 (0.59s) - user error (JSON Decoding of test/conc_eval/sync/mutex_multi.all.mir failed: Error in $.fns[1079].body.blocks[0].block.data[2].rhs: unsupported RValue Just (String "ThreadLocalRef")) + Oracle output: 3 (2.70s) + user error (JSON Decoding of test/conc_eval/sync/mutex_multi.all.mir failed: Error in $.statics[12].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) Use -p '/mutex_multi/' to rerun this test only. consts - struct_val: OK (0.17s) + struct_val: OK (2.18s) Compiling and running oracle program (0.17s) - Oracle output: Foo { x: true } + Oracle output: Foo { x: true } (2.02s) Crux output: Foo { x: true } - struct_unit: FAIL (0.20s) - Compiling and running oracle program (0.18s) - Oracle output: Err(Foo { x: () }) (0.02s) - Crux output: - failures: - - ---- struct_unit/c2581b77::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in struct_unit/c2581b77::crux_test[0] - [Crux] Translation error in struct_unit/c2581b77::crux_test[0]: error building variant core/5a1f6960::result[0]::Result[0]::Err[0]: not enough expressions -- [] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/struct_unit/' to rerun this test only. - local_key: FAIL (0.17s) + struct_unit: OK (2.17s) + Compiling and running oracle program (0.17s) + Oracle output: Err(Foo { x: () }) (2.00s) + Crux output: Err(Foo { x: () }) + local_key: FAIL (2.15s) Compiling and running oracle program (0.17s) - Oracle output: 1 + Oracle output: 1 (1.97s) user error (JSON Decoding of test/conc_eval/consts/local_key.all.mir failed: Error in $.statics[0].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) Use -p '/local_key/' to rerun this test only. - fn_def: FAIL (0.16s) - Compiling and running oracle program (0.16s) - Oracle output: 1 + fn_def: FAIL (2.17s) + Compiling and running oracle program (0.19s) + Oracle output: 1 (1.98s) user error (JSON Decoding of test/conc_eval/consts/fn_def.all.mir failed: Error in $.fns[0].body.blocks[0].block.terminator.func.data.rendered: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) Use -p '/fn_def/' to rerun this test only. - enum_val: OK (0.17s) + enum_val: OK (2.19s) Compiling and running oracle program (0.17s) - Oracle output: None + Oracle output: None (2.02s) Crux output: None crypto - add: OK (0.24s) + add: OK (2.22s) Compiling and running oracle program (0.17s) - Oracle output: true (0.06s) + Oracle output: true (2.04s) Crux output: true - add_noL: OK (0.21s) + add_noL: OK (2.22s) Compiling and running oracle program (0.17s) - Oracle output: false (0.04s) + Oracle output: false (2.05s) Crux output: false cell - cell: OK (0.19s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (0.03s) + cell: OK (2.17s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (2.00s) + Crux output: 2 + ref_cell: OK (2.76s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (2.59s) + Crux output: 2 + ref_cell2: OK (2.89s) + Compiling and running oracle program (0.18s) + Oracle output: 2 (2.70s) Crux output: 2 - ref_cell: FAIL (0.83s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (0.65s) - Crux output: - failures: - - ---- ref_cell/67706a18::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/5a1f6960::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyInt B32) Mut - [Crux] as: TyRawPtr (TyUint B8) Mut - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '$0=="crux-mir.crux concrete..cell.ref_cell"' to rerun this test only. - ref_cell2: FAIL (0.78s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (0.61s) - Crux output: - failures: - - ---- ref_cell2/d5dc162d::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/5a1f6960::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyInt B32) Mut - [Crux] as: TyRawPtr (TyUint B8) Mut - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/ref_cell2/' to rerun this test only. fnptr - call: OK (0.16s) + call: OK (2.16s) Compiling and running oracle program (0.16s) - Oracle output: 2 + Oracle output: 2 (2.00s) Crux output: 2 - field: OK (0.17s) - Compiling and running oracle program (0.17s) - Oracle output: 2 + field: OK (2.16s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (2.00s) Crux output: 2 - make: OK (0.17s) + make: OK (2.14s) Compiling and running oracle program (0.17s) - Oracle output: 0 + Oracle output: 0 (1.97s) Crux output: 0 custom: rustc compilation failed for custom error output: @@ -895,16 +664,16 @@ FAIL (expected: taking address of an overridden function) (0.06s) failed to compile and run (expected failure) num - overflow: FAIL (0.20s) + overflow: FAIL (2.20s) Compiling and running oracle program (0.17s) - Oracle output: () (0.03s) + Oracle output: () (2.03s) Crux output: failures: - ---- overflow/0ff004e4::crux_test[0] counterexamples ---- + ---- overflow/9fad2f92::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/uint_macros.rs:1578:74: 1578:77 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:305:5: 306:27: error: in core/5a1f6960::num[0]::{impl#7}[0]::overflowing_sub[0] - [Crux] Translation error in core/5a1f6960::num[0]::{impl#7}[0]::overflowing_sub[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::sub_with_overflow[0]::_instaddce72e1232152c[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/uint_macros.rs:1578:74: 1578:77 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:305:5: 306:27: error: in core/1c79c753::num[0]::{impl#7}[0]::overflowing_sub[0] + [Crux] Translation error in core/1c79c753::num[0]::{impl#7}[0]::overflowing_sub[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::sub_with_overflow[0]::_instaddce72e1232152c[0] [Crux] Overall status: Invalid. @@ -912,16 +681,16 @@ FAIL (expected: taking address of an overridden function) (0.06s) crux doesn't match oracle Use -p '/num.overflow/' to rerun this test only. - from_bytes: FAIL (0.19s) - Compiling and running oracle program (0.16s) - Oracle output: () (0.03s) + from_bytes: FAIL (2.19s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.02s) Crux output: failures: - ---- from_bytes/84ee4e7f::f[0] counterexamples ---- + ---- from_bytes/2b08070a::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/uint_macros.rs:2417:37: 2417:42 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:890:5: 891:49: error: in core/5a1f6960::num[0]::{impl#8}[0]::from_ne_bytes[0] - [Crux] Translation error in core/5a1f6960::num[0]::{impl#8}[0]::from_ne_bytes[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_inst850c595be4adc72f[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/uint_macros.rs:2417:37: 2417:42 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:890:5: 891:49: error: in core/1c79c753::num[0]::{impl#8}[0]::from_ne_bytes[0] + [Crux] Translation error in core/1c79c753::num[0]::{impl#8}[0]::from_ne_bytes[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_inst850c595be4adc72f[0] [Crux] Overall status: Invalid. @@ -929,30 +698,30 @@ FAIL (expected: taking address of an overridden function) (0.06s) crux doesn't match oracle Use -p '/from_bytes/' to rerun this test only. - saturate: OK (0.76s) - Compiling and running oracle program (0.19s) - Oracle output: () (0.57s) + saturate: OK (2.79s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.62s) Crux output: () prim - bool: OK (0.17s) - Compiling and running oracle program (0.17s) - Oracle output: false + bool: OK (2.15s) + Compiling and running oracle program (0.18s) + Oracle output: false (1.97s) Crux output: false - shift3: OK (0.16s) - Compiling and running oracle program (0.16s) - Oracle output: -9223372036854775808 + shift3: OK (2.20s) + Compiling and running oracle program (0.18s) + Oracle output: -9223372036854775808 (2.02s) Crux output: -9223372036854775808 - div: OK (0.16s) + div: OK (2.13s) Compiling and running oracle program (0.16s) - Oracle output: 4 + Oracle output: 4 (1.97s) Crux output: 4 - ge: OK (0.16s) + ge: OK (2.16s) Compiling and running oracle program (0.16s) - Oracle output: true + Oracle output: true (1.99s) Crux output: true - add1: OK (0.16s) + add1: OK (2.12s) Compiling and running oracle program (0.16s) - Oracle output: 2 + Oracle output: 2 (1.96s) Crux output: 2 shift_exceeding: rustc compilation failed for shift_exceeding error output: @@ -984,21 +753,21 @@ error: aborting due to previous error; 1 warning emitted For more information about this error, try `rustc --explain E0425`. -FAIL (expected: Should panic, but doesn't) (0.06s) - Compiling and running oracle program (0.06s) +FAIL (expected: Should panic, but doesn't) (0.07s) + Compiling and running oracle program (0.07s) test/Test.hs:107: failed to compile and run (expected failure) - char_from_u32: FAIL (0.19s) - Compiling and running oracle program (0.16s) - Oracle output: 'A' (0.03s) + char_from_u32: FAIL (2.27s) + Compiling and running oracle program (0.17s) + Oracle output: 'A' (2.10s) Crux output: failures: - ---- char_from_u32/2f2f276c::crux_test[0] counterexamples ---- + ---- char_from_u32/c5a421fb::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/char/convert.rs:215:31: 215:32: error: in core/5a1f6960::char[0]::convert[0]::char_try_from_u32[0] - [Crux] Translation error in core/5a1f6960::char[0]::convert[0]::char_try_from_u32[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instdc3291f36c45d6de[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/char/convert.rs:215:31: 215:32: error: in core/1c79c753::char[0]::convert[0]::char_try_from_u32[0] + [Crux] Translation error in core/1c79c753::char[0]::convert[0]::char_try_from_u32[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instdc3291f36c45d6de[0] [Crux] Overall status: Invalid. @@ -1006,24 +775,24 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/char_from_u32/' to rerun this test only. - ffs: OK (0.20s) - Compiling and running oracle program (0.17s) - Oracle output: true (0.03s) + ffs: OK (2.24s) + Compiling and running oracle program (0.19s) + Oracle output: true (2.06s) Crux output: true - mut_arg: OK (0.17s) + mut_arg: OK (2.16s) Compiling and running oracle program (0.17s) - Oracle output: 1 + Oracle output: 1 (1.99s) Crux output: 1 - litstring: FAIL (0.19s) - Compiling and running oracle program (0.17s) - Oracle output: true (0.02s) + litstring: FAIL (2.19s) + Compiling and running oracle program (0.20s) + Oracle output: true (2.00s) Crux output: failures: - ---- litstring/dd559cbf::crux_test[0] counterexamples ---- + ---- litstring/17414821::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/5a1f6960::str[0]::{impl#0}[0]::as_bytes[0] - [Crux] Translation error in core/5a1f6960::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0] + [Crux] Translation error in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] [Crux] Overall status: Invalid. @@ -1031,58 +800,58 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/litstring/' to rerun this test only. - lit: OK (0.17s) - Compiling and running oracle program (0.16s) - Oracle output: false - Crux output: false - litbstring: OK (0.17s) + lit: OK (2.17s) Compiling and running oracle program (0.17s) - Oracle output: true - Crux output: true - shift1: OK (0.16s) + Oracle output: false (2.00s) + Crux output: false + litbstring: OK (2.29s) Compiling and running oracle program (0.16s) - Oracle output: 2 + Oracle output: true (2.13s) + Crux output: true + shift1: OK (2.13s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (1.97s) Crux output: 2 - shift2: OK (0.17s) + shift2: OK (2.17s) Compiling and running oracle program (0.17s) - Oracle output: 2 + Oracle output: 2 (2.00s) Crux output: 2 - shift4: OK (0.16s) - Compiling and running oracle program (0.16s) - Oracle output: -9223372036854775808 + shift4: OK (2.16s) + Compiling and running oracle program (0.19s) + Oracle output: -9223372036854775808 (1.97s) Crux output: -9223372036854775808 - mut: OK (0.17s) + mut: OK (2.15s) Compiling and running oracle program (0.17s) - Oracle output: 14 + Oracle output: 14 (1.98s) Crux output: 14 - wrapping_sub: OK (0.18s) - Compiling and running oracle program (0.17s) - Oracle output: true + wrapping_sub: OK (2.14s) + Compiling and running oracle program (0.16s) + Oracle output: true (1.98s) Crux output: true impl - self: OK (0.17s) - Compiling and running oracle program (0.16s) - Oracle output: 42 + self: OK (2.15s) + Compiling and running oracle program (0.17s) + Oracle output: 42 (1.98s) Crux output: 42 - simple: OK (0.16s) - Compiling and running oracle program (0.16s) - Oracle output: 42 + simple: OK (2.23s) + Compiling and running oracle program (0.18s) + Oracle output: 42 (2.05s) Crux output: 42 - self_mut: OK (0.16s) - Compiling and running oracle program (0.16s) - Oracle output: 42 + self_mut: OK (2.25s) + Compiling and running oracle program (0.19s) + Oracle output: 42 (2.07s) Crux output: 42 box - new: FAIL (0.25s) + new: FAIL (2.26s) Compiling and running oracle program (0.17s) - Oracle output: () (0.08s) + Oracle output: () (2.09s) Crux output: failures: - ---- new/f7f026f5::f[0] counterexamples ---- + ---- new/b5aad3cf::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -1090,16 +859,16 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/box.new/' to rerun this test only. - mut_ref: FAIL (0.25s) - Compiling and running oracle program (0.17s) - Oracle output: () (0.09s) + mut_ref: FAIL (2.32s) + Compiling and running oracle program (0.20s) + Oracle output: () (2.13s) Crux output: failures: - ---- mut_ref/975c3f15::f[0] counterexamples ---- + ---- mut_ref/ac037107::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -1107,16 +876,16 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/box.mut_ref/' to rerun this test only. - mut: FAIL (0.27s) - Compiling and running oracle program (0.19s) - Oracle output: () (0.08s) + mut: FAIL (2.41s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.23s) Crux output: failures: - ---- mut/5b3bcc1f::f[0] counterexamples ---- + ---- mut/b7651654::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -1124,16 +893,16 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..box.mut"' to rerun this test only. - unsize: FAIL (0.25s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (0.09s) + unsize: FAIL (2.27s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (2.11s) Crux output: failures: - ---- unsize/18882248::f[0] counterexamples ---- + ---- unsize/d08bfbca::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -1141,16 +910,16 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/box.unsize/' to rerun this test only. - struct: FAIL (0.25s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (0.08s) + struct: FAIL (2.30s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (2.13s) Crux output: failures: - ---- struct/c7e3a7f5::crux_test[0] counterexamples ---- + ---- struct/b1eca756::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -1159,57 +928,55 @@ FAIL (expected: Should panic, but doesn't) (0.06s) Use -p '/box.struct/' to rerun this test only. ptr - is_null_slice: FAIL (expected: can't unsize null pointers) (0.18s) + is_null_slice: FAIL (expected: can't unsize null pointers) (2.20s) Compiling and running oracle program (0.16s) - Oracle output: (false, true) (0.02s) + Oracle output: (false, true) (2.03s) Crux output: failures: - ---- is_null_slice/994d2033::crux_test[0] counterexamples ---- + ---- is_null_slice/b07afa7d::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/ptr/is_null_slice.rs:7:13: 7:33: error: in is_null_slice/994d2033::crux_test[0] + [Crux] test/conc_eval/ptr/is_null_slice.rs:7:13: 7:33: error: in is_null_slice/b07afa7d::crux_test[0] [Crux] attempted subindex on the result of an integer-to-pointer cast [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst55af81a2cef2701c[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TySlice (TyInt B32)) Immut - [Crux] as: TyRawPtr (TyUint B8) Immut + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:38:38: 38:42: error: in core/1c79c753::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst55af81a2cef2701c[0] + [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle (expected failure) - struct_eq: OK (0.17s) + struct_eq: OK (2.17s) Compiling and running oracle program (0.16s) - Oracle output: () (0.01s) + Oracle output: () (2.01s) Crux output: () - read_write: OK (0.20s) - Compiling and running oracle program (0.16s) - Oracle output: [1, 3, 1] (0.04s) + read_write: OK (2.23s) + Compiling and running oracle program (0.17s) + Oracle output: [1, 3, 1] (2.06s) Crux output: [1, 3, 1] - offset_mut: OK (0.19s) - Compiling and running oracle program (0.19s) - Oracle output: 3 + offset_mut: OK (2.23s) + Compiling and running oracle program (0.18s) + Oracle output: 3 (2.05s) Crux output: 3 - copy: OK (0.20s) - Compiling and running oracle program (0.17s) - Oracle output: [1, 2, 3, 1, 2, 3] (0.03s) + copy: OK (2.21s) + Compiling and running oracle program (0.18s) + Oracle output: [1, 2, 3, 1, 2, 3] (2.03s) Crux output: [1, 2, 3, 1, 2, 3] - unsize_slice: OK (0.17s) - Compiling and running oracle program (0.16s) - Oracle output: (1, 2) + unsize_slice: OK (2.16s) + Compiling and running oracle program (0.17s) + Oracle output: (1, 2) (2.00s) Crux output: (1, 2) - dangling_eq: FAIL (0.20s) - Compiling and running oracle program (0.17s) - Oracle output: () (0.03s) + dangling_eq: FAIL (2.18s) + Compiling and running oracle program (0.16s) + Oracle output: () (2.02s) Crux output: failures: - ---- dangling_eq/a15fc55d::crux_test[0] counterexamples ---- + ---- dangling_eq/82e2c977::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/mod.rs:599:29: 599:33: error: in core/5a1f6960::ptr[0]::invalid_mut[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::invalid_mut[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_inst102627303ebf2823[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:135:14: 135:78: error: in core/1c79c753::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/1c79c753::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1217,24 +984,24 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/dangling_eq/' to rerun this test only. - cast_eq: OK (0.18s) - Compiling and running oracle program (0.17s) - Oracle output: () + cast_eq: OK (2.17s) + Compiling and running oracle program (0.16s) + Oracle output: () (2.00s) Crux output: () - offset_from: OK (0.21s) - Compiling and running oracle program (0.20s) - Oracle output: -2 (0.01s) + offset_from: OK (2.18s) + Compiling and running oracle program (0.17s) + Oracle output: -2 (2.01s) Crux output: -2 - null_eq: FAIL (0.21s) - Compiling and running oracle program (0.19s) - Oracle output: 1 (0.03s) + null_eq: FAIL (2.34s) + Compiling and running oracle program (0.20s) + Oracle output: 1 (2.15s) Crux output: failures: - ---- null_eq/9be1a456::crux_test[0] counterexamples ---- + ---- null_eq/c0291176::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/mod.rs:568:29: 568:33: error: in core/5a1f6960::ptr[0]::invalid[0]::_instf59dc84daa91a404[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::invalid[0]::_instf59dc84daa91a404[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_insta5d0a99e3b08f3bd[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:118:14: 118:78: error: in core/1c79c753::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/1c79c753::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1242,131 +1009,116 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/null_eq/' to rerun this test only. - coerce_unsized: OK (0.17s) + coerce_unsized: OK (2.14s) Compiling and running oracle program (0.16s) - Oracle output: (1, 2) + Oracle output: (1, 2) (1.98s) Crux output: (1, 2) - offset: OK (0.16s) + offset: OK (2.14s) Compiling and running oracle program (0.16s) - Oracle output: 3 + Oracle output: 3 (1.99s) Crux output: 3 - is_null: FAIL (0.18s) - Compiling and running oracle program (0.16s) - Oracle output: (false, true) (0.02s) - Crux output: - failures: - - ---- is_null/7b991371::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyInt B32) Immut - [Crux] as: TyRawPtr (TyUint B8) Immut - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '$0=="crux-mir.crux concrete..ptr.is_null"' to rerun this test only. - valid_eq: OK (0.17s) + is_null: OK (2.19s) Compiling and running oracle program (0.16s) - Oracle output: () (0.02s) + Oracle output: (false, true) (2.02s) + Crux output: (false, true) + valid_eq: OK (2.17s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.00s) Crux output: () ops - deref2: OK (0.18s) + deref2: OK (2.17s) Compiling and running oracle program (0.16s) - Oracle output: () (0.02s) + Oracle output: () (2.00s) Crux output: () - index2: OK (0.19s) - Compiling and running oracle program (0.17s) - Oracle output: () (0.01s) + index2: OK (2.15s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.98s) Crux output: () - arith1: OK (0.19s) + arith1: OK (2.28s) Compiling and running oracle program (0.17s) - Oracle output: () (0.02s) + Oracle output: () (2.12s) Crux output: () - index3: OK (0.18s) + index3: OK (2.14s) Compiling and running oracle program (0.17s) - Oracle output: () (0.01s) + Oracle output: () (1.97s) Crux output: () - deref1: OK (0.16s) - Compiling and running oracle program (0.16s) - Oracle output: () + deref1: OK (2.18s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.00s) Crux output: () - index1: OK (0.16s) + index1: OK (2.11s) Compiling and running oracle program (0.16s) - Oracle output: () + Oracle output: () (1.95s) Crux output: () - deref3: OK (0.18s) - Compiling and running oracle program (0.17s) - Oracle output: () (0.01s) + deref3: OK (2.18s) + Compiling and running oracle program (0.16s) + Oracle output: () (2.01s) Crux output: () statics - promoted_fn: OK (0.16s) - Compiling and running oracle program (0.16s) - Oracle output: 1 + promoted_fn: OK (2.13s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (1.97s) Crux output: 1 - promoted_static: OK (0.16s) - Compiling and running oracle program (0.16s) - Oracle output: 1 + promoted_static: OK (2.17s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (2.01s) Crux output: 1 io - cursor_write2: FAIL (0.72s) + cursor_write2: FAIL (2.68s) Compiling and running oracle program (0.18s) - Oracle output: 0 (0.54s) - user error (JSON Decoding of test/conc_eval/io/cursor_write2.all.mir failed: Error in $.statics[61].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) + Oracle output: 0 (2.50s) + user error (JSON Decoding of test/conc_eval/io/cursor_write2.all.mir failed: Error in $.statics[13].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) Use -p '/cursor_write2/' to rerun this test only. - cursor_write: FAIL (0.66s) + cursor_write: FAIL (2.78s) Compiling and running oracle program (0.19s) - Oracle output: 0 (0.47s) - standalone use of `dyn` is not supported: TyDynamic core/5a1f6960::error[0]::Error[0]::_trait56353a7b840403b4[0] + Oracle output: 0 (2.59s) + standalone use of `dyn` is not supported: TyDynamic core/1c79c753::error[0]::Error[0]::_trait56353a7b840403b4[0] CallStack (from HasCallStack): - error, called at src/Mir/TransTy.hs:225:25 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/Trans.hs:833:12 in crucible-mir-0.1-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:901:5 in crucible-mir-0.1-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:953:30 in crucible-mir-0.1-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1165:11 in crucible-mir-0.1-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1764:68 in crucible-mir-0.1-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1773:28 in crucible-mir-0.1-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1813:10 in crucible-mir-0.1-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1859:24 in crucible-mir-0.1-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2293:30 in crucible-mir-0.1-inplace:Mir.Trans + error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/Trans.hs:824:12 in crucible-mir-0.1-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:892:5 in crucible-mir-0.1-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:944:30 in crucible-mir-0.1-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1157:11 in crucible-mir-0.1-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1602:68 in crucible-mir-0.1-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1611:28 in crucible-mir-0.1-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1651:10 in crucible-mir-0.1-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1697:24 in crucible-mir-0.1-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2131:30 in crucible-mir-0.1-inplace:Mir.Trans transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate - translateMIR, called at src/Mir/Language.hs:255:16 in crux-mir-0.6.0.99-inplace:Mir.Language + translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '$0=="crux-mir.crux concrete..io.cursor_write"' to rerun this test only. - cursor_read: FAIL (0.74s) - Compiling and running oracle program (0.20s) - Oracle output: 0 (0.55s) - standalone use of `dyn` is not supported: TyDynamic core/5a1f6960::error[0]::Error[0]::_trait56353a7b840403b4[0] + cursor_read: FAIL (2.75s) + Compiling and running oracle program (0.19s) + Oracle output: 0 (2.56s) + standalone use of `dyn` is not supported: TyDynamic core/1c79c753::error[0]::Error[0]::_trait56353a7b840403b4[0] CallStack (from HasCallStack): - error, called at src/Mir/TransTy.hs:225:25 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/Trans.hs:833:12 in crucible-mir-0.1-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:901:5 in crucible-mir-0.1-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:953:30 in crucible-mir-0.1-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1165:11 in crucible-mir-0.1-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1764:68 in crucible-mir-0.1-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1773:28 in crucible-mir-0.1-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1813:10 in crucible-mir-0.1-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1859:24 in crucible-mir-0.1-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2293:30 in crucible-mir-0.1-inplace:Mir.Trans + error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/Trans.hs:824:12 in crucible-mir-0.1-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:892:5 in crucible-mir-0.1-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:944:30 in crucible-mir-0.1-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1157:11 in crucible-mir-0.1-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1602:68 in crucible-mir-0.1-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1611:28 in crucible-mir-0.1-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1651:10 in crucible-mir-0.1-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1697:24 in crucible-mir-0.1-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2131:30 in crucible-mir-0.1-inplace:Mir.Trans transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate - translateMIR, called at src/Mir/Language.hs:255:16 in crux-mir-0.6.0.99-inplace:Mir.Language + translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '/io.cursor_read/' to rerun this test only. mem - maybe_uninit: FAIL (0.19s) - Compiling and running oracle program (0.18s) - Oracle output: 1 (0.02s) + maybe_uninit: FAIL (2.17s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (2.02s) Crux output: failures: - ---- maybe_uninit/419c4a2a::crux_test[0] counterexamples ---- + ---- maybe_uninit/e2dcf2ce::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in maybe_uninit/419c4a2a::crux_test[0] - [Crux] Translation error in maybe_uninit/419c4a2a::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyInt B32]) + [Crux] internal: error: in maybe_uninit/e2dcf2ce::crux_test[0] + [Crux] Translation error in maybe_uninit/e2dcf2ce::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyInt B32]) [Crux] Overall status: Invalid. @@ -1374,16 +1126,16 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..mem.maybe_uninit"' to rerun this test only. - maybe_uninit_array_cast: FAIL (0.19s) - Compiling and running oracle program (0.17s) - Oracle output: [1, 2] (0.02s) + maybe_uninit_array_cast: FAIL (2.21s) + Compiling and running oracle program (0.16s) + Oracle output: [1, 2] (2.05s) Crux output: failures: - ---- maybe_uninit_array_cast/c4460e75::crux_test[0] counterexamples ---- + ---- maybe_uninit_array_cast/97f384c1::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in maybe_uninit_array_cast/c4460e75::crux_test[0] - [Crux] Translation error in maybe_uninit_array_cast/c4460e75::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyArray (TyInt B32) 2]) + [Crux] internal: error: in maybe_uninit_array_cast/97f384c1::crux_test[0] + [Crux] Translation error in maybe_uninit_array_cast/97f384c1::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyArray (TyInt B32) 2]) [Crux] Overall status: Invalid. @@ -1392,43 +1144,43 @@ FAIL (expected: Should panic, but doesn't) (0.06s) Use -p '/maybe_uninit_array_cast/' to rerun this test only. time - instant: FAIL (0.64s) - Compiling and running oracle program (0.17s) - Oracle output: () (0.47s) - user error (JSON Decoding of test/conc_eval/time/instant.all.mir failed: Error in $.statics[61].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) + instant: FAIL (2.66s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.48s) + user error (JSON Decoding of test/conc_eval/time/instant.all.mir failed: Error in $.statics[12].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) Use -p '/instant/' to rerun this test only. struct - field_order: OK (0.18s) + field_order: OK (2.19s) Compiling and running oracle program (0.17s) - Oracle output: () + Oracle output: () (2.02s) Crux output: () - arg: OK (0.17s) + arg: OK (2.19s) Compiling and running oracle program (0.17s) - Oracle output: 42 + Oracle output: 42 (2.02s) Crux output: 42 - tup: OK (0.17s) - Compiling and running oracle program (0.17s) - Oracle output: () + tup: OK (2.26s) + Compiling and running oracle program (0.16s) + Oracle output: () (2.09s) Crux output: () - proj: OK (0.18s) - Compiling and running oracle program (0.17s) - Oracle output: 42 + proj: OK (2.14s) + Compiling and running oracle program (0.18s) + Oracle output: 42 (1.95s) Crux output: 42 - ret: OK (0.19s) - Compiling and running oracle program (0.19s) - Oracle output: S { x: 42, y: 120 } + ret: OK (2.14s) + Compiling and running oracle program (0.16s) + Oracle output: S { x: 42, y: 120 } (1.97s) Crux output: S { x: 42, y: 120 } - repr_transparent: FAIL (0.21s) + repr_transparent: FAIL (2.24s) Compiling and running oracle program (0.18s) - Oracle output: 6 (0.03s) + Oracle output: 6 (2.06s) Crux output: failures: - ---- repr_transparent/020bf42b::crux_test[0] counterexamples ---- + ---- repr_transparent/9888f20e::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:61:9: 61:18: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0]: unimplemented cast: Misc + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:46:9: 46:18: error: in core/1c79c753::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0] + [Crux] Translation error in core/1c79c753::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyInt B32) Immut [Crux] as: TyRawPtr (TyArray (TyInt B32) 2) Immut @@ -1438,119 +1190,104 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..struct.repr_transparent"' to rerun this test only. - repr_transparent_const: OK (0.18s) - Compiling and running oracle program (0.18s) - Oracle output: 124 + repr_transparent_const: OK (2.13s) + Compiling and running oracle program (0.16s) + Oracle output: 124 (1.97s) Crux output: 124 dyn - assoc_ty: OK (0.19s) - Compiling and running oracle program (0.19s) - Oracle output: 100 - Crux output: 100 - trait_param: OK (0.17s) + assoc_ty: OK (2.13s) Compiling and running oracle program (0.17s) - Oracle output: 100 + Oracle output: 100 (1.96s) + Crux output: 100 + trait_param: OK (2.14s) + Compiling and running oracle program (0.16s) + Oracle output: 100 (1.98s) Crux output: 100 - plain_trait: OK (0.17s) + plain_trait: OK (2.13s) Compiling and running oracle program (0.16s) - Oracle output: 100 + Oracle output: 100 (1.97s) Crux output: 100 - inherit: OK (0.18s) + inherit: OK (2.17s) Compiling and running oracle program (0.17s) - Oracle output: 4 + Oracle output: 4 (2.00s) Crux output: 4 stdlib - option: OK (0.16s) + option: OK (2.12s) Compiling and running oracle program (0.16s) - Oracle output: true + Oracle output: true (1.96s) Crux output: true - option2: OK (0.18s) - Compiling and running oracle program (0.18s) - Oracle output: 0 + option2: OK (2.13s) + Compiling and running oracle program (0.16s) + Oracle output: 0 (1.98s) Crux output: 0 - result: OK (0.19s) - Compiling and running oracle program (0.19s) - Oracle output: 27 - Crux output: 27 - default: OK (0.17s) + result: OK (2.17s) Compiling and running oracle program (0.17s) - Oracle output: true + Oracle output: 27 (2.00s) + Crux output: 27 + default: OK (2.17s) + Compiling and running oracle program (0.18s) + Oracle output: true (1.99s) Crux output: true - cvt: OK (0.16s) - Compiling and running oracle program (0.16s) - Oracle output: 0 + cvt: OK (2.14s) + Compiling and running oracle program (0.17s) + Oracle output: 0 (1.98s) Crux output: 0 - range: OK (0.16s) - Compiling and running oracle program (0.16s) - Oracle output: 10 + range: OK (2.13s) + Compiling and running oracle program (0.17s) + Oracle output: 10 (1.97s) Crux output: 10 - poly: OK (0.17s) + poly: OK (2.14s) Compiling and running oracle program (0.17s) - Oracle output: 1 + Oracle output: 1 (1.97s) Crux output: 1 - result_interior: OK (0.18s) + result_interior: OK (2.27s) Compiling and running oracle program (0.17s) - Oracle output: 3 + Oracle output: 3 (2.10s) Crux output: 3 - default_impl: OK (0.16s) - Compiling and running oracle program (0.16s) - Oracle output: () + default_impl: OK (2.16s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.00s) Crux output: () - option3: OK (0.19s) - Compiling and running oracle program (0.19s) - Oracle output: 27 + option3: OK (2.15s) + Compiling and running oracle program (0.18s) + Oracle output: 27 (1.97s) Crux output: 27 - teq: OK (0.16s) - Compiling and running oracle program (0.16s) - Oracle output: false + teq: OK (2.15s) + Compiling and running oracle program (0.17s) + Oracle output: false (1.98s) Crux output: false array - wick2: OK (0.17s) - Compiling and running oracle program (0.16s) - Oracle output: true + wick2: OK (2.14s) + Compiling and running oracle program (0.17s) + Oracle output: true (1.97s) Crux output: true - arg: OK (0.17s) - Compiling and running oracle program (0.16s) - Oracle output: 2 + arg: OK (2.13s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (1.96s) Crux output: 2 - iter: FAIL (0.22s) - Compiling and running oracle program (0.17s) - Oracle output: 3 (0.05s) - Crux output: - failures: - - ---- iter/d4aa120f::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:53:37: 53:54: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyInt B32) Immut - [Crux] as: TyRawPtr (TyUint B8) Immut - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/array.iter/' to rerun this test only. - mk_and_proj: OK (0.17s) - Compiling and running oracle program (0.16s) - Oracle output: 42 + iter: OK (2.24s) + Compiling and running oracle program (0.17s) + Oracle output: 3 (2.07s) + Crux output: 3 + mk_and_proj: OK (2.15s) + Compiling and running oracle program (0.17s) + Oracle output: 42 (1.98s) Crux output: 42 - wick3: FAIL (expected: needs Vec data structure from stdlib) (0.24s) + wick3: FAIL (expected: needs Vec data structure from stdlib) (0.23s) Compiling and running oracle program (0.19s) Oracle output: true (0.05s) user error (Error 101 while running mir-json on test/conc_eval/array/wick3.rs) (expected failure) - clone: FAIL (0.20s) + clone: FAIL (2.16s) Compiling and running oracle program (0.17s) - Oracle output: () (0.03s) + Oracle output: () (1.99s) Crux output: failures: - ---- clone/12e7688c::f[0] counterexamples ---- + ---- clone/a68cf198::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:61:9: 61:18: error: in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0]: unimplemented cast: Misc + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:46:9: 46:18: error: in core/1c79c753::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0] + [Crux] Translation error in core/1c79c753::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyInt B32) Immut [Crux] as: TyRawPtr (TyArray (TyInt B32) 3) Immut @@ -1565,16 +1302,16 @@ FAIL (expected: Should panic, but doesn't) (0.06s) Oracle output: true (0.05s) user error (Error 101 while running mir-json on test/conc_eval/array/wick1.rs) (expected failure) - mut_index: FAIL (0.76s) + mut_index: FAIL (2.76s) Compiling and running oracle program (0.17s) - Oracle output: 7 (0.60s) + Oracle output: 7 (2.59s) Crux output: failures: - ---- mut_index/00778a49::crux_test[0] counterexamples ---- + ---- mut_index/e0856e73::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1582,24 +1319,26 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/mut_index/' to rerun this test only. - mut_arg: OK (0.18s) - Compiling and running oracle program (0.18s) - Oracle output: 42 + mut_arg: OK (2.14s) + Compiling and running oracle program (0.16s) + Oracle output: 42 (1.98s) Crux output: 42 - const: OK (0.16s) + const: OK (2.13s) Compiling and running oracle program (0.16s) - Oracle output: 1 + Oracle output: 1 (1.97s) Crux output: 1 - from_slice: FAIL (0.87s) - Compiling and running oracle program (0.16s) - Oracle output: () (0.71s) + from_slice: FAIL (2.93s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.75s) Crux output: failures: - ---- from_slice/f9c79bee::f[0] counterexamples ---- + ---- from_slice/e6be3dab::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in core/5a1f6960::array[0]::{impl#10}[0]::try_from[0]::_instb3c76ad3dd4c5256[0] - [Crux] Translation error in core/5a1f6960::array[0]::{impl#10}[0]::try_from[0]::_instb3c76ad3dd4c5256[0]: error building variant core/5a1f6960::result[0]::Result[0]::Err[0]: not enough expressions -- [] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/array/mod.rs:266:23: 266:54: error: in core/1c79c753::array[0]::{impl#10}[0]::try_from[0]::_inste494657dbbea865b[0] + [Crux] Translation error in core/1c79c753::array[0]::{impl#10}[0]::try_from[0]::_inste494657dbbea865b[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyUint B8) Immut + [Crux] as: TyRawPtr (TyArray (TyUint B8) 4) Immut [Crux] Overall status: Invalid. @@ -1607,26 +1346,26 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/array.from_slice/' to rerun this test only. - const_impl: OK (0.17s) - Compiling and running oracle program (0.17s) - Oracle output: 5 + const_impl: OK (2.17s) + Compiling and running oracle program (0.16s) + Oracle output: 5 (2.00s) Crux output: 5 enum - match: OK (0.17s) + match: OK (2.14s) Compiling and running oracle program (0.17s) - Oracle output: 42 + Oracle output: 42 (1.97s) Crux output: 42 - field_order: OK (0.18s) - Compiling and running oracle program (0.17s) - Oracle output: () + field_order: OK (2.17s) + Compiling and running oracle program (0.16s) + Oracle output: () (2.01s) Crux output: () - mixed_discrs: OK (0.17s) - Compiling and running oracle program (0.17s) - Oracle output: () + mixed_discrs: OK (2.15s) + Compiling and running oracle program (0.18s) + Oracle output: () (1.98s) Crux output: () - arg: OK (0.17s) - Compiling and running oracle program (0.16s) - Oracle output: 0 + arg: OK (2.14s) + Compiling and running oracle program (0.17s) + Oracle output: 0 (1.97s) Crux output: 0 arg2: rustc compilation failed for arg2 error output: @@ -1656,16 +1395,16 @@ FAIL (0.05s) failed to compile and run Use -p '/arg2/' to rerun this test only. - cow: FAIL (0.32s) - Compiling and running oracle program (0.18s) - Oracle output: 200 (0.15s) + cow: FAIL (2.37s) + Compiling and running oracle program (0.17s) + Oracle output: 200 (2.19s) Crux output: failures: - ---- cow/95160fea::crux_test[0] counterexamples ---- + ---- cow/d3e6425a::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1673,33 +1412,33 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/cow/' to rerun this test only. - ret: OK (0.17s) + ret: OK (2.14s) Compiling and running oracle program (0.17s) - Oracle output: (A(42), B { x: 42 }, C) + Oracle output: (A(42), B { x: 42 }, C) (1.96s) Crux output: (A(42), B { x: 42 }, C) - eq: OK (0.19s) - Compiling and running oracle program (0.17s) - Oracle output: () (0.01s) + eq: OK (2.25s) + Compiling and running oracle program (0.18s) + Oracle output: () (2.07s) Crux output: () - cmp: OK (0.18s) + cmp: OK (2.13s) Compiling and running oracle program (0.17s) - Oracle output: -23 + Oracle output: -23 (1.97s) Crux output: -23 - inner: OK (0.17s) + inner: OK (2.18s) Compiling and running oracle program (0.17s) - Oracle output: 4 + Oracle output: 4 (2.01s) Crux output: 4 vec - drop: FAIL (0.29s) + drop: FAIL (2.33s) Compiling and running oracle program (0.18s) - Oracle output: () (0.11s) + Oracle output: () (2.15s) Crux output: failures: - ---- drop/af39379f::crux_test[0] counterexamples ---- + ---- drop/99a52740::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -1707,16 +1446,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/drop/' to rerun this test only. - set_len: FAIL (0.30s) + set_len: FAIL (2.31s) Compiling and running oracle program (0.18s) - Oracle output: 2 (0.12s) + Oracle output: 2 (2.13s) Crux output: failures: - ---- set_len/ce70a406::crux_test[0] counterexamples ---- + ---- set_len/44e827e2::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -1724,33 +1463,20 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/set_len/' to rerun this test only. - push: FAIL (0.38s) - Compiling and running oracle program (0.18s) - Oracle output: (1, 2) (0.20s) - Crux output: - failures: - - ---- push/1ea71252::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/cmp.rs:1300:11: 1300:28: error: in core/5a1f6960::cmp[0]::max_by[0]::_insta10905add4aa20a5[0] - [Crux] Translation error in core/5a1f6960::cmp[0]::max_by[0]::_insta10905add4aa20a5[0]: illegal or unimplemented coercion from TyInt USize (concretely BVRepr 32) to TyInt B8 - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/vec.push/' to rerun this test only. - extend: FAIL (0.41s) - Compiling and running oracle program (0.19s) - Oracle output: (1, 10) (0.23s) + push: OK (2.44s) + Compiling and running oracle program (0.19s) + Oracle output: (1, 2) (2.24s) + Crux output: (1, 2) + extend: FAIL (2.53s) + Compiling and running oracle program (0.20s) + Oracle output: (1, 10) (2.34s) Crux output: failures: - ---- extend/842dc1c4::crux_test[0] counterexamples ---- + ---- extend/97a81c1b::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/cmp.rs:1300:11: 1300:28: error: in core/5a1f6960::cmp[0]::max_by[0]::_insta10905add4aa20a5[0] - [Crux] Translation error in core/5a1f6960::cmp[0]::max_by[0]::_insta10905add4aa20a5[0]: illegal or unimplemented coercion from TyInt USize (concretely BVRepr 32) to TyInt B8 + [Crux] internal: error: in core/1c79c753::iter[0]::traits[0]::iterator[0]::Iterator[0]::try_fold[0]::_inst647f2421f6e4ebd6[0] + [Crux] Translation error in core/1c79c753::iter[0]::traits[0]::iterator[0]::Iterator[0]::try_fold[0]::_inst647f2421f6e4ebd6[0]: error building variant core/1c79c753::ops[0]::control_flow[0]::ControlFlow[0]::Continue[0]: type mismatch: expected AnyRepr but got BVRepr 32 in field 0: ([FieldRepr (FkInit AnyRepr)],[Just bVLit(32, BV 0)]) -- [Just bVLit(32, BV 0): BVRepr 32] [Crux] Overall status: Invalid. @@ -1758,65 +1484,48 @@ FAIL (0.05s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..vec.extend"' to rerun this test only. - collect: FAIL (1.07s) - Compiling and running oracle program (0.18s) - Oracle output: 45 (0.89s) - Crux output: - failures: - - ---- collect/f7c87f57::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/5a1f6960::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyInt B32) Mut - [Crux] as: TyRawPtr (TyUint B8) Mut - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/collect/' to rerun this test only. - extend_trusted_len: FAIL (1.00s) + collect: FAIL (3.15s) Compiling and running oracle program (0.19s) - Oracle output: (1, 10) (0.81s) + Oracle output: 45 (2.96s) Crux output: failures: - ---- extend_trusted_len/87428b97::crux_test[0] counterexamples ---- + ---- collect/f1727714::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/cmp.rs:1300:11: 1300:28: error: in core/5a1f6960::cmp[0]::max_by[0]::_insta10905add4aa20a5[0] - [Crux] Translation error in core/5a1f6960::cmp[0]::max_by[0]::_insta10905add4aa20a5[0]: illegal or unimplemented coercion from TyInt USize (concretely BVRepr 32) to TyInt B8 + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:786:58: 786:72: error: in core/1c79c753::ptr[0]::const_ptr[0]::{impl#0}[0]::sub_ptr[0]::runtime[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/1c79c753::ptr[0]::const_ptr[0]::{impl#0}[0]::sub_ptr[0]::runtime[0]::_inst1e2825177cd3b608[0]: No translation for pointer binop: >= [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/extend_trusted_len/' to rerun this test only. - from_elem_zero: thread 'main' panicked at 'called `Result::unwrap()` on an `Err` value: ErrorImpl { code: Io(Custom { kind: Other, error: "numeric field was not a number: when getting cksum for crate.json" }), offset: 0 }', src/bin/mir-json-dce.rs:38:44 -note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (0.22s) - Compiling and running oracle program (0.19s) - Oracle output: 0 (0.03s) - user error (Error 101 while running mir-json on ["test/conc_eval/vec/from_elem_zero.mir","rlibs/libcore.mir","rlibs/librustc_std_workspace_core.mir","rlibs/liblibc.mir","rlibs/libcompiler_builtins.mir","rlibs/liballoc.mir","rlibs/libcfg_if.mir","rlibs/libmemchr.mir","rlibs/libadler.mir","rlibs/librustc_demangle.mir","rlibs/libunwind.mir","rlibs/libpanic_unwind.mir","rlibs/librustc_std_workspace_alloc.mir","rlibs/libpanic_abort.mir","rlibs/libgimli.mir","rlibs/libstd_detect.mir","rlibs/libobject.mir","rlibs/libminiz_oxide.mir","rlibs/libhashbrown.mir","rlibs/libaddr2line.mir","rlibs/libstd.mir","rlibs/libcrucible.mir","rlibs/libint512.mir","rlibs/libbyteorder.mir","rlibs/libbytes.mir"]) + Use -p '/collect/' to rerun this test only. + extend_trusted_len: OK (3.11s) + Compiling and running oracle program (0.18s) + Oracle output: (1, 10) (2.93s) + Crux output: (1, 10) + from_elem_zero: FAIL (0.24s) + Compiling and running oracle program (0.18s) + Oracle output: 0 (0.06s) + user error (Error 101 while running mir-json on test/conc_eval/vec/from_elem_zero.rs) Use -p '/from_elem_zero/' to rerun this test only. clos - promoted: OK (0.18s) - Compiling and running oracle program (0.18s) - Oracle output: 33 + promoted: OK (2.18s) + Compiling and running oracle program (0.17s) + Oracle output: 33 (2.02s) Crux output: 33 - as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (0.18s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (0.02s) + as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (2.21s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (2.05s) Crux output: failures: - ---- as_fn_ptr/a835ceec::crux_test[0] counterexamples ---- + ---- as_fn_ptr/a97a9eab::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/a835ceec::crux_test[0] - [Crux] Translation error in as_fn_ptr/a835ceec::crux_test[0]: unimplemented cast: ClosureFnPointer + [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/a97a9eab::crux_test[0] + [Crux] Translation error in as_fn_ptr/a97a9eab::crux_test[0]: unimplemented cast: ClosureFnPointer [Crux] ty: TyClosure [] [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [], _fsreturn_ty = TyInt B32, _fsabi = RustAbi, _fsspreadarg = Nothing}) @@ -1825,102 +1534,102 @@ FAIL (0.22s) test/Test.hs:124: crux doesn't match oracle (expected failure) - fnptr_fnmut: OK (0.18s) - Compiling and running oracle program (0.18s) - Oracle output: 4 + fnptr_fnmut: OK (2.21s) + Compiling and running oracle program (0.17s) + Oracle output: 4 (2.04s) Crux output: 4 - direct_fnonce: OK (0.16s) - Compiling and running oracle program (0.16s) - Oracle output: 2 + direct_fnonce: OK (2.15s) + Compiling and running oracle program (0.19s) + Oracle output: 2 (1.97s) Crux output: 2 - conv_fnonce_fnmut: OK (0.16s) + conv_fnonce_fnmut: OK (2.18s) Compiling and running oracle program (0.16s) - Oracle output: 2 + Oracle output: 2 (2.02s) Crux output: 2 - fn_static_poly: OK (0.19s) - Compiling and running oracle program (0.19s) - Oracle output: 3 + fn_static_poly: OK (2.13s) + Compiling and running oracle program (0.17s) + Oracle output: 3 (1.96s) Crux output: 3 - fnptr_fnonce: OK (0.18s) + fnptr_fnonce: OK (2.16s) Compiling and running oracle program (0.17s) - Oracle output: 4 + Oracle output: 4 (1.99s) Crux output: 4 - fn_static: OK (0.16s) + fn_static: OK (2.16s) Compiling and running oracle program (0.16s) - Oracle output: 3 + Oracle output: 3 (1.99s) Crux output: 3 fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (0.20s) - Compiling and running oracle program (0.17s) + Compiling and running oracle program (0.16s) Oracle output: 3 (0.03s) user error (Error 101 while running mir-json on test/conc_eval/clos/fn_dyn.rs) (expected failure) - fnonce: OK (0.16s) + fnonce: OK (2.13s) Compiling and running oracle program (0.16s) - Oracle output: false + Oracle output: false (1.96s) Crux output: false - conv_fnmut_fn: OK (0.17s) + conv_fnmut_fn: OK (2.16s) Compiling and running oracle program (0.17s) - Oracle output: 2 + Oracle output: 2 (1.99s) Crux output: 2 - direct_fnmut2: OK (0.18s) - Compiling and running oracle program (0.17s) - Oracle output: 7 + direct_fnmut2: OK (2.17s) + Compiling and running oracle program (0.19s) + Oracle output: 7 (1.98s) Crux output: 7 - fo: OK (0.18s) - Compiling and running oracle program (0.18s) - Oracle output: 29 + fo: OK (2.17s) + Compiling and running oracle program (0.16s) + Oracle output: 29 (2.01s) Crux output: 29 - unique_borrow: OK (0.17s) + unique_borrow: OK (2.18s) Compiling and running oracle program (0.17s) - Oracle output: 3 + Oracle output: 3 (2.00s) Crux output: 3 - conv_fnonce_fn: OK (0.16s) + conv_fnonce_fn: OK (2.25s) Compiling and running oracle program (0.16s) - Oracle output: 2 + Oracle output: 2 (2.09s) Crux output: 2 - fnptr_fn: OK (0.17s) + fnptr_fn: OK (2.17s) Compiling and running oracle program (0.17s) - Oracle output: 4 + Oracle output: 4 (2.00s) Crux output: 4 - direct_fnmut: OK (0.19s) - Compiling and running oracle program (0.18s) - Oracle output: 7 + direct_fnmut: OK (2.16s) + Compiling and running oracle program (0.17s) + Oracle output: 7 (1.98s) Crux output: 7 - dispatch_fnmut: OK (0.18s) + dispatch_fnmut: OK (2.16s) Compiling and running oracle program (0.17s) - Oracle output: 7 (0.01s) + Oracle output: 7 (1.99s) Crux output: 7 - poly: OK (0.16s) - Compiling and running oracle program (0.16s) - Oracle output: 3 + poly: OK (2.13s) + Compiling and running oracle program (0.17s) + Oracle output: 3 (1.96s) Crux output: 3 - direct_fn: OK (0.17s) - Compiling and running oracle program (0.16s) - Oracle output: 2 + direct_fn: OK (2.14s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (1.97s) Crux output: 2 - fnptr_closure: OK (0.20s) - Compiling and running oracle program (0.20s) - Oracle output: 7 + fnptr_closure: OK (2.15s) + Compiling and running oracle program (0.16s) + Oracle output: 7 (1.98s) Crux output: 7 - fnonce1: OK (0.17s) + fnonce1: OK (2.27s) Compiling and running oracle program (0.17s) - Oracle output: 3 + Oracle output: 3 (2.10s) Crux output: 3 - ref_fnmut: OK (0.17s) - Compiling and running oracle program (0.16s) - Oracle output: 7 + ref_fnmut: OK (2.21s) + Compiling and running oracle program (0.18s) + Oracle output: 7 (2.03s) Crux output: 7 vec_deque - pop: FAIL (0.92s) + pop: FAIL (2.92s) Compiling and running oracle program (0.18s) - Oracle output: [5, 4, 3, 1, 2] (0.73s) + Oracle output: [5, 4, 3, 1, 2] (2.74s) Crux output: failures: - ---- pop/becc159b::crux_test[0] counterexamples ---- + ---- pop/b657f8fe::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -1928,16 +1637,16 @@ FAIL (0.22s) crux doesn't match oracle Use -p '/vec_deque.pop/' to rerun this test only. - iter_clone: FAIL (0.97s) - Compiling and running oracle program (0.21s) - Oracle output: [1, 2, 3, 2, 3] (0.76s) + iter_clone: FAIL (3.04s) + Compiling and running oracle program (0.19s) + Oracle output: [1, 2, 3, 2, 3] (2.85s) Crux output: failures: - ---- iter_clone/07a0e881::crux_test[0] counterexamples ---- + ---- iter_clone/aeac0d43::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -1945,16 +1654,16 @@ FAIL (0.22s) crux doesn't match oracle Use -p '/iter_clone/' to rerun this test only. - push: FAIL (0.89s) - Compiling and running oracle program (0.18s) - Oracle output: [4, 1, 2, 3, 5] (0.72s) + push: FAIL (3.16s) + Compiling and running oracle program (0.21s) + Oracle output: [4, 1, 2, 3, 5] (2.95s) Crux output: failures: - ---- push/2bbd650b::crux_test[0] counterexamples ---- + ---- push/0f2c3e92::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/cmp.rs:1300:11: 1300:28: error: in core/5a1f6960::cmp[0]::max_by[0]::_insta10905add4aa20a5[0] - [Crux] Translation error in core/5a1f6960::cmp[0]::max_by[0]::_insta10905add4aa20a5[0]: illegal or unimplemented coercion from TyInt USize (concretely BVRepr 32) to TyInt B8 + [Crux] internal: error: in alloc/8ccd0c40::collections[0]::vec_deque[0]::{impl#5}[0]::grow[0]::_inst6708c5c8c5ed40f8[0] + [Crux] panicking::panic, called from alloc/8ccd0c40::collections[0]::vec_deque[0]::{impl#5}[0]::grow[0]::_inst6708c5c8c5ed40f8[0] [Crux] Overall status: Invalid. @@ -1962,16 +1671,16 @@ FAIL (0.22s) crux doesn't match oracle Use -p '/vec_deque.push/' to rerun this test only. - retain: FAIL (0.93s) - Compiling and running oracle program (0.17s) - Oracle output: [1, 4] (0.75s) + retain: FAIL (3.11s) + Compiling and running oracle program (0.19s) + Oracle output: [1, 4] (2.92s) Crux output: failures: - ---- retain/71809604::crux_test[0] counterexamples ---- + ---- retain/9ac0c4dd::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -1979,16 +1688,16 @@ FAIL (0.22s) crux doesn't match oracle Use -p '/retain/' to rerun this test only. - rotate_left: FAIL (0.97s) + rotate_left: FAIL (3.17s) Compiling and running oracle program (0.20s) - Oracle output: [3, 4, 5, 1, 2] (0.77s) + Oracle output: [3, 4, 5, 1, 2] (2.97s) Crux output: failures: - ---- rotate_left/c536d376::crux_test[0] counterexamples ---- + ---- rotate_left/3a6979d7::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -1996,16 +1705,16 @@ FAIL (0.22s) crux doesn't match oracle Use -p '/rotate_left/' to rerun this test only. - rotate_right: FAIL (0.91s) - Compiling and running oracle program (0.18s) - Oracle output: [4, 5, 1, 2, 3] (0.73s) + rotate_right: FAIL (2.99s) + Compiling and running oracle program (0.19s) + Oracle output: [4, 5, 1, 2, 3] (2.81s) Crux output: failures: - ---- rotate_right/7b40a8a9::crux_test[0] counterexamples ---- + ---- rotate_right/a923d682::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -2014,20 +1723,20 @@ FAIL (0.22s) Use -p '/rotate_right/' to rerun this test only. slice - len: OK (0.17s) - Compiling and running oracle program (0.17s) - Oracle output: 5 + len: OK (2.13s) + Compiling and running oracle program (0.16s) + Oracle output: 5 (1.97s) Crux output: 5 - mut_range: FAIL (0.80s) - Compiling and running oracle program (0.17s) - Oracle output: 86 (0.63s) + mut_range: FAIL (2.77s) + Compiling and running oracle program (0.18s) + Oracle output: 86 (2.59s) Crux output: failures: - ---- mut_range/037e427f::crux_test[0] counterexamples ---- + ---- mut_range/92e0cd57::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -2035,16 +1744,16 @@ FAIL (0.22s) crux doesn't match oracle Use -p '/mut_range/' to rerun this test only. - range_len: FAIL (0.72s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (0.55s) + range_len: FAIL (2.91s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (2.74s) Crux output: failures: - ---- range_len/d549cefb::crux_test[0] counterexamples ---- + ---- range_len/57a23c29::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -2052,16 +1761,16 @@ FAIL (0.22s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..slice.range_len"' to rerun this test only. - range_len_mut: FAIL (0.76s) - Compiling and running oracle program (0.18s) - Oracle output: 1 (0.59s) + range_len_mut: FAIL (2.99s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (2.81s) Crux output: failures: - ---- range_len_mut/efebb8fd::crux_test[0] counterexamples ---- + ---- range_len_mut/b9e32f41::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -2069,24 +1778,24 @@ FAIL (0.22s) crux doesn't match oracle Use -p '/range_len_mut/' to rerun this test only. - mk_and_proj: OK (0.17s) - Compiling and running oracle program (0.17s) - Oracle output: 42 + mk_and_proj: OK (2.26s) + Compiling and running oracle program (0.16s) + Oracle output: 42 (2.10s) Crux output: 42 - swap: OK (0.20s) + swap: OK (2.20s) Compiling and running oracle program (0.17s) - Oracle output: 2001 (0.03s) + Oracle output: 2001 (2.03s) Crux output: 2001 - eq: FAIL (0.19s) - Compiling and running oracle program (0.16s) - Oracle output: () (0.03s) + eq: FAIL (2.23s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.06s) Crux output: failures: - ---- eq/13e22240::f[0] counterexamples ---- + ---- eq/394a586c::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/mem/mod.rs:338:38: 338:41: error: in core/5a1f6960::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/5a1f6960::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::size_of_val[0]::_inst55af81a2cef2701c[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/mem/mod.rs:338:38: 338:41: error: in core/1c79c753::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/1c79c753::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::size_of_val[0]::_inst55af81a2cef2701c[0] [Crux] Overall status: Invalid. @@ -2094,18 +1803,18 @@ FAIL (0.22s) crux doesn't match oracle Use -p '/slice.eq/' to rerun this test only. - iter_mut: FAIL (0.21s) + iter_mut: FAIL (2.25s) Compiling and running oracle program (0.17s) - Oracle output: () (0.05s) + Oracle output: () (2.07s) Crux output: failures: - ---- iter_mut/af588650::f[0] counterexamples ---- + ---- iter_mut/05baf6c7::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/mut_ptr.rs:52:37: 52:52: error: in core/5a1f6960::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::mut_ptr[0]::{impl#0}[0]::is_null[0]::_inst1e2825177cd3b608[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyInt B32) Mut - [Crux] as: TyRawPtr (TyUint B8) Mut + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/array/mod.rs:266:23: 266:54: error: in core/1c79c753::array[0]::{impl#10}[0]::try_from[0]::_insta17aeaef2aec3782[0] + [Crux] Translation error in core/1c79c753::array[0]::{impl#10}[0]::try_from[0]::_insta17aeaef2aec3782[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyInt B32) Immut + [Crux] as: TyRawPtr (TyArray (TyInt B32) 3) Immut [Crux] Overall status: Invalid. @@ -2113,16 +1822,16 @@ FAIL (0.22s) crux doesn't match oracle Use -p '/iter_mut/' to rerun this test only. - get: FAIL (0.20s) - Compiling and running oracle program (0.16s) - Oracle output: true (0.04s) + get: FAIL (2.22s) + Compiling and running oracle program (0.17s) + Oracle output: true (2.05s) Crux output: failures: - ---- get/8d7d266a::crux_test[0] counterexamples ---- + ---- get/0730ccf8::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -2130,895 +1839,862 @@ FAIL (0.22s) crux doesn't match oracle Use -p '/slice.get/' to rerun this test only. - mut: OK (0.18s) + mut: OK (2.17s) Compiling and running oracle program (0.17s) - Oracle output: 42 + Oracle output: 42 (2.00s) Crux output: 42 - last: OK (0.17s) - Compiling and running oracle program (0.16s) - Oracle output: 3 + last: OK (2.18s) + Compiling and running oracle program (0.17s) + Oracle output: 3 (2.02s) Crux output: 3 crux symbolic Output testing - mux_init_mut: FAIL (0.10s) + mux_init_mut: FAIL (2.11s) files test/symb_eval/refs/mux_init_mut.good and test/symb_eval/refs/mux_init_mut.out differ; test/symb_eval/refs/mux_init_mut.out contains: - test mux_init_mut/ba0f083a::crux_test[0]: FAILED + test mux_init_mut/822b14fd::crux_test[0]: FAILED failures: - ---- mux_init_mut/ba0f083a::crux_test[0] counterexamples ---- + ---- mux_init_mut/822b14fd::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/mux_init_mut/' to rerun this test only. - mux_init_imm: FAIL (0.11s) + mux_init_imm: FAIL (2.14s) files test/symb_eval/refs/mux_init_imm.good and test/symb_eval/refs/mux_init_imm.out differ; test/symb_eval/refs/mux_init_imm.out contains: - test mux_init_imm/f397f768::crux_test[0]: FAILED + test mux_init_imm/156ea6ac::crux_test[0]: FAILED failures: - ---- mux_init_imm/f397f768::crux_test[0] counterexamples ---- + ---- mux_init_imm/156ea6ac::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/mux_init_imm/' to rerun this test only. - early_fail: FAIL (0.13s) + early_fail: FAIL (2.14s) files test/symb_eval/crux/early_fail.good and test/symb_eval/crux/early_fail.out differ; test/symb_eval/crux/early_fail.out contains: - test early_fail/aaff3446::fail1[0]: FAILED - test early_fail/aaff3446::fail2[0]: FAILED + test early_fail/d7997ce8::fail1[0]: FAILED + test early_fail/d7997ce8::fail2[0]: FAILED failures: - ---- early_fail/aaff3446::fail1[0] counterexamples ---- + ---- early_fail/d7997ce8::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in early_fail/aaff3446::fail1[0] - [Crux] panicking::panic_fmt, called from early_fail/aaff3446::fail1[0] + [Crux] internal: error: in early_fail/d7997ce8::fail1[0] + [Crux] panicking::panic_fmt, called from early_fail/d7997ce8::fail1[0] - ---- early_fail/aaff3446::fail2[0] counterexamples ---- + ---- early_fail/d7997ce8::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/early_fail/' to rerun this test only. - mixed_fail: FAIL (0.17s) + mixed_fail: FAIL (2.25s) files test/symb_eval/crux/mixed_fail.good and test/symb_eval/crux/mixed_fail.out differ; test/symb_eval/crux/mixed_fail.out contains: - test mixed_fail/927be8d8::fail1[0]: FAILED - test mixed_fail/927be8d8::fail2[0]: FAILED - test mixed_fail/927be8d8::pass1[0]: FAILED - test mixed_fail/927be8d8::pass2[0]: FAILED + test mixed_fail/276aa1e8::fail1[0]: FAILED + test mixed_fail/276aa1e8::fail2[0]: FAILED + test mixed_fail/276aa1e8::pass1[0]: FAILED + test mixed_fail/276aa1e8::pass2[0]: FAILED failures: - ---- mixed_fail/927be8d8::fail1[0] counterexamples ---- + ---- mixed_fail/276aa1e8::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - ---- mixed_fail/927be8d8::fail2[0] counterexamples ---- + ---- mixed_fail/276aa1e8::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - ---- mixed_fail/927be8d8::pass1[0] counterexamples ---- + ---- mixed_fail/276aa1e8::pass1[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - ---- mixed_fail/927be8d8::pass2[0] counterexamples ---- + ---- mixed_fail/276aa1e8::pass2[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/mixed_fail/' to rerun this test only. - multi: FAIL (0.16s) + multi: FAIL (2.10s) files test/symb_eval/crux/multi.good and test/symb_eval/crux/multi.out differ; test/symb_eval/crux/multi.out contains: - test multi/6273896c::fail1[0]: FAILED - test multi/6273896c::fail2[0]: FAILED - test multi/6273896c::fail3[0]: FAILED + test multi/a87e06c3::fail1[0]: FAILED + test multi/a87e06c3::fail2[0]: FAILED + test multi/a87e06c3::fail3[0]: FAILED failures: - ---- multi/6273896c::fail1[0] counterexamples ---- + ---- multi/a87e06c3::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - ---- multi/6273896c::fail2[0] counterexamples ---- + ---- multi/a87e06c3::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - ---- multi/6273896c::fail3[0] counterexamples ---- + ---- multi/a87e06c3::fail3[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/Output testing.multi/' to rerun this test only. - fail_return: FAIL (0.14s) + fail_return: FAIL (2.13s) files test/symb_eval/crux/fail_return.good and test/symb_eval/crux/fail_return.out differ; test/symb_eval/crux/fail_return.out contains: - test fail_return/bbd9602c::fail1[0]: FAILED - test fail_return/bbd9602c::fail2[0]: FAILED + test fail_return/06c77a43::fail1[0]: FAILED + test fail_return/06c77a43::fail2[0]: FAILED failures: - ---- fail_return/bbd9602c::fail1[0] counterexamples ---- + ---- fail_return/06c77a43::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - ---- fail_return/bbd9602c::fail2[0] counterexamples ---- + ---- fail_return/06c77a43::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/fail_return/' to rerun this test only. - no_conc: FAIL (0.13s) + no_conc: FAIL (2.14s) files test/symb_eval/concretize/no_conc.good and test/symb_eval/concretize/no_conc.out differ; test/symb_eval/concretize/no_conc.out contains: - test no_conc/5b356412::crux_test[0]: FAILED + test no_conc/d856aa0d::crux_test[0]: FAILED failures: - ---- no_conc/5b356412::crux_test[0] counterexamples ---- + ---- no_conc/d856aa0d::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/no_conc/' to rerun this test only. - conc: FAIL (0.13s) + conc: FAIL (2.12s) files test/symb_eval/concretize/conc.good and test/symb_eval/concretize/conc.out differ; test/symb_eval/concretize/conc.out contains: - test conc/b85a9e47::crux_test[0]: FAILED + test conc/e48cfc0a::crux_test[0]: FAILED failures: - ---- conc/b85a9e47::crux_test[0] counterexamples ---- + ---- conc/e48cfc0a::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.conc"' to rerun this test only. - array: FAIL (0.13s) + array: FAIL (2.16s) files test/symb_eval/concretize/array.good and test/symb_eval/concretize/array.out differ; test/symb_eval/concretize/array.out contains: - test array/b294edd1::crux_test[0]: FAILED + test array/bf76a5b2::crux_test[0]: FAILED failures: - ---- array/b294edd1::crux_test[0] counterexamples ---- + ---- array/bf76a5b2::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::array[0]::symbolic[0]::_inst1e2825177cd3b608[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::array[0]::symbolic[0]::_inst1e2825177cd3b608[0] + [Crux] internal: error: in crucible/bfc64a60::array[0]::symbolic[0]::_inst1e2825177cd3b608[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::array[0]::symbolic[0]::_inst1e2825177cd3b608[0] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.array"' to rerun this test only. - assert_ok: FAIL (0.78s) + assert_ok: FAIL (2.73s) files test/symb_eval/concretize/assert_ok.good and test/symb_eval/concretize/assert_ok.out differ; test/symb_eval/concretize/assert_ok.out contains: - test assert_ok/223e01b8::crux_test[0]: FAILED + test assert_ok/bc7b1378::crux_test[0]: FAILED failures: - ---- assert_ok/223e01b8::crux_test[0] counterexamples ---- + ---- assert_ok/bc7b1378::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/assert_ok/' to rerun this test only. - assert: FAIL (0.73s) + assert: FAIL (2.77s) files test/symb_eval/concretize/assert.good and test/symb_eval/concretize/assert.out differ; test/symb_eval/concretize/assert.out contains: - test assert/4d446bb2::crux_test[0]: FAILED + test assert/6f5b63e9::crux_test[0]: FAILED failures: - ---- assert/4d446bb2::crux_test[0] counterexamples ---- + ---- assert/6f5b63e9::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.assert"' to rerun this test only. - bytes2: FAIL (0.17s) + bytes2: FAIL (2.23s) files test/symb_eval/crypto/bytes2.good and test/symb_eval/crypto/bytes2.out differ; test/symb_eval/crypto/bytes2.out contains: - test bytes2/cec6fb92::f[0]: FAILED + test bytes2/01526193::f[0]: FAILED failures: - ---- bytes2/cec6fb92::f[0] counterexamples ---- + ---- bytes2/01526193::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/bytes2/' to rerun this test only. - double: FAIL (0.11s) + double: FAIL (2.05s) files test/symb_eval/crypto/double.good and test/symb_eval/crypto/double.out differ; test/symb_eval/crypto/double.out contains: - test double/da43fdac::f[0]: FAILED + test double/e982a639::f[0]: FAILED failures: - ---- double/da43fdac::f[0] counterexamples ---- + ---- double/e982a639::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u32[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u32[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u32[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u32[0] [Crux] Overall status: Invalid. Use -p '/double/' to rerun this test only. - ffs: FAIL (0.13s) + ffs: FAIL (2.07s) files test/symb_eval/crypto/ffs.good and test/symb_eval/crypto/ffs.out differ; test/symb_eval/crypto/ffs.out contains: - test ffs/be5003b6::f[0]: FAILED + test ffs/f4a00052::f[0]: FAILED failures: - ---- ffs/be5003b6::f[0] counterexamples ---- + ---- ffs/f4a00052::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u32[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u32[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u32[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u32[0] [Crux] Overall status: Invalid. Use -p '/Output testing.ffs/' to rerun this test only. - bytes: FAIL (0.17s) + bytes: FAIL (2.21s) files test/symb_eval/crypto/bytes.good and test/symb_eval/crypto/bytes.out differ; test/symb_eval/crypto/bytes.out contains: - test bytes/39b68e7d::f[0]: FAILED + test bytes/eccaff21::f[0]: FAILED failures: - ---- bytes/39b68e7d::f[0] counterexamples ---- + ---- bytes/eccaff21::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u64[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u64[0] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.bytes"' to rerun this test only. - bad_symb1: FAIL (0.14s) + bad_symb1: FAIL (2.10s) files test/symb_eval/overrides/bad_symb1.good and test/symb_eval/overrides/bad_symb1.out differ; test/symb_eval/overrides/bad_symb1.out contains: - test bad_symb1/6eace5d9::crux_test[0]: FAILED + test bad_symb1/716328c8::crux_test[0]: FAILED failures: - ---- bad_symb1/6eace5d9::crux_test[0] counterexamples ---- + ---- bad_symb1/716328c8::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/bad_symb1/' to rerun this test only. - bad_symb2: FAIL (0.13s) + bad_symb2: FAIL (2.07s) files test/symb_eval/overrides/bad_symb2.good and test/symb_eval/overrides/bad_symb2.out differ; test/symb_eval/overrides/bad_symb2.out contains: - test bad_symb2/0e44e20a::crux_test[0]: FAILED + test bad_symb2/bfa11db6::crux_test[0]: FAILED failures: - ---- bad_symb2/0e44e20a::crux_test[0] counterexamples ---- + ---- bad_symb2/bfa11db6::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/bad_symb2/' to rerun this test only. - override3: FAIL (0.11s) + override3: FAIL (2.05s) files test/symb_eval/overrides/override3.good and test/symb_eval/overrides/override3.out differ; test/symb_eval/overrides/override3.out contains: - test override3/406699f3::f[0]: FAILED + test override3/337f1596::f[0]: FAILED failures: - ---- override3/406699f3::f[0] counterexamples ---- + ---- override3/337f1596::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/override3/' to rerun this test only. - override1: FAIL (0.02s) + override1: FAIL (1.93s) files test/symb_eval/overrides/override1.good and test/symb_eval/overrides/override1.out differ; test/symb_eval/overrides/override1.out contains: - test override1/84f90b58::f[0]: FAILED + test override1/774ba047::f[0]: FAILED failures: - ---- override1/84f90b58::f[0] counterexamples ---- + ---- override1/774ba047::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::one[0] - [Crux] panicking::panic, called from crucible/6eda2caa::one[0] + [Crux] internal: error: in crucible/bfc64a60::one[0] + [Crux] panicking::panic, called from crucible/bfc64a60::one[0] [Crux] Overall status: Invalid. Use -p '/override1/' to rerun this test only. - override4: FAIL (0.13s) + override4: FAIL (2.13s) files test/symb_eval/overrides/override4.good and test/symb_eval/overrides/override4.out differ; test/symb_eval/overrides/override4.out contains: - test override4/2a387b8d::f[0]: FAILED + test override4/d6e2b714::f[0]: FAILED failures: - ---- override4/2a387b8d::f[0] counterexamples ---- + ---- override4/d6e2b714::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/override4/' to rerun this test only. - override_rust: FAIL (0.11s) + override_rust: FAIL (2.08s) files test/symb_eval/overrides/override_rust.good and test/symb_eval/overrides/override_rust.out differ; test/symb_eval/overrides/override_rust.out contains: - test override_rust/81c3ff88::crux_test[0]: FAILED + test override_rust/fd4f3e57::crux_test[0]: FAILED failures: - ---- override_rust/81c3ff88::crux_test[0] counterexamples ---- + ---- override_rust/fd4f3e57::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::override_[0]::_inst28abcb7c380bf102[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::override_[0]::_inst28abcb7c380bf102[0] + [Crux] internal: error: in crucible/bfc64a60::override_[0]::_inst28abcb7c380bf102[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::override_[0]::_inst28abcb7c380bf102[0] [Crux] Overall status: Invalid. Use -p '/override_rust/' to rerun this test only. - override2: FAIL (0.11s) + override2: FAIL (2.08s) files test/symb_eval/overrides/override2.good and test/symb_eval/overrides/override2.out differ; test/symb_eval/overrides/override2.out contains: - test override2/7910119c::f[0]: FAILED + test override2/4376491c::f[0]: FAILED failures: - ---- override2/7910119c::f[0] counterexamples ---- + ---- override2/4376491c::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/override2/' to rerun this test only. - override5: FAIL (0.11s) + override5: FAIL (2.29s) files test/symb_eval/overrides/override5.good and test/symb_eval/overrides/override5.out differ; test/symb_eval/overrides/override5.out contains: - test override5/5f8d6467::f[0]: FAILED + test override5/dd33ced4::f[0]: FAILED failures: - ---- override5/5f8d6467::f[0] counterexamples ---- + ---- override5/dd33ced4::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u64[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u64[0] [Crux] Overall status: Invalid. Use -p '/override5/' to rerun this test only. - mux: FAIL (0.11s) + mux: FAIL (2.13s) files test/symb_eval/fnptr/mux.good and test/symb_eval/fnptr/mux.out differ; test/symb_eval/fnptr/mux.out contains: - test mux/3918b96f::crux_test[0]: FAILED + test mux/42d63c18::crux_test[0]: FAILED failures: - ---- mux/3918b96f::crux_test[0] counterexamples ---- + ---- mux/42d63c18::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.mux"' to rerun this test only. - checked_mul_signed: FAIL (1.93s) - files test/symb_eval/num/checked_mul_signed.good and test/symb_eval/num/checked_mul_signed.out differ; test/symb_eval/num/checked_mul_signed.out contains: - test checked_mul_signed/aea9eef4::crux_test[0]: returned 44, FAILED - - failures: - - ---- checked_mul_signed/aea9eef4::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/num/checked_mul_signed.rs:4:5: 4:13: error: in checked_mul_signed/aea9eef4::crux_test[0] - [Crux] attempt to compute `const -100_i8 * move _2`, which would overflow - - [Crux] Overall status: Invalid. - + checked_mul_signed: FAIL (0.05s) + test/symb_eval/num/checked_mul_signed.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/num/checked_mul_signed.rs) Use -p '/checked_mul_signed/' to rerun this test only. - checked_mul: FAIL (2.02s) - files test/symb_eval/num/checked_mul.good and test/symb_eval/num/checked_mul.out differ; test/symb_eval/num/checked_mul.out contains: - test checked_mul/840048c9::crux_test[0]: returned 44, FAILED - - failures: - - ---- checked_mul/840048c9::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/num/checked_mul.rs:4:5: 4:12: error: in checked_mul/840048c9::crux_test[0] - [Crux] attempt to compute `const 100_u8 * move _2`, which would overflow - - [Crux] Overall status: Invalid. - + checked_mul: FAIL + test/symb_eval/num/checked_mul.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/num/checked_mul.rs) Use -p '$0=="crux-mir.crux symbolic.Output testing.checked_mul"' to rerun this test only. - checked_div: FAIL (0.02s) + checked_div: FAIL (2.04s) files test/symb_eval/num/checked_div.good and test/symb_eval/num/checked_div.out differ; test/symb_eval/num/checked_div.out contains: - test checked_div/258a4803::crux_test[0]: returned 65, FAILED + test checked_div/5f132fcb::crux_test[0]: returned 65, FAILED failures: - ---- checked_div/258a4803::crux_test[0] counterexamples ---- + ---- checked_div/5f132fcb::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/258a4803::div_signed[0] + [Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/5f132fcb::div_signed[0] [Crux] attempt to compute `_3 / _4`, which would overflow [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/258a4803::div_signed[0] + [Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/5f132fcb::div_signed[0] [Crux] attempt to divide `_3` by zero [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/258a4803::div_unsigned[0] + [Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/5f132fcb::div_unsigned[0] [Crux] attempt to divide `_3` by zero [Crux] Overall status: Invalid. Use -p '/checked_div/' to rerun this test only. - checked_add: FAIL (1.94s) - files test/symb_eval/num/checked_add.good and test/symb_eval/num/checked_add.out differ; test/symb_eval/num/checked_add.out contains: - test checked_add/b2604487::crux_test[0]: returned 44, FAILED - - failures: - - ---- checked_add/b2604487::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/num/checked_add.rs:5:5: 5:12: error: in checked_add/b2604487::crux_test[0] - [Crux] attempt to compute `const 100_u8 + move _2`, which would overflow - - [Crux] Overall status: Invalid. - + checked_add: FAIL (0.05s) + test/symb_eval/num/checked_add.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/num/checked_add.rs) Use -p '/checked_add/' to rerun this test only. - test1: FAIL (0.23s) + test1: FAIL (2.37s) files test/symb_eval/scalar/test1.good and test/symb_eval/scalar/test1.out differ; test/symb_eval/scalar/test1.out contains: - test test1/b6abe038::f[0]: FAILED + test test1/51764b01::f[0]: FAILED failures: - ---- test1/b6abe038::f[0] counterexamples ---- + ---- test1/51764b01::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in int512/11aae4f6::symbolic[0] - [Crux] panicking::panic, called from int512/11aae4f6::symbolic[0] + [Crux] internal: error: in int512/e9689d74::symbolic[0] + [Crux] panicking::panic, called from int512/e9689d74::symbolic[0] [Crux] Overall status: Invalid. Use -p '/test1/' to rerun this test only. - pop: FAIL (0.13s) + pop: FAIL (2.14s) files test/symb_eval/vector/pop.good and test/symb_eval/vector/pop.out differ; test/symb_eval/vector/pop.out contains: - test pop/34bcb08d::f[0]: FAILED + test pop/d7c48ae6::f[0]: FAILED failures: - ---- pop/34bcb08d::f[0] counterexamples ---- + ---- pop/d7c48ae6::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in pop/34bcb08d::f[0] - [Crux] Translation error in pop/34bcb08d::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in pop/d7c48ae6::f[0] + [Crux] Translation error in pop/d7c48ae6::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/Output testing.pop/' to rerun this test only. - as_mut_slice: FAIL (0.13s) + as_mut_slice: FAIL (2.18s) files test/symb_eval/vector/as_mut_slice.good and test/symb_eval/vector/as_mut_slice.out differ; test/symb_eval/vector/as_mut_slice.out contains: - test as_mut_slice/7ce532be::f[0]: FAILED + test as_mut_slice/1fe505fc::f[0]: FAILED failures: - ---- as_mut_slice/7ce532be::f[0] counterexamples ---- + ---- as_mut_slice/1fe505fc::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in as_mut_slice/7ce532be::f[0] - [Crux] Translation error in as_mut_slice/7ce532be::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in as_mut_slice/1fe505fc::f[0] + [Crux] Translation error in as_mut_slice/1fe505fc::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/as_mut_slice/' to rerun this test only. - push: FAIL (0.12s) + push: FAIL (2.15s) files test/symb_eval/vector/push.good and test/symb_eval/vector/push.out differ; test/symb_eval/vector/push.out contains: - test push/9484b802::f[0]: FAILED + test push/35673522::f[0]: FAILED failures: - ---- push/9484b802::f[0] counterexamples ---- + ---- push/35673522::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in push/9484b802::f[0] - [Crux] Translation error in push/9484b802::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in push/35673522::f[0] + [Crux] Translation error in push/35673522::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/Output testing.push/' to rerun this test only. - copy_from_slice: FAIL (0.13s) + copy_from_slice: FAIL (2.15s) files test/symb_eval/vector/copy_from_slice.good and test/symb_eval/vector/copy_from_slice.out differ; test/symb_eval/vector/copy_from_slice.out contains: - test copy_from_slice/891db15e::f[0]: FAILED + test copy_from_slice/089d2837::f[0]: FAILED failures: - ---- copy_from_slice/891db15e::f[0] counterexamples ---- + ---- copy_from_slice/089d2837::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/vector/copy_from_slice.rs:7:43: 7:56: error: in copy_from_slice/891db15e::f[0] - [Crux] Translation error in copy_from_slice/891db15e::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] test/symb_eval/vector/copy_from_slice.rs:7:43: 7:56: error: in copy_from_slice/089d2837::f[0] + [Crux] Translation error in copy_from_slice/089d2837::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/copy_from_slice/' to rerun this test only. - new: FAIL (0.12s) + new: FAIL (2.19s) files test/symb_eval/vector/new.good and test/symb_eval/vector/new.out differ; test/symb_eval/vector/new.out contains: - test new/7c232b74::f[0]: FAILED + test new/bbccbf1b::f[0]: FAILED failures: - ---- new/7c232b74::f[0] counterexamples ---- + ---- new/bbccbf1b::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in new/7c232b74::f[0] - [Crux] Translation error in new/7c232b74::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in new/bbccbf1b::f[0] + [Crux] Translation error in new/bbccbf1b::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.new"' to rerun this test only. - mut: FAIL (0.12s) + mut: FAIL (2.10s) files test/symb_eval/vector/mut.good and test/symb_eval/vector/mut.out differ; test/symb_eval/vector/mut.out contains: - test mut/6449c4fb::f[0]: FAILED + test mut/9fe4b5d9::f[0]: FAILED failures: - ---- mut/6449c4fb::f[0] counterexamples ---- + ---- mut/9fe4b5d9::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in mut/6449c4fb::f[0] - [Crux] Translation error in mut/6449c4fb::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in mut/9fe4b5d9::f[0] + [Crux] Translation error in mut/9fe4b5d9::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/Output testing.mut/' to rerun this test only. - concat: FAIL (0.12s) + concat: FAIL (2.21s) files test/symb_eval/vector/concat.good and test/symb_eval/vector/concat.out differ; test/symb_eval/vector/concat.out contains: - test concat/8ea0a338::f[0]: FAILED + test concat/6204baae::f[0]: FAILED failures: - ---- concat/8ea0a338::f[0] counterexamples ---- + ---- concat/6204baae::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in concat/8ea0a338::f[0] - [Crux] Translation error in concat/8ea0a338::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in concat/6204baae::f[0] + [Crux] Translation error in concat/6204baae::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/concat/' to rerun this test only. - split_at: FAIL (0.13s) + split_at: FAIL (2.17s) files test/symb_eval/vector/split_at.good and test/symb_eval/vector/split_at.out differ; test/symb_eval/vector/split_at.out contains: - test split_at/096c0611::f[0]: FAILED + test split_at/ce3992d1::f[0]: FAILED failures: - ---- split_at/096c0611::f[0] counterexamples ---- + ---- split_at/ce3992d1::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/vector/split_at.rs:7:43: 7:56: error: in split_at/096c0611::f[0] - [Crux] Translation error in split_at/096c0611::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] test/symb_eval/vector/split_at.rs:7:43: 7:56: error: in split_at/ce3992d1::f[0] + [Crux] Translation error in split_at/ce3992d1::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/split_at/' to rerun this test only. - replicate: FAIL (0.13s) + replicate: FAIL (2.24s) files test/symb_eval/vector/replicate.good and test/symb_eval/vector/replicate.out differ; test/symb_eval/vector/replicate.out contains: - test replicate/776d13b1::f[0]: FAILED + test replicate/a34bdb46::f[0]: FAILED failures: - ---- replicate/776d13b1::f[0] counterexamples ---- + ---- replicate/a34bdb46::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in replicate/776d13b1::f[0] - [Crux] Translation error in replicate/776d13b1::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in replicate/a34bdb46::f[0] + [Crux] Translation error in replicate/a34bdb46::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/replicate/' to rerun this test only. - as_slice: FAIL (0.13s) + as_slice: FAIL (2.19s) files test/symb_eval/vector/as_slice.good and test/symb_eval/vector/as_slice.out differ; test/symb_eval/vector/as_slice.out contains: - test as_slice/a27e71ac::f[0]: FAILED + test as_slice/190f97f7::f[0]: FAILED failures: - ---- as_slice/a27e71ac::f[0] counterexamples ---- + ---- as_slice/190f97f7::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in as_slice/a27e71ac::f[0] - [Crux] Translation error in as_slice/a27e71ac::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in as_slice/190f97f7::f[0] + [Crux] Translation error in as_slice/190f97f7::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/as_slice/' to rerun this test only. - array_mut: FAIL (0.11s) + array_mut: FAIL (2.17s) files test/symb_eval/mux/array_mut.good and test/symb_eval/mux/array_mut.out differ; test/symb_eval/mux/array_mut.out contains: - test array_mut/57ba5703::crux_test[0]: FAILED + test array_mut/c60a5b51::crux_test[0]: FAILED failures: - ---- array_mut/57ba5703::crux_test[0] counterexamples ---- + ---- array_mut/c60a5b51::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/array_mut/' to rerun this test only. - array: FAIL (0.11s) + array: FAIL (2.20s) files test/symb_eval/mux/array.good and test/symb_eval/mux/array.out differ; test/symb_eval/mux/array.out contains: - test array/0ee8d2fc::crux_test[0]: FAILED + test array/457c9e83::crux_test[0]: FAILED failures: - ---- array/0ee8d2fc::crux_test[0] counterexamples ---- + ---- array/457c9e83::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.array"' to rerun this test only. - extend_bytes: FAIL (0.13s) + extend_bytes: FAIL (2.24s) files test/symb_eval/bytes/extend_bytes.good and test/symb_eval/bytes/extend_bytes.out differ; test/symb_eval/bytes/extend_bytes.out contains: - test extend_bytes/3e4d1139::f[0]: FAILED + test extend_bytes/cbf12b9c::f[0]: FAILED failures: - ---- extend_bytes/3e4d1139::f[0] counterexamples ---- + ---- extend_bytes/cbf12b9c::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312a3ac0::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/312a3ac0::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/0b664e91::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/0b664e91::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/extend_bytes/' to rerun this test only. - put: FAIL (0.58s) + put: FAIL (2.65s) files test/symb_eval/bytes/put.good and test/symb_eval/bytes/put.out differ; test/symb_eval/bytes/put.out contains: - test put/da785c5a::f[0]: FAILED + test put/e40a3486::f[0]: FAILED failures: - ---- put/da785c5a::f[0] counterexamples ---- + ---- put/e40a3486::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312a3ac0::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/312a3ac0::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/0b664e91::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/0b664e91::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.put"' to rerun this test only. - split_to: FAIL (0.60s) + split_to: FAIL (2.59s) files test/symb_eval/bytes/split_to.good and test/symb_eval/bytes/split_to.out differ; test/symb_eval/bytes/split_to.out contains: - test split_to/99272bb6::f[0]: FAILED + test split_to/8021023a::f[0]: FAILED failures: - ---- split_to/99272bb6::f[0] counterexamples ---- + ---- split_to/8021023a::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312a3ac0::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/312a3ac0::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/0b664e91::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/0b664e91::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/split_to/' to rerun this test only. - new: FAIL (0.10s) + new: FAIL (2.13s) files test/symb_eval/bytes/new.good and test/symb_eval/bytes/new.out differ; test/symb_eval/bytes/new.out contains: - test new/6f028dc4::f[0]: FAILED + test new/08a0233f::f[0]: FAILED failures: - ---- new/6f028dc4::f[0] counterexamples ---- + ---- new/08a0233f::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312a3ac0::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/312a3ac0::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/0b664e91::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/0b664e91::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.new"' to rerun this test only. - split_off: FAIL (0.57s) + split_off: FAIL (2.60s) files test/symb_eval/bytes/split_off.good and test/symb_eval/bytes/split_off.out differ; test/symb_eval/bytes/split_off.out contains: - test split_off/4bbd6335::f[0]: FAILED + test split_off/f72c19fe::f[0]: FAILED failures: - ---- split_off/4bbd6335::f[0] counterexamples ---- + ---- split_off/f72c19fe::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312a3ac0::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/312a3ac0::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/0b664e91::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/0b664e91::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/split_off/' to rerun this test only. - sym_len: FAIL (0.62s) + sym_len: FAIL (2.62s) files test/symb_eval/bytes/sym_len.good and test/symb_eval/bytes/sym_len.out differ; test/symb_eval/bytes/sym_len.out contains: - test sym_len/c70884d6::f[0]: FAILED + test sym_len/d62ca140::f[0]: FAILED failures: - ---- sym_len/c70884d6::f[0] counterexamples ---- + ---- sym_len/d62ca140::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312a3ac0::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/312a3ac0::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/0b664e91::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/0b664e91::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/sym_len/' to rerun this test only. - put_overflow: FAIL (0.63s) + put_overflow: FAIL (2.54s) files test/symb_eval/bytes/put_overflow.good and test/symb_eval/bytes/put_overflow.out differ; test/symb_eval/bytes/put_overflow.out contains: - test put_overflow/26c5f05d::f[0]: FAILED + test put_overflow/42b7073f::f[0]: FAILED failures: - ---- put_overflow/26c5f05d::f[0] counterexamples ---- + ---- put_overflow/42b7073f::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312a3ac0::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/312a3ac0::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/0b664e91::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/0b664e91::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/put_overflow/' to rerun this test only. - vec_cursor_read: FAIL (0.43s) - test/symb_eval/io/vec_cursor_read.out: withFile: user error (JSON Decoding of test/symb_eval/io/vec_cursor_read.all.mir failed: Error in $.statics[60].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) + vec_cursor_read: FAIL (2.55s) + test/symb_eval/io/vec_cursor_read.out: withFile: user error (JSON Decoding of test/symb_eval/io/vec_cursor_read.all.mir failed: Error in $.statics[12].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) Use -p '/vec_cursor_read/' to rerun this test only. - vec_write: FAIL (0.48s) - standalone use of `dyn` is not supported: TyDynamic core/5a1f6960::error[0]::Error[0]::_trait56353a7b840403b4[0] + vec_write: FAIL (2.52s) + standalone use of `dyn` is not supported: TyDynamic core/1c79c753::error[0]::Error[0]::_trait56353a7b840403b4[0] CallStack (from HasCallStack): - error, called at src/Mir/TransTy.hs:225:25 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/Trans.hs:833:12 in crucible-mir-0.1-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:901:5 in crucible-mir-0.1-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:953:30 in crucible-mir-0.1-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1165:11 in crucible-mir-0.1-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1764:68 in crucible-mir-0.1-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1773:28 in crucible-mir-0.1-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1813:10 in crucible-mir-0.1-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1859:24 in crucible-mir-0.1-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2293:30 in crucible-mir-0.1-inplace:Mir.Trans + error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/Trans.hs:824:12 in crucible-mir-0.1-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:892:5 in crucible-mir-0.1-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:944:30 in crucible-mir-0.1-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1157:11 in crucible-mir-0.1-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1602:68 in crucible-mir-0.1-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1611:28 in crucible-mir-0.1-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1651:10 in crucible-mir-0.1-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1697:24 in crucible-mir-0.1-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2131:30 in crucible-mir-0.1-inplace:Mir.Trans transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate - translateMIR, called at src/Mir/Language.hs:255:16 in crux-mir-0.6.0.99-inplace:Mir.Language + translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '/vec_write/' to rerun this test only. - write: FAIL + write: FAIL (0.06s) test/symb_eval/byteorder/write.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/write.rs) Use -p '/Output testing.write/' to rerun this test only. read: FAIL test/symb_eval/byteorder/read.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/read.rs) Use -p '/Output testing.read/' to rerun this test only. - slice_mut: FAIL (0.12s) + slice_mut: FAIL (2.13s) files test/symb_eval/array/slice_mut.good and test/symb_eval/array/slice_mut.out differ; test/symb_eval/array/slice_mut.out contains: - test slice_mut/9abd2b5d::crux_test[0]: FAILED + test slice_mut/4fd64ade::crux_test[0]: FAILED failures: - ---- slice_mut/9abd2b5d::crux_test[0] counterexamples ---- + ---- slice_mut/4fd64ade::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in slice_mut/9abd2b5d::crux_test[0] - [Crux] Translation error in slice_mut/9abd2b5d::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/6eda2caa::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/6eda2caa::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/6eda2caa::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) + [Crux] internal: error: in slice_mut/4fd64ade::crux_test[0] + [Crux] Translation error in slice_mut/4fd64ade::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/bfc64a60::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/bfc64a60::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/bfc64a60::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/slice_mut/' to rerun this test only. - basic: FAIL (0.13s) + basic: FAIL (2.15s) files test/symb_eval/array/basic.good and test/symb_eval/array/basic.out differ; test/symb_eval/array/basic.out contains: - test basic/79cad869::crux_test[0]: FAILED + test basic/6ffe7bac::crux_test[0]: FAILED failures: - ---- basic/79cad869::crux_test[0] counterexamples ---- + ---- basic/6ffe7bac::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in basic/79cad869::crux_test[0] - [Crux] Translation error in basic/79cad869::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/6eda2caa::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/6eda2caa::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/6eda2caa::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/6eda2caa::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) + [Crux] internal: error: in basic/6ffe7bac::crux_test[0] + [Crux] Translation error in basic/6ffe7bac::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/bfc64a60::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/bfc64a60::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/bfc64a60::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/bfc64a60::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/Output testing.basic/' to rerun this test only. - slice: FAIL (0.15s) + slice: FAIL (2.11s) files test/symb_eval/array/slice.good and test/symb_eval/array/slice.out differ; test/symb_eval/array/slice.out contains: - test slice/0e7bff2f::crux_test[0]: FAILED + test slice/3d09fb76::crux_test[0]: FAILED failures: - ---- slice/0e7bff2f::crux_test[0] counterexamples ---- + ---- slice/3d09fb76::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in slice/0e7bff2f::crux_test[0] - [Crux] Translation error in slice/0e7bff2f::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/6eda2caa::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/6eda2caa::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/6eda2caa::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) + [Crux] internal: error: in slice/3d09fb76::crux_test[0] + [Crux] Translation error in slice/3d09fb76::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/bfc64a60::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/bfc64a60::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/bfc64a60::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.slice"' to rerun this test only. - mux_slice: FAIL (0.15s) + mux_slice: FAIL (2.11s) files test/symb_eval/array/mux_slice.good and test/symb_eval/array/mux_slice.out differ; test/symb_eval/array/mux_slice.out contains: - test mux_slice/508aacc9::crux_test[0]: FAILED + test mux_slice/a488aa59::crux_test[0]: FAILED failures: - ---- mux_slice/508aacc9::crux_test[0] counterexamples ---- + ---- mux_slice/a488aa59::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in mux_slice/508aacc9::crux_test[0] - [Crux] Translation error in mux_slice/508aacc9::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/6eda2caa::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/6eda2caa::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/6eda2caa::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) + [Crux] internal: error: in mux_slice/a488aa59::crux_test[0] + [Crux] Translation error in mux_slice/a488aa59::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/bfc64a60::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/bfc64a60::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/bfc64a60::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/mux_slice/' to rerun this test only. - mux: FAIL (0.11s) + mux: FAIL (2.13s) files test/symb_eval/enum/mux.good and test/symb_eval/enum/mux.out differ; test/symb_eval/enum/mux.out contains: - test mux/2bd00921::test[0]: FAILED + test mux/f411b525::test[0]: FAILED failures: - ---- mux/2bd00921::test[0] counterexamples ---- + ---- mux/f411b525::test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.mux"' to rerun this test only. - uninit_read: FAIL (0.12s) + uninit_read: FAIL (2.15s) files test/symb_eval/alloc/uninit_read.good and test/symb_eval/alloc/uninit_read.out differ; test/symb_eval/alloc/uninit_read.out contains: - test uninit_read/e9c1be4c::crux_test[0]: FAILED + test uninit_read/1ca33845::crux_test[0]: FAILED failures: - ---- uninit_read/e9c1be4c::crux_test[0] counterexamples ---- + ---- uninit_read/1ca33845::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/e9c1be4c::crux_test[0] + [Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/1ca33845::crux_test[0] [Crux] Attempted to read uninitialized vector index [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/e9c1be4c::crux_test[0] + [Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/1ca33845::crux_test[0] [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. Use -p '/uninit_read/' to rerun this test only. - valid_read: FAIL (0.14s) + valid_read: FAIL (2.13s) files test/symb_eval/alloc/valid_read.good and test/symb_eval/alloc/valid_read.out differ; test/symb_eval/alloc/valid_read.out contains: - test valid_read/b81b29bc::crux_test[0]: returned 45, ok + test valid_read/8c374ae6::crux_test[0]: returned 45, ok [Crux] Overall status: Valid. Use -p '/valid_read/' to rerun this test only. - out_of_bounds: FAIL (0.14s) + out_of_bounds: FAIL (2.25s) files test/symb_eval/alloc/out_of_bounds.good and test/symb_eval/alloc/out_of_bounds.out differ; test/symb_eval/alloc/out_of_bounds.out contains: - test out_of_bounds/a4641c13::crux_test[0]: FAILED + test out_of_bounds/21a0dca0::crux_test[0]: FAILED failures: - ---- out_of_bounds/a4641c13::crux_test[0] counterexamples ---- + ---- out_of_bounds/21a0dca0::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/a4641c13::crux_test[0] + [Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/21a0dca0::crux_test[0] [Crux] vector index out of range: the length is 10 but the index is BV 12 [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/a4641c13::crux_test[0] + [Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/21a0dca0::crux_test[0] [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. Use -p '/out_of_bounds/' to rerun this test only. - zero_length: FAIL (0.11s) + zero_length: FAIL (2.13s) files test/symb_eval/alloc/zero_length.good and test/symb_eval/alloc/zero_length.out differ; test/symb_eval/alloc/zero_length.out contains: - test zero_length/a53db669::crux_test[0]: ok + test zero_length/6d5f0f4c::crux_test[0]: ok [Crux] Overall status: Valid. Use -p '/zero_length/' to rerun this test only. - downcast: FAIL (0.10s) + downcast: FAIL (2.04s) files test/symb_eval/any/downcast.good and test/symb_eval/any/downcast.out differ; test/symb_eval/any/downcast.out contains: - test downcast/cc755b1e::crux_test[0]: returned 1, ok + test downcast/0fb4b328::crux_test[0]: returned 1, ok [Crux] Overall status: Valid. Use -p '$0=="crux-mir.crux symbolic.Output testing.downcast"' to rerun this test only. - downcast_fail: FAIL (0.10s) + downcast_fail: FAIL (2.11s) files test/symb_eval/any/downcast_fail.good and test/symb_eval/any/downcast_fail.out differ; test/symb_eval/any/downcast_fail.out contains: - test downcast_fail/adccf677::crux_test[0]: FAILED + test downcast_fail/222b7fba::crux_test[0]: FAILED failures: - ---- downcast_fail/adccf677::crux_test[0] counterexamples ---- + ---- downcast_fail/222b7fba::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/any/downcast_fail.rs:10:14: 10:15: error: in downcast_fail/adccf677::crux_test[0] + [Crux] test/symb_eval/any/downcast_fail.rs:10:14: 10:15: error: in downcast_fail/222b7fba::crux_test[0] [Crux] failed to downcast Any as BVRepr 32 [Crux] Overall status: Invalid. @@ -3027,184 +2703,184 @@ FAIL (0.22s) conditional: FAIL (0.05s) test/symb_eval/any/conditional.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/any/conditional.rs) Use -p '/conditional/' to rerun this test only. - literals: FAIL (0.02s) + literals: FAIL (2.15s) files test/symb_eval/bitvector/literals.good and test/symb_eval/bitvector/literals.out differ; test/symb_eval/bitvector/literals.out contains: - test literals/bc9c2ca9::crux_test[0]: FAILED + test literals/a1226597::crux_test[0]: FAILED failures: - ---- literals/bc9c2ca9::crux_test[0] counterexamples ---- + ---- literals/a1226597::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:91:29: 91:30 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:125:1: 125:53: error: in crucible/6eda2caa::bitvector[0]::{impl#27}[0]::from[0]::_inst9ea2f935eda83dab[0] - [Crux] Translation error in crucible/6eda2caa::bitvector[0]::{impl#27}[0]::from[0]::_inst9ea2f935eda83dab[0]: BUG: invalid arguments to bv_convert: [$38: AnyRepr] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:91:29: 91:30 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:125:1: 125:53: error: in crucible/bfc64a60::bitvector[0]::{impl#27}[0]::from[0]::_inst9ea2f935eda83dab[0] + [Crux] Translation error in crucible/bfc64a60::bitvector[0]::{impl#27}[0]::from[0]::_inst9ea2f935eda83dab[0]: BUG: invalid arguments to bv_convert: [$38: AnyRepr] [Crux] Overall status: Invalid. Use -p '/literals/' to rerun this test only. - arith: FAIL (0.17s) + arith: FAIL (2.22s) files test/symb_eval/bitvector/arith.good and test/symb_eval/bitvector/arith.out differ; test/symb_eval/bitvector/arith.out contains: - test arith/58d00074::crux_test[0]: FAILED + test arith/f5fb78e6::crux_test[0]: FAILED failures: - ---- arith/58d00074::crux_test[0] counterexamples ---- + ---- arith/f5fb78e6::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u64[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u64[0] [Crux] Overall status: Invalid. Use -p '/Output testing.arith/' to rerun this test only. - leading_zeros: FAIL (0.16s) + leading_zeros: FAIL (2.13s) files test/symb_eval/bitvector/leading_zeros.good and test/symb_eval/bitvector/leading_zeros.out differ; test/symb_eval/bitvector/leading_zeros.out contains: - test leading_zeros/5d113916::crux_test[0]: FAILED + test leading_zeros/38d9f72d::crux_test[0]: FAILED failures: - ---- leading_zeros/5d113916::crux_test[0] counterexamples ---- + ---- leading_zeros/38d9f72d::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u64[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u64[0] [Crux] Overall status: Invalid. Use -p '/leading_zeros/' to rerun this test only. - overflowing_sub: FAIL (0.14s) + overflowing_sub: FAIL (2.08s) files test/symb_eval/bitvector/overflowing_sub.good and test/symb_eval/bitvector/overflowing_sub.out differ; test/symb_eval/bitvector/overflowing_sub.out contains: - test overflowing_sub/0e7bfc8b::crux_test[0]: FAILED + test overflowing_sub/f396bf39::crux_test[0]: FAILED failures: - ---- overflowing_sub/0e7bfc8b::crux_test[0] counterexamples ---- + ---- overflowing_sub/f396bf39::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u64[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u64[0] [Crux] Overall status: Invalid. Use -p '/overflowing_sub/' to rerun this test only. - cmp: FAIL (0.13s) + cmp: FAIL (2.14s) files test/symb_eval/bitvector/cmp.good and test/symb_eval/bitvector/cmp.out differ; test/symb_eval/bitvector/cmp.out contains: - test cmp/de4d4043::crux_test[0]: FAILED + test cmp/38a002d0::crux_test[0]: FAILED failures: - ---- cmp/de4d4043::crux_test[0] counterexamples ---- + ---- cmp/38a002d0::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::symbolic[0]::symbolic_u64[0] + [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u64[0] [Crux] Overall status: Invalid. Use -p '/Output testing.cmp/' to rerun this test only. - symbolic: FAIL (0.04s) + symbolic: FAIL (2.03s) files test/symb_eval/bitvector/symbolic.good and test/symb_eval/bitvector/symbolic.out differ; test/symb_eval/bitvector/symbolic.out contains: - test symbolic/6a0ffa6e::crux_test[0]: FAILED + test symbolic/9263414f::crux_test[0]: FAILED failures: - ---- symbolic/6a0ffa6e::crux_test[0] counterexamples ---- + ---- symbolic/9263414f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::bitvector[0]::make_symbolic_256[0] - [Crux] panicking::panic, called from crucible/6eda2caa::bitvector[0]::make_symbolic_256[0] + [Crux] internal: error: in crucible/bfc64a60::bitvector[0]::make_symbolic_256[0] + [Crux] panicking::panic, called from crucible/bfc64a60::bitvector[0]::make_symbolic_256[0] [Crux] Overall status: Invalid. Use -p '/Output testing.symbolic/' to rerun this test only. - from_to: FAIL (0.02s) + from_to: FAIL (1.99s) files test/symb_eval/bitvector/from_to.good and test/symb_eval/bitvector/from_to.out differ; test/symb_eval/bitvector/from_to.out contains: - test from_to/dcb07b82::crux_test[0]: FAILED + test from_to/a5531674::crux_test[0]: FAILED failures: - ---- from_to/dcb07b82::crux_test[0] counterexamples ---- + ---- from_to/a5531674::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:103:29: 103:30 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:125:1: 125:53: error: in crucible/6eda2caa::bitvector[0]::{impl#21}[0]::from[0]::_inst9ea2f935eda83dab[0] - [Crux] Translation error in crucible/6eda2caa::bitvector[0]::{impl#21}[0]::from[0]::_inst9ea2f935eda83dab[0]: BUG: invalid arguments to bv_convert: [$34: BVRepr 64] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:103:29: 103:30 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:125:1: 125:53: error: in crucible/bfc64a60::bitvector[0]::{impl#21}[0]::from[0]::_inst9ea2f935eda83dab[0] + [Crux] Translation error in crucible/bfc64a60::bitvector[0]::{impl#21}[0]::from[0]::_inst9ea2f935eda83dab[0]: BUG: invalid arguments to bv_convert: [$34: BVRepr 64] [Crux] Overall status: Invalid. Use -p '/from_to/' to rerun this test only. - clone: FAIL (0.24s) + clone: FAIL (2.35s) files test/symb_eval/vec/clone.good and test/symb_eval/vec/clone.out differ; test/symb_eval/vec/clone.out contains: - test clone/dfc182b4::f[0]: FAILED + test clone/f8a5c837::f[0]: FAILED failures: - ---- clone/dfc182b4::f[0] counterexamples ---- + ---- clone/f8a5c837::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. Use -p '/Output testing.clone/' to rerun this test only. - sort_by_key: FAIL (0.97s) + sort_by_key: FAIL (3.00s) files test/symb_eval/vec/sort_by_key.good and test/symb_eval/vec/sort_by_key.out differ; test/symb_eval/vec/sort_by_key.out contains: - test sort_by_key/ea977c8a::f[0]: FAILED + test sort_by_key/a72e2459::f[0]: FAILED failures: - ---- sort_by_key/ea977c8a::f[0] counterexamples ---- + ---- sort_by_key/a72e2459::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. Use -p '/sort_by_key/' to rerun this test only. - into_iter: FAIL (0.82s) + into_iter: FAIL (2.90s) files test/symb_eval/vec/into_iter.good and test/symb_eval/vec/into_iter.out differ; test/symb_eval/vec/into_iter.out contains: - test into_iter/e3aefb85::f[0]: FAILED + test into_iter/35485644::f[0]: FAILED failures: - ---- into_iter/e3aefb85::f[0] counterexamples ---- + ---- into_iter/35485644::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. Use -p '/into_iter/' to rerun this test only. - macro: FAIL (0.11s) + macro: FAIL (2.14s) files test/symb_eval/vec/macro.good and test/symb_eval/vec/macro.out differ; test/symb_eval/vec/macro.out contains: - test macro/e7284038::f[0]: FAILED + test macro/f7b50ef5::f[0]: FAILED failures: - ---- macro/e7284038::f[0] counterexamples ---- + ---- macro/f7b50ef5::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/5a1f6960::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/5a1f6960::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. Use -p '/Output testing.macro/' to rerun this test only. - construct: FAIL (0.12s) + construct: FAIL (2.25s) files test/symb_eval/sym_bytes/construct.good and test/symb_eval/sym_bytes/construct.out differ; test/symb_eval/sym_bytes/construct.out contains: - test construct/63aa86a0::crux_test[0]: FAILED + test construct/2811c902::crux_test[0]: FAILED failures: - ---- construct/63aa86a0::crux_test[0] counterexamples ---- + ---- construct/2811c902::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::sym_bytes[0]::{impl#0}[0]::zeroed[0] - [Crux] Translation error in crucible/6eda2caa::sym_bytes[0]::{impl#0}[0]::zeroed[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 8) to AnyRepr (TyAdt crucible/6eda2caa::array[0]::Array[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::array[0]::Array[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/6eda2caa::array[0]::Array[0]::_adtaddce72e1232152c[0] crucible/6eda2caa::array[0]::Array[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in crucible/bfc64a60::sym_bytes[0]::{impl#0}[0]::zeroed[0] + [Crux] Translation error in crucible/bfc64a60::sym_bytes[0]::{impl#0}[0]::zeroed[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::array[0]::Array[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::array[0]::Array[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::array[0]::Array[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::array[0]::Array[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/construct/' to rerun this test only. - deserialize: FAIL (0.14s) + deserialize: FAIL (2.07s) files test/symb_eval/sym_bytes/deserialize.good and test/symb_eval/sym_bytes/deserialize.out differ; test/symb_eval/sym_bytes/deserialize.out contains: - test deserialize/40308d9e::crux_test[0]: FAILED + test deserialize/098471bc::crux_test[0]: FAILED failures: - ---- deserialize/40308d9e::crux_test[0] counterexamples ---- + ---- deserialize/098471bc::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/6eda2caa::array[0]::symbolic[0]::_instaddce72e1232152c[0] - [Crux] panicking::panic_fmt, called from crucible/6eda2caa::array[0]::symbolic[0]::_instaddce72e1232152c[0] + [Crux] internal: error: in crucible/bfc64a60::array[0]::symbolic[0]::_instaddce72e1232152c[0] + [Crux] panicking::panic_fmt, called from crucible/bfc64a60::array[0]::symbolic[0]::_instaddce72e1232152c[0] [Crux] Overall status: Invalid. @@ -3225,7 +2901,11 @@ FAIL (0.22s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (0.08s) +thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_dual/4ed2e432::crux_test1[0]/report_data.js" and "test/coverage/out/coverage_dual/30f9fe77::crux_test1[0]/report_data.js" do not match', src/main.rs:825:13 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAIL (2.07s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_dual/30f9fe77::crux_test1[0]/report_data.js" "test/coverage/out/coverage_dual/4ed2e432::crux_test1[0]/report_data.js" "test/coverage/out/coverage_dual/30f9fe77::crux_test2[0]/report_data.js" "test/coverage/out/coverage_dual/4ed2e432::crux_test2[0]/report_data.js" (exit 101): failed + Use -p '/coverage_dual/' to rerun this test only. coverage: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | @@ -3240,21 +2920,10 @@ FAIL (0.22s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (0.14s) - files test/coverage/coverage.good and test/coverage/coverage.out differ; test/coverage/coverage.out contains: - warning: branch condition never has value true - ┌─ lib/core/src/fmt/mod.rs:400:12 - │ - 400 │ if pieces.len() < args.len() || pieces.len() > args.len() + 1 { - │ ^^^^^^^^^^^^^^^^^^^^^^^^^ - - warning: branch condition never has value true - ┌─ lib/core/src/fmt/mod.rs:400:12 - │ - 400 │ if pieces.len() < args.len() || pieces.len() > args.len() + 1 { - │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - +thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage/cb3943dc::crux_test[0]/report_data.js" and "test/coverage/out/coverage/0fe46bcc::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAIL (2.17s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage/0fe46bcc::crux_test[0]/report_data.js" "test/coverage/out/coverage/cb3943dc::crux_test[0]/report_data.js" (exit 101): failed Use -p '$0=="crux-mir.crux coverage.Output testing.coverage"' to rerun this test only. coverage_try: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 @@ -3270,21 +2939,10 @@ FAIL (0.22s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (0.20s) - files test/coverage/coverage_try.good and test/coverage/coverage_try.out differ; test/coverage/coverage_try.out contains: - warning: branch condition never has value true - ┌─ lib/core/src/fmt/mod.rs:400:12 - │ - 400 │ if pieces.len() < args.len() || pieces.len() > args.len() + 1 { - │ ^^^^^^^^^^^^^^^^^^^^^^^^^ - - warning: branch condition never has value true - ┌─ lib/core/src/fmt/mod.rs:400:12 - │ - 400 │ if pieces.len() < args.len() || pieces.len() > args.len() + 1 { - │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - +thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_try/a9dc15f2::crux_test[0]/report_data.js" and "test/coverage/out/coverage_try/d0e6f11f::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAIL (2.18s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_try/d0e6f11f::crux_test[0]/report_data.js" "test/coverage/out/coverage_try/a9dc15f2::crux_test[0]/report_data.js" (exit 101): failed Use -p '/coverage_try/' to rerun this test only. coverage_cond: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 @@ -3300,7 +2958,11 @@ FAIL (0.22s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (0.08s) +thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_cond/3f380757::crux_test[0]/report_data.js" and "test/coverage/out/coverage_cond/3dfc8004::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAIL (2.07s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_cond/3dfc8004::crux_test[0]/report_data.js" "test/coverage/out/coverage_cond/3f380757::crux_test[0]/report_data.js" (exit 101): failed + Use -p '/coverage_cond/' to rerun this test only. coverage_loop: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | @@ -3315,15 +2977,10 @@ FAIL (0.22s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (0.10s) - files test/coverage/coverage_loop.good and test/coverage/coverage_loop.out differ; test/coverage/coverage_loop.out contains: - warning: branch condition never has value true - ┌─ test/coverage/coverage_loop.rs:12:19 - │ - 12 │ } else if i > 99 { - │ ^^^^^^ - - +thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_loop/81f6cac8::crux_test[0]/report_data.js" and "test/coverage/out/coverage_loop/bf94e0f1::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAIL (2.09s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_loop/bf94e0f1::crux_test[0]/report_data.js" "test/coverage/out/coverage_loop/81f6cac8::crux_test[0]/report_data.js" (exit 101): failed Use -p '/coverage_loop/' to rerun this test only. coverage_shortcircuit: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 @@ -3339,21 +2996,10 @@ FAIL (0.22s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (0.16s) - files test/coverage/coverage_shortcircuit.good and test/coverage/coverage_shortcircuit.out differ; test/coverage/coverage_shortcircuit.out contains: - warning: branch condition never has value true - ┌─ lib/core/src/fmt/mod.rs:400:12 - │ - 400 │ if pieces.len() < args.len() || pieces.len() > args.len() + 1 { - │ ^^^^^^^^^^^^^^^^^^^^^^^^^ - - warning: branch condition never has value true - ┌─ lib/core/src/fmt/mod.rs:400:12 - │ - 400 │ if pieces.len() < args.len() || pieces.len() > args.len() + 1 { - │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - +thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_shortcircuit/f90f762e::crux_test[0]/report_data.js" and "test/coverage/out/coverage_shortcircuit/5762c1e1::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAIL (2.21s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_shortcircuit/5762c1e1::crux_test[0]/report_data.js" "test/coverage/out/coverage_shortcircuit/f90f762e::crux_test[0]/report_data.js" (exit 101): failed Use -p '/coverage_shortcircuit/' to rerun this test only. coverage_mono: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 @@ -3369,7 +3015,11 @@ FAIL (0.22s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (0.07s) +thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_mono/94654c01::crux_test[0]/report_data.js" and "test/coverage/out/coverage_mono/564877e0::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAIL (2.05s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_mono/564877e0::crux_test[0]/report_data.js" "test/coverage/out/coverage_mono/94654c01::crux_test[0]/report_data.js" (exit 101): failed + Use -p '/coverage_mono/' to rerun this test only. coverage_match: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | @@ -3384,21 +3034,10 @@ FAIL (0.22s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (0.18s) - files test/coverage/coverage_match.good and test/coverage/coverage_match.out differ; test/coverage/coverage_match.out contains: - warning: branch condition never has value true - ┌─ lib/core/src/fmt/mod.rs:400:12 - │ - 400 │ if pieces.len() < args.len() || pieces.len() > args.len() + 1 { - │ ^^^^^^^^^^^^^^^^^^^^^^^^^ - - warning: branch condition never has value true - ┌─ lib/core/src/fmt/mod.rs:400:12 - │ - 400 │ if pieces.len() < args.len() || pieces.len() > args.len() + 1 { - │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - +thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_match/2b40efd1::crux_test[0]/report_data.js" and "test/coverage/out/coverage_match/e0a88aee::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAIL (2.11s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_match/e0a88aee::crux_test[0]/report_data.js" "test/coverage/out/coverage_match/2b40efd1::crux_test[0]/report_data.js" (exit 101): failed Use -p '/coverage_match/' to rerun this test only. coverage_macro: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 @@ -3414,21 +3053,10 @@ FAIL (0.22s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (0.17s) - files test/coverage/coverage_macro.good and test/coverage/coverage_macro.out differ; test/coverage/coverage_macro.out contains: - warning: branch condition never has value true - ┌─ lib/core/src/fmt/mod.rs:400:12 - │ - 400 │ if pieces.len() < args.len() || pieces.len() > args.len() + 1 { - │ ^^^^^^^^^^^^^^^^^^^^^^^^^ - - warning: branch condition never has value true - ┌─ lib/core/src/fmt/mod.rs:400:12 - │ - 400 │ if pieces.len() < args.len() || pieces.len() > args.len() + 1 { - │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - +thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_macro/ff936904::crux_test[0]/report_data.js" and "test/coverage/out/coverage_macro/e538cb6e::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAIL (2.10s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_macro/e538cb6e::crux_test[0]/report_data.js" "test/coverage/out/coverage_macro/ff936904::crux_test[0]/report_data.js" (exit 101): failed Use -p '/coverage_macro/' to rerun this test only. -171 out of 339 tests failed (93.19s) +151 out of 339 tests failed (738.26s) From 2995603942ecfae60a29a04a488ff3e63dddf226 Mon Sep 17 00:00:00 2001 From: James LaMar Date: Wed, 10 May 2023 14:06:12 -0700 Subject: [PATCH 033/114] update `mir-json` submodule --- dependencies/mir-json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies/mir-json b/dependencies/mir-json index a4d0ebefb..4a1ff3012 160000 --- a/dependencies/mir-json +++ b/dependencies/mir-json @@ -1 +1 @@ -Subproject commit a4d0ebefb5665857a916f74d86a681e8b4895dac +Subproject commit 4a1ff30120b5d8e10c104211e9e8233c6357f926 From 0f293cf23f1a1d84cfd3b338ce7a1fc794011486 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 11 May 2023 14:13:49 -0400 Subject: [PATCH 034/114] Add missing pretty-printer case for ThreadLocalRef --- crucible-mir/src/Mir/PP.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/crucible-mir/src/Mir/PP.hs b/crucible-mir/src/Mir/PP.hs index d331b7d84..aa6222e2c 100644 --- a/crucible-mir/src/Mir/PP.hs +++ b/crucible-mir/src/Mir/PP.hs @@ -207,6 +207,7 @@ instance Pretty Rvalue where pretty (RAdtAg a) = pretty a pretty (ShallowInitBox ptr ty) = pretty_fn2 "ShallowInitBox" ptr ty pretty (CopyForDeref lv) = pretty_fn1 "CopyForDeref" lv + pretty (ThreadLocalRef a) = pretty_fn1 "ThreadLocalRef" a instance Pretty AdtAg where pretty (AdtAg (Adt nm _kind _vs _ _ _ _) i ops _) = pretty_fn3 "AdtAg" nm i ops From 621ffc6fca9e5a272ae95fd8a3252c3eed9fe01c Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Tue, 16 May 2023 16:26:31 -0400 Subject: [PATCH 035/114] Translate slice constants --- crucible-mir/src/Mir/JSON.hs | 3 +++ crucible-mir/src/Mir/Mir.hs | 1 + crucible-mir/src/Mir/PP.hs | 1 + crucible-mir/src/Mir/Trans.hs | 19 +++++++++++++++++-- dependencies/mir-json | 2 +- 5 files changed, 23 insertions(+), 3 deletions(-) diff --git a/crucible-mir/src/Mir/JSON.hs b/crucible-mir/src/Mir/JSON.hs index 9e590f598..971d0806a 100644 --- a/crucible-mir/src/Mir/JSON.hs +++ b/crucible-mir/src/Mir/JSON.hs @@ -488,6 +488,9 @@ instance FromJSON ConstVal where _ -> fail $ "bad size " ++ show size ++ " for float literal" pure $ ConstFloat $ FloatLit fk (T.unpack val) + Just (String "slice") -> do + ConstSlice <$> v .: "elements" + Just (String "str") -> do val <- v .: "val" let f sci = case Scientific.toBoundedInteger sci of diff --git a/crucible-mir/src/Mir/Mir.hs b/crucible-mir/src/Mir/Mir.hs index 40228c91f..9e11fa432 100644 --- a/crucible-mir/src/Mir/Mir.hs +++ b/crucible-mir/src/Mir/Mir.hs @@ -495,6 +495,7 @@ data FloatLit data ConstVal = ConstFloat FloatLit | ConstInt IntLit + | ConstSlice [ConstVal] | ConstStr B.ByteString | ConstByteStr B.ByteString | ConstBool Bool diff --git a/crucible-mir/src/Mir/PP.hs b/crucible-mir/src/Mir/PP.hs index aa6222e2c..5c8a2c86f 100644 --- a/crucible-mir/src/Mir/PP.hs +++ b/crucible-mir/src/Mir/PP.hs @@ -327,6 +327,7 @@ instance Pretty ConstVal where pretty (ConstRawPtr a) = pretty a pretty (ConstStruct fs) = pretty "struct" <> list (map pretty fs) pretty (ConstEnum v fs) = pretty "enum" <> list ((pretty "variant" <+> pretty v) : map pretty fs) + pretty (ConstSlice cs) = list (map pretty cs) instance Pretty AggregateKind where pretty (AKArray t) = brackets (pretty t) diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index 7a289f21e..4c3b96dad 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -149,9 +149,24 @@ transConstVal _ty (Some (UsizeRepr)) (M.ConstInt i) = return $ MirExp UsizeRepr (S.app $ usizeLit n) transConstVal _ty (Some (IsizeRepr)) (ConstInt i) = return $ MirExp IsizeRepr (S.app $ isizeLit (fromIntegerLit i)) -transConstVal _ty (Some (MirSliceRepr (C.BVRepr w))) (M.ConstStr bs) +transConstVal (M.TyRef (M.TySlice ty) _) (Some (MirSliceRepr tpr)) (M.ConstSlice cs) = do + cs' <- Trav.for cs $ \c -> do + MirExp tpr' c' <- transConstVal ty (Some tpr) c + Refl <- testEqualityOrFail tpr tpr' $ + "transConstVal (ConstSlice): returned wrong type: expected " ++ + show tpr ++ ", got " ++ show tpr' + pure c' + vec <- mirVector_fromVector tpr $ R.App $ E.VectorLit tpr $ V.fromList cs' + -- return $ MirExp (MirVectorRepr tpr) vec + vecRef <- constMirRef (MirVectorRepr tpr) vec + ref <- subindexRef tpr vecRef (R.App $ usizeLit 0) + let len = R.App $ usizeLit $ fromIntegral $ length cs + let struct = S.mkStruct + (mirSliceCtxRepr tpr) + (Ctx.Empty Ctx.:> ref Ctx.:> len) + return $ MirExp (MirSliceRepr tpr) struct +transConstVal _ty (Some (MirSliceRepr u8Repr@(C.BVRepr w))) (M.ConstStr bs) | Just Refl <- testEquality w (knownNat @8) = do - let u8Repr = C.BVRepr $ knownNat @8 let bytes = map (\b -> R.App (eBVLit (knownNat @8) (toInteger b))) (BS.unpack bs) let vec = R.App $ E.VectorLit u8Repr (V.fromList bytes) mirVec <- mirVector_fromVector u8Repr vec diff --git a/dependencies/mir-json b/dependencies/mir-json index 4a1ff3012..819fa0392 160000 --- a/dependencies/mir-json +++ b/dependencies/mir-json @@ -1 +1 @@ -Subproject commit 4a1ff30120b5d8e10c104211e9e8233c6357f926 +Subproject commit 819fa0392d36bf49bed7eff52acd3d9bc4c5b515 From 0f52c5d3aa52c5ac4528f6166993efe602525e59 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Tue, 16 May 2023 16:47:49 -0400 Subject: [PATCH 036/114] Reapply implement str::as_bytes via crucible_identity_transmute Originally from commit 63f8524ae96ff6ee5e1b2ec66c40fa24f825eb40 --- crux-mir/lib/Patches.md | 4 ++++ crux-mir/lib/core/src/mem/mod.rs | 14 ++++++++++++++ crux-mir/lib/core/src/str/mod.rs | 9 ++------- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index c004af389..24e384e42 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -51,3 +51,7 @@ identify all of the code that was changed in each patch. a simple workaround: include an explicit `len` field, which is updated in sync with `ptr` and `end`, and avoids the need for pointer arithmetic. + +* Implement `str::as_bytes` via `crucible_identity_transmute` (last applied: May 16, 2023) + + This is necessary to avoid a gnarly use of `transmute`. diff --git a/crux-mir/lib/core/src/mem/mod.rs b/crux-mir/lib/core/src/mem/mod.rs index 5e01ccc07..5a31f12d5 100644 --- a/crux-mir/lib/core/src/mem/mod.rs +++ b/crux-mir/lib/core/src/mem/mod.rs @@ -1273,3 +1273,17 @@ pub trait SizedTypeProperties: Sized { #[doc(hidden)] #[unstable(feature = "sized_type_properties", issue = "none")] impl SizedTypeProperties for T {} + +/// Convert `T` to `U`. Both types must have the same Crucible representation. +#[unstable(feature = "crucible_intrinsics", issue = "none")] +#[rustc_const_stable(feature = "stable_crucible_intrinsics", since = "1.0.0")] +#[inline(never)] +#[allow(unused_attributes)] +#[allow_internal_unstable(const_fn_union)] +pub const unsafe fn crucible_identity_transmute(x: T) -> U { + union Transmute { + x: ManuallyDrop, + y: ManuallyDrop, + } + unsafe { ManuallyDrop::into_inner(Transmute { x: ManuallyDrop::new(x) }.y) } +} diff --git a/crux-mir/lib/core/src/str/mod.rs b/crux-mir/lib/core/src/str/mod.rs index ab2f8520e..4f2e57aae 100644 --- a/crux-mir/lib/core/src/str/mod.rs +++ b/crux-mir/lib/core/src/str/mod.rs @@ -323,8 +323,7 @@ impl str { #[inline(always)] #[allow(unused_attributes)] pub const fn as_bytes(&self) -> &[u8] { - // SAFETY: const sound because we transmute two types with the same layout - unsafe { mem::transmute(self) } + unsafe { mem::crucible_identity_transmute(self) } } /// Converts a mutable string slice to a mutable byte slice. @@ -367,11 +366,7 @@ impl str { #[must_use] #[inline(always)] pub unsafe fn as_bytes_mut(&mut self) -> &mut [u8] { - // SAFETY: the cast from `&str` to `&[u8]` is safe since `str` - // has the same layout as `&[u8]` (only std can make this guarantee). - // The pointer dereference is safe since it comes from a mutable reference which - // is guaranteed to be valid for writes. - unsafe { &mut *(self as *mut str as *mut [u8]) } + unsafe { mem::crucible_identity_transmute(self) } } /// Converts a string slice to a raw pointer. From 54f7db6b0ddd0739551a09e20583657b8c5aa368 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 17 May 2023 11:28:52 -0400 Subject: [PATCH 037/114] Compute lengths of &str constants properly --- crux-mir/test_output.log | 2607 +++++++++++++++++++------------------- 1 file changed, 1297 insertions(+), 1310 deletions(-) diff --git a/crux-mir/test_output.log b/crux-mir/test_output.log index 3fd7819b8..3802ed705 100644 --- a/crux-mir/test_output.log +++ b/crux-mir/test_output.log @@ -2,85 +2,85 @@ crux-mir crux concrete refs - promoted_imm: OK (2.17s) - Compiling and running oracle program (0.16s) - Oracle output: 0 (2.01s) + promoted_imm: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.74s) Crux output: 0 - mut_raw: OK (2.14s) - Compiling and running oracle program (0.18s) - Oracle output: 123 (1.96s) + mut_raw: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 123 (1.72s) Crux output: 123 - mut_nested: OK (2.20s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.03s) + mut_nested: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.72s) Crux output: () - fn_ptr_mut: OK (2.23s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (2.07s) + fn_ptr_mut: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.72s) Crux output: 1 - never: OK (2.17s) - Compiling and running oracle program (0.19s) - Oracle output: 1 (1.98s) + never: OK (1.87s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.72s) Crux output: 1 - mut_ref: OK (2.14s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (1.97s) + mut_ref: OK (1.86s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.71s) Crux output: 1 - temp: OK (2.16s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (2.00s) + temp: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.72s) Crux output: 1 - mut_arg: OK (2.14s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (1.96s) + mut_arg: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.74s) Crux output: 1 - fn_ptr: OK (2.14s) + fn_ptr: OK (1.99s) Compiling and running oracle program (0.16s) - Oracle output: 1 (1.98s) + Oracle output: 1 (1.83s) Crux output: 1 - promoted_mut: OK (2.15s) - Compiling and running oracle program (0.17s) - Oracle output: 0 (1.98s) + promoted_mut: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.74s) Crux output: 0 - never_mut: OK (2.12s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (1.96s) + never_mut: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.71s) Crux output: 1 - mut_tuple_field: OK (2.19s) - Compiling and running oracle program (0.20s) - Oracle output: () (1.99s) + mut_tuple_field: OK (1.86s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.72s) Crux output: () - imm_raw: OK (2.20s) - Compiling and running oracle program (0.18s) - Oracle output: 123 (2.02s) + imm_raw: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 123 (1.71s) Crux output: 123 - static_mut: OK (2.16s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (2.00s) + static_mut: OK (1.87s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.72s) Crux output: 2 - imm_ref: OK (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: 123 (2.00s) + imm_ref: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 123 (1.72s) Crux output: 123 - imm_arg: OK (2.15s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (1.98s) + imm_arg: OK (1.86s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.71s) Crux output: 1 hash_map - insert_multi: OK (2.62s) - Compiling and running oracle program (0.20s) - Oracle output: 100 (2.43s) + insert_multi: OK (2.20s) + Compiling and running oracle program (0.18s) + Oracle output: 100 (2.02s) Crux output: 100 - insert_remove: FAIL (3.32s) - Compiling and running oracle program (0.23s) - Oracle output: 12 (3.10s) + insert_remove: FAIL (2.63s) + Compiling and running oracle program (0.18s) + Oracle output: 12 (2.45s) Crux output: failures: - ---- insert_remove/faf037d0::crux_test[0] counterexamples ---- + ---- insert_remove/00c7ee06::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in alloc/8ccd0c40::vec[0]::{impl#1}[0]::set_len[0]::_inst1b10a593fa289fda[0] - [Crux] panicking::panic, called from alloc/8ccd0c40::vec[0]::{impl#1}[0]::set_len[0]::_inst1b10a593fa289fda[0] + [Crux] internal: error: in alloc/fed0e039::vec[0]::{impl#1}[0]::set_len[0]::_inst1b10a593fa289fda[0] + [Crux] panicking::panic, called from alloc/fed0e039::vec[0]::{impl#1}[0]::set_len[0]::_inst1b10a593fa289fda[0] [Crux] Overall status: Invalid. @@ -88,207 +88,207 @@ crux-mir crux doesn't match oracle Use -p '/insert_remove/' to rerun this test only. - insert_iter: OK (2.56s) - Compiling and running oracle program (0.21s) - Oracle output: [11, 12] (2.35s) + insert_iter: OK (2.22s) + Compiling and running oracle program (0.18s) + Oracle output: [11, 12] (2.04s) Crux output: [11, 12] - insert_get: OK (2.55s) - Compiling and running oracle program (0.21s) - Oracle output: (11, 12) (2.34s) + insert_get: OK (2.20s) + Compiling and running oracle program (0.18s) + Oracle output: (11, 12) (2.02s) Crux output: (11, 12) traits - params: OK (2.15s) - Compiling and running oracle program (0.16s) - Oracle output: 25 (1.98s) + params: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 25 (1.72s) Crux output: 25 - generic3: OK (2.14s) - Compiling and running oracle program (0.17s) - Oracle output: () (1.97s) + generic3: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.72s) Crux output: () - static_three: OK (2.15s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (1.98s) + static_three: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.72s) Crux output: 2 - generic1: OK (2.20s) - Compiling and running oracle program (0.19s) - Oracle output: () (2.01s) + generic1: OK (1.86s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.72s) Crux output: () - subtrait: OK (2.16s) - Compiling and running oracle program (0.17s) - Oracle output: 73 (1.99s) + subtrait: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 73 (1.71s) Crux output: 73 - gen_trait: OK (2.14s) - Compiling and running oracle program (0.17s) - Oracle output: -69 (1.97s) + gen_trait: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: -69 (1.71s) Crux output: -69 - dynamic_branch: FAIL (0.20s) - Compiling and running oracle program (0.16s) + dynamic_branch: FAIL (0.17s) + Compiling and running oracle program (0.14s) Oracle output: 1 (0.03s) user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_branch.rs) Use -p '/dynamic_branch/' to rerun this test only. - static_two: OK (2.15s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (1.98s) + static_two: OK (1.85s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.71s) Crux output: 1 - tyfam5: OK (2.18s) - Compiling and running oracle program (0.16s) - Oracle output: () (2.01s) + tyfam5: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.70s) Crux output: () - generic2: OK (2.20s) - Compiling and running oracle program (0.19s) - Oracle output: () (2.01s) + generic2: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.73s) Crux output: () - assoc3: OK (2.16s) - Compiling and running oracle program (0.19s) - Oracle output: () (1.96s) + assoc3: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.72s) Crux output: () - tyfam4: OK (2.27s) - Compiling and running oracle program (0.16s) - Oracle output: 0 (2.11s) + tyfam4: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.72s) Crux output: 0 - static_self: OK (2.15s) - Compiling and running oracle program (0.16s) - Oracle output: 42 (1.99s) + static_self: OK (1.84s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (1.70s) Crux output: 42 - conv: OK (2.20s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (2.03s) + conv: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.71s) Crux output: 1 - dynamic_poly: FAIL (0.21s) - Compiling and running oracle program (0.17s) + dynamic_poly: FAIL (0.17s) + Compiling and running oracle program (0.14s) Oracle output: 3 (0.03s) user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_poly.rs) Use -p '/dynamic_poly/' to rerun this test only. - dynamic_simple: OK (2.29s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (2.11s) + dynamic_simple: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.75s) Crux output: 1 - gen_trait_poly: OK (2.19s) - Compiling and running oracle program (0.20s) - Oracle output: 12 (2.00s) + gen_trait_poly: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 12 (1.72s) Crux output: 12 - default: OK (2.18s) - Compiling and running oracle program (0.16s) - Oracle output: 12 (2.01s) + default: OK (1.84s) + Compiling and running oracle program (0.14s) + Oracle output: 12 (1.70s) Crux output: 12 - assoc1: OK (2.22s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.04s) + assoc1: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.76s) Crux output: () - subtrait2: OK (2.28s) - Compiling and running oracle program (0.20s) - Oracle output: 64 (2.08s) + subtrait2: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: 64 (1.76s) Crux output: 64 - bounds1: OK (2.22s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.05s) + bounds1: OK (1.92s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.78s) Crux output: () - bounds3: OK (2.30s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.12s) + bounds3: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.73s) Crux output: () - static_eq: OK (2.22s) - Compiling and running oracle program (0.18s) - Oracle output: true (2.05s) + static_eq: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: true (1.71s) Crux output: true - tyfam3: OK (2.32s) - Compiling and running oracle program (0.18s) - Oracle output: 23 (2.14s) + tyfam3: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 23 (1.71s) Crux output: 23 - dict_med: OK (2.13s) - Compiling and running oracle program (0.16s) - Oracle output: 42 (1.97s) + dict_med: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (1.71s) Crux output: 42 - basics1: OK (2.28s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.11s) + basics1: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.75s) Crux output: () - assoc2: OK (2.20s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.03s) + assoc2: OK (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.76s) Crux output: () - dict_polymem: OK (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: 4 (2.00s) + dict_polymem: OK (1.95s) + Compiling and running oracle program (0.14s) + Oracle output: 4 (1.81s) Crux output: 4 - bounds2: OK (2.20s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.03s) + bounds2: OK (1.99s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.84s) Crux output: () - intoiter: OK (2.16s) - Compiling and running oracle program (0.17s) - Oracle output: 0 (1.99s) + intoiter: OK (1.99s) + Compiling and running oracle program (0.16s) + Oracle output: 0 (1.83s) Crux output: 0 - static: OK (2.17s) - Compiling and running oracle program (0.18s) - Oracle output: 42 (1.99s) - Crux output: 42 - dynamic_two: FAIL (0.20s) + static: OK (1.91s) Compiling and running oracle program (0.16s) - Oracle output: 1 (0.04s) + Oracle output: 42 (1.75s) + Crux output: 42 + dynamic_two: FAIL (0.17s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (0.03s) user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_two.rs) Use -p '/dynamic_two/' to rerun this test only. - bounds4: OK (2.32s) - Compiling and running oracle program (0.16s) - Oracle output: () (2.15s) + bounds4: OK (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.76s) Crux output: () - dict_poly: OK (2.18s) - Compiling and running oracle program (0.19s) - Oracle output: 1 (1.99s) + dict_poly: OK (2.01s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (1.85s) Crux output: 1 - dict_simple: OK (2.18s) - Compiling and running oracle program (0.19s) - Oracle output: 32 (1.99s) + dict_simple: OK (1.98s) + Compiling and running oracle program (0.15s) + Oracle output: 32 (1.83s) Crux output: 32 - tyfam: OK (2.19s) - Compiling and running oracle program (0.18s) - Oracle output: 23 (2.02s) + tyfam: OK (1.96s) + Compiling and running oracle program (0.16s) + Oracle output: 23 (1.80s) Crux output: 23 - bounds5: OK (2.21s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.03s) + bounds5: OK (1.93s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.78s) Crux output: () - tyfam2: OK (2.19s) - Compiling and running oracle program (0.17s) - Oracle output: 23 (2.02s) + tyfam2: OK (1.92s) + Compiling and running oracle program (0.15s) + Oracle output: 23 (1.77s) Crux output: 23 - dynamic_med: FAIL (0.20s) - Compiling and running oracle program (0.17s) + dynamic_med: FAIL (0.17s) + Compiling and running oracle program (0.14s) Oracle output: 1 (0.03s) user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_med.rs) Use -p '/dynamic_med/' to rerun this test only. intTest - test0039: OK (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: 7 (2.00s) + test0039: OK (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: 7 (1.76s) Crux output: 7 - test0038: OK (2.36s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.19s) + test0038: OK (1.99s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.84s) Crux output: () tuple - clone_rec: OK (2.17s) - Compiling and running oracle program (0.16s) - Oracle output: () (2.01s) + clone_rec: OK (1.92s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.77s) Crux output: () - clone: OK (2.19s) - Compiling and running oracle program (0.19s) - Oracle output: () (1.99s) + clone: OK (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.76s) Crux output: () - clone_from: FAIL (2.19s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.02s) + clone_from: FAIL (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.76s) Crux output: failures: - ---- clone_from/e1b836b1::f[0] counterexamples ---- + ---- clone_from/ca9e7ae4::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/clone.rs:136:17: 136:31: error: in core/1c79c753::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0] - [Crux] Translation error in core/1c79c753::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0]: don't know how to generate CloneShim for unknown method core/1c79c753::clone[0]::Clone[0]::clone[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/clone.rs:136:17: 136:31: error: in core/95754f0e::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0] + [Crux] Translation error in core/95754f0e::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0]: don't know how to generate CloneShim for unknown method core/95754f0e::clone[0]::Clone[0]::clone[0] [Crux] Overall status: Invalid. @@ -296,16 +296,16 @@ crux-mir crux doesn't match oracle Use -p '/clone_from/' to rerun this test only. - clone_struct: FAIL (2.23s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.05s) + clone_struct: FAIL (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.76s) Crux output: failures: - ---- clone_struct/96042f83::f[0] counterexamples ---- + ---- clone_struct/b6c12504::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/tuple/clone_struct.rs:8:13: 8:22: error: in clone_struct/96042f83::f[0] - [Crux] Translation error in clone_struct/96042f83::f[0]: don't know how to generate CloneShim for unknown method core/1c79c753::clone[0]::Clone[0]::clone[0] + [Crux] test/conc_eval/tuple/clone_struct.rs:8:13: 8:22: error: in clone_struct/b6c12504::f[0] + [Crux] Translation error in clone_struct/b6c12504::f[0]: don't know how to generate CloneShim for unknown method core/95754f0e::clone[0]::Clone[0]::clone[0] [Crux] Overall status: Invalid. @@ -314,49 +314,49 @@ crux-mir Use -p '/clone_struct/' to rerun this test only. iter - for: OK (2.20s) - Compiling and running oracle program (0.17s) - Oracle output: 47 (2.03s) + for: OK (1.90s) + Compiling and running oracle program (0.15s) + Oracle output: 47 (1.75s) Crux output: 47 - sum: OK (2.28s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.09s) + sum: OK (1.93s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.78s) Crux output: () - filter_chain: OK (2.36s) - Compiling and running oracle program (0.19s) - Oracle output: 0 (2.16s) - Crux output: 0 - from_fn: OK (2.16s) + filter_chain: OK (2.05s) Compiling and running oracle program (0.17s) - Oracle output: () (1.99s) + Oracle output: 0 (1.88s) + Crux output: 0 + from_fn: OK (1.96s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.81s) Crux output: () - peek: OK (2.29s) - Compiling and running oracle program (0.18s) - Oracle output: 3 (2.11s) + peek: OK (1.99s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (1.84s) Crux output: 3 - zip: OK (2.39s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.22s) + zip: OK (1.98s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.82s) Crux output: () - loop: OK (2.11s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (1.95s) + loop: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.74s) Crux output: 2 - cloned: OK (2.25s) - Compiling and running oracle program (0.17s) - Oracle output: 6 (2.08s) + cloned: OK (1.95s) + Compiling and running oracle program (0.15s) + Oracle output: 6 (1.80s) Crux output: 6 str - format_hex: FAIL (3.17s) - Compiling and running oracle program (0.17s) - Oracle output: true (3.00s) + format_hex: FAIL (2.62s) + Compiling and running oracle program (0.15s) + Oracle output: true (2.46s) Crux output: failures: - ---- format_hex/f0662c51::crux_test[0] counterexamples ---- + ---- format_hex/ab76c909::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0] - [Crux] Translation error in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] + [Crux] lib/core/src/num/uint_macros.rs:1677:74: 1677:89 !lib/core/src/num/mod.rs:965:5: 970:27: error: in core/95754f0e::num[0]::{impl#12}[0]::overflowing_mul[0] + [Crux] Translation error in core/95754f0e::num[0]::{impl#12}[0]::overflowing_mul[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::mul_with_overflow[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. @@ -364,16 +364,16 @@ crux-mir crux doesn't match oracle Use -p '/format_hex/' to rerun this test only. - format_int: FAIL (2.97s) - Compiling and running oracle program (0.17s) - Oracle output: true (2.79s) + format_int: FAIL (2.61s) + Compiling and running oracle program (0.15s) + Oracle output: true (2.46s) Crux output: failures: - ---- format_int/b92db7e2::crux_test[0] counterexamples ---- + ---- format_int/f1c217ad::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0] - [Crux] Translation error in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] + [Crux] lib/core/src/num/uint_macros.rs:1677:74: 1677:89 !lib/core/src/num/mod.rs:965:5: 970:27: error: in core/95754f0e::num[0]::{impl#12}[0]::overflowing_mul[0] + [Crux] Translation error in core/95754f0e::num[0]::{impl#12}[0]::overflowing_mul[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::mul_with_overflow[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. @@ -381,16 +381,16 @@ crux-mir crux doesn't match oracle Use -p '/format_int/' to rerun this test only. - format_struct: FAIL (3.18s) - Compiling and running oracle program (0.21s) - Oracle output: true (2.97s) + format_struct: FAIL (2.74s) + Compiling and running oracle program (0.16s) + Oracle output: true (2.58s) Crux output: failures: - ---- format_struct/62367108::crux_test[0] counterexamples ---- + ---- format_struct/f8c3e1dc::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0] - [Crux] Translation error in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/95754f0e::fmt[0]::write[0] + [Crux] Translation error in core/95754f0e::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/95754f0e::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. @@ -398,16 +398,16 @@ crux-mir crux doesn't match oracle Use -p '/format_struct/' to rerun this test only. - format: FAIL (2.98s) + format: FAIL (2.85s) Compiling and running oracle program (0.18s) - Oracle output: true (2.80s) + Oracle output: true (2.67s) Crux output: failures: - ---- format/ab353d66::crux_test[0] counterexamples ---- + ---- format/a6953732::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0] - [Crux] Translation error in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] + [Crux] lib/core/src/num/uint_macros.rs:1677:74: 1677:89 !lib/core/src/num/mod.rs:965:5: 970:27: error: in core/95754f0e::num[0]::{impl#12}[0]::overflowing_mul[0] + [Crux] Translation error in core/95754f0e::num[0]::{impl#12}[0]::overflowing_mul[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::mul_with_overflow[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. @@ -415,16 +415,16 @@ crux-mir crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..str.format"' to rerun this test only. - to_owned: FAIL (2.42s) - Compiling and running oracle program (0.17s) - Oracle output: true (2.25s) + to_owned: FAIL (2.26s) + Compiling and running oracle program (0.16s) + Oracle output: true (2.11s) Crux output: failures: - ---- to_owned/5974b99d::crux_test[0] counterexamples ---- + ---- to_owned/9b9b876a::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0] - [Crux] Translation error in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -432,16 +432,16 @@ crux-mir crux doesn't match oracle Use -p '/to_owned/' to rerun this test only. - string_push: FAIL (2.97s) - Compiling and running oracle program (0.18s) - Oracle output: true (2.79s) + string_push: FAIL (2.66s) + Compiling and running oracle program (0.17s) + Oracle output: true (2.50s) Crux output: failures: - ---- string_push/c3ceb3d7::crux_test[0] counterexamples ---- + ---- string_push/82be0c4f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -449,16 +449,16 @@ crux-mir crux doesn't match oracle Use -p '/string_push/' to rerun this test only. - format_array: FAIL (3.41s) - Compiling and running oracle program (0.18s) - Oracle output: true (3.23s) + format_array: FAIL (2.99s) + Compiling and running oracle program (0.17s) + Oracle output: true (2.81s) Crux output: failures: - ---- format_array/cd770658::crux_test[0] counterexamples ---- + ---- format_array/73672e87::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0] - [Crux] Translation error in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/95754f0e::fmt[0]::write[0] + [Crux] Translation error in core/95754f0e::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/95754f0e::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. @@ -467,22 +467,22 @@ crux-mir Use -p '/format_array/' to rerun this test only. sync - mutex: FAIL (2.77s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (2.59s) - user error (JSON Decoding of test/conc_eval/sync/mutex.all.mir failed: Error in $.statics[12].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) + mutex: FAIL (2.65s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (2.49s) + user error (JSON Decoding of test/conc_eval/sync/mutex.all.mir failed: Error in $.statics[6].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) Use -p '$0=="crux-mir.crux concrete..sync.mutex"' to rerun this test only. - arc_cell: FAIL (2.34s) - Compiling and running oracle program (0.17s) - Oracle output: 4 (2.16s) + arc_cell: FAIL (2.14s) + Compiling and running oracle program (0.16s) + Oracle output: 4 (1.98s) Crux output: failures: - ---- arc_cell/34dc4cdc::crux_test[0] counterexamples ---- + ---- arc_cell/7c4dad53::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -490,16 +490,16 @@ crux-mir crux doesn't match oracle Use -p '/arc_cell/' to rerun this test only. - arc: FAIL (2.33s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (2.16s) + arc: FAIL (2.15s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (2.00s) Crux output: failures: - ---- arc/cc226314::crux_test[0] counterexamples ---- + ---- arc/bb69dff5::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -507,16 +507,16 @@ crux-mir crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..sync.arc"' to rerun this test only. - atomic_add: FAIL (2.24s) - Compiling and running oracle program (0.18s) - Oracle output: 6 (2.06s) + atomic_add: FAIL (2.15s) + Compiling and running oracle program (0.16s) + Oracle output: 6 (1.99s) Crux output: failures: - ---- atomic_add/4834fad5::crux_test[0] counterexamples ---- + ---- atomic_add/39afb391::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/sync/atomic.rs:3058:61: 3058:64: error: in core/1c79c753::sync[0]::atomic[0]::atomic_add[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/1c79c753::sync[0]::atomic[0]::atomic_add[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::atomic_xadd_release[0]::_inst1e2825177cd3b608[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/sync/atomic.rs:3058:61: 3058:64: error: in core/95754f0e::sync[0]::atomic[0]::atomic_add[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/95754f0e::sync[0]::atomic[0]::atomic_add[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::atomic_xadd_release[0]::_inst1e2825177cd3b608[0] [Crux] Overall status: Invalid. @@ -524,22 +524,22 @@ crux-mir crux doesn't match oracle Use -p '/atomic_add/' to rerun this test only. - rwlock: FAIL (2.99s) - Compiling and running oracle program (0.19s) - Oracle output: 2 (2.80s) - user error (JSON Decoding of test/conc_eval/sync/rwlock.all.mir failed: Error in $.statics[12].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) + rwlock: FAIL (2.70s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (2.53s) + user error (JSON Decoding of test/conc_eval/sync/rwlock.all.mir failed: Error in $.statics[6].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) Use -p '$0=="crux-mir.crux concrete..sync.rwlock"' to rerun this test only. - atomic_cxchg: FAIL (2.31s) - Compiling and running oracle program (0.19s) - Oracle output: 6 (2.12s) + atomic_cxchg: FAIL (2.17s) + Compiling and running oracle program (0.17s) + Oracle output: 6 (2.00s) Crux output: failures: - ---- atomic_cxchg/ac838f7f::crux_test[0] counterexamples ---- + ---- atomic_cxchg/d34db99d::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/sync/atomic.rs:3095:86: 3095:89: error: in core/1c79c753::sync[0]::atomic[0]::atomic_compare_exchange[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/1c79c753::sync[0]::atomic[0]::atomic_compare_exchange[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::atomic_cxchg_relaxed_relaxed[0]::_inst1e2825177cd3b608[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/sync/atomic.rs:3095:86: 3095:89: error: in core/95754f0e::sync[0]::atomic[0]::atomic_compare_exchange[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/95754f0e::sync[0]::atomic[0]::atomic_compare_exchange[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::atomic_cxchg_relaxed_relaxed[0]::_inst1e2825177cd3b608[0] [Crux] Overall status: Invalid. @@ -547,16 +547,16 @@ crux-mir crux doesn't match oracle Use -p '/atomic_cxchg/' to rerun this test only. - arc_clone: FAIL (2.36s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (2.19s) + arc_clone: FAIL (2.21s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (2.04s) Crux output: failures: - ---- arc_clone/0035f3b8::crux_test[0] counterexamples ---- + ---- arc_clone/494b2b01::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -564,85 +564,85 @@ crux-mir crux doesn't match oracle Use -p '/arc_clone/' to rerun this test only. - atomic_swap: OK (2.22s) + atomic_swap: OK (2.14s) Compiling and running oracle program (0.16s) - Oracle output: (2, 1) (2.06s) + Oracle output: (2, 1) (1.98s) Crux output: (2, 1) - rwlock_multi: FAIL (2.86s) - Compiling and running oracle program (0.19s) - Oracle output: 3 (2.67s) - user error (JSON Decoding of test/conc_eval/sync/rwlock_multi.all.mir failed: Error in $.statics[12].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) + rwlock_multi: FAIL (2.81s) + Compiling and running oracle program (0.17s) + Oracle output: 3 (2.63s) + user error (JSON Decoding of test/conc_eval/sync/rwlock_multi.all.mir failed: Error in $.statics[6].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) Use -p '/rwlock_multi/' to rerun this test only. - atomic_fence: OK (2.21s) + atomic_fence: OK (2.14s) Compiling and running oracle program (0.16s) - Oracle output: 2 (2.05s) + Oracle output: 2 (1.98s) Crux output: 2 - mutex_multi: FAIL (2.88s) - Compiling and running oracle program (0.18s) - Oracle output: 3 (2.70s) - user error (JSON Decoding of test/conc_eval/sync/mutex_multi.all.mir failed: Error in $.statics[12].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) + mutex_multi: FAIL (2.81s) + Compiling and running oracle program (0.16s) + Oracle output: 3 (2.65s) + user error (JSON Decoding of test/conc_eval/sync/mutex_multi.all.mir failed: Error in $.statics[6].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) Use -p '/mutex_multi/' to rerun this test only. consts - struct_val: OK (2.18s) - Compiling and running oracle program (0.17s) - Oracle output: Foo { x: true } (2.02s) + struct_val: OK (2.05s) + Compiling and running oracle program (0.16s) + Oracle output: Foo { x: true } (1.90s) Crux output: Foo { x: true } - struct_unit: OK (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: Err(Foo { x: () }) (2.00s) + struct_unit: OK (2.04s) + Compiling and running oracle program (0.16s) + Oracle output: Err(Foo { x: () }) (1.88s) Crux output: Err(Foo { x: () }) - local_key: FAIL (2.15s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (1.97s) + local_key: FAIL (2.02s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (1.86s) user error (JSON Decoding of test/conc_eval/consts/local_key.all.mir failed: Error in $.statics[0].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) Use -p '/local_key/' to rerun this test only. - fn_def: FAIL (2.17s) - Compiling and running oracle program (0.19s) - Oracle output: 1 (1.98s) + fn_def: FAIL (2.03s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (1.87s) user error (JSON Decoding of test/conc_eval/consts/fn_def.all.mir failed: Error in $.fns[0].body.blocks[0].block.terminator.func.data.rendered: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) Use -p '/fn_def/' to rerun this test only. - enum_val: OK (2.19s) - Compiling and running oracle program (0.17s) - Oracle output: None (2.02s) + enum_val: OK (2.06s) + Compiling and running oracle program (0.16s) + Oracle output: None (1.91s) Crux output: None crypto - add: OK (2.22s) + add: OK (2.15s) Compiling and running oracle program (0.17s) - Oracle output: true (2.04s) + Oracle output: true (1.98s) Crux output: true - add_noL: OK (2.22s) + add_noL: OK (2.08s) Compiling and running oracle program (0.17s) - Oracle output: false (2.05s) + Oracle output: false (1.91s) Crux output: false cell - cell: OK (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (2.00s) + cell: OK (2.11s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (1.95s) Crux output: 2 - ref_cell: OK (2.76s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (2.59s) + ref_cell: OK (2.65s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (2.50s) Crux output: 2 - ref_cell2: OK (2.89s) - Compiling and running oracle program (0.18s) - Oracle output: 2 (2.70s) + ref_cell2: OK (2.70s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (2.53s) Crux output: 2 fnptr - call: OK (2.16s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (2.00s) + call: OK (1.99s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.84s) Crux output: 2 - field: OK (2.16s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (2.00s) + field: OK (1.98s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.83s) Crux output: 2 - make: OK (2.14s) - Compiling and running oracle program (0.17s) - Oracle output: 0 (1.97s) + make: OK (1.99s) + Compiling and running oracle program (0.16s) + Oracle output: 0 (1.84s) Crux output: 0 custom: rustc compilation failed for custom error output: @@ -658,22 +658,22 @@ error: aborting due to previous error For more information about this error, try `rustc --explain E0432`. -FAIL (expected: taking address of an overridden function) (0.06s) - Compiling and running oracle program (0.06s) +FAIL (expected: taking address of an overridden function) (0.05s) + Compiling and running oracle program (0.05s) test/Test.hs:107: failed to compile and run (expected failure) num - overflow: FAIL (2.20s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.03s) + overflow: FAIL (2.03s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.87s) Crux output: failures: - ---- overflow/9fad2f92::crux_test[0] counterexamples ---- + ---- overflow/6b711274::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/uint_macros.rs:1578:74: 1578:77 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:305:5: 306:27: error: in core/1c79c753::num[0]::{impl#7}[0]::overflowing_sub[0] - [Crux] Translation error in core/1c79c753::num[0]::{impl#7}[0]::overflowing_sub[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::sub_with_overflow[0]::_instaddce72e1232152c[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/uint_macros.rs:1578:74: 1578:77 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:305:5: 306:27: error: in core/95754f0e::num[0]::{impl#7}[0]::overflowing_sub[0] + [Crux] Translation error in core/95754f0e::num[0]::{impl#7}[0]::overflowing_sub[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::sub_with_overflow[0]::_instaddce72e1232152c[0] [Crux] Overall status: Invalid. @@ -681,16 +681,16 @@ FAIL (expected: taking address of an overridden function) (0.06s) crux doesn't match oracle Use -p '/num.overflow/' to rerun this test only. - from_bytes: FAIL (2.19s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.02s) + from_bytes: FAIL (2.02s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.86s) Crux output: failures: - ---- from_bytes/2b08070a::f[0] counterexamples ---- + ---- from_bytes/848b3106::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/uint_macros.rs:2417:37: 2417:42 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:890:5: 891:49: error: in core/1c79c753::num[0]::{impl#8}[0]::from_ne_bytes[0] - [Crux] Translation error in core/1c79c753::num[0]::{impl#8}[0]::from_ne_bytes[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_inst850c595be4adc72f[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/uint_macros.rs:2417:37: 2417:42 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:890:5: 891:49: error: in core/95754f0e::num[0]::{impl#8}[0]::from_ne_bytes[0] + [Crux] Translation error in core/95754f0e::num[0]::{impl#8}[0]::from_ne_bytes[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_inst850c595be4adc72f[0] [Crux] Overall status: Invalid. @@ -698,30 +698,30 @@ FAIL (expected: taking address of an overridden function) (0.06s) crux doesn't match oracle Use -p '/from_bytes/' to rerun this test only. - saturate: OK (2.79s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.62s) + saturate: OK (2.56s) + Compiling and running oracle program (0.15s) + Oracle output: () (2.41s) Crux output: () prim - bool: OK (2.15s) - Compiling and running oracle program (0.18s) - Oracle output: false (1.97s) + bool: OK (1.96s) + Compiling and running oracle program (0.15s) + Oracle output: false (1.81s) Crux output: false - shift3: OK (2.20s) - Compiling and running oracle program (0.18s) - Oracle output: -9223372036854775808 (2.02s) + shift3: OK (1.95s) + Compiling and running oracle program (0.15s) + Oracle output: -9223372036854775808 (1.80s) Crux output: -9223372036854775808 - div: OK (2.13s) - Compiling and running oracle program (0.16s) - Oracle output: 4 (1.97s) + div: OK (1.95s) + Compiling and running oracle program (0.15s) + Oracle output: 4 (1.80s) Crux output: 4 - ge: OK (2.16s) - Compiling and running oracle program (0.16s) - Oracle output: true (1.99s) + ge: OK (1.96s) + Compiling and running oracle program (0.15s) + Oracle output: true (1.82s) Crux output: true - add1: OK (2.12s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (1.96s) + add1: OK (1.97s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.82s) Crux output: 2 shift_exceeding: rustc compilation failed for shift_exceeding error output: @@ -753,21 +753,21 @@ error: aborting due to previous error; 1 warning emitted For more information about this error, try `rustc --explain E0425`. -FAIL (expected: Should panic, but doesn't) (0.07s) - Compiling and running oracle program (0.07s) +FAIL (expected: Should panic, but doesn't) (0.06s) + Compiling and running oracle program (0.06s) test/Test.hs:107: failed to compile and run (expected failure) - char_from_u32: FAIL (2.27s) - Compiling and running oracle program (0.17s) - Oracle output: 'A' (2.10s) + char_from_u32: FAIL (1.98s) + Compiling and running oracle program (0.15s) + Oracle output: 'A' (1.83s) Crux output: failures: - ---- char_from_u32/c5a421fb::crux_test[0] counterexamples ---- + ---- char_from_u32/d563b221::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/char/convert.rs:215:31: 215:32: error: in core/1c79c753::char[0]::convert[0]::char_try_from_u32[0] - [Crux] Translation error in core/1c79c753::char[0]::convert[0]::char_try_from_u32[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instdc3291f36c45d6de[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/char/convert.rs:215:31: 215:32: error: in core/95754f0e::char[0]::convert[0]::char_try_from_u32[0] + [Crux] Translation error in core/95754f0e::char[0]::convert[0]::char_try_from_u32[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instdc3291f36c45d6de[0] [Crux] Overall status: Invalid. @@ -775,83 +775,70 @@ FAIL (expected: Should panic, but doesn't) (0.07s) crux doesn't match oracle Use -p '/char_from_u32/' to rerun this test only. - ffs: OK (2.24s) - Compiling and running oracle program (0.19s) - Oracle output: true (2.06s) + ffs: OK (2.02s) + Compiling and running oracle program (0.16s) + Oracle output: true (1.86s) Crux output: true - mut_arg: OK (2.16s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (1.99s) + mut_arg: OK (1.96s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.81s) Crux output: 1 - litstring: FAIL (2.19s) - Compiling and running oracle program (0.20s) - Oracle output: true (2.00s) - Crux output: - failures: - - ---- litstring/17414821::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/str/mod.rs:327:33: 327:37: error: in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0] - [Crux] Translation error in core/1c79c753::str[0]::{impl#0}[0]::as_bytes[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instf78f029a53f4941f[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/litstring/' to rerun this test only. - lit: OK (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: false (2.00s) - Crux output: false - litbstring: OK (2.29s) + litstring: OK (1.96s) + Compiling and running oracle program (0.15s) + Oracle output: true (1.81s) + Crux output: true + lit: OK (1.98s) Compiling and running oracle program (0.16s) - Oracle output: true (2.13s) + Oracle output: false (1.82s) + Crux output: false + litbstring: OK (1.96s) + Compiling and running oracle program (0.15s) + Oracle output: true (1.82s) Crux output: true - shift1: OK (2.13s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (1.97s) + shift1: OK (1.97s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.82s) Crux output: 2 - shift2: OK (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (2.00s) + shift2: OK (1.97s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.83s) Crux output: 2 - shift4: OK (2.16s) - Compiling and running oracle program (0.19s) - Oracle output: -9223372036854775808 (1.97s) + shift4: OK (1.98s) + Compiling and running oracle program (0.15s) + Oracle output: -9223372036854775808 (1.83s) Crux output: -9223372036854775808 - mut: OK (2.15s) - Compiling and running oracle program (0.17s) - Oracle output: 14 (1.98s) + mut: OK (1.97s) + Compiling and running oracle program (0.15s) + Oracle output: 14 (1.82s) Crux output: 14 - wrapping_sub: OK (2.14s) - Compiling and running oracle program (0.16s) - Oracle output: true (1.98s) + wrapping_sub: OK (1.96s) + Compiling and running oracle program (0.15s) + Oracle output: true (1.81s) Crux output: true impl - self: OK (2.15s) - Compiling and running oracle program (0.17s) - Oracle output: 42 (1.98s) + self: OK (1.98s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.83s) Crux output: 42 - simple: OK (2.23s) - Compiling and running oracle program (0.18s) - Oracle output: 42 (2.05s) + simple: OK (1.97s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.82s) Crux output: 42 - self_mut: OK (2.25s) - Compiling and running oracle program (0.19s) - Oracle output: 42 (2.07s) + self_mut: OK (2.00s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.85s) Crux output: 42 box - new: FAIL (2.26s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.09s) + new: FAIL (2.11s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.96s) Crux output: failures: - ---- new/b5aad3cf::f[0] counterexamples ---- + ---- new/2c6bd102::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -859,16 +846,16 @@ FAIL (expected: Should panic, but doesn't) (0.07s) crux doesn't match oracle Use -p '/box.new/' to rerun this test only. - mut_ref: FAIL (2.32s) - Compiling and running oracle program (0.20s) - Oracle output: () (2.13s) + mut_ref: FAIL (2.08s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.93s) Crux output: failures: - ---- mut_ref/ac037107::f[0] counterexamples ---- + ---- mut_ref/b4fa115d::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -876,16 +863,16 @@ FAIL (expected: Should panic, but doesn't) (0.07s) crux doesn't match oracle Use -p '/box.mut_ref/' to rerun this test only. - mut: FAIL (2.41s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.23s) + mut: FAIL (2.10s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.94s) Crux output: failures: - ---- mut/b7651654::f[0] counterexamples ---- + ---- mut/2ecd7f0f::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -893,16 +880,16 @@ FAIL (expected: Should panic, but doesn't) (0.07s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..box.mut"' to rerun this test only. - unsize: FAIL (2.27s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (2.11s) + unsize: FAIL (2.07s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (1.91s) Crux output: failures: - ---- unsize/d08bfbca::f[0] counterexamples ---- + ---- unsize/6c690c2b::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -910,16 +897,16 @@ FAIL (expected: Should panic, but doesn't) (0.07s) crux doesn't match oracle Use -p '/box.unsize/' to rerun this test only. - struct: FAIL (2.30s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (2.13s) + struct: FAIL (2.09s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.94s) Crux output: failures: - ---- struct/b1eca756::crux_test[0] counterexamples ---- + ---- struct/da6961f6::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -928,18 +915,18 @@ FAIL (expected: Should panic, but doesn't) (0.07s) Use -p '/box.struct/' to rerun this test only. ptr - is_null_slice: FAIL (expected: can't unsize null pointers) (2.20s) - Compiling and running oracle program (0.16s) - Oracle output: (false, true) (2.03s) + is_null_slice: FAIL (expected: can't unsize null pointers) (2.03s) + Compiling and running oracle program (0.15s) + Oracle output: (false, true) (1.88s) Crux output: failures: - ---- is_null_slice/b07afa7d::crux_test[0] counterexamples ---- + ---- is_null_slice/2e26bc81::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/ptr/is_null_slice.rs:7:13: 7:33: error: in is_null_slice/b07afa7d::crux_test[0] + [Crux] test/conc_eval/ptr/is_null_slice.rs:7:13: 7:33: error: in is_null_slice/2e26bc81::crux_test[0] [Crux] attempted subindex on the result of an integer-to-pointer cast [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:38:38: 38:42: error: in core/1c79c753::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst55af81a2cef2701c[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:38:38: 38:42: error: in core/95754f0e::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst55af81a2cef2701c[0] [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. @@ -947,36 +934,36 @@ FAIL (expected: Should panic, but doesn't) (0.07s) test/Test.hs:124: crux doesn't match oracle (expected failure) - struct_eq: OK (2.17s) - Compiling and running oracle program (0.16s) - Oracle output: () (2.01s) + struct_eq: OK (2.03s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.88s) Crux output: () - read_write: OK (2.23s) - Compiling and running oracle program (0.17s) - Oracle output: [1, 3, 1] (2.06s) + read_write: OK (2.06s) + Compiling and running oracle program (0.15s) + Oracle output: [1, 3, 1] (1.91s) Crux output: [1, 3, 1] - offset_mut: OK (2.23s) - Compiling and running oracle program (0.18s) - Oracle output: 3 (2.05s) + offset_mut: OK (1.97s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (1.82s) Crux output: 3 - copy: OK (2.21s) - Compiling and running oracle program (0.18s) - Oracle output: [1, 2, 3, 1, 2, 3] (2.03s) + copy: OK (2.03s) + Compiling and running oracle program (0.16s) + Oracle output: [1, 2, 3, 1, 2, 3] (1.88s) Crux output: [1, 2, 3, 1, 2, 3] - unsize_slice: OK (2.16s) - Compiling and running oracle program (0.17s) - Oracle output: (1, 2) (2.00s) + unsize_slice: OK (1.97s) + Compiling and running oracle program (0.15s) + Oracle output: (1, 2) (1.82s) Crux output: (1, 2) - dangling_eq: FAIL (2.18s) - Compiling and running oracle program (0.16s) - Oracle output: () (2.02s) + dangling_eq: FAIL (2.01s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.86s) Crux output: failures: - ---- dangling_eq/82e2c977::crux_test[0] counterexamples ---- + ---- dangling_eq/ed769482::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:135:14: 135:78: error: in core/1c79c753::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/1c79c753::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:135:14: 135:78: error: in core/95754f0e::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/95754f0e::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -984,24 +971,24 @@ FAIL (expected: Should panic, but doesn't) (0.07s) crux doesn't match oracle Use -p '/dangling_eq/' to rerun this test only. - cast_eq: OK (2.17s) - Compiling and running oracle program (0.16s) - Oracle output: () (2.00s) + cast_eq: OK (2.00s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.86s) Crux output: () - offset_from: OK (2.18s) - Compiling and running oracle program (0.17s) - Oracle output: -2 (2.01s) + offset_from: OK (2.02s) + Compiling and running oracle program (0.15s) + Oracle output: -2 (1.86s) Crux output: -2 - null_eq: FAIL (2.34s) - Compiling and running oracle program (0.20s) - Oracle output: 1 (2.15s) + null_eq: FAIL (2.14s) + Compiling and running oracle program (0.18s) + Oracle output: 1 (1.96s) Crux output: failures: - ---- null_eq/c0291176::crux_test[0] counterexamples ---- + ---- null_eq/bd64c304::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:118:14: 118:78: error: in core/1c79c753::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/1c79c753::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:118:14: 118:78: error: in core/95754f0e::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/95754f0e::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1009,71 +996,71 @@ FAIL (expected: Should panic, but doesn't) (0.07s) crux doesn't match oracle Use -p '/null_eq/' to rerun this test only. - coerce_unsized: OK (2.14s) - Compiling and running oracle program (0.16s) - Oracle output: (1, 2) (1.98s) + coerce_unsized: OK (2.02s) + Compiling and running oracle program (0.17s) + Oracle output: (1, 2) (1.85s) Crux output: (1, 2) - offset: OK (2.14s) - Compiling and running oracle program (0.16s) - Oracle output: 3 (1.99s) + offset: OK (1.94s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (1.79s) Crux output: 3 - is_null: OK (2.19s) - Compiling and running oracle program (0.16s) - Oracle output: (false, true) (2.02s) + is_null: OK (1.95s) + Compiling and running oracle program (0.15s) + Oracle output: (false, true) (1.80s) Crux output: (false, true) - valid_eq: OK (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.00s) + valid_eq: OK (1.97s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.83s) Crux output: () ops - deref2: OK (2.17s) - Compiling and running oracle program (0.16s) - Oracle output: () (2.00s) + deref2: OK (1.94s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.80s) Crux output: () - index2: OK (2.15s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.98s) + index2: OK (1.95s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.80s) Crux output: () - arith1: OK (2.28s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.12s) + arith1: OK (1.95s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.80s) Crux output: () - index3: OK (2.14s) - Compiling and running oracle program (0.17s) - Oracle output: () (1.97s) + index3: OK (1.95s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.80s) Crux output: () - deref1: OK (2.18s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.00s) + deref1: OK (1.94s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.79s) Crux output: () - index1: OK (2.11s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.95s) + index1: OK (1.92s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.77s) Crux output: () - deref3: OK (2.18s) - Compiling and running oracle program (0.16s) - Oracle output: () (2.01s) + deref3: OK (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.77s) Crux output: () statics - promoted_fn: OK (2.13s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (1.97s) + promoted_fn: OK (1.94s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.80s) Crux output: 1 - promoted_static: OK (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (2.01s) + promoted_static: OK (1.96s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.82s) Crux output: 1 io - cursor_write2: FAIL (2.68s) - Compiling and running oracle program (0.18s) - Oracle output: 0 (2.50s) - user error (JSON Decoding of test/conc_eval/io/cursor_write2.all.mir failed: Error in $.statics[13].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) + cursor_write2: FAIL (2.44s) + Compiling and running oracle program (0.16s) + Oracle output: 0 (2.28s) + user error (JSON Decoding of test/conc_eval/io/cursor_write2.all.mir failed: Error in $.statics[7].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) Use -p '/cursor_write2/' to rerun this test only. - cursor_write: FAIL (2.78s) - Compiling and running oracle program (0.19s) - Oracle output: 0 (2.59s) - standalone use of `dyn` is not supported: TyDynamic core/1c79c753::error[0]::Error[0]::_trait56353a7b840403b4[0] + cursor_write: FAIL (2.60s) + Compiling and running oracle program (0.17s) + Oracle output: 0 (2.43s) + standalone use of `dyn` is not supported: TyDynamic core/95754f0e::error[0]::Error[0]::_trait56353a7b840403b4[0] CallStack (from HasCallStack): error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy tyToRepr, called at src/Mir/Trans.hs:824:12 in crucible-mir-0.1-inplace:Mir.Trans @@ -1089,10 +1076,10 @@ FAIL (expected: Should panic, but doesn't) (0.07s) translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '$0=="crux-mir.crux concrete..io.cursor_write"' to rerun this test only. - cursor_read: FAIL (2.75s) - Compiling and running oracle program (0.19s) - Oracle output: 0 (2.56s) - standalone use of `dyn` is not supported: TyDynamic core/1c79c753::error[0]::Error[0]::_trait56353a7b840403b4[0] + cursor_read: FAIL (2.44s) + Compiling and running oracle program (0.17s) + Oracle output: 0 (2.27s) + standalone use of `dyn` is not supported: TyDynamic core/95754f0e::error[0]::Error[0]::_trait56353a7b840403b4[0] CallStack (from HasCallStack): error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy tyToRepr, called at src/Mir/Trans.hs:824:12 in crucible-mir-0.1-inplace:Mir.Trans @@ -1109,16 +1096,16 @@ FAIL (expected: Should panic, but doesn't) (0.07s) Use -p '/io.cursor_read/' to rerun this test only. mem - maybe_uninit: FAIL (2.17s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (2.02s) + maybe_uninit: FAIL (2.01s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.87s) Crux output: failures: - ---- maybe_uninit/e2dcf2ce::crux_test[0] counterexamples ---- + ---- maybe_uninit/da71da36::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in maybe_uninit/e2dcf2ce::crux_test[0] - [Crux] Translation error in maybe_uninit/e2dcf2ce::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyInt B32]) + [Crux] internal: error: in maybe_uninit/da71da36::crux_test[0] + [Crux] Translation error in maybe_uninit/da71da36::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyInt B32]) [Crux] Overall status: Invalid. @@ -1126,16 +1113,16 @@ FAIL (expected: Should panic, but doesn't) (0.07s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..mem.maybe_uninit"' to rerun this test only. - maybe_uninit_array_cast: FAIL (2.21s) - Compiling and running oracle program (0.16s) - Oracle output: [1, 2] (2.05s) + maybe_uninit_array_cast: FAIL (2.02s) + Compiling and running oracle program (0.15s) + Oracle output: [1, 2] (1.87s) Crux output: failures: - ---- maybe_uninit_array_cast/97f384c1::crux_test[0] counterexamples ---- + ---- maybe_uninit_array_cast/c7f09dc0::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in maybe_uninit_array_cast/97f384c1::crux_test[0] - [Crux] Translation error in maybe_uninit_array_cast/97f384c1::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyArray (TyInt B32) 2]) + [Crux] internal: error: in maybe_uninit_array_cast/c7f09dc0::crux_test[0] + [Crux] Translation error in maybe_uninit_array_cast/c7f09dc0::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyArray (TyInt B32) 2]) [Crux] Overall status: Invalid. @@ -1144,43 +1131,43 @@ FAIL (expected: Should panic, but doesn't) (0.07s) Use -p '/maybe_uninit_array_cast/' to rerun this test only. time - instant: FAIL (2.66s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.48s) - user error (JSON Decoding of test/conc_eval/time/instant.all.mir failed: Error in $.statics[12].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) + instant: FAIL (2.31s) + Compiling and running oracle program (0.15s) + Oracle output: () (2.16s) + user error (JSON Decoding of test/conc_eval/time/instant.all.mir failed: Error in $.statics[6].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) Use -p '/instant/' to rerun this test only. struct - field_order: OK (2.19s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.02s) + field_order: OK (1.92s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.78s) Crux output: () - arg: OK (2.19s) - Compiling and running oracle program (0.17s) - Oracle output: 42 (2.02s) + arg: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.75s) Crux output: 42 - tup: OK (2.26s) - Compiling and running oracle program (0.16s) - Oracle output: () (2.09s) + tup: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.75s) Crux output: () - proj: OK (2.14s) - Compiling and running oracle program (0.18s) - Oracle output: 42 (1.95s) + proj: OK (1.85s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.71s) Crux output: 42 - ret: OK (2.14s) - Compiling and running oracle program (0.16s) - Oracle output: S { x: 42, y: 120 } (1.97s) + ret: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: S { x: 42, y: 120 } (1.71s) Crux output: S { x: 42, y: 120 } - repr_transparent: FAIL (2.24s) - Compiling and running oracle program (0.18s) - Oracle output: 6 (2.06s) + repr_transparent: FAIL (1.93s) + Compiling and running oracle program (0.15s) + Oracle output: 6 (1.78s) Crux output: failures: - ---- repr_transparent/9888f20e::crux_test[0] counterexamples ---- + ---- repr_transparent/5cbd4118::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:46:9: 46:18: error: in core/1c79c753::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0] - [Crux] Translation error in core/1c79c753::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0]: unimplemented cast: Misc + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:46:9: 46:18: error: in core/95754f0e::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0] + [Crux] Translation error in core/95754f0e::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyInt B32) Immut [Crux] as: TyRawPtr (TyArray (TyInt B32) 2) Immut @@ -1190,104 +1177,104 @@ FAIL (expected: Should panic, but doesn't) (0.07s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..struct.repr_transparent"' to rerun this test only. - repr_transparent_const: OK (2.13s) - Compiling and running oracle program (0.16s) - Oracle output: 124 (1.97s) + repr_transparent_const: OK (1.90s) + Compiling and running oracle program (0.15s) + Oracle output: 124 (1.75s) Crux output: 124 dyn - assoc_ty: OK (2.13s) - Compiling and running oracle program (0.17s) - Oracle output: 100 (1.96s) + assoc_ty: OK (1.88s) + Compiling and running oracle program (0.15s) + Oracle output: 100 (1.73s) Crux output: 100 - trait_param: OK (2.14s) - Compiling and running oracle program (0.16s) - Oracle output: 100 (1.98s) + trait_param: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 100 (1.71s) Crux output: 100 - plain_trait: OK (2.13s) - Compiling and running oracle program (0.16s) - Oracle output: 100 (1.97s) + plain_trait: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 100 (1.73s) Crux output: 100 - inherit: OK (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: 4 (2.00s) + inherit: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 4 (1.74s) Crux output: 4 stdlib - option: OK (2.12s) - Compiling and running oracle program (0.16s) - Oracle output: true (1.96s) + option: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: true (1.71s) Crux output: true - option2: OK (2.13s) - Compiling and running oracle program (0.16s) - Oracle output: 0 (1.98s) + option2: OK (1.88s) + Compiling and running oracle program (0.15s) + Oracle output: 0 (1.73s) Crux output: 0 - result: OK (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: 27 (2.00s) + result: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 27 (1.72s) Crux output: 27 - default: OK (2.17s) - Compiling and running oracle program (0.18s) - Oracle output: true (1.99s) + default: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: true (1.74s) Crux output: true - cvt: OK (2.14s) - Compiling and running oracle program (0.17s) - Oracle output: 0 (1.98s) + cvt: OK (1.91s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.77s) Crux output: 0 - range: OK (2.13s) - Compiling and running oracle program (0.17s) - Oracle output: 10 (1.97s) + range: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: 10 (1.75s) Crux output: 10 - poly: OK (2.14s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (1.97s) + poly: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.74s) Crux output: 1 - result_interior: OK (2.27s) - Compiling and running oracle program (0.17s) - Oracle output: 3 (2.10s) + result_interior: OK (1.88s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (1.73s) Crux output: 3 - default_impl: OK (2.16s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.00s) + default_impl: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.71s) Crux output: () - option3: OK (2.15s) - Compiling and running oracle program (0.18s) - Oracle output: 27 (1.97s) + option3: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 27 (1.73s) Crux output: 27 - teq: OK (2.15s) - Compiling and running oracle program (0.17s) - Oracle output: false (1.98s) + teq: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: false (1.74s) Crux output: false array - wick2: OK (2.14s) - Compiling and running oracle program (0.17s) - Oracle output: true (1.97s) + wick2: OK (1.90s) + Compiling and running oracle program (0.15s) + Oracle output: true (1.76s) Crux output: true - arg: OK (2.13s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (1.96s) + arg: OK (1.88s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.74s) Crux output: 2 - iter: OK (2.24s) - Compiling and running oracle program (0.17s) - Oracle output: 3 (2.07s) + iter: OK (1.91s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.77s) Crux output: 3 - mk_and_proj: OK (2.15s) - Compiling and running oracle program (0.17s) - Oracle output: 42 (1.98s) + mk_and_proj: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (1.72s) Crux output: 42 - wick3: FAIL (expected: needs Vec data structure from stdlib) (0.23s) - Compiling and running oracle program (0.19s) - Oracle output: true (0.05s) + wick3: FAIL (expected: needs Vec data structure from stdlib) (0.20s) + Compiling and running oracle program (0.16s) + Oracle output: true (0.04s) user error (Error 101 while running mir-json on test/conc_eval/array/wick3.rs) (expected failure) - clone: FAIL (2.16s) - Compiling and running oracle program (0.17s) - Oracle output: () (1.99s) + clone: FAIL (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.76s) Crux output: failures: - ---- clone/a68cf198::f[0] counterexamples ---- + ---- clone/64b1e1e8::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:46:9: 46:18: error: in core/1c79c753::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0] - [Crux] Translation error in core/1c79c753::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0]: unimplemented cast: Misc + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:46:9: 46:18: error: in core/95754f0e::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0] + [Crux] Translation error in core/95754f0e::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyInt B32) Immut [Crux] as: TyRawPtr (TyArray (TyInt B32) 3) Immut @@ -1297,21 +1284,21 @@ FAIL (expected: Should panic, but doesn't) (0.07s) crux doesn't match oracle Use -p '/array.clone/' to rerun this test only. - wick1: FAIL (expected: needs Vec data structure from stdlib) (0.23s) - Compiling and running oracle program (0.18s) + wick1: FAIL (expected: needs Vec data structure from stdlib) (0.21s) + Compiling and running oracle program (0.16s) Oracle output: true (0.05s) user error (Error 101 while running mir-json on test/conc_eval/array/wick1.rs) (expected failure) - mut_index: FAIL (2.76s) - Compiling and running oracle program (0.17s) - Oracle output: 7 (2.59s) + mut_index: FAIL (2.41s) + Compiling and running oracle program (0.15s) + Oracle output: 7 (2.26s) Crux output: failures: - ---- mut_index/e0856e73::crux_test[0] counterexamples ---- + ---- mut_index/4ff817fe::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1319,24 +1306,24 @@ FAIL (expected: Should panic, but doesn't) (0.07s) crux doesn't match oracle Use -p '/mut_index/' to rerun this test only. - mut_arg: OK (2.14s) - Compiling and running oracle program (0.16s) - Oracle output: 42 (1.98s) + mut_arg: OK (1.92s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (1.77s) Crux output: 42 - const: OK (2.13s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.97s) + const: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.72s) Crux output: 1 - from_slice: FAIL (2.93s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.75s) + from_slice: FAIL (2.55s) + Compiling and running oracle program (0.15s) + Oracle output: () (2.41s) Crux output: failures: - ---- from_slice/e6be3dab::f[0] counterexamples ---- + ---- from_slice/21321bd2::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/array/mod.rs:266:23: 266:54: error: in core/1c79c753::array[0]::{impl#10}[0]::try_from[0]::_inste494657dbbea865b[0] - [Crux] Translation error in core/1c79c753::array[0]::{impl#10}[0]::try_from[0]::_inste494657dbbea865b[0]: unimplemented cast: Misc + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/array/mod.rs:266:23: 266:54: error: in core/95754f0e::array[0]::{impl#10}[0]::try_from[0]::_inste494657dbbea865b[0] + [Crux] Translation error in core/95754f0e::array[0]::{impl#10}[0]::try_from[0]::_inste494657dbbea865b[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyUint B8) Immut [Crux] as: TyRawPtr (TyArray (TyUint B8) 4) Immut @@ -1346,26 +1333,26 @@ FAIL (expected: Should panic, but doesn't) (0.07s) crux doesn't match oracle Use -p '/array.from_slice/' to rerun this test only. - const_impl: OK (2.17s) - Compiling and running oracle program (0.16s) - Oracle output: 5 (2.00s) + const_impl: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 5 (1.74s) Crux output: 5 enum - match: OK (2.14s) - Compiling and running oracle program (0.17s) - Oracle output: 42 (1.97s) + match: OK (1.90s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.75s) Crux output: 42 - field_order: OK (2.17s) - Compiling and running oracle program (0.16s) - Oracle output: () (2.01s) + field_order: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.72s) Crux output: () - mixed_discrs: OK (2.15s) - Compiling and running oracle program (0.18s) - Oracle output: () (1.98s) + mixed_discrs: OK (1.88s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.73s) Crux output: () - arg: OK (2.14s) - Compiling and running oracle program (0.17s) - Oracle output: 0 (1.97s) + arg: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.74s) Crux output: 0 arg2: rustc compilation failed for arg2 error output: @@ -1395,16 +1382,16 @@ FAIL (0.05s) failed to compile and run Use -p '/arg2/' to rerun this test only. - cow: FAIL (2.37s) - Compiling and running oracle program (0.17s) - Oracle output: 200 (2.19s) + cow: FAIL (2.03s) + Compiling and running oracle program (0.15s) + Oracle output: 200 (1.87s) Crux output: failures: - ---- cow/d3e6425a::crux_test[0] counterexamples ---- + ---- cow/64514981::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1412,33 +1399,33 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/cow/' to rerun this test only. - ret: OK (2.14s) - Compiling and running oracle program (0.17s) - Oracle output: (A(42), B { x: 42 }, C) (1.96s) + ret: OK (1.88s) + Compiling and running oracle program (0.15s) + Oracle output: (A(42), B { x: 42 }, C) (1.73s) Crux output: (A(42), B { x: 42 }, C) - eq: OK (2.25s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.07s) + eq: OK (1.90s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.75s) Crux output: () - cmp: OK (2.13s) - Compiling and running oracle program (0.17s) - Oracle output: -23 (1.97s) + cmp: OK (1.87s) + Compiling and running oracle program (0.15s) + Oracle output: -23 (1.72s) Crux output: -23 - inner: OK (2.18s) - Compiling and running oracle program (0.17s) - Oracle output: 4 (2.01s) + inner: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 4 (1.74s) Crux output: 4 vec - drop: FAIL (2.33s) - Compiling and running oracle program (0.18s) - Oracle output: () (2.15s) + drop: FAIL (2.04s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.89s) Crux output: failures: - ---- drop/99a52740::crux_test[0] counterexamples ---- + ---- drop/42c23083::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -1446,16 +1433,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/drop/' to rerun this test only. - set_len: FAIL (2.31s) - Compiling and running oracle program (0.18s) - Oracle output: 2 (2.13s) + set_len: FAIL (2.03s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.88s) Crux output: failures: - ---- set_len/44e827e2::crux_test[0] counterexamples ---- + ---- set_len/7feb2720::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -1463,20 +1450,20 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/set_len/' to rerun this test only. - push: OK (2.44s) - Compiling and running oracle program (0.19s) - Oracle output: (1, 2) (2.24s) + push: OK (2.10s) + Compiling and running oracle program (0.16s) + Oracle output: (1, 2) (1.94s) Crux output: (1, 2) - extend: FAIL (2.53s) - Compiling and running oracle program (0.20s) - Oracle output: (1, 10) (2.34s) + extend: FAIL (2.15s) + Compiling and running oracle program (0.16s) + Oracle output: (1, 10) (1.99s) Crux output: failures: - ---- extend/97a81c1b::crux_test[0] counterexamples ---- + ---- extend/bf33c8cd::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in core/1c79c753::iter[0]::traits[0]::iterator[0]::Iterator[0]::try_fold[0]::_inst647f2421f6e4ebd6[0] - [Crux] Translation error in core/1c79c753::iter[0]::traits[0]::iterator[0]::Iterator[0]::try_fold[0]::_inst647f2421f6e4ebd6[0]: error building variant core/1c79c753::ops[0]::control_flow[0]::ControlFlow[0]::Continue[0]: type mismatch: expected AnyRepr but got BVRepr 32 in field 0: ([FieldRepr (FkInit AnyRepr)],[Just bVLit(32, BV 0)]) -- [Just bVLit(32, BV 0): BVRepr 32] + [Crux] internal: error: in core/95754f0e::iter[0]::traits[0]::iterator[0]::Iterator[0]::try_fold[0]::_inst647f2421f6e4ebd6[0] + [Crux] Translation error in core/95754f0e::iter[0]::traits[0]::iterator[0]::Iterator[0]::try_fold[0]::_inst647f2421f6e4ebd6[0]: error building variant core/95754f0e::ops[0]::control_flow[0]::ControlFlow[0]::Continue[0]: type mismatch: expected AnyRepr but got BVRepr 32 in field 0: ([FieldRepr (FkInit AnyRepr)],[Just bVLit(32, BV 0)]) -- [Just bVLit(32, BV 0): BVRepr 32] [Crux] Overall status: Invalid. @@ -1484,16 +1471,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..vec.extend"' to rerun this test only. - collect: FAIL (3.15s) - Compiling and running oracle program (0.19s) - Oracle output: 45 (2.96s) + collect: FAIL (2.78s) + Compiling and running oracle program (0.16s) + Oracle output: 45 (2.62s) Crux output: failures: - ---- collect/f1727714::crux_test[0] counterexamples ---- + ---- collect/0932e133::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:786:58: 786:72: error: in core/1c79c753::ptr[0]::const_ptr[0]::{impl#0}[0]::sub_ptr[0]::runtime[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/1c79c753::ptr[0]::const_ptr[0]::{impl#0}[0]::sub_ptr[0]::runtime[0]::_inst1e2825177cd3b608[0]: No translation for pointer binop: >= + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:786:58: 786:72: error: in core/95754f0e::ptr[0]::const_ptr[0]::{impl#0}[0]::sub_ptr[0]::runtime[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/95754f0e::ptr[0]::const_ptr[0]::{impl#0}[0]::sub_ptr[0]::runtime[0]::_inst1e2825177cd3b608[0]: No translation for pointer binop: >= [Crux] Overall status: Invalid. @@ -1501,31 +1488,31 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/collect/' to rerun this test only. - extend_trusted_len: OK (3.11s) - Compiling and running oracle program (0.18s) - Oracle output: (1, 10) (2.93s) + extend_trusted_len: OK (2.66s) + Compiling and running oracle program (0.16s) + Oracle output: (1, 10) (2.50s) Crux output: (1, 10) - from_elem_zero: FAIL (0.24s) - Compiling and running oracle program (0.18s) - Oracle output: 0 (0.06s) + from_elem_zero: FAIL (0.21s) + Compiling and running oracle program (0.16s) + Oracle output: 0 (0.05s) user error (Error 101 while running mir-json on test/conc_eval/vec/from_elem_zero.rs) Use -p '/from_elem_zero/' to rerun this test only. clos - promoted: OK (2.18s) - Compiling and running oracle program (0.17s) - Oracle output: 33 (2.02s) + promoted: OK (1.92s) + Compiling and running oracle program (0.15s) + Oracle output: 33 (1.77s) Crux output: 33 - as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (2.21s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (2.05s) + as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.74s) Crux output: failures: - ---- as_fn_ptr/a97a9eab::crux_test[0] counterexamples ---- + ---- as_fn_ptr/7cd8b443::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/a97a9eab::crux_test[0] - [Crux] Translation error in as_fn_ptr/a97a9eab::crux_test[0]: unimplemented cast: ClosureFnPointer + [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/7cd8b443::crux_test[0] + [Crux] Translation error in as_fn_ptr/7cd8b443::crux_test[0]: unimplemented cast: ClosureFnPointer [Crux] ty: TyClosure [] [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [], _fsreturn_ty = TyInt B32, _fsabi = RustAbi, _fsspreadarg = Nothing}) @@ -1534,102 +1521,102 @@ FAIL (0.05s) test/Test.hs:124: crux doesn't match oracle (expected failure) - fnptr_fnmut: OK (2.21s) - Compiling and running oracle program (0.17s) - Oracle output: 4 (2.04s) + fnptr_fnmut: OK (1.87s) + Compiling and running oracle program (0.15s) + Oracle output: 4 (1.72s) Crux output: 4 - direct_fnonce: OK (2.15s) - Compiling and running oracle program (0.19s) - Oracle output: 2 (1.97s) + direct_fnonce: OK (1.86s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.71s) Crux output: 2 - conv_fnonce_fnmut: OK (2.18s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (2.02s) + conv_fnonce_fnmut: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.73s) Crux output: 2 - fn_static_poly: OK (2.13s) - Compiling and running oracle program (0.17s) - Oracle output: 3 (1.96s) + fn_static_poly: OK (1.91s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.77s) Crux output: 3 - fnptr_fnonce: OK (2.16s) - Compiling and running oracle program (0.17s) - Oracle output: 4 (1.99s) + fnptr_fnonce: OK (1.88s) + Compiling and running oracle program (0.15s) + Oracle output: 4 (1.73s) Crux output: 4 - fn_static: OK (2.16s) - Compiling and running oracle program (0.16s) - Oracle output: 3 (1.99s) + fn_static: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.73s) Crux output: 3 - fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (0.20s) - Compiling and running oracle program (0.16s) + fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (0.18s) + Compiling and running oracle program (0.15s) Oracle output: 3 (0.03s) user error (Error 101 while running mir-json on test/conc_eval/clos/fn_dyn.rs) (expected failure) - fnonce: OK (2.13s) - Compiling and running oracle program (0.16s) - Oracle output: false (1.96s) + fnonce: OK (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: false (1.76s) Crux output: false - conv_fnmut_fn: OK (2.16s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (1.99s) + conv_fnmut_fn: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.76s) Crux output: 2 - direct_fnmut2: OK (2.17s) - Compiling and running oracle program (0.19s) - Oracle output: 7 (1.98s) + direct_fnmut2: OK (1.90s) + Compiling and running oracle program (0.15s) + Oracle output: 7 (1.75s) Crux output: 7 - fo: OK (2.17s) - Compiling and running oracle program (0.16s) - Oracle output: 29 (2.01s) + fo: OK (1.93s) + Compiling and running oracle program (0.15s) + Oracle output: 29 (1.78s) Crux output: 29 - unique_borrow: OK (2.18s) - Compiling and running oracle program (0.17s) - Oracle output: 3 (2.00s) + unique_borrow: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.76s) Crux output: 3 - conv_fnonce_fn: OK (2.25s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (2.09s) + conv_fnonce_fn: OK (1.90s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.75s) Crux output: 2 - fnptr_fn: OK (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: 4 (2.00s) + fnptr_fn: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: 4 (1.75s) Crux output: 4 - direct_fnmut: OK (2.16s) - Compiling and running oracle program (0.17s) - Oracle output: 7 (1.98s) + direct_fnmut: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: 7 (1.75s) Crux output: 7 - dispatch_fnmut: OK (2.16s) - Compiling and running oracle program (0.17s) - Oracle output: 7 (1.99s) + dispatch_fnmut: OK (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: 7 (1.76s) Crux output: 7 - poly: OK (2.13s) - Compiling and running oracle program (0.17s) - Oracle output: 3 (1.96s) + poly: OK (1.94s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (1.79s) Crux output: 3 - direct_fn: OK (2.14s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (1.97s) + direct_fn: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.74s) Crux output: 2 - fnptr_closure: OK (2.15s) - Compiling and running oracle program (0.16s) - Oracle output: 7 (1.98s) + fnptr_closure: OK (1.91s) + Compiling and running oracle program (0.14s) + Oracle output: 7 (1.77s) Crux output: 7 - fnonce1: OK (2.27s) - Compiling and running oracle program (0.17s) - Oracle output: 3 (2.10s) + fnonce1: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.72s) Crux output: 3 - ref_fnmut: OK (2.21s) - Compiling and running oracle program (0.18s) - Oracle output: 7 (2.03s) + ref_fnmut: OK (2.01s) + Compiling and running oracle program (0.15s) + Oracle output: 7 (1.87s) Crux output: 7 vec_deque - pop: FAIL (2.92s) - Compiling and running oracle program (0.18s) - Oracle output: [5, 4, 3, 1, 2] (2.74s) + pop: FAIL (2.77s) + Compiling and running oracle program (0.17s) + Oracle output: [5, 4, 3, 1, 2] (2.60s) Crux output: failures: - ---- pop/b657f8fe::crux_test[0] counterexamples ---- + ---- pop/3a129246::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -1637,16 +1624,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/vec_deque.pop/' to rerun this test only. - iter_clone: FAIL (3.04s) - Compiling and running oracle program (0.19s) - Oracle output: [1, 2, 3, 2, 3] (2.85s) + iter_clone: FAIL (2.83s) + Compiling and running oracle program (0.16s) + Oracle output: [1, 2, 3, 2, 3] (2.66s) Crux output: failures: - ---- iter_clone/aeac0d43::crux_test[0] counterexamples ---- + ---- iter_clone/ffad8245::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -1654,16 +1641,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/iter_clone/' to rerun this test only. - push: FAIL (3.16s) - Compiling and running oracle program (0.21s) - Oracle output: [4, 1, 2, 3, 5] (2.95s) + push: FAIL (2.70s) + Compiling and running oracle program (0.16s) + Oracle output: [4, 1, 2, 3, 5] (2.54s) Crux output: failures: - ---- push/0f2c3e92::crux_test[0] counterexamples ---- + ---- push/7760ccfb::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in alloc/8ccd0c40::collections[0]::vec_deque[0]::{impl#5}[0]::grow[0]::_inst6708c5c8c5ed40f8[0] - [Crux] panicking::panic, called from alloc/8ccd0c40::collections[0]::vec_deque[0]::{impl#5}[0]::grow[0]::_inst6708c5c8c5ed40f8[0] + [Crux] internal: error: in alloc/fed0e039::collections[0]::vec_deque[0]::{impl#5}[0]::grow[0]::_inst6708c5c8c5ed40f8[0] + [Crux] panicking::panic, called from alloc/fed0e039::collections[0]::vec_deque[0]::{impl#5}[0]::grow[0]::_inst6708c5c8c5ed40f8[0] [Crux] Overall status: Invalid. @@ -1671,16 +1658,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/vec_deque.push/' to rerun this test only. - retain: FAIL (3.11s) - Compiling and running oracle program (0.19s) - Oracle output: [1, 4] (2.92s) + retain: FAIL (2.70s) + Compiling and running oracle program (0.17s) + Oracle output: [1, 4] (2.53s) Crux output: failures: - ---- retain/9ac0c4dd::crux_test[0] counterexamples ---- + ---- retain/0ae7e799::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -1688,16 +1675,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/retain/' to rerun this test only. - rotate_left: FAIL (3.17s) - Compiling and running oracle program (0.20s) - Oracle output: [3, 4, 5, 1, 2] (2.97s) + rotate_left: FAIL (2.82s) + Compiling and running oracle program (0.16s) + Oracle output: [3, 4, 5, 1, 2] (2.66s) Crux output: failures: - ---- rotate_left/3a6979d7::crux_test[0] counterexamples ---- + ---- rotate_left/b0c4e843::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -1705,16 +1692,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/rotate_left/' to rerun this test only. - rotate_right: FAIL (2.99s) - Compiling and running oracle program (0.19s) - Oracle output: [4, 5, 1, 2, 3] (2.81s) + rotate_right: FAIL (2.84s) + Compiling and running oracle program (0.16s) + Oracle output: [4, 5, 1, 2, 3] (2.68s) Crux output: failures: - ---- rotate_right/a923d682::crux_test[0] counterexamples ---- + ---- rotate_right/de4668b4::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. @@ -1723,20 +1710,20 @@ FAIL (0.05s) Use -p '/rotate_right/' to rerun this test only. slice - len: OK (2.13s) + len: OK (2.02s) Compiling and running oracle program (0.16s) - Oracle output: 5 (1.97s) + Oracle output: 5 (1.86s) Crux output: 5 - mut_range: FAIL (2.77s) - Compiling and running oracle program (0.18s) - Oracle output: 86 (2.59s) + mut_range: FAIL (2.49s) + Compiling and running oracle program (0.17s) + Oracle output: 86 (2.33s) Crux output: failures: - ---- mut_range/92e0cd57::crux_test[0] counterexamples ---- + ---- mut_range/a64fa57c::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1744,16 +1731,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/mut_range/' to rerun this test only. - range_len: FAIL (2.91s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (2.74s) + range_len: FAIL (2.47s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (2.32s) Crux output: failures: - ---- range_len/57a23c29::crux_test[0] counterexamples ---- + ---- range_len/d3e86403::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1761,16 +1748,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..slice.range_len"' to rerun this test only. - range_len_mut: FAIL (2.99s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (2.81s) + range_len_mut: FAIL (2.43s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (2.29s) Crux output: failures: - ---- range_len_mut/b9e32f41::crux_test[0] counterexamples ---- + ---- range_len_mut/0c7dec71::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1778,24 +1765,24 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/range_len_mut/' to rerun this test only. - mk_and_proj: OK (2.26s) - Compiling and running oracle program (0.16s) - Oracle output: 42 (2.10s) + mk_and_proj: OK (1.95s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.79s) Crux output: 42 - swap: OK (2.20s) - Compiling and running oracle program (0.17s) - Oracle output: 2001 (2.03s) + swap: OK (1.96s) + Compiling and running oracle program (0.15s) + Oracle output: 2001 (1.81s) Crux output: 2001 - eq: FAIL (2.23s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.06s) + eq: FAIL (1.94s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.79s) Crux output: failures: - ---- eq/394a586c::f[0] counterexamples ---- + ---- eq/336948bb::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/mem/mod.rs:338:38: 338:41: error: in core/1c79c753::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/1c79c753::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::size_of_val[0]::_inst55af81a2cef2701c[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/mem/mod.rs:338:38: 338:41: error: in core/95754f0e::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/95754f0e::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::size_of_val[0]::_inst55af81a2cef2701c[0] [Crux] Overall status: Invalid. @@ -1803,16 +1790,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/slice.eq/' to rerun this test only. - iter_mut: FAIL (2.25s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.07s) + iter_mut: FAIL (2.01s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.86s) Crux output: failures: - ---- iter_mut/05baf6c7::f[0] counterexamples ---- + ---- iter_mut/1188a615::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/array/mod.rs:266:23: 266:54: error: in core/1c79c753::array[0]::{impl#10}[0]::try_from[0]::_insta17aeaef2aec3782[0] - [Crux] Translation error in core/1c79c753::array[0]::{impl#10}[0]::try_from[0]::_insta17aeaef2aec3782[0]: unimplemented cast: Misc + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/array/mod.rs:266:23: 266:54: error: in core/95754f0e::array[0]::{impl#10}[0]::try_from[0]::_insta17aeaef2aec3782[0] + [Crux] Translation error in core/95754f0e::array[0]::{impl#10}[0]::try_from[0]::_insta17aeaef2aec3782[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyInt B32) Immut [Crux] as: TyRawPtr (TyArray (TyInt B32) 3) Immut @@ -1822,16 +1809,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/iter_mut/' to rerun this test only. - get: FAIL (2.22s) - Compiling and running oracle program (0.17s) - Oracle output: true (2.05s) + get: FAIL (1.97s) + Compiling and running oracle program (0.15s) + Oracle output: true (1.82s) Crux output: failures: - ---- get/0730ccf8::crux_test[0] counterexamples ---- + ---- get/39caa45e::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/1c79c753::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1839,390 +1826,390 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/slice.get/' to rerun this test only. - mut: OK (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: 42 (2.00s) + mut: OK (1.95s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.80s) Crux output: 42 - last: OK (2.18s) - Compiling and running oracle program (0.17s) - Oracle output: 3 (2.02s) + last: OK (2.05s) + Compiling and running oracle program (0.16s) + Oracle output: 3 (1.89s) Crux output: 3 crux symbolic Output testing - mux_init_mut: FAIL (2.11s) + mux_init_mut: FAIL (2.05s) files test/symb_eval/refs/mux_init_mut.good and test/symb_eval/refs/mux_init_mut.out differ; test/symb_eval/refs/mux_init_mut.out contains: - test mux_init_mut/822b14fd::crux_test[0]: FAILED + test mux_init_mut/c7bde8a8::crux_test[0]: FAILED failures: - ---- mux_init_mut/822b14fd::crux_test[0] counterexamples ---- + ---- mux_init_mut/c7bde8a8::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/mux_init_mut/' to rerun this test only. - mux_init_imm: FAIL (2.14s) + mux_init_imm: FAIL (1.98s) files test/symb_eval/refs/mux_init_imm.good and test/symb_eval/refs/mux_init_imm.out differ; test/symb_eval/refs/mux_init_imm.out contains: - test mux_init_imm/156ea6ac::crux_test[0]: FAILED + test mux_init_imm/45896388::crux_test[0]: FAILED failures: - ---- mux_init_imm/156ea6ac::crux_test[0] counterexamples ---- + ---- mux_init_imm/45896388::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/mux_init_imm/' to rerun this test only. - early_fail: FAIL (2.14s) + early_fail: FAIL (2.10s) files test/symb_eval/crux/early_fail.good and test/symb_eval/crux/early_fail.out differ; test/symb_eval/crux/early_fail.out contains: - test early_fail/d7997ce8::fail1[0]: FAILED - test early_fail/d7997ce8::fail2[0]: FAILED + test early_fail/a3644e81::fail1[0]: FAILED + test early_fail/a3644e81::fail2[0]: FAILED failures: - ---- early_fail/d7997ce8::fail1[0] counterexamples ---- + ---- early_fail/a3644e81::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in early_fail/d7997ce8::fail1[0] - [Crux] panicking::panic_fmt, called from early_fail/d7997ce8::fail1[0] + [Crux] internal: error: in early_fail/a3644e81::fail1[0] + [Crux] panicking::panic_fmt, called from early_fail/a3644e81::fail1[0] - ---- early_fail/d7997ce8::fail2[0] counterexamples ---- + ---- early_fail/a3644e81::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/early_fail/' to rerun this test only. - mixed_fail: FAIL (2.25s) + mixed_fail: FAIL (2.08s) files test/symb_eval/crux/mixed_fail.good and test/symb_eval/crux/mixed_fail.out differ; test/symb_eval/crux/mixed_fail.out contains: - test mixed_fail/276aa1e8::fail1[0]: FAILED - test mixed_fail/276aa1e8::fail2[0]: FAILED - test mixed_fail/276aa1e8::pass1[0]: FAILED - test mixed_fail/276aa1e8::pass2[0]: FAILED + test mixed_fail/d1feb05c::fail1[0]: FAILED + test mixed_fail/d1feb05c::fail2[0]: FAILED + test mixed_fail/d1feb05c::pass1[0]: FAILED + test mixed_fail/d1feb05c::pass2[0]: FAILED failures: - ---- mixed_fail/276aa1e8::fail1[0] counterexamples ---- + ---- mixed_fail/d1feb05c::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - ---- mixed_fail/276aa1e8::fail2[0] counterexamples ---- + ---- mixed_fail/d1feb05c::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - ---- mixed_fail/276aa1e8::pass1[0] counterexamples ---- + ---- mixed_fail/d1feb05c::pass1[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - ---- mixed_fail/276aa1e8::pass2[0] counterexamples ---- + ---- mixed_fail/d1feb05c::pass2[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/mixed_fail/' to rerun this test only. - multi: FAIL (2.10s) + multi: FAIL (2.00s) files test/symb_eval/crux/multi.good and test/symb_eval/crux/multi.out differ; test/symb_eval/crux/multi.out contains: - test multi/a87e06c3::fail1[0]: FAILED - test multi/a87e06c3::fail2[0]: FAILED - test multi/a87e06c3::fail3[0]: FAILED + test multi/2eb0a9e7::fail1[0]: FAILED + test multi/2eb0a9e7::fail2[0]: FAILED + test multi/2eb0a9e7::fail3[0]: FAILED failures: - ---- multi/a87e06c3::fail1[0] counterexamples ---- + ---- multi/2eb0a9e7::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - ---- multi/a87e06c3::fail2[0] counterexamples ---- + ---- multi/2eb0a9e7::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - ---- multi/a87e06c3::fail3[0] counterexamples ---- + ---- multi/2eb0a9e7::fail3[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/Output testing.multi/' to rerun this test only. - fail_return: FAIL (2.13s) + fail_return: FAIL (2.02s) files test/symb_eval/crux/fail_return.good and test/symb_eval/crux/fail_return.out differ; test/symb_eval/crux/fail_return.out contains: - test fail_return/06c77a43::fail1[0]: FAILED - test fail_return/06c77a43::fail2[0]: FAILED + test fail_return/8b108ddc::fail1[0]: FAILED + test fail_return/8b108ddc::fail2[0]: FAILED failures: - ---- fail_return/06c77a43::fail1[0] counterexamples ---- + ---- fail_return/8b108ddc::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - ---- fail_return/06c77a43::fail2[0] counterexamples ---- + ---- fail_return/8b108ddc::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/fail_return/' to rerun this test only. - no_conc: FAIL (2.14s) + no_conc: FAIL (1.93s) files test/symb_eval/concretize/no_conc.good and test/symb_eval/concretize/no_conc.out differ; test/symb_eval/concretize/no_conc.out contains: - test no_conc/d856aa0d::crux_test[0]: FAILED + test no_conc/fd84226e::crux_test[0]: FAILED failures: - ---- no_conc/d856aa0d::crux_test[0] counterexamples ---- + ---- no_conc/fd84226e::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/no_conc/' to rerun this test only. - conc: FAIL (2.12s) + conc: FAIL (2.03s) files test/symb_eval/concretize/conc.good and test/symb_eval/concretize/conc.out differ; test/symb_eval/concretize/conc.out contains: - test conc/e48cfc0a::crux_test[0]: FAILED + test conc/daa4ef6b::crux_test[0]: FAILED failures: - ---- conc/e48cfc0a::crux_test[0] counterexamples ---- + ---- conc/daa4ef6b::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.conc"' to rerun this test only. - array: FAIL (2.16s) + array: FAIL (2.14s) files test/symb_eval/concretize/array.good and test/symb_eval/concretize/array.out differ; test/symb_eval/concretize/array.out contains: - test array/bf76a5b2::crux_test[0]: FAILED + test array/ddc0118b::crux_test[0]: FAILED failures: - ---- array/bf76a5b2::crux_test[0] counterexamples ---- + ---- array/ddc0118b::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::array[0]::symbolic[0]::_inst1e2825177cd3b608[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::array[0]::symbolic[0]::_inst1e2825177cd3b608[0] + [Crux] internal: error: in crucible/7940c2ae::array[0]::symbolic[0]::_inst1e2825177cd3b608[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::array[0]::symbolic[0]::_inst1e2825177cd3b608[0] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.array"' to rerun this test only. - assert_ok: FAIL (2.73s) + assert_ok: FAIL (2.61s) files test/symb_eval/concretize/assert_ok.good and test/symb_eval/concretize/assert_ok.out differ; test/symb_eval/concretize/assert_ok.out contains: - test assert_ok/bc7b1378::crux_test[0]: FAILED + test assert_ok/22df3d32::crux_test[0]: FAILED failures: - ---- assert_ok/bc7b1378::crux_test[0] counterexamples ---- + ---- assert_ok/22df3d32::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/assert_ok/' to rerun this test only. - assert: FAIL (2.77s) + assert: FAIL (2.44s) files test/symb_eval/concretize/assert.good and test/symb_eval/concretize/assert.out differ; test/symb_eval/concretize/assert.out contains: - test assert/6f5b63e9::crux_test[0]: FAILED + test assert/be1de2be::crux_test[0]: FAILED failures: - ---- assert/6f5b63e9::crux_test[0] counterexamples ---- + ---- assert/be1de2be::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.assert"' to rerun this test only. - bytes2: FAIL (2.23s) + bytes2: FAIL (1.96s) files test/symb_eval/crypto/bytes2.good and test/symb_eval/crypto/bytes2.out differ; test/symb_eval/crypto/bytes2.out contains: - test bytes2/01526193::f[0]: FAILED + test bytes2/529826ca::f[0]: FAILED failures: - ---- bytes2/01526193::f[0] counterexamples ---- + ---- bytes2/529826ca::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/bytes2/' to rerun this test only. - double: FAIL (2.05s) + double: FAIL (1.83s) files test/symb_eval/crypto/double.good and test/symb_eval/crypto/double.out differ; test/symb_eval/crypto/double.out contains: - test double/e982a639::f[0]: FAILED + test double/afc4527c::f[0]: FAILED failures: - ---- double/e982a639::f[0] counterexamples ---- + ---- double/afc4527c::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u32[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u32[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u32[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u32[0] [Crux] Overall status: Invalid. Use -p '/double/' to rerun this test only. - ffs: FAIL (2.07s) + ffs: FAIL (1.92s) files test/symb_eval/crypto/ffs.good and test/symb_eval/crypto/ffs.out differ; test/symb_eval/crypto/ffs.out contains: - test ffs/f4a00052::f[0]: FAILED + test ffs/35c1efc1::f[0]: FAILED failures: - ---- ffs/f4a00052::f[0] counterexamples ---- + ---- ffs/35c1efc1::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u32[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u32[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u32[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u32[0] [Crux] Overall status: Invalid. Use -p '/Output testing.ffs/' to rerun this test only. - bytes: FAIL (2.21s) + bytes: FAIL (2.20s) files test/symb_eval/crypto/bytes.good and test/symb_eval/crypto/bytes.out differ; test/symb_eval/crypto/bytes.out contains: - test bytes/eccaff21::f[0]: FAILED + test bytes/10105f30::f[0]: FAILED failures: - ---- bytes/eccaff21::f[0] counterexamples ---- + ---- bytes/10105f30::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u64[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u64[0] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.bytes"' to rerun this test only. - bad_symb1: FAIL (2.10s) + bad_symb1: FAIL (1.90s) files test/symb_eval/overrides/bad_symb1.good and test/symb_eval/overrides/bad_symb1.out differ; test/symb_eval/overrides/bad_symb1.out contains: - test bad_symb1/716328c8::crux_test[0]: FAILED + test bad_symb1/89147cbd::crux_test[0]: FAILED failures: - ---- bad_symb1/716328c8::crux_test[0] counterexamples ---- + ---- bad_symb1/89147cbd::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/bad_symb1/' to rerun this test only. - bad_symb2: FAIL (2.07s) + bad_symb2: FAIL (1.91s) files test/symb_eval/overrides/bad_symb2.good and test/symb_eval/overrides/bad_symb2.out differ; test/symb_eval/overrides/bad_symb2.out contains: - test bad_symb2/bfa11db6::crux_test[0]: FAILED + test bad_symb2/96a88eca::crux_test[0]: FAILED failures: - ---- bad_symb2/bfa11db6::crux_test[0] counterexamples ---- + ---- bad_symb2/96a88eca::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/bad_symb2/' to rerun this test only. - override3: FAIL (2.05s) + override3: FAIL (1.81s) files test/symb_eval/overrides/override3.good and test/symb_eval/overrides/override3.out differ; test/symb_eval/overrides/override3.out contains: - test override3/337f1596::f[0]: FAILED + test override3/e13ebb8f::f[0]: FAILED failures: - ---- override3/337f1596::f[0] counterexamples ---- + ---- override3/e13ebb8f::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/override3/' to rerun this test only. - override1: FAIL (1.93s) + override1: FAIL (1.84s) files test/symb_eval/overrides/override1.good and test/symb_eval/overrides/override1.out differ; test/symb_eval/overrides/override1.out contains: - test override1/774ba047::f[0]: FAILED + test override1/33ca3dcf::f[0]: FAILED failures: - ---- override1/774ba047::f[0] counterexamples ---- + ---- override1/33ca3dcf::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::one[0] - [Crux] panicking::panic, called from crucible/bfc64a60::one[0] + [Crux] internal: error: in crucible/7940c2ae::one[0] + [Crux] panicking::panic, called from crucible/7940c2ae::one[0] [Crux] Overall status: Invalid. Use -p '/override1/' to rerun this test only. - override4: FAIL (2.13s) + override4: FAIL (1.98s) files test/symb_eval/overrides/override4.good and test/symb_eval/overrides/override4.out differ; test/symb_eval/overrides/override4.out contains: - test override4/d6e2b714::f[0]: FAILED + test override4/f0e88173::f[0]: FAILED failures: - ---- override4/d6e2b714::f[0] counterexamples ---- + ---- override4/f0e88173::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/override4/' to rerun this test only. - override_rust: FAIL (2.08s) + override_rust: FAIL (1.91s) files test/symb_eval/overrides/override_rust.good and test/symb_eval/overrides/override_rust.out differ; test/symb_eval/overrides/override_rust.out contains: - test override_rust/fd4f3e57::crux_test[0]: FAILED + test override_rust/81682348::crux_test[0]: FAILED failures: - ---- override_rust/fd4f3e57::crux_test[0] counterexamples ---- + ---- override_rust/81682348::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::override_[0]::_inst28abcb7c380bf102[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::override_[0]::_inst28abcb7c380bf102[0] + [Crux] internal: error: in crucible/7940c2ae::override_[0]::_inst28abcb7c380bf102[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::override_[0]::_inst28abcb7c380bf102[0] [Crux] Overall status: Invalid. Use -p '/override_rust/' to rerun this test only. - override2: FAIL (2.08s) + override2: FAIL (1.96s) files test/symb_eval/overrides/override2.good and test/symb_eval/overrides/override2.out differ; test/symb_eval/overrides/override2.out contains: - test override2/4376491c::f[0]: FAILED + test override2/23599c8e::f[0]: FAILED failures: - ---- override2/4376491c::f[0] counterexamples ---- + ---- override2/23599c8e::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/override2/' to rerun this test only. - override5: FAIL (2.29s) + override5: FAIL (1.95s) files test/symb_eval/overrides/override5.good and test/symb_eval/overrides/override5.out differ; test/symb_eval/overrides/override5.out contains: - test override5/dd33ced4::f[0]: FAILED + test override5/ff4062ad::f[0]: FAILED failures: - ---- override5/dd33ced4::f[0] counterexamples ---- + ---- override5/ff4062ad::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u64[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u64[0] [Crux] Overall status: Invalid. Use -p '/override5/' to rerun this test only. - mux: FAIL (2.13s) + mux: FAIL (2.18s) files test/symb_eval/fnptr/mux.good and test/symb_eval/fnptr/mux.out differ; test/symb_eval/fnptr/mux.out contains: - test mux/42d63c18::crux_test[0]: FAILED + test mux/c7b9f0dd::crux_test[0]: FAILED failures: - ---- mux/42d63c18::crux_test[0] counterexamples ---- + ---- mux/c7b9f0dd::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. @@ -2233,21 +2220,21 @@ FAIL (0.05s) checked_mul: FAIL test/symb_eval/num/checked_mul.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/num/checked_mul.rs) Use -p '$0=="crux-mir.crux symbolic.Output testing.checked_mul"' to rerun this test only. - checked_div: FAIL (2.04s) + checked_div: FAIL (1.98s) files test/symb_eval/num/checked_div.good and test/symb_eval/num/checked_div.out differ; test/symb_eval/num/checked_div.out contains: - test checked_div/5f132fcb::crux_test[0]: returned 65, FAILED + test checked_div/5776f60a::crux_test[0]: returned 65, FAILED failures: - ---- checked_div/5f132fcb::crux_test[0] counterexamples ---- + ---- checked_div/5776f60a::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/5f132fcb::div_signed[0] + [Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/5776f60a::div_signed[0] [Crux] attempt to compute `_3 / _4`, which would overflow [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/5f132fcb::div_signed[0] + [Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/5776f60a::div_signed[0] [Crux] attempt to divide `_3` by zero [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/5f132fcb::div_unsigned[0] + [Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/5776f60a::div_unsigned[0] [Crux] attempt to divide `_3` by zero [Crux] Overall status: Invalid. @@ -2256,291 +2243,291 @@ FAIL (0.05s) checked_add: FAIL (0.05s) test/symb_eval/num/checked_add.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/num/checked_add.rs) Use -p '/checked_add/' to rerun this test only. - test1: FAIL (2.37s) + test1: FAIL (2.01s) files test/symb_eval/scalar/test1.good and test/symb_eval/scalar/test1.out differ; test/symb_eval/scalar/test1.out contains: - test test1/51764b01::f[0]: FAILED + test test1/ac4e99fd::f[0]: FAILED failures: - ---- test1/51764b01::f[0] counterexamples ---- + ---- test1/ac4e99fd::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in int512/e9689d74::symbolic[0] - [Crux] panicking::panic, called from int512/e9689d74::symbolic[0] + [Crux] internal: error: in int512/8a4ba723::symbolic[0] + [Crux] panicking::panic, called from int512/8a4ba723::symbolic[0] [Crux] Overall status: Invalid. Use -p '/test1/' to rerun this test only. - pop: FAIL (2.14s) + pop: FAIL (2.00s) files test/symb_eval/vector/pop.good and test/symb_eval/vector/pop.out differ; test/symb_eval/vector/pop.out contains: - test pop/d7c48ae6::f[0]: FAILED + test pop/54b55322::f[0]: FAILED failures: - ---- pop/d7c48ae6::f[0] counterexamples ---- + ---- pop/54b55322::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in pop/d7c48ae6::f[0] - [Crux] Translation error in pop/d7c48ae6::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in pop/54b55322::f[0] + [Crux] Translation error in pop/54b55322::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/Output testing.pop/' to rerun this test only. - as_mut_slice: FAIL (2.18s) + as_mut_slice: FAIL (1.93s) files test/symb_eval/vector/as_mut_slice.good and test/symb_eval/vector/as_mut_slice.out differ; test/symb_eval/vector/as_mut_slice.out contains: - test as_mut_slice/1fe505fc::f[0]: FAILED + test as_mut_slice/152d9811::f[0]: FAILED failures: - ---- as_mut_slice/1fe505fc::f[0] counterexamples ---- + ---- as_mut_slice/152d9811::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in as_mut_slice/1fe505fc::f[0] - [Crux] Translation error in as_mut_slice/1fe505fc::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in as_mut_slice/152d9811::f[0] + [Crux] Translation error in as_mut_slice/152d9811::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/as_mut_slice/' to rerun this test only. - push: FAIL (2.15s) + push: FAIL (1.91s) files test/symb_eval/vector/push.good and test/symb_eval/vector/push.out differ; test/symb_eval/vector/push.out contains: - test push/35673522::f[0]: FAILED + test push/f42c220a::f[0]: FAILED failures: - ---- push/35673522::f[0] counterexamples ---- + ---- push/f42c220a::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in push/35673522::f[0] - [Crux] Translation error in push/35673522::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in push/f42c220a::f[0] + [Crux] Translation error in push/f42c220a::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/Output testing.push/' to rerun this test only. - copy_from_slice: FAIL (2.15s) + copy_from_slice: FAIL (1.92s) files test/symb_eval/vector/copy_from_slice.good and test/symb_eval/vector/copy_from_slice.out differ; test/symb_eval/vector/copy_from_slice.out contains: - test copy_from_slice/089d2837::f[0]: FAILED + test copy_from_slice/97f90fa9::f[0]: FAILED failures: - ---- copy_from_slice/089d2837::f[0] counterexamples ---- + ---- copy_from_slice/97f90fa9::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/vector/copy_from_slice.rs:7:43: 7:56: error: in copy_from_slice/089d2837::f[0] - [Crux] Translation error in copy_from_slice/089d2837::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] test/symb_eval/vector/copy_from_slice.rs:7:43: 7:56: error: in copy_from_slice/97f90fa9::f[0] + [Crux] Translation error in copy_from_slice/97f90fa9::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/copy_from_slice/' to rerun this test only. - new: FAIL (2.19s) + new: FAIL (1.90s) files test/symb_eval/vector/new.good and test/symb_eval/vector/new.out differ; test/symb_eval/vector/new.out contains: - test new/bbccbf1b::f[0]: FAILED + test new/85d35851::f[0]: FAILED failures: - ---- new/bbccbf1b::f[0] counterexamples ---- + ---- new/85d35851::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in new/bbccbf1b::f[0] - [Crux] Translation error in new/bbccbf1b::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in new/85d35851::f[0] + [Crux] Translation error in new/85d35851::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.new"' to rerun this test only. - mut: FAIL (2.10s) + mut: FAIL (1.85s) files test/symb_eval/vector/mut.good and test/symb_eval/vector/mut.out differ; test/symb_eval/vector/mut.out contains: - test mut/9fe4b5d9::f[0]: FAILED + test mut/27dffc0f::f[0]: FAILED failures: - ---- mut/9fe4b5d9::f[0] counterexamples ---- + ---- mut/27dffc0f::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in mut/9fe4b5d9::f[0] - [Crux] Translation error in mut/9fe4b5d9::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in mut/27dffc0f::f[0] + [Crux] Translation error in mut/27dffc0f::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/Output testing.mut/' to rerun this test only. - concat: FAIL (2.21s) + concat: FAIL (2.00s) files test/symb_eval/vector/concat.good and test/symb_eval/vector/concat.out differ; test/symb_eval/vector/concat.out contains: - test concat/6204baae::f[0]: FAILED + test concat/ab6becf8::f[0]: FAILED failures: - ---- concat/6204baae::f[0] counterexamples ---- + ---- concat/ab6becf8::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in concat/6204baae::f[0] - [Crux] Translation error in concat/6204baae::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in concat/ab6becf8::f[0] + [Crux] Translation error in concat/ab6becf8::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/concat/' to rerun this test only. - split_at: FAIL (2.17s) + split_at: FAIL (1.90s) files test/symb_eval/vector/split_at.good and test/symb_eval/vector/split_at.out differ; test/symb_eval/vector/split_at.out contains: - test split_at/ce3992d1::f[0]: FAILED + test split_at/ee475d7d::f[0]: FAILED failures: - ---- split_at/ce3992d1::f[0] counterexamples ---- + ---- split_at/ee475d7d::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/vector/split_at.rs:7:43: 7:56: error: in split_at/ce3992d1::f[0] - [Crux] Translation error in split_at/ce3992d1::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] test/symb_eval/vector/split_at.rs:7:43: 7:56: error: in split_at/ee475d7d::f[0] + [Crux] Translation error in split_at/ee475d7d::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/split_at/' to rerun this test only. - replicate: FAIL (2.24s) + replicate: FAIL (1.94s) files test/symb_eval/vector/replicate.good and test/symb_eval/vector/replicate.out differ; test/symb_eval/vector/replicate.out contains: - test replicate/a34bdb46::f[0]: FAILED + test replicate/6efeb740::f[0]: FAILED failures: - ---- replicate/a34bdb46::f[0] counterexamples ---- + ---- replicate/6efeb740::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in replicate/a34bdb46::f[0] - [Crux] Translation error in replicate/a34bdb46::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in replicate/6efeb740::f[0] + [Crux] Translation error in replicate/6efeb740::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/replicate/' to rerun this test only. - as_slice: FAIL (2.19s) + as_slice: FAIL (1.93s) files test/symb_eval/vector/as_slice.good and test/symb_eval/vector/as_slice.out differ; test/symb_eval/vector/as_slice.out contains: - test as_slice/190f97f7::f[0]: FAILED + test as_slice/b6304c8a::f[0]: FAILED failures: - ---- as_slice/190f97f7::f[0] counterexamples ---- + ---- as_slice/b6304c8a::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in as_slice/190f97f7::f[0] - [Crux] Translation error in as_slice/190f97f7::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in as_slice/b6304c8a::f[0] + [Crux] Translation error in as_slice/b6304c8a::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/as_slice/' to rerun this test only. - array_mut: FAIL (2.17s) + array_mut: FAIL (1.88s) files test/symb_eval/mux/array_mut.good and test/symb_eval/mux/array_mut.out differ; test/symb_eval/mux/array_mut.out contains: - test array_mut/c60a5b51::crux_test[0]: FAILED + test array_mut/75102df4::crux_test[0]: FAILED failures: - ---- array_mut/c60a5b51::crux_test[0] counterexamples ---- + ---- array_mut/75102df4::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/array_mut/' to rerun this test only. - array: FAIL (2.20s) + array: FAIL (1.99s) files test/symb_eval/mux/array.good and test/symb_eval/mux/array.out differ; test/symb_eval/mux/array.out contains: - test array/457c9e83::crux_test[0]: FAILED + test array/a00cfa61::crux_test[0]: FAILED failures: - ---- array/457c9e83::crux_test[0] counterexamples ---- + ---- array/a00cfa61::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.array"' to rerun this test only. - extend_bytes: FAIL (2.24s) + extend_bytes: FAIL (1.98s) files test/symb_eval/bytes/extend_bytes.good and test/symb_eval/bytes/extend_bytes.out differ; test/symb_eval/bytes/extend_bytes.out contains: - test extend_bytes/cbf12b9c::f[0]: FAILED + test extend_bytes/d546bdb8::f[0]: FAILED failures: - ---- extend_bytes/cbf12b9c::f[0] counterexamples ---- + ---- extend_bytes/d546bdb8::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/0b664e91::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/0b664e91::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312c069f::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/312c069f::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/extend_bytes/' to rerun this test only. - put: FAIL (2.65s) + put: FAIL (2.37s) files test/symb_eval/bytes/put.good and test/symb_eval/bytes/put.out differ; test/symb_eval/bytes/put.out contains: - test put/e40a3486::f[0]: FAILED + test put/6d809441::f[0]: FAILED failures: - ---- put/e40a3486::f[0] counterexamples ---- + ---- put/6d809441::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/0b664e91::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/0b664e91::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312c069f::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/312c069f::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.put"' to rerun this test only. - split_to: FAIL (2.59s) + split_to: FAIL (2.34s) files test/symb_eval/bytes/split_to.good and test/symb_eval/bytes/split_to.out differ; test/symb_eval/bytes/split_to.out contains: - test split_to/8021023a::f[0]: FAILED + test split_to/015724c2::f[0]: FAILED failures: - ---- split_to/8021023a::f[0] counterexamples ---- + ---- split_to/015724c2::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/0b664e91::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/0b664e91::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312c069f::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/312c069f::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/split_to/' to rerun this test only. - new: FAIL (2.13s) + new: FAIL (1.92s) files test/symb_eval/bytes/new.good and test/symb_eval/bytes/new.out differ; test/symb_eval/bytes/new.out contains: - test new/08a0233f::f[0]: FAILED + test new/91796bdc::f[0]: FAILED failures: - ---- new/08a0233f::f[0] counterexamples ---- + ---- new/91796bdc::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/0b664e91::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/0b664e91::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312c069f::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/312c069f::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.new"' to rerun this test only. - split_off: FAIL (2.60s) + split_off: FAIL (2.24s) files test/symb_eval/bytes/split_off.good and test/symb_eval/bytes/split_off.out differ; test/symb_eval/bytes/split_off.out contains: - test split_off/f72c19fe::f[0]: FAILED + test split_off/55a6890f::f[0]: FAILED failures: - ---- split_off/f72c19fe::f[0] counterexamples ---- + ---- split_off/55a6890f::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/0b664e91::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/0b664e91::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312c069f::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/312c069f::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/split_off/' to rerun this test only. - sym_len: FAIL (2.62s) + sym_len: FAIL (2.39s) files test/symb_eval/bytes/sym_len.good and test/symb_eval/bytes/sym_len.out differ; test/symb_eval/bytes/sym_len.out contains: - test sym_len/d62ca140::f[0]: FAILED + test sym_len/5c8517fc::f[0]: FAILED failures: - ---- sym_len/d62ca140::f[0] counterexamples ---- + ---- sym_len/5c8517fc::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/0b664e91::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/0b664e91::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312c069f::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/312c069f::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/sym_len/' to rerun this test only. - put_overflow: FAIL (2.54s) + put_overflow: FAIL (2.31s) files test/symb_eval/bytes/put_overflow.good and test/symb_eval/bytes/put_overflow.out differ; test/symb_eval/bytes/put_overflow.out contains: - test put_overflow/42b7073f::f[0]: FAILED + test put_overflow/382878d8::f[0]: FAILED failures: - ---- put_overflow/42b7073f::f[0] counterexamples ---- + ---- put_overflow/382878d8::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/0b664e91::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/0b664e91::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312c069f::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/312c069f::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/put_overflow/' to rerun this test only. - vec_cursor_read: FAIL (2.55s) - test/symb_eval/io/vec_cursor_read.out: withFile: user error (JSON Decoding of test/symb_eval/io/vec_cursor_read.all.mir failed: Error in $.statics[12].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) + vec_cursor_read: FAIL (2.14s) + test/symb_eval/io/vec_cursor_read.out: withFile: user error (JSON Decoding of test/symb_eval/io/vec_cursor_read.all.mir failed: Error in $.statics[6].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) Use -p '/vec_cursor_read/' to rerun this test only. - vec_write: FAIL (2.52s) - standalone use of `dyn` is not supported: TyDynamic core/1c79c753::error[0]::Error[0]::_trait56353a7b840403b4[0] + vec_write: FAIL (2.24s) + standalone use of `dyn` is not supported: TyDynamic core/95754f0e::error[0]::Error[0]::_trait56353a7b840403b4[0] CallStack (from HasCallStack): error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy tyToRepr, called at src/Mir/Trans.hs:824:12 in crucible-mir-0.1-inplace:Mir.Trans @@ -2555,332 +2542,332 @@ FAIL (0.05s) transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '/vec_write/' to rerun this test only. - write: FAIL (0.06s) + write: FAIL test/symb_eval/byteorder/write.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/write.rs) Use -p '/Output testing.write/' to rerun this test only. read: FAIL test/symb_eval/byteorder/read.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/read.rs) Use -p '/Output testing.read/' to rerun this test only. - slice_mut: FAIL (2.13s) + slice_mut: FAIL (1.93s) files test/symb_eval/array/slice_mut.good and test/symb_eval/array/slice_mut.out differ; test/symb_eval/array/slice_mut.out contains: - test slice_mut/4fd64ade::crux_test[0]: FAILED + test slice_mut/04d49b74::crux_test[0]: FAILED failures: - ---- slice_mut/4fd64ade::crux_test[0] counterexamples ---- + ---- slice_mut/04d49b74::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in slice_mut/4fd64ade::crux_test[0] - [Crux] Translation error in slice_mut/4fd64ade::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/bfc64a60::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/bfc64a60::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/bfc64a60::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) + [Crux] internal: error: in slice_mut/04d49b74::crux_test[0] + [Crux] Translation error in slice_mut/04d49b74::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/7940c2ae::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/7940c2ae::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/7940c2ae::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/slice_mut/' to rerun this test only. - basic: FAIL (2.15s) + basic: FAIL (1.91s) files test/symb_eval/array/basic.good and test/symb_eval/array/basic.out differ; test/symb_eval/array/basic.out contains: - test basic/6ffe7bac::crux_test[0]: FAILED + test basic/e501fd67::crux_test[0]: FAILED failures: - ---- basic/6ffe7bac::crux_test[0] counterexamples ---- + ---- basic/e501fd67::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in basic/6ffe7bac::crux_test[0] - [Crux] Translation error in basic/6ffe7bac::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/bfc64a60::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/bfc64a60::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/bfc64a60::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/bfc64a60::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) + [Crux] internal: error: in basic/e501fd67::crux_test[0] + [Crux] Translation error in basic/e501fd67::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/7940c2ae::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/7940c2ae::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/7940c2ae::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/7940c2ae::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/Output testing.basic/' to rerun this test only. - slice: FAIL (2.11s) + slice: FAIL (1.89s) files test/symb_eval/array/slice.good and test/symb_eval/array/slice.out differ; test/symb_eval/array/slice.out contains: - test slice/3d09fb76::crux_test[0]: FAILED + test slice/84738852::crux_test[0]: FAILED failures: - ---- slice/3d09fb76::crux_test[0] counterexamples ---- + ---- slice/84738852::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in slice/3d09fb76::crux_test[0] - [Crux] Translation error in slice/3d09fb76::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/bfc64a60::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/bfc64a60::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/bfc64a60::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) + [Crux] internal: error: in slice/84738852::crux_test[0] + [Crux] Translation error in slice/84738852::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/7940c2ae::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/7940c2ae::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/7940c2ae::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.slice"' to rerun this test only. - mux_slice: FAIL (2.11s) + mux_slice: FAIL (1.83s) files test/symb_eval/array/mux_slice.good and test/symb_eval/array/mux_slice.out differ; test/symb_eval/array/mux_slice.out contains: - test mux_slice/a488aa59::crux_test[0]: FAILED + test mux_slice/7df60048::crux_test[0]: FAILED failures: - ---- mux_slice/a488aa59::crux_test[0] counterexamples ---- + ---- mux_slice/7df60048::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in mux_slice/a488aa59::crux_test[0] - [Crux] Translation error in mux_slice/a488aa59::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/bfc64a60::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/bfc64a60::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/bfc64a60::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) + [Crux] internal: error: in mux_slice/7df60048::crux_test[0] + [Crux] Translation error in mux_slice/7df60048::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/7940c2ae::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/7940c2ae::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/7940c2ae::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/mux_slice/' to rerun this test only. - mux: FAIL (2.13s) + mux: FAIL (1.88s) files test/symb_eval/enum/mux.good and test/symb_eval/enum/mux.out differ; test/symb_eval/enum/mux.out contains: - test mux/f411b525::test[0]: FAILED + test mux/b88b2c60::test[0]: FAILED failures: - ---- mux/f411b525::test[0] counterexamples ---- + ---- mux/b88b2c60::test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.mux"' to rerun this test only. - uninit_read: FAIL (2.15s) + uninit_read: FAIL (1.88s) files test/symb_eval/alloc/uninit_read.good and test/symb_eval/alloc/uninit_read.out differ; test/symb_eval/alloc/uninit_read.out contains: - test uninit_read/1ca33845::crux_test[0]: FAILED + test uninit_read/1541c194::crux_test[0]: FAILED failures: - ---- uninit_read/1ca33845::crux_test[0] counterexamples ---- + ---- uninit_read/1541c194::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/1ca33845::crux_test[0] + [Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/1541c194::crux_test[0] [Crux] Attempted to read uninitialized vector index [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/1ca33845::crux_test[0] + [Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/1541c194::crux_test[0] [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. Use -p '/uninit_read/' to rerun this test only. - valid_read: FAIL (2.13s) + valid_read: FAIL (1.82s) files test/symb_eval/alloc/valid_read.good and test/symb_eval/alloc/valid_read.out differ; test/symb_eval/alloc/valid_read.out contains: - test valid_read/8c374ae6::crux_test[0]: returned 45, ok + test valid_read/959d5d3a::crux_test[0]: returned 45, ok [Crux] Overall status: Valid. Use -p '/valid_read/' to rerun this test only. - out_of_bounds: FAIL (2.25s) + out_of_bounds: FAIL (1.95s) files test/symb_eval/alloc/out_of_bounds.good and test/symb_eval/alloc/out_of_bounds.out differ; test/symb_eval/alloc/out_of_bounds.out contains: - test out_of_bounds/21a0dca0::crux_test[0]: FAILED + test out_of_bounds/b6cdb317::crux_test[0]: FAILED failures: - ---- out_of_bounds/21a0dca0::crux_test[0] counterexamples ---- + ---- out_of_bounds/b6cdb317::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/21a0dca0::crux_test[0] + [Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/b6cdb317::crux_test[0] [Crux] vector index out of range: the length is 10 but the index is BV 12 [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/21a0dca0::crux_test[0] + [Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/b6cdb317::crux_test[0] [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. Use -p '/out_of_bounds/' to rerun this test only. - zero_length: FAIL (2.13s) + zero_length: FAIL (1.89s) files test/symb_eval/alloc/zero_length.good and test/symb_eval/alloc/zero_length.out differ; test/symb_eval/alloc/zero_length.out contains: - test zero_length/6d5f0f4c::crux_test[0]: ok + test zero_length/fb4e0bb6::crux_test[0]: ok [Crux] Overall status: Valid. Use -p '/zero_length/' to rerun this test only. - downcast: FAIL (2.04s) + downcast: FAIL (1.77s) files test/symb_eval/any/downcast.good and test/symb_eval/any/downcast.out differ; test/symb_eval/any/downcast.out contains: - test downcast/0fb4b328::crux_test[0]: returned 1, ok + test downcast/547ed877::crux_test[0]: returned 1, ok [Crux] Overall status: Valid. Use -p '$0=="crux-mir.crux symbolic.Output testing.downcast"' to rerun this test only. - downcast_fail: FAIL (2.11s) + downcast_fail: FAIL (1.86s) files test/symb_eval/any/downcast_fail.good and test/symb_eval/any/downcast_fail.out differ; test/symb_eval/any/downcast_fail.out contains: - test downcast_fail/222b7fba::crux_test[0]: FAILED + test downcast_fail/78cdfd33::crux_test[0]: FAILED failures: - ---- downcast_fail/222b7fba::crux_test[0] counterexamples ---- + ---- downcast_fail/78cdfd33::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/any/downcast_fail.rs:10:14: 10:15: error: in downcast_fail/222b7fba::crux_test[0] + [Crux] test/symb_eval/any/downcast_fail.rs:10:14: 10:15: error: in downcast_fail/78cdfd33::crux_test[0] [Crux] failed to downcast Any as BVRepr 32 [Crux] Overall status: Invalid. Use -p '/downcast_fail/' to rerun this test only. - conditional: FAIL (0.05s) + conditional: FAIL (0.04s) test/symb_eval/any/conditional.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/any/conditional.rs) Use -p '/conditional/' to rerun this test only. - literals: FAIL (2.15s) + literals: FAIL (1.79s) files test/symb_eval/bitvector/literals.good and test/symb_eval/bitvector/literals.out differ; test/symb_eval/bitvector/literals.out contains: - test literals/a1226597::crux_test[0]: FAILED + test literals/2f1b5a7b::crux_test[0]: FAILED failures: - ---- literals/a1226597::crux_test[0] counterexamples ---- + ---- literals/2f1b5a7b::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:91:29: 91:30 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:125:1: 125:53: error: in crucible/bfc64a60::bitvector[0]::{impl#27}[0]::from[0]::_inst9ea2f935eda83dab[0] - [Crux] Translation error in crucible/bfc64a60::bitvector[0]::{impl#27}[0]::from[0]::_inst9ea2f935eda83dab[0]: BUG: invalid arguments to bv_convert: [$38: AnyRepr] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:91:29: 91:30 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:125:1: 125:53: error: in crucible/7940c2ae::bitvector[0]::{impl#27}[0]::from[0]::_inst9ea2f935eda83dab[0] + [Crux] Translation error in crucible/7940c2ae::bitvector[0]::{impl#27}[0]::from[0]::_inst9ea2f935eda83dab[0]: BUG: invalid arguments to bv_convert: [$38: AnyRepr] [Crux] Overall status: Invalid. Use -p '/literals/' to rerun this test only. - arith: FAIL (2.22s) + arith: FAIL (1.92s) files test/symb_eval/bitvector/arith.good and test/symb_eval/bitvector/arith.out differ; test/symb_eval/bitvector/arith.out contains: - test arith/f5fb78e6::crux_test[0]: FAILED + test arith/dd407607::crux_test[0]: FAILED failures: - ---- arith/f5fb78e6::crux_test[0] counterexamples ---- + ---- arith/dd407607::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u64[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u64[0] [Crux] Overall status: Invalid. Use -p '/Output testing.arith/' to rerun this test only. - leading_zeros: FAIL (2.13s) + leading_zeros: FAIL (1.85s) files test/symb_eval/bitvector/leading_zeros.good and test/symb_eval/bitvector/leading_zeros.out differ; test/symb_eval/bitvector/leading_zeros.out contains: - test leading_zeros/38d9f72d::crux_test[0]: FAILED + test leading_zeros/fde29bba::crux_test[0]: FAILED failures: - ---- leading_zeros/38d9f72d::crux_test[0] counterexamples ---- + ---- leading_zeros/fde29bba::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u64[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u64[0] [Crux] Overall status: Invalid. Use -p '/leading_zeros/' to rerun this test only. - overflowing_sub: FAIL (2.08s) + overflowing_sub: FAIL (1.83s) files test/symb_eval/bitvector/overflowing_sub.good and test/symb_eval/bitvector/overflowing_sub.out differ; test/symb_eval/bitvector/overflowing_sub.out contains: - test overflowing_sub/f396bf39::crux_test[0]: FAILED + test overflowing_sub/308b68e3::crux_test[0]: FAILED failures: - ---- overflowing_sub/f396bf39::crux_test[0] counterexamples ---- + ---- overflowing_sub/308b68e3::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u64[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u64[0] [Crux] Overall status: Invalid. Use -p '/overflowing_sub/' to rerun this test only. - cmp: FAIL (2.14s) + cmp: FAIL (1.85s) files test/symb_eval/bitvector/cmp.good and test/symb_eval/bitvector/cmp.out differ; test/symb_eval/bitvector/cmp.out contains: - test cmp/38a002d0::crux_test[0]: FAILED + test cmp/76eaaca4::crux_test[0]: FAILED failures: - ---- cmp/38a002d0::crux_test[0] counterexamples ---- + ---- cmp/76eaaca4::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::symbolic[0]::symbolic_u64[0] + [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u64[0] [Crux] Overall status: Invalid. Use -p '/Output testing.cmp/' to rerun this test only. - symbolic: FAIL (2.03s) + symbolic: FAIL (1.72s) files test/symb_eval/bitvector/symbolic.good and test/symb_eval/bitvector/symbolic.out differ; test/symb_eval/bitvector/symbolic.out contains: - test symbolic/9263414f::crux_test[0]: FAILED + test symbolic/998167ef::crux_test[0]: FAILED failures: - ---- symbolic/9263414f::crux_test[0] counterexamples ---- + ---- symbolic/998167ef::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::bitvector[0]::make_symbolic_256[0] - [Crux] panicking::panic, called from crucible/bfc64a60::bitvector[0]::make_symbolic_256[0] + [Crux] internal: error: in crucible/7940c2ae::bitvector[0]::make_symbolic_256[0] + [Crux] panicking::panic, called from crucible/7940c2ae::bitvector[0]::make_symbolic_256[0] [Crux] Overall status: Invalid. Use -p '/Output testing.symbolic/' to rerun this test only. - from_to: FAIL (1.99s) + from_to: FAIL (1.71s) files test/symb_eval/bitvector/from_to.good and test/symb_eval/bitvector/from_to.out differ; test/symb_eval/bitvector/from_to.out contains: - test from_to/a5531674::crux_test[0]: FAILED + test from_to/43b66088::crux_test[0]: FAILED failures: - ---- from_to/a5531674::crux_test[0] counterexamples ---- + ---- from_to/43b66088::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:103:29: 103:30 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:125:1: 125:53: error: in crucible/bfc64a60::bitvector[0]::{impl#21}[0]::from[0]::_inst9ea2f935eda83dab[0] - [Crux] Translation error in crucible/bfc64a60::bitvector[0]::{impl#21}[0]::from[0]::_inst9ea2f935eda83dab[0]: BUG: invalid arguments to bv_convert: [$34: BVRepr 64] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:103:29: 103:30 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:125:1: 125:53: error: in crucible/7940c2ae::bitvector[0]::{impl#21}[0]::from[0]::_inst9ea2f935eda83dab[0] + [Crux] Translation error in crucible/7940c2ae::bitvector[0]::{impl#21}[0]::from[0]::_inst9ea2f935eda83dab[0]: BUG: invalid arguments to bv_convert: [$34: BVRepr 64] [Crux] Overall status: Invalid. Use -p '/from_to/' to rerun this test only. - clone: FAIL (2.35s) + clone: FAIL (2.02s) files test/symb_eval/vec/clone.good and test/symb_eval/vec/clone.out differ; test/symb_eval/vec/clone.out contains: - test clone/f8a5c837::f[0]: FAILED + test clone/e5273a68::f[0]: FAILED failures: - ---- clone/f8a5c837::f[0] counterexamples ---- + ---- clone/e5273a68::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. Use -p '/Output testing.clone/' to rerun this test only. - sort_by_key: FAIL (3.00s) + sort_by_key: FAIL (2.58s) files test/symb_eval/vec/sort_by_key.good and test/symb_eval/vec/sort_by_key.out differ; test/symb_eval/vec/sort_by_key.out contains: - test sort_by_key/a72e2459::f[0]: FAILED + test sort_by_key/ee1d6dec::f[0]: FAILED failures: - ---- sort_by_key/a72e2459::f[0] counterexamples ---- + ---- sort_by_key/ee1d6dec::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. Use -p '/sort_by_key/' to rerun this test only. - into_iter: FAIL (2.90s) + into_iter: FAIL (2.55s) files test/symb_eval/vec/into_iter.good and test/symb_eval/vec/into_iter.out differ; test/symb_eval/vec/into_iter.out contains: - test into_iter/35485644::f[0]: FAILED + test into_iter/f0e581e8::f[0]: FAILED failures: - ---- into_iter/35485644::f[0] counterexamples ---- + ---- into_iter/f0e581e8::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. Use -p '/into_iter/' to rerun this test only. - macro: FAIL (2.14s) + macro: FAIL (1.88s) files test/symb_eval/vec/macro.good and test/symb_eval/vec/macro.out differ; test/symb_eval/vec/macro.out contains: - test macro/f7b50ef5::f[0]: FAILED + test macro/4069544e::f[0]: FAILED failures: - ---- macro/f7b50ef5::f[0] counterexamples ---- + ---- macro/4069544e::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/1c79c753::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/1c79c753::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] + [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] [Crux] Overall status: Invalid. Use -p '/Output testing.macro/' to rerun this test only. - construct: FAIL (2.25s) + construct: FAIL (1.91s) files test/symb_eval/sym_bytes/construct.good and test/symb_eval/sym_bytes/construct.out differ; test/symb_eval/sym_bytes/construct.out contains: - test construct/2811c902::crux_test[0]: FAILED + test construct/4a1d0528::crux_test[0]: FAILED failures: - ---- construct/2811c902::crux_test[0] counterexamples ---- + ---- construct/4a1d0528::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::sym_bytes[0]::{impl#0}[0]::zeroed[0] - [Crux] Translation error in crucible/bfc64a60::sym_bytes[0]::{impl#0}[0]::zeroed[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 8) to AnyRepr (TyAdt crucible/bfc64a60::array[0]::Array[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::array[0]::Array[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/bfc64a60::array[0]::Array[0]::_adtaddce72e1232152c[0] crucible/bfc64a60::array[0]::Array[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in crucible/7940c2ae::sym_bytes[0]::{impl#0}[0]::zeroed[0] + [Crux] Translation error in crucible/7940c2ae::sym_bytes[0]::{impl#0}[0]::zeroed[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::array[0]::Array[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::array[0]::Array[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::array[0]::Array[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::array[0]::Array[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/construct/' to rerun this test only. - deserialize: FAIL (2.07s) + deserialize: FAIL (1.82s) files test/symb_eval/sym_bytes/deserialize.good and test/symb_eval/sym_bytes/deserialize.out differ; test/symb_eval/sym_bytes/deserialize.out contains: - test deserialize/098471bc::crux_test[0]: FAILED + test deserialize/34a68134::crux_test[0]: FAILED failures: - ---- deserialize/098471bc::crux_test[0] counterexamples ---- + ---- deserialize/34a68134::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/bfc64a60::array[0]::symbolic[0]::_instaddce72e1232152c[0] - [Crux] panicking::panic_fmt, called from crucible/bfc64a60::array[0]::symbolic[0]::_instaddce72e1232152c[0] + [Crux] internal: error: in crucible/7940c2ae::array[0]::symbolic[0]::_instaddce72e1232152c[0] + [Crux] panicking::panic_fmt, called from crucible/7940c2ae::array[0]::symbolic[0]::_instaddce72e1232152c[0] [Crux] Overall status: Invalid. @@ -2901,10 +2888,10 @@ FAIL (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_dual/4ed2e432::crux_test1[0]/report_data.js" and "test/coverage/out/coverage_dual/30f9fe77::crux_test1[0]/report_data.js" do not match', src/main.rs:825:13 +thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_dual/2d5bca87::crux_test2[0]/report_data.js" and "test/coverage/out/coverage_dual/30f9fe77::crux_test1[0]/report_data.js" do not match', src/main.rs:825:13 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (2.07s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_dual/30f9fe77::crux_test1[0]/report_data.js" "test/coverage/out/coverage_dual/4ed2e432::crux_test1[0]/report_data.js" "test/coverage/out/coverage_dual/30f9fe77::crux_test2[0]/report_data.js" "test/coverage/out/coverage_dual/4ed2e432::crux_test2[0]/report_data.js" (exit 101): failed +FAIL (1.81s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_dual/30f9fe77::crux_test1[0]/report_data.js" "test/coverage/out/coverage_dual/2d5bca87::crux_test2[0]/report_data.js" "test/coverage/out/coverage_dual/4ed2e432::crux_test1[0]/report_data.js" "test/coverage/out/coverage_dual/2d5bca87::crux_test1[0]/report_data.js" "test/coverage/out/coverage_dual/30f9fe77::crux_test2[0]/report_data.js" "test/coverage/out/coverage_dual/4ed2e432::crux_test2[0]/report_data.js" (exit 101): failed Use -p '/coverage_dual/' to rerun this test only. coverage: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 @@ -2920,10 +2907,10 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage/cb3943dc::crux_test[0]/report_data.js" and "test/coverage/out/coverage/0fe46bcc::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 +thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage/c30e4ff4::crux_test[0]/report_data.js" and "test/coverage/out/coverage/0fe46bcc::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (2.17s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage/0fe46bcc::crux_test[0]/report_data.js" "test/coverage/out/coverage/cb3943dc::crux_test[0]/report_data.js" (exit 101): failed +FAIL (2.04s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage/0fe46bcc::crux_test[0]/report_data.js" "test/coverage/out/coverage/c30e4ff4::crux_test[0]/report_data.js" "test/coverage/out/coverage/cb3943dc::crux_test[0]/report_data.js" (exit 101): failed Use -p '$0=="crux-mir.crux coverage.Output testing.coverage"' to rerun this test only. coverage_try: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 @@ -2939,10 +2926,10 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_try/a9dc15f2::crux_test[0]/report_data.js" and "test/coverage/out/coverage_try/d0e6f11f::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 +thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_try/853a6496::crux_test[0]/report_data.js" and "test/coverage/out/coverage_try/d0e6f11f::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (2.18s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_try/d0e6f11f::crux_test[0]/report_data.js" "test/coverage/out/coverage_try/a9dc15f2::crux_test[0]/report_data.js" (exit 101): failed +FAIL (1.94s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_try/d0e6f11f::crux_test[0]/report_data.js" "test/coverage/out/coverage_try/853a6496::crux_test[0]/report_data.js" "test/coverage/out/coverage_try/a9dc15f2::crux_test[0]/report_data.js" (exit 101): failed Use -p '/coverage_try/' to rerun this test only. coverage_cond: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 @@ -2958,10 +2945,10 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_cond/3f380757::crux_test[0]/report_data.js" and "test/coverage/out/coverage_cond/3dfc8004::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 +thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_cond/2c579738::crux_test[0]/report_data.js" and "test/coverage/out/coverage_cond/3dfc8004::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (2.07s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_cond/3dfc8004::crux_test[0]/report_data.js" "test/coverage/out/coverage_cond/3f380757::crux_test[0]/report_data.js" (exit 101): failed +FAIL (1.79s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_cond/3dfc8004::crux_test[0]/report_data.js" "test/coverage/out/coverage_cond/2c579738::crux_test[0]/report_data.js" "test/coverage/out/coverage_cond/3f380757::crux_test[0]/report_data.js" (exit 101): failed Use -p '/coverage_cond/' to rerun this test only. coverage_loop: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 @@ -2977,10 +2964,10 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_loop/81f6cac8::crux_test[0]/report_data.js" and "test/coverage/out/coverage_loop/bf94e0f1::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 +thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_loop/bf94e0f1::crux_test[0]/report_data.js" and "test/coverage/out/coverage_loop/fff32a45::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (2.09s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_loop/bf94e0f1::crux_test[0]/report_data.js" "test/coverage/out/coverage_loop/81f6cac8::crux_test[0]/report_data.js" (exit 101): failed +FAIL (1.87s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_loop/fff32a45::crux_test[0]/report_data.js" "test/coverage/out/coverage_loop/bf94e0f1::crux_test[0]/report_data.js" "test/coverage/out/coverage_loop/81f6cac8::crux_test[0]/report_data.js" (exit 101): failed Use -p '/coverage_loop/' to rerun this test only. coverage_shortcircuit: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 @@ -2996,10 +2983,10 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_shortcircuit/f90f762e::crux_test[0]/report_data.js" and "test/coverage/out/coverage_shortcircuit/5762c1e1::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 +thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_shortcircuit/e95f8b07::crux_test[0]/report_data.js" and "test/coverage/out/coverage_shortcircuit/5762c1e1::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (2.21s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_shortcircuit/5762c1e1::crux_test[0]/report_data.js" "test/coverage/out/coverage_shortcircuit/f90f762e::crux_test[0]/report_data.js" (exit 101): failed +FAIL (1.90s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_shortcircuit/5762c1e1::crux_test[0]/report_data.js" "test/coverage/out/coverage_shortcircuit/e95f8b07::crux_test[0]/report_data.js" "test/coverage/out/coverage_shortcircuit/f90f762e::crux_test[0]/report_data.js" (exit 101): failed Use -p '/coverage_shortcircuit/' to rerun this test only. coverage_mono: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 @@ -3017,8 +3004,8 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_mono/94654c01::crux_test[0]/report_data.js" and "test/coverage/out/coverage_mono/564877e0::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (2.05s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_mono/564877e0::crux_test[0]/report_data.js" "test/coverage/out/coverage_mono/94654c01::crux_test[0]/report_data.js" (exit 101): failed +FAIL (1.81s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_mono/564877e0::crux_test[0]/report_data.js" "test/coverage/out/coverage_mono/94654c01::crux_test[0]/report_data.js" "test/coverage/out/coverage_mono/d7f56dd4::crux_test[0]/report_data.js" (exit 101): failed Use -p '/coverage_mono/' to rerun this test only. coverage_match: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 @@ -3034,10 +3021,10 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_match/2b40efd1::crux_test[0]/report_data.js" and "test/coverage/out/coverage_match/e0a88aee::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 +thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_match/63e02e94::crux_test[0]/report_data.js" and "test/coverage/out/coverage_match/e0a88aee::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (2.11s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_match/e0a88aee::crux_test[0]/report_data.js" "test/coverage/out/coverage_match/2b40efd1::crux_test[0]/report_data.js" (exit 101): failed +FAIL (1.92s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_match/e0a88aee::crux_test[0]/report_data.js" "test/coverage/out/coverage_match/63e02e94::crux_test[0]/report_data.js" "test/coverage/out/coverage_match/2b40efd1::crux_test[0]/report_data.js" (exit 101): failed Use -p '/coverage_match/' to rerun this test only. coverage_macro: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 @@ -3055,8 +3042,8 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_macro/ff936904::crux_test[0]/report_data.js" and "test/coverage/out/coverage_macro/e538cb6e::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (2.10s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_macro/e538cb6e::crux_test[0]/report_data.js" "test/coverage/out/coverage_macro/ff936904::crux_test[0]/report_data.js" (exit 101): failed +FAIL (1.92s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_macro/e538cb6e::crux_test[0]/report_data.js" "test/coverage/out/coverage_macro/ff936904::crux_test[0]/report_data.js" "test/coverage/out/coverage_macro/54bd500f::crux_test[0]/report_data.js" (exit 101): failed Use -p '/coverage_macro/' to rerun this test only. -151 out of 339 tests failed (738.26s) +150 out of 339 tests failed (656.22s) From 0b5105b09515c7a1075c46ef9b747fa753673d28 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 17 May 2023 14:57:42 -0400 Subject: [PATCH 038/114] Tighten up treatment of atomic intrinsics --- crucible-mir/src/Mir/TransCustom.hs | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/crucible-mir/src/Mir/TransCustom.hs b/crucible-mir/src/Mir/TransCustom.hs index 0151a00c0..f1be610c7 100644 --- a/crucible-mir/src/Mir/TransCustom.hs +++ b/crucible-mir/src/Mir/TransCustom.hs @@ -1499,12 +1499,17 @@ makeAtomicRMW :: MirGenerator h s ret (R.Expr MIR s (C.BVType w))) -> [(ExplodedDefId, CustomRHS)] makeAtomicRMW name rmw = - makeAtomicIntrinsics (Text.pack name) ["acq", "rel", "acqrel", "relaxed", "seqcst"] $ + makeAtomicIntrinsics (Text.pack name) allAtomicOrderings $ atomic_rmw_impl name rmw +-- These names are taken from +-- https://github.com/rust-lang/rust/blob/22b4c688956de0925f7a10a79cb0e1ca35f55425/library/core/src/sync/atomic.rs#L3039-L3043 +allAtomicOrderings :: [Text] +allAtomicOrderings = ["acquire", "release", "acqrel", "relaxed", "seqcst"] + atomic_funcs = - makeAtomicIntrinsics "store" ["rel", "relaxed", "seqcst"] atomic_store_impl ++ - makeAtomicIntrinsics "load" ["acq", "relaxed", "seqcst"] atomic_load_impl ++ + makeAtomicIntrinsics "store" storeVariants atomic_store_impl ++ + makeAtomicIntrinsics "load" loadVariants atomic_load_impl ++ makeAtomicIntrinsics "cxchg" compareExchangeVariants atomic_cxchg_impl ++ makeAtomicIntrinsics "cxchgweak" compareExchangeVariants atomic_cxchg_impl ++ makeAtomicIntrinsics "fence" fenceVariants atomic_fence_impl ++ @@ -1524,9 +1529,18 @@ atomic_funcs = makeAtomicRMW "umin" $ \w old val -> return $ R.App $ E.BVUMin w old val ] where - compareExchangeVariants = ["acq", "rel", "acqrel", "relaxed", - "acq_failrelaxed", "acqrel_failrelaxed", "failrelaxed", "failacq"] - fenceVariants = ["acq", "rel", "acqrel", "seqcst"] + -- See https://github.com/rust-lang/rust/blob/22b4c688956de0925f7a10a79cb0e1ca35f55425/library/core/src/sync/atomic.rs#L3008-L3012 + storeVariants = ["release", "relaxed", "seqcst"] + -- See https://github.com/rust-lang/rust/blob/22b4c688956de0925f7a10a79cb0e1ca35f55425/library/core/src/sync/atomic.rs#L3023-L3027 + loadVariants = ["acquire", "relaxed", "seqcst"] + -- See https://github.com/rust-lang/rust/blob/22b4c688956de0925f7a10a79cb0e1ca35f55425/library/core/src/sync/atomic.rs#L3095-L3111 + compareExchangeVariants = [ success <> "_" <> failure + | success <- allAtomicOrderings + , failure <- allAtomicOrderings + , failure `notElem` ["acqrel", "release"] + ] + -- See https://github.com/rust-lang/rust/blob/22b4c688956de0925f7a10a79cb0e1ca35f55425/library/core/src/sync/atomic.rs#L3366-L3370 + fenceVariants = ["acquire", "release", "acqrel", "seqcst"] -------------------------------------------------------------------------------------------------------------------------- From b54a15e3db09fd09c7cfec4f97bed31fae5b1582 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 17 May 2023 14:58:03 -0400 Subject: [PATCH 039/114] boxed.rs: Use crucible's allocator --- crux-mir/lib/alloc/src/boxed.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crux-mir/lib/alloc/src/boxed.rs b/crux-mir/lib/alloc/src/boxed.rs index a563b2587..31170b24b 100644 --- a/crux-mir/lib/alloc/src/boxed.rs +++ b/crux-mir/lib/alloc/src/boxed.rs @@ -184,6 +184,8 @@ use crate::vec::Vec; #[unstable(feature = "thin_box", issue = "92791")] pub use thin::ThinBox; +use crucible; + mod thin; /// A pointer type that uniquely owns a heap allocation of type `T`. @@ -215,8 +217,11 @@ impl Box { #[stable(feature = "rust1", since = "1.0.0")] #[must_use] pub fn new(x: T) -> Self { - #[rustc_box] - Box::new(x) + unsafe { + let ptr = crucible::alloc::allocate::(1); + ptr.write(x); + Box::from_raw(ptr) + } } /// Constructs a new box with uninitialized contents. From 26de04409aa09fd0febd7f534c2ab193bad75a5f Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 17 May 2023 16:59:54 -0400 Subject: [PATCH 040/114] lib/Patches.md: Document "boxed.rs: Use crucible's allocator" --- crux-mir/lib/Patches.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index 24e384e42..3f03db46f 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -20,6 +20,10 @@ identify all of the code that was changed in each patch. We also make sure to avoid the `Layout::array` function, which has a particularly tricky use of `transmute` that we do not currently support. +* `boxed.rs`: Use crucible's allocator (last applied: May 17, 2023) + + Same reasoning as above. + * Reimplement `core::fmt` using `crucible::any::Any` (last applied: May 2, 2023) TODO: Describe why this is necessary From 7274762ea25059181a35988e2125faeb220a44fa Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 17 May 2023 17:02:02 -0400 Subject: [PATCH 041/114] Don't deallocate in box_free and drop This reapplies the changes originally from commit 2424b443de967886e99a2d67b20cda67b0583d86 --- crux-mir/lib/Patches.md | 4 ++++ crux-mir/lib/alloc/src/alloc.rs | 7 +------ crux-mir/lib/alloc/src/sync.rs | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index 3f03db46f..66ae01d27 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -59,3 +59,7 @@ identify all of the code that was changed in each patch. * Implement `str::as_bytes` via `crucible_identity_transmute` (last applied: May 16, 2023) This is necessary to avoid a gnarly use of `transmute`. + +* Don't deallocate in `box_free` and `drop` (last applied: May 17, 2023) + + Crucible doesn't support a `deallocate` operation. diff --git a/crux-mir/lib/alloc/src/alloc.rs b/crux-mir/lib/alloc/src/alloc.rs index 3a797bd5e..65cfb79ac 100644 --- a/crux-mir/lib/alloc/src/alloc.rs +++ b/crux-mir/lib/alloc/src/alloc.rs @@ -341,12 +341,7 @@ pub(crate) const unsafe fn box_free, alloc: A, ) { - unsafe { - let size = size_of_val(ptr.as_ref()); - let align = min_align_of_val(ptr.as_ref()); - let layout = Layout::from_size_align_unchecked(size, align); - alloc.deallocate(From::from(ptr.cast()), layout) - } + // Crucible: currently we don't implement `deallocate`, so this is a no-op. } // # Allocation error handler diff --git a/crux-mir/lib/alloc/src/sync.rs b/crux-mir/lib/alloc/src/sync.rs index bab7f5f53..18244d11c 100644 --- a/crux-mir/lib/alloc/src/sync.rs +++ b/crux-mir/lib/alloc/src/sync.rs @@ -2247,7 +2247,7 @@ unsafe impl<#[may_dangle] T: ?Sized> Drop for Weak { if inner.weak.fetch_sub(1, Release) == 1 { acquire!(inner.weak); - unsafe { Global.deallocate(self.ptr.cast(), Layout::for_value_raw(self.ptr.as_ptr())) } + // Crucible: we should deallocate `self.ptr` here, but we don't support `deallocate` yet. } } } From e59402c331b7855883cfa6b3c905e9f8e2737c41 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 18 May 2023 09:48:10 -0400 Subject: [PATCH 042/114] Make {sub,mul}_with_overflow use {extern} --- crucible-mir/src/Mir/TransCustom.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crucible-mir/src/Mir/TransCustom.hs b/crucible-mir/src/Mir/TransCustom.hs index f1be610c7..db95d1696 100644 --- a/crucible-mir/src/Mir/TransCustom.hs +++ b/crucible-mir/src/Mir/TransCustom.hs @@ -688,13 +688,13 @@ add_with_overflow = sub_with_overflow :: (ExplodedDefId, CustomRHS) sub_with_overflow = - ( ["core","intrinsics", "", "sub_with_overflow"] + ( ["core","intrinsics", "{extern}", "sub_with_overflow"] , makeArithWithOverflow "sub_with_overflow" Nothing Sub ) mul_with_overflow :: (ExplodedDefId, CustomRHS) mul_with_overflow = - ( ["core","intrinsics", "", "mul_with_overflow"] + ( ["core","intrinsics", "{extern}", "mul_with_overflow"] , makeArithWithOverflow "mul_with_overflow" Nothing Mul ) From b9fec04c75fb3658fd18d019af6093715a9f82ab Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 18 May 2023 10:32:14 -0400 Subject: [PATCH 043/114] Reapply "reimplement from_{le,be}_bytes" Originally from commit 13ba0e9dc9aedb0881ee9cb3ad08cac89ee42368 --- crux-mir/lib/Patches.md | 5 ++++ crux-mir/lib/core/src/num/uint_macros.rs | 38 ++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index 66ae01d27..421b767f2 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -63,3 +63,8 @@ identify all of the code that was changed in each patch. * Don't deallocate in `box_free` and `drop` (last applied: May 17, 2023) Crucible doesn't support a `deallocate` operation. + +* Reimplement `from_{le,be}_bytes` (last applied: May 18, 2023) + + The actual implementations of these functions involve gnarly uses of + `transmute`. diff --git a/crux-mir/lib/core/src/num/uint_macros.rs b/crux-mir/lib/core/src/num/uint_macros.rs index 1c97c4686..23af576a2 100644 --- a/crux-mir/lib/core/src/num/uint_macros.rs +++ b/crux-mir/lib/core/src/num/uint_macros.rs @@ -2342,7 +2342,24 @@ macro_rules! uint_impl { #[must_use] #[inline] pub const fn from_be_bytes(bytes: [u8; mem::size_of::()]) -> Self { - Self::from_be(Self::from_ne_bytes(bytes)) + let mut x = 0; + x |= (bytes[15 % ($BITS / 8)] as Self) << ( 0 % $BITS); + x |= (bytes[14 % ($BITS / 8)] as Self) << ( 8 % $BITS); + x |= (bytes[13 % ($BITS / 8)] as Self) << ( 16 % $BITS); + x |= (bytes[12 % ($BITS / 8)] as Self) << ( 24 % $BITS); + x |= (bytes[11 % ($BITS / 8)] as Self) << ( 32 % $BITS); + x |= (bytes[10 % ($BITS / 8)] as Self) << ( 40 % $BITS); + x |= (bytes[ 9 % ($BITS / 8)] as Self) << ( 48 % $BITS); + x |= (bytes[ 8 % ($BITS / 8)] as Self) << ( 56 % $BITS); + x |= (bytes[ 7 % ($BITS / 8)] as Self) << ( 64 % $BITS); + x |= (bytes[ 6 % ($BITS / 8)] as Self) << ( 72 % $BITS); + x |= (bytes[ 5 % ($BITS / 8)] as Self) << ( 80 % $BITS); + x |= (bytes[ 4 % ($BITS / 8)] as Self) << ( 88 % $BITS); + x |= (bytes[ 3 % ($BITS / 8)] as Self) << ( 96 % $BITS); + x |= (bytes[ 2 % ($BITS / 8)] as Self) << (104 % $BITS); + x |= (bytes[ 1 % ($BITS / 8)] as Self) << (112 % $BITS); + x |= (bytes[ 0 % ($BITS / 8)] as Self) << (120 % $BITS); + x } /// Create a native endian integer value from its representation @@ -2371,7 +2388,24 @@ macro_rules! uint_impl { #[must_use] #[inline] pub const fn from_le_bytes(bytes: [u8; mem::size_of::()]) -> Self { - Self::from_le(Self::from_ne_bytes(bytes)) + let mut x = 0; + x |= (bytes[ 0 % ($BITS / 8)] as Self) << ( 0 % $BITS); + x |= (bytes[ 1 % ($BITS / 8)] as Self) << ( 8 % $BITS); + x |= (bytes[ 2 % ($BITS / 8)] as Self) << ( 16 % $BITS); + x |= (bytes[ 3 % ($BITS / 8)] as Self) << ( 24 % $BITS); + x |= (bytes[ 4 % ($BITS / 8)] as Self) << ( 32 % $BITS); + x |= (bytes[ 5 % ($BITS / 8)] as Self) << ( 40 % $BITS); + x |= (bytes[ 6 % ($BITS / 8)] as Self) << ( 48 % $BITS); + x |= (bytes[ 7 % ($BITS / 8)] as Self) << ( 56 % $BITS); + x |= (bytes[ 8 % ($BITS / 8)] as Self) << ( 64 % $BITS); + x |= (bytes[ 9 % ($BITS / 8)] as Self) << ( 72 % $BITS); + x |= (bytes[10 % ($BITS / 8)] as Self) << ( 80 % $BITS); + x |= (bytes[11 % ($BITS / 8)] as Self) << ( 88 % $BITS); + x |= (bytes[12 % ($BITS / 8)] as Self) << ( 96 % $BITS); + x |= (bytes[13 % ($BITS / 8)] as Self) << (104 % $BITS); + x |= (bytes[14 % ($BITS / 8)] as Self) << (112 % $BITS); + x |= (bytes[15 % ($BITS / 8)] as Self) << (120 % $BITS); + x } /// Create a native endian integer value from its memory representation From 7db42e78e196eedbf74669f73cc4766fbfdc00e0 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 18 May 2023 10:36:01 -0400 Subject: [PATCH 044/114] Reapply "reimplement to_{le,be}_bytes" Originally from commit c804deb73c4894fb17f936ee85ff3c8163ce7c7c --- crux-mir/lib/Patches.md | 4 +++ crux-mir/lib/core/src/num/int_macros.rs | 38 ++++++++++++++++++++++-- crux-mir/lib/core/src/num/uint_macros.rs | 38 ++++++++++++++++++++++-- 3 files changed, 76 insertions(+), 4 deletions(-) diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index 421b767f2..d0bb4dd53 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -68,3 +68,7 @@ identify all of the code that was changed in each patch. The actual implementations of these functions involve gnarly uses of `transmute`. + +* Reimplement `to_{le,be}_bytes` (last applied: May 18, 2023) + + Same reasoning as above. diff --git a/crux-mir/lib/core/src/num/int_macros.rs b/crux-mir/lib/core/src/num/int_macros.rs index 2cae98b8e..fbb35024b 100644 --- a/crux-mir/lib/core/src/num/int_macros.rs +++ b/crux-mir/lib/core/src/num/int_macros.rs @@ -2633,7 +2633,24 @@ macro_rules! int_impl { without modifying the original"] #[inline] pub const fn to_be_bytes(self) -> [u8; mem::size_of::()] { - self.to_be().to_ne_bytes() + let mut buf = [0; $BITS / 8]; + buf[15 % ($BITS / 8)] = (self >> ( 0 % $BITS)) as u8; + buf[14 % ($BITS / 8)] = (self >> ( 8 % $BITS)) as u8; + buf[13 % ($BITS / 8)] = (self >> ( 16 % $BITS)) as u8; + buf[12 % ($BITS / 8)] = (self >> ( 24 % $BITS)) as u8; + buf[11 % ($BITS / 8)] = (self >> ( 32 % $BITS)) as u8; + buf[10 % ($BITS / 8)] = (self >> ( 40 % $BITS)) as u8; + buf[ 9 % ($BITS / 8)] = (self >> ( 48 % $BITS)) as u8; + buf[ 8 % ($BITS / 8)] = (self >> ( 56 % $BITS)) as u8; + buf[ 7 % ($BITS / 8)] = (self >> ( 64 % $BITS)) as u8; + buf[ 6 % ($BITS / 8)] = (self >> ( 72 % $BITS)) as u8; + buf[ 5 % ($BITS / 8)] = (self >> ( 80 % $BITS)) as u8; + buf[ 4 % ($BITS / 8)] = (self >> ( 88 % $BITS)) as u8; + buf[ 3 % ($BITS / 8)] = (self >> ( 96 % $BITS)) as u8; + buf[ 2 % ($BITS / 8)] = (self >> (104 % $BITS)) as u8; + buf[ 1 % ($BITS / 8)] = (self >> (112 % $BITS)) as u8; + buf[ 0 % ($BITS / 8)] = (self >> (120 % $BITS)) as u8; + buf } /// Return the memory representation of this integer as a byte array in @@ -2653,7 +2670,24 @@ macro_rules! int_impl { without modifying the original"] #[inline] pub const fn to_le_bytes(self) -> [u8; mem::size_of::()] { - self.to_le().to_ne_bytes() + let mut buf = [0; $BITS / 8]; + buf[ 0 % ($BITS / 8)] = (self >> ( 0 % $BITS)) as u8; + buf[ 1 % ($BITS / 8)] = (self >> ( 8 % $BITS)) as u8; + buf[ 2 % ($BITS / 8)] = (self >> ( 16 % $BITS)) as u8; + buf[ 3 % ($BITS / 8)] = (self >> ( 24 % $BITS)) as u8; + buf[ 4 % ($BITS / 8)] = (self >> ( 32 % $BITS)) as u8; + buf[ 5 % ($BITS / 8)] = (self >> ( 40 % $BITS)) as u8; + buf[ 6 % ($BITS / 8)] = (self >> ( 48 % $BITS)) as u8; + buf[ 7 % ($BITS / 8)] = (self >> ( 56 % $BITS)) as u8; + buf[ 8 % ($BITS / 8)] = (self >> ( 64 % $BITS)) as u8; + buf[ 9 % ($BITS / 8)] = (self >> ( 72 % $BITS)) as u8; + buf[10 % ($BITS / 8)] = (self >> ( 80 % $BITS)) as u8; + buf[11 % ($BITS / 8)] = (self >> ( 88 % $BITS)) as u8; + buf[12 % ($BITS / 8)] = (self >> ( 96 % $BITS)) as u8; + buf[13 % ($BITS / 8)] = (self >> (104 % $BITS)) as u8; + buf[14 % ($BITS / 8)] = (self >> (112 % $BITS)) as u8; + buf[15 % ($BITS / 8)] = (self >> (120 % $BITS)) as u8; + buf } /// Return the memory representation of this integer as a byte array in diff --git a/crux-mir/lib/core/src/num/uint_macros.rs b/crux-mir/lib/core/src/num/uint_macros.rs index 23af576a2..77d559f8f 100644 --- a/crux-mir/lib/core/src/num/uint_macros.rs +++ b/crux-mir/lib/core/src/num/uint_macros.rs @@ -2255,7 +2255,24 @@ macro_rules! uint_impl { without modifying the original"] #[inline] pub const fn to_be_bytes(self) -> [u8; mem::size_of::()] { - self.to_be().to_ne_bytes() + let mut buf = [0; $BITS / 8]; + buf[15 % ($BITS / 8)] = (self >> ( 0 % $BITS)) as u8; + buf[14 % ($BITS / 8)] = (self >> ( 8 % $BITS)) as u8; + buf[13 % ($BITS / 8)] = (self >> ( 16 % $BITS)) as u8; + buf[12 % ($BITS / 8)] = (self >> ( 24 % $BITS)) as u8; + buf[11 % ($BITS / 8)] = (self >> ( 32 % $BITS)) as u8; + buf[10 % ($BITS / 8)] = (self >> ( 40 % $BITS)) as u8; + buf[ 9 % ($BITS / 8)] = (self >> ( 48 % $BITS)) as u8; + buf[ 8 % ($BITS / 8)] = (self >> ( 56 % $BITS)) as u8; + buf[ 7 % ($BITS / 8)] = (self >> ( 64 % $BITS)) as u8; + buf[ 6 % ($BITS / 8)] = (self >> ( 72 % $BITS)) as u8; + buf[ 5 % ($BITS / 8)] = (self >> ( 80 % $BITS)) as u8; + buf[ 4 % ($BITS / 8)] = (self >> ( 88 % $BITS)) as u8; + buf[ 3 % ($BITS / 8)] = (self >> ( 96 % $BITS)) as u8; + buf[ 2 % ($BITS / 8)] = (self >> (104 % $BITS)) as u8; + buf[ 1 % ($BITS / 8)] = (self >> (112 % $BITS)) as u8; + buf[ 0 % ($BITS / 8)] = (self >> (120 % $BITS)) as u8; + buf } /// Return the memory representation of this integer as a byte array in @@ -2275,7 +2292,24 @@ macro_rules! uint_impl { without modifying the original"] #[inline] pub const fn to_le_bytes(self) -> [u8; mem::size_of::()] { - self.to_le().to_ne_bytes() + let mut buf = [0; $BITS / 8]; + buf[ 0 % ($BITS / 8)] = (self >> ( 0 % $BITS)) as u8; + buf[ 1 % ($BITS / 8)] = (self >> ( 8 % $BITS)) as u8; + buf[ 2 % ($BITS / 8)] = (self >> ( 16 % $BITS)) as u8; + buf[ 3 % ($BITS / 8)] = (self >> ( 24 % $BITS)) as u8; + buf[ 4 % ($BITS / 8)] = (self >> ( 32 % $BITS)) as u8; + buf[ 5 % ($BITS / 8)] = (self >> ( 40 % $BITS)) as u8; + buf[ 6 % ($BITS / 8)] = (self >> ( 48 % $BITS)) as u8; + buf[ 7 % ($BITS / 8)] = (self >> ( 56 % $BITS)) as u8; + buf[ 8 % ($BITS / 8)] = (self >> ( 64 % $BITS)) as u8; + buf[ 9 % ($BITS / 8)] = (self >> ( 72 % $BITS)) as u8; + buf[10 % ($BITS / 8)] = (self >> ( 80 % $BITS)) as u8; + buf[11 % ($BITS / 8)] = (self >> ( 88 % $BITS)) as u8; + buf[12 % ($BITS / 8)] = (self >> ( 96 % $BITS)) as u8; + buf[13 % ($BITS / 8)] = (self >> (104 % $BITS)) as u8; + buf[14 % ($BITS / 8)] = (self >> (112 % $BITS)) as u8; + buf[15 % ($BITS / 8)] = (self >> (120 % $BITS)) as u8; + buf } /// Return the memory representation of this integer as a byte array in From 7b2f9249b3eb96516aa22ebe84e18a37790c99df Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 18 May 2023 10:46:33 -0400 Subject: [PATCH 045/114] Use {extern} in transmute custom op --- crucible-mir/src/Mir/TransCustom.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crucible-mir/src/Mir/TransCustom.hs b/crucible-mir/src/Mir/TransCustom.hs index db95d1696..78e4d8fa7 100644 --- a/crucible-mir/src/Mir/TransCustom.hs +++ b/crucible-mir/src/Mir/TransCustom.hs @@ -973,7 +973,7 @@ mem_crucible_identity_transmute = (["core","mem", "crucible_identity_transmute"] ) mem_transmute :: (ExplodedDefId, CustomRHS) -mem_transmute = (["core", "intrinsics", "", "transmute"], +mem_transmute = (["core", "intrinsics", "{extern}", "transmute"], \ substs -> case substs of Substs [tyT, tyU] -> Just $ CustomOp $ \ _ ops -> case ops of [e@(MirExp argTy _)] -> do From c2aa0838c600355fb534c044355f5301675dfa4f Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 18 May 2023 13:52:33 -0400 Subject: [PATCH 046/114] raw_vec.rs: Make sure to always set capacity --- crux-mir/lib/alloc/src/raw_vec.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crux-mir/lib/alloc/src/raw_vec.rs b/crux-mir/lib/alloc/src/raw_vec.rs index 6b23f8fa3..f7c09a9e9 100644 --- a/crux-mir/lib/alloc/src/raw_vec.rs +++ b/crux-mir/lib/alloc/src/raw_vec.rs @@ -375,8 +375,8 @@ impl RawVec { } } else { crucible::alloc::reallocate::(self.ptr.as_ptr(), cap); - self.cap = cap; } + self.cap = cap; Ok(()) } @@ -398,8 +398,8 @@ impl RawVec { } } else { crucible::alloc::reallocate::(self.ptr.as_ptr(), cap); - self.cap = cap; } + self.cap = cap; Ok(()) } @@ -413,8 +413,8 @@ impl RawVec { } } else { crucible::alloc::reallocate::(self.ptr.as_ptr(), cap); - self.cap = cap; } + self.cap = cap; Ok(()) } } From 00435375debb2c0ea05f798121c3edf56dd3cbdc Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 18 May 2023 14:11:09 -0400 Subject: [PATCH 047/114] Expunge uses of `rustc_box` and the `box` keyword These will desugar to direct uses of `exchange_malloc`, which is too complicated for Crucible to handle. Let's use `Box::new` instead. --- crux-mir/lib/Patches.md | 2 +- crux-mir/lib/alloc/src/boxed.rs | 5 +---- crux-mir/lib/alloc/src/macros.rs | 1 - crux-mir/lib/alloc/src/vec/mod.rs | 1 - crux-mir/test/conc_eval/box/struct.rs | 2 +- 5 files changed, 3 insertions(+), 8 deletions(-) diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index d0bb4dd53..0b11ca2e7 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -20,7 +20,7 @@ identify all of the code that was changed in each patch. We also make sure to avoid the `Layout::array` function, which has a particularly tricky use of `transmute` that we do not currently support. -* `boxed.rs`: Use crucible's allocator (last applied: May 17, 2023) +* Use crucible's allocator instead of `rustc_box` (last applied: May 17, 2023) Same reasoning as above. diff --git a/crux-mir/lib/alloc/src/boxed.rs b/crux-mir/lib/alloc/src/boxed.rs index 31170b24b..a06c2ba8d 100644 --- a/crux-mir/lib/alloc/src/boxed.rs +++ b/crux-mir/lib/alloc/src/boxed.rs @@ -288,8 +288,7 @@ impl Box { #[must_use] #[inline(always)] pub fn pin(x: T) -> Pin> { - (#[rustc_box] - Box::new(x)) + Box::new(x) .into() } @@ -1248,7 +1247,6 @@ unsafe impl<#[may_dangle] T: ?Sized, A: Allocator> Drop for Box { impl Default for Box { /// Creates a `Box`, with the `Default` value for T. fn default() -> Self { - #[rustc_box] Box::new(T::default()) } } @@ -1621,7 +1619,6 @@ impl From<[T; N]> for Box<[T]> { /// println!("{boxed:?}"); /// ``` fn from(array: [T; N]) -> Box<[T]> { - #[rustc_box] Box::new(array) } } diff --git a/crux-mir/lib/alloc/src/macros.rs b/crux-mir/lib/alloc/src/macros.rs index 5198bf297..7f2ebb227 100644 --- a/crux-mir/lib/alloc/src/macros.rs +++ b/crux-mir/lib/alloc/src/macros.rs @@ -48,7 +48,6 @@ macro_rules! vec { ); ($($x:expr),+ $(,)?) => ( $crate::__rust_force_expr!(<[_]>::into_vec( - #[rustc_box] $crate::boxed::Box::new([$($x),+]) )) ); diff --git a/crux-mir/lib/alloc/src/vec/mod.rs b/crux-mir/lib/alloc/src/vec/mod.rs index 36b0b3c9e..3e1f1dd50 100644 --- a/crux-mir/lib/alloc/src/vec/mod.rs +++ b/crux-mir/lib/alloc/src/vec/mod.rs @@ -3161,7 +3161,6 @@ impl From<[T; N]> for Vec { #[cfg(not(test))] fn from(s: [T; N]) -> Vec { <[T]>::into_vec( - #[rustc_box] Box::new(s), ) } diff --git a/crux-mir/test/conc_eval/box/struct.rs b/crux-mir/test/conc_eval/box/struct.rs index 56942c0d9..789ef3dd3 100644 --- a/crux-mir/test/conc_eval/box/struct.rs +++ b/crux-mir/test/conc_eval/box/struct.rs @@ -4,7 +4,7 @@ struct Test(i32); #[cfg_attr(crux, crux::test)] fn crux_test() -> i32 { - let x = box Test(1); + let x = Box::new(Test(1)); x.0 } From a7988e8198cc02e6e444bd8534272ac1a04d93ad Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 18 May 2023 14:49:01 -0400 Subject: [PATCH 048/114] Update test_output.log --- crux-mir/test_output.log | 2657 ++++++++++++++++++-------------------- 1 file changed, 1244 insertions(+), 1413 deletions(-) diff --git a/crux-mir/test_output.log b/crux-mir/test_output.log index 3802ed705..ee03c9a0c 100644 --- a/crux-mir/test_output.log +++ b/crux-mir/test_output.log @@ -2,293 +2,280 @@ crux-mir crux concrete refs - promoted_imm: OK (1.89s) + promoted_imm: OK (1.97s) Compiling and running oracle program (0.14s) - Oracle output: 0 (1.74s) + Oracle output: 0 (1.83s) Crux output: 0 - mut_raw: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 123 (1.72s) + mut_raw: OK (1.98s) + Compiling and running oracle program (0.15s) + Oracle output: 123 (1.83s) Crux output: 123 - mut_nested: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.72s) + mut_nested: OK (2.05s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.89s) Crux output: () - fn_ptr_mut: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.72s) - Crux output: 1 - never: OK (1.87s) + fn_ptr_mut: OK (2.00s) Compiling and running oracle program (0.15s) - Oracle output: 1 (1.72s) + Oracle output: 1 (1.85s) Crux output: 1 - mut_ref: OK (1.86s) + never: OK (1.99s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (1.84s) + Crux output: 1 + mut_ref: OK (2.03s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (1.87s) + Crux output: 1 + temp: OK (1.94s) Compiling and running oracle program (0.15s) - Oracle output: 1 (1.71s) + Oracle output: 1 (1.80s) Crux output: 1 - temp: OK (1.86s) + mut_arg: OK (1.87s) Compiling and running oracle program (0.14s) Oracle output: 1 (1.72s) Crux output: 1 - mut_arg: OK (1.88s) + fn_ptr: OK (1.92s) Compiling and running oracle program (0.14s) - Oracle output: 1 (1.74s) - Crux output: 1 - fn_ptr: OK (1.99s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.83s) + Oracle output: 1 (1.77s) Crux output: 1 - promoted_mut: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.74s) + promoted_mut: OK (2.01s) + Compiling and running oracle program (0.18s) + Oracle output: 0 (1.84s) Crux output: 0 - never_mut: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.71s) + never_mut: OK (1.96s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.81s) Crux output: 1 - mut_tuple_field: OK (1.86s) + mut_tuple_field: OK (2.02s) Compiling and running oracle program (0.15s) - Oracle output: () (1.72s) + Oracle output: () (1.86s) Crux output: () - imm_raw: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 123 (1.71s) + imm_raw: OK (1.99s) + Compiling and running oracle program (0.16s) + Oracle output: 123 (1.83s) Crux output: 123 - static_mut: OK (1.87s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.72s) + static_mut: OK (2.03s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (1.87s) Crux output: 2 - imm_ref: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 123 (1.72s) - Crux output: 123 - imm_arg: OK (1.86s) + imm_ref: OK (1.93s) Compiling and running oracle program (0.15s) - Oracle output: 1 (1.71s) + Oracle output: 123 (1.78s) + Crux output: 123 + imm_arg: OK (1.99s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (1.83s) Crux output: 1 hash_map - insert_multi: OK (2.20s) - Compiling and running oracle program (0.18s) - Oracle output: 100 (2.02s) + insert_multi: OK (2.32s) + Compiling and running oracle program (0.19s) + Oracle output: 100 (2.13s) Crux output: 100 - insert_remove: FAIL (2.63s) - Compiling and running oracle program (0.18s) - Oracle output: 12 (2.45s) - Crux output: - failures: - - ---- insert_remove/00c7ee06::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in alloc/fed0e039::vec[0]::{impl#1}[0]::set_len[0]::_inst1b10a593fa289fda[0] - [Crux] panicking::panic, called from alloc/fed0e039::vec[0]::{impl#1}[0]::set_len[0]::_inst1b10a593fa289fda[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/insert_remove/' to rerun this test only. - insert_iter: OK (2.22s) - Compiling and running oracle program (0.18s) - Oracle output: [11, 12] (2.04s) + insert_remove: OK (2.89s) + Compiling and running oracle program (0.21s) + Oracle output: 12 (2.68s) + Crux output: 12 + insert_iter: OK (2.31s) + Compiling and running oracle program (0.19s) + Oracle output: [11, 12] (2.12s) Crux output: [11, 12] - insert_get: OK (2.20s) - Compiling and running oracle program (0.18s) - Oracle output: (11, 12) (2.02s) + insert_get: OK (2.32s) + Compiling and running oracle program (0.19s) + Oracle output: (11, 12) (2.14s) Crux output: (11, 12) traits - params: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 25 (1.72s) + params: OK (2.10s) + Compiling and running oracle program (0.15s) + Oracle output: 25 (1.95s) Crux output: 25 - generic3: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.72s) + generic3: OK (1.99s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.83s) Crux output: () - static_three: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.72s) + static_three: OK (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.76s) Crux output: 2 - generic1: OK (1.86s) + generic1: OK (1.93s) Compiling and running oracle program (0.15s) - Oracle output: () (1.72s) + Oracle output: () (1.78s) Crux output: () - subtrait: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 73 (1.71s) + subtrait: OK (2.03s) + Compiling and running oracle program (0.16s) + Oracle output: 73 (1.88s) Crux output: 73 - gen_trait: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: -69 (1.71s) + gen_trait: OK (1.96s) + Compiling and running oracle program (0.15s) + Oracle output: -69 (1.81s) Crux output: -69 - dynamic_branch: FAIL (0.17s) - Compiling and running oracle program (0.14s) + dynamic_branch: FAIL (0.18s) + Compiling and running oracle program (0.15s) Oracle output: 1 (0.03s) user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_branch.rs) Use -p '/dynamic_branch/' to rerun this test only. - static_two: OK (1.85s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.71s) + static_two: OK (2.00s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (1.85s) Crux output: 1 - tyfam5: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.70s) + tyfam5: OK (1.95s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.80s) Crux output: () - generic2: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.73s) + generic2: OK (2.03s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.87s) Crux output: () - assoc3: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.72s) + assoc3: OK (2.03s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.88s) Crux output: () - tyfam4: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.72s) + tyfam4: OK (1.99s) + Compiling and running oracle program (0.16s) + Oracle output: 0 (1.82s) Crux output: 0 - static_self: OK (1.84s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.70s) + static_self: OK (1.95s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.80s) Crux output: 42 - conv: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.71s) + conv: OK (1.98s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.83s) Crux output: 1 - dynamic_poly: FAIL (0.17s) - Compiling and running oracle program (0.14s) + dynamic_poly: FAIL (0.19s) + Compiling and running oracle program (0.16s) Oracle output: 3 (0.03s) user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_poly.rs) Use -p '/dynamic_poly/' to rerun this test only. - dynamic_simple: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.75s) + dynamic_simple: OK (1.88s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (1.73s) Crux output: 1 - gen_trait_poly: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 12 (1.72s) + gen_trait_poly: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: 12 (1.75s) Crux output: 12 - default: OK (1.84s) - Compiling and running oracle program (0.14s) - Oracle output: 12 (1.70s) + default: OK (1.98s) + Compiling and running oracle program (0.15s) + Oracle output: 12 (1.83s) Crux output: 12 - assoc1: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.76s) + assoc1: OK (2.03s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.88s) Crux output: () - subtrait2: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: 64 (1.76s) + subtrait2: OK (1.99s) + Compiling and running oracle program (0.15s) + Oracle output: 64 (1.84s) Crux output: 64 - bounds1: OK (1.92s) + bounds1: OK (2.05s) Compiling and running oracle program (0.15s) - Oracle output: () (1.78s) + Oracle output: () (1.90s) Crux output: () - bounds3: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.73s) + bounds3: OK (1.96s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.80s) Crux output: () - static_eq: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: true (1.71s) + static_eq: OK (1.97s) + Compiling and running oracle program (0.15s) + Oracle output: true (1.82s) Crux output: true - tyfam3: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 23 (1.71s) + tyfam3: OK (1.95s) + Compiling and running oracle program (0.15s) + Oracle output: 23 (1.80s) Crux output: 23 - dict_med: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.71s) + dict_med: OK (1.97s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.82s) Crux output: 42 - basics1: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.75s) + basics1: OK (1.97s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.83s) Crux output: () - assoc2: OK (1.91s) + assoc2: OK (1.93s) Compiling and running oracle program (0.15s) - Oracle output: () (1.76s) + Oracle output: () (1.78s) Crux output: () - dict_polymem: OK (1.95s) - Compiling and running oracle program (0.14s) - Oracle output: 4 (1.81s) + dict_polymem: OK (1.94s) + Compiling and running oracle program (0.15s) + Oracle output: 4 (1.80s) Crux output: 4 - bounds2: OK (1.99s) + bounds2: OK (1.96s) Compiling and running oracle program (0.15s) - Oracle output: () (1.84s) + Oracle output: () (1.81s) Crux output: () - intoiter: OK (1.99s) - Compiling and running oracle program (0.16s) - Oracle output: 0 (1.83s) + intoiter: OK (1.95s) + Compiling and running oracle program (0.15s) + Oracle output: 0 (1.80s) Crux output: 0 - static: OK (1.91s) - Compiling and running oracle program (0.16s) - Oracle output: 42 (1.75s) + static: OK (1.93s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.78s) Crux output: 42 - dynamic_two: FAIL (0.17s) - Compiling and running oracle program (0.14s) + dynamic_two: FAIL (0.18s) + Compiling and running oracle program (0.15s) Oracle output: 1 (0.03s) user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_two.rs) Use -p '/dynamic_two/' to rerun this test only. - bounds4: OK (1.91s) + bounds4: OK (1.96s) Compiling and running oracle program (0.15s) - Oracle output: () (1.76s) + Oracle output: () (1.81s) Crux output: () - dict_poly: OK (2.01s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.85s) + dict_poly: OK (1.96s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.81s) Crux output: 1 - dict_simple: OK (1.98s) + dict_simple: OK (1.99s) Compiling and running oracle program (0.15s) - Oracle output: 32 (1.83s) + Oracle output: 32 (1.84s) Crux output: 32 - tyfam: OK (1.96s) + tyfam: OK (2.00s) Compiling and running oracle program (0.16s) - Oracle output: 23 (1.80s) + Oracle output: 23 (1.84s) Crux output: 23 - bounds5: OK (1.93s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.78s) + bounds5: OK (2.12s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.97s) Crux output: () - tyfam2: OK (1.92s) - Compiling and running oracle program (0.15s) - Oracle output: 23 (1.77s) + tyfam2: OK (2.02s) + Compiling and running oracle program (0.16s) + Oracle output: 23 (1.86s) Crux output: 23 - dynamic_med: FAIL (0.17s) - Compiling and running oracle program (0.14s) + dynamic_med: FAIL (0.18s) + Compiling and running oracle program (0.15s) Oracle output: 1 (0.03s) user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_med.rs) Use -p '/dynamic_med/' to rerun this test only. intTest - test0039: OK (1.91s) - Compiling and running oracle program (0.15s) - Oracle output: 7 (1.76s) + test0039: OK (2.07s) + Compiling and running oracle program (0.16s) + Oracle output: 7 (1.91s) Crux output: 7 - test0038: OK (1.99s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.84s) + test0038: OK (2.10s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.94s) Crux output: () tuple - clone_rec: OK (1.92s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.77s) + clone_rec: OK (2.08s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.92s) Crux output: () - clone: OK (1.91s) + clone: OK (1.97s) Compiling and running oracle program (0.15s) - Oracle output: () (1.76s) + Oracle output: () (1.82s) Crux output: () - clone_from: FAIL (1.91s) + clone_from: FAIL (1.94s) Compiling and running oracle program (0.15s) - Oracle output: () (1.76s) + Oracle output: () (1.79s) Crux output: failures: - ---- clone_from/ca9e7ae4::f[0] counterexamples ---- + ---- clone_from/f3686bfc::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/clone.rs:136:17: 136:31: error: in core/95754f0e::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0] - [Crux] Translation error in core/95754f0e::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0]: don't know how to generate CloneShim for unknown method core/95754f0e::clone[0]::Clone[0]::clone[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/clone.rs:136:17: 136:31: error: in core/feeb1386::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0] + [Crux] Translation error in core/feeb1386::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0]: don't know how to generate CloneShim for unknown method core/feeb1386::clone[0]::Clone[0]::clone[0] [Crux] Overall status: Invalid. @@ -296,16 +283,16 @@ crux-mir crux doesn't match oracle Use -p '/clone_from/' to rerun this test only. - clone_struct: FAIL (1.91s) + clone_struct: FAIL (1.96s) Compiling and running oracle program (0.15s) - Oracle output: () (1.76s) + Oracle output: () (1.81s) Crux output: failures: - ---- clone_struct/b6c12504::f[0] counterexamples ---- + ---- clone_struct/5c02078b::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/tuple/clone_struct.rs:8:13: 8:22: error: in clone_struct/b6c12504::f[0] - [Crux] Translation error in clone_struct/b6c12504::f[0]: don't know how to generate CloneShim for unknown method core/95754f0e::clone[0]::Clone[0]::clone[0] + [Crux] test/conc_eval/tuple/clone_struct.rs:8:13: 8:22: error: in clone_struct/5c02078b::f[0] + [Crux] Translation error in clone_struct/5c02078b::f[0]: don't know how to generate CloneShim for unknown method core/feeb1386::clone[0]::Clone[0]::clone[0] [Crux] Overall status: Invalid. @@ -314,49 +301,49 @@ crux-mir Use -p '/clone_struct/' to rerun this test only. iter - for: OK (1.90s) - Compiling and running oracle program (0.15s) - Oracle output: 47 (1.75s) + for: OK (2.07s) + Compiling and running oracle program (0.16s) + Oracle output: 47 (1.92s) Crux output: 47 - sum: OK (1.93s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.78s) + sum: OK (2.12s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.96s) Crux output: () - filter_chain: OK (2.05s) - Compiling and running oracle program (0.17s) - Oracle output: 0 (1.88s) + filter_chain: OK (2.21s) + Compiling and running oracle program (0.18s) + Oracle output: 0 (2.03s) Crux output: 0 - from_fn: OK (1.96s) + from_fn: OK (1.94s) Compiling and running oracle program (0.15s) - Oracle output: () (1.81s) + Oracle output: () (1.80s) Crux output: () - peek: OK (1.99s) + peek: OK (2.00s) Compiling and running oracle program (0.15s) - Oracle output: 3 (1.84s) + Oracle output: 3 (1.85s) Crux output: 3 - zip: OK (1.98s) + zip: OK (1.97s) Compiling and running oracle program (0.15s) Oracle output: () (1.82s) Crux output: () - loop: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.74s) + loop: OK (1.94s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.79s) Crux output: 2 - cloned: OK (1.95s) + cloned: OK (1.94s) Compiling and running oracle program (0.15s) - Oracle output: 6 (1.80s) + Oracle output: 6 (1.79s) Crux output: 6 str - format_hex: FAIL (2.62s) + format_hex: FAIL (2.65s) Compiling and running oracle program (0.15s) - Oracle output: true (2.46s) + Oracle output: true (2.49s) Crux output: failures: - ---- format_hex/ab76c909::crux_test[0] counterexamples ---- + ---- format_hex/92b3ccb6::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/num/uint_macros.rs:1677:74: 1677:89 !lib/core/src/num/mod.rs:965:5: 970:27: error: in core/95754f0e::num[0]::{impl#12}[0]::overflowing_mul[0] - [Crux] Translation error in core/95754f0e::num[0]::{impl#12}[0]::overflowing_mul[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::mul_with_overflow[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/feeb1386::fmt[0]::write[0] + [Crux] Translation error in core/feeb1386::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/feeb1386::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. @@ -364,16 +351,16 @@ crux-mir crux doesn't match oracle Use -p '/format_hex/' to rerun this test only. - format_int: FAIL (2.61s) - Compiling and running oracle program (0.15s) - Oracle output: true (2.46s) + format_int: FAIL (2.67s) + Compiling and running oracle program (0.16s) + Oracle output: true (2.51s) Crux output: failures: - ---- format_int/f1c217ad::crux_test[0] counterexamples ---- + ---- format_int/c0e1710f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/num/uint_macros.rs:1677:74: 1677:89 !lib/core/src/num/mod.rs:965:5: 970:27: error: in core/95754f0e::num[0]::{impl#12}[0]::overflowing_mul[0] - [Crux] Translation error in core/95754f0e::num[0]::{impl#12}[0]::overflowing_mul[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::mul_with_overflow[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/feeb1386::fmt[0]::write[0] + [Crux] Translation error in core/feeb1386::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/feeb1386::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. @@ -381,16 +368,16 @@ crux-mir crux doesn't match oracle Use -p '/format_int/' to rerun this test only. - format_struct: FAIL (2.74s) + format_struct: FAIL (2.65s) Compiling and running oracle program (0.16s) - Oracle output: true (2.58s) + Oracle output: true (2.49s) Crux output: failures: - ---- format_struct/f8c3e1dc::crux_test[0] counterexamples ---- + ---- format_struct/4d89e395::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/95754f0e::fmt[0]::write[0] - [Crux] Translation error in core/95754f0e::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/95754f0e::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/feeb1386::fmt[0]::write[0] + [Crux] Translation error in core/feeb1386::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/feeb1386::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. @@ -398,16 +385,16 @@ crux-mir crux doesn't match oracle Use -p '/format_struct/' to rerun this test only. - format: FAIL (2.85s) - Compiling and running oracle program (0.18s) - Oracle output: true (2.67s) + format: FAIL (2.61s) + Compiling and running oracle program (0.15s) + Oracle output: true (2.46s) Crux output: failures: - ---- format/a6953732::crux_test[0] counterexamples ---- + ---- format/78eb465c::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/num/uint_macros.rs:1677:74: 1677:89 !lib/core/src/num/mod.rs:965:5: 970:27: error: in core/95754f0e::num[0]::{impl#12}[0]::overflowing_mul[0] - [Crux] Translation error in core/95754f0e::num[0]::{impl#12}[0]::overflowing_mul[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::mul_with_overflow[0]::_inst11a9dd2a3cfdbd73[0] + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/feeb1386::fmt[0]::write[0] + [Crux] Translation error in core/feeb1386::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/feeb1386::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. @@ -415,16 +402,16 @@ crux-mir crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..str.format"' to rerun this test only. - to_owned: FAIL (2.26s) - Compiling and running oracle program (0.16s) - Oracle output: true (2.11s) + to_owned: FAIL (1.99s) + Compiling and running oracle program (0.15s) + Oracle output: true (1.84s) Crux output: failures: - ---- to_owned/9b9b876a::crux_test[0] counterexamples ---- + ---- to_owned/0e7e1577::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -432,16 +419,16 @@ crux-mir crux doesn't match oracle Use -p '/to_owned/' to rerun this test only. - string_push: FAIL (2.66s) - Compiling and running oracle program (0.17s) - Oracle output: true (2.50s) + string_push: FAIL (2.54s) + Compiling and running oracle program (0.16s) + Oracle output: true (2.39s) Crux output: failures: - ---- string_push/82be0c4f::crux_test[0] counterexamples ---- + ---- string_push/320b1149::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -449,16 +436,16 @@ crux-mir crux doesn't match oracle Use -p '/string_push/' to rerun this test only. - format_array: FAIL (2.99s) - Compiling and running oracle program (0.17s) - Oracle output: true (2.81s) + format_array: FAIL (2.69s) + Compiling and running oracle program (0.16s) + Oracle output: true (2.53s) Crux output: failures: - ---- format_array/73672e87::crux_test[0] counterexamples ---- + ---- format_array/8db6fb19::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/95754f0e::fmt[0]::write[0] - [Crux] Translation error in core/95754f0e::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/95754f0e::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/feeb1386::fmt[0]::write[0] + [Crux] Translation error in core/feeb1386::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/feeb1386::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. @@ -467,182 +454,117 @@ crux-mir Use -p '/format_array/' to rerun this test only. sync - mutex: FAIL (2.65s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (2.49s) - user error (JSON Decoding of test/conc_eval/sync/mutex.all.mir failed: Error in $.statics[6].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) + mutex: FAIL (2.48s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (2.33s) + user error (JSON Decoding of test/conc_eval/sync/mutex.all.mir failed: Error in $.statics[171].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) Use -p '$0=="crux-mir.crux concrete..sync.mutex"' to rerun this test only. - arc_cell: FAIL (2.14s) - Compiling and running oracle program (0.16s) - Oracle output: 4 (1.98s) - Crux output: - failures: - - ---- arc_cell/7c4dad53::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/arc_cell/' to rerun this test only. - arc: FAIL (2.15s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (2.00s) - Crux output: - failures: - - ---- arc/bb69dff5::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '$0=="crux-mir.crux concrete..sync.arc"' to rerun this test only. - atomic_add: FAIL (2.15s) - Compiling and running oracle program (0.16s) - Oracle output: 6 (1.99s) - Crux output: - failures: - - ---- atomic_add/39afb391::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/sync/atomic.rs:3058:61: 3058:64: error: in core/95754f0e::sync[0]::atomic[0]::atomic_add[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/95754f0e::sync[0]::atomic[0]::atomic_add[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::atomic_xadd_release[0]::_inst1e2825177cd3b608[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/atomic_add/' to rerun this test only. - rwlock: FAIL (2.70s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (2.53s) - user error (JSON Decoding of test/conc_eval/sync/rwlock.all.mir failed: Error in $.statics[6].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) + arc_cell: OK (2.15s) + Compiling and running oracle program (0.16s) + Oracle output: 4 (1.99s) + Crux output: 4 + arc: OK (2.07s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (1.91s) + Crux output: 1 + atomic_add: OK (2.08s) + Compiling and running oracle program (0.15s) + Oracle output: 6 (1.93s) + Crux output: 6 + rwlock: FAIL (2.68s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (2.52s) + user error (JSON Decoding of test/conc_eval/sync/rwlock.all.mir failed: Error in $.statics[172].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) Use -p '$0=="crux-mir.crux concrete..sync.rwlock"' to rerun this test only. - atomic_cxchg: FAIL (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: 6 (2.00s) - Crux output: - failures: - - ---- atomic_cxchg/d34db99d::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/sync/atomic.rs:3095:86: 3095:89: error: in core/95754f0e::sync[0]::atomic[0]::atomic_compare_exchange[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/95754f0e::sync[0]::atomic[0]::atomic_compare_exchange[0]::_inst1e2825177cd3b608[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::atomic_cxchg_relaxed_relaxed[0]::_inst1e2825177cd3b608[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/atomic_cxchg/' to rerun this test only. - arc_clone: FAIL (2.21s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (2.04s) - Crux output: - failures: - - ---- arc_clone/494b2b01::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/arc_clone/' to rerun this test only. - atomic_swap: OK (2.14s) + atomic_cxchg: OK (2.27s) + Compiling and running oracle program (0.17s) + Oracle output: 6 (2.10s) + Crux output: 6 + arc_clone: OK (2.24s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (2.07s) + Crux output: 2 + atomic_swap: OK (2.12s) Compiling and running oracle program (0.16s) - Oracle output: (2, 1) (1.98s) + Oracle output: (2, 1) (1.97s) Crux output: (2, 1) - rwlock_multi: FAIL (2.81s) + rwlock_multi: FAIL (2.68s) Compiling and running oracle program (0.17s) - Oracle output: 3 (2.63s) - user error (JSON Decoding of test/conc_eval/sync/rwlock_multi.all.mir failed: Error in $.statics[6].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) + Oracle output: 3 (2.51s) + user error (JSON Decoding of test/conc_eval/sync/rwlock_multi.all.mir failed: Error in $.statics[172].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) Use -p '/rwlock_multi/' to rerun this test only. - atomic_fence: OK (2.14s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (1.98s) + atomic_fence: OK (2.07s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.92s) Crux output: 2 mutex_multi: FAIL (2.81s) - Compiling and running oracle program (0.16s) - Oracle output: 3 (2.65s) - user error (JSON Decoding of test/conc_eval/sync/mutex_multi.all.mir failed: Error in $.statics[6].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) + Compiling and running oracle program (0.18s) + Oracle output: 3 (2.63s) + user error (JSON Decoding of test/conc_eval/sync/mutex_multi.all.mir failed: Error in $.statics[171].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) Use -p '/mutex_multi/' to rerun this test only. consts - struct_val: OK (2.05s) + struct_val: OK (2.13s) Compiling and running oracle program (0.16s) - Oracle output: Foo { x: true } (1.90s) + Oracle output: Foo { x: true } (1.97s) Crux output: Foo { x: true } - struct_unit: OK (2.04s) + struct_unit: OK (2.11s) Compiling and running oracle program (0.16s) - Oracle output: Err(Foo { x: () }) (1.88s) + Oracle output: Err(Foo { x: () }) (1.95s) Crux output: Err(Foo { x: () }) - local_key: FAIL (2.02s) + local_key: FAIL (2.04s) Compiling and running oracle program (0.16s) - Oracle output: 1 (1.86s) + Oracle output: 1 (1.88s) user error (JSON Decoding of test/conc_eval/consts/local_key.all.mir failed: Error in $.statics[0].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) Use -p '/local_key/' to rerun this test only. - fn_def: FAIL (2.03s) + fn_def: FAIL (2.07s) Compiling and running oracle program (0.16s) - Oracle output: 1 (1.87s) + Oracle output: 1 (1.91s) user error (JSON Decoding of test/conc_eval/consts/fn_def.all.mir failed: Error in $.fns[0].body.blocks[0].block.terminator.func.data.rendered: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) Use -p '/fn_def/' to rerun this test only. - enum_val: OK (2.06s) - Compiling and running oracle program (0.16s) - Oracle output: None (1.91s) + enum_val: OK (2.00s) + Compiling and running oracle program (0.17s) + Oracle output: None (1.82s) Crux output: None crypto - add: OK (2.15s) - Compiling and running oracle program (0.17s) - Oracle output: true (1.98s) + add: OK (2.06s) + Compiling and running oracle program (0.16s) + Oracle output: true (1.90s) Crux output: true - add_noL: OK (2.08s) - Compiling and running oracle program (0.17s) - Oracle output: false (1.91s) + add_noL: OK (2.16s) + Compiling and running oracle program (0.16s) + Oracle output: false (2.00s) Crux output: false cell - cell: OK (2.11s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (1.95s) - Crux output: 2 - ref_cell: OK (2.65s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (2.50s) - Crux output: 2 - ref_cell2: OK (2.70s) + cell: OK (2.10s) Compiling and running oracle program (0.17s) - Oracle output: 2 (2.53s) + Oracle output: 2 (1.94s) Crux output: 2 - fnptr - call: OK (1.99s) + ref_cell: OK (2.54s) Compiling and running oracle program (0.15s) - Oracle output: 2 (1.84s) + Oracle output: 2 (2.38s) Crux output: 2 - field: OK (1.98s) + ref_cell2: OK (2.67s) Compiling and running oracle program (0.15s) - Oracle output: 2 (1.83s) + Oracle output: 2 (2.52s) Crux output: 2 - make: OK (1.99s) + fnptr + call: OK (2.16s) Compiling and running oracle program (0.16s) - Oracle output: 0 (1.84s) + Oracle output: 2 (2.00s) + Crux output: 2 + field: OK (2.07s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (1.91s) + Crux output: 2 + make: OK (2.05s) + Compiling and running oracle program (0.16s) + Oracle output: 0 (1.89s) Crux output: 0 custom: rustc compilation failed for custom error output: @@ -658,70 +580,44 @@ error: aborting due to previous error For more information about this error, try `rustc --explain E0432`. -FAIL (expected: taking address of an overridden function) (0.05s) - Compiling and running oracle program (0.05s) +FAIL (expected: taking address of an overridden function) (0.06s) + Compiling and running oracle program (0.06s) test/Test.hs:107: failed to compile and run (expected failure) num - overflow: FAIL (2.03s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.87s) - Crux output: - failures: - - ---- overflow/6b711274::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/uint_macros.rs:1578:74: 1578:77 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:305:5: 306:27: error: in core/95754f0e::num[0]::{impl#7}[0]::overflowing_sub[0] - [Crux] Translation error in core/95754f0e::num[0]::{impl#7}[0]::overflowing_sub[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::sub_with_overflow[0]::_instaddce72e1232152c[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/num.overflow/' to rerun this test only. - from_bytes: FAIL (2.02s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.86s) - Crux output: - failures: - - ---- from_bytes/848b3106::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/uint_macros.rs:2417:37: 2417:42 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:890:5: 891:49: error: in core/95754f0e::num[0]::{impl#8}[0]::from_ne_bytes[0] - [Crux] Translation error in core/95754f0e::num[0]::{impl#8}[0]::from_ne_bytes[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_inst850c595be4adc72f[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/from_bytes/' to rerun this test only. - saturate: OK (2.56s) - Compiling and running oracle program (0.15s) - Oracle output: () (2.41s) + overflow: OK (2.17s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.00s) + Crux output: () + from_bytes: OK (2.12s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.96s) + Crux output: () + saturate: OK (2.71s) + Compiling and running oracle program (0.16s) + Oracle output: () (2.56s) Crux output: () prim - bool: OK (1.96s) - Compiling and running oracle program (0.15s) - Oracle output: false (1.81s) + bool: OK (2.10s) + Compiling and running oracle program (0.16s) + Oracle output: false (1.94s) Crux output: false - shift3: OK (1.95s) + shift3: OK (2.02s) Compiling and running oracle program (0.15s) - Oracle output: -9223372036854775808 (1.80s) + Oracle output: -9223372036854775808 (1.87s) Crux output: -9223372036854775808 - div: OK (1.95s) + div: OK (2.01s) Compiling and running oracle program (0.15s) - Oracle output: 4 (1.80s) + Oracle output: 4 (1.85s) Crux output: 4 - ge: OK (1.96s) + ge: OK (2.02s) Compiling and running oracle program (0.15s) - Oracle output: true (1.82s) + Oracle output: true (1.87s) Crux output: true - add1: OK (1.97s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.82s) + add1: OK (2.10s) + Compiling and running oracle program (0.18s) + Oracle output: 2 (1.93s) Crux output: 2 shift_exceeding: rustc compilation failed for shift_exceeding error output: @@ -758,175 +654,97 @@ FAIL (expected: Should panic, but doesn't) (0.06s) test/Test.hs:107: failed to compile and run (expected failure) - char_from_u32: FAIL (1.98s) - Compiling and running oracle program (0.15s) - Oracle output: 'A' (1.83s) - Crux output: - failures: - - ---- char_from_u32/d563b221::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/char/convert.rs:215:31: 215:32: error: in core/95754f0e::char[0]::convert[0]::char_try_from_u32[0] - [Crux] Translation error in core/95754f0e::char[0]::convert[0]::char_try_from_u32[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instdc3291f36c45d6de[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/char_from_u32/' to rerun this test only. - ffs: OK (2.02s) + char_from_u32: OK (2.05s) + Compiling and running oracle program (0.15s) + Oracle output: 'A' (1.89s) + Crux output: 'A' + ffs: OK (2.09s) Compiling and running oracle program (0.16s) - Oracle output: true (1.86s) + Oracle output: true (1.93s) Crux output: true - mut_arg: OK (1.96s) + mut_arg: OK (2.03s) Compiling and running oracle program (0.15s) - Oracle output: 1 (1.81s) + Oracle output: 1 (1.88s) Crux output: 1 - litstring: OK (1.96s) + litstring: OK (2.00s) Compiling and running oracle program (0.15s) - Oracle output: true (1.81s) + Oracle output: true (1.85s) Crux output: true - lit: OK (1.98s) + lit: OK (2.04s) Compiling and running oracle program (0.16s) - Oracle output: false (1.82s) + Oracle output: false (1.88s) Crux output: false - litbstring: OK (1.96s) + litbstring: OK (2.00s) Compiling and running oracle program (0.15s) - Oracle output: true (1.82s) + Oracle output: true (1.85s) Crux output: true - shift1: OK (1.97s) + shift1: OK (1.99s) Compiling and running oracle program (0.15s) - Oracle output: 2 (1.82s) + Oracle output: 2 (1.84s) Crux output: 2 - shift2: OK (1.97s) + shift2: OK (2.01s) Compiling and running oracle program (0.15s) - Oracle output: 2 (1.83s) + Oracle output: 2 (1.86s) Crux output: 2 - shift4: OK (1.98s) - Compiling and running oracle program (0.15s) - Oracle output: -9223372036854775808 (1.83s) + shift4: OK (2.06s) + Compiling and running oracle program (0.18s) + Oracle output: -9223372036854775808 (1.88s) Crux output: -9223372036854775808 - mut: OK (1.97s) + mut: OK (1.99s) Compiling and running oracle program (0.15s) - Oracle output: 14 (1.82s) + Oracle output: 14 (1.84s) Crux output: 14 - wrapping_sub: OK (1.96s) + wrapping_sub: OK (1.98s) Compiling and running oracle program (0.15s) - Oracle output: true (1.81s) + Oracle output: true (1.83s) Crux output: true impl - self: OK (1.98s) - Compiling and running oracle program (0.15s) + self: OK (1.99s) + Compiling and running oracle program (0.16s) Oracle output: 42 (1.83s) Crux output: 42 - simple: OK (1.97s) + simple: OK (1.99s) Compiling and running oracle program (0.15s) - Oracle output: 42 (1.82s) + Oracle output: 42 (1.84s) Crux output: 42 - self_mut: OK (2.00s) + self_mut: OK (1.98s) Compiling and running oracle program (0.15s) - Oracle output: 42 (1.85s) + Oracle output: 42 (1.83s) Crux output: 42 box - new: FAIL (2.11s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.96s) - Crux output: - failures: - - ---- new/2c6bd102::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/box.new/' to rerun this test only. - mut_ref: FAIL (2.08s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.93s) - Crux output: - failures: - - ---- mut_ref/b4fa115d::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/box.mut_ref/' to rerun this test only. - mut: FAIL (2.10s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.94s) - Crux output: - failures: - - ---- mut/2ecd7f0f::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '$0=="crux-mir.crux concrete..box.mut"' to rerun this test only. - unsize: FAIL (2.07s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (1.91s) - Crux output: - failures: - - ---- unsize/6c690c2b::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/box.unsize/' to rerun this test only. - struct: FAIL (2.09s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.94s) - Crux output: - failures: - - ---- struct/da6961f6::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/box.struct/' to rerun this test only. + new: OK (2.16s) + Compiling and running oracle program (0.16s) + Oracle output: () (2.01s) + Crux output: () + mut_ref: OK (2.15s) + Compiling and running oracle program (0.16s) + Oracle output: () (2.00s) + Crux output: () + mut: OK (2.15s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.98s) + Crux output: () + unsize: OK (2.14s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.99s) + Crux output: 2 + struct: OK (2.17s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (2.01s) + Crux output: 1 ptr - is_null_slice: FAIL (expected: can't unsize null pointers) (2.03s) + is_null_slice: FAIL (expected: can't unsize null pointers) (2.06s) Compiling and running oracle program (0.15s) - Oracle output: (false, true) (1.88s) + Oracle output: (false, true) (1.90s) Crux output: failures: - ---- is_null_slice/2e26bc81::crux_test[0] counterexamples ---- + ---- is_null_slice/afb4cb63::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/ptr/is_null_slice.rs:7:13: 7:33: error: in is_null_slice/2e26bc81::crux_test[0] + [Crux] test/conc_eval/ptr/is_null_slice.rs:7:13: 7:33: error: in is_null_slice/afb4cb63::crux_test[0] [Crux] attempted subindex on the result of an integer-to-pointer cast [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:38:38: 38:42: error: in core/95754f0e::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst55af81a2cef2701c[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:38:38: 38:42: error: in core/feeb1386::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst55af81a2cef2701c[0] [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. @@ -934,36 +752,36 @@ FAIL (expected: Should panic, but doesn't) (0.06s) test/Test.hs:124: crux doesn't match oracle (expected failure) - struct_eq: OK (2.03s) + struct_eq: OK (2.01s) Compiling and running oracle program (0.15s) - Oracle output: () (1.88s) + Oracle output: () (1.86s) Crux output: () - read_write: OK (2.06s) - Compiling and running oracle program (0.15s) - Oracle output: [1, 3, 1] (1.91s) + read_write: OK (2.05s) + Compiling and running oracle program (0.16s) + Oracle output: [1, 3, 1] (1.90s) Crux output: [1, 3, 1] - offset_mut: OK (1.97s) + offset_mut: OK (2.05s) Compiling and running oracle program (0.15s) - Oracle output: 3 (1.82s) + Oracle output: 3 (1.89s) Crux output: 3 - copy: OK (2.03s) - Compiling and running oracle program (0.16s) - Oracle output: [1, 2, 3, 1, 2, 3] (1.88s) + copy: OK (2.09s) + Compiling and running oracle program (0.17s) + Oracle output: [1, 2, 3, 1, 2, 3] (1.92s) Crux output: [1, 2, 3, 1, 2, 3] - unsize_slice: OK (1.97s) - Compiling and running oracle program (0.15s) - Oracle output: (1, 2) (1.82s) + unsize_slice: OK (2.07s) + Compiling and running oracle program (0.16s) + Oracle output: (1, 2) (1.91s) Crux output: (1, 2) - dangling_eq: FAIL (2.01s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.86s) + dangling_eq: FAIL (2.09s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.93s) Crux output: failures: - ---- dangling_eq/ed769482::crux_test[0] counterexamples ---- + ---- dangling_eq/7ae9455f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:135:14: 135:78: error: in core/95754f0e::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/95754f0e::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:135:14: 135:78: error: in core/feeb1386::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -971,24 +789,24 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/dangling_eq/' to rerun this test only. - cast_eq: OK (2.00s) + cast_eq: OK (2.10s) Compiling and running oracle program (0.15s) - Oracle output: () (1.86s) + Oracle output: () (1.95s) Crux output: () - offset_from: OK (2.02s) - Compiling and running oracle program (0.15s) - Oracle output: -2 (1.86s) + offset_from: OK (2.07s) + Compiling and running oracle program (0.16s) + Oracle output: -2 (1.91s) Crux output: -2 - null_eq: FAIL (2.14s) - Compiling and running oracle program (0.18s) - Oracle output: 1 (1.96s) + null_eq: FAIL (2.07s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (1.91s) Crux output: failures: - ---- null_eq/bd64c304::crux_test[0] counterexamples ---- + ---- null_eq/3dc65e3a::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:118:14: 118:78: error: in core/95754f0e::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/95754f0e::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:118:14: 118:78: error: in core/feeb1386::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -996,116 +814,125 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/null_eq/' to rerun this test only. - coerce_unsized: OK (2.02s) - Compiling and running oracle program (0.17s) - Oracle output: (1, 2) (1.85s) + coerce_unsized: OK (2.11s) + Compiling and running oracle program (0.18s) + Oracle output: (1, 2) (1.93s) Crux output: (1, 2) - offset: OK (1.94s) + offset: OK (2.00s) Compiling and running oracle program (0.15s) - Oracle output: 3 (1.79s) + Oracle output: 3 (1.85s) Crux output: 3 - is_null: OK (1.95s) + is_null: OK (2.02s) Compiling and running oracle program (0.15s) - Oracle output: (false, true) (1.80s) + Oracle output: (false, true) (1.87s) Crux output: (false, true) - valid_eq: OK (1.97s) + valid_eq: OK (2.01s) Compiling and running oracle program (0.15s) - Oracle output: () (1.83s) + Oracle output: () (1.85s) Crux output: () ops - deref2: OK (1.94s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.80s) - Crux output: () - index2: OK (1.95s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.80s) - Crux output: () - arith1: OK (1.95s) + deref2: OK (2.01s) Compiling and running oracle program (0.15s) - Oracle output: () (1.80s) + Oracle output: () (1.86s) Crux output: () - index3: OK (1.95s) + index2: OK (2.00s) Compiling and running oracle program (0.15s) - Oracle output: () (1.80s) + Oracle output: () (1.85s) Crux output: () - deref1: OK (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.79s) + arith1: OK (2.01s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.86s) Crux output: () - index1: OK (1.92s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.77s) + index3: OK (2.03s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.87s) Crux output: () - deref3: OK (1.91s) + deref1: OK (2.03s) Compiling and running oracle program (0.15s) - Oracle output: () (1.77s) + Oracle output: () (1.88s) + Crux output: () + index1: OK (2.10s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.95s) + Crux output: () + deref3: OK (2.04s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.88s) Crux output: () statics - promoted_fn: OK (1.94s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.80s) + promoted_fn: OK (2.02s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.87s) Crux output: 1 - promoted_static: OK (1.96s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.82s) + promoted_static: OK (2.04s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (1.88s) Crux output: 1 io - cursor_write2: FAIL (2.44s) - Compiling and running oracle program (0.16s) - Oracle output: 0 (2.28s) - user error (JSON Decoding of test/conc_eval/io/cursor_write2.all.mir failed: Error in $.statics[7].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) - - Use -p '/cursor_write2/' to rerun this test only. - cursor_write: FAIL (2.60s) + cursor_write2: FAIL (3.35s) Compiling and running oracle program (0.17s) - Oracle output: 0 (2.43s) - standalone use of `dyn` is not supported: TyDynamic core/95754f0e::error[0]::Error[0]::_trait56353a7b840403b4[0] + Oracle output: 0 (3.18s) + standalone use of `dyn` is not supported: TyDynamic core/feeb1386::any[0]::Erased[0]::_trait58656ccc69864acf[0] CallStack (from HasCallStack): error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/Trans.hs:824:12 in crucible-mir-0.1-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:892:5 in crucible-mir-0.1-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:944:30 in crucible-mir-0.1-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1157:11 in crucible-mir-0.1-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1602:68 in crucible-mir-0.1-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1611:28 in crucible-mir-0.1-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1651:10 in crucible-mir-0.1-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1697:24 in crucible-mir-0.1-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2131:30 in crucible-mir-0.1-inplace:Mir.Trans + tyToRepr, called at src/Mir/TransTy.hs:213:7 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:402:29 in crucible-mir-0.1-inplace:Mir.TransTy + tyToReprCont, called at src/Mir/TransTy.hs:202:24 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy + tyListToCtx, called at src/Mir/TransTy.hs:221:6 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy + tyListToCtx, called at src/Mir/TransTy.hs:1066:16 in crucible-mir-0.1-inplace:Mir.TransTy + traitVtableType, called at src/Mir/Trans.hs:2030:18 in crucible-mir-0.1-inplace:Mir.Trans + transVirtCall, called at src/Mir/Trans.hs:2153:31 in crucible-mir-0.1-inplace:Mir.Trans transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language + Use -p '/cursor_write2/' to rerun this test only. + cursor_write: FAIL (2.77s) + Compiling and running oracle program (0.18s) + Oracle output: 0 (2.59s) + Crux output: + failures: + + ---- cursor_write/fc3f6778::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + + [Crux] Overall status: Invalid. + + test/Test.hs:124: + crux doesn't match oracle + Use -p '$0=="crux-mir.crux concrete..io.cursor_write"' to rerun this test only. - cursor_read: FAIL (2.44s) - Compiling and running oracle program (0.17s) - Oracle output: 0 (2.27s) - standalone use of `dyn` is not supported: TyDynamic core/95754f0e::error[0]::Error[0]::_trait56353a7b840403b4[0] - CallStack (from HasCallStack): - error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/Trans.hs:824:12 in crucible-mir-0.1-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:892:5 in crucible-mir-0.1-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:944:30 in crucible-mir-0.1-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1157:11 in crucible-mir-0.1-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1602:68 in crucible-mir-0.1-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1611:28 in crucible-mir-0.1-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1651:10 in crucible-mir-0.1-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1697:24 in crucible-mir-0.1-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2131:30 in crucible-mir-0.1-inplace:Mir.Trans - transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate - translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language + cursor_read: FAIL (2.74s) + Compiling and running oracle program (0.17s) + Oracle output: 0 (2.56s) + Crux output: + failures: + + ---- cursor_read/337c2c0d::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + + [Crux] Overall status: Invalid. + + test/Test.hs:124: + crux doesn't match oracle Use -p '/io.cursor_read/' to rerun this test only. mem - maybe_uninit: FAIL (2.01s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.87s) + maybe_uninit: FAIL (2.06s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (1.89s) Crux output: failures: - ---- maybe_uninit/da71da36::crux_test[0] counterexamples ---- + ---- maybe_uninit/d8947d0b::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in maybe_uninit/da71da36::crux_test[0] - [Crux] Translation error in maybe_uninit/da71da36::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyInt B32]) + [Crux] internal: error: in maybe_uninit/d8947d0b::crux_test[0] + [Crux] Translation error in maybe_uninit/d8947d0b::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyInt B32]) [Crux] Overall status: Invalid. @@ -1113,16 +940,16 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..mem.maybe_uninit"' to rerun this test only. - maybe_uninit_array_cast: FAIL (2.02s) - Compiling and running oracle program (0.15s) - Oracle output: [1, 2] (1.87s) + maybe_uninit_array_cast: FAIL (2.06s) + Compiling and running oracle program (0.16s) + Oracle output: [1, 2] (1.91s) Crux output: failures: - ---- maybe_uninit_array_cast/c7f09dc0::crux_test[0] counterexamples ---- + ---- maybe_uninit_array_cast/0c1e07d1::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in maybe_uninit_array_cast/c7f09dc0::crux_test[0] - [Crux] Translation error in maybe_uninit_array_cast/c7f09dc0::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyArray (TyInt B32) 2]) + [Crux] internal: error: in maybe_uninit_array_cast/0c1e07d1::crux_test[0] + [Crux] Translation error in maybe_uninit_array_cast/0c1e07d1::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyArray (TyInt B32) 2]) [Crux] Overall status: Invalid. @@ -1131,43 +958,56 @@ FAIL (expected: Should panic, but doesn't) (0.06s) Use -p '/maybe_uninit_array_cast/' to rerun this test only. time - instant: FAIL (2.31s) - Compiling and running oracle program (0.15s) - Oracle output: () (2.16s) - user error (JSON Decoding of test/conc_eval/time/instant.all.mir failed: Error in $.statics[6].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) + instant: FAIL (3.19s) + Compiling and running oracle program (0.16s) + Oracle output: () (3.03s) + standalone use of `dyn` is not supported: TyDynamic core/feeb1386::any[0]::Erased[0]::_trait58656ccc69864acf[0] + CallStack (from HasCallStack): + error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:213:7 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:402:29 in crucible-mir-0.1-inplace:Mir.TransTy + tyToReprCont, called at src/Mir/TransTy.hs:202:24 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy + tyListToCtx, called at src/Mir/TransTy.hs:221:6 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy + tyListToCtx, called at src/Mir/TransTy.hs:1066:16 in crucible-mir-0.1-inplace:Mir.TransTy + traitVtableType, called at src/Mir/Trans.hs:2030:18 in crucible-mir-0.1-inplace:Mir.Trans + transVirtCall, called at src/Mir/Trans.hs:2153:31 in crucible-mir-0.1-inplace:Mir.Trans + transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate + translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '/instant/' to rerun this test only. struct - field_order: OK (1.92s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.78s) + field_order: OK (2.02s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.87s) Crux output: () - arg: OK (1.89s) + arg: OK (2.03s) Compiling and running oracle program (0.15s) - Oracle output: 42 (1.75s) + Oracle output: 42 (1.87s) Crux output: 42 - tup: OK (1.89s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.75s) + tup: OK (2.09s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.92s) Crux output: () - proj: OK (1.85s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.71s) + proj: OK (2.06s) + Compiling and running oracle program (0.17s) + Oracle output: 42 (1.90s) Crux output: 42 - ret: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: S { x: 42, y: 120 } (1.71s) + ret: OK (2.02s) + Compiling and running oracle program (0.15s) + Oracle output: S { x: 42, y: 120 } (1.86s) Crux output: S { x: 42, y: 120 } - repr_transparent: FAIL (1.93s) - Compiling and running oracle program (0.15s) - Oracle output: 6 (1.78s) + repr_transparent: FAIL (2.08s) + Compiling and running oracle program (0.16s) + Oracle output: 6 (1.92s) Crux output: failures: - ---- repr_transparent/5cbd4118::crux_test[0] counterexamples ---- + ---- repr_transparent/ae0007db::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:46:9: 46:18: error: in core/95754f0e::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0] - [Crux] Translation error in core/95754f0e::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0]: unimplemented cast: Misc + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:46:9: 46:18: error: in core/feeb1386::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0] + [Crux] Translation error in core/feeb1386::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyInt B32) Immut [Crux] as: TyRawPtr (TyArray (TyInt B32) 2) Immut @@ -1177,104 +1017,104 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..struct.repr_transparent"' to rerun this test only. - repr_transparent_const: OK (1.90s) + repr_transparent_const: OK (2.02s) Compiling and running oracle program (0.15s) - Oracle output: 124 (1.75s) + Oracle output: 124 (1.87s) Crux output: 124 dyn - assoc_ty: OK (1.88s) + assoc_ty: OK (2.05s) Compiling and running oracle program (0.15s) - Oracle output: 100 (1.73s) + Oracle output: 100 (1.90s) Crux output: 100 - trait_param: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 100 (1.71s) + trait_param: OK (2.08s) + Compiling and running oracle program (0.16s) + Oracle output: 100 (1.92s) Crux output: 100 - plain_trait: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 100 (1.73s) + plain_trait: OK (2.01s) + Compiling and running oracle program (0.16s) + Oracle output: 100 (1.86s) Crux output: 100 - inherit: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 4 (1.74s) + inherit: OK (2.02s) + Compiling and running oracle program (0.15s) + Oracle output: 4 (1.87s) Crux output: 4 stdlib - option: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: true (1.71s) + option: OK (2.00s) + Compiling and running oracle program (0.15s) + Oracle output: true (1.84s) Crux output: true - option2: OK (1.88s) + option2: OK (1.99s) Compiling and running oracle program (0.15s) - Oracle output: 0 (1.73s) + Oracle output: 0 (1.84s) Crux output: 0 - result: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 27 (1.72s) + result: OK (2.00s) + Compiling and running oracle program (0.15s) + Oracle output: 27 (1.84s) Crux output: 27 - default: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: true (1.74s) + default: OK (2.01s) + Compiling and running oracle program (0.15s) + Oracle output: true (1.86s) Crux output: true - cvt: OK (1.91s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.77s) + cvt: OK (2.05s) + Compiling and running oracle program (0.15s) + Oracle output: 0 (1.89s) Crux output: 0 - range: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: 10 (1.75s) + range: OK (2.01s) + Compiling and running oracle program (0.16s) + Oracle output: 10 (1.86s) Crux output: 10 - poly: OK (1.89s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.74s) + poly: OK (2.05s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (1.89s) Crux output: 1 - result_interior: OK (1.88s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (1.73s) + result_interior: OK (2.03s) + Compiling and running oracle program (0.16s) + Oracle output: 3 (1.87s) Crux output: 3 - default_impl: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.71s) + default_impl: OK (2.05s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.90s) Crux output: () - option3: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 27 (1.73s) - Crux output: 27 - teq: OK (1.89s) + option3: OK (2.15s) Compiling and running oracle program (0.15s) - Oracle output: false (1.74s) + Oracle output: 27 (2.00s) + Crux output: 27 + teq: OK (2.07s) + Compiling and running oracle program (0.17s) + Oracle output: false (1.90s) Crux output: false array - wick2: OK (1.90s) + wick2: OK (2.01s) Compiling and running oracle program (0.15s) - Oracle output: true (1.76s) + Oracle output: true (1.86s) Crux output: true - arg: OK (1.88s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.74s) + arg: OK (1.99s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (1.84s) Crux output: 2 - iter: OK (1.91s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.77s) + iter: OK (2.05s) + Compiling and running oracle program (0.16s) + Oracle output: 3 (1.89s) Crux output: 3 - mk_and_proj: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.72s) + mk_and_proj: OK (2.03s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.87s) Crux output: 42 - wick3: FAIL (expected: needs Vec data structure from stdlib) (0.20s) - Compiling and running oracle program (0.16s) - Oracle output: true (0.04s) + wick3: FAIL (expected: needs Vec data structure from stdlib) (0.22s) + Compiling and running oracle program (0.17s) + Oracle output: true (0.05s) user error (Error 101 while running mir-json on test/conc_eval/array/wick3.rs) (expected failure) - clone: FAIL (1.91s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.76s) + clone: FAIL (2.15s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.98s) Crux output: failures: - ---- clone/64b1e1e8::f[0] counterexamples ---- + ---- clone/93fbc27f::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:46:9: 46:18: error: in core/95754f0e::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0] - [Crux] Translation error in core/95754f0e::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0]: unimplemented cast: Misc + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:46:9: 46:18: error: in core/feeb1386::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0] + [Crux] Translation error in core/feeb1386::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyInt B32) Immut [Crux] as: TyRawPtr (TyArray (TyInt B32) 3) Immut @@ -1284,21 +1124,21 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/array.clone/' to rerun this test only. - wick1: FAIL (expected: needs Vec data structure from stdlib) (0.21s) - Compiling and running oracle program (0.16s) + wick1: FAIL (expected: needs Vec data structure from stdlib) (0.22s) + Compiling and running oracle program (0.17s) Oracle output: true (0.05s) user error (Error 101 while running mir-json on test/conc_eval/array/wick1.rs) (expected failure) - mut_index: FAIL (2.41s) - Compiling and running oracle program (0.15s) - Oracle output: 7 (2.26s) + mut_index: FAIL (2.60s) + Compiling and running oracle program (0.16s) + Oracle output: 7 (2.44s) Crux output: failures: - ---- mut_index/4ff817fe::crux_test[0] counterexamples ---- + ---- mut_index/2ae4141f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1306,24 +1146,24 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/mut_index/' to rerun this test only. - mut_arg: OK (1.92s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.77s) + mut_arg: OK (1.99s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.83s) Crux output: 42 - const: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.72s) + const: OK (2.05s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.90s) Crux output: 1 - from_slice: FAIL (2.55s) - Compiling and running oracle program (0.15s) - Oracle output: () (2.41s) + from_slice: FAIL (2.82s) + Compiling and running oracle program (0.16s) + Oracle output: () (2.66s) Crux output: failures: - ---- from_slice/21321bd2::f[0] counterexamples ---- + ---- from_slice/9d05249b::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/array/mod.rs:266:23: 266:54: error: in core/95754f0e::array[0]::{impl#10}[0]::try_from[0]::_inste494657dbbea865b[0] - [Crux] Translation error in core/95754f0e::array[0]::{impl#10}[0]::try_from[0]::_inste494657dbbea865b[0]: unimplemented cast: Misc + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/array/mod.rs:266:23: 266:54: error: in core/feeb1386::array[0]::{impl#10}[0]::try_from[0]::_inste494657dbbea865b[0] + [Crux] Translation error in core/feeb1386::array[0]::{impl#10}[0]::try_from[0]::_inste494657dbbea865b[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyUint B8) Immut [Crux] as: TyRawPtr (TyArray (TyUint B8) 4) Immut @@ -1333,26 +1173,26 @@ FAIL (expected: Should panic, but doesn't) (0.06s) crux doesn't match oracle Use -p '/array.from_slice/' to rerun this test only. - const_impl: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 5 (1.74s) + const_impl: OK (2.13s) + Compiling and running oracle program (0.17s) + Oracle output: 5 (1.96s) Crux output: 5 enum - match: OK (1.90s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.75s) + match: OK (2.08s) + Compiling and running oracle program (0.16s) + Oracle output: 42 (1.92s) Crux output: 42 - field_order: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.72s) + field_order: OK (2.11s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.96s) Crux output: () - mixed_discrs: OK (1.88s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.73s) + mixed_discrs: OK (2.01s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.85s) Crux output: () - arg: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.74s) + arg: OK (2.00s) + Compiling and running oracle program (0.16s) + Oracle output: 0 (1.85s) Crux output: 0 arg2: rustc compilation failed for arg2 error output: @@ -1382,16 +1222,16 @@ FAIL (0.05s) failed to compile and run Use -p '/arg2/' to rerun this test only. - cow: FAIL (2.03s) - Compiling and running oracle program (0.15s) - Oracle output: 200 (1.87s) + cow: FAIL (2.16s) + Compiling and running oracle program (0.16s) + Oracle output: 200 (2.00s) Crux output: failures: - ---- cow/64514981::crux_test[0] counterexamples ---- + ---- cow/f1ac0cae::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1399,33 +1239,33 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/cow/' to rerun this test only. - ret: OK (1.88s) - Compiling and running oracle program (0.15s) - Oracle output: (A(42), B { x: 42 }, C) (1.73s) + ret: OK (2.05s) + Compiling and running oracle program (0.16s) + Oracle output: (A(42), B { x: 42 }, C) (1.89s) Crux output: (A(42), B { x: 42 }, C) - eq: OK (1.90s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.75s) + eq: OK (2.09s) + Compiling and running oracle program (0.17s) + Oracle output: () (1.93s) Crux output: () - cmp: OK (1.87s) - Compiling and running oracle program (0.15s) - Oracle output: -23 (1.72s) + cmp: OK (2.08s) + Compiling and running oracle program (0.16s) + Oracle output: -23 (1.92s) Crux output: -23 - inner: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 4 (1.74s) + inner: OK (2.04s) + Compiling and running oracle program (0.15s) + Oracle output: 4 (1.89s) Crux output: 4 vec - drop: FAIL (2.04s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.89s) + drop: FAIL (2.25s) + Compiling and running oracle program (0.16s) + Oracle output: () (2.09s) Crux output: failures: - ---- drop/42c23083::crux_test[0] counterexamples ---- + ---- drop/97adc623::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_instb944086d0b551051[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_instb944086d0b551051[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1433,16 +1273,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/drop/' to rerun this test only. - set_len: FAIL (2.03s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.88s) + set_len: FAIL (2.24s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (2.07s) Crux output: failures: - ---- set_len/7feb2720::crux_test[0] counterexamples ---- + ---- set_len/3f5b6000::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1450,20 +1290,20 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/set_len/' to rerun this test only. - push: OK (2.10s) + push: OK (2.27s) Compiling and running oracle program (0.16s) - Oracle output: (1, 2) (1.94s) + Oracle output: (1, 2) (2.11s) Crux output: (1, 2) - extend: FAIL (2.15s) - Compiling and running oracle program (0.16s) - Oracle output: (1, 10) (1.99s) + extend: FAIL (2.33s) + Compiling and running oracle program (0.17s) + Oracle output: (1, 10) (2.16s) Crux output: failures: - ---- extend/bf33c8cd::crux_test[0] counterexamples ---- + ---- extend/3b321784::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in core/95754f0e::iter[0]::traits[0]::iterator[0]::Iterator[0]::try_fold[0]::_inst647f2421f6e4ebd6[0] - [Crux] Translation error in core/95754f0e::iter[0]::traits[0]::iterator[0]::Iterator[0]::try_fold[0]::_inst647f2421f6e4ebd6[0]: error building variant core/95754f0e::ops[0]::control_flow[0]::ControlFlow[0]::Continue[0]: type mismatch: expected AnyRepr but got BVRepr 32 in field 0: ([FieldRepr (FkInit AnyRepr)],[Just bVLit(32, BV 0)]) -- [Just bVLit(32, BV 0): BVRepr 32] + [Crux] internal: error: in core/feeb1386::iter[0]::traits[0]::iterator[0]::Iterator[0]::try_fold[0]::_inst647f2421f6e4ebd6[0] + [Crux] Translation error in core/feeb1386::iter[0]::traits[0]::iterator[0]::Iterator[0]::try_fold[0]::_inst647f2421f6e4ebd6[0]: error building variant core/feeb1386::ops[0]::control_flow[0]::ControlFlow[0]::Continue[0]: type mismatch: expected AnyRepr but got BVRepr 32 in field 0: ([FieldRepr (FkInit AnyRepr)],[Just bVLit(32, BV 0)]) -- [Just bVLit(32, BV 0): BVRepr 32] [Crux] Overall status: Invalid. @@ -1471,16 +1311,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..vec.extend"' to rerun this test only. - collect: FAIL (2.78s) - Compiling and running oracle program (0.16s) - Oracle output: 45 (2.62s) + collect: FAIL (3.06s) + Compiling and running oracle program (0.17s) + Oracle output: 45 (2.89s) Crux output: failures: - ---- collect/0932e133::crux_test[0] counterexamples ---- + ---- collect/e38ca0df::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:786:58: 786:72: error: in core/95754f0e::ptr[0]::const_ptr[0]::{impl#0}[0]::sub_ptr[0]::runtime[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/95754f0e::ptr[0]::const_ptr[0]::{impl#0}[0]::sub_ptr[0]::runtime[0]::_inst1e2825177cd3b608[0]: No translation for pointer binop: >= + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:786:58: 786:72: error: in core/feeb1386::ptr[0]::const_ptr[0]::{impl#0}[0]::sub_ptr[0]::runtime[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/feeb1386::ptr[0]::const_ptr[0]::{impl#0}[0]::sub_ptr[0]::runtime[0]::_inst1e2825177cd3b608[0]: No translation for pointer binop: >= [Crux] Overall status: Invalid. @@ -1488,31 +1328,31 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/collect/' to rerun this test only. - extend_trusted_len: OK (2.66s) - Compiling and running oracle program (0.16s) - Oracle output: (1, 10) (2.50s) + extend_trusted_len: OK (3.00s) + Compiling and running oracle program (0.18s) + Oracle output: (1, 10) (2.82s) Crux output: (1, 10) - from_elem_zero: FAIL (0.21s) - Compiling and running oracle program (0.16s) + from_elem_zero: FAIL (0.22s) + Compiling and running oracle program (0.17s) Oracle output: 0 (0.05s) user error (Error 101 while running mir-json on test/conc_eval/vec/from_elem_zero.rs) Use -p '/from_elem_zero/' to rerun this test only. clos - promoted: OK (1.92s) + promoted: OK (2.02s) Compiling and running oracle program (0.15s) - Oracle output: 33 (1.77s) + Oracle output: 33 (1.87s) Crux output: 33 - as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.74s) + as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (2.03s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.88s) Crux output: failures: - ---- as_fn_ptr/7cd8b443::crux_test[0] counterexamples ---- + ---- as_fn_ptr/61af895c::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/7cd8b443::crux_test[0] - [Crux] Translation error in as_fn_ptr/7cd8b443::crux_test[0]: unimplemented cast: ClosureFnPointer + [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/61af895c::crux_test[0] + [Crux] Translation error in as_fn_ptr/61af895c::crux_test[0]: unimplemented cast: ClosureFnPointer [Crux] ty: TyClosure [] [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [], _fsreturn_ty = TyInt B32, _fsabi = RustAbi, _fsspreadarg = Nothing}) @@ -1521,102 +1361,102 @@ FAIL (0.05s) test/Test.hs:124: crux doesn't match oracle (expected failure) - fnptr_fnmut: OK (1.87s) - Compiling and running oracle program (0.15s) - Oracle output: 4 (1.72s) + fnptr_fnmut: OK (2.06s) + Compiling and running oracle program (0.16s) + Oracle output: 4 (1.89s) Crux output: 4 - direct_fnonce: OK (1.86s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.71s) + direct_fnonce: OK (2.09s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (1.94s) Crux output: 2 - conv_fnonce_fnmut: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.73s) + conv_fnonce_fnmut: OK (2.04s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (1.89s) Crux output: 2 - fn_static_poly: OK (1.91s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.77s) + fn_static_poly: OK (2.03s) + Compiling and running oracle program (0.16s) + Oracle output: 3 (1.87s) Crux output: 3 - fnptr_fnonce: OK (1.88s) - Compiling and running oracle program (0.15s) - Oracle output: 4 (1.73s) + fnptr_fnonce: OK (2.01s) + Compiling and running oracle program (0.16s) + Oracle output: 4 (1.86s) Crux output: 4 - fn_static: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.73s) + fn_static: OK (2.02s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (1.87s) Crux output: 3 fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (0.18s) Compiling and running oracle program (0.15s) Oracle output: 3 (0.03s) user error (Error 101 while running mir-json on test/conc_eval/clos/fn_dyn.rs) (expected failure) - fnonce: OK (1.91s) + fnonce: OK (2.00s) Compiling and running oracle program (0.15s) - Oracle output: false (1.76s) + Oracle output: false (1.85s) Crux output: false - conv_fnmut_fn: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.76s) + conv_fnmut_fn: OK (2.01s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (1.85s) Crux output: 2 - direct_fnmut2: OK (1.90s) + direct_fnmut2: OK (2.03s) Compiling and running oracle program (0.15s) - Oracle output: 7 (1.75s) + Oracle output: 7 (1.87s) Crux output: 7 - fo: OK (1.93s) + fo: OK (2.03s) Compiling and running oracle program (0.15s) - Oracle output: 29 (1.78s) + Oracle output: 29 (1.88s) Crux output: 29 - unique_borrow: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.76s) + unique_borrow: OK (2.02s) + Compiling and running oracle program (0.16s) + Oracle output: 3 (1.86s) Crux output: 3 - conv_fnonce_fn: OK (1.90s) + conv_fnonce_fn: OK (2.01s) Compiling and running oracle program (0.15s) - Oracle output: 2 (1.75s) + Oracle output: 2 (1.85s) Crux output: 2 - fnptr_fn: OK (1.89s) - Compiling and running oracle program (0.15s) - Oracle output: 4 (1.75s) + fnptr_fn: OK (2.02s) + Compiling and running oracle program (0.16s) + Oracle output: 4 (1.87s) Crux output: 4 - direct_fnmut: OK (1.89s) - Compiling and running oracle program (0.15s) - Oracle output: 7 (1.75s) + direct_fnmut: OK (2.06s) + Compiling and running oracle program (0.16s) + Oracle output: 7 (1.90s) Crux output: 7 - dispatch_fnmut: OK (1.91s) - Compiling and running oracle program (0.15s) - Oracle output: 7 (1.76s) + dispatch_fnmut: OK (2.08s) + Compiling and running oracle program (0.19s) + Oracle output: 7 (1.90s) Crux output: 7 - poly: OK (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (1.79s) + poly: OK (2.03s) + Compiling and running oracle program (0.16s) + Oracle output: 3 (1.87s) Crux output: 3 - direct_fn: OK (1.89s) + direct_fn: OK (2.03s) Compiling and running oracle program (0.15s) - Oracle output: 2 (1.74s) + Oracle output: 2 (1.87s) Crux output: 2 - fnptr_closure: OK (1.91s) - Compiling and running oracle program (0.14s) - Oracle output: 7 (1.77s) + fnptr_closure: OK (2.02s) + Compiling and running oracle program (0.16s) + Oracle output: 7 (1.86s) Crux output: 7 - fnonce1: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.72s) + fnonce1: OK (2.01s) + Compiling and running oracle program (0.16s) + Oracle output: 3 (1.85s) Crux output: 3 - ref_fnmut: OK (2.01s) - Compiling and running oracle program (0.15s) - Oracle output: 7 (1.87s) + ref_fnmut: OK (2.05s) + Compiling and running oracle program (0.16s) + Oracle output: 7 (1.90s) Crux output: 7 vec_deque - pop: FAIL (2.77s) + pop: FAIL (2.71s) Compiling and running oracle program (0.17s) - Oracle output: [5, 4, 3, 1, 2] (2.60s) + Oracle output: [5, 4, 3, 1, 2] (2.54s) Crux output: failures: - ---- pop/3a129246::crux_test[0] counterexamples ---- + ---- pop/89bcc818::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1624,16 +1464,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/vec_deque.pop/' to rerun this test only. - iter_clone: FAIL (2.83s) - Compiling and running oracle program (0.16s) - Oracle output: [1, 2, 3, 2, 3] (2.66s) + iter_clone: FAIL (2.80s) + Compiling and running oracle program (0.17s) + Oracle output: [1, 2, 3, 2, 3] (2.63s) Crux output: failures: - ---- iter_clone/ffad8245::crux_test[0] counterexamples ---- + ---- iter_clone/d8b5a2df::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1641,33 +1481,20 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/iter_clone/' to rerun this test only. - push: FAIL (2.70s) - Compiling and running oracle program (0.16s) - Oracle output: [4, 1, 2, 3, 5] (2.54s) - Crux output: - failures: - - ---- push/7760ccfb::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in alloc/fed0e039::collections[0]::vec_deque[0]::{impl#5}[0]::grow[0]::_inst6708c5c8c5ed40f8[0] - [Crux] panicking::panic, called from alloc/fed0e039::collections[0]::vec_deque[0]::{impl#5}[0]::grow[0]::_inst6708c5c8c5ed40f8[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/vec_deque.push/' to rerun this test only. - retain: FAIL (2.70s) + push: OK (2.78s) + Compiling and running oracle program (0.17s) + Oracle output: [4, 1, 2, 3, 5] (2.61s) + Crux output: [4, 1, 2, 3, 5] + retain: FAIL (2.80s) Compiling and running oracle program (0.17s) - Oracle output: [1, 4] (2.53s) + Oracle output: [1, 4] (2.63s) Crux output: failures: - ---- retain/0ae7e799::crux_test[0] counterexamples ---- + ---- retain/b9dbae7f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1675,16 +1502,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/retain/' to rerun this test only. - rotate_left: FAIL (2.82s) - Compiling and running oracle program (0.16s) - Oracle output: [3, 4, 5, 1, 2] (2.66s) + rotate_left: FAIL (2.88s) + Compiling and running oracle program (0.17s) + Oracle output: [3, 4, 5, 1, 2] (2.71s) Crux output: failures: - ---- rotate_left/b0c4e843::crux_test[0] counterexamples ---- + ---- rotate_left/74a68a05::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1693,15 +1520,15 @@ FAIL (0.05s) Use -p '/rotate_left/' to rerun this test only. rotate_right: FAIL (2.84s) - Compiling and running oracle program (0.16s) - Oracle output: [4, 5, 1, 2, 3] (2.68s) + Compiling and running oracle program (0.19s) + Oracle output: [4, 5, 1, 2, 3] (2.66s) Crux output: failures: - ---- rotate_right/de4668b4::crux_test[0] counterexamples ---- + ---- rotate_right/959d304a::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1714,16 +1541,16 @@ FAIL (0.05s) Compiling and running oracle program (0.16s) Oracle output: 5 (1.86s) Crux output: 5 - mut_range: FAIL (2.49s) - Compiling and running oracle program (0.17s) - Oracle output: 86 (2.33s) + mut_range: FAIL (2.57s) + Compiling and running oracle program (0.16s) + Oracle output: 86 (2.41s) Crux output: failures: - ---- mut_range/a64fa57c::crux_test[0] counterexamples ---- + ---- mut_range/702f154c::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1731,16 +1558,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/mut_range/' to rerun this test only. - range_len: FAIL (2.47s) + range_len: FAIL (2.54s) Compiling and running oracle program (0.15s) - Oracle output: 1 (2.32s) + Oracle output: 1 (2.39s) Crux output: failures: - ---- range_len/d3e86403::crux_test[0] counterexamples ---- + ---- range_len/04ce374c::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1748,16 +1575,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..slice.range_len"' to rerun this test only. - range_len_mut: FAIL (2.43s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (2.29s) + range_len_mut: FAIL (2.66s) + Compiling and running oracle program (0.17s) + Oracle output: 1 (2.49s) Crux output: failures: - ---- range_len_mut/0c7dec71::crux_test[0] counterexamples ---- + ---- range_len_mut/d4b69d12::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1765,24 +1592,24 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/range_len_mut/' to rerun this test only. - mk_and_proj: OK (1.95s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.79s) + mk_and_proj: OK (2.02s) + Compiling and running oracle program (0.16s) + Oracle output: 42 (1.86s) Crux output: 42 - swap: OK (1.96s) + swap: OK (2.06s) Compiling and running oracle program (0.15s) - Oracle output: 2001 (1.81s) + Oracle output: 2001 (1.90s) Crux output: 2001 - eq: FAIL (1.94s) + eq: FAIL (2.02s) Compiling and running oracle program (0.15s) - Oracle output: () (1.79s) + Oracle output: () (1.87s) Crux output: failures: - ---- eq/336948bb::f[0] counterexamples ---- + ---- eq/6323bc6f::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/mem/mod.rs:338:38: 338:41: error: in core/95754f0e::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/95754f0e::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::size_of_val[0]::_inst55af81a2cef2701c[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/mem/mod.rs:338:38: 338:41: error: in core/feeb1386::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/feeb1386::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0]: callExp: Don't know how to call core/feeb1386::intrinsics[0]::{extern#0}[0]::size_of_val[0]::_inst55af81a2cef2701c[0] [Crux] Overall status: Invalid. @@ -1790,16 +1617,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/slice.eq/' to rerun this test only. - iter_mut: FAIL (2.01s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.86s) + iter_mut: FAIL (2.08s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.92s) Crux output: failures: - ---- iter_mut/1188a615::f[0] counterexamples ---- + ---- iter_mut/8b0c4648::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/array/mod.rs:266:23: 266:54: error: in core/95754f0e::array[0]::{impl#10}[0]::try_from[0]::_insta17aeaef2aec3782[0] - [Crux] Translation error in core/95754f0e::array[0]::{impl#10}[0]::try_from[0]::_insta17aeaef2aec3782[0]: unimplemented cast: Misc + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/array/mod.rs:266:23: 266:54: error: in core/feeb1386::array[0]::{impl#10}[0]::try_from[0]::_insta17aeaef2aec3782[0] + [Crux] Translation error in core/feeb1386::array[0]::{impl#10}[0]::try_from[0]::_insta17aeaef2aec3782[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyInt B32) Immut [Crux] as: TyRawPtr (TyArray (TyInt B32) 3) Immut @@ -1809,16 +1636,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/iter_mut/' to rerun this test only. - get: FAIL (1.97s) - Compiling and running oracle program (0.15s) - Oracle output: true (1.82s) + get: FAIL (2.03s) + Compiling and running oracle program (0.16s) + Oracle output: true (1.88s) Crux output: failures: - ---- get/39caa45e::crux_test[0] counterexamples ---- + ---- get/54cacd53::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/95754f0e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1826,834 +1653,838 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/slice.get/' to rerun this test only. - mut: OK (1.95s) + mut: OK (2.00s) Compiling and running oracle program (0.15s) - Oracle output: 42 (1.80s) + Oracle output: 42 (1.85s) Crux output: 42 - last: OK (2.05s) - Compiling and running oracle program (0.16s) - Oracle output: 3 (1.89s) + last: OK (2.01s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (1.86s) Crux output: 3 crux symbolic Output testing - mux_init_mut: FAIL (2.05s) + mux_init_mut: FAIL (1.97s) files test/symb_eval/refs/mux_init_mut.good and test/symb_eval/refs/mux_init_mut.out differ; test/symb_eval/refs/mux_init_mut.out contains: - test mux_init_mut/c7bde8a8::crux_test[0]: FAILED + test mux_init_mut/59f4ede3::crux_test[0]: FAILED failures: - ---- mux_init_mut/c7bde8a8::crux_test[0] counterexamples ---- + ---- mux_init_mut/59f4ede3::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/mux_init_mut/' to rerun this test only. mux_init_imm: FAIL (1.98s) files test/symb_eval/refs/mux_init_imm.good and test/symb_eval/refs/mux_init_imm.out differ; test/symb_eval/refs/mux_init_imm.out contains: - test mux_init_imm/45896388::crux_test[0]: FAILED + test mux_init_imm/c11c6d2f::crux_test[0]: FAILED failures: - ---- mux_init_imm/45896388::crux_test[0] counterexamples ---- + ---- mux_init_imm/c11c6d2f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/mux_init_imm/' to rerun this test only. - early_fail: FAIL (2.10s) + early_fail: FAIL (1.98s) files test/symb_eval/crux/early_fail.good and test/symb_eval/crux/early_fail.out differ; test/symb_eval/crux/early_fail.out contains: - test early_fail/a3644e81::fail1[0]: FAILED - test early_fail/a3644e81::fail2[0]: FAILED + test early_fail/aa1c593a::fail1[0]: FAILED + test early_fail/aa1c593a::fail2[0]: FAILED failures: - ---- early_fail/a3644e81::fail1[0] counterexamples ---- + ---- early_fail/aa1c593a::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in early_fail/a3644e81::fail1[0] - [Crux] panicking::panic_fmt, called from early_fail/a3644e81::fail1[0] + [Crux] internal: error: in early_fail/aa1c593a::fail1[0] + [Crux] panicking::panic_fmt, called from early_fail/aa1c593a::fail1[0] - ---- early_fail/a3644e81::fail2[0] counterexamples ---- + ---- early_fail/aa1c593a::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/early_fail/' to rerun this test only. - mixed_fail: FAIL (2.08s) + mixed_fail: FAIL (2.01s) files test/symb_eval/crux/mixed_fail.good and test/symb_eval/crux/mixed_fail.out differ; test/symb_eval/crux/mixed_fail.out contains: - test mixed_fail/d1feb05c::fail1[0]: FAILED - test mixed_fail/d1feb05c::fail2[0]: FAILED - test mixed_fail/d1feb05c::pass1[0]: FAILED - test mixed_fail/d1feb05c::pass2[0]: FAILED + test mixed_fail/6c0add63::fail1[0]: FAILED + test mixed_fail/6c0add63::fail2[0]: FAILED + test mixed_fail/6c0add63::pass1[0]: FAILED + test mixed_fail/6c0add63::pass2[0]: FAILED failures: - ---- mixed_fail/d1feb05c::fail1[0] counterexamples ---- + ---- mixed_fail/6c0add63::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - ---- mixed_fail/d1feb05c::fail2[0] counterexamples ---- + ---- mixed_fail/6c0add63::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - ---- mixed_fail/d1feb05c::pass1[0] counterexamples ---- + ---- mixed_fail/6c0add63::pass1[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - ---- mixed_fail/d1feb05c::pass2[0] counterexamples ---- + ---- mixed_fail/6c0add63::pass2[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/mixed_fail/' to rerun this test only. - multi: FAIL (2.00s) + multi: FAIL (1.94s) files test/symb_eval/crux/multi.good and test/symb_eval/crux/multi.out differ; test/symb_eval/crux/multi.out contains: - test multi/2eb0a9e7::fail1[0]: FAILED - test multi/2eb0a9e7::fail2[0]: FAILED - test multi/2eb0a9e7::fail3[0]: FAILED + test multi/608d8b6e::fail1[0]: FAILED + test multi/608d8b6e::fail2[0]: FAILED + test multi/608d8b6e::fail3[0]: FAILED failures: - ---- multi/2eb0a9e7::fail1[0] counterexamples ---- + ---- multi/608d8b6e::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - ---- multi/2eb0a9e7::fail2[0] counterexamples ---- + ---- multi/608d8b6e::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - ---- multi/2eb0a9e7::fail3[0] counterexamples ---- + ---- multi/608d8b6e::fail3[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/Output testing.multi/' to rerun this test only. - fail_return: FAIL (2.02s) + fail_return: FAIL (1.98s) files test/symb_eval/crux/fail_return.good and test/symb_eval/crux/fail_return.out differ; test/symb_eval/crux/fail_return.out contains: - test fail_return/8b108ddc::fail1[0]: FAILED - test fail_return/8b108ddc::fail2[0]: FAILED + test fail_return/ac160af8::fail1[0]: FAILED + test fail_return/ac160af8::fail2[0]: FAILED failures: - ---- fail_return/8b108ddc::fail1[0] counterexamples ---- + ---- fail_return/ac160af8::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - ---- fail_return/8b108ddc::fail2[0] counterexamples ---- + ---- fail_return/ac160af8::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/fail_return/' to rerun this test only. - no_conc: FAIL (1.93s) + no_conc: FAIL (1.97s) files test/symb_eval/concretize/no_conc.good and test/symb_eval/concretize/no_conc.out differ; test/symb_eval/concretize/no_conc.out contains: - test no_conc/fd84226e::crux_test[0]: FAILED + test no_conc/7d5e52dc::crux_test[0]: FAILED failures: - ---- no_conc/fd84226e::crux_test[0] counterexamples ---- + ---- no_conc/7d5e52dc::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/no_conc/' to rerun this test only. - conc: FAIL (2.03s) + conc: FAIL (1.96s) files test/symb_eval/concretize/conc.good and test/symb_eval/concretize/conc.out differ; test/symb_eval/concretize/conc.out contains: - test conc/daa4ef6b::crux_test[0]: FAILED + test conc/d6426254::crux_test[0]: FAILED failures: - ---- conc/daa4ef6b::crux_test[0] counterexamples ---- + ---- conc/d6426254::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.conc"' to rerun this test only. - array: FAIL (2.14s) + array: FAIL (2.01s) files test/symb_eval/concretize/array.good and test/symb_eval/concretize/array.out differ; test/symb_eval/concretize/array.out contains: - test array/ddc0118b::crux_test[0]: FAILED + test array/4739c244::crux_test[0]: FAILED failures: - ---- array/ddc0118b::crux_test[0] counterexamples ---- + ---- array/4739c244::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::array[0]::symbolic[0]::_inst1e2825177cd3b608[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::array[0]::symbolic[0]::_inst1e2825177cd3b608[0] + [Crux] internal: error: in crucible/2c868646::array[0]::symbolic[0]::_inst1e2825177cd3b608[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::array[0]::symbolic[0]::_inst1e2825177cd3b608[0] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.array"' to rerun this test only. - assert_ok: FAIL (2.61s) + assert_ok: FAIL (2.52s) files test/symb_eval/concretize/assert_ok.good and test/symb_eval/concretize/assert_ok.out differ; test/symb_eval/concretize/assert_ok.out contains: - test assert_ok/22df3d32::crux_test[0]: FAILED + test assert_ok/05b748f4::crux_test[0]: FAILED failures: - ---- assert_ok/22df3d32::crux_test[0] counterexamples ---- + ---- assert_ok/05b748f4::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/assert_ok/' to rerun this test only. - assert: FAIL (2.44s) + assert: FAIL (2.52s) files test/symb_eval/concretize/assert.good and test/symb_eval/concretize/assert.out differ; test/symb_eval/concretize/assert.out contains: - test assert/be1de2be::crux_test[0]: FAILED + test assert/d4658be4::crux_test[0]: FAILED failures: - ---- assert/be1de2be::crux_test[0] counterexamples ---- + ---- assert/d4658be4::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.assert"' to rerun this test only. - bytes2: FAIL (1.96s) + bytes2: FAIL (2.03s) files test/symb_eval/crypto/bytes2.good and test/symb_eval/crypto/bytes2.out differ; test/symb_eval/crypto/bytes2.out contains: - test bytes2/529826ca::f[0]: FAILED + test bytes2/bf8b6d44::f[0]: FAILED failures: - ---- bytes2/529826ca::f[0] counterexamples ---- + ---- bytes2/bf8b6d44::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/bytes2/' to rerun this test only. - double: FAIL (1.83s) + double: FAIL (1.90s) files test/symb_eval/crypto/double.good and test/symb_eval/crypto/double.out differ; test/symb_eval/crypto/double.out contains: - test double/afc4527c::f[0]: FAILED + test double/dba12598::f[0]: FAILED failures: - ---- double/afc4527c::f[0] counterexamples ---- + ---- double/dba12598::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u32[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u32[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u32[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u32[0] [Crux] Overall status: Invalid. Use -p '/double/' to rerun this test only. - ffs: FAIL (1.92s) + ffs: FAIL (2.01s) files test/symb_eval/crypto/ffs.good and test/symb_eval/crypto/ffs.out differ; test/symb_eval/crypto/ffs.out contains: - test ffs/35c1efc1::f[0]: FAILED + test ffs/7ab601f7::f[0]: FAILED failures: - ---- ffs/35c1efc1::f[0] counterexamples ---- + ---- ffs/7ab601f7::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u32[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u32[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u32[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u32[0] [Crux] Overall status: Invalid. Use -p '/Output testing.ffs/' to rerun this test only. - bytes: FAIL (2.20s) + bytes: FAIL (2.03s) files test/symb_eval/crypto/bytes.good and test/symb_eval/crypto/bytes.out differ; test/symb_eval/crypto/bytes.out contains: - test bytes/10105f30::f[0]: FAILED + test bytes/335dfb55::f[0]: FAILED failures: - ---- bytes/10105f30::f[0] counterexamples ---- + ---- bytes/335dfb55::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u64[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u64[0] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.bytes"' to rerun this test only. - bad_symb1: FAIL (1.90s) + bad_symb1: FAIL (2.01s) files test/symb_eval/overrides/bad_symb1.good and test/symb_eval/overrides/bad_symb1.out differ; test/symb_eval/overrides/bad_symb1.out contains: - test bad_symb1/89147cbd::crux_test[0]: FAILED + test bad_symb1/20a92dea::crux_test[0]: FAILED failures: - ---- bad_symb1/89147cbd::crux_test[0] counterexamples ---- + ---- bad_symb1/20a92dea::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/bad_symb1/' to rerun this test only. bad_symb2: FAIL (1.91s) files test/symb_eval/overrides/bad_symb2.good and test/symb_eval/overrides/bad_symb2.out differ; test/symb_eval/overrides/bad_symb2.out contains: - test bad_symb2/96a88eca::crux_test[0]: FAILED + test bad_symb2/4f8712a5::crux_test[0]: FAILED failures: - ---- bad_symb2/96a88eca::crux_test[0] counterexamples ---- + ---- bad_symb2/4f8712a5::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/bad_symb2/' to rerun this test only. - override3: FAIL (1.81s) + override3: FAIL (1.91s) files test/symb_eval/overrides/override3.good and test/symb_eval/overrides/override3.out differ; test/symb_eval/overrides/override3.out contains: - test override3/e13ebb8f::f[0]: FAILED + test override3/2aec5606::f[0]: FAILED failures: - ---- override3/e13ebb8f::f[0] counterexamples ---- + ---- override3/2aec5606::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/override3/' to rerun this test only. - override1: FAIL (1.84s) + override1: FAIL (1.82s) files test/symb_eval/overrides/override1.good and test/symb_eval/overrides/override1.out differ; test/symb_eval/overrides/override1.out contains: - test override1/33ca3dcf::f[0]: FAILED + test override1/6487bdad::f[0]: FAILED failures: - ---- override1/33ca3dcf::f[0] counterexamples ---- + ---- override1/6487bdad::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::one[0] - [Crux] panicking::panic, called from crucible/7940c2ae::one[0] + [Crux] internal: error: in crucible/2c868646::one[0] + [Crux] panicking::panic, called from crucible/2c868646::one[0] [Crux] Overall status: Invalid. Use -p '/override1/' to rerun this test only. - override4: FAIL (1.98s) + override4: FAIL (1.96s) files test/symb_eval/overrides/override4.good and test/symb_eval/overrides/override4.out differ; test/symb_eval/overrides/override4.out contains: - test override4/f0e88173::f[0]: FAILED + test override4/6bc9b63e::f[0]: FAILED failures: - ---- override4/f0e88173::f[0] counterexamples ---- + ---- override4/6bc9b63e::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/override4/' to rerun this test only. - override_rust: FAIL (1.91s) + override_rust: FAIL (1.92s) files test/symb_eval/overrides/override_rust.good and test/symb_eval/overrides/override_rust.out differ; test/symb_eval/overrides/override_rust.out contains: - test override_rust/81682348::crux_test[0]: FAILED + test override_rust/7782f5ae::crux_test[0]: FAILED failures: - ---- override_rust/81682348::crux_test[0] counterexamples ---- + ---- override_rust/7782f5ae::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::override_[0]::_inst28abcb7c380bf102[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::override_[0]::_inst28abcb7c380bf102[0] + [Crux] internal: error: in crucible/2c868646::override_[0]::_inst28abcb7c380bf102[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::override_[0]::_inst28abcb7c380bf102[0] [Crux] Overall status: Invalid. Use -p '/override_rust/' to rerun this test only. - override2: FAIL (1.96s) + override2: FAIL (1.92s) files test/symb_eval/overrides/override2.good and test/symb_eval/overrides/override2.out differ; test/symb_eval/overrides/override2.out contains: - test override2/23599c8e::f[0]: FAILED + test override2/2d082460::f[0]: FAILED failures: - ---- override2/23599c8e::f[0] counterexamples ---- + ---- override2/2d082460::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/override2/' to rerun this test only. - override5: FAIL (1.95s) + override5: FAIL (1.92s) files test/symb_eval/overrides/override5.good and test/symb_eval/overrides/override5.out differ; test/symb_eval/overrides/override5.out contains: - test override5/ff4062ad::f[0]: FAILED + test override5/82e21c84::f[0]: FAILED failures: - ---- override5/ff4062ad::f[0] counterexamples ---- + ---- override5/82e21c84::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u64[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u64[0] [Crux] Overall status: Invalid. Use -p '/override5/' to rerun this test only. - mux: FAIL (2.18s) + mux: FAIL (1.94s) files test/symb_eval/fnptr/mux.good and test/symb_eval/fnptr/mux.out differ; test/symb_eval/fnptr/mux.out contains: - test mux/c7b9f0dd::crux_test[0]: FAILED + test mux/bb0881be::crux_test[0]: FAILED failures: - ---- mux/c7b9f0dd::crux_test[0] counterexamples ---- + ---- mux/bb0881be::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.mux"' to rerun this test only. - checked_mul_signed: FAIL (0.05s) + checked_mul_signed: FAIL (0.04s) test/symb_eval/num/checked_mul_signed.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/num/checked_mul_signed.rs) Use -p '/checked_mul_signed/' to rerun this test only. checked_mul: FAIL test/symb_eval/num/checked_mul.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/num/checked_mul.rs) Use -p '$0=="crux-mir.crux symbolic.Output testing.checked_mul"' to rerun this test only. - checked_div: FAIL (1.98s) + checked_div: FAIL (1.87s) files test/symb_eval/num/checked_div.good and test/symb_eval/num/checked_div.out differ; test/symb_eval/num/checked_div.out contains: - test checked_div/5776f60a::crux_test[0]: returned 65, FAILED + test checked_div/2f0ffa29::crux_test[0]: returned 65, FAILED failures: - ---- checked_div/5776f60a::crux_test[0] counterexamples ---- + ---- checked_div/2f0ffa29::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/5776f60a::div_signed[0] + [Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/2f0ffa29::div_signed[0] [Crux] attempt to compute `_3 / _4`, which would overflow [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/5776f60a::div_signed[0] + [Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/2f0ffa29::div_signed[0] [Crux] attempt to divide `_3` by zero [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/5776f60a::div_unsigned[0] + [Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/2f0ffa29::div_unsigned[0] [Crux] attempt to divide `_3` by zero [Crux] Overall status: Invalid. Use -p '/checked_div/' to rerun this test only. - checked_add: FAIL (0.05s) + checked_add: FAIL (0.04s) test/symb_eval/num/checked_add.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/num/checked_add.rs) Use -p '/checked_add/' to rerun this test only. - test1: FAIL (2.01s) + test1: FAIL (2.14s) files test/symb_eval/scalar/test1.good and test/symb_eval/scalar/test1.out differ; test/symb_eval/scalar/test1.out contains: - test test1/ac4e99fd::f[0]: FAILED + test test1/19389055::f[0]: FAILED failures: - ---- test1/ac4e99fd::f[0] counterexamples ---- + ---- test1/19389055::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in int512/8a4ba723::symbolic[0] - [Crux] panicking::panic, called from int512/8a4ba723::symbolic[0] + [Crux] internal: error: in int512/e5326267::symbolic[0] + [Crux] panicking::panic, called from int512/e5326267::symbolic[0] [Crux] Overall status: Invalid. Use -p '/test1/' to rerun this test only. - pop: FAIL (2.00s) + pop: FAIL (1.97s) files test/symb_eval/vector/pop.good and test/symb_eval/vector/pop.out differ; test/symb_eval/vector/pop.out contains: - test pop/54b55322::f[0]: FAILED + test pop/3f4f93b2::f[0]: FAILED failures: - ---- pop/54b55322::f[0] counterexamples ---- + ---- pop/3f4f93b2::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in pop/54b55322::f[0] - [Crux] Translation error in pop/54b55322::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in pop/3f4f93b2::f[0] + [Crux] Translation error in pop/3f4f93b2::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/Output testing.pop/' to rerun this test only. - as_mut_slice: FAIL (1.93s) + as_mut_slice: FAIL (1.97s) files test/symb_eval/vector/as_mut_slice.good and test/symb_eval/vector/as_mut_slice.out differ; test/symb_eval/vector/as_mut_slice.out contains: - test as_mut_slice/152d9811::f[0]: FAILED + test as_mut_slice/cbac53b5::f[0]: FAILED failures: - ---- as_mut_slice/152d9811::f[0] counterexamples ---- + ---- as_mut_slice/cbac53b5::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in as_mut_slice/152d9811::f[0] - [Crux] Translation error in as_mut_slice/152d9811::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in as_mut_slice/cbac53b5::f[0] + [Crux] Translation error in as_mut_slice/cbac53b5::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/as_mut_slice/' to rerun this test only. - push: FAIL (1.91s) + push: FAIL (1.96s) files test/symb_eval/vector/push.good and test/symb_eval/vector/push.out differ; test/symb_eval/vector/push.out contains: - test push/f42c220a::f[0]: FAILED + test push/ac65ef68::f[0]: FAILED failures: - ---- push/f42c220a::f[0] counterexamples ---- + ---- push/ac65ef68::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in push/f42c220a::f[0] - [Crux] Translation error in push/f42c220a::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in push/ac65ef68::f[0] + [Crux] Translation error in push/ac65ef68::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/Output testing.push/' to rerun this test only. - copy_from_slice: FAIL (1.92s) + copy_from_slice: FAIL (1.97s) files test/symb_eval/vector/copy_from_slice.good and test/symb_eval/vector/copy_from_slice.out differ; test/symb_eval/vector/copy_from_slice.out contains: - test copy_from_slice/97f90fa9::f[0]: FAILED + test copy_from_slice/f4a95669::f[0]: FAILED failures: - ---- copy_from_slice/97f90fa9::f[0] counterexamples ---- + ---- copy_from_slice/f4a95669::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/vector/copy_from_slice.rs:7:43: 7:56: error: in copy_from_slice/97f90fa9::f[0] - [Crux] Translation error in copy_from_slice/97f90fa9::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] test/symb_eval/vector/copy_from_slice.rs:7:43: 7:56: error: in copy_from_slice/f4a95669::f[0] + [Crux] Translation error in copy_from_slice/f4a95669::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/copy_from_slice/' to rerun this test only. - new: FAIL (1.90s) + new: FAIL (1.95s) files test/symb_eval/vector/new.good and test/symb_eval/vector/new.out differ; test/symb_eval/vector/new.out contains: - test new/85d35851::f[0]: FAILED + test new/c088771b::f[0]: FAILED failures: - ---- new/85d35851::f[0] counterexamples ---- + ---- new/c088771b::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in new/85d35851::f[0] - [Crux] Translation error in new/85d35851::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in new/c088771b::f[0] + [Crux] Translation error in new/c088771b::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.new"' to rerun this test only. - mut: FAIL (1.85s) + mut: FAIL (1.90s) files test/symb_eval/vector/mut.good and test/symb_eval/vector/mut.out differ; test/symb_eval/vector/mut.out contains: - test mut/27dffc0f::f[0]: FAILED + test mut/cc259414::f[0]: FAILED failures: - ---- mut/27dffc0f::f[0] counterexamples ---- + ---- mut/cc259414::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in mut/27dffc0f::f[0] - [Crux] Translation error in mut/27dffc0f::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in mut/cc259414::f[0] + [Crux] Translation error in mut/cc259414::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/Output testing.mut/' to rerun this test only. - concat: FAIL (2.00s) + concat: FAIL (1.98s) files test/symb_eval/vector/concat.good and test/symb_eval/vector/concat.out differ; test/symb_eval/vector/concat.out contains: - test concat/ab6becf8::f[0]: FAILED + test concat/fc772d1c::f[0]: FAILED failures: - ---- concat/ab6becf8::f[0] counterexamples ---- + ---- concat/fc772d1c::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in concat/ab6becf8::f[0] - [Crux] Translation error in concat/ab6becf8::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in concat/fc772d1c::f[0] + [Crux] Translation error in concat/fc772d1c::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/concat/' to rerun this test only. - split_at: FAIL (1.90s) + split_at: FAIL (1.92s) files test/symb_eval/vector/split_at.good and test/symb_eval/vector/split_at.out differ; test/symb_eval/vector/split_at.out contains: - test split_at/ee475d7d::f[0]: FAILED + test split_at/9b0f1336::f[0]: FAILED failures: - ---- split_at/ee475d7d::f[0] counterexamples ---- + ---- split_at/9b0f1336::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/vector/split_at.rs:7:43: 7:56: error: in split_at/ee475d7d::f[0] - [Crux] Translation error in split_at/ee475d7d::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] test/symb_eval/vector/split_at.rs:7:43: 7:56: error: in split_at/9b0f1336::f[0] + [Crux] Translation error in split_at/9b0f1336::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/split_at/' to rerun this test only. - replicate: FAIL (1.94s) + replicate: FAIL (1.93s) files test/symb_eval/vector/replicate.good and test/symb_eval/vector/replicate.out differ; test/symb_eval/vector/replicate.out contains: - test replicate/6efeb740::f[0]: FAILED + test replicate/eb069cc4::f[0]: FAILED failures: - ---- replicate/6efeb740::f[0] counterexamples ---- + ---- replicate/eb069cc4::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in replicate/6efeb740::f[0] - [Crux] Translation error in replicate/6efeb740::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in replicate/eb069cc4::f[0] + [Crux] Translation error in replicate/eb069cc4::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/replicate/' to rerun this test only. - as_slice: FAIL (1.93s) + as_slice: FAIL (1.97s) files test/symb_eval/vector/as_slice.good and test/symb_eval/vector/as_slice.out differ; test/symb_eval/vector/as_slice.out contains: - test as_slice/b6304c8a::f[0]: FAILED + test as_slice/d7157304::f[0]: FAILED failures: - ---- as_slice/b6304c8a::f[0] counterexamples ---- + ---- as_slice/d7157304::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in as_slice/b6304c8a::f[0] - [Crux] Translation error in as_slice/b6304c8a::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in as_slice/d7157304::f[0] + [Crux] Translation error in as_slice/d7157304::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/as_slice/' to rerun this test only. - array_mut: FAIL (1.88s) + array_mut: FAIL (1.91s) files test/symb_eval/mux/array_mut.good and test/symb_eval/mux/array_mut.out differ; test/symb_eval/mux/array_mut.out contains: - test array_mut/75102df4::crux_test[0]: FAILED + test array_mut/44a64061::crux_test[0]: FAILED failures: - ---- array_mut/75102df4::crux_test[0] counterexamples ---- + ---- array_mut/44a64061::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '/array_mut/' to rerun this test only. - array: FAIL (1.99s) + array: FAIL (1.98s) files test/symb_eval/mux/array.good and test/symb_eval/mux/array.out differ; test/symb_eval/mux/array.out contains: - test array/a00cfa61::crux_test[0]: FAILED + test array/dd1d0c70::crux_test[0]: FAILED failures: - ---- array/a00cfa61::crux_test[0] counterexamples ---- + ---- array/dd1d0c70::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.array"' to rerun this test only. extend_bytes: FAIL (1.98s) files test/symb_eval/bytes/extend_bytes.good and test/symb_eval/bytes/extend_bytes.out differ; test/symb_eval/bytes/extend_bytes.out contains: - test extend_bytes/d546bdb8::f[0]: FAILED + test extend_bytes/824fdbfb::f[0]: FAILED failures: - ---- extend_bytes/d546bdb8::f[0] counterexamples ---- + ---- extend_bytes/824fdbfb::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312c069f::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/312c069f::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/27dbd76c::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/27dbd76c::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/extend_bytes/' to rerun this test only. - put: FAIL (2.37s) + put: FAIL (2.35s) files test/symb_eval/bytes/put.good and test/symb_eval/bytes/put.out differ; test/symb_eval/bytes/put.out contains: - test put/6d809441::f[0]: FAILED + test put/7a504e32::f[0]: FAILED failures: - ---- put/6d809441::f[0] counterexamples ---- + ---- put/7a504e32::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312c069f::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/312c069f::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/27dbd76c::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/27dbd76c::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.put"' to rerun this test only. - split_to: FAIL (2.34s) + split_to: FAIL (2.39s) files test/symb_eval/bytes/split_to.good and test/symb_eval/bytes/split_to.out differ; test/symb_eval/bytes/split_to.out contains: - test split_to/015724c2::f[0]: FAILED + test split_to/6e6b755e::f[0]: FAILED failures: - ---- split_to/015724c2::f[0] counterexamples ---- + ---- split_to/6e6b755e::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312c069f::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/312c069f::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/27dbd76c::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/27dbd76c::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/split_to/' to rerun this test only. - new: FAIL (1.92s) + new: FAIL (1.96s) files test/symb_eval/bytes/new.good and test/symb_eval/bytes/new.out differ; test/symb_eval/bytes/new.out contains: - test new/91796bdc::f[0]: FAILED + test new/917441ba::f[0]: FAILED failures: - ---- new/91796bdc::f[0] counterexamples ---- + ---- new/917441ba::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312c069f::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/312c069f::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/27dbd76c::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/27dbd76c::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.new"' to rerun this test only. - split_off: FAIL (2.24s) + split_off: FAIL (2.35s) files test/symb_eval/bytes/split_off.good and test/symb_eval/bytes/split_off.out differ; test/symb_eval/bytes/split_off.out contains: - test split_off/55a6890f::f[0]: FAILED + test split_off/1f9c9570::f[0]: FAILED failures: - ---- split_off/55a6890f::f[0] counterexamples ---- + ---- split_off/1f9c9570::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312c069f::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/312c069f::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/27dbd76c::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/27dbd76c::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/split_off/' to rerun this test only. - sym_len: FAIL (2.39s) + sym_len: FAIL (2.46s) files test/symb_eval/bytes/sym_len.good and test/symb_eval/bytes/sym_len.out differ; test/symb_eval/bytes/sym_len.out contains: - test sym_len/5c8517fc::f[0]: FAILED + test sym_len/394525b8::f[0]: FAILED failures: - ---- sym_len/5c8517fc::f[0] counterexamples ---- + ---- sym_len/394525b8::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312c069f::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/312c069f::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/27dbd76c::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/27dbd76c::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/sym_len/' to rerun this test only. - put_overflow: FAIL (2.31s) + put_overflow: FAIL (2.32s) files test/symb_eval/bytes/put_overflow.good and test/symb_eval/bytes/put_overflow.out differ; test/symb_eval/bytes/put_overflow.out contains: - test put_overflow/382878d8::f[0]: FAILED + test put_overflow/bab83c29::f[0]: FAILED failures: - ---- put_overflow/382878d8::f[0] counterexamples ---- + ---- put_overflow/bab83c29::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/312c069f::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/312c069f::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/27dbd76c::{impl#1}[0]::with_capacity[0] + [Crux] Translation error in bytes/27dbd76c::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/put_overflow/' to rerun this test only. - vec_cursor_read: FAIL (2.14s) - test/symb_eval/io/vec_cursor_read.out: withFile: user error (JSON Decoding of test/symb_eval/io/vec_cursor_read.all.mir failed: Error in $.statics[6].rendered: parseJSON - bad rendered constant kind: Just (String "slice")) - Use -p '/vec_cursor_read/' to rerun this test only. - vec_write: FAIL (2.24s) - standalone use of `dyn` is not supported: TyDynamic core/95754f0e::error[0]::Error[0]::_trait56353a7b840403b4[0] + vec_cursor_read: FAIL (2.91s) + standalone use of `dyn` is not supported: TyDynamic core/feeb1386::any[0]::Erased[0]::_trait58656ccc69864acf[0] CallStack (from HasCallStack): error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/Trans.hs:824:12 in crucible-mir-0.1-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:892:5 in crucible-mir-0.1-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:944:30 in crucible-mir-0.1-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1157:11 in crucible-mir-0.1-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1602:68 in crucible-mir-0.1-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1611:28 in crucible-mir-0.1-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1651:10 in crucible-mir-0.1-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1697:24 in crucible-mir-0.1-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2131:30 in crucible-mir-0.1-inplace:Mir.Trans + tyToRepr, called at src/Mir/TransTy.hs:213:7 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:402:29 in crucible-mir-0.1-inplace:Mir.TransTy + tyToReprCont, called at src/Mir/TransTy.hs:202:24 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy + tyListToCtx, called at src/Mir/TransTy.hs:221:6 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy + tyListToCtx, called at src/Mir/TransTy.hs:1066:16 in crucible-mir-0.1-inplace:Mir.TransTy + traitVtableType, called at src/Mir/Trans.hs:2030:18 in crucible-mir-0.1-inplace:Mir.Trans + transVirtCall, called at src/Mir/Trans.hs:2153:31 in crucible-mir-0.1-inplace:Mir.Trans transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language + Use -p '/vec_cursor_read/' to rerun this test only. + vec_write: FAIL (2.50s) + files test/symb_eval/io/vec_write.good and test/symb_eval/io/vec_write.out differ; test/symb_eval/io/vec_write.out contains: + test vec_write/f46d72c4::f[0]: ok + + [Crux] Overall status: Valid. + Use -p '/vec_write/' to rerun this test only. write: FAIL test/symb_eval/byteorder/write.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/write.rs) Use -p '/Output testing.write/' to rerun this test only. - read: FAIL + read: FAIL (0.06s) test/symb_eval/byteorder/read.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/read.rs) Use -p '/Output testing.read/' to rerun this test only. slice_mut: FAIL (1.93s) files test/symb_eval/array/slice_mut.good and test/symb_eval/array/slice_mut.out differ; test/symb_eval/array/slice_mut.out contains: - test slice_mut/04d49b74::crux_test[0]: FAILED + test slice_mut/1cfe8832::crux_test[0]: FAILED failures: - ---- slice_mut/04d49b74::crux_test[0] counterexamples ---- + ---- slice_mut/1cfe8832::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in slice_mut/04d49b74::crux_test[0] - [Crux] Translation error in slice_mut/04d49b74::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/7940c2ae::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/7940c2ae::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/7940c2ae::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) + [Crux] internal: error: in slice_mut/1cfe8832::crux_test[0] + [Crux] Translation error in slice_mut/1cfe8832::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/2c868646::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/2c868646::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/2c868646::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/2c868646::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/slice_mut/' to rerun this test only. - basic: FAIL (1.91s) + basic: FAIL (1.98s) files test/symb_eval/array/basic.good and test/symb_eval/array/basic.out differ; test/symb_eval/array/basic.out contains: - test basic/e501fd67::crux_test[0]: FAILED + test basic/e9644cdf::crux_test[0]: FAILED failures: - ---- basic/e501fd67::crux_test[0] counterexamples ---- + ---- basic/e9644cdf::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in basic/e501fd67::crux_test[0] - [Crux] Translation error in basic/e501fd67::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/7940c2ae::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/7940c2ae::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/7940c2ae::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/7940c2ae::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) + [Crux] internal: error: in basic/e9644cdf::crux_test[0] + [Crux] Translation error in basic/e9644cdf::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/2c868646::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/2c868646::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/2c868646::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/2c868646::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/Output testing.basic/' to rerun this test only. - slice: FAIL (1.89s) + slice: FAIL (1.97s) files test/symb_eval/array/slice.good and test/symb_eval/array/slice.out differ; test/symb_eval/array/slice.out contains: - test slice/84738852::crux_test[0]: FAILED + test slice/22115682::crux_test[0]: FAILED failures: - ---- slice/84738852::crux_test[0] counterexamples ---- + ---- slice/22115682::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in slice/84738852::crux_test[0] - [Crux] Translation error in slice/84738852::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/7940c2ae::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/7940c2ae::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/7940c2ae::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) + [Crux] internal: error: in slice/22115682::crux_test[0] + [Crux] Translation error in slice/22115682::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/2c868646::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/2c868646::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/2c868646::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/2c868646::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.slice"' to rerun this test only. - mux_slice: FAIL (1.83s) + mux_slice: FAIL (1.95s) files test/symb_eval/array/mux_slice.good and test/symb_eval/array/mux_slice.out differ; test/symb_eval/array/mux_slice.out contains: - test mux_slice/7df60048::crux_test[0]: FAILED + test mux_slice/3c0a096f::crux_test[0]: FAILED failures: - ---- mux_slice/7df60048::crux_test[0] counterexamples ---- + ---- mux_slice/3c0a096f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in mux_slice/7df60048::crux_test[0] - [Crux] Translation error in mux_slice/7df60048::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/7940c2ae::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/7940c2ae::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/7940c2ae::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) + [Crux] internal: error: in mux_slice/3c0a096f::crux_test[0] + [Crux] Translation error in mux_slice/3c0a096f::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/2c868646::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/2c868646::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/2c868646::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/2c868646::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/mux_slice/' to rerun this test only. - mux: FAIL (1.88s) + mux: FAIL (1.96s) files test/symb_eval/enum/mux.good and test/symb_eval/enum/mux.out differ; test/symb_eval/enum/mux.out contains: - test mux/b88b2c60::test[0]: FAILED + test mux/cb5d0cbe::test[0]: FAILED failures: - ---- mux/b88b2c60::test[0] counterexamples ---- + ---- mux/cb5d0cbe::test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u8[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.mux"' to rerun this test only. - uninit_read: FAIL (1.88s) + uninit_read: FAIL (1.92s) files test/symb_eval/alloc/uninit_read.good and test/symb_eval/alloc/uninit_read.out differ; test/symb_eval/alloc/uninit_read.out contains: - test uninit_read/1541c194::crux_test[0]: FAILED + test uninit_read/9ace772b::crux_test[0]: FAILED failures: - ---- uninit_read/1541c194::crux_test[0] counterexamples ---- + ---- uninit_read/9ace772b::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/1541c194::crux_test[0] + [Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/9ace772b::crux_test[0] [Crux] Attempted to read uninitialized vector index [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/1541c194::crux_test[0] + [Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/9ace772b::crux_test[0] [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. Use -p '/uninit_read/' to rerun this test only. - valid_read: FAIL (1.82s) + valid_read: FAIL (1.94s) files test/symb_eval/alloc/valid_read.good and test/symb_eval/alloc/valid_read.out differ; test/symb_eval/alloc/valid_read.out contains: - test valid_read/959d5d3a::crux_test[0]: returned 45, ok + test valid_read/f240ed17::crux_test[0]: returned 45, ok [Crux] Overall status: Valid. Use -p '/valid_read/' to rerun this test only. - out_of_bounds: FAIL (1.95s) + out_of_bounds: FAIL (1.99s) files test/symb_eval/alloc/out_of_bounds.good and test/symb_eval/alloc/out_of_bounds.out differ; test/symb_eval/alloc/out_of_bounds.out contains: - test out_of_bounds/b6cdb317::crux_test[0]: FAILED + test out_of_bounds/1e583a40::crux_test[0]: FAILED failures: - ---- out_of_bounds/b6cdb317::crux_test[0] counterexamples ---- + ---- out_of_bounds/1e583a40::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/b6cdb317::crux_test[0] + [Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/1e583a40::crux_test[0] [Crux] vector index out of range: the length is 10 but the index is BV 12 [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/b6cdb317::crux_test[0] + [Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/1e583a40::crux_test[0] [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. @@ -2661,27 +2492,27 @@ FAIL (0.05s) Use -p '/out_of_bounds/' to rerun this test only. zero_length: FAIL (1.89s) files test/symb_eval/alloc/zero_length.good and test/symb_eval/alloc/zero_length.out differ; test/symb_eval/alloc/zero_length.out contains: - test zero_length/fb4e0bb6::crux_test[0]: ok + test zero_length/54f30b76::crux_test[0]: ok [Crux] Overall status: Valid. Use -p '/zero_length/' to rerun this test only. - downcast: FAIL (1.77s) + downcast: FAIL (1.89s) files test/symb_eval/any/downcast.good and test/symb_eval/any/downcast.out differ; test/symb_eval/any/downcast.out contains: - test downcast/547ed877::crux_test[0]: returned 1, ok + test downcast/8170112c::crux_test[0]: returned 1, ok [Crux] Overall status: Valid. Use -p '$0=="crux-mir.crux symbolic.Output testing.downcast"' to rerun this test only. - downcast_fail: FAIL (1.86s) + downcast_fail: FAIL (1.90s) files test/symb_eval/any/downcast_fail.good and test/symb_eval/any/downcast_fail.out differ; test/symb_eval/any/downcast_fail.out contains: - test downcast_fail/78cdfd33::crux_test[0]: FAILED + test downcast_fail/21cf5aa1::crux_test[0]: FAILED failures: - ---- downcast_fail/78cdfd33::crux_test[0] counterexamples ---- + ---- downcast_fail/21cf5aa1::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/any/downcast_fail.rs:10:14: 10:15: error: in downcast_fail/78cdfd33::crux_test[0] + [Crux] test/symb_eval/any/downcast_fail.rs:10:14: 10:15: error: in downcast_fail/21cf5aa1::crux_test[0] [Crux] failed to downcast Any as BVRepr 32 [Crux] Overall status: Invalid. @@ -2690,184 +2521,184 @@ FAIL (0.05s) conditional: FAIL (0.04s) test/symb_eval/any/conditional.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/any/conditional.rs) Use -p '/conditional/' to rerun this test only. - literals: FAIL (1.79s) + literals: FAIL (1.88s) files test/symb_eval/bitvector/literals.good and test/symb_eval/bitvector/literals.out differ; test/symb_eval/bitvector/literals.out contains: - test literals/2f1b5a7b::crux_test[0]: FAILED + test literals/9e53c8fd::crux_test[0]: FAILED failures: - ---- literals/2f1b5a7b::crux_test[0] counterexamples ---- + ---- literals/9e53c8fd::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:91:29: 91:30 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:125:1: 125:53: error: in crucible/7940c2ae::bitvector[0]::{impl#27}[0]::from[0]::_inst9ea2f935eda83dab[0] - [Crux] Translation error in crucible/7940c2ae::bitvector[0]::{impl#27}[0]::from[0]::_inst9ea2f935eda83dab[0]: BUG: invalid arguments to bv_convert: [$38: AnyRepr] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:91:29: 91:30 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:125:1: 125:53: error: in crucible/2c868646::bitvector[0]::{impl#27}[0]::from[0]::_inst9ea2f935eda83dab[0] + [Crux] Translation error in crucible/2c868646::bitvector[0]::{impl#27}[0]::from[0]::_inst9ea2f935eda83dab[0]: BUG: invalid arguments to bv_convert: [$38: AnyRepr] [Crux] Overall status: Invalid. Use -p '/literals/' to rerun this test only. - arith: FAIL (1.92s) + arith: FAIL (1.96s) files test/symb_eval/bitvector/arith.good and test/symb_eval/bitvector/arith.out differ; test/symb_eval/bitvector/arith.out contains: - test arith/dd407607::crux_test[0]: FAILED + test arith/92eb28c3::crux_test[0]: FAILED failures: - ---- arith/dd407607::crux_test[0] counterexamples ---- + ---- arith/92eb28c3::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u64[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u64[0] [Crux] Overall status: Invalid. Use -p '/Output testing.arith/' to rerun this test only. - leading_zeros: FAIL (1.85s) + leading_zeros: FAIL (1.94s) files test/symb_eval/bitvector/leading_zeros.good and test/symb_eval/bitvector/leading_zeros.out differ; test/symb_eval/bitvector/leading_zeros.out contains: - test leading_zeros/fde29bba::crux_test[0]: FAILED + test leading_zeros/a2716535::crux_test[0]: FAILED failures: - ---- leading_zeros/fde29bba::crux_test[0] counterexamples ---- + ---- leading_zeros/a2716535::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u64[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u64[0] [Crux] Overall status: Invalid. Use -p '/leading_zeros/' to rerun this test only. - overflowing_sub: FAIL (1.83s) + overflowing_sub: FAIL (1.93s) files test/symb_eval/bitvector/overflowing_sub.good and test/symb_eval/bitvector/overflowing_sub.out differ; test/symb_eval/bitvector/overflowing_sub.out contains: - test overflowing_sub/308b68e3::crux_test[0]: FAILED + test overflowing_sub/041a4211::crux_test[0]: FAILED failures: - ---- overflowing_sub/308b68e3::crux_test[0] counterexamples ---- + ---- overflowing_sub/041a4211::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u64[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u64[0] [Crux] Overall status: Invalid. Use -p '/overflowing_sub/' to rerun this test only. - cmp: FAIL (1.85s) + cmp: FAIL (1.98s) files test/symb_eval/bitvector/cmp.good and test/symb_eval/bitvector/cmp.out differ; test/symb_eval/bitvector/cmp.out contains: - test cmp/76eaaca4::crux_test[0]: FAILED + test cmp/c8db4f55::crux_test[0]: FAILED failures: - ---- cmp/76eaaca4::crux_test[0] counterexamples ---- + ---- cmp/c8db4f55::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::symbolic[0]::symbolic_u64[0] + [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u64[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u64[0] [Crux] Overall status: Invalid. Use -p '/Output testing.cmp/' to rerun this test only. - symbolic: FAIL (1.72s) + symbolic: FAIL (1.88s) files test/symb_eval/bitvector/symbolic.good and test/symb_eval/bitvector/symbolic.out differ; test/symb_eval/bitvector/symbolic.out contains: - test symbolic/998167ef::crux_test[0]: FAILED + test symbolic/87036093::crux_test[0]: FAILED failures: - ---- symbolic/998167ef::crux_test[0] counterexamples ---- + ---- symbolic/87036093::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::bitvector[0]::make_symbolic_256[0] - [Crux] panicking::panic, called from crucible/7940c2ae::bitvector[0]::make_symbolic_256[0] + [Crux] internal: error: in crucible/2c868646::bitvector[0]::make_symbolic_256[0] + [Crux] panicking::panic, called from crucible/2c868646::bitvector[0]::make_symbolic_256[0] [Crux] Overall status: Invalid. Use -p '/Output testing.symbolic/' to rerun this test only. - from_to: FAIL (1.71s) + from_to: FAIL (1.82s) files test/symb_eval/bitvector/from_to.good and test/symb_eval/bitvector/from_to.out differ; test/symb_eval/bitvector/from_to.out contains: - test from_to/43b66088::crux_test[0]: FAILED + test from_to/a99ea3e1::crux_test[0]: FAILED failures: - ---- from_to/43b66088::crux_test[0] counterexamples ---- + ---- from_to/a99ea3e1::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:103:29: 103:30 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:125:1: 125:53: error: in crucible/7940c2ae::bitvector[0]::{impl#21}[0]::from[0]::_inst9ea2f935eda83dab[0] - [Crux] Translation error in crucible/7940c2ae::bitvector[0]::{impl#21}[0]::from[0]::_inst9ea2f935eda83dab[0]: BUG: invalid arguments to bv_convert: [$34: BVRepr 64] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:103:29: 103:30 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:125:1: 125:53: error: in crucible/2c868646::bitvector[0]::{impl#21}[0]::from[0]::_inst9ea2f935eda83dab[0] + [Crux] Translation error in crucible/2c868646::bitvector[0]::{impl#21}[0]::from[0]::_inst9ea2f935eda83dab[0]: BUG: invalid arguments to bv_convert: [$34: BVRepr 64] [Crux] Overall status: Invalid. Use -p '/from_to/' to rerun this test only. - clone: FAIL (2.02s) + clone: FAIL (2.01s) files test/symb_eval/vec/clone.good and test/symb_eval/vec/clone.out differ; test/symb_eval/vec/clone.out contains: - test clone/e5273a68::f[0]: FAILED + test clone/725f93b1::f[0]: FAILED failures: - ---- clone/e5273a68::f[0] counterexamples ---- + ---- clone/725f93b1::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. Use -p '/Output testing.clone/' to rerun this test only. - sort_by_key: FAIL (2.58s) + sort_by_key: FAIL (2.83s) files test/symb_eval/vec/sort_by_key.good and test/symb_eval/vec/sort_by_key.out differ; test/symb_eval/vec/sort_by_key.out contains: - test sort_by_key/ee1d6dec::f[0]: FAILED + test sort_by_key/525a0c32::f[0]: FAILED failures: - ---- sort_by_key/ee1d6dec::f[0] counterexamples ---- + ---- sort_by_key/525a0c32::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. Use -p '/sort_by_key/' to rerun this test only. - into_iter: FAIL (2.55s) + into_iter: FAIL (2.59s) files test/symb_eval/vec/into_iter.good and test/symb_eval/vec/into_iter.out differ; test/symb_eval/vec/into_iter.out contains: - test into_iter/f0e581e8::f[0]: FAILED + test into_iter/addfe790::f[0]: FAILED failures: - ---- into_iter/f0e581e8::f[0] counterexamples ---- + ---- into_iter/addfe790::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. Use -p '/into_iter/' to rerun this test only. - macro: FAIL (1.88s) + macro: FAIL (2.06s) files test/symb_eval/vec/macro.good and test/symb_eval/vec/macro.out differ; test/symb_eval/vec/macro.out contains: - test macro/4069544e::f[0]: FAILED + test macro/5e6d3881::f[0]: FAILED failures: - ---- macro/4069544e::f[0] counterexamples ---- + ---- macro/5e6d3881::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/alignment.rs:89:53: 89:58: error: in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0] - [Crux] Translation error in core/95754f0e::ptr[0]::alignment[0]::{impl#0}[0]::new_unchecked[0]: callExp: Don't know how to call core/95754f0e::intrinsics[0]::{extern#0}[0]::transmute[0]::_instfab2666815496a23[0] + [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. Use -p '/Output testing.macro/' to rerun this test only. - construct: FAIL (1.91s) + construct: FAIL (1.96s) files test/symb_eval/sym_bytes/construct.good and test/symb_eval/sym_bytes/construct.out differ; test/symb_eval/sym_bytes/construct.out contains: - test construct/4a1d0528::crux_test[0]: FAILED + test construct/e7a21663::crux_test[0]: FAILED failures: - ---- construct/4a1d0528::crux_test[0] counterexamples ---- + ---- construct/e7a21663::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::sym_bytes[0]::{impl#0}[0]::zeroed[0] - [Crux] Translation error in crucible/7940c2ae::sym_bytes[0]::{impl#0}[0]::zeroed[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 8) to AnyRepr (TyAdt crucible/7940c2ae::array[0]::Array[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::array[0]::Array[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/7940c2ae::array[0]::Array[0]::_adtaddce72e1232152c[0] crucible/7940c2ae::array[0]::Array[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in crucible/2c868646::sym_bytes[0]::{impl#0}[0]::zeroed[0] + [Crux] Translation error in crucible/2c868646::sym_bytes[0]::{impl#0}[0]::zeroed[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 8) to AnyRepr (TyAdt crucible/2c868646::array[0]::Array[0]::_adtaddce72e1232152c[0] crucible/2c868646::array[0]::Array[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/2c868646::array[0]::Array[0]::_adtaddce72e1232152c[0] crucible/2c868646::array[0]::Array[0] (Substs [TyUint B8]), _varIsZST = False}) [Crux] Overall status: Invalid. Use -p '/construct/' to rerun this test only. - deserialize: FAIL (1.82s) + deserialize: FAIL (1.91s) files test/symb_eval/sym_bytes/deserialize.good and test/symb_eval/sym_bytes/deserialize.out differ; test/symb_eval/sym_bytes/deserialize.out contains: - test deserialize/34a68134::crux_test[0]: FAILED + test deserialize/f2eb315f::crux_test[0]: FAILED failures: - ---- deserialize/34a68134::crux_test[0] counterexamples ---- + ---- deserialize/f2eb315f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/7940c2ae::array[0]::symbolic[0]::_instaddce72e1232152c[0] - [Crux] panicking::panic_fmt, called from crucible/7940c2ae::array[0]::symbolic[0]::_instaddce72e1232152c[0] + [Crux] internal: error: in crucible/2c868646::array[0]::symbolic[0]::_instaddce72e1232152c[0] + [Crux] panicking::panic_fmt, called from crucible/2c868646::array[0]::symbolic[0]::_instaddce72e1232152c[0] [Crux] Overall status: Invalid. @@ -2890,8 +2721,8 @@ FAIL (0.05s) thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_dual/2d5bca87::crux_test2[0]/report_data.js" and "test/coverage/out/coverage_dual/30f9fe77::crux_test1[0]/report_data.js" do not match', src/main.rs:825:13 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (1.81s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_dual/30f9fe77::crux_test1[0]/report_data.js" "test/coverage/out/coverage_dual/2d5bca87::crux_test2[0]/report_data.js" "test/coverage/out/coverage_dual/4ed2e432::crux_test1[0]/report_data.js" "test/coverage/out/coverage_dual/2d5bca87::crux_test1[0]/report_data.js" "test/coverage/out/coverage_dual/30f9fe77::crux_test2[0]/report_data.js" "test/coverage/out/coverage_dual/4ed2e432::crux_test2[0]/report_data.js" (exit 101): failed +FAIL (1.88s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_dual/30f9fe77::crux_test1[0]/report_data.js" "test/coverage/out/coverage_dual/2d5bca87::crux_test2[0]/report_data.js" "test/coverage/out/coverage_dual/4ed2e432::crux_test1[0]/report_data.js" "test/coverage/out/coverage_dual/cbc8b50e::crux_test2[0]/report_data.js" "test/coverage/out/coverage_dual/2d5bca87::crux_test1[0]/report_data.js" "test/coverage/out/coverage_dual/83032ba6::crux_test1[0]/report_data.js" "test/coverage/out/coverage_dual/30f9fe77::crux_test2[0]/report_data.js" "test/coverage/out/coverage_dual/4ed2e432::crux_test2[0]/report_data.js" "test/coverage/out/coverage_dual/83032ba6::crux_test2[0]/report_data.js" "test/coverage/out/coverage_dual/cbc8b50e::crux_test1[0]/report_data.js" (exit 101): failed Use -p '/coverage_dual/' to rerun this test only. coverage: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 @@ -2909,8 +2740,8 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage/c30e4ff4::crux_test[0]/report_data.js" and "test/coverage/out/coverage/0fe46bcc::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (2.04s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage/0fe46bcc::crux_test[0]/report_data.js" "test/coverage/out/coverage/c30e4ff4::crux_test[0]/report_data.js" "test/coverage/out/coverage/cb3943dc::crux_test[0]/report_data.js" (exit 101): failed +FAIL (2.01s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage/0fe46bcc::crux_test[0]/report_data.js" "test/coverage/out/coverage/c30e4ff4::crux_test[0]/report_data.js" "test/coverage/out/coverage/ba917dd2::crux_test[0]/report_data.js" "test/coverage/out/coverage/cb3943dc::crux_test[0]/report_data.js" "test/coverage/out/coverage/b5488ae7::crux_test[0]/report_data.js" (exit 101): failed Use -p '$0=="crux-mir.crux coverage.Output testing.coverage"' to rerun this test only. coverage_try: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 @@ -2928,8 +2759,8 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_try/853a6496::crux_test[0]/report_data.js" and "test/coverage/out/coverage_try/d0e6f11f::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (1.94s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_try/d0e6f11f::crux_test[0]/report_data.js" "test/coverage/out/coverage_try/853a6496::crux_test[0]/report_data.js" "test/coverage/out/coverage_try/a9dc15f2::crux_test[0]/report_data.js" (exit 101): failed +FAIL (2.01s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_try/d0e6f11f::crux_test[0]/report_data.js" "test/coverage/out/coverage_try/853a6496::crux_test[0]/report_data.js" "test/coverage/out/coverage_try/c9abcb4f::crux_test[0]/report_data.js" "test/coverage/out/coverage_try/a9dc15f2::crux_test[0]/report_data.js" "test/coverage/out/coverage_try/28ff1843::crux_test[0]/report_data.js" (exit 101): failed Use -p '/coverage_try/' to rerun this test only. coverage_cond: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 @@ -2945,10 +2776,10 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_cond/2c579738::crux_test[0]/report_data.js" and "test/coverage/out/coverage_cond/3dfc8004::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 +thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_cond/133a307f::crux_test[0]/report_data.js" and "test/coverage/out/coverage_cond/3dfc8004::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (1.79s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_cond/3dfc8004::crux_test[0]/report_data.js" "test/coverage/out/coverage_cond/2c579738::crux_test[0]/report_data.js" "test/coverage/out/coverage_cond/3f380757::crux_test[0]/report_data.js" (exit 101): failed +FAIL (1.90s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_cond/3dfc8004::crux_test[0]/report_data.js" "test/coverage/out/coverage_cond/133a307f::crux_test[0]/report_data.js" "test/coverage/out/coverage_cond/2c579738::crux_test[0]/report_data.js" "test/coverage/out/coverage_cond/3f380757::crux_test[0]/report_data.js" "test/coverage/out/coverage_cond/b4b5a400::crux_test[0]/report_data.js" (exit 101): failed Use -p '/coverage_cond/' to rerun this test only. coverage_loop: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 @@ -2964,10 +2795,10 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_loop/bf94e0f1::crux_test[0]/report_data.js" and "test/coverage/out/coverage_loop/fff32a45::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 +thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_loop/fff32a45::crux_test[0]/report_data.js" and "test/coverage/out/coverage_loop/04b47bd3::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (1.87s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_loop/fff32a45::crux_test[0]/report_data.js" "test/coverage/out/coverage_loop/bf94e0f1::crux_test[0]/report_data.js" "test/coverage/out/coverage_loop/81f6cac8::crux_test[0]/report_data.js" (exit 101): failed +FAIL (1.94s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_loop/04b47bd3::crux_test[0]/report_data.js" "test/coverage/out/coverage_loop/fff32a45::crux_test[0]/report_data.js" "test/coverage/out/coverage_loop/bf94e0f1::crux_test[0]/report_data.js" "test/coverage/out/coverage_loop/09255ce2::crux_test[0]/report_data.js" "test/coverage/out/coverage_loop/81f6cac8::crux_test[0]/report_data.js" (exit 101): failed Use -p '/coverage_loop/' to rerun this test only. coverage_shortcircuit: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 @@ -2983,10 +2814,10 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_shortcircuit/e95f8b07::crux_test[0]/report_data.js" and "test/coverage/out/coverage_shortcircuit/5762c1e1::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 +thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_shortcircuit/5762c1e1::crux_test[0]/report_data.js" and "test/coverage/out/coverage_shortcircuit/86a0f0fa::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (1.90s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_shortcircuit/5762c1e1::crux_test[0]/report_data.js" "test/coverage/out/coverage_shortcircuit/e95f8b07::crux_test[0]/report_data.js" "test/coverage/out/coverage_shortcircuit/f90f762e::crux_test[0]/report_data.js" (exit 101): failed +FAIL (1.98s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_shortcircuit/86a0f0fa::crux_test[0]/report_data.js" "test/coverage/out/coverage_shortcircuit/5762c1e1::crux_test[0]/report_data.js" "test/coverage/out/coverage_shortcircuit/466e729a::crux_test[0]/report_data.js" "test/coverage/out/coverage_shortcircuit/e95f8b07::crux_test[0]/report_data.js" "test/coverage/out/coverage_shortcircuit/f90f762e::crux_test[0]/report_data.js" (exit 101): failed Use -p '/coverage_shortcircuit/' to rerun this test only. coverage_mono: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 @@ -3004,8 +2835,8 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_mono/94654c01::crux_test[0]/report_data.js" and "test/coverage/out/coverage_mono/564877e0::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (1.81s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_mono/564877e0::crux_test[0]/report_data.js" "test/coverage/out/coverage_mono/94654c01::crux_test[0]/report_data.js" "test/coverage/out/coverage_mono/d7f56dd4::crux_test[0]/report_data.js" (exit 101): failed +FAIL (1.89s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_mono/564877e0::crux_test[0]/report_data.js" "test/coverage/out/coverage_mono/94654c01::crux_test[0]/report_data.js" "test/coverage/out/coverage_mono/03e46ae4::crux_test[0]/report_data.js" "test/coverage/out/coverage_mono/d8d1fa00::crux_test[0]/report_data.js" "test/coverage/out/coverage_mono/d7f56dd4::crux_test[0]/report_data.js" (exit 101): failed Use -p '/coverage_mono/' to rerun this test only. coverage_match: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 @@ -3021,10 +2852,10 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_match/63e02e94::crux_test[0]/report_data.js" and "test/coverage/out/coverage_match/e0a88aee::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 +thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_match/de962e25::crux_test[0]/report_data.js" and "test/coverage/out/coverage_match/eeac3067::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (1.92s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_match/e0a88aee::crux_test[0]/report_data.js" "test/coverage/out/coverage_match/63e02e94::crux_test[0]/report_data.js" "test/coverage/out/coverage_match/2b40efd1::crux_test[0]/report_data.js" (exit 101): failed +FAIL (1.96s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_match/eeac3067::crux_test[0]/report_data.js" "test/coverage/out/coverage_match/de962e25::crux_test[0]/report_data.js" "test/coverage/out/coverage_match/e0a88aee::crux_test[0]/report_data.js" "test/coverage/out/coverage_match/63e02e94::crux_test[0]/report_data.js" "test/coverage/out/coverage_match/2b40efd1::crux_test[0]/report_data.js" (exit 101): failed Use -p '/coverage_match/' to rerun this test only. coverage_macro: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 @@ -3042,8 +2873,8 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_macro/ff936904::crux_test[0]/report_data.js" and "test/coverage/out/coverage_macro/e538cb6e::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (1.92s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_macro/e538cb6e::crux_test[0]/report_data.js" "test/coverage/out/coverage_macro/ff936904::crux_test[0]/report_data.js" "test/coverage/out/coverage_macro/54bd500f::crux_test[0]/report_data.js" (exit 101): failed +FAIL (2.01s) + readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_macro/e538cb6e::crux_test[0]/report_data.js" "test/coverage/out/coverage_macro/ff936904::crux_test[0]/report_data.js" "test/coverage/out/coverage_macro/115b4d60::crux_test[0]/report_data.js" "test/coverage/out/coverage_macro/b70cc90f::crux_test[0]/report_data.js" "test/coverage/out/coverage_macro/54bd500f::crux_test[0]/report_data.js" (exit 101): failed Use -p '/coverage_macro/' to rerun this test only. -150 out of 339 tests failed (656.22s) +135 out of 339 tests failed (682.81s) From 76c5d1926cf60c655e27a3b2f6321065e140552a Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 18 May 2023 15:01:02 -0400 Subject: [PATCH 049/114] crux-mir: Repair some test cases with &dyn --- crux-mir/test/conc_eval/traits/dynamic_branch.rs | 2 +- crux-mir/test/conc_eval/traits/dynamic_med.rs | 2 +- crux-mir/test/conc_eval/traits/dynamic_poly.rs | 2 +- crux-mir/test/conc_eval/traits/dynamic_two.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crux-mir/test/conc_eval/traits/dynamic_branch.rs b/crux-mir/test/conc_eval/traits/dynamic_branch.rs index 9e02324e6..aa22ff491 100644 --- a/crux-mir/test/conc_eval/traits/dynamic_branch.rs +++ b/crux-mir/test/conc_eval/traits/dynamic_branch.rs @@ -19,7 +19,7 @@ impl Foo for () { } -fn fun(f: &Foo) -> u32 { +fn fun(f: &dyn Foo) -> u32 { return f.foo(); } diff --git a/crux-mir/test/conc_eval/traits/dynamic_med.rs b/crux-mir/test/conc_eval/traits/dynamic_med.rs index 13c715502..c8d0ea973 100644 --- a/crux-mir/test/conc_eval/traits/dynamic_med.rs +++ b/crux-mir/test/conc_eval/traits/dynamic_med.rs @@ -21,7 +21,7 @@ impl Foo for Data { -fn fun(f: &Foo) -> u32 { +fn fun(f: &dyn Foo) -> u32 { f.foo() } diff --git a/crux-mir/test/conc_eval/traits/dynamic_poly.rs b/crux-mir/test/conc_eval/traits/dynamic_poly.rs index 57c4ae687..c6e535a1c 100644 --- a/crux-mir/test/conc_eval/traits/dynamic_poly.rs +++ b/crux-mir/test/conc_eval/traits/dynamic_poly.rs @@ -13,7 +13,7 @@ impl Foo for Data { -fn h(f: &Foo) -> u32 { +fn h(f: &dyn Foo) -> u32 { f.foo(2) } diff --git a/crux-mir/test/conc_eval/traits/dynamic_two.rs b/crux-mir/test/conc_eval/traits/dynamic_two.rs index 33b9efaeb..850ffd010 100644 --- a/crux-mir/test/conc_eval/traits/dynamic_two.rs +++ b/crux-mir/test/conc_eval/traits/dynamic_two.rs @@ -26,7 +26,7 @@ impl Bar for Data { -fn fun(f: &Foo) -> u32 { +fn fun(f: &dyn Foo) -> u32 { f.foo() } From 098d34172e7ec530ab145d656c0518b3a4e0510b Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 18 May 2023 15:28:34 -0400 Subject: [PATCH 050/114] fn_dyn.rs: Use &dyn --- crux-mir/test/conc_eval/clos/fn_dyn.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crux-mir/test/conc_eval/clos/fn_dyn.rs b/crux-mir/test/conc_eval/clos/fn_dyn.rs index a98152fb6..0a3fef8cf 100644 --- a/crux-mir/test/conc_eval/clos/fn_dyn.rs +++ b/crux-mir/test/conc_eval/clos/fn_dyn.rs @@ -1,6 +1,6 @@ // FAIL: call_once shim (in `dyn Fn` vtable) #![cfg_attr(not(with_main), no_std)] -fn call_with_one(some_closure: &Fn(i32) -> i32) -> i32 { +fn call_with_one(some_closure: &dyn Fn(i32) -> i32) -> i32 { some_closure(1) } From ea56c95e1f475c3b62dc2bcba2af92c30e746602 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 18 May 2023 15:52:56 -0400 Subject: [PATCH 051/114] crux-mir: Repair checked_* test cases --- crux-mir/test/symb_eval/num/checked_add.good | 8 ++++---- crux-mir/test/symb_eval/num/checked_add.rs | 1 + crux-mir/test/symb_eval/num/checked_div.good | 16 ++++++++-------- crux-mir/test/symb_eval/num/checked_mul.good | 8 ++++---- crux-mir/test/symb_eval/num/checked_mul.rs | 2 ++ .../test/symb_eval/num/checked_mul_signed.good | 8 ++++---- .../test/symb_eval/num/checked_mul_signed.rs | 2 ++ 7 files changed, 25 insertions(+), 20 deletions(-) diff --git a/crux-mir/test/symb_eval/num/checked_add.good b/crux-mir/test/symb_eval/num/checked_add.good index 6c0c35192..996a449f9 100644 --- a/crux-mir/test/symb_eval/num/checked_add.good +++ b/crux-mir/test/symb_eval/num/checked_add.good @@ -1,10 +1,10 @@ -test checked_add/3a1fbbbh::crux_test[0]: returned 44, FAILED +test checked_add/ec6fd7f2::crux_test[0]: returned 44, FAILED failures: ----- checked_add/3a1fbbbh::crux_test[0] counterexamples ---- +---- checked_add/ec6fd7f2::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_add.rs:5:5: 5:12: error: in checked_add/3a1fbbbh::crux_test[0] -[Crux] attempt to add with overflow +[Crux] test/symb_eval/num/checked_add.rs:6:5: 6:12: error: in checked_add/ec6fd7f2::crux_test[0] +[Crux] attempt to compute `const 100_u8 + move _2`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/num/checked_add.rs b/crux-mir/test/symb_eval/num/checked_add.rs index 75a1643f5..d5980d345 100644 --- a/crux-mir/test/symb_eval/num/checked_add.rs +++ b/crux-mir/test/symb_eval/num/checked_add.rs @@ -1,3 +1,4 @@ +#![allow(arithmetic_overflow)] #[cfg_attr(crux, crux::test)] fn crux_test() -> u8 { diff --git a/crux-mir/test/symb_eval/num/checked_div.good b/crux-mir/test/symb_eval/num/checked_div.good index 04b9ed52c..3e096ae44 100644 --- a/crux-mir/test/symb_eval/num/checked_div.good +++ b/crux-mir/test/symb_eval/num/checked_div.good @@ -1,16 +1,16 @@ -test checked_div/3a1fbbbh::crux_test[0]: returned 65, FAILED +test checked_div/df889fa7::crux_test[0]: returned 65, FAILED failures: ----- checked_div/3a1fbbbh::crux_test[0] counterexamples ---- +---- checked_div/df889fa7::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/3a1fbbbh::div_signed[0] -[Crux] attempt to divide with overflow +[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/df889fa7::div_signed[0] +[Crux] attempt to compute `_3 / _4`, which would overflow [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/3a1fbbbh::div_signed[0] -[Crux] attempt to divide by zero +[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/df889fa7::div_signed[0] +[Crux] attempt to divide `_3` by zero [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/3a1fbbbh::div_unsigned[0] -[Crux] attempt to divide by zero +[Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/df889fa7::div_unsigned[0] +[Crux] attempt to divide `_3` by zero [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/num/checked_mul.good b/crux-mir/test/symb_eval/num/checked_mul.good index 66144abe8..c0be8fe62 100644 --- a/crux-mir/test/symb_eval/num/checked_mul.good +++ b/crux-mir/test/symb_eval/num/checked_mul.good @@ -1,10 +1,10 @@ -test checked_mul/3a1fbbbh::crux_test[0]: returned 44, FAILED +test checked_mul/14b4dd3a::crux_test[0]: returned 44, FAILED failures: ----- checked_mul/3a1fbbbh::crux_test[0] counterexamples ---- +---- checked_mul/14b4dd3a::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_mul.rs:4:5: 4:12: error: in checked_mul/3a1fbbbh::crux_test[0] -[Crux] attempt to multiply with overflow +[Crux] test/symb_eval/num/checked_mul.rs:6:5: 6:12: error: in checked_mul/14b4dd3a::crux_test[0] +[Crux] attempt to compute `const 100_u8 * move _2`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/num/checked_mul.rs b/crux-mir/test/symb_eval/num/checked_mul.rs index 7bad23ec6..0c3e1b58a 100644 --- a/crux-mir/test/symb_eval/num/checked_mul.rs +++ b/crux-mir/test/symb_eval/num/checked_mul.rs @@ -1,3 +1,5 @@ +#![allow(arithmetic_overflow)] + #[cfg_attr(crux, crux::test)] fn crux_test() -> u8 { let x = 3; diff --git a/crux-mir/test/symb_eval/num/checked_mul_signed.good b/crux-mir/test/symb_eval/num/checked_mul_signed.good index dc3587bb6..2bd3c9ffa 100644 --- a/crux-mir/test/symb_eval/num/checked_mul_signed.good +++ b/crux-mir/test/symb_eval/num/checked_mul_signed.good @@ -1,10 +1,10 @@ -test checked_mul_signed/3a1fbbbh::crux_test[0]: returned 44, FAILED +test checked_mul_signed/3d1be011::crux_test[0]: returned 44, FAILED failures: ----- checked_mul_signed/3a1fbbbh::crux_test[0] counterexamples ---- +---- checked_mul_signed/3d1be011::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_mul_signed.rs:4:5: 4:13: error: in checked_mul_signed/3a1fbbbh::crux_test[0] -[Crux] attempt to multiply with overflow +[Crux] test/symb_eval/num/checked_mul_signed.rs:6:5: 6:13: error: in checked_mul_signed/3d1be011::crux_test[0] +[Crux] attempt to compute `const -100_i8 * move _2`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/num/checked_mul_signed.rs b/crux-mir/test/symb_eval/num/checked_mul_signed.rs index 67aa95093..409c0f9e0 100644 --- a/crux-mir/test/symb_eval/num/checked_mul_signed.rs +++ b/crux-mir/test/symb_eval/num/checked_mul_signed.rs @@ -1,3 +1,5 @@ +#![allow(arithmetic_overflow)] + #[cfg_attr(crux, crux::test)] fn crux_test() -> i8 { let x = -3; From ea9066b5436db9bb5be37f12ddf505bfdb79a571 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 18 May 2023 16:11:08 -0400 Subject: [PATCH 052/114] conditional.rs: Modernize crux::test usage --- crux-mir/test/symb_eval/any/conditional.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crux-mir/test/symb_eval/any/conditional.rs b/crux-mir/test/symb_eval/any/conditional.rs index f77c69f2c..8663dfbf6 100644 --- a/crux-mir/test/symb_eval/any/conditional.rs +++ b/crux-mir/test/symb_eval/any/conditional.rs @@ -7,7 +7,7 @@ extern crate crucible; use crucible::*; use crucible::any::Any; -#[crux_test] +#[cfg_attr(crux, crux::test)] fn crux_test() { let mut x = bool::symbolic("x"); if x { From 643ce40ca44ad0d41b1706119532aae02d74f89c Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Mon, 22 May 2023 15:38:33 -0400 Subject: [PATCH 053/114] Remove normDefId, compare against ExplodedDefIds Practically all uses of `normDefId` are quite dangerous in a modern setting, as even sysroot crates now have full disambiguators. This means that using the default disambiguator will not work. As a result, we now compare special `DefId`s (e.g., for `crux-mir` overrides) against `ExplodedDefId`s, which strip away disambiguator information. --- crucible-mir/src/Mir/DefId.hs | 37 ++++++++--------- crucible-mir/src/Mir/Mir.hs | 4 +- crucible-mir/src/Mir/TransTy.hs | 30 +++++++------- crux-mir/crux-mir.cabal | 1 + crux-mir/src/Mir/Overrides.hs | 70 ++++++++++++++++++++------------- 5 files changed, 79 insertions(+), 63 deletions(-) diff --git a/crucible-mir/src/Mir/DefId.hs b/crucible-mir/src/Mir/DefId.hs index cbef5284b..11473eca1 100644 --- a/crucible-mir/src/Mir/DefId.hs +++ b/crucible-mir/src/Mir/DefId.hs @@ -1,4 +1,6 @@ {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TemplateHaskellQuotes #-} +{-# LANGUAGE ViewPatterns #-} {-# LANGUAGE DeriveGeneric, DeriveAnyClass, DefaultSignatures #-} {-# OPTIONS_GHC -Wincomplete-patterns -Wall #-} @@ -13,13 +15,12 @@ module Mir.DefId , ExplodedDefId , idKey +, explodedDefIdPat +, textIdKey , getTraitName , cleanVariantName , parseFieldName - -, normDefId -, normDefIdPat ) where import Data.Aeson @@ -58,8 +59,8 @@ data DefId = DefId deriving (Eq, Ord, Generic) -- | The crate disambiguator hash produced when the crate metadata string is --- empty. This is the disambiguator for all sysroot crates, which are the only --- ones we override. +-- empty. This is useful for creating 'DefId's for thing that do not have a +-- disambiguator associated with it (e.g., Crucible names). defaultDisambiguator :: Text defaultDisambiguator = "3a1fbbbh" @@ -69,8 +70,8 @@ defaultDisambiguator = "3a1fbbbh" -- of disambiguators are optional. If the crate disambiguator is omitted, then -- it's assumed to be `defaultDisambiguator`, and if a segment disambiguator is -- omitted elsewhere in the path, it's assumed to be zero. So you can write, --- for example, `core::option::Option`, and parsing will expand it to the --- canonical form `core/3a1fbbbh::option[0]::Option[0]`. +-- for example, `foo::bar::Baz`, and parsing will expand it to the +-- canonical form `foo/3a1fbbbh::bar[0]::Baz[0]`. textId :: Text -> DefId textId s = DefId crate disambig segs where @@ -109,11 +110,21 @@ instance FromJSON DefId where instance Pretty DefId where pretty = viaShow - +-- | The individual 'DefId' components of a 'DefId'. The first element is the +-- crate name, and the subsequent elements are the path segments. By convention, +-- 'ExplodedDefId's never contain disambiguators, which make them a useful way +-- to refer to identifiers in a slightly more stable format that does not depend +-- on the particulars of how a package is hashed. type ExplodedDefId = [Text] + idKey :: DefId -> ExplodedDefId idKey did = did_crate did : map fst (did_path did) +explodedDefIdPat :: ExplodedDefId -> TH.Q TH.Pat +explodedDefIdPat edid = [p| ((\did -> idKey did == edid) -> True) |] + +textIdKey :: Text -> ExplodedDefId +textIdKey = idKey . textId idInit :: DefId -> DefId idInit (DefId crate disambig segs) = DefId crate disambig (init segs) @@ -133,13 +144,3 @@ cleanVariantName = idLast parseFieldName :: DefId -> Maybe Text parseFieldName (DefId _ _ []) = Nothing parseFieldName did = Just $ idLast did - --- | Normalize a DefId string. This allows writing down path strings in a more --- readable form. -normDefId :: Text -> Text -normDefId s = idText $ textId s - --- | Like `normDefId`, but produces a Template Haskell pattern. Useful for --- defining pattern synonyms that match a specific `TyAdt` type constructor. -normDefIdPat :: Text -> TH.Q TH.Pat -normDefIdPat s = return $ TH.LitP $ TH.StringL $ T.unpack $ normDefId s diff --git a/crucible-mir/src/Mir/Mir.hs b/crucible-mir/src/Mir/Mir.hs index 9e11fa432..9f0bd747a 100644 --- a/crucible-mir/src/Mir/Mir.hs +++ b/crucible-mir/src/Mir/Mir.hs @@ -20,7 +20,7 @@ License : BSD3 {-# LANGUAGE GeneralizedNewtypeDeriving #-} {-# LANGUAGE DerivingStrategies #-} {-# LANGUAGE TypeFamilies #-} - +{-# LANGUAGE ViewPatterns #-} {-# LANGUAGE OverloadedStrings #-} @@ -664,7 +664,7 @@ typeOfProj elm baseTy = case elm of peelRef :: Ty -> Ty peelRef (TyRef t _) = t peelRef (TyRawPtr t _) = t - peelRef (TyAdt _ $(normDefIdPat "alloc::boxed::Box") (Substs [t])) = t + peelRef (TyAdt _ $(explodedDefIdPat ["alloc", "boxed", "Box"]) (Substs [t])) = t peelRef t = t peelIdx :: Ty -> Ty diff --git a/crucible-mir/src/Mir/TransTy.hs b/crucible-mir/src/Mir/TransTy.hs index 66f2c6f35..3bad56863 100644 --- a/crucible-mir/src/Mir/TransTy.hs +++ b/crucible-mir/src/Mir/TransTy.hs @@ -98,46 +98,46 @@ baseSizeToNatCont M.USize k = k (knownNat :: NatRepr SizeBits) -- Custom type aliases -pattern CTyInt512 <- M.TyAdt _ $(M.normDefIdPat "int512::Int512") (M.Substs []) +pattern CTyInt512 <- M.TyAdt _ $(M.explodedDefIdPat ["int512", "Int512"]) (M.Substs []) where CTyInt512 = M.TyAdt (M.textId "type::adt") (M.textId "int512::Int512") (M.Substs []) -pattern CTyBox t <- M.TyAdt _ $(M.normDefIdPat "alloc::boxed::Box") (M.Substs [t]) +pattern CTyBox t <- M.TyAdt _ $(M.explodedDefIdPat ["alloc", "boxed", "Box"]) (M.Substs [t]) where CTyBox t = M.TyAdt (M.textId "type::adt") (M.textId "alloc::boxed::Box") (M.Substs [t]) -pattern CTyMaybeUninit t <- M.TyAdt _ $(M.normDefIdPat "core::mem::maybe_uninit::MaybeUninit") (M.Substs [t]) +pattern CTyMaybeUninit t <- M.TyAdt _ $(M.explodedDefIdPat ["core", "mem", "maybe_uninit", "MaybeUninit"]) (M.Substs [t]) where CTyMaybeUninit t = M.TyAdt (M.textId "type::adt") (M.textId "core::mem::maybe_uninit::MaybeUninit") (M.Substs [t]) --- `UnsafeCell` isn't handled specially inside baseline `crux-mir`, but +-- `UnsafeCell` isn't handled specially inside baseline `crucible-mir`, but -- `crux-mir-comp` looks for it (using this pattern synonym). -pattern CTyUnsafeCell t <- M.TyAdt _ $(M.normDefIdPat "core::cell::UnsafeCell") (M.Substs [t]) +pattern CTyUnsafeCell t <- M.TyAdt _ $(M.explodedDefIdPat ["core", "cell", "UnsafeCell"]) (M.Substs [t]) where CTyUnsafeCell t = M.TyAdt (M.textId "type::adt") (M.textId "core::cell::UnsafeCell") (M.Substs [t]) -pattern CTyVector t <- M.TyAdt _ $(M.normDefIdPat "crucible::vector::Vector") (M.Substs [t]) +pattern CTyVector t <- M.TyAdt _ $(M.explodedDefIdPat ["crucible", "vector", "Vector"]) (M.Substs [t]) where CTyVector t = M.TyAdt (M.textId "type::adt") (M.textId "crucible::vector::Vector") (M.Substs [t]) -pattern CTyArray t <- M.TyAdt _ $(M.normDefIdPat "crucible::array::Array") (M.Substs [t]) +pattern CTyArray t <- M.TyAdt _ $(M.explodedDefIdPat ["crucible", "array", "Array"]) (M.Substs [t]) where CTyArray t = M.TyAdt (M.textId "type::adt") (M.textId "crucible::array::Array") (M.Substs [t]) -pattern CTyBvSize128 <- M.TyAdt _ $(M.normDefIdPat "crucible::bitvector::_128") (M.Substs []) +pattern CTyBvSize128 <- M.TyAdt _ $(M.explodedDefIdPat ["crucible", "bitvector", "_128"]) (M.Substs []) where CTyBvSize128 = M.TyAdt (M.textId "type::adt") (M.textId "crucible::bitvector::_128") (M.Substs []) -pattern CTyBvSize256 <- M.TyAdt _ $(M.normDefIdPat "crucible::bitvector::_256") (M.Substs []) +pattern CTyBvSize256 <- M.TyAdt _ $(M.explodedDefIdPat ["crucible", "bitvector", "_256"]) (M.Substs []) where CTyBvSize256 = M.TyAdt (M.textId "type::adt") (M.textId "crucible::bitvector::_256") (M.Substs []) -pattern CTyBvSize512 <- M.TyAdt _ $(M.normDefIdPat "crucible::bitvector::_512") (M.Substs []) +pattern CTyBvSize512 <- M.TyAdt _ $(M.explodedDefIdPat ["crucible", "bitvector", "_512"]) (M.Substs []) where CTyBvSize512 = M.TyAdt (M.textId "type::adt") (M.textId "crucible::bitvector::_512") (M.Substs []) -pattern CTyBv t <- M.TyAdt _ $(M.normDefIdPat "crucible::bitvector::Bv") (M.Substs [t]) +pattern CTyBv t <- M.TyAdt _ $(M.explodedDefIdPat ["crucible", "bitvector", "Bv"]) (M.Substs [t]) where CTyBv t = M.TyAdt (M.textId "type::adt") (M.textId "crucible::bitvector::Bv") (M.Substs [t]) pattern CTyBv128 = CTyBv CTyBvSize128 pattern CTyBv256 = CTyBv CTyBvSize256 pattern CTyBv512 = CTyBv CTyBvSize512 -pattern CTyAny <- M.TyAdt _ $(M.normDefIdPat "core::crucible::any::Any") (M.Substs []) +pattern CTyAny <- M.TyAdt _ $(M.explodedDefIdPat ["core", "crucible", "any", "Any"]) (M.Substs []) where CTyAny = M.TyAdt (M.textId "type::adt") (M.textId "core::crucible::any::Any") (M.Substs []) -pattern CTyMethodSpec <- M.TyAdt _ $(M.normDefIdPat "crucible::method_spec::raw::MethodSpec") (M.Substs []) +pattern CTyMethodSpec <- M.TyAdt _ $(M.explodedDefIdPat ["crucible", "method_spec", "raw", "MethodSpec"]) (M.Substs []) where CTyMethodSpec = M.TyAdt (M.textId "type::adt") (M.textId "crucible::method_spec::raw::MethodSpec") (M.Substs []) -pattern CTyMethodSpecBuilder <- M.TyAdt _ $(M.normDefIdPat "crucible::method_spec::raw::MethodSpecBuilder") (M.Substs []) +pattern CTyMethodSpecBuilder <- M.TyAdt _ $(M.explodedDefIdPat ["crucible", "method_spec", "raw", "MethodSpecBuilder"]) (M.Substs []) where CTyMethodSpecBuilder = M.TyAdt (M.textId "type::adt") (M.textId "crucible::method_spec::raw::MethodSpecBuilder") (M.Substs []) -- These don't have custom representation, but are referenced in various -- places. -pattern CTyOption t <- M.TyAdt _ $(M.normDefIdPat "core::option::Option") (M.Substs [t]) +pattern CTyOption t <- M.TyAdt _ $(M.explodedDefIdPat ["core", "option", "Option"]) (M.Substs [t]) where CTyOption t = M.TyAdt (M.textId "type::adt") (M.textId "core::option::Option") (M.Substs [t]) optionDefId :: M.DefId diff --git a/crux-mir/crux-mir.cabal b/crux-mir/crux-mir.cabal index 2fc8a1194..7d56e4327 100644 --- a/crux-mir/crux-mir.cabal +++ b/crux-mir/crux-mir.cabal @@ -24,6 +24,7 @@ library ansi-terminal, bv-sized, bytestring, + extra, prettyprinter >= 1.7.0, text, time, diff --git a/crux-mir/src/Mir/Overrides.hs b/crux-mir/src/Mir/Overrides.hs index 1ca09c170..6c3c53fc5 100644 --- a/crux-mir/src/Mir/Overrides.hs +++ b/crux-mir/src/Mir/Overrides.hs @@ -22,6 +22,7 @@ import Control.Monad.Trans.Maybe (MaybeT(..)) import qualified Data.BitVector.Sized as BV import qualified Data.ByteString as BS +import Data.List.Extra (unsnoc) import Data.Map (Map, fromList) import qualified Data.Map as Map import Data.Maybe (fromMaybe) @@ -402,25 +403,25 @@ bindFn :: CFG MIR blocks args ret -> OverrideSim (p sym) sym MIR rtp a r () bindFn symOnline cs name cfg - | (normDefId "crucible::array::symbolic" <> "::_inst") `Text.isPrefixOf` name + | hasInstPrefix ["crucible", "array", "symbolic"] explodedName , Empty :> MirSliceRepr (BVRepr w) <- cfgArgTypes cfg , UsizeArrayRepr btpr <- cfgReturnType cfg , Just Refl <- testEquality w (knownNat @8) = bindFnHandle (cfgHandle cfg) $ UseOverride $ mkOverride' "array::symbolic" (UsizeArrayRepr btpr) (array_symbolic btpr) - | (normDefId "crucible::concretize" <> "::_inst") `Text.isPrefixOf` name + | hasInstPrefix ["crucible", "concretize"] explodedName , Empty :> tpr <- cfgArgTypes cfg , Just Refl <- testEquality tpr (cfgReturnType cfg) = bindFnHandle (cfgHandle cfg) $ UseOverride $ mkOverride' "concretize" tpr $ concretize symOnline - | (normDefId "crucible::override_" <> "::_inst") `Text.isPrefixOf` name + | hasInstPrefix ["crucible", "override_"] explodedName , Empty :> UnitRepr :> UnitRepr <- cfgArgTypes cfg , UnitRepr <- cfgReturnType cfg = bindFnHandle (cfgHandle cfg) $ UseOverride $ mkOverride' "crucible_override_" UnitRepr $ overrideRust cs name - | (normDefId "crucible::dump_what4" <> "::_inst") `Text.isPrefixOf` name + | hasInstPrefix ["crucible", "dump_what4"] explodedName , Empty :> MirSliceRepr (BVRepr w) :> (asBaseType -> AsBaseType btpr) <- cfgArgTypes cfg , Just Refl <- testEquality w (knownNat @8) , UnitRepr <- cfgReturnType cfg @@ -431,12 +432,23 @@ bindFn symOnline cs name cfg Just x -> return x Nothing -> fail $ "dump_what4: desc string must be concrete" liftIO $ putStrLn $ Text.unpack str ++ " = " ++ show (printSymExpr expr) - + where + explodedName = textIdKey name + + -- | Check if @edid@ has the same path as @pfxInit@, plus a final path + -- component that begins with the name @_inst@. + hasInstPrefix :: [Text] -> ExplodedDefId -> Bool + hasInstPrefix pfxInit edid = + case unsnoc edid of + Nothing -> False + Just (edidInit, edidLast) -> + pfxInit == edidInit && + "_inst" `Text.isPrefixOf` edidLast bindFn _symOnline _cs fn cfg = ovrWithBackend $ \bak -> let s = backendGetSym bak in - case Map.lookup fn (overrides bak) of + case Map.lookup (textIdKey fn) (overrides bak) of Nothing -> bindFnHandle (cfgHandle cfg) $ UseCFG cfg (postdomInfo cfg) Just (($ functionNameFromText fn) -> SomeOverride argTys retTy f) -> @@ -450,13 +462,11 @@ bindFn _symOnline _cs fn cfg = where override :: forall args ret . - Text -> CtxRepr args -> TypeRepr ret -> + ExplodedDefId -> CtxRepr args -> TypeRepr ret -> (forall rtp . OverrideSim (p sym) sym MIR rtp args ret (RegValue sym ret)) -> - (Text, FunctionName -> SomeOverride (p sym) sym) - override n args ret act = - -- Round-trip through `DefId` to normalize the path. In particular, - -- this adds the current `defaultDisambiguator` and any missing `[0]`s. - ( normDefId n + (ExplodedDefId, FunctionName -> SomeOverride (p sym) sym) + override edid args ret act = + ( edid , \funName -> SomeOverride args ret (mkOverride' funName ret act) ) @@ -469,32 +479,36 @@ bindFn _symOnline _cs fn cfg = strrepr :: TypeRepr (MirSlice (BVType 8)) strrepr = knownRepr - symb_bv :: forall n . (1 <= n) => Text -> NatRepr n -> (Text, FunctionName -> SomeOverride (p sym) sym) - symb_bv name n = - override name (Empty :> strrepr) (BVRepr n) $ + symb_bv :: forall n . (1 <= n) + => ExplodedDefId -> NatRepr n + -> (ExplodedDefId, FunctionName -> SomeOverride (p sym) sym) + symb_bv edid n = + override edid (Empty :> strrepr) (BVRepr n) $ do RegMap (Empty :> str) <- getOverrideArgs makeSymbolicVar str $ BaseBVRepr n - overrides :: IsSymBackend sym bak' => bak' -> Map Text (FunctionName -> SomeOverride (p sym) sym) + overrides :: IsSymBackend sym bak' + => bak' + -> Map ExplodedDefId (FunctionName -> SomeOverride (p sym) sym) overrides bak = let sym = backendGetSym bak in - fromList [ override "crucible::one" Empty (BVRepr (knownNat @8)) $ + fromList [ override ["crucible", "one"] Empty (BVRepr (knownNat @8)) $ do h <- printHandle <$> getContext liftIO (hPutStrLn h "Hello, I'm an override") v <- liftIO $ bvLit sym knownNat (BV.mkBV knownNat 1) return v - , symb_bv "crucible::symbolic::symbolic_u8" (knownNat @8) - , symb_bv "crucible::symbolic::symbolic_u16" (knownNat @16) - , symb_bv "crucible::symbolic::symbolic_u32" (knownNat @32) - , symb_bv "crucible::symbolic::symbolic_u64" (knownNat @64) - , symb_bv "crucible::symbolic::symbolic_u128" (knownNat @128) - , symb_bv "int512::symbolic" (knownNat @512) - , symb_bv "crucible::bitvector::make_symbolic_128" (knownNat @128) - , symb_bv "crucible::bitvector::make_symbolic_256" (knownNat @256) - , symb_bv "crucible::bitvector::make_symbolic_512" (knownNat @512) + , symb_bv ["crucible", "symbolic", "symbolic_u8"] (knownNat @8) + , symb_bv ["crucible", "symbolic", "symbolic_u16"] (knownNat @16) + , symb_bv ["crucible", "symbolic", "symbolic_u32"] (knownNat @32) + , symb_bv ["crucible", "symbolic", "symbolic_u64"] (knownNat @64) + , symb_bv ["crucible", "symbolic", "symbolic_u128"] (knownNat @128) + , symb_bv ["int512", "symbolic"] (knownNat @512) + , symb_bv ["crucible", "bitvector", "make_symbolic_128"] (knownNat @128) + , symb_bv ["crucible", "bitvector", "make_symbolic_256"] (knownNat @256) + , symb_bv ["crucible", "bitvector", "make_symbolic_512"] (knownNat @512) , let argTys = (Empty :> BoolRepr :> strrepr :> strrepr :> u32repr :> u32repr) - in override "crucible::crucible_assert_impl" argTys UnitRepr $ + in override ["crucible", "crucible_assert_impl"] argTys UnitRepr $ do RegMap (Empty :> c :> srcArg :> fileArg :> lineArg :> colArg) <- getOverrideArgs src <- maybe (fail "not a constant src string") (pure . Text.unpack) @@ -509,7 +523,7 @@ bindFn _symOnline _cs fn cfg = liftIO $ assert bak (regValue c) reason return () , let argTys = (Empty :> BoolRepr :> strrepr :> strrepr :> u32repr :> u32repr) - in override "crucible::crucible_assume_impl" argTys UnitRepr $ + in override ["crucible", "crucible_assume_impl"] argTys UnitRepr $ do RegMap (Empty :> c :> srcArg :> fileArg :> lineArg :> colArg) <- getOverrideArgs loc <- liftIO $ getCurrentProgramLoc sym src <- maybe (fail "not a constant src string") From 5d35c27f91b1bb46dc72fbf2a71f99c36cd30caa Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Tue, 23 May 2023 18:33:44 -0400 Subject: [PATCH 054/114] crux-mir: Accept new golden output for some symbolic tests These test cases now have different hashes in their disambiguators. --- crux-mir/test/symb_eval/alloc/out_of_bounds.good | 8 ++++---- crux-mir/test/symb_eval/alloc/uninit_read.good | 8 ++++---- crux-mir/test/symb_eval/alloc/valid_read.good | 2 +- crux-mir/test/symb_eval/alloc/zero_length.good | 2 +- crux-mir/test/symb_eval/any/conditional.good | 2 +- crux-mir/test/symb_eval/any/downcast.good | 2 +- crux-mir/test/symb_eval/array/basic.good | 2 +- crux-mir/test/symb_eval/array/mux_slice.good | 2 +- crux-mir/test/symb_eval/array/slice.good | 2 +- crux-mir/test/symb_eval/array/slice_mut.good | 2 +- crux-mir/test/symb_eval/bitvector/arith.good | 2 +- crux-mir/test/symb_eval/bitvector/cmp.good | 2 +- crux-mir/test/symb_eval/bitvector/from_to.good | 2 +- crux-mir/test/symb_eval/bitvector/literals.good | 2 +- crux-mir/test/symb_eval/bitvector/overflowing_sub.good | 2 +- crux-mir/test/symb_eval/bitvector/symbolic.good | 2 +- crux-mir/test/symb_eval/bytes/extend_bytes.good | 2 +- crux-mir/test/symb_eval/bytes/new.good | 2 +- crux-mir/test/symb_eval/concretize/array.good | 2 +- crux-mir/test/symb_eval/concretize/assert_ok.good | 2 +- crux-mir/test/symb_eval/concretize/conc.good | 2 +- crux-mir/test/symb_eval/concretize/no_conc.good | 2 +- crux-mir/test/symb_eval/crypto/bytes2.good | 2 +- crux-mir/test/symb_eval/crypto/double.good | 2 +- crux-mir/test/symb_eval/crypto/ffs.good | 2 +- crux-mir/test/symb_eval/enum/mux.good | 2 +- crux-mir/test/symb_eval/fnptr/mux.good | 2 +- crux-mir/test/symb_eval/io/vec_write.good | 2 +- crux-mir/test/symb_eval/mux/array.good | 2 +- crux-mir/test/symb_eval/mux/array_mut.good | 2 +- crux-mir/test/symb_eval/num/checked_add.good | 6 +++--- crux-mir/test/symb_eval/num/checked_div.good | 10 +++++----- crux-mir/test/symb_eval/num/checked_mul.good | 6 +++--- crux-mir/test/symb_eval/num/checked_mul_signed.good | 6 +++--- crux-mir/test/symb_eval/overrides/bad_symb1.good | 6 +++--- crux-mir/test/symb_eval/overrides/bad_symb2.good | 6 +++--- crux-mir/test/symb_eval/overrides/override1.good | 2 +- crux-mir/test/symb_eval/overrides/override3.good | 2 +- crux-mir/test/symb_eval/overrides/override4.good | 2 +- crux-mir/test/symb_eval/overrides/override_rust.good | 2 +- crux-mir/test/symb_eval/refs/mux_init_imm.good | 2 +- crux-mir/test/symb_eval/refs/mux_init_mut.good | 2 +- crux-mir/test/symb_eval/scalar/test1.good | 2 +- crux-mir/test/symb_eval/sym_bytes/deserialize.good | 8 ++++---- crux-mir/test/symb_eval/vector/as_mut_slice.good | 2 +- crux-mir/test/symb_eval/vector/as_slice.good | 2 +- crux-mir/test/symb_eval/vector/concat.good | 2 +- crux-mir/test/symb_eval/vector/copy_from_slice.good | 2 +- crux-mir/test/symb_eval/vector/mut.good | 2 +- crux-mir/test/symb_eval/vector/new.good | 2 +- crux-mir/test/symb_eval/vector/push.good | 2 +- crux-mir/test/symb_eval/vector/replicate.good | 2 +- crux-mir/test/symb_eval/vector/split_at.good | 2 +- 53 files changed, 76 insertions(+), 76 deletions(-) diff --git a/crux-mir/test/symb_eval/alloc/out_of_bounds.good b/crux-mir/test/symb_eval/alloc/out_of_bounds.good index 689cb4e08..ef03f0762 100644 --- a/crux-mir/test/symb_eval/alloc/out_of_bounds.good +++ b/crux-mir/test/symb_eval/alloc/out_of_bounds.good @@ -1,13 +1,13 @@ -test out_of_bounds/3a1fbbbh::crux_test[0]: FAILED +test out_of_bounds/b521303c::crux_test[0]: FAILED failures: ----- out_of_bounds/3a1fbbbh::crux_test[0] counterexamples ---- +---- out_of_bounds/b521303c::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/3a1fbbbh::crux_test[0] +[Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/b521303c::crux_test[0] [Crux] vector index out of range: the length is 10 but the index is BV 12 [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/3a1fbbbh::crux_test[0] +[Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/b521303c::crux_test[0] [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/alloc/uninit_read.good b/crux-mir/test/symb_eval/alloc/uninit_read.good index b184913f6..4ce8bdfc4 100644 --- a/crux-mir/test/symb_eval/alloc/uninit_read.good +++ b/crux-mir/test/symb_eval/alloc/uninit_read.good @@ -1,13 +1,13 @@ -test uninit_read/3a1fbbbh::crux_test[0]: FAILED +test uninit_read/e86da728::crux_test[0]: FAILED failures: ----- uninit_read/3a1fbbbh::crux_test[0] counterexamples ---- +---- uninit_read/e86da728::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/3a1fbbbh::crux_test[0] +[Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/e86da728::crux_test[0] [Crux] Attempted to read uninitialized vector index [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/3a1fbbbh::crux_test[0] +[Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/e86da728::crux_test[0] [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/alloc/valid_read.good b/crux-mir/test/symb_eval/alloc/valid_read.good index c71223602..f9a9f3619 100644 --- a/crux-mir/test/symb_eval/alloc/valid_read.good +++ b/crux-mir/test/symb_eval/alloc/valid_read.good @@ -1,3 +1,3 @@ -test valid_read/3a1fbbbh::crux_test[0]: returned 45, ok +test valid_read/f9c1ade1::crux_test[0]: returned 45, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/alloc/zero_length.good b/crux-mir/test/symb_eval/alloc/zero_length.good index 6f47af06f..924d71647 100644 --- a/crux-mir/test/symb_eval/alloc/zero_length.good +++ b/crux-mir/test/symb_eval/alloc/zero_length.good @@ -1,3 +1,3 @@ -test zero_length/3a1fbbbh::crux_test[0]: ok +test zero_length/94b92a2d::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/any/conditional.good b/crux-mir/test/symb_eval/any/conditional.good index 66b30437f..3bc0f4fb0 100644 --- a/crux-mir/test/symb_eval/any/conditional.good +++ b/crux-mir/test/symb_eval/any/conditional.good @@ -1,3 +1,3 @@ -test conditional/3a1fbbbh::crux_test[0]: ok +test conditional/84afefc9::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/any/downcast.good b/crux-mir/test/symb_eval/any/downcast.good index def32f4d9..3591a46a0 100644 --- a/crux-mir/test/symb_eval/any/downcast.good +++ b/crux-mir/test/symb_eval/any/downcast.good @@ -1,3 +1,3 @@ -test downcast/3a1fbbbh::crux_test[0]: returned 1, ok +test downcast/870d31b0::crux_test[0]: returned 1, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/array/basic.good b/crux-mir/test/symb_eval/array/basic.good index 6a570da6d..f1439da79 100644 --- a/crux-mir/test/symb_eval/array/basic.good +++ b/crux-mir/test/symb_eval/array/basic.good @@ -1,3 +1,3 @@ -test basic/3a1fbbbh::crux_test[0]: returned 3, ok +test basic/bfb477c2::crux_test[0]: returned 3, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/array/mux_slice.good b/crux-mir/test/symb_eval/array/mux_slice.good index 15c6fc94c..03022fdb1 100644 --- a/crux-mir/test/symb_eval/array/mux_slice.good +++ b/crux-mir/test/symb_eval/array/mux_slice.good @@ -1,3 +1,3 @@ -test mux_slice/3a1fbbbh::crux_test[0]: returned Symbolic BV, ok +test mux_slice/fe5e891b::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/array/slice.good b/crux-mir/test/symb_eval/array/slice.good index 0476c9aa0..7d0e5b30c 100644 --- a/crux-mir/test/symb_eval/array/slice.good +++ b/crux-mir/test/symb_eval/array/slice.good @@ -1,3 +1,3 @@ -test slice/3a1fbbbh::crux_test[0]: returned 5, ok +test slice/ea2efd47::crux_test[0]: returned 5, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/array/slice_mut.good b/crux-mir/test/symb_eval/array/slice_mut.good index ce4d4c819..03b8d75c2 100644 --- a/crux-mir/test/symb_eval/array/slice_mut.good +++ b/crux-mir/test/symb_eval/array/slice_mut.good @@ -1,3 +1,3 @@ -test slice_mut/3a1fbbbh::crux_test[0]: returned 5, ok +test slice_mut/1361be29::crux_test[0]: returned 5, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/arith.good b/crux-mir/test/symb_eval/bitvector/arith.good index 76e789d24..1586f7ebe 100644 --- a/crux-mir/test/symb_eval/bitvector/arith.good +++ b/crux-mir/test/symb_eval/bitvector/arith.good @@ -1,3 +1,3 @@ -test arith/3a1fbbbh::crux_test[0]: ok +test arith/382b0162::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/cmp.good b/crux-mir/test/symb_eval/bitvector/cmp.good index b26916f30..971f62ad6 100644 --- a/crux-mir/test/symb_eval/bitvector/cmp.good +++ b/crux-mir/test/symb_eval/bitvector/cmp.good @@ -1,3 +1,3 @@ -test cmp/3a1fbbbh::crux_test[0]: ok +test cmp/a77f9ebc::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/from_to.good b/crux-mir/test/symb_eval/bitvector/from_to.good index b4b557ad5..458a40add 100644 --- a/crux-mir/test/symb_eval/bitvector/from_to.good +++ b/crux-mir/test/symb_eval/bitvector/from_to.good @@ -1,3 +1,3 @@ -test from_to/3a1fbbbh::crux_test[0]: ok +test from_to/cecfc226::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/literals.good b/crux-mir/test/symb_eval/bitvector/literals.good index 86b273e7c..966cd907b 100644 --- a/crux-mir/test/symb_eval/bitvector/literals.good +++ b/crux-mir/test/symb_eval/bitvector/literals.good @@ -1,3 +1,3 @@ -test literals/3a1fbbbh::crux_test[0]: ok +test literals/06825ef9::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/overflowing_sub.good b/crux-mir/test/symb_eval/bitvector/overflowing_sub.good index eccd2b401..0e029bc46 100644 --- a/crux-mir/test/symb_eval/bitvector/overflowing_sub.good +++ b/crux-mir/test/symb_eval/bitvector/overflowing_sub.good @@ -1,3 +1,3 @@ -test overflowing_sub/3a1fbbbh::crux_test[0]: ok +test overflowing_sub/30a11e11::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/symbolic.good b/crux-mir/test/symb_eval/bitvector/symbolic.good index da33604e7..ea6f17081 100644 --- a/crux-mir/test/symb_eval/bitvector/symbolic.good +++ b/crux-mir/test/symb_eval/bitvector/symbolic.good @@ -1,3 +1,3 @@ -test symbolic/3a1fbbbh::crux_test[0]: ok +test symbolic/d4690595::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/extend_bytes.good b/crux-mir/test/symb_eval/bytes/extend_bytes.good index 87d39bb04..4b4c681a8 100644 --- a/crux-mir/test/symb_eval/bytes/extend_bytes.good +++ b/crux-mir/test/symb_eval/bytes/extend_bytes.good @@ -1,3 +1,3 @@ -test extend_bytes/3a1fbbbh::f[0]: ok +test extend_bytes/fe7bcbf0::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/new.good b/crux-mir/test/symb_eval/bytes/new.good index 1d5ebf83b..1e78a772e 100644 --- a/crux-mir/test/symb_eval/bytes/new.good +++ b/crux-mir/test/symb_eval/bytes/new.good @@ -1,3 +1,3 @@ -test new/3a1fbbbh::f[0]: ok +test new/dec1e542::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/concretize/array.good b/crux-mir/test/symb_eval/concretize/array.good index 466652519..399cfc9fe 100644 --- a/crux-mir/test/symb_eval/concretize/array.good +++ b/crux-mir/test/symb_eval/concretize/array.good @@ -1,3 +1,3 @@ -test array/3a1fbbbh::crux_test[0]: returned (1, 2, 3), ok +test array/aa48835b::crux_test[0]: returned (1, 2, 3), ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/concretize/assert_ok.good b/crux-mir/test/symb_eval/concretize/assert_ok.good index 55f2e9e73..50dbc0720 100644 --- a/crux-mir/test/symb_eval/concretize/assert_ok.good +++ b/crux-mir/test/symb_eval/concretize/assert_ok.good @@ -1,3 +1,3 @@ -test assert_ok/3a1fbbbh::crux_test[0]: returned 1, ok +test assert_ok/e60ec1ab::crux_test[0]: returned 1, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/concretize/conc.good b/crux-mir/test/symb_eval/concretize/conc.good index db4bb75fa..47946de64 100644 --- a/crux-mir/test/symb_eval/concretize/conc.good +++ b/crux-mir/test/symb_eval/concretize/conc.good @@ -1,3 +1,3 @@ -test conc/3a1fbbbh::crux_test[0]: returned (1, 0), ok +test conc/58237648::crux_test[0]: returned (1, 0), ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/concretize/no_conc.good b/crux-mir/test/symb_eval/concretize/no_conc.good index 0369b343d..7a2d78170 100644 --- a/crux-mir/test/symb_eval/concretize/no_conc.good +++ b/crux-mir/test/symb_eval/concretize/no_conc.good @@ -1,3 +1,3 @@ -test no_conc/3a1fbbbh::crux_test[0]: returned (Symbolic BV, Symbolic BV), ok +test no_conc/46b8f4c7::crux_test[0]: returned (Symbolic BV, Symbolic BV), ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/crypto/bytes2.good b/crux-mir/test/symb_eval/crypto/bytes2.good index cf1da1fc1..ce0d54c6d 100644 --- a/crux-mir/test/symb_eval/crypto/bytes2.good +++ b/crux-mir/test/symb_eval/crypto/bytes2.good @@ -1,3 +1,3 @@ -test bytes2/3a1fbbbh::f[0]: ok +test bytes2/c529255b::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/crypto/double.good b/crux-mir/test/symb_eval/crypto/double.good index 022f2d2c2..f3febce52 100644 --- a/crux-mir/test/symb_eval/crypto/double.good +++ b/crux-mir/test/symb_eval/crypto/double.good @@ -1,3 +1,3 @@ -test double/3a1fbbbh::f[0]: ok +test double/e6e26f62::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/crypto/ffs.good b/crux-mir/test/symb_eval/crypto/ffs.good index 95fdc8edb..b364ebbcc 100644 --- a/crux-mir/test/symb_eval/crypto/ffs.good +++ b/crux-mir/test/symb_eval/crypto/ffs.good @@ -1,3 +1,3 @@ -test ffs/3a1fbbbh::f[0]: ok +test ffs/12bb9465::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/enum/mux.good b/crux-mir/test/symb_eval/enum/mux.good index 5cf040b26..c15770986 100644 --- a/crux-mir/test/symb_eval/enum/mux.good +++ b/crux-mir/test/symb_eval/enum/mux.good @@ -1,3 +1,3 @@ -test mux/3a1fbbbh::test[0]: ok +test mux/e3e33eba::test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/fnptr/mux.good b/crux-mir/test/symb_eval/fnptr/mux.good index fb263322c..efcb25700 100644 --- a/crux-mir/test/symb_eval/fnptr/mux.good +++ b/crux-mir/test/symb_eval/fnptr/mux.good @@ -1,3 +1,3 @@ -test mux/3a1fbbbh::crux_test[0]: returned 2, ok +test mux/97bbb42d::crux_test[0]: returned 2, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/io/vec_write.good b/crux-mir/test/symb_eval/io/vec_write.good index 6a93cb6e9..a3f6a7fa6 100644 --- a/crux-mir/test/symb_eval/io/vec_write.good +++ b/crux-mir/test/symb_eval/io/vec_write.good @@ -1,3 +1,3 @@ -test vec_write/3a1fbbbh::f[0]: ok +test vec_write/b46f4310::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/mux/array.good b/crux-mir/test/symb_eval/mux/array.good index 537ea6a23..914c5f91e 100644 --- a/crux-mir/test/symb_eval/mux/array.good +++ b/crux-mir/test/symb_eval/mux/array.good @@ -1,3 +1,3 @@ -test array/3a1fbbbh::crux_test[0]: returned Symbolic BV, ok +test array/8d6b099f::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/mux/array_mut.good b/crux-mir/test/symb_eval/mux/array_mut.good index 925dc5212..4a2d5e64f 100644 --- a/crux-mir/test/symb_eval/mux/array_mut.good +++ b/crux-mir/test/symb_eval/mux/array_mut.good @@ -1,3 +1,3 @@ -test array_mut/3a1fbbbh::crux_test[0]: returned Symbolic BV, ok +test array_mut/706b4322::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/num/checked_add.good b/crux-mir/test/symb_eval/num/checked_add.good index 996a449f9..ddab6eeff 100644 --- a/crux-mir/test/symb_eval/num/checked_add.good +++ b/crux-mir/test/symb_eval/num/checked_add.good @@ -1,10 +1,10 @@ -test checked_add/ec6fd7f2::crux_test[0]: returned 44, FAILED +test checked_add/b10d29cb::crux_test[0]: returned 44, FAILED failures: ----- checked_add/ec6fd7f2::crux_test[0] counterexamples ---- +---- checked_add/b10d29cb::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_add.rs:6:5: 6:12: error: in checked_add/ec6fd7f2::crux_test[0] +[Crux] test/symb_eval/num/checked_add.rs:6:5: 6:12: error: in checked_add/b10d29cb::crux_test[0] [Crux] attempt to compute `const 100_u8 + move _2`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/num/checked_div.good b/crux-mir/test/symb_eval/num/checked_div.good index 3e096ae44..625342130 100644 --- a/crux-mir/test/symb_eval/num/checked_div.good +++ b/crux-mir/test/symb_eval/num/checked_div.good @@ -1,16 +1,16 @@ -test checked_div/df889fa7::crux_test[0]: returned 65, FAILED +test checked_div/56a4cdf6::crux_test[0]: returned 65, FAILED failures: ----- checked_div/df889fa7::crux_test[0] counterexamples ---- +---- checked_div/56a4cdf6::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/df889fa7::div_signed[0] +[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/56a4cdf6::div_signed[0] [Crux] attempt to compute `_3 / _4`, which would overflow [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/df889fa7::div_signed[0] +[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/56a4cdf6::div_signed[0] [Crux] attempt to divide `_3` by zero [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/df889fa7::div_unsigned[0] +[Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/56a4cdf6::div_unsigned[0] [Crux] attempt to divide `_3` by zero [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/num/checked_mul.good b/crux-mir/test/symb_eval/num/checked_mul.good index c0be8fe62..aea18fe6f 100644 --- a/crux-mir/test/symb_eval/num/checked_mul.good +++ b/crux-mir/test/symb_eval/num/checked_mul.good @@ -1,10 +1,10 @@ -test checked_mul/14b4dd3a::crux_test[0]: returned 44, FAILED +test checked_mul/6db35fea::crux_test[0]: returned 44, FAILED failures: ----- checked_mul/14b4dd3a::crux_test[0] counterexamples ---- +---- checked_mul/6db35fea::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_mul.rs:6:5: 6:12: error: in checked_mul/14b4dd3a::crux_test[0] +[Crux] test/symb_eval/num/checked_mul.rs:6:5: 6:12: error: in checked_mul/6db35fea::crux_test[0] [Crux] attempt to compute `const 100_u8 * move _2`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/num/checked_mul_signed.good b/crux-mir/test/symb_eval/num/checked_mul_signed.good index 2bd3c9ffa..2222c909c 100644 --- a/crux-mir/test/symb_eval/num/checked_mul_signed.good +++ b/crux-mir/test/symb_eval/num/checked_mul_signed.good @@ -1,10 +1,10 @@ -test checked_mul_signed/3d1be011::crux_test[0]: returned 44, FAILED +test checked_mul_signed/1cb83c82::crux_test[0]: returned 44, FAILED failures: ----- checked_mul_signed/3d1be011::crux_test[0] counterexamples ---- +---- checked_mul_signed/1cb83c82::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_mul_signed.rs:6:5: 6:13: error: in checked_mul_signed/3d1be011::crux_test[0] +[Crux] test/symb_eval/num/checked_mul_signed.rs:6:5: 6:13: error: in checked_mul_signed/1cb83c82::crux_test[0] [Crux] attempt to compute `const -100_i8 * move _2`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/overrides/bad_symb1.good b/crux-mir/test/symb_eval/overrides/bad_symb1.good index 961e57159..79016bc4e 100644 --- a/crux-mir/test/symb_eval/overrides/bad_symb1.good +++ b/crux-mir/test/symb_eval/overrides/bad_symb1.good @@ -1,10 +1,10 @@ -test bad_symb1/3a1fbbbh::crux_test[0]: FAILED +test bad_symb1/fe491089::crux_test[0]: FAILED failures: ----- bad_symb1/3a1fbbbh::crux_test[0] counterexamples ---- +---- bad_symb1/fe491089::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] lib/crucible/symbolic.rs:24:64: 24:68 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/3a1fbbbh::symbolic[0]::{{impl}}[1]::symbolic[0] +[Crux] lib/crucible/symbolic.rs:24:64: 24:68 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/57b9ba51::symbolic[0]::{impl#1}[0]::symbolic[0] [Crux] symbolic variable name must be a concrete string [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/overrides/bad_symb2.good b/crux-mir/test/symb_eval/overrides/bad_symb2.good index c8af1add3..643bfb40a 100644 --- a/crux-mir/test/symb_eval/overrides/bad_symb2.good +++ b/crux-mir/test/symb_eval/overrides/bad_symb2.good @@ -1,10 +1,10 @@ -test bad_symb2/3a1fbbbh::crux_test[0]: FAILED +test bad_symb2/8202e334::crux_test[0]: FAILED failures: ----- bad_symb2/3a1fbbbh::crux_test[0] counterexamples ---- +---- bad_symb2/8202e334::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] lib/crucible/symbolic.rs:24:64: 24:68 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/3a1fbbbh::symbolic[0]::{{impl}}[1]::symbolic[0] +[Crux] lib/crucible/symbolic.rs:24:64: 24:68 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/57b9ba51::symbolic[0]::{impl#1}[0]::symbolic[0] [Crux] invalid symbolic variable name "\NUL:., /": Identifier must start with a letter. [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/overrides/override1.good b/crux-mir/test/symb_eval/overrides/override1.good index 7933f335e..514c4e67c 100644 --- a/crux-mir/test/symb_eval/overrides/override1.good +++ b/crux-mir/test/symb_eval/overrides/override1.good @@ -1,4 +1,4 @@ -test override1/3a1fbbbh::f[0]: Hello, I'm an override +test override1/abe84667::f[0]: Hello, I'm an override returned 2, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/overrides/override3.good b/crux-mir/test/symb_eval/overrides/override3.good index 08c4ad7bd..8f662efcc 100644 --- a/crux-mir/test/symb_eval/overrides/override3.good +++ b/crux-mir/test/symb_eval/overrides/override3.good @@ -1,3 +1,3 @@ -test override3/3a1fbbbh::f[0]: ok +test override3/e63a77d1::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/overrides/override4.good b/crux-mir/test/symb_eval/overrides/override4.good index e4367d48f..1ad5aa98e 100644 --- a/crux-mir/test/symb_eval/overrides/override4.good +++ b/crux-mir/test/symb_eval/overrides/override4.good @@ -1,3 +1,3 @@ -test override4/3a1fbbbh::f[0]: ok +test override4/cf8d07e7::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/overrides/override_rust.good b/crux-mir/test/symb_eval/overrides/override_rust.good index 0ba725f2d..82d82c98a 100644 --- a/crux-mir/test/symb_eval/overrides/override_rust.good +++ b/crux-mir/test/symb_eval/overrides/override_rust.good @@ -1,3 +1,3 @@ -test override_rust/3a1fbbbh::crux_test[0]: ok +test override_rust/b13aa63f::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/refs/mux_init_imm.good b/crux-mir/test/symb_eval/refs/mux_init_imm.good index ec97e9a26..3a9aa5ac3 100644 --- a/crux-mir/test/symb_eval/refs/mux_init_imm.good +++ b/crux-mir/test/symb_eval/refs/mux_init_imm.good @@ -1,3 +1,3 @@ -test mux_init_imm/3a1fbbbh::crux_test[0]: returned Symbolic BV, ok +test mux_init_imm/425f8639::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/refs/mux_init_mut.good b/crux-mir/test/symb_eval/refs/mux_init_mut.good index 7e95d4177..305f5d5fb 100644 --- a/crux-mir/test/symb_eval/refs/mux_init_mut.good +++ b/crux-mir/test/symb_eval/refs/mux_init_mut.good @@ -1,3 +1,3 @@ -test mux_init_mut/3a1fbbbh::crux_test[0]: returned Symbolic BV, ok +test mux_init_mut/a10fc099::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/scalar/test1.good b/crux-mir/test/symb_eval/scalar/test1.good index f73bfe2e7..3b0523bf5 100644 --- a/crux-mir/test/symb_eval/scalar/test1.good +++ b/crux-mir/test/symb_eval/scalar/test1.good @@ -1,3 +1,3 @@ -test test1/3a1fbbbh::f[0]: ok +test test1/87e099b9::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/sym_bytes/deserialize.good b/crux-mir/test/symb_eval/sym_bytes/deserialize.good index 481c7d740..3b3ed63a0 100644 --- a/crux-mir/test/symb_eval/sym_bytes/deserialize.good +++ b/crux-mir/test/symb_eval/sym_bytes/deserialize.good @@ -1,10 +1,10 @@ -test deserialize/3a1fbbbh::crux_test[0]: returned Symbolic BV, FAILED +test deserialize/21c6f4ae::crux_test[0]: returned Symbolic BV, FAILED failures: ----- deserialize/3a1fbbbh::crux_test[0] counterexamples ---- +---- deserialize/21c6f4ae::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/sym_bytes/deserialize.rs:19:21: 19:44: error: in deserialize/3a1fbbbh::deserialize[0] -[Crux] attempt to add with overflow +[Crux] test/symb_eval/sym_bytes/deserialize.rs:19:21: 19:44: error: in deserialize/21c6f4ae::deserialize[0] +[Crux] attempt to compute `move _61 + move _62`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/vector/as_mut_slice.good b/crux-mir/test/symb_eval/vector/as_mut_slice.good index 78bdee9f0..9812fd1be 100644 --- a/crux-mir/test/symb_eval/vector/as_mut_slice.good +++ b/crux-mir/test/symb_eval/vector/as_mut_slice.good @@ -1,3 +1,3 @@ -test as_mut_slice/3a1fbbbh::f[0]: ok +test as_mut_slice/23bdd9be::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/as_slice.good b/crux-mir/test/symb_eval/vector/as_slice.good index 987334325..68f7fd280 100644 --- a/crux-mir/test/symb_eval/vector/as_slice.good +++ b/crux-mir/test/symb_eval/vector/as_slice.good @@ -1,3 +1,3 @@ -test as_slice/3a1fbbbh::f[0]: ok +test as_slice/0226cf7b::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/concat.good b/crux-mir/test/symb_eval/vector/concat.good index 03d7cf42d..b1473c41b 100644 --- a/crux-mir/test/symb_eval/vector/concat.good +++ b/crux-mir/test/symb_eval/vector/concat.good @@ -1,3 +1,3 @@ -test concat/3a1fbbbh::f[0]: ok +test concat/a9c87648::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/copy_from_slice.good b/crux-mir/test/symb_eval/vector/copy_from_slice.good index 62b8bcbf5..c48ae65d4 100644 --- a/crux-mir/test/symb_eval/vector/copy_from_slice.good +++ b/crux-mir/test/symb_eval/vector/copy_from_slice.good @@ -1,3 +1,3 @@ -test copy_from_slice/3a1fbbbh::f[0]: ok +test copy_from_slice/c0ec1446::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/mut.good b/crux-mir/test/symb_eval/vector/mut.good index 057dae709..7f712fc6d 100644 --- a/crux-mir/test/symb_eval/vector/mut.good +++ b/crux-mir/test/symb_eval/vector/mut.good @@ -1,3 +1,3 @@ -test mut/3a1fbbbh::f[0]: ok +test mut/7326e6fa::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/new.good b/crux-mir/test/symb_eval/vector/new.good index 1d5ebf83b..1fc92321a 100644 --- a/crux-mir/test/symb_eval/vector/new.good +++ b/crux-mir/test/symb_eval/vector/new.good @@ -1,3 +1,3 @@ -test new/3a1fbbbh::f[0]: ok +test new/ed554471::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/push.good b/crux-mir/test/symb_eval/vector/push.good index 04ca2f3d2..99fd1905f 100644 --- a/crux-mir/test/symb_eval/vector/push.good +++ b/crux-mir/test/symb_eval/vector/push.good @@ -1,3 +1,3 @@ -test push/3a1fbbbh::f[0]: ok +test push/f463b0b2::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/replicate.good b/crux-mir/test/symb_eval/vector/replicate.good index a26f7f2ba..2d01d8952 100644 --- a/crux-mir/test/symb_eval/vector/replicate.good +++ b/crux-mir/test/symb_eval/vector/replicate.good @@ -1,3 +1,3 @@ -test replicate/3a1fbbbh::f[0]: ok +test replicate/99a2adac::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/split_at.good b/crux-mir/test/symb_eval/vector/split_at.good index 46c7f836e..da9830bc7 100644 --- a/crux-mir/test/symb_eval/vector/split_at.good +++ b/crux-mir/test/symb_eval/vector/split_at.good @@ -1,3 +1,3 @@ -test split_at/3a1fbbbh::f[0]: ok +test split_at/3be09e25::f[0]: ok [Crux] Overall status: Valid. From a29b2ff3c77c1c452cb8c4cab3c3d0cb656aaf8f Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Tue, 23 May 2023 19:13:48 -0400 Subject: [PATCH 055/114] crux-mir: Update test_output.log --- crux-mir/test_output.log | 3838 ++++++++++++++++---------------------- 1 file changed, 1635 insertions(+), 2203 deletions(-) diff --git a/crux-mir/test_output.log b/crux-mir/test_output.log index ee03c9a0c..38873032c 100644 --- a/crux-mir/test_output.log +++ b/crux-mir/test_output.log @@ -1,1199 +1,1036 @@ crux-mir crux concrete - refs - promoted_imm: OK (1.97s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.83s) - Crux output: 0 - mut_raw: OK (1.98s) - Compiling and running oracle program (0.15s) - Oracle output: 123 (1.83s) - Crux output: 123 - mut_nested: OK (2.05s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.89s) - Crux output: () - fn_ptr_mut: OK (2.00s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.85s) - Crux output: 1 - never: OK (1.99s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.84s) - Crux output: 1 - mut_ref: OK (2.03s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (1.87s) - Crux output: 1 - temp: OK (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.80s) - Crux output: 1 - mut_arg: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.72s) - Crux output: 1 - fn_ptr: OK (1.92s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.77s) - Crux output: 1 - promoted_mut: OK (2.01s) - Compiling and running oracle program (0.18s) - Oracle output: 0 (1.84s) - Crux output: 0 - never_mut: OK (1.96s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.81s) - Crux output: 1 - mut_tuple_field: OK (2.02s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.86s) - Crux output: () - imm_raw: OK (1.99s) - Compiling and running oracle program (0.16s) - Oracle output: 123 (1.83s) - Crux output: 123 - static_mut: OK (2.03s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (1.87s) + clos + fn_static_poly: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: 3 (0.84s) + Crux output: 3 + fnptr_closure: OK (0.99s) + Compiling and running oracle program (0.12s) + Oracle output: 7 (0.86s) + Crux output: 7 + fnptr_fn: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 4 (0.85s) + Crux output: 4 + conv_fnonce_fnmut: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: 2 (0.84s) Crux output: 2 - imm_ref: OK (1.93s) - Compiling and running oracle program (0.15s) - Oracle output: 123 (1.78s) - Crux output: 123 - imm_arg: OK (1.99s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.83s) + as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.86s) + Crux output: + failures: + + ---- as_fn_ptr/bd4bdeb8::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/bd4bdeb8::crux_test[0] + [Crux] Translation error in as_fn_ptr/bd4bdeb8::crux_test[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [], _fsreturn_ty = TyInt B32, _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:124: + crux doesn't match oracle + (expected failure) + fnonce1: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: 3 (0.84s) + Crux output: 3 + direct_fn: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 2 (0.84s) + Crux output: 2 + fn_static: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: 3 (0.84s) + Crux output: 3 + poly: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.84s) + Crux output: 3 + direct_fnmut: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 7 (0.85s) + Crux output: 7 + direct_fnmut2: OK (1.00s) + Compiling and running oracle program (0.12s) + Oracle output: 7 (0.87s) + Crux output: 7 + promoted: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 33 (0.84s) + Crux output: 33 + direct_fnonce: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 2 (0.85s) + Crux output: 2 + fo: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 29 (0.84s) + Crux output: 29 + fnonce: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: false (0.85s) + Crux output: false + fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.84s) + vtable signature mismatch for vtable core/9a570a5e::ops[0]::function[0]::Fn[0]::_vtbl570ffd80324c2ca9[0] of trait core/9a570a5e::ops[0]::function[0]::Fn[0]::_trait42f8b6475ee85722[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32)] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32)] + CallStack (from HasCallStack): + error, called at src/Mir/Trans.hs:933:20 in crucible-mir-0.1-inplace:Mir.Trans + mkTraitObject, called at src/Mir/Trans.hs:776:11 in crucible-mir-0.1-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:907:5 in crucible-mir-0.1-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:959:30 in crucible-mir-0.1-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1172:11 in crucible-mir-0.1-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1617:68 in crucible-mir-0.1-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1626:28 in crucible-mir-0.1-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1666:10 in crucible-mir-0.1-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1712:24 in crucible-mir-0.1-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2146:30 in crucible-mir-0.1-inplace:Mir.Trans + transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate + translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language + (expected failure) + ref_fnmut: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 7 (0.85s) + Crux output: 7 + dispatch_fnmut: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: 7 (0.88s) + Crux output: 7 + unique_borrow: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.85s) + Crux output: 3 + conv_fnmut_fn: OK (1.00s) + Compiling and running oracle program (0.12s) + Oracle output: 2 (0.88s) + Crux output: 2 + fnptr_fnmut: OK (1.02s) + Compiling and running oracle program (0.13s) + Oracle output: 4 (0.89s) + Crux output: 4 + fnptr_fnonce: OK (1.03s) + Compiling and running oracle program (0.13s) + Oracle output: 4 (0.90s) + Crux output: 4 + conv_fnonce_fn: OK (1.00s) + Compiling and running oracle program (0.12s) + Oracle output: 2 (0.87s) + Crux output: 2 + prim + bool: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: false (0.87s) + Crux output: false + litbstring: OK (1.00s) + Compiling and running oracle program (0.12s) + Oracle output: true (0.88s) + Crux output: true + shift3: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: -9223372036854775808 (0.84s) + Crux output: -9223372036854775808 + litstring: OK (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: true (0.88s) + Crux output: true + ge: OK (1.01s) + Compiling and running oracle program (0.12s) + Oracle output: true (0.89s) + Crux output: true + shift_exceeding: rustc compilation failed for shift_exceeding +error output: +error[E0425]: cannot find function `catch_unwind` in module `panic` + --> test/conc_eval/prim/shift_exceeding.rs:14:25 + | +14 | let result = panic::catch_unwind(|| { + | ^^^^^^^^^^^^ not found in `panic` + | +help: consider importing this function + | +3 | use std::panic::catch_unwind; + | +help: if you import `catch_unwind`, refer to it directly + | +14 - let result = panic::catch_unwind(|| { +14 + let result = catch_unwind(|| { + | + +warning: lint `exceeding_bitshifts` has been renamed to `arithmetic_overflow` + --> test/conc_eval/prim/shift_exceeding.rs:4:9 + | +4 | #[allow(exceeding_bitshifts)] + | ^^^^^^^^^^^^^^^^^^^ help: use the new name: `arithmetic_overflow` + | + = note: `#[warn(renamed_and_removed_lints)]` on by default + +error: aborting due to previous error; 1 warning emitted + +For more information about this error, try `rustc --explain E0425`. + +FAIL (expected: Should panic, but doesn't) (0.05s) + Compiling and running oracle program (0.05s) + test/Test.hs:107: + failed to compile and run + (expected failure) + shift4: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: -9223372036854775808 (0.88s) + Crux output: -9223372036854775808 + mut_arg: OK (0.99s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.86s) Crux output: 1 - hash_map - insert_multi: OK (2.32s) - Compiling and running oracle program (0.19s) - Oracle output: 100 (2.13s) - Crux output: 100 - insert_remove: OK (2.89s) - Compiling and running oracle program (0.21s) - Oracle output: 12 (2.68s) - Crux output: 12 - insert_iter: OK (2.31s) - Compiling and running oracle program (0.19s) - Oracle output: [11, 12] (2.12s) - Crux output: [11, 12] - insert_get: OK (2.32s) - Compiling and running oracle program (0.19s) - Oracle output: (11, 12) (2.14s) - Crux output: (11, 12) - traits - params: OK (2.10s) - Compiling and running oracle program (0.15s) - Oracle output: 25 (1.95s) - Crux output: 25 - generic3: OK (1.99s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.83s) - Crux output: () - static_three: OK (1.91s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.76s) + wrapping_sub: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: true (0.84s) + Crux output: true + lit: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: false (0.85s) + Crux output: false + mut: OK (0.99s) + Compiling and running oracle program (0.13s) + Oracle output: 14 (0.86s) + Crux output: 14 + shift1: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.84s) Crux output: 2 - generic1: OK (1.93s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.78s) - Crux output: () - subtrait: OK (2.03s) - Compiling and running oracle program (0.16s) - Oracle output: 73 (1.88s) - Crux output: 73 - gen_trait: OK (1.96s) - Compiling and running oracle program (0.15s) - Oracle output: -69 (1.81s) - Crux output: -69 - dynamic_branch: FAIL (0.18s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (0.03s) - user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_branch.rs) - - Use -p '/dynamic_branch/' to rerun this test only. - static_two: OK (2.00s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.85s) + add1: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: 2 (0.84s) + Crux output: 2 + div: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: 4 (0.84s) + Crux output: 4 + shift2: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: 2 (0.84s) + Crux output: 2 + ffs: OK (1.03s) + Compiling and running oracle program (0.13s) + Oracle output: true (0.90s) + Crux output: true + char_from_u32: OK (0.99s) + Compiling and running oracle program (0.12s) + Oracle output: 'A' (0.87s) + Crux output: 'A' + traits + tyfam: OK (0.98s) + Compiling and running oracle program (0.12s) + Oracle output: 23 (0.86s) + Crux output: 23 + dynamic_branch: OK (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.88s) Crux output: 1 - tyfam5: OK (1.95s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.80s) + static: OK (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.88s) + Crux output: 42 + bounds3: OK (1.02s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.88s) Crux output: () - generic2: OK (2.03s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.87s) + tyfam3: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: 23 (0.86s) + Crux output: 23 + static_two: OK (1.00s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (0.87s) + Crux output: 1 + bounds2: OK (0.99s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.87s) Crux output: () - assoc3: OK (2.03s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.88s) + generic2: OK (0.99s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.87s) Crux output: () - tyfam4: OK (1.99s) - Compiling and running oracle program (0.16s) - Oracle output: 0 (1.82s) - Crux output: 0 - static_self: OK (1.95s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.80s) - Crux output: 42 - conv: OK (1.98s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.83s) - Crux output: 1 - dynamic_poly: FAIL (0.19s) - Compiling and running oracle program (0.16s) - Oracle output: 3 (0.03s) - user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_poly.rs) - - Use -p '/dynamic_poly/' to rerun this test only. - dynamic_simple: OK (1.88s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.73s) - Crux output: 1 - gen_trait_poly: OK (1.89s) - Compiling and running oracle program (0.15s) - Oracle output: 12 (1.75s) - Crux output: 12 - default: OK (1.98s) - Compiling and running oracle program (0.15s) - Oracle output: 12 (1.83s) + gen_trait_poly: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 12 (0.85s) Crux output: 12 - assoc1: OK (2.03s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.88s) - Crux output: () - subtrait2: OK (1.99s) - Compiling and running oracle program (0.15s) - Oracle output: 64 (1.84s) - Crux output: 64 - bounds1: OK (2.05s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.90s) + dict_simple: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 32 (0.85s) + Crux output: 32 + static_self: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.83s) + Crux output: 42 + tyfam4: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 0 (0.84s) + Crux output: 0 + gen_trait: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: -69 (0.84s) + Crux output: -69 + generic3: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.85s) Crux output: () - bounds3: OK (1.96s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.80s) + subtrait: OK (0.99s) + Compiling and running oracle program (0.13s) + Oracle output: 73 (0.86s) + Crux output: 73 + default: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 12 (0.84s) + Crux output: 12 + bounds1: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.85s) Crux output: () - static_eq: OK (1.97s) - Compiling and running oracle program (0.15s) - Oracle output: true (1.82s) + intoiter: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: 0 (0.84s) + Crux output: 0 + static_eq: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: true (0.85s) Crux output: true - tyfam3: OK (1.95s) - Compiling and running oracle program (0.15s) - Oracle output: 23 (1.80s) - Crux output: 23 - dict_med: OK (1.97s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.82s) - Crux output: 42 - basics1: OK (1.97s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.83s) + assoc3: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.85s) Crux output: () - assoc2: OK (1.93s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.78s) + dynamic_poly: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.84s) + Crux output: 3 + dynamic_med: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.83s) + Crux output: 1 + static_three: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.85s) + Crux output: 2 + dynamic_two: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (0.83s) + Crux output: 1 + bounds4: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.87s) Crux output: () - dict_polymem: OK (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: 4 (1.80s) - Crux output: 4 - bounds2: OK (1.96s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.81s) + bounds5: OK (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.88s) Crux output: () - intoiter: OK (1.95s) - Compiling and running oracle program (0.15s) - Oracle output: 0 (1.80s) - Crux output: 0 - static: OK (1.93s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.78s) + dict_med: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 42 (0.84s) Crux output: 42 - dynamic_two: FAIL (0.18s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (0.03s) - user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_two.rs) - - Use -p '/dynamic_two/' to rerun this test only. - bounds4: OK (1.96s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.81s) + dynamic_simple: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.84s) + Crux output: 1 + assoc1: OK (0.98s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.86s) Crux output: () - dict_poly: OK (1.96s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.81s) + params: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: 25 (0.84s) + Crux output: 25 + conv: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (0.84s) Crux output: 1 - dict_simple: OK (1.99s) - Compiling and running oracle program (0.15s) - Oracle output: 32 (1.84s) - Crux output: 32 - tyfam: OK (2.00s) - Compiling and running oracle program (0.16s) - Oracle output: 23 (1.84s) - Crux output: 23 - bounds5: OK (2.12s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.97s) + basics1: OK (0.99s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.87s) Crux output: () - tyfam2: OK (2.02s) - Compiling and running oracle program (0.16s) - Oracle output: 23 (1.86s) + dict_poly: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.85s) + Crux output: 1 + dict_polymem: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 4 (0.84s) + Crux output: 4 + subtrait2: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 64 (0.85s) + Crux output: 64 + tyfam2: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 23 (0.85s) Crux output: 23 - dynamic_med: FAIL (0.18s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (0.03s) - user error (Error 101 while running mir-json on test/conc_eval/traits/dynamic_med.rs) - - Use -p '/dynamic_med/' to rerun this test only. - intTest - test0039: OK (2.07s) - Compiling and running oracle program (0.16s) - Oracle output: 7 (1.91s) - Crux output: 7 - test0038: OK (2.10s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.94s) + tyfam5: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.84s) Crux output: () - tuple - clone_rec: OK (2.08s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.92s) + assoc2: OK (0.99s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.86s) Crux output: () - clone: OK (1.97s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.82s) + generic1: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.86s) Crux output: () - clone_from: FAIL (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.79s) + impl + self_mut: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 42 (0.85s) + Crux output: 42 + self: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: 42 (0.83s) + Crux output: 42 + simple: OK (0.95s) + Compiling and running oracle program (0.12s) + Oracle output: 42 (0.83s) + Crux output: 42 + vec + drop: FAIL (1.19s) + Compiling and running oracle program (0.13s) + Oracle output: () (1.06s) Crux output: failures: - ---- clone_from/f3686bfc::f[0] counterexamples ---- + ---- drop/b09762c6::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/clone.rs:136:17: 136:31: error: in core/feeb1386::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0] - [Crux] Translation error in core/feeb1386::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0]: don't know how to generate CloneShim for unknown method core/feeb1386::clone[0]::Clone[0]::clone[0] + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_instb944086d0b551051[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_instb944086d0b551051[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/clone_from/' to rerun this test only. - clone_struct: FAIL (1.96s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.81s) + Use -p '/drop/' to rerun this test only. + from_elem_zero: FAIL (0.18s) + Compiling and running oracle program (0.13s) + Oracle output: 0 (0.05s) + user error (Error 101 while running mir-json on test/conc_eval/vec/from_elem_zero.rs) + + Use -p '/from_elem_zero/' to rerun this test only. + extend_trusted_len: OK (1.80s) + Compiling and running oracle program (0.14s) + Oracle output: (1, 10) (1.66s) + Crux output: (1, 10) + extend: FAIL (1.28s) + Compiling and running oracle program (0.14s) + Oracle output: (1, 10) (1.15s) Crux output: failures: - ---- clone_struct/5c02078b::f[0] counterexamples ---- + ---- extend/e200844a::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/tuple/clone_struct.rs:8:13: 8:22: error: in clone_struct/5c02078b::f[0] - [Crux] Translation error in clone_struct/5c02078b::f[0]: don't know how to generate CloneShim for unknown method core/feeb1386::clone[0]::Clone[0]::clone[0] + [Crux] internal: error: in core/9a570a5e::iter[0]::traits[0]::iterator[0]::Iterator[0]::try_fold[0]::_inst647f2421f6e4ebd6[0] + [Crux] Translation error in core/9a570a5e::iter[0]::traits[0]::iterator[0]::Iterator[0]::try_fold[0]::_inst647f2421f6e4ebd6[0]: error building variant core/9a570a5e::ops[0]::control_flow[0]::ControlFlow[0]::Continue[0]: type mismatch: expected AnyRepr but got BVRepr 32 in field 0: ([FieldRepr (FkInit AnyRepr)],[Just bVLit(32, BV 0)]) -- [Just bVLit(32, BV 0): BVRepr 32] [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/clone_struct/' to rerun this test only. - iter - for: OK (2.07s) - Compiling and running oracle program (0.16s) - Oracle output: 47 (1.92s) - Crux output: 47 - sum: OK (2.12s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.96s) - Crux output: () - filter_chain: OK (2.21s) - Compiling and running oracle program (0.18s) - Oracle output: 0 (2.03s) - Crux output: 0 - from_fn: OK (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.80s) - Crux output: () - peek: OK (2.00s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (1.85s) - Crux output: 3 - zip: OK (1.97s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.82s) - Crux output: () - loop: OK (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.79s) - Crux output: 2 - cloned: OK (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: 6 (1.79s) - Crux output: 6 - str - format_hex: FAIL (2.65s) - Compiling and running oracle program (0.15s) - Oracle output: true (2.49s) + Use -p '$0=="crux-mir.crux concrete..vec.extend"' to rerun this test only. + set_len: FAIL (1.16s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (1.04s) Crux output: failures: - ---- format_hex/92b3ccb6::crux_test[0] counterexamples ---- + ---- set_len/3493ee41::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/feeb1386::fmt[0]::write[0] - [Crux] Translation error in core/feeb1386::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/feeb1386::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/format_hex/' to rerun this test only. - format_int: FAIL (2.67s) - Compiling and running oracle program (0.16s) - Oracle output: true (2.51s) + Use -p '/set_len/' to rerun this test only. + collect: FAIL (1.86s) + Compiling and running oracle program (0.13s) + Oracle output: 45 (1.73s) Crux output: failures: - ---- format_int/c0e1710f::crux_test[0] counterexamples ---- + ---- collect/3459f282::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/feeb1386::fmt[0]::write[0] - [Crux] Translation error in core/feeb1386::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/feeb1386::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:786:58: 786:72: error: in core/9a570a5e::ptr[0]::const_ptr[0]::{impl#0}[0]::sub_ptr[0]::runtime[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::const_ptr[0]::{impl#0}[0]::sub_ptr[0]::runtime[0]::_inst1e2825177cd3b608[0]: No translation for pointer binop: >= [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/format_int/' to rerun this test only. - format_struct: FAIL (2.65s) - Compiling and running oracle program (0.16s) - Oracle output: true (2.49s) + Use -p '/collect/' to rerun this test only. + push: OK (1.19s) + Compiling and running oracle program (0.13s) + Oracle output: (1, 2) (1.06s) + Crux output: (1, 2) + slice + len: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 5 (0.85s) + Crux output: 5 + mut_range: FAIL (1.52s) + Compiling and running oracle program (0.12s) + Oracle output: 86 (1.39s) Crux output: failures: - ---- format_struct/4d89e395::crux_test[0] counterexamples ---- + ---- mut_range/1fd9f010::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/feeb1386::fmt[0]::write[0] - [Crux] Translation error in core/feeb1386::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/feeb1386::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/format_struct/' to rerun this test only. - format: FAIL (2.61s) - Compiling and running oracle program (0.15s) - Oracle output: true (2.46s) + Use -p '/mut_range/' to rerun this test only. + range_len: FAIL (1.53s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (1.41s) Crux output: failures: - ---- format/78eb465c::crux_test[0] counterexamples ---- + ---- range_len/c3db385f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/feeb1386::fmt[0]::write[0] - [Crux] Translation error in core/feeb1386::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/feeb1386::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '$0=="crux-mir.crux concrete..str.format"' to rerun this test only. - to_owned: FAIL (1.99s) - Compiling and running oracle program (0.15s) - Oracle output: true (1.84s) + Use -p '$0=="crux-mir.crux concrete..slice.range_len"' to rerun this test only. + eq: FAIL (1.02s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.90s) Crux output: failures: - ---- to_owned/0e7e1577::crux_test[0] counterexamples ---- + ---- eq/6780a00b::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/mem/mod.rs:338:38: 338:41: error: in core/9a570a5e::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/9a570a5e::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0]: callExp: Don't know how to call core/9a570a5e::intrinsics[0]::{extern#0}[0]::size_of_val[0]::_inst55af81a2cef2701c[0] [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/to_owned/' to rerun this test only. - string_push: FAIL (2.54s) - Compiling and running oracle program (0.16s) - Oracle output: true (2.39s) + Use -p '/slice.eq/' to rerun this test only. + iter_mut: FAIL (1.08s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.95s) Crux output: failures: - ---- string_push/320b1149::crux_test[0] counterexamples ---- + ---- iter_mut/43e13e25::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/array/mod.rs:266:23: 266:54: error: in core/9a570a5e::array[0]::{impl#10}[0]::try_from[0]::_insta17aeaef2aec3782[0] + [Crux] Translation error in core/9a570a5e::array[0]::{impl#10}[0]::try_from[0]::_insta17aeaef2aec3782[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyInt B32) Immut + [Crux] as: TyRawPtr (TyArray (TyInt B32) 3) Immut [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/string_push/' to rerun this test only. - format_array: FAIL (2.69s) - Compiling and running oracle program (0.16s) - Oracle output: true (2.53s) + Use -p '/iter_mut/' to rerun this test only. + get: FAIL (1.03s) + Compiling and running oracle program (0.13s) + Oracle output: true (0.90s) Crux output: failures: - ---- format_array/8db6fb19::crux_test[0] counterexamples ---- + ---- get/6397f384::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/feeb1386::fmt[0]::write[0] - [Crux] Translation error in core/feeb1386::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/feeb1386::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/format_array/' to rerun this test only. - sync - mutex: FAIL (2.48s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (2.33s) - user error (JSON Decoding of test/conc_eval/sync/mutex.all.mir failed: Error in $.statics[171].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) + Use -p '/slice.get/' to rerun this test only. + last: OK (0.99s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.87s) + Crux output: 3 + range_len_mut: FAIL (1.57s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (1.44s) + Crux output: + failures: - Use -p '$0=="crux-mir.crux concrete..sync.mutex"' to rerun this test only. - arc_cell: OK (2.15s) - Compiling and running oracle program (0.16s) - Oracle output: 4 (1.99s) - Crux output: 4 - arc: OK (2.07s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.91s) - Crux output: 1 - atomic_add: OK (2.08s) - Compiling and running oracle program (0.15s) - Oracle output: 6 (1.93s) - Crux output: 6 - rwlock: FAIL (2.68s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (2.52s) - user error (JSON Decoding of test/conc_eval/sync/rwlock.all.mir failed: Error in $.statics[172].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) + ---- range_len_mut/84f21ce0::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI - Use -p '$0=="crux-mir.crux concrete..sync.rwlock"' to rerun this test only. - atomic_cxchg: OK (2.27s) - Compiling and running oracle program (0.17s) - Oracle output: 6 (2.10s) - Crux output: 6 - arc_clone: OK (2.24s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (2.07s) - Crux output: 2 - atomic_swap: OK (2.12s) - Compiling and running oracle program (0.16s) - Oracle output: (2, 1) (1.97s) - Crux output: (2, 1) - rwlock_multi: FAIL (2.68s) - Compiling and running oracle program (0.17s) - Oracle output: 3 (2.51s) - user error (JSON Decoding of test/conc_eval/sync/rwlock_multi.all.mir failed: Error in $.statics[172].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) - - Use -p '/rwlock_multi/' to rerun this test only. - atomic_fence: OK (2.07s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.92s) - Crux output: 2 - mutex_multi: FAIL (2.81s) - Compiling and running oracle program (0.18s) - Oracle output: 3 (2.63s) - user error (JSON Decoding of test/conc_eval/sync/mutex_multi.all.mir failed: Error in $.statics[171].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) - - Use -p '/mutex_multi/' to rerun this test only. - consts - struct_val: OK (2.13s) - Compiling and running oracle program (0.16s) - Oracle output: Foo { x: true } (1.97s) - Crux output: Foo { x: true } - struct_unit: OK (2.11s) - Compiling and running oracle program (0.16s) - Oracle output: Err(Foo { x: () }) (1.95s) - Crux output: Err(Foo { x: () }) - local_key: FAIL (2.04s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.88s) - user error (JSON Decoding of test/conc_eval/consts/local_key.all.mir failed: Error in $.statics[0].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) + [Crux] Overall status: Invalid. - Use -p '/local_key/' to rerun this test only. - fn_def: FAIL (2.07s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.91s) - user error (JSON Decoding of test/conc_eval/consts/fn_def.all.mir failed: Error in $.fns[0].body.blocks[0].block.terminator.func.data.rendered: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) + test/Test.hs:124: + crux doesn't match oracle - Use -p '/fn_def/' to rerun this test only. - enum_val: OK (2.00s) - Compiling and running oracle program (0.17s) - Oracle output: None (1.82s) - Crux output: None - crypto - add: OK (2.06s) - Compiling and running oracle program (0.16s) - Oracle output: true (1.90s) - Crux output: true - add_noL: OK (2.16s) - Compiling and running oracle program (0.16s) - Oracle output: false (2.00s) - Crux output: false - cell - cell: OK (2.10s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (1.94s) - Crux output: 2 - ref_cell: OK (2.54s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (2.38s) - Crux output: 2 - ref_cell2: OK (2.67s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (2.52s) - Crux output: 2 - fnptr - call: OK (2.16s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (2.00s) - Crux output: 2 - field: OK (2.07s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (1.91s) - Crux output: 2 - make: OK (2.05s) - Compiling and running oracle program (0.16s) - Oracle output: 0 (1.89s) - Crux output: 0 - custom: rustc compilation failed for custom -error output: -error[E0432]: unresolved import `core` - --> test/conc_eval/fnptr/custom.rs:2:5 - | -2 | use core::mem; - | ^^^^ maybe a missing crate `core`? - | - = help: consider adding `extern crate core` to use the `core` crate - -error: aborting due to previous error - -For more information about this error, try `rustc --explain E0432`. - -FAIL (expected: taking address of an overridden function) (0.06s) - Compiling and running oracle program (0.06s) - test/Test.hs:107: - failed to compile and run - (expected failure) + Use -p '/range_len_mut/' to rerun this test only. + mut: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 42 (0.84s) + Crux output: 42 + swap: OK (1.02s) + Compiling and running oracle program (0.13s) + Oracle output: 2001 (0.89s) + Crux output: 2001 + mk_and_proj: OK (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.87s) + Crux output: 42 num - overflow: OK (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.00s) + overflow: OK (1.07s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.94s) Crux output: () - from_bytes: OK (2.12s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.96s) + saturate: OK (1.57s) + Compiling and running oracle program (0.13s) + Oracle output: () (1.45s) Crux output: () - saturate: OK (2.71s) - Compiling and running oracle program (0.16s) - Oracle output: () (2.56s) + from_bytes: OK (1.02s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.89s) Crux output: () - prim - bool: OK (2.10s) - Compiling and running oracle program (0.16s) - Oracle output: false (1.94s) - Crux output: false - shift3: OK (2.02s) - Compiling and running oracle program (0.15s) - Oracle output: -9223372036854775808 (1.87s) - Crux output: -9223372036854775808 - div: OK (2.01s) - Compiling and running oracle program (0.15s) - Oracle output: 4 (1.85s) + dyn + inherit: OK (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: 4 (0.89s) Crux output: 4 - ge: OK (2.02s) - Compiling and running oracle program (0.15s) - Oracle output: true (1.87s) - Crux output: true - add1: OK (2.10s) - Compiling and running oracle program (0.18s) - Oracle output: 2 (1.93s) - Crux output: 2 - shift_exceeding: rustc compilation failed for shift_exceeding -error output: -error[E0425]: cannot find function `catch_unwind` in module `panic` - --> test/conc_eval/prim/shift_exceeding.rs:14:25 - | -14 | let result = panic::catch_unwind(|| { - | ^^^^^^^^^^^^ not found in `panic` - | -help: consider importing this function - | -3 | use std::panic::catch_unwind; - | -help: if you import `catch_unwind`, refer to it directly - | -14 - let result = panic::catch_unwind(|| { -14 + let result = catch_unwind(|| { - | - -warning: lint `exceeding_bitshifts` has been renamed to `arithmetic_overflow` - --> test/conc_eval/prim/shift_exceeding.rs:4:9 - | -4 | #[allow(exceeding_bitshifts)] - | ^^^^^^^^^^^^^^^^^^^ help: use the new name: `arithmetic_overflow` - | - = note: `#[warn(renamed_and_removed_lints)]` on by default - -error: aborting due to previous error; 1 warning emitted - -For more information about this error, try `rustc --explain E0425`. - -FAIL (expected: Should panic, but doesn't) (0.06s) - Compiling and running oracle program (0.06s) - test/Test.hs:107: - failed to compile and run - (expected failure) - char_from_u32: OK (2.05s) - Compiling and running oracle program (0.15s) - Oracle output: 'A' (1.89s) - Crux output: 'A' - ffs: OK (2.09s) - Compiling and running oracle program (0.16s) - Oracle output: true (1.93s) - Crux output: true - mut_arg: OK (2.03s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.88s) - Crux output: 1 - litstring: OK (2.00s) - Compiling and running oracle program (0.15s) - Oracle output: true (1.85s) - Crux output: true - lit: OK (2.04s) - Compiling and running oracle program (0.16s) - Oracle output: false (1.88s) - Crux output: false - litbstring: OK (2.00s) - Compiling and running oracle program (0.15s) - Oracle output: true (1.85s) - Crux output: true - shift1: OK (1.99s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.84s) - Crux output: 2 - shift2: OK (2.01s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.86s) - Crux output: 2 - shift4: OK (2.06s) - Compiling and running oracle program (0.18s) - Oracle output: -9223372036854775808 (1.88s) - Crux output: -9223372036854775808 - mut: OK (1.99s) - Compiling and running oracle program (0.15s) - Oracle output: 14 (1.84s) - Crux output: 14 - wrapping_sub: OK (1.98s) - Compiling and running oracle program (0.15s) - Oracle output: true (1.83s) - Crux output: true - impl - self: OK (1.99s) - Compiling and running oracle program (0.16s) - Oracle output: 42 (1.83s) - Crux output: 42 - simple: OK (1.99s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.84s) - Crux output: 42 - self_mut: OK (1.98s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.83s) - Crux output: 42 + plain_trait: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 100 (0.85s) + Crux output: 100 + trait_param: OK (0.99s) + Compiling and running oracle program (0.12s) + Oracle output: 100 (0.86s) + Crux output: 100 + assoc_ty: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 100 (0.84s) + Crux output: 100 box - new: OK (2.16s) - Compiling and running oracle program (0.16s) - Oracle output: () (2.01s) + mut_ref: OK (1.12s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.99s) Crux output: () - mut_ref: OK (2.15s) - Compiling and running oracle program (0.16s) - Oracle output: () (2.00s) + new: OK (1.15s) + Compiling and running oracle program (0.13s) + Oracle output: () (1.02s) Crux output: () - mut: OK (2.15s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.98s) + struct: OK (1.15s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (1.03s) + Crux output: 1 + mut: OK (1.11s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.98s) Crux output: () - unsize: OK (2.14s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.99s) + unsize: OK (1.13s) + Compiling and running oracle program (0.12s) + Oracle output: 2 (1.01s) Crux output: 2 - struct: OK (2.17s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (2.01s) - Crux output: 1 - ptr - is_null_slice: FAIL (expected: can't unsize null pointers) (2.06s) - Compiling and running oracle program (0.15s) - Oracle output: (false, true) (1.90s) + vec_deque + rotate_right: FAIL (1.72s) + Compiling and running oracle program (0.13s) + Oracle output: [4, 5, 1, 2, 3] (1.59s) Crux output: failures: - ---- is_null_slice/afb4cb63::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/ptr/is_null_slice.rs:7:13: 7:33: error: in is_null_slice/afb4cb63::crux_test[0] - [Crux] attempted subindex on the result of an integer-to-pointer cast + ---- rotate_right/91384e0f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:38:38: 38:42: error: in core/feeb1386::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst55af81a2cef2701c[0] - [Crux] attempted to read empty mux tree + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - (expected failure) - struct_eq: OK (2.01s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.86s) - Crux output: () - read_write: OK (2.05s) - Compiling and running oracle program (0.16s) - Oracle output: [1, 3, 1] (1.90s) - Crux output: [1, 3, 1] - offset_mut: OK (2.05s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (1.89s) - Crux output: 3 - copy: OK (2.09s) - Compiling and running oracle program (0.17s) - Oracle output: [1, 2, 3, 1, 2, 3] (1.92s) - Crux output: [1, 2, 3, 1, 2, 3] - unsize_slice: OK (2.07s) - Compiling and running oracle program (0.16s) - Oracle output: (1, 2) (1.91s) - Crux output: (1, 2) - dangling_eq: FAIL (2.09s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.93s) + + Use -p '/rotate_right/' to rerun this test only. + retain: FAIL (1.75s) + Compiling and running oracle program (0.14s) + Oracle output: [1, 4] (1.62s) Crux output: failures: - ---- dangling_eq/7ae9455f::crux_test[0] counterexamples ---- + ---- retain/12c8a6d0::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:135:14: 135:78: error: in core/feeb1386::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/dangling_eq/' to rerun this test only. - cast_eq: OK (2.10s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.95s) - Crux output: () - offset_from: OK (2.07s) - Compiling and running oracle program (0.16s) - Oracle output: -2 (1.91s) - Crux output: -2 - null_eq: FAIL (2.07s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.91s) + Use -p '/retain/' to rerun this test only. + push: OK (1.73s) + Compiling and running oracle program (0.13s) + Oracle output: [4, 1, 2, 3, 5] (1.60s) + Crux output: [4, 1, 2, 3, 5] + rotate_left: FAIL (1.69s) + Compiling and running oracle program (0.14s) + Oracle output: [3, 4, 5, 1, 2] (1.55s) Crux output: failures: - ---- null_eq/3dc65e3a::crux_test[0] counterexamples ---- + ---- rotate_left/864001d8::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:118:14: 118:78: error: in core/feeb1386::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/null_eq/' to rerun this test only. - coerce_unsized: OK (2.11s) - Compiling and running oracle program (0.18s) - Oracle output: (1, 2) (1.93s) - Crux output: (1, 2) - offset: OK (2.00s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (1.85s) - Crux output: 3 - is_null: OK (2.02s) - Compiling and running oracle program (0.15s) - Oracle output: (false, true) (1.87s) - Crux output: (false, true) - valid_eq: OK (2.01s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.85s) - Crux output: () - ops - deref2: OK (2.01s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.86s) - Crux output: () - index2: OK (2.00s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.85s) - Crux output: () - arith1: OK (2.01s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.86s) - Crux output: () - index3: OK (2.03s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.87s) - Crux output: () - deref1: OK (2.03s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.88s) - Crux output: () - index1: OK (2.10s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.95s) - Crux output: () - deref3: OK (2.04s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.88s) - Crux output: () - statics - promoted_fn: OK (2.02s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.87s) - Crux output: 1 - promoted_static: OK (2.04s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.88s) - Crux output: 1 - io - cursor_write2: FAIL (3.35s) - Compiling and running oracle program (0.17s) - Oracle output: 0 (3.18s) - standalone use of `dyn` is not supported: TyDynamic core/feeb1386::any[0]::Erased[0]::_trait58656ccc69864acf[0] - CallStack (from HasCallStack): - error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:213:7 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:402:29 in crucible-mir-0.1-inplace:Mir.TransTy - tyToReprCont, called at src/Mir/TransTy.hs:202:24 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy - tyListToCtx, called at src/Mir/TransTy.hs:221:6 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy - tyListToCtx, called at src/Mir/TransTy.hs:1066:16 in crucible-mir-0.1-inplace:Mir.TransTy - traitVtableType, called at src/Mir/Trans.hs:2030:18 in crucible-mir-0.1-inplace:Mir.Trans - transVirtCall, called at src/Mir/Trans.hs:2153:31 in crucible-mir-0.1-inplace:Mir.Trans - transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate - translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language - - Use -p '/cursor_write2/' to rerun this test only. - cursor_write: FAIL (2.77s) - Compiling and running oracle program (0.18s) - Oracle output: 0 (2.59s) + Use -p '/rotate_left/' to rerun this test only. + pop: FAIL (1.65s) + Compiling and running oracle program (0.13s) + Oracle output: [5, 4, 3, 1, 2] (1.52s) Crux output: failures: - ---- cursor_write/fc3f6778::crux_test[0] counterexamples ---- + ---- pop/795717d2::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '$0=="crux-mir.crux concrete..io.cursor_write"' to rerun this test only. - cursor_read: FAIL (2.74s) - Compiling and running oracle program (0.17s) - Oracle output: 0 (2.56s) + Use -p '/vec_deque.pop/' to rerun this test only. + iter_clone: FAIL (1.76s) + Compiling and running oracle program (0.13s) + Oracle output: [1, 2, 3, 2, 3] (1.62s) Crux output: failures: - ---- cursor_read/337c2c0d::crux_test[0] counterexamples ---- + ---- iter_clone/17f65018::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/io.cursor_read/' to rerun this test only. - mem - maybe_uninit: FAIL (2.06s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.89s) + Use -p '/iter_clone/' to rerun this test only. + iter + loop: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: 2 (0.84s) + Crux output: 2 + from_fn: OK (1.03s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.90s) + Crux output: () + filter_chain: OK (1.15s) + Compiling and running oracle program (0.15s) + Oracle output: 0 (1.00s) + Crux output: 0 + for: OK (1.06s) + Compiling and running oracle program (0.13s) + Oracle output: 47 (0.93s) + Crux output: 47 + zip: OK (1.11s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.98s) + Crux output: () + peek: OK (1.09s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.96s) + Crux output: 3 + cloned: OK (1.02s) + Compiling and running oracle program (0.13s) + Oracle output: 6 (0.89s) + Crux output: 6 + sum: OK (1.05s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.92s) + Crux output: () + crypto + add: OK (1.05s) + Compiling and running oracle program (0.13s) + Oracle output: true (0.92s) + Crux output: true + add_noL: OK (1.05s) + Compiling and running oracle program (0.14s) + Oracle output: false (0.91s) + Crux output: false + statics + promoted_static: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (0.85s) + Crux output: 1 + promoted_fn: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.85s) + Crux output: 1 + ptr + offset_mut: OK (0.99s) + Compiling and running oracle program (0.12s) + Oracle output: 3 (0.87s) + Crux output: 3 + dangling_eq: FAIL (1.02s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.89s) Crux output: failures: - ---- maybe_uninit/d8947d0b::crux_test[0] counterexamples ---- + ---- dangling_eq/c854fd1c::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in maybe_uninit/d8947d0b::crux_test[0] - [Crux] Translation error in maybe_uninit/d8947d0b::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyInt B32]) + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:135:14: 135:78: error: in core/9a570a5e::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '$0=="crux-mir.crux concrete..mem.maybe_uninit"' to rerun this test only. - maybe_uninit_array_cast: FAIL (2.06s) - Compiling and running oracle program (0.16s) - Oracle output: [1, 2] (1.91s) + Use -p '/dangling_eq/' to rerun this test only. + is_null_slice: FAIL (expected: can't unsize null pointers) (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: (false, true) (0.89s) Crux output: failures: - ---- maybe_uninit_array_cast/0c1e07d1::crux_test[0] counterexamples ---- + ---- is_null_slice/fec67c8e::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/ptr/is_null_slice.rs:7:13: 7:33: error: in is_null_slice/fec67c8e::crux_test[0] + [Crux] attempted subindex on the result of an integer-to-pointer cast [Crux] Found counterexample for verification goal - [Crux] internal: error: in maybe_uninit_array_cast/0c1e07d1::crux_test[0] - [Crux] Translation error in maybe_uninit_array_cast/0c1e07d1::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyArray (TyInt B32) 2]) + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:38:38: 38:42: error: in core/9a570a5e::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst55af81a2cef2701c[0] + [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - - Use -p '/maybe_uninit_array_cast/' to rerun this test only. - time - instant: FAIL (3.19s) - Compiling and running oracle program (0.16s) - Oracle output: () (3.03s) - standalone use of `dyn` is not supported: TyDynamic core/feeb1386::any[0]::Erased[0]::_trait58656ccc69864acf[0] - CallStack (from HasCallStack): - error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:213:7 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:402:29 in crucible-mir-0.1-inplace:Mir.TransTy - tyToReprCont, called at src/Mir/TransTy.hs:202:24 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy - tyListToCtx, called at src/Mir/TransTy.hs:221:6 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy - tyListToCtx, called at src/Mir/TransTy.hs:1066:16 in crucible-mir-0.1-inplace:Mir.TransTy - traitVtableType, called at src/Mir/Trans.hs:2030:18 in crucible-mir-0.1-inplace:Mir.Trans - transVirtCall, called at src/Mir/Trans.hs:2153:31 in crucible-mir-0.1-inplace:Mir.Trans - transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate - translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language - - Use -p '/instant/' to rerun this test only. - struct - field_order: OK (2.02s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.87s) + (expected failure) + coerce_unsized: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: (1, 2) (0.85s) + Crux output: (1, 2) + is_null: OK (0.99s) + Compiling and running oracle program (0.12s) + Oracle output: (false, true) (0.87s) + Crux output: (false, true) + offset: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.87s) + Crux output: 3 + offset_from: OK (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: -2 (0.89s) + Crux output: -2 + struct_eq: OK (1.01s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.88s) Crux output: () - arg: OK (2.03s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.87s) - Crux output: 42 - tup: OK (2.09s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.92s) + valid_eq: OK (1.01s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.89s) Crux output: () - proj: OK (2.06s) - Compiling and running oracle program (0.17s) - Oracle output: 42 (1.90s) - Crux output: 42 - ret: OK (2.02s) - Compiling and running oracle program (0.15s) - Oracle output: S { x: 42, y: 120 } (1.86s) - Crux output: S { x: 42, y: 120 } - repr_transparent: FAIL (2.08s) - Compiling and running oracle program (0.16s) - Oracle output: 6 (1.92s) + cast_eq: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.87s) + Crux output: () + copy: OK (1.06s) + Compiling and running oracle program (0.13s) + Oracle output: [1, 2, 3, 1, 2, 3] (0.93s) + Crux output: [1, 2, 3, 1, 2, 3] + unsize_slice: OK (1.02s) + Compiling and running oracle program (0.13s) + Oracle output: (1, 2) (0.89s) + Crux output: (1, 2) + read_write: OK (1.05s) + Compiling and running oracle program (0.13s) + Oracle output: [1, 3, 1] (0.92s) + Crux output: [1, 3, 1] + null_eq: FAIL (1.04s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.92s) Crux output: failures: - ---- repr_transparent/ae0007db::crux_test[0] counterexamples ---- + ---- null_eq/b1679e97::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:46:9: 46:18: error: in core/feeb1386::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0] - [Crux] Translation error in core/feeb1386::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyInt B32) Immut - [Crux] as: TyRawPtr (TyArray (TyInt B32) 2) Immut + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:118:14: 118:78: error: in core/9a570a5e::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '$0=="crux-mir.crux concrete..struct.repr_transparent"' to rerun this test only. - repr_transparent_const: OK (2.02s) - Compiling and running oracle program (0.15s) - Oracle output: 124 (1.87s) - Crux output: 124 - dyn - assoc_ty: OK (2.05s) - Compiling and running oracle program (0.15s) - Oracle output: 100 (1.90s) - Crux output: 100 - trait_param: OK (2.08s) - Compiling and running oracle program (0.16s) - Oracle output: 100 (1.92s) - Crux output: 100 - plain_trait: OK (2.01s) - Compiling and running oracle program (0.16s) - Oracle output: 100 (1.86s) - Crux output: 100 - inherit: OK (2.02s) - Compiling and running oracle program (0.15s) - Oracle output: 4 (1.87s) - Crux output: 4 - stdlib - option: OK (2.00s) - Compiling and running oracle program (0.15s) - Oracle output: true (1.84s) - Crux output: true - option2: OK (1.99s) - Compiling and running oracle program (0.15s) - Oracle output: 0 (1.84s) - Crux output: 0 - result: OK (2.00s) - Compiling and running oracle program (0.15s) - Oracle output: 27 (1.84s) - Crux output: 27 - default: OK (2.01s) - Compiling and running oracle program (0.15s) - Oracle output: true (1.86s) - Crux output: true - cvt: OK (2.05s) - Compiling and running oracle program (0.15s) - Oracle output: 0 (1.89s) - Crux output: 0 - range: OK (2.01s) - Compiling and running oracle program (0.16s) - Oracle output: 10 (1.86s) - Crux output: 10 - poly: OK (2.05s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.89s) - Crux output: 1 - result_interior: OK (2.03s) - Compiling and running oracle program (0.16s) - Oracle output: 3 (1.87s) - Crux output: 3 - default_impl: OK (2.05s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.90s) - Crux output: () - option3: OK (2.15s) - Compiling and running oracle program (0.15s) - Oracle output: 27 (2.00s) - Crux output: 27 - teq: OK (2.07s) - Compiling and running oracle program (0.17s) - Oracle output: false (1.90s) - Crux output: false - array - wick2: OK (2.01s) - Compiling and running oracle program (0.15s) - Oracle output: true (1.86s) - Crux output: true - arg: OK (1.99s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (1.84s) - Crux output: 2 - iter: OK (2.05s) - Compiling and running oracle program (0.16s) - Oracle output: 3 (1.89s) - Crux output: 3 - mk_and_proj: OK (2.03s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.87s) - Crux output: 42 - wick3: FAIL (expected: needs Vec data structure from stdlib) (0.22s) - Compiling and running oracle program (0.17s) - Oracle output: true (0.05s) - user error (Error 101 while running mir-json on test/conc_eval/array/wick3.rs) - (expected failure) - clone: FAIL (2.15s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.98s) + Use -p '/null_eq/' to rerun this test only. + str + to_owned: FAIL (1.21s) + Compiling and running oracle program (0.13s) + Oracle output: true (1.08s) Crux output: failures: - ---- clone/93fbc27f::f[0] counterexamples ---- + ---- to_owned/54c3c5eb::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:46:9: 46:18: error: in core/feeb1386::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0] - [Crux] Translation error in core/feeb1386::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyInt B32) Immut - [Crux] as: TyRawPtr (TyArray (TyInt B32) 3) Immut + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/array.clone/' to rerun this test only. - wick1: FAIL (expected: needs Vec data structure from stdlib) (0.22s) - Compiling and running oracle program (0.17s) - Oracle output: true (0.05s) - user error (Error 101 while running mir-json on test/conc_eval/array/wick1.rs) - (expected failure) - mut_index: FAIL (2.60s) - Compiling and running oracle program (0.16s) - Oracle output: 7 (2.44s) + Use -p '/to_owned/' to rerun this test only. + format_struct: FAIL (1.94s) + Compiling and running oracle program (0.14s) + Oracle output: true (1.80s) Crux output: failures: - ---- mut_index/2ae4141f::crux_test[0] counterexamples ---- + ---- format_struct/8f121570::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/9a570a5e::fmt[0]::write[0] + [Crux] Translation error in core/9a570a5e::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/9a570a5e::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/mut_index/' to rerun this test only. - mut_arg: OK (1.99s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.83s) - Crux output: 42 - const: OK (2.05s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.90s) - Crux output: 1 - from_slice: FAIL (2.82s) - Compiling and running oracle program (0.16s) - Oracle output: () (2.66s) + Use -p '/format_struct/' to rerun this test only. + format_array: FAIL (1.88s) + Compiling and running oracle program (0.13s) + Oracle output: true (1.75s) Crux output: failures: - ---- from_slice/9d05249b::f[0] counterexamples ---- + ---- format_array/fa5f4b6c::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/array/mod.rs:266:23: 266:54: error: in core/feeb1386::array[0]::{impl#10}[0]::try_from[0]::_inste494657dbbea865b[0] - [Crux] Translation error in core/feeb1386::array[0]::{impl#10}[0]::try_from[0]::_inste494657dbbea865b[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyUint B8) Immut - [Crux] as: TyRawPtr (TyArray (TyUint B8) 4) Immut + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/9a570a5e::fmt[0]::write[0] + [Crux] Translation error in core/9a570a5e::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/9a570a5e::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/array.from_slice/' to rerun this test only. - const_impl: OK (2.13s) - Compiling and running oracle program (0.17s) - Oracle output: 5 (1.96s) - Crux output: 5 + Use -p '/format_array/' to rerun this test only. + format_int: FAIL (1.75s) + Compiling and running oracle program (0.13s) + Oracle output: true (1.61s) + Crux output: + failures: + + ---- format_int/85a95142::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/9a570a5e::fmt[0]::write[0] + [Crux] Translation error in core/9a570a5e::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/9a570a5e::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + + [Crux] Overall status: Invalid. + + test/Test.hs:124: + crux doesn't match oracle + + Use -p '/format_int/' to rerun this test only. + format: FAIL (1.72s) + Compiling and running oracle program (0.13s) + Oracle output: true (1.59s) + Crux output: + failures: + + ---- format/5293190a::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/9a570a5e::fmt[0]::write[0] + [Crux] Translation error in core/9a570a5e::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/9a570a5e::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + + [Crux] Overall status: Invalid. + + test/Test.hs:124: + crux doesn't match oracle + + Use -p '$0=="crux-mir.crux concrete..str.format"' to rerun this test only. + format_hex: FAIL (1.74s) + Compiling and running oracle program (0.13s) + Oracle output: true (1.61s) + Crux output: + failures: + + ---- format_hex/314ca526::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/9a570a5e::fmt[0]::write[0] + [Crux] Translation error in core/9a570a5e::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/9a570a5e::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + + [Crux] Overall status: Invalid. + + test/Test.hs:124: + crux doesn't match oracle + + Use -p '/format_hex/' to rerun this test only. + string_push: FAIL (1.68s) + Compiling and running oracle program (0.13s) + Oracle output: true (1.55s) + Crux output: + failures: + + ---- string_push/59079431::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + + [Crux] Overall status: Invalid. + + test/Test.hs:124: + crux doesn't match oracle + + Use -p '/string_push/' to rerun this test only. + time + instant: FAIL (2.09s) + Compiling and running oracle program (0.12s) + Oracle output: () (1.97s) + standalone use of `dyn` is not supported: TyDynamic core/9a570a5e::any[0]::Erased[0]::_trait58656ccc69864acf[0] + CallStack (from HasCallStack): + error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:213:7 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:402:29 in crucible-mir-0.1-inplace:Mir.TransTy + tyToReprCont, called at src/Mir/TransTy.hs:202:24 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy + tyListToCtx, called at src/Mir/TransTy.hs:221:6 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy + tyListToCtx, called at src/Mir/TransTy.hs:1066:16 in crucible-mir-0.1-inplace:Mir.TransTy + traitVtableType, called at src/Mir/Trans.hs:2030:18 in crucible-mir-0.1-inplace:Mir.Trans + transVirtCall, called at src/Mir/Trans.hs:2153:31 in crucible-mir-0.1-inplace:Mir.Trans + transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate + translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language + + Use -p '/instant/' to rerun this test only. enum - match: OK (2.08s) - Compiling and running oracle program (0.16s) - Oracle output: 42 (1.92s) - Crux output: 42 - field_order: OK (2.11s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.96s) - Crux output: () - mixed_discrs: OK (2.01s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.85s) - Crux output: () - arg: OK (2.00s) - Compiling and running oracle program (0.16s) - Oracle output: 0 (1.85s) - Crux output: 0 arg2: rustc compilation failed for arg2 error output: error[E0518]: attribute should be applied to function or closure @@ -1222,16 +1059,32 @@ FAIL (0.05s) failed to compile and run Use -p '/arg2/' to rerun this test only. - cow: FAIL (2.16s) - Compiling and running oracle program (0.16s) - Oracle output: 200 (2.00s) + field_order: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.85s) + Crux output: () + ret: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: (A(42), B { x: 42 }, C) (0.87s) + Crux output: (A(42), B { x: 42 }, C) + eq: OK (1.06s) + Compiling and running oracle program (0.14s) + Oracle output: () (0.91s) + Crux output: () + inner: OK (1.04s) + Compiling and running oracle program (0.14s) + Oracle output: 4 (0.91s) + Crux output: 4 + cow: FAIL (1.21s) + Compiling and running oracle program (0.13s) + Oracle output: 200 (1.08s) Crux output: failures: - ---- cow/f1ac0cae::crux_test[0] counterexamples ---- + ---- cow/daae3f2d::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1239,1473 +1092,1028 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/cow/' to rerun this test only. - ret: OK (2.05s) - Compiling and running oracle program (0.16s) - Oracle output: (A(42), B { x: 42 }, C) (1.89s) - Crux output: (A(42), B { x: 42 }, C) - eq: OK (2.09s) - Compiling and running oracle program (0.17s) - Oracle output: () (1.93s) - Crux output: () - cmp: OK (2.08s) - Compiling and running oracle program (0.16s) - Oracle output: -23 (1.92s) + cmp: OK (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: -23 (0.88s) Crux output: -23 - inner: OK (2.04s) - Compiling and running oracle program (0.15s) - Oracle output: 4 (1.89s) + match: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.87s) + Crux output: 42 + arg: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 0 (0.84s) + Crux output: 0 + mixed_discrs: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.84s) + Crux output: () + sync + arc: OK (1.13s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (1.00s) + Crux output: 1 + atomic_swap: OK (1.00s) + Compiling and running oracle program (0.12s) + Oracle output: (2, 1) (0.87s) + Crux output: (2, 1) + rwlock_multi: FAIL (1.52s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.39s) + user error (JSON Decoding of test/conc_eval/sync/rwlock_multi.all.mir failed: Error in $.statics[165].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) + + Use -p '/rwlock_multi/' to rerun this test only. + rwlock: FAIL (1.50s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (1.37s) + user error (JSON Decoding of test/conc_eval/sync/rwlock.all.mir failed: Error in $.statics[165].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) + + Use -p '$0=="crux-mir.crux concrete..sync.rwlock"' to rerun this test only. + atomic_fence: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.88s) + Crux output: 2 + mutex_multi: FAIL (1.51s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (1.38s) + user error (JSON Decoding of test/conc_eval/sync/mutex_multi.all.mir failed: Error in $.statics[163].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) + + Use -p '/mutex_multi/' to rerun this test only. + atomic_add: OK (1.06s) + Compiling and running oracle program (0.13s) + Oracle output: 6 (0.94s) + Crux output: 6 + arc_clone: OK (1.14s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (1.01s) + Crux output: 2 + arc_cell: OK (1.16s) + Compiling and running oracle program (0.13s) + Oracle output: 4 (1.03s) Crux output: 4 - vec - drop: FAIL (2.25s) - Compiling and running oracle program (0.16s) - Oracle output: () (2.09s) + mutex: FAIL (1.46s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (1.33s) + user error (JSON Decoding of test/conc_eval/sync/mutex.all.mir failed: Error in $.statics[163].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) + + Use -p '$0=="crux-mir.crux concrete..sync.mutex"' to rerun this test only. + atomic_cxchg: OK (1.10s) + Compiling and running oracle program (0.13s) + Oracle output: 6 (0.97s) + Crux output: 6 + intTest + test0038: OK (1.07s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.95s) + Crux output: () + test0039: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: 7 (0.83s) + Crux output: 7 + stdlib + result_interior: OK (0.98s) + Compiling and running oracle program (0.12s) + Oracle output: 3 (0.85s) + Crux output: 3 + option3: OK (0.95s) + Compiling and running oracle program (0.12s) + Oracle output: 27 (0.82s) + Crux output: 27 + result: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 27 (0.86s) + Crux output: 27 + poly: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.84s) + Crux output: 1 + default_impl: OK (0.95s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.83s) + Crux output: () + cvt: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 0 (0.84s) + Crux output: 0 + default: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: true (0.85s) + Crux output: true + option: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: true (0.85s) + Crux output: true + option2: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 0 (0.83s) + Crux output: 0 + teq: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: false (0.85s) + Crux output: false + range: OK (0.98s) + Compiling and running oracle program (0.12s) + Oracle output: 10 (0.85s) + Crux output: 10 + hash_map + insert_get: OK (1.32s) + Compiling and running oracle program (0.15s) + Oracle output: (11, 12) (1.17s) + Crux output: (11, 12) + insert_iter: OK (1.31s) + Compiling and running oracle program (0.15s) + Oracle output: [11, 12] (1.15s) + Crux output: [11, 12] + insert_multi: OK (1.32s) + Compiling and running oracle program (0.15s) + Oracle output: 100 (1.17s) + Crux output: 100 + insert_remove: OK (1.76s) + Compiling and running oracle program (0.15s) + Oracle output: 12 (1.61s) + Crux output: 12 + tuple + clone: OK (1.01s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.88s) + Crux output: () + clone_from: FAIL (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.88s) Crux output: failures: - ---- drop/97adc623::crux_test[0] counterexamples ---- + ---- clone_from/b523f2dc::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_instb944086d0b551051[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_instb944086d0b551051[0]: evalPlace (PField, Union) NYI + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/clone.rs:136:17: 136:31: error: in core/9a570a5e::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0] + [Crux] Translation error in core/9a570a5e::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0]: don't know how to generate CloneShim for unknown method core/9a570a5e::clone[0]::Clone[0]::clone[0] [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/drop/' to rerun this test only. - set_len: FAIL (2.24s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (2.07s) + Use -p '/clone_from/' to rerun this test only. + clone_rec: OK (0.98s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.85s) + Crux output: () + clone_struct: FAIL (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.87s) Crux output: failures: - ---- set_len/3f5b6000::crux_test[0] counterexamples ---- + ---- clone_struct/e620880a::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI + [Crux] test/conc_eval/tuple/clone_struct.rs:8:13: 8:22: error: in clone_struct/e620880a::f[0] + [Crux] Translation error in clone_struct/e620880a::f[0]: don't know how to generate CloneShim for unknown method core/9a570a5e::clone[0]::Clone[0]::clone[0] [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/set_len/' to rerun this test only. - push: OK (2.27s) - Compiling and running oracle program (0.16s) - Oracle output: (1, 2) (2.11s) - Crux output: (1, 2) - extend: FAIL (2.33s) - Compiling and running oracle program (0.17s) - Oracle output: (1, 10) (2.16s) + Use -p '/clone_struct/' to rerun this test only. + consts + struct_val: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: Foo { x: true } (0.84s) + Crux output: Foo { x: true } + struct_unit: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: Err(Foo { x: () }) (0.85s) + Crux output: Err(Foo { x: () }) + enum_val: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: None (0.83s) + Crux output: None + local_key: FAIL (0.99s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.87s) + user error (JSON Decoding of test/conc_eval/consts/local_key.all.mir failed: Error in $.statics[0].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) + + Use -p '/local_key/' to rerun this test only. + fn_def: FAIL (0.95s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (0.83s) + user error (JSON Decoding of test/conc_eval/consts/fn_def.all.mir failed: Error in $.fns[0].body.blocks[0].block.terminator.func.data.rendered: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) + + Use -p '/fn_def/' to rerun this test only. + array + iter: OK (1.05s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.92s) + Crux output: 3 + const_impl: OK (0.99s) + Compiling and running oracle program (0.12s) + Oracle output: 5 (0.86s) + Crux output: 5 + clone: FAIL (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.88s) Crux output: failures: - ---- extend/3b321784::crux_test[0] counterexamples ---- + ---- clone/c1f28444::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in core/feeb1386::iter[0]::traits[0]::iterator[0]::Iterator[0]::try_fold[0]::_inst647f2421f6e4ebd6[0] - [Crux] Translation error in core/feeb1386::iter[0]::traits[0]::iterator[0]::Iterator[0]::try_fold[0]::_inst647f2421f6e4ebd6[0]: error building variant core/feeb1386::ops[0]::control_flow[0]::ControlFlow[0]::Continue[0]: type mismatch: expected AnyRepr but got BVRepr 32 in field 0: ([FieldRepr (FkInit AnyRepr)],[Just bVLit(32, BV 0)]) -- [Just bVLit(32, BV 0): BVRepr 32] + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:46:9: 46:18: error: in core/9a570a5e::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyInt B32) Immut + [Crux] as: TyRawPtr (TyArray (TyInt B32) 3) Immut [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '$0=="crux-mir.crux concrete..vec.extend"' to rerun this test only. - collect: FAIL (3.06s) - Compiling and running oracle program (0.17s) - Oracle output: 45 (2.89s) + Use -p '/array.clone/' to rerun this test only. + mut_index: FAIL (1.49s) + Compiling and running oracle program (0.12s) + Oracle output: 7 (1.37s) Crux output: failures: - ---- collect/e38ca0df::crux_test[0] counterexamples ---- + ---- mut_index/92366878::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:786:58: 786:72: error: in core/feeb1386::ptr[0]::const_ptr[0]::{impl#0}[0]::sub_ptr[0]::runtime[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/feeb1386::ptr[0]::const_ptr[0]::{impl#0}[0]::sub_ptr[0]::runtime[0]::_inst1e2825177cd3b608[0]: No translation for pointer binop: >= + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/collect/' to rerun this test only. - extend_trusted_len: OK (3.00s) - Compiling and running oracle program (0.18s) - Oracle output: (1, 10) (2.82s) - Crux output: (1, 10) - from_elem_zero: FAIL (0.22s) - Compiling and running oracle program (0.17s) - Oracle output: 0 (0.05s) - user error (Error 101 while running mir-json on test/conc_eval/vec/from_elem_zero.rs) - - Use -p '/from_elem_zero/' to rerun this test only. - clos - promoted: OK (2.02s) - Compiling and running oracle program (0.15s) - Oracle output: 33 (1.87s) - Crux output: 33 - as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (2.03s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.88s) + Use -p '/mut_index/' to rerun this test only. + mut_arg: OK (0.98s) + Compiling and running oracle program (0.12s) + Oracle output: 42 (0.86s) + Crux output: 42 + from_slice: FAIL (1.62s) + Compiling and running oracle program (0.13s) + Oracle output: () (1.49s) Crux output: failures: - ---- as_fn_ptr/61af895c::crux_test[0] counterexamples ---- + ---- from_slice/47fdae33::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/61af895c::crux_test[0] - [Crux] Translation error in as_fn_ptr/61af895c::crux_test[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [], _fsreturn_ty = TyInt B32, _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/array/mod.rs:266:23: 266:54: error: in core/9a570a5e::array[0]::{impl#10}[0]::try_from[0]::_inste494657dbbea865b[0] + [Crux] Translation error in core/9a570a5e::array[0]::{impl#10}[0]::try_from[0]::_inste494657dbbea865b[0]: unimplemented cast: Misc + [Crux] ty: TyRawPtr (TyUint B8) Immut + [Crux] as: TyRawPtr (TyArray (TyUint B8) 4) Immut [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle + + Use -p '/array.from_slice/' to rerun this test only. + const: OK (0.95s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (0.83s) + Crux output: 1 + wick2: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: true (0.85s) + Crux output: true + wick1: FAIL (expected: needs Vec data structure from stdlib) (0.18s) + Compiling and running oracle program (0.14s) + Oracle output: true (0.04s) + user error (Error 101 while running mir-json on test/conc_eval/array/wick1.rs) (expected failure) - fnptr_fnmut: OK (2.06s) - Compiling and running oracle program (0.16s) - Oracle output: 4 (1.89s) - Crux output: 4 - direct_fnonce: OK (2.09s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (1.94s) - Crux output: 2 - conv_fnonce_fnmut: OK (2.04s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (1.89s) + arg: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: 2 (0.84s) Crux output: 2 - fn_static_poly: OK (2.03s) - Compiling and running oracle program (0.16s) - Oracle output: 3 (1.87s) - Crux output: 3 - fnptr_fnonce: OK (2.01s) - Compiling and running oracle program (0.16s) - Oracle output: 4 (1.86s) - Crux output: 4 - fn_static: OK (2.02s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (1.87s) - Crux output: 3 - fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (0.18s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (0.03s) - user error (Error 101 while running mir-json on test/conc_eval/clos/fn_dyn.rs) + mk_and_proj: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 42 (0.84s) + Crux output: 42 + wick3: FAIL (expected: needs Vec data structure from stdlib) (0.17s) + Compiling and running oracle program (0.13s) + Oracle output: true (0.04s) + user error (Error 101 while running mir-json on test/conc_eval/array/wick3.rs) (expected failure) - fnonce: OK (2.00s) - Compiling and running oracle program (0.15s) - Oracle output: false (1.85s) - Crux output: false - conv_fnmut_fn: OK (2.01s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (1.85s) - Crux output: 2 - direct_fnmut2: OK (2.03s) - Compiling and running oracle program (0.15s) - Oracle output: 7 (1.87s) - Crux output: 7 - fo: OK (2.03s) - Compiling and running oracle program (0.15s) - Oracle output: 29 (1.88s) - Crux output: 29 - unique_borrow: OK (2.02s) - Compiling and running oracle program (0.16s) - Oracle output: 3 (1.86s) - Crux output: 3 - conv_fnonce_fn: OK (2.01s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.85s) - Crux output: 2 - fnptr_fn: OK (2.02s) - Compiling and running oracle program (0.16s) - Oracle output: 4 (1.87s) - Crux output: 4 - direct_fnmut: OK (2.06s) - Compiling and running oracle program (0.16s) - Oracle output: 7 (1.90s) - Crux output: 7 - dispatch_fnmut: OK (2.08s) - Compiling and running oracle program (0.19s) - Oracle output: 7 (1.90s) - Crux output: 7 - poly: OK (2.03s) - Compiling and running oracle program (0.16s) - Oracle output: 3 (1.87s) - Crux output: 3 - direct_fn: OK (2.03s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.87s) - Crux output: 2 - fnptr_closure: OK (2.02s) - Compiling and running oracle program (0.16s) - Oracle output: 7 (1.86s) - Crux output: 7 - fnonce1: OK (2.01s) - Compiling and running oracle program (0.16s) - Oracle output: 3 (1.85s) - Crux output: 3 - ref_fnmut: OK (2.05s) - Compiling and running oracle program (0.16s) - Oracle output: 7 (1.90s) - Crux output: 7 - vec_deque - pop: FAIL (2.71s) - Compiling and running oracle program (0.17s) - Oracle output: [5, 4, 3, 1, 2] (2.54s) + mem + maybe_uninit_array_cast: FAIL (0.99s) + Compiling and running oracle program (0.12s) + Oracle output: [1, 2] (0.86s) Crux output: failures: - ---- pop/89bcc818::crux_test[0] counterexamples ---- + ---- maybe_uninit_array_cast/9824cebb::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI + [Crux] internal: error: in maybe_uninit_array_cast/9824cebb::crux_test[0] + [Crux] Translation error in maybe_uninit_array_cast/9824cebb::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyArray (TyInt B32) 2]) [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/vec_deque.pop/' to rerun this test only. - iter_clone: FAIL (2.80s) - Compiling and running oracle program (0.17s) - Oracle output: [1, 2, 3, 2, 3] (2.63s) + Use -p '/maybe_uninit_array_cast/' to rerun this test only. + maybe_uninit: FAIL (0.99s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.86s) Crux output: failures: - ---- iter_clone/d8b5a2df::crux_test[0] counterexamples ---- + ---- maybe_uninit/de6620f2::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI + [Crux] internal: error: in maybe_uninit/de6620f2::crux_test[0] + [Crux] Translation error in maybe_uninit/de6620f2::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyInt B32]) [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/iter_clone/' to rerun this test only. - push: OK (2.78s) - Compiling and running oracle program (0.17s) - Oracle output: [4, 1, 2, 3, 5] (2.61s) - Crux output: [4, 1, 2, 3, 5] - retain: FAIL (2.80s) - Compiling and running oracle program (0.17s) - Oracle output: [1, 4] (2.63s) - Crux output: - failures: - - ---- retain/b9dbae7f::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/retain/' to rerun this test only. - rotate_left: FAIL (2.88s) - Compiling and running oracle program (0.17s) - Oracle output: [3, 4, 5, 1, 2] (2.71s) - Crux output: - failures: - - ---- rotate_left/74a68a05::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/rotate_left/' to rerun this test only. - rotate_right: FAIL (2.84s) - Compiling and running oracle program (0.19s) - Oracle output: [4, 5, 1, 2, 3] (2.66s) - Crux output: - failures: - - ---- rotate_right/959d304a::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/rotate_right/' to rerun this test only. - slice - len: OK (2.02s) - Compiling and running oracle program (0.16s) - Oracle output: 5 (1.86s) - Crux output: 5 - mut_range: FAIL (2.57s) - Compiling and running oracle program (0.16s) - Oracle output: 86 (2.41s) - Crux output: - failures: - - ---- mut_range/702f154c::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/mut_range/' to rerun this test only. - range_len: FAIL (2.54s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (2.39s) - Crux output: - failures: - - ---- range_len/04ce374c::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle + Use -p '$0=="crux-mir.crux concrete..mem.maybe_uninit"' to rerun this test only. + ops + index2: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.85s) + Crux output: () + deref1: OK (0.98s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.86s) + Crux output: () + index1: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.85s) + Crux output: () + deref3: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.85s) + Crux output: () + arith1: OK (0.99s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.86s) + Crux output: () + deref2: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.84s) + Crux output: () + index3: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.84s) + Crux output: () + fnptr + field: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 2 (0.85s) + Crux output: 2 + custom: rustc compilation failed for custom +error output: +error[E0432]: unresolved import `core` + --> test/conc_eval/fnptr/custom.rs:2:5 + | +2 | use core::mem; + | ^^^^ maybe a missing crate `core`? + | + = help: consider adding `extern crate core` to use the `core` crate + +error: aborting due to previous error + +For more information about this error, try `rustc --explain E0432`. + +FAIL (expected: taking address of an overridden function) (0.05s) + Compiling and running oracle program (0.05s) + test/Test.hs:107: + failed to compile and run + (expected failure) + make: OK (0.98s) + Compiling and running oracle program (0.12s) + Oracle output: 0 (0.86s) + Crux output: 0 + call: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 2 (0.85s) + Crux output: 2 + io + cursor_write2: FAIL (2.05s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.92s) + standalone use of `dyn` is not supported: TyDynamic core/9a570a5e::any[0]::Erased[0]::_trait58656ccc69864acf[0] + CallStack (from HasCallStack): + error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:213:7 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:402:29 in crucible-mir-0.1-inplace:Mir.TransTy + tyToReprCont, called at src/Mir/TransTy.hs:202:24 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy + tyListToCtx, called at src/Mir/TransTy.hs:221:6 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy + tyListToCtx, called at src/Mir/TransTy.hs:1066:16 in crucible-mir-0.1-inplace:Mir.TransTy + traitVtableType, called at src/Mir/Trans.hs:2030:18 in crucible-mir-0.1-inplace:Mir.Trans + transVirtCall, called at src/Mir/Trans.hs:2153:31 in crucible-mir-0.1-inplace:Mir.Trans + transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate + translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language - Use -p '$0=="crux-mir.crux concrete..slice.range_len"' to rerun this test only. - range_len_mut: FAIL (2.66s) - Compiling and running oracle program (0.17s) - Oracle output: 1 (2.49s) + Use -p '/cursor_write2/' to rerun this test only. + cursor_read: FAIL (1.64s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.51s) Crux output: failures: - ---- range_len_mut/d4b69d12::crux_test[0] counterexamples ---- + ---- cursor_read/aadc00f0::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/range_len_mut/' to rerun this test only. - mk_and_proj: OK (2.02s) - Compiling and running oracle program (0.16s) - Oracle output: 42 (1.86s) - Crux output: 42 - swap: OK (2.06s) - Compiling and running oracle program (0.15s) - Oracle output: 2001 (1.90s) - Crux output: 2001 - eq: FAIL (2.02s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.87s) + Use -p '/io.cursor_read/' to rerun this test only. + cursor_write: FAIL (1.67s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.53s) Crux output: failures: - ---- eq/6323bc6f::f[0] counterexamples ---- + ---- cursor_write/2389e4f2::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/mem/mod.rs:338:38: 338:41: error: in core/feeb1386::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/feeb1386::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0]: callExp: Don't know how to call core/feeb1386::intrinsics[0]::{extern#0}[0]::size_of_val[0]::_inst55af81a2cef2701c[0] + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/slice.eq/' to rerun this test only. - iter_mut: FAIL (2.08s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.92s) + Use -p '$0=="crux-mir.crux concrete..io.cursor_write"' to rerun this test only. + cell + ref_cell2: OK (1.56s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (1.43s) + Crux output: 2 + cell: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 2 (0.85s) + Crux output: 2 + ref_cell: OK (1.60s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (1.47s) + Crux output: 2 + struct + repr_transparent: FAIL (1.04s) + Compiling and running oracle program (0.13s) + Oracle output: 6 (0.91s) Crux output: failures: - ---- iter_mut/8b0c4648::f[0] counterexamples ---- + ---- repr_transparent/f9a814ae::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/array/mod.rs:266:23: 266:54: error: in core/feeb1386::array[0]::{impl#10}[0]::try_from[0]::_insta17aeaef2aec3782[0] - [Crux] Translation error in core/feeb1386::array[0]::{impl#10}[0]::try_from[0]::_insta17aeaef2aec3782[0]: unimplemented cast: Misc + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:46:9: 46:18: error: in core/9a570a5e::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyInt B32) Immut - [Crux] as: TyRawPtr (TyArray (TyInt B32) 3) Immut - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/iter_mut/' to rerun this test only. - get: FAIL (2.03s) - Compiling and running oracle program (0.16s) - Oracle output: true (1.88s) - Crux output: - failures: - - ---- get/54cacd53::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] as: TyRawPtr (TyArray (TyInt B32) 2) Immut [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/slice.get/' to rerun this test only. - mut: OK (2.00s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.85s) + Use -p '$0=="crux-mir.crux concrete..struct.repr_transparent"' to rerun this test only. + tup: OK (0.95s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.83s) + Crux output: () + repr_transparent_const: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 124 (0.86s) + Crux output: 124 + field_order: OK (0.98s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.86s) + Crux output: () + ret: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: S { x: 42, y: 120 } (0.83s) + Crux output: S { x: 42, y: 120 } + proj: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.83s) Crux output: 42 - last: OK (2.01s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (1.86s) - Crux output: 3 + arg: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.84s) + Crux output: 42 + refs + temp: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (0.85s) + Crux output: 1 + static_mut: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.85s) + Crux output: 2 + imm_raw: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 123 (0.85s) + Crux output: 123 + mut_tuple_field: OK (0.99s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.86s) + Crux output: () + mut_ref: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (0.83s) + Crux output: 1 + mut_arg: OK (0.95s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (0.83s) + Crux output: 1 + never: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.85s) + Crux output: 1 + mut_raw: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: 123 (0.84s) + Crux output: 123 + promoted_imm: OK (0.98s) + Compiling and running oracle program (0.12s) + Oracle output: 0 (0.86s) + Crux output: 0 + mut_nested: OK (0.98s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.86s) + Crux output: () + imm_ref: OK (0.95s) + Compiling and running oracle program (0.12s) + Oracle output: 123 (0.83s) + Crux output: 123 + fn_ptr_mut: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.84s) + Crux output: 1 + promoted_mut: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 0 (0.85s) + Crux output: 0 + imm_arg: OK (0.95s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (0.83s) + Crux output: 1 + fn_ptr: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (0.84s) + Crux output: 1 + never_mut: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (0.84s) + Crux output: 1 crux symbolic - Output testing - mux_init_mut: FAIL (1.97s) - files test/symb_eval/refs/mux_init_mut.good and test/symb_eval/refs/mux_init_mut.out differ; test/symb_eval/refs/mux_init_mut.out contains: - test mux_init_mut/59f4ede3::crux_test[0]: FAILED - - failures: - - ---- mux_init_mut/59f4ede3::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - [Crux] Overall status: Invalid. - - Use -p '/mux_init_mut/' to rerun this test only. - mux_init_imm: FAIL (1.98s) - files test/symb_eval/refs/mux_init_imm.good and test/symb_eval/refs/mux_init_imm.out differ; test/symb_eval/refs/mux_init_imm.out contains: - test mux_init_imm/c11c6d2f::crux_test[0]: FAILED - - failures: - - ---- mux_init_imm/c11c6d2f::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - [Crux] Overall status: Invalid. - - Use -p '/mux_init_imm/' to rerun this test only. - early_fail: FAIL (1.98s) - files test/symb_eval/crux/early_fail.good and test/symb_eval/crux/early_fail.out differ; test/symb_eval/crux/early_fail.out contains: - test early_fail/aa1c593a::fail1[0]: FAILED - test early_fail/aa1c593a::fail2[0]: FAILED - - failures: - - ---- early_fail/aa1c593a::fail1[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in early_fail/aa1c593a::fail1[0] - [Crux] panicking::panic_fmt, called from early_fail/aa1c593a::fail1[0] - - ---- early_fail/aa1c593a::fail2[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - [Crux] Overall status: Invalid. - - Use -p '/early_fail/' to rerun this test only. - mixed_fail: FAIL (2.01s) - files test/symb_eval/crux/mixed_fail.good and test/symb_eval/crux/mixed_fail.out differ; test/symb_eval/crux/mixed_fail.out contains: - test mixed_fail/6c0add63::fail1[0]: FAILED - test mixed_fail/6c0add63::fail2[0]: FAILED - test mixed_fail/6c0add63::pass1[0]: FAILED - test mixed_fail/6c0add63::pass2[0]: FAILED - - failures: - - ---- mixed_fail/6c0add63::fail1[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - ---- mixed_fail/6c0add63::fail2[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - ---- mixed_fail/6c0add63::pass1[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - ---- mixed_fail/6c0add63::pass2[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - [Crux] Overall status: Invalid. - - Use -p '/mixed_fail/' to rerun this test only. - multi: FAIL (1.94s) - files test/symb_eval/crux/multi.good and test/symb_eval/crux/multi.out differ; test/symb_eval/crux/multi.out contains: - test multi/608d8b6e::fail1[0]: FAILED - test multi/608d8b6e::fail2[0]: FAILED - test multi/608d8b6e::fail3[0]: FAILED - - failures: - - ---- multi/608d8b6e::fail1[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - ---- multi/608d8b6e::fail2[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - ---- multi/608d8b6e::fail3[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - [Crux] Overall status: Invalid. - - Use -p '/Output testing.multi/' to rerun this test only. - fail_return: FAIL (1.98s) - files test/symb_eval/crux/fail_return.good and test/symb_eval/crux/fail_return.out differ; test/symb_eval/crux/fail_return.out contains: - test fail_return/ac160af8::fail1[0]: FAILED - test fail_return/ac160af8::fail2[0]: FAILED - - failures: - - ---- fail_return/ac160af8::fail1[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - ---- fail_return/ac160af8::fail2[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - [Crux] Overall status: Invalid. - - Use -p '/fail_return/' to rerun this test only. - no_conc: FAIL (1.97s) - files test/symb_eval/concretize/no_conc.good and test/symb_eval/concretize/no_conc.out differ; test/symb_eval/concretize/no_conc.out contains: - test no_conc/7d5e52dc::crux_test[0]: FAILED - - failures: - - ---- no_conc/7d5e52dc::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - [Crux] Overall status: Invalid. - - Use -p '/no_conc/' to rerun this test only. - conc: FAIL (1.96s) - files test/symb_eval/concretize/conc.good and test/symb_eval/concretize/conc.out differ; test/symb_eval/concretize/conc.out contains: - test conc/d6426254::crux_test[0]: FAILED - - failures: - - ---- conc/d6426254::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.conc"' to rerun this test only. - array: FAIL (2.01s) - files test/symb_eval/concretize/array.good and test/symb_eval/concretize/array.out differ; test/symb_eval/concretize/array.out contains: - test array/4739c244::crux_test[0]: FAILED - - failures: - - ---- array/4739c244::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::array[0]::symbolic[0]::_inst1e2825177cd3b608[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::array[0]::symbolic[0]::_inst1e2825177cd3b608[0] - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.array"' to rerun this test only. - assert_ok: FAIL (2.52s) - files test/symb_eval/concretize/assert_ok.good and test/symb_eval/concretize/assert_ok.out differ; test/symb_eval/concretize/assert_ok.out contains: - test assert_ok/05b748f4::crux_test[0]: FAILED - - failures: - - ---- assert_ok/05b748f4::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - [Crux] Overall status: Invalid. - - Use -p '/assert_ok/' to rerun this test only. - assert: FAIL (2.52s) - files test/symb_eval/concretize/assert.good and test/symb_eval/concretize/assert.out differ; test/symb_eval/concretize/assert.out contains: - test assert/d4658be4::crux_test[0]: FAILED - - failures: - - ---- assert/d4658be4::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.assert"' to rerun this test only. - bytes2: FAIL (2.03s) - files test/symb_eval/crypto/bytes2.good and test/symb_eval/crypto/bytes2.out differ; test/symb_eval/crypto/bytes2.out contains: - test bytes2/bf8b6d44::f[0]: FAILED - - failures: - - ---- bytes2/bf8b6d44::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - [Crux] Overall status: Invalid. - - Use -p '/bytes2/' to rerun this test only. - double: FAIL (1.90s) - files test/symb_eval/crypto/double.good and test/symb_eval/crypto/double.out differ; test/symb_eval/crypto/double.out contains: - test double/dba12598::f[0]: FAILED - - failures: - - ---- double/dba12598::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u32[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u32[0] - - [Crux] Overall status: Invalid. - - Use -p '/double/' to rerun this test only. - ffs: FAIL (2.01s) - files test/symb_eval/crypto/ffs.good and test/symb_eval/crypto/ffs.out differ; test/symb_eval/crypto/ffs.out contains: - test ffs/7ab601f7::f[0]: FAILED - - failures: - - ---- ffs/7ab601f7::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u32[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u32[0] - - [Crux] Overall status: Invalid. - - Use -p '/Output testing.ffs/' to rerun this test only. - bytes: FAIL (2.03s) - files test/symb_eval/crypto/bytes.good and test/symb_eval/crypto/bytes.out differ; test/symb_eval/crypto/bytes.out contains: - test bytes/335dfb55::f[0]: FAILED - - failures: - - ---- bytes/335dfb55::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u64[0] - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.bytes"' to rerun this test only. - bad_symb1: FAIL (2.01s) - files test/symb_eval/overrides/bad_symb1.good and test/symb_eval/overrides/bad_symb1.out differ; test/symb_eval/overrides/bad_symb1.out contains: - test bad_symb1/20a92dea::crux_test[0]: FAILED - - failures: - - ---- bad_symb1/20a92dea::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - [Crux] Overall status: Invalid. - - Use -p '/bad_symb1/' to rerun this test only. - bad_symb2: FAIL (1.91s) - files test/symb_eval/overrides/bad_symb2.good and test/symb_eval/overrides/bad_symb2.out differ; test/symb_eval/overrides/bad_symb2.out contains: - test bad_symb2/4f8712a5::crux_test[0]: FAILED - - failures: - - ---- bad_symb2/4f8712a5::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - [Crux] Overall status: Invalid. - - Use -p '/bad_symb2/' to rerun this test only. - override3: FAIL (1.91s) - files test/symb_eval/overrides/override3.good and test/symb_eval/overrides/override3.out differ; test/symb_eval/overrides/override3.out contains: - test override3/2aec5606::f[0]: FAILED - - failures: - - ---- override3/2aec5606::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - [Crux] Overall status: Invalid. - - Use -p '/override3/' to rerun this test only. - override1: FAIL (1.82s) - files test/symb_eval/overrides/override1.good and test/symb_eval/overrides/override1.out differ; test/symb_eval/overrides/override1.out contains: - test override1/6487bdad::f[0]: FAILED - - failures: - - ---- override1/6487bdad::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::one[0] - [Crux] panicking::panic, called from crucible/2c868646::one[0] - - [Crux] Overall status: Invalid. - - Use -p '/override1/' to rerun this test only. - override4: FAIL (1.96s) - files test/symb_eval/overrides/override4.good and test/symb_eval/overrides/override4.out differ; test/symb_eval/overrides/override4.out contains: - test override4/6bc9b63e::f[0]: FAILED - - failures: - - ---- override4/6bc9b63e::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - [Crux] Overall status: Invalid. - - Use -p '/override4/' to rerun this test only. - override_rust: FAIL (1.92s) - files test/symb_eval/overrides/override_rust.good and test/symb_eval/overrides/override_rust.out differ; test/symb_eval/overrides/override_rust.out contains: - test override_rust/7782f5ae::crux_test[0]: FAILED - - failures: - - ---- override_rust/7782f5ae::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::override_[0]::_inst28abcb7c380bf102[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::override_[0]::_inst28abcb7c380bf102[0] - - [Crux] Overall status: Invalid. - - Use -p '/override_rust/' to rerun this test only. - override2: FAIL (1.92s) - files test/symb_eval/overrides/override2.good and test/symb_eval/overrides/override2.out differ; test/symb_eval/overrides/override2.out contains: - test override2/2d082460::f[0]: FAILED - - failures: - - ---- override2/2d082460::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - [Crux] Overall status: Invalid. - - Use -p '/override2/' to rerun this test only. - override5: FAIL (1.92s) - files test/symb_eval/overrides/override5.good and test/symb_eval/overrides/override5.out differ; test/symb_eval/overrides/override5.out contains: - test override5/82e21c84::f[0]: FAILED - - failures: - - ---- override5/82e21c84::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u64[0] - - [Crux] Overall status: Invalid. - - Use -p '/override5/' to rerun this test only. - mux: FAIL (1.94s) - files test/symb_eval/fnptr/mux.good and test/symb_eval/fnptr/mux.out differ; test/symb_eval/fnptr/mux.out contains: - test mux/bb0881be::crux_test[0]: FAILED - - failures: - - ---- mux/bb0881be::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.mux"' to rerun this test only. - checked_mul_signed: FAIL (0.04s) - test/symb_eval/num/checked_mul_signed.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/num/checked_mul_signed.rs) - Use -p '/checked_mul_signed/' to rerun this test only. - checked_mul: FAIL - test/symb_eval/num/checked_mul.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/num/checked_mul.rs) - Use -p '$0=="crux-mir.crux symbolic.Output testing.checked_mul"' to rerun this test only. - checked_div: FAIL (1.87s) - files test/symb_eval/num/checked_div.good and test/symb_eval/num/checked_div.out differ; test/symb_eval/num/checked_div.out contains: - test checked_div/2f0ffa29::crux_test[0]: returned 65, FAILED - - failures: - - ---- checked_div/2f0ffa29::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/2f0ffa29::div_signed[0] - [Crux] attempt to compute `_3 / _4`, which would overflow - [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/2f0ffa29::div_signed[0] - [Crux] attempt to divide `_3` by zero - [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/2f0ffa29::div_unsigned[0] - [Crux] attempt to divide `_3` by zero - - [Crux] Overall status: Invalid. - - Use -p '/checked_div/' to rerun this test only. - checked_add: FAIL (0.04s) - test/symb_eval/num/checked_add.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/num/checked_add.rs) - Use -p '/checked_add/' to rerun this test only. - test1: FAIL (2.14s) - files test/symb_eval/scalar/test1.good and test/symb_eval/scalar/test1.out differ; test/symb_eval/scalar/test1.out contains: - test test1/19389055::f[0]: FAILED - - failures: - - ---- test1/19389055::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in int512/e5326267::symbolic[0] - [Crux] panicking::panic, called from int512/e5326267::symbolic[0] - - [Crux] Overall status: Invalid. - - Use -p '/test1/' to rerun this test only. - pop: FAIL (1.97s) - files test/symb_eval/vector/pop.good and test/symb_eval/vector/pop.out differ; test/symb_eval/vector/pop.out contains: - test pop/3f4f93b2::f[0]: FAILED - - failures: - - ---- pop/3f4f93b2::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in pop/3f4f93b2::f[0] - [Crux] Translation error in pop/3f4f93b2::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) - - [Crux] Overall status: Invalid. - - Use -p '/Output testing.pop/' to rerun this test only. - as_mut_slice: FAIL (1.97s) - files test/symb_eval/vector/as_mut_slice.good and test/symb_eval/vector/as_mut_slice.out differ; test/symb_eval/vector/as_mut_slice.out contains: - test as_mut_slice/cbac53b5::f[0]: FAILED + Output testing + macro: FAIL (0.99s) + files test/symb_eval/vec/macro.good and test/symb_eval/vec/macro.out differ; test/symb_eval/vec/macro.out contains: + test macro/4fbe54d8::f[0]: FAILED failures: - ---- as_mut_slice/cbac53b5::f[0] counterexamples ---- + ---- macro/4fbe54d8::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in as_mut_slice/cbac53b5::f[0] - [Crux] Translation error in as_mut_slice/cbac53b5::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. - Use -p '/as_mut_slice/' to rerun this test only. - push: FAIL (1.96s) - files test/symb_eval/vector/push.good and test/symb_eval/vector/push.out differ; test/symb_eval/vector/push.out contains: - test push/ac65ef68::f[0]: FAILED + Use -p '/Output testing.macro/' to rerun this test only. + clone: FAIL (0.98s) + files test/symb_eval/vec/clone.good and test/symb_eval/vec/clone.out differ; test/symb_eval/vec/clone.out contains: + test clone/19414d1e::f[0]: FAILED failures: - ---- push/ac65ef68::f[0] counterexamples ---- + ---- clone/19414d1e::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in push/ac65ef68::f[0] - [Crux] Translation error in push/ac65ef68::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. - Use -p '/Output testing.push/' to rerun this test only. - copy_from_slice: FAIL (1.97s) - files test/symb_eval/vector/copy_from_slice.good and test/symb_eval/vector/copy_from_slice.out differ; test/symb_eval/vector/copy_from_slice.out contains: - test copy_from_slice/f4a95669::f[0]: FAILED + Use -p '/Output testing.clone/' to rerun this test only. + sort_by_key: FAIL (1.79s) + files test/symb_eval/vec/sort_by_key.good and test/symb_eval/vec/sort_by_key.out differ; test/symb_eval/vec/sort_by_key.out contains: + test sort_by_key/c0695800::f[0]: FAILED failures: - ---- copy_from_slice/f4a95669::f[0] counterexamples ---- + ---- sort_by_key/c0695800::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/vector/copy_from_slice.rs:7:43: 7:56: error: in copy_from_slice/f4a95669::f[0] - [Crux] Translation error in copy_from_slice/f4a95669::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. - Use -p '/copy_from_slice/' to rerun this test only. - new: FAIL (1.95s) - files test/symb_eval/vector/new.good and test/symb_eval/vector/new.out differ; test/symb_eval/vector/new.out contains: - test new/c088771b::f[0]: FAILED + Use -p '/sort_by_key/' to rerun this test only. + into_iter: FAIL (1.50s) + files test/symb_eval/vec/into_iter.good and test/symb_eval/vec/into_iter.out differ; test/symb_eval/vec/into_iter.out contains: + test into_iter/23759b38::f[0]: FAILED failures: - ---- new/c088771b::f[0] counterexamples ---- + ---- into_iter/23759b38::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in new/c088771b::f[0] - [Crux] Translation error in new/c088771b::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. - Use -p '$0=="crux-mir.crux symbolic.Output testing.new"' to rerun this test only. - mut: FAIL (1.90s) - files test/symb_eval/vector/mut.good and test/symb_eval/vector/mut.out differ; test/symb_eval/vector/mut.out contains: - test mut/cc259414::f[0]: FAILED + Use -p '/into_iter/' to rerun this test only. + as_slice: OK (0.96s) + replicate: OK (0.97s) + split_at: OK (0.96s) + concat: OK (0.97s) + copy_from_slice: OK (0.94s) + new: OK (0.95s) + as_mut_slice: OK (0.91s) + mut: OK (0.96s) + push: OK (0.99s) + pop: FAIL (0.95s) + files test/symb_eval/vector/pop.good and test/symb_eval/vector/pop.out differ; test/symb_eval/vector/pop.out contains: + test pop/30fddbda::f[0]: FAILED failures: - ---- mut/cc259414::f[0] counterexamples ---- + ---- pop/30fddbda::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in mut/cc259414::f[0] - [Crux] Translation error in mut/cc259414::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] test/symb_eval/vector/pop.rs:9:18: 9:19: error: in pop/30fddbda::f[0] + [Crux] Translation error in pop/30fddbda::f[0]: unknown ADT (core/3a1fbbbh::option[0]::Option[0],Substs [TyUint B8]) [Crux] Overall status: Invalid. - Use -p '/Output testing.mut/' to rerun this test only. - concat: FAIL (1.98s) - files test/symb_eval/vector/concat.good and test/symb_eval/vector/concat.out differ; test/symb_eval/vector/concat.out contains: - test concat/fc772d1c::f[0]: FAILED + Use -p '/Output testing.pop/' to rerun this test only. + checked_div: OK (0.90s) + checked_add: OK (0.90s) + checked_mul_signed: OK (0.93s) + checked_mul: OK (0.85s) + array_mut: OK (1.01s) + array: OK (0.97s) + bad_symb1: OK (0.96s) + override5: FAIL (0.97s) + files test/symb_eval/overrides/override5.good and test/symb_eval/overrides/override5.out differ; test/symb_eval/overrides/override5.out contains: + test override5/bcc96f90::f[0]: FAILED failures: - ---- concat/fc772d1c::f[0] counterexamples ---- + ---- override5/bcc96f90::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in concat/fc772d1c::f[0] - [Crux] Translation error in concat/fc772d1c::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/overrides/override5.rs:10:5: 10:47: error: in override5/bcc96f90::f[0] + [Crux] MIR assertion at test/symb_eval/overrides/override5.rs:10:5: + [Crux] foo.wrapping_add(1) != 0 [Crux] Overall status: Invalid. - Use -p '/concat/' to rerun this test only. - split_at: FAIL (1.92s) - files test/symb_eval/vector/split_at.good and test/symb_eval/vector/split_at.out differ; test/symb_eval/vector/split_at.out contains: - test split_at/9b0f1336::f[0]: FAILED + Use -p '/override5/' to rerun this test only. + override3: OK (0.97s) + override_rust: OK (0.95s) + override1: OK (0.86s) + override2: FAIL (0.97s) + files test/symb_eval/overrides/override2.good and test/symb_eval/overrides/override2.out differ; test/symb_eval/overrides/override2.out contains: + test override2/3ba46755::f[0]: FAILED failures: - ---- split_at/9b0f1336::f[0] counterexamples ---- + ---- override2/3ba46755::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/vector/split_at.rs:7:43: 7:56: error: in split_at/9b0f1336::f[0] - [Crux] Translation error in split_at/9b0f1336::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/overrides/override2.rs:9:5: 9:49: error: in override2/3ba46755::f[0] + [Crux] MIR assertion at test/symb_eval/overrides/override2.rs:9:5: + [Crux] foo.wrapping_add(1) == foo [Crux] Overall status: Invalid. - Use -p '/split_at/' to rerun this test only. - replicate: FAIL (1.93s) - files test/symb_eval/vector/replicate.good and test/symb_eval/vector/replicate.out differ; test/symb_eval/vector/replicate.out contains: - test replicate/eb069cc4::f[0]: FAILED + Use -p '/override2/' to rerun this test only. + bad_symb2: OK (0.93s) + override4: OK (0.98s) + write: FAIL + test/symb_eval/byteorder/write.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/write.rs) + Use -p '/Output testing.write/' to rerun this test only. + read: FAIL (0.05s) + test/symb_eval/byteorder/read.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/read.rs) + Use -p '/Output testing.read/' to rerun this test only. + deserialize: OK (0.98s) + construct: FAIL (0.98s) + files test/symb_eval/sym_bytes/construct.good and test/symb_eval/sym_bytes/construct.out differ; test/symb_eval/sym_bytes/construct.out contains: + test construct/64b2e785::crux_test[0]: returned Symbolic BV, FAILED failures: - ---- replicate/eb069cc4::f[0] counterexamples ---- + ---- construct/64b2e785::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in replicate/eb069cc4::f[0] - [Crux] Translation error in replicate/eb069cc4::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/sym_bytes/construct.rs:13:5: 13:35: error: in construct/64b2e785::crux_test[0] + [Crux] MIR assertion at test/symb_eval/sym_bytes/construct.rs:13:5: + [Crux] sym2[0] == 0 [Crux] Overall status: Invalid. - Use -p '/replicate/' to rerun this test only. - as_slice: FAIL (1.97s) - files test/symb_eval/vector/as_slice.good and test/symb_eval/vector/as_slice.out differ; test/symb_eval/vector/as_slice.out contains: - test as_slice/d7157304::f[0]: FAILED + Use -p '/construct/' to rerun this test only. + double: OK (0.95s) + bytes: FAIL (1.04s) + files test/symb_eval/crypto/bytes.good and test/symb_eval/crypto/bytes.out differ; test/symb_eval/crypto/bytes.out contains: + test bytes/3d5d9475::f[0]: FAILED failures: - ---- as_slice/d7157304::f[0] counterexamples ---- + ---- bytes/3d5d9475::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in as_slice/d7157304::f[0] - [Crux] Translation error in as_slice/d7157304::f[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_3", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) - - [Crux] Overall status: Invalid. - - Use -p '/as_slice/' to rerun this test only. - array_mut: FAIL (1.91s) - files test/symb_eval/mux/array_mut.good and test/symb_eval/mux/array_mut.out differ; test/symb_eval/mux/array_mut.out contains: - test array_mut/44a64061::crux_test[0]: FAILED - - failures: - - ---- array_mut/44a64061::crux_test[0] counterexamples ---- + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/3d5d9475::f[0] + [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: + [Crux] a[i] == b[i] [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - [Crux] Overall status: Invalid. - - Use -p '/array_mut/' to rerun this test only. - array: FAIL (1.98s) - files test/symb_eval/mux/array.good and test/symb_eval/mux/array.out differ; test/symb_eval/mux/array.out contains: - test array/dd1d0c70::crux_test[0]: FAILED - - failures: - - ---- array/dd1d0c70::crux_test[0] counterexamples ---- + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/3d5d9475::f[0] + [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: + [Crux] a[i] == b[i] + [Crux] Found counterexample for verification goal + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/3d5d9475::f[0] + [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: + [Crux] a[i] == b[i] [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/3d5d9475::f[0] + [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: + [Crux] a[i] == b[i] + [Crux] Found counterexample for verification goal + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/3d5d9475::f[0] + [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: + [Crux] a[i] == b[i] [Crux] Overall status: Invalid. - Use -p '$0=="crux-mir.crux symbolic.Output testing.array"' to rerun this test only. - extend_bytes: FAIL (1.98s) - files test/symb_eval/bytes/extend_bytes.good and test/symb_eval/bytes/extend_bytes.out differ; test/symb_eval/bytes/extend_bytes.out contains: - test extend_bytes/824fdbfb::f[0]: FAILED + Use -p '$0=="crux-mir.crux symbolic.Output testing.bytes"' to rerun this test only. + ffs: OK (1.02s) + bytes2: OK (1.10s) + from_to: OK (0.84s) + overflowing_sub: OK (0.99s) + literals: OK (0.83s) + arith: OK (1.03s) + leading_zeros: FAIL (0.99s) + files test/symb_eval/bitvector/leading_zeros.good and test/symb_eval/bitvector/leading_zeros.out differ; test/symb_eval/bitvector/leading_zeros.out contains: + test leading_zeros/85b8f496::crux_test[0]: FAILED failures: - ---- extend_bytes/824fdbfb::f[0] counterexamples ---- + ---- leading_zeros/85b8f496::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/27dbd76c::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/27dbd76c::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/uint_macros.rs:131:30: 131:34 !/home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:927:5: 931:16: error: in core/9a570a5e::num[0]::{impl#10}[0]::leading_zeros[0] + [Crux] Translation error in core/9a570a5e::num[0]::{impl#10}[0]::leading_zeros[0]: callExp: Don't know how to call core/9a570a5e::intrinsics[0]::{extern#0}[0]::ctlz[0]::_inst11a9dd2a3cfdbd73[0] [Crux] Overall status: Invalid. - Use -p '/extend_bytes/' to rerun this test only. - put: FAIL (2.35s) - files test/symb_eval/bytes/put.good and test/symb_eval/bytes/put.out differ; test/symb_eval/bytes/put.out contains: - test put/7a504e32::f[0]: FAILED + Use -p '/leading_zeros/' to rerun this test only. + cmp: OK (0.98s) + symbolic: OK (0.87s) + mux: OK (0.99s) + test1: OK (101.62s) + split_off: FAIL (1.37s) + files test/symb_eval/bytes/split_off.good and test/symb_eval/bytes/split_off.out differ; test/symb_eval/bytes/split_off.out contains: + test split_off/17077784::f[0]: FAILED failures: - ---- put/7a504e32::f[0] counterexamples ---- + ---- split_off/17077784::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/27dbd76c::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/27dbd76c::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. - Use -p '$0=="crux-mir.crux symbolic.Output testing.put"' to rerun this test only. - split_to: FAIL (2.39s) + Use -p '/split_off/' to rerun this test only. + split_to: FAIL (1.45s) files test/symb_eval/bytes/split_to.good and test/symb_eval/bytes/split_to.out differ; test/symb_eval/bytes/split_to.out contains: - test split_to/6e6b755e::f[0]: FAILED + test split_to/2da81cd0::f[0]: FAILED failures: - ---- split_to/6e6b755e::f[0] counterexamples ---- + ---- split_to/2da81cd0::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/27dbd76c::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/27dbd76c::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. Use -p '/split_to/' to rerun this test only. - new: FAIL (1.96s) - files test/symb_eval/bytes/new.good and test/symb_eval/bytes/new.out differ; test/symb_eval/bytes/new.out contains: - test new/917441ba::f[0]: FAILED - - failures: - - ---- new/917441ba::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/27dbd76c::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/27dbd76c::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.new"' to rerun this test only. - split_off: FAIL (2.35s) - files test/symb_eval/bytes/split_off.good and test/symb_eval/bytes/split_off.out differ; test/symb_eval/bytes/split_off.out contains: - test split_off/1f9c9570::f[0]: FAILED + new: OK (0.91s) + put: FAIL (1.45s) + files test/symb_eval/bytes/put.good and test/symb_eval/bytes/put.out differ; test/symb_eval/bytes/put.out contains: + test put/6bf8bf24::f[0]: FAILED failures: - ---- split_off/1f9c9570::f[0] counterexamples ---- + ---- put/6bf8bf24::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/27dbd76c::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/27dbd76c::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. - Use -p '/split_off/' to rerun this test only. - sym_len: FAIL (2.46s) + Use -p '$0=="crux-mir.crux symbolic.Output testing.put"' to rerun this test only. + sym_len: FAIL (1.49s) files test/symb_eval/bytes/sym_len.good and test/symb_eval/bytes/sym_len.out differ; test/symb_eval/bytes/sym_len.out contains: - test sym_len/394525b8::f[0]: FAILED + test sym_len/94f46a68::f[0]: FAILED failures: - ---- sym_len/394525b8::f[0] counterexamples ---- + ---- sym_len/94f46a68::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/27dbd76c::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/27dbd76c::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. Use -p '/sym_len/' to rerun this test only. - put_overflow: FAIL (2.32s) + extend_bytes: OK (0.96s) + put_overflow: FAIL (1.35s) files test/symb_eval/bytes/put_overflow.good and test/symb_eval/bytes/put_overflow.out differ; test/symb_eval/bytes/put_overflow.out contains: - test put_overflow/bab83c29::f[0]: FAILED + test put_overflow/793a6d43::f[0]: FAILED failures: - ---- put_overflow/bab83c29::f[0] counterexamples ---- + ---- put_overflow/793a6d43::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/bytes.rs:73:40: 73:43: error: in bytes/27dbd76c::{impl#1}[0]::with_capacity[0] - [Crux] Translation error in bytes/27dbd76c::{impl#1}[0]::with_capacity[0]: ill-typed assignment of VectorRepr (BVRepr 8) to AnyRepr (TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/2c868646::vector[0]::Vector[0]::_adtaddce72e1232152c[0] crucible/2c868646::vector[0]::Vector[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] internal: error: in bytes/e3cb607d::{impl#4}[0]::put_slice[0] + [Crux] panicking::panic_fmt, called from bytes/e3cb607d::{impl#4}[0]::put_slice[0] [Crux] Overall status: Invalid. Use -p '/put_overflow/' to rerun this test only. - vec_cursor_read: FAIL (2.91s) - standalone use of `dyn` is not supported: TyDynamic core/feeb1386::any[0]::Erased[0]::_trait58656ccc69864acf[0] - CallStack (from HasCallStack): - error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:213:7 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:402:29 in crucible-mir-0.1-inplace:Mir.TransTy - tyToReprCont, called at src/Mir/TransTy.hs:202:24 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy - tyListToCtx, called at src/Mir/TransTy.hs:221:6 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy - tyListToCtx, called at src/Mir/TransTy.hs:1066:16 in crucible-mir-0.1-inplace:Mir.TransTy - traitVtableType, called at src/Mir/Trans.hs:2030:18 in crucible-mir-0.1-inplace:Mir.Trans - transVirtCall, called at src/Mir/Trans.hs:2153:31 in crucible-mir-0.1-inplace:Mir.Trans - transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate - translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language - Use -p '/vec_cursor_read/' to rerun this test only. - vec_write: FAIL (2.50s) - files test/symb_eval/io/vec_write.good and test/symb_eval/io/vec_write.out differ; test/symb_eval/io/vec_write.out contains: - test vec_write/f46d72c4::f[0]: ok - - [Crux] Overall status: Valid. - - Use -p '/vec_write/' to rerun this test only. - write: FAIL - test/symb_eval/byteorder/write.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/write.rs) - Use -p '/Output testing.write/' to rerun this test only. - read: FAIL (0.06s) - test/symb_eval/byteorder/read.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/read.rs) - Use -p '/Output testing.read/' to rerun this test only. - slice_mut: FAIL (1.93s) - files test/symb_eval/array/slice_mut.good and test/symb_eval/array/slice_mut.out differ; test/symb_eval/array/slice_mut.out contains: - test slice_mut/1cfe8832::crux_test[0]: FAILED - - failures: - - ---- slice_mut/1cfe8832::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in slice_mut/1cfe8832::crux_test[0] - [Crux] Translation error in slice_mut/1cfe8832::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/2c868646::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/2c868646::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/2c868646::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/2c868646::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) - - [Crux] Overall status: Invalid. - - Use -p '/slice_mut/' to rerun this test only. - basic: FAIL (1.98s) - files test/symb_eval/array/basic.good and test/symb_eval/array/basic.out differ; test/symb_eval/array/basic.out contains: - test basic/e9644cdf::crux_test[0]: FAILED - - failures: - - ---- basic/e9644cdf::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in basic/e9644cdf::crux_test[0] - [Crux] Translation error in basic/e9644cdf::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/2c868646::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/2c868646::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Immut, _varty = TyAdt crucible/2c868646::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/2c868646::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) - - [Crux] Overall status: Invalid. - - Use -p '/Output testing.basic/' to rerun this test only. - slice: FAIL (1.97s) - files test/symb_eval/array/slice.good and test/symb_eval/array/slice.out differ; test/symb_eval/array/slice.out contains: - test slice/22115682::crux_test[0]: FAILED - - failures: - - ---- slice/22115682::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in slice/22115682::crux_test[0] - [Crux] Translation error in slice/22115682::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/2c868646::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/2c868646::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/2c868646::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/2c868646::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.slice"' to rerun this test only. - mux_slice: FAIL (1.95s) - files test/symb_eval/array/mux_slice.good and test/symb_eval/array/mux_slice.out differ; test/symb_eval/array/mux_slice.out contains: - test mux_slice/3c0a096f::crux_test[0]: FAILED - - failures: - - ---- mux_slice/3c0a096f::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in mux_slice/3c0a096f::crux_test[0] - [Crux] Translation error in mux_slice/3c0a096f::crux_test[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 32) to AnyRepr (TyAdt crucible/2c868646::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/2c868646::array[0]::Array[0] (Substs [TyInt B32])) LBase (Var {_varname = "_1", _varmut = Mut, _varty = TyAdt crucible/2c868646::array[0]::Array[0]::_adt1e2825177cd3b608[0] crucible/2c868646::array[0]::Array[0] (Substs [TyInt B32]), _varIsZST = False}) - - [Crux] Overall status: Invalid. - - Use -p '/mux_slice/' to rerun this test only. - mux: FAIL (1.96s) - files test/symb_eval/enum/mux.good and test/symb_eval/enum/mux.out differ; test/symb_eval/enum/mux.out contains: - test mux/cb5d0cbe::test[0]: FAILED - - failures: - - ---- mux/cb5d0cbe::test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u8[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u8[0] - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.mux"' to rerun this test only. - uninit_read: FAIL (1.92s) - files test/symb_eval/alloc/uninit_read.good and test/symb_eval/alloc/uninit_read.out differ; test/symb_eval/alloc/uninit_read.out contains: - test uninit_read/9ace772b::crux_test[0]: FAILED - - failures: - - ---- uninit_read/9ace772b::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/9ace772b::crux_test[0] - [Crux] Attempted to read uninitialized vector index - [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/9ace772b::crux_test[0] - [Crux] attempted to read empty mux tree - - [Crux] Overall status: Invalid. - - Use -p '/uninit_read/' to rerun this test only. - valid_read: FAIL (1.94s) - files test/symb_eval/alloc/valid_read.good and test/symb_eval/alloc/valid_read.out differ; test/symb_eval/alloc/valid_read.out contains: - test valid_read/f240ed17::crux_test[0]: returned 45, ok - - [Crux] Overall status: Valid. - - Use -p '/valid_read/' to rerun this test only. - out_of_bounds: FAIL (1.99s) - files test/symb_eval/alloc/out_of_bounds.good and test/symb_eval/alloc/out_of_bounds.out differ; test/symb_eval/alloc/out_of_bounds.out contains: - test out_of_bounds/1e583a40::crux_test[0]: FAILED + multi: FAIL (0.96s) + files test/symb_eval/crux/multi.good and test/symb_eval/crux/multi.out differ; test/symb_eval/crux/multi.out contains: + test multi/42855c12::fail1[0]: FAILED + test multi/42855c12::fail2[0]: FAILED + test multi/42855c12::fail3[0]: FAILED failures: - ---- out_of_bounds/1e583a40::crux_test[0] counterexamples ---- + ---- multi/42855c12::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/1e583a40::crux_test[0] - [Crux] vector index out of range: the length is 10 but the index is BV 12 + [Crux] test/symb_eval/crux/multi.rs:8:22: 8:27: error: in multi/42855c12::fail1[0] + [Crux] attempt to compute `move _5 + const 1_u8`, which would overflow [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/1e583a40::crux_test[0] - [Crux] attempted to read empty mux tree - - [Crux] Overall status: Invalid. - - Use -p '/out_of_bounds/' to rerun this test only. - zero_length: FAIL (1.89s) - files test/symb_eval/alloc/zero_length.good and test/symb_eval/alloc/zero_length.out differ; test/symb_eval/alloc/zero_length.out contains: - test zero_length/54f30b76::crux_test[0]: ok - - [Crux] Overall status: Valid. - - Use -p '/zero_length/' to rerun this test only. - downcast: FAIL (1.89s) - files test/symb_eval/any/downcast.good and test/symb_eval/any/downcast.out differ; test/symb_eval/any/downcast.out contains: - test downcast/8170112c::crux_test[0]: returned 1, ok - - [Crux] Overall status: Valid. + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/multi.rs:8:5: 8:32: error: in multi/42855c12::fail1[0] + [Crux] MIR assertion at test/symb_eval/crux/multi.rs:8:5: + [Crux] x + 1 > x - Use -p '$0=="crux-mir.crux symbolic.Output testing.downcast"' to rerun this test only. - downcast_fail: FAIL (1.90s) - files test/symb_eval/any/downcast_fail.good and test/symb_eval/any/downcast_fail.out differ; test/symb_eval/any/downcast_fail.out contains: - test downcast_fail/21cf5aa1::crux_test[0]: FAILED - - failures: - - ---- downcast_fail/21cf5aa1::crux_test[0] counterexamples ---- + ---- multi/42855c12::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/any/downcast_fail.rs:10:14: 10:15: error: in downcast_fail/21cf5aa1::crux_test[0] - [Crux] failed to downcast Any as BVRepr 32 - - [Crux] Overall status: Invalid. - - Use -p '/downcast_fail/' to rerun this test only. - conditional: FAIL (0.04s) - test/symb_eval/any/conditional.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/any/conditional.rs) - Use -p '/conditional/' to rerun this test only. - literals: FAIL (1.88s) - files test/symb_eval/bitvector/literals.good and test/symb_eval/bitvector/literals.out differ; test/symb_eval/bitvector/literals.out contains: - test literals/9e53c8fd::crux_test[0]: FAILED - - failures: + [Crux] internal: error: in multi/42855c12::fail2[0] + [Crux] panicking::panic_fmt, called from multi/42855c12::fail2[0] - ---- literals/9e53c8fd::crux_test[0] counterexamples ---- + ---- multi/42855c12::fail3[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:91:29: 91:30 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:125:1: 125:53: error: in crucible/2c868646::bitvector[0]::{impl#27}[0]::from[0]::_inst9ea2f935eda83dab[0] - [Crux] Translation error in crucible/2c868646::bitvector[0]::{impl#27}[0]::from[0]::_inst9ea2f935eda83dab[0]: BUG: invalid arguments to bv_convert: [$38: AnyRepr] + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/multi.rs:20:5: 20:29: error: in multi/42855c12::assert_zero[0] + [Crux] MIR assertion at test/symb_eval/crux/multi.rs:20:5: + [Crux] x == 0 [Crux] Overall status: Invalid. - Use -p '/literals/' to rerun this test only. - arith: FAIL (1.96s) - files test/symb_eval/bitvector/arith.good and test/symb_eval/bitvector/arith.out differ; test/symb_eval/bitvector/arith.out contains: - test arith/92eb28c3::crux_test[0]: FAILED + Use -p '/Output testing.multi/' to rerun this test only. + fail_return: FAIL (0.98s) + files test/symb_eval/crux/fail_return.good and test/symb_eval/crux/fail_return.out differ; test/symb_eval/crux/fail_return.out contains: + test fail_return/c4d3092b::fail1[0]: returned Symbolic BV, FAILED + test fail_return/c4d3092b::fail2[0]: returned 123, FAILED failures: - ---- arith/92eb28c3::crux_test[0] counterexamples ---- + ---- fail_return/c4d3092b::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u64[0] - - [Crux] Overall status: Invalid. - - Use -p '/Output testing.arith/' to rerun this test only. - leading_zeros: FAIL (1.94s) - files test/symb_eval/bitvector/leading_zeros.good and test/symb_eval/bitvector/leading_zeros.out differ; test/symb_eval/bitvector/leading_zeros.out contains: - test leading_zeros/a2716535::crux_test[0]: FAILED - - failures: - - ---- leading_zeros/a2716535::crux_test[0] counterexamples ---- + [Crux] test/symb_eval/crux/fail_return.rs:8:22: 8:27: error: in fail_return/c4d3092b::fail1[0] + [Crux] attempt to compute `move _4 + const 1_u8`, which would overflow [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u64[0] - - [Crux] Overall status: Invalid. - - Use -p '/leading_zeros/' to rerun this test only. - overflowing_sub: FAIL (1.93s) - files test/symb_eval/bitvector/overflowing_sub.good and test/symb_eval/bitvector/overflowing_sub.out differ; test/symb_eval/bitvector/overflowing_sub.out contains: - test overflowing_sub/041a4211::crux_test[0]: FAILED + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/fail_return.rs:8:5: 8:32: error: in fail_return/c4d3092b::fail1[0] + [Crux] MIR assertion at test/symb_eval/crux/fail_return.rs:8:5: + [Crux] x + 1 > x - failures: - - ---- overflowing_sub/041a4211::crux_test[0] counterexamples ---- + ---- fail_return/c4d3092b::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u64[0] - - [Crux] Overall status: Invalid. - - Use -p '/overflowing_sub/' to rerun this test only. - cmp: FAIL (1.98s) - files test/symb_eval/bitvector/cmp.good and test/symb_eval/bitvector/cmp.out differ; test/symb_eval/bitvector/cmp.out contains: - test cmp/c8db4f55::crux_test[0]: FAILED - - failures: - - ---- cmp/c8db4f55::crux_test[0] counterexamples ---- + [Crux] test/symb_eval/crux/fail_return.rs:15:22: 15:27: error: in fail_return/c4d3092b::fail2[0] + [Crux] attempt to compute `move _5 + const 1_u8`, which would overflow [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::symbolic[0]::symbolic_u64[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::symbolic[0]::symbolic_u64[0] + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/fail_return.rs:15:5: 15:32: error: in fail_return/c4d3092b::fail2[0] + [Crux] MIR assertion at test/symb_eval/crux/fail_return.rs:15:5: + [Crux] x + 1 > x [Crux] Overall status: Invalid. - Use -p '/Output testing.cmp/' to rerun this test only. - symbolic: FAIL (1.88s) - files test/symb_eval/bitvector/symbolic.good and test/symb_eval/bitvector/symbolic.out differ; test/symb_eval/bitvector/symbolic.out contains: - test symbolic/87036093::crux_test[0]: FAILED + Use -p '/fail_return/' to rerun this test only. + mixed_fail: FAIL (1.00s) + files test/symb_eval/crux/mixed_fail.good and test/symb_eval/crux/mixed_fail.out differ; test/symb_eval/crux/mixed_fail.out contains: + test mixed_fail/d59a8d8b::fail1[0]: FAILED + test mixed_fail/d59a8d8b::fail2[0]: FAILED + test mixed_fail/d59a8d8b::pass1[0]: ok + test mixed_fail/d59a8d8b::pass2[0]: ok failures: - ---- symbolic/87036093::crux_test[0] counterexamples ---- + ---- mixed_fail/d59a8d8b::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::bitvector[0]::make_symbolic_256[0] - [Crux] panicking::panic, called from crucible/2c868646::bitvector[0]::make_symbolic_256[0] - - [Crux] Overall status: Invalid. - - Use -p '/Output testing.symbolic/' to rerun this test only. - from_to: FAIL (1.82s) - files test/symb_eval/bitvector/from_to.good and test/symb_eval/bitvector/from_to.out differ; test/symb_eval/bitvector/from_to.out contains: - test from_to/a99ea3e1::crux_test[0]: FAILED - - failures: - - ---- from_to/a99ea3e1::crux_test[0] counterexamples ---- + [Crux] test/symb_eval/crux/mixed_fail.rs:8:22: 8:27: error: in mixed_fail/d59a8d8b::fail1[0] + [Crux] attempt to compute `move _5 + const 1_u8`, which would overflow [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:103:29: 103:30 !/home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/bitvector.rs:125:1: 125:53: error: in crucible/2c868646::bitvector[0]::{impl#21}[0]::from[0]::_inst9ea2f935eda83dab[0] - [Crux] Translation error in crucible/2c868646::bitvector[0]::{impl#21}[0]::from[0]::_inst9ea2f935eda83dab[0]: BUG: invalid arguments to bv_convert: [$34: BVRepr 64] + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/mixed_fail.rs:8:5: 8:32: error: in mixed_fail/d59a8d8b::fail1[0] + [Crux] MIR assertion at test/symb_eval/crux/mixed_fail.rs:8:5: + [Crux] x + 1 > x - [Crux] Overall status: Invalid. - - Use -p '/from_to/' to rerun this test only. - clone: FAIL (2.01s) - files test/symb_eval/vec/clone.good and test/symb_eval/vec/clone.out differ; test/symb_eval/vec/clone.out contains: - test clone/725f93b1::f[0]: FAILED - - failures: - - ---- clone/725f93b1::f[0] counterexamples ---- + ---- mixed_fail/d59a8d8b::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - Use -p '/Output testing.clone/' to rerun this test only. - sort_by_key: FAIL (2.83s) - files test/symb_eval/vec/sort_by_key.good and test/symb_eval/vec/sort_by_key.out differ; test/symb_eval/vec/sort_by_key.out contains: - test sort_by_key/525a0c32::f[0]: FAILED - - failures: - - ---- sort_by_key/525a0c32::f[0] counterexamples ---- + [Crux] test/symb_eval/crux/mixed_fail.rs:14:22: 14:27: error: in mixed_fail/d59a8d8b::fail2[0] + [Crux] attempt to compute `move _5 + const 2_u8`, which would overflow [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/mixed_fail.rs:14:5: 14:32: error: in mixed_fail/d59a8d8b::fail2[0] + [Crux] MIR assertion at test/symb_eval/crux/mixed_fail.rs:14:5: + [Crux] x + 2 > x [Crux] Overall status: Invalid. - Use -p '/sort_by_key/' to rerun this test only. - into_iter: FAIL (2.59s) - files test/symb_eval/vec/into_iter.good and test/symb_eval/vec/into_iter.out differ; test/symb_eval/vec/into_iter.out contains: - test into_iter/addfe790::f[0]: FAILED + Use -p '/mixed_fail/' to rerun this test only. + early_fail: FAIL (0.98s) + files test/symb_eval/crux/early_fail.good and test/symb_eval/crux/early_fail.out differ; test/symb_eval/crux/early_fail.out contains: + test early_fail/09aa9433::fail1[0]: FAILED + test early_fail/09aa9433::fail2[0]: FAILED failures: - ---- into_iter/addfe790::f[0] counterexamples ---- + ---- early_fail/09aa9433::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - Use -p '/into_iter/' to rerun this test only. - macro: FAIL (2.06s) - files test/symb_eval/vec/macro.good and test/symb_eval/vec/macro.out differ; test/symb_eval/vec/macro.out contains: - test macro/5e6d3881::f[0]: FAILED - - failures: + [Crux] internal: error: in early_fail/09aa9433::fail1[0] + [Crux] panicking::panic_fmt, called from early_fail/09aa9433::fail1[0] - ---- macro/5e6d3881::f[0] counterexamples ---- + ---- early_fail/09aa9433::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/feeb1386::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI + [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/early_fail.rs:17:5: 17:29: error: in early_fail/09aa9433::fail2[0] + [Crux] MIR assertion at test/symb_eval/crux/early_fail.rs:17:5: + [Crux] x == 0 [Crux] Overall status: Invalid. - Use -p '/Output testing.macro/' to rerun this test only. - construct: FAIL (1.96s) - files test/symb_eval/sym_bytes/construct.good and test/symb_eval/sym_bytes/construct.out differ; test/symb_eval/sym_bytes/construct.out contains: - test construct/e7a21663::crux_test[0]: FAILED + Use -p '/early_fail/' to rerun this test only. + basic: OK (0.99s) + slice: OK (1.02s) + slice_mut: OK (1.02s) + mux_slice: OK (1.02s) + downcast_fail: FAIL (0.93s) + files test/symb_eval/any/downcast_fail.good and test/symb_eval/any/downcast_fail.out differ; test/symb_eval/any/downcast_fail.out contains: + test downcast_fail/474c9422::crux_test[0]: FAILED failures: - ---- construct/e7a21663::crux_test[0] counterexamples ---- + ---- downcast_fail/474c9422::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::sym_bytes[0]::{impl#0}[0]::zeroed[0] - [Crux] Translation error in crucible/2c868646::sym_bytes[0]::{impl#0}[0]::zeroed[0]: ill-typed assignment of SymbolicArrayRepr [BaseBVRepr 32] (BaseBVRepr 8) to AnyRepr (TyAdt crucible/2c868646::array[0]::Array[0]::_adtaddce72e1232152c[0] crucible/2c868646::array[0]::Array[0] (Substs [TyUint B8])) LBase (Var {_varname = "_2", _varmut = Mut, _varty = TyAdt crucible/2c868646::array[0]::Array[0]::_adtaddce72e1232152c[0] crucible/2c868646::array[0]::Array[0] (Substs [TyUint B8]), _varIsZST = False}) + [Crux] test/symb_eval/any/downcast_fail.rs:10:14: 10:15: error: in downcast_fail/474c9422::crux_test[0] + [Crux] failed to downcast Any as BVRepr 32 [Crux] Overall status: Invalid. - Use -p '/construct/' to rerun this test only. - deserialize: FAIL (1.91s) - files test/symb_eval/sym_bytes/deserialize.good and test/symb_eval/sym_bytes/deserialize.out differ; test/symb_eval/sym_bytes/deserialize.out contains: - test deserialize/f2eb315f::crux_test[0]: FAILED + Use -p '/downcast_fail/' to rerun this test only. + downcast: OK (0.96s) + conditional: OK (0.90s) + mux: OK (0.91s) + vec_cursor_read: FAIL (1.92s) + standalone use of `dyn` is not supported: TyDynamic core/9a570a5e::any[0]::Erased[0]::_trait58656ccc69864acf[0] + CallStack (from HasCallStack): + error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:213:7 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:402:29 in crucible-mir-0.1-inplace:Mir.TransTy + tyToReprCont, called at src/Mir/TransTy.hs:202:24 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy + tyListToCtx, called at src/Mir/TransTy.hs:221:6 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy + tyListToCtx, called at src/Mir/TransTy.hs:1066:16 in crucible-mir-0.1-inplace:Mir.TransTy + traitVtableType, called at src/Mir/Trans.hs:2030:18 in crucible-mir-0.1-inplace:Mir.Trans + transVirtCall, called at src/Mir/Trans.hs:2153:31 in crucible-mir-0.1-inplace:Mir.Trans + transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate + translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language + Use -p '/vec_cursor_read/' to rerun this test only. + vec_write: OK (1.45s) + mux_init_mut: OK (0.95s) + mux_init_imm: OK (0.98s) + uninit_read: OK (0.99s) + valid_read: OK (0.98s) + zero_length: OK (0.94s) + out_of_bounds: OK (1.01s) + no_conc: OK (1.00s) + assert: FAIL (1.64s) + files test/symb_eval/concretize/assert.good and test/symb_eval/concretize/assert.out differ; test/symb_eval/concretize/assert.out contains: + test assert/58758234::crux_test[0]: returned 1, FAILED failures: - ---- deserialize/f2eb315f::crux_test[0] counterexamples ---- + ---- assert/58758234::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in crucible/2c868646::array[0]::symbolic[0]::_instaddce72e1232152c[0] - [Crux] panicking::panic_fmt, called from crucible/2c868646::array[0]::symbolic[0]::_instaddce72e1232152c[0] + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/9a570a5e::fmt[0]::write[0] + [Crux] Translation error in core/9a570a5e::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/9a570a5e::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. - Use -p '/deserialize/' to rerun this test only. + Use -p '$0=="crux-mir.crux symbolic.Output testing.assert"' to rerun this test only. + array: OK (1.00s) + conc: OK (0.94s) + assert_ok: OK (1.52s) crux coverage Output testing - coverage_dual: warning: trailing semicolon in macro used in expression position + coverage_try: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -2719,12 +2127,8 @@ FAIL (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_dual/2d5bca87::crux_test2[0]/report_data.js" and "test/coverage/out/coverage_dual/30f9fe77::crux_test1[0]/report_data.js" do not match', src/main.rs:825:13 -note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (1.88s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_dual/30f9fe77::crux_test1[0]/report_data.js" "test/coverage/out/coverage_dual/2d5bca87::crux_test2[0]/report_data.js" "test/coverage/out/coverage_dual/4ed2e432::crux_test1[0]/report_data.js" "test/coverage/out/coverage_dual/cbc8b50e::crux_test2[0]/report_data.js" "test/coverage/out/coverage_dual/2d5bca87::crux_test1[0]/report_data.js" "test/coverage/out/coverage_dual/83032ba6::crux_test1[0]/report_data.js" "test/coverage/out/coverage_dual/30f9fe77::crux_test2[0]/report_data.js" "test/coverage/out/coverage_dual/4ed2e432::crux_test2[0]/report_data.js" "test/coverage/out/coverage_dual/83032ba6::crux_test2[0]/report_data.js" "test/coverage/out/coverage_dual/cbc8b50e::crux_test1[0]/report_data.js" (exit 101): failed - Use -p '/coverage_dual/' to rerun this test only. - coverage: warning: trailing semicolon in macro used in expression position +OK (1.04s) + coverage_cond: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -2738,12 +2142,8 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage/c30e4ff4::crux_test[0]/report_data.js" and "test/coverage/out/coverage/0fe46bcc::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 -note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (2.01s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage/0fe46bcc::crux_test[0]/report_data.js" "test/coverage/out/coverage/c30e4ff4::crux_test[0]/report_data.js" "test/coverage/out/coverage/ba917dd2::crux_test[0]/report_data.js" "test/coverage/out/coverage/cb3943dc::crux_test[0]/report_data.js" "test/coverage/out/coverage/b5488ae7::crux_test[0]/report_data.js" (exit 101): failed - Use -p '$0=="crux-mir.crux coverage.Output testing.coverage"' to rerun this test only. - coverage_try: warning: trailing semicolon in macro used in expression position +OK (0.90s) + coverage: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -2757,12 +2157,17 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_try/853a6496::crux_test[0]/report_data.js" and "test/coverage/out/coverage_try/d0e6f11f::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 -note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (2.01s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_try/d0e6f11f::crux_test[0]/report_data.js" "test/coverage/out/coverage_try/853a6496::crux_test[0]/report_data.js" "test/coverage/out/coverage_try/c9abcb4f::crux_test[0]/report_data.js" "test/coverage/out/coverage_try/a9dc15f2::crux_test[0]/report_data.js" "test/coverage/out/coverage_try/28ff1843::crux_test[0]/report_data.js" (exit 101): failed - Use -p '/coverage_try/' to rerun this test only. - coverage_cond: warning: trailing semicolon in macro used in expression position +FAIL (1.00s) + files test/coverage/coverage.good and test/coverage/coverage.out differ; test/coverage/coverage.out contains: + warning: branch condition never has a value other than [1] + ┌─ test/coverage/coverage.rs:11:15 + │ + 11 │ } else if x == 1 { + │ ^ + + + Use -p '$0=="crux-mir.crux coverage.Output testing.coverage"' to rerun this test only. + coverage_mono: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -2776,12 +2181,8 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_cond/133a307f::crux_test[0]/report_data.js" and "test/coverage/out/coverage_cond/3dfc8004::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 -note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (1.90s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_cond/3dfc8004::crux_test[0]/report_data.js" "test/coverage/out/coverage_cond/133a307f::crux_test[0]/report_data.js" "test/coverage/out/coverage_cond/2c579738::crux_test[0]/report_data.js" "test/coverage/out/coverage_cond/3f380757::crux_test[0]/report_data.js" "test/coverage/out/coverage_cond/b4b5a400::crux_test[0]/report_data.js" (exit 101): failed - Use -p '/coverage_cond/' to rerun this test only. - coverage_loop: warning: trailing semicolon in macro used in expression position +OK (0.85s) + coverage_macro: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -2795,12 +2196,20 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_loop/fff32a45::crux_test[0]/report_data.js" and "test/coverage/out/coverage_loop/04b47bd3::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 -note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (1.94s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_loop/04b47bd3::crux_test[0]/report_data.js" "test/coverage/out/coverage_loop/fff32a45::crux_test[0]/report_data.js" "test/coverage/out/coverage_loop/bf94e0f1::crux_test[0]/report_data.js" "test/coverage/out/coverage_loop/09255ce2::crux_test[0]/report_data.js" "test/coverage/out/coverage_loop/81f6cac8::crux_test[0]/report_data.js" (exit 101): failed - Use -p '/coverage_loop/' to rerun this test only. - coverage_shortcircuit: warning: trailing semicolon in macro used in expression position +FAIL (1.01s) + files test/coverage/coverage_macro.good and test/coverage/coverage_macro.out differ; test/coverage/coverage_macro.out contains: + warning: branch condition never has value true + ┌─ test/coverage/coverage_macro.rs:7:12 + │ + 7 │ if !$cond { + │ ^^^^^^ + · + 18 │ check!(u8::try_from(y_32).is_ok()); + │ ---------------------------------- in this macro invocation + + + Use -p '/coverage_macro/' to rerun this test only. + coverage_dual: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -2814,12 +2223,8 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_shortcircuit/5762c1e1::crux_test[0]/report_data.js" and "test/coverage/out/coverage_shortcircuit/86a0f0fa::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 -note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (1.98s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_shortcircuit/86a0f0fa::crux_test[0]/report_data.js" "test/coverage/out/coverage_shortcircuit/5762c1e1::crux_test[0]/report_data.js" "test/coverage/out/coverage_shortcircuit/466e729a::crux_test[0]/report_data.js" "test/coverage/out/coverage_shortcircuit/e95f8b07::crux_test[0]/report_data.js" "test/coverage/out/coverage_shortcircuit/f90f762e::crux_test[0]/report_data.js" (exit 101): failed - Use -p '/coverage_shortcircuit/' to rerun this test only. - coverage_mono: warning: trailing semicolon in macro used in expression position +OK (0.89s) + coverage_match: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -2833,12 +2238,23 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_mono/94654c01::crux_test[0]/report_data.js" and "test/coverage/out/coverage_mono/564877e0::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 -note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (1.89s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_mono/564877e0::crux_test[0]/report_data.js" "test/coverage/out/coverage_mono/94654c01::crux_test[0]/report_data.js" "test/coverage/out/coverage_mono/03e46ae4::crux_test[0]/report_data.js" "test/coverage/out/coverage_mono/d8d1fa00::crux_test[0]/report_data.js" "test/coverage/out/coverage_mono/d7f56dd4::crux_test[0]/report_data.js" (exit 101): failed - Use -p '/coverage_mono/' to rerun this test only. - coverage_match: warning: trailing semicolon in macro used in expression position +FAIL (1.00s) + files test/coverage/coverage_match.good and test/coverage/coverage_match.out differ; test/coverage/coverage_match.out contains: + warning: branch condition never has value 1 + ┌─ test/coverage/coverage_match.rs:12:9 + │ + 12 │ let x = u8::symbolic("x"); + │ ^ + + warning: branch condition never has value 20 + ┌─ test/coverage/coverage_match.rs:22:9 + │ + 22 │ E::A => 1, + │ ^^^^ + + + Use -p '/coverage_match/' to rerun this test only. + coverage_shortcircuit: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -2852,12 +2268,23 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_match/de962e25::crux_test[0]/report_data.js" and "test/coverage/out/coverage_match/eeac3067::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 -note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (1.96s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_match/eeac3067::crux_test[0]/report_data.js" "test/coverage/out/coverage_match/de962e25::crux_test[0]/report_data.js" "test/coverage/out/coverage_match/e0a88aee::crux_test[0]/report_data.js" "test/coverage/out/coverage_match/63e02e94::crux_test[0]/report_data.js" "test/coverage/out/coverage_match/2b40efd1::crux_test[0]/report_data.js" (exit 101): failed - Use -p '/coverage_match/' to rerun this test only. - coverage_macro: warning: trailing semicolon in macro used in expression position +FAIL (0.96s) + files test/coverage/coverage_shortcircuit.good and test/coverage/coverage_shortcircuit.out differ; test/coverage/coverage_shortcircuit.out contains: + warning: branch condition never has value false + ┌─ test/coverage/coverage_shortcircuit.rs:9:8 + │ + 9 │ if x == 0 || x == 1 { + │ ^^^^^^^^^^^^^^^^ + + warning: branch condition never has a value other than [0] + ┌─ test/coverage/coverage_shortcircuit.rs:9:8 + │ + 9 │ if x == 0 || x == 1 { + │ ^ + + + Use -p '/coverage_shortcircuit/' to rerun this test only. + coverage_loop: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -2871,10 +2298,15 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -thread 'main' panicked at 'translation hashes for "test/coverage/out/coverage_macro/ff936904::crux_test[0]/report_data.js" and "test/coverage/out/coverage_macro/e538cb6e::crux_test[0]/report_data.js" do not match', src/main.rs:825:13 -note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace -FAIL (2.01s) - readCreateProcess: cargo "run" "--manifest-path" "report-coverage/Cargo.toml" "--quiet" "--" "--no-color" "test/coverage/out/coverage_macro/e538cb6e::crux_test[0]/report_data.js" "test/coverage/out/coverage_macro/ff936904::crux_test[0]/report_data.js" "test/coverage/out/coverage_macro/115b4d60::crux_test[0]/report_data.js" "test/coverage/out/coverage_macro/b70cc90f::crux_test[0]/report_data.js" "test/coverage/out/coverage_macro/54bd500f::crux_test[0]/report_data.js" (exit 101): failed - Use -p '/coverage_macro/' to rerun this test only. +FAIL (0.95s) + files test/coverage/coverage_loop.good and test/coverage/coverage_loop.out differ; test/coverage/coverage_loop.out contains: + warning: branch condition never has value true + ┌─ test/coverage/coverage_loop.rs:12:19 + │ + 12 │ } else if i > 99 { + │ ^^^^^^ + + + Use -p '/coverage_loop/' to rerun this test only. -135 out of 339 tests failed (682.81s) +74 out of 339 tests failed (459.89s) From d7a4776fec99d61f7be43e6d1ba4edb3edfbe079 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Tue, 23 May 2023 19:22:51 -0400 Subject: [PATCH 056/114] crux-mir: Override ctlz properly --- crucible-mir/src/Mir/TransCustom.hs | 4 ++-- crux-mir/test/symb_eval/bitvector/leading_zeros.good | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crucible-mir/src/Mir/TransCustom.hs b/crucible-mir/src/Mir/TransCustom.hs index 78e4d8fa7..3e4183433 100644 --- a/crucible-mir/src/Mir/TransCustom.hs +++ b/crucible-mir/src/Mir/TransCustom.hs @@ -873,12 +873,12 @@ ctlz_impl name optFixedWidth _substs = Just $ CustomOp $ \_optys ops -> case ops ctlz :: (ExplodedDefId, CustomRHS) ctlz = - ( ["core","intrinsics", "", "ctlz"] + ( ["core","intrinsics", "{extern}", "ctlz"] , ctlz_impl "ctlz" Nothing ) ctlz_nonzero :: (ExplodedDefId, CustomRHS) ctlz_nonzero = - ( ["core","intrinsics", "", "ctlz_nonzero"] + ( ["core","intrinsics", "{extern}", "ctlz_nonzero"] , ctlz_impl "ctlz_nonzero" Nothing ) rotate_left :: (ExplodedDefId, CustomRHS) diff --git a/crux-mir/test/symb_eval/bitvector/leading_zeros.good b/crux-mir/test/symb_eval/bitvector/leading_zeros.good index 8cfe59ba4..2ed62e38a 100644 --- a/crux-mir/test/symb_eval/bitvector/leading_zeros.good +++ b/crux-mir/test/symb_eval/bitvector/leading_zeros.good @@ -1,3 +1,3 @@ -test leading_zeros/3a1fbbbh::crux_test[0]: ok +test leading_zeros/85b8f496::crux_test[0]: ok [Crux] Overall status: Valid. From 0be2b4a724eeea52ccc5f2c40a92bd1d57dda6c4 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Tue, 23 May 2023 20:31:45 -0400 Subject: [PATCH 057/114] build.sh: Use --remap-path-prefix This ensures that full paths do not leak into MIR output. --- crux-mir/build.sh | 54 ++++++++++--------- .../test/symb_eval/alloc/out_of_bounds.good | 8 +-- .../test/symb_eval/alloc/uninit_read.good | 8 +-- crux-mir/test/symb_eval/alloc/valid_read.good | 2 +- .../test/symb_eval/alloc/zero_length.good | 2 +- crux-mir/test/symb_eval/any/conditional.good | 2 +- crux-mir/test/symb_eval/any/downcast.good | 2 +- crux-mir/test/symb_eval/array/basic.good | 2 +- crux-mir/test/symb_eval/array/mux_slice.good | 2 +- crux-mir/test/symb_eval/array/slice.good | 2 +- crux-mir/test/symb_eval/array/slice_mut.good | 2 +- crux-mir/test/symb_eval/bitvector/arith.good | 2 +- crux-mir/test/symb_eval/bitvector/cmp.good | 2 +- .../test/symb_eval/bitvector/from_to.good | 2 +- .../symb_eval/bitvector/leading_zeros.good | 2 +- .../test/symb_eval/bitvector/literals.good | 2 +- .../symb_eval/bitvector/overflowing_sub.good | 2 +- .../test/symb_eval/bitvector/symbolic.good | 2 +- .../test/symb_eval/bytes/extend_bytes.good | 2 +- crux-mir/test/symb_eval/bytes/new.good | 2 +- .../test/symb_eval/bytes/put_overflow.good | 8 +-- crux-mir/test/symb_eval/concretize/array.good | 2 +- .../test/symb_eval/concretize/assert_ok.good | 2 +- crux-mir/test/symb_eval/concretize/conc.good | 2 +- .../test/symb_eval/concretize/no_conc.good | 2 +- crux-mir/test/symb_eval/crux/early_fail.good | 14 ++--- crux-mir/test/symb_eval/crux/fail_return.good | 20 +++---- crux-mir/test/symb_eval/crux/mixed_fail.good | 24 ++++----- crux-mir/test/symb_eval/crux/multi.good | 24 ++++----- crux-mir/test/symb_eval/crypto/bytes.good | 14 ++--- crux-mir/test/symb_eval/crypto/bytes2.good | 2 +- crux-mir/test/symb_eval/crypto/double.good | 2 +- crux-mir/test/symb_eval/crypto/ffs.good | 2 +- crux-mir/test/symb_eval/enum/mux.good | 2 +- crux-mir/test/symb_eval/fnptr/mux.good | 2 +- crux-mir/test/symb_eval/io/vec_write.good | 2 +- crux-mir/test/symb_eval/mux/array.good | 2 +- crux-mir/test/symb_eval/mux/array_mut.good | 2 +- crux-mir/test/symb_eval/num/checked_add.good | 6 +-- crux-mir/test/symb_eval/num/checked_div.good | 10 ++-- crux-mir/test/symb_eval/num/checked_mul.good | 6 +-- .../symb_eval/num/checked_mul_signed.good | 6 +-- .../test/symb_eval/overrides/bad_symb1.good | 6 +-- .../test/symb_eval/overrides/bad_symb2.good | 6 +-- .../test/symb_eval/overrides/override1.good | 2 +- .../test/symb_eval/overrides/override2.good | 6 +-- .../test/symb_eval/overrides/override3.good | 2 +- .../test/symb_eval/overrides/override4.good | 2 +- .../test/symb_eval/overrides/override5.good | 6 +-- .../symb_eval/overrides/override_rust.good | 2 +- .../test/symb_eval/refs/mux_init_imm.good | 2 +- .../test/symb_eval/refs/mux_init_mut.good | 2 +- crux-mir/test/symb_eval/scalar/test1.good | 2 +- .../test/symb_eval/sym_bytes/construct.good | 6 +-- .../test/symb_eval/sym_bytes/deserialize.good | 6 +-- .../test/symb_eval/vector/as_mut_slice.good | 2 +- crux-mir/test/symb_eval/vector/as_slice.good | 2 +- crux-mir/test/symb_eval/vector/concat.good | 2 +- .../symb_eval/vector/copy_from_slice.good | 2 +- crux-mir/test/symb_eval/vector/mut.good | 2 +- crux-mir/test/symb_eval/vector/new.good | 2 +- crux-mir/test/symb_eval/vector/push.good | 2 +- crux-mir/test/symb_eval/vector/replicate.good | 2 +- crux-mir/test/symb_eval/vector/split_at.good | 2 +- 64 files changed, 166 insertions(+), 162 deletions(-) diff --git a/crux-mir/build.sh b/crux-mir/build.sh index 3b30a684a..6c7093f4c 100755 --- a/crux-mir/build.sh +++ b/crux-mir/build.sh @@ -2,80 +2,84 @@ set -e export STD_ENV_ARCH=$(uname -m) +translate() { + mir-json -L rlibs --out-dir rlibs --crate-type rlib --remap-path-prefix "$(pwd -P)=." "$@" +} + echo 'Building core...' -mir-json lib/core/src/lib.rs --edition=2021 --crate-name core -L rlibs --out-dir rlibs --crate-type rlib +translate lib/core/src/lib.rs --edition=2021 --crate-name core echo 'Building rustc_std_workspace_core...' -mir-json lib/rustc_std_workspace_core/lib.rs --edition=2021 --crate-name rustc_std_workspace_core -L rlibs --out-dir rlibs --crate-type rlib --extern core=rlibs/libcore.rlib +translate lib/rustc_std_workspace_core/lib.rs --edition=2021 --crate-name rustc_std_workspace_core --extern core=rlibs/libcore.rlib echo 'Building libc...' -mir-json lib/libc/src/lib.rs --crate-name libc -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="align"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="rustc-std-workspace-core"' --cfg freebsd11 --cfg libc_priv_mod_use --cfg libc_union --cfg libc_const_size_of --cfg libc_align --cfg libc_int128 --cfg libc_core_cvoid --cfg libc_packedN --cfg libc_cfg_target_vendor --cfg libc_non_exhaustive --cfg libc_ptr_addr_of --cfg libc_underscore_const_names --cfg libc_thread_local --cfg 'libc_const_extern_fn`' --extern rustc_std_workspace_core=rlibs/librustc_std_workspace_core.rlib +translate lib/libc/src/lib.rs --crate-name libc --cfg 'feature="align"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="rustc-std-workspace-core"' --cfg freebsd11 --cfg libc_priv_mod_use --cfg libc_union --cfg libc_const_size_of --cfg libc_align --cfg libc_int128 --cfg libc_core_cvoid --cfg libc_packedN --cfg libc_cfg_target_vendor --cfg libc_non_exhaustive --cfg libc_ptr_addr_of --cfg libc_underscore_const_names --cfg libc_thread_local --cfg 'libc_const_extern_fn`' --extern rustc_std_workspace_core=rlibs/librustc_std_workspace_core.rlib echo 'Building compiler_builtins...' -mir-json lib/compiler_builtins/src/lib.rs --crate-name compiler_builtins -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="compiler-builtins"' --cfg 'feature="core"' --cfg 'feature="default"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="unstable"' --cfg 'feature="mem-unaligned"`' --extern core=rlibs/libcore.rlib +translate lib/compiler_builtins/src/lib.rs --crate-name compiler_builtins --cfg 'feature="compiler-builtins"' --cfg 'feature="core"' --cfg 'feature="default"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="unstable"' --cfg 'feature="mem-unaligned"`' --extern core=rlibs/libcore.rlib # extra lib (added manually) echo "Building crucible..." -mir-json lib/crucible/lib.rs --crate-name crucible --edition=2021 -L rlibs --out-dir rlibs --crate-type rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib +translate lib/crucible/lib.rs --crate-name crucible --edition=2021 --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib echo 'Building alloc...' -mir-json lib/alloc/src/lib.rs --edition=2021 --crate-name alloc -L rlibs --out-dir rlibs --crate-type rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern crucible=rlibs/libcrucible.rlib +translate lib/alloc/src/lib.rs --edition=2021 --crate-name alloc --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern crucible=rlibs/libcrucible.rlib echo 'Building cfg_if...' -mir-json lib/cfg_if/src/lib.rs --edition=2018 --crate-name cfg_if -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib +translate lib/cfg_if/src/lib.rs --edition=2018 --crate-name cfg_if --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib echo 'Building memchr...' -mir-json lib/memchr/src/lib.rs --edition=2018 --crate-name memchr -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --cfg memchr_runtime_simd --cfg memchr_runtime_sse2 --cfg memchr_runtime_sse42 --cfg 'memchr_runtime_avx`' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib +translate lib/memchr/src/lib.rs --edition=2018 --crate-name memchr --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --cfg memchr_runtime_simd --cfg memchr_runtime_sse2 --cfg memchr_runtime_sse42 --cfg 'memchr_runtime_avx`' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib echo 'Building adler...' -mir-json lib/adler/src/lib.rs --crate-name adler -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib +translate lib/adler/src/lib.rs --crate-name adler --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib echo 'Building rustc_demangle...' -mir-json lib/rustc_demangle/src/lib.rs --crate-name rustc_demangle -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib +translate lib/rustc_demangle/src/lib.rs --crate-name rustc_demangle --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib echo 'Building unwind...' -mir-json lib/unwind/src/lib.rs --edition=2021 --crate-name unwind -L rlibs --out-dir rlibs --crate-type rlib --extern cfg_if=rlibs/libcfg_if.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern libc=rlibs/liblibc.rlib +translate lib/unwind/src/lib.rs --edition=2021 --crate-name unwind --extern cfg_if=rlibs/libcfg_if.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern libc=rlibs/liblibc.rlib echo 'Building panic_unwind...' -mir-json lib/panic_unwind/src/lib.rs --edition=2021 --crate-name panic_unwind -L rlibs --out-dir rlibs --crate-type rlib --extern alloc=rlibs/liballoc.rlib --extern cfg_if=rlibs/libcfg_if.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern libc=rlibs/liblibc.rlib --extern unwind=rlibs/libunwind.rlib +translate lib/panic_unwind/src/lib.rs --edition=2021 --crate-name panic_unwind --extern alloc=rlibs/liballoc.rlib --extern cfg_if=rlibs/libcfg_if.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern libc=rlibs/liblibc.rlib --extern unwind=rlibs/libunwind.rlib echo 'Building rustc_std_workspace_alloc...' -mir-json lib/rustc_std_workspace_alloc/lib.rs --edition=2021 --crate-name rustc_std_workspace_alloc -L rlibs --out-dir rlibs --crate-type rlib --extern alloc=rlibs/liballoc.rlib +translate lib/rustc_std_workspace_alloc/lib.rs --edition=2021 --crate-name rustc_std_workspace_alloc --extern alloc=rlibs/liballoc.rlib echo 'Building panic_abort...' -mir-json lib/panic_abort/src/lib.rs --edition=2021 --crate-name panic_abort -L rlibs --out-dir rlibs --crate-type rlib --extern alloc=rlibs/liballoc.rlib --extern cfg_if=rlibs/libcfg_if.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern libc=rlibs/liblibc.rlib +translate lib/panic_abort/src/lib.rs --edition=2021 --crate-name panic_abort --extern alloc=rlibs/liballoc.rlib --extern cfg_if=rlibs/libcfg_if.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern libc=rlibs/liblibc.rlib echo 'Building gimli...' -mir-json lib/gimli/src/lib.rs --edition=2018 --crate-name gimli -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="read"' --cfg 'feature="read-core"' --cfg 'feature="rustc-dep-of-std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib +translate lib/gimli/src/lib.rs --edition=2018 --crate-name gimli --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="read"' --cfg 'feature="read-core"' --cfg 'feature="rustc-dep-of-std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib echo 'Building std_detect...' -mir-json lib/std_detect/src/lib.rs --edition=2018 --crate-name std_detect -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="libc"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="std_detect_dlsym_getauxval"' --cfg 'feature="std_detect_file_io"' --extern cfg_if=rlibs/libcfg_if.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern libc=rlibs/liblibc.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib +translate lib/std_detect/src/lib.rs --edition=2018 --crate-name std_detect --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="libc"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="std_detect_dlsym_getauxval"' --cfg 'feature="std_detect_file_io"' --extern cfg_if=rlibs/libcfg_if.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern libc=rlibs/liblibc.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib echo 'Building object...' -mir-json lib/object/src/lib.rs --edition=2018 --crate-name object -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="alloc"' --cfg 'feature="archive"' --cfg 'feature="coff"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="elf"' --cfg 'feature="macho"' --cfg 'feature="pe"' --cfg 'feature="read_core"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="unaligned"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern memchr=rlibs/libmemchr.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib +translate lib/object/src/lib.rs --edition=2018 --crate-name object --cfg 'feature="alloc"' --cfg 'feature="archive"' --cfg 'feature="coff"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="elf"' --cfg 'feature="macho"' --cfg 'feature="pe"' --cfg 'feature="read_core"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="unaligned"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern memchr=rlibs/libmemchr.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib echo 'Building miniz_oxide...' -mir-json lib/miniz_oxide/src/lib.rs --edition=2018 --crate-name miniz_oxide -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern adler=rlibs/libadler.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib +translate lib/miniz_oxide/src/lib.rs --edition=2018 --crate-name miniz_oxide --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern adler=rlibs/libadler.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib echo 'Building hashbrown...' -mir-json lib/hashbrown/src/lib.rs --edition=2021 --crate-name hashbrown -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="nightly"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="rustc-internal-api"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib +translate lib/hashbrown/src/lib.rs --edition=2021 --crate-name hashbrown --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="nightly"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="rustc-internal-api"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib echo 'Building addr2line...' -mir-json lib/addr2line/src/lib.rs --crate-name addr2line -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern gimli=rlibs/libgimli.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib +translate lib/addr2line/src/lib.rs --crate-name addr2line --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern gimli=rlibs/libgimli.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib echo 'Building std...' -mir-json lib/std/src/lib.rs --edition=2021 --crate-name std -L rlibs --out-dir rlibs --crate-type rlib --cfg 'feature="addr2line"' --cfg 'feature="backtrace"' --cfg 'feature="gimli-symbolize"' --cfg 'feature="miniz_oxide"' --cfg 'feature="object"' --cfg 'feature="panic_unwind"' --cfg 'feature="std_detect_dlsym_getauxval"' --cfg 'feature="std_detect_file_io"' --cfg 'backtrace_in_libstd`' --extern addr2line=rlibs/libaddr2line.rlib --extern alloc=rlibs/liballoc.rlib --extern cfg_if=rlibs/libcfg_if.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern hashbrown=rlibs/libhashbrown.rlib --extern libc=rlibs/liblibc.rlib --extern miniz_oxide=rlibs/libminiz_oxide.rlib --extern object=rlibs/libobject.rlib --extern panic_abort=rlibs/libpanic_abort.rlib --extern panic_unwind=rlibs/libpanic_unwind.rlib --extern rustc_demangle=rlibs/librustc_demangle.rlib --extern std_detect=rlibs/libstd_detect.rlib --extern unwind=rlibs/libunwind.rlib +translate lib/std/src/lib.rs --edition=2021 --crate-name std --cfg 'feature="addr2line"' --cfg 'feature="backtrace"' --cfg 'feature="gimli-symbolize"' --cfg 'feature="miniz_oxide"' --cfg 'feature="object"' --cfg 'feature="panic_unwind"' --cfg 'feature="std_detect_dlsym_getauxval"' --cfg 'feature="std_detect_file_io"' --cfg 'backtrace_in_libstd`' --extern addr2line=rlibs/libaddr2line.rlib --extern alloc=rlibs/liballoc.rlib --extern cfg_if=rlibs/libcfg_if.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern hashbrown=rlibs/libhashbrown.rlib --extern libc=rlibs/liblibc.rlib --extern miniz_oxide=rlibs/libminiz_oxide.rlib --extern object=rlibs/libobject.rlib --extern panic_abort=rlibs/libpanic_abort.rlib --extern panic_unwind=rlibs/libpanic_unwind.rlib --extern rustc_demangle=rlibs/librustc_demangle.rlib --extern std_detect=rlibs/libstd_detect.rlib --extern unwind=rlibs/libunwind.rlib echo 'Building proc_macro...' -mir-json lib/proc_macro/src/lib.rs --edition=2021 --crate-name proc_macro -L rlibs --out-dir rlibs --crate-type rlib --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib +translate lib/proc_macro/src/lib.rs --edition=2021 --crate-name proc_macro --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib # extra libs (added manually) echo 'Building int512...' -mir-json lib/int512.rs --crate-name int512 -L rlibs --out-dir rlibs --crate-type rlib --extern core=rlibs/libcore.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib +translate lib/int512.rs --crate-name int512 --extern core=rlibs/libcore.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib echo 'Building bytes...' -mir-json lib/bytes.rs --edition=2021 --crate-name bytes -L rlibs --out-dir rlibs --crate-type rlib --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern crucible=rlibs/libcrucible.rlib +translate lib/bytes.rs --edition=2021 --crate-name bytes --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern crucible=rlibs/libcrucible.rlib echo 'Building byteorder...' -mir-json lib/byteorder/lib.rs --edition=2021 --crate-name byteorder -L rlibs --out-dir rlibs --crate-type rlib --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib +translate lib/byteorder/lib.rs --edition=2021 --crate-name byteorder --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib diff --git a/crux-mir/test/symb_eval/alloc/out_of_bounds.good b/crux-mir/test/symb_eval/alloc/out_of_bounds.good index ef03f0762..188f578fc 100644 --- a/crux-mir/test/symb_eval/alloc/out_of_bounds.good +++ b/crux-mir/test/symb_eval/alloc/out_of_bounds.good @@ -1,13 +1,13 @@ -test out_of_bounds/b521303c::crux_test[0]: FAILED +test out_of_bounds/ca0644b7::crux_test[0]: FAILED failures: ----- out_of_bounds/b521303c::crux_test[0] counterexamples ---- +---- out_of_bounds/ca0644b7::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/b521303c::crux_test[0] +[Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/ca0644b7::crux_test[0] [Crux] vector index out of range: the length is 10 but the index is BV 12 [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/b521303c::crux_test[0] +[Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/ca0644b7::crux_test[0] [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/alloc/uninit_read.good b/crux-mir/test/symb_eval/alloc/uninit_read.good index 4ce8bdfc4..4dbddb17d 100644 --- a/crux-mir/test/symb_eval/alloc/uninit_read.good +++ b/crux-mir/test/symb_eval/alloc/uninit_read.good @@ -1,13 +1,13 @@ -test uninit_read/e86da728::crux_test[0]: FAILED +test uninit_read/851ea4ad::crux_test[0]: FAILED failures: ----- uninit_read/e86da728::crux_test[0] counterexamples ---- +---- uninit_read/851ea4ad::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/e86da728::crux_test[0] +[Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/851ea4ad::crux_test[0] [Crux] Attempted to read uninitialized vector index [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/e86da728::crux_test[0] +[Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/851ea4ad::crux_test[0] [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/alloc/valid_read.good b/crux-mir/test/symb_eval/alloc/valid_read.good index f9a9f3619..8955502eb 100644 --- a/crux-mir/test/symb_eval/alloc/valid_read.good +++ b/crux-mir/test/symb_eval/alloc/valid_read.good @@ -1,3 +1,3 @@ -test valid_read/f9c1ade1::crux_test[0]: returned 45, ok +test valid_read/c82bffb5::crux_test[0]: returned 45, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/alloc/zero_length.good b/crux-mir/test/symb_eval/alloc/zero_length.good index 924d71647..d3abeb0a7 100644 --- a/crux-mir/test/symb_eval/alloc/zero_length.good +++ b/crux-mir/test/symb_eval/alloc/zero_length.good @@ -1,3 +1,3 @@ -test zero_length/94b92a2d::crux_test[0]: ok +test zero_length/d9fb4927::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/any/conditional.good b/crux-mir/test/symb_eval/any/conditional.good index 3bc0f4fb0..2dd35b976 100644 --- a/crux-mir/test/symb_eval/any/conditional.good +++ b/crux-mir/test/symb_eval/any/conditional.good @@ -1,3 +1,3 @@ -test conditional/84afefc9::crux_test[0]: ok +test conditional/e22ae149::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/any/downcast.good b/crux-mir/test/symb_eval/any/downcast.good index 3591a46a0..6f08340fc 100644 --- a/crux-mir/test/symb_eval/any/downcast.good +++ b/crux-mir/test/symb_eval/any/downcast.good @@ -1,3 +1,3 @@ -test downcast/870d31b0::crux_test[0]: returned 1, ok +test downcast/c8ce9e8e::crux_test[0]: returned 1, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/array/basic.good b/crux-mir/test/symb_eval/array/basic.good index f1439da79..ec102490b 100644 --- a/crux-mir/test/symb_eval/array/basic.good +++ b/crux-mir/test/symb_eval/array/basic.good @@ -1,3 +1,3 @@ -test basic/bfb477c2::crux_test[0]: returned 3, ok +test basic/a10c5e44::crux_test[0]: returned 3, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/array/mux_slice.good b/crux-mir/test/symb_eval/array/mux_slice.good index 03022fdb1..f59baf7b3 100644 --- a/crux-mir/test/symb_eval/array/mux_slice.good +++ b/crux-mir/test/symb_eval/array/mux_slice.good @@ -1,3 +1,3 @@ -test mux_slice/fe5e891b::crux_test[0]: returned Symbolic BV, ok +test mux_slice/df68b584::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/array/slice.good b/crux-mir/test/symb_eval/array/slice.good index 7d0e5b30c..2d240a3a6 100644 --- a/crux-mir/test/symb_eval/array/slice.good +++ b/crux-mir/test/symb_eval/array/slice.good @@ -1,3 +1,3 @@ -test slice/ea2efd47::crux_test[0]: returned 5, ok +test slice/ab67a8fd::crux_test[0]: returned 5, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/array/slice_mut.good b/crux-mir/test/symb_eval/array/slice_mut.good index 03b8d75c2..06ec78240 100644 --- a/crux-mir/test/symb_eval/array/slice_mut.good +++ b/crux-mir/test/symb_eval/array/slice_mut.good @@ -1,3 +1,3 @@ -test slice_mut/1361be29::crux_test[0]: returned 5, ok +test slice_mut/cf0e1c49::crux_test[0]: returned 5, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/arith.good b/crux-mir/test/symb_eval/bitvector/arith.good index 1586f7ebe..9d27f4d34 100644 --- a/crux-mir/test/symb_eval/bitvector/arith.good +++ b/crux-mir/test/symb_eval/bitvector/arith.good @@ -1,3 +1,3 @@ -test arith/382b0162::crux_test[0]: ok +test arith/909ad774::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/cmp.good b/crux-mir/test/symb_eval/bitvector/cmp.good index 971f62ad6..e82ae15ed 100644 --- a/crux-mir/test/symb_eval/bitvector/cmp.good +++ b/crux-mir/test/symb_eval/bitvector/cmp.good @@ -1,3 +1,3 @@ -test cmp/a77f9ebc::crux_test[0]: ok +test cmp/79f545fc::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/from_to.good b/crux-mir/test/symb_eval/bitvector/from_to.good index 458a40add..d54bb2266 100644 --- a/crux-mir/test/symb_eval/bitvector/from_to.good +++ b/crux-mir/test/symb_eval/bitvector/from_to.good @@ -1,3 +1,3 @@ -test from_to/cecfc226::crux_test[0]: ok +test from_to/81d915d3::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/leading_zeros.good b/crux-mir/test/symb_eval/bitvector/leading_zeros.good index 2ed62e38a..bce2fc5b1 100644 --- a/crux-mir/test/symb_eval/bitvector/leading_zeros.good +++ b/crux-mir/test/symb_eval/bitvector/leading_zeros.good @@ -1,3 +1,3 @@ -test leading_zeros/85b8f496::crux_test[0]: ok +test leading_zeros/a181beeb::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/literals.good b/crux-mir/test/symb_eval/bitvector/literals.good index 966cd907b..118642e14 100644 --- a/crux-mir/test/symb_eval/bitvector/literals.good +++ b/crux-mir/test/symb_eval/bitvector/literals.good @@ -1,3 +1,3 @@ -test literals/06825ef9::crux_test[0]: ok +test literals/9d6d8593::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/overflowing_sub.good b/crux-mir/test/symb_eval/bitvector/overflowing_sub.good index 0e029bc46..a658401c3 100644 --- a/crux-mir/test/symb_eval/bitvector/overflowing_sub.good +++ b/crux-mir/test/symb_eval/bitvector/overflowing_sub.good @@ -1,3 +1,3 @@ -test overflowing_sub/30a11e11::crux_test[0]: ok +test overflowing_sub/d297e8c6::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/symbolic.good b/crux-mir/test/symb_eval/bitvector/symbolic.good index ea6f17081..295ff7e2a 100644 --- a/crux-mir/test/symb_eval/bitvector/symbolic.good +++ b/crux-mir/test/symb_eval/bitvector/symbolic.good @@ -1,3 +1,3 @@ -test symbolic/d4690595::crux_test[0]: ok +test symbolic/8802d91d::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/extend_bytes.good b/crux-mir/test/symb_eval/bytes/extend_bytes.good index 4b4c681a8..9efeeecb9 100644 --- a/crux-mir/test/symb_eval/bytes/extend_bytes.good +++ b/crux-mir/test/symb_eval/bytes/extend_bytes.good @@ -1,3 +1,3 @@ -test extend_bytes/fe7bcbf0::f[0]: ok +test extend_bytes/724ff396::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/new.good b/crux-mir/test/symb_eval/bytes/new.good index 1e78a772e..6eaf95355 100644 --- a/crux-mir/test/symb_eval/bytes/new.good +++ b/crux-mir/test/symb_eval/bytes/new.good @@ -1,3 +1,3 @@ -test new/dec1e542::f[0]: ok +test new/7e2f4495::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/put_overflow.good b/crux-mir/test/symb_eval/bytes/put_overflow.good index 93b3a0633..453a375dd 100644 --- a/crux-mir/test/symb_eval/bytes/put_overflow.good +++ b/crux-mir/test/symb_eval/bytes/put_overflow.good @@ -1,10 +1,10 @@ -test put_overflow/3a1fbbbh::f[0]: FAILED +test put_overflow/57f18178::f[0]: FAILED failures: ----- put_overflow/3a1fbbbh::f[0] counterexamples ---- +---- put_overflow/57f18178::f[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] internal: error: in bytes/3a1fbbbh::{{impl}}[4]::put_slice[0] -[Crux] panicking::begin_panic, called from bytes/3a1fbbbh::{{impl}}[4]::put_slice[0] +[Crux] internal: error: in bytes/c14f6c84::{impl#4}[0]::put_slice[0] +[Crux] panicking::panic_fmt, called from bytes/c14f6c84::{impl#4}[0]::put_slice[0] [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/concretize/array.good b/crux-mir/test/symb_eval/concretize/array.good index 399cfc9fe..bcc382540 100644 --- a/crux-mir/test/symb_eval/concretize/array.good +++ b/crux-mir/test/symb_eval/concretize/array.good @@ -1,3 +1,3 @@ -test array/aa48835b::crux_test[0]: returned (1, 2, 3), ok +test array/191383c3::crux_test[0]: returned (1, 2, 3), ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/concretize/assert_ok.good b/crux-mir/test/symb_eval/concretize/assert_ok.good index 50dbc0720..d34d68332 100644 --- a/crux-mir/test/symb_eval/concretize/assert_ok.good +++ b/crux-mir/test/symb_eval/concretize/assert_ok.good @@ -1,3 +1,3 @@ -test assert_ok/e60ec1ab::crux_test[0]: returned 1, ok +test assert_ok/e20e387d::crux_test[0]: returned 1, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/concretize/conc.good b/crux-mir/test/symb_eval/concretize/conc.good index 47946de64..e3f3d2ad5 100644 --- a/crux-mir/test/symb_eval/concretize/conc.good +++ b/crux-mir/test/symb_eval/concretize/conc.good @@ -1,3 +1,3 @@ -test conc/58237648::crux_test[0]: returned (1, 0), ok +test conc/86e91a75::crux_test[0]: returned (1, 0), ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/concretize/no_conc.good b/crux-mir/test/symb_eval/concretize/no_conc.good index 7a2d78170..fdf6884dd 100644 --- a/crux-mir/test/symb_eval/concretize/no_conc.good +++ b/crux-mir/test/symb_eval/concretize/no_conc.good @@ -1,3 +1,3 @@ -test no_conc/46b8f4c7::crux_test[0]: returned (Symbolic BV, Symbolic BV), ok +test no_conc/27ee0b55::crux_test[0]: returned (Symbolic BV, Symbolic BV), ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/crux/early_fail.good b/crux-mir/test/symb_eval/crux/early_fail.good index 3c1c1cde8..036a6003d 100644 --- a/crux-mir/test/symb_eval/crux/early_fail.good +++ b/crux-mir/test/symb_eval/crux/early_fail.good @@ -1,16 +1,16 @@ -test early_fail/3a1fbbbh::fail1[0]: FAILED -test early_fail/3a1fbbbh::fail2[0]: FAILED +test early_fail/233440b0::fail1[0]: FAILED +test early_fail/233440b0::fail2[0]: FAILED failures: ----- early_fail/3a1fbbbh::fail1[0] counterexamples ---- +---- early_fail/233440b0::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] internal: error: in early_fail/3a1fbbbh::fail1[0] -[Crux] panicking::begin_panic, called from early_fail/3a1fbbbh::fail1[0] +[Crux] internal: error: in early_fail/233440b0::fail1[0] +[Crux] panicking::panic_fmt, called from early_fail/233440b0::fail1[0] ----- early_fail/3a1fbbbh::fail2[0] counterexamples ---- +---- early_fail/233440b0::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:37:41: 37:58 !test/symb_eval/crux/early_fail.rs:17:5: 17:30: error: in early_fail/3a1fbbbh::fail2[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/early_fail.rs:17:5: 17:29: error: in early_fail/233440b0::fail2[0] [Crux] MIR assertion at test/symb_eval/crux/early_fail.rs:17:5: [Crux] x == 0 diff --git a/crux-mir/test/symb_eval/crux/fail_return.good b/crux-mir/test/symb_eval/crux/fail_return.good index 9774a9644..a4cb6b748 100644 --- a/crux-mir/test/symb_eval/crux/fail_return.good +++ b/crux-mir/test/symb_eval/crux/fail_return.good @@ -1,23 +1,23 @@ -test fail_return/3a1fbbbh::fail1[0]: returned Symbolic BV, FAILED -test fail_return/3a1fbbbh::fail2[0]: returned 123, FAILED +test fail_return/10c6efbb::fail1[0]: returned Symbolic BV, FAILED +test fail_return/10c6efbb::fail2[0]: returned 123, FAILED failures: ----- fail_return/3a1fbbbh::fail1[0] counterexamples ---- +---- fail_return/10c6efbb::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/crux/fail_return.rs:8:22: 8:27: error: in fail_return/3a1fbbbh::fail1[0] -[Crux] attempt to add with overflow +[Crux] test/symb_eval/crux/fail_return.rs:8:22: 8:27: error: in fail_return/10c6efbb::fail1[0] +[Crux] attempt to compute `move _4 + const 1_u8`, which would overflow [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:37:41: 37:58 !test/symb_eval/crux/fail_return.rs:8:5: 8:33: error: in fail_return/3a1fbbbh::fail1[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/fail_return.rs:8:5: 8:32: error: in fail_return/10c6efbb::fail1[0] [Crux] MIR assertion at test/symb_eval/crux/fail_return.rs:8:5: [Crux] x + 1 > x ----- fail_return/3a1fbbbh::fail2[0] counterexamples ---- +---- fail_return/10c6efbb::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/crux/fail_return.rs:15:22: 15:27: error: in fail_return/3a1fbbbh::fail2[0] -[Crux] attempt to add with overflow +[Crux] test/symb_eval/crux/fail_return.rs:15:22: 15:27: error: in fail_return/10c6efbb::fail2[0] +[Crux] attempt to compute `move _5 + const 1_u8`, which would overflow [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:37:41: 37:58 !test/symb_eval/crux/fail_return.rs:15:5: 15:33: error: in fail_return/3a1fbbbh::fail2[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/fail_return.rs:15:5: 15:32: error: in fail_return/10c6efbb::fail2[0] [Crux] MIR assertion at test/symb_eval/crux/fail_return.rs:15:5: [Crux] x + 1 > x diff --git a/crux-mir/test/symb_eval/crux/mixed_fail.good b/crux-mir/test/symb_eval/crux/mixed_fail.good index 72b88d2de..918bdf3ed 100644 --- a/crux-mir/test/symb_eval/crux/mixed_fail.good +++ b/crux-mir/test/symb_eval/crux/mixed_fail.good @@ -1,25 +1,25 @@ -test mixed_fail/3a1fbbbh::fail1[0]: FAILED -test mixed_fail/3a1fbbbh::fail2[0]: FAILED -test mixed_fail/3a1fbbbh::pass1[0]: ok -test mixed_fail/3a1fbbbh::pass2[0]: ok +test mixed_fail/0b094ced::fail1[0]: FAILED +test mixed_fail/0b094ced::fail2[0]: FAILED +test mixed_fail/0b094ced::pass1[0]: ok +test mixed_fail/0b094ced::pass2[0]: ok failures: ----- mixed_fail/3a1fbbbh::fail1[0] counterexamples ---- +---- mixed_fail/0b094ced::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/crux/mixed_fail.rs:8:22: 8:27: error: in mixed_fail/3a1fbbbh::fail1[0] -[Crux] attempt to add with overflow +[Crux] test/symb_eval/crux/mixed_fail.rs:8:22: 8:27: error: in mixed_fail/0b094ced::fail1[0] +[Crux] attempt to compute `move _5 + const 1_u8`, which would overflow [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:37:41: 37:58 !test/symb_eval/crux/mixed_fail.rs:8:5: 8:33: error: in mixed_fail/3a1fbbbh::fail1[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/mixed_fail.rs:8:5: 8:32: error: in mixed_fail/0b094ced::fail1[0] [Crux] MIR assertion at test/symb_eval/crux/mixed_fail.rs:8:5: [Crux] x + 1 > x ----- mixed_fail/3a1fbbbh::fail2[0] counterexamples ---- +---- mixed_fail/0b094ced::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/crux/mixed_fail.rs:14:22: 14:27: error: in mixed_fail/3a1fbbbh::fail2[0] -[Crux] attempt to add with overflow +[Crux] test/symb_eval/crux/mixed_fail.rs:14:22: 14:27: error: in mixed_fail/0b094ced::fail2[0] +[Crux] attempt to compute `move _5 + const 2_u8`, which would overflow [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:37:41: 37:58 !test/symb_eval/crux/mixed_fail.rs:14:5: 14:33: error: in mixed_fail/3a1fbbbh::fail2[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/mixed_fail.rs:14:5: 14:32: error: in mixed_fail/0b094ced::fail2[0] [Crux] MIR assertion at test/symb_eval/crux/mixed_fail.rs:14:5: [Crux] x + 2 > x diff --git a/crux-mir/test/symb_eval/crux/multi.good b/crux-mir/test/symb_eval/crux/multi.good index f137ea092..eb87b1f1a 100644 --- a/crux-mir/test/symb_eval/crux/multi.good +++ b/crux-mir/test/symb_eval/crux/multi.good @@ -1,26 +1,26 @@ -test multi/3a1fbbbh::fail1[0]: FAILED -test multi/3a1fbbbh::fail2[0]: FAILED -test multi/3a1fbbbh::fail3[0]: FAILED +test multi/017eda4c::fail1[0]: FAILED +test multi/017eda4c::fail2[0]: FAILED +test multi/017eda4c::fail3[0]: FAILED failures: ----- multi/3a1fbbbh::fail1[0] counterexamples ---- +---- multi/017eda4c::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/crux/multi.rs:8:22: 8:27: error: in multi/3a1fbbbh::fail1[0] -[Crux] attempt to add with overflow +[Crux] test/symb_eval/crux/multi.rs:8:22: 8:27: error: in multi/017eda4c::fail1[0] +[Crux] attempt to compute `move _5 + const 1_u8`, which would overflow [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:37:41: 37:58 !test/symb_eval/crux/multi.rs:8:5: 8:33: error: in multi/3a1fbbbh::fail1[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/multi.rs:8:5: 8:32: error: in multi/017eda4c::fail1[0] [Crux] MIR assertion at test/symb_eval/crux/multi.rs:8:5: [Crux] x + 1 > x ----- multi/3a1fbbbh::fail2[0] counterexamples ---- +---- multi/017eda4c::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] internal: error: in multi/3a1fbbbh::fail2[0] -[Crux] panicking::begin_panic, called from multi/3a1fbbbh::fail2[0] +[Crux] internal: error: in multi/017eda4c::fail2[0] +[Crux] panicking::panic_fmt, called from multi/017eda4c::fail2[0] ----- multi/3a1fbbbh::fail3[0] counterexamples ---- +---- multi/017eda4c::fail3[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:37:41: 37:58 !test/symb_eval/crux/multi.rs:20:5: 20:30: error: in multi/3a1fbbbh::assert_zero[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/multi.rs:20:5: 20:29: error: in multi/017eda4c::assert_zero[0] [Crux] MIR assertion at test/symb_eval/crux/multi.rs:20:5: [Crux] x == 0 diff --git a/crux-mir/test/symb_eval/crypto/bytes.good b/crux-mir/test/symb_eval/crypto/bytes.good index affed8b85..03b9a1559 100644 --- a/crux-mir/test/symb_eval/crypto/bytes.good +++ b/crux-mir/test/symb_eval/crypto/bytes.good @@ -1,26 +1,26 @@ -test bytes/3a1fbbbh::f[0]: FAILED +test bytes/0910acd4::f[0]: FAILED failures: ----- bytes/3a1fbbbh::f[0] counterexamples ---- +---- bytes/0910acd4::f[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:37:41: 37:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:38: error: in bytes/3a1fbbbh::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/0910acd4::f[0] [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: [Crux] a[i] == b[i] [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:37:41: 37:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:38: error: in bytes/3a1fbbbh::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/0910acd4::f[0] [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: [Crux] a[i] == b[i] [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:37:41: 37:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:38: error: in bytes/3a1fbbbh::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/0910acd4::f[0] [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: [Crux] a[i] == b[i] [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:37:41: 37:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:38: error: in bytes/3a1fbbbh::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/0910acd4::f[0] [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: [Crux] a[i] == b[i] [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:37:41: 37:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:38: error: in bytes/3a1fbbbh::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/0910acd4::f[0] [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: [Crux] a[i] == b[i] diff --git a/crux-mir/test/symb_eval/crypto/bytes2.good b/crux-mir/test/symb_eval/crypto/bytes2.good index ce0d54c6d..567b95ba6 100644 --- a/crux-mir/test/symb_eval/crypto/bytes2.good +++ b/crux-mir/test/symb_eval/crypto/bytes2.good @@ -1,3 +1,3 @@ -test bytes2/c529255b::f[0]: ok +test bytes2/4bb09208::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/crypto/double.good b/crux-mir/test/symb_eval/crypto/double.good index f3febce52..34ac8fbec 100644 --- a/crux-mir/test/symb_eval/crypto/double.good +++ b/crux-mir/test/symb_eval/crypto/double.good @@ -1,3 +1,3 @@ -test double/e6e26f62::f[0]: ok +test double/872a3d11::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/crypto/ffs.good b/crux-mir/test/symb_eval/crypto/ffs.good index b364ebbcc..0c4ead1a8 100644 --- a/crux-mir/test/symb_eval/crypto/ffs.good +++ b/crux-mir/test/symb_eval/crypto/ffs.good @@ -1,3 +1,3 @@ -test ffs/12bb9465::f[0]: ok +test ffs/fd27844b::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/enum/mux.good b/crux-mir/test/symb_eval/enum/mux.good index c15770986..65fce7fa1 100644 --- a/crux-mir/test/symb_eval/enum/mux.good +++ b/crux-mir/test/symb_eval/enum/mux.good @@ -1,3 +1,3 @@ -test mux/e3e33eba::test[0]: ok +test mux/48499f53::test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/fnptr/mux.good b/crux-mir/test/symb_eval/fnptr/mux.good index efcb25700..dc9293257 100644 --- a/crux-mir/test/symb_eval/fnptr/mux.good +++ b/crux-mir/test/symb_eval/fnptr/mux.good @@ -1,3 +1,3 @@ -test mux/97bbb42d::crux_test[0]: returned 2, ok +test mux/08dfe1f8::crux_test[0]: returned 2, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/io/vec_write.good b/crux-mir/test/symb_eval/io/vec_write.good index a3f6a7fa6..7770b625b 100644 --- a/crux-mir/test/symb_eval/io/vec_write.good +++ b/crux-mir/test/symb_eval/io/vec_write.good @@ -1,3 +1,3 @@ -test vec_write/b46f4310::f[0]: ok +test vec_write/ee788fed::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/mux/array.good b/crux-mir/test/symb_eval/mux/array.good index 914c5f91e..b0f2799e2 100644 --- a/crux-mir/test/symb_eval/mux/array.good +++ b/crux-mir/test/symb_eval/mux/array.good @@ -1,3 +1,3 @@ -test array/8d6b099f::crux_test[0]: returned Symbolic BV, ok +test array/283f281e::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/mux/array_mut.good b/crux-mir/test/symb_eval/mux/array_mut.good index 4a2d5e64f..108141347 100644 --- a/crux-mir/test/symb_eval/mux/array_mut.good +++ b/crux-mir/test/symb_eval/mux/array_mut.good @@ -1,3 +1,3 @@ -test array_mut/706b4322::crux_test[0]: returned Symbolic BV, ok +test array_mut/78781ed1::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/num/checked_add.good b/crux-mir/test/symb_eval/num/checked_add.good index ddab6eeff..2a1c040a1 100644 --- a/crux-mir/test/symb_eval/num/checked_add.good +++ b/crux-mir/test/symb_eval/num/checked_add.good @@ -1,10 +1,10 @@ -test checked_add/b10d29cb::crux_test[0]: returned 44, FAILED +test checked_add/00902a32::crux_test[0]: returned 44, FAILED failures: ----- checked_add/b10d29cb::crux_test[0] counterexamples ---- +---- checked_add/00902a32::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_add.rs:6:5: 6:12: error: in checked_add/b10d29cb::crux_test[0] +[Crux] test/symb_eval/num/checked_add.rs:6:5: 6:12: error: in checked_add/00902a32::crux_test[0] [Crux] attempt to compute `const 100_u8 + move _2`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/num/checked_div.good b/crux-mir/test/symb_eval/num/checked_div.good index 625342130..9f8a448fe 100644 --- a/crux-mir/test/symb_eval/num/checked_div.good +++ b/crux-mir/test/symb_eval/num/checked_div.good @@ -1,16 +1,16 @@ -test checked_div/56a4cdf6::crux_test[0]: returned 65, FAILED +test checked_div/1a8a798f::crux_test[0]: returned 65, FAILED failures: ----- checked_div/56a4cdf6::crux_test[0] counterexamples ---- +---- checked_div/1a8a798f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/56a4cdf6::div_signed[0] +[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/1a8a798f::div_signed[0] [Crux] attempt to compute `_3 / _4`, which would overflow [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/56a4cdf6::div_signed[0] +[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/1a8a798f::div_signed[0] [Crux] attempt to divide `_3` by zero [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/56a4cdf6::div_unsigned[0] +[Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/1a8a798f::div_unsigned[0] [Crux] attempt to divide `_3` by zero [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/num/checked_mul.good b/crux-mir/test/symb_eval/num/checked_mul.good index aea18fe6f..8048f0231 100644 --- a/crux-mir/test/symb_eval/num/checked_mul.good +++ b/crux-mir/test/symb_eval/num/checked_mul.good @@ -1,10 +1,10 @@ -test checked_mul/6db35fea::crux_test[0]: returned 44, FAILED +test checked_mul/aca814e3::crux_test[0]: returned 44, FAILED failures: ----- checked_mul/6db35fea::crux_test[0] counterexamples ---- +---- checked_mul/aca814e3::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_mul.rs:6:5: 6:12: error: in checked_mul/6db35fea::crux_test[0] +[Crux] test/symb_eval/num/checked_mul.rs:6:5: 6:12: error: in checked_mul/aca814e3::crux_test[0] [Crux] attempt to compute `const 100_u8 * move _2`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/num/checked_mul_signed.good b/crux-mir/test/symb_eval/num/checked_mul_signed.good index 2222c909c..1ee8e5654 100644 --- a/crux-mir/test/symb_eval/num/checked_mul_signed.good +++ b/crux-mir/test/symb_eval/num/checked_mul_signed.good @@ -1,10 +1,10 @@ -test checked_mul_signed/1cb83c82::crux_test[0]: returned 44, FAILED +test checked_mul_signed/2f330b6c::crux_test[0]: returned 44, FAILED failures: ----- checked_mul_signed/1cb83c82::crux_test[0] counterexamples ---- +---- checked_mul_signed/2f330b6c::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_mul_signed.rs:6:5: 6:13: error: in checked_mul_signed/1cb83c82::crux_test[0] +[Crux] test/symb_eval/num/checked_mul_signed.rs:6:5: 6:13: error: in checked_mul_signed/2f330b6c::crux_test[0] [Crux] attempt to compute `const -100_i8 * move _2`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/overrides/bad_symb1.good b/crux-mir/test/symb_eval/overrides/bad_symb1.good index 79016bc4e..775c4e29a 100644 --- a/crux-mir/test/symb_eval/overrides/bad_symb1.good +++ b/crux-mir/test/symb_eval/overrides/bad_symb1.good @@ -1,10 +1,10 @@ -test bad_symb1/fe491089::crux_test[0]: FAILED +test bad_symb1/92726a7e::crux_test[0]: FAILED failures: ----- bad_symb1/fe491089::crux_test[0] counterexamples ---- +---- bad_symb1/92726a7e::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] lib/crucible/symbolic.rs:24:64: 24:68 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/57b9ba51::symbolic[0]::{impl#1}[0]::symbolic[0] +[Crux] lib/crucible/symbolic.rs:24:64: 24:68 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/6430d6c4::symbolic[0]::{impl#1}[0]::symbolic[0] [Crux] symbolic variable name must be a concrete string [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/overrides/bad_symb2.good b/crux-mir/test/symb_eval/overrides/bad_symb2.good index 643bfb40a..ec5d5a482 100644 --- a/crux-mir/test/symb_eval/overrides/bad_symb2.good +++ b/crux-mir/test/symb_eval/overrides/bad_symb2.good @@ -1,10 +1,10 @@ -test bad_symb2/8202e334::crux_test[0]: FAILED +test bad_symb2/4edd5907::crux_test[0]: FAILED failures: ----- bad_symb2/8202e334::crux_test[0] counterexamples ---- +---- bad_symb2/4edd5907::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] lib/crucible/symbolic.rs:24:64: 24:68 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/57b9ba51::symbolic[0]::{impl#1}[0]::symbolic[0] +[Crux] lib/crucible/symbolic.rs:24:64: 24:68 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/6430d6c4::symbolic[0]::{impl#1}[0]::symbolic[0] [Crux] invalid symbolic variable name "\NUL:., /": Identifier must start with a letter. [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/overrides/override1.good b/crux-mir/test/symb_eval/overrides/override1.good index 514c4e67c..99b4231d9 100644 --- a/crux-mir/test/symb_eval/overrides/override1.good +++ b/crux-mir/test/symb_eval/overrides/override1.good @@ -1,4 +1,4 @@ -test override1/abe84667::f[0]: Hello, I'm an override +test override1/59e81ac9::f[0]: Hello, I'm an override returned 2, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/overrides/override2.good b/crux-mir/test/symb_eval/overrides/override2.good index 0ddb174b0..46e8f9344 100644 --- a/crux-mir/test/symb_eval/overrides/override2.good +++ b/crux-mir/test/symb_eval/overrides/override2.good @@ -1,10 +1,10 @@ -test override2/3a1fbbbh::f[0]: FAILED +test override2/cadd74c9::f[0]: FAILED failures: ----- override2/3a1fbbbh::f[0] counterexamples ---- +---- override2/cadd74c9::f[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:37:41: 37:58 !test/symb_eval/overrides/override2.rs:9:5: 9:50: error: in override2/3a1fbbbh::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/overrides/override2.rs:9:5: 9:49: error: in override2/cadd74c9::f[0] [Crux] MIR assertion at test/symb_eval/overrides/override2.rs:9:5: [Crux] foo.wrapping_add(1) == foo diff --git a/crux-mir/test/symb_eval/overrides/override3.good b/crux-mir/test/symb_eval/overrides/override3.good index 8f662efcc..00eb584ab 100644 --- a/crux-mir/test/symb_eval/overrides/override3.good +++ b/crux-mir/test/symb_eval/overrides/override3.good @@ -1,3 +1,3 @@ -test override3/e63a77d1::f[0]: ok +test override3/503ceed2::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/overrides/override4.good b/crux-mir/test/symb_eval/overrides/override4.good index 1ad5aa98e..acdc6b932 100644 --- a/crux-mir/test/symb_eval/overrides/override4.good +++ b/crux-mir/test/symb_eval/overrides/override4.good @@ -1,3 +1,3 @@ -test override4/cf8d07e7::f[0]: ok +test override4/8240a34c::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/overrides/override5.good b/crux-mir/test/symb_eval/overrides/override5.good index be7c0684d..29687eb77 100644 --- a/crux-mir/test/symb_eval/overrides/override5.good +++ b/crux-mir/test/symb_eval/overrides/override5.good @@ -1,10 +1,10 @@ -test override5/3a1fbbbh::f[0]: FAILED +test override5/31b4d207::f[0]: FAILED failures: ----- override5/3a1fbbbh::f[0] counterexamples ---- +---- override5/31b4d207::f[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:37:41: 37:58 !test/symb_eval/overrides/override5.rs:10:5: 10:48: error: in override5/3a1fbbbh::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/overrides/override5.rs:10:5: 10:47: error: in override5/31b4d207::f[0] [Crux] MIR assertion at test/symb_eval/overrides/override5.rs:10:5: [Crux] foo.wrapping_add(1) != 0 diff --git a/crux-mir/test/symb_eval/overrides/override_rust.good b/crux-mir/test/symb_eval/overrides/override_rust.good index 82d82c98a..c26c1a437 100644 --- a/crux-mir/test/symb_eval/overrides/override_rust.good +++ b/crux-mir/test/symb_eval/overrides/override_rust.good @@ -1,3 +1,3 @@ -test override_rust/b13aa63f::crux_test[0]: ok +test override_rust/3f867a7c::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/refs/mux_init_imm.good b/crux-mir/test/symb_eval/refs/mux_init_imm.good index 3a9aa5ac3..37931b749 100644 --- a/crux-mir/test/symb_eval/refs/mux_init_imm.good +++ b/crux-mir/test/symb_eval/refs/mux_init_imm.good @@ -1,3 +1,3 @@ -test mux_init_imm/425f8639::crux_test[0]: returned Symbolic BV, ok +test mux_init_imm/af1ab5c2::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/refs/mux_init_mut.good b/crux-mir/test/symb_eval/refs/mux_init_mut.good index 305f5d5fb..952e82e81 100644 --- a/crux-mir/test/symb_eval/refs/mux_init_mut.good +++ b/crux-mir/test/symb_eval/refs/mux_init_mut.good @@ -1,3 +1,3 @@ -test mux_init_mut/a10fc099::crux_test[0]: returned Symbolic BV, ok +test mux_init_mut/ba52aab3::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/scalar/test1.good b/crux-mir/test/symb_eval/scalar/test1.good index 3b0523bf5..0c2afaefb 100644 --- a/crux-mir/test/symb_eval/scalar/test1.good +++ b/crux-mir/test/symb_eval/scalar/test1.good @@ -1,3 +1,3 @@ -test test1/87e099b9::f[0]: ok +test test1/38b24463::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/sym_bytes/construct.good b/crux-mir/test/symb_eval/sym_bytes/construct.good index db1a66a4c..c14372be6 100644 --- a/crux-mir/test/symb_eval/sym_bytes/construct.good +++ b/crux-mir/test/symb_eval/sym_bytes/construct.good @@ -1,10 +1,10 @@ -test construct/3a1fbbbh::crux_test[0]: returned Symbolic BV, FAILED +test construct/961e3d80::crux_test[0]: returned Symbolic BV, FAILED failures: ----- construct/3a1fbbbh::crux_test[0] counterexamples ---- +---- construct/961e3d80::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:37:41: 37:58 !test/symb_eval/sym_bytes/construct.rs:13:5: 13:36: error: in construct/3a1fbbbh::crux_test[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/sym_bytes/construct.rs:13:5: 13:35: error: in construct/961e3d80::crux_test[0] [Crux] MIR assertion at test/symb_eval/sym_bytes/construct.rs:13:5: [Crux] sym2[0] == 0 diff --git a/crux-mir/test/symb_eval/sym_bytes/deserialize.good b/crux-mir/test/symb_eval/sym_bytes/deserialize.good index 3b3ed63a0..43e3a3443 100644 --- a/crux-mir/test/symb_eval/sym_bytes/deserialize.good +++ b/crux-mir/test/symb_eval/sym_bytes/deserialize.good @@ -1,10 +1,10 @@ -test deserialize/21c6f4ae::crux_test[0]: returned Symbolic BV, FAILED +test deserialize/9453da6c::crux_test[0]: returned Symbolic BV, FAILED failures: ----- deserialize/21c6f4ae::crux_test[0] counterexamples ---- +---- deserialize/9453da6c::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/sym_bytes/deserialize.rs:19:21: 19:44: error: in deserialize/21c6f4ae::deserialize[0] +[Crux] test/symb_eval/sym_bytes/deserialize.rs:19:21: 19:44: error: in deserialize/9453da6c::deserialize[0] [Crux] attempt to compute `move _61 + move _62`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/vector/as_mut_slice.good b/crux-mir/test/symb_eval/vector/as_mut_slice.good index 9812fd1be..c8c97f037 100644 --- a/crux-mir/test/symb_eval/vector/as_mut_slice.good +++ b/crux-mir/test/symb_eval/vector/as_mut_slice.good @@ -1,3 +1,3 @@ -test as_mut_slice/23bdd9be::f[0]: ok +test as_mut_slice/2fb2b2e1::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/as_slice.good b/crux-mir/test/symb_eval/vector/as_slice.good index 68f7fd280..d385b1f74 100644 --- a/crux-mir/test/symb_eval/vector/as_slice.good +++ b/crux-mir/test/symb_eval/vector/as_slice.good @@ -1,3 +1,3 @@ -test as_slice/0226cf7b::f[0]: ok +test as_slice/c1cfd521::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/concat.good b/crux-mir/test/symb_eval/vector/concat.good index b1473c41b..cb93061db 100644 --- a/crux-mir/test/symb_eval/vector/concat.good +++ b/crux-mir/test/symb_eval/vector/concat.good @@ -1,3 +1,3 @@ -test concat/a9c87648::f[0]: ok +test concat/580b9ab8::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/copy_from_slice.good b/crux-mir/test/symb_eval/vector/copy_from_slice.good index c48ae65d4..46a80119e 100644 --- a/crux-mir/test/symb_eval/vector/copy_from_slice.good +++ b/crux-mir/test/symb_eval/vector/copy_from_slice.good @@ -1,3 +1,3 @@ -test copy_from_slice/c0ec1446::f[0]: ok +test copy_from_slice/43f9bf00::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/mut.good b/crux-mir/test/symb_eval/vector/mut.good index 7f712fc6d..798ccfca5 100644 --- a/crux-mir/test/symb_eval/vector/mut.good +++ b/crux-mir/test/symb_eval/vector/mut.good @@ -1,3 +1,3 @@ -test mut/7326e6fa::f[0]: ok +test mut/33f66d86::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/new.good b/crux-mir/test/symb_eval/vector/new.good index 1fc92321a..b40921c17 100644 --- a/crux-mir/test/symb_eval/vector/new.good +++ b/crux-mir/test/symb_eval/vector/new.good @@ -1,3 +1,3 @@ -test new/ed554471::f[0]: ok +test new/b6f18315::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/push.good b/crux-mir/test/symb_eval/vector/push.good index 99fd1905f..85b6630a9 100644 --- a/crux-mir/test/symb_eval/vector/push.good +++ b/crux-mir/test/symb_eval/vector/push.good @@ -1,3 +1,3 @@ -test push/f463b0b2::f[0]: ok +test push/6f8aba04::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/replicate.good b/crux-mir/test/symb_eval/vector/replicate.good index 2d01d8952..e495481ad 100644 --- a/crux-mir/test/symb_eval/vector/replicate.good +++ b/crux-mir/test/symb_eval/vector/replicate.good @@ -1,3 +1,3 @@ -test replicate/99a2adac::f[0]: ok +test replicate/fc4d884f::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/split_at.good b/crux-mir/test/symb_eval/vector/split_at.good index da9830bc7..8c4b42226 100644 --- a/crux-mir/test/symb_eval/vector/split_at.good +++ b/crux-mir/test/symb_eval/vector/split_at.good @@ -1,3 +1,3 @@ -test split_at/3be09e25::f[0]: ok +test split_at/cf146783::f[0]: ok [Crux] Overall status: Valid. From b7a30efbc251f9acf0b8f2800e885f88a1d5bd94 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Tue, 23 May 2023 20:52:29 -0400 Subject: [PATCH 058/114] crux-mir: Update test_output.log --- crux-mir/test_output.log | 1730 +++++++++++++++++--------------------- 1 file changed, 763 insertions(+), 967 deletions(-) diff --git a/crux-mir/test_output.log b/crux-mir/test_output.log index 38873032c..6dea76706 100644 --- a/crux-mir/test_output.log +++ b/crux-mir/test_output.log @@ -2,32 +2,32 @@ crux-mir crux concrete clos - fn_static_poly: OK (0.96s) + fn_static_poly: OK (0.97s) Compiling and running oracle program (0.12s) - Oracle output: 3 (0.84s) + Oracle output: 3 (0.85s) Crux output: 3 - fnptr_closure: OK (0.99s) - Compiling and running oracle program (0.12s) - Oracle output: 7 (0.86s) + fnptr_closure: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: 7 (0.87s) Crux output: 7 - fnptr_fn: OK (0.98s) + fnptr_fn: OK (0.99s) Compiling and running oracle program (0.13s) - Oracle output: 4 (0.85s) + Oracle output: 4 (0.87s) Crux output: 4 - conv_fnonce_fnmut: OK (0.96s) - Compiling and running oracle program (0.12s) - Oracle output: 2 (0.84s) + conv_fnonce_fnmut: OK (0.99s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (0.86s) Crux output: 2 - as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.86s) + as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (1.04s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (0.92s) Crux output: failures: - ---- as_fn_ptr/bd4bdeb8::crux_test[0] counterexamples ---- + ---- as_fn_ptr/c6e389f5::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/bd4bdeb8::crux_test[0] - [Crux] Translation error in as_fn_ptr/bd4bdeb8::crux_test[0]: unimplemented cast: ClosureFnPointer + [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/c6e389f5::crux_test[0] + [Crux] Translation error in as_fn_ptr/c6e389f5::crux_test[0]: unimplemented cast: ClosureFnPointer [Crux] ty: TyClosure [] [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [], _fsreturn_ty = TyInt B32, _fsabi = RustAbi, _fsspreadarg = Nothing}) @@ -36,50 +36,50 @@ crux-mir test/Test.hs:124: crux doesn't match oracle (expected failure) - fnonce1: OK (0.96s) - Compiling and running oracle program (0.12s) - Oracle output: 3 (0.84s) + fnonce1: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.85s) Crux output: 3 - direct_fn: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 2 (0.84s) + direct_fn: OK (0.99s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.86s) Crux output: 2 - fn_static: OK (0.96s) - Compiling and running oracle program (0.12s) - Oracle output: 3 (0.84s) + fn_static: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.88s) Crux output: 3 - poly: OK (0.96s) + poly: OK (0.97s) Compiling and running oracle program (0.13s) Oracle output: 3 (0.84s) Crux output: 3 - direct_fnmut: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 7 (0.85s) - Crux output: 7 - direct_fnmut2: OK (1.00s) - Compiling and running oracle program (0.12s) + direct_fnmut: OK (1.00s) + Compiling and running oracle program (0.13s) Oracle output: 7 (0.87s) Crux output: 7 - promoted: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 33 (0.84s) + direct_fnmut2: OK (1.02s) + Compiling and running oracle program (0.13s) + Oracle output: 7 (0.89s) + Crux output: 7 + promoted: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 33 (0.86s) Crux output: 33 - direct_fnonce: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 2 (0.85s) + direct_fnonce: OK (1.05s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.93s) Crux output: 2 - fo: OK (0.97s) + fo: OK (0.96s) Compiling and running oracle program (0.13s) - Oracle output: 29 (0.84s) + Oracle output: 29 (0.83s) Crux output: 29 - fnonce: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: false (0.85s) + fnonce: OK (0.99s) + Compiling and running oracle program (0.13s) + Oracle output: false (0.86s) Crux output: false - fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (0.97s) + fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (0.98s) Compiling and running oracle program (0.13s) - Oracle output: 3 (0.84s) - vtable signature mismatch for vtable core/9a570a5e::ops[0]::function[0]::Fn[0]::_vtbl570ffd80324c2ca9[0] of trait core/9a570a5e::ops[0]::function[0]::Fn[0]::_trait42f8b6475ee85722[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32)] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32)] + Oracle output: 3 (0.86s) + vtable signature mismatch for vtable core/beffc212::ops[0]::function[0]::Fn[0]::_vtbl570ffd80324c2ca9[0] of trait core/beffc212::ops[0]::function[0]::Fn[0]::_trait42f8b6475ee85722[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32)] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32)] CallStack (from HasCallStack): error, called at src/Mir/Trans.hs:933:20 in crucible-mir-0.1-inplace:Mir.Trans mkTraitObject, called at src/Mir/Trans.hs:776:11 in crucible-mir-0.1-inplace:Mir.Trans @@ -94,54 +94,54 @@ crux-mir transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language (expected failure) - ref_fnmut: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 7 (0.85s) + ref_fnmut: OK (1.03s) + Compiling and running oracle program (0.17s) + Oracle output: 7 (0.87s) Crux output: 7 dispatch_fnmut: OK (1.00s) Compiling and running oracle program (0.13s) Oracle output: 7 (0.88s) Crux output: 7 - unique_borrow: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.85s) + unique_borrow: OK (1.00s) + Compiling and running oracle program (0.12s) + Oracle output: 3 (0.87s) Crux output: 3 - conv_fnmut_fn: OK (1.00s) + conv_fnmut_fn: OK (0.97s) Compiling and running oracle program (0.12s) - Oracle output: 2 (0.88s) + Oracle output: 2 (0.85s) Crux output: 2 - fnptr_fnmut: OK (1.02s) + fnptr_fnmut: OK (1.00s) Compiling and running oracle program (0.13s) - Oracle output: 4 (0.89s) + Oracle output: 4 (0.87s) Crux output: 4 - fnptr_fnonce: OK (1.03s) + fnptr_fnonce: OK (0.97s) Compiling and running oracle program (0.13s) - Oracle output: 4 (0.90s) + Oracle output: 4 (0.84s) Crux output: 4 - conv_fnonce_fn: OK (1.00s) + conv_fnonce_fn: OK (0.98s) Compiling and running oracle program (0.12s) - Oracle output: 2 (0.87s) + Oracle output: 2 (0.85s) Crux output: 2 prim - bool: OK (1.00s) + bool: OK (0.96s) Compiling and running oracle program (0.13s) - Oracle output: false (0.87s) + Oracle output: false (0.83s) Crux output: false - litbstring: OK (1.00s) + litbstring: OK (0.99s) Compiling and running oracle program (0.12s) - Oracle output: true (0.88s) + Oracle output: true (0.87s) Crux output: true - shift3: OK (0.96s) - Compiling and running oracle program (0.12s) + shift3: OK (0.97s) + Compiling and running oracle program (0.13s) Oracle output: -9223372036854775808 (0.84s) Crux output: -9223372036854775808 - litstring: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: true (0.88s) + litstring: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: true (0.85s) Crux output: true - ge: OK (1.01s) + ge: OK (0.96s) Compiling and running oracle program (0.12s) - Oracle output: true (0.89s) + Oracle output: true (0.84s) Crux output: true shift_exceeding: rustc compilation failed for shift_exceeding error output: @@ -178,231 +178,231 @@ FAIL (expected: Should panic, but doesn't) (0.05s) test/Test.hs:107: failed to compile and run (expected failure) - shift4: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: -9223372036854775808 (0.88s) + shift4: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: -9223372036854775808 (0.85s) Crux output: -9223372036854775808 - mut_arg: OK (0.99s) + mut_arg: OK (0.98s) Compiling and running oracle program (0.13s) - Oracle output: 1 (0.86s) + Oracle output: 1 (0.85s) Crux output: 1 wrapping_sub: OK (0.97s) - Compiling and running oracle program (0.13s) + Compiling and running oracle program (0.12s) Oracle output: true (0.84s) Crux output: true - lit: OK (0.98s) + lit: OK (0.97s) Compiling and running oracle program (0.13s) Oracle output: false (0.85s) Crux output: false - mut: OK (0.99s) + mut: OK (0.98s) Compiling and running oracle program (0.13s) - Oracle output: 14 (0.86s) + Oracle output: 14 (0.85s) Crux output: 14 - shift1: OK (0.96s) + shift1: OK (0.99s) Compiling and running oracle program (0.13s) - Oracle output: 2 (0.84s) + Oracle output: 2 (0.86s) Crux output: 2 add1: OK (0.96s) Compiling and running oracle program (0.12s) Oracle output: 2 (0.84s) Crux output: 2 - div: OK (0.96s) - Compiling and running oracle program (0.12s) + div: OK (0.97s) + Compiling and running oracle program (0.13s) Oracle output: 4 (0.84s) Crux output: 4 - shift2: OK (0.96s) + shift2: OK (0.98s) Compiling and running oracle program (0.12s) - Oracle output: 2 (0.84s) + Oracle output: 2 (0.86s) Crux output: 2 - ffs: OK (1.03s) + ffs: OK (1.06s) Compiling and running oracle program (0.13s) - Oracle output: true (0.90s) + Oracle output: true (0.93s) Crux output: true char_from_u32: OK (0.99s) - Compiling and running oracle program (0.12s) + Compiling and running oracle program (0.13s) Oracle output: 'A' (0.87s) Crux output: 'A' traits - tyfam: OK (0.98s) - Compiling and running oracle program (0.12s) - Oracle output: 23 (0.86s) + tyfam: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 23 (0.85s) Crux output: 23 - dynamic_branch: OK (1.01s) + dynamic_branch: OK (0.97s) Compiling and running oracle program (0.13s) - Oracle output: 1 (0.88s) + Oracle output: 1 (0.84s) Crux output: 1 - static: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.88s) + static: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 42 (0.85s) Crux output: 42 - bounds3: OK (1.02s) + bounds3: OK (0.99s) Compiling and running oracle program (0.13s) - Oracle output: () (0.88s) + Oracle output: () (0.86s) Crux output: () - tyfam3: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: 23 (0.86s) - Crux output: 23 - static_two: OK (1.00s) + tyfam3: OK (0.96s) Compiling and running oracle program (0.12s) - Oracle output: 1 (0.87s) + Oracle output: 23 (0.84s) + Crux output: 23 + static_two: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.85s) Crux output: 1 - bounds2: OK (0.99s) + bounds2: OK (1.00s) Compiling and running oracle program (0.13s) Oracle output: () (0.87s) Crux output: () - generic2: OK (0.99s) + generic2: OK (1.00s) Compiling and running oracle program (0.13s) Oracle output: () (0.87s) Crux output: () - gen_trait_poly: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 12 (0.85s) + gen_trait_poly: OK (0.98s) + Compiling and running oracle program (0.12s) + Oracle output: 12 (0.86s) Crux output: 12 - dict_simple: OK (0.98s) + dict_simple: OK (0.97s) Compiling and running oracle program (0.13s) - Oracle output: 32 (0.85s) + Oracle output: 32 (0.84s) Crux output: 32 static_self: OK (0.96s) - Compiling and running oracle program (0.13s) + Compiling and running oracle program (0.12s) Oracle output: 42 (0.83s) Crux output: 42 tyfam4: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 0 (0.84s) + Compiling and running oracle program (0.13s) + Oracle output: 0 (0.85s) Crux output: 0 gen_trait: OK (0.97s) Compiling and running oracle program (0.13s) Oracle output: -69 (0.84s) Crux output: -69 - generic3: OK (0.98s) + generic3: OK (0.99s) Compiling and running oracle program (0.13s) - Oracle output: () (0.85s) + Oracle output: () (0.87s) Crux output: () - subtrait: OK (0.99s) + subtrait: OK (0.97s) Compiling and running oracle program (0.13s) - Oracle output: 73 (0.86s) + Oracle output: 73 (0.84s) Crux output: 73 default: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 12 (0.84s) + Compiling and running oracle program (0.12s) + Oracle output: 12 (0.85s) Crux output: 12 - bounds1: OK (0.98s) + bounds1: OK (1.01s) Compiling and running oracle program (0.13s) - Oracle output: () (0.85s) + Oracle output: () (0.88s) Crux output: () - intoiter: OK (0.96s) - Compiling and running oracle program (0.12s) - Oracle output: 0 (0.84s) - Crux output: 0 - static_eq: OK (0.98s) + intoiter: OK (0.98s) Compiling and running oracle program (0.13s) + Oracle output: 0 (0.86s) + Crux output: 0 + static_eq: OK (0.97s) + Compiling and running oracle program (0.12s) Oracle output: true (0.85s) Crux output: true assoc3: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.85s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.84s) Crux output: () - dynamic_poly: OK (0.97s) + dynamic_poly: OK (0.98s) Compiling and running oracle program (0.13s) - Oracle output: 3 (0.84s) + Oracle output: 3 (0.85s) Crux output: 3 - dynamic_med: OK (0.96s) + dynamic_med: OK (0.98s) Compiling and running oracle program (0.13s) - Oracle output: 1 (0.83s) + Oracle output: 1 (0.86s) Crux output: 1 static_three: OK (0.97s) Compiling and running oracle program (0.13s) - Oracle output: 2 (0.85s) + Oracle output: 2 (0.84s) Crux output: 2 - dynamic_two: OK (0.96s) - Compiling and running oracle program (0.12s) - Oracle output: 1 (0.83s) + dynamic_two: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.86s) Crux output: 1 - bounds4: OK (1.00s) + bounds4: OK (0.99s) Compiling and running oracle program (0.13s) Oracle output: () (0.87s) Crux output: () - bounds5: OK (1.01s) + bounds5: OK (1.00s) Compiling and running oracle program (0.13s) - Oracle output: () (0.88s) + Oracle output: () (0.87s) Crux output: () - dict_med: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 42 (0.84s) + dict_med: OK (0.99s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.86s) Crux output: 42 dynamic_simple: OK (0.97s) Compiling and running oracle program (0.13s) Oracle output: 1 (0.84s) Crux output: 1 - assoc1: OK (0.98s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.86s) + assoc1: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.88s) Crux output: () - params: OK (0.96s) - Compiling and running oracle program (0.12s) - Oracle output: 25 (0.84s) + params: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 25 (0.85s) Crux output: 25 - conv: OK (0.96s) - Compiling and running oracle program (0.12s) - Oracle output: 1 (0.84s) + conv: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.85s) Crux output: 1 - basics1: OK (0.99s) + basics1: OK (1.00s) Compiling and running oracle program (0.13s) Oracle output: () (0.87s) Crux output: () - dict_poly: OK (0.98s) + dict_poly: OK (0.97s) Compiling and running oracle program (0.13s) - Oracle output: 1 (0.85s) + Oracle output: 1 (0.84s) Crux output: 1 dict_polymem: OK (0.96s) - Compiling and running oracle program (0.13s) + Compiling and running oracle program (0.12s) Oracle output: 4 (0.84s) Crux output: 4 - subtrait2: OK (0.97s) - Compiling and running oracle program (0.12s) + subtrait2: OK (0.98s) + Compiling and running oracle program (0.13s) Oracle output: 64 (0.85s) Crux output: 64 - tyfam2: OK (0.98s) + tyfam2: OK (0.96s) Compiling and running oracle program (0.13s) - Oracle output: 23 (0.85s) + Oracle output: 23 (0.84s) Crux output: 23 tyfam5: OK (0.97s) Compiling and running oracle program (0.12s) - Oracle output: () (0.84s) + Oracle output: () (0.85s) Crux output: () - assoc2: OK (0.99s) + assoc2: OK (0.97s) Compiling and running oracle program (0.13s) - Oracle output: () (0.86s) + Oracle output: () (0.84s) Crux output: () - generic1: OK (0.98s) + generic1: OK (0.97s) Compiling and running oracle program (0.13s) - Oracle output: () (0.86s) + Oracle output: () (0.85s) Crux output: () impl - self_mut: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 42 (0.85s) + self_mut: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.86s) Crux output: 42 - self: OK (0.96s) - Compiling and running oracle program (0.12s) - Oracle output: 42 (0.83s) + self: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.84s) Crux output: 42 - simple: OK (0.95s) - Compiling and running oracle program (0.12s) - Oracle output: 42 (0.83s) + simple: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.84s) Crux output: 42 vec drop: FAIL (1.19s) Compiling and running oracle program (0.13s) - Oracle output: () (1.06s) + Oracle output: () (1.05s) Crux output: failures: - ---- drop/b09762c6::crux_test[0] counterexamples ---- + ---- drop/883706d7::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_instb944086d0b551051[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_instb944086d0b551051[0]: evalPlace (PField, Union) NYI + [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_instb944086d0b551051[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_instb944086d0b551051[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -412,24 +412,24 @@ FAIL (expected: Should panic, but doesn't) (0.05s) Use -p '/drop/' to rerun this test only. from_elem_zero: FAIL (0.18s) Compiling and running oracle program (0.13s) - Oracle output: 0 (0.05s) + Oracle output: 0 (0.04s) user error (Error 101 while running mir-json on test/conc_eval/vec/from_elem_zero.rs) Use -p '/from_elem_zero/' to rerun this test only. - extend_trusted_len: OK (1.80s) + extend_trusted_len: OK (1.84s) Compiling and running oracle program (0.14s) - Oracle output: (1, 10) (1.66s) + Oracle output: (1, 10) (1.70s) Crux output: (1, 10) - extend: FAIL (1.28s) + extend: FAIL (1.26s) Compiling and running oracle program (0.14s) - Oracle output: (1, 10) (1.15s) + Oracle output: (1, 10) (1.12s) Crux output: failures: - ---- extend/e200844a::crux_test[0] counterexamples ---- + ---- extend/e55977a0::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in core/9a570a5e::iter[0]::traits[0]::iterator[0]::Iterator[0]::try_fold[0]::_inst647f2421f6e4ebd6[0] - [Crux] Translation error in core/9a570a5e::iter[0]::traits[0]::iterator[0]::Iterator[0]::try_fold[0]::_inst647f2421f6e4ebd6[0]: error building variant core/9a570a5e::ops[0]::control_flow[0]::ControlFlow[0]::Continue[0]: type mismatch: expected AnyRepr but got BVRepr 32 in field 0: ([FieldRepr (FkInit AnyRepr)],[Just bVLit(32, BV 0)]) -- [Just bVLit(32, BV 0): BVRepr 32] + [Crux] internal: error: in core/beffc212::iter[0]::traits[0]::iterator[0]::Iterator[0]::try_fold[0]::_inst647f2421f6e4ebd6[0] + [Crux] Translation error in core/beffc212::iter[0]::traits[0]::iterator[0]::Iterator[0]::try_fold[0]::_inst647f2421f6e4ebd6[0]: error building variant core/beffc212::ops[0]::control_flow[0]::ControlFlow[0]::Continue[0]: type mismatch: expected AnyRepr but got BVRepr 32 in field 0: ([FieldRepr (FkInit AnyRepr)],[Just bVLit(32, BV 0)]) -- [Just bVLit(32, BV 0): BVRepr 32] [Crux] Overall status: Invalid. @@ -437,16 +437,16 @@ FAIL (expected: Should panic, but doesn't) (0.05s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..vec.extend"' to rerun this test only. - set_len: FAIL (1.16s) + set_len: FAIL (1.20s) Compiling and running oracle program (0.13s) - Oracle output: 2 (1.04s) + Oracle output: 2 (1.08s) Crux output: failures: - ---- set_len/3493ee41::crux_test[0] counterexamples ---- + ---- set_len/225fd37f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI + [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -454,16 +454,16 @@ FAIL (expected: Should panic, but doesn't) (0.05s) crux doesn't match oracle Use -p '/set_len/' to rerun this test only. - collect: FAIL (1.86s) - Compiling and running oracle program (0.13s) - Oracle output: 45 (1.73s) + collect: FAIL (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 45 (1.72s) Crux output: failures: - ---- collect/3459f282::crux_test[0] counterexamples ---- + ---- collect/1633cf1b::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:786:58: 786:72: error: in core/9a570a5e::ptr[0]::const_ptr[0]::{impl#0}[0]::sub_ptr[0]::runtime[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::const_ptr[0]::{impl#0}[0]::sub_ptr[0]::runtime[0]::_inst1e2825177cd3b608[0]: No translation for pointer binop: >= + [Crux] ./lib/core/src/ptr/const_ptr.rs:786:58: 786:72: error: in core/beffc212::ptr[0]::const_ptr[0]::{impl#0}[0]::sub_ptr[0]::runtime[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/beffc212::ptr[0]::const_ptr[0]::{impl#0}[0]::sub_ptr[0]::runtime[0]::_inst1e2825177cd3b608[0]: No translation for pointer binop: >= [Crux] Overall status: Invalid. @@ -471,25 +471,25 @@ FAIL (expected: Should panic, but doesn't) (0.05s) crux doesn't match oracle Use -p '/collect/' to rerun this test only. - push: OK (1.19s) + push: OK (1.20s) Compiling and running oracle program (0.13s) - Oracle output: (1, 2) (1.06s) + Oracle output: (1, 2) (1.07s) Crux output: (1, 2) slice - len: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 5 (0.85s) + len: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 5 (0.84s) Crux output: 5 mut_range: FAIL (1.52s) - Compiling and running oracle program (0.12s) + Compiling and running oracle program (0.13s) Oracle output: 86 (1.39s) Crux output: failures: - ---- mut_range/1fd9f010::crux_test[0] counterexamples ---- + ---- mut_range/b5c421ea::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -497,16 +497,16 @@ FAIL (expected: Should panic, but doesn't) (0.05s) crux doesn't match oracle Use -p '/mut_range/' to rerun this test only. - range_len: FAIL (1.53s) + range_len: FAIL (1.50s) Compiling and running oracle program (0.13s) - Oracle output: 1 (1.41s) + Oracle output: 1 (1.37s) Crux output: failures: - ---- range_len/c3db385f::crux_test[0] counterexamples ---- + ---- range_len/5665dfa7::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -514,16 +514,16 @@ FAIL (expected: Should panic, but doesn't) (0.05s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..slice.range_len"' to rerun this test only. - eq: FAIL (1.02s) + eq: FAIL (1.01s) Compiling and running oracle program (0.12s) - Oracle output: () (0.90s) + Oracle output: () (0.89s) Crux output: failures: - ---- eq/6780a00b::f[0] counterexamples ---- + ---- eq/b2461ea6::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/mem/mod.rs:338:38: 338:41: error: in core/9a570a5e::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/9a570a5e::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0]: callExp: Don't know how to call core/9a570a5e::intrinsics[0]::{extern#0}[0]::size_of_val[0]::_inst55af81a2cef2701c[0] + [Crux] ./lib/core/src/mem/mod.rs:338:38: 338:41: error: in core/beffc212::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/beffc212::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0]: callExp: Don't know how to call core/beffc212::intrinsics[0]::{extern#0}[0]::size_of_val[0]::_inst55af81a2cef2701c[0] [Crux] Overall status: Invalid. @@ -531,16 +531,16 @@ FAIL (expected: Should panic, but doesn't) (0.05s) crux doesn't match oracle Use -p '/slice.eq/' to rerun this test only. - iter_mut: FAIL (1.08s) + iter_mut: FAIL (1.06s) Compiling and running oracle program (0.13s) - Oracle output: () (0.95s) + Oracle output: () (0.93s) Crux output: failures: - ---- iter_mut/43e13e25::f[0] counterexamples ---- + ---- iter_mut/bf566c00::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/array/mod.rs:266:23: 266:54: error: in core/9a570a5e::array[0]::{impl#10}[0]::try_from[0]::_insta17aeaef2aec3782[0] - [Crux] Translation error in core/9a570a5e::array[0]::{impl#10}[0]::try_from[0]::_insta17aeaef2aec3782[0]: unimplemented cast: Misc + [Crux] ./lib/core/src/array/mod.rs:266:23: 266:54: error: in core/beffc212::array[0]::{impl#10}[0]::try_from[0]::_insta17aeaef2aec3782[0] + [Crux] Translation error in core/beffc212::array[0]::{impl#10}[0]::try_from[0]::_insta17aeaef2aec3782[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyInt B32) Immut [Crux] as: TyRawPtr (TyArray (TyInt B32) 3) Immut @@ -551,15 +551,15 @@ FAIL (expected: Should panic, but doesn't) (0.05s) Use -p '/iter_mut/' to rerun this test only. get: FAIL (1.03s) - Compiling and running oracle program (0.13s) - Oracle output: true (0.90s) + Compiling and running oracle program (0.12s) + Oracle output: true (0.91s) Crux output: failures: - ---- get/6397f384::crux_test[0] counterexamples ---- + ---- get/56903f09::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -568,19 +568,19 @@ FAIL (expected: Should panic, but doesn't) (0.05s) Use -p '/slice.get/' to rerun this test only. last: OK (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.87s) + Compiling and running oracle program (0.12s) + Oracle output: 3 (0.86s) Crux output: 3 - range_len_mut: FAIL (1.57s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (1.44s) + range_len_mut: FAIL (1.54s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (1.42s) Crux output: failures: - ---- range_len_mut/84f21ce0::crux_test[0] counterexamples ---- + ---- range_len_mut/dc7450de::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -588,80 +588,80 @@ FAIL (expected: Should panic, but doesn't) (0.05s) crux doesn't match oracle Use -p '/range_len_mut/' to rerun this test only. - mut: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 42 (0.84s) + mut: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.86s) Crux output: 42 - swap: OK (1.02s) + swap: OK (1.01s) Compiling and running oracle program (0.13s) - Oracle output: 2001 (0.89s) + Oracle output: 2001 (0.88s) Crux output: 2001 - mk_and_proj: OK (1.01s) + mk_and_proj: OK (0.96s) Compiling and running oracle program (0.13s) - Oracle output: 42 (0.87s) + Oracle output: 42 (0.83s) Crux output: 42 num - overflow: OK (1.07s) + overflow: OK (1.03s) Compiling and running oracle program (0.13s) - Oracle output: () (0.94s) + Oracle output: () (0.90s) Crux output: () - saturate: OK (1.57s) + saturate: OK (1.53s) Compiling and running oracle program (0.13s) - Oracle output: () (1.45s) + Oracle output: () (1.40s) Crux output: () - from_bytes: OK (1.02s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.89s) + from_bytes: OK (1.03s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.91s) Crux output: () dyn inherit: OK (1.01s) Compiling and running oracle program (0.13s) - Oracle output: 4 (0.89s) + Oracle output: 4 (0.88s) Crux output: 4 - plain_trait: OK (0.97s) + plain_trait: OK (0.98s) Compiling and running oracle program (0.13s) - Oracle output: 100 (0.85s) - Crux output: 100 - trait_param: OK (0.99s) - Compiling and running oracle program (0.12s) Oracle output: 100 (0.86s) Crux output: 100 - assoc_ty: OK (0.97s) + trait_param: OK (0.96s) Compiling and running oracle program (0.13s) Oracle output: 100 (0.84s) Crux output: 100 + assoc_ty: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 100 (0.86s) + Crux output: 100 box - mut_ref: OK (1.12s) + mut_ref: OK (1.13s) Compiling and running oracle program (0.13s) - Oracle output: () (0.99s) + Oracle output: () (1.00s) Crux output: () - new: OK (1.15s) - Compiling and running oracle program (0.13s) - Oracle output: () (1.02s) + new: OK (1.11s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.99s) Crux output: () - struct: OK (1.15s) + struct: OK (1.10s) Compiling and running oracle program (0.13s) - Oracle output: 1 (1.03s) + Oracle output: 1 (0.98s) Crux output: 1 - mut: OK (1.11s) + mut: OK (1.09s) Compiling and running oracle program (0.13s) - Oracle output: () (0.98s) + Oracle output: () (0.96s) Crux output: () - unsize: OK (1.13s) - Compiling and running oracle program (0.12s) - Oracle output: 2 (1.01s) + unsize: OK (1.12s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.99s) Crux output: 2 vec_deque - rotate_right: FAIL (1.72s) - Compiling and running oracle program (0.13s) - Oracle output: [4, 5, 1, 2, 3] (1.59s) + rotate_right: FAIL (1.75s) + Compiling and running oracle program (0.14s) + Oracle output: [4, 5, 1, 2, 3] (1.62s) Crux output: failures: - ---- rotate_right/91384e0f::crux_test[0] counterexamples ---- + ---- rotate_right/b7a01f9c::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI + [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -669,16 +669,16 @@ FAIL (expected: Should panic, but doesn't) (0.05s) crux doesn't match oracle Use -p '/rotate_right/' to rerun this test only. - retain: FAIL (1.75s) + retain: FAIL (1.70s) Compiling and running oracle program (0.14s) - Oracle output: [1, 4] (1.62s) + Oracle output: [1, 4] (1.57s) Crux output: failures: - ---- retain/12c8a6d0::crux_test[0] counterexamples ---- + ---- retain/16d9d612::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI + [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -686,20 +686,20 @@ FAIL (expected: Should panic, but doesn't) (0.05s) crux doesn't match oracle Use -p '/retain/' to rerun this test only. - push: OK (1.73s) + push: OK (1.72s) Compiling and running oracle program (0.13s) - Oracle output: [4, 1, 2, 3, 5] (1.60s) + Oracle output: [4, 1, 2, 3, 5] (1.59s) Crux output: [4, 1, 2, 3, 5] - rotate_left: FAIL (1.69s) - Compiling and running oracle program (0.14s) - Oracle output: [3, 4, 5, 1, 2] (1.55s) + rotate_left: FAIL (1.71s) + Compiling and running oracle program (0.13s) + Oracle output: [3, 4, 5, 1, 2] (1.57s) Crux output: failures: - ---- rotate_left/864001d8::crux_test[0] counterexamples ---- + ---- rotate_left/791d5fca::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI + [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -707,16 +707,16 @@ FAIL (expected: Should panic, but doesn't) (0.05s) crux doesn't match oracle Use -p '/rotate_left/' to rerun this test only. - pop: FAIL (1.65s) + pop: FAIL (1.62s) Compiling and running oracle program (0.13s) - Oracle output: [5, 4, 3, 1, 2] (1.52s) + Oracle output: [5, 4, 3, 1, 2] (1.49s) Crux output: failures: - ---- pop/795717d2::crux_test[0] counterexamples ---- + ---- pop/f75a69e4::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI + [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -724,16 +724,16 @@ FAIL (expected: Should panic, but doesn't) (0.05s) crux doesn't match oracle Use -p '/vec_deque.pop/' to rerun this test only. - iter_clone: FAIL (1.76s) - Compiling and running oracle program (0.13s) - Oracle output: [1, 2, 3, 2, 3] (1.62s) + iter_clone: FAIL (1.73s) + Compiling and running oracle program (0.14s) + Oracle output: [1, 2, 3, 2, 3] (1.60s) Crux output: failures: - ---- iter_clone/17f65018::crux_test[0] counterexamples ---- + ---- iter_clone/fecbd0b0::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI + [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -742,29 +742,29 @@ FAIL (expected: Should panic, but doesn't) (0.05s) Use -p '/iter_clone/' to rerun this test only. iter - loop: OK (0.96s) + loop: OK (1.00s) Compiling and running oracle program (0.12s) - Oracle output: 2 (0.84s) + Oracle output: 2 (0.87s) Crux output: 2 - from_fn: OK (1.03s) + from_fn: OK (1.00s) Compiling and running oracle program (0.13s) - Oracle output: () (0.90s) + Oracle output: () (0.88s) Crux output: () - filter_chain: OK (1.15s) + filter_chain: OK (1.13s) Compiling and running oracle program (0.15s) - Oracle output: 0 (1.00s) + Oracle output: 0 (0.98s) Crux output: 0 - for: OK (1.06s) + for: OK (1.02s) Compiling and running oracle program (0.13s) - Oracle output: 47 (0.93s) + Oracle output: 47 (0.90s) Crux output: 47 - zip: OK (1.11s) + zip: OK (1.07s) Compiling and running oracle program (0.13s) - Oracle output: () (0.98s) + Oracle output: () (0.94s) Crux output: () - peek: OK (1.09s) + peek: OK (1.07s) Compiling and running oracle program (0.13s) - Oracle output: 3 (0.96s) + Oracle output: 3 (0.94s) Crux output: 3 cloned: OK (1.02s) Compiling and running oracle program (0.13s) @@ -775,38 +775,38 @@ FAIL (expected: Should panic, but doesn't) (0.05s) Oracle output: () (0.92s) Crux output: () crypto - add: OK (1.05s) + add: OK (1.03s) Compiling and running oracle program (0.13s) - Oracle output: true (0.92s) + Oracle output: true (0.90s) Crux output: true - add_noL: OK (1.05s) + add_noL: OK (1.07s) Compiling and running oracle program (0.14s) - Oracle output: false (0.91s) + Oracle output: false (0.93s) Crux output: false statics - promoted_static: OK (0.97s) - Compiling and running oracle program (0.12s) + promoted_static: OK (0.98s) + Compiling and running oracle program (0.13s) Oracle output: 1 (0.85s) Crux output: 1 - promoted_fn: OK (0.98s) + promoted_fn: OK (0.97s) Compiling and running oracle program (0.13s) - Oracle output: 1 (0.85s) + Oracle output: 1 (0.84s) Crux output: 1 ptr - offset_mut: OK (0.99s) + offset_mut: OK (0.98s) Compiling and running oracle program (0.12s) - Oracle output: 3 (0.87s) + Oracle output: 3 (0.86s) Crux output: 3 - dangling_eq: FAIL (1.02s) + dangling_eq: FAIL (1.01s) Compiling and running oracle program (0.13s) Oracle output: () (0.89s) Crux output: failures: - ---- dangling_eq/c854fd1c::crux_test[0] counterexamples ---- + ---- dangling_eq/d3f79519::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:135:14: 135:78: error: in core/9a570a5e::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI + [Crux] ./lib/core/src/ptr/metadata.rs:135:14: 135:78: error: in core/beffc212::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -814,18 +814,18 @@ FAIL (expected: Should panic, but doesn't) (0.05s) crux doesn't match oracle Use -p '/dangling_eq/' to rerun this test only. - is_null_slice: FAIL (expected: can't unsize null pointers) (1.01s) - Compiling and running oracle program (0.13s) + is_null_slice: FAIL (expected: can't unsize null pointers) (1.02s) + Compiling and running oracle program (0.12s) Oracle output: (false, true) (0.89s) Crux output: failures: - ---- is_null_slice/fec67c8e::crux_test[0] counterexamples ---- + ---- is_null_slice/819ce3b2::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/ptr/is_null_slice.rs:7:13: 7:33: error: in is_null_slice/fec67c8e::crux_test[0] + [Crux] test/conc_eval/ptr/is_null_slice.rs:7:13: 7:33: error: in is_null_slice/819ce3b2::crux_test[0] [Crux] attempted subindex on the result of an integer-to-pointer cast [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:38:38: 38:42: error: in core/9a570a5e::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst55af81a2cef2701c[0] + [Crux] ./lib/core/src/ptr/const_ptr.rs:38:38: 38:42: error: in core/beffc212::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst55af81a2cef2701c[0] [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. @@ -833,56 +833,56 @@ FAIL (expected: Should panic, but doesn't) (0.05s) test/Test.hs:124: crux doesn't match oracle (expected failure) - coerce_unsized: OK (0.97s) + coerce_unsized: OK (0.99s) Compiling and running oracle program (0.12s) - Oracle output: (1, 2) (0.85s) + Oracle output: (1, 2) (0.86s) Crux output: (1, 2) - is_null: OK (0.99s) - Compiling and running oracle program (0.12s) - Oracle output: (false, true) (0.87s) + is_null: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: (false, true) (0.88s) Crux output: (false, true) - offset: OK (1.00s) + offset: OK (0.98s) Compiling and running oracle program (0.13s) - Oracle output: 3 (0.87s) + Oracle output: 3 (0.85s) Crux output: 3 - offset_from: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: -2 (0.89s) + offset_from: OK (1.00s) + Compiling and running oracle program (0.12s) + Oracle output: -2 (0.87s) Crux output: -2 - struct_eq: OK (1.01s) + struct_eq: OK (1.00s) Compiling and running oracle program (0.12s) Oracle output: () (0.88s) Crux output: () - valid_eq: OK (1.01s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.89s) - Crux output: () - cast_eq: OK (1.00s) + valid_eq: OK (0.99s) Compiling and running oracle program (0.13s) Oracle output: () (0.87s) Crux output: () - copy: OK (1.06s) + cast_eq: OK (0.99s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.87s) + Crux output: () + copy: OK (1.03s) Compiling and running oracle program (0.13s) - Oracle output: [1, 2, 3, 1, 2, 3] (0.93s) + Oracle output: [1, 2, 3, 1, 2, 3] (0.91s) Crux output: [1, 2, 3, 1, 2, 3] - unsize_slice: OK (1.02s) + unsize_slice: OK (0.98s) Compiling and running oracle program (0.13s) - Oracle output: (1, 2) (0.89s) + Oracle output: (1, 2) (0.86s) Crux output: (1, 2) - read_write: OK (1.05s) + read_write: OK (1.06s) Compiling and running oracle program (0.13s) - Oracle output: [1, 3, 1] (0.92s) + Oracle output: [1, 3, 1] (0.93s) Crux output: [1, 3, 1] - null_eq: FAIL (1.04s) + null_eq: FAIL (1.03s) Compiling and running oracle program (0.13s) - Oracle output: 1 (0.92s) + Oracle output: 1 (0.90s) Crux output: failures: - ---- null_eq/b1679e97::crux_test[0] counterexamples ---- + ---- null_eq/1dae14de::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:118:14: 118:78: error: in core/9a570a5e::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI + [Crux] ./lib/core/src/ptr/metadata.rs:118:14: 118:78: error: in core/beffc212::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -891,16 +891,16 @@ FAIL (expected: Should panic, but doesn't) (0.05s) Use -p '/null_eq/' to rerun this test only. str - to_owned: FAIL (1.21s) + to_owned: FAIL (1.17s) Compiling and running oracle program (0.13s) - Oracle output: true (1.08s) + Oracle output: true (1.05s) Crux output: failures: - ---- to_owned/54c3c5eb::crux_test[0] counterexamples ---- + ---- to_owned/ac2d2c7c::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -908,16 +908,16 @@ FAIL (expected: Should panic, but doesn't) (0.05s) crux doesn't match oracle Use -p '/to_owned/' to rerun this test only. - format_struct: FAIL (1.94s) + format_struct: FAIL (1.86s) Compiling and running oracle program (0.14s) - Oracle output: true (1.80s) + Oracle output: true (1.72s) Crux output: failures: - ---- format_struct/8f121570::crux_test[0] counterexamples ---- + ---- format_struct/d1c28a08::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/9a570a5e::fmt[0]::write[0] - [Crux] Translation error in core/9a570a5e::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/9a570a5e::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/beffc212::fmt[0]::write[0] + [Crux] Translation error in core/beffc212::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/beffc212::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. @@ -925,16 +925,16 @@ FAIL (expected: Should panic, but doesn't) (0.05s) crux doesn't match oracle Use -p '/format_struct/' to rerun this test only. - format_array: FAIL (1.88s) + format_array: FAIL (1.84s) Compiling and running oracle program (0.13s) - Oracle output: true (1.75s) + Oracle output: true (1.71s) Crux output: failures: - ---- format_array/fa5f4b6c::crux_test[0] counterexamples ---- + ---- format_array/69f3b2e7::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/9a570a5e::fmt[0]::write[0] - [Crux] Translation error in core/9a570a5e::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/9a570a5e::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/beffc212::fmt[0]::write[0] + [Crux] Translation error in core/beffc212::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/beffc212::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. @@ -942,16 +942,16 @@ FAIL (expected: Should panic, but doesn't) (0.05s) crux doesn't match oracle Use -p '/format_array/' to rerun this test only. - format_int: FAIL (1.75s) + format_int: FAIL (1.71s) Compiling and running oracle program (0.13s) - Oracle output: true (1.61s) + Oracle output: true (1.58s) Crux output: failures: - ---- format_int/85a95142::crux_test[0] counterexamples ---- + ---- format_int/050dadd3::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/9a570a5e::fmt[0]::write[0] - [Crux] Translation error in core/9a570a5e::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/9a570a5e::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/beffc212::fmt[0]::write[0] + [Crux] Translation error in core/beffc212::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/beffc212::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. @@ -959,16 +959,16 @@ FAIL (expected: Should panic, but doesn't) (0.05s) crux doesn't match oracle Use -p '/format_int/' to rerun this test only. - format: FAIL (1.72s) + format: FAIL (1.75s) Compiling and running oracle program (0.13s) - Oracle output: true (1.59s) + Oracle output: true (1.62s) Crux output: failures: - ---- format/5293190a::crux_test[0] counterexamples ---- + ---- format/1ac747dd::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/9a570a5e::fmt[0]::write[0] - [Crux] Translation error in core/9a570a5e::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/9a570a5e::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/beffc212::fmt[0]::write[0] + [Crux] Translation error in core/beffc212::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/beffc212::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. @@ -976,16 +976,16 @@ FAIL (expected: Should panic, but doesn't) (0.05s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..str.format"' to rerun this test only. - format_hex: FAIL (1.74s) + format_hex: FAIL (1.78s) Compiling and running oracle program (0.13s) - Oracle output: true (1.61s) + Oracle output: true (1.65s) Crux output: failures: - ---- format_hex/314ca526::crux_test[0] counterexamples ---- + ---- format_hex/23d72fd8::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/9a570a5e::fmt[0]::write[0] - [Crux] Translation error in core/9a570a5e::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/9a570a5e::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/beffc212::fmt[0]::write[0] + [Crux] Translation error in core/beffc212::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/beffc212::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. @@ -993,16 +993,16 @@ FAIL (expected: Should panic, but doesn't) (0.05s) crux doesn't match oracle Use -p '/format_hex/' to rerun this test only. - string_push: FAIL (1.68s) + string_push: FAIL (1.64s) Compiling and running oracle program (0.13s) - Oracle output: true (1.55s) + Oracle output: true (1.51s) Crux output: failures: - ---- string_push/59079431::crux_test[0] counterexamples ---- + ---- string_push/210232c5::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1014,7 +1014,7 @@ FAIL (expected: Should panic, but doesn't) (0.05s) instant: FAIL (2.09s) Compiling and running oracle program (0.12s) Oracle output: () (1.97s) - standalone use of `dyn` is not supported: TyDynamic core/9a570a5e::any[0]::Erased[0]::_trait58656ccc69864acf[0] + standalone use of `dyn` is not supported: TyDynamic core/beffc212::any[0]::Erased[0]::_trait58656ccc69864acf[0] CallStack (from HasCallStack): error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy tyToRepr, called at src/Mir/TransTy.hs:213:7 in crucible-mir-0.1-inplace:Mir.TransTy @@ -1059,32 +1059,32 @@ FAIL (0.05s) failed to compile and run Use -p '/arg2/' to rerun this test only. - field_order: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.85s) + field_order: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.86s) Crux output: () - ret: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: (A(42), B { x: 42 }, C) (0.87s) + ret: OK (0.99s) + Compiling and running oracle program (0.12s) + Oracle output: (A(42), B { x: 42 }, C) (0.86s) Crux output: (A(42), B { x: 42 }, C) - eq: OK (1.06s) - Compiling and running oracle program (0.14s) - Oracle output: () (0.91s) + eq: OK (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.88s) Crux output: () - inner: OK (1.04s) - Compiling and running oracle program (0.14s) - Oracle output: 4 (0.91s) + inner: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 4 (0.85s) Crux output: 4 - cow: FAIL (1.21s) + cow: FAIL (1.11s) Compiling and running oracle program (0.13s) - Oracle output: 200 (1.08s) + Oracle output: 200 (0.98s) Crux output: failures: - ---- cow/daae3f2d::crux_test[0] counterexamples ---- + ---- cow/f92ce4f1::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1092,161 +1092,161 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/cow/' to rerun this test only. - cmp: OK (1.01s) + cmp: OK (0.98s) Compiling and running oracle program (0.13s) - Oracle output: -23 (0.88s) + Oracle output: -23 (0.85s) Crux output: -23 - match: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.87s) + match: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 42 (0.84s) Crux output: 42 arg: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 0 (0.84s) + Compiling and running oracle program (0.12s) + Oracle output: 0 (0.85s) Crux output: 0 mixed_discrs: OK (0.97s) Compiling and running oracle program (0.13s) Oracle output: () (0.84s) Crux output: () sync - arc: OK (1.13s) + arc: OK (1.12s) Compiling and running oracle program (0.13s) - Oracle output: 1 (1.00s) + Oracle output: 1 (0.99s) Crux output: 1 - atomic_swap: OK (1.00s) + atomic_swap: OK (1.03s) Compiling and running oracle program (0.12s) - Oracle output: (2, 1) (0.87s) + Oracle output: (2, 1) (0.90s) Crux output: (2, 1) - rwlock_multi: FAIL (1.52s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.39s) - user error (JSON Decoding of test/conc_eval/sync/rwlock_multi.all.mir failed: Error in $.statics[165].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) + rwlock_multi: FAIL (1.55s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (1.42s) + user error (JSON Decoding of test/conc_eval/sync/rwlock_multi.all.mir failed: Error in $.statics[162].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) Use -p '/rwlock_multi/' to rerun this test only. - rwlock: FAIL (1.50s) + rwlock: FAIL (1.60s) Compiling and running oracle program (0.13s) - Oracle output: 2 (1.37s) - user error (JSON Decoding of test/conc_eval/sync/rwlock.all.mir failed: Error in $.statics[165].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) + Oracle output: 2 (1.47s) + user error (JSON Decoding of test/conc_eval/sync/rwlock.all.mir failed: Error in $.statics[162].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) Use -p '$0=="crux-mir.crux concrete..sync.rwlock"' to rerun this test only. - atomic_fence: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.88s) + atomic_fence: OK (1.03s) + Compiling and running oracle program (0.12s) + Oracle output: 2 (0.91s) Crux output: 2 - mutex_multi: FAIL (1.51s) + mutex_multi: FAIL (1.45s) Compiling and running oracle program (0.13s) - Oracle output: 3 (1.38s) - user error (JSON Decoding of test/conc_eval/sync/mutex_multi.all.mir failed: Error in $.statics[163].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) + Oracle output: 3 (1.32s) + user error (JSON Decoding of test/conc_eval/sync/mutex_multi.all.mir failed: Error in $.statics[161].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) Use -p '/mutex_multi/' to rerun this test only. - atomic_add: OK (1.06s) + atomic_add: OK (1.09s) Compiling and running oracle program (0.13s) - Oracle output: 6 (0.94s) + Oracle output: 6 (0.96s) Crux output: 6 - arc_clone: OK (1.14s) + arc_clone: OK (1.18s) Compiling and running oracle program (0.13s) - Oracle output: 2 (1.01s) + Oracle output: 2 (1.05s) Crux output: 2 - arc_cell: OK (1.16s) + arc_cell: OK (1.18s) Compiling and running oracle program (0.13s) - Oracle output: 4 (1.03s) + Oracle output: 4 (1.05s) Crux output: 4 - mutex: FAIL (1.46s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (1.33s) - user error (JSON Decoding of test/conc_eval/sync/mutex.all.mir failed: Error in $.statics[163].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) + mutex: FAIL (1.60s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.46s) + user error (JSON Decoding of test/conc_eval/sync/mutex.all.mir failed: Error in $.statics[161].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) Use -p '$0=="crux-mir.crux concrete..sync.mutex"' to rerun this test only. - atomic_cxchg: OK (1.10s) - Compiling and running oracle program (0.13s) - Oracle output: 6 (0.97s) + atomic_cxchg: OK (1.09s) + Compiling and running oracle program (0.14s) + Oracle output: 6 (0.95s) Crux output: 6 intTest - test0038: OK (1.07s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.95s) + test0038: OK (1.06s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.93s) Crux output: () test0039: OK (0.96s) Compiling and running oracle program (0.12s) - Oracle output: 7 (0.83s) + Oracle output: 7 (0.84s) Crux output: 7 stdlib - result_interior: OK (0.98s) + result_interior: OK (0.97s) Compiling and running oracle program (0.12s) Oracle output: 3 (0.85s) Crux output: 3 - option3: OK (0.95s) + option3: OK (0.99s) Compiling and running oracle program (0.12s) - Oracle output: 27 (0.82s) + Oracle output: 27 (0.87s) Crux output: 27 - result: OK (0.98s) + result: OK (0.97s) Compiling and running oracle program (0.13s) - Oracle output: 27 (0.86s) + Oracle output: 27 (0.84s) Crux output: 27 poly: OK (0.97s) Compiling and running oracle program (0.13s) Oracle output: 1 (0.84s) Crux output: 1 - default_impl: OK (0.95s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.83s) + default_impl: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.84s) Crux output: () - cvt: OK (0.96s) + cvt: OK (0.99s) Compiling and running oracle program (0.13s) - Oracle output: 0 (0.84s) + Oracle output: 0 (0.86s) Crux output: 0 - default: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: true (0.85s) - Crux output: true - option: OK (0.97s) + default: OK (0.98s) Compiling and running oracle program (0.13s) Oracle output: true (0.85s) Crux output: true - option2: OK (0.96s) + option: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: true (0.84s) + Crux output: true + option2: OK (0.97s) Compiling and running oracle program (0.13s) - Oracle output: 0 (0.83s) + Oracle output: 0 (0.84s) Crux output: 0 - teq: OK (0.97s) + teq: OK (0.98s) Compiling and running oracle program (0.13s) Oracle output: false (0.85s) Crux output: false - range: OK (0.98s) + range: OK (0.97s) Compiling and running oracle program (0.12s) Oracle output: 10 (0.85s) Crux output: 10 hash_map - insert_get: OK (1.32s) + insert_get: OK (1.30s) Compiling and running oracle program (0.15s) - Oracle output: (11, 12) (1.17s) + Oracle output: (11, 12) (1.14s) Crux output: (11, 12) insert_iter: OK (1.31s) - Compiling and running oracle program (0.15s) + Compiling and running oracle program (0.16s) Oracle output: [11, 12] (1.15s) Crux output: [11, 12] - insert_multi: OK (1.32s) + insert_multi: OK (1.35s) Compiling and running oracle program (0.15s) - Oracle output: 100 (1.17s) + Oracle output: 100 (1.20s) Crux output: 100 - insert_remove: OK (1.76s) + insert_remove: OK (1.80s) Compiling and running oracle program (0.15s) - Oracle output: 12 (1.61s) + Oracle output: 12 (1.65s) Crux output: 12 tuple - clone: OK (1.01s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.88s) + clone: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.87s) Crux output: () - clone_from: FAIL (1.00s) + clone_from: FAIL (1.02s) Compiling and running oracle program (0.13s) - Oracle output: () (0.88s) + Oracle output: () (0.89s) Crux output: failures: - ---- clone_from/b523f2dc::f[0] counterexamples ---- + ---- clone_from/44ff9844::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/clone.rs:136:17: 136:31: error: in core/9a570a5e::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0] - [Crux] Translation error in core/9a570a5e::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0]: don't know how to generate CloneShim for unknown method core/9a570a5e::clone[0]::Clone[0]::clone[0] + [Crux] ./lib/core/src/clone.rs:136:17: 136:31: error: in core/beffc212::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0] + [Crux] Translation error in core/beffc212::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0]: don't know how to generate CloneShim for unknown method core/beffc212::clone[0]::Clone[0]::clone[0] [Crux] Overall status: Invalid. @@ -1254,20 +1254,20 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/clone_from/' to rerun this test only. - clone_rec: OK (0.98s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.85s) + clone_rec: OK (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.88s) Crux output: () - clone_struct: FAIL (1.00s) + clone_struct: FAIL (1.02s) Compiling and running oracle program (0.13s) - Oracle output: () (0.87s) + Oracle output: () (0.90s) Crux output: failures: - ---- clone_struct/e620880a::f[0] counterexamples ---- + ---- clone_struct/473e1c76::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/tuple/clone_struct.rs:8:13: 8:22: error: in clone_struct/e620880a::f[0] - [Crux] Translation error in clone_struct/e620880a::f[0]: don't know how to generate CloneShim for unknown method core/9a570a5e::clone[0]::Clone[0]::clone[0] + [Crux] test/conc_eval/tuple/clone_struct.rs:8:13: 8:22: error: in clone_struct/473e1c76::f[0] + [Crux] Translation error in clone_struct/473e1c76::f[0]: don't know how to generate CloneShim for unknown method core/beffc212::clone[0]::Clone[0]::clone[0] [Crux] Overall status: Invalid. @@ -1276,49 +1276,49 @@ FAIL (0.05s) Use -p '/clone_struct/' to rerun this test only. consts - struct_val: OK (0.96s) - Compiling and running oracle program (0.12s) - Oracle output: Foo { x: true } (0.84s) + struct_val: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: Foo { x: true } (0.85s) Crux output: Foo { x: true } - struct_unit: OK (0.97s) + struct_unit: OK (1.00s) Compiling and running oracle program (0.13s) - Oracle output: Err(Foo { x: () }) (0.85s) + Oracle output: Err(Foo { x: () }) (0.87s) Crux output: Err(Foo { x: () }) - enum_val: OK (0.96s) - Compiling and running oracle program (0.12s) - Oracle output: None (0.83s) - Crux output: None - local_key: FAIL (0.99s) + enum_val: OK (1.00s) Compiling and running oracle program (0.13s) - Oracle output: 1 (0.87s) + Oracle output: None (0.87s) + Crux output: None + local_key: FAIL (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (0.85s) user error (JSON Decoding of test/conc_eval/consts/local_key.all.mir failed: Error in $.statics[0].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) Use -p '/local_key/' to rerun this test only. - fn_def: FAIL (0.95s) + fn_def: FAIL (1.00s) Compiling and running oracle program (0.12s) - Oracle output: 1 (0.83s) + Oracle output: 1 (0.87s) user error (JSON Decoding of test/conc_eval/consts/fn_def.all.mir failed: Error in $.fns[0].body.blocks[0].block.terminator.func.data.rendered: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) Use -p '/fn_def/' to rerun this test only. array - iter: OK (1.05s) + iter: OK (1.04s) Compiling and running oracle program (0.13s) - Oracle output: 3 (0.92s) + Oracle output: 3 (0.91s) Crux output: 3 - const_impl: OK (0.99s) - Compiling and running oracle program (0.12s) - Oracle output: 5 (0.86s) + const_impl: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 5 (0.85s) Crux output: 5 clone: FAIL (1.01s) Compiling and running oracle program (0.13s) - Oracle output: () (0.88s) + Oracle output: () (0.89s) Crux output: failures: - ---- clone/c1f28444::f[0] counterexamples ---- + ---- clone/43838f2d::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:46:9: 46:18: error: in core/9a570a5e::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0]: unimplemented cast: Misc + [Crux] ./lib/core/src/ptr/const_ptr.rs:46:9: 46:18: error: in core/beffc212::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0] + [Crux] Translation error in core/beffc212::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyInt B32) Immut [Crux] as: TyRawPtr (TyArray (TyInt B32) 3) Immut @@ -1328,16 +1328,16 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/array.clone/' to rerun this test only. - mut_index: FAIL (1.49s) - Compiling and running oracle program (0.12s) - Oracle output: 7 (1.37s) + mut_index: FAIL (1.53s) + Compiling and running oracle program (0.13s) + Oracle output: 7 (1.40s) Crux output: failures: - ---- mut_index/92366878::crux_test[0] counterexamples ---- + ---- mut_index/209c678d::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1345,20 +1345,20 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/mut_index/' to rerun this test only. - mut_arg: OK (0.98s) + mut_arg: OK (0.99s) Compiling and running oracle program (0.12s) - Oracle output: 42 (0.86s) + Oracle output: 42 (0.87s) Crux output: 42 - from_slice: FAIL (1.62s) + from_slice: FAIL (1.63s) Compiling and running oracle program (0.13s) - Oracle output: () (1.49s) + Oracle output: () (1.50s) Crux output: failures: - ---- from_slice/47fdae33::f[0] counterexamples ---- + ---- from_slice/09f81d88::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/array/mod.rs:266:23: 266:54: error: in core/9a570a5e::array[0]::{impl#10}[0]::try_from[0]::_inste494657dbbea865b[0] - [Crux] Translation error in core/9a570a5e::array[0]::{impl#10}[0]::try_from[0]::_inste494657dbbea865b[0]: unimplemented cast: Misc + [Crux] ./lib/core/src/array/mod.rs:266:23: 266:54: error: in core/beffc212::array[0]::{impl#10}[0]::try_from[0]::_inste494657dbbea865b[0] + [Crux] Translation error in core/beffc212::array[0]::{impl#10}[0]::try_from[0]::_inste494657dbbea865b[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyUint B8) Immut [Crux] as: TyRawPtr (TyArray (TyUint B8) 4) Immut @@ -1368,43 +1368,43 @@ FAIL (0.05s) crux doesn't match oracle Use -p '/array.from_slice/' to rerun this test only. - const: OK (0.95s) - Compiling and running oracle program (0.12s) - Oracle output: 1 (0.83s) + const: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.84s) Crux output: 1 - wick2: OK (0.98s) + wick2: OK (0.99s) Compiling and running oracle program (0.13s) - Oracle output: true (0.85s) + Oracle output: true (0.86s) Crux output: true - wick1: FAIL (expected: needs Vec data structure from stdlib) (0.18s) + wick1: FAIL (expected: needs Vec data structure from stdlib) (0.17s) Compiling and running oracle program (0.14s) Oracle output: true (0.04s) user error (Error 101 while running mir-json on test/conc_eval/array/wick1.rs) (expected failure) - arg: OK (0.96s) + arg: OK (0.98s) Compiling and running oracle program (0.12s) - Oracle output: 2 (0.84s) + Oracle output: 2 (0.86s) Crux output: 2 - mk_and_proj: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 42 (0.84s) - Crux output: 42 - wick3: FAIL (expected: needs Vec data structure from stdlib) (0.17s) + mk_and_proj: OK (0.99s) Compiling and running oracle program (0.13s) + Oracle output: 42 (0.86s) + Crux output: 42 + wick3: FAIL (expected: needs Vec data structure from stdlib) (0.18s) + Compiling and running oracle program (0.14s) Oracle output: true (0.04s) user error (Error 101 while running mir-json on test/conc_eval/array/wick3.rs) (expected failure) mem - maybe_uninit_array_cast: FAIL (0.99s) - Compiling and running oracle program (0.12s) - Oracle output: [1, 2] (0.86s) + maybe_uninit_array_cast: FAIL (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: [1, 2] (0.87s) Crux output: failures: - ---- maybe_uninit_array_cast/9824cebb::crux_test[0] counterexamples ---- + ---- maybe_uninit_array_cast/5024e15a::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in maybe_uninit_array_cast/9824cebb::crux_test[0] - [Crux] Translation error in maybe_uninit_array_cast/9824cebb::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyArray (TyInt B32) 2]) + [Crux] internal: error: in maybe_uninit_array_cast/5024e15a::crux_test[0] + [Crux] Translation error in maybe_uninit_array_cast/5024e15a::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyArray (TyInt B32) 2]) [Crux] Overall status: Invalid. @@ -1414,14 +1414,14 @@ FAIL (0.05s) Use -p '/maybe_uninit_array_cast/' to rerun this test only. maybe_uninit: FAIL (0.99s) Compiling and running oracle program (0.13s) - Oracle output: 1 (0.86s) + Oracle output: 1 (0.87s) Crux output: failures: - ---- maybe_uninit/de6620f2::crux_test[0] counterexamples ---- + ---- maybe_uninit/4270f58e::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in maybe_uninit/de6620f2::crux_test[0] - [Crux] Translation error in maybe_uninit/de6620f2::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyInt B32]) + [Crux] internal: error: in maybe_uninit/4270f58e::crux_test[0] + [Crux] Translation error in maybe_uninit/4270f58e::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyInt B32]) [Crux] Overall status: Invalid. @@ -1430,36 +1430,36 @@ FAIL (0.05s) Use -p '$0=="crux-mir.crux concrete..mem.maybe_uninit"' to rerun this test only. ops - index2: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.85s) - Crux output: () - deref1: OK (0.98s) + index2: OK (0.99s) Compiling and running oracle program (0.12s) - Oracle output: () (0.86s) + Oracle output: () (0.87s) Crux output: () - index1: OK (0.97s) + deref1: OK (0.97s) Compiling and running oracle program (0.12s) Oracle output: () (0.85s) Crux output: () - deref3: OK (0.97s) - Compiling and running oracle program (0.12s) + index1: OK (1.02s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.89s) + Crux output: () + deref3: OK (0.98s) + Compiling and running oracle program (0.13s) Oracle output: () (0.85s) Crux output: () - arith1: OK (0.99s) + arith1: OK (1.03s) Compiling and running oracle program (0.13s) - Oracle output: () (0.86s) + Oracle output: () (0.90s) Crux output: () - deref2: OK (0.96s) + deref2: OK (0.99s) Compiling and running oracle program (0.12s) - Oracle output: () (0.84s) + Oracle output: () (0.87s) Crux output: () - index3: OK (0.97s) + index3: OK (1.01s) Compiling and running oracle program (0.13s) - Oracle output: () (0.84s) + Oracle output: () (0.87s) Crux output: () fnptr - field: OK (0.97s) + field: OK (0.98s) Compiling and running oracle program (0.12s) Oracle output: 2 (0.85s) Crux output: 2 @@ -1482,19 +1482,19 @@ FAIL (expected: taking address of an overridden function) (0.05s) test/Test.hs:107: failed to compile and run (expected failure) - make: OK (0.98s) + make: OK (1.00s) Compiling and running oracle program (0.12s) - Oracle output: 0 (0.86s) + Oracle output: 0 (0.87s) Crux output: 0 call: OK (0.97s) - Compiling and running oracle program (0.12s) + Compiling and running oracle program (0.13s) Oracle output: 2 (0.85s) Crux output: 2 io - cursor_write2: FAIL (2.05s) + cursor_write2: FAIL (2.07s) Compiling and running oracle program (0.14s) - Oracle output: 0 (1.92s) - standalone use of `dyn` is not supported: TyDynamic core/9a570a5e::any[0]::Erased[0]::_trait58656ccc69864acf[0] + Oracle output: 0 (1.93s) + standalone use of `dyn` is not supported: TyDynamic core/beffc212::any[0]::Erased[0]::_trait58656ccc69864acf[0] CallStack (from HasCallStack): error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy tyToRepr, called at src/Mir/TransTy.hs:213:7 in crucible-mir-0.1-inplace:Mir.TransTy @@ -1510,16 +1510,16 @@ FAIL (expected: taking address of an overridden function) (0.05s) translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '/cursor_write2/' to rerun this test only. - cursor_read: FAIL (1.64s) + cursor_read: FAIL (1.65s) Compiling and running oracle program (0.14s) Oracle output: 0 (1.51s) Crux output: failures: - ---- cursor_read/aadc00f0::crux_test[0] counterexamples ---- + ---- cursor_read/a3a3296c::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1527,16 +1527,16 @@ FAIL (expected: taking address of an overridden function) (0.05s) crux doesn't match oracle Use -p '/io.cursor_read/' to rerun this test only. - cursor_write: FAIL (1.67s) + cursor_write: FAIL (1.69s) Compiling and running oracle program (0.14s) - Oracle output: 0 (1.53s) + Oracle output: 0 (1.55s) Crux output: failures: - ---- cursor_write/2389e4f2::crux_test[0] counterexamples ---- + ---- cursor_write/0e9225ad::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. @@ -1545,29 +1545,29 @@ FAIL (expected: taking address of an overridden function) (0.05s) Use -p '$0=="crux-mir.crux concrete..io.cursor_write"' to rerun this test only. cell - ref_cell2: OK (1.56s) + ref_cell2: OK (1.58s) Compiling and running oracle program (0.13s) - Oracle output: 2 (1.43s) + Oracle output: 2 (1.45s) Crux output: 2 - cell: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 2 (0.85s) + cell: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.87s) Crux output: 2 - ref_cell: OK (1.60s) + ref_cell: OK (1.56s) Compiling and running oracle program (0.13s) - Oracle output: 2 (1.47s) + Oracle output: 2 (1.44s) Crux output: 2 struct - repr_transparent: FAIL (1.04s) + repr_transparent: FAIL (1.03s) Compiling and running oracle program (0.13s) - Oracle output: 6 (0.91s) + Oracle output: 6 (0.90s) Crux output: failures: - ---- repr_transparent/f9a814ae::crux_test[0] counterexamples ---- + ---- repr_transparent/ac707a40::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/const_ptr.rs:46:9: 46:18: error: in core/9a570a5e::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0]: unimplemented cast: Misc + [Crux] ./lib/core/src/ptr/const_ptr.rs:46:9: 46:18: error: in core/beffc212::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0] + [Crux] Translation error in core/beffc212::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0]: unimplemented cast: Misc [Crux] ty: TyRawPtr (TyInt B32) Immut [Crux] as: TyRawPtr (TyArray (TyInt B32) 2) Immut @@ -1577,501 +1577,297 @@ FAIL (expected: taking address of an overridden function) (0.05s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..struct.repr_transparent"' to rerun this test only. - tup: OK (0.95s) + tup: OK (0.97s) Compiling and running oracle program (0.12s) - Oracle output: () (0.83s) + Oracle output: () (0.84s) Crux output: () - repr_transparent_const: OK (0.98s) + repr_transparent_const: OK (1.00s) Compiling and running oracle program (0.13s) - Oracle output: 124 (0.86s) + Oracle output: 124 (0.87s) Crux output: 124 field_order: OK (0.98s) Compiling and running oracle program (0.12s) Oracle output: () (0.86s) Crux output: () - ret: OK (0.96s) + ret: OK (0.97s) Compiling and running oracle program (0.13s) - Oracle output: S { x: 42, y: 120 } (0.83s) + Oracle output: S { x: 42, y: 120 } (0.84s) Crux output: S { x: 42, y: 120 } - proj: OK (0.96s) + proj: OK (0.97s) Compiling and running oracle program (0.13s) - Oracle output: 42 (0.83s) + Oracle output: 42 (0.84s) Crux output: 42 - arg: OK (0.97s) + arg: OK (0.98s) Compiling and running oracle program (0.13s) - Oracle output: 42 (0.84s) + Oracle output: 42 (0.86s) Crux output: 42 refs - temp: OK (0.97s) + temp: OK (0.99s) Compiling and running oracle program (0.12s) - Oracle output: 1 (0.85s) + Oracle output: 1 (0.87s) Crux output: 1 - static_mut: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.85s) - Crux output: 2 - imm_raw: OK (0.97s) + static_mut: OK (0.99s) Compiling and running oracle program (0.12s) - Oracle output: 123 (0.85s) + Oracle output: 2 (0.86s) + Crux output: 2 + imm_raw: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 123 (0.84s) Crux output: 123 mut_tuple_field: OK (0.99s) Compiling and running oracle program (0.12s) - Oracle output: () (0.86s) + Oracle output: () (0.87s) Crux output: () - mut_ref: OK (0.96s) + mut_ref: OK (0.99s) Compiling and running oracle program (0.12s) - Oracle output: 1 (0.83s) + Oracle output: 1 (0.87s) Crux output: 1 - mut_arg: OK (0.95s) + mut_arg: OK (0.99s) Compiling and running oracle program (0.12s) - Oracle output: 1 (0.83s) + Oracle output: 1 (0.86s) Crux output: 1 never: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.85s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (0.86s) Crux output: 1 - mut_raw: OK (0.96s) + mut_raw: OK (0.97s) Compiling and running oracle program (0.12s) - Oracle output: 123 (0.84s) + Oracle output: 123 (0.85s) Crux output: 123 - promoted_imm: OK (0.98s) - Compiling and running oracle program (0.12s) - Oracle output: 0 (0.86s) + promoted_imm: OK (0.99s) + Compiling and running oracle program (0.13s) + Oracle output: 0 (0.87s) Crux output: 0 - mut_nested: OK (0.98s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.86s) + mut_nested: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.88s) Crux output: () - imm_ref: OK (0.95s) - Compiling and running oracle program (0.12s) - Oracle output: 123 (0.83s) + imm_ref: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 123 (0.85s) Crux output: 123 - fn_ptr_mut: OK (0.97s) + fn_ptr_mut: OK (0.98s) Compiling and running oracle program (0.13s) - Oracle output: 1 (0.84s) + Oracle output: 1 (0.85s) Crux output: 1 - promoted_mut: OK (0.98s) - Compiling and running oracle program (0.13s) + promoted_mut: OK (0.97s) + Compiling and running oracle program (0.12s) Oracle output: 0 (0.85s) Crux output: 0 - imm_arg: OK (0.95s) - Compiling and running oracle program (0.12s) - Oracle output: 1 (0.83s) - Crux output: 1 - fn_ptr: OK (0.96s) + imm_arg: OK (0.96s) Compiling and running oracle program (0.12s) Oracle output: 1 (0.84s) Crux output: 1 - never_mut: OK (0.97s) + fn_ptr: OK (0.99s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.87s) + Crux output: 1 + never_mut: OK (0.98s) Compiling and running oracle program (0.12s) - Oracle output: 1 (0.84s) + Oracle output: 1 (0.86s) Crux output: 1 crux symbolic Output testing - macro: FAIL (0.99s) + macro: FAIL (1.03s) files test/symb_eval/vec/macro.good and test/symb_eval/vec/macro.out differ; test/symb_eval/vec/macro.out contains: - test macro/4fbe54d8::f[0]: FAILED + test macro/9050e48f::f[0]: FAILED failures: - ---- macro/4fbe54d8::f[0] counterexamples ---- + ---- macro/9050e48f::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI + [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. Use -p '/Output testing.macro/' to rerun this test only. - clone: FAIL (0.98s) + clone: FAIL (1.00s) files test/symb_eval/vec/clone.good and test/symb_eval/vec/clone.out differ; test/symb_eval/vec/clone.out contains: - test clone/19414d1e::f[0]: FAILED + test clone/6db5b217::f[0]: FAILED failures: - ---- clone/19414d1e::f[0] counterexamples ---- + ---- clone/6db5b217::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI + [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. Use -p '/Output testing.clone/' to rerun this test only. - sort_by_key: FAIL (1.79s) + sort_by_key: FAIL (1.81s) files test/symb_eval/vec/sort_by_key.good and test/symb_eval/vec/sort_by_key.out differ; test/symb_eval/vec/sort_by_key.out contains: - test sort_by_key/c0695800::f[0]: FAILED + test sort_by_key/5a8cbf18::f[0]: FAILED failures: - ---- sort_by_key/c0695800::f[0] counterexamples ---- + ---- sort_by_key/5a8cbf18::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI + [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. Use -p '/sort_by_key/' to rerun this test only. - into_iter: FAIL (1.50s) + into_iter: FAIL (1.61s) files test/symb_eval/vec/into_iter.good and test/symb_eval/vec/into_iter.out differ; test/symb_eval/vec/into_iter.out contains: - test into_iter/23759b38::f[0]: FAILED + test into_iter/772ec3dc::f[0]: FAILED failures: - ---- into_iter/23759b38::f[0] counterexamples ---- + ---- into_iter/772ec3dc::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI + [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. Use -p '/into_iter/' to rerun this test only. - as_slice: OK (0.96s) - replicate: OK (0.97s) - split_at: OK (0.96s) - concat: OK (0.97s) + as_slice: OK (0.95s) + replicate: OK (0.93s) + split_at: OK (0.98s) + concat: OK (1.00s) copy_from_slice: OK (0.94s) new: OK (0.95s) - as_mut_slice: OK (0.91s) - mut: OK (0.96s) - push: OK (0.99s) - pop: FAIL (0.95s) + as_mut_slice: OK (0.98s) + mut: OK (0.95s) + push: OK (0.97s) + pop: FAIL (0.97s) files test/symb_eval/vector/pop.good and test/symb_eval/vector/pop.out differ; test/symb_eval/vector/pop.out contains: - test pop/30fddbda::f[0]: FAILED + test pop/d48bfd10::f[0]: FAILED failures: - ---- pop/30fddbda::f[0] counterexamples ---- + ---- pop/d48bfd10::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/vector/pop.rs:9:18: 9:19: error: in pop/30fddbda::f[0] - [Crux] Translation error in pop/30fddbda::f[0]: unknown ADT (core/3a1fbbbh::option[0]::Option[0],Substs [TyUint B8]) + [Crux] test/symb_eval/vector/pop.rs:9:18: 9:19: error: in pop/d48bfd10::f[0] + [Crux] Translation error in pop/d48bfd10::f[0]: unknown ADT (core/3a1fbbbh::option[0]::Option[0],Substs [TyUint B8]) [Crux] Overall status: Invalid. Use -p '/Output testing.pop/' to rerun this test only. - checked_div: OK (0.90s) - checked_add: OK (0.90s) - checked_mul_signed: OK (0.93s) - checked_mul: OK (0.85s) + checked_div: OK (0.86s) + checked_add: OK (0.83s) + checked_mul_signed: OK (0.87s) + checked_mul: OK (0.84s) array_mut: OK (1.01s) - array: OK (0.97s) - bad_symb1: OK (0.96s) - override5: FAIL (0.97s) - files test/symb_eval/overrides/override5.good and test/symb_eval/overrides/override5.out differ; test/symb_eval/overrides/override5.out contains: - test override5/bcc96f90::f[0]: FAILED - - failures: - - ---- override5/bcc96f90::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/overrides/override5.rs:10:5: 10:47: error: in override5/bcc96f90::f[0] - [Crux] MIR assertion at test/symb_eval/overrides/override5.rs:10:5: - [Crux] foo.wrapping_add(1) != 0 - - [Crux] Overall status: Invalid. - - Use -p '/override5/' to rerun this test only. - override3: OK (0.97s) - override_rust: OK (0.95s) - override1: OK (0.86s) - override2: FAIL (0.97s) - files test/symb_eval/overrides/override2.good and test/symb_eval/overrides/override2.out differ; test/symb_eval/overrides/override2.out contains: - test override2/3ba46755::f[0]: FAILED - - failures: - - ---- override2/3ba46755::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/overrides/override2.rs:9:5: 9:49: error: in override2/3ba46755::f[0] - [Crux] MIR assertion at test/symb_eval/overrides/override2.rs:9:5: - [Crux] foo.wrapping_add(1) == foo - - [Crux] Overall status: Invalid. - - Use -p '/override2/' to rerun this test only. - bad_symb2: OK (0.93s) - override4: OK (0.98s) - write: FAIL + array: OK (0.96s) + bad_symb1: OK (0.92s) + override5: OK (1.00s) + override3: OK (0.99s) + override_rust: OK (0.96s) + override1: OK (0.82s) + override2: OK (0.92s) + bad_symb2: OK (0.95s) + override4: OK (0.99s) + write: FAIL (0.05s) test/symb_eval/byteorder/write.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/write.rs) Use -p '/Output testing.write/' to rerun this test only. - read: FAIL (0.05s) + read: FAIL test/symb_eval/byteorder/read.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/read.rs) Use -p '/Output testing.read/' to rerun this test only. - deserialize: OK (0.98s) - construct: FAIL (0.98s) - files test/symb_eval/sym_bytes/construct.good and test/symb_eval/sym_bytes/construct.out differ; test/symb_eval/sym_bytes/construct.out contains: - test construct/64b2e785::crux_test[0]: returned Symbolic BV, FAILED - - failures: - - ---- construct/64b2e785::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/sym_bytes/construct.rs:13:5: 13:35: error: in construct/64b2e785::crux_test[0] - [Crux] MIR assertion at test/symb_eval/sym_bytes/construct.rs:13:5: - [Crux] sym2[0] == 0 - - [Crux] Overall status: Invalid. - - Use -p '/construct/' to rerun this test only. - double: OK (0.95s) - bytes: FAIL (1.04s) - files test/symb_eval/crypto/bytes.good and test/symb_eval/crypto/bytes.out differ; test/symb_eval/crypto/bytes.out contains: - test bytes/3d5d9475::f[0]: FAILED - - failures: - - ---- bytes/3d5d9475::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/3d5d9475::f[0] - [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: - [Crux] a[i] == b[i] - [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/3d5d9475::f[0] - [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: - [Crux] a[i] == b[i] - [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/3d5d9475::f[0] - [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: - [Crux] a[i] == b[i] - [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/3d5d9475::f[0] - [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: - [Crux] a[i] == b[i] - [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/3d5d9475::f[0] - [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: - [Crux] a[i] == b[i] - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.bytes"' to rerun this test only. - ffs: OK (1.02s) - bytes2: OK (1.10s) - from_to: OK (0.84s) - overflowing_sub: OK (0.99s) - literals: OK (0.83s) + deserialize: OK (1.04s) + construct: OK (1.00s) + double: OK (0.93s) + bytes: OK (1.08s) + ffs: OK (1.03s) + bytes2: OK (1.07s) + from_to: OK (0.85s) + overflowing_sub: OK (0.98s) + literals: OK (0.84s) arith: OK (1.03s) - leading_zeros: FAIL (0.99s) - files test/symb_eval/bitvector/leading_zeros.good and test/symb_eval/bitvector/leading_zeros.out differ; test/symb_eval/bitvector/leading_zeros.out contains: - test leading_zeros/85b8f496::crux_test[0]: FAILED - - failures: - - ---- leading_zeros/85b8f496::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/uint_macros.rs:131:30: 131:34 !/home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/core/src/num/mod.rs:927:5: 931:16: error: in core/9a570a5e::num[0]::{impl#10}[0]::leading_zeros[0] - [Crux] Translation error in core/9a570a5e::num[0]::{impl#10}[0]::leading_zeros[0]: callExp: Don't know how to call core/9a570a5e::intrinsics[0]::{extern#0}[0]::ctlz[0]::_inst11a9dd2a3cfdbd73[0] - - [Crux] Overall status: Invalid. - - Use -p '/leading_zeros/' to rerun this test only. - cmp: OK (0.98s) - symbolic: OK (0.87s) - mux: OK (0.99s) - test1: OK (101.62s) - split_off: FAIL (1.37s) + leading_zeros: OK (1.04s) + cmp: OK (0.96s) + symbolic: OK (0.84s) + mux: OK (0.98s) + test1: OK (102.62s) + split_off: FAIL (1.38s) files test/symb_eval/bytes/split_off.good and test/symb_eval/bytes/split_off.out differ; test/symb_eval/bytes/split_off.out contains: - test split_off/17077784::f[0]: FAILED + test split_off/72b60843::f[0]: FAILED failures: - ---- split_off/17077784::f[0] counterexamples ---- + ---- split_off/72b60843::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. Use -p '/split_off/' to rerun this test only. - split_to: FAIL (1.45s) + split_to: FAIL (1.36s) files test/symb_eval/bytes/split_to.good and test/symb_eval/bytes/split_to.out differ; test/symb_eval/bytes/split_to.out contains: - test split_to/2da81cd0::f[0]: FAILED + test split_to/c31c803f::f[0]: FAILED failures: - ---- split_to/2da81cd0::f[0] counterexamples ---- + ---- split_to/c31c803f::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. Use -p '/split_to/' to rerun this test only. - new: OK (0.91s) + new: OK (0.96s) put: FAIL (1.45s) files test/symb_eval/bytes/put.good and test/symb_eval/bytes/put.out differ; test/symb_eval/bytes/put.out contains: - test put/6bf8bf24::f[0]: FAILED + test put/d816ba96::f[0]: FAILED failures: - ---- put/6bf8bf24::f[0] counterexamples ---- + ---- put/d816ba96::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.put"' to rerun this test only. - sym_len: FAIL (1.49s) + sym_len: FAIL (1.51s) files test/symb_eval/bytes/sym_len.good and test/symb_eval/bytes/sym_len.out differ; test/symb_eval/bytes/sym_len.out contains: - test sym_len/94f46a68::f[0]: FAILED + test sym_len/006a1393::f[0]: FAILED failures: - ---- sym_len/94f46a68::f[0] counterexamples ---- + ---- sym_len/006a1393::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/9a570a5e::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] + [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. Use -p '/sym_len/' to rerun this test only. - extend_bytes: OK (0.96s) - put_overflow: FAIL (1.35s) - files test/symb_eval/bytes/put_overflow.good and test/symb_eval/bytes/put_overflow.out differ; test/symb_eval/bytes/put_overflow.out contains: - test put_overflow/793a6d43::f[0]: FAILED - - failures: - - ---- put_overflow/793a6d43::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in bytes/e3cb607d::{impl#4}[0]::put_slice[0] - [Crux] panicking::panic_fmt, called from bytes/e3cb607d::{impl#4}[0]::put_slice[0] - - [Crux] Overall status: Invalid. - - Use -p '/put_overflow/' to rerun this test only. - multi: FAIL (0.96s) - files test/symb_eval/crux/multi.good and test/symb_eval/crux/multi.out differ; test/symb_eval/crux/multi.out contains: - test multi/42855c12::fail1[0]: FAILED - test multi/42855c12::fail2[0]: FAILED - test multi/42855c12::fail3[0]: FAILED - - failures: - - ---- multi/42855c12::fail1[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/crux/multi.rs:8:22: 8:27: error: in multi/42855c12::fail1[0] - [Crux] attempt to compute `move _5 + const 1_u8`, which would overflow - [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/multi.rs:8:5: 8:32: error: in multi/42855c12::fail1[0] - [Crux] MIR assertion at test/symb_eval/crux/multi.rs:8:5: - [Crux] x + 1 > x - - ---- multi/42855c12::fail2[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in multi/42855c12::fail2[0] - [Crux] panicking::panic_fmt, called from multi/42855c12::fail2[0] - - ---- multi/42855c12::fail3[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/multi.rs:20:5: 20:29: error: in multi/42855c12::assert_zero[0] - [Crux] MIR assertion at test/symb_eval/crux/multi.rs:20:5: - [Crux] x == 0 - - [Crux] Overall status: Invalid. - - Use -p '/Output testing.multi/' to rerun this test only. - fail_return: FAIL (0.98s) - files test/symb_eval/crux/fail_return.good and test/symb_eval/crux/fail_return.out differ; test/symb_eval/crux/fail_return.out contains: - test fail_return/c4d3092b::fail1[0]: returned Symbolic BV, FAILED - test fail_return/c4d3092b::fail2[0]: returned 123, FAILED - - failures: - - ---- fail_return/c4d3092b::fail1[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/crux/fail_return.rs:8:22: 8:27: error: in fail_return/c4d3092b::fail1[0] - [Crux] attempt to compute `move _4 + const 1_u8`, which would overflow - [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/fail_return.rs:8:5: 8:32: error: in fail_return/c4d3092b::fail1[0] - [Crux] MIR assertion at test/symb_eval/crux/fail_return.rs:8:5: - [Crux] x + 1 > x - - ---- fail_return/c4d3092b::fail2[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/crux/fail_return.rs:15:22: 15:27: error: in fail_return/c4d3092b::fail2[0] - [Crux] attempt to compute `move _5 + const 1_u8`, which would overflow - [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/fail_return.rs:15:5: 15:32: error: in fail_return/c4d3092b::fail2[0] - [Crux] MIR assertion at test/symb_eval/crux/fail_return.rs:15:5: - [Crux] x + 1 > x - - [Crux] Overall status: Invalid. - - Use -p '/fail_return/' to rerun this test only. - mixed_fail: FAIL (1.00s) - files test/symb_eval/crux/mixed_fail.good and test/symb_eval/crux/mixed_fail.out differ; test/symb_eval/crux/mixed_fail.out contains: - test mixed_fail/d59a8d8b::fail1[0]: FAILED - test mixed_fail/d59a8d8b::fail2[0]: FAILED - test mixed_fail/d59a8d8b::pass1[0]: ok - test mixed_fail/d59a8d8b::pass2[0]: ok - - failures: - - ---- mixed_fail/d59a8d8b::fail1[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/crux/mixed_fail.rs:8:22: 8:27: error: in mixed_fail/d59a8d8b::fail1[0] - [Crux] attempt to compute `move _5 + const 1_u8`, which would overflow - [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/mixed_fail.rs:8:5: 8:32: error: in mixed_fail/d59a8d8b::fail1[0] - [Crux] MIR assertion at test/symb_eval/crux/mixed_fail.rs:8:5: - [Crux] x + 1 > x - - ---- mixed_fail/d59a8d8b::fail2[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/crux/mixed_fail.rs:14:22: 14:27: error: in mixed_fail/d59a8d8b::fail2[0] - [Crux] attempt to compute `move _5 + const 2_u8`, which would overflow - [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/mixed_fail.rs:14:5: 14:32: error: in mixed_fail/d59a8d8b::fail2[0] - [Crux] MIR assertion at test/symb_eval/crux/mixed_fail.rs:14:5: - [Crux] x + 2 > x - - [Crux] Overall status: Invalid. - - Use -p '/mixed_fail/' to rerun this test only. - early_fail: FAIL (0.98s) - files test/symb_eval/crux/early_fail.good and test/symb_eval/crux/early_fail.out differ; test/symb_eval/crux/early_fail.out contains: - test early_fail/09aa9433::fail1[0]: FAILED - test early_fail/09aa9433::fail2[0]: FAILED - - failures: - - ---- early_fail/09aa9433::fail1[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in early_fail/09aa9433::fail1[0] - [Crux] panicking::panic_fmt, called from early_fail/09aa9433::fail1[0] - - ---- early_fail/09aa9433::fail2[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] /home/ryanglscott/Documents/Hacking/Haskell/sandbox/crucible/crux-mir/lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/early_fail.rs:17:5: 17:29: error: in early_fail/09aa9433::fail2[0] - [Crux] MIR assertion at test/symb_eval/crux/early_fail.rs:17:5: - [Crux] x == 0 - - [Crux] Overall status: Invalid. - - Use -p '/early_fail/' to rerun this test only. + extend_bytes: OK (1.00s) + put_overflow: OK (1.38s) + multi: OK (0.97s) + fail_return: OK (0.96s) + mixed_fail: OK (1.03s) + early_fail: OK (0.99s) basic: OK (0.99s) - slice: OK (1.02s) - slice_mut: OK (1.02s) - mux_slice: OK (1.02s) - downcast_fail: FAIL (0.93s) + slice: OK (1.09s) + slice_mut: OK (1.04s) + mux_slice: OK (1.01s) + downcast_fail: FAIL (0.96s) files test/symb_eval/any/downcast_fail.good and test/symb_eval/any/downcast_fail.out differ; test/symb_eval/any/downcast_fail.out contains: - test downcast_fail/474c9422::crux_test[0]: FAILED + test downcast_fail/a1506e51::crux_test[0]: FAILED failures: - ---- downcast_fail/474c9422::crux_test[0] counterexamples ---- + ---- downcast_fail/a1506e51::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/any/downcast_fail.rs:10:14: 10:15: error: in downcast_fail/474c9422::crux_test[0] + [Crux] test/symb_eval/any/downcast_fail.rs:10:14: 10:15: error: in downcast_fail/a1506e51::crux_test[0] [Crux] failed to downcast Any as BVRepr 32 [Crux] Overall status: Invalid. Use -p '/downcast_fail/' to rerun this test only. - downcast: OK (0.96s) - conditional: OK (0.90s) - mux: OK (0.91s) - vec_cursor_read: FAIL (1.92s) - standalone use of `dyn` is not supported: TyDynamic core/9a570a5e::any[0]::Erased[0]::_trait58656ccc69864acf[0] + downcast: OK (1.01s) + conditional: OK (0.98s) + mux: OK (0.93s) + vec_cursor_read: FAIL (1.94s) + standalone use of `dyn` is not supported: TyDynamic core/beffc212::any[0]::Erased[0]::_trait58656ccc69864acf[0] CallStack (from HasCallStack): error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy tyToRepr, called at src/Mir/TransTy.hs:213:7 in crucible-mir-0.1-inplace:Mir.TransTy @@ -2086,31 +1882,31 @@ FAIL (expected: taking address of an overridden function) (0.05s) transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '/vec_cursor_read/' to rerun this test only. - vec_write: OK (1.45s) - mux_init_mut: OK (0.95s) - mux_init_imm: OK (0.98s) - uninit_read: OK (0.99s) - valid_read: OK (0.98s) - zero_length: OK (0.94s) - out_of_bounds: OK (1.01s) - no_conc: OK (1.00s) + vec_write: OK (1.47s) + mux_init_mut: OK (0.99s) + mux_init_imm: OK (0.96s) + uninit_read: OK (0.91s) + valid_read: OK (0.96s) + zero_length: OK (0.91s) + out_of_bounds: OK (1.02s) + no_conc: OK (0.99s) assert: FAIL (1.64s) files test/symb_eval/concretize/assert.good and test/symb_eval/concretize/assert.out differ; test/symb_eval/concretize/assert.out contains: - test assert/58758234::crux_test[0]: returned 1, FAILED + test assert/e68faa4a::crux_test[0]: returned 1, FAILED failures: - ---- assert/58758234::crux_test[0] counterexamples ---- + ---- assert/e68faa4a::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/9a570a5e::fmt[0]::write[0] - [Crux] Translation error in core/9a570a5e::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/9a570a5e::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/beffc212::fmt[0]::write[0] + [Crux] Translation error in core/beffc212::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/beffc212::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.assert"' to rerun this test only. - array: OK (1.00s) - conc: OK (0.94s) - assert_ok: OK (1.52s) + array: OK (1.04s) + conc: OK (0.98s) + assert_ok: OK (1.55s) crux coverage Output testing coverage_try: warning: trailing semicolon in macro used in expression position @@ -2127,7 +1923,7 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (1.04s) +OK (1.01s) coverage_cond: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | @@ -2157,7 +1953,7 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (1.00s) +FAIL (0.97s) files test/coverage/coverage.good and test/coverage/coverage.out differ; test/coverage/coverage.out contains: warning: branch condition never has a value other than [1] ┌─ test/coverage/coverage.rs:11:15 @@ -2181,7 +1977,7 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (0.85s) +OK (0.91s) coverage_macro: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | @@ -2196,7 +1992,7 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (1.01s) +FAIL (1.06s) files test/coverage/coverage_macro.good and test/coverage/coverage_macro.out differ; test/coverage/coverage_macro.out contains: warning: branch condition never has value true ┌─ test/coverage/coverage_macro.rs:7:12 @@ -2223,7 +2019,7 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (0.89s) +OK (0.88s) coverage_match: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | @@ -2238,7 +2034,7 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (1.00s) +FAIL (1.06s) files test/coverage/coverage_match.good and test/coverage/coverage_match.out differ; test/coverage/coverage_match.out contains: warning: branch condition never has value 1 ┌─ test/coverage/coverage_match.rs:12:9 @@ -2268,7 +2064,7 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (0.96s) +FAIL (0.99s) files test/coverage/coverage_shortcircuit.good and test/coverage/coverage_shortcircuit.out differ; test/coverage/coverage_shortcircuit.out contains: warning: branch condition never has value false ┌─ test/coverage/coverage_shortcircuit.rs:9:8 @@ -2298,7 +2094,7 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (0.95s) +FAIL (0.91s) files test/coverage/coverage_loop.good and test/coverage/coverage_loop.out differ; test/coverage/coverage_loop.out contains: warning: branch condition never has value true ┌─ test/coverage/coverage_loop.rs:12:19 @@ -2309,4 +2105,4 @@ FAIL (expected: taking address of an overridden function) (0.05s) Use -p '/coverage_loop/' to rerun this test only. -74 out of 339 tests failed (459.89s) +64 out of 339 tests failed (461.96s) From 917fb765e407c80b809338c5e4dac982067c7aea Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 24 May 2023 16:07:24 -0400 Subject: [PATCH 059/114] Handle closure constants properly --- crucible-mir/src/Mir/JSON.hs | 3 +++ crucible-mir/src/Mir/Mir.hs | 1 + crucible-mir/src/Mir/PP.hs | 1 + crucible-mir/src/Mir/Trans.hs | 29 ++++++++++++++++++----------- dependencies/mir-json | 2 +- 5 files changed, 24 insertions(+), 12 deletions(-) diff --git a/crucible-mir/src/Mir/JSON.hs b/crucible-mir/src/Mir/JSON.hs index 971d0806a..25d923bb1 100644 --- a/crucible-mir/src/Mir/JSON.hs +++ b/crucible-mir/src/Mir/JSON.hs @@ -527,6 +527,9 @@ instance FromJSON ConstVal where Just (String "tuple") -> ConstTuple <$> v .: "elements" + Just (String "closure") -> + ConstClosure <$> v .: "upvars" + o -> do fail $ "parseJSON - bad rendered constant kind: " ++ show o diff --git a/crucible-mir/src/Mir/Mir.hs b/crucible-mir/src/Mir/Mir.hs index 9f0bd747a..1c20ddfa2 100644 --- a/crucible-mir/src/Mir/Mir.hs +++ b/crucible-mir/src/Mir/Mir.hs @@ -503,6 +503,7 @@ data ConstVal = | ConstVariant DefId | ConstFunction DefId | ConstTuple [ConstVal] + | ConstClosure [ConstVal] | ConstArray [ConstVal] | ConstRepeat ConstVal Int | ConstInitializer DefId diff --git a/crucible-mir/src/Mir/PP.hs b/crucible-mir/src/Mir/PP.hs index 5c8a2c86f..423aa582b 100644 --- a/crucible-mir/src/Mir/PP.hs +++ b/crucible-mir/src/Mir/PP.hs @@ -318,6 +318,7 @@ instance Pretty ConstVal where pretty (ConstChar i) = pretty i pretty (ConstVariant i) = pr_id i pretty (ConstTuple cs) = tupled (map pretty cs) + pretty (ConstClosure us) = pretty "closure" <> list (map pretty us) pretty (ConstArray cs) = list (map pretty cs) pretty (ConstRepeat cv i) = brackets (pretty cv <> semi <+> pretty i) pretty (ConstFunction a) = pr_id a diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index 4c3b96dad..c65219108 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -193,17 +193,10 @@ transConstVal _ty (Some C.UnitRepr) (M.ConstFunction _did) = return $ MirExp C.UnitRepr $ S.app E.EmptyApp transConstVal _ty (Some C.UnitRepr) (M.ConstTuple []) = return $ MirExp C.UnitRepr $ S.app E.EmptyApp -transConstVal (M.TyTuple tys) (Some (C.StructRepr tprs)) (M.ConstTuple vals) = do - col <- use $ cs . collection - vals' <- zipWith3M - (\ty (Some tpr) val -> - case tpr of - C.MaybeRepr valTpr -> do - transConstVal ty (Some valTpr) val - _ -> - mirFail $ "transConstVal (ConstTuple): expected tuple field to have MaybeType, but got " ++ show tpr) - tys (toListFC Some tprs) vals - return $ buildTupleMaybe col tys $ map Just vals' +transConstVal (M.TyTuple tys) (Some (C.StructRepr tprs)) (M.ConstTuple vals) = + transConstTuple tys tprs vals +transConstVal (M.TyClosure upvar_tys) (Some (C.StructRepr upvar_tprs)) (M.ConstClosure upvar_vals) = + transConstTuple upvar_tys upvar_tprs upvar_vals transConstVal _ty (Some (C.RealValRepr)) (M.ConstFloat (M.FloatLit _ str)) = case reads str of @@ -257,6 +250,20 @@ transConstVal ty (Some (MirReferenceRepr tpr)) init = do transConstVal ty tp cv = mirFail $ "fail or unimp constant: " ++ show ty ++ " (" ++ show tp ++ ") " ++ show cv +-- Translate a constant (non-empty) tuple or constant closure value. +transConstTuple :: [M.Ty] -> C.CtxRepr ctx -> [ConstVal] -> MirGenerator h s ret (MirExp s) +transConstTuple tys tprs vals = do + col <- use $ cs . collection + vals' <- zipWith3M + (\ty (Some tpr) val -> + case tpr of + C.MaybeRepr valTpr -> do + transConstVal ty (Some valTpr) val + _ -> + mirFail $ "transConstTuple: expected tuple field to have MaybeType, but got " ++ show tpr) + tys (toListFC Some tprs) vals + return $ buildTupleMaybe col tys $ map Just vals' + -- Taken from GHC's source code, which is BSD-3 licensed. zipWith3M :: Monad m => (a -> b -> c -> m d) -> [a] -> [b] -> [c] -> m [d] {-# INLINE zipWith3M #-} diff --git a/dependencies/mir-json b/dependencies/mir-json index 819fa0392..0e9882883 160000 --- a/dependencies/mir-json +++ b/dependencies/mir-json @@ -1 +1 @@ -Subproject commit 819fa0392d36bf49bed7eff52acd3d9bc4c5b515 +Subproject commit 0e988288342da6320a2e38ea31e7b42020dd3caf From 141803b4ff300971f4e4ddcb1be755da27e7b539 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 24 May 2023 16:08:11 -0400 Subject: [PATCH 060/114] Reapply "implement allocate_zeroed to fix vec![0; len]" Originally from commit 5bd1bcfd941660a540ea5d5a20c2b545944dd6d3 --- crux-mir/lib/Patches.md | 4 ++++ crux-mir/lib/alloc/src/raw_vec.rs | 9 +++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index 0b11ca2e7..558198612 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -72,3 +72,7 @@ identify all of the code that was changed in each patch. * Reimplement `to_{le,be}_bytes` (last applied: May 18, 2023) Same reasoning as above. + +* Use `allocate_zeroed` in `RawVec`'s `allocate_in` (last applied: May 24, 2023) + + This is needed to make `vec![0; len]` work as expected. diff --git a/crux-mir/lib/alloc/src/raw_vec.rs b/crux-mir/lib/alloc/src/raw_vec.rs index f7c09a9e9..5c7a99eef 100644 --- a/crux-mir/lib/alloc/src/raw_vec.rs +++ b/crux-mir/lib/alloc/src/raw_vec.rs @@ -173,8 +173,13 @@ impl RawVec { Self::new_in(alloc) } else { let ptr = { - // NB: This ignores both the choice of allocator and the `init` argument - let ptr = crucible::alloc::allocate::(capacity); + // NB: This ignores the choice of allocator + let ptr = match init { + AllocInit::Zeroed => + crucible::alloc::allocate_zeroed::(capacity), + AllocInit::Uninitialized => + crucible::alloc::allocate::(capacity), + }; unsafe { NonNull::new_unchecked(ptr) } }; From b153e90acae2d35d55f4ee0574617577035817e5 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 25 May 2023 12:34:59 -0400 Subject: [PATCH 061/114] crucible-mir: Handle constant function pointers --- crucible-mir/src/Mir/JSON.hs | 3 +++ crucible-mir/src/Mir/Mir.hs | 1 + crucible-mir/src/Mir/PP.hs | 1 + crucible-mir/src/Mir/Trans.hs | 18 ++++++++++++++++++ dependencies/mir-json | 2 +- 5 files changed, 24 insertions(+), 1 deletion(-) diff --git a/crucible-mir/src/Mir/JSON.hs b/crucible-mir/src/Mir/JSON.hs index 25d923bb1..6744caf96 100644 --- a/crucible-mir/src/Mir/JSON.hs +++ b/crucible-mir/src/Mir/JSON.hs @@ -530,6 +530,9 @@ instance FromJSON ConstVal where Just (String "closure") -> ConstClosure <$> v .: "upvars" + Just (String "fn_ptr") -> + ConstFnPtr <$> v .: "instance" + o -> do fail $ "parseJSON - bad rendered constant kind: " ++ show o diff --git a/crucible-mir/src/Mir/Mir.hs b/crucible-mir/src/Mir/Mir.hs index 1c20ddfa2..f400f7e02 100644 --- a/crucible-mir/src/Mir/Mir.hs +++ b/crucible-mir/src/Mir/Mir.hs @@ -513,6 +513,7 @@ data ConstVal = | ConstRawPtr Integer | ConstStruct [ConstVal] | ConstEnum Int [ConstVal] + | ConstFnPtr Instance deriving (Show,Eq, Ord, Generic) data AggregateKind = diff --git a/crucible-mir/src/Mir/PP.hs b/crucible-mir/src/Mir/PP.hs index 423aa582b..c827cca4a 100644 --- a/crucible-mir/src/Mir/PP.hs +++ b/crucible-mir/src/Mir/PP.hs @@ -329,6 +329,7 @@ instance Pretty ConstVal where pretty (ConstStruct fs) = pretty "struct" <> list (map pretty fs) pretty (ConstEnum v fs) = pretty "enum" <> list ((pretty "variant" <+> pretty v) : map pretty fs) pretty (ConstSlice cs) = list (map pretty cs) + pretty (ConstFnPtr i) = pretty "fn_ptr" <> brackets (pretty i) instance Pretty AggregateKind where pretty (AKArray t) = brackets (pretty t) diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index c65219108..cabf7c618 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -247,6 +247,24 @@ transConstVal ty (Some (MirReferenceRepr tpr)) init = do "transConstVal returned wrong type: expected " ++ show tpr ++ ", got " ++ show tpr' ref <- constMirRef tpr val return $ MirExp (MirReferenceRepr tpr) ref +transConstVal _ty (Some tpr@(C.FunctionHandleRepr argTys retTy)) (ConstFnPtr inst) = do + let did = inst^.inDefId + mbHndl <- resolveFn did + case mbHndl of + Just (MirHandle _ _ hndl) -> do + Refl <- testEqualityOrFail argTys (FH.handleArgTypes hndl) $ unlines + [ "transConstVal (ConstFnPtr): argument types mismatch" + , "expected: " ++ show argTys + , "actual: " ++ show (FH.handleArgTypes hndl) + ] + Refl <- testEqualityOrFail retTy (FH.handleReturnType hndl) $ unlines + [ "transConstVal (ConstFnPtr): return type mismatch" + , "expected: " ++ show retTy + , "actual: " ++ show (FH.handleReturnType hndl) + ] + return $ MirExp tpr $ R.App $ E.HandleLit hndl + Nothing -> mirFail $ + "transConstVal (ConstFnPtr): Couldn't resolve function " ++ show did transConstVal ty tp cv = mirFail $ "fail or unimp constant: " ++ show ty ++ " (" ++ show tp ++ ") " ++ show cv diff --git a/dependencies/mir-json b/dependencies/mir-json index 0e9882883..3b0f12244 160000 --- a/dependencies/mir-json +++ b/dependencies/mir-json @@ -1 +1 @@ -Subproject commit 0e988288342da6320a2e38ea31e7b42020dd3caf +Subproject commit 3b0f12244ef993e108f348bdbd3e1829543fc7a1 From 5df73b3f6e69c838408a4430dcba56cfd08c695b Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 25 May 2023 14:50:32 -0400 Subject: [PATCH 062/114] Reapply "reenable old override for slice::len" Originally from commit 7e5c01df86287f7c11eaa52f160dce9a3751b07d --- crux-mir/lib/Patches.md | 7 +++++++ crux-mir/lib/core/src/slice/mod.rs | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index 558198612..67944e145 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -76,3 +76,10 @@ identify all of the code that was changed in each patch. * Use `allocate_zeroed` in `RawVec`'s `allocate_in` (last applied: May 24, 2023) This is needed to make `vec![0; len]` work as expected. + +* Use `crucible_slice_len_hook` when computing slice `len` (last applied: May 25, 2023) + + The usual implementation of slice length requires using the `PtrRepr` union + type. Currently, `crucible-mir` does not support Rust unions, so we instead + implement slice `len` in terms of our own `crucible_slice_len_hook` function + that we override. diff --git a/crux-mir/lib/core/src/slice/mod.rs b/crux-mir/lib/core/src/slice/mod.rs index d93a3a57e..136f58799 100644 --- a/crux-mir/lib/core/src/slice/mod.rs +++ b/crux-mir/lib/core/src/slice/mod.rs @@ -134,7 +134,10 @@ impl [T] { #[inline] #[must_use] pub const fn len(&self) -> usize { - ptr::metadata(self) + const fn crucible_slice_len_hook(slice: &[T]) -> usize { + ptr::metadata(slice) + } + crucible_slice_len_hook(self) } /// Returns `true` if the slice has a length of 0. From 8d99d39516c51837b8e7c9b9bcd9ec54638d576d Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 25 May 2023 15:25:01 -0400 Subject: [PATCH 063/114] crucible-mir: Add sub_ptr custom ops These play the same role as `offset_from`. --- crucible-mir/src/Mir/TransCustom.hs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crucible-mir/src/Mir/TransCustom.hs b/crucible-mir/src/Mir/TransCustom.hs index 3e4183433..a289da164 100644 --- a/crucible-mir/src/Mir/TransCustom.hs +++ b/crucible-mir/src/Mir/TransCustom.hs @@ -137,6 +137,8 @@ customOpDefs = Map.fromList $ [ , ptr_wrapping_offset_mut , ptr_offset_from , ptr_offset_from_mut + , sub_ptr + , sub_ptr_mut , ptr_compare_usize , is_aligned_and_not_null , ptr_slice_from_raw_parts @@ -481,6 +483,11 @@ ptr_offset_from = (["core", "ptr", "const_ptr", "{impl}", "offset_from"], ptr_of ptr_offset_from_mut :: (ExplodedDefId, CustomRHS) ptr_offset_from_mut = (["core", "ptr", "mut_ptr", "{impl}", "offset_from"], ptr_offset_from_impl) +sub_ptr :: (ExplodedDefId, CustomRHS) +sub_ptr = (["core", "ptr", "const_ptr", "{impl}", "sub_ptr"], ptr_offset_from_impl) +sub_ptr_mut :: (ExplodedDefId, CustomRHS) +sub_ptr_mut = (["core", "ptr", "mut_ptr", "{impl}", "sub_ptr"], ptr_offset_from_impl) + ptr_compare_usize :: (ExplodedDefId, CustomRHS) ptr_compare_usize = (["core", "crucible", "ptr", "compare_usize"], \substs -> case substs of From 6308cc86a5de313ee9c76b4095e77afc9b0973b0 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 25 May 2023 16:25:57 -0400 Subject: [PATCH 064/114] Remove invalid inline pragmas `#[inline(...)]` pragmas can now only be attached to functions, not enum variants. It is unclear what utility these pragmas were serving in the first place, so let's just remove them. --- crux-mir/test/conc_eval/enum/arg2.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crux-mir/test/conc_eval/enum/arg2.rs b/crux-mir/test/conc_eval/enum/arg2.rs index 687fd89d9..3d6f771e2 100644 --- a/crux-mir/test/conc_eval/enum/arg2.rs +++ b/crux-mir/test/conc_eval/enum/arg2.rs @@ -1,8 +1,6 @@ #![cfg_attr(not(with_main), no_std)] enum E { - #[inline(never)] A(u8), - #[inline(never)] B(i32,i32), } From 22e1e07f55642c8f8fba29df84c8a1e857159665 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Fri, 26 May 2023 07:55:55 -0400 Subject: [PATCH 065/114] crux-mir: Override const and mut slice len as well --- crucible-mir/src/Mir/TransCustom.hs | 28 ++++++++++++++++++++------ crux-mir/lib/Patches.md | 2 +- crux-mir/lib/core/src/ptr/const_ptr.rs | 5 ++++- crux-mir/lib/core/src/ptr/mut_ptr.rs | 5 ++++- 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/crucible-mir/src/Mir/TransCustom.hs b/crucible-mir/src/Mir/TransCustom.hs index a289da164..c82dd166d 100644 --- a/crucible-mir/src/Mir/TransCustom.hs +++ b/crucible-mir/src/Mir/TransCustom.hs @@ -75,6 +75,8 @@ customOpDefs = Map.fromList $ [ , slice_index_usize_get_unchecked_mut , slice_index_range_get_unchecked_mut , slice_len + , const_slice_len + , mut_slice_len -- core::intrinsics , discriminant_value @@ -1049,12 +1051,26 @@ array_from_slice = (["core","array", "{impl}", "try_from", "crucible_array_from_ slice_len :: (ExplodedDefId, CustomRHS) slice_len = - (["core","slice","{impl}","len", "crucible_slice_len_hook"] - , \(Substs [_]) -> Just $ CustomOp $ \ _optys ops -> - case ops of - [MirExp (MirSliceRepr _) e] -> do - return $ MirExp UsizeRepr $ getSliceLen e - _ -> mirFail $ "BUG: invalid arguments to " ++ "slice_len") + ( ["core","slice","{impl}","len", "crucible_slice_len_hook"] + , slice_len_impl ) + +const_slice_len :: (ExplodedDefId, CustomRHS) +const_slice_len = + ( ["core","ptr","const_ptr","{impl}","len", "crucible_slice_len_hook"] + , slice_len_impl ) + +mut_slice_len :: (ExplodedDefId, CustomRHS) +mut_slice_len = + ( ["core","ptr","mut_ptr","{impl}","len", "crucible_slice_len_hook"] + , slice_len_impl ) + +slice_len_impl :: CustomRHS +slice_len_impl (Substs [_]) = + Just $ CustomOp $ \ _optys ops -> + case ops of + [MirExp (MirSliceRepr _) e] -> do + return $ MirExp UsizeRepr $ getSliceLen e + _ -> mirFail $ "BUG: invalid arguments to " ++ "slice_len" -- These four custom ops implement mutable and immutable unchecked indexing by -- usize and by Range. All other indexing dispatches to one of these. Note diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index 67944e145..b445f5a23 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -77,7 +77,7 @@ identify all of the code that was changed in each patch. This is needed to make `vec![0; len]` work as expected. -* Use `crucible_slice_len_hook` when computing slice `len` (last applied: May 25, 2023) +* Use `crucible_slice_len_hook` when computing slice `len` (last applied: May 26, 2023) The usual implementation of slice length requires using the `PtrRepr` union type. Currently, `crucible-mir` does not support Rust unions, so we instead diff --git a/crux-mir/lib/core/src/ptr/const_ptr.rs b/crux-mir/lib/core/src/ptr/const_ptr.rs index 935605b79..e75bd56a9 100644 --- a/crux-mir/lib/core/src/ptr/const_ptr.rs +++ b/crux-mir/lib/core/src/ptr/const_ptr.rs @@ -1612,7 +1612,10 @@ impl *const [T] { #[unstable(feature = "slice_ptr_len", issue = "71146")] #[rustc_const_unstable(feature = "const_slice_ptr_len", issue = "71146")] pub const fn len(self) -> usize { - metadata(self) + const fn crucible_slice_len_hook(slice: *const [T]) -> usize { + metadata(slice) + } + crucible_slice_len_hook(self) } /// Returns a raw pointer to the slice's buffer. diff --git a/crux-mir/lib/core/src/ptr/mut_ptr.rs b/crux-mir/lib/core/src/ptr/mut_ptr.rs index 6a4fa75c7..13849798d 100644 --- a/crux-mir/lib/core/src/ptr/mut_ptr.rs +++ b/crux-mir/lib/core/src/ptr/mut_ptr.rs @@ -1883,7 +1883,10 @@ impl *mut [T] { #[unstable(feature = "slice_ptr_len", issue = "71146")] #[rustc_const_unstable(feature = "const_slice_ptr_len", issue = "71146")] pub const fn len(self) -> usize { - metadata(self) + const fn crucible_slice_len_hook(slice: *mut [T]) -> usize { + metadata(slice) + } + crucible_slice_len_hook(self) } /// Returns `true` if the raw slice has a length of 0. From 2272bc374f770c6b409b0dfa599a70777b15af70 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Fri, 26 May 2023 08:15:25 -0400 Subject: [PATCH 066/114] Reapply "update &[T] -> &[T; N] TryFrom impl" Originally from commit e645cee0315584bb9371a60b266d11ab1f156a91 --- crux-mir/lib/Patches.md | 4 ++++ crux-mir/lib/core/src/array/mod.rs | 20 ++++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index b445f5a23..cad1cb89c 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -83,3 +83,7 @@ identify all of the code that was changed in each patch. type. Currently, `crucible-mir` does not support Rust unions, so we instead implement slice `len` in terms of our own `crucible_slice_len_hook` function that we override. + +* Use `crucible_array_from_slice_hook` in `&[T] -> &[T; N]` `TryFrom` impl (last applied: May 26, 2023) + + The actual implementation uses a pointer cast that Crucible can't handle. diff --git a/crux-mir/lib/core/src/array/mod.rs b/crux-mir/lib/core/src/array/mod.rs index 2825e0bbb..1bbecc309 100644 --- a/crux-mir/lib/core/src/array/mod.rs +++ b/crux-mir/lib/core/src/array/mod.rs @@ -262,13 +262,20 @@ impl<'a, T, const N: usize> TryFrom<&'a [T]> for &'a [T; N] { type Error = TryFromSliceError; fn try_from(slice: &[T]) -> Result<&[T; N], TryFromSliceError> { - if slice.len() == N { - let ptr = slice.as_ptr() as *const [T; N]; - // SAFETY: ok because we just checked that the length fits - unsafe { Ok(&*ptr) } - } else { - Err(TryFromSliceError(())) + fn crucible_array_from_slice_hook( + slice: &[T], + len: usize, + ) -> Option<&[T; N]> { + if slice.len() == N { + let ptr = slice.as_ptr() as *const [T; N]; + // SAFETY: ok because we just checked that the length fits + unsafe { Some(&*ptr) } + } else { + None + } } + crucible_array_from_slice_hook(slice, N) + .ok_or(TryFromSliceError(())) } } @@ -289,6 +296,7 @@ impl<'a, T, const N: usize> TryFrom<&'a mut [T]> for &'a mut [T; N] { type Error = TryFromSliceError; fn try_from(slice: &mut [T]) -> Result<&mut [T; N], TryFromSliceError> { + // Mut version is not supported by crucible - see comment in TransCustom.array_from_slice if slice.len() == N { let ptr = slice.as_mut_ptr() as *mut [T; N]; // SAFETY: ok because we just checked that the length fits From e629734b9fbc2d3870fd227f656b4b8aaf6bcc27 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Fri, 26 May 2023 08:17:42 -0400 Subject: [PATCH 067/114] Disable `IsRawEqComparable`-based `SpecArrayEq` instances These require pointer casts that Crucible can't support. We instead fall back on the other `SpecArrayEq` instances that are slower (but easier to translate). --- crux-mir/lib/Patches.md | 6 ++++++ crux-mir/lib/core/src/array/equality.rs | 2 ++ 2 files changed, 8 insertions(+) diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index cad1cb89c..38634a59c 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -87,3 +87,9 @@ identify all of the code that was changed in each patch. * Use `crucible_array_from_slice_hook` in `&[T] -> &[T; N]` `TryFrom` impl (last applied: May 26, 2023) The actual implementation uses a pointer cast that Crucible can't handle. + +* Disable `IsRawEqComparable`-based `SpecArrayEq` instances (last applied: May 26, 2023) + + These require pointer casts that Crucible can't support. We instead fall back + on the other `SpecArrayEq` instances that are slower (but easier to + translate). diff --git a/crux-mir/lib/core/src/array/equality.rs b/crux-mir/lib/core/src/array/equality.rs index b2c895f88..f5f7e5663 100644 --- a/crux-mir/lib/core/src/array/equality.rs +++ b/crux-mir/lib/core/src/array/equality.rs @@ -144,6 +144,7 @@ impl, Other, const N: usize> SpecArrayEq for T { } } +/* impl, U, const N: usize> SpecArrayEq for T { fn spec_eq(a: &[T; N], b: &[U; N]) -> bool { // SAFETY: This is why `IsRawEqComparable` is an `unsafe trait`. @@ -156,6 +157,7 @@ impl, U, const N: usize> SpecArrayEq for T { !Self::spec_eq(a, b) } } +*/ /// `U` exists on here mostly because `min_specialization` didn't let me /// repeat the `T` type parameter in the above specialization, so instead From 6dde6467f1cc7a5dff2688292628fc4ef8ddb59e Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Fri, 26 May 2023 08:30:51 -0400 Subject: [PATCH 068/114] Reapply "disable bytewise equality comparisons for [T]" Originally from commit e6dddfb1804c283728322ff7c282cda518e4e952 --- crux-mir/lib/Patches.md | 4 ++++ crux-mir/lib/core/src/slice/cmp.rs | 2 ++ 2 files changed, 6 insertions(+) diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index 38634a59c..ca7673999 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -93,3 +93,7 @@ identify all of the code that was changed in each patch. These require pointer casts that Crucible can't support. We instead fall back on the other `SpecArrayEq` instances that are slower (but easier to translate). + +* Disable bytewise equality comparisons for `[T]` (last applied: May 26, 2023) + + These require the `size_of_val` intrinsic, which isn't current supported. diff --git a/crux-mir/lib/core/src/slice/cmp.rs b/crux-mir/lib/core/src/slice/cmp.rs index 5e1b218e5..34d2635b3 100644 --- a/crux-mir/lib/core/src/slice/cmp.rs +++ b/crux-mir/lib/core/src/slice/cmp.rs @@ -74,6 +74,7 @@ where } } +/* // Use memcmp for bytewise equality when the types allow impl SlicePartialEq for [A] where @@ -92,6 +93,7 @@ where } } } +*/ #[doc(hidden)] // intermediate trait for specialization of slice's PartialOrd From 8db861ee1923ec0cab92d0ae7db20b2b12bf977f Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Fri, 26 May 2023 14:29:59 -0400 Subject: [PATCH 069/114] crucible-mir: Compute discriminant for initial enum values properly Previously, we were hard-coding `0` as the discriminant, but this is just plain wrong for enums with multiple variants. --- crucible-mir/src/Mir/TransTy.hs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/crucible-mir/src/Mir/TransTy.hs b/crucible-mir/src/Mir/TransTy.hs index 3bad56863..4e50fd016 100644 --- a/crucible-mir/src/Mir/TransTy.hs +++ b/crucible-mir/src/Mir/TransTy.hs @@ -921,7 +921,7 @@ buildEnum' adt i es = do es' <- inferElidedVariantFields ftys es asn <- case buildStructAssign' fctx' $ map (fmap (\(MirExp _ e) -> Some e)) es' of Left err -> - mirFail $ "error building variant " ++ show (var^.M.vname) ++ ": " ++ err ++ " -- " ++ show es + mirFail $ "error building variant " ++ show (var^.M.vname) ++ ": " ++ err ++ " -- " ++ show es' Right x -> return x Refl <- testEqualityOrFail (fieldCtxType fctx') ctx' $ "got wrong fields for " ++ show (adt ^. M.adtname, i) ++ "?" @@ -1200,16 +1200,14 @@ initialValue (M.TyAdt nm _ _) = do fldExps <- mapM initField (var^.M.vfields) Just <$> buildStruct' adt fldExps M.Enum _ -> do - case inhabited (adt ^. M.adtvariants) of + case ifind (\_ vars -> vars ^. M.vinhabited) (adt ^. M.adtvariants) of -- Uninhabited enums can't be initialized. - [] -> return Nothing + Nothing -> return Nothing -- Inhabited enums get initialized to their first inhabited variant. - vs@(var : _) -> do + Just (discr, var) -> do fldExps <- mapM initField (var^.M.vfields) - Just <$> buildEnum' adt 0 fldExps + Just <$> buildEnum' adt discr fldExps M.Union -> return Nothing - where - inhabited vars = filter M._vinhabited vars From 087ec43b4b9abb80fe13167c6a01832020154bfc Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Fri, 26 May 2023 18:52:47 -0400 Subject: [PATCH 070/114] crux-mir-test: Update symbolic golden test output --- .../test/symb_eval/alloc/out_of_bounds.good | 8 +++---- .../test/symb_eval/alloc/uninit_read.good | 8 +++---- crux-mir/test/symb_eval/alloc/valid_read.good | 2 +- .../test/symb_eval/alloc/zero_length.good | 2 +- crux-mir/test/symb_eval/any/conditional.good | 2 +- crux-mir/test/symb_eval/any/downcast.good | 2 +- .../test/symb_eval/any/downcast_fail.good | 6 ++--- crux-mir/test/symb_eval/array/basic.good | 2 +- crux-mir/test/symb_eval/array/mux_slice.good | 2 +- crux-mir/test/symb_eval/array/slice.good | 2 +- crux-mir/test/symb_eval/array/slice_mut.good | 2 +- crux-mir/test/symb_eval/bitvector/arith.good | 2 +- crux-mir/test/symb_eval/bitvector/cmp.good | 2 +- .../test/symb_eval/bitvector/from_to.good | 2 +- .../symb_eval/bitvector/leading_zeros.good | 2 +- .../test/symb_eval/bitvector/literals.good | 2 +- .../symb_eval/bitvector/overflowing_sub.good | 2 +- .../test/symb_eval/bitvector/symbolic.good | 2 +- .../test/symb_eval/bytes/extend_bytes.good | 2 +- crux-mir/test/symb_eval/bytes/new.good | 2 +- crux-mir/test/symb_eval/bytes/put.good | 2 +- .../test/symb_eval/bytes/put_overflow.good | 8 +++---- crux-mir/test/symb_eval/bytes/split_off.good | 2 +- crux-mir/test/symb_eval/bytes/split_to.good | 2 +- crux-mir/test/symb_eval/bytes/sym_len.good | 2 +- crux-mir/test/symb_eval/concretize/array.good | 2 +- .../test/symb_eval/concretize/assert_ok.good | 2 +- crux-mir/test/symb_eval/concretize/conc.good | 2 +- .../test/symb_eval/concretize/no_conc.good | 2 +- crux-mir/test/symb_eval/crux/early_fail.good | 14 ++++++------ crux-mir/test/symb_eval/crux/fail_return.good | 16 +++++++------- crux-mir/test/symb_eval/crux/mixed_fail.good | 20 ++++++++--------- crux-mir/test/symb_eval/crux/multi.good | 22 +++++++++---------- crux-mir/test/symb_eval/crypto/bytes.good | 14 ++++++------ crux-mir/test/symb_eval/crypto/bytes2.good | 2 +- crux-mir/test/symb_eval/crypto/double.good | 2 +- crux-mir/test/symb_eval/crypto/ffs.good | 2 +- crux-mir/test/symb_eval/enum/mux.good | 2 +- crux-mir/test/symb_eval/fnptr/mux.good | 2 +- crux-mir/test/symb_eval/io/vec_write.good | 2 +- crux-mir/test/symb_eval/mux/array.good | 2 +- crux-mir/test/symb_eval/mux/array_mut.good | 2 +- crux-mir/test/symb_eval/num/checked_add.good | 6 ++--- crux-mir/test/symb_eval/num/checked_div.good | 10 ++++----- crux-mir/test/symb_eval/num/checked_mul.good | 6 ++--- .../symb_eval/num/checked_mul_signed.good | 6 ++--- .../test/symb_eval/overrides/bad_symb1.good | 6 ++--- .../test/symb_eval/overrides/bad_symb2.good | 6 ++--- .../test/symb_eval/overrides/override1.good | 2 +- .../test/symb_eval/overrides/override2.good | 6 ++--- .../test/symb_eval/overrides/override3.good | 2 +- .../test/symb_eval/overrides/override4.good | 2 +- .../test/symb_eval/overrides/override5.good | 6 ++--- .../symb_eval/overrides/override_rust.good | 2 +- .../test/symb_eval/refs/mux_init_imm.good | 2 +- .../test/symb_eval/refs/mux_init_mut.good | 2 +- crux-mir/test/symb_eval/scalar/test1.good | 2 +- .../test/symb_eval/sym_bytes/construct.good | 6 ++--- .../test/symb_eval/sym_bytes/deserialize.good | 6 ++--- crux-mir/test/symb_eval/vec/clone.good | 2 +- crux-mir/test/symb_eval/vec/into_iter.good | 2 +- crux-mir/test/symb_eval/vec/macro.good | 2 +- crux-mir/test/symb_eval/vec/sort_by_key.good | 2 +- .../test/symb_eval/vector/as_mut_slice.good | 2 +- crux-mir/test/symb_eval/vector/as_slice.good | 2 +- crux-mir/test/symb_eval/vector/concat.good | 2 +- .../symb_eval/vector/copy_from_slice.good | 2 +- crux-mir/test/symb_eval/vector/mut.good | 2 +- crux-mir/test/symb_eval/vector/new.good | 2 +- crux-mir/test/symb_eval/vector/push.good | 2 +- crux-mir/test/symb_eval/vector/replicate.good | 2 +- crux-mir/test/symb_eval/vector/split_at.good | 2 +- 72 files changed, 143 insertions(+), 143 deletions(-) diff --git a/crux-mir/test/symb_eval/alloc/out_of_bounds.good b/crux-mir/test/symb_eval/alloc/out_of_bounds.good index 188f578fc..e4abadfbf 100644 --- a/crux-mir/test/symb_eval/alloc/out_of_bounds.good +++ b/crux-mir/test/symb_eval/alloc/out_of_bounds.good @@ -1,13 +1,13 @@ -test out_of_bounds/ca0644b7::crux_test[0]: FAILED +test out_of_bounds/fb36419e::crux_test[0]: FAILED failures: ----- out_of_bounds/ca0644b7::crux_test[0] counterexamples ---- +---- out_of_bounds/fb36419e::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/ca0644b7::crux_test[0] +[Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/fb36419e::crux_test[0] [Crux] vector index out of range: the length is 10 but the index is BV 12 [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/ca0644b7::crux_test[0] +[Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/fb36419e::crux_test[0] [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/alloc/uninit_read.good b/crux-mir/test/symb_eval/alloc/uninit_read.good index 4dbddb17d..8203fd082 100644 --- a/crux-mir/test/symb_eval/alloc/uninit_read.good +++ b/crux-mir/test/symb_eval/alloc/uninit_read.good @@ -1,13 +1,13 @@ -test uninit_read/851ea4ad::crux_test[0]: FAILED +test uninit_read/9bd6592b::crux_test[0]: FAILED failures: ----- uninit_read/851ea4ad::crux_test[0] counterexamples ---- +---- uninit_read/9bd6592b::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/851ea4ad::crux_test[0] +[Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/9bd6592b::crux_test[0] [Crux] Attempted to read uninitialized vector index [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/851ea4ad::crux_test[0] +[Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/9bd6592b::crux_test[0] [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/alloc/valid_read.good b/crux-mir/test/symb_eval/alloc/valid_read.good index 8955502eb..e01406bff 100644 --- a/crux-mir/test/symb_eval/alloc/valid_read.good +++ b/crux-mir/test/symb_eval/alloc/valid_read.good @@ -1,3 +1,3 @@ -test valid_read/c82bffb5::crux_test[0]: returned 45, ok +test valid_read/73b28820::crux_test[0]: returned 45, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/alloc/zero_length.good b/crux-mir/test/symb_eval/alloc/zero_length.good index d3abeb0a7..992b092db 100644 --- a/crux-mir/test/symb_eval/alloc/zero_length.good +++ b/crux-mir/test/symb_eval/alloc/zero_length.good @@ -1,3 +1,3 @@ -test zero_length/d9fb4927::crux_test[0]: ok +test zero_length/aeacedba::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/any/conditional.good b/crux-mir/test/symb_eval/any/conditional.good index 2dd35b976..cfe6b55e7 100644 --- a/crux-mir/test/symb_eval/any/conditional.good +++ b/crux-mir/test/symb_eval/any/conditional.good @@ -1,3 +1,3 @@ -test conditional/e22ae149::crux_test[0]: ok +test conditional/1f766d14::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/any/downcast.good b/crux-mir/test/symb_eval/any/downcast.good index 6f08340fc..636e88cba 100644 --- a/crux-mir/test/symb_eval/any/downcast.good +++ b/crux-mir/test/symb_eval/any/downcast.good @@ -1,3 +1,3 @@ -test downcast/c8ce9e8e::crux_test[0]: returned 1, ok +test downcast/d939fa83::crux_test[0]: returned 1, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/any/downcast_fail.good b/crux-mir/test/symb_eval/any/downcast_fail.good index 342800fae..0f2d4da64 100644 --- a/crux-mir/test/symb_eval/any/downcast_fail.good +++ b/crux-mir/test/symb_eval/any/downcast_fail.good @@ -1,10 +1,10 @@ -test downcast_fail/3a1fbbbh::crux_test[0]: FAILED +test downcast_fail/a71ba9d2::crux_test[0]: FAILED failures: ----- downcast_fail/3a1fbbbh::crux_test[0] counterexamples ---- +---- downcast_fail/a71ba9d2::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/any/downcast_fail.rs:10:14: 10:15: error: in downcast_fail/3a1fbbbh::crux_test[0] +[Crux] test/symb_eval/any/downcast_fail.rs:10:14: 10:15: error: in downcast_fail/a71ba9d2::crux_test[0] [Crux] failed to downcast Any as BVRepr 32 [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/array/basic.good b/crux-mir/test/symb_eval/array/basic.good index ec102490b..9f0721a59 100644 --- a/crux-mir/test/symb_eval/array/basic.good +++ b/crux-mir/test/symb_eval/array/basic.good @@ -1,3 +1,3 @@ -test basic/a10c5e44::crux_test[0]: returned 3, ok +test basic/0010ffa4::crux_test[0]: returned 3, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/array/mux_slice.good b/crux-mir/test/symb_eval/array/mux_slice.good index f59baf7b3..745d81cd6 100644 --- a/crux-mir/test/symb_eval/array/mux_slice.good +++ b/crux-mir/test/symb_eval/array/mux_slice.good @@ -1,3 +1,3 @@ -test mux_slice/df68b584::crux_test[0]: returned Symbolic BV, ok +test mux_slice/e736ab73::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/array/slice.good b/crux-mir/test/symb_eval/array/slice.good index 2d240a3a6..c08294483 100644 --- a/crux-mir/test/symb_eval/array/slice.good +++ b/crux-mir/test/symb_eval/array/slice.good @@ -1,3 +1,3 @@ -test slice/ab67a8fd::crux_test[0]: returned 5, ok +test slice/9ed070ad::crux_test[0]: returned 5, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/array/slice_mut.good b/crux-mir/test/symb_eval/array/slice_mut.good index 06ec78240..ad14f864d 100644 --- a/crux-mir/test/symb_eval/array/slice_mut.good +++ b/crux-mir/test/symb_eval/array/slice_mut.good @@ -1,3 +1,3 @@ -test slice_mut/cf0e1c49::crux_test[0]: returned 5, ok +test slice_mut/d42dc8bf::crux_test[0]: returned 5, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/arith.good b/crux-mir/test/symb_eval/bitvector/arith.good index 9d27f4d34..7bbf853f0 100644 --- a/crux-mir/test/symb_eval/bitvector/arith.good +++ b/crux-mir/test/symb_eval/bitvector/arith.good @@ -1,3 +1,3 @@ -test arith/909ad774::crux_test[0]: ok +test arith/7bb6b57e::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/cmp.good b/crux-mir/test/symb_eval/bitvector/cmp.good index e82ae15ed..78dc12dfc 100644 --- a/crux-mir/test/symb_eval/bitvector/cmp.good +++ b/crux-mir/test/symb_eval/bitvector/cmp.good @@ -1,3 +1,3 @@ -test cmp/79f545fc::crux_test[0]: ok +test cmp/9d74ce07::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/from_to.good b/crux-mir/test/symb_eval/bitvector/from_to.good index d54bb2266..920eed437 100644 --- a/crux-mir/test/symb_eval/bitvector/from_to.good +++ b/crux-mir/test/symb_eval/bitvector/from_to.good @@ -1,3 +1,3 @@ -test from_to/81d915d3::crux_test[0]: ok +test from_to/1c910e9c::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/leading_zeros.good b/crux-mir/test/symb_eval/bitvector/leading_zeros.good index bce2fc5b1..e2c33c0f4 100644 --- a/crux-mir/test/symb_eval/bitvector/leading_zeros.good +++ b/crux-mir/test/symb_eval/bitvector/leading_zeros.good @@ -1,3 +1,3 @@ -test leading_zeros/a181beeb::crux_test[0]: ok +test leading_zeros/8a721fcc::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/literals.good b/crux-mir/test/symb_eval/bitvector/literals.good index 118642e14..2de3d3fb5 100644 --- a/crux-mir/test/symb_eval/bitvector/literals.good +++ b/crux-mir/test/symb_eval/bitvector/literals.good @@ -1,3 +1,3 @@ -test literals/9d6d8593::crux_test[0]: ok +test literals/ce6d61a1::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/overflowing_sub.good b/crux-mir/test/symb_eval/bitvector/overflowing_sub.good index a658401c3..d60697e57 100644 --- a/crux-mir/test/symb_eval/bitvector/overflowing_sub.good +++ b/crux-mir/test/symb_eval/bitvector/overflowing_sub.good @@ -1,3 +1,3 @@ -test overflowing_sub/d297e8c6::crux_test[0]: ok +test overflowing_sub/d9667423::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/symbolic.good b/crux-mir/test/symb_eval/bitvector/symbolic.good index 295ff7e2a..327bc23b3 100644 --- a/crux-mir/test/symb_eval/bitvector/symbolic.good +++ b/crux-mir/test/symb_eval/bitvector/symbolic.good @@ -1,3 +1,3 @@ -test symbolic/8802d91d::crux_test[0]: ok +test symbolic/10258fd3::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/extend_bytes.good b/crux-mir/test/symb_eval/bytes/extend_bytes.good index 9efeeecb9..b73531704 100644 --- a/crux-mir/test/symb_eval/bytes/extend_bytes.good +++ b/crux-mir/test/symb_eval/bytes/extend_bytes.good @@ -1,3 +1,3 @@ -test extend_bytes/724ff396::f[0]: ok +test extend_bytes/aa56259e::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/new.good b/crux-mir/test/symb_eval/bytes/new.good index 6eaf95355..13d4da34e 100644 --- a/crux-mir/test/symb_eval/bytes/new.good +++ b/crux-mir/test/symb_eval/bytes/new.good @@ -1,3 +1,3 @@ -test new/7e2f4495::f[0]: ok +test new/b3e2709a::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/put.good b/crux-mir/test/symb_eval/bytes/put.good index 038295946..5a5a70eee 100644 --- a/crux-mir/test/symb_eval/bytes/put.good +++ b/crux-mir/test/symb_eval/bytes/put.good @@ -1,3 +1,3 @@ -test put/3a1fbbbh::f[0]: ok +test put/0fcb517b::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/put_overflow.good b/crux-mir/test/symb_eval/bytes/put_overflow.good index 453a375dd..f261a5442 100644 --- a/crux-mir/test/symb_eval/bytes/put_overflow.good +++ b/crux-mir/test/symb_eval/bytes/put_overflow.good @@ -1,10 +1,10 @@ -test put_overflow/57f18178::f[0]: FAILED +test put_overflow/84c199d7::f[0]: FAILED failures: ----- put_overflow/57f18178::f[0] counterexamples ---- +---- put_overflow/84c199d7::f[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] internal: error: in bytes/c14f6c84::{impl#4}[0]::put_slice[0] -[Crux] panicking::panic_fmt, called from bytes/c14f6c84::{impl#4}[0]::put_slice[0] +[Crux] internal: error: in bytes/91c2c7c6::{impl#4}[0]::put_slice[0] +[Crux] panicking::panic_fmt, called from bytes/91c2c7c6::{impl#4}[0]::put_slice[0] [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/bytes/split_off.good b/crux-mir/test/symb_eval/bytes/split_off.good index 0d0c568ca..4bd6be310 100644 --- a/crux-mir/test/symb_eval/bytes/split_off.good +++ b/crux-mir/test/symb_eval/bytes/split_off.good @@ -1,3 +1,3 @@ -test split_off/3a1fbbbh::f[0]: ok +test split_off/7c95a5fb::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/split_to.good b/crux-mir/test/symb_eval/bytes/split_to.good index 9832b893f..19e596453 100644 --- a/crux-mir/test/symb_eval/bytes/split_to.good +++ b/crux-mir/test/symb_eval/bytes/split_to.good @@ -1,3 +1,3 @@ -test split_to/3a1fbbbh::f[0]: ok +test split_to/2ad59b12::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/sym_len.good b/crux-mir/test/symb_eval/bytes/sym_len.good index 961e9684c..235917a45 100644 --- a/crux-mir/test/symb_eval/bytes/sym_len.good +++ b/crux-mir/test/symb_eval/bytes/sym_len.good @@ -1,3 +1,3 @@ -test sym_len/3a1fbbbh::f[0]: ok +test sym_len/4bc70337::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/concretize/array.good b/crux-mir/test/symb_eval/concretize/array.good index bcc382540..6c171d270 100644 --- a/crux-mir/test/symb_eval/concretize/array.good +++ b/crux-mir/test/symb_eval/concretize/array.good @@ -1,3 +1,3 @@ -test array/191383c3::crux_test[0]: returned (1, 2, 3), ok +test array/f99e13ca::crux_test[0]: returned (1, 2, 3), ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/concretize/assert_ok.good b/crux-mir/test/symb_eval/concretize/assert_ok.good index d34d68332..2fc4024a0 100644 --- a/crux-mir/test/symb_eval/concretize/assert_ok.good +++ b/crux-mir/test/symb_eval/concretize/assert_ok.good @@ -1,3 +1,3 @@ -test assert_ok/e20e387d::crux_test[0]: returned 1, ok +test assert_ok/d68c5200::crux_test[0]: returned 1, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/concretize/conc.good b/crux-mir/test/symb_eval/concretize/conc.good index e3f3d2ad5..b3250b486 100644 --- a/crux-mir/test/symb_eval/concretize/conc.good +++ b/crux-mir/test/symb_eval/concretize/conc.good @@ -1,3 +1,3 @@ -test conc/86e91a75::crux_test[0]: returned (1, 0), ok +test conc/9dcb4d77::crux_test[0]: returned (1, 0), ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/concretize/no_conc.good b/crux-mir/test/symb_eval/concretize/no_conc.good index fdf6884dd..a1baa2660 100644 --- a/crux-mir/test/symb_eval/concretize/no_conc.good +++ b/crux-mir/test/symb_eval/concretize/no_conc.good @@ -1,3 +1,3 @@ -test no_conc/27ee0b55::crux_test[0]: returned (Symbolic BV, Symbolic BV), ok +test no_conc/65b2bc21::crux_test[0]: returned (Symbolic BV, Symbolic BV), ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/crux/early_fail.good b/crux-mir/test/symb_eval/crux/early_fail.good index 036a6003d..fc064b12b 100644 --- a/crux-mir/test/symb_eval/crux/early_fail.good +++ b/crux-mir/test/symb_eval/crux/early_fail.good @@ -1,16 +1,16 @@ -test early_fail/233440b0::fail1[0]: FAILED -test early_fail/233440b0::fail2[0]: FAILED +test early_fail/9e7e7cc7::fail1[0]: FAILED +test early_fail/9e7e7cc7::fail2[0]: FAILED failures: ----- early_fail/233440b0::fail1[0] counterexamples ---- +---- early_fail/9e7e7cc7::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] internal: error: in early_fail/233440b0::fail1[0] -[Crux] panicking::panic_fmt, called from early_fail/233440b0::fail1[0] +[Crux] internal: error: in early_fail/9e7e7cc7::fail1[0] +[Crux] panicking::panic_fmt, called from early_fail/9e7e7cc7::fail1[0] ----- early_fail/233440b0::fail2[0] counterexamples ---- +---- early_fail/9e7e7cc7::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/early_fail.rs:17:5: 17:29: error: in early_fail/233440b0::fail2[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/early_fail.rs:17:5: 17:29: error: in early_fail/9e7e7cc7::fail2[0] [Crux] MIR assertion at test/symb_eval/crux/early_fail.rs:17:5: [Crux] x == 0 diff --git a/crux-mir/test/symb_eval/crux/fail_return.good b/crux-mir/test/symb_eval/crux/fail_return.good index a4cb6b748..08690aff0 100644 --- a/crux-mir/test/symb_eval/crux/fail_return.good +++ b/crux-mir/test/symb_eval/crux/fail_return.good @@ -1,23 +1,23 @@ -test fail_return/10c6efbb::fail1[0]: returned Symbolic BV, FAILED -test fail_return/10c6efbb::fail2[0]: returned 123, FAILED +test fail_return/b8a81eca::fail1[0]: returned Symbolic BV, FAILED +test fail_return/b8a81eca::fail2[0]: returned 123, FAILED failures: ----- fail_return/10c6efbb::fail1[0] counterexamples ---- +---- fail_return/b8a81eca::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/crux/fail_return.rs:8:22: 8:27: error: in fail_return/10c6efbb::fail1[0] +[Crux] test/symb_eval/crux/fail_return.rs:8:22: 8:27: error: in fail_return/b8a81eca::fail1[0] [Crux] attempt to compute `move _4 + const 1_u8`, which would overflow [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/fail_return.rs:8:5: 8:32: error: in fail_return/10c6efbb::fail1[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/fail_return.rs:8:5: 8:32: error: in fail_return/b8a81eca::fail1[0] [Crux] MIR assertion at test/symb_eval/crux/fail_return.rs:8:5: [Crux] x + 1 > x ----- fail_return/10c6efbb::fail2[0] counterexamples ---- +---- fail_return/b8a81eca::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/crux/fail_return.rs:15:22: 15:27: error: in fail_return/10c6efbb::fail2[0] +[Crux] test/symb_eval/crux/fail_return.rs:15:22: 15:27: error: in fail_return/b8a81eca::fail2[0] [Crux] attempt to compute `move _5 + const 1_u8`, which would overflow [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/fail_return.rs:15:5: 15:32: error: in fail_return/10c6efbb::fail2[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/fail_return.rs:15:5: 15:32: error: in fail_return/b8a81eca::fail2[0] [Crux] MIR assertion at test/symb_eval/crux/fail_return.rs:15:5: [Crux] x + 1 > x diff --git a/crux-mir/test/symb_eval/crux/mixed_fail.good b/crux-mir/test/symb_eval/crux/mixed_fail.good index 918bdf3ed..33e7ec2b4 100644 --- a/crux-mir/test/symb_eval/crux/mixed_fail.good +++ b/crux-mir/test/symb_eval/crux/mixed_fail.good @@ -1,25 +1,25 @@ -test mixed_fail/0b094ced::fail1[0]: FAILED -test mixed_fail/0b094ced::fail2[0]: FAILED -test mixed_fail/0b094ced::pass1[0]: ok -test mixed_fail/0b094ced::pass2[0]: ok +test mixed_fail/b1edd9fb::fail1[0]: FAILED +test mixed_fail/b1edd9fb::fail2[0]: FAILED +test mixed_fail/b1edd9fb::pass1[0]: ok +test mixed_fail/b1edd9fb::pass2[0]: ok failures: ----- mixed_fail/0b094ced::fail1[0] counterexamples ---- +---- mixed_fail/b1edd9fb::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/crux/mixed_fail.rs:8:22: 8:27: error: in mixed_fail/0b094ced::fail1[0] +[Crux] test/symb_eval/crux/mixed_fail.rs:8:22: 8:27: error: in mixed_fail/b1edd9fb::fail1[0] [Crux] attempt to compute `move _5 + const 1_u8`, which would overflow [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/mixed_fail.rs:8:5: 8:32: error: in mixed_fail/0b094ced::fail1[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/mixed_fail.rs:8:5: 8:32: error: in mixed_fail/b1edd9fb::fail1[0] [Crux] MIR assertion at test/symb_eval/crux/mixed_fail.rs:8:5: [Crux] x + 1 > x ----- mixed_fail/0b094ced::fail2[0] counterexamples ---- +---- mixed_fail/b1edd9fb::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/crux/mixed_fail.rs:14:22: 14:27: error: in mixed_fail/0b094ced::fail2[0] +[Crux] test/symb_eval/crux/mixed_fail.rs:14:22: 14:27: error: in mixed_fail/b1edd9fb::fail2[0] [Crux] attempt to compute `move _5 + const 2_u8`, which would overflow [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/mixed_fail.rs:14:5: 14:32: error: in mixed_fail/0b094ced::fail2[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/mixed_fail.rs:14:5: 14:32: error: in mixed_fail/b1edd9fb::fail2[0] [Crux] MIR assertion at test/symb_eval/crux/mixed_fail.rs:14:5: [Crux] x + 2 > x diff --git a/crux-mir/test/symb_eval/crux/multi.good b/crux-mir/test/symb_eval/crux/multi.good index eb87b1f1a..22733dbdd 100644 --- a/crux-mir/test/symb_eval/crux/multi.good +++ b/crux-mir/test/symb_eval/crux/multi.good @@ -1,26 +1,26 @@ -test multi/017eda4c::fail1[0]: FAILED -test multi/017eda4c::fail2[0]: FAILED -test multi/017eda4c::fail3[0]: FAILED +test multi/e2c1a28d::fail1[0]: FAILED +test multi/e2c1a28d::fail2[0]: FAILED +test multi/e2c1a28d::fail3[0]: FAILED failures: ----- multi/017eda4c::fail1[0] counterexamples ---- +---- multi/e2c1a28d::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/crux/multi.rs:8:22: 8:27: error: in multi/017eda4c::fail1[0] +[Crux] test/symb_eval/crux/multi.rs:8:22: 8:27: error: in multi/e2c1a28d::fail1[0] [Crux] attempt to compute `move _5 + const 1_u8`, which would overflow [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/multi.rs:8:5: 8:32: error: in multi/017eda4c::fail1[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/multi.rs:8:5: 8:32: error: in multi/e2c1a28d::fail1[0] [Crux] MIR assertion at test/symb_eval/crux/multi.rs:8:5: [Crux] x + 1 > x ----- multi/017eda4c::fail2[0] counterexamples ---- +---- multi/e2c1a28d::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] internal: error: in multi/017eda4c::fail2[0] -[Crux] panicking::panic_fmt, called from multi/017eda4c::fail2[0] +[Crux] internal: error: in multi/e2c1a28d::fail2[0] +[Crux] panicking::panic_fmt, called from multi/e2c1a28d::fail2[0] ----- multi/017eda4c::fail3[0] counterexamples ---- +---- multi/e2c1a28d::fail3[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/multi.rs:20:5: 20:29: error: in multi/017eda4c::assert_zero[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/multi.rs:20:5: 20:29: error: in multi/e2c1a28d::assert_zero[0] [Crux] MIR assertion at test/symb_eval/crux/multi.rs:20:5: [Crux] x == 0 diff --git a/crux-mir/test/symb_eval/crypto/bytes.good b/crux-mir/test/symb_eval/crypto/bytes.good index 03b9a1559..788f51b23 100644 --- a/crux-mir/test/symb_eval/crypto/bytes.good +++ b/crux-mir/test/symb_eval/crypto/bytes.good @@ -1,26 +1,26 @@ -test bytes/0910acd4::f[0]: FAILED +test bytes/55748dbe::f[0]: FAILED failures: ----- bytes/0910acd4::f[0] counterexamples ---- +---- bytes/55748dbe::f[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/0910acd4::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/55748dbe::f[0] [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: [Crux] a[i] == b[i] [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/0910acd4::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/55748dbe::f[0] [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: [Crux] a[i] == b[i] [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/0910acd4::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/55748dbe::f[0] [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: [Crux] a[i] == b[i] [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/0910acd4::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/55748dbe::f[0] [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: [Crux] a[i] == b[i] [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/0910acd4::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/55748dbe::f[0] [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: [Crux] a[i] == b[i] diff --git a/crux-mir/test/symb_eval/crypto/bytes2.good b/crux-mir/test/symb_eval/crypto/bytes2.good index 567b95ba6..d3d14d16e 100644 --- a/crux-mir/test/symb_eval/crypto/bytes2.good +++ b/crux-mir/test/symb_eval/crypto/bytes2.good @@ -1,3 +1,3 @@ -test bytes2/4bb09208::f[0]: ok +test bytes2/595e306d::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/crypto/double.good b/crux-mir/test/symb_eval/crypto/double.good index 34ac8fbec..9ba9b704b 100644 --- a/crux-mir/test/symb_eval/crypto/double.good +++ b/crux-mir/test/symb_eval/crypto/double.good @@ -1,3 +1,3 @@ -test double/872a3d11::f[0]: ok +test double/1b95a34b::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/crypto/ffs.good b/crux-mir/test/symb_eval/crypto/ffs.good index 0c4ead1a8..b39d58de3 100644 --- a/crux-mir/test/symb_eval/crypto/ffs.good +++ b/crux-mir/test/symb_eval/crypto/ffs.good @@ -1,3 +1,3 @@ -test ffs/fd27844b::f[0]: ok +test ffs/28606b89::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/enum/mux.good b/crux-mir/test/symb_eval/enum/mux.good index 65fce7fa1..20f7f6db8 100644 --- a/crux-mir/test/symb_eval/enum/mux.good +++ b/crux-mir/test/symb_eval/enum/mux.good @@ -1,3 +1,3 @@ -test mux/48499f53::test[0]: ok +test mux/2b41b8c9::test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/fnptr/mux.good b/crux-mir/test/symb_eval/fnptr/mux.good index dc9293257..f57657e49 100644 --- a/crux-mir/test/symb_eval/fnptr/mux.good +++ b/crux-mir/test/symb_eval/fnptr/mux.good @@ -1,3 +1,3 @@ -test mux/08dfe1f8::crux_test[0]: returned 2, ok +test mux/a669714b::crux_test[0]: returned 2, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/io/vec_write.good b/crux-mir/test/symb_eval/io/vec_write.good index 7770b625b..7e1bf9697 100644 --- a/crux-mir/test/symb_eval/io/vec_write.good +++ b/crux-mir/test/symb_eval/io/vec_write.good @@ -1,3 +1,3 @@ -test vec_write/ee788fed::f[0]: ok +test vec_write/e955fa73::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/mux/array.good b/crux-mir/test/symb_eval/mux/array.good index b0f2799e2..bd143e4e0 100644 --- a/crux-mir/test/symb_eval/mux/array.good +++ b/crux-mir/test/symb_eval/mux/array.good @@ -1,3 +1,3 @@ -test array/283f281e::crux_test[0]: returned Symbolic BV, ok +test array/c70863b9::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/mux/array_mut.good b/crux-mir/test/symb_eval/mux/array_mut.good index 108141347..d719ba6ca 100644 --- a/crux-mir/test/symb_eval/mux/array_mut.good +++ b/crux-mir/test/symb_eval/mux/array_mut.good @@ -1,3 +1,3 @@ -test array_mut/78781ed1::crux_test[0]: returned Symbolic BV, ok +test array_mut/faf5a48b::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/num/checked_add.good b/crux-mir/test/symb_eval/num/checked_add.good index 2a1c040a1..6f515fc09 100644 --- a/crux-mir/test/symb_eval/num/checked_add.good +++ b/crux-mir/test/symb_eval/num/checked_add.good @@ -1,10 +1,10 @@ -test checked_add/00902a32::crux_test[0]: returned 44, FAILED +test checked_add/948ef81f::crux_test[0]: returned 44, FAILED failures: ----- checked_add/00902a32::crux_test[0] counterexamples ---- +---- checked_add/948ef81f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_add.rs:6:5: 6:12: error: in checked_add/00902a32::crux_test[0] +[Crux] test/symb_eval/num/checked_add.rs:6:5: 6:12: error: in checked_add/948ef81f::crux_test[0] [Crux] attempt to compute `const 100_u8 + move _2`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/num/checked_div.good b/crux-mir/test/symb_eval/num/checked_div.good index 9f8a448fe..deabab3e6 100644 --- a/crux-mir/test/symb_eval/num/checked_div.good +++ b/crux-mir/test/symb_eval/num/checked_div.good @@ -1,16 +1,16 @@ -test checked_div/1a8a798f::crux_test[0]: returned 65, FAILED +test checked_div/85dae97d::crux_test[0]: returned 65, FAILED failures: ----- checked_div/1a8a798f::crux_test[0] counterexamples ---- +---- checked_div/85dae97d::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/1a8a798f::div_signed[0] +[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/85dae97d::div_signed[0] [Crux] attempt to compute `_3 / _4`, which would overflow [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/1a8a798f::div_signed[0] +[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/85dae97d::div_signed[0] [Crux] attempt to divide `_3` by zero [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/1a8a798f::div_unsigned[0] +[Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/85dae97d::div_unsigned[0] [Crux] attempt to divide `_3` by zero [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/num/checked_mul.good b/crux-mir/test/symb_eval/num/checked_mul.good index 8048f0231..ce3335b79 100644 --- a/crux-mir/test/symb_eval/num/checked_mul.good +++ b/crux-mir/test/symb_eval/num/checked_mul.good @@ -1,10 +1,10 @@ -test checked_mul/aca814e3::crux_test[0]: returned 44, FAILED +test checked_mul/0eadbb9f::crux_test[0]: returned 44, FAILED failures: ----- checked_mul/aca814e3::crux_test[0] counterexamples ---- +---- checked_mul/0eadbb9f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_mul.rs:6:5: 6:12: error: in checked_mul/aca814e3::crux_test[0] +[Crux] test/symb_eval/num/checked_mul.rs:6:5: 6:12: error: in checked_mul/0eadbb9f::crux_test[0] [Crux] attempt to compute `const 100_u8 * move _2`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/num/checked_mul_signed.good b/crux-mir/test/symb_eval/num/checked_mul_signed.good index 1ee8e5654..ecea74575 100644 --- a/crux-mir/test/symb_eval/num/checked_mul_signed.good +++ b/crux-mir/test/symb_eval/num/checked_mul_signed.good @@ -1,10 +1,10 @@ -test checked_mul_signed/2f330b6c::crux_test[0]: returned 44, FAILED +test checked_mul_signed/6750aee6::crux_test[0]: returned 44, FAILED failures: ----- checked_mul_signed/2f330b6c::crux_test[0] counterexamples ---- +---- checked_mul_signed/6750aee6::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_mul_signed.rs:6:5: 6:13: error: in checked_mul_signed/2f330b6c::crux_test[0] +[Crux] test/symb_eval/num/checked_mul_signed.rs:6:5: 6:13: error: in checked_mul_signed/6750aee6::crux_test[0] [Crux] attempt to compute `const -100_i8 * move _2`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/overrides/bad_symb1.good b/crux-mir/test/symb_eval/overrides/bad_symb1.good index 775c4e29a..a80153382 100644 --- a/crux-mir/test/symb_eval/overrides/bad_symb1.good +++ b/crux-mir/test/symb_eval/overrides/bad_symb1.good @@ -1,10 +1,10 @@ -test bad_symb1/92726a7e::crux_test[0]: FAILED +test bad_symb1/3a80a96c::crux_test[0]: FAILED failures: ----- bad_symb1/92726a7e::crux_test[0] counterexamples ---- +---- bad_symb1/3a80a96c::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] lib/crucible/symbolic.rs:24:64: 24:68 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/6430d6c4::symbolic[0]::{impl#1}[0]::symbolic[0] +[Crux] lib/crucible/symbolic.rs:24:64: 24:68 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/4ab3f93d::symbolic[0]::{impl#1}[0]::symbolic[0] [Crux] symbolic variable name must be a concrete string [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/overrides/bad_symb2.good b/crux-mir/test/symb_eval/overrides/bad_symb2.good index ec5d5a482..4808605e0 100644 --- a/crux-mir/test/symb_eval/overrides/bad_symb2.good +++ b/crux-mir/test/symb_eval/overrides/bad_symb2.good @@ -1,10 +1,10 @@ -test bad_symb2/4edd5907::crux_test[0]: FAILED +test bad_symb2/e656e2a4::crux_test[0]: FAILED failures: ----- bad_symb2/4edd5907::crux_test[0] counterexamples ---- +---- bad_symb2/e656e2a4::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] lib/crucible/symbolic.rs:24:64: 24:68 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/6430d6c4::symbolic[0]::{impl#1}[0]::symbolic[0] +[Crux] lib/crucible/symbolic.rs:24:64: 24:68 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/4ab3f93d::symbolic[0]::{impl#1}[0]::symbolic[0] [Crux] invalid symbolic variable name "\NUL:., /": Identifier must start with a letter. [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/overrides/override1.good b/crux-mir/test/symb_eval/overrides/override1.good index 99b4231d9..3f054d543 100644 --- a/crux-mir/test/symb_eval/overrides/override1.good +++ b/crux-mir/test/symb_eval/overrides/override1.good @@ -1,4 +1,4 @@ -test override1/59e81ac9::f[0]: Hello, I'm an override +test override1/3ff54103::f[0]: Hello, I'm an override returned 2, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/overrides/override2.good b/crux-mir/test/symb_eval/overrides/override2.good index 46e8f9344..2fcf8dd4a 100644 --- a/crux-mir/test/symb_eval/overrides/override2.good +++ b/crux-mir/test/symb_eval/overrides/override2.good @@ -1,10 +1,10 @@ -test override2/cadd74c9::f[0]: FAILED +test override2/6aa6cea9::f[0]: FAILED failures: ----- override2/cadd74c9::f[0] counterexamples ---- +---- override2/6aa6cea9::f[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/overrides/override2.rs:9:5: 9:49: error: in override2/cadd74c9::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/overrides/override2.rs:9:5: 9:49: error: in override2/6aa6cea9::f[0] [Crux] MIR assertion at test/symb_eval/overrides/override2.rs:9:5: [Crux] foo.wrapping_add(1) == foo diff --git a/crux-mir/test/symb_eval/overrides/override3.good b/crux-mir/test/symb_eval/overrides/override3.good index 00eb584ab..7b9cadfd8 100644 --- a/crux-mir/test/symb_eval/overrides/override3.good +++ b/crux-mir/test/symb_eval/overrides/override3.good @@ -1,3 +1,3 @@ -test override3/503ceed2::f[0]: ok +test override3/a07e766d::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/overrides/override4.good b/crux-mir/test/symb_eval/overrides/override4.good index acdc6b932..261847069 100644 --- a/crux-mir/test/symb_eval/overrides/override4.good +++ b/crux-mir/test/symb_eval/overrides/override4.good @@ -1,3 +1,3 @@ -test override4/8240a34c::f[0]: ok +test override4/4e5d7bb4::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/overrides/override5.good b/crux-mir/test/symb_eval/overrides/override5.good index 29687eb77..9281d7301 100644 --- a/crux-mir/test/symb_eval/overrides/override5.good +++ b/crux-mir/test/symb_eval/overrides/override5.good @@ -1,10 +1,10 @@ -test override5/31b4d207::f[0]: FAILED +test override5/48be6bb4::f[0]: FAILED failures: ----- override5/31b4d207::f[0] counterexamples ---- +---- override5/48be6bb4::f[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/overrides/override5.rs:10:5: 10:47: error: in override5/31b4d207::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/overrides/override5.rs:10:5: 10:47: error: in override5/48be6bb4::f[0] [Crux] MIR assertion at test/symb_eval/overrides/override5.rs:10:5: [Crux] foo.wrapping_add(1) != 0 diff --git a/crux-mir/test/symb_eval/overrides/override_rust.good b/crux-mir/test/symb_eval/overrides/override_rust.good index c26c1a437..35cfb1f01 100644 --- a/crux-mir/test/symb_eval/overrides/override_rust.good +++ b/crux-mir/test/symb_eval/overrides/override_rust.good @@ -1,3 +1,3 @@ -test override_rust/3f867a7c::crux_test[0]: ok +test override_rust/cfc3a2ca::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/refs/mux_init_imm.good b/crux-mir/test/symb_eval/refs/mux_init_imm.good index 37931b749..bc2db2185 100644 --- a/crux-mir/test/symb_eval/refs/mux_init_imm.good +++ b/crux-mir/test/symb_eval/refs/mux_init_imm.good @@ -1,3 +1,3 @@ -test mux_init_imm/af1ab5c2::crux_test[0]: returned Symbolic BV, ok +test mux_init_imm/a53a5a3d::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/refs/mux_init_mut.good b/crux-mir/test/symb_eval/refs/mux_init_mut.good index 952e82e81..7bb9f588e 100644 --- a/crux-mir/test/symb_eval/refs/mux_init_mut.good +++ b/crux-mir/test/symb_eval/refs/mux_init_mut.good @@ -1,3 +1,3 @@ -test mux_init_mut/ba52aab3::crux_test[0]: returned Symbolic BV, ok +test mux_init_mut/7e854109::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/scalar/test1.good b/crux-mir/test/symb_eval/scalar/test1.good index 0c2afaefb..1586648de 100644 --- a/crux-mir/test/symb_eval/scalar/test1.good +++ b/crux-mir/test/symb_eval/scalar/test1.good @@ -1,3 +1,3 @@ -test test1/38b24463::f[0]: ok +test test1/5496ba4f::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/sym_bytes/construct.good b/crux-mir/test/symb_eval/sym_bytes/construct.good index c14372be6..2d29e549f 100644 --- a/crux-mir/test/symb_eval/sym_bytes/construct.good +++ b/crux-mir/test/symb_eval/sym_bytes/construct.good @@ -1,10 +1,10 @@ -test construct/961e3d80::crux_test[0]: returned Symbolic BV, FAILED +test construct/55406696::crux_test[0]: returned Symbolic BV, FAILED failures: ----- construct/961e3d80::crux_test[0] counterexamples ---- +---- construct/55406696::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/sym_bytes/construct.rs:13:5: 13:35: error: in construct/961e3d80::crux_test[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/sym_bytes/construct.rs:13:5: 13:35: error: in construct/55406696::crux_test[0] [Crux] MIR assertion at test/symb_eval/sym_bytes/construct.rs:13:5: [Crux] sym2[0] == 0 diff --git a/crux-mir/test/symb_eval/sym_bytes/deserialize.good b/crux-mir/test/symb_eval/sym_bytes/deserialize.good index 43e3a3443..8c63325d3 100644 --- a/crux-mir/test/symb_eval/sym_bytes/deserialize.good +++ b/crux-mir/test/symb_eval/sym_bytes/deserialize.good @@ -1,10 +1,10 @@ -test deserialize/9453da6c::crux_test[0]: returned Symbolic BV, FAILED +test deserialize/2a8b0d13::crux_test[0]: returned Symbolic BV, FAILED failures: ----- deserialize/9453da6c::crux_test[0] counterexamples ---- +---- deserialize/2a8b0d13::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/sym_bytes/deserialize.rs:19:21: 19:44: error: in deserialize/9453da6c::deserialize[0] +[Crux] test/symb_eval/sym_bytes/deserialize.rs:19:21: 19:44: error: in deserialize/2a8b0d13::deserialize[0] [Crux] attempt to compute `move _61 + move _62`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/vec/clone.good b/crux-mir/test/symb_eval/vec/clone.good index 2d7417a69..e0bfa9d43 100644 --- a/crux-mir/test/symb_eval/vec/clone.good +++ b/crux-mir/test/symb_eval/vec/clone.good @@ -1,3 +1,3 @@ -test clone/3a1fbbbh::f[0]: ok +test clone/e141ef81::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vec/into_iter.good b/crux-mir/test/symb_eval/vec/into_iter.good index d3b7714b9..9eea163b4 100644 --- a/crux-mir/test/symb_eval/vec/into_iter.good +++ b/crux-mir/test/symb_eval/vec/into_iter.good @@ -1,3 +1,3 @@ -test into_iter/3a1fbbbh::f[0]: ok +test into_iter/82ce8ccb::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vec/macro.good b/crux-mir/test/symb_eval/vec/macro.good index 814167275..6ba0c94c2 100644 --- a/crux-mir/test/symb_eval/vec/macro.good +++ b/crux-mir/test/symb_eval/vec/macro.good @@ -1,3 +1,3 @@ -test macro/3a1fbbbh::f[0]: ok +test macro/a1425499::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vec/sort_by_key.good b/crux-mir/test/symb_eval/vec/sort_by_key.good index 59f459b68..bbbe037aa 100644 --- a/crux-mir/test/symb_eval/vec/sort_by_key.good +++ b/crux-mir/test/symb_eval/vec/sort_by_key.good @@ -1,3 +1,3 @@ -test sort_by_key/3a1fbbbh::f[0]: ok +test sort_by_key/c5f23e33::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/as_mut_slice.good b/crux-mir/test/symb_eval/vector/as_mut_slice.good index c8c97f037..d723038f5 100644 --- a/crux-mir/test/symb_eval/vector/as_mut_slice.good +++ b/crux-mir/test/symb_eval/vector/as_mut_slice.good @@ -1,3 +1,3 @@ -test as_mut_slice/2fb2b2e1::f[0]: ok +test as_mut_slice/caa8fcae::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/as_slice.good b/crux-mir/test/symb_eval/vector/as_slice.good index d385b1f74..9ac9b6c81 100644 --- a/crux-mir/test/symb_eval/vector/as_slice.good +++ b/crux-mir/test/symb_eval/vector/as_slice.good @@ -1,3 +1,3 @@ -test as_slice/c1cfd521::f[0]: ok +test as_slice/71764b65::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/concat.good b/crux-mir/test/symb_eval/vector/concat.good index cb93061db..237177715 100644 --- a/crux-mir/test/symb_eval/vector/concat.good +++ b/crux-mir/test/symb_eval/vector/concat.good @@ -1,3 +1,3 @@ -test concat/580b9ab8::f[0]: ok +test concat/a65f2287::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/copy_from_slice.good b/crux-mir/test/symb_eval/vector/copy_from_slice.good index 46a80119e..89ed1ebef 100644 --- a/crux-mir/test/symb_eval/vector/copy_from_slice.good +++ b/crux-mir/test/symb_eval/vector/copy_from_slice.good @@ -1,3 +1,3 @@ -test copy_from_slice/43f9bf00::f[0]: ok +test copy_from_slice/9cf6b878::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/mut.good b/crux-mir/test/symb_eval/vector/mut.good index 798ccfca5..5cffb915e 100644 --- a/crux-mir/test/symb_eval/vector/mut.good +++ b/crux-mir/test/symb_eval/vector/mut.good @@ -1,3 +1,3 @@ -test mut/33f66d86::f[0]: ok +test mut/8d39d367::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/new.good b/crux-mir/test/symb_eval/vector/new.good index b40921c17..1fe5985b8 100644 --- a/crux-mir/test/symb_eval/vector/new.good +++ b/crux-mir/test/symb_eval/vector/new.good @@ -1,3 +1,3 @@ -test new/b6f18315::f[0]: ok +test new/a9927192::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/push.good b/crux-mir/test/symb_eval/vector/push.good index 85b6630a9..75e530d8b 100644 --- a/crux-mir/test/symb_eval/vector/push.good +++ b/crux-mir/test/symb_eval/vector/push.good @@ -1,3 +1,3 @@ -test push/6f8aba04::f[0]: ok +test push/cb01e03b::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/replicate.good b/crux-mir/test/symb_eval/vector/replicate.good index e495481ad..a598e00fc 100644 --- a/crux-mir/test/symb_eval/vector/replicate.good +++ b/crux-mir/test/symb_eval/vector/replicate.good @@ -1,3 +1,3 @@ -test replicate/fc4d884f::f[0]: ok +test replicate/1d9f8903::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/split_at.good b/crux-mir/test/symb_eval/vector/split_at.good index 8c4b42226..ecedbd0f9 100644 --- a/crux-mir/test/symb_eval/vector/split_at.good +++ b/crux-mir/test/symb_eval/vector/split_at.good @@ -1,3 +1,3 @@ -test split_at/cf146783::f[0]: ok +test split_at/9315f323::f[0]: ok [Crux] Overall status: Valid. From e9dac2860a36b488dac042b531e4ff245045ed6a Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Fri, 26 May 2023 19:06:56 -0400 Subject: [PATCH 071/114] Update test_output.log --- crux-mir/test_output.log | 3022 +++++++++++++++++--------------------- 1 file changed, 1317 insertions(+), 1705 deletions(-) diff --git a/crux-mir/test_output.log b/crux-mir/test_output.log index 6dea76706..ea31085b4 100644 --- a/crux-mir/test_output.log +++ b/crux-mir/test_output.log @@ -1,1500 +1,906 @@ crux-mir crux concrete - clos - fn_static_poly: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 3 (0.85s) - Crux output: 3 - fnptr_closure: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: 7 (0.87s) - Crux output: 7 - fnptr_fn: OK (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: 4 (0.87s) - Crux output: 4 - conv_fnonce_fnmut: OK (0.99s) + refs + promoted_imm: OK (1.94s) Compiling and running oracle program (0.14s) - Oracle output: 2 (0.86s) - Crux output: 2 - as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (1.04s) - Compiling and running oracle program (0.12s) - Oracle output: 1 (0.92s) - Crux output: - failures: - - ---- as_fn_ptr/c6e389f5::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/c6e389f5::crux_test[0] - [Crux] Translation error in as_fn_ptr/c6e389f5::crux_test[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [], _fsreturn_ty = TyInt B32, _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - (expected failure) - fnonce1: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.85s) - Crux output: 3 - direct_fn: OK (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.86s) - Crux output: 2 - fn_static: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.88s) - Crux output: 3 - poly: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.84s) - Crux output: 3 - direct_fnmut: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: 7 (0.87s) - Crux output: 7 - direct_fnmut2: OK (1.02s) - Compiling and running oracle program (0.13s) - Oracle output: 7 (0.89s) - Crux output: 7 - promoted: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 33 (0.86s) - Crux output: 33 - direct_fnonce: OK (1.05s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.93s) - Crux output: 2 - fo: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: 29 (0.83s) - Crux output: 29 - fnonce: OK (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: false (0.86s) - Crux output: false - fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.86s) - vtable signature mismatch for vtable core/beffc212::ops[0]::function[0]::Fn[0]::_vtbl570ffd80324c2ca9[0] of trait core/beffc212::ops[0]::function[0]::Fn[0]::_trait42f8b6475ee85722[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32)] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32)] - CallStack (from HasCallStack): - error, called at src/Mir/Trans.hs:933:20 in crucible-mir-0.1-inplace:Mir.Trans - mkTraitObject, called at src/Mir/Trans.hs:776:11 in crucible-mir-0.1-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:907:5 in crucible-mir-0.1-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:959:30 in crucible-mir-0.1-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1172:11 in crucible-mir-0.1-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1617:68 in crucible-mir-0.1-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1626:28 in crucible-mir-0.1-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1666:10 in crucible-mir-0.1-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1712:24 in crucible-mir-0.1-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2146:30 in crucible-mir-0.1-inplace:Mir.Trans - transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate - translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language - (expected failure) - ref_fnmut: OK (1.03s) + Oracle output: 0 (1.80s) + Crux output: 0 + mut_raw: OK (2.01s) + Compiling and running oracle program (0.14s) + Oracle output: 123 (1.86s) + Crux output: 123 + mut_nested: OK (2.00s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.84s) + Crux output: () + fn_ptr_mut: OK (2.01s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.85s) + Crux output: 1 + never: OK (2.04s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.89s) + Crux output: 1 + mut_ref: OK (2.11s) + Compiling and running oracle program (0.18s) + Oracle output: 1 (1.93s) + Crux output: 1 + temp: OK (2.05s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.90s) + Crux output: 1 + mut_arg: OK (1.99s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.84s) + Crux output: 1 + fn_ptr: OK (2.06s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.91s) + Crux output: 1 + promoted_mut: OK (2.09s) Compiling and running oracle program (0.17s) - Oracle output: 7 (0.87s) - Crux output: 7 - dispatch_fnmut: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: 7 (0.88s) - Crux output: 7 - unique_borrow: OK (1.00s) - Compiling and running oracle program (0.12s) - Oracle output: 3 (0.87s) - Crux output: 3 - conv_fnmut_fn: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 2 (0.85s) - Crux output: 2 - fnptr_fnmut: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: 4 (0.87s) - Crux output: 4 - fnptr_fnonce: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 4 (0.84s) - Crux output: 4 - conv_fnonce_fn: OK (0.98s) - Compiling and running oracle program (0.12s) - Oracle output: 2 (0.85s) - Crux output: 2 - prim - bool: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: false (0.83s) - Crux output: false - litbstring: OK (0.99s) - Compiling and running oracle program (0.12s) - Oracle output: true (0.87s) - Crux output: true - shift3: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: -9223372036854775808 (0.84s) - Crux output: -9223372036854775808 - litstring: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: true (0.85s) - Crux output: true - ge: OK (0.96s) - Compiling and running oracle program (0.12s) - Oracle output: true (0.84s) - Crux output: true - shift_exceeding: rustc compilation failed for shift_exceeding -error output: -error[E0425]: cannot find function `catch_unwind` in module `panic` - --> test/conc_eval/prim/shift_exceeding.rs:14:25 - | -14 | let result = panic::catch_unwind(|| { - | ^^^^^^^^^^^^ not found in `panic` - | -help: consider importing this function - | -3 | use std::panic::catch_unwind; - | -help: if you import `catch_unwind`, refer to it directly - | -14 - let result = panic::catch_unwind(|| { -14 + let result = catch_unwind(|| { - | - -warning: lint `exceeding_bitshifts` has been renamed to `arithmetic_overflow` - --> test/conc_eval/prim/shift_exceeding.rs:4:9 - | -4 | #[allow(exceeding_bitshifts)] - | ^^^^^^^^^^^^^^^^^^^ help: use the new name: `arithmetic_overflow` - | - = note: `#[warn(renamed_and_removed_lints)]` on by default - -error: aborting due to previous error; 1 warning emitted - -For more information about this error, try `rustc --explain E0425`. - -FAIL (expected: Should panic, but doesn't) (0.05s) - Compiling and running oracle program (0.05s) - test/Test.hs:107: - failed to compile and run - (expected failure) - shift4: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: -9223372036854775808 (0.85s) - Crux output: -9223372036854775808 - mut_arg: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.85s) + Oracle output: 0 (1.92s) + Crux output: 0 + never_mut: OK (2.09s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.94s) Crux output: 1 - wrapping_sub: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: true (0.84s) - Crux output: true - lit: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: false (0.85s) - Crux output: false - mut: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 14 (0.85s) - Crux output: 14 - shift1: OK (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.86s) - Crux output: 2 - add1: OK (0.96s) - Compiling and running oracle program (0.12s) - Oracle output: 2 (0.84s) - Crux output: 2 - div: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 4 (0.84s) - Crux output: 4 - shift2: OK (0.98s) - Compiling and running oracle program (0.12s) - Oracle output: 2 (0.86s) + mut_tuple_field: OK (1.98s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.84s) + Crux output: () + imm_raw: OK (2.01s) + Compiling and running oracle program (0.15s) + Oracle output: 123 (1.86s) + Crux output: 123 + static_mut: OK (2.05s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (1.89s) Crux output: 2 - ffs: OK (1.06s) - Compiling and running oracle program (0.13s) - Oracle output: true (0.93s) - Crux output: true - char_from_u32: OK (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: 'A' (0.87s) - Crux output: 'A' - traits - tyfam: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 23 (0.85s) - Crux output: 23 - dynamic_branch: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.84s) + imm_ref: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: 123 (1.76s) + Crux output: 123 + imm_arg: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.74s) Crux output: 1 - static: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 42 (0.85s) - Crux output: 42 - bounds3: OK (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.86s) + hash_map + insert_multi: OK (2.37s) + Compiling and running oracle program (0.20s) + Oracle output: 100 (2.17s) + Crux output: 100 + insert_remove: OK (2.82s) + Compiling and running oracle program (0.18s) + Oracle output: 12 (2.64s) + Crux output: 12 + insert_iter: OK (2.46s) + Compiling and running oracle program (0.20s) + Oracle output: [11, 12] (2.26s) + Crux output: [11, 12] + insert_get: OK (2.38s) + Compiling and running oracle program (0.19s) + Oracle output: (11, 12) (2.20s) + Crux output: (11, 12) + traits + params: OK (2.03s) + Compiling and running oracle program (0.16s) + Oracle output: 25 (1.88s) + Crux output: 25 + generic3: OK (2.01s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.85s) Crux output: () - tyfam3: OK (0.96s) - Compiling and running oracle program (0.12s) - Oracle output: 23 (0.84s) - Crux output: 23 - static_two: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.85s) + static_three: OK (2.05s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.90s) + Crux output: 2 + generic1: OK (1.93s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.78s) + Crux output: () + subtrait: OK (1.92s) + Compiling and running oracle program (0.14s) + Oracle output: 73 (1.77s) + Crux output: 73 + gen_trait: OK (1.86s) + Compiling and running oracle program (0.15s) + Oracle output: -69 (1.72s) + Crux output: -69 + dynamic_branch: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.74s) + Crux output: 1 + static_two: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.72s) Crux output: 1 - bounds2: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.87s) + tyfam5: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.73s) Crux output: () - generic2: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.87s) + generic2: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.75s) Crux output: () - gen_trait_poly: OK (0.98s) - Compiling and running oracle program (0.12s) - Oracle output: 12 (0.86s) - Crux output: 12 - dict_simple: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 32 (0.84s) - Crux output: 32 - static_self: OK (0.96s) - Compiling and running oracle program (0.12s) - Oracle output: 42 (0.83s) - Crux output: 42 - tyfam4: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 0 (0.85s) - Crux output: 0 - gen_trait: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: -69 (0.84s) - Crux output: -69 - generic3: OK (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.87s) + assoc3: OK (1.92s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.77s) Crux output: () - subtrait: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 73 (0.84s) - Crux output: 73 - default: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 12 (0.85s) + tyfam4: OK (1.95s) + Compiling and running oracle program (0.15s) + Oracle output: 0 (1.81s) + Crux output: 0 + static_self: OK (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.76s) + Crux output: 42 + conv: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.75s) + Crux output: 1 + dynamic_poly: OK (1.88s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (1.74s) + Crux output: 3 + dynamic_simple: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.76s) + Crux output: 1 + gen_trait_poly: OK (1.93s) + Compiling and running oracle program (0.15s) + Oracle output: 12 (1.78s) + Crux output: 12 + default: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 12 (1.74s) Crux output: 12 - bounds1: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.88s) + assoc1: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.74s) Crux output: () - intoiter: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 0 (0.86s) - Crux output: 0 - static_eq: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: true (0.85s) + subtrait2: OK (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: 64 (1.77s) + Crux output: 64 + bounds1: OK (2.05s) + Compiling and running oracle program (0.18s) + Oracle output: () (1.87s) + Crux output: () + bounds3: OK (2.02s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.86s) + Crux output: () + static_eq: OK (1.98s) + Compiling and running oracle program (0.15s) + Oracle output: true (1.82s) Crux output: true - assoc3: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.84s) + tyfam3: OK (2.03s) + Compiling and running oracle program (0.16s) + Oracle output: 23 (1.88s) + Crux output: 23 + dict_med: OK (2.03s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.88s) + Crux output: 42 + basics1: OK (2.04s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.89s) Crux output: () - dynamic_poly: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.85s) - Crux output: 3 - dynamic_med: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.86s) - Crux output: 1 - static_three: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.84s) - Crux output: 2 - dynamic_two: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.86s) - Crux output: 1 - bounds4: OK (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.87s) + assoc2: OK (1.96s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.82s) Crux output: () - bounds5: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.87s) + dict_polymem: OK (2.04s) + Compiling and running oracle program (0.15s) + Oracle output: 4 (1.89s) + Crux output: 4 + bounds2: OK (2.02s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.87s) Crux output: () - dict_med: OK (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.86s) + intoiter: OK (2.03s) + Compiling and running oracle program (0.15s) + Oracle output: 0 (1.88s) + Crux output: 0 + static: OK (2.00s) + Compiling and running oracle program (0.16s) + Oracle output: 42 (1.84s) Crux output: 42 - dynamic_simple: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.84s) + dynamic_two: OK (2.01s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.86s) Crux output: 1 - assoc1: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.88s) + bounds4: OK (1.95s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.81s) Crux output: () - params: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 25 (0.85s) - Crux output: 25 - conv: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.85s) + dict_poly: OK (2.01s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.86s) Crux output: 1 - basics1: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.87s) + dict_simple: OK (2.10s) + Compiling and running oracle program (0.16s) + Oracle output: 32 (1.94s) + Crux output: 32 + tyfam: OK (2.01s) + Compiling and running oracle program (0.15s) + Oracle output: 23 (1.86s) + Crux output: 23 + bounds5: OK (2.04s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.88s) Crux output: () - dict_poly: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.84s) - Crux output: 1 - dict_polymem: OK (0.96s) - Compiling and running oracle program (0.12s) - Oracle output: 4 (0.84s) - Crux output: 4 - subtrait2: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 64 (0.85s) - Crux output: 64 - tyfam2: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: 23 (0.84s) + tyfam2: OK (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: 23 (1.76s) Crux output: 23 - tyfam5: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.85s) + dynamic_med: OK (1.93s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.78s) + Crux output: 1 + intTest + test0039: OK (1.94s) + Compiling and running oracle program (0.15s) + Oracle output: 7 (1.79s) + Crux output: 7 + test0038: OK (1.96s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.82s) Crux output: () - assoc2: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.84s) + tuple + clone_rec: OK (1.90s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.75s) Crux output: () - generic1: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.85s) + clone: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.75s) Crux output: () - impl - self_mut: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.86s) - Crux output: 42 - self: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.84s) - Crux output: 42 - simple: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.84s) - Crux output: 42 - vec - drop: FAIL (1.19s) - Compiling and running oracle program (0.13s) - Oracle output: () (1.05s) + clone_from: FAIL (1.94s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.79s) Crux output: failures: - ---- drop/883706d7::crux_test[0] counterexamples ---- + ---- clone_from/8fca6705::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_instb944086d0b551051[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_instb944086d0b551051[0]: evalPlace (PField, Union) NYI + [Crux] ./lib/core/src/clone.rs:136:17: 136:31: error: in core/1cdf94ce::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0] + [Crux] Translation error in core/1cdf94ce::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0]: don't know how to generate CloneShim for unknown method core/1cdf94ce::clone[0]::Clone[0]::clone[0] [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/drop/' to rerun this test only. - from_elem_zero: FAIL (0.18s) - Compiling and running oracle program (0.13s) - Oracle output: 0 (0.04s) - user error (Error 101 while running mir-json on test/conc_eval/vec/from_elem_zero.rs) - - Use -p '/from_elem_zero/' to rerun this test only. - extend_trusted_len: OK (1.84s) - Compiling and running oracle program (0.14s) - Oracle output: (1, 10) (1.70s) - Crux output: (1, 10) - extend: FAIL (1.26s) - Compiling and running oracle program (0.14s) - Oracle output: (1, 10) (1.12s) + Use -p '/clone_from/' to rerun this test only. + clone_struct: FAIL (1.94s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.79s) Crux output: failures: - ---- extend/e55977a0::crux_test[0] counterexamples ---- + ---- clone_struct/8dcd765f::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in core/beffc212::iter[0]::traits[0]::iterator[0]::Iterator[0]::try_fold[0]::_inst647f2421f6e4ebd6[0] - [Crux] Translation error in core/beffc212::iter[0]::traits[0]::iterator[0]::Iterator[0]::try_fold[0]::_inst647f2421f6e4ebd6[0]: error building variant core/beffc212::ops[0]::control_flow[0]::ControlFlow[0]::Continue[0]: type mismatch: expected AnyRepr but got BVRepr 32 in field 0: ([FieldRepr (FkInit AnyRepr)],[Just bVLit(32, BV 0)]) -- [Just bVLit(32, BV 0): BVRepr 32] + [Crux] test/conc_eval/tuple/clone_struct.rs:8:13: 8:22: error: in clone_struct/8dcd765f::f[0] + [Crux] Translation error in clone_struct/8dcd765f::f[0]: don't know how to generate CloneShim for unknown method core/1cdf94ce::clone[0]::Clone[0]::clone[0] [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '$0=="crux-mir.crux concrete..vec.extend"' to rerun this test only. - set_len: FAIL (1.20s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (1.08s) + Use -p '/clone_struct/' to rerun this test only. + iter + for: OK (1.92s) + Compiling and running oracle program (0.15s) + Oracle output: 47 (1.77s) + Crux output: 47 + sum: OK (1.92s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.77s) + Crux output: () + filter_chain: OK (2.06s) + Compiling and running oracle program (0.17s) + Oracle output: 0 (1.89s) + Crux output: 0 + from_fn: OK (1.96s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.81s) + Crux output: () + peek: OK (2.06s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (1.91s) + Crux output: 3 + zip: OK (2.01s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.85s) + Crux output: () + loop: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.73s) + Crux output: 2 + cloned: OK (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: 6 (1.77s) + Crux output: 6 + str + format_hex: FAIL (2.65s) + Compiling and running oracle program (0.15s) + Oracle output: true (2.50s) Crux output: failures: - ---- set_len/225fd37f::crux_test[0] counterexamples ---- + ---- format_hex/ff25ab16::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/1cdf94ce::fmt[0]::write[0] + [Crux] Translation error in core/1cdf94ce::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/1cdf94ce::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/set_len/' to rerun this test only. - collect: FAIL (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 45 (1.72s) + Use -p '/format_hex/' to rerun this test only. + format_int: FAIL (2.62s) + Compiling and running oracle program (0.15s) + Oracle output: true (2.47s) Crux output: failures: - ---- collect/1633cf1b::crux_test[0] counterexamples ---- + ---- format_int/8280e1f2::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/ptr/const_ptr.rs:786:58: 786:72: error: in core/beffc212::ptr[0]::const_ptr[0]::{impl#0}[0]::sub_ptr[0]::runtime[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/beffc212::ptr[0]::const_ptr[0]::{impl#0}[0]::sub_ptr[0]::runtime[0]::_inst1e2825177cd3b608[0]: No translation for pointer binop: >= + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/1cdf94ce::fmt[0]::write[0] + [Crux] Translation error in core/1cdf94ce::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/1cdf94ce::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/collect/' to rerun this test only. - push: OK (1.20s) - Compiling and running oracle program (0.13s) - Oracle output: (1, 2) (1.07s) - Crux output: (1, 2) - slice - len: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: 5 (0.84s) - Crux output: 5 - mut_range: FAIL (1.52s) - Compiling and running oracle program (0.13s) - Oracle output: 86 (1.39s) + Use -p '/format_int/' to rerun this test only. + format_struct: FAIL (2.69s) + Compiling and running oracle program (0.16s) + Oracle output: true (2.53s) Crux output: failures: - ---- mut_range/b5c421ea::crux_test[0] counterexamples ---- + ---- format_struct/e77dd717::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/1cdf94ce::fmt[0]::write[0] + [Crux] Translation error in core/1cdf94ce::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/1cdf94ce::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/mut_range/' to rerun this test only. - range_len: FAIL (1.50s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (1.37s) + Use -p '/format_struct/' to rerun this test only. + format: FAIL (2.66s) + Compiling and running oracle program (0.15s) + Oracle output: true (2.51s) Crux output: failures: - ---- range_len/5665dfa7::crux_test[0] counterexamples ---- + ---- format/6d842288::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/1cdf94ce::fmt[0]::write[0] + [Crux] Translation error in core/1cdf94ce::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/1cdf94ce::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '$0=="crux-mir.crux concrete..slice.range_len"' to rerun this test only. - eq: FAIL (1.01s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.89s) + Use -p '$0=="crux-mir.crux concrete..str.format"' to rerun this test only. + to_owned: FAIL (2.09s) + Compiling and running oracle program (0.15s) + Oracle output: true (1.94s) Crux output: failures: - ---- eq/b2461ea6::f[0] counterexamples ---- + ---- to_owned/81422a79::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/mem/mod.rs:338:38: 338:41: error: in core/beffc212::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/beffc212::mem[0]::size_of_val[0]::_inst55af81a2cef2701c[0]: callExp: Don't know how to call core/beffc212::intrinsics[0]::{extern#0}[0]::size_of_val[0]::_inst55af81a2cef2701c[0] + [Crux] lib/core/src/iter/traits/iterator.rs:2607:40: 2607:61: error: in core/1cdf94ce::iter[0]::traits[0]::iterator[0]::Iterator[0]::all[0]::_instf9a28a1fbcee32be[0] + [Crux] Translation error in core/1cdf94ce::iter[0]::traits[0]::iterator[0]::Iterator[0]::all[0]::_instf9a28a1fbcee32be[0]: ill-typed assignment of IntrinsicRepr MirReference [BVRepr 8] to IntrinsicRepr MirReference [AnyRepr] (TyRef (TyAdt core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0]::_adtdfc2b6f78a281997[0] core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0] (Substs [TyTuple [],TyTuple []])) Immut) LBase (Var {_varname = "_10", _varmut = Mut, _varty = TyRef (TyAdt core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0]::_adtdfc2b6f78a281997[0] core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0] (Substs [TyTuple [],TyTuple []])) Immut, _varIsZST = False}) [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/slice.eq/' to rerun this test only. - iter_mut: FAIL (1.06s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.93s) + Use -p '/to_owned/' to rerun this test only. + string_push: FAIL (2.58s) + Compiling and running oracle program (0.16s) + Oracle output: true (2.42s) Crux output: failures: - ---- iter_mut/bf566c00::f[0] counterexamples ---- + ---- string_push/f3da471e::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/array/mod.rs:266:23: 266:54: error: in core/beffc212::array[0]::{impl#10}[0]::try_from[0]::_insta17aeaef2aec3782[0] - [Crux] Translation error in core/beffc212::array[0]::{impl#10}[0]::try_from[0]::_insta17aeaef2aec3782[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyInt B32) Immut - [Crux] as: TyRawPtr (TyArray (TyInt B32) 3) Immut + [Crux] lib/core/src/iter/traits/iterator.rs:2607:40: 2607:61: error: in core/1cdf94ce::iter[0]::traits[0]::iterator[0]::Iterator[0]::all[0]::_instf9a28a1fbcee32be[0] + [Crux] Translation error in core/1cdf94ce::iter[0]::traits[0]::iterator[0]::Iterator[0]::all[0]::_instf9a28a1fbcee32be[0]: ill-typed assignment of IntrinsicRepr MirReference [BVRepr 8] to IntrinsicRepr MirReference [AnyRepr] (TyRef (TyAdt core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0]::_adtdfc2b6f78a281997[0] core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0] (Substs [TyTuple [],TyTuple []])) Immut) LBase (Var {_varname = "_10", _varmut = Mut, _varty = TyRef (TyAdt core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0]::_adtdfc2b6f78a281997[0] core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0] (Substs [TyTuple [],TyTuple []])) Immut, _varIsZST = False}) [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/iter_mut/' to rerun this test only. - get: FAIL (1.03s) - Compiling and running oracle program (0.12s) - Oracle output: true (0.91s) + Use -p '/string_push/' to rerun this test only. + format_array: FAIL (2.73s) + Compiling and running oracle program (0.16s) + Oracle output: true (2.58s) Crux output: failures: - ---- get/56903f09::crux_test[0] counterexamples ---- + ---- format_array/5c87c800::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/1cdf94ce::fmt[0]::write[0] + [Crux] Translation error in core/1cdf94ce::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/1cdf94ce::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/slice.get/' to rerun this test only. - last: OK (0.99s) - Compiling and running oracle program (0.12s) - Oracle output: 3 (0.86s) - Crux output: 3 - range_len_mut: FAIL (1.54s) - Compiling and running oracle program (0.12s) - Oracle output: 1 (1.42s) - Crux output: - failures: + Use -p '/format_array/' to rerun this test only. + sync + mutex: FAIL (3.18s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (3.03s) + TODO RGS evalRval ThreadLocalRef + CallStack (from HasCallStack): + error, called at src/Mir/Trans.hs:1034:33 in crucible-mir-0.1-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1197:11 in crucible-mir-0.1-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1642:68 in crucible-mir-0.1-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1651:28 in crucible-mir-0.1-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1691:10 in crucible-mir-0.1-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1737:24 in crucible-mir-0.1-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2171:30 in crucible-mir-0.1-inplace:Mir.Trans + transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate + translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language - ---- range_len_mut/dc7450de::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/range_len_mut/' to rerun this test only. - mut: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.86s) - Crux output: 42 - swap: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: 2001 (0.88s) - Crux output: 2001 - mk_and_proj: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.83s) - Crux output: 42 - num - overflow: OK (1.03s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.90s) - Crux output: () - saturate: OK (1.53s) - Compiling and running oracle program (0.13s) - Oracle output: () (1.40s) - Crux output: () - from_bytes: OK (1.03s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.91s) - Crux output: () - dyn - inherit: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: 4 (0.88s) + Use -p '$0=="crux-mir.crux concrete..sync.mutex"' to rerun this test only. + arc_cell: OK (2.10s) + Compiling and running oracle program (0.16s) + Oracle output: 4 (1.94s) Crux output: 4 - plain_trait: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 100 (0.86s) - Crux output: 100 - trait_param: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: 100 (0.84s) - Crux output: 100 - assoc_ty: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 100 (0.86s) - Crux output: 100 - box - mut_ref: OK (1.13s) - Compiling and running oracle program (0.13s) - Oracle output: () (1.00s) - Crux output: () - new: OK (1.11s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.99s) - Crux output: () - struct: OK (1.10s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.98s) + arc: OK (2.08s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.94s) Crux output: 1 - mut: OK (1.09s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.96s) - Crux output: () - unsize: OK (1.12s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.99s) - Crux output: 2 - vec_deque - rotate_right: FAIL (1.75s) - Compiling and running oracle program (0.14s) - Oracle output: [4, 5, 1, 2, 3] (1.62s) - Crux output: - failures: - - ---- rotate_right/b7a01f9c::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/rotate_right/' to rerun this test only. - retain: FAIL (1.70s) - Compiling and running oracle program (0.14s) - Oracle output: [1, 4] (1.57s) - Crux output: - failures: - - ---- retain/16d9d612::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/retain/' to rerun this test only. - push: OK (1.72s) - Compiling and running oracle program (0.13s) - Oracle output: [4, 1, 2, 3, 5] (1.59s) - Crux output: [4, 1, 2, 3, 5] - rotate_left: FAIL (1.71s) - Compiling and running oracle program (0.13s) - Oracle output: [3, 4, 5, 1, 2] (1.57s) - Crux output: - failures: - - ---- rotate_left/791d5fca::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/rotate_left/' to rerun this test only. - pop: FAIL (1.62s) - Compiling and running oracle program (0.13s) - Oracle output: [5, 4, 3, 1, 2] (1.49s) - Crux output: - failures: - - ---- pop/f75a69e4::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/vec_deque.pop/' to rerun this test only. - iter_clone: FAIL (1.73s) - Compiling and running oracle program (0.14s) - Oracle output: [1, 2, 3, 2, 3] (1.60s) - Crux output: - failures: - - ---- iter_clone/fecbd0b0::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle + atomic_add: OK (1.97s) + Compiling and running oracle program (0.15s) + Oracle output: 6 (1.82s) + Crux output: 6 + rwlock: FAIL (3.25s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (3.10s) + TODO RGS evalRval ThreadLocalRef + CallStack (from HasCallStack): + error, called at src/Mir/Trans.hs:1034:33 in crucible-mir-0.1-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1197:11 in crucible-mir-0.1-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1642:68 in crucible-mir-0.1-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1651:28 in crucible-mir-0.1-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1691:10 in crucible-mir-0.1-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1737:24 in crucible-mir-0.1-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2171:30 in crucible-mir-0.1-inplace:Mir.Trans + transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate + translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language - Use -p '/iter_clone/' to rerun this test only. - iter - loop: OK (1.00s) - Compiling and running oracle program (0.12s) - Oracle output: 2 (0.87s) - Crux output: 2 - from_fn: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.88s) - Crux output: () - filter_chain: OK (1.13s) + Use -p '$0=="crux-mir.crux concrete..sync.rwlock"' to rerun this test only. + atomic_cxchg: OK (2.05s) Compiling and running oracle program (0.15s) - Oracle output: 0 (0.98s) - Crux output: 0 - for: OK (1.02s) - Compiling and running oracle program (0.13s) - Oracle output: 47 (0.90s) - Crux output: 47 - zip: OK (1.07s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.94s) - Crux output: () - peek: OK (1.07s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.94s) - Crux output: 3 - cloned: OK (1.02s) - Compiling and running oracle program (0.13s) - Oracle output: 6 (0.89s) + Oracle output: 6 (1.89s) Crux output: 6 - sum: OK (1.05s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.92s) - Crux output: () - crypto - add: OK (1.03s) - Compiling and running oracle program (0.13s) - Oracle output: true (0.90s) - Crux output: true - add_noL: OK (1.07s) + arc_clone: OK (2.11s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.96s) + Crux output: 2 + atomic_swap: OK (1.93s) Compiling and running oracle program (0.14s) - Oracle output: false (0.93s) - Crux output: false - statics - promoted_static: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.85s) - Crux output: 1 - promoted_fn: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.84s) - Crux output: 1 - ptr - offset_mut: OK (0.98s) - Compiling and running oracle program (0.12s) - Oracle output: 3 (0.86s) - Crux output: 3 - dangling_eq: FAIL (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.89s) - Crux output: - failures: - - ---- dangling_eq/d3f79519::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/ptr/metadata.rs:135:14: 135:78: error: in core/beffc212::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/dangling_eq/' to rerun this test only. - is_null_slice: FAIL (expected: can't unsize null pointers) (1.02s) - Compiling and running oracle program (0.12s) - Oracle output: (false, true) (0.89s) - Crux output: - failures: - - ---- is_null_slice/819ce3b2::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/ptr/is_null_slice.rs:7:13: 7:33: error: in is_null_slice/819ce3b2::crux_test[0] - [Crux] attempted subindex on the result of an integer-to-pointer cast - [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/ptr/const_ptr.rs:38:38: 38:42: error: in core/beffc212::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst55af81a2cef2701c[0] - [Crux] attempted to read empty mux tree - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - (expected failure) - coerce_unsized: OK (0.99s) - Compiling and running oracle program (0.12s) - Oracle output: (1, 2) (0.86s) - Crux output: (1, 2) - is_null: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: (false, true) (0.88s) - Crux output: (false, true) - offset: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.85s) - Crux output: 3 - offset_from: OK (1.00s) - Compiling and running oracle program (0.12s) - Oracle output: -2 (0.87s) - Crux output: -2 - struct_eq: OK (1.00s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.88s) - Crux output: () - valid_eq: OK (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.87s) - Crux output: () - cast_eq: OK (0.99s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.87s) - Crux output: () - copy: OK (1.03s) - Compiling and running oracle program (0.13s) - Oracle output: [1, 2, 3, 1, 2, 3] (0.91s) - Crux output: [1, 2, 3, 1, 2, 3] - unsize_slice: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: (1, 2) (0.86s) - Crux output: (1, 2) - read_write: OK (1.06s) - Compiling and running oracle program (0.13s) - Oracle output: [1, 3, 1] (0.93s) - Crux output: [1, 3, 1] - null_eq: FAIL (1.03s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.90s) - Crux output: - failures: - - ---- null_eq/1dae14de::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/ptr/metadata.rs:118:14: 118:78: error: in core/beffc212::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/null_eq/' to rerun this test only. - str - to_owned: FAIL (1.17s) - Compiling and running oracle program (0.13s) - Oracle output: true (1.05s) - Crux output: - failures: - - ---- to_owned/ac2d2c7c::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/to_owned/' to rerun this test only. - format_struct: FAIL (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: true (1.72s) - Crux output: - failures: - - ---- format_struct/d1c28a08::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/beffc212::fmt[0]::write[0] - [Crux] Translation error in core/beffc212::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/beffc212::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/format_struct/' to rerun this test only. - format_array: FAIL (1.84s) - Compiling and running oracle program (0.13s) - Oracle output: true (1.71s) - Crux output: - failures: - - ---- format_array/69f3b2e7::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/beffc212::fmt[0]::write[0] - [Crux] Translation error in core/beffc212::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/beffc212::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/format_array/' to rerun this test only. - format_int: FAIL (1.71s) - Compiling and running oracle program (0.13s) - Oracle output: true (1.58s) - Crux output: - failures: - - ---- format_int/050dadd3::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/beffc212::fmt[0]::write[0] - [Crux] Translation error in core/beffc212::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/beffc212::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/format_int/' to rerun this test only. - format: FAIL (1.75s) - Compiling and running oracle program (0.13s) - Oracle output: true (1.62s) - Crux output: - failures: - - ---- format/1ac747dd::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/beffc212::fmt[0]::write[0] - [Crux] Translation error in core/beffc212::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/beffc212::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '$0=="crux-mir.crux concrete..str.format"' to rerun this test only. - format_hex: FAIL (1.78s) - Compiling and running oracle program (0.13s) - Oracle output: true (1.65s) - Crux output: - failures: - - ---- format_hex/23d72fd8::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/beffc212::fmt[0]::write[0] - [Crux] Translation error in core/beffc212::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/beffc212::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/format_hex/' to rerun this test only. - string_push: FAIL (1.64s) - Compiling and running oracle program (0.13s) - Oracle output: true (1.51s) - Crux output: - failures: - - ---- string_push/210232c5::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle + Oracle output: (2, 1) (1.78s) + Crux output: (2, 1) + rwlock_multi: FAIL (3.31s) + Compiling and running oracle program (0.16s) + Oracle output: 3 (3.16s) + TODO RGS evalRval ThreadLocalRef + CallStack (from HasCallStack): + error, called at src/Mir/Trans.hs:1034:33 in crucible-mir-0.1-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1197:11 in crucible-mir-0.1-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1642:68 in crucible-mir-0.1-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1651:28 in crucible-mir-0.1-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1691:10 in crucible-mir-0.1-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1737:24 in crucible-mir-0.1-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2171:30 in crucible-mir-0.1-inplace:Mir.Trans + transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate + translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language - Use -p '/string_push/' to rerun this test only. - time - instant: FAIL (2.09s) - Compiling and running oracle program (0.12s) - Oracle output: () (1.97s) - standalone use of `dyn` is not supported: TyDynamic core/beffc212::any[0]::Erased[0]::_trait58656ccc69864acf[0] + Use -p '/rwlock_multi/' to rerun this test only. + atomic_fence: OK (2.00s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.85s) + Crux output: 2 + mutex_multi: FAIL (3.28s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (3.13s) + TODO RGS evalRval ThreadLocalRef CallStack (from HasCallStack): - error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:213:7 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:402:29 in crucible-mir-0.1-inplace:Mir.TransTy - tyToReprCont, called at src/Mir/TransTy.hs:202:24 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy - tyListToCtx, called at src/Mir/TransTy.hs:221:6 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy - tyListToCtx, called at src/Mir/TransTy.hs:1066:16 in crucible-mir-0.1-inplace:Mir.TransTy - traitVtableType, called at src/Mir/Trans.hs:2030:18 in crucible-mir-0.1-inplace:Mir.Trans - transVirtCall, called at src/Mir/Trans.hs:2153:31 in crucible-mir-0.1-inplace:Mir.Trans + error, called at src/Mir/Trans.hs:1034:33 in crucible-mir-0.1-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1197:11 in crucible-mir-0.1-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1642:68 in crucible-mir-0.1-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1651:28 in crucible-mir-0.1-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1691:10 in crucible-mir-0.1-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1737:24 in crucible-mir-0.1-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2171:30 in crucible-mir-0.1-inplace:Mir.Trans transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language - Use -p '/instant/' to rerun this test only. - enum - arg2: rustc compilation failed for arg2 + Use -p '/mutex_multi/' to rerun this test only. + consts + struct_val: OK (1.96s) + Compiling and running oracle program (0.15s) + Oracle output: Foo { x: true } (1.81s) + Crux output: Foo { x: true } + struct_unit: OK (1.95s) + Compiling and running oracle program (0.15s) + Oracle output: Err(Foo { x: () }) (1.81s) + Crux output: Err(Foo { x: () }) + local_key: OK (1.98s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.83s) + Crux output: 1 + fn_def: OK (1.92s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.78s) + Crux output: 1 + enum_val: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: None (1.72s) + Crux output: None + crypto + add: OK (2.03s) + Compiling and running oracle program (0.15s) + Oracle output: true (1.88s) + Crux output: true + add_noL: OK (2.02s) + Compiling and running oracle program (0.16s) + Oracle output: false (1.86s) + Crux output: false + cell + cell: OK (1.95s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.81s) + Crux output: 2 + ref_cell: OK (2.49s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (2.34s) + Crux output: 2 + ref_cell2: OK (2.61s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (2.46s) + Crux output: 2 + fnptr + call: OK (1.95s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.80s) + Crux output: 2 + field: OK (1.99s) + Compiling and running oracle program (0.17s) + Oracle output: 2 (1.83s) + Crux output: 2 + make: OK (1.98s) + Compiling and running oracle program (0.15s) + Oracle output: 0 (1.84s) + Crux output: 0 + custom: rustc compilation failed for custom error output: -error[E0518]: attribute should be applied to function or closure - --> test/conc_eval/enum/arg2.rs:3:5 +error[E0432]: unresolved import `core` + --> test/conc_eval/fnptr/custom.rs:2:5 | -3 | #[inline(never)] - | ^^^^^^^^^^^^^^^^ -4 | A(u8), - | ----- not a function or closure - -error[E0518]: attribute should be applied to function or closure - --> test/conc_eval/enum/arg2.rs:5:5 +2 | use core::mem; + | ^^^^ maybe a missing crate `core`? | -5 | #[inline(never)] - | ^^^^^^^^^^^^^^^^ -6 | B(i32,i32), - | ---------- not a function or closure + = help: consider adding `extern crate core` to use the `core` crate -error: aborting due to 2 previous errors +error: aborting due to previous error -For more information about this error, try `rustc --explain E0518`. +For more information about this error, try `rustc --explain E0432`. -FAIL (0.05s) - Compiling and running oracle program (0.05s) +FAIL (expected: taking address of an overridden function) (0.06s) + Compiling and running oracle program (0.06s) test/Test.hs:107: failed to compile and run - - Use -p '/arg2/' to rerun this test only. - field_order: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.86s) + (expected failure) + num + overflow: OK (2.00s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.85s) Crux output: () - ret: OK (0.99s) - Compiling and running oracle program (0.12s) - Oracle output: (A(42), B { x: 42 }, C) (0.86s) - Crux output: (A(42), B { x: 42 }, C) - eq: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.88s) + from_bytes: OK (1.96s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.82s) Crux output: () - inner: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 4 (0.85s) - Crux output: 4 - cow: FAIL (1.11s) - Compiling and running oracle program (0.13s) - Oracle output: 200 (0.98s) - Crux output: - failures: - - ---- cow/f92ce4f1::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/cow/' to rerun this test only. - cmp: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: -23 (0.85s) - Crux output: -23 - match: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 42 (0.84s) - Crux output: 42 - arg: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 0 (0.85s) - Crux output: 0 - mixed_discrs: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.84s) + saturate: OK (2.54s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.37s) Crux output: () - sync - arc: OK (1.12s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.99s) - Crux output: 1 - atomic_swap: OK (1.03s) - Compiling and running oracle program (0.12s) - Oracle output: (2, 1) (0.90s) - Crux output: (2, 1) - rwlock_multi: FAIL (1.55s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (1.42s) - user error (JSON Decoding of test/conc_eval/sync/rwlock_multi.all.mir failed: Error in $.statics[162].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) - - Use -p '/rwlock_multi/' to rerun this test only. - rwlock: FAIL (1.60s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (1.47s) - user error (JSON Decoding of test/conc_eval/sync/rwlock.all.mir failed: Error in $.statics[162].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) - - Use -p '$0=="crux-mir.crux concrete..sync.rwlock"' to rerun this test only. - atomic_fence: OK (1.03s) - Compiling and running oracle program (0.12s) - Oracle output: 2 (0.91s) - Crux output: 2 - mutex_multi: FAIL (1.45s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (1.32s) - user error (JSON Decoding of test/conc_eval/sync/mutex_multi.all.mir failed: Error in $.statics[161].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) - - Use -p '/mutex_multi/' to rerun this test only. - atomic_add: OK (1.09s) - Compiling and running oracle program (0.13s) - Oracle output: 6 (0.96s) - Crux output: 6 - arc_clone: OK (1.18s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (1.05s) - Crux output: 2 - arc_cell: OK (1.18s) - Compiling and running oracle program (0.13s) - Oracle output: 4 (1.05s) + prim + bool: OK (1.92s) + Compiling and running oracle program (0.14s) + Oracle output: false (1.78s) + Crux output: false + shift3: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: -9223372036854775808 (1.76s) + Crux output: -9223372036854775808 + div: OK (1.92s) + Compiling and running oracle program (0.14s) + Oracle output: 4 (1.78s) Crux output: 4 - mutex: FAIL (1.60s) + ge: OK (1.91s) Compiling and running oracle program (0.14s) - Oracle output: 1 (1.46s) - user error (JSON Decoding of test/conc_eval/sync/mutex.all.mir failed: Error in $.statics[161].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) - - Use -p '$0=="crux-mir.crux concrete..sync.mutex"' to rerun this test only. - atomic_cxchg: OK (1.09s) + Oracle output: true (1.77s) + Crux output: true + add1: OK (1.93s) Compiling and running oracle program (0.14s) - Oracle output: 6 (0.95s) - Crux output: 6 - intTest - test0038: OK (1.06s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.93s) - Crux output: () - test0039: OK (0.96s) - Compiling and running oracle program (0.12s) - Oracle output: 7 (0.84s) - Crux output: 7 - stdlib - result_interior: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 3 (0.85s) - Crux output: 3 - option3: OK (0.99s) - Compiling and running oracle program (0.12s) - Oracle output: 27 (0.87s) - Crux output: 27 - result: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 27 (0.84s) - Crux output: 27 - poly: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.84s) - Crux output: 1 - default_impl: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.84s) - Crux output: () - cvt: OK (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: 0 (0.86s) - Crux output: 0 - default: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: true (0.85s) + Oracle output: 2 (1.79s) + Crux output: 2 + shift_exceeding: rustc compilation failed for shift_exceeding +error output: +error[E0425]: cannot find function `catch_unwind` in module `panic` + --> test/conc_eval/prim/shift_exceeding.rs:14:25 + | +14 | let result = panic::catch_unwind(|| { + | ^^^^^^^^^^^^ not found in `panic` + | +help: consider importing this function + | +3 | use std::panic::catch_unwind; + | +help: if you import `catch_unwind`, refer to it directly + | +14 - let result = panic::catch_unwind(|| { +14 + let result = catch_unwind(|| { + | + +warning: lint `exceeding_bitshifts` has been renamed to `arithmetic_overflow` + --> test/conc_eval/prim/shift_exceeding.rs:4:9 + | +4 | #[allow(exceeding_bitshifts)] + | ^^^^^^^^^^^^^^^^^^^ help: use the new name: `arithmetic_overflow` + | + = note: `#[warn(renamed_and_removed_lints)]` on by default + +error: aborting due to previous error; 1 warning emitted + +For more information about this error, try `rustc --explain E0425`. + +FAIL (expected: Should panic, but doesn't) (0.05s) + Compiling and running oracle program (0.05s) + test/Test.hs:107: + failed to compile and run + (expected failure) + char_from_u32: OK (1.91s) + Compiling and running oracle program (0.14s) + Oracle output: 'A' (1.76s) + Crux output: 'A' + ffs: OK (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: true (1.76s) Crux output: true - option: OK (0.96s) - Compiling and running oracle program (0.12s) - Oracle output: true (0.84s) + mut_arg: OK (1.87s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.72s) + Crux output: 1 + litstring: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: true (1.73s) Crux output: true - option2: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 0 (0.84s) - Crux output: 0 - teq: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: false (0.85s) + lit: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: false (1.75s) Crux output: false - range: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 10 (0.85s) - Crux output: 10 - hash_map - insert_get: OK (1.30s) + litbstring: OK (1.94s) + Compiling and running oracle program (0.14s) + Oracle output: true (1.80s) + Crux output: true + shift1: OK (1.91s) Compiling and running oracle program (0.15s) - Oracle output: (11, 12) (1.14s) - Crux output: (11, 12) - insert_iter: OK (1.31s) - Compiling and running oracle program (0.16s) - Oracle output: [11, 12] (1.15s) - Crux output: [11, 12] - insert_multi: OK (1.35s) + Oracle output: 2 (1.76s) + Crux output: 2 + shift2: OK (1.92s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.77s) + Crux output: 2 + shift4: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: -9223372036854775808 (1.73s) + Crux output: -9223372036854775808 + mut: OK (1.93s) + Compiling and running oracle program (0.14s) + Oracle output: 14 (1.79s) + Crux output: 14 + wrapping_sub: OK (1.91s) + Compiling and running oracle program (0.14s) + Oracle output: true (1.77s) + Crux output: true + impl + self: OK (1.97s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (1.82s) + Crux output: 42 + simple: OK (1.97s) + Compiling and running oracle program (0.17s) + Oracle output: 42 (1.81s) + Crux output: 42 + self_mut: OK (1.92s) Compiling and running oracle program (0.15s) - Oracle output: 100 (1.20s) - Crux output: 100 - insert_remove: OK (1.80s) + Oracle output: 42 (1.77s) + Crux output: 42 + box + new: OK (2.05s) Compiling and running oracle program (0.15s) - Oracle output: 12 (1.65s) - Crux output: 12 - tuple - clone: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.87s) + Oracle output: () (1.90s) Crux output: () - clone_from: FAIL (1.02s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.89s) - Crux output: - failures: - - ---- clone_from/44ff9844::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/clone.rs:136:17: 136:31: error: in core/beffc212::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0] - [Crux] Translation error in core/beffc212::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0]: don't know how to generate CloneShim for unknown method core/beffc212::clone[0]::Clone[0]::clone[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/clone_from/' to rerun this test only. - clone_rec: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.88s) + mut_ref: OK (2.04s) + Compiling and running oracle program (0.17s) + Oracle output: () (1.86s) Crux output: () - clone_struct: FAIL (1.02s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.90s) - Crux output: - failures: - - ---- clone_struct/473e1c76::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/tuple/clone_struct.rs:8:13: 8:22: error: in clone_struct/473e1c76::f[0] - [Crux] Translation error in clone_struct/473e1c76::f[0]: don't know how to generate CloneShim for unknown method core/beffc212::clone[0]::Clone[0]::clone[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/clone_struct/' to rerun this test only. - consts - struct_val: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: Foo { x: true } (0.85s) - Crux output: Foo { x: true } - struct_unit: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: Err(Foo { x: () }) (0.87s) - Crux output: Err(Foo { x: () }) - enum_val: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: None (0.87s) - Crux output: None - local_key: FAIL (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 1 (0.85s) - user error (JSON Decoding of test/conc_eval/consts/local_key.all.mir failed: Error in $.statics[0].rendered.fields[0]: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) - - Use -p '/local_key/' to rerun this test only. - fn_def: FAIL (1.00s) - Compiling and running oracle program (0.12s) - Oracle output: 1 (0.87s) - user error (JSON Decoding of test/conc_eval/consts/fn_def.all.mir failed: Error in $.fns[0].body.blocks[0].block.terminator.func.data.rendered: parseJSON - bad rendered constant kind: Just (String "fn_ptr")) - - Use -p '/fn_def/' to rerun this test only. - array - iter: OK (1.04s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.91s) - Crux output: 3 - const_impl: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 5 (0.85s) - Crux output: 5 - clone: FAIL (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.89s) - Crux output: - failures: - - ---- clone/43838f2d::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/ptr/const_ptr.rs:46:9: 46:18: error: in core/beffc212::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0] - [Crux] Translation error in core/beffc212::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst6cd22c7344b2c333[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyInt B32) Immut - [Crux] as: TyRawPtr (TyArray (TyInt B32) 3) Immut - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/array.clone/' to rerun this test only. - mut_index: FAIL (1.53s) - Compiling and running oracle program (0.13s) - Oracle output: 7 (1.40s) - Crux output: - failures: - - ---- mut_index/209c678d::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/mut_index/' to rerun this test only. - mut_arg: OK (0.99s) - Compiling and running oracle program (0.12s) - Oracle output: 42 (0.87s) - Crux output: 42 - from_slice: FAIL (1.63s) - Compiling and running oracle program (0.13s) - Oracle output: () (1.50s) + mut: OK (2.02s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.87s) + Crux output: () + unsize: OK (2.05s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.90s) + Crux output: 2 + struct: OK (2.02s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.87s) + Crux output: 1 + ptr + is_null_slice: FAIL (expected: can't unsize null pointers) (1.94s) + Compiling and running oracle program (0.14s) + Oracle output: (false, true) (1.80s) Crux output: failures: - ---- from_slice/09f81d88::f[0] counterexamples ---- + ---- is_null_slice/7fe66064::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/ptr/is_null_slice.rs:7:13: 7:33: error: in is_null_slice/7fe66064::crux_test[0] + [Crux] attempted subindex on the result of an integer-to-pointer cast [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/array/mod.rs:266:23: 266:54: error: in core/beffc212::array[0]::{impl#10}[0]::try_from[0]::_inste494657dbbea865b[0] - [Crux] Translation error in core/beffc212::array[0]::{impl#10}[0]::try_from[0]::_inste494657dbbea865b[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyUint B8) Immut - [Crux] as: TyRawPtr (TyArray (TyUint B8) 4) Immut + [Crux] ./lib/core/src/ptr/const_ptr.rs:38:38: 38:42: error: in core/1cdf94ce::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst55af81a2cef2701c[0] + [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - - Use -p '/array.from_slice/' to rerun this test only. - const: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.84s) - Crux output: 1 - wick2: OK (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: true (0.86s) - Crux output: true - wick1: FAIL (expected: needs Vec data structure from stdlib) (0.17s) - Compiling and running oracle program (0.14s) - Oracle output: true (0.04s) - user error (Error 101 while running mir-json on test/conc_eval/array/wick1.rs) (expected failure) - arg: OK (0.98s) - Compiling and running oracle program (0.12s) - Oracle output: 2 (0.86s) - Crux output: 2 - mk_and_proj: OK (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.86s) - Crux output: 42 - wick3: FAIL (expected: needs Vec data structure from stdlib) (0.18s) + struct_eq: OK (1.93s) Compiling and running oracle program (0.14s) - Oracle output: true (0.04s) - user error (Error 101 while running mir-json on test/conc_eval/array/wick3.rs) - (expected failure) - mem - maybe_uninit_array_cast: FAIL (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: [1, 2] (0.87s) + Oracle output: () (1.79s) + Crux output: () + read_write: OK (1.98s) + Compiling and running oracle program (0.15s) + Oracle output: [1, 3, 1] (1.82s) + Crux output: [1, 3, 1] + offset_mut: OK (1.92s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.78s) + Crux output: 3 + copy: OK (1.97s) + Compiling and running oracle program (0.15s) + Oracle output: [1, 2, 3, 1, 2, 3] (1.81s) + Crux output: [1, 2, 3, 1, 2, 3] + unsize_slice: OK (1.92s) + Compiling and running oracle program (0.14s) + Oracle output: (1, 2) (1.78s) + Crux output: (1, 2) + dangling_eq: FAIL (1.94s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.80s) Crux output: failures: - ---- maybe_uninit_array_cast/5024e15a::crux_test[0] counterexamples ---- + ---- dangling_eq/3f4bcd46::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in maybe_uninit_array_cast/5024e15a::crux_test[0] - [Crux] Translation error in maybe_uninit_array_cast/5024e15a::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyArray (TyInt B32) 2]) + [Crux] ./lib/core/src/ptr/metadata.rs:135:14: 135:78: error: in core/1cdf94ce::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/1cdf94ce::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/maybe_uninit_array_cast/' to rerun this test only. - maybe_uninit: FAIL (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.87s) + Use -p '/dangling_eq/' to rerun this test only. + cast_eq: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.75s) + Crux output: () + offset_from: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: -2 (1.74s) + Crux output: -2 + null_eq: FAIL (1.94s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.79s) Crux output: failures: - ---- maybe_uninit/4270f58e::crux_test[0] counterexamples ---- + ---- null_eq/7504a468::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in maybe_uninit/4270f58e::crux_test[0] - [Crux] Translation error in maybe_uninit/4270f58e::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyInt B32]) + [Crux] ./lib/core/src/ptr/metadata.rs:118:14: 118:78: error: in core/1cdf94ce::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0] + [Crux] Translation error in core/1cdf94ce::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '$0=="crux-mir.crux concrete..mem.maybe_uninit"' to rerun this test only. + Use -p '/null_eq/' to rerun this test only. + coerce_unsized: OK (1.88s) + Compiling and running oracle program (0.15s) + Oracle output: (1, 2) (1.73s) + Crux output: (1, 2) + offset: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.75s) + Crux output: 3 + is_null: OK (1.90s) + Compiling and running oracle program (0.15s) + Oracle output: (false, true) (1.75s) + Crux output: (false, true) + valid_eq: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.74s) + Crux output: () ops - index2: OK (0.99s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.87s) + deref2: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.75s) Crux output: () - deref1: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.85s) + index2: OK (1.88s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.73s) Crux output: () - index1: OK (1.02s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.89s) + arith1: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.74s) Crux output: () - deref3: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.85s) + index3: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.74s) Crux output: () - arith1: OK (1.03s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.90s) + deref1: OK (1.90s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.75s) Crux output: () - deref2: OK (0.99s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.87s) + index1: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.75s) Crux output: () - index3: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.87s) + deref3: OK (2.02s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.86s) Crux output: () - fnptr - field: OK (0.98s) - Compiling and running oracle program (0.12s) - Oracle output: 2 (0.85s) - Crux output: 2 - custom: rustc compilation failed for custom -error output: -error[E0432]: unresolved import `core` - --> test/conc_eval/fnptr/custom.rs:2:5 - | -2 | use core::mem; - | ^^^^ maybe a missing crate `core`? - | - = help: consider adding `extern crate core` to use the `core` crate - -error: aborting due to previous error - -For more information about this error, try `rustc --explain E0432`. - -FAIL (expected: taking address of an overridden function) (0.05s) - Compiling and running oracle program (0.05s) - test/Test.hs:107: - failed to compile and run - (expected failure) - make: OK (1.00s) - Compiling and running oracle program (0.12s) - Oracle output: 0 (0.87s) - Crux output: 0 - call: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.85s) - Crux output: 2 - io - cursor_write2: FAIL (2.07s) + statics + promoted_fn: OK (1.94s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.80s) + Crux output: 1 + promoted_static: OK (1.89s) Compiling and running oracle program (0.14s) - Oracle output: 0 (1.93s) - standalone use of `dyn` is not supported: TyDynamic core/beffc212::any[0]::Erased[0]::_trait58656ccc69864acf[0] + Oracle output: 1 (1.75s) + Crux output: 1 + io + cursor_write2: FAIL (3.04s) + Compiling and running oracle program (0.16s) + Oracle output: 0 (2.88s) + standalone use of `dyn` is not supported: TyDynamic core/1cdf94ce::any[0]::Erased[0]::_trait58656ccc69864acf[0] CallStack (from HasCallStack): error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy tyToRepr, called at src/Mir/TransTy.hs:213:7 in crucible-mir-0.1-inplace:Mir.TransTy @@ -1504,370 +910,569 @@ FAIL (expected: taking address of an overridden function) (0.05s) tyListToCtx, called at src/Mir/TransTy.hs:221:6 in crucible-mir-0.1-inplace:Mir.TransTy tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy tyListToCtx, called at src/Mir/TransTy.hs:1066:16 in crucible-mir-0.1-inplace:Mir.TransTy - traitVtableType, called at src/Mir/Trans.hs:2030:18 in crucible-mir-0.1-inplace:Mir.Trans - transVirtCall, called at src/Mir/Trans.hs:2153:31 in crucible-mir-0.1-inplace:Mir.Trans + traitVtableType, called at src/Mir/Trans.hs:2055:18 in crucible-mir-0.1-inplace:Mir.Trans + transVirtCall, called at src/Mir/Trans.hs:2178:31 in crucible-mir-0.1-inplace:Mir.Trans transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '/cursor_write2/' to rerun this test only. - cursor_read: FAIL (1.65s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.51s) + cursor_write: OK (2.52s) + Compiling and running oracle program (0.17s) + Oracle output: 0 (2.35s) + Crux output: 0 + cursor_read: OK (2.54s) + Compiling and running oracle program (0.16s) + Oracle output: 0 (2.38s) + Crux output: 0 + mem + maybe_uninit: FAIL (2.00s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.85s) Crux output: failures: - ---- cursor_read/a3a3296c::crux_test[0] counterexamples ---- + ---- maybe_uninit/9a4e7f6f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] internal: error: in maybe_uninit/9a4e7f6f::crux_test[0] + [Crux] Translation error in maybe_uninit/9a4e7f6f::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyInt B32]) [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '/io.cursor_read/' to rerun this test only. - cursor_write: FAIL (1.69s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.55s) + Use -p '$0=="crux-mir.crux concrete..mem.maybe_uninit"' to rerun this test only. + maybe_uninit_array_cast: FAIL (1.99s) + Compiling and running oracle program (0.15s) + Oracle output: [1, 2] (1.84s) Crux output: failures: - ---- cursor_write/0e9225ad::crux_test[0] counterexamples ---- + ---- maybe_uninit_array_cast/54dba2bc::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI + [Crux] internal: error: in maybe_uninit_array_cast/54dba2bc::crux_test[0] + [Crux] Translation error in maybe_uninit_array_cast/54dba2bc::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyArray (TyInt B32) 2]) [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '$0=="crux-mir.crux concrete..io.cursor_write"' to rerun this test only. - cell - ref_cell2: OK (1.58s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (1.45s) - Crux output: 2 - cell: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.87s) - Crux output: 2 - ref_cell: OK (1.56s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (1.44s) - Crux output: 2 + Use -p '/maybe_uninit_array_cast/' to rerun this test only. + time + instant: FAIL (3.14s) + Compiling and running oracle program (0.15s) + Oracle output: () (2.99s) + standalone use of `dyn` is not supported: TyDynamic core/1cdf94ce::any[0]::Erased[0]::_trait58656ccc69864acf[0] + CallStack (from HasCallStack): + error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:213:7 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:402:29 in crucible-mir-0.1-inplace:Mir.TransTy + tyToReprCont, called at src/Mir/TransTy.hs:202:24 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy + tyListToCtx, called at src/Mir/TransTy.hs:221:6 in crucible-mir-0.1-inplace:Mir.TransTy + tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy + tyListToCtx, called at src/Mir/TransTy.hs:1066:16 in crucible-mir-0.1-inplace:Mir.TransTy + traitVtableType, called at src/Mir/Trans.hs:2055:18 in crucible-mir-0.1-inplace:Mir.Trans + transVirtCall, called at src/Mir/Trans.hs:2178:31 in crucible-mir-0.1-inplace:Mir.Trans + transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate + translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language + + Use -p '/instant/' to rerun this test only. struct - repr_transparent: FAIL (1.03s) - Compiling and running oracle program (0.13s) - Oracle output: 6 (0.90s) + field_order: OK (2.01s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.86s) + Crux output: () + arg: OK (1.92s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.78s) + Crux output: 42 + tup: OK (1.96s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.81s) + Crux output: () + proj: OK (2.00s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.85s) + Crux output: 42 + ret: OK (1.97s) + Compiling and running oracle program (0.15s) + Oracle output: S { x: 42, y: 120 } (1.83s) + Crux output: S { x: 42, y: 120 } + repr_transparent: OK (2.07s) + Compiling and running oracle program (0.16s) + Oracle output: 6 (1.91s) + Crux output: 6 + repr_transparent_const: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: 124 (1.75s) + Crux output: 124 + dyn + assoc_ty: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: 100 (1.75s) + Crux output: 100 + trait_param: OK (1.95s) + Compiling and running oracle program (0.14s) + Oracle output: 100 (1.80s) + Crux output: 100 + plain_trait: OK (2.00s) + Compiling and running oracle program (0.16s) + Oracle output: 100 (1.84s) + Crux output: 100 + inherit: OK (1.98s) + Compiling and running oracle program (0.15s) + Oracle output: 4 (1.83s) + Crux output: 4 + stdlib + option: OK (1.95s) + Compiling and running oracle program (0.16s) + Oracle output: true (1.79s) + Crux output: true + option2: OK (1.97s) + Compiling and running oracle program (0.16s) + Oracle output: 0 (1.81s) + Crux output: 0 + result: OK (1.95s) + Compiling and running oracle program (0.15s) + Oracle output: 27 (1.80s) + Crux output: 27 + default: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: true (1.74s) + Crux output: true + cvt: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.73s) + Crux output: 0 + range: OK (2.01s) + Compiling and running oracle program (0.14s) + Oracle output: 10 (1.87s) + Crux output: 10 + poly: OK (2.04s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (1.88s) + Crux output: 1 + result_interior: OK (2.01s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (1.86s) + Crux output: 3 + default_impl: OK (1.94s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.79s) + Crux output: () + option3: OK (1.92s) + Compiling and running oracle program (0.14s) + Oracle output: 27 (1.78s) + Crux output: 27 + teq: OK (1.97s) + Compiling and running oracle program (0.15s) + Oracle output: false (1.82s) + Crux output: false + array + wick2: OK (2.01s) + Compiling and running oracle program (0.16s) + Oracle output: true (1.85s) + Crux output: true + arg: OK (2.04s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.88s) + Crux output: 2 + iter: OK (2.14s) + Compiling and running oracle program (0.17s) + Oracle output: 3 (1.97s) + Crux output: 3 + mk_and_proj: OK (2.04s) + Compiling and running oracle program (0.16s) + Oracle output: 42 (1.89s) + Crux output: 42 + wick3: FAIL (expected: needs Vec data structure from stdlib) (0.21s) + Compiling and running oracle program (0.17s) + Oracle output: true (0.04s) + user error (Error 101 while running mir-json on test/conc_eval/array/wick3.rs) + (expected failure) + clone: OK (2.10s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.94s) + Crux output: () + wick1: FAIL (expected: needs Vec data structure from stdlib) (0.23s) + Compiling and running oracle program (0.18s) + Oracle output: true (0.05s) + user error (Error 101 while running mir-json on test/conc_eval/array/wick1.rs) + (expected failure) + mut_index: OK (2.46s) + Compiling and running oracle program (0.16s) + Oracle output: 7 (2.31s) + Crux output: 7 + mut_arg: OK (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.76s) + Crux output: 42 + const: OK (1.86s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.71s) + Crux output: 1 + from_slice: FAIL (2.58s) + Compiling and running oracle program (0.15s) + Oracle output: () (2.44s) Crux output: failures: - ---- repr_transparent/ac707a40::crux_test[0] counterexamples ---- + ---- from_slice/4075ba45::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/ptr/const_ptr.rs:46:9: 46:18: error: in core/beffc212::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0] - [Crux] Translation error in core/beffc212::ptr[0]::const_ptr[0]::{impl#0}[0]::cast[0]::_inst1fe84093a10a77db[0]: unimplemented cast: Misc - [Crux] ty: TyRawPtr (TyInt B32) Immut - [Crux] as: TyRawPtr (TyArray (TyInt B32) 2) Immut + [Crux] lib/core/src/iter/traits/iterator.rs:2607:40: 2607:61: error: in core/1cdf94ce::iter[0]::traits[0]::iterator[0]::Iterator[0]::all[0]::_instf9a28a1fbcee32be[0] + [Crux] Translation error in core/1cdf94ce::iter[0]::traits[0]::iterator[0]::Iterator[0]::all[0]::_instf9a28a1fbcee32be[0]: ill-typed assignment of IntrinsicRepr MirReference [BVRepr 8] to IntrinsicRepr MirReference [AnyRepr] (TyRef (TyAdt core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0]::_adtdfc2b6f78a281997[0] core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0] (Substs [TyTuple [],TyTuple []])) Immut) LBase (Var {_varname = "_10", _varmut = Mut, _varty = TyRef (TyAdt core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0]::_adtdfc2b6f78a281997[0] core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0] (Substs [TyTuple [],TyTuple []])) Immut, _varIsZST = False}) [Crux] Overall status: Invalid. test/Test.hs:124: crux doesn't match oracle - Use -p '$0=="crux-mir.crux concrete..struct.repr_transparent"' to rerun this test only. - tup: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.84s) + Use -p '/array.from_slice/' to rerun this test only. + const_impl: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 5 (1.73s) + Crux output: 5 + enum + match: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (1.76s) + Crux output: 42 + field_order: OK (1.91s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.77s) Crux output: () - repr_transparent_const: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: 124 (0.87s) - Crux output: 124 - field_order: OK (0.98s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.86s) + mixed_discrs: OK (1.92s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.78s) Crux output: () - ret: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: S { x: 42, y: 120 } (0.84s) - Crux output: S { x: 42, y: 120 } - proj: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.84s) - Crux output: 42 - arg: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.86s) - Crux output: 42 - refs - temp: OK (0.99s) - Compiling and running oracle program (0.12s) - Oracle output: 1 (0.87s) - Crux output: 1 - static_mut: OK (0.99s) - Compiling and running oracle program (0.12s) - Oracle output: 2 (0.86s) + arg: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.73s) + Crux output: 0 + arg2: OK (1.87s) + Compiling and running oracle program (0.15s) + Oracle output: 0 (1.73s) + Crux output: 0 + cow: OK (2.01s) + Compiling and running oracle program (0.15s) + Oracle output: 200 (1.86s) + Crux output: 200 + ret: OK (1.86s) + Compiling and running oracle program (0.15s) + Oracle output: (A(42), B { x: 42 }, C) (1.72s) + Crux output: (A(42), B { x: 42 }, C) + eq: OK (1.90s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.75s) + Crux output: () + cmp: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: -23 (1.74s) + Crux output: -23 + inner: OK (1.91s) + Compiling and running oracle program (0.14s) + Oracle output: 4 (1.77s) + Crux output: 4 + vec + drop: OK (2.13s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.97s) + Crux output: () + set_len: OK (2.15s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (1.99s) + Crux output: 2 + push: OK (2.17s) + Compiling and running oracle program (0.17s) + Oracle output: (1, 2) (2.00s) + Crux output: (1, 2) + extend: OK (2.25s) + Compiling and running oracle program (0.17s) + Oracle output: (1, 10) (2.08s) + Crux output: (1, 10) + collect: OK (2.85s) + Compiling and running oracle program (0.16s) + Oracle output: 45 (2.69s) + Crux output: 45 + extend_trusted_len: OK (2.76s) + Compiling and running oracle program (0.16s) + Oracle output: (1, 10) (2.59s) + Crux output: (1, 10) + from_elem_zero: OK (2.13s) + Compiling and running oracle program (0.16s) + Oracle output: 0 (1.97s) + Crux output: 0 + clos + promoted: OK (1.95s) + Compiling and running oracle program (0.16s) + Oracle output: 33 (1.80s) + Crux output: 33 + as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (1.97s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.82s) + Crux output: + failures: + + ---- as_fn_ptr/dd580d0d::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/dd580d0d::crux_test[0] + [Crux] Translation error in as_fn_ptr/dd580d0d::crux_test[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [], _fsreturn_ty = TyInt B32, _fsabi = RustAbi, _fsspreadarg = Nothing}) + + [Crux] Overall status: Invalid. + + test/Test.hs:124: + crux doesn't match oracle + (expected failure) + fnptr_fnmut: OK (1.96s) + Compiling and running oracle program (0.15s) + Oracle output: 4 (1.81s) + Crux output: 4 + direct_fnonce: OK (1.93s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.78s) + Crux output: 2 + conv_fnonce_fnmut: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.73s) + Crux output: 2 + fn_static_poly: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.73s) + Crux output: 3 + fnptr_fnonce: OK (1.88s) + Compiling and running oracle program (0.15s) + Oracle output: 4 (1.73s) + Crux output: 4 + fn_static: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.72s) + Crux output: 3 + fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.73s) + vtable signature mismatch for vtable core/1cdf94ce::ops[0]::function[0]::Fn[0]::_vtbl570ffd80324c2ca9[0] of trait core/1cdf94ce::ops[0]::function[0]::Fn[0]::_trait42f8b6475ee85722[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32)] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32)] + CallStack (from HasCallStack): + error, called at src/Mir/Trans.hs:958:20 in crucible-mir-0.1-inplace:Mir.Trans + mkTraitObject, called at src/Mir/Trans.hs:801:11 in crucible-mir-0.1-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:932:5 in crucible-mir-0.1-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:984:30 in crucible-mir-0.1-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1197:11 in crucible-mir-0.1-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1642:68 in crucible-mir-0.1-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1651:28 in crucible-mir-0.1-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1691:10 in crucible-mir-0.1-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1737:24 in crucible-mir-0.1-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2171:30 in crucible-mir-0.1-inplace:Mir.Trans + transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate + translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language + (expected failure) + fnonce: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: false (1.72s) + Crux output: false + conv_fnmut_fn: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.72s) Crux output: 2 - imm_raw: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: 123 (0.84s) - Crux output: 123 - mut_tuple_field: OK (0.99s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.87s) - Crux output: () - mut_ref: OK (0.99s) - Compiling and running oracle program (0.12s) - Oracle output: 1 (0.87s) - Crux output: 1 - mut_arg: OK (0.99s) - Compiling and running oracle program (0.12s) - Oracle output: 1 (0.86s) + direct_fnmut2: OK (1.87s) + Compiling and running oracle program (0.15s) + Oracle output: 7 (1.73s) + Crux output: 7 + fo: OK (1.93s) + Compiling and running oracle program (0.15s) + Oracle output: 29 (1.78s) + Crux output: 29 + unique_borrow: OK (1.92s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.77s) + Crux output: 3 + conv_fnonce_fn: OK (1.94s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.80s) + Crux output: 2 + fnptr_fn: OK (2.02s) + Compiling and running oracle program (0.15s) + Oracle output: 4 (1.88s) + Crux output: 4 + direct_fnmut: OK (1.97s) + Compiling and running oracle program (0.16s) + Oracle output: 7 (1.81s) + Crux output: 7 + dispatch_fnmut: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: 7 (1.75s) + Crux output: 7 + poly: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.73s) + Crux output: 3 + direct_fn: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.74s) + Crux output: 2 + fnptr_closure: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: 7 (1.74s) + Crux output: 7 + fnonce1: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.73s) + Crux output: 3 + ref_fnmut: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: 7 (1.74s) + Crux output: 7 + vec_deque + pop: OK (2.51s) + Compiling and running oracle program (0.16s) + Oracle output: [5, 4, 3, 1, 2] (2.35s) + Crux output: [5, 4, 3, 1, 2] + iter_clone: OK (2.67s) + Compiling and running oracle program (0.16s) + Oracle output: [1, 2, 3, 2, 3] (2.51s) + Crux output: [1, 2, 3, 2, 3] + push: OK (2.56s) + Compiling and running oracle program (0.16s) + Oracle output: [4, 1, 2, 3, 5] (2.40s) + Crux output: [4, 1, 2, 3, 5] + retain: OK (2.57s) + Compiling and running oracle program (0.16s) + Oracle output: [1, 4] (2.41s) + Crux output: [1, 4] + rotate_left: OK (2.59s) + Compiling and running oracle program (0.16s) + Oracle output: [3, 4, 5, 1, 2] (2.43s) + Crux output: [3, 4, 5, 1, 2] + rotate_right: OK (2.63s) + Compiling and running oracle program (0.16s) + Oracle output: [4, 5, 1, 2, 3] (2.47s) + Crux output: [4, 5, 1, 2, 3] + slice + len: OK (1.87s) + Compiling and running oracle program (0.15s) + Oracle output: 5 (1.73s) + Crux output: 5 + mut_range: OK (2.34s) + Compiling and running oracle program (0.15s) + Oracle output: 86 (2.19s) + Crux output: 86 + range_len: OK (2.35s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (2.21s) Crux output: 1 - never: OK (0.98s) - Compiling and running oracle program (0.12s) - Oracle output: 1 (0.86s) + range_len_mut: OK (2.33s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (2.19s) Crux output: 1 - mut_raw: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 123 (0.85s) - Crux output: 123 - promoted_imm: OK (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: 0 (0.87s) - Crux output: 0 - mut_nested: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.88s) + mk_and_proj: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (1.72s) + Crux output: 42 + swap: OK (1.94s) + Compiling and running oracle program (0.14s) + Oracle output: 2001 (1.79s) + Crux output: 2001 + eq: OK (1.97s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.82s) Crux output: () - imm_ref: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 123 (0.85s) - Crux output: 123 - fn_ptr_mut: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.85s) - Crux output: 1 - promoted_mut: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: 0 (0.85s) - Crux output: 0 - imm_arg: OK (0.96s) - Compiling and running oracle program (0.12s) - Oracle output: 1 (0.84s) - Crux output: 1 - fn_ptr: OK (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.87s) - Crux output: 1 - never_mut: OK (0.98s) - Compiling and running oracle program (0.12s) - Oracle output: 1 (0.86s) - Crux output: 1 + iter_mut: OK (1.97s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.82s) + Crux output: () + get: OK (1.90s) + Compiling and running oracle program (0.15s) + Oracle output: true (1.75s) + Crux output: true + mut: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (1.72s) + Crux output: 42 + last: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.73s) + Crux output: 3 crux symbolic Output testing - macro: FAIL (1.03s) - files test/symb_eval/vec/macro.good and test/symb_eval/vec/macro.out differ; test/symb_eval/vec/macro.out contains: - test macro/9050e48f::f[0]: FAILED - - failures: - - ---- macro/9050e48f::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - Use -p '/Output testing.macro/' to rerun this test only. - clone: FAIL (1.00s) - files test/symb_eval/vec/clone.good and test/symb_eval/vec/clone.out differ; test/symb_eval/vec/clone.out contains: - test clone/6db5b217::f[0]: FAILED - - failures: - - ---- clone/6db5b217::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - Use -p '/Output testing.clone/' to rerun this test only. - sort_by_key: FAIL (1.81s) - files test/symb_eval/vec/sort_by_key.good and test/symb_eval/vec/sort_by_key.out differ; test/symb_eval/vec/sort_by_key.out contains: - test sort_by_key/5a8cbf18::f[0]: FAILED - - failures: - - ---- sort_by_key/5a8cbf18::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - Use -p '/sort_by_key/' to rerun this test only. - into_iter: FAIL (1.61s) - files test/symb_eval/vec/into_iter.good and test/symb_eval/vec/into_iter.out differ; test/symb_eval/vec/into_iter.out contains: - test into_iter/772ec3dc::f[0]: FAILED + mux_init_mut: OK (1.83s) + mux_init_imm: OK (1.78s) + early_fail: OK (1.86s) + mixed_fail: OK (1.89s) + multi: OK (1.89s) + fail_return: OK (1.80s) + no_conc: OK (1.86s) + conc: OK (1.91s) + array: OK (1.88s) + assert_ok: OK (2.40s) + assert: FAIL (2.50s) + files test/symb_eval/concretize/assert.good and test/symb_eval/concretize/assert.out differ; test/symb_eval/concretize/assert.out contains: + test assert/b59efe3c::crux_test[0]: returned 1, FAILED failures: - ---- into_iter/772ec3dc::f[0] counterexamples ---- + ---- assert/b59efe3c::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst55af81a2cef2701c[0]: evalPlace (PField, Union) NYI + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/1cdf94ce::fmt[0]::write[0] + [Crux] Translation error in core/1cdf94ce::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/1cdf94ce::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. - Use -p '/into_iter/' to rerun this test only. - as_slice: OK (0.95s) - replicate: OK (0.93s) - split_at: OK (0.98s) - concat: OK (1.00s) - copy_from_slice: OK (0.94s) - new: OK (0.95s) - as_mut_slice: OK (0.98s) - mut: OK (0.95s) - push: OK (0.97s) - pop: FAIL (0.97s) + Use -p '$0=="crux-mir.crux symbolic.Output testing.assert"' to rerun this test only. + bytes2: OK (1.88s) + double: OK (1.79s) + ffs: OK (1.92s) + bytes: OK (1.89s) + bad_symb1: OK (1.78s) + bad_symb2: OK (1.81s) + override3: OK (1.82s) + override1: OK (1.76s) + override4: OK (1.83s) + override_rust: OK (1.82s) + override2: OK (1.85s) + override5: OK (1.79s) + mux: OK (1.83s) + checked_mul_signed: OK (1.70s) + checked_mul: OK (1.69s) + checked_div: OK (1.75s) + checked_add: OK (1.70s) + test1: OK (109.12s) + pop: FAIL (1.79s) files test/symb_eval/vector/pop.good and test/symb_eval/vector/pop.out differ; test/symb_eval/vector/pop.out contains: - test pop/d48bfd10::f[0]: FAILED + test pop/9a0a9424::f[0]: FAILED failures: - ---- pop/d48bfd10::f[0] counterexamples ---- + ---- pop/9a0a9424::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/vector/pop.rs:9:18: 9:19: error: in pop/d48bfd10::f[0] - [Crux] Translation error in pop/d48bfd10::f[0]: unknown ADT (core/3a1fbbbh::option[0]::Option[0],Substs [TyUint B8]) + [Crux] test/symb_eval/vector/pop.rs:9:18: 9:19: error: in pop/9a0a9424::f[0] + [Crux] Translation error in pop/9a0a9424::f[0]: unknown ADT (core/3a1fbbbh::option[0]::Option[0],Substs [TyUint B8]) [Crux] Overall status: Invalid. Use -p '/Output testing.pop/' to rerun this test only. - checked_div: OK (0.86s) - checked_add: OK (0.83s) - checked_mul_signed: OK (0.87s) - checked_mul: OK (0.84s) - array_mut: OK (1.01s) - array: OK (0.96s) - bad_symb1: OK (0.92s) - override5: OK (1.00s) - override3: OK (0.99s) - override_rust: OK (0.96s) - override1: OK (0.82s) - override2: OK (0.92s) - bad_symb2: OK (0.95s) - override4: OK (0.99s) - write: FAIL (0.05s) - test/symb_eval/byteorder/write.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/write.rs) - Use -p '/Output testing.write/' to rerun this test only. - read: FAIL - test/symb_eval/byteorder/read.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/read.rs) - Use -p '/Output testing.read/' to rerun this test only. - deserialize: OK (1.04s) - construct: OK (1.00s) - double: OK (0.93s) - bytes: OK (1.08s) - ffs: OK (1.03s) - bytes2: OK (1.07s) - from_to: OK (0.85s) - overflowing_sub: OK (0.98s) - literals: OK (0.84s) - arith: OK (1.03s) - leading_zeros: OK (1.04s) - cmp: OK (0.96s) - symbolic: OK (0.84s) - mux: OK (0.98s) - test1: OK (102.62s) - split_off: FAIL (1.38s) - files test/symb_eval/bytes/split_off.good and test/symb_eval/bytes/split_off.out differ; test/symb_eval/bytes/split_off.out contains: - test split_off/72b60843::f[0]: FAILED - - failures: - - ---- split_off/72b60843::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - Use -p '/split_off/' to rerun this test only. - split_to: FAIL (1.36s) - files test/symb_eval/bytes/split_to.good and test/symb_eval/bytes/split_to.out differ; test/symb_eval/bytes/split_to.out contains: - test split_to/c31c803f::f[0]: FAILED - - failures: - - ---- split_to/c31c803f::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - Use -p '/split_to/' to rerun this test only. - new: OK (0.96s) - put: FAIL (1.45s) - files test/symb_eval/bytes/put.good and test/symb_eval/bytes/put.out differ; test/symb_eval/bytes/put.out contains: - test put/d816ba96::f[0]: FAILED - - failures: - - ---- put/d816ba96::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.put"' to rerun this test only. - sym_len: FAIL (1.51s) - files test/symb_eval/bytes/sym_len.good and test/symb_eval/bytes/sym_len.out differ; test/symb_eval/bytes/sym_len.out contains: - test sym_len/006a1393::f[0]: FAILED - - failures: - - ---- sym_len/006a1393::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ptr/metadata.rs:98:14: 98:40: error: in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0] - [Crux] Translation error in core/beffc212::ptr[0]::metadata[0]::metadata[0]::_inst1743054c4d18d3c5[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - Use -p '/sym_len/' to rerun this test only. - extend_bytes: OK (1.00s) - put_overflow: OK (1.38s) - multi: OK (0.97s) - fail_return: OK (0.96s) - mixed_fail: OK (1.03s) - early_fail: OK (0.99s) - basic: OK (0.99s) - slice: OK (1.09s) - slice_mut: OK (1.04s) - mux_slice: OK (1.01s) - downcast_fail: FAIL (0.96s) - files test/symb_eval/any/downcast_fail.good and test/symb_eval/any/downcast_fail.out differ; test/symb_eval/any/downcast_fail.out contains: - test downcast_fail/a1506e51::crux_test[0]: FAILED - - failures: - - ---- downcast_fail/a1506e51::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/any/downcast_fail.rs:10:14: 10:15: error: in downcast_fail/a1506e51::crux_test[0] - [Crux] failed to downcast Any as BVRepr 32 - - [Crux] Overall status: Invalid. - - Use -p '/downcast_fail/' to rerun this test only. - downcast: OK (1.01s) - conditional: OK (0.98s) - mux: OK (0.93s) - vec_cursor_read: FAIL (1.94s) - standalone use of `dyn` is not supported: TyDynamic core/beffc212::any[0]::Erased[0]::_trait58656ccc69864acf[0] + as_mut_slice: OK (1.83s) + push: OK (1.87s) + copy_from_slice: OK (1.79s) + new: OK (1.84s) + mut: OK (1.79s) + concat: OK (1.85s) + split_at: OK (1.83s) + replicate: OK (1.85s) + as_slice: OK (1.77s) + array_mut: OK (1.79s) + array: OK (1.80s) + extend_bytes: OK (1.81s) + put: OK (2.25s) + split_to: OK (2.22s) + new: OK (1.84s) + split_off: OK (2.18s) + sym_len: OK (2.36s) + put_overflow: OK (2.25s) + vec_cursor_read: FAIL (2.72s) + standalone use of `dyn` is not supported: TyDynamic core/1cdf94ce::any[0]::Erased[0]::_trait58656ccc69864acf[0] CallStack (from HasCallStack): error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy tyToRepr, called at src/Mir/TransTy.hs:213:7 in crucible-mir-0.1-inplace:Mir.TransTy @@ -1877,39 +1482,46 @@ FAIL (expected: taking address of an overridden function) (0.05s) tyListToCtx, called at src/Mir/TransTy.hs:221:6 in crucible-mir-0.1-inplace:Mir.TransTy tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy tyListToCtx, called at src/Mir/TransTy.hs:1066:16 in crucible-mir-0.1-inplace:Mir.TransTy - traitVtableType, called at src/Mir/Trans.hs:2030:18 in crucible-mir-0.1-inplace:Mir.Trans - transVirtCall, called at src/Mir/Trans.hs:2153:31 in crucible-mir-0.1-inplace:Mir.Trans + traitVtableType, called at src/Mir/Trans.hs:2055:18 in crucible-mir-0.1-inplace:Mir.Trans + transVirtCall, called at src/Mir/Trans.hs:2178:31 in crucible-mir-0.1-inplace:Mir.Trans transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language Use -p '/vec_cursor_read/' to rerun this test only. - vec_write: OK (1.47s) - mux_init_mut: OK (0.99s) - mux_init_imm: OK (0.96s) - uninit_read: OK (0.91s) - valid_read: OK (0.96s) - zero_length: OK (0.91s) - out_of_bounds: OK (1.02s) - no_conc: OK (0.99s) - assert: FAIL (1.64s) - files test/symb_eval/concretize/assert.good and test/symb_eval/concretize/assert.out differ; test/symb_eval/concretize/assert.out contains: - test assert/e68faa4a::crux_test[0]: returned 1, FAILED - - failures: - - ---- assert/e68faa4a::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/beffc212::fmt[0]::write[0] - [Crux] Translation error in core/beffc212::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/beffc212::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.assert"' to rerun this test only. - array: OK (1.04s) - conc: OK (0.98s) - assert_ok: OK (1.55s) + vec_write: OK (2.32s) + write: FAIL (0.06s) + test/symb_eval/byteorder/write.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/write.rs) + Use -p '/Output testing.write/' to rerun this test only. + read: FAIL + test/symb_eval/byteorder/read.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/read.rs) + Use -p '/Output testing.read/' to rerun this test only. + slice_mut: OK (1.87s) + basic: OK (1.78s) + slice: OK (1.95s) + mux_slice: OK (2.04s) + mux: OK (1.85s) + uninit_read: OK (1.88s) + valid_read: OK (1.89s) + out_of_bounds: OK (1.81s) + zero_length: OK (1.84s) + downcast: OK (1.86s) + downcast_fail: OK (1.78s) + conditional: OK (1.84s) + literals: OK (1.72s) + arith: OK (1.87s) + leading_zeros: OK (1.90s) + overflowing_sub: OK (1.84s) + cmp: OK (1.79s) + symbolic: OK (1.71s) + from_to: OK (1.76s) + clone: OK (2.00s) + sort_by_key: OK (2.69s) + into_iter: OK (2.39s) + macro: OK (1.83s) + construct: OK (1.84s) + deserialize: OK (1.83s) crux coverage Output testing - coverage_try: warning: trailing semicolon in macro used in expression position + coverage_dual: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1923,8 +1535,8 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (1.01s) - coverage_cond: warning: trailing semicolon in macro used in expression position +OK (1.83s) + coverage: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1938,8 +1550,17 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (0.90s) - coverage: warning: trailing semicolon in macro used in expression position +FAIL (1.86s) + files test/coverage/coverage.good and test/coverage/coverage.out differ; test/coverage/coverage.out contains: + warning: branch condition never has a value other than [1] + ┌─ test/coverage/coverage.rs:11:15 + │ + 11 │ } else if x == 1 { + │ ^ + + + Use -p '$0=="crux-mir.crux coverage.Output testing.coverage"' to rerun this test only. + coverage_try: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1953,17 +1574,8 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (0.97s) - files test/coverage/coverage.good and test/coverage/coverage.out differ; test/coverage/coverage.out contains: - warning: branch condition never has a value other than [1] - ┌─ test/coverage/coverage.rs:11:15 - │ - 11 │ } else if x == 1 { - │ ^ - - - Use -p '$0=="crux-mir.crux coverage.Output testing.coverage"' to rerun this test only. - coverage_mono: warning: trailing semicolon in macro used in expression position +OK (1.93s) + coverage_cond: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1977,8 +1589,8 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (0.91s) - coverage_macro: warning: trailing semicolon in macro used in expression position +OK (1.78s) + coverage_loop: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1992,20 +1604,17 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (1.06s) - files test/coverage/coverage_macro.good and test/coverage/coverage_macro.out differ; test/coverage/coverage_macro.out contains: +FAIL (1.83s) + files test/coverage/coverage_loop.good and test/coverage/coverage_loop.out differ; test/coverage/coverage_loop.out contains: warning: branch condition never has value true - ┌─ test/coverage/coverage_macro.rs:7:12 + ┌─ test/coverage/coverage_loop.rs:12:19 │ - 7 │ if !$cond { - │ ^^^^^^ - · - 18 │ check!(u8::try_from(y_32).is_ok()); - │ ---------------------------------- in this macro invocation + 12 │ } else if i > 99 { + │ ^^^^^^ - Use -p '/coverage_macro/' to rerun this test only. - coverage_dual: warning: trailing semicolon in macro used in expression position + Use -p '/coverage_loop/' to rerun this test only. + coverage_shortcircuit: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -2019,8 +1628,23 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (0.88s) - coverage_match: warning: trailing semicolon in macro used in expression position +FAIL (1.88s) + files test/coverage/coverage_shortcircuit.good and test/coverage/coverage_shortcircuit.out differ; test/coverage/coverage_shortcircuit.out contains: + warning: branch condition never has value false + ┌─ test/coverage/coverage_shortcircuit.rs:9:8 + │ + 9 │ if x == 0 || x == 1 { + │ ^^^^^^^^^^^^^^^^ + + warning: branch condition never has a value other than [0] + ┌─ test/coverage/coverage_shortcircuit.rs:9:8 + │ + 9 │ if x == 0 || x == 1 { + │ ^ + + + Use -p '/coverage_shortcircuit/' to rerun this test only. + coverage_mono: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -2034,23 +1658,8 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (1.06s) - files test/coverage/coverage_match.good and test/coverage/coverage_match.out differ; test/coverage/coverage_match.out contains: - warning: branch condition never has value 1 - ┌─ test/coverage/coverage_match.rs:12:9 - │ - 12 │ let x = u8::symbolic("x"); - │ ^ - - warning: branch condition never has value 20 - ┌─ test/coverage/coverage_match.rs:22:9 - │ - 22 │ E::A => 1, - │ ^^^^ - - - Use -p '/coverage_match/' to rerun this test only. - coverage_shortcircuit: warning: trailing semicolon in macro used in expression position +OK (1.78s) + coverage_match: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -2064,23 +1673,23 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (0.99s) - files test/coverage/coverage_shortcircuit.good and test/coverage/coverage_shortcircuit.out differ; test/coverage/coverage_shortcircuit.out contains: - warning: branch condition never has value false - ┌─ test/coverage/coverage_shortcircuit.rs:9:8 - │ - 9 │ if x == 0 || x == 1 { - │ ^^^^^^^^^^^^^^^^ +FAIL (1.90s) + files test/coverage/coverage_match.good and test/coverage/coverage_match.out differ; test/coverage/coverage_match.out contains: + warning: branch condition never has value 1 + ┌─ test/coverage/coverage_match.rs:12:9 + │ + 12 │ let x = u8::symbolic("x"); + │ ^ - warning: branch condition never has a value other than [0] - ┌─ test/coverage/coverage_shortcircuit.rs:9:8 - │ - 9 │ if x == 0 || x == 1 { - │ ^ + warning: branch condition never has value 20 + ┌─ test/coverage/coverage_match.rs:22:9 + │ + 22 │ E::A => 1, + │ ^^^^ - Use -p '/coverage_shortcircuit/' to rerun this test only. - coverage_loop: warning: trailing semicolon in macro used in expression position + Use -p '/coverage_match/' to rerun this test only. + coverage_macro: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -2094,15 +1703,18 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (0.91s) - files test/coverage/coverage_loop.good and test/coverage/coverage_loop.out differ; test/coverage/coverage_loop.out contains: +FAIL (1.89s) + files test/coverage/coverage_macro.good and test/coverage/coverage_macro.out differ; test/coverage/coverage_macro.out contains: warning: branch condition never has value true - ┌─ test/coverage/coverage_loop.rs:12:19 + ┌─ test/coverage/coverage_macro.rs:7:12 │ - 12 │ } else if i > 99 { - │ ^^^^^^ + 7 │ if !$cond { + │ ^^^^^^ + · + 18 │ check!(u8::try_from(y_32).is_ok()); + │ ---------------------------------- in this macro invocation - Use -p '/coverage_loop/' to rerun this test only. + Use -p '/coverage_macro/' to rerun this test only. -64 out of 339 tests failed (461.96s) +30 out of 339 tests failed (781.97s) From faef7f1e4521a36d01e0c0dcab9d3ba7fbf03159 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Tue, 30 May 2023 14:50:40 -0400 Subject: [PATCH 072/114] Use crucible_null_hook to implement null/null_mut --- crucible-mir/src/Mir/TransCustom.hs | 19 +++++++++++++++++++ crux-mir/lib/Patches.md | 8 ++++++++ crux-mir/lib/core/src/ptr/mod.rs | 10 ++++++++-- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/crucible-mir/src/Mir/TransCustom.hs b/crucible-mir/src/Mir/TransCustom.hs index c82dd166d..939554771 100644 --- a/crucible-mir/src/Mir/TransCustom.hs +++ b/crucible-mir/src/Mir/TransCustom.hs @@ -149,6 +149,8 @@ customOpDefs = Map.fromList $ [ , ptr_read , ptr_write , ptr_swap + , ptr_null + , ptr_null_mut , intrinsics_copy , intrinsics_copy_nonoverlapping @@ -564,6 +566,23 @@ ptr_swap = ( ["core", "ptr", "swap"], \substs -> case substs of _ -> mirFail $ "bad arguments for ptr::swap: " ++ show ops _ -> Nothing) +ptr_null :: (ExplodedDefId, CustomRHS) +ptr_null = ( ["core", "ptr", "null", "crucible_null_hook"] + , null_ptr_impl "ptr::null" ) + +ptr_null_mut :: (ExplodedDefId, CustomRHS) +ptr_null_mut = ( ["core", "ptr", "null_mut", "crucible_null_hook"] + , null_ptr_impl "ptr::null_mut" ) + +null_ptr_impl :: String -> CustomRHS +null_ptr_impl what substs = case substs of + Substs [ty] -> Just $ CustomOp $ \_ ops -> case ops of + [] -> do + Some tpr <- tyToReprM ty + ref <- integerToMirRef tpr $ R.App $ eBVLit knownNat 0 + return $ MirExp (MirReferenceRepr tpr) ref + _ -> mirFail $ "expected no arguments for " ++ what ++ ", received: " ++ show ops + _ -> Nothing intrinsics_copy :: (ExplodedDefId, CustomRHS) intrinsics_copy = ( ["core", "intrinsics", "copy"], \substs -> case substs of diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index ca7673999..afda945dd 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -97,3 +97,11 @@ identify all of the code that was changed in each patch. * Disable bytewise equality comparisons for `[T]` (last applied: May 26, 2023) These require the `size_of_val` intrinsic, which isn't current supported. + +* Use `crucible_null_hook` to implement `null`/`null_mut` (last applied: May 30, 2023) + + Morally, the `null`/`null_mut` functions behave like `0 as *const`/`0 as + *mut`, but due to Strict Pointer Provenance, their actual implementations are + much trickier than that. Unfortunately, we can't just inline `0 as *const`/`0 + as *mut`, as recent versions of `rustc` won't typecheck that. We resort to + overriding custom `crucible_null_hook` functions instead. diff --git a/crux-mir/lib/core/src/ptr/mod.rs b/crux-mir/lib/core/src/ptr/mod.rs index a2c72e670..dbb74eddf 100644 --- a/crux-mir/lib/core/src/ptr/mod.rs +++ b/crux-mir/lib/core/src/ptr/mod.rs @@ -513,7 +513,10 @@ pub unsafe fn drop_in_place(to_drop: *mut T) { #[rustc_allow_const_fn_unstable(ptr_metadata)] #[rustc_diagnostic_item = "ptr_null"] pub const fn null() -> *const T { - from_raw_parts(invalid(0), ()) + const fn crucible_null_hook() -> *const T { + from_raw_parts(invalid(0), ()) + } + crucible_null_hook() } /// Creates a null mutable raw pointer. @@ -534,7 +537,10 @@ pub const fn null() -> *const T { #[rustc_allow_const_fn_unstable(ptr_metadata)] #[rustc_diagnostic_item = "ptr_null_mut"] pub const fn null_mut() -> *mut T { - from_raw_parts_mut(invalid_mut(0), ()) + const fn crucible_null_hook() -> *mut T { + from_raw_parts_mut(invalid_mut(0), ()) + } + crucible_null_hook() } /// Creates an invalid pointer with the given address. From 8a599bc8f480589f5382821fded5ae13ace39798 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 31 May 2023 11:20:24 -0400 Subject: [PATCH 073/114] crux-mir-test: Accept new coverage output --- crux-mir/test/coverage/coverage.good | 4 ++-- crux-mir/test/coverage/coverage_loop.good | 24 ------------------- crux-mir/test/coverage/coverage_macro.good | 2 +- crux-mir/test/coverage/coverage_match.good | 12 +++++----- .../test/coverage/coverage_shortcircuit.good | 6 ++--- 5 files changed, 12 insertions(+), 36 deletions(-) diff --git a/crux-mir/test/coverage/coverage.good b/crux-mir/test/coverage/coverage.good index 7a9bcd967..2296f4bf5 100644 --- a/crux-mir/test/coverage/coverage.good +++ b/crux-mir/test/coverage/coverage.good @@ -1,6 +1,6 @@ -warning: branch condition never has value false +warning: branch condition never has a value other than [1] ┌─ test/coverage/coverage.rs:11:15 │ 11 │ } else if x == 1 { - │ ^^^^^^ + │ ^ diff --git a/crux-mir/test/coverage/coverage_loop.good b/crux-mir/test/coverage/coverage_loop.good index cf35fff09..0a6082833 100644 --- a/crux-mir/test/coverage/coverage_loop.good +++ b/crux-mir/test/coverage/coverage_loop.good @@ -1,27 +1,3 @@ -warning: branch condition never has value true - ┌─ ./lib/libcore/convert/num.rs:220:20 - │ -220 │ if u > (Self::max_value() as $source) { - │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: branch condition never has a value other than [1] - ┌─ ./lib/libcore/iter/range.rs:217:20 - │ -217 │ if let Some(mut n) = self.start.add_usize(1) { - │ ^^^^^^^^^^^ - -warning: branch condition never has value 1 - ┌─ ./lib/libcore/iter/range.rs:135:21 - │ -135 │ Ok(n_as_unsigned) => { - │ ^^^^^^^^^^^^^^^^^ - -warning: branch condition never has value false - ┌─ ./lib/libcore/iter/range.rs:140:28 - │ -140 │ if wrapped >= *self { - │ ^^^^^^^^^^^^^^^^ - warning: branch condition never has value true ┌─ test/coverage/coverage_loop.rs:12:19 │ diff --git a/crux-mir/test/coverage/coverage_macro.good b/crux-mir/test/coverage/coverage_macro.good index d9a1473f0..69b680f22 100644 --- a/crux-mir/test/coverage/coverage_macro.good +++ b/crux-mir/test/coverage/coverage_macro.good @@ -5,5 +5,5 @@ warning: branch condition never has value true │ ^^^^^^ · 18 │ check!(u8::try_from(y_32).is_ok()); - │ ----------------------------------- in this macro invocation + │ ---------------------------------- in this macro invocation diff --git a/crux-mir/test/coverage/coverage_match.good b/crux-mir/test/coverage/coverage_match.good index 32179a9a1..79c9f9872 100644 --- a/crux-mir/test/coverage/coverage_match.good +++ b/crux-mir/test/coverage/coverage_match.good @@ -1,12 +1,12 @@ warning: branch condition never has value 1 - ┌─ test/coverage/coverage_match.rs:14:19 + ┌─ test/coverage/coverage_match.rs:12:9 │ -14 │ let e = match x { - │ ^ +12 │ let x = u8::symbolic("x"); + │ ^ warning: branch condition never has value 20 - ┌─ test/coverage/coverage_match.rs:21:19 + ┌─ test/coverage/coverage_match.rs:22:9 │ -21 │ let y = match e { - │ ^ +22 │ E::A => 1, + │ ^^^^ diff --git a/crux-mir/test/coverage/coverage_shortcircuit.good b/crux-mir/test/coverage/coverage_shortcircuit.good index a45dec04e..e61365a96 100644 --- a/crux-mir/test/coverage/coverage_shortcircuit.good +++ b/crux-mir/test/coverage/coverage_shortcircuit.good @@ -2,11 +2,11 @@ warning: branch condition never has value false ┌─ test/coverage/coverage_shortcircuit.rs:9:8 │ 9 │ if x == 0 || x == 1 { - │ ^^^^^^ + │ ^^^^^^^^^^^^^^^^ -warning: branch condition never has value false +warning: branch condition never has a value other than [0] ┌─ test/coverage/coverage_shortcircuit.rs:9:8 │ 9 │ if x == 0 || x == 1 { - │ ^^^^^^^^^^^^^^^^ + │ ^ From d6ab13ae686d30271490faa3b11ee64fd815eafe Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 31 May 2023 11:31:49 -0400 Subject: [PATCH 074/114] crucible-mir: Override clone/clone_from properly --- crucible-mir/src/Mir/Generator.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crucible-mir/src/Mir/Generator.hs b/crucible-mir/src/Mir/Generator.hs index 0895ffa11..7263ecabd 100644 --- a/crucible-mir/src/Mir/Generator.hs +++ b/crucible-mir/src/Mir/Generator.hs @@ -489,10 +489,10 @@ resolveCustom instDefId = do f <- use $ customOps . fnPtrShimOp return $ Just $ f ty IkCloneShim ty parts - | intr ^. intrInst . inDefId == textId "core::clone::Clone::clone" -> do + | idKey (intr ^. intrInst . inDefId) == ["core", "clone", "Clone", "clone"] -> do f <- use $ customOps . cloneShimOp return $ Just $ f ty parts - | intr ^. intrInst . inDefId == textId "core::clone::Clone::clone_from" -> do + | idKey (intr ^. intrInst . inDefId) == ["core", "clone", "Clone", "clone_from"] -> do f <- use $ customOps . cloneFromShimOp return $ Just $ f ty parts | otherwise -> mirFail $ From 29f737dfca24413fab8bf04e43218dd5e6830875 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 1 Jun 2023 13:14:16 -0400 Subject: [PATCH 075/114] Implement Demand using PhantomData instead of a DST --- crux-mir/lib/Patches.md | 4 ++++ crux-mir/lib/core/src/any.rs | 16 ++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index afda945dd..fea10f545 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -105,3 +105,7 @@ identify all of the code that was changed in each patch. much trickier than that. Unfortunately, we can't just inline `0 as *const`/`0 as *mut`, as recent versions of `rustc` won't typecheck that. We resort to overriding custom `crucible_null_hook` functions instead. + +* Implement `Demand` using `PhantomData` instead of a dynamically sized `dyn` (last applied: June 1, 2023) + + We do not current support dynamically sized types in crucible-mir. diff --git a/crux-mir/lib/core/src/any.rs b/crux-mir/lib/core/src/any.rs index c0fb0d993..c8595d9a2 100644 --- a/crux-mir/lib/core/src/any.rs +++ b/crux-mir/lib/core/src/any.rs @@ -154,6 +154,7 @@ use crate::fmt; use crate::intrinsics; +use crate::marker::PhantomData; /////////////////////////////////////////////////////////////////////////////// // Any trait @@ -868,7 +869,12 @@ where /// A data provider provides values by calling this type's provide methods. #[unstable(feature = "provide_any", issue = "96024")] #[repr(transparent)] -pub struct Demand<'a>(dyn Erased<'a> + 'a); +// NB: We have replaced a use of `dyn Erased` with `PhantomData` here. The +// former is a dynamically sized type, which we do not yet support in +// crucible-mir. We have to make some questionable changes to the `impl` block +// below to accommodate this change of type, but we do not aim to support this +// sort of code yet anyway. +pub struct Demand<'a>(PhantomData + 'a>); impl<'a> Demand<'a> { /// Create a new `&mut Demand` from a `&mut dyn Erased` trait object. @@ -993,9 +999,6 @@ impl<'a> Demand<'a> { where I: tags::Type<'a>, { - if let Some(res @ TaggedOption(None)) = self.0.downcast_mut::() { - res.0 = Some(value); - } self } @@ -1004,9 +1007,6 @@ impl<'a> Demand<'a> { where I: tags::Type<'a>, { - if let Some(res @ TaggedOption(None)) = self.0.downcast_mut::() { - res.0 = Some(fulfil()); - } self } @@ -1157,7 +1157,7 @@ impl<'a> Demand<'a> { where I: tags::Type<'a>, { - matches!(self.0.downcast::(), Some(TaggedOption(None))) + true } } From b70354485fa257a83f26240318e059ad066fcc3e Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 1 Jun 2023 14:17:59 -0400 Subject: [PATCH 076/114] crux-mir: Re-implement timing Partially adapted from cd3cea5b147d81b7792891827c1afa7f4de15ff3 --- crux-mir/lib/Patches.md | 6 ++ crux-mir/lib/std/src/fs.rs | 3 +- crux-mir/lib/std/src/lib.rs | 1 + crux-mir/lib/std/src/sys/crux/condvar.rs | 44 +++++++++++ crux-mir/lib/std/src/sys/crux/mod.rs | 4 + crux-mir/lib/std/src/sys/crux/mutex.rs | 94 ++++++++++++++++++++++++ crux-mir/lib/std/src/sys/crux/rwlock.rs | 70 ++++++++++++++++++ crux-mir/lib/std/src/sys/crux/time.rs | 79 ++++++++++++++++++++ crux-mir/lib/std/src/sys/mod.rs | 1 + crux-mir/lib/std/src/time.rs | 9 ++- 10 files changed, 309 insertions(+), 2 deletions(-) create mode 100644 crux-mir/lib/std/src/sys/crux/condvar.rs create mode 100644 crux-mir/lib/std/src/sys/crux/mod.rs create mode 100644 crux-mir/lib/std/src/sys/crux/mutex.rs create mode 100644 crux-mir/lib/std/src/sys/crux/rwlock.rs create mode 100644 crux-mir/lib/std/src/sys/crux/time.rs diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index fea10f545..4fe9cfe58 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -109,3 +109,9 @@ identify all of the code that was changed in each patch. * Implement `Demand` using `PhantomData` instead of a dynamically sized `dyn` (last applied: June 1, 2023) We do not current support dynamically sized types in crucible-mir. + +* Replace `sys::time` with Crux-specific implementation (last applied: June 1, 2023) + + Crux's version is not suitable for doing actual timing (it hard-codes the + time to a fixed date), but it does simulate much more easily than the actual + implementation. diff --git a/crux-mir/lib/std/src/fs.rs b/crux-mir/lib/std/src/fs.rs index 286ad68fd..94c7289fa 100644 --- a/crux-mir/lib/std/src/fs.rs +++ b/crux-mir/lib/std/src/fs.rs @@ -17,7 +17,8 @@ use crate::io::{self, BorrowedCursor, IoSlice, IoSliceMut, Read, Seek, SeekFrom, use crate::path::{Path, PathBuf}; use crate::sys::fs as fs_imp; use crate::sys_common::{AsInner, AsInnerMut, FromInner, IntoInner}; -use crate::time::SystemTime; +use crate::time; +use crate::sys::crux::time::SystemTime; /// An object providing access to an open file on the filesystem. /// diff --git a/crux-mir/lib/std/src/lib.rs b/crux-mir/lib/std/src/lib.rs index 99a258cfe..a55aec91c 100644 --- a/crux-mir/lib/std/src/lib.rs +++ b/crux-mir/lib/std/src/lib.rs @@ -365,6 +365,7 @@ #![feature(thread_local_internals)] // #![default_lib_allocator] +#![feature(crucible_intrinsics)] // Explicitly import the prelude. The compiler uses this same unstable attribute // to import the prelude implicitly when building crates that depend on std. diff --git a/crux-mir/lib/std/src/sys/crux/condvar.rs b/crux-mir/lib/std/src/sys/crux/condvar.rs new file mode 100644 index 000000000..8b7e42468 --- /dev/null +++ b/crux-mir/lib/std/src/sys/crux/condvar.rs @@ -0,0 +1,44 @@ +use crate::sys::crux::mutex::Mutex; +use crate::time::Duration; + +pub struct Condvar { +} + +unsafe impl Send for Condvar {} +unsafe impl Sync for Condvar {} + +impl Condvar { + pub const fn new() -> Condvar { + Condvar {} + } + + pub unsafe fn init(&mut self) { + // No-op + } + + #[inline] + pub unsafe fn notify_one(&self) { + // No-op + } + + #[inline] + pub unsafe fn notify_all(&self) { + // No-op + } + + #[inline] + pub unsafe fn wait(&self, mutex: &Mutex) { + panic!("deadlock: single-threaded program is waiting on a condvar"); + } + + #[inline] + pub unsafe fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool { + // Pretend to have timed out, as there's no other way to end a wait. + false + } + + #[inline] + pub unsafe fn destroy(&self) { + // No-op + } +} diff --git a/crux-mir/lib/std/src/sys/crux/mod.rs b/crux-mir/lib/std/src/sys/crux/mod.rs new file mode 100644 index 000000000..08110995f --- /dev/null +++ b/crux-mir/lib/std/src/sys/crux/mod.rs @@ -0,0 +1,4 @@ +pub mod condvar; +pub mod mutex; +pub mod rwlock; +pub mod time; diff --git a/crux-mir/lib/std/src/sys/crux/mutex.rs b/crux-mir/lib/std/src/sys/crux/mutex.rs new file mode 100644 index 000000000..792ddef0d --- /dev/null +++ b/crux-mir/lib/std/src/sys/crux/mutex.rs @@ -0,0 +1,94 @@ +#![allow(dead_code)] +use crate::cell::Cell; +use crate::mem; +use core::crucible::concurrency; + +pub struct Mutex { + locked: Cell, +} + +unsafe impl Send for Mutex {} +unsafe impl Sync for Mutex {} + +impl Mutex { + pub const fn new() -> Mutex { + Mutex { + locked: Cell::new(false), + } + } + #[inline] + pub unsafe fn init(&mut self) { + // No-op + } + #[inline] + pub unsafe fn lock(&self) { + // TODO: Currently, we never run `drop` code, so locks are never released. Once we support + // drops, we can enable this assertion to check for (invalid) reentrant locking. + // assert!(!self.locked.get()); + concurrency::mutex_lock(self); + self.locked.set(true); + } + #[inline] + pub unsafe fn unlock(&self) { + // TODO: Currently, we never run `drop` code, so locks are never released. Once we support + // drops, we can enable this assertion to check for invalid usage. + concurrency::mutex_unlock(self); + // assert!(self.locked.get()); + self.locked.set(false); + } + #[inline] + pub unsafe fn try_lock(&self) -> bool { + if self.locked.get() { + false + } else { + self.locked.set(true); + true + } + } + #[inline] + pub unsafe fn destroy(&self) { + // No-op + } +} + +pub struct ReentrantMutex { + lock_count: Cell, +} + +unsafe impl Send for ReentrantMutex {} +unsafe impl Sync for ReentrantMutex {} + +impl ReentrantMutex { + pub const unsafe fn uninitialized() -> ReentrantMutex { + ReentrantMutex { + lock_count: Cell::new(0), + } + } + + #[inline] + pub unsafe fn init(&self) { + // No-op + } + + #[inline] + pub unsafe fn lock(&self) { + self.lock_count.set(self.lock_count.get() + 1); + } + + #[inline] + pub unsafe fn try_lock(&self) -> bool { + // There is only one thread, so locking cannot fail. + self.lock(); + true + } + + #[inline] + pub unsafe fn unlock(&self) { + self.lock_count.set(self.lock_count.get() - 1); + } + + #[inline] + pub unsafe fn destroy(&self) { + // No-op + } +} diff --git a/crux-mir/lib/std/src/sys/crux/rwlock.rs b/crux-mir/lib/std/src/sys/crux/rwlock.rs new file mode 100644 index 000000000..99b060290 --- /dev/null +++ b/crux-mir/lib/std/src/sys/crux/rwlock.rs @@ -0,0 +1,70 @@ +use crate::cell::Cell; + +pub struct RWLock { + num_readers: Cell, + write_locked: Cell, +} + +unsafe impl Send for RWLock {} +unsafe impl Sync for RWLock {} + +impl RWLock { + pub const fn new() -> RWLock { + RWLock { + num_readers: Cell::new(0), + write_locked: Cell::new(false), + } + } + #[inline] + pub unsafe fn read(&self) { + // TODO: Currently, we never run `drop` code, so locks are never released. Once we support + // drops, we can enable this assertion to check for invalid usage. + //assert!(!self.write_locked.get()); + self.num_readers.set(self.num_readers.get() + 1); + } + + #[inline] + pub unsafe fn try_read(&self) -> bool { + if self.write_locked.get() { + false + } else { + self.read(); + true + } + } + + #[inline] + pub unsafe fn write(&self) { + // TODO: Currently, we never run `drop` code, so locks are never released. Once we support + // drops, we can enable this assertion to check for invalid usage. + //assert!(!self.write_locked.get()); + //assert!(self.num_readers.get() == 0); + self.write_locked.set(true); + } + + #[inline] + pub unsafe fn try_write(&self) -> bool { + if self.write_locked.get() || self.num_readers.get() > 0 { + false + } else { + self.write(); + true + } + } + + #[inline] + pub unsafe fn read_unlock(&self) { + self.num_readers.set(self.num_readers.get() - 1); + } + + #[inline] + pub unsafe fn write_unlock(&self) { + assert!(self.write_locked.get()); + self.write_locked.set(false); + } + + #[inline] + pub unsafe fn destroy(&self) { + // No-op + } +} diff --git a/crux-mir/lib/std/src/sys/crux/time.rs b/crux-mir/lib/std/src/sys/crux/time.rs new file mode 100644 index 000000000..68437f6b0 --- /dev/null +++ b/crux-mir/lib/std/src/sys/crux/time.rs @@ -0,0 +1,79 @@ +#![unstable(feature = "crucible_intrinsics", issue = "none")] + +use core::convert::TryInto; +use core::time::Duration; +use crate::sys; +use crate::sys_common::{FromInner, IntoInner}; + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] +pub struct Instant(u64); + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] +pub struct SystemTime(i64); + +pub const UNIX_EPOCH: SystemTime = SystemTime(0); + +impl Instant { + pub fn now() -> Instant { + Instant(0) + } + + pub const fn zero() -> Instant { + Instant(0) + } + + pub fn actually_monotonic() -> bool { + true + } + + pub fn checked_sub_instant(&self, other: &Instant) -> Option { + Some(Duration::from_nanos(self.0.checked_sub(other.0)?)) + } + + pub fn checked_add_duration(&self, other: &Duration) -> Option { + Some(Instant(self.0.checked_add(other.as_nanos().try_into().ok()?)?)) + } + + pub fn checked_sub_duration(&self, other: &Duration) -> Option { + Some(Instant(self.0.checked_sub(other.as_nanos().try_into().ok()?)?)) + } +} + +impl SystemTime { + pub fn now() -> SystemTime { + SystemTime(0) + } + + pub fn sub_time(&self, other: &SystemTime) -> Result { + if self.0 >= other.0 { + Ok(Duration::from_nanos((self.0 - other.0) as u64)) + } else { + Err(Duration::from_nanos((other.0 - self.0) as u64)) + } + } + + pub fn checked_add_duration(&self, other: &Duration) -> Option { + Some(SystemTime(self.0.checked_add(other.as_nanos().try_into().ok()?)?)) + } + + pub fn checked_sub_duration(&self, other: &Duration) -> Option { + Some(SystemTime(self.0.checked_sub(other.as_nanos().try_into().ok()?)?)) + } +} + +impl FromInner for SystemTime { + fn from_inner(time: sys::time::SystemTime) -> SystemTime { + match time.sub_time(&sys::time::UNIX_EPOCH) { + Ok(pos) => UNIX_EPOCH.checked_add_duration(&pos).unwrap(), + Err(neg) => UNIX_EPOCH.checked_sub_duration(&neg).unwrap(), + } + } +} + +impl IntoInner for SystemTime { + fn into_inner(self) -> sys::time::SystemTime { + // NB: A bit skeevy, but much easier than actually converting to a + // sys::time::SystemTime + sys::time::UNIX_EPOCH + } +} diff --git a/crux-mir/lib/std/src/sys/mod.rs b/crux-mir/lib/std/src/sys/mod.rs index c080c176a..c75b5e669 100644 --- a/crux-mir/lib/std/src/sys/mod.rs +++ b/crux-mir/lib/std/src/sys/mod.rs @@ -23,6 +23,7 @@ #![allow(missing_debug_implementations)] pub mod common; +pub mod crux; cfg_if::cfg_if! { if #[cfg(unix)] { diff --git a/crux-mir/lib/std/src/time.rs b/crux-mir/lib/std/src/time.rs index ecd06ebf7..4344a2cd4 100644 --- a/crux-mir/lib/std/src/time.rs +++ b/crux-mir/lib/std/src/time.rs @@ -37,7 +37,8 @@ mod tests; use crate::error::Error; use crate::fmt; use crate::ops::{Add, AddAssign, Sub, SubAssign}; -use crate::sys::time; +use crate::sys; +use crate::sys::crux::time; use crate::sys_common::{FromInner, IntoInner}; #[stable(feature = "time", since = "1.3.0")] @@ -692,3 +693,9 @@ impl IntoInner for SystemTime { self.0 } } + +impl FromInner for SystemTime { + fn from_inner(time: sys::time::SystemTime) -> SystemTime { + SystemTime(time::SystemTime::from_inner(time)) + } +} From 9e34725d70d71ba8d0ecb5a9de494ce4ecccd036 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 1 Jun 2023 15:53:32 -0400 Subject: [PATCH 077/114] crucible-mir: Translate ThreadLocalRefs properly We treat them as though they are references to statics in a single-threaded environment. --- crucible-mir/src/Mir/JSON.hs | 2 +- crucible-mir/src/Mir/Mir.hs | 5 ++--- crucible-mir/src/Mir/PP.hs | 2 +- crucible-mir/src/Mir/Trans.hs | 2 +- dependencies/mir-json | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/crucible-mir/src/Mir/JSON.hs b/crucible-mir/src/Mir/JSON.hs index 6744caf96..0cbcfadfe 100644 --- a/crucible-mir/src/Mir/JSON.hs +++ b/crucible-mir/src/Mir/JSON.hs @@ -311,7 +311,7 @@ instance FromJSON Rvalue where Just (String "Aggregate") -> Aggregate <$> v .: "akind" <*> v .: "ops" Just (String "ShallowInitBox") -> ShallowInitBox <$> v .: "ptr" <*> v .: "ty" Just (String "CopyForDeref") -> CopyForDeref <$> v .: "place" - Just (String "ThreadLocalRef") -> ThreadLocalRef <$> v .: "def_id" + Just (String "ThreadLocalRef") -> ThreadLocalRef <$> v .: "def_id" <*> v .: "ty" k -> fail $ "unsupported RValue " ++ show k instance FromJSON Terminator where diff --git a/crucible-mir/src/Mir/Mir.hs b/crucible-mir/src/Mir/Mir.hs index f400f7e02..4a77707d8 100644 --- a/crucible-mir/src/Mir/Mir.hs +++ b/crucible-mir/src/Mir/Mir.hs @@ -350,8 +350,7 @@ data Rvalue = | RAdtAg AdtAg | ShallowInitBox { _sibptr :: Operand, _sibty :: Ty } | CopyForDeref Lvalue - | ThreadLocalRef DefId - -- ^ TODO RGS: Is this right? + | ThreadLocalRef DefId Ty deriving (Show,Eq, Ord, Generic) data AdtAg = AdtAg { _agadt :: Adt, _avgariant :: Integer, _aops :: [Operand], _adtagty :: Ty } @@ -723,7 +722,7 @@ instance TypeOf Rvalue where typeOf (RAdtAg (AdtAg _ _ _ ty)) = ty typeOf (ShallowInitBox _ ty) = ty typeOf (CopyForDeref lv) = typeOf lv - typeOf (ThreadLocalRef _) = error "TODO RGS typeOf ThreadLocalRef" + typeOf (ThreadLocalRef _ ty) = ty instance TypeOf Operand where typeOf (Move lv) = typeOf lv diff --git a/crucible-mir/src/Mir/PP.hs b/crucible-mir/src/Mir/PP.hs index c827cca4a..fce38a1ad 100644 --- a/crucible-mir/src/Mir/PP.hs +++ b/crucible-mir/src/Mir/PP.hs @@ -207,7 +207,7 @@ instance Pretty Rvalue where pretty (RAdtAg a) = pretty a pretty (ShallowInitBox ptr ty) = pretty_fn2 "ShallowInitBox" ptr ty pretty (CopyForDeref lv) = pretty_fn1 "CopyForDeref" lv - pretty (ThreadLocalRef a) = pretty_fn1 "ThreadLocalRef" a + pretty (ThreadLocalRef a b) = pretty_fn2 "ThreadLocalRef" a b instance Pretty AdtAg where pretty (AdtAg (Adt nm _kind _vs _ _ _ _) i ops _) = pretty_fn3 "AdtAg" nm i ops diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index cabf7c618..d586b917f 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -1027,11 +1027,11 @@ evalRval rv@(M.RAdtAg (M.AdtAg adt agv ops ty)) = do M.Union -> do mirFail $ "evalRval: Union types are unsupported, for " ++ show (adt ^. adtname) _ -> mirFail $ "evalRval: unsupported type for AdtAg: " ++ show ty +evalRval (M.ThreadLocalRef did _) = staticPlace did >>= addrOfPlace -- TODO: are these correct? evalRval rv@(M.ShallowInitBox op ty) = evalOperand op evalRval rv@(M.CopyForDeref lv) = evalLvalue lv -evalRval (M.ThreadLocalRef _) = error "TODO RGS evalRval ThreadLocalRef" evalLvalue :: HasCallStack => M.Lvalue -> MirGenerator h s ret (MirExp s) evalLvalue lv = evalPlace lv >>= readPlace diff --git a/dependencies/mir-json b/dependencies/mir-json index 3b0f12244..861d52a74 160000 --- a/dependencies/mir-json +++ b/dependencies/mir-json @@ -1 +1 @@ -Subproject commit 3b0f12244ef993e108f348bdbd3e1829543fc7a1 +Subproject commit 861d52a74048e93edd139b48ffe576adbf516218 From 550bb9cf8820ac36863adeb8f255ab4e447756e9 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 1 Jun 2023 15:58:24 -0400 Subject: [PATCH 078/114] Replace sys::{condvar,mutex,rwlock} with Crux-specific implementations --- crux-mir/lib/Patches.md | 5 +++++ crux-mir/lib/std/src/sync/condvar.rs | 2 +- crux-mir/lib/std/src/sync/mutex.rs | 2 +- crux-mir/lib/std/src/sync/remutex.rs | 2 +- crux-mir/lib/std/src/sync/rwlock.rs | 2 +- crux-mir/lib/std/src/sys/crux/condvar.rs | 12 ++++++------ crux-mir/lib/std/src/sys/crux/rwlock.rs | 12 ++++++------ 7 files changed, 21 insertions(+), 16 deletions(-) diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index 4fe9cfe58..a1e415de5 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -115,3 +115,8 @@ identify all of the code that was changed in each patch. Crux's version is not suitable for doing actual timing (it hard-codes the time to a fixed date), but it does simulate much more easily than the actual implementation. + +* Replace `sys::{condvar,mutex,rwlock}` with Crux-specific implementation (last applied: June 1, 2023) + + Because Crucible is effectively single-threaded, we can safely replace these + with much simpler implementations that aren't nearly as tricky to simulate. diff --git a/crux-mir/lib/std/src/sync/condvar.rs b/crux-mir/lib/std/src/sync/condvar.rs index 76a1b4a2a..424160588 100644 --- a/crux-mir/lib/std/src/sync/condvar.rs +++ b/crux-mir/lib/std/src/sync/condvar.rs @@ -3,7 +3,7 @@ mod tests; use crate::fmt; use crate::sync::{mutex, poison, LockResult, MutexGuard, PoisonError}; -use crate::sys::locks as sys; +use crate::sys::crux::condvar as sys; use crate::time::{Duration, Instant}; /// A type indicating whether a timed wait on a condition variable returned diff --git a/crux-mir/lib/std/src/sync/mutex.rs b/crux-mir/lib/std/src/sync/mutex.rs index 065045f44..1c5e2e895 100644 --- a/crux-mir/lib/std/src/sync/mutex.rs +++ b/crux-mir/lib/std/src/sync/mutex.rs @@ -5,7 +5,7 @@ use crate::cell::UnsafeCell; use crate::fmt; use crate::ops::{Deref, DerefMut}; use crate::sync::{poison, LockResult, TryLockError, TryLockResult}; -use crate::sys::locks as sys; +use crate::sys::crux::mutex as sys; /// A mutual exclusion primitive useful for protecting shared data /// diff --git a/crux-mir/lib/std/src/sync/remutex.rs b/crux-mir/lib/std/src/sync/remutex.rs index 4c054da64..7396a398d 100644 --- a/crux-mir/lib/std/src/sync/remutex.rs +++ b/crux-mir/lib/std/src/sync/remutex.rs @@ -5,7 +5,7 @@ use crate::cell::UnsafeCell; use crate::ops::Deref; use crate::panic::{RefUnwindSafe, UnwindSafe}; use crate::sync::atomic::{AtomicUsize, Ordering::Relaxed}; -use crate::sys::locks as sys; +use crate::sys::crux::mutex as sys; /// A re-entrant mutual exclusion /// diff --git a/crux-mir/lib/std/src/sync/rwlock.rs b/crux-mir/lib/std/src/sync/rwlock.rs index 7c409cb3e..971469d1a 100644 --- a/crux-mir/lib/std/src/sync/rwlock.rs +++ b/crux-mir/lib/std/src/sync/rwlock.rs @@ -6,7 +6,7 @@ use crate::fmt; use crate::ops::{Deref, DerefMut}; use crate::ptr::NonNull; use crate::sync::{poison, LockResult, TryLockError, TryLockResult}; -use crate::sys::locks as sys; +use crate::sys::crux::rwlock as sys; /// A reader-writer lock /// diff --git a/crux-mir/lib/std/src/sys/crux/condvar.rs b/crux-mir/lib/std/src/sys/crux/condvar.rs index 8b7e42468..68e7992f7 100644 --- a/crux-mir/lib/std/src/sys/crux/condvar.rs +++ b/crux-mir/lib/std/src/sys/crux/condvar.rs @@ -12,33 +12,33 @@ impl Condvar { Condvar {} } - pub unsafe fn init(&mut self) { + pub fn init(&mut self) { // No-op } #[inline] - pub unsafe fn notify_one(&self) { + pub fn notify_one(&self) { // No-op } #[inline] - pub unsafe fn notify_all(&self) { + pub fn notify_all(&self) { // No-op } #[inline] - pub unsafe fn wait(&self, mutex: &Mutex) { + pub fn wait(&self, mutex: &Mutex) { panic!("deadlock: single-threaded program is waiting on a condvar"); } #[inline] - pub unsafe fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool { + pub fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool { // Pretend to have timed out, as there's no other way to end a wait. false } #[inline] - pub unsafe fn destroy(&self) { + pub fn destroy(&self) { // No-op } } diff --git a/crux-mir/lib/std/src/sys/crux/rwlock.rs b/crux-mir/lib/std/src/sys/crux/rwlock.rs index 99b060290..7f5516462 100644 --- a/crux-mir/lib/std/src/sys/crux/rwlock.rs +++ b/crux-mir/lib/std/src/sys/crux/rwlock.rs @@ -1,16 +1,16 @@ use crate::cell::Cell; -pub struct RWLock { +pub struct RwLock { num_readers: Cell, write_locked: Cell, } -unsafe impl Send for RWLock {} -unsafe impl Sync for RWLock {} +unsafe impl Send for RwLock {} +unsafe impl Sync for RwLock {} -impl RWLock { - pub const fn new() -> RWLock { - RWLock { +impl RwLock { + pub const fn new() -> RwLock { + RwLock { num_readers: Cell::new(0), write_locked: Cell::new(false), } From 76784e1e7a452a47ed7db99c2ac1c66aafa11438 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 1 Jun 2023 16:33:53 -0400 Subject: [PATCH 079/114] crux-mir-test: Update golden output for symbolic tests --- .../test/symb_eval/alloc/out_of_bounds.good | 8 +++---- .../test/symb_eval/alloc/uninit_read.good | 8 +++---- crux-mir/test/symb_eval/alloc/valid_read.good | 2 +- .../test/symb_eval/alloc/zero_length.good | 2 +- crux-mir/test/symb_eval/any/conditional.good | 2 +- crux-mir/test/symb_eval/any/downcast.good | 2 +- .../test/symb_eval/any/downcast_fail.good | 6 ++--- crux-mir/test/symb_eval/array/basic.good | 2 +- crux-mir/test/symb_eval/array/mux_slice.good | 2 +- crux-mir/test/symb_eval/array/slice.good | 2 +- crux-mir/test/symb_eval/array/slice_mut.good | 2 +- crux-mir/test/symb_eval/bitvector/arith.good | 2 +- crux-mir/test/symb_eval/bitvector/cmp.good | 2 +- .../test/symb_eval/bitvector/from_to.good | 2 +- .../symb_eval/bitvector/leading_zeros.good | 2 +- .../test/symb_eval/bitvector/literals.good | 2 +- .../symb_eval/bitvector/overflowing_sub.good | 2 +- .../test/symb_eval/bitvector/symbolic.good | 2 +- .../test/symb_eval/bytes/extend_bytes.good | 2 +- crux-mir/test/symb_eval/bytes/new.good | 2 +- crux-mir/test/symb_eval/bytes/put.good | 2 +- .../test/symb_eval/bytes/put_overflow.good | 8 +++---- crux-mir/test/symb_eval/bytes/split_off.good | 2 +- crux-mir/test/symb_eval/bytes/split_to.good | 2 +- crux-mir/test/symb_eval/bytes/sym_len.good | 2 +- crux-mir/test/symb_eval/concretize/array.good | 2 +- .../test/symb_eval/concretize/assert_ok.good | 2 +- crux-mir/test/symb_eval/concretize/conc.good | 2 +- .../test/symb_eval/concretize/no_conc.good | 2 +- crux-mir/test/symb_eval/crux/early_fail.good | 14 ++++++------ crux-mir/test/symb_eval/crux/fail_return.good | 16 +++++++------- crux-mir/test/symb_eval/crux/mixed_fail.good | 20 ++++++++--------- crux-mir/test/symb_eval/crux/multi.good | 22 +++++++++---------- crux-mir/test/symb_eval/crypto/bytes.good | 14 ++++++------ crux-mir/test/symb_eval/crypto/bytes2.good | 2 +- crux-mir/test/symb_eval/crypto/double.good | 2 +- crux-mir/test/symb_eval/crypto/ffs.good | 2 +- crux-mir/test/symb_eval/enum/mux.good | 2 +- crux-mir/test/symb_eval/fnptr/mux.good | 2 +- .../test/symb_eval/io/vec_cursor_read.good | 2 +- crux-mir/test/symb_eval/io/vec_write.good | 2 +- crux-mir/test/symb_eval/mux/array.good | 2 +- crux-mir/test/symb_eval/mux/array_mut.good | 2 +- crux-mir/test/symb_eval/num/checked_add.good | 6 ++--- crux-mir/test/symb_eval/num/checked_div.good | 10 ++++----- crux-mir/test/symb_eval/num/checked_mul.good | 6 ++--- .../symb_eval/num/checked_mul_signed.good | 6 ++--- .../test/symb_eval/overrides/bad_symb1.good | 6 ++--- .../test/symb_eval/overrides/bad_symb2.good | 6 ++--- .../test/symb_eval/overrides/override1.good | 2 +- .../test/symb_eval/overrides/override2.good | 6 ++--- .../test/symb_eval/overrides/override3.good | 2 +- .../test/symb_eval/overrides/override4.good | 2 +- .../test/symb_eval/overrides/override5.good | 6 ++--- .../symb_eval/overrides/override_rust.good | 2 +- .../test/symb_eval/refs/mux_init_imm.good | 2 +- .../test/symb_eval/refs/mux_init_mut.good | 2 +- crux-mir/test/symb_eval/scalar/test1.good | 2 +- .../test/symb_eval/sym_bytes/construct.good | 6 ++--- .../test/symb_eval/sym_bytes/deserialize.good | 6 ++--- crux-mir/test/symb_eval/vec/clone.good | 2 +- crux-mir/test/symb_eval/vec/into_iter.good | 2 +- crux-mir/test/symb_eval/vec/macro.good | 2 +- crux-mir/test/symb_eval/vec/sort_by_key.good | 2 +- .../test/symb_eval/vector/as_mut_slice.good | 2 +- crux-mir/test/symb_eval/vector/as_slice.good | 2 +- crux-mir/test/symb_eval/vector/concat.good | 2 +- .../symb_eval/vector/copy_from_slice.good | 2 +- crux-mir/test/symb_eval/vector/mut.good | 2 +- crux-mir/test/symb_eval/vector/new.good | 2 +- crux-mir/test/symb_eval/vector/push.good | 2 +- crux-mir/test/symb_eval/vector/replicate.good | 2 +- crux-mir/test/symb_eval/vector/split_at.good | 2 +- 73 files changed, 144 insertions(+), 144 deletions(-) diff --git a/crux-mir/test/symb_eval/alloc/out_of_bounds.good b/crux-mir/test/symb_eval/alloc/out_of_bounds.good index e4abadfbf..17df011f0 100644 --- a/crux-mir/test/symb_eval/alloc/out_of_bounds.good +++ b/crux-mir/test/symb_eval/alloc/out_of_bounds.good @@ -1,13 +1,13 @@ -test out_of_bounds/fb36419e::crux_test[0]: FAILED +test out_of_bounds/2674d8f8::crux_test[0]: FAILED failures: ----- out_of_bounds/fb36419e::crux_test[0] counterexamples ---- +---- out_of_bounds/2674d8f8::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/fb36419e::crux_test[0] +[Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/2674d8f8::crux_test[0] [Crux] vector index out of range: the length is 10 but the index is BV 12 [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/fb36419e::crux_test[0] +[Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/2674d8f8::crux_test[0] [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/alloc/uninit_read.good b/crux-mir/test/symb_eval/alloc/uninit_read.good index 8203fd082..1813c0935 100644 --- a/crux-mir/test/symb_eval/alloc/uninit_read.good +++ b/crux-mir/test/symb_eval/alloc/uninit_read.good @@ -1,13 +1,13 @@ -test uninit_read/9bd6592b::crux_test[0]: FAILED +test uninit_read/2bff4d2d::crux_test[0]: FAILED failures: ----- uninit_read/9bd6592b::crux_test[0] counterexamples ---- +---- uninit_read/2bff4d2d::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/9bd6592b::crux_test[0] +[Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/2bff4d2d::crux_test[0] [Crux] Attempted to read uninitialized vector index [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/9bd6592b::crux_test[0] +[Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/2bff4d2d::crux_test[0] [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/alloc/valid_read.good b/crux-mir/test/symb_eval/alloc/valid_read.good index e01406bff..c85bdcfab 100644 --- a/crux-mir/test/symb_eval/alloc/valid_read.good +++ b/crux-mir/test/symb_eval/alloc/valid_read.good @@ -1,3 +1,3 @@ -test valid_read/73b28820::crux_test[0]: returned 45, ok +test valid_read/3ad5949c::crux_test[0]: returned 45, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/alloc/zero_length.good b/crux-mir/test/symb_eval/alloc/zero_length.good index 992b092db..58a24d9ad 100644 --- a/crux-mir/test/symb_eval/alloc/zero_length.good +++ b/crux-mir/test/symb_eval/alloc/zero_length.good @@ -1,3 +1,3 @@ -test zero_length/aeacedba::crux_test[0]: ok +test zero_length/251728a4::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/any/conditional.good b/crux-mir/test/symb_eval/any/conditional.good index cfe6b55e7..039679893 100644 --- a/crux-mir/test/symb_eval/any/conditional.good +++ b/crux-mir/test/symb_eval/any/conditional.good @@ -1,3 +1,3 @@ -test conditional/1f766d14::crux_test[0]: ok +test conditional/6c557b21::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/any/downcast.good b/crux-mir/test/symb_eval/any/downcast.good index 636e88cba..0216b4f45 100644 --- a/crux-mir/test/symb_eval/any/downcast.good +++ b/crux-mir/test/symb_eval/any/downcast.good @@ -1,3 +1,3 @@ -test downcast/d939fa83::crux_test[0]: returned 1, ok +test downcast/a79fa030::crux_test[0]: returned 1, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/any/downcast_fail.good b/crux-mir/test/symb_eval/any/downcast_fail.good index 0f2d4da64..c4dc035ad 100644 --- a/crux-mir/test/symb_eval/any/downcast_fail.good +++ b/crux-mir/test/symb_eval/any/downcast_fail.good @@ -1,10 +1,10 @@ -test downcast_fail/a71ba9d2::crux_test[0]: FAILED +test downcast_fail/0c70442e::crux_test[0]: FAILED failures: ----- downcast_fail/a71ba9d2::crux_test[0] counterexamples ---- +---- downcast_fail/0c70442e::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/any/downcast_fail.rs:10:14: 10:15: error: in downcast_fail/a71ba9d2::crux_test[0] +[Crux] test/symb_eval/any/downcast_fail.rs:10:14: 10:15: error: in downcast_fail/0c70442e::crux_test[0] [Crux] failed to downcast Any as BVRepr 32 [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/array/basic.good b/crux-mir/test/symb_eval/array/basic.good index 9f0721a59..cd61dc7e1 100644 --- a/crux-mir/test/symb_eval/array/basic.good +++ b/crux-mir/test/symb_eval/array/basic.good @@ -1,3 +1,3 @@ -test basic/0010ffa4::crux_test[0]: returned 3, ok +test basic/b603207c::crux_test[0]: returned 3, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/array/mux_slice.good b/crux-mir/test/symb_eval/array/mux_slice.good index 745d81cd6..4f7bb7bbb 100644 --- a/crux-mir/test/symb_eval/array/mux_slice.good +++ b/crux-mir/test/symb_eval/array/mux_slice.good @@ -1,3 +1,3 @@ -test mux_slice/e736ab73::crux_test[0]: returned Symbolic BV, ok +test mux_slice/93b6889e::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/array/slice.good b/crux-mir/test/symb_eval/array/slice.good index c08294483..90588b935 100644 --- a/crux-mir/test/symb_eval/array/slice.good +++ b/crux-mir/test/symb_eval/array/slice.good @@ -1,3 +1,3 @@ -test slice/9ed070ad::crux_test[0]: returned 5, ok +test slice/04ec98ca::crux_test[0]: returned 5, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/array/slice_mut.good b/crux-mir/test/symb_eval/array/slice_mut.good index ad14f864d..a4f7cd4d2 100644 --- a/crux-mir/test/symb_eval/array/slice_mut.good +++ b/crux-mir/test/symb_eval/array/slice_mut.good @@ -1,3 +1,3 @@ -test slice_mut/d42dc8bf::crux_test[0]: returned 5, ok +test slice_mut/5c718432::crux_test[0]: returned 5, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/arith.good b/crux-mir/test/symb_eval/bitvector/arith.good index 7bbf853f0..14d6dfa48 100644 --- a/crux-mir/test/symb_eval/bitvector/arith.good +++ b/crux-mir/test/symb_eval/bitvector/arith.good @@ -1,3 +1,3 @@ -test arith/7bb6b57e::crux_test[0]: ok +test arith/2d4acabe::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/cmp.good b/crux-mir/test/symb_eval/bitvector/cmp.good index 78dc12dfc..c640fff18 100644 --- a/crux-mir/test/symb_eval/bitvector/cmp.good +++ b/crux-mir/test/symb_eval/bitvector/cmp.good @@ -1,3 +1,3 @@ -test cmp/9d74ce07::crux_test[0]: ok +test cmp/e92927a6::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/from_to.good b/crux-mir/test/symb_eval/bitvector/from_to.good index 920eed437..58973ead8 100644 --- a/crux-mir/test/symb_eval/bitvector/from_to.good +++ b/crux-mir/test/symb_eval/bitvector/from_to.good @@ -1,3 +1,3 @@ -test from_to/1c910e9c::crux_test[0]: ok +test from_to/0179ef0b::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/leading_zeros.good b/crux-mir/test/symb_eval/bitvector/leading_zeros.good index e2c33c0f4..c0462400e 100644 --- a/crux-mir/test/symb_eval/bitvector/leading_zeros.good +++ b/crux-mir/test/symb_eval/bitvector/leading_zeros.good @@ -1,3 +1,3 @@ -test leading_zeros/8a721fcc::crux_test[0]: ok +test leading_zeros/26d4da72::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/literals.good b/crux-mir/test/symb_eval/bitvector/literals.good index 2de3d3fb5..568a3a2ae 100644 --- a/crux-mir/test/symb_eval/bitvector/literals.good +++ b/crux-mir/test/symb_eval/bitvector/literals.good @@ -1,3 +1,3 @@ -test literals/ce6d61a1::crux_test[0]: ok +test literals/5d523333::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/overflowing_sub.good b/crux-mir/test/symb_eval/bitvector/overflowing_sub.good index d60697e57..a12d68b8c 100644 --- a/crux-mir/test/symb_eval/bitvector/overflowing_sub.good +++ b/crux-mir/test/symb_eval/bitvector/overflowing_sub.good @@ -1,3 +1,3 @@ -test overflowing_sub/d9667423::crux_test[0]: ok +test overflowing_sub/3e1181ed::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/symbolic.good b/crux-mir/test/symb_eval/bitvector/symbolic.good index 327bc23b3..7d6ed9301 100644 --- a/crux-mir/test/symb_eval/bitvector/symbolic.good +++ b/crux-mir/test/symb_eval/bitvector/symbolic.good @@ -1,3 +1,3 @@ -test symbolic/10258fd3::crux_test[0]: ok +test symbolic/ab685979::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/extend_bytes.good b/crux-mir/test/symb_eval/bytes/extend_bytes.good index b73531704..2d76adbf3 100644 --- a/crux-mir/test/symb_eval/bytes/extend_bytes.good +++ b/crux-mir/test/symb_eval/bytes/extend_bytes.good @@ -1,3 +1,3 @@ -test extend_bytes/aa56259e::f[0]: ok +test extend_bytes/87b45525::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/new.good b/crux-mir/test/symb_eval/bytes/new.good index 13d4da34e..37d4827e1 100644 --- a/crux-mir/test/symb_eval/bytes/new.good +++ b/crux-mir/test/symb_eval/bytes/new.good @@ -1,3 +1,3 @@ -test new/b3e2709a::f[0]: ok +test new/3e06dc75::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/put.good b/crux-mir/test/symb_eval/bytes/put.good index 5a5a70eee..4c1c47e3f 100644 --- a/crux-mir/test/symb_eval/bytes/put.good +++ b/crux-mir/test/symb_eval/bytes/put.good @@ -1,3 +1,3 @@ -test put/0fcb517b::f[0]: ok +test put/b1ab63fb::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/put_overflow.good b/crux-mir/test/symb_eval/bytes/put_overflow.good index f261a5442..9ba04a77e 100644 --- a/crux-mir/test/symb_eval/bytes/put_overflow.good +++ b/crux-mir/test/symb_eval/bytes/put_overflow.good @@ -1,10 +1,10 @@ -test put_overflow/84c199d7::f[0]: FAILED +test put_overflow/df566fcd::f[0]: FAILED failures: ----- put_overflow/84c199d7::f[0] counterexamples ---- +---- put_overflow/df566fcd::f[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] internal: error: in bytes/91c2c7c6::{impl#4}[0]::put_slice[0] -[Crux] panicking::panic_fmt, called from bytes/91c2c7c6::{impl#4}[0]::put_slice[0] +[Crux] internal: error: in bytes/dba3271f::{impl#4}[0]::put_slice[0] +[Crux] panicking::panic_fmt, called from bytes/dba3271f::{impl#4}[0]::put_slice[0] [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/bytes/split_off.good b/crux-mir/test/symb_eval/bytes/split_off.good index 4bd6be310..4b8a0522a 100644 --- a/crux-mir/test/symb_eval/bytes/split_off.good +++ b/crux-mir/test/symb_eval/bytes/split_off.good @@ -1,3 +1,3 @@ -test split_off/7c95a5fb::f[0]: ok +test split_off/50a75a9e::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/split_to.good b/crux-mir/test/symb_eval/bytes/split_to.good index 19e596453..4ea7fdeb2 100644 --- a/crux-mir/test/symb_eval/bytes/split_to.good +++ b/crux-mir/test/symb_eval/bytes/split_to.good @@ -1,3 +1,3 @@ -test split_to/2ad59b12::f[0]: ok +test split_to/b8cd8da6::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/sym_len.good b/crux-mir/test/symb_eval/bytes/sym_len.good index 235917a45..6a7d88b4d 100644 --- a/crux-mir/test/symb_eval/bytes/sym_len.good +++ b/crux-mir/test/symb_eval/bytes/sym_len.good @@ -1,3 +1,3 @@ -test sym_len/4bc70337::f[0]: ok +test sym_len/c7960ac7::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/concretize/array.good b/crux-mir/test/symb_eval/concretize/array.good index 6c171d270..45e78956d 100644 --- a/crux-mir/test/symb_eval/concretize/array.good +++ b/crux-mir/test/symb_eval/concretize/array.good @@ -1,3 +1,3 @@ -test array/f99e13ca::crux_test[0]: returned (1, 2, 3), ok +test array/a41ef166::crux_test[0]: returned (1, 2, 3), ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/concretize/assert_ok.good b/crux-mir/test/symb_eval/concretize/assert_ok.good index 2fc4024a0..4db821c59 100644 --- a/crux-mir/test/symb_eval/concretize/assert_ok.good +++ b/crux-mir/test/symb_eval/concretize/assert_ok.good @@ -1,3 +1,3 @@ -test assert_ok/d68c5200::crux_test[0]: returned 1, ok +test assert_ok/dec2bc1d::crux_test[0]: returned 1, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/concretize/conc.good b/crux-mir/test/symb_eval/concretize/conc.good index b3250b486..1944b12c6 100644 --- a/crux-mir/test/symb_eval/concretize/conc.good +++ b/crux-mir/test/symb_eval/concretize/conc.good @@ -1,3 +1,3 @@ -test conc/9dcb4d77::crux_test[0]: returned (1, 0), ok +test conc/8f6163ae::crux_test[0]: returned (1, 0), ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/concretize/no_conc.good b/crux-mir/test/symb_eval/concretize/no_conc.good index a1baa2660..18906e1d6 100644 --- a/crux-mir/test/symb_eval/concretize/no_conc.good +++ b/crux-mir/test/symb_eval/concretize/no_conc.good @@ -1,3 +1,3 @@ -test no_conc/65b2bc21::crux_test[0]: returned (Symbolic BV, Symbolic BV), ok +test no_conc/ab98a20c::crux_test[0]: returned (Symbolic BV, Symbolic BV), ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/crux/early_fail.good b/crux-mir/test/symb_eval/crux/early_fail.good index fc064b12b..e923d5233 100644 --- a/crux-mir/test/symb_eval/crux/early_fail.good +++ b/crux-mir/test/symb_eval/crux/early_fail.good @@ -1,16 +1,16 @@ -test early_fail/9e7e7cc7::fail1[0]: FAILED -test early_fail/9e7e7cc7::fail2[0]: FAILED +test early_fail/9ed3a747::fail1[0]: FAILED +test early_fail/9ed3a747::fail2[0]: FAILED failures: ----- early_fail/9e7e7cc7::fail1[0] counterexamples ---- +---- early_fail/9ed3a747::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] internal: error: in early_fail/9e7e7cc7::fail1[0] -[Crux] panicking::panic_fmt, called from early_fail/9e7e7cc7::fail1[0] +[Crux] internal: error: in early_fail/9ed3a747::fail1[0] +[Crux] panicking::panic_fmt, called from early_fail/9ed3a747::fail1[0] ----- early_fail/9e7e7cc7::fail2[0] counterexamples ---- +---- early_fail/9ed3a747::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/early_fail.rs:17:5: 17:29: error: in early_fail/9e7e7cc7::fail2[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/early_fail.rs:17:5: 17:29: error: in early_fail/9ed3a747::fail2[0] [Crux] MIR assertion at test/symb_eval/crux/early_fail.rs:17:5: [Crux] x == 0 diff --git a/crux-mir/test/symb_eval/crux/fail_return.good b/crux-mir/test/symb_eval/crux/fail_return.good index 08690aff0..bdeb641c6 100644 --- a/crux-mir/test/symb_eval/crux/fail_return.good +++ b/crux-mir/test/symb_eval/crux/fail_return.good @@ -1,23 +1,23 @@ -test fail_return/b8a81eca::fail1[0]: returned Symbolic BV, FAILED -test fail_return/b8a81eca::fail2[0]: returned 123, FAILED +test fail_return/29cbbbb6::fail1[0]: returned Symbolic BV, FAILED +test fail_return/29cbbbb6::fail2[0]: returned 123, FAILED failures: ----- fail_return/b8a81eca::fail1[0] counterexamples ---- +---- fail_return/29cbbbb6::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/crux/fail_return.rs:8:22: 8:27: error: in fail_return/b8a81eca::fail1[0] +[Crux] test/symb_eval/crux/fail_return.rs:8:22: 8:27: error: in fail_return/29cbbbb6::fail1[0] [Crux] attempt to compute `move _4 + const 1_u8`, which would overflow [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/fail_return.rs:8:5: 8:32: error: in fail_return/b8a81eca::fail1[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/fail_return.rs:8:5: 8:32: error: in fail_return/29cbbbb6::fail1[0] [Crux] MIR assertion at test/symb_eval/crux/fail_return.rs:8:5: [Crux] x + 1 > x ----- fail_return/b8a81eca::fail2[0] counterexamples ---- +---- fail_return/29cbbbb6::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/crux/fail_return.rs:15:22: 15:27: error: in fail_return/b8a81eca::fail2[0] +[Crux] test/symb_eval/crux/fail_return.rs:15:22: 15:27: error: in fail_return/29cbbbb6::fail2[0] [Crux] attempt to compute `move _5 + const 1_u8`, which would overflow [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/fail_return.rs:15:5: 15:32: error: in fail_return/b8a81eca::fail2[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/fail_return.rs:15:5: 15:32: error: in fail_return/29cbbbb6::fail2[0] [Crux] MIR assertion at test/symb_eval/crux/fail_return.rs:15:5: [Crux] x + 1 > x diff --git a/crux-mir/test/symb_eval/crux/mixed_fail.good b/crux-mir/test/symb_eval/crux/mixed_fail.good index 33e7ec2b4..03c53fd4a 100644 --- a/crux-mir/test/symb_eval/crux/mixed_fail.good +++ b/crux-mir/test/symb_eval/crux/mixed_fail.good @@ -1,25 +1,25 @@ -test mixed_fail/b1edd9fb::fail1[0]: FAILED -test mixed_fail/b1edd9fb::fail2[0]: FAILED -test mixed_fail/b1edd9fb::pass1[0]: ok -test mixed_fail/b1edd9fb::pass2[0]: ok +test mixed_fail/8469a3ad::fail1[0]: FAILED +test mixed_fail/8469a3ad::fail2[0]: FAILED +test mixed_fail/8469a3ad::pass1[0]: ok +test mixed_fail/8469a3ad::pass2[0]: ok failures: ----- mixed_fail/b1edd9fb::fail1[0] counterexamples ---- +---- mixed_fail/8469a3ad::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/crux/mixed_fail.rs:8:22: 8:27: error: in mixed_fail/b1edd9fb::fail1[0] +[Crux] test/symb_eval/crux/mixed_fail.rs:8:22: 8:27: error: in mixed_fail/8469a3ad::fail1[0] [Crux] attempt to compute `move _5 + const 1_u8`, which would overflow [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/mixed_fail.rs:8:5: 8:32: error: in mixed_fail/b1edd9fb::fail1[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/mixed_fail.rs:8:5: 8:32: error: in mixed_fail/8469a3ad::fail1[0] [Crux] MIR assertion at test/symb_eval/crux/mixed_fail.rs:8:5: [Crux] x + 1 > x ----- mixed_fail/b1edd9fb::fail2[0] counterexamples ---- +---- mixed_fail/8469a3ad::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/crux/mixed_fail.rs:14:22: 14:27: error: in mixed_fail/b1edd9fb::fail2[0] +[Crux] test/symb_eval/crux/mixed_fail.rs:14:22: 14:27: error: in mixed_fail/8469a3ad::fail2[0] [Crux] attempt to compute `move _5 + const 2_u8`, which would overflow [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/mixed_fail.rs:14:5: 14:32: error: in mixed_fail/b1edd9fb::fail2[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/mixed_fail.rs:14:5: 14:32: error: in mixed_fail/8469a3ad::fail2[0] [Crux] MIR assertion at test/symb_eval/crux/mixed_fail.rs:14:5: [Crux] x + 2 > x diff --git a/crux-mir/test/symb_eval/crux/multi.good b/crux-mir/test/symb_eval/crux/multi.good index 22733dbdd..14e9c957a 100644 --- a/crux-mir/test/symb_eval/crux/multi.good +++ b/crux-mir/test/symb_eval/crux/multi.good @@ -1,26 +1,26 @@ -test multi/e2c1a28d::fail1[0]: FAILED -test multi/e2c1a28d::fail2[0]: FAILED -test multi/e2c1a28d::fail3[0]: FAILED +test multi/0fbcaa4a::fail1[0]: FAILED +test multi/0fbcaa4a::fail2[0]: FAILED +test multi/0fbcaa4a::fail3[0]: FAILED failures: ----- multi/e2c1a28d::fail1[0] counterexamples ---- +---- multi/0fbcaa4a::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/crux/multi.rs:8:22: 8:27: error: in multi/e2c1a28d::fail1[0] +[Crux] test/symb_eval/crux/multi.rs:8:22: 8:27: error: in multi/0fbcaa4a::fail1[0] [Crux] attempt to compute `move _5 + const 1_u8`, which would overflow [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/multi.rs:8:5: 8:32: error: in multi/e2c1a28d::fail1[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/multi.rs:8:5: 8:32: error: in multi/0fbcaa4a::fail1[0] [Crux] MIR assertion at test/symb_eval/crux/multi.rs:8:5: [Crux] x + 1 > x ----- multi/e2c1a28d::fail2[0] counterexamples ---- +---- multi/0fbcaa4a::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] internal: error: in multi/e2c1a28d::fail2[0] -[Crux] panicking::panic_fmt, called from multi/e2c1a28d::fail2[0] +[Crux] internal: error: in multi/0fbcaa4a::fail2[0] +[Crux] panicking::panic_fmt, called from multi/0fbcaa4a::fail2[0] ----- multi/e2c1a28d::fail3[0] counterexamples ---- +---- multi/0fbcaa4a::fail3[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/multi.rs:20:5: 20:29: error: in multi/e2c1a28d::assert_zero[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/multi.rs:20:5: 20:29: error: in multi/0fbcaa4a::assert_zero[0] [Crux] MIR assertion at test/symb_eval/crux/multi.rs:20:5: [Crux] x == 0 diff --git a/crux-mir/test/symb_eval/crypto/bytes.good b/crux-mir/test/symb_eval/crypto/bytes.good index 788f51b23..3f774cfbe 100644 --- a/crux-mir/test/symb_eval/crypto/bytes.good +++ b/crux-mir/test/symb_eval/crypto/bytes.good @@ -1,26 +1,26 @@ -test bytes/55748dbe::f[0]: FAILED +test bytes/3f231946::f[0]: FAILED failures: ----- bytes/55748dbe::f[0] counterexamples ---- +---- bytes/3f231946::f[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/55748dbe::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/3f231946::f[0] [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: [Crux] a[i] == b[i] [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/55748dbe::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/3f231946::f[0] [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: [Crux] a[i] == b[i] [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/55748dbe::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/3f231946::f[0] [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: [Crux] a[i] == b[i] [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/55748dbe::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/3f231946::f[0] [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: [Crux] a[i] == b[i] [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/55748dbe::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/3f231946::f[0] [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: [Crux] a[i] == b[i] diff --git a/crux-mir/test/symb_eval/crypto/bytes2.good b/crux-mir/test/symb_eval/crypto/bytes2.good index d3d14d16e..57f37573d 100644 --- a/crux-mir/test/symb_eval/crypto/bytes2.good +++ b/crux-mir/test/symb_eval/crypto/bytes2.good @@ -1,3 +1,3 @@ -test bytes2/595e306d::f[0]: ok +test bytes2/930baf50::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/crypto/double.good b/crux-mir/test/symb_eval/crypto/double.good index 9ba9b704b..48b3b135d 100644 --- a/crux-mir/test/symb_eval/crypto/double.good +++ b/crux-mir/test/symb_eval/crypto/double.good @@ -1,3 +1,3 @@ -test double/1b95a34b::f[0]: ok +test double/b109b0aa::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/crypto/ffs.good b/crux-mir/test/symb_eval/crypto/ffs.good index b39d58de3..cc6b7fe63 100644 --- a/crux-mir/test/symb_eval/crypto/ffs.good +++ b/crux-mir/test/symb_eval/crypto/ffs.good @@ -1,3 +1,3 @@ -test ffs/28606b89::f[0]: ok +test ffs/8450a4c1::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/enum/mux.good b/crux-mir/test/symb_eval/enum/mux.good index 20f7f6db8..697132333 100644 --- a/crux-mir/test/symb_eval/enum/mux.good +++ b/crux-mir/test/symb_eval/enum/mux.good @@ -1,3 +1,3 @@ -test mux/2b41b8c9::test[0]: ok +test mux/72a0da7e::test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/fnptr/mux.good b/crux-mir/test/symb_eval/fnptr/mux.good index f57657e49..5a28542c1 100644 --- a/crux-mir/test/symb_eval/fnptr/mux.good +++ b/crux-mir/test/symb_eval/fnptr/mux.good @@ -1,3 +1,3 @@ -test mux/a669714b::crux_test[0]: returned 2, ok +test mux/f7f94a9c::crux_test[0]: returned 2, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/io/vec_cursor_read.good b/crux-mir/test/symb_eval/io/vec_cursor_read.good index 110506132..0e032b8f2 100644 --- a/crux-mir/test/symb_eval/io/vec_cursor_read.good +++ b/crux-mir/test/symb_eval/io/vec_cursor_read.good @@ -1,3 +1,3 @@ -test vec_cursor_read/3a1fbbbh::f[0]: ok +test vec_cursor_read/3c45b63f::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/io/vec_write.good b/crux-mir/test/symb_eval/io/vec_write.good index 7e1bf9697..c735c5aa4 100644 --- a/crux-mir/test/symb_eval/io/vec_write.good +++ b/crux-mir/test/symb_eval/io/vec_write.good @@ -1,3 +1,3 @@ -test vec_write/e955fa73::f[0]: ok +test vec_write/6974c82c::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/mux/array.good b/crux-mir/test/symb_eval/mux/array.good index bd143e4e0..512e9e2a6 100644 --- a/crux-mir/test/symb_eval/mux/array.good +++ b/crux-mir/test/symb_eval/mux/array.good @@ -1,3 +1,3 @@ -test array/c70863b9::crux_test[0]: returned Symbolic BV, ok +test array/e80be074::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/mux/array_mut.good b/crux-mir/test/symb_eval/mux/array_mut.good index d719ba6ca..8c45ab5e9 100644 --- a/crux-mir/test/symb_eval/mux/array_mut.good +++ b/crux-mir/test/symb_eval/mux/array_mut.good @@ -1,3 +1,3 @@ -test array_mut/faf5a48b::crux_test[0]: returned Symbolic BV, ok +test array_mut/1330af4a::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/num/checked_add.good b/crux-mir/test/symb_eval/num/checked_add.good index 6f515fc09..3d1804215 100644 --- a/crux-mir/test/symb_eval/num/checked_add.good +++ b/crux-mir/test/symb_eval/num/checked_add.good @@ -1,10 +1,10 @@ -test checked_add/948ef81f::crux_test[0]: returned 44, FAILED +test checked_add/65d1ac2f::crux_test[0]: returned 44, FAILED failures: ----- checked_add/948ef81f::crux_test[0] counterexamples ---- +---- checked_add/65d1ac2f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_add.rs:6:5: 6:12: error: in checked_add/948ef81f::crux_test[0] +[Crux] test/symb_eval/num/checked_add.rs:6:5: 6:12: error: in checked_add/65d1ac2f::crux_test[0] [Crux] attempt to compute `const 100_u8 + move _2`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/num/checked_div.good b/crux-mir/test/symb_eval/num/checked_div.good index deabab3e6..e12a6282d 100644 --- a/crux-mir/test/symb_eval/num/checked_div.good +++ b/crux-mir/test/symb_eval/num/checked_div.good @@ -1,16 +1,16 @@ -test checked_div/85dae97d::crux_test[0]: returned 65, FAILED +test checked_div/03613482::crux_test[0]: returned 65, FAILED failures: ----- checked_div/85dae97d::crux_test[0] counterexamples ---- +---- checked_div/03613482::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/85dae97d::div_signed[0] +[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/03613482::div_signed[0] [Crux] attempt to compute `_3 / _4`, which would overflow [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/85dae97d::div_signed[0] +[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/03613482::div_signed[0] [Crux] attempt to divide `_3` by zero [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/85dae97d::div_unsigned[0] +[Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/03613482::div_unsigned[0] [Crux] attempt to divide `_3` by zero [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/num/checked_mul.good b/crux-mir/test/symb_eval/num/checked_mul.good index ce3335b79..7940a75a0 100644 --- a/crux-mir/test/symb_eval/num/checked_mul.good +++ b/crux-mir/test/symb_eval/num/checked_mul.good @@ -1,10 +1,10 @@ -test checked_mul/0eadbb9f::crux_test[0]: returned 44, FAILED +test checked_mul/849ca86a::crux_test[0]: returned 44, FAILED failures: ----- checked_mul/0eadbb9f::crux_test[0] counterexamples ---- +---- checked_mul/849ca86a::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_mul.rs:6:5: 6:12: error: in checked_mul/0eadbb9f::crux_test[0] +[Crux] test/symb_eval/num/checked_mul.rs:6:5: 6:12: error: in checked_mul/849ca86a::crux_test[0] [Crux] attempt to compute `const 100_u8 * move _2`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/num/checked_mul_signed.good b/crux-mir/test/symb_eval/num/checked_mul_signed.good index ecea74575..579039c26 100644 --- a/crux-mir/test/symb_eval/num/checked_mul_signed.good +++ b/crux-mir/test/symb_eval/num/checked_mul_signed.good @@ -1,10 +1,10 @@ -test checked_mul_signed/6750aee6::crux_test[0]: returned 44, FAILED +test checked_mul_signed/0175bcf1::crux_test[0]: returned 44, FAILED failures: ----- checked_mul_signed/6750aee6::crux_test[0] counterexamples ---- +---- checked_mul_signed/0175bcf1::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_mul_signed.rs:6:5: 6:13: error: in checked_mul_signed/6750aee6::crux_test[0] +[Crux] test/symb_eval/num/checked_mul_signed.rs:6:5: 6:13: error: in checked_mul_signed/0175bcf1::crux_test[0] [Crux] attempt to compute `const -100_i8 * move _2`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/overrides/bad_symb1.good b/crux-mir/test/symb_eval/overrides/bad_symb1.good index a80153382..0101f31bb 100644 --- a/crux-mir/test/symb_eval/overrides/bad_symb1.good +++ b/crux-mir/test/symb_eval/overrides/bad_symb1.good @@ -1,10 +1,10 @@ -test bad_symb1/3a80a96c::crux_test[0]: FAILED +test bad_symb1/9b84185b::crux_test[0]: FAILED failures: ----- bad_symb1/3a80a96c::crux_test[0] counterexamples ---- +---- bad_symb1/9b84185b::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] lib/crucible/symbolic.rs:24:64: 24:68 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/4ab3f93d::symbolic[0]::{impl#1}[0]::symbolic[0] +[Crux] lib/crucible/symbolic.rs:24:64: 24:68 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/c130157c::symbolic[0]::{impl#1}[0]::symbolic[0] [Crux] symbolic variable name must be a concrete string [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/overrides/bad_symb2.good b/crux-mir/test/symb_eval/overrides/bad_symb2.good index 4808605e0..2d4adf3da 100644 --- a/crux-mir/test/symb_eval/overrides/bad_symb2.good +++ b/crux-mir/test/symb_eval/overrides/bad_symb2.good @@ -1,10 +1,10 @@ -test bad_symb2/e656e2a4::crux_test[0]: FAILED +test bad_symb2/67e9f90f::crux_test[0]: FAILED failures: ----- bad_symb2/e656e2a4::crux_test[0] counterexamples ---- +---- bad_symb2/67e9f90f::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] lib/crucible/symbolic.rs:24:64: 24:68 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/4ab3f93d::symbolic[0]::{impl#1}[0]::symbolic[0] +[Crux] lib/crucible/symbolic.rs:24:64: 24:68 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/c130157c::symbolic[0]::{impl#1}[0]::symbolic[0] [Crux] invalid symbolic variable name "\NUL:., /": Identifier must start with a letter. [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/overrides/override1.good b/crux-mir/test/symb_eval/overrides/override1.good index 3f054d543..7c4a56ccd 100644 --- a/crux-mir/test/symb_eval/overrides/override1.good +++ b/crux-mir/test/symb_eval/overrides/override1.good @@ -1,4 +1,4 @@ -test override1/3ff54103::f[0]: Hello, I'm an override +test override1/5f575d95::f[0]: Hello, I'm an override returned 2, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/overrides/override2.good b/crux-mir/test/symb_eval/overrides/override2.good index 2fcf8dd4a..fe66852a0 100644 --- a/crux-mir/test/symb_eval/overrides/override2.good +++ b/crux-mir/test/symb_eval/overrides/override2.good @@ -1,10 +1,10 @@ -test override2/6aa6cea9::f[0]: FAILED +test override2/980aac96::f[0]: FAILED failures: ----- override2/6aa6cea9::f[0] counterexamples ---- +---- override2/980aac96::f[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/overrides/override2.rs:9:5: 9:49: error: in override2/6aa6cea9::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/overrides/override2.rs:9:5: 9:49: error: in override2/980aac96::f[0] [Crux] MIR assertion at test/symb_eval/overrides/override2.rs:9:5: [Crux] foo.wrapping_add(1) == foo diff --git a/crux-mir/test/symb_eval/overrides/override3.good b/crux-mir/test/symb_eval/overrides/override3.good index 7b9cadfd8..938c23e93 100644 --- a/crux-mir/test/symb_eval/overrides/override3.good +++ b/crux-mir/test/symb_eval/overrides/override3.good @@ -1,3 +1,3 @@ -test override3/a07e766d::f[0]: ok +test override3/19279091::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/overrides/override4.good b/crux-mir/test/symb_eval/overrides/override4.good index 261847069..d8c0a1a88 100644 --- a/crux-mir/test/symb_eval/overrides/override4.good +++ b/crux-mir/test/symb_eval/overrides/override4.good @@ -1,3 +1,3 @@ -test override4/4e5d7bb4::f[0]: ok +test override4/083fe82b::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/overrides/override5.good b/crux-mir/test/symb_eval/overrides/override5.good index 9281d7301..3d18a0506 100644 --- a/crux-mir/test/symb_eval/overrides/override5.good +++ b/crux-mir/test/symb_eval/overrides/override5.good @@ -1,10 +1,10 @@ -test override5/48be6bb4::f[0]: FAILED +test override5/19f870ba::f[0]: FAILED failures: ----- override5/48be6bb4::f[0] counterexamples ---- +---- override5/19f870ba::f[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/overrides/override5.rs:10:5: 10:47: error: in override5/48be6bb4::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/overrides/override5.rs:10:5: 10:47: error: in override5/19f870ba::f[0] [Crux] MIR assertion at test/symb_eval/overrides/override5.rs:10:5: [Crux] foo.wrapping_add(1) != 0 diff --git a/crux-mir/test/symb_eval/overrides/override_rust.good b/crux-mir/test/symb_eval/overrides/override_rust.good index 35cfb1f01..813b304a8 100644 --- a/crux-mir/test/symb_eval/overrides/override_rust.good +++ b/crux-mir/test/symb_eval/overrides/override_rust.good @@ -1,3 +1,3 @@ -test override_rust/cfc3a2ca::crux_test[0]: ok +test override_rust/5ae83a2f::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/refs/mux_init_imm.good b/crux-mir/test/symb_eval/refs/mux_init_imm.good index bc2db2185..a86449a3e 100644 --- a/crux-mir/test/symb_eval/refs/mux_init_imm.good +++ b/crux-mir/test/symb_eval/refs/mux_init_imm.good @@ -1,3 +1,3 @@ -test mux_init_imm/a53a5a3d::crux_test[0]: returned Symbolic BV, ok +test mux_init_imm/9274f69e::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/refs/mux_init_mut.good b/crux-mir/test/symb_eval/refs/mux_init_mut.good index 7bb9f588e..aa8bb547d 100644 --- a/crux-mir/test/symb_eval/refs/mux_init_mut.good +++ b/crux-mir/test/symb_eval/refs/mux_init_mut.good @@ -1,3 +1,3 @@ -test mux_init_mut/7e854109::crux_test[0]: returned Symbolic BV, ok +test mux_init_mut/13769ed2::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/scalar/test1.good b/crux-mir/test/symb_eval/scalar/test1.good index 1586648de..08fa2b6e2 100644 --- a/crux-mir/test/symb_eval/scalar/test1.good +++ b/crux-mir/test/symb_eval/scalar/test1.good @@ -1,3 +1,3 @@ -test test1/5496ba4f::f[0]: ok +test test1/e0d3b1cd::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/sym_bytes/construct.good b/crux-mir/test/symb_eval/sym_bytes/construct.good index 2d29e549f..647eec8c3 100644 --- a/crux-mir/test/symb_eval/sym_bytes/construct.good +++ b/crux-mir/test/symb_eval/sym_bytes/construct.good @@ -1,10 +1,10 @@ -test construct/55406696::crux_test[0]: returned Symbolic BV, FAILED +test construct/4a537371::crux_test[0]: returned Symbolic BV, FAILED failures: ----- construct/55406696::crux_test[0] counterexamples ---- +---- construct/4a537371::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/sym_bytes/construct.rs:13:5: 13:35: error: in construct/55406696::crux_test[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/sym_bytes/construct.rs:13:5: 13:35: error: in construct/4a537371::crux_test[0] [Crux] MIR assertion at test/symb_eval/sym_bytes/construct.rs:13:5: [Crux] sym2[0] == 0 diff --git a/crux-mir/test/symb_eval/sym_bytes/deserialize.good b/crux-mir/test/symb_eval/sym_bytes/deserialize.good index 8c63325d3..e1f8bbdc7 100644 --- a/crux-mir/test/symb_eval/sym_bytes/deserialize.good +++ b/crux-mir/test/symb_eval/sym_bytes/deserialize.good @@ -1,10 +1,10 @@ -test deserialize/2a8b0d13::crux_test[0]: returned Symbolic BV, FAILED +test deserialize/a0a785f7::crux_test[0]: returned Symbolic BV, FAILED failures: ----- deserialize/2a8b0d13::crux_test[0] counterexamples ---- +---- deserialize/a0a785f7::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/sym_bytes/deserialize.rs:19:21: 19:44: error: in deserialize/2a8b0d13::deserialize[0] +[Crux] test/symb_eval/sym_bytes/deserialize.rs:19:21: 19:44: error: in deserialize/a0a785f7::deserialize[0] [Crux] attempt to compute `move _61 + move _62`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/vec/clone.good b/crux-mir/test/symb_eval/vec/clone.good index e0bfa9d43..450fcf559 100644 --- a/crux-mir/test/symb_eval/vec/clone.good +++ b/crux-mir/test/symb_eval/vec/clone.good @@ -1,3 +1,3 @@ -test clone/e141ef81::f[0]: ok +test clone/a6d38a22::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vec/into_iter.good b/crux-mir/test/symb_eval/vec/into_iter.good index 9eea163b4..d6fb1920a 100644 --- a/crux-mir/test/symb_eval/vec/into_iter.good +++ b/crux-mir/test/symb_eval/vec/into_iter.good @@ -1,3 +1,3 @@ -test into_iter/82ce8ccb::f[0]: ok +test into_iter/17035813::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vec/macro.good b/crux-mir/test/symb_eval/vec/macro.good index 6ba0c94c2..b93da9bd9 100644 --- a/crux-mir/test/symb_eval/vec/macro.good +++ b/crux-mir/test/symb_eval/vec/macro.good @@ -1,3 +1,3 @@ -test macro/a1425499::f[0]: ok +test macro/36f16f2e::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vec/sort_by_key.good b/crux-mir/test/symb_eval/vec/sort_by_key.good index bbbe037aa..5c0bc10a3 100644 --- a/crux-mir/test/symb_eval/vec/sort_by_key.good +++ b/crux-mir/test/symb_eval/vec/sort_by_key.good @@ -1,3 +1,3 @@ -test sort_by_key/c5f23e33::f[0]: ok +test sort_by_key/31d3f947::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/as_mut_slice.good b/crux-mir/test/symb_eval/vector/as_mut_slice.good index d723038f5..4bd1380de 100644 --- a/crux-mir/test/symb_eval/vector/as_mut_slice.good +++ b/crux-mir/test/symb_eval/vector/as_mut_slice.good @@ -1,3 +1,3 @@ -test as_mut_slice/caa8fcae::f[0]: ok +test as_mut_slice/65a3c7cd::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/as_slice.good b/crux-mir/test/symb_eval/vector/as_slice.good index 9ac9b6c81..659ab2a7f 100644 --- a/crux-mir/test/symb_eval/vector/as_slice.good +++ b/crux-mir/test/symb_eval/vector/as_slice.good @@ -1,3 +1,3 @@ -test as_slice/71764b65::f[0]: ok +test as_slice/0cc3eb84::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/concat.good b/crux-mir/test/symb_eval/vector/concat.good index 237177715..7510ee189 100644 --- a/crux-mir/test/symb_eval/vector/concat.good +++ b/crux-mir/test/symb_eval/vector/concat.good @@ -1,3 +1,3 @@ -test concat/a65f2287::f[0]: ok +test concat/e9e1d873::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/copy_from_slice.good b/crux-mir/test/symb_eval/vector/copy_from_slice.good index 89ed1ebef..b3e56af73 100644 --- a/crux-mir/test/symb_eval/vector/copy_from_slice.good +++ b/crux-mir/test/symb_eval/vector/copy_from_slice.good @@ -1,3 +1,3 @@ -test copy_from_slice/9cf6b878::f[0]: ok +test copy_from_slice/d3fda69b::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/mut.good b/crux-mir/test/symb_eval/vector/mut.good index 5cffb915e..c35d63d85 100644 --- a/crux-mir/test/symb_eval/vector/mut.good +++ b/crux-mir/test/symb_eval/vector/mut.good @@ -1,3 +1,3 @@ -test mut/8d39d367::f[0]: ok +test mut/8471053f::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/new.good b/crux-mir/test/symb_eval/vector/new.good index 1fe5985b8..6b0633c92 100644 --- a/crux-mir/test/symb_eval/vector/new.good +++ b/crux-mir/test/symb_eval/vector/new.good @@ -1,3 +1,3 @@ -test new/a9927192::f[0]: ok +test new/833ad47b::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/push.good b/crux-mir/test/symb_eval/vector/push.good index 75e530d8b..7a83b24f2 100644 --- a/crux-mir/test/symb_eval/vector/push.good +++ b/crux-mir/test/symb_eval/vector/push.good @@ -1,3 +1,3 @@ -test push/cb01e03b::f[0]: ok +test push/415cb1ec::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/replicate.good b/crux-mir/test/symb_eval/vector/replicate.good index a598e00fc..17848fb32 100644 --- a/crux-mir/test/symb_eval/vector/replicate.good +++ b/crux-mir/test/symb_eval/vector/replicate.good @@ -1,3 +1,3 @@ -test replicate/1d9f8903::f[0]: ok +test replicate/4ba185ef::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/split_at.good b/crux-mir/test/symb_eval/vector/split_at.good index ecedbd0f9..c382274ca 100644 --- a/crux-mir/test/symb_eval/vector/split_at.good +++ b/crux-mir/test/symb_eval/vector/split_at.good @@ -1,3 +1,3 @@ -test split_at/9315f323::f[0]: ok +test split_at/23248c64::f[0]: ok [Crux] Overall status: Valid. From 74c7e1b25d94e03eb9de51d0542ddc701db2e8a2 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 1 Jun 2023 16:34:19 -0400 Subject: [PATCH 080/114] Update test_output.log --- crux-mir/test_output.log | 1701 +++++++++++++++++--------------------- 1 file changed, 748 insertions(+), 953 deletions(-) diff --git a/crux-mir/test_output.log b/crux-mir/test_output.log index ea31085b4..3baf84ffe 100644 --- a/crux-mir/test_output.log +++ b/crux-mir/test_output.log @@ -2,340 +2,314 @@ crux-mir crux concrete refs - promoted_imm: OK (1.94s) + promoted_imm: OK (1.86s) Compiling and running oracle program (0.14s) - Oracle output: 0 (1.80s) + Oracle output: 0 (1.72s) Crux output: 0 - mut_raw: OK (2.01s) + mut_raw: OK (1.86s) Compiling and running oracle program (0.14s) - Oracle output: 123 (1.86s) + Oracle output: 123 (1.72s) Crux output: 123 - mut_nested: OK (2.00s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.84s) + mut_nested: OK (1.92s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.78s) Crux output: () - fn_ptr_mut: OK (2.01s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.85s) + fn_ptr_mut: OK (1.97s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.83s) Crux output: 1 - never: OK (2.04s) + never: OK (2.01s) Compiling and running oracle program (0.15s) - Oracle output: 1 (1.89s) + Oracle output: 1 (1.85s) Crux output: 1 - mut_ref: OK (2.11s) - Compiling and running oracle program (0.18s) - Oracle output: 1 (1.93s) + mut_ref: OK (2.10s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (1.94s) Crux output: 1 - temp: OK (2.05s) + temp: OK (2.15s) Compiling and running oracle program (0.15s) - Oracle output: 1 (1.90s) + Oracle output: 1 (2.00s) Crux output: 1 - mut_arg: OK (1.99s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.84s) + mut_arg: OK (2.22s) + Compiling and running oracle program (0.18s) + Oracle output: 1 (2.04s) Crux output: 1 - fn_ptr: OK (2.06s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.91s) + fn_ptr: OK (2.14s) + Compiling and running oracle program (0.20s) + Oracle output: 1 (1.95s) Crux output: 1 - promoted_mut: OK (2.09s) - Compiling and running oracle program (0.17s) + promoted_mut: OK (2.08s) + Compiling and running oracle program (0.16s) Oracle output: 0 (1.92s) Crux output: 0 - never_mut: OK (2.09s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.94s) + never_mut: OK (2.00s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (1.83s) Crux output: 1 - mut_tuple_field: OK (1.98s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.84s) + mut_tuple_field: OK (2.06s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.90s) Crux output: () - imm_raw: OK (2.01s) + imm_raw: OK (2.03s) Compiling and running oracle program (0.15s) - Oracle output: 123 (1.86s) + Oracle output: 123 (1.88s) Crux output: 123 - static_mut: OK (2.05s) + static_mut: OK (2.23s) Compiling and running oracle program (0.17s) - Oracle output: 2 (1.89s) + Oracle output: 2 (2.06s) Crux output: 2 - imm_ref: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: 123 (1.76s) + imm_ref: OK (1.98s) + Compiling and running oracle program (0.15s) + Oracle output: 123 (1.83s) Crux output: 123 - imm_arg: OK (1.89s) + imm_arg: OK (1.97s) Compiling and running oracle program (0.15s) - Oracle output: 1 (1.74s) + Oracle output: 1 (1.83s) Crux output: 1 hash_map - insert_multi: OK (2.37s) + insert_multi: OK (2.42s) Compiling and running oracle program (0.20s) - Oracle output: 100 (2.17s) + Oracle output: 100 (2.22s) Crux output: 100 - insert_remove: OK (2.82s) - Compiling and running oracle program (0.18s) - Oracle output: 12 (2.64s) - Crux output: 12 - insert_iter: OK (2.46s) + insert_remove: OK (2.95s) Compiling and running oracle program (0.20s) - Oracle output: [11, 12] (2.26s) - Crux output: [11, 12] - insert_get: OK (2.38s) + Oracle output: 12 (2.75s) + Crux output: 12 + insert_iter: OK (2.28s) Compiling and running oracle program (0.19s) - Oracle output: (11, 12) (2.20s) + Oracle output: [11, 12] (2.09s) + Crux output: [11, 12] + insert_get: OK (2.26s) + Compiling and running oracle program (0.18s) + Oracle output: (11, 12) (2.08s) Crux output: (11, 12) traits - params: OK (2.03s) - Compiling and running oracle program (0.16s) - Oracle output: 25 (1.88s) + params: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: 25 (1.74s) Crux output: 25 - generic3: OK (2.01s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.85s) - Crux output: () - static_three: OK (2.05s) + generic3: OK (1.94s) Compiling and running oracle program (0.15s) - Oracle output: 2 (1.90s) + Oracle output: () (1.79s) + Crux output: () + static_three: OK (2.04s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (1.88s) Crux output: 2 - generic1: OK (1.93s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.78s) + generic1: OK (2.01s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.86s) Crux output: () - subtrait: OK (1.92s) - Compiling and running oracle program (0.14s) - Oracle output: 73 (1.77s) + subtrait: OK (1.97s) + Compiling and running oracle program (0.15s) + Oracle output: 73 (1.82s) Crux output: 73 - gen_trait: OK (1.86s) + gen_trait: OK (1.89s) Compiling and running oracle program (0.15s) - Oracle output: -69 (1.72s) + Oracle output: -69 (1.74s) Crux output: -69 - dynamic_branch: OK (1.88s) + dynamic_branch: OK (2.01s) Compiling and running oracle program (0.14s) - Oracle output: 1 (1.74s) + Oracle output: 1 (1.86s) Crux output: 1 - static_two: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.72s) + static_two: OK (2.02s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (1.86s) Crux output: 1 - tyfam5: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.73s) - Crux output: () - generic2: OK (1.89s) + tyfam5: OK (2.02s) Compiling and running oracle program (0.15s) - Oracle output: () (1.75s) + Oracle output: () (1.87s) Crux output: () - assoc3: OK (1.92s) + generic2: OK (1.97s) Compiling and running oracle program (0.15s) - Oracle output: () (1.77s) + Oracle output: () (1.82s) Crux output: () - tyfam4: OK (1.95s) - Compiling and running oracle program (0.15s) - Oracle output: 0 (1.81s) + assoc3: OK (1.94s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.78s) + Crux output: () + tyfam4: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.76s) Crux output: 0 - static_self: OK (1.91s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.76s) + static_self: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (1.72s) Crux output: 42 - conv: OK (1.89s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.75s) + conv: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.71s) Crux output: 1 - dynamic_poly: OK (1.88s) + dynamic_poly: OK (1.87s) Compiling and running oracle program (0.15s) - Oracle output: 3 (1.74s) + Oracle output: 3 (1.73s) Crux output: 3 - dynamic_simple: OK (1.90s) + dynamic_simple: OK (1.86s) Compiling and running oracle program (0.14s) - Oracle output: 1 (1.76s) + Oracle output: 1 (1.72s) Crux output: 1 - gen_trait_poly: OK (1.93s) - Compiling and running oracle program (0.15s) - Oracle output: 12 (1.78s) + gen_trait_poly: OK (1.97s) + Compiling and running oracle program (0.14s) + Oracle output: 12 (1.82s) Crux output: 12 - default: OK (1.88s) + default: OK (1.91s) Compiling and running oracle program (0.14s) - Oracle output: 12 (1.74s) + Oracle output: 12 (1.76s) Crux output: 12 - assoc1: OK (1.89s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.74s) + assoc1: OK (1.92s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.77s) Crux output: () - subtrait2: OK (1.91s) + subtrait2: OK (1.90s) Compiling and running oracle program (0.15s) - Oracle output: 64 (1.77s) + Oracle output: 64 (1.76s) Crux output: 64 - bounds1: OK (2.05s) - Compiling and running oracle program (0.18s) - Oracle output: () (1.87s) - Crux output: () - bounds3: OK (2.02s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.86s) + bounds1: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.74s) Crux output: () - static_eq: OK (1.98s) + bounds3: OK (1.97s) Compiling and running oracle program (0.15s) - Oracle output: true (1.82s) + Oracle output: () (1.82s) + Crux output: () + static_eq: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: true (1.75s) Crux output: true - tyfam3: OK (2.03s) - Compiling and running oracle program (0.16s) - Oracle output: 23 (1.88s) + tyfam3: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 23 (1.73s) Crux output: 23 - dict_med: OK (2.03s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.88s) + dict_med: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (1.73s) Crux output: 42 - basics1: OK (2.04s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.89s) + basics1: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.75s) Crux output: () - assoc2: OK (1.96s) + assoc2: OK (1.94s) Compiling and running oracle program (0.15s) - Oracle output: () (1.82s) + Oracle output: () (1.79s) Crux output: () - dict_polymem: OK (2.04s) - Compiling and running oracle program (0.15s) - Oracle output: 4 (1.89s) + dict_polymem: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 4 (1.74s) Crux output: 4 - bounds2: OK (2.02s) + bounds2: OK (1.90s) Compiling and running oracle program (0.15s) - Oracle output: () (1.87s) + Oracle output: () (1.75s) Crux output: () - intoiter: OK (2.03s) - Compiling and running oracle program (0.15s) - Oracle output: 0 (1.88s) + intoiter: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.73s) Crux output: 0 - static: OK (2.00s) - Compiling and running oracle program (0.16s) - Oracle output: 42 (1.84s) + static: OK (1.94s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (1.80s) Crux output: 42 - dynamic_two: OK (2.01s) + dynamic_two: OK (1.89s) Compiling and running oracle program (0.15s) - Oracle output: 1 (1.86s) + Oracle output: 1 (1.75s) Crux output: 1 - bounds4: OK (1.95s) + bounds4: OK (2.01s) Compiling and running oracle program (0.15s) - Oracle output: () (1.81s) + Oracle output: () (1.87s) Crux output: () - dict_poly: OK (2.01s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.86s) - Crux output: 1 - dict_simple: OK (2.10s) + dict_poly: OK (2.08s) Compiling and running oracle program (0.16s) - Oracle output: 32 (1.94s) - Crux output: 32 - tyfam: OK (2.01s) + Oracle output: 1 (1.92s) + Crux output: 1 + dict_simple: OK (1.98s) Compiling and running oracle program (0.15s) - Oracle output: 23 (1.86s) + Oracle output: 32 (1.82s) + Crux output: 32 + tyfam: OK (2.00s) + Compiling and running oracle program (0.16s) + Oracle output: 23 (1.84s) Crux output: 23 - bounds5: OK (2.04s) + bounds5: OK (2.02s) Compiling and running oracle program (0.16s) - Oracle output: () (1.88s) + Oracle output: () (1.86s) Crux output: () - tyfam2: OK (1.91s) + tyfam2: OK (1.95s) Compiling and running oracle program (0.15s) - Oracle output: 23 (1.76s) + Oracle output: 23 (1.80s) Crux output: 23 - dynamic_med: OK (1.93s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.78s) + dynamic_med: OK (1.99s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.84s) Crux output: 1 intTest - test0039: OK (1.94s) + test0039: OK (1.95s) Compiling and running oracle program (0.15s) - Oracle output: 7 (1.79s) + Oracle output: 7 (1.80s) Crux output: 7 test0038: OK (1.96s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.82s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.81s) Crux output: () tuple - clone_rec: OK (1.90s) + clone_rec: OK (1.88s) Compiling and running oracle program (0.15s) - Oracle output: () (1.75s) + Oracle output: () (1.73s) Crux output: () - clone: OK (1.89s) + clone: OK (1.88s) Compiling and running oracle program (0.15s) - Oracle output: () (1.75s) + Oracle output: () (1.73s) + Crux output: () + clone_from: OK (1.98s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.84s) + Crux output: () + clone_struct: OK (2.00s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.86s) Crux output: () - clone_from: FAIL (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.79s) - Crux output: - failures: - - ---- clone_from/8fca6705::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/clone.rs:136:17: 136:31: error: in core/1cdf94ce::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0] - [Crux] Translation error in core/1cdf94ce::clone[0]::Clone[0]::clone_from[0]::_insta1f0a334420ce0e0[0]: don't know how to generate CloneShim for unknown method core/1cdf94ce::clone[0]::Clone[0]::clone[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/clone_from/' to rerun this test only. - clone_struct: FAIL (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.79s) - Crux output: - failures: - - ---- clone_struct/8dcd765f::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/tuple/clone_struct.rs:8:13: 8:22: error: in clone_struct/8dcd765f::f[0] - [Crux] Translation error in clone_struct/8dcd765f::f[0]: don't know how to generate CloneShim for unknown method core/1cdf94ce::clone[0]::Clone[0]::clone[0] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/clone_struct/' to rerun this test only. iter - for: OK (1.92s) + for: OK (1.94s) Compiling and running oracle program (0.15s) - Oracle output: 47 (1.77s) + Oracle output: 47 (1.79s) Crux output: 47 - sum: OK (1.92s) + sum: OK (1.96s) Compiling and running oracle program (0.15s) - Oracle output: () (1.77s) + Oracle output: () (1.81s) Crux output: () filter_chain: OK (2.06s) Compiling and running oracle program (0.17s) Oracle output: 0 (1.89s) Crux output: 0 - from_fn: OK (1.96s) + from_fn: OK (1.91s) Compiling and running oracle program (0.15s) - Oracle output: () (1.81s) + Oracle output: () (1.76s) Crux output: () - peek: OK (2.06s) + peek: OK (1.98s) Compiling and running oracle program (0.15s) - Oracle output: 3 (1.91s) + Oracle output: 3 (1.83s) Crux output: 3 - zip: OK (2.01s) + zip: OK (1.99s) Compiling and running oracle program (0.16s) - Oracle output: () (1.85s) + Oracle output: () (1.83s) Crux output: () - loop: OK (1.88s) + loop: OK (1.93s) Compiling and running oracle program (0.14s) - Oracle output: 2 (1.73s) + Oracle output: 2 (1.79s) Crux output: 2 - cloned: OK (1.91s) - Compiling and running oracle program (0.15s) - Oracle output: 6 (1.77s) + cloned: OK (1.98s) + Compiling and running oracle program (0.16s) + Oracle output: 6 (1.82s) Crux output: 6 str - format_hex: FAIL (2.65s) - Compiling and running oracle program (0.15s) - Oracle output: true (2.50s) + format_hex: FAIL (2.70s) + Compiling and running oracle program (0.17s) + Oracle output: true (2.53s) Crux output: failures: - ---- format_hex/ff25ab16::crux_test[0] counterexamples ---- + ---- format_hex/45df7be1::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/1cdf94ce::fmt[0]::write[0] - [Crux] Translation error in core/1cdf94ce::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/1cdf94ce::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] + [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. @@ -343,16 +317,16 @@ crux-mir crux doesn't match oracle Use -p '/format_hex/' to rerun this test only. - format_int: FAIL (2.62s) - Compiling and running oracle program (0.15s) - Oracle output: true (2.47s) + format_int: FAIL (2.65s) + Compiling and running oracle program (0.16s) + Oracle output: true (2.49s) Crux output: failures: - ---- format_int/8280e1f2::crux_test[0] counterexamples ---- + ---- format_int/5be019fc::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/1cdf94ce::fmt[0]::write[0] - [Crux] Translation error in core/1cdf94ce::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/1cdf94ce::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] + [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. @@ -360,16 +334,16 @@ crux-mir crux doesn't match oracle Use -p '/format_int/' to rerun this test only. - format_struct: FAIL (2.69s) + format_struct: FAIL (2.80s) Compiling and running oracle program (0.16s) - Oracle output: true (2.53s) + Oracle output: true (2.64s) Crux output: failures: - ---- format_struct/e77dd717::crux_test[0] counterexamples ---- + ---- format_struct/6a3af932::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/1cdf94ce::fmt[0]::write[0] - [Crux] Translation error in core/1cdf94ce::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/1cdf94ce::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] + [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. @@ -377,16 +351,16 @@ crux-mir crux doesn't match oracle Use -p '/format_struct/' to rerun this test only. - format: FAIL (2.66s) + format: FAIL (2.64s) Compiling and running oracle program (0.15s) - Oracle output: true (2.51s) + Oracle output: true (2.49s) Crux output: failures: - ---- format/6d842288::crux_test[0] counterexamples ---- + ---- format/9cdf0325::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/1cdf94ce::fmt[0]::write[0] - [Crux] Translation error in core/1cdf94ce::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/1cdf94ce::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] + [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. @@ -395,15 +369,15 @@ crux-mir Use -p '$0=="crux-mir.crux concrete..str.format"' to rerun this test only. to_owned: FAIL (2.09s) - Compiling and running oracle program (0.15s) - Oracle output: true (1.94s) + Compiling and running oracle program (0.16s) + Oracle output: true (1.93s) Crux output: failures: - ---- to_owned/81422a79::crux_test[0] counterexamples ---- + ---- to_owned/ddf383ae::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/iter/traits/iterator.rs:2607:40: 2607:61: error: in core/1cdf94ce::iter[0]::traits[0]::iterator[0]::Iterator[0]::all[0]::_instf9a28a1fbcee32be[0] - [Crux] Translation error in core/1cdf94ce::iter[0]::traits[0]::iterator[0]::Iterator[0]::all[0]::_instf9a28a1fbcee32be[0]: ill-typed assignment of IntrinsicRepr MirReference [BVRepr 8] to IntrinsicRepr MirReference [AnyRepr] (TyRef (TyAdt core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0]::_adtdfc2b6f78a281997[0] core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0] (Substs [TyTuple [],TyTuple []])) Immut) LBase (Var {_varname = "_10", _varmut = Mut, _varty = TyRef (TyAdt core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0]::_adtdfc2b6f78a281997[0] core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0] (Substs [TyTuple [],TyTuple []])) Immut, _varIsZST = False}) + [Crux] lib/core/src/ops/control_flow.rs:84:30: 84:39 !lib/core/src/ops/control_flow.rs:84:30: 84:39: error: in core/092bc89a::ops[0]::control_flow[0]::{impl#11}[0]::eq[0]::_instdfc2b6f78a281997[0] + [Crux] bad unpack: Any as StructRepr [BVRepr 32, VariantRepr [StructRepr [UnitRepr], StructRepr [UnitRepr]]] [Crux] Overall status: Invalid. @@ -411,16 +385,16 @@ crux-mir crux doesn't match oracle Use -p '/to_owned/' to rerun this test only. - string_push: FAIL (2.58s) + string_push: FAIL (2.64s) Compiling and running oracle program (0.16s) - Oracle output: true (2.42s) + Oracle output: true (2.47s) Crux output: failures: - ---- string_push/f3da471e::crux_test[0] counterexamples ---- + ---- string_push/03bb3176::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/iter/traits/iterator.rs:2607:40: 2607:61: error: in core/1cdf94ce::iter[0]::traits[0]::iterator[0]::Iterator[0]::all[0]::_instf9a28a1fbcee32be[0] - [Crux] Translation error in core/1cdf94ce::iter[0]::traits[0]::iterator[0]::Iterator[0]::all[0]::_instf9a28a1fbcee32be[0]: ill-typed assignment of IntrinsicRepr MirReference [BVRepr 8] to IntrinsicRepr MirReference [AnyRepr] (TyRef (TyAdt core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0]::_adtdfc2b6f78a281997[0] core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0] (Substs [TyTuple [],TyTuple []])) Immut) LBase (Var {_varname = "_10", _varmut = Mut, _varty = TyRef (TyAdt core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0]::_adtdfc2b6f78a281997[0] core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0] (Substs [TyTuple [],TyTuple []])) Immut, _varIsZST = False}) + [Crux] lib/core/src/ops/control_flow.rs:84:30: 84:39 !lib/core/src/ops/control_flow.rs:84:30: 84:39: error: in core/092bc89a::ops[0]::control_flow[0]::{impl#11}[0]::eq[0]::_instdfc2b6f78a281997[0] + [Crux] bad unpack: Any as StructRepr [BVRepr 32, VariantRepr [StructRepr [UnitRepr], StructRepr [UnitRepr]]] [Crux] Overall status: Invalid. @@ -428,16 +402,16 @@ crux-mir crux doesn't match oracle Use -p '/string_push/' to rerun this test only. - format_array: FAIL (2.73s) + format_array: FAIL (2.74s) Compiling and running oracle program (0.16s) Oracle output: true (2.58s) Crux output: failures: - ---- format_array/5c87c800::crux_test[0] counterexamples ---- + ---- format_array/fd6e4004::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/1cdf94ce::fmt[0]::write[0] - [Crux] Translation error in core/1cdf94ce::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/1cdf94ce::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] + [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. @@ -446,153 +420,105 @@ crux-mir Use -p '/format_array/' to rerun this test only. sync - mutex: FAIL (3.18s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (3.03s) - TODO RGS evalRval ThreadLocalRef - CallStack (from HasCallStack): - error, called at src/Mir/Trans.hs:1034:33 in crucible-mir-0.1-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1197:11 in crucible-mir-0.1-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1642:68 in crucible-mir-0.1-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1651:28 in crucible-mir-0.1-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1691:10 in crucible-mir-0.1-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1737:24 in crucible-mir-0.1-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2171:30 in crucible-mir-0.1-inplace:Mir.Trans - transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate - translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language - - Use -p '$0=="crux-mir.crux concrete..sync.mutex"' to rerun this test only. - arc_cell: OK (2.10s) + mutex: OK (2.92s) Compiling and running oracle program (0.16s) - Oracle output: 4 (1.94s) + Oracle output: 1 (2.77s) + Crux output: 1 + arc_cell: OK (2.11s) + Compiling and running oracle program (0.16s) + Oracle output: 4 (1.95s) Crux output: 4 - arc: OK (2.08s) + arc: OK (2.03s) Compiling and running oracle program (0.15s) - Oracle output: 1 (1.94s) + Oracle output: 1 (1.88s) Crux output: 1 - atomic_add: OK (1.97s) + atomic_add: OK (1.98s) Compiling and running oracle program (0.15s) - Oracle output: 6 (1.82s) + Oracle output: 6 (1.83s) Crux output: 6 - rwlock: FAIL (3.25s) + rwlock: OK (2.96s) Compiling and running oracle program (0.15s) - Oracle output: 2 (3.10s) - TODO RGS evalRval ThreadLocalRef - CallStack (from HasCallStack): - error, called at src/Mir/Trans.hs:1034:33 in crucible-mir-0.1-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1197:11 in crucible-mir-0.1-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1642:68 in crucible-mir-0.1-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1651:28 in crucible-mir-0.1-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1691:10 in crucible-mir-0.1-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1737:24 in crucible-mir-0.1-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2171:30 in crucible-mir-0.1-inplace:Mir.Trans - transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate - translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language - - Use -p '$0=="crux-mir.crux concrete..sync.rwlock"' to rerun this test only. - atomic_cxchg: OK (2.05s) + Oracle output: 2 (2.81s) + Crux output: 2 + atomic_cxchg: OK (1.99s) Compiling and running oracle program (0.15s) - Oracle output: 6 (1.89s) + Oracle output: 6 (1.84s) Crux output: 6 - arc_clone: OK (2.11s) - Compiling and running oracle program (0.15s) + arc_clone: OK (2.12s) + Compiling and running oracle program (0.16s) Oracle output: 2 (1.96s) Crux output: 2 - atomic_swap: OK (1.93s) - Compiling and running oracle program (0.14s) - Oracle output: (2, 1) (1.78s) + atomic_swap: OK (1.94s) + Compiling and running oracle program (0.15s) + Oracle output: (2, 1) (1.79s) Crux output: (2, 1) - rwlock_multi: FAIL (3.31s) + rwlock_multi: OK (3.07s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (2.92s) + Crux output: 3 + atomic_fence: OK (1.99s) Compiling and running oracle program (0.16s) - Oracle output: 3 (3.16s) - TODO RGS evalRval ThreadLocalRef - CallStack (from HasCallStack): - error, called at src/Mir/Trans.hs:1034:33 in crucible-mir-0.1-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1197:11 in crucible-mir-0.1-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1642:68 in crucible-mir-0.1-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1651:28 in crucible-mir-0.1-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1691:10 in crucible-mir-0.1-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1737:24 in crucible-mir-0.1-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2171:30 in crucible-mir-0.1-inplace:Mir.Trans - transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate - translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language - - Use -p '/rwlock_multi/' to rerun this test only. - atomic_fence: OK (2.00s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.85s) + Oracle output: 2 (1.83s) Crux output: 2 - mutex_multi: FAIL (3.28s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (3.13s) - TODO RGS evalRval ThreadLocalRef - CallStack (from HasCallStack): - error, called at src/Mir/Trans.hs:1034:33 in crucible-mir-0.1-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1197:11 in crucible-mir-0.1-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1642:68 in crucible-mir-0.1-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1651:28 in crucible-mir-0.1-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1691:10 in crucible-mir-0.1-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1737:24 in crucible-mir-0.1-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2171:30 in crucible-mir-0.1-inplace:Mir.Trans - transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate - translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language - - Use -p '/mutex_multi/' to rerun this test only. + mutex_multi: OK (2.98s) + Compiling and running oracle program (0.16s) + Oracle output: 3 (2.82s) + Crux output: 3 consts - struct_val: OK (1.96s) + struct_val: OK (1.91s) Compiling and running oracle program (0.15s) - Oracle output: Foo { x: true } (1.81s) + Oracle output: Foo { x: true } (1.76s) Crux output: Foo { x: true } - struct_unit: OK (1.95s) - Compiling and running oracle program (0.15s) - Oracle output: Err(Foo { x: () }) (1.81s) + struct_unit: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: Err(Foo { x: () }) (1.74s) Crux output: Err(Foo { x: () }) - local_key: OK (1.98s) + local_key: OK (1.91s) Compiling and running oracle program (0.15s) - Oracle output: 1 (1.83s) + Oracle output: 1 (1.76s) Crux output: 1 - fn_def: OK (1.92s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.78s) + fn_def: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.72s) Crux output: 1 enum_val: OK (1.87s) Compiling and running oracle program (0.14s) - Oracle output: None (1.72s) + Oracle output: None (1.73s) Crux output: None crypto - add: OK (2.03s) - Compiling and running oracle program (0.15s) - Oracle output: true (1.88s) - Crux output: true - add_noL: OK (2.02s) + add: OK (1.94s) Compiling and running oracle program (0.16s) - Oracle output: false (1.86s) + Oracle output: true (1.78s) + Crux output: true + add_noL: OK (1.96s) + Compiling and running oracle program (0.15s) + Oracle output: false (1.81s) Crux output: false cell - cell: OK (1.95s) + cell: OK (1.98s) Compiling and running oracle program (0.15s) - Oracle output: 2 (1.81s) + Oracle output: 2 (1.84s) Crux output: 2 - ref_cell: OK (2.49s) + ref_cell: OK (2.48s) Compiling and running oracle program (0.15s) Oracle output: 2 (2.34s) Crux output: 2 - ref_cell2: OK (2.61s) + ref_cell2: OK (2.49s) Compiling and running oracle program (0.15s) - Oracle output: 2 (2.46s) + Oracle output: 2 (2.34s) Crux output: 2 fnptr - call: OK (1.95s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.80s) + call: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.76s) Crux output: 2 - field: OK (1.99s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (1.83s) + field: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.74s) Crux output: 2 - make: OK (1.98s) - Compiling and running oracle program (0.15s) - Oracle output: 0 (1.84s) + make: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.73s) Crux output: 0 custom: rustc compilation failed for custom error output: @@ -608,44 +534,44 @@ error: aborting due to previous error For more information about this error, try `rustc --explain E0432`. -FAIL (expected: taking address of an overridden function) (0.06s) - Compiling and running oracle program (0.06s) +FAIL (expected: taking address of an overridden function) (0.05s) + Compiling and running oracle program (0.05s) test/Test.hs:107: failed to compile and run (expected failure) num - overflow: OK (2.00s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.85s) + overflow: OK (1.98s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.84s) Crux output: () - from_bytes: OK (1.96s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.82s) + from_bytes: OK (1.99s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.85s) Crux output: () - saturate: OK (2.54s) + saturate: OK (2.49s) Compiling and running oracle program (0.17s) - Oracle output: () (2.37s) + Oracle output: () (2.32s) Crux output: () prim - bool: OK (1.92s) + bool: OK (1.88s) Compiling and running oracle program (0.14s) - Oracle output: false (1.78s) + Oracle output: false (1.73s) Crux output: false - shift3: OK (1.90s) + shift3: OK (1.87s) Compiling and running oracle program (0.14s) - Oracle output: -9223372036854775808 (1.76s) + Oracle output: -9223372036854775808 (1.73s) Crux output: -9223372036854775808 - div: OK (1.92s) + div: OK (1.90s) Compiling and running oracle program (0.14s) - Oracle output: 4 (1.78s) + Oracle output: 4 (1.75s) Crux output: 4 ge: OK (1.91s) Compiling and running oracle program (0.14s) Oracle output: true (1.77s) Crux output: true - add1: OK (1.93s) + add1: OK (1.85s) Compiling and running oracle program (0.14s) - Oracle output: 2 (1.79s) + Oracle output: 2 (1.71s) Crux output: 2 shift_exceeding: rustc compilation failed for shift_exceeding error output: @@ -684,95 +610,95 @@ FAIL (expected: Should panic, but doesn't) (0.05s) (expected failure) char_from_u32: OK (1.91s) Compiling and running oracle program (0.14s) - Oracle output: 'A' (1.76s) + Oracle output: 'A' (1.77s) Crux output: 'A' - ffs: OK (1.91s) + ffs: OK (1.97s) Compiling and running oracle program (0.15s) - Oracle output: true (1.76s) + Oracle output: true (1.81s) Crux output: true - mut_arg: OK (1.87s) + mut_arg: OK (1.92s) Compiling and running oracle program (0.15s) - Oracle output: 1 (1.72s) + Oracle output: 1 (1.77s) Crux output: 1 - litstring: OK (1.87s) + litstring: OK (1.95s) Compiling and running oracle program (0.14s) - Oracle output: true (1.73s) + Oracle output: true (1.80s) Crux output: true - lit: OK (1.90s) + lit: OK (1.91s) Compiling and running oracle program (0.14s) - Oracle output: false (1.75s) + Oracle output: false (1.76s) Crux output: false - litbstring: OK (1.94s) + litbstring: OK (1.89s) Compiling and running oracle program (0.14s) - Oracle output: true (1.80s) + Oracle output: true (1.75s) Crux output: true - shift1: OK (1.91s) + shift1: OK (1.87s) Compiling and running oracle program (0.15s) - Oracle output: 2 (1.76s) + Oracle output: 2 (1.73s) Crux output: 2 - shift2: OK (1.92s) + shift2: OK (1.90s) Compiling and running oracle program (0.14s) - Oracle output: 2 (1.77s) + Oracle output: 2 (1.76s) Crux output: 2 - shift4: OK (1.88s) + shift4: OK (1.90s) Compiling and running oracle program (0.14s) - Oracle output: -9223372036854775808 (1.73s) + Oracle output: -9223372036854775808 (1.76s) Crux output: -9223372036854775808 - mut: OK (1.93s) - Compiling and running oracle program (0.14s) - Oracle output: 14 (1.79s) + mut: OK (1.88s) + Compiling and running oracle program (0.15s) + Oracle output: 14 (1.74s) Crux output: 14 - wrapping_sub: OK (1.91s) + wrapping_sub: OK (1.89s) Compiling and running oracle program (0.14s) - Oracle output: true (1.77s) + Oracle output: true (1.75s) Crux output: true impl - self: OK (1.97s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.82s) + self: OK (1.86s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.71s) Crux output: 42 - simple: OK (1.97s) - Compiling and running oracle program (0.17s) - Oracle output: 42 (1.81s) + simple: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (1.74s) Crux output: 42 - self_mut: OK (1.92s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.77s) + self_mut: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (1.73s) Crux output: 42 box - new: OK (2.05s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.90s) + new: OK (2.01s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.87s) Crux output: () - mut_ref: OK (2.04s) - Compiling and running oracle program (0.17s) - Oracle output: () (1.86s) + mut_ref: OK (2.02s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.88s) Crux output: () - mut: OK (2.02s) + mut: OK (2.01s) Compiling and running oracle program (0.15s) - Oracle output: () (1.87s) + Oracle output: () (1.86s) Crux output: () - unsize: OK (2.05s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.90s) + unsize: OK (1.99s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.85s) Crux output: 2 - struct: OK (2.02s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.87s) + struct: OK (2.05s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (1.89s) Crux output: 1 ptr - is_null_slice: FAIL (expected: can't unsize null pointers) (1.94s) - Compiling and running oracle program (0.14s) + is_null_slice: FAIL (expected: can't unsize null pointers) (1.95s) + Compiling and running oracle program (0.15s) Oracle output: (false, true) (1.80s) Crux output: failures: - ---- is_null_slice/7fe66064::crux_test[0] counterexamples ---- + ---- is_null_slice/3549f883::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/ptr/is_null_slice.rs:7:13: 7:33: error: in is_null_slice/7fe66064::crux_test[0] + [Crux] test/conc_eval/ptr/is_null_slice.rs:7:13: 7:33: error: in is_null_slice/3549f883::crux_test[0] [Crux] attempted subindex on the result of an integer-to-pointer cast [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/ptr/const_ptr.rs:38:38: 38:42: error: in core/1cdf94ce::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst55af81a2cef2701c[0] + [Crux] ./lib/core/src/ptr/const_ptr.rs:38:38: 38:42: error: in core/092bc89a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst55af81a2cef2701c[0] [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. @@ -780,149 +706,108 @@ FAIL (expected: Should panic, but doesn't) (0.05s) test/Test.hs:124: crux doesn't match oracle (expected failure) - struct_eq: OK (1.93s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.79s) + struct_eq: OK (1.88s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.74s) Crux output: () - read_write: OK (1.98s) + read_write: OK (1.94s) Compiling and running oracle program (0.15s) - Oracle output: [1, 3, 1] (1.82s) + Oracle output: [1, 3, 1] (1.80s) Crux output: [1, 3, 1] - offset_mut: OK (1.92s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.78s) + offset_mut: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (1.75s) Crux output: 3 - copy: OK (1.97s) + copy: OK (1.94s) Compiling and running oracle program (0.15s) - Oracle output: [1, 2, 3, 1, 2, 3] (1.81s) + Oracle output: [1, 2, 3, 1, 2, 3] (1.78s) Crux output: [1, 2, 3, 1, 2, 3] - unsize_slice: OK (1.92s) + unsize_slice: OK (1.88s) Compiling and running oracle program (0.14s) - Oracle output: (1, 2) (1.78s) + Oracle output: (1, 2) (1.73s) Crux output: (1, 2) - dangling_eq: FAIL (1.94s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.80s) - Crux output: - failures: - - ---- dangling_eq/3f4bcd46::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/ptr/metadata.rs:135:14: 135:78: error: in core/1cdf94ce::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/1cdf94ce::ptr[0]::metadata[0]::from_raw_parts_mut[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/dangling_eq/' to rerun this test only. - cast_eq: OK (1.89s) + dangling_eq: OK (1.94s) Compiling and running oracle program (0.14s) - Oracle output: () (1.75s) + Oracle output: () (1.80s) Crux output: () - offset_from: OK (1.89s) + cast_eq: OK (1.91s) Compiling and running oracle program (0.15s) - Oracle output: -2 (1.74s) + Oracle output: () (1.77s) + Crux output: () + offset_from: OK (1.95s) + Compiling and running oracle program (0.14s) + Oracle output: -2 (1.81s) Crux output: -2 - null_eq: FAIL (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.79s) - Crux output: - failures: - - ---- null_eq/7504a468::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/ptr/metadata.rs:118:14: 118:78: error: in core/1cdf94ce::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0] - [Crux] Translation error in core/1cdf94ce::ptr[0]::metadata[0]::from_raw_parts[0]::_inst1e2825177cd3b608[0]: evalPlace (PField, Union) NYI - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/null_eq/' to rerun this test only. - coerce_unsized: OK (1.88s) - Compiling and running oracle program (0.15s) + null_eq: OK (1.91s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.77s) + Crux output: 1 + coerce_unsized: OK (1.87s) + Compiling and running oracle program (0.14s) Oracle output: (1, 2) (1.73s) Crux output: (1, 2) - offset: OK (1.89s) + offset: OK (1.96s) Compiling and running oracle program (0.14s) - Oracle output: 3 (1.75s) + Oracle output: 3 (1.81s) Crux output: 3 - is_null: OK (1.90s) - Compiling and running oracle program (0.15s) - Oracle output: (false, true) (1.75s) + is_null: OK (2.14s) + Compiling and running oracle program (0.16s) + Oracle output: (false, true) (1.98s) Crux output: (false, true) - valid_eq: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.74s) + valid_eq: OK (2.03s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.87s) Crux output: () ops - deref2: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.75s) - Crux output: () - index2: OK (1.88s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.73s) + deref2: OK (2.50s) + Compiling and running oracle program (0.16s) + Oracle output: () (2.35s) Crux output: () - arith1: OK (1.89s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.74s) + index2: OK (2.40s) + Compiling and running oracle program (0.22s) + Oracle output: () (2.18s) Crux output: () - index3: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.74s) + arith1: OK (2.00s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.84s) Crux output: () - deref1: OK (1.90s) + index3: OK (1.98s) Compiling and running oracle program (0.15s) - Oracle output: () (1.75s) + Oracle output: () (1.82s) Crux output: () - index1: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.75s) + deref1: OK (1.98s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.83s) Crux output: () - deref3: OK (2.02s) + index1: OK (1.97s) Compiling and running oracle program (0.15s) - Oracle output: () (1.86s) + Oracle output: () (1.82s) + Crux output: () + deref3: OK (2.10s) + Compiling and running oracle program (0.18s) + Oracle output: () (1.92s) Crux output: () statics - promoted_fn: OK (1.94s) + promoted_fn: OK (1.95s) Compiling and running oracle program (0.15s) Oracle output: 1 (1.80s) Crux output: 1 - promoted_static: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.75s) + promoted_static: OK (2.01s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.86s) Crux output: 1 io - cursor_write2: FAIL (3.04s) - Compiling and running oracle program (0.16s) - Oracle output: 0 (2.88s) - standalone use of `dyn` is not supported: TyDynamic core/1cdf94ce::any[0]::Erased[0]::_trait58656ccc69864acf[0] - CallStack (from HasCallStack): - error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:213:7 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:402:29 in crucible-mir-0.1-inplace:Mir.TransTy - tyToReprCont, called at src/Mir/TransTy.hs:202:24 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy - tyListToCtx, called at src/Mir/TransTy.hs:221:6 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy - tyListToCtx, called at src/Mir/TransTy.hs:1066:16 in crucible-mir-0.1-inplace:Mir.TransTy - traitVtableType, called at src/Mir/Trans.hs:2055:18 in crucible-mir-0.1-inplace:Mir.Trans - transVirtCall, called at src/Mir/Trans.hs:2178:31 in crucible-mir-0.1-inplace:Mir.Trans - transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate - translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language - - Use -p '/cursor_write2/' to rerun this test only. - cursor_write: OK (2.52s) + cursor_write2: OK (3.26s) Compiling and running oracle program (0.17s) - Oracle output: 0 (2.35s) + Oracle output: 0 (3.09s) Crux output: 0 - cursor_read: OK (2.54s) - Compiling and running oracle program (0.16s) - Oracle output: 0 (2.38s) + cursor_write: OK (2.63s) + Compiling and running oracle program (0.17s) + Oracle output: 0 (2.46s) + Crux output: 0 + cursor_read: OK (2.70s) + Compiling and running oracle program (0.17s) + Oracle output: 0 (2.53s) Crux output: 0 mem maybe_uninit: FAIL (2.00s) @@ -931,10 +816,10 @@ FAIL (expected: Should panic, but doesn't) (0.05s) Crux output: failures: - ---- maybe_uninit/9a4e7f6f::crux_test[0] counterexamples ---- + ---- maybe_uninit/00b53fab::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in maybe_uninit/9a4e7f6f::crux_test[0] - [Crux] Translation error in maybe_uninit/9a4e7f6f::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyInt B32]) + [Crux] internal: error: in maybe_uninit/00b53fab::crux_test[0] + [Crux] Translation error in maybe_uninit/00b53fab::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyInt B32]) [Crux] Overall status: Invalid. @@ -942,16 +827,16 @@ FAIL (expected: Should panic, but doesn't) (0.05s) crux doesn't match oracle Use -p '$0=="crux-mir.crux concrete..mem.maybe_uninit"' to rerun this test only. - maybe_uninit_array_cast: FAIL (1.99s) + maybe_uninit_array_cast: FAIL (2.05s) Compiling and running oracle program (0.15s) - Oracle output: [1, 2] (1.84s) + Oracle output: [1, 2] (1.89s) Crux output: failures: - ---- maybe_uninit_array_cast/54dba2bc::crux_test[0] counterexamples ---- + ---- maybe_uninit_array_cast/8abbf92b::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in maybe_uninit_array_cast/54dba2bc::crux_test[0] - [Crux] Translation error in maybe_uninit_array_cast/54dba2bc::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyArray (TyInt B32) 2]) + [Crux] internal: error: in maybe_uninit_array_cast/8abbf92b::crux_test[0] + [Crux] Translation error in maybe_uninit_array_cast/8abbf92b::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyArray (TyInt B32) 2]) [Crux] Overall status: Invalid. @@ -960,169 +845,154 @@ FAIL (expected: Should panic, but doesn't) (0.05s) Use -p '/maybe_uninit_array_cast/' to rerun this test only. time - instant: FAIL (3.14s) + instant: OK (1.98s) Compiling and running oracle program (0.15s) - Oracle output: () (2.99s) - standalone use of `dyn` is not supported: TyDynamic core/1cdf94ce::any[0]::Erased[0]::_trait58656ccc69864acf[0] - CallStack (from HasCallStack): - error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:213:7 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:402:29 in crucible-mir-0.1-inplace:Mir.TransTy - tyToReprCont, called at src/Mir/TransTy.hs:202:24 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy - tyListToCtx, called at src/Mir/TransTy.hs:221:6 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy - tyListToCtx, called at src/Mir/TransTy.hs:1066:16 in crucible-mir-0.1-inplace:Mir.TransTy - traitVtableType, called at src/Mir/Trans.hs:2055:18 in crucible-mir-0.1-inplace:Mir.Trans - transVirtCall, called at src/Mir/Trans.hs:2178:31 in crucible-mir-0.1-inplace:Mir.Trans - transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate - translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language - - Use -p '/instant/' to rerun this test only. + Oracle output: () (1.84s) + Crux output: () struct - field_order: OK (2.01s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.86s) + field_order: OK (2.00s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.85s) Crux output: () - arg: OK (1.92s) + arg: OK (2.03s) Compiling and running oracle program (0.15s) - Oracle output: 42 (1.78s) + Oracle output: 42 (1.88s) Crux output: 42 - tup: OK (1.96s) + tup: OK (2.11s) Compiling and running oracle program (0.15s) - Oracle output: () (1.81s) + Oracle output: () (1.96s) Crux output: () - proj: OK (2.00s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.85s) + proj: OK (2.23s) + Compiling and running oracle program (0.16s) + Oracle output: 42 (2.06s) Crux output: 42 - ret: OK (1.97s) - Compiling and running oracle program (0.15s) - Oracle output: S { x: 42, y: 120 } (1.83s) + ret: OK (2.09s) + Compiling and running oracle program (0.17s) + Oracle output: S { x: 42, y: 120 } (1.91s) Crux output: S { x: 42, y: 120 } - repr_transparent: OK (2.07s) - Compiling and running oracle program (0.16s) - Oracle output: 6 (1.91s) + repr_transparent: OK (2.54s) + Compiling and running oracle program (0.17s) + Oracle output: 6 (2.37s) Crux output: 6 - repr_transparent_const: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: 124 (1.75s) + repr_transparent_const: OK (2.33s) + Compiling and running oracle program (0.18s) + Oracle output: 124 (2.15s) Crux output: 124 dyn - assoc_ty: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: 100 (1.75s) + assoc_ty: OK (2.10s) + Compiling and running oracle program (0.17s) + Oracle output: 100 (1.93s) Crux output: 100 - trait_param: OK (1.95s) - Compiling and running oracle program (0.14s) - Oracle output: 100 (1.80s) + trait_param: OK (2.11s) + Compiling and running oracle program (0.15s) + Oracle output: 100 (1.96s) Crux output: 100 - plain_trait: OK (2.00s) + plain_trait: OK (2.09s) Compiling and running oracle program (0.16s) - Oracle output: 100 (1.84s) + Oracle output: 100 (1.94s) Crux output: 100 - inherit: OK (1.98s) + inherit: OK (2.10s) Compiling and running oracle program (0.15s) - Oracle output: 4 (1.83s) + Oracle output: 4 (1.95s) Crux output: 4 stdlib - option: OK (1.95s) - Compiling and running oracle program (0.16s) - Oracle output: true (1.79s) + option: OK (2.12s) + Compiling and running oracle program (0.18s) + Oracle output: true (1.94s) Crux output: true - option2: OK (1.97s) + option2: OK (2.14s) Compiling and running oracle program (0.16s) - Oracle output: 0 (1.81s) + Oracle output: 0 (1.98s) Crux output: 0 - result: OK (1.95s) - Compiling and running oracle program (0.15s) - Oracle output: 27 (1.80s) + result: OK (2.10s) + Compiling and running oracle program (0.17s) + Oracle output: 27 (1.93s) Crux output: 27 - default: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: true (1.74s) + default: OK (2.06s) + Compiling and running oracle program (0.16s) + Oracle output: true (1.90s) Crux output: true - cvt: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.73s) + cvt: OK (2.09s) + Compiling and running oracle program (0.15s) + Oracle output: 0 (1.93s) Crux output: 0 - range: OK (2.01s) - Compiling and running oracle program (0.14s) - Oracle output: 10 (1.87s) - Crux output: 10 - poly: OK (2.04s) + range: OK (2.02s) Compiling and running oracle program (0.16s) - Oracle output: 1 (1.88s) + Oracle output: 10 (1.85s) + Crux output: 10 + poly: OK (2.09s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.93s) Crux output: 1 - result_interior: OK (2.01s) + result_interior: OK (2.09s) Compiling and running oracle program (0.15s) - Oracle output: 3 (1.86s) + Oracle output: 3 (1.94s) Crux output: 3 - default_impl: OK (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.79s) + default_impl: OK (2.34s) + Compiling and running oracle program (0.19s) + Oracle output: () (2.15s) Crux output: () - option3: OK (1.92s) - Compiling and running oracle program (0.14s) - Oracle output: 27 (1.78s) + option3: OK (2.15s) + Compiling and running oracle program (0.16s) + Oracle output: 27 (1.99s) Crux output: 27 - teq: OK (1.97s) - Compiling and running oracle program (0.15s) - Oracle output: false (1.82s) + teq: OK (2.38s) + Compiling and running oracle program (0.18s) + Oracle output: false (2.21s) Crux output: false array - wick2: OK (2.01s) - Compiling and running oracle program (0.16s) - Oracle output: true (1.85s) + wick2: OK (2.24s) + Compiling and running oracle program (0.19s) + Oracle output: true (2.04s) Crux output: true - arg: OK (2.04s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.88s) + arg: OK (2.16s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (2.00s) Crux output: 2 - iter: OK (2.14s) - Compiling and running oracle program (0.17s) - Oracle output: 3 (1.97s) + iter: OK (2.20s) + Compiling and running oracle program (0.19s) + Oracle output: 3 (2.02s) Crux output: 3 - mk_and_proj: OK (2.04s) - Compiling and running oracle program (0.16s) - Oracle output: 42 (1.89s) + mk_and_proj: OK (2.02s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.87s) Crux output: 42 wick3: FAIL (expected: needs Vec data structure from stdlib) (0.21s) Compiling and running oracle program (0.17s) Oracle output: true (0.04s) user error (Error 101 while running mir-json on test/conc_eval/array/wick3.rs) (expected failure) - clone: OK (2.10s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.94s) + clone: OK (2.06s) + Compiling and running oracle program (0.16s) + Oracle output: () (1.90s) Crux output: () - wick1: FAIL (expected: needs Vec data structure from stdlib) (0.23s) - Compiling and running oracle program (0.18s) + wick1: FAIL (expected: needs Vec data structure from stdlib) (0.21s) + Compiling and running oracle program (0.17s) Oracle output: true (0.05s) user error (Error 101 while running mir-json on test/conc_eval/array/wick1.rs) (expected failure) - mut_index: OK (2.46s) + mut_index: OK (2.54s) Compiling and running oracle program (0.16s) - Oracle output: 7 (2.31s) + Oracle output: 7 (2.38s) Crux output: 7 - mut_arg: OK (1.91s) + mut_arg: OK (2.05s) Compiling and running oracle program (0.15s) - Oracle output: 42 (1.76s) + Oracle output: 42 (1.89s) Crux output: 42 - const: OK (1.86s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.71s) + const: OK (1.99s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (1.83s) Crux output: 1 - from_slice: FAIL (2.58s) - Compiling and running oracle program (0.15s) - Oracle output: () (2.44s) + from_slice: FAIL (2.66s) + Compiling and running oracle program (0.17s) + Oracle output: () (2.48s) Crux output: failures: - ---- from_slice/4075ba45::f[0] counterexamples ---- + ---- from_slice/0eab1383::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/iter/traits/iterator.rs:2607:40: 2607:61: error: in core/1cdf94ce::iter[0]::traits[0]::iterator[0]::Iterator[0]::all[0]::_instf9a28a1fbcee32be[0] - [Crux] Translation error in core/1cdf94ce::iter[0]::traits[0]::iterator[0]::Iterator[0]::all[0]::_instf9a28a1fbcee32be[0]: ill-typed assignment of IntrinsicRepr MirReference [BVRepr 8] to IntrinsicRepr MirReference [AnyRepr] (TyRef (TyAdt core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0]::_adtdfc2b6f78a281997[0] core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0] (Substs [TyTuple [],TyTuple []])) Immut) LBase (Var {_varname = "_10", _varmut = Mut, _varty = TyRef (TyAdt core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0]::_adtdfc2b6f78a281997[0] core/1cdf94ce::ops[0]::control_flow[0]::ControlFlow[0] (Substs [TyTuple [],TyTuple []])) Immut, _varIsZST = False}) + [Crux] lib/core/src/ops/control_flow.rs:84:30: 84:39 !lib/core/src/ops/control_flow.rs:84:30: 84:39: error: in core/092bc89a::ops[0]::control_flow[0]::{impl#11}[0]::eq[0]::_instdfc2b6f78a281997[0] + [Crux] bad unpack: Any as StructRepr [BVRepr 32, VariantRepr [StructRepr [UnitRepr], StructRepr [UnitRepr]]] [Crux] Overall status: Invalid. @@ -1130,95 +1000,95 @@ FAIL (expected: Should panic, but doesn't) (0.05s) crux doesn't match oracle Use -p '/array.from_slice/' to rerun this test only. - const_impl: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 5 (1.73s) + const_impl: OK (1.96s) + Compiling and running oracle program (0.15s) + Oracle output: 5 (1.81s) Crux output: 5 enum - match: OK (1.90s) - Compiling and running oracle program (0.14s) + match: OK (1.92s) + Compiling and running oracle program (0.16s) Oracle output: 42 (1.76s) Crux output: 42 - field_order: OK (1.91s) + field_order: OK (1.89s) Compiling and running oracle program (0.14s) - Oracle output: () (1.77s) + Oracle output: () (1.75s) Crux output: () - mixed_discrs: OK (1.92s) + mixed_discrs: OK (1.93s) Compiling and running oracle program (0.15s) Oracle output: () (1.78s) Crux output: () - arg: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.73s) - Crux output: 0 - arg2: OK (1.87s) + arg: OK (1.89s) Compiling and running oracle program (0.15s) - Oracle output: 0 (1.73s) + Oracle output: 0 (1.74s) Crux output: 0 - cow: OK (2.01s) + arg2: OK (1.95s) Compiling and running oracle program (0.15s) - Oracle output: 200 (1.86s) + Oracle output: 0 (1.80s) + Crux output: 0 + cow: OK (2.09s) + Compiling and running oracle program (0.16s) + Oracle output: 200 (1.93s) Crux output: 200 - ret: OK (1.86s) + ret: OK (1.85s) Compiling and running oracle program (0.15s) - Oracle output: (A(42), B { x: 42 }, C) (1.72s) + Oracle output: (A(42), B { x: 42 }, C) (1.70s) Crux output: (A(42), B { x: 42 }, C) - eq: OK (1.90s) + eq: OK (1.96s) Compiling and running oracle program (0.15s) - Oracle output: () (1.75s) + Oracle output: () (1.81s) Crux output: () - cmp: OK (1.89s) - Compiling and running oracle program (0.15s) - Oracle output: -23 (1.74s) - Crux output: -23 - inner: OK (1.91s) + cmp: OK (1.92s) Compiling and running oracle program (0.14s) + Oracle output: -23 (1.78s) + Crux output: -23 + inner: OK (1.92s) + Compiling and running oracle program (0.15s) Oracle output: 4 (1.77s) Crux output: 4 vec - drop: OK (2.13s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.97s) + drop: OK (2.09s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.94s) Crux output: () - set_len: OK (2.15s) + set_len: OK (2.11s) Compiling and running oracle program (0.16s) - Oracle output: 2 (1.99s) + Oracle output: 2 (1.95s) Crux output: 2 - push: OK (2.17s) - Compiling and running oracle program (0.17s) - Oracle output: (1, 2) (2.00s) + push: OK (2.10s) + Compiling and running oracle program (0.15s) + Oracle output: (1, 2) (1.94s) Crux output: (1, 2) - extend: OK (2.25s) + extend: OK (2.19s) Compiling and running oracle program (0.17s) - Oracle output: (1, 10) (2.08s) + Oracle output: (1, 10) (2.02s) Crux output: (1, 10) - collect: OK (2.85s) + collect: OK (2.80s) Compiling and running oracle program (0.16s) - Oracle output: 45 (2.69s) + Oracle output: 45 (2.64s) Crux output: 45 - extend_trusted_len: OK (2.76s) - Compiling and running oracle program (0.16s) - Oracle output: (1, 10) (2.59s) + extend_trusted_len: OK (2.74s) + Compiling and running oracle program (0.17s) + Oracle output: (1, 10) (2.57s) Crux output: (1, 10) - from_elem_zero: OK (2.13s) + from_elem_zero: OK (2.17s) Compiling and running oracle program (0.16s) - Oracle output: 0 (1.97s) + Oracle output: 0 (2.01s) Crux output: 0 clos - promoted: OK (1.95s) - Compiling and running oracle program (0.16s) - Oracle output: 33 (1.80s) + promoted: OK (1.92s) + Compiling and running oracle program (0.15s) + Oracle output: 33 (1.78s) Crux output: 33 - as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (1.97s) + as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (1.89s) Compiling and running oracle program (0.14s) - Oracle output: 1 (1.82s) + Oracle output: 1 (1.75s) Crux output: failures: - ---- as_fn_ptr/dd580d0d::crux_test[0] counterexamples ---- + ---- as_fn_ptr/ab0e7632::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/dd580d0d::crux_test[0] - [Crux] Translation error in as_fn_ptr/dd580d0d::crux_test[0]: unimplemented cast: ClosureFnPointer + [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/ab0e7632::crux_test[0] + [Crux] Translation error in as_fn_ptr/ab0e7632::crux_test[0]: unimplemented cast: ClosureFnPointer [Crux] ty: TyClosure [] [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [], _fsreturn_ty = TyInt B32, _fsabi = RustAbi, _fsspreadarg = Nothing}) @@ -1227,34 +1097,34 @@ FAIL (expected: Should panic, but doesn't) (0.05s) test/Test.hs:124: crux doesn't match oracle (expected failure) - fnptr_fnmut: OK (1.96s) + fnptr_fnmut: OK (1.88s) Compiling and running oracle program (0.15s) - Oracle output: 4 (1.81s) + Oracle output: 4 (1.73s) Crux output: 4 - direct_fnonce: OK (1.93s) + direct_fnonce: OK (1.91s) Compiling and running oracle program (0.14s) - Oracle output: 2 (1.78s) + Oracle output: 2 (1.76s) Crux output: 2 - conv_fnonce_fnmut: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.73s) + conv_fnonce_fnmut: OK (1.90s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.75s) Crux output: 2 - fn_static_poly: OK (1.87s) + fn_static_poly: OK (1.88s) Compiling and running oracle program (0.14s) - Oracle output: 3 (1.73s) + Oracle output: 3 (1.74s) Crux output: 3 fnptr_fnonce: OK (1.88s) Compiling and running oracle program (0.15s) Oracle output: 4 (1.73s) Crux output: 4 - fn_static: OK (1.86s) + fn_static: OK (1.87s) Compiling and running oracle program (0.14s) Oracle output: 3 (1.72s) Crux output: 3 - fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (1.87s) + fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (1.88s) Compiling and running oracle program (0.14s) - Oracle output: 3 (1.73s) - vtable signature mismatch for vtable core/1cdf94ce::ops[0]::function[0]::Fn[0]::_vtbl570ffd80324c2ca9[0] of trait core/1cdf94ce::ops[0]::function[0]::Fn[0]::_trait42f8b6475ee85722[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32)] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32)] + Oracle output: 3 (1.74s) + vtable signature mismatch for vtable core/092bc89a::ops[0]::function[0]::Fn[0]::_vtbl570ffd80324c2ca9[0] of trait core/092bc89a::ops[0]::function[0]::Fn[0]::_trait42f8b6475ee85722[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32)] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32)] CallStack (from HasCallStack): error, called at src/Mir/Trans.hs:958:20 in crucible-mir-0.1-inplace:Mir.Trans mkTraitObject, called at src/Mir/Trans.hs:801:11 in crucible-mir-0.1-inplace:Mir.Trans @@ -1269,256 +1139,241 @@ FAIL (expected: Should panic, but doesn't) (0.05s) transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language (expected failure) - fnonce: OK (1.86s) + fnonce: OK (1.91s) Compiling and running oracle program (0.14s) - Oracle output: false (1.72s) + Oracle output: false (1.77s) Crux output: false conv_fnmut_fn: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.72s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.71s) Crux output: 2 direct_fnmut2: OK (1.87s) Compiling and running oracle program (0.15s) Oracle output: 7 (1.73s) Crux output: 7 - fo: OK (1.93s) + fo: OK (1.88s) Compiling and running oracle program (0.15s) - Oracle output: 29 (1.78s) + Oracle output: 29 (1.73s) Crux output: 29 - unique_borrow: OK (1.92s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.77s) + unique_borrow: OK (1.94s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (1.79s) Crux output: 3 - conv_fnonce_fn: OK (1.94s) + conv_fnonce_fn: OK (1.92s) Compiling and running oracle program (0.14s) - Oracle output: 2 (1.80s) + Oracle output: 2 (1.77s) Crux output: 2 - fnptr_fn: OK (2.02s) + fnptr_fn: OK (1.91s) Compiling and running oracle program (0.15s) - Oracle output: 4 (1.88s) + Oracle output: 4 (1.76s) Crux output: 4 - direct_fnmut: OK (1.97s) - Compiling and running oracle program (0.16s) - Oracle output: 7 (1.81s) - Crux output: 7 - dispatch_fnmut: OK (1.89s) + direct_fnmut: OK (2.01s) Compiling and running oracle program (0.15s) - Oracle output: 7 (1.75s) + Oracle output: 7 (1.85s) Crux output: 7 - poly: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.73s) + dispatch_fnmut: OK (2.10s) + Compiling and running oracle program (0.17s) + Oracle output: 7 (1.94s) + Crux output: 7 + poly: OK (2.21s) + Compiling and running oracle program (0.16s) + Oracle output: 3 (2.05s) Crux output: 3 - direct_fn: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.74s) + direct_fn: OK (2.00s) + Compiling and running oracle program (0.16s) + Oracle output: 2 (1.84s) Crux output: 2 - fnptr_closure: OK (1.89s) - Compiling and running oracle program (0.15s) - Oracle output: 7 (1.74s) + fnptr_closure: OK (2.18s) + Compiling and running oracle program (0.17s) + Oracle output: 7 (2.01s) Crux output: 7 - fnonce1: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.73s) + fnonce1: OK (1.98s) + Compiling and running oracle program (0.16s) + Oracle output: 3 (1.82s) Crux output: 3 - ref_fnmut: OK (1.89s) + ref_fnmut: OK (1.97s) Compiling and running oracle program (0.15s) - Oracle output: 7 (1.74s) + Oracle output: 7 (1.82s) Crux output: 7 vec_deque - pop: OK (2.51s) - Compiling and running oracle program (0.16s) - Oracle output: [5, 4, 3, 1, 2] (2.35s) + pop: OK (2.72s) + Compiling and running oracle program (0.17s) + Oracle output: [5, 4, 3, 1, 2] (2.55s) Crux output: [5, 4, 3, 1, 2] - iter_clone: OK (2.67s) - Compiling and running oracle program (0.16s) - Oracle output: [1, 2, 3, 2, 3] (2.51s) + iter_clone: OK (2.70s) + Compiling and running oracle program (0.17s) + Oracle output: [1, 2, 3, 2, 3] (2.53s) Crux output: [1, 2, 3, 2, 3] - push: OK (2.56s) + push: OK (2.72s) Compiling and running oracle program (0.16s) - Oracle output: [4, 1, 2, 3, 5] (2.40s) + Oracle output: [4, 1, 2, 3, 5] (2.56s) Crux output: [4, 1, 2, 3, 5] - retain: OK (2.57s) - Compiling and running oracle program (0.16s) - Oracle output: [1, 4] (2.41s) + retain: OK (2.68s) + Compiling and running oracle program (0.17s) + Oracle output: [1, 4] (2.50s) Crux output: [1, 4] - rotate_left: OK (2.59s) + rotate_left: OK (2.76s) Compiling and running oracle program (0.16s) - Oracle output: [3, 4, 5, 1, 2] (2.43s) + Oracle output: [3, 4, 5, 1, 2] (2.60s) Crux output: [3, 4, 5, 1, 2] - rotate_right: OK (2.63s) - Compiling and running oracle program (0.16s) - Oracle output: [4, 5, 1, 2, 3] (2.47s) + rotate_right: OK (2.89s) + Compiling and running oracle program (0.17s) + Oracle output: [4, 5, 1, 2, 3] (2.72s) Crux output: [4, 5, 1, 2, 3] slice - len: OK (1.87s) - Compiling and running oracle program (0.15s) - Oracle output: 5 (1.73s) + len: OK (2.09s) + Compiling and running oracle program (0.16s) + Oracle output: 5 (1.93s) Crux output: 5 - mut_range: OK (2.34s) - Compiling and running oracle program (0.15s) - Oracle output: 86 (2.19s) + mut_range: OK (2.70s) + Compiling and running oracle program (0.16s) + Oracle output: 86 (2.54s) Crux output: 86 - range_len: OK (2.35s) + range_len: OK (2.51s) Compiling and running oracle program (0.15s) - Oracle output: 1 (2.21s) + Oracle output: 1 (2.36s) Crux output: 1 - range_len_mut: OK (2.33s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (2.19s) + range_len_mut: OK (2.54s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (2.38s) Crux output: 1 - mk_and_proj: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.72s) + mk_and_proj: OK (1.98s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.83s) Crux output: 42 - swap: OK (1.94s) - Compiling and running oracle program (0.14s) - Oracle output: 2001 (1.79s) + swap: OK (2.03s) + Compiling and running oracle program (0.15s) + Oracle output: 2001 (1.88s) Crux output: 2001 - eq: OK (1.97s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.82s) - Crux output: () - iter_mut: OK (1.97s) + eq: OK (2.06s) Compiling and running oracle program (0.15s) - Oracle output: () (1.82s) + Oracle output: () (1.90s) Crux output: () - get: OK (1.90s) - Compiling and running oracle program (0.15s) - Oracle output: true (1.75s) + iter_mut: OK (2.16s) + Compiling and running oracle program (0.16s) + Oracle output: () (2.00s) + Crux output: () + get: OK (2.08s) + Compiling and running oracle program (0.17s) + Oracle output: true (1.91s) Crux output: true - mut: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.72s) + mut: OK (1.97s) + Compiling and running oracle program (0.16s) + Oracle output: 42 (1.82s) Crux output: 42 - last: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.73s) + last: OK (2.27s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (2.12s) Crux output: 3 crux symbolic Output testing - mux_init_mut: OK (1.83s) - mux_init_imm: OK (1.78s) - early_fail: OK (1.86s) + mux_init_mut: OK (2.04s) + mux_init_imm: OK (1.98s) + early_fail: OK (1.93s) mixed_fail: OK (1.89s) - multi: OK (1.89s) - fail_return: OK (1.80s) - no_conc: OK (1.86s) + multi: OK (1.98s) + fail_return: OK (1.88s) + no_conc: OK (1.91s) conc: OK (1.91s) - array: OK (1.88s) - assert_ok: OK (2.40s) - assert: FAIL (2.50s) + array: OK (1.89s) + assert_ok: OK (2.39s) + assert: FAIL (2.47s) files test/symb_eval/concretize/assert.good and test/symb_eval/concretize/assert.out differ; test/symb_eval/concretize/assert.out contains: - test assert/b59efe3c::crux_test[0]: returned 1, FAILED + test assert/9ec99c21::crux_test[0]: returned 1, FAILED failures: - ---- assert/b59efe3c::crux_test[0] counterexamples ---- + ---- assert/9ec99c21::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/1cdf94ce::fmt[0]::write[0] - [Crux] Translation error in core/1cdf94ce::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/1cdf94ce::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] + [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. Use -p '$0=="crux-mir.crux symbolic.Output testing.assert"' to rerun this test only. bytes2: OK (1.88s) - double: OK (1.79s) - ffs: OK (1.92s) - bytes: OK (1.89s) - bad_symb1: OK (1.78s) - bad_symb2: OK (1.81s) + double: OK (1.77s) + ffs: OK (1.93s) + bytes: OK (1.87s) + bad_symb1: OK (1.81s) + bad_symb2: OK (1.82s) override3: OK (1.82s) - override1: OK (1.76s) - override4: OK (1.83s) - override_rust: OK (1.82s) - override2: OK (1.85s) - override5: OK (1.79s) - mux: OK (1.83s) - checked_mul_signed: OK (1.70s) - checked_mul: OK (1.69s) - checked_div: OK (1.75s) - checked_add: OK (1.70s) - test1: OK (109.12s) - pop: FAIL (1.79s) + override1: OK (1.78s) + override4: OK (1.94s) + override_rust: OK (2.06s) + override2: OK (1.99s) + override5: OK (2.12s) + mux: OK (2.02s) + checked_mul_signed: OK (1.86s) + checked_mul: OK (1.76s) + checked_div: OK (1.82s) + checked_add: OK (1.74s) + test1: OK (112.50s) + pop: FAIL (1.83s) files test/symb_eval/vector/pop.good and test/symb_eval/vector/pop.out differ; test/symb_eval/vector/pop.out contains: - test pop/9a0a9424::f[0]: FAILED + test pop/5b5ade0c::f[0]: FAILED failures: - ---- pop/9a0a9424::f[0] counterexamples ---- + ---- pop/5b5ade0c::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/vector/pop.rs:9:18: 9:19: error: in pop/9a0a9424::f[0] - [Crux] Translation error in pop/9a0a9424::f[0]: unknown ADT (core/3a1fbbbh::option[0]::Option[0],Substs [TyUint B8]) + [Crux] test/symb_eval/vector/pop.rs:9:18: 9:19: error: in pop/5b5ade0c::f[0] + [Crux] Translation error in pop/5b5ade0c::f[0]: unknown ADT (core/3a1fbbbh::option[0]::Option[0],Substs [TyUint B8]) [Crux] Overall status: Invalid. Use -p '/Output testing.pop/' to rerun this test only. - as_mut_slice: OK (1.83s) - push: OK (1.87s) - copy_from_slice: OK (1.79s) + as_mut_slice: OK (1.82s) + push: OK (1.80s) + copy_from_slice: OK (1.78s) new: OK (1.84s) - mut: OK (1.79s) - concat: OK (1.85s) - split_at: OK (1.83s) + mut: OK (1.83s) + concat: OK (1.84s) + split_at: OK (1.80s) replicate: OK (1.85s) - as_slice: OK (1.77s) - array_mut: OK (1.79s) - array: OK (1.80s) - extend_bytes: OK (1.81s) - put: OK (2.25s) - split_to: OK (2.22s) - new: OK (1.84s) - split_off: OK (2.18s) - sym_len: OK (2.36s) - put_overflow: OK (2.25s) - vec_cursor_read: FAIL (2.72s) - standalone use of `dyn` is not supported: TyDynamic core/1cdf94ce::any[0]::Erased[0]::_trait58656ccc69864acf[0] - CallStack (from HasCallStack): - error, called at src/Mir/TransTy.hs:228:25 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:213:7 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:402:29 in crucible-mir-0.1-inplace:Mir.TransTy - tyToReprCont, called at src/Mir/TransTy.hs:202:24 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy - tyListToCtx, called at src/Mir/TransTy.hs:221:6 in crucible-mir-0.1-inplace:Mir.TransTy - tyToRepr, called at src/Mir/TransTy.hs:418:46 in crucible-mir-0.1-inplace:Mir.TransTy - tyListToCtx, called at src/Mir/TransTy.hs:1066:16 in crucible-mir-0.1-inplace:Mir.TransTy - traitVtableType, called at src/Mir/Trans.hs:2055:18 in crucible-mir-0.1-inplace:Mir.Trans - transVirtCall, called at src/Mir/Trans.hs:2178:31 in crucible-mir-0.1-inplace:Mir.Trans - transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate - translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language - Use -p '/vec_cursor_read/' to rerun this test only. - vec_write: OK (2.32s) + as_slice: OK (1.83s) + array_mut: OK (1.84s) + array: OK (1.79s) + extend_bytes: OK (1.78s) + put: OK (2.22s) + split_to: OK (2.21s) + new: OK (1.82s) + split_off: OK (2.27s) + sym_len: OK (2.35s) + put_overflow: OK (2.30s) + vec_cursor_read: OK (2.79s) + vec_write: OK (2.37s) write: FAIL (0.06s) test/symb_eval/byteorder/write.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/write.rs) Use -p '/Output testing.write/' to rerun this test only. read: FAIL test/symb_eval/byteorder/read.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/read.rs) Use -p '/Output testing.read/' to rerun this test only. - slice_mut: OK (1.87s) + slice_mut: OK (1.85s) basic: OK (1.78s) - slice: OK (1.95s) - mux_slice: OK (2.04s) - mux: OK (1.85s) - uninit_read: OK (1.88s) - valid_read: OK (1.89s) - out_of_bounds: OK (1.81s) + slice: OK (1.93s) + mux_slice: OK (1.93s) + mux: OK (1.79s) + uninit_read: OK (1.84s) + valid_read: OK (1.90s) + out_of_bounds: OK (1.84s) zero_length: OK (1.84s) - downcast: OK (1.86s) - downcast_fail: OK (1.78s) - conditional: OK (1.84s) - literals: OK (1.72s) - arith: OK (1.87s) - leading_zeros: OK (1.90s) - overflowing_sub: OK (1.84s) - cmp: OK (1.79s) - symbolic: OK (1.71s) - from_to: OK (1.76s) - clone: OK (2.00s) - sort_by_key: OK (2.69s) - into_iter: OK (2.39s) - macro: OK (1.83s) - construct: OK (1.84s) - deserialize: OK (1.83s) + downcast: OK (1.77s) + downcast_fail: OK (1.84s) + conditional: OK (1.82s) + literals: OK (1.73s) + arith: OK (1.93s) + leading_zeros: OK (1.91s) + overflowing_sub: OK (1.82s) + cmp: OK (1.84s) + symbolic: OK (1.70s) + from_to: OK (1.72s) + clone: OK (1.99s) + sort_by_key: OK (2.65s) + into_iter: OK (2.41s) + macro: OK (1.81s) + construct: OK (1.83s) + deserialize: OK (1.84s) crux coverage Output testing coverage_dual: warning: trailing semicolon in macro used in expression position @@ -1535,7 +1390,7 @@ FAIL (expected: Should panic, but doesn't) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (1.83s) +OK (1.82s) coverage: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | @@ -1550,16 +1405,7 @@ FAIL (expected: Should panic, but doesn't) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (1.86s) - files test/coverage/coverage.good and test/coverage/coverage.out differ; test/coverage/coverage.out contains: - warning: branch condition never has a value other than [1] - ┌─ test/coverage/coverage.rs:11:15 - │ - 11 │ } else if x == 1 { - │ ^ - - - Use -p '$0=="crux-mir.crux coverage.Output testing.coverage"' to rerun this test only. +OK (1.84s) coverage_try: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | @@ -1574,7 +1420,7 @@ FAIL (expected: Should panic, but doesn't) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (1.93s) +OK (1.86s) coverage_cond: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | @@ -1589,7 +1435,7 @@ FAIL (expected: Should panic, but doesn't) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (1.78s) +OK (1.77s) coverage_loop: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | @@ -1604,16 +1450,7 @@ FAIL (expected: Should panic, but doesn't) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (1.83s) - files test/coverage/coverage_loop.good and test/coverage/coverage_loop.out differ; test/coverage/coverage_loop.out contains: - warning: branch condition never has value true - ┌─ test/coverage/coverage_loop.rs:12:19 - │ - 12 │ } else if i > 99 { - │ ^^^^^^ - - - Use -p '/coverage_loop/' to rerun this test only. +OK (1.85s) coverage_shortcircuit: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | @@ -1628,22 +1465,7 @@ FAIL (expected: Should panic, but doesn't) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (1.88s) - files test/coverage/coverage_shortcircuit.good and test/coverage/coverage_shortcircuit.out differ; test/coverage/coverage_shortcircuit.out contains: - warning: branch condition never has value false - ┌─ test/coverage/coverage_shortcircuit.rs:9:8 - │ - 9 │ if x == 0 || x == 1 { - │ ^^^^^^^^^^^^^^^^ - - warning: branch condition never has a value other than [0] - ┌─ test/coverage/coverage_shortcircuit.rs:9:8 - │ - 9 │ if x == 0 || x == 1 { - │ ^ - - - Use -p '/coverage_shortcircuit/' to rerun this test only. +OK (1.90s) coverage_mono: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | @@ -1658,7 +1480,7 @@ FAIL (expected: Should panic, but doesn't) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (1.78s) +OK (1.79s) coverage_match: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | @@ -1673,22 +1495,7 @@ FAIL (expected: Should panic, but doesn't) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (1.90s) - files test/coverage/coverage_match.good and test/coverage/coverage_match.out differ; test/coverage/coverage_match.out contains: - warning: branch condition never has value 1 - ┌─ test/coverage/coverage_match.rs:12:9 - │ - 12 │ let x = u8::symbolic("x"); - │ ^ - - warning: branch condition never has value 20 - ┌─ test/coverage/coverage_match.rs:22:9 - │ - 22 │ E::A => 1, - │ ^^^^ - - - Use -p '/coverage_match/' to rerun this test only. +OK (1.91s) coverage_macro: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | @@ -1703,18 +1510,6 @@ FAIL (expected: Should panic, but doesn't) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -FAIL (1.89s) - files test/coverage/coverage_macro.good and test/coverage/coverage_macro.out differ; test/coverage/coverage_macro.out contains: - warning: branch condition never has value true - ┌─ test/coverage/coverage_macro.rs:7:12 - │ - 7 │ if !$cond { - │ ^^^^^^ - · - 18 │ check!(u8::try_from(y_32).is_ok()); - │ ---------------------------------- in this macro invocation - - - Use -p '/coverage_macro/' to rerun this test only. +OK (1.94s) -30 out of 339 tests failed (781.97s) +14 out of 339 tests failed (794.60s) From 519f321d76552b0abfc55cc5b44faa58db4cd583 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 1 Jun 2023 20:31:15 -0400 Subject: [PATCH 081/114] crux-mir-test: Sanitize disambiguators in golden test output This makes the golden tests much more resilient to changes in the underlying Rust libraries, which can affect what the disambiguator values become. --- crux-mir/crux-mir.cabal | 2 ++ crux-mir/test/Test.hs | 50 ++++++++++++++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/crux-mir/crux-mir.cabal b/crux-mir/crux-mir.cabal index 7d56e4327..3ce861fc2 100644 --- a/crux-mir/crux-mir.cabal +++ b/crux-mir/crux-mir.cabal @@ -92,6 +92,8 @@ test-suite test config-schema, config-value, bytestring, + regex-base, + regex-posix, utf8-string default-language: Haskell2010 diff --git a/crux-mir/test/Test.hs b/crux-mir/test/Test.hs index d6b4174c6..644f64347 100644 --- a/crux-mir/test/Test.hs +++ b/crux-mir/test/Test.hs @@ -1,7 +1,9 @@ {-# LANGUAGE LambdaCase #-} {-# LANGUAGE ImplicitParams #-} {-# Language OverloadedStrings #-} -{-# OPTIONS_GHC -Wall -fno-warn-unused-top-binds #-} +{-# Language ScopedTypeVariables #-} +{-# OPTIONS_GHC -Wall #-} +module Main (main) where import qualified Data.ByteString as BS import qualified Data.ByteString.UTF8 as BS8 @@ -22,6 +24,8 @@ import Test.Tasty.HUnit (Assertion, testCaseSteps, assertBool, assertF import Test.Tasty.Golden (findByExtension) import Test.Tasty.Golden.Advanced (goldenTest) import Test.Tasty.ExpectedFailure (expectFailBecause) +import Text.Regex.Base ( makeRegex, matchM ) +import Text.Regex.Posix.ByteString.Lazy ( Regex ) import qualified Mir.Language as Mir @@ -34,11 +38,6 @@ import qualified Config.Schema as Config type OracleTest = FilePath -> String -> (String -> IO ()) -> Assertion --- Don't show any debug output when testing (SAWInterface) -debugLevel :: Int -debugLevel = 0 - - -- | Check whether an input file is expected to fail based on a comment in the first line. expectedFail :: FilePath -> IO (Maybe String) expectedFail fn = @@ -130,8 +129,9 @@ symbTest dir = return $ testGroup "Output testing" [ doGoldenTest (takeBaseName rustFile) goodFile outFile $ - withFile outFile WriteMode $ \h -> - runCrux rustFile h RcmSymbolic + do withFile outFile WriteMode $ \h -> + runCrux rustFile h RcmSymbolic + sanitizeGoldenOutputFile outFile | rustFile <- rustFiles -- Skip hidden files, such as editor swap files , not $ "." `isPrefixOf` takeFileName rustFile @@ -173,6 +173,40 @@ doGoldenTest rustFile goodFile outFile act = goldenTest (takeBaseName rustFile) outFile ++ " contains:\n" ++ BS8.toString out) (\out -> BS.writeFile goodFile out) +-- | Sanitize the golden output in a file. +sanitizeGoldenOutputFile :: FilePath -> IO () +sanitizeGoldenOutputFile file = do + out <- BS.readFile file + BS.writeFile file $ sanitizeGoldenTestOutput out + +-- | Replace occurrences of crate disambiguators (which are highly sensitive to +-- the contents of crates' source code) with more stable output to reduce churn +-- in the golden tests. +sanitizeGoldenTestOutput :: BS.ByteString -> BS.ByteString +sanitizeGoldenTestOutput = go + where + go :: BS.ByteString -> BS.ByteString + go out + | Just (before, _matched :: BS.ByteString, after, [preDisamb]) + <- matchM disambRE out + = before <> preDisamb <> "::" <> go after + | otherwise + = out + + disambRE :: Regex + disambRE = makeRegex disambBS + + -- A disambiguator looks something like foo/a1b4j89a, i.e., an alphanumeric + -- string that starts with an alphabetic character, followed by a forward + -- slash, followed by eight alphanumeric characters. To reduce the rate of + -- false positives, we also match two trailing colons, which would be + -- present when printing out a full DefId (e.g., foo/a1b4j89a::Bar::Baz). + disambBS :: BS.ByteString + disambBS = "([A-Za-z_]" <> alphaNum <> "*/)" <> mconcat (replicate 8 alphaNum) <> "::" + + alphaNum :: BS.ByteString + alphaNum = "[A-Za-z0-9_]" + main :: IO () main = defaultMain =<< suite From afbf93998c3b6a2d1c450f13b404e7ddf8a3b752 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 1 Jun 2023 20:31:59 -0400 Subject: [PATCH 082/114] crux-mir-test: Update golden test output --- .../test/symb_eval/alloc/out_of_bounds.good | 8 +++---- .../test/symb_eval/alloc/uninit_read.good | 8 +++---- crux-mir/test/symb_eval/alloc/valid_read.good | 2 +- .../test/symb_eval/alloc/zero_length.good | 2 +- crux-mir/test/symb_eval/any/conditional.good | 2 +- crux-mir/test/symb_eval/any/downcast.good | 2 +- .../test/symb_eval/any/downcast_fail.good | 6 ++--- crux-mir/test/symb_eval/array/basic.good | 2 +- crux-mir/test/symb_eval/array/mux_slice.good | 2 +- crux-mir/test/symb_eval/array/slice.good | 2 +- crux-mir/test/symb_eval/array/slice_mut.good | 2 +- crux-mir/test/symb_eval/bitvector/arith.good | 2 +- crux-mir/test/symb_eval/bitvector/cmp.good | 2 +- .../test/symb_eval/bitvector/from_to.good | 2 +- .../symb_eval/bitvector/leading_zeros.good | 2 +- .../test/symb_eval/bitvector/literals.good | 2 +- .../symb_eval/bitvector/overflowing_sub.good | 2 +- .../test/symb_eval/bitvector/symbolic.good | 2 +- .../test/symb_eval/bytes/extend_bytes.good | 2 +- crux-mir/test/symb_eval/bytes/new.good | 2 +- crux-mir/test/symb_eval/bytes/put.good | 2 +- .../test/symb_eval/bytes/put_overflow.good | 8 +++---- crux-mir/test/symb_eval/bytes/split_off.good | 2 +- crux-mir/test/symb_eval/bytes/split_to.good | 2 +- crux-mir/test/symb_eval/bytes/sym_len.good | 2 +- crux-mir/test/symb_eval/concretize/array.good | 2 +- .../test/symb_eval/concretize/assert_ok.good | 2 +- crux-mir/test/symb_eval/concretize/conc.good | 2 +- .../test/symb_eval/concretize/no_conc.good | 2 +- crux-mir/test/symb_eval/crux/early_fail.good | 14 ++++++------ crux-mir/test/symb_eval/crux/fail_return.good | 16 +++++++------- crux-mir/test/symb_eval/crux/mixed_fail.good | 20 ++++++++--------- crux-mir/test/symb_eval/crux/multi.good | 22 +++++++++---------- crux-mir/test/symb_eval/crypto/bytes.good | 14 ++++++------ crux-mir/test/symb_eval/crypto/bytes2.good | 2 +- crux-mir/test/symb_eval/crypto/double.good | 2 +- crux-mir/test/symb_eval/crypto/ffs.good | 2 +- crux-mir/test/symb_eval/enum/mux.good | 2 +- crux-mir/test/symb_eval/fnptr/mux.good | 2 +- .../test/symb_eval/io/vec_cursor_read.good | 2 +- crux-mir/test/symb_eval/io/vec_write.good | 2 +- crux-mir/test/symb_eval/mux/array.good | 2 +- crux-mir/test/symb_eval/mux/array_mut.good | 2 +- crux-mir/test/symb_eval/num/checked_add.good | 6 ++--- crux-mir/test/symb_eval/num/checked_div.good | 10 ++++----- crux-mir/test/symb_eval/num/checked_mul.good | 6 ++--- .../symb_eval/num/checked_mul_signed.good | 6 ++--- .../test/symb_eval/overrides/bad_symb1.good | 6 ++--- .../test/symb_eval/overrides/bad_symb2.good | 6 ++--- .../test/symb_eval/overrides/override1.good | 2 +- .../test/symb_eval/overrides/override2.good | 6 ++--- .../test/symb_eval/overrides/override3.good | 2 +- .../test/symb_eval/overrides/override4.good | 2 +- .../test/symb_eval/overrides/override5.good | 6 ++--- .../symb_eval/overrides/override_rust.good | 2 +- .../test/symb_eval/refs/mux_init_imm.good | 2 +- .../test/symb_eval/refs/mux_init_mut.good | 2 +- crux-mir/test/symb_eval/scalar/test1.good | 2 +- .../test/symb_eval/sym_bytes/construct.good | 6 ++--- .../test/symb_eval/sym_bytes/deserialize.good | 6 ++--- crux-mir/test/symb_eval/vec/clone.good | 2 +- crux-mir/test/symb_eval/vec/into_iter.good | 2 +- crux-mir/test/symb_eval/vec/macro.good | 2 +- crux-mir/test/symb_eval/vec/sort_by_key.good | 2 +- .../test/symb_eval/vector/as_mut_slice.good | 2 +- crux-mir/test/symb_eval/vector/as_slice.good | 2 +- crux-mir/test/symb_eval/vector/concat.good | 2 +- .../symb_eval/vector/copy_from_slice.good | 2 +- crux-mir/test/symb_eval/vector/mut.good | 2 +- crux-mir/test/symb_eval/vector/new.good | 2 +- crux-mir/test/symb_eval/vector/push.good | 2 +- crux-mir/test/symb_eval/vector/replicate.good | 2 +- crux-mir/test/symb_eval/vector/split_at.good | 2 +- 73 files changed, 144 insertions(+), 144 deletions(-) diff --git a/crux-mir/test/symb_eval/alloc/out_of_bounds.good b/crux-mir/test/symb_eval/alloc/out_of_bounds.good index 17df011f0..83291b406 100644 --- a/crux-mir/test/symb_eval/alloc/out_of_bounds.good +++ b/crux-mir/test/symb_eval/alloc/out_of_bounds.good @@ -1,13 +1,13 @@ -test out_of_bounds/2674d8f8::crux_test[0]: FAILED +test out_of_bounds/::crux_test[0]: FAILED failures: ----- out_of_bounds/2674d8f8::crux_test[0] counterexamples ---- +---- out_of_bounds/::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/2674d8f8::crux_test[0] +[Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/::crux_test[0] [Crux] vector index out of range: the length is 10 but the index is BV 12 [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/2674d8f8::crux_test[0] +[Crux] test/symb_eval/alloc/out_of_bounds.rs:12:9: 12:24: error: in out_of_bounds/::crux_test[0] [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/alloc/uninit_read.good b/crux-mir/test/symb_eval/alloc/uninit_read.good index 1813c0935..1654dcc7f 100644 --- a/crux-mir/test/symb_eval/alloc/uninit_read.good +++ b/crux-mir/test/symb_eval/alloc/uninit_read.good @@ -1,13 +1,13 @@ -test uninit_read/2bff4d2d::crux_test[0]: FAILED +test uninit_read/::crux_test[0]: FAILED failures: ----- uninit_read/2bff4d2d::crux_test[0] counterexamples ---- +---- uninit_read/::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/2bff4d2d::crux_test[0] +[Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/::crux_test[0] [Crux] Attempted to read uninitialized vector index [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/2bff4d2d::crux_test[0] +[Crux] test/symb_eval/alloc/uninit_read.rs:9:9: 9:23: error: in uninit_read/::crux_test[0] [Crux] attempted to read empty mux tree [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/alloc/valid_read.good b/crux-mir/test/symb_eval/alloc/valid_read.good index c85bdcfab..f7e7f3ee6 100644 --- a/crux-mir/test/symb_eval/alloc/valid_read.good +++ b/crux-mir/test/symb_eval/alloc/valid_read.good @@ -1,3 +1,3 @@ -test valid_read/3ad5949c::crux_test[0]: returned 45, ok +test valid_read/::crux_test[0]: returned 45, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/alloc/zero_length.good b/crux-mir/test/symb_eval/alloc/zero_length.good index 58a24d9ad..57035fcd8 100644 --- a/crux-mir/test/symb_eval/alloc/zero_length.good +++ b/crux-mir/test/symb_eval/alloc/zero_length.good @@ -1,3 +1,3 @@ -test zero_length/251728a4::crux_test[0]: ok +test zero_length/::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/any/conditional.good b/crux-mir/test/symb_eval/any/conditional.good index 039679893..56dc9b108 100644 --- a/crux-mir/test/symb_eval/any/conditional.good +++ b/crux-mir/test/symb_eval/any/conditional.good @@ -1,3 +1,3 @@ -test conditional/6c557b21::crux_test[0]: ok +test conditional/::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/any/downcast.good b/crux-mir/test/symb_eval/any/downcast.good index 0216b4f45..7582601c8 100644 --- a/crux-mir/test/symb_eval/any/downcast.good +++ b/crux-mir/test/symb_eval/any/downcast.good @@ -1,3 +1,3 @@ -test downcast/a79fa030::crux_test[0]: returned 1, ok +test downcast/::crux_test[0]: returned 1, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/any/downcast_fail.good b/crux-mir/test/symb_eval/any/downcast_fail.good index c4dc035ad..8d1c1488d 100644 --- a/crux-mir/test/symb_eval/any/downcast_fail.good +++ b/crux-mir/test/symb_eval/any/downcast_fail.good @@ -1,10 +1,10 @@ -test downcast_fail/0c70442e::crux_test[0]: FAILED +test downcast_fail/::crux_test[0]: FAILED failures: ----- downcast_fail/0c70442e::crux_test[0] counterexamples ---- +---- downcast_fail/::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/any/downcast_fail.rs:10:14: 10:15: error: in downcast_fail/0c70442e::crux_test[0] +[Crux] test/symb_eval/any/downcast_fail.rs:10:14: 10:15: error: in downcast_fail/::crux_test[0] [Crux] failed to downcast Any as BVRepr 32 [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/array/basic.good b/crux-mir/test/symb_eval/array/basic.good index cd61dc7e1..31c8bad63 100644 --- a/crux-mir/test/symb_eval/array/basic.good +++ b/crux-mir/test/symb_eval/array/basic.good @@ -1,3 +1,3 @@ -test basic/b603207c::crux_test[0]: returned 3, ok +test basic/::crux_test[0]: returned 3, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/array/mux_slice.good b/crux-mir/test/symb_eval/array/mux_slice.good index 4f7bb7bbb..275efd1d8 100644 --- a/crux-mir/test/symb_eval/array/mux_slice.good +++ b/crux-mir/test/symb_eval/array/mux_slice.good @@ -1,3 +1,3 @@ -test mux_slice/93b6889e::crux_test[0]: returned Symbolic BV, ok +test mux_slice/::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/array/slice.good b/crux-mir/test/symb_eval/array/slice.good index 90588b935..5a4a40f36 100644 --- a/crux-mir/test/symb_eval/array/slice.good +++ b/crux-mir/test/symb_eval/array/slice.good @@ -1,3 +1,3 @@ -test slice/04ec98ca::crux_test[0]: returned 5, ok +test slice/::crux_test[0]: returned 5, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/array/slice_mut.good b/crux-mir/test/symb_eval/array/slice_mut.good index a4f7cd4d2..6771b9fec 100644 --- a/crux-mir/test/symb_eval/array/slice_mut.good +++ b/crux-mir/test/symb_eval/array/slice_mut.good @@ -1,3 +1,3 @@ -test slice_mut/5c718432::crux_test[0]: returned 5, ok +test slice_mut/::crux_test[0]: returned 5, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/arith.good b/crux-mir/test/symb_eval/bitvector/arith.good index 14d6dfa48..c336ba370 100644 --- a/crux-mir/test/symb_eval/bitvector/arith.good +++ b/crux-mir/test/symb_eval/bitvector/arith.good @@ -1,3 +1,3 @@ -test arith/2d4acabe::crux_test[0]: ok +test arith/::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/cmp.good b/crux-mir/test/symb_eval/bitvector/cmp.good index c640fff18..680a7c6ae 100644 --- a/crux-mir/test/symb_eval/bitvector/cmp.good +++ b/crux-mir/test/symb_eval/bitvector/cmp.good @@ -1,3 +1,3 @@ -test cmp/e92927a6::crux_test[0]: ok +test cmp/::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/from_to.good b/crux-mir/test/symb_eval/bitvector/from_to.good index 58973ead8..91c9c0dcf 100644 --- a/crux-mir/test/symb_eval/bitvector/from_to.good +++ b/crux-mir/test/symb_eval/bitvector/from_to.good @@ -1,3 +1,3 @@ -test from_to/0179ef0b::crux_test[0]: ok +test from_to/::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/leading_zeros.good b/crux-mir/test/symb_eval/bitvector/leading_zeros.good index c0462400e..ddec072fd 100644 --- a/crux-mir/test/symb_eval/bitvector/leading_zeros.good +++ b/crux-mir/test/symb_eval/bitvector/leading_zeros.good @@ -1,3 +1,3 @@ -test leading_zeros/26d4da72::crux_test[0]: ok +test leading_zeros/::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/literals.good b/crux-mir/test/symb_eval/bitvector/literals.good index 568a3a2ae..2bce0e525 100644 --- a/crux-mir/test/symb_eval/bitvector/literals.good +++ b/crux-mir/test/symb_eval/bitvector/literals.good @@ -1,3 +1,3 @@ -test literals/5d523333::crux_test[0]: ok +test literals/::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/overflowing_sub.good b/crux-mir/test/symb_eval/bitvector/overflowing_sub.good index a12d68b8c..2c0dd7b27 100644 --- a/crux-mir/test/symb_eval/bitvector/overflowing_sub.good +++ b/crux-mir/test/symb_eval/bitvector/overflowing_sub.good @@ -1,3 +1,3 @@ -test overflowing_sub/3e1181ed::crux_test[0]: ok +test overflowing_sub/::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bitvector/symbolic.good b/crux-mir/test/symb_eval/bitvector/symbolic.good index 7d6ed9301..f7216a7f7 100644 --- a/crux-mir/test/symb_eval/bitvector/symbolic.good +++ b/crux-mir/test/symb_eval/bitvector/symbolic.good @@ -1,3 +1,3 @@ -test symbolic/ab685979::crux_test[0]: ok +test symbolic/::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/extend_bytes.good b/crux-mir/test/symb_eval/bytes/extend_bytes.good index 2d76adbf3..6bf46db6d 100644 --- a/crux-mir/test/symb_eval/bytes/extend_bytes.good +++ b/crux-mir/test/symb_eval/bytes/extend_bytes.good @@ -1,3 +1,3 @@ -test extend_bytes/87b45525::f[0]: ok +test extend_bytes/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/new.good b/crux-mir/test/symb_eval/bytes/new.good index 37d4827e1..32ebe4def 100644 --- a/crux-mir/test/symb_eval/bytes/new.good +++ b/crux-mir/test/symb_eval/bytes/new.good @@ -1,3 +1,3 @@ -test new/3e06dc75::f[0]: ok +test new/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/put.good b/crux-mir/test/symb_eval/bytes/put.good index 4c1c47e3f..99f1392f7 100644 --- a/crux-mir/test/symb_eval/bytes/put.good +++ b/crux-mir/test/symb_eval/bytes/put.good @@ -1,3 +1,3 @@ -test put/b1ab63fb::f[0]: ok +test put/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/put_overflow.good b/crux-mir/test/symb_eval/bytes/put_overflow.good index 9ba04a77e..1c430f8cd 100644 --- a/crux-mir/test/symb_eval/bytes/put_overflow.good +++ b/crux-mir/test/symb_eval/bytes/put_overflow.good @@ -1,10 +1,10 @@ -test put_overflow/df566fcd::f[0]: FAILED +test put_overflow/::f[0]: FAILED failures: ----- put_overflow/df566fcd::f[0] counterexamples ---- +---- put_overflow/::f[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] internal: error: in bytes/dba3271f::{impl#4}[0]::put_slice[0] -[Crux] panicking::panic_fmt, called from bytes/dba3271f::{impl#4}[0]::put_slice[0] +[Crux] internal: error: in bytes/::{impl#4}[0]::put_slice[0] +[Crux] panicking::panic_fmt, called from bytes/::{impl#4}[0]::put_slice[0] [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/bytes/split_off.good b/crux-mir/test/symb_eval/bytes/split_off.good index 4b8a0522a..a0c086e7e 100644 --- a/crux-mir/test/symb_eval/bytes/split_off.good +++ b/crux-mir/test/symb_eval/bytes/split_off.good @@ -1,3 +1,3 @@ -test split_off/50a75a9e::f[0]: ok +test split_off/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/split_to.good b/crux-mir/test/symb_eval/bytes/split_to.good index 4ea7fdeb2..160415e0d 100644 --- a/crux-mir/test/symb_eval/bytes/split_to.good +++ b/crux-mir/test/symb_eval/bytes/split_to.good @@ -1,3 +1,3 @@ -test split_to/b8cd8da6::f[0]: ok +test split_to/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/bytes/sym_len.good b/crux-mir/test/symb_eval/bytes/sym_len.good index 6a7d88b4d..da2887b42 100644 --- a/crux-mir/test/symb_eval/bytes/sym_len.good +++ b/crux-mir/test/symb_eval/bytes/sym_len.good @@ -1,3 +1,3 @@ -test sym_len/c7960ac7::f[0]: ok +test sym_len/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/concretize/array.good b/crux-mir/test/symb_eval/concretize/array.good index 45e78956d..e632b4173 100644 --- a/crux-mir/test/symb_eval/concretize/array.good +++ b/crux-mir/test/symb_eval/concretize/array.good @@ -1,3 +1,3 @@ -test array/a41ef166::crux_test[0]: returned (1, 2, 3), ok +test array/::crux_test[0]: returned (1, 2, 3), ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/concretize/assert_ok.good b/crux-mir/test/symb_eval/concretize/assert_ok.good index 4db821c59..6382f97d3 100644 --- a/crux-mir/test/symb_eval/concretize/assert_ok.good +++ b/crux-mir/test/symb_eval/concretize/assert_ok.good @@ -1,3 +1,3 @@ -test assert_ok/dec2bc1d::crux_test[0]: returned 1, ok +test assert_ok/::crux_test[0]: returned 1, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/concretize/conc.good b/crux-mir/test/symb_eval/concretize/conc.good index 1944b12c6..f002b45ff 100644 --- a/crux-mir/test/symb_eval/concretize/conc.good +++ b/crux-mir/test/symb_eval/concretize/conc.good @@ -1,3 +1,3 @@ -test conc/8f6163ae::crux_test[0]: returned (1, 0), ok +test conc/::crux_test[0]: returned (1, 0), ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/concretize/no_conc.good b/crux-mir/test/symb_eval/concretize/no_conc.good index 18906e1d6..7525c61cc 100644 --- a/crux-mir/test/symb_eval/concretize/no_conc.good +++ b/crux-mir/test/symb_eval/concretize/no_conc.good @@ -1,3 +1,3 @@ -test no_conc/ab98a20c::crux_test[0]: returned (Symbolic BV, Symbolic BV), ok +test no_conc/::crux_test[0]: returned (Symbolic BV, Symbolic BV), ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/crux/early_fail.good b/crux-mir/test/symb_eval/crux/early_fail.good index e923d5233..8e0412676 100644 --- a/crux-mir/test/symb_eval/crux/early_fail.good +++ b/crux-mir/test/symb_eval/crux/early_fail.good @@ -1,16 +1,16 @@ -test early_fail/9ed3a747::fail1[0]: FAILED -test early_fail/9ed3a747::fail2[0]: FAILED +test early_fail/::fail1[0]: FAILED +test early_fail/::fail2[0]: FAILED failures: ----- early_fail/9ed3a747::fail1[0] counterexamples ---- +---- early_fail/::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] internal: error: in early_fail/9ed3a747::fail1[0] -[Crux] panicking::panic_fmt, called from early_fail/9ed3a747::fail1[0] +[Crux] internal: error: in early_fail/::fail1[0] +[Crux] panicking::panic_fmt, called from early_fail/::fail1[0] ----- early_fail/9ed3a747::fail2[0] counterexamples ---- +---- early_fail/::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/early_fail.rs:17:5: 17:29: error: in early_fail/9ed3a747::fail2[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/early_fail.rs:17:5: 17:29: error: in early_fail/::fail2[0] [Crux] MIR assertion at test/symb_eval/crux/early_fail.rs:17:5: [Crux] x == 0 diff --git a/crux-mir/test/symb_eval/crux/fail_return.good b/crux-mir/test/symb_eval/crux/fail_return.good index bdeb641c6..a1f5c23c9 100644 --- a/crux-mir/test/symb_eval/crux/fail_return.good +++ b/crux-mir/test/symb_eval/crux/fail_return.good @@ -1,23 +1,23 @@ -test fail_return/29cbbbb6::fail1[0]: returned Symbolic BV, FAILED -test fail_return/29cbbbb6::fail2[0]: returned 123, FAILED +test fail_return/::fail1[0]: returned Symbolic BV, FAILED +test fail_return/::fail2[0]: returned 123, FAILED failures: ----- fail_return/29cbbbb6::fail1[0] counterexamples ---- +---- fail_return/::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/crux/fail_return.rs:8:22: 8:27: error: in fail_return/29cbbbb6::fail1[0] +[Crux] test/symb_eval/crux/fail_return.rs:8:22: 8:27: error: in fail_return/::fail1[0] [Crux] attempt to compute `move _4 + const 1_u8`, which would overflow [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/fail_return.rs:8:5: 8:32: error: in fail_return/29cbbbb6::fail1[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/fail_return.rs:8:5: 8:32: error: in fail_return/::fail1[0] [Crux] MIR assertion at test/symb_eval/crux/fail_return.rs:8:5: [Crux] x + 1 > x ----- fail_return/29cbbbb6::fail2[0] counterexamples ---- +---- fail_return/::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/crux/fail_return.rs:15:22: 15:27: error: in fail_return/29cbbbb6::fail2[0] +[Crux] test/symb_eval/crux/fail_return.rs:15:22: 15:27: error: in fail_return/::fail2[0] [Crux] attempt to compute `move _5 + const 1_u8`, which would overflow [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/fail_return.rs:15:5: 15:32: error: in fail_return/29cbbbb6::fail2[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/fail_return.rs:15:5: 15:32: error: in fail_return/::fail2[0] [Crux] MIR assertion at test/symb_eval/crux/fail_return.rs:15:5: [Crux] x + 1 > x diff --git a/crux-mir/test/symb_eval/crux/mixed_fail.good b/crux-mir/test/symb_eval/crux/mixed_fail.good index 03c53fd4a..b9b8890dd 100644 --- a/crux-mir/test/symb_eval/crux/mixed_fail.good +++ b/crux-mir/test/symb_eval/crux/mixed_fail.good @@ -1,25 +1,25 @@ -test mixed_fail/8469a3ad::fail1[0]: FAILED -test mixed_fail/8469a3ad::fail2[0]: FAILED -test mixed_fail/8469a3ad::pass1[0]: ok -test mixed_fail/8469a3ad::pass2[0]: ok +test mixed_fail/::fail1[0]: FAILED +test mixed_fail/::fail2[0]: FAILED +test mixed_fail/::pass1[0]: ok +test mixed_fail/::pass2[0]: ok failures: ----- mixed_fail/8469a3ad::fail1[0] counterexamples ---- +---- mixed_fail/::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/crux/mixed_fail.rs:8:22: 8:27: error: in mixed_fail/8469a3ad::fail1[0] +[Crux] test/symb_eval/crux/mixed_fail.rs:8:22: 8:27: error: in mixed_fail/::fail1[0] [Crux] attempt to compute `move _5 + const 1_u8`, which would overflow [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/mixed_fail.rs:8:5: 8:32: error: in mixed_fail/8469a3ad::fail1[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/mixed_fail.rs:8:5: 8:32: error: in mixed_fail/::fail1[0] [Crux] MIR assertion at test/symb_eval/crux/mixed_fail.rs:8:5: [Crux] x + 1 > x ----- mixed_fail/8469a3ad::fail2[0] counterexamples ---- +---- mixed_fail/::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/crux/mixed_fail.rs:14:22: 14:27: error: in mixed_fail/8469a3ad::fail2[0] +[Crux] test/symb_eval/crux/mixed_fail.rs:14:22: 14:27: error: in mixed_fail/::fail2[0] [Crux] attempt to compute `move _5 + const 2_u8`, which would overflow [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/mixed_fail.rs:14:5: 14:32: error: in mixed_fail/8469a3ad::fail2[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/mixed_fail.rs:14:5: 14:32: error: in mixed_fail/::fail2[0] [Crux] MIR assertion at test/symb_eval/crux/mixed_fail.rs:14:5: [Crux] x + 2 > x diff --git a/crux-mir/test/symb_eval/crux/multi.good b/crux-mir/test/symb_eval/crux/multi.good index 14e9c957a..cd45e9679 100644 --- a/crux-mir/test/symb_eval/crux/multi.good +++ b/crux-mir/test/symb_eval/crux/multi.good @@ -1,26 +1,26 @@ -test multi/0fbcaa4a::fail1[0]: FAILED -test multi/0fbcaa4a::fail2[0]: FAILED -test multi/0fbcaa4a::fail3[0]: FAILED +test multi/::fail1[0]: FAILED +test multi/::fail2[0]: FAILED +test multi/::fail3[0]: FAILED failures: ----- multi/0fbcaa4a::fail1[0] counterexamples ---- +---- multi/::fail1[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/crux/multi.rs:8:22: 8:27: error: in multi/0fbcaa4a::fail1[0] +[Crux] test/symb_eval/crux/multi.rs:8:22: 8:27: error: in multi/::fail1[0] [Crux] attempt to compute `move _5 + const 1_u8`, which would overflow [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/multi.rs:8:5: 8:32: error: in multi/0fbcaa4a::fail1[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/multi.rs:8:5: 8:32: error: in multi/::fail1[0] [Crux] MIR assertion at test/symb_eval/crux/multi.rs:8:5: [Crux] x + 1 > x ----- multi/0fbcaa4a::fail2[0] counterexamples ---- +---- multi/::fail2[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] internal: error: in multi/0fbcaa4a::fail2[0] -[Crux] panicking::panic_fmt, called from multi/0fbcaa4a::fail2[0] +[Crux] internal: error: in multi/::fail2[0] +[Crux] panicking::panic_fmt, called from multi/::fail2[0] ----- multi/0fbcaa4a::fail3[0] counterexamples ---- +---- multi/::fail3[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/multi.rs:20:5: 20:29: error: in multi/0fbcaa4a::assert_zero[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crux/multi.rs:20:5: 20:29: error: in multi/::assert_zero[0] [Crux] MIR assertion at test/symb_eval/crux/multi.rs:20:5: [Crux] x == 0 diff --git a/crux-mir/test/symb_eval/crypto/bytes.good b/crux-mir/test/symb_eval/crypto/bytes.good index 3f774cfbe..90c3d5dc6 100644 --- a/crux-mir/test/symb_eval/crypto/bytes.good +++ b/crux-mir/test/symb_eval/crypto/bytes.good @@ -1,26 +1,26 @@ -test bytes/3f231946::f[0]: FAILED +test bytes/::f[0]: FAILED failures: ----- bytes/3f231946::f[0] counterexamples ---- +---- bytes/::f[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/3f231946::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/::f[0] [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: [Crux] a[i] == b[i] [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/3f231946::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/::f[0] [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: [Crux] a[i] == b[i] [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/3f231946::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/::f[0] [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: [Crux] a[i] == b[i] [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/3f231946::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/::f[0] [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: [Crux] a[i] == b[i] [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/3f231946::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/crypto/bytes.rs:85:7: 85:37: error: in bytes/::f[0] [Crux] MIR assertion at test/symb_eval/crypto/bytes.rs:85:7: [Crux] a[i] == b[i] diff --git a/crux-mir/test/symb_eval/crypto/bytes2.good b/crux-mir/test/symb_eval/crypto/bytes2.good index 57f37573d..52964c94c 100644 --- a/crux-mir/test/symb_eval/crypto/bytes2.good +++ b/crux-mir/test/symb_eval/crypto/bytes2.good @@ -1,3 +1,3 @@ -test bytes2/930baf50::f[0]: ok +test bytes2/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/crypto/double.good b/crux-mir/test/symb_eval/crypto/double.good index 48b3b135d..924c4d0eb 100644 --- a/crux-mir/test/symb_eval/crypto/double.good +++ b/crux-mir/test/symb_eval/crypto/double.good @@ -1,3 +1,3 @@ -test double/b109b0aa::f[0]: ok +test double/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/crypto/ffs.good b/crux-mir/test/symb_eval/crypto/ffs.good index cc6b7fe63..fd44c0aba 100644 --- a/crux-mir/test/symb_eval/crypto/ffs.good +++ b/crux-mir/test/symb_eval/crypto/ffs.good @@ -1,3 +1,3 @@ -test ffs/8450a4c1::f[0]: ok +test ffs/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/enum/mux.good b/crux-mir/test/symb_eval/enum/mux.good index 697132333..4571b8891 100644 --- a/crux-mir/test/symb_eval/enum/mux.good +++ b/crux-mir/test/symb_eval/enum/mux.good @@ -1,3 +1,3 @@ -test mux/72a0da7e::test[0]: ok +test mux/::test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/fnptr/mux.good b/crux-mir/test/symb_eval/fnptr/mux.good index 5a28542c1..4601d065c 100644 --- a/crux-mir/test/symb_eval/fnptr/mux.good +++ b/crux-mir/test/symb_eval/fnptr/mux.good @@ -1,3 +1,3 @@ -test mux/f7f94a9c::crux_test[0]: returned 2, ok +test mux/::crux_test[0]: returned 2, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/io/vec_cursor_read.good b/crux-mir/test/symb_eval/io/vec_cursor_read.good index 0e032b8f2..95d2c9d18 100644 --- a/crux-mir/test/symb_eval/io/vec_cursor_read.good +++ b/crux-mir/test/symb_eval/io/vec_cursor_read.good @@ -1,3 +1,3 @@ -test vec_cursor_read/3c45b63f::f[0]: ok +test vec_cursor_read/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/io/vec_write.good b/crux-mir/test/symb_eval/io/vec_write.good index c735c5aa4..2b8c02e11 100644 --- a/crux-mir/test/symb_eval/io/vec_write.good +++ b/crux-mir/test/symb_eval/io/vec_write.good @@ -1,3 +1,3 @@ -test vec_write/6974c82c::f[0]: ok +test vec_write/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/mux/array.good b/crux-mir/test/symb_eval/mux/array.good index 512e9e2a6..17f5f4b37 100644 --- a/crux-mir/test/symb_eval/mux/array.good +++ b/crux-mir/test/symb_eval/mux/array.good @@ -1,3 +1,3 @@ -test array/e80be074::crux_test[0]: returned Symbolic BV, ok +test array/::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/mux/array_mut.good b/crux-mir/test/symb_eval/mux/array_mut.good index 8c45ab5e9..8d8616b47 100644 --- a/crux-mir/test/symb_eval/mux/array_mut.good +++ b/crux-mir/test/symb_eval/mux/array_mut.good @@ -1,3 +1,3 @@ -test array_mut/1330af4a::crux_test[0]: returned Symbolic BV, ok +test array_mut/::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/num/checked_add.good b/crux-mir/test/symb_eval/num/checked_add.good index 3d1804215..61a115f4f 100644 --- a/crux-mir/test/symb_eval/num/checked_add.good +++ b/crux-mir/test/symb_eval/num/checked_add.good @@ -1,10 +1,10 @@ -test checked_add/65d1ac2f::crux_test[0]: returned 44, FAILED +test checked_add/::crux_test[0]: returned 44, FAILED failures: ----- checked_add/65d1ac2f::crux_test[0] counterexamples ---- +---- checked_add/::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_add.rs:6:5: 6:12: error: in checked_add/65d1ac2f::crux_test[0] +[Crux] test/symb_eval/num/checked_add.rs:6:5: 6:12: error: in checked_add/::crux_test[0] [Crux] attempt to compute `const 100_u8 + move _2`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/num/checked_div.good b/crux-mir/test/symb_eval/num/checked_div.good index e12a6282d..b13ca362d 100644 --- a/crux-mir/test/symb_eval/num/checked_div.good +++ b/crux-mir/test/symb_eval/num/checked_div.good @@ -1,16 +1,16 @@ -test checked_div/03613482::crux_test[0]: returned 65, FAILED +test checked_div/::crux_test[0]: returned 65, FAILED failures: ----- checked_div/03613482::crux_test[0] counterexamples ---- +---- checked_div/::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/03613482::div_signed[0] +[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/::div_signed[0] [Crux] attempt to compute `_3 / _4`, which would overflow [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/03613482::div_signed[0] +[Crux] test/symb_eval/num/checked_div.rs:3:5: 3:10: error: in checked_div/::div_signed[0] [Crux] attempt to divide `_3` by zero [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/03613482::div_unsigned[0] +[Crux] test/symb_eval/num/checked_div.rs:8:5: 8:10: error: in checked_div/::div_unsigned[0] [Crux] attempt to divide `_3` by zero [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/num/checked_mul.good b/crux-mir/test/symb_eval/num/checked_mul.good index 7940a75a0..681564e22 100644 --- a/crux-mir/test/symb_eval/num/checked_mul.good +++ b/crux-mir/test/symb_eval/num/checked_mul.good @@ -1,10 +1,10 @@ -test checked_mul/849ca86a::crux_test[0]: returned 44, FAILED +test checked_mul/::crux_test[0]: returned 44, FAILED failures: ----- checked_mul/849ca86a::crux_test[0] counterexamples ---- +---- checked_mul/::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_mul.rs:6:5: 6:12: error: in checked_mul/849ca86a::crux_test[0] +[Crux] test/symb_eval/num/checked_mul.rs:6:5: 6:12: error: in checked_mul/::crux_test[0] [Crux] attempt to compute `const 100_u8 * move _2`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/num/checked_mul_signed.good b/crux-mir/test/symb_eval/num/checked_mul_signed.good index 579039c26..1d0181d28 100644 --- a/crux-mir/test/symb_eval/num/checked_mul_signed.good +++ b/crux-mir/test/symb_eval/num/checked_mul_signed.good @@ -1,10 +1,10 @@ -test checked_mul_signed/0175bcf1::crux_test[0]: returned 44, FAILED +test checked_mul_signed/::crux_test[0]: returned 44, FAILED failures: ----- checked_mul_signed/0175bcf1::crux_test[0] counterexamples ---- +---- checked_mul_signed/::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/num/checked_mul_signed.rs:6:5: 6:13: error: in checked_mul_signed/0175bcf1::crux_test[0] +[Crux] test/symb_eval/num/checked_mul_signed.rs:6:5: 6:13: error: in checked_mul_signed/::crux_test[0] [Crux] attempt to compute `const -100_i8 * move _2`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/overrides/bad_symb1.good b/crux-mir/test/symb_eval/overrides/bad_symb1.good index 0101f31bb..14454566a 100644 --- a/crux-mir/test/symb_eval/overrides/bad_symb1.good +++ b/crux-mir/test/symb_eval/overrides/bad_symb1.good @@ -1,10 +1,10 @@ -test bad_symb1/9b84185b::crux_test[0]: FAILED +test bad_symb1/::crux_test[0]: FAILED failures: ----- bad_symb1/9b84185b::crux_test[0] counterexamples ---- +---- bad_symb1/::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] lib/crucible/symbolic.rs:24:64: 24:68 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/c130157c::symbolic[0]::{impl#1}[0]::symbolic[0] +[Crux] lib/crucible/symbolic.rs:24:64: 24:68 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/::symbolic[0]::{impl#1}[0]::symbolic[0] [Crux] symbolic variable name must be a concrete string [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/overrides/bad_symb2.good b/crux-mir/test/symb_eval/overrides/bad_symb2.good index 2d4adf3da..c51777b6e 100644 --- a/crux-mir/test/symb_eval/overrides/bad_symb2.good +++ b/crux-mir/test/symb_eval/overrides/bad_symb2.good @@ -1,10 +1,10 @@ -test bad_symb2/67e9f90f::crux_test[0]: FAILED +test bad_symb2/::crux_test[0]: FAILED failures: ----- bad_symb2/67e9f90f::crux_test[0] counterexamples ---- +---- bad_symb2/::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] lib/crucible/symbolic.rs:24:64: 24:68 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/c130157c::symbolic[0]::{impl#1}[0]::symbolic[0] +[Crux] lib/crucible/symbolic.rs:24:64: 24:68 !lib/crucible/symbolic.rs:30:1: 36:2: error: in crucible/::symbolic[0]::{impl#1}[0]::symbolic[0] [Crux] invalid symbolic variable name "\NUL:., /": Identifier must start with a letter. [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/overrides/override1.good b/crux-mir/test/symb_eval/overrides/override1.good index 7c4a56ccd..ccdb3b071 100644 --- a/crux-mir/test/symb_eval/overrides/override1.good +++ b/crux-mir/test/symb_eval/overrides/override1.good @@ -1,4 +1,4 @@ -test override1/5f575d95::f[0]: Hello, I'm an override +test override1/::f[0]: Hello, I'm an override returned 2, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/overrides/override2.good b/crux-mir/test/symb_eval/overrides/override2.good index fe66852a0..08ca7db6e 100644 --- a/crux-mir/test/symb_eval/overrides/override2.good +++ b/crux-mir/test/symb_eval/overrides/override2.good @@ -1,10 +1,10 @@ -test override2/980aac96::f[0]: FAILED +test override2/::f[0]: FAILED failures: ----- override2/980aac96::f[0] counterexamples ---- +---- override2/::f[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/overrides/override2.rs:9:5: 9:49: error: in override2/980aac96::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/overrides/override2.rs:9:5: 9:49: error: in override2/::f[0] [Crux] MIR assertion at test/symb_eval/overrides/override2.rs:9:5: [Crux] foo.wrapping_add(1) == foo diff --git a/crux-mir/test/symb_eval/overrides/override3.good b/crux-mir/test/symb_eval/overrides/override3.good index 938c23e93..05069c8ab 100644 --- a/crux-mir/test/symb_eval/overrides/override3.good +++ b/crux-mir/test/symb_eval/overrides/override3.good @@ -1,3 +1,3 @@ -test override3/19279091::f[0]: ok +test override3/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/overrides/override4.good b/crux-mir/test/symb_eval/overrides/override4.good index d8c0a1a88..718d39ea8 100644 --- a/crux-mir/test/symb_eval/overrides/override4.good +++ b/crux-mir/test/symb_eval/overrides/override4.good @@ -1,3 +1,3 @@ -test override4/083fe82b::f[0]: ok +test override4/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/overrides/override5.good b/crux-mir/test/symb_eval/overrides/override5.good index 3d18a0506..04409860f 100644 --- a/crux-mir/test/symb_eval/overrides/override5.good +++ b/crux-mir/test/symb_eval/overrides/override5.good @@ -1,10 +1,10 @@ -test override5/19f870ba::f[0]: FAILED +test override5/::f[0]: FAILED failures: ----- override5/19f870ba::f[0] counterexamples ---- +---- override5/::f[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/overrides/override5.rs:10:5: 10:47: error: in override5/19f870ba::f[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/overrides/override5.rs:10:5: 10:47: error: in override5/::f[0] [Crux] MIR assertion at test/symb_eval/overrides/override5.rs:10:5: [Crux] foo.wrapping_add(1) != 0 diff --git a/crux-mir/test/symb_eval/overrides/override_rust.good b/crux-mir/test/symb_eval/overrides/override_rust.good index 813b304a8..1e302e282 100644 --- a/crux-mir/test/symb_eval/overrides/override_rust.good +++ b/crux-mir/test/symb_eval/overrides/override_rust.good @@ -1,3 +1,3 @@ -test override_rust/5ae83a2f::crux_test[0]: ok +test override_rust/::crux_test[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/refs/mux_init_imm.good b/crux-mir/test/symb_eval/refs/mux_init_imm.good index a86449a3e..eb83c81c3 100644 --- a/crux-mir/test/symb_eval/refs/mux_init_imm.good +++ b/crux-mir/test/symb_eval/refs/mux_init_imm.good @@ -1,3 +1,3 @@ -test mux_init_imm/9274f69e::crux_test[0]: returned Symbolic BV, ok +test mux_init_imm/::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/refs/mux_init_mut.good b/crux-mir/test/symb_eval/refs/mux_init_mut.good index aa8bb547d..dd503691d 100644 --- a/crux-mir/test/symb_eval/refs/mux_init_mut.good +++ b/crux-mir/test/symb_eval/refs/mux_init_mut.good @@ -1,3 +1,3 @@ -test mux_init_mut/13769ed2::crux_test[0]: returned Symbolic BV, ok +test mux_init_mut/::crux_test[0]: returned Symbolic BV, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/scalar/test1.good b/crux-mir/test/symb_eval/scalar/test1.good index 08fa2b6e2..85fc13297 100644 --- a/crux-mir/test/symb_eval/scalar/test1.good +++ b/crux-mir/test/symb_eval/scalar/test1.good @@ -1,3 +1,3 @@ -test test1/e0d3b1cd::f[0]: ok +test test1/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/sym_bytes/construct.good b/crux-mir/test/symb_eval/sym_bytes/construct.good index 647eec8c3..68c51d4a9 100644 --- a/crux-mir/test/symb_eval/sym_bytes/construct.good +++ b/crux-mir/test/symb_eval/sym_bytes/construct.good @@ -1,10 +1,10 @@ -test construct/4a537371::crux_test[0]: returned Symbolic BV, FAILED +test construct/::crux_test[0]: returned Symbolic BV, FAILED failures: ----- construct/4a537371::crux_test[0] counterexamples ---- +---- construct/::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/sym_bytes/construct.rs:13:5: 13:35: error: in construct/4a537371::crux_test[0] +[Crux] ./lib/crucible/lib.rs:38:41: 38:58 !test/symb_eval/sym_bytes/construct.rs:13:5: 13:35: error: in construct/::crux_test[0] [Crux] MIR assertion at test/symb_eval/sym_bytes/construct.rs:13:5: [Crux] sym2[0] == 0 diff --git a/crux-mir/test/symb_eval/sym_bytes/deserialize.good b/crux-mir/test/symb_eval/sym_bytes/deserialize.good index e1f8bbdc7..7f11de136 100644 --- a/crux-mir/test/symb_eval/sym_bytes/deserialize.good +++ b/crux-mir/test/symb_eval/sym_bytes/deserialize.good @@ -1,10 +1,10 @@ -test deserialize/a0a785f7::crux_test[0]: returned Symbolic BV, FAILED +test deserialize/::crux_test[0]: returned Symbolic BV, FAILED failures: ----- deserialize/a0a785f7::crux_test[0] counterexamples ---- +---- deserialize/::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] test/symb_eval/sym_bytes/deserialize.rs:19:21: 19:44: error: in deserialize/a0a785f7::deserialize[0] +[Crux] test/symb_eval/sym_bytes/deserialize.rs:19:21: 19:44: error: in deserialize/::deserialize[0] [Crux] attempt to compute `move _61 + move _62`, which would overflow [Crux] Overall status: Invalid. diff --git a/crux-mir/test/symb_eval/vec/clone.good b/crux-mir/test/symb_eval/vec/clone.good index 450fcf559..48dcb06a2 100644 --- a/crux-mir/test/symb_eval/vec/clone.good +++ b/crux-mir/test/symb_eval/vec/clone.good @@ -1,3 +1,3 @@ -test clone/a6d38a22::f[0]: ok +test clone/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vec/into_iter.good b/crux-mir/test/symb_eval/vec/into_iter.good index d6fb1920a..608ab1680 100644 --- a/crux-mir/test/symb_eval/vec/into_iter.good +++ b/crux-mir/test/symb_eval/vec/into_iter.good @@ -1,3 +1,3 @@ -test into_iter/17035813::f[0]: ok +test into_iter/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vec/macro.good b/crux-mir/test/symb_eval/vec/macro.good index b93da9bd9..db1e3408e 100644 --- a/crux-mir/test/symb_eval/vec/macro.good +++ b/crux-mir/test/symb_eval/vec/macro.good @@ -1,3 +1,3 @@ -test macro/36f16f2e::f[0]: ok +test macro/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vec/sort_by_key.good b/crux-mir/test/symb_eval/vec/sort_by_key.good index 5c0bc10a3..efd849c57 100644 --- a/crux-mir/test/symb_eval/vec/sort_by_key.good +++ b/crux-mir/test/symb_eval/vec/sort_by_key.good @@ -1,3 +1,3 @@ -test sort_by_key/31d3f947::f[0]: ok +test sort_by_key/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/as_mut_slice.good b/crux-mir/test/symb_eval/vector/as_mut_slice.good index 4bd1380de..bc190605f 100644 --- a/crux-mir/test/symb_eval/vector/as_mut_slice.good +++ b/crux-mir/test/symb_eval/vector/as_mut_slice.good @@ -1,3 +1,3 @@ -test as_mut_slice/65a3c7cd::f[0]: ok +test as_mut_slice/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/as_slice.good b/crux-mir/test/symb_eval/vector/as_slice.good index 659ab2a7f..035d64a37 100644 --- a/crux-mir/test/symb_eval/vector/as_slice.good +++ b/crux-mir/test/symb_eval/vector/as_slice.good @@ -1,3 +1,3 @@ -test as_slice/0cc3eb84::f[0]: ok +test as_slice/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/concat.good b/crux-mir/test/symb_eval/vector/concat.good index 7510ee189..7b31032f9 100644 --- a/crux-mir/test/symb_eval/vector/concat.good +++ b/crux-mir/test/symb_eval/vector/concat.good @@ -1,3 +1,3 @@ -test concat/e9e1d873::f[0]: ok +test concat/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/copy_from_slice.good b/crux-mir/test/symb_eval/vector/copy_from_slice.good index b3e56af73..05e78e7e7 100644 --- a/crux-mir/test/symb_eval/vector/copy_from_slice.good +++ b/crux-mir/test/symb_eval/vector/copy_from_slice.good @@ -1,3 +1,3 @@ -test copy_from_slice/d3fda69b::f[0]: ok +test copy_from_slice/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/mut.good b/crux-mir/test/symb_eval/vector/mut.good index c35d63d85..cce6acdd6 100644 --- a/crux-mir/test/symb_eval/vector/mut.good +++ b/crux-mir/test/symb_eval/vector/mut.good @@ -1,3 +1,3 @@ -test mut/8471053f::f[0]: ok +test mut/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/new.good b/crux-mir/test/symb_eval/vector/new.good index 6b0633c92..32ebe4def 100644 --- a/crux-mir/test/symb_eval/vector/new.good +++ b/crux-mir/test/symb_eval/vector/new.good @@ -1,3 +1,3 @@ -test new/833ad47b::f[0]: ok +test new/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/push.good b/crux-mir/test/symb_eval/vector/push.good index 7a83b24f2..bfcc32b98 100644 --- a/crux-mir/test/symb_eval/vector/push.good +++ b/crux-mir/test/symb_eval/vector/push.good @@ -1,3 +1,3 @@ -test push/415cb1ec::f[0]: ok +test push/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/replicate.good b/crux-mir/test/symb_eval/vector/replicate.good index 17848fb32..62eea09fa 100644 --- a/crux-mir/test/symb_eval/vector/replicate.good +++ b/crux-mir/test/symb_eval/vector/replicate.good @@ -1,3 +1,3 @@ -test replicate/4ba185ef::f[0]: ok +test replicate/::f[0]: ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/vector/split_at.good b/crux-mir/test/symb_eval/vector/split_at.good index c382274ca..bd71b3e07 100644 --- a/crux-mir/test/symb_eval/vector/split_at.good +++ b/crux-mir/test/symb_eval/vector/split_at.good @@ -1,3 +1,3 @@ -test split_at/23248c64::f[0]: ok +test split_at/::f[0]: ok [Crux] Overall status: Valid. From 65e75fe5842e68899dfe87e81a0571be10be8e42 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 1 Jun 2023 20:43:16 -0400 Subject: [PATCH 083/114] Regenerate test_output.log --- crux-mir/test_output.log | 2389 +++++++++++++++++++------------------- 1 file changed, 1175 insertions(+), 1214 deletions(-) diff --git a/crux-mir/test_output.log b/crux-mir/test_output.log index 3baf84ffe..e31d84715 100644 --- a/crux-mir/test_output.log +++ b/crux-mir/test_output.log @@ -1,578 +1,148 @@ crux-mir crux concrete - refs - promoted_imm: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.72s) - Crux output: 0 - mut_raw: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 123 (1.72s) - Crux output: 123 - mut_nested: OK (1.92s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.78s) - Crux output: () - fn_ptr_mut: OK (1.97s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.83s) - Crux output: 1 - never: OK (2.01s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.85s) - Crux output: 1 - mut_ref: OK (2.10s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.94s) - Crux output: 1 - temp: OK (2.15s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (2.00s) - Crux output: 1 - mut_arg: OK (2.22s) - Compiling and running oracle program (0.18s) - Oracle output: 1 (2.04s) - Crux output: 1 - fn_ptr: OK (2.14s) - Compiling and running oracle program (0.20s) - Oracle output: 1 (1.95s) - Crux output: 1 - promoted_mut: OK (2.08s) - Compiling and running oracle program (0.16s) - Oracle output: 0 (1.92s) - Crux output: 0 - never_mut: OK (2.00s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.83s) - Crux output: 1 - mut_tuple_field: OK (2.06s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.90s) - Crux output: () - imm_raw: OK (2.03s) - Compiling and running oracle program (0.15s) - Oracle output: 123 (1.88s) - Crux output: 123 - static_mut: OK (2.23s) - Compiling and running oracle program (0.17s) - Oracle output: 2 (2.06s) - Crux output: 2 - imm_ref: OK (1.98s) - Compiling and running oracle program (0.15s) - Oracle output: 123 (1.83s) - Crux output: 123 - imm_arg: OK (1.97s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.83s) - Crux output: 1 - hash_map - insert_multi: OK (2.42s) - Compiling and running oracle program (0.20s) - Oracle output: 100 (2.22s) - Crux output: 100 - insert_remove: OK (2.95s) - Compiling and running oracle program (0.20s) - Oracle output: 12 (2.75s) - Crux output: 12 - insert_iter: OK (2.28s) - Compiling and running oracle program (0.19s) - Oracle output: [11, 12] (2.09s) - Crux output: [11, 12] - insert_get: OK (2.26s) - Compiling and running oracle program (0.18s) - Oracle output: (11, 12) (2.08s) - Crux output: (11, 12) - traits - params: OK (1.89s) - Compiling and running oracle program (0.15s) - Oracle output: 25 (1.74s) - Crux output: 25 - generic3: OK (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.79s) - Crux output: () - static_three: OK (2.04s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (1.88s) - Crux output: 2 - generic1: OK (2.01s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.86s) - Crux output: () - subtrait: OK (1.97s) - Compiling and running oracle program (0.15s) - Oracle output: 73 (1.82s) - Crux output: 73 - gen_trait: OK (1.89s) - Compiling and running oracle program (0.15s) - Oracle output: -69 (1.74s) - Crux output: -69 - dynamic_branch: OK (2.01s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.86s) - Crux output: 1 - static_two: OK (2.02s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.86s) - Crux output: 1 - tyfam5: OK (2.02s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.87s) - Crux output: () - generic2: OK (1.97s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.82s) - Crux output: () - assoc3: OK (1.94s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.78s) - Crux output: () - tyfam4: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.76s) - Crux output: 0 - static_self: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.72s) - Crux output: 42 - conv: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.71s) - Crux output: 1 - dynamic_poly: OK (1.87s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (1.73s) + clos + fn_static_poly: OK (0.94s) + Compiling and running oracle program (0.12s) + Oracle output: 3 (0.82s) Crux output: 3 - dynamic_simple: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.72s) - Crux output: 1 - gen_trait_poly: OK (1.97s) - Compiling and running oracle program (0.14s) - Oracle output: 12 (1.82s) - Crux output: 12 - default: OK (1.91s) - Compiling and running oracle program (0.14s) - Oracle output: 12 (1.76s) - Crux output: 12 - assoc1: OK (1.92s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.77s) - Crux output: () - subtrait2: OK (1.90s) - Compiling and running oracle program (0.15s) - Oracle output: 64 (1.76s) - Crux output: 64 - bounds1: OK (1.89s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.74s) - Crux output: () - bounds3: OK (1.97s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.82s) - Crux output: () - static_eq: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: true (1.75s) - Crux output: true - tyfam3: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 23 (1.73s) - Crux output: 23 - dict_med: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.73s) - Crux output: 42 - basics1: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.75s) - Crux output: () - assoc2: OK (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.79s) - Crux output: () - dict_polymem: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 4 (1.74s) - Crux output: 4 - bounds2: OK (1.90s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.75s) - Crux output: () - intoiter: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.73s) - Crux output: 0 - static: OK (1.94s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.80s) - Crux output: 42 - dynamic_two: OK (1.89s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.75s) - Crux output: 1 - bounds4: OK (2.01s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.87s) - Crux output: () - dict_poly: OK (2.08s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.92s) - Crux output: 1 - dict_simple: OK (1.98s) - Compiling and running oracle program (0.15s) - Oracle output: 32 (1.82s) - Crux output: 32 - tyfam: OK (2.00s) - Compiling and running oracle program (0.16s) - Oracle output: 23 (1.84s) - Crux output: 23 - bounds5: OK (2.02s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.86s) - Crux output: () - tyfam2: OK (1.95s) - Compiling and running oracle program (0.15s) - Oracle output: 23 (1.80s) - Crux output: 23 - dynamic_med: OK (1.99s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.84s) - Crux output: 1 - intTest - test0039: OK (1.95s) - Compiling and running oracle program (0.15s) - Oracle output: 7 (1.80s) + fnptr_closure: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 7 (0.84s) Crux output: 7 - test0038: OK (1.96s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.81s) - Crux output: () - tuple - clone_rec: OK (1.88s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.73s) - Crux output: () - clone: OK (1.88s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.73s) - Crux output: () - clone_from: OK (1.98s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.84s) - Crux output: () - clone_struct: OK (2.00s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.86s) - Crux output: () - iter - for: OK (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: 47 (1.79s) - Crux output: 47 - sum: OK (1.96s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.81s) - Crux output: () - filter_chain: OK (2.06s) - Compiling and running oracle program (0.17s) - Oracle output: 0 (1.89s) - Crux output: 0 - from_fn: OK (1.91s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.76s) - Crux output: () - peek: OK (1.98s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (1.83s) - Crux output: 3 - zip: OK (1.99s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.83s) - Crux output: () - loop: OK (1.93s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.79s) + fnptr_fn: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: 4 (0.81s) + Crux output: 4 + conv_fnonce_fnmut: OK (0.95s) + Compiling and running oracle program (0.12s) + Oracle output: 2 (0.83s) Crux output: 2 - cloned: OK (1.98s) - Compiling and running oracle program (0.16s) - Oracle output: 6 (1.82s) - Crux output: 6 - str - format_hex: FAIL (2.70s) - Compiling and running oracle program (0.17s) - Oracle output: true (2.53s) - Crux output: - failures: - - ---- format_hex/45df7be1::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] - [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/format_hex/' to rerun this test only. - format_int: FAIL (2.65s) - Compiling and running oracle program (0.16s) - Oracle output: true (2.49s) + as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.83s) Crux output: failures: - ---- format_int/5be019fc::crux_test[0] counterexamples ---- + ---- as_fn_ptr/d8484d39::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] - [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/format_int/' to rerun this test only. - format_struct: FAIL (2.80s) - Compiling and running oracle program (0.16s) - Oracle output: true (2.64s) - Crux output: - failures: - - ---- format_struct/6a3af932::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] - [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/format_struct/' to rerun this test only. - format: FAIL (2.64s) - Compiling and running oracle program (0.15s) - Oracle output: true (2.49s) - Crux output: - failures: - - ---- format/9cdf0325::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] - [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '$0=="crux-mir.crux concrete..str.format"' to rerun this test only. - to_owned: FAIL (2.09s) - Compiling and running oracle program (0.16s) - Oracle output: true (1.93s) - Crux output: - failures: - - ---- to_owned/ddf383ae::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ops/control_flow.rs:84:30: 84:39 !lib/core/src/ops/control_flow.rs:84:30: 84:39: error: in core/092bc89a::ops[0]::control_flow[0]::{impl#11}[0]::eq[0]::_instdfc2b6f78a281997[0] - [Crux] bad unpack: Any as StructRepr [BVRepr 32, VariantRepr [StructRepr [UnitRepr], StructRepr [UnitRepr]]] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/to_owned/' to rerun this test only. - string_push: FAIL (2.64s) - Compiling and running oracle program (0.16s) - Oracle output: true (2.47s) - Crux output: - failures: - - ---- string_push/03bb3176::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ops/control_flow.rs:84:30: 84:39 !lib/core/src/ops/control_flow.rs:84:30: 84:39: error: in core/092bc89a::ops[0]::control_flow[0]::{impl#11}[0]::eq[0]::_instdfc2b6f78a281997[0] - [Crux] bad unpack: Any as StructRepr [BVRepr 32, VariantRepr [StructRepr [UnitRepr], StructRepr [UnitRepr]]] - - [Crux] Overall status: Invalid. - - test/Test.hs:124: - crux doesn't match oracle - - Use -p '/string_push/' to rerun this test only. - format_array: FAIL (2.74s) - Compiling and running oracle program (0.16s) - Oracle output: true (2.58s) - Crux output: - failures: - - ---- format_array/fd6e4004::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] - [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/d8484d39::crux_test[0] + [Crux] Translation error in as_fn_ptr/d8484d39::crux_test[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [], _fsreturn_ty = TyInt B32, _fsabi = RustAbi, _fsspreadarg = Nothing}) [Crux] Overall status: Invalid. - test/Test.hs:124: + test/Test.hs:123: crux doesn't match oracle - - Use -p '/format_array/' to rerun this test only. - sync - mutex: OK (2.92s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (2.77s) - Crux output: 1 - arc_cell: OK (2.11s) - Compiling and running oracle program (0.16s) - Oracle output: 4 (1.95s) - Crux output: 4 - arc: OK (2.03s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.88s) - Crux output: 1 - atomic_add: OK (1.98s) - Compiling and running oracle program (0.15s) - Oracle output: 6 (1.83s) - Crux output: 6 - rwlock: OK (2.96s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (2.81s) - Crux output: 2 - atomic_cxchg: OK (1.99s) - Compiling and running oracle program (0.15s) - Oracle output: 6 (1.84s) - Crux output: 6 - arc_clone: OK (2.12s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (1.96s) - Crux output: 2 - atomic_swap: OK (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: (2, 1) (1.79s) - Crux output: (2, 1) - rwlock_multi: OK (3.07s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (2.92s) + (expected failure) + fnonce1: OK (0.91s) + Compiling and running oracle program (0.12s) + Oracle output: 3 (0.79s) Crux output: 3 - atomic_fence: OK (1.99s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (1.83s) + direct_fn: OK (0.93s) + Compiling and running oracle program (0.12s) + Oracle output: 2 (0.80s) Crux output: 2 - mutex_multi: OK (2.98s) - Compiling and running oracle program (0.16s) - Oracle output: 3 (2.82s) + fn_static: OK (0.92s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.79s) Crux output: 3 - consts - struct_val: OK (1.91s) - Compiling and running oracle program (0.15s) - Oracle output: Foo { x: true } (1.76s) - Crux output: Foo { x: true } - struct_unit: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: Err(Foo { x: () }) (1.74s) - Crux output: Err(Foo { x: () }) - local_key: OK (1.91s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.76s) - Crux output: 1 - fn_def: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.72s) - Crux output: 1 - enum_val: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: None (1.73s) - Crux output: None - crypto - add: OK (1.94s) - Compiling and running oracle program (0.16s) - Oracle output: true (1.78s) - Crux output: true - add_noL: OK (1.96s) - Compiling and running oracle program (0.15s) - Oracle output: false (1.81s) - Crux output: false - cell - cell: OK (1.98s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.84s) - Crux output: 2 - ref_cell: OK (2.48s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (2.34s) + poly: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.80s) + Crux output: 3 + direct_fnmut: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: 7 (0.82s) + Crux output: 7 + direct_fnmut2: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 7 (0.85s) + Crux output: 7 + promoted: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 33 (0.84s) + Crux output: 33 + direct_fnonce: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.83s) Crux output: 2 - ref_cell2: OK (2.49s) + fo: OK (0.99s) Compiling and running oracle program (0.15s) - Oracle output: 2 (2.34s) - Crux output: 2 - fnptr - call: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.76s) + Oracle output: 29 (0.84s) + Crux output: 29 + fnonce: OK (0.91s) + Compiling and running oracle program (0.12s) + Oracle output: false (0.79s) + Crux output: false + fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (0.93s) + Compiling and running oracle program (0.12s) + Oracle output: 3 (0.81s) + vtable signature mismatch for vtable core/092bc89a::ops[0]::function[0]::Fn[0]::_vtbl570ffd80324c2ca9[0] of trait core/092bc89a::ops[0]::function[0]::Fn[0]::_trait42f8b6475ee85722[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32)] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32)] + CallStack (from HasCallStack): + error, called at src/Mir/Trans.hs:958:20 in crucible-mir-0.1-inplace:Mir.Trans + mkTraitObject, called at src/Mir/Trans.hs:801:11 in crucible-mir-0.1-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:932:5 in crucible-mir-0.1-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:984:30 in crucible-mir-0.1-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1197:11 in crucible-mir-0.1-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1642:68 in crucible-mir-0.1-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1651:28 in crucible-mir-0.1-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1691:10 in crucible-mir-0.1-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1737:24 in crucible-mir-0.1-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2171:30 in crucible-mir-0.1-inplace:Mir.Trans + transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate + translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language + (expected failure) + ref_fnmut: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 7 (0.81s) + Crux output: 7 + dispatch_fnmut: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 7 (0.83s) + Crux output: 7 + unique_borrow: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.81s) + Crux output: 3 + conv_fnmut_fn: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.80s) Crux output: 2 - field: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.74s) + fnptr_fnmut: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 4 (0.80s) + Crux output: 4 + fnptr_fnonce: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 4 (0.81s) + Crux output: 4 + conv_fnonce_fn: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.80s) Crux output: 2 - make: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.73s) - Crux output: 0 - custom: rustc compilation failed for custom -error output: -error[E0432]: unresolved import `core` - --> test/conc_eval/fnptr/custom.rs:2:5 - | -2 | use core::mem; - | ^^^^ maybe a missing crate `core`? - | - = help: consider adding `extern crate core` to use the `core` crate - -error: aborting due to previous error - -For more information about this error, try `rustc --explain E0432`. - -FAIL (expected: taking address of an overridden function) (0.05s) - Compiling and running oracle program (0.05s) - test/Test.hs:107: - failed to compile and run - (expected failure) - num - overflow: OK (1.98s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.84s) - Crux output: () - from_bytes: OK (1.99s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.85s) - Crux output: () - saturate: OK (2.49s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.32s) - Crux output: () prim - bool: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: false (1.73s) + bool: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: false (0.80s) Crux output: false - shift3: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: -9223372036854775808 (1.73s) + litbstring: OK (0.92s) + Compiling and running oracle program (0.13s) + Oracle output: true (0.79s) + Crux output: true + shift3: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: -9223372036854775808 (0.81s) Crux output: -9223372036854775808 - div: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: 4 (1.75s) - Crux output: 4 - ge: OK (1.91s) - Compiling and running oracle program (0.14s) - Oracle output: true (1.77s) + litstring: OK (0.92s) + Compiling and running oracle program (0.13s) + Oracle output: true (0.80s) + Crux output: true + ge: OK (0.92s) + Compiling and running oracle program (0.12s) + Oracle output: true (0.80s) Crux output: true - add1: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.71s) - Crux output: 2 shift_exceeding: rustc compilation failed for shift_exceeding error output: error[E0425]: cannot find function `catch_unwind` in module `panic` @@ -604,98 +174,443 @@ error: aborting due to previous error; 1 warning emitted For more information about this error, try `rustc --explain E0425`. FAIL (expected: Should panic, but doesn't) (0.05s) - Compiling and running oracle program (0.05s) - test/Test.hs:107: + Compiling and running oracle program (0.06s) + test/Test.hs:106: failed to compile and run (expected failure) - char_from_u32: OK (1.91s) - Compiling and running oracle program (0.14s) - Oracle output: 'A' (1.77s) + shift4: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: -9223372036854775808 (0.80s) + Crux output: -9223372036854775808 + mut_arg: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.81s) + Crux output: 1 + wrapping_sub: OK (0.92s) + Compiling and running oracle program (0.13s) + Oracle output: true (0.79s) + Crux output: true + lit: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: false (0.81s) + Crux output: false + mut: OK (0.93s) + Compiling and running oracle program (0.12s) + Oracle output: 14 (0.80s) + Crux output: 14 + shift1: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.81s) + Crux output: 2 + add1: OK (0.93s) + Compiling and running oracle program (0.12s) + Oracle output: 2 (0.81s) + Crux output: 2 + div: OK (0.93s) + Compiling and running oracle program (0.12s) + Oracle output: 4 (0.81s) + Crux output: 4 + shift2: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.80s) + Crux output: 2 + ffs: OK (1.02s) + Compiling and running oracle program (0.13s) + Oracle output: true (0.89s) + Crux output: true + char_from_u32: OK (0.94s) + Compiling and running oracle program (0.12s) + Oracle output: 'A' (0.81s) Crux output: 'A' - ffs: OK (1.97s) - Compiling and running oracle program (0.15s) - Oracle output: true (1.81s) + traits + tyfam: OK (0.92s) + Compiling and running oracle program (0.13s) + Oracle output: 23 (0.80s) + Crux output: 23 + dynamic_branch: OK (0.92s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.79s) + Crux output: 1 + static: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.83s) + Crux output: 42 + bounds3: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.83s) + Crux output: () + tyfam3: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: 23 (0.82s) + Crux output: 23 + static_two: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.81s) + Crux output: 1 + bounds2: OK (0.99s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.86s) + Crux output: () + generic2: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.84s) + Crux output: () + gen_trait_poly: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 12 (0.84s) + Crux output: 12 + dict_simple: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 32 (0.81s) + Crux output: 32 + static_self: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.84s) + Crux output: 42 + tyfam4: OK (0.97s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (0.83s) + Crux output: 0 + gen_trait: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: -69 (0.82s) + Crux output: -69 + generic3: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.82s) + Crux output: () + subtrait: OK (0.92s) + Compiling and running oracle program (0.13s) + Oracle output: 73 (0.80s) + Crux output: 73 + default: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 12 (0.81s) + Crux output: 12 + bounds1: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.81s) + Crux output: () + intoiter: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 0 (0.80s) + Crux output: 0 + static_eq: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: true (0.81s) Crux output: true - mut_arg: OK (1.92s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.77s) + assoc3: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.80s) + Crux output: () + dynamic_poly: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.81s) + Crux output: 3 + dynamic_med: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.80s) + Crux output: 1 + static_three: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.80s) + Crux output: 2 + dynamic_two: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.80s) Crux output: 1 - litstring: OK (1.95s) + bounds4: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.83s) + Crux output: () + bounds5: OK (0.98s) Compiling and running oracle program (0.14s) - Oracle output: true (1.80s) - Crux output: true - lit: OK (1.91s) + Oracle output: () (0.84s) + Crux output: () + dict_med: OK (0.92s) + Compiling and running oracle program (0.12s) + Oracle output: 42 (0.80s) + Crux output: 42 + dynamic_simple: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.80s) + Crux output: 1 + assoc1: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.82s) + Crux output: () + params: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: 25 (0.82s) + Crux output: 25 + conv: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.80s) + Crux output: 1 + basics1: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.83s) + Crux output: () + dict_poly: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.81s) + Crux output: 1 + dict_polymem: OK (0.94s) + Compiling and running oracle program (0.12s) + Oracle output: 4 (0.82s) + Crux output: 4 + subtrait2: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 64 (0.80s) + Crux output: 64 + tyfam2: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 23 (0.83s) + Crux output: 23 + tyfam5: OK (0.92s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.79s) + Crux output: () + assoc2: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.82s) + Crux output: () + generic1: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.83s) + Crux output: () + impl + self_mut: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.81s) + Crux output: 42 + self: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.83s) + Crux output: 42 + simple: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.80s) + Crux output: 42 + vec + drop: OK (1.11s) Compiling and running oracle program (0.14s) - Oracle output: false (1.76s) - Crux output: false - litbstring: OK (1.89s) + Oracle output: () (0.98s) + Crux output: () + from_elem_zero: OK (1.23s) Compiling and running oracle program (0.14s) - Oracle output: true (1.75s) - Crux output: true - shift1: OK (1.87s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.73s) - Crux output: 2 - shift2: OK (1.90s) + Oracle output: 0 (1.09s) + Crux output: 0 + extend_trusted_len: OK (1.82s) Compiling and running oracle program (0.14s) - Oracle output: 2 (1.76s) + Oracle output: (1, 10) (1.68s) + Crux output: (1, 10) + extend: OK (1.23s) + Compiling and running oracle program (0.14s) + Oracle output: (1, 10) (1.10s) + Crux output: (1, 10) + set_len: OK (1.15s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (1.02s) Crux output: 2 - shift4: OK (1.90s) + collect: OK (1.86s) Compiling and running oracle program (0.14s) - Oracle output: -9223372036854775808 (1.76s) - Crux output: -9223372036854775808 - mut: OK (1.88s) - Compiling and running oracle program (0.15s) - Oracle output: 14 (1.74s) - Crux output: 14 - wrapping_sub: OK (1.89s) + Oracle output: 45 (1.72s) + Crux output: 45 + push: OK (1.14s) + Compiling and running oracle program (0.13s) + Oracle output: (1, 2) (1.01s) + Crux output: (1, 2) + slice + len: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 5 (0.81s) + Crux output: 5 + mut_range: OK (1.50s) + Compiling and running oracle program (0.13s) + Oracle output: 86 (1.37s) + Crux output: 86 + range_len: OK (1.47s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (1.34s) + Crux output: 1 + eq: OK (1.05s) Compiling and running oracle program (0.14s) - Oracle output: true (1.75s) + Oracle output: () (0.91s) + Crux output: () + iter_mut: OK (1.06s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.93s) + Crux output: () + get: OK (0.97s) + Compiling and running oracle program (0.12s) + Oracle output: true (0.85s) Crux output: true - impl - self: OK (1.86s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.71s) - Crux output: 42 - simple: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.74s) + last: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.84s) + Crux output: 3 + range_len_mut: OK (1.44s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (1.31s) + Crux output: 1 + mut: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.82s) Crux output: 42 - self_mut: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.73s) + swap: OK (0.99s) + Compiling and running oracle program (0.13s) + Oracle output: 2001 (0.86s) + Crux output: 2001 + mk_and_proj: OK (0.92s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.80s) Crux output: 42 + num + overflow: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.87s) + Crux output: () + saturate: OK (1.53s) + Compiling and running oracle program (0.13s) + Oracle output: () (1.40s) + Crux output: () + from_bytes: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.88s) + Crux output: () + dyn + inherit: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: 4 (0.82s) + Crux output: 4 + plain_trait: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 100 (0.80s) + Crux output: 100 + trait_param: OK (0.91s) + Compiling and running oracle program (0.13s) + Oracle output: 100 (0.79s) + Crux output: 100 + assoc_ty: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 100 (0.80s) + Crux output: 100 box - new: OK (2.01s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.87s) + mut_ref: OK (1.08s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.95s) Crux output: () - mut_ref: OK (2.02s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.88s) + new: OK (1.09s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.96s) Crux output: () - mut: OK (2.01s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.86s) + struct: OK (1.09s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.96s) + Crux output: 1 + mut: OK (1.09s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.96s) Crux output: () - unsize: OK (1.99s) + unsize: OK (1.09s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.96s) + Crux output: 2 + vec_deque + rotate_right: OK (1.69s) + Compiling and running oracle program (0.14s) + Oracle output: [4, 5, 1, 2, 3] (1.55s) + Crux output: [4, 5, 1, 2, 3] + retain: OK (1.69s) + Compiling and running oracle program (0.14s) + Oracle output: [1, 4] (1.55s) + Crux output: [1, 4] + push: OK (1.68s) + Compiling and running oracle program (0.13s) + Oracle output: [4, 1, 2, 3, 5] (1.55s) + Crux output: [4, 1, 2, 3, 5] + rotate_left: OK (1.70s) Compiling and running oracle program (0.14s) - Oracle output: 2 (1.85s) + Oracle output: [3, 4, 5, 1, 2] (1.57s) + Crux output: [3, 4, 5, 1, 2] + pop: OK (1.60s) + Compiling and running oracle program (0.13s) + Oracle output: [5, 4, 3, 1, 2] (1.46s) + Crux output: [5, 4, 3, 1, 2] + iter_clone: OK (1.69s) + Compiling and running oracle program (0.15s) + Oracle output: [1, 2, 3, 2, 3] (1.54s) + Crux output: [1, 2, 3, 2, 3] + iter + loop: OK (0.94s) + Compiling and running oracle program (0.12s) + Oracle output: 2 (0.82s) Crux output: 2 - struct: OK (2.05s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.89s) + from_fn: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.85s) + Crux output: () + filter_chain: OK (1.08s) + Compiling and running oracle program (0.15s) + Oracle output: 0 (0.93s) + Crux output: 0 + for: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: 47 (0.87s) + Crux output: 47 + zip: OK (1.06s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.92s) + Crux output: () + peek: OK (1.05s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.91s) + Crux output: 3 + cloned: OK (1.02s) + Compiling and running oracle program (0.13s) + Oracle output: 6 (0.89s) + Crux output: 6 + sum: OK (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.88s) + Crux output: () + crypto + add: OK (1.04s) + Compiling and running oracle program (0.13s) + Oracle output: true (0.91s) + Crux output: true + add_noL: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: false (0.85s) + Crux output: false + statics + promoted_static: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.81s) + Crux output: 1 + promoted_fn: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.81s) Crux output: 1 ptr - is_null_slice: FAIL (expected: can't unsize null pointers) (1.95s) - Compiling and running oracle program (0.15s) - Oracle output: (false, true) (1.80s) + offset_mut: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.82s) + Crux output: 3 + dangling_eq: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.85s) + Crux output: () + is_null_slice: FAIL (expected: can't unsize null pointers) (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: (false, true) (0.84s) Crux output: failures: - ---- is_null_slice/3549f883::crux_test[0] counterexamples ---- + ---- is_null_slice/d474a1e5::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/ptr/is_null_slice.rs:7:13: 7:33: error: in is_null_slice/3549f883::crux_test[0] + [Crux] test/conc_eval/ptr/is_null_slice.rs:7:13: 7:33: error: in is_null_slice/d474a1e5::crux_test[0] [Crux] attempted subindex on the result of an integer-to-pointer cast [Crux] Found counterexample for verification goal [Crux] ./lib/core/src/ptr/const_ptr.rs:38:38: 38:42: error: in core/092bc89a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst55af81a2cef2701c[0] @@ -703,680 +618,726 @@ FAIL (expected: Should panic, but doesn't) (0.05s) [Crux] Overall status: Invalid. - test/Test.hs:124: + test/Test.hs:123: crux doesn't match oracle (expected failure) - struct_eq: OK (1.88s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.74s) - Crux output: () - read_write: OK (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: [1, 3, 1] (1.80s) - Crux output: [1, 3, 1] - offset_mut: OK (1.89s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (1.75s) - Crux output: 3 - copy: OK (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: [1, 2, 3, 1, 2, 3] (1.78s) - Crux output: [1, 2, 3, 1, 2, 3] - unsize_slice: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: (1, 2) (1.73s) - Crux output: (1, 2) - dangling_eq: OK (1.94s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.80s) - Crux output: () - cast_eq: OK (1.91s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.77s) - Crux output: () - offset_from: OK (1.95s) - Compiling and running oracle program (0.14s) - Oracle output: -2 (1.81s) - Crux output: -2 - null_eq: OK (1.91s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.77s) - Crux output: 1 - coerce_unsized: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: (1, 2) (1.73s) + coerce_unsized: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: (1, 2) (0.82s) Crux output: (1, 2) - offset: OK (1.96s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.81s) - Crux output: 3 - is_null: OK (2.14s) - Compiling and running oracle program (0.16s) - Oracle output: (false, true) (1.98s) + is_null: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: (false, true) (0.81s) Crux output: (false, true) - valid_eq: OK (2.03s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.87s) - Crux output: () - ops - deref2: OK (2.50s) - Compiling and running oracle program (0.16s) - Oracle output: () (2.35s) - Crux output: () - index2: OK (2.40s) - Compiling and running oracle program (0.22s) - Oracle output: () (2.18s) - Crux output: () - arith1: OK (2.00s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.84s) - Crux output: () - index3: OK (1.98s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.82s) - Crux output: () - deref1: OK (1.98s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.83s) + offset: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.81s) + Crux output: 3 + offset_from: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: -2 (0.83s) + Crux output: -2 + struct_eq: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.83s) Crux output: () - index1: OK (1.97s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.82s) + valid_eq: OK (0.95s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.82s) Crux output: () - deref3: OK (2.10s) - Compiling and running oracle program (0.18s) - Oracle output: () (1.92s) + cast_eq: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.84s) Crux output: () - statics - promoted_fn: OK (1.95s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.80s) - Crux output: 1 - promoted_static: OK (2.01s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.86s) + copy: OK (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: [1, 2, 3, 1, 2, 3] (0.88s) + Crux output: [1, 2, 3, 1, 2, 3] + unsize_slice: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: (1, 2) (0.83s) + Crux output: (1, 2) + read_write: OK (1.01s) + Compiling and running oracle program (0.13s) + Oracle output: [1, 3, 1] (0.88s) + Crux output: [1, 3, 1] + null_eq: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.83s) Crux output: 1 - io - cursor_write2: OK (3.26s) - Compiling and running oracle program (0.17s) - Oracle output: 0 (3.09s) - Crux output: 0 - cursor_write: OK (2.63s) - Compiling and running oracle program (0.17s) - Oracle output: 0 (2.46s) - Crux output: 0 - cursor_read: OK (2.70s) - Compiling and running oracle program (0.17s) - Oracle output: 0 (2.53s) - Crux output: 0 - mem - maybe_uninit: FAIL (2.00s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.85s) + str + to_owned: OK (1.17s) + Compiling and running oracle program (0.13s) + Oracle output: true (1.04s) + Crux output: true + format_struct: FAIL (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: true (1.72s) Crux output: failures: - ---- maybe_uninit/00b53fab::crux_test[0] counterexamples ---- + ---- format_struct/af5dc5a7::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in maybe_uninit/00b53fab::crux_test[0] - [Crux] Translation error in maybe_uninit/00b53fab::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyInt B32]) + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] + [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. - test/Test.hs:124: + test/Test.hs:123: crux doesn't match oracle - Use -p '$0=="crux-mir.crux concrete..mem.maybe_uninit"' to rerun this test only. - maybe_uninit_array_cast: FAIL (2.05s) - Compiling and running oracle program (0.15s) - Oracle output: [1, 2] (1.89s) + Use -p '/format_struct/' to rerun this test only. + format_array: FAIL (1.78s) + Compiling and running oracle program (0.13s) + Oracle output: true (1.64s) Crux output: failures: - ---- maybe_uninit_array_cast/8abbf92b::crux_test[0] counterexamples ---- + ---- format_array/15683c6d::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in maybe_uninit_array_cast/8abbf92b::crux_test[0] - [Crux] Translation error in maybe_uninit_array_cast/8abbf92b::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyArray (TyInt B32) 2]) + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] + [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. - test/Test.hs:124: + test/Test.hs:123: crux doesn't match oracle - Use -p '/maybe_uninit_array_cast/' to rerun this test only. + Use -p '/format_array/' to rerun this test only. + format_int: FAIL (1.71s) + Compiling and running oracle program (0.14s) + Oracle output: true (1.57s) + Crux output: + failures: + + ---- format_int/66f79d5f::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] + [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + + [Crux] Overall status: Invalid. + + test/Test.hs:123: + crux doesn't match oracle + + Use -p '/format_int/' to rerun this test only. + format: FAIL (1.74s) + Compiling and running oracle program (0.14s) + Oracle output: true (1.60s) + Crux output: + failures: + + ---- format/fa538280::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] + [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + + [Crux] Overall status: Invalid. + + test/Test.hs:123: + crux doesn't match oracle + + Use -p '$0=="crux-mir.crux concrete..str.format"' to rerun this test only. + format_hex: FAIL (1.70s) + Compiling and running oracle program (0.13s) + Oracle output: true (1.57s) + Crux output: + failures: + + ---- format_hex/efa14f18::crux_test[0] counterexamples ---- + [Crux] Found counterexample for verification goal + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] + [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + + [Crux] Overall status: Invalid. + + test/Test.hs:123: + crux doesn't match oracle + + Use -p '/format_hex/' to rerun this test only. + string_push: OK (1.65s) + Compiling and running oracle program (0.14s) + Oracle output: true (1.51s) + Crux output: true time - instant: OK (1.98s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.84s) + instant: OK (0.99s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.86s) Crux output: () - struct - field_order: OK (2.00s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.85s) + enum + arg2: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 0 (0.84s) + Crux output: 0 + field_order: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.81s) Crux output: () - arg: OK (2.03s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.88s) - Crux output: 42 - tup: OK (2.11s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.96s) + ret: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: (A(42), B { x: 42 }, C) (0.80s) + Crux output: (A(42), B { x: 42 }, C) + eq: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.84s) Crux output: () - proj: OK (2.23s) - Compiling and running oracle program (0.16s) - Oracle output: 42 (2.06s) + inner: OK (0.94s) + Compiling and running oracle program (0.12s) + Oracle output: 4 (0.81s) + Crux output: 4 + cow: OK (1.09s) + Compiling and running oracle program (0.14s) + Oracle output: 200 (0.96s) + Crux output: 200 + cmp: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: -23 (0.80s) + Crux output: -23 + match: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.81s) Crux output: 42 - ret: OK (2.09s) - Compiling and running oracle program (0.17s) - Oracle output: S { x: 42, y: 120 } (1.91s) - Crux output: S { x: 42, y: 120 } - repr_transparent: OK (2.54s) - Compiling and running oracle program (0.17s) - Oracle output: 6 (2.37s) + arg: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: 0 (0.82s) + Crux output: 0 + mixed_discrs: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.83s) + Crux output: () + sync + arc: OK (1.11s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.99s) + Crux output: 1 + atomic_swap: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: (2, 1) (0.86s) + Crux output: (2, 1) + rwlock_multi: OK (2.18s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (2.04s) + Crux output: 3 + rwlock: OK (2.11s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.97s) + Crux output: 2 + atomic_fence: OK (1.00s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.87s) + Crux output: 2 + mutex_multi: OK (2.08s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.95s) + Crux output: 3 + atomic_add: OK (1.07s) + Compiling and running oracle program (0.13s) + Oracle output: 6 (0.93s) Crux output: 6 - repr_transparent_const: OK (2.33s) - Compiling and running oracle program (0.18s) - Oracle output: 124 (2.15s) - Crux output: 124 - dyn - assoc_ty: OK (2.10s) - Compiling and running oracle program (0.17s) - Oracle output: 100 (1.93s) - Crux output: 100 - trait_param: OK (2.11s) - Compiling and running oracle program (0.15s) - Oracle output: 100 (1.96s) - Crux output: 100 - plain_trait: OK (2.09s) - Compiling and running oracle program (0.16s) - Oracle output: 100 (1.94s) - Crux output: 100 - inherit: OK (2.10s) - Compiling and running oracle program (0.15s) - Oracle output: 4 (1.95s) + arc_clone: OK (1.11s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.98s) + Crux output: 2 + arc_cell: OK (1.13s) + Compiling and running oracle program (0.14s) + Oracle output: 4 (1.00s) Crux output: 4 + mutex: OK (2.05s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (1.91s) + Crux output: 1 + atomic_cxchg: OK (1.09s) + Compiling and running oracle program (0.15s) + Oracle output: 6 (0.94s) + Crux output: 6 + intTest + test0038: OK (1.05s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.92s) + Crux output: () + test0039: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: 7 (0.81s) + Crux output: 7 stdlib - option: OK (2.12s) - Compiling and running oracle program (0.18s) - Oracle output: true (1.94s) - Crux output: true - option2: OK (2.14s) - Compiling and running oracle program (0.16s) - Oracle output: 0 (1.98s) - Crux output: 0 - result: OK (2.10s) - Compiling and running oracle program (0.17s) - Oracle output: 27 (1.93s) + result_interior: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: 3 (0.83s) + Crux output: 3 + option3: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 27 (0.84s) Crux output: 27 - default: OK (2.06s) - Compiling and running oracle program (0.16s) - Oracle output: true (1.90s) + result: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 27 (0.84s) + Crux output: 27 + poly: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.81s) + Crux output: 1 + default_impl: OK (0.95s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.83s) + Crux output: () + cvt: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 0 (0.84s) + Crux output: 0 + default: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: true (0.82s) Crux output: true - cvt: OK (2.09s) - Compiling and running oracle program (0.15s) - Oracle output: 0 (1.93s) + option: OK (0.96s) + Compiling and running oracle program (0.14s) + Oracle output: true (0.82s) + Crux output: true + option2: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 0 (0.83s) Crux output: 0 - range: OK (2.02s) + teq: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: false (0.84s) + Crux output: false + range: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 10 (0.81s) + Crux output: 10 + hash_map + insert_get: OK (1.32s) + Compiling and running oracle program (0.16s) + Oracle output: (11, 12) (1.16s) + Crux output: (11, 12) + insert_iter: OK (1.28s) + Compiling and running oracle program (0.16s) + Oracle output: [11, 12] (1.13s) + Crux output: [11, 12] + insert_multi: OK (1.30s) Compiling and running oracle program (0.16s) - Oracle output: 10 (1.85s) - Crux output: 10 - poly: OK (2.09s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.93s) + Oracle output: 100 (1.15s) + Crux output: 100 + insert_remove: OK (1.74s) + Compiling and running oracle program (0.16s) + Oracle output: 12 (1.58s) + Crux output: 12 + tuple + clone: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.83s) + Crux output: () + clone_from: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.84s) + Crux output: () + clone_rec: OK (0.99s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.86s) + Crux output: () + clone_struct: OK (0.99s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.86s) + Crux output: () + consts + struct_val: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: Foo { x: true } (0.82s) + Crux output: Foo { x: true } + struct_unit: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: Err(Foo { x: () }) (0.83s) + Crux output: Err(Foo { x: () }) + enum_val: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: None (0.82s) + Crux output: None + local_key: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.83s) Crux output: 1 - result_interior: OK (2.09s) + fn_def: OK (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.84s) + Crux output: 1 + array + iter: OK (1.02s) Compiling and running oracle program (0.15s) - Oracle output: 3 (1.94s) + Oracle output: 3 (0.87s) Crux output: 3 - default_impl: OK (2.34s) - Compiling and running oracle program (0.19s) - Oracle output: () (2.15s) + const_impl: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 5 (0.81s) + Crux output: 5 + clone: OK (1.06s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.93s) Crux output: () - option3: OK (2.15s) - Compiling and running oracle program (0.16s) - Oracle output: 27 (1.99s) - Crux output: 27 - teq: OK (2.38s) - Compiling and running oracle program (0.18s) - Oracle output: false (2.21s) - Crux output: false - array - wick2: OK (2.24s) - Compiling and running oracle program (0.19s) - Oracle output: true (2.04s) - Crux output: true - arg: OK (2.16s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (2.00s) - Crux output: 2 - iter: OK (2.20s) - Compiling and running oracle program (0.19s) - Oracle output: 3 (2.02s) - Crux output: 3 - mk_and_proj: OK (2.02s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.87s) + mut_index: OK (1.50s) + Compiling and running oracle program (0.13s) + Oracle output: 7 (1.37s) + Crux output: 7 + mut_arg: OK (0.94s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (0.81s) Crux output: 42 - wick3: FAIL (expected: needs Vec data structure from stdlib) (0.21s) - Compiling and running oracle program (0.17s) - Oracle output: true (0.04s) - user error (Error 101 while running mir-json on test/conc_eval/array/wick3.rs) - (expected failure) - clone: OK (2.06s) - Compiling and running oracle program (0.16s) - Oracle output: () (1.90s) + from_slice: OK (1.65s) + Compiling and running oracle program (0.13s) + Oracle output: () (1.52s) Crux output: () - wick1: FAIL (expected: needs Vec data structure from stdlib) (0.21s) - Compiling and running oracle program (0.17s) + const: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.80s) + Crux output: 1 + wick2: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: true (0.83s) + Crux output: true + wick1: FAIL (expected: needs Vec data structure from stdlib) (0.19s) + Compiling and running oracle program (0.14s) Oracle output: true (0.05s) user error (Error 101 while running mir-json on test/conc_eval/array/wick1.rs) (expected failure) - mut_index: OK (2.54s) - Compiling and running oracle program (0.16s) - Oracle output: 7 (2.38s) - Crux output: 7 - mut_arg: OK (2.05s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.89s) + arg: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.80s) + Crux output: 2 + mk_and_proj: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.80s) Crux output: 42 - const: OK (1.99s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.83s) - Crux output: 1 - from_slice: FAIL (2.66s) - Compiling and running oracle program (0.17s) - Oracle output: () (2.48s) + wick3: FAIL (expected: needs Vec data structure from stdlib) (0.18s) + Compiling and running oracle program (0.14s) + Oracle output: true (0.04s) + user error (Error 101 while running mir-json on test/conc_eval/array/wick3.rs) + (expected failure) + mem + maybe_uninit_array_cast: FAIL (0.97s) + Compiling and running oracle program (0.13s) + Oracle output: [1, 2] (0.84s) Crux output: failures: - ---- from_slice/0eab1383::f[0] counterexamples ---- + ---- maybe_uninit_array_cast/171ff09d::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/ops/control_flow.rs:84:30: 84:39 !lib/core/src/ops/control_flow.rs:84:30: 84:39: error: in core/092bc89a::ops[0]::control_flow[0]::{impl#11}[0]::eq[0]::_instdfc2b6f78a281997[0] - [Crux] bad unpack: Any as StructRepr [BVRepr 32, VariantRepr [StructRepr [UnitRepr], StructRepr [UnitRepr]]] + [Crux] internal: error: in maybe_uninit_array_cast/171ff09d::crux_test[0] + [Crux] Translation error in maybe_uninit_array_cast/171ff09d::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyArray (TyInt B32) 2]) [Crux] Overall status: Invalid. - test/Test.hs:124: + test/Test.hs:123: crux doesn't match oracle - Use -p '/array.from_slice/' to rerun this test only. - const_impl: OK (1.96s) - Compiling and running oracle program (0.15s) - Oracle output: 5 (1.81s) - Crux output: 5 - enum - match: OK (1.92s) - Compiling and running oracle program (0.16s) - Oracle output: 42 (1.76s) - Crux output: 42 - field_order: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.75s) - Crux output: () - mixed_discrs: OK (1.93s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.78s) - Crux output: () - arg: OK (1.89s) - Compiling and running oracle program (0.15s) - Oracle output: 0 (1.74s) - Crux output: 0 - arg2: OK (1.95s) - Compiling and running oracle program (0.15s) - Oracle output: 0 (1.80s) - Crux output: 0 - cow: OK (2.09s) - Compiling and running oracle program (0.16s) - Oracle output: 200 (1.93s) - Crux output: 200 - ret: OK (1.85s) - Compiling and running oracle program (0.15s) - Oracle output: (A(42), B { x: 42 }, C) (1.70s) - Crux output: (A(42), B { x: 42 }, C) - eq: OK (1.96s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.81s) - Crux output: () - cmp: OK (1.92s) - Compiling and running oracle program (0.14s) - Oracle output: -23 (1.78s) - Crux output: -23 - inner: OK (1.92s) - Compiling and running oracle program (0.15s) - Oracle output: 4 (1.77s) - Crux output: 4 - vec - drop: OK (2.09s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.94s) - Crux output: () - set_len: OK (2.11s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (1.95s) - Crux output: 2 - push: OK (2.10s) - Compiling and running oracle program (0.15s) - Oracle output: (1, 2) (1.94s) - Crux output: (1, 2) - extend: OK (2.19s) - Compiling and running oracle program (0.17s) - Oracle output: (1, 10) (2.02s) - Crux output: (1, 10) - collect: OK (2.80s) - Compiling and running oracle program (0.16s) - Oracle output: 45 (2.64s) - Crux output: 45 - extend_trusted_len: OK (2.74s) - Compiling and running oracle program (0.17s) - Oracle output: (1, 10) (2.57s) - Crux output: (1, 10) - from_elem_zero: OK (2.17s) - Compiling and running oracle program (0.16s) - Oracle output: 0 (2.01s) - Crux output: 0 - clos - promoted: OK (1.92s) - Compiling and running oracle program (0.15s) - Oracle output: 33 (1.78s) - Crux output: 33 - as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.75s) + Use -p '/maybe_uninit_array_cast/' to rerun this test only. + maybe_uninit: FAIL (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (0.84s) Crux output: failures: - ---- as_fn_ptr/ab0e7632::crux_test[0] counterexamples ---- + ---- maybe_uninit/a2fd5c40::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/ab0e7632::crux_test[0] - [Crux] Translation error in as_fn_ptr/ab0e7632::crux_test[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [], _fsreturn_ty = TyInt B32, _fsabi = RustAbi, _fsspreadarg = Nothing}) + [Crux] internal: error: in maybe_uninit/a2fd5c40::crux_test[0] + [Crux] Translation error in maybe_uninit/a2fd5c40::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyInt B32]) [Crux] Overall status: Invalid. - test/Test.hs:124: + test/Test.hs:123: crux doesn't match oracle - (expected failure) - fnptr_fnmut: OK (1.88s) - Compiling and running oracle program (0.15s) - Oracle output: 4 (1.73s) - Crux output: 4 - direct_fnonce: OK (1.91s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.76s) + + Use -p '$0=="crux-mir.crux concrete..mem.maybe_uninit"' to rerun this test only. + ops + index2: OK (0.95s) + Compiling and running oracle program (0.12s) + Oracle output: () (0.83s) + Crux output: () + deref1: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.81s) + Crux output: () + index1: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.81s) + Crux output: () + deref3: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.84s) + Crux output: () + arith1: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.82s) + Crux output: () + deref2: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.83s) + Crux output: () + index3: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.83s) + Crux output: () + fnptr + field: OK (0.96s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.83s) Crux output: 2 - conv_fnonce_fnmut: OK (1.90s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.75s) + custom: rustc compilation failed for custom +error output: +error[E0432]: unresolved import `core` + --> test/conc_eval/fnptr/custom.rs:2:5 + | +2 | use core::mem; + | ^^^^ maybe a missing crate `core`? + | + = help: consider adding `extern crate core` to use the `core` crate + +error: aborting due to previous error + +For more information about this error, try `rustc --explain E0432`. + +FAIL (expected: taking address of an overridden function) (0.05s) + Compiling and running oracle program (0.05s) + test/Test.hs:106: + failed to compile and run + (expected failure) + make: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 0 (0.81s) + Crux output: 0 + call: OK (0.96s) + Compiling and running oracle program (0.12s) + Oracle output: 2 (0.84s) Crux output: 2 - fn_static_poly: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.74s) - Crux output: 3 - fnptr_fnonce: OK (1.88s) - Compiling and running oracle program (0.15s) - Oracle output: 4 (1.73s) - Crux output: 4 - fn_static: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.72s) - Crux output: 3 - fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (1.88s) + io + cursor_write2: OK (2.19s) Compiling and running oracle program (0.14s) - Oracle output: 3 (1.74s) - vtable signature mismatch for vtable core/092bc89a::ops[0]::function[0]::Fn[0]::_vtbl570ffd80324c2ca9[0] of trait core/092bc89a::ops[0]::function[0]::Fn[0]::_trait42f8b6475ee85722[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32)] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32)] - CallStack (from HasCallStack): - error, called at src/Mir/Trans.hs:958:20 in crucible-mir-0.1-inplace:Mir.Trans - mkTraitObject, called at src/Mir/Trans.hs:801:11 in crucible-mir-0.1-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:932:5 in crucible-mir-0.1-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:984:30 in crucible-mir-0.1-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1197:11 in crucible-mir-0.1-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1642:68 in crucible-mir-0.1-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1651:28 in crucible-mir-0.1-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1691:10 in crucible-mir-0.1-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1737:24 in crucible-mir-0.1-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2171:30 in crucible-mir-0.1-inplace:Mir.Trans - transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate - translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language - (expected failure) - fnonce: OK (1.91s) + Oracle output: 0 (2.05s) + Crux output: 0 + cursor_read: OK (1.60s) + Compiling and running oracle program (0.16s) + Oracle output: 0 (1.44s) + Crux output: 0 + cursor_write: OK (1.64s) Compiling and running oracle program (0.14s) - Oracle output: false (1.77s) - Crux output: false - conv_fnmut_fn: OK (1.86s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.71s) + Oracle output: 0 (1.49s) + Crux output: 0 + cell + ref_cell2: OK (1.52s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (1.39s) Crux output: 2 - direct_fnmut2: OK (1.87s) - Compiling and running oracle program (0.15s) - Oracle output: 7 (1.73s) - Crux output: 7 - fo: OK (1.88s) - Compiling and running oracle program (0.15s) - Oracle output: 29 (1.73s) - Crux output: 29 - unique_borrow: OK (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (1.79s) - Crux output: 3 - conv_fnonce_fn: OK (1.92s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.77s) + cell: OK (0.98s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.86s) Crux output: 2 - fnptr_fn: OK (1.91s) - Compiling and running oracle program (0.15s) - Oracle output: 4 (1.76s) - Crux output: 4 - direct_fnmut: OK (2.01s) - Compiling and running oracle program (0.15s) - Oracle output: 7 (1.85s) - Crux output: 7 - dispatch_fnmut: OK (2.10s) - Compiling and running oracle program (0.17s) - Oracle output: 7 (1.94s) - Crux output: 7 - poly: OK (2.21s) - Compiling and running oracle program (0.16s) - Oracle output: 3 (2.05s) - Crux output: 3 - direct_fn: OK (2.00s) - Compiling and running oracle program (0.16s) - Oracle output: 2 (1.84s) + ref_cell: OK (1.53s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (1.40s) Crux output: 2 - fnptr_closure: OK (2.18s) - Compiling and running oracle program (0.17s) - Oracle output: 7 (2.01s) - Crux output: 7 - fnonce1: OK (1.98s) - Compiling and running oracle program (0.16s) - Oracle output: 3 (1.82s) - Crux output: 3 - ref_fnmut: OK (1.97s) - Compiling and running oracle program (0.15s) - Oracle output: 7 (1.82s) - Crux output: 7 - vec_deque - pop: OK (2.72s) - Compiling and running oracle program (0.17s) - Oracle output: [5, 4, 3, 1, 2] (2.55s) - Crux output: [5, 4, 3, 1, 2] - iter_clone: OK (2.70s) - Compiling and running oracle program (0.17s) - Oracle output: [1, 2, 3, 2, 3] (2.53s) - Crux output: [1, 2, 3, 2, 3] - push: OK (2.72s) - Compiling and running oracle program (0.16s) - Oracle output: [4, 1, 2, 3, 5] (2.56s) - Crux output: [4, 1, 2, 3, 5] - retain: OK (2.68s) - Compiling and running oracle program (0.17s) - Oracle output: [1, 4] (2.50s) - Crux output: [1, 4] - rotate_left: OK (2.76s) - Compiling and running oracle program (0.16s) - Oracle output: [3, 4, 5, 1, 2] (2.60s) - Crux output: [3, 4, 5, 1, 2] - rotate_right: OK (2.89s) - Compiling and running oracle program (0.17s) - Oracle output: [4, 5, 1, 2, 3] (2.72s) - Crux output: [4, 5, 1, 2, 3] - slice - len: OK (2.09s) - Compiling and running oracle program (0.16s) - Oracle output: 5 (1.93s) - Crux output: 5 - mut_range: OK (2.70s) - Compiling and running oracle program (0.16s) - Oracle output: 86 (2.54s) - Crux output: 86 - range_len: OK (2.51s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (2.36s) - Crux output: 1 - range_len_mut: OK (2.54s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (2.38s) - Crux output: 1 - mk_and_proj: OK (1.98s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.83s) - Crux output: 42 - swap: OK (2.03s) - Compiling and running oracle program (0.15s) - Oracle output: 2001 (1.88s) - Crux output: 2001 - eq: OK (2.06s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.90s) + struct + repr_transparent: OK (1.03s) + Compiling and running oracle program (0.13s) + Oracle output: 6 (0.90s) + Crux output: 6 + tup: OK (0.92s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.79s) Crux output: () - iter_mut: OK (2.16s) - Compiling and running oracle program (0.16s) - Oracle output: () (2.00s) + repr_transparent_const: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 124 (0.81s) + Crux output: 124 + field_order: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.82s) Crux output: () - get: OK (2.08s) - Compiling and running oracle program (0.17s) - Oracle output: true (1.91s) - Crux output: true - mut: OK (1.97s) - Compiling and running oracle program (0.16s) - Oracle output: 42 (1.82s) + ret: OK (0.92s) + Compiling and running oracle program (0.13s) + Oracle output: S { x: 42, y: 120 } (0.80s) + Crux output: S { x: 42, y: 120 } + proj: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.80s) Crux output: 42 - last: OK (2.27s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (2.12s) - Crux output: 3 + arg: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 42 (0.80s) + Crux output: 42 + refs + temp: OK (0.95s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (0.82s) + Crux output: 1 + static_mut: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: 2 (0.82s) + Crux output: 2 + imm_raw: OK (0.93s) + Compiling and running oracle program (0.12s) + Oracle output: 123 (0.81s) + Crux output: 123 + mut_tuple_field: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.82s) + Crux output: () + mut_ref: OK (0.92s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.80s) + Crux output: 1 + mut_arg: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.82s) + Crux output: 1 + never: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.81s) + Crux output: 1 + mut_raw: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 123 (0.81s) + Crux output: 123 + promoted_imm: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 0 (0.81s) + Crux output: 0 + mut_nested: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: () (0.82s) + Crux output: () + imm_ref: OK (0.93s) + Compiling and running oracle program (0.12s) + Oracle output: 123 (0.80s) + Crux output: 123 + fn_ptr_mut: OK (0.93s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.80s) + Crux output: 1 + promoted_mut: OK (0.95s) + Compiling and running oracle program (0.13s) + Oracle output: 0 (0.82s) + Crux output: 0 + imm_arg: OK (0.92s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (0.79s) + Crux output: 1 + fn_ptr: OK (0.95s) + Compiling and running oracle program (0.12s) + Oracle output: 1 (0.83s) + Crux output: 1 + never_mut: OK (0.94s) + Compiling and running oracle program (0.13s) + Oracle output: 1 (0.81s) + Crux output: 1 crux symbolic Output testing - mux_init_mut: OK (2.04s) - mux_init_imm: OK (1.98s) - early_fail: OK (1.93s) - mixed_fail: OK (1.89s) - multi: OK (1.98s) - fail_return: OK (1.88s) - no_conc: OK (1.91s) - conc: OK (1.91s) - array: OK (1.89s) - assert_ok: OK (2.39s) - assert: FAIL (2.47s) - files test/symb_eval/concretize/assert.good and test/symb_eval/concretize/assert.out differ; test/symb_eval/concretize/assert.out contains: - test assert/9ec99c21::crux_test[0]: returned 1, FAILED + macro: OK (1.02s) + clone: OK (1.08s) + sort_by_key: OK (1.64s) + into_iter: OK (1.53s) + as_slice: OK (0.89s) + replicate: OK (0.93s) + split_at: OK (0.92s) + concat: OK (0.89s) + copy_from_slice: OK (0.88s) + new: OK (0.95s) + as_mut_slice: OK (0.91s) + mut: OK (0.93s) + push: OK (0.94s) + pop: FAIL (0.95s) + files test/symb_eval/vector/pop.good and test/symb_eval/vector/pop.out differ; test/symb_eval/vector/pop.out contains: + test pop/::f[0]: FAILED failures: - ---- assert/9ec99c21::crux_test[0] counterexamples ---- + ---- pop/::f[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] - [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] + [Crux] test/symb_eval/vector/pop.rs:9:18: 9:19: error: in pop/::f[0] + [Crux] Translation error in pop/::f[0]: unknown ADT (core/::option[0]::Option[0],Substs [TyUint B8]) [Crux] Overall status: Invalid. - Use -p '$0=="crux-mir.crux symbolic.Output testing.assert"' to rerun this test only. - bytes2: OK (1.88s) - double: OK (1.77s) - ffs: OK (1.93s) - bytes: OK (1.87s) - bad_symb1: OK (1.81s) - bad_symb2: OK (1.82s) - override3: OK (1.82s) - override1: OK (1.78s) - override4: OK (1.94s) - override_rust: OK (2.06s) - override2: OK (1.99s) - override5: OK (2.12s) - mux: OK (2.02s) - checked_mul_signed: OK (1.86s) - checked_mul: OK (1.76s) - checked_div: OK (1.82s) - checked_add: OK (1.74s) - test1: OK (112.50s) - pop: FAIL (1.83s) - files test/symb_eval/vector/pop.good and test/symb_eval/vector/pop.out differ; test/symb_eval/vector/pop.out contains: - test pop/5b5ade0c::f[0]: FAILED + Use -p '/Output testing.pop/' to rerun this test only. + checked_div: OK (0.85s) + checked_add: OK (0.84s) + checked_mul_signed: OK (0.83s) + checked_mul: OK (0.82s) + array_mut: OK (0.96s) + array: OK (0.92s) + bad_symb1: OK (0.90s) + override5: OK (0.95s) + override3: OK (0.94s) + override_rust: OK (0.88s) + override1: OK (0.83s) + override2: OK (0.94s) + bad_symb2: OK (0.89s) + override4: OK (0.90s) + write: FAIL + test/symb_eval/byteorder/write.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/write.rs) + Use -p '/Output testing.write/' to rerun this test only. + read: FAIL (0.05s) + test/symb_eval/byteorder/read.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/read.rs) + Use -p '/Output testing.read/' to rerun this test only. + deserialize: OK (0.92s) + construct: OK (0.92s) + double: OK (0.89s) + bytes: OK (1.01s) + ffs: OK (1.05s) + bytes2: OK (1.07s) + from_to: OK (0.85s) + overflowing_sub: OK (0.93s) + literals: OK (0.83s) + arith: OK (1.06s) + leading_zeros: OK (1.08s) + cmp: OK (0.92s) + symbolic: OK (0.88s) + mux: OK (0.90s) + test1: OK (100.82s) + split_off: OK (1.39s) + split_to: OK (1.37s) + new: OK (0.89s) + put: OK (1.33s) + sym_len: OK (1.40s) + extend_bytes: OK (0.96s) + put_overflow: OK (1.44s) + multi: OK (1.01s) + fail_return: OK (0.95s) + mixed_fail: OK (0.99s) + early_fail: OK (0.97s) + basic: OK (0.90s) + slice: OK (1.05s) + slice_mut: OK (0.93s) + mux_slice: OK (1.05s) + downcast_fail: OK (0.98s) + downcast: OK (0.96s) + conditional: OK (0.85s) + mux: OK (0.96s) + vec_cursor_read: OK (1.97s) + vec_write: OK (1.44s) + mux_init_mut: OK (0.91s) + mux_init_imm: OK (0.94s) + uninit_read: OK (0.91s) + valid_read: OK (0.92s) + zero_length: OK (0.91s) + out_of_bounds: OK (0.98s) + no_conc: OK (0.90s) + assert: FAIL (1.58s) + files test/symb_eval/concretize/assert.good and test/symb_eval/concretize/assert.out differ; test/symb_eval/concretize/assert.out contains: + test assert/::crux_test[0]: returned 1, FAILED failures: - ---- pop/5b5ade0c::f[0] counterexamples ---- + ---- assert/::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/vector/pop.rs:9:18: 9:19: error: in pop/5b5ade0c::f[0] - [Crux] Translation error in pop/5b5ade0c::f[0]: unknown ADT (core/3a1fbbbh::option[0]::Option[0],Substs [TyUint B8]) + [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/::fmt[0]::write[0] + [Crux] Translation error in core/::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] [Crux] Overall status: Invalid. - Use -p '/Output testing.pop/' to rerun this test only. - as_mut_slice: OK (1.82s) - push: OK (1.80s) - copy_from_slice: OK (1.78s) - new: OK (1.84s) - mut: OK (1.83s) - concat: OK (1.84s) - split_at: OK (1.80s) - replicate: OK (1.85s) - as_slice: OK (1.83s) - array_mut: OK (1.84s) - array: OK (1.79s) - extend_bytes: OK (1.78s) - put: OK (2.22s) - split_to: OK (2.21s) - new: OK (1.82s) - split_off: OK (2.27s) - sym_len: OK (2.35s) - put_overflow: OK (2.30s) - vec_cursor_read: OK (2.79s) - vec_write: OK (2.37s) - write: FAIL (0.06s) - test/symb_eval/byteorder/write.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/write.rs) - Use -p '/Output testing.write/' to rerun this test only. - read: FAIL - test/symb_eval/byteorder/read.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/read.rs) - Use -p '/Output testing.read/' to rerun this test only. - slice_mut: OK (1.85s) - basic: OK (1.78s) - slice: OK (1.93s) - mux_slice: OK (1.93s) - mux: OK (1.79s) - uninit_read: OK (1.84s) - valid_read: OK (1.90s) - out_of_bounds: OK (1.84s) - zero_length: OK (1.84s) - downcast: OK (1.77s) - downcast_fail: OK (1.84s) - conditional: OK (1.82s) - literals: OK (1.73s) - arith: OK (1.93s) - leading_zeros: OK (1.91s) - overflowing_sub: OK (1.82s) - cmp: OK (1.84s) - symbolic: OK (1.70s) - from_to: OK (1.72s) - clone: OK (1.99s) - sort_by_key: OK (2.65s) - into_iter: OK (2.41s) - macro: OK (1.81s) - construct: OK (1.83s) - deserialize: OK (1.84s) + Use -p '$0=="crux-mir.crux symbolic.Output testing.assert"' to rerun this test only. + array: OK (1.04s) + conc: OK (0.99s) + assert_ok: OK (1.56s) crux coverage Output testing - coverage_dual: warning: trailing semicolon in macro used in expression position + coverage_try: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1390,8 +1351,8 @@ FAIL (expected: Should panic, but doesn't) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (1.82s) - coverage: warning: trailing semicolon in macro used in expression position +OK (1.02s) + coverage_cond: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1405,8 +1366,8 @@ FAIL (expected: Should panic, but doesn't) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (1.84s) - coverage_try: warning: trailing semicolon in macro used in expression position +OK (0.83s) + coverage: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1420,8 +1381,8 @@ FAIL (expected: Should panic, but doesn't) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (1.86s) - coverage_cond: warning: trailing semicolon in macro used in expression position +OK (0.97s) + coverage_mono: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1435,8 +1396,8 @@ FAIL (expected: Should panic, but doesn't) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (1.77s) - coverage_loop: warning: trailing semicolon in macro used in expression position +OK (0.87s) + coverage_macro: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1450,8 +1411,8 @@ FAIL (expected: Should panic, but doesn't) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (1.85s) - coverage_shortcircuit: warning: trailing semicolon in macro used in expression position +OK (0.97s) + coverage_dual: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1465,8 +1426,8 @@ FAIL (expected: Should panic, but doesn't) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (1.90s) - coverage_mono: warning: trailing semicolon in macro used in expression position +OK (0.90s) + coverage_match: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1480,8 +1441,8 @@ FAIL (expected: Should panic, but doesn't) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (1.79s) - coverage_match: warning: trailing semicolon in macro used in expression position +OK (1.03s) + coverage_shortcircuit: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1495,8 +1456,8 @@ FAIL (expected: Should panic, but doesn't) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (1.91s) - coverage_macro: warning: trailing semicolon in macro used in expression position +OK (1.01s) + coverage_loop: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1510,6 +1471,6 @@ FAIL (expected: Should panic, but doesn't) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (1.94s) +OK (0.95s) -14 out of 339 tests failed (794.60s) +11 out of 339 tests failed (452.57s) From d29bb61d6a9e4992bc71de8a91bfc5761b0a1b43 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Fri, 2 Jun 2023 07:10:34 -0400 Subject: [PATCH 084/114] Upgrade to byteorder v1.4.3 --- crux-mir/build.sh | 2 +- .../lib/byteorder/.github/workflows/ci.yml | 153 ++ crux-mir/lib/byteorder/.gitignore | 6 + crux-mir/lib/byteorder/CHANGELOG.md | 139 ++ crux-mir/lib/byteorder/COPYING | 3 + crux-mir/lib/byteorder/Cargo.toml | 34 + crux-mir/lib/byteorder/README.md | 77 + crux-mir/lib/byteorder/UNLICENSE | 24 + crux-mir/lib/byteorder/benches/bench.rs | 324 +++ crux-mir/lib/byteorder/rustfmt.toml | 2 + crux-mir/lib/byteorder/{ => src}/io.rs | 95 +- crux-mir/lib/byteorder/{ => src}/lib.rs | 2068 ++++++++++++----- 12 files changed, 2262 insertions(+), 665 deletions(-) create mode 100644 crux-mir/lib/byteorder/.github/workflows/ci.yml create mode 100644 crux-mir/lib/byteorder/.gitignore create mode 100644 crux-mir/lib/byteorder/CHANGELOG.md create mode 100644 crux-mir/lib/byteorder/COPYING create mode 100644 crux-mir/lib/byteorder/Cargo.toml create mode 100644 crux-mir/lib/byteorder/README.md create mode 100644 crux-mir/lib/byteorder/UNLICENSE create mode 100644 crux-mir/lib/byteorder/benches/bench.rs create mode 100644 crux-mir/lib/byteorder/rustfmt.toml rename crux-mir/lib/byteorder/{ => src}/io.rs (96%) rename crux-mir/lib/byteorder/{ => src}/lib.rs (67%) diff --git a/crux-mir/build.sh b/crux-mir/build.sh index 6c7093f4c..2201adbd8 100755 --- a/crux-mir/build.sh +++ b/crux-mir/build.sh @@ -81,5 +81,5 @@ echo 'Building bytes...' translate lib/bytes.rs --edition=2021 --crate-name bytes --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern crucible=rlibs/libcrucible.rlib echo 'Building byteorder...' -translate lib/byteorder/lib.rs --edition=2021 --crate-name byteorder --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib +translate lib/byteorder/src/lib.rs --edition=2021 --crate-name byteorder --cfg 'feature="std"' --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib diff --git a/crux-mir/lib/byteorder/.github/workflows/ci.yml b/crux-mir/lib/byteorder/.github/workflows/ci.yml new file mode 100644 index 000000000..076834a86 --- /dev/null +++ b/crux-mir/lib/byteorder/.github/workflows/ci.yml @@ -0,0 +1,153 @@ +name: ci +on: + pull_request: + push: + branches: + - master + schedule: + - cron: '00 01 * * *' +jobs: + test: + name: test + env: + # For some builds, we use cross to test on 32-bit and big-endian + # systems. + CARGO: cargo + # When CARGO is set to CROSS, TARGET is set to `--target matrix.target`. + TARGET: + runs-on: ${{ matrix.os }} + strategy: + matrix: + build: + - pinned + - stable + - stable-32 + - stable-mips + - beta + - nightly + - macos + - win-msvc + - win-gnu + include: + - build: pinned + os: ubuntu-18.04 + rust: 1.41.1 + - build: stable + os: ubuntu-18.04 + rust: stable + - build: stable-32 + os: ubuntu-18.04 + rust: stable + target: i686-unknown-linux-gnu + - build: stable-mips + os: ubuntu-18.04 + rust: stable + target: mips64-unknown-linux-gnuabi64 + - build: beta + os: ubuntu-18.04 + rust: beta + - build: nightly + os: ubuntu-18.04 + rust: nightly + - build: macos + os: macos-latest + rust: stable + - build: win-msvc + os: windows-2019 + rust: stable + - build: win-gnu + os: windows-2019 + rust: stable-x86_64-gnu + steps: + - name: Checkout repository + uses: actions/checkout@v1 + with: + fetch-depth: 1 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.rust }} + profile: minimal + override: true + + - name: Use Cross + if: matrix.target != '' + run: | + # We used to install 'cross' from master, but it kept failing. So now + # we build from a known-good version until 'cross' becomes more stable + # or we find an alternative. Notably, between v0.2.1 and current + # master (2022-06-14), the number of Cross's dependencies has doubled. + cargo install --bins --git https://github.com/cross-rs/cross --tag v0.2.1 + echo "CARGO=cross" >> $GITHUB_ENV + echo "TARGET=--target ${{ matrix.target }}" >> $GITHUB_ENV + + - name: Show command used for Cargo + run: | + echo "cargo command is: ${{ env.CARGO }}" + echo "target flag is: ${{ env.TARGET }}" + + - name: Show CPU info for debugging + if: matrix.os == 'ubuntu-18.04' + run: lscpu + + - name: Build + run: ${{ env.CARGO }} build --verbose $TARGET + + - name: Build (no default) + run: ${{ env.CARGO }} build --verbose $TARGET --no-default-features + + - name: Build docs + run: ${{ env.CARGO }} doc --verbose $TARGET + + # Our dev dependencies evolve more rapidly than we'd like, so only run + # tests when we aren't pinning the Rust version. + - name: Tests + if: matrix.build != 'pinned' + run: ${{ env.CARGO }} test --verbose $TARGET + + - name: Tests (no default, lib only) + if: matrix.build != 'pinned' + run: ${{ env.CARGO }} test --verbose --no-default-features --lib $TARGET + + - name: Tests (i128) + if: matrix.build != 'pinned' + run: ${{ env.CARGO }} test --verbose --features i128 $TARGET + + - name: Tests (no default, lib only, i128) + if: matrix.build != 'pinned' + run: ${{ env.CARGO }} test --verbose --no-default-features --features i128 --lib $TARGET + + - name: Compile benchmarks + if: matrix.build == 'nightly' + run: cargo bench --verbose --no-run $TARGET + + - name: Compile benchmarks (no default) + if: matrix.build == 'nightly' + run: cargo bench --verbose --no-run --no-default-features $TARGET + + - name: Compile benchmarks (i128) + if: matrix.build == 'nightly' + run: cargo bench --verbose --no-run --features i128 $TARGET + + - name: Compile benchmarks (no default, i128) + if: matrix.build == 'nightly' + run: cargo bench --verbose --no-run --no-default-features --features i128 $TARGET + + rustfmt: + name: rustfmt + runs-on: ubuntu-18.04 + steps: + - name: Checkout repository + uses: actions/checkout@v1 + with: + fetch-depth: 1 + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + profile: minimal + components: rustfmt + - name: Check formatting + run: cargo fmt -- --check diff --git a/crux-mir/lib/byteorder/.gitignore b/crux-mir/lib/byteorder/.gitignore new file mode 100644 index 000000000..1fec0efb8 --- /dev/null +++ b/crux-mir/lib/byteorder/.gitignore @@ -0,0 +1,6 @@ +.*.swp +doc +tags +build +target +Cargo.lock diff --git a/crux-mir/lib/byteorder/CHANGELOG.md b/crux-mir/lib/byteorder/CHANGELOG.md new file mode 100644 index 000000000..6b51eb076 --- /dev/null +++ b/crux-mir/lib/byteorder/CHANGELOG.md @@ -0,0 +1,139 @@ +1.3.4 +===== +This patch release squashes deprecation warnings for the `try!` macro, in +accordance with byteorder's minimum supported Rust version (currently at Rust +1.12.0). + + +1.3.3 +===== +This patch release adds `ByteOrder::write_i8_into()` as a simple, safe interface +for ordinarily unsafe or tedious code. + + +1.3.2 +===== +This patch release adds `ReadBytesExt::read_i8_into()` as a simple, safe interface +for ordinarily unsafe or tedious code. + + +1.3.1 +===== +This minor release performs mostly small internal changes. Going forward, these +are not going to be incorporated into the changelog. + + +1.3.0 +===== +This new minor release now enables `i128` support automatically on Rust +compilers that support 128-bit integers. The `i128` feature is now a no-op, but +continues to exist for backward compatibility purposes. The crate continues to +maintain compatibility with Rust 1.12.0. + +This release also deprecates the `ByteOrder` trait methods +`read_f32_into_unchecked` and `read_f64_into_unchecked` in favor of +`read_f32_into` and `read_f64_into`. This was an oversight from the 1.2 release +where the corresponding methods on `ReadBytesExt` were deprecated. + +`quickcheck` and `rand` were bumped to `0.8` and `0.6`, respectively. + +A few small documentation related bugs have been fixed. + + +1.2.7 +===== +This patch release excludes some CI files from the crate release and updates +the license field to use `OR` instead of `/`. + + +1.2.6 +===== +This patch release fixes some test compilation errors introduced by an +over-eager release of 1.2.5. + + +1.2.5 +===== +This patch release fixes some typos in the docs, adds doc tests to methods on +`WriteByteExt` and bumps the quickcheck dependency to `0.7`. + + +1.2.4 +===== +This patch release adds support for 48-bit integers by adding the following +methods to the `ByteOrder` trait: `read_u48`, `read_i48`, `write_u48` and +`write_i48`. Corresponding methods have been added to the `ReadBytesExt` and +`WriteBytesExt` traits as well. + + +1.2.3 +===== +This patch release removes the use of `feature(i128_type)` from byteorder, +since it has been stabilized. We leave byteorder's `i128` feature in place +in order to continue supporting compilation on older versions of Rust. + + +1.2.2 +===== +This patch release only consists of internal improvements and refactorings. +Notably, this removes all uses of `transmute` and instead uses pointer casts. + + +1.2.1 +===== +This patch release removes more unnecessary uses of `unsafe` that +were overlooked in the prior `1.2.0` release. In particular, the +`ReadBytesExt::read_{f32,f64}_into_checked` methods have been deprecated and +replaced by more appropriately named `read_{f32,f64}_into` methods. + + +1.2.0 +===== +The most prominent change in this release of `byteorder` is the removal of +unnecessary signaling NaN masking, and in turn, the `unsafe` annotations +associated with methods that didn't do masking. See +[#103](https://github.com/BurntSushi/byteorder/issues/103) +for more details. + +* [BUG #102](https://github.com/BurntSushi/byteorder/issues/102): + Fix big endian tests. +* [BUG #103](https://github.com/BurntSushi/byteorder/issues/103): + Remove sNaN masking. + + +1.1.0 +===== +This release of `byteorder` features a number of fixes and improvements, mostly +as a result of the +[Litz Blitz evaluation](https://public.etherpad-mozilla.org/p/rust-crate-eval-byteorder). + +Feature enhancements: + +* [FEATURE #63](https://github.com/BurntSushi/byteorder/issues/63): + Add methods for reading/writing slices of numbers for a specific + endianness. +* [FEATURE #65](https://github.com/BurntSushi/byteorder/issues/65): + Add support for `u128`/`i128` types. (Behind the nightly only `i128` + feature.) +* [FEATURE #72](https://github.com/BurntSushi/byteorder/issues/72): + Add "panics" and "errors" sections for each relevant public API item. +* [FEATURE #74](https://github.com/BurntSushi/byteorder/issues/74): + Add CI badges to Cargo.toml. +* [FEATURE #75](https://github.com/BurntSushi/byteorder/issues/75): + Add more examples to public API items. +* Add 24-bit read/write methods. +* Add `BE` and `LE` type aliases for `BigEndian` and `LittleEndian`, + respectively. + +Bug fixes: + +* [BUG #68](https://github.com/BurntSushi/byteorder/issues/68): + Panic in {BigEndian,LittleEndian}::default. +* [BUG #69](https://github.com/BurntSushi/byteorder/issues/69): + Seal the `ByteOrder` trait to prevent out-of-crate implementations. +* [BUG #71](https://github.com/BurntSushi/byteorder/issues/71): + Guarantee that the results of `read_f32`/`read_f64` are always defined. +* [BUG #73](https://github.com/BurntSushi/byteorder/issues/73): + Add crates.io categories. +* [BUG #77](https://github.com/BurntSushi/byteorder/issues/77): + Add `html_root` doc attribute. diff --git a/crux-mir/lib/byteorder/COPYING b/crux-mir/lib/byteorder/COPYING new file mode 100644 index 000000000..bb9c20a09 --- /dev/null +++ b/crux-mir/lib/byteorder/COPYING @@ -0,0 +1,3 @@ +This project is dual-licensed under the Unlicense and MIT licenses. + +You may use this code under the terms of either license. diff --git a/crux-mir/lib/byteorder/Cargo.toml b/crux-mir/lib/byteorder/Cargo.toml new file mode 100644 index 000000000..728f66805 --- /dev/null +++ b/crux-mir/lib/byteorder/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "byteorder" +version = "1.4.3" #:version +authors = ["Andrew Gallant "] +description = "Library for reading/writing numbers in big-endian and little-endian." +documentation = "https://docs.rs/byteorder" +homepage = "https://github.com/BurntSushi/byteorder" +repository = "https://github.com/BurntSushi/byteorder" +readme = "README.md" +categories = ["encoding", "parsing", "no-std"] +keywords = ["byte", "endian", "big-endian", "little-endian", "binary"] +license = "Unlicense OR MIT" +exclude = ["/ci/*"] +edition = "2018" + +[lib] +name = "byteorder" +bench = false + +[dev-dependencies] +quickcheck = { version = "0.9.2", default-features = false } +rand = "0.7" + +[features] +default = ["std"] +std = [] + +# This feature is no longer used and is DEPRECATED. This crate now +# automatically enables i128 support for Rust compilers that support it. The +# feature will be removed if and when a new major version is released. +i128 = [] + +[profile.bench] +opt-level = 3 diff --git a/crux-mir/lib/byteorder/README.md b/crux-mir/lib/byteorder/README.md new file mode 100644 index 000000000..6069899ba --- /dev/null +++ b/crux-mir/lib/byteorder/README.md @@ -0,0 +1,77 @@ +byteorder +========= +This crate provides convenience methods for encoding and decoding +numbers in either big-endian or little-endian order. + +[![Build status](https://github.com/BurntSushi/byteorder/workflows/ci/badge.svg)](https://github.com/BurntSushi/byteorder/actions) +[![](https://meritbadge.herokuapp.com/byteorder)](https://crates.io/crates/byteorder) + +Dual-licensed under MIT or the [UNLICENSE](https://unlicense.org/). + + +### Documentation + +https://docs.rs/byteorder + + +### Installation + +This crate works with Cargo and is on +[crates.io](https://crates.io/crates/byteorder). Add it to your `Cargo.toml` +like so: + +```toml +[dependencies] +byteorder = "1" +``` + +If you want to augment existing `Read` and `Write` traits, then import the +extension methods like so: + +```rust +use byteorder::{ReadBytesExt, WriteBytesExt, BigEndian, LittleEndian}; +``` + +For example: + +```rust +use std::io::Cursor; +use byteorder::{BigEndian, ReadBytesExt}; + +let mut rdr = Cursor::new(vec![2, 5, 3, 0]); +// Note that we use type parameters to indicate which kind of byte order +// we want! +assert_eq!(517, rdr.read_u16::().unwrap()); +assert_eq!(768, rdr.read_u16::().unwrap()); +``` + +### `no_std` crates + +This crate has a feature, `std`, that is enabled by default. To use this crate +in a `no_std` context, add the following to your `Cargo.toml`: + +```toml +[dependencies] +byteorder = { version = "1", default-features = false } +``` + + +### Minimum Rust version policy + +This crate's minimum supported `rustc` version is `1.41.1`. + +The current policy is that the minimum Rust version required to use this crate +can be increased in minor version updates. For example, if `crate 1.0` requires +Rust 1.20.0, then `crate 1.0.z` for all values of `z` will also require Rust +1.20.0 or newer. However, `crate 1.y` for `y > 0` may require a newer minimum +version of Rust. + +In general, this crate will be conservative with respect to the minimum +supported version of Rust. + + +### Alternatives + +Note that as of Rust 1.32, the standard numeric types provide built-in methods +like `to_le_bytes` and `from_le_bytes`, which support some of the same use +cases. diff --git a/crux-mir/lib/byteorder/UNLICENSE b/crux-mir/lib/byteorder/UNLICENSE new file mode 100644 index 000000000..68a49daad --- /dev/null +++ b/crux-mir/lib/byteorder/UNLICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/crux-mir/lib/byteorder/benches/bench.rs b/crux-mir/lib/byteorder/benches/bench.rs new file mode 100644 index 000000000..bb00422f7 --- /dev/null +++ b/crux-mir/lib/byteorder/benches/bench.rs @@ -0,0 +1,324 @@ +#![feature(test)] + +extern crate test; + +macro_rules! bench_num { + ($name:ident, $read:ident, $bytes:expr, $data:expr) => { + mod $name { + use byteorder::{ + BigEndian, ByteOrder, LittleEndian, NativeEndian, + }; + use test::black_box as bb; + use test::Bencher; + + const NITER: usize = 100_000; + + #[bench] + fn read_big_endian(b: &mut Bencher) { + let buf = $data; + b.iter(|| { + for _ in 0..NITER { + bb(BigEndian::$read(&buf, $bytes)); + } + }); + } + + #[bench] + fn read_little_endian(b: &mut Bencher) { + let buf = $data; + b.iter(|| { + for _ in 0..NITER { + bb(LittleEndian::$read(&buf, $bytes)); + } + }); + } + + #[bench] + fn read_native_endian(b: &mut Bencher) { + let buf = $data; + b.iter(|| { + for _ in 0..NITER { + bb(NativeEndian::$read(&buf, $bytes)); + } + }); + } + } + }; + ($ty:ident, $max:ident, + $read:ident, $write:ident, $size:expr, $data:expr) => { + mod $ty { + use byteorder::{ + BigEndian, ByteOrder, LittleEndian, NativeEndian, + }; + use std::$ty; + use test::black_box as bb; + use test::Bencher; + + const NITER: usize = 100_000; + + #[bench] + fn read_big_endian(b: &mut Bencher) { + let buf = $data; + b.iter(|| { + for _ in 0..NITER { + bb(BigEndian::$read(&buf)); + } + }); + } + + #[bench] + fn read_little_endian(b: &mut Bencher) { + let buf = $data; + b.iter(|| { + for _ in 0..NITER { + bb(LittleEndian::$read(&buf)); + } + }); + } + + #[bench] + fn read_native_endian(b: &mut Bencher) { + let buf = $data; + b.iter(|| { + for _ in 0..NITER { + bb(NativeEndian::$read(&buf)); + } + }); + } + + #[bench] + fn write_big_endian(b: &mut Bencher) { + let mut buf = $data; + let n = $ty::$max; + b.iter(|| { + for _ in 0..NITER { + bb(BigEndian::$write(&mut buf, n)); + } + }); + } + + #[bench] + fn write_little_endian(b: &mut Bencher) { + let mut buf = $data; + let n = $ty::$max; + b.iter(|| { + for _ in 0..NITER { + bb(LittleEndian::$write(&mut buf, n)); + } + }); + } + + #[bench] + fn write_native_endian(b: &mut Bencher) { + let mut buf = $data; + let n = $ty::$max; + b.iter(|| { + for _ in 0..NITER { + bb(NativeEndian::$write(&mut buf, n)); + } + }); + } + } + }; +} + +bench_num!(u16, MAX, read_u16, write_u16, 2, [1, 2]); +bench_num!(i16, MAX, read_i16, write_i16, 2, [1, 2]); +bench_num!(u32, MAX, read_u32, write_u32, 4, [1, 2, 3, 4]); +bench_num!(i32, MAX, read_i32, write_i32, 4, [1, 2, 3, 4]); +bench_num!(u64, MAX, read_u64, write_u64, 8, [1, 2, 3, 4, 5, 6, 7, 8]); +bench_num!(i64, MAX, read_i64, write_i64, 8, [1, 2, 3, 4, 5, 6, 7, 8]); +bench_num!(f32, MAX, read_f32, write_f32, 4, [1, 2, 3, 4]); +bench_num!(f64, MAX, read_f64, write_f64, 8, [1, 2, 3, 4, 5, 6, 7, 8]); + +bench_num!(uint_1, read_uint, 1, [1]); +bench_num!(uint_2, read_uint, 2, [1, 2]); +bench_num!(uint_3, read_uint, 3, [1, 2, 3]); +bench_num!(uint_4, read_uint, 4, [1, 2, 3, 4]); +bench_num!(uint_5, read_uint, 5, [1, 2, 3, 4, 5]); +bench_num!(uint_6, read_uint, 6, [1, 2, 3, 4, 5, 6]); +bench_num!(uint_7, read_uint, 7, [1, 2, 3, 4, 5, 6, 7]); +bench_num!(uint_8, read_uint, 8, [1, 2, 3, 4, 5, 6, 7, 8]); + +bench_num!(int_1, read_int, 1, [1]); +bench_num!(int_2, read_int, 2, [1, 2]); +bench_num!(int_3, read_int, 3, [1, 2, 3]); +bench_num!(int_4, read_int, 4, [1, 2, 3, 4]); +bench_num!(int_5, read_int, 5, [1, 2, 3, 4, 5]); +bench_num!(int_6, read_int, 6, [1, 2, 3, 4, 5, 6]); +bench_num!(int_7, read_int, 7, [1, 2, 3, 4, 5, 6, 7]); +bench_num!(int_8, read_int, 8, [1, 2, 3, 4, 5, 6, 7, 8]); + +bench_num!( + u128, + MAX, + read_u128, + write_u128, + 16, + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] +); +bench_num!( + i128, + MAX, + read_i128, + write_i128, + 16, + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] +); + +bench_num!(uint128_1, read_uint128, 1, [1]); +bench_num!(uint128_2, read_uint128, 2, [1, 2]); +bench_num!(uint128_3, read_uint128, 3, [1, 2, 3]); +bench_num!(uint128_4, read_uint128, 4, [1, 2, 3, 4]); +bench_num!(uint128_5, read_uint128, 5, [1, 2, 3, 4, 5]); +bench_num!(uint128_6, read_uint128, 6, [1, 2, 3, 4, 5, 6]); +bench_num!(uint128_7, read_uint128, 7, [1, 2, 3, 4, 5, 6, 7]); +bench_num!(uint128_8, read_uint128, 8, [1, 2, 3, 4, 5, 6, 7, 8]); +bench_num!(uint128_9, read_uint128, 9, [1, 2, 3, 4, 5, 6, 7, 8, 9]); +bench_num!(uint128_10, read_uint128, 10, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); +bench_num!(uint128_11, read_uint128, 11, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); +bench_num!( + uint128_12, + read_uint128, + 12, + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] +); +bench_num!( + uint128_13, + read_uint128, + 13, + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] +); +bench_num!( + uint128_14, + read_uint128, + 14, + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] +); +bench_num!( + uint128_15, + read_uint128, + 15, + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] +); +bench_num!( + uint128_16, + read_uint128, + 16, + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] +); + +bench_num!(int128_1, read_int128, 1, [1]); +bench_num!(int128_2, read_int128, 2, [1, 2]); +bench_num!(int128_3, read_int128, 3, [1, 2, 3]); +bench_num!(int128_4, read_int128, 4, [1, 2, 3, 4]); +bench_num!(int128_5, read_int128, 5, [1, 2, 3, 4, 5]); +bench_num!(int128_6, read_int128, 6, [1, 2, 3, 4, 5, 6]); +bench_num!(int128_7, read_int128, 7, [1, 2, 3, 4, 5, 6, 7]); +bench_num!(int128_8, read_int128, 8, [1, 2, 3, 4, 5, 6, 7, 8]); +bench_num!(int128_9, read_int128, 9, [1, 2, 3, 4, 5, 6, 7, 8, 9]); +bench_num!(int128_10, read_int128, 10, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); +bench_num!(int128_11, read_int128, 11, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); +bench_num!( + int128_12, + read_int128, + 12, + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] +); +bench_num!( + int128_13, + read_int128, + 13, + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] +); +bench_num!( + int128_14, + read_int128, + 14, + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] +); +bench_num!( + int128_15, + read_int128, + 15, + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] +); +bench_num!( + int128_16, + read_int128, + 16, + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] +); + +macro_rules! bench_slice { + ($name:ident, $numty:ty, $read:ident, $write:ident) => { + mod $name { + use std::mem::size_of; + + use byteorder::{BigEndian, ByteOrder, LittleEndian}; + use rand::distributions; + use rand::{self, Rng}; + use test::Bencher; + + #[bench] + fn read_big_endian(b: &mut Bencher) { + let mut numbers: Vec<$numty> = rand::thread_rng() + .sample_iter(&distributions::Standard) + .take(100000) + .collect(); + let mut bytes = vec![0; numbers.len() * size_of::<$numty>()]; + BigEndian::$write(&numbers, &mut bytes); + + b.bytes = bytes.len() as u64; + b.iter(|| { + BigEndian::$read(&bytes, &mut numbers); + }); + } + + #[bench] + fn read_little_endian(b: &mut Bencher) { + let mut numbers: Vec<$numty> = rand::thread_rng() + .sample_iter(&distributions::Standard) + .take(100000) + .collect(); + let mut bytes = vec![0; numbers.len() * size_of::<$numty>()]; + LittleEndian::$write(&numbers, &mut bytes); + + b.bytes = bytes.len() as u64; + b.iter(|| { + LittleEndian::$read(&bytes, &mut numbers); + }); + } + + #[bench] + fn write_big_endian(b: &mut Bencher) { + let numbers: Vec<$numty> = rand::thread_rng() + .sample_iter(&distributions::Standard) + .take(100000) + .collect(); + let mut bytes = vec![0; numbers.len() * size_of::<$numty>()]; + + b.bytes = bytes.len() as u64; + b.iter(|| { + BigEndian::$write(&numbers, &mut bytes); + }); + } + + #[bench] + fn write_little_endian(b: &mut Bencher) { + let numbers: Vec<$numty> = rand::thread_rng() + .sample_iter(&distributions::Standard) + .take(100000) + .collect(); + let mut bytes = vec![0; numbers.len() * size_of::<$numty>()]; + + b.bytes = bytes.len() as u64; + b.iter(|| { + LittleEndian::$write(&numbers, &mut bytes); + }); + } + } + }; +} + +bench_slice!(slice_u64, u64, read_u64_into, write_u64_into); diff --git a/crux-mir/lib/byteorder/rustfmt.toml b/crux-mir/lib/byteorder/rustfmt.toml new file mode 100644 index 000000000..aa37a218b --- /dev/null +++ b/crux-mir/lib/byteorder/rustfmt.toml @@ -0,0 +1,2 @@ +max_width = 79 +use_small_heuristics = "max" diff --git a/crux-mir/lib/byteorder/io.rs b/crux-mir/lib/byteorder/src/io.rs similarity index 96% rename from crux-mir/lib/byteorder/io.rs rename to crux-mir/lib/byteorder/src/io.rs index ed6a84845..dfad2ca39 100644 --- a/crux-mir/lib/byteorder/io.rs +++ b/crux-mir/lib/byteorder/src/io.rs @@ -1,7 +1,9 @@ -use std::io::{self, Result}; -use std::slice; +use std::{ + io::{self, Result}, + slice, +}; -use ByteOrder; +use crate::ByteOrder; /// Extends [`Read`] with methods for reading numbers. (For `std::io`.) /// @@ -52,7 +54,7 @@ pub trait ReadBytesExt: io::Read { #[inline] fn read_u8(&mut self) -> Result { let mut buf = [0; 1]; - try!(self.read_exact(&mut buf)); + self.read_exact(&mut buf)?; Ok(buf[0]) } @@ -82,7 +84,7 @@ pub trait ReadBytesExt: io::Read { #[inline] fn read_i8(&mut self) -> Result { let mut buf = [0; 1]; - try!(self.read_exact(&mut buf)); + self.read_exact(&mut buf)?; Ok(buf[0] as i8) } @@ -109,7 +111,7 @@ pub trait ReadBytesExt: io::Read { #[inline] fn read_u16(&mut self) -> Result { let mut buf = [0; 2]; - try!(self.read_exact(&mut buf)); + self.read_exact(&mut buf)?; Ok(T::read_u16(&buf)) } @@ -136,7 +138,7 @@ pub trait ReadBytesExt: io::Read { #[inline] fn read_i16(&mut self) -> Result { let mut buf = [0; 2]; - try!(self.read_exact(&mut buf)); + self.read_exact(&mut buf)?; Ok(T::read_i16(&buf)) } @@ -162,7 +164,7 @@ pub trait ReadBytesExt: io::Read { #[inline] fn read_u24(&mut self) -> Result { let mut buf = [0; 3]; - try!(self.read_exact(&mut buf)); + self.read_exact(&mut buf)?; Ok(T::read_u24(&buf)) } @@ -188,7 +190,7 @@ pub trait ReadBytesExt: io::Read { #[inline] fn read_i24(&mut self) -> Result { let mut buf = [0; 3]; - try!(self.read_exact(&mut buf)); + self.read_exact(&mut buf)?; Ok(T::read_i24(&buf)) } @@ -214,7 +216,7 @@ pub trait ReadBytesExt: io::Read { #[inline] fn read_u32(&mut self) -> Result { let mut buf = [0; 4]; - try!(self.read_exact(&mut buf)); + self.read_exact(&mut buf)?; Ok(T::read_u32(&buf)) } @@ -240,7 +242,7 @@ pub trait ReadBytesExt: io::Read { #[inline] fn read_i32(&mut self) -> Result { let mut buf = [0; 4]; - try!(self.read_exact(&mut buf)); + self.read_exact(&mut buf)?; Ok(T::read_i32(&buf)) } @@ -266,7 +268,7 @@ pub trait ReadBytesExt: io::Read { #[inline] fn read_u48(&mut self) -> Result { let mut buf = [0; 6]; - try!(self.read_exact(&mut buf)); + self.read_exact(&mut buf)?; Ok(T::read_u48(&buf)) } @@ -292,7 +294,7 @@ pub trait ReadBytesExt: io::Read { #[inline] fn read_i48(&mut self) -> Result { let mut buf = [0; 6]; - try!(self.read_exact(&mut buf)); + self.read_exact(&mut buf)?; Ok(T::read_i48(&buf)) } @@ -318,7 +320,7 @@ pub trait ReadBytesExt: io::Read { #[inline] fn read_u64(&mut self) -> Result { let mut buf = [0; 8]; - try!(self.read_exact(&mut buf)); + self.read_exact(&mut buf)?; Ok(T::read_u64(&buf)) } @@ -344,7 +346,7 @@ pub trait ReadBytesExt: io::Read { #[inline] fn read_i64(&mut self) -> Result { let mut buf = [0; 8]; - try!(self.read_exact(&mut buf)); + self.read_exact(&mut buf)?; Ok(T::read_i64(&buf)) } @@ -370,11 +372,10 @@ pub trait ReadBytesExt: io::Read { /// ]); /// assert_eq!(16947640962301618749969007319746179, rdr.read_u128::().unwrap()); /// ``` - #[cfg(byteorder_i128)] #[inline] fn read_u128(&mut self) -> Result { let mut buf = [0; 16]; - try!(self.read_exact(&mut buf)); + self.read_exact(&mut buf)?; Ok(T::read_u128(&buf)) } @@ -397,11 +398,10 @@ pub trait ReadBytesExt: io::Read { /// let mut rdr = Cursor::new(vec![0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); /// assert_eq!(i128::min_value(), rdr.read_i128::().unwrap()); /// ``` - #[cfg(byteorder_i128)] #[inline] fn read_i128(&mut self) -> Result { let mut buf = [0; 16]; - try!(self.read_exact(&mut buf)); + self.read_exact(&mut buf)?; Ok(T::read_i128(&buf)) } @@ -426,7 +426,7 @@ pub trait ReadBytesExt: io::Read { #[inline] fn read_uint(&mut self, nbytes: usize) -> Result { let mut buf = [0; 8]; - try!(self.read_exact(&mut buf[..nbytes])); + self.read_exact(&mut buf[..nbytes])?; Ok(T::read_uint(&buf[..nbytes], nbytes)) } @@ -451,25 +451,23 @@ pub trait ReadBytesExt: io::Read { #[inline] fn read_int(&mut self, nbytes: usize) -> Result { let mut buf = [0; 8]; - try!(self.read_exact(&mut buf[..nbytes])); + self.read_exact(&mut buf[..nbytes])?; Ok(T::read_int(&buf[..nbytes], nbytes)) } /// Reads an unsigned n-bytes integer from the underlying reader. - #[cfg(byteorder_i128)] #[inline] fn read_uint128(&mut self, nbytes: usize) -> Result { let mut buf = [0; 16]; - try!(self.read_exact(&mut buf[..nbytes])); + self.read_exact(&mut buf[..nbytes])?; Ok(T::read_uint128(&buf[..nbytes], nbytes)) } /// Reads a signed n-bytes integer from the underlying reader. - #[cfg(byteorder_i128)] #[inline] fn read_int128(&mut self, nbytes: usize) -> Result { let mut buf = [0; 16]; - try!(self.read_exact(&mut buf[..nbytes])); + self.read_exact(&mut buf[..nbytes])?; Ok(T::read_int128(&buf[..nbytes], nbytes)) } @@ -500,7 +498,7 @@ pub trait ReadBytesExt: io::Read { #[inline] fn read_f32(&mut self) -> Result { let mut buf = [0; 4]; - try!(self.read_exact(&mut buf)); + self.read_exact(&mut buf)?; Ok(T::read_f32(&buf)) } @@ -531,7 +529,7 @@ pub trait ReadBytesExt: io::Read { #[inline] fn read_f64(&mut self) -> Result { let mut buf = [0; 8]; - try!(self.read_exact(&mut buf)); + self.read_exact(&mut buf)?; Ok(T::read_f64(&buf)) } @@ -564,7 +562,7 @@ pub trait ReadBytesExt: io::Read { fn read_u16_into(&mut self, dst: &mut [u16]) -> Result<()> { { let buf = unsafe { slice_to_u8_mut(dst) }; - try!(self.read_exact(buf)); + self.read_exact(buf)?; } T::from_slice_u16(dst); Ok(()) @@ -599,7 +597,7 @@ pub trait ReadBytesExt: io::Read { fn read_u32_into(&mut self, dst: &mut [u32]) -> Result<()> { { let buf = unsafe { slice_to_u8_mut(dst) }; - try!(self.read_exact(buf)); + self.read_exact(buf)?; } T::from_slice_u32(dst); Ok(()) @@ -637,7 +635,7 @@ pub trait ReadBytesExt: io::Read { fn read_u64_into(&mut self, dst: &mut [u64]) -> Result<()> { { let buf = unsafe { slice_to_u8_mut(dst) }; - try!(self.read_exact(buf)); + self.read_exact(buf)?; } T::from_slice_u64(dst); Ok(()) @@ -671,7 +669,6 @@ pub trait ReadBytesExt: io::Read { /// rdr.read_u128_into::(&mut dst).unwrap(); /// assert_eq!([517, 768], dst); /// ``` - #[cfg(byteorder_i128)] #[inline] fn read_u128_into( &mut self, @@ -679,7 +676,7 @@ pub trait ReadBytesExt: io::Read { ) -> Result<()> { { let buf = unsafe { slice_to_u8_mut(dst) }; - try!(self.read_exact(buf)); + self.read_exact(buf)?; } T::from_slice_u128(dst); Ok(()) @@ -750,7 +747,7 @@ pub trait ReadBytesExt: io::Read { fn read_i16_into(&mut self, dst: &mut [i16]) -> Result<()> { { let buf = unsafe { slice_to_u8_mut(dst) }; - try!(self.read_exact(buf)); + self.read_exact(buf)?; } T::from_slice_i16(dst); Ok(()) @@ -785,7 +782,7 @@ pub trait ReadBytesExt: io::Read { fn read_i32_into(&mut self, dst: &mut [i32]) -> Result<()> { { let buf = unsafe { slice_to_u8_mut(dst) }; - try!(self.read_exact(buf)); + self.read_exact(buf)?; } T::from_slice_i32(dst); Ok(()) @@ -823,7 +820,7 @@ pub trait ReadBytesExt: io::Read { fn read_i64_into(&mut self, dst: &mut [i64]) -> Result<()> { { let buf = unsafe { slice_to_u8_mut(dst) }; - try!(self.read_exact(buf)); + self.read_exact(buf)?; } T::from_slice_i64(dst); Ok(()) @@ -857,7 +854,6 @@ pub trait ReadBytesExt: io::Read { /// rdr.read_i128_into::(&mut dst).unwrap(); /// assert_eq!([517, 768], dst); /// ``` - #[cfg(byteorder_i128)] #[inline] fn read_i128_into( &mut self, @@ -865,7 +861,7 @@ pub trait ReadBytesExt: io::Read { ) -> Result<()> { { let buf = unsafe { slice_to_u8_mut(dst) }; - try!(self.read_exact(buf)); + self.read_exact(buf)?; } T::from_slice_i128(dst); Ok(()) @@ -903,13 +899,10 @@ pub trait ReadBytesExt: io::Read { /// assert_eq!([f32::consts::PI, 1.0], dst); /// ``` #[inline] - fn read_f32_into( - &mut self, - dst: &mut [f32], - ) -> Result<()> { + fn read_f32_into(&mut self, dst: &mut [f32]) -> Result<()> { { let buf = unsafe { slice_to_u8_mut(dst) }; - try!(self.read_exact(buf)); + self.read_exact(buf)?; } T::from_slice_f32(dst); Ok(()) @@ -951,7 +944,7 @@ pub trait ReadBytesExt: io::Read { /// assert_eq!([f32::consts::PI, 1.0], dst); /// ``` #[inline] - #[deprecated(since="1.2.0", note="please use `read_f32_into` instead")] + #[deprecated(since = "1.2.0", note = "please use `read_f32_into` instead")] fn read_f32_into_unchecked( &mut self, dst: &mut [f32], @@ -991,13 +984,10 @@ pub trait ReadBytesExt: io::Read { /// assert_eq!([f64::consts::PI, 1.0], dst); /// ``` #[inline] - fn read_f64_into( - &mut self, - dst: &mut [f64], - ) -> Result<()> { + fn read_f64_into(&mut self, dst: &mut [f64]) -> Result<()> { { let buf = unsafe { slice_to_u8_mut(dst) }; - try!(self.read_exact(buf)); + self.read_exact(buf)?; } T::from_slice_f64(dst); Ok(()) @@ -1045,7 +1035,7 @@ pub trait ReadBytesExt: io::Read { /// assert_eq!([f64::consts::PI, 1.0], dst); /// ``` #[inline] - #[deprecated(since="1.2.0", note="please use `read_f64_into` instead")] + #[deprecated(since = "1.2.0", note = "please use `read_f64_into` instead")] fn read_f64_into_unchecked( &mut self, dst: &mut [f64], @@ -1408,7 +1398,6 @@ pub trait WriteBytesExt: io::Write { } /// Writes an unsigned 128 bit integer to the underlying writer. - #[cfg(byteorder_i128)] #[inline] fn write_u128(&mut self, n: u128) -> Result<()> { let mut buf = [0; 16]; @@ -1417,7 +1406,6 @@ pub trait WriteBytesExt: io::Write { } /// Writes a signed 128 bit integer to the underlying writer. - #[cfg(byteorder_i128)] #[inline] fn write_i128(&mut self, n: i128) -> Result<()> { let mut buf = [0; 16]; @@ -1501,7 +1489,6 @@ pub trait WriteBytesExt: io::Write { /// /// If the given integer is not representable in the given number of bytes, /// this method panics. If `nbytes > 16`, this method panics. - #[cfg(byteorder_i128)] #[inline] fn write_uint128( &mut self, @@ -1517,7 +1504,6 @@ pub trait WriteBytesExt: io::Write { /// /// If the given integer is not representable in the given number of bytes, /// this method panics. If `nbytes > 16`, this method panics. - #[cfg(byteorder_i128)] #[inline] fn write_int128( &mut self, @@ -1596,7 +1582,8 @@ impl WriteBytesExt for W {} /// representation. /// /// This function is wildly unsafe because it permits arbitrary modification of -/// the binary representation of any `Copy` type. Use with care. +/// the binary representation of any `Copy` type. Use with care. It's intended +/// to be called only where `T` is a numeric type. unsafe fn slice_to_u8_mut(slice: &mut [T]) -> &mut [u8] { use std::mem::size_of; diff --git a/crux-mir/lib/byteorder/lib.rs b/crux-mir/lib/byteorder/src/lib.rs similarity index 67% rename from crux-mir/lib/byteorder/lib.rs rename to crux-mir/lib/byteorder/src/lib.rs index 0909a3b95..cc37cca6a 100644 --- a/crux-mir/lib/byteorder/lib.rs +++ b/crux-mir/lib/byteorder/src/lib.rs @@ -67,42 +67,25 @@ cases. [`Write`]: https://doc.rust-lang.org/std/io/trait.Write.html */ -#![cfg_attr(crux, feature(crucible_intrinsics))] #![deny(missing_docs)] #![cfg_attr(not(feature = "std"), no_std)] -#[cfg(feature = "std")] -extern crate core; - -#[cfg(test)] -#[macro_use] -extern crate doc_comment; - -#[cfg(test)] -doctest!("../README.md"); - -use core::fmt::Debug; -use core::hash::Hash; -use core::mem; -use core::ptr::copy_nonoverlapping; -use core::slice; +use core::{ + convert::TryInto, fmt::Debug, hash::Hash, ptr::copy_nonoverlapping, slice, +}; #[cfg(feature = "std")] -pub use io::{ReadBytesExt, WriteBytesExt}; +pub use crate::io::{ReadBytesExt, WriteBytesExt}; #[cfg(feature = "std")] mod io; -#[cfg(crux)] use core::mem::crucible_identity_transmute as identity_transmute; -#[cfg(not(crux))] use core::mem::transmute as identity_transmute; - #[inline] fn extend_sign(val: u64, nbytes: usize) -> i64 { let shift = (8 - nbytes) * 8; (val << shift) as i64 >> shift } -#[cfg(byteorder_i128)] #[inline] fn extend_sign128(val: u128, nbytes: usize) -> i128 { let shift = (16 - nbytes) * 8; @@ -115,7 +98,6 @@ fn unextend_sign(val: i64, nbytes: usize) -> u64 { (val << shift) as u64 >> shift } -#[cfg(byteorder_i128)] #[inline] fn unextend_sign128(val: i128, nbytes: usize) -> u128 { let shift = (16 - nbytes) * 8; @@ -143,7 +125,6 @@ fn pack_size(n: u64) -> usize { } } -#[cfg(byteorder_i128)] #[inline] fn pack_size128(n: u128) -> usize { if n < 1 << 8 { @@ -184,7 +165,7 @@ fn pack_size128(n: u128) -> usize { mod private { /// Sealed stops crates other than byteorder from implementing any traits /// that use it. - pub trait Sealed{} + pub trait Sealed {} impl Sealed for super::LittleEndian {} impl Sealed for super::BigEndian {} } @@ -224,8 +205,16 @@ mod private { /// /// [`BigEndian`]: enum.BigEndian.html /// [`LittleEndian`]: enum.LittleEndian.html -pub trait ByteOrder - : Clone + Copy + Debug + Default + Eq + Hash + Ord + PartialEq + PartialOrd +pub trait ByteOrder: + Clone + + Copy + + Debug + + Default + + Eq + + Hash + + Ord + + PartialEq + + PartialOrd + private::Sealed { /// Reads an unsigned 16 bit integer from `buf`. @@ -332,7 +321,6 @@ pub trait ByteOrder /// LittleEndian::write_u128(&mut buf, 1_000_000); /// assert_eq!(1_000_000, LittleEndian::read_u128(&buf)); /// ``` - #[cfg(byteorder_i128)] fn read_u128(buf: &[u8]) -> u128; /// Reads an unsigned n-bytes integer from `buf`. @@ -373,7 +361,6 @@ pub trait ByteOrder /// LittleEndian::write_uint128(&mut buf, 1_000_000, 3); /// assert_eq!(1_000_000, LittleEndian::read_uint128(&buf, 3)); /// ``` - #[cfg(byteorder_i128)] fn read_uint128(buf: &[u8], nbytes: usize) -> u128; /// Writes an unsigned 16 bit integer `n` to `buf`. @@ -492,7 +479,6 @@ pub trait ByteOrder /// LittleEndian::write_u128(&mut buf, 1_000_000); /// assert_eq!(1_000_000, LittleEndian::read_u128(&buf)); /// ``` - #[cfg(byteorder_i128)] fn write_u128(buf: &mut [u8], n: u128); /// Writes an unsigned integer `n` to `buf` using only `nbytes`. @@ -533,7 +519,6 @@ pub trait ByteOrder /// LittleEndian::write_uint128(&mut buf, 1_000_000, 3); /// assert_eq!(1_000_000, LittleEndian::read_uint128(&buf, 3)); /// ``` - #[cfg(byteorder_i128)] fn write_uint128(buf: &mut [u8], n: u128, nbytes: usize); /// Reads a signed 16 bit integer from `buf`. @@ -663,7 +648,6 @@ pub trait ByteOrder /// LittleEndian::write_i128(&mut buf, -1_000_000_000); /// assert_eq!(-1_000_000_000, LittleEndian::read_i128(&buf)); /// ``` - #[cfg(byteorder_i128)] #[inline] fn read_i128(buf: &[u8]) -> i128 { Self::read_u128(buf) as i128 @@ -710,7 +694,6 @@ pub trait ByteOrder /// LittleEndian::write_int128(&mut buf, -1_000, 3); /// assert_eq!(-1_000, LittleEndian::read_int128(&buf, 3)); /// ``` - #[cfg(byteorder_i128)] #[inline] fn read_int128(buf: &[u8], nbytes: usize) -> i128 { extend_sign128(Self::read_uint128(buf, nbytes), nbytes) @@ -736,7 +719,7 @@ pub trait ByteOrder /// ``` #[inline] fn read_f32(buf: &[u8]) -> f32 { - unimplemented!() + f32::from_bits(Self::read_u32(buf)) } /// Reads a IEEE754 double-precision (8 bytes) floating point number. @@ -759,7 +742,7 @@ pub trait ByteOrder /// ``` #[inline] fn read_f64(buf: &[u8]) -> f64 { - unimplemented!() + f64::from_bits(Self::read_u64(buf)) } /// Writes a signed 16 bit integer `n` to `buf`. @@ -889,7 +872,6 @@ pub trait ByteOrder /// LittleEndian::write_i128(&mut buf, -1_000_000_000); /// assert_eq!(-1_000_000_000, LittleEndian::read_i128(&buf)); /// ``` - #[cfg(byteorder_i128)] #[inline] fn write_i128(buf: &mut [u8], n: i128) { Self::write_u128(buf, n as u128) @@ -936,7 +918,6 @@ pub trait ByteOrder /// LittleEndian::write_int128(&mut buf, -1_000, 3); /// assert_eq!(-1_000, LittleEndian::read_int128(&buf, 3)); /// ``` - #[cfg(byteorder_i128)] #[inline] fn write_int128(buf: &mut [u8], n: i128, nbytes: usize) { Self::write_uint128(buf, unextend_sign128(n, nbytes), nbytes) @@ -962,7 +943,7 @@ pub trait ByteOrder /// ``` #[inline] fn write_f32(buf: &mut [u8], n: f32) { - unimplemented!() + Self::write_u32(buf, n.to_bits()) } /// Writes a IEEE754 double-precision (8 bytes) floating point number. @@ -985,7 +966,7 @@ pub trait ByteOrder /// ``` #[inline] fn write_f64(buf: &mut [u8], n: f64) { - unimplemented!() + Self::write_u64(buf, n.to_bits()) } /// Reads unsigned 16 bit integers from `src` into `dst`. @@ -1078,7 +1059,6 @@ pub trait ByteOrder /// LittleEndian::read_u128_into(&bytes, &mut numbers_got); /// assert_eq!(numbers_given, numbers_got); /// ``` - #[cfg(byteorder_i128)] fn read_u128_into(src: &[u8], dst: &mut [u128]); /// Reads signed 16 bit integers from `src` to `dst`. @@ -1105,7 +1085,7 @@ pub trait ByteOrder #[inline] fn read_i16_into(src: &[u8], dst: &mut [i16]) { let dst = unsafe { - identity_transmute::<&mut [i16], &mut [u16]>(dst) + slice::from_raw_parts_mut(dst.as_mut_ptr() as *mut u16, dst.len()) }; Self::read_u16_into(src, dst) } @@ -1134,7 +1114,7 @@ pub trait ByteOrder #[inline] fn read_i32_into(src: &[u8], dst: &mut [i32]) { let dst = unsafe { - identity_transmute::<&mut [i32], &mut [u32]>(dst) + slice::from_raw_parts_mut(dst.as_mut_ptr() as *mut u32, dst.len()) }; Self::read_u32_into(src, dst); } @@ -1163,7 +1143,7 @@ pub trait ByteOrder #[inline] fn read_i64_into(src: &[u8], dst: &mut [i64]) { let dst = unsafe { - identity_transmute::<&mut [i64], &mut [u64]>(dst) + slice::from_raw_parts_mut(dst.as_mut_ptr() as *mut u64, dst.len()) }; Self::read_u64_into(src, dst); } @@ -1189,11 +1169,10 @@ pub trait ByteOrder /// LittleEndian::read_i128_into(&bytes, &mut numbers_got); /// assert_eq!(numbers_given, numbers_got); /// ``` - #[cfg(byteorder_i128)] #[inline] fn read_i128_into(src: &[u8], dst: &mut [i128]) { let dst = unsafe { - identity_transmute::<&mut [i128], &mut [u128]>(dst) + slice::from_raw_parts_mut(dst.as_mut_ptr() as *mut u128, dst.len()) }; Self::read_u128_into(src, dst); } @@ -1222,7 +1201,10 @@ pub trait ByteOrder /// ``` #[inline] fn read_f32_into(src: &[u8], dst: &mut [f32]) { - unimplemented!() + let dst = unsafe { + slice::from_raw_parts_mut(dst.as_mut_ptr() as *mut u32, dst.len()) + }; + Self::read_u32_into(src, dst); } /// **DEPRECATED**. @@ -1251,7 +1233,7 @@ pub trait ByteOrder /// assert_eq!(numbers_given, numbers_got); /// ``` #[inline] - #[deprecated(since="1.3.0", note="please use `read_f32_into` instead")] + #[deprecated(since = "1.3.0", note = "please use `read_f32_into` instead")] fn read_f32_into_unchecked(src: &[u8], dst: &mut [f32]) { Self::read_f32_into(src, dst); } @@ -1280,7 +1262,10 @@ pub trait ByteOrder /// ``` #[inline] fn read_f64_into(src: &[u8], dst: &mut [f64]) { - unimplemented!() + let dst = unsafe { + slice::from_raw_parts_mut(dst.as_mut_ptr() as *mut u64, dst.len()) + }; + Self::read_u64_into(src, dst); } /// **DEPRECATED**. @@ -1310,7 +1295,7 @@ pub trait ByteOrder /// assert_eq!(numbers_given, numbers_got); /// ``` #[inline] - #[deprecated(since="1.3.0", note="please use `read_f64_into` instead")] + #[deprecated(since = "1.3.0", note = "please use `read_f64_into` instead")] fn read_f64_into_unchecked(src: &[u8], dst: &mut [f64]) { Self::read_f64_into(src, dst); } @@ -1405,9 +1390,42 @@ pub trait ByteOrder /// LittleEndian::read_u128_into(&bytes, &mut numbers_got); /// assert_eq!(numbers_given, numbers_got); /// ``` - #[cfg(byteorder_i128)] fn write_u128_into(src: &[u128], dst: &mut [u8]); + /// Writes signed 8 bit integers from `src` into `dst`. + /// + /// Note that since each `i8` is a single byte, no byte order conversions + /// are used. This method is included because it provides a safe, simple + /// way for the caller to write from a `&[i8]` buffer. (Without this + /// method, the caller would have to either use `unsafe` code or convert + /// each byte to `u8` individually.) + /// + /// # Panics + /// + /// Panics when `buf.len() != src.len()`. + /// + /// # Examples + /// + /// Write and read `i8` numbers in little endian order: + /// + /// ```rust + /// use byteorder::{ByteOrder, LittleEndian, ReadBytesExt}; + /// + /// let mut bytes = [0; 4]; + /// let numbers_given = [1, 2, 0xf, 0xe]; + /// LittleEndian::write_i8_into(&numbers_given, &mut bytes); + /// + /// let mut numbers_got = [0; 4]; + /// bytes.as_ref().read_i8_into(&mut numbers_got); + /// assert_eq!(numbers_given, numbers_got); + /// ``` + fn write_i8_into(src: &[i8], dst: &mut [u8]) { + let src = unsafe { + slice::from_raw_parts(src.as_ptr() as *const u8, src.len()) + }; + dst.copy_from_slice(src); + } + /// Writes signed 16 bit integers from `src` into `dst`. /// /// # Panics @@ -1431,7 +1449,7 @@ pub trait ByteOrder /// ``` fn write_i16_into(src: &[i16], dst: &mut [u8]) { let src = unsafe { - identity_transmute::<&[i16], &[u16]>(src) + slice::from_raw_parts(src.as_ptr() as *const u16, src.len()) }; Self::write_u16_into(src, dst); } @@ -1459,7 +1477,7 @@ pub trait ByteOrder /// ``` fn write_i32_into(src: &[i32], dst: &mut [u8]) { let src = unsafe { - identity_transmute::<&[i32], &[u32]>(src) + slice::from_raw_parts(src.as_ptr() as *const u32, src.len()) }; Self::write_u32_into(src, dst); } @@ -1487,7 +1505,7 @@ pub trait ByteOrder /// ``` fn write_i64_into(src: &[i64], dst: &mut [u8]) { let src = unsafe { - identity_transmute::<&[i64], &[u64]>(src) + slice::from_raw_parts(src.as_ptr() as *const u64, src.len()) }; Self::write_u64_into(src, dst); } @@ -1513,10 +1531,9 @@ pub trait ByteOrder /// LittleEndian::read_i128_into(&bytes, &mut numbers_got); /// assert_eq!(numbers_given, numbers_got); /// ``` - #[cfg(byteorder_i128)] fn write_i128_into(src: &[i128], dst: &mut [u8]) { let src = unsafe { - identity_transmute::<&[i128], &[u128]>(src) + slice::from_raw_parts(src.as_ptr() as *const u128, src.len()) }; Self::write_u128_into(src, dst); } @@ -1540,13 +1557,14 @@ pub trait ByteOrder /// LittleEndian::write_f32_into(&numbers_given, &mut bytes); /// /// let mut numbers_got = [0.0; 4]; - /// unsafe { - /// LittleEndian::read_f32_into(&bytes, &mut numbers_got); - /// } + /// LittleEndian::read_f32_into(&bytes, &mut numbers_got); /// assert_eq!(numbers_given, numbers_got); /// ``` fn write_f32_into(src: &[f32], dst: &mut [u8]) { - unimplemented!() + let src = unsafe { + slice::from_raw_parts(src.as_ptr() as *const u32, src.len()) + }; + Self::write_u32_into(src, dst); } /// Writes IEEE754 double-precision (8 bytes) floating point numbers from @@ -1568,13 +1586,14 @@ pub trait ByteOrder /// LittleEndian::write_f64_into(&numbers_given, &mut bytes); /// /// let mut numbers_got = [0.0; 4]; - /// unsafe { - /// LittleEndian::read_f64_into(&bytes, &mut numbers_got); - /// } + /// LittleEndian::read_f64_into(&bytes, &mut numbers_got); /// assert_eq!(numbers_given, numbers_got); /// ``` fn write_f64_into(src: &[f64], dst: &mut [u8]) { - unimplemented!() + let src = unsafe { + slice::from_raw_parts(src.as_ptr() as *const u64, src.len()) + }; + Self::write_u64_into(src, dst); } /// Converts the given slice of unsigned 16 bit integers to a particular @@ -1651,7 +1670,6 @@ pub trait ByteOrder /// BigEndian::from_slice_u128(&mut numbers); /// assert_eq!(numbers, [5u128.to_be(), 65000u128.to_be()]); /// ``` - #[cfg(byteorder_i128)] fn from_slice_u128(numbers: &mut [u128]); /// Converts the given slice of signed 16 bit integers to a particular @@ -1674,7 +1692,7 @@ pub trait ByteOrder #[inline] fn from_slice_i16(src: &mut [i16]) { let src = unsafe { - identity_transmute::<&mut [i16], &mut [u16]>(src) + slice::from_raw_parts_mut(src.as_mut_ptr() as *mut u16, src.len()) }; Self::from_slice_u16(src); } @@ -1699,7 +1717,7 @@ pub trait ByteOrder #[inline] fn from_slice_i32(src: &mut [i32]) { let src = unsafe { - identity_transmute::<&mut [i32], &mut [u32]>(src) + slice::from_raw_parts_mut(src.as_mut_ptr() as *mut u32, src.len()) }; Self::from_slice_u32(src); } @@ -1724,7 +1742,7 @@ pub trait ByteOrder #[inline] fn from_slice_i64(src: &mut [i64]) { let src = unsafe { - identity_transmute::<&mut [i64], &mut [u64]>(src) + slice::from_raw_parts_mut(src.as_mut_ptr() as *mut u64, src.len()) }; Self::from_slice_u64(src); } @@ -1746,11 +1764,10 @@ pub trait ByteOrder /// BigEndian::from_slice_i128(&mut numbers); /// assert_eq!(numbers, [5i128.to_be(), 65000i128.to_be()]); /// ``` - #[cfg(byteorder_i128)] #[inline] fn from_slice_i128(src: &mut [i128]) { let src = unsafe { - identity_transmute::<&mut [i128], &mut [u128]>(src) + slice::from_raw_parts_mut(src.as_mut_ptr() as *mut u128, src.len()) }; Self::from_slice_u128(src); } @@ -1878,169 +1895,231 @@ pub type NativeEndian = LittleEndian; #[cfg(target_endian = "big")] pub type NativeEndian = BigEndian; -macro_rules! safe_read_bytes { - ($src:expr, $ty:ty, $range:expr) => ({ - let mut result: $ty = 0; - for i in $range { - result = (result << 8) | ($src[i] as $ty); +/// Copies $size bytes from a number $n to a &mut [u8] $dst. $ty represents the +/// numeric type of $n and $which must be either to_be or to_le, depending on +/// which endianness one wants to use when writing to $dst. +/// +/// This macro is only safe to call when $ty is a numeric type and $size == +/// size_of::<$ty>() and where $dst is a &mut [u8]. +macro_rules! unsafe_write_num_bytes { + ($ty:ty, $size:expr, $n:expr, $dst:expr, $which:ident) => {{ + assert!($size <= $dst.len()); + unsafe { + // N.B. https://github.com/rust-lang/rust/issues/22776 + let bytes = *(&$n.$which() as *const _ as *const [u8; $size]); + copy_nonoverlapping((&bytes).as_ptr(), $dst.as_mut_ptr(), $size); } - result - }); + }}; } -macro_rules! safe_write_bytes { - ($val:expr, $dst:expr, $range:expr) => ({ - let mut j = 0; - for i in ($range).rev() { - $dst[i] = ($val >> (j * 8)) as u8; - j += 1; +/// Copies a &[u8] $src into a &mut [] $dst for the endianness given +/// by $which (must be either to_be or to_le). +/// +/// This macro is only safe to call when $src and $dst are &[u8] and &mut [u8], +/// respectively. The macro will panic if $src.len() != $size * $dst.len(), +/// where $size represents the size of the integers encoded in $src. +macro_rules! unsafe_read_slice { + ($src:expr, $dst:expr, $size:expr, $which:ident) => {{ + assert_eq!($src.len(), $size * $dst.len()); + + unsafe { + copy_nonoverlapping( + $src.as_ptr(), + $dst.as_mut_ptr() as *mut u8, + $src.len(), + ); } - }); -} - -macro_rules! safe_read_bytes_rev { - ($src:expr, $ty:ty, $range:expr) => (safe_read_bytes!($src, $ty, $range.rev())); -} - -macro_rules! safe_write_bytes_rev { - ($val:expr, $dst:expr, $range:expr) => (safe_write_bytes!($val, $dst, $range.rev())); + for v in $dst.iter_mut() { + *v = v.$which(); + } + }}; } -macro_rules! safe_read_slice { - ($src:expr, $dst:expr, $read_one:ident, $step:expr) => ({ - assert!($src.len() == $dst.len() * $step); - let mut base = 0; - for one_dst in $dst { - let one_src = &$src[base .. base + $step]; - *one_dst = Self::$read_one(one_src); - base += $step; +/// Copies a &[$ty] $src into a &mut [u8] $dst, where $ty must be a numeric +/// type. This panics if size_of::<$ty>() * $src.len() != $dst.len(). +/// +/// This macro is only safe to call when $src is a slice of numeric types and +/// $dst is a &mut [u8] and where $ty represents the type of the integers in +/// $src. +macro_rules! unsafe_write_slice_native { + ($src:expr, $dst:expr, $ty:ty) => {{ + let size = core::mem::size_of::<$ty>(); + assert_eq!(size * $src.len(), $dst.len()); + + unsafe { + copy_nonoverlapping( + $src.as_ptr() as *const u8, + $dst.as_mut_ptr(), + $dst.len(), + ); } - }); + }}; } -macro_rules! safe_write_slice { - ($src:expr, $dst:expr, $write_one:ident, $step:expr) => ({ - assert!($dst.len() == $src.len() * $step); - let mut base = 0; - for one_src in $src { - let one_dst = &mut $dst[base .. base + $step]; - Self::$write_one(one_dst, *one_src); - base += $step; +macro_rules! write_slice { + ($src:expr, $dst:expr, $ty:ty, $size:expr, $write:expr) => {{ + assert!($size == ::core::mem::size_of::<$ty>()); + assert_eq!($size * $src.len(), $dst.len()); + + for (&n, chunk) in $src.iter().zip($dst.chunks_mut($size)) { + $write(chunk, n); } - }); + }}; } impl ByteOrder for BigEndian { #[inline] fn read_u16(buf: &[u8]) -> u16 { - safe_read_bytes!(buf, u16, 0..2) + u16::from_be_bytes(buf[..2].try_into().unwrap()) } #[inline] fn read_u32(buf: &[u8]) -> u32 { - safe_read_bytes!(buf, u32, 0..4) + u32::from_be_bytes(buf[..4].try_into().unwrap()) } #[inline] fn read_u64(buf: &[u8]) -> u64 { - safe_read_bytes!(buf, u64, 0..8) + u64::from_be_bytes(buf[..8].try_into().unwrap()) } - #[cfg(byteorder_i128)] #[inline] fn read_u128(buf: &[u8]) -> u128 { - safe_read_bytes!(buf, u128, 0..16) + u128::from_be_bytes(buf[..16].try_into().unwrap()) } #[inline] fn read_uint(buf: &[u8], nbytes: usize) -> u64 { assert!(1 <= nbytes && nbytes <= 8 && nbytes <= buf.len()); - safe_read_bytes!(buf, u64, 0..nbytes) + let mut out = 0u64; + let ptr_out = &mut out as *mut u64 as *mut u8; + unsafe { + copy_nonoverlapping( + buf.as_ptr(), + ptr_out.offset((8 - nbytes) as isize), + nbytes, + ); + } + out.to_be() } - #[cfg(byteorder_i128)] #[inline] fn read_uint128(buf: &[u8], nbytes: usize) -> u128 { assert!(1 <= nbytes && nbytes <= 16 && nbytes <= buf.len()); - safe_read_bytes!(buf, u128, 0..nbytes) + let mut out: u128 = 0; + let ptr_out = &mut out as *mut u128 as *mut u8; + unsafe { + copy_nonoverlapping( + buf.as_ptr(), + ptr_out.offset((16 - nbytes) as isize), + nbytes, + ); + } + out.to_be() } #[inline] fn write_u16(buf: &mut [u8], n: u16) { - safe_write_bytes!(n, buf, 0..2) + unsafe_write_num_bytes!(u16, 2, n, buf, to_be); } #[inline] fn write_u32(buf: &mut [u8], n: u32) { - safe_write_bytes!(n, buf, 0..4) + unsafe_write_num_bytes!(u32, 4, n, buf, to_be); } #[inline] fn write_u64(buf: &mut [u8], n: u64) { - safe_write_bytes!(n, buf, 0..8) + unsafe_write_num_bytes!(u64, 8, n, buf, to_be); } - #[cfg(byteorder_i128)] #[inline] fn write_u128(buf: &mut [u8], n: u128) { - safe_write_bytes!(n, buf, 0..16) + unsafe_write_num_bytes!(u128, 16, n, buf, to_be); } #[inline] fn write_uint(buf: &mut [u8], n: u64, nbytes: usize) { assert!(pack_size(n) <= nbytes && nbytes <= 8); assert!(nbytes <= buf.len()); - safe_write_bytes!(n, buf, 0..nbytes) + unsafe { + let bytes = *(&n.to_be() as *const u64 as *const [u8; 8]); + copy_nonoverlapping( + bytes.as_ptr().offset((8 - nbytes) as isize), + buf.as_mut_ptr(), + nbytes, + ); + } } - #[cfg(byteorder_i128)] #[inline] fn write_uint128(buf: &mut [u8], n: u128, nbytes: usize) { assert!(pack_size128(n) <= nbytes && nbytes <= 16); assert!(nbytes <= buf.len()); - safe_write_bytes!(n, buf, 0..nbytes) + unsafe { + let bytes = *(&n.to_be() as *const u128 as *const [u8; 16]); + copy_nonoverlapping( + bytes.as_ptr().offset((16 - nbytes) as isize), + buf.as_mut_ptr(), + nbytes, + ); + } } #[inline] fn read_u16_into(src: &[u8], dst: &mut [u16]) { - safe_read_slice!(src, dst, read_u16, 2); + unsafe_read_slice!(src, dst, 2, to_be); } #[inline] fn read_u32_into(src: &[u8], dst: &mut [u32]) { - safe_read_slice!(src, dst, read_u32, 4); + unsafe_read_slice!(src, dst, 4, to_be); } #[inline] fn read_u64_into(src: &[u8], dst: &mut [u64]) { - safe_read_slice!(src, dst, read_u64, 8); + unsafe_read_slice!(src, dst, 8, to_be); } - #[cfg(byteorder_i128)] #[inline] fn read_u128_into(src: &[u8], dst: &mut [u128]) { - safe_read_slice!(src, dst, read_u128, 16); + unsafe_read_slice!(src, dst, 16, to_be); } #[inline] fn write_u16_into(src: &[u16], dst: &mut [u8]) { - safe_write_slice!(src, dst, write_u16, 2); + if cfg!(target_endian = "big") { + unsafe_write_slice_native!(src, dst, u16); + } else { + write_slice!(src, dst, u16, 2, Self::write_u16); + } } #[inline] fn write_u32_into(src: &[u32], dst: &mut [u8]) { - safe_write_slice!(src, dst, write_u32, 4); + if cfg!(target_endian = "big") { + unsafe_write_slice_native!(src, dst, u32); + } else { + write_slice!(src, dst, u32, 4, Self::write_u32); + } } #[inline] fn write_u64_into(src: &[u64], dst: &mut [u8]) { - safe_write_slice!(src, dst, write_u64, 8); + if cfg!(target_endian = "big") { + unsafe_write_slice_native!(src, dst, u64); + } else { + write_slice!(src, dst, u64, 8, Self::write_u64); + } } - #[cfg(byteorder_i128)] #[inline] fn write_u128_into(src: &[u128], dst: &mut [u8]) { - safe_write_slice!(src, dst, write_u128, 16); + if cfg!(target_endian = "big") { + unsafe_write_slice_native!(src, dst, u128); + } else { + write_slice!(src, dst, u128, 16, Self::write_u128); + } } #[inline] @@ -2070,7 +2149,6 @@ impl ByteOrder for BigEndian { } } - #[cfg(byteorder_i128)] #[inline] fn from_slice_u128(numbers: &mut [u128]) { if cfg!(target_endian = "little") { @@ -2082,126 +2160,166 @@ impl ByteOrder for BigEndian { #[inline] fn from_slice_f32(numbers: &mut [f32]) { - unimplemented!() + if cfg!(target_endian = "little") { + for n in numbers { + unsafe { + let int = *(n as *const f32 as *const u32); + *n = *(&int.to_be() as *const u32 as *const f32); + } + } + } } #[inline] fn from_slice_f64(numbers: &mut [f64]) { - unimplemented!() + if cfg!(target_endian = "little") { + for n in numbers { + unsafe { + let int = *(n as *const f64 as *const u64); + *n = *(&int.to_be() as *const u64 as *const f64); + } + } + } } } impl ByteOrder for LittleEndian { #[inline] fn read_u16(buf: &[u8]) -> u16 { - safe_read_bytes_rev!(buf, u16, 0..2) + u16::from_le_bytes(buf[..2].try_into().unwrap()) } #[inline] fn read_u32(buf: &[u8]) -> u32 { - safe_read_bytes_rev!(buf, u32, 0..4) + u32::from_le_bytes(buf[..4].try_into().unwrap()) } #[inline] fn read_u64(buf: &[u8]) -> u64 { - safe_read_bytes_rev!(buf, u64, 0..8) + u64::from_le_bytes(buf[..8].try_into().unwrap()) } - #[cfg(byteorder_i128)] #[inline] fn read_u128(buf: &[u8]) -> u128 { - safe_read_bytes_rev!(buf, u128, 0..16) + u128::from_le_bytes(buf[..16].try_into().unwrap()) } #[inline] fn read_uint(buf: &[u8], nbytes: usize) -> u64 { assert!(1 <= nbytes && nbytes <= 8 && nbytes <= buf.len()); - safe_read_bytes_rev!(buf, u64, 0..nbytes) + let mut out = 0u64; + let ptr_out = &mut out as *mut u64 as *mut u8; + unsafe { + copy_nonoverlapping(buf.as_ptr(), ptr_out, nbytes); + } + out.to_le() } - #[cfg(byteorder_i128)] #[inline] fn read_uint128(buf: &[u8], nbytes: usize) -> u128 { assert!(1 <= nbytes && nbytes <= 16 && nbytes <= buf.len()); - safe_read_bytes_rev!(buf, u128, 0..nbytes) + let mut out: u128 = 0; + let ptr_out = &mut out as *mut u128 as *mut u8; + unsafe { + copy_nonoverlapping(buf.as_ptr(), ptr_out, nbytes); + } + out.to_le() } #[inline] fn write_u16(buf: &mut [u8], n: u16) { - safe_write_bytes_rev!(n, buf, 0..2) + unsafe_write_num_bytes!(u16, 2, n, buf, to_le); } #[inline] fn write_u32(buf: &mut [u8], n: u32) { - safe_write_bytes_rev!(n, buf, 0..4) + unsafe_write_num_bytes!(u32, 4, n, buf, to_le); } #[inline] fn write_u64(buf: &mut [u8], n: u64) { - safe_write_bytes_rev!(n, buf, 0..8) + unsafe_write_num_bytes!(u64, 8, n, buf, to_le); } - #[cfg(byteorder_i128)] #[inline] fn write_u128(buf: &mut [u8], n: u128) { - safe_write_bytes_rev!(n, buf, 0..16) + unsafe_write_num_bytes!(u128, 16, n, buf, to_le); } #[inline] fn write_uint(buf: &mut [u8], n: u64, nbytes: usize) { - assert!(pack_size(n) <= nbytes && nbytes <= 8); + assert!(pack_size(n as u64) <= nbytes && nbytes <= 8); assert!(nbytes <= buf.len()); - safe_write_bytes_rev!(n, buf, 0..nbytes) + unsafe { + let bytes = *(&n.to_le() as *const u64 as *const [u8; 8]); + copy_nonoverlapping(bytes.as_ptr(), buf.as_mut_ptr(), nbytes); + } } - #[cfg(byteorder_i128)] #[inline] fn write_uint128(buf: &mut [u8], n: u128, nbytes: usize) { - assert!(pack_size128(n) <= nbytes && nbytes <= 16); + assert!(pack_size128(n as u128) <= nbytes && nbytes <= 16); assert!(nbytes <= buf.len()); - safe_write_bytes_rev!(n, buf, 0..nbytes) + unsafe { + let bytes = *(&n.to_le() as *const u128 as *const [u8; 16]); + copy_nonoverlapping(bytes.as_ptr(), buf.as_mut_ptr(), nbytes); + } } #[inline] fn read_u16_into(src: &[u8], dst: &mut [u16]) { - safe_read_slice!(src, dst, read_u16, 2); + unsafe_read_slice!(src, dst, 2, to_le); } #[inline] fn read_u32_into(src: &[u8], dst: &mut [u32]) { - safe_read_slice!(src, dst, read_u32, 4); + unsafe_read_slice!(src, dst, 4, to_le); } #[inline] fn read_u64_into(src: &[u8], dst: &mut [u64]) { - safe_read_slice!(src, dst, read_u64, 8); + unsafe_read_slice!(src, dst, 8, to_le); } - #[cfg(byteorder_i128)] #[inline] fn read_u128_into(src: &[u8], dst: &mut [u128]) { - safe_read_slice!(src, dst, read_u128, 16); + unsafe_read_slice!(src, dst, 16, to_le); } #[inline] fn write_u16_into(src: &[u16], dst: &mut [u8]) { - safe_write_slice!(src, dst, write_u16, 2); + if cfg!(target_endian = "little") { + unsafe_write_slice_native!(src, dst, u16); + } else { + write_slice!(src, dst, u16, 2, Self::write_u16); + } } #[inline] fn write_u32_into(src: &[u32], dst: &mut [u8]) { - safe_write_slice!(src, dst, write_u32, 4); + if cfg!(target_endian = "little") { + unsafe_write_slice_native!(src, dst, u32); + } else { + write_slice!(src, dst, u32, 4, Self::write_u32); + } } #[inline] fn write_u64_into(src: &[u64], dst: &mut [u8]) { - safe_write_slice!(src, dst, write_u64, 8); + if cfg!(target_endian = "little") { + unsafe_write_slice_native!(src, dst, u64); + } else { + write_slice!(src, dst, u64, 8, Self::write_u64); + } } - #[cfg(byteorder_i128)] #[inline] fn write_u128_into(src: &[u128], dst: &mut [u8]) { - safe_write_slice!(src, dst, write_u128, 16); + if cfg!(target_endian = "little") { + unsafe_write_slice_native!(src, dst, u128); + } else { + write_slice!(src, dst, u128, 16, Self::write_u128); + } } #[inline] @@ -2231,7 +2349,6 @@ impl ByteOrder for LittleEndian { } } - #[cfg(byteorder_i128)] #[inline] fn from_slice_u128(numbers: &mut [u128]) { if cfg!(target_endian = "big") { @@ -2243,26 +2360,33 @@ impl ByteOrder for LittleEndian { #[inline] fn from_slice_f32(numbers: &mut [f32]) { - unimplemented!() + if cfg!(target_endian = "big") { + for n in numbers { + unsafe { + let int = *(n as *const f32 as *const u32); + *n = *(&int.to_le() as *const u32 as *const f32); + } + } + } } #[inline] fn from_slice_f64(numbers: &mut [f64]) { - unimplemented!() + if cfg!(target_endian = "big") { + for n in numbers { + unsafe { + let int = *(n as *const f64 as *const u64); + *n = *(&int.to_le() as *const u64 as *const f64); + } + } + } } } #[cfg(test)] mod test { - extern crate quickcheck; - extern crate rand; - - use self::quickcheck::{QuickCheck, StdGen, Testable}; - use self::rand::thread_rng; - #[cfg(byteorder_i128)] - use self::rand::Rng; - #[cfg(byteorder_i128)] - use self::quickcheck::{Arbitrary, Gen}; + use quickcheck::{Arbitrary, Gen, QuickCheck, StdGen, Testable}; + use rand::{thread_rng, Rng}; pub const U24_MAX: u32 = 16_777_215; pub const I24_MAX: i32 = 8_388_607; @@ -2273,7 +2397,9 @@ mod test { pub const I64_MAX: u64 = ::core::i64::MAX as u64; macro_rules! calc_max { - ($max:expr, $bytes:expr) => { calc_max!($max, $bytes, 8) }; + ($max:expr, $bytes:expr) => { + calc_max!($max, $bytes, 8) + }; ($max:expr, $bytes:expr, $maxbytes:expr) => { ($max - 1) >> (8 * ($maxbytes - $bytes)) }; @@ -2282,7 +2408,6 @@ mod test { #[derive(Clone, Debug)] pub struct Wi128(pub T); - #[cfg(byteorder_i128)] impl Wi128 { pub fn clone(&self) -> T { self.0.clone() @@ -2295,24 +2420,20 @@ mod test { } } - #[cfg(byteorder_i128)] impl Arbitrary for Wi128 { fn arbitrary(gen: &mut G) -> Wi128 { let max = calc_max!(::core::u128::MAX, gen.size(), 16); - let output = - (gen.gen::() as u128) | - ((gen.gen::() as u128) << 64); + let output = (gen.gen::() as u128) + | ((gen.gen::() as u128) << 64); Wi128(output & (max - 1)) } } - #[cfg(byteorder_i128)] impl Arbitrary for Wi128 { fn arbitrary(gen: &mut G) -> Wi128 { let max = calc_max!(::core::i128::MAX, gen.size(), 16); - let output = - (gen.gen::() as i128) | - ((gen.gen::() as i128) << 64); + let output = (gen.gen::() as i128) + | ((gen.gen::() as i128) << 64); Wi128(output & (max - 1)) } } @@ -2327,17 +2448,20 @@ mod test { macro_rules! qc_byte_order { ($name:ident, $ty_int:ty, $max:expr, - $bytes:expr, $read:ident, $write:ident) => ( + $bytes:expr, $read:ident, $write:ident) => { mod $name { - use {BigEndian, ByteOrder, NativeEndian, LittleEndian}; - #[allow(unused_imports)] use super::{ qc_sized, Wi128 }; + #[allow(unused_imports)] + use super::{qc_sized, Wi128}; + use crate::{ + BigEndian, ByteOrder, LittleEndian, NativeEndian, + }; #[test] fn big_endian() { fn prop(n: $ty_int) -> bool { let mut buf = [0; 16]; BigEndian::$write(&mut buf, n.clone(), $bytes); - n == BigEndian::$read(&mut buf[..$bytes], $bytes) + n == BigEndian::$read(&buf[..$bytes], $bytes) } qc_sized(prop as fn($ty_int) -> bool, $max); } @@ -2347,7 +2471,7 @@ mod test { fn prop(n: $ty_int) -> bool { let mut buf = [0; 16]; LittleEndian::$write(&mut buf, n.clone(), $bytes); - n == LittleEndian::$read(&mut buf[..$bytes], $bytes) + n == LittleEndian::$read(&buf[..$bytes], $bytes) } qc_sized(prop as fn($ty_int) -> bool, $max); } @@ -2357,18 +2481,21 @@ mod test { fn prop(n: $ty_int) -> bool { let mut buf = [0; 16]; NativeEndian::$write(&mut buf, n.clone(), $bytes); - n == NativeEndian::$read(&mut buf[..$bytes], $bytes) + n == NativeEndian::$read(&buf[..$bytes], $bytes) } qc_sized(prop as fn($ty_int) -> bool, $max); } } - ); + }; ($name:ident, $ty_int:ty, $max:expr, - $read:ident, $write:ident) => ( + $read:ident, $write:ident) => { mod $name { + #[allow(unused_imports)] + use super::{qc_sized, Wi128}; + use crate::{ + BigEndian, ByteOrder, LittleEndian, NativeEndian, + }; use core::mem::size_of; - use {BigEndian, ByteOrder, NativeEndian, LittleEndian}; - #[allow(unused_imports)] use super::{ qc_sized, Wi128 }; #[test] fn big_endian() { @@ -2376,7 +2503,7 @@ mod test { let bytes = size_of::<$ty_int>(); let mut buf = [0; 16]; BigEndian::$write(&mut buf[16 - bytes..], n.clone()); - n == BigEndian::$read(&mut buf[16 - bytes..]) + n == BigEndian::$read(&buf[16 - bytes..]) } qc_sized(prop as fn($ty_int) -> bool, $max - 1); } @@ -2387,7 +2514,7 @@ mod test { let bytes = size_of::<$ty_int>(); let mut buf = [0; 16]; LittleEndian::$write(&mut buf[..bytes], n.clone()); - n == LittleEndian::$read(&mut buf[..bytes]) + n == LittleEndian::$read(&buf[..bytes]) } qc_sized(prop as fn($ty_int) -> bool, $max - 1); } @@ -2398,164 +2525,489 @@ mod test { let bytes = size_of::<$ty_int>(); let mut buf = [0; 16]; NativeEndian::$write(&mut buf[..bytes], n.clone()); - n == NativeEndian::$read(&mut buf[..bytes]) + n == NativeEndian::$read(&buf[..bytes]) } qc_sized(prop as fn($ty_int) -> bool, $max - 1); } } - ); - } - - qc_byte_order!(prop_u16, u16, ::core::u16::MAX as u64, read_u16, write_u16); - qc_byte_order!(prop_i16, i16, ::core::i16::MAX as u64, read_i16, write_i16); - qc_byte_order!(prop_u24, u32, ::test::U24_MAX as u64, read_u24, write_u24); - qc_byte_order!(prop_i24, i32, ::test::I24_MAX as u64, read_i24, write_i24); - qc_byte_order!(prop_u32, u32, ::core::u32::MAX as u64, read_u32, write_u32); - qc_byte_order!(prop_i32, i32, ::core::i32::MAX as u64, read_i32, write_i32); - qc_byte_order!(prop_u48, u64, ::test::U48_MAX as u64, read_u48, write_u48); - qc_byte_order!(prop_i48, i64, ::test::I48_MAX as u64, read_i48, write_i48); - qc_byte_order!(prop_u64, u64, ::core::u64::MAX as u64, read_u64, write_u64); - qc_byte_order!(prop_i64, i64, ::core::i64::MAX as u64, read_i64, write_i64); - //qc_byte_order!(prop_f32, f32, ::core::u64::MAX as u64, read_f32, write_f32); - //qc_byte_order!(prop_f64, f64, ::core::i64::MAX as u64, read_f64, write_f64); - - #[cfg(byteorder_i128)] + }; + } + + qc_byte_order!( + prop_u16, + u16, + ::core::u16::MAX as u64, + read_u16, + write_u16 + ); + qc_byte_order!( + prop_i16, + i16, + ::core::i16::MAX as u64, + read_i16, + write_i16 + ); + qc_byte_order!( + prop_u24, + u32, + crate::test::U24_MAX as u64, + read_u24, + write_u24 + ); + qc_byte_order!( + prop_i24, + i32, + crate::test::I24_MAX as u64, + read_i24, + write_i24 + ); + qc_byte_order!( + prop_u32, + u32, + ::core::u32::MAX as u64, + read_u32, + write_u32 + ); + qc_byte_order!( + prop_i32, + i32, + ::core::i32::MAX as u64, + read_i32, + write_i32 + ); + qc_byte_order!( + prop_u48, + u64, + crate::test::U48_MAX as u64, + read_u48, + write_u48 + ); + qc_byte_order!( + prop_i48, + i64, + crate::test::I48_MAX as u64, + read_i48, + write_i48 + ); + qc_byte_order!( + prop_u64, + u64, + ::core::u64::MAX as u64, + read_u64, + write_u64 + ); + qc_byte_order!( + prop_i64, + i64, + ::core::i64::MAX as u64, + read_i64, + write_i64 + ); + qc_byte_order!( + prop_f32, + f32, + ::core::u64::MAX as u64, + read_f32, + write_f32 + ); + qc_byte_order!( + prop_f64, + f64, + ::core::i64::MAX as u64, + read_f64, + write_f64 + ); + qc_byte_order!(prop_u128, Wi128, 16 + 1, read_u128, write_u128); - #[cfg(byteorder_i128)] qc_byte_order!(prop_i128, Wi128, 16 + 1, read_i128, write_i128); - qc_byte_order!(prop_uint_1, - u64, calc_max!(super::U64_MAX, 1), 1, read_uint, write_uint); - qc_byte_order!(prop_uint_2, - u64, calc_max!(super::U64_MAX, 2), 2, read_uint, write_uint); - qc_byte_order!(prop_uint_3, - u64, calc_max!(super::U64_MAX, 3), 3, read_uint, write_uint); - qc_byte_order!(prop_uint_4, - u64, calc_max!(super::U64_MAX, 4), 4, read_uint, write_uint); - qc_byte_order!(prop_uint_5, - u64, calc_max!(super::U64_MAX, 5), 5, read_uint, write_uint); - qc_byte_order!(prop_uint_6, - u64, calc_max!(super::U64_MAX, 6), 6, read_uint, write_uint); - qc_byte_order!(prop_uint_7, - u64, calc_max!(super::U64_MAX, 7), 7, read_uint, write_uint); - qc_byte_order!(prop_uint_8, - u64, calc_max!(super::U64_MAX, 8), 8, read_uint, write_uint); - - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_1, - Wi128, 1, 1, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_2, - Wi128, 2, 2, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_3, - Wi128, 3, 3, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_4, - Wi128, 4, 4, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_5, - Wi128, 5, 5, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_6, - Wi128, 6, 6, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_7, - Wi128, 7, 7, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_8, - Wi128, 8, 8, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_9, - Wi128, 9, 9, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_10, - Wi128, 10, 10, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_11, - Wi128, 11, 11, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_12, - Wi128, 12, 12, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_13, - Wi128, 13, 13, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_14, - Wi128, 14, 14, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_15, - Wi128, 15, 15, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_16, - Wi128, 16, 16, read_uint128, write_uint128); - - qc_byte_order!(prop_int_1, - i64, calc_max!(super::I64_MAX, 1), 1, read_int, write_int); - qc_byte_order!(prop_int_2, - i64, calc_max!(super::I64_MAX, 2), 2, read_int, write_int); - qc_byte_order!(prop_int_3, - i64, calc_max!(super::I64_MAX, 3), 3, read_int, write_int); - qc_byte_order!(prop_int_4, - i64, calc_max!(super::I64_MAX, 4), 4, read_int, write_int); - qc_byte_order!(prop_int_5, - i64, calc_max!(super::I64_MAX, 5), 5, read_int, write_int); - qc_byte_order!(prop_int_6, - i64, calc_max!(super::I64_MAX, 6), 6, read_int, write_int); - qc_byte_order!(prop_int_7, - i64, calc_max!(super::I64_MAX, 7), 7, read_int, write_int); - qc_byte_order!(prop_int_8, - i64, calc_max!(super::I64_MAX, 8), 8, read_int, write_int); - - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_1, - Wi128, 1, 1, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_2, - Wi128, 2, 2, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_3, - Wi128, 3, 3, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_4, - Wi128, 4, 4, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_5, - Wi128, 5, 5, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_6, - Wi128, 6, 6, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_7, - Wi128, 7, 7, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_8, - Wi128, 8, 8, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_9, - Wi128, 9, 9, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_10, - Wi128, 10, 10, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_11, - Wi128, 11, 11, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_12, - Wi128, 12, 12, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_13, - Wi128, 13, 13, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_14, - Wi128, 14, 14, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_15, - Wi128, 15, 15, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_16, - Wi128, 16, 16, read_int128, write_int128); - + qc_byte_order!( + prop_uint_1, + u64, + calc_max!(super::U64_MAX, 1), + 1, + read_uint, + write_uint + ); + qc_byte_order!( + prop_uint_2, + u64, + calc_max!(super::U64_MAX, 2), + 2, + read_uint, + write_uint + ); + qc_byte_order!( + prop_uint_3, + u64, + calc_max!(super::U64_MAX, 3), + 3, + read_uint, + write_uint + ); + qc_byte_order!( + prop_uint_4, + u64, + calc_max!(super::U64_MAX, 4), + 4, + read_uint, + write_uint + ); + qc_byte_order!( + prop_uint_5, + u64, + calc_max!(super::U64_MAX, 5), + 5, + read_uint, + write_uint + ); + qc_byte_order!( + prop_uint_6, + u64, + calc_max!(super::U64_MAX, 6), + 6, + read_uint, + write_uint + ); + qc_byte_order!( + prop_uint_7, + u64, + calc_max!(super::U64_MAX, 7), + 7, + read_uint, + write_uint + ); + qc_byte_order!( + prop_uint_8, + u64, + calc_max!(super::U64_MAX, 8), + 8, + read_uint, + write_uint + ); + + qc_byte_order!( + prop_uint128_1, + Wi128, + 1, + 1, + read_uint128, + write_uint128 + ); + qc_byte_order!( + prop_uint128_2, + Wi128, + 2, + 2, + read_uint128, + write_uint128 + ); + qc_byte_order!( + prop_uint128_3, + Wi128, + 3, + 3, + read_uint128, + write_uint128 + ); + qc_byte_order!( + prop_uint128_4, + Wi128, + 4, + 4, + read_uint128, + write_uint128 + ); + qc_byte_order!( + prop_uint128_5, + Wi128, + 5, + 5, + read_uint128, + write_uint128 + ); + qc_byte_order!( + prop_uint128_6, + Wi128, + 6, + 6, + read_uint128, + write_uint128 + ); + qc_byte_order!( + prop_uint128_7, + Wi128, + 7, + 7, + read_uint128, + write_uint128 + ); + qc_byte_order!( + prop_uint128_8, + Wi128, + 8, + 8, + read_uint128, + write_uint128 + ); + qc_byte_order!( + prop_uint128_9, + Wi128, + 9, + 9, + read_uint128, + write_uint128 + ); + qc_byte_order!( + prop_uint128_10, + Wi128, + 10, + 10, + read_uint128, + write_uint128 + ); + qc_byte_order!( + prop_uint128_11, + Wi128, + 11, + 11, + read_uint128, + write_uint128 + ); + qc_byte_order!( + prop_uint128_12, + Wi128, + 12, + 12, + read_uint128, + write_uint128 + ); + qc_byte_order!( + prop_uint128_13, + Wi128, + 13, + 13, + read_uint128, + write_uint128 + ); + qc_byte_order!( + prop_uint128_14, + Wi128, + 14, + 14, + read_uint128, + write_uint128 + ); + qc_byte_order!( + prop_uint128_15, + Wi128, + 15, + 15, + read_uint128, + write_uint128 + ); + qc_byte_order!( + prop_uint128_16, + Wi128, + 16, + 16, + read_uint128, + write_uint128 + ); + + qc_byte_order!( + prop_int_1, + i64, + calc_max!(super::I64_MAX, 1), + 1, + read_int, + write_int + ); + qc_byte_order!( + prop_int_2, + i64, + calc_max!(super::I64_MAX, 2), + 2, + read_int, + write_int + ); + qc_byte_order!( + prop_int_3, + i64, + calc_max!(super::I64_MAX, 3), + 3, + read_int, + write_int + ); + qc_byte_order!( + prop_int_4, + i64, + calc_max!(super::I64_MAX, 4), + 4, + read_int, + write_int + ); + qc_byte_order!( + prop_int_5, + i64, + calc_max!(super::I64_MAX, 5), + 5, + read_int, + write_int + ); + qc_byte_order!( + prop_int_6, + i64, + calc_max!(super::I64_MAX, 6), + 6, + read_int, + write_int + ); + qc_byte_order!( + prop_int_7, + i64, + calc_max!(super::I64_MAX, 7), + 7, + read_int, + write_int + ); + qc_byte_order!( + prop_int_8, + i64, + calc_max!(super::I64_MAX, 8), + 8, + read_int, + write_int + ); + + qc_byte_order!( + prop_int128_1, + Wi128, + 1, + 1, + read_int128, + write_int128 + ); + qc_byte_order!( + prop_int128_2, + Wi128, + 2, + 2, + read_int128, + write_int128 + ); + qc_byte_order!( + prop_int128_3, + Wi128, + 3, + 3, + read_int128, + write_int128 + ); + qc_byte_order!( + prop_int128_4, + Wi128, + 4, + 4, + read_int128, + write_int128 + ); + qc_byte_order!( + prop_int128_5, + Wi128, + 5, + 5, + read_int128, + write_int128 + ); + qc_byte_order!( + prop_int128_6, + Wi128, + 6, + 6, + read_int128, + write_int128 + ); + qc_byte_order!( + prop_int128_7, + Wi128, + 7, + 7, + read_int128, + write_int128 + ); + qc_byte_order!( + prop_int128_8, + Wi128, + 8, + 8, + read_int128, + write_int128 + ); + qc_byte_order!( + prop_int128_9, + Wi128, + 9, + 9, + read_int128, + write_int128 + ); + qc_byte_order!( + prop_int128_10, + Wi128, + 10, + 10, + read_int128, + write_int128 + ); + qc_byte_order!( + prop_int128_11, + Wi128, + 11, + 11, + read_int128, + write_int128 + ); + qc_byte_order!( + prop_int128_12, + Wi128, + 12, + 12, + read_int128, + write_int128 + ); + qc_byte_order!( + prop_int128_13, + Wi128, + 13, + 13, + read_int128, + write_int128 + ); + qc_byte_order!( + prop_int128_14, + Wi128, + 14, + 14, + read_int128, + write_int128 + ); + qc_byte_order!( + prop_int128_15, + Wi128, + 15, + 15, + read_int128, + write_int128 + ); + qc_byte_order!( + prop_int128_16, + Wi128, + 16, + 16, + read_int128, + write_int128 + ); // Test that all of the byte conversion functions panic when given a // buffer that is too small. @@ -2564,9 +3016,11 @@ mod test { // with a buffer overflow. macro_rules! too_small { ($name:ident, $maximally_small:expr, $zero:expr, - $read:ident, $write:ident) => ( + $read:ident, $write:ident) => { mod $name { - use {BigEndian, ByteOrder, NativeEndian, LittleEndian}; + use crate::{ + BigEndian, ByteOrder, LittleEndian, NativeEndian, + }; #[test] #[should_panic] @@ -2610,10 +3064,12 @@ mod test { NativeEndian::$write(&mut buf, $zero); } } - ); - ($name:ident, $maximally_small:expr, $read:ident) => ( + }; + ($name:ident, $maximally_small:expr, $read:ident) => { mod $name { - use {BigEndian, ByteOrder, NativeEndian, LittleEndian}; + use crate::{ + BigEndian, ByteOrder, LittleEndian, NativeEndian, + }; #[test] #[should_panic] @@ -2636,7 +3092,7 @@ mod test { NativeEndian::$read(&buf, $maximally_small + 1); } } - ); + }; } too_small!(small_u16, 1, 0, read_u16, write_u16); @@ -2645,11 +3101,9 @@ mod test { too_small!(small_i32, 3, 0, read_i32, write_i32); too_small!(small_u64, 7, 0, read_u64, write_u64); too_small!(small_i64, 7, 0, read_i64, write_i64); - //too_small!(small_f32, 3, 0.0, read_f32, write_f32); - //too_small!(small_f64, 7, 0.0, read_f64, write_f64); - #[cfg(byteorder_i128)] + too_small!(small_f32, 3, 0.0, read_f32, write_f32); + too_small!(small_f64, 7, 0.0, read_f64, write_f64); too_small!(small_u128, 15, 0, read_u128, write_u128); - #[cfg(byteorder_i128)] too_small!(small_i128, 15, 0, read_i128, write_i128); too_small!(small_uint_1, 1, read_uint); @@ -2660,35 +3114,20 @@ mod test { too_small!(small_uint_6, 6, read_uint); too_small!(small_uint_7, 7, read_uint); - #[cfg(byteorder_i128)] too_small!(small_uint128_1, 1, read_uint128); - #[cfg(byteorder_i128)] too_small!(small_uint128_2, 2, read_uint128); - #[cfg(byteorder_i128)] too_small!(small_uint128_3, 3, read_uint128); - #[cfg(byteorder_i128)] too_small!(small_uint128_4, 4, read_uint128); - #[cfg(byteorder_i128)] too_small!(small_uint128_5, 5, read_uint128); - #[cfg(byteorder_i128)] too_small!(small_uint128_6, 6, read_uint128); - #[cfg(byteorder_i128)] too_small!(small_uint128_7, 7, read_uint128); - #[cfg(byteorder_i128)] too_small!(small_uint128_8, 8, read_uint128); - #[cfg(byteorder_i128)] too_small!(small_uint128_9, 9, read_uint128); - #[cfg(byteorder_i128)] too_small!(small_uint128_10, 10, read_uint128); - #[cfg(byteorder_i128)] too_small!(small_uint128_11, 11, read_uint128); - #[cfg(byteorder_i128)] too_small!(small_uint128_12, 12, read_uint128); - #[cfg(byteorder_i128)] too_small!(small_uint128_13, 13, read_uint128); - #[cfg(byteorder_i128)] too_small!(small_uint128_14, 14, read_uint128); - #[cfg(byteorder_i128)] too_small!(small_uint128_15, 15, read_uint128); too_small!(small_int_1, 1, read_int); @@ -2699,35 +3138,20 @@ mod test { too_small!(small_int_6, 6, read_int); too_small!(small_int_7, 7, read_int); - #[cfg(byteorder_i128)] too_small!(small_int128_1, 1, read_int128); - #[cfg(byteorder_i128)] too_small!(small_int128_2, 2, read_int128); - #[cfg(byteorder_i128)] too_small!(small_int128_3, 3, read_int128); - #[cfg(byteorder_i128)] too_small!(small_int128_4, 4, read_int128); - #[cfg(byteorder_i128)] too_small!(small_int128_5, 5, read_int128); - #[cfg(byteorder_i128)] too_small!(small_int128_6, 6, read_int128); - #[cfg(byteorder_i128)] too_small!(small_int128_7, 7, read_int128); - #[cfg(byteorder_i128)] too_small!(small_int128_8, 8, read_int128); - #[cfg(byteorder_i128)] too_small!(small_int128_9, 9, read_int128); - #[cfg(byteorder_i128)] too_small!(small_int128_10, 10, read_int128); - #[cfg(byteorder_i128)] too_small!(small_int128_11, 11, read_int128); - #[cfg(byteorder_i128)] too_small!(small_int128_12, 12, read_int128); - #[cfg(byteorder_i128)] too_small!(small_int128_13, 13, read_int128); - #[cfg(byteorder_i128)] too_small!(small_int128_14, 14, read_int128); - #[cfg(byteorder_i128)] too_small!(small_int128_15, 15, read_int128); // Test that reading/writing slices enforces the correct lengths. @@ -2735,7 +3159,9 @@ mod test { ($name:ident, $read:ident, $write:ident, $num_bytes:expr, $numbers:expr) => { mod $name { - use {ByteOrder, BigEndian, NativeEndian, LittleEndian}; + use crate::{ + BigEndian, ByteOrder, LittleEndian, NativeEndian, + }; #[test] #[should_panic] @@ -2785,54 +3211,171 @@ mod test { NativeEndian::$write(&numbers, &mut bytes); } } - } + }; } slice_lengths!( - slice_len_too_small_u16, read_u16_into, write_u16_into, 3, [0, 0]); + slice_len_too_small_u16, + read_u16_into, + write_u16_into, + 3, + [0, 0] + ); slice_lengths!( - slice_len_too_big_u16, read_u16_into, write_u16_into, 5, [0, 0]); + slice_len_too_big_u16, + read_u16_into, + write_u16_into, + 5, + [0, 0] + ); slice_lengths!( - slice_len_too_small_i16, read_i16_into, write_i16_into, 3, [0, 0]); + slice_len_too_small_i16, + read_i16_into, + write_i16_into, + 3, + [0, 0] + ); slice_lengths!( - slice_len_too_big_i16, read_i16_into, write_i16_into, 5, [0, 0]); + slice_len_too_big_i16, + read_i16_into, + write_i16_into, + 5, + [0, 0] + ); slice_lengths!( - slice_len_too_small_u32, read_u32_into, write_u32_into, 7, [0, 0]); + slice_len_too_small_u32, + read_u32_into, + write_u32_into, + 7, + [0, 0] + ); slice_lengths!( - slice_len_too_big_u32, read_u32_into, write_u32_into, 9, [0, 0]); + slice_len_too_big_u32, + read_u32_into, + write_u32_into, + 9, + [0, 0] + ); slice_lengths!( - slice_len_too_small_i32, read_i32_into, write_i32_into, 7, [0, 0]); + slice_len_too_small_i32, + read_i32_into, + write_i32_into, + 7, + [0, 0] + ); slice_lengths!( - slice_len_too_big_i32, read_i32_into, write_i32_into, 9, [0, 0]); + slice_len_too_big_i32, + read_i32_into, + write_i32_into, + 9, + [0, 0] + ); slice_lengths!( - slice_len_too_small_u64, read_u64_into, write_u64_into, 15, [0, 0]); + slice_len_too_small_u64, + read_u64_into, + write_u64_into, + 15, + [0, 0] + ); slice_lengths!( - slice_len_too_big_u64, read_u64_into, write_u64_into, 17, [0, 0]); + slice_len_too_big_u64, + read_u64_into, + write_u64_into, + 17, + [0, 0] + ); slice_lengths!( - slice_len_too_small_i64, read_i64_into, write_i64_into, 15, [0, 0]); + slice_len_too_small_i64, + read_i64_into, + write_i64_into, + 15, + [0, 0] + ); slice_lengths!( - slice_len_too_big_i64, read_i64_into, write_i64_into, 17, [0, 0]); + slice_len_too_big_i64, + read_i64_into, + write_i64_into, + 17, + [0, 0] + ); - #[cfg(byteorder_i128)] slice_lengths!( - slice_len_too_small_u128, read_u128_into, write_u128_into, 31, [0, 0]); - #[cfg(byteorder_i128)] + slice_len_too_small_u128, + read_u128_into, + write_u128_into, + 31, + [0, 0] + ); slice_lengths!( - slice_len_too_big_u128, read_u128_into, write_u128_into, 33, [0, 0]); - #[cfg(byteorder_i128)] + slice_len_too_big_u128, + read_u128_into, + write_u128_into, + 33, + [0, 0] + ); slice_lengths!( - slice_len_too_small_i128, read_i128_into, write_i128_into, 31, [0, 0]); - #[cfg(byteorder_i128)] + slice_len_too_small_i128, + read_i128_into, + write_i128_into, + 31, + [0, 0] + ); slice_lengths!( - slice_len_too_big_i128, read_i128_into, write_i128_into, 33, [0, 0]); + slice_len_too_big_i128, + read_i128_into, + write_i128_into, + 33, + [0, 0] + ); #[test] fn uint_bigger_buffer() { - use {ByteOrder, LittleEndian}; + use crate::{ByteOrder, LittleEndian}; let n = LittleEndian::read_uint(&[1, 2, 3, 4, 5, 6, 7, 8], 5); - assert_eq!(n, 0x0504030201); + assert_eq!(n, 0x05_0403_0201); + } + + #[test] + fn regression173_array_impl() { + use crate::{BigEndian, ByteOrder, LittleEndian}; + + let xs = [0; 100]; + + let x = BigEndian::read_u16(&xs); + assert_eq!(x, 0); + let x = BigEndian::read_u32(&xs); + assert_eq!(x, 0); + let x = BigEndian::read_u64(&xs); + assert_eq!(x, 0); + let x = BigEndian::read_u128(&xs); + assert_eq!(x, 0); + let x = BigEndian::read_i16(&xs); + assert_eq!(x, 0); + let x = BigEndian::read_i32(&xs); + assert_eq!(x, 0); + let x = BigEndian::read_i64(&xs); + assert_eq!(x, 0); + let x = BigEndian::read_i128(&xs); + assert_eq!(x, 0); + + let x = LittleEndian::read_u16(&xs); + assert_eq!(x, 0); + let x = LittleEndian::read_u32(&xs); + assert_eq!(x, 0); + let x = LittleEndian::read_u64(&xs); + assert_eq!(x, 0); + let x = LittleEndian::read_u128(&xs); + assert_eq!(x, 0); + let x = LittleEndian::read_i16(&xs); + assert_eq!(x, 0); + let x = LittleEndian::read_i32(&xs); + assert_eq!(x, 0); + let x = LittleEndian::read_i64(&xs); + assert_eq!(x, 0); + let x = LittleEndian::read_i128(&xs); + assert_eq!(x, 0); } } @@ -2846,7 +3389,6 @@ mod stdtests { use self::rand::thread_rng; fn qc_unsized(f: A) { - QuickCheck::new() .gen(StdGen::new(thread_rng(), 16)) .tests(1_00) @@ -2855,19 +3397,22 @@ mod stdtests { } macro_rules! calc_max { - ($max:expr, $bytes:expr) => { ($max - 1) >> (8 * (8 - $bytes)) }; + ($max:expr, $bytes:expr) => { + ($max - 1) >> (8 * (8 - $bytes)) + }; } macro_rules! qc_bytes_ext { ($name:ident, $ty_int:ty, $max:expr, - $bytes:expr, $read:ident, $write:ident) => ( + $bytes:expr, $read:ident, $write:ident) => { mod $name { - use std::io::Cursor; - use { - ReadBytesExt, WriteBytesExt, - BigEndian, NativeEndian, LittleEndian, + #[allow(unused_imports)] + use crate::test::{qc_sized, Wi128}; + use crate::{ + BigEndian, LittleEndian, NativeEndian, ReadBytesExt, + WriteBytesExt, }; - #[allow(unused_imports)] use test::{qc_sized, Wi128}; + use std::io::Cursor; #[test] fn big_endian() { @@ -2908,15 +3453,16 @@ mod stdtests { qc_sized(prop as fn($ty_int) -> bool, $max); } } - ); - ($name:ident, $ty_int:ty, $max:expr, $read:ident, $write:ident) => ( + }; + ($name:ident, $ty_int:ty, $max:expr, $read:ident, $write:ident) => { mod $name { - use std::io::Cursor; - use { - ReadBytesExt, WriteBytesExt, - BigEndian, NativeEndian, LittleEndian, + #[allow(unused_imports)] + use crate::test::{qc_sized, Wi128}; + use crate::{ + BigEndian, LittleEndian, NativeEndian, ReadBytesExt, + WriteBytesExt, }; - #[allow(unused_imports)] use test::{qc_sized, Wi128}; + use std::io::Cursor; #[test] fn big_endian() { @@ -2951,188 +3497,484 @@ mod stdtests { qc_sized(prop as fn($ty_int) -> bool, $max - 1); } } - ); - } - - qc_bytes_ext!(prop_ext_u16, - u16, ::std::u16::MAX as u64, read_u16, write_u16); - qc_bytes_ext!(prop_ext_i16, - i16, ::std::i16::MAX as u64, read_i16, write_i16); - qc_bytes_ext!(prop_ext_u32, - u32, ::std::u32::MAX as u64, read_u32, write_u32); - qc_bytes_ext!(prop_ext_i32, - i32, ::std::i32::MAX as u64, read_i32, write_i32); - qc_bytes_ext!(prop_ext_u64, - u64, ::std::u64::MAX as u64, read_u64, write_u64); - qc_bytes_ext!(prop_ext_i64, - i64, ::std::i64::MAX as u64, read_i64, write_i64); - //qc_bytes_ext!(prop_ext_f32, - // f32, ::std::u64::MAX as u64, read_f32, write_f32); - //qc_bytes_ext!(prop_ext_f64, - // f64, ::std::i64::MAX as u64, read_f64, write_f64); - - #[cfg(byteorder_i128)] + }; + } + + qc_bytes_ext!( + prop_ext_u16, + u16, + ::std::u16::MAX as u64, + read_u16, + write_u16 + ); + qc_bytes_ext!( + prop_ext_i16, + i16, + ::std::i16::MAX as u64, + read_i16, + write_i16 + ); + qc_bytes_ext!( + prop_ext_u32, + u32, + ::std::u32::MAX as u64, + read_u32, + write_u32 + ); + qc_bytes_ext!( + prop_ext_i32, + i32, + ::std::i32::MAX as u64, + read_i32, + write_i32 + ); + qc_bytes_ext!( + prop_ext_u64, + u64, + ::std::u64::MAX as u64, + read_u64, + write_u64 + ); + qc_bytes_ext!( + prop_ext_i64, + i64, + ::std::i64::MAX as u64, + read_i64, + write_i64 + ); + qc_bytes_ext!( + prop_ext_f32, + f32, + ::std::u64::MAX as u64, + read_f32, + write_f32 + ); + qc_bytes_ext!( + prop_ext_f64, + f64, + ::std::i64::MAX as u64, + read_f64, + write_f64 + ); + qc_bytes_ext!(prop_ext_u128, Wi128, 16 + 1, read_u128, write_u128); - #[cfg(byteorder_i128)] qc_bytes_ext!(prop_ext_i128, Wi128, 16 + 1, read_i128, write_i128); - qc_bytes_ext!(prop_ext_uint_1, - u64, calc_max!(::test::U64_MAX, 1), 1, read_uint, write_u64); - qc_bytes_ext!(prop_ext_uint_2, - u64, calc_max!(::test::U64_MAX, 2), 2, read_uint, write_u64); - qc_bytes_ext!(prop_ext_uint_3, - u64, calc_max!(::test::U64_MAX, 3), 3, read_uint, write_u64); - qc_bytes_ext!(prop_ext_uint_4, - u64, calc_max!(::test::U64_MAX, 4), 4, read_uint, write_u64); - qc_bytes_ext!(prop_ext_uint_5, - u64, calc_max!(::test::U64_MAX, 5), 5, read_uint, write_u64); - qc_bytes_ext!(prop_ext_uint_6, - u64, calc_max!(::test::U64_MAX, 6), 6, read_uint, write_u64); - qc_bytes_ext!(prop_ext_uint_7, - u64, calc_max!(::test::U64_MAX, 7), 7, read_uint, write_u64); - qc_bytes_ext!(prop_ext_uint_8, - u64, calc_max!(::test::U64_MAX, 8), 8, read_uint, write_u64); - - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_1, - Wi128, 1, 1, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_2, - Wi128, 2, 2, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_3, - Wi128, 3, 3, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_4, - Wi128, 4, 4, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_5, - Wi128, 5, 5, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_6, - Wi128, 6, 6, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_7, - Wi128, 7, 7, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_8, - Wi128, 8, 8, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_9, - Wi128, 9, 9, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_10, - Wi128, 10, 10, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_11, - Wi128, 11, 11, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_12, - Wi128, 12, 12, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_13, - Wi128, 13, 13, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_14, - Wi128, 14, 14, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_15, - Wi128, 15, 15, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_16, - Wi128, 16, 16, read_uint128, write_u128); - - qc_bytes_ext!(prop_ext_int_1, - i64, calc_max!(::test::I64_MAX, 1), 1, read_int, write_i64); - qc_bytes_ext!(prop_ext_int_2, - i64, calc_max!(::test::I64_MAX, 2), 2, read_int, write_i64); - qc_bytes_ext!(prop_ext_int_3, - i64, calc_max!(::test::I64_MAX, 3), 3, read_int, write_i64); - qc_bytes_ext!(prop_ext_int_4, - i64, calc_max!(::test::I64_MAX, 4), 4, read_int, write_i64); - qc_bytes_ext!(prop_ext_int_5, - i64, calc_max!(::test::I64_MAX, 5), 5, read_int, write_i64); - qc_bytes_ext!(prop_ext_int_6, - i64, calc_max!(::test::I64_MAX, 6), 6, read_int, write_i64); - qc_bytes_ext!(prop_ext_int_7, - i64, calc_max!(::test::I64_MAX, 1), 7, read_int, write_i64); - qc_bytes_ext!(prop_ext_int_8, - i64, calc_max!(::test::I64_MAX, 8), 8, read_int, write_i64); - - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_1, - Wi128, 1, 1, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_2, - Wi128, 2, 2, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_3, - Wi128, 3, 3, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_4, - Wi128, 4, 4, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_5, - Wi128, 5, 5, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_6, - Wi128, 6, 6, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_7, - Wi128, 7, 7, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_8, - Wi128, 8, 8, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_9, - Wi128, 9, 9, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_10, - Wi128, 10, 10, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_11, - Wi128, 11, 11, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_12, - Wi128, 12, 12, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_13, - Wi128, 13, 13, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_14, - Wi128, 14, 14, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_15, - Wi128, 15, 15, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_16, - Wi128, 16, 16, read_int128, write_i128); + qc_bytes_ext!( + prop_ext_uint_1, + u64, + calc_max!(crate::test::U64_MAX, 1), + 1, + read_uint, + write_u64 + ); + qc_bytes_ext!( + prop_ext_uint_2, + u64, + calc_max!(crate::test::U64_MAX, 2), + 2, + read_uint, + write_u64 + ); + qc_bytes_ext!( + prop_ext_uint_3, + u64, + calc_max!(crate::test::U64_MAX, 3), + 3, + read_uint, + write_u64 + ); + qc_bytes_ext!( + prop_ext_uint_4, + u64, + calc_max!(crate::test::U64_MAX, 4), + 4, + read_uint, + write_u64 + ); + qc_bytes_ext!( + prop_ext_uint_5, + u64, + calc_max!(crate::test::U64_MAX, 5), + 5, + read_uint, + write_u64 + ); + qc_bytes_ext!( + prop_ext_uint_6, + u64, + calc_max!(crate::test::U64_MAX, 6), + 6, + read_uint, + write_u64 + ); + qc_bytes_ext!( + prop_ext_uint_7, + u64, + calc_max!(crate::test::U64_MAX, 7), + 7, + read_uint, + write_u64 + ); + qc_bytes_ext!( + prop_ext_uint_8, + u64, + calc_max!(crate::test::U64_MAX, 8), + 8, + read_uint, + write_u64 + ); + + qc_bytes_ext!( + prop_ext_uint128_1, + Wi128, + 1, + 1, + read_uint128, + write_u128 + ); + qc_bytes_ext!( + prop_ext_uint128_2, + Wi128, + 2, + 2, + read_uint128, + write_u128 + ); + qc_bytes_ext!( + prop_ext_uint128_3, + Wi128, + 3, + 3, + read_uint128, + write_u128 + ); + qc_bytes_ext!( + prop_ext_uint128_4, + Wi128, + 4, + 4, + read_uint128, + write_u128 + ); + qc_bytes_ext!( + prop_ext_uint128_5, + Wi128, + 5, + 5, + read_uint128, + write_u128 + ); + qc_bytes_ext!( + prop_ext_uint128_6, + Wi128, + 6, + 6, + read_uint128, + write_u128 + ); + qc_bytes_ext!( + prop_ext_uint128_7, + Wi128, + 7, + 7, + read_uint128, + write_u128 + ); + qc_bytes_ext!( + prop_ext_uint128_8, + Wi128, + 8, + 8, + read_uint128, + write_u128 + ); + qc_bytes_ext!( + prop_ext_uint128_9, + Wi128, + 9, + 9, + read_uint128, + write_u128 + ); + qc_bytes_ext!( + prop_ext_uint128_10, + Wi128, + 10, + 10, + read_uint128, + write_u128 + ); + qc_bytes_ext!( + prop_ext_uint128_11, + Wi128, + 11, + 11, + read_uint128, + write_u128 + ); + qc_bytes_ext!( + prop_ext_uint128_12, + Wi128, + 12, + 12, + read_uint128, + write_u128 + ); + qc_bytes_ext!( + prop_ext_uint128_13, + Wi128, + 13, + 13, + read_uint128, + write_u128 + ); + qc_bytes_ext!( + prop_ext_uint128_14, + Wi128, + 14, + 14, + read_uint128, + write_u128 + ); + qc_bytes_ext!( + prop_ext_uint128_15, + Wi128, + 15, + 15, + read_uint128, + write_u128 + ); + qc_bytes_ext!( + prop_ext_uint128_16, + Wi128, + 16, + 16, + read_uint128, + write_u128 + ); + + qc_bytes_ext!( + prop_ext_int_1, + i64, + calc_max!(crate::test::I64_MAX, 1), + 1, + read_int, + write_i64 + ); + qc_bytes_ext!( + prop_ext_int_2, + i64, + calc_max!(crate::test::I64_MAX, 2), + 2, + read_int, + write_i64 + ); + qc_bytes_ext!( + prop_ext_int_3, + i64, + calc_max!(crate::test::I64_MAX, 3), + 3, + read_int, + write_i64 + ); + qc_bytes_ext!( + prop_ext_int_4, + i64, + calc_max!(crate::test::I64_MAX, 4), + 4, + read_int, + write_i64 + ); + qc_bytes_ext!( + prop_ext_int_5, + i64, + calc_max!(crate::test::I64_MAX, 5), + 5, + read_int, + write_i64 + ); + qc_bytes_ext!( + prop_ext_int_6, + i64, + calc_max!(crate::test::I64_MAX, 6), + 6, + read_int, + write_i64 + ); + qc_bytes_ext!( + prop_ext_int_7, + i64, + calc_max!(crate::test::I64_MAX, 1), + 7, + read_int, + write_i64 + ); + qc_bytes_ext!( + prop_ext_int_8, + i64, + calc_max!(crate::test::I64_MAX, 8), + 8, + read_int, + write_i64 + ); + + qc_bytes_ext!( + prop_ext_int128_1, + Wi128, + 1, + 1, + read_int128, + write_i128 + ); + qc_bytes_ext!( + prop_ext_int128_2, + Wi128, + 2, + 2, + read_int128, + write_i128 + ); + qc_bytes_ext!( + prop_ext_int128_3, + Wi128, + 3, + 3, + read_int128, + write_i128 + ); + qc_bytes_ext!( + prop_ext_int128_4, + Wi128, + 4, + 4, + read_int128, + write_i128 + ); + qc_bytes_ext!( + prop_ext_int128_5, + Wi128, + 5, + 5, + read_int128, + write_i128 + ); + qc_bytes_ext!( + prop_ext_int128_6, + Wi128, + 6, + 6, + read_int128, + write_i128 + ); + qc_bytes_ext!( + prop_ext_int128_7, + Wi128, + 7, + 7, + read_int128, + write_i128 + ); + qc_bytes_ext!( + prop_ext_int128_8, + Wi128, + 8, + 8, + read_int128, + write_i128 + ); + qc_bytes_ext!( + prop_ext_int128_9, + Wi128, + 9, + 9, + read_int128, + write_i128 + ); + qc_bytes_ext!( + prop_ext_int128_10, + Wi128, + 10, + 10, + read_int128, + write_i128 + ); + qc_bytes_ext!( + prop_ext_int128_11, + Wi128, + 11, + 11, + read_int128, + write_i128 + ); + qc_bytes_ext!( + prop_ext_int128_12, + Wi128, + 12, + 12, + read_int128, + write_i128 + ); + qc_bytes_ext!( + prop_ext_int128_13, + Wi128, + 13, + 13, + read_int128, + write_i128 + ); + qc_bytes_ext!( + prop_ext_int128_14, + Wi128, + 14, + 14, + read_int128, + write_i128 + ); + qc_bytes_ext!( + prop_ext_int128_15, + Wi128, + 15, + 15, + read_int128, + write_i128 + ); + qc_bytes_ext!( + prop_ext_int128_16, + Wi128, + 16, + 16, + read_int128, + write_i128 + ); // Test slice serialization/deserialization. macro_rules! qc_slice { ($name:ident, $ty_int:ty, $read:ident, $write:ident, $zero:expr) => { mod $name { - use core::mem::size_of; - use {ByteOrder, BigEndian, NativeEndian, LittleEndian}; use super::qc_unsized; #[allow(unused_imports)] - use test::Wi128; + use crate::test::Wi128; + use crate::{ + BigEndian, ByteOrder, LittleEndian, NativeEndian, + }; + use core::mem::size_of; #[test] fn big_endian() { #[allow(unused_unsafe)] fn prop(numbers: Vec<$ty_int>) -> bool { - let numbers: Vec<_> = numbers - .into_iter() - .map(|x| x.clone()) - .collect(); + let numbers: Vec<_> = + numbers.into_iter().map(|x| x.clone()).collect(); let num_bytes = size_of::<$ty_int>() * numbers.len(); let mut bytes = vec![0; num_bytes]; BigEndian::$write(&numbers, &mut bytes); let mut got = vec![$zero; numbers.len()]; - unsafe { BigEndian::$read(&bytes, &mut got); } + unsafe { + BigEndian::$read(&bytes, &mut got); + } numbers == got } @@ -3143,17 +3985,17 @@ mod stdtests { fn little_endian() { #[allow(unused_unsafe)] fn prop(numbers: Vec<$ty_int>) -> bool { - let numbers: Vec<_> = numbers - .into_iter() - .map(|x| x.clone()) - .collect(); + let numbers: Vec<_> = + numbers.into_iter().map(|x| x.clone()).collect(); let num_bytes = size_of::<$ty_int>() * numbers.len(); let mut bytes = vec![0; num_bytes]; LittleEndian::$write(&numbers, &mut bytes); let mut got = vec![$zero; numbers.len()]; - unsafe { LittleEndian::$read(&bytes, &mut got); } + unsafe { + LittleEndian::$read(&bytes, &mut got); + } numbers == got } @@ -3164,24 +4006,24 @@ mod stdtests { fn native_endian() { #[allow(unused_unsafe)] fn prop(numbers: Vec<$ty_int>) -> bool { - let numbers: Vec<_> = numbers - .into_iter() - .map(|x| x.clone()) - .collect(); + let numbers: Vec<_> = + numbers.into_iter().map(|x| x.clone()).collect(); let num_bytes = size_of::<$ty_int>() * numbers.len(); let mut bytes = vec![0; num_bytes]; NativeEndian::$write(&numbers, &mut bytes); let mut got = vec![$zero; numbers.len()]; - unsafe { NativeEndian::$read(&bytes, &mut got); } + unsafe { + NativeEndian::$read(&bytes, &mut got); + } numbers == got } qc_unsized(prop as fn(_) -> bool); } } - } + }; } qc_slice!(prop_slice_u16, u16, read_u16_into, write_u16_into, 0); @@ -3190,15 +4032,21 @@ mod stdtests { qc_slice!(prop_slice_i32, i32, read_i32_into, write_i32_into, 0); qc_slice!(prop_slice_u64, u64, read_u64_into, write_u64_into, 0); qc_slice!(prop_slice_i64, i64, read_i64_into, write_i64_into, 0); - #[cfg(byteorder_i128)] qc_slice!( - prop_slice_u128, Wi128, read_u128_into, write_u128_into, 0); - #[cfg(byteorder_i128)] + prop_slice_u128, + Wi128, + read_u128_into, + write_u128_into, + 0 + ); qc_slice!( - prop_slice_i128, Wi128, read_i128_into, write_i128_into, 0); - - //qc_slice!( - // prop_slice_f32, f32, read_f32_into, write_f32_into, 0.0); - //qc_slice!( - // prop_slice_f64, f64, read_f64_into, write_f64_into, 0.0); + prop_slice_i128, + Wi128, + read_i128_into, + write_i128_into, + 0 + ); + + qc_slice!(prop_slice_f32, f32, read_f32_into, write_f32_into, 0.0); + qc_slice!(prop_slice_f64, f64, read_f64_into, write_f64_into, 0.0); } From 2236a8e2b4e3fcda5b79146adfdab022c054f88d Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Fri, 2 Jun 2023 07:46:03 -0400 Subject: [PATCH 085/114] Use Crucible-friendly implementations of byteorder functions Adapted from the original commits in c9392a18604d72fa807a433415fe25eb0abff957 and f361e11419c0a231c76229e850572283ab5d7d29 --- crux-mir/lib/Patches.md | 5 + crux-mir/lib/byteorder/src/lib.rs | 393 ++++++++++-------------------- 2 files changed, 134 insertions(+), 264 deletions(-) diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index a1e415de5..a978ac67d 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -120,3 +120,8 @@ identify all of the code that was changed in each patch. Because Crucible is effectively single-threaded, we can safely replace these with much simpler implementations that aren't nearly as tricky to simulate. + +* Use Crucible-friendly implementations of `byteorder` functions (last applied: June 2, 2023) + + Much of `byteorder` is implemented on top of unsafe code that is tricky to + simulate. diff --git a/crux-mir/lib/byteorder/src/lib.rs b/crux-mir/lib/byteorder/src/lib.rs index cc37cca6a..e778143e8 100644 --- a/crux-mir/lib/byteorder/src/lib.rs +++ b/crux-mir/lib/byteorder/src/lib.rs @@ -67,12 +67,14 @@ cases. [`Write`]: https://doc.rust-lang.org/std/io/trait.Write.html */ +#![cfg_attr(crux, feature(crucible_intrinsics))] #![deny(missing_docs)] #![cfg_attr(not(feature = "std"), no_std)] use core::{ convert::TryInto, fmt::Debug, hash::Hash, ptr::copy_nonoverlapping, slice, }; +use core::mem; #[cfg(feature = "std")] pub use crate::io::{ReadBytesExt, WriteBytesExt}; @@ -80,6 +82,9 @@ pub use crate::io::{ReadBytesExt, WriteBytesExt}; #[cfg(feature = "std")] mod io; +#[cfg(crux)] use core::mem::crucible_identity_transmute as identity_transmute; +#[cfg(not(crux))] use core::mem::transmute as identity_transmute; + #[inline] fn extend_sign(val: u64, nbytes: usize) -> i64 { let shift = (8 - nbytes) * 8; @@ -719,7 +724,7 @@ pub trait ByteOrder: /// ``` #[inline] fn read_f32(buf: &[u8]) -> f32 { - f32::from_bits(Self::read_u32(buf)) + unimplemented!() } /// Reads a IEEE754 double-precision (8 bytes) floating point number. @@ -742,7 +747,7 @@ pub trait ByteOrder: /// ``` #[inline] fn read_f64(buf: &[u8]) -> f64 { - f64::from_bits(Self::read_u64(buf)) + unimplemented!() } /// Writes a signed 16 bit integer `n` to `buf`. @@ -943,7 +948,7 @@ pub trait ByteOrder: /// ``` #[inline] fn write_f32(buf: &mut [u8], n: f32) { - Self::write_u32(buf, n.to_bits()) + unimplemented!() } /// Writes a IEEE754 double-precision (8 bytes) floating point number. @@ -966,7 +971,7 @@ pub trait ByteOrder: /// ``` #[inline] fn write_f64(buf: &mut [u8], n: f64) { - Self::write_u64(buf, n.to_bits()) + unimplemented!() } /// Reads unsigned 16 bit integers from `src` into `dst`. @@ -1085,7 +1090,7 @@ pub trait ByteOrder: #[inline] fn read_i16_into(src: &[u8], dst: &mut [i16]) { let dst = unsafe { - slice::from_raw_parts_mut(dst.as_mut_ptr() as *mut u16, dst.len()) + identity_transmute::<&mut [i16], &mut [u16]>(dst) }; Self::read_u16_into(src, dst) } @@ -1114,7 +1119,7 @@ pub trait ByteOrder: #[inline] fn read_i32_into(src: &[u8], dst: &mut [i32]) { let dst = unsafe { - slice::from_raw_parts_mut(dst.as_mut_ptr() as *mut u32, dst.len()) + identity_transmute::<&mut [i32], &mut [u32]>(dst) }; Self::read_u32_into(src, dst); } @@ -1143,7 +1148,7 @@ pub trait ByteOrder: #[inline] fn read_i64_into(src: &[u8], dst: &mut [i64]) { let dst = unsafe { - slice::from_raw_parts_mut(dst.as_mut_ptr() as *mut u64, dst.len()) + identity_transmute::<&mut [i64], &mut [u64]>(dst) }; Self::read_u64_into(src, dst); } @@ -1172,7 +1177,7 @@ pub trait ByteOrder: #[inline] fn read_i128_into(src: &[u8], dst: &mut [i128]) { let dst = unsafe { - slice::from_raw_parts_mut(dst.as_mut_ptr() as *mut u128, dst.len()) + identity_transmute::<&mut [i128], &mut [u128]>(dst) }; Self::read_u128_into(src, dst); } @@ -1201,10 +1206,7 @@ pub trait ByteOrder: /// ``` #[inline] fn read_f32_into(src: &[u8], dst: &mut [f32]) { - let dst = unsafe { - slice::from_raw_parts_mut(dst.as_mut_ptr() as *mut u32, dst.len()) - }; - Self::read_u32_into(src, dst); + unimplemented!() } /// **DEPRECATED**. @@ -1262,10 +1264,7 @@ pub trait ByteOrder: /// ``` #[inline] fn read_f64_into(src: &[u8], dst: &mut [f64]) { - let dst = unsafe { - slice::from_raw_parts_mut(dst.as_mut_ptr() as *mut u64, dst.len()) - }; - Self::read_u64_into(src, dst); + unimplemented!() } /// **DEPRECATED**. @@ -1449,7 +1448,7 @@ pub trait ByteOrder: /// ``` fn write_i16_into(src: &[i16], dst: &mut [u8]) { let src = unsafe { - slice::from_raw_parts(src.as_ptr() as *const u16, src.len()) + identity_transmute::<&[i16], &[u16]>(src) }; Self::write_u16_into(src, dst); } @@ -1477,7 +1476,7 @@ pub trait ByteOrder: /// ``` fn write_i32_into(src: &[i32], dst: &mut [u8]) { let src = unsafe { - slice::from_raw_parts(src.as_ptr() as *const u32, src.len()) + identity_transmute::<&[i32], &[u32]>(src) }; Self::write_u32_into(src, dst); } @@ -1505,7 +1504,7 @@ pub trait ByteOrder: /// ``` fn write_i64_into(src: &[i64], dst: &mut [u8]) { let src = unsafe { - slice::from_raw_parts(src.as_ptr() as *const u64, src.len()) + identity_transmute::<&[i64], &[u64]>(src) }; Self::write_u64_into(src, dst); } @@ -1533,7 +1532,7 @@ pub trait ByteOrder: /// ``` fn write_i128_into(src: &[i128], dst: &mut [u8]) { let src = unsafe { - slice::from_raw_parts(src.as_ptr() as *const u128, src.len()) + identity_transmute::<&[i128], &[u128]>(src) }; Self::write_u128_into(src, dst); } @@ -1561,10 +1560,7 @@ pub trait ByteOrder: /// assert_eq!(numbers_given, numbers_got); /// ``` fn write_f32_into(src: &[f32], dst: &mut [u8]) { - let src = unsafe { - slice::from_raw_parts(src.as_ptr() as *const u32, src.len()) - }; - Self::write_u32_into(src, dst); + unimplemented!() } /// Writes IEEE754 double-precision (8 bytes) floating point numbers from @@ -1590,10 +1586,7 @@ pub trait ByteOrder: /// assert_eq!(numbers_given, numbers_got); /// ``` fn write_f64_into(src: &[f64], dst: &mut [u8]) { - let src = unsafe { - slice::from_raw_parts(src.as_ptr() as *const u64, src.len()) - }; - Self::write_u64_into(src, dst); + unimplemented!() } /// Converts the given slice of unsigned 16 bit integers to a particular @@ -1692,7 +1685,7 @@ pub trait ByteOrder: #[inline] fn from_slice_i16(src: &mut [i16]) { let src = unsafe { - slice::from_raw_parts_mut(src.as_mut_ptr() as *mut u16, src.len()) + identity_transmute::<&mut [i16], &mut [u16]>(src) }; Self::from_slice_u16(src); } @@ -1717,7 +1710,7 @@ pub trait ByteOrder: #[inline] fn from_slice_i32(src: &mut [i32]) { let src = unsafe { - slice::from_raw_parts_mut(src.as_mut_ptr() as *mut u32, src.len()) + identity_transmute::<&mut [i32], &mut [u32]>(src) }; Self::from_slice_u32(src); } @@ -1742,7 +1735,7 @@ pub trait ByteOrder: #[inline] fn from_slice_i64(src: &mut [i64]) { let src = unsafe { - slice::from_raw_parts_mut(src.as_mut_ptr() as *mut u64, src.len()) + identity_transmute::<&mut [i64], &mut [u64]>(src) }; Self::from_slice_u64(src); } @@ -1767,7 +1760,7 @@ pub trait ByteOrder: #[inline] fn from_slice_i128(src: &mut [i128]) { let src = unsafe { - slice::from_raw_parts_mut(src.as_mut_ptr() as *mut u128, src.len()) + identity_transmute::<&mut [i128], &mut [u128]>(src) }; Self::from_slice_u128(src); } @@ -1895,231 +1888,163 @@ pub type NativeEndian = LittleEndian; #[cfg(target_endian = "big")] pub type NativeEndian = BigEndian; -/// Copies $size bytes from a number $n to a &mut [u8] $dst. $ty represents the -/// numeric type of $n and $which must be either to_be or to_le, depending on -/// which endianness one wants to use when writing to $dst. -/// -/// This macro is only safe to call when $ty is a numeric type and $size == -/// size_of::<$ty>() and where $dst is a &mut [u8]. -macro_rules! unsafe_write_num_bytes { - ($ty:ty, $size:expr, $n:expr, $dst:expr, $which:ident) => {{ - assert!($size <= $dst.len()); - unsafe { - // N.B. https://github.com/rust-lang/rust/issues/22776 - let bytes = *(&$n.$which() as *const _ as *const [u8; $size]); - copy_nonoverlapping((&bytes).as_ptr(), $dst.as_mut_ptr(), $size); +macro_rules! safe_read_bytes { + ($src:expr, $ty:ty, $range:expr) => ({ + let mut result: $ty = 0; + for i in $range { + result = (result << 8) | ($src[i] as $ty); } - }}; + result + }); } -/// Copies a &[u8] $src into a &mut [] $dst for the endianness given -/// by $which (must be either to_be or to_le). -/// -/// This macro is only safe to call when $src and $dst are &[u8] and &mut [u8], -/// respectively. The macro will panic if $src.len() != $size * $dst.len(), -/// where $size represents the size of the integers encoded in $src. -macro_rules! unsafe_read_slice { - ($src:expr, $dst:expr, $size:expr, $which:ident) => {{ - assert_eq!($src.len(), $size * $dst.len()); - - unsafe { - copy_nonoverlapping( - $src.as_ptr(), - $dst.as_mut_ptr() as *mut u8, - $src.len(), - ); +macro_rules! safe_write_bytes { + ($val:expr, $dst:expr, $range:expr) => ({ + let mut j = 0; + for i in ($range).rev() { + $dst[i] = ($val >> (j * 8)) as u8; + j += 1; } - for v in $dst.iter_mut() { - *v = v.$which(); - } - }}; + }); } -/// Copies a &[$ty] $src into a &mut [u8] $dst, where $ty must be a numeric -/// type. This panics if size_of::<$ty>() * $src.len() != $dst.len(). -/// -/// This macro is only safe to call when $src is a slice of numeric types and -/// $dst is a &mut [u8] and where $ty represents the type of the integers in -/// $src. -macro_rules! unsafe_write_slice_native { - ($src:expr, $dst:expr, $ty:ty) => {{ - let size = core::mem::size_of::<$ty>(); - assert_eq!(size * $src.len(), $dst.len()); - - unsafe { - copy_nonoverlapping( - $src.as_ptr() as *const u8, - $dst.as_mut_ptr(), - $dst.len(), - ); - } - }}; +macro_rules! safe_read_bytes_rev { + ($src:expr, $ty:ty, $range:expr) => (safe_read_bytes!($src, $ty, $range.rev())); +} + +macro_rules! safe_write_bytes_rev { + ($val:expr, $dst:expr, $range:expr) => (safe_write_bytes!($val, $dst, $range.rev())); } -macro_rules! write_slice { - ($src:expr, $dst:expr, $ty:ty, $size:expr, $write:expr) => {{ - assert!($size == ::core::mem::size_of::<$ty>()); - assert_eq!($size * $src.len(), $dst.len()); +macro_rules! safe_read_slice { + ($src:expr, $dst:expr, $read_one:ident, $step:expr) => ({ + assert!($src.len() == $dst.len() * $step); + let mut base = 0; + for one_dst in $dst { + let one_src = &$src[base .. base + $step]; + *one_dst = Self::$read_one(one_src); + base += $step; + } + }); +} - for (&n, chunk) in $src.iter().zip($dst.chunks_mut($size)) { - $write(chunk, n); +macro_rules! safe_write_slice { + ($src:expr, $dst:expr, $write_one:ident, $step:expr) => ({ + assert!($dst.len() == $src.len() * $step); + let mut base = 0; + for one_src in $src { + let one_dst = &mut $dst[base .. base + $step]; + Self::$write_one(one_dst, *one_src); + base += $step; } - }}; + }); } impl ByteOrder for BigEndian { #[inline] fn read_u16(buf: &[u8]) -> u16 { - u16::from_be_bytes(buf[..2].try_into().unwrap()) + safe_read_bytes!(buf, u16, 0..2) } #[inline] fn read_u32(buf: &[u8]) -> u32 { - u32::from_be_bytes(buf[..4].try_into().unwrap()) + safe_read_bytes!(buf, u32, 0..4) } #[inline] fn read_u64(buf: &[u8]) -> u64 { - u64::from_be_bytes(buf[..8].try_into().unwrap()) + safe_read_bytes!(buf, u64, 0..8) } #[inline] fn read_u128(buf: &[u8]) -> u128 { - u128::from_be_bytes(buf[..16].try_into().unwrap()) + safe_read_bytes!(buf, u128, 0..16) } #[inline] fn read_uint(buf: &[u8], nbytes: usize) -> u64 { assert!(1 <= nbytes && nbytes <= 8 && nbytes <= buf.len()); - let mut out = 0u64; - let ptr_out = &mut out as *mut u64 as *mut u8; - unsafe { - copy_nonoverlapping( - buf.as_ptr(), - ptr_out.offset((8 - nbytes) as isize), - nbytes, - ); - } - out.to_be() + safe_read_bytes!(buf, u64, 0..nbytes) } #[inline] fn read_uint128(buf: &[u8], nbytes: usize) -> u128 { assert!(1 <= nbytes && nbytes <= 16 && nbytes <= buf.len()); - let mut out: u128 = 0; - let ptr_out = &mut out as *mut u128 as *mut u8; - unsafe { - copy_nonoverlapping( - buf.as_ptr(), - ptr_out.offset((16 - nbytes) as isize), - nbytes, - ); - } - out.to_be() + safe_read_bytes!(buf, u128, 0..nbytes) } #[inline] fn write_u16(buf: &mut [u8], n: u16) { - unsafe_write_num_bytes!(u16, 2, n, buf, to_be); + safe_write_bytes!(n, buf, 0..2) } #[inline] fn write_u32(buf: &mut [u8], n: u32) { - unsafe_write_num_bytes!(u32, 4, n, buf, to_be); + safe_write_bytes!(n, buf, 0..4) } #[inline] fn write_u64(buf: &mut [u8], n: u64) { - unsafe_write_num_bytes!(u64, 8, n, buf, to_be); + safe_write_bytes!(n, buf, 0..8) } #[inline] fn write_u128(buf: &mut [u8], n: u128) { - unsafe_write_num_bytes!(u128, 16, n, buf, to_be); + safe_write_bytes!(n, buf, 0..16) } #[inline] fn write_uint(buf: &mut [u8], n: u64, nbytes: usize) { assert!(pack_size(n) <= nbytes && nbytes <= 8); assert!(nbytes <= buf.len()); - unsafe { - let bytes = *(&n.to_be() as *const u64 as *const [u8; 8]); - copy_nonoverlapping( - bytes.as_ptr().offset((8 - nbytes) as isize), - buf.as_mut_ptr(), - nbytes, - ); - } + safe_write_bytes!(n, buf, 0..nbytes) } #[inline] fn write_uint128(buf: &mut [u8], n: u128, nbytes: usize) { assert!(pack_size128(n) <= nbytes && nbytes <= 16); assert!(nbytes <= buf.len()); - unsafe { - let bytes = *(&n.to_be() as *const u128 as *const [u8; 16]); - copy_nonoverlapping( - bytes.as_ptr().offset((16 - nbytes) as isize), - buf.as_mut_ptr(), - nbytes, - ); - } + safe_write_bytes!(n, buf, 0..nbytes) } #[inline] fn read_u16_into(src: &[u8], dst: &mut [u16]) { - unsafe_read_slice!(src, dst, 2, to_be); + safe_read_slice!(src, dst, read_u16, 2); } #[inline] fn read_u32_into(src: &[u8], dst: &mut [u32]) { - unsafe_read_slice!(src, dst, 4, to_be); + safe_read_slice!(src, dst, read_u32, 4); } #[inline] fn read_u64_into(src: &[u8], dst: &mut [u64]) { - unsafe_read_slice!(src, dst, 8, to_be); + safe_read_slice!(src, dst, read_u64, 8); } #[inline] fn read_u128_into(src: &[u8], dst: &mut [u128]) { - unsafe_read_slice!(src, dst, 16, to_be); + safe_read_slice!(src, dst, read_u128, 16); } #[inline] fn write_u16_into(src: &[u16], dst: &mut [u8]) { - if cfg!(target_endian = "big") { - unsafe_write_slice_native!(src, dst, u16); - } else { - write_slice!(src, dst, u16, 2, Self::write_u16); - } + safe_write_slice!(src, dst, write_u16, 2); } #[inline] fn write_u32_into(src: &[u32], dst: &mut [u8]) { - if cfg!(target_endian = "big") { - unsafe_write_slice_native!(src, dst, u32); - } else { - write_slice!(src, dst, u32, 4, Self::write_u32); - } + safe_write_slice!(src, dst, write_u32, 4); } #[inline] fn write_u64_into(src: &[u64], dst: &mut [u8]) { - if cfg!(target_endian = "big") { - unsafe_write_slice_native!(src, dst, u64); - } else { - write_slice!(src, dst, u64, 8, Self::write_u64); - } + safe_write_slice!(src, dst, write_u64, 8); } #[inline] fn write_u128_into(src: &[u128], dst: &mut [u8]) { - if cfg!(target_endian = "big") { - unsafe_write_slice_native!(src, dst, u128); - } else { - write_slice!(src, dst, u128, 16, Self::write_u128); - } + safe_write_slice!(src, dst, write_u128, 16); } #[inline] @@ -2160,166 +2085,120 @@ impl ByteOrder for BigEndian { #[inline] fn from_slice_f32(numbers: &mut [f32]) { - if cfg!(target_endian = "little") { - for n in numbers { - unsafe { - let int = *(n as *const f32 as *const u32); - *n = *(&int.to_be() as *const u32 as *const f32); - } - } - } + unimplemented!() } #[inline] fn from_slice_f64(numbers: &mut [f64]) { - if cfg!(target_endian = "little") { - for n in numbers { - unsafe { - let int = *(n as *const f64 as *const u64); - *n = *(&int.to_be() as *const u64 as *const f64); - } - } - } + unimplemented!() } } impl ByteOrder for LittleEndian { #[inline] fn read_u16(buf: &[u8]) -> u16 { - u16::from_le_bytes(buf[..2].try_into().unwrap()) + safe_read_bytes_rev!(buf, u16, 0..2) } #[inline] fn read_u32(buf: &[u8]) -> u32 { - u32::from_le_bytes(buf[..4].try_into().unwrap()) + safe_read_bytes_rev!(buf, u32, 0..4) } #[inline] fn read_u64(buf: &[u8]) -> u64 { - u64::from_le_bytes(buf[..8].try_into().unwrap()) + safe_read_bytes_rev!(buf, u64, 0..8) } #[inline] fn read_u128(buf: &[u8]) -> u128 { - u128::from_le_bytes(buf[..16].try_into().unwrap()) + safe_read_bytes_rev!(buf, u128, 0..16) } #[inline] fn read_uint(buf: &[u8], nbytes: usize) -> u64 { assert!(1 <= nbytes && nbytes <= 8 && nbytes <= buf.len()); - let mut out = 0u64; - let ptr_out = &mut out as *mut u64 as *mut u8; - unsafe { - copy_nonoverlapping(buf.as_ptr(), ptr_out, nbytes); - } - out.to_le() + safe_read_bytes_rev!(buf, u64, 0..nbytes) } #[inline] fn read_uint128(buf: &[u8], nbytes: usize) -> u128 { assert!(1 <= nbytes && nbytes <= 16 && nbytes <= buf.len()); - let mut out: u128 = 0; - let ptr_out = &mut out as *mut u128 as *mut u8; - unsafe { - copy_nonoverlapping(buf.as_ptr(), ptr_out, nbytes); - } - out.to_le() + safe_read_bytes_rev!(buf, u128, 0..nbytes) } #[inline] fn write_u16(buf: &mut [u8], n: u16) { - unsafe_write_num_bytes!(u16, 2, n, buf, to_le); + safe_write_bytes_rev!(n, buf, 0..2) } #[inline] fn write_u32(buf: &mut [u8], n: u32) { - unsafe_write_num_bytes!(u32, 4, n, buf, to_le); + safe_write_bytes_rev!(n, buf, 0..4) } #[inline] fn write_u64(buf: &mut [u8], n: u64) { - unsafe_write_num_bytes!(u64, 8, n, buf, to_le); + safe_write_bytes_rev!(n, buf, 0..8) } #[inline] fn write_u128(buf: &mut [u8], n: u128) { - unsafe_write_num_bytes!(u128, 16, n, buf, to_le); + safe_write_bytes_rev!(n, buf, 0..16) } #[inline] fn write_uint(buf: &mut [u8], n: u64, nbytes: usize) { - assert!(pack_size(n as u64) <= nbytes && nbytes <= 8); + assert!(pack_size(n) <= nbytes && nbytes <= 8); assert!(nbytes <= buf.len()); - unsafe { - let bytes = *(&n.to_le() as *const u64 as *const [u8; 8]); - copy_nonoverlapping(bytes.as_ptr(), buf.as_mut_ptr(), nbytes); - } + safe_write_bytes_rev!(n, buf, 0..nbytes) } #[inline] fn write_uint128(buf: &mut [u8], n: u128, nbytes: usize) { - assert!(pack_size128(n as u128) <= nbytes && nbytes <= 16); + assert!(pack_size128(n) <= nbytes && nbytes <= 16); assert!(nbytes <= buf.len()); - unsafe { - let bytes = *(&n.to_le() as *const u128 as *const [u8; 16]); - copy_nonoverlapping(bytes.as_ptr(), buf.as_mut_ptr(), nbytes); - } + safe_write_bytes_rev!(n, buf, 0..nbytes) } #[inline] fn read_u16_into(src: &[u8], dst: &mut [u16]) { - unsafe_read_slice!(src, dst, 2, to_le); + safe_read_slice!(src, dst, read_u16, 2); } #[inline] fn read_u32_into(src: &[u8], dst: &mut [u32]) { - unsafe_read_slice!(src, dst, 4, to_le); + safe_read_slice!(src, dst, read_u32, 4); } #[inline] fn read_u64_into(src: &[u8], dst: &mut [u64]) { - unsafe_read_slice!(src, dst, 8, to_le); + safe_read_slice!(src, dst, read_u64, 8); } #[inline] fn read_u128_into(src: &[u8], dst: &mut [u128]) { - unsafe_read_slice!(src, dst, 16, to_le); + safe_read_slice!(src, dst, read_u128, 16); } #[inline] fn write_u16_into(src: &[u16], dst: &mut [u8]) { - if cfg!(target_endian = "little") { - unsafe_write_slice_native!(src, dst, u16); - } else { - write_slice!(src, dst, u16, 2, Self::write_u16); - } + safe_write_slice!(src, dst, write_u16, 2); } #[inline] fn write_u32_into(src: &[u32], dst: &mut [u8]) { - if cfg!(target_endian = "little") { - unsafe_write_slice_native!(src, dst, u32); - } else { - write_slice!(src, dst, u32, 4, Self::write_u32); - } + safe_write_slice!(src, dst, write_u32, 4); } #[inline] fn write_u64_into(src: &[u64], dst: &mut [u8]) { - if cfg!(target_endian = "little") { - unsafe_write_slice_native!(src, dst, u64); - } else { - write_slice!(src, dst, u64, 8, Self::write_u64); - } + safe_write_slice!(src, dst, write_u64, 8); } #[inline] fn write_u128_into(src: &[u128], dst: &mut [u8]) { - if cfg!(target_endian = "little") { - unsafe_write_slice_native!(src, dst, u128); - } else { - write_slice!(src, dst, u128, 16, Self::write_u128); - } + safe_write_slice!(src, dst, write_u128, 16); } #[inline] @@ -2360,26 +2239,12 @@ impl ByteOrder for LittleEndian { #[inline] fn from_slice_f32(numbers: &mut [f32]) { - if cfg!(target_endian = "big") { - for n in numbers { - unsafe { - let int = *(n as *const f32 as *const u32); - *n = *(&int.to_le() as *const u32 as *const f32); - } - } - } + unimplemented!() } #[inline] fn from_slice_f64(numbers: &mut [f64]) { - if cfg!(target_endian = "big") { - for n in numbers { - unsafe { - let int = *(n as *const f64 as *const u64); - *n = *(&int.to_le() as *const u64 as *const f64); - } - } - } + unimplemented!() } } @@ -3101,8 +2966,8 @@ mod test { too_small!(small_i32, 3, 0, read_i32, write_i32); too_small!(small_u64, 7, 0, read_u64, write_u64); too_small!(small_i64, 7, 0, read_i64, write_i64); - too_small!(small_f32, 3, 0.0, read_f32, write_f32); - too_small!(small_f64, 7, 0.0, read_f64, write_f64); + //too_small!(small_f32, 3, 0.0, read_f32, write_f32); + //too_small!(small_f64, 7, 0.0, read_f64, write_f64); too_small!(small_u128, 15, 0, read_u128, write_u128); too_small!(small_i128, 15, 0, read_i128, write_i128); @@ -3542,20 +3407,20 @@ mod stdtests { read_i64, write_i64 ); - qc_bytes_ext!( - prop_ext_f32, - f32, - ::std::u64::MAX as u64, - read_f32, - write_f32 - ); - qc_bytes_ext!( - prop_ext_f64, - f64, - ::std::i64::MAX as u64, - read_f64, - write_f64 - ); + //qc_bytes_ext!( + // prop_ext_f32, + // f32, + // ::std::u64::MAX as u64, + // read_f32, + // write_f32 + //); + //qc_bytes_ext!( + // prop_ext_f64, + // f64, + // ::std::i64::MAX as u64, + // read_f64, + // write_f64 + //); qc_bytes_ext!(prop_ext_u128, Wi128, 16 + 1, read_u128, write_u128); qc_bytes_ext!(prop_ext_i128, Wi128, 16 + 1, read_i128, write_i128); @@ -4047,6 +3912,6 @@ mod stdtests { 0 ); - qc_slice!(prop_slice_f32, f32, read_f32_into, write_f32_into, 0.0); - qc_slice!(prop_slice_f64, f64, read_f64_into, write_f64_into, 0.0); + //qc_slice!(prop_slice_f32, f32, read_f32_into, write_f32_into, 0.0); + //qc_slice!(prop_slice_f64, f64, read_f64_into, write_f64_into, 0.0); } From e526e9a686b8579d19a62d1dcee735e73806f54d Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Fri, 2 Jun 2023 13:14:33 -0400 Subject: [PATCH 086/114] Bump mir-json submodule --- dependencies/mir-json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies/mir-json b/dependencies/mir-json index 861d52a74..a65ab6703 160000 --- a/dependencies/mir-json +++ b/dependencies/mir-json @@ -1 +1 @@ -Subproject commit 861d52a74048e93edd139b48ffe576adbf516218 +Subproject commit a65ab670394e90feb84650f82868a7bbdaed71c8 From d8853f0e1b5ce989d00607f92ae56148bc7112f4 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Fri, 2 Jun 2023 13:15:16 -0400 Subject: [PATCH 087/114] Accept new golden output for byteorder tests --- crux-mir/test/symb_eval/byteorder/read.good | 2 +- crux-mir/test/symb_eval/byteorder/write.good | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crux-mir/test/symb_eval/byteorder/read.good b/crux-mir/test/symb_eval/byteorder/read.good index 8970881bc..e5d65b995 100644 --- a/crux-mir/test/symb_eval/byteorder/read.good +++ b/crux-mir/test/symb_eval/byteorder/read.good @@ -1,3 +1,3 @@ -test read/3a1fbbbh::crux_test[0]: returned 0, ok +test read/::crux_test[0]: returned 0, ok [Crux] Overall status: Valid. diff --git a/crux-mir/test/symb_eval/byteorder/write.good b/crux-mir/test/symb_eval/byteorder/write.good index 75a9441d4..009352cef 100644 --- a/crux-mir/test/symb_eval/byteorder/write.good +++ b/crux-mir/test/symb_eval/byteorder/write.good @@ -1,3 +1,3 @@ -test write/3a1fbbbh::crux_test[0]: returned 0, ok +test write/::crux_test[0]: returned 0, ok [Crux] Overall status: Valid. From 2302923561bd5cb12ddeeaf385e0192b3394752b Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 21 Jun 2023 15:29:44 -0400 Subject: [PATCH 088/114] Hackily support `&dyn Trait` references For now, we hackily extend `MirPlace` with a custom representation for `&dyn Trait` references, enabling `crucible-mir` to compute `addrOfPlace` (but no other operations). We leave the task of doing this robustly to #1092. --- crucible-mir/src/Mir/Generator.hs | 16 ++++++++++++++++ crucible-mir/src/Mir/Trans.hs | 13 ++++++++++++- crucible-mir/src/Mir/TransTy.hs | 8 ++++---- crux-mir/test/symb_eval/concretize/assert.good | 6 +++--- 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/crucible-mir/src/Mir/Generator.hs b/crucible-mir/src/Mir/Generator.hs index 7263ecabd..8717bb74b 100644 --- a/crucible-mir/src/Mir/Generator.hs +++ b/crucible-mir/src/Mir/Generator.hs @@ -136,6 +136,21 @@ data MirExp s where -- result of lvalue evaluation. data MirPlace s where MirPlace :: C.TypeRepr ty -> R.Expr MIR s (MirReferenceType ty) -> PtrMetadata s -> MirPlace s + -- | This is a hack used to support @&dyn Trait@ references. Unlike other + -- reference types, trait objects do not use 'MirReferenceType', instead + -- using a custom 'AnyType' representation that wraps a + -- @'MirReferenceType' t@ (for some unknown @t@). The only 'MirPlaceDynRef' + -- operation that is currently supported is @addrOfPlace@, which supports + -- code that looks like @&*x@ (where @x: &dyn Trait@). This sort of + -- operation does arise in @rustc@-compiled MIR, even in the standard + -- libraries, so we must have /some/ level of support for it. + -- + -- All other 'MirPlaceDynRef' operations (e.g., @readPlace@ and + -- @evalPlaceProj@) are unsupported and will throw an exception if + -- encountered. To implement these operations, we will need to change the + -- encoding of trait objects to use 'MirReferenceType' in a more standard + -- way. See . + MirPlaceDynRef :: R.Expr MIR s DynRefType -> MirPlace s -- | MIR supports a notion of "unsized places" - for example, it generates code -- like `(*s)[i]` where `s` is a slice. To handle this, we attach the metadata @@ -389,6 +404,7 @@ instance Show (MirExp s) where instance Show (MirPlace s) where show (MirPlace tr e m) = show e ++ ", " ++ show m ++ ": & " ++ show tr + show (MirPlaceDynRef e) = "dyn reference " ++ show e instance Show MirHandle where show (MirHandle _nm sig c) = diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index d586b917f..d48d76c78 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -374,11 +374,16 @@ readPlace (MirPlace tpr r NoMeta) = MirExp tpr <$> readMirRef tpr r readPlace (MirPlace tpr _ meta) = mirFail $ "don't know how to read from place with metadata " ++ show meta ++ " (type " ++ show tpr ++ ")" +readPlace MirPlaceDynRef{} = + -- See https://github.com/GaloisInc/crucible/issues/1092 + mirFail "readPlace not supported for dyn references" addrOfPlace :: HasCallStack => MirPlace s -> MirGenerator h s ret (MirExp s) addrOfPlace (MirPlace tpr r NoMeta) = return $ MirExp (MirReferenceRepr tpr) r addrOfPlace (MirPlace tpr r (SliceMeta len)) = return $ MirExp (MirSliceRepr tpr) $ mkSlice tpr r len +addrOfPlace (MirPlaceDynRef dynRef) = + return $ MirExp DynRefRepr dynRef @@ -1053,6 +1058,9 @@ evalPlaceProj ty pl@(MirPlace tpr ref NoMeta) M.Deref = do where doRef (M.TySlice _) | MirSliceRepr tpr' <- tpr = doSlice tpr' ref doRef M.TyStr | MirSliceRepr tpr' <- tpr = doSlice tpr' ref + doRef (M.TyDynamic _) | DynRefRepr <- tpr = do + dynRef <- readMirRef tpr ref + return $ MirPlaceDynRef dynRef doRef _ | MirReferenceRepr tpr' <- tpr = do MirPlace tpr' <$> readMirRef tpr ref <*> pure NoMeta doRef _ = mirFail $ "deref: bad repr for " ++ show ty ++ ": " ++ show tpr @@ -1145,6 +1153,9 @@ evalPlaceProj ty (MirPlace tpr ref meta) (M.ConstantIndex idx _minLen fromEnd) = evalPlaceProj _ pl (M.Downcast _idx) = return pl evalPlaceProj ty (MirPlace _ _ meta) proj = mirFail $ "projection " ++ show proj ++ " not yet implemented for " ++ show (ty, meta) +evalPlaceProj _ MirPlaceDynRef{} _ = + -- See https://github.com/GaloisInc/crucible/issues/1092 + mirFail "evalPlaceProj not supported for dyn references" -------------------------------------------------------------------------------------- -- ** Statements @@ -2002,7 +2013,7 @@ transVirtCall colState intrName methName dynTraitName methIndex $ \eqMethArgs@Refl recvTy argTys -> let retTy = FH.handleReturnType methFH in - checkEq recvTy dynRefRepr (die ["method receiver is not `&dyn`/`&mut dyn`"]) + checkEq recvTy DynRefRepr (die ["method receiver is not `&dyn`/`&mut dyn`"]) $ \eqRecvTy@Refl -> -- Unpack vtable type diff --git a/crucible-mir/src/Mir/TransTy.hs b/crucible-mir/src/Mir/TransTy.hs index 4e50fd016..e0ec0dcdb 100644 --- a/crucible-mir/src/Mir/TransTy.hs +++ b/crucible-mir/src/Mir/TransTy.hs @@ -235,11 +235,11 @@ tyToRepr col t0 = case t0 of _ -> error $ unwords ["unknown type?", show t0] -dynRefCtx :: Ctx.Assignment C.TypeRepr (Ctx.EmptyCtx Ctx.::> C.AnyType Ctx.::> C.AnyType) -dynRefCtx = Ctx.empty Ctx.:> C.AnyRepr Ctx.:> C.AnyRepr +pattern DynRefCtx :: () => ctx ~ Ctx.EmptyCtx Ctx.::> C.AnyType Ctx.::> C.AnyType => Ctx.Assignment C.TypeRepr ctx +pattern DynRefCtx = Ctx.Empty Ctx.:> C.AnyRepr Ctx.:> C.AnyRepr -dynRefRepr :: C.TypeRepr DynRefType -dynRefRepr = C.StructRepr dynRefCtx +pattern DynRefRepr :: () => tp ~ DynRefType => C.TypeRepr tp +pattern DynRefRepr = C.StructRepr DynRefCtx tyToReprM :: M.Ty -> MirGenerator h s ret (Some C.TypeRepr) diff --git a/crux-mir/test/symb_eval/concretize/assert.good b/crux-mir/test/symb_eval/concretize/assert.good index 899626ca9..e78402882 100644 --- a/crux-mir/test/symb_eval/concretize/assert.good +++ b/crux-mir/test/symb_eval/concretize/assert.good @@ -1,10 +1,10 @@ -test assert/3a1fbbbh::crux_test[0]: returned 1, FAILED +test assert/::crux_test[0]: returned 1, FAILED failures: ----- assert/3a1fbbbh::crux_test[0] counterexamples ---- +---- assert/::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal -[Crux] ./lib/crucible/lib.rs:47:17: 47:82 !test/symb_eval/concretize/assert.rs:10:5: 10:82: error: in assert/3a1fbbbh::crux_test[0] +[Crux] ./lib/crucible/lib.rs:48:17: 48:82 !test/symb_eval/concretize/assert.rs:10:5: 10:81: error: in assert/::crux_test[0] [Crux] MIR assertion at test/symb_eval/concretize/assert.rs:10:5: [Crux] 100 + 157 == 1 From 5a1153582839cfe6cbf49a27c864dceae2f7d93f Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 21 Jun 2023 16:17:35 -0400 Subject: [PATCH 089/114] Track crate names and their hashes, use it to look up wired-in names Unlike in the previous Rust nightly, where the disambiguators for crates in the standard library were always hard-coded to a known constant, the disambiguators in the new Rust nightly are always unique, even for standard library crates. As a result, in order to determine what disambiguator a crate like `core` has, we must look at the MIR JSON and map each crate name to its disambiguator. See the new `crateHashesMap` field in `CollectionState`. --- crucible-mir/src/Mir/DefId.hs | 16 ++++++---- crucible-mir/src/Mir/Generator.hs | 40 +++++++++++++++++++++++-- crucible-mir/src/Mir/Trans.hs | 30 +++++++++++++++++-- crucible-mir/src/Mir/TransCustom.hs | 6 ++-- crucible-mir/src/Mir/TransTy.hs | 5 ++-- crux-mir/test/symb_eval/vector/pop.good | 2 +- 6 files changed, 83 insertions(+), 16 deletions(-) diff --git a/crucible-mir/src/Mir/DefId.hs b/crucible-mir/src/Mir/DefId.hs index 11473eca1..c3b18d53c 100644 --- a/crucible-mir/src/Mir/DefId.hs +++ b/crucible-mir/src/Mir/DefId.hs @@ -1,5 +1,5 @@ {-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE TemplateHaskellQuotes #-} +{-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE ViewPatterns #-} {-# LANGUAGE DeriveGeneric, DeriveAnyClass, DefaultSignatures #-} @@ -10,6 +10,9 @@ module Mir.DefId ( DefId +, didCrate +, didCrateDisambig +, didPath , textId , idText @@ -23,6 +26,7 @@ module Mir.DefId , parseFieldName ) where +import Control.Lens (makeLenses) import Data.Aeson import qualified Language.Haskell.TH.Syntax as TH @@ -48,16 +52,18 @@ type Segment = (Text, Int) data DefId = DefId { -- | The name of the enclosing crate. - did_crate :: Text + _didCrate :: Text -- | The disambiguator of the enclosing crate. These are strings, in a -- different format than the integer disambiguators used for normal path -- segments. - , did_crate_disambig :: Text + , _didCrateDisambig :: Text -- | The segments of the path. - , did_path :: [Segment] + , _didPath :: [Segment] } deriving (Eq, Ord, Generic) +makeLenses ''DefId + -- | The crate disambiguator hash produced when the crate metadata string is -- empty. This is useful for creating 'DefId's for thing that do not have a -- disambiguator associated with it (e.g., Crucible names). @@ -118,7 +124,7 @@ instance Pretty DefId where type ExplodedDefId = [Text] idKey :: DefId -> ExplodedDefId -idKey did = did_crate did : map fst (did_path did) +idKey did = _didCrate did : map fst (_didPath did) explodedDefIdPat :: ExplodedDefId -> TH.Q TH.Pat explodedDefIdPat edid = [p| ((\did -> idKey did == edid) -> True) |] diff --git a/crucible-mir/src/Mir/Generator.hs b/crucible-mir/src/Mir/Generator.hs index 8717bb74b..bb5307d6c 100644 --- a/crucible-mir/src/Mir/Generator.hs +++ b/crucible-mir/src/Mir/Generator.hs @@ -65,7 +65,9 @@ where import Data.Kind(Type) import qualified Data.Aeson as Aeson +import qualified Data.Foldable as F import qualified Data.List as List +import Data.List.NonEmpty (NonEmpty(..)) import qualified Data.Maybe as Maybe import Data.Map.Strict(Map) import qualified Data.Map.Strict as Map @@ -205,6 +207,16 @@ data CollectionState _staticMap :: !(Map DefId StaticVar), -- | For Enums, gives the discriminant value for each variant. _discrMap :: !(Map AdtName [Integer]), + -- | Map crate names to their respective crate hashes, the latter of + -- which are used to disambiguate identifier names. We consult this 'Map' + -- when looking up wired-in names (e.g., 'Option' or 'MaybeUninit' in + -- @core@) to determine what disambiguator to use. + -- + -- Note that the range of the 'Map' is a 'NonEmpty' list because it is + -- possible to depend on two different crates with the same crate name, + -- but with different hashes. Most of the time, however, this list will + -- contain exactly one disambiguator per crate name. + _crateHashesMap :: !(Map Text (NonEmpty Text)), _collection :: !Collection } @@ -393,10 +405,10 @@ instance Monoid RustModule where mempty = RustModule mempty mempty mempty instance Semigroup CollectionState where - (CollectionState hm1 vm1 sm1 dm1 col1) <> (CollectionState hm2 vm2 sm2 dm2 col2) = - (CollectionState (hm1 <> hm2) (vm1 <> vm2) (sm1 <> sm2) (dm1 <> dm2) (col1 <> col2)) + (CollectionState hm1 vm1 sm1 dm1 chm1 col1) <> (CollectionState hm2 vm2 sm2 dm2 chm2 col2) = + (CollectionState (hm1 <> hm2) (vm1 <> vm2) (sm1 <> sm2) (dm1 <> dm2) (Map.unionWith (<>) chm1 chm2) (col1 <> col2)) instance Monoid CollectionState where - mempty = CollectionState mempty mempty mempty mempty mempty + mempty = CollectionState mempty mempty mempty mempty mempty mempty instance Show (MirExp s) where @@ -461,6 +473,28 @@ findAdtInst origName substs = do Just x -> return x Nothing -> mirFail $ "unknown ADT " ++ show (origName, substs) +-- | Find the 'DefId' corresponding to the supplied text. This consults the +-- 'crateHashesMap' to ensure that the crate's disambiguator is correct. If a +-- crate name is ambiguous (i.e., if there are multiple disambiguators +-- associated with the crate name), this will throw an error. +findDefId :: Text -> MirGenerator h s ret DefId +findDefId str = do + crateDisambigs <- use $ cs . crateHashesMap + case Map.lookup crate crateDisambigs of + Just allDisambigs@(disambig :| otherDisambigs) + | F.null otherDisambigs + -> pure $ partialDefId & didCrateDisambig .~ disambig + | otherwise + -> mirFail $ unlines $ + [ "ambiguous crate " ++ crateStr + , "crate disambiguators:" + ] ++ F.toList (Text.unpack <$> allDisambigs) + Nothing -> mirFail $ "unknown crate " ++ crateStr + where + partialDefId = textId str + crate = partialDefId^.didCrate + crateStr = Text.unpack crate + -- | What to do when the translation fails. mirFail :: String -> MirGenerator h s ret a mirFail str = do diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index d48d76c78..561f4129a 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -52,6 +52,7 @@ import Data.Bits (shift, shiftL) import qualified Data.ByteString as BS import qualified Data.Char as Char import qualified Data.List as List +import Data.List.NonEmpty (NonEmpty(..)) import qualified Data.List.NonEmpty as NE import Data.Map.Strict (Map) import qualified Data.Map.Strict as Map @@ -2135,7 +2136,30 @@ mkDiscrMap col = mconcat [ Map.singleton (adt^.M.adtname) (adtIndices adt col) | adt <- Map.elems $ col^.M.adts, Lens.is _Enum (adt^.M.adtkind) ] - +-- | Gather all of the 'M.DefId's in a 'M.Collection' and construct a +-- 'crateHashesMap' from it. To accomplish this, it suffices to look at the +-- domain of each map in a 'M.Collection' (with a handful of exceptions—see +-- the comments below), which ranges over 'M.DefId's. +mkCrateHashesMap :: M.Collection -> Map Text (NonEmpty Text) +mkCrateHashesMap + (Collection functionsM adtsM adtsOrigM traitsM + staticsM vtablesM intrinsicsM + -- namedTys ranges over type names, which aren't full DefIds. + _namedTysM + -- The roots are duplicates of other Maps' DefIds. + _rootsM) = + Map.fromList $ + f functionsM + ++ f adtsM + ++ f adtsOrigM + ++ f traitsM + ++ f staticsM + ++ f vtablesM + ++ f intrinsicsM + where + f :: Map M.DefId a -> [(Text, NonEmpty Text)] + f m = map (\did -> (did ^. M.didCrate, did ^. M.didCrateDisambig :| [])) + (Map.keys m) --------------------------------------------------------------------------- @@ -2173,10 +2197,10 @@ transCollection col halloc = do sm <- foldrM allocateStatic Map.empty (col^.statics) let dm = mkDiscrMap col - + let chm = mkCrateHashesMap col let colState :: CollectionState - colState = CollectionState hmap vm sm dm col + colState = CollectionState hmap vm sm dm chm col -- translate all of the functions fnInfo <- mapM (stToIO . transDefine (?libCS <> colState)) (Map.elems (col^.M.functions)) diff --git a/crucible-mir/src/Mir/TransCustom.hs b/crucible-mir/src/Mir/TransCustom.hs index 939554771..9c020bde0 100644 --- a/crucible-mir/src/Mir/TransCustom.hs +++ b/crucible-mir/src/Mir/TransCustom.hs @@ -1026,7 +1026,7 @@ intrinsics_assume = (["core", "intrinsics", "", "assume"], \_substs -> -- TODO: needs layout info from mir-json assert_inhabited :: (ExplodedDefId, CustomRHS) -assert_inhabited = (["core", "intrinsics", "", "assert_inhabited"], \_substs -> +assert_inhabited = (["core", "intrinsics", "{extern}", "assert_inhabited"], \_substs -> Just $ CustomOp $ \_ _ -> return $ MirExp C.UnitRepr $ R.App E.EmptyApp) array_from_slice :: (ExplodedDefId, CustomRHS) @@ -1601,7 +1601,8 @@ maybe_uninit_uninit :: (ExplodedDefId, CustomRHS) maybe_uninit_uninit = (["core", "mem", "maybe_uninit", "{impl}", "uninit"], \substs -> case substs of Substs [t] -> Just $ CustomOp $ \_ _ -> do - adt <- findAdtInst (M.textId "core::mem::maybe_uninit::MaybeUninit") (Substs [t]) + maybeUninitDefId <- findDefId "core::mem::maybe_uninit::MaybeUninit" + adt <- findAdtInst maybeUninitDefId (Substs [t]) let t = TyAdt (adt ^. adtname) (adt ^. adtOrigDefId) (adt ^. adtOrigSubsts) initialValue t >>= \mv -> case mv of Just v -> return v @@ -1705,6 +1706,7 @@ unwrapMirExp tpr (MirExp tpr' e) maybeToOption :: Ty -> C.TypeRepr tp -> R.Expr MIR s (C.MaybeType tp) -> MirGenerator h s ret (MirExp s) maybeToOption ty tpr e = do + optionDefId <- findDefId optionDefIdText adt <- findAdtInst optionDefId (Substs [ty]) e' <- G.caseMaybe e C.AnyRepr $ G.MatchMaybe (\val -> buildEnum adt optionDiscrSome [MirExp tpr val] >>= unwrapMirExp C.AnyRepr) diff --git a/crucible-mir/src/Mir/TransTy.hs b/crucible-mir/src/Mir/TransTy.hs index e0ec0dcdb..39a6c6a71 100644 --- a/crucible-mir/src/Mir/TransTy.hs +++ b/crucible-mir/src/Mir/TransTy.hs @@ -29,6 +29,7 @@ import Control.Lens import qualified Data.BitVector.Sized as BV import Data.List (findIndices) import Data.String (fromString) +import Data.Text (Text) import qualified Data.Vector as V import GHC.Stack @@ -140,8 +141,8 @@ pattern CTyMethodSpecBuilder <- M.TyAdt _ $(M.explodedDefIdPat ["crucible", "met pattern CTyOption t <- M.TyAdt _ $(M.explodedDefIdPat ["core", "option", "Option"]) (M.Substs [t]) where CTyOption t = M.TyAdt (M.textId "type::adt") (M.textId "core::option::Option") (M.Substs [t]) -optionDefId :: M.DefId -optionDefId = M.textId "core::option::Option" +optionDefIdText :: Text +optionDefIdText = "core::option::Option" optionDiscrNone :: Int optionDiscrNone = 0 optionDiscrSome :: Int diff --git a/crux-mir/test/symb_eval/vector/pop.good b/crux-mir/test/symb_eval/vector/pop.good index ac5d37589..1d0f9bbcc 100644 --- a/crux-mir/test/symb_eval/vector/pop.good +++ b/crux-mir/test/symb_eval/vector/pop.good @@ -1,3 +1,3 @@ -test pop/3a1fbbbh::f[0]: ok +test pop/::f[0]: ok [Crux] Overall status: Valid. From 2740aac4a46e3e4018c48078fb0fc2630f84e4a0 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 21 Jun 2023 16:56:19 -0400 Subject: [PATCH 090/114] Update test_output.log --- crux-mir/test_output.log | 2312 ++++++++++++++++++-------------------- 1 file changed, 1099 insertions(+), 1213 deletions(-) diff --git a/crux-mir/test_output.log b/crux-mir/test_output.log index e31d84715..24d279b87 100644 --- a/crux-mir/test_output.log +++ b/crux-mir/test_output.log @@ -1,148 +1,494 @@ +Build profile: -w ghc-9.2.7 -O1 +In order, the following will be built (use -v for more details): + - crux-mir-0.6.0.99 (test:test) (file /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/dist-newstyle/build/x86_64-linux/ghc-9.2.7/crux-mir-0.6.0.99/cache/build changed) +Preprocessing test suite 'test' for crux-mir-0.6.0.99.. +Building test suite 'test' for crux-mir-0.6.0.99.. +[1 of 1] Compiling Main ( test/Test.hs, /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/dist-newstyle/build/x86_64-linux/ghc-9.2.7/crux-mir-0.6.0.99/t/test/build/test/test-tmp/Main.o ) [Mir.Language changed] +Linking /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/dist-newstyle/build/x86_64-linux/ghc-9.2.7/crux-mir-0.6.0.99/t/test/build/test/test ... crux-mir crux concrete - clos - fn_static_poly: OK (0.94s) - Compiling and running oracle program (0.12s) - Oracle output: 3 (0.82s) + refs + promoted_imm: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.75s) + Crux output: 0 + mut_raw: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: 123 (1.75s) + Crux output: 123 + mut_nested: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.74s) + Crux output: () + fn_ptr_mut: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.75s) + Crux output: 1 + never: OK (1.90s) + Compiling and running oracle program (0.16s) + Oracle output: 1 (1.75s) + Crux output: 1 + mut_ref: OK (1.88s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.74s) + Crux output: 1 + temp: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.76s) + Crux output: 1 + mut_arg: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.72s) + Crux output: 1 + fn_ptr: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.75s) + Crux output: 1 + promoted_mut: OK (1.93s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.79s) + Crux output: 0 + never_mut: OK (1.98s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.84s) + Crux output: 1 + mut_tuple_field: OK (1.99s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.85s) + Crux output: () + imm_raw: OK (2.04s) + Compiling and running oracle program (0.16s) + Oracle output: 123 (1.88s) + Crux output: 123 + static_mut: OK (1.98s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.83s) + Crux output: 2 + imm_ref: OK (2.05s) + Compiling and running oracle program (0.15s) + Oracle output: 123 (1.90s) + Crux output: 123 + imm_arg: OK (1.92s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.78s) + Crux output: 1 + hash_map + insert_multi: OK (2.27s) + Compiling and running oracle program (0.18s) + Oracle output: 100 (2.09s) + Crux output: 100 + insert_remove: OK (2.68s) + Compiling and running oracle program (0.18s) + Oracle output: 12 (2.51s) + Crux output: 12 + insert_iter: OK (2.23s) + Compiling and running oracle program (0.18s) + Oracle output: [11, 12] (2.05s) + Crux output: [11, 12] + insert_get: OK (2.25s) + Compiling and running oracle program (0.18s) + Oracle output: (11, 12) (2.07s) + Crux output: (11, 12) + traits + params: OK (1.96s) + Compiling and running oracle program (0.14s) + Oracle output: 25 (1.82s) + Crux output: 25 + generic3: OK (2.00s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.85s) + Crux output: () + static_three: OK (1.90s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.74s) + Crux output: 2 + generic1: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.74s) + Crux output: () + subtrait: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 73 (1.72s) + Crux output: 73 + gen_trait: OK (1.92s) + Compiling and running oracle program (0.15s) + Oracle output: -69 (1.78s) + Crux output: -69 + dynamic_branch: OK (1.93s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.79s) + Crux output: 1 + static_two: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.72s) + Crux output: 1 + tyfam5: OK (1.84s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.70s) + Crux output: () + generic2: OK (1.87s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.73s) + Crux output: () + assoc3: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.72s) + Crux output: () + tyfam4: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.71s) + Crux output: 0 + static_self: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (1.71s) + Crux output: 42 + conv: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.71s) + Crux output: 1 + dynamic_poly: OK (1.85s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (1.71s) Crux output: 3 - fnptr_closure: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 7 (0.84s) + dynamic_simple: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.71s) + Crux output: 1 + gen_trait_poly: OK (1.85s) + Compiling and running oracle program (0.15s) + Oracle output: 12 (1.70s) + Crux output: 12 + default: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 12 (1.71s) + Crux output: 12 + assoc1: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.72s) + Crux output: () + subtrait2: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 64 (1.72s) + Crux output: 64 + bounds1: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.73s) + Crux output: () + bounds3: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.72s) + Crux output: () + static_eq: OK (1.84s) + Compiling and running oracle program (0.14s) + Oracle output: true (1.70s) + Crux output: true + tyfam3: OK (1.84s) + Compiling and running oracle program (0.14s) + Oracle output: 23 (1.70s) + Crux output: 23 + dict_med: OK (1.84s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (1.70s) + Crux output: 42 + basics1: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.75s) + Crux output: () + assoc2: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.76s) + Crux output: () + dict_polymem: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 4 (1.72s) + Crux output: 4 + bounds2: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.71s) + Crux output: () + intoiter: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.71s) + Crux output: 0 + static: OK (1.84s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (1.70s) + Crux output: 42 + dynamic_two: OK (1.84s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.70s) + Crux output: 1 + bounds4: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.72s) + Crux output: () + dict_poly: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.76s) + Crux output: 1 + dict_simple: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 32 (1.74s) + Crux output: 32 + tyfam: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 23 (1.71s) + Crux output: 23 + bounds5: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.72s) + Crux output: () + tyfam2: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 23 (1.70s) + Crux output: 23 + dynamic_med: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.70s) + Crux output: 1 + intTest + test0039: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 7 (1.72s) Crux output: 7 - fnptr_fn: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: 4 (0.81s) + test0038: OK (1.94s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.79s) + Crux output: () + tuple + clone_rec: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.73s) + Crux output: () + clone: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.73s) + Crux output: () + clone_from: OK (1.90s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.75s) + Crux output: () + clone_struct: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.72s) + Crux output: () + iter + for: OK (1.93s) + Compiling and running oracle program (0.14s) + Oracle output: 47 (1.79s) + Crux output: 47 + sum: OK (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.76s) + Crux output: () + filter_chain: OK (2.01s) + Compiling and running oracle program (0.17s) + Oracle output: 0 (1.85s) + Crux output: 0 + from_fn: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.74s) + Crux output: () + peek: OK (1.95s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (1.80s) + Crux output: 3 + zip: OK (1.94s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.80s) + Crux output: () + loop: OK (1.84s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.70s) + Crux output: 2 + cloned: OK (1.90s) + Compiling and running oracle program (0.15s) + Oracle output: 6 (1.75s) + Crux output: 6 + str + format_hex: OK (2.61s) + Compiling and running oracle program (0.15s) + Oracle output: true (2.46s) + Crux output: true + format_int: OK (2.61s) + Compiling and running oracle program (0.15s) + Oracle output: true (2.46s) + Crux output: true + format_struct: OK (2.76s) + Compiling and running oracle program (0.15s) + Oracle output: true (2.61s) + Crux output: true + format: OK (2.59s) + Compiling and running oracle program (0.15s) + Oracle output: true (2.44s) + Crux output: true + to_owned: OK (2.05s) + Compiling and running oracle program (0.15s) + Oracle output: true (1.90s) + Crux output: true + string_push: OK (2.59s) + Compiling and running oracle program (0.15s) + Oracle output: true (2.44s) + Crux output: true + format_array: OK (2.74s) + Compiling and running oracle program (0.15s) + Oracle output: true (2.59s) + Crux output: true + sync + mutex: OK (2.88s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (2.73s) + Crux output: 1 + arc_cell: OK (2.09s) + Compiling and running oracle program (0.15s) + Oracle output: 4 (1.95s) Crux output: 4 - conv_fnonce_fnmut: OK (0.95s) - Compiling and running oracle program (0.12s) - Oracle output: 2 (0.83s) + arc: OK (2.01s) + Compiling and running oracle program (0.15s) + Oracle output: 1 (1.86s) + Crux output: 1 + atomic_add: OK (1.96s) + Compiling and running oracle program (0.14s) + Oracle output: 6 (1.82s) + Crux output: 6 + rwlock: OK (2.91s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (2.77s) Crux output: 2 - as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.83s) - Crux output: - failures: - - ---- as_fn_ptr/d8484d39::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/d8484d39::crux_test[0] - [Crux] Translation error in as_fn_ptr/d8484d39::crux_test[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [], _fsreturn_ty = TyInt B32, _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - test/Test.hs:123: - crux doesn't match oracle - (expected failure) - fnonce1: OK (0.91s) - Compiling and running oracle program (0.12s) - Oracle output: 3 (0.79s) - Crux output: 3 - direct_fn: OK (0.93s) - Compiling and running oracle program (0.12s) - Oracle output: 2 (0.80s) + atomic_cxchg: OK (1.97s) + Compiling and running oracle program (0.15s) + Oracle output: 6 (1.82s) + Crux output: 6 + arc_clone: OK (2.01s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.86s) Crux output: 2 - fn_static: OK (0.92s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.79s) - Crux output: 3 - poly: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.80s) + atomic_swap: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: (2, 1) (1.75s) + Crux output: (2, 1) + rwlock_multi: OK (2.99s) + Compiling and running oracle program (0.15s) + Oracle output: 3 (2.83s) Crux output: 3 - direct_fnmut: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: 7 (0.82s) - Crux output: 7 - direct_fnmut2: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 7 (0.85s) - Crux output: 7 - promoted: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: 33 (0.84s) - Crux output: 33 - direct_fnonce: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.83s) + atomic_fence: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.74s) Crux output: 2 - fo: OK (0.99s) + mutex_multi: OK (2.97s) Compiling and running oracle program (0.15s) - Oracle output: 29 (0.84s) - Crux output: 29 - fnonce: OK (0.91s) - Compiling and running oracle program (0.12s) - Oracle output: false (0.79s) - Crux output: false - fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (0.93s) - Compiling and running oracle program (0.12s) - Oracle output: 3 (0.81s) - vtable signature mismatch for vtable core/092bc89a::ops[0]::function[0]::Fn[0]::_vtbl570ffd80324c2ca9[0] of trait core/092bc89a::ops[0]::function[0]::Fn[0]::_trait42f8b6475ee85722[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32)] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32)] - CallStack (from HasCallStack): - error, called at src/Mir/Trans.hs:958:20 in crucible-mir-0.1-inplace:Mir.Trans - mkTraitObject, called at src/Mir/Trans.hs:801:11 in crucible-mir-0.1-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:932:5 in crucible-mir-0.1-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:984:30 in crucible-mir-0.1-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1197:11 in crucible-mir-0.1-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1642:68 in crucible-mir-0.1-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1651:28 in crucible-mir-0.1-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1691:10 in crucible-mir-0.1-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1737:24 in crucible-mir-0.1-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2171:30 in crucible-mir-0.1-inplace:Mir.Trans - transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate - translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language - (expected failure) - ref_fnmut: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 7 (0.81s) - Crux output: 7 - dispatch_fnmut: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: 7 (0.83s) - Crux output: 7 - unique_borrow: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.81s) + Oracle output: 3 (2.82s) Crux output: 3 - conv_fnmut_fn: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.80s) + consts + struct_val: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: Foo { x: true } (1.72s) + Crux output: Foo { x: true } + struct_unit: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: Err(Foo { x: () }) (1.72s) + Crux output: Err(Foo { x: () }) + local_key: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.71s) + Crux output: 1 + fn_def: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.70s) + Crux output: 1 + enum_val: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: None (1.76s) + Crux output: None + crypto + add: OK (1.95s) + Compiling and running oracle program (0.15s) + Oracle output: true (1.80s) + Crux output: true + add_noL: OK (1.93s) + Compiling and running oracle program (0.15s) + Oracle output: false (1.78s) + Crux output: false + cell + cell: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.74s) Crux output: 2 - fnptr_fnmut: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 4 (0.80s) - Crux output: 4 - fnptr_fnonce: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 4 (0.81s) - Crux output: 4 - conv_fnonce_fn: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.80s) + ref_cell: OK (2.37s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (2.22s) + Crux output: 2 + ref_cell2: OK (2.41s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (2.27s) + Crux output: 2 + fnptr + call: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.73s) + Crux output: 2 + field: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.72s) Crux output: 2 + make: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.71s) + Crux output: 0 + custom: rustc compilation failed for custom +error output: +error[E0432]: unresolved import `core` + --> test/conc_eval/fnptr/custom.rs:2:5 + | +2 | use core::mem; + | ^^^^ maybe a missing crate `core`? + | + = help: consider adding `extern crate core` to use the `core` crate + +error: aborting due to previous error + +For more information about this error, try `rustc --explain E0432`. + +FAIL (expected: taking address of an overridden function) (0.05s) + Compiling and running oracle program (0.05s) + test/Test.hs:106: + failed to compile and run + (expected failure) + num + overflow: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.76s) + Crux output: () + from_bytes: OK (1.93s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.79s) + Crux output: () + saturate: OK (2.42s) + Compiling and running oracle program (0.15s) + Oracle output: () (2.28s) + Crux output: () prim - bool: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: false (0.80s) + bool: OK (1.84s) + Compiling and running oracle program (0.14s) + Oracle output: false (1.70s) Crux output: false - litbstring: OK (0.92s) - Compiling and running oracle program (0.13s) - Oracle output: true (0.79s) - Crux output: true - shift3: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: -9223372036854775808 (0.81s) + shift3: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: -9223372036854775808 (1.71s) Crux output: -9223372036854775808 - litstring: OK (0.92s) - Compiling and running oracle program (0.13s) - Oracle output: true (0.80s) - Crux output: true - ge: OK (0.92s) - Compiling and running oracle program (0.12s) - Oracle output: true (0.80s) + div: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 4 (1.71s) + Crux output: 4 + ge: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: true (1.71s) Crux output: true + add1: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.75s) + Crux output: 2 shift_exceeding: rustc compilation failed for shift_exceeding error output: error[E0425]: cannot find function `catch_unwind` in module `panic` @@ -174,443 +520,98 @@ error: aborting due to previous error; 1 warning emitted For more information about this error, try `rustc --explain E0425`. FAIL (expected: Should panic, but doesn't) (0.05s) - Compiling and running oracle program (0.06s) + Compiling and running oracle program (0.05s) test/Test.hs:106: failed to compile and run (expected failure) - shift4: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: -9223372036854775808 (0.80s) - Crux output: -9223372036854775808 - mut_arg: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.81s) - Crux output: 1 - wrapping_sub: OK (0.92s) - Compiling and running oracle program (0.13s) - Oracle output: true (0.79s) - Crux output: true - lit: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: false (0.81s) - Crux output: false - mut: OK (0.93s) - Compiling and running oracle program (0.12s) - Oracle output: 14 (0.80s) - Crux output: 14 - shift1: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.81s) - Crux output: 2 - add1: OK (0.93s) - Compiling and running oracle program (0.12s) - Oracle output: 2 (0.81s) - Crux output: 2 - div: OK (0.93s) - Compiling and running oracle program (0.12s) - Oracle output: 4 (0.81s) - Crux output: 4 - shift2: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.80s) - Crux output: 2 - ffs: OK (1.02s) - Compiling and running oracle program (0.13s) - Oracle output: true (0.89s) - Crux output: true - char_from_u32: OK (0.94s) - Compiling and running oracle program (0.12s) - Oracle output: 'A' (0.81s) - Crux output: 'A' - traits - tyfam: OK (0.92s) - Compiling and running oracle program (0.13s) - Oracle output: 23 (0.80s) - Crux output: 23 - dynamic_branch: OK (0.92s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.79s) - Crux output: 1 - static: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.83s) - Crux output: 42 - bounds3: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.83s) - Crux output: () - tyfam3: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: 23 (0.82s) - Crux output: 23 - static_two: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.81s) - Crux output: 1 - bounds2: OK (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.86s) - Crux output: () - generic2: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.84s) - Crux output: () - gen_trait_poly: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: 12 (0.84s) - Crux output: 12 - dict_simple: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 32 (0.81s) - Crux output: 32 - static_self: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.84s) - Crux output: 42 - tyfam4: OK (0.97s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (0.83s) - Crux output: 0 - gen_trait: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: -69 (0.82s) - Crux output: -69 - generic3: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.82s) - Crux output: () - subtrait: OK (0.92s) - Compiling and running oracle program (0.13s) - Oracle output: 73 (0.80s) - Crux output: 73 - default: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 12 (0.81s) - Crux output: 12 - bounds1: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.81s) - Crux output: () - intoiter: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 0 (0.80s) - Crux output: 0 - static_eq: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: true (0.81s) - Crux output: true - assoc3: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.80s) - Crux output: () - dynamic_poly: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.81s) - Crux output: 3 - dynamic_med: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.80s) - Crux output: 1 - static_three: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.80s) - Crux output: 2 - dynamic_two: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.80s) - Crux output: 1 - bounds4: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.83s) - Crux output: () - bounds5: OK (0.98s) + char_from_u32: OK (1.87s) Compiling and running oracle program (0.14s) - Oracle output: () (0.84s) - Crux output: () - dict_med: OK (0.92s) - Compiling and running oracle program (0.12s) - Oracle output: 42 (0.80s) - Crux output: 42 - dynamic_simple: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.80s) - Crux output: 1 - assoc1: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.82s) - Crux output: () - params: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: 25 (0.82s) - Crux output: 25 - conv: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.80s) - Crux output: 1 - basics1: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.83s) - Crux output: () - dict_poly: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.81s) + Oracle output: 'A' (1.73s) + Crux output: 'A' + ffs: OK (1.94s) + Compiling and running oracle program (0.15s) + Oracle output: true (1.79s) + Crux output: true + mut_arg: OK (1.91s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.77s) Crux output: 1 - dict_polymem: OK (0.94s) - Compiling and running oracle program (0.12s) - Oracle output: 4 (0.82s) - Crux output: 4 - subtrait2: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 64 (0.80s) - Crux output: 64 - tyfam2: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: 23 (0.83s) - Crux output: 23 - tyfam5: OK (0.92s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.79s) - Crux output: () - assoc2: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.82s) - Crux output: () - generic1: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.83s) - Crux output: () - impl - self_mut: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.81s) - Crux output: 42 - self: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.83s) - Crux output: 42 - simple: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.80s) - Crux output: 42 - vec - drop: OK (1.11s) + litstring: OK (1.83s) Compiling and running oracle program (0.14s) - Oracle output: () (0.98s) - Crux output: () - from_elem_zero: OK (1.23s) + Oracle output: true (1.69s) + Crux output: true + lit: OK (1.86s) Compiling and running oracle program (0.14s) - Oracle output: 0 (1.09s) - Crux output: 0 - extend_trusted_len: OK (1.82s) + Oracle output: false (1.71s) + Crux output: false + litbstring: OK (1.83s) Compiling and running oracle program (0.14s) - Oracle output: (1, 10) (1.68s) - Crux output: (1, 10) - extend: OK (1.23s) + Oracle output: true (1.69s) + Crux output: true + shift1: OK (1.85s) Compiling and running oracle program (0.14s) - Oracle output: (1, 10) (1.10s) - Crux output: (1, 10) - set_len: OK (1.15s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (1.02s) + Oracle output: 2 (1.71s) Crux output: 2 - collect: OK (1.86s) + shift2: OK (1.84s) Compiling and running oracle program (0.14s) - Oracle output: 45 (1.72s) - Crux output: 45 - push: OK (1.14s) - Compiling and running oracle program (0.13s) - Oracle output: (1, 2) (1.01s) - Crux output: (1, 2) - slice - len: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 5 (0.81s) - Crux output: 5 - mut_range: OK (1.50s) - Compiling and running oracle program (0.13s) - Oracle output: 86 (1.37s) - Crux output: 86 - range_len: OK (1.47s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (1.34s) - Crux output: 1 - eq: OK (1.05s) + Oracle output: 2 (1.69s) + Crux output: 2 + shift4: OK (1.83s) Compiling and running oracle program (0.14s) - Oracle output: () (0.91s) - Crux output: () - iter_mut: OK (1.06s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.93s) - Crux output: () - get: OK (0.97s) - Compiling and running oracle program (0.12s) - Oracle output: true (0.85s) + Oracle output: -9223372036854775808 (1.70s) + Crux output: -9223372036854775808 + mut: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 14 (1.71s) + Crux output: 14 + wrapping_sub: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: true (1.71s) Crux output: true - last: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.84s) - Crux output: 3 - range_len_mut: OK (1.44s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (1.31s) - Crux output: 1 - mut: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.82s) + impl + self: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (1.71s) Crux output: 42 - swap: OK (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: 2001 (0.86s) - Crux output: 2001 - mk_and_proj: OK (0.92s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.80s) + simple: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (1.75s) + Crux output: 42 + self_mut: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (1.72s) Crux output: 42 - num - overflow: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.87s) - Crux output: () - saturate: OK (1.53s) - Compiling and running oracle program (0.13s) - Oracle output: () (1.40s) - Crux output: () - from_bytes: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.88s) - Crux output: () - dyn - inherit: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: 4 (0.82s) - Crux output: 4 - plain_trait: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 100 (0.80s) - Crux output: 100 - trait_param: OK (0.91s) - Compiling and running oracle program (0.13s) - Oracle output: 100 (0.79s) - Crux output: 100 - assoc_ty: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 100 (0.80s) - Crux output: 100 box - mut_ref: OK (1.08s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.95s) - Crux output: () - new: OK (1.09s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.96s) - Crux output: () - struct: OK (1.09s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.96s) - Crux output: 1 - mut: OK (1.09s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.96s) + new: OK (1.97s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.83s) Crux output: () - unsize: OK (1.09s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.96s) - Crux output: 2 - vec_deque - rotate_right: OK (1.69s) + mut_ref: OK (1.97s) Compiling and running oracle program (0.14s) - Oracle output: [4, 5, 1, 2, 3] (1.55s) - Crux output: [4, 5, 1, 2, 3] - retain: OK (1.69s) + Oracle output: () (1.83s) + Crux output: () + mut: OK (1.98s) Compiling and running oracle program (0.14s) - Oracle output: [1, 4] (1.55s) - Crux output: [1, 4] - push: OK (1.68s) - Compiling and running oracle program (0.13s) - Oracle output: [4, 1, 2, 3, 5] (1.55s) - Crux output: [4, 1, 2, 3, 5] - rotate_left: OK (1.70s) + Oracle output: () (1.84s) + Crux output: () + unsize: OK (2.00s) Compiling and running oracle program (0.14s) - Oracle output: [3, 4, 5, 1, 2] (1.57s) - Crux output: [3, 4, 5, 1, 2] - pop: OK (1.60s) - Compiling and running oracle program (0.13s) - Oracle output: [5, 4, 3, 1, 2] (1.46s) - Crux output: [5, 4, 3, 1, 2] - iter_clone: OK (1.69s) - Compiling and running oracle program (0.15s) - Oracle output: [1, 2, 3, 2, 3] (1.54s) - Crux output: [1, 2, 3, 2, 3] - iter - loop: OK (0.94s) - Compiling and running oracle program (0.12s) - Oracle output: 2 (0.82s) + Oracle output: 2 (1.86s) Crux output: 2 - from_fn: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.85s) - Crux output: () - filter_chain: OK (1.08s) - Compiling and running oracle program (0.15s) - Oracle output: 0 (0.93s) - Crux output: 0 - for: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: 47 (0.87s) - Crux output: 47 - zip: OK (1.06s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.92s) - Crux output: () - peek: OK (1.05s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.91s) - Crux output: 3 - cloned: OK (1.02s) - Compiling and running oracle program (0.13s) - Oracle output: 6 (0.89s) - Crux output: 6 - sum: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.88s) - Crux output: () - crypto - add: OK (1.04s) - Compiling and running oracle program (0.13s) - Oracle output: true (0.91s) - Crux output: true - add_noL: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: false (0.85s) - Crux output: false - statics - promoted_static: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.81s) - Crux output: 1 - promoted_fn: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.81s) + struct: OK (1.99s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.84s) Crux output: 1 ptr - offset_mut: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.82s) - Crux output: 3 - dangling_eq: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.85s) - Crux output: () - is_null_slice: FAIL (expected: can't unsize null pointers) (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: (false, true) (0.84s) + is_null_slice: FAIL (expected: can't unsize null pointers) (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: (false, true) (1.75s) Crux output: failures: - ---- is_null_slice/d474a1e5::crux_test[0] counterexamples ---- + ---- is_null_slice/3549f883::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/ptr/is_null_slice.rs:7:13: 7:33: error: in is_null_slice/d474a1e5::crux_test[0] + [Crux] test/conc_eval/ptr/is_null_slice.rs:7:13: 7:33: error: in is_null_slice/3549f883::crux_test[0] [Crux] attempted subindex on the result of an integer-to-pointer cast [Crux] Found counterexample for verification goal [Crux] ./lib/core/src/ptr/const_ptr.rs:38:38: 38:42: error: in core/092bc89a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst55af81a2cef2701c[0] @@ -621,723 +622,608 @@ FAIL (expected: Should panic, but doesn't) (0.05s) test/Test.hs:123: crux doesn't match oracle (expected failure) - coerce_unsized: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: (1, 2) (0.82s) - Crux output: (1, 2) - is_null: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: (false, true) (0.81s) - Crux output: (false, true) - offset: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.81s) + struct_eq: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.72s) + Crux output: () + read_write: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: [1, 3, 1] (1.76s) + Crux output: [1, 3, 1] + offset_mut: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.72s) Crux output: 3 - offset_from: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: -2 (0.83s) + copy: OK (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: [1, 2, 3, 1, 2, 3] (1.76s) + Crux output: [1, 2, 3, 1, 2, 3] + unsize_slice: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: (1, 2) (1.72s) + Crux output: (1, 2) + dangling_eq: OK (1.92s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.78s) + Crux output: () + cast_eq: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.73s) + Crux output: () + offset_from: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: -2 (1.74s) Crux output: -2 - struct_eq: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.83s) + null_eq: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.73s) + Crux output: 1 + coerce_unsized: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: (1, 2) (1.72s) + Crux output: (1, 2) + offset: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.72s) + Crux output: 3 + is_null: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: (false, true) (1.73s) + Crux output: (false, true) + valid_eq: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.72s) + Crux output: () + ops + deref2: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.74s) Crux output: () - valid_eq: OK (0.95s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.82s) + index2: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.72s) Crux output: () - cast_eq: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.84s) + arith1: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.73s) Crux output: () - copy: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: [1, 2, 3, 1, 2, 3] (0.88s) - Crux output: [1, 2, 3, 1, 2, 3] - unsize_slice: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: (1, 2) (0.83s) - Crux output: (1, 2) - read_write: OK (1.01s) - Compiling and running oracle program (0.13s) - Oracle output: [1, 3, 1] (0.88s) - Crux output: [1, 3, 1] - null_eq: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.83s) + index3: OK (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.76s) + Crux output: () + deref1: OK (1.84s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.70s) + Crux output: () + index1: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.73s) + Crux output: () + deref3: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.74s) + Crux output: () + statics + promoted_fn: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.76s) Crux output: 1 - str - to_owned: OK (1.17s) - Compiling and running oracle program (0.13s) - Oracle output: true (1.04s) - Crux output: true - format_struct: FAIL (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: true (1.72s) - Crux output: - failures: - - ---- format_struct/af5dc5a7::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] - [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] - - [Crux] Overall status: Invalid. - - test/Test.hs:123: - crux doesn't match oracle - - Use -p '/format_struct/' to rerun this test only. - format_array: FAIL (1.78s) - Compiling and running oracle program (0.13s) - Oracle output: true (1.64s) - Crux output: - failures: - - ---- format_array/15683c6d::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] - [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] - - [Crux] Overall status: Invalid. - - test/Test.hs:123: - crux doesn't match oracle - - Use -p '/format_array/' to rerun this test only. - format_int: FAIL (1.71s) - Compiling and running oracle program (0.14s) - Oracle output: true (1.57s) - Crux output: - failures: - - ---- format_int/66f79d5f::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] - [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] - - [Crux] Overall status: Invalid. - - test/Test.hs:123: - crux doesn't match oracle - - Use -p '/format_int/' to rerun this test only. - format: FAIL (1.74s) - Compiling and running oracle program (0.14s) - Oracle output: true (1.60s) - Crux output: - failures: - - ---- format/fa538280::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] - [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] - - [Crux] Overall status: Invalid. - - test/Test.hs:123: - crux doesn't match oracle - - Use -p '$0=="crux-mir.crux concrete..str.format"' to rerun this test only. - format_hex: FAIL (1.70s) - Compiling and running oracle program (0.13s) - Oracle output: true (1.57s) - Crux output: - failures: - - ---- format_hex/efa14f18::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/092bc89a::fmt[0]::write[0] - [Crux] Translation error in core/092bc89a::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/092bc89a::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] - - [Crux] Overall status: Invalid. - - test/Test.hs:123: - crux doesn't match oracle - - Use -p '/format_hex/' to rerun this test only. - string_push: OK (1.65s) + promoted_static: OK (1.86s) Compiling and running oracle program (0.14s) - Oracle output: true (1.51s) - Crux output: true + Oracle output: 1 (1.72s) + Crux output: 1 + io + cursor_write2: OK (3.09s) + Compiling and running oracle program (0.16s) + Oracle output: 0 (2.93s) + Crux output: 0 + cursor_write: OK (2.52s) + Compiling and running oracle program (0.16s) + Oracle output: 0 (2.36s) + Crux output: 0 + cursor_read: OK (2.51s) + Compiling and running oracle program (0.16s) + Oracle output: 0 (2.35s) + Crux output: 0 + mem + maybe_uninit: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.72s) + Crux output: 1 + maybe_uninit_array_cast: OK (1.87s) + Compiling and running oracle program (0.15s) + Oracle output: [1, 2] (1.72s) + Crux output: [1, 2] time - instant: OK (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.86s) + instant: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.75s) Crux output: () - enum - arg2: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: 0 (0.84s) - Crux output: 0 - field_order: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.81s) + struct + field_order: OK (1.86s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.71s) Crux output: () - ret: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: (A(42), B { x: 42 }, C) (0.80s) - Crux output: (A(42), B { x: 42 }, C) - eq: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.84s) + arg: OK (1.87s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.72s) + Crux output: 42 + tup: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.70s) Crux output: () - inner: OK (0.94s) - Compiling and running oracle program (0.12s) - Oracle output: 4 (0.81s) - Crux output: 4 - cow: OK (1.09s) + proj: OK (1.84s) Compiling and running oracle program (0.14s) - Oracle output: 200 (0.96s) - Crux output: 200 - cmp: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: -23 (0.80s) - Crux output: -23 - match: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.81s) + Oracle output: 42 (1.70s) Crux output: 42 - arg: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: 0 (0.82s) + ret: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: S { x: 42, y: 120 } (1.71s) + Crux output: S { x: 42, y: 120 } + repr_transparent: OK (1.98s) + Compiling and running oracle program (0.14s) + Oracle output: 6 (1.84s) + Crux output: 6 + repr_transparent_const: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 124 (1.73s) + Crux output: 124 + dyn + assoc_ty: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 100 (1.73s) + Crux output: 100 + trait_param: OK (1.90s) + Compiling and running oracle program (0.15s) + Oracle output: 100 (1.75s) + Crux output: 100 + plain_trait: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 100 (1.73s) + Crux output: 100 + inherit: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 4 (1.74s) + Crux output: 4 + stdlib + option: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: true (1.71s) + Crux output: true + option2: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.71s) Crux output: 0 - mixed_discrs: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.83s) - Crux output: () - sync - arc: OK (1.11s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.99s) + result: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 27 (1.72s) + Crux output: 27 + default: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: true (1.71s) + Crux output: true + cvt: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.73s) + Crux output: 0 + range: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 10 (1.73s) + Crux output: 10 + poly: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.71s) Crux output: 1 - atomic_swap: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: (2, 1) (0.86s) - Crux output: (2, 1) - rwlock_multi: OK (2.18s) + result_interior: OK (1.86s) Compiling and running oracle program (0.14s) - Oracle output: 3 (2.04s) + Oracle output: 3 (1.72s) Crux output: 3 - rwlock: OK (2.11s) + default_impl: OK (1.86s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.71s) + Crux output: () + option3: OK (1.84s) Compiling and running oracle program (0.14s) - Oracle output: 2 (1.97s) - Crux output: 2 - atomic_fence: OK (1.00s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.87s) + Oracle output: 27 (1.70s) + Crux output: 27 + teq: OK (1.84s) + Compiling and running oracle program (0.15s) + Oracle output: false (1.70s) + Crux output: false + array + wick2: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: true (1.72s) + Crux output: true + arg: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.72s) Crux output: 2 - mutex_multi: OK (2.08s) + iter: OK (1.91s) Compiling and running oracle program (0.14s) - Oracle output: 3 (1.95s) + Oracle output: 3 (1.77s) Crux output: 3 - atomic_add: OK (1.07s) - Compiling and running oracle program (0.13s) - Oracle output: 6 (0.93s) - Crux output: 6 - arc_clone: OK (1.11s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.98s) - Crux output: 2 - arc_cell: OK (1.13s) + mk_and_proj: OK (1.85s) Compiling and running oracle program (0.14s) - Oracle output: 4 (1.00s) - Crux output: 4 - mutex: OK (2.05s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (1.91s) - Crux output: 1 - atomic_cxchg: OK (1.09s) + Oracle output: 42 (1.72s) + Crux output: 42 + wick3: FAIL (expected: needs Vec data structure from stdlib) (0.20s) + Compiling and running oracle program (0.16s) + Oracle output: true (0.04s) + user error (Error 101 while running mir-json on test/conc_eval/array/wick3.rs) + (expected failure) + clone: OK (1.99s) Compiling and running oracle program (0.15s) - Oracle output: 6 (0.94s) - Crux output: 6 - intTest - test0038: OK (1.05s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.92s) + Oracle output: () (1.84s) Crux output: () - test0039: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: 7 (0.81s) + wick1: FAIL (expected: needs Vec data structure from stdlib) (0.20s) + Compiling and running oracle program (0.16s) + Oracle output: true (0.04s) + user error (Error 101 while running mir-json on test/conc_eval/array/wick1.rs) + (expected failure) + mut_index: OK (2.34s) + Compiling and running oracle program (0.15s) + Oracle output: 7 (2.19s) Crux output: 7 - stdlib - result_interior: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: 3 (0.83s) - Crux output: 3 - option3: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 27 (0.84s) - Crux output: 27 - result: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 27 (0.84s) - Crux output: 27 - poly: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.81s) + mut_arg: OK (1.86s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.72s) + Crux output: 42 + const: OK (1.84s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.70s) Crux output: 1 - default_impl: OK (0.95s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.83s) - Crux output: () - cvt: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: 0 (0.84s) - Crux output: 0 - default: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: true (0.82s) - Crux output: true - option: OK (0.96s) + from_slice: OK (2.51s) Compiling and running oracle program (0.14s) - Oracle output: true (0.82s) - Crux output: true - option2: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: 0 (0.83s) - Crux output: 0 - teq: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: false (0.84s) - Crux output: false - range: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 10 (0.81s) - Crux output: 10 - hash_map - insert_get: OK (1.32s) - Compiling and running oracle program (0.16s) - Oracle output: (11, 12) (1.16s) - Crux output: (11, 12) - insert_iter: OK (1.28s) - Compiling and running oracle program (0.16s) - Oracle output: [11, 12] (1.13s) - Crux output: [11, 12] - insert_multi: OK (1.30s) - Compiling and running oracle program (0.16s) - Oracle output: 100 (1.15s) - Crux output: 100 - insert_remove: OK (1.74s) - Compiling and running oracle program (0.16s) - Oracle output: 12 (1.58s) - Crux output: 12 - tuple - clone: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.83s) + Oracle output: () (2.37s) Crux output: () - clone_from: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.84s) - Crux output: () - clone_rec: OK (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.86s) - Crux output: () - clone_struct: OK (0.99s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.86s) + const_impl: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: 5 (1.76s) + Crux output: 5 + enum + match: OK (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: 42 (1.76s) + Crux output: 42 + field_order: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.76s) Crux output: () - consts - struct_val: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: Foo { x: true } (0.82s) - Crux output: Foo { x: true } - struct_unit: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: Err(Foo { x: () }) (0.83s) - Crux output: Err(Foo { x: () }) - enum_val: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: None (0.82s) - Crux output: None - local_key: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.83s) - Crux output: 1 - fn_def: OK (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.84s) - Crux output: 1 - array - iter: OK (1.02s) + mixed_discrs: OK (1.87s) Compiling and running oracle program (0.15s) - Oracle output: 3 (0.87s) - Crux output: 3 - const_impl: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 5 (0.81s) - Crux output: 5 - clone: OK (1.06s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.93s) + Oracle output: () (1.73s) Crux output: () - mut_index: OK (1.50s) - Compiling and running oracle program (0.13s) - Oracle output: 7 (1.37s) - Crux output: 7 - mut_arg: OK (0.94s) + arg: OK (1.85s) Compiling and running oracle program (0.14s) - Oracle output: 42 (0.81s) - Crux output: 42 - from_slice: OK (1.65s) - Compiling and running oracle program (0.13s) - Oracle output: () (1.52s) + Oracle output: 0 (1.70s) + Crux output: 0 + arg2: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 0 (1.71s) + Crux output: 0 + cow: OK (2.00s) + Compiling and running oracle program (0.15s) + Oracle output: 200 (1.85s) + Crux output: 200 + ret: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: (A(42), B { x: 42 }, C) (1.75s) + Crux output: (A(42), B { x: 42 }, C) + eq: OK (1.89s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.74s) Crux output: () - const: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.80s) - Crux output: 1 - wick2: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: true (0.83s) - Crux output: true - wick1: FAIL (expected: needs Vec data structure from stdlib) (0.19s) + cmp: OK (1.86s) Compiling and running oracle program (0.14s) - Oracle output: true (0.05s) - user error (Error 101 while running mir-json on test/conc_eval/array/wick1.rs) - (expected failure) - arg: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.80s) + Oracle output: -23 (1.72s) + Crux output: -23 + inner: OK (1.91s) + Compiling and running oracle program (0.15s) + Oracle output: 4 (1.76s) + Crux output: 4 + vec + drop: OK (2.04s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.89s) + Crux output: () + set_len: OK (2.03s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.89s) Crux output: 2 - mk_and_proj: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.80s) - Crux output: 42 - wick3: FAIL (expected: needs Vec data structure from stdlib) (0.18s) + push: OK (2.12s) + Compiling and running oracle program (0.15s) + Oracle output: (1, 2) (1.97s) + Crux output: (1, 2) + extend: OK (2.12s) + Compiling and running oracle program (0.16s) + Oracle output: (1, 10) (1.96s) + Crux output: (1, 10) + collect: OK (2.77s) + Compiling and running oracle program (0.16s) + Oracle output: 45 (2.61s) + Crux output: 45 + extend_trusted_len: OK (2.65s) + Compiling and running oracle program (0.16s) + Oracle output: (1, 10) (2.49s) + Crux output: (1, 10) + from_elem_zero: OK (2.11s) + Compiling and running oracle program (0.15s) + Oracle output: 0 (1.96s) + Crux output: 0 + clos + promoted: OK (1.87s) Compiling and running oracle program (0.14s) - Oracle output: true (0.04s) - user error (Error 101 while running mir-json on test/conc_eval/array/wick3.rs) - (expected failure) - mem - maybe_uninit_array_cast: FAIL (0.97s) - Compiling and running oracle program (0.13s) - Oracle output: [1, 2] (0.84s) - Crux output: - failures: - - ---- maybe_uninit_array_cast/171ff09d::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] internal: error: in maybe_uninit_array_cast/171ff09d::crux_test[0] - [Crux] Translation error in maybe_uninit_array_cast/171ff09d::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyArray (TyInt B32) 2]) - - [Crux] Overall status: Invalid. - - test/Test.hs:123: - crux doesn't match oracle - - Use -p '/maybe_uninit_array_cast/' to rerun this test only. - maybe_uninit: FAIL (0.96s) - Compiling and running oracle program (0.12s) - Oracle output: 1 (0.84s) + Oracle output: 33 (1.73s) + Crux output: 33 + as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (1.73s) Crux output: failures: - ---- maybe_uninit/a2fd5c40::crux_test[0] counterexamples ---- + ---- as_fn_ptr/ab0e7632::crux_test[0] counterexamples ---- [Crux] Found counterexample for verification goal - [Crux] internal: error: in maybe_uninit/a2fd5c40::crux_test[0] - [Crux] Translation error in maybe_uninit/a2fd5c40::crux_test[0]: unknown ADT (core/3a1fbbbh::mem[0]::maybe_uninit[0]::MaybeUninit[0],Substs [TyInt B32]) + [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/ab0e7632::crux_test[0] + [Crux] Translation error in as_fn_ptr/ab0e7632::crux_test[0]: unimplemented cast: ClosureFnPointer + [Crux] ty: TyClosure [] + [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [], _fsreturn_ty = TyInt B32, _fsabi = RustAbi, _fsspreadarg = Nothing}) [Crux] Overall status: Invalid. test/Test.hs:123: crux doesn't match oracle - - Use -p '$0=="crux-mir.crux concrete..mem.maybe_uninit"' to rerun this test only. - ops - index2: OK (0.95s) - Compiling and running oracle program (0.12s) - Oracle output: () (0.83s) - Crux output: () - deref1: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.81s) - Crux output: () - index1: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.81s) - Crux output: () - deref3: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.84s) - Crux output: () - arith1: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.82s) - Crux output: () - deref2: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.83s) - Crux output: () - index3: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.83s) - Crux output: () - fnptr - field: OK (0.96s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.83s) - Crux output: 2 - custom: rustc compilation failed for custom -error output: -error[E0432]: unresolved import `core` - --> test/conc_eval/fnptr/custom.rs:2:5 - | -2 | use core::mem; - | ^^^^ maybe a missing crate `core`? - | - = help: consider adding `extern crate core` to use the `core` crate - -error: aborting due to previous error - -For more information about this error, try `rustc --explain E0432`. - -FAIL (expected: taking address of an overridden function) (0.05s) - Compiling and running oracle program (0.05s) - test/Test.hs:106: - failed to compile and run (expected failure) - make: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 0 (0.81s) - Crux output: 0 - call: OK (0.96s) - Compiling and running oracle program (0.12s) - Oracle output: 2 (0.84s) - Crux output: 2 - io - cursor_write2: OK (2.19s) + fnptr_fnmut: OK (1.90s) + Compiling and running oracle program (0.15s) + Oracle output: 4 (1.76s) + Crux output: 4 + direct_fnonce: OK (1.86s) Compiling and running oracle program (0.14s) - Oracle output: 0 (2.05s) - Crux output: 0 - cursor_read: OK (1.60s) - Compiling and running oracle program (0.16s) - Oracle output: 0 (1.44s) - Crux output: 0 - cursor_write: OK (1.64s) + Oracle output: 2 (1.72s) + Crux output: 2 + conv_fnonce_fnmut: OK (1.85s) Compiling and running oracle program (0.14s) - Oracle output: 0 (1.49s) - Crux output: 0 - cell - ref_cell2: OK (1.52s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (1.39s) + Oracle output: 2 (1.71s) Crux output: 2 - cell: OK (0.98s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.86s) + fn_static_poly: OK (1.90s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.76s) + Crux output: 3 + fnptr_fnonce: OK (1.86s) + Compiling and running oracle program (0.15s) + Oracle output: 4 (1.71s) + Crux output: 4 + fn_static: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.73s) + Crux output: 3 + fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.73s) + vtable signature mismatch for vtable core/092bc89a::ops[0]::function[0]::Fn[0]::_vtbl570ffd80324c2ca9[0] of trait core/092bc89a::ops[0]::function[0]::Fn[0]::_trait42f8b6475ee85722[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32)] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32)] + CallStack (from HasCallStack): + error, called at src/Mir/Trans.hs:964:20 in crucible-mir-0.1-inplace:Mir.Trans + mkTraitObject, called at src/Mir/Trans.hs:807:11 in crucible-mir-0.1-inplace:Mir.Trans + evalCast', called at src/Mir/Trans.hs:938:5 in crucible-mir-0.1-inplace:Mir.Trans + evalCast, called at src/Mir/Trans.hs:990:30 in crucible-mir-0.1-inplace:Mir.Trans + evalRval, called at src/Mir/Trans.hs:1209:11 in crucible-mir-0.1-inplace:Mir.Trans + transStatement, called at src/Mir/Trans.hs:1654:68 in crucible-mir-0.1-inplace:Mir.Trans + translateBlockBody, called at src/Mir/Trans.hs:1663:28 in crucible-mir-0.1-inplace:Mir.Trans + registerBlock, called at src/Mir/Trans.hs:1703:10 in crucible-mir-0.1-inplace:Mir.Trans + genFn, called at src/Mir/Trans.hs:1749:24 in crucible-mir-0.1-inplace:Mir.Trans + transDefine, called at src/Mir/Trans.hs:2206:30 in crucible-mir-0.1-inplace:Mir.Trans + transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate + translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language + (expected failure) + fnonce: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: false (1.72s) + Crux output: false + conv_fnmut_fn: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.72s) Crux output: 2 - ref_cell: OK (1.53s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (1.40s) + direct_fnmut2: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 7 (1.72s) + Crux output: 7 + fo: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 29 (1.72s) + Crux output: 29 + unique_borrow: OK (1.86s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.72s) + Crux output: 3 + conv_fnonce_fn: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 2 (1.73s) Crux output: 2 - struct - repr_transparent: OK (1.03s) - Compiling and running oracle program (0.13s) - Oracle output: 6 (0.90s) - Crux output: 6 - tup: OK (0.92s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.79s) - Crux output: () - repr_transparent_const: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 124 (0.81s) - Crux output: 124 - field_order: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.82s) - Crux output: () - ret: OK (0.92s) - Compiling and running oracle program (0.13s) - Oracle output: S { x: 42, y: 120 } (0.80s) - Crux output: S { x: 42, y: 120 } - proj: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.80s) - Crux output: 42 - arg: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 42 (0.80s) - Crux output: 42 - refs - temp: OK (0.95s) - Compiling and running oracle program (0.12s) - Oracle output: 1 (0.82s) - Crux output: 1 - static_mut: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: 2 (0.82s) + fnptr_fn: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 4 (1.73s) + Crux output: 4 + direct_fnmut: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 7 (1.71s) + Crux output: 7 + dispatch_fnmut: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 7 (1.73s) + Crux output: 7 + poly: OK (1.84s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.71s) + Crux output: 3 + direct_fn: OK (1.87s) + Compiling and running oracle program (0.15s) + Oracle output: 2 (1.73s) Crux output: 2 - imm_raw: OK (0.93s) - Compiling and running oracle program (0.12s) - Oracle output: 123 (0.81s) - Crux output: 123 - mut_tuple_field: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.82s) - Crux output: () - mut_ref: OK (0.92s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.80s) - Crux output: 1 - mut_arg: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.82s) + fnptr_closure: OK (1.90s) + Compiling and running oracle program (0.15s) + Oracle output: 7 (1.75s) + Crux output: 7 + fnonce1: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.70s) + Crux output: 3 + ref_fnmut: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 7 (1.73s) + Crux output: 7 + vec_deque + pop: OK (2.53s) + Compiling and running oracle program (0.15s) + Oracle output: [5, 4, 3, 1, 2] (2.37s) + Crux output: [5, 4, 3, 1, 2] + iter_clone: OK (2.56s) + Compiling and running oracle program (0.15s) + Oracle output: [1, 2, 3, 2, 3] (2.40s) + Crux output: [1, 2, 3, 2, 3] + push: OK (2.56s) + Compiling and running oracle program (0.15s) + Oracle output: [4, 1, 2, 3, 5] (2.41s) + Crux output: [4, 1, 2, 3, 5] + retain: OK (2.60s) + Compiling and running oracle program (0.15s) + Oracle output: [1, 4] (2.45s) + Crux output: [1, 4] + rotate_left: OK (2.56s) + Compiling and running oracle program (0.15s) + Oracle output: [3, 4, 5, 1, 2] (2.41s) + Crux output: [3, 4, 5, 1, 2] + rotate_right: OK (2.64s) + Compiling and running oracle program (0.15s) + Oracle output: [4, 5, 1, 2, 3] (2.49s) + Crux output: [4, 5, 1, 2, 3] + slice + len: OK (1.87s) + Compiling and running oracle program (0.14s) + Oracle output: 5 (1.73s) + Crux output: 5 + mut_range: OK (2.35s) + Compiling and running oracle program (0.14s) + Oracle output: 86 (2.21s) + Crux output: 86 + range_len: OK (2.34s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (2.20s) Crux output: 1 - never: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.81s) + range_len_mut: OK (2.33s) + Compiling and running oracle program (0.14s) + Oracle output: 1 (2.18s) Crux output: 1 - mut_raw: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 123 (0.81s) - Crux output: 123 - promoted_imm: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 0 (0.81s) - Crux output: 0 - mut_nested: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: () (0.82s) + mk_and_proj: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (1.75s) + Crux output: 42 + swap: OK (1.89s) + Compiling and running oracle program (0.14s) + Oracle output: 2001 (1.75s) + Crux output: 2001 + eq: OK (1.93s) + Compiling and running oracle program (0.14s) + Oracle output: () (1.79s) Crux output: () - imm_ref: OK (0.93s) - Compiling and running oracle program (0.12s) - Oracle output: 123 (0.80s) - Crux output: 123 - fn_ptr_mut: OK (0.93s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.80s) - Crux output: 1 - promoted_mut: OK (0.95s) - Compiling and running oracle program (0.13s) - Oracle output: 0 (0.82s) - Crux output: 0 - imm_arg: OK (0.92s) - Compiling and running oracle program (0.12s) - Oracle output: 1 (0.79s) - Crux output: 1 - fn_ptr: OK (0.95s) - Compiling and running oracle program (0.12s) - Oracle output: 1 (0.83s) - Crux output: 1 - never_mut: OK (0.94s) - Compiling and running oracle program (0.13s) - Oracle output: 1 (0.81s) - Crux output: 1 + iter_mut: OK (1.98s) + Compiling and running oracle program (0.15s) + Oracle output: () (1.83s) + Crux output: () + get: OK (1.88s) + Compiling and running oracle program (0.15s) + Oracle output: true (1.74s) + Crux output: true + mut: OK (1.85s) + Compiling and running oracle program (0.14s) + Oracle output: 42 (1.71s) + Crux output: 42 + last: OK (1.88s) + Compiling and running oracle program (0.14s) + Oracle output: 3 (1.74s) + Crux output: 3 crux symbolic Output testing - macro: OK (1.02s) - clone: OK (1.08s) - sort_by_key: OK (1.64s) - into_iter: OK (1.53s) - as_slice: OK (0.89s) - replicate: OK (0.93s) - split_at: OK (0.92s) - concat: OK (0.89s) - copy_from_slice: OK (0.88s) - new: OK (0.95s) - as_mut_slice: OK (0.91s) - mut: OK (0.93s) - push: OK (0.94s) - pop: FAIL (0.95s) - files test/symb_eval/vector/pop.good and test/symb_eval/vector/pop.out differ; test/symb_eval/vector/pop.out contains: - test pop/::f[0]: FAILED - - failures: - - ---- pop/::f[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/symb_eval/vector/pop.rs:9:18: 9:19: error: in pop/::f[0] - [Crux] Translation error in pop/::f[0]: unknown ADT (core/::option[0]::Option[0],Substs [TyUint B8]) - - [Crux] Overall status: Invalid. - - Use -p '/Output testing.pop/' to rerun this test only. - checked_div: OK (0.85s) - checked_add: OK (0.84s) - checked_mul_signed: OK (0.83s) - checked_mul: OK (0.82s) - array_mut: OK (0.96s) - array: OK (0.92s) - bad_symb1: OK (0.90s) - override5: OK (0.95s) - override3: OK (0.94s) - override_rust: OK (0.88s) - override1: OK (0.83s) - override2: OK (0.94s) - bad_symb2: OK (0.89s) - override4: OK (0.90s) - write: FAIL - test/symb_eval/byteorder/write.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/write.rs) - Use -p '/Output testing.write/' to rerun this test only. - read: FAIL (0.05s) - test/symb_eval/byteorder/read.out: withFile: user error (Error 101 while running mir-json on test/symb_eval/byteorder/read.rs) - Use -p '/Output testing.read/' to rerun this test only. - deserialize: OK (0.92s) - construct: OK (0.92s) - double: OK (0.89s) - bytes: OK (1.01s) - ffs: OK (1.05s) - bytes2: OK (1.07s) - from_to: OK (0.85s) - overflowing_sub: OK (0.93s) - literals: OK (0.83s) - arith: OK (1.06s) - leading_zeros: OK (1.08s) - cmp: OK (0.92s) - symbolic: OK (0.88s) - mux: OK (0.90s) - test1: OK (100.82s) - split_off: OK (1.39s) - split_to: OK (1.37s) - new: OK (0.89s) - put: OK (1.33s) - sym_len: OK (1.40s) - extend_bytes: OK (0.96s) - put_overflow: OK (1.44s) - multi: OK (1.01s) - fail_return: OK (0.95s) - mixed_fail: OK (0.99s) - early_fail: OK (0.97s) - basic: OK (0.90s) - slice: OK (1.05s) - slice_mut: OK (0.93s) - mux_slice: OK (1.05s) - downcast_fail: OK (0.98s) - downcast: OK (0.96s) - conditional: OK (0.85s) - mux: OK (0.96s) - vec_cursor_read: OK (1.97s) - vec_write: OK (1.44s) - mux_init_mut: OK (0.91s) - mux_init_imm: OK (0.94s) - uninit_read: OK (0.91s) - valid_read: OK (0.92s) - zero_length: OK (0.91s) - out_of_bounds: OK (0.98s) - no_conc: OK (0.90s) - assert: FAIL (1.58s) - files test/symb_eval/concretize/assert.good and test/symb_eval/concretize/assert.out differ; test/symb_eval/concretize/assert.out contains: - test assert/::crux_test[0]: returned 1, FAILED - - failures: - - ---- assert/::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] lib/core/src/fmt/mod.rs:1200:40: 1200:46: error: in core/::fmt[0]::write[0] - [Crux] Translation error in core/::fmt[0]::write[0]: deref: bad repr for TyRef (TyDynamic core/::fmt[0]::Write[0]::_traite97b8e14d91a3b61[0]) Mut: StructRepr [AnyRepr, AnyRepr] - - [Crux] Overall status: Invalid. - - Use -p '$0=="crux-mir.crux symbolic.Output testing.assert"' to rerun this test only. - array: OK (1.04s) - conc: OK (0.99s) - assert_ok: OK (1.56s) + mux_init_mut: OK (1.77s) + mux_init_imm: OK (1.86s) + early_fail: OK (1.86s) + mixed_fail: OK (1.86s) + multi: OK (1.88s) + fail_return: OK (1.82s) + no_conc: OK (1.84s) + conc: OK (1.90s) + array: OK (1.83s) + assert_ok: OK (2.64s) + assert: OK (2.67s) + bytes2: OK (2.07s) + double: OK (1.88s) + ffs: OK (2.02s) + bytes: OK (2.15s) + bad_symb1: OK (1.88s) + bad_symb2: OK (1.89s) + override3: OK (1.86s) + override1: OK (1.78s) + override4: OK (1.90s) + override_rust: OK (1.86s) + override2: OK (1.86s) + override5: OK (1.93s) + mux: OK (2.12s) + checked_mul_signed: OK (1.94s) + checked_mul: OK (1.84s) + checked_div: OK (1.81s) + checked_add: OK (1.76s) + test1: OK (120.41s) + pop: OK (1.98s) + as_mut_slice: OK (2.00s) + push: OK (1.96s) + copy_from_slice: OK (2.05s) + new: OK (2.03s) + mut: OK (1.87s) + concat: OK (1.89s) + split_at: OK (1.96s) + replicate: OK (1.97s) + as_slice: OK (1.84s) + array_mut: OK (1.85s) + array: OK (1.86s) + extend_bytes: OK (1.86s) + put: OK (2.48s) + split_to: OK (2.62s) + new: OK (2.08s) + split_off: OK (2.48s) + sym_len: OK (2.57s) + put_overflow: OK (2.57s) + vec_cursor_read: OK (3.17s) + vec_write: OK (2.54s) + write: OK (3.36s) + read: OK (3.02s) + slice_mut: OK (1.91s) + basic: OK (1.95s) + slice: OK (2.12s) + mux_slice: OK (1.93s) + mux: OK (2.07s) + uninit_read: OK (1.94s) + valid_read: OK (1.87s) + out_of_bounds: OK (1.84s) + zero_length: OK (1.88s) + downcast: OK (1.95s) + downcast_fail: OK (1.88s) + conditional: OK (1.92s) + literals: OK (1.73s) + arith: OK (2.00s) + leading_zeros: OK (1.94s) + overflowing_sub: OK (1.86s) + cmp: OK (1.86s) + symbolic: OK (1.78s) + from_to: OK (1.73s) + clone: OK (2.04s) + sort_by_key: OK (2.57s) + into_iter: OK (2.39s) + macro: OK (1.85s) + construct: OK (1.87s) + deserialize: OK (1.84s) crux coverage Output testing - coverage_try: warning: trailing semicolon in macro used in expression position + coverage_dual: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1351,8 +1237,8 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (1.02s) - coverage_cond: warning: trailing semicolon in macro used in expression position +OK (1.78s) + coverage: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1366,8 +1252,8 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (0.83s) - coverage: warning: trailing semicolon in macro used in expression position +OK (1.93s) + coverage_try: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1381,8 +1267,8 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (0.97s) - coverage_mono: warning: trailing semicolon in macro used in expression position +OK (1.90s) + coverage_cond: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1396,8 +1282,8 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (0.87s) - coverage_macro: warning: trailing semicolon in macro used in expression position +OK (1.79s) + coverage_loop: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1411,8 +1297,8 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (0.97s) - coverage_dual: warning: trailing semicolon in macro used in expression position +OK (1.82s) + coverage_shortcircuit: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1426,8 +1312,8 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (0.90s) - coverage_match: warning: trailing semicolon in macro used in expression position +OK (1.87s) + coverage_mono: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1441,8 +1327,8 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (1.03s) - coverage_shortcircuit: warning: trailing semicolon in macro used in expression position +OK (1.81s) + coverage_match: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1456,8 +1342,8 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (1.01s) - coverage_loop: warning: trailing semicolon in macro used in expression position +OK (1.88s) + coverage_macro: warning: trailing semicolon in macro used in expression position --> src/main.rs:45:46 | 45 | return Err(format!($($args)*).into()); @@ -1471,6 +1357,6 @@ FAIL (expected: taking address of an overridden function) (0.05s) = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) -OK (0.95s) +OK (1.90s) -11 out of 339 tests failed (452.57s) +All 339 tests passed (788.49s) From 370cb619b7ba70d65e1c9bf5c194eff27ea6e4ab Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 29 Jun 2023 10:43:21 -0400 Subject: [PATCH 091/114] crux-mir: Include test crate This is needed to power `mir-json`'s `cargo crux-test` command. --- crux-mir/build.sh | 9 + crux-mir/lib/getopts/.cargo-ok | 1 + crux-mir/lib/getopts/.cargo_vcs_info.json | 5 + .../lib/getopts/.github/workflows/main.yml | 15 + crux-mir/lib/getopts/.gitignore | 6 + crux-mir/lib/getopts/Cargo.toml | 40 + crux-mir/lib/getopts/Cargo.toml.orig | 25 + crux-mir/lib/getopts/LICENSE-APACHE | 201 +++ crux-mir/lib/getopts/LICENSE-MIT | 25 + crux-mir/lib/getopts/README.md | 15 + crux-mir/lib/getopts/src/lib.rs | 1046 ++++++++++++++ crux-mir/lib/getopts/src/tests/mod.rs | 1254 +++++++++++++++++ crux-mir/lib/getopts/tests/smoke.rs | 8 + crux-mir/lib/test/Cargo.toml | 36 + crux-mir/lib/test/src/bench.rs | 243 ++++ crux-mir/lib/test/src/cli.rs | 494 +++++++ crux-mir/lib/test/src/console.rs | 307 ++++ crux-mir/lib/test/src/event.rs | 36 + crux-mir/lib/test/src/formatters/json.rs | 259 ++++ crux-mir/lib/test/src/formatters/junit.rs | 180 +++ crux-mir/lib/test/src/formatters/mod.rs | 42 + crux-mir/lib/test/src/formatters/pretty.rs | 281 ++++ crux-mir/lib/test/src/formatters/terse.rs | 259 ++++ crux-mir/lib/test/src/helpers/concurrency.rs | 14 + crux-mir/lib/test/src/helpers/exit_code.rs | 20 + crux-mir/lib/test/src/helpers/metrics.rs | 50 + crux-mir/lib/test/src/helpers/mod.rs | 7 + crux-mir/lib/test/src/helpers/shuffle.rs | 67 + crux-mir/lib/test/src/lib.rs | 798 +++++++++++ crux-mir/lib/test/src/options.rs | 82 ++ crux-mir/lib/test/src/stats.rs | 302 ++++ crux-mir/lib/test/src/stats/tests.rs | 591 ++++++++ crux-mir/lib/test/src/term.rs | 85 ++ crux-mir/lib/test/src/term/terminfo/mod.rs | 205 +++ crux-mir/lib/test/src/term/terminfo/parm.rs | 532 +++++++ .../lib/test/src/term/terminfo/parm/tests.rs | 124 ++ .../test/src/term/terminfo/parser/compiled.rs | 336 +++++ .../term/terminfo/parser/compiled/tests.rs | 8 + .../lib/test/src/term/terminfo/searcher.rs | 69 + .../test/src/term/terminfo/searcher/tests.rs | 19 + crux-mir/lib/test/src/term/win.rs | 169 +++ crux-mir/lib/test/src/test_result.rs | 107 ++ crux-mir/lib/test/src/tests.rs | 835 +++++++++++ crux-mir/lib/test/src/time.rs | 195 +++ crux-mir/lib/test/src/types.rs | 168 +++ crux-mir/lib/unicode_width/.cargo-ok | 1 + .../lib/unicode_width/.cargo_vcs_info.json | 6 + crux-mir/lib/unicode_width/.gitignore | 3 + crux-mir/lib/unicode_width/.travis.yml | 28 + crux-mir/lib/unicode_width/COPYRIGHT | 7 + crux-mir/lib/unicode_width/Cargo.toml | 60 + crux-mir/lib/unicode_width/Cargo.toml.orig | 29 + crux-mir/lib/unicode_width/LICENSE-APACHE | 201 +++ crux-mir/lib/unicode_width/LICENSE-MIT | 25 + crux-mir/lib/unicode_width/README.md | 58 + crux-mir/lib/unicode_width/scripts/unicode.py | 505 +++++++ crux-mir/lib/unicode_width/src/lib.rs | 131 ++ crux-mir/lib/unicode_width/src/tables.rs | 540 +++++++ crux-mir/lib/unicode_width/src/tests.rs | 191 +++ 59 files changed, 11355 insertions(+) create mode 100644 crux-mir/lib/getopts/.cargo-ok create mode 100644 crux-mir/lib/getopts/.cargo_vcs_info.json create mode 100644 crux-mir/lib/getopts/.github/workflows/main.yml create mode 100644 crux-mir/lib/getopts/.gitignore create mode 100644 crux-mir/lib/getopts/Cargo.toml create mode 100644 crux-mir/lib/getopts/Cargo.toml.orig create mode 100644 crux-mir/lib/getopts/LICENSE-APACHE create mode 100644 crux-mir/lib/getopts/LICENSE-MIT create mode 100644 crux-mir/lib/getopts/README.md create mode 100644 crux-mir/lib/getopts/src/lib.rs create mode 100644 crux-mir/lib/getopts/src/tests/mod.rs create mode 100644 crux-mir/lib/getopts/tests/smoke.rs create mode 100644 crux-mir/lib/test/Cargo.toml create mode 100644 crux-mir/lib/test/src/bench.rs create mode 100644 crux-mir/lib/test/src/cli.rs create mode 100644 crux-mir/lib/test/src/console.rs create mode 100644 crux-mir/lib/test/src/event.rs create mode 100644 crux-mir/lib/test/src/formatters/json.rs create mode 100644 crux-mir/lib/test/src/formatters/junit.rs create mode 100644 crux-mir/lib/test/src/formatters/mod.rs create mode 100644 crux-mir/lib/test/src/formatters/pretty.rs create mode 100644 crux-mir/lib/test/src/formatters/terse.rs create mode 100644 crux-mir/lib/test/src/helpers/concurrency.rs create mode 100644 crux-mir/lib/test/src/helpers/exit_code.rs create mode 100644 crux-mir/lib/test/src/helpers/metrics.rs create mode 100644 crux-mir/lib/test/src/helpers/mod.rs create mode 100644 crux-mir/lib/test/src/helpers/shuffle.rs create mode 100644 crux-mir/lib/test/src/lib.rs create mode 100644 crux-mir/lib/test/src/options.rs create mode 100644 crux-mir/lib/test/src/stats.rs create mode 100644 crux-mir/lib/test/src/stats/tests.rs create mode 100644 crux-mir/lib/test/src/term.rs create mode 100644 crux-mir/lib/test/src/term/terminfo/mod.rs create mode 100644 crux-mir/lib/test/src/term/terminfo/parm.rs create mode 100644 crux-mir/lib/test/src/term/terminfo/parm/tests.rs create mode 100644 crux-mir/lib/test/src/term/terminfo/parser/compiled.rs create mode 100644 crux-mir/lib/test/src/term/terminfo/parser/compiled/tests.rs create mode 100644 crux-mir/lib/test/src/term/terminfo/searcher.rs create mode 100644 crux-mir/lib/test/src/term/terminfo/searcher/tests.rs create mode 100644 crux-mir/lib/test/src/term/win.rs create mode 100644 crux-mir/lib/test/src/test_result.rs create mode 100644 crux-mir/lib/test/src/tests.rs create mode 100644 crux-mir/lib/test/src/time.rs create mode 100644 crux-mir/lib/test/src/types.rs create mode 100644 crux-mir/lib/unicode_width/.cargo-ok create mode 100644 crux-mir/lib/unicode_width/.cargo_vcs_info.json create mode 100644 crux-mir/lib/unicode_width/.gitignore create mode 100644 crux-mir/lib/unicode_width/.travis.yml create mode 100644 crux-mir/lib/unicode_width/COPYRIGHT create mode 100644 crux-mir/lib/unicode_width/Cargo.toml create mode 100644 crux-mir/lib/unicode_width/Cargo.toml.orig create mode 100644 crux-mir/lib/unicode_width/LICENSE-APACHE create mode 100644 crux-mir/lib/unicode_width/LICENSE-MIT create mode 100644 crux-mir/lib/unicode_width/README.md create mode 100755 crux-mir/lib/unicode_width/scripts/unicode.py create mode 100644 crux-mir/lib/unicode_width/src/lib.rs create mode 100644 crux-mir/lib/unicode_width/src/tables.rs create mode 100644 crux-mir/lib/unicode_width/src/tests.rs diff --git a/crux-mir/build.sh b/crux-mir/build.sh index 2201adbd8..e530fab12 100755 --- a/crux-mir/build.sh +++ b/crux-mir/build.sh @@ -73,6 +73,15 @@ translate lib/std/src/lib.rs --edition=2021 --crate-name std --cfg 'feature="add echo 'Building proc_macro...' translate lib/proc_macro/src/lib.rs --edition=2021 --crate-name proc_macro --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib +echo 'Building unicode_width...' +translate lib/unicode_width/src/lib.rs --crate-name unicode_width --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="default"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib + +echo 'Building getopts...' +translate lib/getopts/src/lib.rs --crate-name getopts --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="std"' --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib --extern unicode_width=rlibs/libunicode_width.rlib + +echo 'Building test...' +translate lib/test/src/lib.rs --edition=2021 --crate-name test --cfg 'feature="backtrace"' --cfg 'feature="default"' --cfg 'feature="panic-unwind"' --cfg 'feature="std_detect_dlsym_getauxval"' --cfg 'feature="std_detect_file_io"' --extern cfg_if=rlibs/libcfg_if.rlib --extern core=rlibs/libcore.rlib --extern getopts=rlibs/libgetopts.rlib --extern libc=rlibs/liblibc.rlib --extern panic_abort=rlibs/libpanic_abort.rlib --extern panic_unwind=rlibs/libpanic_unwind.rlib --extern proc_macro=rlibs/libproc_macro.rlib --extern std=rlibs/libstd.rlib + # extra libs (added manually) echo 'Building int512...' translate lib/int512.rs --crate-name int512 --extern core=rlibs/libcore.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib diff --git a/crux-mir/lib/getopts/.cargo-ok b/crux-mir/lib/getopts/.cargo-ok new file mode 100644 index 000000000..b5754e203 --- /dev/null +++ b/crux-mir/lib/getopts/.cargo-ok @@ -0,0 +1 @@ +ok \ No newline at end of file diff --git a/crux-mir/lib/getopts/.cargo_vcs_info.json b/crux-mir/lib/getopts/.cargo_vcs_info.json new file mode 100644 index 000000000..8e9a38a0f --- /dev/null +++ b/crux-mir/lib/getopts/.cargo_vcs_info.json @@ -0,0 +1,5 @@ +{ + "git": { + "sha1": "be919b8323f57ba238ea9cd6e68190029809e278" + } +} diff --git a/crux-mir/lib/getopts/.github/workflows/main.yml b/crux-mir/lib/getopts/.github/workflows/main.yml new file mode 100644 index 000000000..2abb4bd62 --- /dev/null +++ b/crux-mir/lib/getopts/.github/workflows/main.yml @@ -0,0 +1,15 @@ +name: CI +on: [push, pull_request] +jobs: + test: + name: Test + runs-on: ubuntu-latest + strategy: + matrix: + rust: [stable, beta, nightly] + steps: + - uses: actions/checkout@master + - name: Install Rust + run: rustup update ${{ matrix.rust }} && rustup default ${{ matrix.rust }} + - run: cargo test + - run: cargo doc diff --git a/crux-mir/lib/getopts/.gitignore b/crux-mir/lib/getopts/.gitignore new file mode 100644 index 000000000..f0d90b95b --- /dev/null +++ b/crux-mir/lib/getopts/.gitignore @@ -0,0 +1,6 @@ +# Cargo +/target +/Cargo.lock + +#IDE +.vscode \ No newline at end of file diff --git a/crux-mir/lib/getopts/Cargo.toml b/crux-mir/lib/getopts/Cargo.toml new file mode 100644 index 000000000..db224fcf5 --- /dev/null +++ b/crux-mir/lib/getopts/Cargo.toml @@ -0,0 +1,40 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies +# +# If you believe there's an error in this file please file an +# issue against the rust-lang/cargo repository. If you're +# editing this file be aware that the upstream Cargo.toml +# will likely look very different (and much more reasonable) + +[package] +name = "getopts" +version = "0.2.21" +authors = ["The Rust Project Developers"] +description = "getopts-like option parsing.\n" +homepage = "https://github.com/rust-lang/getopts" +documentation = "https://doc.rust-lang.org/getopts" +readme = "README.md" +categories = ["command-line-interface"] +license = "MIT/Apache-2.0" +repository = "https://github.com/rust-lang/getopts" +[dependencies.core] +version = "1.0" +optional = true +package = "rustc-std-workspace-core" + +[dependencies.std] +version = "1.0" +optional = true +package = "rustc-std-workspace-std" + +[dependencies.unicode-width] +version = "0.1.5" +[dev-dependencies.log] +version = "0.4" + +[features] +rustc-dep-of-std = ["unicode-width/rustc-dep-of-std", "std", "core"] diff --git a/crux-mir/lib/getopts/Cargo.toml.orig b/crux-mir/lib/getopts/Cargo.toml.orig new file mode 100644 index 000000000..b55900245 --- /dev/null +++ b/crux-mir/lib/getopts/Cargo.toml.orig @@ -0,0 +1,25 @@ +[package] + +name = "getopts" +version = "0.2.21" # don't forget to update html_root_url +authors = ["The Rust Project Developers"] +license = "MIT/Apache-2.0" +readme = "README.md" +repository = "https://github.com/rust-lang/getopts" +documentation = "https://doc.rust-lang.org/getopts" +homepage = "https://github.com/rust-lang/getopts" +description = """ +getopts-like option parsing. +""" +categories = ["command-line-interface"] + +[dependencies] +unicode-width = "0.1.5" +std = { version = "1.0", package = "rustc-std-workspace-std", optional = true } +core = { version = "1.0", package = "rustc-std-workspace-core", optional = true } + +[dev-dependencies] +log = "0.4" + +[features] +rustc-dep-of-std = ['unicode-width/rustc-dep-of-std', 'std', 'core'] diff --git a/crux-mir/lib/getopts/LICENSE-APACHE b/crux-mir/lib/getopts/LICENSE-APACHE new file mode 100644 index 000000000..16fe87b06 --- /dev/null +++ b/crux-mir/lib/getopts/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/crux-mir/lib/getopts/LICENSE-MIT b/crux-mir/lib/getopts/LICENSE-MIT new file mode 100644 index 000000000..39d4bdb5a --- /dev/null +++ b/crux-mir/lib/getopts/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2014 The Rust Project Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/crux-mir/lib/getopts/README.md b/crux-mir/lib/getopts/README.md new file mode 100644 index 000000000..065a05638 --- /dev/null +++ b/crux-mir/lib/getopts/README.md @@ -0,0 +1,15 @@ +getopts +=== + +A Rust library for option parsing for CLI utilities. + +[Documentation](https://docs.rs/getopts) + +## Usage + +Add this to your `Cargo.toml`: + +```toml +[dependencies] +getopts = "0.2" +``` diff --git a/crux-mir/lib/getopts/src/lib.rs b/crux-mir/lib/getopts/src/lib.rs new file mode 100644 index 000000000..fd45b9508 --- /dev/null +++ b/crux-mir/lib/getopts/src/lib.rs @@ -0,0 +1,1046 @@ +// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// ignore-lexer-test FIXME #15677 + +//! Simple getopt alternative. +//! +//! Construct a vector of options, either by using `reqopt`, `optopt`, and +//! `optflag` or by building them from components yourself, and pass them to +//! `getopts`, along with a vector of actual arguments (not including +//! `argv[0]`). You'll either get a failure code back, or a match. You'll have +//! to verify whether the amount of 'free' arguments in the match is what you +//! expect. Use `opt_*` accessors to get argument values out of the matches +//! object. +//! +//! Single-character options are expected to appear on the command line with a +//! single preceding dash; multiple-character options are expected to be +//! proceeded by two dashes. Options that expect an argument accept their +//! argument following either a space or an equals sign. Single-character +//! options don't require the space. +//! +//! # Usage +//! +//! This crate is [on crates.io](https://crates.io/crates/getopts) and can be +//! used by adding `getopts` to the dependencies in your project's `Cargo.toml`. +//! +//! ```toml +//! [dependencies] +//! getopts = "0.2" +//! ``` +//! +//! and this to your crate root: +//! +//! ```rust +//! extern crate getopts; +//! ``` +//! +//! # Example +//! +//! The following example shows simple command line parsing for an application +//! that requires an input file to be specified, accepts an optional output file +//! name following `-o`, and accepts both `-h` and `--help` as optional flags. +//! +//! ```{.rust} +//! extern crate getopts; +//! use getopts::Options; +//! use std::env; +//! +//! fn do_work(inp: &str, out: Option) { +//! println!("{}", inp); +//! match out { +//! Some(x) => println!("{}", x), +//! None => println!("No Output"), +//! } +//! } +//! +//! fn print_usage(program: &str, opts: Options) { +//! let brief = format!("Usage: {} FILE [options]", program); +//! print!("{}", opts.usage(&brief)); +//! } +//! +//! fn main() { +//! let args: Vec = env::args().collect(); +//! let program = args[0].clone(); +//! +//! let mut opts = Options::new(); +//! opts.optopt("o", "", "set output file name", "NAME"); +//! opts.optflag("h", "help", "print this help menu"); +//! let matches = match opts.parse(&args[1..]) { +//! Ok(m) => { m } +//! Err(f) => { panic!(f.to_string()) } +//! }; +//! if matches.opt_present("h") { +//! print_usage(&program, opts); +//! return; +//! } +//! let output = matches.opt_str("o"); +//! let input = if !matches.free.is_empty() { +//! matches.free[0].clone() +//! } else { +//! print_usage(&program, opts); +//! return; +//! }; +//! do_work(&input, output); +//! } +//! ``` + +#![doc( + html_logo_url = "https://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png", + html_favicon_url = "https://www.rust-lang.org/favicon.ico", + html_root_url = "https://docs.rs/getopts/0.2.20" +)] +#![deny(missing_docs)] +#![cfg_attr(test, deny(warnings))] + +#[cfg(test)] +#[macro_use] +extern crate log; +extern crate unicode_width; + +use self::Fail::*; +use self::HasArg::*; +use self::Name::*; +use self::Occur::*; +use self::Optval::*; + +use std::error::Error; +use std::ffi::OsStr; +use std::fmt; +use std::iter::{repeat, IntoIterator}; +use std::result; +use std::str::FromStr; + +use unicode_width::UnicodeWidthStr; + +#[cfg(test)] +mod tests; + +/// A description of the options that a program can handle. +pub struct Options { + grps: Vec, + parsing_style: ParsingStyle, + long_only: bool, +} + +impl Default for Options { + fn default() -> Self { + Self::new() + } +} + +impl Options { + /// Create a blank set of options. + pub fn new() -> Options { + Options { + grps: Vec::new(), + parsing_style: ParsingStyle::FloatingFrees, + long_only: false, + } + } + + /// Set the parsing style. + pub fn parsing_style(&mut self, style: ParsingStyle) -> &mut Options { + self.parsing_style = style; + self + } + + /// Set or clear "long options only" mode. + /// + /// In "long options only" mode, short options cannot be clustered + /// together, and long options can be given with either a single + /// "-" or the customary "--". This mode also changes the meaning + /// of "-a=b"; in the ordinary mode this will parse a short option + /// "-a" with argument "=b"; whereas in long-options-only mode the + /// argument will be simply "b". + pub fn long_only(&mut self, long_only: bool) -> &mut Options { + self.long_only = long_only; + self + } + + /// Create a generic option group, stating all parameters explicitly. + pub fn opt( + &mut self, + short_name: &str, + long_name: &str, + desc: &str, + hint: &str, + hasarg: HasArg, + occur: Occur, + ) -> &mut Options { + validate_names(short_name, long_name); + self.grps.push(OptGroup { + short_name: short_name.to_string(), + long_name: long_name.to_string(), + hint: hint.to_string(), + desc: desc.to_string(), + hasarg, + occur, + }); + self + } + + /// Create a long option that is optional and does not take an argument. + /// + /// * `short_name` - e.g. `"h"` for a `-h` option, or `""` for none + /// * `long_name` - e.g. `"help"` for a `--help` option, or `""` for none + /// * `desc` - Description for usage help + pub fn optflag(&mut self, short_name: &str, long_name: &str, desc: &str) -> &mut Options { + validate_names(short_name, long_name); + self.grps.push(OptGroup { + short_name: short_name.to_string(), + long_name: long_name.to_string(), + hint: "".to_string(), + desc: desc.to_string(), + hasarg: No, + occur: Optional, + }); + self + } + + /// Create a long option that can occur more than once and does not + /// take an argument. + /// + /// * `short_name` - e.g. `"h"` for a `-h` option, or `""` for none + /// * `long_name` - e.g. `"help"` for a `--help` option, or `""` for none + /// * `desc` - Description for usage help + pub fn optflagmulti(&mut self, short_name: &str, long_name: &str, desc: &str) -> &mut Options { + validate_names(short_name, long_name); + self.grps.push(OptGroup { + short_name: short_name.to_string(), + long_name: long_name.to_string(), + hint: "".to_string(), + desc: desc.to_string(), + hasarg: No, + occur: Multi, + }); + self + } + + /// Create a long option that is optional and takes an optional argument. + /// + /// * `short_name` - e.g. `"h"` for a `-h` option, or `""` for none + /// * `long_name` - e.g. `"help"` for a `--help` option, or `""` for none + /// * `desc` - Description for usage help + /// * `hint` - Hint that is used in place of the argument in the usage help, + /// e.g. `"FILE"` for a `-o FILE` option + pub fn optflagopt( + &mut self, + short_name: &str, + long_name: &str, + desc: &str, + hint: &str, + ) -> &mut Options { + validate_names(short_name, long_name); + self.grps.push(OptGroup { + short_name: short_name.to_string(), + long_name: long_name.to_string(), + hint: hint.to_string(), + desc: desc.to_string(), + hasarg: Maybe, + occur: Optional, + }); + self + } + + /// Create a long option that is optional, takes an argument, and may occur + /// multiple times. + /// + /// * `short_name` - e.g. `"h"` for a `-h` option, or `""` for none + /// * `long_name` - e.g. `"help"` for a `--help` option, or `""` for none + /// * `desc` - Description for usage help + /// * `hint` - Hint that is used in place of the argument in the usage help, + /// e.g. `"FILE"` for a `-o FILE` option + pub fn optmulti( + &mut self, + short_name: &str, + long_name: &str, + desc: &str, + hint: &str, + ) -> &mut Options { + validate_names(short_name, long_name); + self.grps.push(OptGroup { + short_name: short_name.to_string(), + long_name: long_name.to_string(), + hint: hint.to_string(), + desc: desc.to_string(), + hasarg: Yes, + occur: Multi, + }); + self + } + + /// Create a long option that is optional and takes an argument. + /// + /// * `short_name` - e.g. `"h"` for a `-h` option, or `""` for none + /// * `long_name` - e.g. `"help"` for a `--help` option, or `""` for none + /// * `desc` - Description for usage help + /// * `hint` - Hint that is used in place of the argument in the usage help, + /// e.g. `"FILE"` for a `-o FILE` option + pub fn optopt( + &mut self, + short_name: &str, + long_name: &str, + desc: &str, + hint: &str, + ) -> &mut Options { + validate_names(short_name, long_name); + self.grps.push(OptGroup { + short_name: short_name.to_string(), + long_name: long_name.to_string(), + hint: hint.to_string(), + desc: desc.to_string(), + hasarg: Yes, + occur: Optional, + }); + self + } + + /// Create a long option that is required and takes an argument. + /// + /// * `short_name` - e.g. `"h"` for a `-h` option, or `""` for none + /// * `long_name` - e.g. `"help"` for a `--help` option, or `""` for none + /// * `desc` - Description for usage help + /// * `hint` - Hint that is used in place of the argument in the usage help, + /// e.g. `"FILE"` for a `-o FILE` option + pub fn reqopt( + &mut self, + short_name: &str, + long_name: &str, + desc: &str, + hint: &str, + ) -> &mut Options { + validate_names(short_name, long_name); + self.grps.push(OptGroup { + short_name: short_name.to_string(), + long_name: long_name.to_string(), + hint: hint.to_string(), + desc: desc.to_string(), + hasarg: Yes, + occur: Req, + }); + self + } + + /// Parse command line arguments according to the provided options. + /// + /// On success returns `Ok(Matches)`. Use methods such as `opt_present` + /// `opt_str`, etc. to interrogate results. + /// # Panics + /// + /// Returns `Err(Fail)` on failure: use the `Debug` implementation of `Fail` + /// to display information about it. + pub fn parse(&self, args: C) -> Result + where + C::Item: AsRef, + { + let opts: Vec = self.grps.iter().map(|x| x.long_to_short()).collect(); + + let mut vals = (0..opts.len()) + .map(|_| Vec::new()) + .collect::>>(); + let mut free: Vec = Vec::new(); + let args = args + .into_iter() + .map(|i| { + i.as_ref() + .to_str() + .ok_or_else(|| Fail::UnrecognizedOption(format!("{:?}", i.as_ref()))) + .map(|s| s.to_owned()) + }) + .collect::<::std::result::Result, _>>()?; + let mut args = args.into_iter().peekable(); + let mut arg_pos = 0; + while let Some(cur) = args.next() { + if !is_arg(&cur) { + free.push(cur); + match self.parsing_style { + ParsingStyle::FloatingFrees => {} + ParsingStyle::StopAtFirstFree => { + free.extend(args); + break; + } + } + } else if cur == "--" { + free.extend(args); + break; + } else { + let mut names; + let mut i_arg = None; + let mut was_long = true; + if cur.as_bytes()[1] == b'-' || self.long_only { + let tail = if cur.as_bytes()[1] == b'-' { + &cur[2..] + } else { + assert!(self.long_only); + &cur[1..] + }; + let mut parts = tail.splitn(2, '='); + names = vec![Name::from_str(parts.next().unwrap())]; + if let Some(rest) = parts.next() { + i_arg = Some(rest.to_string()); + } + } else { + was_long = false; + names = Vec::new(); + for (j, ch) in cur.char_indices().skip(1) { + let opt = Short(ch); + + /* In a series of potential options (eg. -aheJ), if we + see one which takes an argument, we assume all + subsequent characters make up the argument. This + allows options such as -L/usr/local/lib/foo to be + interpreted correctly + */ + + let opt_id = match find_opt(&opts, &opt) { + Some(id) => id, + None => return Err(UnrecognizedOption(opt.to_string())), + }; + + names.push(opt); + + let arg_follows = match opts[opt_id].hasarg { + Yes | Maybe => true, + No => false, + }; + + if arg_follows { + let next = j + ch.len_utf8(); + if next < cur.len() { + i_arg = Some(cur[next..].to_string()); + break; + } + } + } + } + let mut name_pos = 0; + for nm in names.iter() { + name_pos += 1; + let optid = match find_opt(&opts, &nm) { + Some(id) => id, + None => return Err(UnrecognizedOption(nm.to_string())), + }; + match opts[optid].hasarg { + No => { + if name_pos == names.len() && i_arg.is_some() { + return Err(UnexpectedArgument(nm.to_string())); + } + vals[optid].push((arg_pos, Given)); + } + Maybe => { + // Note that here we do not handle `--arg value`. + // This matches GNU getopt behavior; but also + // makes sense, because if this were accepted, + // then users could only write a "Maybe" long + // option at the end of the arguments when + // FloatingFrees is in use. + if let Some(i_arg) = i_arg.take() { + vals[optid].push((arg_pos, Val(i_arg))); + } else if was_long + || name_pos < names.len() + || args.peek().map_or(true, |n| is_arg(&n)) + { + vals[optid].push((arg_pos, Given)); + } else { + vals[optid].push((arg_pos, Val(args.next().unwrap()))); + } + } + Yes => { + if let Some(i_arg) = i_arg.take() { + vals[optid].push((arg_pos, Val(i_arg))); + } else if let Some(n) = args.next() { + vals[optid].push((arg_pos, Val(n))); + } else { + return Err(ArgumentMissing(nm.to_string())); + } + } + } + } + } + arg_pos += 1; + } + debug_assert_eq!(vals.len(), opts.len()); + for (vals, opt) in vals.iter().zip(opts.iter()) { + if opt.occur == Req && vals.is_empty() { + return Err(OptionMissing(opt.name.to_string())); + } + if opt.occur != Multi && vals.len() > 1 { + return Err(OptionDuplicated(opt.name.to_string())); + } + } + Ok(Matches { opts, vals, free }) + } + + /// Derive a short one-line usage summary from a set of long options. + pub fn short_usage(&self, program_name: &str) -> String { + let mut line = format!("Usage: {} ", program_name); + line.push_str( + &self + .grps + .iter() + .map(format_option) + .collect::>() + .join(" "), + ); + line + } + + /// Derive a formatted message from a set of options. + pub fn usage(&self, brief: &str) -> String { + self.usage_with_format(|opts| { + format!( + "{}\n\nOptions:\n{}\n", + brief, + opts.collect::>().join("\n") + ) + }) + } + + /// Derive a custom formatted message from a set of options. The formatted options provided to + /// a closure as an iterator. + pub fn usage_with_format) -> String>( + &self, + mut formatter: F, + ) -> String { + formatter(&mut self.usage_items()) + } + + /// Derive usage items from a set of options. + fn usage_items<'a>(&'a self) -> Box + 'a> { + let desc_sep = format!("\n{}", repeat(" ").take(24).collect::()); + + let any_short = self.grps.iter().any(|optref| !optref.short_name.is_empty()); + + let rows = self.grps.iter().map(move |optref| { + let OptGroup { + short_name, + long_name, + hint, + desc, + hasarg, + .. + } = (*optref).clone(); + + let mut row = " ".to_string(); + + // short option + match short_name.width() { + 0 => { + if any_short { + row.push_str(" "); + } + } + 1 => { + row.push('-'); + row.push_str(&short_name); + if long_name.width() > 0 { + row.push_str(", "); + } else { + // Only a single space here, so that any + // argument is printed in the correct spot. + row.push(' '); + } + } + // FIXME: refer issue #7. + _ => panic!("the short name should only be 1 ascii char long"), + } + + // long option + match long_name.width() { + 0 => {} + _ => { + row.push_str(if self.long_only { "-" } else { "--" }); + row.push_str(&long_name); + row.push(' '); + } + } + + // arg + match hasarg { + No => {} + Yes => row.push_str(&hint), + Maybe => { + row.push('['); + row.push_str(&hint); + row.push(']'); + } + } + + let rowlen = row.width(); + if rowlen < 24 { + for _ in 0..24 - rowlen { + row.push(' '); + } + } else { + row.push_str(&desc_sep) + } + + let desc_rows = each_split_within(&desc, 54); + row.push_str(&desc_rows.join(&desc_sep)); + + row + }); + + Box::new(rows) + } +} + +fn validate_names(short_name: &str, long_name: &str) { + let len = short_name.len(); + assert!( + len == 1 || len == 0, + "the short_name (first argument) should be a single character, \ + or an empty string for none" + ); + let len = long_name.len(); + assert!( + len == 0 || len > 1, + "the long_name (second argument) should be longer than a single \ + character, or an empty string for none" + ); +} + +/// What parsing style to use when parsing arguments. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum ParsingStyle { + /// Flags and "free" arguments can be freely inter-mixed. + FloatingFrees, + /// As soon as a "free" argument (i.e. non-flag) is encountered, stop + /// considering any remaining arguments as flags. + StopAtFirstFree, +} + +/// Name of an option. Either a string or a single char. +#[derive(Clone, Debug, PartialEq, Eq)] +enum Name { + /// A string representing the long name of an option. + /// For example: "help" + Long(String), + /// A char representing the short name of an option. + /// For example: 'h' + Short(char), +} + +/// Describes whether an option has an argument. +#[derive(Clone, Debug, Copy, PartialEq, Eq)] +pub enum HasArg { + /// The option requires an argument. + Yes, + /// The option takes no argument. + No, + /// The option argument is optional. + Maybe, +} + +/// Describes how often an option may occur. +#[derive(Clone, Debug, Copy, PartialEq, Eq)] +pub enum Occur { + /// The option occurs once. + Req, + /// The option occurs at most once. + Optional, + /// The option occurs zero or more times. + Multi, +} + +/// A description of a possible option. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Opt { + /// Name of the option + name: Name, + /// Whether it has an argument + hasarg: HasArg, + /// How often it can occur + occur: Occur, + /// Which options it aliases + aliases: Vec, +} + +/// One group of options, e.g., both `-h` and `--help`, along with +/// their shared description and properties. +#[derive(Clone, PartialEq, Eq)] +struct OptGroup { + /// Short name of the option, e.g. `h` for a `-h` option + short_name: String, + /// Long name of the option, e.g. `help` for a `--help` option + long_name: String, + /// Hint for argument, e.g. `FILE` for a `-o FILE` option + hint: String, + /// Description for usage help text + desc: String, + /// Whether option has an argument + hasarg: HasArg, + /// How often it can occur + occur: Occur, +} + +/// Describes whether an option is given at all or has a value. +#[derive(Clone, Debug, PartialEq, Eq)] +enum Optval { + Val(String), + Given, +} + +/// The result of checking command line arguments. Contains a vector +/// of matches and a vector of free strings. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Matches { + /// Options that matched + opts: Vec, + /// Values of the Options that matched and their positions + vals: Vec>, + /// Free string fragments + pub free: Vec, +} + +/// The type returned when the command line does not conform to the +/// expected format. Use the `Debug` implementation to output detailed +/// information. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Fail { + /// The option requires an argument but none was passed. + ArgumentMissing(String), + /// The passed option is not declared among the possible options. + UnrecognizedOption(String), + /// A required option is not present. + OptionMissing(String), + /// A single occurrence option is being used multiple times. + OptionDuplicated(String), + /// There's an argument being passed to a non-argument option. + UnexpectedArgument(String), +} + +impl Error for Fail { + fn description(&self) -> &str { + match *self { + ArgumentMissing(_) => "missing argument", + UnrecognizedOption(_) => "unrecognized option", + OptionMissing(_) => "missing option", + OptionDuplicated(_) => "duplicated option", + UnexpectedArgument(_) => "unexpected argument", + } + } +} + +/// The result of parsing a command line with a set of options. +pub type Result = result::Result; + +impl Name { + fn from_str(nm: &str) -> Name { + if nm.len() == 1 { + Short(nm.as_bytes()[0] as char) + } else { + Long(nm.to_string()) + } + } + + fn to_string(&self) -> String { + match *self { + Short(ch) => ch.to_string(), + Long(ref s) => s.to_string(), + } + } +} + +impl OptGroup { + /// Translate OptGroup into Opt. + /// (Both short and long names correspond to different Opts). + fn long_to_short(&self) -> Opt { + let OptGroup { + short_name, + long_name, + hasarg, + occur, + .. + } = (*self).clone(); + + match (short_name.len(), long_name.len()) { + (0, 0) => panic!("this long-format option was given no name"), + (0, _) => Opt { + name: Long(long_name), + hasarg, + occur, + aliases: Vec::new(), + }, + (1, 0) => Opt { + name: Short(short_name.as_bytes()[0] as char), + hasarg, + occur, + aliases: Vec::new(), + }, + (1, _) => Opt { + name: Long(long_name), + hasarg, + occur, + aliases: vec![Opt { + name: Short(short_name.as_bytes()[0] as char), + hasarg: hasarg, + occur: occur, + aliases: Vec::new(), + }], + }, + (_, _) => panic!("something is wrong with the long-form opt"), + } + } +} + +impl Matches { + fn opt_vals(&self, nm: &str) -> Vec<(usize, Optval)> { + match find_opt(&self.opts, &Name::from_str(nm)) { + Some(id) => self.vals[id].clone(), + None => panic!("No option '{}' defined", nm), + } + } + + fn opt_val(&self, nm: &str) -> Option { + self.opt_vals(nm).into_iter().map(|(_, o)| o).next() + } + /// Returns true if an option was defined + pub fn opt_defined(&self, nm: &str) -> bool { + find_opt(&self.opts, &Name::from_str(nm)).is_some() + } + + /// Returns true if an option was matched. + pub fn opt_present(&self, nm: &str) -> bool { + !self.opt_vals(nm).is_empty() + } + + /// Returns the number of times an option was matched. + pub fn opt_count(&self, nm: &str) -> usize { + self.opt_vals(nm).len() + } + + /// Returns a vector of all the positions in which an option was matched. + pub fn opt_positions(&self, nm: &str) -> Vec { + self.opt_vals(nm).into_iter().map(|(pos, _)| pos).collect() + } + + /// Returns true if any of several options were matched. + pub fn opts_present(&self, names: &[String]) -> bool { + names + .iter() + .any(|nm| match find_opt(&self.opts, &Name::from_str(&nm)) { + Some(id) if !self.vals[id].is_empty() => true, + _ => false, + }) + } + + /// Returns the string argument supplied to one of several matching options or `None`. + pub fn opts_str(&self, names: &[String]) -> Option { + names + .iter() + .filter_map(|nm| match self.opt_val(&nm) { + Some(Val(s)) => Some(s), + _ => None, + }) + .next() + } + + /// Returns a vector of the arguments provided to all matches of the given + /// option. + /// + /// Used when an option accepts multiple values. + pub fn opt_strs(&self, nm: &str) -> Vec { + self.opt_vals(nm) + .into_iter() + .filter_map(|(_, v)| match v { + Val(s) => Some(s), + _ => None, + }) + .collect() + } + + /// Returns a vector of the arguments provided to all matches of the given + /// option, together with their positions. + /// + /// Used when an option accepts multiple values. + pub fn opt_strs_pos(&self, nm: &str) -> Vec<(usize, String)> { + self.opt_vals(nm) + .into_iter() + .filter_map(|(p, v)| match v { + Val(s) => Some((p, s)), + _ => None, + }) + .collect() + } + + /// Returns the string argument supplied to a matching option or `None`. + pub fn opt_str(&self, nm: &str) -> Option { + match self.opt_val(nm) { + Some(Val(s)) => Some(s), + _ => None, + } + } + + /// Returns the matching string, a default, or `None`. + /// + /// Returns `None` if the option was not present, `def` if the option was + /// present but no argument was provided, and the argument if the option was + /// present and an argument was provided. + pub fn opt_default(&self, nm: &str, def: &str) -> Option { + match self.opt_val(nm) { + Some(Val(s)) => Some(s), + Some(_) => Some(def.to_string()), + None => None, + } + } + + /// Returns some matching value or `None`. + /// + /// Similar to opt_str, also converts matching argument using FromStr. + pub fn opt_get(&self, nm: &str) -> result::Result, T::Err> + where + T: FromStr, + { + match self.opt_val(nm) { + Some(Val(s)) => Ok(Some(s.parse()?)), + Some(Given) => Ok(None), + None => Ok(None), + } + } + + /// Returns a matching value or default. + /// + /// Similar to opt_default, except the two differences. + /// Instead of returning None when argument was not present, return `def`. + /// Instead of returning &str return type T, parsed using str::parse(). + pub fn opt_get_default(&self, nm: &str, def: T) -> result::Result + where + T: FromStr, + { + match self.opt_val(nm) { + Some(Val(s)) => s.parse(), + Some(Given) => Ok(def), + None => Ok(def), + } + } +} + +fn is_arg(arg: &str) -> bool { + arg.as_bytes().get(0) == Some(&b'-') && arg.len() > 1 +} + +fn find_opt(opts: &[Opt], nm: &Name) -> Option { + // Search main options. + let pos = opts.iter().position(|opt| &opt.name == nm); + if pos.is_some() { + return pos; + } + + // Search in aliases. + for candidate in opts.iter() { + if candidate.aliases.iter().any(|opt| &opt.name == nm) { + return opts.iter().position(|opt| opt.name == candidate.name); + } + } + + None +} + +impl fmt::Display for Fail { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + ArgumentMissing(ref nm) => write!(f, "Argument to option '{}' missing", *nm), + UnrecognizedOption(ref nm) => write!(f, "Unrecognized option: '{}'", *nm), + OptionMissing(ref nm) => write!(f, "Required option '{}' missing", *nm), + OptionDuplicated(ref nm) => write!(f, "Option '{}' given more than once", *nm), + UnexpectedArgument(ref nm) => write!(f, "Option '{}' does not take an argument", *nm), + } + } +} + +fn format_option(opt: &OptGroup) -> String { + let mut line = String::new(); + + if opt.occur != Req { + line.push('['); + } + + // Use short_name if possible, but fall back to long_name. + if !opt.short_name.is_empty() { + line.push('-'); + line.push_str(&opt.short_name); + } else { + line.push_str("--"); + line.push_str(&opt.long_name); + } + + if opt.hasarg != No { + line.push(' '); + if opt.hasarg == Maybe { + line.push('['); + } + line.push_str(&opt.hint); + if opt.hasarg == Maybe { + line.push(']'); + } + } + + if opt.occur != Req { + line.push(']'); + } + if opt.occur == Multi { + line.push_str(".."); + } + + line +} + +/// Splits a string into substrings with possibly internal whitespace, +/// each of them at most `lim` bytes long, if possible. The substrings +/// have leading and trailing whitespace removed, and are only cut at +/// whitespace boundaries. +fn each_split_within(desc: &str, lim: usize) -> Vec { + let mut rows = Vec::new(); + for line in desc.trim().lines() { + let line_chars = line.chars().chain(Some(' ')); + let words = line_chars + .fold((Vec::new(), 0, 0), |(mut words, a, z), c| { + let idx = z + c.len_utf8(); // Get the current byte offset + + // If the char is whitespace, advance the word start and maybe push a word + if c.is_whitespace() { + if a != z { + words.push(&line[a..z]); + } + (words, idx, idx) + } + // If the char is not whitespace, continue, retaining the current + else { + (words, a, idx) + } + }) + .0; + + let mut row = String::new(); + for word in words.iter() { + let sep = if !row.is_empty() { Some(" ") } else { None }; + let width = row.width() + word.width() + sep.map(UnicodeWidthStr::width).unwrap_or(0); + + if width <= lim { + if let Some(sep) = sep { + row.push_str(sep) + } + row.push_str(word); + continue; + } + if !row.is_empty() { + rows.push(row.clone()); + row.clear(); + } + row.push_str(word); + } + if !row.is_empty() { + rows.push(row); + } + } + rows +} diff --git a/crux-mir/lib/getopts/src/tests/mod.rs b/crux-mir/lib/getopts/src/tests/mod.rs new file mode 100644 index 000000000..f1fb39098 --- /dev/null +++ b/crux-mir/lib/getopts/src/tests/mod.rs @@ -0,0 +1,1254 @@ +use super::each_split_within; +use super::Fail::*; +use super::{HasArg, Name, Occur, Opt, Options, ParsingStyle}; + +#[test] +fn test_split_within() { + fn t(s: &str, i: usize, u: &[String]) { + let v = each_split_within(&(s.to_string()), i); + assert!(v.iter().zip(u.iter()).all(|(a, b)| a == b)); + } + t("", 0, &[]); + t("", 15, &[]); + t("hello", 15, &["hello".to_string()]); + t( + "\nMary had a little lamb\nLittle lamb\n", + 15, + &[ + "Mary had a".to_string(), + "little lamb".to_string(), + "Little lamb".to_string(), + ], + ); + t( + "\nMary had a little lamb\nLittle lamb\n", + ::std::usize::MAX, + &[ + "Mary had a little lamb".to_string(), + "Little lamb".to_string(), + ], + ); +} + +// Tests for reqopt +#[test] +fn test_reqopt() { + let long_args = vec!["--test=20".to_string()]; + let mut opts = Options::new(); + opts.reqopt("t", "test", "testing", "TEST"); + match opts.parse(&long_args) { + Ok(ref m) => { + assert!(m.opt_present("test")); + assert_eq!(m.opt_str("test").unwrap(), "20"); + assert!(m.opt_present("t")); + assert_eq!(m.opt_str("t").unwrap(), "20"); + } + _ => { + panic!("test_reqopt failed (long arg)"); + } + } + let short_args = vec!["-t".to_string(), "20".to_string()]; + match opts.parse(&short_args) { + Ok(ref m) => { + assert!((m.opt_present("test"))); + assert_eq!(m.opt_str("test").unwrap(), "20"); + assert!((m.opt_present("t"))); + assert_eq!(m.opt_str("t").unwrap(), "20"); + } + _ => { + panic!("test_reqopt failed (short arg)"); + } + } +} + +#[test] +fn test_reqopt_missing() { + let args = vec!["blah".to_string()]; + match Options::new() + .reqopt("t", "test", "testing", "TEST") + .parse(&args) + { + Err(OptionMissing(_)) => {} + _ => panic!(), + } +} + +#[test] +fn test_reqopt_no_arg() { + let long_args = vec!["--test".to_string()]; + let mut opts = Options::new(); + opts.reqopt("t", "test", "testing", "TEST"); + match opts.parse(&long_args) { + Err(ArgumentMissing(_)) => {} + _ => panic!(), + } + let short_args = vec!["-t".to_string()]; + match opts.parse(&short_args) { + Err(ArgumentMissing(_)) => {} + _ => panic!(), + } +} + +#[test] +fn test_reqopt_multi() { + let args = vec!["--test=20".to_string(), "-t".to_string(), "30".to_string()]; + match Options::new() + .reqopt("t", "test", "testing", "TEST") + .parse(&args) + { + Err(OptionDuplicated(_)) => {} + _ => panic!(), + } +} + +// Tests for optopt +#[test] +fn test_optopt() { + let long_args = vec!["--test=20".to_string()]; + let mut opts = Options::new(); + opts.optopt("t", "test", "testing", "TEST"); + match opts.parse(&long_args) { + Ok(ref m) => { + assert!(m.opt_present("test")); + assert_eq!(m.opt_str("test").unwrap(), "20"); + assert!((m.opt_present("t"))); + assert_eq!(m.opt_str("t").unwrap(), "20"); + } + _ => panic!(), + } + let short_args = vec!["-t".to_string(), "20".to_string()]; + match opts.parse(&short_args) { + Ok(ref m) => { + assert!((m.opt_present("test"))); + assert_eq!(m.opt_str("test").unwrap(), "20"); + assert!((m.opt_present("t"))); + assert_eq!(m.opt_str("t").unwrap(), "20"); + } + _ => panic!(), + } +} + +#[test] +fn test_optopt_missing() { + let args = vec!["blah".to_string()]; + match Options::new() + .optopt("t", "test", "testing", "TEST") + .parse(&args) + { + Ok(ref m) => { + assert!(!m.opt_present("test")); + assert!(!m.opt_present("t")); + } + _ => panic!(), + } +} + +#[test] +fn test_optopt_no_arg() { + let long_args = vec!["--test".to_string()]; + let mut opts = Options::new(); + opts.optopt("t", "test", "testing", "TEST"); + match opts.parse(&long_args) { + Err(ArgumentMissing(_)) => {} + _ => panic!(), + } + let short_args = vec!["-t".to_string()]; + match opts.parse(&short_args) { + Err(ArgumentMissing(_)) => {} + _ => panic!(), + } +} + +#[test] +fn test_optopt_multi() { + let args = vec!["--test=20".to_string(), "-t".to_string(), "30".to_string()]; + match Options::new() + .optopt("t", "test", "testing", "TEST") + .parse(&args) + { + Err(OptionDuplicated(_)) => {} + _ => panic!(), + } +} + +// Tests for optflag +#[test] +fn test_optflag() { + let long_args = vec!["--test".to_string()]; + let mut opts = Options::new(); + opts.optflag("t", "test", "testing"); + match opts.parse(&long_args) { + Ok(ref m) => { + assert!(m.opt_present("test")); + assert!(m.opt_present("t")); + } + _ => panic!(), + } + let short_args = vec!["-t".to_string()]; + match opts.parse(&short_args) { + Ok(ref m) => { + assert!(m.opt_present("test")); + assert!(m.opt_present("t")); + } + _ => panic!(), + } +} + +#[test] +fn test_optflag_missing() { + let args = vec!["blah".to_string()]; + match Options::new().optflag("t", "test", "testing").parse(&args) { + Ok(ref m) => { + assert!(!m.opt_present("test")); + assert!(!m.opt_present("t")); + } + _ => panic!(), + } +} + +#[test] +fn test_opt_end() { + let args = vec!["--".to_owned(), "-t".to_owned()]; + match Options::new().optflag("t", "test", "testing").parse(&args) { + Ok(ref m) => { + assert!(!m.opt_present("test")); + assert!(!m.opt_present("t")); + assert_eq!(m.free.len(), 1); + assert_eq!(m.free[0], "-t"); + } + _ => panic!(), + } +} + +#[test] +fn test_opt_only_end() { + let args = vec!["--".to_owned()]; + match Options::new().optflag("t", "test", "testing").parse(&args) { + Ok(ref m) => { + assert!(!m.opt_present("test")); + assert!(!m.opt_present("t")); + assert_eq!(m.free.len(), 0); + } + _ => panic!(), + } +} + +#[test] +fn test_optflag_long_arg() { + let args = vec!["--test=20".to_string()]; + match Options::new().optflag("t", "test", "testing").parse(&args) { + Err(UnexpectedArgument(_)) => {} + _ => panic!(), + } +} + +#[test] +fn test_optflag_multi() { + let args = vec!["--test".to_string(), "-t".to_string()]; + match Options::new().optflag("t", "test", "testing").parse(&args) { + Err(OptionDuplicated(_)) => {} + _ => panic!(), + } +} + +#[test] +fn test_optflag_short_arg() { + let args = vec!["-t".to_string(), "20".to_string()]; + match Options::new().optflag("t", "test", "testing").parse(&args) { + Ok(ref m) => { + // The next variable after the flag is just a free argument + + assert!(m.free[0] == "20"); + } + _ => panic!(), + } +} + +// Tests for optflagmulti +#[test] +fn test_optflagmulti_short1() { + let args = vec!["-v".to_string()]; + match Options::new() + .optflagmulti("v", "verbose", "verbosity") + .parse(&args) + { + Ok(ref m) => { + assert_eq!(m.opt_count("v"), 1); + } + _ => panic!(), + } +} + +#[test] +fn test_optflagmulti_short2a() { + let args = vec!["-v".to_string(), "-v".to_string()]; + match Options::new() + .optflagmulti("v", "verbose", "verbosity") + .parse(&args) + { + Ok(ref m) => { + assert_eq!(m.opt_count("v"), 2); + } + _ => panic!(), + } +} + +#[test] +fn test_optflagmulti_short2b() { + let args = vec!["-vv".to_string()]; + match Options::new() + .optflagmulti("v", "verbose", "verbosity") + .parse(&args) + { + Ok(ref m) => { + assert_eq!(m.opt_count("v"), 2); + } + _ => panic!(), + } +} + +#[test] +fn test_optflagmulti_long1() { + let args = vec!["--verbose".to_string()]; + match Options::new() + .optflagmulti("v", "verbose", "verbosity") + .parse(&args) + { + Ok(ref m) => { + assert_eq!(m.opt_count("verbose"), 1); + } + _ => panic!(), + } +} + +#[test] +fn test_optflagmulti_long2() { + let args = vec!["--verbose".to_string(), "--verbose".to_string()]; + match Options::new() + .optflagmulti("v", "verbose", "verbosity") + .parse(&args) + { + Ok(ref m) => { + assert_eq!(m.opt_count("verbose"), 2); + } + _ => panic!(), + } +} + +#[test] +fn test_optflagmulti_mix() { + let args = vec![ + "--verbose".to_string(), + "-v".to_string(), + "-vv".to_string(), + "verbose".to_string(), + ]; + match Options::new() + .optflagmulti("v", "verbose", "verbosity") + .parse(&args) + { + Ok(ref m) => { + assert_eq!(m.opt_count("verbose"), 4); + assert_eq!(m.opt_count("v"), 4); + } + _ => panic!(), + } +} + +// Tests for optflagopt +#[test] +fn test_optflagopt() { + let long_args = vec!["--test".to_string()]; + let mut opts = Options::new(); + opts.optflagopt("t", "test", "testing", "ARG"); + match opts.parse(&long_args) { + Ok(ref m) => { + assert!(m.opt_present("test")); + assert!(m.opt_present("t")); + } + _ => panic!(), + } + let short_args = vec!["-t".to_string()]; + match opts.parse(&short_args) { + Ok(ref m) => { + assert!(m.opt_present("test")); + assert!(m.opt_present("t")); + } + _ => panic!(), + } + let short_args = vec!["-t".to_string(), "x".to_string()]; + match opts.parse(&short_args) { + Ok(ref m) => { + assert_eq!(m.opt_str("t").unwrap(), "x"); + assert_eq!(m.opt_str("test").unwrap(), "x"); + } + _ => panic!(), + } + let long_args = vec!["--test=x".to_string()]; + match opts.parse(&long_args) { + Ok(ref m) => { + assert_eq!(m.opt_str("t").unwrap(), "x"); + assert_eq!(m.opt_str("test").unwrap(), "x"); + } + _ => panic!(), + } + let long_args = vec!["--test".to_string(), "x".to_string()]; + match opts.parse(&long_args) { + Ok(ref m) => { + assert_eq!(m.opt_str("t"), None); + assert_eq!(m.opt_str("test"), None); + } + _ => panic!(), + } + let no_args: Vec = vec![]; + match opts.parse(&no_args) { + Ok(ref m) => { + assert!(!m.opt_present("test")); + assert!(!m.opt_present("t")); + } + _ => panic!(), + } +} + +// Tests for optmulti +#[test] +fn test_optmulti() { + let long_args = vec!["--test=20".to_string()]; + let mut opts = Options::new(); + opts.optmulti("t", "test", "testing", "TEST"); + match opts.parse(&long_args) { + Ok(ref m) => { + assert!((m.opt_present("test"))); + assert_eq!(m.opt_str("test").unwrap(), "20"); + assert!((m.opt_present("t"))); + assert_eq!(m.opt_str("t").unwrap(), "20"); + } + _ => panic!(), + } + let short_args = vec!["-t".to_string(), "20".to_string()]; + match opts.parse(&short_args) { + Ok(ref m) => { + assert!((m.opt_present("test"))); + assert_eq!(m.opt_str("test").unwrap(), "20"); + assert!((m.opt_present("t"))); + assert_eq!(m.opt_str("t").unwrap(), "20"); + } + _ => panic!(), + } +} + +#[test] +fn test_optmulti_missing() { + let args = vec!["blah".to_string()]; + match Options::new() + .optmulti("t", "test", "testing", "TEST") + .parse(&args) + { + Ok(ref m) => { + assert!(!m.opt_present("test")); + assert!(!m.opt_present("t")); + } + _ => panic!(), + } +} + +#[test] +fn test_optmulti_no_arg() { + let long_args = vec!["--test".to_string()]; + let mut opts = Options::new(); + opts.optmulti("t", "test", "testing", "TEST"); + match opts.parse(&long_args) { + Err(ArgumentMissing(_)) => {} + _ => panic!(), + } + let short_args = vec!["-t".to_string()]; + match opts.parse(&short_args) { + Err(ArgumentMissing(_)) => {} + _ => panic!(), + } +} + +#[test] +fn test_optmulti_multi() { + let args = vec!["--test=20".to_string(), "-t".to_string(), "30".to_string()]; + match Options::new() + .optmulti("t", "test", "testing", "TEST") + .parse(&args) + { + Ok(ref m) => { + assert!(m.opt_present("test")); + assert_eq!(m.opt_str("test").unwrap(), "20"); + assert!(m.opt_present("t")); + assert_eq!(m.opt_str("t").unwrap(), "20"); + let pair = m.opt_strs("test"); + assert!(pair[0] == "20"); + assert!(pair[1] == "30"); + } + _ => panic!(), + } +} + +#[test] +fn test_free_argument_is_hyphen() { + let args = vec!["-".to_string()]; + match Options::new().parse(&args) { + Ok(ref m) => { + assert_eq!(m.free.len(), 1); + assert_eq!(m.free[0], "-"); + } + _ => panic!(), + } +} + +#[test] +fn test_unrecognized_option() { + let long_args = vec!["--untest".to_string()]; + let mut opts = Options::new(); + opts.optmulti("t", "test", "testing", "TEST"); + match opts.parse(&long_args) { + Err(UnrecognizedOption(_)) => {} + _ => panic!(), + } + let short_args = vec!["-u".to_string()]; + match opts.parse(&short_args) { + Err(UnrecognizedOption(_)) => {} + _ => panic!(), + } +} + +#[test] +fn test_combined() { + let args = vec![ + "prog".to_string(), + "free1".to_string(), + "-s".to_string(), + "20".to_string(), + "free2".to_string(), + "--flag".to_string(), + "--long=30".to_string(), + "-f".to_string(), + "-m".to_string(), + "40".to_string(), + "-m".to_string(), + "50".to_string(), + "-n".to_string(), + "-A B".to_string(), + "-n".to_string(), + "-60 70".to_string(), + ]; + match Options::new() + .optopt("s", "something", "something", "SOMETHING") + .optflag("", "flag", "a flag") + .reqopt("", "long", "hi", "LONG") + .optflag("f", "", "another flag") + .optmulti("m", "", "mmmmmm", "YUM") + .optmulti("n", "", "nothing", "NOTHING") + .optopt("", "notpresent", "nothing to see here", "NOPE") + .parse(&args) + { + Ok(ref m) => { + assert!(m.free[0] == "prog"); + assert!(m.free[1] == "free1"); + assert_eq!(m.opt_str("s").unwrap(), "20"); + assert!(m.free[2] == "free2"); + assert!((m.opt_present("flag"))); + assert_eq!(m.opt_str("long").unwrap(), "30"); + assert!((m.opt_present("f"))); + let pair = m.opt_strs("m"); + assert!(pair[0] == "40"); + assert!(pair[1] == "50"); + let pair = m.opt_strs("n"); + assert!(pair[0] == "-A B"); + assert!(pair[1] == "-60 70"); + assert!((!m.opt_present("notpresent"))); + } + _ => panic!(), + } +} + +#[test] +fn test_mixed_stop() { + let args = vec![ + "-a".to_string(), + "b".to_string(), + "-c".to_string(), + "d".to_string(), + ]; + match Options::new() + .parsing_style(ParsingStyle::StopAtFirstFree) + .optflag("a", "", "") + .optopt("c", "", "", "") + .parse(&args) + { + Ok(ref m) => { + println!("{}", m.opt_present("c")); + assert!(m.opt_present("a")); + assert!(!m.opt_present("c")); + assert_eq!(m.free.len(), 3); + assert_eq!(m.free[0], "b"); + assert_eq!(m.free[1], "-c"); + assert_eq!(m.free[2], "d"); + } + _ => panic!(), + } +} + +#[test] +fn test_mixed_stop_hyphen() { + let args = vec![ + "-a".to_string(), + "-".to_string(), + "-c".to_string(), + "d".to_string(), + ]; + match Options::new() + .parsing_style(ParsingStyle::StopAtFirstFree) + .optflag("a", "", "") + .optopt("c", "", "", "") + .parse(&args) + { + Ok(ref m) => { + println!("{}", m.opt_present("c")); + assert!(m.opt_present("a")); + assert!(!m.opt_present("c")); + assert_eq!(m.free.len(), 3); + assert_eq!(m.free[0], "-"); + assert_eq!(m.free[1], "-c"); + assert_eq!(m.free[2], "d"); + } + _ => panic!(), + } +} + +#[test] +fn test_multi() { + let mut opts = Options::new(); + opts.optopt("e", "", "encrypt", "ENCRYPT"); + opts.optopt("", "encrypt", "encrypt", "ENCRYPT"); + opts.optopt("f", "", "flag", "FLAG"); + + let args_single = vec!["-e".to_string(), "foo".to_string()]; + let matches_single = &match opts.parse(&args_single) { + Ok(m) => m, + _ => panic!(), + }; + assert!(matches_single.opts_present(&["e".to_string()])); + assert!(matches_single.opts_present(&["encrypt".to_string(), "e".to_string()])); + assert!(matches_single.opts_present(&["e".to_string(), "encrypt".to_string()])); + assert!(!matches_single.opts_present(&["encrypt".to_string()])); + assert!(!matches_single.opts_present(&["thing".to_string()])); + assert!(!matches_single.opts_present(&[])); + + assert_eq!(matches_single.opts_str(&["e".to_string()]).unwrap(), "foo"); + assert_eq!( + matches_single + .opts_str(&["e".to_string(), "encrypt".to_string()]) + .unwrap(), + "foo" + ); + assert_eq!( + matches_single + .opts_str(&["encrypt".to_string(), "e".to_string()]) + .unwrap(), + "foo" + ); + + let args_both = vec![ + "-e".to_string(), + "foo".to_string(), + "--encrypt".to_string(), + "foo".to_string(), + ]; + let matches_both = &match opts.parse(&args_both) { + Ok(m) => m, + _ => panic!(), + }; + assert!(matches_both.opts_present(&["e".to_string()])); + assert!(matches_both.opts_present(&["encrypt".to_string()])); + assert!(matches_both.opts_present(&["encrypt".to_string(), "e".to_string()])); + assert!(matches_both.opts_present(&["e".to_string(), "encrypt".to_string()])); + assert!(!matches_both.opts_present(&["f".to_string()])); + assert!(!matches_both.opts_present(&["thing".to_string()])); + assert!(!matches_both.opts_present(&[])); + + assert_eq!(matches_both.opts_str(&["e".to_string()]).unwrap(), "foo"); + assert_eq!( + matches_both.opts_str(&["encrypt".to_string()]).unwrap(), + "foo" + ); + assert_eq!( + matches_both + .opts_str(&["e".to_string(), "encrypt".to_string()]) + .unwrap(), + "foo" + ); + assert_eq!( + matches_both + .opts_str(&["encrypt".to_string(), "e".to_string()]) + .unwrap(), + "foo" + ); +} + +#[test] +fn test_nospace() { + let args = vec!["-Lfoo".to_string(), "-M.".to_string()]; + let matches = &match Options::new() + .optmulti("L", "", "library directory", "LIB") + .optmulti("M", "", "something", "MMMM") + .parse(&args) + { + Ok(m) => m, + _ => panic!(), + }; + assert!(matches.opts_present(&["L".to_string()])); + assert_eq!(matches.opts_str(&["L".to_string()]).unwrap(), "foo"); + assert!(matches.opts_present(&["M".to_string()])); + assert_eq!(matches.opts_str(&["M".to_string()]).unwrap(), "."); +} + +#[test] +fn test_nospace_conflict() { + let args = vec!["-vvLverbose".to_string(), "-v".to_string()]; + let matches = &match Options::new() + .optmulti("L", "", "library directory", "LIB") + .optflagmulti("v", "verbose", "Verbose") + .parse(&args) + { + Ok(m) => m, + Err(e) => panic!("{}", e), + }; + assert!(matches.opts_present(&["L".to_string()])); + assert_eq!(matches.opts_str(&["L".to_string()]).unwrap(), "verbose"); + assert!(matches.opts_present(&["v".to_string()])); + assert_eq!(3, matches.opt_count("v")); +} + +#[test] +fn test_long_to_short() { + let mut short = Opt { + name: Name::Long("banana".to_string()), + hasarg: HasArg::Yes, + occur: Occur::Req, + aliases: Vec::new(), + }; + short.aliases = vec![Opt { + name: Name::Short('b'), + hasarg: HasArg::Yes, + occur: Occur::Req, + aliases: Vec::new(), + }]; + let mut opts = Options::new(); + opts.reqopt("b", "banana", "some bananas", "VAL"); + let verbose = &opts.grps[0]; + assert!(verbose.long_to_short() == short); +} + +#[test] +fn test_aliases_long_and_short() { + let args = vec!["-a".to_string(), "--apple".to_string(), "-a".to_string()]; + + let matches = Options::new() + .optflagmulti("a", "apple", "Desc") + .parse(&args) + .unwrap(); + assert_eq!(3, matches.opt_count("a")); + assert_eq!(3, matches.opt_count("apple")); +} + +#[test] +fn test_usage() { + let mut opts = Options::new(); + opts.reqopt("b", "banana", "Desc", "VAL"); + opts.optopt("a", "012345678901234567890123456789", "Desc", "VAL"); + opts.optflag("k", "kiwi", "Desc"); + opts.optflagopt("p", "", "Desc", "VAL"); + opts.optmulti("l", "", "Desc", "VAL"); + opts.optflag("", "starfruit", "Starfruit"); + + let expected = "Usage: fruits + +Options: + -b, --banana VAL Desc + -a, --012345678901234567890123456789 VAL + Desc + -k, --kiwi Desc + -p [VAL] Desc + -l VAL Desc + --starfruit Starfruit +"; + + let generated_usage = opts.usage("Usage: fruits"); + + debug!("expected: <<{}>>", expected); + debug!("generated: <<{}>>", generated_usage); + assert_eq!(generated_usage, expected); +} + +#[test] +fn test_usage_description_wrapping() { + // indentation should be 24 spaces + // lines wrap after 78: or rather descriptions wrap after 54 + + let mut opts = Options::new(); + opts.optflag( + "k", + "kiwi", + "This is a long description which won't be wrapped..+..", + ); // 54 + opts.optflag( + "a", + "apple", + "This is a long description which _will_ be wrapped..+..", + ); + opts.optflag( + "b", + "banana", + "HereWeNeedOneSingleWordThatIsLongerThanTheWrappingLengthAndThisIsIt", + ); + + let expected = "Usage: fruits + +Options: + -k, --kiwi This is a long description which won't be wrapped..+.. + -a, --apple This is a long description which _will_ be + wrapped..+.. + -b, --banana HereWeNeedOneSingleWordThatIsLongerThanTheWrappingLengthAndThisIsIt +"; + + let usage = opts.usage("Usage: fruits"); + + debug!("expected: <<{}>>", expected); + debug!("generated: <<{}>>", usage); + assert!(usage == expected) +} + +#[test] +fn test_usage_description_multibyte_handling() { + let mut opts = Options::new(); + opts.optflag( + "k", + "k\u{2013}w\u{2013}", + "The word kiwi is normally spelled with two i's", + ); + opts.optflag( + "a", + "apple", + "This \u{201C}description\u{201D} has some characters that could \ + confuse the line wrapping; an apple costs 0.51€ in some parts of Europe.", + ); + + let expected = "Usage: fruits + +Options: + -k, --k–w– The word kiwi is normally spelled with two i's + -a, --apple This “description” has some characters that could + confuse the line wrapping; an apple costs 0.51€ in + some parts of Europe. +"; + + let usage = opts.usage("Usage: fruits"); + + debug!("expected: <<{}>>", expected); + debug!("generated: <<{}>>", usage); + assert!(usage == expected) +} + +#[test] +fn test_usage_description_newline_handling() { + let mut opts = Options::new(); + opts.optflag( + "k", + "k\u{2013}w\u{2013}", + "The word kiwi is normally spelled with two i's", + ); + opts.optflag( + "a", + "apple", + "This description forces a new line.\n Here is a premature\n\ + newline", + ); + + let expected = "Usage: fruits + +Options: + -k, --k–w– The word kiwi is normally spelled with two i's + -a, --apple This description forces a new line. + Here is a premature + newline +"; + + let usage = opts.usage("Usage: fruits"); + + debug!("expected: <<{}>>", expected); + debug!("generated: <<{}>>", usage); + assert!(usage == expected) +} + +#[test] +fn test_usage_multiwidth() { + let mut opts = Options::new(); + opts.optflag("a", "apple", "apple description"); + opts.optflag("b", "banana\u{00AB}", "banana description"); + opts.optflag("c", "brûlée", "brûlée quite long description"); + opts.optflag("k", "kiwi\u{20AC}", "kiwi description"); + opts.optflag("o", "orange\u{2039}", "orange description"); + opts.optflag( + "r", + "raspberry-but-making-this-option-way-too-long", + "raspberry description is also quite long indeed longer than \ + every other piece of text we might encounter here and thus will \ + be automatically broken up", + ); + + let expected = "Usage: fruits + +Options: + -a, --apple apple description + -b, --banana« banana description + -c, --brûlée brûlée quite long description + -k, --kiwi€ kiwi description + -o, --orange‹ orange description + -r, --raspberry-but-making-this-option-way-too-long\u{0020} + raspberry description is also quite long indeed longer + than every other piece of text we might encounter here + and thus will be automatically broken up +"; + + let usage = opts.usage("Usage: fruits"); + + debug!("expected: <<{}>>", expected); + debug!("generated: <<{}>>", usage); + assert!(usage == expected) +} + +#[test] +fn test_usage_short_only() { + let mut opts = Options::new(); + opts.optopt("k", "", "Kiwi", "VAL"); + opts.optflag("s", "", "Starfruit"); + opts.optflagopt("a", "", "Apple", "TYPE"); + + let expected = "Usage: fruits + +Options: + -k VAL Kiwi + -s Starfruit + -a [TYPE] Apple +"; + + let usage = opts.usage("Usage: fruits"); + debug!("expected: <<{}>>", expected); + debug!("generated: <<{}>>", usage); + assert!(usage == expected) +} + +#[test] +fn test_usage_long_only() { + let mut opts = Options::new(); + opts.optopt("", "kiwi", "Kiwi", "VAL"); + opts.optflag("", "starfruit", "Starfruit"); + opts.optflagopt("", "apple", "Apple", "TYPE"); + + let expected = "Usage: fruits + +Options: + --kiwi VAL Kiwi + --starfruit Starfruit + --apple [TYPE] Apple +"; + + let usage = opts.usage("Usage: fruits"); + debug!("expected: <<{}>>", expected); + debug!("generated: <<{}>>", usage); + assert!(usage == expected) +} + +#[test] +fn test_short_usage() { + let mut opts = Options::new(); + opts.reqopt("b", "banana", "Desc", "VAL"); + opts.optopt("a", "012345678901234567890123456789", "Desc", "VAL"); + opts.optflag("k", "kiwi", "Desc"); + opts.optflagopt("p", "", "Desc", "VAL"); + opts.optmulti("l", "", "Desc", "VAL"); + + let expected = "Usage: fruits -b VAL [-a VAL] [-k] [-p [VAL]] [-l VAL]..".to_string(); + let generated_usage = opts.short_usage("fruits"); + + debug!("expected: <<{}>>", expected); + debug!("generated: <<{}>>", generated_usage); + assert_eq!(generated_usage, expected); +} +#[test] +fn test_nonexistant_opt() { + let mut opts = Options::new(); + opts.optflag("b", "bar", "Desc"); + let args: Vec = Vec::new(); + let matches = opts.parse(&args).unwrap(); + assert_eq!(matches.opt_defined("foo"), false); + assert_eq!(matches.opt_defined("bar"), true); +} +#[test] +fn test_args_with_equals() { + let mut opts = Options::new(); + opts.optopt("o", "one", "One", "INFO"); + opts.optopt("t", "two", "Two", "INFO"); + + let args = vec![ + "--one".to_string(), + "A=B".to_string(), + "--two=C=D".to_string(), + ]; + let matches = &match opts.parse(&args) { + Ok(m) => m, + Err(e) => panic!("{}", e), + }; + assert_eq!(matches.opts_str(&["o".to_string()]).unwrap(), "A=B"); + assert_eq!(matches.opts_str(&["t".to_string()]).unwrap(), "C=D"); +} + +#[test] +fn test_long_only_usage() { + let mut opts = Options::new(); + opts.long_only(true); + opts.optflag("k", "kiwi", "Description"); + opts.optflag("a", "apple", "Description"); + + let expected = "Usage: fruits + +Options: + -k, -kiwi Description + -a, -apple Description +"; + + let usage = opts.usage("Usage: fruits"); + + debug!("expected: <<{}>>", expected); + debug!("generated: <<{}>>", usage); + assert!(usage == expected) +} + +#[test] +fn test_long_only_mode() { + let mut opts = Options::new(); + opts.long_only(true); + opts.optopt("a", "apple", "Description", "X"); + opts.optopt("b", "banana", "Description", "X"); + opts.optopt("c", "currant", "Description", "X"); + opts.optopt("", "durian", "Description", "X"); + opts.optopt("e", "", "Description", "X"); + opts.optopt("", "fruit", "Description", "X"); + + let args = vec![ + "-a", + "A", + "-b=B", + "--c=C", + "-durian", + "D", + "--e", + "E", + "-fruit=any", + ]; + let matches = &match opts.parse(&args) { + Ok(m) => m, + Err(e) => panic!("{}", e), + }; + assert_eq!(matches.opts_str(&["a".to_string()]).unwrap(), "A"); + assert_eq!(matches.opts_str(&["b".to_string()]).unwrap(), "B"); + assert_eq!(matches.opts_str(&["c".to_string()]).unwrap(), "C"); + assert_eq!(matches.opts_str(&["durian".to_string()]).unwrap(), "D"); + assert_eq!(matches.opts_str(&["e".to_string()]).unwrap(), "E"); + assert_eq!(matches.opts_str(&["fruit".to_string()]).unwrap(), "any"); +} + +#[test] +fn test_long_only_mode_no_short_parse() { + let mut opts = Options::new(); + opts.long_only(true); + opts.optflag("h", "help", "Description"); + opts.optflag("i", "ignore", "Description"); + opts.optflag("", "hi", "Description"); + + let args = vec!["-hi"]; + let matches = &match opts.parse(&args) { + Ok(m) => m, + Err(e) => panic!("{}", e), + }; + assert!(matches.opt_present("hi")); + assert!(!matches.opt_present("h")); + assert!(!matches.opt_present("i")); +} + +#[test] +fn test_normal_mode_no_long_parse() { + // Like test_long_only_mode_no_short_parse, but we make sure + // that long_only can be disabled, and the right thing + // happens. + let mut opts = Options::new(); + opts.long_only(true); + opts.optflag("h", "help", "Description"); + opts.optflag("i", "ignore", "Description"); + opts.optflag("", "hi", "Description"); + opts.long_only(false); + + let args = vec!["-hi"]; + let matches = &match opts.parse(&args) { + Ok(m) => m, + Err(e) => panic!("{}", e), + }; + assert!(!matches.opt_present("hi")); + assert!(matches.opt_present("h")); + assert!(matches.opt_present("i")); +} + +#[test] +#[should_panic] +fn test_long_name_too_short() { + let mut opts = Options::new(); + opts.optflag("", "a", "Oops, long option too short"); +} + +#[test] +#[should_panic] +fn test_undefined_opt_present() { + let mut opts = Options::new(); + opts.optflag("h", "help", "Description"); + let args = vec!["-h"]; + match opts.parse(args) { + Ok(matches) => assert!(!matches.opt_present("undefined")), + Err(e) => panic!("{}", e), + } +} + +#[test] +fn test_opt_default() { + let mut opts = Options::new(); + opts.optflag("h", "help", "Description"); + opts.optflag("i", "ignore", "Description"); + opts.optflag("r", "run", "Description"); + opts.long_only(false); + + let args: Vec = ["-i", "-r", "10"].iter().map(|x| x.to_string()).collect(); + let matches = &match opts.parse(&args) { + Ok(m) => m, + Err(e) => panic!("{}", e), + }; + assert_eq!(matches.opt_default("help", ""), None); + assert_eq!(matches.opt_default("i", "def"), Some("def".to_string())); +} + +#[test] +fn test_opt_get() { + let mut opts = Options::new(); + opts.optflag("h", "help", "Description"); + opts.optflagopt("i", "ignore", "Description", "true | false"); + opts.optflagopt("r", "run", "Description", "0 .. 10"); + opts.optflagopt("p", "percent", "Description", "0.0 .. 10.0"); + opts.long_only(false); + + let args: Vec = ["-i", "true", "-p", "1.1"] + .iter() + .map(|x| x.to_string()) + .collect(); + let matches = &match opts.parse(&args) { + Ok(m) => m, + Err(e) => panic!("{}", e), + }; + let h_arg = matches.opt_get::("help"); + assert_eq!(h_arg, Ok(None)); + let i_arg = matches.opt_get("i"); + assert_eq!(i_arg, Ok(Some(true))); + let p_arg = matches.opt_get("p"); + assert_eq!(p_arg, Ok(Some(1.1))); +} + +#[test] +fn test_opt_get_default() { + let mut opts = Options::new(); + opts.optflag("h", "help", "Description"); + opts.optflagopt("i", "ignore", "Description", "true | false"); + opts.optflagopt("r", "run", "Description", "0 .. 10"); + opts.optflagopt("p", "percent", "Description", "0.0 .. 10.0"); + opts.long_only(false); + + let args: Vec = ["-i", "true", "-p", "1.1"] + .iter() + .map(|x| x.to_string()) + .collect(); + let matches = &match opts.parse(&args) { + Ok(m) => m, + Err(e) => panic!("{}", e), + }; + let h_arg = matches.opt_get_default("help", 10); + assert_eq!(h_arg, Ok(10)); + let i_arg = matches.opt_get_default("i", false); + assert_eq!(i_arg, Ok(true)); + let p_arg = matches.opt_get_default("p", 10.2); + assert_eq!(p_arg, Ok(1.1)); +} + +#[test] +fn test_opt_positions() { + let mut opts = Options::new(); + opts.optflagmulti("a", "act", "Description"); + opts.optflagmulti("e", "enact", "Description"); + opts.optflagmulti("r", "react", "Description"); + + let args: Vec = ["-a", "-a", "-r", "-a", "-r", "-r"] + .iter() + .map(|x| x.to_string()) + .collect(); + + let matches = &match opts.parse(&args) { + Ok(m) => m, + Err(e) => panic!("{}", e), + }; + + let a_pos = matches.opt_positions("a"); + assert_eq!(a_pos, vec![0, 1, 3]); + let e_pos = matches.opt_positions("e"); + assert_eq!(e_pos, vec![]); + let r_pos = matches.opt_positions("r"); + assert_eq!(r_pos, vec![2, 4, 5]); +} + +#[test] +fn test_opt_strs_pos() { + let mut opts = Options::new(); + opts.optmulti("a", "act", "Description", "NUM"); + opts.optmulti("e", "enact", "Description", "NUM"); + opts.optmulti("r", "react", "Description", "NUM"); + + let args: Vec = ["-a1", "-a2", "-r3", "-a4", "-r5", "-r6"] + .iter() + .map(|x| x.to_string()) + .collect(); + + let matches = &match opts.parse(&args) { + Ok(m) => m, + Err(e) => panic!("{}", e), + }; + + let a_pos = matches.opt_strs_pos("a"); + assert_eq!( + a_pos, + vec![ + (0, "1".to_string()), + (1, "2".to_string()), + (3, "4".to_string()) + ] + ); + let e_pos = matches.opt_strs_pos("e"); + assert_eq!(e_pos, vec![]); + let r_pos = matches.opt_strs_pos("r"); + assert_eq!( + r_pos, + vec![ + (2, "3".to_string()), + (4, "5".to_string()), + (5, "6".to_string()) + ] + ); +} diff --git a/crux-mir/lib/getopts/tests/smoke.rs b/crux-mir/lib/getopts/tests/smoke.rs new file mode 100644 index 000000000..a46f9c016 --- /dev/null +++ b/crux-mir/lib/getopts/tests/smoke.rs @@ -0,0 +1,8 @@ +extern crate getopts; + +use std::env; + +#[test] +fn main() { + getopts::Options::new().parse(env::args()).unwrap(); +} diff --git a/crux-mir/lib/test/Cargo.toml b/crux-mir/lib/test/Cargo.toml new file mode 100644 index 000000000..61b6f33bc --- /dev/null +++ b/crux-mir/lib/test/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "test" +version = "0.0.0" +edition = "2021" + +[lib] +crate-type = ["dylib", "rlib"] + +[dependencies] +cfg-if = { version = "1.0", features = ['rustc-dep-of-std'] } +getopts = { version = "0.2.21", features = ['rustc-dep-of-std'] } +std = { path = "../std" } +core = { path = "../core" } +libc = { version = "0.2", default-features = false } +panic_unwind = { path = "../panic_unwind" } +panic_abort = { path = "../panic_abort" } + +# not actually used but needed to always have proc_macro in the sysroot +proc_macro = { path = "../proc_macro" } + +# Forward features to the `std` crate as necessary +[features] +default = ["std_detect_file_io", "std_detect_dlsym_getauxval", "panic-unwind"] +backtrace = ["std/backtrace"] +compiler-builtins-c = ["std/compiler-builtins-c"] +compiler-builtins-mem = ["std/compiler-builtins-mem"] +compiler-builtins-no-asm = ["std/compiler-builtins-no-asm"] +compiler-builtins-mangled-names = ["std/compiler-builtins-mangled-names"] +llvm-libunwind = ["std/llvm-libunwind"] +system-llvm-libunwind = ["std/system-llvm-libunwind"] +panic-unwind = ["std/panic_unwind"] +panic_immediate_abort = ["std/panic_immediate_abort"] +profiler = ["std/profiler"] +std_detect_file_io = ["std/std_detect_file_io"] +std_detect_dlsym_getauxval = ["std/std_detect_dlsym_getauxval"] +std_detect_env_override = ["std/std_detect_env_override"] diff --git a/crux-mir/lib/test/src/bench.rs b/crux-mir/lib/test/src/bench.rs new file mode 100644 index 000000000..23925e6ea --- /dev/null +++ b/crux-mir/lib/test/src/bench.rs @@ -0,0 +1,243 @@ +//! Benchmarking module. +use super::{ + event::CompletedTest, + options::BenchMode, + test_result::TestResult, + types::{TestDesc, TestId}, + Sender, +}; + +use crate::stats; +use std::cmp; +use std::io; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +/// An identity function that *__hints__* to the compiler to be maximally pessimistic about what +/// `black_box` could do. +/// +/// See [`std::hint::black_box`] for details. +#[inline(always)] +pub fn black_box(dummy: T) -> T { + std::hint::black_box(dummy) +} + +/// Manager of the benchmarking runs. +/// +/// This is fed into functions marked with `#[bench]` to allow for +/// set-up & tear-down before running a piece of code repeatedly via a +/// call to `iter`. +#[derive(Clone)] +pub struct Bencher { + mode: BenchMode, + summary: Option, + pub bytes: u64, +} + +impl Bencher { + /// Callback for benchmark functions to run in their body. + pub fn iter(&mut self, mut inner: F) + where + F: FnMut() -> T, + { + if self.mode == BenchMode::Single { + ns_iter_inner(&mut inner, 1); + return; + } + + self.summary = Some(iter(&mut inner)); + } + + pub fn bench(&mut self, mut f: F) -> Result, String> + where + F: FnMut(&mut Bencher) -> Result<(), String>, + { + let result = f(self); + result.map(|_| self.summary) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct BenchSamples { + pub ns_iter_summ: stats::Summary, + pub mb_s: usize, +} + +pub fn fmt_bench_samples(bs: &BenchSamples) -> String { + use std::fmt::Write; + let mut output = String::new(); + + let median = bs.ns_iter_summ.median as usize; + let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize; + + write!( + output, + "{:>11} ns/iter (+/- {})", + fmt_thousands_sep(median, ','), + fmt_thousands_sep(deviation, ',') + ) + .unwrap(); + if bs.mb_s != 0 { + write!(output, " = {} MB/s", bs.mb_s).unwrap(); + } + output +} + +// Format a number with thousands separators +fn fmt_thousands_sep(mut n: usize, sep: char) -> String { + use std::fmt::Write; + let mut output = String::new(); + let mut trailing = false; + for &pow in &[9, 6, 3, 0] { + let base = 10_usize.pow(pow); + if pow == 0 || trailing || n / base != 0 { + if !trailing { + write!(output, "{}", n / base).unwrap(); + } else { + write!(output, "{:03}", n / base).unwrap(); + } + if pow != 0 { + output.push(sep); + } + trailing = true; + } + n %= base; + } + + output +} + +fn ns_iter_inner(inner: &mut F, k: u64) -> u64 +where + F: FnMut() -> T, +{ + let start = Instant::now(); + for _ in 0..k { + black_box(inner()); + } + start.elapsed().as_nanos() as u64 +} + +pub fn iter(inner: &mut F) -> stats::Summary +where + F: FnMut() -> T, +{ + // Initial bench run to get ballpark figure. + let ns_single = ns_iter_inner(inner, 1); + + // Try to estimate iter count for 1ms falling back to 1m + // iterations if first run took < 1ns. + let ns_target_total = 1_000_000; // 1ms + let mut n = ns_target_total / cmp::max(1, ns_single); + + // if the first run took more than 1ms we don't want to just + // be left doing 0 iterations on every loop. The unfortunate + // side effect of not being able to do as many runs is + // automatically handled by the statistical analysis below + // (i.e., larger error bars). + n = cmp::max(1, n); + + let mut total_run = Duration::new(0, 0); + let samples: &mut [f64] = &mut [0.0_f64; 50]; + loop { + let loop_start = Instant::now(); + + for p in &mut *samples { + *p = ns_iter_inner(inner, n) as f64 / n as f64; + } + + stats::winsorize(samples, 5.0); + let summ = stats::Summary::new(samples); + + for p in &mut *samples { + let ns = ns_iter_inner(inner, 5 * n); + *p = ns as f64 / (5 * n) as f64; + } + + stats::winsorize(samples, 5.0); + let summ5 = stats::Summary::new(samples); + + let loop_run = loop_start.elapsed(); + + // If we've run for 100ms and seem to have converged to a + // stable median. + if loop_run > Duration::from_millis(100) + && summ.median_abs_dev_pct < 1.0 + && summ.median - summ5.median < summ5.median_abs_dev + { + return summ5; + } + + total_run += loop_run; + // Longest we ever run for is 3s. + if total_run > Duration::from_secs(3) { + return summ5; + } + + // If we overflow here just return the results so far. We check a + // multiplier of 10 because we're about to multiply by 2 and the + // next iteration of the loop will also multiply by 5 (to calculate + // the summ5 result) + n = match n.checked_mul(10) { + Some(_) => n * 2, + None => { + return summ5; + } + }; + } +} + +pub fn benchmark( + id: TestId, + desc: TestDesc, + monitor_ch: Sender, + nocapture: bool, + f: F, +) where + F: FnMut(&mut Bencher) -> Result<(), String>, +{ + let mut bs = Bencher { mode: BenchMode::Auto, summary: None, bytes: 0 }; + + let data = Arc::new(Mutex::new(Vec::new())); + + if !nocapture { + io::set_output_capture(Some(data.clone())); + } + + let result = catch_unwind(AssertUnwindSafe(|| bs.bench(f))); + + io::set_output_capture(None); + + let test_result = match result { + //bs.bench(f) { + Ok(Ok(Some(ns_iter_summ))) => { + let ns_iter = cmp::max(ns_iter_summ.median as u64, 1); + let mb_s = bs.bytes * 1000 / ns_iter; + + let bs = BenchSamples { ns_iter_summ, mb_s: mb_s as usize }; + TestResult::TrBench(bs) + } + Ok(Ok(None)) => { + // iter not called, so no data. + // FIXME: error in this case? + let samples: &mut [f64] = &mut [0.0_f64; 1]; + let bs = BenchSamples { ns_iter_summ: stats::Summary::new(samples), mb_s: 0 }; + TestResult::TrBench(bs) + } + Err(_) => TestResult::TrFailed, + Ok(Err(_)) => TestResult::TrFailed, + }; + + let stdout = data.lock().unwrap().to_vec(); + let message = CompletedTest::new(id, desc, test_result, None, stdout); + monitor_ch.send(message).unwrap(); +} + +pub fn run_once(f: F) -> Result<(), String> +where + F: FnMut(&mut Bencher) -> Result<(), String>, +{ + let mut bs = Bencher { mode: BenchMode::Single, summary: None, bytes: 0 }; + bs.bench(f).map(|_| ()) +} diff --git a/crux-mir/lib/test/src/cli.rs b/crux-mir/lib/test/src/cli.rs new file mode 100644 index 000000000..796796e07 --- /dev/null +++ b/crux-mir/lib/test/src/cli.rs @@ -0,0 +1,494 @@ +//! Module converting command-line arguments into test configuration. + +use std::env; +use std::path::PathBuf; + +use super::options::{ColorConfig, Options, OutputFormat, RunIgnored}; +use super::time::TestTimeOptions; +use std::io::{self, IsTerminal}; + +#[derive(Debug)] +pub struct TestOpts { + pub list: bool, + pub filters: Vec, + pub filter_exact: bool, + pub force_run_in_process: bool, + pub exclude_should_panic: bool, + pub run_ignored: RunIgnored, + pub run_tests: bool, + pub bench_benchmarks: bool, + pub logfile: Option, + pub nocapture: bool, + pub color: ColorConfig, + pub format: OutputFormat, + pub shuffle: bool, + pub shuffle_seed: Option, + pub test_threads: Option, + pub skip: Vec, + pub time_options: Option, + /// Stop at first failing test. + /// May run a few more tests due to threading, but will + /// abort as soon as possible. + pub fail_fast: bool, + pub options: Options, +} + +impl TestOpts { + pub fn use_color(&self) -> bool { + match self.color { + ColorConfig::AutoColor => !self.nocapture && io::stdout().is_terminal(), + ColorConfig::AlwaysColor => true, + ColorConfig::NeverColor => false, + } + } +} + +/// Result of parsing the options. +pub type OptRes = Result; +/// Result of parsing the option part. +type OptPartRes = Result; + +fn optgroups() -> getopts::Options { + let mut opts = getopts::Options::new(); + opts.optflag("", "include-ignored", "Run ignored and not ignored tests") + .optflag("", "ignored", "Run only ignored tests") + .optflag("", "force-run-in-process", "Forces tests to run in-process when panic=abort") + .optflag("", "exclude-should-panic", "Excludes tests marked as should_panic") + .optflag("", "test", "Run tests and not benchmarks") + .optflag("", "bench", "Run benchmarks instead of tests") + .optflag("", "list", "List all tests and benchmarks") + .optflag("h", "help", "Display this message") + .optopt("", "logfile", "Write logs to the specified file", "PATH") + .optflag( + "", + "nocapture", + "don't capture stdout/stderr of each \ + task, allow printing directly", + ) + .optopt( + "", + "test-threads", + "Number of threads used for running tests \ + in parallel", + "n_threads", + ) + .optmulti( + "", + "skip", + "Skip tests whose names contain FILTER (this flag can \ + be used multiple times)", + "FILTER", + ) + .optflag( + "q", + "quiet", + "Display one character per test instead of one line. \ + Alias to --format=terse", + ) + .optflag("", "exact", "Exactly match filters rather than by substring") + .optopt( + "", + "color", + "Configure coloring of output: + auto = colorize if stdout is a tty and tests are run on serially (default); + always = always colorize output; + never = never colorize output;", + "auto|always|never", + ) + .optopt( + "", + "format", + "Configure formatting of output: + pretty = Print verbose output; + terse = Display one character per test; + json = Output a json document; + junit = Output a JUnit document", + "pretty|terse|json|junit", + ) + .optflag("", "show-output", "Show captured stdout of successful tests") + .optopt( + "Z", + "", + "Enable nightly-only flags: + unstable-options = Allow use of experimental features", + "unstable-options", + ) + .optflag( + "", + "report-time", + "Show execution time of each test. + + Threshold values for colorized output can be configured via + `RUST_TEST_TIME_UNIT`, `RUST_TEST_TIME_INTEGRATION` and + `RUST_TEST_TIME_DOCTEST` environment variables. + + Expected format of environment variable is `VARIABLE=WARN_TIME,CRITICAL_TIME`. + Durations must be specified in milliseconds, e.g. `500,2000` means that the warn time + is 0.5 seconds, and the critical time is 2 seconds. + + Not available for --format=terse", + ) + .optflag( + "", + "ensure-time", + "Treat excess of the test execution time limit as error. + + Threshold values for this option can be configured via + `RUST_TEST_TIME_UNIT`, `RUST_TEST_TIME_INTEGRATION` and + `RUST_TEST_TIME_DOCTEST` environment variables. + + Expected format of environment variable is `VARIABLE=WARN_TIME,CRITICAL_TIME`. + + `CRITICAL_TIME` here means the limit that should not be exceeded by test. + ", + ) + .optflag("", "shuffle", "Run tests in random order") + .optopt( + "", + "shuffle-seed", + "Run tests in random order; seed the random number generator with SEED", + "SEED", + ); + opts +} + +fn usage(binary: &str, options: &getopts::Options) { + let message = format!("Usage: {binary} [OPTIONS] [FILTERS...]"); + println!( + r#"{usage} + +The FILTER string is tested against the name of all tests, and only those +tests whose names contain the filter are run. Multiple filter strings may +be passed, which will run all tests matching any of the filters. + +By default, all tests are run in parallel. This can be altered with the +--test-threads flag or the RUST_TEST_THREADS environment variable when running +tests (set it to 1). + +By default, the tests are run in alphabetical order. Use --shuffle or set +RUST_TEST_SHUFFLE to run the tests in random order. Pass the generated +"shuffle seed" to --shuffle-seed (or set RUST_TEST_SHUFFLE_SEED) to run the +tests in the same order again. Note that --shuffle and --shuffle-seed do not +affect whether the tests are run in parallel. + +All tests have their standard output and standard error captured by default. +This can be overridden with the --nocapture flag or setting RUST_TEST_NOCAPTURE +environment variable to a value other than "0". Logging is not captured by default. + +Test Attributes: + + `#[test]` - Indicates a function is a test to be run. This function + takes no arguments. + `#[bench]` - Indicates a function is a benchmark to be run. This + function takes one argument (test::Bencher). + `#[should_panic]` - This function (also labeled with `#[test]`) will only pass if + the code causes a panic (an assertion failure or panic!) + A message may be provided, which the failure string must + contain: #[should_panic(expected = "foo")]. + `#[ignore]` - When applied to a function which is already attributed as a + test, then the test runner will ignore these tests during + normal test runs. Running with --ignored or --include-ignored will run + these tests."#, + usage = options.usage(&message) + ); +} + +/// Parses command line arguments into test options. +/// Returns `None` if help was requested (since we only show help message and don't run tests), +/// returns `Some(Err(..))` if provided arguments are incorrect, +/// otherwise creates a `TestOpts` object and returns it. +pub fn parse_opts(args: &[String]) -> Option { + // Parse matches. + let opts = optgroups(); + let binary = args.get(0).map(|c| &**c).unwrap_or("..."); + let args = args.get(1..).unwrap_or(args); + let matches = match opts.parse(args) { + Ok(m) => m, + Err(f) => return Some(Err(f.to_string())), + }; + + // Check if help was requested. + if matches.opt_present("h") { + // Show help and do nothing more. + usage(binary, &opts); + return None; + } + + // Actually parse the opts. + let opts_result = parse_opts_impl(matches); + + Some(opts_result) +} + +// Gets the option value and checks if unstable features are enabled. +macro_rules! unstable_optflag { + ($matches:ident, $allow_unstable:ident, $option_name:literal) => {{ + let opt = $matches.opt_present($option_name); + if !$allow_unstable && opt { + return Err(format!( + "The \"{}\" flag is only accepted on the nightly compiler with -Z unstable-options", + $option_name + )); + } + + opt + }}; +} + +// Gets the option value and checks if unstable features are enabled. +macro_rules! unstable_optopt { + ($matches:ident, $allow_unstable:ident, $option_name:literal) => {{ + let opt = $matches.opt_str($option_name); + if !$allow_unstable && opt.is_some() { + return Err(format!( + "The \"{}\" option is only accepted on the nightly compiler with -Z unstable-options", + $option_name + )); + } + + opt + }}; +} + +// Implementation of `parse_opts` that doesn't care about help message +// and returns a `Result`. +fn parse_opts_impl(matches: getopts::Matches) -> OptRes { + let allow_unstable = get_allow_unstable(&matches)?; + + // Unstable flags + let force_run_in_process = unstable_optflag!(matches, allow_unstable, "force-run-in-process"); + let exclude_should_panic = unstable_optflag!(matches, allow_unstable, "exclude-should-panic"); + let time_options = get_time_options(&matches, allow_unstable)?; + let shuffle = get_shuffle(&matches, allow_unstable)?; + let shuffle_seed = get_shuffle_seed(&matches, allow_unstable)?; + + let include_ignored = matches.opt_present("include-ignored"); + let quiet = matches.opt_present("quiet"); + let exact = matches.opt_present("exact"); + let list = matches.opt_present("list"); + let skip = matches.opt_strs("skip"); + + let bench_benchmarks = matches.opt_present("bench"); + let run_tests = !bench_benchmarks || matches.opt_present("test"); + + let logfile = get_log_file(&matches)?; + let run_ignored = get_run_ignored(&matches, include_ignored)?; + let filters = matches.free.clone(); + let nocapture = get_nocapture(&matches)?; + let test_threads = get_test_threads(&matches)?; + let color = get_color_config(&matches)?; + let format = get_format(&matches, quiet, allow_unstable)?; + + let options = Options::new().display_output(matches.opt_present("show-output")); + + let test_opts = TestOpts { + list, + filters, + filter_exact: exact, + force_run_in_process, + exclude_should_panic, + run_ignored, + run_tests, + bench_benchmarks, + logfile, + nocapture, + color, + format, + shuffle, + shuffle_seed, + test_threads, + skip, + time_options, + options, + fail_fast: false, + }; + + Ok(test_opts) +} + +// FIXME: Copied from librustc_ast until linkage errors are resolved. Issue #47566 +fn is_nightly() -> bool { + // Whether this is a feature-staged build, i.e., on the beta or stable channel + let disable_unstable_features = option_env!("CFG_DISABLE_UNSTABLE_FEATURES").is_some(); + // Whether we should enable unstable features for bootstrapping + let bootstrap = env::var("RUSTC_BOOTSTRAP").is_ok(); + + bootstrap || !disable_unstable_features +} + +// Gets the CLI options associated with `report-time` feature. +fn get_time_options( + matches: &getopts::Matches, + allow_unstable: bool, +) -> OptPartRes> { + let report_time = unstable_optflag!(matches, allow_unstable, "report-time"); + let ensure_test_time = unstable_optflag!(matches, allow_unstable, "ensure-time"); + + // If `ensure-test-time` option is provided, time output is enforced, + // so user won't be confused if any of tests will silently fail. + let options = if report_time || ensure_test_time { + Some(TestTimeOptions::new_from_env(ensure_test_time)) + } else { + None + }; + + Ok(options) +} + +fn get_shuffle(matches: &getopts::Matches, allow_unstable: bool) -> OptPartRes { + let mut shuffle = unstable_optflag!(matches, allow_unstable, "shuffle"); + if !shuffle && allow_unstable { + shuffle = match env::var("RUST_TEST_SHUFFLE") { + Ok(val) => &val != "0", + Err(_) => false, + }; + } + + Ok(shuffle) +} + +fn get_shuffle_seed(matches: &getopts::Matches, allow_unstable: bool) -> OptPartRes> { + let mut shuffle_seed = match unstable_optopt!(matches, allow_unstable, "shuffle-seed") { + Some(n_str) => match n_str.parse::() { + Ok(n) => Some(n), + Err(e) => { + return Err(format!( + "argument for --shuffle-seed must be a number \ + (error: {e})" + )); + } + }, + None => None, + }; + + if shuffle_seed.is_none() && allow_unstable { + shuffle_seed = match env::var("RUST_TEST_SHUFFLE_SEED") { + Ok(val) => match val.parse::() { + Ok(n) => Some(n), + Err(_) => panic!("RUST_TEST_SHUFFLE_SEED is `{val}`, should be a number."), + }, + Err(_) => None, + }; + } + + Ok(shuffle_seed) +} + +fn get_test_threads(matches: &getopts::Matches) -> OptPartRes> { + let test_threads = match matches.opt_str("test-threads") { + Some(n_str) => match n_str.parse::() { + Ok(0) => return Err("argument for --test-threads must not be 0".to_string()), + Ok(n) => Some(n), + Err(e) => { + return Err(format!( + "argument for --test-threads must be a number > 0 \ + (error: {e})" + )); + } + }, + None => None, + }; + + Ok(test_threads) +} + +fn get_format( + matches: &getopts::Matches, + quiet: bool, + allow_unstable: bool, +) -> OptPartRes { + let format = match matches.opt_str("format").as_deref() { + None if quiet => OutputFormat::Terse, + Some("pretty") | None => OutputFormat::Pretty, + Some("terse") => OutputFormat::Terse, + Some("json") => { + if !allow_unstable { + return Err("The \"json\" format is only accepted on the nightly compiler".into()); + } + OutputFormat::Json + } + Some("junit") => { + if !allow_unstable { + return Err("The \"junit\" format is only accepted on the nightly compiler".into()); + } + OutputFormat::Junit + } + Some(v) => { + return Err(format!( + "argument for --format must be pretty, terse, json or junit (was \ + {v})" + )); + } + }; + + Ok(format) +} + +fn get_color_config(matches: &getopts::Matches) -> OptPartRes { + let color = match matches.opt_str("color").as_deref() { + Some("auto") | None => ColorConfig::AutoColor, + Some("always") => ColorConfig::AlwaysColor, + Some("never") => ColorConfig::NeverColor, + + Some(v) => { + return Err(format!( + "argument for --color must be auto, always, or never (was \ + {v})" + )); + } + }; + + Ok(color) +} + +fn get_nocapture(matches: &getopts::Matches) -> OptPartRes { + let mut nocapture = matches.opt_present("nocapture"); + if !nocapture { + nocapture = match env::var("RUST_TEST_NOCAPTURE") { + Ok(val) => &val != "0", + Err(_) => false, + }; + } + + Ok(nocapture) +} + +fn get_run_ignored(matches: &getopts::Matches, include_ignored: bool) -> OptPartRes { + let run_ignored = match (include_ignored, matches.opt_present("ignored")) { + (true, true) => { + return Err("the options --include-ignored and --ignored are mutually exclusive".into()); + } + (true, false) => RunIgnored::Yes, + (false, true) => RunIgnored::Only, + (false, false) => RunIgnored::No, + }; + + Ok(run_ignored) +} + +fn get_allow_unstable(matches: &getopts::Matches) -> OptPartRes { + let mut allow_unstable = false; + + if let Some(opt) = matches.opt_str("Z") { + if !is_nightly() { + return Err("the option `Z` is only accepted on the nightly compiler".into()); + } + + match &*opt { + "unstable-options" => { + allow_unstable = true; + } + _ => { + return Err("Unrecognized option to `Z`".into()); + } + } + }; + + Ok(allow_unstable) +} + +fn get_log_file(matches: &getopts::Matches) -> OptPartRes> { + let logfile = matches.opt_str("logfile").map(|s| PathBuf::from(&s)); + + Ok(logfile) +} diff --git a/crux-mir/lib/test/src/console.rs b/crux-mir/lib/test/src/console.rs new file mode 100644 index 000000000..24cbe035f --- /dev/null +++ b/crux-mir/lib/test/src/console.rs @@ -0,0 +1,307 @@ +//! Module providing interface for running tests in the console. + +use std::fs::File; +use std::io; +use std::io::prelude::Write; +use std::time::Instant; + +use super::{ + bench::fmt_bench_samples, + cli::TestOpts, + event::{CompletedTest, TestEvent}, + filter_tests, + formatters::{JsonFormatter, JunitFormatter, OutputFormatter, PrettyFormatter, TerseFormatter}, + helpers::{concurrency::get_concurrency, metrics::MetricMap}, + options::{Options, OutputFormat}, + run_tests, term, + test_result::TestResult, + time::{TestExecTime, TestSuiteExecTime}, + types::{NamePadding, TestDesc, TestDescAndFn}, +}; + +/// Generic wrapper over stdout. +pub enum OutputLocation { + Pretty(Box), + Raw(T), +} + +impl Write for OutputLocation { + fn write(&mut self, buf: &[u8]) -> io::Result { + match *self { + OutputLocation::Pretty(ref mut term) => term.write(buf), + OutputLocation::Raw(ref mut stdout) => stdout.write(buf), + } + } + + fn flush(&mut self) -> io::Result<()> { + match *self { + OutputLocation::Pretty(ref mut term) => term.flush(), + OutputLocation::Raw(ref mut stdout) => stdout.flush(), + } + } +} + +pub struct ConsoleTestState { + pub log_out: Option, + pub total: usize, + pub passed: usize, + pub failed: usize, + pub ignored: usize, + pub filtered_out: usize, + pub measured: usize, + pub exec_time: Option, + pub metrics: MetricMap, + pub failures: Vec<(TestDesc, Vec)>, + pub not_failures: Vec<(TestDesc, Vec)>, + pub time_failures: Vec<(TestDesc, Vec)>, + pub options: Options, +} + +impl ConsoleTestState { + pub fn new(opts: &TestOpts) -> io::Result { + let log_out = match opts.logfile { + Some(ref path) => Some(File::create(path)?), + None => None, + }; + + Ok(ConsoleTestState { + log_out, + total: 0, + passed: 0, + failed: 0, + ignored: 0, + filtered_out: 0, + measured: 0, + exec_time: None, + metrics: MetricMap::new(), + failures: Vec::new(), + not_failures: Vec::new(), + time_failures: Vec::new(), + options: opts.options, + }) + } + + pub fn write_log(&mut self, msg: F) -> io::Result<()> + where + S: AsRef, + F: FnOnce() -> S, + { + match self.log_out { + None => Ok(()), + Some(ref mut o) => { + let msg = msg(); + let msg = msg.as_ref(); + o.write_all(msg.as_bytes()) + } + } + } + + pub fn write_log_result( + &mut self, + test: &TestDesc, + result: &TestResult, + exec_time: Option<&TestExecTime>, + ) -> io::Result<()> { + self.write_log(|| { + let TestDesc { name, ignore_message, .. } = test; + format!( + "{} {}", + match *result { + TestResult::TrOk => "ok".to_owned(), + TestResult::TrFailed => "failed".to_owned(), + TestResult::TrFailedMsg(ref msg) => format!("failed: {msg}"), + TestResult::TrIgnored => { + if let Some(msg) = ignore_message { + format!("ignored: {msg}") + } else { + "ignored".to_owned() + } + } + TestResult::TrBench(ref bs) => fmt_bench_samples(bs), + TestResult::TrTimedFail => "failed (time limit exceeded)".to_owned(), + }, + name, + ) + })?; + if let Some(exec_time) = exec_time { + self.write_log(|| format!(" <{exec_time}>"))?; + } + self.write_log(|| "\n") + } + + fn current_test_count(&self) -> usize { + self.passed + self.failed + self.ignored + self.measured + } +} + +// List the tests to console, and optionally to logfile. Filters are honored. +pub fn list_tests_console(opts: &TestOpts, tests: Vec) -> io::Result<()> { + let mut output = match term::stdout() { + None => OutputLocation::Raw(io::stdout().lock()), + Some(t) => OutputLocation::Pretty(t), + }; + + let quiet = opts.format == OutputFormat::Terse; + let mut st = ConsoleTestState::new(opts)?; + + let mut ntest = 0; + let mut nbench = 0; + + for test in filter_tests(opts, tests).into_iter() { + use crate::TestFn::*; + + let TestDescAndFn { desc: TestDesc { name, .. }, testfn } = test; + + let fntype = match testfn { + StaticTestFn(..) | DynTestFn(..) => { + ntest += 1; + "test" + } + StaticBenchFn(..) | DynBenchFn(..) => { + nbench += 1; + "benchmark" + } + }; + + writeln!(output, "{name}: {fntype}")?; + st.write_log(|| format!("{fntype} {name}\n"))?; + } + + fn plural(count: u32, s: &str) -> String { + match count { + 1 => format!("1 {s}"), + n => format!("{n} {s}s"), + } + } + + if !quiet { + if ntest != 0 || nbench != 0 { + writeln!(output)?; + } + + writeln!(output, "{}, {}", plural(ntest, "test"), plural(nbench, "benchmark"))?; + } + + Ok(()) +} + +// Updates `ConsoleTestState` depending on result of the test execution. +fn handle_test_result(st: &mut ConsoleTestState, completed_test: CompletedTest) { + let test = completed_test.desc; + let stdout = completed_test.stdout; + match completed_test.result { + TestResult::TrOk => { + st.passed += 1; + st.not_failures.push((test, stdout)); + } + TestResult::TrIgnored => st.ignored += 1, + TestResult::TrBench(bs) => { + st.metrics.insert_metric( + test.name.as_slice(), + bs.ns_iter_summ.median, + bs.ns_iter_summ.max - bs.ns_iter_summ.min, + ); + st.measured += 1 + } + TestResult::TrFailed => { + st.failed += 1; + st.failures.push((test, stdout)); + } + TestResult::TrFailedMsg(msg) => { + st.failed += 1; + let mut stdout = stdout; + stdout.extend_from_slice(format!("note: {msg}").as_bytes()); + st.failures.push((test, stdout)); + } + TestResult::TrTimedFail => { + st.failed += 1; + st.time_failures.push((test, stdout)); + } + } +} + +// Handler for events that occur during test execution. +// It is provided as a callback to the `run_tests` function. +fn on_test_event( + event: &TestEvent, + st: &mut ConsoleTestState, + out: &mut dyn OutputFormatter, +) -> io::Result<()> { + match (*event).clone() { + TestEvent::TeFiltered(filtered_tests, shuffle_seed) => { + st.total = filtered_tests; + out.write_run_start(filtered_tests, shuffle_seed)?; + } + TestEvent::TeFilteredOut(filtered_out) => { + st.filtered_out = filtered_out; + } + TestEvent::TeWait(ref test) => out.write_test_start(test)?, + TestEvent::TeTimeout(ref test) => out.write_timeout(test)?, + TestEvent::TeResult(completed_test) => { + let test = &completed_test.desc; + let result = &completed_test.result; + let exec_time = &completed_test.exec_time; + let stdout = &completed_test.stdout; + + st.write_log_result(test, result, exec_time.as_ref())?; + out.write_result(test, result, exec_time.as_ref(), stdout, st)?; + handle_test_result(st, completed_test); + } + } + + Ok(()) +} + +/// A simple console test runner. +/// Runs provided tests reporting process and results to the stdout. +pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Result { + let output = match term::stdout() { + None => OutputLocation::Raw(io::stdout()), + Some(t) => OutputLocation::Pretty(t), + }; + + let max_name_len = tests + .iter() + .max_by_key(|t| len_if_padded(t)) + .map(|t| t.desc.name.as_slice().len()) + .unwrap_or(0); + + let is_multithreaded = opts.test_threads.unwrap_or_else(get_concurrency) > 1; + + let mut out: Box = match opts.format { + OutputFormat::Pretty => Box::new(PrettyFormatter::new( + output, + opts.use_color(), + max_name_len, + is_multithreaded, + opts.time_options, + )), + OutputFormat::Terse => { + Box::new(TerseFormatter::new(output, opts.use_color(), max_name_len, is_multithreaded)) + } + OutputFormat::Json => Box::new(JsonFormatter::new(output)), + OutputFormat::Junit => Box::new(JunitFormatter::new(output)), + }; + let mut st = ConsoleTestState::new(opts)?; + + // Prevent the usage of `Instant` in some cases: + // - It's currently not supported for wasm targets. + // - We disable it for miri because it's not available when isolation is enabled. + let is_instant_supported = !cfg!(target_family = "wasm") && !cfg!(miri); + + let start_time = is_instant_supported.then(Instant::now); + run_tests(opts, tests, |x| on_test_event(&x, &mut st, &mut *out))?; + st.exec_time = start_time.map(|t| TestSuiteExecTime(t.elapsed())); + + assert!(opts.fail_fast || st.current_test_count() == st.total); + + out.write_run_finish(&st) +} + +// Calculates padding for given test description. +fn len_if_padded(t: &TestDescAndFn) -> usize { + match t.testfn.padding() { + NamePadding::PadNone => 0, + NamePadding::PadOnRight => t.desc.name.as_slice().len(), + } +} diff --git a/crux-mir/lib/test/src/event.rs b/crux-mir/lib/test/src/event.rs new file mode 100644 index 000000000..80281ebd2 --- /dev/null +++ b/crux-mir/lib/test/src/event.rs @@ -0,0 +1,36 @@ +//! Module containing different events that can occur +//! during tests execution process. + +use super::test_result::TestResult; +use super::time::TestExecTime; +use super::types::{TestDesc, TestId}; + +#[derive(Debug, Clone)] +pub struct CompletedTest { + pub id: TestId, + pub desc: TestDesc, + pub result: TestResult, + pub exec_time: Option, + pub stdout: Vec, +} + +impl CompletedTest { + pub fn new( + id: TestId, + desc: TestDesc, + result: TestResult, + exec_time: Option, + stdout: Vec, + ) -> Self { + Self { id, desc, result, exec_time, stdout } + } +} + +#[derive(Debug, Clone)] +pub enum TestEvent { + TeFiltered(usize, Option), + TeWait(TestDesc), + TeResult(CompletedTest), + TeTimeout(TestDesc), + TeFilteredOut(usize), +} diff --git a/crux-mir/lib/test/src/formatters/json.rs b/crux-mir/lib/test/src/formatters/json.rs new file mode 100644 index 000000000..95d2faf25 --- /dev/null +++ b/crux-mir/lib/test/src/formatters/json.rs @@ -0,0 +1,259 @@ +use std::{borrow::Cow, io, io::prelude::Write}; + +use super::OutputFormatter; +use crate::{ + console::{ConsoleTestState, OutputLocation}, + test_result::TestResult, + time, + types::TestDesc, +}; + +pub(crate) struct JsonFormatter { + out: OutputLocation, +} + +impl JsonFormatter { + pub fn new(out: OutputLocation) -> Self { + Self { out } + } + + fn writeln_message(&mut self, s: &str) -> io::Result<()> { + assert!(!s.contains('\n')); + + self.out.write_all(s.as_ref())?; + self.out.write_all(b"\n") + } + + fn write_message(&mut self, s: &str) -> io::Result<()> { + assert!(!s.contains('\n')); + + self.out.write_all(s.as_ref()) + } + + fn write_event( + &mut self, + ty: &str, + name: &str, + evt: &str, + exec_time: Option<&time::TestExecTime>, + stdout: Option>, + extra: Option<&str>, + ) -> io::Result<()> { + // A doc test's name includes a filename which must be escaped for correct json. + self.write_message(&format!( + r#"{{ "type": "{}", "name": "{}", "event": "{}""#, + ty, + EscapedString(name), + evt + ))?; + if let Some(exec_time) = exec_time { + self.write_message(&format!(r#", "exec_time": {}"#, exec_time.0.as_secs_f64()))?; + } + if let Some(stdout) = stdout { + self.write_message(&format!(r#", "stdout": "{}""#, EscapedString(stdout)))?; + } + if let Some(extra) = extra { + self.write_message(&format!(r#", {extra}"#))?; + } + self.writeln_message(" }") + } +} + +impl OutputFormatter for JsonFormatter { + fn write_run_start(&mut self, test_count: usize, shuffle_seed: Option) -> io::Result<()> { + let shuffle_seed_json = if let Some(shuffle_seed) = shuffle_seed { + format!(r#", "shuffle_seed": {shuffle_seed}"#) + } else { + String::new() + }; + self.writeln_message(&format!( + r#"{{ "type": "suite", "event": "started", "test_count": {test_count}{shuffle_seed_json} }}"# + )) + } + + fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> { + self.writeln_message(&format!( + r#"{{ "type": "test", "event": "started", "name": "{}" }}"#, + EscapedString(desc.name.as_slice()) + )) + } + + fn write_result( + &mut self, + desc: &TestDesc, + result: &TestResult, + exec_time: Option<&time::TestExecTime>, + stdout: &[u8], + state: &ConsoleTestState, + ) -> io::Result<()> { + let display_stdout = state.options.display_output || *result != TestResult::TrOk; + let stdout = if display_stdout && !stdout.is_empty() { + Some(String::from_utf8_lossy(stdout)) + } else { + None + }; + match *result { + TestResult::TrOk => { + self.write_event("test", desc.name.as_slice(), "ok", exec_time, stdout, None) + } + + TestResult::TrFailed => { + self.write_event("test", desc.name.as_slice(), "failed", exec_time, stdout, None) + } + + TestResult::TrTimedFail => self.write_event( + "test", + desc.name.as_slice(), + "failed", + exec_time, + stdout, + Some(r#""reason": "time limit exceeded""#), + ), + + TestResult::TrFailedMsg(ref m) => self.write_event( + "test", + desc.name.as_slice(), + "failed", + exec_time, + stdout, + Some(&*format!(r#""message": "{}""#, EscapedString(m))), + ), + + TestResult::TrIgnored => self.write_event( + "test", + desc.name.as_slice(), + "ignored", + exec_time, + stdout, + desc.ignore_message + .map(|msg| format!(r#""message": "{}""#, EscapedString(msg))) + .as_deref(), + ), + + TestResult::TrBench(ref bs) => { + let median = bs.ns_iter_summ.median as usize; + let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize; + + let mbps = if bs.mb_s == 0 { + String::new() + } else { + format!(r#", "mib_per_second": {}"#, bs.mb_s) + }; + + let line = format!( + "{{ \"type\": \"bench\", \ + \"name\": \"{}\", \ + \"median\": {}, \ + \"deviation\": {}{} }}", + EscapedString(desc.name.as_slice()), + median, + deviation, + mbps + ); + + self.writeln_message(&line) + } + } + } + + fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { + self.writeln_message(&format!( + r#"{{ "type": "test", "event": "timeout", "name": "{}" }}"#, + EscapedString(desc.name.as_slice()) + )) + } + + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { + self.write_message(&format!( + "{{ \"type\": \"suite\", \ + \"event\": \"{}\", \ + \"passed\": {}, \ + \"failed\": {}, \ + \"ignored\": {}, \ + \"measured\": {}, \ + \"filtered_out\": {}", + if state.failed == 0 { "ok" } else { "failed" }, + state.passed, + state.failed, + state.ignored, + state.measured, + state.filtered_out, + ))?; + + if let Some(ref exec_time) = state.exec_time { + let time_str = format!(", \"exec_time\": {}", exec_time.0.as_secs_f64()); + self.write_message(&time_str)?; + } + + self.writeln_message(" }")?; + + Ok(state.failed == 0) + } +} + +/// A formatting utility used to print strings with characters in need of escaping. +/// Base code taken form `libserialize::json::escape_str` +struct EscapedString>(S); + +impl> std::fmt::Display for EscapedString { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> ::std::fmt::Result { + let mut start = 0; + + for (i, byte) in self.0.as_ref().bytes().enumerate() { + let escaped = match byte { + b'"' => "\\\"", + b'\\' => "\\\\", + b'\x00' => "\\u0000", + b'\x01' => "\\u0001", + b'\x02' => "\\u0002", + b'\x03' => "\\u0003", + b'\x04' => "\\u0004", + b'\x05' => "\\u0005", + b'\x06' => "\\u0006", + b'\x07' => "\\u0007", + b'\x08' => "\\b", + b'\t' => "\\t", + b'\n' => "\\n", + b'\x0b' => "\\u000b", + b'\x0c' => "\\f", + b'\r' => "\\r", + b'\x0e' => "\\u000e", + b'\x0f' => "\\u000f", + b'\x10' => "\\u0010", + b'\x11' => "\\u0011", + b'\x12' => "\\u0012", + b'\x13' => "\\u0013", + b'\x14' => "\\u0014", + b'\x15' => "\\u0015", + b'\x16' => "\\u0016", + b'\x17' => "\\u0017", + b'\x18' => "\\u0018", + b'\x19' => "\\u0019", + b'\x1a' => "\\u001a", + b'\x1b' => "\\u001b", + b'\x1c' => "\\u001c", + b'\x1d' => "\\u001d", + b'\x1e' => "\\u001e", + b'\x1f' => "\\u001f", + b'\x7f' => "\\u007f", + _ => { + continue; + } + }; + + if start < i { + f.write_str(&self.0.as_ref()[start..i])?; + } + + f.write_str(escaped)?; + + start = i + 1; + } + + if start != self.0.as_ref().len() { + f.write_str(&self.0.as_ref()[start..])?; + } + + Ok(()) + } +} diff --git a/crux-mir/lib/test/src/formatters/junit.rs b/crux-mir/lib/test/src/formatters/junit.rs new file mode 100644 index 000000000..7a40ce33c --- /dev/null +++ b/crux-mir/lib/test/src/formatters/junit.rs @@ -0,0 +1,180 @@ +use std::io::{self, prelude::Write}; +use std::time::Duration; + +use super::OutputFormatter; +use crate::{ + console::{ConsoleTestState, OutputLocation}, + test_result::TestResult, + time, + types::{TestDesc, TestType}, +}; + +pub struct JunitFormatter { + out: OutputLocation, + results: Vec<(TestDesc, TestResult, Duration)>, +} + +impl JunitFormatter { + pub fn new(out: OutputLocation) -> Self { + Self { out, results: Vec::new() } + } + + fn write_message(&mut self, s: &str) -> io::Result<()> { + assert!(!s.contains('\n')); + + self.out.write_all(s.as_ref()) + } +} + +impl OutputFormatter for JunitFormatter { + fn write_run_start( + &mut self, + _test_count: usize, + _shuffle_seed: Option, + ) -> io::Result<()> { + // We write xml header on run start + self.write_message("") + } + + fn write_test_start(&mut self, _desc: &TestDesc) -> io::Result<()> { + // We do not output anything on test start. + Ok(()) + } + + fn write_timeout(&mut self, _desc: &TestDesc) -> io::Result<()> { + // We do not output anything on test timeout. + Ok(()) + } + + fn write_result( + &mut self, + desc: &TestDesc, + result: &TestResult, + exec_time: Option<&time::TestExecTime>, + _stdout: &[u8], + _state: &ConsoleTestState, + ) -> io::Result<()> { + // Because the testsuite node holds some of the information as attributes, we can't write it + // until all of the tests have finished. Instead of writing every result as they come in, we add + // them to a Vec and write them all at once when run is complete. + let duration = exec_time.map(|t| t.0).unwrap_or_default(); + self.results.push((desc.clone(), result.clone(), duration)); + Ok(()) + } + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { + self.write_message("")?; + + self.write_message(&format!( + "", + state.failed, state.total, state.ignored + ))?; + for (desc, result, duration) in std::mem::take(&mut self.results) { + let (class_name, test_name) = parse_class_name(&desc); + match result { + TestResult::TrIgnored => { /* no-op */ } + TestResult::TrFailed => { + self.write_message(&format!( + "", + class_name, + test_name, + duration.as_secs_f64() + ))?; + self.write_message("")?; + self.write_message("")?; + } + + TestResult::TrFailedMsg(ref m) => { + self.write_message(&format!( + "", + class_name, + test_name, + duration.as_secs_f64() + ))?; + self.write_message(&format!(""))?; + self.write_message("")?; + } + + TestResult::TrTimedFail => { + self.write_message(&format!( + "", + class_name, + test_name, + duration.as_secs_f64() + ))?; + self.write_message("")?; + self.write_message("")?; + } + + TestResult::TrBench(ref b) => { + self.write_message(&format!( + "", + class_name, test_name, b.ns_iter_summ.sum + ))?; + } + + TestResult::TrOk => { + self.write_message(&format!( + "", + class_name, + test_name, + duration.as_secs_f64() + ))?; + } + } + } + self.write_message("")?; + self.write_message("")?; + self.write_message("")?; + self.write_message("")?; + + self.out.write_all(b"\n")?; + + Ok(state.failed == 0) + } +} + +fn parse_class_name(desc: &TestDesc) -> (String, String) { + match desc.test_type { + TestType::UnitTest => parse_class_name_unit(desc), + TestType::DocTest => parse_class_name_doc(desc), + TestType::IntegrationTest => parse_class_name_integration(desc), + TestType::Unknown => (String::from("unknown"), String::from(desc.name.as_slice())), + } +} + +fn parse_class_name_unit(desc: &TestDesc) -> (String, String) { + // Module path => classname + // Function name => name + let module_segments: Vec<&str> = desc.name.as_slice().split("::").collect(); + let (class_name, test_name) = match module_segments[..] { + [test] => (String::from("crate"), String::from(test)), + [ref path @ .., test] => (path.join("::"), String::from(test)), + [..] => unreachable!(), + }; + (class_name, test_name) +} + +fn parse_class_name_doc(desc: &TestDesc) -> (String, String) { + // File path => classname + // Line # => test name + let segments: Vec<&str> = desc.name.as_slice().split(" - ").collect(); + let (class_name, test_name) = match segments[..] { + [file, line] => (String::from(file.trim()), String::from(line.trim())), + [..] => unreachable!(), + }; + (class_name, test_name) +} + +fn parse_class_name_integration(desc: &TestDesc) -> (String, String) { + (String::from("integration"), String::from(desc.name.as_slice())) +} diff --git a/crux-mir/lib/test/src/formatters/mod.rs b/crux-mir/lib/test/src/formatters/mod.rs new file mode 100644 index 000000000..cb67b6491 --- /dev/null +++ b/crux-mir/lib/test/src/formatters/mod.rs @@ -0,0 +1,42 @@ +use std::{io, io::prelude::Write}; + +use crate::{ + console::ConsoleTestState, + test_result::TestResult, + time, + types::{TestDesc, TestName}, +}; + +mod json; +mod junit; +mod pretty; +mod terse; + +pub(crate) use self::json::JsonFormatter; +pub(crate) use self::junit::JunitFormatter; +pub(crate) use self::pretty::PrettyFormatter; +pub(crate) use self::terse::TerseFormatter; + +pub(crate) trait OutputFormatter { + fn write_run_start(&mut self, test_count: usize, shuffle_seed: Option) -> io::Result<()>; + fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()>; + fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()>; + fn write_result( + &mut self, + desc: &TestDesc, + result: &TestResult, + exec_time: Option<&time::TestExecTime>, + stdout: &[u8], + state: &ConsoleTestState, + ) -> io::Result<()>; + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result; +} + +pub(crate) fn write_stderr_delimiter(test_output: &mut Vec, test_name: &TestName) { + match test_output.last() { + Some(b'\n') => (), + Some(_) => test_output.push(b'\n'), + None => (), + } + writeln!(test_output, "---- {test_name} stderr ----").unwrap(); +} diff --git a/crux-mir/lib/test/src/formatters/pretty.rs b/crux-mir/lib/test/src/formatters/pretty.rs new file mode 100644 index 000000000..247778e51 --- /dev/null +++ b/crux-mir/lib/test/src/formatters/pretty.rs @@ -0,0 +1,281 @@ +use std::{io, io::prelude::Write}; + +use super::OutputFormatter; +use crate::{ + bench::fmt_bench_samples, + console::{ConsoleTestState, OutputLocation}, + term, + test_result::TestResult, + time, + types::TestDesc, +}; + +pub(crate) struct PrettyFormatter { + out: OutputLocation, + use_color: bool, + time_options: Option, + + /// Number of columns to fill when aligning names + max_name_len: usize, + + is_multithreaded: bool, +} + +impl PrettyFormatter { + pub fn new( + out: OutputLocation, + use_color: bool, + max_name_len: usize, + is_multithreaded: bool, + time_options: Option, + ) -> Self { + PrettyFormatter { out, use_color, max_name_len, is_multithreaded, time_options } + } + + #[cfg(test)] + pub fn output_location(&self) -> &OutputLocation { + &self.out + } + + pub fn write_ok(&mut self) -> io::Result<()> { + self.write_short_result("ok", term::color::GREEN) + } + + pub fn write_failed(&mut self) -> io::Result<()> { + self.write_short_result("FAILED", term::color::RED) + } + + pub fn write_ignored(&mut self, message: Option<&'static str>) -> io::Result<()> { + if let Some(message) = message { + self.write_short_result(&format!("ignored, {message}"), term::color::YELLOW) + } else { + self.write_short_result("ignored", term::color::YELLOW) + } + } + + pub fn write_time_failed(&mut self) -> io::Result<()> { + self.write_short_result("FAILED (time limit exceeded)", term::color::RED) + } + + pub fn write_bench(&mut self) -> io::Result<()> { + self.write_pretty("bench", term::color::CYAN) + } + + pub fn write_short_result( + &mut self, + result: &str, + color: term::color::Color, + ) -> io::Result<()> { + self.write_pretty(result, color) + } + + pub fn write_pretty(&mut self, word: &str, color: term::color::Color) -> io::Result<()> { + match self.out { + OutputLocation::Pretty(ref mut term) => { + if self.use_color { + term.fg(color)?; + } + term.write_all(word.as_bytes())?; + if self.use_color { + term.reset()?; + } + term.flush() + } + OutputLocation::Raw(ref mut stdout) => { + stdout.write_all(word.as_bytes())?; + stdout.flush() + } + } + } + + pub fn write_plain>(&mut self, s: S) -> io::Result<()> { + let s = s.as_ref(); + self.out.write_all(s.as_bytes())?; + self.out.flush() + } + + fn write_time( + &mut self, + desc: &TestDesc, + exec_time: Option<&time::TestExecTime>, + ) -> io::Result<()> { + if let (Some(opts), Some(time)) = (self.time_options, exec_time) { + let time_str = format!(" <{time}>"); + + let color = if self.use_color { + if opts.is_critical(desc, time) { + Some(term::color::RED) + } else if opts.is_warn(desc, time) { + Some(term::color::YELLOW) + } else { + None + } + } else { + None + }; + + match color { + Some(color) => self.write_pretty(&time_str, color)?, + None => self.write_plain(&time_str)?, + } + } + + Ok(()) + } + + fn write_results( + &mut self, + inputs: &Vec<(TestDesc, Vec)>, + results_type: &str, + ) -> io::Result<()> { + let results_out_str = format!("\n{results_type}:\n"); + + self.write_plain(&results_out_str)?; + + let mut results = Vec::new(); + let mut stdouts = String::new(); + for (f, stdout) in inputs { + results.push(f.name.to_string()); + if !stdout.is_empty() { + stdouts.push_str(&format!("---- {} stdout ----\n", f.name)); + let output = String::from_utf8_lossy(stdout); + stdouts.push_str(&output); + stdouts.push('\n'); + } + } + if !stdouts.is_empty() { + self.write_plain("\n")?; + self.write_plain(&stdouts)?; + } + + self.write_plain(&results_out_str)?; + results.sort(); + for name in &results { + self.write_plain(&format!(" {name}\n"))?; + } + Ok(()) + } + + pub fn write_successes(&mut self, state: &ConsoleTestState) -> io::Result<()> { + self.write_results(&state.not_failures, "successes") + } + + pub fn write_failures(&mut self, state: &ConsoleTestState) -> io::Result<()> { + self.write_results(&state.failures, "failures") + } + + pub fn write_time_failures(&mut self, state: &ConsoleTestState) -> io::Result<()> { + self.write_results(&state.time_failures, "failures (time limit exceeded)") + } + + fn write_test_name(&mut self, desc: &TestDesc) -> io::Result<()> { + let name = desc.padded_name(self.max_name_len, desc.name.padding()); + if let Some(test_mode) = desc.test_mode() { + self.write_plain(format!("test {name} - {test_mode} ... "))?; + } else { + self.write_plain(format!("test {name} ... "))?; + } + + Ok(()) + } +} + +impl OutputFormatter for PrettyFormatter { + fn write_run_start(&mut self, test_count: usize, shuffle_seed: Option) -> io::Result<()> { + let noun = if test_count != 1 { "tests" } else { "test" }; + let shuffle_seed_msg = if let Some(shuffle_seed) = shuffle_seed { + format!(" (shuffle seed: {shuffle_seed})") + } else { + String::new() + }; + self.write_plain(format!("\nrunning {test_count} {noun}{shuffle_seed_msg}\n")) + } + + fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> { + // When running tests concurrently, we should not print + // the test's name as the result will be mis-aligned. + // When running the tests serially, we print the name here so + // that the user can see which test hangs. + if !self.is_multithreaded { + self.write_test_name(desc)?; + } + + Ok(()) + } + + fn write_result( + &mut self, + desc: &TestDesc, + result: &TestResult, + exec_time: Option<&time::TestExecTime>, + _: &[u8], + _: &ConsoleTestState, + ) -> io::Result<()> { + if self.is_multithreaded { + self.write_test_name(desc)?; + } + + match *result { + TestResult::TrOk => self.write_ok()?, + TestResult::TrFailed | TestResult::TrFailedMsg(_) => self.write_failed()?, + TestResult::TrIgnored => self.write_ignored(desc.ignore_message)?, + TestResult::TrBench(ref bs) => { + self.write_bench()?; + self.write_plain(format!(": {}", fmt_bench_samples(bs)))?; + } + TestResult::TrTimedFail => self.write_time_failed()?, + } + + self.write_time(desc, exec_time)?; + self.write_plain("\n") + } + + fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { + self.write_plain(format!( + "test {} has been running for over {} seconds\n", + desc.name, + time::TEST_WARN_TIMEOUT_S + )) + } + + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { + if state.options.display_output { + self.write_successes(state)?; + } + let success = state.failed == 0; + if !success { + if !state.failures.is_empty() { + self.write_failures(state)?; + } + + if !state.time_failures.is_empty() { + self.write_time_failures(state)?; + } + } + + self.write_plain("\ntest result: ")?; + + if success { + // There's no parallelism at this point so it's safe to use color + self.write_pretty("ok", term::color::GREEN)?; + } else { + self.write_pretty("FAILED", term::color::RED)?; + } + + let s = format!( + ". {} passed; {} failed; {} ignored; {} measured; {} filtered out", + state.passed, state.failed, state.ignored, state.measured, state.filtered_out + ); + + self.write_plain(s)?; + + if let Some(ref exec_time) = state.exec_time { + let time_str = format!("; finished in {exec_time}"); + self.write_plain(time_str)?; + } + + self.write_plain("\n\n")?; + + Ok(success) + } +} diff --git a/crux-mir/lib/test/src/formatters/terse.rs b/crux-mir/lib/test/src/formatters/terse.rs new file mode 100644 index 000000000..0837ab169 --- /dev/null +++ b/crux-mir/lib/test/src/formatters/terse.rs @@ -0,0 +1,259 @@ +use std::{io, io::prelude::Write}; + +use super::OutputFormatter; +use crate::{ + bench::fmt_bench_samples, + console::{ConsoleTestState, OutputLocation}, + term, + test_result::TestResult, + time, + types::NamePadding, + types::TestDesc, +}; + +// We insert a '\n' when the output hits 100 columns in quiet mode. 88 test +// result chars leaves 12 chars for a progress count like " 11704/12853". +const QUIET_MODE_MAX_COLUMN: usize = 88; + +pub(crate) struct TerseFormatter { + out: OutputLocation, + use_color: bool, + is_multithreaded: bool, + /// Number of columns to fill when aligning names + max_name_len: usize, + + test_count: usize, + total_test_count: usize, +} + +impl TerseFormatter { + pub fn new( + out: OutputLocation, + use_color: bool, + max_name_len: usize, + is_multithreaded: bool, + ) -> Self { + TerseFormatter { + out, + use_color, + max_name_len, + is_multithreaded, + test_count: 0, + total_test_count: 0, // initialized later, when write_run_start is called + } + } + + pub fn write_ok(&mut self) -> io::Result<()> { + self.write_short_result(".", term::color::GREEN) + } + + pub fn write_failed(&mut self) -> io::Result<()> { + self.write_short_result("F", term::color::RED) + } + + pub fn write_ignored(&mut self) -> io::Result<()> { + self.write_short_result("i", term::color::YELLOW) + } + + pub fn write_bench(&mut self) -> io::Result<()> { + self.write_pretty("bench", term::color::CYAN) + } + + pub fn write_short_result( + &mut self, + result: &str, + color: term::color::Color, + ) -> io::Result<()> { + self.write_pretty(result, color)?; + if self.test_count % QUIET_MODE_MAX_COLUMN == QUIET_MODE_MAX_COLUMN - 1 { + // We insert a new line regularly in order to flush the + // screen when dealing with line-buffered output (e.g., piping to + // `stamp` in the rust CI). + let out = format!(" {}/{}\n", self.test_count + 1, self.total_test_count); + self.write_plain(out)?; + } + + self.test_count += 1; + Ok(()) + } + + pub fn write_pretty(&mut self, word: &str, color: term::color::Color) -> io::Result<()> { + match self.out { + OutputLocation::Pretty(ref mut term) => { + if self.use_color { + term.fg(color)?; + } + term.write_all(word.as_bytes())?; + if self.use_color { + term.reset()?; + } + term.flush() + } + OutputLocation::Raw(ref mut stdout) => { + stdout.write_all(word.as_bytes())?; + stdout.flush() + } + } + } + + pub fn write_plain>(&mut self, s: S) -> io::Result<()> { + let s = s.as_ref(); + self.out.write_all(s.as_bytes())?; + self.out.flush() + } + + pub fn write_outputs(&mut self, state: &ConsoleTestState) -> io::Result<()> { + self.write_plain("\nsuccesses:\n")?; + let mut successes = Vec::new(); + let mut stdouts = String::new(); + for (f, stdout) in &state.not_failures { + successes.push(f.name.to_string()); + if !stdout.is_empty() { + stdouts.push_str(&format!("---- {} stdout ----\n", f.name)); + let output = String::from_utf8_lossy(stdout); + stdouts.push_str(&output); + stdouts.push('\n'); + } + } + if !stdouts.is_empty() { + self.write_plain("\n")?; + self.write_plain(&stdouts)?; + } + + self.write_plain("\nsuccesses:\n")?; + successes.sort(); + for name in &successes { + self.write_plain(&format!(" {name}\n"))?; + } + Ok(()) + } + + pub fn write_failures(&mut self, state: &ConsoleTestState) -> io::Result<()> { + self.write_plain("\nfailures:\n")?; + let mut failures = Vec::new(); + let mut fail_out = String::new(); + for (f, stdout) in &state.failures { + failures.push(f.name.to_string()); + if !stdout.is_empty() { + fail_out.push_str(&format!("---- {} stdout ----\n", f.name)); + let output = String::from_utf8_lossy(stdout); + fail_out.push_str(&output); + fail_out.push('\n'); + } + } + if !fail_out.is_empty() { + self.write_plain("\n")?; + self.write_plain(&fail_out)?; + } + + self.write_plain("\nfailures:\n")?; + failures.sort(); + for name in &failures { + self.write_plain(&format!(" {name}\n"))?; + } + Ok(()) + } + + fn write_test_name(&mut self, desc: &TestDesc) -> io::Result<()> { + let name = desc.padded_name(self.max_name_len, desc.name.padding()); + if let Some(test_mode) = desc.test_mode() { + self.write_plain(format!("test {name} - {test_mode} ... "))?; + } else { + self.write_plain(format!("test {name} ... "))?; + } + + Ok(()) + } +} + +impl OutputFormatter for TerseFormatter { + fn write_run_start(&mut self, test_count: usize, shuffle_seed: Option) -> io::Result<()> { + self.total_test_count = test_count; + let noun = if test_count != 1 { "tests" } else { "test" }; + let shuffle_seed_msg = if let Some(shuffle_seed) = shuffle_seed { + format!(" (shuffle seed: {shuffle_seed})") + } else { + String::new() + }; + self.write_plain(format!("\nrunning {test_count} {noun}{shuffle_seed_msg}\n")) + } + + fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> { + // Remnants from old libtest code that used the padding value + // in order to indicate benchmarks. + // When running benchmarks, terse-mode should still print their name as if + // it is the Pretty formatter. + if !self.is_multithreaded && desc.name.padding() == NamePadding::PadOnRight { + self.write_test_name(desc)?; + } + + Ok(()) + } + + fn write_result( + &mut self, + desc: &TestDesc, + result: &TestResult, + _: Option<&time::TestExecTime>, + _: &[u8], + _: &ConsoleTestState, + ) -> io::Result<()> { + match *result { + TestResult::TrOk => self.write_ok(), + TestResult::TrFailed | TestResult::TrFailedMsg(_) | TestResult::TrTimedFail => { + self.write_failed() + } + TestResult::TrIgnored => self.write_ignored(), + TestResult::TrBench(ref bs) => { + if self.is_multithreaded { + self.write_test_name(desc)?; + } + self.write_bench()?; + self.write_plain(format!(": {}\n", fmt_bench_samples(bs))) + } + } + } + + fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { + self.write_plain(format!( + "test {} has been running for over {} seconds\n", + desc.name, + time::TEST_WARN_TIMEOUT_S + )) + } + + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { + if state.options.display_output { + self.write_outputs(state)?; + } + let success = state.failed == 0; + if !success { + self.write_failures(state)?; + } + + self.write_plain("\ntest result: ")?; + + if success { + // There's no parallelism at this point so it's safe to use color + self.write_pretty("ok", term::color::GREEN)?; + } else { + self.write_pretty("FAILED", term::color::RED)?; + } + + let s = format!( + ". {} passed; {} failed; {} ignored; {} measured; {} filtered out", + state.passed, state.failed, state.ignored, state.measured, state.filtered_out + ); + + self.write_plain(s)?; + + if let Some(ref exec_time) = state.exec_time { + let time_str = format!("; finished in {exec_time}"); + self.write_plain(time_str)?; + } + + self.write_plain("\n\n")?; + + Ok(success) + } +} diff --git a/crux-mir/lib/test/src/helpers/concurrency.rs b/crux-mir/lib/test/src/helpers/concurrency.rs new file mode 100644 index 000000000..eb2111573 --- /dev/null +++ b/crux-mir/lib/test/src/helpers/concurrency.rs @@ -0,0 +1,14 @@ +//! Helper module which helps to determine amount of threads to be used +//! during tests execution. +use std::{env, num::NonZeroUsize, thread}; + +pub fn get_concurrency() -> usize { + if let Ok(value) = env::var("RUST_TEST_THREADS") { + match value.parse::().ok() { + Some(n) => n.get(), + _ => panic!("RUST_TEST_THREADS is `{value}`, should be a positive integer."), + } + } else { + thread::available_parallelism().map(|n| n.get()).unwrap_or(1) + } +} diff --git a/crux-mir/lib/test/src/helpers/exit_code.rs b/crux-mir/lib/test/src/helpers/exit_code.rs new file mode 100644 index 000000000..f762f8881 --- /dev/null +++ b/crux-mir/lib/test/src/helpers/exit_code.rs @@ -0,0 +1,20 @@ +//! Helper module to detect subprocess exit code. + +use std::process::ExitStatus; + +#[cfg(not(unix))] +pub fn get_exit_code(status: ExitStatus) -> Result { + status.code().ok_or_else(|| "received no exit code from child process".into()) +} + +#[cfg(unix)] +pub fn get_exit_code(status: ExitStatus) -> Result { + use std::os::unix::process::ExitStatusExt; + match status.code() { + Some(code) => Ok(code), + None => match status.signal() { + Some(signal) => Err(format!("child process exited with signal {signal}")), + None => Err("child process exited with unknown signal".into()), + }, + } +} diff --git a/crux-mir/lib/test/src/helpers/metrics.rs b/crux-mir/lib/test/src/helpers/metrics.rs new file mode 100644 index 000000000..f77a23e68 --- /dev/null +++ b/crux-mir/lib/test/src/helpers/metrics.rs @@ -0,0 +1,50 @@ +//! Benchmark metrics. +use std::collections::BTreeMap; + +#[derive(Clone, PartialEq, Debug, Copy)] +pub struct Metric { + value: f64, + noise: f64, +} + +impl Metric { + pub fn new(value: f64, noise: f64) -> Metric { + Metric { value, noise } + } +} + +#[derive(Clone, PartialEq)] +pub struct MetricMap(BTreeMap); + +impl MetricMap { + pub fn new() -> MetricMap { + MetricMap(BTreeMap::new()) + } + + /// Insert a named `value` (+/- `noise`) metric into the map. The value + /// must be non-negative. The `noise` indicates the uncertainty of the + /// metric, which doubles as the "noise range" of acceptable + /// pairwise-regressions on this named value, when comparing from one + /// metric to the next using `compare_to_old`. + /// + /// If `noise` is positive, then it means this metric is of a value + /// you want to see grow smaller, so a change larger than `noise` in the + /// positive direction represents a regression. + /// + /// If `noise` is negative, then it means this metric is of a value + /// you want to see grow larger, so a change larger than `noise` in the + /// negative direction represents a regression. + pub fn insert_metric(&mut self, name: &str, value: f64, noise: f64) { + let m = Metric { value, noise }; + self.0.insert(name.to_owned(), m); + } + + pub fn fmt_metrics(&self) -> String { + let v = self + .0 + .iter() + .map(|(k, v)| format!("{}: {} (+/- {})", *k, v.value, v.noise)) + .collect::>(); + v.join(", ") + } +} diff --git a/crux-mir/lib/test/src/helpers/mod.rs b/crux-mir/lib/test/src/helpers/mod.rs new file mode 100644 index 000000000..6f366a911 --- /dev/null +++ b/crux-mir/lib/test/src/helpers/mod.rs @@ -0,0 +1,7 @@ +//! Module with common helpers not directly related to tests +//! but used in `libtest`. + +pub mod concurrency; +pub mod exit_code; +pub mod metrics; +pub mod shuffle; diff --git a/crux-mir/lib/test/src/helpers/shuffle.rs b/crux-mir/lib/test/src/helpers/shuffle.rs new file mode 100644 index 000000000..ca503106c --- /dev/null +++ b/crux-mir/lib/test/src/helpers/shuffle.rs @@ -0,0 +1,67 @@ +use crate::cli::TestOpts; +use crate::types::{TestDescAndFn, TestId, TestName}; +use std::collections::hash_map::DefaultHasher; +use std::hash::Hasher; +use std::time::{SystemTime, UNIX_EPOCH}; + +pub fn get_shuffle_seed(opts: &TestOpts) -> Option { + opts.shuffle_seed.or_else(|| { + if opts.shuffle { + Some( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Failed to get system time") + .as_nanos() as u64, + ) + } else { + None + } + }) +} + +pub fn shuffle_tests(shuffle_seed: u64, tests: &mut [(TestId, TestDescAndFn)]) { + let test_names: Vec<&TestName> = tests.iter().map(|test| &test.1.desc.name).collect(); + let test_names_hash = calculate_hash(&test_names); + let mut rng = Rng::new(shuffle_seed, test_names_hash); + shuffle(&mut rng, tests); +} + +// `shuffle` is from `rust-analyzer/src/cli/analysis_stats.rs`. +fn shuffle(rng: &mut Rng, slice: &mut [T]) { + for i in 0..slice.len() { + randomize_first(rng, &mut slice[i..]); + } + + fn randomize_first(rng: &mut Rng, slice: &mut [T]) { + assert!(!slice.is_empty()); + let idx = rng.rand_range(0..slice.len() as u64) as usize; + slice.swap(0, idx); + } +} + +struct Rng { + state: u64, + extra: u64, +} + +impl Rng { + fn new(seed: u64, extra: u64) -> Self { + Self { state: seed, extra } + } + + fn rand_range(&mut self, range: core::ops::Range) -> u64 { + self.rand_u64() % (range.end - range.start) + range.start + } + + fn rand_u64(&mut self) -> u64 { + self.state = calculate_hash(&(self.state, self.extra)); + self.state + } +} + +// `calculate_hash` is from `core/src/hash/mod.rs`. +fn calculate_hash(t: &T) -> u64 { + let mut s = DefaultHasher::new(); + t.hash(&mut s); + s.finish() +} diff --git a/crux-mir/lib/test/src/lib.rs b/crux-mir/lib/test/src/lib.rs new file mode 100644 index 000000000..69fb529d7 --- /dev/null +++ b/crux-mir/lib/test/src/lib.rs @@ -0,0 +1,798 @@ +//! Support code for rustc's built in unit-test and micro-benchmarking +//! framework. +//! +//! Almost all user code will only be interested in `Bencher` and +//! `black_box`. All other interactions (such as writing tests and +//! benchmarks themselves) should be done via the `#[test]` and +//! `#[bench]` attributes. +//! +//! See the [Testing Chapter](../book/ch11-00-testing.html) of the book for more +//! details. + +// Currently, not much of this is meant for users. It is intended to +// support the simplest interface possible for representing and +// running tests while providing a base that other test frameworks may +// build off of. + +#![unstable(feature = "test", issue = "50297")] +#![doc(test(attr(deny(warnings))))] +#![feature(internal_output_capture)] +#![feature(is_terminal)] +#![feature(staged_api)] +#![feature(process_exitcode_internals)] +#![feature(panic_can_unwind)] +#![feature(test)] + +// Public reexports +pub use self::bench::{black_box, Bencher}; +pub use self::console::run_tests_console; +pub use self::options::{ColorConfig, Options, OutputFormat, RunIgnored, ShouldPanic}; +pub use self::types::TestName::*; +pub use self::types::*; +pub use self::ColorConfig::*; +pub use cli::TestOpts; + +// Module to be used by rustc to compile tests in libtest +pub mod test { + pub use crate::{ + assert_test_result, + bench::Bencher, + cli::{parse_opts, TestOpts}, + filter_tests, + helpers::metrics::{Metric, MetricMap}, + options::{Options, RunIgnored, RunStrategy, ShouldPanic}, + run_test, test_main, test_main_static, + test_result::{TestResult, TrFailed, TrFailedMsg, TrIgnored, TrOk}, + time::{TestExecTime, TestTimeOptions}, + types::{ + DynTestFn, DynTestName, StaticBenchFn, StaticTestFn, StaticTestName, TestDesc, + TestDescAndFn, TestId, TestName, TestType, + }, + }; +} + +use std::{ + collections::VecDeque, + env, io, + io::prelude::Write, + mem::ManuallyDrop, + panic::{self, catch_unwind, AssertUnwindSafe, PanicInfo}, + process::{self, Command, Termination}, + sync::mpsc::{channel, Sender}, + sync::{Arc, Mutex}, + thread, + time::{Duration, Instant}, +}; + +pub mod bench; +mod cli; +mod console; +mod event; +mod formatters; +mod helpers; +mod options; +pub mod stats; +mod term; +mod test_result; +mod time; +mod types; + +#[cfg(test)] +mod tests; + +use core::any::Any; +use event::{CompletedTest, TestEvent}; +use helpers::concurrency::get_concurrency; +use helpers::exit_code::get_exit_code; +use helpers::shuffle::{get_shuffle_seed, shuffle_tests}; +use options::RunStrategy; +use test_result::*; +use time::TestExecTime; + +// Process exit code to be used to indicate test failures. +const ERROR_EXIT_CODE: i32 = 101; + +const SECONDARY_TEST_INVOKER_VAR: &str = "__RUST_TEST_INVOKE"; + +// The default console test runner. It accepts the command line +// arguments and a vector of test_descs. +pub fn test_main(args: &[String], tests: Vec, options: Option) { + let mut opts = match cli::parse_opts(args) { + Some(Ok(o)) => o, + Some(Err(msg)) => { + eprintln!("error: {msg}"); + process::exit(ERROR_EXIT_CODE); + } + None => return, + }; + if let Some(options) = options { + opts.options = options; + } + if opts.list { + if let Err(e) = console::list_tests_console(&opts, tests) { + eprintln!("error: io error when listing tests: {e:?}"); + process::exit(ERROR_EXIT_CODE); + } + } else { + if !opts.nocapture { + // If we encounter a non-unwinding panic, flush any captured output from the current test, + // and stop capturing output to ensure that the non-unwinding panic message is visible. + // We also acquire the locks for both output streams to prevent output from other threads + // from interleaving with the panic message or appearing after it. + let builtin_panic_hook = panic::take_hook(); + let hook = Box::new({ + move |info: &'_ PanicInfo<'_>| { + if !info.can_unwind() { + std::mem::forget(std::io::stderr().lock()); + let mut stdout = ManuallyDrop::new(std::io::stdout().lock()); + if let Some(captured) = io::set_output_capture(None) { + if let Ok(data) = captured.lock() { + let _ = stdout.write_all(&data); + let _ = stdout.flush(); + } + } + } + builtin_panic_hook(info); + } + }); + panic::set_hook(hook); + } + match console::run_tests_console(&opts, tests) { + Ok(true) => {} + Ok(false) => process::exit(ERROR_EXIT_CODE), + Err(e) => { + eprintln!("error: io error when listing tests: {e:?}"); + process::exit(ERROR_EXIT_CODE); + } + } + } +} + +/// A variant optimized for invocation with a static test vector. +/// This will panic (intentionally) when fed any dynamic tests. +/// +/// This is the entry point for the main function generated by `rustc --test` +/// when panic=unwind. +pub fn test_main_static(tests: &[&TestDescAndFn]) { + let args = env::args().collect::>(); + let owned_tests: Vec<_> = tests.iter().map(make_owned_test).collect(); + test_main(&args, owned_tests, None) +} + +/// A variant optimized for invocation with a static test vector. +/// This will panic (intentionally) when fed any dynamic tests. +/// +/// Runs tests in panic=abort mode, which involves spawning subprocesses for +/// tests. +/// +/// This is the entry point for the main function generated by `rustc --test` +/// when panic=abort. +pub fn test_main_static_abort(tests: &[&TestDescAndFn]) { + // If we're being run in SpawnedSecondary mode, run the test here. run_test + // will then exit the process. + if let Ok(name) = env::var(SECONDARY_TEST_INVOKER_VAR) { + env::remove_var(SECONDARY_TEST_INVOKER_VAR); + let test = tests + .iter() + .filter(|test| test.desc.name.as_slice() == name) + .map(make_owned_test) + .next() + .unwrap_or_else(|| panic!("couldn't find a test with the provided name '{name}'")); + let TestDescAndFn { desc, testfn } = test; + let testfn = match testfn { + StaticTestFn(f) => f, + _ => panic!("only static tests are supported"), + }; + run_test_in_spawned_subprocess(desc, Box::new(testfn)); + } + + let args = env::args().collect::>(); + let owned_tests: Vec<_> = tests.iter().map(make_owned_test).collect(); + test_main(&args, owned_tests, Some(Options::new().panic_abort(true))) +} + +/// Clones static values for putting into a dynamic vector, which test_main() +/// needs to hand out ownership of tests to parallel test runners. +/// +/// This will panic when fed any dynamic tests, because they cannot be cloned. +fn make_owned_test(test: &&TestDescAndFn) -> TestDescAndFn { + match test.testfn { + StaticTestFn(f) => TestDescAndFn { testfn: StaticTestFn(f), desc: test.desc.clone() }, + StaticBenchFn(f) => TestDescAndFn { testfn: StaticBenchFn(f), desc: test.desc.clone() }, + _ => panic!("non-static tests passed to test::test_main_static"), + } +} + +/// Invoked when unit tests terminate. Returns `Result::Err` if the test is +/// considered a failure. By default, invokes `report() and checks for a `0` +/// result. +pub fn assert_test_result(result: T) -> Result<(), String> { + let code = result.report().to_i32(); + if code == 0 { + Ok(()) + } else { + Err(format!( + "the test returned a termination value with a non-zero status code \ + ({code}) which indicates a failure" + )) + } +} + +struct FilteredTests { + tests: Vec<(TestId, TestDescAndFn)>, + benchs: Vec<(TestId, TestDescAndFn)>, + next_id: usize, +} + +impl FilteredTests { + fn add_bench(&mut self, desc: TestDesc, testfn: TestFn) { + let test = TestDescAndFn { desc, testfn }; + self.benchs.push((TestId(self.next_id), test)); + self.next_id += 1; + } + fn add_test(&mut self, desc: TestDesc, testfn: TestFn) { + let test = TestDescAndFn { desc, testfn }; + self.tests.push((TestId(self.next_id), test)); + self.next_id += 1; + } + fn add_bench_as_test( + &mut self, + desc: TestDesc, + benchfn: impl Fn(&mut Bencher) -> Result<(), String> + Send + 'static, + ) { + let testfn = DynTestFn(Box::new(move || { + bench::run_once(|b| __rust_begin_short_backtrace(|| benchfn(b))) + })); + self.add_test(desc, testfn); + } + fn total_len(&self) -> usize { + self.tests.len() + self.benchs.len() + } +} + +pub fn run_tests( + opts: &TestOpts, + tests: Vec, + mut notify_about_test_event: F, +) -> io::Result<()> +where + F: FnMut(TestEvent) -> io::Result<()>, +{ + use std::collections::{self, HashMap}; + use std::hash::BuildHasherDefault; + use std::sync::mpsc::RecvTimeoutError; + + struct RunningTest { + join_handle: Option>, + } + + impl RunningTest { + fn join(self, completed_test: &mut CompletedTest) { + if let Some(join_handle) = self.join_handle { + if let Err(_) = join_handle.join() { + if let TrOk = completed_test.result { + completed_test.result = + TrFailedMsg("panicked after reporting success".to_string()); + } + } + } + } + } + + // Use a deterministic hasher + type TestMap = + HashMap>; + + struct TimeoutEntry { + id: TestId, + desc: TestDesc, + timeout: Instant, + } + + let tests_len = tests.len(); + + let mut filtered = FilteredTests { tests: Vec::new(), benchs: Vec::new(), next_id: 0 }; + + for test in filter_tests(opts, tests) { + let mut desc = test.desc; + desc.name = desc.name.with_padding(test.testfn.padding()); + + match test.testfn { + DynBenchFn(benchfn) => { + if opts.bench_benchmarks { + filtered.add_bench(desc, DynBenchFn(benchfn)); + } else { + filtered.add_bench_as_test(desc, benchfn); + } + } + StaticBenchFn(benchfn) => { + if opts.bench_benchmarks { + filtered.add_bench(desc, StaticBenchFn(benchfn)); + } else { + filtered.add_bench_as_test(desc, benchfn); + } + } + testfn => { + filtered.add_test(desc, testfn); + } + }; + } + + let filtered_out = tests_len - filtered.total_len(); + let event = TestEvent::TeFilteredOut(filtered_out); + notify_about_test_event(event)?; + + let shuffle_seed = get_shuffle_seed(opts); + + let event = TestEvent::TeFiltered(filtered.total_len(), shuffle_seed); + notify_about_test_event(event)?; + + let concurrency = opts.test_threads.unwrap_or_else(get_concurrency); + + let mut remaining = filtered.tests; + if let Some(shuffle_seed) = shuffle_seed { + shuffle_tests(shuffle_seed, &mut remaining); + } + // Store the tests in a VecDeque so we can efficiently remove the first element to run the + // tests in the order they were passed (unless shuffled). + let mut remaining = VecDeque::from(remaining); + let mut pending = 0; + + let (tx, rx) = channel::(); + let run_strategy = if opts.options.panic_abort && !opts.force_run_in_process { + RunStrategy::SpawnPrimary + } else { + RunStrategy::InProcess + }; + + let mut running_tests: TestMap = HashMap::default(); + let mut timeout_queue: VecDeque = VecDeque::new(); + + fn get_timed_out_tests( + running_tests: &TestMap, + timeout_queue: &mut VecDeque, + ) -> Vec { + let now = Instant::now(); + let mut timed_out = Vec::new(); + while let Some(timeout_entry) = timeout_queue.front() { + if now < timeout_entry.timeout { + break; + } + let timeout_entry = timeout_queue.pop_front().unwrap(); + if running_tests.contains_key(&timeout_entry.id) { + timed_out.push(timeout_entry.desc); + } + } + timed_out + } + + fn calc_timeout(timeout_queue: &VecDeque) -> Option { + timeout_queue.front().map(|&TimeoutEntry { timeout: next_timeout, .. }| { + let now = Instant::now(); + if next_timeout >= now { next_timeout - now } else { Duration::new(0, 0) } + }) + } + + if concurrency == 1 { + while !remaining.is_empty() { + let (id, test) = remaining.pop_front().unwrap(); + let event = TestEvent::TeWait(test.desc.clone()); + notify_about_test_event(event)?; + let join_handle = run_test(opts, !opts.run_tests, id, test, run_strategy, tx.clone()); + // Wait for the test to complete. + let mut completed_test = rx.recv().unwrap(); + RunningTest { join_handle }.join(&mut completed_test); + + let fail_fast = match completed_test.result { + TrIgnored | TrOk | TrBench(_) => false, + TrFailed | TrFailedMsg(_) | TrTimedFail => opts.fail_fast, + }; + + let event = TestEvent::TeResult(completed_test); + notify_about_test_event(event)?; + + if fail_fast { + return Ok(()); + } + } + } else { + while pending > 0 || !remaining.is_empty() { + while pending < concurrency && !remaining.is_empty() { + let (id, test) = remaining.pop_front().unwrap(); + let timeout = time::get_default_test_timeout(); + let desc = test.desc.clone(); + + let event = TestEvent::TeWait(desc.clone()); + notify_about_test_event(event)?; //here no pad + let join_handle = + run_test(opts, !opts.run_tests, id, test, run_strategy, tx.clone()); + running_tests.insert(id, RunningTest { join_handle }); + timeout_queue.push_back(TimeoutEntry { id, desc, timeout }); + pending += 1; + } + + let mut res; + loop { + if let Some(timeout) = calc_timeout(&timeout_queue) { + res = rx.recv_timeout(timeout); + for test in get_timed_out_tests(&running_tests, &mut timeout_queue) { + let event = TestEvent::TeTimeout(test); + notify_about_test_event(event)?; + } + + match res { + Err(RecvTimeoutError::Timeout) => { + // Result is not yet ready, continue waiting. + } + _ => { + // We've got a result, stop the loop. + break; + } + } + } else { + res = rx.recv().map_err(|_| RecvTimeoutError::Disconnected); + break; + } + } + + let mut completed_test = res.unwrap(); + let running_test = running_tests.remove(&completed_test.id).unwrap(); + running_test.join(&mut completed_test); + + let fail_fast = match completed_test.result { + TrIgnored | TrOk | TrBench(_) => false, + TrFailed | TrFailedMsg(_) | TrTimedFail => opts.fail_fast, + }; + + let event = TestEvent::TeResult(completed_test); + notify_about_test_event(event)?; + pending -= 1; + + if fail_fast { + // Prevent remaining test threads from panicking + std::mem::forget(rx); + return Ok(()); + } + } + } + + if opts.bench_benchmarks { + // All benchmarks run at the end, in serial. + for (id, b) in filtered.benchs { + let event = TestEvent::TeWait(b.desc.clone()); + notify_about_test_event(event)?; + let join_handle = run_test(opts, false, id, b, run_strategy, tx.clone()); + // Wait for the test to complete. + let mut completed_test = rx.recv().unwrap(); + RunningTest { join_handle }.join(&mut completed_test); + + let event = TestEvent::TeResult(completed_test); + notify_about_test_event(event)?; + } + } + Ok(()) +} + +pub fn filter_tests(opts: &TestOpts, tests: Vec) -> Vec { + let mut filtered = tests; + let matches_filter = |test: &TestDescAndFn, filter: &str| { + let test_name = test.desc.name.as_slice(); + + match opts.filter_exact { + true => test_name == filter, + false => test_name.contains(filter), + } + }; + + // Remove tests that don't match the test filter + if !opts.filters.is_empty() { + filtered.retain(|test| opts.filters.iter().any(|filter| matches_filter(test, filter))); + } + + // Skip tests that match any of the skip filters + if !opts.skip.is_empty() { + filtered.retain(|test| !opts.skip.iter().any(|sf| matches_filter(test, sf))); + } + + // Excludes #[should_panic] tests + if opts.exclude_should_panic { + filtered.retain(|test| test.desc.should_panic == ShouldPanic::No); + } + + // maybe unignore tests + match opts.run_ignored { + RunIgnored::Yes => { + filtered.iter_mut().for_each(|test| test.desc.ignore = false); + } + RunIgnored::Only => { + filtered.retain(|test| test.desc.ignore); + filtered.iter_mut().for_each(|test| test.desc.ignore = false); + } + RunIgnored::No => {} + } + + filtered +} + +pub fn convert_benchmarks_to_tests(tests: Vec) -> Vec { + // convert benchmarks to tests, if we're not benchmarking them + tests + .into_iter() + .map(|x| { + let testfn = match x.testfn { + DynBenchFn(benchfn) => DynTestFn(Box::new(move || { + bench::run_once(|b| __rust_begin_short_backtrace(|| benchfn(b))) + })), + StaticBenchFn(benchfn) => DynTestFn(Box::new(move || { + bench::run_once(|b| __rust_begin_short_backtrace(|| benchfn(b))) + })), + f => f, + }; + TestDescAndFn { desc: x.desc, testfn } + }) + .collect() +} + +pub fn run_test( + opts: &TestOpts, + force_ignore: bool, + id: TestId, + test: TestDescAndFn, + strategy: RunStrategy, + monitor_ch: Sender, +) -> Option> { + let TestDescAndFn { desc, testfn } = test; + + // Emscripten can catch panics but other wasm targets cannot + let ignore_because_no_process_support = desc.should_panic != ShouldPanic::No + && cfg!(target_family = "wasm") + && !cfg!(target_os = "emscripten"); + + if force_ignore || desc.ignore || ignore_because_no_process_support { + let message = CompletedTest::new(id, desc, TrIgnored, None, Vec::new()); + monitor_ch.send(message).unwrap(); + return None; + } + + struct TestRunOpts { + pub strategy: RunStrategy, + pub nocapture: bool, + pub time: Option, + } + + fn run_test_inner( + id: TestId, + desc: TestDesc, + monitor_ch: Sender, + testfn: Box Result<(), String> + Send>, + opts: TestRunOpts, + ) -> Option> { + let name = desc.name.clone(); + + let runtest = move || match opts.strategy { + RunStrategy::InProcess => run_test_in_process( + id, + desc, + opts.nocapture, + opts.time.is_some(), + testfn, + monitor_ch, + opts.time, + ), + RunStrategy::SpawnPrimary => spawn_test_subprocess( + id, + desc, + opts.nocapture, + opts.time.is_some(), + monitor_ch, + opts.time, + ), + }; + + // If the platform is single-threaded we're just going to run + // the test synchronously, regardless of the concurrency + // level. + let supports_threads = !cfg!(target_os = "emscripten") && !cfg!(target_family = "wasm"); + if supports_threads { + let cfg = thread::Builder::new().name(name.as_slice().to_owned()); + let mut runtest = Arc::new(Mutex::new(Some(runtest))); + let runtest2 = runtest.clone(); + match cfg.spawn(move || runtest2.lock().unwrap().take().unwrap()()) { + Ok(handle) => Some(handle), + Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + // `ErrorKind::WouldBlock` means hitting the thread limit on some + // platforms, so run the test synchronously here instead. + Arc::get_mut(&mut runtest).unwrap().get_mut().unwrap().take().unwrap()(); + None + } + Err(e) => panic!("failed to spawn thread to run test: {e}"), + } + } else { + runtest(); + None + } + } + + let test_run_opts = + TestRunOpts { strategy, nocapture: opts.nocapture, time: opts.time_options }; + + match testfn { + DynBenchFn(benchfn) => { + // Benchmarks aren't expected to panic, so we run them all in-process. + crate::bench::benchmark(id, desc, monitor_ch, opts.nocapture, benchfn); + None + } + StaticBenchFn(benchfn) => { + // Benchmarks aren't expected to panic, so we run them all in-process. + crate::bench::benchmark(id, desc, monitor_ch, opts.nocapture, benchfn); + None + } + DynTestFn(f) => { + match strategy { + RunStrategy::InProcess => (), + _ => panic!("Cannot run dynamic test fn out-of-process"), + }; + run_test_inner( + id, + desc, + monitor_ch, + Box::new(move || __rust_begin_short_backtrace(f)), + test_run_opts, + ) + } + StaticTestFn(f) => run_test_inner( + id, + desc, + monitor_ch, + Box::new(move || __rust_begin_short_backtrace(f)), + test_run_opts, + ), + } +} + +/// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`. +#[inline(never)] +fn __rust_begin_short_backtrace T>(f: F) -> T { + let result = f(); + + // prevent this frame from being tail-call optimised away + black_box(result) +} + +fn run_test_in_process( + id: TestId, + desc: TestDesc, + nocapture: bool, + report_time: bool, + testfn: Box Result<(), String> + Send>, + monitor_ch: Sender, + time_opts: Option, +) { + // Buffer for capturing standard I/O + let data = Arc::new(Mutex::new(Vec::new())); + + if !nocapture { + io::set_output_capture(Some(data.clone())); + } + + let start = report_time.then(Instant::now); + let result = fold_err(catch_unwind(AssertUnwindSafe(testfn))); + let exec_time = start.map(|start| { + let duration = start.elapsed(); + TestExecTime(duration) + }); + + io::set_output_capture(None); + + let test_result = match result { + Ok(()) => calc_result(&desc, Ok(()), &time_opts, &exec_time), + Err(e) => calc_result(&desc, Err(e.as_ref()), &time_opts, &exec_time), + }; + let stdout = data.lock().unwrap_or_else(|e| e.into_inner()).to_vec(); + let message = CompletedTest::new(id, desc, test_result, exec_time, stdout); + monitor_ch.send(message).unwrap(); +} + +fn fold_err( + result: Result, Box>, +) -> Result> +where + E: Send + 'static, +{ + match result { + Ok(Err(e)) => Err(Box::new(e)), + Ok(Ok(v)) => Ok(v), + Err(e) => Err(e), + } +} + +fn spawn_test_subprocess( + id: TestId, + desc: TestDesc, + nocapture: bool, + report_time: bool, + monitor_ch: Sender, + time_opts: Option, +) { + let (result, test_output, exec_time) = (|| { + let args = env::args().collect::>(); + let current_exe = &args[0]; + + let mut command = Command::new(current_exe); + command.env(SECONDARY_TEST_INVOKER_VAR, desc.name.as_slice()); + if nocapture { + command.stdout(process::Stdio::inherit()); + command.stderr(process::Stdio::inherit()); + } + + let start = report_time.then(Instant::now); + let output = match command.output() { + Ok(out) => out, + Err(e) => { + let err = format!("Failed to spawn {} as child for test: {:?}", args[0], e); + return (TrFailed, err.into_bytes(), None); + } + }; + let exec_time = start.map(|start| { + let duration = start.elapsed(); + TestExecTime(duration) + }); + + let std::process::Output { stdout, stderr, status } = output; + let mut test_output = stdout; + formatters::write_stderr_delimiter(&mut test_output, &desc.name); + test_output.extend_from_slice(&stderr); + + let result = match (|| -> Result { + let exit_code = get_exit_code(status)?; + Ok(get_result_from_exit_code(&desc, exit_code, &time_opts, &exec_time)) + })() { + Ok(r) => r, + Err(e) => { + write!(&mut test_output, "Unexpected error: {e}").unwrap(); + TrFailed + } + }; + + (result, test_output, exec_time) + })(); + + let message = CompletedTest::new(id, desc, result, exec_time, test_output); + monitor_ch.send(message).unwrap(); +} + +fn run_test_in_spawned_subprocess( + desc: TestDesc, + testfn: Box Result<(), String> + Send>, +) -> ! { + let builtin_panic_hook = panic::take_hook(); + let record_result = Arc::new(move |panic_info: Option<&'_ PanicInfo<'_>>| { + let test_result = match panic_info { + Some(info) => calc_result(&desc, Err(info.payload()), &None, &None), + None => calc_result(&desc, Ok(()), &None, &None), + }; + + // We don't support serializing TrFailedMsg, so just + // print the message out to stderr. + if let TrFailedMsg(msg) = &test_result { + eprintln!("{msg}"); + } + + if let Some(info) = panic_info { + builtin_panic_hook(info); + } + + if let TrOk = test_result { + process::exit(test_result::TR_OK); + } else { + process::exit(test_result::TR_FAILED); + } + }); + let record_result2 = record_result.clone(); + panic::set_hook(Box::new(move |info| record_result2(Some(info)))); + if let Err(message) = testfn() { + panic!("{}", message); + } + record_result(None); + unreachable!("panic=abort callback should have exited the process") +} diff --git a/crux-mir/lib/test/src/options.rs b/crux-mir/lib/test/src/options.rs new file mode 100644 index 000000000..75ec0b616 --- /dev/null +++ b/crux-mir/lib/test/src/options.rs @@ -0,0 +1,82 @@ +//! Enums denoting options for test execution. + +/// Number of times to run a benchmarked function +#[derive(Clone, PartialEq, Eq)] +pub enum BenchMode { + Auto, + Single, +} + +/// Whether test is expected to panic or not +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub enum ShouldPanic { + No, + Yes, + YesWithMessage(&'static str), +} + +/// Whether should console output be colored or not +#[derive(Copy, Clone, Debug)] +pub enum ColorConfig { + AutoColor, + AlwaysColor, + NeverColor, +} + +/// Format of the test results output +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum OutputFormat { + /// Verbose output + Pretty, + /// Quiet output + Terse, + /// JSON output + Json, + /// JUnit output + Junit, +} + +/// Whether ignored test should be run or not +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum RunIgnored { + Yes, + No, + /// Run only ignored tests + Only, +} + +#[derive(Clone, Copy)] +pub enum RunStrategy { + /// Runs the test in the current process, and sends the result back over the + /// supplied channel. + InProcess, + + /// Spawns a subprocess to run the test, and sends the result back over the + /// supplied channel. Requires `argv[0]` to exist and point to the binary + /// that's currently running. + SpawnPrimary, +} + +/// Options for the test run defined by the caller (instead of CLI arguments). +/// In case we want to add other options as well, just add them in this struct. +#[derive(Copy, Clone, Debug)] +pub struct Options { + pub display_output: bool, + pub panic_abort: bool, +} + +impl Options { + pub fn new() -> Options { + Options { display_output: false, panic_abort: false } + } + + pub fn display_output(mut self, display_output: bool) -> Options { + self.display_output = display_output; + self + } + + pub fn panic_abort(mut self, panic_abort: bool) -> Options { + self.panic_abort = panic_abort; + self + } +} diff --git a/crux-mir/lib/test/src/stats.rs b/crux-mir/lib/test/src/stats.rs new file mode 100644 index 000000000..b33b08012 --- /dev/null +++ b/crux-mir/lib/test/src/stats.rs @@ -0,0 +1,302 @@ +#![allow(missing_docs)] + +use std::mem; + +#[cfg(test)] +mod tests; + +fn local_sort(v: &mut [f64]) { + v.sort_by(|x: &f64, y: &f64| x.total_cmp(y)); +} + +/// Trait that provides simple descriptive statistics on a univariate set of numeric samples. +pub trait Stats { + /// Sum of the samples. + /// + /// Note: this method sacrifices performance at the altar of accuracy + /// Depends on IEEE 754 arithmetic guarantees. See proof of correctness at: + /// ["Adaptive Precision Floating-Point Arithmetic and Fast Robust Geometric + /// Predicates"][paper] + /// + /// [paper]: https://www.cs.cmu.edu/~quake-papers/robust-arithmetic.ps + fn sum(&self) -> f64; + + /// Minimum value of the samples. + fn min(&self) -> f64; + + /// Maximum value of the samples. + fn max(&self) -> f64; + + /// Arithmetic mean (average) of the samples: sum divided by sample-count. + /// + /// See: + fn mean(&self) -> f64; + + /// Median of the samples: value separating the lower half of the samples from the higher half. + /// Equal to `self.percentile(50.0)`. + /// + /// See: + fn median(&self) -> f64; + + /// Variance of the samples: bias-corrected mean of the squares of the differences of each + /// sample from the sample mean. Note that this calculates the _sample variance_ rather than the + /// population variance, which is assumed to be unknown. It therefore corrects the `(n-1)/n` + /// bias that would appear if we calculated a population variance, by dividing by `(n-1)` rather + /// than `n`. + /// + /// See: + fn var(&self) -> f64; + + /// Standard deviation: the square root of the sample variance. + /// + /// Note: this is not a robust statistic for non-normal distributions. Prefer the + /// `median_abs_dev` for unknown distributions. + /// + /// See: + fn std_dev(&self) -> f64; + + /// Standard deviation as a percent of the mean value. See `std_dev` and `mean`. + /// + /// Note: this is not a robust statistic for non-normal distributions. Prefer the + /// `median_abs_dev_pct` for unknown distributions. + fn std_dev_pct(&self) -> f64; + + /// Scaled median of the absolute deviations of each sample from the sample median. This is a + /// robust (distribution-agnostic) estimator of sample variability. Use this in preference to + /// `std_dev` if you cannot assume your sample is normally distributed. Note that this is scaled + /// by the constant `1.4826` to allow its use as a consistent estimator for the standard + /// deviation. + /// + /// See: + fn median_abs_dev(&self) -> f64; + + /// Median absolute deviation as a percent of the median. See `median_abs_dev` and `median`. + fn median_abs_dev_pct(&self) -> f64; + + /// Percentile: the value below which `pct` percent of the values in `self` fall. For example, + /// percentile(95.0) will return the value `v` such that 95% of the samples `s` in `self` + /// satisfy `s <= v`. + /// + /// Calculated by linear interpolation between closest ranks. + /// + /// See: + fn percentile(&self, pct: f64) -> f64; + + /// Quartiles of the sample: three values that divide the sample into four equal groups, each + /// with 1/4 of the data. The middle value is the median. See `median` and `percentile`. This + /// function may calculate the 3 quartiles more efficiently than 3 calls to `percentile`, but + /// is otherwise equivalent. + /// + /// See also: + fn quartiles(&self) -> (f64, f64, f64); + + /// Inter-quartile range: the difference between the 25th percentile (1st quartile) and the 75th + /// percentile (3rd quartile). See `quartiles`. + /// + /// See also: + fn iqr(&self) -> f64; +} + +/// Extracted collection of all the summary statistics of a sample set. +#[derive(Debug, Clone, PartialEq, Copy)] +#[allow(missing_docs)] +pub struct Summary { + pub sum: f64, + pub min: f64, + pub max: f64, + pub mean: f64, + pub median: f64, + pub var: f64, + pub std_dev: f64, + pub std_dev_pct: f64, + pub median_abs_dev: f64, + pub median_abs_dev_pct: f64, + pub quartiles: (f64, f64, f64), + pub iqr: f64, +} + +impl Summary { + /// Construct a new summary of a sample set. + pub fn new(samples: &[f64]) -> Summary { + Summary { + sum: samples.sum(), + min: samples.min(), + max: samples.max(), + mean: samples.mean(), + median: samples.median(), + var: samples.var(), + std_dev: samples.std_dev(), + std_dev_pct: samples.std_dev_pct(), + median_abs_dev: samples.median_abs_dev(), + median_abs_dev_pct: samples.median_abs_dev_pct(), + quartiles: samples.quartiles(), + iqr: samples.iqr(), + } + } +} + +impl Stats for [f64] { + // FIXME #11059 handle NaN, inf and overflow + fn sum(&self) -> f64 { + let mut partials = vec![]; + + for &x in self { + let mut x = x; + let mut j = 0; + // This inner loop applies `hi`/`lo` summation to each + // partial so that the list of partial sums remains exact. + for i in 0..partials.len() { + let mut y: f64 = partials[i]; + if x.abs() < y.abs() { + mem::swap(&mut x, &mut y); + } + // Rounded `x+y` is stored in `hi` with round-off stored in + // `lo`. Together `hi+lo` are exactly equal to `x+y`. + let hi = x + y; + let lo = y - (hi - x); + if lo != 0.0 { + partials[j] = lo; + j += 1; + } + x = hi; + } + if j >= partials.len() { + partials.push(x); + } else { + partials[j] = x; + partials.truncate(j + 1); + } + } + let zero: f64 = 0.0; + partials.iter().fold(zero, |p, q| p + *q) + } + + fn min(&self) -> f64 { + assert!(!self.is_empty()); + self.iter().fold(self[0], |p, q| p.min(*q)) + } + + fn max(&self) -> f64 { + assert!(!self.is_empty()); + self.iter().fold(self[0], |p, q| p.max(*q)) + } + + fn mean(&self) -> f64 { + assert!(!self.is_empty()); + self.sum() / (self.len() as f64) + } + + fn median(&self) -> f64 { + self.percentile(50_f64) + } + + fn var(&self) -> f64 { + if self.len() < 2 { + 0.0 + } else { + let mean = self.mean(); + let mut v: f64 = 0.0; + for s in self { + let x = *s - mean; + v += x * x; + } + // N.B., this is _supposed to be_ len-1, not len. If you + // change it back to len, you will be calculating a + // population variance, not a sample variance. + let denom = (self.len() - 1) as f64; + v / denom + } + } + + fn std_dev(&self) -> f64 { + self.var().sqrt() + } + + fn std_dev_pct(&self) -> f64 { + let hundred = 100_f64; + (self.std_dev() / self.mean()) * hundred + } + + fn median_abs_dev(&self) -> f64 { + let med = self.median(); + let abs_devs: Vec = self.iter().map(|&v| (med - v).abs()).collect(); + // This constant is derived by smarter statistics brains than me, but it is + // consistent with how R and other packages treat the MAD. + let number = 1.4826; + abs_devs.median() * number + } + + fn median_abs_dev_pct(&self) -> f64 { + let hundred = 100_f64; + (self.median_abs_dev() / self.median()) * hundred + } + + fn percentile(&self, pct: f64) -> f64 { + let mut tmp = self.to_vec(); + local_sort(&mut tmp); + percentile_of_sorted(&tmp, pct) + } + + fn quartiles(&self) -> (f64, f64, f64) { + let mut tmp = self.to_vec(); + local_sort(&mut tmp); + let first = 25_f64; + let a = percentile_of_sorted(&tmp, first); + let second = 50_f64; + let b = percentile_of_sorted(&tmp, second); + let third = 75_f64; + let c = percentile_of_sorted(&tmp, third); + (a, b, c) + } + + fn iqr(&self) -> f64 { + let (a, _, c) = self.quartiles(); + c - a + } +} + +// Helper function: extract a value representing the `pct` percentile of a sorted sample-set, using +// linear interpolation. If samples are not sorted, return nonsensical value. +fn percentile_of_sorted(sorted_samples: &[f64], pct: f64) -> f64 { + assert!(!sorted_samples.is_empty()); + if sorted_samples.len() == 1 { + return sorted_samples[0]; + } + let zero: f64 = 0.0; + assert!(zero <= pct); + let hundred = 100_f64; + assert!(pct <= hundred); + if pct == hundred { + return sorted_samples[sorted_samples.len() - 1]; + } + let length = (sorted_samples.len() - 1) as f64; + let rank = (pct / hundred) * length; + let lrank = rank.floor(); + let d = rank - lrank; + let n = lrank as usize; + let lo = sorted_samples[n]; + let hi = sorted_samples[n + 1]; + lo + (hi - lo) * d +} + +/// Winsorize a set of samples, replacing values above the `100-pct` percentile +/// and below the `pct` percentile with those percentiles themselves. This is a +/// way of minimizing the effect of outliers, at the cost of biasing the sample. +/// It differs from trimming in that it does not change the number of samples, +/// just changes the values of those that are outliers. +/// +/// See: +pub fn winsorize(samples: &mut [f64], pct: f64) { + let mut tmp = samples.to_vec(); + local_sort(&mut tmp); + let lo = percentile_of_sorted(&tmp, pct); + let hundred = 100_f64; + let hi = percentile_of_sorted(&tmp, hundred - pct); + for samp in samples { + if *samp > hi { + *samp = hi + } else if *samp < lo { + *samp = lo + } + } +} diff --git a/crux-mir/lib/test/src/stats/tests.rs b/crux-mir/lib/test/src/stats/tests.rs new file mode 100644 index 000000000..3a6e8401b --- /dev/null +++ b/crux-mir/lib/test/src/stats/tests.rs @@ -0,0 +1,591 @@ +use super::*; + +extern crate test; +use self::test::test::Bencher; +use std::io; +use std::io::prelude::*; + +// Test vectors generated from R, using the script src/etc/stat-test-vectors.r. + +macro_rules! assert_approx_eq { + ($a: expr, $b: expr) => {{ + let (a, b) = (&$a, &$b); + assert!((*a - *b).abs() < 1.0e-6, "{} is not approximately equal to {}", *a, *b); + }}; +} + +fn check(samples: &[f64], summ: &Summary) { + let summ2 = Summary::new(samples); + + let mut w = io::sink(); + let w = &mut w; + (write!(w, "\n")).unwrap(); + + assert_eq!(summ.sum, summ2.sum); + assert_eq!(summ.min, summ2.min); + assert_eq!(summ.max, summ2.max); + assert_eq!(summ.mean, summ2.mean); + assert_eq!(summ.median, summ2.median); + + // We needed a few more digits to get exact equality on these + // but they're within float epsilon, which is 1.0e-6. + assert_approx_eq!(summ.var, summ2.var); + assert_approx_eq!(summ.std_dev, summ2.std_dev); + assert_approx_eq!(summ.std_dev_pct, summ2.std_dev_pct); + assert_approx_eq!(summ.median_abs_dev, summ2.median_abs_dev); + assert_approx_eq!(summ.median_abs_dev_pct, summ2.median_abs_dev_pct); + + assert_eq!(summ.quartiles, summ2.quartiles); + assert_eq!(summ.iqr, summ2.iqr); +} + +#[test] +fn test_min_max_nan() { + let xs = &[1.0, 2.0, f64::NAN, 3.0, 4.0]; + let summary = Summary::new(xs); + assert_eq!(summary.min, 1.0); + assert_eq!(summary.max, 4.0); +} + +#[test] +fn test_norm2() { + let val = &[958.0000000000, 924.0000000000]; + let summ = &Summary { + sum: 1882.0000000000, + min: 924.0000000000, + max: 958.0000000000, + mean: 941.0000000000, + median: 941.0000000000, + var: 578.0000000000, + std_dev: 24.0416305603, + std_dev_pct: 2.5549022912, + median_abs_dev: 25.2042000000, + median_abs_dev_pct: 2.6784484591, + quartiles: (932.5000000000, 941.0000000000, 949.5000000000), + iqr: 17.0000000000, + }; + check(val, summ); +} +#[test] +fn test_norm10narrow() { + let val = &[ + 966.0000000000, + 985.0000000000, + 1110.0000000000, + 848.0000000000, + 821.0000000000, + 975.0000000000, + 962.0000000000, + 1157.0000000000, + 1217.0000000000, + 955.0000000000, + ]; + let summ = &Summary { + sum: 9996.0000000000, + min: 821.0000000000, + max: 1217.0000000000, + mean: 999.6000000000, + median: 970.5000000000, + var: 16050.7111111111, + std_dev: 126.6914010938, + std_dev_pct: 12.6742097933, + median_abs_dev: 102.2994000000, + median_abs_dev_pct: 10.5408964451, + quartiles: (956.7500000000, 970.5000000000, 1078.7500000000), + iqr: 122.0000000000, + }; + check(val, summ); +} +#[test] +fn test_norm10medium() { + let val = &[ + 954.0000000000, + 1064.0000000000, + 855.0000000000, + 1000.0000000000, + 743.0000000000, + 1084.0000000000, + 704.0000000000, + 1023.0000000000, + 357.0000000000, + 869.0000000000, + ]; + let summ = &Summary { + sum: 8653.0000000000, + min: 357.0000000000, + max: 1084.0000000000, + mean: 865.3000000000, + median: 911.5000000000, + var: 48628.4555555556, + std_dev: 220.5186059170, + std_dev_pct: 25.4846418487, + median_abs_dev: 195.7032000000, + median_abs_dev_pct: 21.4704552935, + quartiles: (771.0000000000, 911.5000000000, 1017.2500000000), + iqr: 246.2500000000, + }; + check(val, summ); +} +#[test] +fn test_norm10wide() { + let val = &[ + 505.0000000000, + 497.0000000000, + 1591.0000000000, + 887.0000000000, + 1026.0000000000, + 136.0000000000, + 1580.0000000000, + 940.0000000000, + 754.0000000000, + 1433.0000000000, + ]; + let summ = &Summary { + sum: 9349.0000000000, + min: 136.0000000000, + max: 1591.0000000000, + mean: 934.9000000000, + median: 913.5000000000, + var: 239208.9888888889, + std_dev: 489.0899599142, + std_dev_pct: 52.3146817750, + median_abs_dev: 611.5725000000, + median_abs_dev_pct: 66.9482758621, + quartiles: (567.2500000000, 913.5000000000, 1331.2500000000), + iqr: 764.0000000000, + }; + check(val, summ); +} +#[test] +fn test_norm25verynarrow() { + let val = &[ + 991.0000000000, + 1018.0000000000, + 998.0000000000, + 1013.0000000000, + 974.0000000000, + 1007.0000000000, + 1014.0000000000, + 999.0000000000, + 1011.0000000000, + 978.0000000000, + 985.0000000000, + 999.0000000000, + 983.0000000000, + 982.0000000000, + 1015.0000000000, + 1002.0000000000, + 977.0000000000, + 948.0000000000, + 1040.0000000000, + 974.0000000000, + 996.0000000000, + 989.0000000000, + 1015.0000000000, + 994.0000000000, + 1024.0000000000, + ]; + let summ = &Summary { + sum: 24926.0000000000, + min: 948.0000000000, + max: 1040.0000000000, + mean: 997.0400000000, + median: 998.0000000000, + var: 393.2066666667, + std_dev: 19.8294393937, + std_dev_pct: 1.9888308788, + median_abs_dev: 22.2390000000, + median_abs_dev_pct: 2.2283567134, + quartiles: (983.0000000000, 998.0000000000, 1013.0000000000), + iqr: 30.0000000000, + }; + check(val, summ); +} +#[test] +fn test_exp10a() { + let val = &[ + 23.0000000000, + 11.0000000000, + 2.0000000000, + 57.0000000000, + 4.0000000000, + 12.0000000000, + 5.0000000000, + 29.0000000000, + 3.0000000000, + 21.0000000000, + ]; + let summ = &Summary { + sum: 167.0000000000, + min: 2.0000000000, + max: 57.0000000000, + mean: 16.7000000000, + median: 11.5000000000, + var: 287.7888888889, + std_dev: 16.9643416875, + std_dev_pct: 101.5828843560, + median_abs_dev: 13.3434000000, + median_abs_dev_pct: 116.0295652174, + quartiles: (4.2500000000, 11.5000000000, 22.5000000000), + iqr: 18.2500000000, + }; + check(val, summ); +} +#[test] +fn test_exp10b() { + let val = &[ + 24.0000000000, + 17.0000000000, + 6.0000000000, + 38.0000000000, + 25.0000000000, + 7.0000000000, + 51.0000000000, + 2.0000000000, + 61.0000000000, + 32.0000000000, + ]; + let summ = &Summary { + sum: 263.0000000000, + min: 2.0000000000, + max: 61.0000000000, + mean: 26.3000000000, + median: 24.5000000000, + var: 383.5666666667, + std_dev: 19.5848580967, + std_dev_pct: 74.4671410520, + median_abs_dev: 22.9803000000, + median_abs_dev_pct: 93.7971428571, + quartiles: (9.5000000000, 24.5000000000, 36.5000000000), + iqr: 27.0000000000, + }; + check(val, summ); +} +#[test] +fn test_exp10c() { + let val = &[ + 71.0000000000, + 2.0000000000, + 32.0000000000, + 1.0000000000, + 6.0000000000, + 28.0000000000, + 13.0000000000, + 37.0000000000, + 16.0000000000, + 36.0000000000, + ]; + let summ = &Summary { + sum: 242.0000000000, + min: 1.0000000000, + max: 71.0000000000, + mean: 24.2000000000, + median: 22.0000000000, + var: 458.1777777778, + std_dev: 21.4050876611, + std_dev_pct: 88.4507754589, + median_abs_dev: 21.4977000000, + median_abs_dev_pct: 97.7168181818, + quartiles: (7.7500000000, 22.0000000000, 35.0000000000), + iqr: 27.2500000000, + }; + check(val, summ); +} +#[test] +fn test_exp25() { + let val = &[ + 3.0000000000, + 24.0000000000, + 1.0000000000, + 19.0000000000, + 7.0000000000, + 5.0000000000, + 30.0000000000, + 39.0000000000, + 31.0000000000, + 13.0000000000, + 25.0000000000, + 48.0000000000, + 1.0000000000, + 6.0000000000, + 42.0000000000, + 63.0000000000, + 2.0000000000, + 12.0000000000, + 108.0000000000, + 26.0000000000, + 1.0000000000, + 7.0000000000, + 44.0000000000, + 25.0000000000, + 11.0000000000, + ]; + let summ = &Summary { + sum: 593.0000000000, + min: 1.0000000000, + max: 108.0000000000, + mean: 23.7200000000, + median: 19.0000000000, + var: 601.0433333333, + std_dev: 24.5161851301, + std_dev_pct: 103.3565983562, + median_abs_dev: 19.2738000000, + median_abs_dev_pct: 101.4410526316, + quartiles: (6.0000000000, 19.0000000000, 31.0000000000), + iqr: 25.0000000000, + }; + check(val, summ); +} +#[test] +fn test_binom25() { + let val = &[ + 18.0000000000, + 17.0000000000, + 27.0000000000, + 15.0000000000, + 21.0000000000, + 25.0000000000, + 17.0000000000, + 24.0000000000, + 25.0000000000, + 24.0000000000, + 26.0000000000, + 26.0000000000, + 23.0000000000, + 15.0000000000, + 23.0000000000, + 17.0000000000, + 18.0000000000, + 18.0000000000, + 21.0000000000, + 16.0000000000, + 15.0000000000, + 31.0000000000, + 20.0000000000, + 17.0000000000, + 15.0000000000, + ]; + let summ = &Summary { + sum: 514.0000000000, + min: 15.0000000000, + max: 31.0000000000, + mean: 20.5600000000, + median: 20.0000000000, + var: 20.8400000000, + std_dev: 4.5650848842, + std_dev_pct: 22.2037202539, + median_abs_dev: 5.9304000000, + median_abs_dev_pct: 29.6520000000, + quartiles: (17.0000000000, 20.0000000000, 24.0000000000), + iqr: 7.0000000000, + }; + check(val, summ); +} +#[test] +fn test_pois25lambda30() { + let val = &[ + 27.0000000000, + 33.0000000000, + 34.0000000000, + 34.0000000000, + 24.0000000000, + 39.0000000000, + 28.0000000000, + 27.0000000000, + 31.0000000000, + 28.0000000000, + 38.0000000000, + 21.0000000000, + 33.0000000000, + 36.0000000000, + 29.0000000000, + 37.0000000000, + 32.0000000000, + 34.0000000000, + 31.0000000000, + 39.0000000000, + 25.0000000000, + 31.0000000000, + 32.0000000000, + 40.0000000000, + 24.0000000000, + ]; + let summ = &Summary { + sum: 787.0000000000, + min: 21.0000000000, + max: 40.0000000000, + mean: 31.4800000000, + median: 32.0000000000, + var: 26.5933333333, + std_dev: 5.1568724372, + std_dev_pct: 16.3814245145, + median_abs_dev: 5.9304000000, + median_abs_dev_pct: 18.5325000000, + quartiles: (28.0000000000, 32.0000000000, 34.0000000000), + iqr: 6.0000000000, + }; + check(val, summ); +} +#[test] +fn test_pois25lambda40() { + let val = &[ + 42.0000000000, + 50.0000000000, + 42.0000000000, + 46.0000000000, + 34.0000000000, + 45.0000000000, + 34.0000000000, + 49.0000000000, + 39.0000000000, + 28.0000000000, + 40.0000000000, + 35.0000000000, + 37.0000000000, + 39.0000000000, + 46.0000000000, + 44.0000000000, + 32.0000000000, + 45.0000000000, + 42.0000000000, + 37.0000000000, + 48.0000000000, + 42.0000000000, + 33.0000000000, + 42.0000000000, + 48.0000000000, + ]; + let summ = &Summary { + sum: 1019.0000000000, + min: 28.0000000000, + max: 50.0000000000, + mean: 40.7600000000, + median: 42.0000000000, + var: 34.4400000000, + std_dev: 5.8685603004, + std_dev_pct: 14.3978417577, + median_abs_dev: 5.9304000000, + median_abs_dev_pct: 14.1200000000, + quartiles: (37.0000000000, 42.0000000000, 45.0000000000), + iqr: 8.0000000000, + }; + check(val, summ); +} +#[test] +fn test_pois25lambda50() { + let val = &[ + 45.0000000000, + 43.0000000000, + 44.0000000000, + 61.0000000000, + 51.0000000000, + 53.0000000000, + 59.0000000000, + 52.0000000000, + 49.0000000000, + 51.0000000000, + 51.0000000000, + 50.0000000000, + 49.0000000000, + 56.0000000000, + 42.0000000000, + 52.0000000000, + 51.0000000000, + 43.0000000000, + 48.0000000000, + 48.0000000000, + 50.0000000000, + 42.0000000000, + 43.0000000000, + 42.0000000000, + 60.0000000000, + ]; + let summ = &Summary { + sum: 1235.0000000000, + min: 42.0000000000, + max: 61.0000000000, + mean: 49.4000000000, + median: 50.0000000000, + var: 31.6666666667, + std_dev: 5.6273143387, + std_dev_pct: 11.3913245723, + median_abs_dev: 4.4478000000, + median_abs_dev_pct: 8.8956000000, + quartiles: (44.0000000000, 50.0000000000, 52.0000000000), + iqr: 8.0000000000, + }; + check(val, summ); +} +#[test] +fn test_unif25() { + let val = &[ + 99.0000000000, + 55.0000000000, + 92.0000000000, + 79.0000000000, + 14.0000000000, + 2.0000000000, + 33.0000000000, + 49.0000000000, + 3.0000000000, + 32.0000000000, + 84.0000000000, + 59.0000000000, + 22.0000000000, + 86.0000000000, + 76.0000000000, + 31.0000000000, + 29.0000000000, + 11.0000000000, + 41.0000000000, + 53.0000000000, + 45.0000000000, + 44.0000000000, + 98.0000000000, + 98.0000000000, + 7.0000000000, + ]; + let summ = &Summary { + sum: 1242.0000000000, + min: 2.0000000000, + max: 99.0000000000, + mean: 49.6800000000, + median: 45.0000000000, + var: 1015.6433333333, + std_dev: 31.8691595957, + std_dev_pct: 64.1488719719, + median_abs_dev: 45.9606000000, + median_abs_dev_pct: 102.1346666667, + quartiles: (29.0000000000, 45.0000000000, 79.0000000000), + iqr: 50.0000000000, + }; + check(val, summ); +} + +#[test] +fn test_sum_f64s() { + assert_eq!([0.5f64, 3.2321f64, 1.5678f64].sum(), 5.2999); +} +#[test] +fn test_sum_f64_between_ints_that_sum_to_0() { + assert_eq!([1e30f64, 1.2f64, -1e30f64].sum(), 1.2); +} + +#[bench] +pub fn sum_three_items(b: &mut Bencher) { + b.iter(|| { + [1e20f64, 1.5f64, -1e20f64].sum(); + }) +} +#[bench] +pub fn sum_many_f64(b: &mut Bencher) { + let nums = [-1e30f64, 1e60, 1e30, 1.0, -1e60]; + let v = (0..500).map(|i| nums[i % 5]).collect::>(); + + b.iter(|| { + v.sum(); + }) +} + +#[bench] +pub fn no_iter(_: &mut Bencher) {} diff --git a/crux-mir/lib/test/src/term.rs b/crux-mir/lib/test/src/term.rs new file mode 100644 index 000000000..a14b0d4f5 --- /dev/null +++ b/crux-mir/lib/test/src/term.rs @@ -0,0 +1,85 @@ +//! Terminal formatting module. +//! +//! This module provides the `Terminal` trait, which abstracts over an [ANSI +//! Terminal][ansi] to provide color printing, among other things. There are two +//! implementations, the `TerminfoTerminal`, which uses control characters from +//! a [terminfo][ti] database, and `WinConsole`, which uses the [Win32 Console +//! API][win]. +//! +//! [ansi]: https://en.wikipedia.org/wiki/ANSI_escape_code +//! [win]: https://docs.microsoft.com/en-us/windows/console/character-mode-applications +//! [ti]: https://en.wikipedia.org/wiki/Terminfo + +#![deny(missing_docs)] + +use std::io::{self, prelude::*}; + +pub(crate) use terminfo::TerminfoTerminal; +#[cfg(windows)] +pub(crate) use win::WinConsole; + +pub(crate) mod terminfo; + +#[cfg(windows)] +mod win; + +/// Alias for stdout terminals. +pub(crate) type StdoutTerminal = dyn Terminal + Send; + +#[cfg(not(windows))] +/// Returns a Terminal wrapping stdout, or None if a terminal couldn't be +/// opened. +pub(crate) fn stdout() -> Option> { + TerminfoTerminal::new(io::stdout()).map(|t| Box::new(t) as Box) +} + +#[cfg(windows)] +/// Returns a Terminal wrapping stdout, or None if a terminal couldn't be +/// opened. +pub(crate) fn stdout() -> Option> { + TerminfoTerminal::new(io::stdout()) + .map(|t| Box::new(t) as Box) + .or_else(|| Some(Box::new(WinConsole::new(io::stdout())) as Box)) +} + +/// Terminal color definitions +#[allow(missing_docs)] +#[cfg_attr(not(windows), allow(dead_code))] +pub(crate) mod color { + /// Number for a terminal color + pub(crate) type Color = u32; + + pub(crate) const BLACK: Color = 0; + pub(crate) const RED: Color = 1; + pub(crate) const GREEN: Color = 2; + pub(crate) const YELLOW: Color = 3; + pub(crate) const BLUE: Color = 4; + pub(crate) const MAGENTA: Color = 5; + pub(crate) const CYAN: Color = 6; + pub(crate) const WHITE: Color = 7; +} + +/// A terminal with similar capabilities to an ANSI Terminal +/// (foreground/background colors etc). +pub trait Terminal: Write { + /// Sets the foreground color to the given color. + /// + /// If the color is a bright color, but the terminal only supports 8 colors, + /// the corresponding normal color will be used instead. + /// + /// Returns `Ok(true)` if the color was set, `Ok(false)` otherwise, and `Err(e)` + /// if there was an I/O error. + fn fg(&mut self, color: color::Color) -> io::Result; + + /// Resets all terminal attributes and colors to their defaults. + /// + /// Returns `Ok(true)` if the terminal was reset, `Ok(false)` otherwise, and `Err(e)` if there + /// was an I/O error. + /// + /// *Note: This does not flush.* + /// + /// That means the reset command may get buffered so, if you aren't planning on doing anything + /// else that might flush stdout's buffer (e.g., writing a line of text), you should flush after + /// calling reset. + fn reset(&mut self) -> io::Result; +} diff --git a/crux-mir/lib/test/src/term/terminfo/mod.rs b/crux-mir/lib/test/src/term/terminfo/mod.rs new file mode 100644 index 000000000..67ba89410 --- /dev/null +++ b/crux-mir/lib/test/src/term/terminfo/mod.rs @@ -0,0 +1,205 @@ +//! Terminfo database interface. + +use std::collections::HashMap; +use std::env; +use std::error; +use std::fmt; +use std::fs::File; +use std::io::{self, prelude::*, BufReader}; +use std::path::Path; + +use super::color; +use super::Terminal; + +use parm::{expand, Param, Variables}; +use parser::compiled::{msys_terminfo, parse}; +use searcher::get_dbpath_for_term; + +/// A parsed terminfo database entry. +#[allow(unused)] +#[derive(Debug)] +pub(crate) struct TermInfo { + /// Names for the terminal + pub(crate) names: Vec, + /// Map of capability name to boolean value + pub(crate) bools: HashMap, + /// Map of capability name to numeric value + pub(crate) numbers: HashMap, + /// Map of capability name to raw (unexpanded) string + pub(crate) strings: HashMap>, +} + +/// A terminfo creation error. +#[derive(Debug)] +pub(crate) enum Error { + /// TermUnset Indicates that the environment doesn't include enough information to find + /// the terminfo entry. + TermUnset, + /// MalformedTerminfo indicates that parsing the terminfo entry failed. + MalformedTerminfo(String), + /// io::Error forwards any io::Errors encountered when finding or reading the terminfo entry. + IoError(io::Error), +} + +impl error::Error for Error { + fn source(&self) -> Option<&(dyn error::Error + 'static)> { + use Error::*; + match self { + IoError(e) => Some(e), + _ => None, + } + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use Error::*; + match *self { + TermUnset => Ok(()), + MalformedTerminfo(ref e) => e.fmt(f), + IoError(ref e) => e.fmt(f), + } + } +} + +impl TermInfo { + /// Creates a TermInfo based on current environment. + pub(crate) fn from_env() -> Result { + let term = match env::var("TERM") { + Ok(name) => TermInfo::from_name(&name), + Err(..) => return Err(Error::TermUnset), + }; + + if term.is_err() && env::var("MSYSCON").map_or(false, |s| "mintty.exe" == s) { + // msys terminal + Ok(msys_terminfo()) + } else { + term + } + } + + /// Creates a TermInfo for the named terminal. + pub(crate) fn from_name(name: &str) -> Result { + if cfg!(miri) { + // Avoid all the work of parsing the terminfo (it's pretty slow under Miri), and just + // assume that the standard color codes work (like e.g. the 'colored' crate). + return Ok(TermInfo { + names: Default::default(), + bools: Default::default(), + numbers: Default::default(), + strings: Default::default(), + }); + } + + get_dbpath_for_term(name) + .ok_or_else(|| { + Error::IoError(io::Error::new(io::ErrorKind::NotFound, "terminfo file not found")) + }) + .and_then(|p| TermInfo::from_path(&(*p))) + } + + /// Parse the given TermInfo. + pub(crate) fn from_path>(path: P) -> Result { + Self::_from_path(path.as_ref()) + } + // Keep the metadata small + fn _from_path(path: &Path) -> Result { + let file = File::open(path).map_err(Error::IoError)?; + let mut reader = BufReader::new(file); + parse(&mut reader, false).map_err(Error::MalformedTerminfo) + } +} + +pub(crate) mod searcher; + +/// TermInfo format parsing. +pub(crate) mod parser { + //! ncurses-compatible compiled terminfo format parsing (term(5)) + pub(crate) mod compiled; +} +pub(crate) mod parm; + +/// A Terminal that knows how many colors it supports, with a reference to its +/// parsed Terminfo database record. +pub(crate) struct TerminfoTerminal { + num_colors: u32, + out: T, + ti: TermInfo, +} + +impl Terminal for TerminfoTerminal { + fn fg(&mut self, color: color::Color) -> io::Result { + let color = self.dim_if_necessary(color); + if cfg!(miri) && color < 8 { + // The Miri logic for this only works for the most basic 8 colors, which we just assume + // the terminal will support. (`num_colors` is always 0 in Miri, so higher colors will + // just fail. But libtest doesn't use any higher colors anyway.) + return write!(self.out, "\x1B[3{color}m").and(Ok(true)); + } + if self.num_colors > color { + return self.apply_cap("setaf", &[Param::Number(color as i32)]); + } + Ok(false) + } + + fn reset(&mut self) -> io::Result { + if cfg!(miri) { + return write!(self.out, "\x1B[0m").and(Ok(true)); + } + // are there any terminals that have color/attrs and not sgr0? + // Try falling back to sgr, then op + let cmd = match ["sgr0", "sgr", "op"].iter().find_map(|cap| self.ti.strings.get(*cap)) { + Some(op) => match expand(op, &[], &mut Variables::new()) { + Ok(cmd) => cmd, + Err(e) => return Err(io::Error::new(io::ErrorKind::InvalidData, e)), + }, + None => return Ok(false), + }; + self.out.write_all(&cmd).and(Ok(true)) + } +} + +impl TerminfoTerminal { + /// Creates a new TerminfoTerminal with the given TermInfo and Write. + pub(crate) fn new_with_terminfo(out: T, terminfo: TermInfo) -> TerminfoTerminal { + let nc = if terminfo.strings.contains_key("setaf") && terminfo.strings.contains_key("setab") + { + terminfo.numbers.get("colors").map_or(0, |&n| n) + } else { + 0 + }; + + TerminfoTerminal { out, ti: terminfo, num_colors: nc } + } + + /// Creates a new TerminfoTerminal for the current environment with the given Write. + /// + /// Returns `None` when the terminfo cannot be found or parsed. + pub(crate) fn new(out: T) -> Option> { + TermInfo::from_env().map(move |ti| TerminfoTerminal::new_with_terminfo(out, ti)).ok() + } + + fn dim_if_necessary(&self, color: color::Color) -> color::Color { + if color >= self.num_colors && (8..16).contains(&color) { color - 8 } else { color } + } + + fn apply_cap(&mut self, cmd: &str, params: &[Param]) -> io::Result { + match self.ti.strings.get(cmd) { + Some(cmd) => match expand(cmd, params, &mut Variables::new()) { + Ok(s) => self.out.write_all(&s).and(Ok(true)), + Err(e) => Err(io::Error::new(io::ErrorKind::InvalidData, e)), + }, + None => Ok(false), + } + } +} + +impl Write for TerminfoTerminal { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.out.write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.out.flush() + } +} diff --git a/crux-mir/lib/test/src/term/terminfo/parm.rs b/crux-mir/lib/test/src/term/terminfo/parm.rs new file mode 100644 index 000000000..2815f6cfc --- /dev/null +++ b/crux-mir/lib/test/src/term/terminfo/parm.rs @@ -0,0 +1,532 @@ +//! Parameterized string expansion + +use self::Param::*; +use self::States::*; + +use std::iter::repeat; + +#[cfg(test)] +mod tests; + +#[derive(Clone, Copy, PartialEq)] +enum States { + Nothing, + Percent, + SetVar, + GetVar, + PushParam, + CharConstant, + CharClose, + IntConstant(i32), + FormatPattern(Flags, FormatState), + SeekIfElse(usize), + SeekIfElsePercent(usize), + SeekIfEnd(usize), + SeekIfEndPercent(usize), +} + +#[derive(Copy, PartialEq, Clone)] +enum FormatState { + Flags, + Width, + Precision, +} + +/// Types of parameters a capability can use +#[allow(missing_docs)] +#[derive(Clone)] +pub(crate) enum Param { + Number(i32), +} + +/// Container for static and dynamic variable arrays +pub(crate) struct Variables { + /// Static variables A-Z + sta_va: [Param; 26], + /// Dynamic variables a-z + dyn_va: [Param; 26], +} + +impl Variables { + /// Returns a new zero-initialized Variables + pub(crate) fn new() -> Variables { + Variables { + sta_va: [ + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + ], + dyn_va: [ + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + ], + } + } +} + +/// Expand a parameterized capability +/// +/// # Arguments +/// * `cap` - string to expand +/// * `params` - vector of params for %p1 etc +/// * `vars` - Variables struct for %Pa etc +/// +/// To be compatible with ncurses, `vars` should be the same between calls to `expand` for +/// multiple capabilities for the same terminal. +pub(crate) fn expand( + cap: &[u8], + params: &[Param], + vars: &mut Variables, +) -> Result, String> { + let mut state = Nothing; + + // expanded cap will only rarely be larger than the cap itself + let mut output = Vec::with_capacity(cap.len()); + + let mut stack: Vec = Vec::new(); + + // Copy parameters into a local vector for mutability + let mut mparams = [ + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + Number(0), + ]; + for (dst, src) in mparams.iter_mut().zip(params.iter()) { + *dst = (*src).clone(); + } + + for &c in cap.iter() { + let cur = c as char; + let mut old_state = state; + match state { + Nothing => { + if cur == '%' { + state = Percent; + } else { + output.push(c); + } + } + Percent => { + match cur { + '%' => { + output.push(c); + state = Nothing + } + 'c' => { + match stack.pop() { + // if c is 0, use 0200 (128) for ncurses compatibility + Some(Number(0)) => output.push(128u8), + // Don't check bounds. ncurses just casts and truncates. + Some(Number(c)) => output.push(c as u8), + None => return Err("stack is empty".to_string()), + } + } + 'p' => state = PushParam, + 'P' => state = SetVar, + 'g' => state = GetVar, + '\'' => state = CharConstant, + '{' => state = IntConstant(0), + 'l' => match stack.pop() { + Some(_) => return Err("a non-str was used with %l".to_string()), + None => return Err("stack is empty".to_string()), + }, + '+' | '-' | '/' | '*' | '^' | '&' | '|' | 'm' => { + match (stack.pop(), stack.pop()) { + (Some(Number(y)), Some(Number(x))) => stack.push(Number(match cur { + '+' => x + y, + '-' => x - y, + '*' => x * y, + '/' => x / y, + '|' => x | y, + '&' => x & y, + '^' => x ^ y, + 'm' => x % y, + _ => unreachable!("All cases handled"), + })), + _ => return Err("stack is empty".to_string()), + } + } + '=' | '>' | '<' | 'A' | 'O' => match (stack.pop(), stack.pop()) { + (Some(Number(y)), Some(Number(x))) => stack.push(Number( + if match cur { + '=' => x == y, + '<' => x < y, + '>' => x > y, + 'A' => x > 0 && y > 0, + 'O' => x > 0 || y > 0, + _ => unreachable!(), + } { + 1 + } else { + 0 + }, + )), + _ => return Err("stack is empty".to_string()), + }, + '!' | '~' => match stack.pop() { + Some(Number(x)) => stack.push(Number(match cur { + '!' if x > 0 => 0, + '!' => 1, + '~' => !x, + _ => unreachable!(), + })), + None => return Err("stack is empty".to_string()), + }, + 'i' => match (&mparams[0], &mparams[1]) { + (&Number(x), &Number(y)) => { + mparams[0] = Number(x + 1); + mparams[1] = Number(y + 1); + } + }, + + // printf-style support for %doxXs + 'd' | 'o' | 'x' | 'X' | 's' => { + if let Some(arg) = stack.pop() { + let flags = Flags::new(); + let res = format(arg, FormatOp::from_char(cur), flags)?; + output.extend(res.iter().cloned()); + } else { + return Err("stack is empty".to_string()); + } + } + ':' | '#' | ' ' | '.' | '0'..='9' => { + let mut flags = Flags::new(); + let mut fstate = FormatState::Flags; + match cur { + ':' => (), + '#' => flags.alternate = true, + ' ' => flags.space = true, + '.' => fstate = FormatState::Precision, + '0'..='9' => { + flags.width = cur as usize - '0' as usize; + fstate = FormatState::Width; + } + _ => unreachable!(), + } + state = FormatPattern(flags, fstate); + } + + // conditionals + '?' => (), + 't' => match stack.pop() { + Some(Number(0)) => state = SeekIfElse(0), + Some(Number(_)) => (), + None => return Err("stack is empty".to_string()), + }, + 'e' => state = SeekIfEnd(0), + ';' => (), + _ => return Err(format!("unrecognized format option {cur}")), + } + } + PushParam => { + // params are 1-indexed + stack.push( + mparams[match cur.to_digit(10) { + Some(d) => d as usize - 1, + None => return Err("bad param number".to_string()), + }] + .clone(), + ); + } + SetVar => { + if cur.is_ascii_uppercase() { + if let Some(arg) = stack.pop() { + let idx = (cur as u8) - b'A'; + vars.sta_va[idx as usize] = arg; + } else { + return Err("stack is empty".to_string()); + } + } else if cur.is_ascii_lowercase() { + if let Some(arg) = stack.pop() { + let idx = (cur as u8) - b'a'; + vars.dyn_va[idx as usize] = arg; + } else { + return Err("stack is empty".to_string()); + } + } else { + return Err("bad variable name in %P".to_string()); + } + } + GetVar => { + if cur.is_ascii_uppercase() { + let idx = (cur as u8) - b'A'; + stack.push(vars.sta_va[idx as usize].clone()); + } else if cur.is_ascii_lowercase() { + let idx = (cur as u8) - b'a'; + stack.push(vars.dyn_va[idx as usize].clone()); + } else { + return Err("bad variable name in %g".to_string()); + } + } + CharConstant => { + stack.push(Number(c as i32)); + state = CharClose; + } + CharClose => { + if cur != '\'' { + return Err("malformed character constant".to_string()); + } + } + IntConstant(i) => { + if cur == '}' { + stack.push(Number(i)); + state = Nothing; + } else if let Some(digit) = cur.to_digit(10) { + match i.checked_mul(10).and_then(|i_ten| i_ten.checked_add(digit as i32)) { + Some(i) => { + state = IntConstant(i); + old_state = Nothing; + } + None => return Err("int constant too large".to_string()), + } + } else { + return Err("bad int constant".to_string()); + } + } + FormatPattern(ref mut flags, ref mut fstate) => { + old_state = Nothing; + match (*fstate, cur) { + (_, 'd') | (_, 'o') | (_, 'x') | (_, 'X') | (_, 's') => { + if let Some(arg) = stack.pop() { + let res = format(arg, FormatOp::from_char(cur), *flags)?; + output.extend(res.iter().cloned()); + // will cause state to go to Nothing + old_state = FormatPattern(*flags, *fstate); + } else { + return Err("stack is empty".to_string()); + } + } + (FormatState::Flags, '#') => { + flags.alternate = true; + } + (FormatState::Flags, '-') => { + flags.left = true; + } + (FormatState::Flags, '+') => { + flags.sign = true; + } + (FormatState::Flags, ' ') => { + flags.space = true; + } + (FormatState::Flags, '0'..='9') => { + flags.width = cur as usize - '0' as usize; + *fstate = FormatState::Width; + } + (FormatState::Flags, '.') => { + *fstate = FormatState::Precision; + } + (FormatState::Width, '0'..='9') => { + let old = flags.width; + flags.width = flags.width * 10 + (cur as usize - '0' as usize); + if flags.width < old { + return Err("format width overflow".to_string()); + } + } + (FormatState::Width, '.') => { + *fstate = FormatState::Precision; + } + (FormatState::Precision, '0'..='9') => { + let old = flags.precision; + flags.precision = flags.precision * 10 + (cur as usize - '0' as usize); + if flags.precision < old { + return Err("format precision overflow".to_string()); + } + } + _ => return Err("invalid format specifier".to_string()), + } + } + SeekIfElse(level) => { + if cur == '%' { + state = SeekIfElsePercent(level); + } + old_state = Nothing; + } + SeekIfElsePercent(level) => { + if cur == ';' { + if level == 0 { + state = Nothing; + } else { + state = SeekIfElse(level - 1); + } + } else if cur == 'e' && level == 0 { + state = Nothing; + } else if cur == '?' { + state = SeekIfElse(level + 1); + } else { + state = SeekIfElse(level); + } + } + SeekIfEnd(level) => { + if cur == '%' { + state = SeekIfEndPercent(level); + } + old_state = Nothing; + } + SeekIfEndPercent(level) => { + if cur == ';' { + if level == 0 { + state = Nothing; + } else { + state = SeekIfEnd(level - 1); + } + } else if cur == '?' { + state = SeekIfEnd(level + 1); + } else { + state = SeekIfEnd(level); + } + } + } + if state == old_state { + state = Nothing; + } + } + Ok(output) +} + +#[derive(Copy, PartialEq, Clone)] +struct Flags { + width: usize, + precision: usize, + alternate: bool, + left: bool, + sign: bool, + space: bool, +} + +impl Flags { + fn new() -> Flags { + Flags { width: 0, precision: 0, alternate: false, left: false, sign: false, space: false } + } +} + +#[derive(Copy, Clone)] +enum FormatOp { + Digit, + Octal, + LowerHex, + UpperHex, + String, +} + +impl FormatOp { + fn from_char(c: char) -> FormatOp { + match c { + 'd' => FormatOp::Digit, + 'o' => FormatOp::Octal, + 'x' => FormatOp::LowerHex, + 'X' => FormatOp::UpperHex, + 's' => FormatOp::String, + _ => panic!("bad FormatOp char"), + } + } +} + +fn format(val: Param, op: FormatOp, flags: Flags) -> Result, String> { + let mut s = match val { + Number(d) => { + match op { + FormatOp::Digit => { + if flags.sign { + format!("{:+01$}", d, flags.precision) + } else if d < 0 { + // C doesn't take sign into account in precision calculation. + format!("{:01$}", d, flags.precision + 1) + } else if flags.space { + format!(" {:01$}", d, flags.precision) + } else { + format!("{:01$}", d, flags.precision) + } + } + FormatOp::Octal => { + if flags.alternate { + // Leading octal zero counts against precision. + format!("0{:01$o}", d, flags.precision.saturating_sub(1)) + } else { + format!("{:01$o}", d, flags.precision) + } + } + FormatOp::LowerHex => { + if flags.alternate && d != 0 { + format!("0x{:01$x}", d, flags.precision) + } else { + format!("{:01$x}", d, flags.precision) + } + } + FormatOp::UpperHex => { + if flags.alternate && d != 0 { + format!("0X{:01$X}", d, flags.precision) + } else { + format!("{:01$X}", d, flags.precision) + } + } + FormatOp::String => return Err("non-number on stack with %s".to_string()), + } + .into_bytes() + } + }; + if flags.width > s.len() { + let n = flags.width - s.len(); + if flags.left { + s.extend(repeat(b' ').take(n)); + } else { + let mut s_ = Vec::with_capacity(flags.width); + s_.extend(repeat(b' ').take(n)); + s_.extend(s.into_iter()); + s = s_; + } + } + Ok(s) +} diff --git a/crux-mir/lib/test/src/term/terminfo/parm/tests.rs b/crux-mir/lib/test/src/term/terminfo/parm/tests.rs new file mode 100644 index 000000000..c738f3ba0 --- /dev/null +++ b/crux-mir/lib/test/src/term/terminfo/parm/tests.rs @@ -0,0 +1,124 @@ +use super::*; + +use std::result::Result::Ok; + +#[test] +fn test_basic_setabf() { + let s = b"\\E[48;5;%p1%dm"; + assert_eq!( + expand(s, &[Number(1)], &mut Variables::new()).unwrap(), + "\\E[48;5;1m".bytes().collect::>() + ); +} + +#[test] +fn test_multiple_int_constants() { + assert_eq!( + expand(b"%{1}%{2}%d%d", &[], &mut Variables::new()).unwrap(), + "21".bytes().collect::>() + ); +} + +#[test] +fn test_op_i() { + let mut vars = Variables::new(); + assert_eq!( + expand(b"%p1%d%p2%d%p3%d%i%p1%d%p2%d%p3%d", &[Number(1), Number(2), Number(3)], &mut vars), + Ok("123233".bytes().collect::>()) + ); + assert_eq!( + expand(b"%p1%d%p2%d%i%p1%d%p2%d", &[], &mut vars), + Ok("0011".bytes().collect::>()) + ); +} + +#[test] +fn test_param_stack_failure_conditions() { + let mut varstruct = Variables::new(); + let vars = &mut varstruct; + fn get_res( + fmt: &str, + cap: &str, + params: &[Param], + vars: &mut Variables, + ) -> Result, String> { + let mut u8v: Vec<_> = fmt.bytes().collect(); + u8v.extend(cap.as_bytes().iter().map(|&b| b)); + expand(&u8v, params, vars) + } + + let caps = ["%d", "%c", "%s", "%Pa", "%l", "%!", "%~"]; + for &cap in caps.iter() { + let res = get_res("", cap, &[], vars); + assert!(res.is_err(), "Op {} succeeded incorrectly with 0 stack entries", cap); + if cap == "%s" || cap == "%l" { + continue; + } + let p = Number(97); + let res = get_res("%p1", cap, &[p], vars); + assert!(res.is_ok(), "Op {} failed with 1 stack entry: {}", cap, res.unwrap_err()); + } + let caps = ["%+", "%-", "%*", "%/", "%m", "%&", "%|", "%A", "%O"]; + for &cap in caps.iter() { + let res = expand(cap.as_bytes(), &[], vars); + assert!(res.is_err(), "Binop {} succeeded incorrectly with 0 stack entries", cap); + let res = get_res("%{1}", cap, &[], vars); + assert!(res.is_err(), "Binop {} succeeded incorrectly with 1 stack entry", cap); + let res = get_res("%{1}%{2}", cap, &[], vars); + assert!(res.is_ok(), "Binop {} failed with 2 stack entries: {}", cap, res.unwrap_err()); + } +} + +#[test] +fn test_push_bad_param() { + assert!(expand(b"%pa", &[], &mut Variables::new()).is_err()); +} + +#[test] +fn test_comparison_ops() { + let v = [('<', [1u8, 0u8, 0u8]), ('=', [0u8, 1u8, 0u8]), ('>', [0u8, 0u8, 1u8])]; + for &(op, bs) in v.iter() { + let s = format!("%{{1}}%{{2}}%{op}%d"); + let res = expand(s.as_bytes(), &[], &mut Variables::new()); + assert!(res.is_ok(), "{}", res.unwrap_err()); + assert_eq!(res.unwrap(), vec![b'0' + bs[0]]); + let s = format!("%{{1}}%{{1}}%{op}%d"); + let res = expand(s.as_bytes(), &[], &mut Variables::new()); + assert!(res.is_ok(), "{}", res.unwrap_err()); + assert_eq!(res.unwrap(), vec![b'0' + bs[1]]); + let s = format!("%{{2}}%{{1}}%{op}%d"); + let res = expand(s.as_bytes(), &[], &mut Variables::new()); + assert!(res.is_ok(), "{}", res.unwrap_err()); + assert_eq!(res.unwrap(), vec![b'0' + bs[2]]); + } +} + +#[test] +fn test_conditionals() { + let mut vars = Variables::new(); + let s = b"\\E[%?%p1%{8}%<%t3%p1%d%e%p1%{16}%<%t9%p1%{8}%-%d%e38;5;%p1%d%;m"; + let res = expand(s, &[Number(1)], &mut vars); + assert!(res.is_ok(), "{}", res.unwrap_err()); + assert_eq!(res.unwrap(), "\\E[31m".bytes().collect::>()); + let res = expand(s, &[Number(8)], &mut vars); + assert!(res.is_ok(), "{}", res.unwrap_err()); + assert_eq!(res.unwrap(), "\\E[90m".bytes().collect::>()); + let res = expand(s, &[Number(42)], &mut vars); + assert!(res.is_ok(), "{}", res.unwrap_err()); + assert_eq!(res.unwrap(), "\\E[38;5;42m".bytes().collect::>()); +} + +#[test] +fn test_format() { + let mut varstruct = Variables::new(); + let vars = &mut varstruct; + + assert_eq!( + expand(b"%p1%d%p1%.3d%p1%5d%p1%:+d", &[Number(1)], vars), + Ok("1001 1+1".bytes().collect::>()) + ); + assert_eq!( + expand(b"%p1%o%p1%#o%p2%6.4x%p2%#6.4X", &[Number(15), Number(27)], vars), + Ok("17017 001b0X001B".bytes().collect::>()) + ); +} diff --git a/crux-mir/lib/test/src/term/terminfo/parser/compiled.rs b/crux-mir/lib/test/src/term/terminfo/parser/compiled.rs new file mode 100644 index 000000000..5d40b7988 --- /dev/null +++ b/crux-mir/lib/test/src/term/terminfo/parser/compiled.rs @@ -0,0 +1,336 @@ +#![allow(non_upper_case_globals, missing_docs)] + +//! ncurses-compatible compiled terminfo format parsing (term(5)) + +use super::super::TermInfo; +use std::collections::HashMap; +use std::io; +use std::io::prelude::*; + +#[cfg(test)] +mod tests; + +// These are the orders ncurses uses in its compiled format (as of 5.9). Not sure if portable. + +#[rustfmt::skip] +pub(crate) static boolfnames: &[&str] = &["auto_left_margin", "auto_right_margin", + "no_esc_ctlc", "ceol_standout_glitch", "eat_newline_glitch", "erase_overstrike", "generic_type", + "hard_copy", "has_meta_key", "has_status_line", "insert_null_glitch", "memory_above", + "memory_below", "move_insert_mode", "move_standout_mode", "over_strike", "status_line_esc_ok", + "dest_tabs_magic_smso", "tilde_glitch", "transparent_underline", "xon_xoff", "needs_xon_xoff", + "prtr_silent", "hard_cursor", "non_rev_rmcup", "no_pad_char", "non_dest_scroll_region", + "can_change", "back_color_erase", "hue_lightness_saturation", "col_addr_glitch", + "cr_cancels_micro_mode", "has_print_wheel", "row_addr_glitch", "semi_auto_right_margin", + "cpi_changes_res", "lpi_changes_res", "backspaces_with_bs", "crt_no_scrolling", + "no_correctly_working_cr", "gnu_has_meta_key", "linefeed_is_newline", "has_hardware_tabs", + "return_does_clr_eol"]; + +#[rustfmt::skip] +pub(crate) static boolnames: &[&str] = &["bw", "am", "xsb", "xhp", "xenl", "eo", + "gn", "hc", "km", "hs", "in", "db", "da", "mir", "msgr", "os", "eslok", "xt", "hz", "ul", "xon", + "nxon", "mc5i", "chts", "nrrmc", "npc", "ndscr", "ccc", "bce", "hls", "xhpa", "crxm", "daisy", + "xvpa", "sam", "cpix", "lpix", "OTbs", "OTns", "OTnc", "OTMT", "OTNL", "OTpt", "OTxr"]; + +#[rustfmt::skip] +pub(crate) static numfnames: &[&str] = &[ "columns", "init_tabs", "lines", + "lines_of_memory", "magic_cookie_glitch", "padding_baud_rate", "virtual_terminal", + "width_status_line", "num_labels", "label_height", "label_width", "max_attributes", + "maximum_windows", "max_colors", "max_pairs", "no_color_video", "buffer_capacity", + "dot_vert_spacing", "dot_horz_spacing", "max_micro_address", "max_micro_jump", "micro_col_size", + "micro_line_size", "number_of_pins", "output_res_char", "output_res_line", + "output_res_horz_inch", "output_res_vert_inch", "print_rate", "wide_char_size", "buttons", + "bit_image_entwining", "bit_image_type", "magic_cookie_glitch_ul", "carriage_return_delay", + "new_line_delay", "backspace_delay", "horizontal_tab_delay", "number_of_function_keys"]; + +#[rustfmt::skip] +pub(crate) static numnames: &[&str] = &[ "cols", "it", "lines", "lm", "xmc", "pb", + "vt", "wsl", "nlab", "lh", "lw", "ma", "wnum", "colors", "pairs", "ncv", "bufsz", "spinv", + "spinh", "maddr", "mjump", "mcs", "mls", "npins", "orc", "orl", "orhi", "orvi", "cps", "widcs", + "btns", "bitwin", "bitype", "UTug", "OTdC", "OTdN", "OTdB", "OTdT", "OTkn"]; + +#[rustfmt::skip] +pub(crate) static stringfnames: &[&str] = &[ "back_tab", "bell", "carriage_return", + "change_scroll_region", "clear_all_tabs", "clear_screen", "clr_eol", "clr_eos", + "column_address", "command_character", "cursor_address", "cursor_down", "cursor_home", + "cursor_invisible", "cursor_left", "cursor_mem_address", "cursor_normal", "cursor_right", + "cursor_to_ll", "cursor_up", "cursor_visible", "delete_character", "delete_line", + "dis_status_line", "down_half_line", "enter_alt_charset_mode", "enter_blink_mode", + "enter_bold_mode", "enter_ca_mode", "enter_delete_mode", "enter_dim_mode", "enter_insert_mode", + "enter_secure_mode", "enter_protected_mode", "enter_reverse_mode", "enter_standout_mode", + "enter_underline_mode", "erase_chars", "exit_alt_charset_mode", "exit_attribute_mode", + "exit_ca_mode", "exit_delete_mode", "exit_insert_mode", "exit_standout_mode", + "exit_underline_mode", "flash_screen", "form_feed", "from_status_line", "init_1string", + "init_2string", "init_3string", "init_file", "insert_character", "insert_line", + "insert_padding", "key_backspace", "key_catab", "key_clear", "key_ctab", "key_dc", "key_dl", + "key_down", "key_eic", "key_eol", "key_eos", "key_f0", "key_f1", "key_f10", "key_f2", "key_f3", + "key_f4", "key_f5", "key_f6", "key_f7", "key_f8", "key_f9", "key_home", "key_ic", "key_il", + "key_left", "key_ll", "key_npage", "key_ppage", "key_right", "key_sf", "key_sr", "key_stab", + "key_up", "keypad_local", "keypad_xmit", "lab_f0", "lab_f1", "lab_f10", "lab_f2", "lab_f3", + "lab_f4", "lab_f5", "lab_f6", "lab_f7", "lab_f8", "lab_f9", "meta_off", "meta_on", "newline", + "pad_char", "parm_dch", "parm_delete_line", "parm_down_cursor", "parm_ich", "parm_index", + "parm_insert_line", "parm_left_cursor", "parm_right_cursor", "parm_rindex", "parm_up_cursor", + "pkey_key", "pkey_local", "pkey_xmit", "print_screen", "prtr_off", "prtr_on", "repeat_char", + "reset_1string", "reset_2string", "reset_3string", "reset_file", "restore_cursor", + "row_address", "save_cursor", "scroll_forward", "scroll_reverse", "set_attributes", "set_tab", + "set_window", "tab", "to_status_line", "underline_char", "up_half_line", "init_prog", "key_a1", + "key_a3", "key_b2", "key_c1", "key_c3", "prtr_non", "char_padding", "acs_chars", "plab_norm", + "key_btab", "enter_xon_mode", "exit_xon_mode", "enter_am_mode", "exit_am_mode", "xon_character", + "xoff_character", "ena_acs", "label_on", "label_off", "key_beg", "key_cancel", "key_close", + "key_command", "key_copy", "key_create", "key_end", "key_enter", "key_exit", "key_find", + "key_help", "key_mark", "key_message", "key_move", "key_next", "key_open", "key_options", + "key_previous", "key_print", "key_redo", "key_reference", "key_refresh", "key_replace", + "key_restart", "key_resume", "key_save", "key_suspend", "key_undo", "key_sbeg", "key_scancel", + "key_scommand", "key_scopy", "key_screate", "key_sdc", "key_sdl", "key_select", "key_send", + "key_seol", "key_sexit", "key_sfind", "key_shelp", "key_shome", "key_sic", "key_sleft", + "key_smessage", "key_smove", "key_snext", "key_soptions", "key_sprevious", "key_sprint", + "key_sredo", "key_sreplace", "key_sright", "key_srsume", "key_ssave", "key_ssuspend", + "key_sundo", "req_for_input", "key_f11", "key_f12", "key_f13", "key_f14", "key_f15", "key_f16", + "key_f17", "key_f18", "key_f19", "key_f20", "key_f21", "key_f22", "key_f23", "key_f24", + "key_f25", "key_f26", "key_f27", "key_f28", "key_f29", "key_f30", "key_f31", "key_f32", + "key_f33", "key_f34", "key_f35", "key_f36", "key_f37", "key_f38", "key_f39", "key_f40", + "key_f41", "key_f42", "key_f43", "key_f44", "key_f45", "key_f46", "key_f47", "key_f48", + "key_f49", "key_f50", "key_f51", "key_f52", "key_f53", "key_f54", "key_f55", "key_f56", + "key_f57", "key_f58", "key_f59", "key_f60", "key_f61", "key_f62", "key_f63", "clr_bol", + "clear_margins", "set_left_margin", "set_right_margin", "label_format", "set_clock", + "display_clock", "remove_clock", "create_window", "goto_window", "hangup", "dial_phone", + "quick_dial", "tone", "pulse", "flash_hook", "fixed_pause", "wait_tone", "user0", "user1", + "user2", "user3", "user4", "user5", "user6", "user7", "user8", "user9", "orig_pair", + "orig_colors", "initialize_color", "initialize_pair", "set_color_pair", "set_foreground", + "set_background", "change_char_pitch", "change_line_pitch", "change_res_horz", + "change_res_vert", "define_char", "enter_doublewide_mode", "enter_draft_quality", + "enter_italics_mode", "enter_leftward_mode", "enter_micro_mode", "enter_near_letter_quality", + "enter_normal_quality", "enter_shadow_mode", "enter_subscript_mode", "enter_superscript_mode", + "enter_upward_mode", "exit_doublewide_mode", "exit_italics_mode", "exit_leftward_mode", + "exit_micro_mode", "exit_shadow_mode", "exit_subscript_mode", "exit_superscript_mode", + "exit_upward_mode", "micro_column_address", "micro_down", "micro_left", "micro_right", + "micro_row_address", "micro_up", "order_of_pins", "parm_down_micro", "parm_left_micro", + "parm_right_micro", "parm_up_micro", "select_char_set", "set_bottom_margin", + "set_bottom_margin_parm", "set_left_margin_parm", "set_right_margin_parm", "set_top_margin", + "set_top_margin_parm", "start_bit_image", "start_char_set_def", "stop_bit_image", + "stop_char_set_def", "subscript_characters", "superscript_characters", "these_cause_cr", + "zero_motion", "char_set_names", "key_mouse", "mouse_info", "req_mouse_pos", "get_mouse", + "set_a_foreground", "set_a_background", "pkey_plab", "device_type", "code_set_init", + "set0_des_seq", "set1_des_seq", "set2_des_seq", "set3_des_seq", "set_lr_margin", + "set_tb_margin", "bit_image_repeat", "bit_image_newline", "bit_image_carriage_return", + "color_names", "define_bit_image_region", "end_bit_image_region", "set_color_band", + "set_page_length", "display_pc_char", "enter_pc_charset_mode", "exit_pc_charset_mode", + "enter_scancode_mode", "exit_scancode_mode", "pc_term_options", "scancode_escape", + "alt_scancode_esc", "enter_horizontal_hl_mode", "enter_left_hl_mode", "enter_low_hl_mode", + "enter_right_hl_mode", "enter_top_hl_mode", "enter_vertical_hl_mode", "set_a_attributes", + "set_pglen_inch", "termcap_init2", "termcap_reset", "linefeed_if_not_lf", "backspace_if_not_bs", + "other_non_function_keys", "arrow_key_map", "acs_ulcorner", "acs_llcorner", "acs_urcorner", + "acs_lrcorner", "acs_ltee", "acs_rtee", "acs_btee", "acs_ttee", "acs_hline", "acs_vline", + "acs_plus", "memory_lock", "memory_unlock", "box_chars_1"]; + +#[rustfmt::skip] +pub(crate) static stringnames: &[&str] = &[ "cbt", "_", "cr", "csr", "tbc", "clear", + "_", "_", "hpa", "cmdch", "cup", "cud1", "home", "civis", "cub1", "mrcup", "cnorm", "cuf1", + "ll", "cuu1", "cvvis", "dch1", "dl1", "dsl", "hd", "smacs", "blink", "bold", "smcup", "smdc", + "dim", "smir", "invis", "prot", "rev", "smso", "smul", "ech", "rmacs", "sgr0", "rmcup", "rmdc", + "rmir", "rmso", "rmul", "flash", "ff", "fsl", "is1", "is2", "is3", "if", "ich1", "il1", "ip", + "kbs", "ktbc", "kclr", "kctab", "_", "_", "kcud1", "_", "_", "_", "_", "_", "_", "_", "_", "_", + "_", "_", "_", "_", "_", "khome", "_", "_", "kcub1", "_", "knp", "kpp", "kcuf1", "_", "_", + "khts", "_", "rmkx", "smkx", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "rmm", "_", + "_", "pad", "dch", "dl", "cud", "ich", "indn", "il", "cub", "cuf", "rin", "cuu", "pfkey", + "pfloc", "pfx", "mc0", "mc4", "_", "rep", "rs1", "rs2", "rs3", "rf", "rc", "vpa", "sc", "ind", + "ri", "sgr", "_", "wind", "_", "tsl", "uc", "hu", "iprog", "_", "_", "_", "_", "_", "mc5p", + "rmp", "acsc", "pln", "kcbt", "smxon", "rmxon", "smam", "rmam", "xonc", "xoffc", "_", "smln", + "rmln", "_", "kcan", "kclo", "kcmd", "kcpy", "kcrt", "_", "kent", "kext", "kfnd", "khlp", + "kmrk", "kmsg", "kmov", "knxt", "kopn", "kopt", "kprv", "kprt", "krdo", "kref", "krfr", "krpl", + "krst", "kres", "ksav", "kspd", "kund", "kBEG", "kCAN", "kCMD", "kCPY", "kCRT", "_", "_", + "kslt", "kEND", "kEOL", "kEXT", "kFND", "kHLP", "kHOM", "_", "kLFT", "kMSG", "kMOV", "kNXT", + "kOPT", "kPRV", "kPRT", "kRDO", "kRPL", "kRIT", "kRES", "kSAV", "kSPD", "kUND", "rfi", "_", "_", + "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", + "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", + "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", "_", + "dclk", "rmclk", "cwin", "wingo", "_", "dial", "qdial", "_", "_", "hook", "pause", "wait", "_", + "_", "_", "_", "_", "_", "_", "_", "_", "_", "op", "oc", "initc", "initp", "scp", "setf", + "setb", "cpi", "lpi", "chr", "cvr", "defc", "swidm", "sdrfq", "sitm", "slm", "smicm", "snlq", + "snrmq", "sshm", "ssubm", "ssupm", "sum", "rwidm", "ritm", "rlm", "rmicm", "rshm", "rsubm", + "rsupm", "rum", "mhpa", "mcud1", "mcub1", "mcuf1", "mvpa", "mcuu1", "porder", "mcud", "mcub", + "mcuf", "mcuu", "scs", "smgb", "smgbp", "smglp", "smgrp", "smgt", "smgtp", "sbim", "scsd", + "rbim", "rcsd", "subcs", "supcs", "docr", "zerom", "csnm", "kmous", "minfo", "reqmp", "getm", + "setaf", "setab", "pfxl", "devt", "csin", "s0ds", "s1ds", "s2ds", "s3ds", "smglr", "smgtb", + "birep", "binel", "bicr", "colornm", "defbi", "endbi", "setcolor", "slines", "dispc", "smpch", + "rmpch", "smsc", "rmsc", "pctrm", "scesc", "scesa", "ehhlm", "elhlm", "elohlm", "erhlm", + "ethlm", "evhlm", "sgr1", "slength", "OTi2", "OTrs", "OTnl", "OTbs", "OTko", "OTma", "OTG2", + "OTG3", "OTG1", "OTG4", "OTGR", "OTGL", "OTGU", "OTGD", "OTGH", "OTGV", "OTGC", "meml", "memu", + "box1"]; + +fn read_le_u16(r: &mut dyn io::Read) -> io::Result { + let mut b = [0; 2]; + r.read_exact(&mut b)?; + Ok((b[0] as u16) | ((b[1] as u16) << 8)) +} + +fn read_le_u32(r: &mut dyn io::Read) -> io::Result { + let mut b = [0; 4]; + r.read_exact(&mut b)?; + Ok((b[0] as u32) | ((b[1] as u32) << 8) | ((b[2] as u32) << 16) | ((b[3] as u32) << 24)) +} + +fn read_byte(r: &mut dyn io::Read) -> io::Result { + match r.bytes().next() { + Some(s) => s, + None => Err(io::Error::new(io::ErrorKind::Other, "end of file")), + } +} + +/// Parse a compiled terminfo entry, using long capability names if `longnames` +/// is true +pub(crate) fn parse(file: &mut dyn io::Read, longnames: bool) -> Result { + macro_rules! t( ($e:expr) => ( + match $e { + Ok(e) => e, + Err(e) => return Err(e.to_string()) + } + ) ); + + let (bnames, snames, nnames) = if longnames { + (boolfnames, stringfnames, numfnames) + } else { + (boolnames, stringnames, numnames) + }; + + // Check magic number + let magic = t!(read_le_u16(file)); + + let extended = match magic { + 0o0432 => false, + 0o01036 => true, + _ => return Err(format!("invalid magic number, found {magic:o}")), + }; + + // According to the spec, these fields must be >= -1 where -1 means that the feature is not + // supported. Using 0 instead of -1 works because we skip sections with length 0. + macro_rules! read_nonneg { + () => {{ + match t!(read_le_u16(file)) as i16 { + n if n >= 0 => n as usize, + -1 => 0, + _ => return Err("incompatible file: length fields must be >= -1".to_string()), + } + }}; + } + + let names_bytes = read_nonneg!(); + let bools_bytes = read_nonneg!(); + let numbers_count = read_nonneg!(); + let string_offsets_count = read_nonneg!(); + let string_table_bytes = read_nonneg!(); + + if names_bytes == 0 { + return Err("incompatible file: names field must be at least 1 byte wide".to_string()); + } + + if bools_bytes > boolnames.len() { + return Err("incompatible file: more booleans than expected".to_string()); + } + + if numbers_count > numnames.len() { + return Err("incompatible file: more numbers than expected".to_string()); + } + + if string_offsets_count > stringnames.len() { + return Err("incompatible file: more string offsets than expected".to_string()); + } + + // don't read NUL + let mut bytes = Vec::new(); + t!(file.take((names_bytes - 1) as u64).read_to_end(&mut bytes)); + let names_str = match String::from_utf8(bytes) { + Ok(s) => s, + Err(_) => return Err("input not utf-8".to_string()), + }; + + let term_names: Vec = names_str.split('|').map(|s| s.to_string()).collect(); + // consume NUL + if t!(read_byte(file)) != b'\0' { + return Err("incompatible file: missing null terminator for names section".to_string()); + } + + let bools_map: HashMap = t! { + (0..bools_bytes).filter_map(|i| match read_byte(file) { + Err(e) => Some(Err(e)), + Ok(1) => Some(Ok((bnames[i].to_string(), true))), + Ok(_) => None + }).collect() + }; + + if (bools_bytes + names_bytes) % 2 == 1 { + t!(read_byte(file)); // compensate for padding + } + + let numbers_map: HashMap = t! { + (0..numbers_count).filter_map(|i| { + let number = if extended { read_le_u32(file) } else { read_le_u16(file).map(Into::into) }; + + match number { + Ok(0xFFFF) => None, + Ok(n) => Some(Ok((nnames[i].to_string(), n))), + Err(e) => Some(Err(e)) + } + }).collect() + }; + + let string_map: HashMap> = if string_offsets_count > 0 { + let string_offsets: Vec = + t!((0..string_offsets_count).map(|_| read_le_u16(file)).collect()); + + let mut string_table = Vec::new(); + t!(file.take(string_table_bytes as u64).read_to_end(&mut string_table)); + + t!(string_offsets + .into_iter() + .enumerate() + .filter(|&(_, offset)| { + // non-entry + offset != 0xFFFF + }) + .map(|(i, offset)| { + let offset = offset as usize; + + let name = if snames[i] == "_" { stringfnames[i] } else { snames[i] }; + + if offset == 0xFFFE { + // undocumented: FFFE indicates cap@, which means the capability is not present + // unsure if the handling for this is correct + return Ok((name.to_string(), Vec::new())); + } + + // Find the offset of the NUL we want to go to + let nulpos = string_table[offset..string_table_bytes].iter().position(|&b| b == 0); + match nulpos { + Some(len) => { + Ok((name.to_string(), string_table[offset..offset + len].to_vec())) + } + None => Err("invalid file: missing NUL in string_table".to_string()), + } + }) + .collect()) + } else { + HashMap::new() + }; + + // And that's all there is to it + Ok(TermInfo { names: term_names, bools: bools_map, numbers: numbers_map, strings: string_map }) +} + +/// Creates a dummy TermInfo struct for msys terminals +pub(crate) fn msys_terminfo() -> TermInfo { + let mut strings = HashMap::new(); + strings.insert("sgr0".to_string(), b"\x1B[0m".to_vec()); + strings.insert("bold".to_string(), b"\x1B[1m".to_vec()); + strings.insert("setaf".to_string(), b"\x1B[3%p1%dm".to_vec()); + strings.insert("setab".to_string(), b"\x1B[4%p1%dm".to_vec()); + + let mut numbers = HashMap::new(); + numbers.insert("colors".to_string(), 8); + + TermInfo { + names: vec!["cygwin".to_string()], // msys is a fork of an older cygwin version + bools: HashMap::new(), + numbers, + strings, + } +} diff --git a/crux-mir/lib/test/src/term/terminfo/parser/compiled/tests.rs b/crux-mir/lib/test/src/term/terminfo/parser/compiled/tests.rs new file mode 100644 index 000000000..8a9187b04 --- /dev/null +++ b/crux-mir/lib/test/src/term/terminfo/parser/compiled/tests.rs @@ -0,0 +1,8 @@ +use super::*; + +#[test] +fn test_veclens() { + assert_eq!(boolfnames.len(), boolnames.len()); + assert_eq!(numfnames.len(), numnames.len()); + assert_eq!(stringfnames.len(), stringnames.len()); +} diff --git a/crux-mir/lib/test/src/term/terminfo/searcher.rs b/crux-mir/lib/test/src/term/terminfo/searcher.rs new file mode 100644 index 000000000..3e8ccc91a --- /dev/null +++ b/crux-mir/lib/test/src/term/terminfo/searcher.rs @@ -0,0 +1,69 @@ +//! ncurses-compatible database discovery. +//! +//! Does not support hashed database, only filesystem! + +use std::env; +use std::fs; +use std::path::PathBuf; + +#[cfg(test)] +mod tests; + +/// Return path to database entry for `term` +#[allow(deprecated)] +pub(crate) fn get_dbpath_for_term(term: &str) -> Option { + let mut dirs_to_search = Vec::new(); + let first_char = term.chars().next()?; + + // Find search directory + if let Some(dir) = env::var_os("TERMINFO") { + dirs_to_search.push(PathBuf::from(dir)); + } + + if let Ok(dirs) = env::var("TERMINFO_DIRS") { + for i in dirs.split(':') { + if i.is_empty() { + dirs_to_search.push(PathBuf::from("/usr/share/terminfo")); + } else { + dirs_to_search.push(PathBuf::from(i)); + } + } + } else { + // Found nothing in TERMINFO_DIRS, use the default paths: + // According to /etc/terminfo/README, after looking at + // ~/.terminfo, ncurses will search /etc/terminfo, then + // /lib/terminfo, and eventually /usr/share/terminfo. + // On Haiku the database can be found at /boot/system/data/terminfo + if let Some(mut homedir) = env::home_dir() { + homedir.push(".terminfo"); + dirs_to_search.push(homedir) + } + + dirs_to_search.push(PathBuf::from("/etc/terminfo")); + dirs_to_search.push(PathBuf::from("/lib/terminfo")); + dirs_to_search.push(PathBuf::from("/usr/share/terminfo")); + dirs_to_search.push(PathBuf::from("/boot/system/data/terminfo")); + } + + // Look for the terminal in all of the search directories + for mut p in dirs_to_search { + if fs::metadata(&p).is_ok() { + p.push(&first_char.to_string()); + p.push(term); + if fs::metadata(&p).is_ok() { + return Some(p); + } + p.pop(); + p.pop(); + + // on some installations the dir is named after the hex of the char + // (e.g., macOS) + p.push(&format!("{:x}", first_char as usize)); + p.push(term); + if fs::metadata(&p).is_ok() { + return Some(p); + } + } + } + None +} diff --git a/crux-mir/lib/test/src/term/terminfo/searcher/tests.rs b/crux-mir/lib/test/src/term/terminfo/searcher/tests.rs new file mode 100644 index 000000000..4227a585e --- /dev/null +++ b/crux-mir/lib/test/src/term/terminfo/searcher/tests.rs @@ -0,0 +1,19 @@ +use super::*; + +#[test] +#[ignore = "buildbots don't have ncurses installed and I can't mock everything I need"] +fn test_get_dbpath_for_term() { + // woefully inadequate test coverage + // note: current tests won't work with non-standard terminfo hierarchies (e.g., macOS's) + use std::env; + // FIXME (#9639): This needs to handle non-utf8 paths + fn x(t: &str) -> String { + let p = get_dbpath_for_term(t).expect("no terminfo entry found"); + p.to_str().unwrap().to_string() + } + assert!(x("screen") == "/usr/share/terminfo/s/screen"); + assert!(get_dbpath_for_term("") == None); + env::set_var("TERMINFO_DIRS", ":"); + assert!(x("screen") == "/usr/share/terminfo/s/screen"); + env::remove_var("TERMINFO_DIRS"); +} diff --git a/crux-mir/lib/test/src/term/win.rs b/crux-mir/lib/test/src/term/win.rs new file mode 100644 index 000000000..55020141a --- /dev/null +++ b/crux-mir/lib/test/src/term/win.rs @@ -0,0 +1,169 @@ +//! Windows console handling + +// FIXME (#13400): this is only a tiny fraction of the Windows console api + +use std::io; +use std::io::prelude::*; + +use super::color; +use super::Terminal; + +/// A Terminal implementation that uses the Win32 Console API. +pub(crate) struct WinConsole { + buf: T, + def_foreground: color::Color, + def_background: color::Color, + foreground: color::Color, + background: color::Color, +} + +type SHORT = i16; +type WORD = u16; +type DWORD = u32; +type BOOL = i32; +type HANDLE = *mut u8; + +#[allow(non_snake_case)] +#[repr(C)] +struct SMALL_RECT { + Left: SHORT, + Top: SHORT, + Right: SHORT, + Bottom: SHORT, +} + +#[allow(non_snake_case)] +#[repr(C)] +struct COORD { + X: SHORT, + Y: SHORT, +} + +#[allow(non_snake_case)] +#[repr(C)] +struct CONSOLE_SCREEN_BUFFER_INFO { + dwSize: COORD, + dwCursorPosition: COORD, + wAttributes: WORD, + srWindow: SMALL_RECT, + dwMaximumWindowSize: COORD, +} + +#[allow(non_snake_case)] +#[link(name = "kernel32")] +extern "system" { + fn SetConsoleTextAttribute(handle: HANDLE, attr: WORD) -> BOOL; + fn GetStdHandle(which: DWORD) -> HANDLE; + fn GetConsoleScreenBufferInfo(handle: HANDLE, info: *mut CONSOLE_SCREEN_BUFFER_INFO) -> BOOL; +} + +fn color_to_bits(color: color::Color) -> u16 { + // magic numbers from mingw-w64's wincon.h + + let bits = match color % 8 { + color::BLACK => 0, + color::BLUE => 0x1, + color::GREEN => 0x2, + color::RED => 0x4, + color::YELLOW => 0x2 | 0x4, + color::MAGENTA => 0x1 | 0x4, + color::CYAN => 0x1 | 0x2, + color::WHITE => 0x1 | 0x2 | 0x4, + _ => unreachable!(), + }; + + if color >= 8 { bits | 0x8 } else { bits } +} + +fn bits_to_color(bits: u16) -> color::Color { + let color = match bits & 0x7 { + 0 => color::BLACK, + 0x1 => color::BLUE, + 0x2 => color::GREEN, + 0x4 => color::RED, + 0x6 => color::YELLOW, + 0x5 => color::MAGENTA, + 0x3 => color::CYAN, + 0x7 => color::WHITE, + _ => unreachable!(), + }; + + color | (u32::from(bits) & 0x8) // copy the hi-intensity bit +} + +impl WinConsole { + fn apply(&mut self) { + let _unused = self.buf.flush(); + let mut accum: WORD = 0; + accum |= color_to_bits(self.foreground); + accum |= color_to_bits(self.background) << 4; + + unsafe { + // Magic -11 means stdout, from + // https://docs.microsoft.com/en-us/windows/console/getstdhandle + // + // You may be wondering, "but what about stderr?", and the answer + // to that is that setting terminal attributes on the stdout + // handle also sets them for stderr, since they go to the same + // terminal! Admittedly, this is fragile, since stderr could be + // redirected to a different console. This is good enough for + // rustc though. See #13400. + let out = GetStdHandle(-11i32 as DWORD); + SetConsoleTextAttribute(out, accum); + } + } + + pub(crate) fn new(out: T) -> WinConsole { + use std::mem::MaybeUninit; + + let fg; + let bg; + unsafe { + let mut buffer_info = MaybeUninit::::uninit(); + if GetConsoleScreenBufferInfo(GetStdHandle(-11i32 as DWORD), buffer_info.as_mut_ptr()) + != 0 + { + let buffer_info = buffer_info.assume_init(); + fg = bits_to_color(buffer_info.wAttributes); + bg = bits_to_color(buffer_info.wAttributes >> 4); + } else { + fg = color::WHITE; + bg = color::BLACK; + } + } + WinConsole { + buf: out, + def_foreground: fg, + def_background: bg, + foreground: fg, + background: bg, + } + } +} + +impl Write for WinConsole { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.buf.write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.buf.flush() + } +} + +impl Terminal for WinConsole { + fn fg(&mut self, color: color::Color) -> io::Result { + self.foreground = color; + self.apply(); + + Ok(true) + } + + fn reset(&mut self) -> io::Result { + self.foreground = self.def_foreground; + self.background = self.def_background; + self.apply(); + + Ok(true) + } +} diff --git a/crux-mir/lib/test/src/test_result.rs b/crux-mir/lib/test/src/test_result.rs new file mode 100644 index 000000000..1da238e3e --- /dev/null +++ b/crux-mir/lib/test/src/test_result.rs @@ -0,0 +1,107 @@ +use std::any::Any; + +use super::bench::BenchSamples; +use super::options::ShouldPanic; +use super::time; +use super::types::TestDesc; + +pub use self::TestResult::*; + +// Return codes for secondary process. +// Start somewhere other than 0 so we know the return code means what we think +// it means. +pub const TR_OK: i32 = 50; +pub const TR_FAILED: i32 = 51; + +#[derive(Debug, Clone, PartialEq)] +pub enum TestResult { + TrOk, + TrFailed, + TrFailedMsg(String), + TrIgnored, + TrBench(BenchSamples), + TrTimedFail, +} + +/// Creates a `TestResult` depending on the raw result of test execution +/// and associated data. +pub fn calc_result<'a>( + desc: &TestDesc, + task_result: Result<(), &'a (dyn Any + 'static + Send)>, + time_opts: &Option, + exec_time: &Option, +) -> TestResult { + let result = match (&desc.should_panic, task_result) { + (&ShouldPanic::No, Ok(())) | (&ShouldPanic::Yes, Err(_)) => TestResult::TrOk, + (&ShouldPanic::YesWithMessage(msg), Err(err)) => { + let maybe_panic_str = err + .downcast_ref::() + .map(|e| &**e) + .or_else(|| err.downcast_ref::<&'static str>().copied()); + + if maybe_panic_str.map(|e| e.contains(msg)).unwrap_or(false) { + TestResult::TrOk + } else if let Some(panic_str) = maybe_panic_str { + TestResult::TrFailedMsg(format!( + r#"panic did not contain expected string + panic message: `{panic_str:?}`, + expected substring: `{msg:?}`"# + )) + } else { + TestResult::TrFailedMsg(format!( + r#"expected panic with string value, + found non-string value: `{:?}` + expected substring: `{:?}`"#, + (*err).type_id(), + msg + )) + } + } + (&ShouldPanic::Yes, Ok(())) | (&ShouldPanic::YesWithMessage(_), Ok(())) => { + TestResult::TrFailedMsg("test did not panic as expected".to_string()) + } + _ => TestResult::TrFailed, + }; + + // If test is already failed (or allowed to fail), do not change the result. + if result != TestResult::TrOk { + return result; + } + + // Check if test is failed due to timeout. + if let (Some(opts), Some(time)) = (time_opts, exec_time) { + if opts.error_on_excess && opts.is_critical(desc, time) { + return TestResult::TrTimedFail; + } + } + + result +} + +/// Creates a `TestResult` depending on the exit code of test subprocess. +pub fn get_result_from_exit_code( + desc: &TestDesc, + code: i32, + time_opts: &Option, + exec_time: &Option, +) -> TestResult { + let result = match code { + TR_OK => TestResult::TrOk, + TR_FAILED => TestResult::TrFailed, + _ => TestResult::TrFailedMsg(format!("got unexpected return code {code}")), + }; + + // If test is already failed (or allowed to fail), do not change the result. + if result != TestResult::TrOk { + return result; + } + + // Check if test is failed due to timeout. + if let (Some(opts), Some(time)) = (time_opts, exec_time) { + if opts.error_on_excess && opts.is_critical(desc, time) { + return TestResult::TrTimedFail; + } + } + + result +} diff --git a/crux-mir/lib/test/src/tests.rs b/crux-mir/lib/test/src/tests.rs new file mode 100644 index 000000000..3a0260f86 --- /dev/null +++ b/crux-mir/lib/test/src/tests.rs @@ -0,0 +1,835 @@ +use super::*; + +use crate::{ + bench::Bencher, + console::OutputLocation, + formatters::PrettyFormatter, + options::OutputFormat, + test::{ + filter_tests, + parse_opts, + run_test, + DynTestFn, + DynTestName, + MetricMap, + RunIgnored, + RunStrategy, + ShouldPanic, + StaticTestName, + TestDesc, + TestDescAndFn, + TestOpts, + TrIgnored, + TrOk, + // FIXME (introduced by #65251) + // ShouldPanic, StaticTestName, TestDesc, TestDescAndFn, TestOpts, TestTimeOptions, + // TestType, TrFailedMsg, TrIgnored, TrOk, + }, + time::{TestTimeOptions, TimeThreshold}, +}; +use std::sync::mpsc::channel; +use std::time::Duration; + +impl TestOpts { + fn new() -> TestOpts { + TestOpts { + list: false, + filters: vec![], + filter_exact: false, + force_run_in_process: false, + exclude_should_panic: false, + run_ignored: RunIgnored::No, + run_tests: false, + bench_benchmarks: false, + logfile: None, + nocapture: false, + color: AutoColor, + format: OutputFormat::Pretty, + shuffle: false, + shuffle_seed: None, + test_threads: None, + skip: vec![], + time_options: None, + options: Options::new(), + fail_fast: false, + } + } +} + +fn one_ignored_one_unignored_test() -> Vec { + vec![ + TestDescAndFn { + desc: TestDesc { + name: StaticTestName("1"), + ignore: true, + ignore_message: None, + should_panic: ShouldPanic::No, + compile_fail: false, + no_run: false, + test_type: TestType::Unknown, + }, + testfn: DynTestFn(Box::new(move || Ok(()))), + }, + TestDescAndFn { + desc: TestDesc { + name: StaticTestName("2"), + ignore: false, + ignore_message: None, + should_panic: ShouldPanic::No, + compile_fail: false, + no_run: false, + test_type: TestType::Unknown, + }, + testfn: DynTestFn(Box::new(move || Ok(()))), + }, + ] +} + +#[test] +pub fn do_not_run_ignored_tests() { + fn f() -> Result<(), String> { + panic!(); + } + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: true, + ignore_message: None, + should_panic: ShouldPanic::No, + compile_fail: false, + no_run: false, + test_type: TestType::Unknown, + }, + testfn: DynTestFn(Box::new(f)), + }; + let (tx, rx) = channel(); + run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx); + let result = rx.recv().unwrap().result; + assert_ne!(result, TrOk); +} + +#[test] +pub fn ignored_tests_result_in_ignored() { + fn f() -> Result<(), String> { + Ok(()) + } + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: true, + ignore_message: None, + should_panic: ShouldPanic::No, + compile_fail: false, + no_run: false, + test_type: TestType::Unknown, + }, + testfn: DynTestFn(Box::new(f)), + }; + let (tx, rx) = channel(); + run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx); + let result = rx.recv().unwrap().result; + assert_eq!(result, TrIgnored); +} + +// FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251) +#[test] +#[cfg(not(target_os = "emscripten"))] +fn test_should_panic() { + fn f() -> Result<(), String> { + panic!(); + } + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: false, + ignore_message: None, + should_panic: ShouldPanic::Yes, + compile_fail: false, + no_run: false, + test_type: TestType::Unknown, + }, + testfn: DynTestFn(Box::new(f)), + }; + let (tx, rx) = channel(); + run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx); + let result = rx.recv().unwrap().result; + assert_eq!(result, TrOk); +} + +// FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251) +#[test] +#[cfg(not(target_os = "emscripten"))] +fn test_should_panic_good_message() { + fn f() -> Result<(), String> { + panic!("an error message"); + } + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: false, + ignore_message: None, + should_panic: ShouldPanic::YesWithMessage("error message"), + compile_fail: false, + no_run: false, + test_type: TestType::Unknown, + }, + testfn: DynTestFn(Box::new(f)), + }; + let (tx, rx) = channel(); + run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx); + let result = rx.recv().unwrap().result; + assert_eq!(result, TrOk); +} + +// FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251) +#[test] +#[cfg(not(target_os = "emscripten"))] +fn test_should_panic_bad_message() { + use crate::tests::TrFailedMsg; + fn f() -> Result<(), String> { + panic!("an error message"); + } + let expected = "foobar"; + let failed_msg = r#"panic did not contain expected string + panic message: `"an error message"`, + expected substring: `"foobar"`"#; + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: false, + ignore_message: None, + should_panic: ShouldPanic::YesWithMessage(expected), + compile_fail: false, + no_run: false, + test_type: TestType::Unknown, + }, + testfn: DynTestFn(Box::new(f)), + }; + let (tx, rx) = channel(); + run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx); + let result = rx.recv().unwrap().result; + assert_eq!(result, TrFailedMsg(failed_msg.to_string())); +} + +// FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251) +#[test] +#[cfg(not(target_os = "emscripten"))] +fn test_should_panic_non_string_message_type() { + use crate::tests::TrFailedMsg; + use std::any::TypeId; + fn f() -> Result<(), String> { + std::panic::panic_any(1i32); + } + let expected = "foobar"; + let failed_msg = format!( + r#"expected panic with string value, + found non-string value: `{:?}` + expected substring: `"foobar"`"#, + TypeId::of::() + ); + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: false, + ignore_message: None, + should_panic: ShouldPanic::YesWithMessage(expected), + compile_fail: false, + no_run: false, + test_type: TestType::Unknown, + }, + testfn: DynTestFn(Box::new(f)), + }; + let (tx, rx) = channel(); + run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx); + let result = rx.recv().unwrap().result; + assert_eq!(result, TrFailedMsg(failed_msg)); +} + +// FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251) +#[test] +#[cfg(not(target_os = "emscripten"))] +fn test_should_panic_but_succeeds() { + let should_panic_variants = [ShouldPanic::Yes, ShouldPanic::YesWithMessage("error message")]; + + for &should_panic in should_panic_variants.iter() { + fn f() -> Result<(), String> { + Ok(()) + } + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: false, + ignore_message: None, + should_panic, + compile_fail: false, + no_run: false, + test_type: TestType::Unknown, + }, + testfn: DynTestFn(Box::new(f)), + }; + let (tx, rx) = channel(); + run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx); + let result = rx.recv().unwrap().result; + assert_eq!( + result, + TrFailedMsg("test did not panic as expected".to_string()), + "should_panic == {:?}", + should_panic + ); + } +} + +fn report_time_test_template(report_time: bool) -> Option { + fn f() -> Result<(), String> { + Ok(()) + } + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: false, + ignore_message: None, + should_panic: ShouldPanic::No, + compile_fail: false, + no_run: false, + test_type: TestType::Unknown, + }, + testfn: DynTestFn(Box::new(f)), + }; + let time_options = if report_time { Some(TestTimeOptions::default()) } else { None }; + + let test_opts = TestOpts { time_options, ..TestOpts::new() }; + let (tx, rx) = channel(); + run_test(&test_opts, false, TestId(0), desc, RunStrategy::InProcess, tx); + let exec_time = rx.recv().unwrap().exec_time; + exec_time +} + +#[test] +fn test_should_not_report_time() { + let exec_time = report_time_test_template(false); + assert!(exec_time.is_none()); +} + +#[test] +fn test_should_report_time() { + let exec_time = report_time_test_template(true); + assert!(exec_time.is_some()); +} + +fn time_test_failure_template(test_type: TestType) -> TestResult { + fn f() -> Result<(), String> { + Ok(()) + } + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: false, + ignore_message: None, + should_panic: ShouldPanic::No, + compile_fail: false, + no_run: false, + test_type, + }, + testfn: DynTestFn(Box::new(f)), + }; + // `Default` will initialize all the thresholds to 0 milliseconds. + let mut time_options = TestTimeOptions::default(); + time_options.error_on_excess = true; + + let test_opts = TestOpts { time_options: Some(time_options), ..TestOpts::new() }; + let (tx, rx) = channel(); + run_test(&test_opts, false, TestId(0), desc, RunStrategy::InProcess, tx); + let result = rx.recv().unwrap().result; + + result +} + +#[test] +fn test_error_on_exceed() { + let types = [TestType::UnitTest, TestType::IntegrationTest, TestType::DocTest]; + + for test_type in types.iter() { + let result = time_test_failure_template(*test_type); + + assert_eq!(result, TestResult::TrTimedFail); + } + + // Check that for unknown tests thresholds aren't applied. + let result = time_test_failure_template(TestType::Unknown); + assert_eq!(result, TestResult::TrOk); +} + +fn typed_test_desc(test_type: TestType) -> TestDesc { + TestDesc { + name: StaticTestName("whatever"), + ignore: false, + ignore_message: None, + should_panic: ShouldPanic::No, + compile_fail: false, + no_run: false, + test_type, + } +} + +fn test_exec_time(millis: u64) -> TestExecTime { + TestExecTime(Duration::from_millis(millis)) +} + +#[test] +fn test_time_options_threshold() { + let unit = TimeThreshold::new(Duration::from_millis(50), Duration::from_millis(100)); + let integration = TimeThreshold::new(Duration::from_millis(500), Duration::from_millis(1000)); + let doc = TimeThreshold::new(Duration::from_millis(5000), Duration::from_millis(10000)); + + let options = TestTimeOptions { + error_on_excess: false, + unit_threshold: unit.clone(), + integration_threshold: integration.clone(), + doctest_threshold: doc.clone(), + }; + + let test_vector = [ + (TestType::UnitTest, unit.warn.as_millis() - 1, false, false), + (TestType::UnitTest, unit.warn.as_millis(), true, false), + (TestType::UnitTest, unit.critical.as_millis(), true, true), + (TestType::IntegrationTest, integration.warn.as_millis() - 1, false, false), + (TestType::IntegrationTest, integration.warn.as_millis(), true, false), + (TestType::IntegrationTest, integration.critical.as_millis(), true, true), + (TestType::DocTest, doc.warn.as_millis() - 1, false, false), + (TestType::DocTest, doc.warn.as_millis(), true, false), + (TestType::DocTest, doc.critical.as_millis(), true, true), + ]; + + for (test_type, time, expected_warn, expected_critical) in test_vector.iter() { + let test_desc = typed_test_desc(*test_type); + let exec_time = test_exec_time(*time as u64); + + assert_eq!(options.is_warn(&test_desc, &exec_time), *expected_warn); + assert_eq!(options.is_critical(&test_desc, &exec_time), *expected_critical); + } +} + +#[test] +fn parse_ignored_flag() { + let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()]; + let opts = parse_opts(&args).unwrap().unwrap(); + assert_eq!(opts.run_ignored, RunIgnored::Only); +} + +#[test] +fn parse_show_output_flag() { + let args = vec!["progname".to_string(), "filter".to_string(), "--show-output".to_string()]; + let opts = parse_opts(&args).unwrap().unwrap(); + assert!(opts.options.display_output); +} + +#[test] +fn parse_include_ignored_flag() { + let args = vec!["progname".to_string(), "filter".to_string(), "--include-ignored".to_string()]; + let opts = parse_opts(&args).unwrap().unwrap(); + assert_eq!(opts.run_ignored, RunIgnored::Yes); +} + +#[test] +pub fn filter_for_ignored_option() { + // When we run ignored tests the test filter should filter out all the + // unignored tests and flip the ignore flag on the rest to false + + let mut opts = TestOpts::new(); + opts.run_tests = true; + opts.run_ignored = RunIgnored::Only; + + let tests = one_ignored_one_unignored_test(); + let filtered = filter_tests(&opts, tests); + + assert_eq!(filtered.len(), 1); + assert_eq!(filtered[0].desc.name.to_string(), "1"); + assert!(!filtered[0].desc.ignore); +} + +#[test] +pub fn run_include_ignored_option() { + // When we "--include-ignored" tests, the ignore flag should be set to false on + // all tests and no test filtered out + + let mut opts = TestOpts::new(); + opts.run_tests = true; + opts.run_ignored = RunIgnored::Yes; + + let tests = one_ignored_one_unignored_test(); + let filtered = filter_tests(&opts, tests); + + assert_eq!(filtered.len(), 2); + assert!(!filtered[0].desc.ignore); + assert!(!filtered[1].desc.ignore); +} + +#[test] +pub fn exclude_should_panic_option() { + let mut opts = TestOpts::new(); + opts.run_tests = true; + opts.exclude_should_panic = true; + + let mut tests = one_ignored_one_unignored_test(); + tests.push(TestDescAndFn { + desc: TestDesc { + name: StaticTestName("3"), + ignore: false, + ignore_message: None, + should_panic: ShouldPanic::Yes, + compile_fail: false, + no_run: false, + test_type: TestType::Unknown, + }, + testfn: DynTestFn(Box::new(move || Ok(()))), + }); + + let filtered = filter_tests(&opts, tests); + + assert_eq!(filtered.len(), 2); + assert!(filtered.iter().all(|test| test.desc.should_panic == ShouldPanic::No)); +} + +#[test] +pub fn exact_filter_match() { + fn tests() -> Vec { + ["base", "base::test", "base::test1", "base::test2"] + .into_iter() + .map(|name| TestDescAndFn { + desc: TestDesc { + name: StaticTestName(name), + ignore: false, + ignore_message: None, + should_panic: ShouldPanic::No, + compile_fail: false, + no_run: false, + test_type: TestType::Unknown, + }, + testfn: DynTestFn(Box::new(move || Ok(()))), + }) + .collect() + } + + let substr = + filter_tests(&TestOpts { filters: vec!["base".into()], ..TestOpts::new() }, tests()); + assert_eq!(substr.len(), 4); + + let substr = + filter_tests(&TestOpts { filters: vec!["bas".into()], ..TestOpts::new() }, tests()); + assert_eq!(substr.len(), 4); + + let substr = + filter_tests(&TestOpts { filters: vec!["::test".into()], ..TestOpts::new() }, tests()); + assert_eq!(substr.len(), 3); + + let substr = + filter_tests(&TestOpts { filters: vec!["base::test".into()], ..TestOpts::new() }, tests()); + assert_eq!(substr.len(), 3); + + let substr = filter_tests( + &TestOpts { filters: vec!["test1".into(), "test2".into()], ..TestOpts::new() }, + tests(), + ); + assert_eq!(substr.len(), 2); + + let exact = filter_tests( + &TestOpts { filters: vec!["base".into()], filter_exact: true, ..TestOpts::new() }, + tests(), + ); + assert_eq!(exact.len(), 1); + + let exact = filter_tests( + &TestOpts { filters: vec!["bas".into()], filter_exact: true, ..TestOpts::new() }, + tests(), + ); + assert_eq!(exact.len(), 0); + + let exact = filter_tests( + &TestOpts { filters: vec!["::test".into()], filter_exact: true, ..TestOpts::new() }, + tests(), + ); + assert_eq!(exact.len(), 0); + + let exact = filter_tests( + &TestOpts { filters: vec!["base::test".into()], filter_exact: true, ..TestOpts::new() }, + tests(), + ); + assert_eq!(exact.len(), 1); + + let exact = filter_tests( + &TestOpts { + filters: vec!["base".into(), "base::test".into()], + filter_exact: true, + ..TestOpts::new() + }, + tests(), + ); + assert_eq!(exact.len(), 2); +} + +fn sample_tests() -> Vec { + let names = vec![ + "sha1::test".to_string(), + "isize::test_to_str".to_string(), + "isize::test_pow".to_string(), + "test::do_not_run_ignored_tests".to_string(), + "test::ignored_tests_result_in_ignored".to_string(), + "test::first_free_arg_should_be_a_filter".to_string(), + "test::parse_ignored_flag".to_string(), + "test::parse_include_ignored_flag".to_string(), + "test::filter_for_ignored_option".to_string(), + "test::run_include_ignored_option".to_string(), + "test::sort_tests".to_string(), + ]; + fn testfn() -> Result<(), String> { + Ok(()) + } + let mut tests = Vec::new(); + for name in &names { + let test = TestDescAndFn { + desc: TestDesc { + name: DynTestName((*name).clone()), + ignore: false, + ignore_message: None, + should_panic: ShouldPanic::No, + compile_fail: false, + no_run: false, + test_type: TestType::Unknown, + }, + testfn: DynTestFn(Box::new(testfn)), + }; + tests.push(test); + } + tests +} + +#[test] +pub fn shuffle_tests() { + let mut opts = TestOpts::new(); + opts.shuffle = true; + + let shuffle_seed = get_shuffle_seed(&opts).unwrap(); + + let left = + sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::>(); + let mut right = + sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::>(); + + assert!(left.iter().zip(&right).all(|(a, b)| a.1.desc.name == b.1.desc.name)); + + helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice()); + + assert!(left.iter().zip(right).any(|(a, b)| a.1.desc.name != b.1.desc.name)); +} + +#[test] +pub fn shuffle_tests_with_seed() { + let mut opts = TestOpts::new(); + opts.shuffle = true; + + let shuffle_seed = get_shuffle_seed(&opts).unwrap(); + + let mut left = + sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::>(); + let mut right = + sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::>(); + + helpers::shuffle::shuffle_tests(shuffle_seed, left.as_mut_slice()); + helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice()); + + assert!(left.iter().zip(right).all(|(a, b)| a.1.desc.name == b.1.desc.name)); +} + +#[test] +pub fn order_depends_on_more_than_seed() { + let mut opts = TestOpts::new(); + opts.shuffle = true; + + let shuffle_seed = get_shuffle_seed(&opts).unwrap(); + + let mut left_tests = sample_tests(); + let mut right_tests = sample_tests(); + + left_tests.pop(); + right_tests.remove(0); + + let mut left = + left_tests.into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::>(); + let mut right = + right_tests.into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::>(); + + assert_eq!(left.len(), right.len()); + + assert!(left.iter().zip(&right).all(|(a, b)| a.0 == b.0)); + + helpers::shuffle::shuffle_tests(shuffle_seed, left.as_mut_slice()); + helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice()); + + assert!(left.iter().zip(right).any(|(a, b)| a.0 != b.0)); +} + +#[test] +pub fn test_metricmap_compare() { + let mut m1 = MetricMap::new(); + let mut m2 = MetricMap::new(); + m1.insert_metric("in-both-noise", 1000.0, 200.0); + m2.insert_metric("in-both-noise", 1100.0, 200.0); + + m1.insert_metric("in-first-noise", 1000.0, 2.0); + m2.insert_metric("in-second-noise", 1000.0, 2.0); + + m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0); + m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0); + + m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0); + m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0); + + m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0); + m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0); + + m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0); + m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0); +} + +#[test] +pub fn test_bench_once_no_iter() { + fn f(_: &mut Bencher) -> Result<(), String> { + Ok(()) + } + bench::run_once(f).unwrap(); +} + +#[test] +pub fn test_bench_once_iter() { + fn f(b: &mut Bencher) -> Result<(), String> { + b.iter(|| {}); + Ok(()) + } + bench::run_once(f).unwrap(); +} + +#[test] +pub fn test_bench_no_iter() { + fn f(_: &mut Bencher) -> Result<(), String> { + Ok(()) + } + + let (tx, rx) = channel(); + + let desc = TestDesc { + name: StaticTestName("f"), + ignore: false, + ignore_message: None, + should_panic: ShouldPanic::No, + compile_fail: false, + no_run: false, + test_type: TestType::Unknown, + }; + + crate::bench::benchmark(TestId(0), desc, tx, true, f); + rx.recv().unwrap(); +} + +#[test] +pub fn test_bench_iter() { + fn f(b: &mut Bencher) -> Result<(), String> { + b.iter(|| {}); + Ok(()) + } + + let (tx, rx) = channel(); + + let desc = TestDesc { + name: StaticTestName("f"), + ignore: false, + ignore_message: None, + should_panic: ShouldPanic::No, + compile_fail: false, + no_run: false, + test_type: TestType::Unknown, + }; + + crate::bench::benchmark(TestId(0), desc, tx, true, f); + rx.recv().unwrap(); +} + +#[test] +fn should_sort_failures_before_printing_them() { + let test_a = TestDesc { + name: StaticTestName("a"), + ignore: false, + ignore_message: None, + should_panic: ShouldPanic::No, + compile_fail: false, + no_run: false, + test_type: TestType::Unknown, + }; + + let test_b = TestDesc { + name: StaticTestName("b"), + ignore: false, + ignore_message: None, + should_panic: ShouldPanic::No, + compile_fail: false, + no_run: false, + test_type: TestType::Unknown, + }; + + let mut out = PrettyFormatter::new(OutputLocation::Raw(Vec::new()), false, 10, false, None); + + let st = console::ConsoleTestState { + log_out: None, + total: 0, + passed: 0, + failed: 0, + ignored: 0, + filtered_out: 0, + measured: 0, + exec_time: None, + metrics: MetricMap::new(), + failures: vec![(test_b, Vec::new()), (test_a, Vec::new())], + options: Options::new(), + not_failures: Vec::new(), + time_failures: Vec::new(), + }; + + out.write_failures(&st).unwrap(); + let s = match out.output_location() { + &OutputLocation::Raw(ref m) => String::from_utf8_lossy(&m[..]), + &OutputLocation::Pretty(_) => unreachable!(), + }; + + let apos = s.find("a").unwrap(); + let bpos = s.find("b").unwrap(); + assert!(apos < bpos); +} + +#[test] +#[cfg(not(target_os = "emscripten"))] +fn test_dyn_bench_returning_err_fails_when_run_as_test() { + fn f(_: &mut Bencher) -> Result<(), String> { + Result::Err("An error".into()) + } + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: false, + ignore_message: None, + should_panic: ShouldPanic::No, + compile_fail: false, + no_run: false, + test_type: TestType::Unknown, + }, + testfn: DynBenchFn(Box::new(f)), + }; + let (tx, rx) = channel(); + let notify = move |event: TestEvent| { + if let TestEvent::TeResult(result) = event { + tx.send(result).unwrap(); + } + Ok(()) + }; + run_tests(&TestOpts { run_tests: true, ..TestOpts::new() }, vec![desc], notify).unwrap(); + let result = rx.recv().unwrap().result; + assert_eq!(result, TrFailed); +} diff --git a/crux-mir/lib/test/src/time.rs b/crux-mir/lib/test/src/time.rs new file mode 100644 index 000000000..7fd69d7f7 --- /dev/null +++ b/crux-mir/lib/test/src/time.rs @@ -0,0 +1,195 @@ +//! Module `time` contains everything related to the time measurement of unit tests +//! execution. +//! The purposes of this module: +//! - Check whether test is timed out. +//! - Provide helpers for `report-time` and `measure-time` options. +//! - Provide newtypes for executions times. + +use std::env; +use std::fmt; +use std::str::FromStr; +use std::time::{Duration, Instant}; + +use super::types::{TestDesc, TestType}; + +pub const TEST_WARN_TIMEOUT_S: u64 = 60; + +/// This small module contains constants used by `report-time` option. +/// Those constants values will be used if corresponding environment variables are not set. +/// +/// To override values for unit-tests, use a constant `RUST_TEST_TIME_UNIT`, +/// To override values for integration tests, use a constant `RUST_TEST_TIME_INTEGRATION`, +/// To override values for doctests, use a constant `RUST_TEST_TIME_DOCTEST`. +/// +/// Example of the expected format is `RUST_TEST_TIME_xxx=100,200`, where 100 means +/// warn time, and 200 means critical time. +pub mod time_constants { + use super::TEST_WARN_TIMEOUT_S; + use std::time::Duration; + + /// Environment variable for overriding default threshold for unit-tests. + pub const UNIT_ENV_NAME: &str = "RUST_TEST_TIME_UNIT"; + + // Unit tests are supposed to be really quick. + pub const UNIT_WARN: Duration = Duration::from_millis(50); + pub const UNIT_CRITICAL: Duration = Duration::from_millis(100); + + /// Environment variable for overriding default threshold for unit-tests. + pub const INTEGRATION_ENV_NAME: &str = "RUST_TEST_TIME_INTEGRATION"; + + // Integration tests may have a lot of work, so they can take longer to execute. + pub const INTEGRATION_WARN: Duration = Duration::from_millis(500); + pub const INTEGRATION_CRITICAL: Duration = Duration::from_millis(1000); + + /// Environment variable for overriding default threshold for unit-tests. + pub const DOCTEST_ENV_NAME: &str = "RUST_TEST_TIME_DOCTEST"; + + // Doctests are similar to integration tests, because they can include a lot of + // initialization code. + pub const DOCTEST_WARN: Duration = INTEGRATION_WARN; + pub const DOCTEST_CRITICAL: Duration = INTEGRATION_CRITICAL; + + // Do not suppose anything about unknown tests, base limits on the + // `TEST_WARN_TIMEOUT_S` constant. + pub const UNKNOWN_WARN: Duration = Duration::from_secs(TEST_WARN_TIMEOUT_S); + pub const UNKNOWN_CRITICAL: Duration = Duration::from_secs(TEST_WARN_TIMEOUT_S * 2); +} + +/// Returns an `Instance` object denoting when the test should be considered +/// timed out. +pub fn get_default_test_timeout() -> Instant { + Instant::now() + Duration::from_secs(TEST_WARN_TIMEOUT_S) +} + +/// The measured execution time of a unit test. +#[derive(Debug, Clone, PartialEq)] +pub struct TestExecTime(pub Duration); + +impl fmt::Display for TestExecTime { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:.3}s", self.0.as_secs_f64()) + } +} + +/// The measured execution time of the whole test suite. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct TestSuiteExecTime(pub Duration); + +impl fmt::Display for TestSuiteExecTime { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:.2}s", self.0.as_secs_f64()) + } +} + +/// Structure denoting time limits for test execution. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub struct TimeThreshold { + pub warn: Duration, + pub critical: Duration, +} + +impl TimeThreshold { + /// Creates a new `TimeThreshold` instance with provided durations. + pub fn new(warn: Duration, critical: Duration) -> Self { + Self { warn, critical } + } + + /// Attempts to create a `TimeThreshold` instance with values obtained + /// from the environment variable, and returns `None` if the variable + /// is not set. + /// Environment variable format is expected to match `\d+,\d+`. + /// + /// # Panics + /// + /// Panics if variable with provided name is set but contains inappropriate + /// value. + pub fn from_env_var(env_var_name: &str) -> Option { + let durations_str = env::var(env_var_name).ok()?; + let (warn_str, critical_str) = durations_str.split_once(',').unwrap_or_else(|| { + panic!( + "Duration variable {env_var_name} expected to have 2 numbers separated by comma, but got {durations_str}" + ) + }); + + let parse_u64 = |v| { + u64::from_str(v).unwrap_or_else(|_| { + panic!( + "Duration value in variable {env_var_name} is expected to be a number, but got {v}" + ) + }) + }; + + let warn = parse_u64(warn_str); + let critical = parse_u64(critical_str); + if warn > critical { + panic!("Test execution warn time should be less or equal to the critical time"); + } + + Some(Self::new(Duration::from_millis(warn), Duration::from_millis(critical))) + } +} + +/// Structure with parameters for calculating test execution time. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub struct TestTimeOptions { + /// Denotes if the test critical execution time limit excess should be considered + /// a test failure. + pub error_on_excess: bool, + pub unit_threshold: TimeThreshold, + pub integration_threshold: TimeThreshold, + pub doctest_threshold: TimeThreshold, +} + +impl TestTimeOptions { + pub fn new_from_env(error_on_excess: bool) -> Self { + let unit_threshold = TimeThreshold::from_env_var(time_constants::UNIT_ENV_NAME) + .unwrap_or_else(Self::default_unit); + + let integration_threshold = + TimeThreshold::from_env_var(time_constants::INTEGRATION_ENV_NAME) + .unwrap_or_else(Self::default_integration); + + let doctest_threshold = TimeThreshold::from_env_var(time_constants::DOCTEST_ENV_NAME) + .unwrap_or_else(Self::default_doctest); + + Self { error_on_excess, unit_threshold, integration_threshold, doctest_threshold } + } + + pub fn is_warn(&self, test: &TestDesc, exec_time: &TestExecTime) -> bool { + exec_time.0 >= self.warn_time(test) + } + + pub fn is_critical(&self, test: &TestDesc, exec_time: &TestExecTime) -> bool { + exec_time.0 >= self.critical_time(test) + } + + fn warn_time(&self, test: &TestDesc) -> Duration { + match test.test_type { + TestType::UnitTest => self.unit_threshold.warn, + TestType::IntegrationTest => self.integration_threshold.warn, + TestType::DocTest => self.doctest_threshold.warn, + TestType::Unknown => time_constants::UNKNOWN_WARN, + } + } + + fn critical_time(&self, test: &TestDesc) -> Duration { + match test.test_type { + TestType::UnitTest => self.unit_threshold.critical, + TestType::IntegrationTest => self.integration_threshold.critical, + TestType::DocTest => self.doctest_threshold.critical, + TestType::Unknown => time_constants::UNKNOWN_CRITICAL, + } + } + + fn default_unit() -> TimeThreshold { + TimeThreshold::new(time_constants::UNIT_WARN, time_constants::UNIT_CRITICAL) + } + + fn default_integration() -> TimeThreshold { + TimeThreshold::new(time_constants::INTEGRATION_WARN, time_constants::INTEGRATION_CRITICAL) + } + + fn default_doctest() -> TimeThreshold { + TimeThreshold::new(time_constants::DOCTEST_WARN, time_constants::DOCTEST_CRITICAL) + } +} diff --git a/crux-mir/lib/test/src/types.rs b/crux-mir/lib/test/src/types.rs new file mode 100644 index 000000000..6f2e03309 --- /dev/null +++ b/crux-mir/lib/test/src/types.rs @@ -0,0 +1,168 @@ +//! Common types used by `libtest`. + +use std::borrow::Cow; +use std::fmt; + +use super::bench::Bencher; +use super::options; + +pub use NamePadding::*; +pub use TestFn::*; +pub use TestName::*; + +/// Type of the test according to the [rust book](https://doc.rust-lang.org/cargo/guide/tests.html) +/// conventions. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub enum TestType { + /// Unit-tests are expected to be in the `src` folder of the crate. + UnitTest, + /// Integration-style tests are expected to be in the `tests` folder of the crate. + IntegrationTest, + /// Doctests are created by the `librustdoc` manually, so it's a different type of test. + DocTest, + /// Tests for the sources that don't follow the project layout convention + /// (e.g. tests in raw `main.rs` compiled by calling `rustc --test` directly). + Unknown, +} + +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +pub enum NamePadding { + PadNone, + PadOnRight, +} + +// The name of a test. By convention this follows the rules for rust +// paths; i.e., it should be a series of identifiers separated by double +// colons. This way if some test runner wants to arrange the tests +// hierarchically it may. +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub enum TestName { + StaticTestName(&'static str), + DynTestName(String), + AlignedTestName(Cow<'static, str>, NamePadding), +} + +impl TestName { + pub fn as_slice(&self) -> &str { + match *self { + StaticTestName(s) => s, + DynTestName(ref s) => s, + AlignedTestName(ref s, _) => s, + } + } + + pub fn padding(&self) -> NamePadding { + match self { + &AlignedTestName(_, p) => p, + _ => PadNone, + } + } + + pub fn with_padding(&self, padding: NamePadding) -> TestName { + let name = match *self { + TestName::StaticTestName(name) => Cow::Borrowed(name), + TestName::DynTestName(ref name) => Cow::Owned(name.clone()), + TestName::AlignedTestName(ref name, _) => name.clone(), + }; + + TestName::AlignedTestName(name, padding) + } +} +impl fmt::Display for TestName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self.as_slice(), f) + } +} + +// A function that runs a test. If the function returns successfully, +// the test succeeds; if the function panics or returns Result::Err +// then the test fails. We may need to come up with a more clever +// definition of test in order to support isolation of tests into +// threads. +pub enum TestFn { + StaticTestFn(fn() -> Result<(), String>), + StaticBenchFn(fn(&mut Bencher) -> Result<(), String>), + DynTestFn(Box Result<(), String> + Send>), + DynBenchFn(Box Result<(), String> + Send>), +} + +impl TestFn { + pub fn padding(&self) -> NamePadding { + match *self { + StaticTestFn(..) => PadNone, + StaticBenchFn(..) => PadOnRight, + DynTestFn(..) => PadNone, + DynBenchFn(..) => PadOnRight, + } + } +} + +impl fmt::Debug for TestFn { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match *self { + StaticTestFn(..) => "StaticTestFn(..)", + StaticBenchFn(..) => "StaticBenchFn(..)", + DynTestFn(..) => "DynTestFn(..)", + DynBenchFn(..) => "DynBenchFn(..)", + }) + } +} + +// A unique integer associated with each test. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct TestId(pub usize); + +// The definition of a single test. A test runner will run a list of +// these. +#[derive(Clone, Debug)] +pub struct TestDesc { + pub name: TestName, + pub ignore: bool, + pub ignore_message: Option<&'static str>, + pub should_panic: options::ShouldPanic, + pub compile_fail: bool, + pub no_run: bool, + pub test_type: TestType, +} + +impl TestDesc { + pub fn padded_name(&self, column_count: usize, align: NamePadding) -> String { + let mut name = String::from(self.name.as_slice()); + let fill = column_count.saturating_sub(name.len()); + let pad = " ".repeat(fill); + match align { + PadNone => name, + PadOnRight => { + name.push_str(&pad); + name + } + } + } + + /// Returns None for ignored test or that that are just run, otherwise give a description of the type of test. + /// Descriptions include "should panic", "compile fail" and "compile". + pub fn test_mode(&self) -> Option<&'static str> { + if self.ignore { + return None; + } + match self.should_panic { + options::ShouldPanic::Yes | options::ShouldPanic::YesWithMessage(_) => { + return Some("should panic"); + } + options::ShouldPanic::No => {} + } + if self.compile_fail { + return Some("compile fail"); + } + if self.no_run { + return Some("compile"); + } + None + } +} + +#[derive(Debug)] +pub struct TestDescAndFn { + pub desc: TestDesc, + pub testfn: TestFn, +} diff --git a/crux-mir/lib/unicode_width/.cargo-ok b/crux-mir/lib/unicode_width/.cargo-ok new file mode 100644 index 000000000..b5754e203 --- /dev/null +++ b/crux-mir/lib/unicode_width/.cargo-ok @@ -0,0 +1 @@ +ok \ No newline at end of file diff --git a/crux-mir/lib/unicode_width/.cargo_vcs_info.json b/crux-mir/lib/unicode_width/.cargo_vcs_info.json new file mode 100644 index 000000000..df61a6cd4 --- /dev/null +++ b/crux-mir/lib/unicode_width/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "f444a314efeda9c130db6d6a96e925db0ea1ed13" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/crux-mir/lib/unicode_width/.gitignore b/crux-mir/lib/unicode_width/.gitignore new file mode 100644 index 000000000..5cdcdba3f --- /dev/null +++ b/crux-mir/lib/unicode_width/.gitignore @@ -0,0 +1,3 @@ +target +Cargo.lock +scripts/tmp diff --git a/crux-mir/lib/unicode_width/.travis.yml b/crux-mir/lib/unicode_width/.travis.yml new file mode 100644 index 000000000..64196faad --- /dev/null +++ b/crux-mir/lib/unicode_width/.travis.yml @@ -0,0 +1,28 @@ +language: rust +rust: 'nightly' +sudo: false +script: + - cargo build --verbose --features bench + - cargo test --verbose --features bench + - cargo bench --verbose --features bench + - cargo clean + - cargo build --verbose + - cargo test --verbose +# next line is an ugly hack to fix an annoying bug where rustdoc tries to use the rustc_private unicode_width crate +# (there is probably a better fix than this) + - rm $(find /home/travis/.rustup -type f -name 'libunicode_width*') + - rustdoc --test README.md -L target/debug -L target/debug/deps + - cargo doc +after_success: | + [ $TRAVIS_BRANCH = master ] && + [ $TRAVIS_PULL_REQUEST = false ] && + echo '' > target/doc/index.html && + pip install ghp-import --user $USER && + $HOME/.local/bin/ghp-import -n target/doc && + git push -qf https://${TOKEN}@github.com/${TRAVIS_REPO_SLUG}.git gh-pages +env: + global: + secure: vHL3zrN8AF+H79jrB8OfzuPqsUHevo6ECzwqXPj2dMSqcSXEeCY/ENAfiyFg+oW8yEVP8X2BS1a/C9yvVQRLqLbm1HbZ/5vUpoggT9S0IhKqZMyAcLYXfIEUDMDQuaSdFndDaHvq8275ScgX1LRv1kcPjQoZHuaXWMH8y/Suvyo= +notifications: + email: + on_success: never diff --git a/crux-mir/lib/unicode_width/COPYRIGHT b/crux-mir/lib/unicode_width/COPYRIGHT new file mode 100644 index 000000000..b286ec16a --- /dev/null +++ b/crux-mir/lib/unicode_width/COPYRIGHT @@ -0,0 +1,7 @@ +Licensed under the Apache License, Version 2.0 + or the MIT +license , +at your option. All files in the project carrying such +notice may not be copied, modified, or distributed except +according to those terms. diff --git a/crux-mir/lib/unicode_width/Cargo.toml b/crux-mir/lib/unicode_width/Cargo.toml new file mode 100644 index 000000000..77aded9fc --- /dev/null +++ b/crux-mir/lib/unicode_width/Cargo.toml @@ -0,0 +1,60 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +name = "unicode-width" +version = "0.1.10" +authors = [ + "kwantam ", + "Manish Goregaokar ", +] +exclude = [ + "target/*", + "Cargo.lock", +] +description = """ +Determine displayed width of `char` and `str` types +according to Unicode Standard Annex #11 rules. +""" +homepage = "https://github.com/unicode-rs/unicode-width" +documentation = "https://unicode-rs.github.io/unicode-width" +readme = "README.md" +keywords = [ + "text", + "width", + "unicode", +] +license = "MIT/Apache-2.0" +repository = "https://github.com/unicode-rs/unicode-width" + +[dependencies.compiler_builtins] +version = "0.1" +optional = true + +[dependencies.core] +version = "1.0" +optional = true +package = "rustc-std-workspace-core" + +[dependencies.std] +version = "1.0" +optional = true +package = "rustc-std-workspace-std" + +[features] +bench = [] +default = [] +no_std = [] +rustc-dep-of-std = [ + "std", + "core", + "compiler_builtins", +] diff --git a/crux-mir/lib/unicode_width/Cargo.toml.orig b/crux-mir/lib/unicode_width/Cargo.toml.orig new file mode 100644 index 000000000..993b35fb4 --- /dev/null +++ b/crux-mir/lib/unicode_width/Cargo.toml.orig @@ -0,0 +1,29 @@ +[package] + +name = "unicode-width" +version = "0.1.10" +authors = ["kwantam ", "Manish Goregaokar "] + +homepage = "https://github.com/unicode-rs/unicode-width" +repository = "https://github.com/unicode-rs/unicode-width" +documentation = "https://unicode-rs.github.io/unicode-width" +license = "MIT/Apache-2.0" +keywords = ["text", "width", "unicode"] +readme = "README.md" +description = """ +Determine displayed width of `char` and `str` types +according to Unicode Standard Annex #11 rules. +""" + +exclude = [ "target/*", "Cargo.lock" ] + +[dependencies] +std = { version = "1.0", package = "rustc-std-workspace-std", optional = true } +core = { version = "1.0", package = "rustc-std-workspace-core", optional = true } +compiler_builtins = { version = "0.1", optional = true } + +[features] +default = [] +no_std = [] +bench = [] +rustc-dep-of-std = ['std', 'core', 'compiler_builtins'] diff --git a/crux-mir/lib/unicode_width/LICENSE-APACHE b/crux-mir/lib/unicode_width/LICENSE-APACHE new file mode 100644 index 000000000..16fe87b06 --- /dev/null +++ b/crux-mir/lib/unicode_width/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/crux-mir/lib/unicode_width/LICENSE-MIT b/crux-mir/lib/unicode_width/LICENSE-MIT new file mode 100644 index 000000000..e69282e38 --- /dev/null +++ b/crux-mir/lib/unicode_width/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2015 The Rust Project Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/crux-mir/lib/unicode_width/README.md b/crux-mir/lib/unicode_width/README.md new file mode 100644 index 000000000..595e16380 --- /dev/null +++ b/crux-mir/lib/unicode_width/README.md @@ -0,0 +1,58 @@ +# unicode-width + +Determine displayed width of `char` and `str` types according to +[Unicode Standard Annex #11][UAX11] rules. + +[UAX11]: http://www.unicode.org/reports/tr11/ + +[![Build Status](https://travis-ci.org/unicode-rs/unicode-width.svg)](https://travis-ci.org/unicode-rs/unicode-width) + +[Documentation](https://unicode-rs.github.io/unicode-width/unicode_width/index.html) + +```rust +extern crate unicode_width; + +use unicode_width::UnicodeWidthStr; + +fn main() { + let teststr = "Ｈｅｌｌｏ, ｗｏｒｌｄ!"; + let width = UnicodeWidthStr::width(teststr); + println!("{}", teststr); + println!("The above string is {} columns wide.", width); + let width = teststr.width_cjk(); + println!("The above string is {} columns wide (CJK).", width); +} +``` + +**NOTE:** The computed width values may not match the actual rendered column +width. For example, the woman scientist emoji comprises of a woman emoji, a +zero-width joiner and a microscope emoji. + +```rust +extern crate unicode_width; +use unicode_width::UnicodeWidthStr; + +fn main() { + assert_eq!(UnicodeWidthStr::width("👩"), 2); // Woman + assert_eq!(UnicodeWidthStr::width("🔬"), 2); // Microscope + assert_eq!(UnicodeWidthStr::width("👩‍🔬"), 4); // Woman scientist +} +``` + +See [Unicode Standard Annex #11][UAX11] for precise details on what is and isn't +covered by this crate. + +## features + +unicode-width does not depend on libstd, so it can be used in crates +with the `#![no_std]` attribute. + +## crates.io + +You can use this package in your project by adding the following +to your `Cargo.toml`: + +```toml +[dependencies] +unicode-width = "0.1.7" +``` diff --git a/crux-mir/lib/unicode_width/scripts/unicode.py b/crux-mir/lib/unicode_width/scripts/unicode.py new file mode 100755 index 000000000..2efb0b63f --- /dev/null +++ b/crux-mir/lib/unicode_width/scripts/unicode.py @@ -0,0 +1,505 @@ +#!/usr/bin/env python3 +# +# Copyright 2011-2022 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +# This script uses the following Unicode tables: +# - EastAsianWidth.txt +# - ReadMe.txt +# - UnicodeData.txt +# +# Since this should not require frequent updates, we just store this +# out-of-line and check the generated module into git. + +import enum +import math +import os +import re +import sys + +NUM_CODEPOINTS = 0x110000 +"""An upper bound for which `range(0, NUM_CODEPOINTS)` contains Unicode's codespace.""" + +MAX_CODEPOINT_BITS = math.ceil(math.log2(NUM_CODEPOINTS - 1)) +"""The maximum number of bits required to represent a Unicode codepoint.""" + + +class OffsetType(enum.IntEnum): + """Represents the data type of a lookup table's offsets. Each variant's value represents the + number of bits required to represent that variant's type.""" + + U2 = 2 + """Offsets are 2-bit unsigned integers, packed four-per-byte.""" + U4 = 4 + """Offsets are 4-bit unsigned integers, packed two-per-byte.""" + U8 = 8 + """Each offset is a single byte (u8).""" + + +TABLE_CFGS = [ + (13, MAX_CODEPOINT_BITS, OffsetType.U8), + (6, 13, OffsetType.U8), + (0, 6, OffsetType.U2), +] +"""Represents the format of each level of the multi-level lookup table. +A level's entry is of the form `(low_bit, cap_bit, offset_type)`. +This means that every sub-table in that level is indexed by bits `low_bit..cap_bit` of the +codepoint and those tables offsets are stored according to `offset_type`. + +If this is edited, you must ensure that `emit_module` reflects your changes.""" + +MODULE_FILENAME = "tables.rs" +"""The filename of the emitted Rust module (will be created in the working directory)""" + +Codepoint = int +BitPos = int + + +def fetch_open(filename: str): + """Opens `filename` and return its corresponding file object. If `filename` isn't on disk, + fetches it from `http://www.unicode.org/Public/UNIDATA/`. Exits with code 1 on failure.""" + if not os.path.exists(os.path.basename(filename)): + os.system(f"curl -O http://www.unicode.org/Public/UNIDATA/{filename}") + try: + return open(filename, encoding="utf-8") + except OSError: + sys.stderr.write(f"cannot load {filename}") + sys.exit(1) + + +def load_unicode_version() -> "tuple[int, int, int]": + """Returns the current Unicode version by fetching and processing `ReadMe.txt`.""" + with fetch_open("ReadMe.txt") as readme: + pattern = r"for Version (\d+)\.(\d+)\.(\d+) of the Unicode" + return tuple(map(int, re.search(pattern, readme.read()).groups())) + + +class EffectiveWidth(enum.IntEnum): + """Represents the width of a Unicode character. All East Asian Width classes resolve into + either `EffectiveWidth.NARROW`, `EffectiveWidth.WIDE`, or `EffectiveWidth.AMBIGUOUS`.""" + + ZERO = 0 + """ Zero columns wide. """ + NARROW = 1 + """ One column wide. """ + WIDE = 2 + """ Two columns wide. """ + AMBIGUOUS = 3 + """ Two columns wide in a CJK context. One column wide in all other contexts. """ + + +def load_east_asian_widths() -> "list[EffectiveWidth]": + """Return a list of effective widths, indexed by codepoint. + Widths are determined by fetching and parsing `EastAsianWidth.txt`. + + `Neutral`, `Narrow`, and `Halfwidth` characters are assigned `EffectiveWidth.NARROW`. + + `Wide` and `Fullwidth` characters are assigned `EffectiveWidth.WIDE`. + + `Ambiguous` chracters are assigned `EffectiveWidth.AMBIGUOUS`.""" + with fetch_open("EastAsianWidth.txt") as eaw: + # matches a width assignment for a single codepoint, i.e. "1F336;N # ..." + single = re.compile(r"^([0-9A-F]+);(\w+) +# (\w+)") + # matches a width assignment for a range of codepoints, i.e. "3001..3003;W # ..." + multiple = re.compile(r"^([0-9A-F]+)\.\.([0-9A-F]+);(\w+) +# (\w+)") + # map between width category code and condensed width + width_codes = { + **{c: EffectiveWidth.NARROW for c in ["N", "Na", "H"]}, + **{c: EffectiveWidth.WIDE for c in ["W", "F"]}, + "A": EffectiveWidth.AMBIGUOUS, + } + + width_map = [] + current = 0 + for line in eaw.readlines(): + raw_data = None # (low, high, width) + if match := single.match(line): + raw_data = (match.group(1), match.group(1), match.group(2)) + elif match := multiple.match(line): + raw_data = (match.group(1), match.group(2), match.group(3)) + else: + continue + low = int(raw_data[0], 16) + high = int(raw_data[1], 16) + width = width_codes[raw_data[2]] + + assert current <= high + while current <= high: + # Some codepoints don't fall into any of the ranges in EastAsianWidth.txt. + # All such codepoints are implicitly given Neural width (resolves to narrow) + width_map.append(EffectiveWidth.NARROW if current < low else width) + current += 1 + + while len(width_map) < NUM_CODEPOINTS: + # Catch any leftover codepoints and assign them implicit Neutral/narrow width. + width_map.append(EffectiveWidth.NARROW) + + return width_map + + +def load_zero_widths() -> "list[bool]": + """Returns a list `l` where `l[c]` is true if codepoint `c` is considered a zero-width + character. `c` is considered a zero-width character if `c` is in general categories + `Cc`, `Cf`, `Mn`, or `Me` (determined by fetching and processing `UnicodeData.txt`).""" + with fetch_open("UnicodeData.txt") as categories: + zw_map = [] + current = 0 + for line in categories.readlines(): + if len(raw_data := line.split(";")) != 15: + continue + [codepoint, name, cat_code] = [ + int(raw_data[0], 16), + raw_data[1], + raw_data[2], + ] + zero_width = cat_code in ["Cc", "Cf", "Mn", "Me"] + + assert current <= codepoint + while current <= codepoint: + if name.endswith(", Last>") or current == codepoint: + # if name ends with Last, we backfill the width value to all codepoints since + # the previous codepoint (aka the start of the range) + zw_map.append(zero_width) + else: + # unassigned characters are implicitly given Neutral width, which is nonzero + zw_map.append(False) + current += 1 + + while len(zw_map) < NUM_CODEPOINTS: + # Catch any leftover codepoints. They must be unassigned (so nonzero width) + zw_map.append(False) + + return zw_map + + +class Bucket: + """A bucket contains a group of codepoints and an ordered width list. If one bucket's width + list overlaps with another's width list, those buckets can be merged via `try_extend`.""" + + def __init__(self): + """Creates an empty bucket.""" + self.entry_set = set() + self.widths = [] + + def append(self, codepoint: Codepoint, width: EffectiveWidth): + """Adds a codepoint/width pair to the bucket, and appends `width` to the width list.""" + self.entry_set.add((codepoint, width)) + self.widths.append(width) + + def try_extend(self, attempt: "Bucket") -> bool: + """If either `self` or `attempt`'s width list starts with the other bucket's width list, + set `self`'s width list to the longer of the two, add all of `attempt`'s codepoints + into `self`, and return `True`. Otherwise, return `False`.""" + (less, more) = (self.widths, attempt.widths) + if len(self.widths) > len(attempt.widths): + (less, more) = (attempt.widths, self.widths) + if less != more[: len(less)]: + return False + self.entry_set |= attempt.entry_set + self.widths = more + return True + + def entries(self) -> "list[tuple[Codepoint, EffectiveWidth]]": + """Return a list of the codepoint/width pairs in this bucket, sorted by codepoint.""" + result = list(self.entry_set) + result.sort() + return result + + def width(self) -> "EffectiveWidth": + """If all codepoints in this bucket have the same width, return that width; otherwise, + return `None`.""" + if len(self.widths) == 0: + return None + potential_width = self.widths[0] + for width in self.widths[1:]: + if potential_width != width: + return None + return potential_width + + +def make_buckets(entries, low_bit: BitPos, cap_bit: BitPos) -> "list[Bucket]": + """Partitions the `(Codepoint, EffectiveWidth)` tuples in `entries` into `Bucket`s. All + codepoints with identical bits from `low_bit` to `cap_bit` (exclusive) are placed in the + same bucket. Returns a list of the buckets in increasing order of those bits.""" + num_bits = cap_bit - low_bit + assert num_bits > 0 + buckets = [Bucket() for _ in range(0, 2 ** num_bits)] + mask = (1 << num_bits) - 1 + for (codepoint, width) in entries: + buckets[(codepoint >> low_bit) & mask].append(codepoint, width) + return buckets + + +class Table: + """Represents a lookup table. Each table contains a certain number of subtables; each + subtable is indexed by a contiguous bit range of the codepoint and contains a list + of `2**(number of bits in bit range)` entries. (The bit range is the same for all subtables.) + + Typically, tables contain a list of buckets of codepoints. Bucket `i`'s codepoints should + be indexed by sub-table `i` in the next-level lookup table. The entries of this table are + indexes into the bucket list (~= indexes into the sub-tables of the next-level table.) The + key to compression is that two different buckets in two different sub-tables may have the + same width list, which means that they can be merged into the same bucket. + + If no bucket contains two codepoints with different widths, calling `indices_to_widths` will + discard the buckets and convert the entries into `EffectiveWidth` values.""" + + def __init__( + self, entry_groups, low_bit: BitPos, cap_bit: BitPos, offset_type: OffsetType + ): + """Create a lookup table with a sub-table for each `(Codepoint, EffectiveWidth)` iterator + in `entry_groups`. Each sub-table is indexed by codepoint bits in `low_bit..cap_bit`, + and each table entry is represented in the format specified by `offset_type`. Asserts + that this table is actually representable with `offset_type`.""" + self.low_bit = low_bit + self.cap_bit = cap_bit + self.offset_type = offset_type + self.entries = [] + self.indexed = [] + + buckets = [] + for entries in entry_groups: + buckets.extend(make_buckets(entries, self.low_bit, self.cap_bit)) + + for bucket in buckets: + for (i, existing) in enumerate(self.indexed): + if existing.try_extend(bucket): + self.entries.append(i) + break + else: + self.entries.append(len(self.indexed)) + self.indexed.append(bucket) + + # Validate offset type + for index in self.entries: + assert index < (1 << int(self.offset_type)) + + def indices_to_widths(self): + """Destructively converts the indices in this table to the `EffectiveWidth` values of + their buckets. Assumes that no bucket contains codepoints with different widths.""" + self.entries = list(map(lambda i: int(self.indexed[i].width()), self.entries)) + del self.indexed + + def buckets(self): + """Returns an iterator over this table's buckets.""" + return self.indexed + + def to_bytes(self) -> "list[int]": + """Returns this table's entries as a list of bytes. The bytes are formatted according to + the `OffsetType` which the table was created with, converting any `EffectiveWidth` entries + to their enum variant's integer value. For example, with `OffsetType.U2`, each byte will + contain four packed 2-bit entries.""" + entries_per_byte = 8 // int(self.offset_type) + byte_array = [] + for i in range(0, len(self.entries), entries_per_byte): + byte = 0 + for j in range(0, entries_per_byte): + byte |= self.entries[i + j] << (j * int(self.offset_type)) + byte_array.append(byte) + return byte_array + + +def make_tables( + table_cfgs: "list[tuple[BitPos, BitPos, OffsetType]]", entries +) -> "list[Table]": + """Creates a table for each configuration in `table_cfgs`, with the first config corresponding + to the top-level lookup table, the second config corresponding to the second-level lookup + table, and so forth. `entries` is an iterator over the `(Codepoint, EffectiveWidth)` pairs + to include in the top-level table.""" + tables = [] + entry_groups = [entries] + for (low_bit, cap_bit, offset_type) in table_cfgs: + table = Table(entry_groups, low_bit, cap_bit, offset_type) + entry_groups = map(lambda bucket: bucket.entries(), table.buckets()) + tables.append(table) + return tables + + +def emit_module( + out_name: str, unicode_version: "tuple[int, int, int]", tables: "list[Table]" +): + """Outputs a Rust module to `out_name` using table data from `tables`. + If `TABLE_CFGS` is edited, you may need to edit the included code for `lookup_width`.""" + if os.path.exists(out_name): + os.remove(out_name) + with open(out_name, "w", newline="\n", encoding="utf-8") as module: + module.write( + """// Copyright 2012-2022 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// NOTE: The following code was generated by "scripts/unicode.py", do not edit directly +""" + ) + module.write( + f""" +/// The version of [Unicode](http://www.unicode.org/) +/// that this version of unicode-width is based on. +pub const UNICODE_VERSION: (u8, u8, u8) = {unicode_version}; +""" + ) + + module.write( + """ +pub mod charwidth { + use core::option::Option::{self, None, Some}; + + /// Returns the [UAX #11](https://www.unicode.org/reports/tr11/) based width of `c` by + /// consulting a multi-level lookup table. + /// If `is_cjk == true`, ambiguous width characters are treated as double width; otherwise, + /// they're treated as single width. + /// + /// # Maintenance + /// The tables themselves are autogenerated but this function is hardcoded. You should have + /// nothing to worry about if you re-run `unicode.py` (for example, when updating Unicode.) + /// However, if you change the *actual structure* of the lookup tables (perhaps by editing the + /// `TABLE_CFGS` global in `unicode.py`) you must ensure that this code reflects those changes. + #[inline] + fn lookup_width(c: char, is_cjk: bool) -> usize { + let cp = c as usize; + + let t1_offset = TABLES_0[cp >> 13 & 0xFF]; + + // Each sub-table in TABLES_1 is 7 bits, and each stored entry is a byte, + // so each sub-table is 128 bytes in size. + // (Sub-tables are selected using the computed offset from the previous table.) + let t2_offset = TABLES_1[128 * usize::from(t1_offset) + (cp >> 6 & 0x7F)]; + + // Each sub-table in TABLES_2 is 6 bits, but each stored entry is 2 bits. + // This is accomplished by packing four stored entries into one byte. + // So each sub-table is 2**(6-2) == 16 bytes in size. + // Since this is the last table, each entry represents an encoded width. + let packed_widths = TABLES_2[16 * usize::from(t2_offset) + (cp >> 2 & 0xF)]; + + // Extract the packed width + let width = packed_widths >> (2 * (cp & 0b11)) & 0b11; + + // A width of 3 signifies that the codepoint is ambiguous width. + if width == 3 { + if is_cjk { + 2 + } else { + 1 + } + } else { + width.into() + } + } +""" + ) + + module.write( + """ + /// Returns the [UAX #11](https://www.unicode.org/reports/tr11/) based width of `c`, or + /// `None` if `c` is a control character other than `'\\x00'`. + /// If `is_cjk == true`, ambiguous width characters are treated as double width; otherwise, + /// they're treated as single width. + #[inline] + pub fn width(c: char, is_cjk: bool) -> Option { + if c < '\\u{7F}' { + if c >= '\\u{20}' { + // U+0020 to U+007F (exclusive) are single-width ASCII codepoints + Some(1) + } else if c == '\\0' { + // U+0000 *is* a control code, but it's special-cased + Some(0) + } else { + // U+0001 to U+0020 (exclusive) are control codes + None + } + } else if c >= '\\u{A0}' { + // No characters >= U+00A0 are control codes, so we can consult the lookup tables + Some(lookup_width(c, is_cjk)) + } else { + // U+007F to U+00A0 (exclusive) are control codes + None + } + } +""" + ) + + subtable_count = 1 + for (i, table) in enumerate(tables): + new_subtable_count = len(table.buckets()) + if i == len(tables) - 1: + table.indices_to_widths() # for the last table, indices == widths + byte_array = table.to_bytes() + module.write( + f""" + /// Autogenerated. {subtable_count} sub-table(s). Consult [`lookup_width`] for layout info. + static TABLES_{i}: [u8; {len(byte_array)}] = [""" + ) + for (j, byte) in enumerate(byte_array): + # Add line breaks for every 15th entry (chosen to match what rustfmt does) + if j % 15 == 0: + module.write("\n ") + module.write(f" 0x{byte:02X},") + module.write("\n ];\n") + subtable_count = new_subtable_count + module.write("}\n") + + +def main(module_filename: str): + """Obtain character data from the latest version of Unicode, transform it into a multi-level + lookup table for character width, and write a Rust module utilizing that table to + `module_filename`. + + We obey the following rules in decreasing order of importance: + - The soft hyphen (`U+00AD`) is single-width. + - Hangul Jamo medial vowels & final consonants (`U+1160..=U+11FF`) are zero-width. + - All codepoints in general categories `Cc`, `Cf`, `Mn`, and `Me` are zero-width. + - All codepoints with an East Asian Width of `Ambigous` are ambiguous-width. + - All codepoints with an East Asian Width of `Wide` or `Fullwidth` are double-width. + - All other codepoints (including unassigned codepoints and codepoints with an East Asian Width + of `Neutral`, `Narrow`, or `Halfwidth`) are single-width. + + These rules are based off of Markus Kuhn's free `wcwidth()` implementation: + http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c""" + version = load_unicode_version() + print(f"Generating module for Unicode {version[0]}.{version[1]}.{version[2]}") + + eaw_map = load_east_asian_widths() + zw_map = load_zero_widths() + + # Characters marked as zero-width in zw_map should be zero-width in the final map + width_map = list( + map(lambda x: EffectiveWidth.ZERO if x[1] else x[0], zip(eaw_map, zw_map)) + ) + + # Override for soft hyphen + width_map[0x00AD] = EffectiveWidth.NARROW + + # Override for Hangul Jamo medial vowels & final consonants + for i in range(0x1160, 0x11FF + 1): + width_map[i] = EffectiveWidth.ZERO + + tables = make_tables(TABLE_CFGS, enumerate(width_map)) + + print("------------------------") + total_size = 0 + for (i, table) in enumerate(tables): + size_bytes = len(table.to_bytes()) + print(f"Table {i} Size: {size_bytes} bytes") + total_size += size_bytes + print("------------------------") + print(f" Total Size: {total_size} bytes") + + emit_module(module_filename, version, tables) + print(f'Wrote to "{module_filename}"') + + +if __name__ == "__main__": + main(MODULE_FILENAME) diff --git a/crux-mir/lib/unicode_width/src/lib.rs b/crux-mir/lib/unicode_width/src/lib.rs new file mode 100644 index 000000000..1ee35c85d --- /dev/null +++ b/crux-mir/lib/unicode_width/src/lib.rs @@ -0,0 +1,131 @@ +// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! Determine displayed width of `char` and `str` types according to +//! [Unicode Standard Annex #11](http://www.unicode.org/reports/tr11/) +//! rules. +//! +//! ```rust +//! extern crate unicode_width; +//! +//! use unicode_width::UnicodeWidthStr; +//! +//! fn main() { +//! let teststr = "Ｈｅｌｌｏ, ｗｏｒｌｄ!"; +//! let width = UnicodeWidthStr::width(teststr); +//! println!("{}", teststr); +//! println!("The above string is {} columns wide.", width); +//! let width = teststr.width_cjk(); +//! println!("The above string is {} columns wide (CJK).", width); +//! } +//! ``` +//! +//! # features +//! +//! unicode-width supports a `no_std` feature. This eliminates dependence +//! on std, and instead uses equivalent functions from core. +//! +//! # crates.io +//! +//! You can use this package in your project by adding the following +//! to your `Cargo.toml`: +//! +//! ```toml +//! [dependencies] +//! unicode-width = "0.1.5" +//! ``` + +#![deny(missing_docs, unsafe_code)] +#![doc(html_logo_url = "https://unicode-rs.github.io/unicode-rs_sm.png", + html_favicon_url = "https://unicode-rs.github.io/unicode-rs_sm.png")] + +#![cfg_attr(feature = "bench", feature(test))] +#![no_std] + +#[cfg(test)] +#[macro_use] +extern crate std; + +#[cfg(feature = "bench")] +extern crate test; + +use tables::charwidth as cw; +pub use tables::UNICODE_VERSION; + +use core::ops::Add; + +mod tables; + +#[cfg(test)] +mod tests; + +/// Methods for determining displayed width of Unicode characters. +pub trait UnicodeWidthChar { + /// Returns the character's displayed width in columns, or `None` if the + /// character is a control character other than `'\x00'`. + /// + /// This function treats characters in the Ambiguous category according + /// to [Unicode Standard Annex #11](http://www.unicode.org/reports/tr11/) + /// as 1 column wide. This is consistent with the recommendations for non-CJK + /// contexts, or when the context cannot be reliably determined. + fn width(self) -> Option; + + /// Returns the character's displayed width in columns, or `None` if the + /// character is a control character other than `'\x00'`. + /// + /// This function treats characters in the Ambiguous category according + /// to [Unicode Standard Annex #11](http://www.unicode.org/reports/tr11/) + /// as 2 columns wide. This is consistent with the recommendations for + /// CJK contexts. + fn width_cjk(self) -> Option; +} + +impl UnicodeWidthChar for char { + #[inline] + fn width(self) -> Option { cw::width(self, false) } + + #[inline] + fn width_cjk(self) -> Option { cw::width(self, true) } +} + +/// Methods for determining displayed width of Unicode strings. +pub trait UnicodeWidthStr { + /// Returns the string's displayed width in columns. + /// + /// Control characters are treated as having zero width. + /// + /// This function treats characters in the Ambiguous category according + /// to [Unicode Standard Annex #11](http://www.unicode.org/reports/tr11/) + /// as 1 column wide. This is consistent with the recommendations for + /// non-CJK contexts, or when the context cannot be reliably determined. + fn width<'a>(&'a self) -> usize; + + /// Returns the string's displayed width in columns. + /// + /// Control characters are treated as having zero width. + /// + /// This function treats characters in the Ambiguous category according + /// to [Unicode Standard Annex #11](http://www.unicode.org/reports/tr11/) + /// as 2 column wide. This is consistent with the recommendations for + /// CJK contexts. + fn width_cjk<'a>(&'a self) -> usize; +} + +impl UnicodeWidthStr for str { + #[inline] + fn width(&self) -> usize { + self.chars().map(|c| cw::width(c, false).unwrap_or(0)).fold(0, Add::add) + } + + #[inline] + fn width_cjk(&self) -> usize { + self.chars().map(|c| cw::width(c, true).unwrap_or(0)).fold(0, Add::add) + } +} diff --git a/crux-mir/lib/unicode_width/src/tables.rs b/crux-mir/lib/unicode_width/src/tables.rs new file mode 100644 index 000000000..439c69c80 --- /dev/null +++ b/crux-mir/lib/unicode_width/src/tables.rs @@ -0,0 +1,540 @@ +// Copyright 2012-2022 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// NOTE: The following code was generated by "scripts/unicode.py", do not edit directly + +/// The version of [Unicode](http://www.unicode.org/) +/// that this version of unicode-width is based on. +pub const UNICODE_VERSION: (u8, u8, u8) = (15, 0, 0); + +pub mod charwidth { + use core::option::Option::{self, None, Some}; + + /// Returns the [UAX #11](https://www.unicode.org/reports/tr11/) based width of `c` by + /// consulting a multi-level lookup table. + /// If `is_cjk == true`, ambiguous width characters are treated as double width; otherwise, + /// they're treated as single width. + /// + /// # Maintenance + /// The tables themselves are autogenerated but this function is hardcoded. You should have + /// nothing to worry about if you re-run `unicode.py` (for example, when updating Unicode.) + /// However, if you change the *actual structure* of the lookup tables (perhaps by editing the + /// `TABLE_CFGS` global in `unicode.py`) you must ensure that this code reflects those changes. + #[inline] + fn lookup_width(c: char, is_cjk: bool) -> usize { + let cp = c as usize; + + let t1_offset = TABLES_0[cp >> 13 & 0xFF]; + + // Each sub-table in TABLES_1 is 7 bits, and each stored entry is a byte, + // so each sub-table is 128 bytes in size. + // (Sub-tables are selected using the computed offset from the previous table.) + let t2_offset = TABLES_1[128 * usize::from(t1_offset) + (cp >> 6 & 0x7F)]; + + // Each sub-table in TABLES_2 is 6 bits, but each stored entry is 2 bits. + // This is accomplished by packing four stored entries into one byte. + // So each sub-table is 2**(6-2) == 16 bytes in size. + // Since this is the last table, each entry represents an encoded width. + let packed_widths = TABLES_2[16 * usize::from(t2_offset) + (cp >> 2 & 0xF)]; + + // Extract the packed width + let width = packed_widths >> (2 * (cp & 0b11)) & 0b11; + + // A width of 3 signifies that the codepoint is ambiguous width. + if width == 3 { + if is_cjk { + 2 + } else { + 1 + } + } else { + width.into() + } + } + + /// Returns the [UAX #11](https://www.unicode.org/reports/tr11/) based width of `c`, or + /// `None` if `c` is a control character other than `'\x00'`. + /// If `is_cjk == true`, ambiguous width characters are treated as double width; otherwise, + /// they're treated as single width. + #[inline] + pub fn width(c: char, is_cjk: bool) -> Option { + if c < '\u{7F}' { + if c >= '\u{20}' { + // U+0020 to U+007F (exclusive) are single-width ASCII codepoints + Some(1) + } else if c == '\0' { + // U+0000 *is* a control code, but it's special-cased + Some(0) + } else { + // U+0001 to U+0020 (exclusive) are control codes + None + } + } else if c >= '\u{A0}' { + // No characters >= U+00A0 are control codes, so we can consult the lookup tables + Some(lookup_width(c, is_cjk)) + } else { + // U+007F to U+00A0 (exclusive) are control codes + None + } + } + + /// Autogenerated. 1 sub-table(s). Consult [`lookup_width`] for layout info. + static TABLES_0: [u8; 256] = [ + 0x00, 0x01, 0x02, 0x03, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, + 0x0E, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x0F, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, + 0x03, 0x0F, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, + 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, + 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, + 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, + 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, + 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x10, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, + 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x12, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + + /// Autogenerated. 19 sub-table(s). Consult [`lookup_width`] for layout info. + static TABLES_1: [u8; 2432] = [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x06, 0x08, 0x06, 0x09, 0x0A, 0x0B, 0x0C, + 0x0D, 0x0E, 0x0F, 0x10, 0x06, 0x06, 0x06, 0x11, 0x12, 0x13, 0x14, 0x06, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x22, 0x24, 0x25, + 0x26, 0x27, 0x28, 0x29, 0x2A, 0x25, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, 0x30, 0x31, 0x32, 0x33, + 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x06, 0x3B, 0x3C, 0x0A, 0x0A, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x3D, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x3E, 0x3F, 0x40, 0x41, 0x42, 0x06, 0x43, 0x06, 0x44, 0x06, 0x06, 0x06, 0x45, + 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x06, 0x06, 0x4E, 0x06, 0x06, 0x06, 0x0A, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, + 0x56, 0x57, 0x58, 0x59, 0x06, 0x5A, 0x06, 0x06, 0x5B, 0x06, 0x5C, 0x5D, 0x5E, 0x5D, 0x5F, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x69, 0x6A, 0x06, 0x06, 0x06, 0x06, 0x06, 0x6B, + 0x06, 0x01, 0x06, 0x6C, 0x06, 0x06, 0x6D, 0x6E, 0x3B, 0x3B, 0x3B, 0x6F, 0x70, 0x71, 0x72, + 0x3B, 0x73, 0x3B, 0x74, 0x75, 0x76, 0x77, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x06, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x78, 0x79, 0x06, 0x06, 0x06, 0x06, 0x06, 0x7A, 0x7B, 0x7C, + 0x06, 0x06, 0x06, 0x06, 0x7D, 0x06, 0x06, 0x7E, 0x7F, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, + 0x86, 0x06, 0x06, 0x06, 0x87, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x88, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x89, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x8A, 0x8B, 0x06, 0x01, 0x71, 0x8C, 0x06, 0x8D, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x8E, 0x06, 0x06, 0x06, 0x8F, 0x06, 0x90, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x91, 0x06, 0x06, 0x92, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x93, 0x06, 0x06, 0x06, 0x06, 0x06, 0x94, 0x95, 0x06, 0x96, 0x97, 0x06, + 0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, 0x9E, 0x9F, 0xA0, 0x2E, 0x06, 0xA1, 0x2C, 0xA2, 0x06, + 0x06, 0xA3, 0xA4, 0xA5, 0xA6, 0x06, 0x06, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0x06, 0xAC, 0x06, + 0x06, 0x06, 0xAD, 0x06, 0x06, 0x06, 0xAE, 0xAF, 0x06, 0xB0, 0xB1, 0xB2, 0xB3, 0x06, 0x06, + 0x06, 0x06, 0x06, 0xB4, 0x06, 0xB5, 0x06, 0xB6, 0xB7, 0xB8, 0x06, 0x06, 0x06, 0x06, 0xB9, + 0xBA, 0xBB, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x47, 0xBC, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0xBD, 0xBE, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0xBF, 0xC0, 0xC1, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0xC2, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0xC3, 0xC4, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0xC5, 0x3B, 0x3B, 0x3B, 0x3B, 0xC6, + 0xC7, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0xC8, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0xC9, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0xCA, + 0xCB, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0xCC, 0xCD, 0x06, 0x06, 0xCE, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0xCF, 0xD0, + 0xD1, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0xD2, 0x06, 0xBF, 0x06, 0xBE, 0x06, 0x06, 0x06, + 0x06, 0x06, 0xD3, 0xD4, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0xD4, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0xD5, 0x06, 0xD6, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0xD7, 0x06, 0x06, 0xD8, + 0xD9, 0xDA, 0xDB, 0x06, 0xDC, 0xDD, 0x06, 0x06, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3, 0x3B, + 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0x3B, 0xE9, 0x3B, 0xEA, 0x06, 0x06, 0x06, 0xEB, 0x06, 0x06, + 0x06, 0x06, 0xEC, 0xED, 0x3B, 0x3B, 0x06, 0xEE, 0xEF, 0xF0, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, + 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0x3B, 0xE5, 0xF1, 0x0A, 0x06, 0x06, 0x0A, 0x0A, 0x0A, + 0x0B, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, 0x5D, + 0x5D, 0xF2, + ]; + + /// Autogenerated. 243 sub-table(s). Consult [`lookup_width`] for layout info. + static TABLES_2: [u8; 3888] = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5D, 0xD7, 0x77, 0x75, 0xFF, + 0xF7, 0x7F, 0xFF, 0x55, 0x75, 0x55, 0x55, 0x57, 0xD5, 0x57, 0xF5, 0x5F, 0x75, 0x7F, 0x5F, + 0xF7, 0xD5, 0x7F, 0x77, 0x5D, 0x55, 0x55, 0x55, 0xDD, 0x55, 0xD5, 0x55, 0x55, 0xF5, 0xD5, + 0x55, 0xFD, 0x55, 0x57, 0xD5, 0x7F, 0x57, 0xFF, 0x5D, 0xF5, 0x55, 0x55, 0x55, 0x55, 0xF5, + 0xD5, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x75, 0x77, 0x77, 0x77, 0x57, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x5D, 0x55, 0x55, + 0x55, 0x5D, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0xD7, 0xFD, 0x5D, 0x57, 0x55, + 0xFF, 0xDD, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0xFD, 0xFF, 0xFF, 0xFF, 0xDF, 0xFF, 0x5F, 0x55, 0xFD, 0xFF, 0xFF, 0xFF, 0xDF, 0xFF, + 0x5F, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x5D, + 0x55, 0x55, 0x55, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x5D, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x15, 0x00, 0x50, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x10, 0x41, 0x10, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x00, 0x50, 0x55, 0x55, 0x00, 0x00, 0x40, 0x54, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x55, + 0x55, 0x55, 0x54, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x05, 0x00, 0x10, 0x00, + 0x14, 0x04, 0x50, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x15, 0x51, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x05, 0x00, 0x00, 0x54, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x15, 0x00, 0x00, 0x55, 0x55, 0x51, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x05, 0x10, 0x00, 0x00, 0x01, 0x01, 0x50, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x01, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x50, 0x55, 0x00, 0x00, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x45, + 0x54, 0x01, 0x00, 0x54, 0x51, 0x01, 0x00, 0x55, 0x55, 0x05, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x51, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x54, 0x01, 0x54, 0x55, 0x51, 0x55, 0x55, 0x55, 0x55, 0x05, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x45, 0x41, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x54, 0x41, 0x15, 0x14, 0x50, 0x51, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x50, 0x51, 0x55, 0x55, 0x01, 0x10, 0x54, 0x51, 0x55, 0x55, 0x55, 0x55, 0x05, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x05, 0x00, 0x51, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x14, 0x01, 0x54, 0x55, 0x51, 0x55, 0x41, 0x55, + 0x55, 0x05, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x45, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x54, 0x55, 0x55, 0x51, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x54, 0x54, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x04, 0x54, 0x05, 0x04, + 0x50, 0x55, 0x41, 0x55, 0x55, 0x05, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x45, + 0x55, 0x50, 0x55, 0x55, 0x55, 0x55, 0x05, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x50, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x15, 0x54, + 0x01, 0x54, 0x55, 0x51, 0x55, 0x55, 0x55, 0x55, 0x05, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x51, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x45, 0x55, 0x05, 0x44, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x51, 0x00, 0x40, 0x55, 0x55, 0x15, 0x00, 0x40, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x51, 0x00, 0x00, 0x54, 0x55, 0x55, 0x00, 0x40, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x50, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x11, 0x51, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x01, 0x00, 0x00, 0x40, 0x00, 0x04, 0x55, 0x01, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x54, 0x55, 0x45, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x01, 0x04, 0x00, 0x41, 0x41, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x50, 0x05, 0x54, 0x55, 0x55, 0x55, 0x01, 0x54, 0x55, 0x55, 0x45, 0x41, + 0x55, 0x51, 0x55, 0x55, 0x55, 0x51, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x01, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x05, 0x54, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x05, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x05, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x05, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x10, 0x00, 0x50, 0x55, 0x45, 0x01, 0x00, 0x00, 0x55, 0x55, 0x51, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x15, 0x00, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x41, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x51, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x40, 0x15, 0x54, 0x55, 0x45, 0x55, 0x01, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x15, + 0x14, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x45, 0x00, 0x40, 0x44, 0x01, 0x00, 0x54, 0x15, 0x00, 0x00, 0x14, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x00, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x04, 0x40, 0x54, 0x45, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x15, 0x00, 0x00, 0x55, 0x55, 0x55, + 0x50, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x05, 0x50, 0x10, 0x50, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x45, 0x50, 0x11, 0x50, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x00, 0x00, + 0x05, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x40, 0x00, 0x00, 0x00, 0x04, 0x00, 0x54, 0x51, + 0x55, 0x54, 0x50, 0x55, 0x55, 0x55, 0x15, 0x00, 0xD7, 0x7F, 0x5F, 0x5F, 0x7F, 0xFF, 0x05, + 0x40, 0xF7, 0x5D, 0xD5, 0x75, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x00, 0x04, + 0x00, 0x00, 0x55, 0x57, 0x55, 0xD5, 0xFD, 0x57, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x57, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x54, 0x55, 0x55, 0x55, 0xD5, 0x5D, 0x5D, 0x55, 0xD5, 0x75, 0x55, + 0x55, 0x7D, 0x75, 0xD5, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0xD5, 0x57, + 0xD5, 0x7F, 0xFF, 0xFF, 0xFF, 0x55, 0xFF, 0xFF, 0x5F, 0x55, 0x55, 0x55, 0x5D, 0x55, 0xFF, + 0xFF, 0x5F, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x5F, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x75, 0x57, 0x55, 0x55, 0x55, 0xD5, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0xF7, 0xD5, 0xD7, + 0xD5, 0x5D, 0x5D, 0x75, 0xFD, 0xD7, 0xDD, 0xFF, 0x77, 0x55, 0xFF, 0x55, 0x5F, 0x55, 0x55, + 0x57, 0x57, 0x75, 0x55, 0x55, 0x55, 0x5F, 0xFF, 0xF5, 0xF5, 0x55, 0x55, 0x55, 0x55, 0xF5, + 0xF5, 0x55, 0x55, 0x55, 0x5D, 0x5D, 0x55, 0x55, 0x5D, 0x55, 0x55, 0x55, 0x55, 0x55, 0xD5, + 0x55, 0x55, 0x55, 0x55, 0x75, 0x55, 0xA5, 0x55, 0x55, 0x55, 0x69, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0xA9, 0x56, 0x96, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xDF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x55, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0x55, 0x55, 0x55, 0xFF, 0xFF, 0xFF, 0xFF, 0xF5, 0x5F, 0x55, 0x55, 0xDF, + 0xFF, 0x5F, 0x55, 0xF5, 0xF5, 0x55, 0x5F, 0x5F, 0xF5, 0xD7, 0xF5, 0x5F, 0x55, 0x55, 0x55, + 0xF5, 0x5F, 0x55, 0xD5, 0x55, 0x55, 0x55, 0x69, 0x55, 0x7D, 0x5D, 0xF5, 0x55, 0x5A, 0x55, + 0x77, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x77, 0x55, 0xAA, 0xAA, 0xAA, 0x55, + 0x55, 0x55, 0xDF, 0xDF, 0x7F, 0xDF, 0x55, 0x55, 0x55, 0x95, 0x55, 0x55, 0x55, 0x55, 0x95, + 0x55, 0x55, 0xF5, 0x59, 0x55, 0xA5, 0x55, 0x55, 0x55, 0x55, 0xE9, 0x55, 0xFA, 0xFF, 0xEF, + 0xFF, 0xFE, 0xFF, 0xFF, 0xDF, 0x55, 0xEF, 0xFF, 0xAF, 0xFB, 0xEF, 0xFB, 0x55, 0x59, 0xA5, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x56, 0x55, 0x55, 0x55, 0x55, 0x5D, 0x55, 0x55, + 0x55, 0x66, 0x95, 0x9A, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0xF5, 0xFF, 0xFF, 0x55, + 0x55, 0x55, 0x55, 0x55, 0xA9, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x56, 0x55, 0x55, 0x95, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x95, 0x56, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x56, 0xF9, 0x5F, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x15, 0x50, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x9A, 0xAA, 0xAA, 0xAA, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, + 0xAA, 0xAA, 0xAA, 0x55, 0x55, 0x55, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x5A, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0xAA, 0xAA, 0xAA, 0x55, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, + 0xAA, 0xAA, 0x0A, 0xA0, 0xAA, 0xAA, 0xAA, 0x6A, 0xA9, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x6A, + 0x81, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x55, 0xA9, 0xAA, 0xAA, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xA9, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x6A, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x55, 0x55, 0x55, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x6A, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, + 0xAA, 0xFF, 0xFF, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, + 0xAA, 0xAA, 0xAA, 0x56, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, + 0xAA, 0xAA, 0x6A, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x15, 0x40, + 0x00, 0x00, 0x50, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x05, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x50, 0x55, 0x55, 0x55, 0x45, 0x45, 0x15, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x41, + 0x55, 0x54, 0x55, 0x55, 0x55, 0x55, 0x55, 0x50, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x00, + 0x00, 0x00, 0x00, 0x50, 0x55, 0x55, 0x15, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x05, 0x00, 0x50, 0x55, 0x55, 0x55, 0x55, 0x55, 0x15, 0x00, 0x00, 0x50, 0x55, 0x55, + 0x55, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x56, 0x40, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x15, 0x05, 0x50, 0x50, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x51, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x01, 0x40, 0x41, 0x41, 0x55, 0x55, 0x15, 0x55, 0x55, + 0x54, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x54, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x04, 0x14, 0x54, 0x05, 0x51, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x50, 0x55, 0x45, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x51, 0x54, 0x51, 0x55, 0x55, 0x55, + 0x55, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x45, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x00, 0x00, 0x00, 0x00, 0xAA, 0xAA, 0x5A, 0x55, 0x00, 0x00, 0x00, 0x00, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x6A, 0xAA, 0xAA, 0xAA, 0xAA, 0x6A, 0xAA, + 0x55, 0x55, 0x55, 0x55, 0x55, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x56, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0xAA, + 0x6A, 0x55, 0x55, 0x55, 0x55, 0x01, 0x5D, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x51, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x54, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x05, 0x40, 0x55, 0x01, 0x41, 0x55, 0x00, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x40, 0x15, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x41, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x00, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x15, 0x54, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x01, + 0x55, 0x05, 0x00, 0x00, 0x54, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x05, 0x50, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x51, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x00, 0x00, 0x00, 0x40, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x14, 0x54, 0x55, 0x15, 0x50, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x15, 0x40, 0x41, 0x51, 0x45, 0x55, 0x55, 0x51, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x40, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x15, 0x00, 0x01, 0x00, 0x54, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x15, 0x55, 0x55, 0x55, 0x50, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x05, 0x00, 0x40, 0x55, 0x55, 0x01, 0x14, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x15, 0x50, 0x04, 0x55, 0x45, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x15, 0x15, 0x00, 0x40, 0x55, 0x55, 0x55, 0x55, 0x55, 0x54, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x05, 0x00, 0x54, 0x00, 0x54, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x00, 0x00, 0x05, + 0x44, 0x55, 0x55, 0x55, 0x55, 0x55, 0x45, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x15, 0x00, 0x44, + 0x15, 0x04, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x05, + 0x50, 0x55, 0x10, 0x54, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x50, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x15, 0x00, 0x40, 0x11, 0x54, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x15, 0x51, 0x00, 0x10, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x01, + 0x05, 0x10, 0x00, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x15, 0x00, 0x00, 0x41, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x15, 0x44, 0x15, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x00, 0x05, 0x55, 0x54, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x01, 0x00, 0x40, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x15, 0x00, 0x14, 0x40, 0x55, 0x15, + 0x55, 0x55, 0x01, 0x40, 0x01, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x05, 0x00, 0x00, 0x40, 0x50, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x00, 0x40, 0x00, + 0x10, 0x55, 0x55, 0x55, 0x55, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x04, 0x41, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x01, + 0x40, 0x45, 0x10, 0x00, 0x10, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x50, 0x11, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x15, 0x54, 0x55, 0x55, 0x50, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x05, 0x40, 0x55, 0x44, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x54, 0x15, 0x00, 0x00, 0x00, 0x50, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x00, 0x54, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x00, 0x40, 0x55, 0x55, 0x55, 0x55, 0x55, 0x15, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x15, 0x40, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0xAA, 0x54, 0x55, 0x55, 0x5A, 0x55, 0x55, 0x55, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x55, 0x55, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x5A, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0xAA, 0xAA, 0x56, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0xAA, + 0xA9, 0xAA, 0x69, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x6A, 0x55, 0x55, 0x55, + 0x65, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x6A, 0x59, 0x55, 0x55, 0x55, 0xAA, 0x55, + 0x55, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x41, 0x00, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x15, 0x50, 0x55, 0x15, 0x00, 0x00, 0x00, 0x40, 0x01, 0x00, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x05, 0x50, 0x55, 0x55, 0x55, 0x55, 0x05, 0x54, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x15, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x54, 0x55, 0x51, 0x55, 0x55, 0x55, + 0x54, 0x55, 0x55, 0x55, 0x55, 0x15, 0x00, 0x01, 0x00, 0x00, 0x00, 0x55, 0x55, 0x55, 0x55, + 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x10, 0x04, 0x40, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x45, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x00, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x00, 0x40, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x00, 0x40, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x56, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x95, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0xFF, 0xFF, 0x7F, 0x55, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0x5F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0x5F, 0x55, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xEF, 0xAB, 0xAA, + 0xEA, 0xFF, 0xFF, 0xFF, 0xFF, 0x57, 0x55, 0x55, 0x55, 0x55, 0x6A, 0x55, 0x55, 0x55, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x55, 0xAA, 0xAA, 0x56, 0x55, + 0x5A, 0x55, 0x55, 0x55, 0xAA, 0x5A, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0xAA, 0xAA, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x56, 0x55, 0x55, 0xA9, 0xAA, 0x9A, 0xAA, 0xAA, 0xAA, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xA6, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0x55, 0x55, 0x55, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, + 0xAA, 0xAA, 0x6A, 0x95, 0xAA, 0x55, 0x55, 0x55, 0xAA, 0xAA, 0xAA, 0xAA, 0x56, 0x56, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, + 0xAA, 0x6A, 0xA6, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, + 0xAA, 0xAA, 0xAA, 0x96, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0x5A, 0x55, 0x55, 0x95, 0x6A, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, + 0x55, 0x55, 0x55, 0x55, 0x65, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x69, 0x55, 0x55, 0x55, + 0x56, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x95, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0xAA, 0x5A, 0x55, 0x56, 0x6A, 0xA9, + 0x55, 0xAA, 0x55, 0x55, 0x95, 0x56, 0x55, 0xAA, 0xAA, 0x56, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0xAA, 0xAA, 0xAA, 0x55, 0x56, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x6A, 0xAA, 0xAA, 0x9A, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0xAA, 0xAA, 0xAA, 0x56, 0xAA, + 0xAA, 0x56, 0x55, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0x9A, + 0xAA, 0x5A, 0x55, 0xA5, 0xAA, 0xAA, 0xAA, 0x55, 0xAA, 0xAA, 0x56, 0x55, 0xAA, 0xAA, 0x56, + 0x55, 0x51, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0x5F, + ]; +} diff --git a/crux-mir/lib/unicode_width/src/tests.rs b/crux-mir/lib/unicode_width/src/tests.rs new file mode 100644 index 000000000..e49b1bfbe --- /dev/null +++ b/crux-mir/lib/unicode_width/src/tests.rs @@ -0,0 +1,191 @@ +// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#[cfg(feature = "bench")] +use std::iter; +#[cfg(feature = "bench")] +use test::{self, Bencher}; +#[cfg(feature = "bench")] +use super::{UnicodeWidthChar, UnicodeWidthStr}; + +use std::prelude::v1::*; + +#[cfg(feature = "bench")] +#[bench] +fn cargo(b: &mut Bencher) { + let string = iter::repeat('a').take(4096).collect::(); + + b.iter(|| { + for c in string.chars() { + test::black_box(UnicodeWidthChar::width(c)); + } + }); +} + +#[cfg(feature = "bench")] +#[bench] +#[allow(deprecated)] +fn stdlib(b: &mut Bencher) { + let string = iter::repeat('a').take(4096).collect::(); + + b.iter(|| { + for c in string.chars() { + test::black_box(c.width()); + } + }); +} + +#[cfg(feature = "bench")] +#[bench] +fn simple_if(b: &mut Bencher) { + let string = iter::repeat('a').take(4096).collect::(); + + b.iter(|| { + for c in string.chars() { + test::black_box(simple_width_if(c)); + } + }); +} + +#[cfg(feature = "bench")] +#[bench] +fn simple_match(b: &mut Bencher) { + let string = iter::repeat('a').take(4096).collect::(); + + b.iter(|| { + for c in string.chars() { + test::black_box(simple_width_match(c)); + } + }); +} + +#[cfg(feature = "bench")] +#[inline] +fn simple_width_if(c: char) -> Option { + let cu = c as u32; + if cu < 127 { + if cu > 31 { + Some(1) + } else if cu == 0 { + Some(0) + } else { + None + } + } else { + UnicodeWidthChar::width(c) + } +} + +#[cfg(feature = "bench")] +#[inline] +fn simple_width_match(c: char) -> Option { + match c as u32 { + cu if cu == 0 => Some(0), + cu if cu < 0x20 => None, + cu if cu < 0x7f => Some(1), + _ => UnicodeWidthChar::width(c) + } +} +#[cfg(all(feature = "bench", not(feature = "no_std")))] +#[bench] +fn enwik8(b: &mut Bencher) { + // To benchmark, download & unzip `enwik8` from https://data.deepai.org/enwik8.zip + let data_path = "bench_data/enwik8"; + let string = std::fs::read_to_string(data_path).unwrap_or_default(); + b.iter(|| test::black_box(UnicodeWidthStr::width(string.as_str()))); +} +#[cfg(all(feature = "bench", not(feature = "no_std")))] +#[bench] +fn jawiki(b: &mut Bencher) { + // To benchmark, download & extract `jawiki-20220501-pages-articles-multistream-index.txt` from + // https://dumps.wikimedia.org/jawiki/20220501/jawiki-20220501-pages-articles-multistream-index.txt.bz2 + let data_path = "bench_data/jawiki-20220501-pages-articles-multistream-index.txt"; + let string = std::fs::read_to_string(data_path).unwrap_or_default(); + b.iter(|| test::black_box(UnicodeWidthStr::width(string.as_str()))); +} +#[test] +fn test_str() { + use super::UnicodeWidthStr; + + assert_eq!(UnicodeWidthStr::width("ｈｅｌｌｏ"), 10); + assert_eq!("ｈｅｌｌｏ".width_cjk(), 10); + assert_eq!(UnicodeWidthStr::width("\0\0\0\x01\x01"), 0); + assert_eq!("\0\0\0\x01\x01".width_cjk(), 0); + assert_eq!(UnicodeWidthStr::width(""), 0); + assert_eq!("".width_cjk(), 0); + assert_eq!(UnicodeWidthStr::width("\u{2081}\u{2082}\u{2083}\u{2084}"), 4); + assert_eq!("\u{2081}\u{2082}\u{2083}\u{2084}".width_cjk(), 8); +} + +#[test] +fn test_emoji() { + // Example from the README. + use super::UnicodeWidthStr; + + assert_eq!(UnicodeWidthStr::width("👩"), 2); // Woman + assert_eq!(UnicodeWidthStr::width("🔬"), 2); // Microscope + assert_eq!(UnicodeWidthStr::width("👩‍🔬"), 4); // Woman scientist +} + +#[test] +fn test_char() { + use super::UnicodeWidthChar; + #[cfg(feature = "no_std")] + use core::option::Option::{Some, None}; + + assert_eq!(UnicodeWidthChar::width('ｈ'), Some(2)); + assert_eq!('ｈ'.width_cjk(), Some(2)); + assert_eq!(UnicodeWidthChar::width('\x00'), Some(0)); + assert_eq!('\x00'.width_cjk(), Some(0)); + assert_eq!(UnicodeWidthChar::width('\x01'), None); + assert_eq!('\x01'.width_cjk(), None); + assert_eq!(UnicodeWidthChar::width('\u{2081}'), Some(1)); + assert_eq!('\u{2081}'.width_cjk(), Some(2)); +} + +#[test] +fn test_char2() { + use super::UnicodeWidthChar; + #[cfg(feature = "no_std")] + use core::option::Option::{Some, None}; + + assert_eq!(UnicodeWidthChar::width('\x00'),Some(0)); + assert_eq!('\x00'.width_cjk(),Some(0)); + + assert_eq!(UnicodeWidthChar::width('\x0A'),None); + assert_eq!('\x0A'.width_cjk(),None); + + assert_eq!(UnicodeWidthChar::width('w'),Some(1)); + assert_eq!('w'.width_cjk(),Some(1)); + + assert_eq!(UnicodeWidthChar::width('ｈ'),Some(2)); + assert_eq!('ｈ'.width_cjk(),Some(2)); + + assert_eq!(UnicodeWidthChar::width('\u{AD}'),Some(1)); + assert_eq!('\u{AD}'.width_cjk(),Some(1)); + + assert_eq!(UnicodeWidthChar::width('\u{1160}'),Some(0)); + assert_eq!('\u{1160}'.width_cjk(),Some(0)); + + assert_eq!(UnicodeWidthChar::width('\u{a1}'),Some(1)); + assert_eq!('\u{a1}'.width_cjk(),Some(2)); + + assert_eq!(UnicodeWidthChar::width('\u{300}'),Some(0)); + assert_eq!('\u{300}'.width_cjk(),Some(0)); +} + +#[test] +fn unicode_12() { + use super::UnicodeWidthChar; + #[cfg(feature = "no_std")] + use core::option::Option::{Some, None}; + + assert_eq!(UnicodeWidthChar::width('\u{1F971}'), Some(2)); +} From 537dfbe0b5bf0e02f711c3bee6132a8a51e56750 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 29 Jun 2023 15:08:26 -0400 Subject: [PATCH 092/114] build.sh: Produce a proper sysroot This is needed to make `rustc`'s `--sysroot` flag behave as expected. --- crux-mir/build.sh | 70 +++++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/crux-mir/build.sh b/crux-mir/build.sh index e530fab12..34f26d15d 100755 --- a/crux-mir/build.sh +++ b/crux-mir/build.sh @@ -1,94 +1,106 @@ +#!/usr/bin/env bash + set -e -export STD_ENV_ARCH=$(uname -m) +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +RLIBS_PARENT="${SCRIPT_DIR}/rlibs" +RLIBS=$(rustc --print target-libdir --sysroot "${RLIBS_PARENT}") +STD_ENV_ARCH=$(uname -m) +export STD_ENV_ARCH translate() { - mir-json -L rlibs --out-dir rlibs --crate-type rlib --remap-path-prefix "$(pwd -P)=." "$@" + mir-json -L "${RLIBS}" --out-dir "${RLIBS}" --crate-type rlib --remap-path-prefix "$(pwd -P)=." "$@" } +mkdir -p "${RLIBS_PARENT}/bin" +mkdir -p "${RLIBS_PARENT}/etc" +mkdir -p "${RLIBS_PARENT}/lib" +mkdir -p "${RLIBS_PARENT}/libexec" +mkdir -p "${RLIBS_PARENT}/share" + echo 'Building core...' translate lib/core/src/lib.rs --edition=2021 --crate-name core echo 'Building rustc_std_workspace_core...' -translate lib/rustc_std_workspace_core/lib.rs --edition=2021 --crate-name rustc_std_workspace_core --extern core=rlibs/libcore.rlib +translate lib/rustc_std_workspace_core/lib.rs --edition=2021 --crate-name rustc_std_workspace_core --extern "core=${RLIBS}/libcore.rlib" echo 'Building libc...' -translate lib/libc/src/lib.rs --crate-name libc --cfg 'feature="align"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="rustc-std-workspace-core"' --cfg freebsd11 --cfg libc_priv_mod_use --cfg libc_union --cfg libc_const_size_of --cfg libc_align --cfg libc_int128 --cfg libc_core_cvoid --cfg libc_packedN --cfg libc_cfg_target_vendor --cfg libc_non_exhaustive --cfg libc_ptr_addr_of --cfg libc_underscore_const_names --cfg libc_thread_local --cfg 'libc_const_extern_fn`' --extern rustc_std_workspace_core=rlibs/librustc_std_workspace_core.rlib +translate lib/libc/src/lib.rs --crate-name libc --cfg 'feature="align"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="rustc-std-workspace-core"' --cfg freebsd11 --cfg libc_priv_mod_use --cfg libc_union --cfg libc_const_size_of --cfg libc_align --cfg libc_int128 --cfg libc_core_cvoid --cfg libc_packedN --cfg libc_cfg_target_vendor --cfg libc_non_exhaustive --cfg libc_ptr_addr_of --cfg libc_underscore_const_names --cfg libc_thread_local --cfg 'libc_const_extern_fn`' --extern "rustc_std_workspace_core=${RLIBS}/librustc_std_workspace_core.rlib" echo 'Building compiler_builtins...' -translate lib/compiler_builtins/src/lib.rs --crate-name compiler_builtins --cfg 'feature="compiler-builtins"' --cfg 'feature="core"' --cfg 'feature="default"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="unstable"' --cfg 'feature="mem-unaligned"`' --extern core=rlibs/libcore.rlib +translate lib/compiler_builtins/src/lib.rs --crate-name compiler_builtins --cfg 'feature="compiler-builtins"' --cfg 'feature="core"' --cfg 'feature="default"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="unstable"' --cfg 'feature="mem-unaligned"`' --extern "core=${RLIBS}/libcore.rlib" # extra lib (added manually) echo "Building crucible..." -translate lib/crucible/lib.rs --crate-name crucible --edition=2021 --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib +translate lib/crucible/lib.rs --crate-name crucible --edition=2021 --extern "compiler_builtins=${RLIBS}/libcompiler_builtins.rlib" --extern "core=${RLIBS}/libcore.rlib" echo 'Building alloc...' -translate lib/alloc/src/lib.rs --edition=2021 --crate-name alloc --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern crucible=rlibs/libcrucible.rlib +translate lib/alloc/src/lib.rs --edition=2021 --crate-name alloc --extern "compiler_builtins=${RLIBS}/libcompiler_builtins.rlib" --extern "core=${RLIBS}/libcore.rlib" --extern "crucible=${RLIBS}/libcrucible.rlib" echo 'Building cfg_if...' -translate lib/cfg_if/src/lib.rs --edition=2018 --crate-name cfg_if --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib +translate lib/cfg_if/src/lib.rs --edition=2018 --crate-name cfg_if --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern "compiler_builtins=${RLIBS}/libcompiler_builtins.rlib" --extern "core=${RLIBS}/libcore.rlib" echo 'Building memchr...' -translate lib/memchr/src/lib.rs --edition=2018 --crate-name memchr --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --cfg memchr_runtime_simd --cfg memchr_runtime_sse2 --cfg memchr_runtime_sse42 --cfg 'memchr_runtime_avx`' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib +translate lib/memchr/src/lib.rs --edition=2018 --crate-name memchr --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --cfg memchr_runtime_simd --cfg memchr_runtime_sse2 --cfg memchr_runtime_sse42 --cfg 'memchr_runtime_avx`' --extern "compiler_builtins=${RLIBS}/libcompiler_builtins.rlib" --extern "core=${RLIBS}/libcore.rlib" echo 'Building adler...' -translate lib/adler/src/lib.rs --crate-name adler --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib +translate lib/adler/src/lib.rs --crate-name adler --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern "compiler_builtins=${RLIBS}/libcompiler_builtins.rlib" --extern "core=${RLIBS}/libcore.rlib" echo 'Building rustc_demangle...' -translate lib/rustc_demangle/src/lib.rs --crate-name rustc_demangle --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib +translate lib/rustc_demangle/src/lib.rs --crate-name rustc_demangle --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern "compiler_builtins=${RLIBS}/libcompiler_builtins.rlib" --extern "core=${RLIBS}/libcore.rlib" echo 'Building unwind...' -translate lib/unwind/src/lib.rs --edition=2021 --crate-name unwind --extern cfg_if=rlibs/libcfg_if.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern libc=rlibs/liblibc.rlib +translate lib/unwind/src/lib.rs --edition=2021 --crate-name unwind --extern "cfg_if=${RLIBS}/libcfg_if.rlib" --extern "compiler_builtins=${RLIBS}/libcompiler_builtins.rlib" --extern "core=${RLIBS}/libcore.rlib" --extern "libc=${RLIBS}/liblibc.rlib" echo 'Building panic_unwind...' -translate lib/panic_unwind/src/lib.rs --edition=2021 --crate-name panic_unwind --extern alloc=rlibs/liballoc.rlib --extern cfg_if=rlibs/libcfg_if.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern libc=rlibs/liblibc.rlib --extern unwind=rlibs/libunwind.rlib +translate lib/panic_unwind/src/lib.rs --edition=2021 --crate-name panic_unwind --extern "alloc=${RLIBS}/liballoc.rlib" --extern "cfg_if=${RLIBS}/libcfg_if.rlib" --extern "compiler_builtins=${RLIBS}/libcompiler_builtins.rlib" --extern "core=${RLIBS}/libcore.rlib" --extern "libc=${RLIBS}/liblibc.rlib" --extern "unwind=${RLIBS}/libunwind.rlib" echo 'Building rustc_std_workspace_alloc...' -translate lib/rustc_std_workspace_alloc/lib.rs --edition=2021 --crate-name rustc_std_workspace_alloc --extern alloc=rlibs/liballoc.rlib +translate lib/rustc_std_workspace_alloc/lib.rs --edition=2021 --crate-name rustc_std_workspace_alloc --extern "alloc=${RLIBS}/liballoc.rlib" echo 'Building panic_abort...' -translate lib/panic_abort/src/lib.rs --edition=2021 --crate-name panic_abort --extern alloc=rlibs/liballoc.rlib --extern cfg_if=rlibs/libcfg_if.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern libc=rlibs/liblibc.rlib +translate lib/panic_abort/src/lib.rs --edition=2021 --crate-name panic_abort --extern "alloc=${RLIBS}/liballoc.rlib" --extern "cfg_if=${RLIBS}/libcfg_if.rlib" --extern "compiler_builtins=${RLIBS}/libcompiler_builtins.rlib" --extern "core=${RLIBS}/libcore.rlib" --extern "libc=${RLIBS}/liblibc.rlib" echo 'Building gimli...' -translate lib/gimli/src/lib.rs --edition=2018 --crate-name gimli --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="read"' --cfg 'feature="read-core"' --cfg 'feature="rustc-dep-of-std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib +translate lib/gimli/src/lib.rs --edition=2018 --crate-name gimli --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="read"' --cfg 'feature="read-core"' --cfg 'feature="rustc-dep-of-std"' --extern "compiler_builtins=${RLIBS}/libcompiler_builtins.rlib" --extern "alloc=${RLIBS}/liballoc.rlib" --extern "core=${RLIBS}/libcore.rlib" echo 'Building std_detect...' -translate lib/std_detect/src/lib.rs --edition=2018 --crate-name std_detect --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="libc"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="std_detect_dlsym_getauxval"' --cfg 'feature="std_detect_file_io"' --extern cfg_if=rlibs/libcfg_if.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern libc=rlibs/liblibc.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib +translate lib/std_detect/src/lib.rs --edition=2018 --crate-name std_detect --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="libc"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="std_detect_dlsym_getauxval"' --cfg 'feature="std_detect_file_io"' --extern "cfg_if=${RLIBS}/libcfg_if.rlib" --extern "compiler_builtins=${RLIBS}/libcompiler_builtins.rlib" --extern "libc=${RLIBS}/liblibc.rlib" --extern "alloc=${RLIBS}/liballoc.rlib" --extern "core=${RLIBS}/libcore.rlib" echo 'Building object...' -translate lib/object/src/lib.rs --edition=2018 --crate-name object --cfg 'feature="alloc"' --cfg 'feature="archive"' --cfg 'feature="coff"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="elf"' --cfg 'feature="macho"' --cfg 'feature="pe"' --cfg 'feature="read_core"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="unaligned"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern memchr=rlibs/libmemchr.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib +translate lib/object/src/lib.rs --edition=2018 --crate-name object --cfg 'feature="alloc"' --cfg 'feature="archive"' --cfg 'feature="coff"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="elf"' --cfg 'feature="macho"' --cfg 'feature="pe"' --cfg 'feature="read_core"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="unaligned"' --extern "compiler_builtins=${RLIBS}/libcompiler_builtins.rlib" --extern "memchr=${RLIBS}/libmemchr.rlib" --extern "alloc=${RLIBS}/liballoc.rlib" --extern "core=${RLIBS}/libcore.rlib" echo 'Building miniz_oxide...' -translate lib/miniz_oxide/src/lib.rs --edition=2018 --crate-name miniz_oxide --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern adler=rlibs/libadler.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib +translate lib/miniz_oxide/src/lib.rs --edition=2018 --crate-name miniz_oxide --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern "adler=${RLIBS}/libadler.rlib" --extern "compiler_builtins=${RLIBS}/libcompiler_builtins.rlib" --extern "alloc=${RLIBS}/liballoc.rlib" --extern "core=${RLIBS}/libcore.rlib" echo 'Building hashbrown...' -translate lib/hashbrown/src/lib.rs --edition=2021 --crate-name hashbrown --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="nightly"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="rustc-internal-api"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib +translate lib/hashbrown/src/lib.rs --edition=2021 --crate-name hashbrown --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="nightly"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="rustc-internal-api"' --extern "compiler_builtins=${RLIBS}/libcompiler_builtins.rlib" --extern "alloc=${RLIBS}/liballoc.rlib" --extern "core=${RLIBS}/libcore.rlib" echo 'Building addr2line...' -translate lib/addr2line/src/lib.rs --crate-name addr2line --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern gimli=rlibs/libgimli.rlib --extern alloc=rlibs/liballoc.rlib --extern core=rlibs/libcore.rlib +translate lib/addr2line/src/lib.rs --crate-name addr2line --cfg 'feature="alloc"' --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --extern "compiler_builtins=${RLIBS}/libcompiler_builtins.rlib" --extern "gimli=${RLIBS}/libgimli.rlib" --extern "alloc=${RLIBS}/liballoc.rlib" --extern "core=${RLIBS}/libcore.rlib" echo 'Building std...' -translate lib/std/src/lib.rs --edition=2021 --crate-name std --cfg 'feature="addr2line"' --cfg 'feature="backtrace"' --cfg 'feature="gimli-symbolize"' --cfg 'feature="miniz_oxide"' --cfg 'feature="object"' --cfg 'feature="panic_unwind"' --cfg 'feature="std_detect_dlsym_getauxval"' --cfg 'feature="std_detect_file_io"' --cfg 'backtrace_in_libstd`' --extern addr2line=rlibs/libaddr2line.rlib --extern alloc=rlibs/liballoc.rlib --extern cfg_if=rlibs/libcfg_if.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern hashbrown=rlibs/libhashbrown.rlib --extern libc=rlibs/liblibc.rlib --extern miniz_oxide=rlibs/libminiz_oxide.rlib --extern object=rlibs/libobject.rlib --extern panic_abort=rlibs/libpanic_abort.rlib --extern panic_unwind=rlibs/libpanic_unwind.rlib --extern rustc_demangle=rlibs/librustc_demangle.rlib --extern std_detect=rlibs/libstd_detect.rlib --extern unwind=rlibs/libunwind.rlib +translate lib/std/src/lib.rs --edition=2021 --crate-name std --cfg 'feature="addr2line"' --cfg 'feature="backtrace"' --cfg 'feature="gimli-symbolize"' --cfg 'feature="miniz_oxide"' --cfg 'feature="object"' --cfg 'feature="panic_unwind"' --cfg 'feature="std_detect_dlsym_getauxval"' --cfg 'feature="std_detect_file_io"' --cfg 'backtrace_in_libstd`' --extern "addr2line=${RLIBS}/libaddr2line.rlib" --extern "alloc=${RLIBS}/liballoc.rlib" --extern "cfg_if=${RLIBS}/libcfg_if.rlib" --extern "compiler_builtins=${RLIBS}/libcompiler_builtins.rlib" --extern "core=${RLIBS}/libcore.rlib" --extern "hashbrown=${RLIBS}/libhashbrown.rlib" --extern "libc=${RLIBS}/liblibc.rlib" --extern "miniz_oxide=${RLIBS}/libminiz_oxide.rlib" --extern "object=${RLIBS}/libobject.rlib" --extern "panic_abort=${RLIBS}/libpanic_abort.rlib" --extern "panic_unwind=${RLIBS}/libpanic_unwind.rlib" --extern "rustc_demangle=${RLIBS}/librustc_demangle.rlib" --extern "std_detect=${RLIBS}/libstd_detect.rlib" --extern "unwind=${RLIBS}/libunwind.rlib" echo 'Building proc_macro...' -translate lib/proc_macro/src/lib.rs --edition=2021 --crate-name proc_macro --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib +translate lib/proc_macro/src/lib.rs --edition=2021 --crate-name proc_macro --extern "core=${RLIBS}/libcore.rlib" --extern "std=${RLIBS}/libstd.rlib" echo 'Building unicode_width...' -translate lib/unicode_width/src/lib.rs --crate-name unicode_width --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="default"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="std"' --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib +translate lib/unicode_width/src/lib.rs --crate-name unicode_width --cfg 'feature="compiler_builtins"' --cfg 'feature="core"' --cfg 'feature="default"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="std"' --extern "compiler_builtins=${RLIBS}/libcompiler_builtins.rlib" --extern "core=${RLIBS}/libcore.rlib" --extern "std=${RLIBS}/libstd.rlib" echo 'Building getopts...' -translate lib/getopts/src/lib.rs --crate-name getopts --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="std"' --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib --extern unicode_width=rlibs/libunicode_width.rlib +translate lib/getopts/src/lib.rs --crate-name getopts --cfg 'feature="core"' --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="std"' --extern "core=${RLIBS}/libcore.rlib" --extern "std=${RLIBS}/libstd.rlib" --extern "unicode_width=${RLIBS}/libunicode_width.rlib" echo 'Building test...' -translate lib/test/src/lib.rs --edition=2021 --crate-name test --cfg 'feature="backtrace"' --cfg 'feature="default"' --cfg 'feature="panic-unwind"' --cfg 'feature="std_detect_dlsym_getauxval"' --cfg 'feature="std_detect_file_io"' --extern cfg_if=rlibs/libcfg_if.rlib --extern core=rlibs/libcore.rlib --extern getopts=rlibs/libgetopts.rlib --extern libc=rlibs/liblibc.rlib --extern panic_abort=rlibs/libpanic_abort.rlib --extern panic_unwind=rlibs/libpanic_unwind.rlib --extern proc_macro=rlibs/libproc_macro.rlib --extern std=rlibs/libstd.rlib +translate lib/test/src/lib.rs --edition=2021 --crate-name test --cfg 'feature="backtrace"' --cfg 'feature="default"' --cfg 'feature="panic-unwind"' --cfg 'feature="std_detect_dlsym_getauxval"' --cfg 'feature="std_detect_file_io"' --extern "cfg_if=${RLIBS}/libcfg_if.rlib" --extern "core=${RLIBS}/libcore.rlib" --extern "getopts=${RLIBS}/libgetopts.rlib" --extern "libc=${RLIBS}/liblibc.rlib" --extern "panic_abort=${RLIBS}/libpanic_abort.rlib" --extern "panic_unwind=${RLIBS}/libpanic_unwind.rlib" --extern "proc_macro=${RLIBS}/libproc_macro.rlib" --extern "std=${RLIBS}/libstd.rlib" # extra libs (added manually) echo 'Building int512...' -translate lib/int512.rs --crate-name int512 --extern core=rlibs/libcore.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib +translate lib/int512.rs --crate-name int512 --extern "core=${RLIBS}/libcore.rlib" --extern "compiler_builtins=${RLIBS}/libcompiler_builtins.rlib" echo 'Building bytes...' -translate lib/bytes.rs --edition=2021 --crate-name bytes --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib --extern crucible=rlibs/libcrucible.rlib +translate lib/bytes.rs --edition=2021 --crate-name bytes --extern "core=${RLIBS}/libcore.rlib" --extern "std=${RLIBS}/libstd.rlib" --extern "compiler_builtins=${RLIBS}/libcompiler_builtins.rlib" --extern "crucible=${RLIBS}/libcrucible.rlib" echo 'Building byteorder...' -translate lib/byteorder/src/lib.rs --edition=2021 --crate-name byteorder --cfg 'feature="std"' --extern core=rlibs/libcore.rlib --extern std=rlibs/libstd.rlib --extern compiler_builtins=rlibs/libcompiler_builtins.rlib +translate lib/byteorder/src/lib.rs --edition=2021 --crate-name byteorder --cfg 'feature="std"' --extern "core=${RLIBS}/libcore.rlib" --extern "std=${RLIBS}/libstd.rlib" --extern "compiler_builtins=${RLIBS}/libcompiler_builtins.rlib" From d4f50746123a1fe1e5a6ba6c0344b00d7afba995 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Mon, 3 Jul 2023 14:36:02 -0400 Subject: [PATCH 093/114] CI: Re-enable macOS build for crux-mir Fixes #1050. --- .github/Dockerfile-crux-mir | 2 -- .github/workflows/crux-mir-build.yml | 13 ++++--------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/Dockerfile-crux-mir b/.github/Dockerfile-crux-mir index 636800818..eae97d597 100644 --- a/.github/Dockerfile-crux-mir +++ b/.github/Dockerfile-crux-mir @@ -2,8 +2,6 @@ FROM rust:1.46.0 AS mir_json ADD dependencies/mir-json /mir-json WORKDIR /mir-json -# Work around https://github.com/rust-lang/cargo/issues/10303 -ENV CARGO_NET_GIT_FETCH_WITH_CLI=true RUN rustup toolchain install nightly-2020-03-22 --force RUN rustup component add --toolchain nightly-2020-03-22 rustc-dev RUN rustup default nightly-2020-03-22 diff --git a/.github/workflows/crux-mir-build.yml b/.github/workflows/crux-mir-build.yml index fa33277ab..d1e5e4c33 100644 --- a/.github/workflows/crux-mir-build.yml +++ b/.github/workflows/crux-mir-build.yml @@ -20,8 +20,6 @@ env: # (symptom: "No suitable image found ... unknown file type, first # eight bytes: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00") CACHE_VERSION: 4 - # Work around https://github.com/rust-lang/cargo/issues/10303 - CARGO_NET_GIT_FETCH_WITH_CLI: true jobs: config: @@ -65,13 +63,10 @@ jobs: os: [ubuntu-22.04] cabal: ["3.8.1.0"] ghc: ["8.10.7", "9.2.7", "9.4.4"] - # include: - # Disable the macOS build for now due to - # https://github.com/GaloisInc/crucible/issues/1050 - # - # - os: macos-12 - # cabal: 3.8.1.0 - # ghc: 9.2.4 + include: + - os: macos-12 + cabal: 3.8.1.0 + ghc: 9.2.7 # We want Windows soon, but it doesn't need to be now name: crux-mir - GHC v${{ matrix.ghc }} - ${{ matrix.os }} From 044ad3df5f0ff817ff993ce18c3140bad2132f14 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Mon, 3 Jul 2023 15:19:30 -0400 Subject: [PATCH 094/114] Remove temporary test_output.log file --- crux-mir/test_output.log | 1362 -------------------------------------- 1 file changed, 1362 deletions(-) delete mode 100644 crux-mir/test_output.log diff --git a/crux-mir/test_output.log b/crux-mir/test_output.log deleted file mode 100644 index 24d279b87..000000000 --- a/crux-mir/test_output.log +++ /dev/null @@ -1,1362 +0,0 @@ -Build profile: -w ghc-9.2.7 -O1 -In order, the following will be built (use -v for more details): - - crux-mir-0.6.0.99 (test:test) (file /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/dist-newstyle/build/x86_64-linux/ghc-9.2.7/crux-mir-0.6.0.99/cache/build changed) -Preprocessing test suite 'test' for crux-mir-0.6.0.99.. -Building test suite 'test' for crux-mir-0.6.0.99.. -[1 of 1] Compiling Main ( test/Test.hs, /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/dist-newstyle/build/x86_64-linux/ghc-9.2.7/crux-mir-0.6.0.99/t/test/build/test/test-tmp/Main.o ) [Mir.Language changed] -Linking /home/rscott/Documents/Hacking/Haskell/sandbox/crucible/dist-newstyle/build/x86_64-linux/ghc-9.2.7/crux-mir-0.6.0.99/t/test/build/test/test ... -crux-mir - crux concrete - - refs - promoted_imm: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.75s) - Crux output: 0 - mut_raw: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: 123 (1.75s) - Crux output: 123 - mut_nested: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.74s) - Crux output: () - fn_ptr_mut: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.75s) - Crux output: 1 - never: OK (1.90s) - Compiling and running oracle program (0.16s) - Oracle output: 1 (1.75s) - Crux output: 1 - mut_ref: OK (1.88s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.74s) - Crux output: 1 - temp: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.76s) - Crux output: 1 - mut_arg: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.72s) - Crux output: 1 - fn_ptr: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.75s) - Crux output: 1 - promoted_mut: OK (1.93s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.79s) - Crux output: 0 - never_mut: OK (1.98s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.84s) - Crux output: 1 - mut_tuple_field: OK (1.99s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.85s) - Crux output: () - imm_raw: OK (2.04s) - Compiling and running oracle program (0.16s) - Oracle output: 123 (1.88s) - Crux output: 123 - static_mut: OK (1.98s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.83s) - Crux output: 2 - imm_ref: OK (2.05s) - Compiling and running oracle program (0.15s) - Oracle output: 123 (1.90s) - Crux output: 123 - imm_arg: OK (1.92s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.78s) - Crux output: 1 - hash_map - insert_multi: OK (2.27s) - Compiling and running oracle program (0.18s) - Oracle output: 100 (2.09s) - Crux output: 100 - insert_remove: OK (2.68s) - Compiling and running oracle program (0.18s) - Oracle output: 12 (2.51s) - Crux output: 12 - insert_iter: OK (2.23s) - Compiling and running oracle program (0.18s) - Oracle output: [11, 12] (2.05s) - Crux output: [11, 12] - insert_get: OK (2.25s) - Compiling and running oracle program (0.18s) - Oracle output: (11, 12) (2.07s) - Crux output: (11, 12) - traits - params: OK (1.96s) - Compiling and running oracle program (0.14s) - Oracle output: 25 (1.82s) - Crux output: 25 - generic3: OK (2.00s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.85s) - Crux output: () - static_three: OK (1.90s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.74s) - Crux output: 2 - generic1: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.74s) - Crux output: () - subtrait: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 73 (1.72s) - Crux output: 73 - gen_trait: OK (1.92s) - Compiling and running oracle program (0.15s) - Oracle output: -69 (1.78s) - Crux output: -69 - dynamic_branch: OK (1.93s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.79s) - Crux output: 1 - static_two: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.72s) - Crux output: 1 - tyfam5: OK (1.84s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.70s) - Crux output: () - generic2: OK (1.87s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.73s) - Crux output: () - assoc3: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.72s) - Crux output: () - tyfam4: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.71s) - Crux output: 0 - static_self: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.71s) - Crux output: 42 - conv: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.71s) - Crux output: 1 - dynamic_poly: OK (1.85s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (1.71s) - Crux output: 3 - dynamic_simple: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.71s) - Crux output: 1 - gen_trait_poly: OK (1.85s) - Compiling and running oracle program (0.15s) - Oracle output: 12 (1.70s) - Crux output: 12 - default: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 12 (1.71s) - Crux output: 12 - assoc1: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.72s) - Crux output: () - subtrait2: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 64 (1.72s) - Crux output: 64 - bounds1: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.73s) - Crux output: () - bounds3: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.72s) - Crux output: () - static_eq: OK (1.84s) - Compiling and running oracle program (0.14s) - Oracle output: true (1.70s) - Crux output: true - tyfam3: OK (1.84s) - Compiling and running oracle program (0.14s) - Oracle output: 23 (1.70s) - Crux output: 23 - dict_med: OK (1.84s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.70s) - Crux output: 42 - basics1: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.75s) - Crux output: () - assoc2: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.76s) - Crux output: () - dict_polymem: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 4 (1.72s) - Crux output: 4 - bounds2: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.71s) - Crux output: () - intoiter: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.71s) - Crux output: 0 - static: OK (1.84s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.70s) - Crux output: 42 - dynamic_two: OK (1.84s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.70s) - Crux output: 1 - bounds4: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.72s) - Crux output: () - dict_poly: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.76s) - Crux output: 1 - dict_simple: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 32 (1.74s) - Crux output: 32 - tyfam: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 23 (1.71s) - Crux output: 23 - bounds5: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.72s) - Crux output: () - tyfam2: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 23 (1.70s) - Crux output: 23 - dynamic_med: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.70s) - Crux output: 1 - intTest - test0039: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 7 (1.72s) - Crux output: 7 - test0038: OK (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.79s) - Crux output: () - tuple - clone_rec: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.73s) - Crux output: () - clone: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.73s) - Crux output: () - clone_from: OK (1.90s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.75s) - Crux output: () - clone_struct: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.72s) - Crux output: () - iter - for: OK (1.93s) - Compiling and running oracle program (0.14s) - Oracle output: 47 (1.79s) - Crux output: 47 - sum: OK (1.91s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.76s) - Crux output: () - filter_chain: OK (2.01s) - Compiling and running oracle program (0.17s) - Oracle output: 0 (1.85s) - Crux output: 0 - from_fn: OK (1.89s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.74s) - Crux output: () - peek: OK (1.95s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (1.80s) - Crux output: 3 - zip: OK (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.80s) - Crux output: () - loop: OK (1.84s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.70s) - Crux output: 2 - cloned: OK (1.90s) - Compiling and running oracle program (0.15s) - Oracle output: 6 (1.75s) - Crux output: 6 - str - format_hex: OK (2.61s) - Compiling and running oracle program (0.15s) - Oracle output: true (2.46s) - Crux output: true - format_int: OK (2.61s) - Compiling and running oracle program (0.15s) - Oracle output: true (2.46s) - Crux output: true - format_struct: OK (2.76s) - Compiling and running oracle program (0.15s) - Oracle output: true (2.61s) - Crux output: true - format: OK (2.59s) - Compiling and running oracle program (0.15s) - Oracle output: true (2.44s) - Crux output: true - to_owned: OK (2.05s) - Compiling and running oracle program (0.15s) - Oracle output: true (1.90s) - Crux output: true - string_push: OK (2.59s) - Compiling and running oracle program (0.15s) - Oracle output: true (2.44s) - Crux output: true - format_array: OK (2.74s) - Compiling and running oracle program (0.15s) - Oracle output: true (2.59s) - Crux output: true - sync - mutex: OK (2.88s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (2.73s) - Crux output: 1 - arc_cell: OK (2.09s) - Compiling and running oracle program (0.15s) - Oracle output: 4 (1.95s) - Crux output: 4 - arc: OK (2.01s) - Compiling and running oracle program (0.15s) - Oracle output: 1 (1.86s) - Crux output: 1 - atomic_add: OK (1.96s) - Compiling and running oracle program (0.14s) - Oracle output: 6 (1.82s) - Crux output: 6 - rwlock: OK (2.91s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (2.77s) - Crux output: 2 - atomic_cxchg: OK (1.97s) - Compiling and running oracle program (0.15s) - Oracle output: 6 (1.82s) - Crux output: 6 - arc_clone: OK (2.01s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.86s) - Crux output: 2 - atomic_swap: OK (1.89s) - Compiling and running oracle program (0.15s) - Oracle output: (2, 1) (1.75s) - Crux output: (2, 1) - rwlock_multi: OK (2.99s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (2.83s) - Crux output: 3 - atomic_fence: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.74s) - Crux output: 2 - mutex_multi: OK (2.97s) - Compiling and running oracle program (0.15s) - Oracle output: 3 (2.82s) - Crux output: 3 - consts - struct_val: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: Foo { x: true } (1.72s) - Crux output: Foo { x: true } - struct_unit: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: Err(Foo { x: () }) (1.72s) - Crux output: Err(Foo { x: () }) - local_key: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.71s) - Crux output: 1 - fn_def: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.70s) - Crux output: 1 - enum_val: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: None (1.76s) - Crux output: None - crypto - add: OK (1.95s) - Compiling and running oracle program (0.15s) - Oracle output: true (1.80s) - Crux output: true - add_noL: OK (1.93s) - Compiling and running oracle program (0.15s) - Oracle output: false (1.78s) - Crux output: false - cell - cell: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.74s) - Crux output: 2 - ref_cell: OK (2.37s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (2.22s) - Crux output: 2 - ref_cell2: OK (2.41s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (2.27s) - Crux output: 2 - fnptr - call: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.73s) - Crux output: 2 - field: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.72s) - Crux output: 2 - make: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.71s) - Crux output: 0 - custom: rustc compilation failed for custom -error output: -error[E0432]: unresolved import `core` - --> test/conc_eval/fnptr/custom.rs:2:5 - | -2 | use core::mem; - | ^^^^ maybe a missing crate `core`? - | - = help: consider adding `extern crate core` to use the `core` crate - -error: aborting due to previous error - -For more information about this error, try `rustc --explain E0432`. - -FAIL (expected: taking address of an overridden function) (0.05s) - Compiling and running oracle program (0.05s) - test/Test.hs:106: - failed to compile and run - (expected failure) - num - overflow: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.76s) - Crux output: () - from_bytes: OK (1.93s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.79s) - Crux output: () - saturate: OK (2.42s) - Compiling and running oracle program (0.15s) - Oracle output: () (2.28s) - Crux output: () - prim - bool: OK (1.84s) - Compiling and running oracle program (0.14s) - Oracle output: false (1.70s) - Crux output: false - shift3: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: -9223372036854775808 (1.71s) - Crux output: -9223372036854775808 - div: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 4 (1.71s) - Crux output: 4 - ge: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: true (1.71s) - Crux output: true - add1: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.75s) - Crux output: 2 - shift_exceeding: rustc compilation failed for shift_exceeding -error output: -error[E0425]: cannot find function `catch_unwind` in module `panic` - --> test/conc_eval/prim/shift_exceeding.rs:14:25 - | -14 | let result = panic::catch_unwind(|| { - | ^^^^^^^^^^^^ not found in `panic` - | -help: consider importing this function - | -3 | use std::panic::catch_unwind; - | -help: if you import `catch_unwind`, refer to it directly - | -14 - let result = panic::catch_unwind(|| { -14 + let result = catch_unwind(|| { - | - -warning: lint `exceeding_bitshifts` has been renamed to `arithmetic_overflow` - --> test/conc_eval/prim/shift_exceeding.rs:4:9 - | -4 | #[allow(exceeding_bitshifts)] - | ^^^^^^^^^^^^^^^^^^^ help: use the new name: `arithmetic_overflow` - | - = note: `#[warn(renamed_and_removed_lints)]` on by default - -error: aborting due to previous error; 1 warning emitted - -For more information about this error, try `rustc --explain E0425`. - -FAIL (expected: Should panic, but doesn't) (0.05s) - Compiling and running oracle program (0.05s) - test/Test.hs:106: - failed to compile and run - (expected failure) - char_from_u32: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 'A' (1.73s) - Crux output: 'A' - ffs: OK (1.94s) - Compiling and running oracle program (0.15s) - Oracle output: true (1.79s) - Crux output: true - mut_arg: OK (1.91s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.77s) - Crux output: 1 - litstring: OK (1.83s) - Compiling and running oracle program (0.14s) - Oracle output: true (1.69s) - Crux output: true - lit: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: false (1.71s) - Crux output: false - litbstring: OK (1.83s) - Compiling and running oracle program (0.14s) - Oracle output: true (1.69s) - Crux output: true - shift1: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.71s) - Crux output: 2 - shift2: OK (1.84s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.69s) - Crux output: 2 - shift4: OK (1.83s) - Compiling and running oracle program (0.14s) - Oracle output: -9223372036854775808 (1.70s) - Crux output: -9223372036854775808 - mut: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 14 (1.71s) - Crux output: 14 - wrapping_sub: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: true (1.71s) - Crux output: true - impl - self: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.71s) - Crux output: 42 - simple: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.75s) - Crux output: 42 - self_mut: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.72s) - Crux output: 42 - box - new: OK (1.97s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.83s) - Crux output: () - mut_ref: OK (1.97s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.83s) - Crux output: () - mut: OK (1.98s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.84s) - Crux output: () - unsize: OK (2.00s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.86s) - Crux output: 2 - struct: OK (1.99s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.84s) - Crux output: 1 - ptr - is_null_slice: FAIL (expected: can't unsize null pointers) (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: (false, true) (1.75s) - Crux output: - failures: - - ---- is_null_slice/3549f883::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/ptr/is_null_slice.rs:7:13: 7:33: error: in is_null_slice/3549f883::crux_test[0] - [Crux] attempted subindex on the result of an integer-to-pointer cast - [Crux] Found counterexample for verification goal - [Crux] ./lib/core/src/ptr/const_ptr.rs:38:38: 38:42: error: in core/092bc89a::ptr[0]::const_ptr[0]::{impl#0}[0]::is_null[0]::_inst55af81a2cef2701c[0] - [Crux] attempted to read empty mux tree - - [Crux] Overall status: Invalid. - - test/Test.hs:123: - crux doesn't match oracle - (expected failure) - struct_eq: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.72s) - Crux output: () - read_write: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: [1, 3, 1] (1.76s) - Crux output: [1, 3, 1] - offset_mut: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.72s) - Crux output: 3 - copy: OK (1.91s) - Compiling and running oracle program (0.15s) - Oracle output: [1, 2, 3, 1, 2, 3] (1.76s) - Crux output: [1, 2, 3, 1, 2, 3] - unsize_slice: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: (1, 2) (1.72s) - Crux output: (1, 2) - dangling_eq: OK (1.92s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.78s) - Crux output: () - cast_eq: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.73s) - Crux output: () - offset_from: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: -2 (1.74s) - Crux output: -2 - null_eq: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.73s) - Crux output: 1 - coerce_unsized: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: (1, 2) (1.72s) - Crux output: (1, 2) - offset: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.72s) - Crux output: 3 - is_null: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: (false, true) (1.73s) - Crux output: (false, true) - valid_eq: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.72s) - Crux output: () - ops - deref2: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.74s) - Crux output: () - index2: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.72s) - Crux output: () - arith1: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.73s) - Crux output: () - index3: OK (1.91s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.76s) - Crux output: () - deref1: OK (1.84s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.70s) - Crux output: () - index1: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.73s) - Crux output: () - deref3: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.74s) - Crux output: () - statics - promoted_fn: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.76s) - Crux output: 1 - promoted_static: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.72s) - Crux output: 1 - io - cursor_write2: OK (3.09s) - Compiling and running oracle program (0.16s) - Oracle output: 0 (2.93s) - Crux output: 0 - cursor_write: OK (2.52s) - Compiling and running oracle program (0.16s) - Oracle output: 0 (2.36s) - Crux output: 0 - cursor_read: OK (2.51s) - Compiling and running oracle program (0.16s) - Oracle output: 0 (2.35s) - Crux output: 0 - mem - maybe_uninit: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.72s) - Crux output: 1 - maybe_uninit_array_cast: OK (1.87s) - Compiling and running oracle program (0.15s) - Oracle output: [1, 2] (1.72s) - Crux output: [1, 2] - time - instant: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.75s) - Crux output: () - struct - field_order: OK (1.86s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.71s) - Crux output: () - arg: OK (1.87s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.72s) - Crux output: 42 - tup: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.70s) - Crux output: () - proj: OK (1.84s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.70s) - Crux output: 42 - ret: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: S { x: 42, y: 120 } (1.71s) - Crux output: S { x: 42, y: 120 } - repr_transparent: OK (1.98s) - Compiling and running oracle program (0.14s) - Oracle output: 6 (1.84s) - Crux output: 6 - repr_transparent_const: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 124 (1.73s) - Crux output: 124 - dyn - assoc_ty: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 100 (1.73s) - Crux output: 100 - trait_param: OK (1.90s) - Compiling and running oracle program (0.15s) - Oracle output: 100 (1.75s) - Crux output: 100 - plain_trait: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 100 (1.73s) - Crux output: 100 - inherit: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 4 (1.74s) - Crux output: 4 - stdlib - option: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: true (1.71s) - Crux output: true - option2: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.71s) - Crux output: 0 - result: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 27 (1.72s) - Crux output: 27 - default: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: true (1.71s) - Crux output: true - cvt: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.73s) - Crux output: 0 - range: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 10 (1.73s) - Crux output: 10 - poly: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.71s) - Crux output: 1 - result_interior: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.72s) - Crux output: 3 - default_impl: OK (1.86s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.71s) - Crux output: () - option3: OK (1.84s) - Compiling and running oracle program (0.14s) - Oracle output: 27 (1.70s) - Crux output: 27 - teq: OK (1.84s) - Compiling and running oracle program (0.15s) - Oracle output: false (1.70s) - Crux output: false - array - wick2: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: true (1.72s) - Crux output: true - arg: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.72s) - Crux output: 2 - iter: OK (1.91s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.77s) - Crux output: 3 - mk_and_proj: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.72s) - Crux output: 42 - wick3: FAIL (expected: needs Vec data structure from stdlib) (0.20s) - Compiling and running oracle program (0.16s) - Oracle output: true (0.04s) - user error (Error 101 while running mir-json on test/conc_eval/array/wick3.rs) - (expected failure) - clone: OK (1.99s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.84s) - Crux output: () - wick1: FAIL (expected: needs Vec data structure from stdlib) (0.20s) - Compiling and running oracle program (0.16s) - Oracle output: true (0.04s) - user error (Error 101 while running mir-json on test/conc_eval/array/wick1.rs) - (expected failure) - mut_index: OK (2.34s) - Compiling and running oracle program (0.15s) - Oracle output: 7 (2.19s) - Crux output: 7 - mut_arg: OK (1.86s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.72s) - Crux output: 42 - const: OK (1.84s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.70s) - Crux output: 1 - from_slice: OK (2.51s) - Compiling and running oracle program (0.14s) - Oracle output: () (2.37s) - Crux output: () - const_impl: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: 5 (1.76s) - Crux output: 5 - enum - match: OK (1.91s) - Compiling and running oracle program (0.15s) - Oracle output: 42 (1.76s) - Crux output: 42 - field_order: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.76s) - Crux output: () - mixed_discrs: OK (1.87s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.73s) - Crux output: () - arg: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.70s) - Crux output: 0 - arg2: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 0 (1.71s) - Crux output: 0 - cow: OK (2.00s) - Compiling and running oracle program (0.15s) - Oracle output: 200 (1.85s) - Crux output: 200 - ret: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: (A(42), B { x: 42 }, C) (1.75s) - Crux output: (A(42), B { x: 42 }, C) - eq: OK (1.89s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.74s) - Crux output: () - cmp: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: -23 (1.72s) - Crux output: -23 - inner: OK (1.91s) - Compiling and running oracle program (0.15s) - Oracle output: 4 (1.76s) - Crux output: 4 - vec - drop: OK (2.04s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.89s) - Crux output: () - set_len: OK (2.03s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.89s) - Crux output: 2 - push: OK (2.12s) - Compiling and running oracle program (0.15s) - Oracle output: (1, 2) (1.97s) - Crux output: (1, 2) - extend: OK (2.12s) - Compiling and running oracle program (0.16s) - Oracle output: (1, 10) (1.96s) - Crux output: (1, 10) - collect: OK (2.77s) - Compiling and running oracle program (0.16s) - Oracle output: 45 (2.61s) - Crux output: 45 - extend_trusted_len: OK (2.65s) - Compiling and running oracle program (0.16s) - Oracle output: (1, 10) (2.49s) - Crux output: (1, 10) - from_elem_zero: OK (2.11s) - Compiling and running oracle program (0.15s) - Oracle output: 0 (1.96s) - Crux output: 0 - clos - promoted: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 33 (1.73s) - Crux output: 33 - as_fn_ptr: FAIL (expected: ClosureFnPointer cast) (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (1.73s) - Crux output: - failures: - - ---- as_fn_ptr/ab0e7632::crux_test[0] counterexamples ---- - [Crux] Found counterexample for verification goal - [Crux] test/conc_eval/clos/as_fn_ptr.rs:9:13: 9:19: error: in as_fn_ptr/ab0e7632::crux_test[0] - [Crux] Translation error in as_fn_ptr/ab0e7632::crux_test[0]: unimplemented cast: ClosureFnPointer - [Crux] ty: TyClosure [] - [Crux] as: TyFnPtr (FnSig {_fsarg_tys = [], _fsreturn_ty = TyInt B32, _fsabi = RustAbi, _fsspreadarg = Nothing}) - - [Crux] Overall status: Invalid. - - test/Test.hs:123: - crux doesn't match oracle - (expected failure) - fnptr_fnmut: OK (1.90s) - Compiling and running oracle program (0.15s) - Oracle output: 4 (1.76s) - Crux output: 4 - direct_fnonce: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.72s) - Crux output: 2 - conv_fnonce_fnmut: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.71s) - Crux output: 2 - fn_static_poly: OK (1.90s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.76s) - Crux output: 3 - fnptr_fnonce: OK (1.86s) - Compiling and running oracle program (0.15s) - Oracle output: 4 (1.71s) - Crux output: 4 - fn_static: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.73s) - Crux output: 3 - fn_dyn: FAIL (expected: call_once shim (in `dyn Fn` vtable)) (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.73s) - vtable signature mismatch for vtable core/092bc89a::ops[0]::function[0]::Fn[0]::_vtbl570ffd80324c2ca9[0] of trait core/092bc89a::ops[0]::function[0]::Fn[0]::_trait42f8b6475ee85722[0] : StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32), FunctionHandleRepr [AnyRepr, BVRepr 32] (BVRepr 32)] != StructRepr [FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32), FunctionHandleRepr [AnyRepr, StructRepr [MaybeRepr (BVRepr 32)]] (BVRepr 32)] - CallStack (from HasCallStack): - error, called at src/Mir/Trans.hs:964:20 in crucible-mir-0.1-inplace:Mir.Trans - mkTraitObject, called at src/Mir/Trans.hs:807:11 in crucible-mir-0.1-inplace:Mir.Trans - evalCast', called at src/Mir/Trans.hs:938:5 in crucible-mir-0.1-inplace:Mir.Trans - evalCast, called at src/Mir/Trans.hs:990:30 in crucible-mir-0.1-inplace:Mir.Trans - evalRval, called at src/Mir/Trans.hs:1209:11 in crucible-mir-0.1-inplace:Mir.Trans - transStatement, called at src/Mir/Trans.hs:1654:68 in crucible-mir-0.1-inplace:Mir.Trans - translateBlockBody, called at src/Mir/Trans.hs:1663:28 in crucible-mir-0.1-inplace:Mir.Trans - registerBlock, called at src/Mir/Trans.hs:1703:10 in crucible-mir-0.1-inplace:Mir.Trans - genFn, called at src/Mir/Trans.hs:1749:24 in crucible-mir-0.1-inplace:Mir.Trans - transDefine, called at src/Mir/Trans.hs:2206:30 in crucible-mir-0.1-inplace:Mir.Trans - transCollection, called at src/Mir/ParseTranslate.hs:77:26 in crucible-mir-0.1-inplace:Mir.ParseTranslate - translateMIR, called at src/Mir/Language.hs:256:16 in crux-mir-0.6.0.99-inplace:Mir.Language - (expected failure) - fnonce: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: false (1.72s) - Crux output: false - conv_fnmut_fn: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.72s) - Crux output: 2 - direct_fnmut2: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 7 (1.72s) - Crux output: 7 - fo: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 29 (1.72s) - Crux output: 29 - unique_borrow: OK (1.86s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.72s) - Crux output: 3 - conv_fnonce_fn: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 2 (1.73s) - Crux output: 2 - fnptr_fn: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 4 (1.73s) - Crux output: 4 - direct_fnmut: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 7 (1.71s) - Crux output: 7 - dispatch_fnmut: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 7 (1.73s) - Crux output: 7 - poly: OK (1.84s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.71s) - Crux output: 3 - direct_fn: OK (1.87s) - Compiling and running oracle program (0.15s) - Oracle output: 2 (1.73s) - Crux output: 2 - fnptr_closure: OK (1.90s) - Compiling and running oracle program (0.15s) - Oracle output: 7 (1.75s) - Crux output: 7 - fnonce1: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.70s) - Crux output: 3 - ref_fnmut: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 7 (1.73s) - Crux output: 7 - vec_deque - pop: OK (2.53s) - Compiling and running oracle program (0.15s) - Oracle output: [5, 4, 3, 1, 2] (2.37s) - Crux output: [5, 4, 3, 1, 2] - iter_clone: OK (2.56s) - Compiling and running oracle program (0.15s) - Oracle output: [1, 2, 3, 2, 3] (2.40s) - Crux output: [1, 2, 3, 2, 3] - push: OK (2.56s) - Compiling and running oracle program (0.15s) - Oracle output: [4, 1, 2, 3, 5] (2.41s) - Crux output: [4, 1, 2, 3, 5] - retain: OK (2.60s) - Compiling and running oracle program (0.15s) - Oracle output: [1, 4] (2.45s) - Crux output: [1, 4] - rotate_left: OK (2.56s) - Compiling and running oracle program (0.15s) - Oracle output: [3, 4, 5, 1, 2] (2.41s) - Crux output: [3, 4, 5, 1, 2] - rotate_right: OK (2.64s) - Compiling and running oracle program (0.15s) - Oracle output: [4, 5, 1, 2, 3] (2.49s) - Crux output: [4, 5, 1, 2, 3] - slice - len: OK (1.87s) - Compiling and running oracle program (0.14s) - Oracle output: 5 (1.73s) - Crux output: 5 - mut_range: OK (2.35s) - Compiling and running oracle program (0.14s) - Oracle output: 86 (2.21s) - Crux output: 86 - range_len: OK (2.34s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (2.20s) - Crux output: 1 - range_len_mut: OK (2.33s) - Compiling and running oracle program (0.14s) - Oracle output: 1 (2.18s) - Crux output: 1 - mk_and_proj: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.75s) - Crux output: 42 - swap: OK (1.89s) - Compiling and running oracle program (0.14s) - Oracle output: 2001 (1.75s) - Crux output: 2001 - eq: OK (1.93s) - Compiling and running oracle program (0.14s) - Oracle output: () (1.79s) - Crux output: () - iter_mut: OK (1.98s) - Compiling and running oracle program (0.15s) - Oracle output: () (1.83s) - Crux output: () - get: OK (1.88s) - Compiling and running oracle program (0.15s) - Oracle output: true (1.74s) - Crux output: true - mut: OK (1.85s) - Compiling and running oracle program (0.14s) - Oracle output: 42 (1.71s) - Crux output: 42 - last: OK (1.88s) - Compiling and running oracle program (0.14s) - Oracle output: 3 (1.74s) - Crux output: 3 - crux symbolic - Output testing - mux_init_mut: OK (1.77s) - mux_init_imm: OK (1.86s) - early_fail: OK (1.86s) - mixed_fail: OK (1.86s) - multi: OK (1.88s) - fail_return: OK (1.82s) - no_conc: OK (1.84s) - conc: OK (1.90s) - array: OK (1.83s) - assert_ok: OK (2.64s) - assert: OK (2.67s) - bytes2: OK (2.07s) - double: OK (1.88s) - ffs: OK (2.02s) - bytes: OK (2.15s) - bad_symb1: OK (1.88s) - bad_symb2: OK (1.89s) - override3: OK (1.86s) - override1: OK (1.78s) - override4: OK (1.90s) - override_rust: OK (1.86s) - override2: OK (1.86s) - override5: OK (1.93s) - mux: OK (2.12s) - checked_mul_signed: OK (1.94s) - checked_mul: OK (1.84s) - checked_div: OK (1.81s) - checked_add: OK (1.76s) - test1: OK (120.41s) - pop: OK (1.98s) - as_mut_slice: OK (2.00s) - push: OK (1.96s) - copy_from_slice: OK (2.05s) - new: OK (2.03s) - mut: OK (1.87s) - concat: OK (1.89s) - split_at: OK (1.96s) - replicate: OK (1.97s) - as_slice: OK (1.84s) - array_mut: OK (1.85s) - array: OK (1.86s) - extend_bytes: OK (1.86s) - put: OK (2.48s) - split_to: OK (2.62s) - new: OK (2.08s) - split_off: OK (2.48s) - sym_len: OK (2.57s) - put_overflow: OK (2.57s) - vec_cursor_read: OK (3.17s) - vec_write: OK (2.54s) - write: OK (3.36s) - read: OK (3.02s) - slice_mut: OK (1.91s) - basic: OK (1.95s) - slice: OK (2.12s) - mux_slice: OK (1.93s) - mux: OK (2.07s) - uninit_read: OK (1.94s) - valid_read: OK (1.87s) - out_of_bounds: OK (1.84s) - zero_length: OK (1.88s) - downcast: OK (1.95s) - downcast_fail: OK (1.88s) - conditional: OK (1.92s) - literals: OK (1.73s) - arith: OK (2.00s) - leading_zeros: OK (1.94s) - overflowing_sub: OK (1.86s) - cmp: OK (1.86s) - symbolic: OK (1.78s) - from_to: OK (1.73s) - clone: OK (2.04s) - sort_by_key: OK (2.57s) - into_iter: OK (2.39s) - macro: OK (1.85s) - construct: OK (1.87s) - deserialize: OK (1.84s) - crux coverage - Output testing - coverage_dual: warning: trailing semicolon in macro used in expression position - --> src/main.rs:45:46 - | -45 | return Err(format!($($args)*).into()); - | ^ -... -309 | _ => die!("unknown tag {:?} for branch", tag), - | ---------------------------------------- in this macro invocation - | - = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! - = note: for more information, see issue #79813 - = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default - = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) - -OK (1.78s) - coverage: warning: trailing semicolon in macro used in expression position - --> src/main.rs:45:46 - | -45 | return Err(format!($($args)*).into()); - | ^ -... -309 | _ => die!("unknown tag {:?} for branch", tag), - | ---------------------------------------- in this macro invocation - | - = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! - = note: for more information, see issue #79813 - = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default - = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) - -OK (1.93s) - coverage_try: warning: trailing semicolon in macro used in expression position - --> src/main.rs:45:46 - | -45 | return Err(format!($($args)*).into()); - | ^ -... -309 | _ => die!("unknown tag {:?} for branch", tag), - | ---------------------------------------- in this macro invocation - | - = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! - = note: for more information, see issue #79813 - = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default - = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) - -OK (1.90s) - coverage_cond: warning: trailing semicolon in macro used in expression position - --> src/main.rs:45:46 - | -45 | return Err(format!($($args)*).into()); - | ^ -... -309 | _ => die!("unknown tag {:?} for branch", tag), - | ---------------------------------------- in this macro invocation - | - = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! - = note: for more information, see issue #79813 - = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default - = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) - -OK (1.79s) - coverage_loop: warning: trailing semicolon in macro used in expression position - --> src/main.rs:45:46 - | -45 | return Err(format!($($args)*).into()); - | ^ -... -309 | _ => die!("unknown tag {:?} for branch", tag), - | ---------------------------------------- in this macro invocation - | - = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! - = note: for more information, see issue #79813 - = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default - = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) - -OK (1.82s) - coverage_shortcircuit: warning: trailing semicolon in macro used in expression position - --> src/main.rs:45:46 - | -45 | return Err(format!($($args)*).into()); - | ^ -... -309 | _ => die!("unknown tag {:?} for branch", tag), - | ---------------------------------------- in this macro invocation - | - = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! - = note: for more information, see issue #79813 - = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default - = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) - -OK (1.87s) - coverage_mono: warning: trailing semicolon in macro used in expression position - --> src/main.rs:45:46 - | -45 | return Err(format!($($args)*).into()); - | ^ -... -309 | _ => die!("unknown tag {:?} for branch", tag), - | ---------------------------------------- in this macro invocation - | - = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! - = note: for more information, see issue #79813 - = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default - = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) - -OK (1.81s) - coverage_match: warning: trailing semicolon in macro used in expression position - --> src/main.rs:45:46 - | -45 | return Err(format!($($args)*).into()); - | ^ -... -309 | _ => die!("unknown tag {:?} for branch", tag), - | ---------------------------------------- in this macro invocation - | - = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! - = note: for more information, see issue #79813 - = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default - = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) - -OK (1.88s) - coverage_macro: warning: trailing semicolon in macro used in expression position - --> src/main.rs:45:46 - | -45 | return Err(format!($($args)*).into()); - | ^ -... -309 | _ => die!("unknown tag {:?} for branch", tag), - | ---------------------------------------- in this macro invocation - | - = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! - = note: for more information, see issue #79813 - = note: `#[warn(semicolon_in_expressions_from_macros)]` on by default - = note: this warning originates in the macro `die` (in Nightly builds, run with -Z macro-backtrace for more info) - -OK (1.90s) - -All 339 tests passed (788.49s) From 5d4453f1ed43de4057ab28318194004424bc47e8 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Mon, 3 Jul 2023 15:50:08 -0400 Subject: [PATCH 095/114] build.sh: Make rlibs a symlink to rlibs_real/.../lib This makes it more convenient to call `mir-json` from `crux-mir`. --- crux-mir/build.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crux-mir/build.sh b/crux-mir/build.sh index 34f26d15d..c054970ce 100755 --- a/crux-mir/build.sh +++ b/crux-mir/build.sh @@ -3,7 +3,8 @@ set -e SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -RLIBS_PARENT="${SCRIPT_DIR}/rlibs" +RLIBS_PARENT="${SCRIPT_DIR}/rlibs_real" +RLIBS_SYMLINK="${SCRIPT_DIR}/rlibs" RLIBS=$(rustc --print target-libdir --sysroot "${RLIBS_PARENT}") STD_ENV_ARCH=$(uname -m) export STD_ENV_ARCH @@ -17,6 +18,10 @@ mkdir -p "${RLIBS_PARENT}/etc" mkdir -p "${RLIBS_PARENT}/lib" mkdir -p "${RLIBS_PARENT}/libexec" mkdir -p "${RLIBS_PARENT}/share" +mkdir -p "${RLIBS}" +if [ ! -d "${RLIBS_SYMLINK}" ]; then + ln -s "${RLIBS}" "${RLIBS_SYMLINK}" +fi echo 'Building core...' translate lib/core/src/lib.rs --edition=2021 --crate-name core From 85e59a92fa30d497a7a3b9420ce9f5fb58296db8 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Mon, 3 Jul 2023 15:50:36 -0400 Subject: [PATCH 096/114] Remove translate_libs.sh --- crux-mir/translate_libs.sh | 138 ------------------------------------- 1 file changed, 138 deletions(-) delete mode 100755 crux-mir/translate_libs.sh diff --git a/crux-mir/translate_libs.sh b/crux-mir/translate_libs.sh deleted file mode 100755 index 7f0e49c48..000000000 --- a/crux-mir/translate_libs.sh +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/sh -set -e - -compile() { - compile_2015 --edition 2018 "$@" -} - -compile_2015() { - # We use `--remap-path-prefix` to keep the full path to the `crux-mir` - # directory from appearing in the generated artifacts. Otherwise, some - # `crux-mir` error messages will contain the path to the `crux-mir` - # directory, which usually contains the user's home directory, and our - # golden-file tests will break. We specifically use `pwd -P` ("physical", - # meaning resolve symlinks) because `rustc` itself resolves symlinks before - # checking for a matching `--remap-path-prefix` rule. - rustc -L rlibs_native --out-dir rlibs_native --crate-type rlib \ - --remap-path-prefix "$(pwd -P)=." \ - "$@" -} - -translate() { - translate_2015 --edition 2018 "$@" -} - -translate_2015() { - mir-json -L rlibs --out-dir rlibs --crate-type rlib \ - --remap-path-prefix "$(pwd -P)=." \ - "$@" -} - -usage() { - echo "Usage: ${0} [FLAGS]" - echo "" - echo "Flags:" - echo "-a --model-internal-atomics Include stdlib internal atomics in concurrency model (default)" - echo " --no-model-internal-atomics Do not include stdlib internal atomics in concurrency model" -} - -SHOULD_MODEL_ATOMICS=1 - -for arg in "$@" -do - case $arg in - -a|--model-internal-atomics) - SHOULD_MODEL_ATOMICS=1 - shift - ;; - --no-model-internal-atomics) - SHOULD_MODEL_ATOMICS=0 - shift - ;; - -h|--help) - usage - exit 0 - ;; - *) - usage - exit 1 - ;; - esac -done - - -if [ $SHOULD_MODEL_ATOMICS -eq "1" ]; then - CONCURRENCY_LIBALLOC_FEATURES=" " - CONCURRENCY_LIBSTD_FEATURES=" " -else - CONCURRENCY_LIBALLOC_FEATURES="--cfg feature=\"opaque-arc-atomics\"" - CONCURRENCY_LIBSTD_FEATURES="--cfg feature=\"opaque-poison-atomics\"" -fi - -translate lib/libcore/lib.rs --crate-name core \ - --cfg 'feature="panic_immediate_abort"' \ - --cfg iter_count --cfg iter_last --cfg iter_min_max \ - --cfg ascii --cfg char --cfg unicode \ - --cfg slice_sort \ - --cfg time --cfg simd --cfg sync \ - --cfg slice_u8 \ - --cfg str --cfg str_lossy --cfg memchr --cfg str_pattern --cfg str_case \ - --cfg flt2dec \ - --cfg from_str --cfg dec2flt \ - --cfg any_downcast \ - --cfg hash \ - --cfg fmt # trouble with integer formatting in bounds check panics - -translate_2015 lib/compiler-builtins/src/lib.rs --crate-name compiler_builtins \ - --cfg 'feature="compiler-builtins"' -translate lib/crucible/lib.rs --crate-name crucible -translate lib/int512.rs -translate lib/liballoc/lib.rs --crate-name alloc \ - ${CONCURRENCY_LIBALLOC_FEATURES} \ - --extern crucible -translate lib/cfg-if/src/lib.rs --crate-name cfg_if --cfg 'feature="rustc-dep-of-std"' \ - --extern rustc_std_workspace_core=rlibs/libcore.rlib \ - -ldl -translate_2015 lib/libc/src/lib.rs --crate-name libc \ - --cfg 'feature="rustc-dep-of-std"' --cfg libc_align \ - --extern rustc_std_workspace_core=rlibs/libcore.rlib - -if [ "$(uname)" = "Linux" ]; then - # Under cargo, this flag would be set by libunwind's `build.rs` file. - libunwind_extra_flags=-lgcc_s -fi -translate lib/libunwind/lib.rs --crate-name unwind \ - --extern cfg_if --extern libc $libunwind_extra_flags - -translate lib/libpanic_abort/lib.rs --crate-name panic_abort -C panic=abort \ - --extern libc -translate lib/libpanic_unwind/lib.rs --crate-name panic_unwind -C panic=unwind \ - --extern alloc --extern cfg_if --extern unwind --extern libc -translate lib/hashbrown/src/lib.rs --crate-name hashbrown \ - --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="nightly"' \ - --cfg 'feature="rustc-internal-api"' --cfg has_extern_crate_alloc \ - --extern alloc -translate_2015 lib/rustc-demangle/src/lib.rs --crate-name rustc_demangle \ - --cfg 'feature="rustc-dep-of-std"' -translate_2015 lib/backtrace/crates/backtrace-sys/src/lib.rs --crate-name backtrace_sys -translate lib/backtrace/src/lib.rs --crate-name backtrace \ - --extern core --extern compiler_builtins --extern libc \ - --extern rustc_demangle --extern cfg_if --extern backtrace_sys -translate lib/libstd/lib.rs --crate-name std \ - --cfg 'feature="panic_immediate_abort"' \ - ${CONCURRENCY_LIBSTD_FEATURES} \ - --extern alloc --extern cfg_if \ - --extern hashbrown --extern backtrace_rs=rlibs/libbacktrace.rlib -translate lib/libterm/lib.rs --crate-name term -translate lib/unicode-width/src/lib.rs --crate-name unicode_width \ - --cfg 'feature="rustc-dep-of-std"' -translate lib/getopts/src/lib.rs --crate-name getopts -translate lib/libtest/lib.rs --crate-name test \ - --extern libc --extern getopts --extern term - -translate_2015 lib/byteorder/lib.rs --crate-name byteorder --cfg 'feature="std"' -translate lib/bytes.rs -translate_2015 lib/bigint/src/lib.rs --crate-name bigint - -# Need native versions of some libs for conc_eval oracle programs -#compile_2015 lib/byteorder/lib.rs --crate-name byteorder --cfg 'feature="std"' From a7c2f6d544760f2d2c5a231e976456f3a9be8dcc Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Mon, 3 Jul 2023 15:51:51 -0400 Subject: [PATCH 097/114] Make build.sh the new translate_libs.sh --- crux-mir/{build.sh => translate_libs.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename crux-mir/{build.sh => translate_libs.sh} (100%) diff --git a/crux-mir/build.sh b/crux-mir/translate_libs.sh similarity index 100% rename from crux-mir/build.sh rename to crux-mir/translate_libs.sh From 4658042c3aed156cb273c56e9a05f842eaa3eb48 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 5 Jul 2023 16:27:53 -0400 Subject: [PATCH 098/114] Document inferElidedVariantFields --- crucible-mir/src/Mir/Mir.hs | 2 ++ crucible-mir/src/Mir/TransTy.hs | 32 +++++++++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/crucible-mir/src/Mir/Mir.hs b/crucible-mir/src/Mir/Mir.hs index 4a77707d8..82fbc7943 100644 --- a/crucible-mir/src/Mir/Mir.hs +++ b/crucible-mir/src/Mir/Mir.hs @@ -353,6 +353,8 @@ data Rvalue = | ThreadLocalRef DefId Ty deriving (Show,Eq, Ord, Generic) +-- | An aggregate ADT expression. Currently, this is only used for enums (see +-- "Mir.Pass.AllocateEnum"). data AdtAg = AdtAg { _agadt :: Adt, _avgariant :: Integer, _aops :: [Operand], _adtagty :: Ty } deriving (Show, Eq, Ord, Generic) diff --git a/crucible-mir/src/Mir/TransTy.hs b/crucible-mir/src/Mir/TransTy.hs index 39a6c6a71..418a68118 100644 --- a/crucible-mir/src/Mir/TransTy.hs +++ b/crucible-mir/src/Mir/TransTy.hs @@ -937,17 +937,43 @@ buildEnum' adt i es = do R.App $ E.InjectVariant ctx idx $ R.App $ E.MkStruct ctx' asn --- | TODO RGS: Add lots of documentation +-- It is possible for an enum to be initialized in MIR without providing +-- explicit assignments for all of its fields. As an example, imagine the value +-- @Ok(())@ of type @Result<(), i32>@. MIR is liable to construct this like so: +-- +-- @ +-- let mut _0 : Result<(), i32>; +-- discr(_0) = 0 +-- @ +-- +-- Note that there is no statement for explicitly initializing the first field +-- of @Ok@ to @()@. This is by design, as @()@ is a zero-sized type (ZST). While +-- ZSTs need not appear explicitly in MIR, we would like to have explicit +-- representations for them in Crucible. This function is responsible for +-- constructing these explicit representations. +-- +-- The approach that this function takes is pretty simple: if we encounter a +-- variant with the same number of types as field values, then return the values +-- unchanged. If there are fewer values than types, then we assume that any ZSTs +-- have elided the fields of the corresponding types, so we insert these values +-- into the list ourselves. We use 'initialValue' to construct a reasonable +-- value of each ZST. +-- +-- Note that we are doing this step somewhat late in the pipeline. An +-- alternative approach would be to infer these missing values in +-- "Mir.Pass.AllocateEnum", when the enum variant initialization is first +-- handled. This would require some additional refactoring, so we have not yet +-- pursued this option. inferElidedVariantFields :: [M.Ty] -> [Maybe (MirExp s)] -> MirGenerator h s ret [Maybe (MirExp s)] inferElidedVariantFields ftys fes - | length ftys == length fes -- TODO RGS: This is fishy. Consider an alternative approach. + | length ftys == length fes = pure fes | otherwise = go ftys fes where go [] [] = pure [] - go [] (_:_) = mirFail $ unlines [ "inferElidedVariantFields: more expressions than types" + go [] (_:_) = mirFail $ unlines [ "inferElidedVariantFields: too many expressions" , "types: " ++ show ftys , "expressions: " ++ show fes ] From a1c0bbcf30d80ac3ea62145da63ffb64358e56fa Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 5 Jul 2023 16:36:14 -0400 Subject: [PATCH 099/114] report-coverage: Avoid triggering rust-lang/rust#79813 warning --- crux-mir/report-coverage/src/main.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crux-mir/report-coverage/src/main.rs b/crux-mir/report-coverage/src/main.rs index 5ffca2a7c..568bc51a8 100644 --- a/crux-mir/report-coverage/src/main.rs +++ b/crux-mir/report-coverage/src/main.rs @@ -306,7 +306,9 @@ fn parse_branch(json: &Value) -> Result { Ok(BranchTrans::DropFlag) }, - _ => die!("unknown tag {:?} for branch", tag), + _ => { + die!("unknown tag {:?} for branch", tag); + }, } } From 352fed489a6324b2f9be148169036b0282e1c335 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 5 Jul 2023 16:46:22 -0400 Subject: [PATCH 100/114] crux-mir: Add Box stress test --- crux-mir/test/conc_eval/box/move_out.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 crux-mir/test/conc_eval/box/move_out.rs diff --git a/crux-mir/test/conc_eval/box/move_out.rs b/crux-mir/test/conc_eval/box/move_out.rs new file mode 100644 index 000000000..fd190c0cc --- /dev/null +++ b/crux-mir/test/conc_eval/box/move_out.rs @@ -0,0 +1,19 @@ +#![cfg_attr(not(with_main), no_std)] + +#[cfg(not(with_main))] extern crate std; +#[cfg(not(with_main))] use std::boxed::Box; +#[cfg(not(with_main))] use std::string::String; + +fn move_out(x: Box) -> String { + *x +} + +#[cfg_attr(crux, crux::test)] +pub fn f() { + let s: String = "Hello, World!".into(); + let b = Box::new(s.clone()); + assert!(move_out(b) == s); +} + + +#[cfg(with_main)] pub fn main() { println!("{:?}", f()); } From ed718aa66f550054759b06dafb42e66e7df89bc7 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 5 Jul 2023 17:08:16 -0400 Subject: [PATCH 101/114] CI: Use up-to-date Rust nightly --- .github/workflows/crux-mir-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/crux-mir-build.yml b/.github/workflows/crux-mir-build.yml index d1e5e4c33..f10761a11 100644 --- a/.github/workflows/crux-mir-build.yml +++ b/.github/workflows/crux-mir-build.yml @@ -101,7 +101,7 @@ jobs: - name: Install latest Rust nightly uses: actions-rs/toolchain@v1 with: - toolchain: nightly-2020-03-22 + toolchain: nightly-2023-01-23 override: true components: rustc-dev From 42a36061cad34150ae3be5b9a89e3c1937dd0ef7 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 5 Jul 2023 17:10:11 -0400 Subject: [PATCH 102/114] crux-mir/lib/Patches.md: Fill in some TODOs --- crux-mir/lib/Patches.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crux-mir/lib/Patches.md b/crux-mir/lib/Patches.md index a978ac67d..4a86a00b5 100644 --- a/crux-mir/lib/Patches.md +++ b/crux-mir/lib/Patches.md @@ -26,11 +26,12 @@ identify all of the code that was changed in each patch. * Reimplement `core::fmt` using `crucible::any::Any` (last applied: May 2, 2023) - TODO: Describe why this is necessary + This avoids some particularly nasty `transmute`s. * Implement `HashMap` in terms of `Vec` (last applied: May 4, 2023) - TODO: Describe why this is necessary + The actual implementation (in terms of `hashbrown`) is too complicated for + Crucible to handle efficiently. * Implement `ptr::invalid` and `ptr::invalid_mut` in terms of casts instead of `transmute` (last applied: May 9, 2023) From ef8fc3769811a2e5ff544dcda25a8697aa69d1f4 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 5 Jul 2023 17:26:49 -0400 Subject: [PATCH 103/114] crucible-mir: Parenthesize some equality constraints --- crucible-mir/src/Mir/TransTy.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crucible-mir/src/Mir/TransTy.hs b/crucible-mir/src/Mir/TransTy.hs index 418a68118..ff59efec7 100644 --- a/crucible-mir/src/Mir/TransTy.hs +++ b/crucible-mir/src/Mir/TransTy.hs @@ -236,10 +236,10 @@ tyToRepr col t0 = case t0 of _ -> error $ unwords ["unknown type?", show t0] -pattern DynRefCtx :: () => ctx ~ Ctx.EmptyCtx Ctx.::> C.AnyType Ctx.::> C.AnyType => Ctx.Assignment C.TypeRepr ctx +pattern DynRefCtx :: () => (ctx ~ Ctx.EmptyCtx Ctx.::> C.AnyType Ctx.::> C.AnyType) => Ctx.Assignment C.TypeRepr ctx pattern DynRefCtx = Ctx.Empty Ctx.:> C.AnyRepr Ctx.:> C.AnyRepr -pattern DynRefRepr :: () => tp ~ DynRefType => C.TypeRepr tp +pattern DynRefRepr :: () => (tp ~ DynRefType) => C.TypeRepr tp pattern DynRefRepr = C.StructRepr DynRefCtx From 9ae78ed8ed39fc0f32d4137e7e3376931a3e81e9 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Wed, 5 Jul 2023 17:31:44 -0400 Subject: [PATCH 104/114] crucible-mir: Avoid some failable do-block matches on ST --- crucible-mir/src/Mir/Trans.hs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index 561f4129a..f3f627adb 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -982,7 +982,12 @@ evalRval (M.Len lv) = M.TyArray _ len -> return $ MirExp UsizeRepr $ R.App $ usizeLit $ fromIntegral len ty@(M.TySlice _) -> do - MirPlace _tpr _ref meta <- evalPlace lv + place <- evalPlace lv + meta <- case place of + MirPlace _tpr _ref meta -> pure meta + MirPlaceDynRef{} -> + -- See https://github.com/GaloisInc/crucible/issues/1092 + mirFail "evalRval (length of slice) not supported for dyn references" case meta of SliceMeta len -> return $ MirExp UsizeRepr len _ -> mirFail $ "bad metadata " ++ show meta ++ " for reference to " ++ show ty @@ -1193,11 +1198,16 @@ doAssignCoerce lv ty exp = doAssign :: HasCallStack => M.Lvalue -> MirExp s -> MirGenerator h s ret () doAssign lv (MirExp tpr val) = do - MirPlace tpr' ref _ <- evalPlace lv - Refl <- testEqualityOrFail tpr tpr' $ - "ill-typed assignment of " ++ show tpr ++ " to " ++ show tpr' - ++ " (" ++ show (M.typeOf lv) ++ ") " ++ show lv - writeMirRef ref val + place <- evalPlace lv + case place of + MirPlaceDynRef{} -> + -- See https://github.com/GaloisInc/crucible/issues/1092 + mirFail "doAssign not supported for dyn references" + MirPlace tpr' ref _ -> do + Refl <- testEqualityOrFail tpr tpr' $ + "ill-typed assignment of " ++ show tpr ++ " to " ++ show tpr' + ++ " (" ++ show (M.typeOf lv) ++ ") " ++ show lv + writeMirRef ref val transStatement :: HasCallStack => M.Statement -> MirGenerator h s ret () From ee39bf09c1557ab57766e3feecf9809fc35ab531 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 6 Jul 2023 08:21:53 -0400 Subject: [PATCH 105/114] crucible-mir: More parentheses --- crucible-mir/src/Mir/TransTy.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crucible-mir/src/Mir/TransTy.hs b/crucible-mir/src/Mir/TransTy.hs index ff59efec7..52bb71c59 100644 --- a/crucible-mir/src/Mir/TransTy.hs +++ b/crucible-mir/src/Mir/TransTy.hs @@ -236,7 +236,7 @@ tyToRepr col t0 = case t0 of _ -> error $ unwords ["unknown type?", show t0] -pattern DynRefCtx :: () => (ctx ~ Ctx.EmptyCtx Ctx.::> C.AnyType Ctx.::> C.AnyType) => Ctx.Assignment C.TypeRepr ctx +pattern DynRefCtx :: () => (ctx ~ (Ctx.EmptyCtx Ctx.::> C.AnyType Ctx.::> C.AnyType)) => Ctx.Assignment C.TypeRepr ctx pattern DynRefCtx = Ctx.Empty Ctx.:> C.AnyRepr Ctx.:> C.AnyRepr pattern DynRefRepr :: () => (tp ~ DynRefType) => C.TypeRepr tp From e0d77afab2b1c204372fd085071dace03cdb9e0f Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Fri, 7 Jul 2023 15:48:53 -0400 Subject: [PATCH 106/114] Use crux::test instead of crux_test --- crux-mir/README.md | 7 ++++--- crux-mir/example/ffs/lib.rs | 3 +-- dependencies/mir-json | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crux-mir/README.md b/crux-mir/README.md index 1c2fd8544..b00b3840a 100644 --- a/crux-mir/README.md +++ b/crux-mir/README.md @@ -40,10 +40,11 @@ version, then rerun both commands. ### Writing test cases -`crux-mir` looks for functions with the `#[crux_test]` attribute and runs them +`crux-mir` looks for functions with the `#[crux::test]` attribute and runs them as tests. You may need to add `#![feature(custom_attribute)]` to the crate -root to use the `#[crux_test]` attribute. These can both be conditionally -compiled by checking for `#[cfg(crux)]`. +root to use the `#[crux::test]` attribute. These can both be conditionally +compiled by checking for the `crux` configuration predicate using +`#[cfg_attr(crux, crux::test)]`. Test cases can create and manipulate symbolic values using the functions in the [`crucible`](lib/crucible) Rust crate. See diff --git a/crux-mir/example/ffs/lib.rs b/crux-mir/example/ffs/lib.rs index ca06d66c6..b857e9174 100644 --- a/crux-mir/example/ffs/lib.rs +++ b/crux-mir/example/ffs/lib.rs @@ -35,8 +35,7 @@ fn test_ffs_correct_concrete() { } /// Check that ffs_fast and ffs_ref produce the same output on *every* input. -#[cfg(crux)] -#[crux_test] +#[cfg_attr(crux, crux::test)] fn test_ffs_correct() { let x = u32::symbolic("x"); let a = ffs_ref(x); diff --git a/dependencies/mir-json b/dependencies/mir-json index a65ab6703..9f1997edb 160000 --- a/dependencies/mir-json +++ b/dependencies/mir-json @@ -1 +1 @@ -Subproject commit a65ab670394e90feb84650f82868a7bbdaed71c8 +Subproject commit 9f1997edb6e25353d741f1ca0872c48111ec3f5a From c8eb7b6175507cbe14936a031ce13661e3d62c68 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 13 Jul 2023 14:10:43 -0400 Subject: [PATCH 107/114] crucible-mir: Remove some commented-out old code --- crucible-mir/src/Mir/Trans.hs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index f3f627adb..549fdb2a3 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -158,7 +158,6 @@ transConstVal (M.TyRef (M.TySlice ty) _) (Some (MirSliceRepr tpr)) (M.ConstSlice show tpr ++ ", got " ++ show tpr' pure c' vec <- mirVector_fromVector tpr $ R.App $ E.VectorLit tpr $ V.fromList cs' - -- return $ MirExp (MirVectorRepr tpr) vec vecRef <- constMirRef (MirVectorRepr tpr) vec ref <- subindexRef tpr vecRef (R.App $ usizeLit 0) let len = R.App $ usizeLit $ fromIntegral $ length cs @@ -1539,10 +1538,6 @@ transTerminator (M.DropAndReplace dlv dop dtarg _ dropFn) _ = do transStatement (M.Assign dlv (M.Use dop) "") jumpToBlock dtarg --- transTerminator (M.Call (M.OpConstant (M.Constant _ (M.ConstFunction funid))) cargs cretdest _) tr = do --- isCustom <- resolveCustom funid --- doCall funid cargs cretdest tr -- cleanup ignored - transTerminator (M.Call (M.OpConstant (M.Constant (M.TyFnDef funid) _)) cargs cretdest _) tr = do isCustom <- resolveCustom funid doCall funid cargs cretdest tr -- cleanup ignored From b794aca58b73f36121b47679a72f4ba3e37a660b Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 13 Jul 2023 14:11:43 -0400 Subject: [PATCH 108/114] crucible-mir: Document reasoning for some evalRval cases --- crucible-mir/src/Mir/Trans.hs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index 549fdb2a3..8ef4fd440 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -1039,8 +1039,12 @@ evalRval rv@(M.RAdtAg (M.AdtAg adt agv ops ty)) = do _ -> mirFail $ "evalRval: unsupported type for AdtAg: " ++ show ty evalRval (M.ThreadLocalRef did _) = staticPlace did >>= addrOfPlace --- TODO: are these correct? +-- ShallowInitBox(T, op) is equivalent to mem::transmute::<*mut T, Box>(op) +-- (modulo some rustc-internal analyses). We represent Box the same as +-- *mut T, so we implement this transmute as the identity function. evalRval rv@(M.ShallowInitBox op ty) = evalOperand op + +-- We treat CopyForDeref(lv) the same as Rvalue::Use(Operand::Copy(lv)). evalRval rv@(M.CopyForDeref lv) = evalLvalue lv evalLvalue :: HasCallStack => M.Lvalue -> MirGenerator h s ret (MirExp s) From 8c11bca669c2f929858aa4fcaafdb98ce86e4428 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 13 Jul 2023 14:54:30 -0400 Subject: [PATCH 109/114] crux-mir: Add test case for ConstKind::Unevaluated --- crux-mir/test/conc_eval/consts/unevaluated.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 crux-mir/test/conc_eval/consts/unevaluated.rs diff --git a/crux-mir/test/conc_eval/consts/unevaluated.rs b/crux-mir/test/conc_eval/consts/unevaluated.rs new file mode 100644 index 000000000..e02242513 --- /dev/null +++ b/crux-mir/test/conc_eval/consts/unevaluated.rs @@ -0,0 +1,14 @@ +const FOO: usize = 5; + +fn f(xs: [i32; FOO]) -> i32 { + xs[2] +} + +#[cfg_attr(crux, crux::test)] +fn crux_test() -> i32 { + f([0, 1, 2, 3, 4]) +} + +pub fn main() { + println!("{:?}", crux_test()); +} From 0afbb96a21c6d4fbf449d90c2591e1b5c997f1f8 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 13 Jul 2023 15:46:01 -0400 Subject: [PATCH 110/114] Properly implement transStatement case for StmtIntrinsic --- crucible-mir/src/Mir/Trans.hs | 60 ++++++++++++++++++++++++++++- crucible-mir/src/Mir/TransCustom.hs | 27 +------------ 2 files changed, 60 insertions(+), 27 deletions(-) diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index 8ef4fd440..44a0dd680 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -34,7 +34,7 @@ module Mir.Trans(transCollection,transStatics,RustModule(..) , writeMirRef , subindexRef , evalBinOp - , vectorCopy, ptrCopy + , vectorCopy, ptrCopy, copyNonOverlapping , evalRval , callExp , derefExp, readPlace, addrOfPlace @@ -1237,7 +1237,32 @@ transStatement (M.SetDiscriminant lv i) = do ty -> mirFail $ "don't know how to set discriminant of " ++ show ty transStatement M.Nop = return () transStatement M.Deinit = return () -transStatement (M.StmtIntrinsic _) = return () --TODO: is this true? +transStatement (M.StmtIntrinsic ndi) = + case ndi of + -- rustc uses assumptions from `assume` to optimize code. If we + -- encounter an occurrence of `assume` in MIR code, we should check + -- that its argument is equal to `true`, as `assume(false)` is UB. + NDIAssume cond -> do + MirExp tpr e <- evalOperand cond + Refl <- testEqualityOrFail tpr C.BoolRepr "expected Assume cond to be BoolType" + G.assertExpr (S.app $ E.BoolEq e (S.app $ E.BoolLit True)) $ + S.app $ E.StringLit $ W4.UnicodeLiteral $ "assume(false) is undefined behavior" + NDICopyNonOverlapping srcOp destOp countOp -> do + srcExp <- evalOperand srcOp + destExp <- evalOperand destOp + countExp <- evalOperand countOp + case (srcExp, destExp, countExp) of + ( MirExp (MirReferenceRepr tpr) src, + MirExp (MirReferenceRepr tpr') dest, + MirExp UsizeRepr count ) + | Just Refl <- testEquality tpr tpr' + -> void $ copyNonOverlapping tpr src dest count + _ -> mirFail $ unlines + [ "bad arguments for intrinsics::copy_nonoverlapping: " + , show srcExp + , show destExp + , show countExp + ] -- | Add a new `BranchTransInfo` entry for the current function. Returns the -- index of the new entry. @@ -2342,6 +2367,37 @@ ptrCopy tpr src dest len = do G.writeRef iRef i') G.dropRef iRef +copyNonOverlapping :: + C.TypeRepr tp -> + R.Expr MIR s (MirReferenceType tp) -> + R.Expr MIR s (MirReferenceType tp) -> + R.Expr MIR s UsizeType -> + MirGenerator h s ret (MirExp s) +copyNonOverlapping tpr src dest count = do + -- Assert that the two regions really are nonoverlapping. + maybeOffset <- mirRef_tryOffsetFrom dest src + + -- `count` must not exceed isize::MAX, else the overlap check + -- will misbehave. + let sizeBits = fromIntegral $ C.intValue (C.knownNat @SizeBits) + let maxCount = R.App $ usizeLit (1 `shift` (sizeBits - 1)) + let countOk = R.App $ usizeLt count maxCount + G.assertExpr countOk $ S.litExpr "count overflow in copy_nonoverlapping" + + -- If `maybeOffset` is Nothing, then src and dest definitely + -- don't overlap, since they come from different allocations. + -- If it's Just, the value must be >= count or <= -count to put + -- the two regions far enough apart. + let count' = usizeToIsize R.App count + let destAbove = \offset -> R.App $ isizeLe count' offset + let destBelow = \offset -> R.App $ isizeLe offset (R.App $ isizeNeg count') + offsetOk <- G.caseMaybe maybeOffset C.BoolRepr $ G.MatchMaybe + (\offset -> return $ R.App $ E.Or (destAbove offset) (destBelow offset)) + (return $ R.App $ E.BoolLit True) + G.assertExpr offsetOk $ S.litExpr "src and dest overlap in copy_nonoverlapping" + + ptrCopy tpr src dest count + return $ MirExp C.UnitRepr $ R.App E.EmptyApp -- LocalWords: params IndexMut FnOnce Fn IntoIterator iter impl -- LocalWords: tovec fromelem tmethsubst MirExp initializer callExp diff --git a/crucible-mir/src/Mir/TransCustom.hs b/crucible-mir/src/Mir/TransCustom.hs index 9c020bde0..70eefbcb9 100644 --- a/crucible-mir/src/Mir/TransCustom.hs +++ b/crucible-mir/src/Mir/TransCustom.hs @@ -613,31 +613,8 @@ intrinsics_copy_nonoverlapping = ( ["core", "intrinsics", "copy_nonoverlapping"] [MirExp (MirReferenceRepr tpr) src, MirExp (MirReferenceRepr tpr') dest, MirExp UsizeRepr count] - | Just Refl <- testEquality tpr tpr' -> do - -- Assert that the two regions really are nonoverlapping. - maybeOffset <- mirRef_tryOffsetFrom dest src - - -- `count` must not exceed isize::MAX, else the overlap check - -- will misbehave. - let sizeBits = fromIntegral $ C.intValue (C.knownNat @SizeBits) - let maxCount = R.App $ usizeLit (1 `shift` (sizeBits - 1)) - let countOk = R.App $ usizeLt count maxCount - G.assertExpr countOk $ S.litExpr "count overflow in copy_nonoverlapping" - - -- If `maybeOffset` is Nothing, then src and dest definitely - -- don't overlap, since they come from different allocations. - -- If it's Just, the value must be >= count or <= -count to put - -- the two regions far enough apart. - let count' = usizeToIsize R.App count - let destAbove = \offset -> R.App $ isizeLe count' offset - let destBelow = \offset -> R.App $ isizeLe offset (R.App $ isizeNeg count') - offsetOk <- G.caseMaybe maybeOffset C.BoolRepr $ G.MatchMaybe - (\offset -> return $ R.App $ E.Or (destAbove offset) (destBelow offset)) - (return $ R.App $ E.BoolLit True) - G.assertExpr offsetOk $ S.litExpr "src and dest overlap in copy_nonoverlapping" - - ptrCopy tpr src dest count - return $ MirExp C.UnitRepr $ R.App E.EmptyApp + | Just Refl <- testEquality tpr tpr' -> + copyNonOverlapping tpr src dest count _ -> mirFail $ "bad arguments for intrinsics::copy_nonoverlapping: " ++ show ops _ -> Nothing) From d2ff2cce568713a97389f1499546cd1ef0ea6de9 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 13 Jul 2023 17:06:06 -0400 Subject: [PATCH 111/114] crucible-mir: Expunge defaultDisambiguator and anything that uses it This require a fair bit of refactoring to avoid problematic uses of `textId` that were relying on `defaultDisambiguator`. --- crucible-mir/src/Mir/DefId.hs | 16 ++++------- crucible-mir/src/Mir/Generator.hs | 37 +++++++++++++++++------- crucible-mir/src/Mir/TransCustom.hs | 30 +++++++++++--------- crucible-mir/src/Mir/TransTy.hs | 44 ++++++++++++----------------- 4 files changed, 67 insertions(+), 60 deletions(-) diff --git a/crucible-mir/src/Mir/DefId.hs b/crucible-mir/src/Mir/DefId.hs index c3b18d53c..77c803175 100644 --- a/crucible-mir/src/Mir/DefId.hs +++ b/crucible-mir/src/Mir/DefId.hs @@ -64,20 +64,14 @@ data DefId = DefId makeLenses ''DefId --- | The crate disambiguator hash produced when the crate metadata string is --- empty. This is useful for creating 'DefId's for thing that do not have a --- disambiguator associated with it (e.g., Crucible names). -defaultDisambiguator :: Text -defaultDisambiguator = "3a1fbbbh" - -- | Parse a string into a `DefId`. -- -- For convenience when writing literal paths in the Haskell source, both types -- of disambiguators are optional. If the crate disambiguator is omitted, then --- it's assumed to be `defaultDisambiguator`, and if a segment disambiguator is --- omitted elsewhere in the path, it's assumed to be zero. So you can write, --- for example, `foo::bar::Baz`, and parsing will expand it to the --- canonical form `foo/3a1fbbbh::bar[0]::Baz[0]`. +-- an exception is thrown. If a segment disambiguator is omitted elsewhere in +-- the path, it's assumed to be zero. So you can write, for example, +-- `foo/3a1fbbbh::bar::Baz`, and parsing will expand it to the canonical form +-- `foo/3a1fbbbh::bar[0]::Baz[0]`. textId :: Text -> DefId textId s = DefId crate disambig segs where @@ -86,7 +80,7 @@ textId s = DefId crate disambig segs x:xs -> (x, xs) (crate, disambig) = case T.splitOn "/" crateStr of - [x] -> (x, defaultDisambiguator) + [x] -> error $ "textId: No crate disambiguator for: " ++ T.unpack x [x, y] -> (x, y) _ -> error $ "textId: malformed crate name " ++ show crateStr diff --git a/crucible-mir/src/Mir/Generator.hs b/crucible-mir/src/Mir/Generator.hs index bb5307d6c..dac192be8 100644 --- a/crucible-mir/src/Mir/Generator.hs +++ b/crucible-mir/src/Mir/Generator.hs @@ -473,17 +473,36 @@ findAdtInst origName substs = do Just x -> return x Nothing -> mirFail $ "unknown ADT " ++ show (origName, substs) --- | Find the 'DefId' corresponding to the supplied text. This consults the --- 'crateHashesMap' to ensure that the crate's disambiguator is correct. If a --- crate name is ambiguous (i.e., if there are multiple disambiguators --- associated with the crate name), this will throw an error. -findDefId :: Text -> MirGenerator h s ret DefId -findDefId str = do +-- Like findAdtInst, but with an `ExplodedDefId` instead of a `DefId`. This uses +-- `findDefId` to compute the `DefId`. +findExplodedAdtInst :: ExplodedDefId -> Substs -> MirGenerator h s ret Adt +findExplodedAdtInst edid substs = do + did <- findDefId edid + findAdtInst did substs + +-- Like findExplodedAdtInst, but returning a `TyAdt`. +findExplodedAdtTy :: ExplodedDefId -> Substs -> MirGenerator h s ret Ty +findExplodedAdtTy edid substs = do + adt <- findExplodedAdtInst edid substs + pure $ TyAdt (adt ^. adtname) (adt ^. adtOrigDefId) (adt ^. adtOrigSubsts) + +-- | Find the 'DefId' corresponding to the supplied 'ExplodedDefId'. This +-- consults the 'crateHashesMap' to ensure that the crate's disambiguator is +-- correct. If a crate name is ambiguous (i.e., if there are multiple +-- disambiguators associated with the crate name), this will throw an error. +findDefId :: ExplodedDefId -> MirGenerator h s ret DefId +findDefId edid = do crateDisambigs <- use $ cs . crateHashesMap + (crate, path) <- + case edid of + crate:path -> pure (crate, path) + [] -> mirFail "findDefId: DefId with no crate" + let crateStr = Text.unpack crate case Map.lookup crate crateDisambigs of Just allDisambigs@(disambig :| otherDisambigs) | F.null otherDisambigs - -> pure $ partialDefId & didCrateDisambig .~ disambig + -> pure $ textId $ Text.intercalate "::" + $ (crate <> "/" <> disambig) : path | otherwise -> mirFail $ unlines $ [ "ambiguous crate " ++ crateStr @@ -491,9 +510,7 @@ findDefId str = do ] ++ F.toList (Text.unpack <$> allDisambigs) Nothing -> mirFail $ "unknown crate " ++ crateStr where - partialDefId = textId str - crate = partialDefId^.didCrate - crateStr = Text.unpack crate + -- partialDefId = textId str -- | What to do when the translation fails. mirFail :: String -> MirGenerator h s ret a diff --git a/crucible-mir/src/Mir/TransCustom.hs b/crucible-mir/src/Mir/TransCustom.hs index 70eefbcb9..fc412d92c 100644 --- a/crucible-mir/src/Mir/TransCustom.hs +++ b/crucible-mir/src/Mir/TransCustom.hs @@ -278,7 +278,9 @@ vector_pop = ( ["crucible","vector","{impl}", "pop"], ) $ \substs -> case substs meInit <- MirExp (C.VectorRepr tpr) <$> vectorInit tpr eVec -- `Option` must exist because it appears in the return type. meLast <- vectorLast tpr eVec >>= maybeToOption t tpr - buildTupleMaybeM [CTyVector t, CTyOption t] [Just meInit, Just meLast] + vectorTy <- findExplodedAdtTy vectorExplodedDefId (Substs [t]) + optionTy <- findExplodedAdtTy optionExplodedDefId (Substs [t]) + buildTupleMaybeM [vectorTy, optionTy] [Just meInit, Just meLast] _ -> mirFail $ "bad arguments for Vector::pop: " ++ show ops _ -> Nothing @@ -289,7 +291,9 @@ vector_pop_front = ( ["crucible","vector","{impl}", "pop_front"], ) $ \substs -> -- `Option` must exist because it appears in the return type. meHead <- vectorHead tpr eVec >>= maybeToOption t tpr meTail <- MirExp (C.VectorRepr tpr) <$> vectorTail tpr eVec - buildTupleMaybeM [CTyOption t, CTyVector t] [Just meHead, Just meTail] + optionTy <- findExplodedAdtTy optionExplodedDefId (Substs [t]) + vectorTy <- findExplodedAdtTy vectorExplodedDefId (Substs [t]) + buildTupleMaybeM [optionTy, vectorTy] [Just meHead, Just meTail] _ -> mirFail $ "bad arguments for Vector::pop_front: " ++ show ops _ -> Nothing @@ -333,7 +337,8 @@ vector_split_at = ( ["crucible","vector","{impl}", "split_at"], ) $ \substs -> c let eIdxNat = R.App $ usizeToNat eIdx mePre <- MirExp (C.VectorRepr tpr) <$> vectorTake tpr eVec eIdxNat meSuf <- MirExp (C.VectorRepr tpr) <$> vectorDrop tpr eVec eIdxNat - buildTupleMaybeM [CTyVector t, CTyVector t] [Just mePre, Just meSuf] + vectorTy <- findExplodedAdtTy vectorExplodedDefId (Substs [t]) + buildTupleMaybeM [vectorTy, vectorTy] [Just mePre, Just meSuf] _ -> mirFail $ "bad arguments for Vector::split_at: " ++ show ops _ -> Nothing @@ -1362,11 +1367,13 @@ type BVMakeLiteral = forall ext f w. bv_literal :: Text -> BVMakeLiteral -> (ExplodedDefId, CustomRHS) bv_literal name op = (["crucible", "bitvector", "{impl}", name], \(Substs [sz]) -> - Just $ CustomOp $ \_optys _ops -> tyToReprM (CTyBv sz) >>= \(Some tpr) -> case tpr of - C.BVRepr w -> - return $ MirExp (C.BVRepr w) $ S.app $ op w - _ -> mirFail $ - "BUG: invalid type param for bv_" ++ Text.unpack name ++ ": " ++ show sz) + Just $ CustomOp $ \_optys _ops -> do + bvTy <- findExplodedAdtTy bvExplodedDefId (Substs [sz]) + tyToReprM bvTy >>= \(Some tpr) -> case tpr of + C.BVRepr w -> + return $ MirExp (C.BVRepr w) $ S.app $ op w + _ -> mirFail $ + "BUG: invalid type param for bv_" ++ Text.unpack name ++ ": " ++ show sz) bv_leading_zeros :: (ExplodedDefId, CustomRHS) bv_leading_zeros = @@ -1578,9 +1585,7 @@ maybe_uninit_uninit :: (ExplodedDefId, CustomRHS) maybe_uninit_uninit = (["core", "mem", "maybe_uninit", "{impl}", "uninit"], \substs -> case substs of Substs [t] -> Just $ CustomOp $ \_ _ -> do - maybeUninitDefId <- findDefId "core::mem::maybe_uninit::MaybeUninit" - adt <- findAdtInst maybeUninitDefId (Substs [t]) - let t = TyAdt (adt ^. adtname) (adt ^. adtOrigDefId) (adt ^. adtOrigSubsts) + t <- findExplodedAdtTy maybeUninitExplodedDefId (Substs [t]) initialValue t >>= \mv -> case mv of Just v -> return v Nothing -> mirFail $ "MaybeUninit::uninit unsupported for " ++ show t @@ -1683,8 +1688,7 @@ unwrapMirExp tpr (MirExp tpr' e) maybeToOption :: Ty -> C.TypeRepr tp -> R.Expr MIR s (C.MaybeType tp) -> MirGenerator h s ret (MirExp s) maybeToOption ty tpr e = do - optionDefId <- findDefId optionDefIdText - adt <- findAdtInst optionDefId (Substs [ty]) + adt <- findExplodedAdtInst optionExplodedDefId (Substs [ty]) e' <- G.caseMaybe e C.AnyRepr $ G.MatchMaybe (\val -> buildEnum adt optionDiscrSome [MirExp tpr val] >>= unwrapMirExp C.AnyRepr) (buildEnum adt optionDiscrNone [] >>= unwrapMirExp C.AnyRepr) diff --git a/crucible-mir/src/Mir/TransTy.hs b/crucible-mir/src/Mir/TransTy.hs index 52bb71c59..22c3fcd53 100644 --- a/crucible-mir/src/Mir/TransTy.hs +++ b/crucible-mir/src/Mir/TransTy.hs @@ -100,49 +100,48 @@ baseSizeToNatCont M.USize k = k (knownNat :: NatRepr SizeBits) -- Custom type aliases pattern CTyInt512 <- M.TyAdt _ $(M.explodedDefIdPat ["int512", "Int512"]) (M.Substs []) - where CTyInt512 = M.TyAdt (M.textId "type::adt") (M.textId "int512::Int512") (M.Substs []) pattern CTyBox t <- M.TyAdt _ $(M.explodedDefIdPat ["alloc", "boxed", "Box"]) (M.Substs [t]) - where CTyBox t = M.TyAdt (M.textId "type::adt") (M.textId "alloc::boxed::Box") (M.Substs [t]) + pattern CTyMaybeUninit t <- M.TyAdt _ $(M.explodedDefIdPat ["core", "mem", "maybe_uninit", "MaybeUninit"]) (M.Substs [t]) - where CTyMaybeUninit t = M.TyAdt (M.textId "type::adt") (M.textId "core::mem::maybe_uninit::MaybeUninit") (M.Substs [t]) + +maybeUninitExplodedDefId :: M.ExplodedDefId +maybeUninitExplodedDefId = ["core", "mem", "maybe_uninit", "MaybeUninit"] + -- `UnsafeCell` isn't handled specially inside baseline `crucible-mir`, but -- `crux-mir-comp` looks for it (using this pattern synonym). pattern CTyUnsafeCell t <- M.TyAdt _ $(M.explodedDefIdPat ["core", "cell", "UnsafeCell"]) (M.Substs [t]) - where CTyUnsafeCell t = M.TyAdt (M.textId "type::adt") (M.textId "core::cell::UnsafeCell") (M.Substs [t]) + pattern CTyVector t <- M.TyAdt _ $(M.explodedDefIdPat ["crucible", "vector", "Vector"]) (M.Substs [t]) - where CTyVector t = M.TyAdt (M.textId "type::adt") (M.textId "crucible::vector::Vector") (M.Substs [t]) + +vectorExplodedDefId :: M.ExplodedDefId +vectorExplodedDefId = ["crucible", "vector", "Vector"] + pattern CTyArray t <- M.TyAdt _ $(M.explodedDefIdPat ["crucible", "array", "Array"]) (M.Substs [t]) - where CTyArray t = M.TyAdt (M.textId "type::adt") (M.textId "crucible::array::Array") (M.Substs [t]) pattern CTyBvSize128 <- M.TyAdt _ $(M.explodedDefIdPat ["crucible", "bitvector", "_128"]) (M.Substs []) - where CTyBvSize128 = M.TyAdt (M.textId "type::adt") (M.textId "crucible::bitvector::_128") (M.Substs []) pattern CTyBvSize256 <- M.TyAdt _ $(M.explodedDefIdPat ["crucible", "bitvector", "_256"]) (M.Substs []) - where CTyBvSize256 = M.TyAdt (M.textId "type::adt") (M.textId "crucible::bitvector::_256") (M.Substs []) pattern CTyBvSize512 <- M.TyAdt _ $(M.explodedDefIdPat ["crucible", "bitvector", "_512"]) (M.Substs []) - where CTyBvSize512 = M.TyAdt (M.textId "type::adt") (M.textId "crucible::bitvector::_512") (M.Substs []) pattern CTyBv t <- M.TyAdt _ $(M.explodedDefIdPat ["crucible", "bitvector", "Bv"]) (M.Substs [t]) - where CTyBv t = M.TyAdt (M.textId "type::adt") (M.textId "crucible::bitvector::Bv") (M.Substs [t]) -pattern CTyBv128 = CTyBv CTyBvSize128 -pattern CTyBv256 = CTyBv CTyBvSize256 -pattern CTyBv512 = CTyBv CTyBvSize512 +pattern CTyBv128 <- CTyBv CTyBvSize128 +pattern CTyBv256 <- CTyBv CTyBvSize256 +pattern CTyBv512 <- CTyBv CTyBvSize512 + +bvExplodedDefId :: M.ExplodedDefId +bvExplodedDefId = ["crucible", "bitvector", "Bv"] pattern CTyAny <- M.TyAdt _ $(M.explodedDefIdPat ["core", "crucible", "any", "Any"]) (M.Substs []) - where CTyAny = M.TyAdt (M.textId "type::adt") (M.textId "core::crucible::any::Any") (M.Substs []) pattern CTyMethodSpec <- M.TyAdt _ $(M.explodedDefIdPat ["crucible", "method_spec", "raw", "MethodSpec"]) (M.Substs []) - where CTyMethodSpec = M.TyAdt (M.textId "type::adt") (M.textId "crucible::method_spec::raw::MethodSpec") (M.Substs []) pattern CTyMethodSpecBuilder <- M.TyAdt _ $(M.explodedDefIdPat ["crucible", "method_spec", "raw", "MethodSpecBuilder"]) (M.Substs []) - where CTyMethodSpecBuilder = M.TyAdt (M.textId "type::adt") (M.textId "crucible::method_spec::raw::MethodSpecBuilder") (M.Substs []) -- These don't have custom representation, but are referenced in various -- places. pattern CTyOption t <- M.TyAdt _ $(M.explodedDefIdPat ["core", "option", "Option"]) (M.Substs [t]) - where CTyOption t = M.TyAdt (M.textId "type::adt") (M.textId "core::option::Option") (M.Substs [t]) -optionDefIdText :: Text -optionDefIdText = "core::option::Option" +optionExplodedDefId :: M.ExplodedDefId +optionExplodedDefId = ["core", "option", "Option"] optionDiscrNone :: Int optionDiscrNone = 0 optionDiscrSome :: Int @@ -305,13 +304,6 @@ tyAdtDef _ _ = Nothing -- | If the `Adt` is a `repr(transparent)` struct with at most one -- non-zero-sized field, return the index of that field. findReprTransparentField :: M.Collection -> M.Adt -> Maybe Int -findReprTransparentField _ adt - -- Special case: Box isn't repr(transparent), but we pretend that it is. - -- Since its inner `Unique` pointer is repr(transparent), this results in - -- Box being represented as a plain `MirReference`. This simplifies some - -- parts of the translation, namely the `Box` operator and the `Deref` - -- projection (which can be applied to boxes just like ordinary refs). - | adt ^. M.adtOrigDefId == M.textId "alloc::boxed::Box" = Just 0 findReprTransparentField col adt = do guard $ adt ^. M.adtReprTransparent [v] <- return $ adt ^. M.adtvariants From 331de03b9c547deebd2d069a02be1e836d1a9926 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 13 Jul 2023 18:42:34 -0400 Subject: [PATCH 112/114] crucible-mir: ShallowInitBox is unsupported for now Because `Box` no longer uses the same `TypeRep` as `mut *T`, it is not sound to translate `ShallowInitBox` as though it were an identity function. Thankfully, nothing in the test suite actually uses `ShallowInitBox`, so we simply leave it unimplemented for now. --- crucible-mir/src/Mir/Trans.hs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/crucible-mir/src/Mir/Trans.hs b/crucible-mir/src/Mir/Trans.hs index 44a0dd680..52fd5661e 100644 --- a/crucible-mir/src/Mir/Trans.hs +++ b/crucible-mir/src/Mir/Trans.hs @@ -1039,14 +1039,12 @@ evalRval rv@(M.RAdtAg (M.AdtAg adt agv ops ty)) = do _ -> mirFail $ "evalRval: unsupported type for AdtAg: " ++ show ty evalRval (M.ThreadLocalRef did _) = staticPlace did >>= addrOfPlace --- ShallowInitBox(T, op) is equivalent to mem::transmute::<*mut T, Box>(op) --- (modulo some rustc-internal analyses). We represent Box the same as --- *mut T, so we implement this transmute as the identity function. -evalRval rv@(M.ShallowInitBox op ty) = evalOperand op - -- We treat CopyForDeref(lv) the same as Rvalue::Use(Operand::Copy(lv)). evalRval rv@(M.CopyForDeref lv) = evalLvalue lv +evalRval rv@(M.ShallowInitBox op ty) = mirFail + "evalRval: ShallowInitBox not supported" + evalLvalue :: HasCallStack => M.Lvalue -> MirGenerator h s ret (MirExp s) evalLvalue lv = evalPlace lv >>= readPlace From e8ac0779c68fe41a131a6e2c1ecd3f8644903757 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 13 Jul 2023 19:06:26 -0400 Subject: [PATCH 113/114] Bump mir-json submodule --- dependencies/mir-json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies/mir-json b/dependencies/mir-json index 9f1997edb..117ec97f9 160000 --- a/dependencies/mir-json +++ b/dependencies/mir-json @@ -1 +1 @@ -Subproject commit 9f1997edb6e25353d741f1ca0872c48111ec3f5a +Subproject commit 117ec97f971171cfac3eea37e5009e1f6bfa9ae8 From 0515e64ab5c39036c6577e06bbe4272d39de5656 Mon Sep 17 00:00:00 2001 From: Ryan Scott Date: Thu, 13 Jul 2023 19:16:37 -0400 Subject: [PATCH 114/114] crux-mir: Use simpler #[crux::test] attribute for symbolic tests There's not much of a point in checking for the `crux` configuration attribute, since one wouldn't be able to compile these tests with bare `rustc` anyway due to their use of the `crucible` crate. --- crux-mir/test/concurrency/atomic/atomic.rs | 2 +- crux-mir/test/concurrency/atomic/atomic_fetch.rs | 2 +- crux-mir/test/concurrency/mutex/mutex.rs | 4 ++-- crux-mir/test/concurrency/mutex/mutex_atomic.rs | 2 +- crux-mir/test/concurrency/mutex/thread_retval.rs | 2 +- crux-mir/test/concurrency/mutex/thread_retval_mutex.rs | 2 +- .../c/pthread-C-DAC/pthread-finding-k-matches.rs | 2 +- .../test/concurrency/sv-benchmarks/c/pthread/bigshot.rs | 6 +++--- .../test/concurrency/sv-benchmarks/c/pthread/fib_bench.rs | 4 ++-- .../test/concurrency/sv-benchmarks/c/pthread/indexer.rs | 2 +- .../test/concurrency/sv-benchmarks/c/pthread/queue.rs | 4 ++-- .../test/concurrency/sv-benchmarks/c/pthread/reorder.rs | 4 ++-- .../test/concurrency/sv-benchmarks/c/pthread/singleton.rs | 2 +- .../test/concurrency/sv-benchmarks/c/pthread/stack.rs | 4 ++-- .../test/concurrency/sv-benchmarks/c/pthread/stateful.rs | 4 ++-- .../concurrency/sv-benchmarks/c/pthread/triangular.rs | 4 ++-- .../sv-benchmarks/c/pthread/triangular_static.rs | 4 ++-- .../test/concurrency/sv-benchmarks/c/pthread/twostage.rs | 2 +- crux-mir/test/coverage/coverage.rs | 2 +- crux-mir/test/coverage/coverage_cond.rs | 2 +- crux-mir/test/coverage/coverage_dual.rs | 4 ++-- crux-mir/test/coverage/coverage_loop.rs | 2 +- crux-mir/test/coverage/coverage_macro.rs | 2 +- crux-mir/test/coverage/coverage_match.rs | 2 +- crux-mir/test/coverage/coverage_mono.rs | 2 +- crux-mir/test/coverage/coverage_shortcircuit.rs | 2 +- crux-mir/test/coverage/coverage_try.rs | 2 +- crux-mir/test/symb_eval/alloc/out_of_bounds.rs | 2 +- crux-mir/test/symb_eval/alloc/uninit_read.rs | 2 +- crux-mir/test/symb_eval/alloc/valid_read.rs | 2 +- crux-mir/test/symb_eval/alloc/zero_length.rs | 2 +- crux-mir/test/symb_eval/any/conditional.rs | 2 +- crux-mir/test/symb_eval/any/downcast.rs | 2 +- crux-mir/test/symb_eval/any/downcast_fail.rs | 2 +- crux-mir/test/symb_eval/array/basic.rs | 2 +- crux-mir/test/symb_eval/array/mux_slice.rs | 2 +- crux-mir/test/symb_eval/array/slice.rs | 2 +- crux-mir/test/symb_eval/array/slice_mut.rs | 2 +- crux-mir/test/symb_eval/bitvector/arith.rs | 2 +- crux-mir/test/symb_eval/bitvector/cmp.rs | 2 +- crux-mir/test/symb_eval/bitvector/from_to.rs | 2 +- crux-mir/test/symb_eval/bitvector/leading_zeros.rs | 2 +- crux-mir/test/symb_eval/bitvector/literals.rs | 2 +- crux-mir/test/symb_eval/bitvector/overflowing_sub.rs | 2 +- crux-mir/test/symb_eval/bitvector/symbolic.rs | 2 +- crux-mir/test/symb_eval/byteorder/read.rs | 2 +- crux-mir/test/symb_eval/byteorder/write.rs | 2 +- crux-mir/test/symb_eval/bytes/extend_bytes.rs | 2 +- crux-mir/test/symb_eval/bytes/new.rs | 2 +- crux-mir/test/symb_eval/bytes/put.rs | 2 +- crux-mir/test/symb_eval/bytes/put_overflow.rs | 2 +- crux-mir/test/symb_eval/bytes/split_off.rs | 2 +- crux-mir/test/symb_eval/bytes/split_to.rs | 2 +- crux-mir/test/symb_eval/bytes/sym_len.rs | 2 +- crux-mir/test/symb_eval/concretize/array.rs | 2 +- crux-mir/test/symb_eval/concretize/assert.rs | 2 +- crux-mir/test/symb_eval/concretize/assert_ok.rs | 2 +- crux-mir/test/symb_eval/concretize/conc.rs | 2 +- crux-mir/test/symb_eval/concretize/no_conc.rs | 2 +- crux-mir/test/symb_eval/crux/early_fail.rs | 4 ++-- crux-mir/test/symb_eval/crux/fail_return.rs | 4 ++-- crux-mir/test/symb_eval/crux/mixed_fail.rs | 8 ++++---- crux-mir/test/symb_eval/crux/multi.rs | 6 +++--- crux-mir/test/symb_eval/crypto/bytes.rs | 2 +- crux-mir/test/symb_eval/crypto/bytes2.rs | 2 +- crux-mir/test/symb_eval/crypto/double.rs | 2 +- crux-mir/test/symb_eval/crypto/ffs.rs | 2 +- crux-mir/test/symb_eval/enum/mux.rs | 2 +- crux-mir/test/symb_eval/fnptr/mux.rs | 2 +- crux-mir/test/symb_eval/io/vec_cursor_read.rs | 2 +- crux-mir/test/symb_eval/io/vec_write.rs | 2 +- crux-mir/test/symb_eval/mux/array.rs | 2 +- crux-mir/test/symb_eval/mux/array_mut.rs | 2 +- crux-mir/test/symb_eval/num/checked_add.rs | 2 +- crux-mir/test/symb_eval/num/checked_div.rs | 2 +- crux-mir/test/symb_eval/num/checked_mul.rs | 2 +- crux-mir/test/symb_eval/num/checked_mul_signed.rs | 2 +- crux-mir/test/symb_eval/overrides/bad_symb1.rs | 2 +- crux-mir/test/symb_eval/overrides/bad_symb2.rs | 2 +- crux-mir/test/symb_eval/overrides/override1.rs | 2 +- crux-mir/test/symb_eval/overrides/override2.rs | 2 +- crux-mir/test/symb_eval/overrides/override3.rs | 2 +- crux-mir/test/symb_eval/overrides/override4.rs | 2 +- crux-mir/test/symb_eval/overrides/override5.rs | 2 +- crux-mir/test/symb_eval/overrides/override_rust.rs | 2 +- crux-mir/test/symb_eval/refs/mux_init_imm.rs | 2 +- crux-mir/test/symb_eval/refs/mux_init_mut.rs | 2 +- crux-mir/test/symb_eval/scalar/test1.rs | 2 +- crux-mir/test/symb_eval/sym_bytes/construct.rs | 2 +- crux-mir/test/symb_eval/sym_bytes/deserialize.rs | 2 +- crux-mir/test/symb_eval/vec/clone.rs | 2 +- crux-mir/test/symb_eval/vec/into_iter.rs | 2 +- crux-mir/test/symb_eval/vec/macro.rs | 2 +- crux-mir/test/symb_eval/vec/sort_by_key.rs | 2 +- crux-mir/test/symb_eval/vector/as_mut_slice.rs | 2 +- crux-mir/test/symb_eval/vector/as_slice.rs | 2 +- crux-mir/test/symb_eval/vector/concat.rs | 2 +- crux-mir/test/symb_eval/vector/copy_from_slice.rs | 2 +- crux-mir/test/symb_eval/vector/mut.rs | 2 +- crux-mir/test/symb_eval/vector/new.rs | 2 +- crux-mir/test/symb_eval/vector/pop.rs | 2 +- crux-mir/test/symb_eval/vector/push.rs | 2 +- crux-mir/test/symb_eval/vector/replicate.rs | 2 +- crux-mir/test/symb_eval/vector/split_at.rs | 2 +- 104 files changed, 122 insertions(+), 122 deletions(-) diff --git a/crux-mir/test/concurrency/atomic/atomic.rs b/crux-mir/test/concurrency/atomic/atomic.rs index 91555dcaa..d266df096 100644 --- a/crux-mir/test/concurrency/atomic/atomic.rs +++ b/crux-mir/test/concurrency/atomic/atomic.rs @@ -4,7 +4,7 @@ use std::thread; use std::sync::{Arc,atomic}; #[cfg(not(with_main))] -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test_nofail() { let N = 3; diff --git a/crux-mir/test/concurrency/atomic/atomic_fetch.rs b/crux-mir/test/concurrency/atomic/atomic_fetch.rs index 317239538..588a4ad38 100644 --- a/crux-mir/test/concurrency/atomic/atomic_fetch.rs +++ b/crux-mir/test/concurrency/atomic/atomic_fetch.rs @@ -4,7 +4,7 @@ use std::thread; use std::sync::{Arc,atomic}; #[cfg(not(with_main))] -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test_nofail() { let N = 3; diff --git a/crux-mir/test/concurrency/mutex/mutex.rs b/crux-mir/test/concurrency/mutex/mutex.rs index 9d262da15..c52b2f148 100644 --- a/crux-mir/test/concurrency/mutex/mutex.rs +++ b/crux-mir/test/concurrency/mutex/mutex.rs @@ -4,7 +4,7 @@ use std::thread; use std::sync::{Arc,Mutex}; #[cfg(not(with_main))] -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test_nofail() { let data = Arc::new(Mutex::new(0 as u32)); let N = 3; @@ -37,7 +37,7 @@ fn crux_test_nofail() { data.crucible_TEMP_unlock(); } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test_fail() { let data = Arc::new(Mutex::new(0 as u32)); let N = 2; diff --git a/crux-mir/test/concurrency/mutex/mutex_atomic.rs b/crux-mir/test/concurrency/mutex/mutex_atomic.rs index f72ce0f73..7503987e7 100644 --- a/crux-mir/test/concurrency/mutex/mutex_atomic.rs +++ b/crux-mir/test/concurrency/mutex/mutex_atomic.rs @@ -6,7 +6,7 @@ use std::sync::atomic::AtomicU16; use std::sync::atomic::Ordering::SeqCst; #[cfg(not(with_main))] -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test_nofail() { let data = Arc::new(Mutex::new(())); let ctr = Arc::new(AtomicU16::new(0)); diff --git a/crux-mir/test/concurrency/mutex/thread_retval.rs b/crux-mir/test/concurrency/mutex/thread_retval.rs index 548f029a8..1cea29eab 100644 --- a/crux-mir/test/concurrency/mutex/thread_retval.rs +++ b/crux-mir/test/concurrency/mutex/thread_retval.rs @@ -4,7 +4,7 @@ use std::thread; use std::sync::{Arc,Mutex}; #[cfg(not(with_main))] -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() { let N = 2; let V = 3; diff --git a/crux-mir/test/concurrency/mutex/thread_retval_mutex.rs b/crux-mir/test/concurrency/mutex/thread_retval_mutex.rs index 87463402c..62a851a33 100644 --- a/crux-mir/test/concurrency/mutex/thread_retval_mutex.rs +++ b/crux-mir/test/concurrency/mutex/thread_retval_mutex.rs @@ -4,7 +4,7 @@ use std::thread; use std::sync::{Arc,Mutex}; #[cfg(not(with_main))] -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() { let data = Arc::new(Mutex::new(0 as u32)); let N = 3; diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread-C-DAC/pthread-finding-k-matches.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread-C-DAC/pthread-finding-k-matches.rs index 85ec50b50..cc0e62989 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread-C-DAC/pthread-finding-k-matches.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread-C-DAC/pthread-finding-k-matches.rs @@ -37,7 +37,7 @@ fn find_entries(tid:usize, count.crucible_TEMP_unlock(); } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn pthread_finding_k_matches() { let mut vals = vec![]; diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/bigshot.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/bigshot.rs index 91406e4cd..1189ae8c1 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/bigshot.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/bigshot.rs @@ -5,7 +5,7 @@ use std::thread; use std::sync::{Arc,Mutex}; #[cfg(not(with_main))] -#[cfg_attr(crux, crux::test)] +#[crux::test] fn bigshot_p() { let v = Arc::new(Mutex::new(None)); @@ -35,7 +35,7 @@ fn bigshot_p() { } #[cfg(not(with_main))] -#[cfg_attr(crux, crux::test)] +#[crux::test] fn bigshot_s() { let v = Arc::new(Mutex::new(None)); @@ -65,7 +65,7 @@ fn bigshot_s() { } #[cfg(not(with_main))] -#[cfg_attr(crux, crux::test)] +#[crux::test] fn bigshot_s2() { let v = Arc::new(Mutex::new(None)); diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/fib_bench.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/fib_bench.rs index e83212809..97f72a02b 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/fib_bench.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/fib_bench.rs @@ -35,7 +35,7 @@ fn calc_fib() -> i32 { j } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test_fail() { let data = Arc::new(Mutex::new((1 as i32, 1 as i32))); @@ -59,7 +59,7 @@ fn crux_test_fail() { data.crucible_TEMP_unlock(); } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() { let data = Arc::new(Mutex::new((1 as i32, 1 as i32))); diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/indexer.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/indexer.rs index 22a24e724..299dd72d2 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/indexer.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/indexer.rs @@ -24,7 +24,7 @@ fn cas(tab:&Arc>>, h:usize, val:i32, new_val:i32) -> i32 } #[cfg(not(with_main))] -#[cfg_attr(crux, crux::test)] +#[crux::test] fn indexer() { let mut tab = vec![]; let mut ts = vec![]; diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/queue.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/queue.rs index b41bac011..93f575548 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/queue.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/queue.rs @@ -125,7 +125,7 @@ fn t2_bad(m : Arc>) { } } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test_fails() { let mut q = Queue::new(); q.enqueue_flag = true; @@ -149,7 +149,7 @@ fn crux_test_fails() { h2.join(); } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test_succeeds() { let mut q = Queue::new(); q.enqueue_flag = true; diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/reorder.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/reorder.rs index 99aa4b51e..87aa7f596 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/reorder.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/reorder.rs @@ -39,12 +39,12 @@ fn run_test(numSet: usize, numCheck: usize) { } } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn reorder2() { run_test(2,2); } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn reorder5() { run_test(4,1); } diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/singleton.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/singleton.rs index c321111f0..7158ba43b 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/singleton.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/singleton.rs @@ -46,7 +46,7 @@ fn thread0(v: Arc>>) { } } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn singleton() { let v = Arc::new(Mutex::new(None)); let vv = v.clone(); diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/stack.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/stack.rs index 75e7b9b2a..94a26e9f5 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/stack.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/stack.rs @@ -85,7 +85,7 @@ fn t2(stack: Arc>, with_flag: bool) } } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn stack_1() { let stk = Arc::new(Mutex::new(Stack::new())); @@ -100,7 +100,7 @@ fn stack_1() h2.join(); } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn stack_2() { let stk = Arc::new(Mutex::new(Stack::new())); diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/stateful.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/stateful.rs index 2578b5acb..f9c65ef71 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/stateful.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/stateful.rs @@ -45,14 +45,14 @@ fn run_threads() -> (i32, i32) return r } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn stateful01_1() { let (d1, d2) = run_threads(); crucible_assert!( !(d1 == 16 && d2 == 5) ); } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn stateful01_2() { let (d1, d2) = run_threads(); diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/triangular.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/triangular.rs index 8687d2802..570fc3890 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/triangular.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/triangular.rs @@ -31,7 +31,7 @@ fn t2(i:Arc, j:Arc, m:Arc>) } } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn triangular_1() { let i = Arc::new(AtomicI32::new(3)); @@ -58,7 +58,7 @@ fn triangular_1() crucible_assert!(! (condI || condJ) ); } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn triangular_2() { let i = Arc::new(AtomicI32::new(3)); diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/triangular_static.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/triangular_static.rs index dfc6b0536..8498743ac 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/triangular_static.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/triangular_static.rs @@ -34,7 +34,7 @@ fn t2(m:Arc>) } } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn triangular_1() { let m = Arc::new(Mutex::new(())); @@ -54,7 +54,7 @@ fn triangular_1() crucible_assert!(! (condI || condJ) ); } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn triangular_2() { let m = Arc::new(Mutex::new(())); diff --git a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/twostage.rs b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/twostage.rs index a0627ce4b..86b27e420 100644 --- a/crux-mir/test/concurrency/sv-benchmarks/c/pthread/twostage.rs +++ b/crux-mir/test/concurrency/sv-benchmarks/c/pthread/twostage.rs @@ -59,7 +59,7 @@ fn twostage_test(t_threads: usize, r_threads: usize) { } } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn twostage_3() { twostage_test(2,1); } diff --git a/crux-mir/test/coverage/coverage.rs b/crux-mir/test/coverage/coverage.rs index c188ce2db..cf915e30b 100644 --- a/crux-mir/test/coverage/coverage.rs +++ b/crux-mir/test/coverage/coverage.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> i32 { let x = u8::symbolic("x"); crucible_assume!(x < 2); diff --git a/crux-mir/test/coverage/coverage_cond.rs b/crux-mir/test/coverage/coverage_cond.rs index 8b866db05..ae13854d9 100644 --- a/crux-mir/test/coverage/coverage_cond.rs +++ b/crux-mir/test/coverage/coverage_cond.rs @@ -10,7 +10,7 @@ fn g(cond: bool) -> u8 { if !cond { 10 } else { 20 } } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> u8 { f(true) + g(true) } diff --git a/crux-mir/test/coverage/coverage_dual.rs b/crux-mir/test/coverage/coverage_dual.rs index b41add186..459f80731 100644 --- a/crux-mir/test/coverage/coverage_dual.rs +++ b/crux-mir/test/coverage/coverage_dual.rs @@ -6,12 +6,12 @@ fn f(c: bool, t: i32, e: i32) -> i32 { if c { t } else { e } } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test1() -> i32 { f(false, 1, 2) } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test2() -> i32 { f(true, 10, 20) } diff --git a/crux-mir/test/coverage/coverage_loop.rs b/crux-mir/test/coverage/coverage_loop.rs index b7e9114b1..bfb27394d 100644 --- a/crux-mir/test/coverage/coverage_loop.rs +++ b/crux-mir/test/coverage/coverage_loop.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> i32 { let mut y = 1; for i in 0..10 { diff --git a/crux-mir/test/coverage/coverage_macro.rs b/crux-mir/test/coverage/coverage_macro.rs index ea30eef3e..b2176dd93 100644 --- a/crux-mir/test/coverage/coverage_macro.rs +++ b/crux-mir/test/coverage/coverage_macro.rs @@ -19,7 +19,7 @@ fn f() -> Option { Some(1) } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> u8 { f().unwrap_or(0) } diff --git a/crux-mir/test/coverage/coverage_match.rs b/crux-mir/test/coverage/coverage_match.rs index c7e7e8fe3..64f450d28 100644 --- a/crux-mir/test/coverage/coverage_match.rs +++ b/crux-mir/test/coverage/coverage_match.rs @@ -7,7 +7,7 @@ enum E { C = 30, } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> i32 { let x = u8::symbolic("x"); crucible_assume!(x != 1); diff --git a/crux-mir/test/coverage/coverage_mono.rs b/crux-mir/test/coverage/coverage_mono.rs index 78563eb1d..32c838dcf 100644 --- a/crux-mir/test/coverage/coverage_mono.rs +++ b/crux-mir/test/coverage/coverage_mono.rs @@ -6,7 +6,7 @@ fn f(c: bool, t: T, e: T) -> T { if c { t } else { e } } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> i32 { f::(false, 1_u8, 2_u8) as i32 + f::(true, 10, 20) } diff --git a/crux-mir/test/coverage/coverage_shortcircuit.rs b/crux-mir/test/coverage/coverage_shortcircuit.rs index 2b9330f15..0c68ea251 100644 --- a/crux-mir/test/coverage/coverage_shortcircuit.rs +++ b/crux-mir/test/coverage/coverage_shortcircuit.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> i32 { let x = u8::symbolic("x"); crucible_assume!(x < 1); diff --git a/crux-mir/test/coverage/coverage_try.rs b/crux-mir/test/coverage/coverage_try.rs index 68a85ea52..c92f01511 100644 --- a/crux-mir/test/coverage/coverage_try.rs +++ b/crux-mir/test/coverage/coverage_try.rs @@ -11,7 +11,7 @@ fn f() -> Option { x.checked_add(y) } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> u8 { f().unwrap_or(0) } diff --git a/crux-mir/test/symb_eval/alloc/out_of_bounds.rs b/crux-mir/test/symb_eval/alloc/out_of_bounds.rs index 1c82780ae..f7450b2f3 100644 --- a/crux-mir/test/symb_eval/alloc/out_of_bounds.rs +++ b/crux-mir/test/symb_eval/alloc/out_of_bounds.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::*; use crucible::alloc::allocate; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> i32 { unsafe { let ptr = allocate::(10); diff --git a/crux-mir/test/symb_eval/alloc/uninit_read.rs b/crux-mir/test/symb_eval/alloc/uninit_read.rs index b024c2c82..e9ee1e68d 100644 --- a/crux-mir/test/symb_eval/alloc/uninit_read.rs +++ b/crux-mir/test/symb_eval/alloc/uninit_read.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::*; use crucible::alloc::allocate; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> i32 { unsafe { let ptr = allocate::(10); diff --git a/crux-mir/test/symb_eval/alloc/valid_read.rs b/crux-mir/test/symb_eval/alloc/valid_read.rs index 8f2def682..831eeb7ee 100644 --- a/crux-mir/test/symb_eval/alloc/valid_read.rs +++ b/crux-mir/test/symb_eval/alloc/valid_read.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::*; use crucible::alloc::allocate; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> i32 { unsafe { let ptr = allocate::(10); diff --git a/crux-mir/test/symb_eval/alloc/zero_length.rs b/crux-mir/test/symb_eval/alloc/zero_length.rs index d9b8ec20c..fac62ff14 100644 --- a/crux-mir/test/symb_eval/alloc/zero_length.rs +++ b/crux-mir/test/symb_eval/alloc/zero_length.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::*; use crucible::alloc::allocate; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() { unsafe { // Make sure the CustomOp succeeds in creating a pointer to the first element of an empty diff --git a/crux-mir/test/symb_eval/any/conditional.rs b/crux-mir/test/symb_eval/any/conditional.rs index 8663dfbf6..951c93603 100644 --- a/crux-mir/test/symb_eval/any/conditional.rs +++ b/crux-mir/test/symb_eval/any/conditional.rs @@ -7,7 +7,7 @@ extern crate crucible; use crucible::*; use crucible::any::Any; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() { let mut x = bool::symbolic("x"); if x { diff --git a/crux-mir/test/symb_eval/any/downcast.rs b/crux-mir/test/symb_eval/any/downcast.rs index a467d2809..4ac7e64c8 100644 --- a/crux-mir/test/symb_eval/any/downcast.rs +++ b/crux-mir/test/symb_eval/any/downcast.rs @@ -3,7 +3,7 @@ extern crate crucible; use crucible::any::Any; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> i32 { let x: i32 = 1; let a = Any::new(x); diff --git a/crux-mir/test/symb_eval/any/downcast_fail.rs b/crux-mir/test/symb_eval/any/downcast_fail.rs index cfc5705cc..70c2474cc 100644 --- a/crux-mir/test/symb_eval/any/downcast_fail.rs +++ b/crux-mir/test/symb_eval/any/downcast_fail.rs @@ -3,7 +3,7 @@ extern crate crucible; use crucible::any::Any; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> i32 { let x: () = (); let a = Any::new(x); diff --git a/crux-mir/test/symb_eval/array/basic.rs b/crux-mir/test/symb_eval/array/basic.rs index 37ad426a4..2deb43247 100644 --- a/crux-mir/test/symb_eval/array/basic.rs +++ b/crux-mir/test/symb_eval/array/basic.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::array::Array; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> i32 { let arr = Array::::zeroed(); let arr = arr.update(0, 0); diff --git a/crux-mir/test/symb_eval/array/mux_slice.rs b/crux-mir/test/symb_eval/array/mux_slice.rs index b9fe40042..940cce634 100644 --- a/crux-mir/test/symb_eval/array/mux_slice.rs +++ b/crux-mir/test/symb_eval/array/mux_slice.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::array::Array; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> i32 { let mut arr1 = Array::::zeroed(); let mut arr2 = Array::::zeroed(); diff --git a/crux-mir/test/symb_eval/array/slice.rs b/crux-mir/test/symb_eval/array/slice.rs index be1e9d260..728665c2a 100644 --- a/crux-mir/test/symb_eval/array/slice.rs +++ b/crux-mir/test/symb_eval/array/slice.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::array::Array; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> i32 { let mut arr = Array::::zeroed(); for i in 0..10 { diff --git a/crux-mir/test/symb_eval/array/slice_mut.rs b/crux-mir/test/symb_eval/array/slice_mut.rs index 35d9b0363..936d2716a 100644 --- a/crux-mir/test/symb_eval/array/slice_mut.rs +++ b/crux-mir/test/symb_eval/array/slice_mut.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::array::Array; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> i32 { let mut arr = Array::::zeroed(); let xs = arr.as_mut_slice(0, 10); diff --git a/crux-mir/test/symb_eval/bitvector/arith.rs b/crux-mir/test/symb_eval/bitvector/arith.rs index 06ddbc14b..3fe81d8a8 100644 --- a/crux-mir/test/symb_eval/bitvector/arith.rs +++ b/crux-mir/test/symb_eval/bitvector/arith.rs @@ -30,7 +30,7 @@ fn test_shift_op(f: impl FnOnce(u64, usize) -> u64, f_bv: impl FnOnce(Bv256, usi crucible_assert!(u64::from(x) == f(a, b)); } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() { test_binop(u64::wrapping_add, Add::add); test_binop(u64::wrapping_sub, Sub::sub); diff --git a/crux-mir/test/symb_eval/bitvector/cmp.rs b/crux-mir/test/symb_eval/bitvector/cmp.rs index 75626bfc2..b57a0f99e 100644 --- a/crux-mir/test/symb_eval/bitvector/cmp.rs +++ b/crux-mir/test/symb_eval/bitvector/cmp.rs @@ -3,7 +3,7 @@ use crucible::bitvector::Bv256; use crucible::crucible_assert; use crucible::Symbolic; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() { { let (a, b) = <(u64, u64)>::symbolic("ab"); diff --git a/crux-mir/test/symb_eval/bitvector/from_to.rs b/crux-mir/test/symb_eval/bitvector/from_to.rs index 7227ab363..1aa6fcf97 100644 --- a/crux-mir/test/symb_eval/bitvector/from_to.rs +++ b/crux-mir/test/symb_eval/bitvector/from_to.rs @@ -8,7 +8,7 @@ fn test_one(i: u64) { crucible_assert!(i == j); } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() { test_one(0); test_one(1); diff --git a/crux-mir/test/symb_eval/bitvector/leading_zeros.rs b/crux-mir/test/symb_eval/bitvector/leading_zeros.rs index e46c7fde4..4d99120cf 100644 --- a/crux-mir/test/symb_eval/bitvector/leading_zeros.rs +++ b/crux-mir/test/symb_eval/bitvector/leading_zeros.rs @@ -5,7 +5,7 @@ use crucible::Symbolic; use std::ops::{Not, Neg, Add, Sub, Mul, Div, Rem, BitAnd, BitOr, BitXor, Shl, Shr}; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() { { let a_64 = u64::symbolic("a"); diff --git a/crux-mir/test/symb_eval/bitvector/literals.rs b/crux-mir/test/symb_eval/bitvector/literals.rs index b9b80e9cc..04bbac9e6 100644 --- a/crux-mir/test/symb_eval/bitvector/literals.rs +++ b/crux-mir/test/symb_eval/bitvector/literals.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::bitvector::Bv256; use crucible::crucible_assert; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() { crucible_assert!(u64::from(Bv256::ZERO) == 0); crucible_assert!(u64::from(Bv256::ONE) == 1); diff --git a/crux-mir/test/symb_eval/bitvector/overflowing_sub.rs b/crux-mir/test/symb_eval/bitvector/overflowing_sub.rs index e5e28acee..d9c4902a7 100644 --- a/crux-mir/test/symb_eval/bitvector/overflowing_sub.rs +++ b/crux-mir/test/symb_eval/bitvector/overflowing_sub.rs @@ -3,7 +3,7 @@ use crucible::bitvector::Bv256; use crucible::crucible_assert; use crucible::Symbolic; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() { let a = u64::symbolic("a"); let b = u64::symbolic("b"); diff --git a/crux-mir/test/symb_eval/bitvector/symbolic.rs b/crux-mir/test/symb_eval/bitvector/symbolic.rs index 76362dd71..445b49be7 100644 --- a/crux-mir/test/symb_eval/bitvector/symbolic.rs +++ b/crux-mir/test/symb_eval/bitvector/symbolic.rs @@ -3,7 +3,7 @@ use crucible::bitvector::Bv256; use crucible::crucible_assert; use crucible::Symbolic; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() { { let a = Bv256::symbolic("a"); diff --git a/crux-mir/test/symb_eval/byteorder/read.rs b/crux-mir/test/symb_eval/byteorder/read.rs index c39a82e08..a449b5970 100644 --- a/crux-mir/test/symb_eval/byteorder/read.rs +++ b/crux-mir/test/symb_eval/byteorder/read.rs @@ -23,4 +23,4 @@ pub fn f(x: u8) -> u8 { pub static ARG: u8 = 1; #[cfg(with_main)] pub fn main() { println!("{:?}", f(ARG)); } -#[cfg(not(with_main))] #[cfg_attr(crux, crux::test)] fn crux_test() -> u8 { f(ARG) } +#[cfg(not(with_main))] #[crux::test] fn crux_test() -> u8 { f(ARG) } diff --git a/crux-mir/test/symb_eval/byteorder/write.rs b/crux-mir/test/symb_eval/byteorder/write.rs index 6a4cc4e4d..16ea457e3 100644 --- a/crux-mir/test/symb_eval/byteorder/write.rs +++ b/crux-mir/test/symb_eval/byteorder/write.rs @@ -24,4 +24,4 @@ pub fn f(x: u8) -> u8 { pub static ARG: u8 = 1; #[cfg(with_main)] pub fn main() { println!("{:?}", f(ARG)); } -#[cfg(not(with_main))] #[cfg_attr(crux, crux::test)] fn crux_test() -> u8 { f(ARG) } +#[cfg(not(with_main))] #[crux::test] fn crux_test() -> u8 { f(ARG) } diff --git a/crux-mir/test/symb_eval/bytes/extend_bytes.rs b/crux-mir/test/symb_eval/bytes/extend_bytes.rs index 8e33bf7d4..49af6cb54 100644 --- a/crux-mir/test/symb_eval/bytes/extend_bytes.rs +++ b/crux-mir/test/symb_eval/bytes/extend_bytes.rs @@ -3,7 +3,7 @@ extern crate bytes; use bytes::{Bytes, BytesMut, Buf, BufMut}; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let mut bm = BytesMut::with_capacity(10); let b = Bytes::from(b"12345" as &[_]); diff --git a/crux-mir/test/symb_eval/bytes/new.rs b/crux-mir/test/symb_eval/bytes/new.rs index e440ac8f8..05785ba0c 100644 --- a/crux-mir/test/symb_eval/bytes/new.rs +++ b/crux-mir/test/symb_eval/bytes/new.rs @@ -3,7 +3,7 @@ extern crate bytes; use bytes::{Bytes, BytesMut, Buf, BufMut}; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let mut b = BytesMut::new(); assert!(b.len() == 0); diff --git a/crux-mir/test/symb_eval/bytes/put.rs b/crux-mir/test/symb_eval/bytes/put.rs index 05cfc6ec4..0f124c828 100644 --- a/crux-mir/test/symb_eval/bytes/put.rs +++ b/crux-mir/test/symb_eval/bytes/put.rs @@ -3,7 +3,7 @@ extern crate bytes; use bytes::{Bytes, BytesMut, Buf, BufMut}; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let mut b = BytesMut::with_capacity(10); b.put_u8(1); diff --git a/crux-mir/test/symb_eval/bytes/put_overflow.rs b/crux-mir/test/symb_eval/bytes/put_overflow.rs index b0946e2b6..9e188703b 100644 --- a/crux-mir/test/symb_eval/bytes/put_overflow.rs +++ b/crux-mir/test/symb_eval/bytes/put_overflow.rs @@ -3,7 +3,7 @@ extern crate bytes; use bytes::{Bytes, BytesMut, Buf, BufMut}; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let mut b = BytesMut::with_capacity(0); b.put_slice(b"1234567-1234567-1234567-1234567-1234567-1234567-"); diff --git a/crux-mir/test/symb_eval/bytes/split_off.rs b/crux-mir/test/symb_eval/bytes/split_off.rs index 35096f039..1bd9853b3 100644 --- a/crux-mir/test/symb_eval/bytes/split_off.rs +++ b/crux-mir/test/symb_eval/bytes/split_off.rs @@ -3,7 +3,7 @@ extern crate bytes; use bytes::{Bytes, BytesMut, Buf, BufMut}; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let mut b = BytesMut::with_capacity(10); b.put_u32_be(0x01020304); diff --git a/crux-mir/test/symb_eval/bytes/split_to.rs b/crux-mir/test/symb_eval/bytes/split_to.rs index 3ee83891d..859002f0a 100644 --- a/crux-mir/test/symb_eval/bytes/split_to.rs +++ b/crux-mir/test/symb_eval/bytes/split_to.rs @@ -3,7 +3,7 @@ extern crate bytes; use bytes::{Bytes, BytesMut, Buf, BufMut}; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let mut b = BytesMut::with_capacity(10); b.put_u32_be(0x01020304); diff --git a/crux-mir/test/symb_eval/bytes/sym_len.rs b/crux-mir/test/symb_eval/bytes/sym_len.rs index e2e100305..1a7601635 100644 --- a/crux-mir/test/symb_eval/bytes/sym_len.rs +++ b/crux-mir/test/symb_eval/bytes/sym_len.rs @@ -5,7 +5,7 @@ extern crate bytes; use bytes::{Bytes, BytesMut, Buf, BufMut}; use core::ops::Deref; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let mut b = BytesMut::with_capacity(10); b.put_u8(1); diff --git a/crux-mir/test/symb_eval/concretize/array.rs b/crux-mir/test/symb_eval/concretize/array.rs index 61b0fc4a1..498188ea3 100644 --- a/crux-mir/test/symb_eval/concretize/array.rs +++ b/crux-mir/test/symb_eval/concretize/array.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::*; use crucible::array::Array; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> (i32, i32, i32) { let arr = Array::symbolic("arr"); let s = arr.as_slice(0, 3); diff --git a/crux-mir/test/symb_eval/concretize/assert.rs b/crux-mir/test/symb_eval/concretize/assert.rs index 3523abdf9..1b696a984 100644 --- a/crux-mir/test/symb_eval/concretize/assert.rs +++ b/crux-mir/test/symb_eval/concretize/assert.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> u8 { let mut x = u8::symbolic("x"); let mut y = u8::symbolic("y"); diff --git a/crux-mir/test/symb_eval/concretize/assert_ok.rs b/crux-mir/test/symb_eval/concretize/assert_ok.rs index e3298a572..c32c7bc4c 100644 --- a/crux-mir/test/symb_eval/concretize/assert_ok.rs +++ b/crux-mir/test/symb_eval/concretize/assert_ok.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> u8 { let mut x = u8::symbolic("x"); let mut y = u8::symbolic("y"); diff --git a/crux-mir/test/symb_eval/concretize/conc.rs b/crux-mir/test/symb_eval/concretize/conc.rs index 7ecc5423b..a599ad5ca 100644 --- a/crux-mir/test/symb_eval/concretize/conc.rs +++ b/crux-mir/test/symb_eval/concretize/conc.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> (u8, u8) { let mut x = u8::symbolic("x"); let mut y = u8::symbolic("y"); diff --git a/crux-mir/test/symb_eval/concretize/no_conc.rs b/crux-mir/test/symb_eval/concretize/no_conc.rs index 80a911533..2366a2c4b 100644 --- a/crux-mir/test/symb_eval/concretize/no_conc.rs +++ b/crux-mir/test/symb_eval/concretize/no_conc.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> (u8, u8) { let mut x = u8::symbolic("x"); let mut y = u8::symbolic("y"); diff --git a/crux-mir/test/symb_eval/crux/early_fail.rs b/crux-mir/test/symb_eval/crux/early_fail.rs index 9efd17cee..fb278b7c1 100644 --- a/crux-mir/test/symb_eval/crux/early_fail.rs +++ b/crux-mir/test/symb_eval/crux/early_fail.rs @@ -6,12 +6,12 @@ use crucible::*; // because `fail1` panics (terminating execution) on all branches and `fail2` never runs. The // desired behavior is to run both `fail1` and `fail2` and report their errors independently. -#[cfg_attr(crux, crux::test)] +#[crux::test] fn fail1() { panic!("bad 1"); } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn fail2() { let x = u8::symbolic("x"); crucible_assert!(x == 0); diff --git a/crux-mir/test/symb_eval/crux/fail_return.rs b/crux-mir/test/symb_eval/crux/fail_return.rs index e8c13060b..ed24b42ca 100644 --- a/crux-mir/test/symb_eval/crux/fail_return.rs +++ b/crux-mir/test/symb_eval/crux/fail_return.rs @@ -2,14 +2,14 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn fail1() -> u8 { let x = u8::symbolic("x"); crucible_assert!(x + 1 > x); x } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn fail2() -> u8 { let x = u8::symbolic("x"); crucible_assert!(x + 1 > x); diff --git a/crux-mir/test/symb_eval/crux/mixed_fail.rs b/crux-mir/test/symb_eval/crux/mixed_fail.rs index 2259c4ff6..9b7bab3b2 100644 --- a/crux-mir/test/symb_eval/crux/mixed_fail.rs +++ b/crux-mir/test/symb_eval/crux/mixed_fail.rs @@ -2,25 +2,25 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn fail1() { let x = u8::symbolic("x"); crucible_assert!(x + 1 > x); } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn fail2() { let x = u8::symbolic("x"); crucible_assert!(x + 2 > x); } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn pass1() { let x = u8::symbolic("x"); crucible_assert!(x > 10 || x <= 10); } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn pass2() { let x = u8::symbolic("x"); crucible_assert!(x > 20 || x <= 20); diff --git a/crux-mir/test/symb_eval/crux/multi.rs b/crux-mir/test/symb_eval/crux/multi.rs index 63204952d..b90883765 100644 --- a/crux-mir/test/symb_eval/crux/multi.rs +++ b/crux-mir/test/symb_eval/crux/multi.rs @@ -2,13 +2,13 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn fail1() { let x = u8::symbolic("x"); crucible_assert!(x + 1 > x); } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn fail2() { let x = u8::symbolic("x"); if x == 0 { @@ -20,7 +20,7 @@ fn assert_zero(x: u8) { crucible_assert!(x == 0); } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn fail3() { let x = u8::symbolic("x"); assert_zero(x); diff --git a/crux-mir/test/symb_eval/crypto/bytes.rs b/crux-mir/test/symb_eval/crypto/bytes.rs index f0ee5b8ea..7e91e1053 100644 --- a/crux-mir/test/symb_eval/crypto/bytes.rs +++ b/crux-mir/test/symb_eval/crypto/bytes.rs @@ -70,7 +70,7 @@ pub fn to_bytes(x :&[u64;5]) -> [u8; 32] { s } -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let a0 = crucible_u64("a0"); let a1 = crucible_u64("a1"); diff --git a/crux-mir/test/symb_eval/crypto/bytes2.rs b/crux-mir/test/symb_eval/crypto/bytes2.rs index 4fcc9c1f2..f3ec31f32 100644 --- a/crux-mir/test/symb_eval/crypto/bytes2.rs +++ b/crux-mir/test/symb_eval/crypto/bytes2.rs @@ -69,7 +69,7 @@ pub fn to_bytes(x :&[u64;5]) -> [u8; 32] { s } -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let mut s = [0u8; 32]; diff --git a/crux-mir/test/symb_eval/crypto/double.rs b/crux-mir/test/symb_eval/crypto/double.rs index 148601a8e..beef414d3 100644 --- a/crux-mir/test/symb_eval/crypto/double.rs +++ b/crux-mir/test/symb_eval/crypto/double.rs @@ -14,7 +14,7 @@ fn double_imp(x : u32) -> u32 { } -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f () { let a0 = crucible_u32("a0"); crucible_assert!(double_ref(a0) == double_imp(a0)); diff --git a/crux-mir/test/symb_eval/crypto/ffs.rs b/crux-mir/test/symb_eval/crypto/ffs.rs index c450510c6..fd0b4804f 100644 --- a/crux-mir/test/symb_eval/crypto/ffs.rs +++ b/crux-mir/test/symb_eval/crypto/ffs.rs @@ -28,7 +28,7 @@ fn ffs_imp(j : u32) -> u32 { } -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f () { let a0 = crucible_u32("a0"); crucible_assert!(ffs_ref(a0) == ffs_imp(a0)); diff --git a/crux-mir/test/symb_eval/enum/mux.rs b/crux-mir/test/symb_eval/enum/mux.rs index 2cdc80cb4..89a3e6426 100644 --- a/crux-mir/test/symb_eval/enum/mux.rs +++ b/crux-mir/test/symb_eval/enum/mux.rs @@ -8,7 +8,7 @@ fn f(x: bool) -> Option { if x { Some(S { val: 1 }) } else { None } } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn test() { let x = crucible::crucible_u8("x") != 0; let y = f(x); diff --git a/crux-mir/test/symb_eval/fnptr/mux.rs b/crux-mir/test/symb_eval/fnptr/mux.rs index 4dd7044e8..6151da402 100644 --- a/crux-mir/test/symb_eval/fnptr/mux.rs +++ b/crux-mir/test/symb_eval/fnptr/mux.rs @@ -9,7 +9,7 @@ fn g(x: i32) -> i32 { x - 1 } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> i32 { let b = bool::symbolic("cond"); let p1: fn(i32) -> i32 = if b { f } else { g }; diff --git a/crux-mir/test/symb_eval/io/vec_cursor_read.rs b/crux-mir/test/symb_eval/io/vec_cursor_read.rs index 33e821129..11d14b1c6 100644 --- a/crux-mir/test/symb_eval/io/vec_cursor_read.rs +++ b/crux-mir/test/symb_eval/io/vec_cursor_read.rs @@ -4,7 +4,7 @@ extern crate std; use std::io::{Read, Write, Cursor}; use std::vec::Vec; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let mut buf = Vec::new(); buf.write(&[1, 2, 3]); diff --git a/crux-mir/test/symb_eval/io/vec_write.rs b/crux-mir/test/symb_eval/io/vec_write.rs index 2adb15b47..4e9a2fbf5 100644 --- a/crux-mir/test/symb_eval/io/vec_write.rs +++ b/crux-mir/test/symb_eval/io/vec_write.rs @@ -4,7 +4,7 @@ extern crate std; use std::io::Write; use std::vec::Vec; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let mut buf = Vec::new(); buf.write(&[1, 2, 3]); diff --git a/crux-mir/test/symb_eval/mux/array.rs b/crux-mir/test/symb_eval/mux/array.rs index 0bca292b9..6d07873e0 100644 --- a/crux-mir/test/symb_eval/mux/array.rs +++ b/crux-mir/test/symb_eval/mux/array.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::array::Array; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> i32 { let arr1 = [1]; let arr2 = [2, 2]; diff --git a/crux-mir/test/symb_eval/mux/array_mut.rs b/crux-mir/test/symb_eval/mux/array_mut.rs index fd4f63546..b6bca18cb 100644 --- a/crux-mir/test/symb_eval/mux/array_mut.rs +++ b/crux-mir/test/symb_eval/mux/array_mut.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::array::Array; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> i32 { let mut arr1 = [1]; let mut arr2 = [2, 2]; diff --git a/crux-mir/test/symb_eval/num/checked_add.rs b/crux-mir/test/symb_eval/num/checked_add.rs index d5980d345..b4af325f6 100644 --- a/crux-mir/test/symb_eval/num/checked_add.rs +++ b/crux-mir/test/symb_eval/num/checked_add.rs @@ -1,6 +1,6 @@ #![allow(arithmetic_overflow)] -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> u8 { let x = 200; 100 + x diff --git a/crux-mir/test/symb_eval/num/checked_div.rs b/crux-mir/test/symb_eval/num/checked_div.rs index 751b82ca1..012b12d16 100644 --- a/crux-mir/test/symb_eval/num/checked_div.rs +++ b/crux-mir/test/symb_eval/num/checked_div.rs @@ -8,7 +8,7 @@ fn div_unsigned(x: u8, y: u8) -> u8 { x / y } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> u8 { let a: i8 = div_signed(1, 1); let b: i8 = div_signed(-128, -1); // Should fail diff --git a/crux-mir/test/symb_eval/num/checked_mul.rs b/crux-mir/test/symb_eval/num/checked_mul.rs index 0c3e1b58a..9ff9e2462 100644 --- a/crux-mir/test/symb_eval/num/checked_mul.rs +++ b/crux-mir/test/symb_eval/num/checked_mul.rs @@ -1,6 +1,6 @@ #![allow(arithmetic_overflow)] -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> u8 { let x = 3; 100 * x diff --git a/crux-mir/test/symb_eval/num/checked_mul_signed.rs b/crux-mir/test/symb_eval/num/checked_mul_signed.rs index 409c0f9e0..fb5ad0275 100644 --- a/crux-mir/test/symb_eval/num/checked_mul_signed.rs +++ b/crux-mir/test/symb_eval/num/checked_mul_signed.rs @@ -1,6 +1,6 @@ #![allow(arithmetic_overflow)] -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> i8 { let x = -3; -100 * x diff --git a/crux-mir/test/symb_eval/overrides/bad_symb1.rs b/crux-mir/test/symb_eval/overrides/bad_symb1.rs index fdb4c9218..daf2a11cf 100644 --- a/crux-mir/test/symb_eval/overrides/bad_symb1.rs +++ b/crux-mir/test/symb_eval/overrides/bad_symb1.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> u8 { let s = if bool::symbolic("cond") { "a" } else { "b" }; let x = u8::symbolic(s); diff --git a/crux-mir/test/symb_eval/overrides/bad_symb2.rs b/crux-mir/test/symb_eval/overrides/bad_symb2.rs index 00251ef08..c125cd876 100644 --- a/crux-mir/test/symb_eval/overrides/bad_symb2.rs +++ b/crux-mir/test/symb_eval/overrides/bad_symb2.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> u8 { let x = u8::symbolic("\0:., /"); x diff --git a/crux-mir/test/symb_eval/overrides/override1.rs b/crux-mir/test/symb_eval/overrides/override1.rs index a635cdc14..db2bd5bf2 100644 --- a/crux-mir/test/symb_eval/overrides/override1.rs +++ b/crux-mir/test/symb_eval/overrides/override1.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() -> u8 { let x: u8 = 1; // This call should be replaced by the test override diff --git a/crux-mir/test/symb_eval/overrides/override2.rs b/crux-mir/test/symb_eval/overrides/override2.rs index 78bedd7d7..b4ee30e2a 100644 --- a/crux-mir/test/symb_eval/overrides/override2.rs +++ b/crux-mir/test/symb_eval/overrides/override2.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { // This call should be replaced by the test override let foo = crucible_i8("foo"); diff --git a/crux-mir/test/symb_eval/overrides/override3.rs b/crux-mir/test/symb_eval/overrides/override3.rs index 465e8cc42..19694e5e3 100644 --- a/crux-mir/test/symb_eval/overrides/override3.rs +++ b/crux-mir/test/symb_eval/overrides/override3.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { // This call should be replaced by the test override let foo = crucible_i8("x"); diff --git a/crux-mir/test/symb_eval/overrides/override4.rs b/crux-mir/test/symb_eval/overrides/override4.rs index b5443abfd..a992b261f 100644 --- a/crux-mir/test/symb_eval/overrides/override4.rs +++ b/crux-mir/test/symb_eval/overrides/override4.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { // This call should be replaced by the test override let foo = crucible_i8("x"); diff --git a/crux-mir/test/symb_eval/overrides/override5.rs b/crux-mir/test/symb_eval/overrides/override5.rs index 7e971c683..b96c5ddf9 100644 --- a/crux-mir/test/symb_eval/overrides/override5.rs +++ b/crux-mir/test/symb_eval/overrides/override5.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { // This call should be replaced by the test override let foo = crucible_u64("foo"); diff --git a/crux-mir/test/symb_eval/overrides/override_rust.rs b/crux-mir/test/symb_eval/overrides/override_rust.rs index a238736b3..8f22feab0 100644 --- a/crux-mir/test/symb_eval/overrides/override_rust.rs +++ b/crux-mir/test/symb_eval/overrides/override_rust.rs @@ -4,7 +4,7 @@ use crucible::*; fn f() -> i32 { 1 } fn g() -> i32 { 2 } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() { crucible_assert!(f() == 1); crucible::override_(f, g); diff --git a/crux-mir/test/symb_eval/refs/mux_init_imm.rs b/crux-mir/test/symb_eval/refs/mux_init_imm.rs index 9e087b733..9574a0d27 100644 --- a/crux-mir/test/symb_eval/refs/mux_init_imm.rs +++ b/crux-mir/test/symb_eval/refs/mux_init_imm.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> i32 { let mut result = 0; let one = 1; diff --git a/crux-mir/test/symb_eval/refs/mux_init_mut.rs b/crux-mir/test/symb_eval/refs/mux_init_mut.rs index d10e81404..c7cdf3ff1 100644 --- a/crux-mir/test/symb_eval/refs/mux_init_mut.rs +++ b/crux-mir/test/symb_eval/refs/mux_init_mut.rs @@ -1,7 +1,7 @@ extern crate crucible; use crucible::*; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> i32 { let mut result = 0; if bool::symbolic("cond") { diff --git a/crux-mir/test/symb_eval/scalar/test1.rs b/crux-mir/test/symb_eval/scalar/test1.rs index 42f59ab40..a12c49654 100644 --- a/crux-mir/test/symb_eval/scalar/test1.rs +++ b/crux-mir/test/symb_eval/scalar/test1.rs @@ -76,7 +76,7 @@ macro_rules! crucible_debug_integer { -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { // Int512 -> Scalar64 -> Int512 conversion is the identity function. { diff --git a/crux-mir/test/symb_eval/sym_bytes/construct.rs b/crux-mir/test/symb_eval/sym_bytes/construct.rs index eb5576aee..8429d93e4 100644 --- a/crux-mir/test/symb_eval/sym_bytes/construct.rs +++ b/crux-mir/test/symb_eval/sym_bytes/construct.rs @@ -2,7 +2,7 @@ extern crate crucible; use crucible::*; use crucible::sym_bytes::SymBytes; -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> u8 { let sym1 = SymBytes::zeroed(5); // Should succeed - sym1[0] is zero. In fact, this should get simplified away and not produce diff --git a/crux-mir/test/symb_eval/sym_bytes/deserialize.rs b/crux-mir/test/symb_eval/sym_bytes/deserialize.rs index 31f85fe27..52a0ae3e3 100644 --- a/crux-mir/test/symb_eval/sym_bytes/deserialize.rs +++ b/crux-mir/test/symb_eval/sym_bytes/deserialize.rs @@ -23,7 +23,7 @@ fn deserialize(b: &[u8]) -> (i16, i16) { } } -#[cfg_attr(crux, crux::test)] +#[crux::test] fn crux_test() -> i32 { let sym = SymBytes::symbolic("sym", 5); crucible_assume!(sym[0] <= 2); diff --git a/crux-mir/test/symb_eval/vec/clone.rs b/crux-mir/test/symb_eval/vec/clone.rs index 6b3b2ef29..5d5c842f5 100644 --- a/crux-mir/test/symb_eval/vec/clone.rs +++ b/crux-mir/test/symb_eval/vec/clone.rs @@ -1,5 +1,5 @@ -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let v = vec![1, 2, 3]; let w = v.clone(); diff --git a/crux-mir/test/symb_eval/vec/into_iter.rs b/crux-mir/test/symb_eval/vec/into_iter.rs index 24e72b852..fde79199b 100644 --- a/crux-mir/test/symb_eval/vec/into_iter.rs +++ b/crux-mir/test/symb_eval/vec/into_iter.rs @@ -1,4 +1,4 @@ -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let v = vec![1, 2, 3]; let mut it = v.into_iter(); diff --git a/crux-mir/test/symb_eval/vec/macro.rs b/crux-mir/test/symb_eval/vec/macro.rs index 396376828..20fe02537 100644 --- a/crux-mir/test/symb_eval/vec/macro.rs +++ b/crux-mir/test/symb_eval/vec/macro.rs @@ -1,5 +1,5 @@ -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let v = vec![1, 2, 3]; assert!(v.len() == 3); diff --git a/crux-mir/test/symb_eval/vec/sort_by_key.rs b/crux-mir/test/symb_eval/vec/sort_by_key.rs index a8b839862..cb794419f 100644 --- a/crux-mir/test/symb_eval/vec/sort_by_key.rs +++ b/crux-mir/test/symb_eval/vec/sort_by_key.rs @@ -1,7 +1,7 @@ #[macro_use] extern crate crucible; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let mut v = vec![1_i32, 3, 2]; v.sort_by_key(|&x| -x); diff --git a/crux-mir/test/symb_eval/vector/as_mut_slice.rs b/crux-mir/test/symb_eval/vector/as_mut_slice.rs index 01fc4e63b..a8dbe4467 100644 --- a/crux-mir/test/symb_eval/vector/as_mut_slice.rs +++ b/crux-mir/test/symb_eval/vector/as_mut_slice.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::vector::Vector; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let mut v = Vector::::new().push(12).push(34); crucible_assert!(v.len() == 2); diff --git a/crux-mir/test/symb_eval/vector/as_slice.rs b/crux-mir/test/symb_eval/vector/as_slice.rs index ec7c385c4..c25bc9422 100644 --- a/crux-mir/test/symb_eval/vector/as_slice.rs +++ b/crux-mir/test/symb_eval/vector/as_slice.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::vector::Vector; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let v = Vector::::new().push(12).push(34); crucible_assert!(v.len() == 2); diff --git a/crux-mir/test/symb_eval/vector/concat.rs b/crux-mir/test/symb_eval/vector/concat.rs index 1d7a6f8ed..562a5271f 100644 --- a/crux-mir/test/symb_eval/vector/concat.rs +++ b/crux-mir/test/symb_eval/vector/concat.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::vector::Vector; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let v1 = Vector::::new().push(1).push(2); let v2 = Vector::::new().push(3).push(4); diff --git a/crux-mir/test/symb_eval/vector/copy_from_slice.rs b/crux-mir/test/symb_eval/vector/copy_from_slice.rs index 945bc9763..315a9846f 100644 --- a/crux-mir/test/symb_eval/vector/copy_from_slice.rs +++ b/crux-mir/test/symb_eval/vector/copy_from_slice.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::vector::Vector; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let v = Vector::::copy_from_slice(&[1, 2, 3, 4]); crucible_assert!(v.len() == 4); diff --git a/crux-mir/test/symb_eval/vector/mut.rs b/crux-mir/test/symb_eval/vector/mut.rs index 4e3a92c56..c093bdbfe 100644 --- a/crux-mir/test/symb_eval/vector/mut.rs +++ b/crux-mir/test/symb_eval/vector/mut.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::vector::Vector; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let mut v = Vector::::new().push(12).push(34); { diff --git a/crux-mir/test/symb_eval/vector/new.rs b/crux-mir/test/symb_eval/vector/new.rs index 175c53be6..2e3418720 100644 --- a/crux-mir/test/symb_eval/vector/new.rs +++ b/crux-mir/test/symb_eval/vector/new.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::vector::Vector; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let v = Vector::::new(); crucible_assert!(v.len() == 0); diff --git a/crux-mir/test/symb_eval/vector/pop.rs b/crux-mir/test/symb_eval/vector/pop.rs index 65df4feea..7c631b47a 100644 --- a/crux-mir/test/symb_eval/vector/pop.rs +++ b/crux-mir/test/symb_eval/vector/pop.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::vector::Vector; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let v = Vector::::new().push(12).push(34); crucible_assert!(v.len() == 2); diff --git a/crux-mir/test/symb_eval/vector/push.rs b/crux-mir/test/symb_eval/vector/push.rs index 9bb3be117..f98c9edde 100644 --- a/crux-mir/test/symb_eval/vector/push.rs +++ b/crux-mir/test/symb_eval/vector/push.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::vector::Vector; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let v = Vector::::new(); crucible_assert!(v.len() == 0); diff --git a/crux-mir/test/symb_eval/vector/replicate.rs b/crux-mir/test/symb_eval/vector/replicate.rs index f45e53dbb..000fb9746 100644 --- a/crux-mir/test/symb_eval/vector/replicate.rs +++ b/crux-mir/test/symb_eval/vector/replicate.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::vector::Vector; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let v = Vector::::replicate(123, 4); crucible_assert!(v.len() == 4); diff --git a/crux-mir/test/symb_eval/vector/split_at.rs b/crux-mir/test/symb_eval/vector/split_at.rs index b1c7868a1..4c3834a66 100644 --- a/crux-mir/test/symb_eval/vector/split_at.rs +++ b/crux-mir/test/symb_eval/vector/split_at.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate crucible; use crucible::vector::Vector; -#[cfg_attr(crux, crux::test)] +#[crux::test] pub fn f() { let v = Vector::::copy_from_slice(&[1, 2, 3, 4]); let (v1, v2) = v.split_at(2);

(&self, mut pred: P) -> usize + where + P: FnMut(&T) -> bool, + { + let (front, back) = self.as_slices(); + + if let Some(true) = back.first().map(|v| pred(v)) { + back.partition_point(pred) + front.len() + } else { + front.partition_point(pred) + } + } +} + +impl VecDeque { + /// Modifies the deque in-place so that `len()` is equal to new_len, + /// either by removing excess elements from the back or by appending clones of `value` + /// to the back. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let mut buf = VecDeque::new(); + /// buf.push_back(5); + /// buf.push_back(10); + /// buf.push_back(15); + /// assert_eq!(buf, [5, 10, 15]); + /// + /// buf.resize(2, 0); + /// assert_eq!(buf, [5, 10]); + /// + /// buf.resize(5, 20); + /// assert_eq!(buf, [5, 10, 20, 20, 20]); + /// ``` + #[stable(feature = "deque_extras", since = "1.16.0")] + pub fn resize(&mut self, new_len: usize, value: T) { + if new_len > self.len() { + let extra = new_len - self.len(); + self.extend(repeat_n(value, extra)) + } else { + self.truncate(new_len); + } + } +} + +/// Returns the index in the underlying buffer for a given logical element index. +#[inline] +fn wrap_index(logical_index: usize, capacity: usize) -> usize { + debug_assert!( + (logical_index == 0 && capacity == 0) + || logical_index < capacity + || (logical_index - capacity) < capacity + ); + if logical_index >= capacity { logical_index - capacity } else { logical_index } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialEq for VecDeque { + fn eq(&self, other: &Self) -> bool { + if self.len != other.len() { + return false; + } + let (sa, sb) = self.as_slices(); + let (oa, ob) = other.as_slices(); + if sa.len() == oa.len() { + sa == oa && sb == ob + } else if sa.len() < oa.len() { + // Always divisible in three sections, for example: + // self: [a b c|d e f] + // other: [0 1 2 3|4 5] + // front = 3, mid = 1, + // [a b c] == [0 1 2] && [d] == [3] && [e f] == [4 5] + let front = sa.len(); + let mid = oa.len() - front; + + let (oa_front, oa_mid) = oa.split_at(front); + let (sb_mid, sb_back) = sb.split_at(mid); + debug_assert_eq!(sa.len(), oa_front.len()); + debug_assert_eq!(sb_mid.len(), oa_mid.len()); + debug_assert_eq!(sb_back.len(), ob.len()); + sa == oa_front && sb_mid == oa_mid && sb_back == ob + } else { + let front = oa.len(); + let mid = sa.len() - front; + + let (sa_front, sa_mid) = sa.split_at(front); + let (ob_mid, ob_back) = ob.split_at(mid); + debug_assert_eq!(sa_front.len(), oa.len()); + debug_assert_eq!(sa_mid.len(), ob_mid.len()); + debug_assert_eq!(sb.len(), ob_back.len()); + sa_front == oa && sa_mid == ob_mid && sb == ob_back + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Eq for VecDeque {} + +__impl_slice_eq1! { [] VecDeque, Vec, } +__impl_slice_eq1! { [] VecDeque, &[U], } +__impl_slice_eq1! { [] VecDeque, &mut [U], } +__impl_slice_eq1! { [const N: usize] VecDeque, [U; N], } +__impl_slice_eq1! { [const N: usize] VecDeque, &[U; N], } +__impl_slice_eq1! { [const N: usize] VecDeque, &mut [U; N], } + +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialOrd for VecDeque { + fn partial_cmp(&self, other: &Self) -> Option { + self.iter().partial_cmp(other.iter()) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Ord for VecDeque { + #[inline] + fn cmp(&self, other: &Self) -> Ordering { + self.iter().cmp(other.iter()) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Hash for VecDeque { + fn hash(&self, state: &mut H) { + state.write_length_prefix(self.len); + // It's not possible to use Hash::hash_slice on slices + // returned by as_slices method as their length can vary + // in otherwise identical deques. + // + // Hasher only guarantees equivalence for the exact same + // set of calls to its methods. + self.iter().for_each(|elem| elem.hash(state)); + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Index for VecDeque { + type Output = T; + + #[inline] + fn index(&self, index: usize) -> &T { + self.get(index).expect("Out of bounds access") + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl IndexMut for VecDeque { + #[inline] + fn index_mut(&mut self, index: usize) -> &mut T { + self.get_mut(index).expect("Out of bounds access") + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl FromIterator for VecDeque { + fn from_iter>(iter: I) -> VecDeque { + SpecFromIter::spec_from_iter(iter.into_iter()) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl IntoIterator for VecDeque { + type Item = T; + type IntoIter = IntoIter; + + /// Consumes the deque into a front-to-back iterator yielding elements by + /// value. + fn into_iter(self) -> IntoIter { + IntoIter::new(self) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T, A: Allocator> IntoIterator for &'a VecDeque { + type Item = &'a T; + type IntoIter = Iter<'a, T>; + + fn into_iter(self) -> Iter<'a, T> { + self.iter() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T, A: Allocator> IntoIterator for &'a mut VecDeque { + type Item = &'a mut T; + type IntoIter = IterMut<'a, T>; + + fn into_iter(self) -> IterMut<'a, T> { + self.iter_mut() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Extend for VecDeque { + fn extend>(&mut self, iter: I) { + >::spec_extend(self, iter.into_iter()); + } + + #[inline] + fn extend_one(&mut self, elem: T) { + self.push_back(elem); + } + + #[inline] + fn extend_reserve(&mut self, additional: usize) { + self.reserve(additional); + } +} + +#[stable(feature = "extend_ref", since = "1.2.0")] +impl<'a, T: 'a + Copy, A: Allocator> Extend<&'a T> for VecDeque { + fn extend>(&mut self, iter: I) { + self.spec_extend(iter.into_iter()); + } + + #[inline] + fn extend_one(&mut self, &elem: &T) { + self.push_back(elem); + } + + #[inline] + fn extend_reserve(&mut self, additional: usize) { + self.reserve(additional); + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Debug for VecDeque { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.iter()).finish() + } +} + +#[stable(feature = "vecdeque_vec_conversions", since = "1.10.0")] +impl From> for VecDeque { + /// Turn a [`Vec`] into a [`VecDeque`]. + /// + /// [`Vec`]: crate::vec::Vec + /// [`VecDeque`]: crate::collections::VecDeque + /// + /// This conversion is guaranteed to run in *O*(1) time + /// and to not re-allocate the `Vec`'s buffer or allocate + /// any additional memory. + #[inline] + fn from(other: Vec) -> Self { + let (ptr, len, cap, alloc) = other.into_raw_parts_with_alloc(); + Self { head: 0, len, buf: unsafe { RawVec::from_raw_parts_in(ptr, cap, alloc) } } + } +} + +#[stable(feature = "vecdeque_vec_conversions", since = "1.10.0")] +impl From> for Vec { + /// Turn a [`VecDeque`] into a [`Vec`]. + /// + /// [`Vec`]: crate::vec::Vec + /// [`VecDeque`]: crate::collections::VecDeque + /// + /// This never needs to re-allocate, but does need to do *O*(*n*) data movement if + /// the circular buffer doesn't happen to be at the beginning of the allocation. + /// + /// # Examples + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// // This one is *O*(1). + /// let deque: VecDeque<_> = (1..5).collect(); + /// let ptr = deque.as_slices().0.as_ptr(); + /// let vec = Vec::from(deque); + /// assert_eq!(vec, [1, 2, 3, 4]); + /// assert_eq!(vec.as_ptr(), ptr); + /// + /// // This one needs data rearranging. + /// let mut deque: VecDeque<_> = (1..5).collect(); + /// deque.push_front(9); + /// deque.push_front(8); + /// let ptr = deque.as_slices().1.as_ptr(); + /// let vec = Vec::from(deque); + /// assert_eq!(vec, [8, 9, 1, 2, 3, 4]); + /// assert_eq!(vec.as_ptr(), ptr); + /// ``` + fn from(mut other: VecDeque) -> Self { + other.make_contiguous(); + + unsafe { + let other = ManuallyDrop::new(other); + let buf = other.buf.ptr(); + let len = other.len(); + let cap = other.capacity(); + let alloc = ptr::read(other.allocator()); + + if other.head != 0 { + ptr::copy(buf.add(other.head), buf, len); + } + Vec::from_raw_parts_in(buf, len, cap, alloc) + } + } +} + +#[stable(feature = "std_collections_from_array", since = "1.56.0")] +impl From<[T; N]> for VecDeque { + /// Converts a `[T; N]` into a `VecDeque`. + /// + /// ``` + /// use std::collections::VecDeque; + /// + /// let deq1 = VecDeque::from([1, 2, 3, 4]); + /// let deq2: VecDeque<_> = [1, 2, 3, 4].into(); + /// assert_eq!(deq1, deq2); + /// ``` + fn from(arr: [T; N]) -> Self { + let mut deq = VecDeque::with_capacity(N); + let arr = ManuallyDrop::new(arr); + if !::IS_ZST { + // SAFETY: VecDeque::with_capacity ensures that there is enough capacity. + unsafe { + ptr::copy_nonoverlapping(arr.as_ptr(), deq.ptr(), N); + } + } + deq.head = 0; + deq.len = N; + deq + } +} diff --git a/crux-mir/lib/alloc/src/collections/vec_deque/spec_extend.rs b/crux-mir/lib/alloc/src/collections/vec_deque/spec_extend.rs new file mode 100644 index 000000000..dccf40ccb --- /dev/null +++ b/crux-mir/lib/alloc/src/collections/vec_deque/spec_extend.rs @@ -0,0 +1,123 @@ +use crate::alloc::Allocator; +use crate::vec; +use core::iter::TrustedLen; +use core::slice; + +use super::VecDeque; + +// Specialization trait used for VecDeque::extend +pub(super) trait SpecExtend { + fn spec_extend(&mut self, iter: I); +} + +impl SpecExtend for VecDeque +where + I: Iterator, +{ + default fn spec_extend(&mut self, mut iter: I) { + // This function should be the moral equivalent of: + // + // for item in iter { + // self.push_back(item); + // } + + // May only be called if `deque.len() < deque.capacity()` + unsafe fn push_unchecked(deque: &mut VecDeque, element: T) { + // SAFETY: Because of the precondition, it's guaranteed that there is space + // in the logical array after the last element. + unsafe { deque.buffer_write(deque.to_physical_idx(deque.len), element) }; + // This can't overflow because `deque.len() < deque.capacity() <= usize::MAX`. + deque.len += 1; + } + + while let Some(element) = iter.next() { + let (lower, _) = iter.size_hint(); + self.reserve(lower.saturating_add(1)); + + // SAFETY: We just reserved space for at least one element. + unsafe { push_unchecked(self, element) }; + + // Inner loop to avoid repeatedly calling `reserve`. + while self.len < self.capacity() { + let Some(element) = iter.next() else { + return; + }; + // SAFETY: The loop condition guarantees that `self.len() < self.capacity()`. + unsafe { push_unchecked(self, element) }; + } + } + } +} + +impl SpecExtend for VecDeque +where + I: TrustedLen, +{ + default fn spec_extend(&mut self, iter: I) { + // This is the case for a TrustedLen iterator. + let (low, high) = iter.size_hint(); + if let Some(additional) = high { + debug_assert_eq!( + low, + additional, + "TrustedLen iterator's size hint is not exact: {:?}", + (low, high) + ); + self.reserve(additional); + + let written = unsafe { + self.write_iter_wrapping(self.to_physical_idx(self.len), iter, additional) + }; + + debug_assert_eq!( + additional, written, + "The number of items written to VecDeque doesn't match the TrustedLen size hint" + ); + } else { + // Per TrustedLen contract a `None` upper bound means that the iterator length + // truly exceeds usize::MAX, which would eventually lead to a capacity overflow anyway. + // Since the other branch already panics eagerly (via `reserve()`) we do the same here. + // This avoids additional codegen for a fallback code path which would eventually + // panic anyway. + panic!("capacity overflow"); + } + } +} + +impl SpecExtend> for VecDeque { + fn spec_extend(&mut self, mut iterator: vec::IntoIter) { + let slice = iterator.as_slice(); + self.reserve(slice.len()); + + unsafe { + self.copy_slice(self.to_physical_idx(self.len), slice); + self.len += slice.len(); + } + iterator.forget_remaining_elements(); + } +} + +impl<'a, T: 'a, I, A: Allocator> SpecExtend<&'a T, I> for VecDeque +where + I: Iterator, + T: Copy, +{ + default fn spec_extend(&mut self, iterator: I) { + self.spec_extend(iterator.copied()) + } +} + +impl<'a, T: 'a, A: Allocator> SpecExtend<&'a T, slice::Iter<'a, T>> for VecDeque +where + T: Copy, +{ + fn spec_extend(&mut self, iterator: slice::Iter<'a, T>) { + let slice = iterator.as_slice(); + self.reserve(slice.len()); + + unsafe { + self.copy_slice(self.to_physical_idx(self.len), slice); + self.len += slice.len(); + } + } +} diff --git a/crux-mir/lib/alloc/src/collections/vec_deque/spec_from_iter.rs b/crux-mir/lib/alloc/src/collections/vec_deque/spec_from_iter.rs new file mode 100644 index 000000000..7650492eb --- /dev/null +++ b/crux-mir/lib/alloc/src/collections/vec_deque/spec_from_iter.rs @@ -0,0 +1,33 @@ +use super::{IntoIter, VecDeque}; + +/// Specialization trait used for `VecDeque::from_iter` +pub(super) trait SpecFromIter { + fn spec_from_iter(iter: I) -> Self; +} + +impl SpecFromIter for VecDeque +where + I: Iterator, +{ + default fn spec_from_iter(iterator: I) -> Self { + // Since converting is O(1) now, just re-use the `Vec` logic for + // anything where we can't do something extra-special for `VecDeque`, + // especially as that could save us some monomorphiziation work + // if one uses the same iterators (like slice ones) with both. + crate::vec::Vec::from_iter(iterator).into() + } +} + +impl SpecFromIter> for VecDeque { + #[inline] + fn spec_from_iter(iterator: crate::vec::IntoIter) -> Self { + iterator.into_vecdeque() + } +} + +impl SpecFromIter> for VecDeque { + #[inline] + fn spec_from_iter(iterator: IntoIter) -> Self { + iterator.into_vecdeque() + } +} diff --git a/crux-mir/lib/alloc/src/collections/vec_deque/tests.rs b/crux-mir/lib/alloc/src/collections/vec_deque/tests.rs index f2ce5b1d1..220ad71be 100644 --- a/crux-mir/lib/alloc/src/collections/vec_deque/tests.rs +++ b/crux-mir/lib/alloc/src/collections/vec_deque/tests.rs @@ -1,9 +1,8 @@ -use super::*; +use core::iter::TrustedLen; -use ::test; +use super::*; #[bench] -#[cfg_attr(miri, ignore)] // Miri does not support benchmarks fn bench_push_back_100(b: &mut test::Bencher) { let mut deq = VecDeque::with_capacity(101); b.iter(|| { @@ -11,12 +10,11 @@ fn bench_push_back_100(b: &mut test::Bencher) { deq.push_back(i); } deq.head = 0; - deq.tail = 0; + deq.len = 0; }) } #[bench] -#[cfg_attr(miri, ignore)] // Miri does not support benchmarks fn bench_push_front_100(b: &mut test::Bencher) { let mut deq = VecDeque::with_capacity(101); b.iter(|| { @@ -24,18 +22,21 @@ fn bench_push_front_100(b: &mut test::Bencher) { deq.push_front(i); } deq.head = 0; - deq.tail = 0; + deq.len = 0; }) } #[bench] -#[cfg_attr(miri, ignore)] // Miri does not support benchmarks fn bench_pop_back_100(b: &mut test::Bencher) { - let mut deq = VecDeque::::with_capacity(101); + let size = 100; + let mut deq = VecDeque::::with_capacity(size + 1); + // We'll mess with private state to pretend like `deq` is filled. + // Make sure the buffer is initialized so that we don't read uninit memory. + unsafe { deq.ptr().write_bytes(0u8, size + 1) }; b.iter(|| { - deq.head = 100; - deq.tail = 0; + deq.head = 0; + deq.len = 100; while !deq.is_empty() { test::black_box(deq.pop_back()); } @@ -43,13 +44,49 @@ fn bench_pop_back_100(b: &mut test::Bencher) { } #[bench] -#[cfg_attr(miri, ignore)] // Miri does not support benchmarks +fn bench_retain_whole_10000(b: &mut test::Bencher) { + let size = if cfg!(miri) { 1000 } else { 100000 }; + let v = (1..size).collect::>(); + + b.iter(|| { + let mut v = v.clone(); + v.retain(|x| *x > 0) + }) +} + +#[bench] +fn bench_retain_odd_10000(b: &mut test::Bencher) { + let size = if cfg!(miri) { 1000 } else { 100000 }; + let v = (1..size).collect::>(); + + b.iter(|| { + let mut v = v.clone(); + v.retain(|x| x & 1 == 0) + }) +} + +#[bench] +fn bench_retain_half_10000(b: &mut test::Bencher) { + let size = if cfg!(miri) { 1000 } else { 100000 }; + let v = (1..size).collect::>(); + + b.iter(|| { + let mut v = v.clone(); + v.retain(|x| *x > size / 2) + }) +} + +#[bench] fn bench_pop_front_100(b: &mut test::Bencher) { - let mut deq = VecDeque::::with_capacity(101); + let size = 100; + let mut deq = VecDeque::::with_capacity(size + 1); + // We'll mess with private state to pretend like `deq` is filled. + // Make sure the buffer is initialized so that we don't read uninit memory. + unsafe { deq.ptr().write_bytes(0u8, size + 1) }; b.iter(|| { - deq.head = 100; - deq.tail = 0; + deq.head = 0; + deq.len = 100; while !deq.is_empty() { test::black_box(deq.pop_front()); } @@ -68,9 +105,9 @@ fn test_swap_front_back_remove() { for len in 0..final_len { let expected: VecDeque<_> = if back { (0..len).collect() } else { (0..len).rev().collect() }; - for tail_pos in 0..usable_cap { - tester.tail = tail_pos; - tester.head = tail_pos; + for head_pos in 0..usable_cap { + tester.head = head_pos; + tester.len = 0; if back { for i in 0..len * 2 { tester.push_front(i); @@ -87,8 +124,8 @@ fn test_swap_front_back_remove() { assert_eq!(tester.swap_remove_front(idx), Some(len * 2 - 1 - i)); } } - assert!(tester.tail < tester.cap()); - assert!(tester.head < tester.cap()); + assert!(tester.head <= tester.capacity()); + assert!(tester.len <= tester.capacity()); assert_eq!(tester, expected); } } @@ -109,27 +146,475 @@ fn test_insert() { let cap = tester.capacity(); // len is the length *after* insertion - for len in 1..cap { + let minlen = if cfg!(miri) { cap - 1 } else { 1 }; // Miri is too slow + for len in minlen..cap { // 0, 1, 2, .., len - 1 let expected = (0..).take(len).collect::>(); - for tail_pos in 0..cap { + for head_pos in 0..cap { for to_insert in 0..len { - tester.tail = tail_pos; - tester.head = tail_pos; + tester.head = head_pos; + tester.len = 0; for i in 0..len { if i != to_insert { tester.push_back(i); } } tester.insert(to_insert, to_insert); - assert!(tester.tail < tester.cap()); - assert!(tester.head < tester.cap()); + assert!(tester.head <= tester.capacity()); + assert!(tester.len <= tester.capacity()); assert_eq!(tester, expected); } } } } +#[test] +fn test_get() { + let mut tester = VecDeque::new(); + tester.push_back(1); + tester.push_back(2); + tester.push_back(3); + + assert_eq!(tester.len(), 3); + + assert_eq!(tester.get(1), Some(&2)); + assert_eq!(tester.get(2), Some(&3)); + assert_eq!(tester.get(0), Some(&1)); + assert_eq!(tester.get(3), None); + + tester.remove(0); + + assert_eq!(tester.len(), 2); + assert_eq!(tester.get(0), Some(&2)); + assert_eq!(tester.get(1), Some(&3)); + assert_eq!(tester.get(2), None); +} + +#[test] +fn test_get_mut() { + let mut tester = VecDeque::new(); + tester.push_back(1); + tester.push_back(2); + tester.push_back(3); + + assert_eq!(tester.len(), 3); + + if let Some(elem) = tester.get_mut(0) { + assert_eq!(*elem, 1); + *elem = 10; + } + + if let Some(elem) = tester.get_mut(2) { + assert_eq!(*elem, 3); + *elem = 30; + } + + assert_eq!(tester.get(0), Some(&10)); + assert_eq!(tester.get(2), Some(&30)); + assert_eq!(tester.get_mut(3), None); + + tester.remove(2); + + assert_eq!(tester.len(), 2); + assert_eq!(tester.get(0), Some(&10)); + assert_eq!(tester.get(1), Some(&2)); + assert_eq!(tester.get(2), None); +} + +#[test] +fn test_swap() { + let mut tester = VecDeque::new(); + tester.push_back(1); + tester.push_back(2); + tester.push_back(3); + + assert_eq!(tester, [1, 2, 3]); + + tester.swap(0, 0); + assert_eq!(tester, [1, 2, 3]); + tester.swap(0, 1); + assert_eq!(tester, [2, 1, 3]); + tester.swap(2, 1); + assert_eq!(tester, [2, 3, 1]); + tester.swap(1, 2); + assert_eq!(tester, [2, 1, 3]); + tester.swap(0, 2); + assert_eq!(tester, [3, 1, 2]); + tester.swap(2, 2); + assert_eq!(tester, [3, 1, 2]); +} + +#[test] +#[should_panic = "assertion failed: j < self.len()"] +fn test_swap_panic() { + let mut tester = VecDeque::new(); + tester.push_back(1); + tester.push_back(2); + tester.push_back(3); + tester.swap(2, 3); +} + +#[test] +fn test_reserve_exact() { + let mut tester: VecDeque = VecDeque::with_capacity(1); + assert_eq!(tester.capacity(), 1); + tester.reserve_exact(50); + assert_eq!(tester.capacity(), 50); + tester.reserve_exact(40); + // reserving won't shrink the buffer + assert_eq!(tester.capacity(), 50); + tester.reserve_exact(200); + assert_eq!(tester.capacity(), 200); +} + +#[test] +#[should_panic = "capacity overflow"] +fn test_reserve_exact_panic() { + let mut tester: VecDeque = VecDeque::new(); + tester.reserve_exact(usize::MAX); +} + +#[test] +fn test_try_reserve_exact() { + let mut tester: VecDeque = VecDeque::with_capacity(1); + assert!(tester.capacity() == 1); + assert_eq!(tester.try_reserve_exact(100), Ok(())); + assert!(tester.capacity() >= 100); + assert_eq!(tester.try_reserve_exact(50), Ok(())); + assert!(tester.capacity() >= 100); + assert_eq!(tester.try_reserve_exact(200), Ok(())); + assert!(tester.capacity() >= 200); + assert_eq!(tester.try_reserve_exact(0), Ok(())); + assert!(tester.capacity() >= 200); + assert!(tester.try_reserve_exact(usize::MAX).is_err()); +} + +#[test] +fn test_try_reserve() { + let mut tester: VecDeque = VecDeque::with_capacity(1); + assert!(tester.capacity() == 1); + assert_eq!(tester.try_reserve(100), Ok(())); + assert!(tester.capacity() >= 100); + assert_eq!(tester.try_reserve(50), Ok(())); + assert!(tester.capacity() >= 100); + assert_eq!(tester.try_reserve(200), Ok(())); + assert!(tester.capacity() >= 200); + assert_eq!(tester.try_reserve(0), Ok(())); + assert!(tester.capacity() >= 200); + assert!(tester.try_reserve(usize::MAX).is_err()); +} + +#[test] +fn test_contains() { + let mut tester = VecDeque::new(); + tester.push_back(1); + tester.push_back(2); + tester.push_back(3); + + assert!(tester.contains(&1)); + assert!(tester.contains(&3)); + assert!(!tester.contains(&0)); + assert!(!tester.contains(&4)); + tester.remove(0); + assert!(!tester.contains(&1)); + assert!(tester.contains(&2)); + assert!(tester.contains(&3)); +} + +#[test] +fn test_rotate_left_right() { + let mut tester: VecDeque<_> = (1..=10).collect(); + tester.reserve(1); + + assert_eq!(tester.len(), 10); + + tester.rotate_left(0); + assert_eq!(tester, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + + tester.rotate_right(0); + assert_eq!(tester, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + + tester.rotate_left(3); + assert_eq!(tester, [4, 5, 6, 7, 8, 9, 10, 1, 2, 3]); + + tester.rotate_right(5); + assert_eq!(tester, [9, 10, 1, 2, 3, 4, 5, 6, 7, 8]); + + tester.rotate_left(tester.len()); + assert_eq!(tester, [9, 10, 1, 2, 3, 4, 5, 6, 7, 8]); + + tester.rotate_right(tester.len()); + assert_eq!(tester, [9, 10, 1, 2, 3, 4, 5, 6, 7, 8]); + + tester.rotate_left(1); + assert_eq!(tester, [10, 1, 2, 3, 4, 5, 6, 7, 8, 9]); +} + +#[test] +#[should_panic = "assertion failed: mid <= self.len()"] +fn test_rotate_left_panic() { + let mut tester: VecDeque<_> = (1..=10).collect(); + tester.rotate_left(tester.len() + 1); +} + +#[test] +#[should_panic = "assertion failed: k <= self.len()"] +fn test_rotate_right_panic() { + let mut tester: VecDeque<_> = (1..=10).collect(); + tester.rotate_right(tester.len() + 1); +} + +#[test] +fn test_binary_search() { + // If the givin VecDeque is not sorted, the returned result is unspecified and meaningless, + // as this method performs a binary search. + + let tester: VecDeque<_> = [0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55].into(); + + assert_eq!(tester.binary_search(&0), Ok(0)); + assert_eq!(tester.binary_search(&5), Ok(5)); + assert_eq!(tester.binary_search(&55), Ok(10)); + assert_eq!(tester.binary_search(&4), Err(5)); + assert_eq!(tester.binary_search(&-1), Err(0)); + assert!(matches!(tester.binary_search(&1), Ok(1..=2))); + + let tester: VecDeque<_> = [1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3].into(); + assert_eq!(tester.binary_search(&1), Ok(0)); + assert!(matches!(tester.binary_search(&2), Ok(1..=4))); + assert!(matches!(tester.binary_search(&3), Ok(5..=13))); + assert_eq!(tester.binary_search(&-2), Err(0)); + assert_eq!(tester.binary_search(&0), Err(0)); + assert_eq!(tester.binary_search(&4), Err(14)); + assert_eq!(tester.binary_search(&5), Err(14)); +} + +#[test] +fn test_binary_search_by() { + // If the givin VecDeque is not sorted, the returned result is unspecified and meaningless, + // as this method performs a binary search. + + let tester: VecDeque<_> = [0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55].into(); + + assert_eq!(tester.binary_search_by(|x| x.cmp(&0)), Ok(0)); + assert_eq!(tester.binary_search_by(|x| x.cmp(&5)), Ok(5)); + assert_eq!(tester.binary_search_by(|x| x.cmp(&55)), Ok(10)); + assert_eq!(tester.binary_search_by(|x| x.cmp(&4)), Err(5)); + assert_eq!(tester.binary_search_by(|x| x.cmp(&-1)), Err(0)); + assert!(matches!(tester.binary_search_by(|x| x.cmp(&1)), Ok(1..=2))); +} + +#[test] +fn test_binary_search_key() { + // If the givin VecDeque is not sorted, the returned result is unspecified and meaningless, + // as this method performs a binary search. + + let tester: VecDeque<_> = [ + (-1, 0), + (2, 10), + (6, 5), + (7, 1), + (8, 10), + (10, 2), + (20, 3), + (24, 5), + (25, 18), + (28, 13), + (31, 21), + (32, 4), + (54, 25), + ] + .into(); + + assert_eq!(tester.binary_search_by_key(&-1, |&(a, _b)| a), Ok(0)); + assert_eq!(tester.binary_search_by_key(&8, |&(a, _b)| a), Ok(4)); + assert_eq!(tester.binary_search_by_key(&25, |&(a, _b)| a), Ok(8)); + assert_eq!(tester.binary_search_by_key(&54, |&(a, _b)| a), Ok(12)); + assert_eq!(tester.binary_search_by_key(&-2, |&(a, _b)| a), Err(0)); + assert_eq!(tester.binary_search_by_key(&1, |&(a, _b)| a), Err(1)); + assert_eq!(tester.binary_search_by_key(&4, |&(a, _b)| a), Err(2)); + assert_eq!(tester.binary_search_by_key(&13, |&(a, _b)| a), Err(6)); + assert_eq!(tester.binary_search_by_key(&55, |&(a, _b)| a), Err(13)); + assert_eq!(tester.binary_search_by_key(&100, |&(a, _b)| a), Err(13)); + + let tester: VecDeque<_> = [ + (0, 0), + (2, 1), + (6, 1), + (5, 1), + (3, 1), + (1, 2), + (2, 3), + (4, 5), + (5, 8), + (8, 13), + (1, 21), + (2, 34), + (4, 55), + ] + .into(); + + assert_eq!(tester.binary_search_by_key(&0, |&(_a, b)| b), Ok(0)); + assert!(matches!(tester.binary_search_by_key(&1, |&(_a, b)| b), Ok(1..=4))); + assert_eq!(tester.binary_search_by_key(&8, |&(_a, b)| b), Ok(8)); + assert_eq!(tester.binary_search_by_key(&13, |&(_a, b)| b), Ok(9)); + assert_eq!(tester.binary_search_by_key(&55, |&(_a, b)| b), Ok(12)); + assert_eq!(tester.binary_search_by_key(&-1, |&(_a, b)| b), Err(0)); + assert_eq!(tester.binary_search_by_key(&4, |&(_a, b)| b), Err(7)); + assert_eq!(tester.binary_search_by_key(&56, |&(_a, b)| b), Err(13)); + assert_eq!(tester.binary_search_by_key(&100, |&(_a, b)| b), Err(13)); +} + +#[test] +fn make_contiguous_big_head() { + let mut tester = VecDeque::with_capacity(15); + + for i in 0..3 { + tester.push_back(i); + } + + for i in 3..10 { + tester.push_front(i); + } + + // 012......9876543 + assert_eq!(tester.capacity(), 15); + assert_eq!((&[9, 8, 7, 6, 5, 4, 3] as &[_], &[0, 1, 2] as &[_]), tester.as_slices()); + + let expected_start = tester.as_slices().1.len(); + tester.make_contiguous(); + assert_eq!(tester.head, expected_start); + assert_eq!((&[9, 8, 7, 6, 5, 4, 3, 0, 1, 2] as &[_], &[] as &[_]), tester.as_slices()); +} + +#[test] +fn make_contiguous_big_tail() { + let mut tester = VecDeque::with_capacity(15); + + for i in 0..8 { + tester.push_back(i); + } + + for i in 8..10 { + tester.push_front(i); + } + + // 01234567......98 + let expected_start = 0; + tester.make_contiguous(); + assert_eq!(tester.head, expected_start); + assert_eq!((&[9, 8, 0, 1, 2, 3, 4, 5, 6, 7] as &[_], &[] as &[_]), tester.as_slices()); +} + +#[test] +fn make_contiguous_small_free() { + let mut tester = VecDeque::with_capacity(16); + + for i in b'A'..b'I' { + tester.push_back(i as char); + } + + for i in b'I'..b'N' { + tester.push_front(i as char); + } + + assert_eq!(tester, ['M', 'L', 'K', 'J', 'I', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H']); + + // ABCDEFGH...MLKJI + let expected_start = 0; + tester.make_contiguous(); + assert_eq!(tester.head, expected_start); + assert_eq!( + (&['M', 'L', 'K', 'J', 'I', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'] as &[_], &[] as &[_]), + tester.as_slices() + ); + + tester.clear(); + for i in b'I'..b'N' { + tester.push_back(i as char); + } + + for i in b'A'..b'I' { + tester.push_front(i as char); + } + + // IJKLM...HGFEDCBA + let expected_start = 3; + tester.make_contiguous(); + assert_eq!(tester.head, expected_start); + assert_eq!( + (&['H', 'G', 'F', 'E', 'D', 'C', 'B', 'A', 'I', 'J', 'K', 'L', 'M'] as &[_], &[] as &[_]), + tester.as_slices() + ); +} + +#[test] +fn make_contiguous_head_to_end() { + let mut tester = VecDeque::with_capacity(16); + + for i in b'A'..b'L' { + tester.push_back(i as char); + } + + for i in b'L'..b'Q' { + tester.push_front(i as char); + } + + assert_eq!( + tester, + ['P', 'O', 'N', 'M', 'L', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K'] + ); + + // ABCDEFGHIJKPONML + let expected_start = 0; + tester.make_contiguous(); + assert_eq!(tester.head, expected_start); + assert_eq!( + ( + &['P', 'O', 'N', 'M', 'L', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K'] + as &[_], + &[] as &[_] + ), + tester.as_slices() + ); + + tester.clear(); + for i in b'L'..b'Q' { + tester.push_back(i as char); + } + + for i in b'A'..b'L' { + tester.push_front(i as char); + } + + // LMNOPKJIHGFEDCBA + let expected_start = 0; + tester.make_contiguous(); + assert_eq!(tester.head, expected_start); + assert_eq!( + ( + &['K', 'J', 'I', 'H', 'G', 'F', 'E', 'D', 'C', 'B', 'A', 'L', 'M', 'N', 'O', 'P'] + as &[_], + &[] as &[_] + ), + tester.as_slices() + ); +} + +#[test] +fn make_contiguous_head_to_end_2() { + // Another test case for #79808, taken from #80293. + + let mut dq = VecDeque::from_iter(0..6); + dq.pop_front(); + dq.pop_front(); + dq.push_back(6); + dq.push_back(7); + dq.push_back(8); + dq.make_contiguous(); + let collected: Vec<_> = dq.iter().copied().collect(); + assert_eq!(dq.as_slices(), (&collected[..], &[] as &[_])); +} + #[test] fn test_remove() { // This test checks that every single combination of tail position, length, and @@ -142,13 +627,14 @@ fn test_remove() { let cap = tester.capacity(); // len is the length *after* removal - for len in 0..cap - 1 { + let minlen = if cfg!(miri) { cap - 2 } else { 0 }; // Miri is too slow + for len in minlen..cap - 1 { // 0, 1, 2, .., len - 1 let expected = (0..).take(len).collect::>(); - for tail_pos in 0..cap { + for head_pos in 0..cap { for to_remove in 0..=len { - tester.tail = tail_pos; - tester.head = tail_pos; + tester.head = head_pos; + tester.len = 0; for i in 0..len { if i == to_remove { tester.push_back(1234); @@ -159,25 +645,85 @@ fn test_remove() { tester.push_back(1234); } tester.remove(to_remove); - assert!(tester.tail < tester.cap()); - assert!(tester.head < tester.cap()); + assert!(tester.head <= tester.capacity()); + assert!(tester.len <= tester.capacity()); assert_eq!(tester, expected); } } } } +#[test] +fn test_range() { + let mut tester: VecDeque = VecDeque::with_capacity(7); + + let cap = tester.capacity(); + let minlen = if cfg!(miri) { cap - 1 } else { 0 }; // Miri is too slow + for len in minlen..=cap { + for head in 0..=cap { + for start in 0..=len { + for end in start..=len { + tester.head = head; + tester.len = 0; + for i in 0..len { + tester.push_back(i); + } + + // Check that we iterate over the correct values + let range: VecDeque<_> = tester.range(start..end).copied().collect(); + let expected: VecDeque<_> = (start..end).collect(); + assert_eq!(range, expected); + } + } + } + } +} + +#[test] +fn test_range_mut() { + let mut tester: VecDeque = VecDeque::with_capacity(7); + + let cap = tester.capacity(); + for len in 0..=cap { + for head in 0..=cap { + for start in 0..=len { + for end in start..=len { + tester.head = head; + tester.len = 0; + for i in 0..len { + tester.push_back(i); + } + + let head_was = tester.head; + let len_was = tester.len; + + // Check that we iterate over the correct values + let range: VecDeque<_> = tester.range_mut(start..end).map(|v| *v).collect(); + let expected: VecDeque<_> = (start..end).collect(); + assert_eq!(range, expected); + + // We shouldn't have changed the capacity or made the + // head or tail out of bounds + assert_eq!(tester.capacity(), cap); + assert_eq!(tester.head, head_was); + assert_eq!(tester.len, len_was); + } + } + } + } +} + #[test] fn test_drain() { let mut tester: VecDeque = VecDeque::with_capacity(7); let cap = tester.capacity(); for len in 0..=cap { - for tail in 0..=cap { + for head in 0..cap { for drain_start in 0..=len { for drain_end in drain_start..=len { - tester.tail = tail; - tester.head = tail; + tester.head = head; + tester.len = 0; for i in 0..len { tester.push_back(i); } @@ -190,8 +736,8 @@ fn test_drain() { // We shouldn't have changed the capacity or made the // head or tail out of bounds assert_eq!(tester.capacity(), cap); - assert!(tester.tail < tester.cap()); - assert!(tester.head < tester.cap()); + assert!(tester.head <= tester.capacity()); + assert!(tester.len <= tester.capacity()); // We should see the correct values in the VecDeque let expected: VecDeque<_> = (0..drain_start).chain(drain_end..len).collect(); @@ -218,17 +764,18 @@ fn test_shrink_to_fit() { for len in 0..=cap { // 0, 1, 2, .., len - 1 let expected = (0..).take(len).collect::>(); - for tail_pos in 0..=max_cap { - tester.tail = tail_pos; - tester.head = tail_pos; + for head_pos in 0..=max_cap { + tester.reserve(head_pos); + tester.head = head_pos; + tester.len = 0; tester.reserve(63); for i in 0..len { tester.push_back(i); } tester.shrink_to_fit(); assert!(tester.capacity() <= cap); - assert!(tester.tail < tester.cap()); - assert!(tester.head < tester.cap()); + assert!(tester.head <= tester.capacity()); + assert!(tester.len <= tester.capacity()); assert_eq!(tester, expected); } } @@ -246,7 +793,8 @@ fn test_split_off() { let cap = tester.capacity(); // len is the length *before* splitting - for len in 0..cap { + let minlen = if cfg!(miri) { cap - 1 } else { 0 }; // Miri is too slow + for len in minlen..cap { // index to split at for at in 0..=len { // 0, 1, 2, .., at - 1 (may be empty) @@ -254,17 +802,17 @@ fn test_split_off() { // at, at + 1, .., len - 1 (may be empty) let expected_other = (at..).take(len - at).collect::>(); - for tail_pos in 0..cap { - tester.tail = tail_pos; - tester.head = tail_pos; + for head_pos in 0..cap { + tester.head = head_pos; + tester.len = 0; for i in 0..len { tester.push_back(i); } let result = tester.split_off(at); - assert!(tester.tail < tester.cap()); - assert!(tester.head < tester.cap()); - assert!(result.tail < result.cap()); - assert!(result.head < result.cap()); + assert!(tester.head <= tester.capacity()); + assert!(tester.len <= tester.capacity()); + assert!(result.head <= result.capacity()); + assert!(result.len <= result.capacity()); assert_eq!(tester, expected_self); assert_eq!(result, expected_other); } @@ -281,13 +829,127 @@ fn test_from_vec() { vec.extend(0..len); let vd = VecDeque::from(vec.clone()); - assert!(vd.cap().is_power_of_two()); assert_eq!(vd.len(), vec.len()); assert!(vd.into_iter().eq(vec)); } } } +#[test] +fn test_extend_basic() { + test_extend_impl(false); +} + +#[test] +fn test_extend_trusted_len() { + test_extend_impl(true); +} + +fn test_extend_impl(trusted_len: bool) { + struct VecDequeTester { + test: VecDeque, + expected: VecDeque, + trusted_len: bool, + } + + impl VecDequeTester { + fn new(trusted_len: bool) -> Self { + Self { test: VecDeque::new(), expected: VecDeque::new(), trusted_len } + } + + fn test_extend(&mut self, iter: I) + where + I: Iterator + TrustedLen + Clone, + { + struct BasicIterator(I); + impl Iterator for BasicIterator + where + I: Iterator, + { + type Item = usize; + + fn next(&mut self) -> Option { + self.0.next() + } + } + + if self.trusted_len { + self.test.extend(iter.clone()); + } else { + self.test.extend(BasicIterator(iter.clone())); + } + + for item in iter { + self.expected.push_back(item) + } + + assert_eq!(self.test, self.expected); + } + + fn drain + Clone>(&mut self, range: R) { + self.test.drain(range.clone()); + self.expected.drain(range); + + assert_eq!(self.test, self.expected); + } + + fn clear(&mut self) { + self.test.clear(); + self.expected.clear(); + } + + fn remaining_capacity(&self) -> usize { + self.test.capacity() - self.test.len() + } + } + + let mut tester = VecDequeTester::new(trusted_len); + + // Initial capacity + tester.test_extend(0..tester.remaining_capacity()); + + // Grow + tester.test_extend(1024..2048); + + // Wrap around + tester.drain(..128); + + tester.test_extend(0..tester.remaining_capacity()); + + // Continue + tester.drain(256..); + tester.test_extend(4096..8196); + + tester.clear(); + + // Start again + tester.test_extend(0..32); +} + +#[test] +fn test_from_array() { + fn test() { + let mut array: [usize; N] = [0; N]; + + for i in 0..N { + array[i] = i; + } + + let deq: VecDeque<_> = array.into(); + + for i in 0..N { + assert_eq!(deq[i], i); + } + + assert_eq!(deq.len(), N); + } + test::<0>(); + test::<1>(); + test::<2>(); + test::<32>(); + test::<35>(); +} + #[test] fn test_vec_from_vecdeque() { use crate::vec::Vec; @@ -305,10 +967,8 @@ fn test_vec_from_vecdeque() { assert!(vec.into_iter().eq(vd)); } - #[cfg(not(miri))] // Miri is too slow - let max_pwr = 7; - #[cfg(miri)] - let max_pwr = 5; + // Miri is too slow + let max_pwr = if cfg!(miri) { 5 } else { 7 }; for cap_pwr in 0..max_pwr { // Make capacity as a (2^x)-1, so that the ring size is 2^x @@ -359,8 +1019,9 @@ fn test_vec_from_vecdeque() { fn test_clone_from() { let m = vec![1; 8]; let n = vec![2; 12]; - for pfv in 0..8 { - for pfu in 0..8 { + let limit = if cfg!(miri) { 4 } else { 8 }; // Miri is too slow + for pfv in 0..limit { + for pfu in 0..limit { for longer in 0..2 { let (vr, ur) = if longer == 0 { (&m, &n) } else { (&n, &m) }; let mut v = VecDeque::from(vr.clone()); @@ -429,3 +1090,43 @@ fn issue_53529() { assert_eq!(*a, 2); } } + +#[test] +fn issue_80303() { + use core::iter; + use core::num::Wrapping; + + // This is a valid, albeit rather bad hash function implementation. + struct SimpleHasher(Wrapping); + + impl Hasher for SimpleHasher { + fn finish(&self) -> u64 { + self.0.0 + } + + fn write(&mut self, bytes: &[u8]) { + // This particular implementation hashes value 24 in addition to bytes. + // Such an implementation is valid as Hasher only guarantees equivalence + // for the exact same set of calls to its methods. + for &v in iter::once(&24).chain(bytes) { + self.0 = Wrapping(31) * self.0 + Wrapping(u64::from(v)); + } + } + } + + fn hash_code(value: impl Hash) -> u64 { + let mut hasher = SimpleHasher(Wrapping(1)); + value.hash(&mut hasher); + hasher.finish() + } + + // This creates two deques for which values returned by as_slices + // method differ. + let vda: VecDeque = (0..10).collect(); + let mut vdb = VecDeque::with_capacity(10); + vdb.extend(5..10); + (0..5).rev().for_each(|elem| vdb.push_front(elem)); + assert_ne!(vda.as_slices(), vdb.as_slices()); + assert_eq!(vda, vdb); + assert_eq!(hash_code(vda), hash_code(vdb)); +} diff --git a/crux-mir/lib/alloc/src/ffi/c_str.rs b/crux-mir/lib/alloc/src/ffi/c_str.rs new file mode 100644 index 000000000..11bd4c4dc --- /dev/null +++ b/crux-mir/lib/alloc/src/ffi/c_str.rs @@ -0,0 +1,1146 @@ +#[cfg(test)] +mod tests; + +use crate::borrow::{Cow, ToOwned}; +use crate::boxed::Box; +use crate::rc::Rc; +use crate::slice::hack::into_vec; +use crate::string::String; +use crate::vec::Vec; +use core::borrow::Borrow; +use core::ffi::{c_char, CStr}; +use core::fmt; +use core::mem; +use core::num::NonZeroU8; +use core::ops; +use core::ptr; +use core::slice; +use core::slice::memchr; +use core::str::{self, Utf8Error}; + +#[cfg(target_has_atomic = "ptr")] +use crate::sync::Arc; + +/// A type representing an owned, C-compatible, nul-terminated string with no nul bytes in the +/// middle. +/// +/// This type serves the purpose of being able to safely generate a +/// C-compatible string from a Rust byte slice or vector. An instance of this +/// type is a static guarantee that the underlying bytes contain no interior 0 +/// bytes ("nul characters") and that the final byte is 0 ("nul terminator"). +/// +/// `CString` is to &[CStr] as [`String`] is to &[str]: the former +/// in each pair are owned strings; the latter are borrowed +/// references. +/// +/// # Creating a `CString` +/// +/// A `CString` is created from either a byte slice or a byte vector, +/// or anything that implements [Into]<[Vec]<[u8]>> (for +/// example, you can build a `CString` straight out of a [`String`] or +/// a &[str], since both implement that trait). +/// +/// The [`CString::new`] method will actually check that the provided &[[u8]] +/// does not have 0 bytes in the middle, and return an error if it +/// finds one. +/// +/// # Extracting a raw pointer to the whole C string +/// +/// `CString` implements an [`as_ptr`][`CStr::as_ptr`] method through the [`Deref`] +/// trait. This method will give you a `*const c_char` which you can +/// feed directly to extern functions that expect a nul-terminated +/// string, like C's `strdup()`. Notice that [`as_ptr`][`CStr::as_ptr`] returns a +/// read-only pointer; if the C code writes to it, that causes +/// undefined behavior. +/// +/// # Extracting a slice of the whole C string +/// +/// Alternatively, you can obtain a &[[u8]] slice from a +/// `CString` with the [`CString::as_bytes`] method. Slices produced in this +/// way do *not* contain the trailing nul terminator. This is useful +/// when you will be calling an extern function that takes a `*const +/// u8` argument which is not necessarily nul-terminated, plus another +/// argument with the length of the string — like C's `strndup()`. +/// You can of course get the slice's length with its +/// [`len`][slice::len] method. +/// +/// If you need a &[[u8]] slice *with* the nul terminator, you +/// can use [`CString::as_bytes_with_nul`] instead. +/// +/// Once you have the kind of slice you need (with or without a nul +/// terminator), you can call the slice's own +/// [`as_ptr`][slice::as_ptr] method to get a read-only raw pointer to pass to +/// extern functions. See the documentation for that function for a +/// discussion on ensuring the lifetime of the raw pointer. +/// +/// [str]: prim@str "str" +/// [`Deref`]: ops::Deref +/// +/// # Examples +/// +/// ```ignore (extern-declaration) +/// # fn main() { +/// use std::ffi::CString; +/// use std::os::raw::c_char; +/// +/// extern "C" { +/// fn my_printer(s: *const c_char); +/// } +/// +/// // We are certain that our string doesn't have 0 bytes in the middle, +/// // so we can .expect() +/// let c_to_print = CString::new("Hello, world!").expect("CString::new failed"); +/// unsafe { +/// my_printer(c_to_print.as_ptr()); +/// } +/// # } +/// ``` +/// +/// # Safety +/// +/// `CString` is intended for working with traditional C-style strings +/// (a sequence of non-nul bytes terminated by a single nul byte); the +/// primary use case for these kinds of strings is interoperating with C-like +/// code. Often you will need to transfer ownership to/from that external +/// code. It is strongly recommended that you thoroughly read through the +/// documentation of `CString` before use, as improper ownership management +/// of `CString` instances can lead to invalid memory accesses, memory leaks, +/// and other memory errors. +#[derive(PartialEq, PartialOrd, Eq, Ord, Hash, Clone)] +#[cfg_attr(not(test), rustc_diagnostic_item = "cstring_type")] +#[stable(feature = "alloc_c_string", since = "1.64.0")] +pub struct CString { + // Invariant 1: the slice ends with a zero byte and has a length of at least one. + // Invariant 2: the slice contains only one zero byte. + // Improper usage of unsafe function can break Invariant 2, but not Invariant 1. + inner: Box<[u8]>, +} + +/// An error indicating that an interior nul byte was found. +/// +/// While Rust strings may contain nul bytes in the middle, C strings +/// can't, as that byte would effectively truncate the string. +/// +/// This error is created by the [`new`][`CString::new`] method on +/// [`CString`]. See its documentation for more. +/// +/// # Examples +/// +/// ``` +/// use std::ffi::{CString, NulError}; +/// +/// let _: NulError = CString::new(b"f\0oo".to_vec()).unwrap_err(); +/// ``` +#[derive(Clone, PartialEq, Eq, Debug)] +#[stable(feature = "alloc_c_string", since = "1.64.0")] +pub struct NulError(usize, Vec); + +#[derive(Clone, PartialEq, Eq, Debug)] +enum FromBytesWithNulErrorKind { + InteriorNul(usize), + NotNulTerminated, +} + +/// An error indicating that a nul byte was not in the expected position. +/// +/// The vector used to create a [`CString`] must have one and only one nul byte, +/// positioned at the end. +/// +/// This error is created by the [`CString::from_vec_with_nul`] method. +/// See its documentation for more. +/// +/// # Examples +/// +/// ``` +/// use std::ffi::{CString, FromVecWithNulError}; +/// +/// let _: FromVecWithNulError = CString::from_vec_with_nul(b"f\0oo".to_vec()).unwrap_err(); +/// ``` +#[derive(Clone, PartialEq, Eq, Debug)] +#[stable(feature = "alloc_c_string", since = "1.64.0")] +pub struct FromVecWithNulError { + error_kind: FromBytesWithNulErrorKind, + bytes: Vec, +} + +#[stable(feature = "cstring_from_vec_with_nul", since = "1.58.0")] +impl FromVecWithNulError { + /// Returns a slice of [`u8`]s bytes that were attempted to convert to a [`CString`]. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use std::ffi::CString; + /// + /// // Some invalid bytes in a vector + /// let bytes = b"f\0oo".to_vec(); + /// + /// let value = CString::from_vec_with_nul(bytes.clone()); + /// + /// assert_eq!(&bytes[..], value.unwrap_err().as_bytes()); + /// ``` + #[must_use] + #[stable(feature = "cstring_from_vec_with_nul", since = "1.58.0")] + pub fn as_bytes(&self) -> &[u8] { + &self.bytes[..] + } + + /// Returns the bytes that were attempted to convert to a [`CString`]. + /// + /// This method is carefully constructed to avoid allocation. It will + /// consume the error, moving out the bytes, so that a copy of the bytes + /// does not need to be made. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use std::ffi::CString; + /// + /// // Some invalid bytes in a vector + /// let bytes = b"f\0oo".to_vec(); + /// + /// let value = CString::from_vec_with_nul(bytes.clone()); + /// + /// assert_eq!(bytes, value.unwrap_err().into_bytes()); + /// ``` + #[must_use = "`self` will be dropped if the result is not used"] + #[stable(feature = "cstring_from_vec_with_nul", since = "1.58.0")] + pub fn into_bytes(self) -> Vec { + self.bytes + } +} + +/// An error indicating invalid UTF-8 when converting a [`CString`] into a [`String`]. +/// +/// `CString` is just a wrapper over a buffer of bytes with a nul terminator; +/// [`CString::into_string`] performs UTF-8 validation on those bytes and may +/// return this error. +/// +/// This `struct` is created by [`CString::into_string()`]. See +/// its documentation for more. +#[derive(Clone, PartialEq, Eq, Debug)] +#[stable(feature = "alloc_c_string", since = "1.64.0")] +pub struct IntoStringError { + inner: CString, + error: Utf8Error, +} + +impl CString { + /// Creates a new C-compatible string from a container of bytes. + /// + /// This function will consume the provided data and use the + /// underlying bytes to construct a new string, ensuring that + /// there is a trailing 0 byte. This trailing 0 byte will be + /// appended by this function; the provided data should *not* + /// contain any 0 bytes in it. + /// + /// # Examples + /// + /// ```ignore (extern-declaration) + /// use std::ffi::CString; + /// use std::os::raw::c_char; + /// + /// extern "C" { fn puts(s: *const c_char); } + /// + /// let to_print = CString::new("Hello!").expect("CString::new failed"); + /// unsafe { + /// puts(to_print.as_ptr()); + /// } + /// ``` + /// + /// # Errors + /// + /// This function will return an error if the supplied bytes contain an + /// internal 0 byte. The [`NulError`] returned will contain the bytes as well as + /// the position of the nul byte. + #[stable(feature = "rust1", since = "1.0.0")] + pub fn new>>(t: T) -> Result { + trait SpecNewImpl { + fn spec_new_impl(self) -> Result; + } + + impl>> SpecNewImpl for T { + default fn spec_new_impl(self) -> Result { + let bytes: Vec = self.into(); + match memchr::memchr(0, &bytes) { + Some(i) => Err(NulError(i, bytes)), + None => Ok(unsafe { CString::_from_vec_unchecked(bytes) }), + } + } + } + + // Specialization for avoiding reallocation + #[inline(always)] // Without that it is not inlined into specializations + fn spec_new_impl_bytes(bytes: &[u8]) -> Result { + // We cannot have such large slice that we would overflow here + // but using `checked_add` allows LLVM to assume that capacity never overflows + // and generate twice shorter code. + // `saturating_add` doesn't help for some reason. + let capacity = bytes.len().checked_add(1).unwrap(); + + // Allocate before validation to avoid duplication of allocation code. + // We still need to allocate and copy memory even if we get an error. + let mut buffer = Vec::with_capacity(capacity); + buffer.extend(bytes); + + // Check memory of self instead of new buffer. + // This allows better optimizations if lto enabled. + match memchr::memchr(0, bytes) { + Some(i) => Err(NulError(i, buffer)), + None => Ok(unsafe { CString::_from_vec_unchecked(buffer) }), + } + } + + impl SpecNewImpl for &'_ [u8] { + fn spec_new_impl(self) -> Result { + spec_new_impl_bytes(self) + } + } + + impl SpecNewImpl for &'_ str { + fn spec_new_impl(self) -> Result { + spec_new_impl_bytes(self.as_bytes()) + } + } + + impl SpecNewImpl for &'_ mut [u8] { + fn spec_new_impl(self) -> Result { + spec_new_impl_bytes(self) + } + } + + t.spec_new_impl() + } + + /// Creates a C-compatible string by consuming a byte vector, + /// without checking for interior 0 bytes. + /// + /// Trailing 0 byte will be appended by this function. + /// + /// This method is equivalent to [`CString::new`] except that no runtime + /// assertion is made that `v` contains no 0 bytes, and it requires an + /// actual byte vector, not anything that can be converted to one with Into. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::CString; + /// + /// let raw = b"foo".to_vec(); + /// unsafe { + /// let c_string = CString::from_vec_unchecked(raw); + /// } + /// ``` + #[must_use] + #[stable(feature = "rust1", since = "1.0.0")] + pub unsafe fn from_vec_unchecked(v: Vec) -> Self { + debug_assert!(memchr::memchr(0, &v).is_none()); + unsafe { Self::_from_vec_unchecked(v) } + } + + unsafe fn _from_vec_unchecked(mut v: Vec) -> Self { + v.reserve_exact(1); + v.push(0); + Self { inner: v.into_boxed_slice() } + } + + /// Retakes ownership of a `CString` that was transferred to C via + /// [`CString::into_raw`]. + /// + /// Additionally, the length of the string will be recalculated from the pointer. + /// + /// # Safety + /// + /// This should only ever be called with a pointer that was earlier + /// obtained by calling [`CString::into_raw`]. Other usage (e.g., trying to take + /// ownership of a string that was allocated by foreign code) is likely to lead + /// to undefined behavior or allocator corruption. + /// + /// It should be noted that the length isn't just "recomputed," but that + /// the recomputed length must match the original length from the + /// [`CString::into_raw`] call. This means the [`CString::into_raw`]/`from_raw` + /// methods should not be used when passing the string to C functions that can + /// modify the string's length. + /// + /// > **Note:** If you need to borrow a string that was allocated by + /// > foreign code, use [`CStr`]. If you need to take ownership of + /// > a string that was allocated by foreign code, you will need to + /// > make your own provisions for freeing it appropriately, likely + /// > with the foreign code's API to do that. + /// + /// # Examples + /// + /// Creates a `CString`, pass ownership to an `extern` function (via raw pointer), then retake + /// ownership with `from_raw`: + /// + /// ```ignore (extern-declaration) + /// use std::ffi::CString; + /// use std::os::raw::c_char; + /// + /// extern "C" { + /// fn some_extern_function(s: *mut c_char); + /// } + /// + /// let c_string = CString::new("Hello!").expect("CString::new failed"); + /// let raw = c_string.into_raw(); + /// unsafe { + /// some_extern_function(raw); + /// let c_string = CString::from_raw(raw); + /// } + /// ``` + #[must_use = "call `drop(from_raw(ptr))` if you intend to drop the `CString`"] + #[stable(feature = "cstr_memory", since = "1.4.0")] + pub unsafe fn from_raw(ptr: *mut c_char) -> CString { + // SAFETY: This is called with a pointer that was obtained from a call + // to `CString::into_raw` and the length has not been modified. As such, + // we know there is a NUL byte (and only one) at the end and that the + // information about the size of the allocation is correct on Rust's + // side. + unsafe { + extern "C" { + /// Provided by libc or compiler_builtins. + fn strlen(s: *const c_char) -> usize; + } + let len = strlen(ptr) + 1; // Including the NUL byte + let slice = slice::from_raw_parts_mut(ptr, len as usize); + CString { inner: Box::from_raw(slice as *mut [c_char] as *mut [u8]) } + } + } + + /// Consumes the `CString` and transfers ownership of the string to a C caller. + /// + /// The pointer which this function returns must be returned to Rust and reconstituted using + /// [`CString::from_raw`] to be properly deallocated. Specifically, one + /// should *not* use the standard C `free()` function to deallocate + /// this string. + /// + /// Failure to call [`CString::from_raw`] will lead to a memory leak. + /// + /// The C side must **not** modify the length of the string (by writing a + /// `null` somewhere inside the string or removing the final one) before + /// it makes it back into Rust using [`CString::from_raw`]. See the safety section + /// in [`CString::from_raw`]. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::CString; + /// + /// let c_string = CString::new("foo").expect("CString::new failed"); + /// + /// let ptr = c_string.into_raw(); + /// + /// unsafe { + /// assert_eq!(b'f', *ptr as u8); + /// assert_eq!(b'o', *ptr.add(1) as u8); + /// assert_eq!(b'o', *ptr.add(2) as u8); + /// assert_eq!(b'\0', *ptr.add(3) as u8); + /// + /// // retake pointer to free memory + /// let _ = CString::from_raw(ptr); + /// } + /// ``` + #[inline] + #[must_use = "`self` will be dropped if the result is not used"] + #[stable(feature = "cstr_memory", since = "1.4.0")] + pub fn into_raw(self) -> *mut c_char { + Box::into_raw(self.into_inner()) as *mut c_char + } + + /// Converts the `CString` into a [`String`] if it contains valid UTF-8 data. + /// + /// On failure, ownership of the original `CString` is returned. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::CString; + /// + /// let valid_utf8 = vec![b'f', b'o', b'o']; + /// let cstring = CString::new(valid_utf8).expect("CString::new failed"); + /// assert_eq!(cstring.into_string().expect("into_string() call failed"), "foo"); + /// + /// let invalid_utf8 = vec![b'f', 0xff, b'o', b'o']; + /// let cstring = CString::new(invalid_utf8).expect("CString::new failed"); + /// let err = cstring.into_string().err().expect("into_string().err() failed"); + /// assert_eq!(err.utf8_error().valid_up_to(), 1); + /// ``` + #[stable(feature = "cstring_into", since = "1.7.0")] + pub fn into_string(self) -> Result { + String::from_utf8(self.into_bytes()).map_err(|e| IntoStringError { + error: e.utf8_error(), + inner: unsafe { Self::_from_vec_unchecked(e.into_bytes()) }, + }) + } + + /// Consumes the `CString` and returns the underlying byte buffer. + /// + /// The returned buffer does **not** contain the trailing nul + /// terminator, and it is guaranteed to not have any interior nul + /// bytes. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::CString; + /// + /// let c_string = CString::new("foo").expect("CString::new failed"); + /// let bytes = c_string.into_bytes(); + /// assert_eq!(bytes, vec![b'f', b'o', b'o']); + /// ``` + #[must_use = "`self` will be dropped if the result is not used"] + #[stable(feature = "cstring_into", since = "1.7.0")] + pub fn into_bytes(self) -> Vec { + let mut vec = into_vec(self.into_inner()); + let _nul = vec.pop(); + debug_assert_eq!(_nul, Some(0u8)); + vec + } + + /// Equivalent to [`CString::into_bytes()`] except that the + /// returned vector includes the trailing nul terminator. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::CString; + /// + /// let c_string = CString::new("foo").expect("CString::new failed"); + /// let bytes = c_string.into_bytes_with_nul(); + /// assert_eq!(bytes, vec![b'f', b'o', b'o', b'\0']); + /// ``` + #[must_use = "`self` will be dropped if the result is not used"] + #[stable(feature = "cstring_into", since = "1.7.0")] + pub fn into_bytes_with_nul(self) -> Vec { + into_vec(self.into_inner()) + } + + /// Returns the contents of this `CString` as a slice of bytes. + /// + /// The returned slice does **not** contain the trailing nul + /// terminator, and it is guaranteed to not have any interior nul + /// bytes. If you need the nul terminator, use + /// [`CString::as_bytes_with_nul`] instead. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::CString; + /// + /// let c_string = CString::new("foo").expect("CString::new failed"); + /// let bytes = c_string.as_bytes(); + /// assert_eq!(bytes, &[b'f', b'o', b'o']); + /// ``` + #[inline] + #[must_use] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn as_bytes(&self) -> &[u8] { + // SAFETY: CString has a length at least 1 + unsafe { self.inner.get_unchecked(..self.inner.len() - 1) } + } + + /// Equivalent to [`CString::as_bytes()`] except that the + /// returned slice includes the trailing nul terminator. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::CString; + /// + /// let c_string = CString::new("foo").expect("CString::new failed"); + /// let bytes = c_string.as_bytes_with_nul(); + /// assert_eq!(bytes, &[b'f', b'o', b'o', b'\0']); + /// ``` + #[inline] + #[must_use] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn as_bytes_with_nul(&self) -> &[u8] { + &self.inner + } + + /// Extracts a [`CStr`] slice containing the entire string. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::{CString, CStr}; + /// + /// let c_string = CString::new(b"foo".to_vec()).expect("CString::new failed"); + /// let cstr = c_string.as_c_str(); + /// assert_eq!(cstr, + /// CStr::from_bytes_with_nul(b"foo\0").expect("CStr::from_bytes_with_nul failed")); + /// ``` + #[inline] + #[must_use] + #[stable(feature = "as_c_str", since = "1.20.0")] + pub fn as_c_str(&self) -> &CStr { + &*self + } + + /// Converts this `CString` into a boxed [`CStr`]. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::{CString, CStr}; + /// + /// let c_string = CString::new(b"foo".to_vec()).expect("CString::new failed"); + /// let boxed = c_string.into_boxed_c_str(); + /// assert_eq!(&*boxed, + /// CStr::from_bytes_with_nul(b"foo\0").expect("CStr::from_bytes_with_nul failed")); + /// ``` + #[must_use = "`self` will be dropped if the result is not used"] + #[stable(feature = "into_boxed_c_str", since = "1.20.0")] + pub fn into_boxed_c_str(self) -> Box { + unsafe { Box::from_raw(Box::into_raw(self.into_inner()) as *mut CStr) } + } + + /// Bypass "move out of struct which implements [`Drop`] trait" restriction. + #[inline] + fn into_inner(self) -> Box<[u8]> { + // Rationale: `mem::forget(self)` invalidates the previous call to `ptr::read(&self.inner)` + // so we use `ManuallyDrop` to ensure `self` is not dropped. + // Then we can return the box directly without invalidating it. + // See https://github.com/rust-lang/rust/issues/62553. + let this = mem::ManuallyDrop::new(self); + unsafe { ptr::read(&this.inner) } + } + + /// Converts a [Vec]<[u8]> to a [`CString`] without checking the + /// invariants on the given [`Vec`]. + /// + /// # Safety + /// + /// The given [`Vec`] **must** have one nul byte as its last element. + /// This means it cannot be empty nor have any other nul byte anywhere else. + /// + /// # Example + /// + /// ``` + /// use std::ffi::CString; + /// assert_eq!( + /// unsafe { CString::from_vec_with_nul_unchecked(b"abc\0".to_vec()) }, + /// unsafe { CString::from_vec_unchecked(b"abc".to_vec()) } + /// ); + /// ``` + #[must_use] + #[stable(feature = "cstring_from_vec_with_nul", since = "1.58.0")] + pub unsafe fn from_vec_with_nul_unchecked(v: Vec) -> Self { + debug_assert!(memchr::memchr(0, &v).unwrap() + 1 == v.len()); + unsafe { Self::_from_vec_with_nul_unchecked(v) } + } + + unsafe fn _from_vec_with_nul_unchecked(v: Vec) -> Self { + Self { inner: v.into_boxed_slice() } + } + + /// Attempts to converts a [Vec]<[u8]> to a [`CString`]. + /// + /// Runtime checks are present to ensure there is only one nul byte in the + /// [`Vec`], its last element. + /// + /// # Errors + /// + /// If a nul byte is present and not the last element or no nul bytes + /// is present, an error will be returned. + /// + /// # Examples + /// + /// A successful conversion will produce the same result as [`CString::new`] + /// when called without the ending nul byte. + /// + /// ``` + /// use std::ffi::CString; + /// assert_eq!( + /// CString::from_vec_with_nul(b"abc\0".to_vec()) + /// .expect("CString::from_vec_with_nul failed"), + /// CString::new(b"abc".to_vec()).expect("CString::new failed") + /// ); + /// ``` + /// + /// An incorrectly formatted [`Vec`] will produce an error. + /// + /// ``` + /// use std::ffi::{CString, FromVecWithNulError}; + /// // Interior nul byte + /// let _: FromVecWithNulError = CString::from_vec_with_nul(b"a\0bc".to_vec()).unwrap_err(); + /// // No nul byte + /// let _: FromVecWithNulError = CString::from_vec_with_nul(b"abc".to_vec()).unwrap_err(); + /// ``` + #[stable(feature = "cstring_from_vec_with_nul", since = "1.58.0")] + pub fn from_vec_with_nul(v: Vec) -> Result { + let nul_pos = memchr::memchr(0, &v); + match nul_pos { + Some(nul_pos) if nul_pos + 1 == v.len() => { + // SAFETY: We know there is only one nul byte, at the end + // of the vec. + Ok(unsafe { Self::_from_vec_with_nul_unchecked(v) }) + } + Some(nul_pos) => Err(FromVecWithNulError { + error_kind: FromBytesWithNulErrorKind::InteriorNul(nul_pos), + bytes: v, + }), + None => Err(FromVecWithNulError { + error_kind: FromBytesWithNulErrorKind::NotNulTerminated, + bytes: v, + }), + } + } +} + +// Turns this `CString` into an empty string to prevent +// memory-unsafe code from working by accident. Inline +// to prevent LLVM from optimizing it away in debug builds. +#[stable(feature = "cstring_drop", since = "1.13.0")] +impl Drop for CString { + #[inline] + fn drop(&mut self) { + unsafe { + *self.inner.get_unchecked_mut(0) = 0; + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ops::Deref for CString { + type Target = CStr; + + #[inline] + fn deref(&self) -> &CStr { + unsafe { CStr::from_bytes_with_nul_unchecked(self.as_bytes_with_nul()) } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Debug for CString { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +#[stable(feature = "cstring_into", since = "1.7.0")] +impl From for Vec { + /// Converts a [`CString`] into a [Vec]<[u8]>. + /// + /// The conversion consumes the [`CString`], and removes the terminating NUL byte. + #[inline] + fn from(s: CString) -> Vec { + s.into_bytes() + } +} + +#[stable(feature = "cstr_default", since = "1.10.0")] +impl Default for CString { + /// Creates an empty `CString`. + fn default() -> CString { + let a: &CStr = Default::default(); + a.to_owned() + } +} + +#[stable(feature = "cstr_borrow", since = "1.3.0")] +impl Borrow for CString { + #[inline] + fn borrow(&self) -> &CStr { + self + } +} + +#[stable(feature = "cstring_from_cow_cstr", since = "1.28.0")] +impl<'a> From> for CString { + /// Converts a `Cow<'a, CStr>` into a `CString`, by copying the contents if they are + /// borrowed. + #[inline] + fn from(s: Cow<'a, CStr>) -> Self { + s.into_owned() + } +} + +#[cfg(not(test))] +#[stable(feature = "box_from_c_str", since = "1.17.0")] +impl From<&CStr> for Box { + /// Converts a `&CStr` into a `Box`, + /// by copying the contents into a newly allocated [`Box`]. + fn from(s: &CStr) -> Box { + let boxed: Box<[u8]> = Box::from(s.to_bytes_with_nul()); + unsafe { Box::from_raw(Box::into_raw(boxed) as *mut CStr) } + } +} + +#[stable(feature = "box_from_cow", since = "1.45.0")] +impl From> for Box { + /// Converts a `Cow<'a, CStr>` into a `Box`, + /// by copying the contents if they are borrowed. + #[inline] + fn from(cow: Cow<'_, CStr>) -> Box { + match cow { + Cow::Borrowed(s) => Box::from(s), + Cow::Owned(s) => Box::from(s), + } + } +} + +#[stable(feature = "c_string_from_box", since = "1.18.0")] +impl From> for CString { + /// Converts a [Box]<[CStr]> into a [`CString`] without copying or allocating. + #[inline] + fn from(s: Box) -> CString { + let raw = Box::into_raw(s) as *mut [u8]; + CString { inner: unsafe { Box::from_raw(raw) } } + } +} + +#[stable(feature = "cstring_from_vec_of_nonzerou8", since = "1.43.0")] +impl From> for CString { + /// Converts a [Vec]<[NonZeroU8]> into a [`CString`] without + /// copying nor checking for inner null bytes. + #[inline] + fn from(v: Vec) -> CString { + unsafe { + // Transmute `Vec` to `Vec`. + let v: Vec = { + // SAFETY: + // - transmuting between `NonZeroU8` and `u8` is sound; + // - `alloc::Layout == alloc::Layout`. + let (ptr, len, cap): (*mut NonZeroU8, _, _) = Vec::into_raw_parts(v); + Vec::from_raw_parts(ptr.cast::(), len, cap) + }; + // SAFETY: `v` cannot contain null bytes, given the type-level + // invariant of `NonZeroU8`. + Self::_from_vec_unchecked(v) + } + } +} + +#[cfg(not(test))] +#[stable(feature = "more_box_slice_clone", since = "1.29.0")] +impl Clone for Box { + #[inline] + fn clone(&self) -> Self { + (**self).into() + } +} + +#[stable(feature = "box_from_c_string", since = "1.20.0")] +impl From for Box { + /// Converts a [`CString`] into a [Box]<[CStr]> without copying or allocating. + #[inline] + fn from(s: CString) -> Box { + s.into_boxed_c_str() + } +} + +#[stable(feature = "cow_from_cstr", since = "1.28.0")] +impl<'a> From for Cow<'a, CStr> { + /// Converts a [`CString`] into an owned [`Cow`] without copying or allocating. + #[inline] + fn from(s: CString) -> Cow<'a, CStr> { + Cow::Owned(s) + } +} + +#[stable(feature = "cow_from_cstr", since = "1.28.0")] +impl<'a> From<&'a CStr> for Cow<'a, CStr> { + /// Converts a [`CStr`] into a borrowed [`Cow`] without copying or allocating. + #[inline] + fn from(s: &'a CStr) -> Cow<'a, CStr> { + Cow::Borrowed(s) + } +} + +#[stable(feature = "cow_from_cstr", since = "1.28.0")] +impl<'a> From<&'a CString> for Cow<'a, CStr> { + /// Converts a `&`[`CString`] into a borrowed [`Cow`] without copying or allocating. + #[inline] + fn from(s: &'a CString) -> Cow<'a, CStr> { + Cow::Borrowed(s.as_c_str()) + } +} + +#[cfg(target_has_atomic = "ptr")] +#[stable(feature = "shared_from_slice2", since = "1.24.0")] +impl From for Arc { + /// Converts a [`CString`] into an [Arc]<[CStr]> by moving the [`CString`] + /// data into a new [`Arc`] buffer. + #[inline] + fn from(s: CString) -> Arc { + let arc: Arc<[u8]> = Arc::from(s.into_inner()); + unsafe { Arc::from_raw(Arc::into_raw(arc) as *const CStr) } + } +} + +#[cfg(target_has_atomic = "ptr")] +#[stable(feature = "shared_from_slice2", since = "1.24.0")] +impl From<&CStr> for Arc { + /// Converts a `&CStr` into a `Arc`, + /// by copying the contents into a newly allocated [`Arc`]. + #[inline] + fn from(s: &CStr) -> Arc { + let arc: Arc<[u8]> = Arc::from(s.to_bytes_with_nul()); + unsafe { Arc::from_raw(Arc::into_raw(arc) as *const CStr) } + } +} + +#[stable(feature = "shared_from_slice2", since = "1.24.0")] +impl From for Rc { + /// Converts a [`CString`] into an [Rc]<[CStr]> by moving the [`CString`] + /// data into a new [`Arc`] buffer. + #[inline] + fn from(s: CString) -> Rc { + let rc: Rc<[u8]> = Rc::from(s.into_inner()); + unsafe { Rc::from_raw(Rc::into_raw(rc) as *const CStr) } + } +} + +#[stable(feature = "shared_from_slice2", since = "1.24.0")] +impl From<&CStr> for Rc { + /// Converts a `&CStr` into a `Rc`, + /// by copying the contents into a newly allocated [`Rc`]. + #[inline] + fn from(s: &CStr) -> Rc { + let rc: Rc<[u8]> = Rc::from(s.to_bytes_with_nul()); + unsafe { Rc::from_raw(Rc::into_raw(rc) as *const CStr) } + } +} + +#[cfg(not(test))] +#[stable(feature = "default_box_extra", since = "1.17.0")] +impl Default for Box { + fn default() -> Box { + let boxed: Box<[u8]> = Box::from([0]); + unsafe { Box::from_raw(Box::into_raw(boxed) as *mut CStr) } + } +} + +impl NulError { + /// Returns the position of the nul byte in the slice that caused + /// [`CString::new`] to fail. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::CString; + /// + /// let nul_error = CString::new("foo\0bar").unwrap_err(); + /// assert_eq!(nul_error.nul_position(), 3); + /// + /// let nul_error = CString::new("foo bar\0").unwrap_err(); + /// assert_eq!(nul_error.nul_position(), 7); + /// ``` + #[must_use] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn nul_position(&self) -> usize { + self.0 + } + + /// Consumes this error, returning the underlying vector of bytes which + /// generated the error in the first place. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::CString; + /// + /// let nul_error = CString::new("foo\0bar").unwrap_err(); + /// assert_eq!(nul_error.into_vec(), b"foo\0bar"); + /// ``` + #[must_use = "`self` will be dropped if the result is not used"] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn into_vec(self) -> Vec { + self.1 + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Display for NulError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "nul byte found in provided data at position: {}", self.0) + } +} + +#[stable(feature = "cstring_from_vec_with_nul", since = "1.58.0")] +impl fmt::Display for FromVecWithNulError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.error_kind { + FromBytesWithNulErrorKind::InteriorNul(pos) => { + write!(f, "data provided contains an interior nul byte at pos {pos}") + } + FromBytesWithNulErrorKind::NotNulTerminated => { + write!(f, "data provided is not nul terminated") + } + } + } +} + +impl IntoStringError { + /// Consumes this error, returning original [`CString`] which generated the + /// error. + #[must_use = "`self` will be dropped if the result is not used"] + #[stable(feature = "cstring_into", since = "1.7.0")] + pub fn into_cstring(self) -> CString { + self.inner + } + + /// Access the underlying UTF-8 error that was the cause of this error. + #[must_use] + #[stable(feature = "cstring_into", since = "1.7.0")] + pub fn utf8_error(&self) -> Utf8Error { + self.error + } + + #[doc(hidden)] + #[unstable(feature = "cstr_internals", issue = "none")] + pub fn __source(&self) -> &Utf8Error { + &self.error + } +} + +impl IntoStringError { + fn description(&self) -> &str { + "C string contained non-utf8 bytes" + } +} + +#[stable(feature = "cstring_into", since = "1.7.0")] +impl fmt::Display for IntoStringError { + #[allow(deprecated, deprecated_in_future)] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.description().fmt(f) + } +} + +#[stable(feature = "cstr_borrow", since = "1.3.0")] +impl ToOwned for CStr { + type Owned = CString; + + fn to_owned(&self) -> CString { + CString { inner: self.to_bytes_with_nul().into() } + } + + fn clone_into(&self, target: &mut CString) { + let mut b = into_vec(mem::take(&mut target.inner)); + self.to_bytes_with_nul().clone_into(&mut b); + target.inner = b.into_boxed_slice(); + } +} + +#[stable(feature = "cstring_asref", since = "1.7.0")] +impl From<&CStr> for CString { + fn from(s: &CStr) -> CString { + s.to_owned() + } +} + +#[stable(feature = "cstring_asref", since = "1.7.0")] +impl ops::Index for CString { + type Output = CStr; + + #[inline] + fn index(&self, _index: ops::RangeFull) -> &CStr { + self + } +} + +#[stable(feature = "cstring_asref", since = "1.7.0")] +impl AsRef for CString { + #[inline] + fn as_ref(&self) -> &CStr { + self + } +} + +#[cfg(not(test))] +impl CStr { + /// Converts a `CStr` into a [Cow]<[str]>. + /// + /// If the contents of the `CStr` are valid UTF-8 data, this + /// function will return a [Cow]::[Borrowed]\(&[str]) + /// with the corresponding &[str] slice. Otherwise, it will + /// replace any invalid UTF-8 sequences with + /// [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD] and return a + /// [Cow]::[Owned]\(&[str]) with the result. + /// + /// [str]: prim@str "str" + /// [Borrowed]: Cow::Borrowed + /// [Owned]: Cow::Owned + /// [U+FFFD]: core::char::REPLACEMENT_CHARACTER "std::char::REPLACEMENT_CHARACTER" + /// + /// # Examples + /// + /// Calling `to_string_lossy` on a `CStr` containing valid UTF-8: + /// + /// ``` + /// use std::borrow::Cow; + /// use std::ffi::CStr; + /// + /// let cstr = CStr::from_bytes_with_nul(b"Hello World\0") + /// .expect("CStr::from_bytes_with_nul failed"); + /// assert_eq!(cstr.to_string_lossy(), Cow::Borrowed("Hello World")); + /// ``` + /// + /// Calling `to_string_lossy` on a `CStr` containing invalid UTF-8: + /// + /// ``` + /// use std::borrow::Cow; + /// use std::ffi::CStr; + /// + /// let cstr = CStr::from_bytes_with_nul(b"Hello \xF0\x90\x80World\0") + /// .expect("CStr::from_bytes_with_nul failed"); + /// assert_eq!( + /// cstr.to_string_lossy(), + /// Cow::Owned(String::from("Hello �World")) as Cow<'_, str> + /// ); + /// ``` + #[rustc_allow_incoherent_impl] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[stable(feature = "cstr_to_str", since = "1.4.0")] + pub fn to_string_lossy(&self) -> Cow<'_, str> { + String::from_utf8_lossy(self.to_bytes()) + } + + /// Converts a [Box]<[CStr]> into a [`CString`] without copying or allocating. + /// + /// # Examples + /// + /// ``` + /// use std::ffi::CString; + /// + /// let c_string = CString::new(b"foo".to_vec()).expect("CString::new failed"); + /// let boxed = c_string.into_boxed_c_str(); + /// assert_eq!(boxed.into_c_string(), CString::new("foo").expect("CString::new failed")); + /// ``` + #[rustc_allow_incoherent_impl] + #[must_use = "`self` will be dropped if the result is not used"] + #[stable(feature = "into_boxed_c_str", since = "1.20.0")] + pub fn into_c_string(self: Box) -> CString { + CString::from(self) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl core::error::Error for NulError { + #[allow(deprecated)] + fn description(&self) -> &str { + "nul byte found in data" + } +} + +#[stable(feature = "cstring_from_vec_with_nul", since = "1.58.0")] +impl core::error::Error for FromVecWithNulError {} + +#[stable(feature = "cstring_into", since = "1.7.0")] +impl core::error::Error for IntoStringError { + #[allow(deprecated)] + fn description(&self) -> &str { + "C string contained non-utf8 bytes" + } + + fn source(&self) -> Option<&(dyn core::error::Error + 'static)> { + Some(self.__source()) + } +} diff --git a/crux-mir/lib/alloc/src/ffi/c_str/tests.rs b/crux-mir/lib/alloc/src/ffi/c_str/tests.rs new file mode 100644 index 000000000..0b7476d5c --- /dev/null +++ b/crux-mir/lib/alloc/src/ffi/c_str/tests.rs @@ -0,0 +1,228 @@ +use super::*; +use crate::rc::Rc; +use crate::sync::Arc; +use core::assert_matches::assert_matches; +use core::ffi::FromBytesUntilNulError; +use core::hash::{Hash, Hasher}; + +#[allow(deprecated)] +use core::hash::SipHasher13 as DefaultHasher; + +#[test] +fn c_to_rust() { + let data = b"123\0"; + let ptr = data.as_ptr() as *const c_char; + unsafe { + assert_eq!(CStr::from_ptr(ptr).to_bytes(), b"123"); + assert_eq!(CStr::from_ptr(ptr).to_bytes_with_nul(), b"123\0"); + } +} + +#[test] +fn simple() { + let s = CString::new("1234").unwrap(); + assert_eq!(s.as_bytes(), b"1234"); + assert_eq!(s.as_bytes_with_nul(), b"1234\0"); +} + +#[test] +fn build_with_zero1() { + assert!(CString::new(&b"\0"[..]).is_err()); +} +#[test] +fn build_with_zero2() { + assert!(CString::new(vec![0]).is_err()); +} + +#[test] +fn formatted() { + let s = CString::new(&b"abc\x01\x02\n\xE2\x80\xA6\xFF"[..]).unwrap(); + assert_eq!(format!("{s:?}"), r#""abc\x01\x02\n\xe2\x80\xa6\xff""#); +} + +#[test] +fn borrowed() { + unsafe { + let s = CStr::from_ptr(b"12\0".as_ptr() as *const _); + assert_eq!(s.to_bytes(), b"12"); + assert_eq!(s.to_bytes_with_nul(), b"12\0"); + } +} + +#[test] +fn to_owned() { + let data = b"123\0"; + let ptr = data.as_ptr() as *const c_char; + + let owned = unsafe { CStr::from_ptr(ptr).to_owned() }; + assert_eq!(owned.as_bytes_with_nul(), data); +} + +#[test] +fn equal_hash() { + let data = b"123\xE2\xFA\xA6\0"; + let ptr = data.as_ptr() as *const c_char; + let cstr: &'static CStr = unsafe { CStr::from_ptr(ptr) }; + + #[allow(deprecated)] + let mut s = DefaultHasher::new(); + cstr.hash(&mut s); + let cstr_hash = s.finish(); + #[allow(deprecated)] + let mut s = DefaultHasher::new(); + CString::new(&data[..data.len() - 1]).unwrap().hash(&mut s); + let cstring_hash = s.finish(); + + assert_eq!(cstr_hash, cstring_hash); +} + +#[test] +fn from_bytes_with_nul() { + let data = b"123\0"; + let cstr = CStr::from_bytes_with_nul(data); + assert_eq!(cstr.map(CStr::to_bytes), Ok(&b"123"[..])); + let cstr = CStr::from_bytes_with_nul(data); + assert_eq!(cstr.map(CStr::to_bytes_with_nul), Ok(&b"123\0"[..])); + + unsafe { + let cstr = CStr::from_bytes_with_nul(data); + let cstr_unchecked = CStr::from_bytes_with_nul_unchecked(data); + assert_eq!(cstr, Ok(cstr_unchecked)); + } +} + +#[test] +fn from_bytes_with_nul_unterminated() { + let data = b"123"; + let cstr = CStr::from_bytes_with_nul(data); + assert!(cstr.is_err()); +} + +#[test] +fn from_bytes_with_nul_interior() { + let data = b"1\023\0"; + let cstr = CStr::from_bytes_with_nul(data); + assert!(cstr.is_err()); +} + +#[test] +fn cstr_from_bytes_until_nul() { + // Test an empty slice. This should fail because it + // does not contain a nul byte. + let b = b""; + assert_matches!(CStr::from_bytes_until_nul(&b[..]), Err(FromBytesUntilNulError { .. })); + + // Test a non-empty slice, that does not contain a nul byte. + let b = b"hello"; + assert_matches!(CStr::from_bytes_until_nul(&b[..]), Err(FromBytesUntilNulError { .. })); + + // Test an empty nul-terminated string + let b = b"\0"; + let r = CStr::from_bytes_until_nul(&b[..]).unwrap(); + assert_eq!(r.to_bytes(), b""); + + // Test a slice with the nul byte in the middle + let b = b"hello\0world!"; + let r = CStr::from_bytes_until_nul(&b[..]).unwrap(); + assert_eq!(r.to_bytes(), b"hello"); + + // Test a slice with the nul byte at the end + let b = b"hello\0"; + let r = CStr::from_bytes_until_nul(&b[..]).unwrap(); + assert_eq!(r.to_bytes(), b"hello"); + + // Test a slice with two nul bytes at the end + let b = b"hello\0\0"; + let r = CStr::from_bytes_until_nul(&b[..]).unwrap(); + assert_eq!(r.to_bytes(), b"hello"); + + // Test a slice containing lots of nul bytes + let b = b"\0\0\0\0"; + let r = CStr::from_bytes_until_nul(&b[..]).unwrap(); + assert_eq!(r.to_bytes(), b""); +} + +#[test] +fn into_boxed() { + let orig: &[u8] = b"Hello, world!\0"; + let cstr = CStr::from_bytes_with_nul(orig).unwrap(); + let boxed: Box = Box::from(cstr); + let cstring = cstr.to_owned().into_boxed_c_str().into_c_string(); + assert_eq!(cstr, &*boxed); + assert_eq!(&*boxed, &*cstring); + assert_eq!(&*cstring, cstr); +} + +#[test] +fn boxed_default() { + let boxed = >::default(); + assert_eq!(boxed.to_bytes_with_nul(), &[0]); +} + +#[test] +fn test_c_str_clone_into() { + let mut c_string = CString::new("lorem").unwrap(); + let c_ptr = c_string.as_ptr(); + let c_str = CStr::from_bytes_with_nul(b"ipsum\0").unwrap(); + c_str.clone_into(&mut c_string); + assert_eq!(c_str, c_string.as_c_str()); + // The exact same size shouldn't have needed to move its allocation + assert_eq!(c_ptr, c_string.as_ptr()); +} + +#[test] +fn into_rc() { + let orig: &[u8] = b"Hello, world!\0"; + let cstr = CStr::from_bytes_with_nul(orig).unwrap(); + let rc: Rc = Rc::from(cstr); + let arc: Arc = Arc::from(cstr); + + assert_eq!(&*rc, cstr); + assert_eq!(&*arc, cstr); + + let rc2: Rc = Rc::from(cstr.to_owned()); + let arc2: Arc = Arc::from(cstr.to_owned()); + + assert_eq!(&*rc2, cstr); + assert_eq!(&*arc2, cstr); +} + +#[test] +fn cstr_const_constructor() { + const CSTR: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"Hello, world!\0") }; + + assert_eq!(CSTR.to_str().unwrap(), "Hello, world!"); +} + +#[test] +fn cstr_index_from() { + let original = b"Hello, world!\0"; + let cstr = CStr::from_bytes_with_nul(original).unwrap(); + let result = CStr::from_bytes_with_nul(&original[7..]).unwrap(); + + assert_eq!(&cstr[7..], result); +} + +#[test] +#[should_panic] +fn cstr_index_from_empty() { + let original = b"Hello, world!\0"; + let cstr = CStr::from_bytes_with_nul(original).unwrap(); + let _ = &cstr[original.len()..]; +} + +#[test] +fn c_string_from_empty_string() { + let original = ""; + let cstring = CString::new(original).unwrap(); + assert_eq!(original.as_bytes(), cstring.as_bytes()); + assert_eq!([b'\0'], cstring.as_bytes_with_nul()); +} + +#[test] +fn c_str_from_empty_string() { + let original = b"\0"; + let cstr = CStr::from_bytes_with_nul(original).unwrap(); + assert_eq!([] as [u8; 0], cstr.to_bytes()); + assert_eq!([b'\0'], cstr.to_bytes_with_nul()); +} diff --git a/crux-mir/lib/alloc/src/ffi/mod.rs b/crux-mir/lib/alloc/src/ffi/mod.rs new file mode 100644 index 000000000..e8530fbc1 --- /dev/null +++ b/crux-mir/lib/alloc/src/ffi/mod.rs @@ -0,0 +1,88 @@ +//! Utilities related to FFI bindings. +//! +//! This module provides utilities to handle data across non-Rust +//! interfaces, like other programming languages and the underlying +//! operating system. It is mainly of use for FFI (Foreign Function +//! Interface) bindings and code that needs to exchange C-like strings +//! with other languages. +//! +//! # Overview +//! +//! Rust represents owned strings with the [`String`] type, and +//! borrowed slices of strings with the [`str`] primitive. Both are +//! always in UTF-8 encoding, and may contain nul bytes in the middle, +//! i.e., if you look at the bytes that make up the string, there may +//! be a `\0` among them. Both `String` and `str` store their length +//! explicitly; there are no nul terminators at the end of strings +//! like in C. +//! +//! C strings are different from Rust strings: +//! +//! * **Encodings** - Rust strings are UTF-8, but C strings may use +//! other encodings. If you are using a string from C, you should +//! check its encoding explicitly, rather than just assuming that it +//! is UTF-8 like you can do in Rust. +//! +//! * **Character size** - C strings may use `char` or `wchar_t`-sized +//! characters; please **note** that C's `char` is different from Rust's. +//! The C standard leaves the actual sizes of those types open to +//! interpretation, but defines different APIs for strings made up of +//! each character type. Rust strings are always UTF-8, so different +//! Unicode characters will be encoded in a variable number of bytes +//! each. The Rust type [`char`] represents a '[Unicode scalar +//! value]', which is similar to, but not the same as, a '[Unicode +//! code point]'. +//! +//! * **Nul terminators and implicit string lengths** - Often, C +//! strings are nul-terminated, i.e., they have a `\0` character at the +//! end. The length of a string buffer is not stored, but has to be +//! calculated; to compute the length of a string, C code must +//! manually call a function like `strlen()` for `char`-based strings, +//! or `wcslen()` for `wchar_t`-based ones. Those functions return +//! the number of characters in the string excluding the nul +//! terminator, so the buffer length is really `len+1` characters. +//! Rust strings don't have a nul terminator; their length is always +//! stored and does not need to be calculated. While in Rust +//! accessing a string's length is an *O*(1) operation (because the +//! length is stored); in C it is an *O*(*n*) operation because the +//! length needs to be computed by scanning the string for the nul +//! terminator. +//! +//! * **Internal nul characters** - When C strings have a nul +//! terminator character, this usually means that they cannot have nul +//! characters in the middle — a nul character would essentially +//! truncate the string. Rust strings *can* have nul characters in +//! the middle, because nul does not have to mark the end of the +//! string in Rust. +//! +//! # Representations of non-Rust strings +//! +//! [`CString`] and [`CStr`] are useful when you need to transfer +//! UTF-8 strings to and from languages with a C ABI, like Python. +//! +//! * **From Rust to C:** [`CString`] represents an owned, C-friendly +//! string: it is nul-terminated, and has no internal nul characters. +//! Rust code can create a [`CString`] out of a normal string (provided +//! that the string doesn't have nul characters in the middle), and +//! then use a variety of methods to obtain a raw \*mut [u8] that can +//! then be passed as an argument to functions which use the C +//! conventions for strings. +//! +//! * **From C to Rust:** [`CStr`] represents a borrowed C string; it +//! is what you would use to wrap a raw \*const [u8] that you got from +//! a C function. A [`CStr`] is guaranteed to be a nul-terminated array +//! of bytes. Once you have a [`CStr`], you can convert it to a Rust +//! &[str] if it's valid UTF-8, or lossily convert it by adding +//! replacement characters. +//! +//! [`String`]: crate::string::String +//! [`CStr`]: core::ffi::CStr + +#![stable(feature = "alloc_ffi", since = "1.64.0")] + +#[stable(feature = "alloc_c_string", since = "1.64.0")] +pub use self::c_str::FromVecWithNulError; +#[stable(feature = "alloc_c_string", since = "1.64.0")] +pub use self::c_str::{CString, IntoStringError, NulError}; + +mod c_str; diff --git a/crux-mir/lib/alloc/src/fmt.rs b/crux-mir/lib/alloc/src/fmt.rs index 13ef2f063..eadb35cb9 100644 --- a/crux-mir/lib/alloc/src/fmt.rs +++ b/crux-mir/lib/alloc/src/fmt.rs @@ -17,8 +17,14 @@ //! format!("The number is {}", 1); // => "The number is 1" //! format!("{:?}", (3, 4)); // => "(3, 4)" //! format!("{value}", value=4); // => "4" +//! let people = "Rustaceans"; +//! format!("Hello {people}!"); // => "Hello Rustaceans!" //! format!("{} {}", 1, 2); // => "1 2" //! format!("{:04}", 42); // => "0042" with leading zeros +//! format!("{:#?}", (100, 200)); // => "( +//! // 100, +//! // 200, +//! // )" //! ``` //! //! From these, you can see that the first argument is a format string. It is @@ -50,8 +56,8 @@ //! The internal iterator over the argument has not been advanced by the time //! the first `{}` is seen, so it prints the first argument. Then upon reaching //! the second `{}`, the iterator has advanced forward to the second argument. -//! Essentially, parameters which explicitly name their argument do not affect -//! parameters which do not name an argument in terms of positional specifiers. +//! Essentially, parameters that explicitly name their argument do not affect +//! parameters that do not name an argument in terms of positional specifiers. //! //! A format string is required to use all of its arguments, otherwise it is a //! compile-time error. You may refer to the same argument more than once in the @@ -60,7 +66,7 @@ //! ## Named parameters //! //! Rust itself does not have a Python-like equivalent of named parameters to a -//! function, but the [`format!`] macro is a syntax extension which allows it to +//! function, but the [`format!`] macro is a syntax extension that allows it to //! leverage named parameters. Named parameters are listed at the end of the //! argument list and have the syntax: //! @@ -68,7 +74,7 @@ //! identifier '=' expression //! ``` //! -//! For example, the following [`format!`] expressions all use named argument: +//! For example, the following [`format!`] expressions all use named arguments: //! //! ``` //! format!("{argument}", argument = "test"); // => "test" @@ -76,14 +82,27 @@ //! format!("{a} {c} {b}", a="a", b='b', c=3); // => "a 3 b" //! ``` //! +//! If a named parameter does not appear in the argument list, `format!` will +//! reference a variable with that name in the current scope. +//! +//! ``` +//! let argument = 2 + 2; +//! format!("{argument}"); // => "4" +//! +//! fn make_string(a: u32, b: &str) -> String { +//! format!("{b} {a}") +//! } +//! make_string(927, "label"); // => "label 927" +//! ``` +//! //! It is not valid to put positional parameters (those without names) after -//! arguments which have names. Like with positional parameters, it is not +//! arguments that have names. Like with positional parameters, it is not //! valid to provide named parameters that are unused by the format string. //! //! # Formatting Parameters //! //! Each argument being formatted can be transformed by a number of formatting -//! parameters (corresponding to `format_spec` in the syntax above). These +//! parameters (corresponding to `format_spec` in [the syntax](#syntax)). These //! parameters affect the string representation of what's being formatted. //! //! ## Width @@ -94,6 +113,8 @@ //! println!("Hello {:1$}!", "x", 5); //! println!("Hello {1:0$}!", 5, "x"); //! println!("Hello {:width$}!", "x", width = 5); +//! let width = 5; +//! println!("Hello {:width$}!", "x"); //! ``` //! //! This is a parameter for the "minimum width" that the format should take up. @@ -130,11 +151,11 @@ //! //! The default [fill/alignment](#fillalignment) for non-numerics is a space and //! left-aligned. The -//! defaults for numeric formatters is also a space but with right-alignment. If +//! default for numeric formatters is also a space character but with right-alignment. If //! the `0` flag (see below) is specified for numerics, then the implicit fill character is //! `0`. //! -//! Note that alignment may not be implemented by some types. In particular, it +//! Note that alignment might not be implemented by some types. In particular, it //! is not generally implemented for the `Debug` trait. A good way to ensure //! padding is applied is to format your input, then pad this resulting string //! to obtain your output: @@ -157,13 +178,12 @@ //! //! * `+` - This is intended for numeric types and indicates that the sign //! should always be printed. Positive signs are never printed by -//! default, and the negative sign is only printed by default for the -//! `Signed` trait. This flag indicates that the correct sign (`+` or `-`) -//! should always be printed. +//! default, and the negative sign is only printed by default for signed values. +//! This flag indicates that the correct sign (`+` or `-`) should always be printed. //! * `-` - Currently not used -//! * `#` - This flag is indicates that the "alternate" form of printing should +//! * `#` - This flag indicates that the "alternate" form of printing should //! be used. The alternate forms are: -//! * `#?` - pretty-print the [`Debug`] formatting +//! * `#?` - pretty-print the [`Debug`] formatting (adds linebreaks and indentation) //! * `#x` - precedes the argument with a `0x` //! * `#X` - precedes the argument with a `0x` //! * `#b` - precedes the argument with a `0b` @@ -173,9 +193,9 @@ //! like `{:08}` would yield `00000001` for the integer `1`, while the //! same format would yield `-0000001` for the integer `-1`. Notice that //! the negative version has one fewer zero than the positive version. -//! Note that padding zeroes are always placed after the sign (if any) +//! Note that padding zeros are always placed after the sign (if any) //! and before the digits. When used together with the `#` flag, a similar -//! rule applies: padding zeroes are inserted after the prefix but before +//! rule applies: padding zeros are inserted after the prefix but before //! the digits. The prefix is included in the total width. //! //! ## Precision @@ -201,10 +221,12 @@ //! //! 3. An asterisk `.*`: //! -//! `.*` means that this `{...}` is associated with *two* format inputs rather than one: the -//! first input holds the `usize` precision, and the second holds the value to print. Note that -//! in this case, if one uses the format string `{:.*}`, then the `` part refers -//! to the *value* to print, and the `precision` must come in the input preceding ``. +//! `.*` means that this `{...}` is associated with *two* format inputs rather than one: +//! - If a format string in the fashion of `{:.*}` is used, then the first input holds +//! the `usize` precision, and the second holds the value to print. +//! - If a format string in the fashion of `{:.*}` is used, then the `` part +//! refers to the value to print, and the `precision` is taken like it was specified with an +//! omitted positional parameter (`{}` instead of `{:}`). //! //! For example, the following calls all print the same thing `Hello x is 0.01000`: //! @@ -218,15 +240,19 @@ //! // Hello {arg 0 ("x")} is {arg 2 (0.01) with precision specified in arg 1 (5)} //! println!("Hello {0} is {2:.1$}", "x", 5, 0.01); //! -//! // Hello {next arg ("x")} is {second of next two args (0.01) with precision -//! // specified in first of next two args (5)} +//! // Hello {next arg -> arg 0 ("x")} is {second of next two args -> arg 2 (0.01) with precision +//! // specified in first of next two args -> arg 1 (5)} //! println!("Hello {} is {:.*}", "x", 5, 0.01); //! -//! // Hello {next arg ("x")} is {arg 2 (0.01) with precision -//! // specified in its predecessor (5)} +//! // Hello {arg 1 ("x")} is {arg 2 (0.01) with precision +//! // specified in next arg -> arg 0 (5)} +//! println!("Hello {1} is {2:.*}", 5, "x", 0.01); +//! +//! // Hello {next arg -> arg 0 ("x")} is {arg 2 (0.01) with precision +//! // specified in next arg -> arg 1 (5)} //! println!("Hello {} is {2:.*}", "x", 5, 0.01); //! -//! // Hello {next arg ("x")} is {arg "number" (0.01) with precision specified +//! // Hello {next arg -> arg 0 ("x")} is {arg "number" (0.01) with precision specified //! // in arg "prec" (5)} //! println!("Hello {} is {number:.prec$}", "x", prec = 5, number = 0.01); //! ``` @@ -239,7 +265,7 @@ //! println!("{}, `{name:>8.*}` has 3 right-aligned characters", "Hello", 3, name="1234.56"); //! ``` //! -//! print two significantly different things: +//! print three significantly different things: //! //! ```text //! Hello, `1234.560` has 3 fractional digits @@ -251,7 +277,7 @@ //! //! In some programming languages, the behavior of string formatting functions //! depends on the operating system's locale setting. The format functions -//! provided by Rust's standard library do not have any concept of locale, and +//! provided by Rust's standard library do not have any concept of locale and //! will produce the same results on all systems regardless of user //! configuration. //! @@ -282,21 +308,27 @@ //! `%`. The actual grammar for the formatting syntax is: //! //! ```text -//! format_string := [ maybe-format ] * -//! maybe-format := '{' '{' | '}' '}' | -//! format := '{' [ argument ] [ ':' format_spec ] '}' +//! format_string := text [ maybe_format text ] * +//! maybe_format := '{' '{' | '}' '}' | format +//! format := '{' [ argument ] [ ':' format_spec ] [ ws ] * '}' //! argument := integer | identifier //! -//! format_spec := [[fill]align][sign]['#']['0'][width]['.' precision][type] +//! format_spec := [[fill]align][sign]['#']['0'][width]['.' precision]type //! fill := character //! align := '<' | '^' | '>' //! sign := '+' | '-' //! width := count //! precision := count | '*' -//! type := identifier | '?' | '' +//! type := '' | '?' | 'x?' | 'X?' | identifier //! count := parameter | integer //! parameter := argument '$' //! ``` +//! In the above grammar, +//! - `text` must not contain any `'{'` or `'}'` characters, +//! - `ws` is any character for which [`char::is_whitespace`] returns `true`, has no semantic +//! meaning and is completely optional, +//! - `integer` is a decimal integer that may contain leading zeroes and must fit into an `usize` and +//! - `identifier` is an `IDENTIFIER_OR_KEYWORD` (not an `IDENTIFIER`) as defined by the [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html). //! //! # Formatting traits //! @@ -309,13 +341,13 @@ //! * `?` ⇒ [`Debug`] //! * `x?` ⇒ [`Debug`] with lower-case hexadecimal integers //! * `X?` ⇒ [`Debug`] with upper-case hexadecimal integers -//! * `o` ⇒ [`Octal`](trait.Octal.html) -//! * `x` ⇒ [`LowerHex`](trait.LowerHex.html) -//! * `X` ⇒ [`UpperHex`](trait.UpperHex.html) -//! * `p` ⇒ [`Pointer`](trait.Pointer.html) +//! * `o` ⇒ [`Octal`] +//! * `x` ⇒ [`LowerHex`] +//! * `X` ⇒ [`UpperHex`] +//! * `p` ⇒ [`Pointer`] //! * `b` ⇒ [`Binary`] -//! * `e` ⇒ [`LowerExp`](trait.LowerExp.html) -//! * `E` ⇒ [`UpperExp`](trait.UpperExp.html) +//! * `e` ⇒ [`LowerExp`] +//! * `E` ⇒ [`UpperExp`] //! //! What this means is that any type of argument which implements the //! [`fmt::Binary`][`Binary`] trait can then be formatted with `{:b}`. Implementations @@ -337,14 +369,14 @@ //! ``` //! //! Your type will be passed as `self` by-reference, and then the function -//! should emit output into the `f.buf` stream. It is up to each format trait -//! implementation to correctly adhere to the requested formatting parameters. -//! The values of these parameters will be listed in the fields of the +//! should emit output into the Formatter `f` which implements `fmt::Write`. It is up to each +//! format trait implementation to correctly adhere to the requested formatting parameters. +//! The values of these parameters can be accessed with methods of the //! [`Formatter`] struct. In order to help with this, the [`Formatter`] struct also //! provides some helper methods. //! //! Additionally, the return value of this function is [`fmt::Result`] which is a -//! type alias of [`Result`]`<(), `[`std::fmt::Error`]`>`. Formatting implementations +//! type alias of [Result]<(), [std::fmt::Error]>. Formatting implementations //! should ensure that they propagate errors from the [`Formatter`] (e.g., when //! calling [`write!`]). However, they should never return errors spuriously. That //! is, a formatting implementation must and may only return an error if the @@ -387,7 +419,7 @@ //! // documentation for details, and the function `pad` can be used //! // to pad strings. //! let decimals = f.precision().unwrap_or(3); -//! let string = format!("{:.*}", decimals, magnitude); +//! let string = format!("{magnitude:.decimals$}"); //! f.pad_integral(true, "", &string) //! } //! } @@ -395,9 +427,9 @@ //! fn main() { //! let myvector = Vector2D { x: 3, y: 4 }; //! -//! println!("{}", myvector); // => "(3, 4)" -//! println!("{:?}", myvector); // => "Vector2D {x: 3, y:4}" -//! println!("{:10.3b}", myvector); // => " 5.000" +//! println!("{myvector}"); // => "(3, 4)" +//! println!("{myvector:?}"); // => "Vector2D {x: 3, y:4}" +//! println!("{myvector:10.3b}"); // => " 5.000" //! } //! ``` //! @@ -428,7 +460,7 @@ //! //! ```ignore (only-for-syntax-highlight) //! format! // described above -//! write! // first argument is a &mut io::Write, the destination +//! write! // first argument is either a &mut io::Write or a &mut fmt::Write, the destination //! writeln! // same as write but appends a newline //! print! // the format string is printed to the standard output //! println! // same as print but appends a newline @@ -439,11 +471,11 @@ //! //! ### `write!` //! -//! This and [`writeln!`] are two macros which are used to emit the format string +//! [`write!`] and [`writeln!`] are two macros which are used to emit the format string //! to a specified stream. This is used to prevent intermediate allocations of //! format strings and instead directly write the output. Under the hood, this //! function is actually invoking the [`write_fmt`] function defined on the -//! [`std::io::Write`] trait. Example usage is: +//! [`std::io::Write`] and the [`std::fmt::Write`] trait. Example usage is: //! //! ``` //! # #![allow(unused_must_use)] @@ -470,7 +502,7 @@ //! //! ### `format_args!` //! -//! This is a curious macro which is used to safely pass around +//! [`format_args!`] is a curious macro used to safely pass around //! an opaque object describing the format string. This object //! does not require any heap allocations to create, and it only //! references information on the stack. Under the hood, all of @@ -486,7 +518,7 @@ //! write!(&mut some_writer, "{}", format_args!("print with a {}", "macro")); //! //! fn my_fmt_fn(args: fmt::Arguments) { -//! write!(&mut io::stdout(), "{}", args); +//! write!(&mut io::stdout(), "{args}"); //! } //! my_fmt_fn(format_args!(", or a {} too", "function")); //! ``` @@ -495,37 +527,27 @@ //! This structure can then be passed to the [`write`] and [`format`] functions //! inside this module in order to process the format string. //! The goal of this macro is to even further prevent intermediate allocations -//! when dealing formatting strings. +//! when dealing with formatting strings. //! //! For example, a logging library could use the standard formatting syntax, but //! it would internally pass around this structure until it has been determined //! where output should go to. //! -//! [`usize`]: ../../std/primitive.usize.html -//! [`isize`]: ../../std/primitive.isize.html -//! [`i8`]: ../../std/primitive.i8.html -//! [`Display`]: trait.Display.html -//! [`Binary`]: trait.Binary.html -//! [`fmt::Result`]: type.Result.html -//! [`Result`]: ../../std/result/enum.Result.html -//! [`std::fmt::Error`]: struct.Error.html -//! [`Formatter`]: struct.Formatter.html -//! [`write!`]: ../../std/macro.write.html -//! [`Debug`]: trait.Debug.html -//! [`format!`]: ../../std/macro.format.html -//! [`to_string`]: ../../std/string/trait.ToString.html -//! [`writeln!`]: ../../std/macro.writeln.html +//! [`fmt::Result`]: Result "fmt::Result" +//! [Result]: core::result::Result "std::result::Result" +//! [std::fmt::Error]: Error "fmt::Error" +//! [`write`]: write() "fmt::write" +//! [`to_string`]: crate::string::ToString::to_string "ToString::to_string" //! [`write_fmt`]: ../../std/io/trait.Write.html#method.write_fmt //! [`std::io::Write`]: ../../std/io/trait.Write.html -//! [`print!`]: ../../std/macro.print.html -//! [`println!`]: ../../std/macro.println.html -//! [`eprint!`]: ../../std/macro.eprint.html -//! [`eprintln!`]: ../../std/macro.eprintln.html -//! [`write!`]: ../../std/macro.write.html -//! [`format_args!`]: ../../std/macro.format_args.html -//! [`fmt::Arguments`]: struct.Arguments.html -//! [`write`]: fn.write.html -//! [`format`]: fn.format.html +//! [`std::fmt::Write`]: ../../std/fmt/trait.Write.html +//! [`print!`]: ../../std/macro.print.html "print!" +//! [`println!`]: ../../std/macro.println.html "println!" +//! [`eprint!`]: ../../std/macro.eprint.html "eprint!" +//! [`eprintln!`]: ../../std/macro.eprintln.html "eprintln!" +//! [`format_args!`]: ../../std/macro.format_args.html "format_args!" +//! [`fmt::Arguments`]: Arguments "fmt::Arguments" +//! [`format`]: format() "fmt::format" #![stable(feature = "rust1", since = "1.0.0")] @@ -550,6 +572,7 @@ pub use core::fmt::{LowerExp, UpperExp}; #[stable(feature = "rust1", since = "1.0.0")] pub use core::fmt::{LowerHex, Pointer, UpperHex}; +#[cfg(not(no_global_oom_handling))] use crate::string; /// The `format` function takes an [`Arguments`] struct and returns the resulting @@ -576,13 +599,19 @@ use crate::string; /// assert_eq!(s, "Hello, world!"); /// ``` /// -/// [`Arguments`]: struct.Arguments.html -/// [`format_args!`]: ../../std/macro.format_args.html -/// [`format!`]: ../../std/macro.format.html +/// [`format_args!`]: core::format_args +/// [`format!`]: crate::format +#[cfg(not(no_global_oom_handling))] +#[must_use] #[stable(feature = "rust1", since = "1.0.0")] +#[inline] pub fn format(args: Arguments<'_>) -> string::String { - let capacity = args.estimated_capacity(); - let mut output = string::String::with_capacity(capacity); - output.write_fmt(args).expect("a formatting trait implementation returned an error"); - output + fn format_inner(args: Arguments<'_>) -> string::String { + let capacity = args.estimated_capacity(); + let mut output = string::String::with_capacity(capacity); + output.write_fmt(args).expect("a formatting trait implementation returned an error"); + output + } + + args.as_str().map_or_else(|| format_inner(args), crate::borrow::ToOwned::to_owned) } diff --git a/crux-mir/lib/alloc/src/lib.rs b/crux-mir/lib/alloc/src/lib.rs index 89e0f61ea..ca75c3895 100644 --- a/crux-mir/lib/alloc/src/lib.rs +++ b/crux-mir/lib/alloc/src/lib.rs @@ -3,7 +3,7 @@ //! This library provides smart pointers and collections for managing //! heap-allocated values. //! -//! This library, like libcore, normally doesn’t need to be used directly +//! This library, like core, normally doesn’t need to be used directly //! since its contents are re-exported in the [`std` crate](../std/index.html). //! Crates that use the `#![no_std]` attribute however will typically //! not depend on `std`, so they’d use this crate instead. @@ -50,92 +50,176 @@ //! The [`alloc`](alloc/index.html) module defines the low-level interface to the //! default global allocator. It is not compatible with the libc allocator API. //! -//! [`Arc`]: sync/index.html -//! [`Box`]: boxed/index.html -//! [`Cell`]: ../core/cell/index.html -//! [`Rc`]: rc/index.html -//! [`RefCell`]: ../core/cell/index.html +//! [`Arc`]: sync +//! [`Box`]: boxed +//! [`Cell`]: core::cell +//! [`Rc`]: rc +//! [`RefCell`]: core::cell #![allow(unused_attributes)] #![stable(feature = "alloc", since = "1.36.0")] #![doc( - html_root_url = "https://doc.rust-lang.org/nightly/", html_playground_url = "https://play.rust-lang.org/", issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/", test(no_crate_inject, attr(allow(unused_variables), deny(warnings))) )] +#![doc(cfg_hide( + not(test), + not(any(test, bootstrap)), + any(not(feature = "miri-test-libstd"), test, doctest), + no_global_oom_handling, + not(no_global_oom_handling), + not(no_rc), + not(no_sync), + target_has_atomic = "ptr" +))] #![no_std] #![needs_allocator] +// To run alloc tests without x.py without ending up with two copies of alloc, Miri needs to be +// able to "empty" this crate. See . +// rustc itself never sets the feature, so this line has no affect there. +#![cfg(any(not(feature = "miri-test-libstd"), test, doctest))] +// +// Lints: +#![deny(unsafe_op_in_unsafe_fn)] +#![deny(fuzzy_provenance_casts)] #![warn(deprecated_in_future)] -#![warn(missing_docs)] #![warn(missing_debug_implementations)] -#![deny(intra_doc_link_resolution_failure)] // rustdoc is run without -D warnings +#![warn(missing_docs)] #![allow(explicit_outlives_requirements)] -#![allow(incomplete_features)] -#![cfg_attr(not(test), feature(generator_trait))] -#![cfg_attr(test, feature(test))] +// +// Library features: +#![feature(alloc_layout_extra)] #![feature(allocator_api)] -#![feature(allow_internal_unstable)] -#![feature(arbitrary_self_types)] -#![feature(box_into_raw_non_null)] -#![feature(box_patterns)] -#![feature(box_syntax)] -#![feature(cfg_sanitize)] -#![feature(cfg_target_has_atomic)] +#![feature(array_chunks)] +#![feature(array_into_iter_constructors)] +#![feature(array_methods)] +#![feature(array_windows)] +#![feature(assert_matches)] +#![feature(async_iterator)] #![feature(coerce_unsized)] -#![feature(const_generic_impls_guard)] -#![feature(const_generics)] -#![feature(const_in_array_repeat_expressions)] -#![feature(const_if_match)] -#![feature(cow_is_borrowed)] -#![feature(crucible_intrinsics)] -#![feature(dispatch_from_dyn)] +#![cfg_attr(not(no_global_oom_handling), feature(const_alloc_error))] +#![feature(const_box)] +#![cfg_attr(not(no_global_oom_handling), feature(const_btree_len))] +#![feature(const_cow_is_borrowed)] +#![feature(const_convert)] +#![feature(const_size_of_val)] +#![feature(const_align_of_val)] +#![feature(const_ptr_read)] +#![feature(const_maybe_uninit_zeroed)] +#![feature(const_maybe_uninit_write)] +#![feature(const_maybe_uninit_as_mut_ptr)] +#![feature(const_refs_to_cell)] #![feature(core_intrinsics)] -#![feature(container_error_extra)] -#![feature(dropck_eyepatch)] +#![feature(core_panic)] +#![feature(const_eval_select)] +#![feature(const_pin)] +#![feature(const_waker)] +#![feature(cstr_from_bytes_until_nul)] +#![feature(dispatch_from_dyn)] +#![feature(error_generic_member_access)] +#![feature(error_in_core)] #![feature(exact_size_is_empty)] +#![feature(extend_one)] #![feature(fmt_internals)] #![feature(fn_traits)] -#![feature(fundamental)] -#![feature(internal_uninit_const)] -#![feature(lang_items)] -#![feature(libc)] -#![feature(nll)] -#![feature(optin_builtin_traits)] +#![feature(hasher_prefixfree_extras)] +#![feature(inline_const)] +#![feature(inplace_iteration)] +#![cfg_attr(test, feature(is_sorted))] +#![feature(iter_advance_by)] +#![feature(iter_next_chunk)] +#![feature(iter_repeat_n)] +#![feature(layout_for_ptr)] +#![feature(maybe_uninit_slice)] +#![feature(maybe_uninit_uninit_array)] +#![feature(maybe_uninit_uninit_array_transpose)] +#![cfg_attr(test, feature(new_uninit))] +#![feature(nonnull_slice_from_raw_parts)] #![feature(pattern)] +#![feature(pointer_byte_offsets)] +#![feature(provide_any)] #![feature(ptr_internals)] -#![feature(ptr_offset_from)] -#![feature(rustc_attrs)] +#![feature(ptr_metadata)] +#![feature(ptr_sub_ptr)] #![feature(receiver_trait)] -#![feature(specialization)] -#![feature(staged_api)] -#![feature(std_internals)] +#![feature(saturating_int_impl)] +#![feature(set_ptr_value)] +#![feature(sized_type_properties)] +#![feature(slice_from_ptr_range)] +#![feature(slice_group_by)] +#![feature(slice_ptr_get)] +#![feature(slice_ptr_len)] +#![feature(slice_range)] #![feature(str_internals)] +#![feature(strict_provenance)] #![feature(trusted_len)] -#![feature(try_reserve)] -#![feature(unboxed_closures)] +#![feature(trusted_random_access)] +#![feature(try_trait_v2)] +#![feature(tuple_trait)] +#![feature(unchecked_math)] #![feature(unicode_internals)] #![feature(unsize)] -#![feature(unsized_locals)] +#![feature(utf8_chunks)] +#![feature(std_internals)] +// +// Language features: #![feature(allocator_internals)] -#![feature(slice_partition_dedup)] -#![feature(maybe_uninit_extra, maybe_uninit_slice)] -#![feature(alloc_layout_extra)] -#![feature(try_trait)] +#![feature(allow_internal_unstable)] #![feature(associated_type_bounds)] +#![feature(cfg_sanitize)] +#![feature(const_deref)] +#![feature(const_mut_refs)] +#![feature(const_ptr_write)] +#![feature(const_precise_live_drops)] +#![feature(const_trait_impl)] +#![feature(const_try)] +#![feature(dropck_eyepatch)] +#![feature(exclusive_range_pattern)] +#![feature(fundamental)] +#![cfg_attr(not(test), feature(generator_trait))] +#![feature(hashmap_internals)] +#![feature(lang_items)] +#![feature(min_specialization)] +#![feature(negative_impls)] +#![feature(never_type)] +#![feature(rustc_allow_const_fn_unstable)] +#![feature(rustc_attrs)] +#![feature(pointer_is_aligned)] +#![feature(slice_internals)] +#![feature(staged_api)] +#![feature(stmt_expr_attributes)] +#![cfg_attr(test, feature(test))] +#![feature(unboxed_closures)] +#![feature(unsized_fn_params)] +#![feature(c_unwind)] +#![feature(with_negative_coherence)] +#![cfg_attr(test, feature(panic_update_hook))] +// +// Rustdoc features: +#![feature(doc_cfg)] +#![feature(doc_cfg_hide)] +// Technically, this is a bug in rustdoc: rustdoc sees the documentation on `#[lang = slice_alloc]` +// blocks is for `&[T]`, which also has documentation using this feature in `core`, and gets mad +// that the feature-gate isn't enabled. Ideally, it wouldn't check for the feature gate for docs +// from other crates, but since this can only appear for lang items, it doesn't seem worth fixing. +#![feature(intra_doc_pointers)] // Allow testing this library - #[cfg(test)] #[macro_use] extern crate std; #[cfg(test)] extern crate test; +#[cfg(test)] +mod testing; // Module with internal macros used by other modules (needs to be included before other modules). #[macro_use] mod macros; +mod raw_vec; + // Heaps provided for low-level allocation strategies pub mod alloc; @@ -153,26 +237,41 @@ mod boxed { } pub mod borrow; pub mod collections; +#[cfg(all(not(no_rc), not(no_sync), not(no_global_oom_handling)))] +pub mod ffi; pub mod fmt; -pub mod prelude; -pub mod raw_vec; +#[cfg(not(no_rc))] pub mod rc; pub mod slice; pub mod str; pub mod string; -#[cfg(target_has_atomic = "ptr")] +#[cfg(all(not(no_rc), not(no_sync), target_has_atomic = "ptr"))] pub mod sync; +#[cfg(all(not(no_global_oom_handling), not(no_rc), not(no_sync), target_has_atomic = "ptr"))] +pub mod task; #[cfg(test)] mod tests; pub mod vec; -#[cfg(not(test))] -mod std { - pub use core::ops; // RangeFull -} - #[doc(hidden)] #[unstable(feature = "liballoc_internals", issue = "none", reason = "implementation detail")] pub mod __export { pub use core::format_args; } + +#[cfg(test)] +#[allow(dead_code)] // Not used in all configurations +pub(crate) mod test_helpers { + /// Copied from `std::test_helpers::test_rng`, since these tests rely on the + /// seed not being the same for every RNG invocation too. + pub(crate) fn test_rng() -> rand_xorshift::XorShiftRng { + use std::hash::{BuildHasher, Hash, Hasher}; + let mut hasher = std::collections::hash_map::RandomState::new().build_hasher(); + std::panic::Location::caller().hash(&mut hasher); + let hc64 = hasher.finish(); + let seed_vec = + hc64.to_le_bytes().into_iter().chain(0u8..8).collect::>(); + let seed: [u8; 16] = seed_vec.as_slice().try_into().unwrap(); + rand::SeedableRng::from_seed(seed) + } +} diff --git a/crux-mir/lib/alloc/src/macros.rs b/crux-mir/lib/alloc/src/macros.rs index 422d3486f..5198bf297 100644 --- a/crux-mir/lib/alloc/src/macros.rs +++ b/crux-mir/lib/alloc/src/macros.rs @@ -29,33 +29,46 @@ /// to the same boxed integer value, not five references pointing to independently /// boxed integers. /// -/// [`Vec`]: ../std/vec/struct.Vec.html -/// [`Clone`]: ../std/clone/trait.Clone.html -#[cfg(not(test))] +/// Also, note that `vec![expr; 0]` is allowed, and produces an empty vector. +/// This will still evaluate `expr`, however, and immediately drop the resulting value, so +/// be mindful of side effects. +/// +/// [`Vec`]: crate::vec::Vec +#[cfg(all(not(no_global_oom_handling), not(test)))] #[macro_export] #[stable(feature = "rust1", since = "1.0.0")] -#[allow_internal_unstable(box_syntax)] +#[rustc_diagnostic_item = "vec_macro"] +#[allow_internal_unstable(rustc_attrs, liballoc_internals)] macro_rules! vec { + () => ( + $crate::__rust_force_expr!($crate::vec::Vec::new()) + ); ($elem:expr; $n:expr) => ( - $crate::vec::from_elem($elem, $n) + $crate::__rust_force_expr!($crate::vec::from_elem($elem, $n)) ); - ($($x:expr),*) => ( - <[_]>::into_vec(box [$($x),*]) + ($($x:expr),+ $(,)?) => ( + $crate::__rust_force_expr!(<[_]>::into_vec( + #[rustc_box] + $crate::boxed::Box::new([$($x),+]) + )) ); - ($($x:expr,)*) => ($crate::vec![$($x),*]) } // HACK(japaric): with cfg(test) the inherent `[T]::into_vec` method, which is // required for this macro definition, is not available. Instead use the // `slice::into_vec` function which is only available with cfg(test) // NB see the slice::hack module in slice.rs for more information -#[cfg(test)] +#[cfg(all(not(no_global_oom_handling), test))] +#[allow(unused_macro_rules)] macro_rules! vec { + () => ( + $crate::vec::Vec::new() + ); ($elem:expr; $n:expr) => ( $crate::vec::from_elem($elem, $n) ); ($($x:expr),*) => ( - $crate::slice::into_vec(box [$($x),*]) + $crate::slice::into_vec($crate::boxed::Box::new([$($x),*])) ); ($($x:expr,)*) => (vec![$($x),*]) } @@ -67,7 +80,7 @@ macro_rules! vec { /// /// Additional parameters passed to `format!` replace the `{}`s within the /// formatting string in the order given unless named or positional parameters -/// are used; see [`std::fmt`][fmt] for more information. +/// are used; see [`std::fmt`] for more information. /// /// A common use for `format!` is concatenation and interpolation of strings. /// The same convention is used with [`print!`] and [`write!`] macros, @@ -76,11 +89,11 @@ macro_rules! vec { /// To convert a single value to a string, use the [`to_string`] method. This /// will use the [`Display`] formatting trait. /// -/// [fmt]: ../std/fmt/index.html +/// [`std::fmt`]: ../std/fmt/index.html /// [`print!`]: ../std/macro.print.html -/// [`write!`]: ../std/macro.write.html -/// [`to_string`]: ../std/string/trait.ToString.html -/// [`Display`]: ../std/fmt/trait.Display.html +/// [`write!`]: core::write +/// [`to_string`]: crate::string::ToString +/// [`Display`]: core::fmt::Display /// /// # Panics /// @@ -94,12 +107,25 @@ macro_rules! vec { /// format!("test"); /// format!("hello {}", "world!"); /// format!("x = {}, y = {y}", 10, y = 30); +/// let (x, y) = (1, 2); +/// format!("{x} + {y} = 3"); /// ``` #[macro_export] #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "format_macro")] macro_rules! format { ($($arg:tt)*) => {{ let res = $crate::fmt::format($crate::__export::format_args!($($arg)*)); res }} } + +/// Force AST node to an expression to improve diagnostics in pattern position. +#[doc(hidden)] +#[macro_export] +#[unstable(feature = "liballoc_internals", issue = "none", reason = "implementation detail")] +macro_rules! __rust_force_expr { + ($e:expr) => { + $e + }; +} diff --git a/crux-mir/lib/alloc/src/prelude/mod.rs b/crux-mir/lib/alloc/src/prelude/mod.rs deleted file mode 100644 index 0534ad3ed..000000000 --- a/crux-mir/lib/alloc/src/prelude/mod.rs +++ /dev/null @@ -1,15 +0,0 @@ -//! The alloc Prelude -//! -//! The purpose of this module is to alleviate imports of commonly-used -//! items of the `alloc` crate by adding a glob import to the top of modules: -//! -//! ``` -//! # #![allow(unused_imports)] -//! #![feature(alloc_prelude)] -//! extern crate alloc; -//! use alloc::prelude::v1::*; -//! ``` - -#![unstable(feature = "alloc_prelude", issue = "58935")] - -pub mod v1; diff --git a/crux-mir/lib/alloc/src/prelude/v1.rs b/crux-mir/lib/alloc/src/prelude/v1.rs deleted file mode 100644 index 6a53b4ca1..000000000 --- a/crux-mir/lib/alloc/src/prelude/v1.rs +++ /dev/null @@ -1,14 +0,0 @@ -//! The first version of the prelude of `alloc` crate. -//! -//! See the [module-level documentation](../index.html) for more. - -#![unstable(feature = "alloc_prelude", issue = "58935")] - -#[unstable(feature = "alloc_prelude", issue = "58935")] -pub use crate::borrow::ToOwned; -#[unstable(feature = "alloc_prelude", issue = "58935")] -pub use crate::boxed::Box; -#[unstable(feature = "alloc_prelude", issue = "58935")] -pub use crate::string::{String, ToString}; -#[unstable(feature = "alloc_prelude", issue = "58935")] -pub use crate::vec::Vec; diff --git a/crux-mir/lib/alloc/src/raw_vec.rs b/crux-mir/lib/alloc/src/raw_vec.rs index 3d6b41ddb..5a10121bb 100644 --- a/crux-mir/lib/alloc/src/raw_vec.rs +++ b/crux-mir/lib/alloc/src/raw_vec.rs @@ -1,113 +1,66 @@ -#![unstable(feature = "raw_vec_internals", reason = "implementation detail", issue = "none")] -#![doc(hidden)] +#![unstable(feature = "raw_vec_internals", reason = "unstable const warnings", issue = "none")] +use core::alloc::LayoutError; use core::cmp; -use core::mem; +use core::intrinsics; +use core::mem::{self, ManuallyDrop, MaybeUninit, SizedTypeProperties}; use core::ops::Drop; use core::ptr::{self, NonNull, Unique}; use core::slice; -use crucible; -use crate::alloc::{handle_alloc_error, AllocErr, AllocRef, Global, Layout}; +#[cfg(not(no_global_oom_handling))] +use crate::alloc::handle_alloc_error; +use crate::alloc::{Allocator, Global, Layout}; use crate::boxed::Box; -use crate::collections::TryReserveError::{self, *}; +use crate::collections::TryReserveError; +use crate::collections::TryReserveErrorKind::*; #[cfg(test)] mod tests; +#[cfg(not(no_global_oom_handling))] +enum AllocInit { + /// The contents of the new memory are uninitialized. + Uninitialized, + /// The new memory is guaranteed to be zeroed. + Zeroed, +} + /// A low-level utility for more ergonomically allocating, reallocating, and deallocating /// a buffer of memory on the heap without having to worry about all the corner cases /// involved. This type is excellent for building your own data structures like Vec and VecDeque. /// In particular: /// -/// * Produces `Unique::empty()` on zero-sized types. -/// * Produces `Unique::empty()` on zero-length allocations. +/// * Produces `Unique::dangling()` on zero-sized types. +/// * Produces `Unique::dangling()` on zero-length allocations. +/// * Avoids freeing `Unique::dangling()`. /// * Catches all overflows in capacity computations (promotes them to "capacity overflow" panics). /// * Guards against 32-bit systems allocating more than isize::MAX bytes. /// * Guards against overflowing your length. -/// * Aborts on OOM or calls `handle_alloc_error` as applicable. -/// * Avoids freeing `Unique::empty()`. +/// * Calls `handle_alloc_error` for fallible allocations. /// * Contains a `ptr::Unique` and thus endows the user with all related benefits. +/// * Uses the excess returned from the allocator to use the largest available capacity. /// /// This type does not in anyway inspect the memory that it manages. When dropped it *will* /// free its memory, but it *won't* try to drop its contents. It is up to the user of `RawVec` /// to handle the actual things *stored* inside of a `RawVec`. /// -/// Note that a `RawVec` always forces its capacity to be `usize::MAX` for zero-sized types. -/// This enables you to use capacity-growing logic catch the overflows in your length -/// that might occur with zero-sized types. -/// -/// The above means that you need to be careful when round-tripping this type with a -/// `Box<[T]>`, since `capacity()` won't yield the length. However, `with_capacity`, -/// `shrink_to_fit`, and `from_box` will actually set `RawVec`'s private capacity -/// field. This allows zero-sized types to not be special-cased by consumers of -/// this type. +/// Note that the excess of a zero-sized types is always infinite, so `capacity()` always returns +/// `usize::MAX`. This means that you need to be careful when round-tripping this type with a +/// `Box<[T]>`, since `capacity()` won't yield the length. #[allow(missing_debug_implementations)] -pub struct RawVec { +pub(crate) struct RawVec { ptr: Unique, cap: usize, - a: A, -} - -impl RawVec { - /// Like `new`, but parameterized over the choice of allocator for - /// the returned `RawVec`. - pub const fn new_in(a: A) -> Self { - let cap = if mem::size_of::() == 0 { core::usize::MAX } else { 0 }; - - // `Unique::empty()` doubles as "unallocated" and "zero-sized allocation". - RawVec { ptr: Unique::empty(), cap, a } - } - - /// Like `with_capacity`, but parameterized over the choice of - /// allocator for the returned `RawVec`. - #[inline] - pub fn with_capacity_in(capacity: usize, a: A) -> Self { - RawVec::allocate_in(capacity, false, a) - } - - /// Like `with_capacity_zeroed`, but parameterized over the choice - /// of allocator for the returned `RawVec`. - #[inline] - pub fn with_capacity_zeroed_in(capacity: usize, a: A) -> Self { - RawVec::allocate_in(capacity, true, a) - } - - fn allocate_in(capacity: usize, zeroed: bool, a: A) -> Self { - let elem_size = mem::size_of::(); - - let alloc_size = capacity.checked_mul(elem_size).unwrap_or_else(|| capacity_overflow()); - alloc_guard(alloc_size).unwrap_or_else(|_| capacity_overflow()); - - // Handles ZSTs and `capacity == 0` alike. - let ptr = if alloc_size == 0 { - NonNull::::dangling() - } else { - // NB: This ignores the choice of allocator - let ptr = if zeroed { - crucible::alloc::allocate_zeroed::(capacity) - } else { - crucible::alloc::allocate::(capacity) - }; - unsafe { NonNull::new_unchecked(ptr) } - }; - - RawVec { ptr: ptr.into(), cap: capacity, a } - } + alloc: A, } impl RawVec { - /// HACK(Centril): This exists because `#[unstable]` `const fn`s needn't conform - /// to `min_const_fn` and so they cannot be called in `min_const_fn`s either. + /// HACK(Centril): This exists because stable `const fn` can only call stable `const fn`, so + /// they cannot call `Self::new()`. /// - /// If you change `RawVec::new` or dependencies, please take care to not - /// introduce anything that would truly violate `min_const_fn`. - /// - /// NOTE: We could avoid this hack and check conformance with some - /// `#[rustc_force_min_const_fn]` attribute which requires conformance - /// with `min_const_fn` but does not necessarily allow calling it in - /// `stable(...) const fn` / user code not enabling `foo` when - /// `#[rustc_const_unstable(feature = "foo", ..)]` is present. + /// If you change `RawVec::new` or dependencies, please take care to not introduce anything + /// that would truly const-call something unstable. pub const NEW: Self = Self::new(); /// Creates the biggest possible `RawVec` (on the system heap) @@ -115,6 +68,7 @@ impl RawVec { /// `RawVec` with capacity `0`. If `T` is zero-sized, then it makes a /// `RawVec` with capacity `usize::MAX`. Useful for implementing /// delayed allocation. + #[must_use] pub const fn new() -> Self { Self::new_in(Global) } @@ -127,64 +81,145 @@ impl RawVec { /// /// # Panics /// - /// * Panics if the requested capacity exceeds `usize::MAX` bytes. - /// * Panics on 32-bit platforms if the requested capacity exceeds - /// `isize::MAX` bytes. + /// Panics if the requested capacity exceeds `isize::MAX` bytes. /// /// # Aborts /// /// Aborts on OOM. + #[cfg(not(any(no_global_oom_handling, test)))] + #[must_use] #[inline] pub fn with_capacity(capacity: usize) -> Self { - RawVec::allocate_in(capacity, false, Global) + Self::with_capacity_in(capacity, Global) } /// Like `with_capacity`, but guarantees the buffer is zeroed. + #[cfg(not(any(no_global_oom_handling, test)))] + #[must_use] #[inline] pub fn with_capacity_zeroed(capacity: usize) -> Self { - RawVec::allocate_in(capacity, true, Global) + Self::with_capacity_zeroed_in(capacity, Global) } } -impl RawVec { - /// Reconstitutes a `RawVec` from a pointer, capacity, and allocator. - /// - /// # Undefined Behavior - /// - /// The `ptr` must be allocated (via the given allocator `a`), and with the given `capacity`. - /// The `capacity` cannot exceed `isize::MAX` (only a concern on 32-bit systems). - /// If the `ptr` and `capacity` come from a `RawVec` created via `a`, then this is guaranteed. - pub unsafe fn from_raw_parts_in(ptr: *mut T, capacity: usize, a: A) -> Self { - RawVec { ptr: Unique::new_unchecked(ptr), cap: capacity, a } +impl RawVec { + // Tiny Vecs are dumb. Skip to: + // - 8 if the element size is 1, because any heap allocators is likely + // to round up a request of less than 8 bytes to at least 8 bytes. + // - 4 if elements are moderate-sized (<= 1 KiB). + // - 1 otherwise, to avoid wasting too much space for very short Vecs. + pub(crate) const MIN_NON_ZERO_CAP: usize = if mem::size_of::() == 1 { + 8 + } else if mem::size_of::() <= 1024 { + 4 + } else { + 1 + }; + + /// Like `new`, but parameterized over the choice of allocator for + /// the returned `RawVec`. + pub const fn new_in(alloc: A) -> Self { + // `cap: 0` means "unallocated". zero-sized types are ignored. + Self { ptr: Unique::dangling(), cap: 0, alloc } } -} -impl RawVec { - /// Reconstitutes a `RawVec` from a pointer and capacity. + /// Like `with_capacity`, but parameterized over the choice of + /// allocator for the returned `RawVec`. + #[cfg(not(no_global_oom_handling))] + #[inline] + pub fn with_capacity_in(capacity: usize, alloc: A) -> Self { + Self::allocate_in(capacity, AllocInit::Uninitialized, alloc) + } + + /// Like `with_capacity_zeroed`, but parameterized over the choice + /// of allocator for the returned `RawVec`. + #[cfg(not(no_global_oom_handling))] + #[inline] + pub fn with_capacity_zeroed_in(capacity: usize, alloc: A) -> Self { + Self::allocate_in(capacity, AllocInit::Zeroed, alloc) + } + + /// Converts the entire buffer into `Box<[MaybeUninit]>` with the specified `len`. /// - /// # Undefined Behavior + /// Note that this will correctly reconstitute any `cap` changes + /// that may have been performed. (See description of type for details.) /// - /// The `ptr` must be allocated (on the system heap), and with the given `capacity`. - /// The `capacity` cannot exceed `isize::MAX` (only a concern on 32-bit systems). - /// If the `ptr` and `capacity` come from a `RawVec`, then this is guaranteed. - pub unsafe fn from_raw_parts(ptr: *mut T, capacity: usize) -> Self { - RawVec { ptr: Unique::new_unchecked(ptr), cap: capacity, a: Global } - } + /// # Safety + /// + /// * `len` must be greater than or equal to the most recently requested capacity, and + /// * `len` must be less than or equal to `self.capacity()`. + /// + /// Note, that the requested capacity and `self.capacity()` could differ, as + /// an allocator could overallocate and return a greater memory block than requested. + pub unsafe fn into_box(self, len: usize) -> Box<[MaybeUninit], A> { + // Sanity-check one half of the safety requirement (we cannot check the other half). + debug_assert!( + len <= self.capacity(), + "`len` must be smaller than or equal to `self.capacity()`" + ); - /// Converts a `Box<[T]>` into a `RawVec`. - pub fn from_box(mut slice: Box<[T]>) -> Self { + let me = ManuallyDrop::new(self); unsafe { - let result = RawVec::from_raw_parts(slice.as_mut_ptr(), slice.len()); - mem::forget(slice); - result + let slice = slice::from_raw_parts_mut(me.ptr() as *mut MaybeUninit, len); + Box::from_raw_in(slice, ptr::read(&me.alloc)) } } -} -impl RawVec { + #[cfg(not(no_global_oom_handling))] + fn allocate_in(capacity: usize, init: AllocInit, alloc: A) -> Self { + // Don't allocate here because `Drop` will not deallocate when `capacity` is 0. + if T::IS_ZST || capacity == 0 { + Self::new_in(alloc) + } else { + // We avoid `unwrap_or_else` here because it bloats the amount of + // LLVM IR generated. + let layout = match Layout::array::(capacity) { + Ok(layout) => layout, + Err(_) => capacity_overflow(), + }; + match alloc_guard(layout.size()) { + Ok(_) => {} + Err(_) => capacity_overflow(), + } + let result = match init { + AllocInit::Uninitialized => alloc.allocate(layout), + AllocInit::Zeroed => alloc.allocate_zeroed(layout), + }; + let ptr = match result { + Ok(ptr) => ptr, + Err(_) => handle_alloc_error(layout), + }; + + // Allocators currently return a `NonNull<[u8]>` whose length + // matches the size requested. If that ever changes, the capacity + // here should change to `ptr.len() / mem::size_of::()`. + Self { + ptr: unsafe { Unique::new_unchecked(ptr.cast().as_ptr()) }, + cap: capacity, + alloc, + } + } + } + + /// Reconstitutes a `RawVec` from a pointer, capacity, and allocator. + /// + /// # Safety + /// + /// The `ptr` must be allocated (via the given allocator `alloc`), and with the given + /// `capacity`. + /// The `capacity` cannot exceed `isize::MAX` for sized types. (only a concern on 32-bit + /// systems). ZST vectors may have a capacity up to `usize::MAX`. + /// If the `ptr` and `capacity` come from a `RawVec` created via `alloc`, then this is + /// guaranteed. + #[inline] + pub unsafe fn from_raw_parts_in(ptr: *mut T, capacity: usize, alloc: A) -> Self { + Self { ptr: unsafe { Unique::new_unchecked(ptr) }, cap: capacity, alloc } + } + /// Gets a raw pointer to the start of the allocation. Note that this is - /// `Unique::empty()` if `capacity == 0` or `T` is zero-sized. In the former case, you must + /// `Unique::dangling()` if `capacity == 0` or `T` is zero-sized. In the former case, you must /// be careful. + #[inline] pub fn ptr(&self) -> *mut T { self.ptr.as_ptr() } @@ -194,294 +229,116 @@ impl RawVec { /// This will always be `usize::MAX` if `T` is zero-sized. #[inline(always)] pub fn capacity(&self) -> usize { - if mem::size_of::() == 0 { !0 } else { self.cap } + if T::IS_ZST { usize::MAX } else { self.cap } } /// Returns a shared reference to the allocator backing this `RawVec`. - pub fn alloc(&self) -> &A { - &self.a - } - - /// Returns a mutable reference to the allocator backing this `RawVec`. - pub fn alloc_mut(&mut self) -> &mut A { - &mut self.a + pub fn allocator(&self) -> &A { + &self.alloc } - fn current_layout(&self) -> Option { - if self.cap == 0 { + fn current_memory(&self) -> Option<(NonNull, Layout)> { + if T::IS_ZST || self.cap == 0 { None } else { // We have an allocated chunk of memory, so we can bypass runtime // checks to get our current layout. unsafe { - let align = mem::align_of::(); - let size = mem::size_of::() * self.cap; - Some(Layout::from_size_align_unchecked(size, align)) + let layout = Layout::array::(self.cap).unwrap_unchecked(); + Some((self.ptr.cast().into(), layout)) } } } - /// Doubles the size of the type's backing allocation. This is common enough - /// to want to do that it's easiest to just have a dedicated method. Slightly - /// more efficient logic can be provided for this than the general case. + /// Ensures that the buffer contains at least enough space to hold `len + + /// additional` elements. If it doesn't already have enough capacity, will + /// reallocate enough space plus comfortable slack space to get amortized + /// *O*(1) behavior. Will limit this behavior if it would needlessly cause + /// itself to panic. /// - /// This function is ideal for when pushing elements one-at-a-time because - /// you don't need to incur the costs of the more general computations - /// reserve needs to do to guard against overflow. You do however need to - /// manually check if your `len == capacity`. - /// - /// # Panics - /// - /// * Panics if `T` is zero-sized on the assumption that you managed to exhaust - /// all `usize::MAX` slots in your imaginary buffer. - /// * Panics on 32-bit platforms if the requested capacity exceeds - /// `isize::MAX` bytes. - /// - /// # Aborts - /// - /// Aborts on OOM - /// - /// # Examples - /// - /// ``` - /// # #![feature(raw_vec_internals)] - /// # extern crate alloc; - /// # use std::ptr; - /// # use alloc::raw_vec::RawVec; - /// struct MyVec { - /// buf: RawVec, - /// len: usize, - /// } - /// - /// impl MyVec { - /// pub fn push(&mut self, elem: T) { - /// if self.len == self.buf.capacity() { self.buf.double(); } - /// // double would have aborted or panicked if the len exceeded - /// // `isize::MAX` so this is safe to do unchecked now. - /// unsafe { - /// ptr::write(self.buf.ptr().add(self.len), elem); - /// } - /// self.len += 1; - /// } - /// } - /// # fn main() { - /// # let mut vec = MyVec { buf: RawVec::new(), len: 0 }; - /// # vec.push(1); - /// # } - /// ``` - #[inline(never)] - #[cold] - pub fn double(&mut self) { - unsafe { - if self.cap == 0 { - self.cap = 4; - self.ptr = NonNull::new_unchecked(crucible::alloc::allocate(self.cap)).into(); - } else { - self.cap *= 2; - crucible::alloc::reallocate(self.ptr.as_ptr(), self.cap); - } - } - } - - /// Attempts to double the size of the type's backing allocation in place. This is common - /// enough to want to do that it's easiest to just have a dedicated method. Slightly - /// more efficient logic can be provided for this than the general case. - /// - /// Returns `true` if the reallocation attempt has succeeded. - /// - /// # Panics - /// - /// * Panics if `T` is zero-sized on the assumption that you managed to exhaust - /// all `usize::MAX` slots in your imaginary buffer. - /// * Panics on 32-bit platforms if the requested capacity exceeds - /// `isize::MAX` bytes. - #[inline(never)] - #[cold] - pub fn double_in_place(&mut self) -> bool { - unsafe { - if self.cap == 0 { - // Can't increase capacity from zero without allocating and changing `self.ptr`. - return false; - } - - // `crucible::alloc::reallocate` always reallocs in-place. - self.cap *= 2; - crucible::alloc::reallocate(self.ptr.as_ptr(), self.cap); - true - } - } - - /// The same as `reserve_exact`, but returns on errors instead of panicking or aborting. - pub fn try_reserve_exact( - &mut self, - used_capacity: usize, - needed_extra_capacity: usize, - ) -> Result<(), TryReserveError> { - self.reserve_internal(used_capacity, needed_extra_capacity, Fallible, Exact) - } - - /// Ensures that the buffer contains at least enough space to hold - /// `used_capacity + needed_extra_capacity` elements. If it doesn't already, - /// will reallocate the minimum possible amount of memory necessary. - /// Generally this will be exactly the amount of memory necessary, - /// but in principle the allocator is free to give back more than - /// we asked for. - /// - /// If `used_capacity` exceeds `self.capacity()`, this may fail to actually allocate + /// If `len` exceeds `self.capacity()`, this may fail to actually allocate /// the requested space. This is not really unsafe, but the unsafe /// code *you* write that relies on the behavior of this function may break. /// + /// This is ideal for implementing a bulk-push operation like `extend`. + /// /// # Panics /// - /// * Panics if the requested capacity exceeds `usize::MAX` bytes. - /// * Panics on 32-bit platforms if the requested capacity exceeds - /// `isize::MAX` bytes. + /// Panics if the new capacity exceeds `isize::MAX` bytes. /// /// # Aborts /// /// Aborts on OOM. - pub fn reserve_exact(&mut self, used_capacity: usize, needed_extra_capacity: usize) { - match self.reserve_internal(used_capacity, needed_extra_capacity, Infallible, Exact) { - Err(CapacityOverflow) => capacity_overflow(), - Err(AllocError { .. }) => unreachable!(), - Ok(()) => { /* yay */ } + #[cfg(not(no_global_oom_handling))] + #[inline] + pub fn reserve(&mut self, len: usize, additional: usize) { + // Callers expect this function to be very cheap when there is already sufficient capacity. + // Therefore, we move all the resizing and error-handling logic from grow_amortized and + // handle_reserve behind a call, while making sure that this function is likely to be + // inlined as just a comparison and a call if the comparison fails. + #[cold] + fn do_reserve_and_handle( + slf: &mut RawVec, + len: usize, + additional: usize, + ) { + handle_reserve(slf.grow_amortized(len, additional)); + } + + if self.needs_to_grow(len, additional) { + do_reserve_and_handle(self, len, additional); } } - /// Calculates the buffer's new size given that it'll hold `used_capacity + - /// needed_extra_capacity` elements. This logic is used in amortized reserve methods. - /// Returns `(new_capacity, new_alloc_size)`. - fn amortized_new_size( - &self, - used_capacity: usize, - needed_extra_capacity: usize, - ) -> Result { - // Nothing we can really do about these checks, sadly. - let required_cap = - used_capacity.checked_add(needed_extra_capacity).ok_or(CapacityOverflow)?; - // Cannot overflow, because `cap <= isize::MAX`, and type of `cap` is `usize`. - let double_cap = self.cap * 2; - // `double_cap` guarantees exponential growth. - Ok(cmp::max(double_cap, required_cap)) + /// A specialized version of `reserve()` used only by the hot and + /// oft-instantiated `Vec::push()`, which does its own capacity check. + #[cfg(not(no_global_oom_handling))] + #[inline(never)] + pub fn reserve_for_push(&mut self, len: usize) { + handle_reserve(self.grow_amortized(len, 1)); } /// The same as `reserve`, but returns on errors instead of panicking or aborting. - pub fn try_reserve( - &mut self, - used_capacity: usize, - needed_extra_capacity: usize, - ) -> Result<(), TryReserveError> { - self.reserve_internal(used_capacity, needed_extra_capacity, Fallible, Amortized) + pub fn try_reserve(&mut self, len: usize, additional: usize) -> Result<(), TryReserveError> { + if self.needs_to_grow(len, additional) { + self.grow_amortized(len, additional) + } else { + Ok(()) + } } - /// Ensures that the buffer contains at least enough space to hold - /// `used_capacity + needed_extra_capacity` elements. If it doesn't already have - /// enough capacity, will reallocate enough space plus comfortable slack - /// space to get amortized `O(1)` behavior. Will limit this behavior - /// if it would needlessly cause itself to panic. + /// Ensures that the buffer contains at least enough space to hold `len + + /// additional` elements. If it doesn't already, will reallocate the + /// minimum possible amount of memory necessary. Generally this will be + /// exactly the amount of memory necessary, but in principle the allocator + /// is free to give back more than we asked for. /// - /// If `used_capacity` exceeds `self.capacity()`, this may fail to actually allocate - /// the requested space. This is not really unsafe, but the unsafe - /// code *you* write that relies on the behavior of this function may break. - /// - /// This is ideal for implementing a bulk-push operation like `extend`. + /// If `len` exceeds `self.capacity()`, this may fail to actually allocate + /// the requested space. This is not really unsafe, but the unsafe code + /// *you* write that relies on the behavior of this function may break. /// /// # Panics /// - /// * Panics if the requested capacity exceeds `usize::MAX` bytes. - /// * Panics on 32-bit platforms if the requested capacity exceeds - /// `isize::MAX` bytes. + /// Panics if the new capacity exceeds `isize::MAX` bytes. /// /// # Aborts /// /// Aborts on OOM. - /// - /// # Examples - /// - /// ``` - /// # #![feature(raw_vec_internals)] - /// # extern crate alloc; - /// # use std::ptr; - /// # use alloc::raw_vec::RawVec; - /// struct MyVec { - /// buf: RawVec, - /// len: usize, - /// } - /// - /// impl MyVec { - /// pub fn push_all(&mut self, elems: &[T]) { - /// self.buf.reserve(self.len, elems.len()); - /// // reserve would have aborted or panicked if the len exceeded - /// // `isize::MAX` so this is safe to do unchecked now. - /// for x in elems { - /// unsafe { - /// ptr::write(self.buf.ptr().add(self.len), x.clone()); - /// } - /// self.len += 1; - /// } - /// } - /// } - /// # fn main() { - /// # let mut vector = MyVec { buf: RawVec::new(), len: 0 }; - /// # vector.push_all(&[1, 3, 5, 7, 9]); - /// # } - /// ``` - pub fn reserve(&mut self, used_capacity: usize, needed_extra_capacity: usize) { - match self.reserve_internal(used_capacity, needed_extra_capacity, Infallible, Amortized) { - Err(CapacityOverflow) => capacity_overflow(), - Err(AllocError { .. }) => unreachable!(), - Ok(()) => { /* yay */ } - } + #[cfg(not(no_global_oom_handling))] + pub fn reserve_exact(&mut self, len: usize, additional: usize) { + handle_reserve(self.try_reserve_exact(len, additional)); } - /// Attempts to ensure that the buffer contains at least enough space to hold - /// `used_capacity + needed_extra_capacity` elements. If it doesn't already have - /// enough capacity, will reallocate in place enough space plus comfortable slack - /// space to get amortized `O(1)` behavior. Will limit this behaviour - /// if it would needlessly cause itself to panic. - /// - /// If `used_capacity` exceeds `self.capacity()`, this may fail to actually allocate - /// the requested space. This is not really unsafe, but the unsafe - /// code *you* write that relies on the behavior of this function may break. - /// - /// Returns `true` if the reallocation attempt has succeeded. - /// - /// # Panics - /// - /// * Panics if the requested capacity exceeds `usize::MAX` bytes. - /// * Panics on 32-bit platforms if the requested capacity exceeds - /// `isize::MAX` bytes. - pub fn reserve_in_place(&mut self, used_capacity: usize, needed_extra_capacity: usize) -> bool { - unsafe { - // NOTE: we don't early branch on ZSTs here because we want this - // to actually catch "asking for more than usize::MAX" in that case. - // If we make it past the first branch then we are guaranteed to - // panic. - - // Don't actually need any more capacity. If the current `cap` is 0, we can't - // reallocate in place. - // Wrapping in case they give a bad `used_capacity` - if self.cap == 0 { - return false; - } - if self.capacity().wrapping_sub(used_capacity) >= needed_extra_capacity { - return false; - } - - let new_cap = self - .amortized_new_size(used_capacity, needed_extra_capacity) - .unwrap_or_else(|_| capacity_overflow()); - - // Here, `cap < used_capacity + needed_extra_capacity <= new_cap` - // (regardless of whether `self.cap - used_capacity` wrapped). - // Therefore, we can safely call `grow_in_place`. - crucible::alloc::reallocate(self.ptr.as_ptr(), new_cap); - self.cap = new_cap; - true - } + /// The same as `reserve_exact`, but returns on errors instead of panicking or aborting. + pub fn try_reserve_exact( + &mut self, + len: usize, + additional: usize, + ) -> Result<(), TryReserveError> { + if self.needs_to_grow(len, additional) { self.grow_exact(len, additional) } else { Ok(()) } } - /// Shrinks the allocation down to the specified amount. If the given amount + /// Shrinks the buffer down to the specified capacity. If the given amount /// is 0, actually completely deallocates. /// /// # Panics @@ -491,126 +348,150 @@ impl RawVec { /// # Aborts /// /// Aborts on OOM. - pub fn shrink_to_fit(&mut self, amount: usize) { - let elem_size = mem::size_of::(); + #[cfg(not(no_global_oom_handling))] + pub fn shrink_to_fit(&mut self, cap: usize) { + handle_reserve(self.shrink(cap)); + } +} - // Set the `cap` because they might be about to promote to a `Box<[T]>` - if elem_size == 0 { - self.cap = amount; - return; - } +impl RawVec { + /// Returns if the buffer needs to grow to fulfill the needed extra capacity. + /// Mainly used to make inlining reserve-calls possible without inlining `grow`. + fn needs_to_grow(&self, len: usize, additional: usize) -> bool { + additional > self.capacity().wrapping_sub(len) + } - // This check is my waterloo; it's the only thing `Vec` wouldn't have to do. - assert!(self.cap >= amount, "Tried to shrink to a larger capacity"); + fn set_ptr_and_cap(&mut self, ptr: NonNull<[u8]>, cap: usize) { + // Allocators currently return a `NonNull<[u8]>` whose length matches + // the size requested. If that ever changes, the capacity here should + // change to `ptr.len() / mem::size_of::()`. + self.ptr = unsafe { Unique::new_unchecked(ptr.cast().as_ptr()) }; + self.cap = cap; + } - if self.cap == 0 { - // `amount` is also zero. There's no current allocation, but also nothing to do. - return; + // This method is usually instantiated many times. So we want it to be as + // small as possible, to improve compile times. But we also want as much of + // its contents to be statically computable as possible, to make the + // generated code run faster. Therefore, this method is carefully written + // so that all of the code that depends on `T` is within it, while as much + // of the code that doesn't depend on `T` as possible is in functions that + // are non-generic over `T`. + fn grow_amortized(&mut self, len: usize, additional: usize) -> Result<(), TryReserveError> { + // This is ensured by the calling contexts. + debug_assert!(additional > 0); + + if T::IS_ZST { + // Since we return a capacity of `usize::MAX` when `elem_size` is + // 0, getting to here necessarily means the `RawVec` is overfull. + return Err(CapacityOverflow.into()); } - unsafe { - crucible::alloc::reallocate(self.ptr.as_ptr(), amount); - if amount == 0 { - self.ptr = NonNull::dangling().into(); - } - self.cap = amount; - } - } -} + // Nothing we can really do about these checks, sadly. + let required_cap = len.checked_add(additional).ok_or(CapacityOverflow)?; -enum Fallibility { - Fallible, - Infallible, -} + // This guarantees exponential growth. The doubling cannot overflow + // because `cap <= isize::MAX` and the type of `cap` is `usize`. + let cap = cmp::max(self.cap * 2, required_cap); + let cap = cmp::max(Self::MIN_NON_ZERO_CAP, cap); -use Fallibility::*; + let new_layout = Layout::array::(cap); -enum ReserveStrategy { - Exact, - Amortized, -} + // `finish_grow` is non-generic over `T`. + let ptr = finish_grow(new_layout, self.current_memory(), &mut self.alloc)?; + self.set_ptr_and_cap(ptr, cap); + Ok(()) + } -use ReserveStrategy::*; + // The constraints on this method are much the same as those on + // `grow_amortized`, but this method is usually instantiated less often so + // it's less critical. + fn grow_exact(&mut self, len: usize, additional: usize) -> Result<(), TryReserveError> { + if T::IS_ZST { + // Since we return a capacity of `usize::MAX` when the type size is + // 0, getting to here necessarily means the `RawVec` is overfull. + return Err(CapacityOverflow.into()); + } -impl RawVec { - fn reserve_internal( - &mut self, - used_capacity: usize, - needed_extra_capacity: usize, - fallibility: Fallibility, - strategy: ReserveStrategy, - ) -> Result<(), TryReserveError> { - let elem_size = mem::size_of::(); + let cap = len.checked_add(additional).ok_or(CapacityOverflow)?; + let new_layout = Layout::array::(cap); - unsafe { - // NOTE: we don't early branch on ZSTs here because we want this - // to actually catch "asking for more than usize::MAX" in that case. - // If we make it past the first branch then we are guaranteed to - // panic. - - // Don't actually need any more capacity. - // Wrapping in case they gave a bad `used_capacity`. - if self.capacity().wrapping_sub(used_capacity) >= needed_extra_capacity { - return Ok(()); - } + // `finish_grow` is non-generic over `T`. + let ptr = finish_grow(new_layout, self.current_memory(), &mut self.alloc)?; + self.set_ptr_and_cap(ptr, cap); + Ok(()) + } - // Nothing we can really do about these checks, sadly. - let new_cap = match strategy { - Exact => { - used_capacity.checked_add(needed_extra_capacity).ok_or(CapacityOverflow)? - } - Amortized => self.amortized_new_size(used_capacity, needed_extra_capacity)?, - }; + #[cfg(not(no_global_oom_handling))] + fn shrink(&mut self, cap: usize) -> Result<(), TryReserveError> { + assert!(cap <= self.capacity(), "Tried to shrink to a larger capacity"); - if self.cap == 0 { - self.ptr = NonNull::new_unchecked(crucible::alloc::allocate::(new_cap)).into(); - } else { - crucible::alloc::reallocate::(self.ptr.as_ptr(), new_cap); - } + let (ptr, layout) = if let Some(mem) = self.current_memory() { mem } else { return Ok(()) }; - self.cap = new_cap; - - Ok(()) - } + let ptr = unsafe { + // `Layout::array` cannot overflow here because it would have + // overflowed earlier when capacity was larger. + let new_layout = Layout::array::(cap).unwrap_unchecked(); + self.alloc + .shrink(ptr, layout, new_layout) + .map_err(|_| AllocError { layout: new_layout, non_exhaustive: () })? + }; + self.set_ptr_and_cap(ptr, cap); + Ok(()) } } -impl RawVec { - /// Converts the entire buffer into `Box<[T]>`. - /// - /// Note that this will correctly reconstitute any `cap` changes - /// that may have been performed. (See description of type for details.) - /// - /// # Undefined Behavior - /// - /// All elements of `RawVec` must be initialized. Notice that - /// the rules around uninitialized boxed values are not finalized yet, - /// but until they are, it is advisable to avoid them. - pub unsafe fn into_box(self) -> Box<[T]> { - // NOTE: not calling `capacity()` here; actually using the real `cap` field! - let slice = slice::from_raw_parts_mut(self.ptr(), self.cap); - let output: Box<[T]> = Box::from_raw(slice); - mem::forget(self); - output - } -} +// This function is outside `RawVec` to minimize compile times. See the comment +// above `RawVec::grow_amortized` for details. (The `A` parameter isn't +// significant, because the number of different `A` types seen in practice is +// much smaller than the number of `T` types.) +#[inline(never)] +fn finish_grow( + new_layout: Result, + current_memory: Option<(NonNull, Layout)>, + alloc: &mut A, +) -> Result, TryReserveError> +where + A: Allocator, +{ + // Check for the error here to minimize the size of `RawVec::grow_*`. + let new_layout = new_layout.map_err(|_| CapacityOverflow)?; + + alloc_guard(new_layout.size())?; + + let memory = if let Some((ptr, old_layout)) = current_memory { + debug_assert_eq!(old_layout.align(), new_layout.align()); + unsafe { + // The allocator checks for alignment equality + intrinsics::assume(old_layout.align() == new_layout.align()); + alloc.grow(ptr, old_layout, new_layout) + } + } else { + alloc.allocate(new_layout) + }; -impl RawVec { - /// Frees the memory owned by the `RawVec` *without* trying to drop its contents. - pub unsafe fn dealloc_buffer(&mut self) { - // Crucible currently doesn't provide a deallocate operation. - } + memory.map_err(|_| AllocError { layout: new_layout, non_exhaustive: () }.into()) } -unsafe impl<#[may_dangle] T, A: AllocRef> Drop for RawVec { +unsafe impl<#[may_dangle] T, A: Allocator> Drop for RawVec { /// Frees the memory owned by the `RawVec` *without* trying to drop its contents. fn drop(&mut self) { - unsafe { - self.dealloc_buffer(); + if let Some((ptr, layout)) = self.current_memory() { + unsafe { self.alloc.deallocate(ptr, layout) } } } } +// Central function for reserve error handling. +#[cfg(not(no_global_oom_handling))] +#[inline] +fn handle_reserve(result: Result<(), TryReserveError>) { + match result.map_err(|e| e.kind()) { + Err(CapacityOverflow) => capacity_overflow(), + Err(AllocError { layout, .. }) => handle_alloc_error(layout), + Ok(()) => { /* yay */ } + } +} + // We need to guarantee the following: // * We don't ever allocate `> isize::MAX` byte-size objects. // * We don't overflow `usize::MAX` and actually allocate too little. @@ -622,8 +503,8 @@ unsafe impl<#[may_dangle] T, A: AllocRef> Drop for RawVec { #[inline] fn alloc_guard(alloc_size: usize) -> Result<(), TryReserveError> { - if mem::size_of::() < 8 && alloc_size > core::isize::MAX as usize { - Err(CapacityOverflow) + if usize::BITS < 64 && alloc_size > isize::MAX as usize { + Err(CapacityOverflow.into()) } else { Ok(()) } @@ -632,6 +513,7 @@ fn alloc_guard(alloc_size: usize) -> Result<(), TryReserveError> { // One central function responsible for reporting capacity overflows. This'll // ensure that the code generation related to these panics is minimal as there's // only one location which panics rather than a bunch throughout the module. +#[cfg(not(no_global_oom_handling))] fn capacity_overflow() -> ! { panic!("capacity overflow"); } diff --git a/crux-mir/lib/alloc/src/raw_vec/tests.rs b/crux-mir/lib/alloc/src/raw_vec/tests.rs index 21a8a76d0..ff322f0da 100644 --- a/crux-mir/lib/alloc/src/raw_vec/tests.rs +++ b/crux-mir/lib/alloc/src/raw_vec/tests.rs @@ -1,8 +1,9 @@ use super::*; +use std::cell::Cell; #[test] fn allocator_param() { - use crate::alloc::AllocErr; + use crate::alloc::AllocError; // Writing a test of integration between third-party // allocators and `RawVec` is a little tricky because the `RawVec` @@ -17,32 +18,32 @@ fn allocator_param() { // A dumb allocator that consumes a fixed amount of fuel // before allocation attempts start failing. struct BoundedAlloc { - fuel: usize, + fuel: Cell, } - unsafe impl AllocRef for BoundedAlloc { - fn alloc(&mut self, layout: Layout) -> Result<(NonNull, usize), AllocErr> { + unsafe impl Allocator for BoundedAlloc { + fn allocate(&self, layout: Layout) -> Result, AllocError> { let size = layout.size(); - if size > self.fuel { - return Err(AllocErr); + if size > self.fuel.get() { + return Err(AllocError); } - match Global.alloc(layout) { + match Global.allocate(layout) { ok @ Ok(_) => { - self.fuel -= size; + self.fuel.set(self.fuel.get() - size); ok } err @ Err(_) => err, } } - unsafe fn dealloc(&mut self, ptr: NonNull, layout: Layout) { - Global.dealloc(ptr, layout) + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + unsafe { Global.deallocate(ptr, layout) } } } - let a = BoundedAlloc { fuel: 500 }; + let a = BoundedAlloc { fuel: Cell::new(500) }; let mut v: RawVec = RawVec::with_capacity_in(50, a); - assert_eq!(v.a.fuel, 450); + assert_eq!(v.alloc.fuel.get(), 450); v.reserve(50, 150); // (causes a realloc, thus using 50 + 150 = 200 units of fuel) - assert_eq!(v.a.fuel, 250); + assert_eq!(v.alloc.fuel.get(), 250); } #[test] @@ -58,7 +59,7 @@ fn reserve_does_not_overallocate() { let mut v: RawVec = RawVec::new(); v.reserve(0, 7); assert_eq!(7, v.capacity()); - // 97 if more than double of 7, so `reserve` should work + // 97 is more than double of 7, so `reserve` should work // like `reserve_exact`. v.reserve(7, 90); assert_eq!(97, v.capacity()); @@ -76,3 +77,87 @@ fn reserve_does_not_overallocate() { assert!(v.capacity() >= 12 + 12 / 2); } } + +struct ZST; + +// A `RawVec` holding zero-sized elements should always look like this. +fn zst_sanity(v: &RawVec) { + assert_eq!(v.capacity(), usize::MAX); + assert_eq!(v.ptr(), core::ptr::Unique::::dangling().as_ptr()); + assert_eq!(v.current_memory(), None); +} + +#[test] +fn zst() { + let cap_err = Err(crate::collections::TryReserveErrorKind::CapacityOverflow.into()); + + assert_eq!(std::mem::size_of::(), 0); + + // All these different ways of creating the RawVec produce the same thing. + + let v: RawVec = RawVec::new(); + zst_sanity(&v); + + let v: RawVec = RawVec::with_capacity_in(100, Global); + zst_sanity(&v); + + let v: RawVec = RawVec::with_capacity_in(100, Global); + zst_sanity(&v); + + let v: RawVec = RawVec::allocate_in(0, AllocInit::Uninitialized, Global); + zst_sanity(&v); + + let v: RawVec = RawVec::allocate_in(100, AllocInit::Uninitialized, Global); + zst_sanity(&v); + + let mut v: RawVec = RawVec::allocate_in(usize::MAX, AllocInit::Uninitialized, Global); + zst_sanity(&v); + + // Check all these operations work as expected with zero-sized elements. + + assert!(!v.needs_to_grow(100, usize::MAX - 100)); + assert!(v.needs_to_grow(101, usize::MAX - 100)); + zst_sanity(&v); + + v.reserve(100, usize::MAX - 100); + //v.reserve(101, usize::MAX - 100); // panics, in `zst_reserve_panic` below + zst_sanity(&v); + + v.reserve_exact(100, usize::MAX - 100); + //v.reserve_exact(101, usize::MAX - 100); // panics, in `zst_reserve_exact_panic` below + zst_sanity(&v); + + assert_eq!(v.try_reserve(100, usize::MAX - 100), Ok(())); + assert_eq!(v.try_reserve(101, usize::MAX - 100), cap_err); + zst_sanity(&v); + + assert_eq!(v.try_reserve_exact(100, usize::MAX - 100), Ok(())); + assert_eq!(v.try_reserve_exact(101, usize::MAX - 100), cap_err); + zst_sanity(&v); + + assert_eq!(v.grow_amortized(100, usize::MAX - 100), cap_err); + assert_eq!(v.grow_amortized(101, usize::MAX - 100), cap_err); + zst_sanity(&v); + + assert_eq!(v.grow_exact(100, usize::MAX - 100), cap_err); + assert_eq!(v.grow_exact(101, usize::MAX - 100), cap_err); + zst_sanity(&v); +} + +#[test] +#[should_panic(expected = "capacity overflow")] +fn zst_reserve_panic() { + let mut v: RawVec = RawVec::new(); + zst_sanity(&v); + + v.reserve(101, usize::MAX - 100); +} + +#[test] +#[should_panic(expected = "capacity overflow")] +fn zst_reserve_exact_panic() { + let mut v: RawVec = RawVec::new(); + zst_sanity(&v); + + v.reserve_exact(101, usize::MAX - 100); +} diff --git a/crux-mir/lib/alloc/src/rc.rs b/crux-mir/lib/alloc/src/rc.rs index 68d1b578d..c9aa23fc4 100644 --- a/crux-mir/lib/alloc/src/rc.rs +++ b/crux-mir/lib/alloc/src/rc.rs @@ -11,7 +11,7 @@ //! is no exception: you cannot generally obtain a mutable reference to //! something inside an [`Rc`]. If you need mutability, put a [`Cell`] //! or [`RefCell`] inside the [`Rc`]; see [an example of mutability -//! inside an Rc][mutability]. +//! inside an `Rc`][mutability]. //! //! [`Rc`] uses non-atomic reference counting. This means that overhead is very //! low, but an [`Rc`] cannot be sent between threads, and consequently [`Rc`] @@ -35,13 +35,27 @@ //! `Rc` automatically dereferences to `T` (via the [`Deref`] trait), //! so you can call `T`'s methods on a value of type [`Rc`][`Rc`]. To avoid name //! clashes with `T`'s methods, the methods of [`Rc`][`Rc`] itself are associated -//! functions, called using function-like syntax: +//! functions, called using [fully qualified syntax]: //! //! ``` //! use std::rc::Rc; +//! //! let my_rc = Rc::new(()); +//! let my_weak = Rc::downgrade(&my_rc); +//! ``` +//! +//! `Rc`'s implementations of traits like `Clone` may also be called using +//! fully qualified syntax. Some people prefer to use fully qualified syntax, +//! while others prefer using method-call syntax. //! -//! Rc::downgrade(&my_rc); +//! ``` +//! use std::rc::Rc; +//! +//! let rc = Rc::new(()); +//! // Method-call syntax +//! let rc2 = rc.clone(); +//! // Fully qualified syntax +//! let rc3 = Rc::clone(&rc); //! ``` //! //! [`Weak`][`Weak`] does not auto-dereference to `T`, because the inner value may have @@ -54,6 +68,7 @@ //! //! ``` //! use std::rc::Rc; +//! //! let foo = Rc::new(vec![1.0, 2.0, 3.0]); //! // The two syntaxes below are equivalent. //! let a = foo.clone(); @@ -214,18 +229,16 @@ //! } //! ``` //! -//! [`Rc`]: struct.Rc.html -//! [`Weak`]: struct.Weak.html -//! [clone]: ../../std/clone/trait.Clone.html#tymethod.clone -//! [`Cell`]: ../../std/cell/struct.Cell.html -//! [`RefCell`]: ../../std/cell/struct.RefCell.html -//! [send]: ../../std/marker/trait.Send.html -//! [arc]: ../../std/sync/struct.Arc.html -//! [`Deref`]: ../../std/ops/trait.Deref.html -//! [downgrade]: struct.Rc.html#method.downgrade -//! [upgrade]: struct.Weak.html#method.upgrade -//! [`None`]: ../../std/option/enum.Option.html#variant.None -//! [mutability]: ../../std/cell/index.html#introducing-mutability-inside-of-something-immutable +//! [clone]: Clone::clone +//! [`Cell`]: core::cell::Cell +//! [`RefCell`]: core::cell::RefCell +//! [send]: core::marker::Send +//! [arc]: crate::sync::Arc +//! [`Deref`]: core::ops::Deref +//! [downgrade]: Rc::downgrade +//! [upgrade]: Weak::upgrade +//! [mutability]: core::cell#introducing-mutability-inside-of-something-immutable +//! [fully qualified syntax]: https://doc.rust-lang.org/book/ch19-03-advanced-traits.html#fully-qualified-syntax-for-disambiguation-calling-methods-with-the-same-name #![stable(feature = "rust1", since = "1.0.0")] @@ -235,7 +248,6 @@ use crate::boxed::Box; use std::boxed::Box; use core::any::Any; -use core::array::LengthAtMost32; use core::borrow; use core::cell::Cell; use core::cmp::Ordering; @@ -243,29 +255,53 @@ use core::convert::{From, TryFrom}; use core::fmt; use core::hash::{Hash, Hasher}; use core::intrinsics::abort; +#[cfg(not(no_global_oom_handling))] use core::iter; use core::marker::{self, PhantomData, Unpin, Unsize}; -use core::mem::{self, align_of, align_of_val, forget, size_of_val}; +#[cfg(not(no_global_oom_handling))] +use core::mem::size_of_val; +use core::mem::{self, align_of_val_raw, forget}; use core::ops::{CoerceUnsized, Deref, DispatchFromDyn, Receiver}; +use core::panic::{RefUnwindSafe, UnwindSafe}; +#[cfg(not(no_global_oom_handling))] use core::pin::Pin; use core::ptr::{self, NonNull}; -use core::slice::{self, from_raw_parts_mut}; -use core::usize; -use crucible; - -use crate::alloc::{box_free, handle_alloc_error, AllocRef, Global, Layout}; +#[cfg(not(no_global_oom_handling))] +use core::slice::from_raw_parts_mut; + +#[cfg(not(no_global_oom_handling))] +use crate::alloc::handle_alloc_error; +#[cfg(not(no_global_oom_handling))] +use crate::alloc::{box_free, WriteCloneIntoRaw}; +use crate::alloc::{AllocError, Allocator, Global, Layout}; +use crate::borrow::{Cow, ToOwned}; +#[cfg(not(no_global_oom_handling))] use crate::string::String; +#[cfg(not(no_global_oom_handling))] use crate::vec::Vec; #[cfg(test)] mod tests; +// This is repr(C) to future-proof against possible field-reordering, which +// would interfere with otherwise safe [into|from]_raw() of transmutable +// inner types. +#[repr(C)] struct RcBox { strong: Cell, weak: Cell, value: T, } +/// Calculate layout for `RcBox` using the inner value's layout +fn rcbox_layout_for_value_layout(layout: Layout) -> Layout { + // Calculate layout using the given value layout. + // Previously, layout was calculated on the expression + // `&*(ptr as *const RcBox)`, but this created a misaligned + // reference (see #54908). + Layout::new::>().extend(layout).unwrap().0.pad_to_align() +} + /// A single-threaded reference-counting pointer. 'Rc' stands for 'Reference /// Counted'. /// @@ -273,12 +309,12 @@ struct RcBox { /// /// The inherent methods of `Rc` are all associated functions, which means /// that you have to call them as e.g., [`Rc::get_mut(&mut value)`][get_mut] instead of -/// `value.get_mut()`. This avoids conflicts with methods of the inner -/// type `T`. +/// `value.get_mut()`. This avoids conflicts with methods of the inner type `T`. /// -/// [get_mut]: #method.get_mut -#[cfg_attr(not(test), lang = "rc")] +/// [get_mut]: Rc::get_mut +#[cfg_attr(not(test), rustc_diagnostic_item = "Rc")] #[stable(feature = "rust1", since = "1.0.0")] +#[rustc_insignificant_dtor] pub struct Rc { ptr: NonNull>, phantom: PhantomData>, @@ -286,22 +322,40 @@ pub struct Rc { #[stable(feature = "rust1", since = "1.0.0")] impl !marker::Send for Rc {} + +// Note that this negative impl isn't strictly necessary for correctness, +// as `Rc` transitively contains a `Cell`, which is itself `!Sync`. +// However, given how important `Rc`'s `!Sync`-ness is, +// having an explicit negative impl is nice for documentation purposes +// and results in nicer error messages. #[stable(feature = "rust1", since = "1.0.0")] impl !marker::Sync for Rc {} -#[unstable(feature = "coerce_unsized", issue = "27732")] +#[stable(feature = "catch_unwind", since = "1.9.0")] +impl UnwindSafe for Rc {} +#[stable(feature = "rc_ref_unwind_safe", since = "1.58.0")] +impl RefUnwindSafe for Rc {} + +#[unstable(feature = "coerce_unsized", issue = "18598")] impl, U: ?Sized> CoerceUnsized> for Rc {} #[unstable(feature = "dispatch_from_dyn", issue = "none")] impl, U: ?Sized> DispatchFromDyn> for Rc {} impl Rc { - fn from_inner(ptr: NonNull>) -> Self { + #[inline(always)] + fn inner(&self) -> &RcBox { + // This unsafety is ok because while this Rc is alive we're guaranteed + // that the inner pointer is valid. + unsafe { self.ptr.as_ref() } + } + + unsafe fn from_inner(ptr: NonNull>) -> Self { Self { ptr, phantom: PhantomData } } unsafe fn from_ptr(ptr: *mut RcBox) -> Self { - Self::from_inner(NonNull::new_unchecked(ptr)) + unsafe { Self::from_inner(NonNull::new_unchecked(ptr)) } } } @@ -315,17 +369,114 @@ impl Rc { /// /// let five = Rc::new(5); /// ``` + #[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] pub fn new(value: T) -> Rc { // There is an implicit weak pointer owned by all the strong // pointers, which ensures that the weak destructor never frees // the allocation while the strong destructor is running, even // if the weak pointer is stored inside the strong one. - Self::from_inner(Box::into_raw_non_null(box RcBox { - strong: Cell::new(1), + unsafe { + Self::from_inner( + Box::leak(Box::new(RcBox { strong: Cell::new(1), weak: Cell::new(1), value })) + .into(), + ) + } + } + + /// Constructs a new `Rc` while giving you a `Weak` to the allocation, + /// to allow you to construct a `T` which holds a weak pointer to itself. + /// + /// Generally, a structure circularly referencing itself, either directly or + /// indirectly, should not hold a strong reference to itself to prevent a memory leak. + /// Using this function, you get access to the weak pointer during the + /// initialization of `T`, before the `Rc` is created, such that you can + /// clone and store it inside the `T`. + /// + /// `new_cyclic` first allocates the managed allocation for the `Rc`, + /// then calls your closure, giving it a `Weak` to this allocation, + /// and only afterwards completes the construction of the `Rc` by placing + /// the `T` returned from your closure into the allocation. + /// + /// Since the new `Rc` is not fully-constructed until `Rc::new_cyclic` + /// returns, calling [`upgrade`] on the weak reference inside your closure will + /// fail and result in a `None` value. + /// + /// # Panics + /// + /// If `data_fn` panics, the panic is propagated to the caller, and the + /// temporary [`Weak`] is dropped normally. + /// + /// # Examples + /// + /// ``` + /// # #![allow(dead_code)] + /// use std::rc::{Rc, Weak}; + /// + /// struct Gadget { + /// me: Weak, + /// } + /// + /// impl Gadget { + /// /// Construct a reference counted Gadget. + /// fn new() -> Rc { + /// // `me` is a `Weak` pointing at the new allocation of the + /// // `Rc` we're constructing. + /// Rc::new_cyclic(|me| { + /// // Create the actual struct here. + /// Gadget { me: me.clone() } + /// }) + /// } + /// + /// /// Return a reference counted pointer to Self. + /// fn me(&self) -> Rc { + /// self.me.upgrade().unwrap() + /// } + /// } + /// ``` + /// [`upgrade`]: Weak::upgrade + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "arc_new_cyclic", since = "1.60.0")] + pub fn new_cyclic(data_fn: F) -> Rc + where + F: FnOnce(&Weak) -> T, + { + // Construct the inner in the "uninitialized" state with a single + // weak reference. + let uninit_ptr: NonNull<_> = Box::leak(Box::new(RcBox { + strong: Cell::new(0), weak: Cell::new(1), - value, + value: mem::MaybeUninit::::uninit(), })) + .into(); + + let init_ptr: NonNull> = uninit_ptr.cast(); + + let weak = Weak { ptr: init_ptr }; + + // It's important we don't give up ownership of the weak pointer, or + // else the memory might be freed by the time `data_fn` returns. If + // we really wanted to pass ownership, we could create an additional + // weak pointer for ourselves, but this would result in additional + // updates to the weak reference count which might not be necessary + // otherwise. + let data = data_fn(&weak); + + let strong = unsafe { + let inner = init_ptr.as_ptr(); + ptr::write(ptr::addr_of_mut!((*inner).value), data); + + let prev_value = (*inner).strong.get(); + debug_assert_eq!(prev_value, 0, "No prior strong references should exist"); + (*inner).strong.set(1); + + Rc::from_inner(init_ptr) + }; + + // Strong references should collectively own a shared weak reference, + // so don't run the destructor for our old weak reference. + mem::forget(weak); + strong } /// Constructs a new `Rc` with uninitialized contents. @@ -340,21 +491,23 @@ impl Rc { /// /// let mut five = Rc::::new_uninit(); /// - /// let five = unsafe { - /// // Deferred initialization: - /// Rc::get_mut_unchecked(&mut five).as_mut_ptr().write(5); + /// // Deferred initialization: + /// Rc::get_mut(&mut five).unwrap().write(5); /// - /// five.assume_init() - /// }; + /// let five = unsafe { five.assume_init() }; /// /// assert_eq!(*five, 5) /// ``` + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] pub fn new_uninit() -> Rc> { unsafe { - Rc::from_ptr(Rc::allocate_for_layout(Layout::new::(), |mem| { - mem as *mut RcBox> - })) + Rc::from_ptr(Rc::allocate_for_layout( + Layout::new::(), + |layout| Global.allocate(layout), + |mem| mem as *mut RcBox>, + )) } } @@ -377,32 +530,125 @@ impl Rc { /// assert_eq!(*zero, 0) /// ``` /// - /// [zeroed]: ../../std/mem/union.MaybeUninit.html#method.zeroed + /// [zeroed]: mem::MaybeUninit::zeroed + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] pub fn new_zeroed() -> Rc> { unsafe { - let mut uninit = Self::new_uninit(); - ptr::write_bytes::(Rc::get_mut_unchecked(&mut uninit).as_mut_ptr(), 0, 1); - uninit + Rc::from_ptr(Rc::allocate_for_layout( + Layout::new::(), + |layout| Global.allocate_zeroed(layout), + |mem| mem as *mut RcBox>, + )) + } + } + + /// Constructs a new `Rc`, returning an error if the allocation fails + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api)] + /// use std::rc::Rc; + /// + /// let five = Rc::try_new(5); + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + #[unstable(feature = "allocator_api", issue = "32838")] + pub fn try_new(value: T) -> Result, AllocError> { + // There is an implicit weak pointer owned by all the strong + // pointers, which ensures that the weak destructor never frees + // the allocation while the strong destructor is running, even + // if the weak pointer is stored inside the strong one. + unsafe { + Ok(Self::from_inner( + Box::leak(Box::try_new(RcBox { strong: Cell::new(1), weak: Cell::new(1), value })?) + .into(), + )) + } + } + + /// Constructs a new `Rc` with uninitialized contents, returning an error if the allocation fails + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// #![feature(get_mut_unchecked)] + /// + /// use std::rc::Rc; + /// + /// let mut five = Rc::::try_new_uninit()?; + /// + /// // Deferred initialization: + /// Rc::get_mut(&mut five).unwrap().write(5); + /// + /// let five = unsafe { five.assume_init() }; + /// + /// assert_eq!(*five, 5); + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + #[unstable(feature = "allocator_api", issue = "32838")] + // #[unstable(feature = "new_uninit", issue = "63291")] + pub fn try_new_uninit() -> Result>, AllocError> { + unsafe { + Ok(Rc::from_ptr(Rc::try_allocate_for_layout( + Layout::new::(), + |layout| Global.allocate(layout), + |mem| mem as *mut RcBox>, + )?)) } } + /// Constructs a new `Rc` with uninitialized contents, with the memory + /// being filled with `0` bytes, returning an error if the allocation fails + /// + /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and + /// incorrect usage of this method. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// + /// use std::rc::Rc; + /// + /// let zero = Rc::::try_new_zeroed()?; + /// let zero = unsafe { zero.assume_init() }; + /// + /// assert_eq!(*zero, 0); + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + /// + /// [zeroed]: mem::MaybeUninit::zeroed + #[unstable(feature = "allocator_api", issue = "32838")] + //#[unstable(feature = "new_uninit", issue = "63291")] + pub fn try_new_zeroed() -> Result>, AllocError> { + unsafe { + Ok(Rc::from_ptr(Rc::try_allocate_for_layout( + Layout::new::(), + |layout| Global.allocate_zeroed(layout), + |mem| mem as *mut RcBox>, + )?)) + } + } /// Constructs a new `Pin>`. If `T` does not implement `Unpin`, then /// `value` will be pinned in memory and unable to be moved. + #[cfg(not(no_global_oom_handling))] #[stable(feature = "pin", since = "1.33.0")] + #[must_use] pub fn pin(value: T) -> Pin> { unsafe { Pin::new_unchecked(Rc::new(value)) } } /// Returns the inner value, if the `Rc` has exactly one strong reference. /// - /// Otherwise, an [`Err`][result] is returned with the same `Rc` that was + /// Otherwise, an [`Err`] is returned with the same `Rc` that was /// passed in. /// /// This will succeed even if there are outstanding weak references. /// - /// [result]: ../../std/result/enum.Result.html - /// /// # Examples /// /// ``` @@ -426,7 +672,7 @@ impl Rc { // the strong count, and then remove the implicit "strong weak" // pointer while also handling drop logic by just crafting a // fake Weak. - this.dec_strong(); + this.inner().dec_strong(); let _weak = Weak { ptr: this.ptr }; forget(this); Ok(val) @@ -450,21 +696,58 @@ impl Rc<[T]> { /// /// let mut values = Rc::<[u32]>::new_uninit_slice(3); /// - /// let values = unsafe { - /// // Deferred initialization: - /// Rc::get_mut_unchecked(&mut values)[0].as_mut_ptr().write(1); - /// Rc::get_mut_unchecked(&mut values)[1].as_mut_ptr().write(2); - /// Rc::get_mut_unchecked(&mut values)[2].as_mut_ptr().write(3); + /// // Deferred initialization: + /// let data = Rc::get_mut(&mut values).unwrap(); + /// data[0].write(1); + /// data[1].write(2); + /// data[2].write(3); /// - /// values.assume_init() - /// }; + /// let values = unsafe { values.assume_init() }; /// /// assert_eq!(*values, [1, 2, 3]) /// ``` + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] pub fn new_uninit_slice(len: usize) -> Rc<[mem::MaybeUninit]> { unsafe { Rc::from_ptr(Rc::allocate_for_slice(len)) } } + + /// Constructs a new reference-counted slice with uninitialized contents, with the memory being + /// filled with `0` bytes. + /// + /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and + /// incorrect usage of this method. + /// + /// # Examples + /// + /// ``` + /// #![feature(new_uninit)] + /// + /// use std::rc::Rc; + /// + /// let values = Rc::<[u32]>::new_zeroed_slice(3); + /// let values = unsafe { values.assume_init() }; + /// + /// assert_eq!(*values, [0, 0, 0]) + /// ``` + /// + /// [zeroed]: mem::MaybeUninit::zeroed + #[cfg(not(no_global_oom_handling))] + #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] + pub fn new_zeroed_slice(len: usize) -> Rc<[mem::MaybeUninit]> { + unsafe { + Rc::from_ptr(Rc::allocate_for_layout( + Layout::array::(len).unwrap(), + |layout| Global.allocate_zeroed(layout), + |mem| { + ptr::slice_from_raw_parts_mut(mem as *mut T, len) + as *mut RcBox<[mem::MaybeUninit]> + }, + )) + } + } } impl Rc> { @@ -478,7 +761,7 @@ impl Rc> { /// Calling this when the content is not yet fully initialized /// causes immediate undefined behavior. /// - /// [`MaybeUninit::assume_init`]: ../../std/mem/union.MaybeUninit.html#method.assume_init + /// [`MaybeUninit::assume_init`]: mem::MaybeUninit::assume_init /// /// # Examples /// @@ -490,19 +773,17 @@ impl Rc> { /// /// let mut five = Rc::::new_uninit(); /// - /// let five = unsafe { - /// // Deferred initialization: - /// Rc::get_mut_unchecked(&mut five).as_mut_ptr().write(5); + /// // Deferred initialization: + /// Rc::get_mut(&mut five).unwrap().write(5); /// - /// five.assume_init() - /// }; + /// let five = unsafe { five.assume_init() }; /// /// assert_eq!(*five, 5) /// ``` #[unstable(feature = "new_uninit", issue = "63291")] #[inline] pub unsafe fn assume_init(self) -> Rc { - Rc::from_inner(mem::ManuallyDrop::new(self).ptr.cast()) + unsafe { Rc::from_inner(mem::ManuallyDrop::new(self).ptr.cast()) } } } @@ -517,7 +798,7 @@ impl Rc<[mem::MaybeUninit]> { /// Calling this when the content is not yet fully initialized /// causes immediate undefined behavior. /// - /// [`MaybeUninit::assume_init`]: ../../std/mem/union.MaybeUninit.html#method.assume_init + /// [`MaybeUninit::assume_init`]: mem::MaybeUninit::assume_init /// /// # Examples /// @@ -529,21 +810,20 @@ impl Rc<[mem::MaybeUninit]> { /// /// let mut values = Rc::<[u32]>::new_uninit_slice(3); /// - /// let values = unsafe { - /// // Deferred initialization: - /// Rc::get_mut_unchecked(&mut values)[0].as_mut_ptr().write(1); - /// Rc::get_mut_unchecked(&mut values)[1].as_mut_ptr().write(2); - /// Rc::get_mut_unchecked(&mut values)[2].as_mut_ptr().write(3); + /// // Deferred initialization: + /// let data = Rc::get_mut(&mut values).unwrap(); + /// data[0].write(1); + /// data[1].write(2); + /// data[2].write(3); /// - /// values.assume_init() - /// }; + /// let values = unsafe { values.assume_init() }; /// /// assert_eq!(*values, [1, 2, 3]) /// ``` #[unstable(feature = "new_uninit", issue = "63291")] #[inline] pub unsafe fn assume_init(self) -> Rc<[T]> { - Rc::from_ptr(mem::ManuallyDrop::new(self).ptr.as_ptr() as _) + unsafe { Rc::from_ptr(mem::ManuallyDrop::new(self).ptr.as_ptr() as _) } } } @@ -551,9 +831,7 @@ impl Rc { /// Consumes the `Rc`, returning the wrapped pointer. /// /// To avoid a memory leak the pointer must be converted back to an `Rc` using - /// [`Rc::from_raw`][from_raw]. - /// - /// [from_raw]: struct.Rc.html#method.from_raw + /// [`Rc::from_raw`]. /// /// # Examples /// @@ -566,30 +844,54 @@ impl Rc { /// ``` #[stable(feature = "rc_raw", since = "1.17.0")] pub fn into_raw(this: Self) -> *const T { - let ptr: *mut RcBox = NonNull::as_ptr(this.ptr); - let fake_ptr = ptr as *mut T; + let ptr = Self::as_ptr(&this); mem::forget(this); + ptr + } - // SAFETY: This cannot go through Deref::deref. - // Instead, we manually offset the pointer rather than manifesting a reference. - // This is so that the returned pointer retains the same provenance as our pointer. - // This is required so that e.g. `get_mut` can write through the pointer - // after the Rc is recovered through `from_raw`. - unsafe { - let offset = data_offset(&(*ptr).value); - set_data_ptr(fake_ptr, (ptr as *mut u8).offset(offset)) - } + /// Provides a raw pointer to the data. + /// + /// The counts are not affected in any way and the `Rc` is not consumed. The pointer is valid + /// for as long there are strong counts in the `Rc`. + /// + /// # Examples + /// + /// ``` + /// use std::rc::Rc; + /// + /// let x = Rc::new("hello".to_owned()); + /// let y = Rc::clone(&x); + /// let x_ptr = Rc::as_ptr(&x); + /// assert_eq!(x_ptr, Rc::as_ptr(&y)); + /// assert_eq!(unsafe { &*x_ptr }, "hello"); + /// ``` + #[stable(feature = "weak_into_raw", since = "1.45.0")] + pub fn as_ptr(this: &Self) -> *const T { + let ptr: *mut RcBox = NonNull::as_ptr(this.ptr); + + // SAFETY: This cannot go through Deref::deref or Rc::inner because + // this is required to retain raw/mut provenance such that e.g. `get_mut` can + // write through the pointer after the Rc is recovered through `from_raw`. + unsafe { ptr::addr_of_mut!((*ptr).value) } } - /// Constructs an `Rc` from a raw pointer. + /// Constructs an `Rc` from a raw pointer. /// - /// The raw pointer must have been previously returned by a call to a - /// [`Rc::into_raw`][into_raw]. + /// The raw pointer must have been previously returned by a call to + /// [`Rc::into_raw`][into_raw] where `U` must have the same size + /// and alignment as `T`. This is trivially true if `U` is `T`. + /// Note that if `U` is not `T` but has the same size and alignment, this is + /// basically like transmuting references of different types. See + /// [`mem::transmute`] for more information on what + /// restrictions apply in this case. /// - /// This function is unsafe because improper use may lead to memory problems. For example, a - /// double-free may occur if the function is called twice on the same raw pointer. + /// The user of `from_raw` has to make sure a specific value of `T` is only + /// dropped once. /// - /// [into_raw]: struct.Rc.html#method.into_raw + /// This function is unsafe because improper use may lead to memory unsafety, + /// even if the returned `Rc` is never accessed. + /// + /// [into_raw]: Rc::into_raw /// /// # Examples /// @@ -611,39 +913,15 @@ impl Rc { /// ``` #[stable(feature = "rc_raw", since = "1.17.0")] pub unsafe fn from_raw(ptr: *const T) -> Self { - let offset = data_offset(ptr); + let offset = unsafe { data_offset(ptr) }; // Reverse the offset to find the original RcBox. - let fake_ptr = ptr as *mut RcBox; - let rc_ptr = set_data_ptr(fake_ptr, (ptr as *mut u8).offset(-offset)); + let rc_ptr = unsafe { ptr.byte_sub(offset) as *mut RcBox }; - Self::from_ptr(rc_ptr) + unsafe { Self::from_ptr(rc_ptr) } } - /// Consumes the `Rc`, returning the wrapped pointer as `NonNull`. - /// - /// # Examples - /// - /// ``` - /// #![feature(rc_into_raw_non_null)] - /// - /// use std::rc::Rc; - /// - /// let x = Rc::new("hello".to_owned()); - /// let ptr = Rc::into_raw_non_null(x); - /// let deref = unsafe { ptr.as_ref() }; - /// assert_eq!(deref, "hello"); - /// ``` - #[unstable(feature = "rc_into_raw_non_null", issue = "47336")] - #[inline] - pub fn into_raw_non_null(this: Self) -> NonNull { - // safe because Rc guarantees its pointer is non-null - unsafe { NonNull::new_unchecked(Rc::into_raw(this) as *mut _) } - } - - /// Creates a new [`Weak`][weak] pointer to this allocation. - /// - /// [weak]: struct.Weak.html + /// Creates a new [`Weak`] pointer to this allocation. /// /// # Examples /// @@ -654,17 +932,17 @@ impl Rc { /// /// let weak_five = Rc::downgrade(&five); /// ``` + #[must_use = "this returns a new `Weak` pointer, \ + without modifying the original `Rc`"] #[stable(feature = "rc_weak", since = "1.4.0")] pub fn downgrade(this: &Self) -> Weak { - this.inc_weak(); + this.inner().inc_weak(); // Make sure we do not create a dangling Weak - debug_assert!(!is_dangling(this.ptr)); + debug_assert!(!is_dangling(this.ptr.as_ptr())); Weak { ptr: this.ptr } } - /// Gets the number of [`Weak`][weak] pointers to this allocation. - /// - /// [weak]: struct.Weak.html + /// Gets the number of [`Weak`] pointers to this allocation. /// /// # Examples /// @@ -679,7 +957,7 @@ impl Rc { #[inline] #[stable(feature = "rc_counts", since = "1.15.0")] pub fn weak_count(this: &Self) -> usize { - this.weak() - 1 + this.inner().weak() - 1 } /// Gets the number of strong (`Rc`) pointers to this allocation. @@ -697,31 +975,94 @@ impl Rc { #[inline] #[stable(feature = "rc_counts", since = "1.15.0")] pub fn strong_count(this: &Self) -> usize { - this.strong() + this.inner().strong() } - /// Returns `true` if there are no other `Rc` or [`Weak`][weak] pointers to - /// this allocation. + /// Increments the strong reference count on the `Rc` associated with the + /// provided pointer by one. + /// + /// # Safety /// - /// [weak]: struct.Weak.html + /// The pointer must have been obtained through `Rc::into_raw`, and the + /// associated `Rc` instance must be valid (i.e. the strong count must be at + /// least 1) for the duration of this method. + /// + /// # Examples + /// + /// ``` + /// use std::rc::Rc; + /// + /// let five = Rc::new(5); + /// + /// unsafe { + /// let ptr = Rc::into_raw(five); + /// Rc::increment_strong_count(ptr); + /// + /// let five = Rc::from_raw(ptr); + /// assert_eq!(2, Rc::strong_count(&five)); + /// } + /// ``` + #[inline] + #[stable(feature = "rc_mutate_strong_count", since = "1.53.0")] + pub unsafe fn increment_strong_count(ptr: *const T) { + // Retain Rc, but don't touch refcount by wrapping in ManuallyDrop + let rc = unsafe { mem::ManuallyDrop::new(Rc::::from_raw(ptr)) }; + // Now increase refcount, but don't drop new refcount either + let _rc_clone: mem::ManuallyDrop<_> = rc.clone(); + } + + /// Decrements the strong reference count on the `Rc` associated with the + /// provided pointer by one. + /// + /// # Safety + /// + /// The pointer must have been obtained through `Rc::into_raw`, and the + /// associated `Rc` instance must be valid (i.e. the strong count must be at + /// least 1) when invoking this method. This method can be used to release + /// the final `Rc` and backing storage, but **should not** be called after + /// the final `Rc` has been released. + /// + /// # Examples + /// + /// ``` + /// use std::rc::Rc; + /// + /// let five = Rc::new(5); + /// + /// unsafe { + /// let ptr = Rc::into_raw(five); + /// Rc::increment_strong_count(ptr); + /// + /// let five = Rc::from_raw(ptr); + /// assert_eq!(2, Rc::strong_count(&five)); + /// Rc::decrement_strong_count(ptr); + /// assert_eq!(1, Rc::strong_count(&five)); + /// } + /// ``` + #[inline] + #[stable(feature = "rc_mutate_strong_count", since = "1.53.0")] + pub unsafe fn decrement_strong_count(ptr: *const T) { + unsafe { mem::drop(Rc::from_raw(ptr)) }; + } + + /// Returns `true` if there are no other `Rc` or [`Weak`] pointers to + /// this allocation. #[inline] fn is_unique(this: &Self) -> bool { Rc::weak_count(this) == 0 && Rc::strong_count(this) == 1 } /// Returns a mutable reference into the given `Rc`, if there are - /// no other `Rc` or [`Weak`][weak] pointers to the same allocation. + /// no other `Rc` or [`Weak`] pointers to the same allocation. /// /// Returns [`None`] otherwise, because it is not safe to /// mutate a shared value. /// /// See also [`make_mut`][make_mut], which will [`clone`][clone] - /// the inner value when there are other pointers. + /// the inner value when there are other `Rc` pointers. /// - /// [weak]: struct.Weak.html - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// [make_mut]: struct.Rc.html#method.make_mut - /// [clone]: ../../std/clone/trait.Clone.html#tymethod.clone + /// [make_mut]: Rc::make_mut + /// [clone]: Clone::clone /// /// # Examples /// @@ -746,14 +1087,15 @@ impl Rc { /// /// See also [`get_mut`], which is safe and does appropriate checks. /// - /// [`get_mut`]: struct.Rc.html#method.get_mut + /// [`get_mut`]: Rc::get_mut /// /// # Safety /// - /// Any other `Rc` or [`Weak`] pointers to the same allocation must not be dereferenced - /// for the duration of the returned borrow. - /// This is trivially the case if no such pointers exist, - /// for example immediately after `Rc::new`. + /// If any other `Rc` or [`Weak`] pointers to the same allocation exist, then + /// they must be must not be dereferenced or have active borrows for the duration + /// of the returned borrow, and their inner type must be exactly the same as the + /// inner type of this Rc (including lifetimes). This is trivially the case if no + /// such pointers exist, for example immediately after `Rc::new`. /// /// # Examples /// @@ -768,16 +1110,50 @@ impl Rc { /// } /// assert_eq!(*x, "foo"); /// ``` + /// Other `Rc` pointers to the same allocation must be to the same type. + /// ```no_run + /// #![feature(get_mut_unchecked)] + /// + /// use std::rc::Rc; + /// + /// let x: Rc = Rc::from("Hello, world!"); + /// let mut y: Rc<[u8]> = x.clone().into(); + /// unsafe { + /// // this is Undefined Behavior, because x's inner type is str, not [u8] + /// Rc::get_mut_unchecked(&mut y).fill(0xff); // 0xff is invalid in UTF-8 + /// } + /// println!("{}", &*x); // Invalid UTF-8 in a str + /// ``` + /// Other `Rc` pointers to the same allocation must be to the exact same type, including lifetimes. + /// ```no_run + /// #![feature(get_mut_unchecked)] + /// + /// use std::rc::Rc; + /// + /// let x: Rc<&str> = Rc::new("Hello, world!"); + /// { + /// let s = String::from("Oh, no!"); + /// let mut y: Rc<&str> = x.clone().into(); + /// unsafe { + /// // this is Undefined Behavior, because x's inner type + /// // is &'long str, not &'short str + /// *Rc::get_mut_unchecked(&mut y) = &s; + /// } + /// } + /// println!("{}", &*x); // Use-after-free + /// ``` #[inline] #[unstable(feature = "get_mut_unchecked", issue = "63292")] pub unsafe fn get_mut_unchecked(this: &mut Self) -> &mut T { - &mut this.ptr.as_mut().value + // We are careful to *not* create a reference covering the "count" fields, as + // this would conflict with accesses to the reference counts (e.g. by `Weak`). + unsafe { &mut (*this.ptr.as_ptr()).value } } #[inline] #[stable(feature = "ptr_eq", since = "1.17.0")] - /// Returns `true` if the two `Rc`s point to the same allocation - /// (in a vein similar to [`ptr::eq`]). + /// Returns `true` if the two `Rc`s point to the same allocation in a vein similar to + /// [`ptr::eq`]. See [that function][`ptr::eq`] for caveats when comparing `dyn Trait` pointers. /// /// # Examples /// @@ -791,8 +1167,6 @@ impl Rc { /// assert!(Rc::ptr_eq(&five, &same_five)); /// assert!(!Rc::ptr_eq(&five, &other_five)); /// ``` - /// - /// [`ptr::eq`]: ../../std/ptr/fn.eq.html pub fn ptr_eq(this: &Self, other: &Self) -> bool { this.ptr.as_ptr() == other.ptr.as_ptr() } @@ -805,14 +1179,15 @@ impl Rc { /// [`clone`] the inner value to a new allocation to ensure unique ownership. This is also /// referred to as clone-on-write. /// - /// If there are no other `Rc` pointers to this allocation, then [`Weak`] - /// pointers to this allocation will be disassociated. + /// However, if there are no other `Rc` pointers to this allocation, but some [`Weak`] + /// pointers, then the [`Weak`] pointers will be disassociated and the inner value will not + /// be cloned. /// - /// See also [`get_mut`], which will fail rather than cloning. + /// See also [`get_mut`], which will fail rather than cloning the inner value + /// or disassociating [`Weak`] pointers. /// - /// [`Weak`]: struct.Weak.html - /// [`clone`]: ../../std/clone/trait.Clone.html#tymethod.clone - /// [`get_mut`]: struct.Rc.html#method.get_mut + /// [`clone`]: Clone::clone + /// [`get_mut`]: Rc::get_mut /// /// # Examples /// @@ -821,11 +1196,11 @@ impl Rc { /// /// let mut data = Rc::new(5); /// - /// *Rc::make_mut(&mut data) += 1; // Won't clone anything - /// let mut other_data = Rc::clone(&data); // Won't clone inner data - /// *Rc::make_mut(&mut data) += 1; // Clones inner data - /// *Rc::make_mut(&mut data) += 1; // Won't clone anything - /// *Rc::make_mut(&mut other_data) *= 2; // Won't clone anything + /// *Rc::make_mut(&mut data) += 1; // Won't clone anything + /// let mut other_data = Rc::clone(&data); // Won't clone inner data + /// *Rc::make_mut(&mut data) += 1; // Clones inner data + /// *Rc::make_mut(&mut data) += 1; // Won't clone anything + /// *Rc::make_mut(&mut other_data) *= 2; // Won't clone anything /// /// // Now `data` and `other_data` point to different allocations. /// assert_eq!(*data, 8); @@ -848,22 +1223,31 @@ impl Rc { /// assert!(76 == *data); /// assert!(weak.upgrade().is_none()); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "rc_unique", since = "1.4.0")] pub fn make_mut(this: &mut Self) -> &mut T { if Rc::strong_count(this) != 1 { - // Gotta clone the data, there are other Rcs - *this = Rc::new((**this).clone()) + // Gotta clone the data, there are other Rcs. + // Pre-allocate memory to allow writing the cloned value directly. + let mut rc = Self::new_uninit(); + unsafe { + let data = Rc::get_mut_unchecked(&mut rc); + (**this).write_clone_into_raw(data.as_mut_ptr()); + *this = rc.assume_init(); + } } else if Rc::weak_count(this) != 0 { // Can just steal the data, all that's left is Weaks + let mut rc = Self::new_uninit(); unsafe { - let mut swap = Rc::new(ptr::read(&this.ptr.as_ref().value)); - mem::swap(this, &mut swap); - swap.dec_strong(); + let data = Rc::get_mut_unchecked(&mut rc); + data.as_mut_ptr().copy_from_nonoverlapping(&**this, 1); + + this.inner().dec_strong(); // Remove implicit strong-weak ref (no need to craft a fake // Weak here -- we know other Weaks can clean up for us) - swap.dec_weak(); - forget(swap); + this.inner().dec_weak(); + ptr::write(this, rc.assume_init()); } } // This unsafety is ok because we're guaranteed that the pointer @@ -873,11 +1257,44 @@ impl Rc { // reference to the allocation. unsafe { &mut this.ptr.as_mut().value } } + + /// If we have the only reference to `T` then unwrap it. Otherwise, clone `T` and return the + /// clone. + /// + /// Assuming `rc_t` is of type `Rc`, this function is functionally equivalent to + /// `(*rc_t).clone()`, but will avoid cloning the inner value where possible. + /// + /// # Examples + /// + /// ``` + /// #![feature(arc_unwrap_or_clone)] + /// # use std::{ptr, rc::Rc}; + /// let inner = String::from("test"); + /// let ptr = inner.as_ptr(); + /// + /// let rc = Rc::new(inner); + /// let inner = Rc::unwrap_or_clone(rc); + /// // The inner value was not cloned + /// assert!(ptr::eq(ptr, inner.as_ptr())); + /// + /// let rc = Rc::new(inner); + /// let rc2 = rc.clone(); + /// let inner = Rc::unwrap_or_clone(rc); + /// // Because there were 2 references, we had to clone the inner value. + /// assert!(!ptr::eq(ptr, inner.as_ptr())); + /// // `rc2` is the last reference, so when we unwrap it we get back + /// // the original `String`. + /// let inner = Rc::unwrap_or_clone(rc2); + /// assert!(ptr::eq(ptr, inner.as_ptr())); + /// ``` + #[inline] + #[unstable(feature = "arc_unwrap_or_clone", issue = "93610")] + pub fn unwrap_or_clone(this: Self) -> T { + Rc::try_unwrap(this).unwrap_or_else(|rc| (*rc).clone()) + } } impl Rc { - #[inline] - #[stable(feature = "rc_downcast", since = "1.29.0")] /// Attempt to downcast the `Rc` to a concrete type. /// /// # Examples @@ -896,15 +1313,55 @@ impl Rc { /// print_if_string(Rc::new(my_string)); /// print_if_string(Rc::new(0i8)); /// ``` + #[inline] + #[stable(feature = "rc_downcast", since = "1.29.0")] pub fn downcast(self) -> Result, Rc> { if (*self).is::() { - let ptr = self.ptr.cast::>(); - forget(self); - Ok(Rc::from_inner(ptr)) + unsafe { + let ptr = self.ptr.cast::>(); + forget(self); + Ok(Rc::from_inner(ptr)) + } } else { Err(self) } } + + /// Downcasts the `Rc` to a concrete type. + /// + /// For a safe alternative see [`downcast`]. + /// + /// # Examples + /// + /// ``` + /// #![feature(downcast_unchecked)] + /// + /// use std::any::Any; + /// use std::rc::Rc; + /// + /// let x: Rc = Rc::new(1_usize); + /// + /// unsafe { + /// assert_eq!(*x.downcast_unchecked::(), 1); + /// } + /// ``` + /// + /// # Safety + /// + /// The contained value must be of type `T`. Calling this method + /// with the incorrect type is *undefined behavior*. + /// + /// + /// [`downcast`]: Self::downcast + #[inline] + #[unstable(feature = "downcast_unchecked", issue = "90850")] + pub unsafe fn downcast_unchecked(self) -> Rc { + unsafe { + let ptr = self.ptr.cast::>(); + mem::forget(self); + Rc::from_inner(ptr) + } + } } impl Rc { @@ -913,40 +1370,65 @@ impl Rc { /// /// The function `mem_to_rcbox` is called with the data pointer /// and must return back a (potentially fat)-pointer for the `RcBox`. + #[cfg(not(no_global_oom_handling))] unsafe fn allocate_for_layout( value_layout: Layout, + allocate: impl FnOnce(Layout) -> Result, AllocError>, mem_to_rcbox: impl FnOnce(*mut u8) -> *mut RcBox, ) -> *mut RcBox { - // Calculate layout using the given value layout. - // Previously, layout was calculated on the expression - // `&*(ptr as *const RcBox)`, but this created a misaligned - // reference (see #54908). - let layout = Layout::new::>().extend(value_layout).unwrap().0.pad_to_align(); + let layout = rcbox_layout_for_value_layout(value_layout); + unsafe { + Rc::try_allocate_for_layout(value_layout, allocate, mem_to_rcbox) + .unwrap_or_else(|_| handle_alloc_error(layout)) + } + } + + /// Allocates an `RcBox` with sufficient space for + /// a possibly-unsized inner value where the value has the layout provided, + /// returning an error if allocation fails. + /// + /// The function `mem_to_rcbox` is called with the data pointer + /// and must return back a (potentially fat)-pointer for the `RcBox`. + #[inline] + unsafe fn try_allocate_for_layout( + value_layout: Layout, + allocate: impl FnOnce(Layout) -> Result, AllocError>, + mem_to_rcbox: impl FnOnce(*mut u8) -> *mut RcBox, + ) -> Result<*mut RcBox, AllocError> { + let layout = rcbox_layout_for_value_layout(value_layout); // Allocate for the layout. - let (mem, _) = Global.alloc(layout).unwrap_or_else(|_| handle_alloc_error(layout)); + let ptr = allocate(layout)?; // Initialize the RcBox - let inner = mem_to_rcbox(mem.as_ptr()); - debug_assert_eq!(Layout::for_value(&*inner), layout); + let inner = mem_to_rcbox(ptr.as_non_null_ptr().as_ptr()); + unsafe { + debug_assert_eq!(Layout::for_value(&*inner), layout); - ptr::write(&mut (*inner).strong, Cell::new(1)); - ptr::write(&mut (*inner).weak, Cell::new(1)); + ptr::write(&mut (*inner).strong, Cell::new(1)); + ptr::write(&mut (*inner).weak, Cell::new(1)); + } - inner + Ok(inner) } /// Allocates an `RcBox` with sufficient space for an unsized inner value + #[cfg(not(no_global_oom_handling))] unsafe fn allocate_for_ptr(ptr: *const T) -> *mut RcBox { // Allocate for the `RcBox` using the given value. - Self::allocate_for_layout(Layout::for_value(&*ptr), |mem| { - set_data_ptr(ptr as *mut T, mem) as *mut RcBox - }) + unsafe { + Self::allocate_for_layout( + Layout::for_value(&*ptr), + |layout| Global.allocate(layout), + |mem| mem.with_metadata_of(ptr as *const RcBox), + ) + } } + #[cfg(not(no_global_oom_handling))] fn from_box(v: Box) -> Rc { unsafe { - let box_unique = Box::into_unique(v); + let (box_unique, alloc) = Box::into_unique(v); let bptr = box_unique.as_ptr(); let value_size = size_of_val(&*bptr); @@ -960,7 +1442,7 @@ impl Rc { ); // Free the allocation without dropping its contents - box_free(box_unique); + box_free(box_unique, alloc); Self::from_ptr(ptr) } @@ -969,37 +1451,33 @@ impl Rc { impl Rc<[T]> { /// Allocates an `RcBox<[T]>` with the given length. + #[cfg(not(no_global_oom_handling))] unsafe fn allocate_for_slice(len: usize) -> *mut RcBox<[T]> { - Self::allocate_for_layout(Layout::array::(len).unwrap(), |mem| { - ptr::slice_from_raw_parts_mut(mem as *mut T, len) as *mut RcBox<[T]> - }) + unsafe { + Self::allocate_for_layout( + Layout::array::(len).unwrap(), + |layout| Global.allocate(layout), + |mem| ptr::slice_from_raw_parts_mut(mem as *mut T, len) as *mut RcBox<[T]>, + ) + } } -} -/// Sets the data pointer of a `?Sized` raw pointer. -/// -/// For a slice/trait object, this sets the `data` field and leaves the rest -/// unchanged. For a sized raw pointer, this simply sets the pointer. -unsafe fn set_data_ptr(mut ptr: *mut T, data: *mut U) -> *mut T { - ptr::write(&mut ptr as *mut _ as *mut *mut u8, data as *mut u8); - ptr -} - -impl Rc<[T]> { - /// Copy elements from slice into newly allocated Rc<[T]> + /// Copy elements from slice into newly allocated `Rc<[T]>` /// /// Unsafe because the caller must either take ownership or bind `T: Copy` + #[cfg(not(no_global_oom_handling))] unsafe fn copy_from_slice(v: &[T]) -> Rc<[T]> { - let ptr = Self::allocate_for_slice(v.len()); - - ptr::copy_nonoverlapping(v.as_ptr(), &mut (*ptr).value as *mut [T] as *mut T, v.len()); - - Self::from_ptr(ptr) + unsafe { + let ptr = Self::allocate_for_slice(v.len()); + ptr::copy_nonoverlapping(v.as_ptr(), &mut (*ptr).value as *mut [T] as *mut T, v.len()); + Self::from_ptr(ptr) + } } /// Constructs an `Rc<[T]>` from an iterator known to be of a certain size. /// /// Behavior is undefined should the size be wrong. + #[cfg(not(no_global_oom_handling))] unsafe fn from_iter_exact(iter: impl iter::Iterator, len: usize) -> Rc<[T]> { // Panic guard while cloning T elements. // In the event of a panic, elements that have been written @@ -1017,30 +1495,32 @@ impl Rc<[T]> { let slice = from_raw_parts_mut(self.elems, self.n_elems); ptr::drop_in_place(slice); - // Crux: we should deallocate here, but we don't support `deallocate` yet. + Global.deallocate(self.mem, self.layout); } } } - let ptr = Self::allocate_for_slice(len); + unsafe { + let ptr = Self::allocate_for_slice(len); - let mem = ptr as *mut _ as *mut u8; - let layout = Layout::for_value(&*ptr); + let mem = ptr as *mut _ as *mut u8; + let layout = Layout::for_value(&*ptr); - // Pointer to first element - let elems = &mut (*ptr).value as *mut [T] as *mut T; + // Pointer to first element + let elems = &mut (*ptr).value as *mut [T] as *mut T; - let mut guard = Guard { mem: NonNull::new_unchecked(mem), elems, layout, n_elems: 0 }; + let mut guard = Guard { mem: NonNull::new_unchecked(mem), elems, layout, n_elems: 0 }; - for (i, item) in iter.enumerate() { - ptr::write(elems.add(i), item); - guard.n_elems += 1; - } + for (i, item) in iter.enumerate() { + ptr::write(elems.add(i), item); + guard.n_elems += 1; + } - // All clear. Forget the guard so it doesn't free the new RcBox. - forget(guard); + // All clear. Forget the guard so it doesn't free the new RcBox. + forget(guard); - Self::from_ptr(ptr) + Self::from_ptr(ptr) + } } } @@ -1049,6 +1529,7 @@ trait RcFromSlice { fn from_slice(slice: &[T]) -> Self; } +#[cfg(not(no_global_oom_handling))] impl RcFromSlice for Rc<[T]> { #[inline] default fn from_slice(v: &[T]) -> Self { @@ -1056,6 +1537,7 @@ impl RcFromSlice for Rc<[T]> { } } +#[cfg(not(no_global_oom_handling))] impl RcFromSlice for Rc<[T]> { #[inline] fn from_slice(v: &[T]) -> Self { @@ -1103,21 +1585,19 @@ unsafe impl<#[may_dangle] T: ?Sized> Drop for Rc { /// drop(foo); // Doesn't print anything /// drop(foo2); // Prints "dropped!" /// ``` - /// - /// [`Weak`]: ../../std/rc/struct.Weak.html fn drop(&mut self) { unsafe { - self.dec_strong(); - if self.strong() == 0 { + self.inner().dec_strong(); + if self.inner().strong() == 0 { // destroy the contained object - ptr::drop_in_place(self.ptr.as_mut()); + ptr::drop_in_place(Self::get_mut_unchecked(self)); // remove the implicit "strong weak" pointer now that we've // destroyed the contents. - self.dec_weak(); + self.inner().dec_weak(); - if self.weak() == 0 { - // Crux: we should deallocate here, but we don't support `deallocate` yet. + if self.inner().weak() == 0 { + Global.deallocate(self.ptr.cast(), Layout::for_value(self.ptr.as_ref())); } } } @@ -1142,11 +1622,14 @@ impl Clone for Rc { /// ``` #[inline] fn clone(&self) -> Rc { - self.inc_strong(); - Self::from_inner(self.ptr) + unsafe { + self.inner().inc_strong(); + Self::from_inner(self.ptr) + } } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl Default for Rc { /// Creates a new `Rc`, with the `Default` value for `T`. @@ -1184,6 +1667,12 @@ impl RcEqIdent for Rc { } } +// Hack to allow specializing on `Eq` even though `Eq` has a method. +#[rustc_unsafe_specialization_marker] +pub(crate) trait MarkerEq: PartialEq {} + +impl MarkerEq for T {} + /// We're doing this specialization here, and not as a more general optimization on `&T`, because it /// would otherwise add a cost to all equality checks on refs. We assume that `Rc`s are used to /// store large values, that are slow to clone, but also heavy to check for equality, causing this @@ -1192,7 +1681,7 @@ impl RcEqIdent for Rc { /// /// We can only do this when `T: Eq` as a `PartialEq` might be deliberately irreflexive. #[stable(feature = "rust1", since = "1.0.0")] -impl RcEqIdent for Rc { +impl RcEqIdent for Rc { #[inline] fn eq(&self, other: &Rc) -> bool { Rc::ptr_eq(self, other) || **self == **other @@ -1399,23 +1888,58 @@ impl fmt::Pointer for Rc { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "from_for_ptrs", since = "1.6.0")] impl From for Rc { + /// Converts a generic type `T` into an `Rc` + /// + /// The conversion allocates on the heap and moves `t` + /// from the stack into it. + /// + /// # Example + /// ```rust + /// # use std::rc::Rc; + /// let x = 5; + /// let rc = Rc::new(5); + /// + /// assert_eq!(Rc::from(x), rc); + /// ``` fn from(t: T) -> Self { Rc::new(t) } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_slice", since = "1.21.0")] impl From<&[T]> for Rc<[T]> { + /// Allocate a reference-counted slice and fill it by cloning `v`'s items. + /// + /// # Example + /// + /// ``` + /// # use std::rc::Rc; + /// let original: &[i32] = &[1, 2, 3]; + /// let shared: Rc<[i32]> = Rc::from(original); + /// assert_eq!(&[1, 2, 3], &shared[..]); + /// ``` #[inline] fn from(v: &[T]) -> Rc<[T]> { >::from_slice(v) } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_slice", since = "1.21.0")] impl From<&str> for Rc { + /// Allocate a reference-counted string slice and copy `v` into it. + /// + /// # Example + /// + /// ``` + /// # use std::rc::Rc; + /// let shared: Rc = Rc::from("statue"); + /// assert_eq!("statue", &shared[..]); + /// ``` #[inline] fn from(v: &str) -> Rc { let rc = Rc::<[u8]>::from(v.as_bytes()); @@ -1423,42 +1947,116 @@ impl From<&str> for Rc { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_slice", since = "1.21.0")] impl From for Rc { + /// Allocate a reference-counted string slice and copy `v` into it. + /// + /// # Example + /// + /// ``` + /// # use std::rc::Rc; + /// let original: String = "statue".to_owned(); + /// let shared: Rc = Rc::from(original); + /// assert_eq!("statue", &shared[..]); + /// ``` #[inline] fn from(v: String) -> Rc { Rc::from(&v[..]) } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_slice", since = "1.21.0")] impl From> for Rc { + /// Move a boxed object to a new, reference counted, allocation. + /// + /// # Example + /// + /// ``` + /// # use std::rc::Rc; + /// let original: Box = Box::new(1); + /// let shared: Rc = Rc::from(original); + /// assert_eq!(1, *shared); + /// ``` #[inline] fn from(v: Box) -> Rc { Rc::from_box(v) } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_slice", since = "1.21.0")] impl From> for Rc<[T]> { + /// Allocate a reference-counted slice and move `v`'s items into it. + /// + /// # Example + /// + /// ``` + /// # use std::rc::Rc; + /// let original: Box> = Box::new(vec![1, 2, 3]); + /// let shared: Rc> = Rc::from(original); + /// assert_eq!(vec![1, 2, 3], *shared); + /// ``` #[inline] fn from(mut v: Vec) -> Rc<[T]> { unsafe { let rc = Rc::copy_from_slice(&v); - // Allow the Vec to free its memory, but not destroy its contents v.set_len(0); - rc } } } -#[stable(feature = "boxed_slice_try_from", since = "1.43.0")] -impl TryFrom> for Rc<[T; N]> +#[stable(feature = "shared_from_cow", since = "1.45.0")] +impl<'a, B> From> for Rc where - [T; N]: LengthAtMost32, + B: ToOwned + ?Sized, + Rc: From<&'a B> + From, { + /// Create a reference-counted pointer from + /// a clone-on-write pointer by copying its content. + /// + /// # Example + /// + /// ```rust + /// # use std::rc::Rc; + /// # use std::borrow::Cow; + /// let cow: Cow = Cow::Borrowed("eggplant"); + /// let shared: Rc = Rc::from(cow); + /// assert_eq!("eggplant", &shared[..]); + /// ``` + #[inline] + fn from(cow: Cow<'a, B>) -> Rc { + match cow { + Cow::Borrowed(s) => Rc::from(s), + Cow::Owned(s) => Rc::from(s), + } + } +} + +#[stable(feature = "shared_from_str", since = "1.62.0")] +impl From> for Rc<[u8]> { + /// Converts a reference-counted string slice into a byte slice. + /// + /// # Example + /// + /// ``` + /// # use std::rc::Rc; + /// let string: Rc = Rc::from("eggplant"); + /// let bytes: Rc<[u8]> = Rc::from(string); + /// assert_eq!("eggplant".as_bytes(), bytes.as_ref()); + /// ``` + #[inline] + fn from(rc: Rc) -> Self { + // SAFETY: `str` has the same layout as `[u8]`. + unsafe { Rc::from_raw(Rc::into_raw(rc) as *const [u8]) } + } +} + +#[stable(feature = "boxed_slice_try_from", since = "1.43.0")] +impl TryFrom> for Rc<[T; N]> { type Error = Rc<[T]>; fn try_from(boxed_slice: Rc<[T]>) -> Result { @@ -1470,6 +2068,7 @@ where } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_iter", since = "1.37.0")] impl iter::FromIterator for Rc<[T]> { /// Takes each element in the `Iterator` and collects it into an `Rc<[T]>`. @@ -1511,25 +2110,28 @@ impl iter::FromIterator for Rc<[T]> { /// # assert_eq!(&*evens, &*(0..10).collect::>()); /// ``` fn from_iter>(iter: I) -> Self { - RcFromIter::from_iter(iter.into_iter()) + ToRcSlice::to_rc_slice(iter.into_iter()) } } /// Specialization trait used for collecting into `Rc<[T]>`. -trait RcFromIter { - fn from_iter(iter: I) -> Self; +#[cfg(not(no_global_oom_handling))] +trait ToRcSlice: Iterator + Sized { + fn to_rc_slice(self) -> Rc<[T]>; } -impl> RcFromIter for Rc<[T]> { - default fn from_iter(iter: I) -> Self { - iter.collect::>().into() +#[cfg(not(no_global_oom_handling))] +impl> ToRcSlice for I { + default fn to_rc_slice(self) -> Rc<[T]> { + self.collect::>().into() } } -impl> RcFromIter for Rc<[T]> { - default fn from_iter(iter: I) -> Self { +#[cfg(not(no_global_oom_handling))] +impl> ToRcSlice for I { + fn to_rc_slice(self) -> Rc<[T]> { // This is the case for a `TrustedLen` iterator. - let (low, high) = iter.size_hint(); + let (low, high) = self.size_hint(); if let Some(high) = high { debug_assert_eq!( low, @@ -1540,32 +2142,21 @@ impl> RcFromIter for Rc<[T]> { unsafe { // SAFETY: We need to ensure that the iterator has an exact length and we have. - Rc::from_iter_exact(iter, low) + Rc::from_iter_exact(self, low) } } else { - // Fall back to normal implementation. - iter.collect::>().into() + // TrustedLen contract guarantees that `upper_bound == `None` implies an iterator + // length exceeding `usize::MAX`. + // The default implementation would collect into a vec which would panic. + // Thus we panic here immediately without invoking `Vec` code. + panic!("capacity overflow"); } } } -impl<'a, T: 'a + Clone> RcFromIter<&'a T, slice::Iter<'a, T>> for Rc<[T]> { - fn from_iter(iter: slice::Iter<'a, T>) -> Self { - // Delegate to `impl From<&[T]> for Rc<[T]>`. - // - // In the case that `T: Copy`, we get to use `ptr::copy_nonoverlapping` - // which is even more performant. - // - // In the fall-back case we have `T: Clone`. This is still better - // than the `TrustedLen` implementation as slices have a known length - // and so we get to avoid calling `size_hint` and avoid the branching. - iter.as_slice().into() - } -} - /// `Weak` is a version of [`Rc`] that holds a non-owning reference to the /// managed allocation. The allocation is accessed by calling [`upgrade`] on the `Weak` -/// pointer, which returns an [`Option`]`<`[`Rc`]`>`. +/// pointer, which returns an [Option]<[Rc]\>. /// /// Since a `Weak` reference does not count towards ownership, it will not /// prevent the value stored in the allocation from being dropped, and `Weak` itself makes no @@ -1582,18 +2173,15 @@ impl<'a, T: 'a + Clone> RcFromIter<&'a T, slice::Iter<'a, T>> for Rc<[T]> { /// /// The typical way to obtain a `Weak` pointer is to call [`Rc::downgrade`]. /// -/// [`Rc`]: struct.Rc.html -/// [`Rc::downgrade`]: struct.Rc.html#method.downgrade -/// [`upgrade`]: struct.Weak.html#method.upgrade -/// [`Option`]: ../../std/option/enum.Option.html -/// [`None`]: ../../std/option/enum.Option.html#variant.None +/// [`upgrade`]: Weak::upgrade #[stable(feature = "rc_weak", since = "1.4.0")] pub struct Weak { // This is a `NonNull` to allow optimizing the size of this type in enums, // but it is not necessarily a valid pointer. // `Weak::new` sets this to `usize::MAX` so that it doesn’t need - // to allocate space on the heap. That's not a value a real pointer + // to allocate space on the heap. That's not a value a real pointer // will ever have because RcBox has alignment at least 2. + // This is only possible when `T: Sized`; unsized `T` never dangle. ptr: NonNull>, } @@ -1602,7 +2190,7 @@ impl !marker::Send for Weak {} #[stable(feature = "rc_weak", since = "1.4.0")] impl !marker::Sync for Weak {} -#[unstable(feature = "coerce_unsized", issue = "27732")] +#[unstable(feature = "coerce_unsized", issue = "18598")] impl, U: ?Sized> CoerceUnsized> for Weak {} #[unstable(feature = "dispatch_from_dyn", issue = "none")] @@ -1612,8 +2200,7 @@ impl Weak { /// Constructs a new `Weak`, without allocating any memory. /// Calling [`upgrade`] on the return value always gives [`None`]. /// - /// [`upgrade`]: #method.upgrade - /// [`None`]: ../../std/option/enum.Option.html + /// [`upgrade`]: Weak::upgrade /// /// # Examples /// @@ -1624,66 +2211,79 @@ impl Weak { /// assert!(empty.upgrade().is_none()); /// ``` #[stable(feature = "downgraded_weak", since = "1.10.0")] - pub fn new() -> Weak { - Weak { ptr: NonNull::new(usize::MAX as *mut RcBox).expect("MAX is not 0") } + #[rustc_const_unstable(feature = "const_weak_new", issue = "95091", reason = "recently added")] + #[must_use] + pub const fn new() -> Weak { + Weak { ptr: unsafe { NonNull::new_unchecked(ptr::invalid_mut::>(usize::MAX)) } } } +} + +pub(crate) fn is_dangling(ptr: *mut T) -> bool { + (ptr as *mut ()).addr() == usize::MAX +} +/// Helper type to allow accessing the reference counts without +/// making any assertions about the data field. +struct WeakInner<'a> { + weak: &'a Cell, + strong: &'a Cell, +} + +impl Weak { /// Returns a raw pointer to the object `T` pointed to by this `Weak`. /// - /// The pointer is valid only if there are some strong references. The pointer may be dangling - /// or even [`null`] otherwise. + /// The pointer is valid only if there are some strong references. The pointer may be dangling, + /// unaligned or even [`null`] otherwise. /// /// # Examples /// /// ``` - /// #![feature(weak_into_raw)] - /// /// use std::rc::Rc; /// use std::ptr; /// /// let strong = Rc::new("hello".to_owned()); /// let weak = Rc::downgrade(&strong); /// // Both point to the same object - /// assert!(ptr::eq(&*strong, weak.as_raw())); + /// assert!(ptr::eq(&*strong, weak.as_ptr())); /// // The strong here keeps it alive, so we can still access the object. - /// assert_eq!("hello", unsafe { &*weak.as_raw() }); + /// assert_eq!("hello", unsafe { &*weak.as_ptr() }); /// /// drop(strong); - /// // But not any more. We can do weak.as_raw(), but accessing the pointer would lead to + /// // But not any more. We can do weak.as_ptr(), but accessing the pointer would lead to /// // undefined behaviour. - /// // assert_eq!("hello", unsafe { &*weak.as_raw() }); - /// ``` - /// - /// [`null`]: ../../std/ptr/fn.null.html - #[unstable(feature = "weak_into_raw", issue = "60728")] - pub fn as_raw(&self) -> *const T { - match self.inner() { - None => ptr::null(), - Some(inner) => { - let offset = data_offset_sized::(); - let ptr = inner as *const RcBox; - // Note: while the pointer we create may already point to dropped value, the - // allocation still lives (it must hold the weak point as long as we are alive). - // Therefore, the offset is OK to do, it won't get out of the allocation. - let ptr = unsafe { (ptr as *const u8).offset(offset) }; - ptr as *const T - } + /// // assert_eq!("hello", unsafe { &*weak.as_ptr() }); + /// ``` + /// + /// [`null`]: ptr::null + #[must_use] + #[stable(feature = "rc_as_ptr", since = "1.45.0")] + pub fn as_ptr(&self) -> *const T { + let ptr: *mut RcBox = NonNull::as_ptr(self.ptr); + + if is_dangling(ptr) { + // If the pointer is dangling, we return the sentinel directly. This cannot be + // a valid payload address, as the payload is at least as aligned as RcBox (usize). + ptr as *const T + } else { + // SAFETY: if is_dangling returns false, then the pointer is dereferenceable. + // The payload may be dropped at this point, and we have to maintain provenance, + // so use raw pointer manipulation. + unsafe { ptr::addr_of_mut!((*ptr).value) } } } /// Consumes the `Weak` and turns it into a raw pointer. /// - /// This converts the weak pointer into a raw pointer, preserving the original weak count. It - /// can be turned back into the `Weak` with [`from_raw`]. + /// This converts the weak pointer into a raw pointer, while still preserving the ownership of + /// one weak reference (the weak count is not modified by this operation). It can be turned + /// back into the `Weak` with [`from_raw`]. /// /// The same restrictions of accessing the target of the pointer as with - /// [`as_raw`] apply. + /// [`as_ptr`] apply. /// /// # Examples /// /// ``` - /// #![feature(weak_into_raw)] - /// /// use std::rc::{Rc, Weak}; /// /// let strong = Rc::new("hello".to_owned()); @@ -1697,11 +2297,12 @@ impl Weak { /// assert_eq!(0, Rc::weak_count(&strong)); /// ``` /// - /// [`from_raw`]: struct.Weak.html#method.from_raw - /// [`as_raw`]: struct.Weak.html#method.as_raw - #[unstable(feature = "weak_into_raw", issue = "60728")] + /// [`from_raw`]: Weak::from_raw + /// [`as_ptr`]: Weak::as_ptr + #[must_use = "`self` will be dropped if the result is not used"] + #[stable(feature = "weak_into_raw", since = "1.45.0")] pub fn into_raw(self) -> *const T { - let result = self.as_raw(); + let result = self.as_ptr(); mem::forget(self); result } @@ -1711,24 +2312,22 @@ impl Weak { /// This can be used to safely get a strong reference (by calling [`upgrade`] /// later) or to deallocate the weak count by dropping the `Weak`. /// - /// It takes ownership of one weak count (with the exception of pointers created by [`new`], - /// as these don't have any corresponding weak count). + /// It takes ownership of one weak reference (with the exception of pointers created by [`new`], + /// as these don't own anything; the method still works on them). /// /// # Safety /// - /// The pointer must have originated from the [`into_raw`] (or [`as_raw`], provided there was - /// a corresponding [`forget`] on the `Weak`) and must still own its potential weak reference - /// count. + /// The pointer must have originated from the [`into_raw`] and must still own its potential + /// weak reference. /// - /// It is allowed for the strong count to be 0 at the time of calling this, but the weak count - /// must be non-zero or the pointer must have originated from a dangling `Weak` (one created - /// by [`new`]). + /// It is allowed for the strong count to be 0 at the time of calling this. Nevertheless, this + /// takes ownership of one weak reference currently represented as a raw pointer (the weak + /// count is not modified by this operation) and therefore it must be paired with a previous + /// call to [`into_raw`]. /// /// # Examples /// /// ``` - /// #![feature(weak_into_raw)] - /// /// use std::rc::{Rc, Weak}; /// /// let strong = Rc::new("hello".to_owned()); @@ -1747,40 +2346,34 @@ impl Weak { /// assert!(unsafe { Weak::from_raw(raw_2) }.upgrade().is_none()); /// ``` /// - /// [`into_raw`]: struct.Weak.html#method.into_raw - /// [`upgrade`]: struct.Weak.html#method.upgrade - /// [`Rc`]: struct.Rc.html - /// [`Weak`]: struct.Weak.html - /// [`as_raw`]: struct.Weak.html#method.as_raw - /// [`new`]: struct.Weak.html#method.new - /// [`forget`]: ../../std/mem/fn.forget.html - #[unstable(feature = "weak_into_raw", issue = "60728")] + /// [`into_raw`]: Weak::into_raw + /// [`upgrade`]: Weak::upgrade + /// [`new`]: Weak::new + #[stable(feature = "weak_into_raw", since = "1.45.0")] pub unsafe fn from_raw(ptr: *const T) -> Self { - if ptr.is_null() { - Self::new() + // See Weak::as_ptr for context on how the input pointer is derived. + + let ptr = if is_dangling(ptr as *mut T) { + // This is a dangling Weak. + ptr as *mut RcBox } else { - // See Rc::from_raw for details - let offset = data_offset(ptr); - let fake_ptr = ptr as *mut RcBox; - let ptr = set_data_ptr(fake_ptr, (ptr as *mut u8).offset(-offset)); - Weak { ptr: NonNull::new(ptr).expect("Invalid pointer passed to from_raw") } - } - } -} + // Otherwise, we're guaranteed the pointer came from a nondangling Weak. + // SAFETY: data_offset is safe to call, as ptr references a real (potentially dropped) T. + let offset = unsafe { data_offset(ptr) }; + // Thus, we reverse the offset to get the whole RcBox. + // SAFETY: the pointer originated from a Weak, so this offset is safe. + unsafe { ptr.byte_sub(offset) as *mut RcBox } + }; -pub(crate) fn is_dangling(ptr: NonNull) -> bool { - crucible::ptr::compare_usize(ptr.as_ptr(), usize::MAX) -} + // SAFETY: we now have recovered the original Weak pointer, so can create the Weak. + Weak { ptr: unsafe { NonNull::new_unchecked(ptr) } } + } -impl Weak { /// Attempts to upgrade the `Weak` pointer to an [`Rc`], delaying /// dropping of the inner value if successful. /// /// Returns [`None`] if the inner value has since been dropped. /// - /// [`Rc`]: struct.Rc.html - /// [`None`]: ../../std/option/enum.Option.html - /// /// # Examples /// /// ``` @@ -1799,22 +2392,26 @@ impl Weak { /// /// assert!(weak_five.upgrade().is_none()); /// ``` + #[must_use = "this returns a new `Rc`, \ + without modifying the original weak pointer"] #[stable(feature = "rc_weak", since = "1.4.0")] pub fn upgrade(&self) -> Option> { let inner = self.inner()?; + if inner.strong() == 0 { None } else { - inner.inc_strong(); - Some(Rc::from_inner(self.ptr)) + unsafe { + inner.inc_strong(); + Some(Rc::from_inner(self.ptr)) + } } } /// Gets the number of strong (`Rc`) pointers pointing to this allocation. /// /// If `self` was created using [`Weak::new`], this will return 0. - /// - /// [`Weak::new`]: #method.new + #[must_use] #[stable(feature = "weak_counts", since = "1.41.0")] pub fn strong_count(&self) -> usize { if let Some(inner) = self.inner() { inner.strong() } else { 0 } @@ -1823,6 +2420,7 @@ impl Weak { /// Gets the number of `Weak` pointers pointing to this allocation. /// /// If no strong pointers remain, this will return zero. + #[must_use] #[stable(feature = "weak_counts", since = "1.41.0")] pub fn weak_count(&self) -> usize { self.inner() @@ -1836,16 +2434,26 @@ impl Weak { .unwrap_or(0) } - /// Returns `None` when the pointer is dangling and there is no allocated `RcBox` + /// Returns `None` when the pointer is dangling and there is no allocated `RcBox`, /// (i.e., when this `Weak` was created by `Weak::new`). #[inline] - fn inner(&self) -> Option<&RcBox> { - if is_dangling(self.ptr) { None } else { Some(unsafe { self.ptr.as_ref() }) } + fn inner(&self) -> Option> { + if is_dangling(self.ptr.as_ptr()) { + None + } else { + // We are careful to *not* create a reference covering the "data" field, as + // the field may be mutated concurrently (for example, if the last `Rc` + // is dropped, the data field will be dropped in-place). + Some(unsafe { + let ptr = self.ptr.as_ptr(); + WeakInner { strong: &(*ptr).strong, weak: &(*ptr).weak } + }) + } } - /// Returns `true` if the two `Weak`s point to the same allocation (similar to - /// [`ptr::eq`]), or if both don't point to any allocation - /// (because they were created with `Weak::new()`). + /// Returns `true` if the two `Weak`s point to the same allocation similar to [`ptr::eq`], or if + /// both don't point to any allocation (because they were created with `Weak::new()`). See [that + /// function][`ptr::eq`] for caveats when comparing `dyn Trait` pointers. /// /// # Notes /// @@ -1882,9 +2490,8 @@ impl Weak { /// let third = Rc::downgrade(&third_rc); /// assert!(!first.ptr_eq(&third)); /// ``` - /// - /// [`ptr::eq`]: ../../std/ptr/fn.eq.html #[inline] + #[must_use] #[stable(feature = "weak_ptr_eq", since = "1.39.0")] pub fn ptr_eq(&self, other: &Self) -> bool { self.ptr.as_ptr() == other.ptr.as_ptr() @@ -1892,7 +2499,7 @@ impl Weak { } #[stable(feature = "rc_weak", since = "1.4.0")] -impl Drop for Weak { +unsafe impl<#[may_dangle] T: ?Sized> Drop for Weak { /// Drops the `Weak` pointer. /// /// # Examples @@ -1918,14 +2525,14 @@ impl Drop for Weak { /// assert!(other_weak_foo.upgrade().is_none()); /// ``` fn drop(&mut self) { - if let Some(inner) = self.inner() { - inner.dec_weak(); - // the weak count starts at 1, and will only go to zero if all - // the strong pointers have disappeared. - if inner.weak() == 0 { - unsafe { - // Crux: we should deallocate here, but we don't support `deallocate` yet. - } + let inner = if let Some(inner) = self.inner() { inner } else { return }; + + inner.dec_weak(); + // the weak count starts at 1, and will only go to zero if all + // the strong pointers have disappeared. + if inner.weak() == 0 { + unsafe { + Global.deallocate(self.ptr.cast(), Layout::for_value_raw(self.ptr.as_ptr())); } } } @@ -1954,7 +2561,7 @@ impl Clone for Weak { } #[stable(feature = "rc_weak", since = "1.4.0")] -impl fmt::Debug for Weak { +impl fmt::Debug for Weak { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "(Weak)") } @@ -1962,11 +2569,10 @@ impl fmt::Debug for Weak { #[stable(feature = "downgraded_weak", since = "1.10.0")] impl Default for Weak { - /// Constructs a new `Weak`, allocating memory for `T` without initializing - /// it. Calling [`upgrade`] on the return value always gives [`None`]. + /// Constructs a new `Weak`, without allocating any memory. + /// Calling [`upgrade`] on the return value always gives [`None`]. /// - /// [`None`]: ../../std/option/enum.Option.html - /// [`upgrade`]: ../../std/rc/struct.Weak.html#method.upgrade + /// [`upgrade`]: Weak::upgrade /// /// # Examples /// @@ -1991,73 +2597,98 @@ impl Default for Weak { // clone these much in Rust thanks to ownership and move-semantics. #[doc(hidden)] -trait RcBoxPtr { - fn inner(&self) -> &RcBox; +trait RcInnerPtr { + fn weak_ref(&self) -> &Cell; + fn strong_ref(&self) -> &Cell; #[inline] fn strong(&self) -> usize { - self.inner().strong.get() + self.strong_ref().get() } #[inline] fn inc_strong(&self) { let strong = self.strong(); + // We insert an `assume` here to hint LLVM at an otherwise + // missed optimization. + // SAFETY: The reference count will never be zero when this is + // called. + unsafe { + core::intrinsics::assume(strong != 0); + } + + let strong = strong.wrapping_add(1); + self.strong_ref().set(strong); + // We want to abort on overflow instead of dropping the value. - // The reference count will never be zero when this is called; - // nevertheless, we insert an abort here to hint LLVM at - // an otherwise missed optimization. - if strong == 0 || strong == usize::max_value() { - unsafe { - abort(); - } + // Checking for overflow after the store instead of before + // allows for slightly better code generation. + if core::intrinsics::unlikely(strong == 0) { + abort(); } - self.inner().strong.set(strong + 1); } #[inline] fn dec_strong(&self) { - self.inner().strong.set(self.strong() - 1); + self.strong_ref().set(self.strong() - 1); } #[inline] fn weak(&self) -> usize { - self.inner().weak.get() + self.weak_ref().get() } #[inline] fn inc_weak(&self) { let weak = self.weak(); + // We insert an `assume` here to hint LLVM at an otherwise + // missed optimization. + // SAFETY: The reference count will never be zero when this is + // called. + unsafe { + core::intrinsics::assume(weak != 0); + } + + let weak = weak.wrapping_add(1); + self.weak_ref().set(weak); + // We want to abort on overflow instead of dropping the value. - // The reference count will never be zero when this is called; - // nevertheless, we insert an abort here to hint LLVM at - // an otherwise missed optimization. - if weak == 0 || weak == usize::max_value() { - unsafe { - abort(); - } + // Checking for overflow after the store instead of before + // allows for slightly better code generation. + if core::intrinsics::unlikely(weak == 0) { + abort(); } - self.inner().weak.set(weak + 1); } #[inline] fn dec_weak(&self) { - self.inner().weak.set(self.weak() - 1); + self.weak_ref().set(self.weak() - 1); } } -impl RcBoxPtr for Rc { +impl RcInnerPtr for RcBox { #[inline(always)] - fn inner(&self) -> &RcBox { - unsafe { self.ptr.as_ref() } + fn weak_ref(&self) -> &Cell { + &self.weak + } + + #[inline(always)] + fn strong_ref(&self) -> &Cell { + &self.strong } } -impl RcBoxPtr for RcBox { +impl<'a> RcInnerPtr for WeakInner<'a> { #[inline(always)] - fn inner(&self) -> &RcBox { - self + fn weak_ref(&self) -> &Cell { + self.weak + } + + #[inline(always)] + fn strong_ref(&self) -> &Cell { + self.strong } } @@ -2078,23 +2709,24 @@ impl AsRef for Rc { #[stable(feature = "pin", since = "1.33.0")] impl Unpin for Rc {} -unsafe fn data_offset(ptr: *const T) -> isize { - // Align the unsized value to the end of the `RcBox`. - // Because it is ?Sized, it will always be the last field in memory. - // Note: This is a detail of the current implementation of the compiler, - // and is not a guaranteed language detail. Do not rely on it outside of std. - data_offset_align(align_of_val(&*ptr)) -} - -/// Computes the offset of the data field within `RcBox`. +/// Get the offset within an `RcBox` for the payload behind a pointer. +/// +/// # Safety /// -/// Unlike [`data_offset`], this doesn't need the pointer, but it works only on `T: Sized`. -fn data_offset_sized() -> isize { - data_offset_align(align_of::()) +/// The pointer must point to (and have valid metadata for) a previously +/// valid instance of T, but the T is allowed to be dropped. +unsafe fn data_offset(ptr: *const T) -> usize { + // Align the unsized value to the end of the RcBox. + // Because RcBox is repr(C), it will always be the last field in memory. + // SAFETY: since the only unsized types possible are slices, trait objects, + // and extern types, the input safety requirement is currently enough to + // satisfy the requirements of align_of_val_raw; this is an implementation + // detail of the language that must not be relied upon outside of std. + unsafe { data_offset_align(align_of_val_raw(ptr)) } } #[inline] -fn data_offset_align(align: usize) -> isize { +fn data_offset_align(align: usize) -> usize { let layout = Layout::new::>(); - (layout.size() + layout.padding_needed_for(align)) as isize + layout.size() + layout.padding_needed_for(align) } diff --git a/crux-mir/lib/alloc/src/rc/tests.rs b/crux-mir/lib/alloc/src/rc/tests.rs index 56788bb56..32433cfbd 100644 --- a/crux-mir/lib/alloc/src/rc/tests.rs +++ b/crux-mir/lib/alloc/src/rc/tests.rs @@ -32,7 +32,7 @@ fn test_simple_clone() { #[test] fn test_destructor() { - let x: Rc> = Rc::new(box 5); + let x: Rc> = Rc::new(Box::new(5)); assert_eq!(**x, 5); } @@ -153,7 +153,7 @@ fn try_unwrap() { #[test] fn into_from_raw() { - let x = Rc::new(box "hello"); + let x = Rc::new(Box::new("hello")); let y = x.clone(); let x_ptr = Rc::into_raw(x); @@ -190,6 +190,48 @@ fn test_into_from_raw_unsized() { assert_eq!(rc2.to_string(), "123"); } +#[test] +fn into_from_weak_raw() { + let x = Rc::new(Box::new("hello")); + let y = Rc::downgrade(&x); + + let y_ptr = Weak::into_raw(y); + unsafe { + assert_eq!(**y_ptr, "hello"); + + let y = Weak::from_raw(y_ptr); + let y_up = Weak::upgrade(&y).unwrap(); + assert_eq!(**y_up, "hello"); + drop(y_up); + + assert_eq!(Rc::try_unwrap(x).map(|x| *x), Ok("hello")); + } +} + +#[test] +fn test_into_from_weak_raw_unsized() { + use std::fmt::Display; + use std::string::ToString; + + let arc: Rc = Rc::from("foo"); + let weak: Weak = Rc::downgrade(&arc); + + let ptr = Weak::into_raw(weak.clone()); + let weak2 = unsafe { Weak::from_raw(ptr) }; + + assert_eq!(unsafe { &*ptr }, "foo"); + assert!(weak.ptr_eq(&weak2)); + + let arc: Rc = Rc::new(123); + let weak: Weak = Rc::downgrade(&arc); + + let ptr = Weak::into_raw(weak.clone()); + let weak2 = unsafe { Weak::from_raw(ptr) }; + + assert_eq!(unsafe { &*ptr }.to_string(), "123"); + assert!(weak.ptr_eq(&weak2)); +} + #[test] fn get_mut() { let mut x = Rc::new(3); @@ -267,7 +309,7 @@ fn test_cowrc_clone_weak() { #[test] fn test_show() { let foo = Rc::new(75); - assert_eq!(format!("{:?}", foo), "75"); + assert_eq!(format!("{foo:?}"), "75"); } #[test] @@ -276,6 +318,23 @@ fn test_unsized() { assert_eq!(foo, foo.clone()); } +#[test] +fn test_maybe_thin_unsized() { + // If/when custom thin DSTs exist, this test should be updated to use one + use std::ffi::{CStr, CString}; + + let x: Rc = Rc::from(CString::new("swordfish").unwrap().into_boxed_c_str()); + assert_eq!(format!("{x:?}"), "\"swordfish\""); + let y: Weak = Rc::downgrade(&x); + drop(x); + + // At this point, the weak points to a dropped DST + assert!(y.upgrade().is_none()); + // But we still need to be able to get the alloc layout to drop. + // CStr has no drop glue, but custom DSTs might, and need to work. + drop(y); +} + #[test] fn test_from_owned() { let foo = 123; @@ -350,7 +409,7 @@ fn test_clone_from_slice_panic() { #[test] fn test_from_box() { - let b: Box = box 123; + let b: Box = Box::new(123); let r: Rc = Rc::from(b); assert_eq!(*r, 123); @@ -379,7 +438,7 @@ fn test_from_box_trait() { use std::fmt::Display; use std::string::ToString; - let b: Box = box 123; + let b: Box = Box::new(123); let r: Rc = Rc::from(b); assert_eq!(r.to_string(), "123"); @@ -389,10 +448,10 @@ fn test_from_box_trait() { fn test_from_box_trait_zero_sized() { use std::fmt::Debug; - let b: Box = box (); + let b: Box = Box::new(()); let r: Rc = Rc::from(b); - assert_eq!(format!("{:?}", r), "()"); + assert_eq!(format!("{r:?}"), "()"); } #[test] @@ -407,14 +466,14 @@ fn test_from_vec() { fn test_downcast() { use std::any::Any; - let r1: Rc = Rc::new(i32::max_value()); + let r1: Rc = Rc::new(i32::MAX); let r2: Rc = Rc::new("abc"); assert!(r1.clone().downcast::().is_err()); let r1i32 = r1.downcast::(); assert!(r1i32.is_ok()); - assert_eq!(r1i32.unwrap(), Rc::new(i32::max_value())); + assert_eq!(r1i32.unwrap(), Rc::new(i32::MAX)); assert!(r2.clone().downcast::().is_err()); @@ -434,3 +493,69 @@ fn test_array_from_slice() { let a: Result, _> = r.clone().try_into(); assert!(a.is_err()); } + +#[test] +fn test_rc_cyclic_with_zero_refs() { + struct ZeroRefs { + inner: Weak, + } + + let zero_refs = Rc::new_cyclic(|inner| { + assert_eq!(inner.strong_count(), 0); + assert!(inner.upgrade().is_none()); + ZeroRefs { inner: Weak::new() } + }); + + assert_eq!(Rc::strong_count(&zero_refs), 1); + assert_eq!(Rc::weak_count(&zero_refs), 0); + assert_eq!(zero_refs.inner.strong_count(), 0); + assert_eq!(zero_refs.inner.weak_count(), 0); +} + +#[test] +fn test_rc_cyclic_with_one_ref() { + struct OneRef { + inner: Weak, + } + + let one_ref = Rc::new_cyclic(|inner| { + assert_eq!(inner.strong_count(), 0); + assert!(inner.upgrade().is_none()); + OneRef { inner: inner.clone() } + }); + + assert_eq!(Rc::strong_count(&one_ref), 1); + assert_eq!(Rc::weak_count(&one_ref), 1); + + let one_ref2 = Weak::upgrade(&one_ref.inner).unwrap(); + assert!(Rc::ptr_eq(&one_ref, &one_ref2)); + + assert_eq!(one_ref.inner.strong_count(), 2); + assert_eq!(one_ref.inner.weak_count(), 1); +} + +#[test] +fn test_rc_cyclic_with_two_ref() { + struct TwoRefs { + inner: Weak, + inner1: Weak, + } + + let two_refs = Rc::new_cyclic(|inner| { + assert_eq!(inner.strong_count(), 0); + assert!(inner.upgrade().is_none()); + TwoRefs { inner: inner.clone(), inner1: inner.clone() } + }); + + assert_eq!(Rc::strong_count(&two_refs), 1); + assert_eq!(Rc::weak_count(&two_refs), 2); + + let two_ref3 = Weak::upgrade(&two_refs.inner).unwrap(); + assert!(Rc::ptr_eq(&two_refs, &two_ref3)); + + let two_ref2 = Weak::upgrade(&two_refs.inner1).unwrap(); + assert!(Rc::ptr_eq(&two_refs, &two_ref2)); + + assert_eq!(Rc::strong_count(&two_refs), 3); + assert_eq!(Rc::weak_count(&two_refs), 2); +} diff --git a/crux-mir/lib/alloc/src/slice.rs b/crux-mir/lib/alloc/src/slice.rs index 5e3f07bde..fecacc2bb 100644 --- a/crux-mir/lib/alloc/src/slice.rs +++ b/crux-mir/lib/alloc/src/slice.rs @@ -1,105 +1,54 @@ -//! A dynamically-sized view into a contiguous sequence, `[T]`. +//! Utilities for the slice primitive type. //! -//! *[See also the slice primitive type](../../std/primitive.slice.html).* +//! *[See also the slice primitive type](slice).* //! -//! Slices are a view into a block of memory represented as a pointer and a -//! length. +//! Most of the structs in this module are iterator types which can only be created +//! using a certain function. For example, `slice.iter()` yields an [`Iter`]. //! -//! ``` -//! // slicing a Vec -//! let vec = vec![1, 2, 3]; -//! let int_slice = &vec[..]; -//! // coercing an array to a slice -//! let str_slice: &[&str] = &["one", "two", "three"]; -//! ``` -//! -//! Slices are either mutable or shared. The shared slice type is `&[T]`, -//! while the mutable slice type is `&mut [T]`, where `T` represents the element -//! type. For example, you can mutate the block of memory that a mutable slice -//! points to: -//! -//! ``` -//! let x = &mut [1, 2, 3]; -//! x[1] = 7; -//! assert_eq!(x, &[1, 7, 3]); -//! ``` -//! -//! Here are some of the things this module contains: -//! -//! ## Structs -//! -//! There are several structs that are useful for slices, such as [`Iter`], which -//! represents iteration over a slice. -//! -//! ## Trait Implementations -//! -//! There are several implementations of common traits for slices. Some examples -//! include: -//! -//! * [`Clone`] -//! * [`Eq`], [`Ord`] - for slices whose element type are [`Eq`] or [`Ord`]. -//! * [`Hash`] - for slices whose element type is [`Hash`]. -//! -//! ## Iteration -//! -//! The slices implement `IntoIterator`. The iterator yields references to the -//! slice elements. -//! -//! ``` -//! let numbers = &[0, 1, 2]; -//! for n in numbers { -//! println!("{} is a number!", n); -//! } -//! ``` -//! -//! The mutable slice yields mutable references to the elements: -//! -//! ``` -//! let mut scores = [7, 8, 9]; -//! for score in &mut scores[..] { -//! *score += 1; -//! } -//! ``` -//! -//! This iterator yields mutable references to the slice's elements, so while -//! the element type of the slice is `i32`, the element type of the iterator is -//! `&mut i32`. -//! -//! * [`.iter`] and [`.iter_mut`] are the explicit methods to return the default -//! iterators. -//! * Further methods that return iterators are [`.split`], [`.splitn`], -//! [`.chunks`], [`.windows`] and more. -//! -//! [`Clone`]: ../../std/clone/trait.Clone.html -//! [`Eq`]: ../../std/cmp/trait.Eq.html -//! [`Ord`]: ../../std/cmp/trait.Ord.html -//! [`Iter`]: struct.Iter.html -//! [`Hash`]: ../../std/hash/trait.Hash.html -//! [`.iter`]: ../../std/primitive.slice.html#method.iter -//! [`.iter_mut`]: ../../std/primitive.slice.html#method.iter_mut -//! [`.split`]: ../../std/primitive.slice.html#method.split -//! [`.splitn`]: ../../std/primitive.slice.html#method.splitn -//! [`.chunks`]: ../../std/primitive.slice.html#method.chunks -//! [`.windows`]: ../../std/primitive.slice.html#method.windows +//! A few functions are provided to create a slice from a value reference +//! or from a raw pointer. #![stable(feature = "rust1", since = "1.0.0")] // Many of the usings in this module are only used in the test configuration. // It's cleaner to just turn off the unused_imports warning than to fix them. #![cfg_attr(test, allow(unused_imports, dead_code))] use core::borrow::{Borrow, BorrowMut}; +#[cfg(not(no_global_oom_handling))] use core::cmp::Ordering::{self, Less}; -use core::mem::{self, size_of}; +#[cfg(not(no_global_oom_handling))] +use core::mem::{self, SizedTypeProperties}; +#[cfg(not(no_global_oom_handling))] use core::ptr; -use core::{u16, u32, u8}; +#[cfg(not(no_global_oom_handling))] +use core::slice::sort; +use crate::alloc::Allocator; +#[cfg(not(no_global_oom_handling))] +use crate::alloc::{self, Global}; +#[cfg(not(no_global_oom_handling))] use crate::borrow::ToOwned; use crate::boxed::Box; use crate::vec::Vec; +#[cfg(test)] +mod tests; + +#[unstable(feature = "slice_range", issue = "76393")] +pub use core::slice::range; +#[unstable(feature = "array_chunks", issue = "74985")] +pub use core::slice::ArrayChunks; +#[unstable(feature = "array_chunks", issue = "74985")] +pub use core::slice::ArrayChunksMut; +#[unstable(feature = "array_windows", issue = "75027")] +pub use core::slice::ArrayWindows; +#[stable(feature = "inherent_ascii_escape", since = "1.60.0")] +pub use core::slice::EscapeAscii; #[stable(feature = "slice_get_slice", since = "1.28.0")] pub use core::slice::SliceIndex; #[stable(feature = "from_ref", since = "1.28.0")] pub use core::slice::{from_mut, from_ref}; +#[unstable(feature = "slice_from_ptr_range", issue = "89792")] +pub use core::slice::{from_mut_ptr_range, from_ptr_range}; #[stable(feature = "rust1", since = "1.0.0")] pub use core::slice::{from_raw_parts, from_raw_parts_mut}; #[stable(feature = "rust1", since = "1.0.0")] @@ -108,6 +57,8 @@ pub use core::slice::{Chunks, Windows}; pub use core::slice::{ChunksExact, ChunksExactMut}; #[stable(feature = "rust1", since = "1.0.0")] pub use core::slice::{ChunksMut, Split, SplitMut}; +#[unstable(feature = "slice_group_by", issue = "80552")] +pub use core::slice::{GroupBy, GroupByMut}; #[stable(feature = "rust1", since = "1.0.0")] pub use core::slice::{Iter, IterMut}; #[stable(feature = "rchunks", since = "1.31.0")] @@ -116,6 +67,8 @@ pub use core::slice::{RChunks, RChunksExact, RChunksExactMut, RChunksMut}; pub use core::slice::{RSplit, RSplitMut}; #[stable(feature = "rust1", since = "1.0.0")] pub use core::slice::{RSplitN, RSplitNMut, SplitN, SplitNMut}; +#[stable(feature = "split_inclusive", since = "1.51.0")] +pub use core::slice::{SplitInclusive, SplitInclusiveMut}; //////////////////////////////////////////////////////////////////////////////// // Basic slice extension methods @@ -135,42 +88,99 @@ pub use hack::to_vec; // functions are actually methods that are in `impl [T]` but not in // `core::slice::SliceExt` - we need to supply these functions for the // `test_permutations` test -mod hack { +pub(crate) mod hack { + use core::alloc::Allocator; + use crate::boxed::Box; - #[cfg(test)] - use crate::string::ToString; use crate::vec::Vec; - pub fn into_vec(b: Box<[T]>) -> Vec { + // We shouldn't add inline attribute to this since this is used in + // `vec!` macro mostly and causes perf regression. See #71204 for + // discussion and perf results. + pub fn into_vec(b: Box<[T], A>) -> Vec { unsafe { let len = b.len(); - let b = Box::into_raw(b); - let xs = Vec::from_raw_parts(b as *mut T, len, len); - xs + let (b, alloc) = Box::into_raw_with_allocator(b); + Vec::from_raw_parts_in(b as *mut T, len, len, alloc) } } + #[cfg(not(no_global_oom_handling))] #[inline] - pub fn to_vec(s: &[T]) -> Vec - where - T: Clone, - { - let mut vector = Vec::with_capacity(s.len()); - vector.extend_from_slice(s); - vector + pub fn to_vec(s: &[T], alloc: A) -> Vec { + T::to_vec(s, alloc) + } + + #[cfg(not(no_global_oom_handling))] + pub trait ConvertVec { + fn to_vec(s: &[Self], alloc: A) -> Vec + where + Self: Sized; + } + + #[cfg(not(no_global_oom_handling))] + impl ConvertVec for T { + #[inline] + default fn to_vec(s: &[Self], alloc: A) -> Vec { + struct DropGuard<'a, T, A: Allocator> { + vec: &'a mut Vec, + num_init: usize, + } + impl<'a, T, A: Allocator> Drop for DropGuard<'a, T, A> { + #[inline] + fn drop(&mut self) { + // SAFETY: + // items were marked initialized in the loop below + unsafe { + self.vec.set_len(self.num_init); + } + } + } + let mut vec = Vec::with_capacity_in(s.len(), alloc); + let mut guard = DropGuard { vec: &mut vec, num_init: 0 }; + let slots = guard.vec.spare_capacity_mut(); + // .take(slots.len()) is necessary for LLVM to remove bounds checks + // and has better codegen than zip. + for (i, b) in s.iter().enumerate().take(slots.len()) { + guard.num_init = i; + slots[i].write(b.clone()); + } + core::mem::forget(guard); + // SAFETY: + // the vec was allocated and initialized above to at least this length. + unsafe { + vec.set_len(s.len()); + } + vec + } + } + + #[cfg(not(no_global_oom_handling))] + impl ConvertVec for T { + #[inline] + fn to_vec(s: &[Self], alloc: A) -> Vec { + let mut v = Vec::with_capacity_in(s.len(), alloc); + // SAFETY: + // allocated above with the capacity of `s`, and initialize to `s.len()` in + // ptr::copy_to_non_overlapping below. + unsafe { + s.as_ptr().copy_to_nonoverlapping(v.as_mut_ptr(), s.len()); + v.set_len(s.len()); + } + v + } } } -#[lang = "slice_alloc"] #[cfg(not(test))] impl [T] { /// Sorts the slice. /// - /// This sort is stable (i.e., does not reorder equal elements) and `O(n log n)` worst-case. + /// This sort is stable (i.e., does not reorder equal elements) and *O*(*n* \* log(*n*)) worst-case. /// /// When applicable, unstable sorting is preferred because it is generally faster than stable /// sorting and it doesn't allocate auxiliary memory. - /// See [`sort_unstable`](#method.sort_unstable). + /// See [`sort_unstable`](slice::sort_unstable). /// /// # Current implementation /// @@ -190,18 +200,20 @@ impl [T] { /// v.sort(); /// assert!(v == [-5, -3, 1, 2, 4]); /// ``` + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn sort(&mut self) where T: Ord, { - merge_sort(self, |a, b| a.lt(b)); + stable_sort(self, T::lt); } /// Sorts the slice with a comparator function. /// - /// This sort is stable (i.e., does not reorder equal elements) and `O(n log n)` worst-case. + /// This sort is stable (i.e., does not reorder equal elements) and *O*(*n* \* log(*n*)) worst-case. /// /// The comparator function must define a total ordering for the elements in the slice. If /// the ordering is not total, the order of the elements is unspecified. An order is a @@ -221,7 +233,7 @@ impl [T] { /// /// When applicable, unstable sorting is preferred because it is generally faster than stable /// sorting and it doesn't allocate auxiliary memory. - /// See [`sort_unstable_by`](#method.sort_unstable_by). + /// See [`sort_unstable_by`](slice::sort_unstable_by). /// /// # Current implementation /// @@ -244,27 +256,29 @@ impl [T] { /// v.sort_by(|a, b| b.cmp(a)); /// assert!(v == [5, 4, 3, 2, 1]); /// ``` + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn sort_by(&mut self, mut compare: F) where F: FnMut(&T, &T) -> Ordering, { - merge_sort(self, |a, b| compare(a, b) == Less); + stable_sort(self, |a, b| compare(a, b) == Less); } /// Sorts the slice with a key extraction function. /// - /// This sort is stable (i.e., does not reorder equal elements) and `O(m n log(m n))` - /// worst-case, where the key function is `O(m)`. + /// This sort is stable (i.e., does not reorder equal elements) and *O*(*m* \* *n* \* log(*n*)) + /// worst-case, where the key function is *O*(*m*). /// /// For expensive key functions (e.g. functions that are not simple property accesses or - /// basic operations), [`sort_by_cached_key`](#method.sort_by_cached_key) is likely to be + /// basic operations), [`sort_by_cached_key`](slice::sort_by_cached_key) is likely to be /// significantly faster, as it does not recompute element keys. /// /// When applicable, unstable sorting is preferred because it is generally faster than stable /// sorting and it doesn't allocate auxiliary memory. - /// See [`sort_unstable_by_key`](#method.sort_unstable_by_key). + /// See [`sort_unstable_by_key`](slice::sort_unstable_by_key). /// /// # Current implementation /// @@ -284,6 +298,8 @@ impl [T] { /// v.sort_by_key(|k| k.abs()); /// assert!(v == [1, 2, -3, 4, -5]); /// ``` + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] #[stable(feature = "slice_sort_by_key", since = "1.7.0")] #[inline] pub fn sort_by_key(&mut self, mut f: F) @@ -291,18 +307,21 @@ impl [T] { F: FnMut(&T) -> K, K: Ord, { - merge_sort(self, |a, b| f(a).lt(&f(b))); + stable_sort(self, |a, b| f(a).lt(&f(b))); } /// Sorts the slice with a key extraction function. /// - /// During sorting, the key function is called only once per element. + /// During sorting, the key function is called at most once per element, by using + /// temporary storage to remember the results of key evaluation. + /// The order of calls to the key function is unspecified and may change in future versions + /// of the standard library. /// - /// This sort is stable (i.e., does not reorder equal elements) and `O(m n + n log n)` - /// worst-case, where the key function is `O(m)`. + /// This sort is stable (i.e., does not reorder equal elements) and *O*(*m* \* *n* + *n* \* log(*n*)) + /// worst-case, where the key function is *O*(*m*). /// /// For simple key functions (e.g., functions that are property accesses or - /// basic operations), [`sort_by_key`](#method.sort_by_key) is likely to be + /// basic operations), [`sort_by_key`](slice::sort_by_key) is likely to be /// faster. /// /// # Current implementation @@ -326,6 +345,8 @@ impl [T] { /// ``` /// /// [pdqsort]: https://github.com/orlp/pdqsort + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] #[stable(feature = "slice_sort_by_cached_key", since = "1.34.0")] #[inline] pub fn sort_by_cached_key(&mut self, f: F) @@ -383,15 +404,41 @@ impl [T] { /// let x = s.to_vec(); /// // Here, `s` and `x` can be modified independently. /// ``` + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] #[rustc_conversion_suggestion] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn to_vec(&self) -> Vec + where + T: Clone, + { + self.to_vec_in(Global) + } + + /// Copies `self` into a new `Vec` with an allocator. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api)] + /// + /// use std::alloc::System; + /// + /// let s = [10, 40, 30]; + /// let x = s.to_vec_in(System); + /// // Here, `s` and `x` can be modified independently. + /// ``` + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] + #[inline] + #[unstable(feature = "allocator_api", issue = "32838")] + pub fn to_vec_in(&self, alloc: A) -> Vec where T: Clone, { // N.B., see the `hack` module in this file for more details. - hack::to_vec(self) + hack::to_vec(self, alloc) } /// Converts `self` into a vector without clones or allocation. @@ -408,14 +455,15 @@ impl [T] { /// /// assert_eq!(x, vec![10, 40, 30]); /// ``` + #[rustc_allow_incoherent_impl] #[stable(feature = "rust1", since = "1.0.0")] #[inline] - pub fn into_vec(self: Box) -> Vec { + pub fn into_vec(self: Box) -> Vec { // N.B., see the `hack` module in this file for more details. hack::into_vec(self) } - /// Creates a vector by repeating a slice `n` times. + /// Creates a vector by copying a slice `n` times. /// /// # Panics /// @@ -433,8 +481,10 @@ impl [T] { /// /// ```should_panic /// // this will panic at runtime - /// b"0123456789abcdef".repeat(usize::max_value()); + /// b"0123456789abcdef".repeat(usize::MAX); /// ``` + #[rustc_allow_incoherent_impl] + #[cfg(not(no_global_oom_handling))] #[stable(feature = "repeat_generic_slice", since = "1.40.0")] pub fn repeat(&self, n: usize) -> Vec where @@ -502,6 +552,7 @@ impl [T] { /// assert_eq!(["hello", "world"].concat(), "helloworld"); /// assert_eq!([[1, 2], [3, 4]].concat(), [1, 2, 3, 4]); /// ``` + #[rustc_allow_incoherent_impl] #[stable(feature = "rust1", since = "1.0.0")] pub fn concat(&self) -> >::Output where @@ -520,6 +571,7 @@ impl [T] { /// assert_eq!([[1, 2], [3, 4]].join(&0), [1, 2, 0, 3, 4]); /// assert_eq!([[1, 2], [3, 4]].join(&[0, 0][..]), [1, 2, 0, 0, 3, 4]); /// ``` + #[rustc_allow_incoherent_impl] #[stable(feature = "rename_connect_to_join", since = "1.3.0")] pub fn join(&self, sep: Separator) -> >::Output where @@ -538,8 +590,9 @@ impl [T] { /// assert_eq!(["hello", "world"].connect(" "), "hello world"); /// assert_eq!([[1, 2], [3, 4]].connect(&0), [1, 2, 0, 3, 4]); /// ``` + #[rustc_allow_incoherent_impl] #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_deprecated(since = "1.3.0", reason = "renamed to join")] + #[deprecated(since = "1.3.0", note = "renamed to join")] pub fn connect(&self, sep: Separator) -> >::Output where Self: Join, @@ -548,7 +601,6 @@ impl [T] { } } -#[lang = "slice_u8_alloc"] #[cfg(not(test))] impl [u8] { /// Returns a vector containing a copy of this slice where each byte @@ -559,7 +611,11 @@ impl [u8] { /// /// To uppercase the value in-place, use [`make_ascii_uppercase`]. /// - /// [`make_ascii_uppercase`]: #method.make_ascii_uppercase + /// [`make_ascii_uppercase`]: slice::make_ascii_uppercase + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] + #[must_use = "this returns the uppercase bytes as a new Vec, \ + without modifying the original"] #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] #[inline] pub fn to_ascii_uppercase(&self) -> Vec { @@ -576,7 +632,11 @@ impl [u8] { /// /// To lowercase the value in-place, use [`make_ascii_lowercase`]. /// - /// [`make_ascii_lowercase`]: #method.make_ascii_lowercase + /// [`make_ascii_lowercase`]: slice::make_ascii_lowercase + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] + #[must_use = "this returns the lowercase bytes as a new Vec, \ + without modifying the original"] #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] #[inline] pub fn to_ascii_lowercase(&self) -> Vec { @@ -590,7 +650,7 @@ impl [u8] { // Extension traits for slices over specific kinds of data //////////////////////////////////////////////////////////////////////////////// -/// Helper trait for [`[T]::concat`](../../std/primitive.slice.html#method.concat). +/// Helper trait for [`[T]::concat`](slice::concat). /// /// Note: the `Item` type parameter is not used in this trait, /// but it allows impls to be more generic. @@ -598,7 +658,7 @@ impl [u8] { /// /// ```error /// error[E0207]: the type parameter `T` is not constrained by the impl trait, self type, or predica -/// --> src/liballoc/slice.rs:608:6 +/// --> library/alloc/src/slice.rs:608:6 /// | /// 608 | impl> Concat for [V] { /// | ^ unconstrained type parameter @@ -625,23 +685,24 @@ pub trait Concat { /// The resulting type after concatenation type Output; - /// Implementation of [`[T]::concat`](../../std/primitive.slice.html#method.concat) + /// Implementation of [`[T]::concat`](slice::concat) #[unstable(feature = "slice_concat_trait", issue = "27747")] fn concat(slice: &Self) -> Self::Output; } -/// Helper trait for [`[T]::join`](../../std/primitive.slice.html#method.join) +/// Helper trait for [`[T]::join`](slice::join) #[unstable(feature = "slice_concat_trait", issue = "27747")] pub trait Join { #[unstable(feature = "slice_concat_trait", issue = "27747")] /// The resulting type after concatenation type Output; - /// Implementation of [`[T]::join`](../../std/primitive.slice.html#method.join) + /// Implementation of [`[T]::join`](slice::join) #[unstable(feature = "slice_concat_trait", issue = "27747")] fn join(slice: &Self, sep: Separator) -> Self::Output; } +#[cfg(not(no_global_oom_handling))] #[unstable(feature = "slice_concat_ext", issue = "27747")] impl> Concat for [V] { type Output = Vec; @@ -656,6 +717,7 @@ impl> Concat for [V] { } } +#[cfg(not(no_global_oom_handling))] #[unstable(feature = "slice_concat_ext", issue = "27747")] impl> Join<&T> for [V] { type Output = Vec; @@ -678,6 +740,7 @@ impl> Join<&T> for [V] { } } +#[cfg(not(no_global_oom_handling))] #[unstable(feature = "slice_concat_ext", issue = "27747")] impl> Join<&[T]> for [V] { type Output = Vec; @@ -706,19 +769,20 @@ impl> Join<&[T]> for [V] { //////////////////////////////////////////////////////////////////////////////// #[stable(feature = "rust1", since = "1.0.0")] -impl Borrow<[T]> for Vec { +impl Borrow<[T]> for Vec { fn borrow(&self) -> &[T] { &self[..] } } #[stable(feature = "rust1", since = "1.0.0")] -impl BorrowMut<[T]> for Vec { +impl BorrowMut<[T]> for Vec { fn borrow_mut(&mut self) -> &mut [T] { &mut self[..] } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl ToOwned for [T] { type Owned = Vec; @@ -729,20 +793,20 @@ impl ToOwned for [T] { #[cfg(test)] fn to_owned(&self) -> Vec { - hack::to_vec(self) + hack::to_vec(self, Global) } fn clone_into(&self, target: &mut Vec) { // drop anything in target that will not be overwritten target.truncate(self.len()); - let len = target.len(); - - // reuse the contained values' allocations/resources. - target.clone_from_slice(&self[..len]); // target.len <= self.len due to the truncate above, so the - // slice here is always in-bounds. - target.extend_from_slice(&self[len..]); + // slices here are always in-bounds. + let (init, tail) = self.split_at(target.len()); + + // reuse the contained values' allocations/resources. + target.clone_from_slice(init); + target.extend_from_slice(tail); } } @@ -750,195 +814,52 @@ impl ToOwned for [T] { // Sorting //////////////////////////////////////////////////////////////////////////////// -/// Inserts `v[0]` into pre-sorted sequence `v[1..]` so that whole `v[..]` becomes sorted. -/// -/// This is the integral subroutine of insertion sort. -fn insert_head(v: &mut [T], is_less: &mut F) +#[inline] +#[cfg(not(no_global_oom_handling))] +fn stable_sort(v: &mut [T], mut is_less: F) where F: FnMut(&T, &T) -> bool, { - if v.len() >= 2 && is_less(&v[1], &v[0]) { - unsafe { - // There are three ways to implement insertion here: - // - // 1. Swap adjacent elements until the first one gets to its final destination. - // However, this way we copy data around more than is necessary. If elements are big - // structures (costly to copy), this method will be slow. - // - // 2. Iterate until the right place for the first element is found. Then shift the - // elements succeeding it to make room for it and finally place it into the - // remaining hole. This is a good method. - // - // 3. Copy the first element into a temporary variable. Iterate until the right place - // for it is found. As we go along, copy every traversed element into the slot - // preceding it. Finally, copy data from the temporary variable into the remaining - // hole. This method is very good. Benchmarks demonstrated slightly better - // performance than with the 2nd method. - // - // All methods were benchmarked, and the 3rd showed best results. So we chose that one. - let mut tmp = mem::ManuallyDrop::new(ptr::read(&v[0])); - - // Intermediate state of the insertion process is always tracked by `hole`, which - // serves two purposes: - // 1. Protects integrity of `v` from panics in `is_less`. - // 2. Fills the remaining hole in `v` in the end. - // - // Panic safety: - // - // If `is_less` panics at any point during the process, `hole` will get dropped and - // fill the hole in `v` with `tmp`, thus ensuring that `v` still holds every object it - // initially held exactly once. - let mut hole = InsertionHole { src: &mut *tmp, dest: &mut v[1] }; - ptr::copy_nonoverlapping(&v[1], &mut v[0], 1); - - for i in 2..v.len() { - if !is_less(&v[i], &*tmp) { - break; - } - ptr::copy_nonoverlapping(&v[i], &mut v[i - 1], 1); - hole.dest = &mut v[i]; - } - // `hole` gets dropped and thus copies `tmp` into the remaining hole in `v`. - } + if T::IS_ZST { + // Sorting has no meaningful behavior on zero-sized types. Do nothing. + return; } - // When dropped, copies from `src` into `dest`. - struct InsertionHole { - src: *mut T, - dest: *mut T, - } - - impl Drop for InsertionHole { - fn drop(&mut self) { - unsafe { - ptr::copy_nonoverlapping(self.src, self.dest, 1); - } + let elem_alloc_fn = |len: usize| -> *mut T { + // SAFETY: Creating the layout is safe as long as merge_sort never calls this with len > + // v.len(). Alloc in general will only be used as 'shadow-region' to store temporary swap + // elements. + unsafe { alloc::alloc(alloc::Layout::array::(len).unwrap_unchecked()) as *mut T } + }; + + let elem_dealloc_fn = |buf_ptr: *mut T, len: usize| { + // SAFETY: Creating the layout is safe as long as merge_sort never calls this with len > + // v.len(). The caller must ensure that buf_ptr was created by elem_alloc_fn with the same + // len. + unsafe { + alloc::dealloc(buf_ptr as *mut u8, alloc::Layout::array::(len).unwrap_unchecked()); } - } -} + }; -/// Merges non-decreasing runs `v[..mid]` and `v[mid..]` using `buf` as temporary storage, and -/// stores the result into `v[..]`. -/// -/// # Safety -/// -/// The two slices must be non-empty and `mid` must be in bounds. Buffer `buf` must be long enough -/// to hold a copy of the shorter slice. Also, `T` must not be a zero-sized type. -unsafe fn merge(v: &mut [T], mid: usize, buf: *mut T, is_less: &mut F) -where - F: FnMut(&T, &T) -> bool, -{ - let len = v.len(); - let v = v.as_mut_ptr(); - let v_mid = v.add(mid); - let v_end = v.add(len); - - // The merge process first copies the shorter run into `buf`. Then it traces the newly copied - // run and the longer run forwards (or backwards), comparing their next unconsumed elements and - // copying the lesser (or greater) one into `v`. - // - // As soon as the shorter run is fully consumed, the process is done. If the longer run gets - // consumed first, then we must copy whatever is left of the shorter run into the remaining - // hole in `v`. - // - // Intermediate state of the process is always tracked by `hole`, which serves two purposes: - // 1. Protects integrity of `v` from panics in `is_less`. - // 2. Fills the remaining hole in `v` if the longer run gets consumed first. - // - // Panic safety: - // - // If `is_less` panics at any point during the process, `hole` will get dropped and fill the - // hole in `v` with the unconsumed range in `buf`, thus ensuring that `v` still holds every - // object it initially held exactly once. - let mut hole; - - if mid <= len - mid { - // The left run is shorter. - ptr::copy_nonoverlapping(v, buf, mid); - hole = MergeHole { start: buf, end: buf.add(mid), dest: v }; - - // Initially, these pointers point to the beginnings of their arrays. - let left = &mut hole.start; - let mut right = v_mid; - let out = &mut hole.dest; - - while *left < hole.end && right < v_end { - // Consume the lesser side. - // If equal, prefer the left run to maintain stability. - let to_copy = if is_less(&*right, &**left) { - get_and_increment(&mut right) - } else { - get_and_increment(left) - }; - ptr::copy_nonoverlapping(to_copy, get_and_increment(out), 1); - } - } else { - // The right run is shorter. - ptr::copy_nonoverlapping(v_mid, buf, len - mid); - hole = MergeHole { start: buf, end: buf.add(len - mid), dest: v_mid }; - - // Initially, these pointers point past the ends of their arrays. - let left = &mut hole.dest; - let right = &mut hole.end; - let mut out = v_end; - - while v < *left && buf < *right { - // Consume the greater side. - // If equal, prefer the right run to maintain stability. - let to_copy = if is_less(&*right.offset(-1), &*left.offset(-1)) { - decrement_and_get(left) - } else { - decrement_and_get(right) - }; - ptr::copy_nonoverlapping(to_copy, decrement_and_get(&mut out), 1); + let run_alloc_fn = |len: usize| -> *mut sort::TimSortRun { + // SAFETY: Creating the layout is safe as long as merge_sort never calls this with an + // obscene length or 0. + unsafe { + alloc::alloc(alloc::Layout::array::(len).unwrap_unchecked()) + as *mut sort::TimSortRun } - } - // Finally, `hole` gets dropped. If the shorter run was not fully consumed, whatever remains of - // it will now be copied into the hole in `v`. - - unsafe fn get_and_increment(ptr: &mut *mut T) -> *mut T { - let old = *ptr; - *ptr = ptr.offset(1); - old - } - - unsafe fn decrement_and_get(ptr: &mut *mut T) -> *mut T { - *ptr = ptr.offset(-1); - *ptr - } - - // When dropped, copies the range `start..end` into `dest..`. - struct MergeHole { - start: *mut T, - end: *mut T, - dest: *mut T, - } + }; - impl Drop for MergeHole { - fn drop(&mut self) { - // `T` is not a zero-sized type, so it's okay to divide by its size. - let len = (self.end as usize - self.start as usize) / mem::size_of::(); - unsafe { - ptr::copy_nonoverlapping(self.start, self.dest, len); - } + let run_dealloc_fn = |buf_ptr: *mut sort::TimSortRun, len: usize| { + // SAFETY: The caller must ensure that buf_ptr was created by elem_alloc_fn with the same + // len. + unsafe { + alloc::dealloc( + buf_ptr as *mut u8, + alloc::Layout::array::(len).unwrap_unchecked(), + ); } - } -} - -// Crux: replace merge sort implementation with a simple selection sort. Parts of the original -// sort implementation rely on `impl Drop`, which we don't support. -fn merge_sort(v: &mut [T], mut is_less: F) -where - F: FnMut(&T, &T) -> bool, -{ + }; - for i in 0 .. v.len() { - let mut best_idx = i; - for j in i + 1 .. v.len() { - if is_less(&v[j], &v[best_idx]) { - best_idx = j; - } - } - v.swap(i, best_idx); - } + sort::merge_sort(v, &mut is_less, elem_alloc_fn, elem_dealloc_fn, run_alloc_fn, run_dealloc_fn); } diff --git a/crux-mir/lib/alloc/src/slice/tests.rs b/crux-mir/lib/alloc/src/slice/tests.rs new file mode 100644 index 000000000..f674530aa --- /dev/null +++ b/crux-mir/lib/alloc/src/slice/tests.rs @@ -0,0 +1,359 @@ +use crate::borrow::ToOwned; +use crate::rc::Rc; +use crate::string::ToString; +use crate::test_helpers::test_rng; +use crate::vec::Vec; + +use core::cell::Cell; +use core::cmp::Ordering::{self, Equal, Greater, Less}; +use core::convert::identity; +use core::fmt; +use core::mem; +use core::sync::atomic::{AtomicUsize, Ordering::Relaxed}; +use rand::{distributions::Standard, prelude::*, Rng, RngCore}; +use std::panic; + +macro_rules! do_test { + ($input:ident, $func:ident) => { + let len = $input.len(); + + // Work out the total number of comparisons required to sort + // this array... + let mut count = 0usize; + $input.to_owned().$func(|a, b| { + count += 1; + a.cmp(b) + }); + + // ... and then panic on each and every single one. + for panic_countdown in 0..count { + // Refresh the counters. + VERSIONS.store(0, Relaxed); + for i in 0..len { + DROP_COUNTS[i].store(0, Relaxed); + } + + let v = $input.to_owned(); + let _ = std::panic::catch_unwind(move || { + let mut v = v; + let mut panic_countdown = panic_countdown; + v.$func(|a, b| { + if panic_countdown == 0 { + SILENCE_PANIC.with(|s| s.set(true)); + panic!(); + } + panic_countdown -= 1; + a.cmp(b) + }) + }); + + // Check that the number of things dropped is exactly + // what we expect (i.e., the contents of `v`). + for (i, c) in DROP_COUNTS.iter().enumerate().take(len) { + let count = c.load(Relaxed); + assert!(count == 1, "found drop count == {} for i == {}, len == {}", count, i, len); + } + + // Check that the most recent versions of values were dropped. + assert_eq!(VERSIONS.load(Relaxed), 0); + } + }; +} + +const MAX_LEN: usize = 80; + +static DROP_COUNTS: [AtomicUsize; MAX_LEN] = [ + // FIXME(RFC 1109): AtomicUsize is not Copy. + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), +]; + +static VERSIONS: AtomicUsize = AtomicUsize::new(0); + +#[derive(Clone, Eq)] +struct DropCounter { + x: u32, + id: usize, + version: Cell, +} + +impl PartialEq for DropCounter { + fn eq(&self, other: &Self) -> bool { + self.partial_cmp(other) == Some(Ordering::Equal) + } +} + +impl PartialOrd for DropCounter { + fn partial_cmp(&self, other: &Self) -> Option { + self.version.set(self.version.get() + 1); + other.version.set(other.version.get() + 1); + VERSIONS.fetch_add(2, Relaxed); + self.x.partial_cmp(&other.x) + } +} + +impl Ord for DropCounter { + fn cmp(&self, other: &Self) -> Ordering { + self.partial_cmp(other).unwrap() + } +} + +impl Drop for DropCounter { + fn drop(&mut self) { + DROP_COUNTS[self.id].fetch_add(1, Relaxed); + VERSIONS.fetch_sub(self.version.get(), Relaxed); + } +} + +std::thread_local!(static SILENCE_PANIC: Cell = Cell::new(false)); + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] // no threads +fn panic_safe() { + panic::update_hook(move |prev, info| { + if !SILENCE_PANIC.with(|s| s.get()) { + prev(info); + } + }); + + let mut rng = test_rng(); + + // Miri is too slow (but still need to `chain` to make the types match) + let lens = if cfg!(miri) { (1..10).chain(0..0) } else { (1..20).chain(70..MAX_LEN) }; + let moduli: &[u32] = if cfg!(miri) { &[5] } else { &[5, 20, 50] }; + + for len in lens { + for &modulus in moduli { + for &has_runs in &[false, true] { + let mut input = (0..len) + .map(|id| DropCounter { + x: rng.next_u32() % modulus, + id: id, + version: Cell::new(0), + }) + .collect::>(); + + if has_runs { + for c in &mut input { + c.x = c.id as u32; + } + + for _ in 0..5 { + let a = rng.gen::() % len; + let b = rng.gen::() % len; + if a < b { + input[a..b].reverse(); + } else { + input.swap(a, b); + } + } + } + + do_test!(input, sort_by); + do_test!(input, sort_unstable_by); + } + } + } + + // Set default panic hook again. + drop(panic::take_hook()); +} + +#[test] +#[cfg_attr(miri, ignore)] // Miri is too slow +fn test_sort() { + let mut rng = test_rng(); + + for len in (2..25).chain(500..510) { + for &modulus in &[5, 10, 100, 1000] { + for _ in 0..10 { + let orig: Vec<_> = (&mut rng) + .sample_iter::(&Standard) + .map(|x| x % modulus) + .take(len) + .collect(); + + // Sort in default order. + let mut v = orig.clone(); + v.sort(); + assert!(v.windows(2).all(|w| w[0] <= w[1])); + + // Sort in ascending order. + let mut v = orig.clone(); + v.sort_by(|a, b| a.cmp(b)); + assert!(v.windows(2).all(|w| w[0] <= w[1])); + + // Sort in descending order. + let mut v = orig.clone(); + v.sort_by(|a, b| b.cmp(a)); + assert!(v.windows(2).all(|w| w[0] >= w[1])); + + // Sort in lexicographic order. + let mut v1 = orig.clone(); + let mut v2 = orig.clone(); + v1.sort_by_key(|x| x.to_string()); + v2.sort_by_cached_key(|x| x.to_string()); + assert!(v1.windows(2).all(|w| w[0].to_string() <= w[1].to_string())); + assert!(v1 == v2); + + // Sort with many pre-sorted runs. + let mut v = orig.clone(); + v.sort(); + v.reverse(); + for _ in 0..5 { + let a = rng.gen::() % len; + let b = rng.gen::() % len; + if a < b { + v[a..b].reverse(); + } else { + v.swap(a, b); + } + } + v.sort(); + assert!(v.windows(2).all(|w| w[0] <= w[1])); + } + } + } + + // Sort using a completely random comparison function. + // This will reorder the elements *somehow*, but won't panic. + let mut v = [0; 500]; + for i in 0..v.len() { + v[i] = i as i32; + } + v.sort_by(|_, _| *[Less, Equal, Greater].choose(&mut rng).unwrap()); + v.sort(); + for i in 0..v.len() { + assert_eq!(v[i], i as i32); + } + + // Should not panic. + [0i32; 0].sort(); + [(); 10].sort(); + [(); 100].sort(); + + let mut v = [0xDEADBEEFu64]; + v.sort(); + assert!(v == [0xDEADBEEF]); +} + +#[test] +fn test_sort_stability() { + // Miri is too slow + let large_range = if cfg!(miri) { 0..0 } else { 500..510 }; + let rounds = if cfg!(miri) { 1 } else { 10 }; + + let mut rng = test_rng(); + for len in (2..25).chain(large_range) { + for _ in 0..rounds { + let mut counts = [0; 10]; + + // create a vector like [(6, 1), (5, 1), (6, 2), ...], + // where the first item of each tuple is random, but + // the second item represents which occurrence of that + // number this element is, i.e., the second elements + // will occur in sorted order. + let orig: Vec<_> = (0..len) + .map(|_| { + let n = rng.gen::() % 10; + counts[n] += 1; + (n, counts[n]) + }) + .collect(); + + let mut v = orig.clone(); + // Only sort on the first element, so an unstable sort + // may mix up the counts. + v.sort_by(|&(a, _), &(b, _)| a.cmp(&b)); + + // This comparison includes the count (the second item + // of the tuple), so elements with equal first items + // will need to be ordered with increasing + // counts... i.e., exactly asserting that this sort is + // stable. + assert!(v.windows(2).all(|w| w[0] <= w[1])); + + let mut v = orig.clone(); + v.sort_by_cached_key(|&(x, _)| x); + assert!(v.windows(2).all(|w| w[0] <= w[1])); + } + } +} diff --git a/crux-mir/lib/alloc/src/str.rs b/crux-mir/lib/alloc/src/str.rs index 843a2f1f8..afbe5cfaf 100644 --- a/crux-mir/lib/alloc/src/str.rs +++ b/crux-mir/lib/alloc/src/str.rs @@ -1,26 +1,6 @@ -//! Unicode string slices. +//! Utilities for the `str` primitive type. //! -//! *[See also the `str` primitive type](../../std/primitive.str.html).* -//! -//! The `&str` type is one of the two main string types, the other being `String`. -//! Unlike its `String` counterpart, its contents are borrowed. -//! -//! # Basic Usage -//! -//! A basic string declaration of `&str` type: -//! -//! ``` -//! let hello_world = "Hello, World!"; -//! ``` -//! -//! Here we have declared a string literal, also known as a string slice. -//! String literals have a static lifetime, which means the string `hello_world` -//! is guaranteed to be valid for the duration of the entire program. -//! We can explicitly specify `hello_world`'s lifetime as well: -//! -//! ``` -//! let hello_world: &'static str = "Hello, world!"; -//! ``` +//! *[See also the `str` primitive type](str).* #![stable(feature = "rust1", since = "1.0.0")] // Many of the usings in this module are only used in the test configuration. @@ -46,6 +26,8 @@ pub use core::str::pattern; pub use core::str::EncodeUtf16; #[stable(feature = "split_ascii_whitespace", since = "1.34.0")] pub use core::str::SplitAsciiWhitespace; +#[stable(feature = "split_inclusive", since = "1.51.0")] +pub use core::str::SplitInclusive; #[stable(feature = "rust1", since = "1.0.0")] pub use core::str::SplitWhitespace; #[stable(feature = "rust1", since = "1.0.0")] @@ -69,9 +51,12 @@ pub use core::str::{RSplit, Split}; pub use core::str::{RSplitN, SplitN}; #[stable(feature = "rust1", since = "1.0.0")] pub use core::str::{RSplitTerminator, SplitTerminator}; +#[unstable(feature = "utf8_chunks", issue = "99543")] +pub use core::str::{Utf8Chunk, Utf8Chunks}; /// Note: `str` in `Concat` is not meaningful here. /// This type parameter of the trait only exists to enable another impl. +#[cfg(not(no_global_oom_handling))] #[unstable(feature = "slice_concat_ext", issue = "27747")] impl> Concat for [S] { type Output = String; @@ -81,6 +66,7 @@ impl> Concat for [S] { } } +#[cfg(not(no_global_oom_handling))] #[unstable(feature = "slice_concat_ext", issue = "27747")] impl> Join<&str> for [S] { type Output = String; @@ -90,8 +76,9 @@ impl> Join<&str> for [S] { } } -macro_rules! spezialize_for_lengths { - ($separator:expr, $target:expr, $iter:expr; $($num:expr),*) => { +#[cfg(not(no_global_oom_handling))] +macro_rules! specialize_for_lengths { + ($separator:expr, $target:expr, $iter:expr; $($num:expr),*) => {{ let mut target = $target; let iter = $iter; let sep_bytes = $separator; @@ -102,7 +89,8 @@ macro_rules! spezialize_for_lengths { $num => { for s in iter { copy_slice_and_advance!(target, sep_bytes); - copy_slice_and_advance!(target, s.borrow().as_ref()); + let content_bytes = s.borrow().as_ref(); + copy_slice_and_advance!(target, content_bytes); } }, )* @@ -110,13 +98,16 @@ macro_rules! spezialize_for_lengths { // arbitrary non-zero size fallback for s in iter { copy_slice_and_advance!(target, sep_bytes); - copy_slice_and_advance!(target, s.borrow().as_ref()); + let content_bytes = s.borrow().as_ref(); + copy_slice_and_advance!(target, content_bytes); } } } - }; + target + }} } +#[cfg(not(no_global_oom_handling))] macro_rules! copy_slice_and_advance { ($target:expr, $bytes:expr) => { let len = $bytes.len(); @@ -134,6 +125,7 @@ macro_rules! copy_slice_and_advance { // the bounds for String-join are S: Borrow and for Vec-join Borrow<[T]> // [T] and str both impl AsRef<[T]> for some T // => s.borrow().as_ref() and we always have slices +#[cfg(not(no_global_oom_handling))] fn join_generic_copy(slice: &[S], sep: &[T]) -> Vec where T: Copy, @@ -153,30 +145,41 @@ where // if the `len` calculation overflows, we'll panic // we would have run out of memory anyway and the rest of the function requires // the entire Vec pre-allocated for safety - let len = sep_len + let reserved_len = sep_len .checked_mul(iter.len()) .and_then(|n| { slice.iter().map(|s| s.borrow().as_ref().len()).try_fold(n, usize::checked_add) }) .expect("attempt to join into collection with len > usize::MAX"); - // crucial for safety - let mut result = Vec::with_capacity(len); - assert!(result.capacity() >= len); + // prepare an uninitialized buffer + let mut result = Vec::with_capacity(reserved_len); + debug_assert!(result.capacity() >= reserved_len); result.extend_from_slice(first.borrow().as_ref()); unsafe { - { - let pos = result.len(); - let target = result.get_unchecked_mut(pos..len); - - // copy separator and slices over without bounds checks - // generate loops with hardcoded offsets for small separators - // massive improvements possible (~ x2) - spezialize_for_lengths!(sep, target, iter; 0, 1, 2, 3, 4); - } - result.set_len(len); + let pos = result.len(); + let target = result.spare_capacity_mut().get_unchecked_mut(..reserved_len - pos); + + // Convert the separator and slices to slices of MaybeUninit + // to simplify implementation in specialize_for_lengths + let sep_uninit = core::slice::from_raw_parts(sep.as_ptr().cast(), sep.len()); + let iter_uninit = iter.map(|it| { + let it = it.borrow().as_ref(); + core::slice::from_raw_parts(it.as_ptr().cast(), it.len()) + }); + + // copy separator and slices over without bounds checks + // generate loops with hardcoded offsets for small separators + // massive improvements possible (~ x2) + let remain = specialize_for_lengths!(sep_uninit, target, iter_uninit; 0, 1, 2, 3, 4); + + // A weird borrow implementation may return different + // slices for the length calculation and the actual copy. + // Make sure we don't expose uninitialized bytes to the caller. + let result_len = reserved_len - remain.len(); + result.set_len(result_len); } result } @@ -197,6 +200,7 @@ impl BorrowMut for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl ToOwned for str { type Owned = String; @@ -213,7 +217,6 @@ impl ToOwned for str { } /// Methods for string slices. -#[lang = "str_alloc"] #[cfg(not(test))] impl str { /// Converts a `Box` into a `Box<[u8]>` without copying or allocating. @@ -228,7 +231,9 @@ impl str { /// let boxed_bytes = boxed_str.into_boxed_bytes(); /// assert_eq!(*boxed_bytes, *s.as_bytes()); /// ``` + #[rustc_allow_incoherent_impl] #[stable(feature = "str_box_extras", since = "1.20.0")] + #[must_use = "`self` will be dropped if the result is not used"] #[inline] pub fn into_boxed_bytes(self: Box) -> Box<[u8]> { self.into() @@ -240,8 +245,6 @@ impl str { /// While doing so, it attempts to find matches of a pattern. If it finds any, it /// replaces them with the replacement string slice. /// - /// [`String`]: string/struct.String.html - /// /// # Examples /// /// Basic usage: @@ -250,6 +253,7 @@ impl str { /// let s = "this is old"; /// /// assert_eq!("this is new", s.replace("old", "new")); + /// assert_eq!("than an old", s.replace("is", "an")); /// ``` /// /// When the pattern doesn't match: @@ -258,6 +262,8 @@ impl str { /// let s = "this is old"; /// assert_eq!(s, s.replace("cookie monster", "little lamb")); /// ``` + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] #[must_use = "this returns the replaced string as a new allocation, \ without modifying the original"] #[stable(feature = "rust1", since = "1.0.0")] @@ -280,8 +286,6 @@ impl str { /// While doing so, it attempts to find matches of a pattern. If it finds any, it /// replaces them with the replacement string slice at most `count` times. /// - /// [`String`]: string/struct.String.html - /// /// # Examples /// /// Basic usage: @@ -299,6 +303,8 @@ impl str { /// let s = "this is old"; /// assert_eq!(s, s.replacen("cookie monster", "little lamb", 10)); /// ``` + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] #[must_use = "this returns the replaced string as a new allocation, \ without modifying the original"] #[stable(feature = "str_replacen", since = "1.16.0")] @@ -324,8 +330,6 @@ impl str { /// the case, this function returns a [`String`] instead of modifying the /// parameter in-place. /// - /// [`String`]: string/struct.String.html - /// /// # Examples /// /// Basic usage: @@ -356,17 +360,29 @@ impl str { /// /// assert_eq!(new_year, new_year.to_lowercase()); /// ``` + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] + #[must_use = "this returns the lowercase string as a new String, \ + without modifying the original"] #[stable(feature = "unicode_case_mapping", since = "1.2.0")] pub fn to_lowercase(&self) -> String { - let mut s = String::with_capacity(self.len()); - for (i, c) in self[..].char_indices() { + let out = convert_while_ascii(self.as_bytes(), u8::to_ascii_lowercase); + + // Safety: we know this is a valid char boundary since + // out.len() is only progressed if ascii bytes are found + let rest = unsafe { self.get_unchecked(out.len()..) }; + + // Safety: We have written only valid ASCII to our vec + let mut s = unsafe { String::from_utf8_unchecked(out) }; + + for (i, c) in rest[..].char_indices() { if c == 'Σ' { // Σ maps to σ, except at the end of a word where it maps to ς. // This is the only conditional (contextual) but language-independent mapping // in `SpecialCasing.txt`, // so hard-code it rather than have a generic "condition" mechanism. // See https://github.com/rust-lang/rust/issues/26035 - map_uppercase_sigma(self, i, &mut s) + map_uppercase_sigma(rest, i, &mut s) } else { match conversions::to_lower(c) { [a, '\0', _] => s.push(a), @@ -385,7 +401,7 @@ impl str { return s; fn map_uppercase_sigma(from: &str, i: usize, to: &mut String) { - // See http://www.unicode.org/versions/Unicode7.0.0/ch03.pdf#G33992 + // See https://www.unicode.org/versions/Unicode7.0.0/ch03.pdf#G33992 // for the definition of `Final_Sigma`. debug_assert!('Σ'.len_utf8() == 2); let is_word_final = case_ignoreable_then_cased(from[..i].chars().rev()) @@ -394,7 +410,7 @@ impl str { } fn case_ignoreable_then_cased>(iter: I) -> bool { - use core::unicode::derived_property::{Case_Ignorable, Cased}; + use core::unicode::{Case_Ignorable, Cased}; match iter.skip_while(|&c| Case_Ignorable(c)).next() { Some(c) => Cased(c), None => false, @@ -411,8 +427,6 @@ impl str { /// the case, this function returns a [`String`] instead of modifying the /// parameter in-place. /// - /// [`String`]: string/struct.String.html - /// /// # Examples /// /// Basic usage: @@ -437,10 +451,22 @@ impl str { /// /// assert_eq!("TSCHÜSS", s.to_uppercase()); /// ``` + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] + #[must_use = "this returns the uppercase string as a new String, \ + without modifying the original"] #[stable(feature = "unicode_case_mapping", since = "1.2.0")] pub fn to_uppercase(&self) -> String { - let mut s = String::with_capacity(self.len()); - for c in self[..].chars() { + let out = convert_while_ascii(self.as_bytes(), u8::to_ascii_uppercase); + + // Safety: we know this is a valid char boundary since + // out.len() is only progressed if ascii bytes are found + let rest = unsafe { self.get_unchecked(out.len()..) }; + + // Safety: We have written only valid ASCII to our vec + let mut s = unsafe { String::from_utf8_unchecked(out) }; + + for c in rest.chars() { match conversions::to_upper(c) { [a, '\0', _] => s.push(a), [a, b, '\0'] => { @@ -459,9 +485,6 @@ impl str { /// Converts a [`Box`] into a [`String`] without copying or allocating. /// - /// [`String`]: string/struct.String.html - /// [`Box`]: boxed/struct.Box.html - /// /// # Examples /// /// Basic usage: @@ -473,6 +496,8 @@ impl str { /// assert_eq!(boxed_str.into_string(), string); /// ``` #[stable(feature = "box_str", since = "1.4.0")] + #[rustc_allow_incoherent_impl] + #[must_use = "`self` will be dropped if the result is not used"] #[inline] pub fn into_string(self: Box) -> String { let slice = Box::<[u8]>::from(self); @@ -485,8 +510,6 @@ impl str { /// /// This function will panic if the capacity would overflow. /// - /// [`String`]: string/struct.String.html - /// /// # Examples /// /// Basic usage: @@ -499,8 +522,11 @@ impl str { /// /// ```should_panic /// // this will panic at runtime - /// "0123456789abcdef".repeat(usize::max_value()); + /// let huge = "0123456789abcdef".repeat(usize::MAX); /// ``` + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] + #[must_use] #[stable(feature = "repeat_str", since = "1.16.0")] pub fn repeat(&self, n: usize) -> String { unsafe { String::from_utf8_unchecked(self.as_bytes().repeat(n)) } @@ -525,15 +551,17 @@ impl str { /// assert_eq!("GRüßE, JüRGEN ❤", s.to_ascii_uppercase()); /// ``` /// - /// [`make_ascii_uppercase`]: #method.make_ascii_uppercase + /// [`make_ascii_uppercase`]: str::make_ascii_uppercase /// [`to_uppercase`]: #method.to_uppercase + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] + #[must_use = "to uppercase the value in-place, use `make_ascii_uppercase()`"] #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] #[inline] pub fn to_ascii_uppercase(&self) -> String { - let mut bytes = self.as_bytes().to_vec(); - bytes.make_ascii_uppercase(); - // make_ascii_uppercase() preserves the UTF-8 invariant. - unsafe { String::from_utf8_unchecked(bytes) } + let mut s = self.to_owned(); + s.make_ascii_uppercase(); + s } /// Returns a copy of this string where each character is mapped to its @@ -555,15 +583,17 @@ impl str { /// assert_eq!("grüße, jürgen ❤", s.to_ascii_lowercase()); /// ``` /// - /// [`make_ascii_lowercase`]: #method.make_ascii_lowercase + /// [`make_ascii_lowercase`]: str::make_ascii_lowercase /// [`to_lowercase`]: #method.to_lowercase + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] + #[must_use = "to lowercase the value in-place, use `make_ascii_lowercase()`"] #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] #[inline] pub fn to_ascii_lowercase(&self) -> String { - let mut bytes = self.as_bytes().to_vec(); - bytes.make_ascii_lowercase(); - // make_ascii_lowercase() preserves the UTF-8 invariant. - unsafe { String::from_utf8_unchecked(bytes) } + let mut s = self.to_owned(); + s.make_ascii_lowercase(); + s } } @@ -581,7 +611,56 @@ impl str { /// assert_eq!("☺", &*smile); /// ``` #[stable(feature = "str_box_extras", since = "1.20.0")] +#[must_use] #[inline] pub unsafe fn from_boxed_utf8_unchecked(v: Box<[u8]>) -> Box { - Box::from_raw(Box::into_raw(v) as *mut str) + unsafe { Box::from_raw(Box::into_raw(v) as *mut str) } +} + +/// Converts the bytes while the bytes are still ascii. +/// For better average performance, this is happens in chunks of `2*size_of::()`. +/// Returns a vec with the converted bytes. +#[inline] +#[cfg(not(test))] +#[cfg(not(no_global_oom_handling))] +fn convert_while_ascii(b: &[u8], convert: fn(&u8) -> u8) -> Vec { + let mut out = Vec::with_capacity(b.len()); + + const USIZE_SIZE: usize = mem::size_of::(); + const MAGIC_UNROLL: usize = 2; + const N: usize = USIZE_SIZE * MAGIC_UNROLL; + const NONASCII_MASK: usize = usize::from_ne_bytes([0x80; USIZE_SIZE]); + + let mut i = 0; + unsafe { + while i + N <= b.len() { + // Safety: we have checks the sizes `b` and `out` to know that our + let in_chunk = b.get_unchecked(i..i + N); + let out_chunk = out.spare_capacity_mut().get_unchecked_mut(i..i + N); + + let mut bits = 0; + for j in 0..MAGIC_UNROLL { + // read the bytes 1 usize at a time (unaligned since we haven't checked the alignment) + // safety: in_chunk is valid bytes in the range + bits |= in_chunk.as_ptr().cast::().add(j).read_unaligned(); + } + // if our chunks aren't ascii, then return only the prior bytes as init + if bits & NONASCII_MASK != 0 { + break; + } + + // perform the case conversions on N bytes (gets heavily autovec'd) + for j in 0..N { + // safety: in_chunk and out_chunk is valid bytes in the range + let out = out_chunk.get_unchecked_mut(j); + out.write(convert(in_chunk.get_unchecked(j))); + } + + // mark these bytes as initialised + i += N; + } + out.set_len(i); + } + + out } diff --git a/crux-mir/lib/alloc/src/string.rs b/crux-mir/lib/alloc/src/string.rs index 0e48f1548..3118c7189 100644 --- a/crux-mir/lib/alloc/src/string.rs +++ b/crux-mir/lib/alloc/src/string.rs @@ -1,10 +1,8 @@ -//! A UTF-8 encoded, growable string. +//! A UTF-8–encoded, growable string. //! -//! This module contains the [`String`] type, a trait for converting -//! [`ToString`]s, and several error types that may result from working with -//! [`String`]s. -//! -//! [`ToString`]: trait.ToString.html +//! This module contains the [`String`] type, the [`ToString`] trait for +//! converting to strings, and several error types that may result from +//! working with [`String`]s. //! //! # Examples //! @@ -20,8 +18,6 @@ //! You can create a new [`String`] from an existing one by concatenating with //! `+`: //! -//! [`String`]: struct.String.html -//! //! ``` //! let s = "Hello".to_string(); //! @@ -46,32 +42,47 @@ #![stable(feature = "rust1", since = "1.0.0")] +#[cfg(not(no_global_oom_handling))] use core::char::{decode_utf16, REPLACEMENT_CHARACTER}; +use core::error::Error; use core::fmt; use core::hash; -use core::iter::{FromIterator, FusedIterator}; +use core::iter::FusedIterator; +#[cfg(not(no_global_oom_handling))] +use core::iter::{from_fn, FromIterator}; +#[cfg(not(no_global_oom_handling))] +use core::ops::Add; +#[cfg(not(no_global_oom_handling))] +use core::ops::AddAssign; +#[cfg(not(no_global_oom_handling))] use core::ops::Bound::{Excluded, Included, Unbounded}; -use core::ops::{self, Add, AddAssign, Index, IndexMut, RangeBounds}; +use core::ops::{self, Index, IndexMut, Range, RangeBounds}; use core::ptr; -use core::str::{lossy, pattern::Pattern}; +use core::slice; +use core::str::pattern::Pattern; +#[cfg(not(no_global_oom_handling))] +use core::str::Utf8Chunks; +#[cfg(not(no_global_oom_handling))] use crate::borrow::{Cow, ToOwned}; use crate::boxed::Box; use crate::collections::TryReserveError; -use crate::str::{self, from_boxed_utf8_unchecked, Chars, FromStr, Utf8Error}; +use crate::str::{self, from_utf8_unchecked_mut, Chars, Utf8Error}; +#[cfg(not(no_global_oom_handling))] +use crate::str::{from_boxed_utf8_unchecked, FromStr}; use crate::vec::Vec; -/// A UTF-8 encoded, growable string. +/// A UTF-8–encoded, growable string. /// /// The `String` type is the most common string type that has ownership over the /// contents of the string. It has a close relationship with its borrowed /// counterpart, the primitive [`str`]. /// -/// [`str`]: ../../std/primitive.str.html -/// /// # Examples /// -/// You can create a `String` from a literal string with [`String::from`]: +/// You can create a `String` from [a literal string][`&str`] with [`String::from`]: +/// +/// [`String::from`]: From::from /// /// ``` /// let hello = String::from("Hello, world!"); @@ -87,10 +98,8 @@ use crate::vec::Vec; /// hello.push_str("orld!"); /// ``` /// -/// [`String::from`]: #method.from -/// [`char`]: ../../std/primitive.char.html -/// [`push`]: #method.push -/// [`push_str`]: #method.push_str +/// [`push`]: String::push +/// [`push_str`]: String::push_str /// /// If you have a vector of UTF-8 bytes, you can create a `String` from it with /// the [`from_utf8`] method: @@ -105,35 +114,107 @@ use crate::vec::Vec; /// assert_eq!("💖", sparkle_heart); /// ``` /// -/// [`from_utf8`]: #method.from_utf8 +/// [`from_utf8`]: String::from_utf8 /// /// # UTF-8 /// -/// `String`s are always valid UTF-8. This has a few implications, the first of -/// which is that if you need a non-UTF-8 string, consider [`OsString`]. It is -/// similar, but without the UTF-8 constraint. The second implication is that -/// you cannot index into a `String`: +/// `String`s are always valid UTF-8. If you need a non-UTF-8 string, consider +/// [`OsString`]. It is similar, but without the UTF-8 constraint. Because UTF-8 +/// is a variable width encoding, `String`s are typically smaller than an array of +/// the same `chars`: +/// +/// ``` +/// use std::mem; +/// +/// // `s` is ASCII which represents each `char` as one byte +/// let s = "hello"; +/// assert_eq!(s.len(), 5); +/// +/// // A `char` array with the same contents would be longer because +/// // every `char` is four bytes +/// let s = ['h', 'e', 'l', 'l', 'o']; +/// let size: usize = s.into_iter().map(|c| mem::size_of_val(&c)).sum(); +/// assert_eq!(size, 20); +/// +/// // However, for non-ASCII strings, the difference will be smaller +/// // and sometimes they are the same +/// let s = "💖💖💖💖💖"; +/// assert_eq!(s.len(), 20); +/// +/// let s = ['💖', '💖', '💖', '💖', '💖']; +/// let size: usize = s.into_iter().map(|c| mem::size_of_val(&c)).sum(); +/// assert_eq!(size, 20); +/// ``` +/// +/// This raises interesting questions as to how `s[i]` should work. +/// What should `i` be here? Several options include byte indices and +/// `char` indices but, because of UTF-8 encoding, only byte indices +/// would provide constant time indexing. Getting the `i`th `char`, for +/// example, is available using [`chars`]: +/// +/// ``` +/// let s = "hello"; +/// let third_character = s.chars().nth(2); +/// assert_eq!(third_character, Some('l')); +/// +/// let s = "💖💖💖💖💖"; +/// let third_character = s.chars().nth(2); +/// assert_eq!(third_character, Some('💖')); +/// ``` +/// +/// Next, what should `s[i]` return? Because indexing returns a reference +/// to underlying data it could be `&u8`, `&[u8]`, or something else similar. +/// Since we're only providing one index, `&u8` makes the most sense but that +/// might not be what the user expects and can be explicitly achieved with +/// [`as_bytes()`]: +/// +/// ``` +/// // The first byte is 104 - the byte value of `'h'` +/// let s = "hello"; +/// assert_eq!(s.as_bytes()[0], 104); +/// // or +/// assert_eq!(s.as_bytes()[0], b'h'); +/// +/// // The first byte is 240 which isn't obviously useful +/// let s = "💖💖💖💖💖"; +/// assert_eq!(s.as_bytes()[0], 240); +/// ``` +/// +/// Due to these ambiguities/restrictions, indexing with a `usize` is simply +/// forbidden: /// /// ```compile_fail,E0277 /// let s = "hello"; /// -/// println!("The first letter of s is {}", s[0]); // ERROR!!! +/// // The following will not compile! +/// println!("The first letter of s is {}", s[0]); /// ``` /// -/// [`OsString`]: ../../std/ffi/struct.OsString.html +/// It is more clear, however, how `&s[i..j]` should work (that is, +/// indexing with a range). It should accept byte indices (to be constant-time) +/// and return a `&str` which is UTF-8 encoded. This is also called "string slicing". +/// Note this will panic if the byte indices provided are not character +/// boundaries - see [`is_char_boundary`] for more details. See the implementations +/// for [`SliceIndex`] for more details on string slicing. For a non-panicking +/// version of string slicing, see [`get`]. +/// +/// [`OsString`]: ../../std/ffi/struct.OsString.html "ffi::OsString" +/// [`SliceIndex`]: core::slice::SliceIndex +/// [`as_bytes()`]: str::as_bytes +/// [`get`]: str::get +/// [`is_char_boundary`]: str::is_char_boundary /// -/// Indexing is intended to be a constant-time operation, but UTF-8 encoding -/// does not allow us to do this. Furthermore, it's not clear what sort of -/// thing the index should return: a byte, a codepoint, or a grapheme cluster. -/// The [`bytes`] and [`chars`] methods return iterators over the first -/// two, respectively. +/// The [`bytes`] and [`chars`] methods return iterators over the bytes and +/// codepoints of the string, respectively. To iterate over codepoints along +/// with byte indices, use [`char_indices`]. /// -/// [`bytes`]: #method.bytes -/// [`chars`]: #method.chars +/// [`bytes`]: str::bytes +/// [`chars`]: str::chars +/// [`char_indices`]: str::char_indices /// /// # Deref /// -/// `String`s implement [`Deref`]``, and so inherit all of [`str`]'s +/// `String` implements [Deref], and so inherits all of [`str`]'s /// methods. In addition, this means that you can pass a `String` to a /// function which takes a [`&str`] by using an ampersand (`&`): /// @@ -174,7 +255,7 @@ use crate::vec::Vec; /// to explicitly extract the string slice containing the string. The second /// way changes `example_func(&example_string);` to /// `example_func(&*example_string);`. In this case we are dereferencing a -/// `String` to a [`str`][`&str`], then referencing the [`str`][`&str`] back to +/// `String` to a [`str`], then referencing the [`str`] back to /// [`&str`]. The second way is more idiomatic, however both work to do the /// conversion explicitly rather than relying on the implicit conversion. /// @@ -215,9 +296,9 @@ use crate::vec::Vec; /// assert_eq!(String::from("Once upon a time..."), s); /// ``` /// -/// [`as_ptr`]: #method.as_ptr -/// [`len`]: #method.len -/// [`capacity`]: #method.capacity +/// [`as_ptr`]: str::as_ptr +/// [`len`]: String::len +/// [`capacity`]: String::capacity /// /// If a `String` has enough capacity, adding elements to it will not /// re-allocate. For example, consider this program: @@ -237,11 +318,11 @@ use crate::vec::Vec; /// /// ```text /// 0 -/// 5 -/// 10 -/// 20 -/// 20 -/// 40 +/// 8 +/// 16 +/// 16 +/// 32 +/// 32 /// ``` /// /// At first, we have no memory allocated at all, but as we append to the @@ -259,7 +340,7 @@ use crate::vec::Vec; /// } /// ``` /// -/// [`with_capacity`]: #method.with_capacity +/// [`with_capacity`]: String::with_capacity /// /// We end up with a different output: /// @@ -274,11 +355,15 @@ use crate::vec::Vec; /// /// Here, there's no need to allocate more memory inside the loop. /// -/// [`&str`]: ../../std/primitive.str.html -/// [`Deref`]: ../../std/ops/trait.Deref.html -/// [`as_str()`]: struct.String.html#method.as_str +/// [str]: prim@str "str" +/// [`str`]: prim@str "str" +/// [`&str`]: prim@str "&str" +/// [Deref]: core::ops::Deref "ops::Deref" +/// [`Deref`]: core::ops::Deref "ops::Deref" +/// [`as_str()`]: String::as_str #[derive(PartialOrd, Eq, Ord)] #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), lang = "String")] pub struct String { vec: Vec, } @@ -290,20 +375,18 @@ pub struct String { /// [`into_bytes`] method will give back the byte vector that was used in the /// conversion attempt. /// -/// [`from_utf8`]: struct.String.html#method.from_utf8 -/// [`String`]: struct.String.html -/// [`into_bytes`]: struct.FromUtf8Error.html#method.into_bytes +/// [`from_utf8`]: String::from_utf8 +/// [`into_bytes`]: FromUtf8Error::into_bytes /// /// The [`Utf8Error`] type provided by [`std::str`] represents an error that may /// occur when converting a slice of [`u8`]s to a [`&str`]. In this sense, it's /// an analogue to `FromUtf8Error`, and you can get one from a `FromUtf8Error` /// through the [`utf8_error`] method. /// -/// [`Utf8Error`]: ../../std/str/struct.Utf8Error.html -/// [`std::str`]: ../../std/str/index.html -/// [`u8`]: ../../std/primitive.u8.html -/// [`&str`]: ../../std/primitive.str.html -/// [`utf8_error`]: #method.utf8_error +/// [`Utf8Error`]: str::Utf8Error "std::str::Utf8Error" +/// [`std::str`]: core::str "std::str" +/// [`&str`]: prim@str "&str" +/// [`utf8_error`]: FromUtf8Error::utf8_error /// /// # Examples /// @@ -319,7 +402,8 @@ pub struct String { /// assert_eq!(vec![0, 159], value.unwrap_err().into_bytes()); /// ``` #[stable(feature = "rust1", since = "1.0.0")] -#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(not(no_global_oom_handling), derive(Clone))] +#[derive(Debug, PartialEq, Eq)] pub struct FromUtf8Error { bytes: Vec, error: Utf8Error, @@ -329,9 +413,7 @@ pub struct FromUtf8Error { /// /// This type is the error type for the [`from_utf16`] method on [`String`]. /// -/// [`from_utf16`]: struct.String.html#method.from_utf16 -/// [`String`]: struct.String.html -/// +/// [`from_utf16`]: String::from_utf16 /// # Examples /// /// Basic usage: @@ -357,7 +439,7 @@ impl String { /// consider the [`with_capacity`] method to prevent excessive /// re-allocation. /// - /// [`with_capacity`]: #method.with_capacity + /// [`with_capacity`]: String::with_capacity /// /// # Examples /// @@ -367,27 +449,28 @@ impl String { /// let s = String::new(); /// ``` #[inline] - #[rustc_const_stable(feature = "const_string_new", since = "1.32.0")] + #[rustc_const_stable(feature = "const_string_new", since = "1.39.0")] #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] pub const fn new() -> String { String { vec: Vec::new() } } - /// Creates a new empty `String` with a particular capacity. + /// Creates a new empty `String` with at least the specified capacity. /// /// `String`s have an internal buffer to hold their data. The capacity is /// the length of that buffer, and can be queried with the [`capacity`] /// method. This method creates an empty `String`, but one with an initial - /// buffer that can hold `capacity` bytes. This is useful when you may be - /// appending a bunch of data to the `String`, reducing the number of + /// buffer that can hold at least `capacity` bytes. This is useful when you + /// may be appending a bunch of data to the `String`, reducing the number of /// reallocations it needs to do. /// - /// [`capacity`]: #method.capacity + /// [`capacity`]: String::capacity /// /// If the given capacity is `0`, no allocation will occur, and this method /// is identical to the [`new`] method. /// - /// [`new`]: #method.new + /// [`new`]: String::new /// /// # Examples /// @@ -410,8 +493,10 @@ impl String { /// // ...but this may make the string reallocate /// s.push('a'); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] pub fn with_capacity(capacity: usize) -> String { String { vec: Vec::with_capacity(capacity) } } @@ -478,14 +563,10 @@ impl String { /// See the docs for [`FromUtf8Error`] for more details on what you can do /// with this error. /// - /// [`from_utf8_unchecked`]: struct.String.html#method.from_utf8_unchecked - /// [`String`]: struct.String.html - /// [`u8`]: ../../std/primitive.u8.html - /// [`Vec`]: ../../std/vec/struct.Vec.html - /// [`str::from_utf8`]: ../../std/str/fn.from_utf8.html - /// [`into_bytes`]: struct.String.html#method.into_bytes - /// [`FromUtf8Error`]: struct.FromUtf8Error.html - /// [`Err`]: ../../std/result/enum.Result.html#variant.Err + /// [`from_utf8_unchecked`]: String::from_utf8_unchecked + /// [`Vec`]: crate::vec::Vec "Vec" + /// [`&str`]: prim@str "&str" + /// [`into_bytes`]: String::into_bytes #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn from_utf8(vec: Vec) -> Result { @@ -504,16 +585,15 @@ impl String { /// `from_utf8_lossy()` will replace any invalid UTF-8 sequences with /// [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD], which looks like this: � /// - /// [`u8`]: ../../std/primitive.u8.html - /// [byteslice]: ../../std/primitive.slice.html - /// [U+FFFD]: ../char/constant.REPLACEMENT_CHARACTER.html + /// [byteslice]: prim@slice + /// [U+FFFD]: core::char::REPLACEMENT_CHARACTER /// /// If you are sure that the byte slice is valid UTF-8, and you don't want /// to incur the overhead of the conversion, there is an unsafe version /// of this function, [`from_utf8_unchecked`], which has the same behavior /// but skips the checks. /// - /// [`from_utf8_unchecked`]: struct.String.html#method.from_utf8_unchecked + /// [`from_utf8_unchecked`]: String::from_utf8_unchecked /// /// This function returns a [`Cow<'a, str>`]. If our byte slice is invalid /// UTF-8, then we need to insert the replacement characters, which will @@ -521,7 +601,7 @@ impl String { /// it's already valid UTF-8, we don't need a new allocation. This return /// type allows us to handle both cases. /// - /// [`Cow<'a, str>`]: ../../std/borrow/enum.Cow.html + /// [`Cow<'a, str>`]: crate::borrow::Cow "borrow::Cow" /// /// # Examples /// @@ -545,17 +625,19 @@ impl String { /// /// assert_eq!("Hello �World", output); /// ``` + #[must_use] + #[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] pub fn from_utf8_lossy(v: &[u8]) -> Cow<'_, str> { - let mut iter = lossy::Utf8Lossy::from_bytes(v).chunks(); + let mut iter = Utf8Chunks::new(v); - let (first_valid, first_broken) = if let Some(chunk) = iter.next() { - let lossy::Utf8LossyChunk { valid, broken } = chunk; - if valid.len() == v.len() { - debug_assert!(broken.is_empty()); + let first_valid = if let Some(chunk) = iter.next() { + let valid = chunk.valid(); + if chunk.invalid().is_empty() { + debug_assert_eq!(valid.len(), v.len()); return Cow::Borrowed(valid); } - (valid, broken) + valid } else { return Cow::Borrowed(""); }; @@ -564,13 +646,11 @@ impl String { let mut res = String::with_capacity(v.len()); res.push_str(first_valid); - if !first_broken.is_empty() { - res.push_str(REPLACEMENT); - } + res.push_str(REPLACEMENT); - for lossy::Utf8LossyChunk { valid, broken } in iter { - res.push_str(valid); - if !broken.is_empty() { + for chunk in iter { + res.push_str(chunk.valid()); + if !chunk.invalid().is_empty() { res.push_str(REPLACEMENT); } } @@ -578,11 +658,9 @@ impl String { Cow::Owned(res) } - /// Decode a UTF-16 encoded vector `v` into a `String`, returning [`Err`] + /// Decode a UTF-16–encoded vector `v` into a `String`, returning [`Err`] /// if `v` contains any invalid data. /// - /// [`Err`]: ../../std/result/enum.Result.html#variant.Err - /// /// # Examples /// /// Basic usage: @@ -599,6 +677,7 @@ impl String { /// 0xD800, 0x0069, 0x0063]; /// assert!(String::from_utf16(v).is_err()); /// ``` + #[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] pub fn from_utf16(v: &[u16]) -> Result { // This isn't done via collect::>() for performance reasons. @@ -614,16 +693,16 @@ impl String { Ok(ret) } - /// Decode a UTF-16 encoded slice `v` into a `String`, replacing + /// Decode a UTF-16–encoded slice `v` into a `String`, replacing /// invalid data with [the replacement character (`U+FFFD`)][U+FFFD]. /// /// Unlike [`from_utf8_lossy`] which returns a [`Cow<'a, str>`], /// `from_utf16_lossy` returns a `String` since the UTF-16 to UTF-8 /// conversion requires a memory allocation. /// - /// [`from_utf8_lossy`]: #method.from_utf8_lossy - /// [`Cow<'a, str>`]: ../borrow/enum.Cow.html - /// [U+FFFD]: ../char/constant.REPLACEMENT_CHARACTER.html + /// [`from_utf8_lossy`]: String::from_utf8_lossy + /// [`Cow<'a, str>`]: crate::borrow::Cow "borrow::Cow" + /// [U+FFFD]: core::char::REPLACEMENT_CHARACTER /// /// # Examples /// @@ -638,6 +717,8 @@ impl String { /// assert_eq!(String::from("𝄞mus\u{FFFD}ic\u{FFFD}"), /// String::from_utf16_lossy(v)); /// ``` + #[cfg(not(no_global_oom_handling))] + #[must_use] #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn from_utf16_lossy(v: &[u16]) -> String { @@ -657,7 +738,7 @@ impl String { /// into a `String` with the [`from_raw_parts`] function, allowing /// the destructor to perform the cleanup. /// - /// [`from_raw_parts`]: #method.from_raw_parts + /// [`from_raw_parts`]: String::from_raw_parts /// /// # Examples /// @@ -670,6 +751,7 @@ impl String { /// let rebuilt = unsafe { String::from_raw_parts(ptr, len, cap) }; /// assert_eq!(rebuilt, "hello"); /// ``` + #[must_use = "`self` will be dropped if the result is not used"] #[unstable(feature = "vec_into_raw_parts", reason = "new API", issue = "65816")] pub fn into_raw_parts(self) -> (*mut u8, usize, usize) { self.vec.into_raw_parts() @@ -682,15 +764,19 @@ impl String { /// This is highly unsafe, due to the number of invariants that aren't /// checked: /// - /// * The memory at `ptr` needs to have been previously allocated by the + /// * The memory at `buf` needs to have been previously allocated by the /// same allocator the standard library uses, with a required alignment of exactly 1. /// * `length` needs to be less than or equal to `capacity`. /// * `capacity` needs to be the correct value. + /// * The first `length` bytes at `buf` need to be valid UTF-8. /// /// Violating these may cause problems like corrupting the allocator's - /// internal data structures. + /// internal data structures. For example, it is normally **not** safe to + /// build a `String` from a pointer to a C `char` array containing UTF-8 + /// _unless_ you are certain that array was originally allocated by the + /// Rust standard library's allocator. /// - /// The ownership of `ptr` is effectively transferred to the + /// The ownership of `buf` is effectively transferred to the /// `String` which may then deallocate, reallocate or change the /// contents of memory pointed to by the pointer at will. Ensure /// that nothing else uses the pointer after calling this @@ -722,7 +808,7 @@ impl String { #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub unsafe fn from_raw_parts(buf: *mut u8, length: usize, capacity: usize) -> String { - String { vec: Vec::from_raw_parts(buf, length, capacity) } + unsafe { String { vec: Vec::from_raw_parts(buf, length, capacity) } } } /// Converts a vector of bytes to a `String` without checking that the @@ -730,7 +816,7 @@ impl String { /// /// See the safe version, [`from_utf8`], for more details. /// - /// [`from_utf8`]: struct.String.html#method.from_utf8 + /// [`from_utf8`]: String::from_utf8 /// /// # Safety /// @@ -754,6 +840,7 @@ impl String { /// assert_eq!("💖", sparkle_heart); /// ``` #[inline] + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub unsafe fn from_utf8_unchecked(bytes: Vec) -> String { String { vec: bytes } @@ -774,6 +861,7 @@ impl String { /// assert_eq!(&[104, 101, 108, 108, 111][..], &bytes[..]); /// ``` #[inline] + #[must_use = "`self` will be dropped if the result is not used"] #[stable(feature = "rust1", since = "1.0.0")] pub fn into_bytes(self) -> Vec { self.vec @@ -791,6 +879,7 @@ impl String { /// assert_eq!("foo", s.as_str()); /// ``` #[inline] + #[must_use] #[stable(feature = "string_as_str", since = "1.7.0")] pub fn as_str(&self) -> &str { self @@ -811,6 +900,7 @@ impl String { /// assert_eq!("FOOBAR", s_mut_str); /// ``` #[inline] + #[must_use] #[stable(feature = "string_as_str", since = "1.7.0")] pub fn as_mut_str(&mut self) -> &mut str { self @@ -829,12 +919,49 @@ impl String { /// /// assert_eq!("foobar", s); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn push_str(&mut self, string: &str) { self.vec.extend_from_slice(string.as_bytes()) } + /// Copies elements from `src` range to the end of the string. + /// + /// ## Panics + /// + /// Panics if the starting point or end point do not lie on a [`char`] + /// boundary, or if they're out of bounds. + /// + /// ## Examples + /// + /// ``` + /// #![feature(string_extend_from_within)] + /// let mut string = String::from("abcde"); + /// + /// string.extend_from_within(2..); + /// assert_eq!(string, "abcdecde"); + /// + /// string.extend_from_within(..2); + /// assert_eq!(string, "abcdecdeab"); + /// + /// string.extend_from_within(4..8); + /// assert_eq!(string, "abcdecdeabecde"); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[unstable(feature = "string_extend_from_within", issue = "103806")] + pub fn extend_from_within(&mut self, src: R) + where + R: RangeBounds, + { + let src @ Range { start, end } = slice::range(src, ..self.len()); + + assert!(self.is_char_boundary(start)); + assert!(self.is_char_boundary(end)); + + self.vec.extend_from_within(src); + } + /// Returns this `String`'s capacity, in bytes. /// /// # Examples @@ -847,27 +974,22 @@ impl String { /// assert!(s.capacity() >= 10); /// ``` #[inline] + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn capacity(&self) -> usize { self.vec.capacity() } - /// Ensures that this `String`'s capacity is at least `additional` bytes - /// larger than its length. - /// - /// The capacity may be increased by more than `additional` bytes if it - /// chooses, to prevent frequent reallocations. - /// - /// If you do not want this "at least" behavior, see the [`reserve_exact`] - /// method. + /// Reserves capacity for at least `additional` bytes more than the + /// current length. The allocator may reserve more space to speculatively + /// avoid frequent allocations. After calling `reserve`, + /// capacity will be greater than or equal to `self.len() + additional`. + /// Does nothing if capacity is already sufficient. /// /// # Panics /// /// Panics if the new capacity overflows [`usize`]. /// - /// [`reserve_exact`]: struct.String.html#method.reserve_exact - /// [`usize`]: ../../std/primitive.usize.html - /// /// # Examples /// /// Basic usage: @@ -880,40 +1002,43 @@ impl String { /// assert!(s.capacity() >= 10); /// ``` /// - /// This may not actually increase the capacity: + /// This might not actually increase the capacity: /// /// ``` /// let mut s = String::with_capacity(10); /// s.push('a'); /// s.push('b'); /// - /// // s now has a length of 2 and a capacity of 10 + /// // s now has a length of 2 and a capacity of at least 10 + /// let capacity = s.capacity(); /// assert_eq!(2, s.len()); - /// assert_eq!(10, s.capacity()); + /// assert!(capacity >= 10); /// - /// // Since we already have an extra 8 capacity, calling this... + /// // Since we already have at least an extra 8 capacity, calling this... /// s.reserve(8); /// /// // ... doesn't actually increase. - /// assert_eq!(10, s.capacity()); + /// assert_eq!(capacity, s.capacity()); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn reserve(&mut self, additional: usize) { self.vec.reserve(additional) } - /// Ensures that this `String`'s capacity is `additional` bytes - /// larger than its length. + /// Reserves the minimum capacity for at least `additional` bytes more than + /// the current length. Unlike [`reserve`], this will not + /// deliberately over-allocate to speculatively avoid frequent allocations. + /// After calling `reserve_exact`, capacity will be greater than or equal to + /// `self.len() + additional`. Does nothing if the capacity is already + /// sufficient. /// - /// Consider using the [`reserve`] method unless you absolutely know - /// better than the allocator. - /// - /// [`reserve`]: #method.reserve + /// [`reserve`]: String::reserve /// /// # Panics /// - /// Panics if the new capacity overflows `usize`. + /// Panics if the new capacity overflows [`usize`]. /// /// # Examples /// @@ -927,34 +1052,37 @@ impl String { /// assert!(s.capacity() >= 10); /// ``` /// - /// This may not actually increase the capacity: + /// This might not actually increase the capacity: /// /// ``` /// let mut s = String::with_capacity(10); /// s.push('a'); /// s.push('b'); /// - /// // s now has a length of 2 and a capacity of 10 + /// // s now has a length of 2 and a capacity of at least 10 + /// let capacity = s.capacity(); /// assert_eq!(2, s.len()); - /// assert_eq!(10, s.capacity()); + /// assert!(capacity >= 10); /// - /// // Since we already have an extra 8 capacity, calling this... + /// // Since we already have at least an extra 8 capacity, calling this... /// s.reserve_exact(8); /// /// // ... doesn't actually increase. - /// assert_eq!(10, s.capacity()); + /// assert_eq!(capacity, s.capacity()); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn reserve_exact(&mut self, additional: usize) { self.vec.reserve_exact(additional) } - /// Tries to reserve capacity for at least `additional` more elements to be inserted - /// in the given `String`. The collection may reserve more space to avoid - /// frequent reallocations. After calling `reserve`, capacity will be - /// greater than or equal to `self.len() + additional`. Does nothing if - /// capacity is already sufficient. + /// Tries to reserve capacity for at least `additional` bytes more than the + /// current length. The allocator may reserve more space to speculatively + /// avoid frequent allocations. After calling `try_reserve`, capacity will be + /// greater than or equal to `self.len() + additional` if it returns + /// `Ok(())`. Does nothing if capacity is already sufficient. This method + /// preserves the contents even if an error occurs. /// /// # Errors /// @@ -964,7 +1092,6 @@ impl String { /// # Examples /// /// ``` - /// #![feature(try_reserve)] /// use std::collections::TryReserveError; /// /// fn process_data(data: &str) -> Result { @@ -980,19 +1107,23 @@ impl String { /// } /// # process_data("rust").expect("why is the test harness OOMing on 4 bytes?"); /// ``` - #[unstable(feature = "try_reserve", reason = "new API", issue = "48043")] + #[stable(feature = "try_reserve", since = "1.57.0")] pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { self.vec.try_reserve(additional) } - /// Tries to reserves the minimum capacity for exactly `additional` more elements to - /// be inserted in the given `String`. After calling `reserve_exact`, - /// capacity will be greater than or equal to `self.len() + additional`. + /// Tries to reserve the minimum capacity for at least `additional` bytes + /// more than the current length. Unlike [`try_reserve`], this will not + /// deliberately over-allocate to speculatively avoid frequent allocations. + /// After calling `try_reserve_exact`, capacity will be greater than or + /// equal to `self.len() + additional` if it returns `Ok(())`. /// Does nothing if the capacity is already sufficient. /// /// Note that the allocator may give the collection more space than it /// requests. Therefore, capacity can not be relied upon to be precisely - /// minimal. Prefer `reserve` if future insertions are expected. + /// minimal. Prefer [`try_reserve`] if future insertions are expected. + /// + /// [`try_reserve`]: String::try_reserve /// /// # Errors /// @@ -1002,14 +1133,13 @@ impl String { /// # Examples /// /// ``` - /// #![feature(try_reserve)] /// use std::collections::TryReserveError; /// /// fn process_data(data: &str) -> Result { /// let mut output = String::new(); /// /// // Pre-reserve the memory, exiting if we can't - /// output.try_reserve(data.len())?; + /// output.try_reserve_exact(data.len())?; /// /// // Now we know this can't OOM in the middle of our complex work /// output.push_str(data); @@ -1018,7 +1148,7 @@ impl String { /// } /// # process_data("rust").expect("why is the test harness OOMing on 4 bytes?"); /// ``` - #[unstable(feature = "try_reserve", reason = "new API", issue = "48043")] + #[stable(feature = "try_reserve", since = "1.57.0")] pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> { self.vec.try_reserve_exact(additional) } @@ -1038,6 +1168,7 @@ impl String { /// s.shrink_to_fit(); /// assert_eq!(3, s.capacity()); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn shrink_to_fit(&mut self) { @@ -1049,13 +1180,11 @@ impl String { /// The capacity will remain at least as large as both the length /// and the supplied value. /// - /// Panics if the current capacity is smaller than the supplied - /// minimum capacity. + /// If the current capacity is less than the lower limit, this is a no-op. /// /// # Examples /// /// ``` - /// #![feature(shrink_to)] /// let mut s = String::from("foo"); /// /// s.reserve(100); @@ -1066,16 +1195,15 @@ impl String { /// s.shrink_to(0); /// assert!(s.capacity() >= 3); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] - #[unstable(feature = "shrink_to", reason = "new API", issue = "56431")] + #[stable(feature = "shrink_to", since = "1.56.0")] pub fn shrink_to(&mut self, min_capacity: usize) { self.vec.shrink_to(min_capacity) } /// Appends the given [`char`] to the end of this `String`. /// - /// [`char`]: ../../std/primitive.char.html - /// /// # Examples /// /// Basic usage: @@ -1089,6 +1217,7 @@ impl String { /// /// assert_eq!("abc123", s); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn push(&mut self, ch: char) { @@ -1102,7 +1231,7 @@ impl String { /// /// The inverse of this method is [`from_utf8`]. /// - /// [`from_utf8`]: #method.from_utf8 + /// [`from_utf8`]: String::from_utf8 /// /// # Examples /// @@ -1114,6 +1243,7 @@ impl String { /// assert_eq!(&[104, 101, 108, 108, 111], s.as_bytes()); /// ``` #[inline] + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn as_bytes(&self) -> &[u8] { &self.vec @@ -1131,8 +1261,6 @@ impl String { /// /// Panics if `new_len` does not lie on a [`char`] boundary. /// - /// [`char`]: ../../std/primitive.char.html - /// /// # Examples /// /// Basic usage: @@ -1157,8 +1285,6 @@ impl String { /// /// Returns [`None`] if this `String` is empty. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// /// # Examples /// /// Basic usage: @@ -1185,7 +1311,7 @@ impl String { /// Removes a [`char`] from this `String` at a byte position and returns it. /// - /// This is an `O(n)` operation, as it requires copying every element in the + /// This is an *O*(*n*) operation, as it requires copying every element in the /// buffer. /// /// # Panics @@ -1193,8 +1319,6 @@ impl String { /// Panics if `idx` is larger than or equal to the `String`'s length, /// or if it does not lie on a [`char`] boundary. /// - /// [`char`]: ../../std/primitive.char.html - /// /// # Examples /// /// Basic usage: @@ -1223,6 +1347,80 @@ impl String { ch } + /// Remove all matches of pattern `pat` in the `String`. + /// + /// # Examples + /// + /// ``` + /// #![feature(string_remove_matches)] + /// let mut s = String::from("Trees are not green, the sky is not blue."); + /// s.remove_matches("not "); + /// assert_eq!("Trees are green, the sky is blue.", s); + /// ``` + /// + /// Matches will be detected and removed iteratively, so in cases where + /// patterns overlap, only the first pattern will be removed: + /// + /// ``` + /// #![feature(string_remove_matches)] + /// let mut s = String::from("banana"); + /// s.remove_matches("ana"); + /// assert_eq!("bna", s); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[unstable(feature = "string_remove_matches", reason = "new API", issue = "72826")] + pub fn remove_matches<'a, P>(&'a mut self, pat: P) + where + P: for<'x> Pattern<'x>, + { + use core::str::pattern::Searcher; + + let rejections = { + let mut searcher = pat.into_searcher(self); + // Per Searcher::next: + // + // A Match result needs to contain the whole matched pattern, + // however Reject results may be split up into arbitrary many + // adjacent fragments. Both ranges may have zero length. + // + // In practice the implementation of Searcher::next_match tends to + // be more efficient, so we use it here and do some work to invert + // matches into rejections since that's what we want to copy below. + let mut front = 0; + let rejections: Vec<_> = from_fn(|| { + let (start, end) = searcher.next_match()?; + let prev_front = front; + front = end; + Some((prev_front, start)) + }) + .collect(); + rejections.into_iter().chain(core::iter::once((front, self.len()))) + }; + + let mut len = 0; + let ptr = self.vec.as_mut_ptr(); + + for (start, end) in rejections { + let count = end - start; + if start != len { + // SAFETY: per Searcher::next: + // + // The stream of Match and Reject values up to a Done will + // contain index ranges that are adjacent, non-overlapping, + // covering the whole haystack, and laying on utf8 + // boundaries. + unsafe { + ptr::copy(ptr.add(start), ptr.add(len), count); + } + } + len += count; + } + + unsafe { + self.vec.set_len(len); + } + } + /// Retains only the characters specified by the predicate. /// /// In other words, remove all characters `c` such that `f(c)` returns `false`. @@ -1239,13 +1437,14 @@ impl String { /// assert_eq!(s, "foobar"); /// ``` /// - /// The exact order may be useful for tracking external state, like an index. + /// Because the elements are visited exactly once in the original order, + /// external state may be used to decide which elements to keep. /// /// ``` /// let mut s = String::from("abcde"); /// let keep = [false, true, true, false, true]; - /// let mut i = 0; - /// s.retain(|_| (keep[i], i += 1).0); + /// let mut iter = keep.iter(); + /// s.retain(|_| *iter.next().unwrap()); /// assert_eq!(s, "bce"); /// ``` #[inline] @@ -1254,40 +1453,58 @@ impl String { where F: FnMut(char) -> bool, { - let len = self.len(); - let mut del_bytes = 0; - let mut idx = 0; + struct SetLenOnDrop<'a> { + s: &'a mut String, + idx: usize, + del_bytes: usize, + } - while idx < len { - let ch = unsafe { self.get_unchecked(idx..len).chars().next().unwrap() }; + impl<'a> Drop for SetLenOnDrop<'a> { + fn drop(&mut self) { + let new_len = self.idx - self.del_bytes; + debug_assert!(new_len <= self.s.len()); + unsafe { self.s.vec.set_len(new_len) }; + } + } + + let len = self.len(); + let mut guard = SetLenOnDrop { s: self, idx: 0, del_bytes: 0 }; + + while guard.idx < len { + let ch = + // SAFETY: `guard.idx` is positive-or-zero and less that len so the `get_unchecked` + // is in bound. `self` is valid UTF-8 like string and the returned slice starts at + // a unicode code point so the `Chars` always return one character. + unsafe { guard.s.get_unchecked(guard.idx..len).chars().next().unwrap_unchecked() }; let ch_len = ch.len_utf8(); if !f(ch) { - del_bytes += ch_len; - } else if del_bytes > 0 { - unsafe { - ptr::copy( - self.vec.as_ptr().add(idx), - self.vec.as_mut_ptr().add(idx - del_bytes), - ch_len, - ); - } + guard.del_bytes += ch_len; + } else if guard.del_bytes > 0 { + // SAFETY: `guard.idx` is in bound and `guard.del_bytes` represent the number of + // bytes that are erased from the string so the resulting `guard.idx - + // guard.del_bytes` always represent a valid unicode code point. + // + // `guard.del_bytes` >= `ch.len_utf8()`, so taking a slice with `ch.len_utf8()` len + // is safe. + ch.encode_utf8(unsafe { + crate::slice::from_raw_parts_mut( + guard.s.as_mut_ptr().add(guard.idx - guard.del_bytes), + ch.len_utf8(), + ) + }); } // Point idx to the next char - idx += ch_len; + guard.idx += ch_len; } - if del_bytes > 0 { - unsafe { - self.vec.set_len(len - del_bytes); - } - } + drop(guard); } /// Inserts a character into this `String` at a byte position. /// - /// This is an `O(n)` operation as it requires copying every element in the + /// This is an *O*(*n*) operation as it requires copying every element in the /// buffer. /// /// # Panics @@ -1295,8 +1512,6 @@ impl String { /// Panics if `idx` is larger than the `String`'s length, or if it does not /// lie on a [`char`] boundary. /// - /// [`char`]: ../../std/primitive.char.html - /// /// # Examples /// /// Basic usage: @@ -1310,6 +1525,7 @@ impl String { /// /// assert_eq!("foo", s); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn insert(&mut self, idx: usize, ch: char) { @@ -1322,19 +1538,22 @@ impl String { } } + #[cfg(not(no_global_oom_handling))] unsafe fn insert_bytes(&mut self, idx: usize, bytes: &[u8]) { let len = self.len(); let amt = bytes.len(); self.vec.reserve(amt); - ptr::copy(self.vec.as_ptr().add(idx), self.vec.as_mut_ptr().add(idx + amt), len - idx); - ptr::copy(bytes.as_ptr(), self.vec.as_mut_ptr().add(idx), amt); - self.vec.set_len(len + amt); + unsafe { + ptr::copy(self.vec.as_ptr().add(idx), self.vec.as_mut_ptr().add(idx + amt), len - idx); + ptr::copy_nonoverlapping(bytes.as_ptr(), self.vec.as_mut_ptr().add(idx), amt); + self.vec.set_len(len + amt); + } } /// Inserts a string slice into this `String` at a byte position. /// - /// This is an `O(n)` operation as it requires copying every element in the + /// This is an *O*(*n*) operation as it requires copying every element in the /// buffer. /// /// # Panics @@ -1342,8 +1561,6 @@ impl String { /// Panics if `idx` is larger than the `String`'s length, or if it does not /// lie on a [`char`] boundary. /// - /// [`char`]: ../../std/primitive.char.html - /// /// # Examples /// /// Basic usage: @@ -1355,6 +1572,7 @@ impl String { /// /// assert_eq!("foobar", s); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "insert_str", since = "1.16.0")] pub fn insert_str(&mut self, idx: usize, string: &str) { @@ -1369,10 +1587,11 @@ impl String { /// /// # Safety /// - /// This function is unsafe because it does not check that the bytes passed - /// to it are valid UTF-8. If this constraint is violated, it may cause - /// memory unsafety issues with future users of the `String`, as the rest of - /// the standard library assumes that `String`s are valid UTF-8. + /// This function is unsafe because the returned `&mut Vec` allows writing + /// bytes which are not valid UTF-8. If this constraint is violated, using + /// the original `String` after dropping the `&mut Vec` may violate memory + /// safety, as the rest of the standard library assumes that `String`s are + /// valid UTF-8. /// /// # Examples /// @@ -1396,7 +1615,7 @@ impl String { } /// Returns the length of this `String`, in bytes, not [`char`]s or - /// graphemes. In other words, it may not be what a human considers the + /// graphemes. In other words, it might not be what a human considers the /// length of the string. /// /// # Examples @@ -1412,6 +1631,7 @@ impl String { /// assert_eq!(fancy_f.chars().count(), 3); /// ``` #[inline] + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn len(&self) -> usize { self.vec.len() @@ -1431,12 +1651,13 @@ impl String { /// assert!(!v.is_empty()); /// ``` #[inline] + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn is_empty(&self) -> bool { self.len() == 0 } - /// Splits the string into two at the given index. + /// Splits the string into two at the given byte index. /// /// Returns a newly allocated `String`. `self` contains bytes `[0, at)`, and /// the returned `String` contains bytes `[at, len)`. `at` must be on the @@ -1459,8 +1680,10 @@ impl String { /// assert_eq!(world, "World!"); /// # } /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "string_split_off", since = "1.16.0")] + #[must_use = "use `.truncate()` if you don't need the other half"] pub fn split_off(&mut self, at: usize) -> String { assert!(self.is_char_boundary(at)); let other = self.vec.split_off(at); @@ -1491,18 +1714,23 @@ impl String { self.vec.clear() } - /// Creates a draining iterator that removes the specified range in the `String` - /// and yields the removed `chars`. + /// Removes the specified range from the string in bulk, returning all + /// removed characters as an iterator. /// - /// Note: The element range is removed even if the iterator is not - /// consumed until the end. + /// The returned iterator keeps a mutable borrow on the string to optimize + /// its implementation. /// /// # Panics /// /// Panics if the starting point or end point do not lie on a [`char`] /// boundary, or if they're out of bounds. /// - /// [`char`]: ../../std/primitive.char.html + /// # Leaking + /// + /// If the returned iterator goes out of scope without being dropped (due to + /// [`core::mem::forget`], for example), the string may still contain a copy + /// of any drained characters, or may have lost characters arbitrarily, + /// including characters outside the range. /// /// # Examples /// @@ -1517,7 +1745,7 @@ impl String { /// assert_eq!(t, "α is alpha, "); /// assert_eq!(s, "β is beta"); /// - /// // A full range clears the string + /// // A full range clears the string, like `clear()` does /// s.drain(..); /// assert_eq!(s, ""); /// ``` @@ -1532,23 +1760,15 @@ impl String { // of the vector version. The data is just plain bytes. // Because the range removal happens in Drop, if the Drain iterator is leaked, // the removal will not happen. - let len = self.len(); - let start = match range.start_bound() { - Included(&n) => n, - Excluded(&n) => n + 1, - Unbounded => 0, - }; - let end = match range.end_bound() { - Included(&n) => n + 1, - Excluded(&n) => n, - Unbounded => len, - }; + let Range { start, end } = slice::range(range, ..self.len()); + assert!(self.is_char_boundary(start)); + assert!(self.is_char_boundary(end)); // Take out two simultaneous borrows. The &mut String won't be accessed // until iteration is over, in Drop. let self_ptr = self as *mut _; - // slicing does the appropriate bounds checks - let chars_iter = self[start..end].chars(); + // SAFETY: `slice::range` and `is_char_boundary` do the appropriate bounds checks. + let chars_iter = unsafe { self.get_unchecked(start..end) }.chars(); Drain { start, end, iter: chars_iter, string: self_ptr } } @@ -1562,9 +1782,6 @@ impl String { /// Panics if the starting point or end point do not lie on a [`char`] /// boundary, or if they're out of bounds. /// - /// [`char`]: ../../std/primitive.char.html - /// [`Vec::splice`]: ../../std/vec/struct.Vec.html#method.splice - /// /// # Examples /// /// Basic usage: @@ -1577,6 +1794,7 @@ impl String { /// s.replace_range(..beta_offset, "Α is capital alpha; "); /// assert_eq!(s, "Α is capital alpha; β is beta"); /// ``` + #[cfg(not(no_global_oom_handling))] #[stable(feature = "splice", since = "1.27.0")] pub fn replace_range(&mut self, range: R, replace_with: &str) where @@ -1587,26 +1805,32 @@ impl String { // Replace_range does not have the memory safety issues of a vector Splice. // of the vector version. The data is just plain bytes. - match range.start_bound() { + // WARNING: Inlining this variable would be unsound (#81138) + let start = range.start_bound(); + match start { Included(&n) => assert!(self.is_char_boundary(n)), Excluded(&n) => assert!(self.is_char_boundary(n + 1)), Unbounded => {} }; - match range.end_bound() { + // WARNING: Inlining this variable would be unsound (#81138) + let end = range.end_bound(); + match end { Included(&n) => assert!(self.is_char_boundary(n + 1)), Excluded(&n) => assert!(self.is_char_boundary(n)), Unbounded => {} }; - unsafe { self.as_mut_vec() }.splice(range, replace_with.bytes()); + // Using `range` again would be unsound (#81138) + // We assume the bounds reported by `range` remain the same, but + // an adversarial implementation could change between calls + unsafe { self.as_mut_vec() }.splice((start, end), replace_with.bytes()); } - /// Converts this `String` into a [`Box`]`<`[`str`]`>`. + /// Converts this `String` into a [Box]<[str]>. /// /// This will drop any excess capacity. /// - /// [`Box`]: ../../std/boxed/struct.Box.html - /// [`str`]: ../../std/primitive.str.html + /// [str]: prim@str "str" /// /// # Examples /// @@ -1617,12 +1841,43 @@ impl String { /// /// let b = s.into_boxed_str(); /// ``` + #[cfg(not(no_global_oom_handling))] #[stable(feature = "box_str", since = "1.4.0")] + #[must_use = "`self` will be dropped if the result is not used"] #[inline] pub fn into_boxed_str(self) -> Box { let slice = self.vec.into_boxed_slice(); unsafe { from_boxed_utf8_unchecked(slice) } } + + /// Consumes and leaks the `String`, returning a mutable reference to the contents, + /// `&'static mut str`. + /// + /// This is mainly useful for data that lives for the remainder of + /// the program's life. Dropping the returned reference will cause a memory + /// leak. + /// + /// It does not reallocate or shrink the `String`, + /// so the leaked allocation may include unused capacity that is not part + /// of the returned slice. + /// + /// # Examples + /// + /// Simple usage: + /// + /// ``` + /// #![feature(string_leak)] + /// + /// let x = String::from("bucket"); + /// let static_ref: &'static mut str = x.leak(); + /// assert_eq!(static_ref, "bucket"); + /// ``` + #[unstable(feature = "string_leak", issue = "102929")] + #[inline] + pub fn leak(self) -> &'static mut str { + let slice = self.vec.leak(); + unsafe { from_utf8_unchecked_mut(slice) } + } } impl FromUtf8Error { @@ -1640,6 +1895,7 @@ impl FromUtf8Error { /// /// assert_eq!(&[0, 159], value.unwrap_err().as_bytes()); /// ``` + #[must_use] #[stable(feature = "from_utf8_error_as_bytes", since = "1.26.0")] pub fn as_bytes(&self) -> &[u8] { &self.bytes[..] @@ -1663,6 +1919,7 @@ impl FromUtf8Error { /// /// assert_eq!(vec![0, 159], value.unwrap_err().into_bytes()); /// ``` + #[must_use = "`self` will be dropped if the result is not used"] #[stable(feature = "rust1", since = "1.0.0")] pub fn into_bytes(self) -> Vec { self.bytes @@ -1675,10 +1932,8 @@ impl FromUtf8Error { /// an analogue to `FromUtf8Error`. See its documentation for more details /// on using it. /// - /// [`Utf8Error`]: ../../std/str/struct.Utf8Error.html - /// [`std::str`]: ../../std/str/index.html - /// [`u8`]: ../../std/primitive.u8.html - /// [`&str`]: ../../std/primitive.str.html + /// [`std::str`]: core::str "std::str" + /// [`&str`]: prim@str "&str" /// /// # Examples /// @@ -1693,6 +1948,7 @@ impl FromUtf8Error { /// // the first byte is invalid here /// assert_eq!(1, error.valid_up_to()); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] pub fn utf8_error(&self) -> Utf8Error { self.error @@ -1713,6 +1969,23 @@ impl fmt::Display for FromUtf16Error { } } +#[stable(feature = "rust1", since = "1.0.0")] +impl Error for FromUtf8Error { + #[allow(deprecated)] + fn description(&self) -> &str { + "invalid utf-8" + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Error for FromUtf16Error { + #[allow(deprecated)] + fn description(&self) -> &str { + "invalid utf-16" + } +} + +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl Clone for String { fn clone(&self) -> Self { @@ -1724,6 +1997,7 @@ impl Clone for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl FromIterator for String { fn from_iter>(iter: I) -> String { @@ -1733,6 +2007,7 @@ impl FromIterator for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "string_from_iter_by_ref", since = "1.17.0")] impl<'a> FromIterator<&'a char> for String { fn from_iter>(iter: I) -> String { @@ -1742,6 +2017,7 @@ impl<'a> FromIterator<&'a char> for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl<'a> FromIterator<&'a str> for String { fn from_iter>(iter: I) -> String { @@ -1751,6 +2027,7 @@ impl<'a> FromIterator<&'a str> for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "extend_string", since = "1.4.0")] impl FromIterator for String { fn from_iter>(iter: I) -> String { @@ -1769,6 +2046,17 @@ impl FromIterator for String { } } +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "box_str2", since = "1.45.0")] +impl FromIterator> for String { + fn from_iter>>(iter: I) -> String { + let mut buf = String::new(); + buf.extend(iter); + buf + } +} + +#[cfg(not(no_global_oom_handling))] #[stable(feature = "herd_cows", since = "1.19.0")] impl<'a> FromIterator> for String { fn from_iter>>(iter: I) -> String { @@ -1788,6 +2076,7 @@ impl<'a> FromIterator> for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl Extend for String { fn extend>(&mut self, iter: I) { @@ -1796,37 +2085,90 @@ impl Extend for String { self.reserve(lower_bound); iterator.for_each(move |c| self.push(c)); } + + #[inline] + fn extend_one(&mut self, c: char) { + self.push(c); + } + + #[inline] + fn extend_reserve(&mut self, additional: usize) { + self.reserve(additional); + } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "extend_ref", since = "1.2.0")] impl<'a> Extend<&'a char> for String { fn extend>(&mut self, iter: I) { self.extend(iter.into_iter().cloned()); } + + #[inline] + fn extend_one(&mut self, &c: &'a char) { + self.push(c); + } + + #[inline] + fn extend_reserve(&mut self, additional: usize) { + self.reserve(additional); + } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl<'a> Extend<&'a str> for String { fn extend>(&mut self, iter: I) { iter.into_iter().for_each(move |s| self.push_str(s)); } + + #[inline] + fn extend_one(&mut self, s: &'a str) { + self.push_str(s); + } } +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "box_str2", since = "1.45.0")] +impl Extend> for String { + fn extend>>(&mut self, iter: I) { + iter.into_iter().for_each(move |s| self.push_str(&s)); + } +} + +#[cfg(not(no_global_oom_handling))] #[stable(feature = "extend_string", since = "1.4.0")] impl Extend for String { fn extend>(&mut self, iter: I) { iter.into_iter().for_each(move |s| self.push_str(&s)); } + + #[inline] + fn extend_one(&mut self, s: String) { + self.push_str(&s); + } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "herd_cows", since = "1.19.0")] impl<'a> Extend> for String { fn extend>>(&mut self, iter: I) { iter.into_iter().for_each(move |s| self.push_str(&s)); } + + #[inline] + fn extend_one(&mut self, s: Cow<'a, str>) { + self.push_str(&s); + } } -/// A convenience impl that delegates to the impl for `&str` +/// A convenience impl that delegates to the impl for `&str`. +/// +/// # Examples +/// +/// ``` +/// assert_eq!(String::from("Hello world").find("world"), Some(6)); +/// ``` #[unstable( feature = "pattern", reason = "API not fully fleshed out and ready to be stabilized", @@ -1848,6 +2190,21 @@ impl<'a, 'b> Pattern<'a> for &'b String { fn is_prefix_of(self, haystack: &'a str) -> bool { self[..].is_prefix_of(haystack) } + + #[inline] + fn strip_prefix_of(self, haystack: &'a str) -> Option<&'a str> { + self[..].strip_prefix_of(haystack) + } + + #[inline] + fn is_suffix_of(self, haystack: &'a str) -> bool { + self[..].is_suffix_of(haystack) + } + + #[inline] + fn strip_suffix_of(self, haystack: &'a str) -> Option<&'a str> { + self[..].strip_suffix_of(haystack) + } } #[stable(feature = "rust1", since = "1.0.0")] @@ -1894,12 +2251,16 @@ macro_rules! impl_eq { impl_eq! { String, str } impl_eq! { String, &'a str } +#[cfg(not(no_global_oom_handling))] impl_eq! { Cow<'a, str>, str } +#[cfg(not(no_global_oom_handling))] impl_eq! { Cow<'a, str>, &'b str } +#[cfg(not(no_global_oom_handling))] impl_eq! { Cow<'a, str>, String } #[stable(feature = "rust1", since = "1.0.0")] -impl Default for String { +#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] +impl const Default for String { /// Creates an empty `String`. #[inline] fn default() -> String { @@ -1935,7 +2296,7 @@ impl hash::Hash for String { /// /// This consumes the `String` on the left-hand side and re-uses its buffer (growing it if /// necessary). This is done to avoid allocating a new `String` and copying the entire contents on -/// every operation, which would lead to `O(n^2)` running time when building an `n`-byte string by +/// every operation, which would lead to *O*(*n*^2) running time when building an *n*-byte string by /// repeated concatenation. /// /// The string on the right-hand side is only borrowed; its contents are copied into the returned @@ -1968,6 +2329,7 @@ impl hash::Hash for String { /// let b = " world"; /// let c = a.to_string() + b; /// ``` +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl Add<&str> for String { type Output = String; @@ -1982,6 +2344,7 @@ impl Add<&str> for String { /// Implements the `+=` operator for appending to a `String`. /// /// This has the same behavior as the [`push_str`][String::push_str] method. +#[cfg(not(no_global_oom_handling))] #[stable(feature = "stringaddassign", since = "1.12.0")] impl AddAssign<&str> for String { #[inline] @@ -2110,10 +2473,11 @@ impl ops::DerefMut for String { /// /// This alias exists for backwards compatibility, and may be eventually deprecated. /// -/// [`Infallible`]: ../../core/convert/enum.Infallible.html +/// [`Infallible`]: core::convert::Infallible "convert::Infallible" #[stable(feature = "str_parse_error", since = "1.5.0")] pub type ParseError = core::convert::Infallible; +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl FromStr for String { type Err = core::convert::Infallible; @@ -2130,7 +2494,8 @@ impl FromStr for String { /// [`Display`] should be implemented instead, and you get the `ToString` /// implementation for free. /// -/// [`Display`]: ../../std/fmt/trait.Display.html +/// [`Display`]: fmt::Display +#[cfg_attr(not(test), rustc_diagnostic_item = "ToString")] #[stable(feature = "rust1", since = "1.0.0")] pub trait ToString { /// Converts the given value to a `String`. @@ -2156,19 +2521,86 @@ pub trait ToString { /// if the `Display` implementation returns an error. /// This indicates an incorrect `Display` implementation /// since `fmt::Write for String` never returns an error itself. +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl ToString for T { + // A common guideline is to not inline generic functions. However, + // removing `#[inline]` from this method causes non-negligible regressions. + // See , the last attempt + // to try to remove it. #[inline] default fn to_string(&self) -> String { - use fmt::Write; let mut buf = String::new(); - buf.write_fmt(format_args!("{}", self)) + let mut formatter = core::fmt::Formatter::new(&mut buf); + // Bypass format_args!() to avoid write_str with zero-length strs + fmt::Display::fmt(self, &mut formatter) .expect("a Display implementation returned an error unexpectedly"); - buf.shrink_to_fit(); buf } } +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "char_to_string_specialization", since = "1.46.0")] +impl ToString for char { + #[inline] + fn to_string(&self) -> String { + String::from(self.encode_utf8(&mut [0; 4])) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "bool_to_string_specialization", since = "CURRENT_RUSTC_VERSION")] +impl ToString for bool { + #[inline] + fn to_string(&self) -> String { + String::from(if *self { "true" } else { "false" }) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "u8_to_string_specialization", since = "1.54.0")] +impl ToString for u8 { + #[inline] + fn to_string(&self) -> String { + let mut buf = String::with_capacity(3); + let mut n = *self; + if n >= 10 { + if n >= 100 { + buf.push((b'0' + n / 100) as char); + n %= 100; + } + buf.push((b'0' + n / 10) as char); + n %= 10; + } + buf.push((b'0' + n) as char); + buf + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "i8_to_string_specialization", since = "1.54.0")] +impl ToString for i8 { + #[inline] + fn to_string(&self) -> String { + let mut buf = String::with_capacity(4); + if self.is_negative() { + buf.push('-'); + } + let mut n = self.unsigned_abs(); + if n >= 10 { + if n >= 100 { + buf.push('1'); + n -= 100; + } + buf.push((b'0' + n / 10) as char); + n %= 10; + } + buf.push((b'0' + n) as char); + buf + } +} + +#[cfg(not(no_global_oom_handling))] #[stable(feature = "str_to_string_specialization", since = "1.9.0")] impl ToString for str { #[inline] @@ -2177,6 +2609,7 @@ impl ToString for str { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "cow_str_to_string_specialization", since = "1.17.0")] impl ToString for Cow<'_, str> { #[inline] @@ -2185,6 +2618,7 @@ impl ToString for Cow<'_, str> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "string_to_string_specialization", since = "1.17.0")] impl ToString for String { #[inline] @@ -2217,17 +2651,22 @@ impl AsRef<[u8]> for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl From<&str> for String { + /// Converts a `&str` into a [`String`]. + /// + /// The result is allocated on the heap. #[inline] fn from(s: &str) -> String { s.to_owned() } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "from_mut_str_for_string", since = "1.44.0")] impl From<&mut str> for String { - /// Converts a `&mut str` into a `String`. + /// Converts a `&mut str` into a [`String`]. /// /// The result is allocated on the heap. #[inline] @@ -2236,19 +2675,23 @@ impl From<&mut str> for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "from_ref_string", since = "1.35.0")] impl From<&String> for String { + /// Converts a `&String` into a [`String`]. + /// + /// This clones `s` and returns the clone. #[inline] fn from(s: &String) -> String { s.clone() } } -// note: test pulls in libstd, which causes errors here +// note: test pulls in std, which causes errors here #[cfg(not(test))] #[stable(feature = "string_from_box", since = "1.18.0")] impl From> for String { - /// Converts the given boxed `str` slice to a `String`. + /// Converts the given boxed `str` slice to a [`String`]. /// It is notable that the `str` slice is owned. /// /// # Examples @@ -2267,9 +2710,10 @@ impl From> for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "box_from_str", since = "1.20.0")] impl From for Box { - /// Converts the given `String` to a boxed `str` slice that is owned. + /// Converts the given [`String`] to a boxed `str` slice that is owned. /// /// # Examples /// @@ -2287,37 +2731,97 @@ impl From for Box { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "string_from_cow_str", since = "1.14.0")] impl<'a> From> for String { + /// Converts a clone-on-write string to an owned + /// instance of [`String`]. + /// + /// This extracts the owned string, + /// clones the string if it is not already owned. + /// + /// # Example + /// + /// ``` + /// # use std::borrow::Cow; + /// // If the string is not owned... + /// let cow: Cow = Cow::Borrowed("eggplant"); + /// // It will allocate on the heap and copy the string. + /// let owned: String = String::from(cow); + /// assert_eq!(&owned[..], "eggplant"); + /// ``` fn from(s: Cow<'a, str>) -> String { s.into_owned() } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl<'a> From<&'a str> for Cow<'a, str> { + /// Converts a string slice into a [`Borrowed`] variant. + /// No heap allocation is performed, and the string + /// is not copied. + /// + /// # Example + /// + /// ``` + /// # use std::borrow::Cow; + /// assert_eq!(Cow::from("eggplant"), Cow::Borrowed("eggplant")); + /// ``` + /// + /// [`Borrowed`]: crate::borrow::Cow::Borrowed "borrow::Cow::Borrowed" #[inline] fn from(s: &'a str) -> Cow<'a, str> { Cow::Borrowed(s) } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl<'a> From for Cow<'a, str> { + /// Converts a [`String`] into an [`Owned`] variant. + /// No heap allocation is performed, and the string + /// is not copied. + /// + /// # Example + /// + /// ``` + /// # use std::borrow::Cow; + /// let s = "eggplant".to_string(); + /// let s2 = "eggplant".to_string(); + /// assert_eq!(Cow::from(s), Cow::<'static, str>::Owned(s2)); + /// ``` + /// + /// [`Owned`]: crate::borrow::Cow::Owned "borrow::Cow::Owned" #[inline] fn from(s: String) -> Cow<'a, str> { Cow::Owned(s) } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "cow_from_string_ref", since = "1.28.0")] impl<'a> From<&'a String> for Cow<'a, str> { + /// Converts a [`String`] reference into a [`Borrowed`] variant. + /// No heap allocation is performed, and the string + /// is not copied. + /// + /// # Example + /// + /// ``` + /// # use std::borrow::Cow; + /// let s = "eggplant".to_string(); + /// assert_eq!(Cow::from(&s), Cow::Borrowed("eggplant")); + /// ``` + /// + /// [`Borrowed`]: crate::borrow::Cow::Borrowed "borrow::Cow::Borrowed" #[inline] fn from(s: &'a String) -> Cow<'a, str> { Cow::Borrowed(s.as_str()) } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "cow_str_from_iter", since = "1.12.0")] impl<'a> FromIterator for Cow<'a, str> { fn from_iter>(it: I) -> Cow<'a, str> { @@ -2325,6 +2829,7 @@ impl<'a> FromIterator for Cow<'a, str> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "cow_str_from_iter", since = "1.12.0")] impl<'a, 'b> FromIterator<&'b str> for Cow<'a, str> { fn from_iter>(it: I) -> Cow<'a, str> { @@ -2332,6 +2837,7 @@ impl<'a, 'b> FromIterator<&'b str> for Cow<'a, str> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "cow_str_from_iter", since = "1.12.0")] impl<'a> FromIterator for Cow<'a, str> { fn from_iter>(it: I) -> Cow<'a, str> { @@ -2341,7 +2847,7 @@ impl<'a> FromIterator for Cow<'a, str> { #[stable(feature = "from_string_for_vec_u8", since = "1.14.0")] impl From for Vec { - /// Converts the given `String` to a vector `Vec` that holds values of type `u8`. + /// Converts the given [`String`] to a vector [`Vec`] that holds values of type [`u8`]. /// /// # Examples /// @@ -2352,7 +2858,7 @@ impl From for Vec { /// let v1 = Vec::from(s1); /// /// for b in v1 { - /// println!("{}", b); + /// println!("{b}"); /// } /// ``` fn from(string: String) -> Vec { @@ -2360,6 +2866,7 @@ impl From for Vec { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl fmt::Write for String { #[inline] @@ -2380,8 +2887,7 @@ impl fmt::Write for String { /// This struct is created by the [`drain`] method on [`String`]. See its /// documentation for more. /// -/// [`drain`]: struct.String.html#method.drain -/// [`String`]: struct.String.html +/// [`drain`]: String::drain #[stable(feature = "drain", since = "1.6.0")] pub struct Drain<'a> { /// Will be used as &'a mut String in the destructor @@ -2397,7 +2903,7 @@ pub struct Drain<'a> { #[stable(feature = "collection_debug", since = "1.17.0")] impl fmt::Debug for Drain<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("Drain { .. }") + f.debug_tuple("Drain").field(&self.as_str()).finish() } } @@ -2420,6 +2926,39 @@ impl Drop for Drain<'_> { } } +impl<'a> Drain<'a> { + /// Returns the remaining (sub)string of this iterator as a slice. + /// + /// # Examples + /// + /// ``` + /// let mut s = String::from("abc"); + /// let mut drain = s.drain(..); + /// assert_eq!(drain.as_str(), "abc"); + /// let _ = drain.next().unwrap(); + /// assert_eq!(drain.as_str(), "bc"); + /// ``` + #[must_use] + #[stable(feature = "string_drain_as_str", since = "1.55.0")] + pub fn as_str(&self) -> &str { + self.iter.as_str() + } +} + +#[stable(feature = "string_drain_as_str", since = "1.55.0")] +impl<'a> AsRef for Drain<'a> { + fn as_ref(&self) -> &str { + self.as_str() + } +} + +#[stable(feature = "string_drain_as_str", since = "1.55.0")] +impl<'a> AsRef<[u8]> for Drain<'a> { + fn as_ref(&self) -> &[u8] { + self.as_str().as_bytes() + } +} + #[stable(feature = "drain", since = "1.6.0")] impl Iterator for Drain<'_> { type Item = char; @@ -2449,3 +2988,20 @@ impl DoubleEndedIterator for Drain<'_> { #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for Drain<'_> {} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "from_char_for_string", since = "1.46.0")] +impl From for String { + /// Allocates an owned [`String`] from a single character. + /// + /// # Example + /// ```rust + /// let c: char = 'a'; + /// let s: String = String::from(c); + /// assert_eq!("a", &s[..]); + /// ``` + #[inline] + fn from(c: char) -> Self { + c.to_string() + } +} diff --git a/crux-mir/lib/alloc/src/sync.rs b/crux-mir/lib/alloc/src/sync.rs index db8235d8e..bab7f5f53 100644 --- a/crux-mir/lib/alloc/src/sync.rs +++ b/crux-mir/lib/alloc/src/sync.rs @@ -2,33 +2,46 @@ //! Thread-safe reference-counting pointers. //! -//! See the [`Arc`][arc] documentation for more details. +//! See the [`Arc`][Arc] documentation for more details. //! -//! [arc]: struct.Arc.html +//! **Note**: This module is only available on platforms that support atomic +//! loads and stores of pointers. This may be detected at compile time using +//! `#[cfg(target_has_atomic = "ptr")]`. use core::any::Any; -use core::array::LengthAtMost32; use core::borrow; use core::cmp::Ordering; use core::convert::{From, TryFrom}; use core::fmt; use core::hash::{Hash, Hasher}; +use core::hint; use core::intrinsics::abort; +#[cfg(not(no_global_oom_handling))] use core::iter; use core::marker::{PhantomData, Unpin, Unsize}; -use core::mem::{self, align_of, align_of_val, size_of_val}; +#[cfg(not(no_global_oom_handling))] +use core::mem::size_of_val; +use core::mem::{self, align_of_val_raw}; use core::ops::{CoerceUnsized, Deref, DispatchFromDyn, Receiver}; +use core::panic::{RefUnwindSafe, UnwindSafe}; use core::pin::Pin; use core::ptr::{self, NonNull}; -use core::slice::{self, from_raw_parts_mut}; +#[cfg(not(no_global_oom_handling))] +use core::slice::from_raw_parts_mut; use core::sync::atomic; -use core::sync::atomic::Ordering::{Acquire, Relaxed, Release, SeqCst}; -use core::{isize, usize}; - -use crate::alloc::{box_free, handle_alloc_error, AllocRef, Global, Layout}; +use core::sync::atomic::Ordering::{Acquire, Relaxed, Release}; + +#[cfg(not(no_global_oom_handling))] +use crate::alloc::handle_alloc_error; +#[cfg(not(no_global_oom_handling))] +use crate::alloc::{box_free, WriteCloneIntoRaw}; +use crate::alloc::{AllocError, Allocator, Global, Layout}; +use crate::borrow::{Cow, ToOwned}; use crate::boxed::Box; use crate::rc::is_dangling; +#[cfg(not(no_global_oom_handling))] use crate::string::String; +#[cfg(not(no_global_oom_handling))] use crate::vec::Vec; #[cfg(test)] @@ -73,6 +86,11 @@ macro_rules! acquire { /// [`Mutex`][mutex], [`RwLock`][rwlock], or one of the [`Atomic`][atomic] /// types. /// +/// **Note**: This type is only available on platforms that support atomic +/// loads and stores of pointers, which includes all platforms that support +/// the `std` crate but not all those which only support [`alloc`](crate). +/// This may be detected at compile time using `#[cfg(target_has_atomic = "ptr")]`. +/// /// ## Thread Safety /// /// Unlike [`Rc`], `Arc` uses atomic operations for its reference @@ -90,8 +108,8 @@ macro_rules! acquire { /// first: after all, isn't the point of `Arc` thread safety? The key is /// this: `Arc` makes it thread safe to have multiple ownership of the same /// data, but it doesn't add thread safety to its data. Consider -/// `Arc<`[`RefCell`]`>`. [`RefCell`] isn't [`Sync`], and if `Arc` was always -/// [`Send`], `Arc<`[`RefCell`]`>` would be as well. But then we'd have a problem: +/// Arc<[RefCell\]>. [`RefCell`] isn't [`Sync`], and if `Arc` was always +/// [`Send`], Arc<[RefCell\]> would be as well. But then we'd have a problem: /// [`RefCell`] is not thread safe; it keeps track of the borrowing count using /// non-atomic operations. /// @@ -101,21 +119,21 @@ macro_rules! acquire { /// ## Breaking cycles with `Weak` /// /// The [`downgrade`][downgrade] method can be used to create a non-owning -/// [`Weak`][weak] pointer. A [`Weak`][weak] pointer can be [`upgrade`][upgrade]d +/// [`Weak`] pointer. A [`Weak`] pointer can be [`upgrade`][upgrade]d /// to an `Arc`, but this will return [`None`] if the value stored in the allocation has /// already been dropped. In other words, `Weak` pointers do not keep the value /// inside the allocation alive; however, they *do* keep the allocation /// (the backing store for the value) alive. /// /// A cycle between `Arc` pointers will never be deallocated. For this reason, -/// [`Weak`][weak] is used to break cycles. For example, a tree could have -/// strong `Arc` pointers from parent nodes to children, and [`Weak`][weak] +/// [`Weak`] is used to break cycles. For example, a tree could have +/// strong `Arc` pointers from parent nodes to children, and [`Weak`] /// pointers from children back to their parents. /// /// # Cloning references /// -/// Creating a new reference from an existing reference counted pointer is done using the -/// `Clone` trait implemented for [`Arc`][arc] and [`Weak`][weak]. +/// Creating a new reference from an existing reference-counted pointer is done using the +/// `Clone` trait implemented for [`Arc`][Arc] and [`Weak`][Weak]. /// /// ``` /// use std::sync::Arc; @@ -131,34 +149,47 @@ macro_rules! acquire { /// `Arc` automatically dereferences to `T` (via the [`Deref`][deref] trait), /// so you can call `T`'s methods on a value of type `Arc`. To avoid name /// clashes with `T`'s methods, the methods of `Arc` itself are associated -/// functions, called using function-like syntax: +/// functions, called using [fully qualified syntax]: /// /// ``` /// use std::sync::Arc; +/// /// let my_arc = Arc::new(()); +/// let my_weak = Arc::downgrade(&my_arc); +/// ``` +/// +/// `Arc`'s implementations of traits like `Clone` may also be called using +/// fully qualified syntax. Some people prefer to use fully qualified syntax, +/// while others prefer using method-call syntax. /// -/// Arc::downgrade(&my_arc); /// ``` +/// use std::sync::Arc; /// -/// [`Weak`][weak] does not auto-dereference to `T`, because the inner value may have +/// let arc = Arc::new(()); +/// // Method-call syntax +/// let arc2 = arc.clone(); +/// // Fully qualified syntax +/// let arc3 = Arc::clone(&arc); +/// ``` +/// +/// [`Weak`][Weak] does not auto-dereference to `T`, because the inner value may have /// already been dropped. /// -/// [arc]: struct.Arc.html -/// [weak]: struct.Weak.html -/// [`Rc`]: ../../std/rc/struct.Rc.html -/// [clone]: ../../std/clone/trait.Clone.html#tymethod.clone +/// [`Rc`]: crate::rc::Rc +/// [clone]: Clone::clone /// [mutex]: ../../std/sync/struct.Mutex.html /// [rwlock]: ../../std/sync/struct.RwLock.html -/// [atomic]: ../../std/sync/atomic/index.html -/// [`Send`]: ../../std/marker/trait.Send.html -/// [`Sync`]: ../../std/marker/trait.Sync.html -/// [deref]: ../../std/ops/trait.Deref.html -/// [downgrade]: struct.Arc.html#method.downgrade -/// [upgrade]: struct.Weak.html#method.upgrade -/// [`None`]: ../../std/option/enum.Option.html#variant.None -/// [`RefCell`]: ../../std/cell/struct.RefCell.html +/// [atomic]: core::sync::atomic +/// [`Send`]: core::marker::Send +/// [`Sync`]: core::marker::Sync +/// [deref]: core::ops::Deref +/// [downgrade]: Arc::downgrade +/// [upgrade]: Weak::upgrade +/// [RefCell\]: core::cell::RefCell +/// [`RefCell`]: core::cell::RefCell /// [`std::sync`]: ../../std/sync/index.html -/// [`Arc::clone(&from)`]: #method.clone +/// [`Arc::clone(&from)`]: Arc::clone +/// [fully qualified syntax]: https://doc.rust-lang.org/book/ch19-03-advanced-traits.html#fully-qualified-syntax-for-disambiguation-calling-methods-with-the-same-name /// /// # Examples /// @@ -178,14 +209,14 @@ macro_rules! acquire { /// let five = Arc::clone(&five); /// /// thread::spawn(move || { -/// println!("{:?}", five); +/// println!("{five:?}"); /// }); /// } /// ``` /// /// Sharing a mutable [`AtomicUsize`]: /// -/// [`AtomicUsize`]: ../../std/sync/atomic/struct.AtomicUsize.html +/// [`AtomicUsize`]: core::sync::atomic::AtomicUsize "sync::atomic::AtomicUsize" /// /// ```no_run /// use std::sync::Arc; @@ -199,7 +230,7 @@ macro_rules! acquire { /// /// thread::spawn(move || { /// let v = val.fetch_add(1, Ordering::SeqCst); -/// println!("{:?}", v); +/// println!("{v:?}"); /// }); /// } /// ``` @@ -207,8 +238,8 @@ macro_rules! acquire { /// See the [`rc` documentation][rc_examples] for more examples of reference /// counting in general. /// -/// [rc_examples]: ../../std/rc/index.html#examples -#[cfg_attr(not(test), lang = "arc")] +/// [rc_examples]: crate::rc#examples +#[cfg_attr(not(test), rustc_diagnostic_item = "Arc")] #[stable(feature = "rust1", since = "1.0.0")] pub struct Arc { ptr: NonNull>, @@ -220,25 +251,28 @@ unsafe impl Send for Arc {} #[stable(feature = "rust1", since = "1.0.0")] unsafe impl Sync for Arc {} -#[unstable(feature = "coerce_unsized", issue = "27732")] +#[stable(feature = "catch_unwind", since = "1.9.0")] +impl UnwindSafe for Arc {} + +#[unstable(feature = "coerce_unsized", issue = "18598")] impl, U: ?Sized> CoerceUnsized> for Arc {} #[unstable(feature = "dispatch_from_dyn", issue = "none")] impl, U: ?Sized> DispatchFromDyn> for Arc {} impl Arc { - fn from_inner(ptr: NonNull>) -> Self { + unsafe fn from_inner(ptr: NonNull>) -> Self { Self { ptr, phantom: PhantomData } } unsafe fn from_ptr(ptr: *mut ArcInner) -> Self { - Self::from_inner(NonNull::new_unchecked(ptr)) + unsafe { Self::from_inner(NonNull::new_unchecked(ptr)) } } } /// `Weak` is a version of [`Arc`] that holds a non-owning reference to the /// managed allocation. The allocation is accessed by calling [`upgrade`] on the `Weak` -/// pointer, which returns an [`Option`]`<`[`Arc`]`>`. +/// pointer, which returns an [Option]<[Arc]\>. /// /// Since a `Weak` reference does not count towards ownership, it will not /// prevent the value stored in the allocation from being dropped, and `Weak` itself makes no @@ -255,18 +289,15 @@ impl Arc { /// /// The typical way to obtain a `Weak` pointer is to call [`Arc::downgrade`]. /// -/// [`Arc`]: struct.Arc.html -/// [`Arc::downgrade`]: struct.Arc.html#method.downgrade -/// [`upgrade`]: struct.Weak.html#method.upgrade -/// [`Option`]: ../../std/option/enum.Option.html -/// [`None`]: ../../std/option/enum.Option.html#variant.None +/// [`upgrade`]: Weak::upgrade #[stable(feature = "arc_weak", since = "1.4.0")] pub struct Weak { // This is a `NonNull` to allow optimizing the size of this type in enums, // but it is not necessarily a valid pointer. // `Weak::new` sets this to `usize::MAX` so that it doesn’t need - // to allocate space on the heap. That's not a value a real pointer + // to allocate space on the heap. That's not a value a real pointer // will ever have because RcBox has alignment at least 2. + // This is only possible when `T: Sized`; unsized `T` never dangle. ptr: NonNull>, } @@ -275,18 +306,22 @@ unsafe impl Send for Weak {} #[stable(feature = "arc_weak", since = "1.4.0")] unsafe impl Sync for Weak {} -#[unstable(feature = "coerce_unsized", issue = "27732")] +#[unstable(feature = "coerce_unsized", issue = "18598")] impl, U: ?Sized> CoerceUnsized> for Weak {} #[unstable(feature = "dispatch_from_dyn", issue = "none")] impl, U: ?Sized> DispatchFromDyn> for Weak {} #[stable(feature = "arc_weak", since = "1.4.0")] -impl fmt::Debug for Weak { +impl fmt::Debug for Weak { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "(Weak)") } } +// This is repr(C) to future-proof against possible field-reordering, which +// would interfere with otherwise safe [into|from]_raw() of transmutable +// inner types. +#[repr(C)] struct ArcInner { strong: atomic::AtomicUsize, @@ -298,6 +333,15 @@ struct ArcInner { data: T, } +/// Calculate layout for `ArcInner` using the inner value's layout +fn arcinner_layout_for_value_layout(layout: Layout) -> Layout { + // Calculate layout using the given value layout. + // Previously, layout was calculated on the expression + // `&*(ptr as *const ArcInner)`, but this created a misaligned + // reference (see #54908). + Layout::new::>().extend(layout).unwrap().0.pad_to_align() +} + unsafe impl Send for ArcInner {} unsafe impl Sync for ArcInner {} @@ -311,32 +355,126 @@ impl Arc { /// /// let five = Arc::new(5); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn new(data: T) -> Arc { // Start the weak pointer count as 1 which is the weak pointer that's // held by all the strong pointers (kinda), see std/rc.rs for more info + let x: Box<_> = Box::new(ArcInner { + strong: atomic::AtomicUsize::new(1), + weak: atomic::AtomicUsize::new(1), + data, + }); + unsafe { Self::from_inner(Box::leak(x).into()) } + } - let x: Box<_> = { - #[cfg(feature = "opaque-arc-atomics")] - { - box ArcInner { - strong: atomic::AtomicUsize::new_unmodeled(1), - weak: atomic::AtomicUsize::new_unmodeled(1), - data, - } - } - - #[cfg(not(feature = "opaque-arc-atomics"))] - { - box ArcInner { - strong: atomic::AtomicUsize::new(1), - weak: atomic::AtomicUsize::new(1), - data, - } - } + /// Constructs a new `Arc` while giving you a `Weak` to the allocation, + /// to allow you to construct a `T` which holds a weak pointer to itself. + /// + /// Generally, a structure circularly referencing itself, either directly or + /// indirectly, should not hold a strong reference to itself to prevent a memory leak. + /// Using this function, you get access to the weak pointer during the + /// initialization of `T`, before the `Arc` is created, such that you can + /// clone and store it inside the `T`. + /// + /// `new_cyclic` first allocates the managed allocation for the `Arc`, + /// then calls your closure, giving it a `Weak` to this allocation, + /// and only afterwards completes the construction of the `Arc` by placing + /// the `T` returned from your closure into the allocation. + /// + /// Since the new `Arc` is not fully-constructed until `Arc::new_cyclic` + /// returns, calling [`upgrade`] on the weak reference inside your closure will + /// fail and result in a `None` value. + /// + /// # Panics + /// + /// If `data_fn` panics, the panic is propagated to the caller, and the + /// temporary [`Weak`] is dropped normally. + /// + /// # Example + /// + /// ``` + /// # #![allow(dead_code)] + /// use std::sync::{Arc, Weak}; + /// + /// struct Gadget { + /// me: Weak, + /// } + /// + /// impl Gadget { + /// /// Construct a reference counted Gadget. + /// fn new() -> Arc { + /// // `me` is a `Weak` pointing at the new allocation of the + /// // `Arc` we're constructing. + /// Arc::new_cyclic(|me| { + /// // Create the actual struct here. + /// Gadget { me: me.clone() } + /// }) + /// } + /// + /// /// Return a reference counted pointer to Self. + /// fn me(&self) -> Arc { + /// self.me.upgrade().unwrap() + /// } + /// } + /// ``` + /// [`upgrade`]: Weak::upgrade + #[cfg(not(no_global_oom_handling))] + #[inline] + #[stable(feature = "arc_new_cyclic", since = "1.60.0")] + pub fn new_cyclic(data_fn: F) -> Arc + where + F: FnOnce(&Weak) -> T, + { + // Construct the inner in the "uninitialized" state with a single + // weak reference. + let uninit_ptr: NonNull<_> = Box::leak(Box::new(ArcInner { + strong: atomic::AtomicUsize::new(0), + weak: atomic::AtomicUsize::new(1), + data: mem::MaybeUninit::::uninit(), + })) + .into(); + let init_ptr: NonNull> = uninit_ptr.cast(); + + let weak = Weak { ptr: init_ptr }; + + // It's important we don't give up ownership of the weak pointer, or + // else the memory might be freed by the time `data_fn` returns. If + // we really wanted to pass ownership, we could create an additional + // weak pointer for ourselves, but this would result in additional + // updates to the weak reference count which might not be necessary + // otherwise. + let data = data_fn(&weak); + + // Now we can properly initialize the inner value and turn our weak + // reference into a strong reference. + let strong = unsafe { + let inner = init_ptr.as_ptr(); + ptr::write(ptr::addr_of_mut!((*inner).data), data); + + // The above write to the data field must be visible to any threads which + // observe a non-zero strong count. Therefore we need at least "Release" ordering + // in order to synchronize with the `compare_exchange_weak` in `Weak::upgrade`. + // + // "Acquire" ordering is not required. When considering the possible behaviours + // of `data_fn` we only need to look at what it could do with a reference to a + // non-upgradeable `Weak`: + // - It can *clone* the `Weak`, increasing the weak reference count. + // - It can drop those clones, decreasing the weak reference count (but never to zero). + // + // These side effects do not impact us in any way, and no other side effects are + // possible with safe code alone. + let prev_value = (*inner).strong.fetch_add(1, Release); + debug_assert_eq!(prev_value, 0, "No prior strong references should exist"); + + Arc::from_inner(init_ptr) }; - Self::from_inner(Box::into_raw_non_null(x)) + + // Strong references should collectively own a shared weak reference, + // so don't run the destructor for our old weak reference. + mem::forget(weak); + strong } /// Constructs a new `Arc` with uninitialized contents. @@ -351,21 +489,23 @@ impl Arc { /// /// let mut five = Arc::::new_uninit(); /// - /// let five = unsafe { - /// // Deferred initialization: - /// Arc::get_mut_unchecked(&mut five).as_mut_ptr().write(5); + /// // Deferred initialization: + /// Arc::get_mut(&mut five).unwrap().write(5); /// - /// five.assume_init() - /// }; + /// let five = unsafe { five.assume_init() }; /// /// assert_eq!(*five, 5) /// ``` + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] pub fn new_uninit() -> Arc> { unsafe { - Arc::from_ptr(Arc::allocate_for_layout(Layout::new::(), |mem| { - mem as *mut ArcInner> - })) + Arc::from_ptr(Arc::allocate_for_layout( + Layout::new::(), + |layout| Global.allocate(layout), + |mem| mem as *mut ArcInner>, + )) } } @@ -388,32 +528,132 @@ impl Arc { /// assert_eq!(*zero, 0) /// ``` /// - /// [zeroed]: ../../std/mem/union.MaybeUninit.html#method.zeroed + /// [zeroed]: mem::MaybeUninit::zeroed + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] pub fn new_zeroed() -> Arc> { unsafe { - let mut uninit = Self::new_uninit(); - ptr::write_bytes::(Arc::get_mut_unchecked(&mut uninit).as_mut_ptr(), 0, 1); - uninit + Arc::from_ptr(Arc::allocate_for_layout( + Layout::new::(), + |layout| Global.allocate_zeroed(layout), + |mem| mem as *mut ArcInner>, + )) } } /// Constructs a new `Pin>`. If `T` does not implement `Unpin`, then /// `data` will be pinned in memory and unable to be moved. + #[cfg(not(no_global_oom_handling))] #[stable(feature = "pin", since = "1.33.0")] + #[must_use] pub fn pin(data: T) -> Pin> { unsafe { Pin::new_unchecked(Arc::new(data)) } } + /// Constructs a new `Pin>`, return an error if allocation fails. + #[unstable(feature = "allocator_api", issue = "32838")] + #[inline] + pub fn try_pin(data: T) -> Result>, AllocError> { + unsafe { Ok(Pin::new_unchecked(Arc::try_new(data)?)) } + } + + /// Constructs a new `Arc`, returning an error if allocation fails. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api)] + /// use std::sync::Arc; + /// + /// let five = Arc::try_new(5)?; + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + #[unstable(feature = "allocator_api", issue = "32838")] + #[inline] + pub fn try_new(data: T) -> Result, AllocError> { + // Start the weak pointer count as 1 which is the weak pointer that's + // held by all the strong pointers (kinda), see std/rc.rs for more info + let x: Box<_> = Box::try_new(ArcInner { + strong: atomic::AtomicUsize::new(1), + weak: atomic::AtomicUsize::new(1), + data, + })?; + unsafe { Ok(Self::from_inner(Box::leak(x).into())) } + } + + /// Constructs a new `Arc` with uninitialized contents, returning an error + /// if allocation fails. + /// + /// # Examples + /// + /// ``` + /// #![feature(new_uninit, allocator_api)] + /// #![feature(get_mut_unchecked)] + /// + /// use std::sync::Arc; + /// + /// let mut five = Arc::::try_new_uninit()?; + /// + /// // Deferred initialization: + /// Arc::get_mut(&mut five).unwrap().write(5); + /// + /// let five = unsafe { five.assume_init() }; + /// + /// assert_eq!(*five, 5); + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + #[unstable(feature = "allocator_api", issue = "32838")] + // #[unstable(feature = "new_uninit", issue = "63291")] + pub fn try_new_uninit() -> Result>, AllocError> { + unsafe { + Ok(Arc::from_ptr(Arc::try_allocate_for_layout( + Layout::new::(), + |layout| Global.allocate(layout), + |mem| mem as *mut ArcInner>, + )?)) + } + } + + /// Constructs a new `Arc` with uninitialized contents, with the memory + /// being filled with `0` bytes, returning an error if allocation fails. + /// + /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage + /// of this method. + /// + /// # Examples + /// + /// ``` + /// #![feature(new_uninit, allocator_api)] + /// + /// use std::sync::Arc; + /// + /// let zero = Arc::::try_new_zeroed()?; + /// let zero = unsafe { zero.assume_init() }; + /// + /// assert_eq!(*zero, 0); + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + /// + /// [zeroed]: mem::MaybeUninit::zeroed + #[unstable(feature = "allocator_api", issue = "32838")] + // #[unstable(feature = "new_uninit", issue = "63291")] + pub fn try_new_zeroed() -> Result>, AllocError> { + unsafe { + Ok(Arc::from_ptr(Arc::try_allocate_for_layout( + Layout::new::(), + |layout| Global.allocate_zeroed(layout), + |mem| mem as *mut ArcInner>, + )?)) + } + } /// Returns the inner value, if the `Arc` has exactly one strong reference. /// - /// Otherwise, an [`Err`][result] is returned with the same `Arc` that was + /// Otherwise, an [`Err`] is returned with the same `Arc` that was /// passed in. /// /// This will succeed even if there are outstanding weak references. /// - /// [result]: ../../std/result/enum.Result.html - /// /// # Examples /// /// ``` @@ -429,8 +669,7 @@ impl Arc { #[inline] #[stable(feature = "arc_unique", since = "1.4.0")] pub fn try_unwrap(this: Self) -> Result { - // See `drop` for why all these atomics are like this - if this.inner().strong.compare_exchange(1, 0, Release, Relaxed).is_err() { + if this.inner().strong.compare_exchange(1, 0, Relaxed, Relaxed).is_err() { return Err(this); } @@ -449,7 +688,7 @@ impl Arc { } impl Arc<[T]> { - /// Constructs a new reference-counted slice with uninitialized contents. + /// Constructs a new atomically reference-counted slice with uninitialized contents. /// /// # Examples /// @@ -461,21 +700,58 @@ impl Arc<[T]> { /// /// let mut values = Arc::<[u32]>::new_uninit_slice(3); /// - /// let values = unsafe { - /// // Deferred initialization: - /// Arc::get_mut_unchecked(&mut values)[0].as_mut_ptr().write(1); - /// Arc::get_mut_unchecked(&mut values)[1].as_mut_ptr().write(2); - /// Arc::get_mut_unchecked(&mut values)[2].as_mut_ptr().write(3); + /// // Deferred initialization: + /// let data = Arc::get_mut(&mut values).unwrap(); + /// data[0].write(1); + /// data[1].write(2); + /// data[2].write(3); /// - /// values.assume_init() - /// }; + /// let values = unsafe { values.assume_init() }; /// /// assert_eq!(*values, [1, 2, 3]) /// ``` + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] pub fn new_uninit_slice(len: usize) -> Arc<[mem::MaybeUninit]> { unsafe { Arc::from_ptr(Arc::allocate_for_slice(len)) } } + + /// Constructs a new atomically reference-counted slice with uninitialized contents, with the memory being + /// filled with `0` bytes. + /// + /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and + /// incorrect usage of this method. + /// + /// # Examples + /// + /// ``` + /// #![feature(new_uninit)] + /// + /// use std::sync::Arc; + /// + /// let values = Arc::<[u32]>::new_zeroed_slice(3); + /// let values = unsafe { values.assume_init() }; + /// + /// assert_eq!(*values, [0, 0, 0]) + /// ``` + /// + /// [zeroed]: mem::MaybeUninit::zeroed + #[cfg(not(no_global_oom_handling))] + #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] + pub fn new_zeroed_slice(len: usize) -> Arc<[mem::MaybeUninit]> { + unsafe { + Arc::from_ptr(Arc::allocate_for_layout( + Layout::array::(len).unwrap(), + |layout| Global.allocate_zeroed(layout), + |mem| { + ptr::slice_from_raw_parts_mut(mem as *mut T, len) + as *mut ArcInner<[mem::MaybeUninit]> + }, + )) + } + } } impl Arc> { @@ -489,7 +765,7 @@ impl Arc> { /// Calling this when the content is not yet fully initialized /// causes immediate undefined behavior. /// - /// [`MaybeUninit::assume_init`]: ../../std/mem/union.MaybeUninit.html#method.assume_init + /// [`MaybeUninit::assume_init`]: mem::MaybeUninit::assume_init /// /// # Examples /// @@ -501,19 +777,18 @@ impl Arc> { /// /// let mut five = Arc::::new_uninit(); /// - /// let five = unsafe { - /// // Deferred initialization: - /// Arc::get_mut_unchecked(&mut five).as_mut_ptr().write(5); + /// // Deferred initialization: + /// Arc::get_mut(&mut five).unwrap().write(5); /// - /// five.assume_init() - /// }; + /// let five = unsafe { five.assume_init() }; /// /// assert_eq!(*five, 5) /// ``` #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use = "`self` will be dropped if the result is not used"] #[inline] pub unsafe fn assume_init(self) -> Arc { - Arc::from_inner(mem::ManuallyDrop::new(self).ptr.cast()) + unsafe { Arc::from_inner(mem::ManuallyDrop::new(self).ptr.cast()) } } } @@ -528,7 +803,7 @@ impl Arc<[mem::MaybeUninit]> { /// Calling this when the content is not yet fully initialized /// causes immediate undefined behavior. /// - /// [`MaybeUninit::assume_init`]: ../../std/mem/union.MaybeUninit.html#method.assume_init + /// [`MaybeUninit::assume_init`]: mem::MaybeUninit::assume_init /// /// # Examples /// @@ -540,21 +815,21 @@ impl Arc<[mem::MaybeUninit]> { /// /// let mut values = Arc::<[u32]>::new_uninit_slice(3); /// - /// let values = unsafe { - /// // Deferred initialization: - /// Arc::get_mut_unchecked(&mut values)[0].as_mut_ptr().write(1); - /// Arc::get_mut_unchecked(&mut values)[1].as_mut_ptr().write(2); - /// Arc::get_mut_unchecked(&mut values)[2].as_mut_ptr().write(3); + /// // Deferred initialization: + /// let data = Arc::get_mut(&mut values).unwrap(); + /// data[0].write(1); + /// data[1].write(2); + /// data[2].write(3); /// - /// values.assume_init() - /// }; + /// let values = unsafe { values.assume_init() }; /// /// assert_eq!(*values, [1, 2, 3]) /// ``` #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use = "`self` will be dropped if the result is not used"] #[inline] pub unsafe fn assume_init(self) -> Arc<[T]> { - Arc::from_ptr(mem::ManuallyDrop::new(self).ptr.as_ptr() as _) + unsafe { Arc::from_ptr(mem::ManuallyDrop::new(self).ptr.as_ptr() as _) } } } @@ -562,9 +837,7 @@ impl Arc { /// Consumes the `Arc`, returning the wrapped pointer. /// /// To avoid a memory leak the pointer must be converted back to an `Arc` using - /// [`Arc::from_raw`][from_raw]. - /// - /// [from_raw]: struct.Arc.html#method.from_raw + /// [`Arc::from_raw`]. /// /// # Examples /// @@ -575,32 +848,59 @@ impl Arc { /// let x_ptr = Arc::into_raw(x); /// assert_eq!(unsafe { &*x_ptr }, "hello"); /// ``` + #[must_use = "losing the pointer will leak memory"] #[stable(feature = "rc_raw", since = "1.17.0")] pub fn into_raw(this: Self) -> *const T { - let ptr: *mut ArcInner = NonNull::as_ptr(this.ptr); - let fake_ptr = ptr as *mut T; + let ptr = Self::as_ptr(&this); mem::forget(this); + ptr + } - // SAFETY: This cannot go through Deref::deref. - // Instead, we manually offset the pointer rather than manifesting a reference. - // This is so that the returned pointer retains the same provenance as our pointer. - // This is required so that e.g. `get_mut` can write through the pointer - // after the Arc is recovered through `from_raw`. - unsafe { - let offset = data_offset(&(*ptr).data); - set_data_ptr(fake_ptr, (ptr as *mut u8).offset(offset)) - } + /// Provides a raw pointer to the data. + /// + /// The counts are not affected in any way and the `Arc` is not consumed. The pointer is valid for + /// as long as there are strong counts in the `Arc`. + /// + /// # Examples + /// + /// ``` + /// use std::sync::Arc; + /// + /// let x = Arc::new("hello".to_owned()); + /// let y = Arc::clone(&x); + /// let x_ptr = Arc::as_ptr(&x); + /// assert_eq!(x_ptr, Arc::as_ptr(&y)); + /// assert_eq!(unsafe { &*x_ptr }, "hello"); + /// ``` + #[must_use] + #[stable(feature = "rc_as_ptr", since = "1.45.0")] + pub fn as_ptr(this: &Self) -> *const T { + let ptr: *mut ArcInner = NonNull::as_ptr(this.ptr); + + // SAFETY: This cannot go through Deref::deref or RcBoxPtr::inner because + // this is required to retain raw/mut provenance such that e.g. `get_mut` can + // write through the pointer after the Rc is recovered through `from_raw`. + unsafe { ptr::addr_of_mut!((*ptr).data) } } - /// Constructs an `Arc` from a raw pointer. + /// Constructs an `Arc` from a raw pointer. + /// + /// The raw pointer must have been previously returned by a call to + /// [`Arc::into_raw`][into_raw] where `U` must have the same size and + /// alignment as `T`. This is trivially true if `U` is `T`. + /// Note that if `U` is not `T` but has the same size and alignment, this is + /// basically like transmuting references of different types. See + /// [`mem::transmute`][transmute] for more information on what + /// restrictions apply in this case. /// - /// The raw pointer must have been previously returned by a call to a - /// [`Arc::into_raw`][into_raw]. + /// The user of `from_raw` has to make sure a specific value of `T` is only + /// dropped once. /// - /// This function is unsafe because improper use may lead to memory problems. For example, a - /// double-free may occur if the function is called twice on the same raw pointer. + /// This function is unsafe because improper use may lead to memory unsafety, + /// even if the returned `Arc` is never accessed. /// - /// [into_raw]: struct.Arc.html#method.into_raw + /// [into_raw]: Arc::into_raw + /// [transmute]: core::mem::transmute /// /// # Examples /// @@ -622,39 +922,17 @@ impl Arc { /// ``` #[stable(feature = "rc_raw", since = "1.17.0")] pub unsafe fn from_raw(ptr: *const T) -> Self { - let offset = data_offset(ptr); - - // Reverse the offset to find the original ArcInner. - let fake_ptr = ptr as *mut ArcInner; - let arc_ptr = set_data_ptr(fake_ptr, (ptr as *mut u8).offset(-offset)); + unsafe { + let offset = data_offset(ptr); - Self::from_ptr(arc_ptr) - } + // Reverse the offset to find the original ArcInner. + let arc_ptr = ptr.byte_sub(offset) as *mut ArcInner; - /// Consumes the `Arc`, returning the wrapped pointer as `NonNull`. - /// - /// # Examples - /// - /// ``` - /// #![feature(rc_into_raw_non_null)] - /// - /// use std::sync::Arc; - /// - /// let x = Arc::new("hello".to_owned()); - /// let ptr = Arc::into_raw_non_null(x); - /// let deref = unsafe { ptr.as_ref() }; - /// assert_eq!(deref, "hello"); - /// ``` - #[unstable(feature = "rc_into_raw_non_null", issue = "47336")] - #[inline] - pub fn into_raw_non_null(this: Self) -> NonNull { - // safe because Arc guarantees its pointer is non-null - unsafe { NonNull::new_unchecked(Arc::into_raw(this) as *mut _) } + Self::from_ptr(arc_ptr) + } } - /// Creates a new [`Weak`][weak] pointer to this allocation. - /// - /// [weak]: struct.Weak.html + /// Creates a new [`Weak`] pointer to this allocation. /// /// # Examples /// @@ -665,6 +943,8 @@ impl Arc { /// /// let weak_five = Arc::downgrade(&five); /// ``` + #[must_use = "this returns a new `Weak` pointer, \ + without modifying the original `Arc`"] #[stable(feature = "arc_weak", since = "1.4.0")] pub fn downgrade(this: &Self) -> Weak { // This Relaxed is OK because we're checking the value in the CAS @@ -674,6 +954,7 @@ impl Arc { loop { // check if the weak counter is currently "locked"; if so, spin. if cur == usize::MAX { + hint::spin_loop(); cur = this.inner().weak.load(Relaxed); continue; } @@ -688,7 +969,7 @@ impl Arc { match this.inner().weak.compare_exchange_weak(cur, cur + 1, Acquire, Relaxed) { Ok(_) => { // Make sure we do not create a dangling Weak - debug_assert!(!is_dangling(this.ptr)); + debug_assert!(!is_dangling(this.ptr.as_ptr())); return Weak { ptr: this.ptr }; } Err(old) => cur = old, @@ -696,9 +977,7 @@ impl Arc { } } - /// Gets the number of [`Weak`][weak] pointers to this allocation. - /// - /// [weak]: struct.Weak.html + /// Gets the number of [`Weak`] pointers to this allocation. /// /// # Safety /// @@ -719,9 +998,10 @@ impl Arc { /// assert_eq!(1, Arc::weak_count(&five)); /// ``` #[inline] + #[must_use] #[stable(feature = "arc_counts", since = "1.15.0")] pub fn weak_count(this: &Self) -> usize { - let cnt = this.inner().weak.load(SeqCst); + let cnt = this.inner().weak.load(Acquire); // If the weak count is currently locked, the value of the // count was 0 just before taking the lock. if cnt == usize::MAX { 0 } else { cnt - 1 } @@ -748,9 +1028,81 @@ impl Arc { /// assert_eq!(2, Arc::strong_count(&five)); /// ``` #[inline] + #[must_use] #[stable(feature = "arc_counts", since = "1.15.0")] pub fn strong_count(this: &Self) -> usize { - this.inner().strong.load(SeqCst) + this.inner().strong.load(Acquire) + } + + /// Increments the strong reference count on the `Arc` associated with the + /// provided pointer by one. + /// + /// # Safety + /// + /// The pointer must have been obtained through `Arc::into_raw`, and the + /// associated `Arc` instance must be valid (i.e. the strong count must be at + /// least 1) for the duration of this method. + /// + /// # Examples + /// + /// ``` + /// use std::sync::Arc; + /// + /// let five = Arc::new(5); + /// + /// unsafe { + /// let ptr = Arc::into_raw(five); + /// Arc::increment_strong_count(ptr); + /// + /// // This assertion is deterministic because we haven't shared + /// // the `Arc` between threads. + /// let five = Arc::from_raw(ptr); + /// assert_eq!(2, Arc::strong_count(&five)); + /// } + /// ``` + #[inline] + #[stable(feature = "arc_mutate_strong_count", since = "1.51.0")] + pub unsafe fn increment_strong_count(ptr: *const T) { + // Retain Arc, but don't touch refcount by wrapping in ManuallyDrop + let arc = unsafe { mem::ManuallyDrop::new(Arc::::from_raw(ptr)) }; + // Now increase refcount, but don't drop new refcount either + let _arc_clone: mem::ManuallyDrop<_> = arc.clone(); + } + + /// Decrements the strong reference count on the `Arc` associated with the + /// provided pointer by one. + /// + /// # Safety + /// + /// The pointer must have been obtained through `Arc::into_raw`, and the + /// associated `Arc` instance must be valid (i.e. the strong count must be at + /// least 1) when invoking this method. This method can be used to release the final + /// `Arc` and backing storage, but **should not** be called after the final `Arc` has been + /// released. + /// + /// # Examples + /// + /// ``` + /// use std::sync::Arc; + /// + /// let five = Arc::new(5); + /// + /// unsafe { + /// let ptr = Arc::into_raw(five); + /// Arc::increment_strong_count(ptr); + /// + /// // Those assertions are deterministic because we haven't shared + /// // the `Arc` between threads. + /// let five = Arc::from_raw(ptr); + /// assert_eq!(2, Arc::strong_count(&five)); + /// Arc::decrement_strong_count(ptr); + /// assert_eq!(1, Arc::strong_count(&five)); + /// } + /// ``` + #[inline] + #[stable(feature = "arc_mutate_strong_count", since = "1.51.0")] + pub unsafe fn decrement_strong_count(ptr: *const T) { + unsafe { mem::drop(Arc::from_raw(ptr)) }; } #[inline] @@ -766,20 +1118,16 @@ impl Arc { // Non-inlined part of `drop`. #[inline(never)] unsafe fn drop_slow(&mut self) { - // Destroy the data at this time, even though we may not free the box - // allocation itself (there may still be weak pointers lying around). - ptr::drop_in_place(&mut self.ptr.as_mut().data); + // Destroy the data at this time, even though we must not free the box + // allocation itself (there might still be weak pointers lying around). + unsafe { ptr::drop_in_place(Self::get_mut_unchecked(self)) }; - if self.inner().weak.fetch_sub(1, Release) == 1 { - acquire!(self.inner().weak); - // Crux: we should deallocate `self.ptr` here, but we don't support `deallocate` yet. - } + // Drop the weak ref collectively held by all strong references + drop(Weak { ptr: self.ptr }); } - #[inline] - #[stable(feature = "ptr_eq", since = "1.17.0")] - /// Returns `true` if the two `Arc`s point to the same allocation - /// (in a vein similar to [`ptr::eq`]). + /// Returns `true` if the two `Arc`s point to the same allocation in a vein similar to + /// [`ptr::eq`]. See [that function][`ptr::eq`] for caveats when comparing `dyn Trait` pointers. /// /// # Examples /// @@ -794,7 +1142,10 @@ impl Arc { /// assert!(!Arc::ptr_eq(&five, &other_five)); /// ``` /// - /// [`ptr::eq`]: ../../std/ptr/fn.eq.html + /// [`ptr::eq`]: core::ptr::eq "ptr::eq" + #[inline] + #[must_use] + #[stable(feature = "ptr_eq", since = "1.17.0")] pub fn ptr_eq(this: &Self, other: &Self) -> bool { this.ptr.as_ptr() == other.ptr.as_ptr() } @@ -806,47 +1157,63 @@ impl Arc { /// /// The function `mem_to_arcinner` is called with the data pointer /// and must return back a (potentially fat)-pointer for the `ArcInner`. + #[cfg(not(no_global_oom_handling))] unsafe fn allocate_for_layout( value_layout: Layout, + allocate: impl FnOnce(Layout) -> Result, AllocError>, mem_to_arcinner: impl FnOnce(*mut u8) -> *mut ArcInner, ) -> *mut ArcInner { - // Calculate layout using the given value layout. - // Previously, layout was calculated on the expression - // `&*(ptr as *const ArcInner)`, but this created a misaligned - // reference (see #54908). - let layout = Layout::new::>().extend(value_layout).unwrap().0.pad_to_align(); + let layout = arcinner_layout_for_value_layout(value_layout); + unsafe { + Arc::try_allocate_for_layout(value_layout, allocate, mem_to_arcinner) + .unwrap_or_else(|_| handle_alloc_error(layout)) + } + } + + /// Allocates an `ArcInner` with sufficient space for + /// a possibly-unsized inner value where the value has the layout provided, + /// returning an error if allocation fails. + /// + /// The function `mem_to_arcinner` is called with the data pointer + /// and must return back a (potentially fat)-pointer for the `ArcInner`. + unsafe fn try_allocate_for_layout( + value_layout: Layout, + allocate: impl FnOnce(Layout) -> Result, AllocError>, + mem_to_arcinner: impl FnOnce(*mut u8) -> *mut ArcInner, + ) -> Result<*mut ArcInner, AllocError> { + let layout = arcinner_layout_for_value_layout(value_layout); - let (mem, _) = Global.alloc(layout).unwrap_or_else(|_| handle_alloc_error(layout)); + let ptr = allocate(layout)?; // Initialize the ArcInner - let inner = mem_to_arcinner(mem.as_ptr()); - debug_assert_eq!(Layout::for_value(&*inner), layout); + let inner = mem_to_arcinner(ptr.as_non_null_ptr().as_ptr()); + debug_assert_eq!(unsafe { Layout::for_value(&*inner) }, layout); - #[cfg(feature = "opaque-arc-atomics")] - { - ptr::write(&mut (*inner).strong, atomic::AtomicUsize::new_unmodeled(1)); - ptr::write(&mut (*inner).weak, atomic::AtomicUsize::new_unmodeled(1)); - } - #[cfg(not(feature = "opaque-arc-atomics"))] - { + unsafe { ptr::write(&mut (*inner).strong, atomic::AtomicUsize::new(1)); ptr::write(&mut (*inner).weak, atomic::AtomicUsize::new(1)); } - inner + Ok(inner) } /// Allocates an `ArcInner` with sufficient space for an unsized inner value. + #[cfg(not(no_global_oom_handling))] unsafe fn allocate_for_ptr(ptr: *const T) -> *mut ArcInner { // Allocate for the `ArcInner` using the given value. - Self::allocate_for_layout(Layout::for_value(&*ptr), |mem| { - set_data_ptr(ptr as *mut T, mem) as *mut ArcInner - }) + unsafe { + Self::allocate_for_layout( + Layout::for_value(&*ptr), + |layout| Global.allocate(layout), + |mem| mem.with_metadata_of(ptr as *const ArcInner), + ) + } } + #[cfg(not(no_global_oom_handling))] fn from_box(v: Box) -> Arc { unsafe { - let box_unique = Box::into_unique(v); + let (box_unique, alloc) = Box::into_unique(v); let bptr = box_unique.as_ptr(); let value_size = size_of_val(&*bptr); @@ -860,7 +1227,7 @@ impl Arc { ); // Free the allocation without dropping its contents - box_free(box_unique); + box_free(box_unique, alloc); Self::from_ptr(ptr) } @@ -869,37 +1236,35 @@ impl Arc { impl Arc<[T]> { /// Allocates an `ArcInner<[T]>` with the given length. + #[cfg(not(no_global_oom_handling))] unsafe fn allocate_for_slice(len: usize) -> *mut ArcInner<[T]> { - Self::allocate_for_layout(Layout::array::(len).unwrap(), |mem| { - ptr::slice_from_raw_parts_mut(mem as *mut T, len) as *mut ArcInner<[T]> - }) + unsafe { + Self::allocate_for_layout( + Layout::array::(len).unwrap(), + |layout| Global.allocate(layout), + |mem| ptr::slice_from_raw_parts_mut(mem as *mut T, len) as *mut ArcInner<[T]>, + ) + } } -} -/// Sets the data pointer of a `?Sized` raw pointer. -/// -/// For a slice/trait object, this sets the `data` field and leaves the rest -/// unchanged. For a sized raw pointer, this simply sets the pointer. -unsafe fn set_data_ptr(mut ptr: *mut T, data: *mut U) -> *mut T { - ptr::write(&mut ptr as *mut _ as *mut *mut u8, data as *mut u8); - ptr -} - -impl Arc<[T]> { - /// Copy elements from slice into newly allocated Arc<[T]> + /// Copy elements from slice into newly allocated `Arc<[T]>` /// /// Unsafe because the caller must either take ownership or bind `T: Copy`. + #[cfg(not(no_global_oom_handling))] unsafe fn copy_from_slice(v: &[T]) -> Arc<[T]> { - let ptr = Self::allocate_for_slice(v.len()); + unsafe { + let ptr = Self::allocate_for_slice(v.len()); - ptr::copy_nonoverlapping(v.as_ptr(), &mut (*ptr).data as *mut [T] as *mut T, v.len()); + ptr::copy_nonoverlapping(v.as_ptr(), &mut (*ptr).data as *mut [T] as *mut T, v.len()); - Self::from_ptr(ptr) + Self::from_ptr(ptr) + } } /// Constructs an `Arc<[T]>` from an iterator known to be of a certain size. /// /// Behavior is undefined should the size be wrong. + #[cfg(not(no_global_oom_handling))] unsafe fn from_iter_exact(iter: impl iter::Iterator, len: usize) -> Arc<[T]> { // Panic guard while cloning T elements. // In the event of a panic, elements that have been written @@ -917,38 +1282,42 @@ impl Arc<[T]> { let slice = from_raw_parts_mut(self.elems, self.n_elems); ptr::drop_in_place(slice); - // Crux: we should deallocate here, but we don't support `deallocate` yet. + Global.deallocate(self.mem, self.layout); } } } - let ptr = Self::allocate_for_slice(len); + unsafe { + let ptr = Self::allocate_for_slice(len); - let mem = ptr as *mut _ as *mut u8; - let layout = Layout::for_value(&*ptr); + let mem = ptr as *mut _ as *mut u8; + let layout = Layout::for_value(&*ptr); - // Pointer to first element - let elems = &mut (*ptr).data as *mut [T] as *mut T; + // Pointer to first element + let elems = &mut (*ptr).data as *mut [T] as *mut T; - let mut guard = Guard { mem: NonNull::new_unchecked(mem), elems, layout, n_elems: 0 }; + let mut guard = Guard { mem: NonNull::new_unchecked(mem), elems, layout, n_elems: 0 }; - for (i, item) in iter.enumerate() { - ptr::write(elems.add(i), item); - guard.n_elems += 1; - } + for (i, item) in iter.enumerate() { + ptr::write(elems.add(i), item); + guard.n_elems += 1; + } - // All clear. Forget the guard so it doesn't free the new ArcInner. - mem::forget(guard); + // All clear. Forget the guard so it doesn't free the new ArcInner. + mem::forget(guard); - Self::from_ptr(ptr) + Self::from_ptr(ptr) + } } } /// Specialization trait used for `From<&[T]>`. +#[cfg(not(no_global_oom_handling))] trait ArcFromSlice { fn from_slice(slice: &[T]) -> Self; } +#[cfg(not(no_global_oom_handling))] impl ArcFromSlice for Arc<[T]> { #[inline] default fn from_slice(v: &[T]) -> Self { @@ -956,6 +1325,7 @@ impl ArcFromSlice for Arc<[T]> { } } +#[cfg(not(no_global_oom_handling))] impl ArcFromSlice for Arc<[T]> { #[inline] fn from_slice(v: &[T]) -> Self { @@ -994,22 +1364,21 @@ impl Clone for Arc { // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html) let old_size = self.inner().strong.fetch_add(1, Relaxed); - // However we need to guard against massive refcounts in case someone - // is `mem::forget`ing Arcs. If we don't do this the count can overflow - // and users will use-after free. We racily saturate to `isize::MAX` on - // the assumption that there aren't ~2 billion threads incrementing - // the reference count at once. This branch will never be taken in - // any realistic program. + // However we need to guard against massive refcounts in case someone is `mem::forget`ing + // Arcs. If we don't do this the count can overflow and users will use-after free. This + // branch will never be taken in any realistic program. We abort because such a program is + // incredibly degenerate, and we don't care to support it. // - // We abort because such a program is incredibly degenerate, and we - // don't care to support it. + // This check is not 100% water-proof: we error when the refcount grows beyond `isize::MAX`. + // But we do that check *after* having done the increment, so there is a chance here that + // the worst already happened and we actually do overflow the `usize` counter. However, that + // requires the counter to grow from `isize::MAX` to `usize::MAX` between the increment + // above and the `abort` below, which seems exceedingly unlikely. if old_size > MAX_REFCOUNT { - unsafe { - abort(); - } + abort(); } - Self::from_inner(self.ptr) + unsafe { Self::from_inner(self.ptr) } } } @@ -1029,19 +1398,19 @@ impl Receiver for Arc {} impl Arc { /// Makes a mutable reference into the given `Arc`. /// - /// If there are other `Arc` or [`Weak`][weak] pointers to the same allocation, - /// then `make_mut` will create a new allocation and invoke [`clone`][clone] on the inner value - /// to ensure unique ownership. This is also referred to as clone-on-write. + /// If there are other `Arc` pointers to the same allocation, then `make_mut` will + /// [`clone`] the inner value to a new allocation to ensure unique ownership. This is also + /// referred to as clone-on-write. /// - /// Note that this differs from the behavior of [`Rc::make_mut`] which disassociates - /// any remaining `Weak` pointers. + /// However, if there are no other `Arc` pointers to this allocation, but some [`Weak`] + /// pointers, then the [`Weak`] pointers will be dissociated and the inner value will not + /// be cloned. /// - /// See also [`get_mut`][get_mut], which will fail rather than cloning. + /// See also [`get_mut`], which will fail rather than cloning the inner value + /// or dissociating [`Weak`] pointers. /// - /// [weak]: struct.Weak.html - /// [clone]: ../../std/clone/trait.Clone.html#tymethod.clone - /// [get_mut]: struct.Arc.html#method.get_mut - /// [`Rc::make_mut`]: ../rc/struct.Rc.html#method.make_mut + /// [`clone`]: Clone::clone + /// [`get_mut`]: Arc::get_mut /// /// # Examples /// @@ -1060,6 +1429,24 @@ impl Arc { /// assert_eq!(*data, 8); /// assert_eq!(*other_data, 12); /// ``` + /// + /// [`Weak`] pointers will be dissociated: + /// + /// ``` + /// use std::sync::Arc; + /// + /// let mut data = Arc::new(75); + /// let weak = Arc::downgrade(&data); + /// + /// assert!(75 == *data); + /// assert!(75 == *weak.upgrade().unwrap()); + /// + /// *Arc::make_mut(&mut data) += 1; + /// + /// assert!(76 == *data); + /// assert!(weak.upgrade().is_none()); + /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "arc_unique", since = "1.4.0")] pub fn make_mut(this: &mut Self) -> &mut T { @@ -1072,8 +1459,14 @@ impl Arc { // weak count, there's no chance the ArcInner itself could be // deallocated. if this.inner().strong.compare_exchange(1, 0, Acquire, Relaxed).is_err() { - // Another strong pointer exists; clone - *this = Arc::new((**this).clone()); + // Another strong pointer exists, so we must clone. + // Pre-allocate memory to allow writing the cloned value directly. + let mut arc = Self::new_uninit(); + unsafe { + let data = Arc::get_mut_unchecked(&mut arc); + (**this).write_clone_into_raw(data.as_mut_ptr()); + *this = arc.assume_init(); + } } else if this.inner().weak.load(Relaxed) != 1 { // Relaxed suffices in the above because this is fundamentally an // optimization: we are always racing with weak pointers being @@ -1089,17 +1482,14 @@ impl Arc { // Materialize our own implicit weak pointer, so that it can clean // up the ArcInner as needed. - let weak = Weak { ptr: this.ptr }; + let _weak = Weak { ptr: this.ptr }; - // mark the data itself as already deallocated + // Can just steal the data, all that's left is Weaks + let mut arc = Self::new_uninit(); unsafe { - // there is no data race in the implicit write caused by `read` - // here (due to zeroing) because data is no longer accessed by - // other threads (due to there being no more strong refs at this - // point). - let mut swap = Arc::new(ptr::read(&weak.ptr.as_ref().data)); - mem::swap(this, &mut swap); - mem::forget(swap); + let data = Arc::get_mut_unchecked(&mut arc); + data.as_mut_ptr().copy_from_nonoverlapping(&**this, 1); + ptr::write(this, arc.assume_init()); } } else { // We were the sole reference of either kind; bump back up the @@ -1109,24 +1499,57 @@ impl Arc { // As with `get_mut()`, the unsafety is ok because our reference was // either unique to begin with, or became one upon cloning the contents. - unsafe { &mut this.ptr.as_mut().data } + unsafe { Self::get_mut_unchecked(this) } + } + + /// If we have the only reference to `T` then unwrap it. Otherwise, clone `T` and return the + /// clone. + /// + /// Assuming `arc_t` is of type `Arc`, this function is functionally equivalent to + /// `(*arc_t).clone()`, but will avoid cloning the inner value where possible. + /// + /// # Examples + /// + /// ``` + /// #![feature(arc_unwrap_or_clone)] + /// # use std::{ptr, sync::Arc}; + /// let inner = String::from("test"); + /// let ptr = inner.as_ptr(); + /// + /// let arc = Arc::new(inner); + /// let inner = Arc::unwrap_or_clone(arc); + /// // The inner value was not cloned + /// assert!(ptr::eq(ptr, inner.as_ptr())); + /// + /// let arc = Arc::new(inner); + /// let arc2 = arc.clone(); + /// let inner = Arc::unwrap_or_clone(arc); + /// // Because there were 2 references, we had to clone the inner value. + /// assert!(!ptr::eq(ptr, inner.as_ptr())); + /// // `arc2` is the last reference, so when we unwrap it we get back + /// // the original `String`. + /// let inner = Arc::unwrap_or_clone(arc2); + /// assert!(ptr::eq(ptr, inner.as_ptr())); + /// ``` + #[inline] + #[unstable(feature = "arc_unwrap_or_clone", issue = "93610")] + pub fn unwrap_or_clone(this: Self) -> T { + Arc::try_unwrap(this).unwrap_or_else(|arc| (*arc).clone()) } } impl Arc { /// Returns a mutable reference into the given `Arc`, if there are - /// no other `Arc` or [`Weak`][weak] pointers to the same allocation. + /// no other `Arc` or [`Weak`] pointers to the same allocation. /// - /// Returns [`None`][option] otherwise, because it is not safe to + /// Returns [`None`] otherwise, because it is not safe to /// mutate a shared value. /// /// See also [`make_mut`][make_mut], which will [`clone`][clone] - /// the inner value when there are other pointers. + /// the inner value when there are other `Arc` pointers. /// - /// [weak]: struct.Weak.html - /// [option]: ../../std/option/enum.Option.html - /// [make_mut]: struct.Arc.html#method.make_mut - /// [clone]: ../../std/clone/trait.Clone.html#tymethod.clone + /// [make_mut]: Arc::make_mut + /// [clone]: Clone::clone /// /// # Examples /// @@ -1160,14 +1583,15 @@ impl Arc { /// /// See also [`get_mut`], which is safe and does appropriate checks. /// - /// [`get_mut`]: struct.Arc.html#method.get_mut + /// [`get_mut`]: Arc::get_mut /// /// # Safety /// - /// Any other `Arc` or [`Weak`] pointers to the same allocation must not be dereferenced - /// for the duration of the returned borrow. - /// This is trivially the case if no such pointers exist, - /// for example immediately after `Arc::new`. + /// If any other `Arc` or [`Weak`] pointers to the same allocation exist, then + /// they must be must not be dereferenced or have active borrows for the duration + /// of the returned borrow, and their inner type must be exactly the same as the + /// inner type of this Rc (including lifetimes). This is trivially the case if no + /// such pointers exist, for example immediately after `Arc::new`. /// /// # Examples /// @@ -1182,10 +1606,44 @@ impl Arc { /// } /// assert_eq!(*x, "foo"); /// ``` + /// Other `Arc` pointers to the same allocation must be to the same type. + /// ```no_run + /// #![feature(get_mut_unchecked)] + /// + /// use std::sync::Arc; + /// + /// let x: Arc = Arc::from("Hello, world!"); + /// let mut y: Arc<[u8]> = x.clone().into(); + /// unsafe { + /// // this is Undefined Behavior, because x's inner type is str, not [u8] + /// Arc::get_mut_unchecked(&mut y).fill(0xff); // 0xff is invalid in UTF-8 + /// } + /// println!("{}", &*x); // Invalid UTF-8 in a str + /// ``` + /// Other `Arc` pointers to the same allocation must be to the exact same type, including lifetimes. + /// ```no_run + /// #![feature(get_mut_unchecked)] + /// + /// use std::sync::Arc; + /// + /// let x: Arc<&str> = Arc::new("Hello, world!"); + /// { + /// let s = String::from("Oh, no!"); + /// let mut y: Arc<&str> = x.clone().into(); + /// unsafe { + /// // this is Undefined Behavior, because x's inner type + /// // is &'long str, not &'short str + /// *Arc::get_mut_unchecked(&mut y) = &s; + /// } + /// } + /// println!("{}", &*x); // Use-after-free + /// ``` #[inline] #[unstable(feature = "get_mut_unchecked", issue = "63292")] pub unsafe fn get_mut_unchecked(this: &mut Self) -> &mut T { - &mut this.ptr.as_mut().data + // We are careful to *not* create a reference covering the "count" fields, as + // this would alias with concurrent access to the reference counts (e.g. by `Weak`). + unsafe { &mut (*this.ptr.as_ptr()).data } } /// Determine whether this is the unique reference (including weak refs) to @@ -1198,7 +1656,7 @@ impl Arc { // // The acquire label here ensures a happens-before relationship with any // writes to `strong` (in particular in `Weak::upgrade`) prior to decrements - // of the `weak` count (via `Weak::drop`, which uses release). If the upgraded + // of the `weak` count (via `Weak::drop`, which uses release). If the upgraded // weak ref was never dropped, the CAS here will fail so we do not care to synchronize. if self.inner().weak.compare_exchange(1, usize::MAX, Acquire, Relaxed).is_ok() { // This needs to be an `Acquire` to synchronize with the decrement of the `strong` @@ -1244,8 +1702,6 @@ unsafe impl<#[may_dangle] T: ?Sized> Drop for Arc { /// drop(foo); // Doesn't print anything /// drop(foo2); // Prints "dropped!" /// ``` - /// - /// [`Weak`]: ../../std/sync/struct.Weak.html #[inline] fn drop(&mut self) { // Because `fetch_sub` is already atomic, we do not need to synchronize @@ -1256,7 +1712,7 @@ unsafe impl<#[may_dangle] T: ?Sized> Drop for Arc { } // This fence is needed to prevent reordering of use of the data and - // deletion of the data. Because it is marked `Release`, the decreasing + // deletion of the data. Because it is marked `Release`, the decreasing // of the reference count synchronizes with this `Acquire` fence. This // means that use of the data happens before decreasing the reference // count, which happens before this fence, which happens before the @@ -1292,8 +1748,6 @@ unsafe impl<#[may_dangle] T: ?Sized> Drop for Arc { } impl Arc { - #[inline] - #[stable(feature = "rc_downcast", since = "1.29.0")] /// Attempt to downcast the `Arc` to a concrete type. /// /// # Examples @@ -1312,26 +1766,68 @@ impl Arc { /// print_if_string(Arc::new(my_string)); /// print_if_string(Arc::new(0i8)); /// ``` + #[inline] + #[stable(feature = "rc_downcast", since = "1.29.0")] pub fn downcast(self) -> Result, Self> where - T: Any + Send + Sync + 'static, + T: Any + Send + Sync, { if (*self).is::() { - let ptr = self.ptr.cast::>(); - mem::forget(self); - Ok(Arc::from_inner(ptr)) + unsafe { + let ptr = self.ptr.cast::>(); + mem::forget(self); + Ok(Arc::from_inner(ptr)) + } } else { Err(self) } } + + /// Downcasts the `Arc` to a concrete type. + /// + /// For a safe alternative see [`downcast`]. + /// + /// # Examples + /// + /// ``` + /// #![feature(downcast_unchecked)] + /// + /// use std::any::Any; + /// use std::sync::Arc; + /// + /// let x: Arc = Arc::new(1_usize); + /// + /// unsafe { + /// assert_eq!(*x.downcast_unchecked::(), 1); + /// } + /// ``` + /// + /// # Safety + /// + /// The contained value must be of type `T`. Calling this method + /// with the incorrect type is *undefined behavior*. + /// + /// + /// [`downcast`]: Self::downcast + #[inline] + #[unstable(feature = "downcast_unchecked", issue = "90850")] + pub unsafe fn downcast_unchecked(self) -> Arc + where + T: Any + Send + Sync, + { + unsafe { + let ptr = self.ptr.cast::>(); + mem::forget(self); + Arc::from_inner(ptr) + } + } } impl Weak { /// Constructs a new `Weak`, without allocating any memory. /// Calling [`upgrade`] on the return value always gives [`None`]. /// - /// [`upgrade`]: struct.Weak.html#method.upgrade - /// [`None`]: ../../std/option/enum.Option.html#variant.None + /// [`upgrade`]: Weak::upgrade /// /// # Examples /// @@ -1342,66 +1838,75 @@ impl Weak { /// assert!(empty.upgrade().is_none()); /// ``` #[stable(feature = "downgraded_weak", since = "1.10.0")] - pub fn new() -> Weak { - Weak { ptr: NonNull::new(usize::MAX as *mut ArcInner).expect("MAX is not 0") } + #[rustc_const_unstable(feature = "const_weak_new", issue = "95091", reason = "recently added")] + #[must_use] + pub const fn new() -> Weak { + Weak { ptr: unsafe { NonNull::new_unchecked(ptr::invalid_mut::>(usize::MAX)) } } } +} + +/// Helper type to allow accessing the reference counts without +/// making any assertions about the data field. +struct WeakInner<'a> { + weak: &'a atomic::AtomicUsize, + strong: &'a atomic::AtomicUsize, +} +impl Weak { /// Returns a raw pointer to the object `T` pointed to by this `Weak`. /// - /// The pointer is valid only if there are some strong references. The pointer may be dangling - /// or even [`null`] otherwise. + /// The pointer is valid only if there are some strong references. The pointer may be dangling, + /// unaligned or even [`null`] otherwise. /// /// # Examples /// /// ``` - /// #![feature(weak_into_raw)] - /// /// use std::sync::Arc; /// use std::ptr; /// /// let strong = Arc::new("hello".to_owned()); /// let weak = Arc::downgrade(&strong); /// // Both point to the same object - /// assert!(ptr::eq(&*strong, weak.as_raw())); + /// assert!(ptr::eq(&*strong, weak.as_ptr())); /// // The strong here keeps it alive, so we can still access the object. - /// assert_eq!("hello", unsafe { &*weak.as_raw() }); + /// assert_eq!("hello", unsafe { &*weak.as_ptr() }); /// /// drop(strong); - /// // But not any more. We can do weak.as_raw(), but accessing the pointer would lead to + /// // But not any more. We can do weak.as_ptr(), but accessing the pointer would lead to /// // undefined behaviour. - /// // assert_eq!("hello", unsafe { &*weak.as_raw() }); - /// ``` - /// - /// [`null`]: ../../std/ptr/fn.null.html - #[unstable(feature = "weak_into_raw", issue = "60728")] - pub fn as_raw(&self) -> *const T { - match self.inner() { - None => ptr::null(), - Some(inner) => { - let offset = data_offset_sized::(); - let ptr = inner as *const ArcInner; - // Note: while the pointer we create may already point to dropped value, the - // allocation still lives (it must hold the weak point as long as we are alive). - // Therefore, the offset is OK to do, it won't get out of the allocation. - let ptr = unsafe { (ptr as *const u8).offset(offset) }; - ptr as *const T - } + /// // assert_eq!("hello", unsafe { &*weak.as_ptr() }); + /// ``` + /// + /// [`null`]: core::ptr::null "ptr::null" + #[must_use] + #[stable(feature = "weak_into_raw", since = "1.45.0")] + pub fn as_ptr(&self) -> *const T { + let ptr: *mut ArcInner = NonNull::as_ptr(self.ptr); + + if is_dangling(ptr) { + // If the pointer is dangling, we return the sentinel directly. This cannot be + // a valid payload address, as the payload is at least as aligned as ArcInner (usize). + ptr as *const T + } else { + // SAFETY: if is_dangling returns false, then the pointer is dereferenceable. + // The payload may be dropped at this point, and we have to maintain provenance, + // so use raw pointer manipulation. + unsafe { ptr::addr_of_mut!((*ptr).data) } } } /// Consumes the `Weak` and turns it into a raw pointer. /// - /// This converts the weak pointer into a raw pointer, preserving the original weak count. It - /// can be turned back into the `Weak` with [`from_raw`]. + /// This converts the weak pointer into a raw pointer, while still preserving the ownership of + /// one weak reference (the weak count is not modified by this operation). It can be turned + /// back into the `Weak` with [`from_raw`]. /// /// The same restrictions of accessing the target of the pointer as with - /// [`as_raw`] apply. + /// [`as_ptr`] apply. /// /// # Examples /// /// ``` - /// #![feature(weak_into_raw)] - /// /// use std::sync::{Arc, Weak}; /// /// let strong = Arc::new("hello".to_owned()); @@ -1415,39 +1920,36 @@ impl Weak { /// assert_eq!(0, Arc::weak_count(&strong)); /// ``` /// - /// [`from_raw`]: struct.Weak.html#method.from_raw - /// [`as_raw`]: struct.Weak.html#method.as_raw - #[unstable(feature = "weak_into_raw", issue = "60728")] + /// [`from_raw`]: Weak::from_raw + /// [`as_ptr`]: Weak::as_ptr + #[must_use = "`self` will be dropped if the result is not used"] + #[stable(feature = "weak_into_raw", since = "1.45.0")] pub fn into_raw(self) -> *const T { - let result = self.as_raw(); + let result = self.as_ptr(); mem::forget(self); result } - /// Converts a raw pointer previously created by [`into_raw`] back into - /// `Weak`. + /// Converts a raw pointer previously created by [`into_raw`] back into `Weak`. /// /// This can be used to safely get a strong reference (by calling [`upgrade`] /// later) or to deallocate the weak count by dropping the `Weak`. /// - /// It takes ownership of one weak count (with the exception of pointers created by [`new`], - /// as these don't have any corresponding weak count). + /// It takes ownership of one weak reference (with the exception of pointers created by [`new`], + /// as these don't own anything; the method still works on them). /// /// # Safety /// - /// The pointer must have originated from the [`into_raw`] (or [`as_raw'], provided there was - /// a corresponding [`forget`] on the `Weak`) and must still own its potential weak reference - /// count. - /// - /// It is allowed for the strong count to be 0 at the time of calling this, but the weak count - /// must be non-zero or the pointer must have originated from a dangling `Weak` (one created - /// by [`new`]). + /// The pointer must have originated from the [`into_raw`] and must still own its potential + /// weak reference. /// + /// It is allowed for the strong count to be 0 at the time of calling this. Nevertheless, this + /// takes ownership of one weak reference currently represented as a raw pointer (the weak + /// count is not modified by this operation) and therefore it must be paired with a previous + /// call to [`into_raw`]. /// # Examples /// /// ``` - /// #![feature(weak_into_raw)] - /// /// use std::sync::{Arc, Weak}; /// /// let strong = Arc::new("hello".to_owned()); @@ -1466,24 +1968,27 @@ impl Weak { /// assert!(unsafe { Weak::from_raw(raw_2) }.upgrade().is_none()); /// ``` /// - /// [`as_raw`]: struct.Weak.html#method.as_raw - /// [`new`]: struct.Weak.html#method.new - /// [`into_raw`]: struct.Weak.html#method.into_raw - /// [`upgrade`]: struct.Weak.html#method.upgrade - /// [`Weak`]: struct.Weak.html - /// [`Arc`]: struct.Arc.html - /// [`forget`]: ../../std/mem/fn.forget.html - #[unstable(feature = "weak_into_raw", issue = "60728")] + /// [`new`]: Weak::new + /// [`into_raw`]: Weak::into_raw + /// [`upgrade`]: Weak::upgrade + #[stable(feature = "weak_into_raw", since = "1.45.0")] pub unsafe fn from_raw(ptr: *const T) -> Self { - if ptr.is_null() { - Self::new() + // See Weak::as_ptr for context on how the input pointer is derived. + + let ptr = if is_dangling(ptr as *mut T) { + // This is a dangling Weak. + ptr as *mut ArcInner } else { - // See Arc::from_raw for details - let offset = data_offset(ptr); - let fake_ptr = ptr as *mut ArcInner; - let ptr = set_data_ptr(fake_ptr, (ptr as *mut u8).offset(-offset)); - Weak { ptr: NonNull::new(ptr).expect("Invalid pointer passed to from_raw") } - } + // Otherwise, we're guaranteed the pointer came from a nondangling Weak. + // SAFETY: data_offset is safe to call, as ptr references a real (potentially dropped) T. + let offset = unsafe { data_offset(ptr) }; + // Thus, we reverse the offset to get the whole RcBox. + // SAFETY: the pointer originated from a Weak, so this offset is safe. + unsafe { ptr.byte_sub(offset) as *mut ArcInner } + }; + + // SAFETY: we now have recovered the original Weak pointer, so can create the Weak. + Weak { ptr: unsafe { NonNull::new_unchecked(ptr) } } } } @@ -1493,9 +1998,6 @@ impl Weak { /// /// Returns [`None`] if the inner value has since been dropped. /// - /// [`Arc`]: struct.Arc.html - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// /// # Examples /// /// ``` @@ -1514,46 +2016,42 @@ impl Weak { /// /// assert!(weak_five.upgrade().is_none()); /// ``` + #[must_use = "this returns a new `Arc`, \ + without modifying the original weak pointer"] #[stable(feature = "arc_weak", since = "1.4.0")] pub fn upgrade(&self) -> Option> { // We use a CAS loop to increment the strong count instead of a - // fetch_add because once the count hits 0 it must never be above 0. - let inner = self.inner()?; - - // Relaxed load because any write of 0 that we can observe - // leaves the field in a permanently zero state (so a - // "stale" read of 0 is fine), and any other value is - // confirmed via the CAS below. - let mut n = inner.strong.load(Relaxed); - - loop { - if n == 0 { - return None; - } - - // See comments in `Arc::clone` for why we do this (for `mem::forget`). - if n > MAX_REFCOUNT { - unsafe { + // fetch_add as this function should never take the reference count + // from zero to one. + self.inner()? + .strong + // Relaxed is fine for the failure case because we don't have any expectations about the new state. + // Acquire is necessary for the success case to synchronise with `Arc::new_cyclic`, when the inner + // value can be initialized after `Weak` references have already been created. In that case, we + // expect to observe the fully initialized value. + .fetch_update(Acquire, Relaxed, |n| { + // Any write of 0 we can observe leaves the field in permanently zero state. + if n == 0 { + return None; + } + // See comments in `Arc::clone` for why we do this (for `mem::forget`). + if n > MAX_REFCOUNT { abort(); } - } - - // Relaxed is valid for the same reason it is on Arc's Clone impl - match inner.strong.compare_exchange_weak(n, n + 1, Relaxed, Relaxed) { - Ok(_) => return Some(Arc::from_inner(self.ptr)), // null checked above - Err(old) => n = old, - } - } + Some(n + 1) + }) + .ok() + // null checked above + .map(|_| unsafe { Arc::from_inner(self.ptr) }) } /// Gets the number of strong (`Arc`) pointers pointing to this allocation. /// /// If `self` was created using [`Weak::new`], this will return 0. - /// - /// [`Weak::new`]: #method.new + #[must_use] #[stable(feature = "weak_counts", since = "1.41.0")] pub fn strong_count(&self) -> usize { - if let Some(inner) = self.inner() { inner.strong.load(SeqCst) } else { 0 } + if let Some(inner) = self.inner() { inner.strong.load(Acquire) } else { 0 } } /// Gets an approximation of the number of `Weak` pointers pointing to this @@ -1567,14 +2065,13 @@ impl Weak { /// Due to implementation details, the returned value can be off by 1 in /// either direction when other threads are manipulating any `Arc`s or /// `Weak`s pointing to the same allocation. - /// - /// [`Weak::new`]: #method.new + #[must_use] #[stable(feature = "weak_counts", since = "1.41.0")] pub fn weak_count(&self) -> usize { self.inner() .map(|inner| { - let weak = inner.weak.load(SeqCst); - let strong = inner.strong.load(SeqCst); + let weak = inner.weak.load(Acquire); + let strong = inner.strong.load(Acquire); if strong == 0 { 0 } else { @@ -1592,13 +2089,23 @@ impl Weak { /// Returns `None` when the pointer is dangling and there is no allocated `ArcInner`, /// (i.e., when this `Weak` was created by `Weak::new`). #[inline] - fn inner(&self) -> Option<&ArcInner> { - if is_dangling(self.ptr) { None } else { Some(unsafe { self.ptr.as_ref() }) } + fn inner(&self) -> Option> { + if is_dangling(self.ptr.as_ptr()) { + None + } else { + // We are careful to *not* create a reference covering the "data" field, as + // the field may be mutated concurrently (for example, if the last `Arc` + // is dropped, the data field will be dropped in-place). + Some(unsafe { + let ptr = self.ptr.as_ptr(); + WeakInner { strong: &(*ptr).strong, weak: &(*ptr).weak } + }) + } } - /// Returns `true` if the two `Weak`s point to the same allocation (similar to - /// [`ptr::eq`]), or if both don't point to any allocation - /// (because they were created with `Weak::new()`). + /// Returns `true` if the two `Weak`s point to the same allocation similar to [`ptr::eq`], or if + /// both don't point to any allocation (because they were created with `Weak::new()`). See [that + /// function][`ptr::eq`] for caveats when comparing `dyn Trait` pointers. /// /// # Notes /// @@ -1636,8 +2143,9 @@ impl Weak { /// assert!(!first.ptr_eq(&third)); /// ``` /// - /// [`ptr::eq`]: ../../std/ptr/fn.eq.html + /// [`ptr::eq`]: core::ptr::eq "ptr::eq" #[inline] + #[must_use] #[stable(feature = "weak_ptr_eq", since = "1.39.0")] pub fn ptr_eq(&self, other: &Self) -> bool { self.ptr.as_ptr() == other.ptr.as_ptr() @@ -1664,7 +2172,7 @@ impl Clone for Weak { } else { return Weak { ptr: self.ptr }; }; - // See comments in Arc::clone() for why this is relaxed. This can use a + // See comments in Arc::clone() for why this is relaxed. This can use a // fetch_add (ignoring the lock) because the weak count is only locked // where are *no other* weak pointers in existence. (So we can't be // running this code in that case). @@ -1672,9 +2180,7 @@ impl Clone for Weak { // See comments in Arc::clone() for why we do this (for mem::forget). if old_size > MAX_REFCOUNT { - unsafe { - abort(); - } + abort(); } Weak { ptr: self.ptr } @@ -1687,8 +2193,7 @@ impl Default for Weak { /// Calling [`upgrade`] on the return value always /// gives [`None`]. /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// [`upgrade`]: ../../std/sync/struct.Weak.html#method.upgrade + /// [`upgrade`]: Weak::upgrade /// /// # Examples /// @@ -1704,7 +2209,7 @@ impl Default for Weak { } #[stable(feature = "arc_weak", since = "1.4.0")] -impl Drop for Weak { +unsafe impl<#[may_dangle] T: ?Sized> Drop for Weak { /// Drops the `Weak` pointer. /// /// # Examples @@ -1742,7 +2247,7 @@ impl Drop for Weak { if inner.weak.fetch_sub(1, Release) == 1 { acquire!(inner.weak); - // Crux: we should deallocate here, but we don't support `deallocate` yet. + unsafe { Global.deallocate(self.ptr.cast(), Layout::for_value_raw(self.ptr.as_ptr())) } } } } @@ -1773,7 +2278,7 @@ impl ArcEqIdent for Arc { /// /// We can only do this when `T: Eq` as a `PartialEq` might be deliberately irreflexive. #[stable(feature = "rust1", since = "1.0.0")] -impl ArcEqIdent for Arc { +impl ArcEqIdent for Arc { #[inline] fn eq(&self, other: &Arc) -> bool { Arc::ptr_eq(self, other) || **self == **other @@ -1963,6 +2468,7 @@ impl fmt::Pointer for Arc { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl Default for Arc { /// Creates a new `Arc`, with the `Default` value for `T`. @@ -1987,23 +2493,59 @@ impl Hash for Arc { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "from_for_ptrs", since = "1.6.0")] impl From for Arc { + /// Converts a `T` into an `Arc` + /// + /// The conversion moves the value into a + /// newly allocated `Arc`. It is equivalent to + /// calling `Arc::new(t)`. + /// + /// # Example + /// ```rust + /// # use std::sync::Arc; + /// let x = 5; + /// let arc = Arc::new(5); + /// + /// assert_eq!(Arc::from(x), arc); + /// ``` fn from(t: T) -> Self { Arc::new(t) } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_slice", since = "1.21.0")] impl From<&[T]> for Arc<[T]> { + /// Allocate a reference-counted slice and fill it by cloning `v`'s items. + /// + /// # Example + /// + /// ``` + /// # use std::sync::Arc; + /// let original: &[i32] = &[1, 2, 3]; + /// let shared: Arc<[i32]> = Arc::from(original); + /// assert_eq!(&[1, 2, 3], &shared[..]); + /// ``` #[inline] fn from(v: &[T]) -> Arc<[T]> { >::from_slice(v) } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_slice", since = "1.21.0")] impl From<&str> for Arc { + /// Allocate a reference-counted `str` and copy `v` into it. + /// + /// # Example + /// + /// ``` + /// # use std::sync::Arc; + /// let shared: Arc = Arc::from("eggplant"); + /// assert_eq!("eggplant", &shared[..]); + /// ``` #[inline] fn from(v: &str) -> Arc { let arc = Arc::<[u8]>::from(v.as_bytes()); @@ -2011,42 +2553,116 @@ impl From<&str> for Arc { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_slice", since = "1.21.0")] impl From for Arc { + /// Allocate a reference-counted `str` and copy `v` into it. + /// + /// # Example + /// + /// ``` + /// # use std::sync::Arc; + /// let unique: String = "eggplant".to_owned(); + /// let shared: Arc = Arc::from(unique); + /// assert_eq!("eggplant", &shared[..]); + /// ``` #[inline] fn from(v: String) -> Arc { Arc::from(&v[..]) } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_slice", since = "1.21.0")] impl From> for Arc { + /// Move a boxed object to a new, reference-counted allocation. + /// + /// # Example + /// + /// ``` + /// # use std::sync::Arc; + /// let unique: Box = Box::from("eggplant"); + /// let shared: Arc = Arc::from(unique); + /// assert_eq!("eggplant", &shared[..]); + /// ``` #[inline] fn from(v: Box) -> Arc { Arc::from_box(v) } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_slice", since = "1.21.0")] impl From> for Arc<[T]> { + /// Allocate a reference-counted slice and move `v`'s items into it. + /// + /// # Example + /// + /// ``` + /// # use std::sync::Arc; + /// let unique: Vec = vec![1, 2, 3]; + /// let shared: Arc<[i32]> = Arc::from(unique); + /// assert_eq!(&[1, 2, 3], &shared[..]); + /// ``` #[inline] fn from(mut v: Vec) -> Arc<[T]> { unsafe { - let arc = Arc::copy_from_slice(&v); - + let rc = Arc::copy_from_slice(&v); // Allow the Vec to free its memory, but not destroy its contents v.set_len(0); - - arc + rc } } } -#[stable(feature = "boxed_slice_try_from", since = "1.43.0")] -impl TryFrom> for Arc<[T; N]> +#[stable(feature = "shared_from_cow", since = "1.45.0")] +impl<'a, B> From> for Arc where - [T; N]: LengthAtMost32, + B: ToOwned + ?Sized, + Arc: From<&'a B> + From, { + /// Create an atomically reference-counted pointer from + /// a clone-on-write pointer by copying its content. + /// + /// # Example + /// + /// ```rust + /// # use std::sync::Arc; + /// # use std::borrow::Cow; + /// let cow: Cow = Cow::Borrowed("eggplant"); + /// let shared: Arc = Arc::from(cow); + /// assert_eq!("eggplant", &shared[..]); + /// ``` + #[inline] + fn from(cow: Cow<'a, B>) -> Arc { + match cow { + Cow::Borrowed(s) => Arc::from(s), + Cow::Owned(s) => Arc::from(s), + } + } +} + +#[stable(feature = "shared_from_str", since = "1.62.0")] +impl From> for Arc<[u8]> { + /// Converts an atomically reference-counted string slice into a byte slice. + /// + /// # Example + /// + /// ``` + /// # use std::sync::Arc; + /// let string: Arc = Arc::from("eggplant"); + /// let bytes: Arc<[u8]> = Arc::from(string); + /// assert_eq!("eggplant".as_bytes(), bytes.as_ref()); + /// ``` + #[inline] + fn from(rc: Arc) -> Self { + // SAFETY: `str` has the same layout as `[u8]`. + unsafe { Arc::from_raw(Arc::into_raw(rc) as *const [u8]) } + } +} + +#[stable(feature = "boxed_slice_try_from", since = "1.43.0")] +impl TryFrom> for Arc<[T; N]> { type Error = Arc<[T]>; fn try_from(boxed_slice: Arc<[T]>) -> Result { @@ -2058,6 +2674,7 @@ where } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_iter", since = "1.37.0")] impl iter::FromIterator for Arc<[T]> { /// Takes each element in the `Iterator` and collects it into an `Arc<[T]>`. @@ -2099,25 +2716,27 @@ impl iter::FromIterator for Arc<[T]> { /// # assert_eq!(&*evens, &*(0..10).collect::>()); /// ``` fn from_iter>(iter: I) -> Self { - ArcFromIter::from_iter(iter.into_iter()) + ToArcSlice::to_arc_slice(iter.into_iter()) } } /// Specialization trait used for collecting into `Arc<[T]>`. -trait ArcFromIter { - fn from_iter(iter: I) -> Self; +trait ToArcSlice: Iterator + Sized { + fn to_arc_slice(self) -> Arc<[T]>; } -impl> ArcFromIter for Arc<[T]> { - default fn from_iter(iter: I) -> Self { - iter.collect::>().into() +#[cfg(not(no_global_oom_handling))] +impl> ToArcSlice for I { + default fn to_arc_slice(self) -> Arc<[T]> { + self.collect::>().into() } } -impl> ArcFromIter for Arc<[T]> { - default fn from_iter(iter: I) -> Self { +#[cfg(not(no_global_oom_handling))] +impl> ToArcSlice for I { + fn to_arc_slice(self) -> Arc<[T]> { // This is the case for a `TrustedLen` iterator. - let (low, high) = iter.size_hint(); + let (low, high) = self.size_hint(); if let Some(high) = high { debug_assert_eq!( low, @@ -2128,29 +2747,18 @@ impl> ArcFromIter for Arc<[T]> { unsafe { // SAFETY: We need to ensure that the iterator has an exact length and we have. - Arc::from_iter_exact(iter, low) + Arc::from_iter_exact(self, low) } } else { - // Fall back to normal implementation. - iter.collect::>().into() + // TrustedLen contract guarantees that `upper_bound == `None` implies an iterator + // length exceeding `usize::MAX`. + // The default implementation would collect into a vec which would panic. + // Thus we panic here immediately without invoking `Vec` code. + panic!("capacity overflow"); } } } -impl<'a, T: 'a + Clone> ArcFromIter<&'a T, slice::Iter<'a, T>> for Arc<[T]> { - fn from_iter(iter: slice::Iter<'a, T>) -> Self { - // Delegate to `impl From<&[T]> for Arc<[T]>`. - // - // In the case that `T: Copy`, we get to use `ptr::copy_nonoverlapping` - // which is even more performant. - // - // In the fall-back case we have `T: Clone`. This is still better - // than the `TrustedLen` implementation as slices have a known length - // and so we get to avoid calling `size_hint` and avoid the branching. - iter.as_slice().into() - } -} - #[stable(feature = "rust1", since = "1.0.0")] impl borrow::Borrow for Arc { fn borrow(&self) -> &T { @@ -2168,24 +2776,45 @@ impl AsRef for Arc { #[stable(feature = "pin", since = "1.33.0")] impl Unpin for Arc {} -/// Computes the offset of the data field within `ArcInner`. -unsafe fn data_offset(ptr: *const T) -> isize { - // Align the unsized value to the end of the `ArcInner`. - // Because it is `?Sized`, it will always be the last field in memory. - // Note: This is a detail of the current implementation of the compiler, - // and is not a guaranteed language detail. Do not rely on it outside of std. - data_offset_align(align_of_val(&*ptr)) -} - -/// Computes the offset of the data field within `ArcInner`. +/// Get the offset within an `ArcInner` for the payload behind a pointer. +/// +/// # Safety /// -/// Unlike [`data_offset`], this doesn't need the pointer, but it works only on `T: Sized`. -fn data_offset_sized() -> isize { - data_offset_align(align_of::()) +/// The pointer must point to (and have valid metadata for) a previously +/// valid instance of T, but the T is allowed to be dropped. +unsafe fn data_offset(ptr: *const T) -> usize { + // Align the unsized value to the end of the ArcInner. + // Because RcBox is repr(C), it will always be the last field in memory. + // SAFETY: since the only unsized types possible are slices, trait objects, + // and extern types, the input safety requirement is currently enough to + // satisfy the requirements of align_of_val_raw; this is an implementation + // detail of the language that must not be relied upon outside of std. + unsafe { data_offset_align(align_of_val_raw(ptr)) } } #[inline] -fn data_offset_align(align: usize) -> isize { +fn data_offset_align(align: usize) -> usize { let layout = Layout::new::>(); - (layout.size() + layout.padding_needed_for(align)) as isize + layout.size() + layout.padding_needed_for(align) +} + +#[stable(feature = "arc_error", since = "1.52.0")] +impl core::error::Error for Arc { + #[allow(deprecated, deprecated_in_future)] + fn description(&self) -> &str { + core::error::Error::description(&**self) + } + + #[allow(deprecated)] + fn cause(&self) -> Option<&dyn core::error::Error> { + core::error::Error::cause(&**self) + } + + fn source(&self) -> Option<&(dyn core::error::Error + 'static)> { + core::error::Error::source(&**self) + } + + fn provide<'a>(&'a self, req: &mut core::any::Demand<'a>) { + core::error::Error::provide(&**self, req); + } } diff --git a/crux-mir/lib/alloc/src/sync/tests.rs b/crux-mir/lib/alloc/src/sync/tests.rs index edc2820ee..0fae8953a 100644 --- a/crux-mir/lib/alloc/src/sync/tests.rs +++ b/crux-mir/lib/alloc/src/sync/tests.rs @@ -32,7 +32,6 @@ impl Drop for Canary { #[test] #[cfg_attr(target_os = "emscripten", ignore)] -#[cfg_attr(miri, ignore)] // Miri does not support threads fn manually_share_arc() { let v = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; let arc_v = Arc::new(v); @@ -104,7 +103,7 @@ fn try_unwrap() { #[test] fn into_from_raw() { - let x = Arc::new(box "hello"); + let x = Arc::new(Box::new("hello")); let y = x.clone(); let x_ptr = Arc::into_raw(x); @@ -141,6 +140,48 @@ fn test_into_from_raw_unsized() { assert_eq!(arc2.to_string(), "123"); } +#[test] +fn into_from_weak_raw() { + let x = Arc::new(Box::new("hello")); + let y = Arc::downgrade(&x); + + let y_ptr = Weak::into_raw(y); + unsafe { + assert_eq!(**y_ptr, "hello"); + + let y = Weak::from_raw(y_ptr); + let y_up = Weak::upgrade(&y).unwrap(); + assert_eq!(**y_up, "hello"); + drop(y_up); + + assert_eq!(Arc::try_unwrap(x).map(|x| *x), Ok("hello")); + } +} + +#[test] +fn test_into_from_weak_raw_unsized() { + use std::fmt::Display; + use std::string::ToString; + + let arc: Arc = Arc::from("foo"); + let weak: Weak = Arc::downgrade(&arc); + + let ptr = Weak::into_raw(weak.clone()); + let weak2 = unsafe { Weak::from_raw(ptr) }; + + assert_eq!(unsafe { &*ptr }, "foo"); + assert!(weak.ptr_eq(&weak2)); + + let arc: Arc = Arc::new(123); + let weak: Weak = Arc::downgrade(&arc); + + let ptr = Weak::into_raw(weak.clone()); + let weak2 = unsafe { Weak::from_raw(ptr) }; + + assert_eq!(unsafe { &*ptr }.to_string(), "123"); + assert!(weak.ptr_eq(&weak2)); +} + #[test] fn test_cowarc_clone_make_mut() { let mut cow0 = Arc::new(75); @@ -294,7 +335,7 @@ fn test_weak_count() { #[test] fn show_arc() { let a = Arc::new(5); - assert_eq!(format!("{:?}", a), "5"); + assert_eq!(format!("{a:?}"), "5"); } // Make sure deriving works with Arc @@ -306,12 +347,29 @@ struct Foo { #[test] fn test_unsized() { let x: Arc<[i32]> = Arc::new([1, 2, 3]); - assert_eq!(format!("{:?}", x), "[1, 2, 3]"); + assert_eq!(format!("{x:?}"), "[1, 2, 3]"); let y = Arc::downgrade(&x.clone()); drop(x); assert!(y.upgrade().is_none()); } +#[test] +fn test_maybe_thin_unsized() { + // If/when custom thin DSTs exist, this test should be updated to use one + use std::ffi::{CStr, CString}; + + let x: Arc = Arc::from(CString::new("swordfish").unwrap().into_boxed_c_str()); + assert_eq!(format!("{x:?}"), "\"swordfish\""); + let y: Weak = Arc::downgrade(&x); + drop(x); + + // At this point, the weak points to a dropped DST + assert!(y.upgrade().is_none()); + // But we still need to be able to get the alloc layout to drop. + // CStr has no drop glue, but custom DSTs might, and need to work. + drop(y); +} + #[test] fn test_from_owned() { let foo = 123; @@ -337,12 +395,13 @@ fn test_ptr_eq() { #[test] #[cfg_attr(target_os = "emscripten", ignore)] -#[cfg_attr(miri, ignore)] // Miri does not support threads fn test_weak_count_locked() { let mut a = Arc::new(atomic::AtomicBool::new(false)); let a2 = a.clone(); let t = thread::spawn(move || { - for _i in 0..1000000 { + // Miri is too slow + let count = if cfg!(miri) { 1000 } else { 1000000 }; + for _i in 0..count { Arc::get_mut(&mut a); } a.store(true, SeqCst); @@ -351,6 +410,8 @@ fn test_weak_count_locked() { while !a2.load(SeqCst) { let n = Arc::weak_count(&a2); assert!(n < 2, "bad weak count: {}", n); + #[cfg(miri)] // Miri's scheduler does not guarantee liveness, and thus needs this hint. + std::hint::spin_loop(); } t.join().unwrap(); } @@ -406,7 +467,7 @@ fn test_clone_from_slice_panic() { #[test] fn test_from_box() { - let b: Box = box 123; + let b: Box = Box::new(123); let r: Arc = Arc::from(b); assert_eq!(*r, 123); @@ -435,7 +496,7 @@ fn test_from_box_trait() { use std::fmt::Display; use std::string::ToString; - let b: Box = box 123; + let b: Box = Box::new(123); let r: Arc = Arc::from(b); assert_eq!(r.to_string(), "123"); @@ -445,10 +506,10 @@ fn test_from_box_trait() { fn test_from_box_trait_zero_sized() { use std::fmt::Debug; - let b: Box = box (); + let b: Box = Box::new(()); let r: Arc = Arc::from(b); - assert_eq!(format!("{:?}", r), "()"); + assert_eq!(format!("{r:?}"), "()"); } #[test] @@ -463,14 +524,14 @@ fn test_from_vec() { fn test_downcast() { use std::any::Any; - let r1: Arc = Arc::new(i32::max_value()); + let r1: Arc = Arc::new(i32::MAX); let r2: Arc = Arc::new("abc"); assert!(r1.clone().downcast::().is_err()); let r1i32 = r1.downcast::(); assert!(r1i32.is_ok()); - assert_eq!(r1i32.unwrap(), Arc::new(i32::max_value())); + assert_eq!(r1i32.unwrap(), Arc::new(i32::MAX)); assert!(r2.clone().downcast::().is_err()); @@ -490,3 +551,89 @@ fn test_array_from_slice() { let a: Result, _> = r.clone().try_into(); assert!(a.is_err()); } + +#[test] +fn test_arc_cyclic_with_zero_refs() { + struct ZeroRefs { + inner: Weak, + } + let zero_refs = Arc::new_cyclic(|inner| { + assert_eq!(inner.strong_count(), 0); + assert!(inner.upgrade().is_none()); + ZeroRefs { inner: Weak::new() } + }); + + assert_eq!(Arc::strong_count(&zero_refs), 1); + assert_eq!(Arc::weak_count(&zero_refs), 0); + assert_eq!(zero_refs.inner.strong_count(), 0); + assert_eq!(zero_refs.inner.weak_count(), 0); +} + +#[test] +fn test_arc_new_cyclic_one_ref() { + struct OneRef { + inner: Weak, + } + let one_ref = Arc::new_cyclic(|inner| { + assert_eq!(inner.strong_count(), 0); + assert!(inner.upgrade().is_none()); + OneRef { inner: inner.clone() } + }); + + assert_eq!(Arc::strong_count(&one_ref), 1); + assert_eq!(Arc::weak_count(&one_ref), 1); + + let one_ref2 = Weak::upgrade(&one_ref.inner).unwrap(); + assert!(Arc::ptr_eq(&one_ref, &one_ref2)); + + assert_eq!(Arc::strong_count(&one_ref), 2); + assert_eq!(Arc::weak_count(&one_ref), 1); +} + +#[test] +fn test_arc_cyclic_two_refs() { + struct TwoRefs { + inner1: Weak, + inner2: Weak, + } + let two_refs = Arc::new_cyclic(|inner| { + assert_eq!(inner.strong_count(), 0); + assert!(inner.upgrade().is_none()); + + let inner1 = inner.clone(); + let inner2 = inner1.clone(); + + TwoRefs { inner1, inner2 } + }); + + assert_eq!(Arc::strong_count(&two_refs), 1); + assert_eq!(Arc::weak_count(&two_refs), 2); + + let two_refs1 = Weak::upgrade(&two_refs.inner1).unwrap(); + assert!(Arc::ptr_eq(&two_refs, &two_refs1)); + + let two_refs2 = Weak::upgrade(&two_refs.inner2).unwrap(); + assert!(Arc::ptr_eq(&two_refs, &two_refs2)); + + assert_eq!(Arc::strong_count(&two_refs), 3); + assert_eq!(Arc::weak_count(&two_refs), 2); +} + +/// Test for Arc::drop bug (https://github.com/rust-lang/rust/issues/55005) +#[test] +#[cfg(miri)] // relies on Stacked Borrows in Miri +fn arc_drop_dereferenceable_race() { + // The bug seems to take up to 700 iterations to reproduce with most seeds (tested 0-9). + for _ in 0..750 { + let arc_1 = Arc::new(()); + let arc_2 = arc_1.clone(); + let thread = thread::spawn(|| drop(arc_2)); + // Spin a bit; makes the race more likely to appear + let mut i = 0; + while i < 256 { + i += 1; + } + drop(arc_1); + thread.join().unwrap(); + } +} diff --git a/crux-mir/lib/alloc/src/task.rs b/crux-mir/lib/alloc/src/task.rs new file mode 100644 index 000000000..9d8e309a9 --- /dev/null +++ b/crux-mir/lib/alloc/src/task.rs @@ -0,0 +1,153 @@ +#![stable(feature = "wake_trait", since = "1.51.0")] + +//! Types and Traits for working with asynchronous tasks. +//! +//! **Note**: This module is only available on platforms that support atomic +//! loads and stores of pointers. This may be detected at compile time using +//! `#[cfg(target_has_atomic = "ptr")]`. + +use core::mem::ManuallyDrop; +use core::task::{RawWaker, RawWakerVTable, Waker}; + +use crate::sync::Arc; + +/// The implementation of waking a task on an executor. +/// +/// This trait can be used to create a [`Waker`]. An executor can define an +/// implementation of this trait, and use that to construct a Waker to pass +/// to the tasks that are executed on that executor. +/// +/// This trait is a memory-safe and ergonomic alternative to constructing a +/// [`RawWaker`]. It supports the common executor design in which the data used +/// to wake up a task is stored in an [`Arc`]. Some executors (especially +/// those for embedded systems) cannot use this API, which is why [`RawWaker`] +/// exists as an alternative for those systems. +/// +/// [arc]: ../../std/sync/struct.Arc.html +/// +/// # Examples +/// +/// A basic `block_on` function that takes a future and runs it to completion on +/// the current thread. +/// +/// **Note:** This example trades correctness for simplicity. In order to prevent +/// deadlocks, production-grade implementations will also need to handle +/// intermediate calls to `thread::unpark` as well as nested invocations. +/// +/// ```rust +/// use std::future::Future; +/// use std::sync::Arc; +/// use std::task::{Context, Poll, Wake}; +/// use std::thread::{self, Thread}; +/// +/// /// A waker that wakes up the current thread when called. +/// struct ThreadWaker(Thread); +/// +/// impl Wake for ThreadWaker { +/// fn wake(self: Arc) { +/// self.0.unpark(); +/// } +/// } +/// +/// /// Run a future to completion on the current thread. +/// fn block_on(fut: impl Future) -> T { +/// // Pin the future so it can be polled. +/// let mut fut = Box::pin(fut); +/// +/// // Create a new context to be passed to the future. +/// let t = thread::current(); +/// let waker = Arc::new(ThreadWaker(t)).into(); +/// let mut cx = Context::from_waker(&waker); +/// +/// // Run the future to completion. +/// loop { +/// match fut.as_mut().poll(&mut cx) { +/// Poll::Ready(res) => return res, +/// Poll::Pending => thread::park(), +/// } +/// } +/// } +/// +/// block_on(async { +/// println!("Hi from inside a future!"); +/// }); +/// ``` +#[stable(feature = "wake_trait", since = "1.51.0")] +pub trait Wake { + /// Wake this task. + #[stable(feature = "wake_trait", since = "1.51.0")] + fn wake(self: Arc); + + /// Wake this task without consuming the waker. + /// + /// If an executor supports a cheaper way to wake without consuming the + /// waker, it should override this method. By default, it clones the + /// [`Arc`] and calls [`wake`] on the clone. + /// + /// [`wake`]: Wake::wake + #[stable(feature = "wake_trait", since = "1.51.0")] + fn wake_by_ref(self: &Arc) { + self.clone().wake(); + } +} + +#[stable(feature = "wake_trait", since = "1.51.0")] +impl From> for Waker { + /// Use a `Wake`-able type as a `Waker`. + /// + /// No heap allocations or atomic operations are used for this conversion. + fn from(waker: Arc) -> Waker { + // SAFETY: This is safe because raw_waker safely constructs + // a RawWaker from Arc. + unsafe { Waker::from_raw(raw_waker(waker)) } + } +} + +#[stable(feature = "wake_trait", since = "1.51.0")] +impl From> for RawWaker { + /// Use a `Wake`-able type as a `RawWaker`. + /// + /// No heap allocations or atomic operations are used for this conversion. + fn from(waker: Arc) -> RawWaker { + raw_waker(waker) + } +} + +// NB: This private function for constructing a RawWaker is used, rather than +// inlining this into the `From> for RawWaker` impl, to ensure that +// the safety of `From> for Waker` does not depend on the correct +// trait dispatch - instead both impls call this function directly and +// explicitly. +#[inline(always)] +fn raw_waker(waker: Arc) -> RawWaker { + // Increment the reference count of the arc to clone it. + unsafe fn clone_waker(waker: *const ()) -> RawWaker { + unsafe { Arc::increment_strong_count(waker as *const W) }; + RawWaker::new( + waker as *const (), + &RawWakerVTable::new(clone_waker::, wake::, wake_by_ref::, drop_waker::), + ) + } + + // Wake by value, moving the Arc into the Wake::wake function + unsafe fn wake(waker: *const ()) { + let waker = unsafe { Arc::from_raw(waker as *const W) }; + ::wake(waker); + } + + // Wake by reference, wrap the waker in ManuallyDrop to avoid dropping it + unsafe fn wake_by_ref(waker: *const ()) { + let waker = unsafe { ManuallyDrop::new(Arc::from_raw(waker as *const W)) }; + ::wake_by_ref(&waker); + } + + // Decrement the reference count of the Arc on drop + unsafe fn drop_waker(waker: *const ()) { + unsafe { Arc::decrement_strong_count(waker as *const W) }; + } + + RawWaker::new( + Arc::into_raw(waker) as *const (), + &RawWakerVTable::new(clone_waker::, wake::, wake_by_ref::, drop_waker::), + ) +} diff --git a/crux-mir/lib/alloc/src/testing/crash_test.rs b/crux-mir/lib/alloc/src/testing/crash_test.rs new file mode 100644 index 000000000..bcf5f5f72 --- /dev/null +++ b/crux-mir/lib/alloc/src/testing/crash_test.rs @@ -0,0 +1,119 @@ +// We avoid relying on anything else in the crate, apart from the `Debug` trait. +use crate::fmt::Debug; +use std::cmp::Ordering; +use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; + +/// A blueprint for crash test dummy instances that monitor particular events. +/// Some instances may be configured to panic at some point. +/// Events are `clone`, `drop` or some anonymous `query`. +/// +/// Crash test dummies are identified and ordered by an id, so they can be used +/// as keys in a BTreeMap. +#[derive(Debug)] +pub struct CrashTestDummy { + pub id: usize, + cloned: AtomicUsize, + dropped: AtomicUsize, + queried: AtomicUsize, +} + +impl CrashTestDummy { + /// Creates a crash test dummy design. The `id` determines order and equality of instances. + pub fn new(id: usize) -> CrashTestDummy { + CrashTestDummy { + id, + cloned: AtomicUsize::new(0), + dropped: AtomicUsize::new(0), + queried: AtomicUsize::new(0), + } + } + + /// Creates an instance of a crash test dummy that records what events it experiences + /// and optionally panics. + pub fn spawn(&self, panic: Panic) -> Instance<'_> { + Instance { origin: self, panic } + } + + /// Returns how many times instances of the dummy have been cloned. + pub fn cloned(&self) -> usize { + self.cloned.load(SeqCst) + } + + /// Returns how many times instances of the dummy have been dropped. + pub fn dropped(&self) -> usize { + self.dropped.load(SeqCst) + } + + /// Returns how many times instances of the dummy have had their `query` member invoked. + pub fn queried(&self) -> usize { + self.queried.load(SeqCst) + } +} + +#[derive(Debug)] +pub struct Instance<'a> { + origin: &'a CrashTestDummy, + panic: Panic, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum Panic { + Never, + InClone, + InDrop, + InQuery, +} + +impl Instance<'_> { + pub fn id(&self) -> usize { + self.origin.id + } + + /// Some anonymous query, the result of which is already given. + pub fn query(&self, result: R) -> R { + self.origin.queried.fetch_add(1, SeqCst); + if self.panic == Panic::InQuery { + panic!("panic in `query`"); + } + result + } +} + +impl Clone for Instance<'_> { + fn clone(&self) -> Self { + self.origin.cloned.fetch_add(1, SeqCst); + if self.panic == Panic::InClone { + panic!("panic in `clone`"); + } + Self { origin: self.origin, panic: Panic::Never } + } +} + +impl Drop for Instance<'_> { + fn drop(&mut self) { + self.origin.dropped.fetch_add(1, SeqCst); + if self.panic == Panic::InDrop { + panic!("panic in `drop`"); + } + } +} + +impl PartialOrd for Instance<'_> { + fn partial_cmp(&self, other: &Self) -> Option { + self.id().partial_cmp(&other.id()) + } +} + +impl Ord for Instance<'_> { + fn cmp(&self, other: &Self) -> Ordering { + self.id().cmp(&other.id()) + } +} + +impl PartialEq for Instance<'_> { + fn eq(&self, other: &Self) -> bool { + self.id().eq(&other.id()) + } +} + +impl Eq for Instance<'_> {} diff --git a/crux-mir/lib/alloc/src/testing/mod.rs b/crux-mir/lib/alloc/src/testing/mod.rs new file mode 100644 index 000000000..7a094f8a5 --- /dev/null +++ b/crux-mir/lib/alloc/src/testing/mod.rs @@ -0,0 +1,3 @@ +pub mod crash_test; +pub mod ord_chaos; +pub mod rng; diff --git a/crux-mir/lib/alloc/src/testing/ord_chaos.rs b/crux-mir/lib/alloc/src/testing/ord_chaos.rs new file mode 100644 index 000000000..96ce7c157 --- /dev/null +++ b/crux-mir/lib/alloc/src/testing/ord_chaos.rs @@ -0,0 +1,81 @@ +use std::cell::Cell; +use std::cmp::Ordering::{self, *}; +use std::ptr; + +// Minimal type with an `Ord` implementation violating transitivity. +#[derive(Debug)] +pub enum Cyclic3 { + A, + B, + C, +} +use Cyclic3::*; + +impl PartialOrd for Cyclic3 { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Cyclic3 { + fn cmp(&self, other: &Self) -> Ordering { + match (self, other) { + (A, A) | (B, B) | (C, C) => Equal, + (A, B) | (B, C) | (C, A) => Less, + (A, C) | (B, A) | (C, B) => Greater, + } + } +} + +impl PartialEq for Cyclic3 { + fn eq(&self, other: &Self) -> bool { + self.cmp(&other) == Equal + } +} + +impl Eq for Cyclic3 {} + +// Controls the ordering of values wrapped by `Governed`. +#[derive(Debug)] +pub struct Governor { + flipped: Cell, +} + +impl Governor { + pub fn new() -> Self { + Governor { flipped: Cell::new(false) } + } + + pub fn flip(&self) { + self.flipped.set(!self.flipped.get()); + } +} + +// Type with an `Ord` implementation that forms a total order at any moment +// (assuming that `T` respects total order), but can suddenly be made to invert +// that total order. +#[derive(Debug)] +pub struct Governed<'a, T>(pub T, pub &'a Governor); + +impl PartialOrd for Governed<'_, T> { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Governed<'_, T> { + fn cmp(&self, other: &Self) -> Ordering { + assert!(ptr::eq(self.1, other.1)); + let ord = self.0.cmp(&other.0); + if self.1.flipped.get() { ord.reverse() } else { ord } + } +} + +impl PartialEq for Governed<'_, T> { + fn eq(&self, other: &Self) -> bool { + assert!(ptr::eq(self.1, other.1)); + self.0.eq(&other.0) + } +} + +impl Eq for Governed<'_, T> {} diff --git a/crux-mir/lib/alloc/src/testing/rng.rs b/crux-mir/lib/alloc/src/testing/rng.rs new file mode 100644 index 000000000..ecf543bee --- /dev/null +++ b/crux-mir/lib/alloc/src/testing/rng.rs @@ -0,0 +1,28 @@ +/// XorShiftRng +pub struct DeterministicRng { + count: usize, + x: u32, + y: u32, + z: u32, + w: u32, +} + +impl DeterministicRng { + pub fn new() -> Self { + DeterministicRng { count: 0, x: 0x193a6754, y: 0xa8a7d469, z: 0x97830e05, w: 0x113ba7bb } + } + + /// Guarantees that each returned number is unique. + pub fn next(&mut self) -> u32 { + self.count += 1; + assert!(self.count <= 70029); + let x = self.x; + let t = x ^ (x << 11); + self.x = self.y; + self.y = self.z; + self.z = self.w; + let w_ = self.w; + self.w = w_ ^ (w_ >> 19) ^ (t ^ (t >> 8)); + self.w + } +} diff --git a/crux-mir/lib/alloc/src/tests.rs b/crux-mir/lib/alloc/src/tests.rs index 1b6e0bb29..299ed156a 100644 --- a/crux-mir/lib/alloc/src/tests.rs +++ b/crux-mir/lib/alloc/src/tests.rs @@ -3,8 +3,6 @@ use core::any::Any; use core::clone::Clone; use core::convert::TryInto; -use core::f64; -use core::i64; use core::ops::Deref; use core::result::Result::{Err, Ok}; @@ -49,19 +47,19 @@ fn any_move() { fn test_show() { let a = Box::new(8) as Box; let b = Box::new(Test) as Box; - let a_str = format!("{:?}", a); - let b_str = format!("{:?}", b); - assert_eq!(a_str, "Any"); - assert_eq!(b_str, "Any"); + let a_str = format!("{a:?}"); + let b_str = format!("{b:?}"); + assert_eq!(a_str, "Any { .. }"); + assert_eq!(b_str, "Any { .. }"); static EIGHT: usize = 8; static TEST: Test = Test; let a = &EIGHT as &dyn Any; let b = &TEST as &dyn Any; - let s = format!("{:?}", a); - assert_eq!(s, "Any"); - let s = format!("{:?}", b); - assert_eq!(s, "Any"); + let s = format!("{a:?}"); + assert_eq!(s, "Any { .. }"); + let s = format!("{b:?}"); + assert_eq!(s, "Any { .. }"); } #[test] diff --git a/crux-mir/lib/alloc/src/vec.rs b/crux-mir/lib/alloc/src/vec.rs deleted file mode 100644 index c280ab03e..000000000 --- a/crux-mir/lib/alloc/src/vec.rs +++ /dev/null @@ -1,2966 +0,0 @@ -//! A contiguous growable array type with heap-allocated contents, written -//! `Vec`. -//! -//! Vectors have `O(1)` indexing, amortized `O(1)` push (to the end) and -//! `O(1)` pop (from the end). -//! -//! Vectors ensure they never allocate more than `isize::MAX` bytes. -//! -//! # Examples -//! -//! You can explicitly create a [`Vec`] with [`new`]: -//! -//! ``` -//! let v: Vec = Vec::new(); -//! ``` -//! -//! ...or by using the [`vec!`] macro: -//! -//! ``` -//! let v: Vec = vec![]; -//! -//! let v = vec![1, 2, 3, 4, 5]; -//! -//! let v = vec![0; 10]; // ten zeroes -//! ``` -//! -//! You can [`push`] values onto the end of a vector (which will grow the vector -//! as needed): -//! -//! ``` -//! let mut v = vec![1, 2]; -//! -//! v.push(3); -//! ``` -//! -//! Popping values works in much the same way: -//! -//! ``` -//! let mut v = vec![1, 2]; -//! -//! let two = v.pop(); -//! ``` -//! -//! Vectors also support indexing (through the [`Index`] and [`IndexMut`] traits): -//! -//! ``` -//! let mut v = vec![1, 2, 3]; -//! let three = v[2]; -//! v[1] = v[1] + 5; -//! ``` -//! -//! [`Vec`]: ../../std/vec/struct.Vec.html -//! [`new`]: ../../std/vec/struct.Vec.html#method.new -//! [`push`]: ../../std/vec/struct.Vec.html#method.push -//! [`Index`]: ../../std/ops/trait.Index.html -//! [`IndexMut`]: ../../std/ops/trait.IndexMut.html -//! [`vec!`]: ../../std/macro.vec.html - -#![stable(feature = "rust1", since = "1.0.0")] - -use core::array::LengthAtMost32; -use core::cmp::{self, Ordering}; -use core::fmt; -use core::hash::{self, Hash}; -use core::intrinsics::{arith_offset, assume}; -use core::iter::{FromIterator, FusedIterator, TrustedLen}; -use core::marker::PhantomData; -use core::mem; -use core::ops::Bound::{Excluded, Included, Unbounded}; -use core::ops::{self, Index, IndexMut, RangeBounds}; -use core::ptr::{self, NonNull}; -use core::slice::{self, SliceIndex}; - -use crate::borrow::{Cow, ToOwned}; -use crate::boxed::Box; -use crate::collections::TryReserveError; -use crate::raw_vec::RawVec; - -/// A contiguous growable array type, written `Vec` but pronounced 'vector'. -/// -/// # Examples -/// -/// ``` -/// let mut vec = Vec::new(); -/// vec.push(1); -/// vec.push(2); -/// -/// assert_eq!(vec.len(), 2); -/// assert_eq!(vec[0], 1); -/// -/// assert_eq!(vec.pop(), Some(2)); -/// assert_eq!(vec.len(), 1); -/// -/// vec[0] = 7; -/// assert_eq!(vec[0], 7); -/// -/// vec.extend([1, 2, 3].iter().copied()); -/// -/// for x in &vec { -/// println!("{}", x); -/// } -/// assert_eq!(vec, [7, 1, 2, 3]); -/// ``` -/// -/// The [`vec!`] macro is provided to make initialization more convenient: -/// -/// ``` -/// let mut vec = vec![1, 2, 3]; -/// vec.push(4); -/// assert_eq!(vec, [1, 2, 3, 4]); -/// ``` -/// -/// It can also initialize each element of a `Vec` with a given value. -/// This may be more efficient than performing allocation and initialization -/// in separate steps, especially when initializing a vector of zeros: -/// -/// ``` -/// let vec = vec![0; 5]; -/// assert_eq!(vec, [0, 0, 0, 0, 0]); -/// -/// // The following is equivalent, but potentially slower: -/// let mut vec1 = Vec::with_capacity(5); -/// vec1.resize(5, 0); -/// ``` -/// -/// Use a `Vec` as an efficient stack: -/// -/// ``` -/// let mut stack = Vec::new(); -/// -/// stack.push(1); -/// stack.push(2); -/// stack.push(3); -/// -/// while let Some(top) = stack.pop() { -/// // Prints 3, 2, 1 -/// println!("{}", top); -/// } -/// ``` -/// -/// # Indexing -/// -/// The `Vec` type allows to access values by index, because it implements the -/// [`Index`] trait. An example will be more explicit: -/// -/// ``` -/// let v = vec![0, 2, 4, 6]; -/// println!("{}", v[1]); // it will display '2' -/// ``` -/// -/// However be careful: if you try to access an index which isn't in the `Vec`, -/// your software will panic! You cannot do this: -/// -/// ```should_panic -/// let v = vec![0, 2, 4, 6]; -/// println!("{}", v[6]); // it will panic! -/// ``` -/// -/// Use [`get`] and [`get_mut`] if you want to check whether the index is in -/// the `Vec`. -/// -/// # Slicing -/// -/// A `Vec` can be mutable. Slices, on the other hand, are read-only objects. -/// To get a slice, use `&`. Example: -/// -/// ``` -/// fn read_slice(slice: &[usize]) { -/// // ... -/// } -/// -/// let v = vec![0, 1]; -/// read_slice(&v); -/// -/// // ... and that's all! -/// // you can also do it like this: -/// let x : &[usize] = &v; -/// ``` -/// -/// In Rust, it's more common to pass slices as arguments rather than vectors -/// when you just want to provide read access. The same goes for [`String`] and -/// [`&str`]. -/// -/// # Capacity and reallocation -/// -/// The capacity of a vector is the amount of space allocated for any future -/// elements that will be added onto the vector. This is not to be confused with -/// the *length* of a vector, which specifies the number of actual elements -/// within the vector. If a vector's length exceeds its capacity, its capacity -/// will automatically be increased, but its elements will have to be -/// reallocated. -/// -/// For example, a vector with capacity 10 and length 0 would be an empty vector -/// with space for 10 more elements. Pushing 10 or fewer elements onto the -/// vector will not change its capacity or cause reallocation to occur. However, -/// if the vector's length is increased to 11, it will have to reallocate, which -/// can be slow. For this reason, it is recommended to use [`Vec::with_capacity`] -/// whenever possible to specify how big the vector is expected to get. -/// -/// # Guarantees -/// -/// Due to its incredibly fundamental nature, `Vec` makes a lot of guarantees -/// about its design. This ensures that it's as low-overhead as possible in -/// the general case, and can be correctly manipulated in primitive ways -/// by unsafe code. Note that these guarantees refer to an unqualified `Vec`. -/// If additional type parameters are added (e.g., to support custom allocators), -/// overriding their defaults may change the behavior. -/// -/// Most fundamentally, `Vec` is and always will be a (pointer, capacity, length) -/// triplet. No more, no less. The order of these fields is completely -/// unspecified, and you should use the appropriate methods to modify these. -/// The pointer will never be null, so this type is null-pointer-optimized. -/// -/// However, the pointer may not actually point to allocated memory. In particular, -/// if you construct a `Vec` with capacity 0 via [`Vec::new`], [`vec![]`][`vec!`], -/// [`Vec::with_capacity(0)`][`Vec::with_capacity`], or by calling [`shrink_to_fit`] -/// on an empty Vec, it will not allocate memory. Similarly, if you store zero-sized -/// types inside a `Vec`, it will not allocate space for them. *Note that in this case -/// the `Vec` may not report a [`capacity`] of 0*. `Vec` will allocate if and only -/// if [`mem::size_of::`]`() * capacity() > 0`. In general, `Vec`'s allocation -/// details are very subtle — if you intend to allocate memory using a `Vec` -/// and use it for something else (either to pass to unsafe code, or to build your -/// own memory-backed collection), be sure to deallocate this memory by using -/// `from_raw_parts` to recover the `Vec` and then dropping it. -/// -/// If a `Vec` *has* allocated memory, then the memory it points to is on the heap -/// (as defined by the allocator Rust is configured to use by default), and its -/// pointer points to [`len`] initialized, contiguous elements in order (what -/// you would see if you coerced it to a slice), followed by [`capacity`]` - -/// `[`len`] logically uninitialized, contiguous elements. -/// -/// `Vec` will never perform a "small optimization" where elements are actually -/// stored on the stack for two reasons: -/// -/// * It would make it more difficult for unsafe code to correctly manipulate -/// a `Vec`. The contents of a `Vec` wouldn't have a stable address if it were -/// only moved, and it would be more difficult to determine if a `Vec` had -/// actually allocated memory. -/// -/// * It would penalize the general case, incurring an additional branch -/// on every access. -/// -/// `Vec` will never automatically shrink itself, even if completely empty. This -/// ensures no unnecessary allocations or deallocations occur. Emptying a `Vec` -/// and then filling it back up to the same [`len`] should incur no calls to -/// the allocator. If you wish to free up unused memory, use -/// [`shrink_to_fit`]. -/// -/// [`push`] and [`insert`] will never (re)allocate if the reported capacity is -/// sufficient. [`push`] and [`insert`] *will* (re)allocate if -/// [`len`]` == `[`capacity`]. That is, the reported capacity is completely -/// accurate, and can be relied on. It can even be used to manually free the memory -/// allocated by a `Vec` if desired. Bulk insertion methods *may* reallocate, even -/// when not necessary. -/// -/// `Vec` does not guarantee any particular growth strategy when reallocating -/// when full, nor when [`reserve`] is called. The current strategy is basic -/// and it may prove desirable to use a non-constant growth factor. Whatever -/// strategy is used will of course guarantee `O(1)` amortized [`push`]. -/// -/// `vec![x; n]`, `vec![a, b, c, d]`, and -/// [`Vec::with_capacity(n)`][`Vec::with_capacity`], will all produce a `Vec` -/// with exactly the requested capacity. If [`len`]` == `[`capacity`], -/// (as is the case for the [`vec!`] macro), then a `Vec` can be converted to -/// and from a [`Box<[T]>`][owned slice] without reallocating or moving the elements. -/// -/// `Vec` will not specifically overwrite any data that is removed from it, -/// but also won't specifically preserve it. Its uninitialized memory is -/// scratch space that it may use however it wants. It will generally just do -/// whatever is most efficient or otherwise easy to implement. Do not rely on -/// removed data to be erased for security purposes. Even if you drop a `Vec`, its -/// buffer may simply be reused by another `Vec`. Even if you zero a `Vec`'s memory -/// first, that may not actually happen because the optimizer does not consider -/// this a side-effect that must be preserved. There is one case which we will -/// not break, however: using `unsafe` code to write to the excess capacity, -/// and then increasing the length to match, is always valid. -/// -/// `Vec` does not currently guarantee the order in which elements are dropped. -/// The order has changed in the past and may change again. -/// -/// [`vec!`]: ../../std/macro.vec.html -/// [`get`]: ../../std/vec/struct.Vec.html#method.get -/// [`get_mut`]: ../../std/vec/struct.Vec.html#method.get_mut -/// [`Index`]: ../../std/ops/trait.Index.html -/// [`String`]: ../../std/string/struct.String.html -/// [`&str`]: ../../std/primitive.str.html -/// [`Vec::with_capacity`]: ../../std/vec/struct.Vec.html#method.with_capacity -/// [`Vec::new`]: ../../std/vec/struct.Vec.html#method.new -/// [`shrink_to_fit`]: ../../std/vec/struct.Vec.html#method.shrink_to_fit -/// [`capacity`]: ../../std/vec/struct.Vec.html#method.capacity -/// [`mem::size_of::`]: ../../std/mem/fn.size_of.html -/// [`len`]: ../../std/vec/struct.Vec.html#method.len -/// [`push`]: ../../std/vec/struct.Vec.html#method.push -/// [`insert`]: ../../std/vec/struct.Vec.html#method.insert -/// [`reserve`]: ../../std/vec/struct.Vec.html#method.reserve -/// [owned slice]: ../../std/boxed/struct.Box.html -#[stable(feature = "rust1", since = "1.0.0")] -#[cfg_attr(not(test), rustc_diagnostic_item = "vec_type")] -pub struct Vec { - buf: RawVec, - len: usize, -} - -//////////////////////////////////////////////////////////////////////////////// -// Inherent methods -//////////////////////////////////////////////////////////////////////////////// - -impl Vec { - /// Constructs a new, empty `Vec`. - /// - /// The vector will not allocate until elements are pushed onto it. - /// - /// # Examples - /// - /// ``` - /// # #![allow(unused_mut)] - /// let mut vec: Vec = Vec::new(); - /// ``` - #[inline] - #[rustc_const_stable(feature = "const_vec_new", since = "1.39.0")] - #[stable(feature = "rust1", since = "1.0.0")] - pub const fn new() -> Vec { - Vec { buf: RawVec::NEW, len: 0 } - } - - /// Constructs a new, empty `Vec` with the specified capacity. - /// - /// The vector will be able to hold exactly `capacity` elements without - /// reallocating. If `capacity` is 0, the vector will not allocate. - /// - /// It is important to note that although the returned vector has the - /// *capacity* specified, the vector will have a zero *length*. For an - /// explanation of the difference between length and capacity, see - /// *[Capacity and reallocation]*. - /// - /// [Capacity and reallocation]: #capacity-and-reallocation - /// - /// # Examples - /// - /// ``` - /// let mut vec = Vec::with_capacity(10); - /// - /// // The vector contains no items, even though it has capacity for more - /// assert_eq!(vec.len(), 0); - /// - /// // These are all done without reallocating... - /// for i in 0..10 { - /// vec.push(i); - /// } - /// - /// // ...but this may make the vector reallocate - /// vec.push(11); - /// ``` - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn with_capacity(capacity: usize) -> Vec { - Vec { buf: RawVec::with_capacity(capacity), len: 0 } - } - - /// Decomposes a `Vec` into its raw components. - /// - /// Returns the raw pointer to the underlying data, the length of - /// the vector (in elements), and the allocated capacity of the - /// data (in elements). These are the same arguments in the same - /// order as the arguments to [`from_raw_parts`]. - /// - /// After calling this function, the caller is responsible for the - /// memory previously managed by the `Vec`. The only way to do - /// this is to convert the raw pointer, length, and capacity back - /// into a `Vec` with the [`from_raw_parts`] function, allowing - /// the destructor to perform the cleanup. - /// - /// [`from_raw_parts`]: #method.from_raw_parts - /// - /// # Examples - /// - /// ``` - /// #![feature(vec_into_raw_parts)] - /// let v: Vec = vec![-1, 0, 1]; - /// - /// let (ptr, len, cap) = v.into_raw_parts(); - /// - /// let rebuilt = unsafe { - /// // We can now make changes to the components, such as - /// // transmuting the raw pointer to a compatible type. - /// let ptr = ptr as *mut u32; - /// - /// Vec::from_raw_parts(ptr, len, cap) - /// }; - /// assert_eq!(rebuilt, [4294967295, 0, 1]); - /// ``` - #[unstable(feature = "vec_into_raw_parts", reason = "new API", issue = "65816")] - pub fn into_raw_parts(self) -> (*mut T, usize, usize) { - let mut me = mem::ManuallyDrop::new(self); - (me.as_mut_ptr(), me.len(), me.capacity()) - } - - /// Creates a `Vec` directly from the raw components of another vector. - /// - /// # Safety - /// - /// This is highly unsafe, due to the number of invariants that aren't - /// checked: - /// - /// * `ptr` needs to have been previously allocated via [`String`]/`Vec` - /// (at least, it's highly likely to be incorrect if it wasn't). - /// * `T` needs to have the same size and alignment as what `ptr` was allocated with. - /// (`T` having a less strict alignment is not sufficient, the alignment really - /// needs to be equal to satsify the [`dealloc`] requirement that memory must be - /// allocated and deallocated with the same layout.) - /// * `length` needs to be less than or equal to `capacity`. - /// * `capacity` needs to be the capacity that the pointer was allocated with. - /// - /// Violating these may cause problems like corrupting the allocator's - /// internal data structures. For example it is **not** safe - /// to build a `Vec` from a pointer to a C `char` array with length `size_t`. - /// It's also not safe to build one from a `Vec` and its length, because - /// the allocator cares about the alignment, and these two types have different - /// alignments. The buffer was allocated with alignment 2 (for `u16`), but after - /// turning it into a `Vec` it'll be deallocated with alignment 1. - /// - /// The ownership of `ptr` is effectively transferred to the - /// `Vec` which may then deallocate, reallocate or change the - /// contents of memory pointed to by the pointer at will. Ensure - /// that nothing else uses the pointer after calling this - /// function. - /// - /// [`String`]: ../../std/string/struct.String.html - /// [`dealloc`]: ../../alloc/alloc/trait.GlobalAlloc.html#tymethod.dealloc - /// - /// # Examples - /// - /// ``` - /// use std::ptr; - /// use std::mem; - /// - /// let v = vec![1, 2, 3]; - /// - // FIXME Update this when vec_into_raw_parts is stabilized - /// // Prevent running `v`'s destructor so we are in complete control - /// // of the allocation. - /// let mut v = mem::ManuallyDrop::new(v); - /// - /// // Pull out the various important pieces of information about `v` - /// let p = v.as_mut_ptr(); - /// let len = v.len(); - /// let cap = v.capacity(); - /// - /// unsafe { - /// // Overwrite memory with 4, 5, 6 - /// for i in 0..len as isize { - /// ptr::write(p.offset(i), 4 + i); - /// } - /// - /// // Put everything back together into a Vec - /// let rebuilt = Vec::from_raw_parts(p, len, cap); - /// assert_eq!(rebuilt, [4, 5, 6]); - /// } - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub unsafe fn from_raw_parts(ptr: *mut T, length: usize, capacity: usize) -> Vec { - Vec { buf: RawVec::from_raw_parts(ptr, capacity), len: length } - } - - /// Returns the number of elements the vector can hold without - /// reallocating. - /// - /// # Examples - /// - /// ``` - /// let vec: Vec = Vec::with_capacity(10); - /// assert_eq!(vec.capacity(), 10); - /// ``` - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn capacity(&self) -> usize { - self.buf.capacity() - } - - /// Reserves capacity for at least `additional` more elements to be inserted - /// in the given `Vec`. The collection may reserve more space to avoid - /// frequent reallocations. After calling `reserve`, capacity will be - /// greater than or equal to `self.len() + additional`. Does nothing if - /// capacity is already sufficient. - /// - /// # Panics - /// - /// Panics if the new capacity overflows `usize`. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1]; - /// vec.reserve(10); - /// assert!(vec.capacity() >= 11); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn reserve(&mut self, additional: usize) { - self.buf.reserve(self.len, additional); - } - - /// Reserves the minimum capacity for exactly `additional` more elements to - /// be inserted in the given `Vec`. After calling `reserve_exact`, - /// capacity will be greater than or equal to `self.len() + additional`. - /// Does nothing if the capacity is already sufficient. - /// - /// Note that the allocator may give the collection more space than it - /// requests. Therefore, capacity can not be relied upon to be precisely - /// minimal. Prefer `reserve` if future insertions are expected. - /// - /// # Panics - /// - /// Panics if the new capacity overflows `usize`. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1]; - /// vec.reserve_exact(10); - /// assert!(vec.capacity() >= 11); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn reserve_exact(&mut self, additional: usize) { - self.buf.reserve_exact(self.len, additional); - } - - /// Tries to reserve capacity for at least `additional` more elements to be inserted - /// in the given `Vec`. The collection may reserve more space to avoid - /// frequent reallocations. After calling `reserve`, capacity will be - /// greater than or equal to `self.len() + additional`. Does nothing if - /// capacity is already sufficient. - /// - /// # Errors - /// - /// If the capacity overflows, or the allocator reports a failure, then an error - /// is returned. - /// - /// # Examples - /// - /// ``` - /// #![feature(try_reserve)] - /// use std::collections::TryReserveError; - /// - /// fn process_data(data: &[u32]) -> Result, TryReserveError> { - /// let mut output = Vec::new(); - /// - /// // Pre-reserve the memory, exiting if we can't - /// output.try_reserve(data.len())?; - /// - /// // Now we know this can't OOM in the middle of our complex work - /// output.extend(data.iter().map(|&val| { - /// val * 2 + 5 // very complicated - /// })); - /// - /// Ok(output) - /// } - /// # process_data(&[1, 2, 3]).expect("why is the test harness OOMing on 12 bytes?"); - /// ``` - #[unstable(feature = "try_reserve", reason = "new API", issue = "48043")] - pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { - self.buf.try_reserve(self.len, additional) - } - - /// Tries to reserves the minimum capacity for exactly `additional` more elements to - /// be inserted in the given `Vec`. After calling `reserve_exact`, - /// capacity will be greater than or equal to `self.len() + additional`. - /// Does nothing if the capacity is already sufficient. - /// - /// Note that the allocator may give the collection more space than it - /// requests. Therefore, capacity can not be relied upon to be precisely - /// minimal. Prefer `reserve` if future insertions are expected. - /// - /// # Errors - /// - /// If the capacity overflows, or the allocator reports a failure, then an error - /// is returned. - /// - /// # Examples - /// - /// ``` - /// #![feature(try_reserve)] - /// use std::collections::TryReserveError; - /// - /// fn process_data(data: &[u32]) -> Result, TryReserveError> { - /// let mut output = Vec::new(); - /// - /// // Pre-reserve the memory, exiting if we can't - /// output.try_reserve(data.len())?; - /// - /// // Now we know this can't OOM in the middle of our complex work - /// output.extend(data.iter().map(|&val| { - /// val * 2 + 5 // very complicated - /// })); - /// - /// Ok(output) - /// } - /// # process_data(&[1, 2, 3]).expect("why is the test harness OOMing on 12 bytes?"); - /// ``` - #[unstable(feature = "try_reserve", reason = "new API", issue = "48043")] - pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> { - self.buf.try_reserve_exact(self.len, additional) - } - - /// Shrinks the capacity of the vector as much as possible. - /// - /// It will drop down as close as possible to the length but the allocator - /// may still inform the vector that there is space for a few more elements. - /// - /// # Examples - /// - /// ``` - /// let mut vec = Vec::with_capacity(10); - /// vec.extend([1, 2, 3].iter().cloned()); - /// assert_eq!(vec.capacity(), 10); - /// vec.shrink_to_fit(); - /// assert!(vec.capacity() >= 3); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn shrink_to_fit(&mut self) { - if self.capacity() != self.len { - self.buf.shrink_to_fit(self.len); - } - } - - /// Shrinks the capacity of the vector with a lower bound. - /// - /// The capacity will remain at least as large as both the length - /// and the supplied value. - /// - /// # Panics - /// - /// Panics if the current capacity is smaller than the supplied - /// minimum capacity. - /// - /// # Examples - /// - /// ``` - /// #![feature(shrink_to)] - /// let mut vec = Vec::with_capacity(10); - /// vec.extend([1, 2, 3].iter().cloned()); - /// assert_eq!(vec.capacity(), 10); - /// vec.shrink_to(4); - /// assert!(vec.capacity() >= 4); - /// vec.shrink_to(0); - /// assert!(vec.capacity() >= 3); - /// ``` - #[unstable(feature = "shrink_to", reason = "new API", issue = "56431")] - pub fn shrink_to(&mut self, min_capacity: usize) { - self.buf.shrink_to_fit(cmp::max(self.len, min_capacity)); - } - - /// Converts the vector into [`Box<[T]>`][owned slice]. - /// - /// Note that this will drop any excess capacity. - /// - /// [owned slice]: ../../std/boxed/struct.Box.html - /// - /// # Examples - /// - /// ``` - /// let v = vec![1, 2, 3]; - /// - /// let slice = v.into_boxed_slice(); - /// ``` - /// - /// Any excess capacity is removed: - /// - /// ``` - /// let mut vec = Vec::with_capacity(10); - /// vec.extend([1, 2, 3].iter().cloned()); - /// - /// assert_eq!(vec.capacity(), 10); - /// let slice = vec.into_boxed_slice(); - /// assert_eq!(slice.into_vec().capacity(), 3); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn into_boxed_slice(mut self) -> Box<[T]> { - unsafe { - self.shrink_to_fit(); - let buf = ptr::read(&self.buf); - mem::forget(self); - buf.into_box() - } - } - - /// Shortens the vector, keeping the first `len` elements and dropping - /// the rest. - /// - /// If `len` is greater than the vector's current length, this has no - /// effect. - /// - /// The [`drain`] method can emulate `truncate`, but causes the excess - /// elements to be returned instead of dropped. - /// - /// Note that this method has no effect on the allocated capacity - /// of the vector. - /// - /// # Examples - /// - /// Truncating a five element vector to two elements: - /// - /// ``` - /// let mut vec = vec![1, 2, 3, 4, 5]; - /// vec.truncate(2); - /// assert_eq!(vec, [1, 2]); - /// ``` - /// - /// No truncation occurs when `len` is greater than the vector's current - /// length: - /// - /// ``` - /// let mut vec = vec![1, 2, 3]; - /// vec.truncate(8); - /// assert_eq!(vec, [1, 2, 3]); - /// ``` - /// - /// Truncating when `len == 0` is equivalent to calling the [`clear`] - /// method. - /// - /// ``` - /// let mut vec = vec![1, 2, 3]; - /// vec.truncate(0); - /// assert_eq!(vec, []); - /// ``` - /// - /// [`clear`]: #method.clear - /// [`drain`]: #method.drain - #[stable(feature = "rust1", since = "1.0.0")] - pub fn truncate(&mut self, len: usize) { - // This is safe because: - // - // * the slice passed to `drop_in_place` is valid; the `len > self.len` - // case avoids creating an invalid slice, and - // * the `len` of the vector is shrunk before calling `drop_in_place`, - // such that no value will be dropped twice in case `drop_in_place` - // were to panic once (if it panics twice, the program aborts). - unsafe { - if len > self.len { - return; - } - let s = self.get_unchecked_mut(len..) as *mut _; - self.len = len; - ptr::drop_in_place(s); - } - } - - /// Extracts a slice containing the entire vector. - /// - /// Equivalent to `&s[..]`. - /// - /// # Examples - /// - /// ``` - /// use std::io::{self, Write}; - /// let buffer = vec![1, 2, 3, 5, 8]; - /// io::sink().write(buffer.as_slice()).unwrap(); - /// ``` - #[inline] - #[stable(feature = "vec_as_slice", since = "1.7.0")] - pub fn as_slice(&self) -> &[T] { - self - } - - /// Extracts a mutable slice of the entire vector. - /// - /// Equivalent to `&mut s[..]`. - /// - /// # Examples - /// - /// ``` - /// use std::io::{self, Read}; - /// let mut buffer = vec![0; 3]; - /// io::repeat(0b101).read_exact(buffer.as_mut_slice()).unwrap(); - /// ``` - #[inline] - #[stable(feature = "vec_as_slice", since = "1.7.0")] - pub fn as_mut_slice(&mut self) -> &mut [T] { - self - } - - /// Returns a raw pointer to the vector's buffer. - /// - /// The caller must ensure that the vector outlives the pointer this - /// function returns, or else it will end up pointing to garbage. - /// Modifying the vector may cause its buffer to be reallocated, - /// which would also make any pointers to it invalid. - /// - /// The caller must also ensure that the memory the pointer (non-transitively) points to - /// is never written to (except inside an `UnsafeCell`) using this pointer or any pointer - /// derived from it. If you need to mutate the contents of the slice, use [`as_mut_ptr`]. - /// - /// # Examples - /// - /// ``` - /// let x = vec![1, 2, 4]; - /// let x_ptr = x.as_ptr(); - /// - /// unsafe { - /// for i in 0..x.len() { - /// assert_eq!(*x_ptr.add(i), 1 << i); - /// } - /// } - /// ``` - /// - /// [`as_mut_ptr`]: #method.as_mut_ptr - #[stable(feature = "vec_as_ptr", since = "1.37.0")] - #[inline] - pub fn as_ptr(&self) -> *const T { - // We shadow the slice method of the same name to avoid going through - // `deref`, which creates an intermediate reference. - let ptr = self.buf.ptr(); - unsafe { - assume(!ptr.is_null()); - } - ptr - } - - /// Returns an unsafe mutable pointer to the vector's buffer. - /// - /// The caller must ensure that the vector outlives the pointer this - /// function returns, or else it will end up pointing to garbage. - /// Modifying the vector may cause its buffer to be reallocated, - /// which would also make any pointers to it invalid. - /// - /// # Examples - /// - /// ``` - /// // Allocate vector big enough for 4 elements. - /// let size = 4; - /// let mut x: Vec = Vec::with_capacity(size); - /// let x_ptr = x.as_mut_ptr(); - /// - /// // Initialize elements via raw pointer writes, then set length. - /// unsafe { - /// for i in 0..size { - /// *x_ptr.add(i) = i as i32; - /// } - /// x.set_len(size); - /// } - /// assert_eq!(&*x, &[0,1,2,3]); - /// ``` - #[stable(feature = "vec_as_ptr", since = "1.37.0")] - #[inline] - pub fn as_mut_ptr(&mut self) -> *mut T { - // We shadow the slice method of the same name to avoid going through - // `deref_mut`, which creates an intermediate reference. - let ptr = self.buf.ptr(); - unsafe { - assume(!ptr.is_null()); - } - ptr - } - - /// Forces the length of the vector to `new_len`. - /// - /// This is a low-level operation that maintains none of the normal - /// invariants of the type. Normally changing the length of a vector - /// is done using one of the safe operations instead, such as - /// [`truncate`], [`resize`], [`extend`], or [`clear`]. - /// - /// [`truncate`]: #method.truncate - /// [`resize`]: #method.resize - /// [`extend`]: ../../std/iter/trait.Extend.html#tymethod.extend - /// [`clear`]: #method.clear - /// - /// # Safety - /// - /// - `new_len` must be less than or equal to [`capacity()`]. - /// - The elements at `old_len..new_len` must be initialized. - /// - /// [`capacity()`]: #method.capacity - /// - /// # Examples - /// - /// This method can be useful for situations in which the vector - /// is serving as a buffer for other code, particularly over FFI: - /// - /// ```no_run - /// # #![allow(dead_code)] - /// # // This is just a minimal skeleton for the doc example; - /// # // don't use this as a starting point for a real library. - /// # pub struct StreamWrapper { strm: *mut std::ffi::c_void } - /// # const Z_OK: i32 = 0; - /// # extern "C" { - /// # fn deflateGetDictionary( - /// # strm: *mut std::ffi::c_void, - /// # dictionary: *mut u8, - /// # dictLength: *mut usize, - /// # ) -> i32; - /// # } - /// # impl StreamWrapper { - /// pub fn get_dictionary(&self) -> Option> { - /// // Per the FFI method's docs, "32768 bytes is always enough". - /// let mut dict = Vec::with_capacity(32_768); - /// let mut dict_length = 0; - /// // SAFETY: When `deflateGetDictionary` returns `Z_OK`, it holds that: - /// // 1. `dict_length` elements were initialized. - /// // 2. `dict_length` <= the capacity (32_768) - /// // which makes `set_len` safe to call. - /// unsafe { - /// // Make the FFI call... - /// let r = deflateGetDictionary(self.strm, dict.as_mut_ptr(), &mut dict_length); - /// if r == Z_OK { - /// // ...and update the length to what was initialized. - /// dict.set_len(dict_length); - /// Some(dict) - /// } else { - /// None - /// } - /// } - /// } - /// # } - /// ``` - /// - /// While the following example is sound, there is a memory leak since - /// the inner vectors were not freed prior to the `set_len` call: - /// - /// ``` - /// let mut vec = vec![vec![1, 0, 0], - /// vec![0, 1, 0], - /// vec![0, 0, 1]]; - /// // SAFETY: - /// // 1. `old_len..0` is empty so no elements need to be initialized. - /// // 2. `0 <= capacity` always holds whatever `capacity` is. - /// unsafe { - /// vec.set_len(0); - /// } - /// ``` - /// - /// Normally, here, one would use [`clear`] instead to correctly drop - /// the contents and thus not leak memory. - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub unsafe fn set_len(&mut self, new_len: usize) { - debug_assert!(new_len <= self.capacity()); - - self.len = new_len; - } - - /// Removes an element from the vector and returns it. - /// - /// The removed element is replaced by the last element of the vector. - /// - /// This does not preserve ordering, but is O(1). - /// - /// # Panics - /// - /// Panics if `index` is out of bounds. - /// - /// # Examples - /// - /// ``` - /// let mut v = vec!["foo", "bar", "baz", "qux"]; - /// - /// assert_eq!(v.swap_remove(1), "bar"); - /// assert_eq!(v, ["foo", "qux", "baz"]); - /// - /// assert_eq!(v.swap_remove(0), "foo"); - /// assert_eq!(v, ["baz", "qux"]); - /// ``` - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn swap_remove(&mut self, index: usize) -> T { - unsafe { - // We replace self[index] with the last element. Note that if the - // bounds check on hole succeeds there must be a last element (which - // can be self[index] itself). - let hole: *mut T = &mut self[index]; - let last = ptr::read(self.get_unchecked(self.len - 1)); - self.len -= 1; - ptr::replace(hole, last) - } - } - - /// Inserts an element at position `index` within the vector, shifting all - /// elements after it to the right. - /// - /// # Panics - /// - /// Panics if `index > len`. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1, 2, 3]; - /// vec.insert(1, 4); - /// assert_eq!(vec, [1, 4, 2, 3]); - /// vec.insert(4, 5); - /// assert_eq!(vec, [1, 4, 2, 3, 5]); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn insert(&mut self, index: usize, element: T) { - let len = self.len(); - assert!(index <= len); - - // space for the new element - if len == self.buf.capacity() { - self.reserve(1); - } - - unsafe { - // infallible - // The spot to put the new value - { - let p = self.as_mut_ptr().add(index); - // Shift everything over to make space. (Duplicating the - // `index`th element into two consecutive places.) - ptr::copy(p, p.offset(1), len - index); - // Write it in, overwriting the first copy of the `index`th - // element. - ptr::write(p, element); - } - self.set_len(len + 1); - } - } - - /// Removes and returns the element at position `index` within the vector, - /// shifting all elements after it to the left. - /// - /// # Panics - /// - /// Panics if `index` is out of bounds. - /// - /// # Examples - /// - /// ``` - /// let mut v = vec![1, 2, 3]; - /// assert_eq!(v.remove(1), 2); - /// assert_eq!(v, [1, 3]); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn remove(&mut self, index: usize) -> T { - let len = self.len(); - assert!(index < len); - unsafe { - // infallible - let ret; - { - // the place we are taking from. - let ptr = self.as_mut_ptr().add(index); - // copy it out, unsafely having a copy of the value on - // the stack and in the vector at the same time. - ret = ptr::read(ptr); - - // Shift everything down to fill in that spot. - ptr::copy(ptr.offset(1), ptr, len - index - 1); - } - self.set_len(len - 1); - ret - } - } - - /// Retains only the elements specified by the predicate. - /// - /// In other words, remove all elements `e` such that `f(&e)` returns `false`. - /// This method operates in place, visiting each element exactly once in the - /// original order, and preserves the order of the retained elements. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1, 2, 3, 4]; - /// vec.retain(|&x| x % 2 == 0); - /// assert_eq!(vec, [2, 4]); - /// ``` - /// - /// The exact order may be useful for tracking external state, like an index. - /// - /// ``` - /// let mut vec = vec![1, 2, 3, 4, 5]; - /// let keep = [false, true, true, false, true]; - /// let mut i = 0; - /// vec.retain(|_| (keep[i], i += 1).0); - /// assert_eq!(vec, [2, 3, 5]); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn retain(&mut self, mut f: F) - where - F: FnMut(&T) -> bool, - { - let len = self.len(); - let mut del = 0; - { - let v = &mut **self; - - for i in 0..len { - if !f(&v[i]) { - del += 1; - } else if del > 0 { - v.swap(i - del, i); - } - } - } - if del > 0 { - self.truncate(len - del); - } - } - - /// Removes all but the first of consecutive elements in the vector that resolve to the same - /// key. - /// - /// If the vector is sorted, this removes all duplicates. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![10, 20, 21, 30, 20]; - /// - /// vec.dedup_by_key(|i| *i / 10); - /// - /// assert_eq!(vec, [10, 20, 30, 20]); - /// ``` - #[stable(feature = "dedup_by", since = "1.16.0")] - #[inline] - pub fn dedup_by_key(&mut self, mut key: F) - where - F: FnMut(&mut T) -> K, - K: PartialEq, - { - self.dedup_by(|a, b| key(a) == key(b)) - } - - /// Removes all but the first of consecutive elements in the vector satisfying a given equality - /// relation. - /// - /// The `same_bucket` function is passed references to two elements from the vector and - /// must determine if the elements compare equal. The elements are passed in opposite order - /// from their order in the slice, so if `same_bucket(a, b)` returns `true`, `a` is removed. - /// - /// If the vector is sorted, this removes all duplicates. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec!["foo", "bar", "Bar", "baz", "bar"]; - /// - /// vec.dedup_by(|a, b| a.eq_ignore_ascii_case(b)); - /// - /// assert_eq!(vec, ["foo", "bar", "baz", "bar"]); - /// ``` - #[stable(feature = "dedup_by", since = "1.16.0")] - pub fn dedup_by(&mut self, same_bucket: F) - where - F: FnMut(&mut T, &mut T) -> bool, - { - let len = { - let (dedup, _) = self.as_mut_slice().partition_dedup_by(same_bucket); - dedup.len() - }; - self.truncate(len); - } - - /// Appends an element to the back of a collection. - /// - /// # Panics - /// - /// Panics if the number of elements in the vector overflows a `usize`. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1, 2]; - /// vec.push(3); - /// assert_eq!(vec, [1, 2, 3]); - /// ``` - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn push(&mut self, value: T) { - // This will panic or abort if we would allocate > isize::MAX bytes - // or if the length increment would overflow for zero-sized types. - if self.len == self.buf.capacity() { - self.reserve(1); - } - unsafe { - let end = self.as_mut_ptr().add(self.len); - ptr::write(end, value); - self.len += 1; - } - } - - /// Removes the last element from a vector and returns it, or [`None`] if it - /// is empty. - /// - /// [`None`]: ../../std/option/enum.Option.html#variant.None - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1, 2, 3]; - /// assert_eq!(vec.pop(), Some(3)); - /// assert_eq!(vec, [1, 2]); - /// ``` - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn pop(&mut self) -> Option { - if self.len == 0 { - None - } else { - unsafe { - self.len -= 1; - Some(ptr::read(self.get_unchecked(self.len()))) - } - } - } - - /// Moves all the elements of `other` into `Self`, leaving `other` empty. - /// - /// # Panics - /// - /// Panics if the number of elements in the vector overflows a `usize`. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1, 2, 3]; - /// let mut vec2 = vec![4, 5, 6]; - /// vec.append(&mut vec2); - /// assert_eq!(vec, [1, 2, 3, 4, 5, 6]); - /// assert_eq!(vec2, []); - /// ``` - #[inline] - #[stable(feature = "append", since = "1.4.0")] - pub fn append(&mut self, other: &mut Self) { - unsafe { - self.append_elements(other.as_slice() as _); - other.set_len(0); - } - } - - /// Appends elements to `Self` from other buffer. - #[inline] - unsafe fn append_elements(&mut self, other: *const [T]) { - let count = (*other).len(); - self.reserve(count); - let len = self.len(); - ptr::copy_nonoverlapping(other as *const T, self.as_mut_ptr().add(len), count); - self.len += count; - } - - /// Creates a draining iterator that removes the specified range in the vector - /// and yields the removed items. - /// - /// Note 1: The element range is removed even if the iterator is only - /// partially consumed or not consumed at all. - /// - /// Note 2: It is unspecified how many elements are removed from the vector - /// if the `Drain` value is leaked. - /// - /// # Panics - /// - /// Panics if the starting point is greater than the end point or if - /// the end point is greater than the length of the vector. - /// - /// # Examples - /// - /// ``` - /// let mut v = vec![1, 2, 3]; - /// let u: Vec<_> = v.drain(1..).collect(); - /// assert_eq!(v, &[1]); - /// assert_eq!(u, &[2, 3]); - /// - /// // A full range clears the vector - /// v.drain(..); - /// assert_eq!(v, &[]); - /// ``` - #[stable(feature = "drain", since = "1.6.0")] - pub fn drain(&mut self, range: R) -> Drain<'_, T> - where - R: RangeBounds, - { - // Memory safety - // - // When the Drain is first created, it shortens the length of - // the source vector to make sure no uninitialized or moved-from elements - // are accessible at all if the Drain's destructor never gets to run. - // - // Drain will ptr::read out the values to remove. - // When finished, remaining tail of the vec is copied back to cover - // the hole, and the vector length is restored to the new length. - // - let len = self.len(); - let start = match range.start_bound() { - Included(&n) => n, - Excluded(&n) => n + 1, - Unbounded => 0, - }; - let end = match range.end_bound() { - Included(&n) => n + 1, - Excluded(&n) => n, - Unbounded => len, - }; - assert!(start <= end); - assert!(end <= len); - - unsafe { - // set self.vec length's to start, to be safe in case Drain is leaked - self.set_len(start); - // Use the borrow in the IterMut to indicate borrowing behavior of the - // whole Drain iterator (like &mut T). - let range_slice = slice::from_raw_parts_mut(self.as_mut_ptr().add(start), end - start); - Drain { - tail_start: end, - tail_len: len - end, - iter: range_slice.iter(), - vec: NonNull::from(self), - } - } - } - - /// Clears the vector, removing all values. - /// - /// Note that this method has no effect on the allocated capacity - /// of the vector. - /// - /// # Examples - /// - /// ``` - /// let mut v = vec![1, 2, 3]; - /// - /// v.clear(); - /// - /// assert!(v.is_empty()); - /// ``` - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn clear(&mut self) { - self.truncate(0) - } - - /// Returns the number of elements in the vector, also referred to - /// as its 'length'. - /// - /// # Examples - /// - /// ``` - /// let a = vec![1, 2, 3]; - /// assert_eq!(a.len(), 3); - /// ``` - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn len(&self) -> usize { - self.len - } - - /// Returns `true` if the vector contains no elements. - /// - /// # Examples - /// - /// ``` - /// let mut v = Vec::new(); - /// assert!(v.is_empty()); - /// - /// v.push(1); - /// assert!(!v.is_empty()); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Splits the collection into two at the given index. - /// - /// Returns a newly allocated vector containing the elements in the range - /// `[at, len)`. After the call, the original vector will be left containing - /// the elements `[0, at)` with its previous capacity unchanged. - /// - /// # Panics - /// - /// Panics if `at > len`. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1,2,3]; - /// let vec2 = vec.split_off(1); - /// assert_eq!(vec, [1]); - /// assert_eq!(vec2, [2, 3]); - /// ``` - #[inline] - #[must_use = "use `.truncate()` if you don't need the other half"] - #[stable(feature = "split_off", since = "1.4.0")] - pub fn split_off(&mut self, at: usize) -> Self { - assert!(at <= self.len(), "`at` out of bounds"); - - let other_len = self.len - at; - let mut other = Vec::with_capacity(other_len); - - // Unsafely `set_len` and copy items to `other`. - unsafe { - self.set_len(at); - other.set_len(other_len); - - ptr::copy_nonoverlapping(self.as_ptr().add(at), other.as_mut_ptr(), other.len()); - } - other - } - - /// Resizes the `Vec` in-place so that `len` is equal to `new_len`. - /// - /// If `new_len` is greater than `len`, the `Vec` is extended by the - /// difference, with each additional slot filled with the result of - /// calling the closure `f`. The return values from `f` will end up - /// in the `Vec` in the order they have been generated. - /// - /// If `new_len` is less than `len`, the `Vec` is simply truncated. - /// - /// This method uses a closure to create new values on every push. If - /// you'd rather [`Clone`] a given value, use [`resize`]. If you want - /// to use the [`Default`] trait to generate values, you can pass - /// [`Default::default()`] as the second argument. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1, 2, 3]; - /// vec.resize_with(5, Default::default); - /// assert_eq!(vec, [1, 2, 3, 0, 0]); - /// - /// let mut vec = vec![]; - /// let mut p = 1; - /// vec.resize_with(4, || { p *= 2; p }); - /// assert_eq!(vec, [2, 4, 8, 16]); - /// ``` - /// - /// [`resize`]: #method.resize - /// [`Clone`]: ../../std/clone/trait.Clone.html - #[stable(feature = "vec_resize_with", since = "1.33.0")] - pub fn resize_with(&mut self, new_len: usize, f: F) - where - F: FnMut() -> T, - { - let len = self.len(); - if new_len > len { - self.extend_with(new_len - len, ExtendFunc(f)); - } else { - self.truncate(new_len); - } - } - - /// Consumes and leaks the `Vec`, returning a mutable reference to the contents, - /// `&'a mut [T]`. Note that the type `T` must outlive the chosen lifetime - /// `'a`. If the type has only static references, or none at all, then this - /// may be chosen to be `'static`. - /// - /// This function is similar to the `leak` function on `Box`. - /// - /// This function is mainly useful for data that lives for the remainder of - /// the program's life. Dropping the returned reference will cause a memory - /// leak. - /// - /// # Examples - /// - /// Simple usage: - /// - /// ``` - /// #![feature(vec_leak)] - /// - /// let x = vec![1, 2, 3]; - /// let static_ref: &'static mut [usize] = Vec::leak(x); - /// static_ref[0] += 1; - /// assert_eq!(static_ref, &[2, 2, 3]); - /// ``` - #[unstable(feature = "vec_leak", issue = "62195")] - #[inline] - pub fn leak<'a>(vec: Vec) -> &'a mut [T] - where - T: 'a, // Technically not needed, but kept to be explicit. - { - Box::leak(vec.into_boxed_slice()) - } -} - -impl Vec { - /// Resizes the `Vec` in-place so that `len` is equal to `new_len`. - /// - /// If `new_len` is greater than `len`, the `Vec` is extended by the - /// difference, with each additional slot filled with `value`. - /// If `new_len` is less than `len`, the `Vec` is simply truncated. - /// - /// This method requires `T` to implement [`Clone`], - /// in order to be able to clone the passed value. - /// If you need more flexibility (or want to rely on [`Default`] instead of - /// [`Clone`]), use [`resize_with`]. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec!["hello"]; - /// vec.resize(3, "world"); - /// assert_eq!(vec, ["hello", "world", "world"]); - /// - /// let mut vec = vec![1, 2, 3, 4]; - /// vec.resize(2, 0); - /// assert_eq!(vec, [1, 2]); - /// ``` - /// - /// [`Clone`]: ../../std/clone/trait.Clone.html - /// [`Default`]: ../../std/default/trait.Default.html - /// [`resize_with`]: #method.resize_with - #[stable(feature = "vec_resize", since = "1.5.0")] - pub fn resize(&mut self, new_len: usize, value: T) { - let len = self.len(); - - if new_len > len { - self.extend_with(new_len - len, ExtendElement(value)) - } else { - self.truncate(new_len); - } - } - - /// Clones and appends all elements in a slice to the `Vec`. - /// - /// Iterates over the slice `other`, clones each element, and then appends - /// it to this `Vec`. The `other` vector is traversed in-order. - /// - /// Note that this function is same as [`extend`] except that it is - /// specialized to work with slices instead. If and when Rust gets - /// specialization this function will likely be deprecated (but still - /// available). - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1]; - /// vec.extend_from_slice(&[2, 3, 4]); - /// assert_eq!(vec, [1, 2, 3, 4]); - /// ``` - /// - /// [`extend`]: #method.extend - #[stable(feature = "vec_extend_from_slice", since = "1.6.0")] - pub fn extend_from_slice(&mut self, other: &[T]) { - self.spec_extend(other.iter()) - } -} - -impl Vec { - /// Resizes the `Vec` in-place so that `len` is equal to `new_len`. - /// - /// If `new_len` is greater than `len`, the `Vec` is extended by the - /// difference, with each additional slot filled with [`Default::default()`]. - /// If `new_len` is less than `len`, the `Vec` is simply truncated. - /// - /// This method uses [`Default`] to create new values on every push. If - /// you'd rather [`Clone`] a given value, use [`resize`]. - /// - /// # Examples - /// - /// ``` - /// # #![allow(deprecated)] - /// #![feature(vec_resize_default)] - /// - /// let mut vec = vec![1, 2, 3]; - /// vec.resize_default(5); - /// assert_eq!(vec, [1, 2, 3, 0, 0]); - /// - /// let mut vec = vec![1, 2, 3, 4]; - /// vec.resize_default(2); - /// assert_eq!(vec, [1, 2]); - /// ``` - /// - /// [`resize`]: #method.resize - /// [`Default::default()`]: ../../std/default/trait.Default.html#tymethod.default - /// [`Default`]: ../../std/default/trait.Default.html - /// [`Clone`]: ../../std/clone/trait.Clone.html - #[unstable(feature = "vec_resize_default", issue = "41758")] - #[rustc_deprecated( - reason = "This is moving towards being removed in favor \ - of `.resize_with(Default::default)`. If you disagree, please comment \ - in the tracking issue.", - since = "1.33.0" - )] - pub fn resize_default(&mut self, new_len: usize) { - let len = self.len(); - - if new_len > len { - self.extend_with(new_len - len, ExtendDefault); - } else { - self.truncate(new_len); - } - } -} - -// This code generalises `extend_with_{element,default}`. -trait ExtendWith { - fn next(&mut self) -> T; - fn last(self) -> T; -} - -struct ExtendElement(T); -impl ExtendWith for ExtendElement { - fn next(&mut self) -> T { - self.0.clone() - } - fn last(self) -> T { - self.0 - } -} - -struct ExtendDefault; -impl ExtendWith for ExtendDefault { - fn next(&mut self) -> T { - Default::default() - } - fn last(self) -> T { - Default::default() - } -} - -struct ExtendFunc(F); -impl T> ExtendWith for ExtendFunc { - fn next(&mut self) -> T { - (self.0)() - } - fn last(mut self) -> T { - (self.0)() - } -} - -impl Vec { - /// Extend the vector by `n` values, using the given generator. - fn extend_with>(&mut self, n: usize, mut value: E) { - self.reserve(n); - - unsafe { - let mut ptr = self.as_mut_ptr().add(self.len()); - let mut local_len = self.len; - - // Write all elements except the last one - for _ in 1..n { - ptr::write(ptr, value.next()); - ptr = ptr.offset(1); - // Increment the length in every step in case next() panics - local_len += 1; - } - - if n > 0 { - // We can write the last element directly without cloning needlessly - ptr::write(ptr, value.last()); - local_len += 1; - } - - self.set_len(local_len); - } - } -} - -impl Vec { - /// Removes consecutive repeated elements in the vector according to the - /// [`PartialEq`] trait implementation. - /// - /// If the vector is sorted, this removes all duplicates. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1, 2, 2, 3, 2]; - /// - /// vec.dedup(); - /// - /// assert_eq!(vec, [1, 2, 3, 2]); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - #[inline] - pub fn dedup(&mut self) { - self.dedup_by(|a, b| a == b) - } -} - -impl Vec { - /// Removes the first instance of `item` from the vector if the item exists. - /// - /// # Examples - /// - /// ``` - /// # #![feature(vec_remove_item)] - /// let mut vec = vec![1, 2, 3, 1]; - /// - /// vec.remove_item(&1); - /// - /// assert_eq!(vec, vec![2, 3, 1]); - /// ``` - #[unstable(feature = "vec_remove_item", reason = "recently added", issue = "40062")] - pub fn remove_item(&mut self, item: &V) -> Option - where - T: PartialEq, - { - let pos = self.iter().position(|x| *x == *item)?; - Some(self.remove(pos)) - } -} - -//////////////////////////////////////////////////////////////////////////////// -// Internal methods and functions -//////////////////////////////////////////////////////////////////////////////// - -#[doc(hidden)] -#[stable(feature = "rust1", since = "1.0.0")] -pub fn from_elem(elem: T, n: usize) -> Vec { - ::from_elem(elem, n) -} - -// Specialization trait used for Vec::from_elem -trait SpecFromElem: Sized { - fn from_elem(elem: Self, n: usize) -> Vec; -} - -impl SpecFromElem for T { - default fn from_elem(elem: Self, n: usize) -> Vec { - let mut v = Vec::with_capacity(n); - v.extend_with(n, ExtendElement(elem)); - v - } -} - -impl SpecFromElem for u8 { - #[inline] - fn from_elem(elem: u8, n: usize) -> Vec { - if elem == 0 { - return Vec { buf: RawVec::with_capacity_zeroed(n), len: n }; - } - unsafe { - let mut v = Vec::with_capacity(n); - ptr::write_bytes(v.as_mut_ptr(), elem, n); - v.set_len(n); - v - } - } -} - -impl SpecFromElem for T { - #[inline] - fn from_elem(elem: T, n: usize) -> Vec { - if elem.is_zero() { - return Vec { buf: RawVec::with_capacity_zeroed(n), len: n }; - } - let mut v = Vec::with_capacity(n); - v.extend_with(n, ExtendElement(elem)); - v - } -} - -unsafe trait IsZero { - /// Whether this value is zero - fn is_zero(&self) -> bool; -} - -macro_rules! impl_is_zero { - ($t: ty, $is_zero: expr) => { - unsafe impl IsZero for $t { - #[inline] - fn is_zero(&self) -> bool { - $is_zero(*self) - } - } - }; -} - -impl_is_zero!(i8, |x| x == 0); -impl_is_zero!(i16, |x| x == 0); -impl_is_zero!(i32, |x| x == 0); -impl_is_zero!(i64, |x| x == 0); -impl_is_zero!(i128, |x| x == 0); -impl_is_zero!(isize, |x| x == 0); - -impl_is_zero!(u16, |x| x == 0); -impl_is_zero!(u32, |x| x == 0); -impl_is_zero!(u64, |x| x == 0); -impl_is_zero!(u128, |x| x == 0); -impl_is_zero!(usize, |x| x == 0); - -impl_is_zero!(bool, |x| x == false); -impl_is_zero!(char, |x| x == '\0'); - -impl_is_zero!(f32, |x: f32| x.to_bits() == 0); -impl_is_zero!(f64, |x: f64| x.to_bits() == 0); - -unsafe impl IsZero for *const T { - #[inline] - fn is_zero(&self) -> bool { - (*self).is_null() - } -} - -unsafe impl IsZero for *mut T { - #[inline] - fn is_zero(&self) -> bool { - (*self).is_null() - } -} - -// `Option<&T>`, `Option<&mut T>` and `Option>` are guaranteed to represent `None` as null. -// For fat pointers, the bytes that would be the pointer metadata in the `Some` variant -// are padding in the `None` variant, so ignoring them and zero-initializing instead is ok. - -unsafe impl IsZero for Option<&T> { - #[inline] - fn is_zero(&self) -> bool { - self.is_none() - } -} - -unsafe impl IsZero for Option<&mut T> { - #[inline] - fn is_zero(&self) -> bool { - self.is_none() - } -} - -unsafe impl IsZero for Option> { - #[inline] - fn is_zero(&self) -> bool { - self.is_none() - } -} - -//////////////////////////////////////////////////////////////////////////////// -// Common trait implementations for Vec -//////////////////////////////////////////////////////////////////////////////// - -#[stable(feature = "rust1", since = "1.0.0")] -impl Clone for Vec { - #[cfg(not(test))] - fn clone(&self) -> Vec { - <[T]>::to_vec(&**self) - } - - // HACK(japaric): with cfg(test) the inherent `[T]::to_vec` method, which is - // required for this method definition, is not available. Instead use the - // `slice::to_vec` function which is only available with cfg(test) - // NB see the slice::hack module in slice.rs for more information - #[cfg(test)] - fn clone(&self) -> Vec { - crate::slice::to_vec(&**self) - } - - fn clone_from(&mut self, other: &Vec) { - other.as_slice().clone_into(self); - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Hash for Vec { - #[inline] - fn hash(&self, state: &mut H) { - Hash::hash(&**self, state) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -#[rustc_on_unimplemented( - message = "vector indices are of type `usize` or ranges of `usize`", - label = "vector indices are of type `usize` or ranges of `usize`" -)] -impl> Index for Vec { - type Output = I::Output; - - #[inline] - fn index(&self, index: I) -> &Self::Output { - Index::index(&**self, index) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -#[rustc_on_unimplemented( - message = "vector indices are of type `usize` or ranges of `usize`", - label = "vector indices are of type `usize` or ranges of `usize`" -)] -impl> IndexMut for Vec { - #[inline] - fn index_mut(&mut self, index: I) -> &mut Self::Output { - IndexMut::index_mut(&mut **self, index) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ops::Deref for Vec { - type Target = [T]; - - fn deref(&self) -> &[T] { - unsafe { slice::from_raw_parts(self.as_ptr(), self.len) } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ops::DerefMut for Vec { - fn deref_mut(&mut self) -> &mut [T] { - unsafe { slice::from_raw_parts_mut(self.as_mut_ptr(), self.len) } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl FromIterator for Vec { - #[inline] - fn from_iter>(iter: I) -> Vec { - >::from_iter(iter.into_iter()) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl IntoIterator for Vec { - type Item = T; - type IntoIter = IntoIter; - - /// Creates a consuming iterator, that is, one that moves each value out of - /// the vector (from start to end). The vector cannot be used after calling - /// this. - /// - /// # Examples - /// - /// ``` - /// let v = vec!["a".to_string(), "b".to_string()]; - /// for s in v.into_iter() { - /// // s has type String, not &String - /// println!("{}", s); - /// } - /// ``` - #[inline] - fn into_iter(mut self) -> IntoIter { - unsafe { - let begin = self.as_mut_ptr(); - let end = if mem::size_of::() == 0 { - arith_offset(begin as *const i8, self.len() as isize) as *const T - } else { - begin.add(self.len()) as *const T - }; - let cap = self.buf.capacity(); - mem::forget(self); - IntoIter { - buf: NonNull::new_unchecked(begin), - phantom: PhantomData, - cap, - ptr: begin, - end, - } - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T> IntoIterator for &'a Vec { - type Item = &'a T; - type IntoIter = slice::Iter<'a, T>; - - fn into_iter(self) -> slice::Iter<'a, T> { - self.iter() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T> IntoIterator for &'a mut Vec { - type Item = &'a mut T; - type IntoIter = slice::IterMut<'a, T>; - - fn into_iter(self) -> slice::IterMut<'a, T> { - self.iter_mut() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Extend for Vec { - #[inline] - fn extend>(&mut self, iter: I) { - >::spec_extend(self, iter.into_iter()) - } -} - -// Specialization trait used for Vec::from_iter and Vec::extend -trait SpecExtend { - fn from_iter(iter: I) -> Self; - fn spec_extend(&mut self, iter: I); -} - -impl SpecExtend for Vec -where - I: Iterator, -{ - default fn from_iter(mut iterator: I) -> Self { - // Unroll the first iteration, as the vector is going to be - // expanded on this iteration in every case when the iterable is not - // empty, but the loop in extend_desugared() is not going to see the - // vector being full in the few subsequent loop iterations. - // So we get better branch prediction. - let mut vector = match iterator.next() { - None => return Vec::new(), - Some(element) => { - let (lower, _) = iterator.size_hint(); - let mut vector = Vec::with_capacity(lower.saturating_add(1)); - unsafe { - ptr::write(vector.get_unchecked_mut(0), element); - vector.set_len(1); - } - vector - } - }; - as SpecExtend>::spec_extend(&mut vector, iterator); - vector - } - - default fn spec_extend(&mut self, iter: I) { - self.extend_desugared(iter) - } -} - -impl SpecExtend for Vec -where - I: TrustedLen, -{ - default fn from_iter(iterator: I) -> Self { - let mut vector = Vec::new(); - vector.spec_extend(iterator); - vector - } - - default fn spec_extend(&mut self, iterator: I) { - // This is the case for a TrustedLen iterator. - let (low, high) = iterator.size_hint(); - if let Some(high_value) = high { - debug_assert_eq!( - low, - high_value, - "TrustedLen iterator's size hint is not exact: {:?}", - (low, high) - ); - } - if let Some(additional) = high { - self.reserve(additional); - unsafe { - let mut ptr = self.as_mut_ptr().add(self.len()); - let mut local_len = self.len; - for element in iterator { - ptr::write(ptr, element); - ptr = ptr.offset(1); - // NB can't overflow since we would have had to alloc the address space - local_len += 1; - } - self.set_len(local_len); - } - } else { - self.extend_desugared(iterator) - } - } -} - -impl SpecExtend> for Vec { - fn from_iter(iterator: IntoIter) -> Self { - // A common case is passing a vector into a function which immediately - // re-collects into a vector. We can short circuit this if the IntoIter - // has not been advanced at all. - if iterator.buf.as_ptr() as *const _ == iterator.ptr { - unsafe { - let vec = Vec::from_raw_parts(iterator.buf.as_ptr(), iterator.len(), iterator.cap); - mem::forget(iterator); - vec - } - } else { - let mut vector = Vec::new(); - vector.spec_extend(iterator); - vector - } - } - - fn spec_extend(&mut self, mut iterator: IntoIter) { - unsafe { - self.append_elements(iterator.as_slice() as _); - } - iterator.ptr = iterator.end; - } -} - -impl<'a, T: 'a, I> SpecExtend<&'a T, I> for Vec -where - I: Iterator, - T: Clone, -{ - default fn from_iter(iterator: I) -> Self { - SpecExtend::from_iter(iterator.cloned()) - } - - default fn spec_extend(&mut self, iterator: I) { - self.spec_extend(iterator.cloned()) - } -} - -impl<'a, T: 'a> SpecExtend<&'a T, slice::Iter<'a, T>> for Vec -where - T: Copy, -{ - fn spec_extend(&mut self, iterator: slice::Iter<'a, T>) { - let slice = iterator.as_slice(); - self.reserve(slice.len()); - unsafe { - let len = self.len(); - self.set_len(len + slice.len()); - self.get_unchecked_mut(len..).copy_from_slice(slice); - } - } -} - -impl Vec { - fn extend_desugared>(&mut self, mut iterator: I) { - // This is the case for a general iterator. - // - // This function should be the moral equivalent of: - // - // for item in iterator { - // self.push(item); - // } - while let Some(element) = iterator.next() { - let len = self.len(); - if len == self.capacity() { - let (lower, _) = iterator.size_hint(); - self.reserve(lower.saturating_add(1)); - } - unsafe { - ptr::write(self.get_unchecked_mut(len), element); - // NB can't overflow since we would have had to alloc the address space - self.set_len(len + 1); - } - } - } - - /// Creates a splicing iterator that replaces the specified range in the vector - /// with the given `replace_with` iterator and yields the removed items. - /// `replace_with` does not need to be the same length as `range`. - /// - /// The element range is removed even if the iterator is not consumed until the end. - /// - /// It is unspecified how many elements are removed from the vector - /// if the `Splice` value is leaked. - /// - /// The input iterator `replace_with` is only consumed when the `Splice` value is dropped. - /// - /// This is optimal if: - /// - /// * The tail (elements in the vector after `range`) is empty, - /// * or `replace_with` yields fewer elements than `range`’s length - /// * or the lower bound of its `size_hint()` is exact. - /// - /// Otherwise, a temporary vector is allocated and the tail is moved twice. - /// - /// # Panics - /// - /// Panics if the starting point is greater than the end point or if - /// the end point is greater than the length of the vector. - /// - /// # Examples - /// - /// ``` - /// let mut v = vec![1, 2, 3]; - /// let new = [7, 8]; - /// let u: Vec<_> = v.splice(..2, new.iter().cloned()).collect(); - /// assert_eq!(v, &[7, 8, 3]); - /// assert_eq!(u, &[1, 2]); - /// ``` - #[inline] - #[stable(feature = "vec_splice", since = "1.21.0")] - pub fn splice(&mut self, range: R, replace_with: I) -> Splice<'_, I::IntoIter> - where - R: RangeBounds, - I: IntoIterator, - { - Splice { drain: self.drain(range), replace_with: replace_with.into_iter() } - } - - /// Creates an iterator which uses a closure to determine if an element should be removed. - /// - /// If the closure returns true, then the element is removed and yielded. - /// If the closure returns false, the element will remain in the vector and will not be yielded - /// by the iterator. - /// - /// Using this method is equivalent to the following code: - /// - /// ``` - /// # let some_predicate = |x: &mut i32| { *x == 2 || *x == 3 || *x == 6 }; - /// # let mut vec = vec![1, 2, 3, 4, 5, 6]; - /// let mut i = 0; - /// while i != vec.len() { - /// if some_predicate(&mut vec[i]) { - /// let val = vec.remove(i); - /// // your code here - /// } else { - /// i += 1; - /// } - /// } - /// - /// # assert_eq!(vec, vec![1, 4, 5]); - /// ``` - /// - /// But `drain_filter` is easier to use. `drain_filter` is also more efficient, - /// because it can backshift the elements of the array in bulk. - /// - /// Note that `drain_filter` also lets you mutate every element in the filter closure, - /// regardless of whether you choose to keep or remove it. - /// - /// - /// # Examples - /// - /// Splitting an array into evens and odds, reusing the original allocation: - /// - /// ``` - /// #![feature(drain_filter)] - /// let mut numbers = vec![1, 2, 3, 4, 5, 6, 8, 9, 11, 13, 14, 15]; - /// - /// let evens = numbers.drain_filter(|x| *x % 2 == 0).collect::>(); - /// let odds = numbers; - /// - /// assert_eq!(evens, vec![2, 4, 6, 8, 14]); - /// assert_eq!(odds, vec![1, 3, 5, 9, 11, 13, 15]); - /// ``` - #[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")] - pub fn drain_filter(&mut self, filter: F) -> DrainFilter<'_, T, F> - where - F: FnMut(&mut T) -> bool, - { - let old_len = self.len(); - - // Guard against us getting leaked (leak amplification) - unsafe { - self.set_len(0); - } - - DrainFilter { vec: self, idx: 0, del: 0, old_len, pred: filter, panic_flag: false } - } -} - -/// Extend implementation that copies elements out of references before pushing them onto the Vec. -/// -/// This implementation is specialized for slice iterators, where it uses [`copy_from_slice`] to -/// append the entire slice at once. -/// -/// [`copy_from_slice`]: ../../std/primitive.slice.html#method.copy_from_slice -#[stable(feature = "extend_ref", since = "1.2.0")] -impl<'a, T: 'a + Copy> Extend<&'a T> for Vec { - fn extend>(&mut self, iter: I) { - self.spec_extend(iter.into_iter()) - } -} - -macro_rules! __impl_slice_eq1 { - ([$($vars:tt)*] $lhs:ty, $rhs:ty, $($constraints:tt)*) => { - #[stable(feature = "rust1", since = "1.0.0")] - impl PartialEq<$rhs> for $lhs - where - A: PartialEq, - $($constraints)* - { - #[inline] - fn eq(&self, other: &$rhs) -> bool { self[..] == other[..] } - #[inline] - fn ne(&self, other: &$rhs) -> bool { self[..] != other[..] } - } - } -} - -__impl_slice_eq1! { [] Vec, Vec, } -__impl_slice_eq1! { [] Vec, &[B], } -__impl_slice_eq1! { [] Vec, &mut [B], } -__impl_slice_eq1! { [] Cow<'_, [A]>, &[B], A: Clone } -__impl_slice_eq1! { [] Cow<'_, [A]>, &mut [B], A: Clone } -__impl_slice_eq1! { [] Cow<'_, [A]>, Vec, A: Clone } -__impl_slice_eq1! { [const N: usize] Vec, [B; N], [B; N]: LengthAtMost32 } -__impl_slice_eq1! { [const N: usize] Vec, &[B; N], [B; N]: LengthAtMost32 } - -// NOTE: some less important impls are omitted to reduce code bloat -// FIXME(Centril): Reconsider this? -//__impl_slice_eq1! { [const N: usize] Vec, &mut [B; N], [B; N]: LengthAtMost32 } -//__impl_slice_eq1! { [const N: usize] Cow<'a, [A]>, [B; N], [B; N]: LengthAtMost32 } -//__impl_slice_eq1! { [const N: usize] Cow<'a, [A]>, &[B; N], [B; N]: LengthAtMost32 } -//__impl_slice_eq1! { [const N: usize] Cow<'a, [A]>, &mut [B; N], [B; N]: LengthAtMost32 } - -/// Implements comparison of vectors, lexicographically. -#[stable(feature = "rust1", since = "1.0.0")] -impl PartialOrd for Vec { - #[inline] - fn partial_cmp(&self, other: &Vec) -> Option { - PartialOrd::partial_cmp(&**self, &**other) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Eq for Vec {} - -/// Implements ordering of vectors, lexicographically. -#[stable(feature = "rust1", since = "1.0.0")] -impl Ord for Vec { - #[inline] - fn cmp(&self, other: &Vec) -> Ordering { - Ord::cmp(&**self, &**other) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -unsafe impl<#[may_dangle] T> Drop for Vec { - fn drop(&mut self) { - unsafe { - // use drop for [T] - ptr::drop_in_place(&mut self[..]); - } - // RawVec handles deallocation - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Default for Vec { - /// Creates an empty `Vec`. - fn default() -> Vec { - Vec::new() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl fmt::Debug for Vec { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&**self, f) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl AsRef> for Vec { - fn as_ref(&self) -> &Vec { - self - } -} - -#[stable(feature = "vec_as_mut", since = "1.5.0")] -impl AsMut> for Vec { - fn as_mut(&mut self) -> &mut Vec { - self - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl AsRef<[T]> for Vec { - fn as_ref(&self) -> &[T] { - self - } -} - -#[stable(feature = "vec_as_mut", since = "1.5.0")] -impl AsMut<[T]> for Vec { - fn as_mut(&mut self) -> &mut [T] { - self - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl From<&[T]> for Vec { - #[cfg(not(test))] - fn from(s: &[T]) -> Vec { - s.to_vec() - } - #[cfg(test)] - fn from(s: &[T]) -> Vec { - crate::slice::to_vec(s) - } -} - -#[stable(feature = "vec_from_mut", since = "1.19.0")] -impl From<&mut [T]> for Vec { - #[cfg(not(test))] - fn from(s: &mut [T]) -> Vec { - s.to_vec() - } - #[cfg(test)] - fn from(s: &mut [T]) -> Vec { - crate::slice::to_vec(s) - } -} - -#[stable(feature = "vec_from_cow_slice", since = "1.14.0")] -impl<'a, T> From> for Vec -where - [T]: ToOwned>, -{ - fn from(s: Cow<'a, [T]>) -> Vec { - s.into_owned() - } -} - -// note: test pulls in libstd, which causes errors here -#[cfg(not(test))] -#[stable(feature = "vec_from_box", since = "1.18.0")] -impl From> for Vec { - fn from(s: Box<[T]>) -> Vec { - s.into_vec() - } -} - -// note: test pulls in libstd, which causes errors here -#[cfg(not(test))] -#[stable(feature = "box_from_vec", since = "1.20.0")] -impl From> for Box<[T]> { - fn from(v: Vec) -> Box<[T]> { - v.into_boxed_slice() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl From<&str> for Vec { - fn from(s: &str) -> Vec { - From::from(s.as_bytes()) - } -} - -//////////////////////////////////////////////////////////////////////////////// -// Clone-on-write -//////////////////////////////////////////////////////////////////////////////// - -#[stable(feature = "cow_from_vec", since = "1.8.0")] -impl<'a, T: Clone> From<&'a [T]> for Cow<'a, [T]> { - fn from(s: &'a [T]) -> Cow<'a, [T]> { - Cow::Borrowed(s) - } -} - -#[stable(feature = "cow_from_vec", since = "1.8.0")] -impl<'a, T: Clone> From> for Cow<'a, [T]> { - fn from(v: Vec) -> Cow<'a, [T]> { - Cow::Owned(v) - } -} - -#[stable(feature = "cow_from_vec_ref", since = "1.28.0")] -impl<'a, T: Clone> From<&'a Vec> for Cow<'a, [T]> { - fn from(v: &'a Vec) -> Cow<'a, [T]> { - Cow::Borrowed(v.as_slice()) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T> FromIterator for Cow<'a, [T]> -where - T: Clone, -{ - fn from_iter>(it: I) -> Cow<'a, [T]> { - Cow::Owned(FromIterator::from_iter(it)) - } -} - -//////////////////////////////////////////////////////////////////////////////// -// Iterators -//////////////////////////////////////////////////////////////////////////////// - -/// An iterator that moves out of a vector. -/// -/// This `struct` is created by the `into_iter` method on [`Vec`] (provided -/// by the [`IntoIterator`] trait). -/// -/// [`Vec`]: struct.Vec.html -/// [`IntoIterator`]: ../../std/iter/trait.IntoIterator.html -#[stable(feature = "rust1", since = "1.0.0")] -pub struct IntoIter { - buf: NonNull, - phantom: PhantomData, - cap: usize, - ptr: *const T, - end: *const T, -} - -#[stable(feature = "vec_intoiter_debug", since = "1.13.0")] -impl fmt::Debug for IntoIter { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("IntoIter").field(&self.as_slice()).finish() - } -} - -impl IntoIter { - /// Returns the remaining items of this iterator as a slice. - /// - /// # Examples - /// - /// ``` - /// let vec = vec!['a', 'b', 'c']; - /// let mut into_iter = vec.into_iter(); - /// assert_eq!(into_iter.as_slice(), &['a', 'b', 'c']); - /// let _ = into_iter.next().unwrap(); - /// assert_eq!(into_iter.as_slice(), &['b', 'c']); - /// ``` - #[stable(feature = "vec_into_iter_as_slice", since = "1.15.0")] - pub fn as_slice(&self) -> &[T] { - unsafe { slice::from_raw_parts(self.ptr, self.len()) } - } - - /// Returns the remaining items of this iterator as a mutable slice. - /// - /// # Examples - /// - /// ``` - /// let vec = vec!['a', 'b', 'c']; - /// let mut into_iter = vec.into_iter(); - /// assert_eq!(into_iter.as_slice(), &['a', 'b', 'c']); - /// into_iter.as_mut_slice()[2] = 'z'; - /// assert_eq!(into_iter.next().unwrap(), 'a'); - /// assert_eq!(into_iter.next().unwrap(), 'b'); - /// assert_eq!(into_iter.next().unwrap(), 'z'); - /// ``` - #[stable(feature = "vec_into_iter_as_slice", since = "1.15.0")] - pub fn as_mut_slice(&mut self) -> &mut [T] { - unsafe { slice::from_raw_parts_mut(self.ptr as *mut T, self.len()) } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -unsafe impl Send for IntoIter {} -#[stable(feature = "rust1", since = "1.0.0")] -unsafe impl Sync for IntoIter {} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for IntoIter { - type Item = T; - - #[inline] - fn next(&mut self) -> Option { - unsafe { - if self.ptr as *const _ == self.end { - None - } else { - if mem::size_of::() == 0 { - // purposefully don't use 'ptr.offset' because for - // vectors with 0-size elements this would return the - // same pointer. - self.ptr = arith_offset(self.ptr as *const i8, 1) as *mut T; - - // Make up a value of this ZST. - Some(mem::zeroed()) - } else { - let old = self.ptr; - self.ptr = self.ptr.offset(1); - - Some(ptr::read(old)) - } - } - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - let exact = if mem::size_of::() == 0 { - (self.end as usize).wrapping_sub(self.ptr as usize) - } else { - unsafe { self.end.offset_from(self.ptr) as usize } - }; - (exact, Some(exact)) - } - - #[inline] - fn count(self) -> usize { - self.len() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl DoubleEndedIterator for IntoIter { - #[inline] - fn next_back(&mut self) -> Option { - unsafe { - if self.end == self.ptr { - None - } else { - if mem::size_of::() == 0 { - // See above for why 'ptr.offset' isn't used - self.end = arith_offset(self.end as *const i8, -1) as *mut T; - - // Make up a value of this ZST. - Some(mem::zeroed()) - } else { - self.end = self.end.offset(-1); - - Some(ptr::read(self.end)) - } - } - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl ExactSizeIterator for IntoIter { - fn is_empty(&self) -> bool { - self.ptr == self.end - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for IntoIter {} - -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for IntoIter {} - -#[stable(feature = "vec_into_iter_clone", since = "1.8.0")] -impl Clone for IntoIter { - fn clone(&self) -> IntoIter { - self.as_slice().to_owned().into_iter() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -unsafe impl<#[may_dangle] T> Drop for IntoIter { - fn drop(&mut self) { - struct DropGuard<'a, T>(&'a mut IntoIter); - - impl Drop for DropGuard<'_, T> { - fn drop(&mut self) { - // RawVec handles deallocation - let _ = unsafe { RawVec::from_raw_parts(self.0.buf.as_ptr(), self.0.cap) }; - } - } - - let guard = DropGuard(self); - // destroy the remaining elements - unsafe { - ptr::drop_in_place(guard.0.as_mut_slice()); - } - // now `guard` will be dropped and do the rest - } -} - -/// A draining iterator for `Vec`. -/// -/// This `struct` is created by the [`drain`] method on [`Vec`]. -/// -/// [`drain`]: struct.Vec.html#method.drain -/// [`Vec`]: struct.Vec.html -#[stable(feature = "drain", since = "1.6.0")] -pub struct Drain<'a, T: 'a> { - /// Index of tail to preserve - tail_start: usize, - /// Length of tail - tail_len: usize, - /// Current remaining range to remove - iter: slice::Iter<'a, T>, - vec: NonNull>, -} - -#[stable(feature = "collection_debug", since = "1.17.0")] -impl fmt::Debug for Drain<'_, T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("Drain").field(&self.iter.as_slice()).finish() - } -} - -impl<'a, T> Drain<'a, T> { - /// Returns the remaining items of this iterator as a slice. - /// - /// # Examples - /// - /// ``` - /// # #![feature(vec_drain_as_slice)] - /// let mut vec = vec!['a', 'b', 'c']; - /// let mut drain = vec.drain(..); - /// assert_eq!(drain.as_slice(), &['a', 'b', 'c']); - /// let _ = drain.next().unwrap(); - /// assert_eq!(drain.as_slice(), &['b', 'c']); - /// ``` - #[unstable(feature = "vec_drain_as_slice", reason = "recently added", issue = "58957")] - pub fn as_slice(&self) -> &[T] { - self.iter.as_slice() - } -} - -#[stable(feature = "drain", since = "1.6.0")] -unsafe impl Sync for Drain<'_, T> {} -#[stable(feature = "drain", since = "1.6.0")] -unsafe impl Send for Drain<'_, T> {} - -#[stable(feature = "drain", since = "1.6.0")] -impl Iterator for Drain<'_, T> { - type Item = T; - - #[inline] - fn next(&mut self) -> Option { - self.iter.next().map(|elt| unsafe { ptr::read(elt as *const _) }) - } - - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() - } -} - -#[stable(feature = "drain", since = "1.6.0")] -impl DoubleEndedIterator for Drain<'_, T> { - #[inline] - fn next_back(&mut self) -> Option { - self.iter.next_back().map(|elt| unsafe { ptr::read(elt as *const _) }) - } -} - -#[stable(feature = "drain", since = "1.6.0")] -impl Drop for Drain<'_, T> { - fn drop(&mut self) { - /// Continues dropping the remaining elements in the `Drain`, then moves back the - /// un-`Drain`ed elements to restore the original `Vec`. - struct DropGuard<'r, 'a, T>(&'r mut Drain<'a, T>); - - impl<'r, 'a, T> Drop for DropGuard<'r, 'a, T> { - fn drop(&mut self) { - // Continue the same loop we have below. If the loop already finished, this does - // nothing. - self.0.for_each(drop); - - if self.0.tail_len > 0 { - unsafe { - let source_vec = self.0.vec.as_mut(); - // memmove back untouched tail, update to new length - let start = source_vec.len(); - let tail = self.0.tail_start; - if tail != start { - let src = source_vec.as_ptr().add(tail); - let dst = source_vec.as_mut_ptr().add(start); - ptr::copy(src, dst, self.0.tail_len); - } - source_vec.set_len(start + self.0.tail_len); - } - } - } - } - - // exhaust self first - while let Some(item) = self.next() { - let guard = DropGuard(self); - drop(item); - mem::forget(guard); - } - - // Drop a `DropGuard` to move back the non-drained tail of `self`. - DropGuard(self); - } -} - -#[stable(feature = "drain", since = "1.6.0")] -impl ExactSizeIterator for Drain<'_, T> { - fn is_empty(&self) -> bool { - self.iter.is_empty() - } -} - -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for Drain<'_, T> {} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for Drain<'_, T> {} - -/// A splicing iterator for `Vec`. -/// -/// This struct is created by the [`splice()`] method on [`Vec`]. See its -/// documentation for more. -/// -/// [`splice()`]: struct.Vec.html#method.splice -/// [`Vec`]: struct.Vec.html -#[derive(Debug)] -#[stable(feature = "vec_splice", since = "1.21.0")] -pub struct Splice<'a, I: Iterator + 'a> { - drain: Drain<'a, I::Item>, - replace_with: I, -} - -#[stable(feature = "vec_splice", since = "1.21.0")] -impl Iterator for Splice<'_, I> { - type Item = I::Item; - - fn next(&mut self) -> Option { - self.drain.next() - } - - fn size_hint(&self) -> (usize, Option) { - self.drain.size_hint() - } -} - -#[stable(feature = "vec_splice", since = "1.21.0")] -impl DoubleEndedIterator for Splice<'_, I> { - fn next_back(&mut self) -> Option { - self.drain.next_back() - } -} - -#[stable(feature = "vec_splice", since = "1.21.0")] -impl ExactSizeIterator for Splice<'_, I> {} - -#[stable(feature = "vec_splice", since = "1.21.0")] -impl Drop for Splice<'_, I> { - fn drop(&mut self) { - self.drain.by_ref().for_each(drop); - - unsafe { - if self.drain.tail_len == 0 { - self.drain.vec.as_mut().extend(self.replace_with.by_ref()); - return; - } - - // First fill the range left by drain(). - if !self.drain.fill(&mut self.replace_with) { - return; - } - - // There may be more elements. Use the lower bound as an estimate. - // FIXME: Is the upper bound a better guess? Or something else? - let (lower_bound, _upper_bound) = self.replace_with.size_hint(); - if lower_bound > 0 { - self.drain.move_tail(lower_bound); - if !self.drain.fill(&mut self.replace_with) { - return; - } - } - - // Collect any remaining elements. - // This is a zero-length vector which does not allocate if `lower_bound` was exact. - let mut collected = self.replace_with.by_ref().collect::>().into_iter(); - // Now we have an exact count. - if collected.len() > 0 { - self.drain.move_tail(collected.len()); - let filled = self.drain.fill(&mut collected); - debug_assert!(filled); - debug_assert_eq!(collected.len(), 0); - } - } - // Let `Drain::drop` move the tail back if necessary and restore `vec.len`. - } -} - -/// Private helper methods for `Splice::drop` -impl Drain<'_, T> { - /// The range from `self.vec.len` to `self.tail_start` contains elements - /// that have been moved out. - /// Fill that range as much as possible with new elements from the `replace_with` iterator. - /// Returns `true` if we filled the entire range. (`replace_with.next()` didn’t return `None`.) - unsafe fn fill>(&mut self, replace_with: &mut I) -> bool { - let vec = self.vec.as_mut(); - let range_start = vec.len; - let range_end = self.tail_start; - let range_slice = - slice::from_raw_parts_mut(vec.as_mut_ptr().add(range_start), range_end - range_start); - - for place in range_slice { - if let Some(new_item) = replace_with.next() { - ptr::write(place, new_item); - vec.len += 1; - } else { - return false; - } - } - true - } - - /// Makes room for inserting more elements before the tail. - unsafe fn move_tail(&mut self, extra_capacity: usize) { - let vec = self.vec.as_mut(); - let used_capacity = self.tail_start + self.tail_len; - vec.buf.reserve(used_capacity, extra_capacity); - - let new_tail_start = self.tail_start + extra_capacity; - let src = vec.as_ptr().add(self.tail_start); - let dst = vec.as_mut_ptr().add(new_tail_start); - ptr::copy(src, dst, self.tail_len); - self.tail_start = new_tail_start; - } -} - -/// An iterator produced by calling `drain_filter` on Vec. -#[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")] -#[derive(Debug)] -pub struct DrainFilter<'a, T, F> -where - F: FnMut(&mut T) -> bool, -{ - vec: &'a mut Vec, - /// The index of the item that will be inspected by the next call to `next`. - idx: usize, - /// The number of items that have been drained (removed) thus far. - del: usize, - /// The original length of `vec` prior to draining. - old_len: usize, - /// The filter test predicate. - pred: F, - /// A flag that indicates a panic has occurred in the filter test prodicate. - /// This is used as a hint in the drop implementation to prevent consumption - /// of the remainder of the `DrainFilter`. Any unprocessed items will be - /// backshifted in the `vec`, but no further items will be dropped or - /// tested by the filter predicate. - panic_flag: bool, -} - -#[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")] -impl Iterator for DrainFilter<'_, T, F> -where - F: FnMut(&mut T) -> bool, -{ - type Item = T; - - fn next(&mut self) -> Option { - unsafe { - while self.idx < self.old_len { - let i = self.idx; - let v = slice::from_raw_parts_mut(self.vec.as_mut_ptr(), self.old_len); - self.panic_flag = true; - let drained = (self.pred)(&mut v[i]); - self.panic_flag = false; - // Update the index *after* the predicate is called. If the index - // is updated prior and the predicate panics, the element at this - // index would be leaked. - self.idx += 1; - if drained { - self.del += 1; - return Some(ptr::read(&v[i])); - } else if self.del > 0 { - let del = self.del; - let src: *const T = &v[i]; - let dst: *mut T = &mut v[i - del]; - ptr::copy_nonoverlapping(src, dst, 1); - } - } - None - } - } - - fn size_hint(&self) -> (usize, Option) { - (0, Some(self.old_len - self.idx)) - } -} - -#[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")] -impl Drop for DrainFilter<'_, T, F> -where - F: FnMut(&mut T) -> bool, -{ - fn drop(&mut self) { - struct BackshiftOnDrop<'a, 'b, T, F> - where - F: FnMut(&mut T) -> bool, - { - drain: &'b mut DrainFilter<'a, T, F>, - } - - impl<'a, 'b, T, F> Drop for BackshiftOnDrop<'a, 'b, T, F> - where - F: FnMut(&mut T) -> bool, - { - fn drop(&mut self) { - unsafe { - if self.drain.idx < self.drain.old_len && self.drain.del > 0 { - // This is a pretty messed up state, and there isn't really an - // obviously right thing to do. We don't want to keep trying - // to execute `pred`, so we just backshift all the unprocessed - // elements and tell the vec that they still exist. The backshift - // is required to prevent a double-drop of the last successfully - // drained item prior to a panic in the predicate. - let ptr = self.drain.vec.as_mut_ptr(); - let src = ptr.add(self.drain.idx); - let dst = src.sub(self.drain.del); - let tail_len = self.drain.old_len - self.drain.idx; - src.copy_to(dst, tail_len); - } - self.drain.vec.set_len(self.drain.old_len - self.drain.del); - } - } - } - - let backshift = BackshiftOnDrop { drain: self }; - - // Attempt to consume any remaining elements if the filter predicate - // has not yet panicked. We'll backshift any remaining elements - // whether we've already panicked or if the consumption here panics. - if !backshift.drain.panic_flag { - backshift.drain.for_each(drop); - } - } -} diff --git a/crux-mir/lib/alloc/src/vec/cow.rs b/crux-mir/lib/alloc/src/vec/cow.rs new file mode 100644 index 000000000..64943a273 --- /dev/null +++ b/crux-mir/lib/alloc/src/vec/cow.rs @@ -0,0 +1,53 @@ +use crate::borrow::Cow; +use core::iter::FromIterator; + +use super::Vec; + +#[stable(feature = "cow_from_vec", since = "1.8.0")] +impl<'a, T: Clone> From<&'a [T]> for Cow<'a, [T]> { + /// Creates a [`Borrowed`] variant of [`Cow`] + /// from a slice. + /// + /// This conversion does not allocate or clone the data. + /// + /// [`Borrowed`]: crate::borrow::Cow::Borrowed + fn from(s: &'a [T]) -> Cow<'a, [T]> { + Cow::Borrowed(s) + } +} + +#[stable(feature = "cow_from_vec", since = "1.8.0")] +impl<'a, T: Clone> From> for Cow<'a, [T]> { + /// Creates an [`Owned`] variant of [`Cow`] + /// from an owned instance of [`Vec`]. + /// + /// This conversion does not allocate or clone the data. + /// + /// [`Owned`]: crate::borrow::Cow::Owned + fn from(v: Vec) -> Cow<'a, [T]> { + Cow::Owned(v) + } +} + +#[stable(feature = "cow_from_vec_ref", since = "1.28.0")] +impl<'a, T: Clone> From<&'a Vec> for Cow<'a, [T]> { + /// Creates a [`Borrowed`] variant of [`Cow`] + /// from a reference to [`Vec`]. + /// + /// This conversion does not allocate or clone the data. + /// + /// [`Borrowed`]: crate::borrow::Cow::Borrowed + fn from(v: &'a Vec) -> Cow<'a, [T]> { + Cow::Borrowed(v.as_slice()) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T> FromIterator for Cow<'a, [T]> +where + T: Clone, +{ + fn from_iter>(it: I) -> Cow<'a, [T]> { + Cow::Owned(FromIterator::from_iter(it)) + } +} diff --git a/crux-mir/lib/alloc/src/vec/drain.rs b/crux-mir/lib/alloc/src/vec/drain.rs new file mode 100644 index 000000000..2b1a787cc --- /dev/null +++ b/crux-mir/lib/alloc/src/vec/drain.rs @@ -0,0 +1,255 @@ +use crate::alloc::{Allocator, Global}; +use core::fmt; +use core::iter::{FusedIterator, TrustedLen}; +use core::mem::{self, ManuallyDrop, SizedTypeProperties}; +use core::ptr::{self, NonNull}; +use core::slice::{self}; + +use super::Vec; + +/// A draining iterator for `Vec`. +/// +/// This `struct` is created by [`Vec::drain`]. +/// See its documentation for more. +/// +/// # Example +/// +/// ``` +/// let mut v = vec![0, 1, 2]; +/// let iter: std::vec::Drain<_> = v.drain(..); +/// ``` +#[stable(feature = "drain", since = "1.6.0")] +pub struct Drain< + 'a, + T: 'a, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + 'a = Global, +> { + /// Index of tail to preserve + pub(super) tail_start: usize, + /// Length of tail + pub(super) tail_len: usize, + /// Current remaining range to remove + pub(super) iter: slice::Iter<'a, T>, + pub(super) vec: NonNull>, +} + +#[stable(feature = "collection_debug", since = "1.17.0")] +impl fmt::Debug for Drain<'_, T, A> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("Drain").field(&self.iter.as_slice()).finish() + } +} + +impl<'a, T, A: Allocator> Drain<'a, T, A> { + /// Returns the remaining items of this iterator as a slice. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec!['a', 'b', 'c']; + /// let mut drain = vec.drain(..); + /// assert_eq!(drain.as_slice(), &['a', 'b', 'c']); + /// let _ = drain.next().unwrap(); + /// assert_eq!(drain.as_slice(), &['b', 'c']); + /// ``` + #[must_use] + #[stable(feature = "vec_drain_as_slice", since = "1.46.0")] + pub fn as_slice(&self) -> &[T] { + self.iter.as_slice() + } + + /// Returns a reference to the underlying allocator. + #[unstable(feature = "allocator_api", issue = "32838")] + #[must_use] + #[inline] + pub fn allocator(&self) -> &A { + unsafe { self.vec.as_ref().allocator() } + } + + /// Keep unyielded elements in the source `Vec`. + /// + /// # Examples + /// + /// ``` + /// #![feature(drain_keep_rest)] + /// + /// let mut vec = vec!['a', 'b', 'c']; + /// let mut drain = vec.drain(..); + /// + /// assert_eq!(drain.next().unwrap(), 'a'); + /// + /// // This call keeps 'b' and 'c' in the vec. + /// drain.keep_rest(); + /// + /// // If we wouldn't call `keep_rest()`, + /// // `vec` would be empty. + /// assert_eq!(vec, ['b', 'c']); + /// ``` + #[unstable(feature = "drain_keep_rest", issue = "101122")] + pub fn keep_rest(self) { + // At this moment layout looks like this: + // + // [head] [yielded by next] [unyielded] [yielded by next_back] [tail] + // ^-- start \_________/-- unyielded_len \____/-- self.tail_len + // ^-- unyielded_ptr ^-- tail + // + // Normally `Drop` impl would drop [unyielded] and then move [tail] to the `start`. + // Here we want to + // 1. Move [unyielded] to `start` + // 2. Move [tail] to a new start at `start + len(unyielded)` + // 3. Update length of the original vec to `len(head) + len(unyielded) + len(tail)` + // a. In case of ZST, this is the only thing we want to do + // 4. Do *not* drop self, as everything is put in a consistent state already, there is nothing to do + let mut this = ManuallyDrop::new(self); + + unsafe { + let source_vec = this.vec.as_mut(); + + let start = source_vec.len(); + let tail = this.tail_start; + + let unyielded_len = this.iter.len(); + let unyielded_ptr = this.iter.as_slice().as_ptr(); + + // ZSTs have no identity, so we don't need to move them around. + let needs_move = mem::size_of::() != 0; + + if needs_move { + let start_ptr = source_vec.as_mut_ptr().add(start); + + // memmove back unyielded elements + if unyielded_ptr != start_ptr { + let src = unyielded_ptr; + let dst = start_ptr; + + ptr::copy(src, dst, unyielded_len); + } + + // memmove back untouched tail + if tail != (start + unyielded_len) { + let src = source_vec.as_ptr().add(tail); + let dst = start_ptr.add(unyielded_len); + ptr::copy(src, dst, this.tail_len); + } + } + + source_vec.set_len(start + unyielded_len + this.tail_len); + } + } +} + +#[stable(feature = "vec_drain_as_slice", since = "1.46.0")] +impl<'a, T, A: Allocator> AsRef<[T]> for Drain<'a, T, A> { + fn as_ref(&self) -> &[T] { + self.as_slice() + } +} + +#[stable(feature = "drain", since = "1.6.0")] +unsafe impl Sync for Drain<'_, T, A> {} +#[stable(feature = "drain", since = "1.6.0")] +unsafe impl Send for Drain<'_, T, A> {} + +#[stable(feature = "drain", since = "1.6.0")] +impl Iterator for Drain<'_, T, A> { + type Item = T; + + #[inline] + fn next(&mut self) -> Option { + self.iter.next().map(|elt| unsafe { ptr::read(elt as *const _) }) + } + + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } +} + +#[stable(feature = "drain", since = "1.6.0")] +impl DoubleEndedIterator for Drain<'_, T, A> { + #[inline] + fn next_back(&mut self) -> Option { + self.iter.next_back().map(|elt| unsafe { ptr::read(elt as *const _) }) + } +} + +#[stable(feature = "drain", since = "1.6.0")] +impl Drop for Drain<'_, T, A> { + fn drop(&mut self) { + /// Moves back the un-`Drain`ed elements to restore the original `Vec`. + struct DropGuard<'r, 'a, T, A: Allocator>(&'r mut Drain<'a, T, A>); + + impl<'r, 'a, T, A: Allocator> Drop for DropGuard<'r, 'a, T, A> { + fn drop(&mut self) { + if self.0.tail_len > 0 { + unsafe { + let source_vec = self.0.vec.as_mut(); + // memmove back untouched tail, update to new length + let start = source_vec.len(); + let tail = self.0.tail_start; + if tail != start { + let src = source_vec.as_ptr().add(tail); + let dst = source_vec.as_mut_ptr().add(start); + ptr::copy(src, dst, self.0.tail_len); + } + source_vec.set_len(start + self.0.tail_len); + } + } + } + } + + let iter = mem::replace(&mut self.iter, (&mut []).iter()); + let drop_len = iter.len(); + + let mut vec = self.vec; + + if T::IS_ZST { + // ZSTs have no identity, so we don't need to move them around, we only need to drop the correct amount. + // this can be achieved by manipulating the Vec length instead of moving values out from `iter`. + unsafe { + let vec = vec.as_mut(); + let old_len = vec.len(); + vec.set_len(old_len + drop_len + self.tail_len); + vec.truncate(old_len + self.tail_len); + } + + return; + } + + // ensure elements are moved back into their appropriate places, even when drop_in_place panics + let _guard = DropGuard(self); + + if drop_len == 0 { + return; + } + + // as_slice() must only be called when iter.len() is > 0 because + // it also gets touched by vec::Splice which may turn it into a dangling pointer + // which would make it and the vec pointer point to different allocations which would + // lead to invalid pointer arithmetic below. + let drop_ptr = iter.as_slice().as_ptr(); + + unsafe { + // drop_ptr comes from a slice::Iter which only gives us a &[T] but for drop_in_place + // a pointer with mutable provenance is necessary. Therefore we must reconstruct + // it from the original vec but also avoid creating a &mut to the front since that could + // invalidate raw pointers to it which some unsafe code might rely on. + let vec_ptr = vec.as_mut().as_mut_ptr(); + let drop_offset = drop_ptr.sub_ptr(vec_ptr); + let to_drop = ptr::slice_from_raw_parts_mut(vec_ptr.add(drop_offset), drop_len); + ptr::drop_in_place(to_drop); + } + } +} + +#[stable(feature = "drain", since = "1.6.0")] +impl ExactSizeIterator for Drain<'_, T, A> { + fn is_empty(&self) -> bool { + self.iter.is_empty() + } +} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for Drain<'_, T, A> {} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Drain<'_, T, A> {} diff --git a/crux-mir/lib/alloc/src/vec/drain_filter.rs b/crux-mir/lib/alloc/src/vec/drain_filter.rs new file mode 100644 index 000000000..8c03f1692 --- /dev/null +++ b/crux-mir/lib/alloc/src/vec/drain_filter.rs @@ -0,0 +1,199 @@ +use crate::alloc::{Allocator, Global}; +use core::mem::{self, ManuallyDrop}; +use core::ptr; +use core::slice; + +use super::Vec; + +/// An iterator which uses a closure to determine if an element should be removed. +/// +/// This struct is created by [`Vec::drain_filter`]. +/// See its documentation for more. +/// +/// # Example +/// +/// ``` +/// #![feature(drain_filter)] +/// +/// let mut v = vec![0, 1, 2]; +/// let iter: std::vec::DrainFilter<_, _> = v.drain_filter(|x| *x % 2 == 0); +/// ``` +#[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")] +#[derive(Debug)] +pub struct DrainFilter< + 'a, + T, + F, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator = Global, +> where + F: FnMut(&mut T) -> bool, +{ + pub(super) vec: &'a mut Vec, + /// The index of the item that will be inspected by the next call to `next`. + pub(super) idx: usize, + /// The number of items that have been drained (removed) thus far. + pub(super) del: usize, + /// The original length of `vec` prior to draining. + pub(super) old_len: usize, + /// The filter test predicate. + pub(super) pred: F, + /// A flag that indicates a panic has occurred in the filter test predicate. + /// This is used as a hint in the drop implementation to prevent consumption + /// of the remainder of the `DrainFilter`. Any unprocessed items will be + /// backshifted in the `vec`, but no further items will be dropped or + /// tested by the filter predicate. + pub(super) panic_flag: bool, +} + +impl DrainFilter<'_, T, F, A> +where + F: FnMut(&mut T) -> bool, +{ + /// Returns a reference to the underlying allocator. + #[unstable(feature = "allocator_api", issue = "32838")] + #[inline] + pub fn allocator(&self) -> &A { + self.vec.allocator() + } + + /// Keep unyielded elements in the source `Vec`. + /// + /// # Examples + /// + /// ``` + /// #![feature(drain_filter)] + /// #![feature(drain_keep_rest)] + /// + /// let mut vec = vec!['a', 'b', 'c']; + /// let mut drain = vec.drain_filter(|_| true); + /// + /// assert_eq!(drain.next().unwrap(), 'a'); + /// + /// // This call keeps 'b' and 'c' in the vec. + /// drain.keep_rest(); + /// + /// // If we wouldn't call `keep_rest()`, + /// // `vec` would be empty. + /// assert_eq!(vec, ['b', 'c']); + /// ``` + #[unstable(feature = "drain_keep_rest", issue = "101122")] + pub fn keep_rest(self) { + // At this moment layout looks like this: + // + // _____________________/-- old_len + // / \ + // [kept] [yielded] [tail] + // \_______/ ^-- idx + // \-- del + // + // Normally `Drop` impl would drop [tail] (via .for_each(drop), ie still calling `pred`) + // + // 1. Move [tail] after [kept] + // 2. Update length of the original vec to `old_len - del` + // a. In case of ZST, this is the only thing we want to do + // 3. Do *not* drop self, as everything is put in a consistent state already, there is nothing to do + let mut this = ManuallyDrop::new(self); + + unsafe { + // ZSTs have no identity, so we don't need to move them around. + let needs_move = mem::size_of::() != 0; + + if needs_move && this.idx < this.old_len && this.del > 0 { + let ptr = this.vec.as_mut_ptr(); + let src = ptr.add(this.idx); + let dst = src.sub(this.del); + let tail_len = this.old_len - this.idx; + src.copy_to(dst, tail_len); + } + + let new_len = this.old_len - this.del; + this.vec.set_len(new_len); + } + } +} + +#[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")] +impl Iterator for DrainFilter<'_, T, F, A> +where + F: FnMut(&mut T) -> bool, +{ + type Item = T; + + fn next(&mut self) -> Option { + unsafe { + while self.idx < self.old_len { + let i = self.idx; + let v = slice::from_raw_parts_mut(self.vec.as_mut_ptr(), self.old_len); + self.panic_flag = true; + let drained = (self.pred)(&mut v[i]); + self.panic_flag = false; + // Update the index *after* the predicate is called. If the index + // is updated prior and the predicate panics, the element at this + // index would be leaked. + self.idx += 1; + if drained { + self.del += 1; + return Some(ptr::read(&v[i])); + } else if self.del > 0 { + let del = self.del; + let src: *const T = &v[i]; + let dst: *mut T = &mut v[i - del]; + ptr::copy_nonoverlapping(src, dst, 1); + } + } + None + } + } + + fn size_hint(&self) -> (usize, Option) { + (0, Some(self.old_len - self.idx)) + } +} + +#[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")] +impl Drop for DrainFilter<'_, T, F, A> +where + F: FnMut(&mut T) -> bool, +{ + fn drop(&mut self) { + struct BackshiftOnDrop<'a, 'b, T, F, A: Allocator> + where + F: FnMut(&mut T) -> bool, + { + drain: &'b mut DrainFilter<'a, T, F, A>, + } + + impl<'a, 'b, T, F, A: Allocator> Drop for BackshiftOnDrop<'a, 'b, T, F, A> + where + F: FnMut(&mut T) -> bool, + { + fn drop(&mut self) { + unsafe { + if self.drain.idx < self.drain.old_len && self.drain.del > 0 { + // This is a pretty messed up state, and there isn't really an + // obviously right thing to do. We don't want to keep trying + // to execute `pred`, so we just backshift all the unprocessed + // elements and tell the vec that they still exist. The backshift + // is required to prevent a double-drop of the last successfully + // drained item prior to a panic in the predicate. + let ptr = self.drain.vec.as_mut_ptr(); + let src = ptr.add(self.drain.idx); + let dst = src.sub(self.drain.del); + let tail_len = self.drain.old_len - self.drain.idx; + src.copy_to(dst, tail_len); + } + self.drain.vec.set_len(self.drain.old_len - self.drain.del); + } + } + } + + let backshift = BackshiftOnDrop { drain: self }; + + // Attempt to consume any remaining elements if the filter predicate + // has not yet panicked. We'll backshift any remaining elements + // whether we've already panicked or if the consumption here panics. + if !backshift.drain.panic_flag { + backshift.drain.for_each(drop); + } + } +} diff --git a/crux-mir/lib/alloc/src/vec/in_place_collect.rs b/crux-mir/lib/alloc/src/vec/in_place_collect.rs new file mode 100644 index 000000000..87d61deb1 --- /dev/null +++ b/crux-mir/lib/alloc/src/vec/in_place_collect.rs @@ -0,0 +1,308 @@ +//! Inplace iterate-and-collect specialization for `Vec` +//! +//! Note: This documents Vec internals, some of the following sections explain implementation +//! details and are best read together with the source of this module. +//! +//! The specialization in this module applies to iterators in the shape of +//! `source.adapter().adapter().adapter().collect::>()` +//! where `source` is an owning iterator obtained from [`Vec`], [`Box<[T]>`][box] (by conversion to `Vec`) +//! or [`BinaryHeap`], the adapters each consume one or more items per step +//! (represented by [`InPlaceIterable`]), provide transitive access to `source` (via [`SourceIter`]) +//! and thus the underlying allocation. And finally the layouts of `T` and `U` must +//! have the same size and alignment, this is currently ensured via const eval instead of trait bounds +//! in the specialized [`SpecFromIter`] implementation. +//! +//! [`BinaryHeap`]: crate::collections::BinaryHeap +//! [box]: crate::boxed::Box +//! +//! By extension some other collections which use `collect::>()` internally in their +//! `FromIterator` implementation benefit from this too. +//! +//! Access to the underlying source goes through a further layer of indirection via the private +//! trait [`AsVecIntoIter`] to hide the implementation detail that other collections may use +//! `vec::IntoIter` internally. +//! +//! In-place iteration depends on the interaction of several unsafe traits, implementation +//! details of multiple parts in the iterator pipeline and often requires holistic reasoning +//! across multiple structs since iterators are executed cooperatively rather than having +//! a central evaluator/visitor struct executing all iterator components. +//! +//! # Reading from and writing to the same allocation +//! +//! By its nature collecting in place means that the reader and writer side of the iterator +//! use the same allocation. Since `try_fold()` (used in [`SpecInPlaceCollect`]) takes a +//! reference to the iterator for the duration of the iteration that means we can't interleave +//! the step of reading a value and getting a reference to write to. Instead raw pointers must be +//! used on the reader and writer side. +//! +//! That writes never clobber a yet-to-be-read item is ensured by the [`InPlaceIterable`] requirements. +//! +//! # Layout constraints +//! +//! [`Allocator`] requires that `allocate()` and `deallocate()` have matching alignment and size. +//! Additionally this specialization doesn't make sense for ZSTs as there is no reallocation to +//! avoid and it would make pointer arithmetic more difficult. +//! +//! [`Allocator`]: core::alloc::Allocator +//! +//! # Drop- and panic-safety +//! +//! Iteration can panic, requiring dropping the already written parts but also the remainder of +//! the source. Iteration can also leave some source items unconsumed which must be dropped. +//! All those drops in turn can panic which then must either leak the allocation or abort to avoid +//! double-drops. +//! +//! This is handled by the [`InPlaceDrop`] guard for sink items (`U`) and by +//! [`vec::IntoIter::forget_allocation_drop_remaining()`] for remaining source items (`T`). +//! +//! If dropping any remaining source item (`T`) panics then [`InPlaceDstBufDrop`] will handle dropping +//! the already collected sink items (`U`) and freeing the allocation. +//! +//! [`vec::IntoIter::forget_allocation_drop_remaining()`]: super::IntoIter::forget_allocation_drop_remaining() +//! +//! # O(1) collect +//! +//! The main iteration itself is further specialized when the iterator implements +//! [`TrustedRandomAccessNoCoerce`] to let the optimizer see that it is a counted loop with a single +//! [induction variable]. This can turn some iterators into a noop, i.e. it reduces them from O(n) to +//! O(1). This particular optimization is quite fickle and doesn't always work, see [#79308] +//! +//! [#79308]: https://github.com/rust-lang/rust/issues/79308 +//! [induction variable]: https://en.wikipedia.org/wiki/Induction_variable +//! +//! Since unchecked accesses through that trait do not advance the read pointer of `IntoIter` +//! this would interact unsoundly with the requirements about dropping the tail described above. +//! But since the normal `Drop` implementation of `IntoIter` would suffer from the same problem it +//! is only correct for `TrustedRandomAccessNoCoerce` to be implemented when the items don't +//! have a destructor. Thus that implicit requirement also makes the specialization safe to use for +//! in-place collection. +//! Note that this safety concern is about the correctness of `impl Drop for IntoIter`, +//! not the guarantees of `InPlaceIterable`. +//! +//! # Adapter implementations +//! +//! The invariants for adapters are documented in [`SourceIter`] and [`InPlaceIterable`], but +//! getting them right can be rather subtle for multiple, sometimes non-local reasons. +//! For example `InPlaceIterable` would be valid to implement for [`Peekable`], except +//! that it is stateful, cloneable and `IntoIter`'s clone implementation shortens the underlying +//! allocation which means if the iterator has been peeked and then gets cloned there no longer is +//! enough room, thus breaking an invariant ([#85322]). +//! +//! [#85322]: https://github.com/rust-lang/rust/issues/85322 +//! [`Peekable`]: core::iter::Peekable +//! +//! +//! # Examples +//! +//! Some cases that are optimized by this specialization, more can be found in the `Vec` +//! benchmarks: +//! +//! ```rust +//! # #[allow(dead_code)] +//! /// Converts a usize vec into an isize one. +//! pub fn cast(vec: Vec) -> Vec { +//! // Does not allocate, free or panic. On optlevel>=2 it does not loop. +//! // Of course this particular case could and should be written with `into_raw_parts` and +//! // `from_raw_parts` instead. +//! vec.into_iter().map(|u| u as isize).collect() +//! } +//! ``` +//! +//! ```rust +//! # #[allow(dead_code)] +//! /// Drops remaining items in `src` and if the layouts of `T` and `U` match it +//! /// returns an empty Vec backed by the original allocation. Otherwise it returns a new +//! /// empty vec. +//! pub fn recycle_allocation(src: Vec) -> Vec { +//! src.into_iter().filter_map(|_| None).collect() +//! } +//! ``` +//! +//! ```rust +//! let vec = vec![13usize; 1024]; +//! let _ = vec.into_iter() +//! .enumerate() +//! .filter_map(|(idx, val)| if idx % 2 == 0 { Some(val+idx) } else {None}) +//! .collect::>(); +//! +//! // is equivalent to the following, but doesn't require bounds checks +//! +//! let mut vec = vec![13usize; 1024]; +//! let mut write_idx = 0; +//! for idx in 0..vec.len() { +//! if idx % 2 == 0 { +//! vec[write_idx] = vec[idx] + idx; +//! write_idx += 1; +//! } +//! } +//! vec.truncate(write_idx); +//! ``` +use core::iter::{InPlaceIterable, SourceIter, TrustedRandomAccessNoCoerce}; +use core::mem::{self, ManuallyDrop, SizedTypeProperties}; +use core::ptr::{self}; + +use super::{InPlaceDrop, InPlaceDstBufDrop, SpecFromIter, SpecFromIterNested, Vec}; + +/// Specialization marker for collecting an iterator pipeline into a Vec while reusing the +/// source allocation, i.e. executing the pipeline in place. +#[rustc_unsafe_specialization_marker] +pub(super) trait InPlaceIterableMarker {} + +impl InPlaceIterableMarker for T where T: InPlaceIterable {} + +impl SpecFromIter for Vec +where + I: Iterator + SourceIter + InPlaceIterableMarker, +{ + default fn from_iter(mut iterator: I) -> Self { + // See "Layout constraints" section in the module documentation. We rely on const + // optimization here since these conditions currently cannot be expressed as trait bounds + if T::IS_ZST + || mem::size_of::() + != mem::size_of::<<::Source as AsVecIntoIter>::Item>() + || mem::align_of::() + != mem::align_of::<<::Source as AsVecIntoIter>::Item>() + { + // fallback to more generic implementations + return SpecFromIterNested::from_iter(iterator); + } + + let (src_buf, src_ptr, dst_buf, dst_end, cap) = unsafe { + let inner = iterator.as_inner().as_into_iter(); + ( + inner.buf.as_ptr(), + inner.ptr, + inner.buf.as_ptr() as *mut T, + inner.end as *const T, + inner.cap, + ) + }; + + let len = SpecInPlaceCollect::collect_in_place(&mut iterator, dst_buf, dst_end); + + let src = unsafe { iterator.as_inner().as_into_iter() }; + // check if SourceIter contract was upheld + // caveat: if they weren't we might not even make it to this point + debug_assert_eq!(src_buf, src.buf.as_ptr()); + // check InPlaceIterable contract. This is only possible if the iterator advanced the + // source pointer at all. If it uses unchecked access via TrustedRandomAccess + // then the source pointer will stay in its initial position and we can't use it as reference + if src.ptr != src_ptr { + debug_assert!( + unsafe { dst_buf.add(len) as *const _ } <= src.ptr, + "InPlaceIterable contract violation, write pointer advanced beyond read pointer" + ); + } + + // The ownership of the allocation and the new `T` values is temporarily moved into `dst_guard`. + // This is safe because `forget_allocation_drop_remaining` immediately forgets the allocation + // before any panic can occur in order to avoid any double free, and then proceeds to drop + // any remaining values at the tail of the source. + // + // Note: This access to the source wouldn't be allowed by the TrustedRandomIteratorNoCoerce + // contract (used by SpecInPlaceCollect below). But see the "O(1) collect" section in the + // module documenttation why this is ok anyway. + let dst_guard = InPlaceDstBufDrop { ptr: dst_buf, len, cap }; + src.forget_allocation_drop_remaining(); + mem::forget(dst_guard); + + let vec = unsafe { Vec::from_raw_parts(dst_buf, len, cap) }; + + vec + } +} + +fn write_in_place_with_drop( + src_end: *const T, +) -> impl FnMut(InPlaceDrop, T) -> Result, !> { + move |mut sink, item| { + unsafe { + // the InPlaceIterable contract cannot be verified precisely here since + // try_fold has an exclusive reference to the source pointer + // all we can do is check if it's still in range + debug_assert!(sink.dst as *const _ <= src_end, "InPlaceIterable contract violation"); + ptr::write(sink.dst, item); + // Since this executes user code which can panic we have to bump the pointer + // after each step. + sink.dst = sink.dst.add(1); + } + Ok(sink) + } +} + +/// Helper trait to hold specialized implementations of the in-place iterate-collect loop +trait SpecInPlaceCollect: Iterator { + /// Collects an iterator (`self`) into the destination buffer (`dst`) and returns the number of items + /// collected. `end` is the last writable element of the allocation and used for bounds checks. + /// + /// This method is specialized and one of its implementations makes use of + /// `Iterator::__iterator_get_unchecked` calls with a `TrustedRandomAccessNoCoerce` bound + /// on `I` which means the caller of this method must take the safety conditions + /// of that trait into consideration. + fn collect_in_place(&mut self, dst: *mut T, end: *const T) -> usize; +} + +impl SpecInPlaceCollect for I +where + I: Iterator, +{ + #[inline] + default fn collect_in_place(&mut self, dst_buf: *mut T, end: *const T) -> usize { + // use try-fold since + // - it vectorizes better for some iterator adapters + // - unlike most internal iteration methods, it only takes a &mut self + // - it lets us thread the write pointer through its innards and get it back in the end + let sink = InPlaceDrop { inner: dst_buf, dst: dst_buf }; + let sink = + self.try_fold::<_, _, Result<_, !>>(sink, write_in_place_with_drop(end)).unwrap(); + // iteration succeeded, don't drop head + unsafe { ManuallyDrop::new(sink).dst.sub_ptr(dst_buf) } + } +} + +impl SpecInPlaceCollect for I +where + I: Iterator + TrustedRandomAccessNoCoerce, +{ + #[inline] + fn collect_in_place(&mut self, dst_buf: *mut T, end: *const T) -> usize { + let len = self.size(); + let mut drop_guard = InPlaceDrop { inner: dst_buf, dst: dst_buf }; + for i in 0..len { + // Safety: InplaceIterable contract guarantees that for every element we read + // one slot in the underlying storage will have been freed up and we can immediately + // write back the result. + unsafe { + let dst = dst_buf.add(i); + debug_assert!(dst as *const _ <= end, "InPlaceIterable contract violation"); + ptr::write(dst, self.__iterator_get_unchecked(i)); + // Since this executes user code which can panic we have to bump the pointer + // after each step. + drop_guard.dst = dst.add(1); + } + } + mem::forget(drop_guard); + len + } +} + +/// Internal helper trait for in-place iteration specialization. +/// +/// Currently this is only implemented by [`vec::IntoIter`] - returning a reference to itself - and +/// [`binary_heap::IntoIter`] which returns a reference to its inner representation. +/// +/// Since this is an internal trait it hides the implementation detail `binary_heap::IntoIter` +/// uses `vec::IntoIter` internally. +/// +/// [`vec::IntoIter`]: super::IntoIter +/// [`binary_heap::IntoIter`]: crate::collections::binary_heap::IntoIter +/// +/// # Safety +/// +/// In-place iteration relies on implementation details of `vec::IntoIter`, most importantly that +/// it does not create references to the whole allocation during iteration, only raw pointers +#[rustc_specialization_trait] +pub(crate) unsafe trait AsVecIntoIter { + type Item; + fn as_into_iter(&mut self) -> &mut super::IntoIter; +} diff --git a/crux-mir/lib/alloc/src/vec/in_place_drop.rs b/crux-mir/lib/alloc/src/vec/in_place_drop.rs new file mode 100644 index 000000000..25ca33c6a --- /dev/null +++ b/crux-mir/lib/alloc/src/vec/in_place_drop.rs @@ -0,0 +1,39 @@ +use core::ptr::{self}; +use core::slice::{self}; + +// A helper struct for in-place iteration that drops the destination slice of iteration, +// i.e. the head. The source slice (the tail) is dropped by IntoIter. +pub(super) struct InPlaceDrop { + pub(super) inner: *mut T, + pub(super) dst: *mut T, +} + +impl InPlaceDrop { + fn len(&self) -> usize { + unsafe { self.dst.sub_ptr(self.inner) } + } +} + +impl Drop for InPlaceDrop { + #[inline] + fn drop(&mut self) { + unsafe { + ptr::drop_in_place(slice::from_raw_parts_mut(self.inner, self.len())); + } + } +} + +// A helper struct for in-place collection that drops the destination allocation and elements, +// to avoid leaking them if some other destructor panics. +pub(super) struct InPlaceDstBufDrop { + pub(super) ptr: *mut T, + pub(super) len: usize, + pub(super) cap: usize, +} + +impl Drop for InPlaceDstBufDrop { + #[inline] + fn drop(&mut self) { + unsafe { super::Vec::from_raw_parts(self.ptr, self.len, self.cap) }; + } +} diff --git a/crux-mir/lib/alloc/src/vec/into_iter.rs b/crux-mir/lib/alloc/src/vec/into_iter.rs new file mode 100644 index 000000000..37966007e --- /dev/null +++ b/crux-mir/lib/alloc/src/vec/into_iter.rs @@ -0,0 +1,433 @@ +#[cfg(not(no_global_oom_handling))] +use super::AsVecIntoIter; +use crate::alloc::{Allocator, Global}; +#[cfg(not(no_global_oom_handling))] +use crate::collections::VecDeque; +use crate::raw_vec::RawVec; +use core::array; +use core::fmt; +use core::iter::{ + FusedIterator, InPlaceIterable, SourceIter, TrustedLen, TrustedRandomAccessNoCoerce, +}; +use core::marker::PhantomData; +use core::mem::{self, ManuallyDrop, MaybeUninit, SizedTypeProperties}; +#[cfg(not(no_global_oom_handling))] +use core::ops::Deref; +use core::ptr::{self, NonNull}; +use core::slice::{self}; + +/// An iterator that moves out of a vector. +/// +/// This `struct` is created by the `into_iter` method on [`Vec`](super::Vec) +/// (provided by the [`IntoIterator`] trait). +/// +/// # Example +/// +/// ``` +/// let v = vec![0, 1, 2]; +/// let iter: std::vec::IntoIter<_> = v.into_iter(); +/// ``` +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_insignificant_dtor] +pub struct IntoIter< + T, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator = Global, +> { + pub(super) buf: NonNull, + pub(super) phantom: PhantomData, + pub(super) cap: usize, + // the drop impl reconstructs a RawVec from buf, cap and alloc + // to avoid dropping the allocator twice we need to wrap it into ManuallyDrop + pub(super) alloc: ManuallyDrop, + pub(super) ptr: *const T, + pub(super) end: *const T, // If T is a ZST, this is actually ptr+len. This encoding is picked so that + // ptr == end is a quick test for the Iterator being empty, that works + // for both ZST and non-ZST. +} + +#[stable(feature = "vec_intoiter_debug", since = "1.13.0")] +impl fmt::Debug for IntoIter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("IntoIter").field(&self.as_slice()).finish() + } +} + +impl IntoIter { + /// Returns the remaining items of this iterator as a slice. + /// + /// # Examples + /// + /// ``` + /// let vec = vec!['a', 'b', 'c']; + /// let mut into_iter = vec.into_iter(); + /// assert_eq!(into_iter.as_slice(), &['a', 'b', 'c']); + /// let _ = into_iter.next().unwrap(); + /// assert_eq!(into_iter.as_slice(), &['b', 'c']); + /// ``` + #[stable(feature = "vec_into_iter_as_slice", since = "1.15.0")] + pub fn as_slice(&self) -> &[T] { + unsafe { slice::from_raw_parts(self.ptr, self.len()) } + } + + /// Returns the remaining items of this iterator as a mutable slice. + /// + /// # Examples + /// + /// ``` + /// let vec = vec!['a', 'b', 'c']; + /// let mut into_iter = vec.into_iter(); + /// assert_eq!(into_iter.as_slice(), &['a', 'b', 'c']); + /// into_iter.as_mut_slice()[2] = 'z'; + /// assert_eq!(into_iter.next().unwrap(), 'a'); + /// assert_eq!(into_iter.next().unwrap(), 'b'); + /// assert_eq!(into_iter.next().unwrap(), 'z'); + /// ``` + #[stable(feature = "vec_into_iter_as_slice", since = "1.15.0")] + pub fn as_mut_slice(&mut self) -> &mut [T] { + unsafe { &mut *self.as_raw_mut_slice() } + } + + /// Returns a reference to the underlying allocator. + #[unstable(feature = "allocator_api", issue = "32838")] + #[inline] + pub fn allocator(&self) -> &A { + &self.alloc + } + + fn as_raw_mut_slice(&mut self) -> *mut [T] { + ptr::slice_from_raw_parts_mut(self.ptr as *mut T, self.len()) + } + + /// Drops remaining elements and relinquishes the backing allocation. + /// This method guarantees it won't panic before relinquishing + /// the backing allocation. + /// + /// This is roughly equivalent to the following, but more efficient + /// + /// ``` + /// # let mut into_iter = Vec::::with_capacity(10).into_iter(); + /// let mut into_iter = std::mem::replace(&mut into_iter, Vec::new().into_iter()); + /// (&mut into_iter).for_each(core::mem::drop); + /// std::mem::forget(into_iter); + /// ``` + /// + /// This method is used by in-place iteration, refer to the vec::in_place_collect + /// documentation for an overview. + #[cfg(not(no_global_oom_handling))] + pub(super) fn forget_allocation_drop_remaining(&mut self) { + let remaining = self.as_raw_mut_slice(); + + // overwrite the individual fields instead of creating a new + // struct and then overwriting &mut self. + // this creates less assembly + self.cap = 0; + self.buf = unsafe { NonNull::new_unchecked(RawVec::NEW.ptr()) }; + self.ptr = self.buf.as_ptr(); + self.end = self.buf.as_ptr(); + + // Dropping the remaining elements can panic, so this needs to be + // done only after updating the other fields. + unsafe { + ptr::drop_in_place(remaining); + } + } + + /// Forgets to Drop the remaining elements while still allowing the backing allocation to be freed. + pub(crate) fn forget_remaining_elements(&mut self) { + // For th ZST case, it is crucial that we mutate `end` here, not `ptr`. + // `ptr` must stay aligned, while `end` may be unaligned. + self.end = self.ptr; + } + + #[cfg(not(no_global_oom_handling))] + #[inline] + pub(crate) fn into_vecdeque(self) -> VecDeque { + // Keep our `Drop` impl from dropping the elements and the allocator + let mut this = ManuallyDrop::new(self); + + // SAFETY: This allocation originally came from a `Vec`, so it passes + // all those checks. We have `this.buf` ≤ `this.ptr` ≤ `this.end`, + // so the `sub_ptr`s below cannot wrap, and will produce a well-formed + // range. `end` ≤ `buf + cap`, so the range will be in-bounds. + // Taking `alloc` is ok because nothing else is going to look at it, + // since our `Drop` impl isn't going to run so there's no more code. + unsafe { + let buf = this.buf.as_ptr(); + let initialized = if T::IS_ZST { + // All the pointers are the same for ZSTs, so it's fine to + // say that they're all at the beginning of the "allocation". + 0..this.len() + } else { + this.ptr.sub_ptr(buf)..this.end.sub_ptr(buf) + }; + let cap = this.cap; + let alloc = ManuallyDrop::take(&mut this.alloc); + VecDeque::from_contiguous_raw_parts_in(buf, initialized, cap, alloc) + } + } +} + +#[stable(feature = "vec_intoiter_as_ref", since = "1.46.0")] +impl AsRef<[T]> for IntoIter { + fn as_ref(&self) -> &[T] { + self.as_slice() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +unsafe impl Send for IntoIter {} +#[stable(feature = "rust1", since = "1.0.0")] +unsafe impl Sync for IntoIter {} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for IntoIter { + type Item = T; + + #[inline] + fn next(&mut self) -> Option { + if self.ptr == self.end { + None + } else if T::IS_ZST { + // `ptr` has to stay where it is to remain aligned, so we reduce the length by 1 by + // reducing the `end`. + self.end = self.end.wrapping_byte_sub(1); + + // Make up a value of this ZST. + Some(unsafe { mem::zeroed() }) + } else { + let old = self.ptr; + self.ptr = unsafe { self.ptr.add(1) }; + + Some(unsafe { ptr::read(old) }) + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let exact = if T::IS_ZST { + self.end.addr().wrapping_sub(self.ptr.addr()) + } else { + unsafe { self.end.sub_ptr(self.ptr) } + }; + (exact, Some(exact)) + } + + #[inline] + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + let step_size = self.len().min(n); + let to_drop = ptr::slice_from_raw_parts_mut(self.ptr as *mut T, step_size); + if T::IS_ZST { + // See `next` for why we sub `end` here. + self.end = self.end.wrapping_byte_sub(step_size); + } else { + // SAFETY: the min() above ensures that step_size is in bounds + self.ptr = unsafe { self.ptr.add(step_size) }; + } + // SAFETY: the min() above ensures that step_size is in bounds + unsafe { + ptr::drop_in_place(to_drop); + } + if step_size < n { + return Err(step_size); + } + Ok(()) + } + + #[inline] + fn count(self) -> usize { + self.len() + } + + #[inline] + fn next_chunk(&mut self) -> Result<[T; N], core::array::IntoIter> { + let mut raw_ary = MaybeUninit::uninit_array(); + + let len = self.len(); + + if T::IS_ZST { + if len < N { + self.forget_remaining_elements(); + // Safety: ZSTs can be conjured ex nihilo, only the amount has to be correct + return Err(unsafe { array::IntoIter::new_unchecked(raw_ary, 0..len) }); + } + + self.end = self.end.wrapping_byte_sub(N); + // Safety: ditto + return Ok(unsafe { raw_ary.transpose().assume_init() }); + } + + if len < N { + // Safety: `len` indicates that this many elements are available and we just checked that + // it fits into the array. + unsafe { + ptr::copy_nonoverlapping(self.ptr, raw_ary.as_mut_ptr() as *mut T, len); + self.forget_remaining_elements(); + return Err(array::IntoIter::new_unchecked(raw_ary, 0..len)); + } + } + + // Safety: `len` is larger than the array size. Copy a fixed amount here to fully initialize + // the array. + return unsafe { + ptr::copy_nonoverlapping(self.ptr, raw_ary.as_mut_ptr() as *mut T, N); + self.ptr = self.ptr.add(N); + Ok(raw_ary.transpose().assume_init()) + }; + } + + unsafe fn __iterator_get_unchecked(&mut self, i: usize) -> Self::Item + where + Self: TrustedRandomAccessNoCoerce, + { + // SAFETY: the caller must guarantee that `i` is in bounds of the + // `Vec`, so `i` cannot overflow an `isize`, and the `self.ptr.add(i)` + // is guaranteed to pointer to an element of the `Vec` and + // thus guaranteed to be valid to dereference. + // + // Also note the implementation of `Self: TrustedRandomAccess` requires + // that `T: Copy` so reading elements from the buffer doesn't invalidate + // them for `Drop`. + unsafe { + if T::IS_ZST { mem::zeroed() } else { ptr::read(self.ptr.add(i)) } + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl DoubleEndedIterator for IntoIter { + #[inline] + fn next_back(&mut self) -> Option { + if self.end == self.ptr { + None + } else if T::IS_ZST { + // See above for why 'ptr.offset' isn't used + self.end = self.end.wrapping_byte_sub(1); + + // Make up a value of this ZST. + Some(unsafe { mem::zeroed() }) + } else { + self.end = unsafe { self.end.sub(1) }; + + Some(unsafe { ptr::read(self.end) }) + } + } + + #[inline] + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + let step_size = self.len().min(n); + if T::IS_ZST { + // SAFETY: same as for advance_by() + self.end = self.end.wrapping_byte_sub(step_size); + } else { + // SAFETY: same as for advance_by() + self.end = unsafe { self.end.sub(step_size) }; + } + let to_drop = ptr::slice_from_raw_parts_mut(self.end as *mut T, step_size); + // SAFETY: same as for advance_by() + unsafe { + ptr::drop_in_place(to_drop); + } + if step_size < n { + return Err(step_size); + } + Ok(()) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ExactSizeIterator for IntoIter { + fn is_empty(&self) -> bool { + self.ptr == self.end + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for IntoIter {} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for IntoIter {} + +#[doc(hidden)] +#[unstable(issue = "none", feature = "std_internals")] +#[rustc_unsafe_specialization_marker] +pub trait NonDrop {} + +// T: Copy as approximation for !Drop since get_unchecked does not advance self.ptr +// and thus we can't implement drop-handling +#[unstable(issue = "none", feature = "std_internals")] +impl NonDrop for T {} + +#[doc(hidden)] +#[unstable(issue = "none", feature = "std_internals")] +// TrustedRandomAccess (without NoCoerce) must not be implemented because +// subtypes/supertypes of `T` might not be `NonDrop` +unsafe impl TrustedRandomAccessNoCoerce for IntoIter +where + T: NonDrop, +{ + const MAY_HAVE_SIDE_EFFECT: bool = false; +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "vec_into_iter_clone", since = "1.8.0")] +impl Clone for IntoIter { + #[cfg(not(test))] + fn clone(&self) -> Self { + self.as_slice().to_vec_in(self.alloc.deref().clone()).into_iter() + } + #[cfg(test)] + fn clone(&self) -> Self { + crate::slice::to_vec(self.as_slice(), self.alloc.deref().clone()).into_iter() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +unsafe impl<#[may_dangle] T, A: Allocator> Drop for IntoIter { + fn drop(&mut self) { + struct DropGuard<'a, T, A: Allocator>(&'a mut IntoIter); + + impl Drop for DropGuard<'_, T, A> { + fn drop(&mut self) { + unsafe { + // `IntoIter::alloc` is not used anymore after this and will be dropped by RawVec + let alloc = ManuallyDrop::take(&mut self.0.alloc); + // RawVec handles deallocation + let _ = RawVec::from_raw_parts_in(self.0.buf.as_ptr(), self.0.cap, alloc); + } + } + } + + let guard = DropGuard(self); + // destroy the remaining elements + unsafe { + ptr::drop_in_place(guard.0.as_raw_mut_slice()); + } + // now `guard` will be dropped and do the rest + } +} + +// In addition to the SAFETY invariants of the following three unsafe traits +// also refer to the vec::in_place_collect module documentation to get an overview +#[unstable(issue = "none", feature = "inplace_iteration")] +#[doc(hidden)] +unsafe impl InPlaceIterable for IntoIter {} + +#[unstable(issue = "none", feature = "inplace_iteration")] +#[doc(hidden)] +unsafe impl SourceIter for IntoIter { + type Source = Self; + + #[inline] + unsafe fn as_inner(&mut self) -> &mut Self::Source { + self + } +} + +#[cfg(not(no_global_oom_handling))] +unsafe impl AsVecIntoIter for IntoIter { + type Item = T; + + fn as_into_iter(&mut self) -> &mut IntoIter { + self + } +} diff --git a/crux-mir/lib/alloc/src/vec/is_zero.rs b/crux-mir/lib/alloc/src/vec/is_zero.rs new file mode 100644 index 000000000..cb9adf05c --- /dev/null +++ b/crux-mir/lib/alloc/src/vec/is_zero.rs @@ -0,0 +1,202 @@ +use core::num::{Saturating, Wrapping}; + +use crate::boxed::Box; + +#[rustc_specialization_trait] +pub(super) unsafe trait IsZero { + /// Whether this value's representation is all zeros, + /// or can be represented with all zeroes. + fn is_zero(&self) -> bool; +} + +macro_rules! impl_is_zero { + ($t:ty, $is_zero:expr) => { + unsafe impl IsZero for $t { + #[inline] + fn is_zero(&self) -> bool { + $is_zero(*self) + } + } + }; +} + +impl_is_zero!(i8, |x| x == 0); // It is needed to impl for arrays and tuples of i8. +impl_is_zero!(i16, |x| x == 0); +impl_is_zero!(i32, |x| x == 0); +impl_is_zero!(i64, |x| x == 0); +impl_is_zero!(i128, |x| x == 0); +impl_is_zero!(isize, |x| x == 0); + +impl_is_zero!(u8, |x| x == 0); // It is needed to impl for arrays and tuples of u8. +impl_is_zero!(u16, |x| x == 0); +impl_is_zero!(u32, |x| x == 0); +impl_is_zero!(u64, |x| x == 0); +impl_is_zero!(u128, |x| x == 0); +impl_is_zero!(usize, |x| x == 0); + +impl_is_zero!(bool, |x| x == false); +impl_is_zero!(char, |x| x == '\0'); + +impl_is_zero!(f32, |x: f32| x.to_bits() == 0); +impl_is_zero!(f64, |x: f64| x.to_bits() == 0); + +unsafe impl IsZero for *const T { + #[inline] + fn is_zero(&self) -> bool { + (*self).is_null() + } +} + +unsafe impl IsZero for *mut T { + #[inline] + fn is_zero(&self) -> bool { + (*self).is_null() + } +} + +unsafe impl IsZero for [T; N] { + #[inline] + fn is_zero(&self) -> bool { + // Because this is generated as a runtime check, it's not obvious that + // it's worth doing if the array is really long. The threshold here + // is largely arbitrary, but was picked because as of 2022-07-01 LLVM + // fails to const-fold the check in `vec![[1; 32]; n]` + // See https://github.com/rust-lang/rust/pull/97581#issuecomment-1166628022 + // Feel free to tweak if you have better evidence. + + N <= 16 && self.iter().all(IsZero::is_zero) + } +} + +// This is recursive macro. +macro_rules! impl_for_tuples { + // Stopper + () => { + // No use for implementing for empty tuple because it is ZST. + }; + ($first_arg:ident $(,$rest:ident)*) => { + unsafe impl <$first_arg: IsZero, $($rest: IsZero,)*> IsZero for ($first_arg, $($rest,)*){ + #[inline] + fn is_zero(&self) -> bool{ + // Destructure tuple to N references + // Rust allows to hide generic params by local variable names. + #[allow(non_snake_case)] + let ($first_arg, $($rest,)*) = self; + + $first_arg.is_zero() + $( && $rest.is_zero() )* + } + } + + impl_for_tuples!($($rest),*); + } +} + +impl_for_tuples!(A, B, C, D, E, F, G, H); + +// `Option<&T>` and `Option>` are guaranteed to represent `None` as null. +// For fat pointers, the bytes that would be the pointer metadata in the `Some` +// variant are padding in the `None` variant, so ignoring them and +// zero-initializing instead is ok. +// `Option<&mut T>` never implements `Clone`, so there's no need for an impl of +// `SpecFromElem`. + +unsafe impl IsZero for Option<&T> { + #[inline] + fn is_zero(&self) -> bool { + self.is_none() + } +} + +unsafe impl IsZero for Option> { + #[inline] + fn is_zero(&self) -> bool { + self.is_none() + } +} + +// `Option` and similar have a representation guarantee that +// they're the same size as the corresponding `u32` type, as well as a guarantee +// that transmuting between `NonZeroU32` and `Option` works. +// While the documentation officially makes it UB to transmute from `None`, +// we're the standard library so we can make extra inferences, and we know that +// the only niche available to represent `None` is the one that's all zeros. + +macro_rules! impl_is_zero_option_of_nonzero { + ($($t:ident,)+) => {$( + unsafe impl IsZero for Option { + #[inline] + fn is_zero(&self) -> bool { + self.is_none() + } + } + )+}; +} + +impl_is_zero_option_of_nonzero!( + NonZeroU8, + NonZeroU16, + NonZeroU32, + NonZeroU64, + NonZeroU128, + NonZeroI8, + NonZeroI16, + NonZeroI32, + NonZeroI64, + NonZeroI128, + NonZeroUsize, + NonZeroIsize, +); + +macro_rules! impl_is_zero_option_of_num { + ($($t:ty,)+) => {$( + unsafe impl IsZero for Option<$t> { + #[inline] + fn is_zero(&self) -> bool { + const { + let none: Self = unsafe { core::mem::MaybeUninit::zeroed().assume_init() }; + assert!(none.is_none()); + } + self.is_none() + } + } + )+}; +} + +impl_is_zero_option_of_num!(u8, u16, u32, u64, u128, i8, i16, i32, i64, i128, usize, isize,); + +unsafe impl IsZero for Wrapping { + #[inline] + fn is_zero(&self) -> bool { + self.0.is_zero() + } +} + +unsafe impl IsZero for Saturating { + #[inline] + fn is_zero(&self) -> bool { + self.0.is_zero() + } +} + +macro_rules! impl_for_optional_bool { + ($($t:ty,)+) => {$( + unsafe impl IsZero for $t { + #[inline] + fn is_zero(&self) -> bool { + // SAFETY: This is *not* a stable layout guarantee, but + // inside `core` we're allowed to rely on the current rustc + // behaviour that options of bools will be one byte with + // no padding, so long as they're nested less than 254 deep. + let raw: u8 = unsafe { core::mem::transmute(*self) }; + raw == 0 + } + } + )+}; +} +impl_for_optional_bool! { + Option, + Option>, + Option>>, + // Could go further, but not worth the metadata overhead +} diff --git a/crux-mir/lib/alloc/src/vec/mod.rs b/crux-mir/lib/alloc/src/vec/mod.rs new file mode 100644 index 000000000..36b0b3c9e --- /dev/null +++ b/crux-mir/lib/alloc/src/vec/mod.rs @@ -0,0 +1,3306 @@ +//! A contiguous growable array type with heap-allocated contents, written +//! `Vec`. +//! +//! Vectors have *O*(1) indexing, amortized *O*(1) push (to the end) and +//! *O*(1) pop (from the end). +//! +//! Vectors ensure they never allocate more than `isize::MAX` bytes. +//! +//! # Examples +//! +//! You can explicitly create a [`Vec`] with [`Vec::new`]: +//! +//! ``` +//! let v: Vec = Vec::new(); +//! ``` +//! +//! ...or by using the [`vec!`] macro: +//! +//! ``` +//! let v: Vec = vec![]; +//! +//! let v = vec![1, 2, 3, 4, 5]; +//! +//! let v = vec![0; 10]; // ten zeroes +//! ``` +//! +//! You can [`push`] values onto the end of a vector (which will grow the vector +//! as needed): +//! +//! ``` +//! let mut v = vec![1, 2]; +//! +//! v.push(3); +//! ``` +//! +//! Popping values works in much the same way: +//! +//! ``` +//! let mut v = vec![1, 2]; +//! +//! let two = v.pop(); +//! ``` +//! +//! Vectors also support indexing (through the [`Index`] and [`IndexMut`] traits): +//! +//! ``` +//! let mut v = vec![1, 2, 3]; +//! let three = v[2]; +//! v[1] = v[1] + 5; +//! ``` +//! +//! [`push`]: Vec::push + +#![stable(feature = "rust1", since = "1.0.0")] + +#[cfg(not(no_global_oom_handling))] +use core::cmp; +use core::cmp::Ordering; +use core::convert::TryFrom; +use core::fmt; +use core::hash::{Hash, Hasher}; +use core::intrinsics::assume; +use core::iter; +#[cfg(not(no_global_oom_handling))] +use core::iter::FromIterator; +use core::marker::PhantomData; +use core::mem::{self, ManuallyDrop, MaybeUninit, SizedTypeProperties}; +use core::ops::{self, Index, IndexMut, Range, RangeBounds}; +use core::ptr::{self, NonNull}; +use core::slice::{self, SliceIndex}; + +use crate::alloc::{Allocator, Global}; +use crate::borrow::{Cow, ToOwned}; +use crate::boxed::Box; +use crate::collections::TryReserveError; +use crate::raw_vec::RawVec; + +#[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")] +pub use self::drain_filter::DrainFilter; + +mod drain_filter; + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "vec_splice", since = "1.21.0")] +pub use self::splice::Splice; + +#[cfg(not(no_global_oom_handling))] +mod splice; + +#[stable(feature = "drain", since = "1.6.0")] +pub use self::drain::Drain; + +mod drain; + +#[cfg(not(no_global_oom_handling))] +mod cow; + +#[cfg(not(no_global_oom_handling))] +pub(crate) use self::in_place_collect::AsVecIntoIter; +#[stable(feature = "rust1", since = "1.0.0")] +pub use self::into_iter::IntoIter; + +mod into_iter; + +#[cfg(not(no_global_oom_handling))] +use self::is_zero::IsZero; + +mod is_zero; + +#[cfg(not(no_global_oom_handling))] +mod in_place_collect; + +mod partial_eq; + +#[cfg(not(no_global_oom_handling))] +use self::spec_from_elem::SpecFromElem; + +#[cfg(not(no_global_oom_handling))] +mod spec_from_elem; + +#[cfg(not(no_global_oom_handling))] +use self::set_len_on_drop::SetLenOnDrop; + +#[cfg(not(no_global_oom_handling))] +mod set_len_on_drop; + +#[cfg(not(no_global_oom_handling))] +use self::in_place_drop::{InPlaceDrop, InPlaceDstBufDrop}; + +#[cfg(not(no_global_oom_handling))] +mod in_place_drop; + +#[cfg(not(no_global_oom_handling))] +use self::spec_from_iter_nested::SpecFromIterNested; + +#[cfg(not(no_global_oom_handling))] +mod spec_from_iter_nested; + +#[cfg(not(no_global_oom_handling))] +use self::spec_from_iter::SpecFromIter; + +#[cfg(not(no_global_oom_handling))] +mod spec_from_iter; + +#[cfg(not(no_global_oom_handling))] +use self::spec_extend::SpecExtend; + +#[cfg(not(no_global_oom_handling))] +mod spec_extend; + +/// A contiguous growable array type, written as `Vec`, short for 'vector'. +/// +/// # Examples +/// +/// ``` +/// let mut vec = Vec::new(); +/// vec.push(1); +/// vec.push(2); +/// +/// assert_eq!(vec.len(), 2); +/// assert_eq!(vec[0], 1); +/// +/// assert_eq!(vec.pop(), Some(2)); +/// assert_eq!(vec.len(), 1); +/// +/// vec[0] = 7; +/// assert_eq!(vec[0], 7); +/// +/// vec.extend([1, 2, 3]); +/// +/// for x in &vec { +/// println!("{x}"); +/// } +/// assert_eq!(vec, [7, 1, 2, 3]); +/// ``` +/// +/// The [`vec!`] macro is provided for convenient initialization: +/// +/// ``` +/// let mut vec1 = vec![1, 2, 3]; +/// vec1.push(4); +/// let vec2 = Vec::from([1, 2, 3, 4]); +/// assert_eq!(vec1, vec2); +/// ``` +/// +/// It can also initialize each element of a `Vec` with a given value. +/// This may be more efficient than performing allocation and initialization +/// in separate steps, especially when initializing a vector of zeros: +/// +/// ``` +/// let vec = vec![0; 5]; +/// assert_eq!(vec, [0, 0, 0, 0, 0]); +/// +/// // The following is equivalent, but potentially slower: +/// let mut vec = Vec::with_capacity(5); +/// vec.resize(5, 0); +/// assert_eq!(vec, [0, 0, 0, 0, 0]); +/// ``` +/// +/// For more information, see +/// [Capacity and Reallocation](#capacity-and-reallocation). +/// +/// Use a `Vec` as an efficient stack: +/// +/// ``` +/// let mut stack = Vec::new(); +/// +/// stack.push(1); +/// stack.push(2); +/// stack.push(3); +/// +/// while let Some(top) = stack.pop() { +/// // Prints 3, 2, 1 +/// println!("{top}"); +/// } +/// ``` +/// +/// # Indexing +/// +/// The `Vec` type allows to access values by index, because it implements the +/// [`Index`] trait. An example will be more explicit: +/// +/// ``` +/// let v = vec![0, 2, 4, 6]; +/// println!("{}", v[1]); // it will display '2' +/// ``` +/// +/// However be careful: if you try to access an index which isn't in the `Vec`, +/// your software will panic! You cannot do this: +/// +/// ```should_panic +/// let v = vec![0, 2, 4, 6]; +/// println!("{}", v[6]); // it will panic! +/// ``` +/// +/// Use [`get`] and [`get_mut`] if you want to check whether the index is in +/// the `Vec`. +/// +/// # Slicing +/// +/// A `Vec` can be mutable. On the other hand, slices are read-only objects. +/// To get a [slice][prim@slice], use [`&`]. Example: +/// +/// ``` +/// fn read_slice(slice: &[usize]) { +/// // ... +/// } +/// +/// let v = vec![0, 1]; +/// read_slice(&v); +/// +/// // ... and that's all! +/// // you can also do it like this: +/// let u: &[usize] = &v; +/// // or like this: +/// let u: &[_] = &v; +/// ``` +/// +/// In Rust, it's more common to pass slices as arguments rather than vectors +/// when you just want to provide read access. The same goes for [`String`] and +/// [`&str`]. +/// +/// # Capacity and reallocation +/// +/// The capacity of a vector is the amount of space allocated for any future +/// elements that will be added onto the vector. This is not to be confused with +/// the *length* of a vector, which specifies the number of actual elements +/// within the vector. If a vector's length exceeds its capacity, its capacity +/// will automatically be increased, but its elements will have to be +/// reallocated. +/// +/// For example, a vector with capacity 10 and length 0 would be an empty vector +/// with space for 10 more elements. Pushing 10 or fewer elements onto the +/// vector will not change its capacity or cause reallocation to occur. However, +/// if the vector's length is increased to 11, it will have to reallocate, which +/// can be slow. For this reason, it is recommended to use [`Vec::with_capacity`] +/// whenever possible to specify how big the vector is expected to get. +/// +/// # Guarantees +/// +/// Due to its incredibly fundamental nature, `Vec` makes a lot of guarantees +/// about its design. This ensures that it's as low-overhead as possible in +/// the general case, and can be correctly manipulated in primitive ways +/// by unsafe code. Note that these guarantees refer to an unqualified `Vec`. +/// If additional type parameters are added (e.g., to support custom allocators), +/// overriding their defaults may change the behavior. +/// +/// Most fundamentally, `Vec` is and always will be a (pointer, capacity, length) +/// triplet. No more, no less. The order of these fields is completely +/// unspecified, and you should use the appropriate methods to modify these. +/// The pointer will never be null, so this type is null-pointer-optimized. +/// +/// However, the pointer might not actually point to allocated memory. In particular, +/// if you construct a `Vec` with capacity 0 via [`Vec::new`], [`vec![]`][`vec!`], +/// [`Vec::with_capacity(0)`][`Vec::with_capacity`], or by calling [`shrink_to_fit`] +/// on an empty Vec, it will not allocate memory. Similarly, if you store zero-sized +/// types inside a `Vec`, it will not allocate space for them. *Note that in this case +/// the `Vec` might not report a [`capacity`] of 0*. `Vec` will allocate if and only +/// if [mem::size_of::\]\() * [capacity]\() > 0. In general, `Vec`'s allocation +/// details are very subtle --- if you intend to allocate memory using a `Vec` +/// and use it for something else (either to pass to unsafe code, or to build your +/// own memory-backed collection), be sure to deallocate this memory by using +/// `from_raw_parts` to recover the `Vec` and then dropping it. +/// +/// If a `Vec` *has* allocated memory, then the memory it points to is on the heap +/// (as defined by the allocator Rust is configured to use by default), and its +/// pointer points to [`len`] initialized, contiguous elements in order (what +/// you would see if you coerced it to a slice), followed by [capacity] - [len] +/// logically uninitialized, contiguous elements. +/// +/// A vector containing the elements `'a'` and `'b'` with capacity 4 can be +/// visualized as below. The top part is the `Vec` struct, it contains a +/// pointer to the head of the allocation in the heap, length and capacity. +/// The bottom part is the allocation on the heap, a contiguous memory block. +/// +/// ```text +/// ptr len capacity +/// +--------+--------+--------+ +/// | 0x0123 | 2 | 4 | +/// +--------+--------+--------+ +/// | +/// v +/// Heap +--------+--------+--------+--------+ +/// | 'a' | 'b' | uninit | uninit | +/// +--------+--------+--------+--------+ +/// ``` +/// +/// - **uninit** represents memory that is not initialized, see [`MaybeUninit`]. +/// - Note: the ABI is not stable and `Vec` makes no guarantees about its memory +/// layout (including the order of fields). +/// +/// `Vec` will never perform a "small optimization" where elements are actually +/// stored on the stack for two reasons: +/// +/// * It would make it more difficult for unsafe code to correctly manipulate +/// a `Vec`. The contents of a `Vec` wouldn't have a stable address if it were +/// only moved, and it would be more difficult to determine if a `Vec` had +/// actually allocated memory. +/// +/// * It would penalize the general case, incurring an additional branch +/// on every access. +/// +/// `Vec` will never automatically shrink itself, even if completely empty. This +/// ensures no unnecessary allocations or deallocations occur. Emptying a `Vec` +/// and then filling it back up to the same [`len`] should incur no calls to +/// the allocator. If you wish to free up unused memory, use +/// [`shrink_to_fit`] or [`shrink_to`]. +/// +/// [`push`] and [`insert`] will never (re)allocate if the reported capacity is +/// sufficient. [`push`] and [`insert`] *will* (re)allocate if +/// [len] == [capacity]. That is, the reported capacity is completely +/// accurate, and can be relied on. It can even be used to manually free the memory +/// allocated by a `Vec` if desired. Bulk insertion methods *may* reallocate, even +/// when not necessary. +/// +/// `Vec` does not guarantee any particular growth strategy when reallocating +/// when full, nor when [`reserve`] is called. The current strategy is basic +/// and it may prove desirable to use a non-constant growth factor. Whatever +/// strategy is used will of course guarantee *O*(1) amortized [`push`]. +/// +/// `vec![x; n]`, `vec![a, b, c, d]`, and +/// [`Vec::with_capacity(n)`][`Vec::with_capacity`], will all produce a `Vec` +/// with exactly the requested capacity. If [len] == [capacity], +/// (as is the case for the [`vec!`] macro), then a `Vec` can be converted to +/// and from a [`Box<[T]>`][owned slice] without reallocating or moving the elements. +/// +/// `Vec` will not specifically overwrite any data that is removed from it, +/// but also won't specifically preserve it. Its uninitialized memory is +/// scratch space that it may use however it wants. It will generally just do +/// whatever is most efficient or otherwise easy to implement. Do not rely on +/// removed data to be erased for security purposes. Even if you drop a `Vec`, its +/// buffer may simply be reused by another allocation. Even if you zero a `Vec`'s memory +/// first, that might not actually happen because the optimizer does not consider +/// this a side-effect that must be preserved. There is one case which we will +/// not break, however: using `unsafe` code to write to the excess capacity, +/// and then increasing the length to match, is always valid. +/// +/// Currently, `Vec` does not guarantee the order in which elements are dropped. +/// The order has changed in the past and may change again. +/// +/// [`get`]: ../../std/vec/struct.Vec.html#method.get +/// [`get_mut`]: ../../std/vec/struct.Vec.html#method.get_mut +/// [`String`]: crate::string::String +/// [`&str`]: type@str +/// [`shrink_to_fit`]: Vec::shrink_to_fit +/// [`shrink_to`]: Vec::shrink_to +/// [capacity]: Vec::capacity +/// [`capacity`]: Vec::capacity +/// [mem::size_of::\]: core::mem::size_of +/// [len]: Vec::len +/// [`len`]: Vec::len +/// [`push`]: Vec::push +/// [`insert`]: Vec::insert +/// [`reserve`]: Vec::reserve +/// [`MaybeUninit`]: core::mem::MaybeUninit +/// [owned slice]: Box +#[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "Vec")] +#[rustc_insignificant_dtor] +pub struct Vec { + buf: RawVec, + len: usize, +} + +//////////////////////////////////////////////////////////////////////////////// +// Inherent methods +//////////////////////////////////////////////////////////////////////////////// + +impl Vec { + /// Constructs a new, empty `Vec`. + /// + /// The vector will not allocate until elements are pushed onto it. + /// + /// # Examples + /// + /// ``` + /// # #![allow(unused_mut)] + /// let mut vec: Vec = Vec::new(); + /// ``` + #[inline] + #[rustc_const_stable(feature = "const_vec_new", since = "1.39.0")] + #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + pub const fn new() -> Self { + Vec { buf: RawVec::NEW, len: 0 } + } + + /// Constructs a new, empty `Vec` with at least the specified capacity. + /// + /// The vector will be able to hold at least `capacity` elements without + /// reallocating. This method is allowed to allocate for more elements than + /// `capacity`. If `capacity` is 0, the vector will not allocate. + /// + /// It is important to note that although the returned vector has the + /// minimum *capacity* specified, the vector will have a zero *length*. For + /// an explanation of the difference between length and capacity, see + /// *[Capacity and reallocation]*. + /// + /// If it is important to know the exact allocated capacity of a `Vec`, + /// always use the [`capacity`] method after construction. + /// + /// For `Vec` where `T` is a zero-sized type, there will be no allocation + /// and the capacity will always be `usize::MAX`. + /// + /// [Capacity and reallocation]: #capacity-and-reallocation + /// [`capacity`]: Vec::capacity + /// + /// # Panics + /// + /// Panics if the new capacity exceeds `isize::MAX` bytes. + /// + /// # Examples + /// + /// ``` + /// let mut vec = Vec::with_capacity(10); + /// + /// // The vector contains no items, even though it has capacity for more + /// assert_eq!(vec.len(), 0); + /// assert!(vec.capacity() >= 10); + /// + /// // These are all done without reallocating... + /// for i in 0..10 { + /// vec.push(i); + /// } + /// assert_eq!(vec.len(), 10); + /// assert!(vec.capacity() >= 10); + /// + /// // ...but this may make the vector reallocate + /// vec.push(11); + /// assert_eq!(vec.len(), 11); + /// assert!(vec.capacity() >= 11); + /// + /// // A vector of a zero-sized type will always over-allocate, since no + /// // allocation is necessary + /// let vec_units = Vec::<()>::with_capacity(10); + /// assert_eq!(vec_units.capacity(), usize::MAX); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + pub fn with_capacity(capacity: usize) -> Self { + Self::with_capacity_in(capacity, Global) + } + + /// Creates a `Vec` directly from a pointer, a capacity, and a length. + /// + /// # Safety + /// + /// This is highly unsafe, due to the number of invariants that aren't + /// checked: + /// + /// * `ptr` must have been allocated using the global allocator, such as via + /// the [`alloc::alloc`] function. + /// * `T` needs to have the same alignment as what `ptr` was allocated with. + /// (`T` having a less strict alignment is not sufficient, the alignment really + /// needs to be equal to satisfy the [`dealloc`] requirement that memory must be + /// allocated and deallocated with the same layout.) + /// * The size of `T` times the `capacity` (ie. the allocated size in bytes) needs + /// to be the same size as the pointer was allocated with. (Because similar to + /// alignment, [`dealloc`] must be called with the same layout `size`.) + /// * `length` needs to be less than or equal to `capacity`. + /// * The first `length` values must be properly initialized values of type `T`. + /// * `capacity` needs to be the capacity that the pointer was allocated with. + /// * The allocated size in bytes must be no larger than `isize::MAX`. + /// See the safety documentation of [`pointer::offset`]. + /// + /// These requirements are always upheld by any `ptr` that has been allocated + /// via `Vec`. Other allocation sources are allowed if the invariants are + /// upheld. + /// + /// Violating these may cause problems like corrupting the allocator's + /// internal data structures. For example it is normally **not** safe + /// to build a `Vec` from a pointer to a C `char` array with length + /// `size_t`, doing so is only safe if the array was initially allocated by + /// a `Vec` or `String`. + /// It's also not safe to build one from a `Vec` and its length, because + /// the allocator cares about the alignment, and these two types have different + /// alignments. The buffer was allocated with alignment 2 (for `u16`), but after + /// turning it into a `Vec` it'll be deallocated with alignment 1. To avoid + /// these issues, it is often preferable to do casting/transmuting using + /// [`slice::from_raw_parts`] instead. + /// + /// The ownership of `ptr` is effectively transferred to the + /// `Vec` which may then deallocate, reallocate or change the + /// contents of memory pointed to by the pointer at will. Ensure + /// that nothing else uses the pointer after calling this + /// function. + /// + /// [`String`]: crate::string::String + /// [`alloc::alloc`]: crate::alloc::alloc + /// [`dealloc`]: crate::alloc::GlobalAlloc::dealloc + /// + /// # Examples + /// + /// ``` + /// use std::ptr; + /// use std::mem; + /// + /// let v = vec![1, 2, 3]; + /// + // FIXME Update this when vec_into_raw_parts is stabilized + /// // Prevent running `v`'s destructor so we are in complete control + /// // of the allocation. + /// let mut v = mem::ManuallyDrop::new(v); + /// + /// // Pull out the various important pieces of information about `v` + /// let p = v.as_mut_ptr(); + /// let len = v.len(); + /// let cap = v.capacity(); + /// + /// unsafe { + /// // Overwrite memory with 4, 5, 6 + /// for i in 0..len { + /// ptr::write(p.add(i), 4 + i); + /// } + /// + /// // Put everything back together into a Vec + /// let rebuilt = Vec::from_raw_parts(p, len, cap); + /// assert_eq!(rebuilt, [4, 5, 6]); + /// } + /// ``` + /// + /// Using memory that was allocated elsewhere: + /// + /// ```rust + /// #![feature(allocator_api)] + /// + /// use std::alloc::{AllocError, Allocator, Global, Layout}; + /// + /// fn main() { + /// let layout = Layout::array::(16).expect("overflow cannot happen"); + /// + /// let vec = unsafe { + /// let mem = match Global.allocate(layout) { + /// Ok(mem) => mem.cast::().as_ptr(), + /// Err(AllocError) => return, + /// }; + /// + /// mem.write(1_000_000); + /// + /// Vec::from_raw_parts_in(mem, 1, 16, Global) + /// }; + /// + /// assert_eq!(vec, &[1_000_000]); + /// assert_eq!(vec.capacity(), 16); + /// } + /// ``` + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub unsafe fn from_raw_parts(ptr: *mut T, length: usize, capacity: usize) -> Self { + unsafe { Self::from_raw_parts_in(ptr, length, capacity, Global) } + } +} + +impl Vec { + /// Constructs a new, empty `Vec`. + /// + /// The vector will not allocate until elements are pushed onto it. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api)] + /// + /// use std::alloc::System; + /// + /// # #[allow(unused_mut)] + /// let mut vec: Vec = Vec::new_in(System); + /// ``` + #[inline] + #[unstable(feature = "allocator_api", issue = "32838")] + pub const fn new_in(alloc: A) -> Self { + Vec { buf: RawVec::new_in(alloc), len: 0 } + } + + /// Constructs a new, empty `Vec` with at least the specified capacity + /// with the provided allocator. + /// + /// The vector will be able to hold at least `capacity` elements without + /// reallocating. This method is allowed to allocate for more elements than + /// `capacity`. If `capacity` is 0, the vector will not allocate. + /// + /// It is important to note that although the returned vector has the + /// minimum *capacity* specified, the vector will have a zero *length*. For + /// an explanation of the difference between length and capacity, see + /// *[Capacity and reallocation]*. + /// + /// If it is important to know the exact allocated capacity of a `Vec`, + /// always use the [`capacity`] method after construction. + /// + /// For `Vec` where `T` is a zero-sized type, there will be no allocation + /// and the capacity will always be `usize::MAX`. + /// + /// [Capacity and reallocation]: #capacity-and-reallocation + /// [`capacity`]: Vec::capacity + /// + /// # Panics + /// + /// Panics if the new capacity exceeds `isize::MAX` bytes. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api)] + /// + /// use std::alloc::System; + /// + /// let mut vec = Vec::with_capacity_in(10, System); + /// + /// // The vector contains no items, even though it has capacity for more + /// assert_eq!(vec.len(), 0); + /// assert_eq!(vec.capacity(), 10); + /// + /// // These are all done without reallocating... + /// for i in 0..10 { + /// vec.push(i); + /// } + /// assert_eq!(vec.len(), 10); + /// assert_eq!(vec.capacity(), 10); + /// + /// // ...but this may make the vector reallocate + /// vec.push(11); + /// assert_eq!(vec.len(), 11); + /// assert!(vec.capacity() >= 11); + /// + /// // A vector of a zero-sized type will always over-allocate, since no + /// // allocation is necessary + /// let vec_units = Vec::<(), System>::with_capacity_in(10, System); + /// assert_eq!(vec_units.capacity(), usize::MAX); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[inline] + #[unstable(feature = "allocator_api", issue = "32838")] + pub fn with_capacity_in(capacity: usize, alloc: A) -> Self { + Vec { buf: RawVec::with_capacity_in(capacity, alloc), len: 0 } + } + + /// Creates a `Vec` directly from a pointer, a capacity, a length, + /// and an allocator. + /// + /// # Safety + /// + /// This is highly unsafe, due to the number of invariants that aren't + /// checked: + /// + /// * `ptr` must be [*currently allocated*] via the given allocator `alloc`. + /// * `T` needs to have the same alignment as what `ptr` was allocated with. + /// (`T` having a less strict alignment is not sufficient, the alignment really + /// needs to be equal to satisfy the [`dealloc`] requirement that memory must be + /// allocated and deallocated with the same layout.) + /// * The size of `T` times the `capacity` (ie. the allocated size in bytes) needs + /// to be the same size as the pointer was allocated with. (Because similar to + /// alignment, [`dealloc`] must be called with the same layout `size`.) + /// * `length` needs to be less than or equal to `capacity`. + /// * The first `length` values must be properly initialized values of type `T`. + /// * `capacity` needs to [*fit*] the layout size that the pointer was allocated with. + /// * The allocated size in bytes must be no larger than `isize::MAX`. + /// See the safety documentation of [`pointer::offset`]. + /// + /// These requirements are always upheld by any `ptr` that has been allocated + /// via `Vec`. Other allocation sources are allowed if the invariants are + /// upheld. + /// + /// Violating these may cause problems like corrupting the allocator's + /// internal data structures. For example it is **not** safe + /// to build a `Vec` from a pointer to a C `char` array with length `size_t`. + /// It's also not safe to build one from a `Vec` and its length, because + /// the allocator cares about the alignment, and these two types have different + /// alignments. The buffer was allocated with alignment 2 (for `u16`), but after + /// turning it into a `Vec` it'll be deallocated with alignment 1. + /// + /// The ownership of `ptr` is effectively transferred to the + /// `Vec` which may then deallocate, reallocate or change the + /// contents of memory pointed to by the pointer at will. Ensure + /// that nothing else uses the pointer after calling this + /// function. + /// + /// [`String`]: crate::string::String + /// [`dealloc`]: crate::alloc::GlobalAlloc::dealloc + /// [*currently allocated*]: crate::alloc::Allocator#currently-allocated-memory + /// [*fit*]: crate::alloc::Allocator#memory-fitting + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api)] + /// + /// use std::alloc::System; + /// + /// use std::ptr; + /// use std::mem; + /// + /// let mut v = Vec::with_capacity_in(3, System); + /// v.push(1); + /// v.push(2); + /// v.push(3); + /// + // FIXME Update this when vec_into_raw_parts is stabilized + /// // Prevent running `v`'s destructor so we are in complete control + /// // of the allocation. + /// let mut v = mem::ManuallyDrop::new(v); + /// + /// // Pull out the various important pieces of information about `v` + /// let p = v.as_mut_ptr(); + /// let len = v.len(); + /// let cap = v.capacity(); + /// let alloc = v.allocator(); + /// + /// unsafe { + /// // Overwrite memory with 4, 5, 6 + /// for i in 0..len { + /// ptr::write(p.add(i), 4 + i); + /// } + /// + /// // Put everything back together into a Vec + /// let rebuilt = Vec::from_raw_parts_in(p, len, cap, alloc.clone()); + /// assert_eq!(rebuilt, [4, 5, 6]); + /// } + /// ``` + /// + /// Using memory that was allocated elsewhere: + /// + /// ```rust + /// use std::alloc::{alloc, Layout}; + /// + /// fn main() { + /// let layout = Layout::array::(16).expect("overflow cannot happen"); + /// let vec = unsafe { + /// let mem = alloc(layout).cast::(); + /// if mem.is_null() { + /// return; + /// } + /// + /// mem.write(1_000_000); + /// + /// Vec::from_raw_parts(mem, 1, 16) + /// }; + /// + /// assert_eq!(vec, &[1_000_000]); + /// assert_eq!(vec.capacity(), 16); + /// } + /// ``` + #[inline] + #[unstable(feature = "allocator_api", issue = "32838")] + pub unsafe fn from_raw_parts_in(ptr: *mut T, length: usize, capacity: usize, alloc: A) -> Self { + unsafe { Vec { buf: RawVec::from_raw_parts_in(ptr, capacity, alloc), len: length } } + } + + /// Decomposes a `Vec` into its raw components. + /// + /// Returns the raw pointer to the underlying data, the length of + /// the vector (in elements), and the allocated capacity of the + /// data (in elements). These are the same arguments in the same + /// order as the arguments to [`from_raw_parts`]. + /// + /// After calling this function, the caller is responsible for the + /// memory previously managed by the `Vec`. The only way to do + /// this is to convert the raw pointer, length, and capacity back + /// into a `Vec` with the [`from_raw_parts`] function, allowing + /// the destructor to perform the cleanup. + /// + /// [`from_raw_parts`]: Vec::from_raw_parts + /// + /// # Examples + /// + /// ``` + /// #![feature(vec_into_raw_parts)] + /// let v: Vec = vec![-1, 0, 1]; + /// + /// let (ptr, len, cap) = v.into_raw_parts(); + /// + /// let rebuilt = unsafe { + /// // We can now make changes to the components, such as + /// // transmuting the raw pointer to a compatible type. + /// let ptr = ptr as *mut u32; + /// + /// Vec::from_raw_parts(ptr, len, cap) + /// }; + /// assert_eq!(rebuilt, [4294967295, 0, 1]); + /// ``` + #[unstable(feature = "vec_into_raw_parts", reason = "new API", issue = "65816")] + pub fn into_raw_parts(self) -> (*mut T, usize, usize) { + let mut me = ManuallyDrop::new(self); + (me.as_mut_ptr(), me.len(), me.capacity()) + } + + /// Decomposes a `Vec` into its raw components. + /// + /// Returns the raw pointer to the underlying data, the length of the vector (in elements), + /// the allocated capacity of the data (in elements), and the allocator. These are the same + /// arguments in the same order as the arguments to [`from_raw_parts_in`]. + /// + /// After calling this function, the caller is responsible for the + /// memory previously managed by the `Vec`. The only way to do + /// this is to convert the raw pointer, length, and capacity back + /// into a `Vec` with the [`from_raw_parts_in`] function, allowing + /// the destructor to perform the cleanup. + /// + /// [`from_raw_parts_in`]: Vec::from_raw_parts_in + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, vec_into_raw_parts)] + /// + /// use std::alloc::System; + /// + /// let mut v: Vec = Vec::new_in(System); + /// v.push(-1); + /// v.push(0); + /// v.push(1); + /// + /// let (ptr, len, cap, alloc) = v.into_raw_parts_with_alloc(); + /// + /// let rebuilt = unsafe { + /// // We can now make changes to the components, such as + /// // transmuting the raw pointer to a compatible type. + /// let ptr = ptr as *mut u32; + /// + /// Vec::from_raw_parts_in(ptr, len, cap, alloc) + /// }; + /// assert_eq!(rebuilt, [4294967295, 0, 1]); + /// ``` + #[unstable(feature = "allocator_api", issue = "32838")] + // #[unstable(feature = "vec_into_raw_parts", reason = "new API", issue = "65816")] + pub fn into_raw_parts_with_alloc(self) -> (*mut T, usize, usize, A) { + let mut me = ManuallyDrop::new(self); + let len = me.len(); + let capacity = me.capacity(); + let ptr = me.as_mut_ptr(); + let alloc = unsafe { ptr::read(me.allocator()) }; + (ptr, len, capacity, alloc) + } + + /// Returns the total number of elements the vector can hold without + /// reallocating. + /// + /// # Examples + /// + /// ``` + /// let mut vec: Vec = Vec::with_capacity(10); + /// vec.push(42); + /// assert_eq!(vec.capacity(), 10); + /// ``` + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn capacity(&self) -> usize { + self.buf.capacity() + } + + /// Reserves capacity for at least `additional` more elements to be inserted + /// in the given `Vec`. The collection may reserve more space to + /// speculatively avoid frequent reallocations. After calling `reserve`, + /// capacity will be greater than or equal to `self.len() + additional`. + /// Does nothing if capacity is already sufficient. + /// + /// # Panics + /// + /// Panics if the new capacity exceeds `isize::MAX` bytes. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1]; + /// vec.reserve(10); + /// assert!(vec.capacity() >= 11); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn reserve(&mut self, additional: usize) { + self.buf.reserve(self.len, additional); + } + + /// Reserves the minimum capacity for at least `additional` more elements to + /// be inserted in the given `Vec`. Unlike [`reserve`], this will not + /// deliberately over-allocate to speculatively avoid frequent allocations. + /// After calling `reserve_exact`, capacity will be greater than or equal to + /// `self.len() + additional`. Does nothing if the capacity is already + /// sufficient. + /// + /// Note that the allocator may give the collection more space than it + /// requests. Therefore, capacity can not be relied upon to be precisely + /// minimal. Prefer [`reserve`] if future insertions are expected. + /// + /// [`reserve`]: Vec::reserve + /// + /// # Panics + /// + /// Panics if the new capacity exceeds `isize::MAX` bytes. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1]; + /// vec.reserve_exact(10); + /// assert!(vec.capacity() >= 11); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn reserve_exact(&mut self, additional: usize) { + self.buf.reserve_exact(self.len, additional); + } + + /// Tries to reserve capacity for at least `additional` more elements to be inserted + /// in the given `Vec`. The collection may reserve more space to speculatively avoid + /// frequent reallocations. After calling `try_reserve`, capacity will be + /// greater than or equal to `self.len() + additional` if it returns + /// `Ok(())`. Does nothing if capacity is already sufficient. This method + /// preserves the contents even if an error occurs. + /// + /// # Errors + /// + /// If the capacity overflows, or the allocator reports a failure, then an error + /// is returned. + /// + /// # Examples + /// + /// ``` + /// use std::collections::TryReserveError; + /// + /// fn process_data(data: &[u32]) -> Result, TryReserveError> { + /// let mut output = Vec::new(); + /// + /// // Pre-reserve the memory, exiting if we can't + /// output.try_reserve(data.len())?; + /// + /// // Now we know this can't OOM in the middle of our complex work + /// output.extend(data.iter().map(|&val| { + /// val * 2 + 5 // very complicated + /// })); + /// + /// Ok(output) + /// } + /// # process_data(&[1, 2, 3]).expect("why is the test harness OOMing on 12 bytes?"); + /// ``` + #[stable(feature = "try_reserve", since = "1.57.0")] + pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.buf.try_reserve(self.len, additional) + } + + /// Tries to reserve the minimum capacity for at least `additional` + /// elements to be inserted in the given `Vec`. Unlike [`try_reserve`], + /// this will not deliberately over-allocate to speculatively avoid frequent + /// allocations. After calling `try_reserve_exact`, capacity will be greater + /// than or equal to `self.len() + additional` if it returns `Ok(())`. + /// Does nothing if the capacity is already sufficient. + /// + /// Note that the allocator may give the collection more space than it + /// requests. Therefore, capacity can not be relied upon to be precisely + /// minimal. Prefer [`try_reserve`] if future insertions are expected. + /// + /// [`try_reserve`]: Vec::try_reserve + /// + /// # Errors + /// + /// If the capacity overflows, or the allocator reports a failure, then an error + /// is returned. + /// + /// # Examples + /// + /// ``` + /// use std::collections::TryReserveError; + /// + /// fn process_data(data: &[u32]) -> Result, TryReserveError> { + /// let mut output = Vec::new(); + /// + /// // Pre-reserve the memory, exiting if we can't + /// output.try_reserve_exact(data.len())?; + /// + /// // Now we know this can't OOM in the middle of our complex work + /// output.extend(data.iter().map(|&val| { + /// val * 2 + 5 // very complicated + /// })); + /// + /// Ok(output) + /// } + /// # process_data(&[1, 2, 3]).expect("why is the test harness OOMing on 12 bytes?"); + /// ``` + #[stable(feature = "try_reserve", since = "1.57.0")] + pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.buf.try_reserve_exact(self.len, additional) + } + + /// Shrinks the capacity of the vector as much as possible. + /// + /// It will drop down as close as possible to the length but the allocator + /// may still inform the vector that there is space for a few more elements. + /// + /// # Examples + /// + /// ``` + /// let mut vec = Vec::with_capacity(10); + /// vec.extend([1, 2, 3]); + /// assert_eq!(vec.capacity(), 10); + /// vec.shrink_to_fit(); + /// assert!(vec.capacity() >= 3); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn shrink_to_fit(&mut self) { + // The capacity is never less than the length, and there's nothing to do when + // they are equal, so we can avoid the panic case in `RawVec::shrink_to_fit` + // by only calling it with a greater capacity. + if self.capacity() > self.len { + self.buf.shrink_to_fit(self.len); + } + } + + /// Shrinks the capacity of the vector with a lower bound. + /// + /// The capacity will remain at least as large as both the length + /// and the supplied value. + /// + /// If the current capacity is less than the lower limit, this is a no-op. + /// + /// # Examples + /// + /// ``` + /// let mut vec = Vec::with_capacity(10); + /// vec.extend([1, 2, 3]); + /// assert_eq!(vec.capacity(), 10); + /// vec.shrink_to(4); + /// assert!(vec.capacity() >= 4); + /// vec.shrink_to(0); + /// assert!(vec.capacity() >= 3); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "shrink_to", since = "1.56.0")] + pub fn shrink_to(&mut self, min_capacity: usize) { + if self.capacity() > min_capacity { + self.buf.shrink_to_fit(cmp::max(self.len, min_capacity)); + } + } + + /// Converts the vector into [`Box<[T]>`][owned slice]. + /// + /// If the vector has excess capacity, its items will be moved into a + /// newly-allocated buffer with exactly the right capacity. + /// + /// [owned slice]: Box + /// + /// # Examples + /// + /// ``` + /// let v = vec![1, 2, 3]; + /// + /// let slice = v.into_boxed_slice(); + /// ``` + /// + /// Any excess capacity is removed: + /// + /// ``` + /// let mut vec = Vec::with_capacity(10); + /// vec.extend([1, 2, 3]); + /// + /// assert_eq!(vec.capacity(), 10); + /// let slice = vec.into_boxed_slice(); + /// assert_eq!(slice.into_vec().capacity(), 3); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn into_boxed_slice(mut self) -> Box<[T], A> { + unsafe { + self.shrink_to_fit(); + let me = ManuallyDrop::new(self); + let buf = ptr::read(&me.buf); + let len = me.len(); + buf.into_box(len).assume_init() + } + } + + /// Shortens the vector, keeping the first `len` elements and dropping + /// the rest. + /// + /// If `len` is greater than the vector's current length, this has no + /// effect. + /// + /// The [`drain`] method can emulate `truncate`, but causes the excess + /// elements to be returned instead of dropped. + /// + /// Note that this method has no effect on the allocated capacity + /// of the vector. + /// + /// # Examples + /// + /// Truncating a five element vector to two elements: + /// + /// ``` + /// let mut vec = vec![1, 2, 3, 4, 5]; + /// vec.truncate(2); + /// assert_eq!(vec, [1, 2]); + /// ``` + /// + /// No truncation occurs when `len` is greater than the vector's current + /// length: + /// + /// ``` + /// let mut vec = vec![1, 2, 3]; + /// vec.truncate(8); + /// assert_eq!(vec, [1, 2, 3]); + /// ``` + /// + /// Truncating when `len == 0` is equivalent to calling the [`clear`] + /// method. + /// + /// ``` + /// let mut vec = vec![1, 2, 3]; + /// vec.truncate(0); + /// assert_eq!(vec, []); + /// ``` + /// + /// [`clear`]: Vec::clear + /// [`drain`]: Vec::drain + #[stable(feature = "rust1", since = "1.0.0")] + pub fn truncate(&mut self, len: usize) { + // This is safe because: + // + // * the slice passed to `drop_in_place` is valid; the `len > self.len` + // case avoids creating an invalid slice, and + // * the `len` of the vector is shrunk before calling `drop_in_place`, + // such that no value will be dropped twice in case `drop_in_place` + // were to panic once (if it panics twice, the program aborts). + unsafe { + // Note: It's intentional that this is `>` and not `>=`. + // Changing it to `>=` has negative performance + // implications in some cases. See #78884 for more. + if len > self.len { + return; + } + let remaining_len = self.len - len; + let s = ptr::slice_from_raw_parts_mut(self.as_mut_ptr().add(len), remaining_len); + self.len = len; + ptr::drop_in_place(s); + } + } + + /// Extracts a slice containing the entire vector. + /// + /// Equivalent to `&s[..]`. + /// + /// # Examples + /// + /// ``` + /// use std::io::{self, Write}; + /// let buffer = vec![1, 2, 3, 5, 8]; + /// io::sink().write(buffer.as_slice()).unwrap(); + /// ``` + #[inline] + #[stable(feature = "vec_as_slice", since = "1.7.0")] + pub fn as_slice(&self) -> &[T] { + self + } + + /// Extracts a mutable slice of the entire vector. + /// + /// Equivalent to `&mut s[..]`. + /// + /// # Examples + /// + /// ``` + /// use std::io::{self, Read}; + /// let mut buffer = vec![0; 3]; + /// io::repeat(0b101).read_exact(buffer.as_mut_slice()).unwrap(); + /// ``` + #[inline] + #[stable(feature = "vec_as_slice", since = "1.7.0")] + pub fn as_mut_slice(&mut self) -> &mut [T] { + self + } + + /// Returns a raw pointer to the vector's buffer, or a dangling raw pointer + /// valid for zero sized reads if the vector didn't allocate. + /// + /// The caller must ensure that the vector outlives the pointer this + /// function returns, or else it will end up pointing to garbage. + /// Modifying the vector may cause its buffer to be reallocated, + /// which would also make any pointers to it invalid. + /// + /// The caller must also ensure that the memory the pointer (non-transitively) points to + /// is never written to (except inside an `UnsafeCell`) using this pointer or any pointer + /// derived from it. If you need to mutate the contents of the slice, use [`as_mut_ptr`]. + /// + /// # Examples + /// + /// ``` + /// let x = vec![1, 2, 4]; + /// let x_ptr = x.as_ptr(); + /// + /// unsafe { + /// for i in 0..x.len() { + /// assert_eq!(*x_ptr.add(i), 1 << i); + /// } + /// } + /// ``` + /// + /// [`as_mut_ptr`]: Vec::as_mut_ptr + #[stable(feature = "vec_as_ptr", since = "1.37.0")] + #[inline] + pub fn as_ptr(&self) -> *const T { + // We shadow the slice method of the same name to avoid going through + // `deref`, which creates an intermediate reference. + let ptr = self.buf.ptr(); + unsafe { + assume(!ptr.is_null()); + } + ptr + } + + /// Returns an unsafe mutable pointer to the vector's buffer, or a dangling + /// raw pointer valid for zero sized reads if the vector didn't allocate. + /// + /// The caller must ensure that the vector outlives the pointer this + /// function returns, or else it will end up pointing to garbage. + /// Modifying the vector may cause its buffer to be reallocated, + /// which would also make any pointers to it invalid. + /// + /// # Examples + /// + /// ``` + /// // Allocate vector big enough for 4 elements. + /// let size = 4; + /// let mut x: Vec = Vec::with_capacity(size); + /// let x_ptr = x.as_mut_ptr(); + /// + /// // Initialize elements via raw pointer writes, then set length. + /// unsafe { + /// for i in 0..size { + /// *x_ptr.add(i) = i as i32; + /// } + /// x.set_len(size); + /// } + /// assert_eq!(&*x, &[0, 1, 2, 3]); + /// ``` + #[stable(feature = "vec_as_ptr", since = "1.37.0")] + #[inline] + pub fn as_mut_ptr(&mut self) -> *mut T { + // We shadow the slice method of the same name to avoid going through + // `deref_mut`, which creates an intermediate reference. + let ptr = self.buf.ptr(); + unsafe { + assume(!ptr.is_null()); + } + ptr + } + + /// Returns a reference to the underlying allocator. + #[unstable(feature = "allocator_api", issue = "32838")] + #[inline] + pub fn allocator(&self) -> &A { + self.buf.allocator() + } + + /// Forces the length of the vector to `new_len`. + /// + /// This is a low-level operation that maintains none of the normal + /// invariants of the type. Normally changing the length of a vector + /// is done using one of the safe operations instead, such as + /// [`truncate`], [`resize`], [`extend`], or [`clear`]. + /// + /// [`truncate`]: Vec::truncate + /// [`resize`]: Vec::resize + /// [`extend`]: Extend::extend + /// [`clear`]: Vec::clear + /// + /// # Safety + /// + /// - `new_len` must be less than or equal to [`capacity()`]. + /// - The elements at `old_len..new_len` must be initialized. + /// + /// [`capacity()`]: Vec::capacity + /// + /// # Examples + /// + /// This method can be useful for situations in which the vector + /// is serving as a buffer for other code, particularly over FFI: + /// + /// ```no_run + /// # #![allow(dead_code)] + /// # // This is just a minimal skeleton for the doc example; + /// # // don't use this as a starting point for a real library. + /// # pub struct StreamWrapper { strm: *mut std::ffi::c_void } + /// # const Z_OK: i32 = 0; + /// # extern "C" { + /// # fn deflateGetDictionary( + /// # strm: *mut std::ffi::c_void, + /// # dictionary: *mut u8, + /// # dictLength: *mut usize, + /// # ) -> i32; + /// # } + /// # impl StreamWrapper { + /// pub fn get_dictionary(&self) -> Option> { + /// // Per the FFI method's docs, "32768 bytes is always enough". + /// let mut dict = Vec::with_capacity(32_768); + /// let mut dict_length = 0; + /// // SAFETY: When `deflateGetDictionary` returns `Z_OK`, it holds that: + /// // 1. `dict_length` elements were initialized. + /// // 2. `dict_length` <= the capacity (32_768) + /// // which makes `set_len` safe to call. + /// unsafe { + /// // Make the FFI call... + /// let r = deflateGetDictionary(self.strm, dict.as_mut_ptr(), &mut dict_length); + /// if r == Z_OK { + /// // ...and update the length to what was initialized. + /// dict.set_len(dict_length); + /// Some(dict) + /// } else { + /// None + /// } + /// } + /// } + /// # } + /// ``` + /// + /// While the following example is sound, there is a memory leak since + /// the inner vectors were not freed prior to the `set_len` call: + /// + /// ``` + /// let mut vec = vec![vec![1, 0, 0], + /// vec![0, 1, 0], + /// vec![0, 0, 1]]; + /// // SAFETY: + /// // 1. `old_len..0` is empty so no elements need to be initialized. + /// // 2. `0 <= capacity` always holds whatever `capacity` is. + /// unsafe { + /// vec.set_len(0); + /// } + /// ``` + /// + /// Normally, here, one would use [`clear`] instead to correctly drop + /// the contents and thus not leak memory. + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub unsafe fn set_len(&mut self, new_len: usize) { + debug_assert!(new_len <= self.capacity()); + + self.len = new_len; + } + + /// Removes an element from the vector and returns it. + /// + /// The removed element is replaced by the last element of the vector. + /// + /// This does not preserve ordering, but is *O*(1). + /// If you need to preserve the element order, use [`remove`] instead. + /// + /// [`remove`]: Vec::remove + /// + /// # Panics + /// + /// Panics if `index` is out of bounds. + /// + /// # Examples + /// + /// ``` + /// let mut v = vec!["foo", "bar", "baz", "qux"]; + /// + /// assert_eq!(v.swap_remove(1), "bar"); + /// assert_eq!(v, ["foo", "qux", "baz"]); + /// + /// assert_eq!(v.swap_remove(0), "foo"); + /// assert_eq!(v, ["baz", "qux"]); + /// ``` + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn swap_remove(&mut self, index: usize) -> T { + #[cold] + #[inline(never)] + fn assert_failed(index: usize, len: usize) -> ! { + panic!("swap_remove index (is {index}) should be < len (is {len})"); + } + + let len = self.len(); + if index >= len { + assert_failed(index, len); + } + unsafe { + // We replace self[index] with the last element. Note that if the + // bounds check above succeeds there must be a last element (which + // can be self[index] itself). + let value = ptr::read(self.as_ptr().add(index)); + let base_ptr = self.as_mut_ptr(); + ptr::copy(base_ptr.add(len - 1), base_ptr.add(index), 1); + self.set_len(len - 1); + value + } + } + + /// Inserts an element at position `index` within the vector, shifting all + /// elements after it to the right. + /// + /// # Panics + /// + /// Panics if `index > len`. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1, 2, 3]; + /// vec.insert(1, 4); + /// assert_eq!(vec, [1, 4, 2, 3]); + /// vec.insert(4, 5); + /// assert_eq!(vec, [1, 4, 2, 3, 5]); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn insert(&mut self, index: usize, element: T) { + #[cold] + #[inline(never)] + fn assert_failed(index: usize, len: usize) -> ! { + panic!("insertion index (is {index}) should be <= len (is {len})"); + } + + let len = self.len(); + + // space for the new element + if len == self.buf.capacity() { + self.reserve(1); + } + + unsafe { + // infallible + // The spot to put the new value + { + let p = self.as_mut_ptr().add(index); + if index < len { + // Shift everything over to make space. (Duplicating the + // `index`th element into two consecutive places.) + ptr::copy(p, p.add(1), len - index); + } else if index == len { + // No elements need shifting. + } else { + assert_failed(index, len); + } + // Write it in, overwriting the first copy of the `index`th + // element. + ptr::write(p, element); + } + self.set_len(len + 1); + } + } + + /// Removes and returns the element at position `index` within the vector, + /// shifting all elements after it to the left. + /// + /// Note: Because this shifts over the remaining elements, it has a + /// worst-case performance of *O*(*n*). If you don't need the order of elements + /// to be preserved, use [`swap_remove`] instead. If you'd like to remove + /// elements from the beginning of the `Vec`, consider using + /// [`VecDeque::pop_front`] instead. + /// + /// [`swap_remove`]: Vec::swap_remove + /// [`VecDeque::pop_front`]: crate::collections::VecDeque::pop_front + /// + /// # Panics + /// + /// Panics if `index` is out of bounds. + /// + /// # Examples + /// + /// ``` + /// let mut v = vec![1, 2, 3]; + /// assert_eq!(v.remove(1), 2); + /// assert_eq!(v, [1, 3]); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[track_caller] + pub fn remove(&mut self, index: usize) -> T { + #[cold] + #[inline(never)] + #[track_caller] + fn assert_failed(index: usize, len: usize) -> ! { + panic!("removal index (is {index}) should be < len (is {len})"); + } + + let len = self.len(); + if index >= len { + assert_failed(index, len); + } + unsafe { + // infallible + let ret; + { + // the place we are taking from. + let ptr = self.as_mut_ptr().add(index); + // copy it out, unsafely having a copy of the value on + // the stack and in the vector at the same time. + ret = ptr::read(ptr); + + // Shift everything down to fill in that spot. + ptr::copy(ptr.add(1), ptr, len - index - 1); + } + self.set_len(len - 1); + ret + } + } + + /// Retains only the elements specified by the predicate. + /// + /// In other words, remove all elements `e` for which `f(&e)` returns `false`. + /// This method operates in place, visiting each element exactly once in the + /// original order, and preserves the order of the retained elements. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1, 2, 3, 4]; + /// vec.retain(|&x| x % 2 == 0); + /// assert_eq!(vec, [2, 4]); + /// ``` + /// + /// Because the elements are visited exactly once in the original order, + /// external state may be used to decide which elements to keep. + /// + /// ``` + /// let mut vec = vec![1, 2, 3, 4, 5]; + /// let keep = [false, true, true, false, true]; + /// let mut iter = keep.iter(); + /// vec.retain(|_| *iter.next().unwrap()); + /// assert_eq!(vec, [2, 3, 5]); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn retain(&mut self, mut f: F) + where + F: FnMut(&T) -> bool, + { + self.retain_mut(|elem| f(elem)); + } + + /// Retains only the elements specified by the predicate, passing a mutable reference to it. + /// + /// In other words, remove all elements `e` such that `f(&mut e)` returns `false`. + /// This method operates in place, visiting each element exactly once in the + /// original order, and preserves the order of the retained elements. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1, 2, 3, 4]; + /// vec.retain_mut(|x| if *x <= 3 { + /// *x += 1; + /// true + /// } else { + /// false + /// }); + /// assert_eq!(vec, [2, 3, 4]); + /// ``` + #[stable(feature = "vec_retain_mut", since = "1.61.0")] + pub fn retain_mut(&mut self, mut f: F) + where + F: FnMut(&mut T) -> bool, + { + let original_len = self.len(); + // Avoid double drop if the drop guard is not executed, + // since we may make some holes during the process. + unsafe { self.set_len(0) }; + + // Vec: [Kept, Kept, Hole, Hole, Hole, Hole, Unchecked, Unchecked] + // |<- processed len ->| ^- next to check + // |<- deleted cnt ->| + // |<- original_len ->| + // Kept: Elements which predicate returns true on. + // Hole: Moved or dropped element slot. + // Unchecked: Unchecked valid elements. + // + // This drop guard will be invoked when predicate or `drop` of element panicked. + // It shifts unchecked elements to cover holes and `set_len` to the correct length. + // In cases when predicate and `drop` never panick, it will be optimized out. + struct BackshiftOnDrop<'a, T, A: Allocator> { + v: &'a mut Vec, + processed_len: usize, + deleted_cnt: usize, + original_len: usize, + } + + impl Drop for BackshiftOnDrop<'_, T, A> { + fn drop(&mut self) { + if self.deleted_cnt > 0 { + // SAFETY: Trailing unchecked items must be valid since we never touch them. + unsafe { + ptr::copy( + self.v.as_ptr().add(self.processed_len), + self.v.as_mut_ptr().add(self.processed_len - self.deleted_cnt), + self.original_len - self.processed_len, + ); + } + } + // SAFETY: After filling holes, all items are in contiguous memory. + unsafe { + self.v.set_len(self.original_len - self.deleted_cnt); + } + } + } + + let mut g = BackshiftOnDrop { v: self, processed_len: 0, deleted_cnt: 0, original_len }; + + fn process_loop( + original_len: usize, + f: &mut F, + g: &mut BackshiftOnDrop<'_, T, A>, + ) where + F: FnMut(&mut T) -> bool, + { + while g.processed_len != original_len { + // SAFETY: Unchecked element must be valid. + let cur = unsafe { &mut *g.v.as_mut_ptr().add(g.processed_len) }; + if !f(cur) { + // Advance early to avoid double drop if `drop_in_place` panicked. + g.processed_len += 1; + g.deleted_cnt += 1; + // SAFETY: We never touch this element again after dropped. + unsafe { ptr::drop_in_place(cur) }; + // We already advanced the counter. + if DELETED { + continue; + } else { + break; + } + } + if DELETED { + // SAFETY: `deleted_cnt` > 0, so the hole slot must not overlap with current element. + // We use copy for move, and never touch this element again. + unsafe { + let hole_slot = g.v.as_mut_ptr().add(g.processed_len - g.deleted_cnt); + ptr::copy_nonoverlapping(cur, hole_slot, 1); + } + } + g.processed_len += 1; + } + } + + // Stage 1: Nothing was deleted. + process_loop::(original_len, &mut f, &mut g); + + // Stage 2: Some elements were deleted. + process_loop::(original_len, &mut f, &mut g); + + // All item are processed. This can be optimized to `set_len` by LLVM. + drop(g); + } + + /// Removes all but the first of consecutive elements in the vector that resolve to the same + /// key. + /// + /// If the vector is sorted, this removes all duplicates. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![10, 20, 21, 30, 20]; + /// + /// vec.dedup_by_key(|i| *i / 10); + /// + /// assert_eq!(vec, [10, 20, 30, 20]); + /// ``` + #[stable(feature = "dedup_by", since = "1.16.0")] + #[inline] + pub fn dedup_by_key(&mut self, mut key: F) + where + F: FnMut(&mut T) -> K, + K: PartialEq, + { + self.dedup_by(|a, b| key(a) == key(b)) + } + + /// Removes all but the first of consecutive elements in the vector satisfying a given equality + /// relation. + /// + /// The `same_bucket` function is passed references to two elements from the vector and + /// must determine if the elements compare equal. The elements are passed in opposite order + /// from their order in the slice, so if `same_bucket(a, b)` returns `true`, `a` is removed. + /// + /// If the vector is sorted, this removes all duplicates. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec!["foo", "bar", "Bar", "baz", "bar"]; + /// + /// vec.dedup_by(|a, b| a.eq_ignore_ascii_case(b)); + /// + /// assert_eq!(vec, ["foo", "bar", "baz", "bar"]); + /// ``` + #[stable(feature = "dedup_by", since = "1.16.0")] + pub fn dedup_by(&mut self, mut same_bucket: F) + where + F: FnMut(&mut T, &mut T) -> bool, + { + let len = self.len(); + if len <= 1 { + return; + } + + /* INVARIANT: vec.len() > read >= write > write-1 >= 0 */ + struct FillGapOnDrop<'a, T, A: core::alloc::Allocator> { + /* Offset of the element we want to check if it is duplicate */ + read: usize, + + /* Offset of the place where we want to place the non-duplicate + * when we find it. */ + write: usize, + + /* The Vec that would need correction if `same_bucket` panicked */ + vec: &'a mut Vec, + } + + impl<'a, T, A: core::alloc::Allocator> Drop for FillGapOnDrop<'a, T, A> { + fn drop(&mut self) { + /* This code gets executed when `same_bucket` panics */ + + /* SAFETY: invariant guarantees that `read - write` + * and `len - read` never overflow and that the copy is always + * in-bounds. */ + unsafe { + let ptr = self.vec.as_mut_ptr(); + let len = self.vec.len(); + + /* How many items were left when `same_bucket` panicked. + * Basically vec[read..].len() */ + let items_left = len.wrapping_sub(self.read); + + /* Pointer to first item in vec[write..write+items_left] slice */ + let dropped_ptr = ptr.add(self.write); + /* Pointer to first item in vec[read..] slice */ + let valid_ptr = ptr.add(self.read); + + /* Copy `vec[read..]` to `vec[write..write+items_left]`. + * The slices can overlap, so `copy_nonoverlapping` cannot be used */ + ptr::copy(valid_ptr, dropped_ptr, items_left); + + /* How many items have been already dropped + * Basically vec[read..write].len() */ + let dropped = self.read.wrapping_sub(self.write); + + self.vec.set_len(len - dropped); + } + } + } + + let mut gap = FillGapOnDrop { read: 1, write: 1, vec: self }; + let ptr = gap.vec.as_mut_ptr(); + + /* Drop items while going through Vec, it should be more efficient than + * doing slice partition_dedup + truncate */ + + /* SAFETY: Because of the invariant, read_ptr, prev_ptr and write_ptr + * are always in-bounds and read_ptr never aliases prev_ptr */ + unsafe { + while gap.read < len { + let read_ptr = ptr.add(gap.read); + let prev_ptr = ptr.add(gap.write.wrapping_sub(1)); + + if same_bucket(&mut *read_ptr, &mut *prev_ptr) { + // Increase `gap.read` now since the drop may panic. + gap.read += 1; + /* We have found duplicate, drop it in-place */ + ptr::drop_in_place(read_ptr); + } else { + let write_ptr = ptr.add(gap.write); + + /* Because `read_ptr` can be equal to `write_ptr`, we either + * have to use `copy` or conditional `copy_nonoverlapping`. + * Looks like the first option is faster. */ + ptr::copy(read_ptr, write_ptr, 1); + + /* We have filled that place, so go further */ + gap.write += 1; + gap.read += 1; + } + } + + /* Technically we could let `gap` clean up with its Drop, but + * when `same_bucket` is guaranteed to not panic, this bloats a little + * the codegen, so we just do it manually */ + gap.vec.set_len(gap.write); + mem::forget(gap); + } + } + + /// Appends an element to the back of a collection. + /// + /// # Panics + /// + /// Panics if the new capacity exceeds `isize::MAX` bytes. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1, 2]; + /// vec.push(3); + /// assert_eq!(vec, [1, 2, 3]); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn push(&mut self, value: T) { + // This will panic or abort if we would allocate > isize::MAX bytes + // or if the length increment would overflow for zero-sized types. + if self.len == self.buf.capacity() { + self.buf.reserve_for_push(self.len); + } + unsafe { + let end = self.as_mut_ptr().add(self.len); + ptr::write(end, value); + self.len += 1; + } + } + + /// Appends an element if there is sufficient spare capacity, otherwise an error is returned + /// with the element. + /// + /// Unlike [`push`] this method will not reallocate when there's insufficient capacity. + /// The caller should use [`reserve`] or [`try_reserve`] to ensure that there is enough capacity. + /// + /// [`push`]: Vec::push + /// [`reserve`]: Vec::reserve + /// [`try_reserve`]: Vec::try_reserve + /// + /// # Examples + /// + /// A manual, panic-free alternative to [`FromIterator`]: + /// + /// ``` + /// #![feature(vec_push_within_capacity)] + /// + /// use std::collections::TryReserveError; + /// fn from_iter_fallible(iter: impl Iterator) -> Result, TryReserveError> { + /// let mut vec = Vec::new(); + /// for value in iter { + /// if let Err(value) = vec.push_within_capacity(value) { + /// vec.try_reserve(1)?; + /// // this cannot fail, the previous line either returned or added at least 1 free slot + /// let _ = vec.push_within_capacity(value); + /// } + /// } + /// Ok(vec) + /// } + /// assert_eq!(from_iter_fallible(0..100), Ok(Vec::from_iter(0..100))); + /// ``` + #[inline] + #[unstable(feature = "vec_push_within_capacity", issue = "100486")] + pub fn push_within_capacity(&mut self, value: T) -> Result<(), T> { + if self.len == self.buf.capacity() { + return Err(value); + } + unsafe { + let end = self.as_mut_ptr().add(self.len); + ptr::write(end, value); + self.len += 1; + } + Ok(()) + } + + /// Removes the last element from a vector and returns it, or [`None`] if it + /// is empty. + /// + /// If you'd like to pop the first element, consider using + /// [`VecDeque::pop_front`] instead. + /// + /// [`VecDeque::pop_front`]: crate::collections::VecDeque::pop_front + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1, 2, 3]; + /// assert_eq!(vec.pop(), Some(3)); + /// assert_eq!(vec, [1, 2]); + /// ``` + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn pop(&mut self) -> Option { + if self.len == 0 { + None + } else { + unsafe { + self.len -= 1; + Some(ptr::read(self.as_ptr().add(self.len()))) + } + } + } + + /// Moves all the elements of `other` into `self`, leaving `other` empty. + /// + /// # Panics + /// + /// Panics if the new capacity exceeds `isize::MAX` bytes. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1, 2, 3]; + /// let mut vec2 = vec![4, 5, 6]; + /// vec.append(&mut vec2); + /// assert_eq!(vec, [1, 2, 3, 4, 5, 6]); + /// assert_eq!(vec2, []); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[inline] + #[stable(feature = "append", since = "1.4.0")] + pub fn append(&mut self, other: &mut Self) { + unsafe { + self.append_elements(other.as_slice() as _); + other.set_len(0); + } + } + + /// Appends elements to `self` from other buffer. + #[cfg(not(no_global_oom_handling))] + #[inline] + unsafe fn append_elements(&mut self, other: *const [T]) { + let count = unsafe { (*other).len() }; + self.reserve(count); + let len = self.len(); + unsafe { ptr::copy_nonoverlapping(other as *const T, self.as_mut_ptr().add(len), count) }; + self.len += count; + } + + /// Removes the specified range from the vector in bulk, returning all + /// removed elements as an iterator. If the iterator is dropped before + /// being fully consumed, it drops the remaining removed elements. + /// + /// The returned iterator keeps a mutable borrow on the vector to optimize + /// its implementation. + /// + /// # Panics + /// + /// Panics if the starting point is greater than the end point or if + /// the end point is greater than the length of the vector. + /// + /// # Leaking + /// + /// If the returned iterator goes out of scope without being dropped (due to + /// [`mem::forget`], for example), the vector may have lost and leaked + /// elements arbitrarily, including elements outside the range. + /// + /// # Examples + /// + /// ``` + /// let mut v = vec![1, 2, 3]; + /// let u: Vec<_> = v.drain(1..).collect(); + /// assert_eq!(v, &[1]); + /// assert_eq!(u, &[2, 3]); + /// + /// // A full range clears the vector, like `clear()` does + /// v.drain(..); + /// assert_eq!(v, &[]); + /// ``` + #[stable(feature = "drain", since = "1.6.0")] + pub fn drain(&mut self, range: R) -> Drain<'_, T, A> + where + R: RangeBounds, + { + // Memory safety + // + // When the Drain is first created, it shortens the length of + // the source vector to make sure no uninitialized or moved-from elements + // are accessible at all if the Drain's destructor never gets to run. + // + // Drain will ptr::read out the values to remove. + // When finished, remaining tail of the vec is copied back to cover + // the hole, and the vector length is restored to the new length. + // + let len = self.len(); + let Range { start, end } = slice::range(range, ..len); + + unsafe { + // set self.vec length's to start, to be safe in case Drain is leaked + self.set_len(start); + let range_slice = slice::from_raw_parts(self.as_ptr().add(start), end - start); + Drain { + tail_start: end, + tail_len: len - end, + iter: range_slice.iter(), + vec: NonNull::from(self), + } + } + } + + /// Clears the vector, removing all values. + /// + /// Note that this method has no effect on the allocated capacity + /// of the vector. + /// + /// # Examples + /// + /// ``` + /// let mut v = vec![1, 2, 3]; + /// + /// v.clear(); + /// + /// assert!(v.is_empty()); + /// ``` + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn clear(&mut self) { + let elems: *mut [T] = self.as_mut_slice(); + + // SAFETY: + // - `elems` comes directly from `as_mut_slice` and is therefore valid. + // - Setting `self.len` before calling `drop_in_place` means that, + // if an element's `Drop` impl panics, the vector's `Drop` impl will + // do nothing (leaking the rest of the elements) instead of dropping + // some twice. + unsafe { + self.len = 0; + ptr::drop_in_place(elems); + } + } + + /// Returns the number of elements in the vector, also referred to + /// as its 'length'. + /// + /// # Examples + /// + /// ``` + /// let a = vec![1, 2, 3]; + /// assert_eq!(a.len(), 3); + /// ``` + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn len(&self) -> usize { + self.len + } + + /// Returns `true` if the vector contains no elements. + /// + /// # Examples + /// + /// ``` + /// let mut v = Vec::new(); + /// assert!(v.is_empty()); + /// + /// v.push(1); + /// assert!(!v.is_empty()); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Splits the collection into two at the given index. + /// + /// Returns a newly allocated vector containing the elements in the range + /// `[at, len)`. After the call, the original vector will be left containing + /// the elements `[0, at)` with its previous capacity unchanged. + /// + /// # Panics + /// + /// Panics if `at > len`. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1, 2, 3]; + /// let vec2 = vec.split_off(1); + /// assert_eq!(vec, [1]); + /// assert_eq!(vec2, [2, 3]); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[inline] + #[must_use = "use `.truncate()` if you don't need the other half"] + #[stable(feature = "split_off", since = "1.4.0")] + pub fn split_off(&mut self, at: usize) -> Self + where + A: Clone, + { + #[cold] + #[inline(never)] + fn assert_failed(at: usize, len: usize) -> ! { + panic!("`at` split index (is {at}) should be <= len (is {len})"); + } + + if at > self.len() { + assert_failed(at, self.len()); + } + + if at == 0 { + // the new vector can take over the original buffer and avoid the copy + return mem::replace( + self, + Vec::with_capacity_in(self.capacity(), self.allocator().clone()), + ); + } + + let other_len = self.len - at; + let mut other = Vec::with_capacity_in(other_len, self.allocator().clone()); + + // Unsafely `set_len` and copy items to `other`. + unsafe { + self.set_len(at); + other.set_len(other_len); + + ptr::copy_nonoverlapping(self.as_ptr().add(at), other.as_mut_ptr(), other.len()); + } + other + } + + /// Resizes the `Vec` in-place so that `len` is equal to `new_len`. + /// + /// If `new_len` is greater than `len`, the `Vec` is extended by the + /// difference, with each additional slot filled with the result of + /// calling the closure `f`. The return values from `f` will end up + /// in the `Vec` in the order they have been generated. + /// + /// If `new_len` is less than `len`, the `Vec` is simply truncated. + /// + /// This method uses a closure to create new values on every push. If + /// you'd rather [`Clone`] a given value, use [`Vec::resize`]. If you + /// want to use the [`Default`] trait to generate values, you can + /// pass [`Default::default`] as the second argument. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1, 2, 3]; + /// vec.resize_with(5, Default::default); + /// assert_eq!(vec, [1, 2, 3, 0, 0]); + /// + /// let mut vec = vec![]; + /// let mut p = 1; + /// vec.resize_with(4, || { p *= 2; p }); + /// assert_eq!(vec, [2, 4, 8, 16]); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "vec_resize_with", since = "1.33.0")] + pub fn resize_with(&mut self, new_len: usize, f: F) + where + F: FnMut() -> T, + { + let len = self.len(); + if new_len > len { + self.extend_trusted(iter::repeat_with(f).take(new_len - len)); + } else { + self.truncate(new_len); + } + } + + /// Consumes and leaks the `Vec`, returning a mutable reference to the contents, + /// `&'a mut [T]`. Note that the type `T` must outlive the chosen lifetime + /// `'a`. If the type has only static references, or none at all, then this + /// may be chosen to be `'static`. + /// + /// As of Rust 1.57, this method does not reallocate or shrink the `Vec`, + /// so the leaked allocation may include unused capacity that is not part + /// of the returned slice. + /// + /// This function is mainly useful for data that lives for the remainder of + /// the program's life. Dropping the returned reference will cause a memory + /// leak. + /// + /// # Examples + /// + /// Simple usage: + /// + /// ``` + /// let x = vec![1, 2, 3]; + /// let static_ref: &'static mut [usize] = x.leak(); + /// static_ref[0] += 1; + /// assert_eq!(static_ref, &[2, 2, 3]); + /// ``` + #[stable(feature = "vec_leak", since = "1.47.0")] + #[inline] + pub fn leak<'a>(self) -> &'a mut [T] + where + A: 'a, + { + let mut me = ManuallyDrop::new(self); + unsafe { slice::from_raw_parts_mut(me.as_mut_ptr(), me.len) } + } + + /// Returns the remaining spare capacity of the vector as a slice of + /// `MaybeUninit`. + /// + /// The returned slice can be used to fill the vector with data (e.g. by + /// reading from a file) before marking the data as initialized using the + /// [`set_len`] method. + /// + /// [`set_len`]: Vec::set_len + /// + /// # Examples + /// + /// ``` + /// // Allocate vector big enough for 10 elements. + /// let mut v = Vec::with_capacity(10); + /// + /// // Fill in the first 3 elements. + /// let uninit = v.spare_capacity_mut(); + /// uninit[0].write(0); + /// uninit[1].write(1); + /// uninit[2].write(2); + /// + /// // Mark the first 3 elements of the vector as being initialized. + /// unsafe { + /// v.set_len(3); + /// } + /// + /// assert_eq!(&v, &[0, 1, 2]); + /// ``` + #[stable(feature = "vec_spare_capacity", since = "1.60.0")] + #[inline] + pub fn spare_capacity_mut(&mut self) -> &mut [MaybeUninit] { + // Note: + // This method is not implemented in terms of `split_at_spare_mut`, + // to prevent invalidation of pointers to the buffer. + unsafe { + slice::from_raw_parts_mut( + self.as_mut_ptr().add(self.len) as *mut MaybeUninit, + self.buf.capacity() - self.len, + ) + } + } + + /// Returns vector content as a slice of `T`, along with the remaining spare + /// capacity of the vector as a slice of `MaybeUninit`. + /// + /// The returned spare capacity slice can be used to fill the vector with data + /// (e.g. by reading from a file) before marking the data as initialized using + /// the [`set_len`] method. + /// + /// [`set_len`]: Vec::set_len + /// + /// Note that this is a low-level API, which should be used with care for + /// optimization purposes. If you need to append data to a `Vec` + /// you can use [`push`], [`extend`], [`extend_from_slice`], + /// [`extend_from_within`], [`insert`], [`append`], [`resize`] or + /// [`resize_with`], depending on your exact needs. + /// + /// [`push`]: Vec::push + /// [`extend`]: Vec::extend + /// [`extend_from_slice`]: Vec::extend_from_slice + /// [`extend_from_within`]: Vec::extend_from_within + /// [`insert`]: Vec::insert + /// [`append`]: Vec::append + /// [`resize`]: Vec::resize + /// [`resize_with`]: Vec::resize_with + /// + /// # Examples + /// + /// ``` + /// #![feature(vec_split_at_spare)] + /// + /// let mut v = vec![1, 1, 2]; + /// + /// // Reserve additional space big enough for 10 elements. + /// v.reserve(10); + /// + /// let (init, uninit) = v.split_at_spare_mut(); + /// let sum = init.iter().copied().sum::(); + /// + /// // Fill in the next 4 elements. + /// uninit[0].write(sum); + /// uninit[1].write(sum * 2); + /// uninit[2].write(sum * 3); + /// uninit[3].write(sum * 4); + /// + /// // Mark the 4 elements of the vector as being initialized. + /// unsafe { + /// let len = v.len(); + /// v.set_len(len + 4); + /// } + /// + /// assert_eq!(&v, &[1, 1, 2, 4, 8, 12, 16]); + /// ``` + #[unstable(feature = "vec_split_at_spare", issue = "81944")] + #[inline] + pub fn split_at_spare_mut(&mut self) -> (&mut [T], &mut [MaybeUninit]) { + // SAFETY: + // - len is ignored and so never changed + let (init, spare, _) = unsafe { self.split_at_spare_mut_with_len() }; + (init, spare) + } + + /// Safety: changing returned .2 (&mut usize) is considered the same as calling `.set_len(_)`. + /// + /// This method provides unique access to all vec parts at once in `extend_from_within`. + unsafe fn split_at_spare_mut_with_len( + &mut self, + ) -> (&mut [T], &mut [MaybeUninit], &mut usize) { + let ptr = self.as_mut_ptr(); + // SAFETY: + // - `ptr` is guaranteed to be valid for `self.len` elements + // - but the allocation extends out to `self.buf.capacity()` elements, possibly + // uninitialized + let spare_ptr = unsafe { ptr.add(self.len) }; + let spare_ptr = spare_ptr.cast::>(); + let spare_len = self.buf.capacity() - self.len; + + // SAFETY: + // - `ptr` is guaranteed to be valid for `self.len` elements + // - `spare_ptr` is pointing one element past the buffer, so it doesn't overlap with `initialized` + unsafe { + let initialized = slice::from_raw_parts_mut(ptr, self.len); + let spare = slice::from_raw_parts_mut(spare_ptr, spare_len); + + (initialized, spare, &mut self.len) + } + } +} + +impl Vec { + /// Resizes the `Vec` in-place so that `len` is equal to `new_len`. + /// + /// If `new_len` is greater than `len`, the `Vec` is extended by the + /// difference, with each additional slot filled with `value`. + /// If `new_len` is less than `len`, the `Vec` is simply truncated. + /// + /// This method requires `T` to implement [`Clone`], + /// in order to be able to clone the passed value. + /// If you need more flexibility (or want to rely on [`Default`] instead of + /// [`Clone`]), use [`Vec::resize_with`]. + /// If you only need to resize to a smaller size, use [`Vec::truncate`]. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec!["hello"]; + /// vec.resize(3, "world"); + /// assert_eq!(vec, ["hello", "world", "world"]); + /// + /// let mut vec = vec![1, 2, 3, 4]; + /// vec.resize(2, 0); + /// assert_eq!(vec, [1, 2]); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "vec_resize", since = "1.5.0")] + pub fn resize(&mut self, new_len: usize, value: T) { + let len = self.len(); + + if new_len > len { + self.extend_with(new_len - len, ExtendElement(value)) + } else { + self.truncate(new_len); + } + } + + /// Clones and appends all elements in a slice to the `Vec`. + /// + /// Iterates over the slice `other`, clones each element, and then appends + /// it to this `Vec`. The `other` slice is traversed in-order. + /// + /// Note that this function is same as [`extend`] except that it is + /// specialized to work with slices instead. If and when Rust gets + /// specialization this function will likely be deprecated (but still + /// available). + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1]; + /// vec.extend_from_slice(&[2, 3, 4]); + /// assert_eq!(vec, [1, 2, 3, 4]); + /// ``` + /// + /// [`extend`]: Vec::extend + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "vec_extend_from_slice", since = "1.6.0")] + pub fn extend_from_slice(&mut self, other: &[T]) { + self.spec_extend(other.iter()) + } + + /// Copies elements from `src` range to the end of the vector. + /// + /// # Panics + /// + /// Panics if the starting point is greater than the end point or if + /// the end point is greater than the length of the vector. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![0, 1, 2, 3, 4]; + /// + /// vec.extend_from_within(2..); + /// assert_eq!(vec, [0, 1, 2, 3, 4, 2, 3, 4]); + /// + /// vec.extend_from_within(..2); + /// assert_eq!(vec, [0, 1, 2, 3, 4, 2, 3, 4, 0, 1]); + /// + /// vec.extend_from_within(4..8); + /// assert_eq!(vec, [0, 1, 2, 3, 4, 2, 3, 4, 0, 1, 4, 2, 3, 4]); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "vec_extend_from_within", since = "1.53.0")] + pub fn extend_from_within(&mut self, src: R) + where + R: RangeBounds, + { + let range = slice::range(src, ..self.len()); + self.reserve(range.len()); + + // SAFETY: + // - `slice::range` guarantees that the given range is valid for indexing self + unsafe { + self.spec_extend_from_within(range); + } + } +} + +impl Vec<[T; N], A> { + /// Takes a `Vec<[T; N]>` and flattens it into a `Vec`. + /// + /// # Panics + /// + /// Panics if the length of the resulting vector would overflow a `usize`. + /// + /// This is only possible when flattening a vector of arrays of zero-sized + /// types, and thus tends to be irrelevant in practice. If + /// `size_of::() > 0`, this will never panic. + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_flatten)] + /// + /// let mut vec = vec![[1, 2, 3], [4, 5, 6], [7, 8, 9]]; + /// assert_eq!(vec.pop(), Some([7, 8, 9])); + /// + /// let mut flattened = vec.into_flattened(); + /// assert_eq!(flattened.pop(), Some(6)); + /// ``` + #[unstable(feature = "slice_flatten", issue = "95629")] + pub fn into_flattened(self) -> Vec { + let (ptr, len, cap, alloc) = self.into_raw_parts_with_alloc(); + let (new_len, new_cap) = if T::IS_ZST { + (len.checked_mul(N).expect("vec len overflow"), usize::MAX) + } else { + // SAFETY: + // - `cap * N` cannot overflow because the allocation is already in + // the address space. + // - Each `[T; N]` has `N` valid elements, so there are `len * N` + // valid elements in the allocation. + unsafe { (len.unchecked_mul(N), cap.unchecked_mul(N)) } + }; + // SAFETY: + // - `ptr` was allocated by `self` + // - `ptr` is well-aligned because `[T; N]` has the same alignment as `T`. + // - `new_cap` refers to the same sized allocation as `cap` because + // `new_cap * size_of::()` == `cap * size_of::<[T; N]>()` + // - `len` <= `cap`, so `len * N` <= `cap * N`. + unsafe { Vec::::from_raw_parts_in(ptr.cast(), new_len, new_cap, alloc) } + } +} + +// This code generalizes `extend_with_{element,default}`. +trait ExtendWith { + fn next(&mut self) -> T; + fn last(self) -> T; +} + +struct ExtendElement(T); +impl ExtendWith for ExtendElement { + fn next(&mut self) -> T { + self.0.clone() + } + fn last(self) -> T { + self.0 + } +} + +impl Vec { + #[cfg(not(no_global_oom_handling))] + /// Extend the vector by `n` values, using the given generator. + fn extend_with>(&mut self, n: usize, mut value: E) { + self.reserve(n); + + unsafe { + let mut ptr = self.as_mut_ptr().add(self.len()); + // Use SetLenOnDrop to work around bug where compiler + // might not realize the store through `ptr` through self.set_len() + // don't alias. + let mut local_len = SetLenOnDrop::new(&mut self.len); + + // Write all elements except the last one + for _ in 1..n { + ptr::write(ptr, value.next()); + ptr = ptr.add(1); + // Increment the length in every step in case next() panics + local_len.increment_len(1); + } + + if n > 0 { + // We can write the last element directly without cloning needlessly + ptr::write(ptr, value.last()); + local_len.increment_len(1); + } + + // len set by scope guard + } + } +} + +impl Vec { + /// Removes consecutive repeated elements in the vector according to the + /// [`PartialEq`] trait implementation. + /// + /// If the vector is sorted, this removes all duplicates. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1, 2, 2, 3, 2]; + /// + /// vec.dedup(); + /// + /// assert_eq!(vec, [1, 2, 3, 2]); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[inline] + pub fn dedup(&mut self) { + self.dedup_by(|a, b| a == b) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Internal methods and functions +//////////////////////////////////////////////////////////////////////////////// + +#[doc(hidden)] +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +pub fn from_elem(elem: T, n: usize) -> Vec { + ::from_elem(elem, n, Global) +} + +#[doc(hidden)] +#[cfg(not(no_global_oom_handling))] +#[unstable(feature = "allocator_api", issue = "32838")] +pub fn from_elem_in(elem: T, n: usize, alloc: A) -> Vec { + ::from_elem(elem, n, alloc) +} + +trait ExtendFromWithinSpec { + /// # Safety + /// + /// - `src` needs to be valid index + /// - `self.capacity() - self.len()` must be `>= src.len()` + unsafe fn spec_extend_from_within(&mut self, src: Range); +} + +impl ExtendFromWithinSpec for Vec { + default unsafe fn spec_extend_from_within(&mut self, src: Range) { + // SAFETY: + // - len is increased only after initializing elements + let (this, spare, len) = unsafe { self.split_at_spare_mut_with_len() }; + + // SAFETY: + // - caller guarantees that src is a valid index + let to_clone = unsafe { this.get_unchecked(src) }; + + iter::zip(to_clone, spare) + .map(|(src, dst)| dst.write(src.clone())) + // Note: + // - Element was just initialized with `MaybeUninit::write`, so it's ok to increase len + // - len is increased after each element to prevent leaks (see issue #82533) + .for_each(|_| *len += 1); + } +} + +impl ExtendFromWithinSpec for Vec { + unsafe fn spec_extend_from_within(&mut self, src: Range) { + let count = src.len(); + { + let (init, spare) = self.split_at_spare_mut(); + + // SAFETY: + // - caller guarantees that `src` is a valid index + let source = unsafe { init.get_unchecked(src) }; + + // SAFETY: + // - Both pointers are created from unique slice references (`&mut [_]`) + // so they are valid and do not overlap. + // - Elements are :Copy so it's OK to copy them, without doing + // anything with the original values + // - `count` is equal to the len of `source`, so source is valid for + // `count` reads + // - `.reserve(count)` guarantees that `spare.len() >= count` so spare + // is valid for `count` writes + unsafe { ptr::copy_nonoverlapping(source.as_ptr(), spare.as_mut_ptr() as _, count) }; + } + + // SAFETY: + // - The elements were just initialized by `copy_nonoverlapping` + self.len += count; + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Common trait implementations for Vec +//////////////////////////////////////////////////////////////////////////////// + +#[stable(feature = "rust1", since = "1.0.0")] +impl ops::Deref for Vec { + type Target = [T]; + + #[inline] + fn deref(&self) -> &[T] { + unsafe { slice::from_raw_parts(self.as_ptr(), self.len) } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ops::DerefMut for Vec { + #[inline] + fn deref_mut(&mut self) -> &mut [T] { + unsafe { slice::from_raw_parts_mut(self.as_mut_ptr(), self.len) } + } +} + +#[cfg(not(no_global_oom_handling))] +trait SpecCloneFrom { + fn clone_from(this: &mut Self, other: &Self); +} + +#[cfg(not(no_global_oom_handling))] +impl SpecCloneFrom for Vec { + default fn clone_from(this: &mut Self, other: &Self) { + // drop anything that will not be overwritten + this.truncate(other.len()); + + // self.len <= other.len due to the truncate above, so the + // slices here are always in-bounds. + let (init, tail) = other.split_at(this.len()); + + // reuse the contained values' allocations/resources. + this.clone_from_slice(init); + this.extend_from_slice(tail); + } +} + +#[cfg(not(no_global_oom_handling))] +impl SpecCloneFrom for Vec { + fn clone_from(this: &mut Self, other: &Self) { + this.clear(); + this.extend_from_slice(other); + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +impl Clone for Vec { + #[cfg(not(test))] + fn clone(&self) -> Self { + let alloc = self.allocator().clone(); + <[T]>::to_vec_in(&**self, alloc) + } + + // HACK(japaric): with cfg(test) the inherent `[T]::to_vec` method, which is + // required for this method definition, is not available. Instead use the + // `slice::to_vec` function which is only available with cfg(test) + // NB see the slice::hack module in slice.rs for more information + #[cfg(test)] + fn clone(&self) -> Self { + let alloc = self.allocator().clone(); + crate::slice::to_vec(&**self, alloc) + } + + fn clone_from(&mut self, other: &Self) { + SpecCloneFrom::clone_from(self, other) + } +} + +/// The hash of a vector is the same as that of the corresponding slice, +/// as required by the `core::borrow::Borrow` implementation. +/// +/// ``` +/// #![feature(build_hasher_simple_hash_one)] +/// use std::hash::BuildHasher; +/// +/// let b = std::collections::hash_map::RandomState::new(); +/// let v: Vec = vec![0xa8, 0x3c, 0x09]; +/// let s: &[u8] = &[0xa8, 0x3c, 0x09]; +/// assert_eq!(b.hash_one(v), b.hash_one(s)); +/// ``` +#[stable(feature = "rust1", since = "1.0.0")] +impl Hash for Vec { + #[inline] + fn hash(&self, state: &mut H) { + Hash::hash(&**self, state) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_on_unimplemented( + message = "vector indices are of type `usize` or ranges of `usize`", + label = "vector indices are of type `usize` or ranges of `usize`" +)] +impl, A: Allocator> Index for Vec { + type Output = I::Output; + + #[inline] + fn index(&self, index: I) -> &Self::Output { + Index::index(&**self, index) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_on_unimplemented( + message = "vector indices are of type `usize` or ranges of `usize`", + label = "vector indices are of type `usize` or ranges of `usize`" +)] +impl, A: Allocator> IndexMut for Vec { + #[inline] + fn index_mut(&mut self, index: I) -> &mut Self::Output { + IndexMut::index_mut(&mut **self, index) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +impl FromIterator for Vec { + #[inline] + fn from_iter>(iter: I) -> Vec { + >::from_iter(iter.into_iter()) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl IntoIterator for Vec { + type Item = T; + type IntoIter = IntoIter; + + /// Creates a consuming iterator, that is, one that moves each value out of + /// the vector (from start to end). The vector cannot be used after calling + /// this. + /// + /// # Examples + /// + /// ``` + /// let v = vec!["a".to_string(), "b".to_string()]; + /// let mut v_iter = v.into_iter(); + /// + /// let first_element: Option = v_iter.next(); + /// + /// assert_eq!(first_element, Some("a".to_string())); + /// assert_eq!(v_iter.next(), Some("b".to_string())); + /// assert_eq!(v_iter.next(), None); + /// ``` + #[inline] + fn into_iter(self) -> Self::IntoIter { + unsafe { + let mut me = ManuallyDrop::new(self); + let alloc = ManuallyDrop::new(ptr::read(me.allocator())); + let begin = me.as_mut_ptr(); + let end = if T::IS_ZST { + begin.wrapping_byte_add(me.len()) + } else { + begin.add(me.len()) as *const T + }; + let cap = me.buf.capacity(); + IntoIter { + buf: NonNull::new_unchecked(begin), + phantom: PhantomData, + cap, + alloc, + ptr: begin, + end, + } + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T, A: Allocator> IntoIterator for &'a Vec { + type Item = &'a T; + type IntoIter = slice::Iter<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T, A: Allocator> IntoIterator for &'a mut Vec { + type Item = &'a mut T; + type IntoIter = slice::IterMut<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.iter_mut() + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +impl Extend for Vec { + #[inline] + fn extend>(&mut self, iter: I) { + >::spec_extend(self, iter.into_iter()) + } + + #[inline] + fn extend_one(&mut self, item: T) { + self.push(item); + } + + #[inline] + fn extend_reserve(&mut self, additional: usize) { + self.reserve(additional); + } +} + +impl Vec { + // leaf method to which various SpecFrom/SpecExtend implementations delegate when + // they have no further optimizations to apply + #[cfg(not(no_global_oom_handling))] + fn extend_desugared>(&mut self, mut iterator: I) { + // This is the case for a general iterator. + // + // This function should be the moral equivalent of: + // + // for item in iterator { + // self.push(item); + // } + while let Some(element) = iterator.next() { + let len = self.len(); + if len == self.capacity() { + let (lower, _) = iterator.size_hint(); + self.reserve(lower.saturating_add(1)); + } + unsafe { + ptr::write(self.as_mut_ptr().add(len), element); + // Since next() executes user code which can panic we have to bump the length + // after each step. + // NB can't overflow since we would have had to alloc the address space + self.set_len(len + 1); + } + } + } + + // specific extend for `TrustedLen` iterators, called both by the specializations + // and internal places where resolving specialization makes compilation slower + #[cfg(not(no_global_oom_handling))] + fn extend_trusted(&mut self, iterator: impl iter::TrustedLen) { + let (low, high) = iterator.size_hint(); + if let Some(additional) = high { + debug_assert_eq!( + low, + additional, + "TrustedLen iterator's size hint is not exact: {:?}", + (low, high) + ); + self.reserve(additional); + unsafe { + let ptr = self.as_mut_ptr(); + let mut local_len = SetLenOnDrop::new(&mut self.len); + iterator.for_each(move |element| { + ptr::write(ptr.add(local_len.current_len()), element); + // Since the loop executes user code which can panic we have to update + // the length every step to correctly drop what we've written. + // NB can't overflow since we would have had to alloc the address space + local_len.increment_len(1); + }); + } + } else { + // Per TrustedLen contract a `None` upper bound means that the iterator length + // truly exceeds usize::MAX, which would eventually lead to a capacity overflow anyway. + // Since the other branch already panics eagerly (via `reserve()`) we do the same here. + // This avoids additional codegen for a fallback code path which would eventually + // panic anyway. + panic!("capacity overflow"); + } + } + + /// Creates a splicing iterator that replaces the specified range in the vector + /// with the given `replace_with` iterator and yields the removed items. + /// `replace_with` does not need to be the same length as `range`. + /// + /// `range` is removed even if the iterator is not consumed until the end. + /// + /// It is unspecified how many elements are removed from the vector + /// if the `Splice` value is leaked. + /// + /// The input iterator `replace_with` is only consumed when the `Splice` value is dropped. + /// + /// This is optimal if: + /// + /// * The tail (elements in the vector after `range`) is empty, + /// * or `replace_with` yields fewer or equal elements than `range`’s length + /// * or the lower bound of its `size_hint()` is exact. + /// + /// Otherwise, a temporary vector is allocated and the tail is moved twice. + /// + /// # Panics + /// + /// Panics if the starting point is greater than the end point or if + /// the end point is greater than the length of the vector. + /// + /// # Examples + /// + /// ``` + /// let mut v = vec![1, 2, 3, 4]; + /// let new = [7, 8, 9]; + /// let u: Vec<_> = v.splice(1..3, new).collect(); + /// assert_eq!(v, &[1, 7, 8, 9, 4]); + /// assert_eq!(u, &[2, 3]); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[inline] + #[stable(feature = "vec_splice", since = "1.21.0")] + pub fn splice(&mut self, range: R, replace_with: I) -> Splice<'_, I::IntoIter, A> + where + R: RangeBounds, + I: IntoIterator, + { + Splice { drain: self.drain(range), replace_with: replace_with.into_iter() } + } + + /// Creates an iterator which uses a closure to determine if an element should be removed. + /// + /// If the closure returns true, then the element is removed and yielded. + /// If the closure returns false, the element will remain in the vector and will not be yielded + /// by the iterator. + /// + /// Using this method is equivalent to the following code: + /// + /// ``` + /// # let some_predicate = |x: &mut i32| { *x == 2 || *x == 3 || *x == 6 }; + /// # let mut vec = vec![1, 2, 3, 4, 5, 6]; + /// let mut i = 0; + /// while i < vec.len() { + /// if some_predicate(&mut vec[i]) { + /// let val = vec.remove(i); + /// // your code here + /// } else { + /// i += 1; + /// } + /// } + /// + /// # assert_eq!(vec, vec![1, 4, 5]); + /// ``` + /// + /// But `drain_filter` is easier to use. `drain_filter` is also more efficient, + /// because it can backshift the elements of the array in bulk. + /// + /// Note that `drain_filter` also lets you mutate every element in the filter closure, + /// regardless of whether you choose to keep or remove it. + /// + /// # Examples + /// + /// Splitting an array into evens and odds, reusing the original allocation: + /// + /// ``` + /// #![feature(drain_filter)] + /// let mut numbers = vec![1, 2, 3, 4, 5, 6, 8, 9, 11, 13, 14, 15]; + /// + /// let evens = numbers.drain_filter(|x| *x % 2 == 0).collect::>(); + /// let odds = numbers; + /// + /// assert_eq!(evens, vec![2, 4, 6, 8, 14]); + /// assert_eq!(odds, vec![1, 3, 5, 9, 11, 13, 15]); + /// ``` + #[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")] + pub fn drain_filter(&mut self, filter: F) -> DrainFilter<'_, T, F, A> + where + F: FnMut(&mut T) -> bool, + { + let old_len = self.len(); + + // Guard against us getting leaked (leak amplification) + unsafe { + self.set_len(0); + } + + DrainFilter { vec: self, idx: 0, del: 0, old_len, pred: filter, panic_flag: false } + } +} + +/// Extend implementation that copies elements out of references before pushing them onto the Vec. +/// +/// This implementation is specialized for slice iterators, where it uses [`copy_from_slice`] to +/// append the entire slice at once. +/// +/// [`copy_from_slice`]: slice::copy_from_slice +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "extend_ref", since = "1.2.0")] +impl<'a, T: Copy + 'a, A: Allocator + 'a> Extend<&'a T> for Vec { + fn extend>(&mut self, iter: I) { + self.spec_extend(iter.into_iter()) + } + + #[inline] + fn extend_one(&mut self, &item: &'a T) { + self.push(item); + } + + #[inline] + fn extend_reserve(&mut self, additional: usize) { + self.reserve(additional); + } +} + +/// Implements comparison of vectors, [lexicographically](core::cmp::Ord#lexicographical-comparison). +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialOrd for Vec { + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + PartialOrd::partial_cmp(&**self, &**other) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Eq for Vec {} + +/// Implements ordering of vectors, [lexicographically](core::cmp::Ord#lexicographical-comparison). +#[stable(feature = "rust1", since = "1.0.0")] +impl Ord for Vec { + #[inline] + fn cmp(&self, other: &Self) -> Ordering { + Ord::cmp(&**self, &**other) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +unsafe impl<#[may_dangle] T, A: Allocator> Drop for Vec { + fn drop(&mut self) { + unsafe { + // use drop for [T] + // use a raw slice to refer to the elements of the vector as weakest necessary type; + // could avoid questions of validity in certain cases + ptr::drop_in_place(ptr::slice_from_raw_parts_mut(self.as_mut_ptr(), self.len)) + } + // RawVec handles deallocation + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] +impl const Default for Vec { + /// Creates an empty `Vec`. + /// + /// The vector will not allocate until elements are pushed onto it. + fn default() -> Vec { + Vec::new() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Debug for Vec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl AsRef> for Vec { + fn as_ref(&self) -> &Vec { + self + } +} + +#[stable(feature = "vec_as_mut", since = "1.5.0")] +impl AsMut> for Vec { + fn as_mut(&mut self) -> &mut Vec { + self + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl AsRef<[T]> for Vec { + fn as_ref(&self) -> &[T] { + self + } +} + +#[stable(feature = "vec_as_mut", since = "1.5.0")] +impl AsMut<[T]> for Vec { + fn as_mut(&mut self) -> &mut [T] { + self + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +impl From<&[T]> for Vec { + /// Allocate a `Vec` and fill it by cloning `s`'s items. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(Vec::from(&[1, 2, 3][..]), vec![1, 2, 3]); + /// ``` + #[cfg(not(test))] + fn from(s: &[T]) -> Vec { + s.to_vec() + } + #[cfg(test)] + fn from(s: &[T]) -> Vec { + crate::slice::to_vec(s, Global) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "vec_from_mut", since = "1.19.0")] +impl From<&mut [T]> for Vec { + /// Allocate a `Vec` and fill it by cloning `s`'s items. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(Vec::from(&mut [1, 2, 3][..]), vec![1, 2, 3]); + /// ``` + #[cfg(not(test))] + fn from(s: &mut [T]) -> Vec { + s.to_vec() + } + #[cfg(test)] + fn from(s: &mut [T]) -> Vec { + crate::slice::to_vec(s, Global) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "vec_from_array", since = "1.44.0")] +impl From<[T; N]> for Vec { + /// Allocate a `Vec` and move `s`'s items into it. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(Vec::from([1, 2, 3]), vec![1, 2, 3]); + /// ``` + #[cfg(not(test))] + fn from(s: [T; N]) -> Vec { + <[T]>::into_vec( + #[rustc_box] + Box::new(s), + ) + } + + #[cfg(test)] + fn from(s: [T; N]) -> Vec { + crate::slice::into_vec(Box::new(s)) + } +} + +#[stable(feature = "vec_from_cow_slice", since = "1.14.0")] +impl<'a, T> From> for Vec +where + [T]: ToOwned>, +{ + /// Convert a clone-on-write slice into a vector. + /// + /// If `s` already owns a `Vec`, it will be returned directly. + /// If `s` is borrowing a slice, a new `Vec` will be allocated and + /// filled by cloning `s`'s items into it. + /// + /// # Examples + /// + /// ``` + /// # use std::borrow::Cow; + /// let o: Cow<[i32]> = Cow::Owned(vec![1, 2, 3]); + /// let b: Cow<[i32]> = Cow::Borrowed(&[1, 2, 3]); + /// assert_eq!(Vec::from(o), Vec::from(b)); + /// ``` + fn from(s: Cow<'a, [T]>) -> Vec { + s.into_owned() + } +} + +// note: test pulls in std, which causes errors here +#[cfg(not(test))] +#[stable(feature = "vec_from_box", since = "1.18.0")] +impl From> for Vec { + /// Convert a boxed slice into a vector by transferring ownership of + /// the existing heap allocation. + /// + /// # Examples + /// + /// ``` + /// let b: Box<[i32]> = vec![1, 2, 3].into_boxed_slice(); + /// assert_eq!(Vec::from(b), vec![1, 2, 3]); + /// ``` + fn from(s: Box<[T], A>) -> Self { + s.into_vec() + } +} + +// note: test pulls in std, which causes errors here +#[cfg(not(no_global_oom_handling))] +#[cfg(not(test))] +#[stable(feature = "box_from_vec", since = "1.20.0")] +impl From> for Box<[T], A> { + /// Convert a vector into a boxed slice. + /// + /// If `v` has excess capacity, its items will be moved into a + /// newly-allocated buffer with exactly the right capacity. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(Box::from(vec![1, 2, 3]), vec![1, 2, 3].into_boxed_slice()); + /// ``` + /// + /// Any excess capacity is removed: + /// ``` + /// let mut vec = Vec::with_capacity(10); + /// vec.extend([1, 2, 3]); + /// + /// assert_eq!(Box::from(vec), vec![1, 2, 3].into_boxed_slice()); + /// ``` + fn from(v: Vec) -> Self { + v.into_boxed_slice() + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +impl From<&str> for Vec { + /// Allocate a `Vec` and fill it with a UTF-8 string. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(Vec::from("123"), vec![b'1', b'2', b'3']); + /// ``` + fn from(s: &str) -> Vec { + From::from(s.as_bytes()) + } +} + +#[stable(feature = "array_try_from_vec", since = "1.48.0")] +impl TryFrom> for [T; N] { + type Error = Vec; + + /// Gets the entire contents of the `Vec` as an array, + /// if its size exactly matches that of the requested array. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(vec![1, 2, 3].try_into(), Ok([1, 2, 3])); + /// assert_eq!(>::new().try_into(), Ok([])); + /// ``` + /// + /// If the length doesn't match, the input comes back in `Err`: + /// ``` + /// let r: Result<[i32; 4], _> = (0..10).collect::>().try_into(); + /// assert_eq!(r, Err(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9])); + /// ``` + /// + /// If you're fine with just getting a prefix of the `Vec`, + /// you can call [`.truncate(N)`](Vec::truncate) first. + /// ``` + /// let mut v = String::from("hello world").into_bytes(); + /// v.sort(); + /// v.truncate(2); + /// let [a, b]: [_; 2] = v.try_into().unwrap(); + /// assert_eq!(a, b' '); + /// assert_eq!(b, b'd'); + /// ``` + fn try_from(mut vec: Vec) -> Result<[T; N], Vec> { + if vec.len() != N { + return Err(vec); + } + + // SAFETY: `.set_len(0)` is always sound. + unsafe { vec.set_len(0) }; + + // SAFETY: A `Vec`'s pointer is always aligned properly, and + // the alignment the array needs is the same as the items. + // We checked earlier that we have sufficient items. + // The items will not double-drop as the `set_len` + // tells the `Vec` not to also drop them. + let array = unsafe { ptr::read(vec.as_ptr() as *const [T; N]) }; + Ok(array) + } +} diff --git a/crux-mir/lib/alloc/src/vec/partial_eq.rs b/crux-mir/lib/alloc/src/vec/partial_eq.rs new file mode 100644 index 000000000..b0cf72577 --- /dev/null +++ b/crux-mir/lib/alloc/src/vec/partial_eq.rs @@ -0,0 +1,47 @@ +use crate::alloc::Allocator; +#[cfg(not(no_global_oom_handling))] +use crate::borrow::Cow; + +use super::Vec; + +macro_rules! __impl_slice_eq1 { + ([$($vars:tt)*] $lhs:ty, $rhs:ty $(where $ty:ty: $bound:ident)?, #[$stability:meta]) => { + #[$stability] + impl PartialEq<$rhs> for $lhs + where + T: PartialEq, + $($ty: $bound)? + { + #[inline] + fn eq(&self, other: &$rhs) -> bool { self[..] == other[..] } + #[inline] + fn ne(&self, other: &$rhs) -> bool { self[..] != other[..] } + } + } +} + +__impl_slice_eq1! { [A1: Allocator, A2: Allocator] Vec, Vec, #[stable(feature = "rust1", since = "1.0.0")] } +__impl_slice_eq1! { [A: Allocator] Vec, &[U], #[stable(feature = "rust1", since = "1.0.0")] } +__impl_slice_eq1! { [A: Allocator] Vec, &mut [U], #[stable(feature = "rust1", since = "1.0.0")] } +__impl_slice_eq1! { [A: Allocator] &[T], Vec, #[stable(feature = "partialeq_vec_for_ref_slice", since = "1.46.0")] } +__impl_slice_eq1! { [A: Allocator] &mut [T], Vec, #[stable(feature = "partialeq_vec_for_ref_slice", since = "1.46.0")] } +__impl_slice_eq1! { [A: Allocator] Vec, [U], #[stable(feature = "partialeq_vec_for_slice", since = "1.48.0")] } +__impl_slice_eq1! { [A: Allocator] [T], Vec, #[stable(feature = "partialeq_vec_for_slice", since = "1.48.0")] } +#[cfg(not(no_global_oom_handling))] +__impl_slice_eq1! { [A: Allocator] Cow<'_, [T]>, Vec where T: Clone, #[stable(feature = "rust1", since = "1.0.0")] } +#[cfg(not(no_global_oom_handling))] +__impl_slice_eq1! { [] Cow<'_, [T]>, &[U] where T: Clone, #[stable(feature = "rust1", since = "1.0.0")] } +#[cfg(not(no_global_oom_handling))] +__impl_slice_eq1! { [] Cow<'_, [T]>, &mut [U] where T: Clone, #[stable(feature = "rust1", since = "1.0.0")] } +__impl_slice_eq1! { [A: Allocator, const N: usize] Vec, [U; N], #[stable(feature = "rust1", since = "1.0.0")] } +__impl_slice_eq1! { [A: Allocator, const N: usize] Vec, &[U; N], #[stable(feature = "rust1", since = "1.0.0")] } + +// NOTE: some less important impls are omitted to reduce code bloat +// FIXME(Centril): Reconsider this? +//__impl_slice_eq1! { [const N: usize] Vec, &mut [B; N], } +//__impl_slice_eq1! { [const N: usize] [A; N], Vec, } +//__impl_slice_eq1! { [const N: usize] &[A; N], Vec, } +//__impl_slice_eq1! { [const N: usize] &mut [A; N], Vec, } +//__impl_slice_eq1! { [const N: usize] Cow<'a, [A]>, [B; N], } +//__impl_slice_eq1! { [const N: usize] Cow<'a, [A]>, &[B; N], } +//__impl_slice_eq1! { [const N: usize] Cow<'a, [A]>, &mut [B; N], } diff --git a/crux-mir/lib/alloc/src/vec/set_len_on_drop.rs b/crux-mir/lib/alloc/src/vec/set_len_on_drop.rs new file mode 100644 index 000000000..6ce5a3a9f --- /dev/null +++ b/crux-mir/lib/alloc/src/vec/set_len_on_drop.rs @@ -0,0 +1,33 @@ +// Set the length of the vec when the `SetLenOnDrop` value goes out of scope. +// +// The idea is: The length field in SetLenOnDrop is a local variable +// that the optimizer will see does not alias with any stores through the Vec's data +// pointer. This is a workaround for alias analysis issue #32155 +pub(super) struct SetLenOnDrop<'a> { + len: &'a mut usize, + local_len: usize, +} + +impl<'a> SetLenOnDrop<'a> { + #[inline] + pub(super) fn new(len: &'a mut usize) -> Self { + SetLenOnDrop { local_len: *len, len } + } + + #[inline] + pub(super) fn increment_len(&mut self, increment: usize) { + self.local_len += increment; + } + + #[inline] + pub(super) fn current_len(&self) -> usize { + self.local_len + } +} + +impl Drop for SetLenOnDrop<'_> { + #[inline] + fn drop(&mut self) { + *self.len = self.local_len; + } +} diff --git a/crux-mir/lib/alloc/src/vec/spec_extend.rs b/crux-mir/lib/alloc/src/vec/spec_extend.rs new file mode 100644 index 000000000..56065ce56 --- /dev/null +++ b/crux-mir/lib/alloc/src/vec/spec_extend.rs @@ -0,0 +1,57 @@ +use crate::alloc::Allocator; +use core::iter::TrustedLen; +use core::slice::{self}; + +use super::{IntoIter, Vec}; + +// Specialization trait used for Vec::extend +pub(super) trait SpecExtend { + fn spec_extend(&mut self, iter: I); +} + +impl SpecExtend for Vec +where + I: Iterator, +{ + default fn spec_extend(&mut self, iter: I) { + self.extend_desugared(iter) + } +} + +impl SpecExtend for Vec +where + I: TrustedLen, +{ + default fn spec_extend(&mut self, iterator: I) { + self.extend_trusted(iterator) + } +} + +impl SpecExtend> for Vec { + fn spec_extend(&mut self, mut iterator: IntoIter) { + unsafe { + self.append_elements(iterator.as_slice() as _); + } + iterator.forget_remaining_elements(); + } +} + +impl<'a, T: 'a, I, A: Allocator + 'a> SpecExtend<&'a T, I> for Vec +where + I: Iterator, + T: Clone, +{ + default fn spec_extend(&mut self, iterator: I) { + self.spec_extend(iterator.cloned()) + } +} + +impl<'a, T: 'a, A: Allocator + 'a> SpecExtend<&'a T, slice::Iter<'a, T>> for Vec +where + T: Copy, +{ + fn spec_extend(&mut self, iterator: slice::Iter<'a, T>) { + let slice = iterator.as_slice(); + unsafe { self.append_elements(slice) }; + } +} diff --git a/crux-mir/lib/alloc/src/vec/spec_from_elem.rs b/crux-mir/lib/alloc/src/vec/spec_from_elem.rs new file mode 100644 index 000000000..ff364c033 --- /dev/null +++ b/crux-mir/lib/alloc/src/vec/spec_from_elem.rs @@ -0,0 +1,61 @@ +use core::ptr; + +use crate::alloc::Allocator; +use crate::raw_vec::RawVec; + +use super::{ExtendElement, IsZero, Vec}; + +// Specialization trait used for Vec::from_elem +pub(super) trait SpecFromElem: Sized { + fn from_elem(elem: Self, n: usize, alloc: A) -> Vec; +} + +impl SpecFromElem for T { + default fn from_elem(elem: Self, n: usize, alloc: A) -> Vec { + let mut v = Vec::with_capacity_in(n, alloc); + v.extend_with(n, ExtendElement(elem)); + v + } +} + +impl SpecFromElem for T { + #[inline] + default fn from_elem(elem: T, n: usize, alloc: A) -> Vec { + if elem.is_zero() { + return Vec { buf: RawVec::with_capacity_zeroed_in(n, alloc), len: n }; + } + let mut v = Vec::with_capacity_in(n, alloc); + v.extend_with(n, ExtendElement(elem)); + v + } +} + +impl SpecFromElem for i8 { + #[inline] + fn from_elem(elem: i8, n: usize, alloc: A) -> Vec { + if elem == 0 { + return Vec { buf: RawVec::with_capacity_zeroed_in(n, alloc), len: n }; + } + unsafe { + let mut v = Vec::with_capacity_in(n, alloc); + ptr::write_bytes(v.as_mut_ptr(), elem as u8, n); + v.set_len(n); + v + } + } +} + +impl SpecFromElem for u8 { + #[inline] + fn from_elem(elem: u8, n: usize, alloc: A) -> Vec { + if elem == 0 { + return Vec { buf: RawVec::with_capacity_zeroed_in(n, alloc), len: n }; + } + unsafe { + let mut v = Vec::with_capacity_in(n, alloc); + ptr::write_bytes(v.as_mut_ptr(), elem, n); + v.set_len(n); + v + } + } +} diff --git a/crux-mir/lib/alloc/src/vec/spec_from_iter.rs b/crux-mir/lib/alloc/src/vec/spec_from_iter.rs new file mode 100644 index 000000000..efa686847 --- /dev/null +++ b/crux-mir/lib/alloc/src/vec/spec_from_iter.rs @@ -0,0 +1,64 @@ +use core::mem::ManuallyDrop; +use core::ptr::{self}; + +use super::{IntoIter, SpecExtend, SpecFromIterNested, Vec}; + +/// Specialization trait used for Vec::from_iter +/// +/// ## The delegation graph: +/// +/// ```text +/// +-------------+ +/// |FromIterator | +/// +-+-----------+ +/// | +/// v +/// +-+-------------------------------+ +---------------------+ +/// |SpecFromIter +---->+SpecFromIterNested | +/// |where I: | | |where I: | +/// | Iterator (default)----------+ | | Iterator (default) | +/// | vec::IntoIter | | | TrustedLen | +/// | SourceIterMarker---fallback-+ | +---------------------+ +/// +---------------------------------+ +/// ``` +pub(super) trait SpecFromIter { + fn from_iter(iter: I) -> Self; +} + +impl SpecFromIter for Vec +where + I: Iterator, +{ + default fn from_iter(iterator: I) -> Self { + SpecFromIterNested::from_iter(iterator) + } +} + +impl SpecFromIter> for Vec { + fn from_iter(iterator: IntoIter) -> Self { + // A common case is passing a vector into a function which immediately + // re-collects into a vector. We can short circuit this if the IntoIter + // has not been advanced at all. + // When it has been advanced We can also reuse the memory and move the data to the front. + // But we only do so when the resulting Vec wouldn't have more unused capacity + // than creating it through the generic FromIterator implementation would. That limitation + // is not strictly necessary as Vec's allocation behavior is intentionally unspecified. + // But it is a conservative choice. + let has_advanced = iterator.buf.as_ptr() as *const _ != iterator.ptr; + if !has_advanced || iterator.len() >= iterator.cap / 2 { + unsafe { + let it = ManuallyDrop::new(iterator); + if has_advanced { + ptr::copy(it.ptr, it.buf.as_ptr(), it.len()); + } + return Vec::from_raw_parts(it.buf.as_ptr(), it.len(), it.cap); + } + } + + let mut vec = Vec::new(); + // must delegate to spec_extend() since extend() itself delegates + // to spec_from for empty Vecs + vec.spec_extend(iterator); + vec + } +} diff --git a/crux-mir/lib/alloc/src/vec/spec_from_iter_nested.rs b/crux-mir/lib/alloc/src/vec/spec_from_iter_nested.rs new file mode 100644 index 000000000..f915ebb86 --- /dev/null +++ b/crux-mir/lib/alloc/src/vec/spec_from_iter_nested.rs @@ -0,0 +1,65 @@ +use core::cmp; +use core::iter::TrustedLen; +use core::ptr; + +use crate::raw_vec::RawVec; + +use super::{SpecExtend, Vec}; + +/// Another specialization trait for Vec::from_iter +/// necessary to manually prioritize overlapping specializations +/// see [`SpecFromIter`](super::SpecFromIter) for details. +pub(super) trait SpecFromIterNested { + fn from_iter(iter: I) -> Self; +} + +impl SpecFromIterNested for Vec +where + I: Iterator, +{ + default fn from_iter(mut iterator: I) -> Self { + // Unroll the first iteration, as the vector is going to be + // expanded on this iteration in every case when the iterable is not + // empty, but the loop in extend_desugared() is not going to see the + // vector being full in the few subsequent loop iterations. + // So we get better branch prediction. + let mut vector = match iterator.next() { + None => return Vec::new(), + Some(element) => { + let (lower, _) = iterator.size_hint(); + let initial_capacity = + cmp::max(RawVec::::MIN_NON_ZERO_CAP, lower.saturating_add(1)); + let mut vector = Vec::with_capacity(initial_capacity); + unsafe { + // SAFETY: We requested capacity at least 1 + ptr::write(vector.as_mut_ptr(), element); + vector.set_len(1); + } + vector + } + }; + // must delegate to spec_extend() since extend() itself delegates + // to spec_from for empty Vecs + as SpecExtend>::spec_extend(&mut vector, iterator); + vector + } +} + +impl SpecFromIterNested for Vec +where + I: TrustedLen, +{ + fn from_iter(iterator: I) -> Self { + let mut vector = match iterator.size_hint() { + (_, Some(upper)) => Vec::with_capacity(upper), + // TrustedLen contract guarantees that `size_hint() == (_, None)` means that there + // are more than `usize::MAX` elements. + // Since the previous branch would eagerly panic if the capacity is too large + // (via `with_capacity`) we do the same here. + _ => panic!("capacity overflow"), + }; + // reuse extend specialization for TrustedLen + vector.spec_extend(iterator); + vector + } +} diff --git a/crux-mir/lib/alloc/src/vec/splice.rs b/crux-mir/lib/alloc/src/vec/splice.rs new file mode 100644 index 000000000..1861147fe --- /dev/null +++ b/crux-mir/lib/alloc/src/vec/splice.rs @@ -0,0 +1,139 @@ +use crate::alloc::{Allocator, Global}; +use core::ptr::{self}; +use core::slice::{self}; + +use super::{Drain, Vec}; + +/// A splicing iterator for `Vec`. +/// +/// This struct is created by [`Vec::splice()`]. +/// See its documentation for more. +/// +/// # Example +/// +/// ``` +/// let mut v = vec![0, 1, 2]; +/// let new = [7, 8]; +/// let iter: std::vec::Splice<_> = v.splice(1.., new); +/// ``` +#[derive(Debug)] +#[stable(feature = "vec_splice", since = "1.21.0")] +pub struct Splice< + 'a, + I: Iterator + 'a, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + 'a = Global, +> { + pub(super) drain: Drain<'a, I::Item, A>, + pub(super) replace_with: I, +} + +#[stable(feature = "vec_splice", since = "1.21.0")] +impl Iterator for Splice<'_, I, A> { + type Item = I::Item; + + fn next(&mut self) -> Option { + self.drain.next() + } + + fn size_hint(&self) -> (usize, Option) { + self.drain.size_hint() + } +} + +#[stable(feature = "vec_splice", since = "1.21.0")] +impl DoubleEndedIterator for Splice<'_, I, A> { + fn next_back(&mut self) -> Option { + self.drain.next_back() + } +} + +#[stable(feature = "vec_splice", since = "1.21.0")] +impl ExactSizeIterator for Splice<'_, I, A> {} + +#[stable(feature = "vec_splice", since = "1.21.0")] +impl Drop for Splice<'_, I, A> { + fn drop(&mut self) { + self.drain.by_ref().for_each(drop); + // At this point draining is done and the only remaining tasks are splicing + // and moving things into the final place. + // Which means we can replace the slice::Iter with pointers that won't point to deallocated + // memory, so that Drain::drop is still allowed to call iter.len(), otherwise it would break + // the ptr.sub_ptr contract. + self.drain.iter = (&[]).iter(); + + unsafe { + if self.drain.tail_len == 0 { + self.drain.vec.as_mut().extend(self.replace_with.by_ref()); + return; + } + + // First fill the range left by drain(). + if !self.drain.fill(&mut self.replace_with) { + return; + } + + // There may be more elements. Use the lower bound as an estimate. + // FIXME: Is the upper bound a better guess? Or something else? + let (lower_bound, _upper_bound) = self.replace_with.size_hint(); + if lower_bound > 0 { + self.drain.move_tail(lower_bound); + if !self.drain.fill(&mut self.replace_with) { + return; + } + } + + // Collect any remaining elements. + // This is a zero-length vector which does not allocate if `lower_bound` was exact. + let mut collected = self.replace_with.by_ref().collect::>().into_iter(); + // Now we have an exact count. + if collected.len() > 0 { + self.drain.move_tail(collected.len()); + let filled = self.drain.fill(&mut collected); + debug_assert!(filled); + debug_assert_eq!(collected.len(), 0); + } + } + // Let `Drain::drop` move the tail back if necessary and restore `vec.len`. + } +} + +/// Private helper methods for `Splice::drop` +impl Drain<'_, T, A> { + /// The range from `self.vec.len` to `self.tail_start` contains elements + /// that have been moved out. + /// Fill that range as much as possible with new elements from the `replace_with` iterator. + /// Returns `true` if we filled the entire range. (`replace_with.next()` didn’t return `None`.) + unsafe fn fill>(&mut self, replace_with: &mut I) -> bool { + let vec = unsafe { self.vec.as_mut() }; + let range_start = vec.len; + let range_end = self.tail_start; + let range_slice = unsafe { + slice::from_raw_parts_mut(vec.as_mut_ptr().add(range_start), range_end - range_start) + }; + + for place in range_slice { + if let Some(new_item) = replace_with.next() { + unsafe { ptr::write(place, new_item) }; + vec.len += 1; + } else { + return false; + } + } + true + } + + /// Makes room for inserting more elements before the tail. + unsafe fn move_tail(&mut self, additional: usize) { + let vec = unsafe { self.vec.as_mut() }; + let len = self.tail_start + self.tail_len; + vec.buf.reserve(len, additional); + + let new_tail_start = self.tail_start + additional; + unsafe { + let src = vec.as_ptr().add(self.tail_start); + let dst = vec.as_mut_ptr().add(new_tail_start); + ptr::copy(src, dst, self.tail_len); + } + self.tail_start = new_tail_start; + } +} diff --git a/crux-mir/lib/alloc/tests/arc.rs b/crux-mir/lib/alloc/tests/arc.rs index 34384cfcb..ce40b5c9b 100644 --- a/crux-mir/lib/alloc/tests/arc.rs +++ b/crux-mir/lib/alloc/tests/arc.rs @@ -50,7 +50,7 @@ fn trait_object() { #[test] fn float_nan_ne() { - let x = Arc::new(std::f32::NAN); + let x = Arc::new(f32::NAN); assert!(x != x); assert!(!(x == x)); } @@ -195,3 +195,18 @@ fn shared_from_iter_trustedlen_no_fuse() { assert_trusted_len(&iter); assert_eq!(&[Box::new(42), Box::new(24)], &*iter.collect::>()); } + +#[test] +fn weak_may_dangle() { + fn hmm<'a>(val: &'a mut Weak<&'a str>) -> Weak<&'a str> { + val.clone() + } + + // Without #[may_dangle] we get: + let mut val = Weak::new(); + hmm(&mut val); + // ~~~~~~~~ borrowed value does not live long enough + // + // `val` dropped here while still borrowed + // borrow might be used here, when `val` is dropped and runs the `Drop` code for type `std::sync::Weak` +} diff --git a/crux-mir/lib/alloc/tests/autotraits.rs b/crux-mir/lib/alloc/tests/autotraits.rs new file mode 100644 index 000000000..879e32b3f --- /dev/null +++ b/crux-mir/lib/alloc/tests/autotraits.rs @@ -0,0 +1,293 @@ +fn require_sync(_: T) {} +fn require_send_sync(_: T) {} + +struct NotSend(*const ()); +unsafe impl Sync for NotSend {} + +#[test] +fn test_btree_map() { + // Tests of this form are prone to https://github.com/rust-lang/rust/issues/64552. + // + // In theory the async block's future would be Send if the value we hold + // across the await point is Send, and Sync if the value we hold across the + // await point is Sync. + // + // We test autotraits in this convoluted way, instead of a straightforward + // `require_send_sync::()`, because the interaction with + // generators exposes some current limitations in rustc's ability to prove a + // lifetime bound on the erased generator witness types. See the above link. + // + // A typical way this would surface in real code is: + // + // fn spawn(_: T) {} + // + // async fn f() { + // let map = BTreeMap::>::new(); + // for _ in &map { + // async {}.await; + // } + // } + // + // fn main() { + // spawn(f()); + // } + // + // where with some unintentionally overconstrained Send impls in alloc's + // internals, the future might incorrectly not be Send even though every + // single type involved in the program is Send and Sync. + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + // Testing like this would not catch all issues that the above form catches. + require_send_sync(None::>); + + require_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::< + alloc::collections::btree_map::DrainFilter< + '_, + &u32, + &u32, + fn(&&u32, &mut &u32) -> bool, + >, + >; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); +} + +#[test] +fn test_btree_set() { + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None:: bool>>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); +} + +#[test] +fn test_binary_heap() { + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); +} + +#[test] +fn test_linked_list() { + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + // FIXME + /* + require_send_sync(async { + let _v = + None:: bool>>; + async {}.await; + }); + */ + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); +} + +#[test] +fn test_vec_deque() { + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); + + require_send_sync(async { + let _v = None::>; + async {}.await; + }); +} diff --git a/crux-mir/lib/alloc/tests/borrow.rs b/crux-mir/lib/alloc/tests/borrow.rs new file mode 100644 index 000000000..57976aa6c --- /dev/null +++ b/crux-mir/lib/alloc/tests/borrow.rs @@ -0,0 +1,60 @@ +use std::borrow::{Cow, ToOwned}; +use std::ffi::{CStr, OsStr}; +use std::path::Path; +use std::rc::Rc; +use std::sync::Arc; + +macro_rules! test_from_cow { + ($value:ident => $($ty:ty),+) => {$( + let borrowed = <$ty>::from(Cow::Borrowed($value)); + let owned = <$ty>::from(Cow::Owned($value.to_owned())); + assert_eq!($value, &*borrowed); + assert_eq!($value, &*owned); + )+}; + ($value:ident : & $ty:ty) => { + test_from_cow!($value => Box<$ty>, Rc<$ty>, Arc<$ty>); + } +} + +#[test] +fn test_from_cow_slice() { + let slice: &[i32] = &[1, 2, 3]; + test_from_cow!(slice: &[i32]); +} + +#[test] +fn test_from_cow_str() { + let string = "hello"; + test_from_cow!(string: &str); +} + +#[test] +fn test_from_cow_c_str() { + let string = CStr::from_bytes_with_nul(b"hello\0").unwrap(); + test_from_cow!(string: &CStr); +} + +#[test] +fn test_from_cow_os_str() { + let string = OsStr::new("hello"); + test_from_cow!(string: &OsStr); +} + +#[test] +fn test_from_cow_path() { + let path = Path::new("hello"); + test_from_cow!(path: &Path); +} + +#[test] +fn cow_const() { + // test that the methods of `Cow` are usable in a const context + + const COW: Cow<'_, str> = Cow::Borrowed("moo"); + + const IS_BORROWED: bool = COW.is_borrowed(); + assert!(IS_BORROWED); + + const IS_OWNED: bool = COW.is_owned(); + assert!(!IS_OWNED); +} diff --git a/crux-mir/lib/alloc/tests/boxed.rs b/crux-mir/lib/alloc/tests/boxed.rs index 66782ecbe..af49826ff 100644 --- a/crux-mir/lib/alloc/tests/boxed.rs +++ b/crux-mir/lib/alloc/tests/boxed.rs @@ -1,8 +1,10 @@ -use std::mem::MaybeUninit; -use std::ptr::NonNull; +use core::alloc::{AllocError, Allocator, Layout}; +use core::cell::Cell; +use core::mem::MaybeUninit; +use core::ptr::NonNull; #[test] -fn unitialized_zero_size_box() { +fn uninitialized_zero_size_box() { assert_eq!( &*Box::<()>::new_uninit() as *const _, NonNull::>::dangling().as_ptr(), @@ -16,3 +18,179 @@ fn unitialized_zero_size_box() { NonNull::>::dangling().as_ptr(), ); } + +#[derive(Clone, PartialEq, Eq, Debug)] +struct Dummy { + _data: u8, +} + +#[test] +fn box_clone_and_clone_from_equivalence() { + for size in (0..8).map(|i| 2usize.pow(i)) { + let control = vec![Dummy { _data: 42 }; size].into_boxed_slice(); + let clone = control.clone(); + let mut copy = vec![Dummy { _data: 84 }; size].into_boxed_slice(); + copy.clone_from(&control); + assert_eq!(control, clone); + assert_eq!(control, copy); + } +} + +/// This test might give a false positive in case the box reallocates, +/// but the allocator keeps the original pointer. +/// +/// On the other hand, it won't give a false negative: If it fails, then the +/// memory was definitely not reused. +#[test] +fn box_clone_from_ptr_stability() { + for size in (0..8).map(|i| 2usize.pow(i)) { + let control = vec![Dummy { _data: 42 }; size].into_boxed_slice(); + let mut copy = vec![Dummy { _data: 84 }; size].into_boxed_slice(); + let copy_raw = copy.as_ptr() as usize; + copy.clone_from(&control); + assert_eq!(copy.as_ptr() as usize, copy_raw); + } +} + +#[test] +fn box_deref_lval() { + let x = Box::new(Cell::new(5)); + x.set(1000); + assert_eq!(x.get(), 1000); +} + +pub struct ConstAllocator; + +unsafe impl const Allocator for ConstAllocator { + fn allocate(&self, layout: Layout) -> Result, AllocError> { + match layout.size() { + 0 => Ok(NonNull::slice_from_raw_parts(layout.dangling(), 0)), + _ => unsafe { + let ptr = core::intrinsics::const_allocate(layout.size(), layout.align()); + Ok(NonNull::new_unchecked(ptr as *mut [u8; 0] as *mut [u8])) + }, + } + } + + unsafe fn deallocate(&self, _ptr: NonNull, layout: Layout) { + match layout.size() { + 0 => { /* do nothing */ } + _ => { /* do nothing too */ } + } + } + + fn allocate_zeroed(&self, layout: Layout) -> Result, AllocError> { + let ptr = self.allocate(layout)?; + if layout.size() > 0 { + unsafe { + ptr.as_mut_ptr().write_bytes(0, layout.size()); + } + } + Ok(ptr) + } + + unsafe fn grow( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + debug_assert!( + new_layout.size() >= old_layout.size(), + "`new_layout.size()` must be greater than or equal to `old_layout.size()`" + ); + + let new_ptr = self.allocate(new_layout)?; + if new_layout.size() > 0 { + // Safety: `new_ptr` is valid for writes and `ptr` for reads of + // `old_layout.size()`, because `new_layout.size() >= + // old_layout.size()` (which is an invariant that must be upheld by + // callers). + unsafe { + new_ptr.as_mut_ptr().copy_from_nonoverlapping(ptr.as_ptr(), old_layout.size()); + } + // Safety: `ptr` is never used again is also an invariant which must + // be upheld by callers. + unsafe { + self.deallocate(ptr, old_layout); + } + } + Ok(new_ptr) + } + + unsafe fn grow_zeroed( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + // Safety: Invariants of `grow_zeroed` and `grow` are the same, and must + // be enforced by callers. + let new_ptr = unsafe { self.grow(ptr, old_layout, new_layout)? }; + if new_layout.size() > 0 { + let old_size = old_layout.size(); + let new_size = new_layout.size(); + let raw_ptr = new_ptr.as_mut_ptr(); + // Safety: + // - `grow` returned Ok, so the returned pointer must be valid for + // `new_size` bytes + // - `new_size` must be larger than `old_size`, which is an + // invariant which must be upheld by callers. + unsafe { + raw_ptr.add(old_size).write_bytes(0, new_size - old_size); + } + } + Ok(new_ptr) + } + + unsafe fn shrink( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + debug_assert!( + new_layout.size() <= old_layout.size(), + "`new_layout.size()` must be smaller than or equal to `old_layout.size()`" + ); + + let new_ptr = self.allocate(new_layout)?; + if new_layout.size() > 0 { + // Safety: `new_ptr` and `ptr` are valid for reads/writes of + // `new_layout.size()` because of the invariants of shrink, which + // include `new_layout.size()` being smaller than (or equal to) + // `old_layout.size()`. + unsafe { + new_ptr.as_mut_ptr().copy_from_nonoverlapping(ptr.as_ptr(), new_layout.size()); + } + // Safety: `ptr` is never used again is also an invariant which must + // be upheld by callers. + unsafe { + self.deallocate(ptr, old_layout); + } + } + Ok(new_ptr) + } + + fn by_ref(&self) -> &Self + where + Self: Sized, + { + self + } +} + +#[test] +fn const_box() { + const VALUE: u32 = { + let mut boxed = Box::new_in(1u32, ConstAllocator); + assert!(*boxed == 1); + + *boxed = 42; + assert!(*boxed == 42); + + *Box::leak(boxed) + }; + + assert!(VALUE == 42); +} diff --git a/crux-mir/lib/alloc/tests/btree/map.rs b/crux-mir/lib/alloc/tests/btree/map.rs deleted file mode 100644 index 3a3462d54..000000000 --- a/crux-mir/lib/alloc/tests/btree/map.rs +++ /dev/null @@ -1,1064 +0,0 @@ -use std::collections::btree_map::Entry::{Occupied, Vacant}; -use std::collections::BTreeMap; -use std::convert::TryFrom; -use std::fmt::Debug; -use std::iter::FromIterator; -use std::ops::Bound::{self, Excluded, Included, Unbounded}; -use std::ops::RangeBounds; -use std::panic::catch_unwind; -use std::rc::Rc; -use std::sync::atomic::{AtomicU32, Ordering}; - -use super::DeterministicRng; - -#[test] -fn test_basic_large() { - let mut map = BTreeMap::new(); - #[cfg(not(miri))] // Miri is too slow - let size = 10000; - #[cfg(miri)] - let size = 144; // to obtain height 3 tree (having edges to both kinds of nodes) - assert_eq!(map.len(), 0); - - for i in 0..size { - assert_eq!(map.insert(i, 10 * i), None); - assert_eq!(map.len(), i + 1); - } - - assert_eq!(map.first_key_value(), Some((&0, &0))); - assert_eq!(map.last_key_value(), Some((&(size - 1), &(10 * (size - 1))))); - assert_eq!(map.first_entry().unwrap().key(), &0); - assert_eq!(map.last_entry().unwrap().key(), &(size - 1)); - - for i in 0..size { - assert_eq!(map.get(&i).unwrap(), &(i * 10)); - } - - for i in size..size * 2 { - assert_eq!(map.get(&i), None); - } - - for i in 0..size { - assert_eq!(map.insert(i, 100 * i), Some(10 * i)); - assert_eq!(map.len(), size); - } - - for i in 0..size { - assert_eq!(map.get(&i).unwrap(), &(i * 100)); - } - - for i in 0..size / 2 { - assert_eq!(map.remove(&(i * 2)), Some(i * 200)); - assert_eq!(map.len(), size - i - 1); - } - - for i in 0..size / 2 { - assert_eq!(map.get(&(2 * i)), None); - assert_eq!(map.get(&(2 * i + 1)).unwrap(), &(i * 200 + 100)); - } - - for i in 0..size / 2 { - assert_eq!(map.remove(&(2 * i)), None); - assert_eq!(map.remove(&(2 * i + 1)), Some(i * 200 + 100)); - assert_eq!(map.len(), size / 2 - i - 1); - } -} - -#[test] -fn test_basic_small() { - let mut map = BTreeMap::new(); - // Empty, root is absent (None): - assert_eq!(map.remove(&1), None); - assert_eq!(map.len(), 0); - assert_eq!(map.get(&1), None); - assert_eq!(map.get_mut(&1), None); - assert_eq!(map.first_key_value(), None); - assert_eq!(map.last_key_value(), None); - assert_eq!(map.keys().count(), 0); - assert_eq!(map.values().count(), 0); - assert_eq!(map.range(..).next(), None); - assert_eq!(map.range(..1).next(), None); - assert_eq!(map.range(1..).next(), None); - assert_eq!(map.range(1..=1).next(), None); - assert_eq!(map.range(1..2).next(), None); - assert_eq!(map.insert(1, 1), None); - - // 1 key-value pair: - assert_eq!(map.len(), 1); - assert_eq!(map.get(&1), Some(&1)); - assert_eq!(map.get_mut(&1), Some(&mut 1)); - assert_eq!(map.first_key_value(), Some((&1, &1))); - assert_eq!(map.last_key_value(), Some((&1, &1))); - assert_eq!(map.keys().collect::>(), vec![&1]); - assert_eq!(map.values().collect::>(), vec![&1]); - assert_eq!(map.insert(1, 2), Some(1)); - assert_eq!(map.len(), 1); - assert_eq!(map.get(&1), Some(&2)); - assert_eq!(map.get_mut(&1), Some(&mut 2)); - assert_eq!(map.first_key_value(), Some((&1, &2))); - assert_eq!(map.last_key_value(), Some((&1, &2))); - assert_eq!(map.keys().collect::>(), vec![&1]); - assert_eq!(map.values().collect::>(), vec![&2]); - assert_eq!(map.insert(2, 4), None); - - // 2 key-value pairs: - assert_eq!(map.len(), 2); - assert_eq!(map.get(&2), Some(&4)); - assert_eq!(map.get_mut(&2), Some(&mut 4)); - assert_eq!(map.first_key_value(), Some((&1, &2))); - assert_eq!(map.last_key_value(), Some((&2, &4))); - assert_eq!(map.keys().collect::>(), vec![&1, &2]); - assert_eq!(map.values().collect::>(), vec![&2, &4]); - assert_eq!(map.remove(&1), Some(2)); - - // 1 key-value pair: - assert_eq!(map.len(), 1); - assert_eq!(map.get(&1), None); - assert_eq!(map.get_mut(&1), None); - assert_eq!(map.get(&2), Some(&4)); - assert_eq!(map.get_mut(&2), Some(&mut 4)); - assert_eq!(map.first_key_value(), Some((&2, &4))); - assert_eq!(map.last_key_value(), Some((&2, &4))); - assert_eq!(map.keys().collect::>(), vec![&2]); - assert_eq!(map.values().collect::>(), vec![&4]); - assert_eq!(map.remove(&2), Some(4)); - - // Empty but root is owned (Some(...)): - assert_eq!(map.len(), 0); - assert_eq!(map.get(&1), None); - assert_eq!(map.get_mut(&1), None); - assert_eq!(map.first_key_value(), None); - assert_eq!(map.last_key_value(), None); - assert_eq!(map.keys().count(), 0); - assert_eq!(map.values().count(), 0); - assert_eq!(map.range(..).next(), None); - assert_eq!(map.range(..1).next(), None); - assert_eq!(map.range(1..).next(), None); - assert_eq!(map.range(1..=1).next(), None); - assert_eq!(map.range(1..2).next(), None); - assert_eq!(map.remove(&1), None); -} - -#[test] -fn test_iter() { - #[cfg(not(miri))] // Miri is too slow - let size = 10000; - #[cfg(miri)] - let size = 200; - - let mut map: BTreeMap<_, _> = (0..size).map(|i| (i, i)).collect(); - - fn test(size: usize, mut iter: T) - where - T: Iterator, - { - for i in 0..size { - assert_eq!(iter.size_hint(), (size - i, Some(size - i))); - assert_eq!(iter.next().unwrap(), (i, i)); - } - assert_eq!(iter.size_hint(), (0, Some(0))); - assert_eq!(iter.next(), None); - } - test(size, map.iter().map(|(&k, &v)| (k, v))); - test(size, map.iter_mut().map(|(&k, &mut v)| (k, v))); - test(size, map.into_iter()); -} - -#[test] -fn test_iter_rev() { - #[cfg(not(miri))] // Miri is too slow - let size = 10000; - #[cfg(miri)] - let size = 200; - - let mut map: BTreeMap<_, _> = (0..size).map(|i| (i, i)).collect(); - - fn test(size: usize, mut iter: T) - where - T: Iterator, - { - for i in 0..size { - assert_eq!(iter.size_hint(), (size - i, Some(size - i))); - assert_eq!(iter.next().unwrap(), (size - i - 1, size - i - 1)); - } - assert_eq!(iter.size_hint(), (0, Some(0))); - assert_eq!(iter.next(), None); - } - test(size, map.iter().rev().map(|(&k, &v)| (k, v))); - test(size, map.iter_mut().rev().map(|(&k, &mut v)| (k, v))); - test(size, map.into_iter().rev()); -} - -/// Specifically tests iter_mut's ability to mutate the value of pairs in-line -fn do_test_iter_mut_mutation(size: usize) -where - T: Copy + Debug + Ord + TryFrom, - >::Error: std::fmt::Debug, -{ - let zero = T::try_from(0).unwrap(); - let mut map: BTreeMap = (0..size).map(|i| (T::try_from(i).unwrap(), zero)).collect(); - - // Forward and backward iteration sees enough pairs (also tested elsewhere) - assert_eq!(map.iter_mut().count(), size); - assert_eq!(map.iter_mut().rev().count(), size); - - // Iterate forwards, trying to mutate to unique values - for (i, (k, v)) in map.iter_mut().enumerate() { - assert_eq!(*k, T::try_from(i).unwrap()); - assert_eq!(*v, zero); - *v = T::try_from(i + 1).unwrap(); - } - - // Iterate backwards, checking that mutations succeeded and trying to mutate again - for (i, (k, v)) in map.iter_mut().rev().enumerate() { - assert_eq!(*k, T::try_from(size - i - 1).unwrap()); - assert_eq!(*v, T::try_from(size - i).unwrap()); - *v = T::try_from(2 * size - i).unwrap(); - } - - // Check that backward mutations succeeded - for (i, (k, v)) in map.iter_mut().enumerate() { - assert_eq!(*k, T::try_from(i).unwrap()); - assert_eq!(*v, T::try_from(size + i + 1).unwrap()); - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, PartialOrd, Ord)] -#[repr(align(32))] -struct Align32(usize); - -impl TryFrom for Align32 { - type Error = (); - - fn try_from(s: usize) -> Result { - Ok(Align32(s)) - } -} - -#[test] -fn test_iter_mut_mutation() { - // Check many alignments because various fields precede array in NodeHeader. - // Check with size 0 which should not iterate at all. - // Check with size 1 for a tree with one kind of node (root = leaf). - // Check with size 12 for a tree with two kinds of nodes (root and leaves). - // Check with size 144 for a tree with all kinds of nodes (root, internals and leaves). - do_test_iter_mut_mutation::(0); - do_test_iter_mut_mutation::(1); - do_test_iter_mut_mutation::(12); - do_test_iter_mut_mutation::(127); // not enough unique values to test 144 - do_test_iter_mut_mutation::(1); - do_test_iter_mut_mutation::(12); - do_test_iter_mut_mutation::(144); - do_test_iter_mut_mutation::(1); - do_test_iter_mut_mutation::(12); - do_test_iter_mut_mutation::(144); - do_test_iter_mut_mutation::(1); - do_test_iter_mut_mutation::(12); - do_test_iter_mut_mutation::(144); - do_test_iter_mut_mutation::(1); - do_test_iter_mut_mutation::(12); - do_test_iter_mut_mutation::(144); - do_test_iter_mut_mutation::(1); - do_test_iter_mut_mutation::(12); - do_test_iter_mut_mutation::(144); -} - -#[test] -fn test_values_mut() { - let mut a = BTreeMap::new(); - a.insert(1, String::from("hello")); - a.insert(2, String::from("goodbye")); - - for value in a.values_mut() { - value.push_str("!"); - } - - let values: Vec = a.values().cloned().collect(); - assert_eq!(values, [String::from("hello!"), String::from("goodbye!")]); -} - -#[test] -fn test_iter_mixed() { - #[cfg(not(miri))] // Miri is too slow - let size = 10000; - #[cfg(miri)] - let size = 200; - - let mut map: BTreeMap<_, _> = (0..size).map(|i| (i, i)).collect(); - - fn test(size: usize, mut iter: T) - where - T: Iterator + DoubleEndedIterator, - { - for i in 0..size / 4 { - assert_eq!(iter.size_hint(), (size - i * 2, Some(size - i * 2))); - assert_eq!(iter.next().unwrap(), (i, i)); - assert_eq!(iter.next_back().unwrap(), (size - i - 1, size - i - 1)); - } - for i in size / 4..size * 3 / 4 { - assert_eq!(iter.size_hint(), (size * 3 / 4 - i, Some(size * 3 / 4 - i))); - assert_eq!(iter.next().unwrap(), (i, i)); - } - assert_eq!(iter.size_hint(), (0, Some(0))); - assert_eq!(iter.next(), None); - } - test(size, map.iter().map(|(&k, &v)| (k, v))); - test(size, map.iter_mut().map(|(&k, &mut v)| (k, v))); - test(size, map.into_iter()); -} - -fn range_keys(map: &BTreeMap, range: impl RangeBounds) -> Vec { - map.range(range) - .map(|(&k, &v)| { - assert_eq!(k, v); - k - }) - .collect() -} - -#[test] -fn test_range_small() { - let size = 4; - - let map: BTreeMap<_, _> = (1..=size).map(|i| (i, i)).collect(); - let all: Vec<_> = (1..=size).collect(); - let (first, last) = (vec![all[0]], vec![all[size as usize - 1]]); - - assert_eq!(range_keys(&map, (Excluded(0), Excluded(size + 1))), all); - assert_eq!(range_keys(&map, (Excluded(0), Included(size + 1))), all); - assert_eq!(range_keys(&map, (Excluded(0), Included(size))), all); - assert_eq!(range_keys(&map, (Excluded(0), Unbounded)), all); - assert_eq!(range_keys(&map, (Included(0), Excluded(size + 1))), all); - assert_eq!(range_keys(&map, (Included(0), Included(size + 1))), all); - assert_eq!(range_keys(&map, (Included(0), Included(size))), all); - assert_eq!(range_keys(&map, (Included(0), Unbounded)), all); - assert_eq!(range_keys(&map, (Included(1), Excluded(size + 1))), all); - assert_eq!(range_keys(&map, (Included(1), Included(size + 1))), all); - assert_eq!(range_keys(&map, (Included(1), Included(size))), all); - assert_eq!(range_keys(&map, (Included(1), Unbounded)), all); - assert_eq!(range_keys(&map, (Unbounded, Excluded(size + 1))), all); - assert_eq!(range_keys(&map, (Unbounded, Included(size + 1))), all); - assert_eq!(range_keys(&map, (Unbounded, Included(size))), all); - assert_eq!(range_keys(&map, ..), all); - - assert_eq!(range_keys(&map, (Excluded(0), Excluded(1))), vec![]); - assert_eq!(range_keys(&map, (Excluded(0), Included(0))), vec![]); - assert_eq!(range_keys(&map, (Included(0), Included(0))), vec![]); - assert_eq!(range_keys(&map, (Included(0), Excluded(1))), vec![]); - assert_eq!(range_keys(&map, (Unbounded, Excluded(1))), vec![]); - assert_eq!(range_keys(&map, (Unbounded, Included(0))), vec![]); - assert_eq!(range_keys(&map, (Excluded(0), Excluded(2))), first); - assert_eq!(range_keys(&map, (Excluded(0), Included(1))), first); - assert_eq!(range_keys(&map, (Included(0), Excluded(2))), first); - assert_eq!(range_keys(&map, (Included(0), Included(1))), first); - assert_eq!(range_keys(&map, (Included(1), Excluded(2))), first); - assert_eq!(range_keys(&map, (Included(1), Included(1))), first); - assert_eq!(range_keys(&map, (Unbounded, Excluded(2))), first); - assert_eq!(range_keys(&map, (Unbounded, Included(1))), first); - assert_eq!(range_keys(&map, (Excluded(size - 1), Excluded(size + 1))), last); - assert_eq!(range_keys(&map, (Excluded(size - 1), Included(size + 1))), last); - assert_eq!(range_keys(&map, (Excluded(size - 1), Included(size))), last); - assert_eq!(range_keys(&map, (Excluded(size - 1), Unbounded)), last); - assert_eq!(range_keys(&map, (Included(size), Excluded(size + 1))), last); - assert_eq!(range_keys(&map, (Included(size), Included(size + 1))), last); - assert_eq!(range_keys(&map, (Included(size), Included(size))), last); - assert_eq!(range_keys(&map, (Included(size), Unbounded)), last); - assert_eq!(range_keys(&map, (Excluded(size), Excluded(size + 1))), vec![]); - assert_eq!(range_keys(&map, (Excluded(size), Included(size))), vec![]); - assert_eq!(range_keys(&map, (Excluded(size), Unbounded)), vec![]); - assert_eq!(range_keys(&map, (Included(size + 1), Excluded(size + 1))), vec![]); - assert_eq!(range_keys(&map, (Included(size + 1), Included(size + 1))), vec![]); - assert_eq!(range_keys(&map, (Included(size + 1), Unbounded)), vec![]); - - assert_eq!(range_keys(&map, ..3), vec![1, 2]); - assert_eq!(range_keys(&map, 3..), vec![3, 4]); - assert_eq!(range_keys(&map, 2..=3), vec![2, 3]); -} - -#[test] -fn test_range_height_2() { - // Assuming that node.CAPACITY is 11, having 12 pairs implies a height 2 tree - // with 2 leaves. Depending on details we don't want or need to rely upon, - // the single key at the root will be 6 or 7. - - let map: BTreeMap<_, _> = (1..=12).map(|i| (i, i)).collect(); - for &root in &[6, 7] { - assert_eq!(range_keys(&map, (Excluded(root), Excluded(root + 1))), vec![]); - assert_eq!(range_keys(&map, (Excluded(root), Included(root + 1))), vec![root + 1]); - assert_eq!(range_keys(&map, (Included(root), Excluded(root + 1))), vec![root]); - assert_eq!(range_keys(&map, (Included(root), Included(root + 1))), vec![root, root + 1]); - - assert_eq!(range_keys(&map, (Excluded(root - 1), Excluded(root))), vec![]); - assert_eq!(range_keys(&map, (Included(root - 1), Excluded(root))), vec![root - 1]); - assert_eq!(range_keys(&map, (Excluded(root - 1), Included(root))), vec![root]); - assert_eq!(range_keys(&map, (Included(root - 1), Included(root))), vec![root - 1, root]); - } -} - -#[test] -fn test_range_large() { - let size = 200; - - let map: BTreeMap<_, _> = (1..=size).map(|i| (i, i)).collect(); - let all: Vec<_> = (1..=size).collect(); - let (first, last) = (vec![all[0]], vec![all[size as usize - 1]]); - - assert_eq!(range_keys(&map, (Excluded(0), Excluded(size + 1))), all); - assert_eq!(range_keys(&map, (Excluded(0), Included(size + 1))), all); - assert_eq!(range_keys(&map, (Excluded(0), Included(size))), all); - assert_eq!(range_keys(&map, (Excluded(0), Unbounded)), all); - assert_eq!(range_keys(&map, (Included(0), Excluded(size + 1))), all); - assert_eq!(range_keys(&map, (Included(0), Included(size + 1))), all); - assert_eq!(range_keys(&map, (Included(0), Included(size))), all); - assert_eq!(range_keys(&map, (Included(0), Unbounded)), all); - assert_eq!(range_keys(&map, (Included(1), Excluded(size + 1))), all); - assert_eq!(range_keys(&map, (Included(1), Included(size + 1))), all); - assert_eq!(range_keys(&map, (Included(1), Included(size))), all); - assert_eq!(range_keys(&map, (Included(1), Unbounded)), all); - assert_eq!(range_keys(&map, (Unbounded, Excluded(size + 1))), all); - assert_eq!(range_keys(&map, (Unbounded, Included(size + 1))), all); - assert_eq!(range_keys(&map, (Unbounded, Included(size))), all); - assert_eq!(range_keys(&map, ..), all); - - assert_eq!(range_keys(&map, (Excluded(0), Excluded(1))), vec![]); - assert_eq!(range_keys(&map, (Excluded(0), Included(0))), vec![]); - assert_eq!(range_keys(&map, (Included(0), Included(0))), vec![]); - assert_eq!(range_keys(&map, (Included(0), Excluded(1))), vec![]); - assert_eq!(range_keys(&map, (Unbounded, Excluded(1))), vec![]); - assert_eq!(range_keys(&map, (Unbounded, Included(0))), vec![]); - assert_eq!(range_keys(&map, (Excluded(0), Excluded(2))), first); - assert_eq!(range_keys(&map, (Excluded(0), Included(1))), first); - assert_eq!(range_keys(&map, (Included(0), Excluded(2))), first); - assert_eq!(range_keys(&map, (Included(0), Included(1))), first); - assert_eq!(range_keys(&map, (Included(1), Excluded(2))), first); - assert_eq!(range_keys(&map, (Included(1), Included(1))), first); - assert_eq!(range_keys(&map, (Unbounded, Excluded(2))), first); - assert_eq!(range_keys(&map, (Unbounded, Included(1))), first); - assert_eq!(range_keys(&map, (Excluded(size - 1), Excluded(size + 1))), last); - assert_eq!(range_keys(&map, (Excluded(size - 1), Included(size + 1))), last); - assert_eq!(range_keys(&map, (Excluded(size - 1), Included(size))), last); - assert_eq!(range_keys(&map, (Excluded(size - 1), Unbounded)), last); - assert_eq!(range_keys(&map, (Included(size), Excluded(size + 1))), last); - assert_eq!(range_keys(&map, (Included(size), Included(size + 1))), last); - assert_eq!(range_keys(&map, (Included(size), Included(size))), last); - assert_eq!(range_keys(&map, (Included(size), Unbounded)), last); - assert_eq!(range_keys(&map, (Excluded(size), Excluded(size + 1))), vec![]); - assert_eq!(range_keys(&map, (Excluded(size), Included(size))), vec![]); - assert_eq!(range_keys(&map, (Excluded(size), Unbounded)), vec![]); - assert_eq!(range_keys(&map, (Included(size + 1), Excluded(size + 1))), vec![]); - assert_eq!(range_keys(&map, (Included(size + 1), Included(size + 1))), vec![]); - assert_eq!(range_keys(&map, (Included(size + 1), Unbounded)), vec![]); - - fn check<'a, L, R>(lhs: L, rhs: R) - where - L: IntoIterator, - R: IntoIterator, - { - let lhs: Vec<_> = lhs.into_iter().collect(); - let rhs: Vec<_> = rhs.into_iter().collect(); - assert_eq!(lhs, rhs); - } - - check(map.range(..=100), map.range(..101)); - check(map.range(5..=8), vec![(&5, &5), (&6, &6), (&7, &7), (&8, &8)]); - check(map.range(-1..=2), vec![(&1, &1), (&2, &2)]); -} - -#[test] -fn test_range_inclusive_max_value() { - let max = std::usize::MAX; - let map: BTreeMap<_, _> = vec![(max, 0)].into_iter().collect(); - - assert_eq!(map.range(max..=max).collect::>(), &[(&max, &0)]); -} - -#[test] -fn test_range_equal_empty_cases() { - let map: BTreeMap<_, _> = (0..5).map(|i| (i, i)).collect(); - assert_eq!(map.range((Included(2), Excluded(2))).next(), None); - assert_eq!(map.range((Excluded(2), Included(2))).next(), None); -} - -#[test] -#[should_panic] -fn test_range_equal_excluded() { - let map: BTreeMap<_, _> = (0..5).map(|i| (i, i)).collect(); - map.range((Excluded(2), Excluded(2))); -} - -#[test] -#[should_panic] -fn test_range_backwards_1() { - let map: BTreeMap<_, _> = (0..5).map(|i| (i, i)).collect(); - map.range((Included(3), Included(2))); -} - -#[test] -#[should_panic] -fn test_range_backwards_2() { - let map: BTreeMap<_, _> = (0..5).map(|i| (i, i)).collect(); - map.range((Included(3), Excluded(2))); -} - -#[test] -#[should_panic] -fn test_range_backwards_3() { - let map: BTreeMap<_, _> = (0..5).map(|i| (i, i)).collect(); - map.range((Excluded(3), Included(2))); -} - -#[test] -#[should_panic] -fn test_range_backwards_4() { - let map: BTreeMap<_, _> = (0..5).map(|i| (i, i)).collect(); - map.range((Excluded(3), Excluded(2))); -} - -#[test] -fn test_range_1000() { - #[cfg(not(miri))] // Miri is too slow - let size = 1000; - #[cfg(miri)] - let size = 144; // to obtain height 3 tree (having edges to both kinds of nodes) - let map: BTreeMap<_, _> = (0..size).map(|i| (i, i)).collect(); - - fn test(map: &BTreeMap, size: u32, min: Bound<&u32>, max: Bound<&u32>) { - let mut kvs = map.range((min, max)).map(|(&k, &v)| (k, v)); - let mut pairs = (0..size).map(|i| (i, i)); - - for (kv, pair) in kvs.by_ref().zip(pairs.by_ref()) { - assert_eq!(kv, pair); - } - assert_eq!(kvs.next(), None); - assert_eq!(pairs.next(), None); - } - test(&map, size, Included(&0), Excluded(&size)); - test(&map, size, Unbounded, Excluded(&size)); - test(&map, size, Included(&0), Included(&(size - 1))); - test(&map, size, Unbounded, Included(&(size - 1))); - test(&map, size, Included(&0), Unbounded); - test(&map, size, Unbounded, Unbounded); -} - -#[test] -fn test_range_borrowed_key() { - let mut map = BTreeMap::new(); - map.insert("aardvark".to_string(), 1); - map.insert("baboon".to_string(), 2); - map.insert("coyote".to_string(), 3); - map.insert("dingo".to_string(), 4); - // NOTE: would like to use simply "b".."d" here... - let mut iter = map.range::((Included("b"), Excluded("d"))); - assert_eq!(iter.next(), Some((&"baboon".to_string(), &2))); - assert_eq!(iter.next(), Some((&"coyote".to_string(), &3))); - assert_eq!(iter.next(), None); -} - -#[test] -fn test_range() { - let size = 200; - #[cfg(not(miri))] // Miri is too slow - let step = 1; - #[cfg(miri)] - let step = 66; - let map: BTreeMap<_, _> = (0..size).map(|i| (i, i)).collect(); - - for i in (0..size).step_by(step) { - for j in (i..size).step_by(step) { - let mut kvs = map.range((Included(&i), Included(&j))).map(|(&k, &v)| (k, v)); - let mut pairs = (i..=j).map(|i| (i, i)); - - for (kv, pair) in kvs.by_ref().zip(pairs.by_ref()) { - assert_eq!(kv, pair); - } - assert_eq!(kvs.next(), None); - assert_eq!(pairs.next(), None); - } - } -} - -#[test] -fn test_range_mut() { - let size = 200; - #[cfg(not(miri))] // Miri is too slow - let step = 1; - #[cfg(miri)] - let step = 66; - let mut map: BTreeMap<_, _> = (0..size).map(|i| (i, i)).collect(); - - for i in (0..size).step_by(step) { - for j in (i..size).step_by(step) { - let mut kvs = map.range_mut((Included(&i), Included(&j))).map(|(&k, &mut v)| (k, v)); - let mut pairs = (i..=j).map(|i| (i, i)); - - for (kv, pair) in kvs.by_ref().zip(pairs.by_ref()) { - assert_eq!(kv, pair); - } - assert_eq!(kvs.next(), None); - assert_eq!(pairs.next(), None); - } - } -} - -#[test] -fn test_borrow() { - // make sure these compile -- using the Borrow trait - { - let mut map = BTreeMap::new(); - map.insert("0".to_string(), 1); - assert_eq!(map["0"], 1); - } - - { - let mut map = BTreeMap::new(); - map.insert(Box::new(0), 1); - assert_eq!(map[&0], 1); - } - - { - let mut map = BTreeMap::new(); - map.insert(Box::new([0, 1]) as Box<[i32]>, 1); - assert_eq!(map[&[0, 1][..]], 1); - } - - { - let mut map = BTreeMap::new(); - map.insert(Rc::new(0), 1); - assert_eq!(map[&0], 1); - } -} - -#[test] -fn test_entry() { - let xs = [(1, 10), (2, 20), (3, 30), (4, 40), (5, 50), (6, 60)]; - - let mut map: BTreeMap<_, _> = xs.iter().cloned().collect(); - - // Existing key (insert) - match map.entry(1) { - Vacant(_) => unreachable!(), - Occupied(mut view) => { - assert_eq!(view.get(), &10); - assert_eq!(view.insert(100), 10); - } - } - assert_eq!(map.get(&1).unwrap(), &100); - assert_eq!(map.len(), 6); - - // Existing key (update) - match map.entry(2) { - Vacant(_) => unreachable!(), - Occupied(mut view) => { - let v = view.get_mut(); - *v *= 10; - } - } - assert_eq!(map.get(&2).unwrap(), &200); - assert_eq!(map.len(), 6); - - // Existing key (take) - match map.entry(3) { - Vacant(_) => unreachable!(), - Occupied(view) => { - assert_eq!(view.remove(), 30); - } - } - assert_eq!(map.get(&3), None); - assert_eq!(map.len(), 5); - - // Inexistent key (insert) - match map.entry(10) { - Occupied(_) => unreachable!(), - Vacant(view) => { - assert_eq!(*view.insert(1000), 1000); - } - } - assert_eq!(map.get(&10).unwrap(), &1000); - assert_eq!(map.len(), 6); -} - -#[test] -fn test_extend_ref() { - let mut a = BTreeMap::new(); - a.insert(1, "one"); - let mut b = BTreeMap::new(); - b.insert(2, "two"); - b.insert(3, "three"); - - a.extend(&b); - - assert_eq!(a.len(), 3); - assert_eq!(a[&1], "one"); - assert_eq!(a[&2], "two"); - assert_eq!(a[&3], "three"); -} - -#[test] -fn test_zst() { - let mut m = BTreeMap::new(); - assert_eq!(m.len(), 0); - - assert_eq!(m.insert((), ()), None); - assert_eq!(m.len(), 1); - - assert_eq!(m.insert((), ()), Some(())); - assert_eq!(m.len(), 1); - assert_eq!(m.iter().count(), 1); - - m.clear(); - assert_eq!(m.len(), 0); - - for _ in 0..100 { - m.insert((), ()); - } - - assert_eq!(m.len(), 1); - assert_eq!(m.iter().count(), 1); -} - -// This test's only purpose is to ensure that zero-sized keys with nonsensical orderings -// do not cause segfaults when used with zero-sized values. All other map behavior is -// undefined. -#[test] -fn test_bad_zst() { - use std::cmp::Ordering; - - struct Bad; - - impl PartialEq for Bad { - fn eq(&self, _: &Self) -> bool { - false - } - } - - impl Eq for Bad {} - - impl PartialOrd for Bad { - fn partial_cmp(&self, _: &Self) -> Option { - Some(Ordering::Less) - } - } - - impl Ord for Bad { - fn cmp(&self, _: &Self) -> Ordering { - Ordering::Less - } - } - - let mut m = BTreeMap::new(); - - for _ in 0..100 { - m.insert(Bad, Bad); - } -} - -#[test] -fn test_clone() { - let mut map = BTreeMap::new(); - let size = 12; // to obtain height 2 tree (having edges to leaf nodes) - assert_eq!(map.len(), 0); - - for i in 0..size { - assert_eq!(map.insert(i, 10 * i), None); - assert_eq!(map.len(), i + 1); - assert_eq!(map, map.clone()); - } - - for i in 0..size { - assert_eq!(map.insert(i, 100 * i), Some(10 * i)); - assert_eq!(map.len(), size); - assert_eq!(map, map.clone()); - } - - for i in 0..size / 2 { - assert_eq!(map.remove(&(i * 2)), Some(i * 200)); - assert_eq!(map.len(), size - i - 1); - assert_eq!(map, map.clone()); - } - - for i in 0..size / 2 { - assert_eq!(map.remove(&(2 * i)), None); - assert_eq!(map.remove(&(2 * i + 1)), Some(i * 200 + 100)); - assert_eq!(map.len(), size / 2 - i - 1); - assert_eq!(map, map.clone()); - } - - // Full 2-level and minimal 3-level tree (sizes 143, 144 -- the only ones we clone for). - for i in 1..=144 { - assert_eq!(map.insert(i, i), None); - assert_eq!(map.len(), i); - if i >= 143 { - assert_eq!(map, map.clone()); - } - } -} - -#[test] -fn test_clone_from() { - let mut map1 = BTreeMap::new(); - let max_size = 12; // to obtain height 2 tree (having edges to leaf nodes) - - // Range to max_size inclusive, because i is the size of map1 being tested. - for i in 0..=max_size { - let mut map2 = BTreeMap::new(); - for j in 0..i { - let mut map1_copy = map2.clone(); - map1_copy.clone_from(&map1); // small cloned from large - assert_eq!(map1_copy, map1); - let mut map2_copy = map1.clone(); - map2_copy.clone_from(&map2); // large cloned from small - assert_eq!(map2_copy, map2); - map2.insert(100 * j + 1, 2 * j + 1); - } - map2.clone_from(&map1); // same length - assert_eq!(map2, map1); - map1.insert(i, 10 * i); - } -} - -#[test] -#[allow(dead_code)] -fn test_variance() { - use std::collections::btree_map::{IntoIter, Iter, Keys, Range, Values}; - - fn map_key<'new>(v: BTreeMap<&'static str, ()>) -> BTreeMap<&'new str, ()> { - v - } - fn map_val<'new>(v: BTreeMap<(), &'static str>) -> BTreeMap<(), &'new str> { - v - } - fn iter_key<'a, 'new>(v: Iter<'a, &'static str, ()>) -> Iter<'a, &'new str, ()> { - v - } - fn iter_val<'a, 'new>(v: Iter<'a, (), &'static str>) -> Iter<'a, (), &'new str> { - v - } - fn into_iter_key<'new>(v: IntoIter<&'static str, ()>) -> IntoIter<&'new str, ()> { - v - } - fn into_iter_val<'new>(v: IntoIter<(), &'static str>) -> IntoIter<(), &'new str> { - v - } - fn range_key<'a, 'new>(v: Range<'a, &'static str, ()>) -> Range<'a, &'new str, ()> { - v - } - fn range_val<'a, 'new>(v: Range<'a, (), &'static str>) -> Range<'a, (), &'new str> { - v - } - fn keys<'a, 'new>(v: Keys<'a, &'static str, ()>) -> Keys<'a, &'new str, ()> { - v - } - fn vals<'a, 'new>(v: Values<'a, (), &'static str>) -> Values<'a, (), &'new str> { - v - } -} - -#[test] -fn test_occupied_entry_key() { - let mut a = BTreeMap::new(); - let key = "hello there"; - let value = "value goes here"; - assert!(a.is_empty()); - a.insert(key.clone(), value.clone()); - assert_eq!(a.len(), 1); - assert_eq!(a[key], value); - - match a.entry(key.clone()) { - Vacant(_) => panic!(), - Occupied(e) => assert_eq!(key, *e.key()), - } - assert_eq!(a.len(), 1); - assert_eq!(a[key], value); -} - -#[test] -fn test_vacant_entry_key() { - let mut a = BTreeMap::new(); - let key = "hello there"; - let value = "value goes here"; - - assert!(a.is_empty()); - match a.entry(key.clone()) { - Occupied(_) => panic!(), - Vacant(e) => { - assert_eq!(key, *e.key()); - e.insert(value.clone()); - } - } - assert_eq!(a.len(), 1); - assert_eq!(a[key], value); -} - -#[test] -fn test_first_last_entry() { - let mut a = BTreeMap::new(); - assert!(a.first_entry().is_none()); - assert!(a.last_entry().is_none()); - a.insert(1, 42); - assert_eq!(a.first_entry().unwrap().key(), &1); - assert_eq!(a.last_entry().unwrap().key(), &1); - a.insert(2, 24); - assert_eq!(a.first_entry().unwrap().key(), &1); - assert_eq!(a.last_entry().unwrap().key(), &2); - a.insert(0, 6); - assert_eq!(a.first_entry().unwrap().key(), &0); - assert_eq!(a.last_entry().unwrap().key(), &2); - let (k1, v1) = a.first_entry().unwrap().remove_entry(); - assert_eq!(k1, 0); - assert_eq!(v1, 6); - let (k2, v2) = a.last_entry().unwrap().remove_entry(); - assert_eq!(k2, 2); - assert_eq!(v2, 24); - assert_eq!(a.first_entry().unwrap().key(), &1); - assert_eq!(a.last_entry().unwrap().key(), &1); -} - -macro_rules! create_append_test { - ($name:ident, $len:expr) => { - #[test] - fn $name() { - let mut a = BTreeMap::new(); - for i in 0..8 { - a.insert(i, i); - } - - let mut b = BTreeMap::new(); - for i in 5..$len { - b.insert(i, 2 * i); - } - - a.append(&mut b); - - assert_eq!(a.len(), $len); - assert_eq!(b.len(), 0); - - for i in 0..$len { - if i < 5 { - assert_eq!(a[&i], i); - } else { - assert_eq!(a[&i], 2 * i); - } - } - - assert_eq!(a.remove(&($len - 1)), Some(2 * ($len - 1))); - assert_eq!(a.insert($len - 1, 20), None); - } - }; -} - -// These are mostly for testing the algorithm that "fixes" the right edge after insertion. -// Single node. -create_append_test!(test_append_9, 9); -// Two leafs that don't need fixing. -create_append_test!(test_append_17, 17); -// Two leafs where the second one ends up underfull and needs stealing at the end. -create_append_test!(test_append_14, 14); -// Two leafs where the second one ends up empty because the insertion finished at the root. -create_append_test!(test_append_12, 12); -// Three levels; insertion finished at the root. -create_append_test!(test_append_144, 144); -// Three levels; insertion finished at leaf while there is an empty node on the second level. -create_append_test!(test_append_145, 145); -// Tests for several randomly chosen sizes. -create_append_test!(test_append_170, 170); -create_append_test!(test_append_181, 181); -#[cfg(not(miri))] // Miri is too slow -create_append_test!(test_append_239, 239); -#[cfg(not(miri))] // Miri is too slow -create_append_test!(test_append_1700, 1700); - -fn rand_data(len: usize) -> Vec<(u32, u32)> { - let mut rng = DeterministicRng::new(); - Vec::from_iter((0..len).map(|_| (rng.next(), rng.next()))) -} - -#[test] -fn test_split_off_empty_right() { - let mut data = rand_data(173); - - let mut map = BTreeMap::from_iter(data.clone()); - let right = map.split_off(&(data.iter().max().unwrap().0 + 1)); - - data.sort(); - assert!(map.into_iter().eq(data)); - assert!(right.into_iter().eq(None)); -} - -#[test] -fn test_split_off_empty_left() { - let mut data = rand_data(314); - - let mut map = BTreeMap::from_iter(data.clone()); - let right = map.split_off(&data.iter().min().unwrap().0); - - data.sort(); - assert!(map.into_iter().eq(None)); - assert!(right.into_iter().eq(data)); -} - -#[test] -fn test_split_off_large_random_sorted() { - #[cfg(not(miri))] // Miri is too slow - let mut data = rand_data(1529); - #[cfg(miri)] - let mut data = rand_data(529); - // special case with maximum height. - data.sort(); - - let mut map = BTreeMap::from_iter(data.clone()); - let key = data[data.len() / 2].0; - let right = map.split_off(&key); - - assert!(map.into_iter().eq(data.clone().into_iter().filter(|x| x.0 < key))); - assert!(right.into_iter().eq(data.into_iter().filter(|x| x.0 >= key))); -} - -#[test] -fn test_into_iter_drop_leak_1() { - static DROPS: AtomicU32 = AtomicU32::new(0); - - struct D; - - impl Drop for D { - fn drop(&mut self) { - if DROPS.fetch_add(1, Ordering::SeqCst) == 3 { - panic!("panic in `drop`"); - } - } - } - - let mut map = BTreeMap::new(); - map.insert("a", D); - map.insert("b", D); - map.insert("c", D); - map.insert("d", D); - map.insert("e", D); - - catch_unwind(move || drop(map.into_iter())).ok(); - - assert_eq!(DROPS.load(Ordering::SeqCst), 5); -} - -#[test] -fn test_into_iter_drop_leak_2() { - let size = 12; // to obtain tree with 2 levels (having edges to leaf nodes) - static DROPS: AtomicU32 = AtomicU32::new(0); - static PANIC_POINT: AtomicU32 = AtomicU32::new(0); - - struct D; - impl Drop for D { - fn drop(&mut self) { - if DROPS.fetch_add(1, Ordering::SeqCst) == PANIC_POINT.load(Ordering::SeqCst) { - panic!("panic in `drop`"); - } - } - } - - for panic_point in vec![0, 1, size - 2, size - 1] { - DROPS.store(0, Ordering::SeqCst); - PANIC_POINT.store(panic_point, Ordering::SeqCst); - let map: BTreeMap<_, _> = (0..size).map(|i| (i, D)).collect(); - catch_unwind(move || drop(map.into_iter())).ok(); - assert_eq!(DROPS.load(Ordering::SeqCst), size); - } -} diff --git a/crux-mir/lib/alloc/tests/btree/mod.rs b/crux-mir/lib/alloc/tests/btree/mod.rs deleted file mode 100644 index 1d08ae13e..000000000 --- a/crux-mir/lib/alloc/tests/btree/mod.rs +++ /dev/null @@ -1,27 +0,0 @@ -mod map; -mod set; - -/// XorShiftRng -struct DeterministicRng { - x: u32, - y: u32, - z: u32, - w: u32, -} - -impl DeterministicRng { - fn new() -> Self { - DeterministicRng { x: 0x193a6754, y: 0xa8a7d469, z: 0x97830e05, w: 0x113ba7bb } - } - - fn next(&mut self) -> u32 { - let x = self.x; - let t = x ^ (x << 11); - self.x = self.y; - self.y = self.z; - self.z = self.w; - let w_ = self.w; - self.w = w_ ^ (w_ >> 19) ^ (t ^ (t >> 8)); - self.w - } -} diff --git a/crux-mir/lib/alloc/tests/btree_set_hash.rs b/crux-mir/lib/alloc/tests/btree_set_hash.rs new file mode 100644 index 000000000..ab275ac43 --- /dev/null +++ b/crux-mir/lib/alloc/tests/btree_set_hash.rs @@ -0,0 +1,29 @@ +use crate::hash; +use std::collections::BTreeSet; + +#[test] +fn test_hash() { + let mut x = BTreeSet::new(); + let mut y = BTreeSet::new(); + + x.insert(1); + x.insert(2); + x.insert(3); + + y.insert(3); + y.insert(2); + y.insert(1); + + assert_eq!(hash(&x), hash(&y)); +} + +#[test] +fn test_prefix_free() { + let x = BTreeSet::from([1, 2, 3]); + let y = BTreeSet::::new(); + + // If hashed by iteration alone, `(x, y)` and `(y, x)` would visit the same + // order of elements, resulting in the same hash. But now that we also hash + // the length, they get distinct sequences of hashed data. + assert_ne!(hash(&(&x, &y)), hash(&(&y, &x))); +} diff --git a/crux-mir/lib/alloc/tests/c_str.rs b/crux-mir/lib/alloc/tests/c_str.rs new file mode 100644 index 000000000..4a5817939 --- /dev/null +++ b/crux-mir/lib/alloc/tests/c_str.rs @@ -0,0 +1,19 @@ +use std::borrow::Cow::{Borrowed, Owned}; +use std::ffi::CStr; +use std::os::raw::c_char; + +#[test] +fn to_str() { + let data = b"123\xE2\x80\xA6\0"; + let ptr = data.as_ptr() as *const c_char; + unsafe { + assert_eq!(CStr::from_ptr(ptr).to_str(), Ok("123…")); + assert_eq!(CStr::from_ptr(ptr).to_string_lossy(), Borrowed("123…")); + } + let data = b"123\xE2\0"; + let ptr = data.as_ptr() as *const c_char; + unsafe { + assert!(CStr::from_ptr(ptr).to_str().is_err()); + assert_eq!(CStr::from_ptr(ptr).to_string_lossy(), Owned::(format!("123\u{FFFD}"))); + } +} diff --git a/crux-mir/lib/alloc/tests/const_fns.rs b/crux-mir/lib/alloc/tests/const_fns.rs new file mode 100644 index 000000000..49b837bec --- /dev/null +++ b/crux-mir/lib/alloc/tests/const_fns.rs @@ -0,0 +1,35 @@ +// Test const functions in the library + +pub const MY_VEC: Vec = Vec::new(); +pub const MY_VEC2: Vec = Default::default(); + +pub const MY_STRING: String = String::new(); +pub const MY_STRING2: String = Default::default(); + +pub const MY_BOXED_SLICE: Box<[usize]> = Default::default(); +pub const MY_BOXED_STR: Box = Default::default(); + +use std::collections::{BTreeMap, BTreeSet}; + +pub const MY_BTREEMAP: BTreeMap = BTreeMap::new(); +pub const MAP: &'static BTreeMap = &MY_BTREEMAP; +pub const MAP_LEN: usize = MAP.len(); +pub const MAP_IS_EMPTY: bool = MAP.is_empty(); + +pub const MY_BTREESET: BTreeSet = BTreeSet::new(); +pub const SET: &'static BTreeSet = &MY_BTREESET; +pub const SET_LEN: usize = SET.len(); +pub const SET_IS_EMPTY: bool = SET.is_empty(); + +#[test] +fn test_const() { + assert_eq!(MY_VEC, MY_VEC2); + assert_eq!(MY_STRING, MY_STRING2); + + assert_eq!(MY_VEC, *MY_BOXED_SLICE); + assert_eq!(MY_STRING, *MY_BOXED_STR); + + assert_eq!(MAP_LEN, 0); + assert_eq!(SET_LEN, 0); + assert!(MAP_IS_EMPTY && SET_IS_EMPTY); +} diff --git a/crux-mir/lib/alloc/tests/fmt.rs b/crux-mir/lib/alloc/tests/fmt.rs index 0ad092b49..04da95bbb 100644 --- a/crux-mir/lib/alloc/tests/fmt.rs +++ b/crux-mir/lib/alloc/tests/fmt.rs @@ -1,7 +1,324 @@ -use std::fmt; +#![deny(warnings)] + +use std::cell::RefCell; +use std::fmt::{self, Write}; +use std::ptr; #[test] fn test_format() { let s = fmt::format(format_args!("Hello, {}!", "world")); assert_eq!(s, "Hello, world!"); } + +struct A; +struct B; +struct C; +struct D; + +impl fmt::LowerHex for A { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("aloha") + } +} +impl fmt::UpperHex for B { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("adios") + } +} +impl fmt::Display for C { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad_integral(true, "☃", "123") + } +} +impl fmt::Binary for D { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("aa")?; + f.write_char('☃')?; + f.write_str("bb") + } +} + +macro_rules! t { + ($a:expr, $b:expr) => { + assert_eq!($a, $b) + }; +} + +#[test] +fn test_format_macro_interface() { + // Various edge cases without formats + t!(format!(""), ""); + t!(format!("hello"), "hello"); + t!(format!("hello {{"), "hello {"); + + // default formatters should work + t!(format!("{}", 1.0f32), "1"); + t!(format!("{}", 1.0f64), "1"); + t!(format!("{}", "a"), "a"); + t!(format!("{}", "a".to_string()), "a"); + t!(format!("{}", false), "false"); + t!(format!("{}", 'a'), "a"); + + // At least exercise all the formats + t!(format!("{}", true), "true"); + t!(format!("{}", '☃'), "☃"); + t!(format!("{}", 10), "10"); + t!(format!("{}", 10_usize), "10"); + t!(format!("{:?}", '☃'), "'☃'"); + t!(format!("{:?}", 10), "10"); + t!(format!("{:?}", 10_usize), "10"); + t!(format!("{:?}", "true"), "\"true\""); + t!(format!("{:?}", "foo\nbar"), "\"foo\\nbar\""); + t!(format!("{:?}", "foo\n\"bar\"\r\n\'baz\'\t\\qux\\"), r#""foo\n\"bar\"\r\n'baz'\t\\qux\\""#); + t!(format!("{:?}", "foo\0bar\x01baz\u{7f}q\u{75}x"), r#""foo\0bar\u{1}baz\u{7f}qux""#); + t!(format!("{:o}", 10_usize), "12"); + t!(format!("{:x}", 10_usize), "a"); + t!(format!("{:X}", 10_usize), "A"); + t!(format!("{}", "foo"), "foo"); + t!(format!("{}", "foo".to_string()), "foo"); + if cfg!(target_pointer_width = "32") { + t!(format!("{:#p}", ptr::invalid::(0x1234)), "0x00001234"); + t!(format!("{:#p}", ptr::invalid_mut::(0x1234)), "0x00001234"); + } else { + t!(format!("{:#p}", ptr::invalid::(0x1234)), "0x0000000000001234"); + t!(format!("{:#p}", ptr::invalid_mut::(0x1234)), "0x0000000000001234"); + } + t!(format!("{:p}", ptr::invalid::(0x1234)), "0x1234"); + t!(format!("{:p}", ptr::invalid_mut::(0x1234)), "0x1234"); + t!(format!("{A:x}"), "aloha"); + t!(format!("{B:X}"), "adios"); + t!(format!("foo {} ☃☃☃☃☃☃", "bar"), "foo bar ☃☃☃☃☃☃"); + t!(format!("{1} {0}", 0, 1), "1 0"); + t!(format!("{foo} {bar}", foo = 0, bar = 1), "0 1"); + t!(format!("{foo} {1} {bar} {0}", 0, 1, foo = 2, bar = 3), "2 1 3 0"); + t!(format!("{} {0}", "a"), "a a"); + t!(format!("{_foo}", _foo = 6usize), "6"); + t!(format!("{foo_bar}", foo_bar = 1), "1"); + t!(format!("{}", 5 + 5), "10"); + t!(format!("{C:#4}"), "☃123"); + t!(format!("{D:b}"), "aa☃bb"); + + let a: &dyn fmt::Debug = &1; + t!(format!("{a:?}"), "1"); + + // Formatting strings and their arguments + t!(format!("{}", "a"), "a"); + t!(format!("{:4}", "a"), "a "); + t!(format!("{:4}", "☃"), "☃ "); + t!(format!("{:>4}", "a"), " a"); + t!(format!("{:<4}", "a"), "a "); + t!(format!("{:^5}", "a"), " a "); + t!(format!("{:^5}", "aa"), " aa "); + t!(format!("{:^4}", "a"), " a "); + t!(format!("{:^4}", "aa"), " aa "); + t!(format!("{:.4}", "a"), "a"); + t!(format!("{:4.4}", "a"), "a "); + t!(format!("{:4.4}", "aaaaaaaaaaaaaaaaaa"), "aaaa"); + t!(format!("{:<4.4}", "aaaaaaaaaaaaaaaaaa"), "aaaa"); + t!(format!("{:>4.4}", "aaaaaaaaaaaaaaaaaa"), "aaaa"); + t!(format!("{:^4.4}", "aaaaaaaaaaaaaaaaaa"), "aaaa"); + t!(format!("{:>10.4}", "aaaaaaaaaaaaaaaaaa"), " aaaa"); + t!(format!("{:2.4}", "aaaaa"), "aaaa"); + t!(format!("{:2.4}", "aaaa"), "aaaa"); + t!(format!("{:2.4}", "aaa"), "aaa"); + t!(format!("{:2.4}", "aa"), "aa"); + t!(format!("{:2.4}", "a"), "a "); + t!(format!("{:0>2}", "a"), "0a"); + t!(format!("{:.*}", 4, "aaaaaaaaaaaaaaaaaa"), "aaaa"); + t!(format!("{:.1$}", "aaaaaaaaaaaaaaaaaa", 4), "aaaa"); + t!(format!("{:.a$}", "aaaaaaaaaaaaaaaaaa", a = 4), "aaaa"); + t!(format!("{:._a$}", "aaaaaaaaaaaaaaaaaa", _a = 4), "aaaa"); + t!(format!("{:1$}", "a", 4), "a "); + t!(format!("{1:0$}", 4, "a"), "a "); + t!(format!("{:a$}", "a", a = 4), "a "); + t!(format!("{:-#}", "a"), "a"); + t!(format!("{:+#}", "a"), "a"); + t!(format!("{:/^10.8}", "1234567890"), "/12345678/"); + + // Some float stuff + t!(format!("{:}", 1.0f32), "1"); + t!(format!("{:}", 1.0f64), "1"); + t!(format!("{:.3}", 1.0f64), "1.000"); + t!(format!("{:10.3}", 1.0f64), " 1.000"); + t!(format!("{:+10.3}", 1.0f64), " +1.000"); + t!(format!("{:+10.3}", -1.0f64), " -1.000"); + + t!(format!("{:e}", 1.2345e6f32), "1.2345e6"); + t!(format!("{:e}", 1.2345e6f64), "1.2345e6"); + t!(format!("{:E}", 1.2345e6f64), "1.2345E6"); + t!(format!("{:.3e}", 1.2345e6f64), "1.234e6"); + t!(format!("{:10.3e}", 1.2345e6f64), " 1.234e6"); + t!(format!("{:+10.3e}", 1.2345e6f64), " +1.234e6"); + t!(format!("{:+10.3e}", -1.2345e6f64), " -1.234e6"); + + // Float edge cases + t!(format!("{}", -0.0), "-0"); + t!(format!("{:?}", 0.0), "0.0"); + + // sign aware zero padding + t!(format!("{:<3}", 1), "1 "); + t!(format!("{:>3}", 1), " 1"); + t!(format!("{:^3}", 1), " 1 "); + t!(format!("{:03}", 1), "001"); + t!(format!("{:<03}", 1), "001"); + t!(format!("{:>03}", 1), "001"); + t!(format!("{:^03}", 1), "001"); + t!(format!("{:+03}", 1), "+01"); + t!(format!("{:<+03}", 1), "+01"); + t!(format!("{:>+03}", 1), "+01"); + t!(format!("{:^+03}", 1), "+01"); + t!(format!("{:#05x}", 1), "0x001"); + t!(format!("{:<#05x}", 1), "0x001"); + t!(format!("{:>#05x}", 1), "0x001"); + t!(format!("{:^#05x}", 1), "0x001"); + t!(format!("{:05}", 1.2), "001.2"); + t!(format!("{:<05}", 1.2), "001.2"); + t!(format!("{:>05}", 1.2), "001.2"); + t!(format!("{:^05}", 1.2), "001.2"); + t!(format!("{:05}", -1.2), "-01.2"); + t!(format!("{:<05}", -1.2), "-01.2"); + t!(format!("{:>05}", -1.2), "-01.2"); + t!(format!("{:^05}", -1.2), "-01.2"); + t!(format!("{:+05}", 1.2), "+01.2"); + t!(format!("{:<+05}", 1.2), "+01.2"); + t!(format!("{:>+05}", 1.2), "+01.2"); + t!(format!("{:^+05}", 1.2), "+01.2"); + + // Ergonomic format_args! + t!(format!("{0:x} {0:X}", 15), "f F"); + t!(format!("{0:x} {0:X} {}", 15), "f F 15"); + t!(format!("{:x}{0:X}{a:x}{:X}{1:x}{a:X}", 13, 14, a = 15), "dDfEeF"); + t!(format!("{a:x} {a:X}", a = 15), "f F"); + + // And its edge cases + t!( + format!( + "{a:.0$} {b:.0$} {0:.0$}\n{a:.c$} {b:.c$} {c:.c$}", + 4, + a = "abcdefg", + b = "hijklmn", + c = 3 + ), + "abcd hijk 4\nabc hij 3" + ); + t!(format!("{a:.*} {0} {:.*}", 4, 3, "efgh", a = "abcdef"), "abcd 4 efg"); + t!(format!("{:.a$} {a} {a:#x}", "aaaaaa", a = 2), "aa 2 0x2"); + + // Test that pointers don't get truncated. + { + let val = usize::MAX; + let exp = format!("{val:#x}"); + t!(format!("{:p}", std::ptr::invalid::(val)), exp); + } + + // Escaping + t!(format!("{{"), "{"); + t!(format!("}}"), "}"); + + // make sure that format! doesn't move out of local variables + let a = Box::new(3); + format!("{a}"); + format!("{a}"); + + // make sure that format! doesn't cause spurious unused-unsafe warnings when + // it's inside of an outer unsafe block + unsafe { + let a: isize = ::std::mem::transmute(3_usize); + format!("{a}"); + } + + // test that trailing commas are acceptable + format!("{}", "test",); + format!("{foo}", foo = "test",); +} + +// Basic test to make sure that we can invoke the `write!` macro with an +// fmt::Write instance. +#[test] +fn test_write() { + let mut buf = String::new(); + let _ = write!(&mut buf, "{}", 3); + { + let w = &mut buf; + let _ = write!(w, "{foo}", foo = 4); + let _ = write!(w, "{}", "hello"); + let _ = writeln!(w, "{}", "line"); + let _ = writeln!(w, "{foo}", foo = "bar"); + let _ = w.write_char('☃'); + let _ = w.write_str("str"); + } + + t!(buf, "34helloline\nbar\n☃str"); +} + +// Just make sure that the macros are defined, there's not really a lot that we +// can do with them just yet (to test the output) +#[test] +fn test_print() { + print!("hi"); + print!("{:?}", vec![0u8]); + println!("hello"); + println!("this is a {}", "test"); + println!("{foo}", foo = "bar"); +} + +// Just make sure that the macros are defined, there's not really a lot that we +// can do with them just yet (to test the output) +#[test] +fn test_format_args() { + let mut buf = String::new(); + { + let w = &mut buf; + let _ = write!(w, "{}", format_args!("{}", 1)); + let _ = write!(w, "{}", format_args!("test")); + let _ = write!(w, "{}", format_args!("{test}", test = 3)); + } + let s = buf; + t!(s, "1test3"); + + let s = fmt::format(format_args!("hello {}", "world")); + t!(s, "hello world"); + let s = format!("{}: {}", "args were", format_args!("hello {}", "world")); + t!(s, "args were: hello world"); +} + +#[test] +fn test_order() { + // Make sure format!() arguments are always evaluated in a left-to-right + // ordering + fn foo() -> isize { + static mut FOO: isize = 0; + unsafe { + FOO += 1; + FOO + } + } + assert_eq!( + format!("{} {} {a} {b} {} {c}", foo(), foo(), foo(), a = foo(), b = foo(), c = foo()), + "1 2 4 5 3 6".to_string() + ); +} + +#[test] +fn test_once() { + // Make sure each argument are evaluated only once even though it may be + // formatted multiple times + fn foo() -> isize { + static mut FOO: isize = 0; + unsafe { + FOO += 1; + FOO + } + } + assert_eq!(format!("{0} {0} {0} {a} {a} {a}", foo(), a = foo()), "1 1 1 2 2 2".to_string()); +} + +#[test] +fn test_refcell() { + let refcell = RefCell::new(5); + assert_eq!(format!("{refcell:?}"), "RefCell { value: 5 }"); + let borrow = refcell.borrow_mut(); + assert_eq!(format!("{refcell:?}"), "RefCell { value: }"); + drop(borrow); + assert_eq!(format!("{refcell:?}"), "RefCell { value: 5 }"); +} diff --git a/crux-mir/lib/alloc/tests/heap.rs b/crux-mir/lib/alloc/tests/heap.rs index d159126f4..246b341ee 100644 --- a/crux-mir/lib/alloc/tests/heap.rs +++ b/crux-mir/lib/alloc/tests/heap.rs @@ -1,4 +1,4 @@ -use std::alloc::{AllocRef, Global, Layout, System}; +use std::alloc::{Allocator, Global, Layout, System}; /// Issue #45955 and #62251. #[test] @@ -11,7 +11,7 @@ fn std_heap_overaligned_request() { check_overalign_requests(Global) } -fn check_overalign_requests(mut allocator: T) { +fn check_overalign_requests(allocator: T) { for &align in &[4, 8, 16, 32] { // less than and bigger than `MIN_ALIGN` for &size in &[align / 2, align - 1] { @@ -20,12 +20,12 @@ fn check_overalign_requests(mut allocator: T) { unsafe { let pointers: Vec<_> = (0..iterations) .map(|_| { - allocator.alloc(Layout::from_size_align(size, align).unwrap()).unwrap().0 + allocator.allocate(Layout::from_size_align(size, align).unwrap()).unwrap() }) .collect(); for &ptr in &pointers { assert_eq!( - (ptr.as_ptr() as usize) % align, + (ptr.as_non_null_ptr().as_ptr() as usize) % align, 0, "Got a pointer less aligned than requested" ) @@ -33,7 +33,10 @@ fn check_overalign_requests(mut allocator: T) { // Clean up for &ptr in &pointers { - allocator.dealloc(ptr, Layout::from_size_align(size, align).unwrap()) + allocator.deallocate( + ptr.as_non_null_ptr(), + Layout::from_size_align(size, align).unwrap(), + ) } } } diff --git a/crux-mir/lib/alloc/tests/lib.rs b/crux-mir/lib/alloc/tests/lib.rs index ea75f8903..2a93a242d 100644 --- a/crux-mir/lib/alloc/tests/lib.rs +++ b/crux-mir/lib/alloc/tests/lib.rs @@ -1,26 +1,64 @@ #![feature(allocator_api)] -#![feature(box_syntax)] +#![feature(alloc_layout_extra)] +#![feature(assert_matches)] +#![feature(btree_drain_filter)] +#![feature(cow_is_borrowed)] +#![feature(const_box)] +#![feature(const_convert)] +#![feature(const_cow_is_borrowed)] +#![feature(const_heap)] +#![feature(const_mut_refs)] +#![feature(const_nonnull_slice_from_raw_parts)] +#![feature(const_ptr_write)] +#![feature(const_try)] +#![feature(core_intrinsics)] #![feature(drain_filter)] #![feature(exact_size_is_empty)] -#![feature(map_first_last)] +#![feature(linked_list_cursors)] +#![feature(map_try_insert)] #![feature(new_uninit)] #![feature(pattern)] #![feature(trusted_len)] -#![feature(try_reserve)] +#![feature(try_reserve_kind)] #![feature(unboxed_closures)] #![feature(associated_type_bounds)] #![feature(binary_heap_into_iter_sorted)] #![feature(binary_heap_drain_sorted)] -#![feature(vec_remove_item)] -#![feature(split_inclusive)] +#![feature(slice_ptr_get)] +#![feature(binary_heap_retain)] +#![feature(binary_heap_as_slice)] +#![feature(inplace_iteration)] +#![feature(iter_advance_by)] +#![feature(iter_next_chunk)] +#![feature(round_char_boundary)] +#![feature(slice_group_by)] +#![feature(slice_partition_dedup)] +#![feature(string_remove_matches)] +#![feature(const_btree_len)] +#![feature(const_default_impls)] +#![feature(const_trait_impl)] +#![feature(const_str_from_utf8)] +#![feature(nonnull_slice_from_raw_parts)] +#![feature(panic_update_hook)] +#![feature(pointer_is_aligned)] +#![feature(slice_flatten)] +#![feature(thin_box)] +#![feature(strict_provenance)] +#![feature(once_cell)] +#![feature(drain_keep_rest)] +#![deny(fuzzy_provenance_casts)] +#![deny(unsafe_op_in_unsafe_fn)] use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; mod arc; -mod binary_heap; +mod autotraits; +mod borrow; mod boxed; -mod btree; +mod btree_set_hash; +mod c_str; +mod const_fns; mod cow_str; mod fmt; mod heap; @@ -29,6 +67,7 @@ mod rc; mod slice; mod str; mod string; +mod thin_box; mod vec; mod vec_deque; diff --git a/crux-mir/lib/alloc/tests/linked_list.rs b/crux-mir/lib/alloc/tests/linked_list.rs index afcb9e03f..65b09cb00 100644 --- a/crux-mir/lib/alloc/tests/linked_list.rs +++ b/crux-mir/lib/alloc/tests/linked_list.rs @@ -1,241 +1,4 @@ use std::collections::LinkedList; -use std::panic::{catch_unwind, AssertUnwindSafe}; - -#[test] -fn test_basic() { - let mut m = LinkedList::>::new(); - assert_eq!(m.pop_front(), None); - assert_eq!(m.pop_back(), None); - assert_eq!(m.pop_front(), None); - m.push_front(box 1); - assert_eq!(m.pop_front(), Some(box 1)); - m.push_back(box 2); - m.push_back(box 3); - assert_eq!(m.len(), 2); - assert_eq!(m.pop_front(), Some(box 2)); - assert_eq!(m.pop_front(), Some(box 3)); - assert_eq!(m.len(), 0); - assert_eq!(m.pop_front(), None); - m.push_back(box 1); - m.push_back(box 3); - m.push_back(box 5); - m.push_back(box 7); - assert_eq!(m.pop_front(), Some(box 1)); - - let mut n = LinkedList::new(); - n.push_front(2); - n.push_front(3); - { - assert_eq!(n.front().unwrap(), &3); - let x = n.front_mut().unwrap(); - assert_eq!(*x, 3); - *x = 0; - } - { - assert_eq!(n.back().unwrap(), &2); - let y = n.back_mut().unwrap(); - assert_eq!(*y, 2); - *y = 1; - } - assert_eq!(n.pop_front(), Some(0)); - assert_eq!(n.pop_front(), Some(1)); -} - -fn generate_test() -> LinkedList { - list_from(&[0, 1, 2, 3, 4, 5, 6]) -} - -fn list_from(v: &[T]) -> LinkedList { - v.iter().cloned().collect() -} - -#[test] -fn test_split_off() { - // singleton - { - let mut m = LinkedList::new(); - m.push_back(1); - - let p = m.split_off(0); - assert_eq!(m.len(), 0); - assert_eq!(p.len(), 1); - assert_eq!(p.back(), Some(&1)); - assert_eq!(p.front(), Some(&1)); - } - - // not singleton, forwards - { - let u = vec![1, 2, 3, 4, 5]; - let mut m = list_from(&u); - let mut n = m.split_off(2); - assert_eq!(m.len(), 2); - assert_eq!(n.len(), 3); - for elt in 1..3 { - assert_eq!(m.pop_front(), Some(elt)); - } - for elt in 3..6 { - assert_eq!(n.pop_front(), Some(elt)); - } - } - // not singleton, backwards - { - let u = vec![1, 2, 3, 4, 5]; - let mut m = list_from(&u); - let mut n = m.split_off(4); - assert_eq!(m.len(), 4); - assert_eq!(n.len(), 1); - for elt in 1..5 { - assert_eq!(m.pop_front(), Some(elt)); - } - for elt in 5..6 { - assert_eq!(n.pop_front(), Some(elt)); - } - } - - // no-op on the last index - { - let mut m = LinkedList::new(); - m.push_back(1); - - let p = m.split_off(1); - assert_eq!(m.len(), 1); - assert_eq!(p.len(), 0); - assert_eq!(m.back(), Some(&1)); - assert_eq!(m.front(), Some(&1)); - } -} - -#[test] -fn test_iterator() { - let m = generate_test(); - for (i, elt) in m.iter().enumerate() { - assert_eq!(i as i32, *elt); - } - let mut n = LinkedList::new(); - assert_eq!(n.iter().next(), None); - n.push_front(4); - let mut it = n.iter(); - assert_eq!(it.size_hint(), (1, Some(1))); - assert_eq!(it.next().unwrap(), &4); - assert_eq!(it.size_hint(), (0, Some(0))); - assert_eq!(it.next(), None); -} - -#[test] -fn test_iterator_clone() { - let mut n = LinkedList::new(); - n.push_back(2); - n.push_back(3); - n.push_back(4); - let mut it = n.iter(); - it.next(); - let mut jt = it.clone(); - assert_eq!(it.next(), jt.next()); - assert_eq!(it.next_back(), jt.next_back()); - assert_eq!(it.next(), jt.next()); -} - -#[test] -fn test_iterator_double_end() { - let mut n = LinkedList::new(); - assert_eq!(n.iter().next(), None); - n.push_front(4); - n.push_front(5); - n.push_front(6); - let mut it = n.iter(); - assert_eq!(it.size_hint(), (3, Some(3))); - assert_eq!(it.next().unwrap(), &6); - assert_eq!(it.size_hint(), (2, Some(2))); - assert_eq!(it.next_back().unwrap(), &4); - assert_eq!(it.size_hint(), (1, Some(1))); - assert_eq!(it.next_back().unwrap(), &5); - assert_eq!(it.next_back(), None); - assert_eq!(it.next(), None); -} - -#[test] -fn test_rev_iter() { - let m = generate_test(); - for (i, elt) in m.iter().rev().enumerate() { - assert_eq!((6 - i) as i32, *elt); - } - let mut n = LinkedList::new(); - assert_eq!(n.iter().rev().next(), None); - n.push_front(4); - let mut it = n.iter().rev(); - assert_eq!(it.size_hint(), (1, Some(1))); - assert_eq!(it.next().unwrap(), &4); - assert_eq!(it.size_hint(), (0, Some(0))); - assert_eq!(it.next(), None); -} - -#[test] -fn test_mut_iter() { - let mut m = generate_test(); - let mut len = m.len(); - for (i, elt) in m.iter_mut().enumerate() { - assert_eq!(i as i32, *elt); - len -= 1; - } - assert_eq!(len, 0); - let mut n = LinkedList::new(); - assert!(n.iter_mut().next().is_none()); - n.push_front(4); - n.push_back(5); - let mut it = n.iter_mut(); - assert_eq!(it.size_hint(), (2, Some(2))); - assert!(it.next().is_some()); - assert!(it.next().is_some()); - assert_eq!(it.size_hint(), (0, Some(0))); - assert!(it.next().is_none()); -} - -#[test] -fn test_iterator_mut_double_end() { - let mut n = LinkedList::new(); - assert!(n.iter_mut().next_back().is_none()); - n.push_front(4); - n.push_front(5); - n.push_front(6); - let mut it = n.iter_mut(); - assert_eq!(it.size_hint(), (3, Some(3))); - assert_eq!(*it.next().unwrap(), 6); - assert_eq!(it.size_hint(), (2, Some(2))); - assert_eq!(*it.next_back().unwrap(), 4); - assert_eq!(it.size_hint(), (1, Some(1))); - assert_eq!(*it.next_back().unwrap(), 5); - assert!(it.next_back().is_none()); - assert!(it.next().is_none()); -} - -#[test] -fn test_mut_rev_iter() { - let mut m = generate_test(); - for (i, elt) in m.iter_mut().rev().enumerate() { - assert_eq!((6 - i) as i32, *elt); - } - let mut n = LinkedList::new(); - assert!(n.iter_mut().rev().next().is_none()); - n.push_front(4); - let mut it = n.iter_mut().rev(); - assert!(it.next().is_some()); - assert!(it.next().is_none()); -} - -#[test] -fn test_eq() { - let mut n = list_from(&[]); - let mut m = list_from(&[]); - assert!(n == m); - n.push_front(1); - assert!(n != m); - m.push_back(1); - assert!(n == m); - - let n = list_from(&[2, 3, 4]); - let m = list_from(&[1, 2, 3]); - assert!(n != m); -} #[test] fn test_hash() { @@ -256,450 +19,3 @@ fn test_hash() { assert!(hash(&x) == hash(&y)); } - -#[test] -fn test_ord() { - let n = list_from(&[]); - let m = list_from(&[1, 2, 3]); - assert!(n < m); - assert!(m > n); - assert!(n <= n); - assert!(n >= n); -} - -#[test] -fn test_ord_nan() { - let nan = 0.0f64 / 0.0; - let n = list_from(&[nan]); - let m = list_from(&[nan]); - assert!(!(n < m)); - assert!(!(n > m)); - assert!(!(n <= m)); - assert!(!(n >= m)); - - let n = list_from(&[nan]); - let one = list_from(&[1.0f64]); - assert!(!(n < one)); - assert!(!(n > one)); - assert!(!(n <= one)); - assert!(!(n >= one)); - - let u = list_from(&[1.0f64, 2.0, nan]); - let v = list_from(&[1.0f64, 2.0, 3.0]); - assert!(!(u < v)); - assert!(!(u > v)); - assert!(!(u <= v)); - assert!(!(u >= v)); - - let s = list_from(&[1.0f64, 2.0, 4.0, 2.0]); - let t = list_from(&[1.0f64, 2.0, 3.0, 2.0]); - assert!(!(s < t)); - assert!(s > one); - assert!(!(s <= one)); - assert!(s >= one); -} - -#[test] -fn test_show() { - let list: LinkedList<_> = (0..10).collect(); - assert_eq!(format!("{:?}", list), "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]"); - - let list: LinkedList<_> = vec!["just", "one", "test", "more"].iter().cloned().collect(); - assert_eq!(format!("{:?}", list), "[\"just\", \"one\", \"test\", \"more\"]"); -} - -#[test] -fn test_extend_ref() { - let mut a = LinkedList::new(); - a.push_back(1); - - a.extend(&[2, 3, 4]); - - assert_eq!(a.len(), 4); - assert_eq!(a, list_from(&[1, 2, 3, 4])); - - let mut b = LinkedList::new(); - b.push_back(5); - b.push_back(6); - a.extend(&b); - - assert_eq!(a.len(), 6); - assert_eq!(a, list_from(&[1, 2, 3, 4, 5, 6])); -} - -#[test] -fn test_extend() { - let mut a = LinkedList::new(); - a.push_back(1); - a.extend(vec![2, 3, 4]); // uses iterator - - assert_eq!(a.len(), 4); - assert!(a.iter().eq(&[1, 2, 3, 4])); - - let b: LinkedList<_> = vec![5, 6, 7].into_iter().collect(); - a.extend(b); // specializes to `append` - - assert_eq!(a.len(), 7); - assert!(a.iter().eq(&[1, 2, 3, 4, 5, 6, 7])); -} - -#[test] -fn test_contains() { - let mut l = LinkedList::new(); - l.extend(&[2, 3, 4]); - - assert!(l.contains(&3)); - assert!(!l.contains(&1)); - - l.clear(); - - assert!(!l.contains(&3)); -} - -#[test] -fn drain_filter_empty() { - let mut list: LinkedList = LinkedList::new(); - - { - let mut iter = list.drain_filter(|_| true); - assert_eq!(iter.size_hint(), (0, Some(0))); - assert_eq!(iter.next(), None); - assert_eq!(iter.size_hint(), (0, Some(0))); - assert_eq!(iter.next(), None); - assert_eq!(iter.size_hint(), (0, Some(0))); - } - - assert_eq!(list.len(), 0); - assert_eq!(list.into_iter().collect::>(), vec![]); -} - -#[test] -fn drain_filter_zst() { - let mut list: LinkedList<_> = vec![(), (), (), (), ()].into_iter().collect(); - let initial_len = list.len(); - let mut count = 0; - - { - let mut iter = list.drain_filter(|_| true); - assert_eq!(iter.size_hint(), (0, Some(initial_len))); - while let Some(_) = iter.next() { - count += 1; - assert_eq!(iter.size_hint(), (0, Some(initial_len - count))); - } - assert_eq!(iter.size_hint(), (0, Some(0))); - assert_eq!(iter.next(), None); - assert_eq!(iter.size_hint(), (0, Some(0))); - } - - assert_eq!(count, initial_len); - assert_eq!(list.len(), 0); - assert_eq!(list.into_iter().collect::>(), vec![]); -} - -#[test] -fn drain_filter_false() { - let mut list: LinkedList<_> = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10].into_iter().collect(); - - let initial_len = list.len(); - let mut count = 0; - - { - let mut iter = list.drain_filter(|_| false); - assert_eq!(iter.size_hint(), (0, Some(initial_len))); - for _ in iter.by_ref() { - count += 1; - } - assert_eq!(iter.size_hint(), (0, Some(0))); - assert_eq!(iter.next(), None); - assert_eq!(iter.size_hint(), (0, Some(0))); - } - - assert_eq!(count, 0); - assert_eq!(list.len(), initial_len); - assert_eq!(list.into_iter().collect::>(), vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); -} - -#[test] -fn drain_filter_true() { - let mut list: LinkedList<_> = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10].into_iter().collect(); - - let initial_len = list.len(); - let mut count = 0; - - { - let mut iter = list.drain_filter(|_| true); - assert_eq!(iter.size_hint(), (0, Some(initial_len))); - while let Some(_) = iter.next() { - count += 1; - assert_eq!(iter.size_hint(), (0, Some(initial_len - count))); - } - assert_eq!(iter.size_hint(), (0, Some(0))); - assert_eq!(iter.next(), None); - assert_eq!(iter.size_hint(), (0, Some(0))); - } - - assert_eq!(count, initial_len); - assert_eq!(list.len(), 0); - assert_eq!(list.into_iter().collect::>(), vec![]); -} - -#[test] -fn drain_filter_complex() { - { - // [+xxx++++++xxxxx++++x+x++] - let mut list = vec![ - 1, 2, 4, 6, 7, 9, 11, 13, 15, 17, 18, 20, 22, 24, 26, 27, 29, 31, 33, 34, 35, 36, 37, - 39, - ] - .into_iter() - .collect::>(); - - let removed = list.drain_filter(|x| *x % 2 == 0).collect::>(); - assert_eq!(removed.len(), 10); - assert_eq!(removed, vec![2, 4, 6, 18, 20, 22, 24, 26, 34, 36]); - - assert_eq!(list.len(), 14); - assert_eq!( - list.into_iter().collect::>(), - vec![1, 7, 9, 11, 13, 15, 17, 27, 29, 31, 33, 35, 37, 39] - ); - } - - { - // [xxx++++++xxxxx++++x+x++] - let mut list = vec![ - 2, 4, 6, 7, 9, 11, 13, 15, 17, 18, 20, 22, 24, 26, 27, 29, 31, 33, 34, 35, 36, 37, 39, - ] - .into_iter() - .collect::>(); - - let removed = list.drain_filter(|x| *x % 2 == 0).collect::>(); - assert_eq!(removed.len(), 10); - assert_eq!(removed, vec![2, 4, 6, 18, 20, 22, 24, 26, 34, 36]); - - assert_eq!(list.len(), 13); - assert_eq!( - list.into_iter().collect::>(), - vec![7, 9, 11, 13, 15, 17, 27, 29, 31, 33, 35, 37, 39] - ); - } - - { - // [xxx++++++xxxxx++++x+x] - let mut list = - vec![2, 4, 6, 7, 9, 11, 13, 15, 17, 18, 20, 22, 24, 26, 27, 29, 31, 33, 34, 35, 36] - .into_iter() - .collect::>(); - - let removed = list.drain_filter(|x| *x % 2 == 0).collect::>(); - assert_eq!(removed.len(), 10); - assert_eq!(removed, vec![2, 4, 6, 18, 20, 22, 24, 26, 34, 36]); - - assert_eq!(list.len(), 11); - assert_eq!( - list.into_iter().collect::>(), - vec![7, 9, 11, 13, 15, 17, 27, 29, 31, 33, 35] - ); - } - - { - // [xxxxxxxxxx+++++++++++] - let mut list = vec![2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19] - .into_iter() - .collect::>(); - - let removed = list.drain_filter(|x| *x % 2 == 0).collect::>(); - assert_eq!(removed.len(), 10); - assert_eq!(removed, vec![2, 4, 6, 8, 10, 12, 14, 16, 18, 20]); - - assert_eq!(list.len(), 10); - assert_eq!(list.into_iter().collect::>(), vec![1, 3, 5, 7, 9, 11, 13, 15, 17, 19]); - } - - { - // [+++++++++++xxxxxxxxxx] - let mut list = vec![1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20] - .into_iter() - .collect::>(); - - let removed = list.drain_filter(|x| *x % 2 == 0).collect::>(); - assert_eq!(removed.len(), 10); - assert_eq!(removed, vec![2, 4, 6, 8, 10, 12, 14, 16, 18, 20]); - - assert_eq!(list.len(), 10); - assert_eq!(list.into_iter().collect::>(), vec![1, 3, 5, 7, 9, 11, 13, 15, 17, 19]); - } -} - -#[test] -fn drain_filter_drop_panic_leak() { - static mut DROPS: i32 = 0; - - struct D(bool); - - impl Drop for D { - fn drop(&mut self) { - unsafe { - DROPS += 1; - } - - if self.0 { - panic!("panic in `drop`"); - } - } - } - - let mut q = LinkedList::new(); - q.push_back(D(false)); - q.push_back(D(false)); - q.push_back(D(false)); - q.push_back(D(false)); - q.push_back(D(false)); - q.push_front(D(false)); - q.push_front(D(true)); - q.push_front(D(false)); - - catch_unwind(AssertUnwindSafe(|| drop(q.drain_filter(|_| true)))).ok(); - - assert_eq!(unsafe { DROPS }, 8); - assert!(q.is_empty()); -} - -#[test] -fn drain_filter_pred_panic_leak() { - static mut DROPS: i32 = 0; - - #[derive(Debug)] - struct D(u32); - - impl Drop for D { - fn drop(&mut self) { - unsafe { - DROPS += 1; - } - } - } - - let mut q = LinkedList::new(); - q.push_back(D(3)); - q.push_back(D(4)); - q.push_back(D(5)); - q.push_back(D(6)); - q.push_back(D(7)); - q.push_front(D(2)); - q.push_front(D(1)); - q.push_front(D(0)); - - catch_unwind(AssertUnwindSafe(|| { - drop(q.drain_filter(|item| if item.0 >= 2 { panic!() } else { true })) - })) - .ok(); - - assert_eq!(unsafe { DROPS }, 2); // 0 and 1 - assert_eq!(q.len(), 6); -} - -#[test] -fn test_drop() { - static mut DROPS: i32 = 0; - struct Elem; - impl Drop for Elem { - fn drop(&mut self) { - unsafe { - DROPS += 1; - } - } - } - - let mut ring = LinkedList::new(); - ring.push_back(Elem); - ring.push_front(Elem); - ring.push_back(Elem); - ring.push_front(Elem); - drop(ring); - - assert_eq!(unsafe { DROPS }, 4); -} - -#[test] -fn test_drop_with_pop() { - static mut DROPS: i32 = 0; - struct Elem; - impl Drop for Elem { - fn drop(&mut self) { - unsafe { - DROPS += 1; - } - } - } - - let mut ring = LinkedList::new(); - ring.push_back(Elem); - ring.push_front(Elem); - ring.push_back(Elem); - ring.push_front(Elem); - - drop(ring.pop_back()); - drop(ring.pop_front()); - assert_eq!(unsafe { DROPS }, 2); - - drop(ring); - assert_eq!(unsafe { DROPS }, 4); -} - -#[test] -fn test_drop_clear() { - static mut DROPS: i32 = 0; - struct Elem; - impl Drop for Elem { - fn drop(&mut self) { - unsafe { - DROPS += 1; - } - } - } - - let mut ring = LinkedList::new(); - ring.push_back(Elem); - ring.push_front(Elem); - ring.push_back(Elem); - ring.push_front(Elem); - ring.clear(); - assert_eq!(unsafe { DROPS }, 4); - - drop(ring); - assert_eq!(unsafe { DROPS }, 4); -} - -#[test] -fn test_drop_panic() { - static mut DROPS: i32 = 0; - - struct D(bool); - - impl Drop for D { - fn drop(&mut self) { - unsafe { - DROPS += 1; - } - - if self.0 { - panic!("panic in `drop`"); - } - } - } - - let mut q = LinkedList::new(); - q.push_back(D(false)); - q.push_back(D(false)); - q.push_back(D(false)); - q.push_back(D(false)); - q.push_back(D(false)); - q.push_front(D(false)); - q.push_front(D(false)); - q.push_front(D(true)); - - catch_unwind(move || drop(q)).ok(); - - assert_eq!(unsafe { DROPS }, 8); -} diff --git a/crux-mir/lib/alloc/tests/rc.rs b/crux-mir/lib/alloc/tests/rc.rs index 884856cd1..efb39a609 100644 --- a/crux-mir/lib/alloc/tests/rc.rs +++ b/crux-mir/lib/alloc/tests/rc.rs @@ -50,7 +50,7 @@ fn trait_object() { #[test] fn float_nan_ne() { - let x = Rc::new(std::f32::NAN); + let x = Rc::new(f32::NAN); assert!(x != x); assert!(!(x == x)); } @@ -191,3 +191,18 @@ fn shared_from_iter_trustedlen_no_fuse() { assert_trusted_len(&iter); assert_eq!(&[Box::new(42), Box::new(24)], &*iter.collect::>()); } + +#[test] +fn weak_may_dangle() { + fn hmm<'a>(val: &'a mut Weak<&'a str>) -> Weak<&'a str> { + val.clone() + } + + // Without #[may_dangle] we get: + let mut val = Weak::new(); + hmm(&mut val); + // ~~~~~~~~ borrowed value does not live long enough + // + // `val` dropped here while still borrowed + // borrow might be used here, when `val` is dropped and runs the `Drop` code for type `std::rc::Weak` +} diff --git a/crux-mir/lib/alloc/tests/slice.rs b/crux-mir/lib/alloc/tests/slice.rs index 8e49e6d8e..0693beb48 100644 --- a/crux-mir/lib/alloc/tests/slice.rs +++ b/crux-mir/lib/alloc/tests/slice.rs @@ -1,13 +1,9 @@ -use std::cell::Cell; -use std::cmp::Ordering::{self, Equal, Greater, Less}; +use std::cmp::Ordering::{Equal, Greater, Less}; +use std::convert::identity; +use std::fmt; use std::mem; use std::panic; use std::rc::Rc; -use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; - -use rand::distributions::Standard; -use rand::seq::SliceRandom; -use rand::{thread_rng, Rng, RngCore}; fn square(n: usize) -> usize { n * n @@ -266,9 +262,9 @@ fn test_swap_remove_fail() { fn test_swap_remove_noncopyable() { // Tests that we don't accidentally run destructors twice. let mut v: Vec> = Vec::new(); - v.push(box 0); - v.push(box 0); - v.push(box 0); + v.push(Box::new(0)); + v.push(Box::new(0)); + v.push(Box::new(0)); let mut _e = v.swap_remove(0); assert_eq!(v.len(), 2); _e = v.swap_remove(1); @@ -294,7 +290,7 @@ fn test_push() { #[test] fn test_truncate() { - let mut v: Vec> = vec![box 6, box 5, box 4]; + let mut v: Vec> = vec![Box::new(6), Box::new(5), Box::new(4)]; v.truncate(1); let v = v; assert_eq!(v.len(), 1); @@ -304,7 +300,7 @@ fn test_truncate() { #[test] fn test_clear() { - let mut v: Vec> = vec![box 6, box 5, box 4]; + let mut v: Vec> = vec![Box::new(6), Box::new(5), Box::new(4)]; v.clear(); assert_eq!(v.len(), 0); // If the unsafe block didn't drop things properly, we blow up here. @@ -386,129 +382,6 @@ fn test_reverse() { assert_eq!(v, (-50..51i16).rev().collect::>()); } -#[test] -#[cfg_attr(miri, ignore)] // Miri is too slow -fn test_sort() { - let mut rng = thread_rng(); - - for len in (2..25).chain(500..510) { - for &modulus in &[5, 10, 100, 1000] { - for _ in 0..10 { - let orig: Vec<_> = - rng.sample_iter::(&Standard).map(|x| x % modulus).take(len).collect(); - - // Sort in default order. - let mut v = orig.clone(); - v.sort(); - assert!(v.windows(2).all(|w| w[0] <= w[1])); - - // Sort in ascending order. - let mut v = orig.clone(); - v.sort_by(|a, b| a.cmp(b)); - assert!(v.windows(2).all(|w| w[0] <= w[1])); - - // Sort in descending order. - let mut v = orig.clone(); - v.sort_by(|a, b| b.cmp(a)); - assert!(v.windows(2).all(|w| w[0] >= w[1])); - - // Sort in lexicographic order. - let mut v1 = orig.clone(); - let mut v2 = orig.clone(); - v1.sort_by_key(|x| x.to_string()); - v2.sort_by_cached_key(|x| x.to_string()); - assert!(v1.windows(2).all(|w| w[0].to_string() <= w[1].to_string())); - assert!(v1 == v2); - - // Sort with many pre-sorted runs. - let mut v = orig.clone(); - v.sort(); - v.reverse(); - for _ in 0..5 { - let a = rng.gen::() % len; - let b = rng.gen::() % len; - if a < b { - v[a..b].reverse(); - } else { - v.swap(a, b); - } - } - v.sort(); - assert!(v.windows(2).all(|w| w[0] <= w[1])); - } - } - } - - // Sort using a completely random comparison function. - // This will reorder the elements *somehow*, but won't panic. - let mut v = [0; 500]; - for i in 0..v.len() { - v[i] = i as i32; - } - v.sort_by(|_, _| *[Less, Equal, Greater].choose(&mut rng).unwrap()); - v.sort(); - for i in 0..v.len() { - assert_eq!(v[i], i as i32); - } - - // Should not panic. - [0i32; 0].sort(); - [(); 10].sort(); - [(); 100].sort(); - - let mut v = [0xDEADBEEFu64]; - v.sort(); - assert!(v == [0xDEADBEEF]); -} - -#[test] -fn test_sort_stability() { - #[cfg(not(miri))] // Miri is too slow - let large_range = 500..510; - #[cfg(not(miri))] // Miri is too slow - let rounds = 10; - - #[cfg(miri)] - let large_range = 0..0; // empty range - #[cfg(miri)] - let rounds = 1; - - for len in (2..25).chain(large_range) { - for _ in 0..rounds { - let mut counts = [0; 10]; - - // create a vector like [(6, 1), (5, 1), (6, 2), ...], - // where the first item of each tuple is random, but - // the second item represents which occurrence of that - // number this element is, i.e., the second elements - // will occur in sorted order. - let orig: Vec<_> = (0..len) - .map(|_| { - let n = thread_rng().gen::() % 10; - counts[n] += 1; - (n, counts[n]) - }) - .collect(); - - let mut v = orig.clone(); - // Only sort on the first element, so an unstable sort - // may mix up the counts. - v.sort_by(|&(a, _), &(b, _)| a.cmp(&b)); - - // This comparison includes the count (the second item - // of the tuple), so elements with equal first items - // will need to be ordered with increasing - // counts... i.e., exactly asserting that this sort is - // stable. - assert!(v.windows(2).all(|w| w[0] <= w[1])); - - let mut v = orig.clone(); - v.sort_by_cached_key(|&(x, _)| x); - assert!(v.windows(2).all(|w| w[0] <= w[1])); - } - } -} - #[test] fn test_rotate_left() { let expected: Vec<_> = (0..13).collect(); @@ -867,7 +740,7 @@ fn test_splitator_inclusive() { assert_eq!(xs.split_inclusive(|_| true).collect::>(), splits); let xs: &[i32] = &[]; - let splits: &[&[i32]] = &[&[]]; + let splits: &[&[i32]] = &[]; assert_eq!(xs.split_inclusive(|x| *x == 5).collect::>(), splits); } @@ -887,7 +760,7 @@ fn test_splitator_inclusive_reverse() { assert_eq!(xs.split_inclusive(|_| true).rev().collect::>(), splits); let xs: &[i32] = &[]; - let splits: &[&[i32]] = &[&[]]; + let splits: &[&[i32]] = &[]; assert_eq!(xs.split_inclusive(|x| *x == 5).rev().collect::>(), splits); } @@ -907,7 +780,7 @@ fn test_splitator_mut_inclusive() { assert_eq!(xs.split_inclusive_mut(|_| true).collect::>(), splits); let xs: &mut [i32] = &mut []; - let splits: &[&[i32]] = &[&[]]; + let splits: &[&[i32]] = &[]; assert_eq!(xs.split_inclusive_mut(|x| *x == 5).collect::>(), splits); } @@ -927,7 +800,7 @@ fn test_splitator_mut_inclusive_reverse() { assert_eq!(xs.split_inclusive_mut(|_| true).rev().collect::>(), splits); let xs: &mut [i32] = &mut []; - let splits: &[&[i32]] = &[&[]]; + let splits: &[&[i32]] = &[]; assert_eq!(xs.split_inclusive_mut(|x| *x == 5).rev().collect::>(), splits); } @@ -998,6 +871,66 @@ fn test_rsplitnator() { assert!(xs.rsplitn(0, |x| *x % 2 == 0).next().is_none()); } +#[test] +fn test_split_iterators_size_hint() { + #[derive(Copy, Clone)] + enum Bounds { + Lower, + Upper, + } + fn assert_tight_size_hints(mut it: impl Iterator, which: Bounds, ctx: impl fmt::Display) { + match which { + Bounds::Lower => { + let mut lower_bounds = vec![it.size_hint().0]; + while let Some(_) = it.next() { + lower_bounds.push(it.size_hint().0); + } + let target: Vec<_> = (0..lower_bounds.len()).rev().collect(); + assert_eq!(lower_bounds, target, "lower bounds incorrect or not tight: {}", ctx); + } + Bounds::Upper => { + let mut upper_bounds = vec![it.size_hint().1]; + while let Some(_) = it.next() { + upper_bounds.push(it.size_hint().1); + } + let target: Vec<_> = (0..upper_bounds.len()).map(Some).rev().collect(); + assert_eq!(upper_bounds, target, "upper bounds incorrect or not tight: {}", ctx); + } + } + } + + for len in 0..=2 { + let mut v: Vec = (0..len).collect(); + + // p: predicate, b: bound selection + for (p, b) in [ + // with a predicate always returning false, the split*-iterators + // become maximally short, so the size_hint lower bounds are tight + ((|_| false) as fn(&_) -> _, Bounds::Lower), + // with a predicate always returning true, the split*-iterators + // become maximally long, so the size_hint upper bounds are tight + ((|_| true) as fn(&_) -> _, Bounds::Upper), + ] { + use assert_tight_size_hints as a; + use format_args as f; + + a(v.split(p), b, "split"); + a(v.split_mut(p), b, "split_mut"); + a(v.split_inclusive(p), b, "split_inclusive"); + a(v.split_inclusive_mut(p), b, "split_inclusive_mut"); + a(v.rsplit(p), b, "rsplit"); + a(v.rsplit_mut(p), b, "rsplit_mut"); + + for n in 0..=3 { + a(v.splitn(n, p), b, f!("splitn, n = {n}")); + a(v.splitn_mut(n, p), b, f!("splitn_mut, n = {n}")); + a(v.rsplitn(n, p), b, f!("rsplitn, n = {n}")); + a(v.rsplitn_mut(n, p), b, f!("rsplitn_mut, n = {n}")); + } + } + } +} + #[test] fn test_windowsator() { let v = &[1, 2, 3, 4]; @@ -1128,8 +1061,8 @@ fn test_show() { macro_rules! test_show_vec { ($x:expr, $x_str:expr) => {{ let (x, x_str) = ($x, $x_str); - assert_eq!(format!("{:?}", x), x_str); - assert_eq!(format!("{:?}", x), x_str); + assert_eq!(format!("{x:?}"), x_str); + assert_eq!(format!("{x:?}"), x_str); }}; } let empty = Vec::::new(); @@ -1460,11 +1393,20 @@ fn test_mut_last() { #[test] fn test_to_vec() { - let xs: Box<_> = box [1, 2, 3]; + let xs: Box<_> = Box::new([1, 2, 3]); let ys = xs.to_vec(); assert_eq!(ys, [1, 2, 3]); } +#[test] +fn test_in_place_iterator_specialization() { + let src: Box<[usize]> = Box::new([1, 2, 3]); + let src_ptr = src.as_ptr(); + let sink: Box<_> = src.into_vec().into_iter().map(std::convert::identity).collect(); + let sink_ptr = sink.as_ptr(); + assert_eq!(src_ptr, sink_ptr); +} + #[test] fn test_box_slice_clone() { let data = vec![vec![0, 1], vec![0], vec![1]]; @@ -1528,7 +1470,7 @@ fn test_copy_from_slice() { } #[test] -#[should_panic(expected = "destination and source slices have different lengths")] +#[should_panic(expected = "source slice length (4) does not match destination slice length (5)")] fn test_copy_from_slice_dst_longer() { let src = [0, 1, 2, 3]; let mut dst = [0; 5]; @@ -1536,248 +1478,194 @@ fn test_copy_from_slice_dst_longer() { } #[test] -#[should_panic(expected = "destination and source slices have different lengths")] +#[should_panic(expected = "source slice length (4) does not match destination slice length (3)")] fn test_copy_from_slice_dst_shorter() { let src = [0, 1, 2, 3]; let mut dst = [0; 3]; dst.copy_from_slice(&src); } -const MAX_LEN: usize = 80; - -static DROP_COUNTS: [AtomicUsize; MAX_LEN] = [ - // FIXME(RFC 1109): AtomicUsize is not Copy. - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), -]; - -static VERSIONS: AtomicUsize = AtomicUsize::new(0); - -#[derive(Clone, Eq)] -struct DropCounter { - x: u32, - id: usize, - version: Cell, -} - -impl PartialEq for DropCounter { - fn eq(&self, other: &Self) -> bool { - self.partial_cmp(other) == Some(Ordering::Equal) - } -} - -impl PartialOrd for DropCounter { - fn partial_cmp(&self, other: &Self) -> Option { - self.version.set(self.version.get() + 1); - other.version.set(other.version.get() + 1); - VERSIONS.fetch_add(2, Relaxed); - self.x.partial_cmp(&other.x) - } -} - -impl Ord for DropCounter { - fn cmp(&self, other: &Self) -> Ordering { - self.partial_cmp(other).unwrap() - } -} - -impl Drop for DropCounter { - fn drop(&mut self) { - DROP_COUNTS[self.id].fetch_add(1, Relaxed); - VERSIONS.fetch_sub(self.version.get(), Relaxed); - } +#[test] +fn repeat_generic_slice() { + assert_eq!([1, 2].repeat(2), vec![1, 2, 1, 2]); + assert_eq!([1, 2, 3, 4].repeat(0), vec![]); + assert_eq!([1, 2, 3, 4].repeat(1), vec![1, 2, 3, 4]); + assert_eq!([1, 2, 3, 4].repeat(3), vec![1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4]); } -macro_rules! test { - ($input:ident, $func:ident) => { - let len = $input.len(); - - // Work out the total number of comparisons required to sort - // this array... - let mut count = 0usize; - $input.to_owned().$func(|a, b| { - count += 1; - a.cmp(b) - }); - - // ... and then panic on each and every single one. - for panic_countdown in 0..count { - // Refresh the counters. - VERSIONS.store(0, Relaxed); - for i in 0..len { - DROP_COUNTS[i].store(0, Relaxed); - } +#[test] +#[allow(unreachable_patterns)] +fn subslice_patterns() { + // This test comprehensively checks the passing static and dynamic semantics + // of subslice patterns `..`, `x @ ..`, `ref x @ ..`, and `ref mut @ ..` + // in slice patterns `[$($pat), $(,)?]` . - let v = $input.to_owned(); - let _ = std::panic::catch_unwind(move || { - let mut v = v; - let mut panic_countdown = panic_countdown; - v.$func(|a, b| { - if panic_countdown == 0 { - SILENCE_PANIC.with(|s| s.set(true)); - panic!(); - } - panic_countdown -= 1; - a.cmp(b) - }) - }); - - // Check that the number of things dropped is exactly - // what we expect (i.e., the contents of `v`). - for (i, c) in DROP_COUNTS.iter().enumerate().take(len) { - let count = c.load(Relaxed); - assert!(count == 1, "found drop count == {} for i == {}, len == {}", count, i, len); - } + #[derive(PartialEq, Debug, Clone)] + struct N(u8); - // Check that the most recent versions of values were dropped. - assert_eq!(VERSIONS.load(Relaxed), 0); + macro_rules! n { + ($($e:expr),* $(,)?) => { + [$(N($e)),*] } - }; -} - -thread_local!(static SILENCE_PANIC: Cell = Cell::new(false)); + } -#[test] -#[cfg_attr(target_os = "emscripten", ignore)] // no threads -fn panic_safe() { - let prev = panic::take_hook(); - panic::set_hook(Box::new(move |info| { - if !SILENCE_PANIC.with(|s| s.get()) { - prev(info); - } - })); - - let mut rng = thread_rng(); - - #[cfg(not(miri))] // Miri is too slow - let lens = (1..20).chain(70..MAX_LEN); - #[cfg(not(miri))] // Miri is too slow - let moduli = &[5, 20, 50]; - - #[cfg(miri)] - let lens = 1..10; - #[cfg(miri)] - let moduli = &[5]; - - for len in lens { - for &modulus in moduli { - for &has_runs in &[false, true] { - let mut input = (0..len) - .map(|id| DropCounter { - x: rng.next_u32() % modulus, - id: id, - version: Cell::new(0), - }) - .collect::>(); - - if has_runs { - for c in &mut input { - c.x = c.id as u32; - } - - for _ in 0..5 { - let a = rng.gen::() % len; - let b = rng.gen::() % len; - if a < b { - input[a..b].reverse(); - } else { - input.swap(a, b); - } - } - } + macro_rules! c { + ($inp:expr, $typ:ty, $out:expr $(,)?) => { + assert_eq!($out, identity::<$typ>($inp)) + }; + } - test!(input, sort_by); - test!(input, sort_unstable_by); + macro_rules! m { + ($e:expr, $p:pat => $b:expr) => { + match $e { + $p => $b, + _ => panic!(), } - } + }; } - // Set default panic hook again. - drop(panic::take_hook()); -} - -#[test] -fn repeat_generic_slice() { - assert_eq!([1, 2].repeat(2), vec![1, 2, 1, 2]); - assert_eq!([1, 2, 3, 4].repeat(0), vec![]); - assert_eq!([1, 2, 3, 4].repeat(1), vec![1, 2, 3, 4]); - assert_eq!([1, 2, 3, 4].repeat(3), vec![1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4]); + // == Slices == + + // Matching slices using `ref` patterns: + let mut v = vec![N(0), N(1), N(2), N(3), N(4)]; + let mut vc = (0..=4).collect::>(); + + let [..] = v[..]; // Always matches. + m!(v[..], [N(0), ref sub @ .., N(4)] => c!(sub, &[N], n![1, 2, 3])); + m!(v[..], [N(0), ref sub @ ..] => c!(sub, &[N], n![1, 2, 3, 4])); + m!(v[..], [ref sub @ .., N(4)] => c!(sub, &[N], n![0, 1, 2, 3])); + m!(v[..], [ref sub @ .., _, _, _, _, _] => c!(sub, &[N], &n![] as &[N])); + m!(v[..], [_, _, _, _, _, ref sub @ ..] => c!(sub, &[N], &n![] as &[N])); + m!(vc[..], [x, .., y] => c!((x, y), (u8, u8), (0, 4))); + + // Matching slices using `ref mut` patterns: + let [..] = v[..]; // Always matches. + m!(v[..], [N(0), ref mut sub @ .., N(4)] => c!(sub, &mut [N], n![1, 2, 3])); + m!(v[..], [N(0), ref mut sub @ ..] => c!(sub, &mut [N], n![1, 2, 3, 4])); + m!(v[..], [ref mut sub @ .., N(4)] => c!(sub, &mut [N], n![0, 1, 2, 3])); + m!(v[..], [ref mut sub @ .., _, _, _, _, _] => c!(sub, &mut [N], &mut n![] as &mut [N])); + m!(v[..], [_, _, _, _, _, ref mut sub @ ..] => c!(sub, &mut [N], &mut n![] as &mut [N])); + m!(vc[..], [x, .., y] => c!((x, y), (u8, u8), (0, 4))); + + // Matching slices using default binding modes (&): + let [..] = &v[..]; // Always matches. + m!(&v[..], [N(0), sub @ .., N(4)] => c!(sub, &[N], n![1, 2, 3])); + m!(&v[..], [N(0), sub @ ..] => c!(sub, &[N], n![1, 2, 3, 4])); + m!(&v[..], [sub @ .., N(4)] => c!(sub, &[N], n![0, 1, 2, 3])); + m!(&v[..], [sub @ .., _, _, _, _, _] => c!(sub, &[N], &n![] as &[N])); + m!(&v[..], [_, _, _, _, _, sub @ ..] => c!(sub, &[N], &n![] as &[N])); + m!(&vc[..], [x, .., y] => c!((x, y), (&u8, &u8), (&0, &4))); + + // Matching slices using default binding modes (&mut): + let [..] = &mut v[..]; // Always matches. + m!(&mut v[..], [N(0), sub @ .., N(4)] => c!(sub, &mut [N], n![1, 2, 3])); + m!(&mut v[..], [N(0), sub @ ..] => c!(sub, &mut [N], n![1, 2, 3, 4])); + m!(&mut v[..], [sub @ .., N(4)] => c!(sub, &mut [N], n![0, 1, 2, 3])); + m!(&mut v[..], [sub @ .., _, _, _, _, _] => c!(sub, &mut [N], &mut n![] as &mut [N])); + m!(&mut v[..], [_, _, _, _, _, sub @ ..] => c!(sub, &mut [N], &mut n![] as &mut [N])); + m!(&mut vc[..], [x, .., y] => c!((x, y), (&mut u8, &mut u8), (&mut 0, &mut 4))); + + // == Arrays == + let mut v = n![0, 1, 2, 3, 4]; + let vc = [0, 1, 2, 3, 4]; + + // Matching arrays by value: + m!(v.clone(), [N(0), sub @ .., N(4)] => c!(sub, [N; 3], n![1, 2, 3])); + m!(v.clone(), [N(0), sub @ ..] => c!(sub, [N; 4], n![1, 2, 3, 4])); + m!(v.clone(), [sub @ .., N(4)] => c!(sub, [N; 4], n![0, 1, 2, 3])); + m!(v.clone(), [sub @ .., _, _, _, _, _] => c!(sub, [N; 0], n![] as [N; 0])); + m!(v.clone(), [_, _, _, _, _, sub @ ..] => c!(sub, [N; 0], n![] as [N; 0])); + m!(v.clone(), [x, .., y] => c!((x, y), (N, N), (N(0), N(4)))); + m!(v.clone(), [..] => ()); + + // Matching arrays by ref patterns: + m!(v, [N(0), ref sub @ .., N(4)] => c!(sub, &[N; 3], &n![1, 2, 3])); + m!(v, [N(0), ref sub @ ..] => c!(sub, &[N; 4], &n![1, 2, 3, 4])); + m!(v, [ref sub @ .., N(4)] => c!(sub, &[N; 4], &n![0, 1, 2, 3])); + m!(v, [ref sub @ .., _, _, _, _, _] => c!(sub, &[N; 0], &n![] as &[N; 0])); + m!(v, [_, _, _, _, _, ref sub @ ..] => c!(sub, &[N; 0], &n![] as &[N; 0])); + m!(vc, [x, .., y] => c!((x, y), (u8, u8), (0, 4))); + + // Matching arrays by ref mut patterns: + m!(v, [N(0), ref mut sub @ .., N(4)] => c!(sub, &mut [N; 3], &mut n![1, 2, 3])); + m!(v, [N(0), ref mut sub @ ..] => c!(sub, &mut [N; 4], &mut n![1, 2, 3, 4])); + m!(v, [ref mut sub @ .., N(4)] => c!(sub, &mut [N; 4], &mut n![0, 1, 2, 3])); + m!(v, [ref mut sub @ .., _, _, _, _, _] => c!(sub, &mut [N; 0], &mut n![] as &mut [N; 0])); + m!(v, [_, _, _, _, _, ref mut sub @ ..] => c!(sub, &mut [N; 0], &mut n![] as &mut [N; 0])); + + // Matching arrays by default binding modes (&): + m!(&v, [N(0), sub @ .., N(4)] => c!(sub, &[N; 3], &n![1, 2, 3])); + m!(&v, [N(0), sub @ ..] => c!(sub, &[N; 4], &n![1, 2, 3, 4])); + m!(&v, [sub @ .., N(4)] => c!(sub, &[N; 4], &n![0, 1, 2, 3])); + m!(&v, [sub @ .., _, _, _, _, _] => c!(sub, &[N; 0], &n![] as &[N; 0])); + m!(&v, [_, _, _, _, _, sub @ ..] => c!(sub, &[N; 0], &n![] as &[N; 0])); + m!(&v, [..] => ()); + m!(&v, [x, .., y] => c!((x, y), (&N, &N), (&N(0), &N(4)))); + + // Matching arrays by default binding modes (&mut): + m!(&mut v, [N(0), sub @ .., N(4)] => c!(sub, &mut [N; 3], &mut n![1, 2, 3])); + m!(&mut v, [N(0), sub @ ..] => c!(sub, &mut [N; 4], &mut n![1, 2, 3, 4])); + m!(&mut v, [sub @ .., N(4)] => c!(sub, &mut [N; 4], &mut n![0, 1, 2, 3])); + m!(&mut v, [sub @ .., _, _, _, _, _] => c!(sub, &mut [N; 0], &mut n![] as &[N; 0])); + m!(&mut v, [_, _, _, _, _, sub @ ..] => c!(sub, &mut [N; 0], &mut n![] as &[N; 0])); + m!(&mut v, [..] => ()); + m!(&mut v, [x, .., y] => c!((x, y), (&mut N, &mut N), (&mut N(0), &mut N(4)))); +} + +#[test] +fn test_group_by() { + let slice = &[1, 1, 1, 3, 3, 2, 2, 2, 1, 0]; + + let mut iter = slice.group_by(|a, b| a == b); + assert_eq!(iter.next(), Some(&[1, 1, 1][..])); + assert_eq!(iter.next(), Some(&[3, 3][..])); + assert_eq!(iter.next(), Some(&[2, 2, 2][..])); + assert_eq!(iter.next(), Some(&[1][..])); + assert_eq!(iter.next(), Some(&[0][..])); + assert_eq!(iter.next(), None); + + let mut iter = slice.group_by(|a, b| a == b); + assert_eq!(iter.next_back(), Some(&[0][..])); + assert_eq!(iter.next_back(), Some(&[1][..])); + assert_eq!(iter.next_back(), Some(&[2, 2, 2][..])); + assert_eq!(iter.next_back(), Some(&[3, 3][..])); + assert_eq!(iter.next_back(), Some(&[1, 1, 1][..])); + assert_eq!(iter.next_back(), None); + + let mut iter = slice.group_by(|a, b| a == b); + assert_eq!(iter.next(), Some(&[1, 1, 1][..])); + assert_eq!(iter.next_back(), Some(&[0][..])); + assert_eq!(iter.next(), Some(&[3, 3][..])); + assert_eq!(iter.next_back(), Some(&[1][..])); + assert_eq!(iter.next(), Some(&[2, 2, 2][..])); + assert_eq!(iter.next_back(), None); +} + +#[test] +fn test_group_by_mut() { + let slice = &mut [1, 1, 1, 3, 3, 2, 2, 2, 1, 0]; + + let mut iter = slice.group_by_mut(|a, b| a == b); + assert_eq!(iter.next(), Some(&mut [1, 1, 1][..])); + assert_eq!(iter.next(), Some(&mut [3, 3][..])); + assert_eq!(iter.next(), Some(&mut [2, 2, 2][..])); + assert_eq!(iter.next(), Some(&mut [1][..])); + assert_eq!(iter.next(), Some(&mut [0][..])); + assert_eq!(iter.next(), None); + + let mut iter = slice.group_by_mut(|a, b| a == b); + assert_eq!(iter.next_back(), Some(&mut [0][..])); + assert_eq!(iter.next_back(), Some(&mut [1][..])); + assert_eq!(iter.next_back(), Some(&mut [2, 2, 2][..])); + assert_eq!(iter.next_back(), Some(&mut [3, 3][..])); + assert_eq!(iter.next_back(), Some(&mut [1, 1, 1][..])); + assert_eq!(iter.next_back(), None); + + let mut iter = slice.group_by_mut(|a, b| a == b); + assert_eq!(iter.next(), Some(&mut [1, 1, 1][..])); + assert_eq!(iter.next_back(), Some(&mut [0][..])); + assert_eq!(iter.next(), Some(&mut [3, 3][..])); + assert_eq!(iter.next_back(), Some(&mut [1][..])); + assert_eq!(iter.next(), Some(&mut [2, 2, 2][..])); + assert_eq!(iter.next_back(), None); } diff --git a/crux-mir/lib/alloc/tests/str.rs b/crux-mir/lib/alloc/tests/str.rs index b703df6f3..4d182be02 100644 --- a/crux-mir/lib/alloc/tests/str.rs +++ b/crux-mir/lib/alloc/tests/str.rs @@ -1,6 +1,7 @@ +use std::assert_matches::assert_matches; use std::borrow::Cow; use std::cmp::Ordering::{Equal, Greater, Less}; -use std::str::from_utf8; +use std::str::{from_utf8, from_utf8_unchecked}; #[test] fn test_le() { @@ -160,6 +161,36 @@ fn test_join_for_different_lengths_with_long_separator() { test_join!("～～～～～a～～～～～bc", ["", "a", "bc"], "～～～～～"); } +#[test] +fn test_join_issue_80335() { + use core::{borrow::Borrow, cell::Cell}; + + struct WeirdBorrow { + state: Cell, + } + + impl Default for WeirdBorrow { + fn default() -> Self { + WeirdBorrow { state: Cell::new(false) } + } + } + + impl Borrow for WeirdBorrow { + fn borrow(&self) -> &str { + let state = self.state.get(); + if state { + "0" + } else { + self.state.set(true); + "123456" + } + } + } + + let arr: [WeirdBorrow; 3] = Default::default(); + test_join!("0-0-0", arr, "-"); +} + #[test] #[cfg_attr(miri, ignore)] // Miri is too slow fn test_unsafe_slice() { @@ -504,7 +535,7 @@ mod slice_index { #[test] #[should_panic] fn test_slice_fail() { - &"中华Việt Nam"[0..2]; + let _ = &"中华Việt Nam"[0..2]; } panic_cases! { @@ -529,6 +560,13 @@ mod slice_index { message: "out of bounds"; } + in mod rangeinclusive_len { + data: "abcdef"; + good: data[0..=5] == "abcdef"; + bad: data[0..=6]; + message: "out of bounds"; + } + in mod range_len_len { data: "abcdef"; good: data[6..6] == ""; @@ -544,6 +582,28 @@ mod slice_index { } } + panic_cases! { + in mod rangeinclusive_exhausted { + data: "abcdef"; + + good: data[0..=5] == "abcdef"; + good: data[{ + let mut iter = 0..=5; + iter.by_ref().count(); // exhaust it + iter + }] == ""; + + // 0..=6 is out of bounds before exhaustion, so it + // stands to reason that it still would be after. + bad: data[{ + let mut iter = 0..=6; + iter.by_ref().count(); // exhaust it + iter + }]; + message: "out of bounds"; + } + } + panic_cases! { in mod range_neg_width { data: "abcdef"; @@ -566,13 +626,13 @@ mod slice_index { data: "hello"; // note: using 0 specifically ensures that the result of overflowing is 0..0, // so that `get` doesn't simply return None for the wrong reason. - bad: data[0..=usize::max_value()]; + bad: data[0..=usize::MAX]; message: "maximum usize"; } in mod rangetoinclusive { data: "hello"; - bad: data[..=usize::max_value()]; + bad: data[..=usize::MAX]; message: "maximum usize"; } } @@ -655,13 +715,13 @@ mod slice_index { #[test] #[should_panic(expected = "byte index 1024 is out of bounds of `Lorem ipsum dolor sit amet")] fn test_slice_fail_truncated_1() { - &LOREM_PARAGRAPH[..1024]; + let _ = &LOREM_PARAGRAPH[..1024]; } // check the truncation in the panic message #[test] #[should_panic(expected = "luctus, im`[...]")] fn test_slice_fail_truncated_2() { - &LOREM_PARAGRAPH[..1024]; + let _ = &LOREM_PARAGRAPH[..1024]; } } @@ -676,7 +736,7 @@ fn test_str_slice_rangetoinclusive_ok() { #[should_panic] fn test_str_slice_rangetoinclusive_notok() { let s = "abcαβγ"; - &s[..=3]; + let _ = &s[..=3]; } #[test] @@ -692,7 +752,7 @@ fn test_str_slicemut_rangetoinclusive_ok() { fn test_str_slicemut_rangetoinclusive_notok() { let mut s = "abcαβγ".to_owned(); let s: &mut str = &mut s; - &mut s[..=3]; + let _ = &mut s[..=3]; } #[test] @@ -824,6 +884,33 @@ fn test_is_utf8() { assert!(from_utf8(&[0xF4, 0x8F, 0xBF, 0xBF]).is_ok()); } +#[test] +fn test_const_is_utf8() { + const _: () = { + // deny overlong encodings + assert!(from_utf8(&[0xc0, 0x80]).is_err()); + assert!(from_utf8(&[0xc0, 0xae]).is_err()); + assert!(from_utf8(&[0xe0, 0x80, 0x80]).is_err()); + assert!(from_utf8(&[0xe0, 0x80, 0xaf]).is_err()); + assert!(from_utf8(&[0xe0, 0x81, 0x81]).is_err()); + assert!(from_utf8(&[0xf0, 0x82, 0x82, 0xac]).is_err()); + assert!(from_utf8(&[0xf4, 0x90, 0x80, 0x80]).is_err()); + + // deny surrogates + assert!(from_utf8(&[0xED, 0xA0, 0x80]).is_err()); + assert!(from_utf8(&[0xED, 0xBF, 0xBF]).is_err()); + + assert!(from_utf8(&[0xC2, 0x80]).is_ok()); + assert!(from_utf8(&[0xDF, 0xBF]).is_ok()); + assert!(from_utf8(&[0xE0, 0xA0, 0x80]).is_ok()); + assert!(from_utf8(&[0xED, 0x9F, 0xBF]).is_ok()); + assert!(from_utf8(&[0xEE, 0x80, 0x80]).is_ok()); + assert!(from_utf8(&[0xEF, 0xBF, 0xBF]).is_ok()); + assert!(from_utf8(&[0xF0, 0x90, 0x80, 0x80]).is_ok()); + assert!(from_utf8(&[0xF4, 0x8F, 0xBF, 0xBF]).is_ok()); + }; +} + #[test] fn from_utf8_mostly_ascii() { // deny invalid bytes embedded in long stretches of ascii @@ -836,13 +923,43 @@ fn from_utf8_mostly_ascii() { } } +#[test] +fn const_from_utf8_mostly_ascii() { + const _: () = { + // deny invalid bytes embedded in long stretches of ascii + let mut i = 32; + while i < 64 { + let mut data = [0; 128]; + data[i] = 0xC0; + assert!(from_utf8(&data).is_err()); + data[i] = 0xC2; + assert!(from_utf8(&data).is_err()); + + i = i + 1; + } + }; +} + #[test] fn from_utf8_error() { macro_rules! test { - ($input: expr, $expected_valid_up_to: expr, $expected_error_len: expr) => { + ($input: expr, $expected_valid_up_to:pat, $expected_error_len:pat) => { let error = from_utf8($input).unwrap_err(); - assert_eq!(error.valid_up_to(), $expected_valid_up_to); - assert_eq!(error.error_len(), $expected_error_len); + assert_matches!(error.valid_up_to(), $expected_valid_up_to); + assert_matches!(error.error_len(), $expected_error_len); + + const _: () = { + match from_utf8($input) { + Err(error) => { + let valid_up_to = error.valid_up_to(); + let error_len = error.error_len(); + + assert!(matches!(valid_up_to, $expected_valid_up_to)); + assert!(matches!(error_len, $expected_error_len)); + } + Ok(_) => unreachable!(), + } + }; }; } test!(b"A\xC3\xA9 \xFF ", 4, Some(1)); @@ -893,11 +1010,11 @@ fn test_as_bytes_fail() { fn test_as_ptr() { let buf = "hello".as_ptr(); unsafe { - assert_eq!(*buf.offset(0), b'h'); - assert_eq!(*buf.offset(1), b'e'); - assert_eq!(*buf.offset(2), b'l'); - assert_eq!(*buf.offset(3), b'l'); - assert_eq!(*buf.offset(4), b'o'); + assert_eq!(*buf.add(0), b'h'); + assert_eq!(*buf.add(1), b'e'); + assert_eq!(*buf.add(2), b'l'); + assert_eq!(*buf.add(3), b'l'); + assert_eq!(*buf.add(4), b'o'); } } @@ -972,7 +1089,7 @@ fn test_split_at_mut() { #[should_panic] fn test_split_at_boundscheck() { let s = "ศไทย中华Việt Nam"; - s.split_at(1); + let _ = s.split_at(1); } #[test] @@ -999,7 +1116,7 @@ fn test_escape_debug() { assert_eq!("abc".escape_debug().to_string(), "abc"); assert_eq!("a c".escape_debug().to_string(), "a c"); assert_eq!("éèê".escape_debug().to_string(), "éèê"); - assert_eq!("\r\n\t".escape_debug().to_string(), "\\r\\n\\t"); + assert_eq!("\0\r\n\t".escape_debug().to_string(), "\\0\\r\\n\\t"); assert_eq!("'\"\\".escape_debug().to_string(), "\\'\\\"\\\\"); assert_eq!("\u{7f}\u{ff}".escape_debug().to_string(), "\\u{7f}\u{ff}"); assert_eq!("\u{100}\u{ffff}".escape_debug().to_string(), "\u{100}\\u{ffff}"); @@ -1066,6 +1183,37 @@ fn test_rev_iterator() { assert_eq!(pos, v.len()); } +#[test] +fn test_to_lowercase_rev_iterator() { + let s = "AÖßÜ💩ΣΤΙΓΜΑΣDžfiİ"; + let v = ['\u{307}', 'i', 'fi', 'dž', 'σ', 'α', 'μ', 'γ', 'ι', 'τ', 'σ', '💩', 'ü', 'ß', 'ö', 'a']; + + let mut pos = 0; + let it = s.chars().flat_map(|c| c.to_lowercase()).rev(); + + for c in it { + assert_eq!(c, v[pos]); + pos += 1; + } + assert_eq!(pos, v.len()); +} + +#[test] +fn test_to_uppercase_rev_iterator() { + let s = "aößü💩στιγμαςDžfiᾀ"; + let v = + ['Ι', 'Ἀ', 'I', 'F', 'DŽ', 'Σ', 'Α', 'Μ', 'Γ', 'Ι', 'Τ', 'Σ', '💩', 'Ü', 'S', 'S', 'Ö', 'A']; + + let mut pos = 0; + let it = s.chars().flat_map(|c| c.to_uppercase()).rev(); + + for c in it { + assert_eq!(c, v[pos]); + pos += 1; + } + assert_eq!(pos, v.len()); +} + #[test] #[cfg_attr(miri, ignore)] // Miri is too slow fn test_chars_decoding() { @@ -1111,7 +1259,7 @@ fn test_chars_debug() { let s = "ศไทย中华Việt Nam"; let c = s.chars(); assert_eq!( - format!("{:?}", c), + format!("{c:?}"), r#"Chars(['ศ', 'ไ', 'ท', 'ย', '中', '华', 'V', 'i', 'ệ', 't', ' ', 'N', 'a', 'm'])"# ); } @@ -1318,6 +1466,30 @@ fn test_rsplitn() { assert_eq!(split, ["mb\n", "\nMäry häd ä little lämb\nLittle l"]); } +#[test] +fn test_split_once() { + assert_eq!("".split_once("->"), None); + assert_eq!("-".split_once("->"), None); + assert_eq!("->".split_once("->"), Some(("", ""))); + assert_eq!("a->".split_once("->"), Some(("a", ""))); + assert_eq!("->b".split_once("->"), Some(("", "b"))); + assert_eq!("a->b".split_once("->"), Some(("a", "b"))); + assert_eq!("a->b->c".split_once("->"), Some(("a", "b->c"))); + assert_eq!("---".split_once("--"), Some(("", "-"))); +} + +#[test] +fn test_rsplit_once() { + assert_eq!("".rsplit_once("->"), None); + assert_eq!("-".rsplit_once("->"), None); + assert_eq!("->".rsplit_once("->"), Some(("", ""))); + assert_eq!("a->".rsplit_once("->"), Some(("a", ""))); + assert_eq!("->b".rsplit_once("->"), Some(("", "b"))); + assert_eq!("a->b".rsplit_once("->"), Some(("a", "b"))); + assert_eq!("a->b->c".rsplit_once("->"), Some(("a->b", "c"))); + assert_eq!("---".rsplit_once("--"), Some(("-", ""))); +} + #[test] fn test_split_whitespace() { let data = "\n \tMäry häd\tä little lämb\nLittle lämb\n"; @@ -1418,11 +1590,27 @@ fn test_bool_from_str() { assert_eq!("not even a boolean".parse::().ok(), None); } -fn check_contains_all_substrings(s: &str) { - assert!(s.contains("")); - for i in 0..s.len() { - for j in i + 1..=s.len() { - assert!(s.contains(&s[i..j])); +fn check_contains_all_substrings(haystack: &str) { + let mut modified_needle = String::new(); + + for i in 0..haystack.len() { + // check different haystack lengths since we special-case short haystacks. + let haystack = &haystack[0..i]; + assert!(haystack.contains("")); + for j in 0..haystack.len() { + for k in j + 1..=haystack.len() { + let needle = &haystack[j..k]; + assert!(haystack.contains(needle)); + modified_needle.clear(); + modified_needle.push_str(needle); + modified_needle.replace_range(0..1, "\0"); + assert!(!haystack.contains(&modified_needle)); + + modified_needle.clear(); + modified_needle.push_str(needle); + modified_needle.replace_range(needle.len() - 1..needle.len(), "\0"); + assert!(!haystack.contains(&modified_needle)); + } } } } @@ -1443,6 +1631,18 @@ fn strslice_issue_16878() { assert!(!"00abc01234567890123456789abc".contains("bcabc")); } +#[test] +fn strslice_issue_104726() { + // Edge-case in the simd_contains impl. + // The first and last byte are the same so it backtracks by one byte + // which aligns with the end of the string. Previously incorrect offset calculations + // lead to out-of-bounds slicing. + #[rustfmt::skip] + let needle = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaba"; + let haystack = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab"; + assert!(!haystack.contains(needle)); +} + #[test] #[cfg_attr(miri, ignore)] // Miri is too slow fn test_strslice_contains() { @@ -1600,6 +1800,20 @@ fn to_lowercase() { assert_eq!("ΑΣΑ".to_lowercase(), "ασα"); assert_eq!("ΑΣ'Α".to_lowercase(), "ασ'α"); assert_eq!("ΑΣ''Α".to_lowercase(), "ασ''α"); + + // a really long string that has it's lowercase form + // even longer. this tests that implementations don't assume + // an incorrect upper bound on allocations + let upper = str::repeat("İ", 512); + let lower = str::repeat("i̇", 512); + assert_eq!(upper.to_lowercase(), lower); + + // a really long ascii-only string. + // This test that the ascii hot-path + // functions correctly + let upper = str::repeat("A", 511); + let lower = str::repeat("a", 511); + assert_eq!(upper.to_lowercase(), lower); } #[test] @@ -1698,7 +1912,7 @@ mod pattern { } if let Some(err) = err { - panic!("Input skipped range at {}", err); + panic!("Input skipped range at {err}"); } if first_index != haystack.len() { @@ -1790,6 +2004,47 @@ mod pattern { "* \t", [Reject(0, 1), Reject(1, 2), Reject(2, 3),] ); + + // See #85462 + #[test] + fn str_searcher_empty_needle_after_done() { + // Empty needle and haystack + { + let mut searcher = "".into_searcher(""); + + assert_eq!(searcher.next(), SearchStep::Match(0, 0)); + assert_eq!(searcher.next(), SearchStep::Done); + assert_eq!(searcher.next(), SearchStep::Done); + assert_eq!(searcher.next(), SearchStep::Done); + + let mut searcher = "".into_searcher(""); + + assert_eq!(searcher.next_back(), SearchStep::Match(0, 0)); + assert_eq!(searcher.next_back(), SearchStep::Done); + assert_eq!(searcher.next_back(), SearchStep::Done); + assert_eq!(searcher.next_back(), SearchStep::Done); + } + // Empty needle and non-empty haystack + { + let mut searcher = "".into_searcher("a"); + + assert_eq!(searcher.next(), SearchStep::Match(0, 0)); + assert_eq!(searcher.next(), SearchStep::Reject(0, 1)); + assert_eq!(searcher.next(), SearchStep::Match(1, 1)); + assert_eq!(searcher.next(), SearchStep::Done); + assert_eq!(searcher.next(), SearchStep::Done); + assert_eq!(searcher.next(), SearchStep::Done); + + let mut searcher = "".into_searcher("a"); + + assert_eq!(searcher.next_back(), SearchStep::Match(1, 1)); + assert_eq!(searcher.next_back(), SearchStep::Reject(0, 1)); + assert_eq!(searcher.next_back(), SearchStep::Match(0, 0)); + assert_eq!(searcher.next_back(), SearchStep::Done); + assert_eq!(searcher.next_back(), SearchStep::Done); + assert_eq!(searcher.next_back(), SearchStep::Done); + } + } } macro_rules! generate_iterator_test { @@ -1897,3 +2152,260 @@ fn different_str_pattern_forwarding_lifetimes() { foo::<&str>("x"); } + +#[test] +fn test_str_multiline() { + let a: String = "this \ +is a test" + .to_string(); + let b: String = "this \ + is \ + another \ + test" + .to_string(); + assert_eq!(a, "this is a test".to_string()); + assert_eq!(b, "this is another test".to_string()); +} + +#[test] +fn test_str_escapes() { + let x = "\\\\\ + "; + assert_eq!(x, r"\\"); // extraneous whitespace stripped +} + +#[test] +fn const_str_ptr() { + const A: [u8; 2] = ['h' as u8, 'i' as u8]; + const B: &'static [u8; 2] = &A; + const C: *const u8 = B as *const u8; + + // Miri does not deduplicate consts (https://github.com/rust-lang/miri/issues/131) + #[cfg(not(miri))] + { + let foo = &A as *const u8; + assert_eq!(foo, C); + } + + unsafe { + assert_eq!(from_utf8_unchecked(&A), "hi"); + assert_eq!(*C, A[0]); + assert_eq!(*(&B[0] as *const u8), A[0]); + } +} + +#[test] +fn utf8() { + let yen: char = '¥'; // 0xa5 + let c_cedilla: char = 'ç'; // 0xe7 + let thorn: char = 'þ'; // 0xfe + let y_diaeresis: char = 'ÿ'; // 0xff + let pi: char = 'Π'; // 0x3a0 + + assert_eq!(yen as isize, 0xa5); + assert_eq!(c_cedilla as isize, 0xe7); + assert_eq!(thorn as isize, 0xfe); + assert_eq!(y_diaeresis as isize, 0xff); + assert_eq!(pi as isize, 0x3a0); + + assert_eq!(pi as isize, '\u{3a0}' as isize); + assert_eq!('\x0a' as isize, '\n' as isize); + + let bhutan: String = "འབྲུག་ཡུལ།".to_string(); + let japan: String = "日本".to_string(); + let uzbekistan: String = "Ўзбекистон".to_string(); + let austria: String = "Österreich".to_string(); + + let bhutan_e: String = + "\u{f60}\u{f56}\u{fb2}\u{f74}\u{f42}\u{f0b}\u{f61}\u{f74}\u{f63}\u{f0d}".to_string(); + let japan_e: String = "\u{65e5}\u{672c}".to_string(); + let uzbekistan_e: String = + "\u{40e}\u{437}\u{431}\u{435}\u{43a}\u{438}\u{441}\u{442}\u{43e}\u{43d}".to_string(); + let austria_e: String = "\u{d6}sterreich".to_string(); + + let oo: char = 'Ö'; + assert_eq!(oo as isize, 0xd6); + + fn check_str_eq(a: String, b: String) { + let mut i: isize = 0; + for ab in a.bytes() { + println!("{i}"); + println!("{ab}"); + let bb: u8 = b.as_bytes()[i as usize]; + println!("{bb}"); + assert_eq!(ab, bb); + i += 1; + } + } + + check_str_eq(bhutan, bhutan_e); + check_str_eq(japan, japan_e); + check_str_eq(uzbekistan, uzbekistan_e); + check_str_eq(austria, austria_e); +} + +#[test] +fn utf8_chars() { + // Chars of 1, 2, 3, and 4 bytes + let chs: Vec = vec!['e', 'é', '€', '\u{10000}']; + let s: String = chs.iter().cloned().collect(); + let schs: Vec = s.chars().collect(); + + assert_eq!(s.len(), 10); + assert_eq!(s.chars().count(), 4); + assert_eq!(schs.len(), 4); + assert_eq!(schs.iter().cloned().collect::(), s); + + assert!((from_utf8(s.as_bytes()).is_ok())); + // invalid prefix + assert!((!from_utf8(&[0x80]).is_ok())); + // invalid 2 byte prefix + assert!((!from_utf8(&[0xc0]).is_ok())); + assert!((!from_utf8(&[0xc0, 0x10]).is_ok())); + // invalid 3 byte prefix + assert!((!from_utf8(&[0xe0]).is_ok())); + assert!((!from_utf8(&[0xe0, 0x10]).is_ok())); + assert!((!from_utf8(&[0xe0, 0xff, 0x10]).is_ok())); + // invalid 4 byte prefix + assert!((!from_utf8(&[0xf0]).is_ok())); + assert!((!from_utf8(&[0xf0, 0x10]).is_ok())); + assert!((!from_utf8(&[0xf0, 0xff, 0x10]).is_ok())); + assert!((!from_utf8(&[0xf0, 0xff, 0xff, 0x10]).is_ok())); +} + +#[test] +fn utf8_char_counts() { + let strs = [("e", 1), ("é", 1), ("€", 1), ("\u{10000}", 1), ("eé€\u{10000}", 4)]; + let spread = if cfg!(miri) { 4 } else { 8 }; + let mut reps = [8, 64, 256, 512] + .iter() + .copied() + .flat_map(|n| n - spread..=n + spread) + .collect::>(); + if cfg!(not(miri)) { + reps.extend([1024, 1 << 16].iter().copied().flat_map(|n| n - spread..=n + spread)); + } + let counts = if cfg!(miri) { 0..1 } else { 0..8 }; + let padding = counts.map(|len| " ".repeat(len)).collect::>(); + + for repeat in reps { + for (tmpl_str, tmpl_char_count) in strs { + for pad_start in &padding { + for pad_end in &padding { + // Create a string with padding... + let with_padding = + format!("{}{}{}", pad_start, tmpl_str.repeat(repeat), pad_end); + // ...and then skip past that padding. This should ensure + // that we test several different alignments for both head + // and tail. + let si = pad_start.len(); + let ei = with_padding.len() - pad_end.len(); + let target = &with_padding[si..ei]; + + assert!(!target.starts_with(" ") && !target.ends_with(" ")); + let expected_count = tmpl_char_count * repeat; + assert_eq!( + expected_count, + target.chars().count(), + "wrong count for `{:?}.repeat({})` (padding: `{:?}`)", + tmpl_str, + repeat, + (pad_start.len(), pad_end.len()), + ); + } + } + } + } +} + +#[test] +fn floor_char_boundary() { + fn check_many(s: &str, arg: impl IntoIterator, ret: usize) { + for idx in arg { + assert_eq!( + s.floor_char_boundary(idx), + ret, + "{:?}.floor_char_boundary({:?}) != {:?}", + s, + idx, + ret + ); + } + } + + // edge case + check_many("", [0, 1, isize::MAX as usize, usize::MAX], 0); + + // basic check + check_many("x", [0], 0); + check_many("x", [1, isize::MAX as usize, usize::MAX], 1); + + // 1-byte chars + check_many("jp", [0], 0); + check_many("jp", [1], 1); + check_many("jp", 2..4, 2); + + // 2-byte chars + check_many("ĵƥ", 0..2, 0); + check_many("ĵƥ", 2..4, 2); + check_many("ĵƥ", 4..6, 4); + + // 3-byte chars + check_many("日本", 0..3, 0); + check_many("日本", 3..6, 3); + check_many("日本", 6..8, 6); + + // 4-byte chars + check_many("🇯🇵", 0..4, 0); + check_many("🇯🇵", 4..8, 4); + check_many("🇯🇵", 8..10, 8); +} + +#[test] +fn ceil_char_boundary() { + fn check_many(s: &str, arg: impl IntoIterator, ret: usize) { + for idx in arg { + assert_eq!( + s.ceil_char_boundary(idx), + ret, + "{:?}.ceil_char_boundary({:?}) != {:?}", + s, + idx, + ret + ); + } + } + + // edge case + check_many("", [0], 0); + + // basic check + check_many("x", [0], 0); + check_many("x", [1], 1); + + // 1-byte chars + check_many("jp", [0], 0); + check_many("jp", [1], 1); + check_many("jp", [2], 2); + + // 2-byte chars + check_many("ĵƥ", 0..=0, 0); + check_many("ĵƥ", 1..=2, 2); + check_many("ĵƥ", 3..=4, 4); + + // 3-byte chars + check_many("日本", 0..=0, 0); + check_many("日本", 1..=3, 3); + check_many("日本", 4..=6, 6); + + // 4-byte chars + check_many("🇯🇵", 0..=0, 0); + check_many("🇯🇵", 1..=4, 4); + check_many("🇯🇵", 5..=8, 8); +} + +#[test] +#[should_panic] +fn ceil_char_boundary_above_len_panic() { + let _ = "x".ceil_char_boundary(2); +} diff --git a/crux-mir/lib/alloc/tests/string.rs b/crux-mir/lib/alloc/tests/string.rs index 08859b2b2..99d1296a4 100644 --- a/crux-mir/lib/alloc/tests/string.rs +++ b/crux-mir/lib/alloc/tests/string.rs @@ -1,7 +1,12 @@ +use std::assert_matches::assert_matches; use std::borrow::Cow; -use std::collections::TryReserveError::*; -use std::mem::size_of; -use std::{isize, usize}; +use std::cell::Cell; +use std::collections::TryReserveErrorKind::*; +use std::ops::Bound; +use std::ops::Bound::*; +use std::ops::RangeBounds; +use std::panic; +use std::str; pub trait IntoCow<'a, B: ?Sized> where @@ -266,30 +271,34 @@ fn test_split_off_empty() { fn test_split_off_past_end() { let orig = "Hello, world!"; let mut split = String::from(orig); - split.split_off(orig.len() + 1); + let _ = split.split_off(orig.len() + 1); } #[test] #[should_panic] fn test_split_off_mid_char() { - let mut orig = String::from("山"); - orig.split_off(1); + let mut shan = String::from("山"); + let _broken_mountain = shan.split_off(1); } #[test] fn test_split_off_ascii() { let mut ab = String::from("ABCD"); + let orig_capacity = ab.capacity(); let cd = ab.split_off(2); assert_eq!(ab, "AB"); assert_eq!(cd, "CD"); + assert_eq!(ab.capacity(), orig_capacity); } #[test] fn test_split_off_unicode() { let mut nihon = String::from("日本語"); + let orig_capacity = nihon.capacity(); let go = nihon.split_off("日本".len()); assert_eq!(nihon, "日本"); assert_eq!(go, "語"); + assert_eq!(nihon.capacity(), orig_capacity); } #[test] @@ -357,6 +366,33 @@ fn remove_bad() { "ศ".to_string().remove(1); } +#[test] +fn test_remove_matches() { + let mut s = "abc".to_string(); + + s.remove_matches('b'); + assert_eq!(s, "ac"); + s.remove_matches('b'); + assert_eq!(s, "ac"); + + let mut s = "abcb".to_string(); + + s.remove_matches('b'); + assert_eq!(s, "ac"); + + let mut s = "ศไทย中华Việt Nam; foobarศ".to_string(); + s.remove_matches('ศ'); + assert_eq!(s, "ไทย中华Việt Nam; foobar"); + + let mut s = "".to_string(); + s.remove_matches(""); + assert_eq!(s, ""); + + let mut s = "aaaaa".to_string(); + s.remove_matches('a'); + assert_eq!(s, ""); +} + #[test] fn test_retain() { let mut s = String::from("α_β_γ"); @@ -375,6 +411,20 @@ fn test_retain() { s.retain(|_| false); assert_eq!(s, ""); + + let mut s = String::from("0è0"); + let _ = panic::catch_unwind(panic::AssertUnwindSafe(|| { + let mut count = 0; + s.retain(|_| { + count += 1; + match count { + 1 => false, + 2 => true, + _ => panic!(), + } + }); + })); + assert!(std::str::from_utf8(s.as_bytes()).is_ok()); } #[test] @@ -420,7 +470,7 @@ fn test_simple_types() { #[test] fn test_vectors() { let x: Vec = vec![]; - assert_eq!(format!("{:?}", x), "[]"); + assert_eq!(format!("{x:?}"), "[]"); assert_eq!(format!("{:?}", vec![1]), "[1]"); assert_eq!(format!("{:?}", vec![1, 2, 3]), "[1, 2, 3]"); assert!(format!("{:?}", vec![vec![], vec![1], vec![1, 1]]) == "[[], [1], [1, 1]]"); @@ -439,7 +489,7 @@ fn test_from_iterator() { b.extend(u.chars()); assert_eq!(s, b); - let c: String = vec![t, u].into_iter().collect(); + let c: String = [t, u].into_iter().collect(); assert_eq!(s, c); let mut d = t.to_string(); @@ -464,6 +514,20 @@ fn test_drain() { assert_eq!(t, ""); } +#[test] +#[should_panic] +fn test_drain_start_overflow() { + let mut s = String::from("abc"); + s.drain((Excluded(usize::MAX), Included(0))); +} + +#[test] +#[should_panic] +fn test_drain_end_overflow() { + let mut s = String::from("abc"); + s.drain((Included(0), Included(usize::MAX))); +} + #[test] fn test_replace_range() { let mut s = "Hello, world!".to_owned(); @@ -501,6 +565,20 @@ fn test_replace_range_inclusive_out_of_bounds() { s.replace_range(5..=5, "789"); } +#[test] +#[should_panic] +fn test_replace_range_start_overflow() { + let mut s = String::from("123"); + s.replace_range((Excluded(usize::MAX), Included(0)), ""); +} + +#[test] +#[should_panic] +fn test_replace_range_end_overflow() { + let mut s = String::from("456"); + s.replace_range((Included(0), Included(usize::MAX)), ""); +} + #[test] fn test_replace_range_empty() { let mut s = String::from("12345"); @@ -515,6 +593,52 @@ fn test_replace_range_unbounded() { assert_eq!(s, ""); } +#[test] +fn test_replace_range_evil_start_bound() { + struct EvilRange(Cell); + + impl RangeBounds for EvilRange { + fn start_bound(&self) -> Bound<&usize> { + Bound::Included(if self.0.get() { + &1 + } else { + self.0.set(true); + &0 + }) + } + fn end_bound(&self) -> Bound<&usize> { + Bound::Unbounded + } + } + + let mut s = String::from("🦀"); + s.replace_range(EvilRange(Cell::new(false)), ""); + assert_eq!(Ok(""), str::from_utf8(s.as_bytes())); +} + +#[test] +fn test_replace_range_evil_end_bound() { + struct EvilRange(Cell); + + impl RangeBounds for EvilRange { + fn start_bound(&self) -> Bound<&usize> { + Bound::Included(&0) + } + fn end_bound(&self) -> Bound<&usize> { + Bound::Excluded(if self.0.get() { + &3 + } else { + self.0.set(true); + &4 + }) + } + } + + let mut s = String::from("🦀"); + s.replace_range(EvilRange(Cell::new(false)), ""); + assert_eq!(Ok(""), str::from_utf8(s.as_bytes())); +} + #[test] fn test_extend_ref() { let mut a = "foo".to_string(); @@ -556,6 +680,7 @@ fn test_reserve_exact() { #[test] #[cfg_attr(miri, ignore)] // Miri does not support signalling OOM +#[cfg_attr(target_os = "android", ignore)] // Android used in CI has a broken dlmalloc fn test_try_reserve() { // These are the interesting cases: // * exactly isize::MAX should never trigger a CapacityOverflow (can be OOM) @@ -568,83 +693,63 @@ fn test_try_reserve() { const MAX_CAP: usize = isize::MAX as usize; const MAX_USIZE: usize = usize::MAX; - // On 16/32-bit, we check that allocations don't exceed isize::MAX, - // on 64-bit, we assume the OS will give an OOM for such a ridiculous size. - // Any platform that succeeds for these requests is technically broken with - // ptr::offset because LLVM is the worst. - let guards_against_isize = size_of::() < 8; - { // Note: basic stuff is checked by test_reserve let mut empty_string: String = String::new(); // Check isize::MAX doesn't count as an overflow - if let Err(CapacityOverflow) = empty_string.try_reserve(MAX_CAP) { + if let Err(CapacityOverflow) = empty_string.try_reserve(MAX_CAP).map_err(|e| e.kind()) { panic!("isize::MAX shouldn't trigger an overflow!"); } // Play it again, frank! (just to be sure) - if let Err(CapacityOverflow) = empty_string.try_reserve(MAX_CAP) { + if let Err(CapacityOverflow) = empty_string.try_reserve(MAX_CAP).map_err(|e| e.kind()) { panic!("isize::MAX shouldn't trigger an overflow!"); } - if guards_against_isize { - // Check isize::MAX + 1 does count as overflow - if let Err(CapacityOverflow) = empty_string.try_reserve(MAX_CAP + 1) { - } else { - panic!("isize::MAX + 1 should trigger an overflow!") - } - - // Check usize::MAX does count as overflow - if let Err(CapacityOverflow) = empty_string.try_reserve(MAX_USIZE) { - } else { - panic!("usize::MAX should trigger an overflow!") - } - } else { - // Check isize::MAX + 1 is an OOM - if let Err(AllocError { .. }) = empty_string.try_reserve(MAX_CAP + 1) { - } else { - panic!("isize::MAX + 1 should trigger an OOM!") - } - - // Check usize::MAX is an OOM - if let Err(AllocError { .. }) = empty_string.try_reserve(MAX_USIZE) { - } else { - panic!("usize::MAX should trigger an OOM!") - } - } + // Check isize::MAX + 1 does count as overflow + assert_matches!( + empty_string.try_reserve(MAX_CAP + 1).map_err(|e| e.kind()), + Err(CapacityOverflow), + "isize::MAX + 1 should trigger an overflow!" + ); + + // Check usize::MAX does count as overflow + assert_matches!( + empty_string.try_reserve(MAX_USIZE).map_err(|e| e.kind()), + Err(CapacityOverflow), + "usize::MAX should trigger an overflow!" + ); } { // Same basic idea, but with non-zero len let mut ten_bytes: String = String::from("0123456789"); - if let Err(CapacityOverflow) = ten_bytes.try_reserve(MAX_CAP - 10) { + if let Err(CapacityOverflow) = ten_bytes.try_reserve(MAX_CAP - 10).map_err(|e| e.kind()) { panic!("isize::MAX shouldn't trigger an overflow!"); } - if let Err(CapacityOverflow) = ten_bytes.try_reserve(MAX_CAP - 10) { + if let Err(CapacityOverflow) = ten_bytes.try_reserve(MAX_CAP - 10).map_err(|e| e.kind()) { panic!("isize::MAX shouldn't trigger an overflow!"); } - if guards_against_isize { - if let Err(CapacityOverflow) = ten_bytes.try_reserve(MAX_CAP - 9) { - } else { - panic!("isize::MAX + 1 should trigger an overflow!"); - } - } else { - if let Err(AllocError { .. }) = ten_bytes.try_reserve(MAX_CAP - 9) { - } else { - panic!("isize::MAX + 1 should trigger an OOM!") - } - } + + assert_matches!( + ten_bytes.try_reserve(MAX_CAP - 9).map_err(|e| e.kind()), + Err(CapacityOverflow), + "isize::MAX + 1 should trigger an overflow!" + ); + // Should always overflow in the add-to-len - if let Err(CapacityOverflow) = ten_bytes.try_reserve(MAX_USIZE) { - } else { - panic!("usize::MAX should trigger an overflow!") - } + assert_matches!( + ten_bytes.try_reserve(MAX_USIZE).map_err(|e| e.kind()), + Err(CapacityOverflow), + "usize::MAX should trigger an overflow!" + ); } } #[test] #[cfg_attr(miri, ignore)] // Miri does not support signalling OOM +#[cfg_attr(target_os = "android", ignore)] // Android used in CI has a broken dlmalloc fn test_try_reserve_exact() { // This is exactly the same as test_try_reserve with the method changed. // See that test for comments. @@ -652,64 +757,70 @@ fn test_try_reserve_exact() { const MAX_CAP: usize = isize::MAX as usize; const MAX_USIZE: usize = usize::MAX; - let guards_against_isize = size_of::() < 8; - { let mut empty_string: String = String::new(); - if let Err(CapacityOverflow) = empty_string.try_reserve_exact(MAX_CAP) { + if let Err(CapacityOverflow) = empty_string.try_reserve_exact(MAX_CAP).map_err(|e| e.kind()) + { panic!("isize::MAX shouldn't trigger an overflow!"); } - if let Err(CapacityOverflow) = empty_string.try_reserve_exact(MAX_CAP) { + if let Err(CapacityOverflow) = empty_string.try_reserve_exact(MAX_CAP).map_err(|e| e.kind()) + { panic!("isize::MAX shouldn't trigger an overflow!"); } - if guards_against_isize { - if let Err(CapacityOverflow) = empty_string.try_reserve_exact(MAX_CAP + 1) { - } else { - panic!("isize::MAX + 1 should trigger an overflow!") - } - - if let Err(CapacityOverflow) = empty_string.try_reserve_exact(MAX_USIZE) { - } else { - panic!("usize::MAX should trigger an overflow!") - } - } else { - if let Err(AllocError { .. }) = empty_string.try_reserve_exact(MAX_CAP + 1) { - } else { - panic!("isize::MAX + 1 should trigger an OOM!") - } - - if let Err(AllocError { .. }) = empty_string.try_reserve_exact(MAX_USIZE) { - } else { - panic!("usize::MAX should trigger an OOM!") - } - } + assert_matches!( + empty_string.try_reserve_exact(MAX_CAP + 1).map_err(|e| e.kind()), + Err(CapacityOverflow), + "isize::MAX + 1 should trigger an overflow!" + ); + + assert_matches!( + empty_string.try_reserve_exact(MAX_USIZE).map_err(|e| e.kind()), + Err(CapacityOverflow), + "usize::MAX should trigger an overflow!" + ); } { let mut ten_bytes: String = String::from("0123456789"); - if let Err(CapacityOverflow) = ten_bytes.try_reserve_exact(MAX_CAP - 10) { + if let Err(CapacityOverflow) = + ten_bytes.try_reserve_exact(MAX_CAP - 10).map_err(|e| e.kind()) + { panic!("isize::MAX shouldn't trigger an overflow!"); } - if let Err(CapacityOverflow) = ten_bytes.try_reserve_exact(MAX_CAP - 10) { + if let Err(CapacityOverflow) = + ten_bytes.try_reserve_exact(MAX_CAP - 10).map_err(|e| e.kind()) + { panic!("isize::MAX shouldn't trigger an overflow!"); } - if guards_against_isize { - if let Err(CapacityOverflow) = ten_bytes.try_reserve_exact(MAX_CAP - 9) { - } else { - panic!("isize::MAX + 1 should trigger an overflow!"); - } - } else { - if let Err(AllocError { .. }) = ten_bytes.try_reserve_exact(MAX_CAP - 9) { - } else { - panic!("isize::MAX + 1 should trigger an OOM!") - } - } - if let Err(CapacityOverflow) = ten_bytes.try_reserve_exact(MAX_USIZE) { - } else { - panic!("usize::MAX should trigger an overflow!") - } + + assert_matches!( + ten_bytes.try_reserve_exact(MAX_CAP - 9).map_err(|e| e.kind()), + Err(CapacityOverflow), + "isize::MAX + 1 should trigger an overflow!" + ); + + assert_matches!( + ten_bytes.try_reserve_exact(MAX_USIZE).map_err(|e| e.kind()), + Err(CapacityOverflow), + "usize::MAX should trigger an overflow!" + ); } } + +#[test] +fn test_from_char() { + assert_eq!(String::from('a'), 'a'.to_string()); + let s: String = 'x'.into(); + assert_eq!(s, 'x'.to_string()); +} + +#[test] +fn test_str_concat() { + let a: String = "hello".to_string(); + let b: String = "world".to_string(); + let s: String = format!("{a}{b}"); + assert_eq!(s.as_bytes()[9], 'd' as u8); +} diff --git a/crux-mir/lib/alloc/tests/thin_box.rs b/crux-mir/lib/alloc/tests/thin_box.rs new file mode 100644 index 000000000..e008b0cc3 --- /dev/null +++ b/crux-mir/lib/alloc/tests/thin_box.rs @@ -0,0 +1,262 @@ +use core::fmt::Debug; +use core::mem::size_of; +use std::boxed::ThinBox; + +#[test] +fn want_niche_optimization() { + fn uses_niche() -> bool { + size_of::<*const ()>() == size_of::>>() + } + + trait Tr {} + assert!(uses_niche::()); + assert!(uses_niche::<[i32]>()); + assert!(uses_niche::()); +} + +#[test] +fn want_thin() { + fn is_thin() -> bool { + size_of::<*const ()>() == size_of::>() + } + + trait Tr {} + assert!(is_thin::()); + assert!(is_thin::<[i32]>()); + assert!(is_thin::()); +} + +#[allow(dead_code)] +fn assert_covariance() { + fn thin_box<'new>(b: ThinBox<[&'static str]>) -> ThinBox<[&'new str]> { + b + } +} + +#[track_caller] +fn verify_aligned(ptr: *const T) { + // Use `black_box` to attempt to obscure the fact that we're calling this + // function on pointers that come from box/references, which the compiler + // would otherwise realize is impossible (because it would mean we've + // already executed UB). + // + // That is, we'd *like* it to be possible for the asserts in this function + // to detect brokenness in the ThinBox impl. + // + // It would probably be better if we instead had these as debug_asserts + // inside `ThinBox`, prior to the point where we do the UB. Anyway, in + // practice these checks are mostly just smoke-detectors for an extremely + // broken `ThinBox` impl, since it's an extremely subtle piece of code. + let ptr = core::hint::black_box(ptr); + assert!( + ptr.is_aligned() && !ptr.is_null(), + "misaligned ThinBox data; valid pointers to `{ty}` should be aligned to {align}: {ptr:p}", + ty = core::any::type_name::(), + align = core::mem::align_of::(), + ); +} + +#[track_caller] +fn check_thin_sized(make: impl FnOnce() -> T) { + let value = make(); + let boxed = ThinBox::new(value.clone()); + let val = &*boxed; + verify_aligned(val as *const T); + assert_eq!(val, &value); +} + +#[track_caller] +fn check_thin_dyn(make: impl FnOnce() -> T) { + let value = make(); + let wanted_debug = format!("{value:?}"); + let boxed: ThinBox = ThinBox::new_unsize(value.clone()); + let val = &*boxed; + // wide reference -> wide pointer -> thin pointer + verify_aligned(val as *const dyn Debug as *const T); + let got_debug = format!("{val:?}"); + assert_eq!(wanted_debug, got_debug); +} + +macro_rules! define_test { + ( + @test_name: $testname:ident; + + $(#[$m:meta])* + struct $Type:ident($inner:ty); + + $($test_stmts:tt)* + ) => { + #[test] + fn $testname() { + use core::sync::atomic::{AtomicIsize, Ordering}; + // Define the type, and implement new/clone/drop in such a way that + // the number of live instances will be counted. + $(#[$m])* + #[derive(Debug, PartialEq)] + struct $Type { + _priv: $inner, + } + + impl Clone for $Type { + fn clone(&self) -> Self { + verify_aligned(self); + Self::new(self._priv.clone()) + } + } + + impl Drop for $Type { + fn drop(&mut self) { + verify_aligned(self); + Self::modify_live(-1); + } + } + + impl $Type { + fn new(i: $inner) -> Self { + Self::modify_live(1); + Self { _priv: i } + } + + fn modify_live(n: isize) -> isize { + static COUNTER: AtomicIsize = AtomicIsize::new(0); + COUNTER.fetch_add(n, Ordering::Relaxed) + n + } + + fn live_objects() -> isize { + Self::modify_live(0) + } + } + // Run the test statements + let _: () = { $($test_stmts)* }; + // Check that we didn't leak anything, or call drop too many times. + assert_eq!( + $Type::live_objects(), 0, + "Wrong number of drops of {}, `initializations - drops` should be 0.", + stringify!($Type), + ); + } + }; +} + +define_test! { + @test_name: align1zst; + struct Align1Zst(()); + + check_thin_sized(|| Align1Zst::new(())); + check_thin_dyn(|| Align1Zst::new(())); +} + +define_test! { + @test_name: align1small; + struct Align1Small(u8); + + check_thin_sized(|| Align1Small::new(50)); + check_thin_dyn(|| Align1Small::new(50)); +} + +define_test! { + @test_name: align1_size_not_pow2; + struct Align64NotPow2Size([u8; 79]); + + check_thin_sized(|| Align64NotPow2Size::new([100; 79])); + check_thin_dyn(|| Align64NotPow2Size::new([100; 79])); +} + +define_test! { + @test_name: align1big; + struct Align1Big([u8; 256]); + + check_thin_sized(|| Align1Big::new([5u8; 256])); + check_thin_dyn(|| Align1Big::new([5u8; 256])); +} + +// Note: `#[repr(align(2))]` is worth testing because +// - can have pointers which are misaligned, unlike align(1) +// - is still expected to have an alignment less than the alignment of a vtable. +define_test! { + @test_name: align2zst; + #[repr(align(2))] + struct Align2Zst(()); + + check_thin_sized(|| Align2Zst::new(())); + check_thin_dyn(|| Align2Zst::new(())); +} + +define_test! { + @test_name: align2small; + #[repr(align(2))] + struct Align2Small(u8); + + check_thin_sized(|| Align2Small::new(60)); + check_thin_dyn(|| Align2Small::new(60)); +} + +define_test! { + @test_name: align2full; + #[repr(align(2))] + struct Align2Full([u8; 2]); + check_thin_sized(|| Align2Full::new([3u8; 2])); + check_thin_dyn(|| Align2Full::new([3u8; 2])); +} + +define_test! { + @test_name: align2_size_not_pow2; + #[repr(align(2))] + struct Align2NotPower2Size([u8; 6]); + + check_thin_sized(|| Align2NotPower2Size::new([3; 6])); + check_thin_dyn(|| Align2NotPower2Size::new([3; 6])); +} + +define_test! { + @test_name: align2big; + #[repr(align(2))] + struct Align2Big([u8; 256]); + + check_thin_sized(|| Align2Big::new([5u8; 256])); + check_thin_dyn(|| Align2Big::new([5u8; 256])); +} + +define_test! { + @test_name: align64zst; + #[repr(align(64))] + struct Align64Zst(()); + + check_thin_sized(|| Align64Zst::new(())); + check_thin_dyn(|| Align64Zst::new(())); +} + +define_test! { + @test_name: align64small; + #[repr(align(64))] + struct Align64Small(u8); + + check_thin_sized(|| Align64Small::new(50)); + check_thin_dyn(|| Align64Small::new(50)); +} + +define_test! { + @test_name: align64med; + #[repr(align(64))] + struct Align64Med([u8; 64]); + check_thin_sized(|| Align64Med::new([10; 64])); + check_thin_dyn(|| Align64Med::new([10; 64])); +} + +define_test! { + @test_name: align64_size_not_pow2; + #[repr(align(64))] + struct Align64NotPow2Size([u8; 192]); + + check_thin_sized(|| Align64NotPow2Size::new([10; 192])); + check_thin_dyn(|| Align64NotPow2Size::new([10; 192])); +} + +define_test! { + @test_name: align64big; + #[repr(align(64))] + struct Align64Big([u8; 256]); + + check_thin_sized(|| Align64Big::new([10; 256])); + check_thin_dyn(|| Align64Big::new([10; 256])); +} diff --git a/crux-mir/lib/alloc/tests/vec.rs b/crux-mir/lib/alloc/tests/vec.rs index 9c4ac52ac..2f07c2911 100644 --- a/crux-mir/lib/alloc/tests/vec.rs +++ b/crux-mir/lib/alloc/tests/vec.rs @@ -1,9 +1,21 @@ +use core::alloc::{Allocator, Layout}; +use core::iter::IntoIterator; +use core::ptr::NonNull; +use std::alloc::System; +use std::assert_matches::assert_matches; use std::borrow::Cow; -use std::collections::TryReserveError::*; -use std::mem::size_of; +use std::cell::Cell; +use std::collections::TryReserveErrorKind::*; +use std::fmt::Debug; +use std::hint; +use std::iter::InPlaceIterable; +use std::mem; +use std::mem::{size_of, swap}; +use std::ops::Bound::*; use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::rc::Rc; +use std::sync::atomic::{AtomicU32, Ordering}; use std::vec::{Drain, IntoIter}; -use std::{isize, usize}; struct DropCounter<'a> { count: &'a mut u32, @@ -17,7 +29,7 @@ impl Drop for DropCounter<'_> { #[test] fn test_small_vec_struct() { - assert!(size_of::>() == size_of::() * 3); + assert_eq!(size_of::>(), size_of::() * 3); } #[test] @@ -69,7 +81,43 @@ fn test_reserve() { #[test] fn test_zst_capacity() { - assert_eq!(Vec::<()>::new().capacity(), usize::max_value()); + assert_eq!(Vec::<()>::new().capacity(), usize::MAX); +} + +#[test] +fn test_indexing() { + let v: Vec = vec![10, 20]; + assert_eq!(v[0], 10); + assert_eq!(v[1], 20); + let mut x: usize = 0; + assert_eq!(v[x], 10); + assert_eq!(v[x + 1], 20); + x = x + 1; + assert_eq!(v[x], 20); + assert_eq!(v[x - 1], 10); +} + +#[test] +fn test_debug_fmt() { + let vec1: Vec = vec![]; + assert_eq!("[]", format!("{:?}", vec1)); + + let vec2 = vec![0, 1]; + assert_eq!("[0, 1]", format!("{:?}", vec2)); + + let slice: &[isize] = &[4, 5]; + assert_eq!("[4, 5]", format!("{slice:?}")); +} + +#[test] +fn test_push() { + let mut v = vec![]; + v.push(1); + assert_eq!(v, [1]); + v.push(2); + assert_eq!(v, [1, 2]); + v.push(3); + assert_eq!(v, [1, 2, 3]); } #[test] @@ -117,6 +165,18 @@ fn test_extend() { assert_eq!(count_x, 1); } +#[test] +fn test_extend_from_slice() { + let a: Vec = vec![1, 2, 3, 4, 5]; + let b: Vec = vec![6, 7, 8, 9, 0]; + + let mut v: Vec = a; + + v.extend_from_slice(&b); + + assert_eq!(v, [1, 2, 3, 4, 5, 6, 7, 8, 9, 0]); +} + #[test] fn test_extend_ref() { let mut v = vec![1, 2]; @@ -133,18 +193,11 @@ fn test_extend_ref() { } #[test] -fn test_remove_item() { - let mut v = vec![1, 2, 3]; - v.remove_item(&1); - - assert_eq!(v.len(), 2); - assert_eq!(v, [2, 3]); +fn test_slice_from_ref() { + let values = vec![1, 2, 3, 4, 5]; + let slice = &values[1..3]; - let mut w = vec![1, 2, 3]; - w.remove_item(&4); - - assert_eq!(w.len(), 3); - w.remove_item(&4); + assert_eq!(slice, [2, 3]); } #[test] @@ -216,8 +269,8 @@ fn test_clone() { #[test] fn test_clone_from() { let mut v = vec![]; - let three: Vec> = vec![box 1, box 2, box 3]; - let two: Vec> = vec![box 4, box 5]; + let three: Vec> = vec![Box::new(1), Box::new(2), Box::new(3)]; + let two: Vec> = vec![Box::new(4), Box::new(5)]; // zero, long v.clone_from(&three); assert_eq!(v, three); @@ -242,6 +295,112 @@ fn test_retain() { assert_eq!(vec, [2, 4]); } +#[test] +fn test_retain_predicate_order() { + for to_keep in [true, false] { + let mut number_of_executions = 0; + let mut vec = vec![1, 2, 3, 4]; + let mut next_expected = 1; + vec.retain(|&x| { + assert_eq!(next_expected, x); + next_expected += 1; + number_of_executions += 1; + to_keep + }); + assert_eq!(number_of_executions, 4); + } +} + +#[test] +fn test_retain_pred_panic_with_hole() { + let v = (0..5).map(Rc::new).collect::>(); + catch_unwind(AssertUnwindSafe(|| { + let mut v = v.clone(); + v.retain(|r| match **r { + 0 => true, + 1 => false, + 2 => true, + _ => panic!(), + }); + })) + .unwrap_err(); + // Everything is dropped when predicate panicked. + assert!(v.iter().all(|r| Rc::strong_count(r) == 1)); +} + +#[test] +fn test_retain_pred_panic_no_hole() { + let v = (0..5).map(Rc::new).collect::>(); + catch_unwind(AssertUnwindSafe(|| { + let mut v = v.clone(); + v.retain(|r| match **r { + 0 | 1 | 2 => true, + _ => panic!(), + }); + })) + .unwrap_err(); + // Everything is dropped when predicate panicked. + assert!(v.iter().all(|r| Rc::strong_count(r) == 1)); +} + +#[test] +fn test_retain_drop_panic() { + struct Wrap(Rc); + + impl Drop for Wrap { + fn drop(&mut self) { + if *self.0 == 3 { + panic!(); + } + } + } + + let v = (0..5).map(|x| Rc::new(x)).collect::>(); + catch_unwind(AssertUnwindSafe(|| { + let mut v = v.iter().map(|r| Wrap(r.clone())).collect::>(); + v.retain(|w| match *w.0 { + 0 => true, + 1 => false, + 2 => true, + 3 => false, // Drop panic. + _ => true, + }); + })) + .unwrap_err(); + // Other elements are dropped when `drop` of one element panicked. + // The panicked wrapper also has its Rc dropped. + assert!(v.iter().all(|r| Rc::strong_count(r) == 1)); +} + +#[test] +fn test_retain_maybeuninits() { + // This test aimed to be run under miri. + use core::mem::MaybeUninit; + let mut vec: Vec<_> = [1i32, 2, 3, 4].map(|v| MaybeUninit::new(vec![v])).into(); + vec.retain(|x| { + // SAFETY: Retain must visit every element of Vec in original order and exactly once. + // Our values is initialized at creation of Vec. + let v = unsafe { x.assume_init_ref()[0] }; + if v & 1 == 0 { + return true; + } + // SAFETY: Value is initialized. + // Value wouldn't be dropped by `Vec::retain` + // because `MaybeUninit` doesn't drop content. + drop(unsafe { x.assume_init_read() }); + false + }); + let vec: Vec = vec + .into_iter() + .map(|x| unsafe { + // SAFETY: All values dropped in retain predicate must be removed by `Vec::retain`. + // Remaining values are initialized. + x.assume_init()[0] + }) + .collect(); + assert_eq!(vec, [2, 4]); +} + #[test] fn test_dedup() { fn case(a: Vec, b: Vec) { @@ -296,11 +455,11 @@ fn test_dedup_by() { #[test] fn test_dedup_unique() { - let mut v0: Vec> = vec![box 1, box 1, box 2, box 3]; + let mut v0: Vec> = vec![Box::new(1), Box::new(1), Box::new(2), Box::new(3)]; v0.dedup(); - let mut v1: Vec> = vec![box 1, box 2, box 2, box 3]; + let mut v1: Vec> = vec![Box::new(1), Box::new(2), Box::new(2), Box::new(3)]; v1.dedup(); - let mut v2: Vec> = vec![box 1, box 2, box 3, box 3]; + let mut v2: Vec> = vec![Box::new(1), Box::new(2), Box::new(3), Box::new(3)]; v2.dedup(); // If the boxed pointers were leaked or otherwise misused, valgrind // and/or rt should raise errors. @@ -341,10 +500,10 @@ fn zero_sized_values() { #[test] fn test_partition() { - assert_eq!(vec![].into_iter().partition(|x: &i32| *x < 3), (vec![], vec![])); - assert_eq!(vec![1, 2, 3].into_iter().partition(|x| *x < 4), (vec![1, 2, 3], vec![])); - assert_eq!(vec![1, 2, 3].into_iter().partition(|x| *x < 2), (vec![1], vec![2, 3])); - assert_eq!(vec![1, 2, 3].into_iter().partition(|x| *x < 0), (vec![], vec![1, 2, 3])); + assert_eq!([].into_iter().partition(|x: &i32| *x < 3), (vec![], vec![])); + assert_eq!([1, 2, 3].into_iter().partition(|x| *x < 4), (vec![1, 2, 3], vec![])); + assert_eq!([1, 2, 3].into_iter().partition(|x| *x < 2), (vec![1], vec![2, 3])); + assert_eq!([1, 2, 3].into_iter().partition(|x| *x < 0), (vec![], vec![1, 2, 3])); } #[test] @@ -358,6 +517,29 @@ fn test_zip_unzip() { assert_eq!((3, 6), (left[2], right[2])); } +#[test] +fn test_cmp() { + let x: &[isize] = &[1, 2, 3, 4, 5]; + let cmp: &[isize] = &[1, 2, 3, 4, 5]; + assert_eq!(&x[..], cmp); + let cmp: &[isize] = &[3, 4, 5]; + assert_eq!(&x[2..], cmp); + let cmp: &[isize] = &[1, 2, 3]; + assert_eq!(&x[..3], cmp); + let cmp: &[isize] = &[2, 3, 4]; + assert_eq!(&x[1..4], cmp); + + let x: Vec = vec![1, 2, 3, 4, 5]; + let cmp: &[isize] = &[1, 2, 3, 4, 5]; + assert_eq!(&x[..], cmp); + let cmp: &[isize] = &[3, 4, 5]; + assert_eq!(&x[2..], cmp); + let cmp: &[isize] = &[1, 2, 3]; + assert_eq!(&x[..3], cmp); + let cmp: &[isize] = &[2, 3, 4]; + assert_eq!(&x[1..4], cmp); +} + #[test] fn test_vec_truncate_drop() { static mut DROPS: u32 = 0; @@ -412,35 +594,35 @@ fn test_index_out_of_bounds() { #[should_panic] fn test_slice_out_of_bounds_1() { let x = vec![1, 2, 3, 4, 5]; - &x[!0..]; + let _ = &x[!0..]; } #[test] #[should_panic] fn test_slice_out_of_bounds_2() { let x = vec![1, 2, 3, 4, 5]; - &x[..6]; + let _ = &x[..6]; } #[test] #[should_panic] fn test_slice_out_of_bounds_3() { let x = vec![1, 2, 3, 4, 5]; - &x[!0..4]; + let _ = &x[!0..4]; } #[test] #[should_panic] fn test_slice_out_of_bounds_4() { let x = vec![1, 2, 3, 4, 5]; - &x[1..6]; + let _ = &x[1..6]; } #[test] #[should_panic] fn test_slice_out_of_bounds_5() { let x = vec![1, 2, 3, 4, 5]; - &x[3..2]; + let _ = &x[3..2]; } #[test] @@ -480,6 +662,17 @@ fn test_move_items_zero_sized() { assert_eq!(vec2, [(), (), ()]); } +#[test] +fn test_drain_empty_vec() { + let mut vec: Vec = vec![]; + let mut vec2: Vec = vec![]; + for i in vec.drain(..) { + vec2.push(i); + } + assert!(vec.is_empty()); + assert!(vec2.is_empty()); +} + #[test] fn test_drain_items() { let mut vec = vec![1, 2, 3]; @@ -564,19 +757,29 @@ fn test_drain_inclusive_range() { #[test] fn test_drain_max_vec_size() { - let mut v = Vec::<()>::with_capacity(usize::max_value()); + let mut v = Vec::<()>::with_capacity(usize::MAX); unsafe { - v.set_len(usize::max_value()); + v.set_len(usize::MAX); } - for _ in v.drain(usize::max_value() - 1..) {} - assert_eq!(v.len(), usize::max_value() - 1); + for _ in v.drain(usize::MAX - 1..) {} + assert_eq!(v.len(), usize::MAX - 1); - let mut v = Vec::<()>::with_capacity(usize::max_value()); + let mut v = Vec::<()>::with_capacity(usize::MAX); unsafe { - v.set_len(usize::max_value()); + v.set_len(usize::MAX); } - for _ in v.drain(usize::max_value() - 1..=usize::max_value() - 1) {} - assert_eq!(v.len(), usize::max_value() - 1); + for _ in v.drain(usize::MAX - 1..=usize::MAX - 1) {} + assert_eq!(v.len(), usize::MAX - 1); +} + +#[test] +#[should_panic] +fn test_drain_index_overflow() { + let mut v = Vec::<()>::with_capacity(usize::MAX); + unsafe { + v.set_len(usize::MAX); + } + v.drain(0..=usize::MAX); } #[test] @@ -586,6 +789,20 @@ fn test_drain_inclusive_out_of_bounds() { v.drain(5..=5); } +#[test] +#[should_panic] +fn test_drain_start_overflow() { + let mut v = vec![1, 2, 3]; + v.drain((Excluded(usize::MAX), Included(0))); +} + +#[test] +#[should_panic] +fn test_drain_end_overflow() { + let mut v = vec![1, 2, 3]; + v.drain((Included(0), Included(usize::MAX))); +} + #[test] fn test_drain_leak() { static mut DROPS: i32 = 0; @@ -624,11 +841,41 @@ fn test_drain_leak() { assert_eq!(v, vec![D(0, false), D(1, false), D(6, false),]); } +#[test] +fn test_drain_keep_rest() { + let mut v = vec![0, 1, 2, 3, 4, 5, 6]; + let mut drain = v.drain(1..6); + assert_eq!(drain.next(), Some(1)); + assert_eq!(drain.next_back(), Some(5)); + assert_eq!(drain.next(), Some(2)); + + drain.keep_rest(); + assert_eq!(v, &[0, 3, 4, 6]); +} + +#[test] +fn test_drain_keep_rest_all() { + let mut v = vec![0, 1, 2, 3, 4, 5, 6]; + v.drain(1..6).keep_rest(); + assert_eq!(v, &[0, 1, 2, 3, 4, 5, 6]); +} + +#[test] +fn test_drain_keep_rest_none() { + let mut v = vec![0, 1, 2, 3, 4, 5, 6]; + let mut drain = v.drain(1..6); + + drain.by_ref().for_each(drop); + + drain.keep_rest(); + assert_eq!(v, &[0, 6]); +} + #[test] fn test_splice() { let mut v = vec![1, 2, 3, 4, 5]; let a = [10, 11, 12]; - v.splice(2..4, a.iter().cloned()); + v.splice(2..4, a); assert_eq!(v, &[1, 2, 10, 11, 12, 5]); v.splice(1..3, Some(20)); assert_eq!(v, &[1, 20, 11, 12, 5]); @@ -638,7 +885,7 @@ fn test_splice() { fn test_splice_inclusive_range() { let mut v = vec![1, 2, 3, 4, 5]; let a = [10, 11, 12]; - let t1: Vec<_> = v.splice(2..=3, a.iter().cloned()).collect(); + let t1: Vec<_> = v.splice(2..=3, a).collect(); assert_eq!(v, &[1, 2, 10, 11, 12, 5]); assert_eq!(t1, &[3, 4]); let t2: Vec<_> = v.splice(1..=2, Some(20)).collect(); @@ -651,7 +898,7 @@ fn test_splice_inclusive_range() { fn test_splice_out_of_bounds() { let mut v = vec![1, 2, 3, 4, 5]; let a = [10, 11, 12]; - v.splice(5..6, a.iter().cloned()); + v.splice(5..6, a); } #[test] @@ -659,7 +906,7 @@ fn test_splice_out_of_bounds() { fn test_splice_inclusive_out_of_bounds() { let mut v = vec![1, 2, 3, 4, 5]; let a = [10, 11, 12]; - v.splice(5..=5, a.iter().cloned()); + v.splice(5..=5, a); } #[test] @@ -683,7 +930,7 @@ fn test_splice_unbounded() { fn test_splice_forget() { let mut v = vec![1, 2, 3, 4, 5]; let a = [10, 11, 12]; - std::mem::forget(v.splice(2..4, a.iter().cloned())); + std::mem::forget(v.splice(2..4, a)); assert_eq!(v, &[1, 2]); } @@ -706,9 +953,23 @@ fn test_append() { #[test] fn test_split_off() { let mut vec = vec![1, 2, 3, 4, 5, 6]; + let orig_capacity = vec.capacity(); let vec2 = vec.split_off(4); assert_eq!(vec, [1, 2, 3, 4]); assert_eq!(vec2, [5, 6]); + assert_eq!(vec.capacity(), orig_capacity); +} + +#[test] +fn test_split_off_take_all() { + let mut vec = vec![1, 2, 3, 4, 5, 6]; + let orig_ptr = vec.as_ptr(); + let orig_capacity = vec.capacity(); + let vec2 = vec.split_off(0); + assert_eq!(vec, []); + assert_eq!(vec2, [1, 2, 3, 4, 5, 6]); + assert_eq!(vec.capacity(), orig_capacity); + assert_eq!(vec2.as_ptr(), orig_ptr); } #[test] @@ -738,13 +999,22 @@ fn test_into_iter_as_mut_slice() { fn test_into_iter_debug() { let vec = vec!['a', 'b', 'c']; let into_iter = vec.into_iter(); - let debug = format!("{:?}", into_iter); + let debug = format!("{into_iter:?}"); assert_eq!(debug, "IntoIter(['a', 'b', 'c'])"); } #[test] fn test_into_iter_count() { - assert_eq!(vec![1, 2, 3].into_iter().count(), 3); + assert_eq!([1, 2, 3].into_iter().count(), 3); +} + +#[test] +fn test_into_iter_next_chunk() { + let mut iter = b"lorem".to_vec().into_iter(); + + assert_eq!(iter.next_chunk().unwrap(), [b'l', b'o']); // N is inferred as 2 + assert_eq!(iter.next_chunk().unwrap(), [b'r', b'e', b'm']); // N is inferred as 3 + assert_eq!(iter.next_chunk::<4>().unwrap_err().as_slice(), &[]); // N is explicitly 4 } #[test] @@ -753,7 +1023,7 @@ fn test_into_iter_clone() { let v: Vec = it.collect(); assert_eq!(&v[..], slice); } - let mut it = vec![1, 2, 3].into_iter(); + let mut it = [1, 2, 3].into_iter(); iter_equal(it.clone(), &[1, 2, 3]); assert_eq!(it.next(), Some(1)); let mut it = it.rev(); @@ -790,6 +1060,224 @@ fn test_into_iter_leak() { assert_eq!(unsafe { DROPS }, 3); } +#[test] +fn test_into_iter_advance_by() { + let mut i = [1, 2, 3, 4, 5].into_iter(); + i.advance_by(0).unwrap(); + i.advance_back_by(0).unwrap(); + assert_eq!(i.as_slice(), [1, 2, 3, 4, 5]); + + i.advance_by(1).unwrap(); + i.advance_back_by(1).unwrap(); + assert_eq!(i.as_slice(), [2, 3, 4]); + + assert_eq!(i.advance_back_by(usize::MAX), Err(3)); + + assert_eq!(i.advance_by(usize::MAX), Err(0)); + + i.advance_by(0).unwrap(); + i.advance_back_by(0).unwrap(); + + assert_eq!(i.len(), 0); +} + +#[test] +fn test_into_iter_drop_allocator() { + struct ReferenceCountedAllocator<'a>(DropCounter<'a>); + + unsafe impl Allocator for ReferenceCountedAllocator<'_> { + fn allocate(&self, layout: Layout) -> Result, core::alloc::AllocError> { + System.allocate(layout) + } + + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + // Safety: Invariants passed to caller. + unsafe { System.deallocate(ptr, layout) } + } + } + + let mut drop_count = 0; + + let allocator = ReferenceCountedAllocator(DropCounter { count: &mut drop_count }); + let _ = Vec::::new_in(allocator); + assert_eq!(drop_count, 1); + + let allocator = ReferenceCountedAllocator(DropCounter { count: &mut drop_count }); + let _ = Vec::::new_in(allocator).into_iter(); + assert_eq!(drop_count, 2); +} + +#[test] +fn test_into_iter_zst() { + #[derive(Debug, Clone)] + struct AlignedZstWithDrop([u64; 0]); + impl Drop for AlignedZstWithDrop { + fn drop(&mut self) { + let addr = self as *mut _ as usize; + assert!(hint::black_box(addr) % mem::align_of::() == 0); + } + } + + const C: AlignedZstWithDrop = AlignedZstWithDrop([0u64; 0]); + + for _ in vec![C].into_iter() {} + for _ in vec![C; 5].into_iter().rev() {} + + let mut it = vec![C, C].into_iter(); + it.advance_by(1).unwrap(); + drop(it); + + let mut it = vec![C, C].into_iter(); + it.next_chunk::<1>().unwrap(); + drop(it); + + let mut it = vec![C, C].into_iter(); + it.next_chunk::<4>().unwrap_err(); + drop(it); +} + +#[test] +fn test_from_iter_specialization() { + let src: Vec = vec![0usize; 1]; + let srcptr = src.as_ptr(); + let sink = src.into_iter().collect::>(); + let sinkptr = sink.as_ptr(); + assert_eq!(srcptr, sinkptr); +} + +#[test] +fn test_from_iter_partially_drained_in_place_specialization() { + let src: Vec = vec![0usize; 10]; + let srcptr = src.as_ptr(); + let mut iter = src.into_iter(); + iter.next(); + iter.next(); + let sink = iter.collect::>(); + let sinkptr = sink.as_ptr(); + assert_eq!(srcptr, sinkptr); +} + +#[test] +fn test_from_iter_specialization_with_iterator_adapters() { + fn assert_in_place_trait(_: &T) {} + let src: Vec = vec![0usize; 256]; + let srcptr = src.as_ptr(); + let iter = src + .into_iter() + .enumerate() + .map(|i| i.0 + i.1) + .zip(std::iter::repeat(1usize)) + .map(|(a, b)| a + b) + .map_while(Option::Some) + .skip(1) + .map(|e| if e != usize::MAX { Ok(std::num::NonZeroUsize::new(e)) } else { Err(()) }); + assert_in_place_trait(&iter); + let sink = iter.collect::, _>>().unwrap(); + let sinkptr = sink.as_ptr(); + assert_eq!(srcptr, sinkptr as *const usize); +} + +#[test] +fn test_from_iter_specialization_head_tail_drop() { + let drop_count: Vec<_> = (0..=2).map(|_| Rc::new(())).collect(); + let src: Vec<_> = drop_count.iter().cloned().collect(); + let srcptr = src.as_ptr(); + let iter = src.into_iter(); + let sink: Vec<_> = iter.skip(1).take(1).collect(); + let sinkptr = sink.as_ptr(); + assert_eq!(srcptr, sinkptr, "specialization was applied"); + assert_eq!(Rc::strong_count(&drop_count[0]), 1, "front was dropped"); + assert_eq!(Rc::strong_count(&drop_count[1]), 2, "one element was collected"); + assert_eq!(Rc::strong_count(&drop_count[2]), 1, "tail was dropped"); + assert_eq!(sink.len(), 1); +} + +#[test] +fn test_from_iter_specialization_panic_during_iteration_drops() { + let drop_count: Vec<_> = (0..=2).map(|_| Rc::new(())).collect(); + let src: Vec<_> = drop_count.iter().cloned().collect(); + let iter = src.into_iter(); + + let _ = std::panic::catch_unwind(AssertUnwindSafe(|| { + let _ = iter + .enumerate() + .filter_map(|(i, e)| { + if i == 1 { + std::panic!("aborting iteration"); + } + Some(e) + }) + .collect::>(); + })); + + assert!( + drop_count.iter().map(Rc::strong_count).all(|count| count == 1), + "all items were dropped once" + ); +} + +#[test] +fn test_from_iter_specialization_panic_during_drop_doesnt_leak() { + static mut DROP_COUNTER_OLD: [usize; 5] = [0; 5]; + static mut DROP_COUNTER_NEW: [usize; 2] = [0; 2]; + + #[derive(Debug)] + struct Old(usize); + + impl Drop for Old { + fn drop(&mut self) { + unsafe { + DROP_COUNTER_OLD[self.0] += 1; + } + + if self.0 == 3 { + panic!(); + } + + println!("Dropped Old: {}", self.0); + } + } + + #[derive(Debug)] + struct New(usize); + + impl Drop for New { + fn drop(&mut self) { + unsafe { + DROP_COUNTER_NEW[self.0] += 1; + } + + println!("Dropped New: {}", self.0); + } + } + + let _ = std::panic::catch_unwind(AssertUnwindSafe(|| { + let v = vec![Old(0), Old(1), Old(2), Old(3), Old(4)]; + let _ = v.into_iter().map(|x| New(x.0)).take(2).collect::>(); + })); + + assert_eq!(unsafe { DROP_COUNTER_OLD[0] }, 1); + assert_eq!(unsafe { DROP_COUNTER_OLD[1] }, 1); + assert_eq!(unsafe { DROP_COUNTER_OLD[2] }, 1); + assert_eq!(unsafe { DROP_COUNTER_OLD[3] }, 1); + assert_eq!(unsafe { DROP_COUNTER_OLD[4] }, 1); + + assert_eq!(unsafe { DROP_COUNTER_NEW[0] }, 1); + assert_eq!(unsafe { DROP_COUNTER_NEW[1] }, 1); +} + +// regression test for issue #85322. Peekable previously implemented InPlaceIterable, +// but due to an interaction with IntoIter's current Clone implementation it failed to uphold +// the contract. +#[test] +fn test_collect_after_iterator_clone() { + let v = vec![0; 5]; + let mut i = v.into_iter().map(|i| i + 1).peekable(); + i.peek(); + let v = i.clone().collect::>(); + assert_eq!(v, [1, 1, 1, 1, 1]); + assert!(v.len() <= v.capacity()); +} #[test] fn test_cow_from() { let borrowed: &[_] = &["borrowed", "(slice)"]; @@ -1011,7 +1499,7 @@ fn drain_filter_consumed_panic() { struct Check { index: usize, drop_counts: Rc>>, - }; + } impl Drop for Check { fn drop(&mut self) { @@ -1063,7 +1551,7 @@ fn drain_filter_unconsumed_panic() { struct Check { index: usize, drop_counts: Rc>>, - }; + } impl Drop for Check { fn drop(&mut self) { @@ -1112,6 +1600,35 @@ fn drain_filter_unconsumed() { assert_eq!(vec, [2, 4]); } +#[test] +fn test_drain_filter_keep_rest() { + let mut v = vec![0, 1, 2, 3, 4, 5, 6]; + let mut drain = v.drain_filter(|&mut x| x % 2 == 0); + assert_eq!(drain.next(), Some(0)); + assert_eq!(drain.next(), Some(2)); + + drain.keep_rest(); + assert_eq!(v, &[1, 3, 4, 5, 6]); +} + +#[test] +fn test_drain_filter_keep_rest_all() { + let mut v = vec![0, 1, 2, 3, 4, 5, 6]; + v.drain_filter(|_| true).keep_rest(); + assert_eq!(v, &[0, 1, 2, 3, 4, 5, 6]); +} + +#[test] +fn test_drain_filter_keep_rest_none() { + let mut v = vec![0, 1, 2, 3, 4, 5, 6]; + let mut drain = v.drain_filter(|_| true); + + drain.by_ref().for_each(drop); + + drain.keep_rest(); + assert_eq!(v, &[]); +} + #[test] fn test_reserve_exact() { // This is all the same as test_reserve @@ -1138,6 +1655,7 @@ fn test_reserve_exact() { #[test] #[cfg_attr(miri, ignore)] // Miri does not support signalling OOM +#[cfg_attr(target_os = "android", ignore)] // Android used in CI has a broken dlmalloc fn test_try_reserve() { // These are the interesting cases: // * exactly isize::MAX should never trigger a CapacityOverflow (can be OOM) @@ -1150,111 +1668,90 @@ fn test_try_reserve() { const MAX_CAP: usize = isize::MAX as usize; const MAX_USIZE: usize = usize::MAX; - // On 16/32-bit, we check that allocations don't exceed isize::MAX, - // on 64-bit, we assume the OS will give an OOM for such a ridiculous size. - // Any platform that succeeds for these requests is technically broken with - // ptr::offset because LLVM is the worst. - let guards_against_isize = size_of::() < 8; - { // Note: basic stuff is checked by test_reserve let mut empty_bytes: Vec = Vec::new(); // Check isize::MAX doesn't count as an overflow - if let Err(CapacityOverflow) = empty_bytes.try_reserve(MAX_CAP) { + if let Err(CapacityOverflow) = empty_bytes.try_reserve(MAX_CAP).map_err(|e| e.kind()) { panic!("isize::MAX shouldn't trigger an overflow!"); } // Play it again, frank! (just to be sure) - if let Err(CapacityOverflow) = empty_bytes.try_reserve(MAX_CAP) { + if let Err(CapacityOverflow) = empty_bytes.try_reserve(MAX_CAP).map_err(|e| e.kind()) { panic!("isize::MAX shouldn't trigger an overflow!"); } - if guards_against_isize { - // Check isize::MAX + 1 does count as overflow - if let Err(CapacityOverflow) = empty_bytes.try_reserve(MAX_CAP + 1) { - } else { - panic!("isize::MAX + 1 should trigger an overflow!") - } - - // Check usize::MAX does count as overflow - if let Err(CapacityOverflow) = empty_bytes.try_reserve(MAX_USIZE) { - } else { - panic!("usize::MAX should trigger an overflow!") - } - } else { - // Check isize::MAX + 1 is an OOM - if let Err(AllocError { .. }) = empty_bytes.try_reserve(MAX_CAP + 1) { - } else { - panic!("isize::MAX + 1 should trigger an OOM!") - } - - // Check usize::MAX is an OOM - if let Err(AllocError { .. }) = empty_bytes.try_reserve(MAX_USIZE) { - } else { - panic!("usize::MAX should trigger an OOM!") - } - } + // Check isize::MAX + 1 does count as overflow + assert_matches!( + empty_bytes.try_reserve(MAX_CAP + 1).map_err(|e| e.kind()), + Err(CapacityOverflow), + "isize::MAX + 1 should trigger an overflow!" + ); + + // Check usize::MAX does count as overflow + assert_matches!( + empty_bytes.try_reserve(MAX_USIZE).map_err(|e| e.kind()), + Err(CapacityOverflow), + "usize::MAX should trigger an overflow!" + ); } { // Same basic idea, but with non-zero len let mut ten_bytes: Vec = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - if let Err(CapacityOverflow) = ten_bytes.try_reserve(MAX_CAP - 10) { + if let Err(CapacityOverflow) = ten_bytes.try_reserve(MAX_CAP - 10).map_err(|e| e.kind()) { panic!("isize::MAX shouldn't trigger an overflow!"); } - if let Err(CapacityOverflow) = ten_bytes.try_reserve(MAX_CAP - 10) { + if let Err(CapacityOverflow) = ten_bytes.try_reserve(MAX_CAP - 10).map_err(|e| e.kind()) { panic!("isize::MAX shouldn't trigger an overflow!"); } - if guards_against_isize { - if let Err(CapacityOverflow) = ten_bytes.try_reserve(MAX_CAP - 9) { - } else { - panic!("isize::MAX + 1 should trigger an overflow!"); - } - } else { - if let Err(AllocError { .. }) = ten_bytes.try_reserve(MAX_CAP - 9) { - } else { - panic!("isize::MAX + 1 should trigger an OOM!") - } - } + + assert_matches!( + ten_bytes.try_reserve(MAX_CAP - 9).map_err(|e| e.kind()), + Err(CapacityOverflow), + "isize::MAX + 1 should trigger an overflow!" + ); + // Should always overflow in the add-to-len - if let Err(CapacityOverflow) = ten_bytes.try_reserve(MAX_USIZE) { - } else { - panic!("usize::MAX should trigger an overflow!") - } + assert_matches!( + ten_bytes.try_reserve(MAX_USIZE).map_err(|e| e.kind()), + Err(CapacityOverflow), + "usize::MAX should trigger an overflow!" + ); } { // Same basic idea, but with interesting type size let mut ten_u32s: Vec = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - if let Err(CapacityOverflow) = ten_u32s.try_reserve(MAX_CAP / 4 - 10) { + if let Err(CapacityOverflow) = ten_u32s.try_reserve(MAX_CAP / 4 - 10).map_err(|e| e.kind()) + { panic!("isize::MAX shouldn't trigger an overflow!"); } - if let Err(CapacityOverflow) = ten_u32s.try_reserve(MAX_CAP / 4 - 10) { + if let Err(CapacityOverflow) = ten_u32s.try_reserve(MAX_CAP / 4 - 10).map_err(|e| e.kind()) + { panic!("isize::MAX shouldn't trigger an overflow!"); } - if guards_against_isize { - if let Err(CapacityOverflow) = ten_u32s.try_reserve(MAX_CAP / 4 - 9) { - } else { - panic!("isize::MAX + 1 should trigger an overflow!"); - } - } else { - if let Err(AllocError { .. }) = ten_u32s.try_reserve(MAX_CAP / 4 - 9) { - } else { - panic!("isize::MAX + 1 should trigger an OOM!") - } - } + + assert_matches!( + ten_u32s.try_reserve(MAX_CAP / 4 - 9).map_err(|e| e.kind()), + Err(CapacityOverflow), + "isize::MAX + 1 should trigger an overflow!" + ); + // Should fail in the mul-by-size - if let Err(CapacityOverflow) = ten_u32s.try_reserve(MAX_USIZE - 20) { - } else { - panic!("usize::MAX should trigger an overflow!"); - } + assert_matches!( + ten_u32s.try_reserve(MAX_USIZE - 20).map_err(|e| e.kind()), + Err(CapacityOverflow), + "usize::MAX should trigger an overflow!" + ); } } #[test] #[cfg_attr(miri, ignore)] // Miri does not support signalling OOM +#[cfg_attr(target_os = "android", ignore)] // Android used in CI has a broken dlmalloc fn test_try_reserve_exact() { // This is exactly the same as test_try_reserve with the method changed. // See that test for comments. @@ -1262,106 +1759,110 @@ fn test_try_reserve_exact() { const MAX_CAP: usize = isize::MAX as usize; const MAX_USIZE: usize = usize::MAX; - let guards_against_isize = size_of::() < 8; - { let mut empty_bytes: Vec = Vec::new(); - if let Err(CapacityOverflow) = empty_bytes.try_reserve_exact(MAX_CAP) { + if let Err(CapacityOverflow) = empty_bytes.try_reserve_exact(MAX_CAP).map_err(|e| e.kind()) + { panic!("isize::MAX shouldn't trigger an overflow!"); } - if let Err(CapacityOverflow) = empty_bytes.try_reserve_exact(MAX_CAP) { + if let Err(CapacityOverflow) = empty_bytes.try_reserve_exact(MAX_CAP).map_err(|e| e.kind()) + { panic!("isize::MAX shouldn't trigger an overflow!"); } - if guards_against_isize { - if let Err(CapacityOverflow) = empty_bytes.try_reserve_exact(MAX_CAP + 1) { - } else { - panic!("isize::MAX + 1 should trigger an overflow!") - } - - if let Err(CapacityOverflow) = empty_bytes.try_reserve_exact(MAX_USIZE) { - } else { - panic!("usize::MAX should trigger an overflow!") - } - } else { - if let Err(AllocError { .. }) = empty_bytes.try_reserve_exact(MAX_CAP + 1) { - } else { - panic!("isize::MAX + 1 should trigger an OOM!") - } - - if let Err(AllocError { .. }) = empty_bytes.try_reserve_exact(MAX_USIZE) { - } else { - panic!("usize::MAX should trigger an OOM!") - } - } + assert_matches!( + empty_bytes.try_reserve_exact(MAX_CAP + 1).map_err(|e| e.kind()), + Err(CapacityOverflow), + "isize::MAX + 1 should trigger an overflow!" + ); + + assert_matches!( + empty_bytes.try_reserve_exact(MAX_USIZE).map_err(|e| e.kind()), + Err(CapacityOverflow), + "usize::MAX should trigger an overflow!" + ); } { let mut ten_bytes: Vec = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - if let Err(CapacityOverflow) = ten_bytes.try_reserve_exact(MAX_CAP - 10) { + if let Err(CapacityOverflow) = + ten_bytes.try_reserve_exact(MAX_CAP - 10).map_err(|e| e.kind()) + { panic!("isize::MAX shouldn't trigger an overflow!"); } - if let Err(CapacityOverflow) = ten_bytes.try_reserve_exact(MAX_CAP - 10) { + if let Err(CapacityOverflow) = + ten_bytes.try_reserve_exact(MAX_CAP - 10).map_err(|e| e.kind()) + { panic!("isize::MAX shouldn't trigger an overflow!"); } - if guards_against_isize { - if let Err(CapacityOverflow) = ten_bytes.try_reserve_exact(MAX_CAP - 9) { - } else { - panic!("isize::MAX + 1 should trigger an overflow!"); - } - } else { - if let Err(AllocError { .. }) = ten_bytes.try_reserve_exact(MAX_CAP - 9) { - } else { - panic!("isize::MAX + 1 should trigger an OOM!") - } - } - if let Err(CapacityOverflow) = ten_bytes.try_reserve_exact(MAX_USIZE) { - } else { - panic!("usize::MAX should trigger an overflow!") - } + + assert_matches!( + ten_bytes.try_reserve_exact(MAX_CAP - 9).map_err(|e| e.kind()), + Err(CapacityOverflow), + "isize::MAX + 1 should trigger an overflow!" + ); + + assert_matches!( + ten_bytes.try_reserve_exact(MAX_USIZE).map_err(|e| e.kind()), + Err(CapacityOverflow), + "usize::MAX should trigger an overflow!" + ); } { let mut ten_u32s: Vec = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - if let Err(CapacityOverflow) = ten_u32s.try_reserve_exact(MAX_CAP / 4 - 10) { + if let Err(CapacityOverflow) = + ten_u32s.try_reserve_exact(MAX_CAP / 4 - 10).map_err(|e| e.kind()) + { panic!("isize::MAX shouldn't trigger an overflow!"); } - if let Err(CapacityOverflow) = ten_u32s.try_reserve_exact(MAX_CAP / 4 - 10) { + if let Err(CapacityOverflow) = + ten_u32s.try_reserve_exact(MAX_CAP / 4 - 10).map_err(|e| e.kind()) + { panic!("isize::MAX shouldn't trigger an overflow!"); } - if guards_against_isize { - if let Err(CapacityOverflow) = ten_u32s.try_reserve_exact(MAX_CAP / 4 - 9) { - } else { - panic!("isize::MAX + 1 should trigger an overflow!"); - } - } else { - if let Err(AllocError { .. }) = ten_u32s.try_reserve_exact(MAX_CAP / 4 - 9) { - } else { - panic!("isize::MAX + 1 should trigger an OOM!") - } - } - if let Err(CapacityOverflow) = ten_u32s.try_reserve_exact(MAX_USIZE - 20) { - } else { - panic!("usize::MAX should trigger an overflow!") - } + + assert_matches!( + ten_u32s.try_reserve_exact(MAX_CAP / 4 - 9).map_err(|e| e.kind()), + Err(CapacityOverflow), + "isize::MAX + 1 should trigger an overflow!" + ); + + assert_matches!( + ten_u32s.try_reserve_exact(MAX_USIZE - 20).map_err(|e| e.kind()), + Err(CapacityOverflow), + "usize::MAX should trigger an overflow!" + ); } } #[test] -fn test_stable_push_pop() { +fn test_stable_pointers() { + /// Pull an element from the iterator, then drop it. + /// Useful to cover both the `next` and `drop` paths of an iterator. + fn next_then_drop(mut i: I) { + i.next().unwrap(); + drop(i); + } + // Test that, if we reserved enough space, adding and removing elements does not - // invalidate references into the vector (such as `v0`). This test also + // invalidate references into the vector (such as `v0`). This test also // runs in Miri, which would detect such problems. - let mut v = Vec::with_capacity(10); + // Note that this test does *not* constitute a stable guarantee that all these functions do not + // reallocate! Only what is explicitly documented at + // is stably guaranteed. + let mut v = Vec::with_capacity(128); v.push(13); - // laundering the lifetime -- we take care that `v` does not reallocate, so that's okay. - let v0 = unsafe { &*(&v[0] as *const _) }; - + // Laundering the lifetime -- we take care that `v` does not reallocate, so that's okay. + let v0 = &mut v[0]; + let v0 = unsafe { &mut *(v0 as *mut _) }; // Now do a bunch of things and occasionally use `v0` again to assert it is still valid. + + // Pushing/inserting and popping/removing v.push(1); v.push(2); v.insert(1, 1); @@ -1369,6 +1870,62 @@ fn test_stable_push_pop() { v.remove(1); v.pop().unwrap(); assert_eq!(*v0, 13); + v.push(1); + v.swap_remove(1); + assert_eq!(v.len(), 2); + v.swap_remove(1); // swap_remove the last element + assert_eq!(*v0, 13); + + // Appending + v.append(&mut vec![27, 19]); + assert_eq!(*v0, 13); + + // Extending + v.extend_from_slice(&[1, 2]); + v.extend(&[1, 2]); // `slice::Iter` (with `T: Copy`) specialization + v.extend(vec![2, 3]); // `vec::IntoIter` specialization + v.extend(std::iter::once(3)); // `TrustedLen` specialization + v.extend(std::iter::empty::()); // `TrustedLen` specialization with empty iterator + v.extend(std::iter::once(3).filter(|_| true)); // base case + v.extend(std::iter::once(&3)); // `cloned` specialization + assert_eq!(*v0, 13); + + // Truncation + v.truncate(2); + assert_eq!(*v0, 13); + + // Resizing + v.resize_with(v.len() + 10, || 42); + assert_eq!(*v0, 13); + v.resize_with(2, || panic!()); + assert_eq!(*v0, 13); + + // No-op reservation + v.reserve(32); + v.reserve_exact(32); + assert_eq!(*v0, 13); + + // Partial draining + v.resize_with(10, || 42); + next_then_drop(v.drain(5..)); + assert_eq!(*v0, 13); + + // Splicing + v.resize_with(10, || 42); + next_then_drop(v.splice(5.., vec![1, 2, 3, 4, 5])); // empty tail after range + assert_eq!(*v0, 13); + next_then_drop(v.splice(5..8, vec![1])); // replacement is smaller than original range + assert_eq!(*v0, 13); + next_then_drop(v.splice(5..6, [1; 10].into_iter().filter(|_| true))); // lower bound not exact + assert_eq!(*v0, 13); + + // spare_capacity_mut + v.spare_capacity_mut(); + assert_eq!(*v0, 13); + + // Smoke test that would fire even outside Miri if an actual relocation happened. + *v0 -= 13; + assert_eq!(v[0], 0); } // https://github.com/rust-lang/rust/pull/49496 introduced specialization based on: @@ -1411,3 +1968,548 @@ fn vec_macro_repeating_null_raw_fat_pointer() { vtable: *mut (), } } + +// This test will likely fail if you change the capacities used in +// `RawVec::grow_amortized`. +#[test] +fn test_push_growth_strategy() { + // If the element size is 1, we jump from 0 to 8, then double. + { + let mut v1: Vec = vec![]; + assert_eq!(v1.capacity(), 0); + + for _ in 0..8 { + v1.push(0); + assert_eq!(v1.capacity(), 8); + } + + for _ in 8..16 { + v1.push(0); + assert_eq!(v1.capacity(), 16); + } + + for _ in 16..32 { + v1.push(0); + assert_eq!(v1.capacity(), 32); + } + + for _ in 32..64 { + v1.push(0); + assert_eq!(v1.capacity(), 64); + } + } + + // If the element size is 2..=1024, we jump from 0 to 4, then double. + { + let mut v2: Vec = vec![]; + let mut v1024: Vec<[u8; 1024]> = vec![]; + assert_eq!(v2.capacity(), 0); + assert_eq!(v1024.capacity(), 0); + + for _ in 0..4 { + v2.push(0); + v1024.push([0; 1024]); + assert_eq!(v2.capacity(), 4); + assert_eq!(v1024.capacity(), 4); + } + + for _ in 4..8 { + v2.push(0); + v1024.push([0; 1024]); + assert_eq!(v2.capacity(), 8); + assert_eq!(v1024.capacity(), 8); + } + + for _ in 8..16 { + v2.push(0); + v1024.push([0; 1024]); + assert_eq!(v2.capacity(), 16); + assert_eq!(v1024.capacity(), 16); + } + + for _ in 16..32 { + v2.push(0); + v1024.push([0; 1024]); + assert_eq!(v2.capacity(), 32); + assert_eq!(v1024.capacity(), 32); + } + + for _ in 32..64 { + v2.push(0); + v1024.push([0; 1024]); + assert_eq!(v2.capacity(), 64); + assert_eq!(v1024.capacity(), 64); + } + } + + // If the element size is > 1024, we jump from 0 to 1, then double. + { + let mut v1025: Vec<[u8; 1025]> = vec![]; + assert_eq!(v1025.capacity(), 0); + + for _ in 0..1 { + v1025.push([0; 1025]); + assert_eq!(v1025.capacity(), 1); + } + + for _ in 1..2 { + v1025.push([0; 1025]); + assert_eq!(v1025.capacity(), 2); + } + + for _ in 2..4 { + v1025.push([0; 1025]); + assert_eq!(v1025.capacity(), 4); + } + + for _ in 4..8 { + v1025.push([0; 1025]); + assert_eq!(v1025.capacity(), 8); + } + + for _ in 8..16 { + v1025.push([0; 1025]); + assert_eq!(v1025.capacity(), 16); + } + + for _ in 16..32 { + v1025.push([0; 1025]); + assert_eq!(v1025.capacity(), 32); + } + + for _ in 32..64 { + v1025.push([0; 1025]); + assert_eq!(v1025.capacity(), 64); + } + } +} + +macro_rules! generate_assert_eq_vec_and_prim { + ($name:ident<$B:ident>($type:ty)) => { + fn $name + Debug, $B: Debug>(a: Vec, b: $type) { + assert!(a == b); + assert_eq!(a, b); + } + }; +} + +generate_assert_eq_vec_and_prim! { assert_eq_vec_and_slice (&[B]) } +generate_assert_eq_vec_and_prim! { assert_eq_vec_and_array_3([B; 3]) } + +#[test] +fn partialeq_vec_and_prim() { + assert_eq_vec_and_slice(vec![1, 2, 3], &[1, 2, 3]); + assert_eq_vec_and_array_3(vec![1, 2, 3], [1, 2, 3]); +} + +macro_rules! assert_partial_eq_valid { + ($a2:expr, $a3:expr; $b2:expr, $b3: expr) => { + assert!($a2 == $b2); + assert!($a2 != $b3); + assert!($a3 != $b2); + assert!($a3 == $b3); + assert_eq!($a2, $b2); + assert_ne!($a2, $b3); + assert_ne!($a3, $b2); + assert_eq!($a3, $b3); + }; +} + +#[test] +fn partialeq_vec_full() { + let vec2: Vec<_> = vec![1, 2]; + let vec3: Vec<_> = vec![1, 2, 3]; + let slice2: &[_] = &[1, 2]; + let slice3: &[_] = &[1, 2, 3]; + let slicemut2: &[_] = &mut [1, 2]; + let slicemut3: &[_] = &mut [1, 2, 3]; + let array2: [_; 2] = [1, 2]; + let array3: [_; 3] = [1, 2, 3]; + let arrayref2: &[_; 2] = &[1, 2]; + let arrayref3: &[_; 3] = &[1, 2, 3]; + + assert_partial_eq_valid!(vec2,vec3; vec2,vec3); + assert_partial_eq_valid!(vec2,vec3; slice2,slice3); + assert_partial_eq_valid!(vec2,vec3; slicemut2,slicemut3); + assert_partial_eq_valid!(slice2,slice3; vec2,vec3); + assert_partial_eq_valid!(slicemut2,slicemut3; vec2,vec3); + assert_partial_eq_valid!(vec2,vec3; array2,array3); + assert_partial_eq_valid!(vec2,vec3; arrayref2,arrayref3); + assert_partial_eq_valid!(vec2,vec3; arrayref2[..],arrayref3[..]); +} + +#[test] +fn test_vec_cycle() { + #[derive(Debug)] + struct C<'a> { + v: Vec>>>, + } + + impl<'a> C<'a> { + fn new() -> C<'a> { + C { v: Vec::new() } + } + } + + let mut c1 = C::new(); + let mut c2 = C::new(); + let mut c3 = C::new(); + + // Push + c1.v.push(Cell::new(None)); + c1.v.push(Cell::new(None)); + + c2.v.push(Cell::new(None)); + c2.v.push(Cell::new(None)); + + c3.v.push(Cell::new(None)); + c3.v.push(Cell::new(None)); + + // Set + c1.v[0].set(Some(&c2)); + c1.v[1].set(Some(&c3)); + + c2.v[0].set(Some(&c2)); + c2.v[1].set(Some(&c3)); + + c3.v[0].set(Some(&c1)); + c3.v[1].set(Some(&c2)); +} + +#[test] +fn test_vec_cycle_wrapped() { + struct Refs<'a> { + v: Vec>>>, + } + + struct C<'a> { + refs: Refs<'a>, + } + + impl<'a> Refs<'a> { + fn new() -> Refs<'a> { + Refs { v: Vec::new() } + } + } + + impl<'a> C<'a> { + fn new() -> C<'a> { + C { refs: Refs::new() } + } + } + + let mut c1 = C::new(); + let mut c2 = C::new(); + let mut c3 = C::new(); + + c1.refs.v.push(Cell::new(None)); + c1.refs.v.push(Cell::new(None)); + c2.refs.v.push(Cell::new(None)); + c2.refs.v.push(Cell::new(None)); + c3.refs.v.push(Cell::new(None)); + c3.refs.v.push(Cell::new(None)); + + c1.refs.v[0].set(Some(&c2)); + c1.refs.v[1].set(Some(&c3)); + c2.refs.v[0].set(Some(&c2)); + c2.refs.v[1].set(Some(&c3)); + c3.refs.v[0].set(Some(&c1)); + c3.refs.v[1].set(Some(&c2)); +} + +#[test] +fn test_zero_sized_capacity() { + for len in [0, 1, 2, 4, 8, 16, 32, 64, 128, 256] { + let v = Vec::<()>::with_capacity(len); + assert_eq!(v.len(), 0); + assert_eq!(v.capacity(), usize::MAX); + } +} + +#[test] +fn test_zero_sized_vec_push() { + const N: usize = 8; + + for len in 0..N { + let mut tester = Vec::with_capacity(len); + assert_eq!(tester.len(), 0); + assert!(tester.capacity() >= len); + for _ in 0..len { + tester.push(()); + } + assert_eq!(tester.len(), len); + assert_eq!(tester.iter().count(), len); + tester.clear(); + } +} + +#[test] +fn test_vec_macro_repeat() { + assert_eq!(vec![1; 3], vec![1, 1, 1]); + assert_eq!(vec![1; 2], vec![1, 1]); + assert_eq!(vec![1; 1], vec![1]); + assert_eq!(vec![1; 0], vec![]); + + // from_elem syntax (see RFC 832) + let el = Box::new(1); + let n = 3; + assert_eq!(vec![el; n], vec![Box::new(1), Box::new(1), Box::new(1)]); +} + +#[test] +fn test_vec_swap() { + let mut a: Vec = vec![0, 1, 2, 3, 4, 5, 6]; + a.swap(2, 4); + assert_eq!(a[2], 4); + assert_eq!(a[4], 2); + let mut n = 42; + swap(&mut n, &mut a[0]); + assert_eq!(a[0], 42); + assert_eq!(n, 0); +} + +#[test] +fn test_extend_from_within_spec() { + #[derive(Copy)] + struct CopyOnly; + + impl Clone for CopyOnly { + fn clone(&self) -> Self { + panic!("extend_from_within must use specialization on copy"); + } + } + + vec![CopyOnly, CopyOnly].extend_from_within(..); +} + +#[test] +fn test_extend_from_within_clone() { + let mut v = vec![String::from("sssss"), String::from("12334567890"), String::from("c")]; + v.extend_from_within(1..); + + assert_eq!(v, ["sssss", "12334567890", "c", "12334567890", "c"]); +} + +#[test] +fn test_extend_from_within_complete_rande() { + let mut v = vec![0, 1, 2, 3]; + v.extend_from_within(..); + + assert_eq!(v, [0, 1, 2, 3, 0, 1, 2, 3]); +} + +#[test] +fn test_extend_from_within_empty_rande() { + let mut v = vec![0, 1, 2, 3]; + v.extend_from_within(1..1); + + assert_eq!(v, [0, 1, 2, 3]); +} + +#[test] +#[should_panic] +fn test_extend_from_within_out_of_rande() { + let mut v = vec![0, 1]; + v.extend_from_within(..3); +} + +#[test] +fn test_extend_from_within_zst() { + let mut v = vec![(); 8]; + v.extend_from_within(3..7); + + assert_eq!(v, [(); 12]); +} + +#[test] +fn test_extend_from_within_empty_vec() { + let mut v = Vec::::new(); + v.extend_from_within(..); + + assert_eq!(v, []); +} + +#[test] +fn test_extend_from_within() { + let mut v = vec![String::from("a"), String::from("b"), String::from("c")]; + v.extend_from_within(1..=2); + v.extend_from_within(..=1); + + assert_eq!(v, ["a", "b", "c", "b", "c", "a", "b"]); +} + +#[test] +fn test_vec_dedup_by() { + let mut vec: Vec = vec![1, -1, 2, 3, 1, -5, 5, -2, 2]; + + vec.dedup_by(|a, b| a.abs() == b.abs()); + + assert_eq!(vec, [1, 2, 3, 1, -5, -2]); +} + +#[test] +fn test_vec_dedup_empty() { + let mut vec: Vec = Vec::new(); + + vec.dedup(); + + assert_eq!(vec, []); +} + +#[test] +fn test_vec_dedup_one() { + let mut vec = vec![12i32]; + + vec.dedup(); + + assert_eq!(vec, [12]); +} + +#[test] +fn test_vec_dedup_multiple_ident() { + let mut vec = vec![12, 12, 12, 12, 12, 11, 11, 11, 11, 11, 11]; + + vec.dedup(); + + assert_eq!(vec, [12, 11]); +} + +#[test] +fn test_vec_dedup_partialeq() { + #[derive(Debug)] + struct Foo(i32, i32); + + impl PartialEq for Foo { + fn eq(&self, other: &Foo) -> bool { + self.0 == other.0 + } + } + + let mut vec = vec![Foo(0, 1), Foo(0, 5), Foo(1, 7), Foo(1, 9)]; + + vec.dedup(); + assert_eq!(vec, [Foo(0, 1), Foo(1, 7)]); +} + +#[test] +fn test_vec_dedup() { + let mut vec: Vec = Vec::with_capacity(8); + let mut template = vec.clone(); + + for x in 0u8..255u8 { + vec.clear(); + template.clear(); + + let iter = (0..8).map(move |bit| (x >> bit) & 1 == 1); + vec.extend(iter); + template.extend_from_slice(&vec); + + let (dedup, _) = template.partition_dedup(); + vec.dedup(); + + assert_eq!(vec, dedup); + } +} + +#[test] +fn test_vec_dedup_panicking() { + #[derive(Debug)] + struct Panic<'a> { + drop_counter: &'a Cell, + value: bool, + index: usize, + } + + impl<'a> PartialEq for Panic<'a> { + fn eq(&self, other: &Self) -> bool { + self.value == other.value + } + } + + impl<'a> Drop for Panic<'a> { + fn drop(&mut self) { + self.drop_counter.set(self.drop_counter.get() + 1); + if !std::thread::panicking() { + assert!(self.index != 4); + } + } + } + + let drop_counter = &Cell::new(0); + let expected = [ + Panic { drop_counter, value: false, index: 0 }, + Panic { drop_counter, value: false, index: 5 }, + Panic { drop_counter, value: true, index: 6 }, + Panic { drop_counter, value: true, index: 7 }, + ]; + let mut vec = vec![ + Panic { drop_counter, value: false, index: 0 }, + // these elements get deduplicated + Panic { drop_counter, value: false, index: 1 }, + Panic { drop_counter, value: false, index: 2 }, + Panic { drop_counter, value: false, index: 3 }, + Panic { drop_counter, value: false, index: 4 }, + // here it panics while dropping the item with index==4 + Panic { drop_counter, value: false, index: 5 }, + Panic { drop_counter, value: true, index: 6 }, + Panic { drop_counter, value: true, index: 7 }, + ]; + + let _ = catch_unwind(AssertUnwindSafe(|| vec.dedup())).unwrap_err(); + + assert_eq!(drop_counter.get(), 4); + + let ok = vec.iter().zip(expected.iter()).all(|(x, y)| x.index == y.index); + + if !ok { + panic!("expected: {expected:?}\ngot: {vec:?}\n"); + } +} + +// Regression test for issue #82533 +#[test] +fn test_extend_from_within_panicing_clone() { + struct Panic<'dc> { + drop_count: &'dc AtomicU32, + aaaaa: bool, + } + + impl Clone for Panic<'_> { + fn clone(&self) -> Self { + if self.aaaaa { + panic!("panic! at the clone"); + } + + Self { ..*self } + } + } + + impl Drop for Panic<'_> { + fn drop(&mut self) { + self.drop_count.fetch_add(1, Ordering::SeqCst); + } + } + + let count = core::sync::atomic::AtomicU32::new(0); + let mut vec = vec![ + Panic { drop_count: &count, aaaaa: false }, + Panic { drop_count: &count, aaaaa: true }, + Panic { drop_count: &count, aaaaa: false }, + ]; + + // This should clone&append one Panic{..} at the end, and then panic while + // cloning second Panic{..}. This means that `Panic::drop` should be called + // 4 times (3 for items already in vector, 1 for just appended). + // + // Previously just appended item was leaked, making drop_count = 3, instead of 4. + std::panic::catch_unwind(move || vec.extend_from_within(..)).unwrap_err(); + + assert_eq!(count.load(Ordering::SeqCst), 4); +} + +#[test] +#[should_panic = "vec len overflow"] +fn test_into_flattened_size_overflow() { + let v = vec![[(); usize::MAX]; 2]; + let _ = v.into_flattened(); +} diff --git a/crux-mir/lib/alloc/tests/vec_deque.rs b/crux-mir/lib/alloc/tests/vec_deque.rs index 101dd67d9..0b8f5281b 100644 --- a/crux-mir/lib/alloc/tests/vec_deque.rs +++ b/crux-mir/lib/alloc/tests/vec_deque.rs @@ -1,9 +1,9 @@ -use std::collections::TryReserveError::*; +use std::assert_matches::assert_matches; +use std::collections::TryReserveErrorKind::*; use std::collections::{vec_deque::Drain, VecDeque}; use std::fmt::Debug; -use std::mem::size_of; +use std::ops::Bound::*; use std::panic::{catch_unwind, AssertUnwindSafe}; -use std::{isize, usize}; use crate::hash; @@ -116,6 +116,20 @@ fn test_index_out_of_bounds() { deq[3]; } +#[test] +#[should_panic] +fn test_range_start_overflow() { + let deq = VecDeque::from(vec![1, 2, 3]); + deq.range((Included(0), Included(usize::MAX))); +} + +#[test] +#[should_panic] +fn test_range_end_overflow() { + let deq = VecDeque::from(vec![1, 2, 3]); + deq.range((Excluded(usize::MAX), Included(0))); +} + #[derive(Clone, PartialEq, Debug)] enum Taggy { One(i32), @@ -451,7 +465,6 @@ fn test_drain() { for i in 6..9 { d.push_front(i); } - assert_eq!(d.drain(..).collect::>(), [8, 7, 6, 0, 1, 2, 3, 4]); assert!(d.is_empty()); } @@ -632,10 +645,10 @@ fn test_ord() { #[test] fn test_show() { let ringbuf: VecDeque<_> = (0..10).collect(); - assert_eq!(format!("{:?}", ringbuf), "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]"); + assert_eq!(format!("{ringbuf:?}"), "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]"); let ringbuf: VecDeque<_> = vec!["just", "one", "test", "more"].iter().cloned().collect(); - assert_eq!(format!("{:?}", ringbuf), "[\"just\", \"one\", \"test\", \"more\"]"); + assert_eq!(format!("{ringbuf:?}"), "[\"just\", \"one\", \"test\", \"more\"]"); } #[test] @@ -912,8 +925,8 @@ fn test_as_mut_slices() { #[test] fn test_append() { - let mut a: VecDeque<_> = vec![1, 2, 3].into_iter().collect(); - let mut b: VecDeque<_> = vec![4, 5, 6].into_iter().collect(); + let mut a: VecDeque<_> = [1, 2, 3].into_iter().collect(); + let mut b: VecDeque<_> = [4, 5, 6].into_iter().collect(); // normal append a.append(&mut b); @@ -955,16 +968,14 @@ fn test_append_permutations() { out } - #[cfg(not(miri))] // Miri is too slow - const MAX: usize = 5; - #[cfg(miri)] - const MAX: usize = 3; + // Miri is too slow + let max = if cfg!(miri) { 3 } else { 5 }; // Many different permutations of both the `VecDeque` getting appended to // and the one getting appended are generated to check `append`. // This ensures all 6 code paths of `append` are tested. - for src_push_back in 0..MAX { - for src_push_front in 0..MAX { + for src_push_back in 0..max { + for src_push_front in 0..max { // doesn't pop more values than are pushed for src_pop_back in 0..(src_push_back + src_push_front) { for src_pop_front in 0..(src_push_back + src_push_front - src_pop_back) { @@ -975,8 +986,8 @@ fn test_append_permutations() { src_pop_front, ); - for dst_push_back in 0..MAX { - for dst_push_front in 0..MAX { + for dst_push_back in 0..max { + for dst_push_front in 0..max { for dst_pop_back in 0..(dst_push_back + dst_push_front) { for dst_pop_front in 0..(dst_push_back + dst_push_front - dst_pop_back) @@ -1130,11 +1141,12 @@ fn test_reserve_exact_2() { v.push_back(16); v.reserve_exact(16); - assert!(v.capacity() >= 48) + assert!(v.capacity() >= 33) } #[test] #[cfg_attr(miri, ignore)] // Miri does not support signalling OOM +#[cfg_attr(target_os = "android", ignore)] // Android used in CI has a broken dlmalloc fn test_try_reserve() { // These are the interesting cases: // * exactly isize::MAX should never trigger a CapacityOverflow (can be OOM) @@ -1144,202 +1156,177 @@ fn test_try_reserve() { // * overflow may trigger when adding `len` to `cap` (in number of elements) // * overflow may trigger when multiplying `new_cap` by size_of:: (to get bytes) - const MAX_CAP: usize = (isize::MAX as usize + 1) / 2 - 1; + const MAX_CAP: usize = isize::MAX as usize; const MAX_USIZE: usize = usize::MAX; - // On 16/32-bit, we check that allocations don't exceed isize::MAX, - // on 64-bit, we assume the OS will give an OOM for such a ridiculous size. - // Any platform that succeeds for these requests is technically broken with - // ptr::offset because LLVM is the worst. - let guards_against_isize = size_of::() < 8; - { // Note: basic stuff is checked by test_reserve let mut empty_bytes: VecDeque = VecDeque::new(); // Check isize::MAX doesn't count as an overflow - if let Err(CapacityOverflow) = empty_bytes.try_reserve(MAX_CAP) { + if let Err(CapacityOverflow) = empty_bytes.try_reserve(MAX_CAP).map_err(|e| e.kind()) { panic!("isize::MAX shouldn't trigger an overflow!"); } // Play it again, frank! (just to be sure) - if let Err(CapacityOverflow) = empty_bytes.try_reserve(MAX_CAP) { + if let Err(CapacityOverflow) = empty_bytes.try_reserve(MAX_CAP).map_err(|e| e.kind()) { panic!("isize::MAX shouldn't trigger an overflow!"); } - if guards_against_isize { - // Check isize::MAX + 1 does count as overflow - if let Err(CapacityOverflow) = empty_bytes.try_reserve(MAX_CAP + 1) { - } else { - panic!("isize::MAX + 1 should trigger an overflow!") - } + // Check isize::MAX + 1 does count as overflow + assert_matches!( + empty_bytes.try_reserve(MAX_CAP + 1).map_err(|e| e.kind()), + Err(CapacityOverflow), + "isize::MAX + 1 should trigger an overflow!" + ); - // Check usize::MAX does count as overflow - if let Err(CapacityOverflow) = empty_bytes.try_reserve(MAX_USIZE) { - } else { - panic!("usize::MAX should trigger an overflow!") - } - } else { - // Check isize::MAX is an OOM - // VecDeque starts with capacity 7, always adds 1 to the capacity - // and also rounds the number to next power of 2 so this is the - // furthest we can go without triggering CapacityOverflow - if let Err(AllocError { .. }) = empty_bytes.try_reserve(MAX_CAP) { - } else { - panic!("isize::MAX + 1 should trigger an OOM!") - } - } + // Check usize::MAX does count as overflow + assert_matches!( + empty_bytes.try_reserve(MAX_USIZE).map_err(|e| e.kind()), + Err(CapacityOverflow), + "usize::MAX should trigger an overflow!" + ); } { // Same basic idea, but with non-zero len - let mut ten_bytes: VecDeque = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10].into_iter().collect(); + let mut ten_bytes: VecDeque = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10].into_iter().collect(); - if let Err(CapacityOverflow) = ten_bytes.try_reserve(MAX_CAP - 10) { + if let Err(CapacityOverflow) = ten_bytes.try_reserve(MAX_CAP - 10).map_err(|e| e.kind()) { panic!("isize::MAX shouldn't trigger an overflow!"); } - if let Err(CapacityOverflow) = ten_bytes.try_reserve(MAX_CAP - 10) { + if let Err(CapacityOverflow) = ten_bytes.try_reserve(MAX_CAP - 10).map_err(|e| e.kind()) { panic!("isize::MAX shouldn't trigger an overflow!"); } - if guards_against_isize { - if let Err(CapacityOverflow) = ten_bytes.try_reserve(MAX_CAP - 9) { - } else { - panic!("isize::MAX + 1 should trigger an overflow!"); - } - } else { - if let Err(AllocError { .. }) = ten_bytes.try_reserve(MAX_CAP - 9) { - } else { - panic!("isize::MAX + 1 should trigger an OOM!") - } - } + + assert_matches!( + ten_bytes.try_reserve(MAX_CAP - 9).map_err(|e| e.kind()), + Err(CapacityOverflow), + "isize::MAX + 1 should trigger an overflow!" + ); + // Should always overflow in the add-to-len - if let Err(CapacityOverflow) = ten_bytes.try_reserve(MAX_USIZE) { - } else { - panic!("usize::MAX should trigger an overflow!") - } + assert_matches!( + ten_bytes.try_reserve(MAX_USIZE).map_err(|e| e.kind()), + Err(CapacityOverflow), + "usize::MAX should trigger an overflow!" + ); } { // Same basic idea, but with interesting type size - let mut ten_u32s: VecDeque = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10].into_iter().collect(); + let mut ten_u32s: VecDeque = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10].into_iter().collect(); - if let Err(CapacityOverflow) = ten_u32s.try_reserve(MAX_CAP / 4 - 10) { + if let Err(CapacityOverflow) = ten_u32s.try_reserve(MAX_CAP / 4 - 10).map_err(|e| e.kind()) + { panic!("isize::MAX shouldn't trigger an overflow!"); } - if let Err(CapacityOverflow) = ten_u32s.try_reserve(MAX_CAP / 4 - 10) { + if let Err(CapacityOverflow) = ten_u32s.try_reserve(MAX_CAP / 4 - 10).map_err(|e| e.kind()) + { panic!("isize::MAX shouldn't trigger an overflow!"); } - if guards_against_isize { - if let Err(CapacityOverflow) = ten_u32s.try_reserve(MAX_CAP / 4 - 9) { - } else { - panic!("isize::MAX + 1 should trigger an overflow!"); - } - } else { - if let Err(AllocError { .. }) = ten_u32s.try_reserve(MAX_CAP / 4 - 9) { - } else { - panic!("isize::MAX + 1 should trigger an OOM!") - } - } + + assert_matches!( + ten_u32s.try_reserve(MAX_CAP / 4 - 9).map_err(|e| e.kind()), + Err(CapacityOverflow), + "isize::MAX + 1 should trigger an overflow!" + ); + // Should fail in the mul-by-size - if let Err(CapacityOverflow) = ten_u32s.try_reserve(MAX_USIZE - 20) { - } else { - panic!("usize::MAX should trigger an overflow!"); - } + assert_matches!( + ten_u32s.try_reserve(MAX_USIZE - 20).map_err(|e| e.kind()), + Err(CapacityOverflow), + "usize::MAX should trigger an overflow!" + ); } } #[test] #[cfg_attr(miri, ignore)] // Miri does not support signalling OOM +#[cfg_attr(target_os = "android", ignore)] // Android used in CI has a broken dlmalloc fn test_try_reserve_exact() { // This is exactly the same as test_try_reserve with the method changed. // See that test for comments. - const MAX_CAP: usize = (isize::MAX as usize + 1) / 2 - 1; + const MAX_CAP: usize = isize::MAX as usize; const MAX_USIZE: usize = usize::MAX; - let guards_against_isize = size_of::() < 8; - { let mut empty_bytes: VecDeque = VecDeque::new(); - if let Err(CapacityOverflow) = empty_bytes.try_reserve_exact(MAX_CAP) { + if let Err(CapacityOverflow) = empty_bytes.try_reserve_exact(MAX_CAP).map_err(|e| e.kind()) + { panic!("isize::MAX shouldn't trigger an overflow!"); } - if let Err(CapacityOverflow) = empty_bytes.try_reserve_exact(MAX_CAP) { + if let Err(CapacityOverflow) = empty_bytes.try_reserve_exact(MAX_CAP).map_err(|e| e.kind()) + { panic!("isize::MAX shouldn't trigger an overflow!"); } - if guards_against_isize { - if let Err(CapacityOverflow) = empty_bytes.try_reserve_exact(MAX_CAP + 1) { - } else { - panic!("isize::MAX + 1 should trigger an overflow!") - } + assert_matches!( + empty_bytes.try_reserve_exact(MAX_CAP + 1).map_err(|e| e.kind()), + Err(CapacityOverflow), + "isize::MAX + 1 should trigger an overflow!" + ); - if let Err(CapacityOverflow) = empty_bytes.try_reserve_exact(MAX_USIZE) { - } else { - panic!("usize::MAX should trigger an overflow!") - } - } else { - // Check isize::MAX is an OOM - // VecDeque starts with capacity 7, always adds 1 to the capacity - // and also rounds the number to next power of 2 so this is the - // furthest we can go without triggering CapacityOverflow - if let Err(AllocError { .. }) = empty_bytes.try_reserve_exact(MAX_CAP) { - } else { - panic!("isize::MAX + 1 should trigger an OOM!") - } - } + assert_matches!( + empty_bytes.try_reserve_exact(MAX_USIZE).map_err(|e| e.kind()), + Err(CapacityOverflow), + "usize::MAX should trigger an overflow!" + ); } { - let mut ten_bytes: VecDeque = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10].into_iter().collect(); + let mut ten_bytes: VecDeque = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10].into_iter().collect(); - if let Err(CapacityOverflow) = ten_bytes.try_reserve_exact(MAX_CAP - 10) { + if let Err(CapacityOverflow) = + ten_bytes.try_reserve_exact(MAX_CAP - 10).map_err(|e| e.kind()) + { panic!("isize::MAX shouldn't trigger an overflow!"); } - if let Err(CapacityOverflow) = ten_bytes.try_reserve_exact(MAX_CAP - 10) { + if let Err(CapacityOverflow) = + ten_bytes.try_reserve_exact(MAX_CAP - 10).map_err(|e| e.kind()) + { panic!("isize::MAX shouldn't trigger an overflow!"); } - if guards_against_isize { - if let Err(CapacityOverflow) = ten_bytes.try_reserve_exact(MAX_CAP - 9) { - } else { - panic!("isize::MAX + 1 should trigger an overflow!"); - } - } else { - if let Err(AllocError { .. }) = ten_bytes.try_reserve_exact(MAX_CAP - 9) { - } else { - panic!("isize::MAX + 1 should trigger an OOM!") - } - } - if let Err(CapacityOverflow) = ten_bytes.try_reserve_exact(MAX_USIZE) { - } else { - panic!("usize::MAX should trigger an overflow!") - } + + assert_matches!( + ten_bytes.try_reserve_exact(MAX_CAP - 9).map_err(|e| e.kind()), + Err(CapacityOverflow), + "isize::MAX + 1 should trigger an overflow!" + ); + + assert_matches!( + ten_bytes.try_reserve_exact(MAX_USIZE).map_err(|e| e.kind()), + Err(CapacityOverflow), + "usize::MAX should trigger an overflow!" + ); } { - let mut ten_u32s: VecDeque = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10].into_iter().collect(); + let mut ten_u32s: VecDeque = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10].into_iter().collect(); - if let Err(CapacityOverflow) = ten_u32s.try_reserve_exact(MAX_CAP / 4 - 10) { + if let Err(CapacityOverflow) = + ten_u32s.try_reserve_exact(MAX_CAP / 4 - 10).map_err(|e| e.kind()) + { panic!("isize::MAX shouldn't trigger an overflow!"); } - if let Err(CapacityOverflow) = ten_u32s.try_reserve_exact(MAX_CAP / 4 - 10) { + if let Err(CapacityOverflow) = + ten_u32s.try_reserve_exact(MAX_CAP / 4 - 10).map_err(|e| e.kind()) + { panic!("isize::MAX shouldn't trigger an overflow!"); } - if guards_against_isize { - if let Err(CapacityOverflow) = ten_u32s.try_reserve_exact(MAX_CAP / 4 - 9) { - } else { - panic!("isize::MAX + 1 should trigger an overflow!"); - } - } else { - if let Err(AllocError { .. }) = ten_u32s.try_reserve_exact(MAX_CAP / 4 - 9) { - } else { - panic!("isize::MAX + 1 should trigger an OOM!") - } - } - if let Err(CapacityOverflow) = ten_u32s.try_reserve_exact(MAX_USIZE - 20) { - } else { - panic!("usize::MAX should trigger an overflow!") - } + + assert_matches!( + ten_u32s.try_reserve_exact(MAX_CAP / 4 - 9).map_err(|e| e.kind()), + Err(CapacityOverflow), + "isize::MAX + 1 should trigger an overflow!" + ); + + assert_matches!( + ten_u32s.try_reserve_exact(MAX_USIZE - 20).map_err(|e| e.kind()), + Err(CapacityOverflow), + "usize::MAX should trigger an overflow!" + ); } } @@ -1403,7 +1390,8 @@ fn test_rotate_nop() { #[test] fn test_rotate_left_parts() { - let mut v: VecDeque<_> = (1..=7).collect(); + let mut v: VecDeque<_> = VecDeque::with_capacity(8); + v.extend(1..=7); v.rotate_left(2); assert_eq!(v.as_slices(), (&[3, 4, 5, 6, 7, 1][..], &[2][..])); v.rotate_left(2); @@ -1422,7 +1410,8 @@ fn test_rotate_left_parts() { #[test] fn test_rotate_right_parts() { - let mut v: VecDeque<_> = (1..=7).collect(); + let mut v: VecDeque<_> = VecDeque::with_capacity(8); + v.extend(1..=7); v.rotate_right(2); assert_eq!(v.as_slices(), (&[6, 7][..], &[1, 2, 3, 4, 5][..])); v.rotate_right(2); @@ -1645,3 +1634,141 @@ fn test_drain_leak() { drop(v); assert_eq!(unsafe { DROPS }, 7); } + +#[test] +fn test_binary_search() { + // Contiguous (front only) search: + let deque: VecDeque<_> = vec![1, 2, 3, 5, 6].into(); + assert!(deque.as_slices().1.is_empty()); + assert_eq!(deque.binary_search(&3), Ok(2)); + assert_eq!(deque.binary_search(&4), Err(3)); + + // Split search (both front & back non-empty): + let mut deque: VecDeque<_> = vec![5, 6].into(); + deque.push_front(3); + deque.push_front(2); + deque.push_front(1); + deque.push_back(10); + assert!(!deque.as_slices().0.is_empty()); + assert!(!deque.as_slices().1.is_empty()); + assert_eq!(deque.binary_search(&0), Err(0)); + assert_eq!(deque.binary_search(&1), Ok(0)); + assert_eq!(deque.binary_search(&5), Ok(3)); + assert_eq!(deque.binary_search(&7), Err(5)); + assert_eq!(deque.binary_search(&20), Err(6)); +} + +#[test] +fn test_binary_search_by() { + let deque: VecDeque<_> = vec![(1,), (2,), (3,), (5,), (6,)].into(); + + assert_eq!(deque.binary_search_by(|&(v,)| v.cmp(&3)), Ok(2)); + assert_eq!(deque.binary_search_by(|&(v,)| v.cmp(&4)), Err(3)); +} + +#[test] +fn test_binary_search_by_key() { + let deque: VecDeque<_> = vec![(1,), (2,), (3,), (5,), (6,)].into(); + + assert_eq!(deque.binary_search_by_key(&3, |&(v,)| v), Ok(2)); + assert_eq!(deque.binary_search_by_key(&4, |&(v,)| v), Err(3)); +} + +#[test] +fn test_partition_point() { + // Contiguous (front only) search: + let deque: VecDeque<_> = vec![1, 2, 3, 5, 6].into(); + assert!(deque.as_slices().1.is_empty()); + assert_eq!(deque.partition_point(|&v| v <= 3), 3); + + // Split search (both front & back non-empty): + let mut deque: VecDeque<_> = vec![5, 6].into(); + deque.push_front(3); + deque.push_front(2); + deque.push_front(1); + deque.push_back(10); + assert!(!deque.as_slices().0.is_empty()); + assert!(!deque.as_slices().1.is_empty()); + assert_eq!(deque.partition_point(|&v| v <= 5), 4); +} + +#[test] +fn test_zero_sized_push() { + const N: usize = 8; + + // Zero sized type + struct Zst; + + // Test that for all possible sequences of push_front / push_back, + // we end up with a deque of the correct size + + for len in 0..N { + let mut tester = VecDeque::with_capacity(len); + assert_eq!(tester.len(), 0); + assert!(tester.capacity() >= len); + for case in 0..(1 << len) { + assert_eq!(tester.len(), 0); + for bit in 0..len { + if case & (1 << bit) != 0 { + tester.push_front(Zst); + } else { + tester.push_back(Zst); + } + } + assert_eq!(tester.len(), len); + assert_eq!(tester.iter().count(), len); + tester.clear(); + } + } +} + +#[test] +fn test_from_zero_sized_vec() { + let v = vec![(); 100]; + let queue = VecDeque::from(v); + assert_eq!(queue.len(), 100); +} + +#[test] +fn test_resize_keeps_reserved_space_from_item() { + let v = Vec::::with_capacity(1234); + let mut d = VecDeque::new(); + d.resize(1, v); + assert_eq!(d[0].capacity(), 1234); +} + +#[test] +fn test_collect_from_into_iter_keeps_allocation() { + let mut v = Vec::with_capacity(13); + v.extend(0..7); + check(v.as_ptr(), v.last().unwrap(), v.into_iter()); + + let mut v = VecDeque::with_capacity(13); + v.extend(0..7); + check(&v[0], &v[v.len() - 1], v.into_iter()); + + fn check(buf: *const i32, last: *const i32, mut it: impl Iterator) { + assert_eq!(it.next(), Some(0)); + assert_eq!(it.next(), Some(1)); + + let mut v: VecDeque = it.collect(); + assert_eq!(v.capacity(), 13); + assert_eq!(v.as_slices().0.as_ptr(), buf.wrapping_add(2)); + assert_eq!(&v[v.len() - 1] as *const _, last); + + assert_eq!(v.as_slices(), ([2, 3, 4, 5, 6].as_slice(), [].as_slice())); + v.push_front(7); + assert_eq!(v.as_slices(), ([7, 2, 3, 4, 5, 6].as_slice(), [].as_slice())); + v.push_front(8); + assert_eq!(v.as_slices(), ([8, 7, 2, 3, 4, 5, 6].as_slice(), [].as_slice())); + + // Now that we've adding thing in place of the two that we removed from + // the front of the iterator, we're back to matching the buffer pointer. + assert_eq!(v.as_slices().0.as_ptr(), buf); + assert_eq!(&v[v.len() - 1] as *const _, last); + + v.push_front(9); + assert_eq!(v.as_slices(), ([9].as_slice(), [8, 7, 2, 3, 4, 5, 6].as_slice())); + assert_eq!(v.capacity(), 13); + } +} diff --git a/crux-mir/lib/backtrace/.github/workflows/main.yml b/crux-mir/lib/backtrace/.github/workflows/main.yml new file mode 100644 index 000000000..c11b08dfd --- /dev/null +++ b/crux-mir/lib/backtrace/.github/workflows/main.yml @@ -0,0 +1,245 @@ +name: CI + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + test: + name: Test + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-latest + rust: stable + - os: ubuntu-latest + rust: beta + - os: ubuntu-latest + rust: nightly + - os: macos-latest + rust: stable + - os: macos-latest + rust: nightly + # Note that these are on nightly due to rust-lang/rust#63700 not being + # on stable yet + - os: windows-latest + rust: stable-x86_64-msvc + - os: windows-latest + rust: stable-i686-msvc + - os: windows-latest + rust: stable-x86_64-gnu + steps: + - uses: actions/checkout@v1 + with: + submodules: true + - name: Install Rust (rustup) + run: rustup update ${{ matrix.rust }} --no-self-update && rustup default ${{ matrix.rust }} + shell: bash + - run: echo RUSTFLAGS=-Dwarnings >> $GITHUB_ENV + shell: bash + + # full fidelity of backtraces on 32-bit msvc requires frame pointers, so + # enable that for our tests + - name: Force frame pointers + run: echo RUSTFLAGS="-Cforce-frame-pointers $RUSTFLAGS" >> $GITHUB_ENV + shell: bash + if: matrix.rust == 'stable-i686-msvc' + + - run: cargo build + - run: cargo test + - run: cargo test --features "serialize-rustc" + - run: cargo test --features "serialize-serde" + - run: cargo test --features "verify-winapi" + - run: cargo test --features "cpp_demangle" + - run: cargo test --no-default-features + - run: cargo test --no-default-features --features "std" + - run: cargo test --manifest-path crates/cpp_smoke_test/Cargo.toml + # This test is specifically about packed debuginfo with `*.dSYM` files + - run: cargo test --manifest-path crates/macos_frames_test/Cargo.toml + env: + CARGO_PROFILE_DEV_SPLIT_DEBUGINFO: packed + CARGO_PROFILE_TEST_SPLIT_DEBUGINFO: packed + - run: cargo test --features gimli-symbolize --manifest-path crates/without_debuginfo/Cargo.toml + - run: cargo test --manifest-path crates/line-tables-only/Cargo.toml --features gimli-symbolize + + # Test debuginfo compression still works + - run: cargo test + if: contains(matrix.os, 'ubuntu') + env: + RUSTFLAGS: "-C link-arg=-Wl,--compress-debug-sections=zlib-gabi" + - run: cargo test + if: contains(matrix.os, 'ubuntu') + env: + RUSTFLAGS: "-C link-arg=-Wl,--compress-debug-sections=zlib-gnu" + + # Test that, on macOS, packed/unpacked debuginfo both work + - run: cargo clean && cargo test + # Test that, on macOS, packed/unpacked debuginfo both work + if: matrix.os == 'macos-latest' + env: + CARGO_PROFILE_DEV_SPLIT_DEBUGINFO: unpacked + CARGO_PROFILE_TEST_SPLIT_DEBUGINFO: unpacked + - run: cargo clean && cargo test + if: matrix.os == 'macos-latest' + env: + CARGO_PROFILE_DEV_SPLIT_DEBUGINFO: packed + CARGO_PROFILE_TEST_SPLIT_DEBUGINFO: packed + # Test that, on macOS, binaries with no UUID work + - run: cargo clean && cargo test + if: matrix.os == 'macos-latest' + env: + RUSTFLAGS: "-C link-arg=-Wl,-no_uuid" + + # Test that, on Linux, packed/unpacked debuginfo both work + - run: cargo clean && cargo test + if: matrix.rust == 'nightly' + env: + RUSTFLAGS: "-C split-debuginfo=unpacked -Zunstable-options" + - run: cargo clean && cargo test + if: matrix.rust == 'nightly' + env: + RUSTFLAGS: "-C split-debuginfo=packed -Zunstable-options" + + # Test that separate debug info works + - run: ./ci/debuglink-docker.sh + if: contains(matrix.os, 'ubuntu') + + # Test that including as a submodule will still work, both with and without + # the `backtrace` feature enabled. + - run: cargo build --manifest-path crates/as-if-std/Cargo.toml + - run: cargo build --manifest-path crates/as-if-std/Cargo.toml --no-default-features + + windows_arm64: + name: Windows AArch64 + runs-on: windows-latest + steps: + - uses: actions/checkout@v1 + with: + submodules: true + - name: Install Rust + run: rustup update stable --no-self-update && rustup default stable + shell: bash + - run: echo RUSTFLAGS=-Dwarnings >> $GITHUB_ENV + shell: bash + - run: rustup target add aarch64-pc-windows-msvc + - run: cargo test --no-run --target aarch64-pc-windows-msvc + - run: cargo test --no-run --target aarch64-pc-windows-msvc --features verify-winapi + + ios: + name: iOS + runs-on: macos-latest + strategy: + matrix: + include: + - target: aarch64-apple-ios + sdk: iphoneos + - target: x86_64-apple-ios + sdk: iphonesimulator + steps: + - uses: actions/checkout@v1 + with: + submodules: true + - run: rustup target add ${{ matrix.target }} + - run: | + export RUSTFLAGS=-Dwarnings + export SDK_PATH=`xcrun --show-sdk-path --sdk ${{ matrix.sdk }}` + export RUSTFLAGS="-C link-arg=-isysroot -C link-arg=$SDK_PATH" + cargo test --no-run --target ${{ matrix.target }} + name: Build tests + + docker: + name: Docker + runs-on: ubuntu-latest + strategy: + matrix: + target: + - aarch64-unknown-linux-gnu + - arm-unknown-linux-gnueabihf + - armv7-unknown-linux-gnueabihf + - i586-unknown-linux-gnu + - i686-unknown-linux-gnu + - powerpc64-unknown-linux-gnu + - s390x-unknown-linux-gnu + - x86_64-pc-windows-gnu + - x86_64-unknown-linux-gnu + - x86_64-unknown-linux-musl + - arm-linux-androideabi + - armv7-linux-androideabi + - aarch64-linux-android + - i686-linux-android + - x86_64-linux-android + steps: + - uses: actions/checkout@v1 + with: + submodules: true + - name: Install Rust + run: rustup update stable && rustup default stable + - run: rustup target add ${{ matrix.target }} + - run: cargo generate-lockfile + - run: echo RUSTFLAGS=-Dwarnings >> $GITHUB_ENV + shell: bash + - run: ./ci/run-docker.sh ${{ matrix.target }} + + rustfmt: + name: Rustfmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + submodules: true + - name: Install Rust + run: rustup update stable && rustup default stable && rustup component add rustfmt + - run: cargo fmt --all -- --check + + build: + name: Build Targets + runs-on: ubuntu-latest + strategy: + matrix: + target: + - wasm32-unknown-unknown + - wasm32-wasi + - x86_64-fuchsia + - x86_64-fortanix-unknown-sgx + - x86_64-unknown-illumos + steps: + - uses: actions/checkout@v1 + with: + submodules: true + - name: Install Rust + run: rustup update nightly && rustup default nightly + - run: rustup target add ${{ matrix.target }} + - run: echo RUSTFLAGS=-Dwarnings >> $GITHUB_ENV + shell: bash + - run: cargo build --target ${{ matrix.target }} + - run: cargo build --manifest-path crates/as-if-std/Cargo.toml --target ${{ matrix.target }} + + msrv: + name: MSRV + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + submodules: true + - name: Install Rust + run: rustup update 1.42.0 && rustup default 1.42.0 + - run: cargo build + + miri: + name: Miri + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + submodules: true + - name: Install Miri + run: | + rustup toolchain install nightly --component miri + rustup override set nightly + cargo miri setup + - run: MIRIFLAGS="-Zmiri-disable-isolation" cargo miri test diff --git a/crux-mir/lib/backtrace/Cargo.toml b/crux-mir/lib/backtrace/Cargo.toml index 24b9ff697..ef1c5ec00 100644 --- a/crux-mir/lib/backtrace/Cargo.toml +++ b/crux-mir/lib/backtrace/Cargo.toml @@ -1,7 +1,8 @@ [package] name = "backtrace" -version = "0.3.44" +version = "0.3.66" authors = ["The Rust Project Developers"] +build = "build.rs" license = "MIT/Apache-2.0" readme = "README.md" repository = "https://github.com/rust-lang/backtrace-rs" @@ -15,89 +16,56 @@ autotests = true edition = "2018" [workspace] -members = ['crates/cpp_smoke_test'] -exclude = ['crates/without_debuginfo'] +members = ['crates/cpp_smoke_test', 'crates/as-if-std'] +exclude = [ + 'crates/without_debuginfo', + 'crates/macos_frames_test', + 'crates/line-tables-only', + 'crates/debuglink', +] [dependencies] -cfg-if = "0.1.10" +cfg-if = "1.0" rustc-demangle = "0.1.4" -backtrace-sys = { path = "crates/backtrace-sys", version = "0.1.32", optional = true } -libc = { version = "0.2.45", default-features = false } -core = { version = "1.0.0", optional = true, package = 'rustc-std-workspace-core' } -compiler_builtins = { version = '0.1.2', optional = true } +libc = { version = "0.2.94", default-features = false } -# Optionally enable the ability to serialize a `Backtrace` +# Optionally enable the ability to serialize a `Backtrace`, controlled through +# the `serialize-*` features below. serde = { version = "1.0", optional = true, features = ['derive'] } rustc-serialize = { version = "0.3", optional = true } # Optionally demangle C++ frames' symbols in backtraces. -cpp_demangle = { default-features = false, version = "0.2.3", optional = true } +cpp_demangle = { default-features = false, version = "0.3.0", optional = true } + -# Optional dependencies enabled through the `gimli-symbolize` feature -addr2line = { version = "0.11.0", optional = true, default-features = false, features = ['std'] } -findshlibs = { version = "0.5.0", optional = true } -memmap = { version = "0.7.0", optional = true } -goblin = { version = "0.1.3", optional = true, default-features = false, features = ['elf32', 'elf64', 'mach32', 'mach64', 'pe32', 'pe64', 'std'] } +# Optional dependencies enabled through the `gimli-symbolize` feature, do not +# use these features directly. +addr2line = { version = "0.17.0", default-features = false } +miniz_oxide = { version = "0.5.0", default-features = false } + +[dependencies.object] +version = "0.29.0" +default-features = false +features = ['read_core', 'elf', 'macho', 'pe', 'unaligned', 'archive'] [target.'cfg(windows)'.dependencies] -winapi = { version = "0.3.3", optional = true } +winapi = { version = "0.3.9", optional = true } -# Each feature controls the two phases of finding a backtrace: getting a -# backtrace and then resolving instruction pointers to symbols. The default -# feature enables all the necessary features for each platform this library -# supports, but it can be disabled to have finer grained control over the -# dependencies. -# -# Note that not all features are available on all platforms, so even though a -# feature is enabled some other feature may be used instead. -[features] -default = ["std", "libunwind", "libbacktrace", "dladdr", "dbghelp"] +[build-dependencies] +# Only needed for Android, but cannot be target dependent +# https://github.com/rust-lang/cargo/issues/4932 +cc = "1.0.67" -# Include std support. -std = [] +[dev-dependencies] +dylib-dep = { path = "crates/dylib-dep" } +libloading = "0.7" -#======================================= -# Methods of acquiring a backtrace -# -# - libunwind: when using this the libgcc library is linked against to get -# the unwinding support. This is generally the most reliable method to get -# a backtrace on unix. -# - unix-backtrace: this uses the backtrace(3) function to acquire a -# backtrace, but is not as reliable as libunwind. It is, however, -# generally found in more locations. -# - dbghelp: on windows this enables usage of dbghelp.dll to find a -# backtrace at runtime -# - kernel32: on windows this enables using RtlCaptureStackBackTrace as the -# function to acquire a backtrace -libunwind = [] -unix-backtrace = [] -dbghelp = [] -kernel32 = [] +[features] +# By default libstd support and gimli-symbolize is used to symbolize addresses. +default = ["std"] -#======================================= -# Methods of resolving symbols -# -# - libbacktrace: this feature activates the `backtrace-sys` dependency, -# building the libbacktrace library found in gcc repos. This library -# parses the DWARF info of ELF executables to find symbol names, and it -# can also provide filename/line number information if debuginfo is -# compiled in. This library currently only primarily works on unixes that -# are not OSX, however. -# - dladdr: this feature uses the dladdr(3) function (a glibc extension) to -# resolve symbol names. This is fairly unreliable on linux, but works well -# enough on OSX. -# - coresymbolication: this feature uses the undocumented core symbolication -# framework on OS X to symbolize. Note that this is not turned on default due -# to various issues, you can see more in #202. -# - gimli-symbolize: use the `gimli-rs/addr2line` crate to symbolicate -# addresses into file, line, and name using DWARF debug information. At -# the moment, this is only possible when targetting Linux, since macOS -# splits DWARF out into a separate object file. Enabling this feature -# means one less C dependency. -libbacktrace = ["backtrace-sys"] -dladdr = [] -coresymbolication = [] -gimli-symbolize = ["addr2line", "findshlibs", "memmap", "goblin"] +# Include std support. This enables types like `Backtrace`. +std = [] #======================================= # Methods of serialization @@ -107,25 +75,30 @@ serialize-rustc = ["rustc-serialize"] serialize-serde = ["serde"] #======================================= -# Internal features for testing and such. +# Deprecated/internal features +# +# Only here for backwards compatibility purposes or for internal testing +# purposes. New code should use none of these features. +coresymbolication = [] +dbghelp = [] +dladdr = [] +gimli-symbolize = [] +kernel32 = [] +libbacktrace = [] +libunwind = [] +unix-backtrace = [] verify-winapi = [ 'winapi/dbghelp', 'winapi/handleapi', 'winapi/libloaderapi', + 'winapi/memoryapi', 'winapi/minwindef', 'winapi/processthreadsapi', 'winapi/synchapi', + 'winapi/tlhelp32', 'winapi/winbase', 'winapi/winnt', ] -rustc-dep-of-std = [ - 'backtrace-sys/rustc-dep-of-std', - 'cfg-if/rustc-dep-of-std', - 'core', - 'compiler_builtins', - 'libc/rustc-dep-of-std', - 'rustc-demangle/rustc-dep-of-std', -] [[example]] name = "backtrace" @@ -150,7 +123,7 @@ edition = '2018' [[test]] name = "accuracy" -required-features = ["std", "dbghelp", "libbacktrace", "libunwind"] +required-features = ["std"] edition = '2018' [[test]] diff --git a/crux-mir/lib/backtrace/README.md b/crux-mir/lib/backtrace/README.md index 9713f0c3c..cd80a697c 100644 --- a/crux-mir/lib/backtrace/README.md +++ b/crux-mir/lib/backtrace/README.md @@ -14,18 +14,12 @@ backtrace like libstd's panics. backtrace = "0.3" ``` -Note that this crate requires `cc` and `ar` to be present on Unix systems when -`libbacktrace` is used (which is the default). For configuring C compilers see -the [`cc` crate documentation](https://github.com/alexcrichton/cc-rs). - ## Usage To simply capture a backtrace and defer dealing with it until a later time, you can use the top-level `Backtrace` type. ```rust -extern crate backtrace; - use backtrace::Backtrace; fn main() { @@ -41,8 +35,6 @@ If, however, you'd like more raw access to the actual tracing functionality, you can use the `trace` and `resolve` functions directly. ```rust -extern crate backtrace; - fn main() { backtrace::trace(|frame| { let ip = frame.ip(); @@ -68,9 +60,9 @@ fn main() { This project is licensed under either of * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or - http://www.apache.org/licenses/LICENSE-2.0) + https://www.apache.org/licenses/LICENSE-2.0) * MIT license ([LICENSE-MIT](LICENSE-MIT) or - http://opensource.org/licenses/MIT) + https://opensource.org/licenses/MIT) at your option. diff --git a/crux-mir/lib/backtrace/benches/benchmarks.rs b/crux-mir/lib/backtrace/benches/benchmarks.rs index ad55788c2..e14e733b8 100644 --- a/crux-mir/lib/backtrace/benches/benchmarks.rs +++ b/crux-mir/lib/backtrace/benches/benchmarks.rs @@ -2,8 +2,6 @@ extern crate test; -extern crate backtrace; - #[cfg(feature = "std")] use backtrace::Backtrace; diff --git a/crux-mir/lib/backtrace/build.rs b/crux-mir/lib/backtrace/build.rs new file mode 100644 index 000000000..812fbb1fe --- /dev/null +++ b/crux-mir/lib/backtrace/build.rs @@ -0,0 +1,41 @@ +extern crate cc; + +use std::env; + +fn main() { + match env::var("CARGO_CFG_TARGET_OS").unwrap_or_default().as_str() { + "android" => build_android(), + _ => {} + } +} + +fn build_android() { + let expansion = match cc::Build::new().file("src/android-api.c").try_expand() { + Ok(result) => result, + Err(e) => { + println!("failed to run C compiler: {}", e); + return; + } + }; + let expansion = match std::str::from_utf8(&expansion) { + Ok(s) => s, + Err(_) => return, + }; + println!("expanded android version detection:\n{}", expansion); + let marker = "APIVERSION"; + let i = match expansion.find(marker) { + Some(i) => i, + None => return, + }; + let version = match expansion[i + marker.len() + 1..].split_whitespace().next() { + Some(s) => s, + None => return, + }; + let version = match version.parse::() { + Ok(n) => n, + Err(_) => return, + }; + if version >= 21 { + println!("cargo:rustc-cfg=feature=\"dl_iterate_phdr\""); + } +} diff --git a/crux-mir/lib/backtrace/ci/android-sdk.sh b/crux-mir/lib/backtrace/ci/android-sdk.sh index aee133e3a..7fde9a97f 100755 --- a/crux-mir/lib/backtrace/ci/android-sdk.sh +++ b/crux-mir/lib/backtrace/ci/android-sdk.sh @@ -1,13 +1,4 @@ #!/usr/bin/env sh -# Copyright 2016 The Rust Project Developers. See the COPYRIGHT -# file at the top-level directory of this distribution and at -# http://rust-lang.org/COPYRIGHT. -# -# Licensed under the Apache License, Version 2.0 or the MIT license -# , at your -# option. This file may not be copied, modified, or distributed -# except according to those terms. set -ex diff --git a/crux-mir/lib/backtrace/ci/debuglink-docker.sh b/crux-mir/lib/backtrace/ci/debuglink-docker.sh new file mode 100755 index 000000000..acb19e98b --- /dev/null +++ b/crux-mir/lib/backtrace/ci/debuglink-docker.sh @@ -0,0 +1,29 @@ +# Small script to run debuglink tests inside a docker image. +# Creates a writable mount on /usr/lib/debug. + +set -ex + +run() { + cargo generate-lockfile --manifest-path crates/debuglink/Cargo.toml + mkdir -p target crates/debuglink/target debug + docker build -t backtrace -f ci/docker/$1/Dockerfile ci + docker run \ + --user `id -u`:`id -g` \ + --rm \ + --init \ + --volume $(dirname $(dirname `which cargo`)):/cargo \ + --env CARGO_HOME=/cargo \ + --volume `rustc --print sysroot`:/rust:ro \ + --env TARGET=$1 \ + --volume `pwd`:/checkout:ro \ + --volume `pwd`/target:/checkout/crates/debuglink/target \ + --workdir /checkout \ + --volume `pwd`/debug:/usr/lib/debug \ + --privileged \ + --env RUSTFLAGS \ + backtrace \ + bash \ + -c 'PATH=$PATH:/rust/bin exec ci/debuglink.sh' +} + +run x86_64-unknown-linux-gnu diff --git a/crux-mir/lib/backtrace/ci/debuglink.sh b/crux-mir/lib/backtrace/ci/debuglink.sh new file mode 100755 index 000000000..b2da2013d --- /dev/null +++ b/crux-mir/lib/backtrace/ci/debuglink.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# Debuglink tests. +# We build crates/debuglink and then move its debuginfo around +# and test that it can still find the debuginfo. + +set -ex + +cratedir=`pwd`/crates/debuglink +exefile=crates/debuglink/target/debug/debuglink + +# Baseline; no separate debug +cargo build --manifest-path crates/debuglink/Cargo.toml +$exefile $cratedir + +# Separate debug in same dir +debugfile1=`dirname $exefile`/debuglink.debug +objcopy --only-keep-debug $exefile $debugfile1 +strip -g $exefile +(cd `dirname $exefile` && objcopy --add-gnu-debuglink=debuglink.debug debuglink) +$exefile $cratedir + +# Separate debug in .debug subdir +debugfile2=`dirname $exefile`/.debug/debuglink.debug +mkdir -p `dirname $debugfile2` +mv $debugfile1 $debugfile2 +$exefile $cratedir + +# Separate debug in /usr/lib/debug subdir +debugfile3="/usr/lib/debug/$cratedir/target/debug/debuglink.debug" +mkdir -p `dirname $debugfile3` +mv $debugfile2 $debugfile3 +$exefile $cratedir + +# Separate debug in /usr/lib/debug/.build-id subdir +id=`readelf -n $exefile | grep '^ Build ID: [0-9a-f]' | cut -b 15-` +idfile="/usr/lib/debug/.build-id/${id:0:2}/${id:2}.debug" +mkdir -p `dirname $idfile` +mv $debugfile3 $idfile +$exefile $cratedir + +# Replace idfile with a symlink (this is the usual arrangement) +mv $idfile $debugfile3 +ln -s $debugfile3 $idfile +$exefile $cratedir + +# Supplementary object file using relative path +dwzfile="/usr/lib/debug/.dwz/debuglink.debug" +mkdir -p `dirname $dwzfile` +cp $debugfile3 $debugfile3.copy +dwz -m $dwzfile -rh $debugfile3 $debugfile3.copy +rm $debugfile3.copy +$exefile $cratedir + +# Supplementary object file using build ID +dwzid=`readelf -n $dwzfile | grep '^ Build ID: [0-9a-f]' | cut -b 15-` +dwzidfile="/usr/lib/debug/.build-id/${dwzid:0:2}/${dwzid:2}.debug" +mkdir -p `dirname $dwzidfile` +mv $dwzfile $dwzidfile +$exefile $cratedir +mv $dwzidfile $dwzfile + +# Missing debug should fail +mv $debugfile3 $debugfile3.tmp +! $exefile $cratedir +mv $debugfile3.tmp $debugfile3 + +# Missing dwz should fail +mv $dwzfile $dwzfile.tmp +! $exefile $cratedir +mv $dwzfile.tmp $dwzfile + +# Cleanup +rm $idfile $debugfile3 $dwzfile +echo Success diff --git a/crux-mir/lib/backtrace/ci/docker/aarch64-linux-android/Dockerfile b/crux-mir/lib/backtrace/ci/docker/aarch64-linux-android/Dockerfile index 81c8208ef..c5655ed5e 100644 --- a/crux-mir/lib/backtrace/ci/docker/aarch64-linux-android/Dockerfile +++ b/crux-mir/lib/backtrace/ci/docker/aarch64-linux-android/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ diff --git a/crux-mir/lib/backtrace/ci/docker/aarch64-unknown-linux-gnu/Dockerfile b/crux-mir/lib/backtrace/ci/docker/aarch64-unknown-linux-gnu/Dockerfile index d691aa46e..871b353c0 100644 --- a/crux-mir/lib/backtrace/ci/docker/aarch64-unknown-linux-gnu/Dockerfile +++ b/crux-mir/lib/backtrace/ci/docker/aarch64-unknown-linux-gnu/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 RUN apt-get update && apt-get install -y --no-install-recommends \ gcc \ ca-certificates \ diff --git a/crux-mir/lib/backtrace/ci/docker/arm-linux-androideabi/Dockerfile b/crux-mir/lib/backtrace/ci/docker/arm-linux-androideabi/Dockerfile index 7cfdae639..446a64cc0 100644 --- a/crux-mir/lib/backtrace/ci/docker/arm-linux-androideabi/Dockerfile +++ b/crux-mir/lib/backtrace/ci/docker/arm-linux-androideabi/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ @@ -11,27 +11,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY android-ndk.sh / RUN /android-ndk.sh arm -WORKDIR /android -COPY android-sdk.sh /android/sdk.sh -RUN ./sdk.sh arm -RUN mv /root/.android /tmp -RUN chmod 777 -R /tmp/.android -RUN chmod 755 /android/sdk/tools/* /android/sdk/emulator/qemu/linux-x86_64/* -ENV PATH=$PATH:/android-toolchain/bin:/android/sdk/platform-tools +ENV PATH=$PATH:/android-toolchain/bin # TODO: run tests in an emulator eventually ENV CARGO_TARGET_ARM_LINUX_ANDROIDEABI_LINKER=arm-linux-androideabi-gcc \ - CARGO_TARGET_ARM_LINUX_ANDROIDEABI_RUNNER=/tmp/runtest \ - HOME=/tmp - -ADD runtest-android.rs /tmp/runtest.rs -ENTRYPOINT [ \ - "bash", \ - "-c", \ - # set SHELL so android can detect a 64bits system, see - # http://stackoverflow.com/a/41789144 - "SHELL=/bin/dash /android/sdk/emulator/emulator @arm -no-window & \ - /rust/bin/rustc /tmp/runtest.rs -o /tmp/runtest && \ - exec \"$@\"", \ - "--" \ -] + CARGO_TARGET_ARM_LINUX_ANDROIDEABI_RUNNER=echo diff --git a/crux-mir/lib/backtrace/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile b/crux-mir/lib/backtrace/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile index 32095e98f..24665972c 100644 --- a/crux-mir/lib/backtrace/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile +++ b/crux-mir/lib/backtrace/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 RUN apt-get update && apt-get install -y --no-install-recommends \ gcc \ ca-certificates \ diff --git a/crux-mir/lib/backtrace/ci/docker/armv7-linux-androideabi/Dockerfile b/crux-mir/lib/backtrace/ci/docker/armv7-linux-androideabi/Dockerfile index c73871603..539bbc494 100644 --- a/crux-mir/lib/backtrace/ci/docker/armv7-linux-androideabi/Dockerfile +++ b/crux-mir/lib/backtrace/ci/docker/armv7-linux-androideabi/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ diff --git a/crux-mir/lib/backtrace/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile b/crux-mir/lib/backtrace/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile index a55fb2792..6f7d0fd36 100644 --- a/crux-mir/lib/backtrace/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile +++ b/crux-mir/lib/backtrace/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 RUN apt-get update && apt-get install -y --no-install-recommends \ gcc \ ca-certificates \ diff --git a/crux-mir/lib/backtrace/ci/docker/i586-unknown-linux-gnu/Dockerfile b/crux-mir/lib/backtrace/ci/docker/i586-unknown-linux-gnu/Dockerfile index d22209295..316a233e3 100644 --- a/crux-mir/lib/backtrace/ci/docker/i586-unknown-linux-gnu/Dockerfile +++ b/crux-mir/lib/backtrace/ci/docker/i586-unknown-linux-gnu/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 RUN apt-get update && apt-get install -y --no-install-recommends \ gcc-multilib \ libc6-dev \ diff --git a/crux-mir/lib/backtrace/ci/docker/i686-linux-android/Dockerfile b/crux-mir/lib/backtrace/ci/docker/i686-linux-android/Dockerfile index ef4752cb3..83ccb2948 100644 --- a/crux-mir/lib/backtrace/ci/docker/i686-linux-android/Dockerfile +++ b/crux-mir/lib/backtrace/ci/docker/i686-linux-android/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ diff --git a/crux-mir/lib/backtrace/ci/docker/i686-unknown-linux-gnu/Dockerfile b/crux-mir/lib/backtrace/ci/docker/i686-unknown-linux-gnu/Dockerfile index d22209295..316a233e3 100644 --- a/crux-mir/lib/backtrace/ci/docker/i686-unknown-linux-gnu/Dockerfile +++ b/crux-mir/lib/backtrace/ci/docker/i686-unknown-linux-gnu/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 RUN apt-get update && apt-get install -y --no-install-recommends \ gcc-multilib \ libc6-dev \ diff --git a/crux-mir/lib/backtrace/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile b/crux-mir/lib/backtrace/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile index b4f5d8a42..7ca5a64bf 100644 --- a/crux-mir/lib/backtrace/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile +++ b/crux-mir/lib/backtrace/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 RUN apt-get update && apt-get install -y --no-install-recommends \ gcc \ diff --git a/crux-mir/lib/backtrace/ci/docker/s390x-unknown-linux-gnu/Dockerfile b/crux-mir/lib/backtrace/ci/docker/s390x-unknown-linux-gnu/Dockerfile new file mode 100644 index 000000000..7c19dcbb4 --- /dev/null +++ b/crux-mir/lib/backtrace/ci/docker/s390x-unknown-linux-gnu/Dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:20.04 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + ca-certificates \ + libc6-dev \ + gcc-s390x-linux-gnu \ + libc6-dev-s390x-cross \ + qemu-user \ + # There seems to be a bug in processing mixed-architecture + # ld.so.cache files that causes crashes in some cases. Work + # around this by simply deleting the cache for now. + && rm /etc/ld.so.cache + +ENV CARGO_TARGET_S390X_UNKNOWN_LINUX_GNU_LINKER=s390x-linux-gnu-gcc \ + CARGO_TARGET_S390X_UNKNOWN_LINUX_GNU_RUNNER="qemu-s390x -L /usr/s390x-linux-gnu" \ + CC=s390x-linux-gnu-gcc diff --git a/crux-mir/lib/backtrace/ci/docker/x86_64-linux-android/Dockerfile b/crux-mir/lib/backtrace/ci/docker/x86_64-linux-android/Dockerfile index 7fea8a163..88a22ce6c 100644 --- a/crux-mir/lib/backtrace/ci/docker/x86_64-linux-android/Dockerfile +++ b/crux-mir/lib/backtrace/ci/docker/x86_64-linux-android/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ diff --git a/crux-mir/lib/backtrace/ci/docker/x86_64-pc-windows-gnu/Dockerfile b/crux-mir/lib/backtrace/ci/docker/x86_64-pc-windows-gnu/Dockerfile index 7a8fbf437..a8e859e67 100644 --- a/crux-mir/lib/backtrace/ci/docker/x86_64-pc-windows-gnu/Dockerfile +++ b/crux-mir/lib/backtrace/ci/docker/x86_64-pc-windows-gnu/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 RUN apt-get update && apt-get install -y --no-install-recommends \ gcc \ libc6-dev \ diff --git a/crux-mir/lib/backtrace/ci/docker/x86_64-unknown-linux-gnu/Dockerfile b/crux-mir/lib/backtrace/ci/docker/x86_64-unknown-linux-gnu/Dockerfile index 864d72e62..551ab1378 100644 --- a/crux-mir/lib/backtrace/ci/docker/x86_64-unknown-linux-gnu/Dockerfile +++ b/crux-mir/lib/backtrace/ci/docker/x86_64-unknown-linux-gnu/Dockerfile @@ -1,5 +1,6 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 RUN apt-get update && apt-get install -y --no-install-recommends \ gcc \ libc6-dev \ - ca-certificates + ca-certificates \ + dwz diff --git a/crux-mir/lib/backtrace/ci/docker/x86_64-unknown-linux-musl/Dockerfile b/crux-mir/lib/backtrace/ci/docker/x86_64-unknown-linux-musl/Dockerfile index 6984dc217..e77e41f5b 100644 --- a/crux-mir/lib/backtrace/ci/docker/x86_64-unknown-linux-musl/Dockerfile +++ b/crux-mir/lib/backtrace/ci/docker/x86_64-unknown-linux-musl/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 RUN apt-get update && apt-get install -y --no-install-recommends \ gcc \ libc6-dev \ diff --git a/crux-mir/lib/backtrace/ci/run-docker.sh b/crux-mir/lib/backtrace/ci/run-docker.sh index 5a9934b41..8aa6d84a4 100755 --- a/crux-mir/lib/backtrace/ci/run-docker.sh +++ b/crux-mir/lib/backtrace/ci/run-docker.sh @@ -18,6 +18,7 @@ run() { --volume `pwd`/target:/checkout/target \ --workdir /checkout \ --privileged \ + --env RUSTFLAGS \ backtrace \ bash \ -c 'PATH=$PATH:/rust/bin exec ci/run.sh' diff --git a/crux-mir/lib/backtrace/ci/run.sh b/crux-mir/lib/backtrace/ci/run.sh index 5cc151507..166b387e4 100755 --- a/crux-mir/lib/backtrace/ci/run.sh +++ b/crux-mir/lib/backtrace/ci/run.sh @@ -3,3 +3,4 @@ set -ex cargo test --target $TARGET +cargo build --target $TARGET --manifest-path crates/as-if-std/Cargo.toml diff --git a/crux-mir/lib/backtrace/crates/as-if-std/Cargo.toml b/crux-mir/lib/backtrace/crates/as-if-std/Cargo.toml new file mode 100644 index 000000000..c763227f2 --- /dev/null +++ b/crux-mir/lib/backtrace/crates/as-if-std/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "as-if-std" +version = "0.1.0" +authors = ["Alex Crichton "] +edition = "2018" +publish = false + +[lib] +test = false +doc = false +doctest = false +bench = false + +[dependencies] +cfg-if = "1.0" +rustc-demangle = "0.1.4" +libc = { version = "0.2.45", default-features = false } +addr2line = { version = "0.16.0", default-features = false, optional = true } +miniz_oxide = { version = "0.4.0", default-features = false } + +[dependencies.object] +version = "0.28" +default-features = false +optional = true +features = ['read_core', 'elf', 'macho', 'pe', 'unaligned', 'archive'] + +[features] +default = ['backtrace'] +backtrace = ['addr2line', 'object'] diff --git a/crux-mir/lib/backtrace/crates/as-if-std/build.rs b/crux-mir/lib/backtrace/crates/as-if-std/build.rs new file mode 100644 index 000000000..7018b1017 --- /dev/null +++ b/crux-mir/lib/backtrace/crates/as-if-std/build.rs @@ -0,0 +1,3 @@ +fn main() { + println!("cargo:rustc-cfg=backtrace_in_libstd"); +} diff --git a/crux-mir/lib/backtrace/crates/as-if-std/src/lib.rs b/crux-mir/lib/backtrace/crates/as-if-std/src/lib.rs new file mode 100644 index 000000000..c0f49b77d --- /dev/null +++ b/crux-mir/lib/backtrace/crates/as-if-std/src/lib.rs @@ -0,0 +1,21 @@ +// A crate which builds the `backtrace` crate as-if it's included as a +// submodule into the standard library. We try to set this crate up similarly +// to the standard library itself to minimize the likelihood of issues when +// updating the `backtrace` crate. + +#![no_std] + +extern crate alloc; + +// We want to `pub use std::*` in the root but we don't want `std` available in +// the root namespace, so do this in a funky inner module. +mod __internal { + extern crate std; + pub use std::*; +} + +pub use __internal::*; + +// This is the magical part which we hope works. +#[path = "../../../src/lib.rs"] +mod the_backtrace_crate; diff --git a/crux-mir/lib/backtrace/crates/backtrace-sys/Cargo.toml b/crux-mir/lib/backtrace/crates/backtrace-sys/Cargo.toml deleted file mode 100644 index 5245d6b1f..000000000 --- a/crux-mir/lib/backtrace/crates/backtrace-sys/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "backtrace-sys" -version = "0.1.32" -authors = ["Alex Crichton "] -build = "build.rs" -license = "MIT/Apache-2.0" -repository = "https://github.com/alexcrichton/backtrace-rs" -homepage = "https://github.com/alexcrichton/backtrace-rs" -documentation = "http://alexcrichton.com/backtrace-rs" -description = """ -Bindings to the libbacktrace gcc library -""" - -[dependencies] -libc = { version = "0.2", default-features = false } -core = { version = '1.0.0', optional = true, package = 'rustc-std-workspace-core' } -compiler_builtins = { version = '0.1.2', optional = true } - -[build-dependencies] -cc = "1.0.37" - -[features] -rustc-dep-of-std = ['core', 'compiler_builtins'] diff --git a/crux-mir/lib/backtrace/crates/backtrace-sys/build.rs b/crux-mir/lib/backtrace/crates/backtrace-sys/build.rs deleted file mode 100644 index 98ab6be05..000000000 --- a/crux-mir/lib/backtrace/crates/backtrace-sys/build.rs +++ /dev/null @@ -1,156 +0,0 @@ -extern crate cc; - -use std::env; -use std::fs::File; -use std::path::PathBuf; - -fn main() { - let target = env::var("TARGET").unwrap(); - - if target.contains("msvc") || // libbacktrace isn't used on MSVC windows - target.contains("emscripten") || // no way this will ever compile for emscripten - target.contains("cloudabi") || - target.contains("hermit") || - target.contains("wasm32") || - target.contains("fuchsia") || - target.contains("uclibc") - { - println!("cargo:rustc-cfg=empty"); - return; - } - - let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()); - - let mut build = cc::Build::new(); - build - .include("src/libbacktrace") - .include(&out_dir) - .warnings(false) - .file("src/libbacktrace/alloc.c") - .file("src/libbacktrace/dwarf.c") - .file("src/libbacktrace/fileline.c") - .file("src/libbacktrace/posix.c") - .file("src/libbacktrace/read.c") - .file("src/libbacktrace/sort.c") - .file("src/libbacktrace/state.c"); - - // No need to have any symbols reexported form shared objects - build.flag("-fvisibility=hidden"); - - if target.contains("darwin") { - build.file("src/libbacktrace/macho.c"); - } else if target.contains("windows") { - build.file("src/libbacktrace/pecoff.c"); - } else { - build.file("src/libbacktrace/elf.c"); - - let pointer_width = env::var("CARGO_CFG_TARGET_POINTER_WIDTH").unwrap(); - if pointer_width == "64" { - build.define("BACKTRACE_ELF_SIZE", "64"); - } else { - build.define("BACKTRACE_ELF_SIZE", "32"); - } - } - - File::create(out_dir.join("backtrace-supported.h")).unwrap(); - build.define("BACKTRACE_SUPPORTED", "1"); - build.define("BACKTRACE_USES_MALLOC", "1"); - build.define("BACKTRACE_SUPPORTS_THREADS", "0"); - build.define("BACKTRACE_SUPPORTS_DATA", "0"); - - File::create(out_dir.join("config.h")).unwrap(); - if target.contains("android") { - maybe_enable_dl_iterate_phdr_android(&mut build); - } else if !target.contains("apple-ios") - && !target.contains("solaris") - && !target.contains("redox") - && !target.contains("haiku") - && !target.contains("vxworks") - { - build.define("HAVE_DL_ITERATE_PHDR", "1"); - } - build.define("_GNU_SOURCE", "1"); - build.define("_LARGE_FILES", "1"); - - // When we're built as part of the Rust compiler, this is used to enable - // debug information in libbacktrace itself. - let any_debug = env::var("RUSTC_DEBUGINFO").unwrap_or_default() == "true" - || env::var("RUSTC_DEBUGINFO_LINES").unwrap_or_default() == "true"; - build.debug(any_debug); - - let syms = [ - "backtrace_full", - "backtrace_dwarf_add", - "backtrace_initialize", - "backtrace_pcinfo", - "backtrace_syminfo", - "backtrace_get_view", - "backtrace_release_view", - "backtrace_alloc", - "backtrace_free", - "backtrace_vector_finish", - "backtrace_vector_grow", - "backtrace_vector_release", - "backtrace_close", - "backtrace_open", - "backtrace_print", - "backtrace_simple", - "backtrace_qsort", - "backtrace_create_state", - "backtrace_uncompress_zdebug", - // These should be `static` in C, but they aren't... - "macho_get_view", - "macho_symbol_type_relevant", - "macho_get_commands", - "macho_try_dsym", - "macho_try_dwarf", - "macho_get_addr_range", - "macho_get_uuid", - "macho_add", - "macho_add_symtab", - "macho_file_to_host_u64", - "macho_file_to_host_u32", - "macho_file_to_host_u16", - ]; - let prefix = if cfg!(feature = "rustc-dep-of-std") { - println!("cargo:rustc-cfg=rdos"); - "__rdos_" - } else { - println!("cargo:rustc-cfg=rbt"); - "__rbt_" - }; - for sym in syms.iter() { - build.define(sym, &format!("{}{}", prefix, sym)[..]); - } - - build.compile("backtrace"); -} - -// The `dl_iterate_phdr` API was added in Android API 21+ (according to #227), -// so if we can dynamically detect an appropriately configured C compiler for -// that then let's enable the `dl_iterate_phdr` API, otherwise Android just -// won't have any information. -fn maybe_enable_dl_iterate_phdr_android(build: &mut cc::Build) { - let expansion = cc::Build::new().file("src/android-api.c").expand(); - let expansion = match std::str::from_utf8(&expansion) { - Ok(s) => s, - Err(_) => return, - }; - println!("expanded android version detection:\n{}", expansion); - let marker = "APIVERSION"; - let i = match expansion.find(marker) { - Some(i) => i, - None => return, - }; - let version = match expansion[i + marker.len() + 1..].split_whitespace().next() { - Some(s) => s, - None => return, - }; - let version = match version.parse::() { - Ok(n) => n, - Err(_) => return, - }; - if version >= 21 { - build.define("HAVE_DL_ITERATE_PHDR", "1"); - } -} diff --git a/crux-mir/lib/backtrace/crates/backtrace-sys/src/lib.rs b/crux-mir/lib/backtrace/crates/backtrace-sys/src/lib.rs deleted file mode 100644 index 98a54c30b..000000000 --- a/crux-mir/lib/backtrace/crates/backtrace-sys/src/lib.rs +++ /dev/null @@ -1,58 +0,0 @@ -#![allow(bad_style)] -#![no_std] - -extern crate libc; - -#[cfg(not(empty))] -pub use self::bindings::*; -#[cfg(not(empty))] -mod bindings { - use libc::{c_char, c_int, c_void, uintptr_t}; - - pub type backtrace_syminfo_callback = extern "C" fn( - data: *mut c_void, - pc: uintptr_t, - symname: *const c_char, - symval: uintptr_t, - symsize: uintptr_t, - ); - pub type backtrace_full_callback = extern "C" fn( - data: *mut c_void, - pc: uintptr_t, - filename: *const c_char, - lineno: c_int, - function: *const c_char, - ) -> c_int; - pub type backtrace_error_callback = - extern "C" fn(data: *mut c_void, msg: *const c_char, errnum: c_int); - pub enum backtrace_state {} - - extern "C" { - #[cfg_attr(rdos, link_name = "__rdos_backtrace_create_state")] - #[cfg_attr(rbt, link_name = "__rbt_backtrace_create_state")] - pub fn backtrace_create_state( - filename: *const c_char, - threaded: c_int, - error: backtrace_error_callback, - data: *mut c_void, - ) -> *mut backtrace_state; - #[cfg_attr(rdos, link_name = "__rdos_backtrace_syminfo")] - #[cfg_attr(rbt, link_name = "__rbt_backtrace_syminfo")] - pub fn backtrace_syminfo( - state: *mut backtrace_state, - addr: uintptr_t, - cb: backtrace_syminfo_callback, - error: backtrace_error_callback, - data: *mut c_void, - ) -> c_int; - #[cfg_attr(rdos, link_name = "__rdos_backtrace_pcinfo")] - #[cfg_attr(rbt, link_name = "__rbt_backtrace_pcinfo")] - pub fn backtrace_pcinfo( - state: *mut backtrace_state, - addr: uintptr_t, - cb: backtrace_full_callback, - error: backtrace_error_callback, - data: *mut c_void, - ) -> c_int; - } -} diff --git a/crux-mir/lib/backtrace/crates/cpp_smoke_test/tests/smoke.rs b/crux-mir/lib/backtrace/crates/cpp_smoke_test/tests/smoke.rs index 3257c98e0..b9aef47f5 100644 --- a/crux-mir/lib/backtrace/crates/cpp_smoke_test/tests/smoke.rs +++ b/crux-mir/lib/backtrace/crates/cpp_smoke_test/tests/smoke.rs @@ -9,7 +9,6 @@ extern "C" { #[test] #[ignore] // fixme(fitzgen/cpp_demangle#73) -#[cfg(not(target_os = "windows"))] fn smoke_test_cpp() { static RAN_ASSERTS: AtomicBool = AtomicBool::new(false); diff --git a/crux-mir/lib/backtrace/crates/debuglink/Cargo.toml b/crux-mir/lib/backtrace/crates/debuglink/Cargo.toml new file mode 100644 index 000000000..6b55b1394 --- /dev/null +++ b/crux-mir/lib/backtrace/crates/debuglink/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "debuglink" +version = "0.1.0" +edition = "2018" + +[dependencies] +backtrace = { path = "../.." } diff --git a/crux-mir/lib/backtrace/crates/debuglink/src/main.rs b/crux-mir/lib/backtrace/crates/debuglink/src/main.rs new file mode 100644 index 000000000..99265ae9a --- /dev/null +++ b/crux-mir/lib/backtrace/crates/debuglink/src/main.rs @@ -0,0 +1,34 @@ +// Test that the debuginfo is being found by checking that the +// backtrace contains `main` and that the source filename uses +// the path given in the command line arguments. +// +// For dwz tests, this assumes that the path string will be moved into +// the dwz file. +fn main() { + let crate_dir = std::env::args().skip(1).next().unwrap(); + let expect = std::path::Path::new(&crate_dir).join("src/main.rs"); + + let bt = backtrace::Backtrace::new(); + println!("{:?}", bt); + + let mut found_main = false; + + for frame in bt.frames() { + let symbols = frame.symbols(); + if symbols.is_empty() { + continue; + } + + if let Some(name) = symbols[0].name() { + let name = format!("{:#}", name); + if name == "debuglink::main" { + found_main = true; + let filename = symbols[0].filename().unwrap(); + assert_eq!(filename, expect); + break; + } + } + } + + assert!(found_main); +} diff --git a/crux-mir/lib/backtrace/crates/dylib-dep/Cargo.toml b/crux-mir/lib/backtrace/crates/dylib-dep/Cargo.toml new file mode 100644 index 000000000..c3d4a8c2f --- /dev/null +++ b/crux-mir/lib/backtrace/crates/dylib-dep/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "dylib-dep" +version = "0.1.0" +edition = "2018" +authors = [] +publish = false + +[lib] +name = "dylib_dep" +crate-type = ["cdylib", "rlib"] diff --git a/crux-mir/lib/backtrace/crates/dylib-dep/src/lib.rs b/crux-mir/lib/backtrace/crates/dylib-dep/src/lib.rs new file mode 100644 index 000000000..201807797 --- /dev/null +++ b/crux-mir/lib/backtrace/crates/dylib-dep/src/lib.rs @@ -0,0 +1,14 @@ +#![allow(improper_ctypes_definitions)] + +type Pos = (&'static str, u32); + +macro_rules! pos { + () => { + (file!(), line!()) + }; +} + +#[no_mangle] +pub extern "C" fn foo(outer: Pos, inner: fn(Pos, Pos)) { + inner(outer, pos!()); +} diff --git a/crux-mir/lib/backtrace/crates/line-tables-only/Cargo.toml b/crux-mir/lib/backtrace/crates/line-tables-only/Cargo.toml new file mode 100644 index 000000000..e2967d3d3 --- /dev/null +++ b/crux-mir/lib/backtrace/crates/line-tables-only/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "line-tables-only" +version = "0.1.0" +edition = "2018" + +[build-dependencies] +cc = "1.0" + +[dependencies] +libc = { version = "0.2", default-features = false } + +[dependencies.backtrace] +path = "../.." +features = [ + 'libunwind', + 'std', +] + +[features] +libbacktrace = ['backtrace/libbacktrace'] +gimli-symbolize = ['backtrace/gimli-symbolize'] diff --git a/crux-mir/lib/backtrace/crates/line-tables-only/build.rs b/crux-mir/lib/backtrace/crates/line-tables-only/build.rs new file mode 100644 index 000000000..125fb7645 --- /dev/null +++ b/crux-mir/lib/backtrace/crates/line-tables-only/build.rs @@ -0,0 +1,10 @@ +fn main() { + println!("cargo:rerun-if-changed=src/callback.c"); + + cc::Build::new() + .opt_level(0) + .debug(false) + .flag("-g1") + .file("src/callback.c") + .compile("libcallback.a"); +} diff --git a/crux-mir/lib/backtrace/crates/line-tables-only/src/callback.c b/crux-mir/lib/backtrace/crates/line-tables-only/src/callback.c new file mode 100644 index 000000000..c9d5d072a --- /dev/null +++ b/crux-mir/lib/backtrace/crates/line-tables-only/src/callback.c @@ -0,0 +1,14 @@ + +typedef void (*callback) (void *data); + +void baz(callback cb, void *data) { + cb(data); +} + +void bar(callback cb, void *data) { + baz(cb, data); +} + +void foo(callback cb, void *data) { + bar(cb, data); +} diff --git a/crux-mir/lib/backtrace/crates/line-tables-only/src/lib.rs b/crux-mir/lib/backtrace/crates/line-tables-only/src/lib.rs new file mode 100644 index 000000000..bd5afcb3a --- /dev/null +++ b/crux-mir/lib/backtrace/crates/line-tables-only/src/lib.rs @@ -0,0 +1,57 @@ +#[cfg(test)] +mod tests { + use std::path::Path; + use backtrace::Backtrace; + use libc::c_void; + + pub type Callback = extern "C" fn(data: *mut c_void); + + extern "C" { + fn foo(cb: Callback, data: *mut c_void); + } + + extern "C" fn store_backtrace(data: *mut c_void) { + let bt = backtrace::Backtrace::new(); + unsafe { *(data as *mut Option) = Some(bt) }; + } + + fn assert_contains(backtrace: &Backtrace, + expected_name: &str, + expected_file: &str, + expected_line: u32) { + + let expected_file = Path::new(expected_file); + + for frame in backtrace.frames() { + for symbol in frame.symbols() { + if let Some(name) = symbol.name() { + if name.as_bytes() == expected_name.as_bytes() { + assert!(symbol.filename().unwrap().ends_with(expected_file)); + assert_eq!(symbol.lineno(), Some(expected_line)); + return; + } + } + } + } + + panic!("symbol {:?} not found in backtrace: {:?}", expected_name, backtrace); + } + + /// Verifies that when debug info includes only lines tables the generated + /// backtrace is still generated successfully. The test exercises behaviour + /// that failed previously when compiling with clang -g1. + /// + /// The test case uses C rather than rust, since at that time when it was + /// written the debug info generated at level 1 in rustc was essentially + /// the same as at level 2. + #[test] + #[cfg_attr(windows, ignore)] + fn backtrace_works_with_line_tables_only() { + let mut backtrace: Option = None; + unsafe { foo(store_backtrace, &mut backtrace as *mut _ as *mut c_void) }; + let backtrace = backtrace.expect("backtrace"); + assert_contains(&backtrace, "foo", "src/callback.c", 13); + assert_contains(&backtrace, "bar", "src/callback.c", 9); + assert_contains(&backtrace, "baz", "src/callback.c", 5); + } +} diff --git a/crux-mir/lib/backtrace/crates/macos_frames_test/Cargo.toml b/crux-mir/lib/backtrace/crates/macos_frames_test/Cargo.toml new file mode 100644 index 000000000..278d51e79 --- /dev/null +++ b/crux-mir/lib/backtrace/crates/macos_frames_test/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "macos_frames_test" +version = "0.1.0" +authors = ["Aaron Hill "] +edition = "2018" + +[dependencies.backtrace] +path = "../.." diff --git a/crux-mir/lib/backtrace/crates/macos_frames_test/src/lib.rs b/crux-mir/lib/backtrace/crates/macos_frames_test/src/lib.rs new file mode 100644 index 000000000..65e2cc340 --- /dev/null +++ b/crux-mir/lib/backtrace/crates/macos_frames_test/src/lib.rs @@ -0,0 +1 @@ +// intentionally blank diff --git a/crux-mir/lib/backtrace/crates/macos_frames_test/tests/main.rs b/crux-mir/lib/backtrace/crates/macos_frames_test/tests/main.rs new file mode 100644 index 000000000..f0e905b24 --- /dev/null +++ b/crux-mir/lib/backtrace/crates/macos_frames_test/tests/main.rs @@ -0,0 +1,30 @@ +// Based on from https://github.com/rust-lang/rust/blob/2cb0b8582ebbf9784db9cec06fff517badbf4553/src/test/ui/issues/issue-45731.rs +// This needs to go in a crate by itself, since it modifies the dSYM for the entire test +// output directory. +// +// Note that this crate is *not* part of the overall `backtrace-rs` workspace, +// so that it gets its own 'target' directory. We manually invoke this test +// in .github/workflows/main.yml by passing `--manifest-path` to Cargo +#[test] +#[cfg(target_os = "macos")] +fn backtrace_no_dsym() { + use std::{env, fs}; + + // Find our dSYM and replace the DWARF binary with an empty file + let mut dsym_path = env::current_exe().unwrap(); + let executable_name = dsym_path.file_name().unwrap().to_str().unwrap().to_string(); + assert!(dsym_path.pop()); // Pop executable + dsym_path.push(format!( + "{}.dSYM/Contents/Resources/DWARF/{0}", + executable_name + )); + let _ = fs::OpenOptions::new() + .read(false) + .write(true) + .truncate(true) + .create(false) + .open(&dsym_path) + .unwrap(); + + backtrace::Backtrace::new(); +} diff --git a/crux-mir/lib/backtrace/crates/without_debuginfo/Cargo.toml b/crux-mir/lib/backtrace/crates/without_debuginfo/Cargo.toml index 70dc204b9..19d76cbec 100644 --- a/crux-mir/lib/backtrace/crates/without_debuginfo/Cargo.toml +++ b/crux-mir/lib/backtrace/crates/without_debuginfo/Cargo.toml @@ -7,17 +7,7 @@ edition = "2018" [dependencies.backtrace] path = "../.." default-features = false -features = [ - # make sure a trace can be acquired - 'libunwind', - 'dbghelp', - - # Allow fallback to dladdr - 'dladdr', - - # Yes, we have `std` - 'std', -] +features = ['std'] [profile.dev] debug = false @@ -27,5 +17,4 @@ debug = false [features] libbacktrace = ['backtrace/libbacktrace'] -coresymbolication = ['backtrace/coresymbolication'] gimli-symbolize = ['backtrace/gimli-symbolize'] diff --git a/crux-mir/lib/backtrace/crates/without_debuginfo/tests/smoke.rs b/crux-mir/lib/backtrace/crates/without_debuginfo/tests/smoke.rs index 85be656c2..5a0dfea15 100644 --- a/crux-mir/lib/backtrace/crates/without_debuginfo/tests/smoke.rs +++ b/crux-mir/lib/backtrace/crates/without_debuginfo/tests/smoke.rs @@ -2,7 +2,8 @@ fn all_frames_have_symbols() { println!("{:?}", backtrace::Backtrace::new()); - let mut all_have_symbols = true; + let mut missing_symbols = 0; + let mut has_symbols = 0; backtrace::trace(|frame| { let mut any = false; backtrace::resolve_frame(frame, |sym| { @@ -10,10 +11,34 @@ fn all_frames_have_symbols() { any = true; } }); - if !any && !frame.ip().is_null() { - all_have_symbols = false; + if any { + has_symbols += 1; + } else if !frame.ip().is_null() { + missing_symbols += 1; } true }); - assert!(all_have_symbols); + + // FIXME(#346) currently on MinGW we can't symbolize kernel32.dll and other + // system libraries, which means we miss the last few symbols. + if cfg!(windows) && cfg!(target_env = "gnu") { + assert!(missing_symbols < has_symbols && has_symbols > 4); + } else { + assert_eq!(missing_symbols, 0); + } +} + +#[test] +fn all_frames_have_module_base_address() { + let mut missing_base_addresses = 0; + backtrace::trace(|frame| { + if frame.module_base_address().is_none() { + missing_base_addresses += 1; + } + true + }); + + if cfg!(windows) { + assert_eq!(missing_base_addresses, 0); + } } diff --git a/crux-mir/lib/backtrace/examples/backtrace.rs b/crux-mir/lib/backtrace/examples/backtrace.rs index 7f9042ed7..7ff6cd39e 100644 --- a/crux-mir/lib/backtrace/examples/backtrace.rs +++ b/crux-mir/lib/backtrace/examples/backtrace.rs @@ -1,5 +1,3 @@ -extern crate backtrace; - use backtrace::Backtrace; fn main() { diff --git a/crux-mir/lib/backtrace/examples/raw.rs b/crux-mir/lib/backtrace/examples/raw.rs index b43cab8ba..d96a127a2 100644 --- a/crux-mir/lib/backtrace/examples/raw.rs +++ b/crux-mir/lib/backtrace/examples/raw.rs @@ -1,5 +1,3 @@ -extern crate backtrace; - fn main() { foo(); } diff --git a/crux-mir/lib/backtrace/crates/backtrace-sys/src/android-api.c b/crux-mir/lib/backtrace/src/android-api.c similarity index 100% rename from crux-mir/lib/backtrace/crates/backtrace-sys/src/android-api.c rename to crux-mir/lib/backtrace/src/android-api.c diff --git a/crux-mir/lib/backtrace/src/backtrace/dbghelp.rs b/crux-mir/lib/backtrace/src/backtrace/dbghelp.rs index 40de1437c..ba0f05f3b 100644 --- a/crux-mir/lib/backtrace/src/backtrace/dbghelp.rs +++ b/crux-mir/lib/backtrace/src/backtrace/dbghelp.rs @@ -1,13 +1,3 @@ -// Copyright 2014 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - //! Backtrace strategy for MSVC platforms. //! //! This module contains the ability to generate a backtrace on MSVC using one @@ -21,17 +11,22 @@ #![allow(bad_style)] -use crate::dbghelp; -use crate::windows::*; +use super::super::{dbghelp, windows::*}; use core::ffi::c_void; use core::mem; #[derive(Clone, Copy)] -pub enum Frame { +pub enum StackFrame { New(STACKFRAME_EX), Old(STACKFRAME64), } +#[derive(Clone, Copy)] +pub struct Frame { + pub(crate) stack_frame: StackFrame, + base_address: *mut c_void, +} + // we're just sending around raw pointers and reading them, never interpreting // them so this should be safe to both send and share across threads. unsafe impl Send for Frame {} @@ -42,35 +37,50 @@ impl Frame { self.addr_pc().Offset as *mut _ } + pub fn sp(&self) -> *mut c_void { + self.addr_stack().Offset as *mut _ + } + pub fn symbol_address(&self) -> *mut c_void { self.ip() } + pub fn module_base_address(&self) -> Option<*mut c_void> { + Some(self.base_address) + } + fn addr_pc(&self) -> &ADDRESS64 { - match self { - Frame::New(new) => &new.AddrPC, - Frame::Old(old) => &old.AddrPC, + match self.stack_frame { + StackFrame::New(ref new) => &new.AddrPC, + StackFrame::Old(ref old) => &old.AddrPC, } } fn addr_pc_mut(&mut self) -> &mut ADDRESS64 { - match self { - Frame::New(new) => &mut new.AddrPC, - Frame::Old(old) => &mut old.AddrPC, + match self.stack_frame { + StackFrame::New(ref mut new) => &mut new.AddrPC, + StackFrame::Old(ref mut old) => &mut old.AddrPC, } } fn addr_frame_mut(&mut self) -> &mut ADDRESS64 { - match self { - Frame::New(new) => &mut new.AddrFrame, - Frame::Old(old) => &mut old.AddrFrame, + match self.stack_frame { + StackFrame::New(ref mut new) => &mut new.AddrFrame, + StackFrame::Old(ref mut old) => &mut old.AddrFrame, + } + } + + fn addr_stack(&self) -> &ADDRESS64 { + match self.stack_frame { + StackFrame::New(ref new) => &new.AddrStack, + StackFrame::Old(ref old) => &old.AddrStack, } } fn addr_stack_mut(&mut self) -> &mut ADDRESS64 { - match self { - Frame::New(new) => &mut new.AddrStack, - Frame::Old(old) => &mut old.AddrStack, + match self.stack_frame { + StackFrame::New(ref mut new) => &mut new.AddrStack, + StackFrame::Old(ref mut old) => &mut old.AddrStack, } } } @@ -79,7 +89,7 @@ impl Frame { struct MyContext(CONTEXT); #[inline(always)] -pub unsafe fn trace(cb: &mut FnMut(&super::Frame) -> bool) { +pub unsafe fn trace(cb: &mut dyn FnMut(&super::Frame) -> bool) { // Allocate necessary structures for doing the stack walk let process = GetCurrentProcess(); let thread = GetCurrentThread(); @@ -121,16 +131,23 @@ pub unsafe fn trace(cb: &mut FnMut(&super::Frame) -> bool) { } } + let process_handle = GetCurrentProcess(); + // Attempt to use `StackWalkEx` if we can, but fall back to `StackWalk64` // since it's in theory supported on more systems. match (*dbghelp.dbghelp()).StackWalkEx() { Some(StackWalkEx) => { + let mut inner: STACKFRAME_EX = mem::zeroed(); + inner.StackFrameSize = mem::size_of::() as DWORD; let mut frame = super::Frame { - inner: Frame::New(mem::zeroed()), + inner: Frame { + stack_frame: StackFrame::New(inner), + base_address: 0 as _, + }, }; let image = init_frame(&mut frame.inner, &context.0); - let frame_ptr = match &mut frame.inner { - Frame::New(ptr) => ptr as *mut STACKFRAME_EX, + let frame_ptr = match &mut frame.inner.stack_frame { + StackFrame::New(ptr) => ptr as *mut STACKFRAME_EX, _ => unreachable!(), }; @@ -147,6 +164,8 @@ pub unsafe fn trace(cb: &mut FnMut(&super::Frame) -> bool) { 0, ) == TRUE { + frame.inner.base_address = get_module_base(process_handle, frame.ip() as _) as _; + if !cb(&frame) { break; } @@ -154,11 +173,14 @@ pub unsafe fn trace(cb: &mut FnMut(&super::Frame) -> bool) { } None => { let mut frame = super::Frame { - inner: Frame::Old(mem::zeroed()), + inner: Frame { + stack_frame: StackFrame::Old(mem::zeroed()), + base_address: 0 as _, + }, }; let image = init_frame(&mut frame.inner, &context.0); - let frame_ptr = match &mut frame.inner { - Frame::Old(ptr) => ptr as *mut STACKFRAME64, + let frame_ptr = match &mut frame.inner.stack_frame { + StackFrame::Old(ptr) => ptr as *mut STACKFRAME64, _ => unreachable!(), }; @@ -174,6 +196,8 @@ pub unsafe fn trace(cb: &mut FnMut(&super::Frame) -> bool) { None, ) == TRUE { + frame.inner.base_address = get_module_base(process_handle, frame.ip() as _) as _; + if !cb(&frame) { break; } diff --git a/crux-mir/lib/backtrace/src/backtrace/libunwind.rs b/crux-mir/lib/backtrace/src/backtrace/libunwind.rs index 21af00c17..ef77edda5 100644 --- a/crux-mir/lib/backtrace/src/backtrace/libunwind.rs +++ b/crux-mir/lib/backtrace/src/backtrace/libunwind.rs @@ -1,13 +1,3 @@ -// Copyright 2014-2015 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - //! Backtrace support using libunwind/gcc_s/etc APIs. //! //! This module contains the ability to unwind the stack using libunwind-style @@ -25,12 +15,14 @@ //! //! This is the default unwinding API for all non-Windows platforms currently. +use super::super::Bomb; use core::ffi::c_void; pub enum Frame { Raw(*mut uw::_Unwind_Context), Cloned { ip: *mut c_void, + sp: *mut c_void, symbol_address: *mut c_void, }, } @@ -51,50 +43,65 @@ impl Frame { unsafe { uw::_Unwind_GetIP(ctx) as *mut c_void } } + pub fn sp(&self) -> *mut c_void { + match *self { + Frame::Raw(ctx) => unsafe { uw::get_sp(ctx) as *mut c_void }, + Frame::Cloned { sp, .. } => sp, + } + } + pub fn symbol_address(&self) -> *mut c_void { if let Frame::Cloned { symbol_address, .. } = *self { return symbol_address; } - // It seems that on OSX `_Unwind_FindEnclosingFunction` returns a - // pointer to... something that's unclear. It's definitely not always - // the enclosing function for whatever reason. It's not entirely clear - // to me what's going on here, so pessimize this for now and just always + // The macOS linker emits a "compact" unwind table that only includes an + // entry for a function if that function either has an LSDA or its + // encoding differs from that of the previous entry. Consequently, on + // macOS, `_Unwind_FindEnclosingFunction` is unreliable (it can return a + // pointer to some totally unrelated function). Instead, we just always // return the ip. // - // Note the `skip_inner_frames.rs` test is skipped on OSX due to this - // clause, and if this is fixed that test in theory can be run on OSX! + // https://github.com/rust-lang/rust/issues/74771#issuecomment-664056788 + // + // Note the `skip_inner_frames.rs` test is skipped on macOS due to this + // clause, and if this is fixed that test in theory can be run on macOS! if cfg!(target_os = "macos") || cfg!(target_os = "ios") { self.ip() } else { unsafe { uw::_Unwind_FindEnclosingFunction(self.ip()) } } } + + pub fn module_base_address(&self) -> Option<*mut c_void> { + None + } } impl Clone for Frame { fn clone(&self) -> Frame { Frame::Cloned { ip: self.ip(), + sp: self.sp(), symbol_address: self.symbol_address(), } } } #[inline(always)] -pub unsafe fn trace(mut cb: &mut FnMut(&super::Frame) -> bool) { +pub unsafe fn trace(mut cb: &mut dyn FnMut(&super::Frame) -> bool) { uw::_Unwind_Backtrace(trace_fn, &mut cb as *mut _ as *mut _); extern "C" fn trace_fn( ctx: *mut uw::_Unwind_Context, arg: *mut c_void, ) -> uw::_Unwind_Reason_Code { - let cb = unsafe { &mut *(arg as *mut &mut FnMut(&super::Frame) -> bool) }; + let cb = unsafe { &mut *(arg as *mut &mut dyn FnMut(&super::Frame) -> bool) }; let cx = super::Frame { inner: Frame::Raw(ctx), }; - let mut bomb = crate::Bomb { enabled: true }; + let mut bomb = Bomb { enabled: true }; let keep_going = cb(&cx); bomb.enabled = false; @@ -139,93 +146,122 @@ mod uw { extern "C" fn(ctx: *mut _Unwind_Context, arg: *mut c_void) -> _Unwind_Reason_Code; extern "C" { - // No native _Unwind_Backtrace on iOS - #[cfg(not(all(target_os = "ios", target_arch = "arm")))] pub fn _Unwind_Backtrace( trace: _Unwind_Trace_Fn, trace_argument: *mut c_void, ) -> _Unwind_Reason_Code; + } + cfg_if::cfg_if! { // available since GCC 4.2.0, should be fine for our purpose - #[cfg(all( + if #[cfg(all( not(all(target_os = "android", target_arch = "arm")), not(all(target_os = "freebsd", target_arch = "arm")), - not(all(target_os = "linux", target_arch = "arm")) - ))] - pub fn _Unwind_GetIP(ctx: *mut _Unwind_Context) -> libc::uintptr_t; + not(all(target_os = "linux", target_arch = "arm")), + not(all(target_os = "horizon", target_arch = "arm")) + ))] { + extern "C" { + pub fn _Unwind_GetIP(ctx: *mut _Unwind_Context) -> libc::uintptr_t; + pub fn _Unwind_FindEnclosingFunction(pc: *mut c_void) -> *mut c_void; - #[cfg(all( - not(target_os = "android"), - not(all(target_os = "freebsd", target_arch = "arm")), - not(all(target_os = "linux", target_arch = "arm")) - ))] - pub fn _Unwind_FindEnclosingFunction(pc: *mut c_void) -> *mut c_void; - } + #[cfg(not(all(target_os = "linux", target_arch = "s390x")))] + // This function is a misnomer: rather than getting this frame's + // Canonical Frame Address (aka the caller frame's SP) it + // returns this frame's SP. + // + // https://github.com/libunwind/libunwind/blob/d32956507cf29d9b1a98a8bce53c78623908f4fe/src/unwind/GetCFA.c#L28-L35 + #[link_name = "_Unwind_GetCFA"] + pub fn get_sp(ctx: *mut _Unwind_Context) -> libc::uintptr_t; - // On android, the function _Unwind_GetIP is a macro, and this is the - // expansion of the macro. This is all copy/pasted directly from the - // header file with the definition of _Unwind_GetIP. - #[cfg(any( - all(target_os = "android", target_arch = "arm"), - all(target_os = "freebsd", target_arch = "arm"), - all(target_os = "linux", target_arch = "arm") - ))] - pub unsafe fn _Unwind_GetIP(ctx: *mut _Unwind_Context) -> libc::uintptr_t { - #[repr(C)] - enum _Unwind_VRS_Result { - _UVRSR_OK = 0, - _UVRSR_NOT_IMPLEMENTED = 1, - _UVRSR_FAILED = 2, - } - #[repr(C)] - enum _Unwind_VRS_RegClass { - _UVRSC_CORE = 0, - _UVRSC_VFP = 1, - _UVRSC_FPA = 2, - _UVRSC_WMMXD = 3, - _UVRSC_WMMXC = 4, - } - #[repr(C)] - enum _Unwind_VRS_DataRepresentation { - _UVRSD_UINT32 = 0, - _UVRSD_VFPX = 1, - _UVRSD_FPAX = 2, - _UVRSD_UINT64 = 3, - _UVRSD_FLOAT = 4, - _UVRSD_DOUBLE = 5, - } + } - type _Unwind_Word = libc::c_uint; - extern "C" { - fn _Unwind_VRS_Get( - ctx: *mut _Unwind_Context, - klass: _Unwind_VRS_RegClass, - word: _Unwind_Word, - repr: _Unwind_VRS_DataRepresentation, - data: *mut c_void, - ) -> _Unwind_VRS_Result; - } + // s390x uses a biased CFA value, therefore we need to use + // _Unwind_GetGR to get the stack pointer register (%r15) + // instead of relying on _Unwind_GetCFA. + #[cfg(all(target_os = "linux", target_arch = "s390x"))] + pub unsafe fn get_sp(ctx: *mut _Unwind_Context) -> libc::uintptr_t { + extern "C" { + pub fn _Unwind_GetGR(ctx: *mut _Unwind_Context, index: libc::c_int) -> libc::uintptr_t; + } + _Unwind_GetGR(ctx, 15) + } + } else { + // On android and arm, the function `_Unwind_GetIP` and a bunch of + // others are macros, so we define functions containing the + // expansion of the macros. + // + // TODO: link to the header file that defines these macros, if you + // can find it. (I, fitzgen, cannot find the header file that some + // of these macro expansions were originally borrowed from.) + #[repr(C)] + enum _Unwind_VRS_Result { + _UVRSR_OK = 0, + _UVRSR_NOT_IMPLEMENTED = 1, + _UVRSR_FAILED = 2, + } + #[repr(C)] + enum _Unwind_VRS_RegClass { + _UVRSC_CORE = 0, + _UVRSC_VFP = 1, + _UVRSC_FPA = 2, + _UVRSC_WMMXD = 3, + _UVRSC_WMMXC = 4, + } + #[repr(C)] + enum _Unwind_VRS_DataRepresentation { + _UVRSD_UINT32 = 0, + _UVRSD_VFPX = 1, + _UVRSD_FPAX = 2, + _UVRSD_UINT64 = 3, + _UVRSD_FLOAT = 4, + _UVRSD_DOUBLE = 5, + } - let mut val: _Unwind_Word = 0; - let ptr = &mut val as *mut _Unwind_Word; - let _ = _Unwind_VRS_Get( - ctx, - _Unwind_VRS_RegClass::_UVRSC_CORE, - 15, - _Unwind_VRS_DataRepresentation::_UVRSD_UINT32, - ptr as *mut c_void, - ); - (val & !1) as libc::uintptr_t - } + type _Unwind_Word = libc::c_uint; + extern "C" { + fn _Unwind_VRS_Get( + ctx: *mut _Unwind_Context, + klass: _Unwind_VRS_RegClass, + word: _Unwind_Word, + repr: _Unwind_VRS_DataRepresentation, + data: *mut c_void, + ) -> _Unwind_VRS_Result; + } + + pub unsafe fn _Unwind_GetIP(ctx: *mut _Unwind_Context) -> libc::uintptr_t { + let mut val: _Unwind_Word = 0; + let ptr = &mut val as *mut _Unwind_Word; + let _ = _Unwind_VRS_Get( + ctx, + _Unwind_VRS_RegClass::_UVRSC_CORE, + 15, + _Unwind_VRS_DataRepresentation::_UVRSD_UINT32, + ptr as *mut c_void, + ); + (val & !1) as libc::uintptr_t + } + + // R13 is the stack pointer on arm. + const SP: _Unwind_Word = 13; - // This function also doesn't exist on Android or ARM/Linux, so make it - // a no-op - #[cfg(any( - target_os = "android", - all(target_os = "freebsd", target_arch = "arm"), - all(target_os = "linux", target_arch = "arm") - ))] - pub unsafe fn _Unwind_FindEnclosingFunction(pc: *mut c_void) -> *mut c_void { - pc + pub unsafe fn get_sp(ctx: *mut _Unwind_Context) -> libc::uintptr_t { + let mut val: _Unwind_Word = 0; + let ptr = &mut val as *mut _Unwind_Word; + let _ = _Unwind_VRS_Get( + ctx, + _Unwind_VRS_RegClass::_UVRSC_CORE, + SP, + _Unwind_VRS_DataRepresentation::_UVRSD_UINT32, + ptr as *mut c_void, + ); + val as libc::uintptr_t + } + + // This function also doesn't exist on Android or ARM/Linux, so make it + // a no-op. + pub unsafe fn _Unwind_FindEnclosingFunction(pc: *mut c_void) -> *mut c_void { + pc + } + } } } diff --git a/crux-mir/lib/backtrace/src/backtrace/miri.rs b/crux-mir/lib/backtrace/src/backtrace/miri.rs new file mode 100644 index 000000000..f8c496428 --- /dev/null +++ b/crux-mir/lib/backtrace/src/backtrace/miri.rs @@ -0,0 +1,109 @@ +use alloc::boxed::Box; +use alloc::vec::Vec; +use core::ffi::c_void; + +extern "Rust" { + fn miri_backtrace_size(flags: u64) -> usize; + fn miri_get_backtrace(flags: u64, buf: *mut *mut ()); + fn miri_resolve_frame(ptr: *mut (), flags: u64) -> MiriFrame; + fn miri_resolve_frame_names(ptr: *mut (), flags: u64, name_buf: *mut u8, filename_buf: *mut u8); +} + +#[repr(C)] +pub struct MiriFrame { + pub name_len: usize, + pub filename_len: usize, + pub lineno: u32, + pub colno: u32, + pub fn_ptr: *mut c_void, +} + +#[derive(Clone, Debug)] +pub struct FullMiriFrame { + pub name: Box<[u8]>, + pub filename: Box<[u8]>, + pub lineno: u32, + pub colno: u32, + pub fn_ptr: *mut c_void, +} + +#[derive(Debug, Clone)] +pub struct Frame { + pub addr: *mut c_void, + pub inner: FullMiriFrame, +} + +// SAFETY: Miri guarantees that the returned pointer +// can be used from any thread. +unsafe impl Send for Frame {} +unsafe impl Sync for Frame {} + +impl Frame { + pub fn ip(&self) -> *mut c_void { + self.addr + } + + pub fn sp(&self) -> *mut c_void { + core::ptr::null_mut() + } + + pub fn symbol_address(&self) -> *mut c_void { + self.inner.fn_ptr + } + + pub fn module_base_address(&self) -> Option<*mut c_void> { + None + } +} + +pub fn trace bool>(cb: F) { + // SAFETY: Miri guarantees that the backtrace API functions + // can be called from any thread. + unsafe { trace_unsynchronized(cb) }; +} + +pub fn resolve_addr(ptr: *mut c_void) -> Frame { + // SAFETY: Miri will stop execution with an error if this pointer + // is invalid. + let frame = unsafe { miri_resolve_frame(ptr as *mut (), 1) }; + + let mut name = Vec::with_capacity(frame.name_len); + let mut filename = Vec::with_capacity(frame.filename_len); + + // SAFETY: name and filename have been allocated with the amount + // of memory miri has asked for, and miri guarantees it will initialize it + unsafe { + miri_resolve_frame_names(ptr as *mut (), 0, name.as_mut_ptr(), filename.as_mut_ptr()); + + name.set_len(frame.name_len); + filename.set_len(frame.filename_len); + } + + Frame { + addr: ptr, + inner: FullMiriFrame { + name: name.into(), + filename: filename.into(), + lineno: frame.lineno, + colno: frame.colno, + fn_ptr: frame.fn_ptr, + }, + } +} + +unsafe fn trace_unsynchronized bool>(mut cb: F) { + let len = miri_backtrace_size(0); + + let mut frames = Vec::with_capacity(len); + + miri_get_backtrace(1, frames.as_mut_ptr()); + + frames.set_len(len); + + for ptr in frames.iter() { + let frame = resolve_addr(*ptr as *mut c_void); + if !cb(&super::Frame { inner: frame }) { + return; + } + } +} diff --git a/crux-mir/lib/backtrace/src/backtrace/mod.rs b/crux-mir/lib/backtrace/src/backtrace/mod.rs index 204974ec4..93355d744 100644 --- a/crux-mir/lib/backtrace/src/backtrace/mod.rs +++ b/crux-mir/lib/backtrace/src/backtrace/mod.rs @@ -55,7 +55,7 @@ pub fn trace bool>(cb: F) { /// Same as `trace`, only unsafe as it's unsynchronized. /// -/// This function does not have synchronization guarentees but is available +/// This function does not have synchronization guarantees but is available /// when the `std` feature of this crate isn't compiled in. See the `trace` /// function for more documentation and examples. /// @@ -90,6 +90,14 @@ impl Frame { self.inner.ip() } + /// Returns the current stack pointer of this frame. + /// + /// In the case that a backend cannot recover the stack pointer for this + /// frame, a null pointer is returned. + pub fn sp(&self) -> *mut c_void { + self.inner.sp() + } + /// Returns the starting symbol address of the frame of this function. /// /// This will attempt to rewind the instruction pointer returned by `ip` to @@ -101,10 +109,15 @@ impl Frame { pub fn symbol_address(&self) -> *mut c_void { self.inner.symbol_address() } + + /// Returns the base address of the module to which the frame belongs. + pub fn module_base_address(&self) -> Option<*mut c_void> { + self.inner.module_base_address() + } } impl fmt::Debug for Frame { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Frame") .field("ip", &self.ip()) .field("symbol_address", &self.symbol_address()) @@ -113,13 +126,18 @@ impl fmt::Debug for Frame { } cfg_if::cfg_if! { - if #[cfg( + // This needs to come first, to ensure that + // Miri takes priority over the host platform + if #[cfg(miri)] { + pub(crate) mod miri; + use self::miri::trace as trace_imp; + pub(crate) use self::miri::Frame as FrameImp; + } else if #[cfg( any( all( unix, not(target_os = "emscripten"), not(all(target_os = "ios", target_arch = "arm")), - feature = "libunwind", ), all( target_env = "sgx", @@ -130,20 +148,12 @@ cfg_if::cfg_if! { mod libunwind; use self::libunwind::trace as trace_imp; pub(crate) use self::libunwind::Frame as FrameImp; - } else if #[cfg( - all( - unix, - not(target_os = "emscripten"), - feature = "unix-backtrace", - ) - )] { - mod unix_backtrace; - use self::unix_backtrace::trace as trace_imp; - pub(crate) use self::unix_backtrace::Frame as FrameImp; - } else if #[cfg(all(windows, feature = "dbghelp", not(target_vendor = "uwp")))] { + } else if #[cfg(all(windows, not(target_vendor = "uwp")))] { mod dbghelp; use self::dbghelp::trace as trace_imp; pub(crate) use self::dbghelp::Frame as FrameImp; + #[cfg(target_env = "msvc")] // only used in dbghelp symbolize + pub(crate) use self::dbghelp::StackFrame; } else { mod noop; use self::noop::trace as trace_imp; diff --git a/crux-mir/lib/backtrace/src/backtrace/noop.rs b/crux-mir/lib/backtrace/src/backtrace/noop.rs index bc2eb0d81..7bcea67aa 100644 --- a/crux-mir/lib/backtrace/src/backtrace/noop.rs +++ b/crux-mir/lib/backtrace/src/backtrace/noop.rs @@ -4,7 +4,7 @@ use core::ffi::c_void; #[inline(always)] -pub fn trace(_cb: &mut FnMut(&super::Frame) -> bool) {} +pub fn trace(_cb: &mut dyn FnMut(&super::Frame) -> bool) {} #[derive(Clone)] pub struct Frame; @@ -14,7 +14,15 @@ impl Frame { 0 as *mut _ } + pub fn sp(&self) -> *mut c_void { + 0 as *mut _ + } + pub fn symbol_address(&self) -> *mut c_void { 0 as *mut _ } + + pub fn module_base_address(&self) -> Option<*mut c_void> { + None + } } diff --git a/crux-mir/lib/backtrace/src/backtrace/unix_backtrace.rs b/crux-mir/lib/backtrace/src/backtrace/unix_backtrace.rs deleted file mode 100644 index 7e3375a6f..000000000 --- a/crux-mir/lib/backtrace/src/backtrace/unix_backtrace.rs +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2014-2015 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -//! Unwinding through the `backtrace` function provided in Unix. -//! -//! This is an alternative unwinding strategy for Unix platforms which don't -//! have support for libunwind but do have support for `backtrace`. Currently -//! there's not a whole lot of those though. This module is a relatively -//! straightforward binding of the `backtrace` API to the `Frame` API that we'd -//! like to have. - -use core::ffi::c_void; -use core::mem; -use libc::c_int; - -#[derive(Clone)] -pub struct Frame { - addr: usize, -} - -impl Frame { - pub fn ip(&self) -> *mut c_void { - self.addr as *mut c_void - } - pub fn symbol_address(&self) -> *mut c_void { - self.ip() - } -} - -extern "C" { - fn backtrace(buf: *mut *mut c_void, sz: c_int) -> c_int; -} - -#[inline(always)] -pub unsafe fn trace(cb: &mut FnMut(&super::Frame) -> bool) { - const SIZE: usize = 100; - - let mut buf: [*mut c_void; SIZE]; - let cnt; - - buf = mem::zeroed(); - cnt = backtrace(buf.as_mut_ptr(), SIZE as c_int); - - for addr in buf[..cnt as usize].iter() { - let cx = super::Frame { - inner: Frame { - addr: *addr as usize, - }, - }; - if !cb(&cx) { - return; - } - } -} diff --git a/crux-mir/lib/backtrace/src/capture.rs b/crux-mir/lib/backtrace/src/capture.rs index 7b37bd1bc..e0dd9c474 100644 --- a/crux-mir/lib/backtrace/src/capture.rs +++ b/crux-mir/lib/backtrace/src/capture.rs @@ -58,6 +58,7 @@ enum Frame { Deserialized { ip: usize, symbol_address: usize, + module_base_address: Option, }, } @@ -75,6 +76,16 @@ impl Frame { Frame::Deserialized { symbol_address, .. } => symbol_address as *mut c_void, } } + + fn module_base_address(&self) -> Option<*mut c_void> { + match *self { + Frame::Raw(ref f) => f.module_base_address(), + Frame::Deserialized { + module_base_address, + .. + } => module_base_address.map(|addr| addr as *mut c_void), + } + } } /// Captured version of a symbol in a backtrace. @@ -94,6 +105,7 @@ pub struct BacktraceSymbol { addr: Option, filename: Option, lineno: Option, + colno: Option, } impl Backtrace { @@ -213,6 +225,7 @@ impl Backtrace { addr: symbol.addr().map(|a| a as usize), filename: symbol.filename().map(|m| m.to_owned()), lineno: symbol.lineno(), + colno: symbol.colno(), }); }; match frame.frame { @@ -236,6 +249,15 @@ impl From> for Backtrace { } } +impl From for BacktraceFrame { + fn from(frame: crate::Frame) -> BacktraceFrame { + BacktraceFrame { + frame: Frame::Raw(frame), + symbols: None, + } + } +} + impl Into> for Backtrace { fn into(self) -> Vec { self.frames @@ -263,6 +285,18 @@ impl BacktraceFrame { self.frame.symbol_address() as *mut c_void } + /// Same as `Frame::module_base_address` + /// + /// # Required features + /// + /// This function requires the `std` feature of the `backtrace` crate to be + /// enabled, and the `std` feature is enabled by default. + pub fn module_base_address(&self) -> Option<*mut c_void> { + self.frame + .module_base_address() + .map(|addr| addr as *mut c_void) + } + /// Returns the list of symbols that this frame corresponds to. /// /// Normally there is only one symbol per frame, but sometimes if a number @@ -289,7 +323,7 @@ impl BacktraceSymbol { /// /// This function requires the `std` feature of the `backtrace` crate to be /// enabled, and the `std` feature is enabled by default. - pub fn name(&self) -> Option { + pub fn name(&self) -> Option> { self.name.as_ref().map(|s| SymbolName::new(s)) } @@ -322,10 +356,20 @@ impl BacktraceSymbol { pub fn lineno(&self) -> Option { self.lineno } + + /// Same as `Symbol::colno` + /// + /// # Required features + /// + /// This function requires the `std` feature of the `backtrace` crate to be + /// enabled, and the `std` feature is enabled by default. + pub fn colno(&self) -> Option { + self.colno + } } impl fmt::Debug for Backtrace { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { let full = fmt.alternate(); let (frames, style) = if full { (&self.frames[..], PrintFmt::Full) @@ -338,17 +382,18 @@ impl fmt::Debug for Backtrace { // short format, because if it's full we presumably want to print // everything. let cwd = std::env::current_dir(); - let mut print_path = move |fmt: &mut fmt::Formatter, path: crate::BytesOrWideString| { - let path = path.into_path_buf(); - if !full { - if let Ok(cwd) = &cwd { - if let Ok(suffix) = path.strip_prefix(cwd) { - return fmt::Display::fmt(&suffix.display(), fmt); + let mut print_path = + move |fmt: &mut fmt::Formatter<'_>, path: crate::BytesOrWideString<'_>| { + let path = path.into_path_buf(); + if !full { + if let Ok(cwd) = &cwd { + if let Ok(suffix) = path.strip_prefix(cwd) { + return fmt::Display::fmt(&suffix.display(), fmt); + } } } - } - fmt::Display::fmt(&path.display(), fmt) - }; + fmt::Display::fmt(&path.display(), fmt) + }; let mut f = BacktraceFmt::new(fmt, style, &mut print_path); f.add_context()?; @@ -367,7 +412,7 @@ impl Default for Backtrace { } impl fmt::Debug for BacktraceFrame { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt.debug_struct("BacktraceFrame") .field("ip", &self.ip()) .field("symbol_address", &self.symbol_address()) @@ -376,12 +421,13 @@ impl fmt::Debug for BacktraceFrame { } impl fmt::Debug for BacktraceSymbol { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt.debug_struct("BacktraceSymbol") .field("name", &self.name()) .field("addr", &self.addr()) .field("filename", &self.filename()) .field("lineno", &self.lineno()) + .field("colno", &self.colno()) .finish() } } @@ -395,6 +441,7 @@ mod rustc_serialize_impls { struct SerializedFrame { ip: usize, symbol_address: usize, + module_base_address: Option, symbols: Option>, } @@ -408,6 +455,7 @@ mod rustc_serialize_impls { frame: Frame::Deserialized { ip: frame.ip, symbol_address: frame.symbol_address, + module_base_address: frame.module_base_address, }, symbols: frame.symbols, }) @@ -423,6 +471,7 @@ mod rustc_serialize_impls { SerializedFrame { ip: frame.ip() as usize, symbol_address: frame.symbol_address() as usize, + module_base_address: frame.module_base_address().map(|addr| addr as usize), symbols: symbols.clone(), } .encode(e) @@ -432,17 +481,16 @@ mod rustc_serialize_impls { #[cfg(feature = "serde")] mod serde_impls { - extern crate serde; - - use self::serde::de::Deserializer; - use self::serde::ser::Serializer; - use self::serde::{Deserialize, Serialize}; use super::*; + use serde::de::Deserializer; + use serde::ser::Serializer; + use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] struct SerializedFrame { ip: usize, symbol_address: usize, + module_base_address: Option, symbols: Option>, } @@ -455,6 +503,7 @@ mod serde_impls { SerializedFrame { ip: frame.ip() as usize, symbol_address: frame.symbol_address() as usize, + module_base_address: frame.module_base_address().map(|addr| addr as usize), symbols: symbols.clone(), } .serialize(s) @@ -471,9 +520,36 @@ mod serde_impls { frame: Frame::Deserialized { ip: frame.ip, symbol_address: frame.symbol_address, + module_base_address: frame.module_base_address, }, symbols: frame.symbols, }) } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_frame_conversion() { + let mut frames = vec![]; + crate::trace(|frame| { + let converted = BacktraceFrame::from(frame.clone()); + frames.push(converted); + true + }); + + let mut manual = Backtrace::from(frames); + manual.resolve(); + let frames = manual.frames(); + + for frame in frames { + println!("{:?}", frame.ip()); + println!("{:?}", frame.symbol_address()); + println!("{:?}", frame.module_base_address()); + println!("{:?}", frame.symbols()); + } + } +} diff --git a/crux-mir/lib/backtrace/src/dbghelp.rs b/crux-mir/lib/backtrace/src/dbghelp.rs index 05ffef2ac..e01002beb 100644 --- a/crux-mir/lib/backtrace/src/dbghelp.rs +++ b/crux-mir/lib/backtrace/src/dbghelp.rs @@ -23,7 +23,7 @@ #![allow(non_snake_case)] -use crate::windows::*; +use super::windows::*; use core::mem; use core::ptr; @@ -34,29 +34,11 @@ use core::ptr; mod dbghelp { use crate::windows::*; pub use winapi::um::dbghelp::{ - StackWalk64, SymCleanup, SymFromAddrW, SymFunctionTableAccess64, SymGetLineFromAddrW64, - SymGetModuleBase64, SymInitializeW, + StackWalk64, StackWalkEx, SymCleanup, SymFromAddrW, SymFunctionTableAccess64, + SymGetLineFromAddrW64, SymGetModuleBase64, SymGetOptions, SymInitializeW, SymSetOptions, }; extern "system" { - // Not defined in winapi yet - pub fn SymGetOptions() -> u32; - pub fn SymSetOptions(_: u32); - - // This is defined in winapi, but it's incorrect (FIXME winapi-rs#768) - pub fn StackWalkEx( - MachineType: DWORD, - hProcess: HANDLE, - hThread: HANDLE, - StackFrame: LPSTACKFRAME_EX, - ContextRecord: PVOID, - ReadMemoryRoutine: PREAD_PROCESS_MEMORY_ROUTINE64, - FunctionTableAccessRoutine: PFUNCTION_TABLE_ACCESS_ROUTINE64, - GetModuleBaseRoutine: PGET_MODULE_BASE_ROUTINE64, - TranslateAddress: PTRANSLATE_ADDRESS_ROUTINE64, - Flags: DWORD, - ) -> BOOL; - // Not defined in winapi yet pub fn SymFromInlineContextW( hProcess: HANDLE, @@ -176,7 +158,7 @@ const SYMOPT_DEFERRED_LOADS: DWORD = 0x00000004; dbghelp! { extern "system" { fn SymGetOptions() -> DWORD; - fn SymSetOptions(options: DWORD) -> (); + fn SymSetOptions(options: DWORD) -> DWORD; fn SymInitializeW( handle: HANDLE, path: PCWSTR, @@ -254,7 +236,6 @@ pub struct Init { /// Note that this function is **safe**, it internally has its own /// synchronization. Also note that it is safe to call this function multiple /// times recursively. -#[cfg(all(windows, feature = "dbghelp"))] pub fn init() -> Result { use core::sync::atomic::{AtomicUsize, Ordering::SeqCst}; diff --git a/crux-mir/lib/backtrace/src/lib.rs b/crux-mir/lib/backtrace/src/lib.rs index 69a283e67..e5dea3387 100644 --- a/crux-mir/lib/backtrace/src/lib.rs +++ b/crux-mir/lib/backtrace/src/lib.rs @@ -6,25 +6,6 @@ //! parsed, for example, and expose the functionality of multiple backend //! implementations. //! -//! # Implementation -//! -//! This library makes use of a number of strategies for actually acquiring a -//! backtrace. For example unix uses libgcc's libunwind bindings by default to -//! acquire a backtrace, but coresymbolication or dladdr is used on OSX to -//! acquire symbol names while linux uses gcc's libbacktrace. -//! -//! When using the default feature set of this library the "most reasonable" set -//! of defaults is chosen for the current platform, but the features activated -//! can also be controlled at a finer granularity. -//! -//! # API Principles -//! -//! This library attempts to be as flexible as possible to accommodate different -//! backend implementations of acquiring a backtrace. Consequently the currently -//! exported functions are closure-based as opposed to the likely expected -//! iterator-based versions. This is done due to limitations of the underlying -//! APIs used from the system. -//! //! # Usage //! //! First, add this to your Cargo.toml @@ -37,8 +18,6 @@ //! Next: //! //! ``` -//! extern crate backtrace; -//! //! fn main() { //! # // Unsafe here so test passes on no_std. //! # #[cfg(feature = "std")] { @@ -61,6 +40,51 @@ //! } //! # } //! ``` +//! +//! # Backtrace accuracy +//! +//! This crate implements best-effort attempts to get the native backtrace. This +//! is not always guaranteed to work, and some platforms don't return any +//! backtrace at all. If your application requires accurate backtraces then it's +//! recommended to closely evaluate this crate to see whether it's suitable +//! for your use case on your target platforms. +//! +//! Even on supported platforms, there's a number of reasons that backtraces may +//! be less-than-accurate, including but not limited to: +//! +//! * Unwind information may not be available. This crate primarily implements +//! backtraces by unwinding the stack, but not all functions may have +//! unwinding information (e.g. DWARF unwinding information). +//! +//! * Rust code may be compiled without unwinding information for some +//! functions. This can also happen for Rust code compiled with +//! `-Cpanic=abort`. You can remedy this, however, with +//! `-Cforce-unwind-tables` as a compiler option. +//! +//! * Unwind information may be inaccurate or corrupt. In the worst case +//! inaccurate unwind information can lead this library to segfault. In the +//! best case inaccurate information will result in a truncated stack trace. +//! +//! * Backtraces may not report filenames/line numbers correctly due to missing +//! or corrupt debug information. This won't lead to segfaults unlike corrupt +//! unwinding information, but missing or malformed debug information will +//! mean that filenames and line numbers will not be available. This may be +//! because debug information wasn't generated by the compiler, or it's just +//! missing on the filesystem. +//! +//! * Not all platforms are supported. For example there's no way to get a +//! backtrace on WebAssembly at the moment. +//! +//! * Crate features may be disabled. Currently this crate supports using Gimli +//! libbacktrace on non-Windows platforms for reading debuginfo for +//! backtraces. If both crate features are disabled, however, then these +//! platforms will generate a backtrace but be unable to generate symbols for +//! it. +//! +//! In most standard workflows for most standard platforms you generally don't +//! need to worry about these caveats. We'll try to fix ones where we can over +//! time, but otherwise it's important to be aware of the limitations of +//! unwinding-based backtraces! #![doc(html_root_url = "https://docs.rs/backtrace")] #![deny(missing_docs)] @@ -69,34 +93,44 @@ all(feature = "std", target_env = "sgx", target_vendor = "fortanix"), feature(sgx_platform) )] -#![allow(bare_trait_objects)] // TODO: remove when updating to 2018 edition -#![allow(rust_2018_idioms)] // TODO: remove when updating to 2018 edition +#![warn(rust_2018_idioms)] +// When we're building as part of libstd, silence all warnings since they're +// irrelevant as this crate is developed out-of-tree. +#![cfg_attr(backtrace_in_libstd, allow(warnings))] +#![cfg_attr(not(feature = "std"), allow(dead_code))] +// We know this is deprecated, it's only here for back-compat reasons. +#![cfg_attr(feature = "rustc-serialize", allow(deprecated))] #[cfg(feature = "std")] #[macro_use] extern crate std; -pub use crate::backtrace::{trace_unsynchronized, Frame}; +// This is only used for gimli right now, which is only used on some platforms, and miri +// so don't worry if it's unused in other configurations. +#[allow(unused_extern_crates)] +extern crate alloc; + +pub use self::backtrace::{trace_unsynchronized, Frame}; mod backtrace; -pub use crate::symbolize::resolve_frame_unsynchronized; -pub use crate::symbolize::{resolve_unsynchronized, Symbol, SymbolName}; +pub use self::symbolize::resolve_frame_unsynchronized; +pub use self::symbolize::{resolve_unsynchronized, Symbol, SymbolName}; mod symbolize; -pub use crate::types::BytesOrWideString; +pub use self::types::BytesOrWideString; mod types; #[cfg(feature = "std")] -pub use crate::symbolize::clear_symbol_cache; +pub use self::symbolize::clear_symbol_cache; mod print; pub use print::{BacktraceFmt, BacktraceFrameFmt, PrintFmt}; cfg_if::cfg_if! { if #[cfg(feature = "std")] { - pub use crate::backtrace::trace; - pub use crate::symbolize::{resolve, resolve_frame}; - pub use crate::capture::{Backtrace, BacktraceFrame, BacktraceSymbol}; + pub use self::backtrace::trace; + pub use self::symbolize::{resolve, resolve_frame}; + pub use self::capture::{Backtrace, BacktraceFrame, BacktraceSymbol}; mod capture; } } @@ -153,7 +187,7 @@ mod lock { } } -#[cfg(all(windows, feature = "dbghelp", not(target_vendor = "uwp")))] +#[cfg(all(windows, not(target_vendor = "uwp")))] mod dbghelp; #[cfg(windows)] mod windows; diff --git a/crux-mir/lib/backtrace/src/print.rs b/crux-mir/lib/backtrace/src/print.rs index bee6717af..cc677122a 100644 --- a/crux-mir/lib/backtrace/src/print.rs +++ b/crux-mir/lib/backtrace/src/print.rs @@ -1,4 +1,6 @@ -use crate::BytesOrWideString; +#[cfg(feature = "std")] +use super::{BacktraceFrame, BacktraceSymbol}; +use super::{BytesOrWideString, Frame, SymbolName}; use core::ffi::c_void; use core::fmt; @@ -16,7 +18,8 @@ pub struct BacktraceFmt<'a, 'b> { fmt: &'a mut fmt::Formatter<'b>, frame_index: usize, format: PrintFmt, - print_path: &'a mut (FnMut(&mut fmt::Formatter, BytesOrWideString) -> fmt::Result + 'b), + print_path: + &'a mut (dyn FnMut(&mut fmt::Formatter<'_>, BytesOrWideString<'_>) -> fmt::Result + 'b), } /// The styles of printing that we can print @@ -41,7 +44,8 @@ impl<'a, 'b> BacktraceFmt<'a, 'b> { pub fn new( fmt: &'a mut fmt::Formatter<'b>, format: PrintFmt, - print_path: &'a mut (FnMut(&mut fmt::Formatter, BytesOrWideString) -> fmt::Result + 'b), + print_path: &'a mut (dyn FnMut(&mut fmt::Formatter<'_>, BytesOrWideString<'_>) -> fmt::Result + + 'b), ) -> Self { BacktraceFmt { fmt, @@ -54,7 +58,7 @@ impl<'a, 'b> BacktraceFmt<'a, 'b> { /// Prints a preamble for the backtrace about to be printed. /// /// This is required on some platforms for backtraces to be fully - /// sumbolicated later, and otherwise this should just be the first method + /// symbolicated later, and otherwise this should just be the first method /// you call after creating a `BacktraceFmt`. pub fn add_context(&mut self) -> fmt::Result { #[cfg(target_os = "fuchsia")] @@ -95,7 +99,7 @@ pub struct BacktraceFrameFmt<'fmt, 'a, 'b> { impl BacktraceFrameFmt<'_, '_, '_> { /// Prints a `BacktraceFrame` with this frame formatter. /// - /// This will recusrively print all `BacktraceSymbol` instances within the + /// This will recursively print all `BacktraceSymbol` instances within the /// `BacktraceFrame`. /// /// # Required features @@ -103,7 +107,7 @@ impl BacktraceFrameFmt<'_, '_, '_> { /// This function requires the `std` feature of the `backtrace` crate to be /// enabled, and the `std` feature is enabled by default. #[cfg(feature = "std")] - pub fn backtrace_frame(&mut self, frame: &crate::BacktraceFrame) -> fmt::Result { + pub fn backtrace_frame(&mut self, frame: &BacktraceFrame) -> fmt::Result { let symbols = frame.symbols(); for symbol in symbols { self.backtrace_symbol(frame, symbol)?; @@ -123,10 +127,10 @@ impl BacktraceFrameFmt<'_, '_, '_> { #[cfg(feature = "std")] pub fn backtrace_symbol( &mut self, - frame: &crate::BacktraceFrame, - symbol: &crate::BacktraceSymbol, + frame: &BacktraceFrame, + symbol: &BacktraceSymbol, ) -> fmt::Result { - self.print_raw( + self.print_raw_with_column( frame.ip(), symbol.name(), // TODO: this isn't great that we don't end up printing anything @@ -136,18 +140,20 @@ impl BacktraceFrameFmt<'_, '_, '_> { .filename() .and_then(|p| Some(BytesOrWideString::Bytes(p.to_str()?.as_bytes()))), symbol.lineno(), + symbol.colno(), )?; Ok(()) } /// Prints a raw traced `Frame` and `Symbol`, typically from within the raw /// callbacks of this crate. - pub fn symbol(&mut self, frame: &crate::Frame, symbol: &crate::Symbol) -> fmt::Result { - self.print_raw( + pub fn symbol(&mut self, frame: &Frame, symbol: &super::Symbol) -> fmt::Result { + self.print_raw_with_column( frame.ip(), symbol.name(), symbol.filename_raw(), symbol.lineno(), + symbol.colno(), )?; Ok(()) } @@ -160,9 +166,25 @@ impl BacktraceFrameFmt<'_, '_, '_> { pub fn print_raw( &mut self, frame_ip: *mut c_void, - symbol_name: Option, - filename: Option, + symbol_name: Option>, + filename: Option>, lineno: Option, + ) -> fmt::Result { + self.print_raw_with_column(frame_ip, symbol_name, filename, lineno, None) + } + + /// Adds a raw frame to the backtrace output, including column information. + /// + /// This method, like the previous, takes the raw arguments in case + /// they're being source from different locations. Note that this may be + /// called multiple times for one frame. + pub fn print_raw_with_column( + &mut self, + frame_ip: *mut c_void, + symbol_name: Option>, + filename: Option>, + lineno: Option, + colno: Option, ) -> fmt::Result { // Fuchsia is unable to symbolize within a process so it has a special // format which can be used to symbolize later. Print that instead of @@ -170,7 +192,7 @@ impl BacktraceFrameFmt<'_, '_, '_> { if cfg!(target_os = "fuchsia") { self.print_raw_fuchsia(frame_ip)?; } else { - self.print_raw_generic(frame_ip, symbol_name, filename, lineno)?; + self.print_raw_generic(frame_ip, symbol_name, filename, lineno, colno)?; } self.symbol_index += 1; Ok(()) @@ -180,9 +202,10 @@ impl BacktraceFrameFmt<'_, '_, '_> { fn print_raw_generic( &mut self, mut frame_ip: *mut c_void, - symbol_name: Option, - filename: Option, + symbol_name: Option>, + filename: Option>, lineno: Option, + colno: Option, ) -> fmt::Result { // No need to print "null" frames, it basically just means that the // system backtrace was a bit eager to trace back super far. @@ -228,13 +251,18 @@ impl BacktraceFrameFmt<'_, '_, '_> { // And last up, print out the filename/line number if they're available. if let (Some(file), Some(line)) = (filename, lineno) { - self.print_fileline(file, line)?; + self.print_fileline(file, line, colno)?; } Ok(()) } - fn print_fileline(&mut self, file: BytesOrWideString, line: u32) -> fmt::Result { + fn print_fileline( + &mut self, + file: BytesOrWideString<'_>, + line: u32, + colno: Option, + ) -> fmt::Result { // Filename/line are printed on lines under the symbol name, so print // some appropriate whitespace to sort of right-align ourselves. if let PrintFmt::Full = self.fmt.format { @@ -245,7 +273,14 @@ impl BacktraceFrameFmt<'_, '_, '_> { // Delegate to our internal callback to print the filename and then // print out the line number. (self.fmt.print_path)(self.fmt.fmt, file)?; - write!(self.fmt.fmt, ":{}\n", line)?; + write!(self.fmt.fmt, ":{}", line)?; + + // Add column number, if available. + if let Some(colno) = colno { + write!(self.fmt.fmt, ":{}", colno)?; + } + + write!(self.fmt.fmt, "\n")?; Ok(()) } diff --git a/crux-mir/lib/backtrace/src/print/fuchsia.rs b/crux-mir/lib/backtrace/src/print/fuchsia.rs index 787954a03..959253acb 100644 --- a/crux-mir/lib/backtrace/src/print/fuchsia.rs +++ b/crux-mir/lib/backtrace/src/print/fuchsia.rs @@ -12,8 +12,8 @@ extern "C" { // callback on each call. 'size' gives the size of the dl_phdr_info. #[allow(improper_ctypes)] fn dl_iterate_phdr( - f: extern "C" fn(info: &dl_phdr_info, size: usize, data: &mut DsoPrinter) -> i32, - data: &mut DsoPrinter, + f: extern "C" fn(info: &dl_phdr_info, size: usize, data: &mut DsoPrinter<'_, '_>) -> i32, + data: &mut DsoPrinter<'_, '_>, ) -> i32; } @@ -224,7 +224,7 @@ const PERM_W: u32 = 0b00000010; const PERM_R: u32 = 0b00000100; impl core::fmt::Display for Perm { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let v = self.0; if v & PERM_R != 0 { f.write_char('r')? @@ -286,7 +286,7 @@ struct Dso<'a> { /// of a shared object it will be the soname (see DT_SONAME). name: &'a str, /// On Fuchsia virtually all binaries have build IDs but this is not a strict - /// requierment. There's no way to match up DSO information with a real ELF + /// requirement. There's no way to match up DSO information with a real ELF /// file afterwards if there is no build_id so we require that every DSO /// have one here. DSO's without a build_id are ignored. build_id: &'a [u8], @@ -310,7 +310,7 @@ struct HexSlice<'a> { } impl fmt::Display for HexSlice<'_> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { for byte in self.bytes { write!(f, "{:02x}", byte)?; } @@ -349,8 +349,12 @@ enum Error { /// # Arguments /// /// * `visitor` - A DsoPrinter that will have one of eats methods called foreach DSO. -fn for_each_dso(mut visitor: &mut DsoPrinter) { - extern "C" fn callback(info: &dl_phdr_info, _size: usize, visitor: &mut DsoPrinter) -> i32 { +fn for_each_dso(mut visitor: &mut DsoPrinter<'_, '_>) { + extern "C" fn callback( + info: &dl_phdr_info, + _size: usize, + visitor: &mut DsoPrinter<'_, '_>, + ) -> i32 { // dl_iterate_phdr ensures that info.name will point to a valid // location. let name_len = unsafe { libc::strlen(info.name) }; diff --git a/crux-mir/lib/backtrace/src/symbolize/coresymbolication.rs b/crux-mir/lib/backtrace/src/symbolize/coresymbolication.rs deleted file mode 100644 index 24dc11a8e..000000000 --- a/crux-mir/lib/backtrace/src/symbolize/coresymbolication.rs +++ /dev/null @@ -1,277 +0,0 @@ -// Copyright 2014-2015 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -//! Symbolication strategy that's OSX-specific and uses the `CoreSymbolication` -//! framework, if possible. -//! -//! This strategy uses internal private APIs that are somewhat undocumented but -//! seem to be widely used on OSX. This is the default symbolication strategy -//! for OSX, but is turned off in release builds for iOS due to reports of apps -//! being rejected due to using these APIs. -//! -//! This would probably be good to get official one day and not using private -//! APIs, but for now it should largely suffice. -//! -//! Note that this module will dynamically load `CoreSymbolication` and its APIs -//! through dlopen/dlsym, and if the loading fails this falls back to `dladdr` -//! as a symbolication strategy. - -#![allow(bad_style)] - -use crate::symbolize::dladdr; -use crate::symbolize::ResolveWhat; -use crate::types::BytesOrWideString; -use crate::SymbolName; -use core::ffi::c_void; -use core::mem; -use core::ptr; -use core::slice; -use libc::{c_char, c_int}; - -#[repr(C)] -#[derive(Copy, Clone, PartialEq)] -pub struct CSTypeRef { - cpp_data: *const c_void, - cpp_obj: *const c_void, -} - -const CS_NOW: u64 = 0x80000000; -const CSREF_NULL: CSTypeRef = CSTypeRef { - cpp_data: 0 as *const c_void, - cpp_obj: 0 as *const c_void, -}; - -pub enum Symbol<'a> { - Core { - path: *const c_char, - lineno: u32, - name: *const c_char, - addr: *mut c_void, - }, - Dladdr(dladdr::Symbol<'a>), -} - -impl Symbol<'_> { - pub fn name(&self) -> Option { - let name = match *self { - Symbol::Core { name, .. } => name, - Symbol::Dladdr(ref info) => return info.name(), - }; - if name.is_null() { - None - } else { - Some(SymbolName::new(unsafe { - let len = libc::strlen(name); - slice::from_raw_parts(name as *const u8, len) - })) - } - } - - pub fn addr(&self) -> Option<*mut c_void> { - match *self { - Symbol::Core { addr, .. } => Some(addr), - Symbol::Dladdr(ref info) => info.addr(), - } - } - - fn filename_bytes(&self) -> Option<&[u8]> { - match *self { - Symbol::Core { path, .. } => { - if path.is_null() { - None - } else { - Some(unsafe { - let len = libc::strlen(path); - slice::from_raw_parts(path as *const u8, len) - }) - } - } - Symbol::Dladdr(_) => None, - } - } - - pub fn filename_raw(&self) -> Option { - self.filename_bytes().map(BytesOrWideString::Bytes) - } - - #[cfg(feature = "std")] - pub fn filename(&self) -> Option<&::std::path::Path> { - use std::ffi::OsStr; - use std::os::unix::prelude::*; - use std::path::Path; - - self.filename_bytes().map(OsStr::from_bytes).map(Path::new) - } - - pub fn lineno(&self) -> Option { - match *self { - Symbol::Core { lineno: 0, .. } => None, - Symbol::Core { lineno, .. } => Some(lineno), - Symbol::Dladdr(_) => None, - } - } -} - -macro_rules! coresymbolication { - (#[load_path = $path:tt] extern "C" { - $(fn $name:ident($($arg:ident: $argty:ty),*) -> $ret: ty;)* - }) => ( - pub struct CoreSymbolication { - // The loaded dynamic library - dll: *mut c_void, - - // Each function pointer for each function we might use - $($name: usize,)* - } - - static mut CORESYMBOLICATION: CoreSymbolication = CoreSymbolication { - // Initially we haven't loaded the dynamic library - dll: 0 as *mut _, - // Initiall all functions are set to zero to say they need to be - // dynamically loaded. - $($name: 0,)* - }; - - // Convenience typedef for each function type. - $(pub type $name = unsafe extern "C" fn($($argty),*) -> $ret;)* - - impl CoreSymbolication { - /// Attempts to open `dbghelp.dll`. Returns `true` if it works or - /// `false` if `dlopen` fails. - fn open(&mut self) -> bool { - if !self.dll.is_null() { - return true; - } - let lib = concat!($path, "\0").as_bytes(); - unsafe { - self.dll = libc::dlopen(lib.as_ptr() as *const _, libc::RTLD_LAZY); - !self.dll.is_null() - } - } - - // Function for each method we'd like to use. When called it will - // either read the cached function pointer or load it and return the - // loaded value. Loads are asserted to succeed. - $(pub fn $name(&mut self) -> $name { - unsafe { - if self.$name == 0 { - let name = concat!(stringify!($name), "\0"); - self.$name = self.symbol(name.as_bytes()) - .expect(concat!("symbol ", stringify!($name), " is missing")); - } - mem::transmute::(self.$name) - } - })* - - fn symbol(&self, symbol: &[u8]) -> Option { - unsafe { - match libc::dlsym(self.dll, symbol.as_ptr() as *const _) as usize { - 0 => None, - n => Some(n), - } - } - } - } - ) -} - -coresymbolication! { - #[load_path = "/System/Library/PrivateFrameworks/CoreSymbolication.framework\ - /Versions/A/CoreSymbolication"] - extern "C" { - fn CSSymbolicatorCreateWithPid(pid: c_int) -> CSTypeRef; - fn CSRelease(rf: CSTypeRef) -> (); - fn CSSymbolicatorGetSymbolWithAddressAtTime( - cs: CSTypeRef, addr: *const c_void, time: u64) -> CSTypeRef; - fn CSSymbolicatorGetSourceInfoWithAddressAtTime( - cs: CSTypeRef, addr: *const c_void, time: u64) -> CSTypeRef; - fn CSSourceInfoGetLineNumber(info: CSTypeRef) -> c_int; - fn CSSourceInfoGetPath(info: CSTypeRef) -> *const c_char; - fn CSSourceInfoGetSymbol(info: CSTypeRef) -> CSTypeRef; - fn CSSymbolGetMangledName(sym: CSTypeRef) -> *const c_char; - fn CSSymbolGetSymbolOwner(sym: CSTypeRef) -> CSTypeRef; - fn CSSymbolOwnerGetBaseAddress(symowner: CSTypeRef) -> *mut c_void; - } -} - -unsafe fn try_resolve(addr: *mut c_void, cb: &mut FnMut(&super::Symbol)) -> bool { - // Note that this is externally synchronized so there's no need for - // synchronization here, making this `static mut` safer. - let lib = &mut CORESYMBOLICATION; - if !lib.open() { - return false; - } - - let cs = lib.CSSymbolicatorCreateWithPid()(libc::getpid()); - if cs == CSREF_NULL { - return false; - } - let _dtor = OwnedCSTypeRef { - ptr: cs, - CSRelease: lib.CSRelease(), - }; - - let info = lib.CSSymbolicatorGetSourceInfoWithAddressAtTime()(cs, addr, CS_NOW); - let sym = if info == CSREF_NULL { - lib.CSSymbolicatorGetSymbolWithAddressAtTime()(cs, addr, CS_NOW) - } else { - lib.CSSourceInfoGetSymbol()(info) - }; - if sym == CSREF_NULL { - return false; - } - let owner = lib.CSSymbolGetSymbolOwner()(sym); - if owner == CSREF_NULL { - return false; - } - - cb(&super::Symbol { - inner: Symbol::Core { - path: if info != CSREF_NULL { - lib.CSSourceInfoGetPath()(info) - } else { - ptr::null() - }, - lineno: if info != CSREF_NULL { - lib.CSSourceInfoGetLineNumber()(info) as u32 - } else { - 0 - }, - name: lib.CSSymbolGetMangledName()(sym), - addr: lib.CSSymbolOwnerGetBaseAddress()(owner), - }, - }); - true -} - -struct OwnedCSTypeRef { - ptr: CSTypeRef, - CSRelease: unsafe extern "C" fn(CSTypeRef), -} - -impl Drop for OwnedCSTypeRef { - fn drop(&mut self) { - unsafe { - (self.CSRelease)(self.ptr); - } - } -} - -pub unsafe fn resolve(what: ResolveWhat, cb: &mut FnMut(&super::Symbol)) { - let addr = what.address_or_ip(); - if try_resolve(addr, cb) { - return; - } - dladdr::resolve(addr, &mut |sym| { - cb(&super::Symbol { - inner: Symbol::Dladdr(sym), - }) - }) -} diff --git a/crux-mir/lib/backtrace/src/symbolize/dbghelp.rs b/crux-mir/lib/backtrace/src/symbolize/dbghelp.rs index 7afcee736..181dba731 100644 --- a/crux-mir/lib/backtrace/src/symbolize/dbghelp.rs +++ b/crux-mir/lib/backtrace/src/symbolize/dbghelp.rs @@ -1,13 +1,3 @@ -// Copyright 2014-2015 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - //! Symbolication strategy using `dbghelp.dll` on Windows, only used for MSVC //! //! This symbolication strategy, like with backtraces, uses dynamically loaded @@ -27,12 +17,8 @@ #![allow(bad_style)] -use crate::backtrace::FrameImp as Frame; -use crate::dbghelp; -use crate::symbolize::ResolveWhat; -use crate::types::BytesOrWideString; -use crate::windows::*; -use crate::SymbolName; +use super::super::{backtrace::StackFrame, dbghelp, windows::*}; +use super::{BytesOrWideString, ResolveWhat, SymbolName}; use core::char; use core::ffi::c_void; use core::marker; @@ -53,7 +39,7 @@ pub struct Symbol<'a> { } impl Symbol<'_> { - pub fn name(&self) -> Option { + pub fn name(&self) -> Option> { Some(SymbolName::new(unsafe { &*self.name })) } @@ -61,11 +47,15 @@ impl Symbol<'_> { Some(self.addr as *mut _) } - pub fn filename_raw(&self) -> Option { + pub fn filename_raw(&self) -> Option> { self.filename .map(|slice| unsafe { BytesOrWideString::Wide(&*slice) }) } + pub fn colno(&self) -> Option { + None + } + pub fn lineno(&self) -> Option { self.line } @@ -81,7 +71,7 @@ impl Symbol<'_> { #[repr(C, align(8))] struct Aligned8(T); -pub unsafe fn resolve(what: ResolveWhat, cb: &mut FnMut(&super::Symbol)) { +pub unsafe fn resolve(what: ResolveWhat<'_>, cb: &mut dyn FnMut(&super::Symbol)) { // Ensure this process's symbols are initialized let dbghelp = match dbghelp::init() { Ok(dbghelp) => dbghelp, @@ -90,9 +80,9 @@ pub unsafe fn resolve(what: ResolveWhat, cb: &mut FnMut(&super::Symbol)) { match what { ResolveWhat::Address(_) => resolve_without_inline(&dbghelp, what.address_or_ip(), cb), - ResolveWhat::Frame(frame) => match &frame.inner { - Frame::New(frame) => resolve_with_inline(&dbghelp, frame, cb), - Frame::Old(_) => resolve_without_inline(&dbghelp, frame.ip(), cb), + ResolveWhat::Frame(frame) => match &frame.inner.stack_frame { + StackFrame::New(frame) => resolve_with_inline(&dbghelp, frame, cb), + StackFrame::Old(_) => resolve_without_inline(&dbghelp, frame.ip(), cb), }, } } @@ -100,7 +90,7 @@ pub unsafe fn resolve(what: ResolveWhat, cb: &mut FnMut(&super::Symbol)) { unsafe fn resolve_with_inline( dbghelp: &dbghelp::Init, frame: &STACKFRAME_EX, - cb: &mut FnMut(&super::Symbol), + cb: &mut dyn FnMut(&super::Symbol), ) { do_resolve( |info| { @@ -129,7 +119,7 @@ unsafe fn resolve_with_inline( unsafe fn resolve_without_inline( dbghelp: &dbghelp::Init, addr: *mut c_void, - cb: &mut FnMut(&super::Symbol), + cb: &mut dyn FnMut(&super::Symbol), ) { do_resolve( |info| dbghelp.SymFromAddrW()(GetCurrentProcess(), addr as DWORD64, &mut 0, info), @@ -141,7 +131,7 @@ unsafe fn resolve_without_inline( unsafe fn do_resolve( sym_from_addr: impl FnOnce(*mut SYMBOL_INFOW) -> BOOL, get_line_from_addr: impl FnOnce(&mut IMAGEHLP_LINEW64) -> BOOL, - cb: &mut FnMut(&super::Symbol), + cb: &mut dyn FnMut(&super::Symbol), ) { const SIZE: usize = 2 * MAX_SYM_NAME + mem::size_of::(); let mut data = Aligned8([0u8; SIZE]); @@ -224,3 +214,5 @@ unsafe fn cache(filename: Option<*const [u16]>) -> Option<::std::ffi::OsString> #[cfg(not(feature = "std"))] unsafe fn cache(_filename: Option<*const [u16]>) {} + +pub unsafe fn clear_symbol_cache() {} diff --git a/crux-mir/lib/backtrace/src/symbolize/dladdr.rs b/crux-mir/lib/backtrace/src/symbolize/dladdr.rs deleted file mode 100644 index 9509e9986..000000000 --- a/crux-mir/lib/backtrace/src/symbolize/dladdr.rs +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2014-2015 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -//! Common support for resolving with `dladdr`, often used as a fallback if -//! other strategies don't work. - -#![allow(dead_code)] - -cfg_if::cfg_if! { - if #[cfg(all(unix, not(target_os = "emscripten"), feature = "dladdr"))] { - use core::ffi::c_void; - use core::marker; - use core::{mem, slice}; - use crate::SymbolName; - use crate::types::BytesOrWideString; - use libc::{self, Dl_info}; - - pub struct Symbol<'a> { - inner: Dl_info, - _marker: marker::PhantomData<&'a i32>, - } - - impl Symbol<'_> { - pub fn name(&self) -> Option { - if self.inner.dli_sname.is_null() { - None - } else { - let ptr = self.inner.dli_sname as *const u8; - unsafe { - let len = libc::strlen(self.inner.dli_sname); - Some(SymbolName::new(slice::from_raw_parts(ptr, len))) - } - } - } - - pub fn addr(&self) -> Option<*mut c_void> { - Some(self.inner.dli_saddr as *mut _) - } - - pub fn filename_raw(&self) -> Option { - None - } - - #[cfg(feature = "std")] - pub fn filename(&self) -> Option<&::std::path::Path> { - None - } - - pub fn lineno(&self) -> Option { - None - } - } - - pub unsafe fn resolve(addr: *mut c_void, cb: &mut FnMut(Symbol<'static>)) { - let mut info = Symbol { - inner: mem::zeroed(), - _marker: marker::PhantomData, - }; - // Skip null addresses to avoid calling into libc and having it do - // things with the dynamic symbol table for no reason. - if !addr.is_null() && libc::dladdr(addr as *mut _, &mut info.inner) != 0 { - cb(info) - } - } - } else { - use core::ffi::c_void; - use core::marker; - use crate::symbolize::SymbolName; - use crate::types::BytesOrWideString; - - pub struct Symbol<'a> { - a: Void, - _b: marker::PhantomData<&'a i32>, - } - - enum Void {} - - impl Symbol<'_> { - pub fn name(&self) -> Option { - match self.a {} - } - - pub fn addr(&self) -> Option<*mut c_void> { - match self.a {} - } - - pub fn filename_raw(&self) -> Option { - match self.a {} - } - - #[cfg(feature = "std")] - pub fn filename(&self) -> Option<&::std::path::Path> { - match self.a {} - } - - pub fn lineno(&self) -> Option { - match self.a {} - } - } - - pub unsafe fn resolve(addr: *mut c_void, cb: &mut FnMut(Symbol<'static>)) { - drop((addr, cb)); - } - } -} diff --git a/crux-mir/lib/backtrace/src/symbolize/dladdr_resolve.rs b/crux-mir/lib/backtrace/src/symbolize/dladdr_resolve.rs deleted file mode 100644 index 7790703ff..000000000 --- a/crux-mir/lib/backtrace/src/symbolize/dladdr_resolve.rs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2014-2015 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -//! Symbolication strategy using `dladdr` -//! -//! The `dladdr` API is available on most Unix implementations but it's quite -//! basic, not handling inline frame information at all. Since it's so prevalent -//! though we have an option to use it! - -use crate::symbolize::{dladdr, ResolveWhat, SymbolName}; -use crate::types::BytesOrWideString; -use core::ffi::c_void; - -pub struct Symbol<'a>(dladdr::Symbol<'a>); - -impl Symbol<'_> { - pub fn name(&self) -> Option { - self.0.name() - } - - pub fn addr(&self) -> Option<*mut c_void> { - self.0.addr() - } - - pub fn filename_raw(&self) -> Option { - self.0.filename_raw() - } - - #[cfg(feature = "std")] - pub fn filename(&self) -> Option<&::std::path::Path> { - self.0.filename() - } - - pub fn lineno(&self) -> Option { - self.0.lineno() - } -} - -pub unsafe fn resolve(what: ResolveWhat, cb: &mut FnMut(&super::Symbol)) { - dladdr::resolve(what.address_or_ip(), &mut |sym| { - cb(&super::Symbol { inner: Symbol(sym) }) - }); -} diff --git a/crux-mir/lib/backtrace/src/symbolize/gimli.rs b/crux-mir/lib/backtrace/src/symbolize/gimli.rs index 1529a90a0..5f10122dd 100644 --- a/crux-mir/lib/backtrace/src/symbolize/gimli.rs +++ b/crux-mir/lib/backtrace/src/symbolize/gimli.rs @@ -1,364 +1,199 @@ //! Support for symbolication using the `gimli` crate on crates.io //! -//! This implementation is largely a work in progress and is off by default for -//! all platforms, but it's hoped to be developed over time! Long-term this is -//! intended to wholesale replace the `libbacktrace.rs` implementation. +//! This is the default symbolication implementation for Rust. use self::gimli::read::EndianSlice; -use self::gimli::LittleEndian as Endian; -use crate::symbolize::dladdr; -use crate::symbolize::ResolveWhat; -use crate::types::BytesOrWideString; -use crate::SymbolName; +use self::gimli::NativeEndian as Endian; +use self::mmap::Mmap; +use self::stash::Stash; +use super::BytesOrWideString; +use super::ResolveWhat; +use super::SymbolName; use addr2line::gimli; +use core::convert::TryInto; use core::mem; use core::u32; -use findshlibs::{self, Segment, SharedLibrary}; use libc::c_void; -use memmap::Mmap; -use std::env; -use std::ffi::OsString; -use std::fs::File; -use std::path::Path; -use std::prelude::v1::*; - -const MAPPINGS_CACHE_SIZE: usize = 4; +use mystd::ffi::OsString; +use mystd::fs::File; +use mystd::path::Path; +use mystd::prelude::v1::*; + +#[cfg(backtrace_in_libstd)] +mod mystd { + pub use crate::*; +} +#[cfg(not(backtrace_in_libstd))] +extern crate std as mystd; -struct Context<'a> { - dwarf: addr2line::Context>, - object: Object<'a>, +cfg_if::cfg_if! { + if #[cfg(windows)] { + #[path = "gimli/mmap_windows.rs"] + mod mmap; + } else if #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "haiku", + target_os = "ios", + target_os = "linux", + target_os = "macos", + target_os = "openbsd", + target_os = "solaris", + target_os = "illumos", + ))] { + #[path = "gimli/mmap_unix.rs"] + mod mmap; + } else { + #[path = "gimli/mmap_fake.rs"] + mod mmap; + } } +mod stash; + +const MAPPINGS_CACHE_SIZE: usize = 4; + struct Mapping { // 'static lifetime is a lie to hack around lack of support for self-referential structs. cx: Context<'static>, _map: Mmap, + _stash: Stash, +} + +enum Either { + #[allow(dead_code)] + A(A), + B(B), } -fn cx<'data>(object: Object<'data>) -> Option> { - fn load_section<'data, S>(obj: &Object<'data>) -> S +impl Mapping { + /// Creates a `Mapping` by ensuring that the `data` specified is used to + /// create a `Context` and it can only borrow from that or the `Stash` of + /// decompressed sections or auxiliary data. + fn mk(data: Mmap, mk: F) -> Option where - S: gimli::Section>, + F: for<'a> FnOnce(&'a [u8], &'a Stash) -> Option>, { - let data = obj.section(S::section_name()).unwrap_or(&[]); - S::from(EndianSlice::new(data, Endian)) + Mapping::mk_or_other(data, move |data, stash| { + let cx = mk(data, stash)?; + Some(Either::B(cx)) + }) } - let dwarf = addr2line::Context::from_sections( - load_section(&object), - load_section(&object), - load_section(&object), - load_section(&object), - load_section(&object), - load_section(&object), - load_section(&object), - load_section(&object), - load_section(&object), - gimli::EndianSlice::new(&[], Endian), - ) - .ok()?; - Some(Context { dwarf, object }) + /// Creates a `Mapping` from `data`, or if the closure decides to, returns a + /// different mapping. + fn mk_or_other(data: Mmap, mk: F) -> Option + where + F: for<'a> FnOnce(&'a [u8], &'a Stash) -> Option>>, + { + let stash = Stash::new(); + let cx = match mk(&data, &stash)? { + Either::A(mapping) => return Some(mapping), + Either::B(cx) => cx, + }; + Some(Mapping { + // Convert to 'static lifetimes since the symbols should + // only borrow `map` and `stash` and we're preserving them below. + cx: unsafe { core::mem::transmute::, Context<'static>>(cx) }, + _map: data, + _stash: stash, + }) + } } -fn assert_lifetimes<'a>(_: &'a Mmap, _: &Context<'a>) {} +struct Context<'a> { + dwarf: addr2line::Context>, + object: Object<'a>, +} -macro_rules! mk { - (Mapping { $map:expr, $inner:expr }) => {{ - assert_lifetimes(&$map, &$inner); - Mapping { - // Convert to 'static lifetimes since the symbols should - // only borrow `map` and we're preserving `map` below. - cx: unsafe { mem::transmute::, Context<'static>>($inner) }, - _map: $map, +impl<'data> Context<'data> { + fn new( + stash: &'data Stash, + object: Object<'data>, + sup: Option>, + ) -> Option> { + let mut sections = gimli::Dwarf::load(|id| -> Result<_, ()> { + let data = object.section(stash, id.name()).unwrap_or(&[]); + Ok(EndianSlice::new(data, Endian)) + }) + .ok()?; + + if let Some(sup) = sup { + sections + .load_sup(|id| -> Result<_, ()> { + let data = sup.section(stash, id.name()).unwrap_or(&[]); + Ok(EndianSlice::new(data, Endian)) + }) + .ok()?; } - }}; + let dwarf = addr2line::Context::from_dwarf(sections).ok()?; + + Some(Context { dwarf, object }) + } } fn mmap(path: &Path) -> Option { let file = File::open(path).ok()?; - // TODO: not completely safe, see https://github.com/danburkert/memmap-rs/issues/25 - unsafe { Mmap::map(&file).ok() } + let len = file.metadata().ok()?.len().try_into().ok()?; + unsafe { Mmap::map(&file, len) } } cfg_if::cfg_if! { if #[cfg(windows)] { - use goblin::pe::{self, PE}; - use goblin::strtab::Strtab; - use std::cmp; - use std::convert::TryFrom; - - struct Object<'a> { - pe: PE<'a>, - data: &'a [u8], - symbols: Vec<(usize, pe::symbol::Symbol)>, - strtab: Strtab<'a>, - } - - impl<'a> Object<'a> { - fn parse(data: &'a [u8]) -> Option> { - let pe = PE::parse(data).ok()?; - let syms = pe.header.coff_header.symbols(data).ok()?; - let strtab = pe.header.coff_header.strings(data).ok()?; - - // Collect all the symbols into a local vector which is sorted - // by address and contains enough data to learn about the symbol - // name. Note that we only look at function symbols and also - // note that the sections are 1-indexed because the zero section - // is special (apparently). - let mut symbols = Vec::new(); - for (_, _, sym) in syms.iter() { - if sym.derived_type() != pe::symbol::IMAGE_SYM_DTYPE_FUNCTION - || sym.section_number == 0 - { - continue; - } - let addr = usize::try_from(sym.value).ok()?; - let section = pe.sections.get(usize::try_from(sym.section_number).ok()? - 1)?; - let va = usize::try_from(section.virtual_address).ok()?; - symbols.push((addr + va + pe.image_base, sym)); - } - symbols.sort_unstable_by_key(|x| x.0); - Some(Object { pe, data, symbols, strtab }) - } - - fn section(&self, name: &str) -> Option<&'a [u8]> { - let section = self.pe - .sections - .iter() - .find(|section| section.name().ok() == Some(name)); - section - .and_then(|section| { - let offset = section.pointer_to_raw_data as usize; - let size = cmp::min(section.virtual_size, section.size_of_raw_data) as usize; - self.data.get(offset..).and_then(|data| data.get(..size)) - }) - } - - fn search_symtab<'b>(&'b self, addr: u64) -> Option<&'b [u8]> { - // Note that unlike other formats COFF doesn't embed the size of - // each symbol. As a last ditch effort search for the *closest* - // symbol to a particular address and return that one. This gets - // really wonky once symbols start getting removed because the - // symbols returned here can be totally incorrect, but we have - // no idea of knowing how to detect that. - let addr = usize::try_from(addr).ok()?; - let i = match self.symbols.binary_search_by_key(&addr, |p| p.0) { - Ok(i) => i, - // typically `addr` isn't in the array, but `i` is where - // we'd insert it, so the previous position must be the - // greatest less than `addr` - Err(i) => i.checked_sub(1)?, - }; - Some(self.symbols[i].1.name(&self.strtab).ok()?.as_bytes()) - } - } - } else if #[cfg(target_os = "macos")] { - use goblin::mach::MachO; - - struct Object<'a> { - macho: MachO<'a>, - dwarf: Option, - } - - impl<'a> Object<'a> { - fn parse(macho: MachO<'a>) -> Option> { - if !macho.little_endian { - return None; - } - let dwarf = macho - .segments - .iter() - .enumerate() - .find(|(_, segment)| segment.name().ok() == Some("__DWARF")) - .map(|p| p.0); - Some(Object { macho, dwarf }) - } - - fn section(&self, name: &str) -> Option<&'a [u8]> { - let dwarf = self.dwarf?; - let dwarf = &self.macho.segments[dwarf]; - dwarf - .into_iter() - .filter_map(|s| s.ok()) - .find(|(section, _data)| { - let section_name = match section.name() { - Ok(s) => s, - Err(_) => return false, - }; - §ion_name[..] == name || { - section_name.starts_with("__") - && name.starts_with(".") - && §ion_name[2..] == &name[1..] - } - }) - .map(|p| p.1) - } - - fn search_symtab<'b>(&'b self, _addr: u64) -> Option<&'b [u8]> { - // So far it seems that we don't need to implement this. Maybe - // `dladdr` on OSX has us covered? Maybe there's not much in the - // symbol table? In any case our relevant tests are passing - // without this being implemented, so let's skip it for now. - None - } - } + mod coff; + use self::coff::Object; + } else if #[cfg(any( + target_os = "macos", + target_os = "ios", + target_os = "tvos", + target_os = "watchos", + ))] { + mod macho; + use self::macho::Object; } else { - use goblin::elf::Elf; - - struct Object<'a> { - elf: Elf<'a>, - data: &'a [u8], - // List of pre-parsed and sorted symbols by base address. The - // boolean indicates whether it comes from the dynamic symbol table - // or the normal symbol table, affecting where it's symbolicated. - syms: Vec<(goblin::elf::Sym, bool)>, - } - - impl<'a> Object<'a> { - fn parse(data: &'a [u8]) -> Option> { - let elf = Elf::parse(data).ok()?; - if !elf.little_endian { - return None; - } - let mut syms = elf - .syms - .iter() - .map(|s| (s, false)) - .chain(elf.dynsyms.iter().map(|s| (s, true))) - // Only look at function/object symbols. This mirrors what - // libbacktrace does and in general we're only symbolicating - // function addresses in theory. Object symbols correspond - // to data, and maybe someone's crazy enough to have a - // function go into static data? - .filter(|(s, _)| { - s.is_function() || s.st_type() == goblin::elf::sym::STT_OBJECT - }) - // skip anything that's in an undefined section header, - // since it means it's an imported function and we're only - // symbolicating with locally defined functions. - .filter(|(s, _)| { - s.st_shndx != goblin::elf::section_header::SHN_UNDEF as usize - }) - .collect::>(); - syms.sort_unstable_by_key(|s| s.0.st_value); - Some(Object { - syms, - elf, - data, - }) - } - - fn section(&self, name: &str) -> Option<&'a [u8]> { - let section = self.elf.section_headers.iter().find(|section| { - match self.elf.shdr_strtab.get(section.sh_name) { - Some(Ok(section_name)) => section_name == name, - _ => false, - } - }); - section - .and_then(|section| { - self.data.get(section.sh_offset as usize..) - .and_then(|data| data.get(..section.sh_size as usize)) - }) - } - - fn search_symtab<'b>(&'b self, addr: u64) -> Option<&'b [u8]> { - // Same sort of binary search as Windows above - let i = match self.syms.binary_search_by_key(&addr, |s| s.0.st_value) { - Ok(i) => i, - Err(i) => i.checked_sub(1)?, - }; - let (sym, dynamic) = self.syms.get(i)?; - if sym.st_value <= addr && addr <= sym.st_value + sym.st_size { - let strtab = if *dynamic { - &self.elf.dynstrtab - } else { - &self.elf.strtab - }; - Some(strtab.get(sym.st_name)?.ok()?.as_bytes()) - } else { - None - } - } - } + mod elf; + use self::elf::Object; } } -impl Mapping { - #[cfg(not(target_os = "macos"))] - fn new(path: &Path) -> Option { - let map = mmap(path)?; - let cx = cx(Object::parse(&map)?)?; - Some(mk!(Mapping { map, cx })) - } - - // The loading path for OSX is is so different we just have a completely - // different implementation of the function here. On OSX we need to go - // probing the filesystem for a bunch of files. - #[cfg(target_os = "macos")] - fn new(path: &Path) -> Option { - // First up we need to load the unique UUID which is stored in the macho - // header of the file we're reading, specified at `path`. - let map = mmap(path)?; - let macho = MachO::parse(&map, 0).ok()?; - let uuid = find_uuid(&macho)?; - - // Next we need to look for a `*.dSYM` file. For now we just probe the - // containing directory and look around for something that matches - // `*.dSYM`. Once it's found we root through the dwarf resources that it - // contains and try to find a macho file which has a matching UUID as - // the one of our own file. If we find a match that's the dwarf file we - // want to return. - let parent = path.parent()?; - for entry in parent.read_dir().ok()? { - let entry = entry.ok()?; - let filename = match entry.file_name().into_string() { - Ok(name) => name, - Err(_) => continue, - }; - if !filename.ends_with(".dSYM") { - continue; - } - let candidates = entry.path().join("Contents/Resources/DWARF"); - if let Some(mapping) = load_dsym(&candidates, &uuid) { - return Some(mapping); - } - } - - // Looks like nothing matched our UUID, so let's at least return our own - // file. This should have the symbol table for at least some - // symbolication purposes. - let inner = cx(Object::parse(macho)?)?; - return Some(mk!(Mapping { map, inner })); - - fn load_dsym(dir: &Path, uuid: &[u8; 16]) -> Option { - for entry in dir.read_dir().ok()? { - let entry = entry.ok()?; - let map = mmap(&entry.path())?; - let macho = MachO::parse(&map, 0).ok()?; - let entry_uuid = find_uuid(&macho)?; - if entry_uuid != uuid { - continue; - } - if let Some(cx) = Object::parse(macho).and_then(cx) { - return Some(mk!(Mapping { map, cx })); - } - } - - None - } - - fn find_uuid<'a>(object: &'a MachO) -> Option<&'a [u8; 16]> { - use goblin::mach::load_command::CommandVariant; - - object - .load_commands - .iter() - .filter_map(|cmd| match &cmd.command { - CommandVariant::Uuid(u) => Some(&u.uuid), - _ => None, - }) - .next() +cfg_if::cfg_if! { + if #[cfg(windows)] { + mod libs_windows; + use libs_windows::native_libraries; + } else if #[cfg(any( + target_os = "macos", + target_os = "ios", + target_os = "tvos", + target_os = "watchos", + ))] { + mod libs_macos; + use libs_macos::native_libraries; + } else if #[cfg(target_os = "illumos")] { + mod libs_illumos; + use libs_illumos::native_libraries; + } else if #[cfg(all( + any( + target_os = "linux", + target_os = "fuchsia", + target_os = "freebsd", + target_os = "openbsd", + all(target_os = "android", feature = "dl_iterate_phdr"), + ), + not(target_env = "uclibc"), + ))] { + mod libs_dl_iterate_phdr; + use libs_dl_iterate_phdr::native_libraries; + } else if #[cfg(target_env = "libnx")] { + mod libs_libnx; + use libs_libnx::native_libraries; + } else if #[cfg(target_os = "haiku")] { + mod libs_haiku; + use libs_haiku::native_libraries; + } else { + // Everything else should doesn't know how to load native libraries. + fn native_libraries() -> Vec { + Vec::new() } } } @@ -370,7 +205,7 @@ struct Cache { /// Mappings cache where we retain parsed dwarf information. /// - /// This list has a fixed capacity for its entire liftime which never + /// This list has a fixed capacity for its entire lifetime which never /// increases. The `usize` element of each pair is an index into `libraries` /// above where `usize::max_value()` represents the current executable. The /// `Mapping` is corresponding parsed dwarf information. @@ -382,13 +217,23 @@ struct Cache { struct Library { name: OsString, + /// Segments of this library loaded into memory, and where they're loaded. segments: Vec, - bias: findshlibs::Bias, + /// The "bias" of this library, typically where it's loaded into memory. + /// This value is added to each segment's stated address to get the actual + /// virtual memory address that the segment is loaded into. Additionally + /// this bias is subtracted from real virtual memory addresses to index into + /// debuginfo and the symbol table. + bias: usize, } struct LibrarySegment { + /// The stated address of this segment in the object file. This is not + /// actually where the segment is loaded, but rather this address plus the + /// containing library's `bias` is where to find it. + stated_virtual_memory_address: usize, + /// The size of this segment in memory. len: usize, - stated_virtual_memory_address: findshlibs::Svma, } // unsafe because this is required to be externally synchronized @@ -398,29 +243,9 @@ pub unsafe fn clear_symbol_cache() { impl Cache { fn new() -> Cache { - let mut libraries = Vec::new(); - // Iterate over all loaded shared libraries and cache information we - // learn about them. This way we only load information here once instead - // of once per symbol. - findshlibs::TargetSharedLibrary::each(|so| { - use findshlibs::IterationControl::*; - libraries.push(Library { - name: so.name().to_owned(), - segments: so - .segments() - .map(|s| LibrarySegment { - len: s.len(), - stated_virtual_memory_address: s.stated_virtual_memory_address(), - }) - .collect(), - bias: so.virtual_memory_bias(), - }); - Continue - }); - Cache { mappings: Vec::with_capacity(MAPPINGS_CACHE_SIZE), - libraries, + libraries: native_libraries(), } } @@ -441,17 +266,7 @@ impl Cache { f(MAPPINGS_CACHE.get_or_insert_with(|| Cache::new())) } - fn avma_to_svma(&self, addr: *const u8) -> Option<(usize, findshlibs::Svma)> { - // Note that for now `findshlibs` is unimplemented on Windows, so we - // just unhelpfully assume that the address is an SVMA. Surprisingly it - // seems to at least somewhat work on Wine on Linux though... - // - // This probably means ASLR on Windows is busted. - if cfg!(windows) { - let addr = findshlibs::Svma(addr); - return Some((usize::max_value(), addr)); - } - + fn avma_to_svma(&self, addr: *const u8) -> Option<(usize, *const u8)> { self.libraries .iter() .enumerate() @@ -460,28 +275,32 @@ impl Cache { // `addr` (handling relocation). If this check passes then we // can continue below and actually translate the address. // - // Note that this is an inlining of `contains_avma` in the - // `findshlibs` crate here. + // Note that we're using `wrapping_add` here to avoid overflow + // checks. It's been seen in the wild that the SVMA + bias + // computation overflows. It seems a bit odd that would happen + // but there's not a huge amount we can do about it other than + // probably just ignore those segments since they're likely + // pointing off into space. This originally came up in + // rust-lang/backtrace-rs#329. if !lib.segments.iter().any(|s| { let svma = s.stated_virtual_memory_address; - let start = unsafe { svma.0.offset(lib.bias.0) as usize }; - let end = start + s.len; + let start = svma.wrapping_add(lib.bias); + let end = start.wrapping_add(s.len); let address = addr as usize; start <= address && address < end }) { return None; } - // Now that we know `lib` contains `addr`, do the equiavlent of - // `avma_to_svma` in the `findshlibs` crate. - let reverse_bias = -lib.bias.0; - let svma = findshlibs::Svma(unsafe { addr.offset(reverse_bias) }); - Some((i, svma)) + // Now that we know `lib` contains `addr`, we can offset with + // the bias to find the stated virtual memory address. + let svma = (addr as usize).wrapping_sub(lib.bias); + Some((i, svma as *const u8)) }) .next() } - fn mapping_for_lib<'a>(&'a mut self, lib: usize) -> Option<&'a Context<'a>> { + fn mapping_for_lib<'a>(&'a mut self, lib: usize) -> Option<&'a mut Context<'a>> { let idx = self.mappings.iter().position(|(idx, _)| *idx == lib); // Invariant: after this conditional completes without early returning @@ -497,15 +316,8 @@ impl Cache { // When the mapping is not in the cache, create a new mapping, // insert it into the front of the cache, and evict the oldest cache // entry if necessary. - let storage; - let path = match self.libraries.get(lib) { - Some(lib) => &lib.name, - None => { - storage = env::current_exe().ok()?.into(); - &storage - } - }; - let mapping = Mapping::new(path.as_ref())?; + let name = &self.libraries[lib].name; + let mapping = Mapping::new(name.as_ref())?; if self.mappings.len() == MAPPINGS_CACHE_SIZE { self.mappings.pop(); @@ -514,19 +326,21 @@ impl Cache { self.mappings.insert(0, (lib, mapping)); } - let cx: &'a Context<'static> = &self.mappings[0].1.cx; + let cx: &'a mut Context<'static> = &mut self.mappings[0].1.cx; // don't leak the `'static` lifetime, make sure it's scoped to just // ourselves - Some(unsafe { mem::transmute::<&'a Context<'static>, &'a Context<'a>>(cx) }) + Some(unsafe { mem::transmute::<&'a mut Context<'static>, &'a mut Context<'a>>(cx) }) } } -pub unsafe fn resolve(what: ResolveWhat, cb: &mut FnMut(&super::Symbol)) { +pub unsafe fn resolve(what: ResolveWhat<'_>, cb: &mut dyn FnMut(&super::Symbol)) { let addr = what.address_or_ip(); - let mut cb = DladdrFallback { - cb, - addr, - called: false, + let mut call = |sym: Symbol<'_>| { + // Extend the lifetime of `sym` to `'static` since we are unfortunately + // required to here, but it's only ever going out as a reference so no + // reference to it should be persisted beyond this frame anyway. + let sym = mem::transmute::, Symbol<'static>>(sym); + (cb)(&super::Symbol { inner: sym }); }; Cache::with_global(|cache| { @@ -541,60 +355,44 @@ pub unsafe fn resolve(what: ResolveWhat, cb: &mut FnMut(&super::Symbol)) { Some(cx) => cx, None => return, }; - if let Ok(mut frames) = cx.dwarf.find_frames(addr.0 as u64) { + let mut any_frames = false; + if let Ok(mut frames) = cx.dwarf.find_frames(addr as u64) { while let Ok(Some(frame)) = frames.next() { - cb.call(Symbol::Frame { - addr: addr.0 as *mut c_void, + any_frames = true; + let name = match frame.function { + Some(f) => Some(f.name.slice()), + None => cx.object.search_symtab(addr as u64), + }; + call(Symbol::Frame { + addr: addr as *mut c_void, location: frame.location, - name: frame.function.map(|f| f.name.slice()), + name, }); } } - - if !cb.called { - if let Some(name) = cx.object.search_symtab(addr.0 as u64) { - cb.call(Symbol::Symtab { - addr: addr.0 as *mut c_void, + if !any_frames { + if let Some((object_cx, object_addr)) = cx.object.search_object_map(addr as u64) { + if let Ok(mut frames) = object_cx.dwarf.find_frames(object_addr) { + while let Ok(Some(frame)) = frames.next() { + any_frames = true; + call(Symbol::Frame { + addr: addr as *mut c_void, + location: frame.location, + name: frame.function.map(|f| f.name.slice()), + }); + } + } + } + } + if !any_frames { + if let Some(name) = cx.object.search_symtab(addr as u64) { + call(Symbol::Symtab { + addr: addr as *mut c_void, name, }); } } }); - - drop(cb); -} - -struct DladdrFallback<'a, 'b> { - addr: *mut c_void, - called: bool, - cb: &'a mut (FnMut(&super::Symbol) + 'b), -} - -impl DladdrFallback<'_, '_> { - fn call(&mut self, sym: Symbol) { - self.called = true; - - // Extend the lifetime of `sym` to `'static` since we are unfortunately - // required to here, but it's ony ever going out as a reference so no - // reference to it should be persisted beyond this frame anyway. - let sym = unsafe { mem::transmute::>(sym) }; - (self.cb)(&super::Symbol { inner: sym }); - } -} - -impl Drop for DladdrFallback<'_, '_> { - fn drop(&mut self) { - if self.called { - return; - } - unsafe { - dladdr::resolve(self.addr, &mut |sym| { - (self.cb)(&super::Symbol { - inner: Symbol::Dladdr(sym), - }) - }); - } - } } pub enum Symbol<'a> { @@ -608,15 +406,11 @@ pub enum Symbol<'a> { /// Couldn't find debug information, but we found it in the symbol table of /// the elf executable. Symtab { addr: *mut c_void, name: &'a [u8] }, - /// We weren't able to find anything in the original file, so we had to fall - /// back to using `dladdr` which had a hit. - Dladdr(dladdr::Symbol<'a>), } impl Symbol<'_> { - pub fn name(&self) -> Option { + pub fn name(&self) -> Option> { match self { - Symbol::Dladdr(s) => s.name(), Symbol::Frame { name, .. } => { let name = name.as_ref()?; Some(SymbolName::new(name)) @@ -627,15 +421,13 @@ impl Symbol<'_> { pub fn addr(&self) -> Option<*mut c_void> { match self { - Symbol::Dladdr(s) => s.addr(), Symbol::Frame { addr, .. } => Some(*addr), Symbol::Symtab { .. } => None, } } - pub fn filename_raw(&self) -> Option { + pub fn filename_raw(&self) -> Option> { match self { - Symbol::Dladdr(s) => return s.filename_raw(), Symbol::Frame { location, .. } => { let file = location.as_ref()?.file?; Some(BytesOrWideString::Bytes(file.as_bytes())) @@ -646,7 +438,6 @@ impl Symbol<'_> { pub fn filename(&self) -> Option<&Path> { match self { - Symbol::Dladdr(s) => return s.filename(), Symbol::Frame { location, .. } => { let file = location.as_ref()?.file?; Some(Path::new(file)) @@ -657,9 +448,15 @@ impl Symbol<'_> { pub fn lineno(&self) -> Option { match self { - Symbol::Dladdr(s) => return s.lineno(), Symbol::Frame { location, .. } => location.as_ref()?.line, Symbol::Symtab { .. } => None, } } + + pub fn colno(&self) -> Option { + match self { + Symbol::Frame { location, .. } => location.as_ref()?.column, + Symbol::Symtab { .. } => None, + } + } } diff --git a/crux-mir/lib/backtrace/src/symbolize/gimli/coff.rs b/crux-mir/lib/backtrace/src/symbolize/gimli/coff.rs new file mode 100644 index 000000000..84d334207 --- /dev/null +++ b/crux-mir/lib/backtrace/src/symbolize/gimli/coff.rs @@ -0,0 +1,108 @@ +use super::{Context, Mapping, Path, Stash, Vec}; +use core::convert::TryFrom; +use object::pe::{ImageDosHeader, ImageSymbol}; +use object::read::pe::{ImageNtHeaders, ImageOptionalHeader, SectionTable}; +use object::read::StringTable; +use object::LittleEndian as LE; + +#[cfg(target_pointer_width = "32")] +type Pe = object::pe::ImageNtHeaders32; +#[cfg(target_pointer_width = "64")] +type Pe = object::pe::ImageNtHeaders64; + +impl Mapping { + pub fn new(path: &Path) -> Option { + let map = super::mmap(path)?; + Mapping::mk(map, |data, stash| { + Context::new(stash, Object::parse(data)?, None) + }) + } +} + +pub struct Object<'a> { + data: &'a [u8], + sections: SectionTable<'a>, + symbols: Vec<(usize, &'a ImageSymbol)>, + strings: StringTable<'a>, +} + +pub fn get_image_base(data: &[u8]) -> Option { + let dos_header = ImageDosHeader::parse(data).ok()?; + let mut offset = dos_header.nt_headers_offset().into(); + let (nt_headers, _) = Pe::parse(data, &mut offset).ok()?; + usize::try_from(nt_headers.optional_header().image_base()).ok() +} + +impl<'a> Object<'a> { + fn parse(data: &'a [u8]) -> Option> { + let dos_header = ImageDosHeader::parse(data).ok()?; + let mut offset = dos_header.nt_headers_offset().into(); + let (nt_headers, _) = Pe::parse(data, &mut offset).ok()?; + let sections = nt_headers.sections(data, offset).ok()?; + let symtab = nt_headers.symbols(data).ok()?; + let strings = symtab.strings(); + let image_base = usize::try_from(nt_headers.optional_header().image_base()).ok()?; + + // Collect all the symbols into a local vector which is sorted + // by address and contains enough data to learn about the symbol + // name. Note that we only look at function symbols and also + // note that the sections are 1-indexed because the zero section + // is special (apparently). + let mut symbols = Vec::new(); + let mut i = 0; + let len = symtab.len(); + while i < len { + let sym = symtab.symbol(i).ok()?; + i += 1 + sym.number_of_aux_symbols as usize; + let section_number = sym.section_number.get(LE); + if sym.derived_type() != object::pe::IMAGE_SYM_DTYPE_FUNCTION || section_number == 0 { + continue; + } + let addr = usize::try_from(sym.value.get(LE)).ok()?; + let section = sections + .section(usize::try_from(section_number).ok()?) + .ok()?; + let va = usize::try_from(section.virtual_address.get(LE)).ok()?; + symbols.push((addr + va + image_base, sym)); + } + symbols.sort_unstable_by_key(|x| x.0); + Some(Object { + data, + sections, + strings, + symbols, + }) + } + + pub fn section(&self, _: &Stash, name: &str) -> Option<&'a [u8]> { + Some( + self.sections + .section_by_name(self.strings, name.as_bytes())? + .1 + .pe_data(self.data) + .ok()?, + ) + } + + pub fn search_symtab<'b>(&'b self, addr: u64) -> Option<&'b [u8]> { + // Note that unlike other formats COFF doesn't embed the size of + // each symbol. As a last ditch effort search for the *closest* + // symbol to a particular address and return that one. This gets + // really wonky once symbols start getting removed because the + // symbols returned here can be totally incorrect, but we have + // no idea of knowing how to detect that. + let addr = usize::try_from(addr).ok()?; + let i = match self.symbols.binary_search_by_key(&addr, |p| p.0) { + Ok(i) => i, + // typically `addr` isn't in the array, but `i` is where + // we'd insert it, so the previous position must be the + // greatest less than `addr` + Err(i) => i.checked_sub(1)?, + }; + self.symbols[i].1.name(self.strings).ok() + } + + pub(super) fn search_object_map(&self, _addr: u64) -> Option<(&Context<'_>, u64)> { + None + } +} diff --git a/crux-mir/lib/backtrace/src/symbolize/gimli/elf.rs b/crux-mir/lib/backtrace/src/symbolize/gimli/elf.rs new file mode 100644 index 000000000..bc71ee2c9 --- /dev/null +++ b/crux-mir/lib/backtrace/src/symbolize/gimli/elf.rs @@ -0,0 +1,423 @@ +use super::mystd::ffi::{OsStr, OsString}; +use super::mystd::fs; +use super::mystd::os::unix::ffi::{OsStrExt, OsStringExt}; +use super::mystd::path::{Path, PathBuf}; +use super::Either; +use super::{Context, Mapping, Stash, Vec}; +use core::convert::{TryFrom, TryInto}; +use core::str; +use object::elf::{ELFCOMPRESS_ZLIB, ELF_NOTE_GNU, NT_GNU_BUILD_ID, SHF_COMPRESSED}; +use object::read::elf::{CompressionHeader, FileHeader, SectionHeader, SectionTable, Sym}; +use object::read::StringTable; +use object::{BigEndian, Bytes, NativeEndian}; + +#[cfg(target_pointer_width = "32")] +type Elf = object::elf::FileHeader32; +#[cfg(target_pointer_width = "64")] +type Elf = object::elf::FileHeader64; + +impl Mapping { + pub fn new(path: &Path) -> Option { + let map = super::mmap(path)?; + Mapping::mk_or_other(map, |map, stash| { + let object = Object::parse(&map)?; + + // Try to locate an external debug file using the build ID. + if let Some(path_debug) = object.build_id().and_then(locate_build_id) { + if let Some(mapping) = Mapping::new_debug(path_debug, None) { + return Some(Either::A(mapping)); + } + } + + // Try to locate an external debug file using the GNU debug link section. + if let Some((path_debug, crc)) = object.gnu_debuglink_path(path) { + if let Some(mapping) = Mapping::new_debug(path_debug, Some(crc)) { + return Some(Either::A(mapping)); + } + } + + Context::new(stash, object, None).map(Either::B) + }) + } + + /// Load debuginfo from an external debug file. + fn new_debug(path: PathBuf, crc: Option) -> Option { + let map = super::mmap(&path)?; + Mapping::mk(map, |map, stash| { + let object = Object::parse(&map)?; + + if let Some(_crc) = crc { + // TODO: check crc + } + + // Try to locate a supplementary object file. + if let Some((path_sup, build_id_sup)) = object.gnu_debugaltlink_path(&path) { + if let Some(map_sup) = super::mmap(&path_sup) { + let map_sup = stash.set_mmap_aux(map_sup); + if let Some(sup) = Object::parse(map_sup) { + if sup.build_id() == Some(build_id_sup) { + return Context::new(stash, object, Some(sup)); + } + } + } + } + + Context::new(stash, object, None) + }) + } +} + +struct ParsedSym { + address: u64, + size: u64, + name: u32, +} + +pub struct Object<'a> { + /// Zero-sized type representing the native endianness. + /// + /// We could use a literal instead, but this helps ensure correctness. + endian: NativeEndian, + /// The entire file data. + data: &'a [u8], + sections: SectionTable<'a, Elf>, + strings: StringTable<'a>, + /// List of pre-parsed and sorted symbols by base address. + syms: Vec, +} + +impl<'a> Object<'a> { + fn parse(data: &'a [u8]) -> Option> { + let elf = Elf::parse(data).ok()?; + let endian = elf.endian().ok()?; + let sections = elf.sections(endian, data).ok()?; + let mut syms = sections + .symbols(endian, data, object::elf::SHT_SYMTAB) + .ok()?; + if syms.is_empty() { + syms = sections + .symbols(endian, data, object::elf::SHT_DYNSYM) + .ok()?; + } + let strings = syms.strings(); + + let mut syms = syms + .iter() + // Only look at function/object symbols. This mirrors what + // libbacktrace does and in general we're only symbolicating + // function addresses in theory. Object symbols correspond + // to data, and maybe someone's crazy enough to have a + // function go into static data? + .filter(|sym| { + let st_type = sym.st_type(); + st_type == object::elf::STT_FUNC || st_type == object::elf::STT_OBJECT + }) + // skip anything that's in an undefined section header, + // since it means it's an imported function and we're only + // symbolicating with locally defined functions. + .filter(|sym| sym.st_shndx(endian) != object::elf::SHN_UNDEF) + .map(|sym| { + let address = sym.st_value(endian).into(); + let size = sym.st_size(endian).into(); + let name = sym.st_name(endian); + ParsedSym { + address, + size, + name, + } + }) + .collect::>(); + syms.sort_unstable_by_key(|s| s.address); + Some(Object { + endian, + data, + sections, + strings, + syms, + }) + } + + pub fn section(&self, stash: &'a Stash, name: &str) -> Option<&'a [u8]> { + if let Some(section) = self.section_header(name) { + let mut data = Bytes(section.data(self.endian, self.data).ok()?); + + // Check for DWARF-standard (gABI) compression, i.e., as generated + // by ld's `--compress-debug-sections=zlib-gabi` flag. + let flags: u64 = section.sh_flags(self.endian).into(); + if (flags & u64::from(SHF_COMPRESSED)) == 0 { + // Not compressed. + return Some(data.0); + } + + let header = data.read::<::CompressionHeader>().ok()?; + if header.ch_type(self.endian) != ELFCOMPRESS_ZLIB { + // Zlib compression is the only known type. + return None; + } + let size = usize::try_from(header.ch_size(self.endian)).ok()?; + let buf = stash.allocate(size); + decompress_zlib(data.0, buf)?; + return Some(buf); + } + + // Check for the nonstandard GNU compression format, i.e., as generated + // by ld's `--compress-debug-sections=zlib-gnu` flag. This means that if + // we're actually asking for `.debug_info` then we need to look up a + // section named `.zdebug_info`. + if !name.starts_with(".debug_") { + return None; + } + let debug_name = name[7..].as_bytes(); + let compressed_section = self + .sections + .iter() + .filter_map(|header| { + let name = self.sections.section_name(self.endian, header).ok()?; + if name.starts_with(b".zdebug_") && &name[8..] == debug_name { + Some(header) + } else { + None + } + }) + .next()?; + let mut data = Bytes(compressed_section.data(self.endian, self.data).ok()?); + if data.read_bytes(8).ok()?.0 != b"ZLIB\0\0\0\0" { + return None; + } + let size = usize::try_from(data.read::>().ok()?.get(BigEndian)).ok()?; + let buf = stash.allocate(size); + decompress_zlib(data.0, buf)?; + Some(buf) + } + + fn section_header(&self, name: &str) -> Option<&::SectionHeader> { + self.sections + .section_by_name(self.endian, name.as_bytes()) + .map(|(_index, section)| section) + } + + pub fn search_symtab<'b>(&'b self, addr: u64) -> Option<&'b [u8]> { + // Same sort of binary search as Windows above + let i = match self.syms.binary_search_by_key(&addr, |sym| sym.address) { + Ok(i) => i, + Err(i) => i.checked_sub(1)?, + }; + let sym = self.syms.get(i)?; + if sym.address <= addr && addr <= sym.address + sym.size { + self.strings.get(sym.name).ok() + } else { + None + } + } + + pub(super) fn search_object_map(&self, _addr: u64) -> Option<(&Context<'_>, u64)> { + None + } + + fn build_id(&self) -> Option<&'a [u8]> { + for section in self.sections.iter() { + if let Ok(Some(mut notes)) = section.notes(self.endian, self.data) { + while let Ok(Some(note)) = notes.next() { + if note.name() == ELF_NOTE_GNU && note.n_type(self.endian) == NT_GNU_BUILD_ID { + return Some(note.desc()); + } + } + } + } + None + } + + // The contents of the ".gnu_debuglink" section is documented at: + // https://sourceware.org/gdb/onlinedocs/gdb/Separate-Debug-Files.html + fn gnu_debuglink_path(&self, path: &Path) -> Option<(PathBuf, u32)> { + let section = self.section_header(".gnu_debuglink")?; + let data = section.data(self.endian, self.data).ok()?; + let len = data.iter().position(|x| *x == 0)?; + let filename = &data[..len]; + let offset = (len + 1 + 3) & !3; + let crc_bytes = data + .get(offset..offset + 4) + .and_then(|bytes| bytes.try_into().ok())?; + let crc = u32::from_ne_bytes(crc_bytes); + let path_debug = locate_debuglink(path, filename)?; + Some((path_debug, crc)) + } + + // The format of the ".gnu_debugaltlink" section is based on gdb. + fn gnu_debugaltlink_path(&self, path: &Path) -> Option<(PathBuf, &'a [u8])> { + let section = self.section_header(".gnu_debugaltlink")?; + let data = section.data(self.endian, self.data).ok()?; + let len = data.iter().position(|x| *x == 0)?; + let filename = &data[..len]; + let build_id = &data[len + 1..]; + let path_sup = locate_debugaltlink(path, filename, build_id)?; + Some((path_sup, build_id)) + } +} + +fn decompress_zlib(input: &[u8], output: &mut [u8]) -> Option<()> { + use miniz_oxide::inflate::core::inflate_flags::{ + TINFL_FLAG_PARSE_ZLIB_HEADER, TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF, + }; + use miniz_oxide::inflate::core::{decompress, DecompressorOxide}; + use miniz_oxide::inflate::TINFLStatus; + + let (status, in_read, out_read) = decompress( + &mut DecompressorOxide::new(), + input, + output, + 0, + TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF | TINFL_FLAG_PARSE_ZLIB_HEADER, + ); + if status == TINFLStatus::Done && in_read == input.len() && out_read == output.len() { + Some(()) + } else { + None + } +} + +const DEBUG_PATH: &[u8] = b"/usr/lib/debug"; + +fn debug_path_exists() -> bool { + cfg_if::cfg_if! { + if #[cfg(any(target_os = "freebsd", target_os = "linux"))] { + use core::sync::atomic::{AtomicU8, Ordering}; + static DEBUG_PATH_EXISTS: AtomicU8 = AtomicU8::new(0); + + let mut exists = DEBUG_PATH_EXISTS.load(Ordering::Relaxed); + if exists == 0 { + exists = if Path::new(OsStr::from_bytes(DEBUG_PATH)).is_dir() { + 1 + } else { + 2 + }; + DEBUG_PATH_EXISTS.store(exists, Ordering::Relaxed); + } + exists == 1 + } else { + false + } + } +} + +/// Locate a debug file based on its build ID. +/// +/// The format of build id paths is documented at: +/// https://sourceware.org/gdb/onlinedocs/gdb/Separate-Debug-Files.html +fn locate_build_id(build_id: &[u8]) -> Option { + const BUILD_ID_PATH: &[u8] = b"/usr/lib/debug/.build-id/"; + const BUILD_ID_SUFFIX: &[u8] = b".debug"; + + if build_id.len() < 2 { + return None; + } + + if !debug_path_exists() { + return None; + } + + let mut path = + Vec::with_capacity(BUILD_ID_PATH.len() + BUILD_ID_SUFFIX.len() + build_id.len() * 2 + 1); + path.extend(BUILD_ID_PATH); + path.push(hex(build_id[0] >> 4)); + path.push(hex(build_id[0] & 0xf)); + path.push(b'/'); + for byte in &build_id[1..] { + path.push(hex(byte >> 4)); + path.push(hex(byte & 0xf)); + } + path.extend(BUILD_ID_SUFFIX); + Some(PathBuf::from(OsString::from_vec(path))) +} + +fn hex(byte: u8) -> u8 { + if byte < 10 { + b'0' + byte + } else { + b'a' + byte - 10 + } +} + +/// Locate a file specified in a `.gnu_debuglink` section. +/// +/// `path` is the file containing the section. +/// `filename` is from the contents of the section. +/// +/// Search order is based on gdb, documented at: +/// https://sourceware.org/gdb/onlinedocs/gdb/Separate-Debug-Files.html +/// +/// gdb also allows the user to customize the debug search path, but we don't. +/// +/// gdb also supports debuginfod, but we don't yet. +fn locate_debuglink(path: &Path, filename: &[u8]) -> Option { + let path = fs::canonicalize(path).ok()?; + let parent = path.parent()?; + let mut f = PathBuf::from(OsString::with_capacity( + DEBUG_PATH.len() + parent.as_os_str().len() + filename.len() + 2, + )); + let filename = Path::new(OsStr::from_bytes(filename)); + + // Try "/parent/filename" if it differs from "path" + f.push(parent); + f.push(filename); + if f != path && f.is_file() { + return Some(f); + } + + // Try "/parent/.debug/filename" + let mut s = OsString::from(f); + s.clear(); + f = PathBuf::from(s); + f.push(parent); + f.push(".debug"); + f.push(filename); + if f.is_file() { + return Some(f); + } + + if debug_path_exists() { + // Try "/usr/lib/debug/parent/filename" + let mut s = OsString::from(f); + s.clear(); + f = PathBuf::from(s); + f.push(OsStr::from_bytes(DEBUG_PATH)); + f.push(parent.strip_prefix("/").unwrap()); + f.push(filename); + if f.is_file() { + return Some(f); + } + } + + None +} + +/// Locate a file specified in a `.gnu_debugaltlink` section. +/// +/// `path` is the file containing the section. +/// `filename` and `build_id` are the contents of the section. +/// +/// Search order is based on gdb: +/// - filename, which is either absolute or relative to `path` +/// - the build ID path under `BUILD_ID_PATH` +/// +/// gdb also allows the user to customize the debug search path, but we don't. +/// +/// gdb also supports debuginfod, but we don't yet. +fn locate_debugaltlink(path: &Path, filename: &[u8], build_id: &[u8]) -> Option { + let filename = Path::new(OsStr::from_bytes(filename)); + if filename.is_absolute() { + if filename.is_file() { + return Some(filename.into()); + } + } else { + let path = fs::canonicalize(path).ok()?; + let parent = path.parent()?; + let mut f = PathBuf::from(parent); + f.push(filename); + if f.is_file() { + return Some(f); + } + } + + locate_build_id(build_id) +} diff --git a/crux-mir/lib/backtrace/src/symbolize/gimli/libs_dl_iterate_phdr.rs b/crux-mir/lib/backtrace/src/symbolize/gimli/libs_dl_iterate_phdr.rs new file mode 100644 index 000000000..a011e6080 --- /dev/null +++ b/crux-mir/lib/backtrace/src/symbolize/gimli/libs_dl_iterate_phdr.rs @@ -0,0 +1,53 @@ +// Other Unix (e.g. Linux) platforms use ELF as an object file format +// and typically implement an API called `dl_iterate_phdr` to load +// native libraries. + +use super::mystd::borrow::ToOwned; +use super::mystd::env; +use super::mystd::ffi::{CStr, OsStr}; +use super::mystd::os::unix::prelude::*; +use super::{Library, LibrarySegment, OsString, Vec}; +use core::slice; + +pub(super) fn native_libraries() -> Vec { + let mut ret = Vec::new(); + unsafe { + libc::dl_iterate_phdr(Some(callback), &mut ret as *mut Vec<_> as *mut _); + } + return ret; +} + +// `info` should be a valid pointers. +// `vec` should be a valid pointer to a `std::Vec`. +unsafe extern "C" fn callback( + info: *mut libc::dl_phdr_info, + _size: libc::size_t, + vec: *mut libc::c_void, +) -> libc::c_int { + let info = &*info; + let libs = &mut *(vec as *mut Vec); + let is_main_prog = info.dlpi_name.is_null() || *info.dlpi_name == 0; + let name = if is_main_prog { + if libs.is_empty() { + env::current_exe().map(|e| e.into()).unwrap_or_default() + } else { + OsString::new() + } + } else { + let bytes = CStr::from_ptr(info.dlpi_name).to_bytes(); + OsStr::from_bytes(bytes).to_owned() + }; + let headers = slice::from_raw_parts(info.dlpi_phdr, info.dlpi_phnum as usize); + libs.push(Library { + name, + segments: headers + .iter() + .map(|header| LibrarySegment { + len: (*header).p_memsz as usize, + stated_virtual_memory_address: (*header).p_vaddr as usize, + }) + .collect(), + bias: info.dlpi_addr as usize, + }); + 0 +} diff --git a/crux-mir/lib/backtrace/src/symbolize/gimli/libs_haiku.rs b/crux-mir/lib/backtrace/src/symbolize/gimli/libs_haiku.rs new file mode 100644 index 000000000..87e023e69 --- /dev/null +++ b/crux-mir/lib/backtrace/src/symbolize/gimli/libs_haiku.rs @@ -0,0 +1,48 @@ +// Haiku implements the image_info struct and the get_next_image_info() +// functions to iterate through the loaded executable images. The +// image_info struct contains a pointer to the start of the .text +// section within the virtual address space, as well as the size of +// that section. All the read-only segments of the ELF-binary are in +// that part of the address space. + +use super::mystd::borrow::ToOwned; +use super::mystd::ffi::{CStr, OsStr}; +use super::mystd::mem::MaybeUninit; +use super::mystd::os::unix::prelude::*; +use super::{Library, LibrarySegment, Vec}; + +pub(super) fn native_libraries() -> Vec { + let mut libraries: Vec = Vec::new(); + + unsafe { + let mut info = MaybeUninit::::zeroed(); + let mut cookie: i32 = 0; + // Load the first image to get a valid info struct + let mut status = + libc::get_next_image_info(libc::B_CURRENT_TEAM, &mut cookie, info.as_mut_ptr()); + if status != libc::B_OK { + return libraries; + } + let mut info = info.assume_init(); + + while status == libc::B_OK { + let mut segments = Vec::new(); + segments.push(LibrarySegment { + stated_virtual_memory_address: 0, + len: info.text_size as usize, + }); + + let bytes = CStr::from_ptr(info.name.as_ptr()).to_bytes(); + let name = OsStr::from_bytes(bytes).to_owned(); + libraries.push(Library { + name: name, + segments: segments, + bias: info.text as usize, + }); + + status = libc::get_next_image_info(libc::B_CURRENT_TEAM, &mut cookie, &mut info); + } + } + + libraries +} diff --git a/crux-mir/lib/backtrace/src/symbolize/gimli/libs_illumos.rs b/crux-mir/lib/backtrace/src/symbolize/gimli/libs_illumos.rs new file mode 100644 index 000000000..e64975e0c --- /dev/null +++ b/crux-mir/lib/backtrace/src/symbolize/gimli/libs_illumos.rs @@ -0,0 +1,99 @@ +use super::mystd::borrow::ToOwned; +use super::mystd::ffi::{CStr, OsStr}; +use super::mystd::os::unix::prelude::*; +use super::{Library, LibrarySegment, Vec}; +use core::mem; +use object::NativeEndian; + +#[cfg(target_pointer_width = "64")] +use object::elf::{FileHeader64 as FileHeader, ProgramHeader64 as ProgramHeader}; + +type EHdr = FileHeader; +type PHdr = ProgramHeader; + +#[repr(C)] +struct LinkMap { + l_addr: libc::c_ulong, + l_name: *const libc::c_char, + l_ld: *const libc::c_void, + l_next: *const LinkMap, + l_prev: *const LinkMap, + l_refname: *const libc::c_char, +} + +const RTLD_SELF: *const libc::c_void = -3isize as *const libc::c_void; +const RTLD_DI_LINKMAP: libc::c_int = 2; + +extern "C" { + fn dlinfo( + handle: *const libc::c_void, + request: libc::c_int, + p: *mut libc::c_void, + ) -> libc::c_int; +} + +pub(super) fn native_libraries() -> Vec { + let mut libs = Vec::new(); + + // Request the current link map from the runtime linker: + let map = unsafe { + let mut map: *const LinkMap = mem::zeroed(); + if dlinfo( + RTLD_SELF, + RTLD_DI_LINKMAP, + (&mut map) as *mut *const LinkMap as *mut libc::c_void, + ) != 0 + { + return libs; + } + map + }; + + // Each entry in the link map represents a loaded object: + let mut l = map; + while !l.is_null() { + // Fetch the fully qualified path of the loaded object: + let bytes = unsafe { CStr::from_ptr((*l).l_name) }.to_bytes(); + let name = OsStr::from_bytes(bytes).to_owned(); + + // The base address of the object loaded into memory: + let addr = unsafe { (*l).l_addr }; + + // Use the ELF header for this object to locate the program + // header: + let e: *const EHdr = unsafe { (*l).l_addr as *const EHdr }; + let phoff = unsafe { (*e).e_phoff }.get(NativeEndian); + let phnum = unsafe { (*e).e_phnum }.get(NativeEndian); + let etype = unsafe { (*e).e_type }.get(NativeEndian); + + let phdr: *const PHdr = (addr + phoff) as *const PHdr; + let phdr = unsafe { core::slice::from_raw_parts(phdr, phnum as usize) }; + + libs.push(Library { + name, + segments: phdr + .iter() + .map(|p| { + let memsz = p.p_memsz.get(NativeEndian); + let vaddr = p.p_vaddr.get(NativeEndian); + LibrarySegment { + len: memsz as usize, + stated_virtual_memory_address: vaddr as usize, + } + }) + .collect(), + bias: if etype == object::elf::ET_EXEC { + // Program header addresses for the base executable are + // already absolute. + 0 + } else { + // Other addresses are relative to the object base. + addr as usize + }, + }); + + l = unsafe { (*l).l_next }; + } + + libs +} diff --git a/crux-mir/lib/backtrace/src/symbolize/gimli/libs_libnx.rs b/crux-mir/lib/backtrace/src/symbolize/gimli/libs_libnx.rs new file mode 100644 index 000000000..93b5ba17e --- /dev/null +++ b/crux-mir/lib/backtrace/src/symbolize/gimli/libs_libnx.rs @@ -0,0 +1,27 @@ +use super::{Library, LibrarySegment, Vec}; + +// DevkitA64 doesn't natively support debug info, but the build system will +// place debug info at the path `romfs:/debug_info.elf`. +pub(super) fn native_libraries() -> Vec { + extern "C" { + static __start__: u8; + } + + let bias = unsafe { &__start__ } as *const u8 as usize; + + let mut ret = Vec::new(); + let mut segments = Vec::new(); + segments.push(LibrarySegment { + stated_virtual_memory_address: 0, + len: usize::max_value() - bias, + }); + + let path = "romfs:/debug_info.elf"; + ret.push(Library { + name: path.into(), + segments, + bias, + }); + + ret +} diff --git a/crux-mir/lib/backtrace/src/symbolize/gimli/libs_macos.rs b/crux-mir/lib/backtrace/src/symbolize/gimli/libs_macos.rs new file mode 100644 index 000000000..17703b88a --- /dev/null +++ b/crux-mir/lib/backtrace/src/symbolize/gimli/libs_macos.rs @@ -0,0 +1,146 @@ +#![allow(deprecated)] + +use super::mystd::ffi::{CStr, OsStr}; +use super::mystd::os::unix::prelude::*; +use super::mystd::prelude::v1::*; +use super::{Library, LibrarySegment}; +use core::convert::TryInto; +use core::mem; + +pub(super) fn native_libraries() -> Vec { + let mut ret = Vec::new(); + let images = unsafe { libc::_dyld_image_count() }; + for i in 0..images { + ret.extend(native_library(i)); + } + return ret; +} + +fn native_library(i: u32) -> Option { + use object::macho; + use object::read::macho::{MachHeader, Segment}; + use object::NativeEndian; + + // Fetch the name of this library which corresponds to the path of + // where to load it as well. + let name = unsafe { + let name = libc::_dyld_get_image_name(i); + if name.is_null() { + return None; + } + CStr::from_ptr(name) + }; + + // Load the image header of this library and delegate to `object` to + // parse all the load commands so we can figure out all the segments + // involved here. + let (mut load_commands, endian) = unsafe { + let header = libc::_dyld_get_image_header(i); + if header.is_null() { + return None; + } + match (*header).magic { + macho::MH_MAGIC => { + let endian = NativeEndian; + let header = &*(header as *const macho::MachHeader32); + let data = core::slice::from_raw_parts( + header as *const _ as *const u8, + mem::size_of_val(header) + header.sizeofcmds.get(endian) as usize, + ); + (header.load_commands(endian, data, 0).ok()?, endian) + } + macho::MH_MAGIC_64 => { + let endian = NativeEndian; + let header = &*(header as *const macho::MachHeader64); + let data = core::slice::from_raw_parts( + header as *const _ as *const u8, + mem::size_of_val(header) + header.sizeofcmds.get(endian) as usize, + ); + (header.load_commands(endian, data, 0).ok()?, endian) + } + _ => return None, + } + }; + + // Iterate over the segments and register known regions for segments + // that we find. Additionally record information bout text segments + // for processing later, see comments below. + let mut segments = Vec::new(); + let mut first_text = 0; + let mut text_fileoff_zero = false; + while let Some(cmd) = load_commands.next().ok()? { + if let Some((seg, _)) = cmd.segment_32().ok()? { + if seg.name() == b"__TEXT" { + first_text = segments.len(); + if seg.fileoff(endian) == 0 && seg.filesize(endian) > 0 { + text_fileoff_zero = true; + } + } + segments.push(LibrarySegment { + len: seg.vmsize(endian).try_into().ok()?, + stated_virtual_memory_address: seg.vmaddr(endian).try_into().ok()?, + }); + } + if let Some((seg, _)) = cmd.segment_64().ok()? { + if seg.name() == b"__TEXT" { + first_text = segments.len(); + if seg.fileoff(endian) == 0 && seg.filesize(endian) > 0 { + text_fileoff_zero = true; + } + } + segments.push(LibrarySegment { + len: seg.vmsize(endian).try_into().ok()?, + stated_virtual_memory_address: seg.vmaddr(endian).try_into().ok()?, + }); + } + } + + // Determine the "slide" for this library which ends up being the + // bias we use to figure out where in memory objects are loaded. + // This is a bit of a weird computation though and is the result of + // trying a few things in the wild and seeing what sticks. + // + // The general idea is that the `bias` plus a segment's + // `stated_virtual_memory_address` is going to be where in the + // actual address space the segment resides. The other thing we rely + // on though is that a real address minus the `bias` is the index to + // look up in the symbol table and debuginfo. + // + // It turns out, though, that for system loaded libraries these + // calculations are incorrect. For native executables, however, it + // appears correct. Lifting some logic from LLDB's source it has + // some special-casing for the first `__TEXT` section loaded from + // file offset 0 with a nonzero size. For whatever reason when this + // is present it appears to mean that the symbol table is relative + // to just the vmaddr slide for the library. If it's *not* present + // then the symbol table is relative to the the vmaddr slide plus + // the segment's stated address. + // + // To handle this situation if we *don't* find a text section at + // file offset zero then we increase the bias by the first text + // sections's stated address and decrease all stated addresses by + // that amount as well. That way the symbol table is always appears + // relative to the library's bias amount. This appears to have the + // right results for symbolizing via the symbol table. + // + // Honestly I'm not entirely sure whether this is right or if + // there's something else that should indicate how to do this. For + // now though this seems to work well enough (?) and we should + // always be able to tweak this over time if necessary. + // + // For some more information see #318 + let mut slide = unsafe { libc::_dyld_get_image_vmaddr_slide(i) as usize }; + if !text_fileoff_zero { + let adjust = segments[first_text].stated_virtual_memory_address; + for segment in segments.iter_mut() { + segment.stated_virtual_memory_address -= adjust; + } + slide += adjust; + } + + Some(Library { + name: OsStr::from_bytes(name.to_bytes()).to_owned(), + segments, + bias: slide, + }) +} diff --git a/crux-mir/lib/backtrace/src/symbolize/gimli/libs_windows.rs b/crux-mir/lib/backtrace/src/symbolize/gimli/libs_windows.rs new file mode 100644 index 000000000..b47ed4245 --- /dev/null +++ b/crux-mir/lib/backtrace/src/symbolize/gimli/libs_windows.rs @@ -0,0 +1,89 @@ +use super::super::super::windows::*; +use super::mystd::os::windows::prelude::*; +use super::{coff, mmap, Library, LibrarySegment, OsString}; +use alloc::vec; +use alloc::vec::Vec; +use core::mem; +use core::mem::MaybeUninit; + +// For loading native libraries on Windows, see some discussion on +// rust-lang/rust#71060 for the various strategies here. +pub(super) fn native_libraries() -> Vec { + let mut ret = Vec::new(); + unsafe { + add_loaded_images(&mut ret); + } + return ret; +} + +unsafe fn add_loaded_images(ret: &mut Vec) { + let snap = CreateToolhelp32Snapshot(TH32CS_SNAPMODULE, 0); + if snap == INVALID_HANDLE_VALUE { + return; + } + + let mut me = MaybeUninit::::zeroed().assume_init(); + me.dwSize = mem::size_of_val(&me) as DWORD; + if Module32FirstW(snap, &mut me) == TRUE { + loop { + if let Some(lib) = load_library(&me) { + ret.push(lib); + } + + if Module32NextW(snap, &mut me) != TRUE { + break; + } + } + } + + CloseHandle(snap); +} + +unsafe fn load_library(me: &MODULEENTRY32W) -> Option { + let pos = me + .szExePath + .iter() + .position(|i| *i == 0) + .unwrap_or(me.szExePath.len()); + let name = OsString::from_wide(&me.szExePath[..pos]); + + // MinGW libraries currently don't support ASLR + // (rust-lang/rust#16514), but DLLs can still be relocated around in + // the address space. It appears that addresses in debug info are + // all as-if this library was loaded at its "image base", which is a + // field in its COFF file headers. Since this is what debuginfo + // seems to list we parse the symbol table and store addresses as if + // the library was loaded at "image base" as well. + // + // The library may not be loaded at "image base", however. + // (presumably something else may be loaded there?) This is where + // the `bias` field comes into play, and we need to figure out the + // value of `bias` here. Unfortunately though it's not clear how to + // acquire this from a loaded module. What we do have, however, is + // the actual load address (`modBaseAddr`). + // + // As a bit of a cop-out for now we mmap the file, read the file + // header information, then drop the mmap. This is wasteful because + // we'll probably reopen the mmap later, but this should work well + // enough for now. + // + // Once we have the `image_base` (desired load location) and the + // `base_addr` (actual load location) we can fill in the `bias` + // (difference between the actual and desired) and then the stated + // address of each segment is the `image_base` since that's what the + // file says. + // + // For now it appears that unlike ELF/MachO we can make do with one + // segment per library, using `modBaseSize` as the whole size. + let mmap = mmap(name.as_ref())?; + let image_base = coff::get_image_base(&mmap)?; + let base_addr = me.modBaseAddr as usize; + Some(Library { + name, + bias: base_addr.wrapping_sub(image_base), + segments: vec![LibrarySegment { + stated_virtual_memory_address: image_base, + len: me.modBaseSize as usize, + }], + }) +} diff --git a/crux-mir/lib/backtrace/src/symbolize/gimli/macho.rs b/crux-mir/lib/backtrace/src/symbolize/gimli/macho.rs new file mode 100644 index 000000000..ec5673843 --- /dev/null +++ b/crux-mir/lib/backtrace/src/symbolize/gimli/macho.rs @@ -0,0 +1,324 @@ +use super::{Box, Context, Mapping, Path, Stash, Vec}; +use core::convert::TryInto; +use object::macho; +use object::read::macho::{MachHeader, Nlist, Section, Segment as _}; +use object::{Bytes, NativeEndian}; + +#[cfg(target_pointer_width = "32")] +type Mach = object::macho::MachHeader32; +#[cfg(target_pointer_width = "64")] +type Mach = object::macho::MachHeader64; +type MachSegment = ::Segment; +type MachSection = ::Section; +type MachNlist = ::Nlist; + +impl Mapping { + // The loading path for OSX is is so different we just have a completely + // different implementation of the function here. On OSX we need to go + // probing the filesystem for a bunch of files. + pub fn new(path: &Path) -> Option { + // First up we need to load the unique UUID which is stored in the macho + // header of the file we're reading, specified at `path`. + let map = super::mmap(path)?; + let (macho, data) = find_header(&map)?; + let endian = macho.endian().ok()?; + let uuid = macho.uuid(endian, data, 0).ok()?; + + // Next we need to look for a `*.dSYM` file. For now we just probe the + // containing directory and look around for something that matches + // `*.dSYM`. Once it's found we root through the dwarf resources that it + // contains and try to find a macho file which has a matching UUID as + // the one of our own file. If we find a match that's the dwarf file we + // want to return. + if let Some(uuid) = uuid { + if let Some(parent) = path.parent() { + if let Some(mapping) = Mapping::load_dsym(parent, uuid) { + return Some(mapping); + } + } + } + + // Looks like nothing matched our UUID, so let's at least return our own + // file. This should have the symbol table for at least some + // symbolication purposes. + Mapping::mk(map, |data, stash| { + let (macho, data) = find_header(data)?; + let endian = macho.endian().ok()?; + let obj = Object::parse(macho, endian, data)?; + Context::new(stash, obj, None) + }) + } + + fn load_dsym(dir: &Path, uuid: [u8; 16]) -> Option { + for entry in dir.read_dir().ok()? { + let entry = entry.ok()?; + let filename = match entry.file_name().into_string() { + Ok(name) => name, + Err(_) => continue, + }; + if !filename.ends_with(".dSYM") { + continue; + } + let candidates = entry.path().join("Contents/Resources/DWARF"); + if let Some(mapping) = Mapping::try_dsym_candidate(&candidates, uuid) { + return Some(mapping); + } + } + None + } + + fn try_dsym_candidate(dir: &Path, uuid: [u8; 16]) -> Option { + // Look for files in the `DWARF` directory which have a matching uuid to + // the original object file. If we find one then we found the debug + // information. + for entry in dir.read_dir().ok()? { + let entry = entry.ok()?; + let map = super::mmap(&entry.path())?; + let candidate = Mapping::mk(map, |data, stash| { + let (macho, data) = find_header(data)?; + let endian = macho.endian().ok()?; + let entry_uuid = macho.uuid(endian, data, 0).ok()??; + if entry_uuid != uuid { + return None; + } + let obj = Object::parse(macho, endian, data)?; + Context::new(stash, obj, None) + }); + if let Some(candidate) = candidate { + return Some(candidate); + } + } + + None + } +} + +fn find_header(data: &'_ [u8]) -> Option<(&'_ Mach, &'_ [u8])> { + use object::endian::BigEndian; + + let desired_cpu = || { + if cfg!(target_arch = "x86") { + Some(macho::CPU_TYPE_X86) + } else if cfg!(target_arch = "x86_64") { + Some(macho::CPU_TYPE_X86_64) + } else if cfg!(target_arch = "arm") { + Some(macho::CPU_TYPE_ARM) + } else if cfg!(target_arch = "aarch64") { + Some(macho::CPU_TYPE_ARM64) + } else { + None + } + }; + + let mut data = Bytes(data); + match data + .clone() + .read::>() + .ok()? + .get(NativeEndian) + { + macho::MH_MAGIC_64 | macho::MH_CIGAM_64 | macho::MH_MAGIC | macho::MH_CIGAM => {} + + macho::FAT_MAGIC | macho::FAT_CIGAM => { + let mut header_data = data; + let endian = BigEndian; + let header = header_data.read::().ok()?; + let nfat = header.nfat_arch.get(endian); + let arch = (0..nfat) + .filter_map(|_| header_data.read::().ok()) + .find(|arch| desired_cpu() == Some(arch.cputype.get(endian)))?; + let offset = arch.offset.get(endian); + let size = arch.size.get(endian); + data = data + .read_bytes_at(offset.try_into().ok()?, size.try_into().ok()?) + .ok()?; + } + + macho::FAT_MAGIC_64 | macho::FAT_CIGAM_64 => { + let mut header_data = data; + let endian = BigEndian; + let header = header_data.read::().ok()?; + let nfat = header.nfat_arch.get(endian); + let arch = (0..nfat) + .filter_map(|_| header_data.read::().ok()) + .find(|arch| desired_cpu() == Some(arch.cputype.get(endian)))?; + let offset = arch.offset.get(endian); + let size = arch.size.get(endian); + data = data + .read_bytes_at(offset.try_into().ok()?, size.try_into().ok()?) + .ok()?; + } + + _ => return None, + } + + Mach::parse(data.0, 0).ok().map(|h| (h, data.0)) +} + +// This is used both for executables/libraries and source object files. +pub struct Object<'a> { + endian: NativeEndian, + data: &'a [u8], + dwarf: Option<&'a [MachSection]>, + syms: Vec<(&'a [u8], u64)>, + syms_sort_by_name: bool, + // Only set for executables/libraries, and not the source object files. + object_map: Option>, + // The outer Option is for lazy loading, and the inner Option allows load errors to be cached. + object_mappings: Box<[Option>]>, +} + +impl<'a> Object<'a> { + fn parse(mach: &'a Mach, endian: NativeEndian, data: &'a [u8]) -> Option> { + let is_object = mach.filetype(endian) == object::macho::MH_OBJECT; + let mut dwarf = None; + let mut syms = Vec::new(); + let mut syms_sort_by_name = false; + let mut commands = mach.load_commands(endian, data, 0).ok()?; + let mut object_map = None; + let mut object_mappings = Vec::new(); + while let Ok(Some(command)) = commands.next() { + if let Some((segment, section_data)) = MachSegment::from_command(command).ok()? { + // Object files should have all sections in a single unnamed segment load command. + if segment.name() == b"__DWARF" || (is_object && segment.name() == b"") { + dwarf = segment.sections(endian, section_data).ok(); + } + } else if let Some(symtab) = command.symtab().ok()? { + let symbols = symtab.symbols::(endian, data).ok()?; + syms = symbols + .iter() + .filter_map(|nlist: &MachNlist| { + let name = nlist.name(endian, symbols.strings()).ok()?; + if name.len() > 0 && nlist.is_definition() { + Some((name, u64::from(nlist.n_value(endian)))) + } else { + None + } + }) + .collect(); + if is_object { + // We never search object file symbols by address. + // Instead, we already know the symbol name from the executable, and we + // need to search by name to find the matching symbol in the object file. + syms.sort_unstable_by_key(|(name, _)| *name); + syms_sort_by_name = true; + } else { + syms.sort_unstable_by_key(|(_, addr)| *addr); + let map = symbols.object_map(endian); + object_mappings.resize_with(map.objects().len(), || None); + object_map = Some(map); + } + } + } + + Some(Object { + endian, + data, + dwarf, + syms, + syms_sort_by_name, + object_map, + object_mappings: object_mappings.into_boxed_slice(), + }) + } + + pub fn section(&self, _: &Stash, name: &str) -> Option<&'a [u8]> { + let name = name.as_bytes(); + let dwarf = self.dwarf?; + let section = dwarf.into_iter().find(|section| { + let section_name = section.name(); + section_name == name || { + section_name.starts_with(b"__") + && name.starts_with(b".") + && §ion_name[2..] == &name[1..] + } + })?; + Some(section.data(self.endian, self.data).ok()?) + } + + pub fn search_symtab<'b>(&'b self, addr: u64) -> Option<&'b [u8]> { + debug_assert!(!self.syms_sort_by_name); + let i = match self.syms.binary_search_by_key(&addr, |(_, addr)| *addr) { + Ok(i) => i, + Err(i) => i.checked_sub(1)?, + }; + let (sym, _addr) = self.syms.get(i)?; + Some(sym) + } + + /// Try to load a context for an object file. + /// + /// If dsymutil was not run, then the DWARF may be found in the source object files. + pub(super) fn search_object_map<'b>(&'b mut self, addr: u64) -> Option<(&Context<'b>, u64)> { + // `object_map` contains a map from addresses to symbols and object paths. + // Look up the address and get a mapping for the object. + let object_map = self.object_map.as_ref()?; + let symbol = object_map.get(addr)?; + let object_index = symbol.object_index(); + let mapping = self.object_mappings.get_mut(object_index)?; + if mapping.is_none() { + // No cached mapping, so create it. + *mapping = Some(object_mapping(object_map.objects().get(object_index)?)); + } + let cx: &'b Context<'static> = &mapping.as_ref()?.as_ref()?.cx; + // Don't leak the `'static` lifetime, make sure it's scoped to just ourselves. + let cx = unsafe { core::mem::transmute::<&'b Context<'static>, &'b Context<'b>>(cx) }; + + // We must translate the address in order to be able to look it up + // in the DWARF in the object file. + debug_assert!(cx.object.syms.is_empty() || cx.object.syms_sort_by_name); + let i = cx + .object + .syms + .binary_search_by_key(&symbol.name(), |(name, _)| *name) + .ok()?; + let object_symbol = cx.object.syms.get(i)?; + let object_addr = addr + .wrapping_sub(symbol.address()) + .wrapping_add(object_symbol.1); + Some((cx, object_addr)) + } +} + +fn object_mapping(path: &[u8]) -> Option { + use super::mystd::ffi::OsStr; + use super::mystd::os::unix::prelude::*; + + let map; + + // `N_OSO` symbol names can be either `/path/to/object.o` or `/path/to/archive.a(object.o)`. + let member_name = if let Some((archive_path, member_name)) = split_archive_path(path) { + map = super::mmap(Path::new(OsStr::from_bytes(archive_path)))?; + Some(member_name) + } else { + map = super::mmap(Path::new(OsStr::from_bytes(path)))?; + None + }; + Mapping::mk(map, |data, stash| { + let data = match member_name { + Some(member_name) => { + let archive = object::read::archive::ArchiveFile::parse(data).ok()?; + let member = archive + .members() + .filter_map(Result::ok) + .find(|m| m.name() == member_name)?; + member.data(data).ok()? + } + None => data, + }; + let (macho, data) = find_header(data)?; + let endian = macho.endian().ok()?; + let obj = Object::parse(macho, endian, data)?; + Context::new(stash, obj, None) + }) +} + +fn split_archive_path(path: &[u8]) -> Option<(&[u8], &[u8])> { + let (last, path) = path.split_last()?; + if *last != b')' { + return None; + } + let index = path.iter().position(|&x| x == b'(')?; + let (archive, rest) = path.split_at(index); + Some((archive, &rest[1..])) +} diff --git a/crux-mir/lib/backtrace/src/symbolize/gimli/mmap_fake.rs b/crux-mir/lib/backtrace/src/symbolize/gimli/mmap_fake.rs new file mode 100644 index 000000000..ce5096415 --- /dev/null +++ b/crux-mir/lib/backtrace/src/symbolize/gimli/mmap_fake.rs @@ -0,0 +1,25 @@ +use super::{mystd::io::Read, File}; +use alloc::vec::Vec; +use core::ops::Deref; + +pub struct Mmap { + vec: Vec, +} + +impl Mmap { + pub unsafe fn map(mut file: &File, len: usize) -> Option { + let mut mmap = Mmap { + vec: Vec::with_capacity(len), + }; + file.read_to_end(&mut mmap.vec).ok()?; + Some(mmap) + } +} + +impl Deref for Mmap { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + &self.vec[..] + } +} diff --git a/crux-mir/lib/backtrace/src/symbolize/gimli/mmap_unix.rs b/crux-mir/lib/backtrace/src/symbolize/gimli/mmap_unix.rs new file mode 100644 index 000000000..5806c9f7e --- /dev/null +++ b/crux-mir/lib/backtrace/src/symbolize/gimli/mmap_unix.rs @@ -0,0 +1,44 @@ +use super::mystd::fs::File; +use super::mystd::os::unix::prelude::*; +use core::ops::Deref; +use core::ptr; +use core::slice; + +pub struct Mmap { + ptr: *mut libc::c_void, + len: usize, +} + +impl Mmap { + pub unsafe fn map(file: &File, len: usize) -> Option { + let ptr = libc::mmap( + ptr::null_mut(), + len, + libc::PROT_READ, + libc::MAP_PRIVATE, + file.as_raw_fd(), + 0, + ); + if ptr == libc::MAP_FAILED { + return None; + } + Some(Mmap { ptr, len }) + } +} + +impl Deref for Mmap { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + unsafe { slice::from_raw_parts(self.ptr as *const u8, self.len) } + } +} + +impl Drop for Mmap { + fn drop(&mut self) { + unsafe { + let r = libc::munmap(self.ptr, self.len); + debug_assert_eq!(r, 0); + } + } +} diff --git a/crux-mir/lib/backtrace/src/symbolize/gimli/mmap_windows.rs b/crux-mir/lib/backtrace/src/symbolize/gimli/mmap_windows.rs new file mode 100644 index 000000000..22f53fe03 --- /dev/null +++ b/crux-mir/lib/backtrace/src/symbolize/gimli/mmap_windows.rs @@ -0,0 +1,57 @@ +use super::super::super::windows::*; +use super::mystd::fs::File; +use super::mystd::os::windows::prelude::*; +use core::ops::Deref; +use core::ptr; +use core::slice; + +pub struct Mmap { + // keep the file alive to prevent it from ebeing deleted which would cause + // us to read bad data. + _file: File, + ptr: *mut c_void, + len: usize, +} + +impl Mmap { + pub unsafe fn map(file: &File, len: usize) -> Option { + let file = file.try_clone().ok()?; + let mapping = CreateFileMappingA( + file.as_raw_handle() as *mut _, + ptr::null_mut(), + PAGE_READONLY, + 0, + 0, + ptr::null(), + ); + if mapping.is_null() { + return None; + } + let ptr = MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, len); + CloseHandle(mapping); + if ptr.is_null() { + return None; + } + Some(Mmap { + _file: file, + ptr, + len, + }) + } +} +impl Deref for Mmap { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + unsafe { slice::from_raw_parts(self.ptr as *const u8, self.len) } + } +} + +impl Drop for Mmap { + fn drop(&mut self) { + unsafe { + let r = UnmapViewOfFile(self.ptr); + debug_assert!(r != 0); + } + } +} diff --git a/crux-mir/lib/backtrace/src/symbolize/gimli/stash.rs b/crux-mir/lib/backtrace/src/symbolize/gimli/stash.rs new file mode 100644 index 000000000..3adfc598a --- /dev/null +++ b/crux-mir/lib/backtrace/src/symbolize/gimli/stash.rs @@ -0,0 +1,52 @@ +// only used on Linux right now, so allow dead code elsewhere +#![cfg_attr(not(target_os = "linux"), allow(dead_code))] + +use super::Mmap; +use alloc::vec; +use alloc::vec::Vec; +use core::cell::UnsafeCell; + +/// A simple arena allocator for byte buffers. +pub struct Stash { + buffers: UnsafeCell>>, + mmap_aux: UnsafeCell>, +} + +impl Stash { + pub fn new() -> Stash { + Stash { + buffers: UnsafeCell::new(Vec::new()), + mmap_aux: UnsafeCell::new(None), + } + } + + /// Allocates a buffer of the specified size and returns a mutable reference + /// to it. + pub fn allocate(&self, size: usize) -> &mut [u8] { + // SAFETY: this is the only function that ever constructs a mutable + // reference to `self.buffers`. + let buffers = unsafe { &mut *self.buffers.get() }; + let i = buffers.len(); + buffers.push(vec![0; size]); + // SAFETY: we never remove elements from `self.buffers`, so a reference + // to the data inside any buffer will live as long as `self` does. + &mut buffers[i] + } + + /// Stores a `Mmap` for the lifetime of this `Stash`, returning a pointer + /// which is scoped to just this lifetime. + pub fn set_mmap_aux(&self, map: Mmap) -> &[u8] { + // SAFETY: this is the only location for a mutable pointer to + // `mmap_aux`, and this structure isn't threadsafe to shared across + // threads either. This also is careful to store at most one `mmap_aux` + // since overwriting a previous one would invalidate the previous + // pointer. Given that though we can safely return a pointer to our + // interior-owned contents. + unsafe { + let mmap_aux = &mut *self.mmap_aux.get(); + assert!(mmap_aux.is_none()); + *mmap_aux = Some(map); + mmap_aux.as_ref().unwrap() + } + } +} diff --git a/crux-mir/lib/backtrace/src/symbolize/libbacktrace.rs b/crux-mir/lib/backtrace/src/symbolize/libbacktrace.rs deleted file mode 100644 index 431878906..000000000 --- a/crux-mir/lib/backtrace/src/symbolize/libbacktrace.rs +++ /dev/null @@ -1,489 +0,0 @@ -// Copyright 2014-2015 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -//! Symbolication strategy using the DWARF-parsing code in libbacktrace. -//! -//! The libbacktrace C library, typically distributed with gcc, supports not -//! only generating a backtrace (which we don't actually use) but also -//! symbolicating the backtrace and handling dwarf debug information about -//! things like inlined frames and whatnot. -//! -//! This is relatively complicated due to lots of various concerns here, but the -//! basic idea is: -//! -//! * First we call `backtrace_syminfo`. This gets symbol information from the -//! dynamic symbol table if we can. -//! * Next we call `backtrace_pcinfo`. This will parse debuginfo tables if -//! they're available and allow us to recover information about inline frames, -//! filenames, line numbers, etc. -//! -//! There's lots of trickery about getting the dwarf tables into libbacktrace, -//! but hopefully it's not the end of the world and is clear enough when reading -//! below. -//! -//! This is the default symbolication strategy for non-MSVC and non-OSX -//! platforms. In libstd though this is the default strategy for OSX. - -#![allow(bad_style)] - -extern crate backtrace_sys as bt; - -use core::{ptr, slice}; -use libc::{self, c_char, c_int, c_void, uintptr_t}; - -use crate::symbolize::dladdr; -use crate::symbolize::{ResolveWhat, SymbolName}; -use crate::types::BytesOrWideString; - -pub enum Symbol<'a> { - Syminfo { - pc: uintptr_t, - symname: *const c_char, - }, - Pcinfo { - pc: uintptr_t, - filename: *const c_char, - lineno: c_int, - function: *const c_char, - symname: *const c_char, - }, - Dladdr(dladdr::Symbol<'a>), -} - -impl Symbol<'_> { - pub fn name(&self) -> Option { - let symbol = |ptr: *const c_char| unsafe { - if ptr.is_null() { - None - } else { - let len = libc::strlen(ptr); - Some(SymbolName::new(slice::from_raw_parts( - ptr as *const u8, - len, - ))) - } - }; - match *self { - Symbol::Syminfo { symname, .. } => symbol(symname), - Symbol::Pcinfo { - function, symname, .. - } => { - // If possible prefer the `function` name which comes from - // debuginfo and can typically be more accurate for inline - // frames for example. If that's not present though fall back to - // the symbol table name specified in `symname`. - // - // Note that sometimes `function` can feel somewhat less - // accurate, for example being listed as `try` - // isntead of `std::panicking::try::do_call`. It's not really - // clear why, but overall the `function` name seems more accurate. - if let Some(sym) = symbol(function) { - return Some(sym); - } - symbol(symname) - } - Symbol::Dladdr(ref s) => s.name(), - } - } - - pub fn addr(&self) -> Option<*mut c_void> { - let pc = match *self { - Symbol::Syminfo { pc, .. } => pc, - Symbol::Pcinfo { pc, .. } => pc, - Symbol::Dladdr(ref s) => return s.addr(), - }; - if pc == 0 { - None - } else { - Some(pc as *mut _) - } - } - - fn filename_bytes(&self) -> Option<&[u8]> { - match *self { - Symbol::Syminfo { .. } => None, - Symbol::Pcinfo { filename, .. } => { - let ptr = filename as *const u8; - unsafe { - let len = libc::strlen(filename); - Some(slice::from_raw_parts(ptr, len)) - } - } - Symbol::Dladdr(_) => None, - } - } - - pub fn filename_raw(&self) -> Option { - self.filename_bytes().map(BytesOrWideString::Bytes) - } - - #[cfg(feature = "std")] - pub fn filename(&self) -> Option<&::std::path::Path> { - use std::path::Path; - - #[cfg(unix)] - fn bytes2path(bytes: &[u8]) -> Option<&Path> { - use std::ffi::OsStr; - use std::os::unix::prelude::*; - Some(Path::new(OsStr::from_bytes(bytes))) - } - - #[cfg(windows)] - fn bytes2path(bytes: &[u8]) -> Option<&Path> { - use std::str; - str::from_utf8(bytes).ok().map(Path::new) - } - - self.filename_bytes().and_then(bytes2path) - } - - pub fn lineno(&self) -> Option { - match *self { - Symbol::Syminfo { .. } => None, - Symbol::Pcinfo { lineno, .. } => Some(lineno as u32), - Symbol::Dladdr(ref s) => s.lineno(), - } - } -} - -extern "C" fn error_cb(_data: *mut c_void, _msg: *const c_char, _errnum: c_int) { - // do nothing for now -} - -/// Type of the `data` pointer passed into `syminfo_cb` -struct SyminfoState<'a> { - cb: &'a mut (FnMut(&super::Symbol) + 'a), - pc: usize, -} - -extern "C" fn syminfo_cb( - data: *mut c_void, - pc: uintptr_t, - symname: *const c_char, - _symval: uintptr_t, - _symsize: uintptr_t, -) { - let mut bomb = crate::Bomb { enabled: true }; - - // Once this callback is invoked from `backtrace_syminfo` when we start - // resolving we go further to call `backtrace_pcinfo`. The - // `backtrace_pcinfo` function will consult debug information and attemp tto - // do things like recover file/line information as well as inlined frames. - // Note though that `backtrace_pcinfo` can fail or not do much if there's - // not debug info, so if that happens we're sure to call the callback with - // at least one symbol from the `syminfo_cb`. - unsafe { - let syminfo_state = &mut *(data as *mut SyminfoState); - let mut pcinfo_state = PcinfoState { - symname, - called: false, - cb: syminfo_state.cb, - }; - bt::backtrace_pcinfo( - init_state(), - syminfo_state.pc as uintptr_t, - pcinfo_cb, - error_cb, - &mut pcinfo_state as *mut _ as *mut _, - ); - if !pcinfo_state.called { - (pcinfo_state.cb)(&super::Symbol { - inner: Symbol::Syminfo { - pc: pc, - symname: symname, - }, - }); - } - } - - bomb.enabled = false; -} - -/// Type of the `data` pointer passed into `pcinfo_cb` -struct PcinfoState<'a> { - cb: &'a mut (FnMut(&super::Symbol) + 'a), - symname: *const c_char, - called: bool, -} - -extern "C" fn pcinfo_cb( - data: *mut c_void, - pc: uintptr_t, - filename: *const c_char, - lineno: c_int, - function: *const c_char, -) -> c_int { - if filename.is_null() || function.is_null() { - return -1; - } - let mut bomb = crate::Bomb { enabled: true }; - - unsafe { - let state = &mut *(data as *mut PcinfoState); - state.called = true; - (state.cb)(&super::Symbol { - inner: Symbol::Pcinfo { - pc: pc, - filename: filename, - lineno: lineno, - symname: state.symname, - function, - }, - }); - } - - bomb.enabled = false; - return 0; -} - -// The libbacktrace API supports creating a state, but it does not -// support destroying a state. I personally take this to mean that a -// state is meant to be created and then live forever. -// -// I would love to register an at_exit() handler which cleans up this -// state, but libbacktrace provides no way to do so. -// -// With these constraints, this function has a statically cached state -// that is calculated the first time this is requested. Remember that -// backtracing all happens serially (one global lock). -// -// Note the lack of synchronization here is due to the requirement that -// `resolve` is externally synchronized. -unsafe fn init_state() -> *mut bt::backtrace_state { - static mut STATE: *mut bt::backtrace_state = 0 as *mut _; - - if !STATE.is_null() { - return STATE; - } - - STATE = bt::backtrace_create_state( - load_filename(), - // Don't exercise threadsafe capabilities of libbacktrace since - // we're always calling it in a synchronized fashion. - 0, - error_cb, - ptr::null_mut(), // no extra data - ); - - return STATE; - - // Note that for libbacktrace to operate at all it needs to find the DWARF - // debug info for the current executable. It typically does that via a - // number of mechanisms including, but not limited to: - // - // * /proc/self/exe on supported platforms - // * The filename passed in explicitly when creating state - // - // The libbacktrace library is a big wad of C code. This naturally means - // it's got memory safety vulnerabilities, especially when handling - // malformed debuginfo. Libstd has run into plenty of these historically. - // - // If /proc/self/exe is used then we can typically ignore these as we - // assume that libbacktrace is "mostly correct" and otherwise doesn't do - // weird things with "attempted to be correct" dwarf debug info. - // - // If we pass in a filename, however, then it's possible on some platforms - // (like BSDs) where a malicious actor can cause an arbitrary file to be - // placed at that location. This means that if we tell libbacktrace about a - // filename it may be using an arbitrary file, possibly causing segfaults. - // If we don't tell libbacktrace anything though then it won't do anything - // on platforms that don't support paths like /proc/self/exe! - // - // Given all that we try as hard as possible to *not* pass in a filename, - // but we must on platforms that don't support /proc/self/exe at all. - cfg_if::cfg_if! { - if #[cfg(any(target_os = "macos", target_os = "ios"))] { - // Note that ideally we'd use `std::env::current_exe`, but we can't - // require `std` here. - // - // Use `_NSGetExecutablePath` to load the current executable path - // into a static area (which if it's too small just give up). - // - // Note that we're seriously trusting libbacktrace here to not die - // on corrupt executables, but it surely does... - unsafe fn load_filename() -> *const libc::c_char { - const N: usize = 256; - static mut BUF: [u8; N] = [0; N]; - extern { - fn _NSGetExecutablePath( - buf: *mut libc::c_char, - bufsize: *mut u32, - ) -> libc::c_int; - } - let mut sz: u32 = BUF.len() as u32; - let ptr = BUF.as_mut_ptr() as *mut libc::c_char; - if _NSGetExecutablePath(ptr, &mut sz) == 0 { - ptr - } else { - ptr::null() - } - } - } else if #[cfg(windows)] { - use crate::windows::*; - - // Windows has a mode of opening files where after it's opened it - // can't be deleted. That's in general what we want here because we - // want to ensure that our executable isn't changing out from under - // us after we hand it off to libbacktrace, hopefully mitigating the - // ability to pass in arbitrary data into libbacktrace (which may be - // mishandled). - // - // Given that we do a bit of a dance here to attempt to get a sort - // of lock on our own image: - // - // * Get a handle to the current process, load its filename. - // * Open a file to that filename with the right flags. - // * Reload the current process's filename, making sure it's the same - // - // If that all passes we in theory have indeed opened our process's - // file and we're guaranteed it won't change. FWIW a bunch of this - // is copied from libstd historically, so this is my best - // interpretation of what was happening. - unsafe fn load_filename() -> *const libc::c_char { - load_filename_opt().unwrap_or(ptr::null()) - } - - unsafe fn load_filename_opt() -> Result<*const libc::c_char, ()> { - const N: usize = 256; - // This lives in static memory so we can return it.. - static mut BUF: [i8; N] = [0; N]; - // ... and this lives on the stack since it's temporary - let mut stack_buf = [0; N]; - let name1 = query_full_name(&mut BUF)?; - - let handle = CreateFileA( - name1.as_ptr(), - GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE, - ptr::null_mut(), - OPEN_EXISTING, - 0, - ptr::null_mut(), - ); - if handle.is_null() { - return Err(()); - } - - let name2 = query_full_name(&mut stack_buf)?; - if name1 != name2 { - CloseHandle(handle); - return Err(()) - } - // intentionally leak `handle` here because having that open - // should preserve our lock on this file name. - Ok(name1.as_ptr()) - } - - unsafe fn query_full_name(buf: &mut [i8]) -> Result<&[i8], ()> { - let dll = GetModuleHandleA(b"kernel32.dll\0".as_ptr() as *const i8); - if dll.is_null() { - return Err(()) - } - let ptrQueryFullProcessImageNameA = - GetProcAddress(dll, b"QueryFullProcessImageNameA\0".as_ptr() as *const _) as usize; - if ptrQueryFullProcessImageNameA == 0 - { - return Err(()); - } - use core::mem; - let p1 = OpenProcess(PROCESS_QUERY_INFORMATION, FALSE, GetCurrentProcessId()); - let mut len = buf.len() as u32; - let pfnQueryFullProcessImageNameA : extern "system" fn( - hProcess: HANDLE, - dwFlags: DWORD, - lpExeName: LPSTR, - lpdwSize: PDWORD, - ) -> BOOL = mem::transmute(ptrQueryFullProcessImageNameA); - - let rc = pfnQueryFullProcessImageNameA(p1, 0, buf.as_mut_ptr(), &mut len); - CloseHandle(p1); - - // We want to return a slice that is nul-terminated, so if - // everything was filled in and it equals the total length - // then equate that to failure. - // - // Otherwise when returning success make sure the nul byte is - // included in the slice. - if rc == 0 || len == buf.len() as u32 { - Err(()) - } else { - assert_eq!(buf[len as usize], 0); - Ok(&buf[..(len + 1) as usize]) - } - } - } else if #[cfg(target_os = "vxworks")] { - unsafe fn load_filename() -> *const libc::c_char { - use libc; - use core::mem; - - const N: usize = libc::VX_RTP_NAME_LENGTH as usize + 1; - static mut BUF: [libc::c_char; N] = [0; N]; - - let mut rtp_desc : libc::RTP_DESC = mem::zeroed(); - if (libc::rtpInfoGet(0, &mut rtp_desc as *mut libc::RTP_DESC) == 0) { - BUF.copy_from_slice(&rtp_desc.pathName); - BUF.as_ptr() - } else { - ptr::null() - } - } - } else { - unsafe fn load_filename() -> *const libc::c_char { - ptr::null() - } - } - } -} - -pub unsafe fn resolve(what: ResolveWhat, cb: &mut FnMut(&super::Symbol)) { - let symaddr = what.address_or_ip() as usize; - - // backtrace errors are currently swept under the rug - let state = init_state(); - if state.is_null() { - return dladdr_fallback(what.address_or_ip(), cb); - } - - // Call the `backtrace_syminfo` API first. This is (from reading the code) - // guaranteed to call `syminfo_cb` exactly once (or fail with an error - // presumably). We then handle more within the `syminfo_cb`. - // - // Note that we do this since `syminfo` will consult the symbol table, - // finding symbol names even if there's no debug information in the binary. - let mut called = false; - { - let mut syminfo_state = SyminfoState { - pc: symaddr, - cb: &mut |sym| { - called = true; - cb(sym); - }, - }; - bt::backtrace_syminfo( - state, - symaddr as uintptr_t, - syminfo_cb, - error_cb, - &mut syminfo_state as *mut _ as *mut _, - ); - } - - if !called { - dladdr_fallback(what.address_or_ip(), cb); - } -} - -unsafe fn dladdr_fallback(addr: *mut c_void, cb: &mut FnMut(&super::Symbol)) { - dladdr::resolve(addr, &mut |sym| { - cb(&super::Symbol { - inner: Symbol::Dladdr(sym), - }) - }); -} diff --git a/crux-mir/lib/backtrace/src/symbolize/miri.rs b/crux-mir/lib/backtrace/src/symbolize/miri.rs new file mode 100644 index 000000000..5b0dc3084 --- /dev/null +++ b/crux-mir/lib/backtrace/src/symbolize/miri.rs @@ -0,0 +1,56 @@ +use core::ffi::c_void; +use core::marker::PhantomData; + +use super::super::backtrace::miri::{resolve_addr, Frame}; +use super::BytesOrWideString; +use super::{ResolveWhat, SymbolName}; + +pub unsafe fn resolve(what: ResolveWhat<'_>, cb: &mut dyn FnMut(&super::Symbol)) { + let sym = match what { + ResolveWhat::Address(addr) => Symbol { + inner: resolve_addr(addr), + _unused: PhantomData, + }, + ResolveWhat::Frame(frame) => Symbol { + inner: frame.inner.clone(), + _unused: PhantomData, + }, + }; + cb(&super::Symbol { inner: sym }) +} + +pub struct Symbol<'a> { + inner: Frame, + _unused: PhantomData<&'a ()>, +} + +impl<'a> Symbol<'a> { + pub fn name(&self) -> Option> { + Some(SymbolName::new(&self.inner.inner.name)) + } + + pub fn addr(&self) -> Option<*mut c_void> { + Some(self.inner.addr) + } + + pub fn filename_raw(&self) -> Option> { + Some(BytesOrWideString::Bytes(&self.inner.inner.filename)) + } + + pub fn lineno(&self) -> Option { + Some(self.inner.inner.lineno) + } + + pub fn colno(&self) -> Option { + Some(self.inner.inner.colno) + } + + #[cfg(feature = "std")] + pub fn filename(&self) -> Option<&std::path::Path> { + Some(std::path::Path::new( + core::str::from_utf8(&self.inner.inner.filename).unwrap(), + )) + } +} + +pub unsafe fn clear_symbol_cache() {} diff --git a/crux-mir/lib/backtrace/src/symbolize/mod.rs b/crux-mir/lib/backtrace/src/symbolize/mod.rs index 36d58769b..dbc346522 100644 --- a/crux-mir/lib/backtrace/src/symbolize/mod.rs +++ b/crux-mir/lib/backtrace/src/symbolize/mod.rs @@ -7,8 +7,8 @@ cfg_if::cfg_if! { } } -use crate::backtrace::Frame; -use crate::types::BytesOrWideString; +use super::backtrace::Frame; +use super::types::BytesOrWideString; use core::ffi::c_void; use rustc_demangle::{try_demangle, Demangle}; @@ -66,7 +66,7 @@ pub fn resolve(addr: *mut c_void, cb: F) { /// Resolve a previously capture frame to a symbol, passing the symbol to the /// specified closure. /// -/// This functin performs the same function as `resolve` except that it takes a +/// This function performs the same function as `resolve` except that it takes a /// `Frame` as an argument instead of an address. This can allow some platform /// implementations of backtracing to provide more accurate symbol information /// or information about inline frames for example. It's recommended to use this @@ -148,7 +148,7 @@ fn adjust_ip(a: *mut c_void) -> *mut c_void { /// Same as `resolve`, only unsafe as it's unsynchronized. /// -/// This function does not have synchronization guarentees but is available when +/// This function does not have synchronization guarantees but is available when /// the `std` feature of this crate isn't compiled in. See the `resolve` /// function for more documentation and examples. /// @@ -159,12 +159,12 @@ pub unsafe fn resolve_unsynchronized(addr: *mut c_void, mut cb: F) where F: FnMut(&Symbol), { - resolve_imp(ResolveWhat::Address(addr), &mut cb) + imp::resolve(ResolveWhat::Address(addr), &mut cb) } /// Same as `resolve_frame`, only unsafe as it's unsynchronized. /// -/// This function does not have synchronization guarentees but is available +/// This function does not have synchronization guarantees but is available /// when the `std` feature of this crate isn't compiled in. See the /// `resolve_frame` function for more documentation and examples. /// @@ -175,7 +175,7 @@ pub unsafe fn resolve_frame_unsynchronized(frame: &Frame, mut cb: F) where F: FnMut(&Symbol), { - resolve_imp(ResolveWhat::Frame(frame), &mut cb) + imp::resolve(ResolveWhat::Frame(frame), &mut cb) } /// A trait representing the resolution of a symbol in a file. @@ -191,7 +191,7 @@ pub struct Symbol { // TODO: this lifetime bound needs to be persisted eventually to `Symbol`, // but that's currently a breaking change. For now this is safe since // `Symbol` is only ever handed out by reference and can't be cloned. - inner: SymbolImp<'static>, + inner: imp::Symbol<'static>, } impl Symbol { @@ -204,7 +204,7 @@ impl Symbol { /// * The raw `str` value of the symbol can be accessed (if it's valid /// utf-8). /// * The raw bytes for the symbol name can be accessed. - pub fn name(&self) -> Option { + pub fn name(&self) -> Option> { self.inner.name() } @@ -215,10 +215,18 @@ impl Symbol { /// Returns the raw filename as a slice. This is mainly useful for `no_std` /// environments. - pub fn filename_raw(&self) -> Option { + pub fn filename_raw(&self) -> Option> { self.inner.filename_raw() } + /// Returns the column number for where this symbol is currently executing. + /// + /// Only gimli currently provides a value here and even then only if `filename` + /// returns `Some`, and so it is then consequently subject to similar caveats. + pub fn colno(&self) -> Option { + self.inner.colno() + } + /// Returns the line number for where this symbol is currently executing. /// /// This return value is typically `Some` if `filename` returns `Some`, and @@ -229,8 +237,8 @@ impl Symbol { /// Returns the file name where this function was defined. /// - /// This is currently only available when libbacktrace is being used (e.g. - /// unix platforms other than OSX) and when a binary is compiled with + /// This is currently only available when libbacktrace or gimli is being + /// used (e.g. unix platforms other) and when a binary is compiled with /// debuginfo. If neither of these conditions is met then this will likely /// return `None`. /// @@ -246,7 +254,7 @@ impl Symbol { } impl fmt::Debug for Symbol { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let mut d = f.debug_struct("Symbol"); if let Some(name) = self.name() { d.field("name", &name); @@ -349,9 +357,9 @@ impl<'a> SymbolName<'a> { } fn format_symbol_name( - fmt: fn(&str, &mut fmt::Formatter) -> fmt::Result, + fmt: fn(&str, &mut fmt::Formatter<'_>) -> fmt::Result, mut bytes: &[u8], - f: &mut fmt::Formatter, + f: &mut fmt::Formatter<'_>, ) -> fmt::Result { while bytes.len() > 0 { match str::from_utf8(bytes) { @@ -375,7 +383,7 @@ fn format_symbol_name( cfg_if::cfg_if! { if #[cfg(feature = "cpp_demangle")] { impl<'a> fmt::Display for SymbolName<'a> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if let Some(ref s) = self.demangled { s.fmt(f) } else if let Some(ref cpp) = self.cpp_demangled.0 { @@ -387,7 +395,7 @@ cfg_if::cfg_if! { } } else { impl<'a> fmt::Display for SymbolName<'a> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if let Some(ref s) = self.demangled { s.fmt(f) } else { @@ -401,7 +409,7 @@ cfg_if::cfg_if! { cfg_if::cfg_if! { if #[cfg(all(feature = "std", feature = "cpp_demangle"))] { impl<'a> fmt::Debug for SymbolName<'a> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { use std::fmt::Write; if let Some(ref s) = self.demangled { @@ -423,7 +431,7 @@ cfg_if::cfg_if! { } } else { impl<'a> fmt::Debug for SymbolName<'a> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if let Some(ref s) = self.demangled { s.fmt(f) } else { @@ -451,64 +459,27 @@ cfg_if::cfg_if! { pub fn clear_symbol_cache() { let _guard = crate::lock::lock(); unsafe { - clear_symbol_cache_imp(); + imp::clear_symbol_cache(); } } -mod dladdr; - cfg_if::cfg_if! { - if #[cfg(all(windows, target_env = "msvc", feature = "dbghelp", not(target_vendor = "uwp")))] { + if #[cfg(miri)] { + mod miri; + use miri as imp; + } else if #[cfg(all(windows, target_env = "msvc", not(target_vendor = "uwp")))] { mod dbghelp; - use self::dbghelp::resolve as resolve_imp; - use self::dbghelp::Symbol as SymbolImp; - unsafe fn clear_symbol_cache_imp() {} + use dbghelp as imp; } else if #[cfg(all( - feature = "std", - feature = "gimli-symbolize", - any( - target_os = "linux", - target_os = "macos", - windows, - ), + any(unix, windows), + not(target_vendor = "uwp"), + not(target_os = "emscripten"), + any(not(backtrace_in_libstd), feature = "backtrace"), ))] { mod gimli; - use self::gimli::resolve as resolve_imp; - use self::gimli::Symbol as SymbolImp; - use self::gimli::clear_symbol_cache as clear_symbol_cache_imp; - // Note that we only enable coresymbolication on iOS when debug assertions - // are enabled because it's helpful in debug mode but it looks like apps get - // rejected from the app store if they use this API, see #92 for more info - } else if #[cfg(all(feature = "coresymbolication", - any(target_os = "macos", - all(target_os = "ios", debug_assertions))))] { - mod coresymbolication; - use self::coresymbolication::resolve as resolve_imp; - use self::coresymbolication::Symbol as SymbolImp; - unsafe fn clear_symbol_cache_imp() {} - } else if #[cfg(all(feature = "libbacktrace", - any(unix, all(windows, not(target_vendor = "uwp"), target_env = "gnu")), - not(target_os = "fuchsia"), - not(target_os = "emscripten"), - not(target_env = "uclibc")))] { - mod libbacktrace; - use self::libbacktrace::resolve as resolve_imp; - use self::libbacktrace::Symbol as SymbolImp; - unsafe fn clear_symbol_cache_imp() {} - } else if #[cfg(all(unix, - not(target_os = "emscripten"), - not(target_os = "fuchsia"), - not(target_env = "uclibc"), - feature = "dladdr"))] { - mod dladdr_resolve; - use self::dladdr_resolve::resolve as resolve_imp; - use self::dladdr_resolve::Symbol as SymbolImp; - unsafe fn clear_symbol_cache_imp() {} + use gimli as imp; } else { mod noop; - use self::noop::resolve as resolve_imp; - use self::noop::Symbol as SymbolImp; - #[allow(unused)] - unsafe fn clear_symbol_cache_imp() {} + use noop as imp; } } diff --git a/crux-mir/lib/backtrace/src/symbolize/noop.rs b/crux-mir/lib/backtrace/src/symbolize/noop.rs index 5c115d5d8..c53336531 100644 --- a/crux-mir/lib/backtrace/src/symbolize/noop.rs +++ b/crux-mir/lib/backtrace/src/symbolize/noop.rs @@ -1,20 +1,18 @@ //! Empty symbolication strategy used to compile for platforms that have no //! support. -use crate::symbolize::ResolveWhat; -use crate::types::BytesOrWideString; -use crate::SymbolName; +use super::{BytesOrWideString, ResolveWhat, SymbolName}; use core::ffi::c_void; use core::marker; -pub unsafe fn resolve(_addr: ResolveWhat, _cb: &mut FnMut(&super::Symbol)) {} +pub unsafe fn resolve(_addr: ResolveWhat<'_>, _cb: &mut dyn FnMut(&super::Symbol)) {} pub struct Symbol<'a> { _marker: marker::PhantomData<&'a i32>, } impl Symbol<'_> { - pub fn name(&self) -> Option { + pub fn name(&self) -> Option> { None } @@ -22,7 +20,7 @@ impl Symbol<'_> { None } - pub fn filename_raw(&self) -> Option { + pub fn filename_raw(&self) -> Option> { None } @@ -34,4 +32,10 @@ impl Symbol<'_> { pub fn lineno(&self) -> Option { None } + + pub fn colno(&self) -> Option { + None + } } + +pub unsafe fn clear_symbol_cache() {} diff --git a/crux-mir/lib/backtrace/src/types.rs b/crux-mir/lib/backtrace/src/types.rs index f7bcb17da..c7e551059 100644 --- a/crux-mir/lib/backtrace/src/types.rs +++ b/crux-mir/lib/backtrace/src/types.rs @@ -77,7 +77,7 @@ impl<'a> BytesOrWideString<'a> { #[cfg(feature = "std")] impl<'a> fmt::Display for BytesOrWideString<'a> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.to_str_lossy().fmt(f) } } diff --git a/crux-mir/lib/backtrace/src/windows.rs b/crux-mir/lib/backtrace/src/windows.rs index 9672af0d2..d091874f1 100644 --- a/crux-mir/lib/backtrace/src/windows.rs +++ b/crux-mir/lib/backtrace/src/windows.rs @@ -25,14 +25,16 @@ cfg_if::cfg_if! { pub use winapi::shared::basetsd::*; pub use winapi::shared::minwindef::*; pub use winapi::um::dbghelp::*; + pub use winapi::um::fileapi::*; pub use winapi::um::handleapi::*; pub use winapi::um::libloaderapi::*; + pub use winapi::um::memoryapi::*; + pub use winapi::um::minwinbase::*; pub use winapi::um::processthreadsapi::*; + pub use winapi::um::synchapi::*; + pub use winapi::um::tlhelp32::*; pub use winapi::um::winbase::*; pub use winapi::um::winnt::*; - pub use winapi::um::fileapi::*; - pub use winapi::um::minwinbase::*; - pub use winapi::um::synchapi::*; } } else { pub use core::ffi::c_void; @@ -310,6 +312,20 @@ ffi! { pub Reserved1: [DWORD64; 4], } + #[repr(C)] + pub struct MODULEENTRY32W { + pub dwSize: DWORD, + pub th32ModuleID: DWORD, + pub th32ProcessID: DWORD, + pub GlblcntUsage: DWORD, + pub ProccntUsage: DWORD, + pub modBaseAddr: *mut u8, + pub modBaseSize: DWORD, + pub hModule: HMODULE, + pub szModule: [WCHAR; MAX_MODULE_NAME32 + 1], + pub szExePath: [WCHAR; MAX_PATH], + } + pub const MAX_SYM_NAME: usize = 2000; pub const AddrModeFlat: ADDRESS_MODE = 3; pub const TRUE: BOOL = 1; @@ -324,6 +340,12 @@ ffi! { pub const OPEN_EXISTING: DWORD = 0x3; pub const GENERIC_READ: DWORD = 0x80000000; pub const INFINITE: DWORD = !0; + pub const PAGE_READONLY: DWORD = 2; + pub const FILE_MAP_READ: DWORD = 4; + pub const TH32CS_SNAPMODULE: DWORD = 0x00000008; + pub const INVALID_HANDLE_VALUE: HANDLE = -1isize as HANDLE; + pub const MAX_MODULE_NAME32: usize = 255; + pub const MAX_PATH: usize = 260; pub type DWORD = u32; pub type PDWORD = *mut u32; @@ -344,6 +366,10 @@ ffi! { pub type LPDWORD = *mut DWORD; pub type DWORDLONG = u64; pub type HMODULE = HINSTANCE; + pub type SIZE_T = usize; + pub type LPVOID = *mut c_void; + pub type LPCVOID = *const c_void; + pub type LPMODULEENTRY32W = *mut MODULEENTRY32W; extern "system" { pub fn GetCurrentProcess() -> HANDLE; @@ -379,6 +405,34 @@ ffi! { dwMilliseconds: DWORD, bAlertable: BOOL, ) -> DWORD; + pub fn CreateFileMappingA( + hFile: HANDLE, + lpFileMappingAttributes: LPSECURITY_ATTRIBUTES, + flProtect: DWORD, + dwMaximumSizeHigh: DWORD, + dwMaximumSizeLow: DWORD, + lpName: LPCSTR, + ) -> HANDLE; + pub fn MapViewOfFile( + hFileMappingObject: HANDLE, + dwDesiredAccess: DWORD, + dwFileOffsetHigh: DWORD, + dwFileOffsetLow: DWORD, + dwNumberOfBytesToMap: SIZE_T, + ) -> LPVOID; + pub fn UnmapViewOfFile(lpBaseAddress: LPCVOID) -> BOOL; + pub fn CreateToolhelp32Snapshot( + dwFlags: DWORD, + th32ProcessID: DWORD, + ) -> HANDLE; + pub fn Module32FirstW( + hSnapshot: HANDLE, + lpme: LPMODULEENTRY32W, + ) -> BOOL; + pub fn Module32NextW( + hSnapshot: HANDLE, + lpme: LPMODULEENTRY32W, + ) -> BOOL; } } diff --git a/crux-mir/lib/backtrace/tests/accuracy/auxiliary.rs b/crux-mir/lib/backtrace/tests/accuracy/auxiliary.rs index 7d0457083..9c8015d9a 100644 --- a/crux-mir/lib/backtrace/tests/accuracy/auxiliary.rs +++ b/crux-mir/lib/backtrace/tests/accuracy/auxiliary.rs @@ -7,7 +7,6 @@ where } #[inline(always)] -#[cfg_attr(feature = "coresymbolication", inline(never))] pub fn callback_inlined(f: F) where F: FnOnce((&'static str, u32)), diff --git a/crux-mir/lib/backtrace/tests/accuracy/main.rs b/crux-mir/lib/backtrace/tests/accuracy/main.rs index 7c484a24e..149203a1b 100644 --- a/crux-mir/lib/backtrace/tests/accuracy/main.rs +++ b/crux-mir/lib/backtrace/tests/accuracy/main.rs @@ -16,6 +16,33 @@ type Pos = (&'static str, u32); #[test] fn doit() { + if + // Skip musl which is by default statically linked and doesn't support + // dynamic libraries. + !cfg!(target_env = "musl") + // Skip Miri, since it doesn't support dynamic libraries. + && !cfg!(miri) + { + // TODO(#238) this shouldn't have to happen first in this function, but + // currently it does. + let mut dir = std::env::current_exe().unwrap(); + dir.pop(); + if cfg!(windows) { + dir.push("dylib_dep.dll"); + } else if cfg!(target_os = "macos") { + dir.push("libdylib_dep.dylib"); + } else { + dir.push("libdylib_dep.so"); + } + unsafe { + let lib = libloading::Library::new(&dir).unwrap(); + let api = lib.get::(b"foo").unwrap(); + api(pos!(), |a, b| { + check!(a, b); + }); + } + } + outer(pos!()); } @@ -39,14 +66,12 @@ fn inner(main_pos: Pos, outer_pos: Pos) { } #[inline(always)] -#[cfg_attr(feature = "coresymbolication", inline(never))] #[rustfmt::skip] fn inner_inlined(main_pos: Pos, outer_pos: Pos) { check!(main_pos, outer_pos); check!(main_pos, outer_pos); #[inline(always)] - #[cfg_attr(feature = "coresymbolication", inline(never))] fn inner_further_inlined(main_pos: Pos, outer_pos: Pos, inner_pos: Pos) { check!(main_pos, outer_pos, inner_pos); } diff --git a/crux-mir/lib/backtrace/tests/concurrent-panics.rs b/crux-mir/lib/backtrace/tests/concurrent-panics.rs index ac1d61d65..470245cc9 100644 --- a/crux-mir/lib/backtrace/tests/concurrent-panics.rs +++ b/crux-mir/lib/backtrace/tests/concurrent-panics.rs @@ -13,7 +13,13 @@ fn main() { // These run in docker containers on CI where they can't re-exec the test, // so just skip these for CI. No other reason this can't run on those // platforms though. - if cfg!(unix) && (cfg!(target_arch = "arm") || cfg!(target_arch = "aarch64")) { + // Miri does not have support for re-execing a file + if cfg!(unix) + && (cfg!(target_arch = "arm") + || cfg!(target_arch = "aarch64") + || cfg!(target_arch = "s390x")) + || cfg!(miri) + { println!("test result: ok"); return; } diff --git a/crux-mir/lib/backtrace/tests/long_fn_name.rs b/crux-mir/lib/backtrace/tests/long_fn_name.rs index a56573482..fa4cfda15 100644 --- a/crux-mir/lib/backtrace/tests/long_fn_name.rs +++ b/crux-mir/lib/backtrace/tests/long_fn_name.rs @@ -1,5 +1,3 @@ -extern crate backtrace; - use backtrace::Backtrace; // 50-character module name @@ -18,7 +16,7 @@ mod _234567890_234567890_234567890_234567890_234567890 { // Long function names must be truncated to (MAX_SYM_NAME - 1) characters. // Only run this test for msvc, since gnu prints "" for all frames. #[test] -#[cfg(all(windows, feature = "dbghelp", target_env = "msvc"))] +#[cfg(all(windows, target_env = "msvc"))] fn test_long_fn_name() { use _234567890_234567890_234567890_234567890_234567890::_234567890_234567890_234567890_234567890_234567890 as S; diff --git a/crux-mir/lib/backtrace/tests/skip_inner_frames.rs b/crux-mir/lib/backtrace/tests/skip_inner_frames.rs index f5043ee6c..8b57bef52 100644 --- a/crux-mir/lib/backtrace/tests/skip_inner_frames.rs +++ b/crux-mir/lib/backtrace/tests/skip_inner_frames.rs @@ -1,5 +1,3 @@ -extern crate backtrace; - use backtrace::Backtrace; // This test only works on platforms which have a working `symbol_address` @@ -9,9 +7,6 @@ const ENABLED: bool = cfg!(all( // Windows hasn't really been tested, and OSX doesn't support actually // finding an enclosing frame, so disable this target_os = "linux", - // This is the only method currently that supports accurate enough - // backtraces for this test to work. - feature = "libunwind", // On ARM finding the enclosing function is simply returning the ip itself. not(target_arch = "arm"), )); diff --git a/crux-mir/lib/backtrace/tests/smoke.rs b/crux-mir/lib/backtrace/tests/smoke.rs index 7f90d2947..683a6f0db 100644 --- a/crux-mir/lib/backtrace/tests/smoke.rs +++ b/crux-mir/lib/backtrace/tests/smoke.rs @@ -1,20 +1,6 @@ -extern crate backtrace; - use backtrace::Frame; use std::thread; -static LIBUNWIND: bool = cfg!(all(unix, feature = "libunwind")); -static UNIX_BACKTRACE: bool = cfg!(all(unix, feature = "unix-backtrace")); -static LIBBACKTRACE: bool = cfg!(feature = "libbacktrace") && !cfg!(target_os = "fuchsia"); -static CORESYMBOLICATION: bool = cfg!(all( - any(target_os = "macos", target_os = "ios"), - feature = "coresymbolication" -)); -static DLADDR: bool = cfg!(all(unix, feature = "dladdr")) && !cfg!(target_os = "fuchsia"); -static DBGHELP: bool = cfg!(all(windows, feature = "dbghelp")); -static MSVC: bool = cfg!(target_env = "msvc"); -static GIMLI_SYMBOLIZE: bool = cfg!(all(feature = "gimli-symbolize", unix, target_os = "linux")); - #[test] // FIXME: shouldn't ignore this test on i686-msvc, unsure why it's failing #[cfg_attr(all(target_arch = "x86", target_env = "msvc"), ignore)] @@ -31,13 +17,6 @@ fn smoke_test_frames() { true }); - if v.len() < 5 { - assert!(!LIBUNWIND); - assert!(!UNIX_BACKTRACE); - assert!(!DBGHELP); - return; - } - // Various platforms have various bits of weirdness about their // backtraces. To find a good starting spot let's search through the // frames @@ -64,6 +43,7 @@ fn smoke_test_frames() { "frame_4", "tests/smoke.rs", start_line + 6, + 9, ); assert_frame( frames.next().unwrap(), @@ -71,6 +51,7 @@ fn smoke_test_frames() { "frame_3", "tests/smoke.rs", start_line + 3, + 52, ); assert_frame( frames.next().unwrap(), @@ -78,6 +59,7 @@ fn smoke_test_frames() { "frame_2", "tests/smoke.rs", start_line + 2, + 52, ); assert_frame( frames.next().unwrap(), @@ -85,6 +67,7 @@ fn smoke_test_frames() { "frame_1", "tests/smoke.rs", start_line + 1, + 52, ); assert_frame( frames.next().unwrap(), @@ -92,6 +75,7 @@ fn smoke_test_frames() { "smoke_test_frames", "", 0, + 0, ); } @@ -101,6 +85,7 @@ fn smoke_test_frames() { expected_name: &str, expected_file: &str, expected_line: u32, + expected_col: u32, ) { backtrace::resolve_frame(frame, |sym| { print!("symbol ip:{:?} address:{:?} ", frame.ip(), frame.symbol_address()); @@ -113,6 +98,9 @@ fn smoke_test_frames() { if let Some(lineno) = sym.lineno() { print!("lineno:{} ", lineno); } + if let Some(colno) = sym.colno() { + print!("colno:{} ", colno); + } println!(); }); @@ -121,65 +109,56 @@ fn smoke_test_frames() { assert!(ip >= sym); assert!( sym >= actual_fn_pointer, - "{:?} < {:?} ({} {}:{})", + "{:?} < {:?} ({} {}:{}:{})", sym as *const usize, actual_fn_pointer as *const usize, expected_name, expected_file, expected_line, + expected_col, ); // windows dbghelp is *quite* liberal (and wrong) in many of its reports // right now... // // This assertion can also fail for release builds, so skip it there - if !DBGHELP && cfg!(debug_assertions) { + if cfg!(debug_assertions) { assert!(sym - actual_fn_pointer < 1024); } let mut resolved = 0; - let can_resolve = DLADDR || LIBBACKTRACE || CORESYMBOLICATION || DBGHELP || GIMLI_SYMBOLIZE; let mut name = None; let mut addr = None; + let mut col = None; let mut line = None; let mut file = None; backtrace::resolve_frame(frame, |sym| { resolved += 1; name = sym.name().map(|v| v.to_string()); addr = sym.addr(); + col = sym.colno(); line = sym.lineno(); file = sym.filename().map(|v| v.to_path_buf()); }); + assert!(resolved > 0); - // dbghelp doesn't always resolve symbols right now - match resolved { - 0 => return assert!(!can_resolve || DBGHELP), - _ => {} - } - - // * linux dladdr doesn't work (only consults local symbol table) - // * windows dbghelp isn't great for GNU - if can_resolve && !(cfg!(target_os = "linux") && DLADDR) && !(DBGHELP && !MSVC) { - let name = name.expect("didn't find a name"); + let name = name.expect("didn't find a name"); - // in release mode names get weird as functions can get merged - // together with `mergefunc`, so only assert this in debug mode - if cfg!(debug_assertions) { - assert!( - name.contains(expected_name), - "didn't find `{}` in `{}`", - expected_name, - name - ); - } + // in release mode names get weird as functions can get merged + // together with `mergefunc`, so only assert this in debug mode + if cfg!(debug_assertions) { + assert!( + name.contains(expected_name), + "didn't find `{}` in `{}`", + expected_name, + name + ); } - if can_resolve { - addr.expect("didn't find a symbol"); - } + addr.expect("didn't find a symbol"); - if (LIBBACKTRACE || CORESYMBOLICATION || (DBGHELP && MSVC)) && cfg!(debug_assertions) { + if cfg!(debug_assertions) { let line = line.expect("didn't find a line number"); let file = file.expect("didn't find a line number"); if !expected_file.is_empty() { @@ -199,6 +178,20 @@ fn smoke_test_frames() { expected_line ); } + + // dbghelp on MSVC doesn't support column numbers + if !cfg!(target_env = "msvc") { + let col = col.expect("didn't find a column number"); + if expected_col != 0 { + assert!( + col == expected_col, + "bad column number on frame for `{}`: {} != {}", + expected_name, + col, + expected_col + ); + } + } } } } @@ -248,3 +241,83 @@ fn is_serde() { is_serialize::(); is_deserialize::(); } + +#[test] +fn sp_smoke_test() { + let mut refs = vec![]; + recursive_stack_references(&mut refs); + return; + + #[inline(never)] + fn recursive_stack_references(refs: &mut Vec) { + assert!(refs.len() < 5); + + let x = refs.len(); + refs.push(&x as *const _ as usize); + + if refs.len() < 5 { + recursive_stack_references(refs); + eprintln!("exiting: {}", x); + return; + } + + backtrace::trace(make_trace_closure(refs)); + eprintln!("exiting: {}", x); + } + + // NB: the following `make_*` functions are pulled out of line, rather than + // defining their results as inline closures at their call sites, so that + // the resulting closures don't have "recursive_stack_references" in their + // mangled names. + + fn make_trace_closure<'a>( + refs: &'a mut Vec, + ) -> impl FnMut(&backtrace::Frame) -> bool + 'a { + let mut child_sp = None; + let mut child_ref = None; + move |frame| { + eprintln!("\n=== frame ==================================="); + + let mut is_recursive_stack_references = false; + backtrace::resolve(frame.ip(), |sym| { + is_recursive_stack_references |= + sym.name() + .and_then(|name| name.as_str()) + .map_or(false, |name| { + eprintln!("name = {}", name); + name.contains("recursive_stack_references") + }) + }); + + let sp = frame.sp() as usize; + eprintln!("sp = {:p}", sp as *const u8); + if sp == 0 { + // If the SP is null, then we don't have an implementation for + // getting the SP on this target. Just keep walking the stack, + // but don't make our assertions about the on-stack pointers and + // SP values. + return true; + } + + // The stack grows down. + if let Some(child_sp) = child_sp { + assert!(child_sp <= sp); + } + + if is_recursive_stack_references { + let r = refs.pop().unwrap(); + eprintln!("ref = {:p}", r as *const u8); + if sp != 0 { + assert!(r > sp); + if let Some(child_ref) = child_ref { + assert!(sp >= child_ref); + } + } + child_ref = Some(r); + } + + child_sp = Some(sp); + true + } + } +} diff --git a/crux-mir/lib/bigint/Cargo.toml b/crux-mir/lib/bigint/Cargo.toml deleted file mode 100644 index 3a38176f3..000000000 --- a/crux-mir/lib/bigint/Cargo.toml +++ /dev/null @@ -1,31 +0,0 @@ -[package] -description = "Large fixed-size integers arithmetics" -homepage = "http://parity.io" -repository = "https://github.com/ethcore/bigint" -license = "MIT/Apache-2.0" -name = "bigint" -version = "4.4.1" -authors = ["Parity Technologies "] - -[dependencies] -rustc-hex = { version = "1.0", optional = true } -heapsize = { version = "0.4", optional = true } -byteorder = { version = "1", default-features = false } -serde = { version = "1.0", optional = true } -serde_derive = { version = "1.0", optional = true } -crunchy = "0.1.5" - -[dev-dependencies] -quickcheck = "0.4" - -[features] -serialize = ["serde", "serde_derive"] -heapsizeof = ["heapsize", "std"] -std = ["rustc-hex"] - -[[example]] -name = "modular" -required-features = ["std"] - -[badges] -maintenance = { status = "deprecated" } \ No newline at end of file diff --git a/crux-mir/lib/bigint/LICENSE-MIT b/crux-mir/lib/bigint/LICENSE-MIT deleted file mode 100644 index 089f3a31c..000000000 --- a/crux-mir/lib/bigint/LICENSE-MIT +++ /dev/null @@ -1,7 +0,0 @@ -Copyright 2015-2017 Parity Technologies - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE \ No newline at end of file diff --git a/crux-mir/lib/bigint/README.md b/crux-mir/lib/bigint/README.md deleted file mode 100644 index 41d6547ea..000000000 --- a/crux-mir/lib/bigint/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# bigint - -[![Build Status](https://travis-ci.org/paritytech/bigint.svg?branch=master)](https://travis-ci.org/paritytech/bigint) - -[API Documentation](https://docs.rs/bigint/) - -## DEPRECATED -This crate is **deprecated** and will not be developed further. Users are invited to prefer the [`uint`](https://crates.io/crates/uint) crate instead. - -### Old readme: -Fixed-sized integers arithmetic - -To specify a dependency, add to `Cargo.toml` - -```toml -[dependencies] -bigint = "4" -``` - -Little example - -```rust -extern crate bigint; -use bigint::U256; - -fn main() { - let mut val: U256 = 1023.into(); - for _ in 0..200 { val = val * 2.into() } - assert_eq!( - &format!("{}", val), - "1643897619276947051879427220465009342380213662639797070513307648" - ); -} -``` - -### `no_std` crates - -This crate has a feature, `std`, that is enabled by default. To use this crate -in a `no_std` context, add the following to your `Cargo.toml`: - -```toml -[dependencies] -bigint = { version = "4", default-features = false } -``` - -# License - -`bigint` is primarily distributed under the terms of both the MIT -license and the Apache License (Version 2.0), at your choice. - -See LICENSE-APACHE, and LICENSE-MIT for details. - -## Contribution - -Unless you explicitly state otherwise, any contribution intentionally submitted -for inclusion in `bigint` by you, as defined in the Apache-2.0 license, shall be -dual licensed as above, without any additional terms or conditions. diff --git a/crux-mir/lib/bigint/benches/bigint.rs b/crux-mir/lib/bigint/benches/bigint.rs deleted file mode 100644 index b9dc9649f..000000000 --- a/crux-mir/lib/bigint/benches/bigint.rs +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright 2015-2017 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -//! benchmarking for bigint -//! should be started with: -//! ```bash -//! rustup run nightly cargo bench -//! ``` - -#![feature(test)] - -extern crate test; -extern crate bigint; - -use test::{Bencher, black_box}; -use bigint::{U256, U512, U128}; - -#[bench] -fn u256_add(b: &mut Bencher) { - b.iter(|| { - let n = black_box(10000); - let zero = black_box(U256::zero()); - (0..n).fold(zero, |old, new| { - old.overflowing_add(U256::from(black_box(new))).0 - }) - }); -} - -#[bench] -fn u256_sub(b: &mut Bencher) { - b.iter(|| { - let n = black_box(10000); - let max = black_box(U256::max_value()); - (0..n).fold(max, |old, new| { - old.overflowing_sub(U256::from(black_box(new))).0 - }) - }); -} - -#[bench] -fn u512_sub(b: &mut Bencher) { - b.iter(|| { - let n = black_box(10000); - let max = black_box(U512::max_value()); - (0..n).fold(max, |old, new| { - let new = black_box(new); - let p = new % 2; - old.overflowing_sub(U512([p, p, p, p, p, p, p, new])).0 - }) - }); -} - -#[bench] -fn u512_add(b: &mut Bencher) { - b.iter(|| { - let n = black_box(10000); - let zero = black_box(U512::zero()); - (0..n).fold(zero, |old, new| { - let new = black_box(new); - old.overflowing_add(U512([new, new, new, new, new, new, new, new])) - .0 - }) - }); -} - -#[bench] -fn u512_mul(b: &mut Bencher) { - b.iter(|| { - (1..10000).fold(black_box(U512::one()), |old, new| { - old.overflowing_mul(U512::from(black_box(new | 1))).0 - }) - }); -} - -#[bench] -fn u512_mul_small(b: &mut Bencher) { - b.iter(|| { - (1..153).fold(black_box(U512::one()), |old, _| { - old.overflowing_mul(U512::from(black_box(10))).0 - }) - }); -} - -#[bench] -fn u256_mul(b: &mut Bencher) { - b.iter(|| { - (1..10000).fold(black_box(U256::one()), |old, new| { - old.overflowing_mul(U256::from(black_box(new | 1))).0 - }) - }); -} - -#[bench] -fn u256_mul_small(b: &mut Bencher) { - b.iter(|| { - (1..77).fold(black_box(U256::one()), |old, _| { - old.overflowing_mul(U256::from(black_box(10))).0 - }) - }); -} - -#[bench] -fn u256_full_mul(b: &mut Bencher) { - b.iter(|| { - let n = black_box(10000); - let one = black_box(U256::one()); - (1..n).map(|n| n | 1).fold(one, |old, new| { - let new = black_box(new); - let U512(ref u512words) = old.full_mul(U256([new, new, new, new])); - U256([u512words[0], u512words[2], u512words[2], u512words[3]]) - }) - }); -} - - -#[bench] -fn u128_mul(b: &mut Bencher) { - b.iter(|| { - let n = black_box(10000); - (1..n).fold(U128([12345u64, 0u64]), |old, new| { - old.overflowing_mul(U128::from(new | 1)).0 - }) - }); -} - -#[bench] -fn u256_from_le(b: &mut Bencher) { - b.iter(|| { - let raw = black_box( - [ - 1u8, - 2, - 3, - 5, - 7, - 11, - 13, - 17, - 19, - 23, - 29, - 31, - 37, - 41, - 43, - 47, - 53, - 59, - 61, - 67, - 71, - 73, - 79, - 83, - 89, - 97, - 101, - 103, - 107, - 109, - 113, - 127, - ], - ); - let _ = U256::from_little_endian(&raw[..]); - }); -} - -#[bench] -fn u256_from_be(b: &mut Bencher) { - b.iter(|| { - let raw = black_box( - [ - 1u8, - 2, - 3, - 5, - 7, - 11, - 13, - 17, - 19, - 23, - 29, - 31, - 37, - 41, - 43, - 47, - 53, - 59, - 61, - 67, - 71, - 73, - 79, - 83, - 89, - 97, - 101, - 103, - 107, - 109, - 113, - 127, - ], - ); - let _ = U256::from_big_endian(&raw[..]); - }); -} diff --git a/crux-mir/lib/bigint/examples/modular.rs b/crux-mir/lib/bigint/examples/modular.rs deleted file mode 100644 index a31ecfc97..000000000 --- a/crux-mir/lib/bigint/examples/modular.rs +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2015-2017 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -extern crate bigint; - -use bigint::U256; - -fn main() { - // Example modular arithmetic using bigint U256 primitives - - // imagine the field 0..p - // where the p is defined below - // (it's a prime!) - let p = U256::from_dec_str( - "38873241744847760218045702002058062581688990428170398542849190507947196700873" - ).expect("p to be a good number in the example"); - - // then, on this field, - // (p-1) + (p+1) = 0 - - // (p - 1) mod p - let p_minus_1 = (p - 1u64.into()) % p; - // (p + 1) mod p - let p_plus_1 = (p + 1u64.into()) % p; - // ((p - 1) mod p + (p + 1) mod p) mod p - let sum = (p_minus_1 + p_plus_1) % p; - assert_eq!(sum, 0.into()); - - // on this field, - // (p-1) + (p-1) = p-2 - let p_minus_1 = (p - 1u64.into()) % p; - let sum = (p_minus_1 + p_minus_1) % p; - assert_eq!(sum, p - 2.into()); - - // on this field, - // (p-1) * 3 = p-3 - let p_minus_1 = (p - 1u64.into()) % p; - - // multiplication is a series of additions - let multiplicator = 3; - let mul = { - let mut result = p_minus_1; - for _ in 0..multiplicator-1 { - result = (p_minus_1 + result) % p; - } - result - }; - - assert_eq!(mul, p - 3.into()); -} \ No newline at end of file diff --git a/crux-mir/lib/bigint/src/lib.rs b/crux-mir/lib/bigint/src/lib.rs deleted file mode 100644 index 845fd2b34..000000000 --- a/crux-mir/lib/bigint/src/lib.rs +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2015-2017 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -//! Efficient large, fixed-size big integers and hashes. - -#![cfg_attr(not(feature="std"), no_std)] -#![cfg_attr(all(not(feature="std"), test), feature(alloc))] - -extern crate crucible; - -pub mod uint; -pub use ::uint::*; diff --git a/crux-mir/lib/bigint/src/uint.rs b/crux-mir/lib/bigint/src/uint.rs deleted file mode 100644 index 5f80dd453..000000000 --- a/crux-mir/lib/bigint/src/uint.rs +++ /dev/null @@ -1,2126 +0,0 @@ -// Copyright 2015-2017 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -// Code derived from original work by Andrew Poelstra - -// Rust Bitcoin Library -// Written in 2014 by -// Andrew Poelstra -// -// To the extent possible under law, the author(s) have dedicated all -// copyright and related and neighboring rights to this software to -// the public domain worldwide. This software is distributed without -// any warranty. -// -// You should have received a copy of the CC0 Public Domain Dedication -// along with this software. -// If not, see . -// - -//! Big unsigned integer types. -//! -//! Implementation of a various large-but-fixed sized unsigned integer types. -//! The functions here are designed to be fast. There are optional `x86_64` -//! implementations for even more speed, hidden behind the `x64_arithmetic` -//! feature flag. - -use core::{str, mem}; -use core::ops::{Shr, Shl, BitAnd, BitOr, BitXor, Not, Div, Rem, Mul, Add, Sub}; - -use crucible::Symbolic; - -mod crucible_uint { - pub use crucible::bitvector::Bv128 as U128; - pub use crucible::bitvector::Bv256 as U256; - pub use crucible::bitvector::Bv512 as U512; -} - -/// Conversion from decimal string error -#[derive(Debug, PartialEq)] -pub enum FromDecStrErr { - /// Char not from range 0-9 - InvalidCharacter, - /// Value does not fit into type - InvalidLength, -} - -macro_rules! impl_map_from { - ($thing:ident, $from:ty, $to:ty) => { - impl From<$from> for $thing { - fn from(value: $from) -> $thing { - From::from(value as $to) - } - } - } -} - -macro_rules! uint_overflowing_add { - ($name:ident, $n_words:tt, $self_expr: expr, $other: expr) => ({ - uint_overflowing_add_reg!($name, $n_words, $self_expr, $other) - }) -} - -macro_rules! uint_overflowing_add_reg { - ($name:ident, $n_words:tt, $self_expr: expr, $other: expr) => ({ - let (raw, over) = crucible_uint::$name::overflowing_add($self_expr.raw, $other.raw); - ($name { raw }, over) - }) -} - -macro_rules! uint_overflowing_sub { - ($name:ident, $n_words:tt, $self_expr: expr, $other: expr) => ({ - uint_overflowing_sub_reg!($name, $n_words, $self_expr, $other) - }) -} - -macro_rules! uint_overflowing_sub_reg { - ($name:ident, $n_words:tt, $self_expr: expr, $other: expr) => ({ - let (raw, over) = crucible_uint::$name::overflowing_sub($self_expr.raw, $other.raw); - ($name { raw }, over) - }) -} - -macro_rules! uint_overflowing_mul { - ($name:ident, $n_words:tt, $self_expr: expr, $other: expr) => ({ - uint_overflowing_mul_reg!($name, $n_words, $self_expr, $other) - }) -} - -macro_rules! uint_overflowing_mul_reg { - ($name:ident, $n_words:tt, $self_expr: expr, $other: expr) => ({ - let (raw, over) = crucible_uint::$name::overflowing_mul($self_expr.raw, $other.raw); - ($name { raw }, over) - }) -} - -macro_rules! overflowing { - ($op: expr, $overflow: expr) => ( - { - let (overflow_x, overflow_overflow) = $op; - $overflow |= overflow_overflow; - overflow_x - } - ); - ($op: expr) => ( - { - let (overflow_x, _overflow_overflow) = $op; - overflow_x - } - ); -} - -macro_rules! panic_on_overflow { - ($name: expr) => { - if $name { - panic!("arithmetic operation overflow") - } - } -} - -#[inline(always)] -fn mul_u32(a: (u64, u64), b: u64, carry: u64) -> (u64, u64) { - let upper = b * a.0; - let lower = b * a.1; - - let (res1, overflow1) = lower.overflowing_add(upper << 32); - let (res2, overflow2) = res1.overflowing_add(carry); - - let carry = (upper >> 32) + overflow1 as u64 + overflow2 as u64; - (res2, carry) -} - -#[inline(always)] -fn split(a: u64) -> (u64, u64) { - (a >> 32, a & 0xFFFFFFFF) -} - -#[inline(always)] -fn split_u128(a: u128) -> (u64, u64) { - ((a >> 64) as _, (a & 0xFFFFFFFFFFFFFFFF) as _) -} - -macro_rules! construct_uint { - ($name:ident, $n_words:tt) => ( - /// Little-endian large integer type - #[repr(C)] - #[derive(Copy, Clone, Eq, PartialEq)] - // TODO: #[derive(Hash)] - #[cfg_attr(feature="serialize", derive(Serialize, Deserialize))] - pub struct $name { - raw: crucible_uint::$name, - } - - impl $name { - pub const MAX: $name = $name { - raw: crucible_uint::$name::MAX, - }; - - /// Convert from a decimal string. - pub fn from_dec_str(value: &str) -> Result { - if !value.bytes().all(|b| b >= 48 && b <= 57) { - return Err(FromDecStrErr::InvalidCharacter) - } - - let mut res = Self::default(); - for b in value.bytes().map(|b| b - 48) { - let (r, overflow) = res.overflowing_mul_u32(10); - if overflow { - return Err(FromDecStrErr::InvalidLength); - } - let (r, overflow) = r.overflowing_add(b.into()); - if overflow { - return Err(FromDecStrErr::InvalidLength); - } - res = r; - } - Ok(res) - } - - /// Conversion to u32 - #[inline] - pub fn low_u32(&self) -> u32 { - self.raw.into() - } - - /// Conversion to u64 - #[inline] - pub fn low_u64(&self) -> u64 { - self.raw.into() - } - - /// Conversion to u32 with overflow checking - /// - /// # Panics - /// - /// Panics if the number is larger than 2^32. - #[inline] - pub fn as_u32(&self) -> u32 { - if self.raw >> 32 != crucible_uint::$name::ZERO { - panic!("Integer overflow when casting U256") - } - self.low_u32() - } - - /// Conversion to u64 with overflow checking - /// - /// # Panics - /// - /// Panics if the number is larger than 2^64. - #[inline] - pub fn as_u64(&self) -> u64 { - if self.raw >> 64 != crucible_uint::$name::ZERO { - panic!("Integer overflow when casting U256") - } - self.low_u64() - } - - /// Whether this is zero. - #[inline] - pub fn is_zero(&self) -> bool { - self.raw == crucible_uint::$name::ZERO - } - - /// Return the least number of bits needed to represent the number - #[inline] - pub fn bits(&self) -> usize { - $n_words * 64 - self.leading_zeros() as usize - } - - /// Return if specific bit is set. - /// - /// # Panics - /// - /// Panics if `index` exceeds the bit width of the number. - #[inline] - pub fn bit(&self, index: usize) -> bool { - u8::from((self.raw >> index) & crucible_uint::$name::ONE) != 0 - } - - /// Returns the number of leading zeros in the binary representation of self. - pub fn leading_zeros(&self) -> u32 { - self.raw.leading_zeros() - } - - /// Returns the number of leading zeros in the binary representation of self. - pub fn trailing_zeros(&self) -> u32 { - unimplemented!() - } - - /// Return specific byte. - /// - /// # Panics - /// - /// Panics if `index` exceeds the byte width of the number. - #[inline] - pub fn byte(&self, index: usize) -> u8 { - (self.raw >> (index * 8)).into() - } - - /// Write to the slice in big-endian format. - #[inline] - pub fn to_big_endian(&self, bytes: &mut [u8]) { - debug_assert!($n_words * 8 == bytes.len()); - for i in 0..bytes.len() { - bytes[bytes.len() - i - 1] = self.byte(i); - } - } - - /// Write to the slice in little-endian format. - #[inline] - pub fn to_little_endian(&self, bytes: &mut [u8]) { - debug_assert!($n_words * 8 == bytes.len()); - for i in 0..bytes.len() { - bytes[i] = self.byte(i); - } - } - - /// Convert to hex string. - #[cfg(feature="std")] - #[inline] - pub fn to_hex(&self) -> String { - use core::cmp; - use rustc_hex::ToHex;; - - if self.is_zero() { return "0".to_owned(); } // special case. - let mut bytes = [0u8; 8 * $n_words]; - self.to_big_endian(&mut bytes); - let bp7 = self.bits() + 7; - let len = cmp::max(bp7 / 8, 1); - let bytes_hex = bytes[bytes.len() - len..].to_hex(); - (&bytes_hex[1 - bp7 % 8 / 4..]).to_owned() - } - - /// Create `10**n` as this type. - /// - /// # Panics - /// - /// Panics if the result overflows the type. - #[inline] - pub fn exp10(n: usize) -> Self { - match n { - 0 => Self::from(1u64), - _ => Self::exp10(n - 1).mul_u32(10) - } - } - - /// Zero (additive identity) of this type. - #[inline] - pub fn zero() -> Self { - Self { - raw: crucible_uint::$name::ZERO, - } - } - - /// One (multiplicative identity) of this type. - #[inline] - pub fn one() -> Self { - Self { - raw: crucible_uint::$name::ONE, - } - } - - /// The maximum value which can be inhabited by this type. - #[inline] - pub fn max_value() -> Self { - !Self::zero() - } - - /// Fast exponentation by squaring - /// https://en.wikipedia.org/wiki/Exponentiation_by_squaring - /// - /// # Panics - /// - /// Panics if the result overflows the type. - pub fn pow(self, expon: Self) -> Self { - if expon.is_zero() { - return Self::one() - } - let is_even = |x : &Self| x.low_u64() & 1 == 0; - - let u_one = Self::one(); - let mut y = u_one; - let mut n = expon; - let mut x = self; - while n > u_one { - if is_even(&n) { - x = x * x; - n = n >> 1; - } else { - y = x * y; - x = x * x; - n = (n - u_one) >> 1; - } - } - x * y - } - - /// Fast exponentation by squaring - /// https://en.wikipedia.org/wiki/Exponentiation_by_squaring - pub fn overflowing_pow(self, expon: Self) -> (Self, bool) { - if expon.is_zero() { return (Self::one(), false) } - - let is_even = |x : &Self| x.low_u64() & 1 == 0; - - let u_one = Self::one(); - let mut y = u_one; - let mut n = expon; - let mut x = self; - let mut overflow = false; - - while n > u_one { - if is_even(&n) { - x = overflowing!(x.overflowing_mul(x), overflow); - n = n >> 1; - } else { - y = overflowing!(x.overflowing_mul(y), overflow); - x = overflowing!(x.overflowing_mul(x), overflow); - n = (n - u_one) >> 1; - } - } - let res = overflowing!(x.overflowing_mul(y), overflow); - (res, overflow) - } - - /// Optimized instructions - #[inline(always)] - pub fn overflowing_add(self, other: $name) -> ($name, bool) { - uint_overflowing_add!($name, $n_words, self, other) - } - - /// Addition which saturates at the maximum value. - pub fn saturating_add(self, other: $name) -> $name { - match self.overflowing_add(other) { - (_, true) => $name::max_value(), - (val, false) => val, - } - } - - /// Subtraction which underflows and returns a flag if it does. - #[inline(always)] - pub fn overflowing_sub(self, other: $name) -> ($name, bool) { - uint_overflowing_sub!($name, $n_words, self, other) - } - - /// Subtraction which saturates at zero. - pub fn saturating_sub(self, other: $name) -> $name { - match self.overflowing_sub(other) { - (_, true) => $name::zero(), - (val, false) => val, - } - } - - /// Multiply with overflow, returning a flag if it does. - #[inline(always)] - pub fn overflowing_mul(self, other: $name) -> ($name, bool) { - uint_overflowing_mul!($name, $n_words, self, other) - } - - /// Multiplication which saturates at the maximum value.. - pub fn saturating_mul(self, other: $name) -> $name { - match self.overflowing_mul(other) { - (_, true) => $name::max_value(), - (val, false) => val, - } - } - - /// Division with overflow - pub fn overflowing_div(self, other: $name) -> ($name, bool) { - (self / other, false) - } - - /// Modulus with overflow. - pub fn overflowing_rem(self, other: $name) -> ($name, bool) { - (self % other, false) - } - - /// Negation with overflow. - pub fn overflowing_neg(self) -> ($name, bool) { - (!self, true) - } - - /// Multiplication by u32 - #[allow(dead_code)] // not used when multiplied with inline assembly - fn mul_u32(self, other: u32) -> Self { - let (ret, overflow) = self.overflowing_mul_u32(other); - panic_on_overflow!(overflow); - ret - } - - /// Overflowing multiplication by u32 - #[allow(dead_code)] // not used when multiplied with inline assembly - fn overflowing_mul_u32(self, other: u32) -> (Self, bool) { - self.overflowing_mul(other.into()) - } - - /// Converts from big endian representation bytes in memory - /// Can also be used as (&slice).into(), as it is default `From` - /// slice implementation for U256 - pub fn from_big_endian(slice: &[u8]) -> Self { - assert!($n_words * 8 >= slice.len()); - let mut result = Self::zero(); - for i in 0..slice.len() { - result = result | Self::from(slice[i]) << (8 * (slice.len() - i - 1)); - } - result - } - - /// Converts from little endian representation bytes in memory - pub fn from_little_endian(slice: &[u8]) -> Self { - assert!($n_words * 8 >= slice.len()); - let mut result = Self::zero(); - for i in 0..slice.len() { - result = result | Self::from(slice[i]) << (8 * i); - } - result - } - } - - impl Default for $name { - fn default() -> Self { - $name::zero() - } - } - - impl From for $name { - fn from(value: u64) -> $name { - $name { - raw: crucible_uint::$name::from(value), - } - } - } - - - impl_map_from!($name, u8, u64); - impl_map_from!($name, u16, u64); - impl_map_from!($name, u32, u64); - impl_map_from!($name, usize, u64); - - impl From for $name { - fn from(value: i64) -> $name { - match value >= 0 { - true => From::from(value as u64), - false => { panic!("Unsigned integer can't be created from negative value"); } - } - } - } - - impl_map_from!($name, i8, i64); - impl_map_from!($name, i16, i64); - impl_map_from!($name, i32, i64); - impl_map_from!($name, isize, i64); - - // Converts from big endian representation of U256 - impl<'a> From<&'a [u8]> for $name { - fn from(bytes: &[u8]) -> $name { - Self::from_big_endian(bytes) - } - } - - #[cfg(feature="std")] - impl str::FromStr for $name { - type Err = ::rustc_hex::FromHexError; - - fn from_str(value: &str) -> Result<$name, Self::Err> { - use rustc_hex::FromHex; - - let bytes: Vec = match value.len() % 2 == 0 { - true => try!(value.from_hex()), - false => try!(("0".to_owned() + value).from_hex()) - }; - - let bytes_ref: &[u8] = &bytes; - Ok(From::from(bytes_ref)) - } - } - - impl Add<$name> for $name { - type Output = $name; - - fn add(self, other: $name) -> $name { - let (result, overflow) = self.overflowing_add(other); - panic_on_overflow!(overflow); - result - } - } - - impl Sub<$name> for $name { - type Output = $name; - - #[inline] - fn sub(self, other: $name) -> $name { - let (result, overflow) = self.overflowing_sub(other); - panic_on_overflow!(overflow); - result - } - } - - impl Mul<$name> for $name { - type Output = $name; - - fn mul(self, other: $name) -> $name { - let (result, overflow) = self.overflowing_mul(other); - panic_on_overflow!(overflow); - result - } - } - - impl Div<$name> for $name { - type Output = $name; - - fn div(self, other: $name) -> $name { - $name { - raw: self.raw.div(other.raw), - } - } - } - - impl Rem<$name> for $name { - type Output = $name; - - fn rem(self, other: $name) -> $name { - $name { - raw: self.raw.rem(other.raw), - } - } - } - - impl BitAnd<$name> for $name { - type Output = $name; - - #[inline] - fn bitand(self, other: $name) -> $name { - $name { - raw: self.raw.bitand(other.raw), - } - } - } - - impl BitXor<$name> for $name { - type Output = $name; - - #[inline] - fn bitxor(self, other: $name) -> $name { - $name { - raw: self.raw.bitxor(other.raw), - } - } - } - - impl BitOr<$name> for $name { - type Output = $name; - - #[inline] - fn bitor(self, other: $name) -> $name { - $name { - raw: self.raw.bitor(other.raw), - } - } - } - - impl Not for $name { - type Output = $name; - - #[inline] - fn not(self) -> $name { - $name { - raw: self.raw.not(), - } - } - } - - impl Shl for $name { - type Output = $name; - - fn shl(self, shift: usize) -> $name { - $name { - raw: self.raw.shl(shift), - } - } - } - - impl Shr for $name { - type Output = $name; - - fn shr(self, shift: usize) -> $name { - $name { - raw: self.raw.shr(shift), - } - } - } - - impl Ord for $name { - fn cmp(&self, other: &$name) -> ::core::cmp::Ordering { - self.raw.cmp(&other.raw) - } - } - - impl PartialOrd for $name { - fn partial_cmp(&self, other: &$name) -> Option<::core::cmp::Ordering> { - Some(self.cmp(other)) - } - } - - impl ::core::fmt::Debug for $name { - fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { - ::core::fmt::Display::fmt(self, f) - } - } - - impl ::core::fmt::Display for $name { - fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { - if self.is_zero() { - return write!(f, "0"); - } - - let mut buf = [0_u8; $n_words*20]; - let mut i = buf.len() - 1; - let mut current = *self; - let ten = $name::from(10); - - loop { - let digit = (current % ten).low_u64() as u8; - buf[i] = digit + b'0'; - current = current / ten; - if current.is_zero() { - break; - } - i -= 1; - } - - // sequence of `'0'..'9'` chars is guaranteed to be a valid UTF8 string - let s = unsafe {::core::str::from_utf8_unchecked(&buf[i..])}; - f.write_str(s) - } - } - - impl ::core::fmt::LowerHex for $name { - fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { - unimplemented!() - } - } - - #[cfg(feature="std")] - impl From<&'static str> for $name { - fn from(s: &'static str) -> Self { - s.parse().unwrap() - } - } - - impl Symbolic for $name { - fn symbolic(desc: &'static str) -> Self { - $name { - raw: crucible_uint::$name::symbolic(desc), - } - } - } - ); -} - -construct_uint!(U128, 2); -construct_uint!(U256, 4); -construct_uint!(U512, 8); - -impl U256 { - /// Multiplies two 256-bit integers to produce full 512-bit integer - /// No overflow possible - #[inline(always)] - pub fn full_mul(self, other: U256) -> U512 { - U512::from(self) * U512::from(other) - } - - /// Find modular inverse by modulo p - pub fn mod_inverse(self, p: Self) -> Self { - let mut mn = (p, self); - let mut xy = (U256::zero(), U256::one()); - - while mn.1 != U256::zero() { - let sb: U256 = ((mn.0 / mn.1).full_mul(xy.1) % U512::from(p)).into(); - if sb > xy.0 { - xy = (xy.1, p - ((sb - xy.0) % p)) - } else { - xy = (xy.1, xy.0 - sb) - } - mn = (mn.1, mn.0 % mn.1); - } - - xy.0 - } -} - -impl From for U512 { - fn from(value: U256) -> U512 { - (&value).into() - } -} - -impl From for U256 { - fn from(value: U512) -> U256 { - (&value).into() - } -} - -impl<'a> From<&'a U256> for U512 { - fn from(value: &'a U256) -> U512 { - U512 { - raw: value.raw.into(), - } - } -} - -impl<'a> From<&'a U512> for U256 { - fn from(value: &'a U512) -> U256 { - assert!(value.raw >> 256 == crucible_uint::U512::ZERO, "Overflow"); - U256 { - raw: value.raw.into(), - } - } -} - -impl From for U128 { - fn from(value: U256) -> U128 { - assert!(value.raw >> 128 == crucible_uint::U256::ZERO, "Overflow"); - U128 { - raw: value.raw.into(), - } - } -} - -impl From for U128 { - fn from(value: U512) -> U128 { - assert!(value.raw >> 128 == crucible_uint::U512::ZERO, "Overflow"); - U128 { - raw: value.raw.into(), - } - } -} - -impl From for U512 { - fn from(value: U128) -> U512 { - U512 { - raw: value.raw.into(), - } - } -} - -impl From for U256 { - fn from(value: U128) -> U256 { - U256 { - raw: value.raw.into(), - } - } -} - -impl From for u64 { - fn from(value: U256) -> u64 { - value.as_u64() - } -} - -impl From for u32 { - fn from(value: U256) -> u32 { - value.as_u32() - } -} - -impl<'a> From<&'a [u8; 32]> for U256 { - fn from(bytes: &[u8; 32]) -> Self { - bytes[..].into() - } -} - -impl From<[u8; 32]> for U256 { - fn from(bytes: [u8; 32]) -> Self { - bytes[..].as_ref().into() - } -} - -impl From for [u8; 32] { - fn from(number: U256) -> Self { - let mut arr = [0u8; 32]; - number.to_big_endian(&mut arr); - arr - } -} - -impl<'a> From<&'a [u8; 16]> for U128 { - fn from(bytes: &[u8; 16]) -> Self { - bytes[..].into() - } -} - -impl From<[u8; 16]> for U128 { - fn from(bytes: [u8; 16]) -> Self { - bytes[..].as_ref().into() - } -} - -impl From for [u8; 16] { - fn from(number: U128) -> Self { - let mut arr = [0u8; 16]; - number.to_big_endian(&mut arr); - arr - } -} - -impl<'a> From<&'a [u8; 64]> for U512 { - fn from(bytes: &[u8; 64]) -> Self { - bytes[..].into() - } -} - -impl From<[u8; 64]> for U512 { - fn from(bytes: [u8; 64]) -> Self { - bytes[..].as_ref().into() - } -} - -impl From for [u8; 64] { - fn from(number: U512) -> Self { - let mut arr = [0u8; 64]; - number.to_big_endian(&mut arr); - arr - } -} - -#[cfg(feature="heapsizeof")] -known_heap_size!(0, U128, U256); - -#[cfg(test)] -mod tests { - use uint::{U128, U256, U512}; - - #[test] - pub fn display_u128() { - let expected = "340282366920938463463374607431768211455"; - let value = U128::MAX; - assert_eq!(format!("{}", value), expected); - assert_eq!(format!("{:?}", value), expected); - } - - #[test] - pub fn display_u256() { - let expected = "115792089237316195423570985008687907853269984665640564039457584007913129639935"; - let value = U256::MAX; - assert_eq!(format!("{}", value), expected); - assert_eq!(format!("{:?}", value), expected); - } - - #[test] - pub fn display_u512() { - let expected = "13407807929942597099574024998205846127479365820592393377723561443721764030073546976801874298166903427690031858186486050853753882811946569946433649006084095"; - let value = U512::MAX; - assert_eq!(format!("{}", value), expected); - assert_eq!(format!("{:?}", value), expected); - } -} - -#[cfg(test)] -#[cfg(feature="std")] -mod std_tests { - use uint::{U128, U256, U512}; - use std::str::FromStr; - use super::FromDecStrErr; - use std::u64::MAX; - - #[test] - pub fn uint256_from() { - let e = U256([10, 0, 0, 0]); - - // test unsigned initialization - let ua = U256::from(10u8); - let ub = U256::from(10u16); - let uc = U256::from(10u32); - let ud = U256::from(10u64); - assert_eq!(e, ua); - assert_eq!(e, ub); - assert_eq!(e, uc); - assert_eq!(e, ud); - - // test initialization from bytes - let va = U256::from(&[10u8][..]); - assert_eq!(e, va); - - // more tests for initialization from bytes - assert_eq!(U256([0x1010, 0, 0, 0]), U256::from(&[0x10u8, 0x10][..])); - assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from(&[0x12u8, 0xf0][..])); - assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from(&[0, 0x12u8, 0xf0][..])); - assert_eq!(U256([0x12f0, 0 , 0, 0]), U256::from(&[0, 0, 0, 0, 0, 0, 0, 0x12u8, 0xf0][..])); - assert_eq!(U256([0x12f0, 1 , 0, 0]), U256::from(&[1, 0, 0, 0, 0, 0, 0, 0x12u8, 0xf0][..])); - assert_eq!( - U256([0x12f0, 1 , 0x0910203040506077, 0x8090a0b0c0d0e0f0]), - U256::from(& - [ - 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, - 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, - 0, 0, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0x12u8, 0xf0 - ][..] - ) - ); - assert_eq!( - U256([0x00192437100019fa, 0x243710, 0, 0]), - U256::from(&[0x24u8, 0x37, 0x10,0, 0x19, 0x24, 0x37, 0x10, 0, 0x19, 0xfa][..]) - ); - - // test initializtion from string - let sa = U256::from_str("0a").unwrap(); - assert_eq!(e, sa); - assert_eq!(U256([0x1010, 0, 0, 0]), U256::from_str("1010").unwrap()); - assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from_str("12f0").unwrap()); - assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from_str("12f0").unwrap()); - assert_eq!(U256([0x12f0, 0 , 0, 0]), U256::from_str("0000000012f0").unwrap()); - assert_eq!(U256([0x12f0, 1 , 0, 0]), U256::from_str("0100000000000012f0").unwrap()); - assert_eq!( - U256([0x12f0, 1 , 0x0910203040506077, 0x8090a0b0c0d0e0f0]), - U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000000100000000000012f0").unwrap() - ); - } - - #[test] - pub fn uint256_to() { - let hex = "8090a0b0c0d0e0f00910203040506077583a2cf8264910e1436bda32571012f0"; - let uint = U256::from_str(hex).unwrap(); - let mut bytes = [0u8; 32]; - uint.to_big_endian(&mut bytes); - let uint2 = U256::from(&bytes[..]); - assert_eq!(uint, uint2); - } - - #[test] - pub fn uint256_bits_test() { - assert_eq!(U256::from(0u64).bits(), 0); - assert_eq!(U256::from(255u64).bits(), 8); - assert_eq!(U256::from(256u64).bits(), 9); - assert_eq!(U256::from(300u64).bits(), 9); - assert_eq!(U256::from(60000u64).bits(), 16); - assert_eq!(U256::from(70000u64).bits(), 17); - - //// Try to read the following lines out loud quickly - let mut shl = U256::from(70000u64); - shl = shl << 100; - assert_eq!(shl.bits(), 117); - shl = shl << 100; - assert_eq!(shl.bits(), 217); - shl = shl << 100; - assert_eq!(shl.bits(), 0); - - //// Bit set check - //// 01010 - assert!(!U256::from(10u8).bit(0)); - assert!(U256::from(10u8).bit(1)); - assert!(!U256::from(10u8).bit(2)); - assert!(U256::from(10u8).bit(3)); - assert!(!U256::from(10u8).bit(4)); - - //// byte check - assert_eq!(U256::from(10u8).byte(0), 10); - assert_eq!(U256::from(0xffu64).byte(0), 0xff); - assert_eq!(U256::from(0xffu64).byte(1), 0); - assert_eq!(U256::from(0x01ffu64).byte(0), 0xff); - assert_eq!(U256::from(0x01ffu64).byte(1), 0x1); - assert_eq!(U256([0u64, 0xfc, 0, 0]).byte(8), 0xfc); - assert_eq!(U256([0u64, 0, 0, u64::max_value()]).byte(31), 0xff); - assert_eq!(U256([0u64, 0, 0, (u64::max_value() >> 8) + 1]).byte(31), 0x01); - } - - #[test] - #[cfg_attr(feature="dev", allow(eq_op))] - pub fn uint256_comp_test() { - let small = U256([10u64, 0, 0, 0]); - let big = U256([0x8C8C3EE70C644118u64, 0x0209E7378231E632, 0, 0]); - let bigger = U256([0x9C8C3EE70C644118u64, 0x0209E7378231E632, 0, 0]); - let biggest = U256([0x5C8C3EE70C644118u64, 0x0209E7378231E632, 0, 1]); - - assert!(small < big); - assert!(big < bigger); - assert!(bigger < biggest); - assert!(bigger <= biggest); - assert!(biggest <= biggest); - assert!(bigger >= big); - assert!(bigger >= small); - assert!(small <= small); - } - - #[test] - pub fn uint256_arithmetic_test() { - let init = U256::from(0xDEADBEEFDEADBEEFu64); - let copy = init; - - let add = init + copy; - assert_eq!(add, U256([0xBD5B7DDFBD5B7DDEu64, 1, 0, 0])); - // Bitshifts - let shl = add << 88; - assert_eq!(shl, U256([0u64, 0xDFBD5B7DDE000000, 0x1BD5B7D, 0])); - let shr = shl >> 40; - assert_eq!(shr, U256([0x7DDE000000000000u64, 0x0001BD5B7DDFBD5B, 0, 0])); - // Increment - let incr = shr + U256::from(1u64); - assert_eq!(incr, U256([0x7DDE000000000001u64, 0x0001BD5B7DDFBD5B, 0, 0])); - // Subtraction - let sub = overflowing!(incr.overflowing_sub(init)); - assert_eq!(sub, U256([0x9F30411021524112u64, 0x0001BD5B7DDFBD5A, 0, 0])); - // Multiplication - let mult = sub.mul_u32(300); - assert_eq!(mult, U256([0x8C8C3EE70C644118u64, 0x0209E7378231E632, 0, 0])); - // Division - assert_eq!(U256::from(105u8) / U256::from(5u8), U256::from(21u8)); - let div = mult / U256::from(300u16); - assert_eq!(div, U256([0x9F30411021524112u64, 0x0001BD5B7DDFBD5A, 0, 0])); - - let a = U256::from_str("ff000000000000000000000000000000000000000000000000000000000000d1").unwrap(); - let b = U256::from_str("00ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff2e").unwrap(); - println!("{:x}", a); - println!("{:x}", b); - assert_eq!(!a, b); - assert_eq!(a, !b); - } - - #[test] - pub fn uint256_simple_mul() { - let a = U256::from_str("10000000000000000").unwrap(); - let b = U256::from_str("10000000000000000").unwrap(); - - let c = U256::from_str("100000000000000000000000000000000").unwrap(); - println!("Multiplying"); - let result = a.overflowing_mul(b); - println!("Got result"); - assert_eq!(result, (c, false)) - } - - #[test] - pub fn uint256_extreme_bitshift_test() { - //// Shifting a u64 by 64 bits gives an undefined value, so make sure that - //// we're doing the Right Thing here - let init = U256::from(0xDEADBEEFDEADBEEFu64); - - assert_eq!(init << 64, U256([0, 0xDEADBEEFDEADBEEF, 0, 0])); - let add = (init << 64) + init; - assert_eq!(add, U256([0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0, 0])); - assert_eq!(add >> 0, U256([0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0, 0])); - assert_eq!(add << 0, U256([0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0, 0])); - assert_eq!(add >> 64, U256([0xDEADBEEFDEADBEEF, 0, 0, 0])); - assert_eq!(add << 64, U256([0, 0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0])); - } - - #[test] - pub fn uint256_exp10() { - assert_eq!(U256::exp10(0), U256::from(1u64)); - println!("\none: {:?}", U256::from(1u64)); - println!("ten: {:?}", U256::from(10u64)); - assert_eq!(U256::from(2u64) * U256::from(10u64), U256::from(20u64)); - assert_eq!(U256::exp10(1), U256::from(10u64)); - assert_eq!(U256::exp10(2), U256::from(100u64)); - assert_eq!(U256::exp10(5), U256::from(100000u64)); - } - - #[test] - pub fn uint256_mul32() { - assert_eq!(U256::from(0u64).mul_u32(2), U256::from(0u64)); - assert_eq!(U256::from(1u64).mul_u32(2), U256::from(2u64)); - assert_eq!(U256::from(10u64).mul_u32(2), U256::from(20u64)); - assert_eq!(U256::from(10u64).mul_u32(5), U256::from(50u64)); - assert_eq!(U256::from(1000u64).mul_u32(50), U256::from(50000u64)); - } - - #[test] - fn uint256_pow() { - assert_eq!(U256::from(10).pow(U256::from(0)), U256::from(1)); - assert_eq!(U256::from(10).pow(U256::from(1)), U256::from(10)); - assert_eq!(U256::from(10).pow(U256::from(2)), U256::from(100)); - assert_eq!(U256::from(10).pow(U256::from(3)), U256::from(1000)); - assert_eq!(U256::from(10).pow(U256::from(20)), U256::exp10(20)); - } - - #[test] - #[should_panic] - fn uint256_pow_overflow_panic() { - U256::from(2).pow(U256::from(0x100)); - } - - #[test] - fn should_format_hex_correctly() { - assert_eq!(&U256::from(0).to_hex(), &"0"); - assert_eq!(&U256::from(0x1).to_hex(), &"1"); - assert_eq!(&U256::from(0xf).to_hex(), &"f"); - assert_eq!(&U256::from(0x10).to_hex(), &"10"); - assert_eq!(&U256::from(0xff).to_hex(), &"ff"); - assert_eq!(&U256::from(0x100).to_hex(), &"100"); - assert_eq!(&U256::from(0xfff).to_hex(), &"fff"); - assert_eq!(&U256::from(0x1000).to_hex(), &"1000"); - } - - #[test] - fn uint256_overflowing_pow() { - // assert_eq!( - // U256::from(2).overflowing_pow(U256::from(0xff)), - // (U256::from_str("8000000000000000000000000000000000000000000000000000000000000000").unwrap(), false) - // ); - assert_eq!( - U256::from(2).overflowing_pow(U256::from(0x100)), - (U256::zero(), true) - ); - } - - #[test] - pub fn uint256_mul1() { - assert_eq!(U256::from(1u64) * U256::from(10u64), U256::from(10u64)); - } - - #[test] - pub fn uint256_mul2() { - let a = U512::from_str("10000000000000000fffffffffffffffe").unwrap(); - let b = U512::from_str("ffffffffffffffffffffffffffffffff").unwrap(); - - assert_eq!(a * b, U512::from_str("10000000000000000fffffffffffffffcffffffffffffffff0000000000000002").unwrap()); - } - - #[test] - pub fn uint256_overflowing_mul() { - assert_eq!( - U256::from_str("100000000000000000000000000000000").unwrap().overflowing_mul( - U256::from_str("100000000000000000000000000000000").unwrap() - ), - (U256::zero(), true) - ); - } - - #[test] - pub fn uint128_add() { - assert_eq!( - U128::from_str("fffffffffffffffff").unwrap() + U128::from_str("fffffffffffffffff").unwrap(), - U128::from_str("1ffffffffffffffffe").unwrap() - ); - } - - #[test] - pub fn uint128_add_overflow() { - assert_eq!( - U128::from_str("ffffffffffffffffffffffffffffffff").unwrap() - .overflowing_add( - U128::from_str("ffffffffffffffffffffffffffffffff").unwrap() - ), - (U128::from_str("fffffffffffffffffffffffffffffffe").unwrap(), true) - ); - } - - #[test] - #[should_panic] - #[cfg(debug_assertions)] - pub fn uint128_add_overflow_panic() { - let _res = U128::from_str("ffffffffffffffffffffffffffffffff").unwrap() - + - U128::from_str("ffffffffffffffffffffffffffffffff").unwrap(); - } - - #[test] - pub fn uint128_mul() { - assert_eq!( - U128::from_str("fffffffff").unwrap() * U128::from_str("fffffffff").unwrap(), - U128::from_str("ffffffffe000000001").unwrap()); - } - - #[test] - pub fn uint512_mul() { - assert_eq!( - U512::from_str("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() - * - U512::from_str("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap(), - U512::from_str("3fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000000000000000001").unwrap() - ); - } - - #[test] - pub fn uint256_mul_overflow() { - assert_eq!( - U256::from_str("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() - .overflowing_mul( - U256::from_str("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() - ), - (U256::from_str("1").unwrap(), true) - ); - } - - #[test] - #[should_panic] - pub fn uint256_mul_overflow_panic() { - let _res = U256::from_str("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() - * - U256::from_str("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap(); - } - - #[test] - pub fn uint256_sub_overflow() { - assert_eq!( - U256::from_str("0").unwrap() - .overflowing_sub( - U256::from_str("1").unwrap() - ), - (U256::from_str("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap(), true) - ); - } - - #[test] - #[should_panic] - pub fn uint256_sub_overflow_panic() { - let _res = U256::from_str("0").unwrap() - - - U256::from_str("1").unwrap(); - } - - #[test] - pub fn uint256_shl() { - assert_eq!( - U256::from_str("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() - << 4, - U256::from_str("fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0").unwrap() - ); - } - - #[test] - pub fn uint256_shl_words() { - assert_eq!( - U256::from_str("0000000000000001ffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() - << 64, - U256::from_str("ffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000").unwrap() - ); - assert_eq!( - U256::from_str("0000000000000000ffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() - << 64, - U256::from_str("ffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000").unwrap() - ); - } - - #[test] - pub fn uint256_mul() { - assert_eq!( - U256::from_str("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() - * - U256::from_str("2").unwrap(), - U256::from_str("fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe").unwrap() - ); - } - - #[test] - fn uint256_div() { - assert_eq!(U256::from(10u64) / U256::from(1u64), U256::from(10u64)); - assert_eq!(U256::from(10u64) / U256::from(2u64), U256::from(5u64)); - assert_eq!(U256::from(10u64) / U256::from(3u64), U256::from(3u64)); - } - - #[test] - fn uint256_rem() { - assert_eq!(U256::from(10u64) % U256::from(1u64), U256::from(0u64)); - assert_eq!(U256::from(10u64) % U256::from(3u64), U256::from(1u64)); - } - - #[test] - fn uint256_from_dec_str() { - assert_eq!(U256::from_dec_str("10").unwrap(), U256::from(10u64)); - assert_eq!(U256::from_dec_str("1024").unwrap(), U256::from(1024u64)); - assert_eq!(U256::from_dec_str("115792089237316195423570985008687907853269984665640564039457584007913129639936"), Err(FromDecStrErr::InvalidLength)); - assert_eq!(U256::from_dec_str("0x11"), Err(FromDecStrErr::InvalidCharacter)); - } - - #[test] - fn display_uint() { - let s = "12345678987654321023456789"; - assert_eq!(format!("{}", U256::from_dec_str(s).unwrap()), s); - } - - #[test] - fn display_uint_zero() { - assert_eq!(format!("{}", U256::from(0)), "0"); - } - - #[test] - fn u512_multi_adds() { - let (result, _) = U512([0, 0, 0, 0, 0, 0, 0, 0]).overflowing_add(U512([0, 0, 0, 0, 0, 0, 0, 0])); - assert_eq!(result, U512([0, 0, 0, 0, 0, 0, 0, 0])); - - let (result, _) = U512([1, 0, 0, 0, 0, 0, 0, 1]).overflowing_add(U512([1, 0, 0, 0, 0, 0, 0, 1])); - assert_eq!(result, U512([2, 0, 0, 0, 0, 0, 0, 2])); - - let (result, _) = U512([0, 0, 0, 0, 0, 0, 0, 1]).overflowing_add(U512([0, 0, 0, 0, 0, 0, 0, 1])); - assert_eq!(result, U512([0, 0, 0, 0, 0, 0, 0, 2])); - - let (result, _) = U512([0, 0, 0, 0, 0, 0, 2, 1]).overflowing_add(U512([0, 0, 0, 0, 0, 0, 3, 1])); - assert_eq!(result, U512([0, 0, 0, 0, 0, 0, 5, 2])); - - let (result, _) = U512([1, 2, 3, 4, 5, 6, 7, 8]).overflowing_add(U512([9, 10, 11, 12, 13, 14, 15, 16])); - assert_eq!(result, U512([10, 12, 14, 16, 18, 20, 22, 24])); - - let (_, overflow) = U512([0, 0, 0, 0, 0, 0, 2, 1]).overflowing_add(U512([0, 0, 0, 0, 0, 0, 3, 1])); - assert!(!overflow); - - let (_, overflow) = U512([MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX]) - .overflowing_add(U512([MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX])); - assert!(overflow); - - let (_, overflow) = U512([0, 0, 0, 0, 0, 0, 0, MAX]) - .overflowing_add(U512([0, 0, 0, 0, 0, 0, 0, MAX])); - assert!(overflow); - - let (_, overflow) = U512([0, 0, 0, 0, 0, 0, 0, MAX]) - .overflowing_add(U512([0, 0, 0, 0, 0, 0, 0, 0])); - assert!(!overflow); - } - - #[test] - fn u256_multi_adds() { - let (result, _) = U256([0, 0, 0, 0]).overflowing_add(U256([0, 0, 0, 0])); - assert_eq!(result, U256([0, 0, 0, 0])); - - let (result, _) = U256([0, 0, 0, 1]).overflowing_add(U256([0, 0, 0, 1])); - assert_eq!(result, U256([0, 0, 0, 2])); - - let (result, overflow) = U256([0, 0, 2, 1]).overflowing_add(U256([0, 0, 3, 1])); - assert_eq!(result, U256([0, 0, 5, 2])); - assert!(!overflow); - - let (_, overflow) = U256([MAX, MAX, MAX, MAX]) - .overflowing_add(U256([MAX, MAX, MAX, MAX])); - assert!(overflow); - - let (_, overflow) = U256([0, 0, 0, MAX]).overflowing_add(U256([0, 0, 0, MAX])); - assert!(overflow); - } - - - #[test] - fn u256_multi_subs() { - let (result, _) = U256([0, 0, 0, 0]).overflowing_sub(U256([0, 0, 0, 0])); - assert_eq!(result, U256([0, 0, 0, 0])); - - let (result, _) = U256([0, 0, 0, 1]).overflowing_sub(U256([0, 0, 0, 1])); - assert_eq!(result, U256([0, 0, 0, 0])); - - let (_, overflow) = U256([0, 0, 2, 1]).overflowing_sub(U256([0, 0, 3, 1])); - assert!(overflow); - - let (result, overflow) = - U256([MAX, MAX, MAX, MAX]) - .overflowing_sub(U256([MAX/2, MAX/2, MAX/2, MAX/2])); - - assert!(!overflow); - assert_eq!(U256([MAX/2+1, MAX/2+1, MAX/2+1, MAX/2+1]), result); - - let (result, overflow) = U256([0, 0, 0, 1]).overflowing_sub(U256([0, 0, 1, 0])); - assert!(!overflow); - assert_eq!(U256([0, 0, MAX, 0]), result); - - let (result, overflow) = U256([0, 0, 0, 1]).overflowing_sub(U256([1, 0, 0, 0])); - assert!(!overflow); - assert_eq!(U256([MAX, MAX, MAX, 0]), result); - } - - #[test] - fn u512_multi_subs() { - let (result, _) = U512([0, 0, 0, 0, 0, 0, 0, 0]).overflowing_sub(U512([0, 0, 0, 0, 0, 0, 0, 0])); - assert_eq!(result, U512([0, 0, 0, 0, 0, 0, 0, 0])); - - let (result, _) = U512([10, 9, 8, 7, 6, 5, 4, 3]).overflowing_sub(U512([9, 8, 7, 6, 5, 4, 3, 2])); - assert_eq!(result, U512([1, 1, 1, 1, 1, 1, 1, 1])); - - let (_, overflow) = U512([10, 9, 8, 7, 6, 5, 4, 3]).overflowing_sub(U512([9, 8, 7, 6, 5, 4, 3, 2])); - assert!(!overflow); - - let (_, overflow) = U512([9, 8, 7, 6, 5, 4, 3, 2]).overflowing_sub(U512([10, 9, 8, 7, 6, 5, 4, 3])); - assert!(overflow); - } - - #[test] - fn u256_multi_carry_all() { - let (result, _) = U256([MAX, 0, 0, 0]).overflowing_mul(U256([MAX, 0, 0, 0])); - assert_eq!(U256([1, MAX-1, 0, 0]), result); - - let (result, _) = U256([0, MAX, 0, 0]).overflowing_mul(U256([MAX, 0, 0, 0])); - assert_eq!(U256([0, 1, MAX-1, 0]), result); - - let (result, _) = U256([MAX, MAX, 0, 0]).overflowing_mul(U256([MAX, 0, 0, 0])); - assert_eq!(U256([1, MAX, MAX-1, 0]), result); - - let (result, _) = U256([MAX, 0, 0, 0]).overflowing_mul(U256([MAX, MAX, 0, 0])); - assert_eq!(U256([1, MAX, MAX-1, 0]), result); - - let (result, _) = U256([MAX, MAX, 0, 0]) - .overflowing_mul(U256([MAX, MAX, 0, 0])); - assert_eq!(U256([1, 0, MAX-1, MAX]), result); - - let (result, _) = U256([MAX, 0, 0, 0]).overflowing_mul(U256([MAX, MAX, MAX, 0])); - assert_eq!(U256([1, MAX, MAX, MAX-1]), result); - - let (result, _) = U256([MAX, MAX, MAX, 0]).overflowing_mul(U256([MAX, 0, 0, 0])); - assert_eq!(U256([1, MAX, MAX, MAX-1]), result); - - let (result, _) = U256([MAX, 0, 0, 0]).overflowing_mul( - U256([MAX, MAX, MAX, MAX])); - assert_eq!(U256([1, MAX, MAX, MAX]), result); - - let (result, _) = U256([MAX, MAX, MAX, MAX]) - .overflowing_mul(U256([MAX, 0, 0, 0])); - assert_eq!(U256([1, MAX, MAX, MAX]), result); - - let (result, _) = U256([MAX, MAX, MAX, 0]) - .overflowing_mul(U256([MAX, MAX, 0, 0])); - assert_eq!(U256([1, 0, MAX, MAX-1]), result); - - let (result, _) = U256([MAX, MAX, 0, 0]) - .overflowing_mul(U256([MAX, MAX, MAX, 0])); - assert_eq!(U256([1, 0, MAX, MAX-1]), result); - - let (result, _) = U256([MAX, MAX, MAX, MAX]) - .overflowing_mul(U256([MAX, MAX, 0, 0])); - assert_eq!(U256([1, 0, MAX, MAX]), result); - - let (result, _) = U256([MAX, MAX, 0, 0]) - .overflowing_mul(U256([MAX, MAX, MAX, MAX])); - assert_eq!(U256([1, 0, MAX, MAX]), result); - - let (result, _) = U256([MAX, MAX, MAX, 0]) - .overflowing_mul(U256([MAX, MAX, MAX, 0])); - assert_eq!(U256([1, 0, 0, MAX-1]), result); - - let (result, _) = U256([MAX, MAX, MAX, 0]) - .overflowing_mul(U256([MAX, MAX, MAX, MAX])); - assert_eq!(U256([1, 0, 0, MAX]), result); - - let (result, _) = U256([MAX, MAX, MAX, MAX]) - .overflowing_mul(U256([MAX, MAX, MAX, 0])); - assert_eq!(U256([1, 0, 0, MAX]), result); - - let (result, _) = U256([0, 0, 0, MAX]).overflowing_mul(U256([0, 0, 0, MAX])); - assert_eq!(U256([0, 0, 0, 0]), result); - - let (result, _) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, MAX])); - assert_eq!(U256([0, 0, 0, MAX]), result); - - let (result, _) = U256([MAX, MAX, MAX, MAX]) - .overflowing_mul(U256([MAX, MAX, MAX, MAX])); - assert_eq!(U256([1, 0, 0, 0]), result); - } - - #[test] - fn u256_multi_muls() { - let (result, _) = U256([0, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, 0])); - assert_eq!(U256([0, 0, 0, 0]), result); - - let (result, _) = U256([1, 0, 0, 0]).overflowing_mul(U256([1, 0, 0, 0])); - assert_eq!(U256([1, 0, 0, 0]), result); - - let (result, _) = U256([5, 0, 0, 0]).overflowing_mul(U256([5, 0, 0, 0])); - assert_eq!(U256([25, 0, 0, 0]), result); - - let (result, _) = U256([0, 5, 0, 0]).overflowing_mul(U256([0, 5, 0, 0])); - assert_eq!(U256([0, 0, 25, 0]), result); - - let (result, _) = U256([0, 0, 0, 1]).overflowing_mul(U256([1, 0, 0, 0])); - assert_eq!(U256([0, 0, 0, 1]), result); - - let (result, _) = U256([0, 0, 0, 5]).overflowing_mul(U256([2, 0, 0, 0])); - assert_eq!(U256([0, 0, 0, 10]), result); - - let (result, _) = U256([0, 0, 1, 0]).overflowing_mul(U256([0, 5, 0, 0])); - assert_eq!(U256([0, 0, 0, 5]), result); - - let (result, _) = U256([0, 0, 8, 0]).overflowing_mul(U256([0, 0, 7, 0])); - assert_eq!(U256([0, 0, 0, 0]), result); - - let (result, _) = U256([2, 0, 0, 0]).overflowing_mul(U256([0, 5, 0, 0])); - assert_eq!(U256([0, 10, 0, 0]), result); - - let (result, _) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, MAX])); - assert_eq!(U256([0, 0, 0, MAX]), result); - } - - #[test] - fn u256_multi_muls_overflow() { - let (_, overflow) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, 0])); - assert!(!overflow); - - let (_, overflow) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, MAX])); - assert!(!overflow); - - let (_, overflow) = U256([0, 1, 0, 0]).overflowing_mul(U256([0, 0, 0, MAX])); - assert!(overflow); - - let (_, overflow) = U256([0, 1, 0, 0]).overflowing_mul(U256([0, 1, 0, 0])); - assert!(!overflow); - - let (_, overflow) = U256([0, 1, 0, MAX]).overflowing_mul(U256([0, 1, 0, MAX])); - assert!(overflow); - - let (_, overflow) = U256([0, MAX, 0, 0]).overflowing_mul(U256([0, MAX, 0, 0])); - assert!(!overflow); - - let (_, overflow) = U256([1, 0, 0, 0]).overflowing_mul(U256([10, 0, 0, 0])); - assert!(!overflow); - - let (_, overflow) = U256([2, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, MAX / 2])); - assert!(!overflow); - - let (_, overflow) = U256([0, 0, 8, 0]).overflowing_mul(U256([0, 0, 7, 0])); - assert!(overflow); - } - - #[test] - fn big_endian() { - let source = U256([1, 0, 0, 0]); - let mut target = vec![0u8; 32]; - - assert_eq!(source, U256::from(1)); - - source.to_big_endian(&mut target); - assert_eq!( - vec![0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8], - target); - - let source = U256([512, 0, 0, 0]); - let mut target = vec![0u8; 32]; - - source.to_big_endian(&mut target); - assert_eq!( - vec![0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8], - target); - - let source = U256([0, 512, 0, 0]); - let mut target = vec![0u8; 32]; - - source.to_big_endian(&mut target); - assert_eq!( - vec![0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8], - target); - - let source = U256::from_str("0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20").unwrap(); - source.to_big_endian(&mut target); - assert_eq!( - vec![0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, - 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20], - target); - } - - #[test] - #[cfg_attr(feature="dev", allow(cyclomatic_complexity))] - fn u256_multi_full_mul() { - let result = U256([0, 0, 0, 0]).full_mul(U256([0, 0, 0, 0])); - assert_eq!(U512([0, 0, 0, 0, 0, 0, 0, 0]), result); - - let result = U256([1, 0, 0, 0]).full_mul(U256([1, 0, 0, 0])); - assert_eq!(U512([1, 0, 0, 0, 0, 0, 0, 0]), result); - - let result = U256([5, 0, 0, 0]).full_mul(U256([5, 0, 0, 0])); - assert_eq!(U512([25, 0, 0, 0, 0, 0, 0, 0]), result); - - let result = U256([0, 5, 0, 0]).full_mul(U256([0, 5, 0, 0])); - assert_eq!(U512([0, 0, 25, 0, 0, 0, 0, 0]), result); - - let result = U256([0, 0, 0, 4]).full_mul(U256([4, 0, 0, 0])); - assert_eq!(U512([0, 0, 0, 16, 0, 0, 0, 0]), result); - - let result = U256([0, 0, 0, 5]).full_mul(U256([2, 0, 0, 0])); - assert_eq!(U512([0, 0, 0, 10, 0, 0, 0, 0]), result); - - let result = U256([0, 0, 2, 0]).full_mul(U256([0, 5, 0, 0])); - assert_eq!(U512([0, 0, 0, 10, 0, 0, 0, 0]), result); - - let result = U256([0, 3, 0, 0]).full_mul(U256([0, 0, 3, 0])); - assert_eq!(U512([0, 0, 0, 9, 0, 0, 0, 0]), result); - - let result = U256([0, 0, 8, 0]).full_mul(U256([0, 0, 6, 0])); - assert_eq!(U512([0, 0, 0, 0, 48, 0, 0, 0]), result); - - let result = U256([9, 0, 0, 0]).full_mul(U256([0, 3, 0, 0])); - assert_eq!(U512([0, 27, 0, 0, 0, 0, 0, 0]), result); - - let result = U256([MAX, 0, 0, 0]).full_mul(U256([MAX, 0, 0, 0])); - assert_eq!(U512([1, MAX-1, 0, 0, 0, 0, 0, 0]), result); - - let result = U256([0, MAX, 0, 0]).full_mul(U256([MAX, 0, 0, 0])); - assert_eq!(U512([0, 1, MAX-1, 0, 0, 0, 0, 0]), result); - - let result = U256([MAX, MAX, 0, 0]).full_mul(U256([MAX, 0, 0, 0])); - assert_eq!(U512([1, MAX, MAX-1, 0, 0, 0, 0, 0]), result); - - let result = U256([MAX, 0, 0, 0]).full_mul(U256([MAX, MAX, 0, 0])); - assert_eq!(U512([1, MAX, MAX-1, 0, 0, 0, 0, 0]), result); - - let result = U256([MAX, MAX, 0, 0]).full_mul(U256([MAX, MAX, 0, 0])); - assert_eq!(U512([1, 0, MAX-1, MAX, 0, 0, 0, 0]), result); - - let result = U256([MAX, 0, 0, 0]).full_mul(U256([MAX, MAX, MAX, 0])); - assert_eq!(U512([1, MAX, MAX, MAX-1, 0, 0, 0, 0]), result); - - let result = U256([MAX, MAX, MAX, 0]).full_mul(U256([MAX, 0, 0, 0])); - assert_eq!(U512([1, MAX, MAX, MAX-1, 0, 0, 0, 0]), result); - - let result = U256([MAX, 0, 0, 0]).full_mul(U256([MAX, MAX, MAX, MAX])); - assert_eq!(U512([1, MAX, MAX, MAX, MAX-1, 0, 0, 0]), result); - - let result = U256([MAX, MAX, MAX, MAX]).full_mul(U256([MAX, 0, 0, 0])); - assert_eq!(U512([1, MAX, MAX, MAX, MAX-1, 0, 0, 0]), result); - - let result = U256([MAX, MAX, MAX, 0]).full_mul(U256([MAX, MAX, 0, 0])); - assert_eq!(U512([1, 0, MAX, MAX-1, MAX, 0, 0, 0]), result); - - let result = U256([MAX, MAX, 0, 0]).full_mul(U256([MAX, MAX, MAX, 0])); - assert_eq!(U512([1, 0, MAX, MAX-1, MAX, 0, 0, 0]), result); - - let result = U256([MAX, MAX, MAX, MAX]).full_mul(U256([MAX, MAX, 0, 0])); - assert_eq!(U512([1, 0, MAX, MAX, MAX-1, MAX, 0, 0]), result); - - let result = U256([MAX, MAX, 0, 0]).full_mul(U256([MAX, MAX, MAX, MAX])); - assert_eq!(U512([1, 0, MAX, MAX, MAX-1, MAX, 0, 0]), result); - - let result = U256([MAX, MAX, MAX, 0]).full_mul(U256([MAX, MAX, MAX, 0])); - assert_eq!(U512([1, 0, 0, MAX-1, MAX, MAX, 0, 0]), result); - - let result = U256([MAX, MAX, MAX, 0]).full_mul(U256([MAX, MAX, MAX, MAX])); - assert_eq!(U512([1, 0, 0, MAX, MAX-1, MAX, MAX, 0]), result); - - let result = U256([MAX, MAX, MAX, MAX]).full_mul(U256([MAX, MAX, MAX, 0])); - assert_eq!(U512([1, 0, 0, MAX, MAX-1, MAX, MAX, 0]), result); - - let result = U256([MAX, MAX, MAX, MAX]).full_mul(U256([MAX, MAX, MAX, MAX])); - assert_eq!(U512([1, 0, 0, 0, MAX-1, MAX, MAX, MAX]), result); - - let result = U256([0, 0, 0, MAX]).full_mul(U256([0, 0, 0, MAX])); - assert_eq!(U512([0, 0, 0, 0, 0, 0, 1, MAX-1]), result); - - let result = U256([1, 0, 0, 0]).full_mul(U256([0, 0, 0, MAX])); - assert_eq!(U512([0, 0, 0, MAX, 0, 0, 0, 0]), result); - - let result = U256([1, 2, 3, 4]).full_mul(U256([5, 0, 0, 0])); - assert_eq!(U512([5, 10, 15, 20, 0, 0, 0, 0]), result); - - let result = U256([1, 2, 3, 4]).full_mul(U256([0, 6, 0, 0])); - assert_eq!(U512([0, 6, 12, 18, 24, 0, 0, 0]), result); - - let result = U256([1, 2, 3, 4]).full_mul(U256([0, 0, 7, 0])); - assert_eq!(U512([0, 0, 7, 14, 21, 28, 0, 0]), result); - - let result = U256([1, 2, 3, 4]).full_mul(U256([0, 0, 0, 8])); - assert_eq!(U512([0, 0, 0, 8, 16, 24, 32, 0]), result); - - let result = U256([1, 2, 3, 4]).full_mul(U256([5, 6, 7, 8])); - assert_eq!(U512([5, 16, 34, 60, 61, 52, 32, 0]), result); - } - - #[test] - fn u256_multi_muls2() { - - let (result, _) = U256([0, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, 0])); - assert_eq!(U256([0, 0, 0, 0]), result); - - let (result, _) = U256([1, 0, 0, 0]).overflowing_mul(U256([1, 0, 0, 0])); - assert_eq!(U256([1, 0, 0, 0]), result); - - let (result, _) = U256([5, 0, 0, 0]).overflowing_mul(U256([5, 0, 0, 0])); - assert_eq!(U256([25, 0, 0, 0]), result); - - let (result, _) = U256([0, 5, 0, 0]).overflowing_mul(U256([0, 5, 0, 0])); - assert_eq!(U256([0, 0, 25, 0]), result); - - let (result, _) = U256([0, 0, 0, 1]).overflowing_mul(U256([1, 0, 0, 0])); - assert_eq!(U256([0, 0, 0, 1]), result); - - let (result, _) = U256([0, 0, 0, 5]).overflowing_mul(U256([2, 0, 0, 0])); - assert_eq!(U256([0, 0, 0, 10]), result); - - let (result, _) = U256([0, 0, 1, 0]).overflowing_mul(U256([0, 5, 0, 0])); - assert_eq!(U256([0, 0, 0, 5]), result); - - let (result, _) = U256([0, 0, 8, 0]).overflowing_mul(U256([0, 0, 7, 0])); - assert_eq!(U256([0, 0, 0, 0]), result); - - let (result, _) = U256([2, 0, 0, 0]).overflowing_mul(U256([0, 5, 0, 0])); - assert_eq!(U256([0, 10, 0, 0]), result); - - let (result, _) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, u64::max_value()])); - assert_eq!(U256([0, 0, 0, u64::max_value()]), result); - - let x1: U256 = "0000000000000000000000000000000000000000000000000000012365124623".into(); - let x2sqr_right: U256 = "000000000000000000000000000000000000000000014baeef72e0378e2328c9".into(); - let x1sqr = x1 * x1; - assert_eq!(x2sqr_right, x1sqr); - - let x1cube = x1sqr * x1; - let x1cube_right: U256 = "0000000000000000000000000000000001798acde139361466f712813717897b".into(); - assert_eq!(x1cube_right, x1cube); - - let x1quad = x1cube * x1; - let x1quad_right: U256 = "000000000000000000000001adbdd6bd6ff027485484b97f8a6a4c7129756dd1".into(); - assert_eq!(x1quad_right, x1quad); - - let x1penta = x1quad * x1; - let x1penta_right: U256 = "00000000000001e92875ac24be246e1c57e0507e8c46cc8d233b77f6f4c72993".into(); - assert_eq!(x1penta_right, x1penta); - - let x1septima = x1penta * x1; - let x1septima_right: U256 = "00022cca1da3f6e5722b7d3cc5bbfb486465ebc5a708dd293042f932d7eee119".into(); - assert_eq!(x1septima_right, x1septima); - } - - #[test] - fn example() { - let mut val: U256 = 1023.into(); - for _ in 0..200 { val = val * 2.into() } - assert_eq!(&format!("{}", val), "1643897619276947051879427220465009342380213662639797070513307648"); - } - - #[test] - fn little_endian() { - let number: U256 = "00022cca1da3f6e5722b7d3cc5bbfb486465ebc5a708dd293042f932d7eee119".into(); - let expected = [ - 0x19, 0xe1, 0xee, 0xd7, - 0x32, 0xf9, 0x42, 0x30, - 0x29, 0xdd, 0x08, 0xa7, - 0xc5, 0xeb, 0x65, 0x64, - 0x48, 0xfb, 0xbb, 0xc5, - 0x3c, 0x7d, 0x2b, 0x72, - 0xe5, 0xf6, 0xa3, 0x1d, - 0xca, 0x2c, 0x02, 0x00 - ]; - let mut result = [0u8; 32]; - number.to_little_endian(&mut result); - assert_eq!(expected, result); - } - - #[test] - fn slice_roundtrip() { - let raw = [ - 1u8, 2, 3, 5, 7, 11, 13, 17, - 19, 23, 29, 31, 37, 41, 43, 47, - 53, 59, 61, 67, 71, 73, 79, 83, - 89, 97, 101, 103, 107, 109, 113, 127 - ]; - - let u256: U256 = (&raw[..]).into(); - - let mut new_raw = [0u8; 32]; - - u256.to_big_endian(&mut new_raw); - - assert_eq!(&raw, &new_raw); - } - - #[test] - fn slice_roundtrip_le() { - let raw = [ - 1u8, 2, 3, 5, 7, 11, 13, 17, - 19, 23, 29, 31, 37, 41, 43, 47, - 53, 59, 61, 67, 71, 73, 79, 83, - 89, 97, 101, 103, 107, 109, 113, 127 - ]; - - let u256 = U256::from_little_endian(&raw[..]); - - let mut new_raw = [0u8; 32]; - - u256.to_little_endian(&mut new_raw); - - assert_eq!(&raw, &new_raw); - } - - #[test] - fn slice_roundtrip_le2() { - let raw = [ - 2, 3, 5, 7, 11, 13, 17, - 19, 23, 29, 31, 37, 41, 43, 47, - 53, 59, 61, 67, 71, 73, 79, 83, - 89, 97, 101, 103, 107, 109, 113, 127 - ]; - - let u256 = U256::from_little_endian(&raw[..]); - - let mut new_raw = [0u8; 32]; - - u256.to_little_endian(&mut new_raw); - - assert_eq!(&raw, &new_raw[..31]); - } - - #[test] - fn modular_inverse() { - let modulo = U256::from_dec_str("115792089210356248762697446949407573530086143415290314195533631308867097853951").unwrap(); - let number = U256::from_dec_str("48439561293906451759052585252797914202762949526041747995844080717082404635286").unwrap(); - let mod_inv = number.mod_inverse(modulo); - assert_eq!( - mod_inv, - U256::from_dec_str("101489101214698129329668954935570020318890663581888936938143465331216272806456").unwrap() - ) - } - - #[test] - fn fixed_arrays_roundtrip() { - let raw: U256 = "7094875209347850239487502394881".into(); - let array: [u8; 32] = raw.into(); - let new_raw = array.into(); - - assert_eq!(raw, new_raw); - } - - #[test] - fn from_little_endian() { - let source: [u8; 32] = [ - 1, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - ]; - - let number = U256::from_little_endian(&source[..]); - - assert_eq!(U256::from(1), number); - } - - #[test] - fn from_big_endian() { - let source: [u8; 32] = [ - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1, - ]; - - let number = U256::from_big_endian(&source[..]); - - assert_eq!(U256::from(1), number); - } - - #[test] - fn leading_zeros() { - assert_eq!(U256::from("000000000000000000000001adbdd6bd6ff027485484b97f8a6a4c7129756dd1").leading_zeros(), 95); - assert_eq!(U256::from("f00000000000000000000001adbdd6bd6ff027485484b97f8a6a4c7129756dd1").leading_zeros(), 0); - assert_eq!(U256::from("0000000000000000000000000000000000000000000000000000000000000001").leading_zeros(), 255); - assert_eq!(U256::from("0000000000000000000000000000000000000000000000000000000000000000").leading_zeros(), 256); - } - - #[test] - fn trailing_zeros() { - assert_eq!(U256::from("1adbdd6bd6ff027485484b97f8a6a4c7129756dd100000000000000000000000").trailing_zeros(), 92); - assert_eq!(U256::from("1adbdd6bd6ff027485484b97f8a6a4c7129756dd10000000000000000000000f").trailing_zeros(), 0); - assert_eq!(U256::from("8000000000000000000000000000000000000000000000000000000000000000").trailing_zeros(), 255); - assert_eq!(U256::from("0000000000000000000000000000000000000000000000000000000000000000").trailing_zeros(), 256); - } - - mod laws { - use uint::{U128, U256, U512}; - - macro_rules! uint_arbitrary { - ($uint:ty, $n_bytes:tt) => { - impl ::quickcheck::Arbitrary for $uint { - fn arbitrary(g: &mut G) -> Self { - let mut res = [0u8; $n_bytes]; - - let p = g.next_f64(); - let range = - if p < 0.1 { - $n_bytes - } else if p < 0.2 { - $n_bytes / 2 - } else { - $n_bytes / 5 - }; - - let size = g.gen_range(0, range); - g.fill_bytes(&mut res[..size]); - - Self::from(res) - } - } - } - } - - uint_arbitrary!(U128, 16); - uint_arbitrary!(U256, 32); - uint_arbitrary!(U512, 64); - - macro_rules! o_try { - ($o:expr) => {{ - let (val, o) = $o; - if o { return ::quickcheck::TestResult::discard(); } - val - }}; - } - - macro_rules! try_expr { - ($any:ident) => { $any }; - ($first:ident + $($second:tt)*) => { - o_try!($first.overflowing_add(try_expr!($($second)*))) - }; - (($($anything:tt)*)) => { - try_expr!($($anything)*) - }; - ($first:ident * $($second:tt)*) => { - o_try!($first.overflowing_mul(try_expr!($($second)*))) - }; - (($($first:tt)*) + $($rest:tt)*) => {{ - let first = try_expr!($($first)*); - try_expr!(first + $($rest)*) - }}; - (($($first:tt)*) * $($rest:tt)*) => {{ - let first = try_expr!($($first)*); - try_expr!(first * $($rest)*) - }}; - } - - macro_rules! uint_laws { - ($mod_name:ident, $uint_ty:ident) => { - mod $mod_name { - use quickcheck::TestResult; - use uint::*; - - quickcheck! { - fn associative_add(x: $uint_ty, y: $uint_ty, z: $uint_ty) -> TestResult { - TestResult::from_bool( - try_expr!((x + y) + z) == - try_expr!(x + (y + z)) - ) - } - } - - quickcheck! { - fn associative_mul(x: $uint_ty, y: $uint_ty, z: $uint_ty) -> TestResult { - TestResult::from_bool( - try_expr!((x * y) * z) == try_expr!(x * (y * z)) - ) - } - } - - quickcheck! { - fn commutative_add(x: $uint_ty, y: $uint_ty) -> TestResult { - TestResult::from_bool( - try_expr!(x + y) == try_expr!(y + x) - ) - } - } - - quickcheck! { - fn commutative_mul(x: $uint_ty, y: $uint_ty) -> TestResult { - TestResult::from_bool( - try_expr!(x * y) == try_expr!(y * x) - ) - } - } - - quickcheck! { - fn identity_add(x: $uint_ty) -> bool { - x + $uint_ty::zero() == x - } - } - - quickcheck! { - fn identity_mul(x: $uint_ty) -> bool { - x * $uint_ty::one() == x - } - } - - quickcheck! { - fn identity_div(x: $uint_ty) -> bool { - x / $uint_ty::one() == x - } - } - - quickcheck! { - fn absorbing_rem(x: $uint_ty) -> bool { - x % $uint_ty::one() == $uint_ty::zero() - } - } - - quickcheck! { - fn absorbing_sub(x: $uint_ty) -> bool { - x - x == $uint_ty::zero() - } - } - - quickcheck! { - fn absorbing_mul(x: $uint_ty) -> bool { - x * $uint_ty::zero() == $uint_ty::zero() - } - } - - quickcheck! { - fn distributive_mul_over_add(x: $uint_ty, y: $uint_ty, z: $uint_ty) -> TestResult { - TestResult::from_bool( - try_expr!(x * (y + z)) == try_expr!((x * y) + (x * z)) && - try_expr!((x + y) * z) == try_expr!((x * z) + (y * z)) - ) - } - } - - quickcheck! { - fn pow_mul(x: $uint_ty) -> TestResult { - let (p2, o) = x.overflowing_pow($uint_ty::from(2)); - if o { return TestResult::discard(); } - let (p3, o) = x.overflowing_pow($uint_ty::from(3)); - if o { return TestResult::discard(); } - - TestResult::from_bool( - p2 == x * x && p3 == x * x * x - ) - } - } - - quickcheck! { - fn add_increases(x: $uint_ty, y: $uint_ty) -> TestResult { - if y.is_zero() { - return TestResult::discard(); - } - - TestResult::from_bool( - try_expr!(x + y) > x - ) - } - } - - quickcheck! { - fn mul_increases(x: $uint_ty, y: $uint_ty) -> TestResult { - if y.is_zero() { - return TestResult::discard(); - } - - TestResult::from_bool( - try_expr!(x * y) >= x - ) - } - } - - quickcheck! { - fn div_decreases_dividend(x: $uint_ty, y: $uint_ty) -> TestResult { - if y.is_zero() { - return TestResult::discard(); - } - - TestResult::from_bool( - x / y <= x - ) - } - } - - quickcheck! { - fn rem_decreases_divisor(x: $uint_ty, y: $uint_ty) -> TestResult { - if y.is_zero() { - return TestResult::discard(); - } - - TestResult::from_bool( - x % y < y - ) - } - } - } - } - } - - uint_laws!(u128, U128); - uint_laws!(u256, U256); - uint_laws!(u512, U512); - } -} diff --git a/crux-mir/lib/byteorder/io.rs b/crux-mir/lib/byteorder/io.rs deleted file mode 100644 index ed6a84845..000000000 --- a/crux-mir/lib/byteorder/io.rs +++ /dev/null @@ -1,1605 +0,0 @@ -use std::io::{self, Result}; -use std::slice; - -use ByteOrder; - -/// Extends [`Read`] with methods for reading numbers. (For `std::io`.) -/// -/// Most of the methods defined here have an unconstrained type parameter that -/// must be explicitly instantiated. Typically, it is instantiated with either -/// the [`BigEndian`] or [`LittleEndian`] types defined in this crate. -/// -/// # Examples -/// -/// Read unsigned 16 bit big-endian integers from a [`Read`]: -/// -/// ```rust -/// use std::io::Cursor; -/// use byteorder::{BigEndian, ReadBytesExt}; -/// -/// let mut rdr = Cursor::new(vec![2, 5, 3, 0]); -/// assert_eq!(517, rdr.read_u16::().unwrap()); -/// assert_eq!(768, rdr.read_u16::().unwrap()); -/// ``` -/// -/// [`BigEndian`]: enum.BigEndian.html -/// [`LittleEndian`]: enum.LittleEndian.html -/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html -pub trait ReadBytesExt: io::Read { - /// Reads an unsigned 8 bit integer from the underlying reader. - /// - /// Note that since this reads a single byte, no byte order conversions - /// are used. It is included for completeness. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read unsigned 8 bit integers from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::ReadBytesExt; - /// - /// let mut rdr = Cursor::new(vec![2, 5]); - /// assert_eq!(2, rdr.read_u8().unwrap()); - /// assert_eq!(5, rdr.read_u8().unwrap()); - /// ``` - #[inline] - fn read_u8(&mut self) -> Result { - let mut buf = [0; 1]; - try!(self.read_exact(&mut buf)); - Ok(buf[0]) - } - - /// Reads a signed 8 bit integer from the underlying reader. - /// - /// Note that since this reads a single byte, no byte order conversions - /// are used. It is included for completeness. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read signed 8 bit integers from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::ReadBytesExt; - /// - /// let mut rdr = Cursor::new(vec![0x02, 0xfb]); - /// assert_eq!(2, rdr.read_i8().unwrap()); - /// assert_eq!(-5, rdr.read_i8().unwrap()); - /// ``` - #[inline] - fn read_i8(&mut self) -> Result { - let mut buf = [0; 1]; - try!(self.read_exact(&mut buf)); - Ok(buf[0] as i8) - } - - /// Reads an unsigned 16 bit integer from the underlying reader. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read unsigned 16 bit big-endian integers from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![2, 5, 3, 0]); - /// assert_eq!(517, rdr.read_u16::().unwrap()); - /// assert_eq!(768, rdr.read_u16::().unwrap()); - /// ``` - #[inline] - fn read_u16(&mut self) -> Result { - let mut buf = [0; 2]; - try!(self.read_exact(&mut buf)); - Ok(T::read_u16(&buf)) - } - - /// Reads a signed 16 bit integer from the underlying reader. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read signed 16 bit big-endian integers from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![0x00, 0xc1, 0xff, 0x7c]); - /// assert_eq!(193, rdr.read_i16::().unwrap()); - /// assert_eq!(-132, rdr.read_i16::().unwrap()); - /// ``` - #[inline] - fn read_i16(&mut self) -> Result { - let mut buf = [0; 2]; - try!(self.read_exact(&mut buf)); - Ok(T::read_i16(&buf)) - } - - /// Reads an unsigned 24 bit integer from the underlying reader. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read unsigned 24 bit big-endian integers from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![0x00, 0x01, 0x0b]); - /// assert_eq!(267, rdr.read_u24::().unwrap()); - /// ``` - #[inline] - fn read_u24(&mut self) -> Result { - let mut buf = [0; 3]; - try!(self.read_exact(&mut buf)); - Ok(T::read_u24(&buf)) - } - - /// Reads a signed 24 bit integer from the underlying reader. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read signed 24 bit big-endian integers from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![0xff, 0x7a, 0x33]); - /// assert_eq!(-34253, rdr.read_i24::().unwrap()); - /// ``` - #[inline] - fn read_i24(&mut self) -> Result { - let mut buf = [0; 3]; - try!(self.read_exact(&mut buf)); - Ok(T::read_i24(&buf)) - } - - /// Reads an unsigned 32 bit integer from the underlying reader. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read unsigned 32 bit big-endian integers from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![0x00, 0x00, 0x01, 0x0b]); - /// assert_eq!(267, rdr.read_u32::().unwrap()); - /// ``` - #[inline] - fn read_u32(&mut self) -> Result { - let mut buf = [0; 4]; - try!(self.read_exact(&mut buf)); - Ok(T::read_u32(&buf)) - } - - /// Reads a signed 32 bit integer from the underlying reader. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read signed 32 bit big-endian integers from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![0xff, 0xff, 0x7a, 0x33]); - /// assert_eq!(-34253, rdr.read_i32::().unwrap()); - /// ``` - #[inline] - fn read_i32(&mut self) -> Result { - let mut buf = [0; 4]; - try!(self.read_exact(&mut buf)); - Ok(T::read_i32(&buf)) - } - - /// Reads an unsigned 48 bit integer from the underlying reader. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read unsigned 48 bit big-endian integers from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![0xb6, 0x71, 0x6b, 0xdc, 0x2b, 0x31]); - /// assert_eq!(200598257150769, rdr.read_u48::().unwrap()); - /// ``` - #[inline] - fn read_u48(&mut self) -> Result { - let mut buf = [0; 6]; - try!(self.read_exact(&mut buf)); - Ok(T::read_u48(&buf)) - } - - /// Reads a signed 48 bit integer from the underlying reader. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read signed 48 bit big-endian integers from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![0x9d, 0x71, 0xab, 0xe7, 0x97, 0x8f]); - /// assert_eq!(-108363435763825, rdr.read_i48::().unwrap()); - /// ``` - #[inline] - fn read_i48(&mut self) -> Result { - let mut buf = [0; 6]; - try!(self.read_exact(&mut buf)); - Ok(T::read_i48(&buf)) - } - - /// Reads an unsigned 64 bit integer from the underlying reader. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read an unsigned 64 bit big-endian integer from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![0x00, 0x03, 0x43, 0x95, 0x4d, 0x60, 0x86, 0x83]); - /// assert_eq!(918733457491587, rdr.read_u64::().unwrap()); - /// ``` - #[inline] - fn read_u64(&mut self) -> Result { - let mut buf = [0; 8]; - try!(self.read_exact(&mut buf)); - Ok(T::read_u64(&buf)) - } - - /// Reads a signed 64 bit integer from the underlying reader. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read a signed 64 bit big-endian integer from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![0x80, 0, 0, 0, 0, 0, 0, 0]); - /// assert_eq!(i64::min_value(), rdr.read_i64::().unwrap()); - /// ``` - #[inline] - fn read_i64(&mut self) -> Result { - let mut buf = [0; 8]; - try!(self.read_exact(&mut buf)); - Ok(T::read_i64(&buf)) - } - - /// Reads an unsigned 128 bit integer from the underlying reader. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read an unsigned 128 bit big-endian integer from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![ - /// 0x00, 0x03, 0x43, 0x95, 0x4d, 0x60, 0x86, 0x83, - /// 0x00, 0x03, 0x43, 0x95, 0x4d, 0x60, 0x86, 0x83 - /// ]); - /// assert_eq!(16947640962301618749969007319746179, rdr.read_u128::().unwrap()); - /// ``` - #[cfg(byteorder_i128)] - #[inline] - fn read_u128(&mut self) -> Result { - let mut buf = [0; 16]; - try!(self.read_exact(&mut buf)); - Ok(T::read_u128(&buf)) - } - - /// Reads a signed 128 bit integer from the underlying reader. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read a signed 128 bit big-endian integer from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); - /// assert_eq!(i128::min_value(), rdr.read_i128::().unwrap()); - /// ``` - #[cfg(byteorder_i128)] - #[inline] - fn read_i128(&mut self) -> Result { - let mut buf = [0; 16]; - try!(self.read_exact(&mut buf)); - Ok(T::read_i128(&buf)) - } - - /// Reads an unsigned n-bytes integer from the underlying reader. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read an unsigned n-byte big-endian integer from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![0x80, 0x74, 0xfa]); - /// assert_eq!(8418554, rdr.read_uint::(3).unwrap()); - #[inline] - fn read_uint(&mut self, nbytes: usize) -> Result { - let mut buf = [0; 8]; - try!(self.read_exact(&mut buf[..nbytes])); - Ok(T::read_uint(&buf[..nbytes], nbytes)) - } - - /// Reads a signed n-bytes integer from the underlying reader. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read an unsigned n-byte big-endian integer from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![0xc1, 0xff, 0x7c]); - /// assert_eq!(-4063364, rdr.read_int::(3).unwrap()); - #[inline] - fn read_int(&mut self, nbytes: usize) -> Result { - let mut buf = [0; 8]; - try!(self.read_exact(&mut buf[..nbytes])); - Ok(T::read_int(&buf[..nbytes], nbytes)) - } - - /// Reads an unsigned n-bytes integer from the underlying reader. - #[cfg(byteorder_i128)] - #[inline] - fn read_uint128(&mut self, nbytes: usize) -> Result { - let mut buf = [0; 16]; - try!(self.read_exact(&mut buf[..nbytes])); - Ok(T::read_uint128(&buf[..nbytes], nbytes)) - } - - /// Reads a signed n-bytes integer from the underlying reader. - #[cfg(byteorder_i128)] - #[inline] - fn read_int128(&mut self, nbytes: usize) -> Result { - let mut buf = [0; 16]; - try!(self.read_exact(&mut buf[..nbytes])); - Ok(T::read_int128(&buf[..nbytes], nbytes)) - } - - /// Reads a IEEE754 single-precision (4 bytes) floating point number from - /// the underlying reader. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read a big-endian single-precision floating point number from a `Read`: - /// - /// ```rust - /// use std::f32; - /// use std::io::Cursor; - /// - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![ - /// 0x40, 0x49, 0x0f, 0xdb, - /// ]); - /// assert_eq!(f32::consts::PI, rdr.read_f32::().unwrap()); - /// ``` - #[inline] - fn read_f32(&mut self) -> Result { - let mut buf = [0; 4]; - try!(self.read_exact(&mut buf)); - Ok(T::read_f32(&buf)) - } - - /// Reads a IEEE754 double-precision (8 bytes) floating point number from - /// the underlying reader. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read a big-endian double-precision floating point number from a `Read`: - /// - /// ```rust - /// use std::f64; - /// use std::io::Cursor; - /// - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![ - /// 0x40, 0x09, 0x21, 0xfb, 0x54, 0x44, 0x2d, 0x18, - /// ]); - /// assert_eq!(f64::consts::PI, rdr.read_f64::().unwrap()); - /// ``` - #[inline] - fn read_f64(&mut self) -> Result { - let mut buf = [0; 8]; - try!(self.read_exact(&mut buf)); - Ok(T::read_f64(&buf)) - } - - /// Reads a sequence of unsigned 16 bit integers from the underlying - /// reader. - /// - /// The given buffer is either filled completely or an error is returned. - /// If an error is returned, the contents of `dst` are unspecified. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read a sequence of unsigned 16 bit big-endian integers from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![2, 5, 3, 0]); - /// let mut dst = [0; 2]; - /// rdr.read_u16_into::(&mut dst).unwrap(); - /// assert_eq!([517, 768], dst); - /// ``` - #[inline] - fn read_u16_into(&mut self, dst: &mut [u16]) -> Result<()> { - { - let buf = unsafe { slice_to_u8_mut(dst) }; - try!(self.read_exact(buf)); - } - T::from_slice_u16(dst); - Ok(()) - } - - /// Reads a sequence of unsigned 32 bit integers from the underlying - /// reader. - /// - /// The given buffer is either filled completely or an error is returned. - /// If an error is returned, the contents of `dst` are unspecified. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read a sequence of unsigned 32 bit big-endian integers from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![0, 0, 2, 5, 0, 0, 3, 0]); - /// let mut dst = [0; 2]; - /// rdr.read_u32_into::(&mut dst).unwrap(); - /// assert_eq!([517, 768], dst); - /// ``` - #[inline] - fn read_u32_into(&mut self, dst: &mut [u32]) -> Result<()> { - { - let buf = unsafe { slice_to_u8_mut(dst) }; - try!(self.read_exact(buf)); - } - T::from_slice_u32(dst); - Ok(()) - } - - /// Reads a sequence of unsigned 64 bit integers from the underlying - /// reader. - /// - /// The given buffer is either filled completely or an error is returned. - /// If an error is returned, the contents of `dst` are unspecified. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read a sequence of unsigned 64 bit big-endian integers from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![ - /// 0, 0, 0, 0, 0, 0, 2, 5, - /// 0, 0, 0, 0, 0, 0, 3, 0, - /// ]); - /// let mut dst = [0; 2]; - /// rdr.read_u64_into::(&mut dst).unwrap(); - /// assert_eq!([517, 768], dst); - /// ``` - #[inline] - fn read_u64_into(&mut self, dst: &mut [u64]) -> Result<()> { - { - let buf = unsafe { slice_to_u8_mut(dst) }; - try!(self.read_exact(buf)); - } - T::from_slice_u64(dst); - Ok(()) - } - - /// Reads a sequence of unsigned 128 bit integers from the underlying - /// reader. - /// - /// The given buffer is either filled completely or an error is returned. - /// If an error is returned, the contents of `dst` are unspecified. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read a sequence of unsigned 128 bit big-endian integers from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![ - /// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, - /// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, - /// ]); - /// let mut dst = [0; 2]; - /// rdr.read_u128_into::(&mut dst).unwrap(); - /// assert_eq!([517, 768], dst); - /// ``` - #[cfg(byteorder_i128)] - #[inline] - fn read_u128_into( - &mut self, - dst: &mut [u128], - ) -> Result<()> { - { - let buf = unsafe { slice_to_u8_mut(dst) }; - try!(self.read_exact(buf)); - } - T::from_slice_u128(dst); - Ok(()) - } - - /// Reads a sequence of signed 8 bit integers from the underlying reader. - /// - /// The given buffer is either filled completely or an error is returned. - /// If an error is returned, the contents of `dst` are unspecified. - /// - /// Note that since each `i8` is a single byte, no byte order conversions - /// are used. This method is included because it provides a safe, simple - /// way for the caller to read into a `&mut [i8]` buffer. (Without this - /// method, the caller would have to either use `unsafe` code or convert - /// each byte to `i8` individually.) - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read a sequence of signed 8 bit integers from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![2, 251, 3]); - /// let mut dst = [0; 3]; - /// rdr.read_i8_into(&mut dst).unwrap(); - /// assert_eq!([2, -5, 3], dst); - /// ``` - #[inline] - fn read_i8_into(&mut self, dst: &mut [i8]) -> Result<()> { - let buf = unsafe { slice_to_u8_mut(dst) }; - self.read_exact(buf) - } - - /// Reads a sequence of signed 16 bit integers from the underlying - /// reader. - /// - /// The given buffer is either filled completely or an error is returned. - /// If an error is returned, the contents of `dst` are unspecified. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read a sequence of signed 16 bit big-endian integers from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![2, 5, 3, 0]); - /// let mut dst = [0; 2]; - /// rdr.read_i16_into::(&mut dst).unwrap(); - /// assert_eq!([517, 768], dst); - /// ``` - #[inline] - fn read_i16_into(&mut self, dst: &mut [i16]) -> Result<()> { - { - let buf = unsafe { slice_to_u8_mut(dst) }; - try!(self.read_exact(buf)); - } - T::from_slice_i16(dst); - Ok(()) - } - - /// Reads a sequence of signed 32 bit integers from the underlying - /// reader. - /// - /// The given buffer is either filled completely or an error is returned. - /// If an error is returned, the contents of `dst` are unspecified. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read a sequence of signed 32 bit big-endian integers from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![0, 0, 2, 5, 0, 0, 3, 0]); - /// let mut dst = [0; 2]; - /// rdr.read_i32_into::(&mut dst).unwrap(); - /// assert_eq!([517, 768], dst); - /// ``` - #[inline] - fn read_i32_into(&mut self, dst: &mut [i32]) -> Result<()> { - { - let buf = unsafe { slice_to_u8_mut(dst) }; - try!(self.read_exact(buf)); - } - T::from_slice_i32(dst); - Ok(()) - } - - /// Reads a sequence of signed 64 bit integers from the underlying - /// reader. - /// - /// The given buffer is either filled completely or an error is returned. - /// If an error is returned, the contents of `dst` are unspecified. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read a sequence of signed 64 bit big-endian integers from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![ - /// 0, 0, 0, 0, 0, 0, 2, 5, - /// 0, 0, 0, 0, 0, 0, 3, 0, - /// ]); - /// let mut dst = [0; 2]; - /// rdr.read_i64_into::(&mut dst).unwrap(); - /// assert_eq!([517, 768], dst); - /// ``` - #[inline] - fn read_i64_into(&mut self, dst: &mut [i64]) -> Result<()> { - { - let buf = unsafe { slice_to_u8_mut(dst) }; - try!(self.read_exact(buf)); - } - T::from_slice_i64(dst); - Ok(()) - } - - /// Reads a sequence of signed 128 bit integers from the underlying - /// reader. - /// - /// The given buffer is either filled completely or an error is returned. - /// If an error is returned, the contents of `dst` are unspecified. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read a sequence of signed 128 bit big-endian integers from a `Read`: - /// - /// ```rust - /// use std::io::Cursor; - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![ - /// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, - /// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, - /// ]); - /// let mut dst = [0; 2]; - /// rdr.read_i128_into::(&mut dst).unwrap(); - /// assert_eq!([517, 768], dst); - /// ``` - #[cfg(byteorder_i128)] - #[inline] - fn read_i128_into( - &mut self, - dst: &mut [i128], - ) -> Result<()> { - { - let buf = unsafe { slice_to_u8_mut(dst) }; - try!(self.read_exact(buf)); - } - T::from_slice_i128(dst); - Ok(()) - } - - /// Reads a sequence of IEEE754 single-precision (4 bytes) floating - /// point numbers from the underlying reader. - /// - /// The given buffer is either filled completely or an error is returned. - /// If an error is returned, the contents of `dst` are unspecified. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read a sequence of big-endian single-precision floating point number - /// from a `Read`: - /// - /// ```rust - /// use std::f32; - /// use std::io::Cursor; - /// - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![ - /// 0x40, 0x49, 0x0f, 0xdb, - /// 0x3f, 0x80, 0x00, 0x00, - /// ]); - /// let mut dst = [0.0; 2]; - /// rdr.read_f32_into::(&mut dst).unwrap(); - /// assert_eq!([f32::consts::PI, 1.0], dst); - /// ``` - #[inline] - fn read_f32_into( - &mut self, - dst: &mut [f32], - ) -> Result<()> { - { - let buf = unsafe { slice_to_u8_mut(dst) }; - try!(self.read_exact(buf)); - } - T::from_slice_f32(dst); - Ok(()) - } - - /// **DEPRECATED**. - /// - /// This method is deprecated. Use `read_f32_into` instead. - /// - /// Reads a sequence of IEEE754 single-precision (4 bytes) floating - /// point numbers from the underlying reader. - /// - /// The given buffer is either filled completely or an error is returned. - /// If an error is returned, the contents of `dst` are unspecified. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read a sequence of big-endian single-precision floating point number - /// from a `Read`: - /// - /// ```rust - /// use std::f32; - /// use std::io::Cursor; - /// - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![ - /// 0x40, 0x49, 0x0f, 0xdb, - /// 0x3f, 0x80, 0x00, 0x00, - /// ]); - /// let mut dst = [0.0; 2]; - /// rdr.read_f32_into_unchecked::(&mut dst).unwrap(); - /// assert_eq!([f32::consts::PI, 1.0], dst); - /// ``` - #[inline] - #[deprecated(since="1.2.0", note="please use `read_f32_into` instead")] - fn read_f32_into_unchecked( - &mut self, - dst: &mut [f32], - ) -> Result<()> { - self.read_f32_into::(dst) - } - - /// Reads a sequence of IEEE754 double-precision (8 bytes) floating - /// point numbers from the underlying reader. - /// - /// The given buffer is either filled completely or an error is returned. - /// If an error is returned, the contents of `dst` are unspecified. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read a sequence of big-endian single-precision floating point number - /// from a `Read`: - /// - /// ```rust - /// use std::f64; - /// use std::io::Cursor; - /// - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![ - /// 0x40, 0x09, 0x21, 0xfb, 0x54, 0x44, 0x2d, 0x18, - /// 0x3f, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - /// ]); - /// let mut dst = [0.0; 2]; - /// rdr.read_f64_into::(&mut dst).unwrap(); - /// assert_eq!([f64::consts::PI, 1.0], dst); - /// ``` - #[inline] - fn read_f64_into( - &mut self, - dst: &mut [f64], - ) -> Result<()> { - { - let buf = unsafe { slice_to_u8_mut(dst) }; - try!(self.read_exact(buf)); - } - T::from_slice_f64(dst); - Ok(()) - } - - /// **DEPRECATED**. - /// - /// This method is deprecated. Use `read_f64_into` instead. - /// - /// Reads a sequence of IEEE754 double-precision (8 bytes) floating - /// point numbers from the underlying reader. - /// - /// The given buffer is either filled completely or an error is returned. - /// If an error is returned, the contents of `dst` are unspecified. - /// - /// # Safety - /// - /// This method is unsafe because there are no guarantees made about the - /// floating point values. In particular, this method does not check for - /// signaling NaNs, which may result in undefined behavior. - /// - /// # Errors - /// - /// This method returns the same errors as [`Read::read_exact`]. - /// - /// [`Read::read_exact`]: https://doc.rust-lang.org/std/io/trait.Read.html#method.read_exact - /// - /// # Examples - /// - /// Read a sequence of big-endian single-precision floating point number - /// from a `Read`: - /// - /// ```rust - /// use std::f64; - /// use std::io::Cursor; - /// - /// use byteorder::{BigEndian, ReadBytesExt}; - /// - /// let mut rdr = Cursor::new(vec![ - /// 0x40, 0x09, 0x21, 0xfb, 0x54, 0x44, 0x2d, 0x18, - /// 0x3f, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - /// ]); - /// let mut dst = [0.0; 2]; - /// rdr.read_f64_into_unchecked::(&mut dst).unwrap(); - /// assert_eq!([f64::consts::PI, 1.0], dst); - /// ``` - #[inline] - #[deprecated(since="1.2.0", note="please use `read_f64_into` instead")] - fn read_f64_into_unchecked( - &mut self, - dst: &mut [f64], - ) -> Result<()> { - self.read_f64_into::(dst) - } -} - -/// All types that implement `Read` get methods defined in `ReadBytesExt` -/// for free. -impl ReadBytesExt for R {} - -/// Extends [`Write`] with methods for writing numbers. (For `std::io`.) -/// -/// Most of the methods defined here have an unconstrained type parameter that -/// must be explicitly instantiated. Typically, it is instantiated with either -/// the [`BigEndian`] or [`LittleEndian`] types defined in this crate. -/// -/// # Examples -/// -/// Write unsigned 16 bit big-endian integers to a [`Write`]: -/// -/// ```rust -/// use byteorder::{BigEndian, WriteBytesExt}; -/// -/// let mut wtr = vec![]; -/// wtr.write_u16::(517).unwrap(); -/// wtr.write_u16::(768).unwrap(); -/// assert_eq!(wtr, vec![2, 5, 3, 0]); -/// ``` -/// -/// [`BigEndian`]: enum.BigEndian.html -/// [`LittleEndian`]: enum.LittleEndian.html -/// [`Write`]: https://doc.rust-lang.org/std/io/trait.Write.html -pub trait WriteBytesExt: io::Write { - /// Writes an unsigned 8 bit integer to the underlying writer. - /// - /// Note that since this writes a single byte, no byte order conversions - /// are used. It is included for completeness. - /// - /// # Errors - /// - /// This method returns the same errors as [`Write::write_all`]. - /// - /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all - /// - /// # Examples - /// - /// Write unsigned 8 bit integers to a `Write`: - /// - /// ```rust - /// use byteorder::WriteBytesExt; - /// - /// let mut wtr = Vec::new(); - /// wtr.write_u8(2).unwrap(); - /// wtr.write_u8(5).unwrap(); - /// assert_eq!(wtr, b"\x02\x05"); - /// ``` - #[inline] - fn write_u8(&mut self, n: u8) -> Result<()> { - self.write_all(&[n]) - } - - /// Writes a signed 8 bit integer to the underlying writer. - /// - /// Note that since this writes a single byte, no byte order conversions - /// are used. It is included for completeness. - /// - /// # Errors - /// - /// This method returns the same errors as [`Write::write_all`]. - /// - /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all - /// - /// # Examples - /// - /// Write signed 8 bit integers to a `Write`: - /// - /// ```rust - /// use byteorder::WriteBytesExt; - /// - /// let mut wtr = Vec::new(); - /// wtr.write_i8(2).unwrap(); - /// wtr.write_i8(-5).unwrap(); - /// assert_eq!(wtr, b"\x02\xfb"); - /// ``` - #[inline] - fn write_i8(&mut self, n: i8) -> Result<()> { - self.write_all(&[n as u8]) - } - - /// Writes an unsigned 16 bit integer to the underlying writer. - /// - /// # Errors - /// - /// This method returns the same errors as [`Write::write_all`]. - /// - /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all - /// - /// # Examples - /// - /// Write unsigned 16 bit big-endian integers to a `Write`: - /// - /// ```rust - /// use byteorder::{BigEndian, WriteBytesExt}; - /// - /// let mut wtr = Vec::new(); - /// wtr.write_u16::(517).unwrap(); - /// wtr.write_u16::(768).unwrap(); - /// assert_eq!(wtr, b"\x02\x05\x03\x00"); - /// ``` - #[inline] - fn write_u16(&mut self, n: u16) -> Result<()> { - let mut buf = [0; 2]; - T::write_u16(&mut buf, n); - self.write_all(&buf) - } - - /// Writes a signed 16 bit integer to the underlying writer. - /// - /// # Errors - /// - /// This method returns the same errors as [`Write::write_all`]. - /// - /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all - /// - /// # Examples - /// - /// Write signed 16 bit big-endian integers to a `Write`: - /// - /// ```rust - /// use byteorder::{BigEndian, WriteBytesExt}; - /// - /// let mut wtr = Vec::new(); - /// wtr.write_i16::(193).unwrap(); - /// wtr.write_i16::(-132).unwrap(); - /// assert_eq!(wtr, b"\x00\xc1\xff\x7c"); - /// ``` - #[inline] - fn write_i16(&mut self, n: i16) -> Result<()> { - let mut buf = [0; 2]; - T::write_i16(&mut buf, n); - self.write_all(&buf) - } - - /// Writes an unsigned 24 bit integer to the underlying writer. - /// - /// # Errors - /// - /// This method returns the same errors as [`Write::write_all`]. - /// - /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all - /// - /// # Examples - /// - /// Write unsigned 24 bit big-endian integers to a `Write`: - /// - /// ```rust - /// use byteorder::{BigEndian, WriteBytesExt}; - /// - /// let mut wtr = Vec::new(); - /// wtr.write_u24::(267).unwrap(); - /// wtr.write_u24::(120111).unwrap(); - /// assert_eq!(wtr, b"\x00\x01\x0b\x01\xd5\x2f"); - /// ``` - #[inline] - fn write_u24(&mut self, n: u32) -> Result<()> { - let mut buf = [0; 3]; - T::write_u24(&mut buf, n); - self.write_all(&buf) - } - - /// Writes a signed 24 bit integer to the underlying writer. - /// - /// # Errors - /// - /// This method returns the same errors as [`Write::write_all`]. - /// - /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all - /// - /// # Examples - /// - /// Write signed 24 bit big-endian integers to a `Write`: - /// - /// ```rust - /// use byteorder::{BigEndian, WriteBytesExt}; - /// - /// let mut wtr = Vec::new(); - /// wtr.write_i24::(-34253).unwrap(); - /// wtr.write_i24::(120111).unwrap(); - /// assert_eq!(wtr, b"\xff\x7a\x33\x01\xd5\x2f"); - /// ``` - #[inline] - fn write_i24(&mut self, n: i32) -> Result<()> { - let mut buf = [0; 3]; - T::write_i24(&mut buf, n); - self.write_all(&buf) - } - - /// Writes an unsigned 32 bit integer to the underlying writer. - /// - /// # Errors - /// - /// This method returns the same errors as [`Write::write_all`]. - /// - /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all - /// - /// # Examples - /// - /// Write unsigned 32 bit big-endian integers to a `Write`: - /// - /// ```rust - /// use byteorder::{BigEndian, WriteBytesExt}; - /// - /// let mut wtr = Vec::new(); - /// wtr.write_u32::(267).unwrap(); - /// wtr.write_u32::(1205419366).unwrap(); - /// assert_eq!(wtr, b"\x00\x00\x01\x0b\x47\xd9\x3d\x66"); - /// ``` - #[inline] - fn write_u32(&mut self, n: u32) -> Result<()> { - let mut buf = [0; 4]; - T::write_u32(&mut buf, n); - self.write_all(&buf) - } - - /// Writes a signed 32 bit integer to the underlying writer. - /// - /// # Errors - /// - /// This method returns the same errors as [`Write::write_all`]. - /// - /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all - /// - /// # Examples - /// - /// Write signed 32 bit big-endian integers to a `Write`: - /// - /// ```rust - /// use byteorder::{BigEndian, WriteBytesExt}; - /// - /// let mut wtr = Vec::new(); - /// wtr.write_i32::(-34253).unwrap(); - /// wtr.write_i32::(1205419366).unwrap(); - /// assert_eq!(wtr, b"\xff\xff\x7a\x33\x47\xd9\x3d\x66"); - /// ``` - #[inline] - fn write_i32(&mut self, n: i32) -> Result<()> { - let mut buf = [0; 4]; - T::write_i32(&mut buf, n); - self.write_all(&buf) - } - - /// Writes an unsigned 48 bit integer to the underlying writer. - /// - /// # Errors - /// - /// This method returns the same errors as [`Write::write_all`]. - /// - /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all - /// - /// # Examples - /// - /// Write unsigned 48 bit big-endian integers to a `Write`: - /// - /// ```rust - /// use byteorder::{BigEndian, WriteBytesExt}; - /// - /// let mut wtr = Vec::new(); - /// wtr.write_u48::(52360336390828).unwrap(); - /// wtr.write_u48::(541).unwrap(); - /// assert_eq!(wtr, b"\x2f\x9f\x17\x40\x3a\xac\x00\x00\x00\x00\x02\x1d"); - /// ``` - #[inline] - fn write_u48(&mut self, n: u64) -> Result<()> { - let mut buf = [0; 6]; - T::write_u48(&mut buf, n); - self.write_all(&buf) - } - - /// Writes a signed 48 bit integer to the underlying writer. - /// - /// # Errors - /// - /// This method returns the same errors as [`Write::write_all`]. - /// - /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all - /// - /// # Examples - /// - /// Write signed 48 bit big-endian integers to a `Write`: - /// - /// ```rust - /// use byteorder::{BigEndian, WriteBytesExt}; - /// - /// let mut wtr = Vec::new(); - /// wtr.write_i48::(-108363435763825).unwrap(); - /// wtr.write_i48::(77).unwrap(); - /// assert_eq!(wtr, b"\x9d\x71\xab\xe7\x97\x8f\x00\x00\x00\x00\x00\x4d"); - /// ``` - #[inline] - fn write_i48(&mut self, n: i64) -> Result<()> { - let mut buf = [0; 6]; - T::write_i48(&mut buf, n); - self.write_all(&buf) - } - - /// Writes an unsigned 64 bit integer to the underlying writer. - /// - /// # Errors - /// - /// This method returns the same errors as [`Write::write_all`]. - /// - /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all - /// - /// # Examples - /// - /// Write unsigned 64 bit big-endian integers to a `Write`: - /// - /// ```rust - /// use byteorder::{BigEndian, WriteBytesExt}; - /// - /// let mut wtr = Vec::new(); - /// wtr.write_u64::(918733457491587).unwrap(); - /// wtr.write_u64::(143).unwrap(); - /// assert_eq!(wtr, b"\x00\x03\x43\x95\x4d\x60\x86\x83\x00\x00\x00\x00\x00\x00\x00\x8f"); - /// ``` - #[inline] - fn write_u64(&mut self, n: u64) -> Result<()> { - let mut buf = [0; 8]; - T::write_u64(&mut buf, n); - self.write_all(&buf) - } - - /// Writes a signed 64 bit integer to the underlying writer. - /// - /// # Errors - /// - /// This method returns the same errors as [`Write::write_all`]. - /// - /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all - /// - /// # Examples - /// - /// Write signed 64 bit big-endian integers to a `Write`: - /// - /// ```rust - /// use byteorder::{BigEndian, WriteBytesExt}; - /// - /// let mut wtr = Vec::new(); - /// wtr.write_i64::(i64::min_value()).unwrap(); - /// wtr.write_i64::(i64::max_value()).unwrap(); - /// assert_eq!(wtr, b"\x80\x00\x00\x00\x00\x00\x00\x00\x7f\xff\xff\xff\xff\xff\xff\xff"); - /// ``` - #[inline] - fn write_i64(&mut self, n: i64) -> Result<()> { - let mut buf = [0; 8]; - T::write_i64(&mut buf, n); - self.write_all(&buf) - } - - /// Writes an unsigned 128 bit integer to the underlying writer. - #[cfg(byteorder_i128)] - #[inline] - fn write_u128(&mut self, n: u128) -> Result<()> { - let mut buf = [0; 16]; - T::write_u128(&mut buf, n); - self.write_all(&buf) - } - - /// Writes a signed 128 bit integer to the underlying writer. - #[cfg(byteorder_i128)] - #[inline] - fn write_i128(&mut self, n: i128) -> Result<()> { - let mut buf = [0; 16]; - T::write_i128(&mut buf, n); - self.write_all(&buf) - } - - /// Writes an unsigned n-bytes integer to the underlying writer. - /// - /// # Errors - /// - /// This method returns the same errors as [`Write::write_all`]. - /// - /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all - /// - /// # Panics - /// - /// If the given integer is not representable in the given number of bytes, - /// this method panics. If `nbytes > 8`, this method panics. - /// - /// # Examples - /// - /// Write unsigned 40 bit big-endian integers to a `Write`: - /// - /// ```rust - /// use byteorder::{BigEndian, WriteBytesExt}; - /// - /// let mut wtr = Vec::new(); - /// wtr.write_uint::(312550384361, 5).unwrap(); - /// wtr.write_uint::(43, 5).unwrap(); - /// assert_eq!(wtr, b"\x48\xc5\x74\x62\xe9\x00\x00\x00\x00\x2b"); - /// ``` - #[inline] - fn write_uint( - &mut self, - n: u64, - nbytes: usize, - ) -> Result<()> { - let mut buf = [0; 8]; - T::write_uint(&mut buf, n, nbytes); - self.write_all(&buf[0..nbytes]) - } - - /// Writes a signed n-bytes integer to the underlying writer. - /// - /// # Errors - /// - /// This method returns the same errors as [`Write::write_all`]. - /// - /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all - /// - /// # Panics - /// - /// If the given integer is not representable in the given number of bytes, - /// this method panics. If `nbytes > 8`, this method panics. - /// - /// # Examples - /// - /// Write signed 56 bit big-endian integers to a `Write`: - /// - /// ```rust - /// use byteorder::{BigEndian, WriteBytesExt}; - /// - /// let mut wtr = Vec::new(); - /// wtr.write_int::(-3548172039376767, 7).unwrap(); - /// wtr.write_int::(43, 7).unwrap(); - /// assert_eq!(wtr, b"\xf3\x64\xf4\xd1\xfd\xb0\x81\x00\x00\x00\x00\x00\x00\x2b"); - /// ``` - #[inline] - fn write_int( - &mut self, - n: i64, - nbytes: usize, - ) -> Result<()> { - let mut buf = [0; 8]; - T::write_int(&mut buf, n, nbytes); - self.write_all(&buf[0..nbytes]) - } - - /// Writes an unsigned n-bytes integer to the underlying writer. - /// - /// If the given integer is not representable in the given number of bytes, - /// this method panics. If `nbytes > 16`, this method panics. - #[cfg(byteorder_i128)] - #[inline] - fn write_uint128( - &mut self, - n: u128, - nbytes: usize, - ) -> Result<()> { - let mut buf = [0; 16]; - T::write_uint128(&mut buf, n, nbytes); - self.write_all(&buf[0..nbytes]) - } - - /// Writes a signed n-bytes integer to the underlying writer. - /// - /// If the given integer is not representable in the given number of bytes, - /// this method panics. If `nbytes > 16`, this method panics. - #[cfg(byteorder_i128)] - #[inline] - fn write_int128( - &mut self, - n: i128, - nbytes: usize, - ) -> Result<()> { - let mut buf = [0; 16]; - T::write_int128(&mut buf, n, nbytes); - self.write_all(&buf[0..nbytes]) - } - - /// Writes a IEEE754 single-precision (4 bytes) floating point number to - /// the underlying writer. - /// - /// # Errors - /// - /// This method returns the same errors as [`Write::write_all`]. - /// - /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all - /// - /// # Examples - /// - /// Write a big-endian single-precision floating point number to a `Write`: - /// - /// ```rust - /// use std::f32; - /// - /// use byteorder::{BigEndian, WriteBytesExt}; - /// - /// let mut wtr = Vec::new(); - /// wtr.write_f32::(f32::consts::PI).unwrap(); - /// assert_eq!(wtr, b"\x40\x49\x0f\xdb"); - /// ``` - #[inline] - fn write_f32(&mut self, n: f32) -> Result<()> { - let mut buf = [0; 4]; - T::write_f32(&mut buf, n); - self.write_all(&buf) - } - - /// Writes a IEEE754 double-precision (8 bytes) floating point number to - /// the underlying writer. - /// - /// # Errors - /// - /// This method returns the same errors as [`Write::write_all`]. - /// - /// [`Write::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all - /// - /// # Examples - /// - /// Write a big-endian double-precision floating point number to a `Write`: - /// - /// ```rust - /// use std::f64; - /// - /// use byteorder::{BigEndian, WriteBytesExt}; - /// - /// let mut wtr = Vec::new(); - /// wtr.write_f64::(f64::consts::PI).unwrap(); - /// assert_eq!(wtr, b"\x40\x09\x21\xfb\x54\x44\x2d\x18"); - /// ``` - #[inline] - fn write_f64(&mut self, n: f64) -> Result<()> { - let mut buf = [0; 8]; - T::write_f64(&mut buf, n); - self.write_all(&buf) - } -} - -/// All types that implement `Write` get methods defined in `WriteBytesExt` -/// for free. -impl WriteBytesExt for W {} - -/// Convert a slice of T (where T is plain old data) to its mutable binary -/// representation. -/// -/// This function is wildly unsafe because it permits arbitrary modification of -/// the binary representation of any `Copy` type. Use with care. -unsafe fn slice_to_u8_mut(slice: &mut [T]) -> &mut [u8] { - use std::mem::size_of; - - let len = size_of::() * slice.len(); - slice::from_raw_parts_mut(slice.as_mut_ptr() as *mut u8, len) -} diff --git a/crux-mir/lib/byteorder/lib.rs b/crux-mir/lib/byteorder/lib.rs deleted file mode 100644 index 0909a3b95..000000000 --- a/crux-mir/lib/byteorder/lib.rs +++ /dev/null @@ -1,3204 +0,0 @@ -/*! -This crate provides convenience methods for encoding and decoding numbers in -either [big-endian or little-endian order]. - -The organization of the crate is pretty simple. A trait, [`ByteOrder`], specifies -byte conversion methods for each type of number in Rust (sans numbers that have -a platform dependent size like `usize` and `isize`). Two types, [`BigEndian`] -and [`LittleEndian`] implement these methods. Finally, [`ReadBytesExt`] and -[`WriteBytesExt`] provide convenience methods available to all types that -implement [`Read`] and [`Write`]. - -An alias, [`NetworkEndian`], for [`BigEndian`] is provided to help improve -code clarity. - -An additional alias, [`NativeEndian`], is provided for the endianness of the -local platform. This is convenient when serializing data for use and -conversions are not desired. - -# Examples - -Read unsigned 16 bit big-endian integers from a [`Read`] type: - -```rust -use std::io::Cursor; -use byteorder::{BigEndian, ReadBytesExt}; - -let mut rdr = Cursor::new(vec![2, 5, 3, 0]); -// Note that we use type parameters to indicate which kind of byte order -// we want! -assert_eq!(517, rdr.read_u16::().unwrap()); -assert_eq!(768, rdr.read_u16::().unwrap()); -``` - -Write unsigned 16 bit little-endian integers to a [`Write`] type: - -```rust -use byteorder::{LittleEndian, WriteBytesExt}; - -let mut wtr = vec![]; -wtr.write_u16::(517).unwrap(); -wtr.write_u16::(768).unwrap(); -assert_eq!(wtr, vec![5, 2, 0, 3]); -``` - -# Optional Features - -This crate optionally provides support for 128 bit values (`i128` and `u128`) -when built with the `i128` feature enabled. - -This crate can also be used without the standard library. - -# Alternatives - -Note that as of Rust 1.32, the standard numeric types provide built-in methods -like `to_le_bytes` and `from_le_bytes`, which support some of the same use -cases. - -[big-endian or little-endian order]: https://en.wikipedia.org/wiki/Endianness -[`ByteOrder`]: trait.ByteOrder.html -[`BigEndian`]: enum.BigEndian.html -[`LittleEndian`]: enum.LittleEndian.html -[`ReadBytesExt`]: trait.ReadBytesExt.html -[`WriteBytesExt`]: trait.WriteBytesExt.html -[`NetworkEndian`]: type.NetworkEndian.html -[`NativeEndian`]: type.NativeEndian.html -[`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html -[`Write`]: https://doc.rust-lang.org/std/io/trait.Write.html -*/ - -#![cfg_attr(crux, feature(crucible_intrinsics))] -#![deny(missing_docs)] -#![cfg_attr(not(feature = "std"), no_std)] - -#[cfg(feature = "std")] -extern crate core; - -#[cfg(test)] -#[macro_use] -extern crate doc_comment; - -#[cfg(test)] -doctest!("../README.md"); - -use core::fmt::Debug; -use core::hash::Hash; -use core::mem; -use core::ptr::copy_nonoverlapping; -use core::slice; - -#[cfg(feature = "std")] -pub use io::{ReadBytesExt, WriteBytesExt}; - -#[cfg(feature = "std")] -mod io; - -#[cfg(crux)] use core::mem::crucible_identity_transmute as identity_transmute; -#[cfg(not(crux))] use core::mem::transmute as identity_transmute; - -#[inline] -fn extend_sign(val: u64, nbytes: usize) -> i64 { - let shift = (8 - nbytes) * 8; - (val << shift) as i64 >> shift -} - -#[cfg(byteorder_i128)] -#[inline] -fn extend_sign128(val: u128, nbytes: usize) -> i128 { - let shift = (16 - nbytes) * 8; - (val << shift) as i128 >> shift -} - -#[inline] -fn unextend_sign(val: i64, nbytes: usize) -> u64 { - let shift = (8 - nbytes) * 8; - (val << shift) as u64 >> shift -} - -#[cfg(byteorder_i128)] -#[inline] -fn unextend_sign128(val: i128, nbytes: usize) -> u128 { - let shift = (16 - nbytes) * 8; - (val << shift) as u128 >> shift -} - -#[inline] -fn pack_size(n: u64) -> usize { - if n < 1 << 8 { - 1 - } else if n < 1 << 16 { - 2 - } else if n < 1 << 24 { - 3 - } else if n < 1 << 32 { - 4 - } else if n < 1 << 40 { - 5 - } else if n < 1 << 48 { - 6 - } else if n < 1 << 56 { - 7 - } else { - 8 - } -} - -#[cfg(byteorder_i128)] -#[inline] -fn pack_size128(n: u128) -> usize { - if n < 1 << 8 { - 1 - } else if n < 1 << 16 { - 2 - } else if n < 1 << 24 { - 3 - } else if n < 1 << 32 { - 4 - } else if n < 1 << 40 { - 5 - } else if n < 1 << 48 { - 6 - } else if n < 1 << 56 { - 7 - } else if n < 1 << 64 { - 8 - } else if n < 1 << 72 { - 9 - } else if n < 1 << 80 { - 10 - } else if n < 1 << 88 { - 11 - } else if n < 1 << 96 { - 12 - } else if n < 1 << 104 { - 13 - } else if n < 1 << 112 { - 14 - } else if n < 1 << 120 { - 15 - } else { - 16 - } -} - -mod private { - /// Sealed stops crates other than byteorder from implementing any traits - /// that use it. - pub trait Sealed{} - impl Sealed for super::LittleEndian {} - impl Sealed for super::BigEndian {} -} - -/// `ByteOrder` describes types that can serialize integers as bytes. -/// -/// Note that `Self` does not appear anywhere in this trait's definition! -/// Therefore, in order to use it, you'll need to use syntax like -/// `T::read_u16(&[0, 1])` where `T` implements `ByteOrder`. -/// -/// This crate provides two types that implement `ByteOrder`: [`BigEndian`] -/// and [`LittleEndian`]. -/// This trait is sealed and cannot be implemented for callers to avoid -/// breaking backwards compatibility when adding new derived traits. -/// -/// # Examples -/// -/// Write and read `u32` numbers in little endian order: -/// -/// ```rust -/// use byteorder::{ByteOrder, LittleEndian}; -/// -/// let mut buf = [0; 4]; -/// LittleEndian::write_u32(&mut buf, 1_000_000); -/// assert_eq!(1_000_000, LittleEndian::read_u32(&buf)); -/// ``` -/// -/// Write and read `i16` numbers in big endian order: -/// -/// ```rust -/// use byteorder::{ByteOrder, BigEndian}; -/// -/// let mut buf = [0; 2]; -/// BigEndian::write_i16(&mut buf, -5_000); -/// assert_eq!(-5_000, BigEndian::read_i16(&buf)); -/// ``` -/// -/// [`BigEndian`]: enum.BigEndian.html -/// [`LittleEndian`]: enum.LittleEndian.html -pub trait ByteOrder - : Clone + Copy + Debug + Default + Eq + Hash + Ord + PartialEq + PartialOrd - + private::Sealed -{ - /// Reads an unsigned 16 bit integer from `buf`. - /// - /// # Panics - /// - /// Panics when `buf.len() < 2`. - fn read_u16(buf: &[u8]) -> u16; - - /// Reads an unsigned 24 bit integer from `buf`, stored in u32. - /// - /// # Panics - /// - /// Panics when `buf.len() < 3`. - /// - /// # Examples - /// - /// Write and read 24 bit `u32` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 3]; - /// LittleEndian::write_u24(&mut buf, 1_000_000); - /// assert_eq!(1_000_000, LittleEndian::read_u24(&buf)); - /// ``` - fn read_u24(buf: &[u8]) -> u32 { - Self::read_uint(buf, 3) as u32 - } - - /// Reads an unsigned 32 bit integer from `buf`. - /// - /// # Panics - /// - /// Panics when `buf.len() < 4`. - /// - /// # Examples - /// - /// Write and read `u32` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 4]; - /// LittleEndian::write_u32(&mut buf, 1_000_000); - /// assert_eq!(1_000_000, LittleEndian::read_u32(&buf)); - /// ``` - fn read_u32(buf: &[u8]) -> u32; - - /// Reads an unsigned 48 bit integer from `buf`, stored in u64. - /// - /// # Panics - /// - /// Panics when `buf.len() < 6`. - /// - /// # Examples - /// - /// Write and read 48 bit `u64` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 6]; - /// LittleEndian::write_u48(&mut buf, 1_000_000_000_000); - /// assert_eq!(1_000_000_000_000, LittleEndian::read_u48(&buf)); - /// ``` - fn read_u48(buf: &[u8]) -> u64 { - Self::read_uint(buf, 6) as u64 - } - - /// Reads an unsigned 64 bit integer from `buf`. - /// - /// # Panics - /// - /// Panics when `buf.len() < 8`. - /// - /// # Examples - /// - /// Write and read `u64` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 8]; - /// LittleEndian::write_u64(&mut buf, 1_000_000); - /// assert_eq!(1_000_000, LittleEndian::read_u64(&buf)); - /// ``` - fn read_u64(buf: &[u8]) -> u64; - - /// Reads an unsigned 128 bit integer from `buf`. - /// - /// # Panics - /// - /// Panics when `buf.len() < 16`. - /// - /// # Examples - /// - /// Write and read `u128` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 16]; - /// LittleEndian::write_u128(&mut buf, 1_000_000); - /// assert_eq!(1_000_000, LittleEndian::read_u128(&buf)); - /// ``` - #[cfg(byteorder_i128)] - fn read_u128(buf: &[u8]) -> u128; - - /// Reads an unsigned n-bytes integer from `buf`. - /// - /// # Panics - /// - /// Panics when `nbytes < 1` or `nbytes > 8` or - /// `buf.len() < nbytes` - /// - /// # Examples - /// - /// Write and read an n-byte number in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 3]; - /// LittleEndian::write_uint(&mut buf, 1_000_000, 3); - /// assert_eq!(1_000_000, LittleEndian::read_uint(&buf, 3)); - /// ``` - fn read_uint(buf: &[u8], nbytes: usize) -> u64; - - /// Reads an unsigned n-bytes integer from `buf`. - /// - /// # Panics - /// - /// Panics when `nbytes < 1` or `nbytes > 16` or - /// `buf.len() < nbytes` - /// - /// # Examples - /// - /// Write and read an n-byte number in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 3]; - /// LittleEndian::write_uint128(&mut buf, 1_000_000, 3); - /// assert_eq!(1_000_000, LittleEndian::read_uint128(&buf, 3)); - /// ``` - #[cfg(byteorder_i128)] - fn read_uint128(buf: &[u8], nbytes: usize) -> u128; - - /// Writes an unsigned 16 bit integer `n` to `buf`. - /// - /// # Panics - /// - /// Panics when `buf.len() < 2`. - /// - /// # Examples - /// - /// Write and read `u16` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 2]; - /// LittleEndian::write_u16(&mut buf, 1_000); - /// assert_eq!(1_000, LittleEndian::read_u16(&buf)); - /// ``` - fn write_u16(buf: &mut [u8], n: u16); - - /// Writes an unsigned 24 bit integer `n` to `buf`, stored in u32. - /// - /// # Panics - /// - /// Panics when `buf.len() < 3`. - /// - /// # Examples - /// - /// Write and read 24 bit `u32` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 3]; - /// LittleEndian::write_u24(&mut buf, 1_000_000); - /// assert_eq!(1_000_000, LittleEndian::read_u24(&buf)); - /// ``` - fn write_u24(buf: &mut [u8], n: u32) { - Self::write_uint(buf, n as u64, 3) - } - - /// Writes an unsigned 32 bit integer `n` to `buf`. - /// - /// # Panics - /// - /// Panics when `buf.len() < 4`. - /// - /// # Examples - /// - /// Write and read `u32` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 4]; - /// LittleEndian::write_u32(&mut buf, 1_000_000); - /// assert_eq!(1_000_000, LittleEndian::read_u32(&buf)); - /// ``` - fn write_u32(buf: &mut [u8], n: u32); - - /// Writes an unsigned 48 bit integer `n` to `buf`, stored in u64. - /// - /// # Panics - /// - /// Panics when `buf.len() < 6`. - /// - /// # Examples - /// - /// Write and read 48 bit `u64` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 6]; - /// LittleEndian::write_u48(&mut buf, 1_000_000_000_000); - /// assert_eq!(1_000_000_000_000, LittleEndian::read_u48(&buf)); - /// ``` - fn write_u48(buf: &mut [u8], n: u64) { - Self::write_uint(buf, n as u64, 6) - } - - /// Writes an unsigned 64 bit integer `n` to `buf`. - /// - /// # Panics - /// - /// Panics when `buf.len() < 8`. - /// - /// # Examples - /// - /// Write and read `u64` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 8]; - /// LittleEndian::write_u64(&mut buf, 1_000_000); - /// assert_eq!(1_000_000, LittleEndian::read_u64(&buf)); - /// ``` - fn write_u64(buf: &mut [u8], n: u64); - - /// Writes an unsigned 128 bit integer `n` to `buf`. - /// - /// # Panics - /// - /// Panics when `buf.len() < 16`. - /// - /// # Examples - /// - /// Write and read `u128` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 16]; - /// LittleEndian::write_u128(&mut buf, 1_000_000); - /// assert_eq!(1_000_000, LittleEndian::read_u128(&buf)); - /// ``` - #[cfg(byteorder_i128)] - fn write_u128(buf: &mut [u8], n: u128); - - /// Writes an unsigned integer `n` to `buf` using only `nbytes`. - /// - /// # Panics - /// - /// If `n` is not representable in `nbytes`, or if `nbytes` is `> 8`, then - /// this method panics. - /// - /// # Examples - /// - /// Write and read an n-byte number in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 3]; - /// LittleEndian::write_uint(&mut buf, 1_000_000, 3); - /// assert_eq!(1_000_000, LittleEndian::read_uint(&buf, 3)); - /// ``` - fn write_uint(buf: &mut [u8], n: u64, nbytes: usize); - - /// Writes an unsigned integer `n` to `buf` using only `nbytes`. - /// - /// # Panics - /// - /// If `n` is not representable in `nbytes`, or if `nbytes` is `> 16`, then - /// this method panics. - /// - /// # Examples - /// - /// Write and read an n-byte number in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 3]; - /// LittleEndian::write_uint128(&mut buf, 1_000_000, 3); - /// assert_eq!(1_000_000, LittleEndian::read_uint128(&buf, 3)); - /// ``` - #[cfg(byteorder_i128)] - fn write_uint128(buf: &mut [u8], n: u128, nbytes: usize); - - /// Reads a signed 16 bit integer from `buf`. - /// - /// # Panics - /// - /// Panics when `buf.len() < 2`. - /// - /// # Examples - /// - /// Write and read `i16` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 2]; - /// LittleEndian::write_i16(&mut buf, -1_000); - /// assert_eq!(-1_000, LittleEndian::read_i16(&buf)); - /// ``` - #[inline] - fn read_i16(buf: &[u8]) -> i16 { - Self::read_u16(buf) as i16 - } - - /// Reads a signed 24 bit integer from `buf`, stored in i32. - /// - /// # Panics - /// - /// Panics when `buf.len() < 3`. - /// - /// # Examples - /// - /// Write and read 24 bit `i32` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 3]; - /// LittleEndian::write_i24(&mut buf, -1_000_000); - /// assert_eq!(-1_000_000, LittleEndian::read_i24(&buf)); - /// ``` - #[inline] - fn read_i24(buf: &[u8]) -> i32 { - Self::read_int(buf, 3) as i32 - } - - /// Reads a signed 32 bit integer from `buf`. - /// - /// # Panics - /// - /// Panics when `buf.len() < 4`. - /// - /// # Examples - /// - /// Write and read `i32` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 4]; - /// LittleEndian::write_i32(&mut buf, -1_000_000); - /// assert_eq!(-1_000_000, LittleEndian::read_i32(&buf)); - /// ``` - #[inline] - fn read_i32(buf: &[u8]) -> i32 { - Self::read_u32(buf) as i32 - } - - /// Reads a signed 48 bit integer from `buf`, stored in i64. - /// - /// # Panics - /// - /// Panics when `buf.len() < 6`. - /// - /// # Examples - /// - /// Write and read 48 bit `i64` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 6]; - /// LittleEndian::write_i48(&mut buf, -1_000_000_000_000); - /// assert_eq!(-1_000_000_000_000, LittleEndian::read_i48(&buf)); - /// ``` - #[inline] - fn read_i48(buf: &[u8]) -> i64 { - Self::read_int(buf, 6) as i64 - } - - /// Reads a signed 64 bit integer from `buf`. - /// - /// # Panics - /// - /// Panics when `buf.len() < 8`. - /// - /// # Examples - /// - /// Write and read `i64` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 8]; - /// LittleEndian::write_i64(&mut buf, -1_000_000_000); - /// assert_eq!(-1_000_000_000, LittleEndian::read_i64(&buf)); - /// ``` - #[inline] - fn read_i64(buf: &[u8]) -> i64 { - Self::read_u64(buf) as i64 - } - - /// Reads a signed 128 bit integer from `buf`. - /// - /// # Panics - /// - /// Panics when `buf.len() < 16`. - /// - /// # Examples - /// - /// Write and read `i128` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 16]; - /// LittleEndian::write_i128(&mut buf, -1_000_000_000); - /// assert_eq!(-1_000_000_000, LittleEndian::read_i128(&buf)); - /// ``` - #[cfg(byteorder_i128)] - #[inline] - fn read_i128(buf: &[u8]) -> i128 { - Self::read_u128(buf) as i128 - } - - /// Reads a signed n-bytes integer from `buf`. - /// - /// # Panics - /// - /// Panics when `nbytes < 1` or `nbytes > 8` or - /// `buf.len() < nbytes` - /// - /// # Examples - /// - /// Write and read n-length signed numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 3]; - /// LittleEndian::write_int(&mut buf, -1_000, 3); - /// assert_eq!(-1_000, LittleEndian::read_int(&buf, 3)); - /// ``` - #[inline] - fn read_int(buf: &[u8], nbytes: usize) -> i64 { - extend_sign(Self::read_uint(buf, nbytes), nbytes) - } - - /// Reads a signed n-bytes integer from `buf`. - /// - /// # Panics - /// - /// Panics when `nbytes < 1` or `nbytes > 16` or - /// `buf.len() < nbytes` - /// - /// # Examples - /// - /// Write and read n-length signed numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 3]; - /// LittleEndian::write_int128(&mut buf, -1_000, 3); - /// assert_eq!(-1_000, LittleEndian::read_int128(&buf, 3)); - /// ``` - #[cfg(byteorder_i128)] - #[inline] - fn read_int128(buf: &[u8], nbytes: usize) -> i128 { - extend_sign128(Self::read_uint128(buf, nbytes), nbytes) - } - - /// Reads a IEEE754 single-precision (4 bytes) floating point number. - /// - /// # Panics - /// - /// Panics when `buf.len() < 4`. - /// - /// # Examples - /// - /// Write and read `f32` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let e = 2.71828; - /// let mut buf = [0; 4]; - /// LittleEndian::write_f32(&mut buf, e); - /// assert_eq!(e, LittleEndian::read_f32(&buf)); - /// ``` - #[inline] - fn read_f32(buf: &[u8]) -> f32 { - unimplemented!() - } - - /// Reads a IEEE754 double-precision (8 bytes) floating point number. - /// - /// # Panics - /// - /// Panics when `buf.len() < 8`. - /// - /// # Examples - /// - /// Write and read `f64` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let phi = 1.6180339887; - /// let mut buf = [0; 8]; - /// LittleEndian::write_f64(&mut buf, phi); - /// assert_eq!(phi, LittleEndian::read_f64(&buf)); - /// ``` - #[inline] - fn read_f64(buf: &[u8]) -> f64 { - unimplemented!() - } - - /// Writes a signed 16 bit integer `n` to `buf`. - /// - /// # Panics - /// - /// Panics when `buf.len() < 2`. - /// - /// # Examples - /// - /// Write and read `i16` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 2]; - /// LittleEndian::write_i16(&mut buf, -1_000); - /// assert_eq!(-1_000, LittleEndian::read_i16(&buf)); - /// ``` - #[inline] - fn write_i16(buf: &mut [u8], n: i16) { - Self::write_u16(buf, n as u16) - } - - /// Writes a signed 24 bit integer `n` to `buf`, stored in i32. - /// - /// # Panics - /// - /// Panics when `buf.len() < 3`. - /// - /// # Examples - /// - /// Write and read 24 bit `i32` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 3]; - /// LittleEndian::write_i24(&mut buf, -1_000_000); - /// assert_eq!(-1_000_000, LittleEndian::read_i24(&buf)); - /// ``` - #[inline] - fn write_i24(buf: &mut [u8], n: i32) { - Self::write_int(buf, n as i64, 3) - } - - /// Writes a signed 32 bit integer `n` to `buf`. - /// - /// # Panics - /// - /// Panics when `buf.len() < 4`. - /// - /// # Examples - /// - /// Write and read `i32` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 4]; - /// LittleEndian::write_i32(&mut buf, -1_000_000); - /// assert_eq!(-1_000_000, LittleEndian::read_i32(&buf)); - /// ``` - #[inline] - fn write_i32(buf: &mut [u8], n: i32) { - Self::write_u32(buf, n as u32) - } - - /// Writes a signed 48 bit integer `n` to `buf`, stored in i64. - /// - /// # Panics - /// - /// Panics when `buf.len() < 6`. - /// - /// # Examples - /// - /// Write and read 48 bit `i64` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 6]; - /// LittleEndian::write_i48(&mut buf, -1_000_000_000_000); - /// assert_eq!(-1_000_000_000_000, LittleEndian::read_i48(&buf)); - /// ``` - #[inline] - fn write_i48(buf: &mut [u8], n: i64) { - Self::write_int(buf, n as i64, 6) - } - - /// Writes a signed 64 bit integer `n` to `buf`. - /// - /// # Panics - /// - /// Panics when `buf.len() < 8`. - /// - /// # Examples - /// - /// Write and read `i64` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 8]; - /// LittleEndian::write_i64(&mut buf, -1_000_000_000); - /// assert_eq!(-1_000_000_000, LittleEndian::read_i64(&buf)); - /// ``` - #[inline] - fn write_i64(buf: &mut [u8], n: i64) { - Self::write_u64(buf, n as u64) - } - - /// Writes a signed 128 bit integer `n` to `buf`. - /// - /// # Panics - /// - /// Panics when `buf.len() < 16`. - /// - /// # Examples - /// - /// Write and read n-byte `i128` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 16]; - /// LittleEndian::write_i128(&mut buf, -1_000_000_000); - /// assert_eq!(-1_000_000_000, LittleEndian::read_i128(&buf)); - /// ``` - #[cfg(byteorder_i128)] - #[inline] - fn write_i128(buf: &mut [u8], n: i128) { - Self::write_u128(buf, n as u128) - } - - /// Writes a signed integer `n` to `buf` using only `nbytes`. - /// - /// # Panics - /// - /// If `n` is not representable in `nbytes`, or if `nbytes` is `> 8`, then - /// this method panics. - /// - /// # Examples - /// - /// Write and read an n-byte number in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 3]; - /// LittleEndian::write_int(&mut buf, -1_000, 3); - /// assert_eq!(-1_000, LittleEndian::read_int(&buf, 3)); - /// ``` - #[inline] - fn write_int(buf: &mut [u8], n: i64, nbytes: usize) { - Self::write_uint(buf, unextend_sign(n, nbytes), nbytes) - } - - /// Writes a signed integer `n` to `buf` using only `nbytes`. - /// - /// # Panics - /// - /// If `n` is not representable in `nbytes`, or if `nbytes` is `> 16`, then - /// this method panics. - /// - /// # Examples - /// - /// Write and read n-length signed numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut buf = [0; 3]; - /// LittleEndian::write_int128(&mut buf, -1_000, 3); - /// assert_eq!(-1_000, LittleEndian::read_int128(&buf, 3)); - /// ``` - #[cfg(byteorder_i128)] - #[inline] - fn write_int128(buf: &mut [u8], n: i128, nbytes: usize) { - Self::write_uint128(buf, unextend_sign128(n, nbytes), nbytes) - } - - /// Writes a IEEE754 single-precision (4 bytes) floating point number. - /// - /// # Panics - /// - /// Panics when `buf.len() < 4`. - /// - /// # Examples - /// - /// Write and read `f32` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let e = 2.71828; - /// let mut buf = [0; 4]; - /// LittleEndian::write_f32(&mut buf, e); - /// assert_eq!(e, LittleEndian::read_f32(&buf)); - /// ``` - #[inline] - fn write_f32(buf: &mut [u8], n: f32) { - unimplemented!() - } - - /// Writes a IEEE754 double-precision (8 bytes) floating point number. - /// - /// # Panics - /// - /// Panics when `buf.len() < 8`. - /// - /// # Examples - /// - /// Write and read `f64` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let phi = 1.6180339887; - /// let mut buf = [0; 8]; - /// LittleEndian::write_f64(&mut buf, phi); - /// assert_eq!(phi, LittleEndian::read_f64(&buf)); - /// ``` - #[inline] - fn write_f64(buf: &mut [u8], n: f64) { - unimplemented!() - } - - /// Reads unsigned 16 bit integers from `src` into `dst`. - /// - /// # Panics - /// - /// Panics when `src.len() != 2*dst.len()`. - /// - /// # Examples - /// - /// Write and read `u16` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 8]; - /// let numbers_given = [1, 2, 0xf00f, 0xffee]; - /// LittleEndian::write_u16_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0; 4]; - /// LittleEndian::read_u16_into(&bytes, &mut numbers_got); - /// assert_eq!(numbers_given, numbers_got); - /// ``` - fn read_u16_into(src: &[u8], dst: &mut [u16]); - - /// Reads unsigned 32 bit integers from `src` into `dst`. - /// - /// # Panics - /// - /// Panics when `src.len() != 4*dst.len()`. - /// - /// # Examples - /// - /// Write and read `u32` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 16]; - /// let numbers_given = [1, 2, 0xf00f, 0xffee]; - /// LittleEndian::write_u32_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0; 4]; - /// LittleEndian::read_u32_into(&bytes, &mut numbers_got); - /// assert_eq!(numbers_given, numbers_got); - /// ``` - fn read_u32_into(src: &[u8], dst: &mut [u32]); - - /// Reads unsigned 64 bit integers from `src` into `dst`. - /// - /// # Panics - /// - /// Panics when `src.len() != 8*dst.len()`. - /// - /// # Examples - /// - /// Write and read `u64` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 32]; - /// let numbers_given = [1, 2, 0xf00f, 0xffee]; - /// LittleEndian::write_u64_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0; 4]; - /// LittleEndian::read_u64_into(&bytes, &mut numbers_got); - /// assert_eq!(numbers_given, numbers_got); - /// ``` - fn read_u64_into(src: &[u8], dst: &mut [u64]); - - /// Reads unsigned 128 bit integers from `src` into `dst`. - /// - /// # Panics - /// - /// Panics when `src.len() != 16*dst.len()`. - /// - /// # Examples - /// - /// Write and read `u128` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 64]; - /// let numbers_given = [1, 2, 0xf00f, 0xffee]; - /// LittleEndian::write_u128_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0; 4]; - /// LittleEndian::read_u128_into(&bytes, &mut numbers_got); - /// assert_eq!(numbers_given, numbers_got); - /// ``` - #[cfg(byteorder_i128)] - fn read_u128_into(src: &[u8], dst: &mut [u128]); - - /// Reads signed 16 bit integers from `src` to `dst`. - /// - /// # Panics - /// - /// Panics when `buf.len() != 2*dst.len()`. - /// - /// # Examples - /// - /// Write and read `i16` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 8]; - /// let numbers_given = [1, 2, 0x0f, 0xee]; - /// LittleEndian::write_i16_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0; 4]; - /// LittleEndian::read_i16_into(&bytes, &mut numbers_got); - /// assert_eq!(numbers_given, numbers_got); - /// ``` - #[inline] - fn read_i16_into(src: &[u8], dst: &mut [i16]) { - let dst = unsafe { - identity_transmute::<&mut [i16], &mut [u16]>(dst) - }; - Self::read_u16_into(src, dst) - } - - /// Reads signed 32 bit integers from `src` into `dst`. - /// - /// # Panics - /// - /// Panics when `src.len() != 4*dst.len()`. - /// - /// # Examples - /// - /// Write and read `i32` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 16]; - /// let numbers_given = [1, 2, 0xf00f, 0xffee]; - /// LittleEndian::write_i32_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0; 4]; - /// LittleEndian::read_i32_into(&bytes, &mut numbers_got); - /// assert_eq!(numbers_given, numbers_got); - /// ``` - #[inline] - fn read_i32_into(src: &[u8], dst: &mut [i32]) { - let dst = unsafe { - identity_transmute::<&mut [i32], &mut [u32]>(dst) - }; - Self::read_u32_into(src, dst); - } - - /// Reads signed 64 bit integers from `src` into `dst`. - /// - /// # Panics - /// - /// Panics when `src.len() != 8*dst.len()`. - /// - /// # Examples - /// - /// Write and read `i64` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 32]; - /// let numbers_given = [1, 2, 0xf00f, 0xffee]; - /// LittleEndian::write_i64_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0; 4]; - /// LittleEndian::read_i64_into(&bytes, &mut numbers_got); - /// assert_eq!(numbers_given, numbers_got); - /// ``` - #[inline] - fn read_i64_into(src: &[u8], dst: &mut [i64]) { - let dst = unsafe { - identity_transmute::<&mut [i64], &mut [u64]>(dst) - }; - Self::read_u64_into(src, dst); - } - - /// Reads signed 128 bit integers from `src` into `dst`. - /// - /// # Panics - /// - /// Panics when `src.len() != 16*dst.len()`. - /// - /// # Examples - /// - /// Write and read `i128` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 64]; - /// let numbers_given = [1, 2, 0xf00f, 0xffee]; - /// LittleEndian::write_i128_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0; 4]; - /// LittleEndian::read_i128_into(&bytes, &mut numbers_got); - /// assert_eq!(numbers_given, numbers_got); - /// ``` - #[cfg(byteorder_i128)] - #[inline] - fn read_i128_into(src: &[u8], dst: &mut [i128]) { - let dst = unsafe { - identity_transmute::<&mut [i128], &mut [u128]>(dst) - }; - Self::read_u128_into(src, dst); - } - - /// Reads IEEE754 single-precision (4 bytes) floating point numbers from - /// `src` into `dst`. - /// - /// # Panics - /// - /// Panics when `src.len() != 4*dst.len()`. - /// - /// # Examples - /// - /// Write and read `f32` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 16]; - /// let numbers_given = [1.0, 2.0, 31.312e31, -11.32e19]; - /// LittleEndian::write_f32_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0.0; 4]; - /// LittleEndian::read_f32_into(&bytes, &mut numbers_got); - /// assert_eq!(numbers_given, numbers_got); - /// ``` - #[inline] - fn read_f32_into(src: &[u8], dst: &mut [f32]) { - unimplemented!() - } - - /// **DEPRECATED**. - /// - /// This method is deprecated. Use `read_f32_into` instead. - /// Reads IEEE754 single-precision (4 bytes) floating point numbers from - /// `src` into `dst`. - /// - /// # Panics - /// - /// Panics when `src.len() != 4*dst.len()`. - /// - /// # Examples - /// - /// Write and read `f32` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 16]; - /// let numbers_given = [1.0, 2.0, 31.312e31, -11.32e19]; - /// LittleEndian::write_f32_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0.0; 4]; - /// LittleEndian::read_f32_into_unchecked(&bytes, &mut numbers_got); - /// assert_eq!(numbers_given, numbers_got); - /// ``` - #[inline] - #[deprecated(since="1.3.0", note="please use `read_f32_into` instead")] - fn read_f32_into_unchecked(src: &[u8], dst: &mut [f32]) { - Self::read_f32_into(src, dst); - } - - /// Reads IEEE754 single-precision (4 bytes) floating point numbers from - /// `src` into `dst`. - /// - /// # Panics - /// - /// Panics when `src.len() != 8*dst.len()`. - /// - /// # Examples - /// - /// Write and read `f64` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 32]; - /// let numbers_given = [1.0, 2.0, 31.312e211, -11.32e91]; - /// LittleEndian::write_f64_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0.0; 4]; - /// LittleEndian::read_f64_into(&bytes, &mut numbers_got); - /// assert_eq!(numbers_given, numbers_got); - /// ``` - #[inline] - fn read_f64_into(src: &[u8], dst: &mut [f64]) { - unimplemented!() - } - - /// **DEPRECATED**. - /// - /// This method is deprecated. Use `read_f64_into` instead. - /// - /// Reads IEEE754 single-precision (4 bytes) floating point numbers from - /// `src` into `dst`. - /// - /// # Panics - /// - /// Panics when `src.len() != 8*dst.len()`. - /// - /// # Examples - /// - /// Write and read `f64` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 32]; - /// let numbers_given = [1.0, 2.0, 31.312e211, -11.32e91]; - /// LittleEndian::write_f64_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0.0; 4]; - /// LittleEndian::read_f64_into_unchecked(&bytes, &mut numbers_got); - /// assert_eq!(numbers_given, numbers_got); - /// ``` - #[inline] - #[deprecated(since="1.3.0", note="please use `read_f64_into` instead")] - fn read_f64_into_unchecked(src: &[u8], dst: &mut [f64]) { - Self::read_f64_into(src, dst); - } - - /// Writes unsigned 16 bit integers from `src` into `dst`. - /// - /// # Panics - /// - /// Panics when `dst.len() != 2*src.len()`. - /// - /// # Examples - /// - /// Write and read `u16` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 8]; - /// let numbers_given = [1, 2, 0xf00f, 0xffee]; - /// LittleEndian::write_u16_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0; 4]; - /// LittleEndian::read_u16_into(&bytes, &mut numbers_got); - /// assert_eq!(numbers_given, numbers_got); - /// ``` - fn write_u16_into(src: &[u16], dst: &mut [u8]); - - /// Writes unsigned 32 bit integers from `src` into `dst`. - /// - /// # Panics - /// - /// Panics when `dst.len() != 4*src.len()`. - /// - /// # Examples - /// - /// Write and read `u32` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 16]; - /// let numbers_given = [1, 2, 0xf00f, 0xffee]; - /// LittleEndian::write_u32_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0; 4]; - /// LittleEndian::read_u32_into(&bytes, &mut numbers_got); - /// assert_eq!(numbers_given, numbers_got); - /// ``` - fn write_u32_into(src: &[u32], dst: &mut [u8]); - - /// Writes unsigned 64 bit integers from `src` into `dst`. - /// - /// # Panics - /// - /// Panics when `dst.len() != 8*src.len()`. - /// - /// # Examples - /// - /// Write and read `u64` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 32]; - /// let numbers_given = [1, 2, 0xf00f, 0xffee]; - /// LittleEndian::write_u64_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0; 4]; - /// LittleEndian::read_u64_into(&bytes, &mut numbers_got); - /// assert_eq!(numbers_given, numbers_got); - /// ``` - fn write_u64_into(src: &[u64], dst: &mut [u8]); - - /// Writes unsigned 128 bit integers from `src` into `dst`. - /// - /// # Panics - /// - /// Panics when `dst.len() != 16*src.len()`. - /// - /// # Examples - /// - /// Write and read `u128` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 64]; - /// let numbers_given = [1, 2, 0xf00f, 0xffee]; - /// LittleEndian::write_u128_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0; 4]; - /// LittleEndian::read_u128_into(&bytes, &mut numbers_got); - /// assert_eq!(numbers_given, numbers_got); - /// ``` - #[cfg(byteorder_i128)] - fn write_u128_into(src: &[u128], dst: &mut [u8]); - - /// Writes signed 16 bit integers from `src` into `dst`. - /// - /// # Panics - /// - /// Panics when `buf.len() != 2*src.len()`. - /// - /// # Examples - /// - /// Write and read `i16` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 8]; - /// let numbers_given = [1, 2, 0x0f, 0xee]; - /// LittleEndian::write_i16_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0; 4]; - /// LittleEndian::read_i16_into(&bytes, &mut numbers_got); - /// assert_eq!(numbers_given, numbers_got); - /// ``` - fn write_i16_into(src: &[i16], dst: &mut [u8]) { - let src = unsafe { - identity_transmute::<&[i16], &[u16]>(src) - }; - Self::write_u16_into(src, dst); - } - - /// Writes signed 32 bit integers from `src` into `dst`. - /// - /// # Panics - /// - /// Panics when `dst.len() != 4*src.len()`. - /// - /// # Examples - /// - /// Write and read `i32` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 16]; - /// let numbers_given = [1, 2, 0xf00f, 0xffee]; - /// LittleEndian::write_i32_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0; 4]; - /// LittleEndian::read_i32_into(&bytes, &mut numbers_got); - /// assert_eq!(numbers_given, numbers_got); - /// ``` - fn write_i32_into(src: &[i32], dst: &mut [u8]) { - let src = unsafe { - identity_transmute::<&[i32], &[u32]>(src) - }; - Self::write_u32_into(src, dst); - } - - /// Writes signed 64 bit integers from `src` into `dst`. - /// - /// # Panics - /// - /// Panics when `dst.len() != 8*src.len()`. - /// - /// # Examples - /// - /// Write and read `i64` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 32]; - /// let numbers_given = [1, 2, 0xf00f, 0xffee]; - /// LittleEndian::write_i64_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0; 4]; - /// LittleEndian::read_i64_into(&bytes, &mut numbers_got); - /// assert_eq!(numbers_given, numbers_got); - /// ``` - fn write_i64_into(src: &[i64], dst: &mut [u8]) { - let src = unsafe { - identity_transmute::<&[i64], &[u64]>(src) - }; - Self::write_u64_into(src, dst); - } - - /// Writes signed 128 bit integers from `src` into `dst`. - /// - /// # Panics - /// - /// Panics when `dst.len() != 16*src.len()`. - /// - /// # Examples - /// - /// Write and read `i128` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 64]; - /// let numbers_given = [1, 2, 0xf00f, 0xffee]; - /// LittleEndian::write_i128_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0; 4]; - /// LittleEndian::read_i128_into(&bytes, &mut numbers_got); - /// assert_eq!(numbers_given, numbers_got); - /// ``` - #[cfg(byteorder_i128)] - fn write_i128_into(src: &[i128], dst: &mut [u8]) { - let src = unsafe { - identity_transmute::<&[i128], &[u128]>(src) - }; - Self::write_u128_into(src, dst); - } - - /// Writes IEEE754 single-precision (4 bytes) floating point numbers from - /// `src` into `dst`. - /// - /// # Panics - /// - /// Panics when `src.len() != 4*dst.len()`. - /// - /// # Examples - /// - /// Write and read `f32` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 16]; - /// let numbers_given = [1.0, 2.0, 31.312e31, -11.32e19]; - /// LittleEndian::write_f32_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0.0; 4]; - /// unsafe { - /// LittleEndian::read_f32_into(&bytes, &mut numbers_got); - /// } - /// assert_eq!(numbers_given, numbers_got); - /// ``` - fn write_f32_into(src: &[f32], dst: &mut [u8]) { - unimplemented!() - } - - /// Writes IEEE754 double-precision (8 bytes) floating point numbers from - /// `src` into `dst`. - /// - /// # Panics - /// - /// Panics when `src.len() != 8*dst.len()`. - /// - /// # Examples - /// - /// Write and read `f64` numbers in little endian order: - /// - /// ```rust - /// use byteorder::{ByteOrder, LittleEndian}; - /// - /// let mut bytes = [0; 32]; - /// let numbers_given = [1.0, 2.0, 31.312e211, -11.32e91]; - /// LittleEndian::write_f64_into(&numbers_given, &mut bytes); - /// - /// let mut numbers_got = [0.0; 4]; - /// unsafe { - /// LittleEndian::read_f64_into(&bytes, &mut numbers_got); - /// } - /// assert_eq!(numbers_given, numbers_got); - /// ``` - fn write_f64_into(src: &[f64], dst: &mut [u8]) { - unimplemented!() - } - - /// Converts the given slice of unsigned 16 bit integers to a particular - /// endianness. - /// - /// If the endianness matches the endianness of the host platform, then - /// this is a no-op. - /// - /// # Examples - /// - /// Convert the host platform's endianness to big-endian: - /// - /// ```rust - /// use byteorder::{ByteOrder, BigEndian}; - /// - /// let mut numbers = [5, 65000]; - /// BigEndian::from_slice_u16(&mut numbers); - /// assert_eq!(numbers, [5u16.to_be(), 65000u16.to_be()]); - /// ``` - fn from_slice_u16(numbers: &mut [u16]); - - /// Converts the given slice of unsigned 32 bit integers to a particular - /// endianness. - /// - /// If the endianness matches the endianness of the host platform, then - /// this is a no-op. - /// - /// # Examples - /// - /// Convert the host platform's endianness to big-endian: - /// - /// ```rust - /// use byteorder::{ByteOrder, BigEndian}; - /// - /// let mut numbers = [5, 65000]; - /// BigEndian::from_slice_u32(&mut numbers); - /// assert_eq!(numbers, [5u32.to_be(), 65000u32.to_be()]); - /// ``` - fn from_slice_u32(numbers: &mut [u32]); - - /// Converts the given slice of unsigned 64 bit integers to a particular - /// endianness. - /// - /// If the endianness matches the endianness of the host platform, then - /// this is a no-op. - /// - /// # Examples - /// - /// Convert the host platform's endianness to big-endian: - /// - /// ```rust - /// use byteorder::{ByteOrder, BigEndian}; - /// - /// let mut numbers = [5, 65000]; - /// BigEndian::from_slice_u64(&mut numbers); - /// assert_eq!(numbers, [5u64.to_be(), 65000u64.to_be()]); - /// ``` - fn from_slice_u64(numbers: &mut [u64]); - - /// Converts the given slice of unsigned 128 bit integers to a particular - /// endianness. - /// - /// If the endianness matches the endianness of the host platform, then - /// this is a no-op. - /// - /// # Examples - /// - /// Convert the host platform's endianness to big-endian: - /// - /// ```rust - /// use byteorder::{ByteOrder, BigEndian}; - /// - /// let mut numbers = [5, 65000]; - /// BigEndian::from_slice_u128(&mut numbers); - /// assert_eq!(numbers, [5u128.to_be(), 65000u128.to_be()]); - /// ``` - #[cfg(byteorder_i128)] - fn from_slice_u128(numbers: &mut [u128]); - - /// Converts the given slice of signed 16 bit integers to a particular - /// endianness. - /// - /// If the endianness matches the endianness of the host platform, then - /// this is a no-op. - /// - /// # Examples - /// - /// Convert the host platform's endianness to big-endian: - /// - /// ```rust - /// use byteorder::{ByteOrder, BigEndian}; - /// - /// let mut numbers = [5, 6500]; - /// BigEndian::from_slice_i16(&mut numbers); - /// assert_eq!(numbers, [5i16.to_be(), 6500i16.to_be()]); - /// ``` - #[inline] - fn from_slice_i16(src: &mut [i16]) { - let src = unsafe { - identity_transmute::<&mut [i16], &mut [u16]>(src) - }; - Self::from_slice_u16(src); - } - - /// Converts the given slice of signed 32 bit integers to a particular - /// endianness. - /// - /// If the endianness matches the endianness of the host platform, then - /// this is a no-op. - /// - /// # Examples - /// - /// Convert the host platform's endianness to big-endian: - /// - /// ```rust - /// use byteorder::{ByteOrder, BigEndian}; - /// - /// let mut numbers = [5, 65000]; - /// BigEndian::from_slice_i32(&mut numbers); - /// assert_eq!(numbers, [5i32.to_be(), 65000i32.to_be()]); - /// ``` - #[inline] - fn from_slice_i32(src: &mut [i32]) { - let src = unsafe { - identity_transmute::<&mut [i32], &mut [u32]>(src) - }; - Self::from_slice_u32(src); - } - - /// Converts the given slice of signed 64 bit integers to a particular - /// endianness. - /// - /// If the endianness matches the endianness of the host platform, then - /// this is a no-op. - /// - /// # Examples - /// - /// Convert the host platform's endianness to big-endian: - /// - /// ```rust - /// use byteorder::{ByteOrder, BigEndian}; - /// - /// let mut numbers = [5, 65000]; - /// BigEndian::from_slice_i64(&mut numbers); - /// assert_eq!(numbers, [5i64.to_be(), 65000i64.to_be()]); - /// ``` - #[inline] - fn from_slice_i64(src: &mut [i64]) { - let src = unsafe { - identity_transmute::<&mut [i64], &mut [u64]>(src) - }; - Self::from_slice_u64(src); - } - - /// Converts the given slice of signed 128 bit integers to a particular - /// endianness. - /// - /// If the endianness matches the endianness of the host platform, then - /// this is a no-op. - /// - /// # Examples - /// - /// Convert the host platform's endianness to big-endian: - /// - /// ```rust - /// use byteorder::{ByteOrder, BigEndian}; - /// - /// let mut numbers = [5, 65000]; - /// BigEndian::from_slice_i128(&mut numbers); - /// assert_eq!(numbers, [5i128.to_be(), 65000i128.to_be()]); - /// ``` - #[cfg(byteorder_i128)] - #[inline] - fn from_slice_i128(src: &mut [i128]) { - let src = unsafe { - identity_transmute::<&mut [i128], &mut [u128]>(src) - }; - Self::from_slice_u128(src); - } - - /// Converts the given slice of IEEE754 single-precision (4 bytes) floating - /// point numbers to a particular endianness. - /// - /// If the endianness matches the endianness of the host platform, then - /// this is a no-op. - fn from_slice_f32(numbers: &mut [f32]); - - /// Converts the given slice of IEEE754 double-precision (8 bytes) floating - /// point numbers to a particular endianness. - /// - /// If the endianness matches the endianness of the host platform, then - /// this is a no-op. - fn from_slice_f64(numbers: &mut [f64]); -} - -/// Defines big-endian serialization. -/// -/// Note that this type has no value constructor. It is used purely at the -/// type level. -/// -/// # Examples -/// -/// Write and read `u32` numbers in big endian order: -/// -/// ```rust -/// use byteorder::{ByteOrder, BigEndian}; -/// -/// let mut buf = [0; 4]; -/// BigEndian::write_u32(&mut buf, 1_000_000); -/// assert_eq!(1_000_000, BigEndian::read_u32(&buf)); -/// ``` -#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub enum BigEndian {} - -impl Default for BigEndian { - fn default() -> BigEndian { - panic!("BigEndian default") - } -} - -/// A type alias for [`BigEndian`]. -/// -/// [`BigEndian`]: enum.BigEndian.html -pub type BE = BigEndian; - -/// Defines little-endian serialization. -/// -/// Note that this type has no value constructor. It is used purely at the -/// type level. -/// -/// # Examples -/// -/// Write and read `u32` numbers in little endian order: -/// -/// ```rust -/// use byteorder::{ByteOrder, LittleEndian}; -/// -/// let mut buf = [0; 4]; -/// LittleEndian::write_u32(&mut buf, 1_000_000); -/// assert_eq!(1_000_000, LittleEndian::read_u32(&buf)); -/// ``` -#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub enum LittleEndian {} - -impl Default for LittleEndian { - fn default() -> LittleEndian { - panic!("LittleEndian default") - } -} - -/// A type alias for [`LittleEndian`]. -/// -/// [`LittleEndian`]: enum.LittleEndian.html -pub type LE = LittleEndian; - -/// Defines network byte order serialization. -/// -/// Network byte order is defined by [RFC 1700][1] to be big-endian, and is -/// referred to in several protocol specifications. This type is an alias of -/// [`BigEndian`]. -/// -/// [1]: https://tools.ietf.org/html/rfc1700 -/// -/// Note that this type has no value constructor. It is used purely at the -/// type level. -/// -/// # Examples -/// -/// Write and read `i16` numbers in big endian order: -/// -/// ```rust -/// use byteorder::{ByteOrder, NetworkEndian, BigEndian}; -/// -/// let mut buf = [0; 2]; -/// BigEndian::write_i16(&mut buf, -5_000); -/// assert_eq!(-5_000, NetworkEndian::read_i16(&buf)); -/// ``` -/// -/// [`BigEndian`]: enum.BigEndian.html -pub type NetworkEndian = BigEndian; - -/// Defines system native-endian serialization. -/// -/// Note that this type has no value constructor. It is used purely at the -/// type level. -/// -/// On this platform, this is an alias for [`LittleEndian`]. -/// -/// [`LittleEndian`]: enum.LittleEndian.html -#[cfg(target_endian = "little")] -pub type NativeEndian = LittleEndian; - -/// Defines system native-endian serialization. -/// -/// Note that this type has no value constructor. It is used purely at the -/// type level. -/// -/// On this platform, this is an alias for [`BigEndian`]. -/// -/// [`BigEndian`]: enum.BigEndian.html -#[cfg(target_endian = "big")] -pub type NativeEndian = BigEndian; - -macro_rules! safe_read_bytes { - ($src:expr, $ty:ty, $range:expr) => ({ - let mut result: $ty = 0; - for i in $range { - result = (result << 8) | ($src[i] as $ty); - } - result - }); -} - -macro_rules! safe_write_bytes { - ($val:expr, $dst:expr, $range:expr) => ({ - let mut j = 0; - for i in ($range).rev() { - $dst[i] = ($val >> (j * 8)) as u8; - j += 1; - } - }); -} - -macro_rules! safe_read_bytes_rev { - ($src:expr, $ty:ty, $range:expr) => (safe_read_bytes!($src, $ty, $range.rev())); -} - -macro_rules! safe_write_bytes_rev { - ($val:expr, $dst:expr, $range:expr) => (safe_write_bytes!($val, $dst, $range.rev())); -} - -macro_rules! safe_read_slice { - ($src:expr, $dst:expr, $read_one:ident, $step:expr) => ({ - assert!($src.len() == $dst.len() * $step); - let mut base = 0; - for one_dst in $dst { - let one_src = &$src[base .. base + $step]; - *one_dst = Self::$read_one(one_src); - base += $step; - } - }); -} - -macro_rules! safe_write_slice { - ($src:expr, $dst:expr, $write_one:ident, $step:expr) => ({ - assert!($dst.len() == $src.len() * $step); - let mut base = 0; - for one_src in $src { - let one_dst = &mut $dst[base .. base + $step]; - Self::$write_one(one_dst, *one_src); - base += $step; - } - }); -} - -impl ByteOrder for BigEndian { - #[inline] - fn read_u16(buf: &[u8]) -> u16 { - safe_read_bytes!(buf, u16, 0..2) - } - - #[inline] - fn read_u32(buf: &[u8]) -> u32 { - safe_read_bytes!(buf, u32, 0..4) - } - - #[inline] - fn read_u64(buf: &[u8]) -> u64 { - safe_read_bytes!(buf, u64, 0..8) - } - - #[cfg(byteorder_i128)] - #[inline] - fn read_u128(buf: &[u8]) -> u128 { - safe_read_bytes!(buf, u128, 0..16) - } - - #[inline] - fn read_uint(buf: &[u8], nbytes: usize) -> u64 { - assert!(1 <= nbytes && nbytes <= 8 && nbytes <= buf.len()); - safe_read_bytes!(buf, u64, 0..nbytes) - } - - #[cfg(byteorder_i128)] - #[inline] - fn read_uint128(buf: &[u8], nbytes: usize) -> u128 { - assert!(1 <= nbytes && nbytes <= 16 && nbytes <= buf.len()); - safe_read_bytes!(buf, u128, 0..nbytes) - } - - #[inline] - fn write_u16(buf: &mut [u8], n: u16) { - safe_write_bytes!(n, buf, 0..2) - } - - #[inline] - fn write_u32(buf: &mut [u8], n: u32) { - safe_write_bytes!(n, buf, 0..4) - } - - #[inline] - fn write_u64(buf: &mut [u8], n: u64) { - safe_write_bytes!(n, buf, 0..8) - } - - #[cfg(byteorder_i128)] - #[inline] - fn write_u128(buf: &mut [u8], n: u128) { - safe_write_bytes!(n, buf, 0..16) - } - - #[inline] - fn write_uint(buf: &mut [u8], n: u64, nbytes: usize) { - assert!(pack_size(n) <= nbytes && nbytes <= 8); - assert!(nbytes <= buf.len()); - safe_write_bytes!(n, buf, 0..nbytes) - } - - #[cfg(byteorder_i128)] - #[inline] - fn write_uint128(buf: &mut [u8], n: u128, nbytes: usize) { - assert!(pack_size128(n) <= nbytes && nbytes <= 16); - assert!(nbytes <= buf.len()); - safe_write_bytes!(n, buf, 0..nbytes) - } - - #[inline] - fn read_u16_into(src: &[u8], dst: &mut [u16]) { - safe_read_slice!(src, dst, read_u16, 2); - } - - #[inline] - fn read_u32_into(src: &[u8], dst: &mut [u32]) { - safe_read_slice!(src, dst, read_u32, 4); - } - - #[inline] - fn read_u64_into(src: &[u8], dst: &mut [u64]) { - safe_read_slice!(src, dst, read_u64, 8); - } - - #[cfg(byteorder_i128)] - #[inline] - fn read_u128_into(src: &[u8], dst: &mut [u128]) { - safe_read_slice!(src, dst, read_u128, 16); - } - - #[inline] - fn write_u16_into(src: &[u16], dst: &mut [u8]) { - safe_write_slice!(src, dst, write_u16, 2); - } - - #[inline] - fn write_u32_into(src: &[u32], dst: &mut [u8]) { - safe_write_slice!(src, dst, write_u32, 4); - } - - #[inline] - fn write_u64_into(src: &[u64], dst: &mut [u8]) { - safe_write_slice!(src, dst, write_u64, 8); - } - - #[cfg(byteorder_i128)] - #[inline] - fn write_u128_into(src: &[u128], dst: &mut [u8]) { - safe_write_slice!(src, dst, write_u128, 16); - } - - #[inline] - fn from_slice_u16(numbers: &mut [u16]) { - if cfg!(target_endian = "little") { - for n in numbers { - *n = n.to_be(); - } - } - } - - #[inline] - fn from_slice_u32(numbers: &mut [u32]) { - if cfg!(target_endian = "little") { - for n in numbers { - *n = n.to_be(); - } - } - } - - #[inline] - fn from_slice_u64(numbers: &mut [u64]) { - if cfg!(target_endian = "little") { - for n in numbers { - *n = n.to_be(); - } - } - } - - #[cfg(byteorder_i128)] - #[inline] - fn from_slice_u128(numbers: &mut [u128]) { - if cfg!(target_endian = "little") { - for n in numbers { - *n = n.to_be(); - } - } - } - - #[inline] - fn from_slice_f32(numbers: &mut [f32]) { - unimplemented!() - } - - #[inline] - fn from_slice_f64(numbers: &mut [f64]) { - unimplemented!() - } -} - -impl ByteOrder for LittleEndian { - #[inline] - fn read_u16(buf: &[u8]) -> u16 { - safe_read_bytes_rev!(buf, u16, 0..2) - } - - #[inline] - fn read_u32(buf: &[u8]) -> u32 { - safe_read_bytes_rev!(buf, u32, 0..4) - } - - #[inline] - fn read_u64(buf: &[u8]) -> u64 { - safe_read_bytes_rev!(buf, u64, 0..8) - } - - #[cfg(byteorder_i128)] - #[inline] - fn read_u128(buf: &[u8]) -> u128 { - safe_read_bytes_rev!(buf, u128, 0..16) - } - - #[inline] - fn read_uint(buf: &[u8], nbytes: usize) -> u64 { - assert!(1 <= nbytes && nbytes <= 8 && nbytes <= buf.len()); - safe_read_bytes_rev!(buf, u64, 0..nbytes) - } - - #[cfg(byteorder_i128)] - #[inline] - fn read_uint128(buf: &[u8], nbytes: usize) -> u128 { - assert!(1 <= nbytes && nbytes <= 16 && nbytes <= buf.len()); - safe_read_bytes_rev!(buf, u128, 0..nbytes) - } - - #[inline] - fn write_u16(buf: &mut [u8], n: u16) { - safe_write_bytes_rev!(n, buf, 0..2) - } - - #[inline] - fn write_u32(buf: &mut [u8], n: u32) { - safe_write_bytes_rev!(n, buf, 0..4) - } - - #[inline] - fn write_u64(buf: &mut [u8], n: u64) { - safe_write_bytes_rev!(n, buf, 0..8) - } - - #[cfg(byteorder_i128)] - #[inline] - fn write_u128(buf: &mut [u8], n: u128) { - safe_write_bytes_rev!(n, buf, 0..16) - } - - #[inline] - fn write_uint(buf: &mut [u8], n: u64, nbytes: usize) { - assert!(pack_size(n) <= nbytes && nbytes <= 8); - assert!(nbytes <= buf.len()); - safe_write_bytes_rev!(n, buf, 0..nbytes) - } - - #[cfg(byteorder_i128)] - #[inline] - fn write_uint128(buf: &mut [u8], n: u128, nbytes: usize) { - assert!(pack_size128(n) <= nbytes && nbytes <= 16); - assert!(nbytes <= buf.len()); - safe_write_bytes_rev!(n, buf, 0..nbytes) - } - - #[inline] - fn read_u16_into(src: &[u8], dst: &mut [u16]) { - safe_read_slice!(src, dst, read_u16, 2); - } - - #[inline] - fn read_u32_into(src: &[u8], dst: &mut [u32]) { - safe_read_slice!(src, dst, read_u32, 4); - } - - #[inline] - fn read_u64_into(src: &[u8], dst: &mut [u64]) { - safe_read_slice!(src, dst, read_u64, 8); - } - - #[cfg(byteorder_i128)] - #[inline] - fn read_u128_into(src: &[u8], dst: &mut [u128]) { - safe_read_slice!(src, dst, read_u128, 16); - } - - #[inline] - fn write_u16_into(src: &[u16], dst: &mut [u8]) { - safe_write_slice!(src, dst, write_u16, 2); - } - - #[inline] - fn write_u32_into(src: &[u32], dst: &mut [u8]) { - safe_write_slice!(src, dst, write_u32, 4); - } - - #[inline] - fn write_u64_into(src: &[u64], dst: &mut [u8]) { - safe_write_slice!(src, dst, write_u64, 8); - } - - #[cfg(byteorder_i128)] - #[inline] - fn write_u128_into(src: &[u128], dst: &mut [u8]) { - safe_write_slice!(src, dst, write_u128, 16); - } - - #[inline] - fn from_slice_u16(numbers: &mut [u16]) { - if cfg!(target_endian = "big") { - for n in numbers { - *n = n.to_le(); - } - } - } - - #[inline] - fn from_slice_u32(numbers: &mut [u32]) { - if cfg!(target_endian = "big") { - for n in numbers { - *n = n.to_le(); - } - } - } - - #[inline] - fn from_slice_u64(numbers: &mut [u64]) { - if cfg!(target_endian = "big") { - for n in numbers { - *n = n.to_le(); - } - } - } - - #[cfg(byteorder_i128)] - #[inline] - fn from_slice_u128(numbers: &mut [u128]) { - if cfg!(target_endian = "big") { - for n in numbers { - *n = n.to_le(); - } - } - } - - #[inline] - fn from_slice_f32(numbers: &mut [f32]) { - unimplemented!() - } - - #[inline] - fn from_slice_f64(numbers: &mut [f64]) { - unimplemented!() - } -} - -#[cfg(test)] -mod test { - extern crate quickcheck; - extern crate rand; - - use self::quickcheck::{QuickCheck, StdGen, Testable}; - use self::rand::thread_rng; - #[cfg(byteorder_i128)] - use self::rand::Rng; - #[cfg(byteorder_i128)] - use self::quickcheck::{Arbitrary, Gen}; - - pub const U24_MAX: u32 = 16_777_215; - pub const I24_MAX: i32 = 8_388_607; - pub const U48_MAX: u64 = 281_474_976_710_655; - pub const I48_MAX: i64 = 140_737_488_355_327; - - pub const U64_MAX: u64 = ::core::u64::MAX; - pub const I64_MAX: u64 = ::core::i64::MAX as u64; - - macro_rules! calc_max { - ($max:expr, $bytes:expr) => { calc_max!($max, $bytes, 8) }; - ($max:expr, $bytes:expr, $maxbytes:expr) => { - ($max - 1) >> (8 * ($maxbytes - $bytes)) - }; - } - - #[derive(Clone, Debug)] - pub struct Wi128(pub T); - - #[cfg(byteorder_i128)] - impl Wi128 { - pub fn clone(&self) -> T { - self.0.clone() - } - } - - impl PartialEq for Wi128 { - fn eq(&self, other: &T) -> bool { - self.0.eq(other) - } - } - - #[cfg(byteorder_i128)] - impl Arbitrary for Wi128 { - fn arbitrary(gen: &mut G) -> Wi128 { - let max = calc_max!(::core::u128::MAX, gen.size(), 16); - let output = - (gen.gen::() as u128) | - ((gen.gen::() as u128) << 64); - Wi128(output & (max - 1)) - } - } - - #[cfg(byteorder_i128)] - impl Arbitrary for Wi128 { - fn arbitrary(gen: &mut G) -> Wi128 { - let max = calc_max!(::core::i128::MAX, gen.size(), 16); - let output = - (gen.gen::() as i128) | - ((gen.gen::() as i128) << 64); - Wi128(output & (max - 1)) - } - } - - pub fn qc_sized(f: A, size: u64) { - QuickCheck::new() - .gen(StdGen::new(thread_rng(), size as usize)) - .tests(1_00) - .max_tests(10_000) - .quickcheck(f); - } - - macro_rules! qc_byte_order { - ($name:ident, $ty_int:ty, $max:expr, - $bytes:expr, $read:ident, $write:ident) => ( - mod $name { - use {BigEndian, ByteOrder, NativeEndian, LittleEndian}; - #[allow(unused_imports)] use super::{ qc_sized, Wi128 }; - - #[test] - fn big_endian() { - fn prop(n: $ty_int) -> bool { - let mut buf = [0; 16]; - BigEndian::$write(&mut buf, n.clone(), $bytes); - n == BigEndian::$read(&mut buf[..$bytes], $bytes) - } - qc_sized(prop as fn($ty_int) -> bool, $max); - } - - #[test] - fn little_endian() { - fn prop(n: $ty_int) -> bool { - let mut buf = [0; 16]; - LittleEndian::$write(&mut buf, n.clone(), $bytes); - n == LittleEndian::$read(&mut buf[..$bytes], $bytes) - } - qc_sized(prop as fn($ty_int) -> bool, $max); - } - - #[test] - fn native_endian() { - fn prop(n: $ty_int) -> bool { - let mut buf = [0; 16]; - NativeEndian::$write(&mut buf, n.clone(), $bytes); - n == NativeEndian::$read(&mut buf[..$bytes], $bytes) - } - qc_sized(prop as fn($ty_int) -> bool, $max); - } - } - ); - ($name:ident, $ty_int:ty, $max:expr, - $read:ident, $write:ident) => ( - mod $name { - use core::mem::size_of; - use {BigEndian, ByteOrder, NativeEndian, LittleEndian}; - #[allow(unused_imports)] use super::{ qc_sized, Wi128 }; - - #[test] - fn big_endian() { - fn prop(n: $ty_int) -> bool { - let bytes = size_of::<$ty_int>(); - let mut buf = [0; 16]; - BigEndian::$write(&mut buf[16 - bytes..], n.clone()); - n == BigEndian::$read(&mut buf[16 - bytes..]) - } - qc_sized(prop as fn($ty_int) -> bool, $max - 1); - } - - #[test] - fn little_endian() { - fn prop(n: $ty_int) -> bool { - let bytes = size_of::<$ty_int>(); - let mut buf = [0; 16]; - LittleEndian::$write(&mut buf[..bytes], n.clone()); - n == LittleEndian::$read(&mut buf[..bytes]) - } - qc_sized(prop as fn($ty_int) -> bool, $max - 1); - } - - #[test] - fn native_endian() { - fn prop(n: $ty_int) -> bool { - let bytes = size_of::<$ty_int>(); - let mut buf = [0; 16]; - NativeEndian::$write(&mut buf[..bytes], n.clone()); - n == NativeEndian::$read(&mut buf[..bytes]) - } - qc_sized(prop as fn($ty_int) -> bool, $max - 1); - } - } - ); - } - - qc_byte_order!(prop_u16, u16, ::core::u16::MAX as u64, read_u16, write_u16); - qc_byte_order!(prop_i16, i16, ::core::i16::MAX as u64, read_i16, write_i16); - qc_byte_order!(prop_u24, u32, ::test::U24_MAX as u64, read_u24, write_u24); - qc_byte_order!(prop_i24, i32, ::test::I24_MAX as u64, read_i24, write_i24); - qc_byte_order!(prop_u32, u32, ::core::u32::MAX as u64, read_u32, write_u32); - qc_byte_order!(prop_i32, i32, ::core::i32::MAX as u64, read_i32, write_i32); - qc_byte_order!(prop_u48, u64, ::test::U48_MAX as u64, read_u48, write_u48); - qc_byte_order!(prop_i48, i64, ::test::I48_MAX as u64, read_i48, write_i48); - qc_byte_order!(prop_u64, u64, ::core::u64::MAX as u64, read_u64, write_u64); - qc_byte_order!(prop_i64, i64, ::core::i64::MAX as u64, read_i64, write_i64); - //qc_byte_order!(prop_f32, f32, ::core::u64::MAX as u64, read_f32, write_f32); - //qc_byte_order!(prop_f64, f64, ::core::i64::MAX as u64, read_f64, write_f64); - - #[cfg(byteorder_i128)] - qc_byte_order!(prop_u128, Wi128, 16 + 1, read_u128, write_u128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_i128, Wi128, 16 + 1, read_i128, write_i128); - - qc_byte_order!(prop_uint_1, - u64, calc_max!(super::U64_MAX, 1), 1, read_uint, write_uint); - qc_byte_order!(prop_uint_2, - u64, calc_max!(super::U64_MAX, 2), 2, read_uint, write_uint); - qc_byte_order!(prop_uint_3, - u64, calc_max!(super::U64_MAX, 3), 3, read_uint, write_uint); - qc_byte_order!(prop_uint_4, - u64, calc_max!(super::U64_MAX, 4), 4, read_uint, write_uint); - qc_byte_order!(prop_uint_5, - u64, calc_max!(super::U64_MAX, 5), 5, read_uint, write_uint); - qc_byte_order!(prop_uint_6, - u64, calc_max!(super::U64_MAX, 6), 6, read_uint, write_uint); - qc_byte_order!(prop_uint_7, - u64, calc_max!(super::U64_MAX, 7), 7, read_uint, write_uint); - qc_byte_order!(prop_uint_8, - u64, calc_max!(super::U64_MAX, 8), 8, read_uint, write_uint); - - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_1, - Wi128, 1, 1, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_2, - Wi128, 2, 2, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_3, - Wi128, 3, 3, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_4, - Wi128, 4, 4, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_5, - Wi128, 5, 5, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_6, - Wi128, 6, 6, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_7, - Wi128, 7, 7, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_8, - Wi128, 8, 8, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_9, - Wi128, 9, 9, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_10, - Wi128, 10, 10, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_11, - Wi128, 11, 11, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_12, - Wi128, 12, 12, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_13, - Wi128, 13, 13, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_14, - Wi128, 14, 14, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_15, - Wi128, 15, 15, read_uint128, write_uint128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_uint128_16, - Wi128, 16, 16, read_uint128, write_uint128); - - qc_byte_order!(prop_int_1, - i64, calc_max!(super::I64_MAX, 1), 1, read_int, write_int); - qc_byte_order!(prop_int_2, - i64, calc_max!(super::I64_MAX, 2), 2, read_int, write_int); - qc_byte_order!(prop_int_3, - i64, calc_max!(super::I64_MAX, 3), 3, read_int, write_int); - qc_byte_order!(prop_int_4, - i64, calc_max!(super::I64_MAX, 4), 4, read_int, write_int); - qc_byte_order!(prop_int_5, - i64, calc_max!(super::I64_MAX, 5), 5, read_int, write_int); - qc_byte_order!(prop_int_6, - i64, calc_max!(super::I64_MAX, 6), 6, read_int, write_int); - qc_byte_order!(prop_int_7, - i64, calc_max!(super::I64_MAX, 7), 7, read_int, write_int); - qc_byte_order!(prop_int_8, - i64, calc_max!(super::I64_MAX, 8), 8, read_int, write_int); - - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_1, - Wi128, 1, 1, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_2, - Wi128, 2, 2, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_3, - Wi128, 3, 3, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_4, - Wi128, 4, 4, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_5, - Wi128, 5, 5, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_6, - Wi128, 6, 6, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_7, - Wi128, 7, 7, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_8, - Wi128, 8, 8, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_9, - Wi128, 9, 9, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_10, - Wi128, 10, 10, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_11, - Wi128, 11, 11, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_12, - Wi128, 12, 12, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_13, - Wi128, 13, 13, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_14, - Wi128, 14, 14, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_15, - Wi128, 15, 15, read_int128, write_int128); - #[cfg(byteorder_i128)] - qc_byte_order!(prop_int128_16, - Wi128, 16, 16, read_int128, write_int128); - - - // Test that all of the byte conversion functions panic when given a - // buffer that is too small. - // - // These tests are critical to ensure safety, otherwise we might end up - // with a buffer overflow. - macro_rules! too_small { - ($name:ident, $maximally_small:expr, $zero:expr, - $read:ident, $write:ident) => ( - mod $name { - use {BigEndian, ByteOrder, NativeEndian, LittleEndian}; - - #[test] - #[should_panic] - fn read_big_endian() { - let buf = [0; $maximally_small]; - BigEndian::$read(&buf); - } - - #[test] - #[should_panic] - fn read_little_endian() { - let buf = [0; $maximally_small]; - LittleEndian::$read(&buf); - } - - #[test] - #[should_panic] - fn read_native_endian() { - let buf = [0; $maximally_small]; - NativeEndian::$read(&buf); - } - - #[test] - #[should_panic] - fn write_big_endian() { - let mut buf = [0; $maximally_small]; - BigEndian::$write(&mut buf, $zero); - } - - #[test] - #[should_panic] - fn write_little_endian() { - let mut buf = [0; $maximally_small]; - LittleEndian::$write(&mut buf, $zero); - } - - #[test] - #[should_panic] - fn write_native_endian() { - let mut buf = [0; $maximally_small]; - NativeEndian::$write(&mut buf, $zero); - } - } - ); - ($name:ident, $maximally_small:expr, $read:ident) => ( - mod $name { - use {BigEndian, ByteOrder, NativeEndian, LittleEndian}; - - #[test] - #[should_panic] - fn read_big_endian() { - let buf = [0; $maximally_small]; - BigEndian::$read(&buf, $maximally_small + 1); - } - - #[test] - #[should_panic] - fn read_little_endian() { - let buf = [0; $maximally_small]; - LittleEndian::$read(&buf, $maximally_small + 1); - } - - #[test] - #[should_panic] - fn read_native_endian() { - let buf = [0; $maximally_small]; - NativeEndian::$read(&buf, $maximally_small + 1); - } - } - ); - } - - too_small!(small_u16, 1, 0, read_u16, write_u16); - too_small!(small_i16, 1, 0, read_i16, write_i16); - too_small!(small_u32, 3, 0, read_u32, write_u32); - too_small!(small_i32, 3, 0, read_i32, write_i32); - too_small!(small_u64, 7, 0, read_u64, write_u64); - too_small!(small_i64, 7, 0, read_i64, write_i64); - //too_small!(small_f32, 3, 0.0, read_f32, write_f32); - //too_small!(small_f64, 7, 0.0, read_f64, write_f64); - #[cfg(byteorder_i128)] - too_small!(small_u128, 15, 0, read_u128, write_u128); - #[cfg(byteorder_i128)] - too_small!(small_i128, 15, 0, read_i128, write_i128); - - too_small!(small_uint_1, 1, read_uint); - too_small!(small_uint_2, 2, read_uint); - too_small!(small_uint_3, 3, read_uint); - too_small!(small_uint_4, 4, read_uint); - too_small!(small_uint_5, 5, read_uint); - too_small!(small_uint_6, 6, read_uint); - too_small!(small_uint_7, 7, read_uint); - - #[cfg(byteorder_i128)] - too_small!(small_uint128_1, 1, read_uint128); - #[cfg(byteorder_i128)] - too_small!(small_uint128_2, 2, read_uint128); - #[cfg(byteorder_i128)] - too_small!(small_uint128_3, 3, read_uint128); - #[cfg(byteorder_i128)] - too_small!(small_uint128_4, 4, read_uint128); - #[cfg(byteorder_i128)] - too_small!(small_uint128_5, 5, read_uint128); - #[cfg(byteorder_i128)] - too_small!(small_uint128_6, 6, read_uint128); - #[cfg(byteorder_i128)] - too_small!(small_uint128_7, 7, read_uint128); - #[cfg(byteorder_i128)] - too_small!(small_uint128_8, 8, read_uint128); - #[cfg(byteorder_i128)] - too_small!(small_uint128_9, 9, read_uint128); - #[cfg(byteorder_i128)] - too_small!(small_uint128_10, 10, read_uint128); - #[cfg(byteorder_i128)] - too_small!(small_uint128_11, 11, read_uint128); - #[cfg(byteorder_i128)] - too_small!(small_uint128_12, 12, read_uint128); - #[cfg(byteorder_i128)] - too_small!(small_uint128_13, 13, read_uint128); - #[cfg(byteorder_i128)] - too_small!(small_uint128_14, 14, read_uint128); - #[cfg(byteorder_i128)] - too_small!(small_uint128_15, 15, read_uint128); - - too_small!(small_int_1, 1, read_int); - too_small!(small_int_2, 2, read_int); - too_small!(small_int_3, 3, read_int); - too_small!(small_int_4, 4, read_int); - too_small!(small_int_5, 5, read_int); - too_small!(small_int_6, 6, read_int); - too_small!(small_int_7, 7, read_int); - - #[cfg(byteorder_i128)] - too_small!(small_int128_1, 1, read_int128); - #[cfg(byteorder_i128)] - too_small!(small_int128_2, 2, read_int128); - #[cfg(byteorder_i128)] - too_small!(small_int128_3, 3, read_int128); - #[cfg(byteorder_i128)] - too_small!(small_int128_4, 4, read_int128); - #[cfg(byteorder_i128)] - too_small!(small_int128_5, 5, read_int128); - #[cfg(byteorder_i128)] - too_small!(small_int128_6, 6, read_int128); - #[cfg(byteorder_i128)] - too_small!(small_int128_7, 7, read_int128); - #[cfg(byteorder_i128)] - too_small!(small_int128_8, 8, read_int128); - #[cfg(byteorder_i128)] - too_small!(small_int128_9, 9, read_int128); - #[cfg(byteorder_i128)] - too_small!(small_int128_10, 10, read_int128); - #[cfg(byteorder_i128)] - too_small!(small_int128_11, 11, read_int128); - #[cfg(byteorder_i128)] - too_small!(small_int128_12, 12, read_int128); - #[cfg(byteorder_i128)] - too_small!(small_int128_13, 13, read_int128); - #[cfg(byteorder_i128)] - too_small!(small_int128_14, 14, read_int128); - #[cfg(byteorder_i128)] - too_small!(small_int128_15, 15, read_int128); - - // Test that reading/writing slices enforces the correct lengths. - macro_rules! slice_lengths { - ($name:ident, $read:ident, $write:ident, - $num_bytes:expr, $numbers:expr) => { - mod $name { - use {ByteOrder, BigEndian, NativeEndian, LittleEndian}; - - #[test] - #[should_panic] - fn read_big_endian() { - let bytes = [0; $num_bytes]; - let mut numbers = $numbers; - BigEndian::$read(&bytes, &mut numbers); - } - - #[test] - #[should_panic] - fn read_little_endian() { - let bytes = [0; $num_bytes]; - let mut numbers = $numbers; - LittleEndian::$read(&bytes, &mut numbers); - } - - #[test] - #[should_panic] - fn read_native_endian() { - let bytes = [0; $num_bytes]; - let mut numbers = $numbers; - NativeEndian::$read(&bytes, &mut numbers); - } - - #[test] - #[should_panic] - fn write_big_endian() { - let mut bytes = [0; $num_bytes]; - let numbers = $numbers; - BigEndian::$write(&numbers, &mut bytes); - } - - #[test] - #[should_panic] - fn write_little_endian() { - let mut bytes = [0; $num_bytes]; - let numbers = $numbers; - LittleEndian::$write(&numbers, &mut bytes); - } - - #[test] - #[should_panic] - fn write_native_endian() { - let mut bytes = [0; $num_bytes]; - let numbers = $numbers; - NativeEndian::$write(&numbers, &mut bytes); - } - } - } - } - - slice_lengths!( - slice_len_too_small_u16, read_u16_into, write_u16_into, 3, [0, 0]); - slice_lengths!( - slice_len_too_big_u16, read_u16_into, write_u16_into, 5, [0, 0]); - slice_lengths!( - slice_len_too_small_i16, read_i16_into, write_i16_into, 3, [0, 0]); - slice_lengths!( - slice_len_too_big_i16, read_i16_into, write_i16_into, 5, [0, 0]); - - slice_lengths!( - slice_len_too_small_u32, read_u32_into, write_u32_into, 7, [0, 0]); - slice_lengths!( - slice_len_too_big_u32, read_u32_into, write_u32_into, 9, [0, 0]); - slice_lengths!( - slice_len_too_small_i32, read_i32_into, write_i32_into, 7, [0, 0]); - slice_lengths!( - slice_len_too_big_i32, read_i32_into, write_i32_into, 9, [0, 0]); - - slice_lengths!( - slice_len_too_small_u64, read_u64_into, write_u64_into, 15, [0, 0]); - slice_lengths!( - slice_len_too_big_u64, read_u64_into, write_u64_into, 17, [0, 0]); - slice_lengths!( - slice_len_too_small_i64, read_i64_into, write_i64_into, 15, [0, 0]); - slice_lengths!( - slice_len_too_big_i64, read_i64_into, write_i64_into, 17, [0, 0]); - - #[cfg(byteorder_i128)] - slice_lengths!( - slice_len_too_small_u128, read_u128_into, write_u128_into, 31, [0, 0]); - #[cfg(byteorder_i128)] - slice_lengths!( - slice_len_too_big_u128, read_u128_into, write_u128_into, 33, [0, 0]); - #[cfg(byteorder_i128)] - slice_lengths!( - slice_len_too_small_i128, read_i128_into, write_i128_into, 31, [0, 0]); - #[cfg(byteorder_i128)] - slice_lengths!( - slice_len_too_big_i128, read_i128_into, write_i128_into, 33, [0, 0]); - - #[test] - fn uint_bigger_buffer() { - use {ByteOrder, LittleEndian}; - let n = LittleEndian::read_uint(&[1, 2, 3, 4, 5, 6, 7, 8], 5); - assert_eq!(n, 0x0504030201); - } -} - -#[cfg(test)] -#[cfg(feature = "std")] -mod stdtests { - extern crate quickcheck; - extern crate rand; - - use self::quickcheck::{QuickCheck, StdGen, Testable}; - use self::rand::thread_rng; - - fn qc_unsized(f: A) { - - QuickCheck::new() - .gen(StdGen::new(thread_rng(), 16)) - .tests(1_00) - .max_tests(10_000) - .quickcheck(f); - } - - macro_rules! calc_max { - ($max:expr, $bytes:expr) => { ($max - 1) >> (8 * (8 - $bytes)) }; - } - - macro_rules! qc_bytes_ext { - ($name:ident, $ty_int:ty, $max:expr, - $bytes:expr, $read:ident, $write:ident) => ( - mod $name { - use std::io::Cursor; - use { - ReadBytesExt, WriteBytesExt, - BigEndian, NativeEndian, LittleEndian, - }; - #[allow(unused_imports)] use test::{qc_sized, Wi128}; - - #[test] - fn big_endian() { - fn prop(n: $ty_int) -> bool { - let mut wtr = vec![]; - wtr.$write::(n.clone()).unwrap(); - let offset = wtr.len() - $bytes; - let mut rdr = Cursor::new(&mut wtr[offset..]); - n == rdr.$read::($bytes).unwrap() - } - qc_sized(prop as fn($ty_int) -> bool, $max); - } - - #[test] - fn little_endian() { - fn prop(n: $ty_int) -> bool { - let mut wtr = vec![]; - wtr.$write::(n.clone()).unwrap(); - let mut rdr = Cursor::new(wtr); - n == rdr.$read::($bytes).unwrap() - } - qc_sized(prop as fn($ty_int) -> bool, $max); - } - - #[test] - fn native_endian() { - fn prop(n: $ty_int) -> bool { - let mut wtr = vec![]; - wtr.$write::(n.clone()).unwrap(); - let offset = if cfg!(target_endian = "big") { - wtr.len() - $bytes - } else { - 0 - }; - let mut rdr = Cursor::new(&mut wtr[offset..]); - n == rdr.$read::($bytes).unwrap() - } - qc_sized(prop as fn($ty_int) -> bool, $max); - } - } - ); - ($name:ident, $ty_int:ty, $max:expr, $read:ident, $write:ident) => ( - mod $name { - use std::io::Cursor; - use { - ReadBytesExt, WriteBytesExt, - BigEndian, NativeEndian, LittleEndian, - }; - #[allow(unused_imports)] use test::{qc_sized, Wi128}; - - #[test] - fn big_endian() { - fn prop(n: $ty_int) -> bool { - let mut wtr = vec![]; - wtr.$write::(n.clone()).unwrap(); - let mut rdr = Cursor::new(wtr); - n == rdr.$read::().unwrap() - } - qc_sized(prop as fn($ty_int) -> bool, $max - 1); - } - - #[test] - fn little_endian() { - fn prop(n: $ty_int) -> bool { - let mut wtr = vec![]; - wtr.$write::(n.clone()).unwrap(); - let mut rdr = Cursor::new(wtr); - n == rdr.$read::().unwrap() - } - qc_sized(prop as fn($ty_int) -> bool, $max - 1); - } - - #[test] - fn native_endian() { - fn prop(n: $ty_int) -> bool { - let mut wtr = vec![]; - wtr.$write::(n.clone()).unwrap(); - let mut rdr = Cursor::new(wtr); - n == rdr.$read::().unwrap() - } - qc_sized(prop as fn($ty_int) -> bool, $max - 1); - } - } - ); - } - - qc_bytes_ext!(prop_ext_u16, - u16, ::std::u16::MAX as u64, read_u16, write_u16); - qc_bytes_ext!(prop_ext_i16, - i16, ::std::i16::MAX as u64, read_i16, write_i16); - qc_bytes_ext!(prop_ext_u32, - u32, ::std::u32::MAX as u64, read_u32, write_u32); - qc_bytes_ext!(prop_ext_i32, - i32, ::std::i32::MAX as u64, read_i32, write_i32); - qc_bytes_ext!(prop_ext_u64, - u64, ::std::u64::MAX as u64, read_u64, write_u64); - qc_bytes_ext!(prop_ext_i64, - i64, ::std::i64::MAX as u64, read_i64, write_i64); - //qc_bytes_ext!(prop_ext_f32, - // f32, ::std::u64::MAX as u64, read_f32, write_f32); - //qc_bytes_ext!(prop_ext_f64, - // f64, ::std::i64::MAX as u64, read_f64, write_f64); - - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_u128, Wi128, 16 + 1, read_u128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_i128, Wi128, 16 + 1, read_i128, write_i128); - - qc_bytes_ext!(prop_ext_uint_1, - u64, calc_max!(::test::U64_MAX, 1), 1, read_uint, write_u64); - qc_bytes_ext!(prop_ext_uint_2, - u64, calc_max!(::test::U64_MAX, 2), 2, read_uint, write_u64); - qc_bytes_ext!(prop_ext_uint_3, - u64, calc_max!(::test::U64_MAX, 3), 3, read_uint, write_u64); - qc_bytes_ext!(prop_ext_uint_4, - u64, calc_max!(::test::U64_MAX, 4), 4, read_uint, write_u64); - qc_bytes_ext!(prop_ext_uint_5, - u64, calc_max!(::test::U64_MAX, 5), 5, read_uint, write_u64); - qc_bytes_ext!(prop_ext_uint_6, - u64, calc_max!(::test::U64_MAX, 6), 6, read_uint, write_u64); - qc_bytes_ext!(prop_ext_uint_7, - u64, calc_max!(::test::U64_MAX, 7), 7, read_uint, write_u64); - qc_bytes_ext!(prop_ext_uint_8, - u64, calc_max!(::test::U64_MAX, 8), 8, read_uint, write_u64); - - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_1, - Wi128, 1, 1, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_2, - Wi128, 2, 2, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_3, - Wi128, 3, 3, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_4, - Wi128, 4, 4, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_5, - Wi128, 5, 5, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_6, - Wi128, 6, 6, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_7, - Wi128, 7, 7, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_8, - Wi128, 8, 8, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_9, - Wi128, 9, 9, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_10, - Wi128, 10, 10, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_11, - Wi128, 11, 11, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_12, - Wi128, 12, 12, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_13, - Wi128, 13, 13, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_14, - Wi128, 14, 14, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_15, - Wi128, 15, 15, read_uint128, write_u128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_uint128_16, - Wi128, 16, 16, read_uint128, write_u128); - - qc_bytes_ext!(prop_ext_int_1, - i64, calc_max!(::test::I64_MAX, 1), 1, read_int, write_i64); - qc_bytes_ext!(prop_ext_int_2, - i64, calc_max!(::test::I64_MAX, 2), 2, read_int, write_i64); - qc_bytes_ext!(prop_ext_int_3, - i64, calc_max!(::test::I64_MAX, 3), 3, read_int, write_i64); - qc_bytes_ext!(prop_ext_int_4, - i64, calc_max!(::test::I64_MAX, 4), 4, read_int, write_i64); - qc_bytes_ext!(prop_ext_int_5, - i64, calc_max!(::test::I64_MAX, 5), 5, read_int, write_i64); - qc_bytes_ext!(prop_ext_int_6, - i64, calc_max!(::test::I64_MAX, 6), 6, read_int, write_i64); - qc_bytes_ext!(prop_ext_int_7, - i64, calc_max!(::test::I64_MAX, 1), 7, read_int, write_i64); - qc_bytes_ext!(prop_ext_int_8, - i64, calc_max!(::test::I64_MAX, 8), 8, read_int, write_i64); - - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_1, - Wi128, 1, 1, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_2, - Wi128, 2, 2, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_3, - Wi128, 3, 3, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_4, - Wi128, 4, 4, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_5, - Wi128, 5, 5, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_6, - Wi128, 6, 6, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_7, - Wi128, 7, 7, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_8, - Wi128, 8, 8, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_9, - Wi128, 9, 9, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_10, - Wi128, 10, 10, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_11, - Wi128, 11, 11, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_12, - Wi128, 12, 12, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_13, - Wi128, 13, 13, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_14, - Wi128, 14, 14, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_15, - Wi128, 15, 15, read_int128, write_i128); - #[cfg(byteorder_i128)] - qc_bytes_ext!(prop_ext_int128_16, - Wi128, 16, 16, read_int128, write_i128); - - // Test slice serialization/deserialization. - macro_rules! qc_slice { - ($name:ident, $ty_int:ty, $read:ident, $write:ident, $zero:expr) => { - mod $name { - use core::mem::size_of; - use {ByteOrder, BigEndian, NativeEndian, LittleEndian}; - use super::qc_unsized; - #[allow(unused_imports)] - use test::Wi128; - - #[test] - fn big_endian() { - #[allow(unused_unsafe)] - fn prop(numbers: Vec<$ty_int>) -> bool { - let numbers: Vec<_> = numbers - .into_iter() - .map(|x| x.clone()) - .collect(); - let num_bytes = size_of::<$ty_int>() * numbers.len(); - let mut bytes = vec![0; num_bytes]; - - BigEndian::$write(&numbers, &mut bytes); - - let mut got = vec![$zero; numbers.len()]; - unsafe { BigEndian::$read(&bytes, &mut got); } - - numbers == got - } - qc_unsized(prop as fn(_) -> bool); - } - - #[test] - fn little_endian() { - #[allow(unused_unsafe)] - fn prop(numbers: Vec<$ty_int>) -> bool { - let numbers: Vec<_> = numbers - .into_iter() - .map(|x| x.clone()) - .collect(); - let num_bytes = size_of::<$ty_int>() * numbers.len(); - let mut bytes = vec![0; num_bytes]; - - LittleEndian::$write(&numbers, &mut bytes); - - let mut got = vec![$zero; numbers.len()]; - unsafe { LittleEndian::$read(&bytes, &mut got); } - - numbers == got - } - qc_unsized(prop as fn(_) -> bool); - } - - #[test] - fn native_endian() { - #[allow(unused_unsafe)] - fn prop(numbers: Vec<$ty_int>) -> bool { - let numbers: Vec<_> = numbers - .into_iter() - .map(|x| x.clone()) - .collect(); - let num_bytes = size_of::<$ty_int>() * numbers.len(); - let mut bytes = vec![0; num_bytes]; - - NativeEndian::$write(&numbers, &mut bytes); - - let mut got = vec![$zero; numbers.len()]; - unsafe { NativeEndian::$read(&bytes, &mut got); } - - numbers == got - } - qc_unsized(prop as fn(_) -> bool); - } - } - } - } - - qc_slice!(prop_slice_u16, u16, read_u16_into, write_u16_into, 0); - qc_slice!(prop_slice_i16, i16, read_i16_into, write_i16_into, 0); - qc_slice!(prop_slice_u32, u32, read_u32_into, write_u32_into, 0); - qc_slice!(prop_slice_i32, i32, read_i32_into, write_i32_into, 0); - qc_slice!(prop_slice_u64, u64, read_u64_into, write_u64_into, 0); - qc_slice!(prop_slice_i64, i64, read_i64_into, write_i64_into, 0); - #[cfg(byteorder_i128)] - qc_slice!( - prop_slice_u128, Wi128, read_u128_into, write_u128_into, 0); - #[cfg(byteorder_i128)] - qc_slice!( - prop_slice_i128, Wi128, read_i128_into, write_i128_into, 0); - - //qc_slice!( - // prop_slice_f32, f32, read_f32_into, write_f32_into, 0.0); - //qc_slice!( - // prop_slice_f64, f64, read_f64_into, write_f64_into, 0.0); -} diff --git a/crux-mir/lib/bytes.rs b/crux-mir/lib/bytes.rs deleted file mode 100644 index fee6ae1c5..000000000 --- a/crux-mir/lib/bytes.rs +++ /dev/null @@ -1,449 +0,0 @@ -//! Verifier-friendly implementations of the `Bytes` and `BytesMut` types from the `bytes` crate. -//! The implementation plays some tricks to provide some support for buffers of symbolic length, -//! even though the underlying `Vector` type requires the length to be concrete. See the doc -//! comments on `Bytes`, `BytesMut`, `reserve`, and `crux_set_fixed` for details. - -extern crate crucible; - -use std::cmp::{self, Ordering}; -use std::fmt; -use std::hash::{Hash, Hasher}; -use std::io; -use std::iter::Extend; -use std::marker::PhantomData; -use std::mem; -use std::ops::{Deref, DerefMut}; - -use crucible::vector::Vector; - - -/// A view into a byte array. The underlying data is stored in a `Vector` of concrete size, -/// but the start and end can be symbolic. -#[derive(Clone)] -pub struct Bytes { - data: Vector, - start: usize, - end: usize, -} - -/// An owned, mutable byte array. The capacity, which is the size of the underlying `Vector`, -/// is always concrete, but the `len` can be symbolic. Only the first `len` bytes of `data` are -/// considered to be initialized. -#[derive(Clone)] -pub struct BytesMut { - data: Vector, - len: usize, - /// When `true`, the capacity cannot actually grow. Useful when calling code that may try to - /// `reserve` a symbolic amount of capacity. - crux_fixed: bool, -} - - -impl Bytes { - pub fn len(&self) -> usize { - self.end - self.start - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - pub fn split_off(&mut self, at: usize) -> Bytes { - let mid = self.start + at; - let end = self.end; - self.end = mid; - Bytes { data: self.data, start: mid, end } - } - - pub fn split_to(&mut self, at: usize) -> Bytes { - let start = self.start; - let mid = self.start + at; - self.start = mid; - Bytes { data: self.data, start, end: mid } - } -} - -impl BytesMut { - pub fn new() -> BytesMut { - Self::with_capacity(0) - } - - pub fn with_capacity(cap: usize) -> BytesMut { - BytesMut { - data: Vector::replicate(0, cap), - len: 0, - crux_fixed: false, - } - } - - pub fn len(&self) -> usize { - self.len - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Freeze the contents of this buffer, producing a read-only view. - pub fn freeze(self) -> Bytes { - Bytes { data: self.data, start: 0, end: self.len } - } - - pub fn reserve(&mut self, amt: usize) { - if !self.crux_fixed { - let excess = self.data.len() - self.len; - if amt > excess { - let more = amt - excess; - self.data = self.data.concat(Vector::replicate(0, more)); - } - } - } - - pub fn truncate(&mut self, len: usize) { - self.len = cmp::min(len, self.len); - } - - pub fn crux_set_fixed(&mut self, fixed: bool) { - self.crux_fixed = fixed; - } -} - - -pub trait Buf { -} - -pub trait BufMut { - fn put_slice(&mut self, xs: &[u8]); - - fn put_u8(&mut self, x: u8); - - fn put_u16_be(&mut self, x: u16) { - self.put_slice(&x.to_be_bytes()) - } - - fn put_u32_be(&mut self, x: u32) { - self.put_slice(&x.to_be_bytes()) - } - - fn put_u64_be(&mut self, x: u64) { - self.put_slice(&x.to_be_bytes()) - } - - fn put_u128_be(&mut self, x: u128) { - self.put_slice(&x.to_be_bytes()) - } - - fn put_u16_le(&mut self, x: u16) { - self.put_slice(&x.to_le_bytes()) - } - - fn put_u32_le(&mut self, x: u32) { - self.put_slice(&x.to_le_bytes()) - } - - fn put_u64_le(&mut self, x: u64) { - self.put_slice(&x.to_le_bytes()) - } - - fn put_u128_le(&mut self, x: u128) { - self.put_slice(&x.to_le_bytes()) - } - - fn put_i8(&mut self, x: i8) { - self.put_u8(x as u8) - } - - fn put_i16_be(&mut self, x: i16) { - self.put_u16_be(x as u16) - } - - fn put_i32_be(&mut self, x: i32) { - self.put_u32_be(x as u32) - } - - fn put_i64_be(&mut self, x: i64) { - self.put_u64_be(x as u64) - } - - fn put_i128_be(&mut self, x: i128) { - self.put_u128_be(x as u128) - } - - fn put_i16_le(&mut self, x: i16) { - self.put_u16_le(x as u16) - } - - fn put_i32_le(&mut self, x: i32) { - self.put_u32_le(x as u32) - } - - fn put_i64_le(&mut self, x: i64) { - self.put_u64_le(x as u64) - } - - fn put_i128_le(&mut self, x: i128) { - self.put_u128_le(x as u128) - } - - fn writer(self) -> Writer - where Self: Sized; -} - - -impl Buf for Bytes { -} - -impl Buf for BytesMut { -} - -impl BufMut for BytesMut { - fn put_slice(&mut self, xs: &[u8]) { - assert!(xs.len() <= self.data.len() - self.len, - "not enough capacity for put_slice"); - self.data.as_mut_slice()[self.len .. self.len + xs.len()].copy_from_slice(xs); - self.len += xs.len(); - } - - fn put_u8(&mut self, x: u8) { - assert!(1 <= self.data.len() - self.len, - "not enough capacity for put_u8"); - self.data.as_mut_slice()[self.len] = x; - self.len += 1; - } - - fn writer(self) -> Writer - where Self: Sized { - Writer { - inner: self, - } - } -} - -impl BufMut for &mut T { - fn put_slice(&mut self, xs: &[u8]) { - ::put_slice(self, xs) - } - - fn put_u8(&mut self, x: u8) { - ::put_u8(self, x) - } - - fn writer(self) -> Writer - where Self: Sized { - Writer { - inner: self, - } - } -} - - -impl PartialEq for Bytes { - fn eq(&self, other: &Bytes) -> bool { - <[u8] as PartialEq>::eq(self, other) - } - fn ne(&self, other: &Bytes) -> bool { - <[u8] as PartialEq>::ne(self, other) - } -} - -impl PartialOrd for Bytes { - fn partial_cmp(&self, other: &Bytes) -> Option { - <[u8] as PartialOrd>::partial_cmp(self, other) - } - fn lt(&self, other: &Bytes) -> bool { - <[u8] as PartialOrd>::lt(self, other) - } - fn le(&self, other: &Bytes) -> bool { - <[u8] as PartialOrd>::le(self, other) - } - fn gt(&self, other: &Bytes) -> bool { - <[u8] as PartialOrd>::gt(self, other) - } - fn ge(&self, other: &Bytes) -> bool { - <[u8] as PartialOrd>::ge(self, other) - } -} - -impl Eq for Bytes {} - -impl Ord for Bytes { - fn cmp(&self, other: &Bytes) -> Ordering { - <[u8] as Ord>::cmp(self, other) - } -} - -impl fmt::Debug for Bytes { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - <[u8] as fmt::Debug>::fmt(self, fmt) - } -} - -impl Hash for Bytes { - fn hash(&self, state: &mut H) { - <[u8] as Hash>::hash(self, state) - } -} - -impl Deref for Bytes { - type Target = [u8]; - fn deref(&self) -> &[u8] { - &self.data.as_slice()[self.start .. self.end] - } -} - - -impl PartialEq for BytesMut { - fn eq(&self, other: &BytesMut) -> bool { - <[u8] as PartialEq>::eq(self, other) - } - fn ne(&self, other: &BytesMut) -> bool { - <[u8] as PartialEq>::ne(self, other) - } -} - -impl PartialOrd for BytesMut { - fn partial_cmp(&self, other: &BytesMut) -> Option { - <[u8] as PartialOrd>::partial_cmp(self, other) - } - fn lt(&self, other: &BytesMut) -> bool { - <[u8] as PartialOrd>::lt(self, other) - } - fn le(&self, other: &BytesMut) -> bool { - <[u8] as PartialOrd>::le(self, other) - } - fn gt(&self, other: &BytesMut) -> bool { - <[u8] as PartialOrd>::gt(self, other) - } - fn ge(&self, other: &BytesMut) -> bool { - <[u8] as PartialOrd>::ge(self, other) - } -} - -impl Eq for BytesMut {} - -impl Ord for BytesMut { - fn cmp(&self, other: &BytesMut) -> Ordering { - <[u8] as Ord>::cmp(self, other) - } -} - -impl fmt::Debug for BytesMut { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - <[u8] as fmt::Debug>::fmt(self, fmt) - } -} - -impl Hash for BytesMut { - fn hash(&self, state: &mut H) { - <[u8] as Hash>::hash(self, state) - } -} - -impl Deref for BytesMut { - type Target = [u8]; - fn deref(&self) -> &[u8] { - &self.data.as_slice()[.. self.len] - } -} - -impl DerefMut for BytesMut { - fn deref_mut(&mut self) -> &mut [u8] { - &mut self.data.as_mut_slice()[.. self.len] - } -} - - -impl From<&[u8]> for Bytes { - fn from(x: &[u8]) -> Bytes { - Bytes { - data: Vector::copy_from_slice(x), - start: 0, - end: x.len(), - } - } -} - -impl From<&[u8]> for BytesMut { - fn from(x: &[u8]) -> BytesMut { - BytesMut { - data: Vector::copy_from_slice(x), - len: x.len(), - crux_fixed: false, - } - } -} - -impl Extend for BytesMut { - fn extend>(&mut self, iter: I) { - for x in iter { - self.put_u8(x); - } - } -} - -impl<'a> Extend<&'a u8> for BytesMut { - fn extend>(&mut self, iter: I) { - for &x in iter { - self.put_u8(x); - } - } -} - -pub struct Iter { - data: Vector, - idx: usize, - end: usize, -} - -impl IntoIterator for Bytes { - type Item = u8; - type IntoIter = Iter; - fn into_iter(self) -> Iter { - Iter { - data: self.data, - idx: self.start, - end: self.end, - } - } -} - -impl IntoIterator for &Bytes { - type Item = u8; - type IntoIter = Iter; - fn into_iter(self) -> Iter { - Iter { - data: self.data, - idx: self.start, - end: self.end, - } - } -} - -impl Iterator for Iter { - type Item = u8; - fn next(&mut self) -> Option { - if self.idx < self.end { - let val = self.data.as_slice()[self.idx]; - self.idx += 1; - Some(val) - } else { - None - } - } -} - - -pub struct Writer { - inner: T, -} - -impl io::Write for Writer { - fn write(&mut self, buf: &[u8]) -> io::Result { - self.inner.put_slice(buf); - Ok(buf.len()) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} diff --git a/crux-mir/lib/cfg_if/.cargo-ok b/crux-mir/lib/cfg_if/.cargo-ok new file mode 100644 index 000000000..b5754e203 --- /dev/null +++ b/crux-mir/lib/cfg_if/.cargo-ok @@ -0,0 +1 @@ +ok \ No newline at end of file diff --git a/crux-mir/lib/cfg_if/.cargo_vcs_info.json b/crux-mir/lib/cfg_if/.cargo_vcs_info.json new file mode 100644 index 000000000..8f2a21a06 --- /dev/null +++ b/crux-mir/lib/cfg_if/.cargo_vcs_info.json @@ -0,0 +1,5 @@ +{ + "git": { + "sha1": "e60fa1efeab0ec6e90c50d93ec526e1410459c23" + } +} diff --git a/crux-mir/lib/cfg_if/.github/workflows/main.yml b/crux-mir/lib/cfg_if/.github/workflows/main.yml new file mode 100644 index 000000000..8eb6b1720 --- /dev/null +++ b/crux-mir/lib/cfg_if/.github/workflows/main.yml @@ -0,0 +1,56 @@ +name: CI +on: [push, pull_request] + +jobs: + test: + name: Test + runs-on: ubuntu-latest + strategy: + matrix: + rust: [stable, beta, nightly] + steps: + - uses: actions/checkout@master + - name: Install Rust Stable + run: | + rustup self update + rustup update ${{ matrix.rust }} + rustup default ${{ matrix.rust }} + rustc -vV + - name: Run tests + run: cargo test + + rustfmt: + name: Rustfmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Install Rust Stable + run: | + rustup update stable + rustup default stable + rustup component add rustfmt + - name: Run rustfmt + run: cargo fmt -- --check + + publish_docs: + name: Publish Documentation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Install Rust Stable + run: | + rustup update stable + rustup default stable + - name: Build documentation + run: cargo doc --no-deps + - name: Publish documentation + run: | + cd target/doc + git init + git remote add origin https://x-access-token:${{ secrets.github_token }}@github.com/${{ github.repository }} + git fetch origin + git reset --hard "origin/gh-pages^" -- + git add . + git -c user.name='ci' -c user.email='ci' commit -m init + git push -f -q origin HEAD:gh-pages + if: github.event_name == 'push' && github.event.ref == 'refs/heads/master' diff --git a/crux-mir/lib/cfg_if/.gitignore b/crux-mir/lib/cfg_if/.gitignore new file mode 100644 index 000000000..a9d37c560 --- /dev/null +++ b/crux-mir/lib/cfg_if/.gitignore @@ -0,0 +1,2 @@ +target +Cargo.lock diff --git a/crux-mir/lib/cfg_if/Cargo.toml b/crux-mir/lib/cfg_if/Cargo.toml new file mode 100644 index 000000000..13a32ba20 --- /dev/null +++ b/crux-mir/lib/cfg_if/Cargo.toml @@ -0,0 +1,36 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies +# +# If you believe there's an error in this file please file an +# issue against the rust-lang/cargo repository. If you're +# editing this file be aware that the upstream Cargo.toml +# will likely look very different (and much more reasonable) + +[package] +edition = "2018" +name = "cfg-if" +version = "1.0.0" +authors = ["Alex Crichton "] +description = "A macro to ergonomically define an item depending on a large number of #[cfg]\nparameters. Structured like an if-else chain, the first matching branch is the\nitem that gets emitted.\n" +homepage = "https://github.com/alexcrichton/cfg-if" +documentation = "https://docs.rs/cfg-if" +readme = "README.md" +license = "MIT/Apache-2.0" +repository = "https://github.com/alexcrichton/cfg-if" +[dependencies.compiler_builtins] +version = "0.1.2" +optional = true + +[dependencies.core] +version = "1.0.0" +optional = true +package = "rustc-std-workspace-core" + +[features] +rustc-dep-of-std = ["core", "compiler_builtins"] +[badges.travis-ci] +repository = "alexcrichton/cfg-if" diff --git a/crux-mir/lib/cfg-if/Cargo.toml b/crux-mir/lib/cfg_if/Cargo.toml.orig similarity index 97% rename from crux-mir/lib/cfg-if/Cargo.toml rename to crux-mir/lib/cfg_if/Cargo.toml.orig index 01bd1a7ab..0a1331609 100644 --- a/crux-mir/lib/cfg-if/Cargo.toml +++ b/crux-mir/lib/cfg_if/Cargo.toml.orig @@ -1,6 +1,6 @@ [package] name = "cfg-if" -version = "0.1.10" +version = "1.0.0" authors = ["Alex Crichton "] license = "MIT/Apache-2.0" readme = "README.md" diff --git a/crux-mir/lib/cfg-if/LICENSE-APACHE b/crux-mir/lib/cfg_if/LICENSE-APACHE similarity index 100% rename from crux-mir/lib/cfg-if/LICENSE-APACHE rename to crux-mir/lib/cfg_if/LICENSE-APACHE diff --git a/crux-mir/lib/backtrace/crates/backtrace-sys/LICENSE-MIT b/crux-mir/lib/cfg_if/LICENSE-MIT similarity index 100% rename from crux-mir/lib/backtrace/crates/backtrace-sys/LICENSE-MIT rename to crux-mir/lib/cfg_if/LICENSE-MIT diff --git a/crux-mir/lib/cfg-if/README.md b/crux-mir/lib/cfg_if/README.md similarity index 100% rename from crux-mir/lib/cfg-if/README.md rename to crux-mir/lib/cfg_if/README.md diff --git a/crux-mir/lib/cfg-if/src/lib.rs b/crux-mir/lib/cfg_if/src/lib.rs similarity index 93% rename from crux-mir/lib/cfg-if/src/lib.rs rename to crux-mir/lib/cfg_if/src/lib.rs index 6c5058dad..52bbbe0f3 100644 --- a/crux-mir/lib/cfg-if/src/lib.rs +++ b/crux-mir/lib/cfg_if/src/lib.rs @@ -34,30 +34,30 @@ macro_rules! cfg_if { // match if/else chains with a final `else` ($( - if #[cfg($($meta:meta),*)] { $($tokens:tt)* } + if #[cfg($meta:meta)] { $($tokens:tt)* } ) else * else { $($tokens2:tt)* }) => { $crate::cfg_if! { @__items () ; - $( ( ($($meta),*) ($($tokens)*) ), )* + $( ( ($meta) ($($tokens)*) ), )* ( () ($($tokens2)*) ), } }; // match if/else chains lacking a final `else` ( - if #[cfg($($i_met:meta),*)] { $($i_tokens:tt)* } + if #[cfg($i_met:meta)] { $($i_tokens:tt)* } $( - else if #[cfg($($e_met:meta),*)] { $($e_tokens:tt)* } + else if #[cfg($e_met:meta)] { $($e_tokens:tt)* } )* ) => { $crate::cfg_if! { @__items () ; - ( ($($i_met),*) ($($i_tokens)*) ), - $( ( ($($e_met),*) ($($e_tokens)*) ), )* + ( ($i_met) ($($i_tokens)*) ), + $( ( ($e_met) ($($e_tokens)*) ), )* ( () () ), } }; diff --git a/crux-mir/lib/cfg-if/tests/xcrate.rs b/crux-mir/lib/cfg_if/tests/xcrate.rs similarity index 100% rename from crux-mir/lib/cfg-if/tests/xcrate.rs rename to crux-mir/lib/cfg_if/tests/xcrate.rs diff --git a/crux-mir/lib/compiler_builtins/.cargo-ok b/crux-mir/lib/compiler_builtins/.cargo-ok new file mode 100644 index 000000000..b5754e203 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/.cargo-ok @@ -0,0 +1 @@ +ok \ No newline at end of file diff --git a/crux-mir/lib/compiler_builtins/.cargo_vcs_info.json b/crux-mir/lib/compiler_builtins/.cargo_vcs_info.json new file mode 100644 index 000000000..f40943670 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "23a74dec400c4f0e6964dd801509486b019844ef" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/crux-mir/lib/compiler_builtins/Cargo.lock b/crux-mir/lib/compiler_builtins/Cargo.lock new file mode 100644 index 000000000..4784f2917 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/Cargo.lock @@ -0,0 +1,23 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "cc" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7db2f146208d7e0fbee761b09cd65a7f51ccc38705d4e7262dad4d73b12a76b1" + +[[package]] +name = "compiler_builtins" +version = "0.1.85" +dependencies = [ + "cc", + "rustc-std-workspace-core", +] + +[[package]] +name = "rustc-std-workspace-core" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1956f5517128a2b6f23ab2dadf1a976f4f5b27962e7724c2bf3d45e539ec098c" diff --git a/crux-mir/lib/compiler_builtins/Cargo.toml b/crux-mir/lib/compiler_builtins/Cargo.toml index 4d5ffa712..0a3bb6df3 100644 --- a/crux-mir/lib/compiler_builtins/Cargo.toml +++ b/crux-mir/lib/compiler_builtins/Cargo.toml @@ -1,74 +1,72 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + [package] -authors = ["Jorge Aparicio "] name = "compiler_builtins" -version = "0.1.22" -license = "MIT/Apache-2.0" -readme = "README.md" -repository = "https://github.com/rust-lang/compiler-builtins" -homepage = "https://github.com/rust-lang/compiler-builtins" -documentation = "https://docs.rs/compiler_builtins" +version = "0.1.85" +authors = ["Jorge Aparicio "] +links = "compiler-rt" +include = [ + "/Cargo.toml", + "/build.rs", + "/src/*", + "/examples/*", + "/LICENSE.txt", + "/README.md", + "/compiler-rt/*", + "/libm/src/math/*", +] description = """ Compiler intrinsics used by the Rust compiler. Also available for other targets if necessary! """ -include = [ - '/Cargo.toml', - '/build.rs', - '/src/*', - '/examples/*', - '/LICENSE.txt', - '/README.md', - '/compiler-rt/*', - '/libm/src/math/*', -] -links = 'compiler-rt' +homepage = "https://github.com/rust-lang/compiler-builtins" +documentation = "https://docs.rs/compiler_builtins" +readme = "README.md" +license = "MIT/Apache-2.0" +repository = "https://github.com/rust-lang/compiler-builtins" + +[profile.dev] +panic = "abort" + +[profile.release] +panic = "abort" [lib] test = false -[dependencies] -# For more information on this dependency see rust-lang/rust's -# `src/tools/rustc-std-workspace` folder -core = { version = "1.0.0", optional = true, package = 'rustc-std-workspace-core' } +[[example]] +name = "intrinsics" +required-features = ["compiler-builtins"] -[build-dependencies] -cc = { optional = true, version = "1.0" } +[dependencies.core] +version = "1.0.0" +optional = true +package = "rustc-std-workspace-core" [dev-dependencies] -panic-handler = { path = 'crates/panic-handler' } -[features] -default = ["compiler-builtins"] +[build-dependencies.cc] +version = "1.0" +optional = true -# Enable compilation of C code in compiler-rt, filling in some more optimized -# implementations and also filling in unimplemented intrinsics +[features] c = ["cc"] - -# Flag this library as the unstable compiler-builtins lib compiler-builtins = [] - -# Generate memory-related intrinsics like memcpy -mem = [] - -# Mangle all names so this can be linked in with other versions or other -# compiler-rt implementations. Also used for testing +default = ["compiler-builtins"] mangled-names = [] - -# Don't generate lang items for i128 intrisnics and such -no-lang-items = [] - -# Only used in the compiler's build system -rustc-dep-of-std = ['compiler-builtins', 'core'] - -[[example]] -name = "intrinsics" -required-features = ["compiler-builtins"] - -[workspace] -members = ["testcrate"] - -[profile.release] -panic = 'abort' - -[profile.dev] -panic = 'abort' +mem = [] +no-asm = [] +public-test-deps = [] +rustc-dep-of-std = [ + "compiler-builtins", + "core", +] diff --git a/crux-mir/lib/compiler_builtins/Cargo.toml.orig b/crux-mir/lib/compiler_builtins/Cargo.toml.orig new file mode 100644 index 000000000..00998e40f --- /dev/null +++ b/crux-mir/lib/compiler_builtins/Cargo.toml.orig @@ -0,0 +1,79 @@ +[package] +authors = ["Jorge Aparicio "] +name = "compiler_builtins" +version = "0.1.85" +license = "MIT/Apache-2.0" +readme = "README.md" +repository = "https://github.com/rust-lang/compiler-builtins" +homepage = "https://github.com/rust-lang/compiler-builtins" +documentation = "https://docs.rs/compiler_builtins" +description = """ +Compiler intrinsics used by the Rust compiler. Also available for other targets +if necessary! +""" +include = [ + '/Cargo.toml', + '/build.rs', + '/src/*', + '/examples/*', + '/LICENSE.txt', + '/README.md', + '/compiler-rt/*', + '/libm/src/math/*', +] +links = 'compiler-rt' + +[lib] +test = false + +[dependencies] +# For more information on this dependency see rust-lang/rust's +# `src/tools/rustc-std-workspace` folder +core = { version = "1.0.0", optional = true, package = 'rustc-std-workspace-core' } + +[build-dependencies] +cc = { optional = true, version = "1.0" } + +[dev-dependencies] +panic-handler = { path = 'crates/panic-handler' } + +[features] +default = ["compiler-builtins"] + +# Enable compilation of C code in compiler-rt, filling in some more optimized +# implementations and also filling in unimplemented intrinsics +c = ["cc"] + +# Workaround for the Cranelift codegen backend. Disables any implementations +# which use inline assembly and fall back to pure Rust versions (if avalible). +no-asm = [] + +# Flag this library as the unstable compiler-builtins lib +compiler-builtins = [] + +# Generate memory-related intrinsics like memcpy +mem = [] + +# Mangle all names so this can be linked in with other versions or other +# compiler-rt implementations. Also used for testing +mangled-names = [] + +# Only used in the compiler's build system +rustc-dep-of-std = ['compiler-builtins', 'core'] + +# This makes certain traits and function specializations public that +# are not normally public but are required by the `testcrate` +public-test-deps = [] + +[[example]] +name = "intrinsics" +required-features = ["compiler-builtins"] + +[workspace] +members = ["testcrate"] + +[profile.release] +panic = 'abort' + +[profile.dev] +panic = 'abort' diff --git a/crux-mir/lib/compiler_builtins/LICENSE.TXT b/crux-mir/lib/compiler_builtins/LICENSE.txt similarity index 100% rename from crux-mir/lib/compiler_builtins/LICENSE.TXT rename to crux-mir/lib/compiler_builtins/LICENSE.txt diff --git a/crux-mir/lib/compiler_builtins/PUBLISHING.md b/crux-mir/lib/compiler_builtins/PUBLISHING.md deleted file mode 100644 index 3df682ab0..000000000 --- a/crux-mir/lib/compiler_builtins/PUBLISHING.md +++ /dev/null @@ -1,16 +0,0 @@ -# Publishing to crates.io - -Publishing `compiler-builtins` to crates.io takes a few steps unfortunately. -It's not great, but it works for now. PRs to improve this process would be -greatly appreciated! - -1. Make sure you've got a clean working tree and it's updated with the latest - changes on `master` -2. Edit `Cargo.toml` to bump the version number -3. Commit this change -4. Run `git tag` to create a tag for this version -5. Delete the `libm/Cargo.toml` file -6. Run `cargo +nightly publish` -7. Push the tag -8. Push the commit -9. Undo changes to `Cargo.toml` and the `libm` submodule diff --git a/crux-mir/lib/compiler_builtins/README.md b/crux-mir/lib/compiler_builtins/README.md index a20d038f8..8b25558a8 100644 --- a/crux-mir/lib/compiler_builtins/README.md +++ b/crux-mir/lib/compiler_builtins/README.md @@ -126,7 +126,7 @@ features = ["c"] - [x] arm/softfloat-alias.list - [x] arm/subdf3vfp.S - [x] arm/subsf3vfp.S -- [ ] arm/truncdfsf2vfp.S +- [x] arm/truncdfsf2vfp.S - [ ] arm/udivmodsi4.S (generic version is done) - [ ] arm/udivsi3.S (generic version is done) - [ ] arm/umodsi3.S (generic version is done) @@ -183,7 +183,7 @@ features = ["c"] - [x] subdf3.c - [x] subsf3.c - [ ] truncdfhf2.c -- [ ] truncdfsf2.c +- [x] truncdfsf2.c - [ ] truncsfhf2.c - [x] udivdi3.c - [x] udivmoddi4.c diff --git a/crux-mir/lib/compiler_builtins/build.rs b/crux-mir/lib/compiler_builtins/build.rs index b520b6247..3f5dbd3ab 100644 --- a/crux-mir/lib/compiler_builtins/build.rs +++ b/crux-mir/lib/compiler_builtins/build.rs @@ -23,14 +23,26 @@ fn main() { return; } - // Forcibly enable memory intrinsics on wasm32 & SGX as we don't have a libc to + // Forcibly enable memory intrinsics on wasm & SGX as we don't have a libc to // provide them. - if (target.contains("wasm32") && !target.contains("wasi")) + if (target.contains("wasm") && !target.contains("wasi")) || (target.contains("sgx") && target.contains("fortanix")) + || target.contains("-none") + || target.contains("nvptx") + || target.contains("uefi") { println!("cargo:rustc-cfg=feature=\"mem\""); } + // These targets have hardware unaligned access support. + if target.contains("x86_64") + || target.contains("i686") + || target.contains("aarch64") + || target.contains("bpf") + { + println!("cargo:rustc-cfg=feature=\"mem-unaligned\""); + } + // NOTE we are going to assume that llvm-target, what determines our codegen option, matches the // target triple. This is usually correct for our built-in targets but can break in presence of // custom targets, which can have arbitrary names. @@ -43,15 +55,17 @@ fn main() { if !cfg!(feature = "mangled-names") && cfg!(feature = "c") { // Don't use a C compiler for these targets: // - // * wasm32 - clang 8 for wasm is somewhat hard to come by and it's + // * wasm - clang for wasm is somewhat hard to come by and it's // unlikely that the C is really that much better than our own Rust. // * nvptx - everything is bitcode, not compatible with mixed C/Rust // * riscv - the rust-lang/rust distribution container doesn't have a C - // compiler nor is cc-rs ready for compilation to riscv (at this - // time). This can probably be removed in the future - if !target.contains("wasm32") && !target.contains("nvptx") && !target.starts_with("riscv") { + // compiler. + if !target.contains("wasm") + && !target.contains("nvptx") + && (!target.starts_with("riscv") || target.contains("xous")) + { #[cfg(feature = "c")] - c::compile(&llvm_target); + c::compile(&llvm_target, &target); } } @@ -67,8 +81,13 @@ fn main() { println!("cargo:rustc-cfg=thumb_1") } - // Only emit the ARM Linux atomic emulation on pre-ARMv6 architectures. - if llvm_target[0] == "armv4t" || llvm_target[0] == "armv5te" { + // Only emit the ARM Linux atomic emulation on pre-ARMv6 architectures. This + // includes the old androideabi. It is deprecated but it is available as a + // rustc target (arm-linux-androideabi). + if llvm_target[0] == "armv4t" + || llvm_target[0] == "armv5te" + || target == "arm-linux-androideabi" + { println!("cargo:rustc-cfg=kernel_user_helpers") } } @@ -77,9 +96,11 @@ fn main() { mod c { extern crate cc; - use std::collections::BTreeMap; + use std::collections::{BTreeMap, HashSet}; use std::env; - use std::path::PathBuf; + use std::fs::{self, File}; + use std::io::Write; + use std::path::{Path, PathBuf}; struct Sources { // SYMBOL -> PATH TO SOURCE @@ -121,13 +142,28 @@ mod c { } /// Compile intrinsics from the compiler-rt C source code - pub fn compile(llvm_target: &[&str]) { + pub fn compile(llvm_target: &[&str], target: &String) { let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); let target_env = env::var("CARGO_CFG_TARGET_ENV").unwrap(); let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap(); let target_vendor = env::var("CARGO_CFG_TARGET_VENDOR").unwrap(); + let mut consider_float_intrinsics = true; let cfg = &mut cc::Build::new(); + // AArch64 GCCs exit with an error condition when they encounter any kind of floating point + // code if the `nofp` and/or `nosimd` compiler flags have been set. + // + // Therefore, evaluate if those flags are present and set a boolean that causes any + // compiler-rt intrinsics that contain floating point source to be excluded for this target. + if target_arch == "aarch64" { + let cflags_key = String::from("CFLAGS_") + &(target.to_owned().replace("-", "_")); + if let Ok(cflags_value) = env::var(cflags_key) { + if cflags_value.contains("+nofp") || cflags_value.contains("+nosimd") { + consider_float_intrinsics = false; + } + } + } + cfg.warnings(false); if target_env == "msvc" { @@ -154,46 +190,64 @@ mod c { cfg.define("VISIBILITY_HIDDEN", None); } + // int_util.c tries to include stdlib.h if `_WIN32` is defined, + // which it is when compiling UEFI targets with clang. This is + // at odds with compiling with `-ffreestanding`, as the header + // may be incompatible or not present. Create a minimal stub + // header to use instead. + if target_os == "uefi" { + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + let include_dir = out_dir.join("include"); + if !include_dir.exists() { + fs::create_dir(&include_dir).unwrap(); + } + fs::write(include_dir.join("stdlib.h"), "#include ").unwrap(); + cfg.flag(&format!("-I{}", include_dir.to_str().unwrap())); + } + let mut sources = Sources::new(); sources.extend(&[ ("__absvdi2", "absvdi2.c"), ("__absvsi2", "absvsi2.c"), ("__addvdi3", "addvdi3.c"), ("__addvsi3", "addvsi3.c"), - ("apple_versioning", "apple_versioning.c"), ("__clzdi2", "clzdi2.c"), ("__clzsi2", "clzsi2.c"), ("__cmpdi2", "cmpdi2.c"), ("__ctzdi2", "ctzdi2.c"), ("__ctzsi2", "ctzsi2.c"), - ("__divdc3", "divdc3.c"), - ("__divsc3", "divsc3.c"), - ("__divxc3", "divxc3.c"), - ("__extendhfsf2", "extendhfsf2.c"), ("__int_util", "int_util.c"), - ("__muldc3", "muldc3.c"), - ("__mulsc3", "mulsc3.c"), ("__mulvdi3", "mulvdi3.c"), ("__mulvsi3", "mulvsi3.c"), - ("__mulxc3", "mulxc3.c"), - ("__negdf2", "negdf2.c"), ("__negdi2", "negdi2.c"), - ("__negsf2", "negsf2.c"), ("__negvdi2", "negvdi2.c"), ("__negvsi2", "negvsi2.c"), ("__paritydi2", "paritydi2.c"), ("__paritysi2", "paritysi2.c"), ("__popcountdi2", "popcountdi2.c"), ("__popcountsi2", "popcountsi2.c"), - ("__powixf2", "powixf2.c"), ("__subvdi3", "subvdi3.c"), ("__subvsi3", "subvsi3.c"), - ("__truncdfhf2", "truncdfhf2.c"), - ("__truncdfsf2", "truncdfsf2.c"), - ("__truncsfhf2", "truncsfhf2.c"), ("__ucmpdi2", "ucmpdi2.c"), ]); + if consider_float_intrinsics { + sources.extend(&[ + ("__divdc3", "divdc3.c"), + ("__divsc3", "divsc3.c"), + ("__divxc3", "divxc3.c"), + ("__extendhfsf2", "extendhfsf2.c"), + ("__muldc3", "muldc3.c"), + ("__mulsc3", "mulsc3.c"), + ("__mulxc3", "mulxc3.c"), + ("__negdf2", "negdf2.c"), + ("__negsf2", "negsf2.c"), + ("__powixf2", "powixf2.c"), + ("__truncdfhf2", "truncdfhf2.c"), + ("__truncsfhf2", "truncsfhf2.c"), + ]); + } + // When compiling in rustbuild (the rust-lang/rust repo) this library // also needs to satisfy intrinsics that jemalloc or C in general may // need, so include a few more that aren't typically needed by @@ -204,7 +258,10 @@ mod c { // On iOS and 32-bit OSX these are all just empty intrinsics, no need to // include them. - if target_os != "ios" && (target_vendor != "apple" || target_arch != "x86") { + if target_os != "ios" + && target_os != "watchos" + && (target_vendor != "apple" || target_arch != "x86") + { sources.extend(&[ ("__absvti2", "absvti2.c"), ("__addvti3", "addvti3.c"), @@ -214,12 +271,15 @@ mod c { ("__ffsti2", "ffsti2.c"), ("__mulvti3", "mulvti3.c"), ("__negti2", "negti2.c"), - ("__negvti2", "negvti2.c"), ("__parityti2", "parityti2.c"), ("__popcountti2", "popcountti2.c"), ("__subvti3", "subvti3.c"), ("__ucmpti2", "ucmpti2.c"), ]); + + if consider_float_intrinsics { + sources.extend(&[("__negvti2", "negvti2.c")]); + } } if target_vendor == "apple" { @@ -238,10 +298,7 @@ mod c { if target_env == "msvc" { if target_arch == "x86_64" { - sources.extend(&[ - ("__floatdisf", "x86_64/floatdisf.c"), - ("__floatdixf", "x86_64/floatdixf.c"), - ]); + sources.extend(&[("__floatdixf", "x86_64/floatdixf.c")]); } } else { // None of these seem to be used on x86_64 windows, and they've all @@ -249,10 +306,7 @@ mod c { if target_os != "windows" { if target_arch == "x86_64" { sources.extend(&[ - ("__floatdisf", "x86_64/floatdisf.c"), ("__floatdixf", "x86_64/floatdixf.c"), - ("__floatundidf", "x86_64/floatundidf.S"), - ("__floatundisf", "x86_64/floatundisf.S"), ("__floatundixf", "x86_64/floatundixf.S"), ]); } @@ -263,11 +317,7 @@ mod c { ("__ashldi3", "i386/ashldi3.S"), ("__ashrdi3", "i386/ashrdi3.S"), ("__divdi3", "i386/divdi3.S"), - ("__floatdidf", "i386/floatdidf.S"), - ("__floatdisf", "i386/floatdisf.S"), ("__floatdixf", "i386/floatdixf.S"), - ("__floatundidf", "i386/floatundidf.S"), - ("__floatundisf", "i386/floatundisf.S"), ("__floatundixf", "i386/floatundixf.S"), ("__lshrdi3", "i386/lshrdi3.S"), ("__moddi3", "i386/moddi3.S"), @@ -278,7 +328,11 @@ mod c { } } - if target_arch == "arm" && target_os != "ios" && target_env != "msvc" { + if target_arch == "arm" + && target_os != "ios" + && target_os != "watchos" + && target_env != "msvc" + { sources.extend(&[ ("__aeabi_div0", "arm/aeabi_div0.c"), ("__aeabi_drsub", "arm/aeabi_drsub.c"), @@ -372,7 +426,7 @@ mod c { ]); } - if target_arch == "aarch64" { + if target_arch == "aarch64" && consider_float_intrinsics { sources.extend(&[ ("__comparetf2", "comparetf2.c"), ("__extenddftf2", "extenddftf2.c"), @@ -389,6 +443,13 @@ mod c { ("__floatunsitf", "floatunsitf.c"), ("__trunctfdf2", "trunctfdf2.c"), ("__trunctfsf2", "trunctfsf2.c"), + ("__addtf3", "addtf3.c"), + ("__multf3", "multf3.c"), + ("__subtf3", "subtf3.c"), + ("__divtf3", "divtf3.c"), + ("__powitf2", "powitf2.c"), + ("__fe_getround", "fp_mode.c"), + ("__fe_raise_inexact", "fp_mode.c"), ]); if target_os != "windows" { @@ -396,8 +457,31 @@ mod c { } } + if target_arch == "mips" { + sources.extend(&[("__bswapsi2", "bswapsi2.c")]); + } + + if target_arch == "mips64" { + sources.extend(&[ + ("__extenddftf2", "extenddftf2.c"), + ("__netf2", "comparetf2.c"), + ("__addtf3", "addtf3.c"), + ("__multf3", "multf3.c"), + ("__subtf3", "subtf3.c"), + ("__fixtfsi", "fixtfsi.c"), + ("__floatsitf", "floatsitf.c"), + ("__fixunstfsi", "fixunstfsi.c"), + ("__floatunsitf", "floatunsitf.c"), + ("__fe_getround", "fp_mode.c"), + ("__divtf3", "divtf3.c"), + ("__trunctfdf2", "trunctfdf2.c"), + ("__trunctfsf2", "trunctfsf2.c"), + ]); + } + // Remove the assembly implementations that won't compile for the target - if llvm_target[0] == "thumbv6m" || llvm_target[0] == "thumbv8m.base" { + if llvm_target[0] == "thumbv6m" || llvm_target[0] == "thumbv8m.base" || target_os == "uefi" + { let mut to_remove = Vec::new(); for (k, v) in sources.map.iter() { if v.ends_with(".S") { @@ -414,6 +498,16 @@ mod c { sources.remove(&["__aeabi_cdcmp", "__aeabi_cfcmp"]); } + // Android uses emulated TLS so we need a runtime support function. + if target_os == "android" { + sources.extend(&[("__emutls_get_address", "emutls.c")]); + + // Work around a bug in the NDK headers (fixed in + // https://r.android.com/2038949 which will be released in a future + // NDK version) by providing a definition of LONG_BIT. + cfg.define("LONG_BIT", "(8 * sizeof(long))"); + } + // When compiling the C code we require the user to tell us where the // source code is, and this is largely done so when we're compiling as // part of rust-lang/rust we can use the same llvm-project repository as @@ -431,14 +525,75 @@ mod c { // use of that macro in lib/builtins/int_util.h in compiler-rt. cfg.flag_if_supported(&format!("-ffile-prefix-map={}=.", root.display())); + // Include out-of-line atomics for aarch64, which are all generated by supplying different + // sets of flags to the same source file. + // Note: Out-of-line aarch64 atomics are not supported by the msvc toolchain (#430). let src_dir = root.join("lib/builtins"); + if target_arch == "aarch64" && target_env != "msvc" { + // See below for why we're building these as separate libraries. + build_aarch64_out_of_line_atomics_libraries(&src_dir, cfg); + + // Some run-time CPU feature detection is necessary, as well. + sources.extend(&[("__aarch64_have_lse_atomics", "cpu_model.c")]); + } + + let mut added_sources = HashSet::new(); for (sym, src) in sources.map.iter() { let src = src_dir.join(src); - cfg.file(&src); - println!("cargo:rerun-if-changed={}", src.display()); + if added_sources.insert(src.clone()) { + cfg.file(&src); + println!("cargo:rerun-if-changed={}", src.display()); + } println!("cargo:rustc-cfg={}=\"optimized-c\"", sym); } cfg.compile("libcompiler-rt.a"); } + + fn build_aarch64_out_of_line_atomics_libraries(builtins_dir: &Path, cfg: &mut cc::Build) { + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + let outlined_atomics_file = builtins_dir.join("aarch64/lse.S"); + println!("cargo:rerun-if-changed={}", outlined_atomics_file.display()); + + cfg.include(&builtins_dir); + + for instruction_type in &["cas", "swp", "ldadd", "ldclr", "ldeor", "ldset"] { + for size in &[1, 2, 4, 8, 16] { + if *size == 16 && *instruction_type != "cas" { + continue; + } + + for (model_number, model_name) in + &[(1, "relax"), (2, "acq"), (3, "rel"), (4, "acq_rel")] + { + // The original compiler-rt build system compiles the same + // source file multiple times with different compiler + // options. Here we do something slightly different: we + // create multiple .S files with the proper #defines and + // then include the original file. + // + // This is needed because the cc crate doesn't allow us to + // override the name of object files and libtool requires + // all objects in an archive to have unique names. + let path = + out_dir.join(format!("lse_{}{}_{}.S", instruction_type, size, model_name)); + let mut file = File::create(&path).unwrap(); + writeln!(file, "#define L_{}", instruction_type).unwrap(); + writeln!(file, "#define SIZE {}", size).unwrap(); + writeln!(file, "#define MODEL {}", model_number).unwrap(); + writeln!( + file, + "#include \"{}\"", + outlined_atomics_file.canonicalize().unwrap().display() + ) + .unwrap(); + drop(file); + cfg.file(path); + + let sym = format!("__aarch64_{}{}_{}", instruction_type, size, model_name); + println!("cargo:rustc-cfg={}=\"optimized-c\"", sym); + } + } + } + } } diff --git a/crux-mir/lib/compiler_builtins/ci/docker/aarch64-unknown-linux-gnu/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/aarch64-unknown-linux-gnu/Dockerfile deleted file mode 100644 index 9e2559f4a..000000000 --- a/crux-mir/lib/compiler_builtins/ci/docker/aarch64-unknown-linux-gnu/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM ubuntu:18.04 -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - gcc libc6-dev ca-certificates \ - gcc-aarch64-linux-gnu libc6-dev-arm64-cross \ - qemu-user-static -ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \ - CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER=qemu-aarch64-static \ - QEMU_LD_PREFIX=/usr/aarch64-linux-gnu \ - RUST_TEST_THREADS=1 diff --git a/crux-mir/lib/compiler_builtins/ci/docker/arm-unknown-linux-gnueabi/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/arm-unknown-linux-gnueabi/Dockerfile deleted file mode 100644 index afab874bc..000000000 --- a/crux-mir/lib/compiler_builtins/ci/docker/arm-unknown-linux-gnueabi/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM ubuntu:18.04 -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - gcc libc6-dev ca-certificates \ - gcc-arm-linux-gnueabi libc6-dev-armel-cross qemu-user-static -ENV CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABI_LINKER=arm-linux-gnueabi-gcc \ - CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABI_RUNNER=qemu-arm-static \ - QEMU_LD_PREFIX=/usr/arm-linux-gnueabi \ - RUST_TEST_THREADS=1 diff --git a/crux-mir/lib/compiler_builtins/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile deleted file mode 100644 index 3ed3602b0..000000000 --- a/crux-mir/lib/compiler_builtins/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM ubuntu:18.04 -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - gcc libc6-dev ca-certificates \ - gcc-arm-linux-gnueabihf libc6-dev-armhf-cross qemu-user-static -ENV CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_LINKER=arm-linux-gnueabihf-gcc \ - CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_RUNNER=qemu-arm-static \ - QEMU_LD_PREFIX=/usr/arm-linux-gnueabihf \ - RUST_TEST_THREADS=1 diff --git a/crux-mir/lib/compiler_builtins/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile deleted file mode 100644 index 6617af155..000000000 --- a/crux-mir/lib/compiler_builtins/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM ubuntu:18.04 -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - gcc libc6-dev ca-certificates \ - gcc-arm-linux-gnueabihf libc6-dev-armhf-cross qemu-user-static -ENV CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER=arm-linux-gnueabihf-gcc \ - CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_RUNNER=qemu-arm-static \ - QEMU_LD_PREFIX=/usr/arm-linux-gnueabihf \ - RUST_TEST_THREADS=1 diff --git a/crux-mir/lib/compiler_builtins/ci/docker/i586-unknown-linux-gnu/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/i586-unknown-linux-gnu/Dockerfile deleted file mode 100644 index 5783e28e1..000000000 --- a/crux-mir/lib/compiler_builtins/ci/docker/i586-unknown-linux-gnu/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM ubuntu:18.04 -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - gcc-multilib libc6-dev ca-certificates diff --git a/crux-mir/lib/compiler_builtins/ci/docker/i686-unknown-linux-gnu/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/i686-unknown-linux-gnu/Dockerfile deleted file mode 100644 index 5783e28e1..000000000 --- a/crux-mir/lib/compiler_builtins/ci/docker/i686-unknown-linux-gnu/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM ubuntu:18.04 -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - gcc-multilib libc6-dev ca-certificates diff --git a/crux-mir/lib/compiler_builtins/ci/docker/mips-unknown-linux-gnu/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/mips-unknown-linux-gnu/Dockerfile deleted file mode 100644 index f47e8f522..000000000 --- a/crux-mir/lib/compiler_builtins/ci/docker/mips-unknown-linux-gnu/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM ubuntu:18.04 - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - gcc libc6-dev ca-certificates \ - gcc-mips-linux-gnu libc6-dev-mips-cross \ - binfmt-support qemu-user-static qemu-system-mips - -ENV CARGO_TARGET_MIPS_UNKNOWN_LINUX_GNU_LINKER=mips-linux-gnu-gcc \ - CARGO_TARGET_MIPS_UNKNOWN_LINUX_GNU_RUNNER=qemu-mips-static \ - QEMU_LD_PREFIX=/usr/mips-linux-gnu \ - RUST_TEST_THREADS=1 diff --git a/crux-mir/lib/compiler_builtins/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile deleted file mode 100644 index 8fa77c7bd..000000000 --- a/crux-mir/lib/compiler_builtins/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM ubuntu:18.04 -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - ca-certificates \ - gcc \ - gcc-mips64-linux-gnuabi64 \ - libc6-dev \ - libc6-dev-mips64-cross \ - qemu-user-static \ - qemu-system-mips -ENV CARGO_TARGET_MIPS64_UNKNOWN_LINUX_GNUABI64_LINKER=mips64-linux-gnuabi64-gcc \ - CARGO_TARGET_MIPS64_UNKNOWN_LINUX_GNUABI64_RUNNER=qemu-mips64-static \ - CC_mips64_unknown_linux_gnuabi64=mips64-linux-gnuabi64-gcc \ - QEMU_LD_PREFIX=/usr/mips64-linux-gnuabi64 \ - RUST_TEST_THREADS=1 diff --git a/crux-mir/lib/compiler_builtins/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile deleted file mode 100644 index c6611d9ac..000000000 --- a/crux-mir/lib/compiler_builtins/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM ubuntu:18.04 -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - ca-certificates \ - gcc \ - gcc-mips64el-linux-gnuabi64 \ - libc6-dev \ - libc6-dev-mips64el-cross \ - qemu-user-static -ENV CARGO_TARGET_MIPS64EL_UNKNOWN_LINUX_GNUABI64_LINKER=mips64el-linux-gnuabi64-gcc \ - CARGO_TARGET_MIPS64EL_UNKNOWN_LINUX_GNUABI64_RUNNER=qemu-mips64el-static \ - CC_mips64el_unknown_linux_gnuabi64=mips64el-linux-gnuabi64-gcc \ - QEMU_LD_PREFIX=/usr/mips64el-linux-gnuabi64 \ - RUST_TEST_THREADS=1 diff --git a/crux-mir/lib/compiler_builtins/ci/docker/mipsel-unknown-linux-gnu/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/mipsel-unknown-linux-gnu/Dockerfile deleted file mode 100644 index 0bc695624..000000000 --- a/crux-mir/lib/compiler_builtins/ci/docker/mipsel-unknown-linux-gnu/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM ubuntu:18.04 - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - gcc libc6-dev ca-certificates \ - gcc-mipsel-linux-gnu libc6-dev-mipsel-cross \ - binfmt-support qemu-user-static - -ENV CARGO_TARGET_MIPSEL_UNKNOWN_LINUX_GNU_LINKER=mipsel-linux-gnu-gcc \ - CARGO_TARGET_MIPSEL_UNKNOWN_LINUX_GNU_RUNNER=qemu-mipsel-static \ - QEMU_LD_PREFIX=/usr/mipsel-linux-gnu \ - RUST_TEST_THREADS=1 diff --git a/crux-mir/lib/compiler_builtins/ci/docker/powerpc-unknown-linux-gnu/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/powerpc-unknown-linux-gnu/Dockerfile deleted file mode 100644 index 2d39fef61..000000000 --- a/crux-mir/lib/compiler_builtins/ci/docker/powerpc-unknown-linux-gnu/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM ubuntu:18.04 - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - gcc libc6-dev qemu-user-static ca-certificates \ - gcc-powerpc-linux-gnu libc6-dev-powerpc-cross \ - qemu-system-ppc - -ENV CARGO_TARGET_POWERPC_UNKNOWN_LINUX_GNU_LINKER=powerpc-linux-gnu-gcc \ - CARGO_TARGET_POWERPC_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc-static \ - QEMU_LD_PREFIX=/usr/powerpc-linux-gnu \ - RUST_TEST_THREADS=1 diff --git a/crux-mir/lib/compiler_builtins/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile deleted file mode 100644 index 653cd3511..000000000 --- a/crux-mir/lib/compiler_builtins/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM ubuntu:18.04 - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - gcc libc6-dev ca-certificates \ - gcc-powerpc64-linux-gnu libc6-dev-ppc64-cross \ - binfmt-support qemu-user-static qemu-system-ppc - -ENV CARGO_TARGET_POWERPC64_UNKNOWN_LINUX_GNU_LINKER=powerpc64-linux-gnu-gcc \ - CARGO_TARGET_POWERPC64_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc64-static \ - CC_powerpc64_unknown_linux_gnu=powerpc64-linux-gnu-gcc \ - QEMU_LD_PREFIX=/usr/powerpc64-linux-gnu \ - RUST_TEST_THREADS=1 diff --git a/crux-mir/lib/compiler_builtins/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile deleted file mode 100644 index 63ea9af9d..000000000 --- a/crux-mir/lib/compiler_builtins/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM ubuntu:18.04 - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - gcc libc6-dev qemu-user-static ca-certificates \ - gcc-powerpc64le-linux-gnu libc6-dev-ppc64el-cross \ - qemu-system-ppc - -ENV CARGO_TARGET_POWERPC64LE_UNKNOWN_LINUX_GNU_LINKER=powerpc64le-linux-gnu-gcc \ - CARGO_TARGET_POWERPC64LE_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc64le-static \ - QEMU_CPU=POWER8 \ - QEMU_LD_PREFIX=/usr/powerpc64le-linux-gnu \ - RUST_TEST_THREADS=1 diff --git a/crux-mir/lib/compiler_builtins/ci/docker/thumbv6m-none-eabi/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/thumbv6m-none-eabi/Dockerfile deleted file mode 100644 index 04d4f4429..000000000 --- a/crux-mir/lib/compiler_builtins/ci/docker/thumbv6m-none-eabi/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM ubuntu:18.04 -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - gcc libc6-dev ca-certificates \ - gcc-arm-none-eabi \ - libnewlib-arm-none-eabi -ENV XARGO=1 diff --git a/crux-mir/lib/compiler_builtins/ci/docker/thumbv7em-none-eabi/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/thumbv7em-none-eabi/Dockerfile deleted file mode 100644 index 04d4f4429..000000000 --- a/crux-mir/lib/compiler_builtins/ci/docker/thumbv7em-none-eabi/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM ubuntu:18.04 -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - gcc libc6-dev ca-certificates \ - gcc-arm-none-eabi \ - libnewlib-arm-none-eabi -ENV XARGO=1 diff --git a/crux-mir/lib/compiler_builtins/ci/docker/thumbv7em-none-eabihf/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/thumbv7em-none-eabihf/Dockerfile deleted file mode 100644 index 04d4f4429..000000000 --- a/crux-mir/lib/compiler_builtins/ci/docker/thumbv7em-none-eabihf/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM ubuntu:18.04 -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - gcc libc6-dev ca-certificates \ - gcc-arm-none-eabi \ - libnewlib-arm-none-eabi -ENV XARGO=1 diff --git a/crux-mir/lib/compiler_builtins/ci/docker/thumbv7m-none-eabi/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/thumbv7m-none-eabi/Dockerfile deleted file mode 100644 index 04d4f4429..000000000 --- a/crux-mir/lib/compiler_builtins/ci/docker/thumbv7m-none-eabi/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM ubuntu:18.04 -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - gcc libc6-dev ca-certificates \ - gcc-arm-none-eabi \ - libnewlib-arm-none-eabi -ENV XARGO=1 diff --git a/crux-mir/lib/compiler_builtins/ci/docker/x86_64-unknown-linux-gnu/Dockerfile b/crux-mir/lib/compiler_builtins/ci/docker/x86_64-unknown-linux-gnu/Dockerfile deleted file mode 100644 index 98000f4eb..000000000 --- a/crux-mir/lib/compiler_builtins/ci/docker/x86_64-unknown-linux-gnu/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM ubuntu:18.04 -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - gcc libc6-dev ca-certificates diff --git a/crux-mir/lib/compiler_builtins/ci/run-docker.sh b/crux-mir/lib/compiler_builtins/ci/run-docker.sh deleted file mode 100755 index 4bb2a78d9..000000000 --- a/crux-mir/lib/compiler_builtins/ci/run-docker.sh +++ /dev/null @@ -1,38 +0,0 @@ -# Small script to run tests for a target (or all targets) inside all the -# respective docker images. - -set -ex - -run() { - local target=$1 - - echo $target - - # This directory needs to exist before calling docker, otherwise docker will create it but it - # will be owned by root - mkdir -p target - - docker build -t $target ci/docker/$target - docker run \ - --rm \ - --user $(id -u):$(id -g) \ - -e CARGO_HOME=/cargo \ - -e CARGO_TARGET_DIR=/target \ - -e RUST_COMPILER_RT_ROOT \ - -v $(dirname $(dirname `which cargo`)):/cargo \ - -v `pwd`/target:/target \ - -v `pwd`:/checkout:ro \ - -v `rustc --print sysroot`:/rust:ro \ - -w /checkout \ - --init \ - $target \ - sh -c "HOME=/tmp PATH=\$PATH:/rust/bin ci/run.sh $target" -} - -if [ -z "$1" ]; then - for d in `ls ci/docker/`; do - run $d - done -else - run $1 -fi diff --git a/crux-mir/lib/compiler_builtins/ci/run.sh b/crux-mir/lib/compiler_builtins/ci/run.sh deleted file mode 100755 index ae32806ec..000000000 --- a/crux-mir/lib/compiler_builtins/ci/run.sh +++ /dev/null @@ -1,103 +0,0 @@ -set -ex - -cargo=cargo - -# Test our implementation -if [ "$XARGO" = "1" ]; then - # FIXME: currently these tests don't work... - echo nothing to do -else - run="cargo test --manifest-path testcrate/Cargo.toml --target $1" - $run - $run --release - $run --features c - $run --features c --release -fi - -cargo build --target $1 -cargo build --target $1 --release -cargo build --target $1 --features c -cargo build --target $1 --release --features c - -PREFIX=$(echo $1 | sed -e 's/unknown-//')- -case $1 in - armv7-*) - PREFIX=arm-linux-gnueabihf- - ;; - thumb*) - PREFIX=arm-none-eabi- - ;; - *86*-*) - PREFIX= - ;; -esac - -NM=nm - -if [ -d /target ]; then - path=/target/${1}/debug/deps/libcompiler_builtins-*.rlib -else - path=target/${1}/debug/deps/libcompiler_builtins-*.rlib -fi - -# Look out for duplicated symbols when we include the compiler-rt (C) implementation -for rlib in $(echo $path); do - set +x - echo "================================================================" - echo checking $rlib for duplicate symbols - echo "================================================================" - - stdout=$($PREFIX$NM -g --defined-only $rlib 2>&1) - - # NOTE On i586, It's normal that the get_pc_thunk symbol appears several - # times so ignore it - # - # FIXME(#167) - we shouldn't ignore `__builtin_cl` style symbols here. - set +e - echo "$stdout" | \ - sort | \ - uniq -d | \ - grep -v __x86.get_pc_thunk | \ - grep -v __builtin_cl | \ - grep -v __builtin_ctz | \ - grep 'T __' - - if test $? = 0; then - exit 1 - fi - - set -ex -done - -rm -f $path - -# Verify that we haven't drop any intrinsic/symbol -build_intrinsics="$cargo build --target $1 -v --example intrinsics" -RUSTFLAGS="-C debug-assertions=no" $build_intrinsics -RUSTFLAGS="-C debug-assertions=no" $build_intrinsics --release -RUSTFLAGS="-C debug-assertions=no" $build_intrinsics --features c -RUSTFLAGS="-C debug-assertions=no" $build_intrinsics --features c --release - -# Verify that there are no undefined symbols to `panic` within our -# implementations -# -# TODO(#79) fix the undefined references problem for debug-assertions+lto -if [ -z "$DEBUG_LTO_BUILD_DOESNT_WORK" ]; then - RUSTFLAGS="-C debug-assertions=no" \ - CARGO_INCREMENTAL=0 \ - $cargo rustc --features "$INTRINSICS_FEATURES" --target $1 --example intrinsics -- -C lto -fi -$cargo rustc --features "$INTRINSICS_FEATURES" --target $1 --example intrinsics --release -- -C lto - -# Ensure no references to a panicking function -for rlib in $(echo $path); do - set +ex - $PREFIX$NM -u $rlib 2>&1 | grep panicking - - if test $? = 0; then - exit 1 - fi - set -ex -done - -true diff --git a/crux-mir/lib/compiler_builtins/crates/panic-handler/Cargo.toml b/crux-mir/lib/compiler_builtins/crates/panic-handler/Cargo.toml deleted file mode 100644 index 1dea613d1..000000000 --- a/crux-mir/lib/compiler_builtins/crates/panic-handler/Cargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[package] -name = "panic-handler" -version = "0.1.0" -authors = ["Alex Crichton "] - -[dependencies] diff --git a/crux-mir/lib/compiler_builtins/crates/panic-handler/src/lib.rs b/crux-mir/lib/compiler_builtins/crates/panic-handler/src/lib.rs deleted file mode 100644 index a75999a4b..000000000 --- a/crux-mir/lib/compiler_builtins/crates/panic-handler/src/lib.rs +++ /dev/null @@ -1,11 +0,0 @@ -// Hack of a crate until rust-lang/rust#51647 is fixed - -#![feature(no_core)] -#![no_core] - -extern crate core; - -#[panic_handler] -fn panic(_: &core::panic::PanicInfo) -> ! { - loop {} -} diff --git a/crux-mir/lib/compiler_builtins/examples/intrinsics.rs b/crux-mir/lib/compiler_builtins/examples/intrinsics.rs index 5ceebe132..0ca30c215 100644 --- a/crux-mir/lib/compiler_builtins/examples/intrinsics.rs +++ b/crux-mir/lib/compiler_builtins/examples/intrinsics.rs @@ -6,7 +6,7 @@ #![allow(unused_features)] #![cfg_attr(thumb, no_main)] #![deny(dead_code)] -#![feature(asm)] +#![feature(bench_black_box)] #![feature(lang_items)] #![feature(start)] #![feature(allocator_api)] @@ -14,7 +14,7 @@ extern crate panic_handler; -#[cfg(all(not(thumb), not(windows)))] +#[cfg(all(not(thumb), not(windows), not(target_arch = "wasm32")))] #[link(name = "c")] extern "C" {} @@ -24,16 +24,9 @@ extern "C" {} // have an additional comment: the function name is the ARM name for the intrinsic and the comment // in the non-ARM name for the intrinsic. mod intrinsics { - // trunccdfsf2 + // truncdfsf2 pub fn aeabi_d2f(x: f64) -> f32 { - // This is only implemented in C currently, so only test it there. - #[cfg(feature = "c")] - return x as f32; - #[cfg(not(feature = "c"))] - { - drop(x); - 0.0 - } + x as f32 } // fixdfsi @@ -276,14 +269,9 @@ mod intrinsics { } fn run() { + use core::hint::black_box as bb; use intrinsics::*; - // A copy of "test::black_box". Used to prevent LLVM from optimizing away the intrinsics during LTO - fn bb(dummy: T) -> T { - unsafe { asm!("" : : "r"(&dummy)) } - dummy - } - bb(aeabi_d2f(bb(2.))); bb(aeabi_d2i(bb(2.))); bb(aeabi_d2l(bb(2.))); @@ -340,11 +328,11 @@ fn run() { something_with_a_dtor(&|| assert_eq!(bb(1), 1)); extern "C" { - fn rust_begin_unwind(); + fn rust_begin_unwind(x: usize); } // if bb(false) { unsafe { - rust_begin_unwind(); + rust_begin_unwind(0); } // } } diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/acos.rs b/crux-mir/lib/compiler_builtins/libm/src/math/acos.rs new file mode 100644 index 000000000..23b13251e --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/acos.rs @@ -0,0 +1,112 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_acos.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunSoft, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ +/* acos(x) + * Method : + * acos(x) = pi/2 - asin(x) + * acos(-x) = pi/2 + asin(x) + * For |x|<=0.5 + * acos(x) = pi/2 - (x + x*x^2*R(x^2)) (see asin.c) + * For x>0.5 + * acos(x) = pi/2 - (pi/2 - 2asin(sqrt((1-x)/2))) + * = 2asin(sqrt((1-x)/2)) + * = 2s + 2s*z*R(z) ...z=(1-x)/2, s=sqrt(z) + * = 2f + (2c + 2s*z*R(z)) + * where f=hi part of s, and c = (z-f*f)/(s+f) is the correction term + * for f so that f+c ~ sqrt(z). + * For x<-0.5 + * acos(x) = pi - 2asin(sqrt((1-|x|)/2)) + * = pi - 0.5*(s+s*z*R(z)), where z=(1-|x|)/2,s=sqrt(z) + * + * Special cases: + * if x is NaN, return x itself; + * if |x|>1, return NaN with invalid signal. + * + * Function needed: sqrt + */ + +use super::sqrt; + +const PIO2_HI: f64 = 1.57079632679489655800e+00; /* 0x3FF921FB, 0x54442D18 */ +const PIO2_LO: f64 = 6.12323399573676603587e-17; /* 0x3C91A626, 0x33145C07 */ +const PS0: f64 = 1.66666666666666657415e-01; /* 0x3FC55555, 0x55555555 */ +const PS1: f64 = -3.25565818622400915405e-01; /* 0xBFD4D612, 0x03EB6F7D */ +const PS2: f64 = 2.01212532134862925881e-01; /* 0x3FC9C155, 0x0E884455 */ +const PS3: f64 = -4.00555345006794114027e-02; /* 0xBFA48228, 0xB5688F3B */ +const PS4: f64 = 7.91534994289814532176e-04; /* 0x3F49EFE0, 0x7501B288 */ +const PS5: f64 = 3.47933107596021167570e-05; /* 0x3F023DE1, 0x0DFDF709 */ +const QS1: f64 = -2.40339491173441421878e+00; /* 0xC0033A27, 0x1C8A2D4B */ +const QS2: f64 = 2.02094576023350569471e+00; /* 0x40002AE5, 0x9C598AC8 */ +const QS3: f64 = -6.88283971605453293030e-01; /* 0xBFE6066C, 0x1B8D0159 */ +const QS4: f64 = 7.70381505559019352791e-02; /* 0x3FB3B8C5, 0xB12E9282 */ + +fn r(z: f64) -> f64 { + let p: f64 = z * (PS0 + z * (PS1 + z * (PS2 + z * (PS3 + z * (PS4 + z * PS5))))); + let q: f64 = 1.0 + z * (QS1 + z * (QS2 + z * (QS3 + z * QS4))); + p / q +} + +/// Arccosine (f64) +/// +/// Computes the inverse cosine (arc cosine) of the input value. +/// Arguments must be in the range -1 to 1. +/// Returns values in radians, in the range of 0 to pi. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn acos(x: f64) -> f64 { + let x1p_120f = f64::from_bits(0x3870000000000000); // 0x1p-120 === 2 ^ -120 + let z: f64; + let w: f64; + let s: f64; + let c: f64; + let df: f64; + let hx: u32; + let ix: u32; + + hx = (x.to_bits() >> 32) as u32; + ix = hx & 0x7fffffff; + /* |x| >= 1 or nan */ + if ix >= 0x3ff00000 { + let lx: u32 = x.to_bits() as u32; + + if ((ix - 0x3ff00000) | lx) == 0 { + /* acos(1)=0, acos(-1)=pi */ + if (hx >> 31) != 0 { + return 2. * PIO2_HI + x1p_120f; + } + return 0.; + } + return 0. / (x - x); + } + /* |x| < 0.5 */ + if ix < 0x3fe00000 { + if ix <= 0x3c600000 { + /* |x| < 2**-57 */ + return PIO2_HI + x1p_120f; + } + return PIO2_HI - (x - (PIO2_LO - x * r(x * x))); + } + /* x < -0.5 */ + if (hx >> 31) != 0 { + z = (1.0 + x) * 0.5; + s = sqrt(z); + w = r(z) * s - PIO2_LO; + return 2. * (PIO2_HI - (s + w)); + } + /* x > 0.5 */ + z = (1.0 - x) * 0.5; + s = sqrt(z); + // Set the low 4 bytes to zero + df = f64::from_bits(s.to_bits() & 0xff_ff_ff_ff_00_00_00_00); + + c = (z - df * df) / (s + df); + w = r(z) * s + c; + 2. * (df + w) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/acosf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/acosf.rs new file mode 100644 index 000000000..1a60479e3 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/acosf.rs @@ -0,0 +1,79 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_acosf.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +use super::sqrtf::sqrtf; + +const PIO2_HI: f32 = 1.5707962513e+00; /* 0x3fc90fda */ +const PIO2_LO: f32 = 7.5497894159e-08; /* 0x33a22168 */ +const P_S0: f32 = 1.6666586697e-01; +const P_S1: f32 = -4.2743422091e-02; +const P_S2: f32 = -8.6563630030e-03; +const Q_S1: f32 = -7.0662963390e-01; + +fn r(z: f32) -> f32 { + let p = z * (P_S0 + z * (P_S1 + z * P_S2)); + let q = 1. + z * Q_S1; + p / q +} + +/// Arccosine (f32) +/// +/// Computes the inverse cosine (arc cosine) of the input value. +/// Arguments must be in the range -1 to 1. +/// Returns values in radians, in the range of 0 to pi. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn acosf(x: f32) -> f32 { + let x1p_120 = f32::from_bits(0x03800000); // 0x1p-120 === 2 ^ (-120) + + let z: f32; + let w: f32; + let s: f32; + + let mut hx = x.to_bits(); + let ix = hx & 0x7fffffff; + /* |x| >= 1 or nan */ + if ix >= 0x3f800000 { + if ix == 0x3f800000 { + if (hx >> 31) != 0 { + return 2. * PIO2_HI + x1p_120; + } + return 0.; + } + return 0. / (x - x); + } + /* |x| < 0.5 */ + if ix < 0x3f000000 { + if ix <= 0x32800000 { + /* |x| < 2**-26 */ + return PIO2_HI + x1p_120; + } + return PIO2_HI - (x - (PIO2_LO - x * r(x * x))); + } + /* x < -0.5 */ + if (hx >> 31) != 0 { + z = (1. + x) * 0.5; + s = sqrtf(z); + w = r(z) * s - PIO2_LO; + return 2. * (PIO2_HI - (s + w)); + } + /* x > 0.5 */ + z = (1. - x) * 0.5; + s = sqrtf(z); + hx = s.to_bits(); + let df = f32::from_bits(hx & 0xfffff000); + let c = (z - df * df) / (s + df); + w = r(z) * s + c; + 2. * (df + w) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/acosh.rs b/crux-mir/lib/compiler_builtins/libm/src/math/acosh.rs new file mode 100644 index 000000000..d1f5b9fa9 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/acosh.rs @@ -0,0 +1,27 @@ +use super::{log, log1p, sqrt}; + +const LN2: f64 = 0.693147180559945309417232121458176568; /* 0x3fe62e42, 0xfefa39ef*/ + +/// Inverse hyperbolic cosine (f64) +/// +/// Calculates the inverse hyperbolic cosine of `x`. +/// Is defined as `log(x + sqrt(x*x-1))`. +/// `x` must be a number greater than or equal to 1. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn acosh(x: f64) -> f64 { + let u = x.to_bits(); + let e = ((u >> 52) as usize) & 0x7ff; + + /* x < 1 domain error is handled in the called functions */ + + if e < 0x3ff + 1 { + /* |x| < 2, up to 2ulp error in [1,1.125] */ + return log1p(x - 1.0 + sqrt((x - 1.0) * (x - 1.0) + 2.0 * (x - 1.0))); + } + if e < 0x3ff + 26 { + /* |x| < 0x1p26 */ + return log(2.0 * x - 1.0 / (x + sqrt(x * x - 1.0))); + } + /* |x| >= 0x1p26 or nan */ + return log(x) + LN2; +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/acoshf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/acoshf.rs new file mode 100644 index 000000000..ad3455fdd --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/acoshf.rs @@ -0,0 +1,26 @@ +use super::{log1pf, logf, sqrtf}; + +const LN2: f32 = 0.693147180559945309417232121458176568; + +/// Inverse hyperbolic cosine (f32) +/// +/// Calculates the inverse hyperbolic cosine of `x`. +/// Is defined as `log(x + sqrt(x*x-1))`. +/// `x` must be a number greater than or equal to 1. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn acoshf(x: f32) -> f32 { + let u = x.to_bits(); + let a = u & 0x7fffffff; + + if a < 0x3f800000 + (1 << 23) { + /* |x| < 2, invalid if x < 1 or nan */ + /* up to 2ulp error in [1,1.125] */ + return log1pf(x - 1.0 + sqrtf((x - 1.0) * (x - 1.0) + 2.0 * (x - 1.0))); + } + if a < 0x3f800000 + (12 << 23) { + /* |x| < 0x1p12 */ + return logf(2.0 * x - 1.0 / (x + sqrtf(x * x - 1.0))); + } + /* x >= 0x1p12 */ + return logf(x) + LN2; +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/asin.rs b/crux-mir/lib/compiler_builtins/libm/src/math/asin.rs new file mode 100644 index 000000000..3e4b7c56e --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/asin.rs @@ -0,0 +1,119 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_asin.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunSoft, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ +/* asin(x) + * Method : + * Since asin(x) = x + x^3/6 + x^5*3/40 + x^7*15/336 + ... + * we approximate asin(x) on [0,0.5] by + * asin(x) = x + x*x^2*R(x^2) + * where + * R(x^2) is a rational approximation of (asin(x)-x)/x^3 + * and its remez error is bounded by + * |(asin(x)-x)/x^3 - R(x^2)| < 2^(-58.75) + * + * For x in [0.5,1] + * asin(x) = pi/2-2*asin(sqrt((1-x)/2)) + * Let y = (1-x), z = y/2, s := sqrt(z), and pio2_hi+pio2_lo=pi/2; + * then for x>0.98 + * asin(x) = pi/2 - 2*(s+s*z*R(z)) + * = pio2_hi - (2*(s+s*z*R(z)) - pio2_lo) + * For x<=0.98, let pio4_hi = pio2_hi/2, then + * f = hi part of s; + * c = sqrt(z) - f = (z-f*f)/(s+f) ...f+c=sqrt(z) + * and + * asin(x) = pi/2 - 2*(s+s*z*R(z)) + * = pio4_hi+(pio4-2s)-(2s*z*R(z)-pio2_lo) + * = pio4_hi+(pio4-2f)-(2s*z*R(z)-(pio2_lo+2c)) + * + * Special cases: + * if x is NaN, return x itself; + * if |x|>1, return NaN with invalid signal. + * + */ + +use super::{fabs, get_high_word, get_low_word, sqrt, with_set_low_word}; + +const PIO2_HI: f64 = 1.57079632679489655800e+00; /* 0x3FF921FB, 0x54442D18 */ +const PIO2_LO: f64 = 6.12323399573676603587e-17; /* 0x3C91A626, 0x33145C07 */ +/* coefficients for R(x^2) */ +const P_S0: f64 = 1.66666666666666657415e-01; /* 0x3FC55555, 0x55555555 */ +const P_S1: f64 = -3.25565818622400915405e-01; /* 0xBFD4D612, 0x03EB6F7D */ +const P_S2: f64 = 2.01212532134862925881e-01; /* 0x3FC9C155, 0x0E884455 */ +const P_S3: f64 = -4.00555345006794114027e-02; /* 0xBFA48228, 0xB5688F3B */ +const P_S4: f64 = 7.91534994289814532176e-04; /* 0x3F49EFE0, 0x7501B288 */ +const P_S5: f64 = 3.47933107596021167570e-05; /* 0x3F023DE1, 0x0DFDF709 */ +const Q_S1: f64 = -2.40339491173441421878e+00; /* 0xC0033A27, 0x1C8A2D4B */ +const Q_S2: f64 = 2.02094576023350569471e+00; /* 0x40002AE5, 0x9C598AC8 */ +const Q_S3: f64 = -6.88283971605453293030e-01; /* 0xBFE6066C, 0x1B8D0159 */ +const Q_S4: f64 = 7.70381505559019352791e-02; /* 0x3FB3B8C5, 0xB12E9282 */ + +fn comp_r(z: f64) -> f64 { + let p = z * (P_S0 + z * (P_S1 + z * (P_S2 + z * (P_S3 + z * (P_S4 + z * P_S5))))); + let q = 1.0 + z * (Q_S1 + z * (Q_S2 + z * (Q_S3 + z * Q_S4))); + p / q +} + +/// Arcsine (f64) +/// +/// Computes the inverse sine (arc sine) of the argument `x`. +/// Arguments to asin must be in the range -1 to 1. +/// Returns values in radians, in the range of -pi/2 to pi/2. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn asin(mut x: f64) -> f64 { + let z: f64; + let r: f64; + let s: f64; + let hx: u32; + let ix: u32; + + hx = get_high_word(x); + ix = hx & 0x7fffffff; + /* |x| >= 1 or nan */ + if ix >= 0x3ff00000 { + let lx: u32; + lx = get_low_word(x); + if ((ix - 0x3ff00000) | lx) == 0 { + /* asin(1) = +-pi/2 with inexact */ + return x * PIO2_HI + f64::from_bits(0x3870000000000000); + } else { + return 0.0 / (x - x); + } + } + /* |x| < 0.5 */ + if ix < 0x3fe00000 { + /* if 0x1p-1022 <= |x| < 0x1p-26, avoid raising underflow */ + if ix < 0x3e500000 && ix >= 0x00100000 { + return x; + } else { + return x + x * comp_r(x * x); + } + } + /* 1 > |x| >= 0.5 */ + z = (1.0 - fabs(x)) * 0.5; + s = sqrt(z); + r = comp_r(z); + if ix >= 0x3fef3333 { + /* if |x| > 0.975 */ + x = PIO2_HI - (2. * (s + s * r) - PIO2_LO); + } else { + let f: f64; + let c: f64; + /* f+c = sqrt(z) */ + f = with_set_low_word(s, 0); + c = (z - f * f) / (s + f); + x = 0.5 * PIO2_HI - (2.0 * s * r - (PIO2_LO - 2.0 * c) - (0.5 * PIO2_HI - 2.0 * f)); + } + if hx >> 31 != 0 { + -x + } else { + x + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/asinf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/asinf.rs new file mode 100644 index 000000000..6ec61b629 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/asinf.rs @@ -0,0 +1,72 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_asinf.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +use super::fabsf::fabsf; +use super::sqrt::sqrt; + +const PIO2: f64 = 1.570796326794896558e+00; + +/* coefficients for R(x^2) */ +const P_S0: f32 = 1.6666586697e-01; +const P_S1: f32 = -4.2743422091e-02; +const P_S2: f32 = -8.6563630030e-03; +const Q_S1: f32 = -7.0662963390e-01; + +fn r(z: f32) -> f32 { + let p = z * (P_S0 + z * (P_S1 + z * P_S2)); + let q = 1. + z * Q_S1; + p / q +} + +/// Arcsine (f32) +/// +/// Computes the inverse sine (arc sine) of the argument `x`. +/// Arguments to asin must be in the range -1 to 1. +/// Returns values in radians, in the range of -pi/2 to pi/2. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn asinf(mut x: f32) -> f32 { + let x1p_120 = f64::from_bits(0x3870000000000000); // 0x1p-120 === 2 ^ (-120) + + let hx = x.to_bits(); + let ix = hx & 0x7fffffff; + + if ix >= 0x3f800000 { + /* |x| >= 1 */ + if ix == 0x3f800000 { + /* |x| == 1 */ + return ((x as f64) * PIO2 + x1p_120) as f32; /* asin(+-1) = +-pi/2 with inexact */ + } + return 0. / (x - x); /* asin(|x|>1) is NaN */ + } + + if ix < 0x3f000000 { + /* |x| < 0.5 */ + /* if 0x1p-126 <= |x| < 0x1p-12, avoid raising underflow */ + if (ix < 0x39800000) && (ix >= 0x00800000) { + return x; + } + return x + x * r(x * x); + } + + /* 1 > |x| >= 0.5 */ + let z = (1. - fabsf(x)) * 0.5; + let s = sqrt(z as f64); + x = (PIO2 - 2. * (s + s * (r(z) as f64))) as f32; + if (hx >> 31) != 0 { + -x + } else { + x + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/asinh.rs b/crux-mir/lib/compiler_builtins/libm/src/math/asinh.rs new file mode 100644 index 000000000..0abd80c2f --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/asinh.rs @@ -0,0 +1,40 @@ +use super::{log, log1p, sqrt}; + +const LN2: f64 = 0.693147180559945309417232121458176568; /* 0x3fe62e42, 0xfefa39ef*/ + +/* asinh(x) = sign(x)*log(|x|+sqrt(x*x+1)) ~= x - x^3/6 + o(x^5) */ +/// Inverse hyperbolic sine (f64) +/// +/// Calculates the inverse hyperbolic sine of `x`. +/// Is defined as `sgn(x)*log(|x|+sqrt(x*x+1))`. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn asinh(mut x: f64) -> f64 { + let mut u = x.to_bits(); + let e = ((u >> 52) as usize) & 0x7ff; + let sign = (u >> 63) != 0; + + /* |x| */ + u &= (!0) >> 1; + x = f64::from_bits(u); + + if e >= 0x3ff + 26 { + /* |x| >= 0x1p26 or inf or nan */ + x = log(x) + LN2; + } else if e >= 0x3ff + 1 { + /* |x| >= 2 */ + x = log(2.0 * x + 1.0 / (sqrt(x * x + 1.0) + x)); + } else if e >= 0x3ff - 26 { + /* |x| >= 0x1p-26, up to 1.6ulp error in [0.125,0.5] */ + x = log1p(x + x * x / (sqrt(x * x + 1.0) + 1.0)); + } else { + /* |x| < 0x1p-26, raise inexact if x != 0 */ + let x1p120 = f64::from_bits(0x4770000000000000); + force_eval!(x + x1p120); + } + + if sign { + -x + } else { + x + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/asinhf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/asinhf.rs new file mode 100644 index 000000000..09c77823e --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/asinhf.rs @@ -0,0 +1,39 @@ +use super::{log1pf, logf, sqrtf}; + +const LN2: f32 = 0.693147180559945309417232121458176568; + +/* asinh(x) = sign(x)*log(|x|+sqrt(x*x+1)) ~= x - x^3/6 + o(x^5) */ +/// Inverse hyperbolic sine (f32) +/// +/// Calculates the inverse hyperbolic sine of `x`. +/// Is defined as `sgn(x)*log(|x|+sqrt(x*x+1))`. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn asinhf(mut x: f32) -> f32 { + let u = x.to_bits(); + let i = u & 0x7fffffff; + let sign = (u >> 31) != 0; + + /* |x| */ + x = f32::from_bits(i); + + if i >= 0x3f800000 + (12 << 23) { + /* |x| >= 0x1p12 or inf or nan */ + x = logf(x) + LN2; + } else if i >= 0x3f800000 + (1 << 23) { + /* |x| >= 2 */ + x = logf(2.0 * x + 1.0 / (sqrtf(x * x + 1.0) + x)); + } else if i >= 0x3f800000 - (12 << 23) { + /* |x| >= 0x1p-12, up to 1.6ulp error in [0.125,0.5] */ + x = log1pf(x + x * x / (sqrtf(x * x + 1.0) + 1.0)); + } else { + /* |x| < 0x1p-12, raise inexact if x!=0 */ + let x1p120 = f32::from_bits(0x7b800000); + force_eval!(x + x1p120); + } + + if sign { + -x + } else { + x + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/atan.rs b/crux-mir/lib/compiler_builtins/libm/src/math/atan.rs new file mode 100644 index 000000000..4259dc71a --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/atan.rs @@ -0,0 +1,184 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/s_atan.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ +/* atan(x) + * Method + * 1. Reduce x to positive by atan(x) = -atan(-x). + * 2. According to the integer k=4t+0.25 chopped, t=x, the argument + * is further reduced to one of the following intervals and the + * arctangent of t is evaluated by the corresponding formula: + * + * [0,7/16] atan(x) = t-t^3*(a1+t^2*(a2+...(a10+t^2*a11)...) + * [7/16,11/16] atan(x) = atan(1/2) + atan( (t-0.5)/(1+t/2) ) + * [11/16.19/16] atan(x) = atan( 1 ) + atan( (t-1)/(1+t) ) + * [19/16,39/16] atan(x) = atan(3/2) + atan( (t-1.5)/(1+1.5t) ) + * [39/16,INF] atan(x) = atan(INF) + atan( -1/t ) + * + * Constants: + * The hexadecimal values are the intended ones for the following + * constants. The decimal values may be used, provided that the + * compiler will convert from decimal to binary accurately enough + * to produce the hexadecimal values shown. + */ + +use super::fabs; +use core::f64; + +const ATANHI: [f64; 4] = [ + 4.63647609000806093515e-01, /* atan(0.5)hi 0x3FDDAC67, 0x0561BB4F */ + 7.85398163397448278999e-01, /* atan(1.0)hi 0x3FE921FB, 0x54442D18 */ + 9.82793723247329054082e-01, /* atan(1.5)hi 0x3FEF730B, 0xD281F69B */ + 1.57079632679489655800e+00, /* atan(inf)hi 0x3FF921FB, 0x54442D18 */ +]; + +const ATANLO: [f64; 4] = [ + 2.26987774529616870924e-17, /* atan(0.5)lo 0x3C7A2B7F, 0x222F65E2 */ + 3.06161699786838301793e-17, /* atan(1.0)lo 0x3C81A626, 0x33145C07 */ + 1.39033110312309984516e-17, /* atan(1.5)lo 0x3C700788, 0x7AF0CBBD */ + 6.12323399573676603587e-17, /* atan(inf)lo 0x3C91A626, 0x33145C07 */ +]; + +const AT: [f64; 11] = [ + 3.33333333333329318027e-01, /* 0x3FD55555, 0x5555550D */ + -1.99999999998764832476e-01, /* 0xBFC99999, 0x9998EBC4 */ + 1.42857142725034663711e-01, /* 0x3FC24924, 0x920083FF */ + -1.11111104054623557880e-01, /* 0xBFBC71C6, 0xFE231671 */ + 9.09088713343650656196e-02, /* 0x3FB745CD, 0xC54C206E */ + -7.69187620504482999495e-02, /* 0xBFB3B0F2, 0xAF749A6D */ + 6.66107313738753120669e-02, /* 0x3FB10D66, 0xA0D03D51 */ + -5.83357013379057348645e-02, /* 0xBFADDE2D, 0x52DEFD9A */ + 4.97687799461593236017e-02, /* 0x3FA97B4B, 0x24760DEB */ + -3.65315727442169155270e-02, /* 0xBFA2B444, 0x2C6A6C2F */ + 1.62858201153657823623e-02, /* 0x3F90AD3A, 0xE322DA11 */ +]; + +/// Arctangent (f64) +/// +/// Computes the inverse tangent (arc tangent) of the input value. +/// Returns a value in radians, in the range of -pi/2 to pi/2. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn atan(x: f64) -> f64 { + let mut x = x; + let mut ix = (x.to_bits() >> 32) as u32; + let sign = ix >> 31; + ix &= 0x7fff_ffff; + if ix >= 0x4410_0000 { + if x.is_nan() { + return x; + } + + let z = ATANHI[3] + f64::from_bits(0x0380_0000); // 0x1p-120f + return if sign != 0 { -z } else { z }; + } + + let id = if ix < 0x3fdc_0000 { + /* |x| < 0.4375 */ + if ix < 0x3e40_0000 { + /* |x| < 2^-27 */ + if ix < 0x0010_0000 { + /* raise underflow for subnormal x */ + force_eval!(x as f32); + } + + return x; + } + + -1 + } else { + x = fabs(x); + if ix < 0x3ff30000 { + /* |x| < 1.1875 */ + if ix < 0x3fe60000 { + /* 7/16 <= |x| < 11/16 */ + x = (2. * x - 1.) / (2. + x); + 0 + } else { + /* 11/16 <= |x| < 19/16 */ + x = (x - 1.) / (x + 1.); + 1 + } + } else if ix < 0x40038000 { + /* |x| < 2.4375 */ + x = (x - 1.5) / (1. + 1.5 * x); + 2 + } else { + /* 2.4375 <= |x| < 2^66 */ + x = -1. / x; + 3 + } + }; + + let z = x * x; + let w = z * z; + /* break sum from i=0 to 10 AT[i]z**(i+1) into odd and even poly */ + let s1 = z * (AT[0] + w * (AT[2] + w * (AT[4] + w * (AT[6] + w * (AT[8] + w * AT[10]))))); + let s2 = w * (AT[1] + w * (AT[3] + w * (AT[5] + w * (AT[7] + w * AT[9])))); + + if id < 0 { + return x - x * (s1 + s2); + } + + let z = i!(ATANHI, id as usize) - (x * (s1 + s2) - i!(ATANLO, id as usize) - x); + + if sign != 0 { + -z + } else { + z + } +} + +#[cfg(test)] +mod tests { + use super::atan; + use core::f64; + + #[test] + fn sanity_check() { + for (input, answer) in [ + (3.0_f64.sqrt() / 3.0, f64::consts::FRAC_PI_6), + (1.0, f64::consts::FRAC_PI_4), + (3.0_f64.sqrt(), f64::consts::FRAC_PI_3), + (-3.0_f64.sqrt() / 3.0, -f64::consts::FRAC_PI_6), + (-1.0, -f64::consts::FRAC_PI_4), + (-3.0_f64.sqrt(), -f64::consts::FRAC_PI_3), + ] + .iter() + { + assert!( + (atan(*input) - answer) / answer < 1e-5, + "\natan({:.4}/16) = {:.4}, actual: {}", + input * 16.0, + answer, + atan(*input) + ); + } + } + + #[test] + fn zero() { + assert_eq!(atan(0.0), 0.0); + } + + #[test] + fn infinity() { + assert_eq!(atan(f64::INFINITY), f64::consts::FRAC_PI_2); + } + + #[test] + fn minus_infinity() { + assert_eq!(atan(f64::NEG_INFINITY), -f64::consts::FRAC_PI_2); + } + + #[test] + fn nan() { + assert!(atan(f64::NAN).is_nan()); + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/atan2.rs b/crux-mir/lib/compiler_builtins/libm/src/math/atan2.rs new file mode 100644 index 000000000..fb2ea4eda --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/atan2.rs @@ -0,0 +1,126 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_atan2.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunSoft, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + * + */ +/* atan2(y,x) + * Method : + * 1. Reduce y to positive by atan2(y,x)=-atan2(-y,x). + * 2. Reduce x to positive by (if x and y are unexceptional): + * ARG (x+iy) = arctan(y/x) ... if x > 0, + * ARG (x+iy) = pi - arctan[y/(-x)] ... if x < 0, + * + * Special cases: + * + * ATAN2((anything), NaN ) is NaN; + * ATAN2(NAN , (anything) ) is NaN; + * ATAN2(+-0, +(anything but NaN)) is +-0 ; + * ATAN2(+-0, -(anything but NaN)) is +-pi ; + * ATAN2(+-(anything but 0 and NaN), 0) is +-pi/2; + * ATAN2(+-(anything but INF and NaN), +INF) is +-0 ; + * ATAN2(+-(anything but INF and NaN), -INF) is +-pi; + * ATAN2(+-INF,+INF ) is +-pi/4 ; + * ATAN2(+-INF,-INF ) is +-3pi/4; + * ATAN2(+-INF, (anything but,0,NaN, and INF)) is +-pi/2; + * + * Constants: + * The hexadecimal values are the intended ones for the following + * constants. The decimal values may be used, provided that the + * compiler will convert from decimal to binary accurately enough + * to produce the hexadecimal values shown. + */ + +use super::atan; +use super::fabs; + +const PI: f64 = 3.1415926535897931160E+00; /* 0x400921FB, 0x54442D18 */ +const PI_LO: f64 = 1.2246467991473531772E-16; /* 0x3CA1A626, 0x33145C07 */ + +/// Arctangent of y/x (f64) +/// +/// Computes the inverse tangent (arc tangent) of `y/x`. +/// Produces the correct result even for angles near pi/2 or -pi/2 (that is, when `x` is near 0). +/// Returns a value in radians, in the range of -pi to pi. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn atan2(y: f64, x: f64) -> f64 { + if x.is_nan() || y.is_nan() { + return x + y; + } + let mut ix = (x.to_bits() >> 32) as u32; + let lx = x.to_bits() as u32; + let mut iy = (y.to_bits() >> 32) as u32; + let ly = y.to_bits() as u32; + if ((ix.wrapping_sub(0x3ff00000)) | lx) == 0 { + /* x = 1.0 */ + return atan(y); + } + let m = ((iy >> 31) & 1) | ((ix >> 30) & 2); /* 2*sign(x)+sign(y) */ + ix &= 0x7fffffff; + iy &= 0x7fffffff; + + /* when y = 0 */ + if (iy | ly) == 0 { + return match m { + 0 | 1 => y, /* atan(+-0,+anything)=+-0 */ + 2 => PI, /* atan(+0,-anything) = PI */ + _ => -PI, /* atan(-0,-anything) =-PI */ + }; + } + /* when x = 0 */ + if (ix | lx) == 0 { + return if m & 1 != 0 { -PI / 2.0 } else { PI / 2.0 }; + } + /* when x is INF */ + if ix == 0x7ff00000 { + if iy == 0x7ff00000 { + return match m { + 0 => PI / 4.0, /* atan(+INF,+INF) */ + 1 => -PI / 4.0, /* atan(-INF,+INF) */ + 2 => 3.0 * PI / 4.0, /* atan(+INF,-INF) */ + _ => -3.0 * PI / 4.0, /* atan(-INF,-INF) */ + }; + } else { + return match m { + 0 => 0.0, /* atan(+...,+INF) */ + 1 => -0.0, /* atan(-...,+INF) */ + 2 => PI, /* atan(+...,-INF) */ + _ => -PI, /* atan(-...,-INF) */ + }; + } + } + /* |y/x| > 0x1p64 */ + if ix.wrapping_add(64 << 20) < iy || iy == 0x7ff00000 { + return if m & 1 != 0 { -PI / 2.0 } else { PI / 2.0 }; + } + + /* z = atan(|y/x|) without spurious underflow */ + let z = if (m & 2 != 0) && iy.wrapping_add(64 << 20) < ix { + /* |y/x| < 0x1p-64, x<0 */ + 0.0 + } else { + atan(fabs(y / x)) + }; + match m { + 0 => z, /* atan(+,+) */ + 1 => -z, /* atan(-,+) */ + 2 => PI - (z - PI_LO), /* atan(+,-) */ + _ => (z - PI_LO) - PI, /* atan(-,-) */ + } +} + +#[test] +fn sanity_check() { + assert_eq!(atan2(0.0, 1.0), 0.0); + assert_eq!(atan2(0.0, -1.0), PI); + assert_eq!(atan2(-0.0, -1.0), -PI); + assert_eq!(atan2(3.0, 2.0), atan(3.0 / 2.0)); + assert_eq!(atan2(2.0, -1.0), atan(2.0 / -1.0) + PI); + assert_eq!(atan2(-2.0, -1.0), atan(-2.0 / -1.0) - PI); +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/atan2f.rs b/crux-mir/lib/compiler_builtins/libm/src/math/atan2f.rs new file mode 100644 index 000000000..eae3b002d --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/atan2f.rs @@ -0,0 +1,91 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_atan2f.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +use super::atanf; +use super::fabsf; + +const PI: f32 = 3.1415927410e+00; /* 0x40490fdb */ +const PI_LO: f32 = -8.7422776573e-08; /* 0xb3bbbd2e */ + +/// Arctangent of y/x (f32) +/// +/// Computes the inverse tangent (arc tangent) of `y/x`. +/// Produces the correct result even for angles near pi/2 or -pi/2 (that is, when `x` is near 0). +/// Returns a value in radians, in the range of -pi to pi. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn atan2f(y: f32, x: f32) -> f32 { + if x.is_nan() || y.is_nan() { + return x + y; + } + let mut ix = x.to_bits(); + let mut iy = y.to_bits(); + + if ix == 0x3f800000 { + /* x=1.0 */ + return atanf(y); + } + let m = ((iy >> 31) & 1) | ((ix >> 30) & 2); /* 2*sign(x)+sign(y) */ + ix &= 0x7fffffff; + iy &= 0x7fffffff; + + /* when y = 0 */ + if iy == 0 { + return match m { + 0 | 1 => y, /* atan(+-0,+anything)=+-0 */ + 2 => PI, /* atan(+0,-anything) = pi */ + 3 | _ => -PI, /* atan(-0,-anything) =-pi */ + }; + } + /* when x = 0 */ + if ix == 0 { + return if m & 1 != 0 { -PI / 2. } else { PI / 2. }; + } + /* when x is INF */ + if ix == 0x7f800000 { + return if iy == 0x7f800000 { + match m { + 0 => PI / 4., /* atan(+INF,+INF) */ + 1 => -PI / 4., /* atan(-INF,+INF) */ + 2 => 3. * PI / 4., /* atan(+INF,-INF)*/ + 3 | _ => -3. * PI / 4., /* atan(-INF,-INF)*/ + } + } else { + match m { + 0 => 0., /* atan(+...,+INF) */ + 1 => -0., /* atan(-...,+INF) */ + 2 => PI, /* atan(+...,-INF) */ + 3 | _ => -PI, /* atan(-...,-INF) */ + } + }; + } + /* |y/x| > 0x1p26 */ + if (ix + (26 << 23) < iy) || (iy == 0x7f800000) { + return if m & 1 != 0 { -PI / 2. } else { PI / 2. }; + } + + /* z = atan(|y/x|) with correct underflow */ + let z = if (m & 2 != 0) && (iy + (26 << 23) < ix) { + /*|y/x| < 0x1p-26, x < 0 */ + 0. + } else { + atanf(fabsf(y / x)) + }; + match m { + 0 => z, /* atan(+,+) */ + 1 => -z, /* atan(-,+) */ + 2 => PI - (z - PI_LO), /* atan(+,-) */ + _ => (z - PI_LO) - PI, /* case 3 */ /* atan(-,-) */ + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/atanf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/atanf.rs new file mode 100644 index 000000000..d042b3bc0 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/atanf.rs @@ -0,0 +1,112 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/s_atanf.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +use super::fabsf; + +const ATAN_HI: [f32; 4] = [ + 4.6364760399e-01, /* atan(0.5)hi 0x3eed6338 */ + 7.8539812565e-01, /* atan(1.0)hi 0x3f490fda */ + 9.8279368877e-01, /* atan(1.5)hi 0x3f7b985e */ + 1.5707962513e+00, /* atan(inf)hi 0x3fc90fda */ +]; + +const ATAN_LO: [f32; 4] = [ + 5.0121582440e-09, /* atan(0.5)lo 0x31ac3769 */ + 3.7748947079e-08, /* atan(1.0)lo 0x33222168 */ + 3.4473217170e-08, /* atan(1.5)lo 0x33140fb4 */ + 7.5497894159e-08, /* atan(inf)lo 0x33a22168 */ +]; + +const A_T: [f32; 5] = [ + 3.3333328366e-01, + -1.9999158382e-01, + 1.4253635705e-01, + -1.0648017377e-01, + 6.1687607318e-02, +]; + +/// Arctangent (f32) +/// +/// Computes the inverse tangent (arc tangent) of the input value. +/// Returns a value in radians, in the range of -pi/2 to pi/2. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn atanf(mut x: f32) -> f32 { + let x1p_120 = f32::from_bits(0x03800000); // 0x1p-120 === 2 ^ (-120) + + let z: f32; + + let mut ix = x.to_bits(); + let sign = (ix >> 31) != 0; + ix &= 0x7fffffff; + + if ix >= 0x4c800000 { + /* if |x| >= 2**26 */ + if x.is_nan() { + return x; + } + z = i!(ATAN_HI, 3) + x1p_120; + return if sign { -z } else { z }; + } + let id = if ix < 0x3ee00000 { + /* |x| < 0.4375 */ + if ix < 0x39800000 { + /* |x| < 2**-12 */ + if ix < 0x00800000 { + /* raise underflow for subnormal x */ + force_eval!(x * x); + } + return x; + } + -1 + } else { + x = fabsf(x); + if ix < 0x3f980000 { + /* |x| < 1.1875 */ + if ix < 0x3f300000 { + /* 7/16 <= |x| < 11/16 */ + x = (2. * x - 1.) / (2. + x); + 0 + } else { + /* 11/16 <= |x| < 19/16 */ + x = (x - 1.) / (x + 1.); + 1 + } + } else if ix < 0x401c0000 { + /* |x| < 2.4375 */ + x = (x - 1.5) / (1. + 1.5 * x); + 2 + } else { + /* 2.4375 <= |x| < 2**26 */ + x = -1. / x; + 3 + } + }; + /* end of argument reduction */ + z = x * x; + let w = z * z; + /* break sum from i=0 to 10 aT[i]z**(i+1) into odd and even poly */ + let s1 = z * (i!(A_T, 0) + w * (i!(A_T, 2) + w * i!(A_T, 4))); + let s2 = w * (i!(A_T, 1) + w * i!(A_T, 3)); + if id < 0 { + return x - x * (s1 + s2); + } + let id = id as usize; + let z = i!(ATAN_HI, id) - ((x * (s1 + s2) - i!(ATAN_LO, id)) - x); + if sign { + -z + } else { + z + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/atanh.rs b/crux-mir/lib/compiler_builtins/libm/src/math/atanh.rs new file mode 100644 index 000000000..b984c4ac6 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/atanh.rs @@ -0,0 +1,37 @@ +use super::log1p; + +/* atanh(x) = log((1+x)/(1-x))/2 = log1p(2x/(1-x))/2 ~= x + x^3/3 + o(x^5) */ +/// Inverse hyperbolic tangent (f64) +/// +/// Calculates the inverse hyperbolic tangent of `x`. +/// Is defined as `log((1+x)/(1-x))/2 = log1p(2x/(1-x))/2`. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn atanh(x: f64) -> f64 { + let u = x.to_bits(); + let e = ((u >> 52) as usize) & 0x7ff; + let sign = (u >> 63) != 0; + + /* |x| */ + let mut y = f64::from_bits(u & 0x7fff_ffff_ffff_ffff); + + if e < 0x3ff - 1 { + if e < 0x3ff - 32 { + /* handle underflow */ + if e == 0 { + force_eval!(y as f32); + } + } else { + /* |x| < 0.5, up to 1.7ulp error */ + y = 0.5 * log1p(2.0 * y + 2.0 * y * y / (1.0 - y)); + } + } else { + /* avoid overflow */ + y = 0.5 * log1p(2.0 * (y / (1.0 - y))); + } + + if sign { + -y + } else { + y + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/atanhf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/atanhf.rs new file mode 100644 index 000000000..a1aa314a5 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/atanhf.rs @@ -0,0 +1,37 @@ +use super::log1pf; + +/* atanh(x) = log((1+x)/(1-x))/2 = log1p(2x/(1-x))/2 ~= x + x^3/3 + o(x^5) */ +/// Inverse hyperbolic tangent (f32) +/// +/// Calculates the inverse hyperbolic tangent of `x`. +/// Is defined as `log((1+x)/(1-x))/2 = log1p(2x/(1-x))/2`. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn atanhf(mut x: f32) -> f32 { + let mut u = x.to_bits(); + let sign = (u >> 31) != 0; + + /* |x| */ + u &= 0x7fffffff; + x = f32::from_bits(u); + + if u < 0x3f800000 - (1 << 23) { + if u < 0x3f800000 - (32 << 23) { + /* handle underflow */ + if u < (1 << 23) { + force_eval!((x * x) as f32); + } + } else { + /* |x| < 0.5, up to 1.7ulp error */ + x = 0.5 * log1pf(2.0 * x + 2.0 * x * x / (1.0 - x)); + } + } else { + /* avoid overflow */ + x = 0.5 * log1pf(2.0 * (x / (1.0 - x))); + } + + if sign { + -x + } else { + x + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/cbrt.rs b/crux-mir/lib/compiler_builtins/libm/src/math/cbrt.rs new file mode 100644 index 000000000..b4e77eaa2 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/cbrt.rs @@ -0,0 +1,113 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/s_cbrt.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + * + * Optimized by Bruce D. Evans. + */ +/* cbrt(x) + * Return cube root of x + */ + +use core::f64; + +const B1: u32 = 715094163; /* B1 = (1023-1023/3-0.03306235651)*2**20 */ +const B2: u32 = 696219795; /* B2 = (1023-1023/3-54/3-0.03306235651)*2**20 */ + +/* |1/cbrt(x) - p(x)| < 2**-23.5 (~[-7.93e-8, 7.929e-8]). */ +const P0: f64 = 1.87595182427177009643; /* 0x3ffe03e6, 0x0f61e692 */ +const P1: f64 = -1.88497979543377169875; /* 0xbffe28e0, 0x92f02420 */ +const P2: f64 = 1.621429720105354466140; /* 0x3ff9f160, 0x4a49d6c2 */ +const P3: f64 = -0.758397934778766047437; /* 0xbfe844cb, 0xbee751d9 */ +const P4: f64 = 0.145996192886612446982; /* 0x3fc2b000, 0xd4e4edd7 */ + +// Cube root (f64) +/// +/// Computes the cube root of the argument. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn cbrt(x: f64) -> f64 { + let x1p54 = f64::from_bits(0x4350000000000000); // 0x1p54 === 2 ^ 54 + + let mut ui: u64 = x.to_bits(); + let mut r: f64; + let s: f64; + let mut t: f64; + let w: f64; + let mut hx: u32 = (ui >> 32) as u32 & 0x7fffffff; + + if hx >= 0x7ff00000 { + /* cbrt(NaN,INF) is itself */ + return x + x; + } + + /* + * Rough cbrt to 5 bits: + * cbrt(2**e*(1+m) ~= 2**(e/3)*(1+(e%3+m)/3) + * where e is integral and >= 0, m is real and in [0, 1), and "/" and + * "%" are integer division and modulus with rounding towards minus + * infinity. The RHS is always >= the LHS and has a maximum relative + * error of about 1 in 16. Adding a bias of -0.03306235651 to the + * (e%3+m)/3 term reduces the error to about 1 in 32. With the IEEE + * floating point representation, for finite positive normal values, + * ordinary integer divison of the value in bits magically gives + * almost exactly the RHS of the above provided we first subtract the + * exponent bias (1023 for doubles) and later add it back. We do the + * subtraction virtually to keep e >= 0 so that ordinary integer + * division rounds towards minus infinity; this is also efficient. + */ + if hx < 0x00100000 { + /* zero or subnormal? */ + ui = (x * x1p54).to_bits(); + hx = (ui >> 32) as u32 & 0x7fffffff; + if hx == 0 { + return x; /* cbrt(0) is itself */ + } + hx = hx / 3 + B2; + } else { + hx = hx / 3 + B1; + } + ui &= 1 << 63; + ui |= (hx as u64) << 32; + t = f64::from_bits(ui); + + /* + * New cbrt to 23 bits: + * cbrt(x) = t*cbrt(x/t**3) ~= t*P(t**3/x) + * where P(r) is a polynomial of degree 4 that approximates 1/cbrt(r) + * to within 2**-23.5 when |r - 1| < 1/10. The rough approximation + * has produced t such than |t/cbrt(x) - 1| ~< 1/32, and cubing this + * gives us bounds for r = t**3/x. + * + * Try to optimize for parallel evaluation as in __tanf.c. + */ + r = (t * t) * (t / x); + t = t * ((P0 + r * (P1 + r * P2)) + ((r * r) * r) * (P3 + r * P4)); + + /* + * Round t away from zero to 23 bits (sloppily except for ensuring that + * the result is larger in magnitude than cbrt(x) but not much more than + * 2 23-bit ulps larger). With rounding towards zero, the error bound + * would be ~5/6 instead of ~4/6. With a maximum error of 2 23-bit ulps + * in the rounded t, the infinite-precision error in the Newton + * approximation barely affects third digit in the final error + * 0.667; the error in the rounded t can be up to about 3 23-bit ulps + * before the final error is larger than 0.667 ulps. + */ + ui = t.to_bits(); + ui = (ui + 0x80000000) & 0xffffffffc0000000; + t = f64::from_bits(ui); + + /* one step Newton iteration to 53 bits with error < 0.667 ulps */ + s = t * t; /* t*t is exact */ + r = x / s; /* error <= 0.5 ulps; |r| < |t| */ + w = t + t; /* t+t is exact */ + r = (r - t) / (w + r); /* r-t is exact; w+r ~= 3*t */ + t = t + t * r; /* error <= 0.5 + 0.5/3 + epsilon */ + t +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/cbrtf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/cbrtf.rs new file mode 100644 index 000000000..9d70305c6 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/cbrtf.rs @@ -0,0 +1,75 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/s_cbrtf.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + * Debugged and optimized by Bruce D. Evans. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ +/* cbrtf(x) + * Return cube root of x + */ + +use core::f32; + +const B1: u32 = 709958130; /* B1 = (127-127.0/3-0.03306235651)*2**23 */ +const B2: u32 = 642849266; /* B2 = (127-127.0/3-24/3-0.03306235651)*2**23 */ + +/// Cube root (f32) +/// +/// Computes the cube root of the argument. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn cbrtf(x: f32) -> f32 { + let x1p24 = f32::from_bits(0x4b800000); // 0x1p24f === 2 ^ 24 + + let mut r: f64; + let mut t: f64; + let mut ui: u32 = x.to_bits(); + let mut hx: u32 = ui & 0x7fffffff; + + if hx >= 0x7f800000 { + /* cbrt(NaN,INF) is itself */ + return x + x; + } + + /* rough cbrt to 5 bits */ + if hx < 0x00800000 { + /* zero or subnormal? */ + if hx == 0 { + return x; /* cbrt(+-0) is itself */ + } + ui = (x * x1p24).to_bits(); + hx = ui & 0x7fffffff; + hx = hx / 3 + B2; + } else { + hx = hx / 3 + B1; + } + ui &= 0x80000000; + ui |= hx; + + /* + * First step Newton iteration (solving t*t-x/t == 0) to 16 bits. In + * double precision so that its terms can be arranged for efficiency + * without causing overflow or underflow. + */ + t = f32::from_bits(ui) as f64; + r = t * t * t; + t = t * (x as f64 + x as f64 + r) / (x as f64 + r + r); + + /* + * Second step Newton iteration to 47 bits. In double precision for + * efficiency and accuracy. + */ + r = t * t * t; + t = t * (x as f64 + x as f64 + r) / (x as f64 + r + r); + + /* rounding to 24 bits is perfect in round-to-nearest mode */ + t as f32 +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/ceil.rs b/crux-mir/lib/compiler_builtins/libm/src/math/ceil.rs new file mode 100644 index 000000000..22d892971 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/ceil.rs @@ -0,0 +1,82 @@ +#![allow(unreachable_code)] +use core::f64; + +const TOINT: f64 = 1. / f64::EPSILON; + +/// Ceil (f64) +/// +/// Finds the nearest integer greater than or equal to `x`. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn ceil(x: f64) -> f64 { + // On wasm32 we know that LLVM's intrinsic will compile to an optimized + // `f64.ceil` native instruction, so we can leverage this for both code size + // and speed. + llvm_intrinsically_optimized! { + #[cfg(target_arch = "wasm32")] { + return unsafe { ::core::intrinsics::ceilf64(x) } + } + } + #[cfg(all(target_arch = "x86", not(target_feature = "sse2")))] + { + //use an alternative implementation on x86, because the + //main implementation fails with the x87 FPU used by + //debian i386, probablly due to excess precision issues. + //basic implementation taken from https://github.com/rust-lang/libm/issues/219 + use super::fabs; + if fabs(x).to_bits() < 4503599627370496.0_f64.to_bits() { + let truncated = x as i64 as f64; + if truncated < x { + return truncated + 1.0; + } else { + return truncated; + } + } else { + return x; + } + } + let u: u64 = x.to_bits(); + let e: i64 = (u >> 52 & 0x7ff) as i64; + let y: f64; + + if e >= 0x3ff + 52 || x == 0. { + return x; + } + // y = int(x) - x, where int(x) is an integer neighbor of x + y = if (u >> 63) != 0 { + x - TOINT + TOINT - x + } else { + x + TOINT - TOINT - x + }; + // special case because of non-nearest rounding modes + if e < 0x3ff { + force_eval!(y); + return if (u >> 63) != 0 { -0. } else { 1. }; + } + if y < 0. { + x + y + 1. + } else { + x + y + } +} + +#[cfg(test)] +mod tests { + use super::*; + use core::f64::*; + + #[test] + fn sanity_check() { + assert_eq!(ceil(1.1), 2.0); + assert_eq!(ceil(2.9), 3.0); + } + + /// The spec: https://en.cppreference.com/w/cpp/numeric/math/ceil + #[test] + fn spec_tests() { + // Not Asserted: that the current rounding mode has no effect. + assert!(ceil(NAN).is_nan()); + for f in [0.0, -0.0, INFINITY, NEG_INFINITY].iter().copied() { + assert_eq!(ceil(f), f); + } + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/ceilf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/ceilf.rs new file mode 100644 index 000000000..7bcc647ca --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/ceilf.rs @@ -0,0 +1,65 @@ +use core::f32; + +/// Ceil (f32) +/// +/// Finds the nearest integer greater than or equal to `x`. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn ceilf(x: f32) -> f32 { + // On wasm32 we know that LLVM's intrinsic will compile to an optimized + // `f32.ceil` native instruction, so we can leverage this for both code size + // and speed. + llvm_intrinsically_optimized! { + #[cfg(target_arch = "wasm32")] { + return unsafe { ::core::intrinsics::ceilf32(x) } + } + } + let mut ui = x.to_bits(); + let e = (((ui >> 23) & 0xff).wrapping_sub(0x7f)) as i32; + + if e >= 23 { + return x; + } + if e >= 0 { + let m = 0x007fffff >> e; + if (ui & m) == 0 { + return x; + } + force_eval!(x + f32::from_bits(0x7b800000)); + if ui >> 31 == 0 { + ui += m; + } + ui &= !m; + } else { + force_eval!(x + f32::from_bits(0x7b800000)); + if ui >> 31 != 0 { + return -0.0; + } else if ui << 1 != 0 { + return 1.0; + } + } + f32::from_bits(ui) +} + +// PowerPC tests are failing on LLVM 13: https://github.com/rust-lang/rust/issues/88520 +#[cfg(not(target_arch = "powerpc64"))] +#[cfg(test)] +mod tests { + use super::*; + use core::f32::*; + + #[test] + fn sanity_check() { + assert_eq!(ceilf(1.1), 2.0); + assert_eq!(ceilf(2.9), 3.0); + } + + /// The spec: https://en.cppreference.com/w/cpp/numeric/math/ceil + #[test] + fn spec_tests() { + // Not Asserted: that the current rounding mode has no effect. + assert!(ceilf(NAN).is_nan()); + for f in [0.0, -0.0, INFINITY, NEG_INFINITY].iter().copied() { + assert_eq!(ceilf(f), f); + } + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/copysign.rs b/crux-mir/lib/compiler_builtins/libm/src/math/copysign.rs new file mode 100644 index 000000000..1f4a35a33 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/copysign.rs @@ -0,0 +1,12 @@ +/// Sign of Y, magnitude of X (f64) +/// +/// Constructs a number with the magnitude (absolute value) of its +/// first argument, `x`, and the sign of its second argument, `y`. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn copysign(x: f64, y: f64) -> f64 { + let mut ux = x.to_bits(); + let uy = y.to_bits(); + ux &= (!0) >> 1; + ux |= uy & (1 << 63); + f64::from_bits(ux) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/copysignf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/copysignf.rs new file mode 100644 index 000000000..6c346e3a5 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/copysignf.rs @@ -0,0 +1,12 @@ +/// Sign of Y, magnitude of X (f32) +/// +/// Constructs a number with the magnitude (absolute value) of its +/// first argument, `x`, and the sign of its second argument, `y`. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn copysignf(x: f32, y: f32) -> f32 { + let mut ux = x.to_bits(); + let uy = y.to_bits(); + ux &= 0x7fffffff; + ux |= uy & 0x80000000; + f32::from_bits(ux) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/cos.rs b/crux-mir/lib/compiler_builtins/libm/src/math/cos.rs new file mode 100644 index 000000000..db8bc4989 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/cos.rs @@ -0,0 +1,73 @@ +// origin: FreeBSD /usr/src/lib/msun/src/s_cos.c */ +// +// ==================================================== +// Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. +// +// Developed at SunPro, a Sun Microsystems, Inc. business. +// Permission to use, copy, modify, and distribute this +// software is freely granted, provided that this notice +// is preserved. +// ==================================================== + +use super::{k_cos, k_sin, rem_pio2}; + +// cos(x) +// Return cosine function of x. +// +// kernel function: +// k_sin ... sine function on [-pi/4,pi/4] +// k_cos ... cosine function on [-pi/4,pi/4] +// rem_pio2 ... argument reduction routine +// +// Method. +// Let S,C and T denote the sin, cos and tan respectively on +// [-PI/4, +PI/4]. Reduce the argument x to y1+y2 = x-k*pi/2 +// in [-pi/4 , +pi/4], and let n = k mod 4. +// We have +// +// n sin(x) cos(x) tan(x) +// ---------------------------------------------------------- +// 0 S C T +// 1 C -S -1/T +// 2 -S -C T +// 3 -C S -1/T +// ---------------------------------------------------------- +// +// Special cases: +// Let trig be any of sin, cos, or tan. +// trig(+-INF) is NaN, with signals; +// trig(NaN) is that NaN; +// +// Accuracy: +// TRIG(x) returns trig(x) nearly rounded +// +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn cos(x: f64) -> f64 { + let ix = (f64::to_bits(x) >> 32) as u32 & 0x7fffffff; + + /* |x| ~< pi/4 */ + if ix <= 0x3fe921fb { + if ix < 0x3e46a09e { + /* if x < 2**-27 * sqrt(2) */ + /* raise inexact if x != 0 */ + if x as i32 == 0 { + return 1.0; + } + } + return k_cos(x, 0.0); + } + + /* cos(Inf or NaN) is NaN */ + if ix >= 0x7ff00000 { + return x - x; + } + + /* argument reduction needed */ + let (n, y0, y1) = rem_pio2(x); + match n & 3 { + 0 => k_cos(y0, y1), + 1 => -k_sin(y0, y1, 1), + 2 => -k_cos(y0, y1), + _ => k_sin(y0, y1, 1), + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/cosf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/cosf.rs new file mode 100644 index 000000000..424fa42ed --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/cosf.rs @@ -0,0 +1,83 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/s_cosf.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + * Optimized by Bruce D. Evans. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +use super::{k_cosf, k_sinf, rem_pio2f}; + +use core::f64::consts::FRAC_PI_2; + +/* Small multiples of pi/2 rounded to double precision. */ +const C1_PIO2: f64 = 1. * FRAC_PI_2; /* 0x3FF921FB, 0x54442D18 */ +const C2_PIO2: f64 = 2. * FRAC_PI_2; /* 0x400921FB, 0x54442D18 */ +const C3_PIO2: f64 = 3. * FRAC_PI_2; /* 0x4012D97C, 0x7F3321D2 */ +const C4_PIO2: f64 = 4. * FRAC_PI_2; /* 0x401921FB, 0x54442D18 */ + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn cosf(x: f32) -> f32 { + let x64 = x as f64; + + let x1p120 = f32::from_bits(0x7b800000); // 0x1p120f === 2 ^ 120 + + let mut ix = x.to_bits(); + let sign = (ix >> 31) != 0; + ix &= 0x7fffffff; + + if ix <= 0x3f490fda { + /* |x| ~<= pi/4 */ + if ix < 0x39800000 { + /* |x| < 2**-12 */ + /* raise inexact if x != 0 */ + force_eval!(x + x1p120); + return 1.; + } + return k_cosf(x64); + } + if ix <= 0x407b53d1 { + /* |x| ~<= 5*pi/4 */ + if ix > 0x4016cbe3 { + /* |x| ~> 3*pi/4 */ + return -k_cosf(if sign { x64 + C2_PIO2 } else { x64 - C2_PIO2 }); + } else if sign { + return k_sinf(x64 + C1_PIO2); + } else { + return k_sinf(C1_PIO2 - x64); + } + } + if ix <= 0x40e231d5 { + /* |x| ~<= 9*pi/4 */ + if ix > 0x40afeddf { + /* |x| ~> 7*pi/4 */ + return k_cosf(if sign { x64 + C4_PIO2 } else { x64 - C4_PIO2 }); + } else if sign { + return k_sinf(-x64 - C3_PIO2); + } else { + return k_sinf(x64 - C3_PIO2); + } + } + + /* cos(Inf or NaN) is NaN */ + if ix >= 0x7f800000 { + return x - x; + } + + /* general argument reduction needed */ + let (n, y) = rem_pio2f(x); + match n & 3 { + 0 => k_cosf(y), + 1 => k_sinf(-y), + 2 => -k_cosf(y), + _ => k_sinf(y), + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/cosh.rs b/crux-mir/lib/compiler_builtins/libm/src/math/cosh.rs new file mode 100644 index 000000000..2fb568ab3 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/cosh.rs @@ -0,0 +1,38 @@ +use super::exp; +use super::expm1; +use super::k_expo2; + +/// Hyperbolic cosine (f64) +/// +/// Computes the hyperbolic cosine of the argument x. +/// Is defined as `(exp(x) + exp(-x))/2` +/// Angles are specified in radians. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn cosh(mut x: f64) -> f64 { + /* |x| */ + let mut ix = x.to_bits(); + ix &= 0x7fffffffffffffff; + x = f64::from_bits(ix); + let w = ix >> 32; + + /* |x| < log(2) */ + if w < 0x3fe62e42 { + if w < 0x3ff00000 - (26 << 20) { + let x1p120 = f64::from_bits(0x4770000000000000); + force_eval!(x + x1p120); + return 1.; + } + let t = expm1(x); // exponential minus 1 + return 1. + t * t / (2. * (1. + t)); + } + + /* |x| < log(DBL_MAX) */ + if w < 0x40862e42 { + let t = exp(x); + /* note: if x>log(0x1p26) then the 1/t is not needed */ + return 0.5 * (t + 1. / t); + } + + /* |x| > log(DBL_MAX) or nan */ + k_expo2(x) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/coshf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/coshf.rs new file mode 100644 index 000000000..e7b684587 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/coshf.rs @@ -0,0 +1,38 @@ +use super::expf; +use super::expm1f; +use super::k_expo2f; + +/// Hyperbolic cosine (f64) +/// +/// Computes the hyperbolic cosine of the argument x. +/// Is defined as `(exp(x) + exp(-x))/2` +/// Angles are specified in radians. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn coshf(mut x: f32) -> f32 { + let x1p120 = f32::from_bits(0x7b800000); // 0x1p120f === 2 ^ 120 + + /* |x| */ + let mut ix = x.to_bits(); + ix &= 0x7fffffff; + x = f32::from_bits(ix); + let w = ix; + + /* |x| < log(2) */ + if w < 0x3f317217 { + if w < (0x3f800000 - (12 << 23)) { + force_eval!(x + x1p120); + return 1.; + } + let t = expm1f(x); + return 1. + t * t / (2. * (1. + t)); + } + + /* |x| < log(FLT_MAX) */ + if w < 0x42b17217 { + let t = expf(x); + return 0.5 * (t + 1. / t); + } + + /* |x| > log(FLT_MAX) or nan */ + k_expo2f(x) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/erf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/erf.rs new file mode 100644 index 000000000..5e21ba578 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/erf.rs @@ -0,0 +1,318 @@ +use super::{exp, fabs, get_high_word, with_set_low_word}; +/* origin: FreeBSD /usr/src/lib/msun/src/s_erf.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ +/* double erf(double x) + * double erfc(double x) + * x + * 2 |\ + * erf(x) = --------- | exp(-t*t)dt + * sqrt(pi) \| + * 0 + * + * erfc(x) = 1-erf(x) + * Note that + * erf(-x) = -erf(x) + * erfc(-x) = 2 - erfc(x) + * + * Method: + * 1. For |x| in [0, 0.84375] + * erf(x) = x + x*R(x^2) + * erfc(x) = 1 - erf(x) if x in [-.84375,0.25] + * = 0.5 + ((0.5-x)-x*R) if x in [0.25,0.84375] + * where R = P/Q where P is an odd poly of degree 8 and + * Q is an odd poly of degree 10. + * -57.90 + * | R - (erf(x)-x)/x | <= 2 + * + * + * Remark. The formula is derived by noting + * erf(x) = (2/sqrt(pi))*(x - x^3/3 + x^5/10 - x^7/42 + ....) + * and that + * 2/sqrt(pi) = 1.128379167095512573896158903121545171688 + * is close to one. The interval is chosen because the fix + * point of erf(x) is near 0.6174 (i.e., erf(x)=x when x is + * near 0.6174), and by some experiment, 0.84375 is chosen to + * guarantee the error is less than one ulp for erf. + * + * 2. For |x| in [0.84375,1.25], let s = |x| - 1, and + * c = 0.84506291151 rounded to single (24 bits) + * erf(x) = sign(x) * (c + P1(s)/Q1(s)) + * erfc(x) = (1-c) - P1(s)/Q1(s) if x > 0 + * 1+(c+P1(s)/Q1(s)) if x < 0 + * |P1/Q1 - (erf(|x|)-c)| <= 2**-59.06 + * Remark: here we use the taylor series expansion at x=1. + * erf(1+s) = erf(1) + s*Poly(s) + * = 0.845.. + P1(s)/Q1(s) + * That is, we use rational approximation to approximate + * erf(1+s) - (c = (single)0.84506291151) + * Note that |P1/Q1|< 0.078 for x in [0.84375,1.25] + * where + * P1(s) = degree 6 poly in s + * Q1(s) = degree 6 poly in s + * + * 3. For x in [1.25,1/0.35(~2.857143)], + * erfc(x) = (1/x)*exp(-x*x-0.5625+R1/S1) + * erf(x) = 1 - erfc(x) + * where + * R1(z) = degree 7 poly in z, (z=1/x^2) + * S1(z) = degree 8 poly in z + * + * 4. For x in [1/0.35,28] + * erfc(x) = (1/x)*exp(-x*x-0.5625+R2/S2) if x > 0 + * = 2.0 - (1/x)*exp(-x*x-0.5625+R2/S2) if -6 x >= 28 + * erf(x) = sign(x) *(1 - tiny) (raise inexact) + * erfc(x) = tiny*tiny (raise underflow) if x > 0 + * = 2 - tiny if x<0 + * + * 7. Special case: + * erf(0) = 0, erf(inf) = 1, erf(-inf) = -1, + * erfc(0) = 1, erfc(inf) = 0, erfc(-inf) = 2, + * erfc/erf(NaN) is NaN + */ + +const ERX: f64 = 8.45062911510467529297e-01; /* 0x3FEB0AC1, 0x60000000 */ +/* + * Coefficients for approximation to erf on [0,0.84375] + */ +const EFX8: f64 = 1.02703333676410069053e+00; /* 0x3FF06EBA, 0x8214DB69 */ +const PP0: f64 = 1.28379167095512558561e-01; /* 0x3FC06EBA, 0x8214DB68 */ +const PP1: f64 = -3.25042107247001499370e-01; /* 0xBFD4CD7D, 0x691CB913 */ +const PP2: f64 = -2.84817495755985104766e-02; /* 0xBF9D2A51, 0xDBD7194F */ +const PP3: f64 = -5.77027029648944159157e-03; /* 0xBF77A291, 0x236668E4 */ +const PP4: f64 = -2.37630166566501626084e-05; /* 0xBEF8EAD6, 0x120016AC */ +const QQ1: f64 = 3.97917223959155352819e-01; /* 0x3FD97779, 0xCDDADC09 */ +const QQ2: f64 = 6.50222499887672944485e-02; /* 0x3FB0A54C, 0x5536CEBA */ +const QQ3: f64 = 5.08130628187576562776e-03; /* 0x3F74D022, 0xC4D36B0F */ +const QQ4: f64 = 1.32494738004321644526e-04; /* 0x3F215DC9, 0x221C1A10 */ +const QQ5: f64 = -3.96022827877536812320e-06; /* 0xBED09C43, 0x42A26120 */ +/* + * Coefficients for approximation to erf in [0.84375,1.25] + */ +const PA0: f64 = -2.36211856075265944077e-03; /* 0xBF6359B8, 0xBEF77538 */ +const PA1: f64 = 4.14856118683748331666e-01; /* 0x3FDA8D00, 0xAD92B34D */ +const PA2: f64 = -3.72207876035701323847e-01; /* 0xBFD7D240, 0xFBB8C3F1 */ +const PA3: f64 = 3.18346619901161753674e-01; /* 0x3FD45FCA, 0x805120E4 */ +const PA4: f64 = -1.10894694282396677476e-01; /* 0xBFBC6398, 0x3D3E28EC */ +const PA5: f64 = 3.54783043256182359371e-02; /* 0x3FA22A36, 0x599795EB */ +const PA6: f64 = -2.16637559486879084300e-03; /* 0xBF61BF38, 0x0A96073F */ +const QA1: f64 = 1.06420880400844228286e-01; /* 0x3FBB3E66, 0x18EEE323 */ +const QA2: f64 = 5.40397917702171048937e-01; /* 0x3FE14AF0, 0x92EB6F33 */ +const QA3: f64 = 7.18286544141962662868e-02; /* 0x3FB2635C, 0xD99FE9A7 */ +const QA4: f64 = 1.26171219808761642112e-01; /* 0x3FC02660, 0xE763351F */ +const QA5: f64 = 1.36370839120290507362e-02; /* 0x3F8BEDC2, 0x6B51DD1C */ +const QA6: f64 = 1.19844998467991074170e-02; /* 0x3F888B54, 0x5735151D */ +/* + * Coefficients for approximation to erfc in [1.25,1/0.35] + */ +const RA0: f64 = -9.86494403484714822705e-03; /* 0xBF843412, 0x600D6435 */ +const RA1: f64 = -6.93858572707181764372e-01; /* 0xBFE63416, 0xE4BA7360 */ +const RA2: f64 = -1.05586262253232909814e+01; /* 0xC0251E04, 0x41B0E726 */ +const RA3: f64 = -6.23753324503260060396e+01; /* 0xC04F300A, 0xE4CBA38D */ +const RA4: f64 = -1.62396669462573470355e+02; /* 0xC0644CB1, 0x84282266 */ +const RA5: f64 = -1.84605092906711035994e+02; /* 0xC067135C, 0xEBCCABB2 */ +const RA6: f64 = -8.12874355063065934246e+01; /* 0xC0545265, 0x57E4D2F2 */ +const RA7: f64 = -9.81432934416914548592e+00; /* 0xC023A0EF, 0xC69AC25C */ +const SA1: f64 = 1.96512716674392571292e+01; /* 0x4033A6B9, 0xBD707687 */ +const SA2: f64 = 1.37657754143519042600e+02; /* 0x4061350C, 0x526AE721 */ +const SA3: f64 = 4.34565877475229228821e+02; /* 0x407B290D, 0xD58A1A71 */ +const SA4: f64 = 6.45387271733267880336e+02; /* 0x40842B19, 0x21EC2868 */ +const SA5: f64 = 4.29008140027567833386e+02; /* 0x407AD021, 0x57700314 */ +const SA6: f64 = 1.08635005541779435134e+02; /* 0x405B28A3, 0xEE48AE2C */ +const SA7: f64 = 6.57024977031928170135e+00; /* 0x401A47EF, 0x8E484A93 */ +const SA8: f64 = -6.04244152148580987438e-02; /* 0xBFAEEFF2, 0xEE749A62 */ +/* + * Coefficients for approximation to erfc in [1/.35,28] + */ +const RB0: f64 = -9.86494292470009928597e-03; /* 0xBF843412, 0x39E86F4A */ +const RB1: f64 = -7.99283237680523006574e-01; /* 0xBFE993BA, 0x70C285DE */ +const RB2: f64 = -1.77579549177547519889e+01; /* 0xC031C209, 0x555F995A */ +const RB3: f64 = -1.60636384855821916062e+02; /* 0xC064145D, 0x43C5ED98 */ +const RB4: f64 = -6.37566443368389627722e+02; /* 0xC083EC88, 0x1375F228 */ +const RB5: f64 = -1.02509513161107724954e+03; /* 0xC0900461, 0x6A2E5992 */ +const RB6: f64 = -4.83519191608651397019e+02; /* 0xC07E384E, 0x9BDC383F */ +const SB1: f64 = 3.03380607434824582924e+01; /* 0x403E568B, 0x261D5190 */ +const SB2: f64 = 3.25792512996573918826e+02; /* 0x40745CAE, 0x221B9F0A */ +const SB3: f64 = 1.53672958608443695994e+03; /* 0x409802EB, 0x189D5118 */ +const SB4: f64 = 3.19985821950859553908e+03; /* 0x40A8FFB7, 0x688C246A */ +const SB5: f64 = 2.55305040643316442583e+03; /* 0x40A3F219, 0xCEDF3BE6 */ +const SB6: f64 = 4.74528541206955367215e+02; /* 0x407DA874, 0xE79FE763 */ +const SB7: f64 = -2.24409524465858183362e+01; /* 0xC03670E2, 0x42712D62 */ + +fn erfc1(x: f64) -> f64 { + let s: f64; + let p: f64; + let q: f64; + + s = fabs(x) - 1.0; + p = PA0 + s * (PA1 + s * (PA2 + s * (PA3 + s * (PA4 + s * (PA5 + s * PA6))))); + q = 1.0 + s * (QA1 + s * (QA2 + s * (QA3 + s * (QA4 + s * (QA5 + s * QA6))))); + + 1.0 - ERX - p / q +} + +fn erfc2(ix: u32, mut x: f64) -> f64 { + let s: f64; + let r: f64; + let big_s: f64; + let z: f64; + + if ix < 0x3ff40000 { + /* |x| < 1.25 */ + return erfc1(x); + } + + x = fabs(x); + s = 1.0 / (x * x); + if ix < 0x4006db6d { + /* |x| < 1/.35 ~ 2.85714 */ + r = RA0 + s * (RA1 + s * (RA2 + s * (RA3 + s * (RA4 + s * (RA5 + s * (RA6 + s * RA7)))))); + big_s = 1.0 + + s * (SA1 + + s * (SA2 + s * (SA3 + s * (SA4 + s * (SA5 + s * (SA6 + s * (SA7 + s * SA8))))))); + } else { + /* |x| > 1/.35 */ + r = RB0 + s * (RB1 + s * (RB2 + s * (RB3 + s * (RB4 + s * (RB5 + s * RB6))))); + big_s = + 1.0 + s * (SB1 + s * (SB2 + s * (SB3 + s * (SB4 + s * (SB5 + s * (SB6 + s * SB7)))))); + } + z = with_set_low_word(x, 0); + + exp(-z * z - 0.5625) * exp((z - x) * (z + x) + r / big_s) / x +} + +/// Error function (f64) +/// +/// Calculates an approximation to the “error function”, which estimates +/// the probability that an observation will fall within x standard +/// deviations of the mean (assuming a normal distribution). +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn erf(x: f64) -> f64 { + let r: f64; + let s: f64; + let z: f64; + let y: f64; + let mut ix: u32; + let sign: usize; + + ix = get_high_word(x); + sign = (ix >> 31) as usize; + ix &= 0x7fffffff; + if ix >= 0x7ff00000 { + /* erf(nan)=nan, erf(+-inf)=+-1 */ + return 1.0 - 2.0 * (sign as f64) + 1.0 / x; + } + if ix < 0x3feb0000 { + /* |x| < 0.84375 */ + if ix < 0x3e300000 { + /* |x| < 2**-28 */ + /* avoid underflow */ + return 0.125 * (8.0 * x + EFX8 * x); + } + z = x * x; + r = PP0 + z * (PP1 + z * (PP2 + z * (PP3 + z * PP4))); + s = 1.0 + z * (QQ1 + z * (QQ2 + z * (QQ3 + z * (QQ4 + z * QQ5)))); + y = r / s; + return x + x * y; + } + if ix < 0x40180000 { + /* 0.84375 <= |x| < 6 */ + y = 1.0 - erfc2(ix, x); + } else { + let x1p_1022 = f64::from_bits(0x0010000000000000); + y = 1.0 - x1p_1022; + } + + if sign != 0 { + -y + } else { + y + } +} + +/// Error function (f64) +/// +/// Calculates the complementary probability. +/// Is `1 - erf(x)`. Is computed directly, so that you can use it to avoid +/// the loss of precision that would result from subtracting +/// large probabilities (on large `x`) from 1. +pub fn erfc(x: f64) -> f64 { + let r: f64; + let s: f64; + let z: f64; + let y: f64; + let mut ix: u32; + let sign: usize; + + ix = get_high_word(x); + sign = (ix >> 31) as usize; + ix &= 0x7fffffff; + if ix >= 0x7ff00000 { + /* erfc(nan)=nan, erfc(+-inf)=0,2 */ + return 2.0 * (sign as f64) + 1.0 / x; + } + if ix < 0x3feb0000 { + /* |x| < 0.84375 */ + if ix < 0x3c700000 { + /* |x| < 2**-56 */ + return 1.0 - x; + } + z = x * x; + r = PP0 + z * (PP1 + z * (PP2 + z * (PP3 + z * PP4))); + s = 1.0 + z * (QQ1 + z * (QQ2 + z * (QQ3 + z * (QQ4 + z * QQ5)))); + y = r / s; + if sign != 0 || ix < 0x3fd00000 { + /* x < 1/4 */ + return 1.0 - (x + x * y); + } + return 0.5 - (x - 0.5 + x * y); + } + if ix < 0x403c0000 { + /* 0.84375 <= |x| < 28 */ + if sign != 0 { + return 2.0 - erfc2(ix, x); + } else { + return erfc2(ix, x); + } + } + + let x1p_1022 = f64::from_bits(0x0010000000000000); + if sign != 0 { + 2.0 - x1p_1022 + } else { + x1p_1022 * x1p_1022 + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/erff.rs b/crux-mir/lib/compiler_builtins/libm/src/math/erff.rs new file mode 100644 index 000000000..f74d4b632 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/erff.rs @@ -0,0 +1,230 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/s_erff.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +use super::{expf, fabsf}; + +const ERX: f32 = 8.4506291151e-01; /* 0x3f58560b */ +/* + * Coefficients for approximation to erf on [0,0.84375] + */ +const EFX8: f32 = 1.0270333290e+00; /* 0x3f8375d4 */ +const PP0: f32 = 1.2837916613e-01; /* 0x3e0375d4 */ +const PP1: f32 = -3.2504209876e-01; /* 0xbea66beb */ +const PP2: f32 = -2.8481749818e-02; /* 0xbce9528f */ +const PP3: f32 = -5.7702702470e-03; /* 0xbbbd1489 */ +const PP4: f32 = -2.3763017452e-05; /* 0xb7c756b1 */ +const QQ1: f32 = 3.9791721106e-01; /* 0x3ecbbbce */ +const QQ2: f32 = 6.5022252500e-02; /* 0x3d852a63 */ +const QQ3: f32 = 5.0813062117e-03; /* 0x3ba68116 */ +const QQ4: f32 = 1.3249473704e-04; /* 0x390aee49 */ +const QQ5: f32 = -3.9602282413e-06; /* 0xb684e21a */ +/* + * Coefficients for approximation to erf in [0.84375,1.25] + */ +const PA0: f32 = -2.3621185683e-03; /* 0xbb1acdc6 */ +const PA1: f32 = 4.1485610604e-01; /* 0x3ed46805 */ +const PA2: f32 = -3.7220788002e-01; /* 0xbebe9208 */ +const PA3: f32 = 3.1834661961e-01; /* 0x3ea2fe54 */ +const PA4: f32 = -1.1089469492e-01; /* 0xbde31cc2 */ +const PA5: f32 = 3.5478305072e-02; /* 0x3d1151b3 */ +const PA6: f32 = -2.1663755178e-03; /* 0xbb0df9c0 */ +const QA1: f32 = 1.0642088205e-01; /* 0x3dd9f331 */ +const QA2: f32 = 5.4039794207e-01; /* 0x3f0a5785 */ +const QA3: f32 = 7.1828655899e-02; /* 0x3d931ae7 */ +const QA4: f32 = 1.2617121637e-01; /* 0x3e013307 */ +const QA5: f32 = 1.3637083583e-02; /* 0x3c5f6e13 */ +const QA6: f32 = 1.1984500103e-02; /* 0x3c445aa3 */ +/* + * Coefficients for approximation to erfc in [1.25,1/0.35] + */ +const RA0: f32 = -9.8649440333e-03; /* 0xbc21a093 */ +const RA1: f32 = -6.9385856390e-01; /* 0xbf31a0b7 */ +const RA2: f32 = -1.0558626175e+01; /* 0xc128f022 */ +const RA3: f32 = -6.2375331879e+01; /* 0xc2798057 */ +const RA4: f32 = -1.6239666748e+02; /* 0xc322658c */ +const RA5: f32 = -1.8460508728e+02; /* 0xc3389ae7 */ +const RA6: f32 = -8.1287437439e+01; /* 0xc2a2932b */ +const RA7: f32 = -9.8143291473e+00; /* 0xc11d077e */ +const SA1: f32 = 1.9651271820e+01; /* 0x419d35ce */ +const SA2: f32 = 1.3765776062e+02; /* 0x4309a863 */ +const SA3: f32 = 4.3456588745e+02; /* 0x43d9486f */ +const SA4: f32 = 6.4538726807e+02; /* 0x442158c9 */ +const SA5: f32 = 4.2900814819e+02; /* 0x43d6810b */ +const SA6: f32 = 1.0863500214e+02; /* 0x42d9451f */ +const SA7: f32 = 6.5702495575e+00; /* 0x40d23f7c */ +const SA8: f32 = -6.0424413532e-02; /* 0xbd777f97 */ +/* + * Coefficients for approximation to erfc in [1/.35,28] + */ +const RB0: f32 = -9.8649431020e-03; /* 0xbc21a092 */ +const RB1: f32 = -7.9928326607e-01; /* 0xbf4c9dd4 */ +const RB2: f32 = -1.7757955551e+01; /* 0xc18e104b */ +const RB3: f32 = -1.6063638306e+02; /* 0xc320a2ea */ +const RB4: f32 = -6.3756646729e+02; /* 0xc41f6441 */ +const RB5: f32 = -1.0250950928e+03; /* 0xc480230b */ +const RB6: f32 = -4.8351919556e+02; /* 0xc3f1c275 */ +const SB1: f32 = 3.0338060379e+01; /* 0x41f2b459 */ +const SB2: f32 = 3.2579251099e+02; /* 0x43a2e571 */ +const SB3: f32 = 1.5367296143e+03; /* 0x44c01759 */ +const SB4: f32 = 3.1998581543e+03; /* 0x4547fdbb */ +const SB5: f32 = 2.5530502930e+03; /* 0x451f90ce */ +const SB6: f32 = 4.7452853394e+02; /* 0x43ed43a7 */ +const SB7: f32 = -2.2440952301e+01; /* 0xc1b38712 */ + +fn erfc1(x: f32) -> f32 { + let s: f32; + let p: f32; + let q: f32; + + s = fabsf(x) - 1.0; + p = PA0 + s * (PA1 + s * (PA2 + s * (PA3 + s * (PA4 + s * (PA5 + s * PA6))))); + q = 1.0 + s * (QA1 + s * (QA2 + s * (QA3 + s * (QA4 + s * (QA5 + s * QA6))))); + return 1.0 - ERX - p / q; +} + +fn erfc2(mut ix: u32, mut x: f32) -> f32 { + let s: f32; + let r: f32; + let big_s: f32; + let z: f32; + + if ix < 0x3fa00000 { + /* |x| < 1.25 */ + return erfc1(x); + } + + x = fabsf(x); + s = 1.0 / (x * x); + if ix < 0x4036db6d { + /* |x| < 1/0.35 */ + r = RA0 + s * (RA1 + s * (RA2 + s * (RA3 + s * (RA4 + s * (RA5 + s * (RA6 + s * RA7)))))); + big_s = 1.0 + + s * (SA1 + + s * (SA2 + s * (SA3 + s * (SA4 + s * (SA5 + s * (SA6 + s * (SA7 + s * SA8))))))); + } else { + /* |x| >= 1/0.35 */ + r = RB0 + s * (RB1 + s * (RB2 + s * (RB3 + s * (RB4 + s * (RB5 + s * RB6))))); + big_s = + 1.0 + s * (SB1 + s * (SB2 + s * (SB3 + s * (SB4 + s * (SB5 + s * (SB6 + s * SB7)))))); + } + ix = x.to_bits(); + z = f32::from_bits(ix & 0xffffe000); + + expf(-z * z - 0.5625) * expf((z - x) * (z + x) + r / big_s) / x +} + +/// Error function (f32) +/// +/// Calculates an approximation to the “error function”, which estimates +/// the probability that an observation will fall within x standard +/// deviations of the mean (assuming a normal distribution). +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn erff(x: f32) -> f32 { + let r: f32; + let s: f32; + let z: f32; + let y: f32; + let mut ix: u32; + let sign: usize; + + ix = x.to_bits(); + sign = (ix >> 31) as usize; + ix &= 0x7fffffff; + if ix >= 0x7f800000 { + /* erf(nan)=nan, erf(+-inf)=+-1 */ + return 1.0 - 2.0 * (sign as f32) + 1.0 / x; + } + if ix < 0x3f580000 { + /* |x| < 0.84375 */ + if ix < 0x31800000 { + /* |x| < 2**-28 */ + /*avoid underflow */ + return 0.125 * (8.0 * x + EFX8 * x); + } + z = x * x; + r = PP0 + z * (PP1 + z * (PP2 + z * (PP3 + z * PP4))); + s = 1.0 + z * (QQ1 + z * (QQ2 + z * (QQ3 + z * (QQ4 + z * QQ5)))); + y = r / s; + return x + x * y; + } + if ix < 0x40c00000 { + /* |x| < 6 */ + y = 1.0 - erfc2(ix, x); + } else { + let x1p_120 = f32::from_bits(0x03800000); + y = 1.0 - x1p_120; + } + + if sign != 0 { + -y + } else { + y + } +} + +/// Error function (f32) +/// +/// Calculates the complementary probability. +/// Is `1 - erf(x)`. Is computed directly, so that you can use it to avoid +/// the loss of precision that would result from subtracting +/// large probabilities (on large `x`) from 1. +pub fn erfcf(x: f32) -> f32 { + let r: f32; + let s: f32; + let z: f32; + let y: f32; + let mut ix: u32; + let sign: usize; + + ix = x.to_bits(); + sign = (ix >> 31) as usize; + ix &= 0x7fffffff; + if ix >= 0x7f800000 { + /* erfc(nan)=nan, erfc(+-inf)=0,2 */ + return 2.0 * (sign as f32) + 1.0 / x; + } + + if ix < 0x3f580000 { + /* |x| < 0.84375 */ + if ix < 0x23800000 { + /* |x| < 2**-56 */ + return 1.0 - x; + } + z = x * x; + r = PP0 + z * (PP1 + z * (PP2 + z * (PP3 + z * PP4))); + s = 1.0 + z * (QQ1 + z * (QQ2 + z * (QQ3 + z * (QQ4 + z * QQ5)))); + y = r / s; + if sign != 0 || ix < 0x3e800000 { + /* x < 1/4 */ + return 1.0 - (x + x * y); + } + return 0.5 - (x - 0.5 + x * y); + } + if ix < 0x41e00000 { + /* |x| < 28 */ + if sign != 0 { + return 2.0 - erfc2(ix, x); + } else { + return erfc2(ix, x); + } + } + + let x1p_120 = f32::from_bits(0x03800000); + if sign != 0 { + 2.0 - x1p_120 + } else { + x1p_120 * x1p_120 + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/exp.rs b/crux-mir/lib/compiler_builtins/libm/src/math/exp.rs new file mode 100644 index 000000000..d4994277f --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/exp.rs @@ -0,0 +1,154 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_exp.c */ +/* + * ==================================================== + * Copyright (C) 2004 by Sun Microsystems, Inc. All rights reserved. + * + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ +/* exp(x) + * Returns the exponential of x. + * + * Method + * 1. Argument reduction: + * Reduce x to an r so that |r| <= 0.5*ln2 ~ 0.34658. + * Given x, find r and integer k such that + * + * x = k*ln2 + r, |r| <= 0.5*ln2. + * + * Here r will be represented as r = hi-lo for better + * accuracy. + * + * 2. Approximation of exp(r) by a special rational function on + * the interval [0,0.34658]: + * Write + * R(r**2) = r*(exp(r)+1)/(exp(r)-1) = 2 + r*r/6 - r**4/360 + ... + * We use a special Remez algorithm on [0,0.34658] to generate + * a polynomial of degree 5 to approximate R. The maximum error + * of this polynomial approximation is bounded by 2**-59. In + * other words, + * R(z) ~ 2.0 + P1*z + P2*z**2 + P3*z**3 + P4*z**4 + P5*z**5 + * (where z=r*r, and the values of P1 to P5 are listed below) + * and + * | 5 | -59 + * | 2.0+P1*z+...+P5*z - R(z) | <= 2 + * | | + * The computation of exp(r) thus becomes + * 2*r + * exp(r) = 1 + ---------- + * R(r) - r + * r*c(r) + * = 1 + r + ----------- (for better accuracy) + * 2 - c(r) + * where + * 2 4 10 + * c(r) = r - (P1*r + P2*r + ... + P5*r ). + * + * 3. Scale back to obtain exp(x): + * From step 1, we have + * exp(x) = 2^k * exp(r) + * + * Special cases: + * exp(INF) is INF, exp(NaN) is NaN; + * exp(-INF) is 0, and + * for finite argument, only exp(0)=1 is exact. + * + * Accuracy: + * according to an error analysis, the error is always less than + * 1 ulp (unit in the last place). + * + * Misc. info. + * For IEEE double + * if x > 709.782712893383973096 then exp(x) overflows + * if x < -745.133219101941108420 then exp(x) underflows + */ + +use super::scalbn; + +const HALF: [f64; 2] = [0.5, -0.5]; +const LN2HI: f64 = 6.93147180369123816490e-01; /* 0x3fe62e42, 0xfee00000 */ +const LN2LO: f64 = 1.90821492927058770002e-10; /* 0x3dea39ef, 0x35793c76 */ +const INVLN2: f64 = 1.44269504088896338700e+00; /* 0x3ff71547, 0x652b82fe */ +const P1: f64 = 1.66666666666666019037e-01; /* 0x3FC55555, 0x5555553E */ +const P2: f64 = -2.77777777770155933842e-03; /* 0xBF66C16C, 0x16BEBD93 */ +const P3: f64 = 6.61375632143793436117e-05; /* 0x3F11566A, 0xAF25DE2C */ +const P4: f64 = -1.65339022054652515390e-06; /* 0xBEBBBD41, 0xC5D26BF1 */ +const P5: f64 = 4.13813679705723846039e-08; /* 0x3E663769, 0x72BEA4D0 */ + +/// Exponential, base *e* (f64) +/// +/// Calculate the exponential of `x`, that is, *e* raised to the power `x` +/// (where *e* is the base of the natural system of logarithms, approximately 2.71828). +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn exp(mut x: f64) -> f64 { + let x1p1023 = f64::from_bits(0x7fe0000000000000); // 0x1p1023 === 2 ^ 1023 + let x1p_149 = f64::from_bits(0x36a0000000000000); // 0x1p-149 === 2 ^ -149 + + let hi: f64; + let lo: f64; + let c: f64; + let xx: f64; + let y: f64; + let k: i32; + let sign: i32; + let mut hx: u32; + + hx = (x.to_bits() >> 32) as u32; + sign = (hx >> 31) as i32; + hx &= 0x7fffffff; /* high word of |x| */ + + /* special cases */ + if hx >= 0x4086232b { + /* if |x| >= 708.39... */ + if x.is_nan() { + return x; + } + if x > 709.782712893383973096 { + /* overflow if x!=inf */ + x *= x1p1023; + return x; + } + if x < -708.39641853226410622 { + /* underflow if x!=-inf */ + force_eval!((-x1p_149 / x) as f32); + if x < -745.13321910194110842 { + return 0.; + } + } + } + + /* argument reduction */ + if hx > 0x3fd62e42 { + /* if |x| > 0.5 ln2 */ + if hx >= 0x3ff0a2b2 { + /* if |x| >= 1.5 ln2 */ + k = (INVLN2 * x + i!(HALF, sign as usize)) as i32; + } else { + k = 1 - sign - sign; + } + hi = x - k as f64 * LN2HI; /* k*ln2hi is exact here */ + lo = k as f64 * LN2LO; + x = hi - lo; + } else if hx > 0x3e300000 { + /* if |x| > 2**-28 */ + k = 0; + hi = x; + lo = 0.; + } else { + /* inexact if x!=0 */ + force_eval!(x1p1023 + x); + return 1. + x; + } + + /* x is now in primary range */ + xx = x * x; + c = x - xx * (P1 + xx * (P2 + xx * (P3 + xx * (P4 + xx * P5)))); + y = 1. + (x * c / (2. - c) - lo + hi); + if k == 0 { + y + } else { + scalbn(y, k) + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/exp10.rs b/crux-mir/lib/compiler_builtins/libm/src/math/exp10.rs new file mode 100644 index 000000000..559930e10 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/exp10.rs @@ -0,0 +1,22 @@ +use super::{exp2, modf, pow}; + +const LN10: f64 = 3.32192809488736234787031942948939; +const P10: &[f64] = &[ + 1e-15, 1e-14, 1e-13, 1e-12, 1e-11, 1e-10, 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, +]; + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn exp10(x: f64) -> f64 { + let (mut y, n) = modf(x); + let u: u64 = n.to_bits(); + /* fabs(n) < 16 without raising invalid on nan */ + if (u >> 52 & 0x7ff) < 0x3ff + 4 { + if y == 0.0 { + return i!(P10, ((n as isize) + 15) as usize); + } + y = exp2(LN10 * y); + return y * i!(P10, ((n as isize) + 15) as usize); + } + return pow(10.0, x); +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/exp10f.rs b/crux-mir/lib/compiler_builtins/libm/src/math/exp10f.rs new file mode 100644 index 000000000..1279bc6c5 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/exp10f.rs @@ -0,0 +1,22 @@ +use super::{exp2, exp2f, modff}; + +const LN10_F32: f32 = 3.32192809488736234787031942948939; +const LN10_F64: f64 = 3.32192809488736234787031942948939; +const P10: &[f32] = &[ + 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, +]; + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn exp10f(x: f32) -> f32 { + let (mut y, n) = modff(x); + let u = n.to_bits(); + /* fabsf(n) < 8 without raising invalid on nan */ + if (u >> 23 & 0xff) < 0x7f + 3 { + if y == 0.0 { + return i!(P10, ((n as isize) + 7) as usize); + } + y = exp2f(LN10_F32 * y); + return y * i!(P10, ((n as isize) + 7) as usize); + } + return exp2(LN10_F64 * (x as f64)) as f32; +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/exp2.rs b/crux-mir/lib/compiler_builtins/libm/src/math/exp2.rs new file mode 100644 index 000000000..e0e385df2 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/exp2.rs @@ -0,0 +1,394 @@ +// origin: FreeBSD /usr/src/lib/msun/src/s_exp2.c */ +//- +// Copyright (c) 2005 David Schultz +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +// OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +// SUCH DAMAGE. + +use super::scalbn; + +const TBLSIZE: usize = 256; + +#[cfg_attr(rustfmt, rustfmt_skip)] +static TBL: [u64; TBLSIZE * 2] = [ + // exp2(z + eps) eps + 0x3fe6a09e667f3d5d, 0x3d39880000000000, + 0x3fe6b052fa751744, 0x3cd8000000000000, + 0x3fe6c012750bd9fe, 0xbd28780000000000, + 0x3fe6cfdcddd476bf, 0x3d1ec00000000000, + 0x3fe6dfb23c651a29, 0xbcd8000000000000, + 0x3fe6ef9298593ae3, 0xbcbc000000000000, + 0x3fe6ff7df9519386, 0xbd2fd80000000000, + 0x3fe70f7466f42da3, 0xbd2c880000000000, + 0x3fe71f75e8ec5fc3, 0x3d13c00000000000, + 0x3fe72f8286eacf05, 0xbd38300000000000, + 0x3fe73f9a48a58152, 0xbd00c00000000000, + 0x3fe74fbd35d7ccfc, 0x3d2f880000000000, + 0x3fe75feb564267f1, 0x3d03e00000000000, + 0x3fe77024b1ab6d48, 0xbd27d00000000000, + 0x3fe780694fde5d38, 0xbcdd000000000000, + 0x3fe790b938ac1d00, 0x3ce3000000000000, + 0x3fe7a11473eb0178, 0xbced000000000000, + 0x3fe7b17b0976d060, 0x3d20400000000000, + 0x3fe7c1ed0130c133, 0x3ca0000000000000, + 0x3fe7d26a62ff8636, 0xbd26900000000000, + 0x3fe7e2f336cf4e3b, 0xbd02e00000000000, + 0x3fe7f3878491c3e8, 0xbd24580000000000, + 0x3fe80427543e1b4e, 0x3d33000000000000, + 0x3fe814d2add1071a, 0x3d0f000000000000, + 0x3fe82589994ccd7e, 0xbd21c00000000000, + 0x3fe8364c1eb942d0, 0x3d29d00000000000, + 0x3fe8471a4623cab5, 0x3d47100000000000, + 0x3fe857f4179f5bbc, 0x3d22600000000000, + 0x3fe868d99b4491af, 0xbd32c40000000000, + 0x3fe879cad931a395, 0xbd23000000000000, + 0x3fe88ac7d98a65b8, 0xbd2a800000000000, + 0x3fe89bd0a4785800, 0xbced000000000000, + 0x3fe8ace5422aa223, 0x3d33280000000000, + 0x3fe8be05bad619fa, 0x3d42b40000000000, + 0x3fe8cf3216b54383, 0xbd2ed00000000000, + 0x3fe8e06a5e08664c, 0xbd20500000000000, + 0x3fe8f1ae99157807, 0x3d28280000000000, + 0x3fe902fed0282c0e, 0xbd1cb00000000000, + 0x3fe9145b0b91ff96, 0xbd05e00000000000, + 0x3fe925c353aa2ff9, 0x3cf5400000000000, + 0x3fe93737b0cdc64a, 0x3d17200000000000, + 0x3fe948b82b5f98ae, 0xbd09000000000000, + 0x3fe95a44cbc852cb, 0x3d25680000000000, + 0x3fe96bdd9a766f21, 0xbd36d00000000000, + 0x3fe97d829fde4e2a, 0xbd01000000000000, + 0x3fe98f33e47a23a3, 0x3d2d000000000000, + 0x3fe9a0f170ca0604, 0xbd38a40000000000, + 0x3fe9b2bb4d53ff89, 0x3d355c0000000000, + 0x3fe9c49182a3f15b, 0x3d26b80000000000, + 0x3fe9d674194bb8c5, 0xbcec000000000000, + 0x3fe9e86319e3238e, 0x3d17d00000000000, + 0x3fe9fa5e8d07f302, 0x3d16400000000000, + 0x3fea0c667b5de54d, 0xbcf5000000000000, + 0x3fea1e7aed8eb8f6, 0x3d09e00000000000, + 0x3fea309bec4a2e27, 0x3d2ad80000000000, + 0x3fea42c980460a5d, 0xbd1af00000000000, + 0x3fea5503b23e259b, 0x3d0b600000000000, + 0x3fea674a8af46213, 0x3d38880000000000, + 0x3fea799e1330b3a7, 0x3d11200000000000, + 0x3fea8bfe53c12e8d, 0x3d06c00000000000, + 0x3fea9e6b5579fcd2, 0xbd29b80000000000, + 0x3feab0e521356fb8, 0x3d2b700000000000, + 0x3feac36bbfd3f381, 0x3cd9000000000000, + 0x3fead5ff3a3c2780, 0x3ce4000000000000, + 0x3feae89f995ad2a3, 0xbd2c900000000000, + 0x3feafb4ce622f367, 0x3d16500000000000, + 0x3feb0e07298db790, 0x3d2fd40000000000, + 0x3feb20ce6c9a89a9, 0x3d12700000000000, + 0x3feb33a2b84f1a4b, 0x3d4d470000000000, + 0x3feb468415b747e7, 0xbd38380000000000, + 0x3feb59728de5593a, 0x3c98000000000000, + 0x3feb6c6e29f1c56a, 0x3d0ad00000000000, + 0x3feb7f76f2fb5e50, 0x3cde800000000000, + 0x3feb928cf22749b2, 0xbd04c00000000000, + 0x3feba5b030a10603, 0xbd0d700000000000, + 0x3febb8e0b79a6f66, 0x3d0d900000000000, + 0x3febcc1e904bc1ff, 0x3d02a00000000000, + 0x3febdf69c3f3a16f, 0xbd1f780000000000, + 0x3febf2c25bd71db8, 0xbd10a00000000000, + 0x3fec06286141b2e9, 0xbd11400000000000, + 0x3fec199bdd8552e0, 0x3d0be00000000000, + 0x3fec2d1cd9fa64ee, 0xbd09400000000000, + 0x3fec40ab5fffd02f, 0xbd0ed00000000000, + 0x3fec544778fafd15, 0x3d39660000000000, + 0x3fec67f12e57d0cb, 0xbd1a100000000000, + 0x3fec7ba88988c1b6, 0xbd58458000000000, + 0x3fec8f6d9406e733, 0xbd1a480000000000, + 0x3feca3405751c4df, 0x3ccb000000000000, + 0x3fecb720dcef9094, 0x3d01400000000000, + 0x3feccb0f2e6d1689, 0x3cf0200000000000, + 0x3fecdf0b555dc412, 0x3cf3600000000000, + 0x3fecf3155b5bab3b, 0xbd06900000000000, + 0x3fed072d4a0789bc, 0x3d09a00000000000, + 0x3fed1b532b08c8fa, 0xbd15e00000000000, + 0x3fed2f87080d8a85, 0x3d1d280000000000, + 0x3fed43c8eacaa203, 0x3d01a00000000000, + 0x3fed5818dcfba491, 0x3cdf000000000000, + 0x3fed6c76e862e6a1, 0xbd03a00000000000, + 0x3fed80e316c9834e, 0xbd0cd80000000000, + 0x3fed955d71ff6090, 0x3cf4c00000000000, + 0x3feda9e603db32ae, 0x3cff900000000000, + 0x3fedbe7cd63a8325, 0x3ce9800000000000, + 0x3fedd321f301b445, 0xbcf5200000000000, + 0x3fede7d5641c05bf, 0xbd1d700000000000, + 0x3fedfc97337b9aec, 0xbd16140000000000, + 0x3fee11676b197d5e, 0x3d0b480000000000, + 0x3fee264614f5a3e7, 0x3d40ce0000000000, + 0x3fee3b333b16ee5c, 0x3d0c680000000000, + 0x3fee502ee78b3fb4, 0xbd09300000000000, + 0x3fee653924676d68, 0xbce5000000000000, + 0x3fee7a51fbc74c44, 0xbd07f80000000000, + 0x3fee8f7977cdb726, 0xbcf3700000000000, + 0x3feea4afa2a490e8, 0x3ce5d00000000000, + 0x3feeb9f4867ccae4, 0x3d161a0000000000, + 0x3feecf482d8e680d, 0x3cf5500000000000, + 0x3feee4aaa2188514, 0x3cc6400000000000, + 0x3feefa1bee615a13, 0xbcee800000000000, + 0x3fef0f9c1cb64106, 0xbcfa880000000000, + 0x3fef252b376bb963, 0xbd2c900000000000, + 0x3fef3ac948dd7275, 0x3caa000000000000, + 0x3fef50765b6e4524, 0xbcf4f00000000000, + 0x3fef6632798844fd, 0x3cca800000000000, + 0x3fef7bfdad9cbe38, 0x3cfabc0000000000, + 0x3fef91d802243c82, 0xbcd4600000000000, + 0x3fefa7c1819e908e, 0xbd0b0c0000000000, + 0x3fefbdba3692d511, 0xbcc0e00000000000, + 0x3fefd3c22b8f7194, 0xbd10de8000000000, + 0x3fefe9d96b2a23ee, 0x3cee430000000000, + 0x3ff0000000000000, 0x0, + 0x3ff00b1afa5abcbe, 0xbcb3400000000000, + 0x3ff0163da9fb3303, 0xbd12170000000000, + 0x3ff02168143b0282, 0x3cba400000000000, + 0x3ff02c9a3e77806c, 0x3cef980000000000, + 0x3ff037d42e11bbca, 0xbcc7400000000000, + 0x3ff04315e86e7f89, 0x3cd8300000000000, + 0x3ff04e5f72f65467, 0xbd1a3f0000000000, + 0x3ff059b0d315855a, 0xbd02840000000000, + 0x3ff0650a0e3c1f95, 0x3cf1600000000000, + 0x3ff0706b29ddf71a, 0x3d15240000000000, + 0x3ff07bd42b72a82d, 0xbce9a00000000000, + 0x3ff0874518759bd0, 0x3ce6400000000000, + 0x3ff092bdf66607c8, 0xbd00780000000000, + 0x3ff09e3ecac6f383, 0xbc98000000000000, + 0x3ff0a9c79b1f3930, 0x3cffa00000000000, + 0x3ff0b5586cf988fc, 0xbcfac80000000000, + 0x3ff0c0f145e46c8a, 0x3cd9c00000000000, + 0x3ff0cc922b724816, 0x3d05200000000000, + 0x3ff0d83b23395dd8, 0xbcfad00000000000, + 0x3ff0e3ec32d3d1f3, 0x3d1bac0000000000, + 0x3ff0efa55fdfa9a6, 0xbd04e80000000000, + 0x3ff0fb66affed2f0, 0xbd0d300000000000, + 0x3ff1073028d7234b, 0x3cf1500000000000, + 0x3ff11301d0125b5b, 0x3cec000000000000, + 0x3ff11edbab5e2af9, 0x3d16bc0000000000, + 0x3ff12abdc06c31d5, 0x3ce8400000000000, + 0x3ff136a814f2047d, 0xbd0ed00000000000, + 0x3ff1429aaea92de9, 0x3ce8e00000000000, + 0x3ff14e95934f3138, 0x3ceb400000000000, + 0x3ff15a98c8a58e71, 0x3d05300000000000, + 0x3ff166a45471c3df, 0x3d03380000000000, + 0x3ff172b83c7d5211, 0x3d28d40000000000, + 0x3ff17ed48695bb9f, 0xbd05d00000000000, + 0x3ff18af9388c8d93, 0xbd1c880000000000, + 0x3ff1972658375d66, 0x3d11f00000000000, + 0x3ff1a35beb6fcba7, 0x3d10480000000000, + 0x3ff1af99f81387e3, 0xbd47390000000000, + 0x3ff1bbe084045d54, 0x3d24e40000000000, + 0x3ff1c82f95281c43, 0xbd0a200000000000, + 0x3ff1d4873168b9b2, 0x3ce3800000000000, + 0x3ff1e0e75eb44031, 0x3ceac00000000000, + 0x3ff1ed5022fcd938, 0x3d01900000000000, + 0x3ff1f9c18438cdf7, 0xbd1b780000000000, + 0x3ff2063b88628d8f, 0x3d2d940000000000, + 0x3ff212be3578a81e, 0x3cd8000000000000, + 0x3ff21f49917ddd41, 0x3d2b340000000000, + 0x3ff22bdda2791323, 0x3d19f80000000000, + 0x3ff2387a6e7561e7, 0xbd19c80000000000, + 0x3ff2451ffb821427, 0x3d02300000000000, + 0x3ff251ce4fb2a602, 0xbd13480000000000, + 0x3ff25e85711eceb0, 0x3d12700000000000, + 0x3ff26b4565e27d16, 0x3d11d00000000000, + 0x3ff2780e341de00f, 0x3d31ee0000000000, + 0x3ff284dfe1f5633e, 0xbd14c00000000000, + 0x3ff291ba7591bb30, 0xbd13d80000000000, + 0x3ff29e9df51fdf09, 0x3d08b00000000000, + 0x3ff2ab8a66d10e9b, 0xbd227c0000000000, + 0x3ff2b87fd0dada3a, 0x3d2a340000000000, + 0x3ff2c57e39771af9, 0xbd10800000000000, + 0x3ff2d285a6e402d9, 0xbd0ed00000000000, + 0x3ff2df961f641579, 0xbcf4200000000000, + 0x3ff2ecafa93e2ecf, 0xbd24980000000000, + 0x3ff2f9d24abd8822, 0xbd16300000000000, + 0x3ff306fe0a31b625, 0xbd32360000000000, + 0x3ff31432edeea50b, 0xbd70df8000000000, + 0x3ff32170fc4cd7b8, 0xbd22480000000000, + 0x3ff32eb83ba8e9a2, 0xbd25980000000000, + 0x3ff33c08b2641766, 0x3d1ed00000000000, + 0x3ff3496266e3fa27, 0xbcdc000000000000, + 0x3ff356c55f929f0f, 0xbd30d80000000000, + 0x3ff36431a2de88b9, 0x3d22c80000000000, + 0x3ff371a7373aaa39, 0x3d20600000000000, + 0x3ff37f26231e74fe, 0xbd16600000000000, + 0x3ff38cae6d05d838, 0xbd0ae00000000000, + 0x3ff39a401b713ec3, 0xbd44720000000000, + 0x3ff3a7db34e5a020, 0x3d08200000000000, + 0x3ff3b57fbfec6e95, 0x3d3e800000000000, + 0x3ff3c32dc313a8f2, 0x3cef800000000000, + 0x3ff3d0e544ede122, 0xbd17a00000000000, + 0x3ff3dea64c1234bb, 0x3d26300000000000, + 0x3ff3ec70df1c4ecc, 0xbd48a60000000000, + 0x3ff3fa4504ac7e8c, 0xbd3cdc0000000000, + 0x3ff40822c367a0bb, 0x3d25b80000000000, + 0x3ff4160a21f72e95, 0x3d1ec00000000000, + 0x3ff423fb27094646, 0xbd13600000000000, + 0x3ff431f5d950a920, 0x3d23980000000000, + 0x3ff43ffa3f84b9eb, 0x3cfa000000000000, + 0x3ff44e0860618919, 0xbcf6c00000000000, + 0x3ff45c2042a7d201, 0xbd0bc00000000000, + 0x3ff46a41ed1d0016, 0xbd12800000000000, + 0x3ff4786d668b3326, 0x3d30e00000000000, + 0x3ff486a2b5c13c00, 0xbd2d400000000000, + 0x3ff494e1e192af04, 0x3d0c200000000000, + 0x3ff4a32af0d7d372, 0xbd1e500000000000, + 0x3ff4b17dea6db801, 0x3d07800000000000, + 0x3ff4bfdad53629e1, 0xbd13800000000000, + 0x3ff4ce41b817c132, 0x3d00800000000000, + 0x3ff4dcb299fddddb, 0x3d2c700000000000, + 0x3ff4eb2d81d8ab96, 0xbd1ce00000000000, + 0x3ff4f9b2769d2d02, 0x3d19200000000000, + 0x3ff508417f4531c1, 0xbd08c00000000000, + 0x3ff516daa2cf662a, 0xbcfa000000000000, + 0x3ff5257de83f51ea, 0x3d4a080000000000, + 0x3ff5342b569d4eda, 0xbd26d80000000000, + 0x3ff542e2f4f6ac1a, 0xbd32440000000000, + 0x3ff551a4ca5d94db, 0x3d483c0000000000, + 0x3ff56070dde9116b, 0x3d24b00000000000, + 0x3ff56f4736b529de, 0x3d415a0000000000, + 0x3ff57e27dbe2c40e, 0xbd29e00000000000, + 0x3ff58d12d497c76f, 0xbd23080000000000, + 0x3ff59c0827ff0b4c, 0x3d4dec0000000000, + 0x3ff5ab07dd485427, 0xbcc4000000000000, + 0x3ff5ba11fba87af4, 0x3d30080000000000, + 0x3ff5c9268a59460b, 0xbd26c80000000000, + 0x3ff5d84590998e3f, 0x3d469a0000000000, + 0x3ff5e76f15ad20e1, 0xbd1b400000000000, + 0x3ff5f6a320dcebca, 0x3d17700000000000, + 0x3ff605e1b976dcb8, 0x3d26f80000000000, + 0x3ff6152ae6cdf715, 0x3d01000000000000, + 0x3ff6247eb03a5531, 0xbd15d00000000000, + 0x3ff633dd1d1929b5, 0xbd12d00000000000, + 0x3ff6434634ccc313, 0xbcea800000000000, + 0x3ff652b9febc8efa, 0xbd28600000000000, + 0x3ff6623882553397, 0x3d71fe0000000000, + 0x3ff671c1c708328e, 0xbd37200000000000, + 0x3ff68155d44ca97e, 0x3ce6800000000000, + 0x3ff690f4b19e9471, 0xbd29780000000000, +]; + +// exp2(x): compute the base 2 exponential of x +// +// Accuracy: Peak error < 0.503 ulp for normalized results. +// +// Method: (accurate tables) +// +// Reduce x: +// x = k + y, for integer k and |y| <= 1/2. +// Thus we have exp2(x) = 2**k * exp2(y). +// +// Reduce y: +// y = i/TBLSIZE + z - eps[i] for integer i near y * TBLSIZE. +// Thus we have exp2(y) = exp2(i/TBLSIZE) * exp2(z - eps[i]), +// with |z - eps[i]| <= 2**-9 + 2**-39 for the table used. +// +// We compute exp2(i/TBLSIZE) via table lookup and exp2(z - eps[i]) via +// a degree-5 minimax polynomial with maximum error under 1.3 * 2**-61. +// The values in exp2t[] and eps[] are chosen such that +// exp2t[i] = exp2(i/TBLSIZE + eps[i]), and eps[i] is a small offset such +// that exp2t[i] is accurate to 2**-64. +// +// Note that the range of i is +-TBLSIZE/2, so we actually index the tables +// by i0 = i + TBLSIZE/2. For cache efficiency, exp2t[] and eps[] are +// virtual tables, interleaved in the real table tbl[]. +// +// This method is due to Gal, with many details due to Gal and Bachelis: +// +// Gal, S. and Bachelis, B. An Accurate Elementary Mathematical Library +// for the IEEE Floating Point Standard. TOMS 17(1), 26-46 (1991). + +/// Exponential, base 2 (f64) +/// +/// Calculate `2^x`, that is, 2 raised to the power `x`. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn exp2(mut x: f64) -> f64 { + let redux = f64::from_bits(0x4338000000000000) / TBLSIZE as f64; + let p1 = f64::from_bits(0x3fe62e42fefa39ef); + let p2 = f64::from_bits(0x3fcebfbdff82c575); + let p3 = f64::from_bits(0x3fac6b08d704a0a6); + let p4 = f64::from_bits(0x3f83b2ab88f70400); + let p5 = f64::from_bits(0x3f55d88003875c74); + + // double_t r, t, z; + // uint32_t ix, i0; + // union {double f; uint64_t i;} u = {x}; + // union {uint32_t u; int32_t i;} k; + let x1p1023 = f64::from_bits(0x7fe0000000000000); + let x1p52 = f64::from_bits(0x4330000000000000); + let _0x1p_149 = f64::from_bits(0xb6a0000000000000); + + /* Filter out exceptional cases. */ + let ui = f64::to_bits(x); + let ix = ui >> 32 & 0x7fffffff; + if ix >= 0x408ff000 { + /* |x| >= 1022 or nan */ + if ix >= 0x40900000 && ui >> 63 == 0 { + /* x >= 1024 or nan */ + /* overflow */ + x *= x1p1023; + return x; + } + if ix >= 0x7ff00000 { + /* -inf or -nan */ + return -1.0 / x; + } + if ui >> 63 != 0 { + /* x <= -1022 */ + /* underflow */ + if x <= -1075.0 || x - x1p52 + x1p52 != x { + force_eval!((_0x1p_149 / x) as f32); + } + if x <= -1075.0 { + return 0.0; + } + } + } else if ix < 0x3c900000 { + /* |x| < 0x1p-54 */ + return 1.0 + x; + } + + /* Reduce x, computing z, i0, and k. */ + let ui = f64::to_bits(x + redux); + let mut i0 = ui as u32; + i0 = i0.wrapping_add(TBLSIZE as u32 / 2); + let ku = i0 / TBLSIZE as u32 * TBLSIZE as u32; + let ki = div!(ku as i32, TBLSIZE as i32); + i0 %= TBLSIZE as u32; + let uf = f64::from_bits(ui) - redux; + let mut z = x - uf; + + /* Compute r = exp2(y) = exp2t[i0] * p(z - eps[i]). */ + let t = f64::from_bits(i!(TBL, 2 * i0 as usize)); /* exp2t[i0] */ + z -= f64::from_bits(i!(TBL, 2 * i0 as usize + 1)); /* eps[i0] */ + let r = t + t * z * (p1 + z * (p2 + z * (p3 + z * (p4 + z * p5)))); + + scalbn(r, ki) +} + +#[test] +fn i0_wrap_test() { + let x = -3.0 / 256.0; + assert_eq!(exp2(x), f64::from_bits(0x3fefbdba3692d514)); +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/exp2f.rs b/crux-mir/lib/compiler_builtins/libm/src/math/exp2f.rs new file mode 100644 index 000000000..f4867b80e --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/exp2f.rs @@ -0,0 +1,135 @@ +// origin: FreeBSD /usr/src/lib/msun/src/s_exp2f.c +//- +// Copyright (c) 2005 David Schultz +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +// OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +// SUCH DAMAGE. + +const TBLSIZE: usize = 16; + +static EXP2FT: [u64; TBLSIZE] = [ + 0x3fe6a09e667f3bcd, + 0x3fe7a11473eb0187, + 0x3fe8ace5422aa0db, + 0x3fe9c49182a3f090, + 0x3feae89f995ad3ad, + 0x3fec199bdd85529c, + 0x3fed5818dcfba487, + 0x3feea4afa2a490da, + 0x3ff0000000000000, + 0x3ff0b5586cf9890f, + 0x3ff172b83c7d517b, + 0x3ff2387a6e756238, + 0x3ff306fe0a31b715, + 0x3ff3dea64c123422, + 0x3ff4bfdad5362a27, + 0x3ff5ab07dd485429, +]; + +// exp2f(x): compute the base 2 exponential of x +// +// Accuracy: Peak error < 0.501 ulp; location of peak: -0.030110927. +// +// Method: (equally-spaced tables) +// +// Reduce x: +// x = k + y, for integer k and |y| <= 1/2. +// Thus we have exp2f(x) = 2**k * exp2(y). +// +// Reduce y: +// y = i/TBLSIZE + z for integer i near y * TBLSIZE. +// Thus we have exp2(y) = exp2(i/TBLSIZE) * exp2(z), +// with |z| <= 2**-(TBLSIZE+1). +// +// We compute exp2(i/TBLSIZE) via table lookup and exp2(z) via a +// degree-4 minimax polynomial with maximum error under 1.4 * 2**-33. +// Using double precision for everything except the reduction makes +// roundoff error insignificant and simplifies the scaling step. +// +// This method is due to Tang, but I do not use his suggested parameters: +// +// Tang, P. Table-driven Implementation of the Exponential Function +// in IEEE Floating-Point Arithmetic. TOMS 15(2), 144-157 (1989). + +/// Exponential, base 2 (f32) +/// +/// Calculate `2^x`, that is, 2 raised to the power `x`. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn exp2f(mut x: f32) -> f32 { + let redux = f32::from_bits(0x4b400000) / TBLSIZE as f32; + let p1 = f32::from_bits(0x3f317218); + let p2 = f32::from_bits(0x3e75fdf0); + let p3 = f32::from_bits(0x3d6359a4); + let p4 = f32::from_bits(0x3c1d964e); + + // double_t t, r, z; + // uint32_t ix, i0, k; + + let x1p127 = f32::from_bits(0x7f000000); + + /* Filter out exceptional cases. */ + let ui = f32::to_bits(x); + let ix = ui & 0x7fffffff; + if ix > 0x42fc0000 { + /* |x| > 126 */ + if ix > 0x7f800000 { + /* NaN */ + return x; + } + if ui >= 0x43000000 && ui < 0x80000000 { + /* x >= 128 */ + x *= x1p127; + return x; + } + if ui >= 0x80000000 { + /* x < -126 */ + if ui >= 0xc3160000 || (ui & 0x0000ffff != 0) { + force_eval!(f32::from_bits(0x80000001) / x); + } + if ui >= 0xc3160000 { + /* x <= -150 */ + return 0.0; + } + } + } else if ix <= 0x33000000 { + /* |x| <= 0x1p-25 */ + return 1.0 + x; + } + + /* Reduce x, computing z, i0, and k. */ + let ui = f32::to_bits(x + redux); + let mut i0 = ui; + i0 += TBLSIZE as u32 / 2; + let k = i0 / TBLSIZE as u32; + let ukf = f64::from_bits(((0x3ff + k) as u64) << 52); + i0 &= TBLSIZE as u32 - 1; + let mut uf = f32::from_bits(ui); + uf -= redux; + let z: f64 = (x - uf) as f64; + /* Compute r = exp2(y) = exp2ft[i0] * p(z). */ + let r: f64 = f64::from_bits(i!(EXP2FT, i0 as usize)); + let t: f64 = r as f64 * z; + let r: f64 = r + t * (p1 as f64 + z * p2 as f64) + t * (z * z) * (p3 as f64 + z * p4 as f64); + + /* Scale by 2**k */ + (r * ukf) as f32 +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/expf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/expf.rs new file mode 100644 index 000000000..a53aa90a6 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/expf.rs @@ -0,0 +1,101 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_expf.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +use super::scalbnf; + +const HALF: [f32; 2] = [0.5, -0.5]; +const LN2_HI: f32 = 6.9314575195e-01; /* 0x3f317200 */ +const LN2_LO: f32 = 1.4286067653e-06; /* 0x35bfbe8e */ +const INV_LN2: f32 = 1.4426950216e+00; /* 0x3fb8aa3b */ +/* + * Domain [-0.34568, 0.34568], range ~[-4.278e-9, 4.447e-9]: + * |x*(exp(x)+1)/(exp(x)-1) - p(x)| < 2**-27.74 + */ +const P1: f32 = 1.6666625440e-1; /* 0xaaaa8f.0p-26 */ +const P2: f32 = -2.7667332906e-3; /* -0xb55215.0p-32 */ + +/// Exponential, base *e* (f32) +/// +/// Calculate the exponential of `x`, that is, *e* raised to the power `x` +/// (where *e* is the base of the natural system of logarithms, approximately 2.71828). +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn expf(mut x: f32) -> f32 { + let x1p127 = f32::from_bits(0x7f000000); // 0x1p127f === 2 ^ 127 + let x1p_126 = f32::from_bits(0x800000); // 0x1p-126f === 2 ^ -126 /*original 0x1p-149f ??????????? */ + let mut hx = x.to_bits(); + let sign = (hx >> 31) as i32; /* sign bit of x */ + let signb: bool = sign != 0; + hx &= 0x7fffffff; /* high word of |x| */ + + /* special cases */ + if hx >= 0x42aeac50 { + /* if |x| >= -87.33655f or NaN */ + if hx > 0x7f800000 { + /* NaN */ + return x; + } + if (hx >= 0x42b17218) && (!signb) { + /* x >= 88.722839f */ + /* overflow */ + x *= x1p127; + return x; + } + if signb { + /* underflow */ + force_eval!(-x1p_126 / x); + if hx >= 0x42cff1b5 { + /* x <= -103.972084f */ + return 0.; + } + } + } + + /* argument reduction */ + let k: i32; + let hi: f32; + let lo: f32; + if hx > 0x3eb17218 { + /* if |x| > 0.5 ln2 */ + if hx > 0x3f851592 { + /* if |x| > 1.5 ln2 */ + k = (INV_LN2 * x + i!(HALF, sign as usize)) as i32; + } else { + k = 1 - sign - sign; + } + let kf = k as f32; + hi = x - kf * LN2_HI; /* k*ln2hi is exact here */ + lo = kf * LN2_LO; + x = hi - lo; + } else if hx > 0x39000000 { + /* |x| > 2**-14 */ + k = 0; + hi = x; + lo = 0.; + } else { + /* raise inexact */ + force_eval!(x1p127 + x); + return 1. + x; + } + + /* x is now in primary range */ + let xx = x * x; + let c = x - xx * (P1 + xx * P2); + let y = 1. + (x * c / (2. - c) - lo + hi); + if k == 0 { + y + } else { + scalbnf(y, k) + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/expm1.rs b/crux-mir/lib/compiler_builtins/libm/src/math/expm1.rs new file mode 100644 index 000000000..42608509a --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/expm1.rs @@ -0,0 +1,144 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/s_expm1.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +use core::f64; + +const O_THRESHOLD: f64 = 7.09782712893383973096e+02; /* 0x40862E42, 0xFEFA39EF */ +const LN2_HI: f64 = 6.93147180369123816490e-01; /* 0x3fe62e42, 0xfee00000 */ +const LN2_LO: f64 = 1.90821492927058770002e-10; /* 0x3dea39ef, 0x35793c76 */ +const INVLN2: f64 = 1.44269504088896338700e+00; /* 0x3ff71547, 0x652b82fe */ +/* Scaled Q's: Qn_here = 2**n * Qn_above, for R(2*z) where z = hxs = x*x/2: */ +const Q1: f64 = -3.33333333333331316428e-02; /* BFA11111 111110F4 */ +const Q2: f64 = 1.58730158725481460165e-03; /* 3F5A01A0 19FE5585 */ +const Q3: f64 = -7.93650757867487942473e-05; /* BF14CE19 9EAADBB7 */ +const Q4: f64 = 4.00821782732936239552e-06; /* 3ED0CFCA 86E65239 */ +const Q5: f64 = -2.01099218183624371326e-07; /* BE8AFDB7 6E09C32D */ + +/// Exponential, base *e*, of x-1 (f64) +/// +/// Calculates the exponential of `x` and subtract 1, that is, *e* raised +/// to the power `x` minus 1 (where *e* is the base of the natural +/// system of logarithms, approximately 2.71828). +/// The result is accurate even for small values of `x`, +/// where using `exp(x)-1` would lose many significant digits. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn expm1(mut x: f64) -> f64 { + let hi: f64; + let lo: f64; + let k: i32; + let c: f64; + let mut t: f64; + let mut y: f64; + + let mut ui = x.to_bits(); + let hx = ((ui >> 32) & 0x7fffffff) as u32; + let sign = (ui >> 63) as i32; + + /* filter out huge and non-finite argument */ + if hx >= 0x4043687A { + /* if |x|>=56*ln2 */ + if x.is_nan() { + return x; + } + if sign != 0 { + return -1.0; + } + if x > O_THRESHOLD { + x *= f64::from_bits(0x7fe0000000000000); + return x; + } + } + + /* argument reduction */ + if hx > 0x3fd62e42 { + /* if |x| > 0.5 ln2 */ + if hx < 0x3FF0A2B2 { + /* and |x| < 1.5 ln2 */ + if sign == 0 { + hi = x - LN2_HI; + lo = LN2_LO; + k = 1; + } else { + hi = x + LN2_HI; + lo = -LN2_LO; + k = -1; + } + } else { + k = (INVLN2 * x + if sign != 0 { -0.5 } else { 0.5 }) as i32; + t = k as f64; + hi = x - t * LN2_HI; /* t*ln2_hi is exact here */ + lo = t * LN2_LO; + } + x = hi - lo; + c = (hi - x) - lo; + } else if hx < 0x3c900000 { + /* |x| < 2**-54, return x */ + if hx < 0x00100000 { + force_eval!(x); + } + return x; + } else { + c = 0.0; + k = 0; + } + + /* x is now in primary range */ + let hfx = 0.5 * x; + let hxs = x * hfx; + let r1 = 1.0 + hxs * (Q1 + hxs * (Q2 + hxs * (Q3 + hxs * (Q4 + hxs * Q5)))); + t = 3.0 - r1 * hfx; + let mut e = hxs * ((r1 - t) / (6.0 - x * t)); + if k == 0 { + /* c is 0 */ + return x - (x * e - hxs); + } + e = x * (e - c) - c; + e -= hxs; + /* exp(x) ~ 2^k (x_reduced - e + 1) */ + if k == -1 { + return 0.5 * (x - e) - 0.5; + } + if k == 1 { + if x < -0.25 { + return -2.0 * (e - (x + 0.5)); + } + return 1.0 + 2.0 * (x - e); + } + ui = ((0x3ff + k) as u64) << 52; /* 2^k */ + let twopk = f64::from_bits(ui); + if k < 0 || k > 56 { + /* suffice to return exp(x)-1 */ + y = x - e + 1.0; + if k == 1024 { + y = y * 2.0 * f64::from_bits(0x7fe0000000000000); + } else { + y = y * twopk; + } + return y - 1.0; + } + ui = ((0x3ff - k) as u64) << 52; /* 2^-k */ + let uf = f64::from_bits(ui); + if k < 20 { + y = (x - e + (1.0 - uf)) * twopk; + } else { + y = (x - (e + uf) + 1.0) * twopk; + } + y +} + +#[cfg(test)] +mod tests { + #[test] + fn sanity_check() { + assert_eq!(super::expm1(1.1), 2.0041660239464334); + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/expm1f.rs b/crux-mir/lib/compiler_builtins/libm/src/math/expm1f.rs new file mode 100644 index 000000000..3fc2a247b --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/expm1f.rs @@ -0,0 +1,134 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/s_expm1f.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +const O_THRESHOLD: f32 = 8.8721679688e+01; /* 0x42b17180 */ +const LN2_HI: f32 = 6.9313812256e-01; /* 0x3f317180 */ +const LN2_LO: f32 = 9.0580006145e-06; /* 0x3717f7d1 */ +const INV_LN2: f32 = 1.4426950216e+00; /* 0x3fb8aa3b */ +/* + * Domain [-0.34568, 0.34568], range ~[-6.694e-10, 6.696e-10]: + * |6 / x * (1 + 2 * (1 / (exp(x) - 1) - 1 / x)) - q(x)| < 2**-30.04 + * Scaled coefficients: Qn_here = 2**n * Qn_for_q (see s_expm1.c): + */ +const Q1: f32 = -3.3333212137e-2; /* -0x888868.0p-28 */ +const Q2: f32 = 1.5807170421e-3; /* 0xcf3010.0p-33 */ + +/// Exponential, base *e*, of x-1 (f32) +/// +/// Calculates the exponential of `x` and subtract 1, that is, *e* raised +/// to the power `x` minus 1 (where *e* is the base of the natural +/// system of logarithms, approximately 2.71828). +/// The result is accurate even for small values of `x`, +/// where using `exp(x)-1` would lose many significant digits. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn expm1f(mut x: f32) -> f32 { + let x1p127 = f32::from_bits(0x7f000000); // 0x1p127f === 2 ^ 127 + + let mut hx = x.to_bits(); + let sign = (hx >> 31) != 0; + hx &= 0x7fffffff; + + /* filter out huge and non-finite argument */ + if hx >= 0x4195b844 { + /* if |x|>=27*ln2 */ + if hx > 0x7f800000 { + /* NaN */ + return x; + } + if sign { + return -1.; + } + if x > O_THRESHOLD { + x *= x1p127; + return x; + } + } + + let k: i32; + let hi: f32; + let lo: f32; + let mut c = 0f32; + /* argument reduction */ + if hx > 0x3eb17218 { + /* if |x| > 0.5 ln2 */ + if hx < 0x3F851592 { + /* and |x| < 1.5 ln2 */ + if !sign { + hi = x - LN2_HI; + lo = LN2_LO; + k = 1; + } else { + hi = x + LN2_HI; + lo = -LN2_LO; + k = -1; + } + } else { + k = (INV_LN2 * x + (if sign { -0.5 } else { 0.5 })) as i32; + let t = k as f32; + hi = x - t * LN2_HI; /* t*ln2_hi is exact here */ + lo = t * LN2_LO; + } + x = hi - lo; + c = (hi - x) - lo; + } else if hx < 0x33000000 { + /* when |x|<2**-25, return x */ + if hx < 0x00800000 { + force_eval!(x * x); + } + return x; + } else { + k = 0; + } + + /* x is now in primary range */ + let hfx = 0.5 * x; + let hxs = x * hfx; + let r1 = 1. + hxs * (Q1 + hxs * Q2); + let t = 3. - r1 * hfx; + let mut e = hxs * ((r1 - t) / (6. - x * t)); + if k == 0 { + /* c is 0 */ + return x - (x * e - hxs); + } + e = x * (e - c) - c; + e -= hxs; + /* exp(x) ~ 2^k (x_reduced - e + 1) */ + if k == -1 { + return 0.5 * (x - e) - 0.5; + } + if k == 1 { + if x < -0.25 { + return -2. * (e - (x + 0.5)); + } + return 1. + 2. * (x - e); + } + let twopk = f32::from_bits(((0x7f + k) << 23) as u32); /* 2^k */ + if (k < 0) || (k > 56) { + /* suffice to return exp(x)-1 */ + let mut y = x - e + 1.; + if k == 128 { + y = y * 2. * x1p127; + } else { + y = y * twopk; + } + return y - 1.; + } + let uf = f32::from_bits(((0x7f - k) << 23) as u32); /* 2^-k */ + if k < 23 { + (x - e + (1. - uf)) * twopk + } else { + (x - (e + uf) + 1.) * twopk + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/expo2.rs b/crux-mir/lib/compiler_builtins/libm/src/math/expo2.rs new file mode 100644 index 000000000..82e9b360a --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/expo2.rs @@ -0,0 +1,14 @@ +use super::{combine_words, exp}; + +/* exp(x)/2 for x >= log(DBL_MAX), slightly better than 0.5*exp(x/2)*exp(x/2) */ +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub(crate) fn expo2(x: f64) -> f64 { + /* k is such that k*ln2 has minimal relative error and x - kln2 > log(DBL_MIN) */ + const K: i32 = 2043; + let kln2 = f64::from_bits(0x40962066151add8b); + + /* note that k is odd and scale*scale overflows */ + let scale = combine_words(((0x3ff + K / 2) as u32) << 20, 0); + /* exp(x - k ln2) * 2**(k-1) */ + exp(x - kln2) * scale * scale +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/fabs.rs b/crux-mir/lib/compiler_builtins/libm/src/math/fabs.rs new file mode 100644 index 000000000..b2255ad32 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/fabs.rs @@ -0,0 +1,41 @@ +use core::u64; + +/// Absolute value (magnitude) (f64) +/// Calculates the absolute value (magnitude) of the argument `x`, +/// by direct manipulation of the bit representation of `x`. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn fabs(x: f64) -> f64 { + // On wasm32 we know that LLVM's intrinsic will compile to an optimized + // `f64.abs` native instruction, so we can leverage this for both code size + // and speed. + llvm_intrinsically_optimized! { + #[cfg(target_arch = "wasm32")] { + return unsafe { ::core::intrinsics::fabsf64(x) } + } + } + f64::from_bits(x.to_bits() & (u64::MAX / 2)) +} + +#[cfg(test)] +mod tests { + use super::*; + use core::f64::*; + + #[test] + fn sanity_check() { + assert_eq!(fabs(-1.0), 1.0); + assert_eq!(fabs(2.8), 2.8); + } + + /// The spec: https://en.cppreference.com/w/cpp/numeric/math/fabs + #[test] + fn spec_tests() { + assert!(fabs(NAN).is_nan()); + for f in [0.0, -0.0].iter().copied() { + assert_eq!(fabs(f), 0.0); + } + for f in [INFINITY, NEG_INFINITY].iter().copied() { + assert_eq!(fabs(f), INFINITY); + } + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/fabsf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/fabsf.rs new file mode 100644 index 000000000..23f3646dc --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/fabsf.rs @@ -0,0 +1,41 @@ +/// Absolute value (magnitude) (f32) +/// Calculates the absolute value (magnitude) of the argument `x`, +/// by direct manipulation of the bit representation of `x`. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn fabsf(x: f32) -> f32 { + // On wasm32 we know that LLVM's intrinsic will compile to an optimized + // `f32.abs` native instruction, so we can leverage this for both code size + // and speed. + llvm_intrinsically_optimized! { + #[cfg(target_arch = "wasm32")] { + return unsafe { ::core::intrinsics::fabsf32(x) } + } + } + f32::from_bits(x.to_bits() & 0x7fffffff) +} + +// PowerPC tests are failing on LLVM 13: https://github.com/rust-lang/rust/issues/88520 +#[cfg(not(target_arch = "powerpc64"))] +#[cfg(test)] +mod tests { + use super::*; + use core::f32::*; + + #[test] + fn sanity_check() { + assert_eq!(fabsf(-1.0), 1.0); + assert_eq!(fabsf(2.8), 2.8); + } + + /// The spec: https://en.cppreference.com/w/cpp/numeric/math/fabs + #[test] + fn spec_tests() { + assert!(fabsf(NAN).is_nan()); + for f in [0.0, -0.0].iter().copied() { + assert_eq!(fabsf(f), 0.0); + } + for f in [INFINITY, NEG_INFINITY].iter().copied() { + assert_eq!(fabsf(f), INFINITY); + } + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/fdim.rs b/crux-mir/lib/compiler_builtins/libm/src/math/fdim.rs new file mode 100644 index 000000000..014930097 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/fdim.rs @@ -0,0 +1,22 @@ +use core::f64; + +/// Positive difference (f64) +/// +/// Determines the positive difference between arguments, returning: +/// * x - y if x > y, or +/// * +0 if x <= y, or +/// * NAN if either argument is NAN. +/// +/// A range error may occur. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn fdim(x: f64, y: f64) -> f64 { + if x.is_nan() { + x + } else if y.is_nan() { + y + } else if x > y { + x - y + } else { + 0.0 + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/fdimf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/fdimf.rs new file mode 100644 index 000000000..ea0b592d7 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/fdimf.rs @@ -0,0 +1,22 @@ +use core::f32; + +/// Positive difference (f32) +/// +/// Determines the positive difference between arguments, returning: +/// * x - y if x > y, or +/// * +0 if x <= y, or +/// * NAN if either argument is NAN. +/// +/// A range error may occur. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn fdimf(x: f32, y: f32) -> f32 { + if x.is_nan() { + x + } else if y.is_nan() { + y + } else if x > y { + x - y + } else { + 0.0 + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/fenv.rs b/crux-mir/lib/compiler_builtins/libm/src/math/fenv.rs new file mode 100644 index 000000000..c91272e82 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/fenv.rs @@ -0,0 +1,27 @@ +// src: musl/src/fenv/fenv.c +/* Dummy functions for archs lacking fenv implementation */ + +pub(crate) const FE_UNDERFLOW: i32 = 0; +pub(crate) const FE_INEXACT: i32 = 0; + +pub(crate) const FE_TONEAREST: i32 = 0; + +#[inline] +pub(crate) fn feclearexcept(_mask: i32) -> i32 { + 0 +} + +#[inline] +pub(crate) fn feraiseexcept(_mask: i32) -> i32 { + 0 +} + +#[inline] +pub(crate) fn fetestexcept(_mask: i32) -> i32 { + 0 +} + +#[inline] +pub(crate) fn fegetround() -> i32 { + FE_TONEAREST +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/floor.rs b/crux-mir/lib/compiler_builtins/libm/src/math/floor.rs new file mode 100644 index 000000000..d09f9a1a1 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/floor.rs @@ -0,0 +1,81 @@ +#![allow(unreachable_code)] +use core::f64; + +const TOINT: f64 = 1. / f64::EPSILON; + +/// Floor (f64) +/// +/// Finds the nearest integer less than or equal to `x`. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn floor(x: f64) -> f64 { + // On wasm32 we know that LLVM's intrinsic will compile to an optimized + // `f64.floor` native instruction, so we can leverage this for both code size + // and speed. + llvm_intrinsically_optimized! { + #[cfg(target_arch = "wasm32")] { + return unsafe { ::core::intrinsics::floorf64(x) } + } + } + #[cfg(all(target_arch = "x86", not(target_feature = "sse2")))] + { + //use an alternative implementation on x86, because the + //main implementation fails with the x87 FPU used by + //debian i386, probablly due to excess precision issues. + //basic implementation taken from https://github.com/rust-lang/libm/issues/219 + use super::fabs; + if fabs(x).to_bits() < 4503599627370496.0_f64.to_bits() { + let truncated = x as i64 as f64; + if truncated > x { + return truncated - 1.0; + } else { + return truncated; + } + } else { + return x; + } + } + let ui = x.to_bits(); + let e = ((ui >> 52) & 0x7ff) as i32; + + if (e >= 0x3ff + 52) || (x == 0.) { + return x; + } + /* y = int(x) - x, where int(x) is an integer neighbor of x */ + let y = if (ui >> 63) != 0 { + x - TOINT + TOINT - x + } else { + x + TOINT - TOINT - x + }; + /* special case because of non-nearest rounding modes */ + if e < 0x3ff { + force_eval!(y); + return if (ui >> 63) != 0 { -1. } else { 0. }; + } + if y > 0. { + x + y - 1. + } else { + x + y + } +} + +#[cfg(test)] +mod tests { + use super::*; + use core::f64::*; + + #[test] + fn sanity_check() { + assert_eq!(floor(1.1), 1.0); + assert_eq!(floor(2.9), 2.0); + } + + /// The spec: https://en.cppreference.com/w/cpp/numeric/math/floor + #[test] + fn spec_tests() { + // Not Asserted: that the current rounding mode has no effect. + assert!(floor(NAN).is_nan()); + for f in [0.0, -0.0, INFINITY, NEG_INFINITY].iter().copied() { + assert_eq!(floor(f), f); + } + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/floorf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/floorf.rs new file mode 100644 index 000000000..dfdab91a0 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/floorf.rs @@ -0,0 +1,66 @@ +use core::f32; + +/// Floor (f32) +/// +/// Finds the nearest integer less than or equal to `x`. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn floorf(x: f32) -> f32 { + // On wasm32 we know that LLVM's intrinsic will compile to an optimized + // `f32.floor` native instruction, so we can leverage this for both code size + // and speed. + llvm_intrinsically_optimized! { + #[cfg(target_arch = "wasm32")] { + return unsafe { ::core::intrinsics::floorf32(x) } + } + } + let mut ui = x.to_bits(); + let e = (((ui >> 23) as i32) & 0xff) - 0x7f; + + if e >= 23 { + return x; + } + if e >= 0 { + let m: u32 = 0x007fffff >> e; + if (ui & m) == 0 { + return x; + } + force_eval!(x + f32::from_bits(0x7b800000)); + if ui >> 31 != 0 { + ui += m; + } + ui &= !m; + } else { + force_eval!(x + f32::from_bits(0x7b800000)); + if ui >> 31 == 0 { + ui = 0; + } else if ui << 1 != 0 { + return -1.0; + } + } + f32::from_bits(ui) +} + +// PowerPC tests are failing on LLVM 13: https://github.com/rust-lang/rust/issues/88520 +#[cfg(not(target_arch = "powerpc64"))] +#[cfg(test)] +mod tests { + use super::*; + use core::f32::*; + + #[test] + fn sanity_check() { + assert_eq!(floorf(0.5), 0.0); + assert_eq!(floorf(1.1), 1.0); + assert_eq!(floorf(2.9), 2.0); + } + + /// The spec: https://en.cppreference.com/w/cpp/numeric/math/floor + #[test] + fn spec_tests() { + // Not Asserted: that the current rounding mode has no effect. + assert!(floorf(NAN).is_nan()); + for f in [0.0, -0.0, INFINITY, NEG_INFINITY].iter().copied() { + assert_eq!(floorf(f), f); + } + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/fma.rs b/crux-mir/lib/compiler_builtins/libm/src/math/fma.rs new file mode 100644 index 000000000..f9a86dc60 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/fma.rs @@ -0,0 +1,243 @@ +use core::{f32, f64}; + +use super::scalbn; + +const ZEROINFNAN: i32 = 0x7ff - 0x3ff - 52 - 1; + +struct Num { + m: u64, + e: i32, + sign: i32, +} + +fn normalize(x: f64) -> Num { + let x1p63: f64 = f64::from_bits(0x43e0000000000000); // 0x1p63 === 2 ^ 63 + + let mut ix: u64 = x.to_bits(); + let mut e: i32 = (ix >> 52) as i32; + let sign: i32 = e & 0x800; + e &= 0x7ff; + if e == 0 { + ix = (x * x1p63).to_bits(); + e = (ix >> 52) as i32 & 0x7ff; + e = if e != 0 { e - 63 } else { 0x800 }; + } + ix &= (1 << 52) - 1; + ix |= 1 << 52; + ix <<= 1; + e -= 0x3ff + 52 + 1; + Num { m: ix, e, sign } +} + +fn mul(x: u64, y: u64) -> (u64, u64) { + let t1: u64; + let t2: u64; + let t3: u64; + let xlo: u64 = x as u32 as u64; + let xhi: u64 = x >> 32; + let ylo: u64 = y as u32 as u64; + let yhi: u64 = y >> 32; + + t1 = xlo * ylo; + t2 = xlo * yhi + xhi * ylo; + t3 = xhi * yhi; + let lo = t1.wrapping_add(t2 << 32); + let hi = t3 + (t2 >> 32) + (t1 > lo) as u64; + (hi, lo) +} + +/// Floating multiply add (f64) +/// +/// Computes `(x*y)+z`, rounded as one ternary operation: +/// Computes the value (as if) to infinite precision and rounds once to the result format, +/// according to the rounding mode characterized by the value of FLT_ROUNDS. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn fma(x: f64, y: f64, z: f64) -> f64 { + let x1p63: f64 = f64::from_bits(0x43e0000000000000); // 0x1p63 === 2 ^ 63 + let x0_ffffff8p_63 = f64::from_bits(0x3bfffffff0000000); // 0x0.ffffff8p-63 + + /* normalize so top 10bits and last bit are 0 */ + let nx = normalize(x); + let ny = normalize(y); + let nz = normalize(z); + + if nx.e >= ZEROINFNAN || ny.e >= ZEROINFNAN { + return x * y + z; + } + if nz.e >= ZEROINFNAN { + if nz.e > ZEROINFNAN { + /* z==0 */ + return x * y + z; + } + return z; + } + + /* mul: r = x*y */ + let zhi: u64; + let zlo: u64; + let (mut rhi, mut rlo) = mul(nx.m, ny.m); + /* either top 20 or 21 bits of rhi and last 2 bits of rlo are 0 */ + + /* align exponents */ + let mut e: i32 = nx.e + ny.e; + let mut d: i32 = nz.e - e; + /* shift bits z<<=kz, r>>=kr, so kz+kr == d, set e = e+kr (== ez-kz) */ + if d > 0 { + if d < 64 { + zlo = nz.m << d; + zhi = nz.m >> (64 - d); + } else { + zlo = 0; + zhi = nz.m; + e = nz.e - 64; + d -= 64; + if d == 0 { + } else if d < 64 { + rlo = rhi << (64 - d) | rlo >> d | ((rlo << (64 - d)) != 0) as u64; + rhi = rhi >> d; + } else { + rlo = 1; + rhi = 0; + } + } + } else { + zhi = 0; + d = -d; + if d == 0 { + zlo = nz.m; + } else if d < 64 { + zlo = nz.m >> d | ((nz.m << (64 - d)) != 0) as u64; + } else { + zlo = 1; + } + } + + /* add */ + let mut sign: i32 = nx.sign ^ ny.sign; + let samesign: bool = (sign ^ nz.sign) == 0; + let mut nonzero: i32 = 1; + if samesign { + /* r += z */ + rlo = rlo.wrapping_add(zlo); + rhi += zhi + (rlo < zlo) as u64; + } else { + /* r -= z */ + let (res, borrow) = rlo.overflowing_sub(zlo); + rlo = res; + rhi = rhi.wrapping_sub(zhi.wrapping_add(borrow as u64)); + if (rhi >> 63) != 0 { + rlo = (rlo as i64).wrapping_neg() as u64; + rhi = (rhi as i64).wrapping_neg() as u64 - (rlo != 0) as u64; + sign = (sign == 0) as i32; + } + nonzero = (rhi != 0) as i32; + } + + /* set rhi to top 63bit of the result (last bit is sticky) */ + if nonzero != 0 { + e += 64; + d = rhi.leading_zeros() as i32 - 1; + /* note: d > 0 */ + rhi = rhi << d | rlo >> (64 - d) | ((rlo << d) != 0) as u64; + } else if rlo != 0 { + d = rlo.leading_zeros() as i32 - 1; + if d < 0 { + rhi = rlo >> 1 | (rlo & 1); + } else { + rhi = rlo << d; + } + } else { + /* exact +-0 */ + return x * y + z; + } + e -= d; + + /* convert to double */ + let mut i: i64 = rhi as i64; /* i is in [1<<62,(1<<63)-1] */ + if sign != 0 { + i = -i; + } + let mut r: f64 = i as f64; /* |r| is in [0x1p62,0x1p63] */ + + if e < -1022 - 62 { + /* result is subnormal before rounding */ + if e == -1022 - 63 { + let mut c: f64 = x1p63; + if sign != 0 { + c = -c; + } + if r == c { + /* min normal after rounding, underflow depends + on arch behaviour which can be imitated by + a double to float conversion */ + let fltmin: f32 = (x0_ffffff8p_63 * f32::MIN_POSITIVE as f64 * r) as f32; + return f64::MIN_POSITIVE / f32::MIN_POSITIVE as f64 * fltmin as f64; + } + /* one bit is lost when scaled, add another top bit to + only round once at conversion if it is inexact */ + if (rhi << 53) != 0 { + i = (rhi >> 1 | (rhi & 1) | 1 << 62) as i64; + if sign != 0 { + i = -i; + } + r = i as f64; + r = 2. * r - c; /* remove top bit */ + + /* raise underflow portably, such that it + cannot be optimized away */ + { + let tiny: f64 = f64::MIN_POSITIVE / f32::MIN_POSITIVE as f64 * r; + r += (tiny * tiny) * (r - r); + } + } + } else { + /* only round once when scaled */ + d = 10; + i = ((rhi >> d | ((rhi << (64 - d)) != 0) as u64) << d) as i64; + if sign != 0 { + i = -i; + } + r = i as f64; + } + } + scalbn(r, e) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn fma_segfault() { + // These two inputs cause fma to segfault on release due to overflow: + assert_eq!( + fma( + -0.0000000000000002220446049250313, + -0.0000000000000002220446049250313, + -0.0000000000000002220446049250313 + ), + -0.00000000000000022204460492503126, + ); + + let result = fma(-0.992, -0.992, -0.992); + //force rounding to storage format on x87 to prevent superious errors. + #[cfg(all(target_arch = "x86", not(target_feature = "sse2")))] + let result = force_eval!(result); + assert_eq!(result, -0.007936000000000007,); + } + + #[test] + fn fma_sbb() { + assert_eq!( + fma(-(1.0 - f64::EPSILON), f64::MIN, f64::MIN), + -3991680619069439e277 + ); + } + + #[test] + fn fma_underflow() { + assert_eq!( + fma(1.1102230246251565e-16, -9.812526705433188e-305, 1.0894e-320), + 0.0, + ); + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/fmaf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/fmaf.rs new file mode 100644 index 000000000..2848f2aee --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/fmaf.rs @@ -0,0 +1,117 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/s_fmaf.c */ +/*- + * Copyright (c) 2005-2011 David Schultz + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +use core::f32; +use core::ptr::read_volatile; + +use super::fenv::{ + feclearexcept, fegetround, feraiseexcept, fetestexcept, FE_INEXACT, FE_TONEAREST, FE_UNDERFLOW, +}; + +/* + * Fused multiply-add: Compute x * y + z with a single rounding error. + * + * A double has more than twice as much precision than a float, so + * direct double-precision arithmetic suffices, except where double + * rounding occurs. + */ + +/// Floating multiply add (f32) +/// +/// Computes `(x*y)+z`, rounded as one ternary operation: +/// Computes the value (as if) to infinite precision and rounds once to the result format, +/// according to the rounding mode characterized by the value of FLT_ROUNDS. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn fmaf(x: f32, y: f32, mut z: f32) -> f32 { + let xy: f64; + let mut result: f64; + let mut ui: u64; + let e: i32; + + xy = x as f64 * y as f64; + result = xy + z as f64; + ui = result.to_bits(); + e = (ui >> 52) as i32 & 0x7ff; + /* Common case: The double precision result is fine. */ + if ( + /* not a halfway case */ + ui & 0x1fffffff) != 0x10000000 || + /* NaN */ + e == 0x7ff || + /* exact */ + (result - xy == z as f64 && result - z as f64 == xy) || + /* not round-to-nearest */ + fegetround() != FE_TONEAREST + { + /* + underflow may not be raised correctly, example: + fmaf(0x1p-120f, 0x1p-120f, 0x1p-149f) + */ + if e < 0x3ff - 126 && e >= 0x3ff - 149 && fetestexcept(FE_INEXACT) != 0 { + feclearexcept(FE_INEXACT); + // prevent `xy + vz` from being CSE'd with `xy + z` above + let vz: f32 = unsafe { read_volatile(&z) }; + result = xy + vz as f64; + if fetestexcept(FE_INEXACT) != 0 { + feraiseexcept(FE_UNDERFLOW); + } else { + feraiseexcept(FE_INEXACT); + } + } + z = result as f32; + return z; + } + + /* + * If result is inexact, and exactly halfway between two float values, + * we need to adjust the low-order bit in the direction of the error. + */ + let neg = ui >> 63 != 0; + let err = if neg == (z as f64 > xy) { + xy - result + z as f64 + } else { + z as f64 - result + xy + }; + if neg == (err < 0.0) { + ui += 1; + } else { + ui -= 1; + } + f64::from_bits(ui) as f32 +} + +#[cfg(test)] +mod tests { + #[test] + fn issue_263() { + let a = f32::from_bits(1266679807); + let b = f32::from_bits(1300234242); + let c = f32::from_bits(1115553792); + let expected = f32::from_bits(1501560833); + assert_eq!(super::fmaf(a, b, c), expected); + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/fmax.rs b/crux-mir/lib/compiler_builtins/libm/src/math/fmax.rs new file mode 100644 index 000000000..93c97bc61 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/fmax.rs @@ -0,0 +1,12 @@ +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn fmax(x: f64, y: f64) -> f64 { + // IEEE754 says: maxNum(x, y) is the canonicalized number y if x < y, x if y < x, the + // canonicalized number if one operand is a number and the other a quiet NaN. Otherwise it + // is either x or y, canonicalized (this means results might differ among implementations). + // When either x or y is a signalingNaN, then the result is according to 6.2. + // + // Since we do not support sNaN in Rust yet, we do not need to handle them. + // FIXME(nagisa): due to https://bugs.llvm.org/show_bug.cgi?id=33303 we canonicalize by + // multiplying by 1.0. Should switch to the `canonicalize` when it works. + (if x.is_nan() || x < y { y } else { x }) * 1.0 +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/fmaxf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/fmaxf.rs new file mode 100644 index 000000000..607746647 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/fmaxf.rs @@ -0,0 +1,12 @@ +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn fmaxf(x: f32, y: f32) -> f32 { + // IEEE754 says: maxNum(x, y) is the canonicalized number y if x < y, x if y < x, the + // canonicalized number if one operand is a number and the other a quiet NaN. Otherwise it + // is either x or y, canonicalized (this means results might differ among implementations). + // When either x or y is a signalingNaN, then the result is according to 6.2. + // + // Since we do not support sNaN in Rust yet, we do not need to handle them. + // FIXME(nagisa): due to https://bugs.llvm.org/show_bug.cgi?id=33303 we canonicalize by + // multiplying by 1.0. Should switch to the `canonicalize` when it works. + (if x.is_nan() || x < y { y } else { x }) * 1.0 +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/fmin.rs b/crux-mir/lib/compiler_builtins/libm/src/math/fmin.rs new file mode 100644 index 000000000..ab1509f34 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/fmin.rs @@ -0,0 +1,12 @@ +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn fmin(x: f64, y: f64) -> f64 { + // IEEE754 says: minNum(x, y) is the canonicalized number x if x < y, y if y < x, the + // canonicalized number if one operand is a number and the other a quiet NaN. Otherwise it + // is either x or y, canonicalized (this means results might differ among implementations). + // When either x or y is a signalingNaN, then the result is according to 6.2. + // + // Since we do not support sNaN in Rust yet, we do not need to handle them. + // FIXME(nagisa): due to https://bugs.llvm.org/show_bug.cgi?id=33303 we canonicalize by + // multiplying by 1.0. Should switch to the `canonicalize` when it works. + (if y.is_nan() || x < y { x } else { y }) * 1.0 +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/fminf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/fminf.rs new file mode 100644 index 000000000..0049e7117 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/fminf.rs @@ -0,0 +1,12 @@ +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn fminf(x: f32, y: f32) -> f32 { + // IEEE754 says: minNum(x, y) is the canonicalized number x if x < y, y if y < x, the + // canonicalized number if one operand is a number and the other a quiet NaN. Otherwise it + // is either x or y, canonicalized (this means results might differ among implementations). + // When either x or y is a signalingNaN, then the result is according to 6.2. + // + // Since we do not support sNaN in Rust yet, we do not need to handle them. + // FIXME(nagisa): due to https://bugs.llvm.org/show_bug.cgi?id=33303 we canonicalize by + // multiplying by 1.0. Should switch to the `canonicalize` when it works. + (if y.is_nan() || x < y { x } else { y }) * 1.0 +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/fmod.rs b/crux-mir/lib/compiler_builtins/libm/src/math/fmod.rs new file mode 100644 index 000000000..d892ffd8b --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/fmod.rs @@ -0,0 +1,80 @@ +use core::u64; + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn fmod(x: f64, y: f64) -> f64 { + let mut uxi = x.to_bits(); + let mut uyi = y.to_bits(); + let mut ex = (uxi >> 52 & 0x7ff) as i64; + let mut ey = (uyi >> 52 & 0x7ff) as i64; + let sx = uxi >> 63; + let mut i; + + if uyi << 1 == 0 || y.is_nan() || ex == 0x7ff { + return (x * y) / (x * y); + } + if uxi << 1 <= uyi << 1 { + if uxi << 1 == uyi << 1 { + return 0.0 * x; + } + return x; + } + + /* normalize x and y */ + if ex == 0 { + i = uxi << 12; + while i >> 63 == 0 { + ex -= 1; + i <<= 1; + } + uxi <<= -ex + 1; + } else { + uxi &= u64::MAX >> 12; + uxi |= 1 << 52; + } + if ey == 0 { + i = uyi << 12; + while i >> 63 == 0 { + ey -= 1; + i <<= 1; + } + uyi <<= -ey + 1; + } else { + uyi &= u64::MAX >> 12; + uyi |= 1 << 52; + } + + /* x mod y */ + while ex > ey { + i = uxi.wrapping_sub(uyi); + if i >> 63 == 0 { + if i == 0 { + return 0.0 * x; + } + uxi = i; + } + uxi <<= 1; + ex -= 1; + } + i = uxi.wrapping_sub(uyi); + if i >> 63 == 0 { + if i == 0 { + return 0.0 * x; + } + uxi = i; + } + while uxi >> 52 == 0 { + uxi <<= 1; + ex -= 1; + } + + /* scale result */ + if ex > 0 { + uxi -= 1 << 52; + uxi |= (ex as u64) << 52; + } else { + uxi >>= -ex + 1; + } + uxi |= (sx as u64) << 63; + + f64::from_bits(uxi) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/fmodf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/fmodf.rs new file mode 100644 index 000000000..c53dc186a --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/fmodf.rs @@ -0,0 +1,89 @@ +use core::f32; +use core::u32; + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn fmodf(x: f32, y: f32) -> f32 { + let mut uxi = x.to_bits(); + let mut uyi = y.to_bits(); + let mut ex = (uxi >> 23 & 0xff) as i32; + let mut ey = (uyi >> 23 & 0xff) as i32; + let sx = uxi & 0x80000000; + let mut i; + + if uyi << 1 == 0 || y.is_nan() || ex == 0xff { + return (x * y) / (x * y); + } + + if uxi << 1 <= uyi << 1 { + if uxi << 1 == uyi << 1 { + return 0.0 * x; + } + + return x; + } + + /* normalize x and y */ + if ex == 0 { + i = uxi << 9; + while i >> 31 == 0 { + ex -= 1; + i <<= 1; + } + + uxi <<= -ex + 1; + } else { + uxi &= u32::MAX >> 9; + uxi |= 1 << 23; + } + + if ey == 0 { + i = uyi << 9; + while i >> 31 == 0 { + ey -= 1; + i <<= 1; + } + + uyi <<= -ey + 1; + } else { + uyi &= u32::MAX >> 9; + uyi |= 1 << 23; + } + + /* x mod y */ + while ex > ey { + i = uxi.wrapping_sub(uyi); + if i >> 31 == 0 { + if i == 0 { + return 0.0 * x; + } + uxi = i; + } + uxi <<= 1; + + ex -= 1; + } + + i = uxi.wrapping_sub(uyi); + if i >> 31 == 0 { + if i == 0 { + return 0.0 * x; + } + uxi = i; + } + + while uxi >> 23 == 0 { + uxi <<= 1; + ex -= 1; + } + + /* scale result up */ + if ex > 0 { + uxi -= 1 << 23; + uxi |= (ex as u32) << 23; + } else { + uxi >>= -ex + 1; + } + uxi |= sx; + + f32::from_bits(uxi) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/frexp.rs b/crux-mir/lib/compiler_builtins/libm/src/math/frexp.rs new file mode 100644 index 000000000..badad786a --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/frexp.rs @@ -0,0 +1,20 @@ +pub fn frexp(x: f64) -> (f64, i32) { + let mut y = x.to_bits(); + let ee = ((y >> 52) & 0x7ff) as i32; + + if ee == 0 { + if x != 0.0 { + let x1p64 = f64::from_bits(0x43f0000000000000); + let (x, e) = frexp(x * x1p64); + return (x, e - 64); + } + return (x, 0); + } else if ee == 0x7ff { + return (x, 0); + } + + let e = ee - 0x3fe; + y &= 0x800fffffffffffff; + y |= 0x3fe0000000000000; + return (f64::from_bits(y), e); +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/frexpf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/frexpf.rs new file mode 100644 index 000000000..2919c0ab0 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/frexpf.rs @@ -0,0 +1,21 @@ +pub fn frexpf(x: f32) -> (f32, i32) { + let mut y = x.to_bits(); + let ee: i32 = ((y >> 23) & 0xff) as i32; + + if ee == 0 { + if x != 0.0 { + let x1p64 = f32::from_bits(0x5f800000); + let (x, e) = frexpf(x * x1p64); + return (x, e - 64); + } else { + return (x, 0); + } + } else if ee == 0xff { + return (x, 0); + } + + let e = ee - 0x7e; + y &= 0x807fffff; + y |= 0x3f000000; + (f32::from_bits(y), e) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/hypot.rs b/crux-mir/lib/compiler_builtins/libm/src/math/hypot.rs new file mode 100644 index 000000000..da458ea1d --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/hypot.rs @@ -0,0 +1,74 @@ +use core::f64; + +use super::sqrt; + +const SPLIT: f64 = 134217728. + 1.; // 0x1p27 + 1 === (2 ^ 27) + 1 + +fn sq(x: f64) -> (f64, f64) { + let xh: f64; + let xl: f64; + let xc: f64; + + xc = x * SPLIT; + xh = x - xc + xc; + xl = x - xh; + let hi = x * x; + let lo = xh * xh - hi + 2. * xh * xl + xl * xl; + (hi, lo) +} + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn hypot(mut x: f64, mut y: f64) -> f64 { + let x1p700 = f64::from_bits(0x6bb0000000000000); // 0x1p700 === 2 ^ 700 + let x1p_700 = f64::from_bits(0x1430000000000000); // 0x1p-700 === 2 ^ -700 + + let mut uxi = x.to_bits(); + let mut uyi = y.to_bits(); + let uti; + let ex: i64; + let ey: i64; + let mut z: f64; + + /* arrange |x| >= |y| */ + uxi &= -1i64 as u64 >> 1; + uyi &= -1i64 as u64 >> 1; + if uxi < uyi { + uti = uxi; + uxi = uyi; + uyi = uti; + } + + /* special cases */ + ex = (uxi >> 52) as i64; + ey = (uyi >> 52) as i64; + x = f64::from_bits(uxi); + y = f64::from_bits(uyi); + /* note: hypot(inf,nan) == inf */ + if ey == 0x7ff { + return y; + } + if ex == 0x7ff || uyi == 0 { + return x; + } + /* note: hypot(x,y) ~= x + y*y/x/2 with inexact for small y/x */ + /* 64 difference is enough for ld80 double_t */ + if ex - ey > 64 { + return x + y; + } + + /* precise sqrt argument in nearest rounding mode without overflow */ + /* xh*xh must not overflow and xl*xl must not underflow in sq */ + z = 1.; + if ex > 0x3ff + 510 { + z = x1p700; + x *= x1p_700; + y *= x1p_700; + } else if ey < 0x3ff - 450 { + z = x1p_700; + x *= x1p700; + y *= x1p700; + } + let (hx, lx) = sq(x); + let (hy, ly) = sq(y); + z * sqrt(ly + lx + hy + hx) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/hypotf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/hypotf.rs new file mode 100644 index 000000000..576eebb33 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/hypotf.rs @@ -0,0 +1,43 @@ +use core::f32; + +use super::sqrtf; + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn hypotf(mut x: f32, mut y: f32) -> f32 { + let x1p90 = f32::from_bits(0x6c800000); // 0x1p90f === 2 ^ 90 + let x1p_90 = f32::from_bits(0x12800000); // 0x1p-90f === 2 ^ -90 + + let mut uxi = x.to_bits(); + let mut uyi = y.to_bits(); + let uti; + let mut z: f32; + + uxi &= -1i32 as u32 >> 1; + uyi &= -1i32 as u32 >> 1; + if uxi < uyi { + uti = uxi; + uxi = uyi; + uyi = uti; + } + + x = f32::from_bits(uxi); + y = f32::from_bits(uyi); + if uyi == 0xff << 23 { + return y; + } + if uxi >= 0xff << 23 || uyi == 0 || uxi - uyi >= 25 << 23 { + return x + y; + } + + z = 1.; + if uxi >= (0x7f + 60) << 23 { + z = x1p90; + x *= x1p_90; + y *= x1p_90; + } else if uyi < (0x7f - 60) << 23 { + z = x1p_90; + x *= x1p90; + y *= x1p90; + } + z * sqrtf((x as f64 * x as f64 + y as f64 * y as f64) as f32) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/ilogb.rs b/crux-mir/lib/compiler_builtins/libm/src/math/ilogb.rs new file mode 100644 index 000000000..7d74dcfb6 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/ilogb.rs @@ -0,0 +1,32 @@ +const FP_ILOGBNAN: i32 = -1 - 0x7fffffff; +const FP_ILOGB0: i32 = FP_ILOGBNAN; + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn ilogb(x: f64) -> i32 { + let mut i: u64 = x.to_bits(); + let e = ((i >> 52) & 0x7ff) as i32; + + if e == 0 { + i <<= 12; + if i == 0 { + force_eval!(0.0 / 0.0); + return FP_ILOGB0; + } + /* subnormal x */ + let mut e = -0x3ff; + while (i >> 63) == 0 { + e -= 1; + i <<= 1; + } + e + } else if e == 0x7ff { + force_eval!(0.0 / 0.0); + if (i << 12) != 0 { + FP_ILOGBNAN + } else { + i32::max_value() + } + } else { + e - 0x3ff + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/ilogbf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/ilogbf.rs new file mode 100644 index 000000000..0fa58748c --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/ilogbf.rs @@ -0,0 +1,32 @@ +const FP_ILOGBNAN: i32 = -1 - 0x7fffffff; +const FP_ILOGB0: i32 = FP_ILOGBNAN; + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn ilogbf(x: f32) -> i32 { + let mut i = x.to_bits(); + let e = ((i >> 23) & 0xff) as i32; + + if e == 0 { + i <<= 9; + if i == 0 { + force_eval!(0.0 / 0.0); + return FP_ILOGB0; + } + /* subnormal x */ + let mut e = -0x7f; + while (i >> 31) == 0 { + e -= 1; + i <<= 1; + } + e + } else if e == 0xff { + force_eval!(0.0 / 0.0); + if (i << 9) != 0 { + FP_ILOGBNAN + } else { + i32::max_value() + } + } else { + e - 0x7f + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/j0.rs b/crux-mir/lib/compiler_builtins/libm/src/math/j0.rs new file mode 100644 index 000000000..c4258ccca --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/j0.rs @@ -0,0 +1,422 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_j0.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunSoft, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ +/* j0(x), y0(x) + * Bessel function of the first and second kinds of order zero. + * Method -- j0(x): + * 1. For tiny x, we use j0(x) = 1 - x^2/4 + x^4/64 - ... + * 2. Reduce x to |x| since j0(x)=j0(-x), and + * for x in (0,2) + * j0(x) = 1-z/4+ z^2*R0/S0, where z = x*x; + * (precision: |j0-1+z/4-z^2R0/S0 |<2**-63.67 ) + * for x in (2,inf) + * j0(x) = sqrt(2/(pi*x))*(p0(x)*cos(x0)-q0(x)*sin(x0)) + * where x0 = x-pi/4. It is better to compute sin(x0),cos(x0) + * as follow: + * cos(x0) = cos(x)cos(pi/4)+sin(x)sin(pi/4) + * = 1/sqrt(2) * (cos(x) + sin(x)) + * sin(x0) = sin(x)cos(pi/4)-cos(x)sin(pi/4) + * = 1/sqrt(2) * (sin(x) - cos(x)) + * (To avoid cancellation, use + * sin(x) +- cos(x) = -cos(2x)/(sin(x) -+ cos(x)) + * to compute the worse one.) + * + * 3 Special cases + * j0(nan)= nan + * j0(0) = 1 + * j0(inf) = 0 + * + * Method -- y0(x): + * 1. For x<2. + * Since + * y0(x) = 2/pi*(j0(x)*(ln(x/2)+Euler) + x^2/4 - ...) + * therefore y0(x)-2/pi*j0(x)*ln(x) is an even function. + * We use the following function to approximate y0, + * y0(x) = U(z)/V(z) + (2/pi)*(j0(x)*ln(x)), z= x^2 + * where + * U(z) = u00 + u01*z + ... + u06*z^6 + * V(z) = 1 + v01*z + ... + v04*z^4 + * with absolute approximation error bounded by 2**-72. + * Note: For tiny x, U/V = u0 and j0(x)~1, hence + * y0(tiny) = u0 + (2/pi)*ln(tiny), (choose tiny<2**-27) + * 2. For x>=2. + * y0(x) = sqrt(2/(pi*x))*(p0(x)*cos(x0)+q0(x)*sin(x0)) + * where x0 = x-pi/4. It is better to compute sin(x0),cos(x0) + * by the method mentioned above. + * 3. Special cases: y0(0)=-inf, y0(x<0)=NaN, y0(inf)=0. + */ + +use super::{cos, fabs, get_high_word, get_low_word, log, sin, sqrt}; +const INVSQRTPI: f64 = 5.64189583547756279280e-01; /* 0x3FE20DD7, 0x50429B6D */ +const TPI: f64 = 6.36619772367581382433e-01; /* 0x3FE45F30, 0x6DC9C883 */ + +/* common method when |x|>=2 */ +fn common(ix: u32, x: f64, y0: bool) -> f64 { + let s: f64; + let mut c: f64; + let mut ss: f64; + let mut cc: f64; + let z: f64; + + /* + * j0(x) = sqrt(2/(pi*x))*(p0(x)*cos(x-pi/4)-q0(x)*sin(x-pi/4)) + * y0(x) = sqrt(2/(pi*x))*(p0(x)*sin(x-pi/4)+q0(x)*cos(x-pi/4)) + * + * sin(x-pi/4) = (sin(x) - cos(x))/sqrt(2) + * cos(x-pi/4) = (sin(x) + cos(x))/sqrt(2) + * sin(x) +- cos(x) = -cos(2x)/(sin(x) -+ cos(x)) + */ + s = sin(x); + c = cos(x); + if y0 { + c = -c; + } + cc = s + c; + /* avoid overflow in 2*x, big ulp error when x>=0x1p1023 */ + if ix < 0x7fe00000 { + ss = s - c; + z = -cos(2.0 * x); + if s * c < 0.0 { + cc = z / ss; + } else { + ss = z / cc; + } + if ix < 0x48000000 { + if y0 { + ss = -ss; + } + cc = pzero(x) * cc - qzero(x) * ss; + } + } + return INVSQRTPI * cc / sqrt(x); +} + +/* R0/S0 on [0, 2.00] */ +const R02: f64 = 1.56249999999999947958e-02; /* 0x3F8FFFFF, 0xFFFFFFFD */ +const R03: f64 = -1.89979294238854721751e-04; /* 0xBF28E6A5, 0xB61AC6E9 */ +const R04: f64 = 1.82954049532700665670e-06; /* 0x3EBEB1D1, 0x0C503919 */ +const R05: f64 = -4.61832688532103189199e-09; /* 0xBE33D5E7, 0x73D63FCE */ +const S01: f64 = 1.56191029464890010492e-02; /* 0x3F8FFCE8, 0x82C8C2A4 */ +const S02: f64 = 1.16926784663337450260e-04; /* 0x3F1EA6D2, 0xDD57DBF4 */ +const S03: f64 = 5.13546550207318111446e-07; /* 0x3EA13B54, 0xCE84D5A9 */ +const S04: f64 = 1.16614003333790000205e-09; /* 0x3E1408BC, 0xF4745D8F */ + +pub fn j0(mut x: f64) -> f64 { + let z: f64; + let r: f64; + let s: f64; + let mut ix: u32; + + ix = get_high_word(x); + ix &= 0x7fffffff; + + /* j0(+-inf)=0, j0(nan)=nan */ + if ix >= 0x7ff00000 { + return 1.0 / (x * x); + } + x = fabs(x); + + if ix >= 0x40000000 { + /* |x| >= 2 */ + /* large ulp error near zeros: 2.4, 5.52, 8.6537,.. */ + return common(ix, x, false); + } + + /* 1 - x*x/4 + x*x*R(x^2)/S(x^2) */ + if ix >= 0x3f200000 { + /* |x| >= 2**-13 */ + /* up to 4ulp error close to 2 */ + z = x * x; + r = z * (R02 + z * (R03 + z * (R04 + z * R05))); + s = 1.0 + z * (S01 + z * (S02 + z * (S03 + z * S04))); + return (1.0 + x / 2.0) * (1.0 - x / 2.0) + z * (r / s); + } + + /* 1 - x*x/4 */ + /* prevent underflow */ + /* inexact should be raised when x!=0, this is not done correctly */ + if ix >= 0x38000000 { + /* |x| >= 2**-127 */ + x = 0.25 * x * x; + } + return 1.0 - x; +} + +const U00: f64 = -7.38042951086872317523e-02; /* 0xBFB2E4D6, 0x99CBD01F */ +const U01: f64 = 1.76666452509181115538e-01; /* 0x3FC69D01, 0x9DE9E3FC */ +const U02: f64 = -1.38185671945596898896e-02; /* 0xBF8C4CE8, 0xB16CFA97 */ +const U03: f64 = 3.47453432093683650238e-04; /* 0x3F36C54D, 0x20B29B6B */ +const U04: f64 = -3.81407053724364161125e-06; /* 0xBECFFEA7, 0x73D25CAD */ +const U05: f64 = 1.95590137035022920206e-08; /* 0x3E550057, 0x3B4EABD4 */ +const U06: f64 = -3.98205194132103398453e-11; /* 0xBDC5E43D, 0x693FB3C8 */ +const V01: f64 = 1.27304834834123699328e-02; /* 0x3F8A1270, 0x91C9C71A */ +const V02: f64 = 7.60068627350353253702e-05; /* 0x3F13ECBB, 0xF578C6C1 */ +const V03: f64 = 2.59150851840457805467e-07; /* 0x3E91642D, 0x7FF202FD */ +const V04: f64 = 4.41110311332675467403e-10; /* 0x3DFE5018, 0x3BD6D9EF */ + +pub fn y0(x: f64) -> f64 { + let z: f64; + let u: f64; + let v: f64; + let ix: u32; + let lx: u32; + + ix = get_high_word(x); + lx = get_low_word(x); + + /* y0(nan)=nan, y0(<0)=nan, y0(0)=-inf, y0(inf)=0 */ + if ((ix << 1) | lx) == 0 { + return -1.0 / 0.0; + } + if (ix >> 31) != 0 { + return 0.0 / 0.0; + } + if ix >= 0x7ff00000 { + return 1.0 / x; + } + + if ix >= 0x40000000 { + /* x >= 2 */ + /* large ulp errors near zeros: 3.958, 7.086,.. */ + return common(ix, x, true); + } + + /* U(x^2)/V(x^2) + (2/pi)*j0(x)*log(x) */ + if ix >= 0x3e400000 { + /* x >= 2**-27 */ + /* large ulp error near the first zero, x ~= 0.89 */ + z = x * x; + u = U00 + z * (U01 + z * (U02 + z * (U03 + z * (U04 + z * (U05 + z * U06))))); + v = 1.0 + z * (V01 + z * (V02 + z * (V03 + z * V04))); + return u / v + TPI * (j0(x) * log(x)); + } + return U00 + TPI * log(x); +} + +/* The asymptotic expansions of pzero is + * 1 - 9/128 s^2 + 11025/98304 s^4 - ..., where s = 1/x. + * For x >= 2, We approximate pzero by + * pzero(x) = 1 + (R/S) + * where R = pR0 + pR1*s^2 + pR2*s^4 + ... + pR5*s^10 + * S = 1 + pS0*s^2 + ... + pS4*s^10 + * and + * | pzero(x)-1-R/S | <= 2 ** ( -60.26) + */ +const PR8: [f64; 6] = [ + /* for x in [inf, 8]=1/[0,0.125] */ + 0.00000000000000000000e+00, /* 0x00000000, 0x00000000 */ + -7.03124999999900357484e-02, /* 0xBFB1FFFF, 0xFFFFFD32 */ + -8.08167041275349795626e+00, /* 0xC02029D0, 0xB44FA779 */ + -2.57063105679704847262e+02, /* 0xC0701102, 0x7B19E863 */ + -2.48521641009428822144e+03, /* 0xC0A36A6E, 0xCD4DCAFC */ + -5.25304380490729545272e+03, /* 0xC0B4850B, 0x36CC643D */ +]; +const PS8: [f64; 5] = [ + 1.16534364619668181717e+02, /* 0x405D2233, 0x07A96751 */ + 3.83374475364121826715e+03, /* 0x40ADF37D, 0x50596938 */ + 4.05978572648472545552e+04, /* 0x40E3D2BB, 0x6EB6B05F */ + 1.16752972564375915681e+05, /* 0x40FC810F, 0x8F9FA9BD */ + 4.76277284146730962675e+04, /* 0x40E74177, 0x4F2C49DC */ +]; + +const PR5: [f64; 6] = [ + /* for x in [8,4.5454]=1/[0.125,0.22001] */ + -1.14125464691894502584e-11, /* 0xBDA918B1, 0x47E495CC */ + -7.03124940873599280078e-02, /* 0xBFB1FFFF, 0xE69AFBC6 */ + -4.15961064470587782438e+00, /* 0xC010A370, 0xF90C6BBF */ + -6.76747652265167261021e+01, /* 0xC050EB2F, 0x5A7D1783 */ + -3.31231299649172967747e+02, /* 0xC074B3B3, 0x6742CC63 */ + -3.46433388365604912451e+02, /* 0xC075A6EF, 0x28A38BD7 */ +]; +const PS5: [f64; 5] = [ + 6.07539382692300335975e+01, /* 0x404E6081, 0x0C98C5DE */ + 1.05125230595704579173e+03, /* 0x40906D02, 0x5C7E2864 */ + 5.97897094333855784498e+03, /* 0x40B75AF8, 0x8FBE1D60 */ + 9.62544514357774460223e+03, /* 0x40C2CCB8, 0xFA76FA38 */ + 2.40605815922939109441e+03, /* 0x40A2CC1D, 0xC70BE864 */ +]; + +const PR3: [f64; 6] = [ + /* for x in [4.547,2.8571]=1/[0.2199,0.35001] */ + -2.54704601771951915620e-09, /* 0xBE25E103, 0x6FE1AA86 */ + -7.03119616381481654654e-02, /* 0xBFB1FFF6, 0xF7C0E24B */ + -2.40903221549529611423e+00, /* 0xC00345B2, 0xAEA48074 */ + -2.19659774734883086467e+01, /* 0xC035F74A, 0x4CB94E14 */ + -5.80791704701737572236e+01, /* 0xC04D0A22, 0x420A1A45 */ + -3.14479470594888503854e+01, /* 0xC03F72AC, 0xA892D80F */ +]; +const PS3: [f64; 5] = [ + 3.58560338055209726349e+01, /* 0x4041ED92, 0x84077DD3 */ + 3.61513983050303863820e+02, /* 0x40769839, 0x464A7C0E */ + 1.19360783792111533330e+03, /* 0x4092A66E, 0x6D1061D6 */ + 1.12799679856907414432e+03, /* 0x40919FFC, 0xB8C39B7E */ + 1.73580930813335754692e+02, /* 0x4065B296, 0xFC379081 */ +]; + +const PR2: [f64; 6] = [ + /* for x in [2.8570,2]=1/[0.3499,0.5] */ + -8.87534333032526411254e-08, /* 0xBE77D316, 0xE927026D */ + -7.03030995483624743247e-02, /* 0xBFB1FF62, 0x495E1E42 */ + -1.45073846780952986357e+00, /* 0xBFF73639, 0x8A24A843 */ + -7.63569613823527770791e+00, /* 0xC01E8AF3, 0xEDAFA7F3 */ + -1.11931668860356747786e+01, /* 0xC02662E6, 0xC5246303 */ + -3.23364579351335335033e+00, /* 0xC009DE81, 0xAF8FE70F */ +]; +const PS2: [f64; 5] = [ + 2.22202997532088808441e+01, /* 0x40363865, 0x908B5959 */ + 1.36206794218215208048e+02, /* 0x4061069E, 0x0EE8878F */ + 2.70470278658083486789e+02, /* 0x4070E786, 0x42EA079B */ + 1.53875394208320329881e+02, /* 0x40633C03, 0x3AB6FAFF */ + 1.46576176948256193810e+01, /* 0x402D50B3, 0x44391809 */ +]; + +fn pzero(x: f64) -> f64 { + let p: &[f64; 6]; + let q: &[f64; 5]; + let z: f64; + let r: f64; + let s: f64; + let mut ix: u32; + + ix = get_high_word(x); + ix &= 0x7fffffff; + if ix >= 0x40200000 { + p = &PR8; + q = &PS8; + } else if ix >= 0x40122E8B { + p = &PR5; + q = &PS5; + } else if ix >= 0x4006DB6D { + p = &PR3; + q = &PS3; + } else + /*ix >= 0x40000000*/ + { + p = &PR2; + q = &PS2; + } + z = 1.0 / (x * x); + r = p[0] + z * (p[1] + z * (p[2] + z * (p[3] + z * (p[4] + z * p[5])))); + s = 1.0 + z * (q[0] + z * (q[1] + z * (q[2] + z * (q[3] + z * q[4])))); + return 1.0 + r / s; +} + +/* For x >= 8, the asymptotic expansions of qzero is + * -1/8 s + 75/1024 s^3 - ..., where s = 1/x. + * We approximate pzero by + * qzero(x) = s*(-1.25 + (R/S)) + * where R = qR0 + qR1*s^2 + qR2*s^4 + ... + qR5*s^10 + * S = 1 + qS0*s^2 + ... + qS5*s^12 + * and + * | qzero(x)/s +1.25-R/S | <= 2 ** ( -61.22) + */ +const QR8: [f64; 6] = [ + /* for x in [inf, 8]=1/[0,0.125] */ + 0.00000000000000000000e+00, /* 0x00000000, 0x00000000 */ + 7.32421874999935051953e-02, /* 0x3FB2BFFF, 0xFFFFFE2C */ + 1.17682064682252693899e+01, /* 0x40278952, 0x5BB334D6 */ + 5.57673380256401856059e+02, /* 0x40816D63, 0x15301825 */ + 8.85919720756468632317e+03, /* 0x40C14D99, 0x3E18F46D */ + 3.70146267776887834771e+04, /* 0x40E212D4, 0x0E901566 */ +]; +const QS8: [f64; 6] = [ + 1.63776026895689824414e+02, /* 0x406478D5, 0x365B39BC */ + 8.09834494656449805916e+03, /* 0x40BFA258, 0x4E6B0563 */ + 1.42538291419120476348e+05, /* 0x41016652, 0x54D38C3F */ + 8.03309257119514397345e+05, /* 0x412883DA, 0x83A52B43 */ + 8.40501579819060512818e+05, /* 0x4129A66B, 0x28DE0B3D */ + -3.43899293537866615225e+05, /* 0xC114FD6D, 0x2C9530C5 */ +]; + +const QR5: [f64; 6] = [ + /* for x in [8,4.5454]=1/[0.125,0.22001] */ + 1.84085963594515531381e-11, /* 0x3DB43D8F, 0x29CC8CD9 */ + 7.32421766612684765896e-02, /* 0x3FB2BFFF, 0xD172B04C */ + 5.83563508962056953777e+00, /* 0x401757B0, 0xB9953DD3 */ + 1.35111577286449829671e+02, /* 0x4060E392, 0x0A8788E9 */ + 1.02724376596164097464e+03, /* 0x40900CF9, 0x9DC8C481 */ + 1.98997785864605384631e+03, /* 0x409F17E9, 0x53C6E3A6 */ +]; +const QS5: [f64; 6] = [ + 8.27766102236537761883e+01, /* 0x4054B1B3, 0xFB5E1543 */ + 2.07781416421392987104e+03, /* 0x40A03BA0, 0xDA21C0CE */ + 1.88472887785718085070e+04, /* 0x40D267D2, 0x7B591E6D */ + 5.67511122894947329769e+04, /* 0x40EBB5E3, 0x97E02372 */ + 3.59767538425114471465e+04, /* 0x40E19118, 0x1F7A54A0 */ + -5.35434275601944773371e+03, /* 0xC0B4EA57, 0xBEDBC609 */ +]; + +const QR3: [f64; 6] = [ + /* for x in [4.547,2.8571]=1/[0.2199,0.35001] */ + 4.37741014089738620906e-09, /* 0x3E32CD03, 0x6ADECB82 */ + 7.32411180042911447163e-02, /* 0x3FB2BFEE, 0x0E8D0842 */ + 3.34423137516170720929e+00, /* 0x400AC0FC, 0x61149CF5 */ + 4.26218440745412650017e+01, /* 0x40454F98, 0x962DAEDD */ + 1.70808091340565596283e+02, /* 0x406559DB, 0xE25EFD1F */ + 1.66733948696651168575e+02, /* 0x4064D77C, 0x81FA21E0 */ +]; +const QS3: [f64; 6] = [ + 4.87588729724587182091e+01, /* 0x40486122, 0xBFE343A6 */ + 7.09689221056606015736e+02, /* 0x40862D83, 0x86544EB3 */ + 3.70414822620111362994e+03, /* 0x40ACF04B, 0xE44DFC63 */ + 6.46042516752568917582e+03, /* 0x40B93C6C, 0xD7C76A28 */ + 2.51633368920368957333e+03, /* 0x40A3A8AA, 0xD94FB1C0 */ + -1.49247451836156386662e+02, /* 0xC062A7EB, 0x201CF40F */ +]; + +const QR2: [f64; 6] = [ + /* for x in [2.8570,2]=1/[0.3499,0.5] */ + 1.50444444886983272379e-07, /* 0x3E84313B, 0x54F76BDB */ + 7.32234265963079278272e-02, /* 0x3FB2BEC5, 0x3E883E34 */ + 1.99819174093815998816e+00, /* 0x3FFFF897, 0xE727779C */ + 1.44956029347885735348e+01, /* 0x402CFDBF, 0xAAF96FE5 */ + 3.16662317504781540833e+01, /* 0x403FAA8E, 0x29FBDC4A */ + 1.62527075710929267416e+01, /* 0x403040B1, 0x71814BB4 */ +]; +const QS2: [f64; 6] = [ + 3.03655848355219184498e+01, /* 0x403E5D96, 0xF7C07AED */ + 2.69348118608049844624e+02, /* 0x4070D591, 0xE4D14B40 */ + 8.44783757595320139444e+02, /* 0x408A6645, 0x22B3BF22 */ + 8.82935845112488550512e+02, /* 0x408B977C, 0x9C5CC214 */ + 2.12666388511798828631e+02, /* 0x406A9553, 0x0E001365 */ + -5.31095493882666946917e+00, /* 0xC0153E6A, 0xF8B32931 */ +]; + +fn qzero(x: f64) -> f64 { + let p: &[f64; 6]; + let q: &[f64; 6]; + let s: f64; + let r: f64; + let z: f64; + let mut ix: u32; + + ix = get_high_word(x); + ix &= 0x7fffffff; + if ix >= 0x40200000 { + p = &QR8; + q = &QS8; + } else if ix >= 0x40122E8B { + p = &QR5; + q = &QS5; + } else if ix >= 0x4006DB6D { + p = &QR3; + q = &QS3; + } else + /*ix >= 0x40000000*/ + { + p = &QR2; + q = &QS2; + } + z = 1.0 / (x * x); + r = p[0] + z * (p[1] + z * (p[2] + z * (p[3] + z * (p[4] + z * p[5])))); + s = 1.0 + z * (q[0] + z * (q[1] + z * (q[2] + z * (q[3] + z * (q[4] + z * q[5]))))); + return (-0.125 + r / s) / x; +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/j0f.rs b/crux-mir/lib/compiler_builtins/libm/src/math/j0f.rs new file mode 100644 index 000000000..91c03dbbc --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/j0f.rs @@ -0,0 +1,359 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_j0f.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +use super::{cosf, fabsf, logf, sinf, sqrtf}; + +const INVSQRTPI: f32 = 5.6418961287e-01; /* 0x3f106ebb */ +const TPI: f32 = 6.3661974669e-01; /* 0x3f22f983 */ + +fn common(ix: u32, x: f32, y0: bool) -> f32 { + let z: f32; + let s: f32; + let mut c: f32; + let mut ss: f32; + let mut cc: f32; + /* + * j0(x) = 1/sqrt(pi) * (P(0,x)*cc - Q(0,x)*ss) / sqrt(x) + * y0(x) = 1/sqrt(pi) * (P(0,x)*ss + Q(0,x)*cc) / sqrt(x) + */ + s = sinf(x); + c = cosf(x); + if y0 { + c = -c; + } + cc = s + c; + if ix < 0x7f000000 { + ss = s - c; + z = -cosf(2.0 * x); + if s * c < 0.0 { + cc = z / ss; + } else { + ss = z / cc; + } + if ix < 0x58800000 { + if y0 { + ss = -ss; + } + cc = pzerof(x) * cc - qzerof(x) * ss; + } + } + return INVSQRTPI * cc / sqrtf(x); +} + +/* R0/S0 on [0, 2.00] */ +const R02: f32 = 1.5625000000e-02; /* 0x3c800000 */ +const R03: f32 = -1.8997929874e-04; /* 0xb947352e */ +const R04: f32 = 1.8295404516e-06; /* 0x35f58e88 */ +const R05: f32 = -4.6183270541e-09; /* 0xb19eaf3c */ +const S01: f32 = 1.5619102865e-02; /* 0x3c7fe744 */ +const S02: f32 = 1.1692678527e-04; /* 0x38f53697 */ +const S03: f32 = 5.1354652442e-07; /* 0x3509daa6 */ +const S04: f32 = 1.1661400734e-09; /* 0x30a045e8 */ + +pub fn j0f(mut x: f32) -> f32 { + let z: f32; + let r: f32; + let s: f32; + let mut ix: u32; + + ix = x.to_bits(); + ix &= 0x7fffffff; + if ix >= 0x7f800000 { + return 1.0 / (x * x); + } + x = fabsf(x); + + if ix >= 0x40000000 { + /* |x| >= 2 */ + /* large ulp error near zeros */ + return common(ix, x, false); + } + if ix >= 0x3a000000 { + /* |x| >= 2**-11 */ + /* up to 4ulp error near 2 */ + z = x * x; + r = z * (R02 + z * (R03 + z * (R04 + z * R05))); + s = 1.0 + z * (S01 + z * (S02 + z * (S03 + z * S04))); + return (1.0 + x / 2.0) * (1.0 - x / 2.0) + z * (r / s); + } + if ix >= 0x21800000 { + /* |x| >= 2**-60 */ + x = 0.25 * x * x; + } + return 1.0 - x; +} + +const U00: f32 = -7.3804296553e-02; /* 0xbd9726b5 */ +const U01: f32 = 1.7666645348e-01; /* 0x3e34e80d */ +const U02: f32 = -1.3818567619e-02; /* 0xbc626746 */ +const U03: f32 = 3.4745343146e-04; /* 0x39b62a69 */ +const U04: f32 = -3.8140706238e-06; /* 0xb67ff53c */ +const U05: f32 = 1.9559013964e-08; /* 0x32a802ba */ +const U06: f32 = -3.9820518410e-11; /* 0xae2f21eb */ +const V01: f32 = 1.2730483897e-02; /* 0x3c509385 */ +const V02: f32 = 7.6006865129e-05; /* 0x389f65e0 */ +const V03: f32 = 2.5915085189e-07; /* 0x348b216c */ +const V04: f32 = 4.4111031494e-10; /* 0x2ff280c2 */ + +pub fn y0f(x: f32) -> f32 { + let z: f32; + let u: f32; + let v: f32; + let ix: u32; + + ix = x.to_bits(); + if (ix & 0x7fffffff) == 0 { + return -1.0 / 0.0; + } + if (ix >> 31) != 0 { + return 0.0 / 0.0; + } + if ix >= 0x7f800000 { + return 1.0 / x; + } + if ix >= 0x40000000 { + /* |x| >= 2.0 */ + /* large ulp error near zeros */ + return common(ix, x, true); + } + if ix >= 0x39000000 { + /* x >= 2**-13 */ + /* large ulp error at x ~= 0.89 */ + z = x * x; + u = U00 + z * (U01 + z * (U02 + z * (U03 + z * (U04 + z * (U05 + z * U06))))); + v = 1.0 + z * (V01 + z * (V02 + z * (V03 + z * V04))); + return u / v + TPI * (j0f(x) * logf(x)); + } + return U00 + TPI * logf(x); +} + +/* The asymptotic expansions of pzero is + * 1 - 9/128 s^2 + 11025/98304 s^4 - ..., where s = 1/x. + * For x >= 2, We approximate pzero by + * pzero(x) = 1 + (R/S) + * where R = pR0 + pR1*s^2 + pR2*s^4 + ... + pR5*s^10 + * S = 1 + pS0*s^2 + ... + pS4*s^10 + * and + * | pzero(x)-1-R/S | <= 2 ** ( -60.26) + */ +const PR8: [f32; 6] = [ + /* for x in [inf, 8]=1/[0,0.125] */ + 0.0000000000e+00, /* 0x00000000 */ + -7.0312500000e-02, /* 0xbd900000 */ + -8.0816707611e+00, /* 0xc1014e86 */ + -2.5706311035e+02, /* 0xc3808814 */ + -2.4852163086e+03, /* 0xc51b5376 */ + -5.2530439453e+03, /* 0xc5a4285a */ +]; +const PS8: [f32; 5] = [ + 1.1653436279e+02, /* 0x42e91198 */ + 3.8337448730e+03, /* 0x456f9beb */ + 4.0597855469e+04, /* 0x471e95db */ + 1.1675296875e+05, /* 0x47e4087c */ + 4.7627726562e+04, /* 0x473a0bba */ +]; +const PR5: [f32; 6] = [ + /* for x in [8,4.5454]=1/[0.125,0.22001] */ + -1.1412546255e-11, /* 0xad48c58a */ + -7.0312492549e-02, /* 0xbd8fffff */ + -4.1596107483e+00, /* 0xc0851b88 */ + -6.7674766541e+01, /* 0xc287597b */ + -3.3123129272e+02, /* 0xc3a59d9b */ + -3.4643338013e+02, /* 0xc3ad3779 */ +]; +const PS5: [f32; 5] = [ + 6.0753936768e+01, /* 0x42730408 */ + 1.0512523193e+03, /* 0x44836813 */ + 5.9789707031e+03, /* 0x45bad7c4 */ + 9.6254453125e+03, /* 0x461665c8 */ + 2.4060581055e+03, /* 0x451660ee */ +]; + +const PR3: [f32; 6] = [ + /* for x in [4.547,2.8571]=1/[0.2199,0.35001] */ + -2.5470459075e-09, /* 0xb12f081b */ + -7.0311963558e-02, /* 0xbd8fffb8 */ + -2.4090321064e+00, /* 0xc01a2d95 */ + -2.1965976715e+01, /* 0xc1afba52 */ + -5.8079170227e+01, /* 0xc2685112 */ + -3.1447946548e+01, /* 0xc1fb9565 */ +]; +const PS3: [f32; 5] = [ + 3.5856033325e+01, /* 0x420f6c94 */ + 3.6151397705e+02, /* 0x43b4c1ca */ + 1.1936077881e+03, /* 0x44953373 */ + 1.1279968262e+03, /* 0x448cffe6 */ + 1.7358093262e+02, /* 0x432d94b8 */ +]; + +const PR2: [f32; 6] = [ + /* for x in [2.8570,2]=1/[0.3499,0.5] */ + -8.8753431271e-08, /* 0xb3be98b7 */ + -7.0303097367e-02, /* 0xbd8ffb12 */ + -1.4507384300e+00, /* 0xbfb9b1cc */ + -7.6356959343e+00, /* 0xc0f4579f */ + -1.1193166733e+01, /* 0xc1331736 */ + -3.2336456776e+00, /* 0xc04ef40d */ +]; +const PS2: [f32; 5] = [ + 2.2220300674e+01, /* 0x41b1c32d */ + 1.3620678711e+02, /* 0x430834f0 */ + 2.7047027588e+02, /* 0x43873c32 */ + 1.5387539673e+02, /* 0x4319e01a */ + 1.4657617569e+01, /* 0x416a859a */ +]; + +fn pzerof(x: f32) -> f32 { + let p: &[f32; 6]; + let q: &[f32; 5]; + let z: f32; + let r: f32; + let s: f32; + let mut ix: u32; + + ix = x.to_bits(); + ix &= 0x7fffffff; + if ix >= 0x41000000 { + p = &PR8; + q = &PS8; + } else if ix >= 0x409173eb { + p = &PR5; + q = &PS5; + } else if ix >= 0x4036d917 { + p = &PR3; + q = &PS3; + } else + /*ix >= 0x40000000*/ + { + p = &PR2; + q = &PS2; + } + z = 1.0 / (x * x); + r = p[0] + z * (p[1] + z * (p[2] + z * (p[3] + z * (p[4] + z * p[5])))); + s = 1.0 + z * (q[0] + z * (q[1] + z * (q[2] + z * (q[3] + z * q[4])))); + return 1.0 + r / s; +} + +/* For x >= 8, the asymptotic expansions of qzero is + * -1/8 s + 75/1024 s^3 - ..., where s = 1/x. + * We approximate pzero by + * qzero(x) = s*(-1.25 + (R/S)) + * where R = qR0 + qR1*s^2 + qR2*s^4 + ... + qR5*s^10 + * S = 1 + qS0*s^2 + ... + qS5*s^12 + * and + * | qzero(x)/s +1.25-R/S | <= 2 ** ( -61.22) + */ +const QR8: [f32; 6] = [ + /* for x in [inf, 8]=1/[0,0.125] */ + 0.0000000000e+00, /* 0x00000000 */ + 7.3242187500e-02, /* 0x3d960000 */ + 1.1768206596e+01, /* 0x413c4a93 */ + 5.5767340088e+02, /* 0x440b6b19 */ + 8.8591972656e+03, /* 0x460a6cca */ + 3.7014625000e+04, /* 0x471096a0 */ +]; +const QS8: [f32; 6] = [ + 1.6377603149e+02, /* 0x4323c6aa */ + 8.0983447266e+03, /* 0x45fd12c2 */ + 1.4253829688e+05, /* 0x480b3293 */ + 8.0330925000e+05, /* 0x49441ed4 */ + 8.4050156250e+05, /* 0x494d3359 */ + -3.4389928125e+05, /* 0xc8a7eb69 */ +]; + +const QR5: [f32; 6] = [ + /* for x in [8,4.5454]=1/[0.125,0.22001] */ + 1.8408595828e-11, /* 0x2da1ec79 */ + 7.3242180049e-02, /* 0x3d95ffff */ + 5.8356351852e+00, /* 0x40babd86 */ + 1.3511157227e+02, /* 0x43071c90 */ + 1.0272437744e+03, /* 0x448067cd */ + 1.9899779053e+03, /* 0x44f8bf4b */ +]; +const QS5: [f32; 6] = [ + 8.2776611328e+01, /* 0x42a58da0 */ + 2.0778142090e+03, /* 0x4501dd07 */ + 1.8847289062e+04, /* 0x46933e94 */ + 5.6751113281e+04, /* 0x475daf1d */ + 3.5976753906e+04, /* 0x470c88c1 */ + -5.3543427734e+03, /* 0xc5a752be */ +]; + +const QR3: [f32; 6] = [ + /* for x in [4.547,2.8571]=1/[0.2199,0.35001] */ + 4.3774099900e-09, /* 0x3196681b */ + 7.3241114616e-02, /* 0x3d95ff70 */ + 3.3442313671e+00, /* 0x405607e3 */ + 4.2621845245e+01, /* 0x422a7cc5 */ + 1.7080809021e+02, /* 0x432acedf */ + 1.6673394775e+02, /* 0x4326bbe4 */ +]; +const QS3: [f32; 6] = [ + 4.8758872986e+01, /* 0x42430916 */ + 7.0968920898e+02, /* 0x44316c1c */ + 3.7041481934e+03, /* 0x4567825f */ + 6.4604252930e+03, /* 0x45c9e367 */ + 2.5163337402e+03, /* 0x451d4557 */ + -1.4924745178e+02, /* 0xc3153f59 */ +]; + +const QR2: [f32; 6] = [ + /* for x in [2.8570,2]=1/[0.3499,0.5] */ + 1.5044444979e-07, /* 0x342189db */ + 7.3223426938e-02, /* 0x3d95f62a */ + 1.9981917143e+00, /* 0x3fffc4bf */ + 1.4495602608e+01, /* 0x4167edfd */ + 3.1666231155e+01, /* 0x41fd5471 */ + 1.6252708435e+01, /* 0x4182058c */ +]; +const QS2: [f32; 6] = [ + 3.0365585327e+01, /* 0x41f2ecb8 */ + 2.6934811401e+02, /* 0x4386ac8f */ + 8.4478375244e+02, /* 0x44533229 */ + 8.8293585205e+02, /* 0x445cbbe5 */ + 2.1266638184e+02, /* 0x4354aa98 */ + -5.3109550476e+00, /* 0xc0a9f358 */ +]; + +fn qzerof(x: f32) -> f32 { + let p: &[f32; 6]; + let q: &[f32; 6]; + let s: f32; + let r: f32; + let z: f32; + let mut ix: u32; + + ix = x.to_bits(); + ix &= 0x7fffffff; + if ix >= 0x41000000 { + p = &QR8; + q = &QS8; + } else if ix >= 0x409173eb { + p = &QR5; + q = &QS5; + } else if ix >= 0x4036d917 { + p = &QR3; + q = &QS3; + } else + /*ix >= 0x40000000*/ + { + p = &QR2; + q = &QS2; + } + z = 1.0 / (x * x); + r = p[0] + z * (p[1] + z * (p[2] + z * (p[3] + z * (p[4] + z * p[5])))); + s = 1.0 + z * (q[0] + z * (q[1] + z * (q[2] + z * (q[3] + z * (q[4] + z * q[5]))))); + return (-0.125 + r / s) / x; +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/j1.rs b/crux-mir/lib/compiler_builtins/libm/src/math/j1.rs new file mode 100644 index 000000000..02a65ca5a --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/j1.rs @@ -0,0 +1,414 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_j1.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunSoft, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ +/* j1(x), y1(x) + * Bessel function of the first and second kinds of order zero. + * Method -- j1(x): + * 1. For tiny x, we use j1(x) = x/2 - x^3/16 + x^5/384 - ... + * 2. Reduce x to |x| since j1(x)=-j1(-x), and + * for x in (0,2) + * j1(x) = x/2 + x*z*R0/S0, where z = x*x; + * (precision: |j1/x - 1/2 - R0/S0 |<2**-61.51 ) + * for x in (2,inf) + * j1(x) = sqrt(2/(pi*x))*(p1(x)*cos(x1)-q1(x)*sin(x1)) + * y1(x) = sqrt(2/(pi*x))*(p1(x)*sin(x1)+q1(x)*cos(x1)) + * where x1 = x-3*pi/4. It is better to compute sin(x1),cos(x1) + * as follow: + * cos(x1) = cos(x)cos(3pi/4)+sin(x)sin(3pi/4) + * = 1/sqrt(2) * (sin(x) - cos(x)) + * sin(x1) = sin(x)cos(3pi/4)-cos(x)sin(3pi/4) + * = -1/sqrt(2) * (sin(x) + cos(x)) + * (To avoid cancellation, use + * sin(x) +- cos(x) = -cos(2x)/(sin(x) -+ cos(x)) + * to compute the worse one.) + * + * 3 Special cases + * j1(nan)= nan + * j1(0) = 0 + * j1(inf) = 0 + * + * Method -- y1(x): + * 1. screen out x<=0 cases: y1(0)=-inf, y1(x<0)=NaN + * 2. For x<2. + * Since + * y1(x) = 2/pi*(j1(x)*(ln(x/2)+Euler)-1/x-x/2+5/64*x^3-...) + * therefore y1(x)-2/pi*j1(x)*ln(x)-1/x is an odd function. + * We use the following function to approximate y1, + * y1(x) = x*U(z)/V(z) + (2/pi)*(j1(x)*ln(x)-1/x), z= x^2 + * where for x in [0,2] (abs err less than 2**-65.89) + * U(z) = U0[0] + U0[1]*z + ... + U0[4]*z^4 + * V(z) = 1 + v0[0]*z + ... + v0[4]*z^5 + * Note: For tiny x, 1/x dominate y1 and hence + * y1(tiny) = -2/pi/tiny, (choose tiny<2**-54) + * 3. For x>=2. + * y1(x) = sqrt(2/(pi*x))*(p1(x)*sin(x1)+q1(x)*cos(x1)) + * where x1 = x-3*pi/4. It is better to compute sin(x1),cos(x1) + * by method mentioned above. + */ + +use super::{cos, fabs, get_high_word, get_low_word, log, sin, sqrt}; + +const INVSQRTPI: f64 = 5.64189583547756279280e-01; /* 0x3FE20DD7, 0x50429B6D */ +const TPI: f64 = 6.36619772367581382433e-01; /* 0x3FE45F30, 0x6DC9C883 */ + +fn common(ix: u32, x: f64, y1: bool, sign: bool) -> f64 { + let z: f64; + let mut s: f64; + let c: f64; + let mut ss: f64; + let mut cc: f64; + + /* + * j1(x) = sqrt(2/(pi*x))*(p1(x)*cos(x-3pi/4)-q1(x)*sin(x-3pi/4)) + * y1(x) = sqrt(2/(pi*x))*(p1(x)*sin(x-3pi/4)+q1(x)*cos(x-3pi/4)) + * + * sin(x-3pi/4) = -(sin(x) + cos(x))/sqrt(2) + * cos(x-3pi/4) = (sin(x) - cos(x))/sqrt(2) + * sin(x) +- cos(x) = -cos(2x)/(sin(x) -+ cos(x)) + */ + s = sin(x); + if y1 { + s = -s; + } + c = cos(x); + cc = s - c; + if ix < 0x7fe00000 { + /* avoid overflow in 2*x */ + ss = -s - c; + z = cos(2.0 * x); + if s * c > 0.0 { + cc = z / ss; + } else { + ss = z / cc; + } + if ix < 0x48000000 { + if y1 { + ss = -ss; + } + cc = pone(x) * cc - qone(x) * ss; + } + } + if sign { + cc = -cc; + } + return INVSQRTPI * cc / sqrt(x); +} + +/* R0/S0 on [0,2] */ +const R00: f64 = -6.25000000000000000000e-02; /* 0xBFB00000, 0x00000000 */ +const R01: f64 = 1.40705666955189706048e-03; /* 0x3F570D9F, 0x98472C61 */ +const R02: f64 = -1.59955631084035597520e-05; /* 0xBEF0C5C6, 0xBA169668 */ +const R03: f64 = 4.96727999609584448412e-08; /* 0x3E6AAAFA, 0x46CA0BD9 */ +const S01: f64 = 1.91537599538363460805e-02; /* 0x3F939D0B, 0x12637E53 */ +const S02: f64 = 1.85946785588630915560e-04; /* 0x3F285F56, 0xB9CDF664 */ +const S03: f64 = 1.17718464042623683263e-06; /* 0x3EB3BFF8, 0x333F8498 */ +const S04: f64 = 5.04636257076217042715e-09; /* 0x3E35AC88, 0xC97DFF2C */ +const S05: f64 = 1.23542274426137913908e-11; /* 0x3DAB2ACF, 0xCFB97ED8 */ + +pub fn j1(x: f64) -> f64 { + let mut z: f64; + let r: f64; + let s: f64; + let mut ix: u32; + let sign: bool; + + ix = get_high_word(x); + sign = (ix >> 31) != 0; + ix &= 0x7fffffff; + if ix >= 0x7ff00000 { + return 1.0 / (x * x); + } + if ix >= 0x40000000 { + /* |x| >= 2 */ + return common(ix, fabs(x), false, sign); + } + if ix >= 0x38000000 { + /* |x| >= 2**-127 */ + z = x * x; + r = z * (R00 + z * (R01 + z * (R02 + z * R03))); + s = 1.0 + z * (S01 + z * (S02 + z * (S03 + z * (S04 + z * S05)))); + z = r / s; + } else { + /* avoid underflow, raise inexact if x!=0 */ + z = x; + } + return (0.5 + z) * x; +} + +const U0: [f64; 5] = [ + -1.96057090646238940668e-01, /* 0xBFC91866, 0x143CBC8A */ + 5.04438716639811282616e-02, /* 0x3FA9D3C7, 0x76292CD1 */ + -1.91256895875763547298e-03, /* 0xBF5F55E5, 0x4844F50F */ + 2.35252600561610495928e-05, /* 0x3EF8AB03, 0x8FA6B88E */ + -9.19099158039878874504e-08, /* 0xBE78AC00, 0x569105B8 */ +]; +const V0: [f64; 5] = [ + 1.99167318236649903973e-02, /* 0x3F94650D, 0x3F4DA9F0 */ + 2.02552581025135171496e-04, /* 0x3F2A8C89, 0x6C257764 */ + 1.35608801097516229404e-06, /* 0x3EB6C05A, 0x894E8CA6 */ + 6.22741452364621501295e-09, /* 0x3E3ABF1D, 0x5BA69A86 */ + 1.66559246207992079114e-11, /* 0x3DB25039, 0xDACA772A */ +]; + +pub fn y1(x: f64) -> f64 { + let z: f64; + let u: f64; + let v: f64; + let ix: u32; + let lx: u32; + + ix = get_high_word(x); + lx = get_low_word(x); + + /* y1(nan)=nan, y1(<0)=nan, y1(0)=-inf, y1(inf)=0 */ + if (ix << 1 | lx) == 0 { + return -1.0 / 0.0; + } + if (ix >> 31) != 0 { + return 0.0 / 0.0; + } + if ix >= 0x7ff00000 { + return 1.0 / x; + } + + if ix >= 0x40000000 { + /* x >= 2 */ + return common(ix, x, true, false); + } + if ix < 0x3c900000 { + /* x < 2**-54 */ + return -TPI / x; + } + z = x * x; + u = U0[0] + z * (U0[1] + z * (U0[2] + z * (U0[3] + z * U0[4]))); + v = 1.0 + z * (V0[0] + z * (V0[1] + z * (V0[2] + z * (V0[3] + z * V0[4])))); + return x * (u / v) + TPI * (j1(x) * log(x) - 1.0 / x); +} + +/* For x >= 8, the asymptotic expansions of pone is + * 1 + 15/128 s^2 - 4725/2^15 s^4 - ..., where s = 1/x. + * We approximate pone by + * pone(x) = 1 + (R/S) + * where R = pr0 + pr1*s^2 + pr2*s^4 + ... + pr5*s^10 + * S = 1 + ps0*s^2 + ... + ps4*s^10 + * and + * | pone(x)-1-R/S | <= 2 ** ( -60.06) + */ + +const PR8: [f64; 6] = [ + /* for x in [inf, 8]=1/[0,0.125] */ + 0.00000000000000000000e+00, /* 0x00000000, 0x00000000 */ + 1.17187499999988647970e-01, /* 0x3FBDFFFF, 0xFFFFFCCE */ + 1.32394806593073575129e+01, /* 0x402A7A9D, 0x357F7FCE */ + 4.12051854307378562225e+02, /* 0x4079C0D4, 0x652EA590 */ + 3.87474538913960532227e+03, /* 0x40AE457D, 0xA3A532CC */ + 7.91447954031891731574e+03, /* 0x40BEEA7A, 0xC32782DD */ +]; +const PS8: [f64; 5] = [ + 1.14207370375678408436e+02, /* 0x405C8D45, 0x8E656CAC */ + 3.65093083420853463394e+03, /* 0x40AC85DC, 0x964D274F */ + 3.69562060269033463555e+04, /* 0x40E20B86, 0x97C5BB7F */ + 9.76027935934950801311e+04, /* 0x40F7D42C, 0xB28F17BB */ + 3.08042720627888811578e+04, /* 0x40DE1511, 0x697A0B2D */ +]; + +const PR5: [f64; 6] = [ + /* for x in [8,4.5454]=1/[0.125,0.22001] */ + 1.31990519556243522749e-11, /* 0x3DAD0667, 0xDAE1CA7D */ + 1.17187493190614097638e-01, /* 0x3FBDFFFF, 0xE2C10043 */ + 6.80275127868432871736e+00, /* 0x401B3604, 0x6E6315E3 */ + 1.08308182990189109773e+02, /* 0x405B13B9, 0x452602ED */ + 5.17636139533199752805e+02, /* 0x40802D16, 0xD052D649 */ + 5.28715201363337541807e+02, /* 0x408085B8, 0xBB7E0CB7 */ +]; +const PS5: [f64; 5] = [ + 5.92805987221131331921e+01, /* 0x404DA3EA, 0xA8AF633D */ + 9.91401418733614377743e+02, /* 0x408EFB36, 0x1B066701 */ + 5.35326695291487976647e+03, /* 0x40B4E944, 0x5706B6FB */ + 7.84469031749551231769e+03, /* 0x40BEA4B0, 0xB8A5BB15 */ + 1.50404688810361062679e+03, /* 0x40978030, 0x036F5E51 */ +]; + +const PR3: [f64; 6] = [ + 3.02503916137373618024e-09, /* 0x3E29FC21, 0xA7AD9EDD */ + 1.17186865567253592491e-01, /* 0x3FBDFFF5, 0x5B21D17B */ + 3.93297750033315640650e+00, /* 0x400F76BC, 0xE85EAD8A */ + 3.51194035591636932736e+01, /* 0x40418F48, 0x9DA6D129 */ + 9.10550110750781271918e+01, /* 0x4056C385, 0x4D2C1837 */ + 4.85590685197364919645e+01, /* 0x4048478F, 0x8EA83EE5 */ +]; +const PS3: [f64; 5] = [ + 3.47913095001251519989e+01, /* 0x40416549, 0xA134069C */ + 3.36762458747825746741e+02, /* 0x40750C33, 0x07F1A75F */ + 1.04687139975775130551e+03, /* 0x40905B7C, 0x5037D523 */ + 8.90811346398256432622e+02, /* 0x408BD67D, 0xA32E31E9 */ + 1.03787932439639277504e+02, /* 0x4059F26D, 0x7C2EED53 */ +]; + +const PR2: [f64; 6] = [ + /* for x in [2.8570,2]=1/[0.3499,0.5] */ + 1.07710830106873743082e-07, /* 0x3E7CE9D4, 0xF65544F4 */ + 1.17176219462683348094e-01, /* 0x3FBDFF42, 0xBE760D83 */ + 2.36851496667608785174e+00, /* 0x4002F2B7, 0xF98FAEC0 */ + 1.22426109148261232917e+01, /* 0x40287C37, 0x7F71A964 */ + 1.76939711271687727390e+01, /* 0x4031B1A8, 0x177F8EE2 */ + 5.07352312588818499250e+00, /* 0x40144B49, 0xA574C1FE */ +]; +const PS2: [f64; 5] = [ + 2.14364859363821409488e+01, /* 0x40356FBD, 0x8AD5ECDC */ + 1.25290227168402751090e+02, /* 0x405F5293, 0x14F92CD5 */ + 2.32276469057162813669e+02, /* 0x406D08D8, 0xD5A2DBD9 */ + 1.17679373287147100768e+02, /* 0x405D6B7A, 0xDA1884A9 */ + 8.36463893371618283368e+00, /* 0x4020BAB1, 0xF44E5192 */ +]; + +fn pone(x: f64) -> f64 { + let p: &[f64; 6]; + let q: &[f64; 5]; + let z: f64; + let r: f64; + let s: f64; + let mut ix: u32; + + ix = get_high_word(x); + ix &= 0x7fffffff; + if ix >= 0x40200000 { + p = &PR8; + q = &PS8; + } else if ix >= 0x40122E8B { + p = &PR5; + q = &PS5; + } else if ix >= 0x4006DB6D { + p = &PR3; + q = &PS3; + } else + /*ix >= 0x40000000*/ + { + p = &PR2; + q = &PS2; + } + z = 1.0 / (x * x); + r = p[0] + z * (p[1] + z * (p[2] + z * (p[3] + z * (p[4] + z * p[5])))); + s = 1.0 + z * (q[0] + z * (q[1] + z * (q[2] + z * (q[3] + z * q[4])))); + return 1.0 + r / s; +} + +/* For x >= 8, the asymptotic expansions of qone is + * 3/8 s - 105/1024 s^3 - ..., where s = 1/x. + * We approximate pone by + * qone(x) = s*(0.375 + (R/S)) + * where R = qr1*s^2 + qr2*s^4 + ... + qr5*s^10 + * S = 1 + qs1*s^2 + ... + qs6*s^12 + * and + * | qone(x)/s -0.375-R/S | <= 2 ** ( -61.13) + */ + +const QR8: [f64; 6] = [ + /* for x in [inf, 8]=1/[0,0.125] */ + 0.00000000000000000000e+00, /* 0x00000000, 0x00000000 */ + -1.02539062499992714161e-01, /* 0xBFBA3FFF, 0xFFFFFDF3 */ + -1.62717534544589987888e+01, /* 0xC0304591, 0xA26779F7 */ + -7.59601722513950107896e+02, /* 0xC087BCD0, 0x53E4B576 */ + -1.18498066702429587167e+04, /* 0xC0C724E7, 0x40F87415 */ + -4.84385124285750353010e+04, /* 0xC0E7A6D0, 0x65D09C6A */ +]; +const QS8: [f64; 6] = [ + 1.61395369700722909556e+02, /* 0x40642CA6, 0xDE5BCDE5 */ + 7.82538599923348465381e+03, /* 0x40BE9162, 0xD0D88419 */ + 1.33875336287249578163e+05, /* 0x4100579A, 0xB0B75E98 */ + 7.19657723683240939863e+05, /* 0x4125F653, 0x72869C19 */ + 6.66601232617776375264e+05, /* 0x412457D2, 0x7719AD5C */ + -2.94490264303834643215e+05, /* 0xC111F969, 0x0EA5AA18 */ +]; + +const QR5: [f64; 6] = [ + /* for x in [8,4.5454]=1/[0.125,0.22001] */ + -2.08979931141764104297e-11, /* 0xBDB6FA43, 0x1AA1A098 */ + -1.02539050241375426231e-01, /* 0xBFBA3FFF, 0xCB597FEF */ + -8.05644828123936029840e+00, /* 0xC0201CE6, 0xCA03AD4B */ + -1.83669607474888380239e+02, /* 0xC066F56D, 0x6CA7B9B0 */ + -1.37319376065508163265e+03, /* 0xC09574C6, 0x6931734F */ + -2.61244440453215656817e+03, /* 0xC0A468E3, 0x88FDA79D */ +]; +const QS5: [f64; 6] = [ + 8.12765501384335777857e+01, /* 0x405451B2, 0xFF5A11B2 */ + 1.99179873460485964642e+03, /* 0x409F1F31, 0xE77BF839 */ + 1.74684851924908907677e+04, /* 0x40D10F1F, 0x0D64CE29 */ + 4.98514270910352279316e+04, /* 0x40E8576D, 0xAABAD197 */ + 2.79480751638918118260e+04, /* 0x40DB4B04, 0xCF7C364B */ + -4.71918354795128470869e+03, /* 0xC0B26F2E, 0xFCFFA004 */ +]; + +const QR3: [f64; 6] = [ + -5.07831226461766561369e-09, /* 0xBE35CFA9, 0xD38FC84F */ + -1.02537829820837089745e-01, /* 0xBFBA3FEB, 0x51AEED54 */ + -4.61011581139473403113e+00, /* 0xC01270C2, 0x3302D9FF */ + -5.78472216562783643212e+01, /* 0xC04CEC71, 0xC25D16DA */ + -2.28244540737631695038e+02, /* 0xC06C87D3, 0x4718D55F */ + -2.19210128478909325622e+02, /* 0xC06B66B9, 0x5F5C1BF6 */ +]; +const QS3: [f64; 6] = [ + 4.76651550323729509273e+01, /* 0x4047D523, 0xCCD367E4 */ + 6.73865112676699709482e+02, /* 0x40850EEB, 0xC031EE3E */ + 3.38015286679526343505e+03, /* 0x40AA684E, 0x448E7C9A */ + 5.54772909720722782367e+03, /* 0x40B5ABBA, 0xA61D54A6 */ + 1.90311919338810798763e+03, /* 0x409DBC7A, 0x0DD4DF4B */ + -1.35201191444307340817e+02, /* 0xC060E670, 0x290A311F */ +]; + +const QR2: [f64; 6] = [ + /* for x in [2.8570,2]=1/[0.3499,0.5] */ + -1.78381727510958865572e-07, /* 0xBE87F126, 0x44C626D2 */ + -1.02517042607985553460e-01, /* 0xBFBA3E8E, 0x9148B010 */ + -2.75220568278187460720e+00, /* 0xC0060484, 0x69BB4EDA */ + -1.96636162643703720221e+01, /* 0xC033A9E2, 0xC168907F */ + -4.23253133372830490089e+01, /* 0xC04529A3, 0xDE104AAA */ + -2.13719211703704061733e+01, /* 0xC0355F36, 0x39CF6E52 */ +]; +const QS2: [f64; 6] = [ + 2.95333629060523854548e+01, /* 0x403D888A, 0x78AE64FF */ + 2.52981549982190529136e+02, /* 0x406F9F68, 0xDB821CBA */ + 7.57502834868645436472e+02, /* 0x4087AC05, 0xCE49A0F7 */ + 7.39393205320467245656e+02, /* 0x40871B25, 0x48D4C029 */ + 1.55949003336666123687e+02, /* 0x40637E5E, 0x3C3ED8D4 */ + -4.95949898822628210127e+00, /* 0xC013D686, 0xE71BE86B */ +]; + +fn qone(x: f64) -> f64 { + let p: &[f64; 6]; + let q: &[f64; 6]; + let s: f64; + let r: f64; + let z: f64; + let mut ix: u32; + + ix = get_high_word(x); + ix &= 0x7fffffff; + if ix >= 0x40200000 { + p = &QR8; + q = &QS8; + } else if ix >= 0x40122E8B { + p = &QR5; + q = &QS5; + } else if ix >= 0x4006DB6D { + p = &QR3; + q = &QS3; + } else + /*ix >= 0x40000000*/ + { + p = &QR2; + q = &QS2; + } + z = 1.0 / (x * x); + r = p[0] + z * (p[1] + z * (p[2] + z * (p[3] + z * (p[4] + z * p[5])))); + s = 1.0 + z * (q[0] + z * (q[1] + z * (q[2] + z * (q[3] + z * (q[4] + z * q[5]))))); + return (0.375 + r / s) / x; +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/j1f.rs b/crux-mir/lib/compiler_builtins/libm/src/math/j1f.rs new file mode 100644 index 000000000..c39f8ff7e --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/j1f.rs @@ -0,0 +1,380 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_j1f.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +use super::{cosf, fabsf, logf, sinf, sqrtf}; + +const INVSQRTPI: f32 = 5.6418961287e-01; /* 0x3f106ebb */ +const TPI: f32 = 6.3661974669e-01; /* 0x3f22f983 */ + +fn common(ix: u32, x: f32, y1: bool, sign: bool) -> f32 { + let z: f64; + let mut s: f64; + let c: f64; + let mut ss: f64; + let mut cc: f64; + + s = sinf(x) as f64; + if y1 { + s = -s; + } + c = cosf(x) as f64; + cc = s - c; + if ix < 0x7f000000 { + ss = -s - c; + z = cosf(2.0 * x) as f64; + if s * c > 0.0 { + cc = z / ss; + } else { + ss = z / cc; + } + if ix < 0x58800000 { + if y1 { + ss = -ss; + } + cc = (ponef(x) as f64) * cc - (qonef(x) as f64) * ss; + } + } + if sign { + cc = -cc; + } + return (((INVSQRTPI as f64) * cc) / (sqrtf(x) as f64)) as f32; +} + +/* R0/S0 on [0,2] */ +const R00: f32 = -6.2500000000e-02; /* 0xbd800000 */ +const R01: f32 = 1.4070566976e-03; /* 0x3ab86cfd */ +const R02: f32 = -1.5995563444e-05; /* 0xb7862e36 */ +const R03: f32 = 4.9672799207e-08; /* 0x335557d2 */ +const S01: f32 = 1.9153760746e-02; /* 0x3c9ce859 */ +const S02: f32 = 1.8594678841e-04; /* 0x3942fab6 */ +const S03: f32 = 1.1771846857e-06; /* 0x359dffc2 */ +const S04: f32 = 5.0463624390e-09; /* 0x31ad6446 */ +const S05: f32 = 1.2354227016e-11; /* 0x2d59567e */ + +pub fn j1f(x: f32) -> f32 { + let mut z: f32; + let r: f32; + let s: f32; + let mut ix: u32; + let sign: bool; + + ix = x.to_bits(); + sign = (ix >> 31) != 0; + ix &= 0x7fffffff; + if ix >= 0x7f800000 { + return 1.0 / (x * x); + } + if ix >= 0x40000000 { + /* |x| >= 2 */ + return common(ix, fabsf(x), false, sign); + } + if ix >= 0x39000000 { + /* |x| >= 2**-13 */ + z = x * x; + r = z * (R00 + z * (R01 + z * (R02 + z * R03))); + s = 1.0 + z * (S01 + z * (S02 + z * (S03 + z * (S04 + z * S05)))); + z = 0.5 + r / s; + } else { + z = 0.5; + } + return z * x; +} + +const U0: [f32; 5] = [ + -1.9605709612e-01, /* 0xbe48c331 */ + 5.0443872809e-02, /* 0x3d4e9e3c */ + -1.9125689287e-03, /* 0xbafaaf2a */ + 2.3525259166e-05, /* 0x37c5581c */ + -9.1909917899e-08, /* 0xb3c56003 */ +]; +const V0: [f32; 5] = [ + 1.9916731864e-02, /* 0x3ca3286a */ + 2.0255257550e-04, /* 0x3954644b */ + 1.3560879779e-06, /* 0x35b602d4 */ + 6.2274145840e-09, /* 0x31d5f8eb */ + 1.6655924903e-11, /* 0x2d9281cf */ +]; + +pub fn y1f(x: f32) -> f32 { + let z: f32; + let u: f32; + let v: f32; + let ix: u32; + + ix = x.to_bits(); + if (ix & 0x7fffffff) == 0 { + return -1.0 / 0.0; + } + if (ix >> 31) != 0 { + return 0.0 / 0.0; + } + if ix >= 0x7f800000 { + return 1.0 / x; + } + if ix >= 0x40000000 { + /* |x| >= 2.0 */ + return common(ix, x, true, false); + } + if ix < 0x33000000 { + /* x < 2**-25 */ + return -TPI / x; + } + z = x * x; + u = U0[0] + z * (U0[1] + z * (U0[2] + z * (U0[3] + z * U0[4]))); + v = 1.0 + z * (V0[0] + z * (V0[1] + z * (V0[2] + z * (V0[3] + z * V0[4])))); + return x * (u / v) + TPI * (j1f(x) * logf(x) - 1.0 / x); +} + +/* For x >= 8, the asymptotic expansions of pone is + * 1 + 15/128 s^2 - 4725/2^15 s^4 - ..., where s = 1/x. + * We approximate pone by + * pone(x) = 1 + (R/S) + * where R = pr0 + pr1*s^2 + pr2*s^4 + ... + pr5*s^10 + * S = 1 + ps0*s^2 + ... + ps4*s^10 + * and + * | pone(x)-1-R/S | <= 2 ** ( -60.06) + */ + +const PR8: [f32; 6] = [ + /* for x in [inf, 8]=1/[0,0.125] */ + 0.0000000000e+00, /* 0x00000000 */ + 1.1718750000e-01, /* 0x3df00000 */ + 1.3239480972e+01, /* 0x4153d4ea */ + 4.1205184937e+02, /* 0x43ce06a3 */ + 3.8747453613e+03, /* 0x45722bed */ + 7.9144794922e+03, /* 0x45f753d6 */ +]; +const PS8: [f32; 5] = [ + 1.1420736694e+02, /* 0x42e46a2c */ + 3.6509309082e+03, /* 0x45642ee5 */ + 3.6956207031e+04, /* 0x47105c35 */ + 9.7602796875e+04, /* 0x47bea166 */ + 3.0804271484e+04, /* 0x46f0a88b */ +]; + +const PR5: [f32; 6] = [ + /* for x in [8,4.5454]=1/[0.125,0.22001] */ + 1.3199052094e-11, /* 0x2d68333f */ + 1.1718749255e-01, /* 0x3defffff */ + 6.8027510643e+00, /* 0x40d9b023 */ + 1.0830818176e+02, /* 0x42d89dca */ + 5.1763616943e+02, /* 0x440168b7 */ + 5.2871520996e+02, /* 0x44042dc6 */ +]; +const PS5: [f32; 5] = [ + 5.9280597687e+01, /* 0x426d1f55 */ + 9.9140142822e+02, /* 0x4477d9b1 */ + 5.3532670898e+03, /* 0x45a74a23 */ + 7.8446904297e+03, /* 0x45f52586 */ + 1.5040468750e+03, /* 0x44bc0180 */ +]; + +const PR3: [f32; 6] = [ + 3.0250391081e-09, /* 0x314fe10d */ + 1.1718686670e-01, /* 0x3defffab */ + 3.9329774380e+00, /* 0x407bb5e7 */ + 3.5119403839e+01, /* 0x420c7a45 */ + 9.1055007935e+01, /* 0x42b61c2a */ + 4.8559066772e+01, /* 0x42423c7c */ +]; +const PS3: [f32; 5] = [ + 3.4791309357e+01, /* 0x420b2a4d */ + 3.3676245117e+02, /* 0x43a86198 */ + 1.0468714600e+03, /* 0x4482dbe3 */ + 8.9081134033e+02, /* 0x445eb3ed */ + 1.0378793335e+02, /* 0x42cf936c */ +]; + +const PR2: [f32; 6] = [ + /* for x in [2.8570,2]=1/[0.3499,0.5] */ + 1.0771083225e-07, /* 0x33e74ea8 */ + 1.1717621982e-01, /* 0x3deffa16 */ + 2.3685150146e+00, /* 0x401795c0 */ + 1.2242610931e+01, /* 0x4143e1bc */ + 1.7693971634e+01, /* 0x418d8d41 */ + 5.0735230446e+00, /* 0x40a25a4d */ +]; +const PS2: [f32; 5] = [ + 2.1436485291e+01, /* 0x41ab7dec */ + 1.2529022980e+02, /* 0x42fa9499 */ + 2.3227647400e+02, /* 0x436846c7 */ + 1.1767937469e+02, /* 0x42eb5bd7 */ + 8.3646392822e+00, /* 0x4105d590 */ +]; + +fn ponef(x: f32) -> f32 { + let p: &[f32; 6]; + let q: &[f32; 5]; + let z: f32; + let r: f32; + let s: f32; + let mut ix: u32; + + ix = x.to_bits(); + ix &= 0x7fffffff; + if ix >= 0x41000000 { + p = &PR8; + q = &PS8; + } else if ix >= 0x409173eb { + p = &PR5; + q = &PS5; + } else if ix >= 0x4036d917 { + p = &PR3; + q = &PS3; + } else + /*ix >= 0x40000000*/ + { + p = &PR2; + q = &PS2; + } + z = 1.0 / (x * x); + r = p[0] + z * (p[1] + z * (p[2] + z * (p[3] + z * (p[4] + z * p[5])))); + s = 1.0 + z * (q[0] + z * (q[1] + z * (q[2] + z * (q[3] + z * q[4])))); + return 1.0 + r / s; +} + +/* For x >= 8, the asymptotic expansions of qone is + * 3/8 s - 105/1024 s^3 - ..., where s = 1/x. + * We approximate pone by + * qone(x) = s*(0.375 + (R/S)) + * where R = qr1*s^2 + qr2*s^4 + ... + qr5*s^10 + * S = 1 + qs1*s^2 + ... + qs6*s^12 + * and + * | qone(x)/s -0.375-R/S | <= 2 ** ( -61.13) + */ + +const QR8: [f32; 6] = [ + /* for x in [inf, 8]=1/[0,0.125] */ + 0.0000000000e+00, /* 0x00000000 */ + -1.0253906250e-01, /* 0xbdd20000 */ + -1.6271753311e+01, /* 0xc1822c8d */ + -7.5960174561e+02, /* 0xc43de683 */ + -1.1849806641e+04, /* 0xc639273a */ + -4.8438511719e+04, /* 0xc73d3683 */ +]; +const QS8: [f32; 6] = [ + 1.6139537048e+02, /* 0x43216537 */ + 7.8253862305e+03, /* 0x45f48b17 */ + 1.3387534375e+05, /* 0x4802bcd6 */ + 7.1965775000e+05, /* 0x492fb29c */ + 6.6660125000e+05, /* 0x4922be94 */ + -2.9449025000e+05, /* 0xc88fcb48 */ +]; + +const QR5: [f32; 6] = [ + /* for x in [8,4.5454]=1/[0.125,0.22001] */ + -2.0897993405e-11, /* 0xadb7d219 */ + -1.0253904760e-01, /* 0xbdd1fffe */ + -8.0564479828e+00, /* 0xc100e736 */ + -1.8366960144e+02, /* 0xc337ab6b */ + -1.3731937256e+03, /* 0xc4aba633 */ + -2.6124443359e+03, /* 0xc523471c */ +]; +const QS5: [f32; 6] = [ + 8.1276550293e+01, /* 0x42a28d98 */ + 1.9917987061e+03, /* 0x44f8f98f */ + 1.7468484375e+04, /* 0x468878f8 */ + 4.9851425781e+04, /* 0x4742bb6d */ + 2.7948074219e+04, /* 0x46da5826 */ + -4.7191835938e+03, /* 0xc5937978 */ +]; + +const QR3: [f32; 6] = [ + -5.0783124372e-09, /* 0xb1ae7d4f */ + -1.0253783315e-01, /* 0xbdd1ff5b */ + -4.6101160049e+00, /* 0xc0938612 */ + -5.7847221375e+01, /* 0xc267638e */ + -2.2824453735e+02, /* 0xc3643e9a */ + -2.1921012878e+02, /* 0xc35b35cb */ +]; +const QS3: [f32; 6] = [ + 4.7665153503e+01, /* 0x423ea91e */ + 6.7386511230e+02, /* 0x4428775e */ + 3.3801528320e+03, /* 0x45534272 */ + 5.5477290039e+03, /* 0x45ad5dd5 */ + 1.9031191406e+03, /* 0x44ede3d0 */ + -1.3520118713e+02, /* 0xc3073381 */ +]; + +const QR2: [f32; 6] = [ + /* for x in [2.8570,2]=1/[0.3499,0.5] */ + -1.7838172539e-07, /* 0xb43f8932 */ + -1.0251704603e-01, /* 0xbdd1f475 */ + -2.7522056103e+00, /* 0xc0302423 */ + -1.9663616180e+01, /* 0xc19d4f16 */ + -4.2325313568e+01, /* 0xc2294d1f */ + -2.1371921539e+01, /* 0xc1aaf9b2 */ +]; +const QS2: [f32; 6] = [ + 2.9533363342e+01, /* 0x41ec4454 */ + 2.5298155212e+02, /* 0x437cfb47 */ + 7.5750280762e+02, /* 0x443d602e */ + 7.3939318848e+02, /* 0x4438d92a */ + 1.5594900513e+02, /* 0x431bf2f2 */ + -4.9594988823e+00, /* 0xc09eb437 */ +]; + +fn qonef(x: f32) -> f32 { + let p: &[f32; 6]; + let q: &[f32; 6]; + let s: f32; + let r: f32; + let z: f32; + let mut ix: u32; + + ix = x.to_bits(); + ix &= 0x7fffffff; + if ix >= 0x41000000 { + p = &QR8; + q = &QS8; + } else if ix >= 0x409173eb { + p = &QR5; + q = &QS5; + } else if ix >= 0x4036d917 { + p = &QR3; + q = &QS3; + } else + /*ix >= 0x40000000*/ + { + p = &QR2; + q = &QS2; + } + z = 1.0 / (x * x); + r = p[0] + z * (p[1] + z * (p[2] + z * (p[3] + z * (p[4] + z * p[5])))); + s = 1.0 + z * (q[0] + z * (q[1] + z * (q[2] + z * (q[3] + z * (q[4] + z * q[5]))))); + return (0.375 + r / s) / x; +} + +// PowerPC tests are failing on LLVM 13: https://github.com/rust-lang/rust/issues/88520 +#[cfg(not(target_arch = "powerpc64"))] +#[cfg(test)] +mod tests { + use super::{j1f, y1f}; + #[test] + fn test_j1f_2488() { + // 0x401F3E49 + assert_eq!(j1f(2.4881766_f32), 0.49999475_f32); + } + #[test] + fn test_y1f_2002() { + //allow slightly different result on x87 + let res = y1f(2.0000002_f32); + if cfg!(all(target_arch = "x86", not(target_feature = "sse2"))) && (res == -0.10703231_f32) + { + return; + } + assert_eq!(res, -0.10703229_f32); + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/jn.rs b/crux-mir/lib/compiler_builtins/libm/src/math/jn.rs new file mode 100644 index 000000000..1be167f84 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/jn.rs @@ -0,0 +1,343 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_jn.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunSoft, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ +/* + * jn(n, x), yn(n, x) + * floating point Bessel's function of the 1st and 2nd kind + * of order n + * + * Special cases: + * y0(0)=y1(0)=yn(n,0) = -inf with division by zero signal; + * y0(-ve)=y1(-ve)=yn(n,-ve) are NaN with invalid signal. + * Note 2. About jn(n,x), yn(n,x) + * For n=0, j0(x) is called, + * for n=1, j1(x) is called, + * for n<=x, forward recursion is used starting + * from values of j0(x) and j1(x). + * for n>x, a continued fraction approximation to + * j(n,x)/j(n-1,x) is evaluated and then backward + * recursion is used starting from a supposed value + * for j(n,x). The resulting value of j(0,x) is + * compared with the actual value to correct the + * supposed value of j(n,x). + * + * yn(n,x) is similar in all respects, except + * that forward recursion is used for all + * values of n>1. + */ + +use super::{cos, fabs, get_high_word, get_low_word, j0, j1, log, sin, sqrt, y0, y1}; + +const INVSQRTPI: f64 = 5.64189583547756279280e-01; /* 0x3FE20DD7, 0x50429B6D */ + +pub fn jn(n: i32, mut x: f64) -> f64 { + let mut ix: u32; + let lx: u32; + let nm1: i32; + let mut i: i32; + let mut sign: bool; + let mut a: f64; + let mut b: f64; + let mut temp: f64; + + ix = get_high_word(x); + lx = get_low_word(x); + sign = (ix >> 31) != 0; + ix &= 0x7fffffff; + + // -lx == !lx + 1 + if (ix | (lx | ((!lx).wrapping_add(1))) >> 31) > 0x7ff00000 { + /* nan */ + return x; + } + + /* J(-n,x) = (-1)^n * J(n, x), J(n, -x) = (-1)^n * J(n, x) + * Thus, J(-n,x) = J(n,-x) + */ + /* nm1 = |n|-1 is used instead of |n| to handle n==INT_MIN */ + if n == 0 { + return j0(x); + } + if n < 0 { + nm1 = -(n + 1); + x = -x; + sign = !sign; + } else { + nm1 = n - 1; + } + if nm1 == 0 { + return j1(x); + } + + sign &= (n & 1) != 0; /* even n: 0, odd n: signbit(x) */ + x = fabs(x); + if (ix | lx) == 0 || ix == 0x7ff00000 { + /* if x is 0 or inf */ + b = 0.0; + } else if (nm1 as f64) < x { + /* Safe to use J(n+1,x)=2n/x *J(n,x)-J(n-1,x) */ + if ix >= 0x52d00000 { + /* x > 2**302 */ + /* (x >> n**2) + * Jn(x) = cos(x-(2n+1)*pi/4)*sqrt(2/x*pi) + * Yn(x) = sin(x-(2n+1)*pi/4)*sqrt(2/x*pi) + * Let s=sin(x), c=cos(x), + * xn=x-(2n+1)*pi/4, sqt2 = sqrt(2),then + * + * n sin(xn)*sqt2 cos(xn)*sqt2 + * ---------------------------------- + * 0 s-c c+s + * 1 -s-c -c+s + * 2 -s+c -c-s + * 3 s+c c-s + */ + temp = match nm1 & 3 { + 0 => -cos(x) + sin(x), + 1 => -cos(x) - sin(x), + 2 => cos(x) - sin(x), + 3 | _ => cos(x) + sin(x), + }; + b = INVSQRTPI * temp / sqrt(x); + } else { + a = j0(x); + b = j1(x); + i = 0; + while i < nm1 { + i += 1; + temp = b; + b = b * (2.0 * (i as f64) / x) - a; /* avoid underflow */ + a = temp; + } + } + } else { + if ix < 0x3e100000 { + /* x < 2**-29 */ + /* x is tiny, return the first Taylor expansion of J(n,x) + * J(n,x) = 1/n!*(x/2)^n - ... + */ + if nm1 > 32 { + /* underflow */ + b = 0.0; + } else { + temp = x * 0.5; + b = temp; + a = 1.0; + i = 2; + while i <= nm1 + 1 { + a *= i as f64; /* a = n! */ + b *= temp; /* b = (x/2)^n */ + i += 1; + } + b = b / a; + } + } else { + /* use backward recurrence */ + /* x x^2 x^2 + * J(n,x)/J(n-1,x) = ---- ------ ------ ..... + * 2n - 2(n+1) - 2(n+2) + * + * 1 1 1 + * (for large x) = ---- ------ ------ ..... + * 2n 2(n+1) 2(n+2) + * -- - ------ - ------ - + * x x x + * + * Let w = 2n/x and h=2/x, then the above quotient + * is equal to the continued fraction: + * 1 + * = ----------------------- + * 1 + * w - ----------------- + * 1 + * w+h - --------- + * w+2h - ... + * + * To determine how many terms needed, let + * Q(0) = w, Q(1) = w(w+h) - 1, + * Q(k) = (w+k*h)*Q(k-1) - Q(k-2), + * When Q(k) > 1e4 good for single + * When Q(k) > 1e9 good for double + * When Q(k) > 1e17 good for quadruple + */ + /* determine k */ + let mut t: f64; + let mut q0: f64; + let mut q1: f64; + let mut w: f64; + let h: f64; + let mut z: f64; + let mut tmp: f64; + let nf: f64; + + let mut k: i32; + + nf = (nm1 as f64) + 1.0; + w = 2.0 * nf / x; + h = 2.0 / x; + z = w + h; + q0 = w; + q1 = w * z - 1.0; + k = 1; + while q1 < 1.0e9 { + k += 1; + z += h; + tmp = z * q1 - q0; + q0 = q1; + q1 = tmp; + } + t = 0.0; + i = k; + while i >= 0 { + t = 1.0 / (2.0 * ((i as f64) + nf) / x - t); + i -= 1; + } + a = t; + b = 1.0; + /* estimate log((2/x)^n*n!) = n*log(2/x)+n*ln(n) + * Hence, if n*(log(2n/x)) > ... + * single 8.8722839355e+01 + * double 7.09782712893383973096e+02 + * long double 1.1356523406294143949491931077970765006170e+04 + * then recurrent value may overflow and the result is + * likely underflow to zero + */ + tmp = nf * log(fabs(w)); + if tmp < 7.09782712893383973096e+02 { + i = nm1; + while i > 0 { + temp = b; + b = b * (2.0 * (i as f64)) / x - a; + a = temp; + i -= 1; + } + } else { + i = nm1; + while i > 0 { + temp = b; + b = b * (2.0 * (i as f64)) / x - a; + a = temp; + /* scale b to avoid spurious overflow */ + let x1p500 = f64::from_bits(0x5f30000000000000); // 0x1p500 == 2^500 + if b > x1p500 { + a /= b; + t /= b; + b = 1.0; + } + i -= 1; + } + } + z = j0(x); + w = j1(x); + if fabs(z) >= fabs(w) { + b = t * z / b; + } else { + b = t * w / a; + } + } + } + + if sign { + -b + } else { + b + } +} + +pub fn yn(n: i32, x: f64) -> f64 { + let mut ix: u32; + let lx: u32; + let mut ib: u32; + let nm1: i32; + let mut sign: bool; + let mut i: i32; + let mut a: f64; + let mut b: f64; + let mut temp: f64; + + ix = get_high_word(x); + lx = get_low_word(x); + sign = (ix >> 31) != 0; + ix &= 0x7fffffff; + + // -lx == !lx + 1 + if (ix | (lx | ((!lx).wrapping_add(1))) >> 31) > 0x7ff00000 { + /* nan */ + return x; + } + if sign && (ix | lx) != 0 { + /* x < 0 */ + return 0.0 / 0.0; + } + if ix == 0x7ff00000 { + return 0.0; + } + + if n == 0 { + return y0(x); + } + if n < 0 { + nm1 = -(n + 1); + sign = (n & 1) != 0; + } else { + nm1 = n - 1; + sign = false; + } + if nm1 == 0 { + if sign { + return -y1(x); + } else { + return y1(x); + } + } + + if ix >= 0x52d00000 { + /* x > 2**302 */ + /* (x >> n**2) + * Jn(x) = cos(x-(2n+1)*pi/4)*sqrt(2/x*pi) + * Yn(x) = sin(x-(2n+1)*pi/4)*sqrt(2/x*pi) + * Let s=sin(x), c=cos(x), + * xn=x-(2n+1)*pi/4, sqt2 = sqrt(2),then + * + * n sin(xn)*sqt2 cos(xn)*sqt2 + * ---------------------------------- + * 0 s-c c+s + * 1 -s-c -c+s + * 2 -s+c -c-s + * 3 s+c c-s + */ + temp = match nm1 & 3 { + 0 => -sin(x) - cos(x), + 1 => -sin(x) + cos(x), + 2 => sin(x) + cos(x), + 3 | _ => sin(x) - cos(x), + }; + b = INVSQRTPI * temp / sqrt(x); + } else { + a = y0(x); + b = y1(x); + /* quit if b is -inf */ + ib = get_high_word(b); + i = 0; + while i < nm1 && ib != 0xfff00000 { + i += 1; + temp = b; + b = (2.0 * (i as f64) / x) * b - a; + ib = get_high_word(b); + a = temp; + } + } + + if sign { + -b + } else { + b + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/jnf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/jnf.rs new file mode 100644 index 000000000..360f62e20 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/jnf.rs @@ -0,0 +1,259 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_jnf.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +use super::{fabsf, j0f, j1f, logf, y0f, y1f}; + +pub fn jnf(n: i32, mut x: f32) -> f32 { + let mut ix: u32; + let mut nm1: i32; + let mut sign: bool; + let mut i: i32; + let mut a: f32; + let mut b: f32; + let mut temp: f32; + + ix = x.to_bits(); + sign = (ix >> 31) != 0; + ix &= 0x7fffffff; + if ix > 0x7f800000 { + /* nan */ + return x; + } + + /* J(-n,x) = J(n,-x), use |n|-1 to avoid overflow in -n */ + if n == 0 { + return j0f(x); + } + if n < 0 { + nm1 = -(n + 1); + x = -x; + sign = !sign; + } else { + nm1 = n - 1; + } + if nm1 == 0 { + return j1f(x); + } + + sign &= (n & 1) != 0; /* even n: 0, odd n: signbit(x) */ + x = fabsf(x); + if ix == 0 || ix == 0x7f800000 { + /* if x is 0 or inf */ + b = 0.0; + } else if (nm1 as f32) < x { + /* Safe to use J(n+1,x)=2n/x *J(n,x)-J(n-1,x) */ + a = j0f(x); + b = j1f(x); + i = 0; + while i < nm1 { + i += 1; + temp = b; + b = b * (2.0 * (i as f32) / x) - a; + a = temp; + } + } else { + if ix < 0x35800000 { + /* x < 2**-20 */ + /* x is tiny, return the first Taylor expansion of J(n,x) + * J(n,x) = 1/n!*(x/2)^n - ... + */ + if nm1 > 8 { + /* underflow */ + nm1 = 8; + } + temp = 0.5 * x; + b = temp; + a = 1.0; + i = 2; + while i <= nm1 + 1 { + a *= i as f32; /* a = n! */ + b *= temp; /* b = (x/2)^n */ + i += 1; + } + b = b / a; + } else { + /* use backward recurrence */ + /* x x^2 x^2 + * J(n,x)/J(n-1,x) = ---- ------ ------ ..... + * 2n - 2(n+1) - 2(n+2) + * + * 1 1 1 + * (for large x) = ---- ------ ------ ..... + * 2n 2(n+1) 2(n+2) + * -- - ------ - ------ - + * x x x + * + * Let w = 2n/x and h=2/x, then the above quotient + * is equal to the continued fraction: + * 1 + * = ----------------------- + * 1 + * w - ----------------- + * 1 + * w+h - --------- + * w+2h - ... + * + * To determine how many terms needed, let + * Q(0) = w, Q(1) = w(w+h) - 1, + * Q(k) = (w+k*h)*Q(k-1) - Q(k-2), + * When Q(k) > 1e4 good for single + * When Q(k) > 1e9 good for double + * When Q(k) > 1e17 good for quadruple + */ + /* determine k */ + let mut t: f32; + let mut q0: f32; + let mut q1: f32; + let mut w: f32; + let h: f32; + let mut z: f32; + let mut tmp: f32; + let nf: f32; + let mut k: i32; + + nf = (nm1 as f32) + 1.0; + w = 2.0 * (nf as f32) / x; + h = 2.0 / x; + z = w + h; + q0 = w; + q1 = w * z - 1.0; + k = 1; + while q1 < 1.0e4 { + k += 1; + z += h; + tmp = z * q1 - q0; + q0 = q1; + q1 = tmp; + } + t = 0.0; + i = k; + while i >= 0 { + t = 1.0 / (2.0 * ((i as f32) + nf) / x - t); + i -= 1; + } + a = t; + b = 1.0; + /* estimate log((2/x)^n*n!) = n*log(2/x)+n*ln(n) + * Hence, if n*(log(2n/x)) > ... + * single 8.8722839355e+01 + * double 7.09782712893383973096e+02 + * long double 1.1356523406294143949491931077970765006170e+04 + * then recurrent value may overflow and the result is + * likely underflow to zero + */ + tmp = nf * logf(fabsf(w)); + if tmp < 88.721679688 { + i = nm1; + while i > 0 { + temp = b; + b = 2.0 * (i as f32) * b / x - a; + a = temp; + i -= 1; + } + } else { + i = nm1; + while i > 0 { + temp = b; + b = 2.0 * (i as f32) * b / x - a; + a = temp; + /* scale b to avoid spurious overflow */ + let x1p60 = f32::from_bits(0x5d800000); // 0x1p60 == 2^60 + if b > x1p60 { + a /= b; + t /= b; + b = 1.0; + } + i -= 1; + } + } + z = j0f(x); + w = j1f(x); + if fabsf(z) >= fabsf(w) { + b = t * z / b; + } else { + b = t * w / a; + } + } + } + + if sign { + -b + } else { + b + } +} + +pub fn ynf(n: i32, x: f32) -> f32 { + let mut ix: u32; + let mut ib: u32; + let nm1: i32; + let mut sign: bool; + let mut i: i32; + let mut a: f32; + let mut b: f32; + let mut temp: f32; + + ix = x.to_bits(); + sign = (ix >> 31) != 0; + ix &= 0x7fffffff; + if ix > 0x7f800000 { + /* nan */ + return x; + } + if sign && ix != 0 { + /* x < 0 */ + return 0.0 / 0.0; + } + if ix == 0x7f800000 { + return 0.0; + } + + if n == 0 { + return y0f(x); + } + if n < 0 { + nm1 = -(n + 1); + sign = (n & 1) != 0; + } else { + nm1 = n - 1; + sign = false; + } + if nm1 == 0 { + if sign { + return -y1f(x); + } else { + return y1f(x); + } + } + + a = y0f(x); + b = y1f(x); + /* quit if b is -inf */ + ib = b.to_bits(); + i = 0; + while i < nm1 && ib != 0xff800000 { + i += 1; + temp = b; + b = (2.0 * (i as f32) / x) * b - a; + ib = b.to_bits(); + a = temp; + } + + if sign { + -b + } else { + b + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/k_cos.rs b/crux-mir/lib/compiler_builtins/libm/src/math/k_cos.rs new file mode 100644 index 000000000..49b2fc64d --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/k_cos.rs @@ -0,0 +1,62 @@ +// origin: FreeBSD /usr/src/lib/msun/src/k_cos.c +// +// ==================================================== +// Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. +// +// Developed at SunSoft, a Sun Microsystems, Inc. business. +// Permission to use, copy, modify, and distribute this +// software is freely granted, provided that this notice +// is preserved. +// ==================================================== + +const C1: f64 = 4.16666666666666019037e-02; /* 0x3FA55555, 0x5555554C */ +const C2: f64 = -1.38888888888741095749e-03; /* 0xBF56C16C, 0x16C15177 */ +const C3: f64 = 2.48015872894767294178e-05; /* 0x3EFA01A0, 0x19CB1590 */ +const C4: f64 = -2.75573143513906633035e-07; /* 0xBE927E4F, 0x809C52AD */ +const C5: f64 = 2.08757232129817482790e-09; /* 0x3E21EE9E, 0xBDB4B1C4 */ +const C6: f64 = -1.13596475577881948265e-11; /* 0xBDA8FAE9, 0xBE8838D4 */ + +// kernel cos function on [-pi/4, pi/4], pi/4 ~ 0.785398164 +// Input x is assumed to be bounded by ~pi/4 in magnitude. +// Input y is the tail of x. +// +// Algorithm +// 1. Since cos(-x) = cos(x), we need only to consider positive x. +// 2. if x < 2^-27 (hx<0x3e400000 0), return 1 with inexact if x!=0. +// 3. cos(x) is approximated by a polynomial of degree 14 on +// [0,pi/4] +// 4 14 +// cos(x) ~ 1 - x*x/2 + C1*x + ... + C6*x +// where the remez error is +// +// | 2 4 6 8 10 12 14 | -58 +// |cos(x)-(1-.5*x +C1*x +C2*x +C3*x +C4*x +C5*x +C6*x )| <= 2 +// | | +// +// 4 6 8 10 12 14 +// 4. let r = C1*x +C2*x +C3*x +C4*x +C5*x +C6*x , then +// cos(x) ~ 1 - x*x/2 + r +// since cos(x+y) ~ cos(x) - sin(x)*y +// ~ cos(x) - x*y, +// a correction term is necessary in cos(x) and hence +// cos(x+y) = 1 - (x*x/2 - (r - x*y)) +// For better accuracy, rearrange to +// cos(x+y) ~ w + (tmp + (r-x*y)) +// where w = 1 - x*x/2 and tmp is a tiny correction term +// (1 - x*x/2 == w + tmp exactly in infinite precision). +// The exactness of w + tmp in infinite precision depends on w +// and tmp having the same precision as x. If they have extra +// precision due to compiler bugs, then the extra precision is +// only good provided it is retained in all terms of the final +// expression for cos(). Retention happens in all cases tested +// under FreeBSD, so don't pessimize things by forcibly clipping +// any extra precision in w. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub(crate) fn k_cos(x: f64, y: f64) -> f64 { + let z = x * x; + let w = z * z; + let r = z * (C1 + z * (C2 + z * C3)) + w * w * (C4 + z * (C5 + z * C6)); + let hz = 0.5 * z; + let w = 1.0 - hz; + w + (((1.0 - w) - hz) + (z * r - x * y)) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/k_cosf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/k_cosf.rs new file mode 100644 index 000000000..e99f2348c --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/k_cosf.rs @@ -0,0 +1,29 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/k_cosf.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + * Debugged and optimized by Bruce D. Evans. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +/* |cos(x) - c(x)| < 2**-34.1 (~[-5.37e-11, 5.295e-11]). */ +const C0: f64 = -0.499999997251031003120; /* -0x1ffffffd0c5e81.0p-54 */ +const C1: f64 = 0.0416666233237390631894; /* 0x155553e1053a42.0p-57 */ +const C2: f64 = -0.00138867637746099294692; /* -0x16c087e80f1e27.0p-62 */ +const C3: f64 = 0.0000243904487962774090654; /* 0x199342e0ee5069.0p-68 */ + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub(crate) fn k_cosf(x: f64) -> f32 { + let z = x * x; + let w = z * z; + let r = C2 + z * C3; + (((1.0 + z * C0) + w * C1) + (w * z) * r) as f32 +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/k_expo2.rs b/crux-mir/lib/compiler_builtins/libm/src/math/k_expo2.rs new file mode 100644 index 000000000..7345075f3 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/k_expo2.rs @@ -0,0 +1,14 @@ +use super::exp; + +/* k is such that k*ln2 has minimal relative error and x - kln2 > log(FLT_MIN) */ +const K: i32 = 2043; + +/* expf(x)/2 for x >= log(FLT_MAX), slightly better than 0.5f*expf(x/2)*expf(x/2) */ +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub(crate) fn k_expo2(x: f64) -> f64 { + let k_ln2 = f64::from_bits(0x40962066151add8b); + /* note that k is odd and scale*scale overflows */ + let scale = f64::from_bits(((((0x3ff + K / 2) as u32) << 20) as u64) << 32); + /* exp(x - k ln2) * 2**(k-1) */ + exp(x - k_ln2) * scale * scale +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/k_expo2f.rs b/crux-mir/lib/compiler_builtins/libm/src/math/k_expo2f.rs new file mode 100644 index 000000000..fbd7b27d5 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/k_expo2f.rs @@ -0,0 +1,14 @@ +use super::expf; + +/* k is such that k*ln2 has minimal relative error and x - kln2 > log(FLT_MIN) */ +const K: i32 = 235; + +/* expf(x)/2 for x >= log(FLT_MAX), slightly better than 0.5f*expf(x/2)*expf(x/2) */ +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub(crate) fn k_expo2f(x: f32) -> f32 { + let k_ln2 = f32::from_bits(0x4322e3bc); + /* note that k is odd and scale*scale overflows */ + let scale = f32::from_bits(((0x7f + K / 2) as u32) << 23); + /* exp(x - k ln2) * 2**(k-1) */ + expf(x - k_ln2) * scale * scale +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/k_sin.rs b/crux-mir/lib/compiler_builtins/libm/src/math/k_sin.rs new file mode 100644 index 000000000..9dd96c944 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/k_sin.rs @@ -0,0 +1,57 @@ +// origin: FreeBSD /usr/src/lib/msun/src/k_sin.c +// +// ==================================================== +// Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. +// +// Developed at SunSoft, a Sun Microsystems, Inc. business. +// Permission to use, copy, modify, and distribute this +// software is freely granted, provided that this notice +// is preserved. +// ==================================================== + +const S1: f64 = -1.66666666666666324348e-01; /* 0xBFC55555, 0x55555549 */ +const S2: f64 = 8.33333333332248946124e-03; /* 0x3F811111, 0x1110F8A6 */ +const S3: f64 = -1.98412698298579493134e-04; /* 0xBF2A01A0, 0x19C161D5 */ +const S4: f64 = 2.75573137070700676789e-06; /* 0x3EC71DE3, 0x57B1FE7D */ +const S5: f64 = -2.50507602534068634195e-08; /* 0xBE5AE5E6, 0x8A2B9CEB */ +const S6: f64 = 1.58969099521155010221e-10; /* 0x3DE5D93A, 0x5ACFD57C */ + +// kernel sin function on ~[-pi/4, pi/4] (except on -0), pi/4 ~ 0.7854 +// Input x is assumed to be bounded by ~pi/4 in magnitude. +// Input y is the tail of x. +// Input iy indicates whether y is 0. (if iy=0, y assume to be 0). +// +// Algorithm +// 1. Since sin(-x) = -sin(x), we need only to consider positive x. +// 2. Callers must return sin(-0) = -0 without calling here since our +// odd polynomial is not evaluated in a way that preserves -0. +// Callers may do the optimization sin(x) ~ x for tiny x. +// 3. sin(x) is approximated by a polynomial of degree 13 on +// [0,pi/4] +// 3 13 +// sin(x) ~ x + S1*x + ... + S6*x +// where +// +// |sin(x) 2 4 6 8 10 12 | -58 +// |----- - (1+S1*x +S2*x +S3*x +S4*x +S5*x +S6*x )| <= 2 +// | x | +// +// 4. sin(x+y) = sin(x) + sin'(x')*y +// ~ sin(x) + (1-x*x/2)*y +// For better accuracy, let +// 3 2 2 2 2 +// r = x *(S2+x *(S3+x *(S4+x *(S5+x *S6)))) +// then 3 2 +// sin(x) = x + (S1*x + (x *(r-y/2)+y)) +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub(crate) fn k_sin(x: f64, y: f64, iy: i32) -> f64 { + let z = x * x; + let w = z * z; + let r = S2 + z * (S3 + z * S4) + z * w * (S5 + z * S6); + let v = z * x; + if iy == 0 { + x + v * (S1 + z * r) + } else { + x - ((z * (0.5 * y - v * r) - y) - v * S1) + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/k_sinf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/k_sinf.rs new file mode 100644 index 000000000..88d10caba --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/k_sinf.rs @@ -0,0 +1,30 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/k_sinf.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + * Optimized by Bruce D. Evans. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +/* |sin(x)/x - s(x)| < 2**-37.5 (~[-4.89e-12, 4.824e-12]). */ +const S1: f64 = -0.166666666416265235595; /* -0x15555554cbac77.0p-55 */ +const S2: f64 = 0.0083333293858894631756; /* 0x111110896efbb2.0p-59 */ +const S3: f64 = -0.000198393348360966317347; /* -0x1a00f9e2cae774.0p-65 */ +const S4: f64 = 0.0000027183114939898219064; /* 0x16cd878c3b46a7.0p-71 */ + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub(crate) fn k_sinf(x: f64) -> f32 { + let z = x * x; + let w = z * z; + let r = S3 + z * S4; + let s = z * x; + ((x + s * (S1 + z * S2)) + s * w * r) as f32 +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/k_tan.rs b/crux-mir/lib/compiler_builtins/libm/src/math/k_tan.rs new file mode 100644 index 000000000..d177010bb --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/k_tan.rs @@ -0,0 +1,105 @@ +// origin: FreeBSD /usr/src/lib/msun/src/k_tan.c */ +// +// ==================================================== +// Copyright 2004 Sun Microsystems, Inc. All Rights Reserved. +// +// Permission to use, copy, modify, and distribute this +// software is freely granted, provided that this notice +// is preserved. +// ==================================================== + +// kernel tan function on ~[-pi/4, pi/4] (except on -0), pi/4 ~ 0.7854 +// Input x is assumed to be bounded by ~pi/4 in magnitude. +// Input y is the tail of x. +// Input odd indicates whether tan (if odd = 0) or -1/tan (if odd = 1) is returned. +// +// Algorithm +// 1. Since tan(-x) = -tan(x), we need only to consider positive x. +// 2. Callers must return tan(-0) = -0 without calling here since our +// odd polynomial is not evaluated in a way that preserves -0. +// Callers may do the optimization tan(x) ~ x for tiny x. +// 3. tan(x) is approximated by a odd polynomial of degree 27 on +// [0,0.67434] +// 3 27 +// tan(x) ~ x + T1*x + ... + T13*x +// where +// +// |tan(x) 2 4 26 | -59.2 +// |----- - (1+T1*x +T2*x +.... +T13*x )| <= 2 +// | x | +// +// Note: tan(x+y) = tan(x) + tan'(x)*y +// ~ tan(x) + (1+x*x)*y +// Therefore, for better accuracy in computing tan(x+y), let +// 3 2 2 2 2 +// r = x *(T2+x *(T3+x *(...+x *(T12+x *T13)))) +// then +// 3 2 +// tan(x+y) = x + (T1*x + (x *(r+y)+y)) +// +// 4. For x in [0.67434,pi/4], let y = pi/4 - x, then +// tan(x) = tan(pi/4-y) = (1-tan(y))/(1+tan(y)) +// = 1 - 2*(tan(y) - (tan(y)^2)/(1+tan(y))) +static T: [f64; 13] = [ + 3.33333333333334091986e-01, /* 3FD55555, 55555563 */ + 1.33333333333201242699e-01, /* 3FC11111, 1110FE7A */ + 5.39682539762260521377e-02, /* 3FABA1BA, 1BB341FE */ + 2.18694882948595424599e-02, /* 3F9664F4, 8406D637 */ + 8.86323982359930005737e-03, /* 3F8226E3, E96E8493 */ + 3.59207910759131235356e-03, /* 3F6D6D22, C9560328 */ + 1.45620945432529025516e-03, /* 3F57DBC8, FEE08315 */ + 5.88041240820264096874e-04, /* 3F4344D8, F2F26501 */ + 2.46463134818469906812e-04, /* 3F3026F7, 1A8D1068 */ + 7.81794442939557092300e-05, /* 3F147E88, A03792A6 */ + 7.14072491382608190305e-05, /* 3F12B80F, 32F0A7E9 */ + -1.85586374855275456654e-05, /* BEF375CB, DB605373 */ + 2.59073051863633712884e-05, /* 3EFB2A70, 74BF7AD4 */ +]; +const PIO4: f64 = 7.85398163397448278999e-01; /* 3FE921FB, 54442D18 */ +const PIO4_LO: f64 = 3.06161699786838301793e-17; /* 3C81A626, 33145C07 */ + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub(crate) fn k_tan(mut x: f64, mut y: f64, odd: i32) -> f64 { + let hx = (f64::to_bits(x) >> 32) as u32; + let big = (hx & 0x7fffffff) >= 0x3FE59428; /* |x| >= 0.6744 */ + if big { + let sign = hx >> 31; + if sign != 0 { + x = -x; + y = -y; + } + x = (PIO4 - x) + (PIO4_LO - y); + y = 0.0; + } + let z = x * x; + let w = z * z; + /* + * Break x^5*(T[1]+x^2*T[2]+...) into + * x^5(T[1]+x^4*T[3]+...+x^20*T[11]) + + * x^5(x^2*(T[2]+x^4*T[4]+...+x^22*[T12])) + */ + let r = T[1] + w * (T[3] + w * (T[5] + w * (T[7] + w * (T[9] + w * T[11])))); + let v = z * (T[2] + w * (T[4] + w * (T[6] + w * (T[8] + w * (T[10] + w * T[12]))))); + let s = z * x; + let r = y + z * (s * (r + v) + y) + s * T[0]; + let w = x + r; + if big { + let sign = hx >> 31; + let s = 1.0 - 2.0 * odd as f64; + let v = s - 2.0 * (x + (r - w * w / (w + s))); + return if sign != 0 { -v } else { v }; + } + if odd == 0 { + return w; + } + /* -1.0/(x+r) has up to 2ulp error, so compute it accurately */ + let w0 = zero_low_word(w); + let v = r - (w0 - x); /* w0+v = r+x */ + let a = -1.0 / w; + let a0 = zero_low_word(a); + a0 + a * (1.0 + a0 * w0 + a0 * v) +} + +fn zero_low_word(x: f64) -> f64 { + f64::from_bits(f64::to_bits(x) & 0xFFFF_FFFF_0000_0000) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/k_tanf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/k_tanf.rs new file mode 100644 index 000000000..af8db539d --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/k_tanf.rs @@ -0,0 +1,46 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/k_tan.c */ +/* + * ==================================================== + * Copyright 2004 Sun Microsystems, Inc. All Rights Reserved. + * + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +/* |tan(x)/x - t(x)| < 2**-25.5 (~[-2e-08, 2e-08]). */ +const T: [f64; 6] = [ + 0.333331395030791399758, /* 0x15554d3418c99f.0p-54 */ + 0.133392002712976742718, /* 0x1112fd38999f72.0p-55 */ + 0.0533812378445670393523, /* 0x1b54c91d865afe.0p-57 */ + 0.0245283181166547278873, /* 0x191df3908c33ce.0p-58 */ + 0.00297435743359967304927, /* 0x185dadfcecf44e.0p-61 */ + 0.00946564784943673166728, /* 0x1362b9bf971bcd.0p-59 */ +]; + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub(crate) fn k_tanf(x: f64, odd: bool) -> f32 { + let z = x * x; + /* + * Split up the polynomial into small independent terms to give + * opportunities for parallel evaluation. The chosen splitting is + * micro-optimized for Athlons (XP, X64). It costs 2 multiplications + * relative to Horner's method on sequential machines. + * + * We add the small terms from lowest degree up for efficiency on + * non-sequential machines (the lowest degree terms tend to be ready + * earlier). Apart from this, we don't care about order of + * operations, and don't need to to care since we have precision to + * spare. However, the chosen splitting is good for accuracy too, + * and would give results as accurate as Horner's method if the + * small terms were added from highest degree down. + */ + let mut r = T[4] + z * T[5]; + let t = T[2] + z * T[3]; + let w = z * z; + let s = z * x; + let u = T[0] + z * T[1]; + r = (x + s * u) + (s * w) * (t + w * r); + (if odd { -1. / r } else { r }) as f32 +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/ldexp.rs b/crux-mir/lib/compiler_builtins/libm/src/math/ldexp.rs new file mode 100644 index 000000000..e46242e55 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/ldexp.rs @@ -0,0 +1,4 @@ +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn ldexp(x: f64, n: i32) -> f64 { + super::scalbn(x, n) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/ldexpf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/ldexpf.rs new file mode 100644 index 000000000..95b27fc49 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/ldexpf.rs @@ -0,0 +1,4 @@ +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn ldexpf(x: f32, n: i32) -> f32 { + super::scalbnf(x, n) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/lgamma.rs b/crux-mir/lib/compiler_builtins/libm/src/math/lgamma.rs new file mode 100644 index 000000000..a08bc5b64 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/lgamma.rs @@ -0,0 +1,6 @@ +use super::lgamma_r; + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn lgamma(x: f64) -> f64 { + lgamma_r(x).0 +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/lgamma_r.rs b/crux-mir/lib/compiler_builtins/libm/src/math/lgamma_r.rs new file mode 100644 index 000000000..b26177e6e --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/lgamma_r.rs @@ -0,0 +1,320 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_lgamma_r.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunSoft, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + * + */ +/* lgamma_r(x, signgamp) + * Reentrant version of the logarithm of the Gamma function + * with user provide pointer for the sign of Gamma(x). + * + * Method: + * 1. Argument Reduction for 0 < x <= 8 + * Since gamma(1+s)=s*gamma(s), for x in [0,8], we may + * reduce x to a number in [1.5,2.5] by + * lgamma(1+s) = log(s) + lgamma(s) + * for example, + * lgamma(7.3) = log(6.3) + lgamma(6.3) + * = log(6.3*5.3) + lgamma(5.3) + * = log(6.3*5.3*4.3*3.3*2.3) + lgamma(2.3) + * 2. Polynomial approximation of lgamma around its + * minimun ymin=1.461632144968362245 to maintain monotonicity. + * On [ymin-0.23, ymin+0.27] (i.e., [1.23164,1.73163]), use + * Let z = x-ymin; + * lgamma(x) = -1.214862905358496078218 + z^2*poly(z) + * where + * poly(z) is a 14 degree polynomial. + * 2. Rational approximation in the primary interval [2,3] + * We use the following approximation: + * s = x-2.0; + * lgamma(x) = 0.5*s + s*P(s)/Q(s) + * with accuracy + * |P/Q - (lgamma(x)-0.5s)| < 2**-61.71 + * Our algorithms are based on the following observation + * + * zeta(2)-1 2 zeta(3)-1 3 + * lgamma(2+s) = s*(1-Euler) + --------- * s - --------- * s + ... + * 2 3 + * + * where Euler = 0.5771... is the Euler constant, which is very + * close to 0.5. + * + * 3. For x>=8, we have + * lgamma(x)~(x-0.5)log(x)-x+0.5*log(2pi)+1/(12x)-1/(360x**3)+.... + * (better formula: + * lgamma(x)~(x-0.5)*(log(x)-1)-.5*(log(2pi)-1) + ...) + * Let z = 1/x, then we approximation + * f(z) = lgamma(x) - (x-0.5)(log(x)-1) + * by + * 3 5 11 + * w = w0 + w1*z + w2*z + w3*z + ... + w6*z + * where + * |w - f(z)| < 2**-58.74 + * + * 4. For negative x, since (G is gamma function) + * -x*G(-x)*G(x) = PI/sin(PI*x), + * we have + * G(x) = PI/(sin(PI*x)*(-x)*G(-x)) + * since G(-x) is positive, sign(G(x)) = sign(sin(PI*x)) for x<0 + * Hence, for x<0, signgam = sign(sin(PI*x)) and + * lgamma(x) = log(|Gamma(x)|) + * = log(PI/(|x*sin(PI*x)|)) - lgamma(-x); + * Note: one should avoid compute PI*(-x) directly in the + * computation of sin(PI*(-x)). + * + * 5. Special Cases + * lgamma(2+s) ~ s*(1-Euler) for tiny s + * lgamma(1) = lgamma(2) = 0 + * lgamma(x) ~ -log(|x|) for tiny x + * lgamma(0) = lgamma(neg.integer) = inf and raise divide-by-zero + * lgamma(inf) = inf + * lgamma(-inf) = inf (bug for bug compatible with C99!?) + * + */ + +use super::{floor, k_cos, k_sin, log}; + +const PI: f64 = 3.14159265358979311600e+00; /* 0x400921FB, 0x54442D18 */ +const A0: f64 = 7.72156649015328655494e-02; /* 0x3FB3C467, 0xE37DB0C8 */ +const A1: f64 = 3.22467033424113591611e-01; /* 0x3FD4A34C, 0xC4A60FAD */ +const A2: f64 = 6.73523010531292681824e-02; /* 0x3FB13E00, 0x1A5562A7 */ +const A3: f64 = 2.05808084325167332806e-02; /* 0x3F951322, 0xAC92547B */ +const A4: f64 = 7.38555086081402883957e-03; /* 0x3F7E404F, 0xB68FEFE8 */ +const A5: f64 = 2.89051383673415629091e-03; /* 0x3F67ADD8, 0xCCB7926B */ +const A6: f64 = 1.19270763183362067845e-03; /* 0x3F538A94, 0x116F3F5D */ +const A7: f64 = 5.10069792153511336608e-04; /* 0x3F40B6C6, 0x89B99C00 */ +const A8: f64 = 2.20862790713908385557e-04; /* 0x3F2CF2EC, 0xED10E54D */ +const A9: f64 = 1.08011567247583939954e-04; /* 0x3F1C5088, 0x987DFB07 */ +const A10: f64 = 2.52144565451257326939e-05; /* 0x3EFA7074, 0x428CFA52 */ +const A11: f64 = 4.48640949618915160150e-05; /* 0x3F07858E, 0x90A45837 */ +const TC: f64 = 1.46163214496836224576e+00; /* 0x3FF762D8, 0x6356BE3F */ +const TF: f64 = -1.21486290535849611461e-01; /* 0xBFBF19B9, 0xBCC38A42 */ +/* tt = -(tail of TF) */ +const TT: f64 = -3.63867699703950536541e-18; /* 0xBC50C7CA, 0xA48A971F */ +const T0: f64 = 4.83836122723810047042e-01; /* 0x3FDEF72B, 0xC8EE38A2 */ +const T1: f64 = -1.47587722994593911752e-01; /* 0xBFC2E427, 0x8DC6C509 */ +const T2: f64 = 6.46249402391333854778e-02; /* 0x3FB08B42, 0x94D5419B */ +const T3: f64 = -3.27885410759859649565e-02; /* 0xBFA0C9A8, 0xDF35B713 */ +const T4: f64 = 1.79706750811820387126e-02; /* 0x3F9266E7, 0x970AF9EC */ +const T5: f64 = -1.03142241298341437450e-02; /* 0xBF851F9F, 0xBA91EC6A */ +const T6: f64 = 6.10053870246291332635e-03; /* 0x3F78FCE0, 0xE370E344 */ +const T7: f64 = -3.68452016781138256760e-03; /* 0xBF6E2EFF, 0xB3E914D7 */ +const T8: f64 = 2.25964780900612472250e-03; /* 0x3F6282D3, 0x2E15C915 */ +const T9: f64 = -1.40346469989232843813e-03; /* 0xBF56FE8E, 0xBF2D1AF1 */ +const T10: f64 = 8.81081882437654011382e-04; /* 0x3F4CDF0C, 0xEF61A8E9 */ +const T11: f64 = -5.38595305356740546715e-04; /* 0xBF41A610, 0x9C73E0EC */ +const T12: f64 = 3.15632070903625950361e-04; /* 0x3F34AF6D, 0x6C0EBBF7 */ +const T13: f64 = -3.12754168375120860518e-04; /* 0xBF347F24, 0xECC38C38 */ +const T14: f64 = 3.35529192635519073543e-04; /* 0x3F35FD3E, 0xE8C2D3F4 */ +const U0: f64 = -7.72156649015328655494e-02; /* 0xBFB3C467, 0xE37DB0C8 */ +const U1: f64 = 6.32827064025093366517e-01; /* 0x3FE4401E, 0x8B005DFF */ +const U2: f64 = 1.45492250137234768737e+00; /* 0x3FF7475C, 0xD119BD6F */ +const U3: f64 = 9.77717527963372745603e-01; /* 0x3FEF4976, 0x44EA8450 */ +const U4: f64 = 2.28963728064692451092e-01; /* 0x3FCD4EAE, 0xF6010924 */ +const U5: f64 = 1.33810918536787660377e-02; /* 0x3F8B678B, 0xBF2BAB09 */ +const V1: f64 = 2.45597793713041134822e+00; /* 0x4003A5D7, 0xC2BD619C */ +const V2: f64 = 2.12848976379893395361e+00; /* 0x40010725, 0xA42B18F5 */ +const V3: f64 = 7.69285150456672783825e-01; /* 0x3FE89DFB, 0xE45050AF */ +const V4: f64 = 1.04222645593369134254e-01; /* 0x3FBAAE55, 0xD6537C88 */ +const V5: f64 = 3.21709242282423911810e-03; /* 0x3F6A5ABB, 0x57D0CF61 */ +const S0: f64 = -7.72156649015328655494e-02; /* 0xBFB3C467, 0xE37DB0C8 */ +const S1: f64 = 2.14982415960608852501e-01; /* 0x3FCB848B, 0x36E20878 */ +const S2: f64 = 3.25778796408930981787e-01; /* 0x3FD4D98F, 0x4F139F59 */ +const S3: f64 = 1.46350472652464452805e-01; /* 0x3FC2BB9C, 0xBEE5F2F7 */ +const S4: f64 = 2.66422703033638609560e-02; /* 0x3F9B481C, 0x7E939961 */ +const S5: f64 = 1.84028451407337715652e-03; /* 0x3F5E26B6, 0x7368F239 */ +const S6: f64 = 3.19475326584100867617e-05; /* 0x3F00BFEC, 0xDD17E945 */ +const R1: f64 = 1.39200533467621045958e+00; /* 0x3FF645A7, 0x62C4AB74 */ +const R2: f64 = 7.21935547567138069525e-01; /* 0x3FE71A18, 0x93D3DCDC */ +const R3: f64 = 1.71933865632803078993e-01; /* 0x3FC601ED, 0xCCFBDF27 */ +const R4: f64 = 1.86459191715652901344e-02; /* 0x3F9317EA, 0x742ED475 */ +const R5: f64 = 7.77942496381893596434e-04; /* 0x3F497DDA, 0xCA41A95B */ +const R6: f64 = 7.32668430744625636189e-06; /* 0x3EDEBAF7, 0xA5B38140 */ +const W0: f64 = 4.18938533204672725052e-01; /* 0x3FDACFE3, 0x90C97D69 */ +const W1: f64 = 8.33333333333329678849e-02; /* 0x3FB55555, 0x5555553B */ +const W2: f64 = -2.77777777728775536470e-03; /* 0xBF66C16C, 0x16B02E5C */ +const W3: f64 = 7.93650558643019558500e-04; /* 0x3F4A019F, 0x98CF38B6 */ +const W4: f64 = -5.95187557450339963135e-04; /* 0xBF4380CB, 0x8C0FE741 */ +const W5: f64 = 8.36339918996282139126e-04; /* 0x3F4B67BA, 0x4CDAD5D1 */ +const W6: f64 = -1.63092934096575273989e-03; /* 0xBF5AB89D, 0x0B9E43E4 */ + +/* sin(PI*x) assuming x > 2^-100, if sin(PI*x)==0 the sign is arbitrary */ +fn sin_pi(mut x: f64) -> f64 { + let mut n: i32; + + /* spurious inexact if odd int */ + x = 2.0 * (x * 0.5 - floor(x * 0.5)); /* x mod 2.0 */ + + n = (x * 4.0) as i32; + n = div!(n + 1, 2); + x -= (n as f64) * 0.5; + x *= PI; + + match n { + 1 => k_cos(x, 0.0), + 2 => k_sin(-x, 0.0, 0), + 3 => -k_cos(x, 0.0), + 0 | _ => k_sin(x, 0.0, 0), + } +} + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn lgamma_r(mut x: f64) -> (f64, i32) { + let u: u64 = x.to_bits(); + let mut t: f64; + let y: f64; + let mut z: f64; + let nadj: f64; + let p: f64; + let p1: f64; + let p2: f64; + let p3: f64; + let q: f64; + let mut r: f64; + let w: f64; + let ix: u32; + let sign: bool; + let i: i32; + let mut signgam: i32; + + /* purge off +-inf, NaN, +-0, tiny and negative arguments */ + signgam = 1; + sign = (u >> 63) != 0; + ix = ((u >> 32) as u32) & 0x7fffffff; + if ix >= 0x7ff00000 { + return (x * x, signgam); + } + if ix < (0x3ff - 70) << 20 { + /* |x|<2**-70, return -log(|x|) */ + if sign { + x = -x; + signgam = -1; + } + return (-log(x), signgam); + } + if sign { + x = -x; + t = sin_pi(x); + if t == 0.0 { + /* -integer */ + return (1.0 / (x - x), signgam); + } + if t > 0.0 { + signgam = -1; + } else { + t = -t; + } + nadj = log(PI / (t * x)); + } else { + nadj = 0.0; + } + + /* purge off 1 and 2 */ + if (ix == 0x3ff00000 || ix == 0x40000000) && (u & 0xffffffff) == 0 { + r = 0.0; + } + /* for x < 2.0 */ + else if ix < 0x40000000 { + if ix <= 0x3feccccc { + /* lgamma(x) = lgamma(x+1)-log(x) */ + r = -log(x); + if ix >= 0x3FE76944 { + y = 1.0 - x; + i = 0; + } else if ix >= 0x3FCDA661 { + y = x - (TC - 1.0); + i = 1; + } else { + y = x; + i = 2; + } + } else { + r = 0.0; + if ix >= 0x3FFBB4C3 { + /* [1.7316,2] */ + y = 2.0 - x; + i = 0; + } else if ix >= 0x3FF3B4C4 { + /* [1.23,1.73] */ + y = x - TC; + i = 1; + } else { + y = x - 1.0; + i = 2; + } + } + match i { + 0 => { + z = y * y; + p1 = A0 + z * (A2 + z * (A4 + z * (A6 + z * (A8 + z * A10)))); + p2 = z * (A1 + z * (A3 + z * (A5 + z * (A7 + z * (A9 + z * A11))))); + p = y * p1 + p2; + r += p - 0.5 * y; + } + 1 => { + z = y * y; + w = z * y; + p1 = T0 + w * (T3 + w * (T6 + w * (T9 + w * T12))); /* parallel comp */ + p2 = T1 + w * (T4 + w * (T7 + w * (T10 + w * T13))); + p3 = T2 + w * (T5 + w * (T8 + w * (T11 + w * T14))); + p = z * p1 - (TT - w * (p2 + y * p3)); + r += TF + p; + } + 2 => { + p1 = y * (U0 + y * (U1 + y * (U2 + y * (U3 + y * (U4 + y * U5))))); + p2 = 1.0 + y * (V1 + y * (V2 + y * (V3 + y * (V4 + y * V5)))); + r += -0.5 * y + p1 / p2; + } + #[cfg(debug_assertions)] + _ => unreachable!(), + #[cfg(not(debug_assertions))] + _ => {} + } + } else if ix < 0x40200000 { + /* x < 8.0 */ + i = x as i32; + y = x - (i as f64); + p = y * (S0 + y * (S1 + y * (S2 + y * (S3 + y * (S4 + y * (S5 + y * S6)))))); + q = 1.0 + y * (R1 + y * (R2 + y * (R3 + y * (R4 + y * (R5 + y * R6))))); + r = 0.5 * y + p / q; + z = 1.0; /* lgamma(1+s) = log(s) + lgamma(s) */ + // TODO: In C, this was implemented using switch jumps with fallthrough. + // Does this implementation have performance problems? + if i >= 7 { + z *= y + 6.0; + } + if i >= 6 { + z *= y + 5.0; + } + if i >= 5 { + z *= y + 4.0; + } + if i >= 4 { + z *= y + 3.0; + } + if i >= 3 { + z *= y + 2.0; + r += log(z); + } + } else if ix < 0x43900000 { + /* 8.0 <= x < 2**58 */ + t = log(x); + z = 1.0 / x; + y = z * z; + w = W0 + z * (W1 + y * (W2 + y * (W3 + y * (W4 + y * (W5 + y * W6))))); + r = (x - 0.5) * (t - 1.0) + w; + } else { + /* 2**58 <= x <= inf */ + r = x * (log(x) - 1.0); + } + if sign { + r = nadj - r; + } + return (r, signgam); +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/lgammaf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/lgammaf.rs new file mode 100644 index 000000000..a9c2da75b --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/lgammaf.rs @@ -0,0 +1,6 @@ +use super::lgammaf_r; + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn lgammaf(x: f32) -> f32 { + lgammaf_r(x).0 +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/lgammaf_r.rs b/crux-mir/lib/compiler_builtins/libm/src/math/lgammaf_r.rs new file mode 100644 index 000000000..723c90daf --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/lgammaf_r.rs @@ -0,0 +1,255 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_lgammaf_r.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +use super::{floorf, k_cosf, k_sinf, logf}; + +const PI: f32 = 3.1415927410e+00; /* 0x40490fdb */ +const A0: f32 = 7.7215664089e-02; /* 0x3d9e233f */ +const A1: f32 = 3.2246702909e-01; /* 0x3ea51a66 */ +const A2: f32 = 6.7352302372e-02; /* 0x3d89f001 */ +const A3: f32 = 2.0580807701e-02; /* 0x3ca89915 */ +const A4: f32 = 7.3855509982e-03; /* 0x3bf2027e */ +const A5: f32 = 2.8905137442e-03; /* 0x3b3d6ec6 */ +const A6: f32 = 1.1927076848e-03; /* 0x3a9c54a1 */ +const A7: f32 = 5.1006977446e-04; /* 0x3a05b634 */ +const A8: f32 = 2.2086278477e-04; /* 0x39679767 */ +const A9: f32 = 1.0801156895e-04; /* 0x38e28445 */ +const A10: f32 = 2.5214456400e-05; /* 0x37d383a2 */ +const A11: f32 = 4.4864096708e-05; /* 0x383c2c75 */ +const TC: f32 = 1.4616321325e+00; /* 0x3fbb16c3 */ +const TF: f32 = -1.2148628384e-01; /* 0xbdf8cdcd */ +/* TT = -(tail of TF) */ +const TT: f32 = 6.6971006518e-09; /* 0x31e61c52 */ +const T0: f32 = 4.8383611441e-01; /* 0x3ef7b95e */ +const T1: f32 = -1.4758771658e-01; /* 0xbe17213c */ +const T2: f32 = 6.4624942839e-02; /* 0x3d845a15 */ +const T3: f32 = -3.2788541168e-02; /* 0xbd064d47 */ +const T4: f32 = 1.7970675603e-02; /* 0x3c93373d */ +const T5: f32 = -1.0314224288e-02; /* 0xbc28fcfe */ +const T6: f32 = 6.1005386524e-03; /* 0x3bc7e707 */ +const T7: f32 = -3.6845202558e-03; /* 0xbb7177fe */ +const T8: f32 = 2.2596477065e-03; /* 0x3b141699 */ +const T9: f32 = -1.4034647029e-03; /* 0xbab7f476 */ +const T10: f32 = 8.8108185446e-04; /* 0x3a66f867 */ +const T11: f32 = -5.3859531181e-04; /* 0xba0d3085 */ +const T12: f32 = 3.1563205994e-04; /* 0x39a57b6b */ +const T13: f32 = -3.1275415677e-04; /* 0xb9a3f927 */ +const T14: f32 = 3.3552918467e-04; /* 0x39afe9f7 */ +const U0: f32 = -7.7215664089e-02; /* 0xbd9e233f */ +const U1: f32 = 6.3282704353e-01; /* 0x3f2200f4 */ +const U2: f32 = 1.4549225569e+00; /* 0x3fba3ae7 */ +const U3: f32 = 9.7771751881e-01; /* 0x3f7a4bb2 */ +const U4: f32 = 2.2896373272e-01; /* 0x3e6a7578 */ +const U5: f32 = 1.3381091878e-02; /* 0x3c5b3c5e */ +const V1: f32 = 2.4559779167e+00; /* 0x401d2ebe */ +const V2: f32 = 2.1284897327e+00; /* 0x4008392d */ +const V3: f32 = 7.6928514242e-01; /* 0x3f44efdf */ +const V4: f32 = 1.0422264785e-01; /* 0x3dd572af */ +const V5: f32 = 3.2170924824e-03; /* 0x3b52d5db */ +const S0: f32 = -7.7215664089e-02; /* 0xbd9e233f */ +const S1: f32 = 2.1498242021e-01; /* 0x3e5c245a */ +const S2: f32 = 3.2577878237e-01; /* 0x3ea6cc7a */ +const S3: f32 = 1.4635047317e-01; /* 0x3e15dce6 */ +const S4: f32 = 2.6642270386e-02; /* 0x3cda40e4 */ +const S5: f32 = 1.8402845599e-03; /* 0x3af135b4 */ +const S6: f32 = 3.1947532989e-05; /* 0x3805ff67 */ +const R1: f32 = 1.3920053244e+00; /* 0x3fb22d3b */ +const R2: f32 = 7.2193557024e-01; /* 0x3f38d0c5 */ +const R3: f32 = 1.7193385959e-01; /* 0x3e300f6e */ +const R4: f32 = 1.8645919859e-02; /* 0x3c98bf54 */ +const R5: f32 = 7.7794247773e-04; /* 0x3a4beed6 */ +const R6: f32 = 7.3266842264e-06; /* 0x36f5d7bd */ +const W0: f32 = 4.1893854737e-01; /* 0x3ed67f1d */ +const W1: f32 = 8.3333335817e-02; /* 0x3daaaaab */ +const W2: f32 = -2.7777778450e-03; /* 0xbb360b61 */ +const W3: f32 = 7.9365057172e-04; /* 0x3a500cfd */ +const W4: f32 = -5.9518753551e-04; /* 0xba1c065c */ +const W5: f32 = 8.3633989561e-04; /* 0x3a5b3dd2 */ +const W6: f32 = -1.6309292987e-03; /* 0xbad5c4e8 */ + +/* sin(PI*x) assuming x > 2^-100, if sin(PI*x)==0 the sign is arbitrary */ +fn sin_pi(mut x: f32) -> f32 { + let mut y: f64; + let mut n: isize; + + /* spurious inexact if odd int */ + x = 2.0 * (x * 0.5 - floorf(x * 0.5)); /* x mod 2.0 */ + + n = (x * 4.0) as isize; + n = div!(n + 1, 2); + y = (x as f64) - (n as f64) * 0.5; + y *= 3.14159265358979323846; + match n { + 1 => k_cosf(y), + 2 => k_sinf(-y), + 3 => -k_cosf(y), + 0 | _ => k_sinf(y), + } +} + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn lgammaf_r(mut x: f32) -> (f32, i32) { + let u = x.to_bits(); + let mut t: f32; + let y: f32; + let mut z: f32; + let nadj: f32; + let p: f32; + let p1: f32; + let p2: f32; + let p3: f32; + let q: f32; + let mut r: f32; + let w: f32; + let ix: u32; + let i: i32; + let sign: bool; + let mut signgam: i32; + + /* purge off +-inf, NaN, +-0, tiny and negative arguments */ + signgam = 1; + sign = (u >> 31) != 0; + ix = u & 0x7fffffff; + if ix >= 0x7f800000 { + return (x * x, signgam); + } + if ix < 0x35000000 { + /* |x| < 2**-21, return -log(|x|) */ + if sign { + signgam = -1; + x = -x; + } + return (-logf(x), signgam); + } + if sign { + x = -x; + t = sin_pi(x); + if t == 0.0 { + /* -integer */ + return (1.0 / (x - x), signgam); + } + if t > 0.0 { + signgam = -1; + } else { + t = -t; + } + nadj = logf(PI / (t * x)); + } else { + nadj = 0.0; + } + + /* purge off 1 and 2 */ + if ix == 0x3f800000 || ix == 0x40000000 { + r = 0.0; + } + /* for x < 2.0 */ + else if ix < 0x40000000 { + if ix <= 0x3f666666 { + /* lgamma(x) = lgamma(x+1)-log(x) */ + r = -logf(x); + if ix >= 0x3f3b4a20 { + y = 1.0 - x; + i = 0; + } else if ix >= 0x3e6d3308 { + y = x - (TC - 1.0); + i = 1; + } else { + y = x; + i = 2; + } + } else { + r = 0.0; + if ix >= 0x3fdda618 { + /* [1.7316,2] */ + y = 2.0 - x; + i = 0; + } else if ix >= 0x3F9da620 { + /* [1.23,1.73] */ + y = x - TC; + i = 1; + } else { + y = x - 1.0; + i = 2; + } + } + match i { + 0 => { + z = y * y; + p1 = A0 + z * (A2 + z * (A4 + z * (A6 + z * (A8 + z * A10)))); + p2 = z * (A1 + z * (A3 + z * (A5 + z * (A7 + z * (A9 + z * A11))))); + p = y * p1 + p2; + r += p - 0.5 * y; + } + 1 => { + z = y * y; + w = z * y; + p1 = T0 + w * (T3 + w * (T6 + w * (T9 + w * T12))); /* parallel comp */ + p2 = T1 + w * (T4 + w * (T7 + w * (T10 + w * T13))); + p3 = T2 + w * (T5 + w * (T8 + w * (T11 + w * T14))); + p = z * p1 - (TT - w * (p2 + y * p3)); + r += TF + p; + } + 2 => { + p1 = y * (U0 + y * (U1 + y * (U2 + y * (U3 + y * (U4 + y * U5))))); + p2 = 1.0 + y * (V1 + y * (V2 + y * (V3 + y * (V4 + y * V5)))); + r += -0.5 * y + p1 / p2; + } + #[cfg(debug_assertions)] + _ => unreachable!(), + #[cfg(not(debug_assertions))] + _ => {} + } + } else if ix < 0x41000000 { + /* x < 8.0 */ + i = x as i32; + y = x - (i as f32); + p = y * (S0 + y * (S1 + y * (S2 + y * (S3 + y * (S4 + y * (S5 + y * S6)))))); + q = 1.0 + y * (R1 + y * (R2 + y * (R3 + y * (R4 + y * (R5 + y * R6))))); + r = 0.5 * y + p / q; + z = 1.0; /* lgamma(1+s) = log(s) + lgamma(s) */ + // TODO: In C, this was implemented using switch jumps with fallthrough. + // Does this implementation have performance problems? + if i >= 7 { + z *= y + 6.0; + } + if i >= 6 { + z *= y + 5.0; + } + if i >= 5 { + z *= y + 4.0; + } + if i >= 4 { + z *= y + 3.0; + } + if i >= 3 { + z *= y + 2.0; + r += logf(z); + } + } else if ix < 0x5c800000 { + /* 8.0 <= x < 2**58 */ + t = logf(x); + z = 1.0 / x; + y = z * z; + w = W0 + z * (W1 + y * (W2 + y * (W3 + y * (W4 + y * (W5 + y * W6))))); + r = (x - 0.5) * (t - 1.0) + w; + } else { + /* 2**58 <= x <= inf */ + r = x * (logf(x) - 1.0); + } + if sign { + r = nadj - r; + } + return (r, signgam); +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/log.rs b/crux-mir/lib/compiler_builtins/libm/src/math/log.rs new file mode 100644 index 000000000..27a26da60 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/log.rs @@ -0,0 +1,117 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_log.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunSoft, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ +/* log(x) + * Return the logarithm of x + * + * Method : + * 1. Argument Reduction: find k and f such that + * x = 2^k * (1+f), + * where sqrt(2)/2 < 1+f < sqrt(2) . + * + * 2. Approximation of log(1+f). + * Let s = f/(2+f) ; based on log(1+f) = log(1+s) - log(1-s) + * = 2s + 2/3 s**3 + 2/5 s**5 + ....., + * = 2s + s*R + * We use a special Remez algorithm on [0,0.1716] to generate + * a polynomial of degree 14 to approximate R The maximum error + * of this polynomial approximation is bounded by 2**-58.45. In + * other words, + * 2 4 6 8 10 12 14 + * R(z) ~ Lg1*s +Lg2*s +Lg3*s +Lg4*s +Lg5*s +Lg6*s +Lg7*s + * (the values of Lg1 to Lg7 are listed in the program) + * and + * | 2 14 | -58.45 + * | Lg1*s +...+Lg7*s - R(z) | <= 2 + * | | + * Note that 2s = f - s*f = f - hfsq + s*hfsq, where hfsq = f*f/2. + * In order to guarantee error in log below 1ulp, we compute log + * by + * log(1+f) = f - s*(f - R) (if f is not too large) + * log(1+f) = f - (hfsq - s*(hfsq+R)). (better accuracy) + * + * 3. Finally, log(x) = k*ln2 + log(1+f). + * = k*ln2_hi+(f-(hfsq-(s*(hfsq+R)+k*ln2_lo))) + * Here ln2 is split into two floating point number: + * ln2_hi + ln2_lo, + * where n*ln2_hi is always exact for |n| < 2000. + * + * Special cases: + * log(x) is NaN with signal if x < 0 (including -INF) ; + * log(+INF) is +INF; log(0) is -INF with signal; + * log(NaN) is that NaN with no signal. + * + * Accuracy: + * according to an error analysis, the error is always less than + * 1 ulp (unit in the last place). + * + * Constants: + * The hexadecimal values are the intended ones for the following + * constants. The decimal values may be used, provided that the + * compiler will convert from decimal to binary accurately enough + * to produce the hexadecimal values shown. + */ + +const LN2_HI: f64 = 6.93147180369123816490e-01; /* 3fe62e42 fee00000 */ +const LN2_LO: f64 = 1.90821492927058770002e-10; /* 3dea39ef 35793c76 */ +const LG1: f64 = 6.666666666666735130e-01; /* 3FE55555 55555593 */ +const LG2: f64 = 3.999999999940941908e-01; /* 3FD99999 9997FA04 */ +const LG3: f64 = 2.857142874366239149e-01; /* 3FD24924 94229359 */ +const LG4: f64 = 2.222219843214978396e-01; /* 3FCC71C5 1D8E78AF */ +const LG5: f64 = 1.818357216161805012e-01; /* 3FC74664 96CB03DE */ +const LG6: f64 = 1.531383769920937332e-01; /* 3FC39A09 D078C69F */ +const LG7: f64 = 1.479819860511658591e-01; /* 3FC2F112 DF3E5244 */ + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn log(mut x: f64) -> f64 { + let x1p54 = f64::from_bits(0x4350000000000000); // 0x1p54 === 2 ^ 54 + + let mut ui = x.to_bits(); + let mut hx: u32 = (ui >> 32) as u32; + let mut k: i32 = 0; + + if (hx < 0x00100000) || ((hx >> 31) != 0) { + /* x < 2**-126 */ + if ui << 1 == 0 { + return -1. / (x * x); /* log(+-0)=-inf */ + } + if hx >> 31 != 0 { + return (x - x) / 0.0; /* log(-#) = NaN */ + } + /* subnormal number, scale x up */ + k -= 54; + x *= x1p54; + ui = x.to_bits(); + hx = (ui >> 32) as u32; + } else if hx >= 0x7ff00000 { + return x; + } else if hx == 0x3ff00000 && ui << 32 == 0 { + return 0.; + } + + /* reduce x into [sqrt(2)/2, sqrt(2)] */ + hx += 0x3ff00000 - 0x3fe6a09e; + k += ((hx >> 20) as i32) - 0x3ff; + hx = (hx & 0x000fffff) + 0x3fe6a09e; + ui = ((hx as u64) << 32) | (ui & 0xffffffff); + x = f64::from_bits(ui); + + let f: f64 = x - 1.0; + let hfsq: f64 = 0.5 * f * f; + let s: f64 = f / (2.0 + f); + let z: f64 = s * s; + let w: f64 = z * z; + let t1: f64 = w * (LG2 + w * (LG4 + w * LG6)); + let t2: f64 = z * (LG1 + w * (LG3 + w * (LG5 + w * LG7))); + let r: f64 = t2 + t1; + let dk: f64 = k as f64; + s * (hfsq + r) + dk * LN2_LO - hfsq + f + dk * LN2_HI +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/log10.rs b/crux-mir/lib/compiler_builtins/libm/src/math/log10.rs new file mode 100644 index 000000000..40dacf2c9 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/log10.rs @@ -0,0 +1,117 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_log10.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunSoft, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ +/* + * Return the base 10 logarithm of x. See log.c for most comments. + * + * Reduce x to 2^k (1+f) and calculate r = log(1+f) - f + f*f/2 + * as in log.c, then combine and scale in extra precision: + * log10(x) = (f - f*f/2 + r)/log(10) + k*log10(2) + */ + +use core::f64; + +const IVLN10HI: f64 = 4.34294481878168880939e-01; /* 0x3fdbcb7b, 0x15200000 */ +const IVLN10LO: f64 = 2.50829467116452752298e-11; /* 0x3dbb9438, 0xca9aadd5 */ +const LOG10_2HI: f64 = 3.01029995663611771306e-01; /* 0x3FD34413, 0x509F6000 */ +const LOG10_2LO: f64 = 3.69423907715893078616e-13; /* 0x3D59FEF3, 0x11F12B36 */ +const LG1: f64 = 6.666666666666735130e-01; /* 3FE55555 55555593 */ +const LG2: f64 = 3.999999999940941908e-01; /* 3FD99999 9997FA04 */ +const LG3: f64 = 2.857142874366239149e-01; /* 3FD24924 94229359 */ +const LG4: f64 = 2.222219843214978396e-01; /* 3FCC71C5 1D8E78AF */ +const LG5: f64 = 1.818357216161805012e-01; /* 3FC74664 96CB03DE */ +const LG6: f64 = 1.531383769920937332e-01; /* 3FC39A09 D078C69F */ +const LG7: f64 = 1.479819860511658591e-01; /* 3FC2F112 DF3E5244 */ + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn log10(mut x: f64) -> f64 { + let x1p54 = f64::from_bits(0x4350000000000000); // 0x1p54 === 2 ^ 54 + + let mut ui: u64 = x.to_bits(); + let hfsq: f64; + let f: f64; + let s: f64; + let z: f64; + let r: f64; + let mut w: f64; + let t1: f64; + let t2: f64; + let dk: f64; + let y: f64; + let mut hi: f64; + let lo: f64; + let mut val_hi: f64; + let mut val_lo: f64; + let mut hx: u32; + let mut k: i32; + + hx = (ui >> 32) as u32; + k = 0; + if hx < 0x00100000 || (hx >> 31) > 0 { + if ui << 1 == 0 { + return -1. / (x * x); /* log(+-0)=-inf */ + } + if (hx >> 31) > 0 { + return (x - x) / 0.0; /* log(-#) = NaN */ + } + /* subnormal number, scale x up */ + k -= 54; + x *= x1p54; + ui = x.to_bits(); + hx = (ui >> 32) as u32; + } else if hx >= 0x7ff00000 { + return x; + } else if hx == 0x3ff00000 && ui << 32 == 0 { + return 0.; + } + + /* reduce x into [sqrt(2)/2, sqrt(2)] */ + hx += 0x3ff00000 - 0x3fe6a09e; + k += (hx >> 20) as i32 - 0x3ff; + hx = (hx & 0x000fffff) + 0x3fe6a09e; + ui = (hx as u64) << 32 | (ui & 0xffffffff); + x = f64::from_bits(ui); + + f = x - 1.0; + hfsq = 0.5 * f * f; + s = f / (2.0 + f); + z = s * s; + w = z * z; + t1 = w * (LG2 + w * (LG4 + w * LG6)); + t2 = z * (LG1 + w * (LG3 + w * (LG5 + w * LG7))); + r = t2 + t1; + + /* See log2.c for details. */ + /* hi+lo = f - hfsq + s*(hfsq+R) ~ log(1+f) */ + hi = f - hfsq; + ui = hi.to_bits(); + ui &= (-1i64 as u64) << 32; + hi = f64::from_bits(ui); + lo = f - hi - hfsq + s * (hfsq + r); + + /* val_hi+val_lo ~ log10(1+f) + k*log10(2) */ + val_hi = hi * IVLN10HI; + dk = k as f64; + y = dk * LOG10_2HI; + val_lo = dk * LOG10_2LO + (lo + hi) * IVLN10LO + lo * IVLN10HI; + + /* + * Extra precision in for adding y is not strictly needed + * since there is no very large cancellation near x = sqrt(2) or + * x = 1/sqrt(2), but we do it anyway since it costs little on CPUs + * with some parallelism and it reduces the error for many args. + */ + w = y + val_hi; + val_lo += (y - w) + val_hi; + val_hi = w; + + val_lo + val_hi +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/log10f.rs b/crux-mir/lib/compiler_builtins/libm/src/math/log10f.rs new file mode 100644 index 000000000..108dfa8b5 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/log10f.rs @@ -0,0 +1,91 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_log10f.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ +/* + * See comments in log10.c. + */ + +use core::f32; + +const IVLN10HI: f32 = 4.3432617188e-01; /* 0x3ede6000 */ +const IVLN10LO: f32 = -3.1689971365e-05; /* 0xb804ead9 */ +const LOG10_2HI: f32 = 3.0102920532e-01; /* 0x3e9a2080 */ +const LOG10_2LO: f32 = 7.9034151668e-07; /* 0x355427db */ +/* |(log(1+s)-log(1-s))/s - Lg(s)| < 2**-34.24 (~[-4.95e-11, 4.97e-11]). */ +const LG1: f32 = 0.66666662693; /* 0xaaaaaa.0p-24 */ +const LG2: f32 = 0.40000972152; /* 0xccce13.0p-25 */ +const LG3: f32 = 0.28498786688; /* 0x91e9ee.0p-25 */ +const LG4: f32 = 0.24279078841; /* 0xf89e26.0p-26 */ + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn log10f(mut x: f32) -> f32 { + let x1p25f = f32::from_bits(0x4c000000); // 0x1p25f === 2 ^ 25 + + let mut ui: u32 = x.to_bits(); + let hfsq: f32; + let f: f32; + let s: f32; + let z: f32; + let r: f32; + let w: f32; + let t1: f32; + let t2: f32; + let dk: f32; + let mut hi: f32; + let lo: f32; + let mut ix: u32; + let mut k: i32; + + ix = ui; + k = 0; + if ix < 0x00800000 || (ix >> 31) > 0 { + /* x < 2**-126 */ + if ix << 1 == 0 { + return -1. / (x * x); /* log(+-0)=-inf */ + } + if (ix >> 31) > 0 { + return (x - x) / 0.0; /* log(-#) = NaN */ + } + /* subnormal number, scale up x */ + k -= 25; + x *= x1p25f; + ui = x.to_bits(); + ix = ui; + } else if ix >= 0x7f800000 { + return x; + } else if ix == 0x3f800000 { + return 0.; + } + + /* reduce x into [sqrt(2)/2, sqrt(2)] */ + ix += 0x3f800000 - 0x3f3504f3; + k += (ix >> 23) as i32 - 0x7f; + ix = (ix & 0x007fffff) + 0x3f3504f3; + ui = ix; + x = f32::from_bits(ui); + + f = x - 1.0; + s = f / (2.0 + f); + z = s * s; + w = z * z; + t1 = w * (LG2 + w * LG4); + t2 = z * (LG1 + w * LG3); + r = t2 + t1; + hfsq = 0.5 * f * f; + + hi = f - hfsq; + ui = hi.to_bits(); + ui &= 0xfffff000; + hi = f32::from_bits(ui); + lo = f - hi - hfsq + s * (hfsq + r); + dk = k as f32; + dk * LOG10_2LO + (lo + hi) * IVLN10LO + lo * IVLN10HI + hi * IVLN10HI + dk * LOG10_2HI +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/log1p.rs b/crux-mir/lib/compiler_builtins/libm/src/math/log1p.rs new file mode 100644 index 000000000..4fd1c73eb --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/log1p.rs @@ -0,0 +1,143 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/s_log1p.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ +/* double log1p(double x) + * Return the natural logarithm of 1+x. + * + * Method : + * 1. Argument Reduction: find k and f such that + * 1+x = 2^k * (1+f), + * where sqrt(2)/2 < 1+f < sqrt(2) . + * + * Note. If k=0, then f=x is exact. However, if k!=0, then f + * may not be representable exactly. In that case, a correction + * term is need. Let u=1+x rounded. Let c = (1+x)-u, then + * log(1+x) - log(u) ~ c/u. Thus, we proceed to compute log(u), + * and add back the correction term c/u. + * (Note: when x > 2**53, one can simply return log(x)) + * + * 2. Approximation of log(1+f): See log.c + * + * 3. Finally, log1p(x) = k*ln2 + log(1+f) + c/u. See log.c + * + * Special cases: + * log1p(x) is NaN with signal if x < -1 (including -INF) ; + * log1p(+INF) is +INF; log1p(-1) is -INF with signal; + * log1p(NaN) is that NaN with no signal. + * + * Accuracy: + * according to an error analysis, the error is always less than + * 1 ulp (unit in the last place). + * + * Constants: + * The hexadecimal values are the intended ones for the following + * constants. The decimal values may be used, provided that the + * compiler will convert from decimal to binary accurately enough + * to produce the hexadecimal values shown. + * + * Note: Assuming log() return accurate answer, the following + * algorithm can be used to compute log1p(x) to within a few ULP: + * + * u = 1+x; + * if(u==1.0) return x ; else + * return log(u)*(x/(u-1.0)); + * + * See HP-15C Advanced Functions Handbook, p.193. + */ + +use core::f64; + +const LN2_HI: f64 = 6.93147180369123816490e-01; /* 3fe62e42 fee00000 */ +const LN2_LO: f64 = 1.90821492927058770002e-10; /* 3dea39ef 35793c76 */ +const LG1: f64 = 6.666666666666735130e-01; /* 3FE55555 55555593 */ +const LG2: f64 = 3.999999999940941908e-01; /* 3FD99999 9997FA04 */ +const LG3: f64 = 2.857142874366239149e-01; /* 3FD24924 94229359 */ +const LG4: f64 = 2.222219843214978396e-01; /* 3FCC71C5 1D8E78AF */ +const LG5: f64 = 1.818357216161805012e-01; /* 3FC74664 96CB03DE */ +const LG6: f64 = 1.531383769920937332e-01; /* 3FC39A09 D078C69F */ +const LG7: f64 = 1.479819860511658591e-01; /* 3FC2F112 DF3E5244 */ + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn log1p(x: f64) -> f64 { + let mut ui: u64 = x.to_bits(); + let hfsq: f64; + let mut f: f64 = 0.; + let mut c: f64 = 0.; + let s: f64; + let z: f64; + let r: f64; + let w: f64; + let t1: f64; + let t2: f64; + let dk: f64; + let hx: u32; + let mut hu: u32; + let mut k: i32; + + hx = (ui >> 32) as u32; + k = 1; + if hx < 0x3fda827a || (hx >> 31) > 0 { + /* 1+x < sqrt(2)+ */ + if hx >= 0xbff00000 { + /* x <= -1.0 */ + if x == -1. { + return x / 0.0; /* log1p(-1) = -inf */ + } + return (x - x) / 0.0; /* log1p(x<-1) = NaN */ + } + if hx << 1 < 0x3ca00000 << 1 { + /* |x| < 2**-53 */ + /* underflow if subnormal */ + if (hx & 0x7ff00000) == 0 { + force_eval!(x as f32); + } + return x; + } + if hx <= 0xbfd2bec4 { + /* sqrt(2)/2- <= 1+x < sqrt(2)+ */ + k = 0; + c = 0.; + f = x; + } + } else if hx >= 0x7ff00000 { + return x; + } + if k > 0 { + ui = (1. + x).to_bits(); + hu = (ui >> 32) as u32; + hu += 0x3ff00000 - 0x3fe6a09e; + k = (hu >> 20) as i32 - 0x3ff; + /* correction term ~ log(1+x)-log(u), avoid underflow in c/u */ + if k < 54 { + c = if k >= 2 { + 1. - (f64::from_bits(ui) - x) + } else { + x - (f64::from_bits(ui) - 1.) + }; + c /= f64::from_bits(ui); + } else { + c = 0.; + } + /* reduce u into [sqrt(2)/2, sqrt(2)] */ + hu = (hu & 0x000fffff) + 0x3fe6a09e; + ui = (hu as u64) << 32 | (ui & 0xffffffff); + f = f64::from_bits(ui) - 1.; + } + hfsq = 0.5 * f * f; + s = f / (2.0 + f); + z = s * s; + w = z * z; + t1 = w * (LG2 + w * (LG4 + w * LG6)); + t2 = z * (LG1 + w * (LG3 + w * (LG5 + w * LG7))); + r = t2 + t1; + dk = k as f64; + s * (hfsq + r) + (dk * LN2_LO + c) - hfsq + f + dk * LN2_HI +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/log1pf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/log1pf.rs new file mode 100644 index 000000000..500e8eeaa --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/log1pf.rs @@ -0,0 +1,98 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/s_log1pf.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +use core::f32; + +const LN2_HI: f32 = 6.9313812256e-01; /* 0x3f317180 */ +const LN2_LO: f32 = 9.0580006145e-06; /* 0x3717f7d1 */ +/* |(log(1+s)-log(1-s))/s - Lg(s)| < 2**-34.24 (~[-4.95e-11, 4.97e-11]). */ +const LG1: f32 = 0.66666662693; /* 0xaaaaaa.0p-24 */ +const LG2: f32 = 0.40000972152; /* 0xccce13.0p-25 */ +const LG3: f32 = 0.28498786688; /* 0x91e9ee.0p-25 */ +const LG4: f32 = 0.24279078841; /* 0xf89e26.0p-26 */ + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn log1pf(x: f32) -> f32 { + let mut ui: u32 = x.to_bits(); + let hfsq: f32; + let mut f: f32 = 0.; + let mut c: f32 = 0.; + let s: f32; + let z: f32; + let r: f32; + let w: f32; + let t1: f32; + let t2: f32; + let dk: f32; + let ix: u32; + let mut iu: u32; + let mut k: i32; + + ix = ui; + k = 1; + if ix < 0x3ed413d0 || (ix >> 31) > 0 { + /* 1+x < sqrt(2)+ */ + if ix >= 0xbf800000 { + /* x <= -1.0 */ + if x == -1. { + return x / 0.0; /* log1p(-1)=+inf */ + } + return (x - x) / 0.0; /* log1p(x<-1)=NaN */ + } + if ix << 1 < 0x33800000 << 1 { + /* |x| < 2**-24 */ + /* underflow if subnormal */ + if (ix & 0x7f800000) == 0 { + force_eval!(x * x); + } + return x; + } + if ix <= 0xbe95f619 { + /* sqrt(2)/2- <= 1+x < sqrt(2)+ */ + k = 0; + c = 0.; + f = x; + } + } else if ix >= 0x7f800000 { + return x; + } + if k > 0 { + ui = (1. + x).to_bits(); + iu = ui; + iu += 0x3f800000 - 0x3f3504f3; + k = (iu >> 23) as i32 - 0x7f; + /* correction term ~ log(1+x)-log(u), avoid underflow in c/u */ + if k < 25 { + c = if k >= 2 { + 1. - (f32::from_bits(ui) - x) + } else { + x - (f32::from_bits(ui) - 1.) + }; + c /= f32::from_bits(ui); + } else { + c = 0.; + } + /* reduce u into [sqrt(2)/2, sqrt(2)] */ + iu = (iu & 0x007fffff) + 0x3f3504f3; + ui = iu; + f = f32::from_bits(ui) - 1.; + } + s = f / (2.0 + f); + z = s * s; + w = z * z; + t1 = w * (LG2 + w * LG4); + t2 = z * (LG1 + w * LG3); + r = t2 + t1; + hfsq = 0.5 * f * f; + dk = k as f32; + s * (hfsq + r) + (dk * LN2_LO + c) - hfsq + f + dk * LN2_HI +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/log2.rs b/crux-mir/lib/compiler_builtins/libm/src/math/log2.rs new file mode 100644 index 000000000..83da3a193 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/log2.rs @@ -0,0 +1,106 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_log2.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunSoft, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ +/* + * Return the base 2 logarithm of x. See log.c for most comments. + * + * Reduce x to 2^k (1+f) and calculate r = log(1+f) - f + f*f/2 + * as in log.c, then combine and scale in extra precision: + * log2(x) = (f - f*f/2 + r)/log(2) + k + */ + +use core::f64; + +const IVLN2HI: f64 = 1.44269504072144627571e+00; /* 0x3ff71547, 0x65200000 */ +const IVLN2LO: f64 = 1.67517131648865118353e-10; /* 0x3de705fc, 0x2eefa200 */ +const LG1: f64 = 6.666666666666735130e-01; /* 3FE55555 55555593 */ +const LG2: f64 = 3.999999999940941908e-01; /* 3FD99999 9997FA04 */ +const LG3: f64 = 2.857142874366239149e-01; /* 3FD24924 94229359 */ +const LG4: f64 = 2.222219843214978396e-01; /* 3FCC71C5 1D8E78AF */ +const LG5: f64 = 1.818357216161805012e-01; /* 3FC74664 96CB03DE */ +const LG6: f64 = 1.531383769920937332e-01; /* 3FC39A09 D078C69F */ +const LG7: f64 = 1.479819860511658591e-01; /* 3FC2F112 DF3E5244 */ + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn log2(mut x: f64) -> f64 { + let x1p54 = f64::from_bits(0x4350000000000000); // 0x1p54 === 2 ^ 54 + + let mut ui: u64 = x.to_bits(); + let hfsq: f64; + let f: f64; + let s: f64; + let z: f64; + let r: f64; + let mut w: f64; + let t1: f64; + let t2: f64; + let y: f64; + let mut hi: f64; + let lo: f64; + let mut val_hi: f64; + let mut val_lo: f64; + let mut hx: u32; + let mut k: i32; + + hx = (ui >> 32) as u32; + k = 0; + if hx < 0x00100000 || (hx >> 31) > 0 { + if ui << 1 == 0 { + return -1. / (x * x); /* log(+-0)=-inf */ + } + if (hx >> 31) > 0 { + return (x - x) / 0.0; /* log(-#) = NaN */ + } + /* subnormal number, scale x up */ + k -= 54; + x *= x1p54; + ui = x.to_bits(); + hx = (ui >> 32) as u32; + } else if hx >= 0x7ff00000 { + return x; + } else if hx == 0x3ff00000 && ui << 32 == 0 { + return 0.; + } + + /* reduce x into [sqrt(2)/2, sqrt(2)] */ + hx += 0x3ff00000 - 0x3fe6a09e; + k += (hx >> 20) as i32 - 0x3ff; + hx = (hx & 0x000fffff) + 0x3fe6a09e; + ui = (hx as u64) << 32 | (ui & 0xffffffff); + x = f64::from_bits(ui); + + f = x - 1.0; + hfsq = 0.5 * f * f; + s = f / (2.0 + f); + z = s * s; + w = z * z; + t1 = w * (LG2 + w * (LG4 + w * LG6)); + t2 = z * (LG1 + w * (LG3 + w * (LG5 + w * LG7))); + r = t2 + t1; + + /* hi+lo = f - hfsq + s*(hfsq+R) ~ log(1+f) */ + hi = f - hfsq; + ui = hi.to_bits(); + ui &= (-1i64 as u64) << 32; + hi = f64::from_bits(ui); + lo = f - hi - hfsq + s * (hfsq + r); + + val_hi = hi * IVLN2HI; + val_lo = (lo + hi) * IVLN2LO + lo * IVLN2HI; + + /* spadd(val_hi, val_lo, y), except for not using double_t: */ + y = k.into(); + w = y + val_hi; + val_lo += (y - w) + val_hi; + val_hi = w; + + val_lo + val_hi +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/log2f.rs b/crux-mir/lib/compiler_builtins/libm/src/math/log2f.rs new file mode 100644 index 000000000..3a20fb15b --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/log2f.rs @@ -0,0 +1,87 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_log2f.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ +/* + * See comments in log2.c. + */ + +use core::f32; + +const IVLN2HI: f32 = 1.4428710938e+00; /* 0x3fb8b000 */ +const IVLN2LO: f32 = -1.7605285393e-04; /* 0xb9389ad4 */ +/* |(log(1+s)-log(1-s))/s - Lg(s)| < 2**-34.24 (~[-4.95e-11, 4.97e-11]). */ +const LG1: f32 = 0.66666662693; /* 0xaaaaaa.0p-24 */ +const LG2: f32 = 0.40000972152; /* 0xccce13.0p-25 */ +const LG3: f32 = 0.28498786688; /* 0x91e9ee.0p-25 */ +const LG4: f32 = 0.24279078841; /* 0xf89e26.0p-26 */ + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn log2f(mut x: f32) -> f32 { + let x1p25f = f32::from_bits(0x4c000000); // 0x1p25f === 2 ^ 25 + + let mut ui: u32 = x.to_bits(); + let hfsq: f32; + let f: f32; + let s: f32; + let z: f32; + let r: f32; + let w: f32; + let t1: f32; + let t2: f32; + let mut hi: f32; + let lo: f32; + let mut ix: u32; + let mut k: i32; + + ix = ui; + k = 0; + if ix < 0x00800000 || (ix >> 31) > 0 { + /* x < 2**-126 */ + if ix << 1 == 0 { + return -1. / (x * x); /* log(+-0)=-inf */ + } + if (ix >> 31) > 0 { + return (x - x) / 0.0; /* log(-#) = NaN */ + } + /* subnormal number, scale up x */ + k -= 25; + x *= x1p25f; + ui = x.to_bits(); + ix = ui; + } else if ix >= 0x7f800000 { + return x; + } else if ix == 0x3f800000 { + return 0.; + } + + /* reduce x into [sqrt(2)/2, sqrt(2)] */ + ix += 0x3f800000 - 0x3f3504f3; + k += (ix >> 23) as i32 - 0x7f; + ix = (ix & 0x007fffff) + 0x3f3504f3; + ui = ix; + x = f32::from_bits(ui); + + f = x - 1.0; + s = f / (2.0 + f); + z = s * s; + w = z * z; + t1 = w * (LG2 + w * LG4); + t2 = z * (LG1 + w * LG3); + r = t2 + t1; + hfsq = 0.5 * f * f; + + hi = f - hfsq; + ui = hi.to_bits(); + ui &= 0xfffff000; + hi = f32::from_bits(ui); + lo = f - hi - hfsq + s * (hfsq + r); + (lo + hi) * IVLN2LO + lo * IVLN2HI + hi * IVLN2HI + k as f32 +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/logf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/logf.rs new file mode 100644 index 000000000..2b57b934f --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/logf.rs @@ -0,0 +1,65 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_logf.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +const LN2_HI: f32 = 6.9313812256e-01; /* 0x3f317180 */ +const LN2_LO: f32 = 9.0580006145e-06; /* 0x3717f7d1 */ +/* |(log(1+s)-log(1-s))/s - Lg(s)| < 2**-34.24 (~[-4.95e-11, 4.97e-11]). */ +const LG1: f32 = 0.66666662693; /* 0xaaaaaa.0p-24*/ +const LG2: f32 = 0.40000972152; /* 0xccce13.0p-25 */ +const LG3: f32 = 0.28498786688; /* 0x91e9ee.0p-25 */ +const LG4: f32 = 0.24279078841; /* 0xf89e26.0p-26 */ + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn logf(mut x: f32) -> f32 { + let x1p25 = f32::from_bits(0x4c000000); // 0x1p25f === 2 ^ 25 + + let mut ix = x.to_bits(); + let mut k = 0i32; + + if (ix < 0x00800000) || ((ix >> 31) != 0) { + /* x < 2**-126 */ + if ix << 1 == 0 { + return -1. / (x * x); /* log(+-0)=-inf */ + } + if (ix >> 31) != 0 { + return (x - x) / 0.; /* log(-#) = NaN */ + } + /* subnormal number, scale up x */ + k -= 25; + x *= x1p25; + ix = x.to_bits(); + } else if ix >= 0x7f800000 { + return x; + } else if ix == 0x3f800000 { + return 0.; + } + + /* reduce x into [sqrt(2)/2, sqrt(2)] */ + ix += 0x3f800000 - 0x3f3504f3; + k += ((ix >> 23) as i32) - 0x7f; + ix = (ix & 0x007fffff) + 0x3f3504f3; + x = f32::from_bits(ix); + + let f = x - 1.; + let s = f / (2. + f); + let z = s * s; + let w = z * z; + let t1 = w * (LG2 + w * LG4); + let t2 = z * (LG1 + w * LG3); + let r = t2 + t1; + let hfsq = 0.5 * f * f; + let dk = k as f32; + s * (hfsq + r) + dk * LN2_LO - hfsq + f + dk * LN2_HI +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/mod.rs b/crux-mir/lib/compiler_builtins/libm/src/math/mod.rs new file mode 100644 index 000000000..05ebb708c --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/mod.rs @@ -0,0 +1,370 @@ +macro_rules! force_eval { + ($e:expr) => { + unsafe { ::core::ptr::read_volatile(&$e) } + }; +} + +#[cfg(not(debug_assertions))] +macro_rules! i { + ($array:expr, $index:expr) => { + unsafe { *$array.get_unchecked($index) } + }; + ($array:expr, $index:expr, = , $rhs:expr) => { + unsafe { + *$array.get_unchecked_mut($index) = $rhs; + } + }; + ($array:expr, $index:expr, += , $rhs:expr) => { + unsafe { + *$array.get_unchecked_mut($index) += $rhs; + } + }; + ($array:expr, $index:expr, -= , $rhs:expr) => { + unsafe { + *$array.get_unchecked_mut($index) -= $rhs; + } + }; + ($array:expr, $index:expr, &= , $rhs:expr) => { + unsafe { + *$array.get_unchecked_mut($index) &= $rhs; + } + }; + ($array:expr, $index:expr, == , $rhs:expr) => { + unsafe { *$array.get_unchecked_mut($index) == $rhs } + }; +} + +#[cfg(debug_assertions)] +macro_rules! i { + ($array:expr, $index:expr) => { + *$array.get($index).unwrap() + }; + ($array:expr, $index:expr, = , $rhs:expr) => { + *$array.get_mut($index).unwrap() = $rhs; + }; + ($array:expr, $index:expr, -= , $rhs:expr) => { + *$array.get_mut($index).unwrap() -= $rhs; + }; + ($array:expr, $index:expr, += , $rhs:expr) => { + *$array.get_mut($index).unwrap() += $rhs; + }; + ($array:expr, $index:expr, &= , $rhs:expr) => { + *$array.get_mut($index).unwrap() &= $rhs; + }; + ($array:expr, $index:expr, == , $rhs:expr) => { + *$array.get_mut($index).unwrap() == $rhs + }; +} + +// Temporary macro to avoid panic codegen for division (in debug mode too). At +// the time of this writing this is only used in a few places, and once +// rust-lang/rust#72751 is fixed then this macro will no longer be necessary and +// the native `/` operator can be used and panics won't be codegen'd. +#[cfg(any(debug_assertions, not(feature = "unstable")))] +macro_rules! div { + ($a:expr, $b:expr) => { + $a / $b + }; +} + +#[cfg(all(not(debug_assertions), feature = "unstable"))] +macro_rules! div { + ($a:expr, $b:expr) => { + unsafe { core::intrinsics::unchecked_div($a, $b) } + }; +} + +macro_rules! llvm_intrinsically_optimized { + (#[cfg($($clause:tt)*)] $e:expr) => { + #[cfg(all(feature = "unstable", $($clause)*))] + { + if true { // thwart the dead code lint + $e + } + } + }; +} + +// Public modules +mod acos; +mod acosf; +mod acosh; +mod acoshf; +mod asin; +mod asinf; +mod asinh; +mod asinhf; +mod atan; +mod atan2; +mod atan2f; +mod atanf; +mod atanh; +mod atanhf; +mod cbrt; +mod cbrtf; +mod ceil; +mod ceilf; +mod copysign; +mod copysignf; +mod cos; +mod cosf; +mod cosh; +mod coshf; +mod erf; +mod erff; +mod exp; +mod exp10; +mod exp10f; +mod exp2; +mod exp2f; +mod expf; +mod expm1; +mod expm1f; +mod fabs; +mod fabsf; +mod fdim; +mod fdimf; +mod floor; +mod floorf; +mod fma; +mod fmaf; +mod fmax; +mod fmaxf; +mod fmin; +mod fminf; +mod fmod; +mod fmodf; +mod frexp; +mod frexpf; +mod hypot; +mod hypotf; +mod ilogb; +mod ilogbf; +mod j0; +mod j0f; +mod j1; +mod j1f; +mod jn; +mod jnf; +mod ldexp; +mod ldexpf; +mod lgamma; +mod lgamma_r; +mod lgammaf; +mod lgammaf_r; +mod log; +mod log10; +mod log10f; +mod log1p; +mod log1pf; +mod log2; +mod log2f; +mod logf; +mod modf; +mod modff; +mod nextafter; +mod nextafterf; +mod pow; +mod powf; +mod remainder; +mod remainderf; +mod remquo; +mod remquof; +mod rint; +mod rintf; +mod round; +mod roundf; +mod scalbn; +mod scalbnf; +mod sin; +mod sincos; +mod sincosf; +mod sinf; +mod sinh; +mod sinhf; +mod sqrt; +mod sqrtf; +mod tan; +mod tanf; +mod tanh; +mod tanhf; +mod tgamma; +mod tgammaf; +mod trunc; +mod truncf; + +// Use separated imports instead of {}-grouped imports for easier merging. +pub use self::acos::acos; +pub use self::acosf::acosf; +pub use self::acosh::acosh; +pub use self::acoshf::acoshf; +pub use self::asin::asin; +pub use self::asinf::asinf; +pub use self::asinh::asinh; +pub use self::asinhf::asinhf; +pub use self::atan::atan; +pub use self::atan2::atan2; +pub use self::atan2f::atan2f; +pub use self::atanf::atanf; +pub use self::atanh::atanh; +pub use self::atanhf::atanhf; +pub use self::cbrt::cbrt; +pub use self::cbrtf::cbrtf; +pub use self::ceil::ceil; +pub use self::ceilf::ceilf; +pub use self::copysign::copysign; +pub use self::copysignf::copysignf; +pub use self::cos::cos; +pub use self::cosf::cosf; +pub use self::cosh::cosh; +pub use self::coshf::coshf; +pub use self::erf::erf; +pub use self::erf::erfc; +pub use self::erff::erfcf; +pub use self::erff::erff; +pub use self::exp::exp; +pub use self::exp10::exp10; +pub use self::exp10f::exp10f; +pub use self::exp2::exp2; +pub use self::exp2f::exp2f; +pub use self::expf::expf; +pub use self::expm1::expm1; +pub use self::expm1f::expm1f; +pub use self::fabs::fabs; +pub use self::fabsf::fabsf; +pub use self::fdim::fdim; +pub use self::fdimf::fdimf; +pub use self::floor::floor; +pub use self::floorf::floorf; +pub use self::fma::fma; +pub use self::fmaf::fmaf; +pub use self::fmax::fmax; +pub use self::fmaxf::fmaxf; +pub use self::fmin::fmin; +pub use self::fminf::fminf; +pub use self::fmod::fmod; +pub use self::fmodf::fmodf; +pub use self::frexp::frexp; +pub use self::frexpf::frexpf; +pub use self::hypot::hypot; +pub use self::hypotf::hypotf; +pub use self::ilogb::ilogb; +pub use self::ilogbf::ilogbf; +pub use self::j0::j0; +pub use self::j0::y0; +pub use self::j0f::j0f; +pub use self::j0f::y0f; +pub use self::j1::j1; +pub use self::j1::y1; +pub use self::j1f::j1f; +pub use self::j1f::y1f; +pub use self::jn::jn; +pub use self::jn::yn; +pub use self::jnf::jnf; +pub use self::jnf::ynf; +pub use self::ldexp::ldexp; +pub use self::ldexpf::ldexpf; +pub use self::lgamma::lgamma; +pub use self::lgamma_r::lgamma_r; +pub use self::lgammaf::lgammaf; +pub use self::lgammaf_r::lgammaf_r; +pub use self::log::log; +pub use self::log10::log10; +pub use self::log10f::log10f; +pub use self::log1p::log1p; +pub use self::log1pf::log1pf; +pub use self::log2::log2; +pub use self::log2f::log2f; +pub use self::logf::logf; +pub use self::modf::modf; +pub use self::modff::modff; +pub use self::nextafter::nextafter; +pub use self::nextafterf::nextafterf; +pub use self::pow::pow; +pub use self::powf::powf; +pub use self::remainder::remainder; +pub use self::remainderf::remainderf; +pub use self::remquo::remquo; +pub use self::remquof::remquof; +pub use self::rint::rint; +pub use self::rintf::rintf; +pub use self::round::round; +pub use self::roundf::roundf; +pub use self::scalbn::scalbn; +pub use self::scalbnf::scalbnf; +pub use self::sin::sin; +pub use self::sincos::sincos; +pub use self::sincosf::sincosf; +pub use self::sinf::sinf; +pub use self::sinh::sinh; +pub use self::sinhf::sinhf; +pub use self::sqrt::sqrt; +pub use self::sqrtf::sqrtf; +pub use self::tan::tan; +pub use self::tanf::tanf; +pub use self::tanh::tanh; +pub use self::tanhf::tanhf; +pub use self::tgamma::tgamma; +pub use self::tgammaf::tgammaf; +pub use self::trunc::trunc; +pub use self::truncf::truncf; + +// Private modules +mod expo2; +mod fenv; +mod k_cos; +mod k_cosf; +mod k_expo2; +mod k_expo2f; +mod k_sin; +mod k_sinf; +mod k_tan; +mod k_tanf; +mod rem_pio2; +mod rem_pio2_large; +mod rem_pio2f; + +// Private re-imports +use self::expo2::expo2; +use self::k_cos::k_cos; +use self::k_cosf::k_cosf; +use self::k_expo2::k_expo2; +use self::k_expo2f::k_expo2f; +use self::k_sin::k_sin; +use self::k_sinf::k_sinf; +use self::k_tan::k_tan; +use self::k_tanf::k_tanf; +use self::rem_pio2::rem_pio2; +use self::rem_pio2_large::rem_pio2_large; +use self::rem_pio2f::rem_pio2f; + +#[inline] +fn get_high_word(x: f64) -> u32 { + (x.to_bits() >> 32) as u32 +} + +#[inline] +fn get_low_word(x: f64) -> u32 { + x.to_bits() as u32 +} + +#[inline] +fn with_set_high_word(f: f64, hi: u32) -> f64 { + let mut tmp = f.to_bits(); + tmp &= 0x00000000_ffffffff; + tmp |= (hi as u64) << 32; + f64::from_bits(tmp) +} + +#[inline] +fn with_set_low_word(f: f64, lo: u32) -> f64 { + let mut tmp = f.to_bits(); + tmp &= 0xffffffff_00000000; + tmp |= lo as u64; + f64::from_bits(tmp) +} + +#[inline] +fn combine_words(hi: u32, lo: u32) -> f64 { + f64::from_bits((hi as u64) << 32 | lo as u64) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/modf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/modf.rs new file mode 100644 index 000000000..bcab33a81 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/modf.rs @@ -0,0 +1,34 @@ +pub fn modf(x: f64) -> (f64, f64) { + let rv2: f64; + let mut u = x.to_bits(); + let mask: u64; + let e = ((u >> 52 & 0x7ff) as i32) - 0x3ff; + + /* no fractional part */ + if e >= 52 { + rv2 = x; + if e == 0x400 && (u << 12) != 0 { + /* nan */ + return (x, rv2); + } + u &= 1 << 63; + return (f64::from_bits(u), rv2); + } + + /* no integral part*/ + if e < 0 { + u &= 1 << 63; + rv2 = f64::from_bits(u); + return (x, rv2); + } + + mask = ((!0) >> 12) >> e; + if (u & mask) == 0 { + rv2 = x; + u &= 1 << 63; + return (f64::from_bits(u), rv2); + } + u &= !mask; + rv2 = f64::from_bits(u); + return (x - rv2, rv2); +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/modff.rs b/crux-mir/lib/compiler_builtins/libm/src/math/modff.rs new file mode 100644 index 000000000..56ece12e3 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/modff.rs @@ -0,0 +1,33 @@ +pub fn modff(x: f32) -> (f32, f32) { + let rv2: f32; + let mut u: u32 = x.to_bits(); + let mask: u32; + let e = ((u >> 23 & 0xff) as i32) - 0x7f; + + /* no fractional part */ + if e >= 23 { + rv2 = x; + if e == 0x80 && (u << 9) != 0 { + /* nan */ + return (x, rv2); + } + u &= 0x80000000; + return (f32::from_bits(u), rv2); + } + /* no integral part */ + if e < 0 { + u &= 0x80000000; + rv2 = f32::from_bits(u); + return (x, rv2); + } + + mask = 0x007fffff >> e; + if (u & mask) == 0 { + rv2 = x; + u &= 0x80000000; + return (f32::from_bits(u), rv2); + } + u &= !mask; + rv2 = f32::from_bits(u); + return (x - rv2, rv2); +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/nextafter.rs b/crux-mir/lib/compiler_builtins/libm/src/math/nextafter.rs new file mode 100644 index 000000000..13094a17c --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/nextafter.rs @@ -0,0 +1,37 @@ +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn nextafter(x: f64, y: f64) -> f64 { + if x.is_nan() || y.is_nan() { + return x + y; + } + + let mut ux_i = x.to_bits(); + let uy_i = y.to_bits(); + if ux_i == uy_i { + return y; + } + + let ax = ux_i & !1_u64 / 2; + let ay = uy_i & !1_u64 / 2; + if ax == 0 { + if ay == 0 { + return y; + } + ux_i = (uy_i & 1_u64 << 63) | 1; + } else if ax > ay || ((ux_i ^ uy_i) & 1_u64 << 63) != 0 { + ux_i -= 1; + } else { + ux_i += 1; + } + + let e = ux_i.wrapping_shr(52 & 0x7ff); + // raise overflow if ux.f is infinite and x is finite + if e == 0x7ff { + force_eval!(x + x); + } + let ux_f = f64::from_bits(ux_i); + // raise underflow if ux.f is subnormal or zero + if e == 0 { + force_eval!(x * x + ux_f * ux_f); + } + ux_f +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/nextafterf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/nextafterf.rs new file mode 100644 index 000000000..df9b10829 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/nextafterf.rs @@ -0,0 +1,37 @@ +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn nextafterf(x: f32, y: f32) -> f32 { + if x.is_nan() || y.is_nan() { + return x + y; + } + + let mut ux_i = x.to_bits(); + let uy_i = y.to_bits(); + if ux_i == uy_i { + return y; + } + + let ax = ux_i & 0x7fff_ffff_u32; + let ay = uy_i & 0x7fff_ffff_u32; + if ax == 0 { + if ay == 0 { + return y; + } + ux_i = (uy_i & 0x8000_0000_u32) | 1; + } else if ax > ay || ((ux_i ^ uy_i) & 0x8000_0000_u32) != 0 { + ux_i -= 1; + } else { + ux_i += 1; + } + + let e = ux_i.wrapping_shr(0x7f80_0000_u32); + // raise overflow if ux_f is infinite and x is finite + if e == 0x7f80_0000_u32 { + force_eval!(x + x); + } + let ux_f = f32::from_bits(ux_i); + // raise underflow if ux_f is subnormal or zero + if e == 0 { + force_eval!(x * x + ux_f * ux_f); + } + ux_f +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/pow.rs b/crux-mir/lib/compiler_builtins/libm/src/math/pow.rs new file mode 100644 index 000000000..6a19ae601 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/pow.rs @@ -0,0 +1,637 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_pow.c */ +/* + * ==================================================== + * Copyright (C) 2004 by Sun Microsystems, Inc. All rights reserved. + * + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +// pow(x,y) return x**y +// +// n +// Method: Let x = 2 * (1+f) +// 1. Compute and return log2(x) in two pieces: +// log2(x) = w1 + w2, +// where w1 has 53-24 = 29 bit trailing zeros. +// 2. Perform y*log2(x) = n+y' by simulating muti-precision +// arithmetic, where |y'|<=0.5. +// 3. Return x**y = 2**n*exp(y'*log2) +// +// Special cases: +// 1. (anything) ** 0 is 1 +// 2. 1 ** (anything) is 1 +// 3. (anything except 1) ** NAN is NAN +// 4. NAN ** (anything except 0) is NAN +// 5. +-(|x| > 1) ** +INF is +INF +// 6. +-(|x| > 1) ** -INF is +0 +// 7. +-(|x| < 1) ** +INF is +0 +// 8. +-(|x| < 1) ** -INF is +INF +// 9. -1 ** +-INF is 1 +// 10. +0 ** (+anything except 0, NAN) is +0 +// 11. -0 ** (+anything except 0, NAN, odd integer) is +0 +// 12. +0 ** (-anything except 0, NAN) is +INF, raise divbyzero +// 13. -0 ** (-anything except 0, NAN, odd integer) is +INF, raise divbyzero +// 14. -0 ** (+odd integer) is -0 +// 15. -0 ** (-odd integer) is -INF, raise divbyzero +// 16. +INF ** (+anything except 0,NAN) is +INF +// 17. +INF ** (-anything except 0,NAN) is +0 +// 18. -INF ** (+odd integer) is -INF +// 19. -INF ** (anything) = -0 ** (-anything), (anything except odd integer) +// 20. (anything) ** 1 is (anything) +// 21. (anything) ** -1 is 1/(anything) +// 22. (-anything) ** (integer) is (-1)**(integer)*(+anything**integer) +// 23. (-anything except 0 and inf) ** (non-integer) is NAN +// +// Accuracy: +// pow(x,y) returns x**y nearly rounded. In particular +// pow(integer,integer) +// always returns the correct integer provided it is +// representable. +// +// Constants : +// The hexadecimal values are the intended ones for the following +// constants. The decimal values may be used, provided that the +// compiler will convert from decimal to binary accurately enough +// to produce the hexadecimal values shown. +// +use super::{fabs, get_high_word, scalbn, sqrt, with_set_high_word, with_set_low_word}; + +const BP: [f64; 2] = [1.0, 1.5]; +const DP_H: [f64; 2] = [0.0, 5.84962487220764160156e-01]; /* 0x3fe2b803_40000000 */ +const DP_L: [f64; 2] = [0.0, 1.35003920212974897128e-08]; /* 0x3E4CFDEB, 0x43CFD006 */ +const TWO53: f64 = 9007199254740992.0; /* 0x43400000_00000000 */ +const HUGE: f64 = 1.0e300; +const TINY: f64 = 1.0e-300; + +// poly coefs for (3/2)*(log(x)-2s-2/3*s**3: +const L1: f64 = 5.99999999999994648725e-01; /* 0x3fe33333_33333303 */ +const L2: f64 = 4.28571428578550184252e-01; /* 0x3fdb6db6_db6fabff */ +const L3: f64 = 3.33333329818377432918e-01; /* 0x3fd55555_518f264d */ +const L4: f64 = 2.72728123808534006489e-01; /* 0x3fd17460_a91d4101 */ +const L5: f64 = 2.30660745775561754067e-01; /* 0x3fcd864a_93c9db65 */ +const L6: f64 = 2.06975017800338417784e-01; /* 0x3fca7e28_4a454eef */ +const P1: f64 = 1.66666666666666019037e-01; /* 0x3fc55555_5555553e */ +const P2: f64 = -2.77777777770155933842e-03; /* 0xbf66c16c_16bebd93 */ +const P3: f64 = 6.61375632143793436117e-05; /* 0x3f11566a_af25de2c */ +const P4: f64 = -1.65339022054652515390e-06; /* 0xbebbbd41_c5d26bf1 */ +const P5: f64 = 4.13813679705723846039e-08; /* 0x3e663769_72bea4d0 */ +const LG2: f64 = 6.93147180559945286227e-01; /* 0x3fe62e42_fefa39ef */ +const LG2_H: f64 = 6.93147182464599609375e-01; /* 0x3fe62e43_00000000 */ +const LG2_L: f64 = -1.90465429995776804525e-09; /* 0xbe205c61_0ca86c39 */ +const OVT: f64 = 8.0085662595372944372e-017; /* -(1024-log2(ovfl+.5ulp)) */ +const CP: f64 = 9.61796693925975554329e-01; /* 0x3feec709_dc3a03fd =2/(3ln2) */ +const CP_H: f64 = 9.61796700954437255859e-01; /* 0x3feec709_e0000000 =(float)cp */ +const CP_L: f64 = -7.02846165095275826516e-09; /* 0xbe3e2fe0_145b01f5 =tail of cp_h*/ +const IVLN2: f64 = 1.44269504088896338700e+00; /* 0x3ff71547_652b82fe =1/ln2 */ +const IVLN2_H: f64 = 1.44269502162933349609e+00; /* 0x3ff71547_60000000 =24b 1/ln2*/ +const IVLN2_L: f64 = 1.92596299112661746887e-08; /* 0x3e54ae0b_f85ddf44 =1/ln2 tail*/ + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn pow(x: f64, y: f64) -> f64 { + let t1: f64; + let t2: f64; + + let (hx, lx): (i32, u32) = ((x.to_bits() >> 32) as i32, x.to_bits() as u32); + let (hy, ly): (i32, u32) = ((y.to_bits() >> 32) as i32, y.to_bits() as u32); + + let mut ix: i32 = (hx & 0x7fffffff) as i32; + let iy: i32 = (hy & 0x7fffffff) as i32; + + /* x**0 = 1, even if x is NaN */ + if ((iy as u32) | ly) == 0 { + return 1.0; + } + + /* 1**y = 1, even if y is NaN */ + if hx == 0x3ff00000 && lx == 0 { + return 1.0; + } + + /* NaN if either arg is NaN */ + if ix > 0x7ff00000 + || (ix == 0x7ff00000 && lx != 0) + || iy > 0x7ff00000 + || (iy == 0x7ff00000 && ly != 0) + { + return x + y; + } + + /* determine if y is an odd int when x < 0 + * yisint = 0 ... y is not an integer + * yisint = 1 ... y is an odd int + * yisint = 2 ... y is an even int + */ + let mut yisint: i32 = 0; + let mut k: i32; + let mut j: i32; + if hx < 0 { + if iy >= 0x43400000 { + yisint = 2; /* even integer y */ + } else if iy >= 0x3ff00000 { + k = (iy >> 20) - 0x3ff; /* exponent */ + + if k > 20 { + j = (ly >> (52 - k)) as i32; + + if (j << (52 - k)) == (ly as i32) { + yisint = 2 - (j & 1); + } + } else if ly == 0 { + j = iy >> (20 - k); + + if (j << (20 - k)) == iy { + yisint = 2 - (j & 1); + } + } + } + } + + if ly == 0 { + /* special value of y */ + if iy == 0x7ff00000 { + /* y is +-inf */ + + return if ((ix - 0x3ff00000) | (lx as i32)) == 0 { + /* (-1)**+-inf is 1 */ + 1.0 + } else if ix >= 0x3ff00000 { + /* (|x|>1)**+-inf = inf,0 */ + if hy >= 0 { + y + } else { + 0.0 + } + } else { + /* (|x|<1)**+-inf = 0,inf */ + if hy >= 0 { + 0.0 + } else { + -y + } + }; + } + + if iy == 0x3ff00000 { + /* y is +-1 */ + return if hy >= 0 { x } else { 1.0 / x }; + } + + if hy == 0x40000000 { + /* y is 2 */ + return x * x; + } + + if hy == 0x3fe00000 { + /* y is 0.5 */ + if hx >= 0 { + /* x >= +0 */ + return sqrt(x); + } + } + } + + let mut ax: f64 = fabs(x); + if lx == 0 { + /* special value of x */ + if ix == 0x7ff00000 || ix == 0 || ix == 0x3ff00000 { + /* x is +-0,+-inf,+-1 */ + let mut z: f64 = ax; + + if hy < 0 { + /* z = (1/|x|) */ + z = 1.0 / z; + } + + if hx < 0 { + if ((ix - 0x3ff00000) | yisint) == 0 { + z = (z - z) / (z - z); /* (-1)**non-int is NaN */ + } else if yisint == 1 { + z = -z; /* (x<0)**odd = -(|x|**odd) */ + } + } + + return z; + } + } + + let mut s: f64 = 1.0; /* sign of result */ + if hx < 0 { + if yisint == 0 { + /* (x<0)**(non-int) is NaN */ + return (x - x) / (x - x); + } + + if yisint == 1 { + /* (x<0)**(odd int) */ + s = -1.0; + } + } + + /* |y| is HUGE */ + if iy > 0x41e00000 { + /* if |y| > 2**31 */ + if iy > 0x43f00000 { + /* if |y| > 2**64, must o/uflow */ + if ix <= 0x3fefffff { + return if hy < 0 { HUGE * HUGE } else { TINY * TINY }; + } + + if ix >= 0x3ff00000 { + return if hy > 0 { HUGE * HUGE } else { TINY * TINY }; + } + } + + /* over/underflow if x is not close to one */ + if ix < 0x3fefffff { + return if hy < 0 { + s * HUGE * HUGE + } else { + s * TINY * TINY + }; + } + if ix > 0x3ff00000 { + return if hy > 0 { + s * HUGE * HUGE + } else { + s * TINY * TINY + }; + } + + /* now |1-x| is TINY <= 2**-20, suffice to compute + log(x) by x-x^2/2+x^3/3-x^4/4 */ + let t: f64 = ax - 1.0; /* t has 20 trailing zeros */ + let w: f64 = (t * t) * (0.5 - t * (0.3333333333333333333333 - t * 0.25)); + let u: f64 = IVLN2_H * t; /* ivln2_h has 21 sig. bits */ + let v: f64 = t * IVLN2_L - w * IVLN2; + t1 = with_set_low_word(u + v, 0); + t2 = v - (t1 - u); + } else { + // double ss,s2,s_h,s_l,t_h,t_l; + let mut n: i32 = 0; + + if ix < 0x00100000 { + /* take care subnormal number */ + ax *= TWO53; + n -= 53; + ix = get_high_word(ax) as i32; + } + + n += (ix >> 20) - 0x3ff; + j = ix & 0x000fffff; + + /* determine interval */ + let k: i32; + ix = j | 0x3ff00000; /* normalize ix */ + if j <= 0x3988E { + /* |x|> 1) | 0x20000000) + 0x00080000 + ((k as u32) << 18), + ); + let t_l: f64 = ax - (t_h - i!(BP, k as usize)); + let s_l: f64 = v * ((u - s_h * t_h) - s_h * t_l); + + /* compute log(ax) */ + let s2: f64 = ss * ss; + let mut r: f64 = s2 * s2 * (L1 + s2 * (L2 + s2 * (L3 + s2 * (L4 + s2 * (L5 + s2 * L6))))); + r += s_l * (s_h + ss); + let s2: f64 = s_h * s_h; + let t_h: f64 = with_set_low_word(3.0 + s2 + r, 0); + let t_l: f64 = r - ((t_h - 3.0) - s2); + + /* u+v = ss*(1+...) */ + let u: f64 = s_h * t_h; + let v: f64 = s_l * t_h + t_l * ss; + + /* 2/(3log2)*(ss+...) */ + let p_h: f64 = with_set_low_word(u + v, 0); + let p_l = v - (p_h - u); + let z_h: f64 = CP_H * p_h; /* cp_h+cp_l = 2/(3*log2) */ + let z_l: f64 = CP_L * p_h + p_l * CP + i!(DP_L, k as usize); + + /* log2(ax) = (ss+..)*2/(3*log2) = n + dp_h + z_h + z_l */ + let t: f64 = n as f64; + t1 = with_set_low_word(((z_h + z_l) + i!(DP_H, k as usize)) + t, 0); + t2 = z_l - (((t1 - t) - i!(DP_H, k as usize)) - z_h); + } + + /* split up y into y1+y2 and compute (y1+y2)*(t1+t2) */ + let y1: f64 = with_set_low_word(y, 0); + let p_l: f64 = (y - y1) * t1 + y * t2; + let mut p_h: f64 = y1 * t1; + let z: f64 = p_l + p_h; + let mut j: i32 = (z.to_bits() >> 32) as i32; + let i: i32 = z.to_bits() as i32; + // let (j, i): (i32, i32) = ((z.to_bits() >> 32) as i32, z.to_bits() as i32); + + if j >= 0x40900000 { + /* z >= 1024 */ + if (j - 0x40900000) | i != 0 { + /* if z > 1024 */ + return s * HUGE * HUGE; /* overflow */ + } + + if p_l + OVT > z - p_h { + return s * HUGE * HUGE; /* overflow */ + } + } else if (j & 0x7fffffff) >= 0x4090cc00 { + /* z <= -1075 */ + // FIXME: instead of abs(j) use unsigned j + + if (((j as u32) - 0xc090cc00) | (i as u32)) != 0 { + /* z < -1075 */ + return s * TINY * TINY; /* underflow */ + } + + if p_l <= z - p_h { + return s * TINY * TINY; /* underflow */ + } + } + + /* compute 2**(p_h+p_l) */ + let i: i32 = j & (0x7fffffff as i32); + k = (i >> 20) - 0x3ff; + let mut n: i32 = 0; + + if i > 0x3fe00000 { + /* if |z| > 0.5, set n = [z+0.5] */ + n = j + (0x00100000 >> (k + 1)); + k = ((n & 0x7fffffff) >> 20) - 0x3ff; /* new k for n */ + let t: f64 = with_set_high_word(0.0, (n & !(0x000fffff >> k)) as u32); + n = ((n & 0x000fffff) | 0x00100000) >> (20 - k); + if j < 0 { + n = -n; + } + p_h -= t; + } + + let t: f64 = with_set_low_word(p_l + p_h, 0); + let u: f64 = t * LG2_H; + let v: f64 = (p_l - (t - p_h)) * LG2 + t * LG2_L; + let mut z: f64 = u + v; + let w: f64 = v - (z - u); + let t: f64 = z * z; + let t1: f64 = z - t * (P1 + t * (P2 + t * (P3 + t * (P4 + t * P5)))); + let r: f64 = (z * t1) / (t1 - 2.0) - (w + z * w); + z = 1.0 - (r - z); + j = get_high_word(z) as i32; + j += n << 20; + + if (j >> 20) <= 0 { + /* subnormal output */ + z = scalbn(z, n); + } else { + z = with_set_high_word(z, j as u32); + } + + s * z +} + +#[cfg(test)] +mod tests { + extern crate core; + + use self::core::f64::consts::{E, PI}; + use self::core::f64::{EPSILON, INFINITY, MAX, MIN, MIN_POSITIVE, NAN, NEG_INFINITY}; + use super::pow; + + const POS_ZERO: &[f64] = &[0.0]; + const NEG_ZERO: &[f64] = &[-0.0]; + const POS_ONE: &[f64] = &[1.0]; + const NEG_ONE: &[f64] = &[-1.0]; + const POS_FLOATS: &[f64] = &[99.0 / 70.0, E, PI]; + const NEG_FLOATS: &[f64] = &[-99.0 / 70.0, -E, -PI]; + const POS_SMALL_FLOATS: &[f64] = &[(1.0 / 2.0), MIN_POSITIVE, EPSILON]; + const NEG_SMALL_FLOATS: &[f64] = &[-(1.0 / 2.0), -MIN_POSITIVE, -EPSILON]; + const POS_EVENS: &[f64] = &[2.0, 6.0, 8.0, 10.0, 22.0, 100.0, MAX]; + const NEG_EVENS: &[f64] = &[MIN, -100.0, -22.0, -10.0, -8.0, -6.0, -2.0]; + const POS_ODDS: &[f64] = &[3.0, 7.0]; + const NEG_ODDS: &[f64] = &[-7.0, -3.0]; + const NANS: &[f64] = &[NAN]; + const POS_INF: &[f64] = &[INFINITY]; + const NEG_INF: &[f64] = &[NEG_INFINITY]; + + const ALL: &[&[f64]] = &[ + POS_ZERO, + NEG_ZERO, + NANS, + NEG_SMALL_FLOATS, + POS_SMALL_FLOATS, + NEG_FLOATS, + POS_FLOATS, + NEG_EVENS, + POS_EVENS, + NEG_ODDS, + POS_ODDS, + NEG_INF, + POS_INF, + NEG_ONE, + POS_ONE, + ]; + const POS: &[&[f64]] = &[POS_ZERO, POS_ODDS, POS_ONE, POS_FLOATS, POS_EVENS, POS_INF]; + const NEG: &[&[f64]] = &[NEG_ZERO, NEG_ODDS, NEG_ONE, NEG_FLOATS, NEG_EVENS, NEG_INF]; + + fn pow_test(base: f64, exponent: f64, expected: f64) { + let res = pow(base, exponent); + assert!( + if expected.is_nan() { + res.is_nan() + } else { + pow(base, exponent) == expected + }, + "{} ** {} was {} instead of {}", + base, + exponent, + res, + expected + ); + } + + fn test_sets_as_base(sets: &[&[f64]], exponent: f64, expected: f64) { + sets.iter() + .for_each(|s| s.iter().for_each(|val| pow_test(*val, exponent, expected))); + } + + fn test_sets_as_exponent(base: f64, sets: &[&[f64]], expected: f64) { + sets.iter() + .for_each(|s| s.iter().for_each(|val| pow_test(base, *val, expected))); + } + + fn test_sets(sets: &[&[f64]], computed: &dyn Fn(f64) -> f64, expected: &dyn Fn(f64) -> f64) { + sets.iter().for_each(|s| { + s.iter().for_each(|val| { + let exp = expected(*val); + let res = computed(*val); + + #[cfg(all(target_arch = "x86", not(target_feature = "sse2")))] + let exp = force_eval!(exp); + #[cfg(all(target_arch = "x86", not(target_feature = "sse2")))] + let res = force_eval!(res); + assert!( + if exp.is_nan() { + res.is_nan() + } else { + exp == res + }, + "test for {} was {} instead of {}", + val, + res, + exp + ); + }) + }); + } + + #[test] + fn zero_as_exponent() { + test_sets_as_base(ALL, 0.0, 1.0); + test_sets_as_base(ALL, -0.0, 1.0); + } + + #[test] + fn one_as_base() { + test_sets_as_exponent(1.0, ALL, 1.0); + } + + #[test] + fn nan_inputs() { + // NAN as the base: + // (NAN ^ anything *but 0* should be NAN) + test_sets_as_exponent(NAN, &ALL[2..], NAN); + + // NAN as the exponent: + // (anything *but 1* ^ NAN should be NAN) + test_sets_as_base(&ALL[..(ALL.len() - 2)], NAN, NAN); + } + + #[test] + fn infinity_as_base() { + // Positive Infinity as the base: + // (+Infinity ^ positive anything but 0 and NAN should be +Infinity) + test_sets_as_exponent(INFINITY, &POS[1..], INFINITY); + + // (+Infinity ^ negative anything except 0 and NAN should be 0.0) + test_sets_as_exponent(INFINITY, &NEG[1..], 0.0); + + // Negative Infinity as the base: + // (-Infinity ^ positive odd ints should be -Infinity) + test_sets_as_exponent(NEG_INFINITY, &[POS_ODDS], NEG_INFINITY); + + // (-Infinity ^ anything but odd ints should be == -0 ^ (-anything)) + // We can lump in pos/neg odd ints here because they don't seem to + // cause panics (div by zero) in release mode (I think). + test_sets(ALL, &|v: f64| pow(NEG_INFINITY, v), &|v: f64| pow(-0.0, -v)); + } + + #[test] + fn infinity_as_exponent() { + // Positive/Negative base greater than 1: + // (pos/neg > 1 ^ Infinity should be Infinity - note this excludes NAN as the base) + test_sets_as_base(&ALL[5..(ALL.len() - 2)], INFINITY, INFINITY); + + // (pos/neg > 1 ^ -Infinity should be 0.0) + test_sets_as_base(&ALL[5..ALL.len() - 2], NEG_INFINITY, 0.0); + + // Positive/Negative base less than 1: + let base_below_one = &[POS_ZERO, NEG_ZERO, NEG_SMALL_FLOATS, POS_SMALL_FLOATS]; + + // (pos/neg < 1 ^ Infinity should be 0.0 - this also excludes NAN as the base) + test_sets_as_base(base_below_one, INFINITY, 0.0); + + // (pos/neg < 1 ^ -Infinity should be Infinity) + test_sets_as_base(base_below_one, NEG_INFINITY, INFINITY); + + // Positive/Negative 1 as the base: + // (pos/neg 1 ^ Infinity should be 1) + test_sets_as_base(&[NEG_ONE, POS_ONE], INFINITY, 1.0); + + // (pos/neg 1 ^ -Infinity should be 1) + test_sets_as_base(&[NEG_ONE, POS_ONE], NEG_INFINITY, 1.0); + } + + #[test] + fn zero_as_base() { + // Positive Zero as the base: + // (+0 ^ anything positive but 0 and NAN should be +0) + test_sets_as_exponent(0.0, &POS[1..], 0.0); + + // (+0 ^ anything negative but 0 and NAN should be Infinity) + // (this should panic because we're dividing by zero) + test_sets_as_exponent(0.0, &NEG[1..], INFINITY); + + // Negative Zero as the base: + // (-0 ^ anything positive but 0, NAN, and odd ints should be +0) + test_sets_as_exponent(-0.0, &POS[3..], 0.0); + + // (-0 ^ anything negative but 0, NAN, and odd ints should be Infinity) + // (should panic because of divide by zero) + test_sets_as_exponent(-0.0, &NEG[3..], INFINITY); + + // (-0 ^ positive odd ints should be -0) + test_sets_as_exponent(-0.0, &[POS_ODDS], -0.0); + + // (-0 ^ negative odd ints should be -Infinity) + // (should panic because of divide by zero) + test_sets_as_exponent(-0.0, &[NEG_ODDS], NEG_INFINITY); + } + + #[test] + fn special_cases() { + // One as the exponent: + // (anything ^ 1 should be anything - i.e. the base) + test_sets(ALL, &|v: f64| pow(v, 1.0), &|v: f64| v); + + // Negative One as the exponent: + // (anything ^ -1 should be 1/anything) + test_sets(ALL, &|v: f64| pow(v, -1.0), &|v: f64| 1.0 / v); + + // Factoring -1 out: + // (negative anything ^ integer should be (-1 ^ integer) * (positive anything ^ integer)) + (&[POS_ZERO, NEG_ZERO, POS_ONE, NEG_ONE, POS_EVENS, NEG_EVENS]) + .iter() + .for_each(|int_set| { + int_set.iter().for_each(|int| { + test_sets(ALL, &|v: f64| pow(-v, *int), &|v: f64| { + pow(-1.0, *int) * pow(v, *int) + }); + }) + }); + + // Negative base (imaginary results): + // (-anything except 0 and Infinity ^ non-integer should be NAN) + (&NEG[1..(NEG.len() - 1)]).iter().for_each(|set| { + set.iter().for_each(|val| { + test_sets(&ALL[3..7], &|v: f64| pow(*val, v), &|_| NAN); + }) + }); + } + + #[test] + fn normal_cases() { + assert_eq!(pow(2.0, 20.0), (1 << 20) as f64); + assert_eq!(pow(-1.0, 9.0), -1.0); + assert!(pow(-1.0, 2.2).is_nan()); + assert!(pow(-1.0, -1.14).is_nan()); + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/powf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/powf.rs new file mode 100644 index 000000000..68d2083bb --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/powf.rs @@ -0,0 +1,342 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_powf.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +use super::{fabsf, scalbnf, sqrtf}; + +const BP: [f32; 2] = [1.0, 1.5]; +const DP_H: [f32; 2] = [0.0, 5.84960938e-01]; /* 0x3f15c000 */ +const DP_L: [f32; 2] = [0.0, 1.56322085e-06]; /* 0x35d1cfdc */ +const TWO24: f32 = 16777216.0; /* 0x4b800000 */ +const HUGE: f32 = 1.0e30; +const TINY: f32 = 1.0e-30; +const L1: f32 = 6.0000002384e-01; /* 0x3f19999a */ +const L2: f32 = 4.2857143283e-01; /* 0x3edb6db7 */ +const L3: f32 = 3.3333334327e-01; /* 0x3eaaaaab */ +const L4: f32 = 2.7272811532e-01; /* 0x3e8ba305 */ +const L5: f32 = 2.3066075146e-01; /* 0x3e6c3255 */ +const L6: f32 = 2.0697501302e-01; /* 0x3e53f142 */ +const P1: f32 = 1.6666667163e-01; /* 0x3e2aaaab */ +const P2: f32 = -2.7777778450e-03; /* 0xbb360b61 */ +const P3: f32 = 6.6137559770e-05; /* 0x388ab355 */ +const P4: f32 = -1.6533901999e-06; /* 0xb5ddea0e */ +const P5: f32 = 4.1381369442e-08; /* 0x3331bb4c */ +const LG2: f32 = 6.9314718246e-01; /* 0x3f317218 */ +const LG2_H: f32 = 6.93145752e-01; /* 0x3f317200 */ +const LG2_L: f32 = 1.42860654e-06; /* 0x35bfbe8c */ +const OVT: f32 = 4.2995665694e-08; /* -(128-log2(ovfl+.5ulp)) */ +const CP: f32 = 9.6179670095e-01; /* 0x3f76384f =2/(3ln2) */ +const CP_H: f32 = 9.6191406250e-01; /* 0x3f764000 =12b cp */ +const CP_L: f32 = -1.1736857402e-04; /* 0xb8f623c6 =tail of cp_h */ +const IVLN2: f32 = 1.4426950216e+00; +const IVLN2_H: f32 = 1.4426879883e+00; +const IVLN2_L: f32 = 7.0526075433e-06; + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn powf(x: f32, y: f32) -> f32 { + let mut z: f32; + let mut ax: f32; + let z_h: f32; + let z_l: f32; + let mut p_h: f32; + let mut p_l: f32; + let y1: f32; + let mut t1: f32; + let t2: f32; + let mut r: f32; + let s: f32; + let mut sn: f32; + let mut t: f32; + let mut u: f32; + let mut v: f32; + let mut w: f32; + let i: i32; + let mut j: i32; + let mut k: i32; + let mut yisint: i32; + let mut n: i32; + let hx: i32; + let hy: i32; + let mut ix: i32; + let iy: i32; + let mut is: i32; + + hx = x.to_bits() as i32; + hy = y.to_bits() as i32; + + ix = hx & 0x7fffffff; + iy = hy & 0x7fffffff; + + /* x**0 = 1, even if x is NaN */ + if iy == 0 { + return 1.0; + } + + /* 1**y = 1, even if y is NaN */ + if hx == 0x3f800000 { + return 1.0; + } + + /* NaN if either arg is NaN */ + if ix > 0x7f800000 || iy > 0x7f800000 { + return x + y; + } + + /* determine if y is an odd int when x < 0 + * yisint = 0 ... y is not an integer + * yisint = 1 ... y is an odd int + * yisint = 2 ... y is an even int + */ + yisint = 0; + if hx < 0 { + if iy >= 0x4b800000 { + yisint = 2; /* even integer y */ + } else if iy >= 0x3f800000 { + k = (iy >> 23) - 0x7f; /* exponent */ + j = iy >> (23 - k); + if (j << (23 - k)) == iy { + yisint = 2 - (j & 1); + } + } + } + + /* special value of y */ + if iy == 0x7f800000 { + /* y is +-inf */ + if ix == 0x3f800000 { + /* (-1)**+-inf is 1 */ + return 1.0; + } else if ix > 0x3f800000 { + /* (|x|>1)**+-inf = inf,0 */ + return if hy >= 0 { y } else { 0.0 }; + } else { + /* (|x|<1)**+-inf = 0,inf */ + return if hy >= 0 { 0.0 } else { -y }; + } + } + if iy == 0x3f800000 { + /* y is +-1 */ + return if hy >= 0 { x } else { 1.0 / x }; + } + + if hy == 0x40000000 { + /* y is 2 */ + return x * x; + } + + if hy == 0x3f000000 + /* y is 0.5 */ + && hx >= 0 + { + /* x >= +0 */ + return sqrtf(x); + } + + ax = fabsf(x); + /* special value of x */ + if ix == 0x7f800000 || ix == 0 || ix == 0x3f800000 { + /* x is +-0,+-inf,+-1 */ + z = ax; + if hy < 0 { + /* z = (1/|x|) */ + z = 1.0 / z; + } + + if hx < 0 { + if ((ix - 0x3f800000) | yisint) == 0 { + z = (z - z) / (z - z); /* (-1)**non-int is NaN */ + } else if yisint == 1 { + z = -z; /* (x<0)**odd = -(|x|**odd) */ + } + } + return z; + } + + sn = 1.0; /* sign of result */ + if hx < 0 { + if yisint == 0 { + /* (x<0)**(non-int) is NaN */ + return (x - x) / (x - x); + } + + if yisint == 1 { + /* (x<0)**(odd int) */ + sn = -1.0; + } + } + + /* |y| is HUGE */ + if iy > 0x4d000000 { + /* if |y| > 2**27 */ + /* over/underflow if x is not close to one */ + if ix < 0x3f7ffff8 { + return if hy < 0 { + sn * HUGE * HUGE + } else { + sn * TINY * TINY + }; + } + + if ix > 0x3f800007 { + return if hy > 0 { + sn * HUGE * HUGE + } else { + sn * TINY * TINY + }; + } + + /* now |1-x| is TINY <= 2**-20, suffice to compute + log(x) by x-x^2/2+x^3/3-x^4/4 */ + t = ax - 1.; /* t has 20 trailing zeros */ + w = (t * t) * (0.5 - t * (0.333333333333 - t * 0.25)); + u = IVLN2_H * t; /* IVLN2_H has 16 sig. bits */ + v = t * IVLN2_L - w * IVLN2; + t1 = u + v; + is = t1.to_bits() as i32; + t1 = f32::from_bits(is as u32 & 0xfffff000); + t2 = v - (t1 - u); + } else { + let mut s2: f32; + let mut s_h: f32; + let s_l: f32; + let mut t_h: f32; + let mut t_l: f32; + + n = 0; + /* take care subnormal number */ + if ix < 0x00800000 { + ax *= TWO24; + n -= 24; + ix = ax.to_bits() as i32; + } + n += ((ix) >> 23) - 0x7f; + j = ix & 0x007fffff; + /* determine interval */ + ix = j | 0x3f800000; /* normalize ix */ + if j <= 0x1cc471 { + /* |x|> 1) & 0xfffff000) | 0x20000000) as i32; + t_h = f32::from_bits(is as u32 + 0x00400000 + ((k as u32) << 21)); + t_l = ax - (t_h - i!(BP, k as usize)); + s_l = v * ((u - s_h * t_h) - s_h * t_l); + /* compute log(ax) */ + s2 = s * s; + r = s2 * s2 * (L1 + s2 * (L2 + s2 * (L3 + s2 * (L4 + s2 * (L5 + s2 * L6))))); + r += s_l * (s_h + s); + s2 = s_h * s_h; + t_h = 3.0 + s2 + r; + is = t_h.to_bits() as i32; + t_h = f32::from_bits(is as u32 & 0xfffff000); + t_l = r - ((t_h - 3.0) - s2); + /* u+v = s*(1+...) */ + u = s_h * t_h; + v = s_l * t_h + t_l * s; + /* 2/(3log2)*(s+...) */ + p_h = u + v; + is = p_h.to_bits() as i32; + p_h = f32::from_bits(is as u32 & 0xfffff000); + p_l = v - (p_h - u); + z_h = CP_H * p_h; /* cp_h+cp_l = 2/(3*log2) */ + z_l = CP_L * p_h + p_l * CP + i!(DP_L, k as usize); + /* log2(ax) = (s+..)*2/(3*log2) = n + dp_h + z_h + z_l */ + t = n as f32; + t1 = ((z_h + z_l) + i!(DP_H, k as usize)) + t; + is = t1.to_bits() as i32; + t1 = f32::from_bits(is as u32 & 0xfffff000); + t2 = z_l - (((t1 - t) - i!(DP_H, k as usize)) - z_h); + }; + + /* split up y into y1+y2 and compute (y1+y2)*(t1+t2) */ + is = y.to_bits() as i32; + y1 = f32::from_bits(is as u32 & 0xfffff000); + p_l = (y - y1) * t1 + y * t2; + p_h = y1 * t1; + z = p_l + p_h; + j = z.to_bits() as i32; + if j > 0x43000000 { + /* if z > 128 */ + return sn * HUGE * HUGE; /* overflow */ + } else if j == 0x43000000 { + /* if z == 128 */ + if p_l + OVT > z - p_h { + return sn * HUGE * HUGE; /* overflow */ + } + } else if (j & 0x7fffffff) > 0x43160000 { + /* z < -150 */ + // FIXME: check should be (uint32_t)j > 0xc3160000 + return sn * TINY * TINY; /* underflow */ + } else if j as u32 == 0xc3160000 + /* z == -150 */ + && p_l <= z - p_h + { + return sn * TINY * TINY; /* underflow */ + } + + /* + * compute 2**(p_h+p_l) + */ + i = j & 0x7fffffff; + k = (i >> 23) - 0x7f; + n = 0; + if i > 0x3f000000 { + /* if |z| > 0.5, set n = [z+0.5] */ + n = j + (0x00800000 >> (k + 1)); + k = ((n & 0x7fffffff) >> 23) - 0x7f; /* new k for n */ + t = f32::from_bits(n as u32 & !(0x007fffff >> k)); + n = ((n & 0x007fffff) | 0x00800000) >> (23 - k); + if j < 0 { + n = -n; + } + p_h -= t; + } + t = p_l + p_h; + is = t.to_bits() as i32; + t = f32::from_bits(is as u32 & 0xffff8000); + u = t * LG2_H; + v = (p_l - (t - p_h)) * LG2 + t * LG2_L; + z = u + v; + w = v - (z - u); + t = z * z; + t1 = z - t * (P1 + t * (P2 + t * (P3 + t * (P4 + t * P5)))); + r = (z * t1) / (t1 - 2.0) - (w + z * w); + z = 1.0 - (r - z); + j = z.to_bits() as i32; + j += n << 23; + if (j >> 23) <= 0 { + /* subnormal output */ + z = scalbnf(z, n); + } else { + z = f32::from_bits(j as u32); + } + sn * z +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/rem_pio2.rs b/crux-mir/lib/compiler_builtins/libm/src/math/rem_pio2.rs new file mode 100644 index 000000000..644616f2d --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/rem_pio2.rs @@ -0,0 +1,233 @@ +// origin: FreeBSD /usr/src/lib/msun/src/e_rem_pio2.c +// +// ==================================================== +// Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. +// +// Developed at SunPro, a Sun Microsystems, Inc. business. +// Permission to use, copy, modify, and distribute this +// software is freely granted, provided that this notice +// is preserved. +// ==================================================== +// +// Optimized by Bruce D. Evans. */ +use super::rem_pio2_large; + +// #if FLT_EVAL_METHOD==0 || FLT_EVAL_METHOD==1 +// #define EPS DBL_EPSILON +const EPS: f64 = 2.2204460492503131e-16; +// #elif FLT_EVAL_METHOD==2 +// #define EPS LDBL_EPSILON +// #endif + +// TODO: Support FLT_EVAL_METHOD? + +const TO_INT: f64 = 1.5 / EPS; +/// 53 bits of 2/pi +const INV_PIO2: f64 = 6.36619772367581382433e-01; /* 0x3FE45F30, 0x6DC9C883 */ +/// first 33 bits of pi/2 +const PIO2_1: f64 = 1.57079632673412561417e+00; /* 0x3FF921FB, 0x54400000 */ +/// pi/2 - PIO2_1 +const PIO2_1T: f64 = 6.07710050650619224932e-11; /* 0x3DD0B461, 0x1A626331 */ +/// second 33 bits of pi/2 +const PIO2_2: f64 = 6.07710050630396597660e-11; /* 0x3DD0B461, 0x1A600000 */ +/// pi/2 - (PIO2_1+PIO2_2) +const PIO2_2T: f64 = 2.02226624879595063154e-21; /* 0x3BA3198A, 0x2E037073 */ +/// third 33 bits of pi/2 +const PIO2_3: f64 = 2.02226624871116645580e-21; /* 0x3BA3198A, 0x2E000000 */ +/// pi/2 - (PIO2_1+PIO2_2+PIO2_3) +const PIO2_3T: f64 = 8.47842766036889956997e-32; /* 0x397B839A, 0x252049C1 */ + +// return the remainder of x rem pi/2 in y[0]+y[1] +// use rem_pio2_large() for large x +// +// caller must handle the case when reduction is not needed: |x| ~<= pi/4 */ +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub(crate) fn rem_pio2(x: f64) -> (i32, f64, f64) { + let x1p24 = f64::from_bits(0x4170000000000000); + + let sign = (f64::to_bits(x) >> 63) as i32; + let ix = (f64::to_bits(x) >> 32) as u32 & 0x7fffffff; + + fn medium(x: f64, ix: u32) -> (i32, f64, f64) { + /* rint(x/(pi/2)), Assume round-to-nearest. */ + let tmp = x as f64 * INV_PIO2 + TO_INT; + // force rounding of tmp to it's storage format on x87 to avoid + // excess precision issues. + #[cfg(all(target_arch = "x86", not(target_feature = "sse2")))] + let tmp = force_eval!(tmp); + let f_n = tmp - TO_INT; + let n = f_n as i32; + let mut r = x - f_n * PIO2_1; + let mut w = f_n * PIO2_1T; /* 1st round, good to 85 bits */ + let mut y0 = r - w; + let ui = f64::to_bits(y0); + let ey = (ui >> 52) as i32 & 0x7ff; + let ex = (ix >> 20) as i32; + if ex - ey > 16 { + /* 2nd round, good to 118 bits */ + let t = r; + w = f_n * PIO2_2; + r = t - w; + w = f_n * PIO2_2T - ((t - r) - w); + y0 = r - w; + let ey = (f64::to_bits(y0) >> 52) as i32 & 0x7ff; + if ex - ey > 49 { + /* 3rd round, good to 151 bits, covers all cases */ + let t = r; + w = f_n * PIO2_3; + r = t - w; + w = f_n * PIO2_3T - ((t - r) - w); + y0 = r - w; + } + } + let y1 = (r - y0) - w; + (n, y0, y1) + } + + if ix <= 0x400f6a7a { + /* |x| ~<= 5pi/4 */ + if (ix & 0xfffff) == 0x921fb { + /* |x| ~= pi/2 or 2pi/2 */ + return medium(x, ix); /* cancellation -- use medium case */ + } + if ix <= 0x4002d97c { + /* |x| ~<= 3pi/4 */ + if sign == 0 { + let z = x - PIO2_1; /* one round good to 85 bits */ + let y0 = z - PIO2_1T; + let y1 = (z - y0) - PIO2_1T; + return (1, y0, y1); + } else { + let z = x + PIO2_1; + let y0 = z + PIO2_1T; + let y1 = (z - y0) + PIO2_1T; + return (-1, y0, y1); + } + } else if sign == 0 { + let z = x - 2.0 * PIO2_1; + let y0 = z - 2.0 * PIO2_1T; + let y1 = (z - y0) - 2.0 * PIO2_1T; + return (2, y0, y1); + } else { + let z = x + 2.0 * PIO2_1; + let y0 = z + 2.0 * PIO2_1T; + let y1 = (z - y0) + 2.0 * PIO2_1T; + return (-2, y0, y1); + } + } + if ix <= 0x401c463b { + /* |x| ~<= 9pi/4 */ + if ix <= 0x4015fdbc { + /* |x| ~<= 7pi/4 */ + if ix == 0x4012d97c { + /* |x| ~= 3pi/2 */ + return medium(x, ix); + } + if sign == 0 { + let z = x - 3.0 * PIO2_1; + let y0 = z - 3.0 * PIO2_1T; + let y1 = (z - y0) - 3.0 * PIO2_1T; + return (3, y0, y1); + } else { + let z = x + 3.0 * PIO2_1; + let y0 = z + 3.0 * PIO2_1T; + let y1 = (z - y0) + 3.0 * PIO2_1T; + return (-3, y0, y1); + } + } else { + if ix == 0x401921fb { + /* |x| ~= 4pi/2 */ + return medium(x, ix); + } + if sign == 0 { + let z = x - 4.0 * PIO2_1; + let y0 = z - 4.0 * PIO2_1T; + let y1 = (z - y0) - 4.0 * PIO2_1T; + return (4, y0, y1); + } else { + let z = x + 4.0 * PIO2_1; + let y0 = z + 4.0 * PIO2_1T; + let y1 = (z - y0) + 4.0 * PIO2_1T; + return (-4, y0, y1); + } + } + } + if ix < 0x413921fb { + /* |x| ~< 2^20*(pi/2), medium size */ + return medium(x, ix); + } + /* + * all other (large) arguments + */ + if ix >= 0x7ff00000 { + /* x is inf or NaN */ + let y0 = x - x; + let y1 = y0; + return (0, y0, y1); + } + /* set z = scalbn(|x|,-ilogb(x)+23) */ + let mut ui = f64::to_bits(x); + ui &= (!1) >> 12; + ui |= (0x3ff + 23) << 52; + let mut z = f64::from_bits(ui); + let mut tx = [0.0; 3]; + for i in 0..2 { + i!(tx,i, =, z as i32 as f64); + z = (z - i!(tx, i)) * x1p24; + } + i!(tx,2, =, z); + /* skip zero terms, first term is non-zero */ + let mut i = 2; + while i != 0 && i!(tx, i) == 0.0 { + i -= 1; + } + let mut ty = [0.0; 3]; + let n = rem_pio2_large(&tx[..=i], &mut ty, ((ix as i32) >> 20) - (0x3ff + 23), 1); + if sign != 0 { + return (-n, -i!(ty, 0), -i!(ty, 1)); + } + (n, i!(ty, 0), i!(ty, 1)) +} + +#[cfg(test)] +mod tests { + use super::rem_pio2; + + #[test] + fn test_near_pi() { + let arg = 3.141592025756836; + let arg = force_eval!(arg); + assert_eq!( + rem_pio2(arg), + (2, -6.278329573009626e-7, -2.1125998133974653e-23) + ); + let arg = 3.141592033207416; + let arg = force_eval!(arg); + assert_eq!( + rem_pio2(arg), + (2, -6.20382377148128e-7, -2.1125998133974653e-23) + ); + let arg = 3.141592144966125; + let arg = force_eval!(arg); + assert_eq!( + rem_pio2(arg), + (2, -5.086236681942706e-7, -2.1125998133974653e-23) + ); + let arg = 3.141592979431152; + let arg = force_eval!(arg); + assert_eq!( + rem_pio2(arg), + (2, 3.2584135866119817e-7, -2.1125998133974653e-23) + ); + } + + #[test] + fn test_overflow_b9b847() { + let _ = rem_pio2(-3054214.5490637687); + } + + #[test] + fn test_overflow_4747b9() { + let _ = rem_pio2(917340800458.2274); + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/rem_pio2_large.rs b/crux-mir/lib/compiler_builtins/libm/src/math/rem_pio2_large.rs new file mode 100644 index 000000000..db97a39d4 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/rem_pio2_large.rs @@ -0,0 +1,470 @@ +#![allow(unused_unsafe)] +/* origin: FreeBSD /usr/src/lib/msun/src/k_rem_pio2.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunSoft, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +use super::floor; +use super::scalbn; + +// initial value for jk +const INIT_JK: [usize; 4] = [3, 4, 4, 6]; + +// Table of constants for 2/pi, 396 Hex digits (476 decimal) of 2/pi +// +// integer array, contains the (24*i)-th to (24*i+23)-th +// bit of 2/pi after binary point. The corresponding +// floating value is +// +// ipio2[i] * 2^(-24(i+1)). +// +// NB: This table must have at least (e0-3)/24 + jk terms. +// For quad precision (e0 <= 16360, jk = 6), this is 686. +#[cfg(any(target_pointer_width = "32", target_pointer_width = "16"))] +const IPIO2: [i32; 66] = [ + 0xA2F983, 0x6E4E44, 0x1529FC, 0x2757D1, 0xF534DD, 0xC0DB62, 0x95993C, 0x439041, 0xFE5163, + 0xABDEBB, 0xC561B7, 0x246E3A, 0x424DD2, 0xE00649, 0x2EEA09, 0xD1921C, 0xFE1DEB, 0x1CB129, + 0xA73EE8, 0x8235F5, 0x2EBB44, 0x84E99C, 0x7026B4, 0x5F7E41, 0x3991D6, 0x398353, 0x39F49C, + 0x845F8B, 0xBDF928, 0x3B1FF8, 0x97FFDE, 0x05980F, 0xEF2F11, 0x8B5A0A, 0x6D1F6D, 0x367ECF, + 0x27CB09, 0xB74F46, 0x3F669E, 0x5FEA2D, 0x7527BA, 0xC7EBE5, 0xF17B3D, 0x0739F7, 0x8A5292, + 0xEA6BFB, 0x5FB11F, 0x8D5D08, 0x560330, 0x46FC7B, 0x6BABF0, 0xCFBC20, 0x9AF436, 0x1DA9E3, + 0x91615E, 0xE61B08, 0x659985, 0x5F14A0, 0x68408D, 0xFFD880, 0x4D7327, 0x310606, 0x1556CA, + 0x73A8C9, 0x60E27B, 0xC08C6B, +]; + +#[cfg(target_pointer_width = "64")] +const IPIO2: [i32; 690] = [ + 0xA2F983, 0x6E4E44, 0x1529FC, 0x2757D1, 0xF534DD, 0xC0DB62, 0x95993C, 0x439041, 0xFE5163, + 0xABDEBB, 0xC561B7, 0x246E3A, 0x424DD2, 0xE00649, 0x2EEA09, 0xD1921C, 0xFE1DEB, 0x1CB129, + 0xA73EE8, 0x8235F5, 0x2EBB44, 0x84E99C, 0x7026B4, 0x5F7E41, 0x3991D6, 0x398353, 0x39F49C, + 0x845F8B, 0xBDF928, 0x3B1FF8, 0x97FFDE, 0x05980F, 0xEF2F11, 0x8B5A0A, 0x6D1F6D, 0x367ECF, + 0x27CB09, 0xB74F46, 0x3F669E, 0x5FEA2D, 0x7527BA, 0xC7EBE5, 0xF17B3D, 0x0739F7, 0x8A5292, + 0xEA6BFB, 0x5FB11F, 0x8D5D08, 0x560330, 0x46FC7B, 0x6BABF0, 0xCFBC20, 0x9AF436, 0x1DA9E3, + 0x91615E, 0xE61B08, 0x659985, 0x5F14A0, 0x68408D, 0xFFD880, 0x4D7327, 0x310606, 0x1556CA, + 0x73A8C9, 0x60E27B, 0xC08C6B, 0x47C419, 0xC367CD, 0xDCE809, 0x2A8359, 0xC4768B, 0x961CA6, + 0xDDAF44, 0xD15719, 0x053EA5, 0xFF0705, 0x3F7E33, 0xE832C2, 0xDE4F98, 0x327DBB, 0xC33D26, + 0xEF6B1E, 0x5EF89F, 0x3A1F35, 0xCAF27F, 0x1D87F1, 0x21907C, 0x7C246A, 0xFA6ED5, 0x772D30, + 0x433B15, 0xC614B5, 0x9D19C3, 0xC2C4AD, 0x414D2C, 0x5D000C, 0x467D86, 0x2D71E3, 0x9AC69B, + 0x006233, 0x7CD2B4, 0x97A7B4, 0xD55537, 0xF63ED7, 0x1810A3, 0xFC764D, 0x2A9D64, 0xABD770, + 0xF87C63, 0x57B07A, 0xE71517, 0x5649C0, 0xD9D63B, 0x3884A7, 0xCB2324, 0x778AD6, 0x23545A, + 0xB91F00, 0x1B0AF1, 0xDFCE19, 0xFF319F, 0x6A1E66, 0x615799, 0x47FBAC, 0xD87F7E, 0xB76522, + 0x89E832, 0x60BFE6, 0xCDC4EF, 0x09366C, 0xD43F5D, 0xD7DE16, 0xDE3B58, 0x929BDE, 0x2822D2, + 0xE88628, 0x4D58E2, 0x32CAC6, 0x16E308, 0xCB7DE0, 0x50C017, 0xA71DF3, 0x5BE018, 0x34132E, + 0x621283, 0x014883, 0x5B8EF5, 0x7FB0AD, 0xF2E91E, 0x434A48, 0xD36710, 0xD8DDAA, 0x425FAE, + 0xCE616A, 0xA4280A, 0xB499D3, 0xF2A606, 0x7F775C, 0x83C2A3, 0x883C61, 0x78738A, 0x5A8CAF, + 0xBDD76F, 0x63A62D, 0xCBBFF4, 0xEF818D, 0x67C126, 0x45CA55, 0x36D9CA, 0xD2A828, 0x8D61C2, + 0x77C912, 0x142604, 0x9B4612, 0xC459C4, 0x44C5C8, 0x91B24D, 0xF31700, 0xAD43D4, 0xE54929, + 0x10D5FD, 0xFCBE00, 0xCC941E, 0xEECE70, 0xF53E13, 0x80F1EC, 0xC3E7B3, 0x28F8C7, 0x940593, + 0x3E71C1, 0xB3092E, 0xF3450B, 0x9C1288, 0x7B20AB, 0x9FB52E, 0xC29247, 0x2F327B, 0x6D550C, + 0x90A772, 0x1FE76B, 0x96CB31, 0x4A1679, 0xE27941, 0x89DFF4, 0x9794E8, 0x84E6E2, 0x973199, + 0x6BED88, 0x365F5F, 0x0EFDBB, 0xB49A48, 0x6CA467, 0x427271, 0x325D8D, 0xB8159F, 0x09E5BC, + 0x25318D, 0x3974F7, 0x1C0530, 0x010C0D, 0x68084B, 0x58EE2C, 0x90AA47, 0x02E774, 0x24D6BD, + 0xA67DF7, 0x72486E, 0xEF169F, 0xA6948E, 0xF691B4, 0x5153D1, 0xF20ACF, 0x339820, 0x7E4BF5, + 0x6863B2, 0x5F3EDD, 0x035D40, 0x7F8985, 0x295255, 0xC06437, 0x10D86D, 0x324832, 0x754C5B, + 0xD4714E, 0x6E5445, 0xC1090B, 0x69F52A, 0xD56614, 0x9D0727, 0x50045D, 0xDB3BB4, 0xC576EA, + 0x17F987, 0x7D6B49, 0xBA271D, 0x296996, 0xACCCC6, 0x5414AD, 0x6AE290, 0x89D988, 0x50722C, + 0xBEA404, 0x940777, 0x7030F3, 0x27FC00, 0xA871EA, 0x49C266, 0x3DE064, 0x83DD97, 0x973FA3, + 0xFD9443, 0x8C860D, 0xDE4131, 0x9D3992, 0x8C70DD, 0xE7B717, 0x3BDF08, 0x2B3715, 0xA0805C, + 0x93805A, 0x921110, 0xD8E80F, 0xAF806C, 0x4BFFDB, 0x0F9038, 0x761859, 0x15A562, 0xBBCB61, + 0xB989C7, 0xBD4010, 0x04F2D2, 0x277549, 0xF6B6EB, 0xBB22DB, 0xAA140A, 0x2F2689, 0x768364, + 0x333B09, 0x1A940E, 0xAA3A51, 0xC2A31D, 0xAEEDAF, 0x12265C, 0x4DC26D, 0x9C7A2D, 0x9756C0, + 0x833F03, 0xF6F009, 0x8C402B, 0x99316D, 0x07B439, 0x15200C, 0x5BC3D8, 0xC492F5, 0x4BADC6, + 0xA5CA4E, 0xCD37A7, 0x36A9E6, 0x9492AB, 0x6842DD, 0xDE6319, 0xEF8C76, 0x528B68, 0x37DBFC, + 0xABA1AE, 0x3115DF, 0xA1AE00, 0xDAFB0C, 0x664D64, 0xB705ED, 0x306529, 0xBF5657, 0x3AFF47, + 0xB9F96A, 0xF3BE75, 0xDF9328, 0x3080AB, 0xF68C66, 0x15CB04, 0x0622FA, 0x1DE4D9, 0xA4B33D, + 0x8F1B57, 0x09CD36, 0xE9424E, 0xA4BE13, 0xB52333, 0x1AAAF0, 0xA8654F, 0xA5C1D2, 0x0F3F0B, + 0xCD785B, 0x76F923, 0x048B7B, 0x721789, 0x53A6C6, 0xE26E6F, 0x00EBEF, 0x584A9B, 0xB7DAC4, + 0xBA66AA, 0xCFCF76, 0x1D02D1, 0x2DF1B1, 0xC1998C, 0x77ADC3, 0xDA4886, 0xA05DF7, 0xF480C6, + 0x2FF0AC, 0x9AECDD, 0xBC5C3F, 0x6DDED0, 0x1FC790, 0xB6DB2A, 0x3A25A3, 0x9AAF00, 0x9353AD, + 0x0457B6, 0xB42D29, 0x7E804B, 0xA707DA, 0x0EAA76, 0xA1597B, 0x2A1216, 0x2DB7DC, 0xFDE5FA, + 0xFEDB89, 0xFDBE89, 0x6C76E4, 0xFCA906, 0x70803E, 0x156E85, 0xFF87FD, 0x073E28, 0x336761, + 0x86182A, 0xEABD4D, 0xAFE7B3, 0x6E6D8F, 0x396795, 0x5BBF31, 0x48D784, 0x16DF30, 0x432DC7, + 0x356125, 0xCE70C9, 0xB8CB30, 0xFD6CBF, 0xA200A4, 0xE46C05, 0xA0DD5A, 0x476F21, 0xD21262, + 0x845CB9, 0x496170, 0xE0566B, 0x015299, 0x375550, 0xB7D51E, 0xC4F133, 0x5F6E13, 0xE4305D, + 0xA92E85, 0xC3B21D, 0x3632A1, 0xA4B708, 0xD4B1EA, 0x21F716, 0xE4698F, 0x77FF27, 0x80030C, + 0x2D408D, 0xA0CD4F, 0x99A520, 0xD3A2B3, 0x0A5D2F, 0x42F9B4, 0xCBDA11, 0xD0BE7D, 0xC1DB9B, + 0xBD17AB, 0x81A2CA, 0x5C6A08, 0x17552E, 0x550027, 0xF0147F, 0x8607E1, 0x640B14, 0x8D4196, + 0xDEBE87, 0x2AFDDA, 0xB6256B, 0x34897B, 0xFEF305, 0x9EBFB9, 0x4F6A68, 0xA82A4A, 0x5AC44F, + 0xBCF82D, 0x985AD7, 0x95C7F4, 0x8D4D0D, 0xA63A20, 0x5F57A4, 0xB13F14, 0x953880, 0x0120CC, + 0x86DD71, 0xB6DEC9, 0xF560BF, 0x11654D, 0x6B0701, 0xACB08C, 0xD0C0B2, 0x485551, 0x0EFB1E, + 0xC37295, 0x3B06A3, 0x3540C0, 0x7BDC06, 0xCC45E0, 0xFA294E, 0xC8CAD6, 0x41F3E8, 0xDE647C, + 0xD8649B, 0x31BED9, 0xC397A4, 0xD45877, 0xC5E369, 0x13DAF0, 0x3C3ABA, 0x461846, 0x5F7555, + 0xF5BDD2, 0xC6926E, 0x5D2EAC, 0xED440E, 0x423E1C, 0x87C461, 0xE9FD29, 0xF3D6E7, 0xCA7C22, + 0x35916F, 0xC5E008, 0x8DD7FF, 0xE26A6E, 0xC6FDB0, 0xC10893, 0x745D7C, 0xB2AD6B, 0x9D6ECD, + 0x7B723E, 0x6A11C6, 0xA9CFF7, 0xDF7329, 0xBAC9B5, 0x5100B7, 0x0DB2E2, 0x24BA74, 0x607DE5, + 0x8AD874, 0x2C150D, 0x0C1881, 0x94667E, 0x162901, 0x767A9F, 0xBEFDFD, 0xEF4556, 0x367ED9, + 0x13D9EC, 0xB9BA8B, 0xFC97C4, 0x27A831, 0xC36EF1, 0x36C594, 0x56A8D8, 0xB5A8B4, 0x0ECCCF, + 0x2D8912, 0x34576F, 0x89562C, 0xE3CE99, 0xB920D6, 0xAA5E6B, 0x9C2A3E, 0xCC5F11, 0x4A0BFD, + 0xFBF4E1, 0x6D3B8E, 0x2C86E2, 0x84D4E9, 0xA9B4FC, 0xD1EEEF, 0xC9352E, 0x61392F, 0x442138, + 0xC8D91B, 0x0AFC81, 0x6A4AFB, 0xD81C2F, 0x84B453, 0x8C994E, 0xCC2254, 0xDC552A, 0xD6C6C0, + 0x96190B, 0xB8701A, 0x649569, 0x605A26, 0xEE523F, 0x0F117F, 0x11B5F4, 0xF5CBFC, 0x2DBC34, + 0xEEBC34, 0xCC5DE8, 0x605EDD, 0x9B8E67, 0xEF3392, 0xB817C9, 0x9B5861, 0xBC57E1, 0xC68351, + 0x103ED8, 0x4871DD, 0xDD1C2D, 0xA118AF, 0x462C21, 0xD7F359, 0x987AD9, 0xC0549E, 0xFA864F, + 0xFC0656, 0xAE79E5, 0x362289, 0x22AD38, 0xDC9367, 0xAAE855, 0x382682, 0x9BE7CA, 0xA40D51, + 0xB13399, 0x0ED7A9, 0x480569, 0xF0B265, 0xA7887F, 0x974C88, 0x36D1F9, 0xB39221, 0x4A827B, + 0x21CF98, 0xDC9F40, 0x5547DC, 0x3A74E1, 0x42EB67, 0xDF9DFE, 0x5FD45E, 0xA4677B, 0x7AACBA, + 0xA2F655, 0x23882B, 0x55BA41, 0x086E59, 0x862A21, 0x834739, 0xE6E389, 0xD49EE5, 0x40FB49, + 0xE956FF, 0xCA0F1C, 0x8A59C5, 0x2BFA94, 0xC5C1D3, 0xCFC50F, 0xAE5ADB, 0x86C547, 0x624385, + 0x3B8621, 0x94792C, 0x876110, 0x7B4C2A, 0x1A2C80, 0x12BF43, 0x902688, 0x893C78, 0xE4C4A8, + 0x7BDBE5, 0xC23AC4, 0xEAF426, 0x8A67F7, 0xBF920D, 0x2BA365, 0xB1933D, 0x0B7CBD, 0xDC51A4, + 0x63DD27, 0xDDE169, 0x19949A, 0x9529A8, 0x28CE68, 0xB4ED09, 0x209F44, 0xCA984E, 0x638270, + 0x237C7E, 0x32B90F, 0x8EF5A7, 0xE75614, 0x08F121, 0x2A9DB5, 0x4D7E6F, 0x5119A5, 0xABF9B5, + 0xD6DF82, 0x61DD96, 0x023616, 0x9F3AC4, 0xA1A283, 0x6DED72, 0x7A8D39, 0xA9B882, 0x5C326B, + 0x5B2746, 0xED3400, 0x7700D2, 0x55F4FC, 0x4D5901, 0x8071E0, +]; + +const PIO2: [f64; 8] = [ + 1.57079625129699707031e+00, /* 0x3FF921FB, 0x40000000 */ + 7.54978941586159635335e-08, /* 0x3E74442D, 0x00000000 */ + 5.39030252995776476554e-15, /* 0x3CF84698, 0x80000000 */ + 3.28200341580791294123e-22, /* 0x3B78CC51, 0x60000000 */ + 1.27065575308067607349e-29, /* 0x39F01B83, 0x80000000 */ + 1.22933308981111328932e-36, /* 0x387A2520, 0x40000000 */ + 2.73370053816464559624e-44, /* 0x36E38222, 0x80000000 */ + 2.16741683877804819444e-51, /* 0x3569F31D, 0x00000000 */ +]; + +// fn rem_pio2_large(x : &[f64], y : &mut [f64], e0 : i32, prec : usize) -> i32 +// +// Input parameters: +// x[] The input value (must be positive) is broken into nx +// pieces of 24-bit integers in double precision format. +// x[i] will be the i-th 24 bit of x. The scaled exponent +// of x[0] is given in input parameter e0 (i.e., x[0]*2^e0 +// match x's up to 24 bits. +// +// Example of breaking a double positive z into x[0]+x[1]+x[2]: +// e0 = ilogb(z)-23 +// z = scalbn(z,-e0) +// for i = 0,1,2 +// x[i] = floor(z) +// z = (z-x[i])*2**24 +// +// y[] ouput result in an array of double precision numbers. +// The dimension of y[] is: +// 24-bit precision 1 +// 53-bit precision 2 +// 64-bit precision 2 +// 113-bit precision 3 +// The actual value is the sum of them. Thus for 113-bit +// precison, one may have to do something like: +// +// long double t,w,r_head, r_tail; +// t = (long double)y[2] + (long double)y[1]; +// w = (long double)y[0]; +// r_head = t+w; +// r_tail = w - (r_head - t); +// +// e0 The exponent of x[0]. Must be <= 16360 or you need to +// expand the ipio2 table. +// +// prec an integer indicating the precision: +// 0 24 bits (single) +// 1 53 bits (double) +// 2 64 bits (extended) +// 3 113 bits (quad) +// +// Here is the description of some local variables: +// +// jk jk+1 is the initial number of terms of ipio2[] needed +// in the computation. The minimum and recommended value +// for jk is 3,4,4,6 for single, double, extended, and quad. +// jk+1 must be 2 larger than you might expect so that our +// recomputation test works. (Up to 24 bits in the integer +// part (the 24 bits of it that we compute) and 23 bits in +// the fraction part may be lost to cancelation before we +// recompute.) +// +// jz local integer variable indicating the number of +// terms of ipio2[] used. +// +// jx nx - 1 +// +// jv index for pointing to the suitable ipio2[] for the +// computation. In general, we want +// ( 2^e0*x[0] * ipio2[jv-1]*2^(-24jv) )/8 +// is an integer. Thus +// e0-3-24*jv >= 0 or (e0-3)/24 >= jv +// Hence jv = max(0,(e0-3)/24). +// +// jp jp+1 is the number of terms in PIo2[] needed, jp = jk. +// +// q[] double array with integral value, representing the +// 24-bits chunk of the product of x and 2/pi. +// +// q0 the corresponding exponent of q[0]. Note that the +// exponent for q[i] would be q0-24*i. +// +// PIo2[] double precision array, obtained by cutting pi/2 +// into 24 bits chunks. +// +// f[] ipio2[] in floating point +// +// iq[] integer array by breaking up q[] in 24-bits chunk. +// +// fq[] final product of x*(2/pi) in fq[0],..,fq[jk] +// +// ih integer. If >0 it indicates q[] is >= 0.5, hence +// it also indicates the *sign* of the result. + +/// Return the last three digits of N with y = x - N*pi/2 +/// so that |y| < pi/2. +/// +/// The method is to compute the integer (mod 8) and fraction parts of +/// (2/pi)*x without doing the full multiplication. In general we +/// skip the part of the product that are known to be a huge integer ( +/// more accurately, = 0 mod 8 ). Thus the number of operations are +/// independent of the exponent of the input. +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub(crate) fn rem_pio2_large(x: &[f64], y: &mut [f64], e0: i32, prec: usize) -> i32 { + let x1p24 = f64::from_bits(0x4170000000000000); // 0x1p24 === 2 ^ 24 + let x1p_24 = f64::from_bits(0x3e70000000000000); // 0x1p_24 === 2 ^ (-24) + + #[cfg(all(target_pointer_width = "64", feature = "checked"))] + assert!(e0 <= 16360); + + let nx = x.len(); + + let mut fw: f64; + let mut n: i32; + let mut ih: i32; + let mut z: f64; + let mut f: [f64; 20] = [0.; 20]; + let mut fq: [f64; 20] = [0.; 20]; + let mut q: [f64; 20] = [0.; 20]; + let mut iq: [i32; 20] = [0; 20]; + + /* initialize jk*/ + let jk = i!(INIT_JK, prec); + let jp = jk; + + /* determine jx,jv,q0, note that 3>q0 */ + let jx = nx - 1; + let mut jv = div!(e0 - 3, 24); + if jv < 0 { + jv = 0; + } + let mut q0 = e0 - 24 * (jv + 1); + let jv = jv as usize; + + /* set up f[0] to f[jx+jk] where f[jx+jk] = ipio2[jv+jk] */ + let mut j = (jv as i32) - (jx as i32); + let m = jx + jk; + for i in 0..=m { + i!(f, i, =, if j < 0 { + 0. + } else { + i!(IPIO2, j as usize) as f64 + }); + j += 1; + } + + /* compute q[0],q[1],...q[jk] */ + for i in 0..=jk { + fw = 0f64; + for j in 0..=jx { + fw += i!(x, j) * i!(f, jx + i - j); + } + i!(q, i, =, fw); + } + + let mut jz = jk; + + 'recompute: loop { + /* distill q[] into iq[] reversingly */ + let mut i = 0i32; + z = i!(q, jz); + for j in (1..=jz).rev() { + fw = (x1p_24 * z) as i32 as f64; + i!(iq, i as usize, =, (z - x1p24 * fw) as i32); + z = i!(q, j - 1) + fw; + i += 1; + } + + /* compute n */ + z = scalbn(z, q0); /* actual value of z */ + z -= 8.0 * floor(z * 0.125); /* trim off integer >= 8 */ + n = z as i32; + z -= n as f64; + ih = 0; + if q0 > 0 { + /* need iq[jz-1] to determine n */ + i = i!(iq, jz - 1) >> (24 - q0); + n += i; + i!(iq, jz - 1, -=, i << (24 - q0)); + ih = i!(iq, jz - 1) >> (23 - q0); + } else if q0 == 0 { + ih = i!(iq, jz - 1) >> 23; + } else if z >= 0.5 { + ih = 2; + } + + if ih > 0 { + /* q > 0.5 */ + n += 1; + let mut carry = 0i32; + for i in 0..jz { + /* compute 1-q */ + let j = i!(iq, i); + if carry == 0 { + if j != 0 { + carry = 1; + i!(iq, i, =, 0x1000000 - j); + } + } else { + i!(iq, i, =, 0xffffff - j); + } + } + if q0 > 0 { + /* rare case: chance is 1 in 12 */ + match q0 { + 1 => { + i!(iq, jz - 1, &=, 0x7fffff); + } + 2 => { + i!(iq, jz - 1, &=, 0x3fffff); + } + _ => {} + } + } + if ih == 2 { + z = 1. - z; + if carry != 0 { + z -= scalbn(1., q0); + } + } + } + + /* check if recomputation is needed */ + if z == 0. { + let mut j = 0; + for i in (jk..=jz - 1).rev() { + j |= i!(iq, i); + } + if j == 0 { + /* need recomputation */ + let mut k = 1; + while i!(iq, jk - k, ==, 0) { + k += 1; /* k = no. of terms needed */ + } + + for i in (jz + 1)..=(jz + k) { + /* add q[jz+1] to q[jz+k] */ + i!(f, jx + i, =, i!(IPIO2, jv + i) as f64); + fw = 0f64; + for j in 0..=jx { + fw += i!(x, j) * i!(f, jx + i - j); + } + i!(q, i, =, fw); + } + jz += k; + continue 'recompute; + } + } + + break; + } + + /* chop off zero terms */ + if z == 0. { + jz -= 1; + q0 -= 24; + while i!(iq, jz) == 0 { + jz -= 1; + q0 -= 24; + } + } else { + /* break z into 24-bit if necessary */ + z = scalbn(z, -q0); + if z >= x1p24 { + fw = (x1p_24 * z) as i32 as f64; + i!(iq, jz, =, (z - x1p24 * fw) as i32); + jz += 1; + q0 += 24; + i!(iq, jz, =, fw as i32); + } else { + i!(iq, jz, =, z as i32); + } + } + + /* convert integer "bit" chunk to floating-point value */ + fw = scalbn(1., q0); + for i in (0..=jz).rev() { + i!(q, i, =, fw * (i!(iq, i) as f64)); + fw *= x1p_24; + } + + /* compute PIo2[0,...,jp]*q[jz,...,0] */ + for i in (0..=jz).rev() { + fw = 0f64; + let mut k = 0; + while (k <= jp) && (k <= jz - i) { + fw += i!(PIO2, k) * i!(q, i + k); + k += 1; + } + i!(fq, jz - i, =, fw); + } + + /* compress fq[] into y[] */ + match prec { + 0 => { + fw = 0f64; + for i in (0..=jz).rev() { + fw += i!(fq, i); + } + i!(y, 0, =, if ih == 0 { fw } else { -fw }); + } + 1 | 2 => { + fw = 0f64; + for i in (0..=jz).rev() { + fw += i!(fq, i); + } + // TODO: drop excess precision here once double_t is used + fw = fw as f64; + i!(y, 0, =, if ih == 0 { fw } else { -fw }); + fw = i!(fq, 0) - fw; + for i in 1..=jz { + fw += i!(fq, i); + } + i!(y, 1, =, if ih == 0 { fw } else { -fw }); + } + 3 => { + /* painful */ + for i in (1..=jz).rev() { + fw = i!(fq, i - 1) + i!(fq, i); + i!(fq, i, +=, i!(fq, i - 1) - fw); + i!(fq, i - 1, =, fw); + } + for i in (2..=jz).rev() { + fw = i!(fq, i - 1) + i!(fq, i); + i!(fq, i, +=, i!(fq, i - 1) - fw); + i!(fq, i - 1, =, fw); + } + fw = 0f64; + for i in (2..=jz).rev() { + fw += i!(fq, i); + } + if ih == 0 { + i!(y, 0, =, i!(fq, 0)); + i!(y, 1, =, i!(fq, 1)); + i!(y, 2, =, fw); + } else { + i!(y, 0, =, -i!(fq, 0)); + i!(y, 1, =, -i!(fq, 1)); + i!(y, 2, =, -fw); + } + } + #[cfg(debug_assertions)] + _ => unreachable!(), + #[cfg(not(debug_assertions))] + _ => {} + } + n & 7 +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/rem_pio2f.rs b/crux-mir/lib/compiler_builtins/libm/src/math/rem_pio2f.rs new file mode 100644 index 000000000..775f5d750 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/rem_pio2f.rs @@ -0,0 +1,67 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_rem_pio2f.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + * Debugged and optimized by Bruce D. Evans. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +use super::rem_pio2_large; + +use core::f64; + +const TOINT: f64 = 1.5 / f64::EPSILON; + +/// 53 bits of 2/pi +const INV_PIO2: f64 = 6.36619772367581382433e-01; /* 0x3FE45F30, 0x6DC9C883 */ +/// first 25 bits of pi/2 +const PIO2_1: f64 = 1.57079631090164184570e+00; /* 0x3FF921FB, 0x50000000 */ +/// pi/2 - pio2_1 +const PIO2_1T: f64 = 1.58932547735281966916e-08; /* 0x3E5110b4, 0x611A6263 */ + +/// Return the remainder of x rem pi/2 in *y +/// +/// use double precision for everything except passing x +/// use __rem_pio2_large() for large x +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub(crate) fn rem_pio2f(x: f32) -> (i32, f64) { + let x64 = x as f64; + + let mut tx: [f64; 1] = [0.]; + let mut ty: [f64; 1] = [0.]; + + let ix = x.to_bits() & 0x7fffffff; + /* 25+53 bit pi is good enough for medium size */ + if ix < 0x4dc90fdb { + /* |x| ~< 2^28*(pi/2), medium size */ + /* Use a specialized rint() to get fn. Assume round-to-nearest. */ + let tmp = x64 * INV_PIO2 + TOINT; + // force rounding of tmp to it's storage format on x87 to avoid + // excess precision issues. + #[cfg(all(target_arch = "x86", not(target_feature = "sse2")))] + let tmp = force_eval!(tmp); + let f_n = tmp - TOINT; + return (f_n as i32, x64 - f_n * PIO2_1 - f_n * PIO2_1T); + } + if ix >= 0x7f800000 { + /* x is inf or NaN */ + return (0, x64 - x64); + } + /* scale x into [2^23, 2^24-1] */ + let sign = (x.to_bits() >> 31) != 0; + let e0 = ((ix >> 23) - (0x7f + 23)) as i32; /* e0 = ilogb(|x|)-23, positive */ + tx[0] = f32::from_bits(ix - (e0 << 23) as u32) as f64; + let n = rem_pio2_large(&tx, &mut ty, e0, 0); + if sign { + return (-n, -ty[0]); + } + (n, ty[0]) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/remainder.rs b/crux-mir/lib/compiler_builtins/libm/src/math/remainder.rs new file mode 100644 index 000000000..9e966c9ed --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/remainder.rs @@ -0,0 +1,5 @@ +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn remainder(x: f64, y: f64) -> f64 { + let (result, _) = super::remquo(x, y); + result +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/remainderf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/remainderf.rs new file mode 100644 index 000000000..b1407cf2a --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/remainderf.rs @@ -0,0 +1,5 @@ +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn remainderf(x: f32, y: f32) -> f32 { + let (result, _) = super::remquof(x, y); + result +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/remquo.rs b/crux-mir/lib/compiler_builtins/libm/src/math/remquo.rs new file mode 100644 index 000000000..0afd1f7f5 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/remquo.rs @@ -0,0 +1,110 @@ +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn remquo(mut x: f64, mut y: f64) -> (f64, i32) { + let ux: u64 = x.to_bits(); + let mut uy: u64 = y.to_bits(); + let mut ex = ((ux >> 52) & 0x7ff) as i32; + let mut ey = ((uy >> 52) & 0x7ff) as i32; + let sx = (ux >> 63) != 0; + let sy = (uy >> 63) != 0; + let mut q: u32; + let mut i: u64; + let mut uxi: u64 = ux; + + if (uy << 1) == 0 || y.is_nan() || ex == 0x7ff { + return ((x * y) / (x * y), 0); + } + if (ux << 1) == 0 { + return (x, 0); + } + + /* normalize x and y */ + if ex == 0 { + i = uxi << 12; + while (i >> 63) == 0 { + ex -= 1; + i <<= 1; + } + uxi <<= -ex + 1; + } else { + uxi &= (!0) >> 12; + uxi |= 1 << 52; + } + if ey == 0 { + i = uy << 12; + while (i >> 63) == 0 { + ey -= 1; + i <<= 1; + } + uy <<= -ey + 1; + } else { + uy &= (!0) >> 12; + uy |= 1 << 52; + } + + q = 0; + + if ex + 1 != ey { + if ex < ey { + return (x, 0); + } + /* x mod y */ + while ex > ey { + i = uxi.wrapping_sub(uy); + if (i >> 63) == 0 { + uxi = i; + q += 1; + } + uxi <<= 1; + q <<= 1; + ex -= 1; + } + i = uxi.wrapping_sub(uy); + if (i >> 63) == 0 { + uxi = i; + q += 1; + } + if uxi == 0 { + ex = -60; + } else { + while (uxi >> 52) == 0 { + uxi <<= 1; + ex -= 1; + } + } + } + + /* scale result and decide between |x| and |x|-|y| */ + if ex > 0 { + uxi -= 1 << 52; + uxi |= (ex as u64) << 52; + } else { + uxi >>= -ex + 1; + } + x = f64::from_bits(uxi); + if sy { + y = -y; + } + if ex == ey || (ex + 1 == ey && (2.0 * x > y || (2.0 * x == y && (q % 2) != 0))) { + x -= y; + // TODO: this matches musl behavior, but it is incorrect + q = q.wrapping_add(1); + } + q &= 0x7fffffff; + let quo = if sx ^ sy { -(q as i32) } else { q as i32 }; + if sx { + (-x, quo) + } else { + (x, quo) + } +} + +#[cfg(test)] +mod tests { + use super::remquo; + + #[test] + fn test_q_overflow() { + // 0xc000000000000001, 0x04c0000000000004 + let _ = remquo(-2.0000000000000004, 8.406091369059082e-286); + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/remquof.rs b/crux-mir/lib/compiler_builtins/libm/src/math/remquof.rs new file mode 100644 index 000000000..d71bd38e3 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/remquof.rs @@ -0,0 +1,97 @@ +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn remquof(mut x: f32, mut y: f32) -> (f32, i32) { + let ux: u32 = x.to_bits(); + let mut uy: u32 = y.to_bits(); + let mut ex = ((ux >> 23) & 0xff) as i32; + let mut ey = ((uy >> 23) & 0xff) as i32; + let sx = (ux >> 31) != 0; + let sy = (uy >> 31) != 0; + let mut q: u32; + let mut i: u32; + let mut uxi: u32 = ux; + + if (uy << 1) == 0 || y.is_nan() || ex == 0xff { + return ((x * y) / (x * y), 0); + } + if (ux << 1) == 0 { + return (x, 0); + } + + /* normalize x and y */ + if ex == 0 { + i = uxi << 9; + while (i >> 31) == 0 { + ex -= 1; + i <<= 1; + } + uxi <<= -ex + 1; + } else { + uxi &= (!0) >> 9; + uxi |= 1 << 23; + } + if ey == 0 { + i = uy << 9; + while (i >> 31) == 0 { + ey -= 1; + i <<= 1; + } + uy <<= -ey + 1; + } else { + uy &= (!0) >> 9; + uy |= 1 << 23; + } + + q = 0; + if ex + 1 != ey { + if ex < ey { + return (x, 0); + } + /* x mod y */ + while ex > ey { + i = uxi.wrapping_sub(uy); + if (i >> 31) == 0 { + uxi = i; + q += 1; + } + uxi <<= 1; + q <<= 1; + ex -= 1; + } + i = uxi.wrapping_sub(uy); + if (i >> 31) == 0 { + uxi = i; + q += 1; + } + if uxi == 0 { + ex = -30; + } else { + while (uxi >> 23) == 0 { + uxi <<= 1; + ex -= 1; + } + } + } + + /* scale result and decide between |x| and |x|-|y| */ + if ex > 0 { + uxi -= 1 << 23; + uxi |= (ex as u32) << 23; + } else { + uxi >>= -ex + 1; + } + x = f32::from_bits(uxi); + if sy { + y = -y; + } + if ex == ey || (ex + 1 == ey && (2.0 * x > y || (2.0 * x == y && (q % 2) != 0))) { + x -= y; + q += 1; + } + q &= 0x7fffffff; + let quo = if sx ^ sy { -(q as i32) } else { q as i32 }; + if sx { + (-x, quo) + } else { + (x, quo) + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/rint.rs b/crux-mir/lib/compiler_builtins/libm/src/math/rint.rs new file mode 100644 index 000000000..0c6025c1f --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/rint.rs @@ -0,0 +1,48 @@ +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn rint(x: f64) -> f64 { + let one_over_e = 1.0 / f64::EPSILON; + let as_u64: u64 = x.to_bits(); + let exponent: u64 = as_u64 >> 52 & 0x7ff; + let is_positive = (as_u64 >> 63) == 0; + if exponent >= 0x3ff + 52 { + x + } else { + let ans = if is_positive { + x + one_over_e - one_over_e + } else { + x - one_over_e + one_over_e + }; + + if ans == 0.0 { + if is_positive { + 0.0 + } else { + -0.0 + } + } else { + ans + } + } +} + +// PowerPC tests are failing on LLVM 13: https://github.com/rust-lang/rust/issues/88520 +#[cfg(not(target_arch = "powerpc64"))] +#[cfg(test)] +mod tests { + use super::rint; + + #[test] + fn negative_zero() { + assert_eq!(rint(-0.0_f64).to_bits(), (-0.0_f64).to_bits()); + } + + #[test] + fn sanity_check() { + assert_eq!(rint(-1.0), -1.0); + assert_eq!(rint(2.8), 3.0); + assert_eq!(rint(-0.5), -0.0); + assert_eq!(rint(0.5), 0.0); + assert_eq!(rint(-1.5), -2.0); + assert_eq!(rint(1.5), 2.0); + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/rintf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/rintf.rs new file mode 100644 index 000000000..d427793f7 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/rintf.rs @@ -0,0 +1,48 @@ +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn rintf(x: f32) -> f32 { + let one_over_e = 1.0 / f32::EPSILON; + let as_u32: u32 = x.to_bits(); + let exponent: u32 = as_u32 >> 23 & 0xff; + let is_positive = (as_u32 >> 31) == 0; + if exponent >= 0x7f + 23 { + x + } else { + let ans = if is_positive { + x + one_over_e - one_over_e + } else { + x - one_over_e + one_over_e + }; + + if ans == 0.0 { + if is_positive { + 0.0 + } else { + -0.0 + } + } else { + ans + } + } +} + +// PowerPC tests are failing on LLVM 13: https://github.com/rust-lang/rust/issues/88520 +#[cfg(not(target_arch = "powerpc64"))] +#[cfg(test)] +mod tests { + use super::rintf; + + #[test] + fn negative_zero() { + assert_eq!(rintf(-0.0_f32).to_bits(), (-0.0_f32).to_bits()); + } + + #[test] + fn sanity_check() { + assert_eq!(rintf(-1.0), -1.0); + assert_eq!(rintf(2.8), 3.0); + assert_eq!(rintf(-0.5), -0.0); + assert_eq!(rintf(0.5), 0.0); + assert_eq!(rintf(-1.5), -2.0); + assert_eq!(rintf(1.5), 2.0); + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/round.rs b/crux-mir/lib/compiler_builtins/libm/src/math/round.rs new file mode 100644 index 000000000..46fabc90f --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/round.rs @@ -0,0 +1,28 @@ +use super::copysign; +use super::trunc; +use core::f64; + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn round(x: f64) -> f64 { + trunc(x + copysign(0.5 - 0.25 * f64::EPSILON, x)) +} + +#[cfg(test)] +mod tests { + use super::round; + + #[test] + fn negative_zero() { + assert_eq!(round(-0.0_f64).to_bits(), (-0.0_f64).to_bits()); + } + + #[test] + fn sanity_check() { + assert_eq!(round(-1.0), -1.0); + assert_eq!(round(2.8), 3.0); + assert_eq!(round(-0.5), -1.0); + assert_eq!(round(0.5), 1.0); + assert_eq!(round(-1.5), -2.0); + assert_eq!(round(1.5), 2.0); + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/roundf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/roundf.rs new file mode 100644 index 000000000..becdb5620 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/roundf.rs @@ -0,0 +1,30 @@ +use super::copysignf; +use super::truncf; +use core::f32; + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn roundf(x: f32) -> f32 { + truncf(x + copysignf(0.5 - 0.25 * f32::EPSILON, x)) +} + +// PowerPC tests are failing on LLVM 13: https://github.com/rust-lang/rust/issues/88520 +#[cfg(not(target_arch = "powerpc64"))] +#[cfg(test)] +mod tests { + use super::roundf; + + #[test] + fn negative_zero() { + assert_eq!(roundf(-0.0_f32).to_bits(), (-0.0_f32).to_bits()); + } + + #[test] + fn sanity_check() { + assert_eq!(roundf(-1.0), -1.0); + assert_eq!(roundf(2.8), 3.0); + assert_eq!(roundf(-0.5), -1.0); + assert_eq!(roundf(0.5), 1.0); + assert_eq!(roundf(-1.5), -2.0); + assert_eq!(roundf(1.5), 2.0); + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/scalbn.rs b/crux-mir/lib/compiler_builtins/libm/src/math/scalbn.rs new file mode 100644 index 000000000..00c455a10 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/scalbn.rs @@ -0,0 +1,33 @@ +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn scalbn(x: f64, mut n: i32) -> f64 { + let x1p1023 = f64::from_bits(0x7fe0000000000000); // 0x1p1023 === 2 ^ 1023 + let x1p53 = f64::from_bits(0x4340000000000000); // 0x1p53 === 2 ^ 53 + let x1p_1022 = f64::from_bits(0x0010000000000000); // 0x1p-1022 === 2 ^ (-1022) + + let mut y = x; + + if n > 1023 { + y *= x1p1023; + n -= 1023; + if n > 1023 { + y *= x1p1023; + n -= 1023; + if n > 1023 { + n = 1023; + } + } + } else if n < -1022 { + /* make sure final n < -53 to avoid double + rounding in the subnormal range */ + y *= x1p_1022 * x1p53; + n += 1022 - 53; + if n < -1022 { + y *= x1p_1022 * x1p53; + n += 1022 - 53; + if n < -1022 { + n = -1022; + } + } + } + y * f64::from_bits(((0x3ff + n) as u64) << 52) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/scalbnf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/scalbnf.rs new file mode 100644 index 000000000..73f4bb57a --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/scalbnf.rs @@ -0,0 +1,29 @@ +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn scalbnf(mut x: f32, mut n: i32) -> f32 { + let x1p127 = f32::from_bits(0x7f000000); // 0x1p127f === 2 ^ 127 + let x1p_126 = f32::from_bits(0x800000); // 0x1p-126f === 2 ^ -126 + let x1p24 = f32::from_bits(0x4b800000); // 0x1p24f === 2 ^ 24 + + if n > 127 { + x *= x1p127; + n -= 127; + if n > 127 { + x *= x1p127; + n -= 127; + if n > 127 { + n = 127; + } + } + } else if n < -126 { + x *= x1p_126 * x1p24; + n += 126 - 24; + if n < -126 { + x *= x1p_126 * x1p24; + n += 126 - 24; + if n < -126 { + n = -126; + } + } + } + x * f32::from_bits(((0x7f + n) as u32) << 23) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/sin.rs b/crux-mir/lib/compiler_builtins/libm/src/math/sin.rs new file mode 100644 index 000000000..a53843dcd --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/sin.rs @@ -0,0 +1,88 @@ +// origin: FreeBSD /usr/src/lib/msun/src/s_sin.c */ +// +// ==================================================== +// Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. +// +// Developed at SunPro, a Sun Microsystems, Inc. business. +// Permission to use, copy, modify, and distribute this +// software is freely granted, provided that this notice +// is preserved. +// ==================================================== + +use super::{k_cos, k_sin, rem_pio2}; + +// sin(x) +// Return sine function of x. +// +// kernel function: +// k_sin ... sine function on [-pi/4,pi/4] +// k_cos ... cose function on [-pi/4,pi/4] +// rem_pio2 ... argument reduction routine +// +// Method. +// Let S,C and T denote the sin, cos and tan respectively on +// [-PI/4, +PI/4]. Reduce the argument x to y1+y2 = x-k*pi/2 +// in [-pi/4 , +pi/4], and let n = k mod 4. +// We have +// +// n sin(x) cos(x) tan(x) +// ---------------------------------------------------------- +// 0 S C T +// 1 C -S -1/T +// 2 -S -C T +// 3 -C S -1/T +// ---------------------------------------------------------- +// +// Special cases: +// Let trig be any of sin, cos, or tan. +// trig(+-INF) is NaN, with signals; +// trig(NaN) is that NaN; +// +// Accuracy: +// TRIG(x) returns trig(x) nearly rounded +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn sin(x: f64) -> f64 { + let x1p120 = f64::from_bits(0x4770000000000000); // 0x1p120f === 2 ^ 120 + + /* High word of x. */ + let ix = (f64::to_bits(x) >> 32) as u32 & 0x7fffffff; + + /* |x| ~< pi/4 */ + if ix <= 0x3fe921fb { + if ix < 0x3e500000 { + /* |x| < 2**-26 */ + /* raise inexact if x != 0 and underflow if subnormal*/ + if ix < 0x00100000 { + force_eval!(x / x1p120); + } else { + force_eval!(x + x1p120); + } + return x; + } + return k_sin(x, 0.0, 0); + } + + /* sin(Inf or NaN) is NaN */ + if ix >= 0x7ff00000 { + return x - x; + } + + /* argument reduction needed */ + let (n, y0, y1) = rem_pio2(x); + match n & 3 { + 0 => k_sin(y0, y1, 1), + 1 => k_cos(y0, y1), + 2 => -k_sin(y0, y1, 1), + _ => -k_cos(y0, y1), + } +} + +#[test] +fn test_near_pi() { + let x = f64::from_bits(0x400921fb000FD5DD); // 3.141592026217707 + let sx = f64::from_bits(0x3ea50d15ced1a4a2); // 6.273720864039205e-7 + let result = sin(x); + #[cfg(all(target_arch = "x86", not(target_feature = "sse2")))] + let result = force_eval!(result); + assert_eq!(result, sx); +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/sincos.rs b/crux-mir/lib/compiler_builtins/libm/src/math/sincos.rs new file mode 100644 index 000000000..ff5d87a1c --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/sincos.rs @@ -0,0 +1,134 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/s_sin.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +use super::{get_high_word, k_cos, k_sin, rem_pio2}; + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn sincos(x: f64) -> (f64, f64) { + let s: f64; + let c: f64; + let mut ix: u32; + + ix = get_high_word(x); + ix &= 0x7fffffff; + + /* |x| ~< pi/4 */ + if ix <= 0x3fe921fb { + /* if |x| < 2**-27 * sqrt(2) */ + if ix < 0x3e46a09e { + /* raise inexact if x!=0 and underflow if subnormal */ + let x1p120 = f64::from_bits(0x4770000000000000); // 0x1p120 == 2^120 + if ix < 0x00100000 { + force_eval!(x / x1p120); + } else { + force_eval!(x + x1p120); + } + return (x, 1.0); + } + return (k_sin(x, 0.0, 0), k_cos(x, 0.0)); + } + + /* sincos(Inf or NaN) is NaN */ + if ix >= 0x7ff00000 { + let rv = x - x; + return (rv, rv); + } + + /* argument reduction needed */ + let (n, y0, y1) = rem_pio2(x); + s = k_sin(y0, y1, 1); + c = k_cos(y0, y1); + match n & 3 { + 0 => (s, c), + 1 => (c, -s), + 2 => (-s, -c), + 3 => (-c, s), + #[cfg(debug_assertions)] + _ => unreachable!(), + #[cfg(not(debug_assertions))] + _ => (0.0, 1.0), + } +} + +// These tests are based on those from sincosf.rs +#[cfg(test)] +mod tests { + use super::sincos; + + const TOLERANCE: f64 = 1e-6; + + #[test] + fn with_pi() { + let (s, c) = sincos(core::f64::consts::PI); + assert!( + (s - 0.0).abs() < TOLERANCE, + "|{} - {}| = {} >= {}", + s, + 0.0, + (s - 0.0).abs(), + TOLERANCE + ); + assert!( + (c + 1.0).abs() < TOLERANCE, + "|{} + {}| = {} >= {}", + c, + 1.0, + (s + 1.0).abs(), + TOLERANCE + ); + } + + #[test] + fn rotational_symmetry() { + use core::f64::consts::PI; + const N: usize = 24; + for n in 0..N { + let theta = 2. * PI * (n as f64) / (N as f64); + let (s, c) = sincos(theta); + let (s_plus, c_plus) = sincos(theta + 2. * PI); + let (s_minus, c_minus) = sincos(theta - 2. * PI); + + assert!( + (s - s_plus).abs() < TOLERANCE, + "|{} - {}| = {} >= {}", + s, + s_plus, + (s - s_plus).abs(), + TOLERANCE + ); + assert!( + (s - s_minus).abs() < TOLERANCE, + "|{} - {}| = {} >= {}", + s, + s_minus, + (s - s_minus).abs(), + TOLERANCE + ); + assert!( + (c - c_plus).abs() < TOLERANCE, + "|{} - {}| = {} >= {}", + c, + c_plus, + (c - c_plus).abs(), + TOLERANCE + ); + assert!( + (c - c_minus).abs() < TOLERANCE, + "|{} - {}| = {} >= {}", + c, + c_minus, + (c - c_minus).abs(), + TOLERANCE + ); + } + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/sincosf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/sincosf.rs new file mode 100644 index 000000000..9a4c36104 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/sincosf.rs @@ -0,0 +1,185 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/s_sinf.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + * Optimized by Bruce D. Evans. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +use super::{k_cosf, k_sinf, rem_pio2f}; + +/* Small multiples of pi/2 rounded to double precision. */ +const PI_2: f32 = 0.5 * 3.1415926535897931160E+00; +const S1PIO2: f32 = 1.0 * PI_2; /* 0x3FF921FB, 0x54442D18 */ +const S2PIO2: f32 = 2.0 * PI_2; /* 0x400921FB, 0x54442D18 */ +const S3PIO2: f32 = 3.0 * PI_2; /* 0x4012D97C, 0x7F3321D2 */ +const S4PIO2: f32 = 4.0 * PI_2; /* 0x401921FB, 0x54442D18 */ + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn sincosf(x: f32) -> (f32, f32) { + let s: f32; + let c: f32; + let mut ix: u32; + let sign: bool; + + ix = x.to_bits(); + sign = (ix >> 31) != 0; + ix &= 0x7fffffff; + + /* |x| ~<= pi/4 */ + if ix <= 0x3f490fda { + /* |x| < 2**-12 */ + if ix < 0x39800000 { + /* raise inexact if x!=0 and underflow if subnormal */ + + let x1p120 = f32::from_bits(0x7b800000); // 0x1p120 == 2^120 + if ix < 0x00100000 { + force_eval!(x / x1p120); + } else { + force_eval!(x + x1p120); + } + return (x, 1.0); + } + return (k_sinf(x as f64), k_cosf(x as f64)); + } + + /* |x| ~<= 5*pi/4 */ + if ix <= 0x407b53d1 { + if ix <= 0x4016cbe3 { + /* |x| ~<= 3pi/4 */ + if sign { + s = -k_cosf((x + S1PIO2) as f64); + c = k_sinf((x + S1PIO2) as f64); + } else { + s = k_cosf((S1PIO2 - x) as f64); + c = k_sinf((S1PIO2 - x) as f64); + } + } + /* -sin(x+c) is not correct if x+c could be 0: -0 vs +0 */ + else { + if sign { + s = -k_sinf((x + S2PIO2) as f64); + c = -k_cosf((x + S2PIO2) as f64); + } else { + s = -k_sinf((x - S2PIO2) as f64); + c = -k_cosf((x - S2PIO2) as f64); + } + } + + return (s, c); + } + + /* |x| ~<= 9*pi/4 */ + if ix <= 0x40e231d5 { + if ix <= 0x40afeddf { + /* |x| ~<= 7*pi/4 */ + if sign { + s = k_cosf((x + S3PIO2) as f64); + c = -k_sinf((x + S3PIO2) as f64); + } else { + s = -k_cosf((x - S3PIO2) as f64); + c = k_sinf((x - S3PIO2) as f64); + } + } else { + if sign { + s = k_sinf((x + S4PIO2) as f64); + c = k_cosf((x + S4PIO2) as f64); + } else { + s = k_sinf((x - S4PIO2) as f64); + c = k_cosf((x - S4PIO2) as f64); + } + } + + return (s, c); + } + + /* sin(Inf or NaN) is NaN */ + if ix >= 0x7f800000 { + let rv = x - x; + return (rv, rv); + } + + /* general argument reduction needed */ + let (n, y) = rem_pio2f(x); + s = k_sinf(y); + c = k_cosf(y); + match n & 3 { + 0 => (s, c), + 1 => (c, -s), + 2 => (-s, -c), + 3 => (-c, s), + #[cfg(debug_assertions)] + _ => unreachable!(), + #[cfg(not(debug_assertions))] + _ => (0.0, 1.0), + } +} + +// PowerPC tests are failing on LLVM 13: https://github.com/rust-lang/rust/issues/88520 +#[cfg(not(target_arch = "powerpc64"))] +#[cfg(test)] +mod tests { + use super::sincosf; + use crate::_eqf; + + #[test] + fn with_pi() { + let (s, c) = sincosf(core::f32::consts::PI); + _eqf(s.abs(), 0.0).unwrap(); + _eqf(c, -1.0).unwrap(); + } + + #[test] + fn rotational_symmetry() { + use core::f32::consts::PI; + const N: usize = 24; + for n in 0..N { + let theta = 2. * PI * (n as f32) / (N as f32); + let (s, c) = sincosf(theta); + let (s_plus, c_plus) = sincosf(theta + 2. * PI); + let (s_minus, c_minus) = sincosf(theta - 2. * PI); + + const TOLERANCE: f32 = 1e-6; + assert!( + (s - s_plus).abs() < TOLERANCE, + "|{} - {}| = {} >= {}", + s, + s_plus, + (s - s_plus).abs(), + TOLERANCE + ); + assert!( + (s - s_minus).abs() < TOLERANCE, + "|{} - {}| = {} >= {}", + s, + s_minus, + (s - s_minus).abs(), + TOLERANCE + ); + assert!( + (c - c_plus).abs() < TOLERANCE, + "|{} - {}| = {} >= {}", + c, + c_plus, + (c - c_plus).abs(), + TOLERANCE + ); + assert!( + (c - c_minus).abs() < TOLERANCE, + "|{} - {}| = {} >= {}", + c, + c_minus, + (c - c_minus).abs(), + TOLERANCE + ); + } + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/sinf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/sinf.rs new file mode 100644 index 000000000..6e20be2ae --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/sinf.rs @@ -0,0 +1,93 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/s_sinf.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + * Optimized by Bruce D. Evans. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +use super::{k_cosf, k_sinf, rem_pio2f}; + +use core::f64::consts::FRAC_PI_2; + +/* Small multiples of pi/2 rounded to double precision. */ +const S1_PIO2: f64 = 1. * FRAC_PI_2; /* 0x3FF921FB, 0x54442D18 */ +const S2_PIO2: f64 = 2. * FRAC_PI_2; /* 0x400921FB, 0x54442D18 */ +const S3_PIO2: f64 = 3. * FRAC_PI_2; /* 0x4012D97C, 0x7F3321D2 */ +const S4_PIO2: f64 = 4. * FRAC_PI_2; /* 0x401921FB, 0x54442D18 */ + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn sinf(x: f32) -> f32 { + let x64 = x as f64; + + let x1p120 = f32::from_bits(0x7b800000); // 0x1p120f === 2 ^ 120 + + let mut ix = x.to_bits(); + let sign = (ix >> 31) != 0; + ix &= 0x7fffffff; + + if ix <= 0x3f490fda { + /* |x| ~<= pi/4 */ + if ix < 0x39800000 { + /* |x| < 2**-12 */ + /* raise inexact if x!=0 and underflow if subnormal */ + force_eval!(if ix < 0x00800000 { + x / x1p120 + } else { + x + x1p120 + }); + return x; + } + return k_sinf(x64); + } + if ix <= 0x407b53d1 { + /* |x| ~<= 5*pi/4 */ + if ix <= 0x4016cbe3 { + /* |x| ~<= 3pi/4 */ + if sign { + return -k_cosf(x64 + S1_PIO2); + } else { + return k_cosf(x64 - S1_PIO2); + } + } + return k_sinf(if sign { + -(x64 + S2_PIO2) + } else { + -(x64 - S2_PIO2) + }); + } + if ix <= 0x40e231d5 { + /* |x| ~<= 9*pi/4 */ + if ix <= 0x40afeddf { + /* |x| ~<= 7*pi/4 */ + if sign { + return k_cosf(x64 + S3_PIO2); + } else { + return -k_cosf(x64 - S3_PIO2); + } + } + return k_sinf(if sign { x64 + S4_PIO2 } else { x64 - S4_PIO2 }); + } + + /* sin(Inf or NaN) is NaN */ + if ix >= 0x7f800000 { + return x - x; + } + + /* general argument reduction needed */ + let (n, y) = rem_pio2f(x); + match n & 3 { + 0 => k_sinf(y), + 1 => k_cosf(y), + 2 => k_sinf(-y), + _ => -k_cosf(y), + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/sinh.rs b/crux-mir/lib/compiler_builtins/libm/src/math/sinh.rs new file mode 100644 index 000000000..fd24fd20c --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/sinh.rs @@ -0,0 +1,49 @@ +use super::{expm1, expo2}; + +// sinh(x) = (exp(x) - 1/exp(x))/2 +// = (exp(x)-1 + (exp(x)-1)/exp(x))/2 +// = x + x^3/6 + o(x^5) +// +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn sinh(x: f64) -> f64 { + // union {double f; uint64_t i;} u = {.f = x}; + // uint32_t w; + // double t, h, absx; + + let mut uf: f64 = x; + let mut ui: u64 = f64::to_bits(uf); + let w: u32; + let t: f64; + let mut h: f64; + let absx: f64; + + h = 0.5; + if ui >> 63 != 0 { + h = -h; + } + /* |x| */ + ui &= !1 / 2; + uf = f64::from_bits(ui); + absx = uf; + w = (ui >> 32) as u32; + + /* |x| < log(DBL_MAX) */ + if w < 0x40862e42 { + t = expm1(absx); + if w < 0x3ff00000 { + if w < 0x3ff00000 - (26 << 20) { + /* note: inexact and underflow are raised by expm1 */ + /* note: this branch avoids spurious underflow */ + return x; + } + return h * (2.0 * t - t * t / (t + 1.0)); + } + /* note: |x|>log(0x1p26)+eps could be just h*exp(x) */ + return h * (t + t / (t + 1.0)); + } + + /* |x| > log(DBL_MAX) or nan */ + /* note: the result is stored to handle overflow */ + t = 2.0 * h * expo2(absx); + t +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/sinhf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/sinhf.rs new file mode 100644 index 000000000..24f863c44 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/sinhf.rs @@ -0,0 +1,30 @@ +use super::expm1f; +use super::k_expo2f; + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn sinhf(x: f32) -> f32 { + let mut h = 0.5f32; + let mut ix = x.to_bits(); + if (ix >> 31) != 0 { + h = -h; + } + /* |x| */ + ix &= 0x7fffffff; + let absx = f32::from_bits(ix); + let w = ix; + + /* |x| < log(FLT_MAX) */ + if w < 0x42b17217 { + let t = expm1f(absx); + if w < 0x3f800000 { + if w < (0x3f800000 - (12 << 23)) { + return x; + } + return h * (2. * t - t * t / (t + 1.)); + } + return h * (t + t / (t + 1.)); + } + + /* |x| > logf(FLT_MAX) or nan */ + 2. * h * k_expo2f(absx) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/sqrt.rs b/crux-mir/lib/compiler_builtins/libm/src/math/sqrt.rs new file mode 100644 index 000000000..f06b209a4 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/sqrt.rs @@ -0,0 +1,264 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_sqrt.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunSoft, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ +/* sqrt(x) + * Return correctly rounded sqrt. + * ------------------------------------------ + * | Use the hardware sqrt if you have one | + * ------------------------------------------ + * Method: + * Bit by bit method using integer arithmetic. (Slow, but portable) + * 1. Normalization + * Scale x to y in [1,4) with even powers of 2: + * find an integer k such that 1 <= (y=x*2^(2k)) < 4, then + * sqrt(x) = 2^k * sqrt(y) + * 2. Bit by bit computation + * Let q = sqrt(y) truncated to i bit after binary point (q = 1), + * i 0 + * i+1 2 + * s = 2*q , and y = 2 * ( y - q ). (1) + * i i i i + * + * To compute q from q , one checks whether + * i+1 i + * + * -(i+1) 2 + * (q + 2 ) <= y. (2) + * i + * -(i+1) + * If (2) is false, then q = q ; otherwise q = q + 2 . + * i+1 i i+1 i + * + * With some algebraic manipulation, it is not difficult to see + * that (2) is equivalent to + * -(i+1) + * s + 2 <= y (3) + * i i + * + * The advantage of (3) is that s and y can be computed by + * i i + * the following recurrence formula: + * if (3) is false + * + * s = s , y = y ; (4) + * i+1 i i+1 i + * + * otherwise, + * -i -(i+1) + * s = s + 2 , y = y - s - 2 (5) + * i+1 i i+1 i i + * + * One may easily use induction to prove (4) and (5). + * Note. Since the left hand side of (3) contain only i+2 bits, + * it does not necessary to do a full (53-bit) comparison + * in (3). + * 3. Final rounding + * After generating the 53 bits result, we compute one more bit. + * Together with the remainder, we can decide whether the + * result is exact, bigger than 1/2ulp, or less than 1/2ulp + * (it will never equal to 1/2ulp). + * The rounding mode can be detected by checking whether + * huge + tiny is equal to huge, and whether huge - tiny is + * equal to huge for some floating point number "huge" and "tiny". + * + * Special cases: + * sqrt(+-0) = +-0 ... exact + * sqrt(inf) = inf + * sqrt(-ve) = NaN ... with invalid signal + * sqrt(NaN) = NaN ... with invalid signal for signaling NaN + */ + +use core::f64; + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn sqrt(x: f64) -> f64 { + // On wasm32 we know that LLVM's intrinsic will compile to an optimized + // `f64.sqrt` native instruction, so we can leverage this for both code size + // and speed. + llvm_intrinsically_optimized! { + #[cfg(target_arch = "wasm32")] { + return if x < 0.0 { + f64::NAN + } else { + unsafe { ::core::intrinsics::sqrtf64(x) } + } + } + } + #[cfg(target_feature = "sse2")] + { + // Note: This path is unlikely since LLVM will usually have already + // optimized sqrt calls into hardware instructions if sse2 is available, + // but if someone does end up here they'll apprected the speed increase. + #[cfg(target_arch = "x86")] + use core::arch::x86::*; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::*; + unsafe { + let m = _mm_set_sd(x); + let m_sqrt = _mm_sqrt_pd(m); + _mm_cvtsd_f64(m_sqrt) + } + } + #[cfg(not(target_feature = "sse2"))] + { + use core::num::Wrapping; + + const TINY: f64 = 1.0e-300; + + let mut z: f64; + let sign: Wrapping = Wrapping(0x80000000); + let mut ix0: i32; + let mut s0: i32; + let mut q: i32; + let mut m: i32; + let mut t: i32; + let mut i: i32; + let mut r: Wrapping; + let mut t1: Wrapping; + let mut s1: Wrapping; + let mut ix1: Wrapping; + let mut q1: Wrapping; + + ix0 = (x.to_bits() >> 32) as i32; + ix1 = Wrapping(x.to_bits() as u32); + + /* take care of Inf and NaN */ + if (ix0 & 0x7ff00000) == 0x7ff00000 { + return x * x + x; /* sqrt(NaN)=NaN, sqrt(+inf)=+inf, sqrt(-inf)=sNaN */ + } + /* take care of zero */ + if ix0 <= 0 { + if ((ix0 & !(sign.0 as i32)) | ix1.0 as i32) == 0 { + return x; /* sqrt(+-0) = +-0 */ + } + if ix0 < 0 { + return (x - x) / (x - x); /* sqrt(-ve) = sNaN */ + } + } + /* normalize x */ + m = ix0 >> 20; + if m == 0 { + /* subnormal x */ + while ix0 == 0 { + m -= 21; + ix0 |= (ix1 >> 11).0 as i32; + ix1 <<= 21; + } + i = 0; + while (ix0 & 0x00100000) == 0 { + i += 1; + ix0 <<= 1; + } + m -= i - 1; + ix0 |= (ix1 >> (32 - i) as usize).0 as i32; + ix1 = ix1 << i as usize; + } + m -= 1023; /* unbias exponent */ + ix0 = (ix0 & 0x000fffff) | 0x00100000; + if (m & 1) == 1 { + /* odd m, double x to make it even */ + ix0 += ix0 + ((ix1 & sign) >> 31).0 as i32; + ix1 += ix1; + } + m >>= 1; /* m = [m/2] */ + + /* generate sqrt(x) bit by bit */ + ix0 += ix0 + ((ix1 & sign) >> 31).0 as i32; + ix1 += ix1; + q = 0; /* [q,q1] = sqrt(x) */ + q1 = Wrapping(0); + s0 = 0; + s1 = Wrapping(0); + r = Wrapping(0x00200000); /* r = moving bit from right to left */ + + while r != Wrapping(0) { + t = s0 + r.0 as i32; + if t <= ix0 { + s0 = t + r.0 as i32; + ix0 -= t; + q += r.0 as i32; + } + ix0 += ix0 + ((ix1 & sign) >> 31).0 as i32; + ix1 += ix1; + r >>= 1; + } + + r = sign; + while r != Wrapping(0) { + t1 = s1 + r; + t = s0; + if t < ix0 || (t == ix0 && t1 <= ix1) { + s1 = t1 + r; + if (t1 & sign) == sign && (s1 & sign) == Wrapping(0) { + s0 += 1; + } + ix0 -= t; + if ix1 < t1 { + ix0 -= 1; + } + ix1 -= t1; + q1 += r; + } + ix0 += ix0 + ((ix1 & sign) >> 31).0 as i32; + ix1 += ix1; + r >>= 1; + } + + /* use floating add to find out rounding direction */ + if (ix0 as u32 | ix1.0) != 0 { + z = 1.0 - TINY; /* raise inexact flag */ + if z >= 1.0 { + z = 1.0 + TINY; + if q1.0 == 0xffffffff { + q1 = Wrapping(0); + q += 1; + } else if z > 1.0 { + if q1.0 == 0xfffffffe { + q += 1; + } + q1 += Wrapping(2); + } else { + q1 += q1 & Wrapping(1); + } + } + } + ix0 = (q >> 1) + 0x3fe00000; + ix1 = q1 >> 1; + if (q & 1) == 1 { + ix1 |= sign; + } + ix0 += m << 20; + f64::from_bits((ix0 as u64) << 32 | ix1.0 as u64) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use core::f64::*; + + #[test] + fn sanity_check() { + assert_eq!(sqrt(100.0), 10.0); + assert_eq!(sqrt(4.0), 2.0); + } + + /// The spec: https://en.cppreference.com/w/cpp/numeric/math/sqrt + #[test] + fn spec_tests() { + // Not Asserted: FE_INVALID exception is raised if argument is negative. + assert!(sqrt(-1.0).is_nan()); + assert!(sqrt(NAN).is_nan()); + for f in [0.0, -0.0, INFINITY].iter().copied() { + assert_eq!(sqrt(f), f); + } + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/sqrtf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/sqrtf.rs new file mode 100644 index 000000000..00b20e578 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/sqrtf.rs @@ -0,0 +1,154 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/e_sqrtf.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn sqrtf(x: f32) -> f32 { + // On wasm32 we know that LLVM's intrinsic will compile to an optimized + // `f32.sqrt` native instruction, so we can leverage this for both code size + // and speed. + llvm_intrinsically_optimized! { + #[cfg(target_arch = "wasm32")] { + return if x < 0.0 { + ::core::f32::NAN + } else { + unsafe { ::core::intrinsics::sqrtf32(x) } + } + } + } + #[cfg(target_feature = "sse")] + { + // Note: This path is unlikely since LLVM will usually have already + // optimized sqrt calls into hardware instructions if sse is available, + // but if someone does end up here they'll apprected the speed increase. + #[cfg(target_arch = "x86")] + use core::arch::x86::*; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::*; + unsafe { + let m = _mm_set_ss(x); + let m_sqrt = _mm_sqrt_ss(m); + _mm_cvtss_f32(m_sqrt) + } + } + #[cfg(not(target_feature = "sse"))] + { + const TINY: f32 = 1.0e-30; + + let mut z: f32; + let sign: i32 = 0x80000000u32 as i32; + let mut ix: i32; + let mut s: i32; + let mut q: i32; + let mut m: i32; + let mut t: i32; + let mut i: i32; + let mut r: u32; + + ix = x.to_bits() as i32; + + /* take care of Inf and NaN */ + if (ix as u32 & 0x7f800000) == 0x7f800000 { + return x * x + x; /* sqrt(NaN)=NaN, sqrt(+inf)=+inf, sqrt(-inf)=sNaN */ + } + + /* take care of zero */ + if ix <= 0 { + if (ix & !sign) == 0 { + return x; /* sqrt(+-0) = +-0 */ + } + if ix < 0 { + return (x - x) / (x - x); /* sqrt(-ve) = sNaN */ + } + } + + /* normalize x */ + m = ix >> 23; + if m == 0 { + /* subnormal x */ + i = 0; + while ix & 0x00800000 == 0 { + ix <<= 1; + i = i + 1; + } + m -= i - 1; + } + m -= 127; /* unbias exponent */ + ix = (ix & 0x007fffff) | 0x00800000; + if m & 1 == 1 { + /* odd m, double x to make it even */ + ix += ix; + } + m >>= 1; /* m = [m/2] */ + + /* generate sqrt(x) bit by bit */ + ix += ix; + q = 0; + s = 0; + r = 0x01000000; /* r = moving bit from right to left */ + + while r != 0 { + t = s + r as i32; + if t <= ix { + s = t + r as i32; + ix -= t; + q += r as i32; + } + ix += ix; + r >>= 1; + } + + /* use floating add to find out rounding direction */ + if ix != 0 { + z = 1.0 - TINY; /* raise inexact flag */ + if z >= 1.0 { + z = 1.0 + TINY; + if z > 1.0 { + q += 2; + } else { + q += q & 1; + } + } + } + + ix = (q >> 1) + 0x3f000000; + ix += m << 23; + f32::from_bits(ix as u32) + } +} + +// PowerPC tests are failing on LLVM 13: https://github.com/rust-lang/rust/issues/88520 +#[cfg(not(target_arch = "powerpc64"))] +#[cfg(test)] +mod tests { + use super::*; + use core::f32::*; + + #[test] + fn sanity_check() { + assert_eq!(sqrtf(100.0), 10.0); + assert_eq!(sqrtf(4.0), 2.0); + } + + /// The spec: https://en.cppreference.com/w/cpp/numeric/math/sqrt + #[test] + fn spec_tests() { + // Not Asserted: FE_INVALID exception is raised if argument is negative. + assert!(sqrtf(-1.0).is_nan()); + assert!(sqrtf(NAN).is_nan()); + for f in [0.0, -0.0, INFINITY].iter().copied() { + assert_eq!(sqrtf(f), f); + } + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/tan.rs b/crux-mir/lib/compiler_builtins/libm/src/math/tan.rs new file mode 100644 index 000000000..5a72f6801 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/tan.rs @@ -0,0 +1,70 @@ +// origin: FreeBSD /usr/src/lib/msun/src/s_tan.c */ +// +// ==================================================== +// Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. +// +// Developed at SunPro, a Sun Microsystems, Inc. business. +// Permission to use, copy, modify, and distribute this +// software is freely granted, provided that this notice +// is preserved. +// ==================================================== + +use super::{k_tan, rem_pio2}; + +// tan(x) +// Return tangent function of x. +// +// kernel function: +// k_tan ... tangent function on [-pi/4,pi/4] +// rem_pio2 ... argument reduction routine +// +// Method. +// Let S,C and T denote the sin, cos and tan respectively on +// [-PI/4, +PI/4]. Reduce the argument x to y1+y2 = x-k*pi/2 +// in [-pi/4 , +pi/4], and let n = k mod 4. +// We have +// +// n sin(x) cos(x) tan(x) +// ---------------------------------------------------------- +// 0 S C T +// 1 C -S -1/T +// 2 -S -C T +// 3 -C S -1/T +// ---------------------------------------------------------- +// +// Special cases: +// Let trig be any of sin, cos, or tan. +// trig(+-INF) is NaN, with signals; +// trig(NaN) is that NaN; +// +// Accuracy: +// TRIG(x) returns trig(x) nearly rounded +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn tan(x: f64) -> f64 { + let x1p120 = f32::from_bits(0x7b800000); // 0x1p120f === 2 ^ 120 + + let ix = (f64::to_bits(x) >> 32) as u32 & 0x7fffffff; + /* |x| ~< pi/4 */ + if ix <= 0x3fe921fb { + if ix < 0x3e400000 { + /* |x| < 2**-27 */ + /* raise inexact if x!=0 and underflow if subnormal */ + force_eval!(if ix < 0x00100000 { + x / x1p120 as f64 + } else { + x + x1p120 as f64 + }); + return x; + } + return k_tan(x, 0.0, 0); + } + + /* tan(Inf or NaN) is NaN */ + if ix >= 0x7ff00000 { + return x - x; + } + + /* argument reduction */ + let (n, y0, y1) = rem_pio2(x); + k_tan(y0, y1, n & 1) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/tanf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/tanf.rs new file mode 100644 index 000000000..10de59c39 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/tanf.rs @@ -0,0 +1,78 @@ +/* origin: FreeBSD /usr/src/lib/msun/src/s_tanf.c */ +/* + * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + * Optimized by Bruce D. Evans. + */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +use super::{k_tanf, rem_pio2f}; + +use core::f64::consts::FRAC_PI_2; + +/* Small multiples of pi/2 rounded to double precision. */ +const T1_PIO2: f64 = 1. * FRAC_PI_2; /* 0x3FF921FB, 0x54442D18 */ +const T2_PIO2: f64 = 2. * FRAC_PI_2; /* 0x400921FB, 0x54442D18 */ +const T3_PIO2: f64 = 3. * FRAC_PI_2; /* 0x4012D97C, 0x7F3321D2 */ +const T4_PIO2: f64 = 4. * FRAC_PI_2; /* 0x401921FB, 0x54442D18 */ + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn tanf(x: f32) -> f32 { + let x64 = x as f64; + + let x1p120 = f32::from_bits(0x7b800000); // 0x1p120f === 2 ^ 120 + + let mut ix = x.to_bits(); + let sign = (ix >> 31) != 0; + ix &= 0x7fffffff; + + if ix <= 0x3f490fda { + /* |x| ~<= pi/4 */ + if ix < 0x39800000 { + /* |x| < 2**-12 */ + /* raise inexact if x!=0 and underflow if subnormal */ + force_eval!(if ix < 0x00800000 { + x / x1p120 + } else { + x + x1p120 + }); + return x; + } + return k_tanf(x64, false); + } + if ix <= 0x407b53d1 { + /* |x| ~<= 5*pi/4 */ + if ix <= 0x4016cbe3 { + /* |x| ~<= 3pi/4 */ + return k_tanf(if sign { x64 + T1_PIO2 } else { x64 - T1_PIO2 }, true); + } else { + return k_tanf(if sign { x64 + T2_PIO2 } else { x64 - T2_PIO2 }, false); + } + } + if ix <= 0x40e231d5 { + /* |x| ~<= 9*pi/4 */ + if ix <= 0x40afeddf { + /* |x| ~<= 7*pi/4 */ + return k_tanf(if sign { x64 + T3_PIO2 } else { x64 - T3_PIO2 }, true); + } else { + return k_tanf(if sign { x64 + T4_PIO2 } else { x64 - T4_PIO2 }, false); + } + } + + /* tan(Inf or NaN) is NaN */ + if ix >= 0x7f800000 { + return x - x; + } + + /* argument reduction */ + let (n, y) = rem_pio2f(x); + k_tanf(y, n & 1 != 0) +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/tanh.rs b/crux-mir/lib/compiler_builtins/libm/src/math/tanh.rs new file mode 100644 index 000000000..980c68554 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/tanh.rs @@ -0,0 +1,53 @@ +use super::expm1; + +/* tanh(x) = (exp(x) - exp(-x))/(exp(x) + exp(-x)) + * = (exp(2*x) - 1)/(exp(2*x) - 1 + 2) + * = (1 - exp(-2*x))/(exp(-2*x) - 1 + 2) + */ +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn tanh(mut x: f64) -> f64 { + let mut uf: f64 = x; + let mut ui: u64 = f64::to_bits(uf); + + let w: u32; + let sign: bool; + let mut t: f64; + + /* x = |x| */ + sign = ui >> 63 != 0; + ui &= !1 / 2; + uf = f64::from_bits(ui); + x = uf; + w = (ui >> 32) as u32; + + if w > 0x3fe193ea { + /* |x| > log(3)/2 ~= 0.5493 or nan */ + if w > 0x40340000 { + /* |x| > 20 or nan */ + /* note: this branch avoids raising overflow */ + t = 1.0 - 0.0 / x; + } else { + t = expm1(2.0 * x); + t = 1.0 - 2.0 / (t + 2.0); + } + } else if w > 0x3fd058ae { + /* |x| > log(5/3)/2 ~= 0.2554 */ + t = expm1(2.0 * x); + t = t / (t + 2.0); + } else if w >= 0x00100000 { + /* |x| >= 0x1p-1022, up to 2ulp error in [0.1,0.2554] */ + t = expm1(-2.0 * x); + t = -t / (t + 2.0); + } else { + /* |x| is subnormal */ + /* note: the branch above would not raise underflow in [0x1p-1023,0x1p-1022) */ + force_eval!(x as f32); + t = x; + } + + if sign { + -t + } else { + t + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/tanhf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/tanhf.rs new file mode 100644 index 000000000..fc94e3ddd --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/tanhf.rs @@ -0,0 +1,39 @@ +use super::expm1f; + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn tanhf(mut x: f32) -> f32 { + /* x = |x| */ + let mut ix = x.to_bits(); + let sign = (ix >> 31) != 0; + ix &= 0x7fffffff; + x = f32::from_bits(ix); + let w = ix; + + let tt = if w > 0x3f0c9f54 { + /* |x| > log(3)/2 ~= 0.5493 or nan */ + if w > 0x41200000 { + /* |x| > 10 */ + 1. + 0. / x + } else { + let t = expm1f(2. * x); + 1. - 2. / (t + 2.) + } + } else if w > 0x3e82c578 { + /* |x| > log(5/3)/2 ~= 0.2554 */ + let t = expm1f(2. * x); + t / (t + 2.) + } else if w >= 0x00800000 { + /* |x| >= 0x1p-126 */ + let t = expm1f(-2. * x); + -t / (t + 2.) + } else { + /* |x| is subnormal */ + force_eval!(x * x); + x + }; + if sign { + -tt + } else { + tt + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/tgamma.rs b/crux-mir/lib/compiler_builtins/libm/src/math/tgamma.rs new file mode 100644 index 000000000..e64eff61f --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/tgamma.rs @@ -0,0 +1,208 @@ +/* +"A Precision Approximation of the Gamma Function" - Cornelius Lanczos (1964) +"Lanczos Implementation of the Gamma Function" - Paul Godfrey (2001) +"An Analysis of the Lanczos Gamma Approximation" - Glendon Ralph Pugh (2004) + +approximation method: + + (x - 0.5) S(x) +Gamma(x) = (x + g - 0.5) * ---------------- + exp(x + g - 0.5) + +with + a1 a2 a3 aN +S(x) ~= [ a0 + ----- + ----- + ----- + ... + ----- ] + x + 1 x + 2 x + 3 x + N + +with a0, a1, a2, a3,.. aN constants which depend on g. + +for x < 0 the following reflection formula is used: + +Gamma(x)*Gamma(-x) = -pi/(x sin(pi x)) + +most ideas and constants are from boost and python +*/ +extern crate core; +use super::{exp, floor, k_cos, k_sin, pow}; + +const PI: f64 = 3.141592653589793238462643383279502884; + +/* sin(pi x) with x > 0x1p-100, if sin(pi*x)==0 the sign is arbitrary */ +fn sinpi(mut x: f64) -> f64 { + let mut n: isize; + + /* argument reduction: x = |x| mod 2 */ + /* spurious inexact when x is odd int */ + x = x * 0.5; + x = 2.0 * (x - floor(x)); + + /* reduce x into [-.25,.25] */ + n = (4.0 * x) as isize; + n = div!(n + 1, 2); + x -= (n as f64) * 0.5; + + x *= PI; + match n { + 1 => k_cos(x, 0.0), + 2 => k_sin(-x, 0.0, 0), + 3 => -k_cos(x, 0.0), + 0 | _ => k_sin(x, 0.0, 0), + } +} + +const N: usize = 12; +//static const double g = 6.024680040776729583740234375; +const GMHALF: f64 = 5.524680040776729583740234375; +const SNUM: [f64; N + 1] = [ + 23531376880.410759688572007674451636754734846804940, + 42919803642.649098768957899047001988850926355848959, + 35711959237.355668049440185451547166705960488635843, + 17921034426.037209699919755754458931112671403265390, + 6039542586.3520280050642916443072979210699388420708, + 1439720407.3117216736632230727949123939715485786772, + 248874557.86205415651146038641322942321632125127801, + 31426415.585400194380614231628318205362874684987640, + 2876370.6289353724412254090516208496135991145378768, + 186056.26539522349504029498971604569928220784236328, + 8071.6720023658162106380029022722506138218516325024, + 210.82427775157934587250973392071336271166969580291, + 2.5066282746310002701649081771338373386264310793408, +]; +const SDEN: [f64; N + 1] = [ + 0.0, + 39916800.0, + 120543840.0, + 150917976.0, + 105258076.0, + 45995730.0, + 13339535.0, + 2637558.0, + 357423.0, + 32670.0, + 1925.0, + 66.0, + 1.0, +]; +/* n! for small integer n */ +const FACT: [f64; 23] = [ + 1.0, + 1.0, + 2.0, + 6.0, + 24.0, + 120.0, + 720.0, + 5040.0, + 40320.0, + 362880.0, + 3628800.0, + 39916800.0, + 479001600.0, + 6227020800.0, + 87178291200.0, + 1307674368000.0, + 20922789888000.0, + 355687428096000.0, + 6402373705728000.0, + 121645100408832000.0, + 2432902008176640000.0, + 51090942171709440000.0, + 1124000727777607680000.0, +]; + +/* S(x) rational function for positive x */ +fn s(x: f64) -> f64 { + let mut num: f64 = 0.0; + let mut den: f64 = 0.0; + + /* to avoid overflow handle large x differently */ + if x < 8.0 { + for i in (0..=N).rev() { + num = num * x + i!(SNUM, i); + den = den * x + i!(SDEN, i); + } + } else { + for i in 0..=N { + num = num / x + i!(SNUM, i); + den = den / x + i!(SDEN, i); + } + } + return num / den; +} + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn tgamma(mut x: f64) -> f64 { + let u: u64 = x.to_bits(); + let absx: f64; + let mut y: f64; + let mut dy: f64; + let mut z: f64; + let mut r: f64; + let ix: u32 = ((u >> 32) as u32) & 0x7fffffff; + let sign: bool = (u >> 63) != 0; + + /* special cases */ + if ix >= 0x7ff00000 { + /* tgamma(nan)=nan, tgamma(inf)=inf, tgamma(-inf)=nan with invalid */ + return x + core::f64::INFINITY; + } + if ix < ((0x3ff - 54) << 20) { + /* |x| < 2^-54: tgamma(x) ~ 1/x, +-0 raises div-by-zero */ + return 1.0 / x; + } + + /* integer arguments */ + /* raise inexact when non-integer */ + if x == floor(x) { + if sign { + return 0.0 / 0.0; + } + if x <= FACT.len() as f64 { + return i!(FACT, (x as usize) - 1); + } + } + + /* x >= 172: tgamma(x)=inf with overflow */ + /* x =< -184: tgamma(x)=+-0 with underflow */ + if ix >= 0x40670000 { + /* |x| >= 184 */ + if sign { + let x1p_126 = f64::from_bits(0x3810000000000000); // 0x1p-126 == 2^-126 + force_eval!((x1p_126 / x) as f32); + if floor(x) * 0.5 == floor(x * 0.5) { + return 0.0; + } else { + return -0.0; + } + } + let x1p1023 = f64::from_bits(0x7fe0000000000000); // 0x1p1023 == 2^1023 + x *= x1p1023; + return x; + } + + absx = if sign { -x } else { x }; + + /* handle the error of x + g - 0.5 */ + y = absx + GMHALF; + if absx > GMHALF { + dy = y - absx; + dy -= GMHALF; + } else { + dy = y - GMHALF; + dy -= absx; + } + + z = absx - 0.5; + r = s(absx) * exp(-y); + if x < 0.0 { + /* reflection formula for negative x */ + /* sinpi(absx) is not 0, integers are already handled */ + r = -PI / (sinpi(absx) * absx * r); + dy = -dy; + z = -z; + } + r += dy * (GMHALF + 0.5) * r / y; + z = pow(y, 0.5 * z); + y = r * z * z; + return y; +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/tgammaf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/tgammaf.rs new file mode 100644 index 000000000..23e3814f9 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/tgammaf.rs @@ -0,0 +1,6 @@ +use super::tgamma; + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn tgammaf(x: f32) -> f32 { + tgamma(x as f64) as f32 +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/trunc.rs b/crux-mir/lib/compiler_builtins/libm/src/math/trunc.rs new file mode 100644 index 000000000..f7892a2c5 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/trunc.rs @@ -0,0 +1,40 @@ +use core::f64; + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn trunc(x: f64) -> f64 { + // On wasm32 we know that LLVM's intrinsic will compile to an optimized + // `f64.trunc` native instruction, so we can leverage this for both code size + // and speed. + llvm_intrinsically_optimized! { + #[cfg(target_arch = "wasm32")] { + return unsafe { ::core::intrinsics::truncf64(x) } + } + } + let x1p120 = f64::from_bits(0x4770000000000000); // 0x1p120f === 2 ^ 120 + + let mut i: u64 = x.to_bits(); + let mut e: i64 = (i >> 52 & 0x7ff) as i64 - 0x3ff + 12; + let m: u64; + + if e >= 52 + 12 { + return x; + } + if e < 12 { + e = 1; + } + m = -1i64 as u64 >> e; + if (i & m) == 0 { + return x; + } + force_eval!(x + x1p120); + i &= !m; + f64::from_bits(i) +} + +#[cfg(test)] +mod tests { + #[test] + fn sanity_check() { + assert_eq!(super::trunc(1.1), 1.0); + } +} diff --git a/crux-mir/lib/compiler_builtins/libm/src/math/truncf.rs b/crux-mir/lib/compiler_builtins/libm/src/math/truncf.rs new file mode 100644 index 000000000..20d5b73bd --- /dev/null +++ b/crux-mir/lib/compiler_builtins/libm/src/math/truncf.rs @@ -0,0 +1,42 @@ +use core::f32; + +#[cfg_attr(all(test, assert_no_panic), no_panic::no_panic)] +pub fn truncf(x: f32) -> f32 { + // On wasm32 we know that LLVM's intrinsic will compile to an optimized + // `f32.trunc` native instruction, so we can leverage this for both code size + // and speed. + llvm_intrinsically_optimized! { + #[cfg(target_arch = "wasm32")] { + return unsafe { ::core::intrinsics::truncf32(x) } + } + } + let x1p120 = f32::from_bits(0x7b800000); // 0x1p120f === 2 ^ 120 + + let mut i: u32 = x.to_bits(); + let mut e: i32 = (i >> 23 & 0xff) as i32 - 0x7f + 9; + let m: u32; + + if e >= 23 + 9 { + return x; + } + if e < 9 { + e = 1; + } + m = -1i32 as u32 >> e; + if (i & m) == 0 { + return x; + } + force_eval!(x + x1p120); + i &= !m; + f32::from_bits(i) +} + +// PowerPC tests are failing on LLVM 13: https://github.com/rust-lang/rust/issues/88520 +#[cfg(not(target_arch = "powerpc64"))] +#[cfg(test)] +mod tests { + #[test] + fn sanity_check() { + assert_eq!(super::truncf(1.1), 1.0); + } +} diff --git a/crux-mir/lib/compiler_builtins/src/arm.rs b/crux-mir/lib/compiler_builtins/src/arm.rs index 4cf73ef37..e517a9ef3 100644 --- a/crux-mir/lib/compiler_builtins/src/arm.rs +++ b/crux-mir/lib/compiler_builtins/src/arm.rs @@ -1,232 +1,187 @@ -use core::intrinsics; - -// NOTE This function and the ones below are implemented using assembly because they using a custom -// calling convention which can't be implemented using a normal Rust function. -// NOTE The only difference between the iOS and non-iOS versions of those functions is that the iOS -// versions use 3 leading underscores in the names of called functions instead of 2. -#[cfg(not(any(target_os = "ios", target_env = "msvc")))] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe fn __aeabi_uidivmod() { - asm!("push {lr} - sub sp, sp, #4 - mov r2, sp - bl __udivmodsi4 - ldr r1, [sp] - add sp, sp, #4 - pop {pc}" ::: "memory" : "volatile"); - intrinsics::unreachable(); -} +#![cfg(not(feature = "no-asm"))] +#![allow(unused_imports)] -#[cfg(target_os = "ios")] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe fn __aeabi_uidivmod() { - asm!("push {lr} - sub sp, sp, #4 - mov r2, sp - bl ___udivmodsi4 - ldr r1, [sp] - add sp, sp, #4 - pop {pc}" ::: "memory" : "volatile"); - intrinsics::unreachable(); -} - -#[cfg(not(target_os = "ios"))] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe fn __aeabi_uldivmod() { - asm!("push {r4, lr} - sub sp, sp, #16 - add r4, sp, #8 - str r4, [sp] - bl __udivmoddi4 - ldr r2, [sp, #8] - ldr r3, [sp, #12] - add sp, sp, #16 - pop {r4, pc}" ::: "memory" : "volatile"); - intrinsics::unreachable(); -} +use core::intrinsics; +// iOS symbols have a leading underscore. #[cfg(target_os = "ios")] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe fn __aeabi_uldivmod() { - asm!("push {r4, lr} - sub sp, sp, #16 - add r4, sp, #8 - str r4, [sp] - bl ___udivmoddi4 - ldr r2, [sp, #8] - ldr r3, [sp, #12] - add sp, sp, #16 - pop {r4, pc}" ::: "memory" : "volatile"); - intrinsics::unreachable(); +macro_rules! bl { + ($func:literal) => { + concat!("bl _", $func) + }; } - #[cfg(not(target_os = "ios"))] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe fn __aeabi_idivmod() { - asm!("push {r0, r1, r4, lr} - bl __aeabi_idiv - pop {r1, r2} - muls r2, r2, r0 - subs r1, r1, r2 - pop {r4, pc}" ::: "memory" : "volatile"); - intrinsics::unreachable(); -} - -#[cfg(target_os = "ios")] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe fn __aeabi_idivmod() { - asm!("push {r0, r1, r4, lr} - bl ___aeabi_idiv - pop {r1, r2} - muls r2, r2, r0 - subs r1, r1, r2 - pop {r4, pc}" ::: "memory" : "volatile"); - intrinsics::unreachable(); -} +macro_rules! bl { + ($func:literal) => { + concat!("bl ", $func) + }; +} + +intrinsics! { + // NOTE This function and the ones below are implemented using assembly because they are using a + // custom calling convention which can't be implemented using a normal Rust function. + #[naked] + #[cfg(not(target_env = "msvc"))] + #[cfg_attr(all(not(windows), not(target_vendor="apple")), linkage = "weak")] + pub unsafe extern "C" fn __aeabi_uidivmod() { + core::arch::asm!( + "push {{lr}}", + "sub sp, sp, #4", + "mov r2, sp", + bl!("__udivmodsi4"), + "ldr r1, [sp]", + "add sp, sp, #4", + "pop {{pc}}", + options(noreturn) + ); + } -#[cfg(not(target_os = "ios"))] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe fn __aeabi_ldivmod() { - asm!("push {r4, lr} - sub sp, sp, #16 - add r4, sp, #8 - str r4, [sp] - bl __divmoddi4 - ldr r2, [sp, #8] - ldr r3, [sp, #12] - add sp, sp, #16 - pop {r4, pc}" ::: "memory" : "volatile"); - intrinsics::unreachable(); -} + #[naked] + #[cfg_attr(all(not(windows), not(target_vendor="apple")), linkage = "weak")] + pub unsafe extern "C" fn __aeabi_uldivmod() { + core::arch::asm!( + "push {{r4, lr}}", + "sub sp, sp, #16", + "add r4, sp, #8", + "str r4, [sp]", + bl!("__udivmoddi4"), + "ldr r2, [sp, #8]", + "ldr r3, [sp, #12]", + "add sp, sp, #16", + "pop {{r4, pc}}", + options(noreturn) + ); + } -#[cfg(target_os = "ios")] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe fn __aeabi_ldivmod() { - asm!("push {r4, lr} - sub sp, sp, #16 - add r4, sp, #8 - str r4, [sp] - bl ___divmoddi4 - ldr r2, [sp, #8] - ldr r3, [sp, #12] - add sp, sp, #16 - pop {r4, pc}" ::: "memory" : "volatile"); - intrinsics::unreachable(); -} + #[naked] + #[cfg_attr(all(not(windows), not(target_vendor="apple")), linkage = "weak")] + pub unsafe extern "C" fn __aeabi_idivmod() { + core::arch::asm!( + "push {{r0, r1, r4, lr}}", + bl!("__aeabi_idiv"), + "pop {{r1, r2}}", + "muls r2, r2, r0", + "subs r1, r1, r2", + "pop {{r4, pc}}", + options(noreturn) + ); + } -// FIXME: The `*4` and `*8` variants should be defined as aliases. + #[naked] + #[cfg_attr(all(not(windows), not(target_vendor="apple")), linkage = "weak")] + pub unsafe extern "C" fn __aeabi_ldivmod() { + core::arch::asm!( + "push {{r4, lr}}", + "sub sp, sp, #16", + "add r4, sp, #8", + "str r4, [sp]", + bl!("__divmoddi4"), + "ldr r2, [sp, #8]", + "ldr r3, [sp, #12]", + "add sp, sp, #16", + "pop {{r4, pc}}", + options(noreturn) + ); + } -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[cfg_attr(thumb, linkage = "weak")] -pub unsafe extern "aapcs" fn __aeabi_memcpy(dest: *mut u8, src: *const u8, n: usize) { - ::mem::memcpy(dest, src, n); -} + // The following functions use weak linkage to allow users to override + // with custom implementation. + // FIXME: The `*4` and `*8` variants should be defined as aliases. -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[cfg_attr(thumb, linkage = "weak")] -pub unsafe extern "aapcs" fn __aeabi_memcpy4(dest: *mut u8, src: *const u8, mut n: usize) { - // We are guaranteed 4-alignment, so accessing at u32 is okay. - let mut dest = dest as *mut u32; - let mut src = src as *mut u32; + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memcpy(dest: *mut u8, src: *const u8, n: usize) { + ::mem::memcpy(dest, src, n); + } - while n >= 4 { - *dest = *src; - dest = dest.offset(1); - src = src.offset(1); - n -= 4; + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memcpy4(dest: *mut u8, src: *const u8, n: usize) { + // We are guaranteed 4-alignment, so accessing at u32 is okay. + let mut dest = dest as *mut u32; + let mut src = src as *mut u32; + let mut n = n; + + while n >= 4 { + *dest = *src; + dest = dest.offset(1); + src = src.offset(1); + n -= 4; + } + + __aeabi_memcpy(dest as *mut u8, src as *const u8, n); } - __aeabi_memcpy(dest as *mut u8, src as *const u8, n); -} + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memcpy8(dest: *mut u8, src: *const u8, n: usize) { + __aeabi_memcpy4(dest, src, n); + } -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[cfg_attr(thumb, linkage = "weak")] -pub unsafe extern "aapcs" fn __aeabi_memcpy8(dest: *mut u8, src: *const u8, n: usize) { - __aeabi_memcpy4(dest, src, n); -} + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memmove(dest: *mut u8, src: *const u8, n: usize) { + ::mem::memmove(dest, src, n); + } -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[cfg_attr(thumb, linkage = "weak")] -pub unsafe extern "aapcs" fn __aeabi_memmove(dest: *mut u8, src: *const u8, n: usize) { - ::mem::memmove(dest, src, n); -} + #[cfg(not(any(target_os = "ios", target_env = "msvc")))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memmove4(dest: *mut u8, src: *const u8, n: usize) { + __aeabi_memmove(dest, src, n); + } -#[cfg(not(any(target_os = "ios", target_env = "msvc")))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[cfg_attr(thumb, linkage = "weak")] -pub unsafe extern "aapcs" fn __aeabi_memmove4(dest: *mut u8, src: *const u8, n: usize) { - __aeabi_memmove(dest, src, n); -} + #[cfg(not(any(target_os = "ios", target_env = "msvc")))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memmove8(dest: *mut u8, src: *const u8, n: usize) { + __aeabi_memmove(dest, src, n); + } -#[cfg(not(any(target_os = "ios", target_env = "msvc")))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[cfg_attr(thumb, linkage = "weak")] -pub unsafe extern "aapcs" fn __aeabi_memmove8(dest: *mut u8, src: *const u8, n: usize) { - __aeabi_memmove(dest, src, n); -} + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memset(dest: *mut u8, n: usize, c: i32) { + // Note the different argument order + ::mem::memset(dest, c, n); + } -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[cfg_attr(thumb, linkage = "weak")] -pub unsafe extern "aapcs" fn __aeabi_memset(dest: *mut u8, n: usize, c: i32) { - // Note the different argument order - ::mem::memset(dest, c, n); -} + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memset4(dest: *mut u8, n: usize, c: i32) { + let mut dest = dest as *mut u32; + let mut n = n; -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[cfg_attr(thumb, linkage = "weak")] -pub unsafe extern "aapcs" fn __aeabi_memset4(dest: *mut u8, mut n: usize, c: i32) { - let mut dest = dest as *mut u32; + let byte = (c as u32) & 0xff; + let c = (byte << 24) | (byte << 16) | (byte << 8) | byte; - let byte = (c as u32) & 0xff; - let c = (byte << 24) | (byte << 16) | (byte << 8) | byte; + while n >= 4 { + *dest = c; + dest = dest.offset(1); + n -= 4; + } - while n >= 4 { - *dest = c; - dest = dest.offset(1); - n -= 4; + __aeabi_memset(dest as *mut u8, n, byte as i32); } - __aeabi_memset(dest as *mut u8, n, byte as i32); -} - -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[cfg_attr(thumb, linkage = "weak")] -pub unsafe extern "aapcs" fn __aeabi_memset8(dest: *mut u8, n: usize, c: i32) { - __aeabi_memset4(dest, n, c); -} + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memset8(dest: *mut u8, n: usize, c: i32) { + __aeabi_memset4(dest, n, c); + } -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[cfg_attr(thumb, linkage = "weak")] -pub unsafe extern "aapcs" fn __aeabi_memclr(dest: *mut u8, n: usize) { - __aeabi_memset(dest, n, 0); -} + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memclr(dest: *mut u8, n: usize) { + __aeabi_memset(dest, n, 0); + } -#[cfg(not(any(target_os = "ios", target_env = "msvc")))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[cfg_attr(thumb, linkage = "weak")] -pub unsafe extern "aapcs" fn __aeabi_memclr4(dest: *mut u8, n: usize) { - __aeabi_memset4(dest, n, 0); -} + #[cfg(not(any(target_os = "ios", target_env = "msvc")))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memclr4(dest: *mut u8, n: usize) { + __aeabi_memset4(dest, n, 0); + } -#[cfg(not(any(target_os = "ios", target_env = "msvc")))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[cfg_attr(thumb, linkage = "weak")] -pub unsafe extern "aapcs" fn __aeabi_memclr8(dest: *mut u8, n: usize) { - __aeabi_memset4(dest, n, 0); + #[cfg(not(any(target_os = "ios", target_env = "msvc")))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memclr8(dest: *mut u8, n: usize) { + __aeabi_memset4(dest, n, 0); + } } diff --git a/crux-mir/lib/compiler_builtins/src/arm_linux.rs b/crux-mir/lib/compiler_builtins/src/arm_linux.rs index e710c1ab9..8f22eb628 100644 --- a/crux-mir/lib/compiler_builtins/src/arm_linux.rs +++ b/crux-mir/lib/compiler_builtins/src/arm_linux.rs @@ -4,11 +4,11 @@ use core::mem; // Kernel-provided user-mode helper functions: // https://www.kernel.org/doc/Documentation/arm/kernel_user_helpers.txt unsafe fn __kuser_cmpxchg(oldval: u32, newval: u32, ptr: *mut u32) -> bool { - let f: extern "C" fn(u32, u32, *mut u32) -> u32 = mem::transmute(0xffff0fc0u32); + let f: extern "C" fn(u32, u32, *mut u32) -> u32 = mem::transmute(0xffff0fc0usize as *const ()); f(oldval, newval, ptr) == 0 } unsafe fn __kuser_memory_barrier() { - let f: extern "C" fn() = mem::transmute(0xffff0fa0u32); + let f: extern "C" fn() = mem::transmute(0xffff0fa0usize as *const ()); f(); } @@ -55,7 +55,7 @@ fn insert_aligned(aligned: u32, val: u32, shift: u32, mask: u32) -> u32 { } // Generic atomic read-modify-write operation -unsafe fn atomic_rmw u32>(ptr: *mut T, f: F) -> u32 { +unsafe fn atomic_rmw u32, G: Fn(u32, u32) -> u32>(ptr: *mut T, f: F, g: G) -> u32 { let aligned_ptr = align_ptr(ptr); let (shift, mask) = get_shift_mask(ptr); @@ -65,7 +65,7 @@ unsafe fn atomic_rmw u32>(ptr: *mut T, f: F) -> u32 { let newval = f(curval); let newval_aligned = insert_aligned(curval_aligned, newval, shift, mask); if __kuser_cmpxchg(curval_aligned, newval_aligned, aligned_ptr) { - return curval; + return g(curval, newval); } } } @@ -89,123 +89,162 @@ unsafe fn atomic_cmpxchg(ptr: *mut T, oldval: u32, newval: u32) -> u32 { } macro_rules! atomic_rmw { - ($name:ident, $ty:ty, $op:expr) => { - #[cfg_attr(not(feature = "mangled-names"), no_mangle)] - pub unsafe extern "C" fn $name(ptr: *mut $ty, val: $ty) -> $ty { - atomic_rmw(ptr, |x| $op(x as $ty, val) as u32) as $ty + ($name:ident, $ty:ty, $op:expr, $fetch:expr) => { + intrinsics! { + pub unsafe extern "C" fn $name(ptr: *mut $ty, val: $ty) -> $ty { + atomic_rmw(ptr, |x| $op(x as $ty, val) as u32, |old, new| $fetch(old, new)) as $ty + } } }; + + (@old $name:ident, $ty:ty, $op:expr) => { + atomic_rmw!($name, $ty, $op, |old, _| old); + }; + + (@new $name:ident, $ty:ty, $op:expr) => { + atomic_rmw!($name, $ty, $op, |_, new| new); + }; } macro_rules! atomic_cmpxchg { ($name:ident, $ty:ty) => { - #[cfg_attr(not(feature = "mangled-names"), no_mangle)] - pub unsafe extern "C" fn $name(ptr: *mut $ty, oldval: $ty, newval: $ty) -> $ty { - atomic_cmpxchg(ptr, oldval as u32, newval as u32) as $ty + intrinsics! { + pub unsafe extern "C" fn $name(ptr: *mut $ty, oldval: $ty, newval: $ty) -> $ty { + atomic_cmpxchg(ptr, oldval as u32, newval as u32) as $ty + } } }; } -atomic_rmw!(__sync_fetch_and_add_1, u8, |a: u8, b: u8| a.wrapping_add(b)); -atomic_rmw!(__sync_fetch_and_add_2, u16, |a: u16, b: u16| a +atomic_rmw!(@old __sync_fetch_and_add_1, u8, |a: u8, b: u8| a.wrapping_add(b)); +atomic_rmw!(@old __sync_fetch_and_add_2, u16, |a: u16, b: u16| a + .wrapping_add(b)); +atomic_rmw!(@old __sync_fetch_and_add_4, u32, |a: u32, b: u32| a + .wrapping_add(b)); + +atomic_rmw!(@new __sync_add_and_fetch_1, u8, |a: u8, b: u8| a.wrapping_add(b)); +atomic_rmw!(@new __sync_add_and_fetch_2, u16, |a: u16, b: u16| a .wrapping_add(b)); -atomic_rmw!(__sync_fetch_and_add_4, u32, |a: u32, b: u32| a +atomic_rmw!(@new __sync_add_and_fetch_4, u32, |a: u32, b: u32| a .wrapping_add(b)); -atomic_rmw!(__sync_fetch_and_sub_1, u8, |a: u8, b: u8| a.wrapping_sub(b)); -atomic_rmw!(__sync_fetch_and_sub_2, u16, |a: u16, b: u16| a +atomic_rmw!(@old __sync_fetch_and_sub_1, u8, |a: u8, b: u8| a.wrapping_sub(b)); +atomic_rmw!(@old __sync_fetch_and_sub_2, u16, |a: u16, b: u16| a .wrapping_sub(b)); -atomic_rmw!(__sync_fetch_and_sub_4, u32, |a: u32, b: u32| a +atomic_rmw!(@old __sync_fetch_and_sub_4, u32, |a: u32, b: u32| a .wrapping_sub(b)); -atomic_rmw!(__sync_fetch_and_and_1, u8, |a: u8, b: u8| a & b); -atomic_rmw!(__sync_fetch_and_and_2, u16, |a: u16, b: u16| a & b); -atomic_rmw!(__sync_fetch_and_and_4, u32, |a: u32, b: u32| a & b); +atomic_rmw!(@new __sync_sub_and_fetch_1, u8, |a: u8, b: u8| a.wrapping_sub(b)); +atomic_rmw!(@new __sync_sub_and_fetch_2, u16, |a: u16, b: u16| a + .wrapping_sub(b)); +atomic_rmw!(@new __sync_sub_and_fetch_4, u32, |a: u32, b: u32| a + .wrapping_sub(b)); + +atomic_rmw!(@old __sync_fetch_and_and_1, u8, |a: u8, b: u8| a & b); +atomic_rmw!(@old __sync_fetch_and_and_2, u16, |a: u16, b: u16| a & b); +atomic_rmw!(@old __sync_fetch_and_and_4, u32, |a: u32, b: u32| a & b); + +atomic_rmw!(@new __sync_and_and_fetch_1, u8, |a: u8, b: u8| a & b); +atomic_rmw!(@new __sync_and_and_fetch_2, u16, |a: u16, b: u16| a & b); +atomic_rmw!(@new __sync_and_and_fetch_4, u32, |a: u32, b: u32| a & b); -atomic_rmw!(__sync_fetch_and_or_1, u8, |a: u8, b: u8| a | b); -atomic_rmw!(__sync_fetch_and_or_2, u16, |a: u16, b: u16| a | b); -atomic_rmw!(__sync_fetch_and_or_4, u32, |a: u32, b: u32| a | b); +atomic_rmw!(@old __sync_fetch_and_or_1, u8, |a: u8, b: u8| a | b); +atomic_rmw!(@old __sync_fetch_and_or_2, u16, |a: u16, b: u16| a | b); +atomic_rmw!(@old __sync_fetch_and_or_4, u32, |a: u32, b: u32| a | b); -atomic_rmw!(__sync_fetch_and_xor_1, u8, |a: u8, b: u8| a ^ b); -atomic_rmw!(__sync_fetch_and_xor_2, u16, |a: u16, b: u16| a ^ b); -atomic_rmw!(__sync_fetch_and_xor_4, u32, |a: u32, b: u32| a ^ b); +atomic_rmw!(@new __sync_or_and_fetch_1, u8, |a: u8, b: u8| a | b); +atomic_rmw!(@new __sync_or_and_fetch_2, u16, |a: u16, b: u16| a | b); +atomic_rmw!(@new __sync_or_and_fetch_4, u32, |a: u32, b: u32| a | b); -atomic_rmw!(__sync_fetch_and_nand_1, u8, |a: u8, b: u8| !(a & b)); -atomic_rmw!(__sync_fetch_and_nand_2, u16, |a: u16, b: u16| !(a & b)); -atomic_rmw!(__sync_fetch_and_nand_4, u32, |a: u32, b: u32| !(a & b)); +atomic_rmw!(@old __sync_fetch_and_xor_1, u8, |a: u8, b: u8| a ^ b); +atomic_rmw!(@old __sync_fetch_and_xor_2, u16, |a: u16, b: u16| a ^ b); +atomic_rmw!(@old __sync_fetch_and_xor_4, u32, |a: u32, b: u32| a ^ b); -atomic_rmw!(__sync_fetch_and_max_1, i8, |a: i8, b: i8| if a > b { +atomic_rmw!(@new __sync_xor_and_fetch_1, u8, |a: u8, b: u8| a ^ b); +atomic_rmw!(@new __sync_xor_and_fetch_2, u16, |a: u16, b: u16| a ^ b); +atomic_rmw!(@new __sync_xor_and_fetch_4, u32, |a: u32, b: u32| a ^ b); + +atomic_rmw!(@old __sync_fetch_and_nand_1, u8, |a: u8, b: u8| !(a & b)); +atomic_rmw!(@old __sync_fetch_and_nand_2, u16, |a: u16, b: u16| !(a & b)); +atomic_rmw!(@old __sync_fetch_and_nand_4, u32, |a: u32, b: u32| !(a & b)); + +atomic_rmw!(@new __sync_nand_and_fetch_1, u8, |a: u8, b: u8| !(a & b)); +atomic_rmw!(@new __sync_nand_and_fetch_2, u16, |a: u16, b: u16| !(a & b)); +atomic_rmw!(@new __sync_nand_and_fetch_4, u32, |a: u32, b: u32| !(a & b)); + +atomic_rmw!(@old __sync_fetch_and_max_1, i8, |a: i8, b: i8| if a > b { a } else { b }); -atomic_rmw!(__sync_fetch_and_max_2, i16, |a: i16, b: i16| if a > b { +atomic_rmw!(@old __sync_fetch_and_max_2, i16, |a: i16, b: i16| if a > b { a } else { b }); -atomic_rmw!(__sync_fetch_and_max_4, i32, |a: i32, b: i32| if a > b { +atomic_rmw!(@old __sync_fetch_and_max_4, i32, |a: i32, b: i32| if a > b { a } else { b }); -atomic_rmw!(__sync_fetch_and_umax_1, u8, |a: u8, b: u8| if a > b { +atomic_rmw!(@old __sync_fetch_and_umax_1, u8, |a: u8, b: u8| if a > b { a } else { b }); -atomic_rmw!(__sync_fetch_and_umax_2, u16, |a: u16, b: u16| if a > b { +atomic_rmw!(@old __sync_fetch_and_umax_2, u16, |a: u16, b: u16| if a > b { a } else { b }); -atomic_rmw!(__sync_fetch_and_umax_4, u32, |a: u32, b: u32| if a > b { +atomic_rmw!(@old __sync_fetch_and_umax_4, u32, |a: u32, b: u32| if a > b { a } else { b }); -atomic_rmw!(__sync_fetch_and_min_1, i8, |a: i8, b: i8| if a < b { +atomic_rmw!(@old __sync_fetch_and_min_1, i8, |a: i8, b: i8| if a < b { a } else { b }); -atomic_rmw!(__sync_fetch_and_min_2, i16, |a: i16, b: i16| if a < b { +atomic_rmw!(@old __sync_fetch_and_min_2, i16, |a: i16, b: i16| if a < b { a } else { b }); -atomic_rmw!(__sync_fetch_and_min_4, i32, |a: i32, b: i32| if a < b { +atomic_rmw!(@old __sync_fetch_and_min_4, i32, |a: i32, b: i32| if a < b { a } else { b }); -atomic_rmw!(__sync_fetch_and_umin_1, u8, |a: u8, b: u8| if a < b { +atomic_rmw!(@old __sync_fetch_and_umin_1, u8, |a: u8, b: u8| if a < b { a } else { b }); -atomic_rmw!(__sync_fetch_and_umin_2, u16, |a: u16, b: u16| if a < b { +atomic_rmw!(@old __sync_fetch_and_umin_2, u16, |a: u16, b: u16| if a < b { a } else { b }); -atomic_rmw!(__sync_fetch_and_umin_4, u32, |a: u32, b: u32| if a < b { +atomic_rmw!(@old __sync_fetch_and_umin_4, u32, |a: u32, b: u32| if a < b { a } else { b }); -atomic_rmw!(__sync_lock_test_and_set_1, u8, |_: u8, b: u8| b); -atomic_rmw!(__sync_lock_test_and_set_2, u16, |_: u16, b: u16| b); -atomic_rmw!(__sync_lock_test_and_set_4, u32, |_: u32, b: u32| b); +atomic_rmw!(@old __sync_lock_test_and_set_1, u8, |_: u8, b: u8| b); +atomic_rmw!(@old __sync_lock_test_and_set_2, u16, |_: u16, b: u16| b); +atomic_rmw!(@old __sync_lock_test_and_set_4, u32, |_: u32, b: u32| b); atomic_cmpxchg!(__sync_val_compare_and_swap_1, u8); atomic_cmpxchg!(__sync_val_compare_and_swap_2, u16); atomic_cmpxchg!(__sync_val_compare_and_swap_4, u32); -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe extern "C" fn __sync_synchronize() { - __kuser_memory_barrier(); +intrinsics! { + pub unsafe extern "C" fn __sync_synchronize() { + __kuser_memory_barrier(); + } } diff --git a/crux-mir/lib/compiler_builtins/src/float/add.rs b/crux-mir/lib/compiler_builtins/src/float/add.rs index e8b9f9e77..67f6c2c14 100644 --- a/crux-mir/lib/compiler_builtins/src/float/add.rs +++ b/crux-mir/lib/compiler_builtins/src/float/add.rs @@ -137,9 +137,8 @@ where a_significand <<= shift; a_exponent -= shift; } - } else - /* addition */ - { + } else { + // addition a_significand += b_significand; // If the addition carried up, we need to right-shift the result and diff --git a/crux-mir/lib/compiler_builtins/src/float/cmp.rs b/crux-mir/lib/compiler_builtins/src/float/cmp.rs index 20ab92e4b..1d4e38433 100644 --- a/crux-mir/lib/compiler_builtins/src/float/cmp.rs +++ b/crux-mir/lib/compiler_builtins/src/float/cmp.rs @@ -1,7 +1,7 @@ #![allow(unreachable_code)] use float::Float; -use int::{CastInto, Int}; +use int::Int; #[derive(Clone, Copy)] enum Result { @@ -31,13 +31,7 @@ impl Result { } } -fn cmp(a: F, b: F) -> Result -where - u32: CastInto, - F::Int: CastInto, - i32: CastInto, - F::Int: CastInto, -{ +fn cmp(a: F, b: F) -> Result { let one = F::Int::ONE; let zero = F::Int::ZERO; let szero = F::SignedInt::ZERO; @@ -69,34 +63,26 @@ where // a and b as signed integers as we would with a fp_ting-point compare. if a_srep & b_srep >= szero { if a_srep < b_srep { - return Result::Less; + Result::Less } else if a_srep == b_srep { - return Result::Equal; + Result::Equal } else { - return Result::Greater; + Result::Greater } - } // Otherwise, both are negative, so we need to flip the sense of the // comparison to get the correct result. (This assumes a twos- or ones- // complement integer representation; if integers are represented in a // sign-magnitude representation, then this flip is incorrect). - else { - if a_srep > b_srep { - return Result::Less; - } else if a_srep == b_srep { - return Result::Equal; - } else { - return Result::Greater; - } + } else if a_srep > b_srep { + Result::Less + } else if a_srep == b_srep { + Result::Equal + } else { + Result::Greater } } -fn unord(a: F, b: F) -> bool -where - u32: CastInto, - F::Int: CastInto, - i32: CastInto, - F::Int: CastInto, -{ + +fn unord(a: F, b: F) -> bool { let one = F::Int::ONE; let sign_bit = F::SIGN_MASK as F::Int; diff --git a/crux-mir/lib/compiler_builtins/src/float/conv.rs b/crux-mir/lib/compiler_builtins/src/float/conv.rs index 8a0fc6cb4..19fdc2fdc 100644 --- a/crux-mir/lib/compiler_builtins/src/float/conv.rs +++ b/crux-mir/lib/compiler_builtins/src/float/conv.rs @@ -1,287 +1,351 @@ -use float::Float; -use int::Int; - -macro_rules! int_to_float { - ($i:expr, $ity:ty, $fty:ty) => {{ - let i = $i; +/// Conversions from integers to floats. +/// +/// These are hand-optimized bit twiddling code, +/// which unfortunately isn't the easiest kind of code to read. +/// +/// The algorithm is explained here: https://blog.m-ou.se/floats/ +mod int_to_float { + pub fn u32_to_f32_bits(i: u32) -> u32 { if i == 0 { - return 0.0; - } - - let mant_dig = <$fty>::SIGNIFICAND_BITS + 1; - let exponent_bias = <$fty>::EXPONENT_BIAS; - - let n = <$ity>::BITS; - let (s, a) = i.extract_sign(); - let mut a = a; - - // number of significant digits - let sd = n - a.leading_zeros(); - - // exponent - let mut e = sd - 1; - - if <$ity>::BITS < mant_dig { - return <$fty>::from_parts( - s, - (e + exponent_bias) as <$fty as Float>::Int, - (a as <$fty as Float>::Int) << (mant_dig - e - 1), - ); + return 0; } - - a = if sd > mant_dig { - /* start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx - * finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR - * 12345678901234567890123456 - * 1 = msb 1 bit - * P = bit MANT_DIG-1 bits to the right of 1 - * Q = bit MANT_DIG bits to the right of 1 - * R = "or" of all bits to the right of Q - */ - let mant_dig_plus_one = mant_dig + 1; - let mant_dig_plus_two = mant_dig + 2; - a = if sd == mant_dig_plus_one { - a << 1 - } else if sd == mant_dig_plus_two { - a - } else { - (a >> (sd - mant_dig_plus_two)) as <$ity as Int>::UnsignedInt - | ((a & <$ity as Int>::UnsignedInt::max_value()) - .wrapping_shl((n + mant_dig_plus_two) - sd) - != 0) as <$ity as Int>::UnsignedInt - }; - - /* finish: */ - a |= ((a & 4) != 0) as <$ity as Int>::UnsignedInt; /* Or P into R */ - a += 1; /* round - this step may add a significant bit */ - a >>= 2; /* dump Q and R */ - - /* a is now rounded to mant_dig or mant_dig+1 bits */ - if (a & (1 << mant_dig)) != 0 { - a >>= 1; - e += 1; - } - a - /* a is now rounded to mant_dig bits */ - } else { - a.wrapping_shl(mant_dig - sd) - /* a is now rounded to mant_dig bits */ - }; - - <$fty>::from_parts( - s, - (e + exponent_bias) as <$fty as Float>::Int, - a as <$fty as Float>::Int, - ) - }}; -} - -intrinsics! { - #[arm_aeabi_alias = __aeabi_i2f] - pub extern "C" fn __floatsisf(i: i32) -> f32 { - int_to_float!(i, i32, f32) + let n = i.leading_zeros(); + let a = (i << n) >> 8; // Significant bits, with bit 24 still in tact. + let b = (i << n) << 24; // Insignificant bits, only relevant for rounding. + let m = a + ((b - (b >> 31 & !a)) >> 31); // Add one when we need to round up. Break ties to even. + let e = 157 - n; // Exponent plus 127, minus one. + (e << 23) + m // + not |, so the mantissa can overflow into the exponent. } - #[arm_aeabi_alias = __aeabi_i2d] - pub extern "C" fn __floatsidf(i: i32) -> f64 { - int_to_float!(i, i32, f64) + pub fn u32_to_f64_bits(i: u32) -> u64 { + if i == 0 { + return 0; + } + let n = i.leading_zeros(); + let m = (i as u64) << (21 + n); // Significant bits, with bit 53 still in tact. + let e = 1053 - n as u64; // Exponent plus 1023, minus one. + (e << 52) + m // Bit 53 of m will overflow into e. } - #[maybe_use_optimized_c_shim] - #[arm_aeabi_alias = __aeabi_l2f] - pub extern "C" fn __floatdisf(i: i64) -> f32 { - // On x86_64 LLVM will use native instructions for this conversion, we - // can just do it directly - if cfg!(target_arch = "x86_64") { - i as f32 - } else { - int_to_float!(i, i64, f32) - } + pub fn u64_to_f32_bits(i: u64) -> u32 { + let n = i.leading_zeros(); + let y = i.wrapping_shl(n); + let a = (y >> 40) as u32; // Significant bits, with bit 24 still in tact. + let b = (y >> 8 | y & 0xFFFF) as u32; // Insignificant bits, only relevant for rounding. + let m = a + ((b - (b >> 31 & !a)) >> 31); // Add one when we need to round up. Break ties to even. + let e = if i == 0 { 0 } else { 189 - n }; // Exponent plus 127, minus one, except for zero. + (e << 23) + m // + not |, so the mantissa can overflow into the exponent. } - #[maybe_use_optimized_c_shim] - #[arm_aeabi_alias = __aeabi_l2d] - pub extern "C" fn __floatdidf(i: i64) -> f64 { - // On x86_64 LLVM will use native instructions for this conversion, we - // can just do it directly - if cfg!(target_arch = "x86_64") { - i as f64 - } else { - int_to_float!(i, i64, f64) + pub fn u64_to_f64_bits(i: u64) -> u64 { + if i == 0 { + return 0; } + let n = i.leading_zeros(); + let a = (i << n) >> 11; // Significant bits, with bit 53 still in tact. + let b = (i << n) << 53; // Insignificant bits, only relevant for rounding. + let m = a + ((b - (b >> 63 & !a)) >> 63); // Add one when we need to round up. Break ties to even. + let e = 1085 - n as u64; // Exponent plus 1023, minus one. + (e << 52) + m // + not |, so the mantissa can overflow into the exponent. } - #[unadjusted_on_win64] - pub extern "C" fn __floattisf(i: i128) -> f32 { - int_to_float!(i, i128, f32) + pub fn u128_to_f32_bits(i: u128) -> u32 { + let n = i.leading_zeros(); + let y = i.wrapping_shl(n); + let a = (y >> 104) as u32; // Significant bits, with bit 24 still in tact. + let b = (y >> 72) as u32 | ((y << 32) >> 32 != 0) as u32; // Insignificant bits, only relevant for rounding. + let m = a + ((b - (b >> 31 & !a)) >> 31); // Add one when we need to round up. Break ties to even. + let e = if i == 0 { 0 } else { 253 - n }; // Exponent plus 127, minus one, except for zero. + (e << 23) + m // + not |, so the mantissa can overflow into the exponent. } - #[unadjusted_on_win64] - pub extern "C" fn __floattidf(i: i128) -> f64 { - int_to_float!(i, i128, f64) + pub fn u128_to_f64_bits(i: u128) -> u64 { + let n = i.leading_zeros(); + let y = i.wrapping_shl(n); + let a = (y >> 75) as u64; // Significant bits, with bit 53 still in tact. + let b = (y >> 11 | y & 0xFFFF_FFFF) as u64; // Insignificant bits, only relevant for rounding. + let m = a + ((b - (b >> 63 & !a)) >> 63); // Add one when we need to round up. Break ties to even. + let e = if i == 0 { 0 } else { 1149 - n as u64 }; // Exponent plus 1023, minus one, except for zero. + (e << 52) + m // + not |, so the mantissa can overflow into the exponent. } +} +// Conversions from unsigned integers to floats. +intrinsics! { #[arm_aeabi_alias = __aeabi_ui2f] pub extern "C" fn __floatunsisf(i: u32) -> f32 { - int_to_float!(i, u32, f32) + f32::from_bits(int_to_float::u32_to_f32_bits(i)) } #[arm_aeabi_alias = __aeabi_ui2d] pub extern "C" fn __floatunsidf(i: u32) -> f64 { - int_to_float!(i, u32, f64) + f64::from_bits(int_to_float::u32_to_f64_bits(i)) } - #[maybe_use_optimized_c_shim] #[arm_aeabi_alias = __aeabi_ul2f] pub extern "C" fn __floatundisf(i: u64) -> f32 { - int_to_float!(i, u64, f32) + f32::from_bits(int_to_float::u64_to_f32_bits(i)) } - #[maybe_use_optimized_c_shim] #[arm_aeabi_alias = __aeabi_ul2d] pub extern "C" fn __floatundidf(i: u64) -> f64 { - int_to_float!(i, u64, f64) + f64::from_bits(int_to_float::u64_to_f64_bits(i)) } - #[unadjusted_on_win64] + #[cfg_attr(any(not(target_feature = "llvm14-builtins-abi"), target_os = "uefi"), unadjusted_on_win64)] pub extern "C" fn __floatuntisf(i: u128) -> f32 { - int_to_float!(i, u128, f32) + f32::from_bits(int_to_float::u128_to_f32_bits(i)) } - #[unadjusted_on_win64] + #[cfg_attr(any(not(target_feature = "llvm14-builtins-abi"), target_os = "uefi"), unadjusted_on_win64)] pub extern "C" fn __floatuntidf(i: u128) -> f64 { - int_to_float!(i, u128, f64) + f64::from_bits(int_to_float::u128_to_f64_bits(i)) } } -#[derive(PartialEq)] -enum Sign { - Positive, - Negative, -} - -macro_rules! float_to_int { - ($f:expr, $fty:ty, $ity:ty) => {{ - let f = $f; - let fixint_min = <$ity>::min_value(); - let fixint_max = <$ity>::max_value(); - let fixint_bits = <$ity>::BITS as usize; - let fixint_unsigned = fixint_min == 0; - - let sign_bit = <$fty>::SIGN_MASK; - let significand_bits = <$fty>::SIGNIFICAND_BITS as usize; - let exponent_bias = <$fty>::EXPONENT_BIAS as usize; - //let exponent_max = <$fty>::exponent_max() as usize; - - // Break a into sign, exponent, significand - let a_rep = <$fty>::repr(f); - let a_abs = a_rep & !sign_bit; - - // this is used to work around -1 not being available for unsigned - let sign = if (a_rep & sign_bit) == 0 { - Sign::Positive - } else { - Sign::Negative - }; - let mut exponent = (a_abs >> significand_bits) as usize; - let significand = (a_abs & <$fty>::SIGNIFICAND_MASK) | <$fty>::IMPLICIT_BIT; - - // if < 1 or unsigned & negative - if exponent < exponent_bias || fixint_unsigned && sign == Sign::Negative { - return 0; - } - exponent -= exponent_bias; - - // If the value is infinity, saturate. - // If the value is too large for the integer type, 0. - if exponent - >= (if fixint_unsigned { - fixint_bits - } else { - fixint_bits - 1 - }) - { - return if sign == Sign::Positive { - fixint_max - } else { - fixint_min - }; - } - // If 0 <= exponent < significand_bits, right shift to get the result. - // Otherwise, shift left. - // (sign - 1) will never overflow as negative signs are already returned as 0 for unsigned - let r = if exponent < significand_bits { - (significand >> (significand_bits - exponent)) as $ity - } else { - (significand as $ity) << (exponent - significand_bits) - }; - - if sign == Sign::Negative { - (!r).wrapping_add(1) - } else { - r - } - }}; -} - +// Conversions from signed integers to floats. intrinsics! { - #[arm_aeabi_alias = __aeabi_f2iz] - pub extern "C" fn __fixsfsi(f: f32) -> i32 { - float_to_int!(f, f32, i32) + #[arm_aeabi_alias = __aeabi_i2f] + pub extern "C" fn __floatsisf(i: i32) -> f32 { + let sign_bit = ((i >> 31) as u32) << 31; + f32::from_bits(int_to_float::u32_to_f32_bits(i.unsigned_abs()) | sign_bit) } - #[arm_aeabi_alias = __aeabi_f2lz] - pub extern "C" fn __fixsfdi(f: f32) -> i64 { - float_to_int!(f, f32, i64) + #[arm_aeabi_alias = __aeabi_i2d] + pub extern "C" fn __floatsidf(i: i32) -> f64 { + let sign_bit = ((i >> 31) as u64) << 63; + f64::from_bits(int_to_float::u32_to_f64_bits(i.unsigned_abs()) | sign_bit) } - #[unadjusted_on_win64] - pub extern "C" fn __fixsfti(f: f32) -> i128 { - float_to_int!(f, f32, i128) + #[arm_aeabi_alias = __aeabi_l2f] + pub extern "C" fn __floatdisf(i: i64) -> f32 { + let sign_bit = ((i >> 63) as u32) << 31; + f32::from_bits(int_to_float::u64_to_f32_bits(i.unsigned_abs()) | sign_bit) } - #[arm_aeabi_alias = __aeabi_d2iz] - pub extern "C" fn __fixdfsi(f: f64) -> i32 { - float_to_int!(f, f64, i32) + #[arm_aeabi_alias = __aeabi_l2d] + pub extern "C" fn __floatdidf(i: i64) -> f64 { + let sign_bit = ((i >> 63) as u64) << 63; + f64::from_bits(int_to_float::u64_to_f64_bits(i.unsigned_abs()) | sign_bit) } - #[arm_aeabi_alias = __aeabi_d2lz] - pub extern "C" fn __fixdfdi(f: f64) -> i64 { - float_to_int!(f, f64, i64) + #[cfg_attr(any(not(target_feature = "llvm14-builtins-abi"), target_os = "uefi"), unadjusted_on_win64)] + pub extern "C" fn __floattisf(i: i128) -> f32 { + let sign_bit = ((i >> 127) as u32) << 31; + f32::from_bits(int_to_float::u128_to_f32_bits(i.unsigned_abs()) | sign_bit) } - #[unadjusted_on_win64] - pub extern "C" fn __fixdfti(f: f64) -> i128 { - float_to_int!(f, f64, i128) + #[cfg_attr(any(not(target_feature = "llvm14-builtins-abi"), target_os = "uefi"), unadjusted_on_win64)] + pub extern "C" fn __floattidf(i: i128) -> f64 { + let sign_bit = ((i >> 127) as u64) << 63; + f64::from_bits(int_to_float::u128_to_f64_bits(i.unsigned_abs()) | sign_bit) } +} +// Conversions from floats to unsigned integers. +intrinsics! { #[arm_aeabi_alias = __aeabi_f2uiz] pub extern "C" fn __fixunssfsi(f: f32) -> u32 { - float_to_int!(f, f32, u32) + let fbits = f.to_bits(); + if fbits < 127 << 23 { // >= 0, < 1 + 0 + } else if fbits < 159 << 23 { // >= 1, < max + let m = 1 << 31 | fbits << 8; // Mantissa and the implicit 1-bit. + let s = 158 - (fbits >> 23); // Shift based on the exponent and bias. + m >> s + } else if fbits <= 255 << 23 { // >= max (incl. inf) + u32::MAX + } else { // Negative or NaN + 0 + } } #[arm_aeabi_alias = __aeabi_f2ulz] pub extern "C" fn __fixunssfdi(f: f32) -> u64 { - float_to_int!(f, f32, u64) + let fbits = f.to_bits(); + if fbits < 127 << 23 { // >= 0, < 1 + 0 + } else if fbits < 191 << 23 { // >= 1, < max + let m = 1 << 63 | (fbits as u64) << 40; // Mantissa and the implicit 1-bit. + let s = 190 - (fbits >> 23); // Shift based on the exponent and bias. + m >> s + } else if fbits <= 255 << 23 { // >= max (incl. inf) + u64::MAX + } else { // Negative or NaN + 0 + } } - #[unadjusted_on_win64] + #[cfg_attr(target_feature = "llvm14-builtins-abi", win64_128bit_abi_hack)] + #[cfg_attr(not(target_feature = "llvm14-builtins-abi"), unadjusted_on_win64)] pub extern "C" fn __fixunssfti(f: f32) -> u128 { - float_to_int!(f, f32, u128) + let fbits = f.to_bits(); + if fbits < 127 << 23 { // >= 0, < 1 + 0 + } else if fbits < 255 << 23 { // >= 1, < inf + let m = 1 << 127 | (fbits as u128) << 104; // Mantissa and the implicit 1-bit. + let s = 254 - (fbits >> 23); // Shift based on the exponent and bias. + m >> s + } else if fbits == 255 << 23 { // == inf + u128::MAX + } else { // Negative or NaN + 0 + } } #[arm_aeabi_alias = __aeabi_d2uiz] pub extern "C" fn __fixunsdfsi(f: f64) -> u32 { - float_to_int!(f, f64, u32) + let fbits = f.to_bits(); + if fbits < 1023 << 52 { // >= 0, < 1 + 0 + } else if fbits < 1055 << 52 { // >= 1, < max + let m = 1 << 31 | (fbits >> 21) as u32; // Mantissa and the implicit 1-bit. + let s = 1054 - (fbits >> 52); // Shift based on the exponent and bias. + m >> s + } else if fbits <= 2047 << 52 { // >= max (incl. inf) + u32::MAX + } else { // Negative or NaN + 0 + } } #[arm_aeabi_alias = __aeabi_d2ulz] pub extern "C" fn __fixunsdfdi(f: f64) -> u64 { - float_to_int!(f, f64, u64) + let fbits = f.to_bits(); + if fbits < 1023 << 52 { // >= 0, < 1 + 0 + } else if fbits < 1087 << 52 { // >= 1, < max + let m = 1 << 63 | fbits << 11; // Mantissa and the implicit 1-bit. + let s = 1086 - (fbits >> 52); // Shift based on the exponent and bias. + m >> s + } else if fbits <= 2047 << 52 { // >= max (incl. inf) + u64::MAX + } else { // Negative or NaN + 0 + } } - #[unadjusted_on_win64] + #[cfg_attr(target_feature = "llvm14-builtins-abi", win64_128bit_abi_hack)] + #[cfg_attr(not(target_feature = "llvm14-builtins-abi"), unadjusted_on_win64)] pub extern "C" fn __fixunsdfti(f: f64) -> u128 { - float_to_int!(f, f64, u128) + let fbits = f.to_bits(); + if fbits < 1023 << 52 { // >= 0, < 1 + 0 + } else if fbits < 1151 << 52 { // >= 1, < max + let m = 1 << 127 | (fbits as u128) << 75; // Mantissa and the implicit 1-bit. + let s = 1150 - (fbits >> 52); // Shift based on the exponent and bias. + m >> s + } else if fbits <= 2047 << 52 { // >= max (incl. inf) + u128::MAX + } else { // Negative or NaN + 0 + } + } +} + +// Conversions from floats to signed integers. +intrinsics! { + #[arm_aeabi_alias = __aeabi_f2iz] + pub extern "C" fn __fixsfsi(f: f32) -> i32 { + let fbits = f.to_bits() & !0 >> 1; // Remove sign bit. + if fbits < 127 << 23 { // >= 0, < 1 + 0 + } else if fbits < 158 << 23 { // >= 1, < max + let m = 1 << 31 | fbits << 8; // Mantissa and the implicit 1-bit. + let s = 158 - (fbits >> 23); // Shift based on the exponent and bias. + let u = (m >> s) as i32; // Unsigned result. + if f.is_sign_negative() { -u } else { u } + } else if fbits <= 255 << 23 { // >= max (incl. inf) + if f.is_sign_negative() { i32::MIN } else { i32::MAX } + } else { // NaN + 0 + } + } + + #[arm_aeabi_alias = __aeabi_f2lz] + pub extern "C" fn __fixsfdi(f: f32) -> i64 { + let fbits = f.to_bits() & !0 >> 1; // Remove sign bit. + if fbits < 127 << 23 { // >= 0, < 1 + 0 + } else if fbits < 190 << 23 { // >= 1, < max + let m = 1 << 63 | (fbits as u64) << 40; // Mantissa and the implicit 1-bit. + let s = 190 - (fbits >> 23); // Shift based on the exponent and bias. + let u = (m >> s) as i64; // Unsigned result. + if f.is_sign_negative() { -u } else { u } + } else if fbits <= 255 << 23 { // >= max (incl. inf) + if f.is_sign_negative() { i64::MIN } else { i64::MAX } + } else { // NaN + 0 + } + } + + #[cfg_attr(target_feature = "llvm14-builtins-abi", win64_128bit_abi_hack)] + #[cfg_attr(not(target_feature = "llvm14-builtins-abi"), unadjusted_on_win64)] + pub extern "C" fn __fixsfti(f: f32) -> i128 { + let fbits = f.to_bits() & !0 >> 1; // Remove sign bit. + if fbits < 127 << 23 { // >= 0, < 1 + 0 + } else if fbits < 254 << 23 { // >= 1, < max + let m = 1 << 127 | (fbits as u128) << 104; // Mantissa and the implicit 1-bit. + let s = 254 - (fbits >> 23); // Shift based on the exponent and bias. + let u = (m >> s) as i128; // Unsigned result. + if f.is_sign_negative() { -u } else { u } + } else if fbits <= 255 << 23 { // >= max (incl. inf) + if f.is_sign_negative() { i128::MIN } else { i128::MAX } + } else { // NaN + 0 + } + } + + #[arm_aeabi_alias = __aeabi_d2iz] + pub extern "C" fn __fixdfsi(f: f64) -> i32 { + let fbits = f.to_bits() & !0 >> 1; // Remove sign bit. + if fbits < 1023 << 52 { // >= 0, < 1 + 0 + } else if fbits < 1054 << 52 { // >= 1, < max + let m = 1 << 31 | (fbits >> 21) as u32; // Mantissa and the implicit 1-bit. + let s = 1054 - (fbits >> 52); // Shift based on the exponent and bias. + let u = (m >> s) as i32; // Unsigned result. + if f.is_sign_negative() { -u } else { u } + } else if fbits <= 2047 << 52 { // >= max (incl. inf) + if f.is_sign_negative() { i32::MIN } else { i32::MAX } + } else { // NaN + 0 + } + } + + #[arm_aeabi_alias = __aeabi_d2lz] + pub extern "C" fn __fixdfdi(f: f64) -> i64 { + let fbits = f.to_bits() & !0 >> 1; // Remove sign bit. + if fbits < 1023 << 52 { // >= 0, < 1 + 0 + } else if fbits < 1086 << 52 { // >= 1, < max + let m = 1 << 63 | fbits << 11; // Mantissa and the implicit 1-bit. + let s = 1086 - (fbits >> 52); // Shift based on the exponent and bias. + let u = (m >> s) as i64; // Unsigned result. + if f.is_sign_negative() { -u } else { u } + } else if fbits <= 2047 << 52 { // >= max (incl. inf) + if f.is_sign_negative() { i64::MIN } else { i64::MAX } + } else { // NaN + 0 + } + } + + #[cfg_attr(target_feature = "llvm14-builtins-abi", win64_128bit_abi_hack)] + #[cfg_attr(not(target_feature = "llvm14-builtins-abi"), unadjusted_on_win64)] + pub extern "C" fn __fixdfti(f: f64) -> i128 { + let fbits = f.to_bits() & !0 >> 1; // Remove sign bit. + if fbits < 1023 << 52 { // >= 0, < 1 + 0 + } else if fbits < 1150 << 52 { // >= 1, < max + let m = 1 << 127 | (fbits as u128) << 75; // Mantissa and the implicit 1-bit. + let s = 1150 - (fbits >> 52); // Shift based on the exponent and bias. + let u = (m >> s) as i128; // Unsigned result. + if f.is_sign_negative() { -u } else { u } + } else if fbits <= 2047 << 52 { // >= max (incl. inf) + if f.is_sign_negative() { i128::MIN } else { i128::MAX } + } else { // NaN + 0 + } } } diff --git a/crux-mir/lib/compiler_builtins/src/float/div.rs b/crux-mir/lib/compiler_builtins/src/float/div.rs index 7c582a440..c2d6c07e7 100644 --- a/crux-mir/lib/compiler_builtins/src/float/div.rs +++ b/crux-mir/lib/compiler_builtins/src/float/div.rs @@ -1,5 +1,9 @@ +// The functions are complex with many branches, and explicit +// `return`s makes it clear where function exit points are +#![allow(clippy::needless_return)] + use float::Float; -use int::{CastInto, Int, WideInt}; +use int::{CastInto, DInt, HInt, Int}; fn div32(a: F, b: F) -> F where @@ -7,7 +11,7 @@ where F::Int: CastInto, i32: CastInto, F::Int: CastInto, - F::Int: WideInt, + F::Int: HInt, { let one = F::Int::ONE; let zero = F::Int::ZERO; @@ -128,13 +132,14 @@ where // This doubles the number of correct binary digits in the approximation // with each iteration, so after three iterations, we have about 28 binary // digits of accuracy. - let mut correction: u32; - correction = negate_u32(((reciprocal as u64).wrapping_mul(q31b as u64) >> 32) as u32); - reciprocal = ((reciprocal as u64).wrapping_mul(correction as u64) as u64 >> 31) as u32; + + let mut correction: u32 = + negate_u32(((reciprocal as u64).wrapping_mul(q31b as u64) >> 32) as u32); + reciprocal = ((reciprocal as u64).wrapping_mul(correction as u64) >> 31) as u32; correction = negate_u32(((reciprocal as u64).wrapping_mul(q31b as u64) >> 32) as u32); - reciprocal = ((reciprocal as u64).wrapping_mul(correction as u64) as u64 >> 31) as u32; + reciprocal = ((reciprocal as u64).wrapping_mul(correction as u64) >> 31) as u32; correction = negate_u32(((reciprocal as u64).wrapping_mul(q31b as u64) >> 32) as u32); - reciprocal = ((reciprocal as u64).wrapping_mul(correction as u64) as u64 >> 31) as u32; + reciprocal = ((reciprocal as u64).wrapping_mul(correction as u64) >> 31) as u32; // Exhaustive testing shows that the error in reciprocal after three steps // is in the interval [-0x1.f58108p-31, 0x1.d0e48cp-29], in line with our @@ -156,7 +161,7 @@ where // is the error in the reciprocal of b scaled by the maximum // possible value of a. As a consequence of this error bound, // either q or nextafter(q) is the correctly rounded - let (mut quotient, _) = ::wide_mul(a_significand << 1, reciprocal.cast()); + let mut quotient = (a_significand << 1).widen_mul(reciprocal.cast()).hi(); // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0). // In either case, we are going to compute a residual of the form @@ -211,7 +216,7 @@ where F::Int: CastInto, i64: CastInto, F::Int: CastInto, - F::Int: WideInt, + F::Int: HInt, { let one = F::Int::ONE; let zero = F::Int::ZERO; @@ -338,8 +343,9 @@ where // This doubles the number of correct binary digits in the approximation // with each iteration, so after three iterations, we have about 28 binary // digits of accuracy. - let mut correction32: u32; - correction32 = negate_u32(((recip32 as u64).wrapping_mul(q31b as u64) >> 32) as u32); + + let mut correction32: u32 = + negate_u32(((recip32 as u64).wrapping_mul(q31b as u64) >> 32) as u32); recip32 = ((recip32 as u64).wrapping_mul(correction32 as u64) >> 31) as u32; correction32 = negate_u32(((recip32 as u64).wrapping_mul(q31b as u64) >> 32) as u32); recip32 = ((recip32 as u64).wrapping_mul(correction32 as u64) >> 31) as u32; @@ -355,16 +361,15 @@ where // We need to perform one more iteration to get us to 56 binary digits; // The last iteration needs to happen with extra precision. let q63blo = CastInto::::cast(b_significand << 11.cast()); - let correction: u64; - let mut reciprocal: u64; - correction = negate_u64( + + let correction: u64 = negate_u64( (recip32 as u64) .wrapping_mul(q31b as u64) .wrapping_add((recip32 as u64).wrapping_mul(q63blo as u64) >> 32), ); let c_hi = (correction >> 32) as u32; let c_lo = correction as u32; - reciprocal = (recip32 as u64) + let mut reciprocal: u64 = (recip32 as u64) .wrapping_mul(c_hi as u64) .wrapping_add((recip32 as u64).wrapping_mul(c_lo as u64) >> 32); @@ -394,7 +399,7 @@ where // We need a 64 x 64 multiply high to compute q, which isn't a basic // operation in C, so we need to be a little bit fussy. - let (mut quotient, _) = ::wide_mul(a_significand << 2, reciprocal.cast()); + let mut quotient = (a_significand << 2).widen_mul(reciprocal.cast()).hi(); // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0). // In either case, we are going to compute a residual of the form diff --git a/crux-mir/lib/compiler_builtins/src/float/mod.rs b/crux-mir/lib/compiler_builtins/src/float/mod.rs index 8b8039452..fdbe9dde3 100644 --- a/crux-mir/lib/compiler_builtins/src/float/mod.rs +++ b/crux-mir/lib/compiler_builtins/src/float/mod.rs @@ -1,4 +1,3 @@ -use core::mem; use core::ops; use super::int::Int; @@ -11,10 +10,13 @@ pub mod extend; pub mod mul; pub mod pow; pub mod sub; +pub mod trunc; +public_test_dep! { /// Trait for some basic operations on floats -pub trait Float: +pub(crate) trait Float: Copy + + core::fmt::Debug + PartialEq + PartialOrd + ops::AddAssign @@ -24,12 +26,15 @@ pub trait Float: + ops::Div + ops::Rem { - /// A uint of the same with as the float + /// A uint of the same width as the float type Int: Int; - /// A int of the same with as the float + /// A int of the same width as the float type SignedInt: Int; + /// An int capable of containing the exponent bits plus a sign bit. This is signed. + type ExpInt: Int; + const ZERO: Self; const ONE: Self; @@ -66,12 +71,23 @@ pub trait Float: /// Returns `self` transmuted to `Self::SignedInt` fn signed_repr(self) -> Self::SignedInt; - #[cfg(test)] /// Checks if two floats have the same bit representation. *Except* for NaNs! NaN can be /// represented in multiple different ways. This method returns `true` if two NaNs are /// compared. fn eq_repr(self, rhs: Self) -> bool; + /// Returns the sign bit + fn sign(self) -> bool; + + /// Returns the exponent with bias + fn exp(self) -> Self::ExpInt; + + /// Returns the significand with no implicit bit (or the "fractional" part) + fn frac(self) -> Self::Int; + + /// Returns the significand with implicit bit + fn imp_frac(self) -> Self::Int; + /// Returns a `Self::Int` transmuted back to `Self` fn from_repr(a: Self::Int) -> Self; @@ -80,15 +96,19 @@ pub trait Float: /// Returns (normalized exponent, normalized significand) fn normalize(significand: Self::Int) -> (i32, Self::Int); + + /// Returns if `self` is subnormal + fn is_subnormal(self) -> bool; +} } -// FIXME: Some of this can be removed if RFC Issue #1424 is resolved -// https://github.com/rust-lang/rfcs/issues/1424 macro_rules! float_impl { - ($ty:ident, $ity:ident, $sity:ident, $bits:expr, $significand_bits:expr) => { + ($ty:ident, $ity:ident, $sity:ident, $expty:ident, $bits:expr, $significand_bits:expr) => { impl Float for $ty { type Int = $ity; type SignedInt = $sity; + type ExpInt = $expty; + const ZERO: Self = 0.0; const ONE: Self = 1.0; @@ -101,12 +121,11 @@ macro_rules! float_impl { const EXPONENT_MASK: Self::Int = !(Self::SIGN_MASK | Self::SIGNIFICAND_MASK); fn repr(self) -> Self::Int { - unsafe { mem::transmute(self) } + self.to_bits() } fn signed_repr(self) -> Self::SignedInt { - unsafe { mem::transmute(self) } + self.to_bits() as Self::SignedInt } - #[cfg(test)] fn eq_repr(self, rhs: Self) -> bool { if self.is_nan() && rhs.is_nan() { true @@ -114,8 +133,20 @@ macro_rules! float_impl { self.repr() == rhs.repr() } } + fn sign(self) -> bool { + self.signed_repr() < Self::SignedInt::ZERO + } + fn exp(self) -> Self::ExpInt { + ((self.to_bits() & Self::EXPONENT_MASK) >> Self::SIGNIFICAND_BITS) as Self::ExpInt + } + fn frac(self) -> Self::Int { + self.to_bits() & Self::SIGNIFICAND_MASK + } + fn imp_frac(self) -> Self::Int { + self.frac() | Self::IMPLICIT_BIT + } fn from_repr(a: Self::Int) -> Self { - unsafe { mem::transmute(a) } + Self::from_bits(a) } fn from_parts(sign: bool, exponent: Self::Int, significand: Self::Int) -> Self { Self::from_repr( @@ -133,9 +164,12 @@ macro_rules! float_impl { significand << shift as Self::Int, ) } + fn is_subnormal(self) -> bool { + (self.repr() & Self::EXPONENT_MASK) == Self::Int::ZERO + } } }; } -float_impl!(f32, u32, i32, 32, 23); -float_impl!(f64, u64, i64, 64, 52); +float_impl!(f32, u32, i32, i16, 32, 23); +float_impl!(f64, u64, i64, i16, 64, 52); diff --git a/crux-mir/lib/compiler_builtins/src/float/mul.rs b/crux-mir/lib/compiler_builtins/src/float/mul.rs index 7b28793c8..c89f22756 100644 --- a/crux-mir/lib/compiler_builtins/src/float/mul.rs +++ b/crux-mir/lib/compiler_builtins/src/float/mul.rs @@ -1,5 +1,5 @@ use float::Float; -use int::{CastInto, Int, WideInt}; +use int::{CastInto, DInt, HInt, Int}; fn mul(a: F, b: F) -> F where @@ -7,7 +7,7 @@ where F::Int: CastInto, i32: CastInto, F::Int: CastInto, - F::Int: WideInt, + F::Int: HInt, { let one = F::Int::ONE; let zero = F::Int::ZERO; @@ -112,8 +112,9 @@ where // have (exponentBits + 2) integral digits, all but two of which must be // zero. Normalizing this result is just a conditional left-shift by one // and bumping the exponent accordingly. - let (mut product_high, mut product_low) = - ::wide_mul(a_significand, b_significand << exponent_bits); + let (mut product_low, mut product_high) = a_significand + .widen_mul(b_significand << exponent_bits) + .lo_hi(); let a_exponent_i32: i32 = a_exponent.cast(); let b_exponent_i32: i32 = b_exponent.cast(); @@ -126,7 +127,8 @@ where if (product_high & implicit_bit) != zero { product_exponent = product_exponent.wrapping_add(1); } else { - ::wide_shift_left(&mut product_high, &mut product_low, 1); + product_high = (product_high << 1) | (product_low >> (bits - 1)); + product_low <<= 1; } // If we have overflowed the type, return +/- infinity. @@ -142,17 +144,23 @@ where // handle this case separately, but we make it a special case to // simplify the shift logic. let shift = one.wrapping_sub(product_exponent.cast()).cast(); - if shift >= bits as i32 { + if shift >= bits { return F::from_repr(product_sign); } // Otherwise, shift the significand of the result so that the round // bit is the high bit of productLo. - ::wide_shift_right_with_sticky( - &mut product_high, - &mut product_low, - shift, - ) + if shift < bits { + let sticky = product_low << (bits - shift); + product_low = product_high << (bits - shift) | product_low >> shift | sticky; + product_high >>= shift; + } else if shift < (2 * bits) { + let sticky = product_high << (2 * bits - shift) | product_low; + product_low = product_high >> (shift - bits) | sticky; + product_high = zero; + } else { + product_high = zero; + } } else { // Result is normal before rounding; insert the exponent. product_high &= significand_mask; @@ -173,7 +181,7 @@ where product_high += product_high & one; } - return F::from_repr(product_high); + F::from_repr(product_high) } intrinsics! { diff --git a/crux-mir/lib/compiler_builtins/src/float/pow.rs b/crux-mir/lib/compiler_builtins/src/float/pow.rs index 2eedf6758..a75340c30 100644 --- a/crux-mir/lib/compiler_builtins/src/float/pow.rs +++ b/crux-mir/lib/compiler_builtins/src/float/pow.rs @@ -1,40 +1,36 @@ use float::Float; use int::Int; -trait Pow: Float { - /// Returns `a` raised to the power `b` - fn pow(self, mut b: i32) -> Self { - let mut a = self; - let recip = b < 0; - let mut r = Self::ONE; - loop { - if (b & 1) != 0 { - r *= a; - } - b = b.aborting_div(2); - if b == 0 { - break; - } - a *= a; +/// Returns `a` raised to the power `b` +fn pow(a: F, b: i32) -> F { + let mut a = a; + let recip = b < 0; + let mut pow = Int::abs_diff(b, 0); + let mut mul = F::ONE; + loop { + if (pow & 1) != 0 { + mul *= a; } - - if recip { - Self::ONE / r - } else { - r + pow >>= 1; + if pow == 0 { + break; } + a *= a; } -} -impl Pow for f32 {} -impl Pow for f64 {} + if recip { + F::ONE / mul + } else { + mul + } +} intrinsics! { pub extern "C" fn __powisf2(a: f32, b: i32) -> f32 { - a.pow(b) + pow(a, b) } pub extern "C" fn __powidf2(a: f64, b: i32) -> f64 { - a.pow(b) + pow(a, b) } } diff --git a/crux-mir/lib/compiler_builtins/src/float/trunc.rs b/crux-mir/lib/compiler_builtins/src/float/trunc.rs new file mode 100644 index 000000000..d73713084 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/src/float/trunc.rs @@ -0,0 +1,125 @@ +use float::Float; +use int::{CastInto, Int}; + +fn trunc(a: F) -> R +where + F::Int: CastInto, + F::Int: CastInto, + u64: CastInto, + u32: CastInto, + + R::Int: CastInto, + u32: CastInto, + F::Int: CastInto, +{ + let src_zero = F::Int::ZERO; + let src_one = F::Int::ONE; + let src_bits = F::BITS; + let src_exp_bias = F::EXPONENT_BIAS; + + let src_min_normal = F::IMPLICIT_BIT; + let src_significand_mask = F::SIGNIFICAND_MASK; + let src_infinity = F::EXPONENT_MASK; + let src_sign_mask = F::SIGN_MASK; + let src_abs_mask = src_sign_mask - src_one; + let round_mask = (src_one << (F::SIGNIFICAND_BITS - R::SIGNIFICAND_BITS)) - src_one; + let halfway = src_one << (F::SIGNIFICAND_BITS - R::SIGNIFICAND_BITS - 1); + let src_qnan = src_one << (F::SIGNIFICAND_BITS - 1); + let src_nan_code = src_qnan - src_one; + + let dst_zero = R::Int::ZERO; + let dst_one = R::Int::ONE; + let dst_bits = R::BITS; + let dst_inf_exp = R::EXPONENT_MAX; + let dst_exp_bias = R::EXPONENT_BIAS; + + let underflow_exponent: F::Int = (src_exp_bias + 1 - dst_exp_bias).cast(); + let overflow_exponent: F::Int = (src_exp_bias + dst_inf_exp - dst_exp_bias).cast(); + let underflow: F::Int = underflow_exponent << F::SIGNIFICAND_BITS; + let overflow: F::Int = overflow_exponent << F::SIGNIFICAND_BITS; + + let dst_qnan = R::Int::ONE << (R::SIGNIFICAND_BITS - 1); + let dst_nan_code = dst_qnan - dst_one; + + let sign_bits_delta = F::SIGNIFICAND_BITS - R::SIGNIFICAND_BITS; + // Break a into a sign and representation of the absolute value. + let a_abs = a.repr() & src_abs_mask; + let sign = a.repr() & src_sign_mask; + let mut abs_result: R::Int; + + if a_abs.wrapping_sub(underflow) < a_abs.wrapping_sub(overflow) { + // The exponent of a is within the range of normal numbers in the + // destination format. We can convert by simply right-shifting with + // rounding and adjusting the exponent. + abs_result = (a_abs >> sign_bits_delta).cast(); + let tmp = src_exp_bias.wrapping_sub(dst_exp_bias) << R::SIGNIFICAND_BITS; + abs_result = abs_result.wrapping_sub(tmp.cast()); + + let round_bits = a_abs & round_mask; + if round_bits > halfway { + // Round to nearest. + abs_result += dst_one; + } else if round_bits == halfway { + // Tie to even. + abs_result += abs_result & dst_one; + }; + } else if a_abs > src_infinity { + // a is NaN. + // Conjure the result by beginning with infinity, setting the qNaN + // bit and inserting the (truncated) trailing NaN field. + abs_result = (dst_inf_exp << R::SIGNIFICAND_BITS).cast(); + abs_result |= dst_qnan; + abs_result |= dst_nan_code + & ((a_abs & src_nan_code) >> (F::SIGNIFICAND_BITS - R::SIGNIFICAND_BITS)).cast(); + } else if a_abs >= overflow { + // a overflows to infinity. + abs_result = (dst_inf_exp << R::SIGNIFICAND_BITS).cast(); + } else { + // a underflows on conversion to the destination type or is an exact + // zero. The result may be a denormal or zero. Extract the exponent + // to get the shift amount for the denormalization. + let a_exp: u32 = (a_abs >> F::SIGNIFICAND_BITS).cast(); + let shift = src_exp_bias - dst_exp_bias - a_exp + 1; + + let significand = (a.repr() & src_significand_mask) | src_min_normal; + + // Right shift by the denormalization amount with sticky. + if shift > F::SIGNIFICAND_BITS { + abs_result = dst_zero; + } else { + let sticky = if (significand << (src_bits - shift)) != src_zero { + src_one + } else { + src_zero + }; + let denormalized_significand: F::Int = significand >> shift | sticky; + abs_result = + (denormalized_significand >> (F::SIGNIFICAND_BITS - R::SIGNIFICAND_BITS)).cast(); + let round_bits = denormalized_significand & round_mask; + // Round to nearest + if round_bits > halfway { + abs_result += dst_one; + } + // Ties to even + else if round_bits == halfway { + abs_result += abs_result & dst_one; + }; + } + } + + // Apply the signbit to the absolute value. + R::from_repr(abs_result | sign.wrapping_shr(src_bits - dst_bits).cast()) +} + +intrinsics! { + #[aapcs_on_arm] + #[arm_aeabi_alias = __aeabi_d2f] + pub extern "C" fn __truncdfsf2(a: f64) -> f32 { + trunc(a) + } + + #[cfg(target_arch = "arm")] + pub extern "C" fn __truncdfsf2vfp(a: f64) -> f32 { + a as f32 + } +} diff --git a/crux-mir/lib/compiler_builtins/src/int/addsub.rs b/crux-mir/lib/compiler_builtins/src/int/addsub.rs index 0a88e2fc8..f4841e90f 100644 --- a/crux-mir/lib/compiler_builtins/src/int/addsub.rs +++ b/crux-mir/lib/compiler_builtins/src/int/addsub.rs @@ -1,25 +1,16 @@ -use int::Int; -use int::LargeInt; +use int::{DInt, Int}; -trait UAddSub: LargeInt { +trait UAddSub: DInt { fn uadd(self, other: Self) -> Self { - let (low, carry) = self.low().overflowing_add(other.low()); - let high = self.high().wrapping_add(other.high()); - let carry = if carry { - Self::HighHalf::ONE - } else { - Self::HighHalf::ZERO - }; - Self::from_parts(low, high.wrapping_add(carry)) + let (lo, carry) = self.lo().overflowing_add(other.lo()); + let hi = self.hi().wrapping_add(other.hi()); + let carry = if carry { Self::H::ONE } else { Self::H::ZERO }; + Self::from_lo_hi(lo, hi.wrapping_add(carry)) } fn uadd_one(self) -> Self { - let (low, carry) = self.low().overflowing_add(Self::LowHalf::ONE); - let carry = if carry { - Self::HighHalf::ONE - } else { - Self::HighHalf::ZERO - }; - Self::from_parts(low, self.high().wrapping_add(carry)) + let (lo, carry) = self.lo().overflowing_add(Self::H::ONE); + let carry = if carry { Self::H::ONE } else { Self::H::ZERO }; + Self::from_lo_hi(lo, self.hi().wrapping_add(carry)) } fn usub(self, other: Self) -> Self { let uneg = (!other).uadd_one(); @@ -48,19 +39,9 @@ trait Addo: AddSub where ::UnsignedInt: UAddSub, { - fn addo(self, other: Self, overflow: &mut i32) -> Self { - *overflow = 0; - let result = AddSub::add(self, other); - if other >= Self::ZERO { - if result < self { - *overflow = 1; - } - } else { - if result >= self { - *overflow = 1; - } - } - result + fn addo(self, other: Self) -> (Self, bool) { + let sum = AddSub::add(self, other); + (sum, (other < Self::ZERO) != (sum < self)) } } @@ -71,19 +52,9 @@ trait Subo: AddSub where ::UnsignedInt: UAddSub, { - fn subo(self, other: Self, overflow: &mut i32) -> Self { - *overflow = 0; - let result = AddSub::sub(self, other); - if other >= Self::ZERO { - if result > self { - *overflow = 1; - } - } else { - if result <= self { - *overflow = 1; - } - } - result + fn subo(self, other: Self) -> (Self, bool) { + let sum = AddSub::sub(self, other); + (sum, (other < Self::ZERO) != (self < sum)) } } @@ -92,43 +63,34 @@ impl Subo for u128 {} intrinsics! { pub extern "C" fn __rust_i128_add(a: i128, b: i128) -> i128 { - __rust_u128_add(a as _, b as _) as _ + AddSub::add(a,b) } pub extern "C" fn __rust_i128_addo(a: i128, b: i128) -> (i128, bool) { - let mut oflow = 0; - let r = a.addo(b, &mut oflow); - (r, oflow != 0) + a.addo(b) } pub extern "C" fn __rust_u128_add(a: u128, b: u128) -> u128 { - a.add(b) + AddSub::add(a,b) } pub extern "C" fn __rust_u128_addo(a: u128, b: u128) -> (u128, bool) { - let mut oflow = 0; - let r = a.addo(b, &mut oflow); - (r, oflow != 0) + a.addo(b) } - pub extern "C" fn __rust_i128_sub(a: i128, b: i128) -> i128 { - __rust_u128_sub(a as _, b as _) as _ + AddSub::sub(a,b) } pub extern "C" fn __rust_i128_subo(a: i128, b: i128) -> (i128, bool) { - let mut oflow = 0; - let r = a.subo(b, &mut oflow); - (r, oflow != 0) + a.subo(b) } pub extern "C" fn __rust_u128_sub(a: u128, b: u128) -> u128 { - a.sub(b) + AddSub::sub(a,b) } pub extern "C" fn __rust_u128_subo(a: u128, b: u128) -> (u128, bool) { - let mut oflow = 0; - let r = a.subo(b, &mut oflow); - (r, oflow != 0) + a.subo(b) } } diff --git a/crux-mir/lib/compiler_builtins/src/int/leading_zeros.rs b/crux-mir/lib/compiler_builtins/src/int/leading_zeros.rs new file mode 100644 index 000000000..9e60ab0d7 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/src/int/leading_zeros.rs @@ -0,0 +1,149 @@ +// Note: these functions happen to produce the correct `usize::leading_zeros(0)` value +// without a explicit zero check. Zero is probably common enough that it could warrant +// adding a zero check at the beginning, but `__clzsi2` has a precondition that `x != 0`. +// Compilers will insert the check for zero in cases where it is needed. + +public_test_dep! { +/// Returns the number of leading binary zeros in `x`. +#[allow(dead_code)] +pub(crate) fn usize_leading_zeros_default(x: usize) -> usize { + // The basic idea is to test if the higher bits of `x` are zero and bisect the number + // of leading zeros. It is possible for all branches of the bisection to use the same + // code path by conditionally shifting the higher parts down to let the next bisection + // step work on the higher or lower parts of `x`. Instead of starting with `z == 0` + // and adding to the number of zeros, it is slightly faster to start with + // `z == usize::MAX.count_ones()` and subtract from the potential number of zeros, + // because it simplifies the final bisection step. + let mut x = x; + // the number of potential leading zeros + let mut z = usize::MAX.count_ones() as usize; + // a temporary + let mut t: usize; + #[cfg(target_pointer_width = "64")] + { + t = x >> 32; + if t != 0 { + z -= 32; + x = t; + } + } + #[cfg(any(target_pointer_width = "32", target_pointer_width = "64"))] + { + t = x >> 16; + if t != 0 { + z -= 16; + x = t; + } + } + t = x >> 8; + if t != 0 { + z -= 8; + x = t; + } + t = x >> 4; + if t != 0 { + z -= 4; + x = t; + } + t = x >> 2; + if t != 0 { + z -= 2; + x = t; + } + // the last two bisections are combined into one conditional + t = x >> 1; + if t != 0 { + z - 2 + } else { + z - x + } + + // We could potentially save a few cycles by using the LUT trick from + // "https://embeddedgurus.com/state-space/2014/09/ + // fast-deterministic-and-portable-counting-leading-zeros/". + // However, 256 bytes for a LUT is too large for embedded use cases. We could remove + // the last 3 bisections and use this 16 byte LUT for the rest of the work: + //const LUT: [u8; 16] = [0, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4]; + //z -= LUT[x] as usize; + //z + // However, it ends up generating about the same number of instructions. When benchmarked + // on x86_64, it is slightly faster to use the LUT, but this is probably because of OOO + // execution effects. Changing to using a LUT and branching is risky for smaller cores. +} +} + +// The above method does not compile well on RISC-V (because of the lack of predicated +// instructions), producing code with many branches or using an excessively long +// branchless solution. This method takes advantage of the set-if-less-than instruction on +// RISC-V that allows `(x >= power-of-two) as usize` to be branchless. + +public_test_dep! { +/// Returns the number of leading binary zeros in `x`. +#[allow(dead_code)] +pub(crate) fn usize_leading_zeros_riscv(x: usize) -> usize { + let mut x = x; + // the number of potential leading zeros + let mut z = usize::MAX.count_ones() as usize; + // a temporary + let mut t: usize; + + // RISC-V does not have a set-if-greater-than-or-equal instruction and + // `(x >= power-of-two) as usize` will get compiled into two instructions, but this is + // still the most optimal method. A conditional set can only be turned into a single + // immediate instruction if `x` is compared with an immediate `imm` (that can fit into + // 12 bits) like `x < imm` but not `imm < x` (because the immediate is always on the + // right). If we try to save an instruction by using `x < imm` for each bisection, we + // have to shift `x` left and compare with powers of two approaching `usize::MAX + 1`, + // but the immediate will never fit into 12 bits and never save an instruction. + #[cfg(target_pointer_width = "64")] + { + // If the upper 32 bits of `x` are not all 0, `t` is set to `1 << 5`, otherwise + // `t` is set to 0. + t = ((x >= (1 << 32)) as usize) << 5; + // If `t` was set to `1 << 5`, then the upper 32 bits are shifted down for the + // next step to process. + x >>= t; + // If `t` was set to `1 << 5`, then we subtract 32 from the number of potential + // leading zeros + z -= t; + } + #[cfg(any(target_pointer_width = "32", target_pointer_width = "64"))] + { + t = ((x >= (1 << 16)) as usize) << 4; + x >>= t; + z -= t; + } + t = ((x >= (1 << 8)) as usize) << 3; + x >>= t; + z -= t; + t = ((x >= (1 << 4)) as usize) << 2; + x >>= t; + z -= t; + t = ((x >= (1 << 2)) as usize) << 1; + x >>= t; + z -= t; + t = (x >= (1 << 1)) as usize; + x >>= t; + z -= t; + // All bits except the LSB are guaranteed to be zero for this final bisection step. + // If `x != 0` then `x == 1` and subtracts one potential zero from `z`. + z - x +} +} + +intrinsics! { + #[maybe_use_optimized_c_shim] + #[cfg(any( + target_pointer_width = "16", + target_pointer_width = "32", + target_pointer_width = "64" + ))] + /// Returns the number of leading binary zeros in `x`. + pub extern "C" fn __clzsi2(x: usize) -> usize { + if cfg!(any(target_arch = "riscv32", target_arch = "riscv64")) { + usize_leading_zeros_riscv(x) + } else { + usize_leading_zeros_default(x) + } + } +} diff --git a/crux-mir/lib/compiler_builtins/src/int/mod.rs b/crux-mir/lib/compiler_builtins/src/int/mod.rs index 7587bc69e..509f9fdae 100644 --- a/crux-mir/lib/compiler_builtins/src/int/mod.rs +++ b/crux-mir/lib/compiler_builtins/src/int/mod.rs @@ -1,31 +1,28 @@ use core::ops; -macro_rules! hty { - ($ty:ty) => { - <$ty as LargeInt>::HighHalf - }; -} - -macro_rules! os_ty { - ($ty:ty) => { - <$ty as Int>::OtherSign - }; -} +mod specialized_div_rem; pub mod addsub; +pub mod leading_zeros; pub mod mul; pub mod sdiv; pub mod shift; pub mod udiv; +pub use self::leading_zeros::__clzsi2; + +public_test_dep! { /// Trait for some basic operations on integers -pub trait Int: +pub(crate) trait Int: Copy + + core::fmt::Debug + PartialEq + PartialOrd + ops::AddAssign + + ops::SubAssign + ops::BitAndAssign + ops::BitOrAssign + + ops::BitXorAssign + ops::ShlAssign + ops::ShrAssign + ops::Add @@ -43,135 +40,198 @@ pub trait Int: /// Unsigned version of Self type UnsignedInt: Int; + /// If `Self` is a signed integer + const SIGNED: bool; + /// The bitwidth of the int type const BITS: u32; const ZERO: Self; const ONE: Self; + const MIN: Self; + const MAX: Self; - /// Extracts the sign from self and returns a tuple. - /// - /// # Examples - /// - /// ```rust,ignore - /// let i = -25_i32; - /// let (sign, u) = i.extract_sign(); - /// assert_eq!(sign, true); - /// assert_eq!(u, 25_u32); - /// ``` - fn extract_sign(self) -> (bool, Self::UnsignedInt); + /// LUT used for maximizing the space covered and minimizing the computational cost of fuzzing + /// in `testcrate`. For example, Self = u128 produces [0,1,2,7,8,15,16,31,32,63,64,95,96,111, + /// 112,119,120,125,126,127]. + const FUZZ_LENGTHS: [u8; 20]; + /// The number of entries of `FUZZ_LENGTHS` actually used. The maximum is 20 for u128. + const FUZZ_NUM: usize; fn unsigned(self) -> Self::UnsignedInt; fn from_unsigned(unsigned: Self::UnsignedInt) -> Self; fn from_bool(b: bool) -> Self; + /// Prevents the need for excessive conversions between signed and unsigned + fn logical_shr(self, other: u32) -> Self; + + /// Absolute difference between two integers. + fn abs_diff(self, other: Self) -> Self::UnsignedInt; + // copied from primitive integers, but put in a trait - fn max_value() -> Self; - fn min_value() -> Self; + fn is_zero(self) -> bool; + fn wrapping_neg(self) -> Self; fn wrapping_add(self, other: Self) -> Self; fn wrapping_mul(self, other: Self) -> Self; fn wrapping_sub(self, other: Self) -> Self; fn wrapping_shl(self, other: u32) -> Self; + fn wrapping_shr(self, other: u32) -> Self; + fn rotate_left(self, other: u32) -> Self; fn overflowing_add(self, other: Self) -> (Self, bool); - fn aborting_div(self, other: Self) -> Self; - fn aborting_rem(self, other: Self) -> Self; fn leading_zeros(self) -> u32; } - -fn unwrap(t: Option) -> T { - match t { - Some(t) => t, - None => ::abort(), - } } macro_rules! int_impl_common { - ($ty:ty, $bits:expr) => { - const BITS: u32 = $bits; - - const ZERO: Self = 0; - const ONE: Self = 1; - - fn from_bool(b: bool) -> Self { - b as $ty + ($ty:ty) => { + const BITS: u32 = ::ZERO.count_zeros(); + const SIGNED: bool = Self::MIN != Self::ZERO; + + const ZERO: Self = 0; + const ONE: Self = 1; + const MIN: Self = ::MIN; + const MAX: Self = ::MAX; + + const FUZZ_LENGTHS: [u8; 20] = { + let bits = ::BITS; + let mut v = [0u8; 20]; + v[0] = 0; + v[1] = 1; + v[2] = 2; // important for parity and the iX::MIN case when reversed + let mut i = 3; + // No need for any more until the byte boundary, because there should be no algorithms + // that are sensitive to anything not next to byte boundaries after 2. We also scale + // in powers of two, which is important to prevent u128 corner tests from getting too + // big. + let mut l = 8; + loop { + if l >= ((bits / 2) as u8) { + break; + } + // get both sides of the byte boundary + v[i] = l - 1; + i += 1; + v[i] = l; + i += 1; + l *= 2; } - fn max_value() -> Self { - ::max_value() + if bits != 8 { + // add the lower side of the middle boundary + v[i] = ((bits / 2) - 1) as u8; + i += 1; } - fn min_value() -> Self { - ::min_value() + // We do not want to jump directly from the Self::BITS/2 boundary to the Self::BITS + // boundary because of algorithms that split the high part up. We reverse the scaling + // as we go to Self::BITS. + let mid = i; + let mut j = 1; + loop { + v[i] = (bits as u8) - (v[mid - j]) - 1; + if j == mid { + break; + } + i += 1; + j += 1; } + v + }; - fn wrapping_add(self, other: Self) -> Self { - ::wrapping_add(self, other) + const FUZZ_NUM: usize = { + let log2 = (::BITS - 1).count_ones() as usize; + if log2 == 3 { + // case for u8 + 6 + } else { + // 3 entries on each extreme, 2 in the middle, and 4 for each scale of intermediate + // boundaries. + 8 + (4 * (log2 - 4)) } + }; - fn wrapping_mul(self, other: Self) -> Self { - ::wrapping_mul(self, other) - } + fn from_bool(b: bool) -> Self { + b as $ty + } - fn wrapping_sub(self, other: Self) -> Self { - ::wrapping_sub(self, other) - } + fn logical_shr(self, other: u32) -> Self { + Self::from_unsigned(self.unsigned().wrapping_shr(other)) + } - fn wrapping_shl(self, other: u32) -> Self { - ::wrapping_shl(self, other) - } + fn is_zero(self) -> bool { + self == Self::ZERO + } - fn overflowing_add(self, other: Self) -> (Self, bool) { - ::overflowing_add(self, other) - } + fn wrapping_neg(self) -> Self { + ::wrapping_neg(self) + } - fn aborting_div(self, other: Self) -> Self { - unwrap(::checked_div(self, other)) - } + fn wrapping_add(self, other: Self) -> Self { + ::wrapping_add(self, other) + } - fn aborting_rem(self, other: Self) -> Self { - unwrap(::checked_rem(self, other)) - } + fn wrapping_mul(self, other: Self) -> Self { + ::wrapping_mul(self, other) + } - fn leading_zeros(self) -> u32 { - ::leading_zeros(self) - } - } + fn wrapping_sub(self, other: Self) -> Self { + ::wrapping_sub(self, other) + } + + fn wrapping_shl(self, other: u32) -> Self { + ::wrapping_shl(self, other) + } + + fn wrapping_shr(self, other: u32) -> Self { + ::wrapping_shr(self, other) + } + + fn rotate_left(self, other: u32) -> Self { + ::rotate_left(self, other) + } + + fn overflowing_add(self, other: Self) -> (Self, bool) { + ::overflowing_add(self, other) + } + + fn leading_zeros(self) -> u32 { + ::leading_zeros(self) + } + }; } macro_rules! int_impl { - ($ity:ty, $uty:ty, $bits:expr) => { + ($ity:ty, $uty:ty) => { impl Int for $uty { type OtherSign = $ity; type UnsignedInt = $uty; - fn extract_sign(self) -> (bool, $uty) { - (false, self) - } - fn unsigned(self) -> $uty { self } + // It makes writing macros easier if this is implemented for both signed and unsigned + #[allow(clippy::wrong_self_convention)] fn from_unsigned(me: $uty) -> Self { me } - int_impl_common!($uty, $bits); + fn abs_diff(self, other: Self) -> Self { + if self < other { + other.wrapping_sub(self) + } else { + self.wrapping_sub(other) + } + } + + int_impl_common!($uty); } impl Int for $ity { type OtherSign = $uty; type UnsignedInt = $uty; - fn extract_sign(self) -> (bool, $uty) { - if self < 0 { - (true, (!(self as $uty)).wrapping_add(1)) - } else { - (false, self as $uty) - } - } - fn unsigned(self) -> $uty { self as $uty } @@ -180,65 +240,132 @@ macro_rules! int_impl { me as $ity } - int_impl_common!($ity, $bits); + fn abs_diff(self, other: Self) -> $uty { + self.wrapping_sub(other).wrapping_abs() as $uty + } + + int_impl_common!($ity); } }; } -int_impl!(i32, u32, 32); -int_impl!(i64, u64, 64); -int_impl!(i128, u128, 128); - -/// Trait to convert an integer to/from smaller parts -pub trait LargeInt: Int { - type LowHalf: Int; - type HighHalf: Int; +int_impl!(isize, usize); +int_impl!(i8, u8); +int_impl!(i16, u16); +int_impl!(i32, u32); +int_impl!(i64, u64); +int_impl!(i128, u128); + +public_test_dep! { +/// Trait for integers twice the bit width of another integer. This is implemented for all +/// primitives except for `u8`, because there is not a smaller primitive. +pub(crate) trait DInt: Int { + /// Integer that is half the bit width of the integer this trait is implemented for + type H: HInt + Int; + + /// Returns the low half of `self` + fn lo(self) -> Self::H; + /// Returns the high half of `self` + fn hi(self) -> Self::H; + /// Returns the low and high halves of `self` as a tuple + fn lo_hi(self) -> (Self::H, Self::H); + /// Constructs an integer using lower and higher half parts + fn from_lo_hi(lo: Self::H, hi: Self::H) -> Self; +} +} - fn low(self) -> Self::LowHalf; - fn low_as_high(low: Self::LowHalf) -> Self::HighHalf; - fn high(self) -> Self::HighHalf; - fn high_as_low(low: Self::HighHalf) -> Self::LowHalf; - fn from_parts(low: Self::LowHalf, high: Self::HighHalf) -> Self; +public_test_dep! { +/// Trait for integers half the bit width of another integer. This is implemented for all +/// primitives except for `u128`, because it there is not a larger primitive. +pub(crate) trait HInt: Int { + /// Integer that is double the bit width of the integer this trait is implemented for + type D: DInt + Int; + + /// Widens (using default extension) the integer to have double bit width + fn widen(self) -> Self::D; + /// Widens (zero extension only) the integer to have double bit width. This is needed to get + /// around problems with associated type bounds (such as `Int`) being unstable + fn zero_widen(self) -> Self::D; + /// Widens the integer to have double bit width and shifts the integer into the higher bits + fn widen_hi(self) -> Self::D; + /// Widening multiplication with zero widening. This cannot overflow. + fn zero_widen_mul(self, rhs: Self) -> Self::D; + /// Widening multiplication. This cannot overflow. + fn widen_mul(self, rhs: Self) -> Self::D; +} } -macro_rules! large_int { - ($ty:ty, $tylow:ty, $tyhigh:ty, $halfbits:expr) => { - impl LargeInt for $ty { - type LowHalf = $tylow; - type HighHalf = $tyhigh; +macro_rules! impl_d_int { + ($($X:ident $D:ident),*) => { + $( + impl DInt for $D { + type H = $X; - fn low(self) -> $tylow { - self as $tylow - } - fn low_as_high(low: $tylow) -> $tyhigh { - low as $tyhigh - } - fn high(self) -> $tyhigh { - (self >> $halfbits) as $tyhigh - } - fn high_as_low(high: $tyhigh) -> $tylow { - high as $tylow - } - fn from_parts(low: $tylow, high: $tyhigh) -> $ty { - low as $ty | ((high as $ty) << $halfbits) + fn lo(self) -> Self::H { + self as $X + } + fn hi(self) -> Self::H { + (self >> <$X as Int>::BITS) as $X + } + fn lo_hi(self) -> (Self::H, Self::H) { + (self.lo(), self.hi()) + } + fn from_lo_hi(lo: Self::H, hi: Self::H) -> Self { + lo.zero_widen() | hi.widen_hi() + } } - } + )* }; } -large_int!(u64, u32, u32, 32); -large_int!(i64, u32, i32, 32); -large_int!(u128, u64, u64, 64); -large_int!(i128, u64, i64, 64); +macro_rules! impl_h_int { + ($($H:ident $uH:ident $X:ident),*) => { + $( + impl HInt for $H { + type D = $X; + fn widen(self) -> Self::D { + self as $X + } + fn zero_widen(self) -> Self::D { + (self as $uH) as $X + } + fn widen_hi(self) -> Self::D { + (self as $X) << <$H as Int>::BITS + } + fn zero_widen_mul(self, rhs: Self) -> Self::D { + self.zero_widen().wrapping_mul(rhs.zero_widen()) + } + fn widen_mul(self, rhs: Self) -> Self::D { + self.widen().wrapping_mul(rhs.widen()) + } + } + )* + }; +} + +impl_d_int!(u8 u16, u16 u32, u32 u64, u64 u128, i8 i16, i16 i32, i32 i64, i64 i128); +impl_h_int!( + u8 u8 u16, + u16 u16 u32, + u32 u32 u64, + u64 u64 u128, + i8 u8 i16, + i16 u16 i32, + i32 u32 i64, + i64 u64 i128 +); + +public_test_dep! { /// Trait to express (possibly lossy) casting of integers -pub trait CastInto: Copy { +pub(crate) trait CastInto: Copy { fn cast(self) -> T; } +} macro_rules! cast_into { ($ty:ty) => { - cast_into!($ty; usize, isize, u32, i32, u64, i64, u128, i128); + cast_into!($ty; usize, isize, u8, i8, u16, i16, u32, i32, u64, i64, u128, i128); }; ($ty:ty; $($into:ty),*) => {$( impl CastInto<$into> for $ty { @@ -249,120 +376,15 @@ macro_rules! cast_into { )*}; } +cast_into!(usize); +cast_into!(isize); +cast_into!(u8); +cast_into!(i8); +cast_into!(u16); +cast_into!(i16); cast_into!(u32); cast_into!(i32); cast_into!(u64); cast_into!(i64); cast_into!(u128); cast_into!(i128); - -pub trait WideInt: Int { - type Output: Int; - - fn wide_mul(self, other: Self) -> (Self, Self); - fn wide_shift_left(&mut self, low: &mut Self, count: i32); - fn wide_shift_right_with_sticky(&mut self, low: &mut Self, count: i32); -} - -macro_rules! impl_wide_int { - ($ty:ty, $tywide:ty, $bits:expr) => { - impl WideInt for $ty { - type Output = $ty; - - fn wide_mul(self, other: Self) -> (Self, Self) { - let product = (self as $tywide).wrapping_mul(other as $tywide); - ((product >> ($bits as $ty)) as $ty, product as $ty) - } - - fn wide_shift_left(&mut self, low: &mut Self, count: i32) { - *self = (*self << count) | (*low >> ($bits - count)); - *low = *low << count; - } - - fn wide_shift_right_with_sticky(&mut self, low: &mut Self, count: i32) { - if count < $bits { - let sticky = *low << ($bits - count); - *low = *self << ($bits - count) | *low >> count | sticky; - *self = *self >> count; - } else if count < 2 * $bits { - let sticky = *self << (2 * $bits - count) | *low; - *low = *self >> (count - $bits) | sticky; - *self = 0; - } else { - let sticky = *self | *low; - *self = sticky; - *self = 0; - } - } - } - }; -} - -impl_wide_int!(u32, u64, 32); -impl_wide_int!(u64, u128, 64); - -intrinsics! { - #[maybe_use_optimized_c_shim] - #[cfg(any( - target_pointer_width = "16", - target_pointer_width = "32", - target_pointer_width = "64" - ))] - pub extern "C" fn __clzsi2(x: usize) -> usize { - // TODO: const this? Would require const-if - // Note(Lokathor): the `intrinsics!` macro can't process mut inputs - let mut x = x; - let mut y: usize; - let mut n: usize = { - #[cfg(target_pointer_width = "64")] - { - 64 - } - #[cfg(target_pointer_width = "32")] - { - 32 - } - #[cfg(target_pointer_width = "16")] - { - 16 - } - }; - #[cfg(target_pointer_width = "64")] - { - y = x >> 32; - if y != 0 { - n -= 32; - x = y; - } - } - #[cfg(any(target_pointer_width = "32", target_pointer_width = "64"))] - { - y = x >> 16; - if y != 0 { - n -= 16; - x = y; - } - } - y = x >> 8; - if y != 0 { - n -= 8; - x = y; - } - y = x >> 4; - if y != 0 { - n -= 4; - x = y; - } - y = x >> 2; - if y != 0 { - n -= 2; - x = y; - } - y = x >> 1; - if y != 0 { - n - 2 - } else { - n - x - } - } -} diff --git a/crux-mir/lib/compiler_builtins/src/int/mul.rs b/crux-mir/lib/compiler_builtins/src/int/mul.rs index 42f13913e..07ce061c9 100644 --- a/crux-mir/lib/compiler_builtins/src/int/mul.rs +++ b/crux-mir/lib/compiler_builtins/src/int/mul.rs @@ -1,122 +1,138 @@ -use core::ops; - -use int::Int; -use int::LargeInt; - -trait Mul: LargeInt { - fn mul(self, other: Self) -> Self { - let half_bits = Self::BITS / 4; - let lower_mask = !<::LowHalf>::ZERO >> half_bits; - let mut low = (self.low() & lower_mask).wrapping_mul(other.low() & lower_mask); - let mut t = low >> half_bits; - low &= lower_mask; - t += (self.low() >> half_bits).wrapping_mul(other.low() & lower_mask); - low += (t & lower_mask) << half_bits; - let mut high = Self::low_as_high(t >> half_bits); - t = low >> half_bits; - low &= lower_mask; - t += (other.low() >> half_bits).wrapping_mul(self.low() & lower_mask); - low += (t & lower_mask) << half_bits; - high += Self::low_as_high(t >> half_bits); - high += Self::low_as_high((self.low() >> half_bits).wrapping_mul(other.low() >> half_bits)); - high = high - .wrapping_add(self.high().wrapping_mul(Self::low_as_high(other.low()))) - .wrapping_add(Self::low_as_high(self.low()).wrapping_mul(other.high())); - Self::from_parts(low, high) +use int::{DInt, HInt, Int}; + +trait Mul: DInt +where + Self::H: DInt, +{ + fn mul(self, rhs: Self) -> Self { + // In order to prevent infinite recursion, we cannot use the `widen_mul` in this: + //self.lo().widen_mul(rhs.lo()) + // .wrapping_add(self.lo().wrapping_mul(rhs.hi()).widen_hi()) + // .wrapping_add(self.hi().wrapping_mul(rhs.lo()).widen_hi()) + + let lhs_lo = self.lo(); + let rhs_lo = rhs.lo(); + // construct the widening multiplication using only `Self::H` sized multiplications + let tmp_0 = lhs_lo.lo().zero_widen_mul(rhs_lo.lo()); + let tmp_1 = lhs_lo.lo().zero_widen_mul(rhs_lo.hi()); + let tmp_2 = lhs_lo.hi().zero_widen_mul(rhs_lo.lo()); + let tmp_3 = lhs_lo.hi().zero_widen_mul(rhs_lo.hi()); + // sum up all widening partials + let mul = Self::from_lo_hi(tmp_0, tmp_3) + .wrapping_add(tmp_1.zero_widen() << (Self::BITS / 4)) + .wrapping_add(tmp_2.zero_widen() << (Self::BITS / 4)); + // add the higher partials + mul.wrapping_add(lhs_lo.wrapping_mul(rhs.hi()).widen_hi()) + .wrapping_add(self.hi().wrapping_mul(rhs_lo).widen_hi()) } } impl Mul for u64 {} impl Mul for i128 {} -trait Mulo: Int + ops::Neg { - fn mulo(self, other: Self, overflow: &mut i32) -> Self { - *overflow = 0; - let result = self.wrapping_mul(other); - if self == Self::min_value() { - if other != Self::ZERO && other != Self::ONE { - *overflow = 1; +pub(crate) trait UMulo: Int + DInt { + fn mulo(self, rhs: Self) -> (Self, bool) { + match (self.hi().is_zero(), rhs.hi().is_zero()) { + // overflow is guaranteed + (false, false) => (self.wrapping_mul(rhs), true), + (true, false) => { + let mul_lo = self.lo().widen_mul(rhs.lo()); + let mul_hi = self.lo().widen_mul(rhs.hi()); + let (mul, o) = mul_lo.overflowing_add(mul_hi.lo().widen_hi()); + (mul, o || !mul_hi.hi().is_zero()) } - return result; - } - if other == Self::min_value() { - if self != Self::ZERO && self != Self::ONE { - *overflow = 1; + (false, true) => { + let mul_lo = rhs.lo().widen_mul(self.lo()); + let mul_hi = rhs.lo().widen_mul(self.hi()); + let (mul, o) = mul_lo.overflowing_add(mul_hi.lo().widen_hi()); + (mul, o || !mul_hi.hi().is_zero()) } - return result; + // overflow is guaranteed to not happen, and use a smaller widening multiplication + (true, true) => (self.lo().widen_mul(rhs.lo()), false), } + } +} - let sa = self >> (Self::BITS - 1); - let abs_a = (self ^ sa) - sa; - let sb = other >> (Self::BITS - 1); - let abs_b = (other ^ sb) - sb; - let two = Self::ONE + Self::ONE; - if abs_a < two || abs_b < two { - return result; - } - if sa == sb { - if abs_a > Self::max_value().aborting_div(abs_b) { - *overflow = 1; +impl UMulo for u32 {} +impl UMulo for u64 {} +impl UMulo for u128 {} + +macro_rules! impl_signed_mulo { + ($fn:ident, $iD:ident, $uD:ident) => { + fn $fn(lhs: $iD, rhs: $iD) -> ($iD, bool) { + let mut lhs = lhs; + let mut rhs = rhs; + // the test against `mul_neg` below fails without this early return + if lhs == 0 || rhs == 0 { + return (0, false); + } + + let lhs_neg = lhs < 0; + let rhs_neg = rhs < 0; + if lhs_neg { + lhs = lhs.wrapping_neg(); } - } else { - if abs_a > Self::min_value().aborting_div(-abs_b) { - *overflow = 1; + if rhs_neg { + rhs = rhs.wrapping_neg(); } - } - result - } -} + let mul_neg = lhs_neg != rhs_neg; -impl Mulo for i32 {} -impl Mulo for i64 {} -impl Mulo for i128 {} + let (mul, o) = (lhs as $uD).mulo(rhs as $uD); + let mut mul = mul as $iD; -trait UMulo: Int { - fn mulo(self, other: Self, overflow: &mut i32) -> Self { - *overflow = 0; - let result = self.wrapping_mul(other); - if self > Self::max_value().aborting_div(other) { - *overflow = 1; + if mul_neg { + mul = mul.wrapping_neg(); + } + if (mul < 0) != mul_neg { + // this one check happens to catch all edge cases related to `$iD::MIN` + (mul, true) + } else { + (mul, o) + } } - result - } + }; } -impl UMulo for u128 {} + +impl_signed_mulo!(i32_overflowing_mul, i32, u32); +impl_signed_mulo!(i64_overflowing_mul, i64, u64); +impl_signed_mulo!(i128_overflowing_mul, i128, u128); intrinsics! { #[maybe_use_optimized_c_shim] #[arm_aeabi_alias = __aeabi_lmul] + #[cfg(any(not(any(target_arch = "riscv32", target_arch = "riscv64")), target_feature = "m"))] pub extern "C" fn __muldi3(a: u64, b: u64) -> u64 { a.mul(b) } - #[aapcs_on_arm] pub extern "C" fn __multi3(a: i128, b: i128) -> i128 { a.mul(b) } pub extern "C" fn __mulosi4(a: i32, b: i32, oflow: &mut i32) -> i32 { - a.mulo(b, oflow) + let (mul, o) = i32_overflowing_mul(a, b); + *oflow = o as i32; + mul } pub extern "C" fn __mulodi4(a: i64, b: i64, oflow: &mut i32) -> i64 { - a.mulo(b, oflow) + let (mul, o) = i64_overflowing_mul(a, b); + *oflow = o as i32; + mul } #[unadjusted_on_win64] pub extern "C" fn __muloti4(a: i128, b: i128, oflow: &mut i32) -> i128 { - a.mulo(b, oflow) + let (mul, o) = i128_overflowing_mul(a, b); + *oflow = o as i32; + mul } pub extern "C" fn __rust_i128_mulo(a: i128, b: i128) -> (i128, bool) { - let mut oflow = 0; - let r = __muloti4(a, b, &mut oflow); - (r, oflow != 0) + i128_overflowing_mul(a, b) } pub extern "C" fn __rust_u128_mulo(a: u128, b: u128) -> (u128, bool) { - let mut oflow = 0; - let r = a.mulo(b, &mut oflow); - (r, oflow != 0) + a.mulo(b) } } diff --git a/crux-mir/lib/compiler_builtins/src/int/sdiv.rs b/crux-mir/lib/compiler_builtins/src/int/sdiv.rs index c9e252cc3..f1822f0f8 100644 --- a/crux-mir/lib/compiler_builtins/src/int/sdiv.rs +++ b/crux-mir/lib/compiler_builtins/src/int/sdiv.rs @@ -1,101 +1,169 @@ -use int::Int; - -trait Div: Int { - /// Returns `a / b` - fn div(self, other: Self) -> Self { - let s_a = self >> (Self::BITS - 1); - let s_b = other >> (Self::BITS - 1); - // NOTE it's OK to overflow here because of the `.unsigned()` below. - // This whole operation is computing the absolute value of the inputs - // So some overflow will happen when dealing with e.g. `i64::MIN` - // where the absolute value is `(-i64::MIN) as u64` - let a = (self ^ s_a).wrapping_sub(s_a); - let b = (other ^ s_b).wrapping_sub(s_b); - let s = s_a ^ s_b; - - let r = a.unsigned().aborting_div(b.unsigned()); - (Self::from_unsigned(r) ^ s) - s +use int::udiv::*; + +macro_rules! sdivmod { + ( + $unsigned_fn:ident, // name of the unsigned division function + $signed_fn:ident, // name of the signed division function + $uX:ident, // unsigned integer type for the inputs and outputs of `$unsigned_name` + $iX:ident, // signed integer type for the inputs and outputs of `$signed_name` + $($attr:tt),* // attributes + ) => { + intrinsics! { + #[avr_skip] + $( + #[$attr] + )* + /// Returns `n / d` and sets `*rem = n % d` + pub extern "C" fn $signed_fn(a: $iX, b: $iX, rem: &mut $iX) -> $iX { + let a_neg = a < 0; + let b_neg = b < 0; + let mut a = a; + let mut b = b; + if a_neg { + a = a.wrapping_neg(); + } + if b_neg { + b = b.wrapping_neg(); + } + let mut r = *rem as $uX; + let t = $unsigned_fn(a as $uX, b as $uX, Some(&mut r)) as $iX; + let mut r = r as $iX; + if a_neg { + r = r.wrapping_neg(); + } + *rem = r; + if a_neg != b_neg { + t.wrapping_neg() + } else { + t + } + } + } } } -impl Div for i32 {} -impl Div for i64 {} -impl Div for i128 {} - -trait Mod: Int { - /// Returns `a % b` - fn mod_(self, other: Self) -> Self { - let s = other >> (Self::BITS - 1); - // NOTE(wrapping_sub) see comment in the `div` - let b = (other ^ s).wrapping_sub(s); - let s = self >> (Self::BITS - 1); - let a = (self ^ s).wrapping_sub(s); - - let r = a.unsigned().aborting_rem(b.unsigned()); - (Self::from_unsigned(r) ^ s) - s +macro_rules! sdiv { + ( + $unsigned_fn:ident, // name of the unsigned division function + $signed_fn:ident, // name of the signed division function + $uX:ident, // unsigned integer type for the inputs and outputs of `$unsigned_name` + $iX:ident, // signed integer type for the inputs and outputs of `$signed_name` + $($attr:tt),* // attributes + ) => { + intrinsics! { + #[avr_skip] + $( + #[$attr] + )* + /// Returns `n / d` + pub extern "C" fn $signed_fn(a: $iX, b: $iX) -> $iX { + let a_neg = a < 0; + let b_neg = b < 0; + let mut a = a; + let mut b = b; + if a_neg { + a = a.wrapping_neg(); + } + if b_neg { + b = b.wrapping_neg(); + } + let t = $unsigned_fn(a as $uX, b as $uX) as $iX; + if a_neg != b_neg { + t.wrapping_neg() + } else { + t + } + } + } } } -impl Mod for i32 {} -impl Mod for i64 {} -impl Mod for i128 {} - -trait Divmod: Int { - /// Returns `a / b` and sets `*rem = n % d` - fn divmod(self, other: Self, rem: &mut Self, div: F) -> Self - where - F: Fn(Self, Self) -> Self, - { - let r = div(self, other); - // NOTE won't overflow because it's using the result from the - // previous division - *rem = self - r.wrapping_mul(other); - r +macro_rules! smod { + ( + $unsigned_fn:ident, // name of the unsigned division function + $signed_fn:ident, // name of the signed division function + $uX:ident, // unsigned integer type for the inputs and outputs of `$unsigned_name` + $iX:ident, // signed integer type for the inputs and outputs of `$signed_name` + $($attr:tt),* // attributes + ) => { + intrinsics! { + #[avr_skip] + $( + #[$attr] + )* + /// Returns `n % d` + pub extern "C" fn $signed_fn(a: $iX, b: $iX) -> $iX { + let a_neg = a < 0; + let b_neg = b < 0; + let mut a = a; + let mut b = b; + if a_neg { + a = a.wrapping_neg(); + } + if b_neg { + b = b.wrapping_neg(); + } + let r = $unsigned_fn(a as $uX, b as $uX) as $iX; + if a_neg { + r.wrapping_neg() + } else { + r + } + } + } } } -impl Divmod for i32 {} -impl Divmod for i64 {} - +sdivmod!( + __udivmodsi4, + __divmodsi4, + u32, + i32, + maybe_use_optimized_c_shim +); +// The `#[arm_aeabi_alias = __aeabi_idiv]` attribute cannot be made to work with `intrinsics!` in macros intrinsics! { #[maybe_use_optimized_c_shim] #[arm_aeabi_alias = __aeabi_idiv] + /// Returns `n / d` pub extern "C" fn __divsi3(a: i32, b: i32) -> i32 { - a.div(b) - } - - #[maybe_use_optimized_c_shim] - pub extern "C" fn __divdi3(a: i64, b: i64) -> i64 { - a.div(b) - } - - #[win64_128bit_abi_hack] - pub extern "C" fn __divti3(a: i128, b: i128) -> i128 { - a.div(b) - } - - #[maybe_use_optimized_c_shim] - pub extern "C" fn __modsi3(a: i32, b: i32) -> i32 { - a.mod_(b) - } - - #[maybe_use_optimized_c_shim] - pub extern "C" fn __moddi3(a: i64, b: i64) -> i64 { - a.mod_(b) - } - - #[win64_128bit_abi_hack] - pub extern "C" fn __modti3(a: i128, b: i128) -> i128 { - a.mod_(b) - } - - #[maybe_use_optimized_c_shim] - pub extern "C" fn __divmodsi4(a: i32, b: i32, rem: &mut i32) -> i32 { - a.divmod(b, rem, |a, b| __divsi3(a, b)) - } - - #[aapcs_on_arm] - pub extern "C" fn __divmoddi4(a: i64, b: i64, rem: &mut i64) -> i64 { - a.divmod(b, rem, |a, b| __divdi3(a, b)) + let a_neg = a < 0; + let b_neg = b < 0; + let mut a = a; + let mut b = b; + if a_neg { + a = a.wrapping_neg(); + } + if b_neg { + b = b.wrapping_neg(); + } + let t = __udivsi3(a as u32, b as u32) as i32; + if a_neg != b_neg { + t.wrapping_neg() + } else { + t + } } } +smod!(__umodsi3, __modsi3, u32, i32, maybe_use_optimized_c_shim); + +sdivmod!( + __udivmoddi4, + __divmoddi4, + u64, + i64, + maybe_use_optimized_c_shim +); +sdiv!(__udivdi3, __divdi3, u64, i64, maybe_use_optimized_c_shim); +smod!(__umoddi3, __moddi3, u64, i64, maybe_use_optimized_c_shim); + +// LLVM does not currently have a `__divmodti4` function, but GCC does +sdivmod!( + __udivmodti4, + __divmodti4, + u128, + i128, + maybe_use_optimized_c_shim +); +sdiv!(__udivti3, __divti3, u128, i128, win64_128bit_abi_hack); +smod!(__umodti3, __modti3, u128, i128, win64_128bit_abi_hack); diff --git a/crux-mir/lib/compiler_builtins/src/int/shift.rs b/crux-mir/lib/compiler_builtins/src/int/shift.rs index 408f8f3cc..908e619e1 100644 --- a/crux-mir/lib/compiler_builtins/src/int/shift.rs +++ b/crux-mir/lib/compiler_builtins/src/int/shift.rs @@ -1,79 +1,79 @@ -use int::{Int, LargeInt}; +use int::{DInt, HInt, Int}; -trait Ashl: Int + LargeInt { +trait Ashl: DInt { /// Returns `a << b`, requires `b < Self::BITS` - fn ashl(self, offset: u32) -> Self - where - Self: LargeInt::LowHalf>, - { - let half_bits = Self::BITS / 2; - if offset & half_bits != 0 { - Self::from_parts(Int::ZERO, self.low() << (offset - half_bits)) - } else if offset == 0 { + fn ashl(self, shl: u32) -> Self { + let n_h = Self::H::BITS; + if shl & n_h != 0 { + // we only need `self.lo()` because `self.hi()` will be shifted out entirely + self.lo().wrapping_shl(shl - n_h).widen_hi() + } else if shl == 0 { self } else { - Self::from_parts( - self.low() << offset, - (self.high() << offset) | (self.low() >> (half_bits - offset)), + Self::from_lo_hi( + self.lo().wrapping_shl(shl), + self.lo().logical_shr(n_h - shl) | self.hi().wrapping_shl(shl), ) } } } +impl Ashl for u32 {} impl Ashl for u64 {} impl Ashl for u128 {} -trait Ashr: Int + LargeInt { +trait Ashr: DInt { /// Returns arithmetic `a >> b`, requires `b < Self::BITS` - fn ashr(self, offset: u32) -> Self - where - Self: LargeInt::HighHalf as Int>::UnsignedInt>, - { - let half_bits = Self::BITS / 2; - if offset & half_bits != 0 { - Self::from_parts( - (self.high() >> (offset - half_bits)).unsigned(), - self.high() >> (half_bits - 1), + fn ashr(self, shr: u32) -> Self { + let n_h = Self::H::BITS; + if shr & n_h != 0 { + Self::from_lo_hi( + self.hi().wrapping_shr(shr - n_h), + // smear the sign bit + self.hi().wrapping_shr(n_h - 1), ) - } else if offset == 0 { + } else if shr == 0 { self } else { - let high_unsigned = self.high().unsigned(); - Self::from_parts( - (high_unsigned << (half_bits - offset)) | (self.low() >> offset), - self.high() >> offset, + Self::from_lo_hi( + self.lo().logical_shr(shr) | self.hi().wrapping_shl(n_h - shr), + self.hi().wrapping_shr(shr), ) } } } +impl Ashr for i32 {} impl Ashr for i64 {} impl Ashr for i128 {} -trait Lshr: Int + LargeInt { +trait Lshr: DInt { /// Returns logical `a >> b`, requires `b < Self::BITS` - fn lshr(self, offset: u32) -> Self - where - Self: LargeInt::LowHalf>, - { - let half_bits = Self::BITS / 2; - if offset & half_bits != 0 { - Self::from_parts(self.high() >> (offset - half_bits), Int::ZERO) - } else if offset == 0 { + fn lshr(self, shr: u32) -> Self { + let n_h = Self::H::BITS; + if shr & n_h != 0 { + self.hi().logical_shr(shr - n_h).zero_widen() + } else if shr == 0 { self } else { - Self::from_parts( - (self.high() << (half_bits - offset)) | (self.low() >> offset), - self.high() >> offset, + Self::from_lo_hi( + self.lo().logical_shr(shr) | self.hi().wrapping_shl(n_h - shr), + self.hi().logical_shr(shr), ) } } } +impl Lshr for u32 {} impl Lshr for u64 {} impl Lshr for u128 {} intrinsics! { + #[maybe_use_optimized_c_shim] + pub extern "C" fn __ashlsi3(a: u32, b: u32) -> u32 { + a.ashl(b) + } + #[maybe_use_optimized_c_shim] #[arm_aeabi_alias = __aeabi_llsl] pub extern "C" fn __ashldi3(a: u64, b: u32) -> u64 { @@ -84,6 +84,11 @@ intrinsics! { a.ashl(b) } + #[maybe_use_optimized_c_shim] + pub extern "C" fn __ashrsi3(a: i32, b: u32) -> i32 { + a.ashr(b) + } + #[maybe_use_optimized_c_shim] #[arm_aeabi_alias = __aeabi_lasr] pub extern "C" fn __ashrdi3(a: i64, b: u32) -> i64 { @@ -94,6 +99,11 @@ intrinsics! { a.ashr(b) } + #[maybe_use_optimized_c_shim] + pub extern "C" fn __lshrsi3(a: u32, b: u32) -> u32 { + a.lshr(b) + } + #[maybe_use_optimized_c_shim] #[arm_aeabi_alias = __aeabi_llsr] pub extern "C" fn __lshrdi3(a: u64, b: u32) -> u64 { @@ -103,20 +113,4 @@ intrinsics! { pub extern "C" fn __lshrti3(a: u128, b: u32) -> u128 { a.lshr(b) } - - pub extern "C" fn __rust_i128_shlo(a: i128, b: u128) -> (i128, bool) { - (__ashlti3(a as _, b as _) as _, b >= 128) - } - - pub extern "C" fn __rust_u128_shlo(a: u128, b: u128) -> (u128, bool) { - (__ashlti3(a, b as _), b >= 128) - } - - pub extern "C" fn __rust_i128_shro(a: i128, b: u128) -> (i128, bool) { - (__ashrti3(a, b as _), b >= 128) - } - - pub extern "C" fn __rust_u128_shro(a: u128, b: u128) -> (u128, bool) { - (__lshrti3(a, b as _), b >= 128) - } } diff --git a/crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/asymmetric.rs b/crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/asymmetric.rs new file mode 100644 index 000000000..56ce188a3 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/asymmetric.rs @@ -0,0 +1,69 @@ +/// Creates an unsigned division function optimized for dividing integers with the same +/// bitwidth as the largest operand in an asymmetrically sized division. For example, x86-64 has an +/// assembly instruction that can divide a 128 bit integer by a 64 bit integer if the quotient fits +/// in 64 bits. The 128 bit version of this algorithm would use that fast hardware division to +/// construct a full 128 bit by 128 bit division. +#[allow(unused_macros)] +macro_rules! impl_asymmetric { + ( + $fn:ident, // name of the unsigned division function + $zero_div_fn:ident, // function called when division by zero is attempted + $half_division:ident, // function for division of a $uX by a $uX + $asymmetric_division:ident, // function for division of a $uD by a $uX + $n_h:expr, // the number of bits in a $iH or $uH + $uH:ident, // unsigned integer with half the bit width of $uX + $uX:ident, // unsigned integer with half the bit width of $uD + $uD:ident // unsigned integer type for the inputs and outputs of `$fn` + ) => { + /// Computes the quotient and remainder of `duo` divided by `div` and returns them as a + /// tuple. + pub fn $fn(duo: $uD, div: $uD) -> ($uD, $uD) { + let n: u32 = $n_h * 2; + + let duo_lo = duo as $uX; + let duo_hi = (duo >> n) as $uX; + let div_lo = div as $uX; + let div_hi = (div >> n) as $uX; + if div_hi == 0 { + if div_lo == 0 { + $zero_div_fn() + } + if duo_hi < div_lo { + // `$uD` by `$uX` division with a quotient that will fit into a `$uX` + let (quo, rem) = unsafe { $asymmetric_division(duo, div_lo) }; + return (quo as $uD, rem as $uD); + } else { + // Short division using the $uD by $uX division + let (quo_hi, rem_hi) = $half_division(duo_hi, div_lo); + let tmp = unsafe { + $asymmetric_division((duo_lo as $uD) | ((rem_hi as $uD) << n), div_lo) + }; + return ((tmp.0 as $uD) | ((quo_hi as $uD) << n), tmp.1 as $uD); + } + } + + // This has been adapted from + // https://www.codeproject.com/tips/785014/uint-division-modulus which was in turn + // adapted from Hacker's Delight. This is similar to the two possibility algorithm + // in that it uses only more significant parts of `duo` and `div` to divide a large + // integer with a smaller division instruction. + let div_lz = div_hi.leading_zeros(); + let div_extra = n - div_lz; + let div_sig_n = (div >> div_extra) as $uX; + let tmp = unsafe { $asymmetric_division(duo >> 1, div_sig_n) }; + + let mut quo = tmp.0 >> ((n - 1) - div_lz); + if quo != 0 { + quo -= 1; + } + + // Note that this is a full `$uD` multiplication being used here + let mut rem = duo - (quo as $uD).wrapping_mul(div); + if div <= rem { + quo += 1; + rem -= div; + } + return (quo as $uD, rem); + } + }; +} diff --git a/crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/binary_long.rs b/crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/binary_long.rs new file mode 100644 index 000000000..0d7822882 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/binary_long.rs @@ -0,0 +1,548 @@ +/// Creates an unsigned division function that uses binary long division, designed for +/// computer architectures without division instructions. These functions have good performance for +/// microarchitectures with large branch miss penalties and architectures without the ability to +/// predicate instructions. For architectures with predicated instructions, one of the algorithms +/// described in the documentation of these functions probably has higher performance, and a custom +/// assembly routine should be used instead. +#[allow(unused_macros)] +macro_rules! impl_binary_long { + ( + $fn:ident, // name of the unsigned division function + $zero_div_fn:ident, // function called when division by zero is attempted + $normalization_shift:ident, // function for finding the normalization shift + $n:tt, // the number of bits in a $iX or $uX + $uX:ident, // unsigned integer type for the inputs and outputs of `$fn` + $iX:ident // signed integer type with same bitwidth as `$uX` + ) => { + /// Computes the quotient and remainder of `duo` divided by `div` and returns them as a + /// tuple. + pub fn $fn(duo: $uX, div: $uX) -> ($uX, $uX) { + let mut duo = duo; + // handle edge cases before calling `$normalization_shift` + if div == 0 { + $zero_div_fn() + } + if duo < div { + return (0, duo); + } + + // There are many variations of binary division algorithm that could be used. This + // documentation gives a tour of different methods so that future readers wanting to + // optimize further do not have to painstakingly derive them. The SWAR variation is + // especially hard to understand without reading the less convoluted methods first. + + // You may notice that a `duo < div_original` check is included in many these + // algorithms. A critical optimization that many algorithms miss is handling of + // quotients that will turn out to have many trailing zeros or many leading zeros. This + // happens in cases of exact or close-to-exact divisions, divisions by power of two, and + // in cases where the quotient is small. The `duo < div_original` check handles these + // cases of early returns and ends up replacing other kinds of mundane checks that + // normally terminate a binary division algorithm. + // + // Something you may see in other algorithms that is not special-cased here is checks + // for division by powers of two. The `duo < div_original` check handles this case and + // more, however it can be checked up front before the bisection using the + // `((div > 0) && ((div & (div - 1)) == 0))` trick. This is not special-cased because + // compilers should handle most cases where divisions by power of two occur, and we do + // not want to add on a few cycles for every division operation just to save a few + // cycles rarely. + + // The following example is the most straightforward translation from the way binary + // long division is typically visualized: + // Dividing 178u8 (0b10110010) by 6u8 (0b110). `div` is shifted left by 5, according to + // the result from `$normalization_shift(duo, div, false)`. + // + // Step 0: `sub` is negative, so there is not full normalization, so no `quo` bit is set + // and `duo` is kept unchanged. + // duo:10110010, div_shifted:11000000, sub:11110010, quo:00000000, shl:5 + // + // Step 1: `sub` is positive, set a `quo` bit and update `duo` for next step. + // duo:10110010, div_shifted:01100000, sub:01010010, quo:00010000, shl:4 + // + // Step 2: Continue based on `sub`. The `quo` bits start accumulating. + // duo:01010010, div_shifted:00110000, sub:00100010, quo:00011000, shl:3 + // duo:00100010, div_shifted:00011000, sub:00001010, quo:00011100, shl:2 + // duo:00001010, div_shifted:00001100, sub:11111110, quo:00011100, shl:1 + // duo:00001010, div_shifted:00000110, sub:00000100, quo:00011100, shl:0 + // The `duo < div_original` check terminates the algorithm with the correct quotient of + // 29u8 and remainder of 4u8 + /* + let div_original = div; + let mut shl = $normalization_shift(duo, div, false); + let mut quo = 0; + loop { + let div_shifted = div << shl; + let sub = duo.wrapping_sub(div_shifted); + // it is recommended to use `println!`s like this if functionality is unclear + /* + println!("duo:{:08b}, div_shifted:{:08b}, sub:{:08b}, quo:{:08b}, shl:{}", + duo, + div_shifted, + sub, + quo, + shl + ); + */ + if 0 <= (sub as $iX) { + duo = sub; + quo += 1 << shl; + if duo < div_original { + // this branch is optional + return (quo, duo) + } + } + if shl == 0 { + return (quo, duo) + } + shl -= 1; + } + */ + + // This restoring binary long division algorithm reduces the number of operations + // overall via: + // - `pow` can be shifted right instead of recalculating from `shl` + // - starting `div` shifted left and shifting it right for each step instead of + // recalculating from `shl` + // - The `duo < div_original` branch is used to terminate the algorithm instead of the + // `shl == 0` branch. This check is strong enough to prevent set bits of `pow` and + // `div` from being shifted off the end. This check also only occurs on half of steps + // on average, since it is behind the `(sub as $iX) >= 0` branch. + // - `shl` is now not needed by any aspect of of the loop and thus only 3 variables are + // being updated between steps + // + // There are many variations of this algorithm, but this encompases the largest number + // of architectures and does not rely on carry flags, add-with-carry, or SWAR + // complications to be decently fast. + /* + let div_original = div; + let shl = $normalization_shift(duo, div, false); + let mut div: $uX = div << shl; + let mut pow: $uX = 1 << shl; + let mut quo: $uX = 0; + loop { + let sub = duo.wrapping_sub(div); + if 0 <= (sub as $iX) { + duo = sub; + quo |= pow; + if duo < div_original { + return (quo, duo) + } + } + div >>= 1; + pow >>= 1; + } + */ + + // If the architecture has flags and predicated arithmetic instructions, it is possible + // to do binary long division without branching and in only 3 or 4 instructions. This is + // a variation of a 3 instruction central loop from + // http://www.chiark.greenend.org.uk/~theom/riscos/docs/ultimate/a252div.txt. + // + // What allows doing division in only 3 instructions is realizing that instead of + // keeping `duo` in place and shifting `div` right to align bits, `div` can be kept in + // place and `duo` can be shifted left. This means `div` does not have to be updated, + // but causes edge case problems and makes `duo < div_original` tests harder. Some + // architectures have an option to shift an argument in an arithmetic operation, which + // means `duo` can be shifted left and subtracted from in one instruction. The other two + // instructions are updating `quo` and undoing the subtraction if it turns out things + // were not normalized. + + /* + // Perform one binary long division step on the already normalized arguments, because + // the main. Note that this does a full normalization since the central loop needs + // `duo.leading_zeros()` to be at least 1 more than `div.leading_zeros()`. The original + // variation only did normalization to the nearest 4 steps, but this makes handling edge + // cases much harder. We do a full normalization and perform a binary long division + // step. In the edge case where the msbs of `duo` and `div` are set, it clears the msb + // of `duo`, then the edge case handler shifts `div` right and does another long + // division step to always insure `duo.leading_zeros() + 1 >= div.leading_zeros()`. + let div_original = div; + let mut shl = $normalization_shift(duo, div, true); + let mut div: $uX = (div << shl); + let mut quo: $uX = 1; + duo = duo.wrapping_sub(div); + if duo < div_original { + return (1 << shl, duo); + } + let div_neg: $uX; + if (div as $iX) < 0 { + // A very ugly edge case where the most significant bit of `div` is set (after + // shifting to match `duo` when its most significant bit is at the sign bit), which + // leads to the sign bit of `div_neg` being cut off and carries not happening when + // they should. This branch performs a long division step that keeps `duo` in place + // and shifts `div` down. + div >>= 1; + div_neg = div.wrapping_neg(); + let (sub, carry) = duo.overflowing_add(div_neg); + duo = sub; + quo = quo.wrapping_add(quo).wrapping_add(carry as $uX); + if !carry { + duo = duo.wrapping_add(div); + } + shl -= 1; + } else { + div_neg = div.wrapping_neg(); + } + // The add-with-carry that updates `quo` needs to have the carry set when a normalized + // subtract happens. Using `duo.wrapping_shl(1).overflowing_sub(div)` to do the + // subtraction generates a carry when an unnormalized subtract happens, which is the + // opposite of what we want. Instead, we use + // `duo.wrapping_shl(1).overflowing_add(div_neg)`, where `div_neg` is negative `div`. + let mut i = shl; + loop { + if i == 0 { + break; + } + i -= 1; + // `ADDS duo, div, duo, LSL #1` + // (add `div` to `duo << 1` and set flags) + let (sub, carry) = duo.wrapping_shl(1).overflowing_add(div_neg); + duo = sub; + // `ADC quo, quo, quo` + // (add with carry). Effectively shifts `quo` left by 1 and sets the least + // significant bit to the carry. + quo = quo.wrapping_add(quo).wrapping_add(carry as $uX); + // `ADDCC duo, duo, div` + // (add if carry clear). Undoes the subtraction if no carry was generated. + if !carry { + duo = duo.wrapping_add(div); + } + } + return (quo, duo >> shl); + */ + + // This is the SWAR (SIMD within in a register) restoring division algorithm. + // This combines several ideas of the above algorithms: + // - If `duo` is shifted left instead of shifting `div` right like in the 3 instruction + // restoring division algorithm, some architectures can do the shifting and + // subtraction step in one instruction. + // - `quo` can be constructed by adding powers-of-two to it or shifting it left by one + // and adding one. + // - Every time `duo` is shifted left, there is another unused 0 bit shifted into the + // LSB, so what if we use those bits to store `quo`? + // Through a complex setup, it is possible to manage `duo` and `quo` in the same + // register, and perform one step with 2 or 3 instructions. The only major downsides are + // that there is significant setup (it is only saves instructions if `shl` is + // approximately more than 4), `duo < div_original` checks are impractical once SWAR is + // initiated, and the number of division steps taken has to be exact (we cannot do more + // division steps than `shl`, because it introduces edge cases where quotient bits in + // `duo` start to collide with the real part of `div`. + /* + // first step. The quotient bit is stored in `quo` for now + let div_original = div; + let mut shl = $normalization_shift(duo, div, true); + let mut div: $uX = (div << shl); + duo = duo.wrapping_sub(div); + let mut quo: $uX = 1 << shl; + if duo < div_original { + return (quo, duo); + } + + let mask: $uX; + if (div as $iX) < 0 { + // deal with same edge case as the 3 instruction restoring division algorithm, but + // the quotient bit from this step also has to be stored in `quo` + div >>= 1; + shl -= 1; + let tmp = 1 << shl; + mask = tmp - 1; + let sub = duo.wrapping_sub(div); + if (sub as $iX) >= 0 { + // restore + duo = sub; + quo |= tmp; + } + if duo < div_original { + return (quo, duo); + } + } else { + mask = quo - 1; + } + // There is now room for quotient bits in `duo`. + + // Note that `div` is already shifted left and has `shl` unset bits. We subtract 1 from + // `div` and end up with the subset of `shl` bits being all being set. This subset acts + // just like a two's complement negative one. The subset of `div` containing the divisor + // had 1 subtracted from it, but a carry will always be generated from the `shl` subset + // as long as the quotient stays positive. + // + // When the modified `div` is subtracted from `duo.wrapping_shl(1)`, the `shl` subset + // adds a quotient bit to the least significant bit. + // For example, 89 (0b01011001) divided by 3 (0b11): + // + // shl:4, div:0b00110000 + // first step: + // duo:0b01011001 + // + div_neg:0b11010000 + // ____________________ + // 0b00101001 + // quo is set to 0b00010000 and mask is set to 0b00001111 for later + // + // 1 is subtracted from `div`. I will differentiate the `shl` part of `div` and the + // quotient part of `duo` with `^`s. + // chars. + // div:0b00110000 + // ^^^^ + // + 0b11111111 + // ________________ + // 0b00101111 + // ^^^^ + // div_neg:0b11010001 + // + // first SWAR step: + // duo_shl1:0b01010010 + // ^ + // + div_neg:0b11010001 + // ____________________ + // 0b00100011 + // ^ + // second: + // duo_shl1:0b01000110 + // ^^ + // + div_neg:0b11010001 + // ____________________ + // 0b00010111 + // ^^ + // third: + // duo_shl1:0b00101110 + // ^^^ + // + div_neg:0b11010001 + // ____________________ + // 0b11111111 + // ^^^ + // 3 steps resulted in the quotient with 3 set bits as expected, but currently the real + // part of `duo` is negative and the third step was an unnormalized step. The restore + // branch then restores `duo`. Note that the restore branch does not shift `duo` left. + // + // duo:0b11111111 + // ^^^ + // + div:0b00101111 + // ^^^^ + // ________________ + // 0b00101110 + // ^^^ + // `duo` is now back in the `duo_shl1` state it was at in the the third step, with an + // unset quotient bit. + // + // final step (`shl` was 4, so exactly 4 steps must be taken) + // duo_shl1:0b01011100 + // ^^^^ + // + div_neg:0b11010001 + // ____________________ + // 0b00101101 + // ^^^^ + // The quotient includes the `^` bits added with the `quo` bits from the beginning that + // contained the first step and potential edge case step, + // `quo:0b00010000 + (duo:0b00101101 & mask:0b00001111) == 0b00011101 == 29u8`. + // The remainder is the bits remaining in `duo` that are not part of the quotient bits, + // `duo:0b00101101 >> shl == 0b0010 == 2u8`. + let div: $uX = div.wrapping_sub(1); + let mut i = shl; + loop { + if i == 0 { + break; + } + i -= 1; + duo = duo.wrapping_shl(1).wrapping_sub(div); + if (duo as $iX) < 0 { + // restore + duo = duo.wrapping_add(div); + } + } + // unpack the results of SWAR + return ((duo & mask) | quo, duo >> shl); + */ + + // The problem with the conditional restoring SWAR algorithm above is that, in practice, + // it requires assembly code to bring out its full unrolled potential (It seems that + // LLVM can't use unrolled conditionals optimally and ends up erasing all the benefit + // that my algorithm intends. On architectures without predicated instructions, the code + // gen is especially bad. We need a default software division algorithm that is + // guaranteed to get decent code gen for the central loop. + + // For non-SWAR algorithms, there is a way to do binary long division without + // predication or even branching. This involves creating a mask from the sign bit and + // performing different kinds of steps using that. + /* + let shl = $normalization_shift(duo, div, true); + let mut div: $uX = div << shl; + let mut pow: $uX = 1 << shl; + let mut quo: $uX = 0; + loop { + let sub = duo.wrapping_sub(div); + let sign_mask = !((sub as $iX).wrapping_shr($n - 1) as $uX); + duo -= div & sign_mask; + quo |= pow & sign_mask; + div >>= 1; + pow >>= 1; + if pow == 0 { + break; + } + } + return (quo, duo); + */ + // However, it requires about 4 extra operations (smearing the sign bit, negating the + // mask, and applying the mask twice) on top of the operations done by the actual + // algorithm. With SWAR however, just 2 extra operations are needed, making it + // practical and even the most optimal algorithm for some architectures. + + // What we do is use custom assembly for predicated architectures that need software + // division, and for the default algorithm use a mask based restoring SWAR algorithm + // without conditionals or branches. On almost all architectures, this Rust code is + // guaranteed to compile down to 5 assembly instructions or less for each step, and LLVM + // will unroll it in a decent way. + + // standard opening for SWAR algorithm with first step and edge case handling + let div_original = div; + let mut shl = $normalization_shift(duo, div, true); + let mut div: $uX = (div << shl); + duo = duo.wrapping_sub(div); + let mut quo: $uX = 1 << shl; + if duo < div_original { + return (quo, duo); + } + let mask: $uX; + if (div as $iX) < 0 { + div >>= 1; + shl -= 1; + let tmp = 1 << shl; + mask = tmp - 1; + let sub = duo.wrapping_sub(div); + if (sub as $iX) >= 0 { + duo = sub; + quo |= tmp; + } + if duo < div_original { + return (quo, duo); + } + } else { + mask = quo - 1; + } + + // central loop + div = div.wrapping_sub(1); + let mut i = shl; + loop { + if i == 0 { + break; + } + i -= 1; + // shift left 1 and subtract + duo = duo.wrapping_shl(1).wrapping_sub(div); + // create mask + let mask = (duo as $iX).wrapping_shr($n - 1) as $uX; + // restore + duo = duo.wrapping_add(div & mask); + } + // unpack + return ((duo & mask) | quo, duo >> shl); + + // miscellanious binary long division algorithms that might be better for specific + // architectures + + // Another kind of long division uses an interesting fact that `div` and `pow` can be + // negated when `duo` is negative to perform a "negated" division step that works in + // place of any normalization mechanism. This is a non-restoring division algorithm that + // is very similar to the non-restoring division algorithms that can be found on the + // internet, except there is only one test for `duo < 0`. The subtraction from `quo` can + // be viewed as shifting the least significant set bit right (e.x. if we enter a series + // of negated binary long division steps starting with `quo == 0b1011_0000` and + // `pow == 0b0000_1000`, `quo` will progress like this: 0b1010_1000, 0b1010_0100, + // 0b1010_0010, 0b1010_0001). + /* + let div_original = div; + let shl = $normalization_shift(duo, div, true); + let mut div: $uX = (div << shl); + let mut pow: $uX = 1 << shl; + let mut quo: $uX = pow; + duo = duo.wrapping_sub(div); + if duo < div_original { + return (quo, duo); + } + div >>= 1; + pow >>= 1; + loop { + if (duo as $iX) < 0 { + // Negated binary long division step. + duo = duo.wrapping_add(div); + quo = quo.wrapping_sub(pow); + } else { + // Normal long division step. + if duo < div_original { + return (quo, duo) + } + duo = duo.wrapping_sub(div); + quo = quo.wrapping_add(pow); + } + pow >>= 1; + div >>= 1; + } + */ + + // This is the Nonrestoring SWAR algorithm, combining the nonrestoring algorithm with + // SWAR techniques that makes the only difference between steps be negation of `div`. + // If there was an architecture with an instruction that negated inputs to an adder + // based on conditionals, and in place shifting (or a three input addition operation + // that can have `duo` as two of the inputs to effectively shift it left by 1), then a + // single instruction central loop is possible. Microarchitectures often have inputs to + // their ALU that can invert the arguments and carry in of adders, but the architectures + // unfortunately do not have an instruction to dynamically invert this input based on + // conditionals. + /* + // SWAR opening + let div_original = div; + let mut shl = $normalization_shift(duo, div, true); + let mut div: $uX = (div << shl); + duo = duo.wrapping_sub(div); + let mut quo: $uX = 1 << shl; + if duo < div_original { + return (quo, duo); + } + let mask: $uX; + if (div as $iX) < 0 { + div >>= 1; + shl -= 1; + let tmp = 1 << shl; + let sub = duo.wrapping_sub(div); + if (sub as $iX) >= 0 { + // restore + duo = sub; + quo |= tmp; + } + if duo < div_original { + return (quo, duo); + } + mask = tmp - 1; + } else { + mask = quo - 1; + } + + // central loop + let div: $uX = div.wrapping_sub(1); + let mut i = shl; + loop { + if i == 0 { + break; + } + i -= 1; + // note: the `wrapping_shl(1)` can be factored out, but would require another + // restoring division step to prevent `(duo as $iX)` from overflowing + if (duo as $iX) < 0 { + // Negated binary long division step. + duo = duo.wrapping_shl(1).wrapping_add(div); + } else { + // Normal long division step. + duo = duo.wrapping_shl(1).wrapping_sub(div); + } + } + if (duo as $iX) < 0 { + // Restore. This was not needed in the original nonrestoring algorithm because of + // the `duo < div_original` checks. + duo = duo.wrapping_add(div); + } + // unpack + return ((duo & mask) | quo, duo >> shl); + */ + } + }; +} diff --git a/crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/delegate.rs b/crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/delegate.rs new file mode 100644 index 000000000..330c6e4f8 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/delegate.rs @@ -0,0 +1,319 @@ +/// Creates an unsigned division function that uses a combination of hardware division and +/// binary long division to divide integers larger than what hardware division by itself can do. This +/// function is intended for microarchitectures that have division hardware, but not fast enough +/// multiplication hardware for `impl_trifecta` to be faster. +#[allow(unused_macros)] +macro_rules! impl_delegate { + ( + $fn:ident, // name of the unsigned division function + $zero_div_fn:ident, // function called when division by zero is attempted + $half_normalization_shift:ident, // function for finding the normalization shift of $uX + $half_division:ident, // function for division of a $uX by a $uX + $n_h:expr, // the number of bits in $iH or $uH + $uH:ident, // unsigned integer with half the bit width of $uX + $uX:ident, // unsigned integer with half the bit width of $uD. + $uD:ident, // unsigned integer type for the inputs and outputs of `$fn` + $iD:ident // signed integer type with the same bitwidth as `$uD` + ) => { + /// Computes the quotient and remainder of `duo` divided by `div` and returns them as a + /// tuple. + pub fn $fn(duo: $uD, div: $uD) -> ($uD, $uD) { + // The two possibility algorithm, undersubtracting long division algorithm, or any kind + // of reciprocal based algorithm will not be fastest, because they involve large + // multiplications that we assume to not be fast enough relative to the divisions to + // outweigh setup times. + + // the number of bits in a $uX + let n = $n_h * 2; + + let duo_lo = duo as $uX; + let duo_hi = (duo >> n) as $uX; + let div_lo = div as $uX; + let div_hi = (div >> n) as $uX; + + match (div_lo == 0, div_hi == 0, duo_hi == 0) { + (true, true, _) => $zero_div_fn(), + (_, false, true) => { + // `duo` < `div` + return (0, duo); + } + (false, true, true) => { + // delegate to smaller division + let tmp = $half_division(duo_lo, div_lo); + return (tmp.0 as $uD, tmp.1 as $uD); + } + (false, true, false) => { + if duo_hi < div_lo { + // `quo_hi` will always be 0. This performs a binary long division algorithm + // to zero `duo_hi` followed by a half division. + + // We can calculate the normalization shift using only `$uX` size functions. + // If we calculated the normalization shift using + // `$half_normalization_shift(duo_hi, div_lo false)`, it would break the + // assumption the function has that the first argument is more than the + // second argument. If the arguments are switched, the assumption holds true + // since `duo_hi < div_lo`. + let norm_shift = $half_normalization_shift(div_lo, duo_hi, false); + let shl = if norm_shift == 0 { + // Consider what happens if the msbs of `duo_hi` and `div_lo` align with + // no shifting. The normalization shift will always return + // `norm_shift == 0` regardless of whether it is fully normalized, + // because `duo_hi < div_lo`. In that edge case, `n - norm_shift` would + // result in shift overflow down the line. For the edge case, because + // both `duo_hi < div_lo` and we are comparing all the significant bits + // of `duo_hi` and `div`, we can make `shl = n - 1`. + n - 1 + } else { + // We also cannot just use `shl = n - norm_shift - 1` in the general + // case, because when we are not in the edge case comparing all the + // significant bits, then the full `duo < div` may not be true and thus + // breaks the division algorithm. + n - norm_shift + }; + + // The 3 variable restoring division algorithm (see binary_long.rs) is ideal + // for this task, since `pow` and `quo` can be `$uX` and the delegation + // check is simple. + let mut div: $uD = div << shl; + let mut pow_lo: $uX = 1 << shl; + let mut quo_lo: $uX = 0; + let mut duo = duo; + loop { + let sub = duo.wrapping_sub(div); + if 0 <= (sub as $iD) { + duo = sub; + quo_lo |= pow_lo; + let duo_hi = (duo >> n) as $uX; + if duo_hi == 0 { + // Delegate to get the rest of the quotient. Note that the + // `div_lo` here is the original unshifted `div`. + let tmp = $half_division(duo as $uX, div_lo); + return ((quo_lo | tmp.0) as $uD, tmp.1 as $uD); + } + } + div >>= 1; + pow_lo >>= 1; + } + } else if duo_hi == div_lo { + // `quo_hi == 1`. This branch is cheap and helps with edge cases. + let tmp = $half_division(duo as $uX, div as $uX); + return ((1 << n) | (tmp.0 as $uD), tmp.1 as $uD); + } else { + // `div_lo < duo_hi` + // `rem_hi == 0` + if (div_lo >> $n_h) == 0 { + // Short division of $uD by a $uH, using $uX by $uX division + let div_0 = div_lo as $uH as $uX; + let (quo_hi, rem_3) = $half_division(duo_hi, div_0); + + let duo_mid = ((duo >> $n_h) as $uH as $uX) | (rem_3 << $n_h); + let (quo_1, rem_2) = $half_division(duo_mid, div_0); + + let duo_lo = (duo as $uH as $uX) | (rem_2 << $n_h); + let (quo_0, rem_1) = $half_division(duo_lo, div_0); + + return ( + (quo_0 as $uD) | ((quo_1 as $uD) << $n_h) | ((quo_hi as $uD) << n), + rem_1 as $uD, + ); + } + + // This is basically a short division composed of a half division for the hi + // part, specialized 3 variable binary long division in the middle, and + // another half division for the lo part. + let duo_lo = duo as $uX; + let tmp = $half_division(duo_hi, div_lo); + let quo_hi = tmp.0; + let mut duo = (duo_lo as $uD) | ((tmp.1 as $uD) << n); + // This check is required to avoid breaking the long division below. + if duo < div { + return ((quo_hi as $uD) << n, duo); + } + + // The half division handled all shift alignments down to `n`, so this + // division can continue with a shift of `n - 1`. + let mut div: $uD = div << (n - 1); + let mut pow_lo: $uX = 1 << (n - 1); + let mut quo_lo: $uX = 0; + loop { + let sub = duo.wrapping_sub(div); + if 0 <= (sub as $iD) { + duo = sub; + quo_lo |= pow_lo; + let duo_hi = (duo >> n) as $uX; + if duo_hi == 0 { + // Delegate to get the rest of the quotient. Note that the + // `div_lo` here is the original unshifted `div`. + let tmp = $half_division(duo as $uX, div_lo); + return ( + (tmp.0) as $uD | (quo_lo as $uD) | ((quo_hi as $uD) << n), + tmp.1 as $uD, + ); + } + } + div >>= 1; + pow_lo >>= 1; + } + } + } + (_, false, false) => { + // Full $uD by $uD binary long division. `quo_hi` will always be 0. + if duo < div { + return (0, duo); + } + let div_original = div; + let shl = $half_normalization_shift(duo_hi, div_hi, false); + let mut duo = duo; + let mut div: $uD = div << shl; + let mut pow_lo: $uX = 1 << shl; + let mut quo_lo: $uX = 0; + loop { + let sub = duo.wrapping_sub(div); + if 0 <= (sub as $iD) { + duo = sub; + quo_lo |= pow_lo; + if duo < div_original { + return (quo_lo as $uD, duo); + } + } + div >>= 1; + pow_lo >>= 1; + } + } + } + } + }; +} + +public_test_dep! { +/// Returns `n / d` and sets `*rem = n % d`. +/// +/// This specialization exists because: +/// - The LLVM backend for 32-bit SPARC cannot compile functions that return `(u128, u128)`, +/// so we have to use an old fashioned `&mut u128` argument to return the remainder. +/// - 64-bit SPARC does not have u64 * u64 => u128 widening multiplication, which makes the +/// delegate algorithm strategy the only reasonably fast way to perform `u128` division. +// used on SPARC +#[allow(dead_code)] +pub(crate) fn u128_divide_sparc(duo: u128, div: u128, rem: &mut u128) -> u128 { + use super::*; + let duo_lo = duo as u64; + let duo_hi = (duo >> 64) as u64; + let div_lo = div as u64; + let div_hi = (div >> 64) as u64; + + match (div_lo == 0, div_hi == 0, duo_hi == 0) { + (true, true, _) => zero_div_fn(), + (_, false, true) => { + *rem = duo; + return 0; + } + (false, true, true) => { + let tmp = u64_by_u64_div_rem(duo_lo, div_lo); + *rem = tmp.1 as u128; + return tmp.0 as u128; + } + (false, true, false) => { + if duo_hi < div_lo { + let norm_shift = u64_normalization_shift(div_lo, duo_hi, false); + let shl = if norm_shift == 0 { + 64 - 1 + } else { + 64 - norm_shift + }; + + let mut div: u128 = div << shl; + let mut pow_lo: u64 = 1 << shl; + let mut quo_lo: u64 = 0; + let mut duo = duo; + loop { + let sub = duo.wrapping_sub(div); + if 0 <= (sub as i128) { + duo = sub; + quo_lo |= pow_lo; + let duo_hi = (duo >> 64) as u64; + if duo_hi == 0 { + let tmp = u64_by_u64_div_rem(duo as u64, div_lo); + *rem = tmp.1 as u128; + return (quo_lo | tmp.0) as u128; + } + } + div >>= 1; + pow_lo >>= 1; + } + } else if duo_hi == div_lo { + let tmp = u64_by_u64_div_rem(duo as u64, div as u64); + *rem = tmp.1 as u128; + return (1 << 64) | (tmp.0 as u128); + } else { + if (div_lo >> 32) == 0 { + let div_0 = div_lo as u32 as u64; + let (quo_hi, rem_3) = u64_by_u64_div_rem(duo_hi, div_0); + + let duo_mid = ((duo >> 32) as u32 as u64) | (rem_3 << 32); + let (quo_1, rem_2) = u64_by_u64_div_rem(duo_mid, div_0); + + let duo_lo = (duo as u32 as u64) | (rem_2 << 32); + let (quo_0, rem_1) = u64_by_u64_div_rem(duo_lo, div_0); + + *rem = rem_1 as u128; + return (quo_0 as u128) | ((quo_1 as u128) << 32) | ((quo_hi as u128) << 64); + } + + let duo_lo = duo as u64; + let tmp = u64_by_u64_div_rem(duo_hi, div_lo); + let quo_hi = tmp.0; + let mut duo = (duo_lo as u128) | ((tmp.1 as u128) << 64); + if duo < div { + *rem = duo; + return (quo_hi as u128) << 64; + } + + let mut div: u128 = div << (64 - 1); + let mut pow_lo: u64 = 1 << (64 - 1); + let mut quo_lo: u64 = 0; + loop { + let sub = duo.wrapping_sub(div); + if 0 <= (sub as i128) { + duo = sub; + quo_lo |= pow_lo; + let duo_hi = (duo >> 64) as u64; + if duo_hi == 0 { + let tmp = u64_by_u64_div_rem(duo as u64, div_lo); + *rem = tmp.1 as u128; + return (tmp.0) as u128 | (quo_lo as u128) | ((quo_hi as u128) << 64); + } + } + div >>= 1; + pow_lo >>= 1; + } + } + } + (_, false, false) => { + if duo < div { + *rem = duo; + return 0; + } + let div_original = div; + let shl = u64_normalization_shift(duo_hi, div_hi, false); + let mut duo = duo; + let mut div: u128 = div << shl; + let mut pow_lo: u64 = 1 << shl; + let mut quo_lo: u64 = 0; + loop { + let sub = duo.wrapping_sub(div); + if 0 <= (sub as i128) { + duo = sub; + quo_lo |= pow_lo; + if duo < div_original { + *rem = duo; + return quo_lo as u128; + } + } + div >>= 1; + pow_lo >>= 1; + } + } + } +} +} diff --git a/crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/mod.rs b/crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/mod.rs new file mode 100644 index 000000000..77034eb54 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/mod.rs @@ -0,0 +1,309 @@ +// TODO: when `unsafe_block_in_unsafe_fn` is stabilized, remove this +#![allow(unused_unsafe)] +// The functions are complex with many branches, and explicit +// `return`s makes it clear where function exit points are +#![allow(clippy::needless_return)] +#![allow(clippy::comparison_chain)] +// Clippy is confused by the complex configuration +#![allow(clippy::if_same_then_else)] +#![allow(clippy::needless_bool)] + +//! This `specialized_div_rem` module is originally from version 1.0.0 of the +//! `specialized-div-rem` crate. Note that `for` loops with ranges are not used in this +//! module, since unoptimized compilation may generate references to `memcpy`. +//! +//! The purpose of these macros is to easily change the both the division algorithm used +//! for a given integer size and the half division used by that algorithm. The way +//! functions call each other is also constructed such that linkers will find the chain of +//! software and hardware divisions needed for every size of signed and unsigned division. +//! For example, most target compilations do the following: +//! +//! - Many 128 bit division functions like `u128::wrapping_div` use +//! `std::intrinsics::unchecked_div`, which gets replaced by `__udivti3` because there +//! is not a 128 bit by 128 bit hardware division function in most architectures. +//! `__udivti3` uses `u128_div_rem` (this extra level of function calls exists because +//! `__umodti3` and `__udivmodti4` also exist, and `specialized_div_rem` supplies just +//! one function to calculate both the quotient and remainder. If configuration flags +//! enable it, `impl_trifecta!` defines `u128_div_rem` to use the trifecta algorithm, +//! which requires the half sized division `u64_by_u64_div_rem`. If the architecture +//! supplies a 64 bit hardware division instruction, `u64_by_u64_div_rem` will be +//! reduced to those instructions. Note that we do not specify the half size division +//! directly to be `__udivdi3`, because hardware division would never be introduced. +//! - If the architecture does not supply a 64 bit hardware division instruction, u64 +//! divisions will use functions such as `__udivdi3`. This will call `u64_div_rem` +//! which is defined by `impl_delegate!`. The half division for this algorithm is +//! `u32_by_u32_div_rem` which in turn becomes hardware division instructions or more +//! software division algorithms. +//! - If the architecture does not supply a 32 bit hardware instruction, linkers will +//! look for `__udivsi3`. `impl_binary_long!` is used, but this algorithm uses no half +//! division, so the chain of calls ends here. +//! +//! On some architectures like x86_64, an asymmetrically sized division is supplied, in +//! which 128 bit numbers can be divided by 64 bit numbers. `impl_asymmetric!` is used to +//! extend the 128 by 64 bit division to a full 128 by 128 bit division. + +// `allow(dead_code)` is used in various places, because the configuration code would otherwise be +// ridiculously complex + +#[macro_use] +mod norm_shift; + +#[macro_use] +mod binary_long; + +#[macro_use] +mod delegate; + +// used on SPARC +#[allow(unused_imports)] +#[cfg(not(feature = "public-test-deps"))] +pub(crate) use self::delegate::u128_divide_sparc; + +#[cfg(feature = "public-test-deps")] +pub use self::delegate::u128_divide_sparc; + +#[macro_use] +mod trifecta; + +#[macro_use] +mod asymmetric; + +/// The behavior of all divisions by zero is controlled by this function. This function should be +/// impossible to reach by Rust users, unless `compiler-builtins` public division functions or +/// `core/std::unchecked_div/rem` are directly used without a zero check in front. +fn zero_div_fn() -> ! { + // Calling the intrinsic directly, to avoid the `assert_unsafe_precondition` that cannot be used + // here because it involves non-`inline` functions + // (https://github.com/rust-lang/compiler-builtins/issues/491). + unsafe { core::intrinsics::unreachable() } +} + +const USE_LZ: bool = { + if cfg!(target_arch = "arm") { + if cfg!(target_feature = "thumb-mode") { + // ARM thumb targets have CLZ instructions if the instruction set of ARMv6T2 is + // supported. This is needed to successfully differentiate between targets like + // `thumbv8.base` and `thumbv8.main`. + cfg!(target_feature = "v6t2") + } else { + // Regular ARM targets have CLZ instructions if the ARMv5TE instruction set is + // supported. Technically, ARMv5T was the first to have CLZ, but the "v5t" target + // feature does not seem to work. + cfg!(target_feature = "v5te") + } + } else if cfg!(any(target_arch = "sparc", target_arch = "sparc64")) { + // LZD or LZCNT on SPARC only exists for the VIS 3 extension and later. + cfg!(target_feature = "vis3") + } else if cfg!(any(target_arch = "riscv32", target_arch = "riscv64")) { + // The `B` extension on RISC-V determines if a CLZ assembly instruction exists + cfg!(target_feature = "b") + } else { + // All other common targets Rust supports should have CLZ instructions + true + } +}; + +impl_normalization_shift!( + u32_normalization_shift, + USE_LZ, + 32, + u32, + i32, + allow(dead_code) +); +impl_normalization_shift!( + u64_normalization_shift, + USE_LZ, + 64, + u64, + i64, + allow(dead_code) +); + +/// Divides `duo` by `div` and returns a tuple of the quotient and the remainder. +/// `checked_div` and `checked_rem` are used to avoid bringing in panic function +/// dependencies. +#[inline] +fn u64_by_u64_div_rem(duo: u64, div: u64) -> (u64, u64) { + if let Some(quo) = duo.checked_div(div) { + if let Some(rem) = duo.checked_rem(div) { + return (quo, rem); + } + } + zero_div_fn() +} + +// Whether `trifecta` or `delegate` is faster for 128 bit division depends on the speed at which a +// microarchitecture can multiply and divide. We decide to be optimistic and assume `trifecta` is +// faster if the target pointer width is at least 64. +#[cfg(all( + not(any(target_pointer_width = "16", target_pointer_width = "32")), + not(all(not(feature = "no-asm"), target_arch = "x86_64")), + not(any(target_arch = "sparc", target_arch = "sparc64")) +))] +impl_trifecta!( + u128_div_rem, + zero_div_fn, + u64_by_u64_div_rem, + 32, + u32, + u64, + u128 +); + +// If the pointer width less than 64, then the target architecture almost certainly does not have +// the fast 64 to 128 bit widening multiplication needed for `trifecta` to be faster. +#[cfg(all( + any(target_pointer_width = "16", target_pointer_width = "32"), + not(all(not(feature = "no-asm"), target_arch = "x86_64")), + not(any(target_arch = "sparc", target_arch = "sparc64")) +))] +impl_delegate!( + u128_div_rem, + zero_div_fn, + u64_normalization_shift, + u64_by_u64_div_rem, + 32, + u32, + u64, + u128, + i128 +); + +/// Divides `duo` by `div` and returns a tuple of the quotient and the remainder. +/// +/// # Safety +/// +/// If the quotient does not fit in a `u64`, a floating point exception occurs. +/// If `div == 0`, then a division by zero exception occurs. +#[cfg(all(not(feature = "no-asm"), target_arch = "x86_64"))] +#[inline] +unsafe fn u128_by_u64_div_rem(duo: u128, div: u64) -> (u64, u64) { + let duo_lo = duo as u64; + let duo_hi = (duo >> 64) as u64; + let quo: u64; + let rem: u64; + unsafe { + // divides the combined registers rdx:rax (`duo` is split into two 64 bit parts to do this) + // by `div`. The quotient is stored in rax and the remainder in rdx. + // FIXME: Use the Intel syntax once we drop LLVM 9 support on rust-lang/rust. + core::arch::asm!( + "div {0}", + in(reg) div, + inlateout("rax") duo_lo => quo, + inlateout("rdx") duo_hi => rem, + options(att_syntax, pure, nomem, nostack) + ); + } + (quo, rem) +} + +// use `asymmetric` instead of `trifecta` on x86_64 +#[cfg(all(not(feature = "no-asm"), target_arch = "x86_64"))] +impl_asymmetric!( + u128_div_rem, + zero_div_fn, + u64_by_u64_div_rem, + u128_by_u64_div_rem, + 32, + u32, + u64, + u128 +); + +/// Divides `duo` by `div` and returns a tuple of the quotient and the remainder. +/// `checked_div` and `checked_rem` are used to avoid bringing in panic function +/// dependencies. +#[inline] +#[allow(dead_code)] +fn u32_by_u32_div_rem(duo: u32, div: u32) -> (u32, u32) { + if let Some(quo) = duo.checked_div(div) { + if let Some(rem) = duo.checked_rem(div) { + return (quo, rem); + } + } + zero_div_fn() +} + +// When not on x86 and the pointer width is not 64, use `delegate` since the division size is larger +// than register size. +#[cfg(all( + not(all(not(feature = "no-asm"), target_arch = "x86")), + not(target_pointer_width = "64") +))] +impl_delegate!( + u64_div_rem, + zero_div_fn, + u32_normalization_shift, + u32_by_u32_div_rem, + 16, + u16, + u32, + u64, + i64 +); + +// When not on x86 and the pointer width is 64, use `binary_long`. +#[cfg(all( + not(all(not(feature = "no-asm"), target_arch = "x86")), + target_pointer_width = "64" +))] +impl_binary_long!( + u64_div_rem, + zero_div_fn, + u64_normalization_shift, + 64, + u64, + i64 +); + +/// Divides `duo` by `div` and returns a tuple of the quotient and the remainder. +/// +/// # Safety +/// +/// If the quotient does not fit in a `u32`, a floating point exception occurs. +/// If `div == 0`, then a division by zero exception occurs. +#[cfg(all(not(feature = "no-asm"), target_arch = "x86"))] +#[inline] +unsafe fn u64_by_u32_div_rem(duo: u64, div: u32) -> (u32, u32) { + let duo_lo = duo as u32; + let duo_hi = (duo >> 32) as u32; + let quo: u32; + let rem: u32; + unsafe { + // divides the combined registers rdx:rax (`duo` is split into two 32 bit parts to do this) + // by `div`. The quotient is stored in rax and the remainder in rdx. + // FIXME: Use the Intel syntax once we drop LLVM 9 support on rust-lang/rust. + core::arch::asm!( + "div {0}", + in(reg) div, + inlateout("rax") duo_lo => quo, + inlateout("rdx") duo_hi => rem, + options(att_syntax, pure, nomem, nostack) + ); + } + (quo, rem) +} + +// use `asymmetric` instead of `delegate` on x86 +#[cfg(all(not(feature = "no-asm"), target_arch = "x86"))] +impl_asymmetric!( + u64_div_rem, + zero_div_fn, + u32_by_u32_div_rem, + u64_by_u32_div_rem, + 16, + u16, + u32, + u64 +); + +// 32 bits is the smallest division used by `compiler-builtins`, so we end with binary long division +impl_binary_long!( + u32_div_rem, + zero_div_fn, + u32_normalization_shift, + 32, + u32, + i32 +); diff --git a/crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/norm_shift.rs b/crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/norm_shift.rs new file mode 100644 index 000000000..61b67b6bc --- /dev/null +++ b/crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/norm_shift.rs @@ -0,0 +1,106 @@ +/// Creates a function used by some division algorithms to compute the "normalization shift". +#[allow(unused_macros)] +macro_rules! impl_normalization_shift { + ( + $name:ident, // name of the normalization shift function + // boolean for if `$uX::leading_zeros` should be used (if an architecture does not have a + // hardware instruction for `usize::leading_zeros`, then this should be `true`) + $use_lz:ident, + $n:tt, // the number of bits in a $iX or $uX + $uX:ident, // unsigned integer type for the inputs of `$name` + $iX:ident, // signed integer type for the inputs of `$name` + $($unsigned_attr:meta),* // attributes for the function + ) => { + /// Finds the shift left that the divisor `div` would need to be normalized for a binary + /// long division step with the dividend `duo`. NOTE: This function assumes that these edge + /// cases have been handled before reaching it: + /// ` + /// if div == 0 { + /// panic!("attempt to divide by zero") + /// } + /// if duo < div { + /// return (0, duo) + /// } + /// ` + /// + /// Normalization is defined as (where `shl` is the output of this function): + /// ` + /// if duo.leading_zeros() != (div << shl).leading_zeros() { + /// // If the most significant bits of `duo` and `div << shl` are not in the same place, + /// // then `div << shl` has one more leading zero than `duo`. + /// assert_eq!(duo.leading_zeros() + 1, (div << shl).leading_zeros()); + /// // Also, `2*(div << shl)` is not more than `duo` (otherwise the first division step + /// // would not be able to clear the msb of `duo`) + /// assert!(duo < (div << (shl + 1))); + /// } + /// if full_normalization { + /// // Some algorithms do not need "full" normalization, which means that `duo` is + /// // larger than `div << shl` when the most significant bits are aligned. + /// assert!((div << shl) <= duo); + /// } + /// ` + /// + /// Note: If the software bisection algorithm is being used in this function, it happens + /// that full normalization always occurs, so be careful that new algorithms are not + /// invisibly depending on this invariant when `full_normalization` is set to `false`. + $( + #[$unsigned_attr] + )* + fn $name(duo: $uX, div: $uX, full_normalization: bool) -> usize { + // We have to find the leading zeros of `div` to know where its msb (most significant + // set bit) is to even begin binary long division. It is also good to know where the msb + // of `duo` is so that useful work can be started instead of shifting `div` for all + // possible quotients (many division steps are wasted if `duo.leading_zeros()` is large + // and `div` starts out being shifted all the way to the msb). Aligning the msbs of + // `div` and `duo` could be done by shifting `div` left by + // `div.leading_zeros() - duo.leading_zeros()`, but some CPUs without division hardware + // also do not have single instructions for calculating `leading_zeros`. Instead of + // software doing two bisections to find the two `leading_zeros`, we do one bisection to + // find `div.leading_zeros() - duo.leading_zeros()` without actually knowing either of + // the leading zeros values. + + let mut shl: usize; + if $use_lz { + shl = (div.leading_zeros() - duo.leading_zeros()) as usize; + if full_normalization { + if duo < (div << shl) { + // when the msb of `duo` and `div` are aligned, the resulting `div` may be + // larger than `duo`, so we decrease the shift by 1. + shl -= 1; + } + } + } else { + let mut test = duo; + shl = 0usize; + let mut lvl = $n >> 1; + loop { + let tmp = test >> lvl; + // It happens that a final `duo < (div << shl)` check is not needed, because the + // `div <= tmp` check insures that the msb of `test` never passes the msb of + // `div`, and any set bits shifted off the end of `test` would still keep + // `div <= tmp` true. + if div <= tmp { + test = tmp; + shl += lvl; + } + // narrow down bisection + lvl >>= 1; + if lvl == 0 { + break + } + } + } + // tests the invariants that should hold before beginning binary long division + /* + if full_normalization { + assert!((div << shl) <= duo); + } + if duo.leading_zeros() != (div << shl).leading_zeros() { + assert_eq!(duo.leading_zeros() + 1, (div << shl).leading_zeros()); + assert!(duo < (div << (shl + 1))); + } + */ + shl + } + } +} diff --git a/crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/trifecta.rs b/crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/trifecta.rs new file mode 100644 index 000000000..7e104053b --- /dev/null +++ b/crux-mir/lib/compiler_builtins/src/int/specialized_div_rem/trifecta.rs @@ -0,0 +1,386 @@ +/// Creates an unsigned division function optimized for division of integers with bitwidths +/// larger than the largest hardware integer division supported. These functions use large radix +/// division algorithms that require both fast division and very fast widening multiplication on the +/// target microarchitecture. Otherwise, `impl_delegate` should be used instead. +#[allow(unused_macros)] +macro_rules! impl_trifecta { + ( + $fn:ident, // name of the unsigned division function + $zero_div_fn:ident, // function called when division by zero is attempted + $half_division:ident, // function for division of a $uX by a $uX + $n_h:expr, // the number of bits in $iH or $uH + $uH:ident, // unsigned integer with half the bit width of $uX + $uX:ident, // unsigned integer with half the bit width of $uD + $uD:ident // unsigned integer type for the inputs and outputs of `$unsigned_name` + ) => { + /// Computes the quotient and remainder of `duo` divided by `div` and returns them as a + /// tuple. + pub fn $fn(duo: $uD, div: $uD) -> ($uD, $uD) { + // This is called the trifecta algorithm because it uses three main algorithms: short + // division for small divisors, the two possibility algorithm for large divisors, and an + // undersubtracting long division algorithm for intermediate cases. + + // This replicates `carrying_mul` (rust-lang rfc #2417). LLVM correctly optimizes this + // to use a widening multiply to 128 bits on the relevant architectures. + fn carrying_mul(lhs: $uX, rhs: $uX) -> ($uX, $uX) { + let tmp = (lhs as $uD).wrapping_mul(rhs as $uD); + (tmp as $uX, (tmp >> ($n_h * 2)) as $uX) + } + fn carrying_mul_add(lhs: $uX, mul: $uX, add: $uX) -> ($uX, $uX) { + let tmp = (lhs as $uD) + .wrapping_mul(mul as $uD) + .wrapping_add(add as $uD); + (tmp as $uX, (tmp >> ($n_h * 2)) as $uX) + } + + // the number of bits in a $uX + let n = $n_h * 2; + + if div == 0 { + $zero_div_fn() + } + + // Trying to use a normalization shift function will cause inelegancies in the code and + // inefficiencies for architectures with a native count leading zeros instruction. The + // undersubtracting algorithm needs both values (keeping the original `div_lz` but + // updating `duo_lz` multiple times), so we assume hardware support for fast + // `leading_zeros` calculation. + let div_lz = div.leading_zeros(); + let mut duo_lz = duo.leading_zeros(); + + // the possible ranges of `duo` and `div` at this point: + // `0 <= duo < 2^n_d` + // `1 <= div < 2^n_d` + + // quotient is 0 or 1 branch + if div_lz <= duo_lz { + // The quotient cannot be more than 1. The highest set bit of `duo` needs to be at + // least one place higher than `div` for the quotient to be more than 1. + if duo >= div { + return (1, duo - div); + } else { + return (0, duo); + } + } + + // `_sb` is the number of significant bits (from the ones place to the highest set bit) + // `{2, 2^div_sb} <= duo < 2^n_d` + // `1 <= div < {2^duo_sb, 2^(n_d - 1)}` + // smaller division branch + if duo_lz >= n { + // `duo < 2^n` so it will fit in a $uX. `div` will also fit in a $uX (because of the + // `div_lz <= duo_lz` branch) so no numerical error. + let (quo, rem) = $half_division(duo as $uX, div as $uX); + return (quo as $uD, rem as $uD); + } + + // `{2^n, 2^div_sb} <= duo < 2^n_d` + // `1 <= div < {2^duo_sb, 2^(n_d - 1)}` + // short division branch + if div_lz >= (n + $n_h) { + // `1 <= div < {2^duo_sb, 2^n_h}` + + // It is barely possible to improve the performance of this by calculating the + // reciprocal and removing one `$half_division`, but only if the CPU can do fast + // multiplications in parallel. Other reciprocal based methods can remove two + // `$half_division`s, but have multiplications that cannot be done in parallel and + // reduce performance. I have decided to use this trivial short division method and + // rely on the CPU having quick divisions. + + let duo_hi = (duo >> n) as $uX; + let div_0 = div as $uH as $uX; + let (quo_hi, rem_3) = $half_division(duo_hi, div_0); + + let duo_mid = ((duo >> $n_h) as $uH as $uX) | (rem_3 << $n_h); + let (quo_1, rem_2) = $half_division(duo_mid, div_0); + + let duo_lo = (duo as $uH as $uX) | (rem_2 << $n_h); + let (quo_0, rem_1) = $half_division(duo_lo, div_0); + + return ( + (quo_0 as $uD) | ((quo_1 as $uD) << $n_h) | ((quo_hi as $uD) << n), + rem_1 as $uD, + ); + } + + // relative leading significant bits, cannot overflow because of above branches + let lz_diff = div_lz - duo_lz; + + // `{2^n, 2^div_sb} <= duo < 2^n_d` + // `2^n_h <= div < {2^duo_sb, 2^(n_d - 1)}` + // `mul` or `mul - 1` branch + if lz_diff < $n_h { + // Two possibility division algorithm + + // The most significant bits of `duo` and `div` are within `$n_h` bits of each + // other. If we take the `n` most significant bits of `duo` and divide them by the + // corresponding bits in `div`, it produces a quotient value `quo`. It happens that + // `quo` or `quo - 1` will always be the correct quotient for the whole number. In + // other words, the bits less significant than the `n` most significant bits of + // `duo` and `div` can only influence the quotient to be one of two values. + // Because there are only two possibilities, there only needs to be one `$uH` sized + // division, a `$uH` by `$uD` multiplication, and only one branch with a few simple + // operations. + // + // Proof that the true quotient can only be `quo` or `quo - 1`. + // All `/` operators here are floored divisions. + // + // `shift` is the number of bits not in the higher `n` significant bits of `duo`. + // (definitions) + // 0. shift = n - duo_lz + // 1. duo_sig_n == duo / 2^shift + // 2. div_sig_n == div / 2^shift + // 3. quo == duo_sig_n / div_sig_n + // + // + // We are trying to find the true quotient, `true_quo`. + // 4. true_quo = duo / div. (definition) + // + // This is true because of the bits that are cut off during the bit shift. + // 5. duo_sig_n * 2^shift <= duo < (duo_sig_n + 1) * 2^shift. + // 6. div_sig_n * 2^shift <= div < (div_sig_n + 1) * 2^shift. + // + // Dividing each bound of (5) by each bound of (6) gives 4 possibilities for what + // `true_quo == duo / div` is bounded by: + // (duo_sig_n * 2^shift) / (div_sig_n * 2^shift) + // (duo_sig_n * 2^shift) / ((div_sig_n + 1) * 2^shift) + // ((duo_sig_n + 1) * 2^shift) / (div_sig_n * 2^shift) + // ((duo_sig_n + 1) * 2^shift) / ((div_sig_n + 1) * 2^shift) + // + // Simplifying each of these four: + // duo_sig_n / div_sig_n + // duo_sig_n / (div_sig_n + 1) + // (duo_sig_n + 1) / div_sig_n + // (duo_sig_n + 1) / (div_sig_n + 1) + // + // Taking the smallest and the largest of these as the low and high bounds + // and replacing `duo / div` with `true_quo`: + // 7. duo_sig_n / (div_sig_n + 1) <= true_quo < (duo_sig_n + 1) / div_sig_n + // + // The `lz_diff < n_h` conditional on this branch makes sure that `div_sig_n` is at + // least `2^n_h`, and the `div_lz <= duo_lz` branch makes sure that the highest bit + // of `div_sig_n` is not the `2^(n - 1)` bit. + // 8. `2^(n - 1) <= duo_sig_n < 2^n` + // 9. `2^n_h <= div_sig_n < 2^(n - 1)` + // + // We want to prove that either + // `(duo_sig_n + 1) / div_sig_n == duo_sig_n / (div_sig_n + 1)` or that + // `(duo_sig_n + 1) / div_sig_n == duo_sig_n / (div_sig_n + 1) + 1`. + // + // We also want to prove that `quo` is one of these: + // `duo_sig_n / div_sig_n == duo_sig_n / (div_sig_n + 1)` or + // `duo_sig_n / div_sig_n == (duo_sig_n + 1) / div_sig_n`. + // + // When 1 is added to the numerator of `duo_sig_n / div_sig_n` to produce + // `(duo_sig_n + 1) / div_sig_n`, it is not possible that the value increases by + // more than 1 with floored integer arithmetic and `div_sig_n != 0`. Consider + // `x/y + 1 < (x + 1)/y` <=> `x/y + 1 < x/y + 1/y` <=> `1 < 1/y` <=> `y < 1`. + // `div_sig_n` is a nonzero integer. Thus, + // 10. `duo_sig_n / div_sig_n == (duo_sig_n + 1) / div_sig_n` or + // `(duo_sig_n / div_sig_n) + 1 == (duo_sig_n + 1) / div_sig_n. + // + // When 1 is added to the denominator of `duo_sig_n / div_sig_n` to produce + // `duo_sig_n / (div_sig_n + 1)`, it is not possible that the value decreases by + // more than 1 with the bounds (8) and (9). Consider `x/y - 1 <= x/(y + 1)` <=> + // `(x - y)/y < x/(y + 1)` <=> `(y + 1)*(x - y) < x*y` <=> `x*y - y*y + x - y < x*y` + // <=> `x < y*y + y`. The smallest value of `div_sig_n` is `2^n_h` and the largest + // value of `duo_sig_n` is `2^n - 1`. Substituting reveals `2^n - 1 < 2^n + 2^n_h`. + // Thus, + // 11. `duo_sig_n / div_sig_n == duo_sig_n / (div_sig_n + 1)` or + // `(duo_sig_n / div_sig_n) - 1` == duo_sig_n / (div_sig_n + 1)` + // + // Combining both (10) and (11), we know that + // `quo - 1 <= duo_sig_n / (div_sig_n + 1) <= true_quo + // < (duo_sig_n + 1) / div_sig_n <= quo + 1` and therefore: + // 12. quo - 1 <= true_quo < quo + 1 + // + // In a lot of division algorithms using smaller divisions to construct a larger + // division, we often encounter a situation where the approximate `quo` value + // calculated from a smaller division is multiple increments away from the true + // `quo` value. In those algorithms, multiple correction steps have to be applied. + // Those correction steps may need more multiplications to test `duo - (quo*div)` + // again. Because of the fact that our `quo` can only be one of two values, we can + // see if `duo - (quo*div)` overflows. If it did overflow, then we know that we have + // the larger of the two values (since the true quotient is unique, and any larger + // quotient will cause `duo - (quo*div)` to be negative). Also because there is only + // one correction needed, we can calculate the remainder `duo - (true_quo*div) == + // duo - ((quo - 1)*div) == duo - (quo*div - div) == duo + div - quo*div`. + // If `duo - (quo*div)` did not overflow, then we have the correct answer. + let shift = n - duo_lz; + let duo_sig_n = (duo >> shift) as $uX; + let div_sig_n = (div >> shift) as $uX; + let quo = $half_division(duo_sig_n, div_sig_n).0; + + // The larger `quo` value can overflow `$uD` in the right circumstances. This is a + // manual `carrying_mul_add` with overflow checking. + let div_lo = div as $uX; + let div_hi = (div >> n) as $uX; + let (tmp_lo, carry) = carrying_mul(quo, div_lo); + let (tmp_hi, overflow) = carrying_mul_add(quo, div_hi, carry); + let tmp = (tmp_lo as $uD) | ((tmp_hi as $uD) << n); + if (overflow != 0) || (duo < tmp) { + return ( + (quo - 1) as $uD, + // Both the addition and subtraction can overflow, but when combined end up + // as a correct positive number. + duo.wrapping_add(div).wrapping_sub(tmp), + ); + } else { + return (quo as $uD, duo - tmp); + } + } + + // Undersubtracting long division algorithm. + // Instead of clearing a minimum of 1 bit from `duo` per iteration via binary long + // division, `n_h - 1` bits are cleared per iteration with this algorithm. It is a more + // complicated version of regular long division. Most integer division algorithms tend + // to guess a part of the quotient, and may have a larger quotient than the true + // quotient (which when multiplied by `div` will "oversubtract" the original dividend). + // They then check if the quotient was in fact too large and then have to correct it. + // This long division algorithm has been carefully constructed to always underguess the + // quotient by slim margins. This allows different subalgorithms to be blindly jumped to + // without needing an extra correction step. + // + // The only problem is that this subalgorithm will not work for many ranges of `duo` and + // `div`. Fortunately, the short division, two possibility algorithm, and other simple + // cases happen to exactly fill these gaps. + // + // For an example, consider the division of 76543210 by 213 and assume that `n_h` is + // equal to two decimal digits (note: we are working with base 10 here for readability). + // The first `sig_n_h` part of the divisor (21) is taken and is incremented by 1 to + // prevent oversubtraction. We also record the number of extra places not a part of + // the `sig_n` or `sig_n_h` parts. + // + // sig_n_h == 2 digits, sig_n == 4 digits + // + // vvvv <- `duo_sig_n` + // 76543210 + // ^^^^ <- extra places in duo, `duo_extra == 4` + // + // vv <- `div_sig_n_h` + // 213 + // ^ <- extra places in div, `div_extra == 1` + // + // The difference in extra places, `duo_extra - div_extra == extra_shl == 3`, is used + // for shifting partial sums in the long division. + // + // In the first step, the first `sig_n` part of duo (7654) is divided by + // `div_sig_n_h_add_1` (22), which results in a partial quotient of 347. This is + // multiplied by the whole divisor to make 73911, which is shifted left by `extra_shl` + // and subtracted from duo. The partial quotient is also shifted left by `extra_shl` to + // be added to `quo`. + // + // 347 + // ________ + // |76543210 + // -73911 + // 2632210 + // + // Variables dependent on duo have to be updated: + // + // vvvv <- `duo_sig_n == 2632` + // 2632210 + // ^^^ <- `duo_extra == 3` + // + // `extra_shl == 2` + // + // Two more steps are taken after this and then duo fits into `n` bits, and then a final + // normal long division step is made. The partial quotients are all progressively added + // to each other in the actual algorithm, but here I have left them all in a tower that + // can be added together to produce the quotient, 359357. + // + // 14 + // 443 + // 119 + // 347 + // ________ + // |76543210 + // -73911 + // 2632210 + // -25347 + // 97510 + // -94359 + // 3151 + // -2982 + // 169 <- the remainder + + let mut duo = duo; + let mut quo: $uD = 0; + + // The number of lesser significant bits not a part of `div_sig_n_h` + let div_extra = (n + $n_h) - div_lz; + + // The most significant `n_h` bits of div + let div_sig_n_h = (div >> div_extra) as $uH; + + // This needs to be a `$uX` in case of overflow from the increment + let div_sig_n_h_add1 = (div_sig_n_h as $uX) + 1; + + // `{2^n, 2^(div_sb + n_h)} <= duo < 2^n_d` + // `2^n_h <= div < {2^(duo_sb - n_h), 2^n}` + loop { + // The number of lesser significant bits not a part of `duo_sig_n` + let duo_extra = n - duo_lz; + + // The most significant `n` bits of `duo` + let duo_sig_n = (duo >> duo_extra) as $uX; + + // the two possibility algorithm requires that the difference between msbs is less + // than `n_h`, so the comparison is `<=` here. + if div_extra <= duo_extra { + // Undersubtracting long division step + let quo_part = $half_division(duo_sig_n, div_sig_n_h_add1).0 as $uD; + let extra_shl = duo_extra - div_extra; + + // Addition to the quotient. + quo += (quo_part << extra_shl); + + // Subtraction from `duo`. At least `n_h - 1` bits are cleared from `duo` here. + duo -= (div.wrapping_mul(quo_part) << extra_shl); + } else { + // Two possibility algorithm + let shift = n - duo_lz; + let duo_sig_n = (duo >> shift) as $uX; + let div_sig_n = (div >> shift) as $uX; + let quo_part = $half_division(duo_sig_n, div_sig_n).0; + let div_lo = div as $uX; + let div_hi = (div >> n) as $uX; + + let (tmp_lo, carry) = carrying_mul(quo_part, div_lo); + // The undersubtracting long division algorithm has already run once, so + // overflow beyond `$uD` bits is not possible here + let (tmp_hi, _) = carrying_mul_add(quo_part, div_hi, carry); + let tmp = (tmp_lo as $uD) | ((tmp_hi as $uD) << n); + + if duo < tmp { + return ( + quo + ((quo_part - 1) as $uD), + duo.wrapping_add(div).wrapping_sub(tmp), + ); + } else { + return (quo + (quo_part as $uD), duo - tmp); + } + } + + duo_lz = duo.leading_zeros(); + + if div_lz <= duo_lz { + // quotient can have 0 or 1 added to it + if div <= duo { + return (quo + 1, duo - div); + } else { + return (quo, duo); + } + } + + // This can only happen if `div_sd < n` (because of previous "quo = 0 or 1" + // branches), but it is not worth it to unroll further. + if n <= duo_lz { + // simple division and addition + let tmp = $half_division(duo as $uX, div as $uX); + return (quo + (tmp.0 as $uD), tmp.1 as $uD); + } + } + } + }; +} diff --git a/crux-mir/lib/compiler_builtins/src/int/udiv.rs b/crux-mir/lib/compiler_builtins/src/int/udiv.rs index b393ac6db..fb09f87d8 100644 --- a/crux-mir/lib/compiler_builtins/src/int/udiv.rs +++ b/crux-mir/lib/compiler_builtins/src/int/udiv.rs @@ -1,270 +1,106 @@ -use int::{Int, LargeInt}; +#[cfg(not(feature = "public-test-deps"))] +pub(crate) use int::specialized_div_rem::*; -macro_rules! udivmod_inner { - ($n:expr, $d:expr, $rem:expr, $ty:ty) => {{ - let (n, d, rem) = ($n, $d, $rem); - // NOTE X is unknown, K != 0 - if n.high() == 0 { - if d.high() == 0 { - // 0 X - // --- - // 0 X - - if let Some(rem) = rem { - *rem = <$ty>::from(n.low().aborting_rem(d.low())); - } - return <$ty>::from(n.low().aborting_div(d.low())) - } else { - // 0 X - // --- - // K X - if let Some(rem) = rem { - *rem = n; - } - return 0; - }; - } - - let mut sr; - let mut q; - let mut r; - - if d.low() == 0 { - if d.high() == 0 { - // K X - // --- - // 0 0 - // NOTE This should be unreachable in safe Rust because the program will panic before - // this intrinsic is called - ::abort(); - } - - if n.low() == 0 { - // K 0 - // --- - // K 0 - if let Some(rem) = rem { - *rem = <$ty>::from_parts(0, n.high().aborting_rem(d.high())); - } - return <$ty>::from(n.high().aborting_div(d.high())) - } - - // K K - // --- - // K 0 - - if d.high().is_power_of_two() { - if let Some(rem) = rem { - *rem = <$ty>::from_parts(n.low(), n.high() & (d.high() - 1)); - } - return <$ty>::from(n.high() >> d.high().trailing_zeros()); - } - - sr = d.high().leading_zeros().wrapping_sub(n.high().leading_zeros()); - - // D > N - if sr > ::BITS - 2 { - if let Some(rem) = rem { - *rem = n; - } - return 0; - } - - sr += 1; - - // 1 <= sr <= ::BITS - 1 - q = n << (<$ty>::BITS - sr); - r = n >> sr; - } else if d.high() == 0 { - // K X - // --- - // 0 K - if d.low().is_power_of_two() { - if let Some(rem) = rem { - *rem = <$ty>::from(n.low() & (d.low() - 1)); - } - - if d.low() == 1 { - return n; - } else { - let sr = d.low().trailing_zeros(); - return n >> sr; - }; - } - - sr = 1 + ::BITS + d.low().leading_zeros() - n.high().leading_zeros(); - - // 2 <= sr <= u64::BITS - 1 - q = n << (<$ty>::BITS - sr); - r = n >> sr; - } else { - // K X - // --- - // K K - sr = d.high().leading_zeros().wrapping_sub(n.high().leading_zeros()); - - // D > N - if sr > ::BITS - 1 { - if let Some(rem) = rem { - *rem = n; - } - return 0; - } - - sr += 1; - - // 1 <= sr <= ::BITS - q = n << (<$ty>::BITS - sr); - r = n >> sr; - } - - // Not a special case - // q and r are initialized with - // q = n << (u64::BITS - sr) - // r = n >> sr - // 1 <= sr <= u64::BITS - 1 - let mut carry = 0; - - // Don't use a range because they may generate references to memcpy in unoptimized code - let mut i = 0; - while i < sr { - i += 1; - - // r:q = ((r:q) << 1) | carry - r = (r << 1) | (q >> (<$ty>::BITS - 1)); - q = (q << 1) | carry as $ty; - - // carry = 0 - // if r >= d { - // r -= d; - // carry = 1; - // } - let s = (d.wrapping_sub(r).wrapping_sub(1)) as os_ty!($ty) >> (<$ty>::BITS - 1); - carry = (s & 1) as hty!($ty); - r -= d & s as $ty; - } - - if let Some(rem) = rem { - *rem = r; - } - (q << 1) | carry as $ty - }} -} +#[cfg(feature = "public-test-deps")] +pub use int::specialized_div_rem::*; intrinsics! { #[maybe_use_optimized_c_shim] #[arm_aeabi_alias = __aeabi_uidiv] /// Returns `n / d` pub extern "C" fn __udivsi3(n: u32, d: u32) -> u32 { - // Special cases - if d == 0 { - // NOTE This should be unreachable in safe Rust because the program will panic before - // this intrinsic is called - ::abort(); - } - - if n == 0 { - return 0; - } - - let mut sr = d.leading_zeros().wrapping_sub(n.leading_zeros()); - - // d > n - if sr > u32::BITS - 1 { - return 0; - } - - // d == 1 - if sr == u32::BITS - 1 { - return n; - } - - sr += 1; - - // 1 <= sr <= u32::BITS - 1 - let mut q = n << (u32::BITS - sr); - let mut r = n >> sr; - - let mut carry = 0; - - // Don't use a range because they may generate references to memcpy in unoptimized code - let mut i = 0; - while i < sr { - i += 1; - - // r:q = ((r:q) << 1) | carry - r = (r << 1) | (q >> (u32::BITS - 1)); - q = (q << 1) | carry; - - // carry = 0; - // if r > d { - // r -= d; - // carry = 1; - // } - - let s = (d.wrapping_sub(r).wrapping_sub(1)) as i32 >> (u32::BITS - 1); - carry = (s & 1) as u32; - r -= d & s as u32; - } - - (q << 1) | carry + u32_div_rem(n, d).0 } #[maybe_use_optimized_c_shim] /// Returns `n % d` pub extern "C" fn __umodsi3(n: u32, d: u32) -> u32 { - let q = __udivsi3(n, d); - n - q * d + u32_div_rem(n, d).1 } + #[avr_skip] #[maybe_use_optimized_c_shim] /// Returns `n / d` and sets `*rem = n % d` pub extern "C" fn __udivmodsi4(n: u32, d: u32, rem: Option<&mut u32>) -> u32 { - let q = __udivsi3(n, d); + let quo_rem = u32_div_rem(n, d); if let Some(rem) = rem { - *rem = n - (q * d); + *rem = quo_rem.1; } - q + quo_rem.0 } + #[avr_skip] #[maybe_use_optimized_c_shim] /// Returns `n / d` pub extern "C" fn __udivdi3(n: u64, d: u64) -> u64 { - __udivmoddi4(n, d, None) + u64_div_rem(n, d).0 } + #[avr_skip] #[maybe_use_optimized_c_shim] /// Returns `n % d` pub extern "C" fn __umoddi3(n: u64, d: u64) -> u64 { - let mut rem = 0; - __udivmoddi4(n, d, Some(&mut rem)); - rem + u64_div_rem(n, d).1 } + #[avr_skip] + #[maybe_use_optimized_c_shim] + /// Returns `n / d` and sets `*rem = n % d` + pub extern "C" fn __udivmoddi4(n: u64, d: u64, rem: Option<&mut u64>) -> u64 { + let quo_rem = u64_div_rem(n, d); + if let Some(rem) = rem { + *rem = quo_rem.1; + } + quo_rem.0 + } + + // Note: we use block configuration and not `if cfg!(...)`, because we need to entirely disable + // the existence of `u128_div_rem` to get 32-bit SPARC to compile, see `u128_divide_sparc` docs. + + #[avr_skip] #[win64_128bit_abi_hack] /// Returns `n / d` pub extern "C" fn __udivti3(n: u128, d: u128) -> u128 { - __udivmodti4(n, d, None) + #[cfg(not(any(target_arch = "sparc", target_arch = "sparc64")))] { + u128_div_rem(n, d).0 + } + #[cfg(any(target_arch = "sparc", target_arch = "sparc64"))] { + u128_divide_sparc(n, d, &mut 0) + } } + #[avr_skip] #[win64_128bit_abi_hack] /// Returns `n % d` pub extern "C" fn __umodti3(n: u128, d: u128) -> u128 { - let mut rem = 0; - __udivmodti4(n, d, Some(&mut rem)); - rem - } - - /// Returns `n / d` and sets `*rem = n % d` - pub extern "C" fn __udivmoddi4(n: u64, d: u64, rem: Option<&mut u64>) -> u64 { - udivmod_inner!(n, d, rem, u64) + #[cfg(not(any(target_arch = "sparc", target_arch = "sparc64")))] { + u128_div_rem(n, d).1 + } + #[cfg(any(target_arch = "sparc", target_arch = "sparc64"))] { + let mut rem = 0; + u128_divide_sparc(n, d, &mut rem); + rem + } } + #[avr_skip] #[win64_128bit_abi_hack] /// Returns `n / d` and sets `*rem = n % d` - pub extern "C" fn __udivmodti4(n: u128, - d: u128, - rem: Option<&mut u128>) -> u128 { - udivmod_inner!(n, d, rem, u128) + pub extern "C" fn __udivmodti4(n: u128, d: u128, rem: Option<&mut u128>) -> u128 { + #[cfg(not(any(target_arch = "sparc", target_arch = "sparc64")))] { + let quo_rem = u128_div_rem(n, d); + if let Some(rem) = rem { + *rem = quo_rem.1; + } + quo_rem.0 + } + #[cfg(any(target_arch = "sparc", target_arch = "sparc64"))] { + let mut tmp = 0; + let quo = u128_divide_sparc(n, d, &mut tmp); + if let Some(rem) = rem { + *rem = tmp; + } + quo + } } } diff --git a/crux-mir/lib/compiler_builtins/src/lib.rs b/crux-mir/lib/compiler_builtins/src/lib.rs index e57a5ef3f..10b4aafec 100644 --- a/crux-mir/lib/compiler_builtins/src/lib.rs +++ b/crux-mir/lib/compiler_builtins/src/lib.rs @@ -1,10 +1,12 @@ #![cfg_attr(feature = "compiler-builtins", compiler_builtins)] +#![cfg_attr(not(feature = "no-asm"), feature(asm))] #![feature(abi_unadjusted)] -#![feature(asm)] -#![feature(global_asm)] +#![cfg_attr(not(feature = "no-asm"), feature(global_asm))] #![feature(cfg_target_has_atomic)] #![feature(compiler_builtins)] +#![feature(core_ffi_c)] #![feature(core_intrinsics)] +#![feature(inline_const)] #![feature(lang_items)] #![feature(linkage)] #![feature(naked_functions)] @@ -15,7 +17,11 @@ // We use `u128` in a whole bunch of places which we currently agree with the // compiler on ABIs and such, so we should be "good enough" for now and changes // to the `u128` ABI will be reflected here. -#![allow(improper_ctypes)] +#![allow(improper_ctypes, improper_ctypes_definitions)] +// `mem::swap` cannot be used because it may generate references to memcpy in unoptimized code. +#![allow(clippy::manual_swap)] +// Support compiling on both stage0 and stage1 which may differ in supported stable features. +#![allow(stable_features)] // We disable #[no_mangle] for tests so that we can verify the test results // against the native compiler-rt implementations of the builtins. @@ -30,10 +36,6 @@ #[cfg(test)] extern crate core; -fn abort() -> ! { - unsafe { core::intrinsics::abort() } -} - #[macro_use] mod macros; @@ -41,8 +43,11 @@ pub mod float; pub mod int; #[cfg(any( - all(target_arch = "wasm32", target_os = "unknown"), + all(target_family = "wasm", target_os = "unknown"), + all(target_arch = "x86_64", target_os = "none"), + all(target_arch = "x86_64", target_os = "uefi"), all(target_arch = "arm", target_os = "none"), + target_os = "xous", all(target_vendor = "fortanix", target_env = "sgx") ))] pub mod math; @@ -51,11 +56,15 @@ pub mod mem; #[cfg(target_arch = "arm")] pub mod arm; -#[cfg(all(kernel_user_helpers, target_os = "linux", target_arch = "arm"))] +#[cfg(all( + kernel_user_helpers, + any(target_os = "linux", target_os = "android"), + target_arch = "arm" +))] pub mod arm_linux; -#[cfg(any(target_arch = "riscv32"))] -pub mod riscv32; +#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] +pub mod riscv; #[cfg(target_arch = "x86")] pub mod x86; diff --git a/crux-mir/lib/compiler_builtins/src/macros.rs b/crux-mir/lib/compiler_builtins/src/macros.rs index 2d11ba622..477c25684 100644 --- a/crux-mir/lib/compiler_builtins/src/macros.rs +++ b/crux-mir/lib/compiler_builtins/src/macros.rs @@ -1,5 +1,21 @@ //! Macros shared throughout the compiler-builtins implementation +/// Changes the visibility to `pub` if feature "public-test-deps" is set +#[cfg(not(feature = "public-test-deps"))] +macro_rules! public_test_dep { + ($(#[$($meta:meta)*])* pub(crate) $ident:ident $($tokens:tt)*) => { + $(#[$($meta)*])* pub(crate) $ident $($tokens)* + }; +} + +/// Changes the visibility to `pub` if feature "public-test-deps" is set +#[cfg(feature = "public-test-deps")] +macro_rules! public_test_dep { + {$(#[$($meta:meta)*])* pub(crate) $ident:ident $($tokens:tt)*} => { + $(#[$($meta)*])* pub $ident $($tokens)* + }; +} + /// The "main macro" used for defining intrinsics. /// /// The compiler-builtins library is super platform-specific with tons of crazy @@ -45,6 +61,35 @@ macro_rules! intrinsics { () => (); + // Support cfg_attr: + ( + #[cfg_attr($e:meta, $($attr:tt)*)] + $(#[$($attrs:tt)*])* + pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { + $($body:tt)* + } + $($rest:tt)* + ) => ( + #[cfg($e)] + intrinsics! { + #[$($attr)*] + $(#[$($attrs)*])* + pub extern $abi fn $name($($argname: $ty),*) $(-> $ret)? { + $($body)* + } + } + + #[cfg(not($e))] + intrinsics! { + $(#[$($attrs)*])* + pub extern $abi fn $name($($argname: $ty),*) $(-> $ret)? { + $($body)* + } + } + + intrinsics!($($rest)*); + ); + // Right now there's a bunch of architecture-optimized intrinsics in the // stock compiler-rt implementation. Not all of these have been ported over // to Rust yet so when the `c` feature of this crate is enabled we fall back @@ -60,17 +105,16 @@ macro_rules! intrinsics { ( #[maybe_use_optimized_c_shim] $(#[$($attr:tt)*])* - pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty { + pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { $($body:tt)* } $($rest:tt)* ) => ( - #[cfg($name = "optimized-c")] - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { extern $abi { - fn $name($($argname: $ty),*) -> $ret; + fn $name($($argname: $ty),*) $(-> $ret)?; } unsafe { $name($($argname),*) @@ -80,7 +124,7 @@ macro_rules! intrinsics { #[cfg(not($name = "optimized-c"))] intrinsics! { $(#[$($attr)*])* - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } } @@ -94,7 +138,7 @@ macro_rules! intrinsics { ( #[aapcs_on_arm] $(#[$($attr:tt)*])* - pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty { + pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { $($body:tt)* } @@ -103,7 +147,7 @@ macro_rules! intrinsics { #[cfg(target_arch = "arm")] intrinsics! { $(#[$($attr)*])* - pub extern "aapcs" fn $name( $($argname: $ty),* ) -> $ret { + pub extern "aapcs" fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } } @@ -111,7 +155,7 @@ macro_rules! intrinsics { #[cfg(not(target_arch = "arm"))] intrinsics! { $(#[$($attr)*])* - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } } @@ -124,24 +168,24 @@ macro_rules! intrinsics { ( #[unadjusted_on_win64] $(#[$($attr:tt)*])* - pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty { + pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { $($body:tt)* } $($rest:tt)* ) => ( - #[cfg(all(windows, target_pointer_width = "64"))] + #[cfg(all(any(windows, all(target_os = "uefi", target_arch = "x86_64")), target_pointer_width = "64"))] intrinsics! { $(#[$($attr)*])* - pub extern "unadjusted" fn $name( $($argname: $ty),* ) -> $ret { + pub extern "unadjusted" fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } } - #[cfg(not(all(windows, target_pointer_width = "64")))] + #[cfg(not(all(any(windows, all(target_os = "uefi", target_arch = "x86_64")), target_pointer_width = "64")))] intrinsics! { $(#[$($attr)*])* - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } } @@ -159,35 +203,33 @@ macro_rules! intrinsics { ( #[win64_128bit_abi_hack] $(#[$($attr:tt)*])* - pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty { + pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { $($body:tt)* } $($rest:tt)* ) => ( - #[cfg(all(windows, target_pointer_width = "64"))] + #[cfg(all(any(windows, target_os = "uefi"), target_arch = "x86_64"))] $(#[$($attr)*])* - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } - #[cfg(all(windows, target_pointer_width = "64"))] + #[cfg(all(any(windows, target_os = "uefi"), target_arch = "x86_64"))] pub mod $name { - - intrinsics! { - pub extern $abi fn $name( $($argname: $ty),* ) - -> ::macros::win64_128bit_abi_hack::U64x2 - { - let e: $ret = super::$name($($argname),*); - ::macros::win64_128bit_abi_hack::U64x2::from(e) - } + #[cfg_attr(not(feature = "mangled-names"), no_mangle)] + pub extern $abi fn $name( $($argname: $ty),* ) + -> ::macros::win64_128bit_abi_hack::U64x2 + { + let e: $($ret)? = super::$name($($argname),*); + ::macros::win64_128bit_abi_hack::U64x2::from(e) } } - #[cfg(not(all(windows, target_pointer_width = "64")))] + #[cfg(not(all(any(windows, target_os = "uefi"), target_arch = "x86_64")))] intrinsics! { $(#[$($attr)*])* - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } } @@ -202,31 +244,38 @@ macro_rules! intrinsics { ( #[arm_aeabi_alias = $alias:ident] $(#[$($attr:tt)*])* - pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty { + pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { $($body:tt)* } $($rest:tt)* ) => ( #[cfg(target_arch = "arm")] - #[cfg_attr(not(feature = "mangled-names"), no_mangle)] - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } #[cfg(target_arch = "arm")] pub mod $name { - intrinsics! { - pub extern "aapcs" fn $alias( $($argname: $ty),* ) -> $ret { - super::$name($($argname),*) - } + #[cfg_attr(not(feature = "mangled-names"), no_mangle)] + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { + super::$name($($argname),*) + } + } + + #[cfg(target_arch = "arm")] + pub mod $alias { + #[cfg_attr(not(feature = "mangled-names"), no_mangle)] + #[cfg_attr(all(not(windows), not(target_vendor="apple")), linkage = "weak")] + pub extern "aapcs" fn $alias( $($argname: $ty),* ) $(-> $ret)? { + super::$name($($argname),*) } } #[cfg(not(target_arch = "arm"))] intrinsics! { $(#[$($attr)*])* - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } } @@ -234,49 +283,167 @@ macro_rules! intrinsics { intrinsics!($($rest)*); ); - // This is the final catch-all rule. At this point we just generate an + // C mem* functions are only generated when the "mem" feature is enabled. + ( + #[mem_builtin] + $(#[$($attr:tt)*])* + pub unsafe extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { + $($body:tt)* + } + + $($rest:tt)* + ) => ( + $(#[$($attr)*])* + pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { + $($body)* + } + + #[cfg(feature = "mem")] + pub mod $name { + $(#[$($attr)*])* + #[cfg_attr(not(feature = "mangled-names"), no_mangle)] + pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { + super::$name($($argname),*) + } + } + + intrinsics!($($rest)*); + ); + + // Naked functions are special: we can't generate wrappers for them since + // they use a custom calling convention. + ( + #[naked] + $(#[$($attr:tt)*])* + pub unsafe extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { + $($body:tt)* + } + + $($rest:tt)* + ) => ( + pub mod $name { + #[naked] + $(#[$($attr)*])* + #[cfg_attr(not(feature = "mangled-names"), no_mangle)] + pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { + $($body)* + } + } + + intrinsics!($($rest)*); + ); + + // For division and modulo, AVR uses a custom calling convention¹ that does + // not match our definitions here. Ideally we would just use hand-written + // naked functions, but that's quite a lot of code to port² - so for the + // time being we are just ignoring the problematic functions, letting + // avr-gcc (which is required to compile to AVR anyway) link them from + // libgcc. + // + // ¹ https://gcc.gnu.org/wiki/avr-gcc (see "Exceptions to the Calling + // Convention") + // ² https://github.com/gcc-mirror/gcc/blob/31048012db98f5ec9c2ba537bfd850374bdd771f/libgcc/config/avr/lib1funcs.S + ( + #[avr_skip] + $(#[$($attr:tt)*])* + pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { + $($body:tt)* + } + + $($rest:tt)* + ) => ( + #[cfg(not(target_arch = "avr"))] + intrinsics! { + $(#[$($attr)*])* + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { + $($body)* + } + } + + intrinsics!($($rest)*); + ); + + // This is the final catch-all rule. At this point we generate an // intrinsic with a conditional `#[no_mangle]` directive to avoid - // interfereing with duplicate symbols and whatnot during testing. + // interfering with duplicate symbols and whatnot during testing. + // + // The implementation is placed in a separate module, to take advantage + // of the fact that rustc partitions functions into code generation + // units based on module they are defined in. As a result we will have + // a separate object file for each intrinsic. For further details see + // corresponding PR in rustc https://github.com/rust-lang/rust/pull/70846 // // After the intrinsic is defined we just continue with the rest of the // input we were given. ( $(#[$($attr:tt)*])* - pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty { + pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { + $($body:tt)* + } + + $($rest:tt)* + ) => ( + $(#[$($attr)*])* + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { + $($body)* + } + + pub mod $name { + $(#[$($attr)*])* + #[cfg_attr(not(feature = "mangled-names"), no_mangle)] + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { + super::$name($($argname),*) + } + } + + intrinsics!($($rest)*); + ); + + // Same as the above for unsafe functions. + ( + $(#[$($attr:tt)*])* + pub unsafe extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { $($body:tt)* } $($rest:tt)* ) => ( $(#[$($attr)*])* - #[cfg_attr(not(feature = "mangled-names"), no_mangle)] - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } + pub mod $name { + $(#[$($attr)*])* + #[cfg_attr(not(feature = "mangled-names"), no_mangle)] + pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { + super::$name($($argname),*) + } + } + intrinsics!($($rest)*); ); } // Hack for LLVM expectations for ABI on windows. This is used by the // `#[win64_128bit_abi_hack]` attribute recognized above -#[cfg(all(windows, target_pointer_width = "64"))] +#[cfg(all(any(windows, target_os = "uefi"), target_pointer_width = "64"))] pub mod win64_128bit_abi_hack { #[repr(simd)] pub struct U64x2(u64, u64); impl From for U64x2 { fn from(i: i128) -> U64x2 { - use int::LargeInt; + use int::DInt; let j = i as u128; - U64x2(j.low(), j.high()) + U64x2(j.lo(), j.hi()) } } impl From for U64x2 { fn from(i: u128) -> U64x2 { - use int::LargeInt; - U64x2(i.low(), i.high()) + use int::DInt; + U64x2(i.lo(), i.hi()) } } } diff --git a/crux-mir/lib/compiler_builtins/src/math.rs b/crux-mir/lib/compiler_builtins/src/math.rs index 4b27cb80f..c64984e9e 100644 --- a/crux-mir/lib/compiler_builtins/src/math.rs +++ b/crux-mir/lib/compiler_builtins/src/math.rs @@ -16,28 +16,23 @@ macro_rules! no_mangle { #[cfg(any( all( - target_arch = "wasm32", + target_family = "wasm", target_os = "unknown", not(target_env = "wasi") ), + target_os = "xous", + all(target_arch = "x86_64", target_os = "uefi"), + all(target_arch = "xtensa", target_os = "none"), all(target_vendor = "fortanix", target_env = "sgx") ))] no_mangle! { fn acos(x: f64) -> f64; fn asin(x: f64) -> f64; - fn atan(x: f64) -> f64; - fn atan2(x: f64, y: f64) -> f64; fn cbrt(x: f64) -> f64; - fn cosh(x: f64) -> f64; fn expm1(x: f64) -> f64; fn hypot(x: f64, y: f64) -> f64; - fn log1p(x: f64) -> f64; - fn sinh(x: f64) -> f64; fn tan(x: f64) -> f64; - fn tanh(x: f64) -> f64; fn cos(x: f64) -> f64; - fn cosf(x: f32) -> f32; - fn exp(x: f64) -> f64; fn expf(x: f32) -> f32; fn log2(x: f64) -> f64; fn log2f(x: f32) -> f32; @@ -51,35 +46,57 @@ no_mangle! { fn fmaxf(x: f32, y: f32) -> f32; fn round(x: f64) -> f64; fn roundf(x: f32) -> f32; + fn rint(x: f64) -> f64; + fn rintf(x: f32) -> f32; fn sin(x: f64) -> f64; - fn sinf(x: f32) -> f32; fn pow(x: f64, y: f64) -> f64; fn powf(x: f32, y: f32) -> f32; - fn exp2(x: f64) -> f64; - fn exp2f(x: f32) -> f32; fn fmod(x: f64, y: f64) -> f64; fn fmodf(x: f32, y: f32) -> f32; - fn fma(x: f64, y: f64, z: f64) -> f64; - fn fmaf(x: f32, y: f32, z: f32) -> f32; fn acosf(n: f32) -> f32; - fn asinf(n: f32) -> f32; fn atan2f(a: f32, b: f32) -> f32; fn atanf(n: f32) -> f32; - fn cbrtf(n: f32) -> f32; fn coshf(n: f32) -> f32; fn expm1f(n: f32) -> f32; fn fdim(a: f64, b: f64) -> f64; fn fdimf(a: f32, b: f32) -> f32; - fn hypotf(x: f32, y: f32) -> f32; fn log1pf(n: f32) -> f32; fn sinhf(n: f32) -> f32; - fn tanf(n: f32) -> f32; fn tanhf(n: f32) -> f32; fn ldexp(f: f64, n: i32) -> f64; fn ldexpf(f: f32, n: i32) -> f32; + fn tgamma(x: f64) -> f64; + fn tgammaf(x: f32) -> f32; + fn atan(x: f64) -> f64; + fn atan2(x: f64, y: f64) -> f64; + fn cosh(x: f64) -> f64; + fn log1p(x: f64) -> f64; + fn sinh(x: f64) -> f64; + fn tanh(x: f64) -> f64; + fn cosf(x: f32) -> f32; + fn exp(x: f64) -> f64; + fn sinf(x: f32) -> f32; + fn exp2(x: f64) -> f64; + fn exp2f(x: f32) -> f32; + fn fma(x: f64, y: f64, z: f64) -> f64; + fn fmaf(x: f32, y: f32, z: f32) -> f32; + fn asinf(n: f32) -> f32; + fn cbrtf(n: f32) -> f32; + fn hypotf(x: f32, y: f32) -> f32; + fn tanf(n: f32) -> f32; } -#[cfg(all(target_vendor = "fortanix", target_env = "sgx"))] +#[cfg(any(target_os = "xous", target_os = "uefi"))] +no_mangle! { + fn sqrtf(x: f32) -> f32; + fn sqrt(x: f64) -> f64; +} + +#[cfg(any( + all(target_vendor = "fortanix", target_env = "sgx"), + target_os = "xous", + target_os = "uefi" +))] no_mangle! { fn ceil(x: f64) -> f64; fn ceilf(x: f32) -> f32; @@ -89,9 +106,17 @@ no_mangle! { fn truncf(x: f32) -> f32; } -// only for the thumb*-none-eabi* targets -#[cfg(all(target_arch = "arm", target_os = "none"))] +// only for the thumb*-none-eabi*, riscv32*-none-elf and x86_64-unknown-none targets that lack the floating point instruction set +#[cfg(any( + all(target_arch = "arm", target_os = "none"), + all(target_arch = "riscv32", not(target_feature = "f"), target_os = "none"), + all(target_arch = "x86_64", target_os = "none") +))] no_mangle! { + fn fmin(x: f64, y: f64) -> f64; + fn fminf(x: f32, y: f32) -> f32; + fn fmax(x: f64, y: f64) -> f64; + fn fmaxf(x: f32, y: f32) -> f32; // `f64 % f64` fn fmod(x: f64, y: f64) -> f64; // `f32 % f32` diff --git a/crux-mir/lib/compiler_builtins/src/mem.rs b/crux-mir/lib/compiler_builtins/src/mem.rs deleted file mode 100644 index 24552ed85..000000000 --- a/crux-mir/lib/compiler_builtins/src/mem.rs +++ /dev/null @@ -1,194 +0,0 @@ -#[allow(warnings)] -#[cfg(target_pointer_width = "16")] -type c_int = i16; -#[allow(warnings)] -#[cfg(not(target_pointer_width = "16"))] -type c_int = i32; - -use core::intrinsics::{atomic_load_unordered, atomic_store_unordered, exact_div}; -use core::mem; -use core::ops::{BitOr, Shl}; - -#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)] -pub unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { - let mut i = 0; - while i < n { - *dest.offset(i as isize) = *src.offset(i as isize); - i += 1; - } - dest -} - -#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)] -pub unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { - if src < dest as *const u8 { - // copy from end - let mut i = n; - while i != 0 { - i -= 1; - *dest.offset(i as isize) = *src.offset(i as isize); - } - } else { - // copy from beginning - let mut i = 0; - while i < n { - *dest.offset(i as isize) = *src.offset(i as isize); - i += 1; - } - } - dest -} - -#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)] -pub unsafe extern "C" fn memset(s: *mut u8, c: c_int, n: usize) -> *mut u8 { - let mut i = 0; - while i < n { - *s.offset(i as isize) = c as u8; - i += 1; - } - s -} - -#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)] -pub unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { - let mut i = 0; - while i < n { - let a = *s1.offset(i as isize); - let b = *s2.offset(i as isize); - if a != b { - return a as i32 - b as i32; - } - i += 1; - } - 0 -} - -#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)] -pub unsafe extern "C" fn bcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { - memcmp(s1, s2, n) -} - -// `bytes` must be a multiple of `mem::size_of::()` -fn memcpy_element_unordered_atomic(dest: *mut T, src: *const T, bytes: usize) { - unsafe { - let n = exact_div(bytes, mem::size_of::()); - let mut i = 0; - while i < n { - atomic_store_unordered(dest.add(i), atomic_load_unordered(src.add(i))); - i += 1; - } - } -} - -// `bytes` must be a multiple of `mem::size_of::()` -fn memmove_element_unordered_atomic(dest: *mut T, src: *const T, bytes: usize) { - unsafe { - let n = exact_div(bytes, mem::size_of::()); - if src < dest as *const T { - // copy from end - let mut i = n; - while i != 0 { - i -= 1; - atomic_store_unordered(dest.add(i), atomic_load_unordered(src.add(i))); - } - } else { - // copy from beginning - let mut i = 0; - while i < n { - atomic_store_unordered(dest.add(i), atomic_load_unordered(src.add(i))); - i += 1; - } - } - } -} - -// `T` must be a primitive integer type, and `bytes` must be a multiple of `mem::size_of::()` -fn memset_element_unordered_atomic(s: *mut T, c: u8, bytes: usize) -where - T: Copy + From + Shl + BitOr, -{ - unsafe { - let n = exact_div(bytes, mem::size_of::()); - - // Construct a value of type `T` consisting of repeated `c` - // bytes, to let us ensure we write each `T` atomically. - let mut x = T::from(c); - let mut i = 1; - while i < mem::size_of::() { - x = x << 8 | T::from(c); - i += 1; - } - - // Write it to `s` - let mut i = 0; - while i < n { - atomic_store_unordered(s.add(i), x); - i += 1; - } - } -} - -intrinsics! { - #[cfg(target_has_atomic_load_store = "8")] - pub extern "C" fn __llvm_memcpy_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () { - memcpy_element_unordered_atomic(dest, src, bytes); - } - #[cfg(target_has_atomic_load_store = "16")] - pub extern "C" fn __llvm_memcpy_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () { - memcpy_element_unordered_atomic(dest, src, bytes); - } - #[cfg(target_has_atomic_load_store = "32")] - pub extern "C" fn __llvm_memcpy_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () { - memcpy_element_unordered_atomic(dest, src, bytes); - } - #[cfg(target_has_atomic_load_store = "64")] - pub extern "C" fn __llvm_memcpy_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () { - memcpy_element_unordered_atomic(dest, src, bytes); - } - #[cfg(target_has_atomic_load_store = "128")] - pub extern "C" fn __llvm_memcpy_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () { - memcpy_element_unordered_atomic(dest, src, bytes); - } - - #[cfg(target_has_atomic_load_store = "8")] - pub extern "C" fn __llvm_memmove_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () { - memmove_element_unordered_atomic(dest, src, bytes); - } - #[cfg(target_has_atomic_load_store = "16")] - pub extern "C" fn __llvm_memmove_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () { - memmove_element_unordered_atomic(dest, src, bytes); - } - #[cfg(target_has_atomic_load_store = "32")] - pub extern "C" fn __llvm_memmove_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () { - memmove_element_unordered_atomic(dest, src, bytes); - } - #[cfg(target_has_atomic_load_store = "64")] - pub extern "C" fn __llvm_memmove_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () { - memmove_element_unordered_atomic(dest, src, bytes); - } - #[cfg(target_has_atomic_load_store = "128")] - pub extern "C" fn __llvm_memmove_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () { - memmove_element_unordered_atomic(dest, src, bytes); - } - - #[cfg(target_has_atomic_load_store = "8")] - pub extern "C" fn __llvm_memset_element_unordered_atomic_1(s: *mut u8, c: u8, bytes: usize) -> () { - memset_element_unordered_atomic(s, c, bytes); - } - #[cfg(target_has_atomic_load_store = "16")] - pub extern "C" fn __llvm_memset_element_unordered_atomic_2(s: *mut u16, c: u8, bytes: usize) -> () { - memset_element_unordered_atomic(s, c, bytes); - } - #[cfg(target_has_atomic_load_store = "32")] - pub extern "C" fn __llvm_memset_element_unordered_atomic_4(s: *mut u32, c: u8, bytes: usize) -> () { - memset_element_unordered_atomic(s, c, bytes); - } - #[cfg(target_has_atomic_load_store = "64")] - pub extern "C" fn __llvm_memset_element_unordered_atomic_8(s: *mut u64, c: u8, bytes: usize) -> () { - memset_element_unordered_atomic(s, c, bytes); - } - #[cfg(target_has_atomic_load_store = "128")] - pub extern "C" fn __llvm_memset_element_unordered_atomic_16(s: *mut u128, c: u8, bytes: usize) -> () { - memset_element_unordered_atomic(s, c, bytes); - } -} diff --git a/crux-mir/lib/compiler_builtins/src/mem/impls.rs b/crux-mir/lib/compiler_builtins/src/mem/impls.rs new file mode 100644 index 000000000..72003a5c4 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/src/mem/impls.rs @@ -0,0 +1,281 @@ +use core::intrinsics::likely; + +const WORD_SIZE: usize = core::mem::size_of::(); +const WORD_MASK: usize = WORD_SIZE - 1; + +// If the number of bytes involved exceed this threshold we will opt in word-wise copy. +// The value here selected is max(2 * WORD_SIZE, 16): +// * We need at least 2 * WORD_SIZE bytes to guarantee that at least 1 word will be copied through +// word-wise copy. +// * The word-wise copy logic needs to perform some checks so it has some small overhead. +// ensures that even on 32-bit platforms we have copied at least 8 bytes through +// word-wise copy so the saving of word-wise copy outweights the fixed overhead. +const WORD_COPY_THRESHOLD: usize = if 2 * WORD_SIZE > 16 { + 2 * WORD_SIZE +} else { + 16 +}; + +#[cfg(feature = "mem-unaligned")] +unsafe fn read_usize_unaligned(x: *const usize) -> usize { + // Do not use `core::ptr::read_unaligned` here, since it calls `copy_nonoverlapping` which + // is translated to memcpy in LLVM. + let x_read = (x as *const [u8; core::mem::size_of::()]).read(); + core::mem::transmute(x_read) +} + +#[inline(always)] +pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize) { + #[inline(always)] + unsafe fn copy_forward_bytes(mut dest: *mut u8, mut src: *const u8, n: usize) { + let dest_end = dest.add(n); + while dest < dest_end { + *dest = *src; + dest = dest.add(1); + src = src.add(1); + } + } + + #[inline(always)] + unsafe fn copy_forward_aligned_words(dest: *mut u8, src: *const u8, n: usize) { + let mut dest_usize = dest as *mut usize; + let mut src_usize = src as *mut usize; + let dest_end = dest.add(n) as *mut usize; + + while dest_usize < dest_end { + *dest_usize = *src_usize; + dest_usize = dest_usize.add(1); + src_usize = src_usize.add(1); + } + } + + #[cfg(not(feature = "mem-unaligned"))] + #[inline(always)] + unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { + let mut dest_usize = dest as *mut usize; + let dest_end = dest.add(n) as *mut usize; + + // Calculate the misalignment offset and shift needed to reassemble value. + let offset = src as usize & WORD_MASK; + let shift = offset * 8; + + // Realign src + let mut src_aligned = (src as usize & !WORD_MASK) as *mut usize; + // This will read (but won't use) bytes out of bound. + // cfg needed because not all targets will have atomic loads that can be lowered + // (e.g. BPF, MSP430), or provided by an external library (e.g. RV32I) + #[cfg(target_has_atomic_load_store = "ptr")] + let mut prev_word = core::intrinsics::atomic_load_unordered(src_aligned); + #[cfg(not(target_has_atomic_load_store = "ptr"))] + let mut prev_word = core::ptr::read_volatile(src_aligned); + + while dest_usize < dest_end { + src_aligned = src_aligned.add(1); + let cur_word = *src_aligned; + #[cfg(target_endian = "little")] + let resembled = prev_word >> shift | cur_word << (WORD_SIZE * 8 - shift); + #[cfg(target_endian = "big")] + let resembled = prev_word << shift | cur_word >> (WORD_SIZE * 8 - shift); + prev_word = cur_word; + + *dest_usize = resembled; + dest_usize = dest_usize.add(1); + } + } + + #[cfg(feature = "mem-unaligned")] + #[inline(always)] + unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { + let mut dest_usize = dest as *mut usize; + let mut src_usize = src as *mut usize; + let dest_end = dest.add(n) as *mut usize; + + while dest_usize < dest_end { + *dest_usize = read_usize_unaligned(src_usize); + dest_usize = dest_usize.add(1); + src_usize = src_usize.add(1); + } + } + + if n >= WORD_COPY_THRESHOLD { + // Align dest + // Because of n >= 2 * WORD_SIZE, dst_misalignment < n + let dest_misalignment = (dest as usize).wrapping_neg() & WORD_MASK; + copy_forward_bytes(dest, src, dest_misalignment); + dest = dest.add(dest_misalignment); + src = src.add(dest_misalignment); + n -= dest_misalignment; + + let n_words = n & !WORD_MASK; + let src_misalignment = src as usize & WORD_MASK; + if likely(src_misalignment == 0) { + copy_forward_aligned_words(dest, src, n_words); + } else { + copy_forward_misaligned_words(dest, src, n_words); + } + dest = dest.add(n_words); + src = src.add(n_words); + n -= n_words; + } + copy_forward_bytes(dest, src, n); +} + +#[inline(always)] +pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) { + // The following backward copy helper functions uses the pointers past the end + // as their inputs instead of pointers to the start! + #[inline(always)] + unsafe fn copy_backward_bytes(mut dest: *mut u8, mut src: *const u8, n: usize) { + let dest_start = dest.sub(n); + while dest_start < dest { + dest = dest.sub(1); + src = src.sub(1); + *dest = *src; + } + } + + #[inline(always)] + unsafe fn copy_backward_aligned_words(dest: *mut u8, src: *const u8, n: usize) { + let mut dest_usize = dest as *mut usize; + let mut src_usize = src as *mut usize; + let dest_start = dest.sub(n) as *mut usize; + + while dest_start < dest_usize { + dest_usize = dest_usize.sub(1); + src_usize = src_usize.sub(1); + *dest_usize = *src_usize; + } + } + + #[cfg(not(feature = "mem-unaligned"))] + #[inline(always)] + unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { + let mut dest_usize = dest as *mut usize; + let dest_start = dest.sub(n) as *mut usize; + + // Calculate the misalignment offset and shift needed to reassemble value. + let offset = src as usize & WORD_MASK; + let shift = offset * 8; + + // Realign src_aligned + let mut src_aligned = (src as usize & !WORD_MASK) as *mut usize; + // This will read (but won't use) bytes out of bound. + // cfg needed because not all targets will have atomic loads that can be lowered + // (e.g. BPF, MSP430), or provided by an external library (e.g. RV32I) + #[cfg(target_has_atomic_load_store = "ptr")] + let mut prev_word = core::intrinsics::atomic_load_unordered(src_aligned); + #[cfg(not(target_has_atomic_load_store = "ptr"))] + let mut prev_word = core::ptr::read_volatile(src_aligned); + + while dest_start < dest_usize { + src_aligned = src_aligned.sub(1); + let cur_word = *src_aligned; + #[cfg(target_endian = "little")] + let resembled = prev_word << (WORD_SIZE * 8 - shift) | cur_word >> shift; + #[cfg(target_endian = "big")] + let resembled = prev_word >> (WORD_SIZE * 8 - shift) | cur_word << shift; + prev_word = cur_word; + + dest_usize = dest_usize.sub(1); + *dest_usize = resembled; + } + } + + #[cfg(feature = "mem-unaligned")] + #[inline(always)] + unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { + let mut dest_usize = dest as *mut usize; + let mut src_usize = src as *mut usize; + let dest_start = dest.sub(n) as *mut usize; + + while dest_start < dest_usize { + dest_usize = dest_usize.sub(1); + src_usize = src_usize.sub(1); + *dest_usize = read_usize_unaligned(src_usize); + } + } + + let mut dest = dest.add(n); + let mut src = src.add(n); + + if n >= WORD_COPY_THRESHOLD { + // Align dest + // Because of n >= 2 * WORD_SIZE, dst_misalignment < n + let dest_misalignment = dest as usize & WORD_MASK; + copy_backward_bytes(dest, src, dest_misalignment); + dest = dest.sub(dest_misalignment); + src = src.sub(dest_misalignment); + n -= dest_misalignment; + + let n_words = n & !WORD_MASK; + let src_misalignment = src as usize & WORD_MASK; + if likely(src_misalignment == 0) { + copy_backward_aligned_words(dest, src, n_words); + } else { + copy_backward_misaligned_words(dest, src, n_words); + } + dest = dest.sub(n_words); + src = src.sub(n_words); + n -= n_words; + } + copy_backward_bytes(dest, src, n); +} + +#[inline(always)] +pub unsafe fn set_bytes(mut s: *mut u8, c: u8, mut n: usize) { + #[inline(always)] + pub unsafe fn set_bytes_bytes(mut s: *mut u8, c: u8, n: usize) { + let end = s.add(n); + while s < end { + *s = c; + s = s.add(1); + } + } + + #[inline(always)] + pub unsafe fn set_bytes_words(s: *mut u8, c: u8, n: usize) { + let mut broadcast = c as usize; + let mut bits = 8; + while bits < WORD_SIZE * 8 { + broadcast |= broadcast << bits; + bits *= 2; + } + + let mut s_usize = s as *mut usize; + let end = s.add(n) as *mut usize; + + while s_usize < end { + *s_usize = broadcast; + s_usize = s_usize.add(1); + } + } + + if likely(n >= WORD_COPY_THRESHOLD) { + // Align s + // Because of n >= 2 * WORD_SIZE, dst_misalignment < n + let misalignment = (s as usize).wrapping_neg() & WORD_MASK; + set_bytes_bytes(s, c, misalignment); + s = s.add(misalignment); + n -= misalignment; + + let n_words = n & !WORD_MASK; + set_bytes_words(s, c, n_words); + s = s.add(n_words); + n -= n_words; + } + set_bytes_bytes(s, c, n); +} + +#[inline(always)] +pub unsafe fn compare_bytes(s1: *const u8, s2: *const u8, n: usize) -> i32 { + let mut i = 0; + while i < n { + let a = *s1.add(i); + let b = *s2.add(i); + if a != b { + return a as i32 - b as i32; + } + i += 1; + } + 0 +} diff --git a/crux-mir/lib/compiler_builtins/src/mem/mod.rs b/crux-mir/lib/compiler_builtins/src/mem/mod.rs new file mode 100644 index 000000000..c5b0ddc16 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/src/mem/mod.rs @@ -0,0 +1,202 @@ +// Trying to satisfy clippy here is hopeless +#![allow(clippy::style)] + +#[allow(warnings)] +#[cfg(target_pointer_width = "16")] +type c_int = i16; +#[allow(warnings)] +#[cfg(not(target_pointer_width = "16"))] +type c_int = i32; + +use core::intrinsics::{atomic_load_unordered, atomic_store_unordered, exact_div}; +use core::mem; +use core::ops::{BitOr, Shl}; + +// memcpy/memmove/memset have optimized implementations on some architectures +#[cfg_attr( + all(not(feature = "no-asm"), target_arch = "x86_64"), + path = "x86_64.rs" +)] +mod impls; + +intrinsics! { + #[mem_builtin] + #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] + pub unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { + impls::copy_forward(dest, src, n); + dest + } + + #[mem_builtin] + #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] + pub unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { + let delta = (dest as usize).wrapping_sub(src as usize); + if delta >= n { + // We can copy forwards because either dest is far enough ahead of src, + // or src is ahead of dest (and delta overflowed). + impls::copy_forward(dest, src, n); + } else { + impls::copy_backward(dest, src, n); + } + dest + } + + #[mem_builtin] + #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] + pub unsafe extern "C" fn memset(s: *mut u8, c: crate::mem::c_int, n: usize) -> *mut u8 { + impls::set_bytes(s, c as u8, n); + s + } + + #[mem_builtin] + #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] + pub unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { + impls::compare_bytes(s1, s2, n) + } + + #[mem_builtin] + #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] + pub unsafe extern "C" fn bcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { + memcmp(s1, s2, n) + } + + #[mem_builtin] + #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] + pub unsafe extern "C" fn strlen(s: *const core::ffi::c_char) -> usize { + let mut n = 0; + let mut s = s; + while *s != 0 { + n += 1; + s = s.offset(1); + } + n + } +} + +// `bytes` must be a multiple of `mem::size_of::()` +#[cfg_attr(not(target_has_atomic_load_store = "8"), allow(dead_code))] +fn memcpy_element_unordered_atomic(dest: *mut T, src: *const T, bytes: usize) { + unsafe { + let n = exact_div(bytes, mem::size_of::()); + let mut i = 0; + while i < n { + atomic_store_unordered(dest.add(i), atomic_load_unordered(src.add(i))); + i += 1; + } + } +} + +// `bytes` must be a multiple of `mem::size_of::()` +#[cfg_attr(not(target_has_atomic_load_store = "8"), allow(dead_code))] +fn memmove_element_unordered_atomic(dest: *mut T, src: *const T, bytes: usize) { + unsafe { + let n = exact_div(bytes, mem::size_of::()); + if src < dest as *const T { + // copy from end + let mut i = n; + while i != 0 { + i -= 1; + atomic_store_unordered(dest.add(i), atomic_load_unordered(src.add(i))); + } + } else { + // copy from beginning + let mut i = 0; + while i < n { + atomic_store_unordered(dest.add(i), atomic_load_unordered(src.add(i))); + i += 1; + } + } + } +} + +// `T` must be a primitive integer type, and `bytes` must be a multiple of `mem::size_of::()` +#[cfg_attr(not(target_has_atomic_load_store = "8"), allow(dead_code))] +fn memset_element_unordered_atomic(s: *mut T, c: u8, bytes: usize) +where + T: Copy + From + Shl + BitOr, +{ + unsafe { + let n = exact_div(bytes, mem::size_of::()); + + // Construct a value of type `T` consisting of repeated `c` + // bytes, to let us ensure we write each `T` atomically. + let mut x = T::from(c); + let mut i = 1; + while i < mem::size_of::() { + x = x << 8 | T::from(c); + i += 1; + } + + // Write it to `s` + let mut i = 0; + while i < n { + atomic_store_unordered(s.add(i), x); + i += 1; + } + } +} + +intrinsics! { + #[cfg(target_has_atomic_load_store = "8")] + pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () { + memcpy_element_unordered_atomic(dest, src, bytes); + } + #[cfg(target_has_atomic_load_store = "16")] + pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () { + memcpy_element_unordered_atomic(dest, src, bytes); + } + #[cfg(target_has_atomic_load_store = "32")] + pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () { + memcpy_element_unordered_atomic(dest, src, bytes); + } + #[cfg(target_has_atomic_load_store = "64")] + pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () { + memcpy_element_unordered_atomic(dest, src, bytes); + } + #[cfg(target_has_atomic_load_store = "128")] + pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () { + memcpy_element_unordered_atomic(dest, src, bytes); + } + + #[cfg(target_has_atomic_load_store = "8")] + pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () { + memmove_element_unordered_atomic(dest, src, bytes); + } + #[cfg(target_has_atomic_load_store = "16")] + pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () { + memmove_element_unordered_atomic(dest, src, bytes); + } + #[cfg(target_has_atomic_load_store = "32")] + pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () { + memmove_element_unordered_atomic(dest, src, bytes); + } + #[cfg(target_has_atomic_load_store = "64")] + pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () { + memmove_element_unordered_atomic(dest, src, bytes); + } + #[cfg(target_has_atomic_load_store = "128")] + pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () { + memmove_element_unordered_atomic(dest, src, bytes); + } + + #[cfg(target_has_atomic_load_store = "8")] + pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_1(s: *mut u8, c: u8, bytes: usize) -> () { + memset_element_unordered_atomic(s, c, bytes); + } + #[cfg(target_has_atomic_load_store = "16")] + pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_2(s: *mut u16, c: u8, bytes: usize) -> () { + memset_element_unordered_atomic(s, c, bytes); + } + #[cfg(target_has_atomic_load_store = "32")] + pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_4(s: *mut u32, c: u8, bytes: usize) -> () { + memset_element_unordered_atomic(s, c, bytes); + } + #[cfg(target_has_atomic_load_store = "64")] + pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_8(s: *mut u64, c: u8, bytes: usize) -> () { + memset_element_unordered_atomic(s, c, bytes); + } + #[cfg(target_has_atomic_load_store = "128")] + pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_16(s: *mut u128, c: u8, bytes: usize) -> () { + memset_element_unordered_atomic(s, c, bytes); + } +} diff --git a/crux-mir/lib/compiler_builtins/src/mem/x86_64.rs b/crux-mir/lib/compiler_builtins/src/mem/x86_64.rs new file mode 100644 index 000000000..17b461f79 --- /dev/null +++ b/crux-mir/lib/compiler_builtins/src/mem/x86_64.rs @@ -0,0 +1,184 @@ +// On most modern Intel and AMD processors, "rep movsq" and "rep stosq" have +// been enhanced to perform better than an simple qword loop, making them ideal +// for implementing memcpy/memset. Note that "rep cmps" has received no such +// enhancement, so it is not used to implement memcmp. +// +// On certain recent Intel processors, "rep movsb" and "rep stosb" have been +// further enhanced to automatically select the best microarchitectural +// implementation based on length and alignment. See the following features from +// the "Intel® 64 and IA-32 Architectures Optimization Reference Manual": +// - ERMSB - Enhanced REP MOVSB and STOSB (Ivy Bridge and later) +// - FSRM - Fast Short REP MOV (Ice Lake and later) +// - Fast Zero-Length MOVSB (On no current hardware) +// - Fast Short STOSB (On no current hardware) +// +// To simplify things, we switch to using the byte-based variants if the "ermsb" +// feature is present at compile-time. We don't bother detecting other features. +// Note that ERMSB does not enhance the backwards (DF=1) "rep movsb". + +use core::arch::asm; +use core::intrinsics; +use core::mem; + +#[inline(always)] +#[cfg(target_feature = "ermsb")] +pub unsafe fn copy_forward(dest: *mut u8, src: *const u8, count: usize) { + // FIXME: Use the Intel syntax once we drop LLVM 9 support on rust-lang/rust. + core::arch::asm!( + "repe movsb (%rsi), (%rdi)", + inout("rcx") count => _, + inout("rdi") dest => _, + inout("rsi") src => _, + options(att_syntax, nostack, preserves_flags) + ); +} + +#[inline(always)] +#[cfg(not(target_feature = "ermsb"))] +pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, count: usize) { + let (pre_byte_count, qword_count, byte_count) = rep_param(dest, count); + // Separating the blocks gives the compiler more freedom to reorder instructions. + asm!( + "rep movsb", + inout("ecx") pre_byte_count => _, + inout("rdi") dest => dest, + inout("rsi") src => src, + options(att_syntax, nostack, preserves_flags) + ); + asm!( + "rep movsq", + inout("rcx") qword_count => _, + inout("rdi") dest => dest, + inout("rsi") src => src, + options(att_syntax, nostack, preserves_flags) + ); + asm!( + "rep movsb", + inout("ecx") byte_count => _, + inout("rdi") dest => _, + inout("rsi") src => _, + options(att_syntax, nostack, preserves_flags) + ); +} + +#[inline(always)] +pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, count: usize) { + let (pre_byte_count, qword_count, byte_count) = rep_param(dest, count); + // We can't separate this block due to std/cld + asm!( + "std", + "rep movsb", + "sub $7, %rsi", + "sub $7, %rdi", + "mov {qword_count}, %rcx", + "rep movsq", + "test {pre_byte_count:e}, {pre_byte_count:e}", + "add $7, %rsi", + "add $7, %rdi", + "mov {pre_byte_count:e}, %ecx", + "rep movsb", + "cld", + pre_byte_count = in(reg) pre_byte_count, + qword_count = in(reg) qword_count, + inout("ecx") byte_count => _, + inout("rdi") dest.add(count - 1) => _, + inout("rsi") src.add(count - 1) => _, + // We modify flags, but we restore it afterwards + options(att_syntax, nostack, preserves_flags) + ); +} + +#[inline(always)] +#[cfg(target_feature = "ermsb")] +pub unsafe fn set_bytes(dest: *mut u8, c: u8, count: usize) { + // FIXME: Use the Intel syntax once we drop LLVM 9 support on rust-lang/rust. + core::arch::asm!( + "repe stosb %al, (%rdi)", + inout("rcx") count => _, + inout("rdi") dest => _, + inout("al") c => _, + options(att_syntax, nostack, preserves_flags) + ) +} + +#[inline(always)] +#[cfg(not(target_feature = "ermsb"))] +pub unsafe fn set_bytes(mut dest: *mut u8, c: u8, count: usize) { + let c = c as u64 * 0x0101_0101_0101_0101; + let (pre_byte_count, qword_count, byte_count) = rep_param(dest, count); + // Separating the blocks gives the compiler more freedom to reorder instructions. + asm!( + "rep stosb", + inout("ecx") pre_byte_count => _, + inout("rdi") dest => dest, + in("rax") c, + options(att_syntax, nostack, preserves_flags) + ); + asm!( + "rep stosq", + inout("rcx") qword_count => _, + inout("rdi") dest => dest, + in("rax") c, + options(att_syntax, nostack, preserves_flags) + ); + asm!( + "rep stosb", + inout("ecx") byte_count => _, + inout("rdi") dest => _, + in("rax") c, + options(att_syntax, nostack, preserves_flags) + ); +} + +#[inline(always)] +pub unsafe fn compare_bytes(a: *const u8, b: *const u8, n: usize) -> i32 { + #[inline(always)] + unsafe fn cmp(mut a: *const T, mut b: *const T, n: usize, f: F) -> i32 + where + T: Clone + Copy + Eq, + U: Clone + Copy + Eq, + F: FnOnce(*const U, *const U, usize) -> i32, + { + // Ensure T is not a ZST. + const { assert!(mem::size_of::() != 0) }; + + let end = a.add(intrinsics::unchecked_div(n, mem::size_of::())); + while a != end { + if a.read_unaligned() != b.read_unaligned() { + return f(a.cast(), b.cast(), mem::size_of::()); + } + a = a.add(1); + b = b.add(1); + } + f( + a.cast(), + b.cast(), + intrinsics::unchecked_rem(n, mem::size_of::()), + ) + } + let c1 = |mut a: *const u8, mut b: *const u8, n| { + for _ in 0..n { + if a.read() != b.read() { + return i32::from(a.read()) - i32::from(b.read()); + } + a = a.add(1); + b = b.add(1); + } + 0 + }; + let c2 = |a: *const u16, b, n| cmp(a, b, n, c1); + let c4 = |a: *const u32, b, n| cmp(a, b, n, c2); + let c8 = |a: *const u64, b, n| cmp(a, b, n, c4); + let c16 = |a: *const u128, b, n| cmp(a, b, n, c8); + c16(a.cast(), b.cast(), n) +} + +/// Determine optimal parameters for a `rep` instruction. +fn rep_param(dest: *mut u8, mut count: usize) -> (usize, usize, usize) { + // Unaligned writes are still slow on modern processors, so align the destination address. + let pre_byte_count = ((8 - (dest as usize & 0b111)) & 0b111).min(count); + count -= pre_byte_count; + let qword_count = count >> 3; + let byte_count = count & 0b111; + (pre_byte_count, qword_count, byte_count) +} diff --git a/crux-mir/lib/compiler_builtins/src/probestack.rs b/crux-mir/lib/compiler_builtins/src/probestack.rs index 933a60dd9..0c30384db 100644 --- a/crux-mir/lib/compiler_builtins/src/probestack.rs +++ b/crux-mir/lib/compiler_builtins/src/probestack.rs @@ -44,6 +44,8 @@ #![cfg(not(feature = "mangled-names"))] // Windows already has builtins to do this. #![cfg(not(windows))] +// All these builtins require assembly +#![cfg(not(feature = "no-asm"))] // We only define stack probing for these architectures today. #![cfg(any(target_arch = "x86_64", target_arch = "x86"))] @@ -56,7 +58,7 @@ extern "C" { // emitted for the function. // // This is the ELF version. -#[cfg(not(target_vendor = "apple"))] +#[cfg(not(any(target_vendor = "apple", target_os = "uefi")))] macro_rules! define_rust_probestack { ($body: expr) => { concat!( @@ -64,6 +66,7 @@ macro_rules! define_rust_probestack { .pushsection .text.__rust_probestack .globl __rust_probestack .type __rust_probestack, @function + .hidden __rust_probestack __rust_probestack: ", $body, @@ -75,7 +78,21 @@ macro_rules! define_rust_probestack { }; } -// Same as above, but for Mach-O. +#[cfg(all(target_os = "uefi", target_arch = "x86_64"))] +macro_rules! define_rust_probestack { + ($body: expr) => { + concat!( + " + .globl __rust_probestack + __rust_probestack: + ", + $body + ) + }; +} + +// Same as above, but for Mach-O. Note that the triple underscore +// is deliberate #[cfg(target_vendor = "apple")] macro_rules! define_rust_probestack { ($body: expr) => { @@ -89,14 +106,34 @@ macro_rules! define_rust_probestack { }; } +// In UEFI x86 arch, triple underscore is deliberate. +#[cfg(all(target_os = "uefi", target_arch = "x86"))] +macro_rules! define_rust_probestack { + ($body: expr) => { + concat!( + " + .globl ___rust_probestack + ___rust_probestack: + ", + $body + ) + }; +} + // Our goal here is to touch each page between %rsp+8 and %rsp+8-%rax, // ensuring that if any pages are unmapped we'll make a page fault. // // The ABI here is that the stack frame size is located in `%rax`. Upon // return we're not supposed to modify `%rsp` or `%rax`. -#[cfg(target_arch = "x86_64")] -global_asm!(define_rust_probestack!( - " +// +// Any changes to this function should be replicated to the SGX version below. +#[cfg(all( + target_arch = "x86_64", + not(all(target_env = "sgx", target_vendor = "fortanix")) +))] +core::arch::global_asm!( + define_rust_probestack!( + " .cfi_startproc pushq %rbp .cfi_adjust_cfa_offset 8 @@ -146,16 +183,85 @@ global_asm!(define_rust_probestack!( ret .cfi_endproc " -)); + ), + options(att_syntax) +); -#[cfg(target_arch = "x86")] +// This function is the same as above, except that some instructions are +// [manually patched for LVI]. +// +// [manually patched for LVI]: https://software.intel.com/security-software-guidance/insights/deep-dive-load-value-injection#specialinstructions +#[cfg(all( + target_arch = "x86_64", + all(target_env = "sgx", target_vendor = "fortanix") +))] +core::arch::global_asm!( + define_rust_probestack!( + " + .cfi_startproc + pushq %rbp + .cfi_adjust_cfa_offset 8 + .cfi_offset %rbp, -16 + movq %rsp, %rbp + .cfi_def_cfa_register %rbp + + mov %rax,%r11 // duplicate %rax as we're clobbering %r11 + + // Main loop, taken in one page increments. We're decrementing rsp by + // a page each time until there's less than a page remaining. We're + // guaranteed that this function isn't called unless there's more than a + // page needed. + // + // Note that we're also testing against `8(%rsp)` to account for the 8 + // bytes pushed on the stack orginally with our return address. Using + // `8(%rsp)` simulates us testing the stack pointer in the caller's + // context. + + // It's usually called when %rax >= 0x1000, but that's not always true. + // Dynamic stack allocation, which is needed to implement unsized + // rvalues, triggers stackprobe even if %rax < 0x1000. + // Thus we have to check %r11 first to avoid segfault. + cmp $0x1000,%r11 + jna 3f +2: + sub $0x1000,%rsp + test %rsp,8(%rsp) + sub $0x1000,%r11 + cmp $0x1000,%r11 + ja 2b + +3: + // Finish up the last remaining stack space requested, getting the last + // bits out of r11 + sub %r11,%rsp + test %rsp,8(%rsp) + + // Restore the stack pointer to what it previously was when entering + // this function. The caller will readjust the stack pointer after we + // return. + add %rax,%rsp + + leave + .cfi_def_cfa_register %rsp + .cfi_adjust_cfa_offset -8 + pop %r11 + lfence + jmp *%r11 + .cfi_endproc + " + ), + options(att_syntax) +); + +#[cfg(all(target_arch = "x86", not(target_os = "uefi")))] // This is the same as x86_64 above, only translated for 32-bit sizes. Note // that on Unix we're expected to restore everything as it was, this // function basically can't tamper with anything. // // The ABI here is the same as x86_64, except everything is 32-bits large. -global_asm!(define_rust_probestack!( - " +core::arch::global_asm!( + define_rust_probestack!( + " .cfi_startproc push %ebp .cfi_adjust_cfa_offset 4 @@ -186,4 +292,59 @@ global_asm!(define_rust_probestack!( ret .cfi_endproc " -)); + ), + options(att_syntax) +); + +#[cfg(all(target_arch = "x86", target_os = "uefi"))] +// UEFI target is windows like target. LLVM will do _chkstk things like windows. +// probestack function will also do things like _chkstk in MSVC. +// So we need to sub %ax %sp in probestack when arch is x86. +// +// REF: Rust commit(74e80468347) +// rust\src\llvm-project\llvm\lib\Target\X86\X86FrameLowering.cpp: 805 +// Comments in LLVM: +// MSVC x32's _chkstk and cygwin/mingw's _alloca adjust %esp themselves. +// MSVC x64's __chkstk and cygwin/mingw's ___chkstk_ms do not adjust %rsp +// themselves. +core::arch::global_asm!( + define_rust_probestack!( + " + .cfi_startproc + push %ebp + .cfi_adjust_cfa_offset 4 + .cfi_offset %ebp, -8 + mov %esp, %ebp + .cfi_def_cfa_register %ebp + push %ecx + push %edx + mov %eax,%ecx + + cmp $0x1000,%ecx + jna 3f +2: + sub $0x1000,%esp + test %esp,8(%esp) + sub $0x1000,%ecx + cmp $0x1000,%ecx + ja 2b + +3: + sub %ecx,%esp + test %esp,8(%esp) + mov 4(%ebp),%edx + mov %edx, 12(%esp) + add %eax,%esp + pop %edx + pop %ecx + leave + + sub %eax, %esp + .cfi_def_cfa_register %esp + .cfi_adjust_cfa_offset -4 + ret + .cfi_endproc + " + ), + options(att_syntax) +); diff --git a/crux-mir/lib/compiler_builtins/src/riscv.rs b/crux-mir/lib/compiler_builtins/src/riscv.rs new file mode 100644 index 000000000..ae361b33a --- /dev/null +++ b/crux-mir/lib/compiler_builtins/src/riscv.rs @@ -0,0 +1,50 @@ +intrinsics! { + // Ancient Egyptian/Ethiopian/Russian multiplication method + // see https://en.wikipedia.org/wiki/Ancient_Egyptian_multiplication + // + // This is a long-available stock algorithm; e.g. it is documented in + // Knuth's "The Art of Computer Programming" volume 2 (under the section + // "Evaluation of Powers") since at least the 2nd edition (1981). + // + // The main attraction of this method is that it implements (software) + // multiplication atop four simple operations: doubling, halving, checking + // if a value is even/odd, and addition. This is *not* considered to be the + // fastest multiplication method, but it may be amongst the simplest (and + // smallest with respect to code size). + // + // for reference, see also implementation from gcc + // https://raw.githubusercontent.com/gcc-mirror/gcc/master/libgcc/config/epiphany/mulsi3.c + // + // and from LLVM (in relatively readable RISC-V assembly): + // https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/riscv/int_mul_impl.inc + pub extern "C" fn __mulsi3(a: u32, b: u32) -> u32 { + let (mut a, mut b) = (a, b); + let mut r = 0; + + while a > 0 { + if a & 1 > 0 { + r += b; + } + a >>= 1; + b <<= 1; + } + + r + } + + #[cfg(not(target_feature = "m"))] + pub extern "C" fn __muldi3(a: u64, b: u64) -> u64 { + let (mut a, mut b) = (a, b); + let mut r = 0; + + while a > 0 { + if a & 1 > 0 { + r += b; + } + a >>= 1; + b <<= 1; + } + + r + } +} diff --git a/crux-mir/lib/compiler_builtins/src/riscv32.rs b/crux-mir/lib/compiler_builtins/src/riscv32.rs deleted file mode 100644 index 9a3c1714c..000000000 --- a/crux-mir/lib/compiler_builtins/src/riscv32.rs +++ /dev/null @@ -1,18 +0,0 @@ -intrinsics! { - // Implementation from gcc - // https://raw.githubusercontent.com/gcc-mirror/gcc/master/libgcc/config/epiphany/mulsi3.c - pub extern "C" fn __mulsi3(a: u32, b: u32) -> u32 { - let (mut a, mut b) = (a, b); - let mut r = 0; - - while a > 0 { - if a & 1 > 0 { - r += b; - } - a >>= 1; - b <<= 1; - } - - r - } -} diff --git a/crux-mir/lib/compiler_builtins/src/x86.rs b/crux-mir/lib/compiler_builtins/src/x86.rs index 035c0a31c..fd1f32e3a 100644 --- a/crux-mir/lib/compiler_builtins/src/x86.rs +++ b/crux-mir/lib/compiler_builtins/src/x86.rs @@ -8,65 +8,78 @@ use core::intrinsics; // NOTE These functions are never mangled as they are not tested against compiler-rt // and mangling ___chkstk would break the `jmp ___chkstk` instruction in __alloca -#[cfg(all(windows, target_env = "gnu", not(feature = "mangled-names")))] -#[naked] -#[no_mangle] -pub unsafe fn ___chkstk_ms() { - asm!(" - push %ecx - push %eax - cmp $$0x1000,%eax - lea 12(%esp),%ecx - jb 1f - 2: - sub $$0x1000,%ecx - test %ecx,(%ecx) - sub $$0x1000,%eax - cmp $$0x1000,%eax - ja 2b - 1: - sub %eax,%ecx - test %ecx,(%ecx) - pop %eax - pop %ecx - ret" ::: "memory" : "volatile"); - intrinsics::unreachable(); -} - -// FIXME: __alloca should be an alias to __chkstk -#[cfg(all(windows, target_env = "gnu", not(feature = "mangled-names")))] -#[naked] -#[no_mangle] -pub unsafe fn __alloca() { - asm!("jmp ___chkstk // Jump to ___chkstk since fallthrough may be unreliable" - ::: "memory" : "volatile"); - intrinsics::unreachable(); -} +intrinsics! { + #[naked] + #[cfg(all( + windows, + target_env = "gnu", + not(feature = "no-asm") + ))] + pub unsafe extern "C" fn ___chkstk_ms() { + core::arch::asm!( + "push %ecx", + "push %eax", + "cmp $0x1000,%eax", + "lea 12(%esp),%ecx", + "jb 1f", + "2:", + "sub $0x1000,%ecx", + "test %ecx,(%ecx)", + "sub $0x1000,%eax", + "cmp $0x1000,%eax", + "ja 2b", + "1:", + "sub %eax,%ecx", + "test %ecx,(%ecx)", + "pop %eax", + "pop %ecx", + "ret", + options(noreturn, att_syntax) + ); + } -#[cfg(all(windows, target_env = "gnu", not(feature = "mangled-names")))] -#[naked] -#[no_mangle] -pub unsafe fn ___chkstk() { - asm!(" - push %ecx - cmp $$0x1000,%eax - lea 8(%esp),%ecx // esp before calling this routine -> ecx - jb 1f - 2: - sub $$0x1000,%ecx - test %ecx,(%ecx) - sub $$0x1000,%eax - cmp $$0x1000,%eax - ja 2b - 1: - sub %eax,%ecx - test %ecx,(%ecx) + // FIXME: __alloca should be an alias to __chkstk + #[naked] + #[cfg(all( + windows, + target_env = "gnu", + not(feature = "no-asm") + ))] + pub unsafe extern "C" fn __alloca() { + core::arch::asm!( + "jmp ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable" + options(noreturn, att_syntax) + ); + } - lea 4(%esp),%eax // load pointer to the return address into eax - mov %ecx,%esp // install the new top of stack pointer into esp - mov -4(%eax),%ecx // restore ecx - push (%eax) // push return address onto the stack - sub %esp,%eax // restore the original value in eax - ret" ::: "memory" : "volatile"); - intrinsics::unreachable(); + #[naked] + #[cfg(all( + windows, + target_env = "gnu", + not(feature = "no-asm") + ))] + pub unsafe extern "C" fn ___chkstk() { + core::arch::asm!( + "push %ecx", + "cmp $0x1000,%eax", + "lea 8(%esp),%ecx", // esp before calling this routine -> ecx + "jb 1f", + "2:", + "sub $0x1000,%ecx", + "test %ecx,(%ecx)", + "sub $0x1000,%eax", + "cmp $0x1000,%eax", + "ja 2b", + "1:", + "sub %eax,%ecx", + "test %ecx,(%ecx)", + "lea 4(%esp),%eax", // load pointer to the return address into eax + "mov %ecx,%esp", // install the new top of stack pointer into esp + "mov -4(%eax),%ecx", // restore ecx + "push (%eax)", // push return address onto the stack + "sub %esp,%eax", // restore the original value in eax + "ret", + options(noreturn, att_syntax) + ); + } } diff --git a/crux-mir/lib/compiler_builtins/src/x86_64.rs b/crux-mir/lib/compiler_builtins/src/x86_64.rs index 6940f8d9d..393eeddd8 100644 --- a/crux-mir/lib/compiler_builtins/src/x86_64.rs +++ b/crux-mir/lib/compiler_builtins/src/x86_64.rs @@ -8,75 +8,87 @@ use core::intrinsics; // NOTE These functions are never mangled as they are not tested against compiler-rt // and mangling ___chkstk would break the `jmp ___chkstk` instruction in __alloca -#[cfg(all(windows, target_env = "gnu", not(feature = "mangled-names")))] -#[naked] -#[no_mangle] -pub unsafe fn ___chkstk_ms() { - asm!(" - push %rcx - push %rax - cmp $$0x1000,%rax - lea 24(%rsp),%rcx - jb 1f - 2: - sub $$0x1000,%rcx - test %rcx,(%rcx) - sub $$0x1000,%rax - cmp $$0x1000,%rax - ja 2b - 1: - sub %rax,%rcx - test %rcx,(%rcx) - pop %rax - pop %rcx - ret" ::: "memory" : "volatile"); - intrinsics::unreachable(); -} - -#[cfg(all(windows, target_env = "gnu", not(feature = "mangled-names")))] -#[naked] -#[no_mangle] -pub unsafe fn __alloca() { - asm!("mov %rcx,%rax // x64 _alloca is a normal function with parameter in rcx - jmp ___chkstk // Jump to ___chkstk since fallthrough may be unreliable" - ::: "memory" : "volatile"); - intrinsics::unreachable(); -} +intrinsics! { + #[naked] + #[cfg(all( + windows, + target_env = "gnu", + not(feature = "no-asm") + ))] + pub unsafe extern "C" fn ___chkstk_ms() { + core::arch::asm!( + "push %rcx", + "push %rax", + "cmp $0x1000,%rax", + "lea 24(%rsp),%rcx", + "jb 1f", + "2:", + "sub $0x1000,%rcx", + "test %rcx,(%rcx)", + "sub $0x1000,%rax", + "cmp $0x1000,%rax", + "ja 2b", + "1:", + "sub %rax,%rcx", + "test %rcx,(%rcx)", + "pop %rax", + "pop %rcx", + "ret", + options(noreturn, att_syntax) + ); + } -#[cfg(all(windows, target_env = "gnu", not(feature = "mangled-names")))] -#[naked] -#[no_mangle] -pub unsafe fn ___chkstk() { - asm!( - " - push %rcx - cmp $$0x1000,%rax - lea 16(%rsp),%rcx // rsp before calling this routine -> rcx - jb 1f - 2: - sub $$0x1000,%rcx - test %rcx,(%rcx) - sub $$0x1000,%rax - cmp $$0x1000,%rax - ja 2b - 1: - sub %rax,%rcx - test %rcx,(%rcx) + #[naked] + #[cfg(all( + windows, + target_env = "gnu", + not(feature = "no-asm") + ))] + pub unsafe extern "C" fn __alloca() { + core::arch::asm!( + "mov %rcx,%rax", // x64 _alloca is a normal function with parameter in rcx + "jmp ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable" + options(noreturn, att_syntax) + ); + } - lea 8(%rsp),%rax // load pointer to the return address into rax - mov %rcx,%rsp // install the new top of stack pointer into rsp - mov -8(%rax),%rcx // restore rcx - push (%rax) // push return address onto the stack - sub %rsp,%rax // restore the original value in rax - ret" - ::: "memory" : "volatile" - ); - intrinsics::unreachable(); + #[naked] + #[cfg(all( + windows, + target_env = "gnu", + not(feature = "no-asm") + ))] + pub unsafe extern "C" fn ___chkstk() { + core::arch::asm!( + "push %rcx", + "cmp $0x1000,%rax", + "lea 16(%rsp),%rcx", // rsp before calling this routine -> rcx + "jb 1f", + "2:", + "sub $0x1000,%rcx", + "test %rcx,(%rcx)", + "sub $0x1000,%rax", + "cmp $0x1000,%rax", + "ja 2b", + "1:", + "sub %rax,%rcx", + "test %rcx,(%rcx)", + "lea 8(%rsp),%rax", // load pointer to the return address into rax + "mov %rcx,%rsp", // install the new top of stack pointer into rsp + "mov -8(%rax),%rcx", // restore rcx + "push (%rax)", // push return address onto the stack + "sub %rsp,%rax", // restore the original value in rax + "ret", + options(noreturn, att_syntax) + ); + } } // HACK(https://github.com/rust-lang/rust/issues/62785): x86_64-unknown-uefi needs special LLVM // support unless we emit the _fltused -#[no_mangle] -#[used] -#[cfg(target_os = "uefi")] -static _fltused: i32 = 0; +mod _fltused { + #[no_mangle] + #[used] + #[cfg(target_os = "uefi")] + static _fltused: i32 = 0; +} diff --git a/crux-mir/lib/compiler_builtins/test.c b/crux-mir/lib/compiler_builtins/test.c deleted file mode 100644 index 5ba8952af..000000000 --- a/crux-mir/lib/compiler_builtins/test.c +++ /dev/null @@ -1,11 +0,0 @@ -#include -int loop(int n) { - int value = n * n; - printf("%d ^2 = %d\n", n, value); - return loop(n + 1); -} - -int main() { - loop(0); - return 0; -} diff --git a/crux-mir/lib/compiler_builtins/testcrate/Cargo.toml b/crux-mir/lib/compiler_builtins/testcrate/Cargo.toml deleted file mode 100644 index 3b99b574e..000000000 --- a/crux-mir/lib/compiler_builtins/testcrate/Cargo.toml +++ /dev/null @@ -1,28 +0,0 @@ -[package] -name = "testcrate" -version = "0.1.0" -authors = ["Alex Crichton "] -edition = "2018" - -[lib] -test = false -doctest = false - -[build-dependencies] -rand = "0.7" - -[dependencies.compiler_builtins] -path = ".." -default-features = false -features = ["no-lang-items"] - -[target.'cfg(all(target_arch = "arm", not(any(target_env = "gnu", target_env = "musl")), target_os = "linux"))'.dev-dependencies] -test = { git = "https://github.com/japaric/utest" } -utest-cortex-m-qemu = { default-features = false, git = "https://github.com/japaric/utest" } -utest-macros = { git = "https://github.com/japaric/utest" } - -[features] -c = ["compiler_builtins/c"] -mem = ["compiler_builtins/mem"] -mangled-names = ["compiler_builtins/mangled-names"] -default = ["mangled-names"] diff --git a/crux-mir/lib/compiler_builtins/testcrate/build.rs b/crux-mir/lib/compiler_builtins/testcrate/build.rs deleted file mode 100644 index e1d4cf9e8..000000000 --- a/crux-mir/lib/compiler_builtins/testcrate/build.rs +++ /dev/null @@ -1,1420 +0,0 @@ -use rand::seq::SliceRandom; -use rand::Rng; -use std::collections::HashMap; -use std::fmt; -use std::fmt::Write as FmtWrite; -use std::fs::{self, OpenOptions}; -use std::hash::{Hash, Hasher}; -use std::io::Write; -use std::path::PathBuf; -use std::{env, mem}; - -const NTESTS: usize = 1_000; - -fn main() { - let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()); - let out_file = out_dir.join("generated.rs"); - drop(fs::remove_file(&out_file)); - - let target = env::var("TARGET").unwrap(); - let target_arch_arm = target.contains("arm") || target.contains("thumb"); - let target_arch_mips = target.contains("mips"); - - // TODO accept NaNs. We don't do that right now because we can't check - // for NaN-ness on the thumb targets (due to missing intrinsics) - - // float/add.rs - gen( - |(a, b): (MyF64, MyF64)| { - let c = a.0 + b.0; - if a.0.is_nan() || b.0.is_nan() || c.is_nan() { - None - } else { - Some(c) - } - }, - "builtins::float::add::__adddf3(a, b)", - ); - gen( - |(a, b): (MyF32, MyF32)| { - let c = a.0 + b.0; - if a.0.is_nan() || b.0.is_nan() || c.is_nan() { - None - } else { - Some(c) - } - }, - "builtins::float::add::__addsf3(a, b)", - ); - - if target_arch_arm { - gen( - |(a, b): (MyF64, MyF64)| { - let c = a.0 + b.0; - if a.0.is_nan() || b.0.is_nan() || c.is_nan() { - None - } else { - Some(c) - } - }, - "builtins::float::add::__adddf3vfp(a, b)", - ); - gen( - |(a, b): (LargeF32, LargeF32)| { - let c = a.0 + b.0; - if a.0.is_nan() || b.0.is_nan() || c.is_nan() { - None - } else { - Some(c) - } - }, - "builtins::float::add::__addsf3vfp(a, b)", - ); - } - - // float/cmp.rs - gen( - |(a, b): (MyF64, MyF64)| { - let (a, b) = (a.0, b.0); - if a.is_nan() || b.is_nan() { - return None; - } - - if a.is_nan() || b.is_nan() { - Some(-1) - } else if a < b { - Some(-1) - } else if a > b { - Some(1) - } else { - Some(0) - } - }, - "builtins::float::cmp::__gedf2(a, b)", - ); - gen( - |(a, b): (MyF32, MyF32)| { - let (a, b) = (a.0, b.0); - if a.is_nan() || b.is_nan() { - return None; - } - - if a.is_nan() || b.is_nan() { - Some(-1) - } else if a < b { - Some(-1) - } else if a > b { - Some(1) - } else { - Some(0) - } - }, - "builtins::float::cmp::__gesf2(a, b)", - ); - gen( - |(a, b): (MyF64, MyF64)| { - let (a, b) = (a.0, b.0); - if a.is_nan() || b.is_nan() { - return None; - } - - if a.is_nan() || b.is_nan() { - Some(1) - } else if a < b { - Some(-1) - } else if a > b { - Some(1) - } else { - Some(0) - } - }, - "builtins::float::cmp::__ledf2(a, b)", - ); - gen( - |(a, b): (MyF32, MyF32)| { - let (a, b) = (a.0, b.0); - if a.is_nan() || b.is_nan() { - return None; - } - - if a.is_nan() || b.is_nan() { - Some(1) - } else if a < b { - Some(-1) - } else if a > b { - Some(1) - } else { - Some(0) - } - }, - "builtins::float::cmp::__lesf2(a, b)", - ); - - gen( - |(a, b): (MyF32, MyF32)| { - let c = a.0.is_nan() || b.0.is_nan(); - Some(c as i32) - }, - "builtins::float::cmp::__unordsf2(a, b)", - ); - - gen( - |(a, b): (MyF64, MyF64)| { - let c = a.0.is_nan() || b.0.is_nan(); - Some(c as i32) - }, - "builtins::float::cmp::__unorddf2(a, b)", - ); - - if target_arch_arm { - gen( - |(a, b): (MyF32, MyF32)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - let c = (a.0 <= b.0) as i32; - Some(c) - }, - "builtins::float::cmp::__aeabi_fcmple(a, b)", - ); - - gen( - |(a, b): (MyF32, MyF32)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - let c = (a.0 >= b.0) as i32; - Some(c) - }, - "builtins::float::cmp::__aeabi_fcmpge(a, b)", - ); - - gen( - |(a, b): (MyF32, MyF32)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - let c = (a.0 == b.0) as i32; - Some(c) - }, - "builtins::float::cmp::__aeabi_fcmpeq(a, b)", - ); - - gen( - |(a, b): (MyF32, MyF32)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - let c = (a.0 < b.0) as i32; - Some(c) - }, - "builtins::float::cmp::__aeabi_fcmplt(a, b)", - ); - - gen( - |(a, b): (MyF32, MyF32)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - let c = (a.0 > b.0) as i32; - Some(c) - }, - "builtins::float::cmp::__aeabi_fcmpgt(a, b)", - ); - - gen( - |(a, b): (MyF64, MyF64)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - let c = (a.0 <= b.0) as i32; - Some(c) - }, - "builtins::float::cmp::__aeabi_dcmple(a, b)", - ); - - gen( - |(a, b): (MyF64, MyF64)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - let c = (a.0 >= b.0) as i32; - Some(c) - }, - "builtins::float::cmp::__aeabi_dcmpge(a, b)", - ); - - gen( - |(a, b): (MyF64, MyF64)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - let c = (a.0 == b.0) as i32; - Some(c) - }, - "builtins::float::cmp::__aeabi_dcmpeq(a, b)", - ); - - gen( - |(a, b): (MyF64, MyF64)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - let c = (a.0 < b.0) as i32; - Some(c) - }, - "builtins::float::cmp::__aeabi_dcmplt(a, b)", - ); - - gen( - |(a, b): (MyF64, MyF64)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - let c = (a.0 > b.0) as i32; - Some(c) - }, - "builtins::float::cmp::__aeabi_dcmpgt(a, b)", - ); - - gen( - |(a, b): (LargeF32, LargeF32)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - Some((a.0 >= b.0) as i32) - }, - "builtins::float::cmp::__gesf2vfp(a, b)", - ); - gen( - |(a, b): (MyF64, MyF64)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - Some((a.0 >= b.0) as i32) - }, - "builtins::float::cmp::__gedf2vfp(a, b)", - ); - gen( - |(a, b): (LargeF32, LargeF32)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - Some((a.0 > b.0) as i32) - }, - "builtins::float::cmp::__gtsf2vfp(a, b)", - ); - gen( - |(a, b): (MyF64, MyF64)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - Some((a.0 > b.0) as i32) - }, - "builtins::float::cmp::__gtdf2vfp(a, b)", - ); - gen( - |(a, b): (LargeF32, LargeF32)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - Some((a.0 < b.0) as i32) - }, - "builtins::float::cmp::__ltsf2vfp(a, b)", - ); - gen( - |(a, b): (MyF64, MyF64)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - Some((a.0 < b.0) as i32) - }, - "builtins::float::cmp::__ltdf2vfp(a, b)", - ); - gen( - |(a, b): (LargeF32, LargeF32)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - Some((a.0 <= b.0) as i32) - }, - "builtins::float::cmp::__lesf2vfp(a, b)", - ); - gen( - |(a, b): (MyF64, MyF64)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - Some((a.0 <= b.0) as i32) - }, - "builtins::float::cmp::__ledf2vfp(a, b)", - ); - gen( - |(a, b): (LargeF32, LargeF32)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - Some((a.0 != b.0) as i32) - }, - "builtins::float::cmp::__nesf2vfp(a, b)", - ); - gen( - |(a, b): (MyF64, MyF64)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - Some((a.0 != b.0) as i32) - }, - "builtins::float::cmp::__nedf2vfp(a, b)", - ); - gen( - |(a, b): (LargeF32, LargeF32)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - Some((a.0 == b.0) as i32) - }, - "builtins::float::cmp::__eqsf2vfp(a, b)", - ); - gen( - |(a, b): (MyF64, MyF64)| { - if a.0.is_nan() || b.0.is_nan() { - return None; - } - Some((a.0 == b.0) as i32) - }, - "builtins::float::cmp::__eqdf2vfp(a, b)", - ); - } - - // float/extend.rs - gen( - |a: MyF32| { - if a.0.is_nan() { - return None; - } - Some(f64::from(a.0)) - }, - "builtins::float::extend::__extendsfdf2(a)", - ); - if target_arch_arm { - gen( - |a: LargeF32| { - if a.0.is_nan() { - return None; - } - Some(f64::from(a.0)) - }, - "builtins::float::extend::__extendsfdf2vfp(a)", - ); - } - - // float/conv.rs - gen( - |a: MyF64| i64::cast(a.0), - "builtins::float::conv::__fixdfdi(a)", - ); - gen( - |a: MyF64| i32::cast(a.0), - "builtins::float::conv::__fixdfsi(a)", - ); - gen( - |a: MyF32| i64::cast(a.0), - "builtins::float::conv::__fixsfdi(a)", - ); - gen( - |a: MyF32| i32::cast(a.0), - "builtins::float::conv::__fixsfsi(a)", - ); - gen( - |a: MyF32| i128::cast(a.0), - "builtins::float::conv::__fixsfti(a)", - ); - gen( - |a: MyF64| i128::cast(a.0), - "builtins::float::conv::__fixdfti(a)", - ); - gen( - |a: MyF64| u64::cast(a.0), - "builtins::float::conv::__fixunsdfdi(a)", - ); - gen( - |a: MyF64| u32::cast(a.0), - "builtins::float::conv::__fixunsdfsi(a)", - ); - gen( - |a: MyF32| u64::cast(a.0), - "builtins::float::conv::__fixunssfdi(a)", - ); - gen( - |a: MyF32| u32::cast(a.0), - "builtins::float::conv::__fixunssfsi(a)", - ); - gen( - |a: MyF32| u128::cast(a.0), - "builtins::float::conv::__fixunssfti(a)", - ); - gen( - |a: MyF64| u128::cast(a.0), - "builtins::float::conv::__fixunsdfti(a)", - ); - gen( - |a: MyI64| Some(a.0 as f64), - "builtins::float::conv::__floatdidf(a)", - ); - gen( - |a: MyI32| Some(a.0 as f64), - "builtins::float::conv::__floatsidf(a)", - ); - gen( - |a: MyI32| Some(a.0 as f32), - "builtins::float::conv::__floatsisf(a)", - ); - gen( - |a: MyU64| Some(a.0 as f64), - "builtins::float::conv::__floatundidf(a)", - ); - gen( - |a: MyU32| Some(a.0 as f64), - "builtins::float::conv::__floatunsidf(a)", - ); - gen( - |a: MyU32| Some(a.0 as f32), - "builtins::float::conv::__floatunsisf(a)", - ); - gen( - |a: MyU128| Some(a.0 as f32), - "builtins::float::conv::__floatuntisf(a)", - ); - if !target_arch_mips { - gen( - |a: MyI128| Some(a.0 as f32), - "builtins::float::conv::__floattisf(a)", - ); - gen( - |a: MyI128| Some(a.0 as f64), - "builtins::float::conv::__floattidf(a)", - ); - gen( - |a: MyU128| Some(a.0 as f64), - "builtins::float::conv::__floatuntidf(a)", - ); - } - - // float/pow.rs - gen( - |(a, b): (MyF64, MyI32)| { - let c = a.0.powi(b.0); - if a.0.is_nan() || c.is_nan() { - None - } else { - Some(c) - } - }, - "builtins::float::pow::__powidf2(a, b)", - ); - gen( - |(a, b): (MyF32, MyI32)| { - let c = a.0.powi(b.0); - if a.0.is_nan() || c.is_nan() { - None - } else { - Some(c) - } - }, - "builtins::float::pow::__powisf2(a, b)", - ); - - // float/sub.rs - gen( - |(a, b): (MyF64, MyF64)| { - let c = a.0 - b.0; - if a.0.is_nan() || b.0.is_nan() || c.is_nan() { - None - } else { - Some(c) - } - }, - "builtins::float::sub::__subdf3(a, b)", - ); - gen( - |(a, b): (MyF32, MyF32)| { - let c = a.0 - b.0; - if a.0.is_nan() || b.0.is_nan() || c.is_nan() { - None - } else { - Some(c) - } - }, - "builtins::float::sub::__subsf3(a, b)", - ); - - if target_arch_arm { - gen( - |(a, b): (MyF64, MyF64)| { - let c = a.0 - b.0; - if a.0.is_nan() || b.0.is_nan() || c.is_nan() { - None - } else { - Some(c) - } - }, - "builtins::float::sub::__subdf3vfp(a, b)", - ); - gen( - |(a, b): (LargeF32, LargeF32)| { - let c = a.0 - b.0; - if a.0.is_nan() || b.0.is_nan() || c.is_nan() { - None - } else { - Some(c) - } - }, - "builtins::float::sub::__subsf3vfp(a, b)", - ); - } - - // float/mul.rs - gen( - |(a, b): (MyF64, MyF64)| { - let c = a.0 * b.0; - if a.0.is_nan() || b.0.is_nan() || c.is_nan() { - None - } else { - Some(c) - } - }, - "builtins::float::mul::__muldf3(a, b)", - ); - gen( - |(a, b): (LargeF32, LargeF32)| { - let c = a.0 * b.0; - if a.0.is_nan() || b.0.is_nan() || c.is_nan() { - None - } else { - Some(c) - } - }, - "builtins::float::mul::__mulsf3(a, b)", - ); - - if target_arch_arm { - gen( - |(a, b): (MyF64, MyF64)| { - let c = a.0 * b.0; - if a.0.is_nan() || b.0.is_nan() || c.is_nan() { - None - } else { - Some(c) - } - }, - "builtins::float::mul::__muldf3vfp(a, b)", - ); - gen( - |(a, b): (LargeF32, LargeF32)| { - let c = a.0 * b.0; - if a.0.is_nan() || b.0.is_nan() || c.is_nan() { - None - } else { - Some(c) - } - }, - "builtins::float::mul::__mulsf3vfp(a, b)", - ); - } - - // float/div.rs - gen( - |(a, b): (MyF64, MyF64)| { - if b.0 == 0.0 { - return None; - } - let c = a.0 / b.0; - if a.0.is_nan() - || b.0.is_nan() - || c.is_nan() - || c.abs() <= unsafe { mem::transmute(4503599627370495u64) } - { - None - } else { - Some(c) - } - }, - "builtins::float::div::__divdf3(a, b)", - ); - gen( - |(a, b): (LargeF32, LargeF32)| { - if b.0 == 0.0 { - return None; - } - let c = a.0 / b.0; - if a.0.is_nan() - || b.0.is_nan() - || c.is_nan() - || c.abs() <= unsafe { mem::transmute(16777215u32) } - { - None - } else { - Some(c) - } - }, - "builtins::float::div::__divsf3(a, b)", - ); - - if target_arch_arm { - gen( - |(a, b): (MyF64, MyF64)| { - if b.0 == 0.0 { - return None; - } - let c = a.0 / b.0; - if a.0.is_nan() - || b.0.is_nan() - || c.is_nan() - || c.abs() <= unsafe { mem::transmute(4503599627370495u64) } - { - None - } else { - Some(c) - } - }, - "builtins::float::div::__divdf3vfp(a, b)", - ); - gen( - |(a, b): (LargeF32, LargeF32)| { - if b.0 == 0.0 { - return None; - } - let c = a.0 / b.0; - if a.0.is_nan() - || b.0.is_nan() - || c.is_nan() - || c.abs() <= unsafe { mem::transmute(16777215u32) } - { - None - } else { - Some(c) - } - }, - "builtins::float::div::__divsf3vfp(a, b)", - ); - } - - // int/addsub.rs - gen( - |(a, b): (MyU128, MyU128)| Some(a.0.wrapping_add(b.0)), - "builtins::int::addsub::__rust_u128_add(a, b)", - ); - gen( - |(a, b): (MyI128, MyI128)| Some(a.0.wrapping_add(b.0)), - "builtins::int::addsub::__rust_i128_add(a, b)", - ); - gen( - |(a, b): (MyU128, MyU128)| Some(a.0.overflowing_add(b.0)), - "builtins::int::addsub::__rust_u128_addo(a, b)", - ); - gen( - |(a, b): (MyI128, MyI128)| Some(a.0.overflowing_add(b.0)), - "builtins::int::addsub::__rust_i128_addo(a, b)", - ); - gen( - |(a, b): (MyU128, MyU128)| Some(a.0.wrapping_sub(b.0)), - "builtins::int::addsub::__rust_u128_sub(a, b)", - ); - gen( - |(a, b): (MyI128, MyI128)| Some(a.0.wrapping_sub(b.0)), - "builtins::int::addsub::__rust_i128_sub(a, b)", - ); - gen( - |(a, b): (MyU128, MyU128)| Some(a.0.overflowing_sub(b.0)), - "builtins::int::addsub::__rust_u128_subo(a, b)", - ); - gen( - |(a, b): (MyI128, MyI128)| Some(a.0.overflowing_sub(b.0)), - "builtins::int::addsub::__rust_i128_subo(a, b)", - ); - - // int/mul.rs - gen( - |(a, b): (MyU64, MyU64)| Some(a.0.wrapping_mul(b.0)), - "builtins::int::mul::__muldi3(a, b)", - ); - gen( - |(a, b): (MyI64, MyI64)| Some(a.0.overflowing_mul(b.0)), - "{ - let mut o = 2; - let c = builtins::int::mul::__mulodi4(a, b, &mut o); - (c, match o { 0 => false, 1 => true, _ => panic!() }) - }", - ); - gen( - |(a, b): (MyI32, MyI32)| Some(a.0.overflowing_mul(b.0)), - "{ - let mut o = 2; - let c = builtins::int::mul::__mulosi4(a, b, &mut o); - (c, match o { 0 => false, 1 => true, _ => panic!() }) - }", - ); - gen( - |(a, b): (MyI128, MyI128)| Some(a.0.wrapping_mul(b.0)), - "builtins::int::mul::__multi3(a, b)", - ); - gen( - |(a, b): (MyI128, MyI128)| Some(a.0.overflowing_mul(b.0)), - "{ - let mut o = 2; - let c = builtins::int::mul::__muloti4(a, b, &mut o); - (c, match o { 0 => false, 1 => true, _ => panic!() }) - }", - ); - - // int/sdiv.rs - gen( - |(a, b): (MyI64, MyI64)| { - if b.0 == 0 { - None - } else { - Some(a.0 / b.0) - } - }, - "builtins::int::sdiv::__divdi3(a, b)", - ); - gen( - |(a, b): (MyI64, MyI64)| { - if b.0 == 0 { - None - } else { - Some((a.0 / b.0, a.0 % b.0)) - } - }, - "{ - let mut r = 0; - (builtins::int::sdiv::__divmoddi4(a, b, &mut r), r) - }", - ); - gen( - |(a, b): (MyI32, MyI32)| { - if b.0 == 0 { - None - } else { - Some((a.0 / b.0, a.0 % b.0)) - } - }, - "{ - let mut r = 0; - (builtins::int::sdiv::__divmodsi4(a, b, &mut r), r) - }", - ); - gen( - |(a, b): (MyI32, MyI32)| { - if b.0 == 0 { - None - } else { - Some(a.0 / b.0) - } - }, - "builtins::int::sdiv::__divsi3(a, b)", - ); - gen( - |(a, b): (MyI32, MyI32)| { - if b.0 == 0 { - None - } else { - Some(a.0 % b.0) - } - }, - "builtins::int::sdiv::__modsi3(a, b)", - ); - gen( - |(a, b): (MyI64, MyI64)| { - if b.0 == 0 { - None - } else { - Some(a.0 % b.0) - } - }, - "builtins::int::sdiv::__moddi3(a, b)", - ); - gen( - |(a, b): (MyI128, MyI128)| { - if b.0 == 0 { - None - } else { - Some(a.0 / b.0) - } - }, - "builtins::int::sdiv::__divti3(a, b)", - ); - gen( - |(a, b): (MyI128, MyI128)| { - if b.0 == 0 { - None - } else { - Some(a.0 % b.0) - } - }, - "builtins::int::sdiv::__modti3(a, b)", - ); - - // int/shift.rs - gen( - |(a, b): (MyU64, MyU32)| Some(a.0 << (b.0 % 64)), - "builtins::int::shift::__ashldi3(a, b % 64)", - ); - gen( - |(a, b): (MyU128, MyU32)| Some(a.0 << (b.0 % 128)), - "builtins::int::shift::__ashlti3(a, b % 128)", - ); - gen( - |(a, b): (MyI64, MyU32)| Some(a.0 >> (b.0 % 64)), - "builtins::int::shift::__ashrdi3(a, b % 64)", - ); - gen( - |(a, b): (MyI128, MyU32)| Some(a.0 >> (b.0 % 128)), - "builtins::int::shift::__ashrti3(a, b % 128)", - ); - gen( - |(a, b): (MyU64, MyU32)| Some(a.0 >> (b.0 % 64)), - "builtins::int::shift::__lshrdi3(a, b % 64)", - ); - gen( - |(a, b): (MyU128, MyU32)| Some(a.0 >> (b.0 % 128)), - "builtins::int::shift::__lshrti3(a, b % 128)", - ); - - // int/udiv.rs - gen( - |(a, b): (MyU64, MyU64)| { - if b.0 == 0 { - None - } else { - Some(a.0 / b.0) - } - }, - "builtins::int::udiv::__udivdi3(a, b)", - ); - gen( - |(a, b): (MyU64, MyU64)| { - if b.0 == 0 { - None - } else { - Some((a.0 / b.0, a.0 % b.0)) - } - }, - "{ - let mut r = 0; - (builtins::int::udiv::__udivmoddi4(a, b, Some(&mut r)), r) - }", - ); - gen( - |(a, b): (MyU32, MyU32)| { - if b.0 == 0 { - None - } else { - Some((a.0 / b.0, a.0 % b.0)) - } - }, - "{ - let mut r = 0; - (builtins::int::udiv::__udivmodsi4(a, b, Some(&mut r)), r) - }", - ); - gen( - |(a, b): (MyU32, MyU32)| { - if b.0 == 0 { - None - } else { - Some(a.0 / b.0) - } - }, - "builtins::int::udiv::__udivsi3(a, b)", - ); - gen( - |(a, b): (MyU32, MyU32)| { - if b.0 == 0 { - None - } else { - Some(a.0 % b.0) - } - }, - "builtins::int::udiv::__umodsi3(a, b)", - ); - gen( - |(a, b): (MyU64, MyU64)| { - if b.0 == 0 { - None - } else { - Some(a.0 % b.0) - } - }, - "builtins::int::udiv::__umoddi3(a, b)", - ); - gen( - |(a, b): (MyU128, MyU128)| { - if b.0 == 0 { - None - } else { - Some(a.0 / b.0) - } - }, - "builtins::int::udiv::__udivti3(a, b)", - ); - gen( - |(a, b): (MyU128, MyU128)| { - if b.0 == 0 { - None - } else { - Some(a.0 % b.0) - } - }, - "builtins::int::udiv::__umodti3(a, b)", - ); - gen( - |(a, b): (MyU128, MyU128)| { - if b.0 == 0 { - None - } else { - Some((a.0 / b.0, a.0 % b.0)) - } - }, - "{ - let mut r = 0; - (builtins::int::udiv::__udivmodti4(a, b, Some(&mut r)), r) - }", - ); -} - -macro_rules! gen_float { - ($name:ident, - $fty:ident, - $uty:ident, - $bits:expr, - $significand_bits:expr) => { - pub fn $name(rng: &mut R) -> $fty - where - R: Rng + ?Sized, - { - const BITS: u8 = $bits; - const SIGNIFICAND_BITS: u8 = $significand_bits; - - const SIGNIFICAND_MASK: $uty = (1 << SIGNIFICAND_BITS) - 1; - const SIGN_MASK: $uty = (1 << (BITS - 1)); - const EXPONENT_MASK: $uty = !(SIGN_MASK | SIGNIFICAND_MASK); - - fn mk_f32(sign: bool, exponent: $uty, significand: $uty) -> $fty { - unsafe { - mem::transmute( - ((sign as $uty) << (BITS - 1)) - | ((exponent & EXPONENT_MASK) << SIGNIFICAND_BITS) - | (significand & SIGNIFICAND_MASK), - ) - } - } - - if rng.gen_range(0, 10) == 1 { - // Special values - *[ - -0.0, - 0.0, - ::std::$fty::MIN, - ::std::$fty::MIN_POSITIVE, - ::std::$fty::MAX, - ::std::$fty::NAN, - ::std::$fty::INFINITY, - -::std::$fty::INFINITY, - ] - .choose(rng) - .unwrap() - } else if rng.gen_range(0, 10) == 1 { - // NaN patterns - mk_f32(rng.gen(), rng.gen(), 0) - } else if rng.gen() { - // Denormalized - mk_f32(rng.gen(), 0, rng.gen()) - } else { - // Random anything - mk_f32(rng.gen(), rng.gen(), rng.gen()) - } - } - }; -} - -gen_float!(gen_f32, f32, u32, 32, 23); -gen_float!(gen_f64, f64, u64, 64, 52); - -macro_rules! gen_large_float { - ($name:ident, - $fty:ident, - $uty:ident, - $bits:expr, - $significand_bits:expr) => { - pub fn $name(rng: &mut R) -> $fty - where - R: Rng + ?Sized, - { - const BITS: u8 = $bits; - const SIGNIFICAND_BITS: u8 = $significand_bits; - - const SIGNIFICAND_MASK: $uty = (1 << SIGNIFICAND_BITS) - 1; - const SIGN_MASK: $uty = (1 << (BITS - 1)); - const EXPONENT_MASK: $uty = !(SIGN_MASK | SIGNIFICAND_MASK); - - fn mk_f32(sign: bool, exponent: $uty, significand: $uty) -> $fty { - unsafe { - mem::transmute( - ((sign as $uty) << (BITS - 1)) - | ((exponent & EXPONENT_MASK) << SIGNIFICAND_BITS) - | (significand & SIGNIFICAND_MASK), - ) - } - } - - if rng.gen_range(0, 10) == 1 { - // Special values - *[ - -0.0, - 0.0, - ::std::$fty::MIN, - ::std::$fty::MIN_POSITIVE, - ::std::$fty::MAX, - ::std::$fty::NAN, - ::std::$fty::INFINITY, - -::std::$fty::INFINITY, - ] - .choose(rng) - .unwrap() - } else if rng.gen_range(0, 10) == 1 { - // NaN patterns - mk_f32(rng.gen(), rng.gen(), 0) - } else if rng.gen() { - // Denormalized - mk_f32(rng.gen(), 0, rng.gen()) - } else { - // Random anything - rng.gen::<$fty>() - } - } - }; -} - -gen_large_float!(gen_large_f32, f32, u32, 32, 23); -gen_large_float!(gen_large_f64, f64, u64, 64, 52); - -trait TestInput: Hash + Eq + fmt::Debug { - fn ty_name() -> String; - fn generate_lets(container: &str, cnt: &mut u8) -> String; - fn generate_static(&self, dst: &mut String); -} - -trait TestOutput { - fn ty_name() -> String; - fn generate_static(&self, dst: &mut String); - fn generate_expr(container: &str) -> String; -} - -fn gen(mut generate: F, test: &str) -where - F: FnMut(A) -> Option, - A: TestInput + Copy, - R: TestOutput, - rand::distributions::Standard: rand::distributions::Distribution, -{ - let rng = &mut rand::thread_rng(); - let testname = test.split("::").last().unwrap().split("(").next().unwrap(); - let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()); - let out_file = out_dir.join("generated.rs"); - - let mut testcases = HashMap::new(); - let mut n = NTESTS; - while n > 0 { - let input: A = rng.gen(); - if testcases.contains_key(&input) { - continue; - } - let output = match generate(input) { - Some(o) => o, - None => continue, - }; - testcases.insert(input, output); - n -= 1; - } - - let mut contents = String::new(); - contents.push_str(&format!("mod {} {{\nuse super::*;\n", testname)); - contents.push_str("#[test]\n"); - contents.push_str("fn test() {\n"); - contents.push_str(&format!( - "static TESTS: [({}, {}); {}] = [\n", - A::ty_name(), - R::ty_name(), - NTESTS - )); - for (input, output) in testcases { - contents.push_str(" ("); - input.generate_static(&mut contents); - contents.push_str(", "); - output.generate_static(&mut contents); - contents.push_str("),\n"); - } - contents.push_str("];\n"); - - contents.push_str(&format!( - r#" - for &(inputs, output) in TESTS.iter() {{ - {} - assert_eq!({}, {}, "inputs {{:?}}", inputs) - }} - "#, - A::generate_lets("inputs", &mut 0), - R::generate_expr("output"), - test, - )); - contents.push_str("\n}\n"); - contents.push_str("\n}\n"); - - OpenOptions::new() - .write(true) - .append(true) - .create(true) - .open(out_file) - .unwrap() - .write_all(contents.as_bytes()) - .unwrap(); -} - -macro_rules! my_float { - ($(struct $name:ident($inner:ident) = $gen:ident;)*) => ($( - #[derive(Debug, Clone, Copy)] - struct $name($inner); - - impl TestInput for $name { - fn ty_name() -> String { - format!("u{}", &stringify!($inner)[1..]) - } - - fn generate_lets(container: &str, cnt: &mut u8) -> String { - let me = *cnt; - *cnt += 1; - format!("let {} = {}::from_bits({});\n", - (b'a' + me) as char, - stringify!($inner), - container) - } - - fn generate_static(&self, dst: &mut String) { - write!(dst, "{}", self.0.to_bits()).unwrap(); - } - } - - impl rand::distributions::Distribution<$name> for rand::distributions::Standard { - fn sample(&self, r: &mut R) -> $name { - $name($gen(r)) - } - } - - impl Hash for $name { - fn hash(&self, h: &mut H) { - self.0.to_bits().hash(h) - } - } - - impl PartialEq for $name { - fn eq(&self, other: &$name) -> bool { - self.0.to_bits() == other.0.to_bits() - } - } - - impl Eq for $name {} - - )*) -} - -my_float! { - struct MyF64(f64) = gen_f64; - struct LargeF64(f64) = gen_large_f64; - struct MyF32(f32) = gen_f32; - struct LargeF32(f32) = gen_large_f32; -} - -macro_rules! my_integer { - ($(struct $name:ident($inner:ident);)*) => ($( - #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] - struct $name($inner); - - impl TestInput for $name { - fn ty_name() -> String { - stringify!($inner).to_string() - } - - fn generate_lets(container: &str, cnt: &mut u8) -> String { - let me = *cnt; - *cnt += 1; - format!("let {} = {};\n", - (b'a' + me) as char, - container) - } - - fn generate_static(&self, dst: &mut String) { - write!(dst, "{}", self.0).unwrap(); - } - } - - impl rand::distributions::Distribution<$name> for rand::distributions::Standard { - fn sample(&self, r: &mut R) -> $name { - let bits = (0 as $inner).count_zeros(); - let mut mk = || { - if r.gen_range(0, 10) == 1 { - *[ - ::std::$inner::MAX >> (bits / 2), - 0, - ::std::$inner::MIN >> (bits / 2), - ].choose(r).unwrap() - } else { - r.gen::<$inner>() - } - }; - let a = mk(); - let b = mk(); - $name((a << (bits / 2)) | (b & (!0 << (bits / 2)))) - } - } - )*) -} - -my_integer! { - struct MyI32(i32); - struct MyI64(i64); - struct MyI128(i128); - struct MyU32(u32); - struct MyU64(u64); - struct MyU128(u128); -} - -impl TestInput for (A, B) -where - A: TestInput, - B: TestInput, -{ - fn ty_name() -> String { - format!("({}, {})", A::ty_name(), B::ty_name()) - } - - fn generate_lets(container: &str, cnt: &mut u8) -> String { - format!( - "{}{}", - A::generate_lets(&format!("{}.0", container), cnt), - B::generate_lets(&format!("{}.1", container), cnt) - ) - } - - fn generate_static(&self, dst: &mut String) { - dst.push_str("("); - self.0.generate_static(dst); - dst.push_str(", "); - self.1.generate_static(dst); - dst.push_str(")"); - } -} - -impl TestOutput for f64 { - fn ty_name() -> String { - "u64".to_string() - } - - fn generate_static(&self, dst: &mut String) { - write!(dst, "{}", self.to_bits()).unwrap(); - } - - fn generate_expr(container: &str) -> String { - format!("f64::from_bits({})", container) - } -} - -impl TestOutput for f32 { - fn ty_name() -> String { - "u32".to_string() - } - - fn generate_static(&self, dst: &mut String) { - write!(dst, "{}", self.to_bits()).unwrap(); - } - - fn generate_expr(container: &str) -> String { - format!("f32::from_bits({})", container) - } -} - -macro_rules! plain_test_output { - ($($i:tt)*) => ($( - impl TestOutput for $i { - fn ty_name() -> String { - stringify!($i).to_string() - } - - fn generate_static(&self, dst: &mut String) { - write!(dst, "{}", self).unwrap(); - } - - fn generate_expr(container: &str) -> String { - container.to_string() - } - } - )*) -} - -plain_test_output!(i32 i64 i128 u32 u64 u128 bool); - -impl TestOutput for (A, B) -where - A: TestOutput, - B: TestOutput, -{ - fn ty_name() -> String { - format!("({}, {})", A::ty_name(), B::ty_name()) - } - - fn generate_static(&self, dst: &mut String) { - dst.push_str("("); - self.0.generate_static(dst); - dst.push_str(", "); - self.1.generate_static(dst); - dst.push_str(")"); - } - - fn generate_expr(container: &str) -> String { - container.to_string() - } -} - -trait FromFloat: Sized { - fn cast(src: T) -> Option; -} - -macro_rules! from_float { - ($($src:ident => $($dst:ident),+);+;) => { - $( - $( - impl FromFloat<$src> for $dst { - fn cast(src: $src) -> Option<$dst> { - use std::{$dst, $src}; - - if src.is_nan() || - src.is_infinite() || - src < std::$dst::MIN as $src || - src > std::$dst::MAX as $src - { - None - } else { - Some(src as $dst) - } - } - } - )+ - )+ - } -} - -from_float! { - f32 => i32, i64, i128, u32, u64, u128; - f64 => i32, i64, i128, u32, u64, u128; -} diff --git a/crux-mir/lib/compiler_builtins/testcrate/src/lib.rs b/crux-mir/lib/compiler_builtins/testcrate/src/lib.rs deleted file mode 100644 index 0c9ac1ac8..000000000 --- a/crux-mir/lib/compiler_builtins/testcrate/src/lib.rs +++ /dev/null @@ -1 +0,0 @@ -#![no_std] diff --git a/crux-mir/lib/compiler_builtins/testcrate/tests/aeabi_memclr.rs b/crux-mir/lib/compiler_builtins/testcrate/tests/aeabi_memclr.rs deleted file mode 100644 index 595076939..000000000 --- a/crux-mir/lib/compiler_builtins/testcrate/tests/aeabi_memclr.rs +++ /dev/null @@ -1,61 +0,0 @@ -#![cfg(all( - target_arch = "arm", - not(any(target_env = "gnu", target_env = "musl")), - target_os = "linux", - feature = "mem" -))] -#![feature(compiler_builtins_lib)] -#![feature(lang_items)] -#![no_std] - -extern crate compiler_builtins; - -// test runner -extern crate utest_cortex_m_qemu; - -// overrides `panic!` -#[macro_use] -extern crate utest_macros; - -use core::mem; - -macro_rules! panic { - ($($tt:tt)*) => { - upanic!($($tt)*); - }; -} - -extern "C" { - fn __aeabi_memclr4(dest: *mut u8, n: usize); - fn __aeabi_memset4(dest: *mut u8, n: usize, c: u32); -} - -struct Aligned { - array: [u8; 8], - _alignment: [u32; 0], -} - -impl Aligned { - fn new() -> Self { - Aligned { - array: [0; 8], - _alignment: [], - } - } -} - -#[test] -fn memclr4() { - let mut aligned = Aligned::new(); - assert_eq!(mem::align_of_val(&aligned), 4); - let xs = &mut aligned.array; - - for n in 0..9 { - unsafe { - __aeabi_memset4(xs.as_mut_ptr(), n, 0xff); - __aeabi_memclr4(xs.as_mut_ptr(), n); - } - - assert!(xs[0..n].iter().all(|x| *x == 0)); - } -} diff --git a/crux-mir/lib/compiler_builtins/testcrate/tests/aeabi_memcpy.rs b/crux-mir/lib/compiler_builtins/testcrate/tests/aeabi_memcpy.rs deleted file mode 100644 index 2d72dfbba..000000000 --- a/crux-mir/lib/compiler_builtins/testcrate/tests/aeabi_memcpy.rs +++ /dev/null @@ -1,72 +0,0 @@ -#![cfg(all( - target_arch = "arm", - not(any(target_env = "gnu", target_env = "musl")), - target_os = "linux", - feature = "mem" -))] -#![feature(compiler_builtins_lib)] -#![feature(lang_items)] -#![no_std] - -extern crate compiler_builtins; - -// test runner -extern crate utest_cortex_m_qemu; - -// overrides `panic!` -#[macro_use] -extern crate utest_macros; - -macro_rules! panic { - ($($tt:tt)*) => { - upanic!($($tt)*); - }; -} - -extern "C" { - fn __aeabi_memcpy(dest: *mut u8, src: *const u8, n: usize); - fn __aeabi_memcpy4(dest: *mut u8, src: *const u8, n: usize); -} - -struct Aligned { - array: [u8; 8], - _alignment: [u32; 0], -} - -impl Aligned { - fn new(array: [u8; 8]) -> Self { - Aligned { - array: array, - _alignment: [], - } - } -} - -#[test] -fn memcpy() { - let mut dest = [0; 4]; - let src = [0xde, 0xad, 0xbe, 0xef]; - - for n in 0..dest.len() { - dest.copy_from_slice(&[0; 4]); - - unsafe { __aeabi_memcpy(dest.as_mut_ptr(), src.as_ptr(), n) } - - assert_eq!(&dest[0..n], &src[0..n]) - } -} - -#[test] -fn memcpy4() { - let mut aligned = Aligned::new([0; 8]); - let dest = &mut aligned.array; - let src = [0xde, 0xad, 0xbe, 0xef, 0xba, 0xad, 0xf0, 0x0d]; - - for n in 0..dest.len() { - dest.copy_from_slice(&[0; 8]); - - unsafe { __aeabi_memcpy4(dest.as_mut_ptr(), src.as_ptr(), n) } - - assert_eq!(&dest[0..n], &src[0..n]) - } -} diff --git a/crux-mir/lib/compiler_builtins/testcrate/tests/aeabi_memset.rs b/crux-mir/lib/compiler_builtins/testcrate/tests/aeabi_memset.rs deleted file mode 100644 index f03729bed..000000000 --- a/crux-mir/lib/compiler_builtins/testcrate/tests/aeabi_memset.rs +++ /dev/null @@ -1,241 +0,0 @@ -#![cfg(all( - target_arch = "arm", - not(any(target_env = "gnu", target_env = "musl")), - target_os = "linux", - feature = "mem" -))] -#![feature(compiler_builtins_lib)] -#![feature(lang_items)] -#![no_std] - -extern crate compiler_builtins; - -// test runner -extern crate utest_cortex_m_qemu; - -// overrides `panic!` -#[macro_use] -extern crate utest_macros; - -use core::mem; - -macro_rules! panic { - ($($tt:tt)*) => { - upanic!($($tt)*); - }; -} - -extern "C" { - fn __aeabi_memset4(dest: *mut u8, n: usize, c: u32); -} - -struct Aligned { - array: [u8; 8], - _alignment: [u32; 0], -} - -impl Aligned { - fn new(array: [u8; 8]) -> Self { - Aligned { - array: array, - _alignment: [], - } - } -} - -#[test] -fn zero() { - let mut aligned = Aligned::new([0u8; 8]); - assert_eq!(mem::align_of_val(&aligned), 4); - let xs = &mut aligned.array; - let c = 0xdeadbeef; - - unsafe { __aeabi_memset4(xs.as_mut_ptr(), 0, c) } - - assert_eq!(*xs, [0; 8]); - - let mut aligned = Aligned::new([1u8; 8]); - assert_eq!(mem::align_of_val(&aligned), 4); - let xs = &mut aligned.array; - let c = 0xdeadbeef; - - unsafe { __aeabi_memset4(xs.as_mut_ptr(), 0, c) } - - assert_eq!(*xs, [1; 8]); -} - -#[test] -fn one() { - let mut aligned = Aligned::new([0u8; 8]); - assert_eq!(mem::align_of_val(&aligned), 4); - let xs = &mut aligned.array; - let n = 1; - let c = 0xdeadbeef; - - unsafe { __aeabi_memset4(xs.as_mut_ptr(), n, c) } - - assert_eq!(*xs, [0xef, 0, 0, 0, 0, 0, 0, 0]); - - let mut aligned = Aligned::new([1u8; 8]); - assert_eq!(mem::align_of_val(&aligned), 4); - let xs = &mut aligned.array; - let c = 0xdeadbeef; - - unsafe { __aeabi_memset4(xs.as_mut_ptr(), n, c) } - - assert_eq!(*xs, [0xef, 1, 1, 1, 1, 1, 1, 1]); -} - -#[test] -fn two() { - let mut aligned = Aligned::new([0u8; 8]); - assert_eq!(mem::align_of_val(&aligned), 4); - let xs = &mut aligned.array; - let n = 2; - let c = 0xdeadbeef; - - unsafe { __aeabi_memset4(xs.as_mut_ptr(), n, c) } - - assert_eq!(*xs, [0xef, 0xef, 0, 0, 0, 0, 0, 0]); - - let mut aligned = Aligned::new([1u8; 8]); - assert_eq!(mem::align_of_val(&aligned), 4); - let xs = &mut aligned.array; - let c = 0xdeadbeef; - - unsafe { __aeabi_memset4(xs.as_mut_ptr(), n, c) } - - assert_eq!(*xs, [0xef, 0xef, 1, 1, 1, 1, 1, 1]); -} - -#[test] -fn three() { - let mut aligned = Aligned::new([0u8; 8]); - assert_eq!(mem::align_of_val(&aligned), 4); - let xs = &mut aligned.array; - let n = 3; - let c = 0xdeadbeef; - - unsafe { __aeabi_memset4(xs.as_mut_ptr(), n, c) } - - assert_eq!(*xs, [0xef, 0xef, 0xef, 0, 0, 0, 0, 0]); - - let mut aligned = Aligned::new([1u8; 8]); - assert_eq!(mem::align_of_val(&aligned), 4); - let xs = &mut aligned.array; - let c = 0xdeadbeef; - - unsafe { __aeabi_memset4(xs.as_mut_ptr(), n, c) } - - assert_eq!(*xs, [0xef, 0xef, 0xef, 1, 1, 1, 1, 1]); -} - -#[test] -fn four() { - let mut aligned = Aligned::new([0u8; 8]); - assert_eq!(mem::align_of_val(&aligned), 4); - let xs = &mut aligned.array; - let n = 4; - let c = 0xdeadbeef; - - unsafe { __aeabi_memset4(xs.as_mut_ptr(), n, c) } - - assert_eq!(*xs, [0xef, 0xef, 0xef, 0xef, 0, 0, 0, 0]); - - let mut aligned = Aligned::new([1u8; 8]); - assert_eq!(mem::align_of_val(&aligned), 4); - let xs = &mut aligned.array; - let c = 0xdeadbeef; - - unsafe { __aeabi_memset4(xs.as_mut_ptr(), n, c) } - - assert_eq!(*xs, [0xef, 0xef, 0xef, 0xef, 1, 1, 1, 1]); -} - -#[test] -fn five() { - let mut aligned = Aligned::new([0u8; 8]); - assert_eq!(mem::align_of_val(&aligned), 4); - let xs = &mut aligned.array; - let n = 5; - let c = 0xdeadbeef; - - unsafe { __aeabi_memset4(xs.as_mut_ptr(), n, c) } - - assert_eq!(*xs, [0xef, 0xef, 0xef, 0xef, 0xef, 0, 0, 0]); - - let mut aligned = Aligned::new([1u8; 8]); - assert_eq!(mem::align_of_val(&aligned), 4); - let xs = &mut aligned.array; - let c = 0xdeadbeef; - - unsafe { __aeabi_memset4(xs.as_mut_ptr(), n, c) } - - assert_eq!(*xs, [0xef, 0xef, 0xef, 0xef, 0xef, 1, 1, 1]); -} - -#[test] -fn six() { - let mut aligned = Aligned::new([0u8; 8]); - assert_eq!(mem::align_of_val(&aligned), 4); - let xs = &mut aligned.array; - let n = 6; - let c = 0xdeadbeef; - - unsafe { __aeabi_memset4(xs.as_mut_ptr(), n, c) } - - assert_eq!(*xs, [0xef, 0xef, 0xef, 0xef, 0xef, 0xef, 0, 0]); - - let mut aligned = Aligned::new([1u8; 8]); - assert_eq!(mem::align_of_val(&aligned), 4); - let xs = &mut aligned.array; - let c = 0xdeadbeef; - - unsafe { __aeabi_memset4(xs.as_mut_ptr(), n, c) } - - assert_eq!(*xs, [0xef, 0xef, 0xef, 0xef, 0xef, 0xef, 1, 1]); -} - -#[test] -fn seven() { - let mut aligned = Aligned::new([0u8; 8]); - assert_eq!(mem::align_of_val(&aligned), 4); - let xs = &mut aligned.array; - let n = 7; - let c = 0xdeadbeef; - - unsafe { __aeabi_memset4(xs.as_mut_ptr(), n, c) } - - assert_eq!(*xs, [0xef, 0xef, 0xef, 0xef, 0xef, 0xef, 0xef, 0]); - - let mut aligned = Aligned::new([1u8; 8]); - assert_eq!(mem::align_of_val(&aligned), 4); - let xs = &mut aligned.array; - let c = 0xdeadbeef; - - unsafe { __aeabi_memset4(xs.as_mut_ptr(), n, c) } - - assert_eq!(*xs, [0xef, 0xef, 0xef, 0xef, 0xef, 0xef, 0xef, 1]); -} - -#[test] -fn eight() { - let mut aligned = Aligned::new([0u8; 8]); - assert_eq!(mem::align_of_val(&aligned), 4); - let xs = &mut aligned.array; - let n = 8; - let c = 0xdeadbeef; - - unsafe { __aeabi_memset4(xs.as_mut_ptr(), n, c) } - - assert_eq!(*xs, [0xef, 0xef, 0xef, 0xef, 0xef, 0xef, 0xef, 0xef]); - - let mut aligned = Aligned::new([1u8; 8]); - assert_eq!(mem::align_of_val(&aligned), 4); - let xs = &mut aligned.array; - let c = 0xdeadbeef; - - unsafe { __aeabi_memset4(xs.as_mut_ptr(), n, c) } - - assert_eq!(*xs, [0xef, 0xef, 0xef, 0xef, 0xef, 0xef, 0xef, 0xef]); -} diff --git a/crux-mir/lib/compiler_builtins/testcrate/tests/count_leading_zeros.rs b/crux-mir/lib/compiler_builtins/testcrate/tests/count_leading_zeros.rs deleted file mode 100644 index b50a7ce84..000000000 --- a/crux-mir/lib/compiler_builtins/testcrate/tests/count_leading_zeros.rs +++ /dev/null @@ -1,25 +0,0 @@ -#![feature(compiler_builtins_lib)] - -extern crate compiler_builtins; - -use compiler_builtins::int::__clzsi2; - -#[test] -fn __clzsi2_test() { - let mut i: usize = core::usize::MAX; - // Check all values above 0 - while i > 0 { - assert_eq!(__clzsi2(i) as u32, i.leading_zeros()); - i >>= 1; - } - // check 0 also - i = 0; - assert_eq!(__clzsi2(i) as u32, i.leading_zeros()); - // double check for bit patterns that aren't just solid 1s - i = 1; - for _ in 0..63 { - assert_eq!(__clzsi2(i) as u32, i.leading_zeros()); - i <<= 2; - i += 1; - } -} diff --git a/crux-mir/lib/compiler_builtins/testcrate/tests/generated.rs b/crux-mir/lib/compiler_builtins/testcrate/tests/generated.rs deleted file mode 100644 index ee575cba8..000000000 --- a/crux-mir/lib/compiler_builtins/testcrate/tests/generated.rs +++ /dev/null @@ -1,38 +0,0 @@ -#![feature(compiler_builtins_lib)] -#![feature(lang_items)] -#![allow(bad_style)] -#![allow(unused_imports)] -#![no_std] - -extern crate compiler_builtins as builtins; - -#[cfg(all( - target_arch = "arm", - not(any(target_env = "gnu", target_env = "musl")), - target_os = "linux", - test -))] -extern crate utest_cortex_m_qemu; - -#[cfg(all( - target_arch = "arm", - not(any(target_env = "gnu", target_env = "musl")), - target_os = "linux", - test -))] -#[macro_use] -extern crate utest_macros; - -#[cfg(all( - target_arch = "arm", - not(any(target_env = "gnu", target_env = "musl")), - target_os = "linux", - test -))] -macro_rules! panic { // overrides `panic!` - ($($tt:tt)*) => { - upanic!($($tt)*); - }; -} - -include!(concat!(env!("OUT_DIR"), "/generated.rs")); diff --git a/crux-mir/lib/compiler_builtins/thumbv6m-linux-eabi.json b/crux-mir/lib/compiler_builtins/thumbv6m-linux-eabi.json deleted file mode 100644 index ac736eae6..000000000 --- a/crux-mir/lib/compiler_builtins/thumbv6m-linux-eabi.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "abi-blacklist": [ - "stdcall", - "fastcall", - "vectorcall", - "win64", - "sysv64" - ], - "arch": "arm", - "data-layout": "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64", - "env": "", - "executables": true, - "features": "+strict-align", - "linker": "arm-none-eabi-gcc", - "linker-flavor": "gcc", - "llvm-target": "thumbv6m-none-eabi", - "max-atomic-width": 0, - "os": "linux", - "panic-strategy": "abort", - "pre-link-args": { - "gcc": ["-nostartfiles"] - }, - "relocation-model": "static", - "target-endian": "little", - "target-pointer-width": "32", - "target-c-int-width": "32", - "vendor": "" -} diff --git a/crux-mir/lib/compiler_builtins/thumbv7em-linux-eabi.json b/crux-mir/lib/compiler_builtins/thumbv7em-linux-eabi.json deleted file mode 100644 index b6d4a6bda..000000000 --- a/crux-mir/lib/compiler_builtins/thumbv7em-linux-eabi.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "abi-blacklist": [ - "stdcall", - "fastcall", - "vectorcall", - "win64", - "sysv64" - ], - "arch": "arm", - "data-layout": "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64", - "env": "", - "executables": true, - "linker": "arm-none-eabi-gcc", - "linker-flavor": "gcc", - "llvm-target": "thumbv7em-none-eabi", - "max-atomic-width": 32, - "os": "linux", - "panic-strategy": "abort", - "pre-link-args": { - "gcc": ["-nostartfiles"] - }, - "relocation-model": "static", - "target-endian": "little", - "target-pointer-width": "32", - "target-c-int-width": "32", - "vendor": "" -} diff --git a/crux-mir/lib/compiler_builtins/thumbv7em-linux-eabihf.json b/crux-mir/lib/compiler_builtins/thumbv7em-linux-eabihf.json deleted file mode 100644 index 81cfcd48d..000000000 --- a/crux-mir/lib/compiler_builtins/thumbv7em-linux-eabihf.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "abi-blacklist": [ - "stdcall", - "fastcall", - "vectorcall", - "win64", - "sysv64" - ], - "arch": "arm", - "data-layout": "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64", - "env": "", - "executables": true, - "features": "+vfp4,+d16,+fp-only-sp", - "linker": "arm-none-eabi-gcc", - "linker-flavor": "gcc", - "llvm-target": "thumbv7em-none-eabihf", - "max-atomic-width": 32, - "os": "linux", - "panic-strategy": "abort", - "pre-link-args": { - "gcc": ["-nostartfiles"] - }, - "relocation-model": "static", - "target-endian": "little", - "target-pointer-width": "32", - "target-c-int-width": "32", - "vendor": "" -} diff --git a/crux-mir/lib/compiler_builtins/thumbv7m-linux-eabi.json b/crux-mir/lib/compiler_builtins/thumbv7m-linux-eabi.json deleted file mode 100644 index abe037c5b..000000000 --- a/crux-mir/lib/compiler_builtins/thumbv7m-linux-eabi.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "abi-blacklist": [ - "stdcall", - "fastcall", - "vectorcall", - "win64", - "sysv64" - ], - "arch": "arm", - "data-layout": "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64", - "env": "", - "executables": true, - "linker": "arm-none-eabi-gcc", - "linker-flavor": "gcc", - "llvm-target": "thumbv7m-none-eabi", - "max-atomic-width": 32, - "os": "linux", - "panic-strategy": "abort", - "pre-link-args": { - "gcc": ["-nostartfiles"] - }, - "relocation-model": "static", - "target-endian": "little", - "target-pointer-width": "32", - "target-c-int-width": "32", - "vendor": "" -} diff --git a/crux-mir/lib/core/Cargo.toml b/crux-mir/lib/core/Cargo.toml new file mode 100644 index 000000000..3dc8c84e0 --- /dev/null +++ b/crux-mir/lib/core/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "core" +version = "0.0.0" +license = "MIT OR Apache-2.0" +repository = "https://github.com/rust-lang/rust.git" +description = "The Rust Core Library" +autotests = false +autobenches = false +# If you update this, be sure to update it in a bunch of other places too! +# As of 2022, it was the ci/pgo.sh script and the core-no-fp-fmt-parse test. +edition = "2021" + +[lib] +test = false +bench = false + +[[test]] +name = "coretests" +path = "tests/lib.rs" + +[[bench]] +name = "corebenches" +path = "benches/lib.rs" +test = true + +[dev-dependencies] +rand = { version = "0.8.5", default-features = false } +rand_xorshift = { version = "0.3.0", default-features = false } + +[features] +# Make panics and failed asserts immediately abort without formatting any message +panic_immediate_abort = [] +# Make `RefCell` store additional debugging information, which is printed out when +# a borrow error occurs +debug_refcell = [] diff --git a/crux-mir/lib/core/benches/ascii.rs b/crux-mir/lib/core/benches/ascii.rs index 76ccd3ddb..64938745a 100644 --- a/crux-mir/lib/core/benches/ascii.rs +++ b/crux-mir/lib/core/benches/ascii.rs @@ -1,3 +1,5 @@ +mod is_ascii; + // Lower-case ASCII 'a' is the first byte that has its highest bit set // after wrap-adding 0x1F: // @@ -64,6 +66,8 @@ macro_rules! benches { use test::black_box; use test::Bencher; +const ASCII_CASE_MASK: u8 = 0b0010_0000; + benches! { fn case00_alloc_only(_bytes: &mut [u8]) {} @@ -202,7 +206,7 @@ benches! { } } for byte in bytes { - *byte &= !((is_ascii_lowercase(*byte) as u8) << 5) + *byte &= !((is_ascii_lowercase(*byte) as u8) * ASCII_CASE_MASK) } } @@ -214,7 +218,7 @@ benches! { } } for byte in bytes { - *byte -= (is_ascii_lowercase(*byte) as u8) << 5 + *byte -= (is_ascii_lowercase(*byte) as u8) * ASCII_CASE_MASK } } @@ -251,9 +255,9 @@ macro_rules! repeat { }; } -const SHORT: &'static str = "Alice's"; -const MEDIUM: &'static str = "Alice's Adventures in Wonderland"; -const LONG: &'static str = repeat!( +const SHORT: &str = "Alice's"; +const MEDIUM: &str = "Alice's Adventures in Wonderland"; +const LONG: &str = repeat!( r#" La Guida di Bragia, a Ballad Opera for the Marionette Theatre (around 1850) Alice's Adventures in Wonderland (1865) diff --git a/crux-mir/lib/core/benches/ascii/is_ascii.rs b/crux-mir/lib/core/benches/ascii/is_ascii.rs new file mode 100644 index 000000000..a42a1dcfe --- /dev/null +++ b/crux-mir/lib/core/benches/ascii/is_ascii.rs @@ -0,0 +1,82 @@ +use super::{LONG, MEDIUM, SHORT}; +use test::black_box; +use test::Bencher; + +macro_rules! benches { + ($( fn $name: ident($arg: ident: &[u8]) $body: block )+) => { + benches!(mod short SHORT[..] $($name $arg $body)+); + benches!(mod medium MEDIUM[..] $($name $arg $body)+); + benches!(mod long LONG[..] $($name $arg $body)+); + // Ensure we benchmark cases where the functions are called with strings + // that are not perfectly aligned or have a length which is not a + // multiple of size_of::() (or both) + benches!(mod unaligned_head MEDIUM[1..] $($name $arg $body)+); + benches!(mod unaligned_tail MEDIUM[..(MEDIUM.len() - 1)] $($name $arg $body)+); + benches!(mod unaligned_both MEDIUM[1..(MEDIUM.len() - 1)] $($name $arg $body)+); + }; + + (mod $mod_name: ident $input: ident [$range: expr] $($name: ident $arg: ident $body: block)+) => { + mod $mod_name { + use super::*; + $( + #[bench] + fn $name(bencher: &mut Bencher) { + bencher.bytes = $input[$range].len() as u64; + let mut vec = $input.as_bytes().to_vec(); + bencher.iter(|| { + let $arg: &[u8] = &black_box(&mut vec)[$range]; + black_box($body) + }) + } + )+ + } + }; +} + +benches! { + fn case00_libcore(bytes: &[u8]) { + bytes.is_ascii() + } + + fn case01_iter_all(bytes: &[u8]) { + bytes.iter().all(|b| b.is_ascii()) + } + + fn case02_align_to(bytes: &[u8]) { + is_ascii_align_to(bytes) + } + + fn case03_align_to_unrolled(bytes: &[u8]) { + is_ascii_align_to_unrolled(bytes) + } +} + +// These are separate since it's easier to debug errors if they don't go through +// macro expansion first. +fn is_ascii_align_to(bytes: &[u8]) -> bool { + if bytes.len() < core::mem::size_of::() { + return bytes.iter().all(|b| b.is_ascii()); + } + // SAFETY: transmuting a sequence of `u8` to `usize` is always fine + let (head, body, tail) = unsafe { bytes.align_to::() }; + head.iter().all(|b| b.is_ascii()) + && body.iter().all(|w| !contains_nonascii(*w)) + && tail.iter().all(|b| b.is_ascii()) +} + +fn is_ascii_align_to_unrolled(bytes: &[u8]) -> bool { + if bytes.len() < core::mem::size_of::() { + return bytes.iter().all(|b| b.is_ascii()); + } + // SAFETY: transmuting a sequence of `u8` to `[usize; 2]` is always fine + let (head, body, tail) = unsafe { bytes.align_to::<[usize; 2]>() }; + head.iter().all(|b| b.is_ascii()) + && body.iter().all(|w| !contains_nonascii(w[0] | w[1])) + && tail.iter().all(|b| b.is_ascii()) +} + +#[inline] +fn contains_nonascii(v: usize) -> bool { + const NONASCII_MASK: usize = usize::from_ne_bytes([0x80; core::mem::size_of::()]); + (NONASCII_MASK & v) != 0 +} diff --git a/crux-mir/lib/core/benches/char/methods.rs b/crux-mir/lib/core/benches/char/methods.rs index a9a08a4d7..9408f83c3 100644 --- a/crux-mir/lib/core/benches/char/methods.rs +++ b/crux-mir/lib/core/benches/char/methods.rs @@ -35,3 +35,43 @@ fn bench_to_digit_radix_var(b: &mut Bencher) { .min() }) } + +#[bench] +fn bench_to_ascii_uppercase(b: &mut Bencher) { + b.iter(|| CHARS.iter().cycle().take(10_000).map(|c| c.to_ascii_uppercase()).min()) +} + +#[bench] +fn bench_to_ascii_lowercase(b: &mut Bencher) { + b.iter(|| CHARS.iter().cycle().take(10_000).map(|c| c.to_ascii_lowercase()).min()) +} + +#[bench] +fn bench_ascii_mix_to_uppercase(b: &mut Bencher) { + b.iter(|| (0..=255).cycle().take(10_000).map(|b| char::from(b).to_uppercase()).count()) +} + +#[bench] +fn bench_ascii_mix_to_lowercase(b: &mut Bencher) { + b.iter(|| (0..=255).cycle().take(10_000).map(|b| char::from(b).to_lowercase()).count()) +} + +#[bench] +fn bench_ascii_char_to_uppercase(b: &mut Bencher) { + b.iter(|| (0..=127).cycle().take(10_000).map(|b| char::from(b).to_uppercase()).count()) +} + +#[bench] +fn bench_ascii_char_to_lowercase(b: &mut Bencher) { + b.iter(|| (0..=127).cycle().take(10_000).map(|b| char::from(b).to_lowercase()).count()) +} + +#[bench] +fn bench_non_ascii_char_to_uppercase(b: &mut Bencher) { + b.iter(|| (128..=255).cycle().take(10_000).map(|b| char::from(b).to_uppercase()).count()) +} + +#[bench] +fn bench_non_ascii_char_to_lowercase(b: &mut Bencher) { + b.iter(|| (128..=255).cycle().take(10_000).map(|b| char::from(b).to_lowercase()).count()) +} diff --git a/crux-mir/lib/core/benches/fmt.rs b/crux-mir/lib/core/benches/fmt.rs index dd72a3399..ff726ff75 100644 --- a/crux-mir/lib/core/benches/fmt.rs +++ b/crux-mir/lib/core/benches/fmt.rs @@ -108,3 +108,43 @@ fn write_str_macro_debug(bh: &mut Bencher) { } }); } + +#[bench] +fn write_str_macro_debug_ascii(bh: &mut Bencher) { + bh.iter(|| { + let mut mem = String::new(); + let wr = &mut mem as &mut dyn fmt::Write; + for _ in 0..1000 { + write!(wr, "{:?}", "Hello, World!").unwrap(); + } + }); +} + +#[bench] +fn write_u128_max(bh: &mut Bencher) { + bh.iter(|| { + test::black_box(format!("{}", u128::MAX)); + }); +} + +#[bench] +fn write_u128_min(bh: &mut Bencher) { + bh.iter(|| { + let s = format!("{}", 0u128); + test::black_box(s); + }); +} + +#[bench] +fn write_u64_max(bh: &mut Bencher) { + bh.iter(|| { + test::black_box(format!("{}", u64::MAX)); + }); +} + +#[bench] +fn write_u64_min(bh: &mut Bencher) { + bh.iter(|| { + test::black_box(format!("{}", 0u64)); + }); +} diff --git a/crux-mir/lib/core/benches/iter.rs b/crux-mir/lib/core/benches/iter.rs index fb6b4b783..9193c79be 100644 --- a/crux-mir/lib/core/benches/iter.rs +++ b/crux-mir/lib/core/benches/iter.rs @@ -1,4 +1,7 @@ +use core::borrow::Borrow; use core::iter::*; +use core::mem; +use core::num::Wrapping; use test::{black_box, Bencher}; #[bench] @@ -45,7 +48,7 @@ fn bench_max_by_key(b: &mut Bencher) { }) } -// http://www.reddit.com/r/rust/comments/31syce/using_iterators_to_find_the_index_of_the_min_or/ +// https://www.reddit.com/r/rust/comments/31syce/using_iterators_to_find_the_index_of_the_min_or/ #[bench] fn bench_max_by_key2(b: &mut Bencher) { fn max_index_iter(array: &[i32]) -> usize { @@ -276,7 +279,29 @@ bench_sums! { bench_sums! { bench_cycle_take_sum, bench_cycle_take_ref_sum, - (0i64..10000).cycle().take(1000000) + (0..10000).cycle().take(1000000) +} + +bench_sums! { + bench_cycle_skip_take_sum, + bench_cycle_skip_take_ref_sum, + (0..100000).cycle().skip(1000000).take(1000000) +} + +bench_sums! { + bench_cycle_take_skip_sum, + bench_cycle_take_skip_ref_sum, + (0..100000).cycle().take(1000000).skip(100000) +} + +bench_sums! { + bench_skip_cycle_skip_zip_add_sum, + bench_skip_cycle_skip_zip_add_ref_sum, + (0..100000).skip(100).cycle().skip(100) + .zip((0..100000).cycle().skip(10)) + .map(|(a,b)| a+b) + .skip(100000) + .take(1000000) } // Checks whether Skip> is as fast as Zip, Skip>, from @@ -341,7 +366,74 @@ fn bench_partial_cmp(b: &mut Bencher) { b.iter(|| (0..100000).map(black_box).partial_cmp((0..100000).map(black_box))) } +#[bench] +fn bench_chain_partial_cmp(b: &mut Bencher) { + b.iter(|| { + (0..50000).chain(50000..100000).map(black_box).partial_cmp((0..100000).map(black_box)) + }) +} + #[bench] fn bench_lt(b: &mut Bencher) { b.iter(|| (0..100000).map(black_box).lt((0..100000).map(black_box))) } + +#[bench] +fn bench_trusted_random_access_adapters(b: &mut Bencher) { + let vec1: Vec<_> = (0usize..100000).collect(); + let vec2 = black_box(vec1.clone()); + b.iter(|| { + let mut iter = vec1 + .iter() + .copied() + .enumerate() + .map(|(idx, e)| idx.wrapping_add(e)) + .zip(vec2.iter().copied()) + .map(|(a, b)| a.wrapping_add(b)) + .fuse(); + let mut acc: usize = 0; + let size = iter.size(); + for i in 0..size { + // SAFETY: TRA requirements are satisfied by 0..size iteration and then dropping the + // iterator. + acc = acc.wrapping_add(unsafe { iter.__iterator_get_unchecked(i) }); + } + acc + }) +} + +/// Exercises the iter::Copied specialization for slice::Iter +#[bench] +fn bench_copied_chunks(b: &mut Bencher) { + let v = vec![1u8; 1024]; + + b.iter(|| { + let mut iter = black_box(&v).iter().copied(); + let mut acc = Wrapping(0); + // This uses a while-let loop to side-step the TRA specialization in ArrayChunks + while let Ok(chunk) = iter.next_chunk::<{ mem::size_of::() }>() { + let d = u64::from_ne_bytes(chunk); + acc += Wrapping(d.rotate_left(7).wrapping_add(1)); + } + acc + }) +} + +/// Exercises the TrustedRandomAccess specialization in ArrayChunks +#[bench] +fn bench_trusted_random_access_chunks(b: &mut Bencher) { + let v = vec![1u8; 1024]; + + b.iter(|| { + black_box(&v) + .iter() + // this shows that we're not relying on the slice::Iter specialization in Copied + .map(|b| *b.borrow()) + .array_chunks::<{ mem::size_of::() }>() + .map(|ary| { + let d = u64::from_ne_bytes(ary); + Wrapping(d.rotate_left(7).wrapping_add(1)) + }) + .sum::>() + }) +} diff --git a/crux-mir/lib/core/benches/lib.rs b/crux-mir/lib/core/benches/lib.rs index 570fc4ab9..f1244d932 100644 --- a/crux-mir/lib/core/benches/lib.rs +++ b/crux-mir/lib/core/benches/lib.rs @@ -1,5 +1,10 @@ +// wasm32 does not support benches (no time). +#![cfg(not(target_arch = "wasm32"))] #![feature(flt2dec)] #![feature(test)] +#![feature(trusted_random_access)] +#![feature(iter_array_chunks)] +#![feature(iter_next_chunk)] extern crate test; @@ -13,3 +18,12 @@ mod num; mod ops; mod pattern; mod slice; +mod str; + +/// Returns a `rand::Rng` seeded with a consistent seed. +/// +/// This is done to avoid introducing nondeterminism in benchmark results. +fn bench_rng() -> rand_xorshift::XorShiftRng { + const SEED: [u8; 16] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; + rand::SeedableRng::from_seed(SEED) +} diff --git a/crux-mir/lib/core/benches/num/dec2flt/mod.rs b/crux-mir/lib/core/benches/num/dec2flt/mod.rs index 561a4bee8..305baa687 100644 --- a/crux-mir/lib/core/benches/num/dec2flt/mod.rs +++ b/crux-mir/lib/core/benches/num/dec2flt/mod.rs @@ -1,4 +1,3 @@ -use std::f64; use test::Bencher; #[bench] diff --git a/crux-mir/lib/core/benches/num/flt2dec/mod.rs b/crux-mir/lib/core/benches/num/flt2dec/mod.rs index b810dd12a..32fd5e626 100644 --- a/crux-mir/lib/core/benches/num/flt2dec/mod.rs +++ b/crux-mir/lib/core/benches/num/flt2dec/mod.rs @@ -5,7 +5,6 @@ mod strategy { use core::num::flt2dec::MAX_SIG_DIGITS; use core::num::flt2dec::{decode, DecodableFloat, Decoded, FullDecoded}; -use std::f64; use std::io::Write; use std::vec::Vec; use test::Bencher; @@ -13,7 +12,7 @@ use test::Bencher; pub fn decode_finite(v: T) -> Decoded { match decode(v).1 { FullDecoded::Finite(decoded) => decoded, - full_decoded => panic!("expected finite, got {:?} instead", full_decoded), + full_decoded => panic!("expected finite, got {full_decoded:?} instead"), } } diff --git a/crux-mir/lib/core/benches/num/flt2dec/strategy/dragon.rs b/crux-mir/lib/core/benches/num/flt2dec/strategy/dragon.rs index 4052fec33..319b9773e 100644 --- a/crux-mir/lib/core/benches/num/flt2dec/strategy/dragon.rs +++ b/crux-mir/lib/core/benches/num/flt2dec/strategy/dragon.rs @@ -1,60 +1,76 @@ use super::super::*; use core::num::flt2dec::strategy::dragon::*; -use std::{f64, i16}; +use std::mem::MaybeUninit; use test::Bencher; #[bench] fn bench_small_shortest(b: &mut Bencher) { let decoded = decode_finite(3.141592f64); - let mut buf = [0; MAX_SIG_DIGITS]; - b.iter(|| format_shortest(&decoded, &mut buf)); + let mut buf = [MaybeUninit::new(0); MAX_SIG_DIGITS]; + b.iter(|| { + format_shortest(&decoded, &mut buf); + }); } #[bench] fn bench_big_shortest(b: &mut Bencher) { let decoded = decode_finite(f64::MAX); - let mut buf = [0; MAX_SIG_DIGITS]; - b.iter(|| format_shortest(&decoded, &mut buf)); + let mut buf = [MaybeUninit::new(0); MAX_SIG_DIGITS]; + b.iter(|| { + format_shortest(&decoded, &mut buf); + }); } #[bench] fn bench_small_exact_3(b: &mut Bencher) { let decoded = decode_finite(3.141592f64); - let mut buf = [0; 3]; - b.iter(|| format_exact(&decoded, &mut buf, i16::MIN)); + let mut buf = [MaybeUninit::new(0); 3]; + b.iter(|| { + format_exact(&decoded, &mut buf, i16::MIN); + }); } #[bench] fn bench_big_exact_3(b: &mut Bencher) { let decoded = decode_finite(f64::MAX); - let mut buf = [0; 3]; - b.iter(|| format_exact(&decoded, &mut buf, i16::MIN)); + let mut buf = [MaybeUninit::new(0); 3]; + b.iter(|| { + format_exact(&decoded, &mut buf, i16::MIN); + }); } #[bench] fn bench_small_exact_12(b: &mut Bencher) { let decoded = decode_finite(3.141592f64); - let mut buf = [0; 12]; - b.iter(|| format_exact(&decoded, &mut buf, i16::MIN)); + let mut buf = [MaybeUninit::new(0); 12]; + b.iter(|| { + format_exact(&decoded, &mut buf, i16::MIN); + }); } #[bench] fn bench_big_exact_12(b: &mut Bencher) { let decoded = decode_finite(f64::MAX); - let mut buf = [0; 12]; - b.iter(|| format_exact(&decoded, &mut buf, i16::MIN)); + let mut buf = [MaybeUninit::new(0); 12]; + b.iter(|| { + format_exact(&decoded, &mut buf, i16::MIN); + }); } #[bench] fn bench_small_exact_inf(b: &mut Bencher) { let decoded = decode_finite(3.141592f64); - let mut buf = [0; 1024]; - b.iter(|| format_exact(&decoded, &mut buf, i16::MIN)); + let mut buf = [MaybeUninit::new(0); 1024]; + b.iter(|| { + format_exact(&decoded, &mut buf, i16::MIN); + }); } #[bench] fn bench_big_exact_inf(b: &mut Bencher) { let decoded = decode_finite(f64::MAX); - let mut buf = [0; 1024]; - b.iter(|| format_exact(&decoded, &mut buf, i16::MIN)); + let mut buf = [MaybeUninit::new(0); 1024]; + b.iter(|| { + format_exact(&decoded, &mut buf, i16::MIN); + }); } diff --git a/crux-mir/lib/core/benches/num/flt2dec/strategy/grisu.rs b/crux-mir/lib/core/benches/num/flt2dec/strategy/grisu.rs index 495074747..8e47a046c 100644 --- a/crux-mir/lib/core/benches/num/flt2dec/strategy/grisu.rs +++ b/crux-mir/lib/core/benches/num/flt2dec/strategy/grisu.rs @@ -1,67 +1,83 @@ use super::super::*; use core::num::flt2dec::strategy::grisu::*; -use std::{f64, i16}; +use std::mem::MaybeUninit; use test::Bencher; pub fn decode_finite(v: T) -> Decoded { match decode(v).1 { FullDecoded::Finite(decoded) => decoded, - full_decoded => panic!("expected finite, got {:?} instead", full_decoded), + full_decoded => panic!("expected finite, got {full_decoded:?} instead"), } } #[bench] fn bench_small_shortest(b: &mut Bencher) { let decoded = decode_finite(3.141592f64); - let mut buf = [0; MAX_SIG_DIGITS]; - b.iter(|| format_shortest(&decoded, &mut buf)); + let mut buf = [MaybeUninit::new(0); MAX_SIG_DIGITS]; + b.iter(|| { + format_shortest(&decoded, &mut buf); + }); } #[bench] fn bench_big_shortest(b: &mut Bencher) { let decoded = decode_finite(f64::MAX); - let mut buf = [0; MAX_SIG_DIGITS]; - b.iter(|| format_shortest(&decoded, &mut buf)); + let mut buf = [MaybeUninit::new(0); MAX_SIG_DIGITS]; + b.iter(|| { + format_shortest(&decoded, &mut buf); + }); } #[bench] fn bench_small_exact_3(b: &mut Bencher) { let decoded = decode_finite(3.141592f64); - let mut buf = [0; 3]; - b.iter(|| format_exact(&decoded, &mut buf, i16::MIN)); + let mut buf = [MaybeUninit::new(0); 3]; + b.iter(|| { + format_exact(&decoded, &mut buf, i16::MIN); + }); } #[bench] fn bench_big_exact_3(b: &mut Bencher) { let decoded = decode_finite(f64::MAX); - let mut buf = [0; 3]; - b.iter(|| format_exact(&decoded, &mut buf, i16::MIN)); + let mut buf = [MaybeUninit::new(0); 3]; + b.iter(|| { + format_exact(&decoded, &mut buf, i16::MIN); + }); } #[bench] fn bench_small_exact_12(b: &mut Bencher) { let decoded = decode_finite(3.141592f64); - let mut buf = [0; 12]; - b.iter(|| format_exact(&decoded, &mut buf, i16::MIN)); + let mut buf = [MaybeUninit::new(0); 12]; + b.iter(|| { + format_exact(&decoded, &mut buf, i16::MIN); + }); } #[bench] fn bench_big_exact_12(b: &mut Bencher) { let decoded = decode_finite(f64::MAX); - let mut buf = [0; 12]; - b.iter(|| format_exact(&decoded, &mut buf, i16::MIN)); + let mut buf = [MaybeUninit::new(0); 12]; + b.iter(|| { + format_exact(&decoded, &mut buf, i16::MIN); + }); } #[bench] fn bench_small_exact_inf(b: &mut Bencher) { let decoded = decode_finite(3.141592f64); - let mut buf = [0; 1024]; - b.iter(|| format_exact(&decoded, &mut buf, i16::MIN)); + let mut buf = [MaybeUninit::new(0); 1024]; + b.iter(|| { + format_exact(&decoded, &mut buf, i16::MIN); + }); } #[bench] fn bench_big_exact_inf(b: &mut Bencher) { let decoded = decode_finite(f64::MAX); - let mut buf = [0; 1024]; - b.iter(|| format_exact(&decoded, &mut buf, i16::MIN)); + let mut buf = [MaybeUninit::new(0); 1024]; + b.iter(|| { + format_exact(&decoded, &mut buf, i16::MIN); + }); } diff --git a/crux-mir/lib/core/benches/num/int_log/mod.rs b/crux-mir/lib/core/benches/num/int_log/mod.rs new file mode 100644 index 000000000..bb61224b5 --- /dev/null +++ b/crux-mir/lib/core/benches/num/int_log/mod.rs @@ -0,0 +1,58 @@ +use rand::Rng; +use test::{black_box, Bencher}; + +macro_rules! int_log_bench { + ($t:ty, $predictable:ident, $random:ident, $random_small:ident) => { + #[bench] + fn $predictable(bench: &mut Bencher) { + bench.iter(|| { + for n in 0..(<$t>::BITS / 8) { + for i in 1..=(100 as $t) { + let x = black_box(i << (n * 8)); + black_box(x.ilog10()); + } + } + }); + } + + #[bench] + fn $random(bench: &mut Bencher) { + let mut rng = crate::bench_rng(); + /* Exponentially distributed random numbers from the whole range of the type. */ + let numbers: Vec<$t> = (0..256) + .map(|_| { + let x = rng.gen::<$t>() >> rng.gen_range(0..<$t>::BITS); + if x != 0 { x } else { 1 } + }) + .collect(); + bench.iter(|| { + for x in &numbers { + black_box(black_box(x).ilog10()); + } + }); + } + + #[bench] + fn $random_small(bench: &mut Bencher) { + let mut rng = crate::bench_rng(); + /* Exponentially distributed random numbers from the range 0..256. */ + let numbers: Vec<$t> = (0..256) + .map(|_| { + let x = (rng.gen::() >> rng.gen_range(0..u8::BITS)) as $t; + if x != 0 { x } else { 1 } + }) + .collect(); + bench.iter(|| { + for x in &numbers { + black_box(black_box(x).ilog10()); + } + }); + } + }; +} + +int_log_bench! {u8, u8_log10_predictable, u8_log10_random, u8_log10_random_small} +int_log_bench! {u16, u16_log10_predictable, u16_log10_random, u16_log10_random_small} +int_log_bench! {u32, u32_log10_predictable, u32_log10_random, u32_log10_random_small} +int_log_bench! {u64, u64_log10_predictable, u64_log10_random, u64_log10_random_small} +int_log_bench! {u128, u128_log10_predictable, u128_log10_random, u128_log10_random_small} diff --git a/crux-mir/lib/core/benches/num/mod.rs b/crux-mir/lib/core/benches/num/mod.rs index 852d4e481..2f9cad272 100644 --- a/crux-mir/lib/core/benches/num/mod.rs +++ b/crux-mir/lib/core/benches/num/mod.rs @@ -1,5 +1,6 @@ mod dec2flt; mod flt2dec; +mod int_log; use std::str::FromStr; use test::Bencher; diff --git a/crux-mir/lib/core/benches/slice.rs b/crux-mir/lib/core/benches/slice.rs index 06b37cb08..9b86a0ca9 100644 --- a/crux-mir/lib/core/benches/slice.rs +++ b/crux-mir/lib/core/benches/slice.rs @@ -7,15 +7,21 @@ enum Cache { L3, } +impl Cache { + fn size(&self) -> usize { + match self { + Cache::L1 => 1000, // 8kb + Cache::L2 => 10_000, // 80kb + Cache::L3 => 1_000_000, // 8Mb + } + } +} + fn binary_search(b: &mut Bencher, cache: Cache, mapper: F) where F: Fn(usize) -> usize, { - let size = match cache { - Cache::L1 => 1000, // 8kb - Cache::L2 => 10_000, // 80kb - Cache::L3 => 1_000_000, // 8Mb - }; + let size = cache.size(); let v = (0..size).map(&mapper).collect::>(); let mut r = 0usize; b.iter(move || { @@ -24,7 +30,18 @@ where // Lookup the whole range to get 50% hits and 50% misses. let i = mapper(r % size); black_box(v.binary_search(&i).is_ok()); - }) + }); +} + +fn binary_search_worst_case(b: &mut Bencher, cache: Cache) { + let size = cache.size(); + + let mut v = vec![0; size]; + let i = 1; + v[size - 1] = i; + b.iter(move || { + black_box(v.binary_search(&i).is_ok()); + }); } #[bench] @@ -57,6 +74,30 @@ fn binary_search_l3_with_dups(b: &mut Bencher) { binary_search(b, Cache::L3, |i| i / 16 * 16); } +#[bench] +fn binary_search_l1_worst_case(b: &mut Bencher) { + binary_search_worst_case(b, Cache::L1); +} + +#[bench] +fn binary_search_l2_worst_case(b: &mut Bencher) { + binary_search_worst_case(b, Cache::L2); +} + +#[bench] +fn binary_search_l3_worst_case(b: &mut Bencher) { + binary_search_worst_case(b, Cache::L3); +} + +#[derive(Clone)] +struct Rgb(u8, u8, u8); + +impl Rgb { + fn gen(i: usize) -> Self { + Rgb(i as u8, (i as u8).wrapping_add(7), (i as u8).wrapping_add(42)) + } +} + macro_rules! rotate { ($fn:ident, $n:expr, $mapper:expr) => { #[bench] @@ -72,13 +113,52 @@ macro_rules! rotate { }; } -#[derive(Clone)] -struct Rgb(u8, u8, u8); - rotate!(rotate_u8, 32, |i| i as u8); -rotate!(rotate_rgb, 32, |i| Rgb(i as u8, (i as u8).wrapping_add(7), (i as u8).wrapping_add(42))); +rotate!(rotate_rgb, 32, Rgb::gen); rotate!(rotate_usize, 32, |i| i); rotate!(rotate_16_usize_4, 16, |i| [i; 4]); rotate!(rotate_16_usize_5, 16, |i| [i; 5]); rotate!(rotate_64_usize_4, 64, |i| [i; 4]); rotate!(rotate_64_usize_5, 64, |i| [i; 5]); + +macro_rules! swap_with_slice { + ($fn:ident, $n:expr, $mapper:expr) => { + #[bench] + fn $fn(b: &mut Bencher) { + let mut x = (0usize..$n).map(&$mapper).collect::>(); + let mut y = ($n..($n * 2)).map(&$mapper).collect::>(); + let mut skip = 0; + b.iter(|| { + for _ in 0..32 { + x[skip..].swap_with_slice(&mut y[..($n - skip)]); + skip = black_box(skip + 1) % 8; + } + black_box((x[$n / 3].clone(), y[$n * 2 / 3].clone())) + }) + } + }; +} + +swap_with_slice!(swap_with_slice_u8_30, 30, |i| i as u8); +swap_with_slice!(swap_with_slice_u8_3000, 3000, |i| i as u8); +swap_with_slice!(swap_with_slice_rgb_30, 30, Rgb::gen); +swap_with_slice!(swap_with_slice_rgb_3000, 3000, Rgb::gen); +swap_with_slice!(swap_with_slice_usize_30, 30, |i| i); +swap_with_slice!(swap_with_slice_usize_3000, 3000, |i| i); +swap_with_slice!(swap_with_slice_4x_usize_30, 30, |i| [i; 4]); +swap_with_slice!(swap_with_slice_4x_usize_3000, 3000, |i| [i; 4]); +swap_with_slice!(swap_with_slice_5x_usize_30, 30, |i| [i; 5]); +swap_with_slice!(swap_with_slice_5x_usize_3000, 3000, |i| [i; 5]); + +#[bench] +fn fill_byte_sized(b: &mut Bencher) { + #[derive(Copy, Clone)] + struct NewType(u8); + + let mut ary = [NewType(0); 1024]; + + b.iter(|| { + let slice = &mut ary[..]; + black_box(slice.fill(black_box(NewType(42)))); + }); +} diff --git a/crux-mir/lib/core/benches/str.rs b/crux-mir/lib/core/benches/str.rs new file mode 100644 index 000000000..78865d81f --- /dev/null +++ b/crux-mir/lib/core/benches/str.rs @@ -0,0 +1,10 @@ +use std::str; +use test::{black_box, Bencher}; + +mod char_count; +mod corpora; + +#[bench] +fn str_validate_emoji(b: &mut Bencher) { + b.iter(|| str::from_utf8(black_box(corpora::emoji::LARGE.as_bytes()))); +} diff --git a/crux-mir/lib/core/benches/str/char_count.rs b/crux-mir/lib/core/benches/str/char_count.rs new file mode 100644 index 000000000..25d9b2e29 --- /dev/null +++ b/crux-mir/lib/core/benches/str/char_count.rs @@ -0,0 +1,107 @@ +use super::corpora::*; +use test::{black_box, Bencher}; + +macro_rules! define_benches { + ($( fn $name: ident($arg: ident: &str) $body: block )+) => { + define_benches!(mod en_tiny, en::TINY, $($name $arg $body)+); + define_benches!(mod en_small, en::SMALL, $($name $arg $body)+); + define_benches!(mod en_medium, en::MEDIUM, $($name $arg $body)+); + define_benches!(mod en_large, en::LARGE, $($name $arg $body)+); + define_benches!(mod en_huge, en::HUGE, $($name $arg $body)+); + + define_benches!(mod zh_tiny, zh::TINY, $($name $arg $body)+); + define_benches!(mod zh_small, zh::SMALL, $($name $arg $body)+); + define_benches!(mod zh_medium, zh::MEDIUM, $($name $arg $body)+); + define_benches!(mod zh_large, zh::LARGE, $($name $arg $body)+); + define_benches!(mod zh_huge, zh::HUGE, $($name $arg $body)+); + + define_benches!(mod ru_tiny, ru::TINY, $($name $arg $body)+); + define_benches!(mod ru_small, ru::SMALL, $($name $arg $body)+); + define_benches!(mod ru_medium, ru::MEDIUM, $($name $arg $body)+); + define_benches!(mod ru_large, ru::LARGE, $($name $arg $body)+); + define_benches!(mod ru_huge, ru::HUGE, $($name $arg $body)+); + + define_benches!(mod emoji_tiny, emoji::TINY, $($name $arg $body)+); + define_benches!(mod emoji_small, emoji::SMALL, $($name $arg $body)+); + define_benches!(mod emoji_medium, emoji::MEDIUM, $($name $arg $body)+); + define_benches!(mod emoji_large, emoji::LARGE, $($name $arg $body)+); + define_benches!(mod emoji_huge, emoji::HUGE, $($name $arg $body)+); + }; + (mod $mod_name: ident, $input: expr, $($name: ident $arg: ident $body: block)+) => { + mod $mod_name { + use super::*; + $( + #[bench] + fn $name(bencher: &mut Bencher) { + let input = $input; + bencher.bytes = input.len() as u64; + let mut input_s = input.to_string(); + bencher.iter(|| { + let $arg: &str = &black_box(&mut input_s); + black_box($body) + }) + } + )+ + } + }; +} + +define_benches! { + fn case00_libcore(s: &str) { + libcore(s) + } + + fn case01_filter_count_cont_bytes(s: &str) { + filter_count_cont_bytes(s) + } + + fn case02_iter_increment(s: &str) { + iterator_increment(s) + } + + fn case03_manual_char_len(s: &str) { + manual_char_len(s) + } +} + +fn libcore(s: &str) -> usize { + s.chars().count() +} + +#[inline] +fn utf8_is_cont_byte(byte: u8) -> bool { + (byte as i8) < -64 +} + +fn filter_count_cont_bytes(s: &str) -> usize { + s.as_bytes().iter().filter(|&&byte| !utf8_is_cont_byte(byte)).count() +} + +fn iterator_increment(s: &str) -> usize { + let mut c = 0; + for _ in s.chars() { + c += 1; + } + c +} + +fn manual_char_len(s: &str) -> usize { + let s = s.as_bytes(); + let mut c = 0; + let mut i = 0; + let l = s.len(); + while i < l { + let b = s[i]; + if b < 0x80 { + i += 1; + } else if b < 0xe0 { + i += 2; + } else if b < 0xf0 { + i += 3; + } else { + i += 4; + } + c += 1; + } + c +} diff --git a/crux-mir/lib/core/benches/str/corpora.rs b/crux-mir/lib/core/benches/str/corpora.rs new file mode 100644 index 000000000..b4ac62506 --- /dev/null +++ b/crux-mir/lib/core/benches/str/corpora.rs @@ -0,0 +1,88 @@ +//! Exposes a number of modules with different kinds of strings. +//! +//! Each module contains `&str` constants named `TINY`, `SMALL`, `MEDIUM`, +//! `LARGE`, and `HUGE`. +//! +//! - The `TINY` string is generally around 8 bytes. +//! - The `SMALL` string is generally around 30-40 bytes. +//! - The `MEDIUM` string is generally around 600-700 bytes. +//! - The `LARGE` string is the `MEDIUM` string repeated 8x, and is around 5kb. +//! - The `HUGE` string is the `LARGE` string repeated 8x (or the `MEDIUM` +//! string repeated 64x), and is around 40kb. +//! +//! Except for `mod emoji` (which is just a bunch of emoji), the strings were +//! pulled from (localizations of) rust-lang.org. + +macro_rules! repeat8 { + ($s:expr) => { + concat!($s, $s, $s, $s, $s, $s, $s, $s) + }; +} + +macro_rules! define_consts { + ($s:literal) => { + pub const MEDIUM: &str = $s; + pub const LARGE: &str = repeat8!($s); + pub const HUGE: &str = repeat8!(repeat8!(repeat8!($s))); + }; +} + +pub mod en { + pub const TINY: &str = "Mary had"; + pub const SMALL: &str = "Mary had a little lamb, Little lamb"; + define_consts! { + "Rust is blazingly fast and memory-efficient: with no runtime or garbage + collector, it can power performance-critical services, run on embedded + devices, and easily integrate with other languages. Rust’s rich type system + and ownership model guarantee memory-safety and thread-safety — enabling you + to eliminate many classes of bugs at compile-time. Rust has great + documentation, a friendly compiler with useful error messages, and top-notch + tooling — an integrated package manager and build tool, smart multi-editor + support with auto-completion and type inspections, an auto-formatter, and + more." + } +} + +pub mod zh { + pub const TINY: &str = "速度惊"; + pub const SMALL: &str = "速度惊人且内存利用率极高"; + define_consts! { + "Rust 速度惊人且内存利用率极高。由于\ + 没有运行时和垃圾回收，它能够胜任对性能要\ + 求特别高的服务，可以在嵌入式设备上运行，\ + 还能轻松和其他语言集成。Rust 丰富的类型\ + 系统和所有权模型保证了内存安全和线程安全，\ + 让您在编译期就能够消除各种各样的错误。\ + Rust 拥有出色的文档、友好的编译器和清晰\ + 的错误提示信息， 还集成了一流的工具——\ + 包管理器和构建工具， 智能地自动补全和类\ + 型检验的多编辑器支持， 以及自动格式化代\ + 码等等。" + } +} + +pub mod ru { + pub const TINY: &str = "Сотни"; + pub const SMALL: &str = "Сотни компаний по"; + define_consts! { + "Сотни компаний по всему миру используют Rust в реальных\ + проектах для быстрых кросс-платформенных решений с\ + ограниченными ресурсами. Такие проекты, как Firefox,\ + Dropbox и Cloudflare, используют Rust. Rust отлично\ + подходит как для стартапов, так и для больших компаний,\ + как для встраиваемых устройств, так и для масштабируемых\ + web-сервисов. Мой самый большой комплимент Rust." + } +} + +pub mod emoji { + pub const TINY: &str = "😀😃"; + pub const SMALL: &str = "😀😃😄😁😆😅🤣😂🙂🙃😉😊😇🥰😍🤩😘"; + define_consts! { + "😀😃😄😁😆😅🤣😂🙂🙃😉😊😇🥰😍🤩😘😗☺😚😙🥲😋😛😜🤪😝🤑🤗🤭🤫🤔🤐🤨😐😑😶😶‍🌫️😏😒\ + 🙄😬😮‍💨🤥😌😔😪🤤😴😷🤒🤕🤢🤮🤧🥵🥶🥴😵😵‍💫🤯��🥳🥸😎🤓🧐😕😟🙁☹😮😯😲😳🥺😦😧😨\ + 😰😥😢😭😱😖😣😞😓😩😫🥱😤😡😠🤬😈👿💀☠💩🤡👹👺👻👽👾🤖😺😸😹😻😼😽🙀😿😾🙈🙉🙊\ + 💋💌💘💝💖💗💓��💕💟❣💔❤️‍🔥❤️‍🩹❤🧡💛💚💙💜🤎🖤🤍💯💢💥💫💦💨🕳💬👁️‍🗨️🗨🗯💭💤👋\ + 🤚🖐✋🖖👌🤌🤏✌" + } +} diff --git a/crux-mir/lib/core/primitive_docs/box_into_raw.md b/crux-mir/lib/core/primitive_docs/box_into_raw.md new file mode 100644 index 000000000..9dd0344c7 --- /dev/null +++ b/crux-mir/lib/core/primitive_docs/box_into_raw.md @@ -0,0 +1 @@ +../std/boxed/struct.Box.html#method.into_raw diff --git a/crux-mir/lib/core/primitive_docs/fs_file.md b/crux-mir/lib/core/primitive_docs/fs_file.md new file mode 100644 index 000000000..4023e340a --- /dev/null +++ b/crux-mir/lib/core/primitive_docs/fs_file.md @@ -0,0 +1 @@ +../std/fs/struct.File.html diff --git a/crux-mir/lib/core/primitive_docs/io_bufread.md b/crux-mir/lib/core/primitive_docs/io_bufread.md new file mode 100644 index 000000000..7beda2cd3 --- /dev/null +++ b/crux-mir/lib/core/primitive_docs/io_bufread.md @@ -0,0 +1 @@ +../std/io/trait.BufRead.html diff --git a/crux-mir/lib/core/primitive_docs/io_read.md b/crux-mir/lib/core/primitive_docs/io_read.md new file mode 100644 index 000000000..b7ecf5e27 --- /dev/null +++ b/crux-mir/lib/core/primitive_docs/io_read.md @@ -0,0 +1 @@ +../std/io/trait.Read.html diff --git a/crux-mir/lib/core/primitive_docs/io_seek.md b/crux-mir/lib/core/primitive_docs/io_seek.md new file mode 100644 index 000000000..db0274d29 --- /dev/null +++ b/crux-mir/lib/core/primitive_docs/io_seek.md @@ -0,0 +1 @@ +../std/io/trait.Seek.html diff --git a/crux-mir/lib/core/primitive_docs/io_write.md b/crux-mir/lib/core/primitive_docs/io_write.md new file mode 100644 index 000000000..92a3b88a7 --- /dev/null +++ b/crux-mir/lib/core/primitive_docs/io_write.md @@ -0,0 +1 @@ +../std/io/trait.Write.html diff --git a/crux-mir/lib/core/primitive_docs/net_tosocketaddrs.md b/crux-mir/lib/core/primitive_docs/net_tosocketaddrs.md new file mode 100644 index 000000000..4daa10ddb --- /dev/null +++ b/crux-mir/lib/core/primitive_docs/net_tosocketaddrs.md @@ -0,0 +1 @@ +../std/net/trait.ToSocketAddrs.html diff --git a/crux-mir/lib/core/primitive_docs/process_exit.md b/crux-mir/lib/core/primitive_docs/process_exit.md new file mode 100644 index 000000000..cae34d12d --- /dev/null +++ b/crux-mir/lib/core/primitive_docs/process_exit.md @@ -0,0 +1 @@ +../std/process/fn.exit.html diff --git a/crux-mir/lib/core/primitive_docs/string_string.md b/crux-mir/lib/core/primitive_docs/string_string.md new file mode 100644 index 000000000..303dc07b1 --- /dev/null +++ b/crux-mir/lib/core/primitive_docs/string_string.md @@ -0,0 +1 @@ +../std/string/struct.String.html diff --git a/crux-mir/lib/core/src/Cargo.toml b/crux-mir/lib/core/src/Cargo.toml deleted file mode 100644 index ac07ffb14..000000000 --- a/crux-mir/lib/core/src/Cargo.toml +++ /dev/null @@ -1,28 +0,0 @@ -[package] -authors = ["The Rust Project Developers"] -name = "core" -version = "0.0.0" -autotests = false -autobenches = false -edition = "2018" - -[lib] -name = "core" -path = "lib.rs" -test = false -bench = false - -[[test]] -name = "coretests" -path = "../libcore/tests/lib.rs" - -[[bench]] -name = "corebenches" -path = "../libcore/benches/lib.rs" - -[dev-dependencies] -rand = "0.7" - -[features] -# Make panics and failed asserts immediately abort without formatting any message -panic_immediate_abort = [] diff --git a/crux-mir/lib/core/src/alloc.rs b/crux-mir/lib/core/src/alloc.rs deleted file mode 100644 index d2a513451..000000000 --- a/crux-mir/lib/core/src/alloc.rs +++ /dev/null @@ -1,1007 +0,0 @@ -//! Memory allocation APIs - -// ignore-tidy-undocumented-unsafe - -#![stable(feature = "alloc_module", since = "1.28.0")] - -use crate::cmp; -use crate::fmt; -use crate::mem; -use crate::num::NonZeroUsize; -use crate::ptr::{self, NonNull}; -use crate::usize; - -const fn size_align() -> (usize, usize) { - (mem::size_of::(), mem::align_of::()) -} - -/// Layout of a block of memory. -/// -/// An instance of `Layout` describes a particular layout of memory. -/// You build a `Layout` up as an input to give to an allocator. -/// -/// All layouts have an associated non-negative size and a -/// power-of-two alignment. -/// -/// (Note however that layouts are *not* required to have positive -/// size, even though many allocators require that all memory -/// requests have positive size. A caller to the `AllocRef::alloc` -/// method must either ensure that conditions like this are met, or -/// use specific allocators with looser requirements.) -#[stable(feature = "alloc_layout", since = "1.28.0")] -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -#[lang = "alloc_layout"] -pub struct Layout { - // size of the requested block of memory, measured in bytes. - size_: usize, - - // alignment of the requested block of memory, measured in bytes. - // we ensure that this is always a power-of-two, because API's - // like `posix_memalign` require it and it is a reasonable - // constraint to impose on Layout constructors. - // - // (However, we do not analogously require `align >= sizeof(void*)`, - // even though that is *also* a requirement of `posix_memalign`.) - align_: NonZeroUsize, -} - -impl Layout { - /// Constructs a `Layout` from a given `size` and `align`, - /// or returns `LayoutErr` if any of the following conditions - /// are not met: - /// - /// * `align` must not be zero, - /// - /// * `align` must be a power of two, - /// - /// * `size`, when rounded up to the nearest multiple of `align`, - /// must not overflow (i.e., the rounded value must be less than - /// `usize::MAX`). - #[stable(feature = "alloc_layout", since = "1.28.0")] - #[rustc_const_unstable(feature = "const_alloc_layout", issue = "67521")] - #[inline] - pub const fn from_size_align(size: usize, align: usize) -> Result { - if !align.is_power_of_two() { - return Err(LayoutErr { private: () }); - } - - // (power-of-two implies align != 0.) - - // Rounded up size is: - // size_rounded_up = (size + align - 1) & !(align - 1); - // - // We know from above that align != 0. If adding (align - 1) - // does not overflow, then rounding up will be fine. - // - // Conversely, &-masking with !(align - 1) will subtract off - // only low-order-bits. Thus if overflow occurs with the sum, - // the &-mask cannot subtract enough to undo that overflow. - // - // Above implies that checking for summation overflow is both - // necessary and sufficient. - if size > usize::MAX - (align - 1) { - return Err(LayoutErr { private: () }); - } - - unsafe { Ok(Layout::from_size_align_unchecked(size, align)) } - } - - /// Creates a layout, bypassing all checks. - /// - /// # Safety - /// - /// This function is unsafe as it does not verify the preconditions from - /// [`Layout::from_size_align`](#method.from_size_align). - #[stable(feature = "alloc_layout", since = "1.28.0")] - #[rustc_const_stable(feature = "alloc_layout", since = "1.28.0")] - #[inline] - pub const unsafe fn from_size_align_unchecked(size: usize, align: usize) -> Self { - Layout { size_: size, align_: NonZeroUsize::new_unchecked(align) } - } - - /// The minimum size in bytes for a memory block of this layout. - #[stable(feature = "alloc_layout", since = "1.28.0")] - #[rustc_const_unstable(feature = "const_alloc_layout", issue = "67521")] - #[inline] - pub const fn size(&self) -> usize { - self.size_ - } - - /// The minimum byte alignment for a memory block of this layout. - #[stable(feature = "alloc_layout", since = "1.28.0")] - #[rustc_const_unstable(feature = "const_alloc_layout", issue = "67521")] - #[inline] - pub const fn align(&self) -> usize { - self.align_.get() - } - - /// Constructs a `Layout` suitable for holding a value of type `T`. - #[stable(feature = "alloc_layout", since = "1.28.0")] - #[rustc_const_stable(feature = "alloc_layout_const_new", since = "1.42.0")] - #[inline] - pub const fn new() -> Self { - let (size, align) = size_align::(); - // Note that the align is guaranteed by rustc to be a power of two and - // the size+align combo is guaranteed to fit in our address space. As a - // result use the unchecked constructor here to avoid inserting code - // that panics if it isn't optimized well enough. - unsafe { Layout::from_size_align_unchecked(size, align) } - } - - /// Produces layout describing a record that could be used to - /// allocate backing structure for `T` (which could be a trait - /// or other unsized type like a slice). - #[stable(feature = "alloc_layout", since = "1.28.0")] - #[inline] - pub fn for_value(t: &T) -> Self { - let (size, align) = (mem::size_of_val(t), mem::align_of_val(t)); - // See rationale in `new` for why this is using an unsafe variant below - debug_assert!(Layout::from_size_align(size, align).is_ok()); - unsafe { Layout::from_size_align_unchecked(size, align) } - } - - /// Creates a `NonNull` that is dangling, but well-aligned for this Layout. - /// - /// Note that the pointer value may potentially represent a valid pointer, - /// which means this must not be used as a "not yet initialized" - /// sentinel value. Types that lazily allocate must track initialization by - /// some other means. - #[unstable(feature = "alloc_layout_extra", issue = "55724")] - pub const fn dangling(&self) -> NonNull { - // align is non-zero and a power of two - unsafe { NonNull::new_unchecked(self.align() as *mut u8) } - } - - /// Creates a layout describing the record that can hold a value - /// of the same layout as `self`, but that also is aligned to - /// alignment `align` (measured in bytes). - /// - /// If `self` already meets the prescribed alignment, then returns - /// `self`. - /// - /// Note that this method does not add any padding to the overall - /// size, regardless of whether the returned layout has a different - /// alignment. In other words, if `K` has size 16, `K.align_to(32)` - /// will *still* have size 16. - /// - /// Returns an error if the combination of `self.size()` and the given - /// `align` violates the conditions listed in - /// [`Layout::from_size_align`](#method.from_size_align). - #[unstable(feature = "alloc_layout_extra", issue = "55724")] - #[inline] - pub fn align_to(&self, align: usize) -> Result { - Layout::from_size_align(self.size(), cmp::max(self.align(), align)) - } - - /// Returns the amount of padding we must insert after `self` - /// to ensure that the following address will satisfy `align` - /// (measured in bytes). - /// - /// e.g., if `self.size()` is 9, then `self.padding_needed_for(4)` - /// returns 3, because that is the minimum number of bytes of - /// padding required to get a 4-aligned address (assuming that the - /// corresponding memory block starts at a 4-aligned address). - /// - /// The return value of this function has no meaning if `align` is - /// not a power-of-two. - /// - /// Note that the utility of the returned value requires `align` - /// to be less than or equal to the alignment of the starting - /// address for the whole allocated block of memory. One way to - /// satisfy this constraint is to ensure `align <= self.align()`. - #[unstable(feature = "alloc_layout_extra", issue = "55724")] - #[rustc_const_unstable(feature = "const_alloc_layout", issue = "67521")] - #[inline] - pub const fn padding_needed_for(&self, align: usize) -> usize { - let len = self.size(); - - // Rounded up value is: - // len_rounded_up = (len + align - 1) & !(align - 1); - // and then we return the padding difference: `len_rounded_up - len`. - // - // We use modular arithmetic throughout: - // - // 1. align is guaranteed to be > 0, so align - 1 is always - // valid. - // - // 2. `len + align - 1` can overflow by at most `align - 1`, - // so the &-mask with `!(align - 1)` will ensure that in the - // case of overflow, `len_rounded_up` will itself be 0. - // Thus the returned padding, when added to `len`, yields 0, - // which trivially satisfies the alignment `align`. - // - // (Of course, attempts to allocate blocks of memory whose - // size and padding overflow in the above manner should cause - // the allocator to yield an error anyway.) - - let len_rounded_up = len.wrapping_add(align).wrapping_sub(1) & !align.wrapping_sub(1); - len_rounded_up.wrapping_sub(len) - } - - /// Creates a layout by rounding the size of this layout up to a multiple - /// of the layout's alignment. - /// - /// This is equivalent to adding the result of `padding_needed_for` - /// to the layout's current size. - #[unstable(feature = "alloc_layout_extra", issue = "55724")] - #[inline] - pub fn pad_to_align(&self) -> Layout { - let pad = self.padding_needed_for(self.align()); - // This cannot overflow. Quoting from the invariant of Layout: - // > `size`, when rounded up to the nearest multiple of `align`, - // > must not overflow (i.e., the rounded value must be less than - // > `usize::MAX`) - let new_size = self.size() + pad; - - Layout::from_size_align(new_size, self.align()).unwrap() - } - - /// Creates a layout describing the record for `n` instances of - /// `self`, with a suitable amount of padding between each to - /// ensure that each instance is given its requested size and - /// alignment. On success, returns `(k, offs)` where `k` is the - /// layout of the array and `offs` is the distance between the start - /// of each element in the array. - /// - /// On arithmetic overflow, returns `LayoutErr`. - #[unstable(feature = "alloc_layout_extra", issue = "55724")] - #[inline] - pub fn repeat(&self, n: usize) -> Result<(Self, usize), LayoutErr> { - // This cannot overflow. Quoting from the invariant of Layout: - // > `size`, when rounded up to the nearest multiple of `align`, - // > must not overflow (i.e., the rounded value must be less than - // > `usize::MAX`) - let padded_size = self.size() + self.padding_needed_for(self.align()); - let alloc_size = padded_size.checked_mul(n).ok_or(LayoutErr { private: () })?; - - unsafe { - // self.align is already known to be valid and alloc_size has been - // padded already. - Ok((Layout::from_size_align_unchecked(alloc_size, self.align()), padded_size)) - } - } - - /// Creates a layout describing the record for `self` followed by - /// `next`, including any necessary padding to ensure that `next` - /// will be properly aligned. Note that the resulting layout will - /// satisfy the alignment properties of both `self` and `next`. - /// - /// The resulting layout will be the same as that of a C struct containing - /// two fields with the layouts of `self` and `next`, in that order. - /// - /// Returns `Some((k, offset))`, where `k` is layout of the concatenated - /// record and `offset` is the relative location, in bytes, of the - /// start of the `next` embedded within the concatenated record - /// (assuming that the record itself starts at offset 0). - /// - /// On arithmetic overflow, returns `LayoutErr`. - #[unstable(feature = "alloc_layout_extra", issue = "55724")] - #[inline] - pub fn extend(&self, next: Self) -> Result<(Self, usize), LayoutErr> { - let new_align = cmp::max(self.align(), next.align()); - let pad = self.padding_needed_for(next.align()); - - let offset = self.size().checked_add(pad).ok_or(LayoutErr { private: () })?; - let new_size = offset.checked_add(next.size()).ok_or(LayoutErr { private: () })?; - - let layout = Layout::from_size_align(new_size, new_align)?; - Ok((layout, offset)) - } - - /// Creates a layout describing the record for `n` instances of - /// `self`, with no padding between each instance. - /// - /// Note that, unlike `repeat`, `repeat_packed` does not guarantee - /// that the repeated instances of `self` will be properly - /// aligned, even if a given instance of `self` is properly - /// aligned. In other words, if the layout returned by - /// `repeat_packed` is used to allocate an array, it is not - /// guaranteed that all elements in the array will be properly - /// aligned. - /// - /// On arithmetic overflow, returns `LayoutErr`. - #[unstable(feature = "alloc_layout_extra", issue = "55724")] - #[inline] - pub fn repeat_packed(&self, n: usize) -> Result { - let size = self.size().checked_mul(n).ok_or(LayoutErr { private: () })?; - Layout::from_size_align(size, self.align()) - } - - /// Creates a layout describing the record for `self` followed by - /// `next` with no additional padding between the two. Since no - /// padding is inserted, the alignment of `next` is irrelevant, - /// and is not incorporated *at all* into the resulting layout. - /// - /// On arithmetic overflow, returns `LayoutErr`. - #[unstable(feature = "alloc_layout_extra", issue = "55724")] - #[inline] - pub fn extend_packed(&self, next: Self) -> Result { - let new_size = self.size().checked_add(next.size()).ok_or(LayoutErr { private: () })?; - Layout::from_size_align(new_size, self.align()) - } - - /// Creates a layout describing the record for a `[T; n]`. - /// - /// On arithmetic overflow, returns `LayoutErr`. - #[unstable(feature = "alloc_layout_extra", issue = "55724")] - #[inline] - pub fn array(n: usize) -> Result { - Layout::new::().repeat(n).map(|(k, offs)| { - debug_assert!(offs == mem::size_of::()); - k - }) - } -} - -/// The parameters given to `Layout::from_size_align` -/// or some other `Layout` constructor -/// do not satisfy its documented constraints. -#[stable(feature = "alloc_layout", since = "1.28.0")] -#[derive(Clone, PartialEq, Eq, Debug)] -pub struct LayoutErr { - private: (), -} - -// (we need this for downstream impl of trait Error) -#[stable(feature = "alloc_layout", since = "1.28.0")] -impl fmt::Display for LayoutErr { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("invalid parameters to Layout::from_size_align") - } -} - -/// The `AllocErr` error indicates an allocation failure -/// that may be due to resource exhaustion or to -/// something wrong when combining the given input arguments with this -/// allocator. -#[unstable(feature = "allocator_api", issue = "32838")] -#[derive(Clone, PartialEq, Eq, Debug)] -pub struct AllocErr; - -// (we need this for downstream impl of trait Error) -#[unstable(feature = "allocator_api", issue = "32838")] -impl fmt::Display for AllocErr { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("memory allocation failed") - } -} - -/// The `CannotReallocInPlace` error is used when [`grow_in_place`] or -/// [`shrink_in_place`] were unable to reuse the given memory block for -/// a requested layout. -/// -/// [`grow_in_place`]: ./trait.AllocRef.html#method.grow_in_place -/// [`shrink_in_place`]: ./trait.AllocRef.html#method.shrink_in_place -#[unstable(feature = "allocator_api", issue = "32838")] -#[derive(Clone, PartialEq, Eq, Debug)] -pub struct CannotReallocInPlace; - -#[unstable(feature = "allocator_api", issue = "32838")] -impl CannotReallocInPlace { - pub fn description(&self) -> &str { - "cannot reallocate allocator's memory in place" - } -} - -// (we need this for downstream impl of trait Error) -#[unstable(feature = "allocator_api", issue = "32838")] -impl fmt::Display for CannotReallocInPlace { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.description()) - } -} - -/// A memory allocator that can be registered as the standard library’s default -/// through the `#[global_allocator]` attribute. -/// -/// Some of the methods require that a memory block be *currently -/// allocated* via an allocator. This means that: -/// -/// * the starting address for that memory block was previously -/// returned by a previous call to an allocation method -/// such as `alloc`, and -/// -/// * the memory block has not been subsequently deallocated, where -/// blocks are deallocated either by being passed to a deallocation -/// method such as `dealloc` or by being -/// passed to a reallocation method that returns a non-null pointer. -/// -/// -/// # Example -/// -/// ```no_run -/// use std::alloc::{GlobalAlloc, Layout, alloc}; -/// use std::ptr::null_mut; -/// -/// struct MyAllocator; -/// -/// unsafe impl GlobalAlloc for MyAllocator { -/// unsafe fn alloc(&self, _layout: Layout) -> *mut u8 { null_mut() } -/// unsafe fn dealloc(&self, _ptr: *mut u8, _layout: Layout) {} -/// } -/// -/// #[global_allocator] -/// static A: MyAllocator = MyAllocator; -/// -/// fn main() { -/// unsafe { -/// assert!(alloc(Layout::new::()).is_null()) -/// } -/// } -/// ``` -/// -/// # Safety -/// -/// The `GlobalAlloc` trait is an `unsafe` trait for a number of reasons, and -/// implementors must ensure that they adhere to these contracts: -/// -/// * It's undefined behavior if global allocators unwind. This restriction may -/// be lifted in the future, but currently a panic from any of these -/// functions may lead to memory unsafety. -/// -/// * `Layout` queries and calculations in general must be correct. Callers of -/// this trait are allowed to rely on the contracts defined on each method, -/// and implementors must ensure such contracts remain true. -#[stable(feature = "global_alloc", since = "1.28.0")] -pub unsafe trait GlobalAlloc { - /// Allocate memory as described by the given `layout`. - /// - /// Returns a pointer to newly-allocated memory, - /// or null to indicate allocation failure. - /// - /// # Safety - /// - /// This function is unsafe because undefined behavior can result - /// if the caller does not ensure that `layout` has non-zero size. - /// - /// (Extension subtraits might provide more specific bounds on - /// behavior, e.g., guarantee a sentinel address or a null pointer - /// in response to a zero-size allocation request.) - /// - /// The allocated block of memory may or may not be initialized. - /// - /// # Errors - /// - /// Returning a null pointer indicates that either memory is exhausted - /// or `layout` does not meet this allocator's size or alignment constraints. - /// - /// Implementations are encouraged to return null on memory - /// exhaustion rather than aborting, but this is not - /// a strict requirement. (Specifically: it is *legal* to - /// implement this trait atop an underlying native allocation - /// library that aborts on memory exhaustion.) - /// - /// Clients wishing to abort computation in response to an - /// allocation error are encouraged to call the [`handle_alloc_error`] function, - /// rather than directly invoking `panic!` or similar. - /// - /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html - #[stable(feature = "global_alloc", since = "1.28.0")] - unsafe fn alloc(&self, layout: Layout) -> *mut u8; - - /// Deallocate the block of memory at the given `ptr` pointer with the given `layout`. - /// - /// # Safety - /// - /// This function is unsafe because undefined behavior can result - /// if the caller does not ensure all of the following: - /// - /// * `ptr` must denote a block of memory currently allocated via - /// this allocator, - /// - /// * `layout` must be the same layout that was used - /// to allocate that block of memory, - #[stable(feature = "global_alloc", since = "1.28.0")] - unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout); - - /// Behaves like `alloc`, but also ensures that the contents - /// are set to zero before being returned. - /// - /// # Safety - /// - /// This function is unsafe for the same reasons that `alloc` is. - /// However the allocated block of memory is guaranteed to be initialized. - /// - /// # Errors - /// - /// Returning a null pointer indicates that either memory is exhausted - /// or `layout` does not meet allocator's size or alignment constraints, - /// just as in `alloc`. - /// - /// Clients wishing to abort computation in response to an - /// allocation error are encouraged to call the [`handle_alloc_error`] function, - /// rather than directly invoking `panic!` or similar. - /// - /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html - #[stable(feature = "global_alloc", since = "1.28.0")] - unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { - let size = layout.size(); - let ptr = self.alloc(layout); - if !ptr.is_null() { - ptr::write_bytes(ptr, 0, size); - } - ptr - } - - /// Shrink or grow a block of memory to the given `new_size`. - /// The block is described by the given `ptr` pointer and `layout`. - /// - /// If this returns a non-null pointer, then ownership of the memory block - /// referenced by `ptr` has been transferred to this allocator. - /// The memory may or may not have been deallocated, - /// and should be considered unusable (unless of course it was - /// transferred back to the caller again via the return value of - /// this method). The new memory block is allocated with `layout`, but - /// with the `size` updated to `new_size`. - /// - /// If this method returns null, then ownership of the memory - /// block has not been transferred to this allocator, and the - /// contents of the memory block are unaltered. - /// - /// # Safety - /// - /// This function is unsafe because undefined behavior can result - /// if the caller does not ensure all of the following: - /// - /// * `ptr` must be currently allocated via this allocator, - /// - /// * `layout` must be the same layout that was used - /// to allocate that block of memory, - /// - /// * `new_size` must be greater than zero. - /// - /// * `new_size`, when rounded up to the nearest multiple of `layout.align()`, - /// must not overflow (i.e., the rounded value must be less than `usize::MAX`). - /// - /// (Extension subtraits might provide more specific bounds on - /// behavior, e.g., guarantee a sentinel address or a null pointer - /// in response to a zero-size allocation request.) - /// - /// # Errors - /// - /// Returns null if the new layout does not meet the size - /// and alignment constraints of the allocator, or if reallocation - /// otherwise fails. - /// - /// Implementations are encouraged to return null on memory - /// exhaustion rather than panicking or aborting, but this is not - /// a strict requirement. (Specifically: it is *legal* to - /// implement this trait atop an underlying native allocation - /// library that aborts on memory exhaustion.) - /// - /// Clients wishing to abort computation in response to a - /// reallocation error are encouraged to call the [`handle_alloc_error`] function, - /// rather than directly invoking `panic!` or similar. - /// - /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html - #[stable(feature = "global_alloc", since = "1.28.0")] - unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - let new_layout = Layout::from_size_align_unchecked(new_size, layout.align()); - let new_ptr = self.alloc(new_layout); - if !new_ptr.is_null() { - ptr::copy_nonoverlapping(ptr, new_ptr, cmp::min(layout.size(), new_size)); - self.dealloc(ptr, layout); - } - new_ptr - } -} - -/// An implementation of `AllocRef` can allocate, reallocate, and -/// deallocate arbitrary blocks of data described via `Layout`. -/// -/// `AllocRef` is designed to be implemented on ZSTs, references, or -/// smart pointers because having an allocator like `MyAlloc([u8; N])` -/// cannot be moved, without updating the pointers to the allocated -/// memory. -/// -/// Some of the methods require that a memory block be *currently -/// allocated* via an allocator. This means that: -/// -/// * the starting address for that memory block was previously -/// returned by a previous call to an allocation method (`alloc`, -/// `alloc_zeroed`) or reallocation method (`realloc`), and -/// -/// * the memory block has not been subsequently deallocated, where -/// blocks are deallocated either by being passed to a deallocation -/// method (`dealloc`) or by being passed to a reallocation method -/// (see above) that returns `Ok`. -/// -/// Unlike [`GlobalAlloc`], zero-sized allocations are allowed in -/// `AllocRef`. If an underlying allocator does not support this (like -/// jemalloc) or return a null pointer (such as `libc::malloc`), this case -/// must be caught. In this case [`Layout::dangling()`] can be used to -/// create a dangling, but aligned `NonNull`. -/// -/// Some of the methods require that a layout *fit* a memory block. -/// What it means for a layout to "fit" a memory block means (or -/// equivalently, for a memory block to "fit" a layout) is that the -/// following two conditions must hold: -/// -/// 1. The block's starting address must be aligned to `layout.align()`. -/// -/// 2. The block's size must fall in the range `[use_min, use_max]`, where: -/// -/// * `use_min` is `layout.size()`, and -/// -/// * `use_max` is the capacity that was returned. -/// -/// Note that: -/// -/// * the size of the layout most recently used to allocate the block -/// is guaranteed to be in the range `[use_min, use_max]`, and -/// -/// * a lower-bound on `use_max` can be safely approximated by a call to -/// `usable_size`. -/// -/// * if a layout `k` fits a memory block (denoted by `ptr`) -/// currently allocated via an allocator `a`, then it is legal to -/// use that layout to deallocate it, i.e., `a.dealloc(ptr, k);`. -/// -/// * if an allocator does not support overallocating, it is fine to -/// simply return `layout.size()` as the allocated size. -/// -/// [`GlobalAlloc`]: self::GlobalAlloc -/// [`Layout::dangling()`]: self::Layout::dangling -/// -/// # Safety -/// -/// The `AllocRef` trait is an `unsafe` trait for a number of reasons, and -/// implementors must ensure that they adhere to these contracts: -/// -/// * Pointers returned from allocation functions must point to valid memory and -/// retain their validity until at least one instance of `AllocRef` is dropped -/// itself. -/// -/// * Cloning or moving the allocator must not invalidate pointers returned -/// from this allocator. Cloning must return a reference to the same allocator. -/// -/// * `Layout` queries and calculations in general must be correct. Callers of -/// this trait are allowed to rely on the contracts defined on each method, -/// and implementors must ensure such contracts remain true. -/// -/// Note that this list may get tweaked over time as clarifications are made in -/// the future. -#[unstable(feature = "allocator_api", issue = "32838")] -pub unsafe trait AllocRef { - /// On success, returns a pointer meeting the size and alignment - /// guarantees of `layout` and the actual size of the allocated block, - /// which must be greater than or equal to `layout.size()`. - /// - /// If this method returns an `Ok(addr)`, then the `addr` returned - /// will be non-null address pointing to a block of storage - /// suitable for holding an instance of `layout`. - /// - /// The returned block of storage may or may not have its contents - /// initialized. (Extension subtraits might restrict this - /// behavior, e.g., to ensure initialization to particular sets of - /// bit patterns.) - /// - /// # Errors - /// - /// Returning `Err` indicates that either memory is exhausted or - /// `layout` does not meet allocator's size or alignment - /// constraints. - /// - /// Implementations are encouraged to return `Err` on memory - /// exhaustion rather than panicking or aborting, but this is not - /// a strict requirement. (Specifically: it is *legal* to - /// implement this trait atop an underlying native allocation - /// library that aborts on memory exhaustion.) - /// - /// Clients wishing to abort computation in response to an - /// allocation error are encouraged to call the [`handle_alloc_error`] function, - /// rather than directly invoking `panic!` or similar. - /// - /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html - fn alloc(&mut self, layout: Layout) -> Result<(NonNull, usize), AllocErr>; - - /// Deallocate the memory referenced by `ptr`. - /// - /// # Safety - /// - /// This function is unsafe because undefined behavior can result - /// if the caller does not ensure all of the following: - /// - /// * `ptr` must denote a block of memory currently allocated via - /// this allocator, - /// - /// * `layout` must *fit* that block of memory, - /// - /// * In addition to fitting the block of memory `layout`, the - /// alignment of the `layout` must match the alignment used - /// to allocate that block of memory. - unsafe fn dealloc(&mut self, ptr: NonNull, layout: Layout); - - /// Behaves like `alloc`, but also ensures that the contents - /// are set to zero before being returned. - /// - /// # Errors - /// - /// Returning `Err` indicates that either memory is exhausted or - /// `layout` does not meet allocator's size or alignment - /// constraints, just as in `alloc`. - /// - /// Clients wishing to abort computation in response to an - /// allocation error are encouraged to call the [`handle_alloc_error`] function, - /// rather than directly invoking `panic!` or similar. - /// - /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html - fn alloc_zeroed(&mut self, layout: Layout) -> Result<(NonNull, usize), AllocErr> { - let size = layout.size(); - let result = self.alloc(layout); - if let Ok((p, _)) = result { - unsafe { ptr::write_bytes(p.as_ptr(), 0, size) } - } - result - } - - // == METHODS FOR MEMORY REUSE == - // realloc, realloc_zeroed, grow_in_place, grow_in_place_zeroed, shrink_in_place - - /// Returns a pointer suitable for holding data described by - /// a new layout with `layout`’s alignment and a size given - /// by `new_size` and the actual size of the allocated block. - /// The latter is greater than or equal to `layout.size()`. - /// To accomplish this, the allocator may extend or shrink - /// the allocation referenced by `ptr` to fit the new layout. - /// - /// If this returns `Ok`, then ownership of the memory block - /// referenced by `ptr` has been transferred to this - /// allocator. The memory may or may not have been freed, and - /// should be considered unusable (unless of course it was - /// transferred back to the caller again via the return value of - /// this method). - /// - /// If this method returns `Err`, then ownership of the memory - /// block has not been transferred to this allocator, and the - /// contents of the memory block are unaltered. - /// - /// # Safety - /// - /// This function is unsafe because undefined behavior can result - /// if the caller does not ensure all of the following: - /// - /// * `ptr` must be currently allocated via this allocator, - /// - /// * `layout` must *fit* the `ptr` (see above). (The `new_size` - /// argument need not fit it.) - /// - /// * `new_size`, when rounded up to the nearest multiple of `layout.align()`, - /// must not overflow (i.e., the rounded value must be less than `usize::MAX`). - /// - /// (Extension subtraits might provide more specific bounds on - /// behavior, e.g., guarantee a sentinel address or a null pointer - /// in response to a zero-size allocation request.) - /// - /// # Errors - /// - /// Returns `Err` only if the new layout - /// does not meet the allocator's size - /// and alignment constraints of the allocator, or if reallocation - /// otherwise fails. - /// - /// Implementations are encouraged to return `Err` on memory - /// exhaustion rather than panicking or aborting, but this is not - /// a strict requirement. (Specifically: it is *legal* to - /// implement this trait atop an underlying native allocation - /// library that aborts on memory exhaustion.) - /// - /// Clients wishing to abort computation in response to a - /// reallocation error are encouraged to call the [`handle_alloc_error`] function, - /// rather than directly invoking `panic!` or similar. - /// - /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html - unsafe fn realloc( - &mut self, - ptr: NonNull, - layout: Layout, - new_size: usize, - ) -> Result<(NonNull, usize), AllocErr> { - let old_size = layout.size(); - - if new_size > old_size { - if let Ok(size) = self.grow_in_place(ptr, layout, new_size) { - return Ok((ptr, size)); - } - } else if new_size < old_size { - if let Ok(size) = self.shrink_in_place(ptr, layout, new_size) { - return Ok((ptr, size)); - } - } else { - return Ok((ptr, new_size)); - } - - // otherwise, fall back on alloc + copy + dealloc. - let new_layout = Layout::from_size_align_unchecked(new_size, layout.align()); - let result = self.alloc(new_layout); - if let Ok((new_ptr, _)) = result { - ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_ptr(), cmp::min(old_size, new_size)); - self.dealloc(ptr, layout); - } - result - } - - /// Behaves like `realloc`, but also ensures that the new contents - /// are set to zero before being returned. - /// - /// # Safety - /// - /// This function is unsafe for the same reasons that `realloc` is. - /// - /// # Errors - /// - /// Returns `Err` only if the new layout - /// does not meet the allocator's size - /// and alignment constraints of the allocator, or if reallocation - /// otherwise fails. - /// - /// Implementations are encouraged to return `Err` on memory - /// exhaustion rather than panicking or aborting, but this is not - /// a strict requirement. (Specifically: it is *legal* to - /// implement this trait atop an underlying native allocation - /// library that aborts on memory exhaustion.) - /// - /// Clients wishing to abort computation in response to a - /// reallocation error are encouraged to call the [`handle_alloc_error`] function, - /// rather than directly invoking `panic!` or similar. - /// - /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html - unsafe fn realloc_zeroed( - &mut self, - ptr: NonNull, - layout: Layout, - new_size: usize, - ) -> Result<(NonNull, usize), AllocErr> { - let old_size = layout.size(); - - if new_size > old_size { - if let Ok(size) = self.grow_in_place_zeroed(ptr, layout, new_size) { - return Ok((ptr, size)); - } - } else if new_size < old_size { - if let Ok(size) = self.shrink_in_place(ptr, layout, new_size) { - return Ok((ptr, size)); - } - } else { - return Ok((ptr, new_size)); - } - - // otherwise, fall back on alloc + copy + dealloc. - let new_layout = Layout::from_size_align_unchecked(new_size, layout.align()); - let result = self.alloc_zeroed(new_layout); - if let Ok((new_ptr, _)) = result { - ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_ptr(), cmp::min(old_size, new_size)); - self.dealloc(ptr, layout); - } - result - } - - /// Attempts to extend the allocation referenced by `ptr` to fit `new_size`. - /// - /// If this returns `Ok`, then the allocator has asserted that the - /// memory block referenced by `ptr` now fits `new_size`, and thus can - /// be used to carry data of a layout of that size and same alignment as - /// `layout`. The returned value is the new size of the allocated block. - /// (The allocator is allowed to expend effort to accomplish this, such - /// as extending the memory block to include successor blocks, or virtual - /// memory tricks.) - /// - /// Regardless of what this method returns, ownership of the - /// memory block referenced by `ptr` has not been transferred, and - /// the contents of the memory block are unaltered. - /// - /// # Safety - /// - /// This function is unsafe because undefined behavior can result - /// if the caller does not ensure all of the following: - /// - /// * `ptr` must be currently allocated via this allocator, - /// - /// * `layout` must *fit* the `ptr` (see above); note the - /// `new_size` argument need not fit it, - /// - /// * `new_size` must not be less than `layout.size()`, - /// - /// # Errors - /// - /// Returns `Err(CannotReallocInPlace)` when the allocator is - /// unable to assert that the memory block referenced by `ptr` - /// could fit `layout`. - /// - /// Note that one cannot pass `CannotReallocInPlace` to the `handle_alloc_error` - /// function; clients are expected either to be able to recover from - /// `grow_in_place` failures without aborting, or to fall back on - /// another reallocation method before resorting to an abort. - #[inline] - unsafe fn grow_in_place( - &mut self, - ptr: NonNull, - layout: Layout, - new_size: usize, - ) -> Result { - let _ = ptr; - let _ = layout; - let _ = new_size; - Err(CannotReallocInPlace) - } - - /// Behaves like `grow_in_place`, but also ensures that the new - /// contents are set to zero before being returned. - /// - /// # Safety - /// - /// This function is unsafe for the same reasons that `grow_in_place` is. - /// - /// # Errors - /// - /// Returns `Err(CannotReallocInPlace)` when the allocator is - /// unable to assert that the memory block referenced by `ptr` - /// could fit `layout`. - /// - /// Note that one cannot pass `CannotReallocInPlace` to the `handle_alloc_error` - /// function; clients are expected either to be able to recover from - /// `grow_in_place` failures without aborting, or to fall back on - /// another reallocation method before resorting to an abort. - unsafe fn grow_in_place_zeroed( - &mut self, - ptr: NonNull, - layout: Layout, - new_size: usize, - ) -> Result { - let size = self.grow_in_place(ptr, layout, new_size)?; - ptr.as_ptr().add(layout.size()).write_bytes(0, new_size - layout.size()); - Ok(size) - } - - /// Attempts to shrink the allocation referenced by `ptr` to fit `new_size`. - /// - /// If this returns `Ok`, then the allocator has asserted that the - /// memory block referenced by `ptr` now fits `new_size`, and - /// thus can only be used to carry data of that smaller - /// layout. The returned value is the new size the allocated block. - /// (The allocator is allowed to take advantage of this, - /// carving off portions of the block for reuse elsewhere.) The - /// truncated contents of the block within the smaller layout are - /// unaltered, and ownership of block has not been transferred. - /// - /// If this returns `Err`, then the memory block is considered to - /// still represent the original (larger) `layout`. None of the - /// block has been carved off for reuse elsewhere, ownership of - /// the memory block has not been transferred, and the contents of - /// the memory block are unaltered. - /// - /// # Safety - /// - /// This function is unsafe because undefined behavior can result - /// if the caller does not ensure all of the following: - /// - /// * `ptr` must be currently allocated via this allocator, - /// - /// * `layout` must *fit* the `ptr` (see above); note the - /// `new_size` argument need not fit it, - /// - /// * `new_size` must not be greater than `layout.size()`, - /// - /// # Errors - /// - /// Returns `Err(CannotReallocInPlace)` when the allocator is - /// unable to assert that the memory block referenced by `ptr` - /// could fit `layout`. - /// - /// Note that one cannot pass `CannotReallocInPlace` to the `handle_alloc_error` - /// function; clients are expected either to be able to recover from - /// `shrink_in_place` failures without aborting, or to fall back - /// on another reallocation method before resorting to an abort. - #[inline] - unsafe fn shrink_in_place( - &mut self, - ptr: NonNull, - layout: Layout, - new_size: usize, - ) -> Result { - let _ = ptr; - let _ = layout; - let _ = new_size; - Err(CannotReallocInPlace) - } -} diff --git a/crux-mir/lib/core/src/alloc/global.rs b/crux-mir/lib/core/src/alloc/global.rs new file mode 100644 index 000000000..1d80b8bf9 --- /dev/null +++ b/crux-mir/lib/core/src/alloc/global.rs @@ -0,0 +1,277 @@ +use crate::alloc::Layout; +use crate::cmp; +use crate::ptr; + +/// A memory allocator that can be registered as the standard library’s default +/// through the `#[global_allocator]` attribute. +/// +/// Some of the methods require that a memory block be *currently +/// allocated* via an allocator. This means that: +/// +/// * the starting address for that memory block was previously +/// returned by a previous call to an allocation method +/// such as `alloc`, and +/// +/// * the memory block has not been subsequently deallocated, where +/// blocks are deallocated either by being passed to a deallocation +/// method such as `dealloc` or by being +/// passed to a reallocation method that returns a non-null pointer. +/// +/// +/// # Example +/// +/// ``` +/// use std::alloc::{GlobalAlloc, Layout}; +/// use std::cell::UnsafeCell; +/// use std::ptr::null_mut; +/// use std::sync::atomic::{ +/// AtomicUsize, +/// Ordering::{Acquire, SeqCst}, +/// }; +/// +/// const ARENA_SIZE: usize = 128 * 1024; +/// const MAX_SUPPORTED_ALIGN: usize = 4096; +/// #[repr(C, align(4096))] // 4096 == MAX_SUPPORTED_ALIGN +/// struct SimpleAllocator { +/// arena: UnsafeCell<[u8; ARENA_SIZE]>, +/// remaining: AtomicUsize, // we allocate from the top, counting down +/// } +/// +/// #[global_allocator] +/// static ALLOCATOR: SimpleAllocator = SimpleAllocator { +/// arena: UnsafeCell::new([0x55; ARENA_SIZE]), +/// remaining: AtomicUsize::new(ARENA_SIZE), +/// }; +/// +/// unsafe impl Sync for SimpleAllocator {} +/// +/// unsafe impl GlobalAlloc for SimpleAllocator { +/// unsafe fn alloc(&self, layout: Layout) -> *mut u8 { +/// let size = layout.size(); +/// let align = layout.align(); +/// +/// // `Layout` contract forbids making a `Layout` with align=0, or align not power of 2. +/// // So we can safely use a mask to ensure alignment without worrying about UB. +/// let align_mask_to_round_down = !(align - 1); +/// +/// if align > MAX_SUPPORTED_ALIGN { +/// return null_mut(); +/// } +/// +/// let mut allocated = 0; +/// if self +/// .remaining +/// .fetch_update(SeqCst, SeqCst, |mut remaining| { +/// if size > remaining { +/// return None; +/// } +/// remaining -= size; +/// remaining &= align_mask_to_round_down; +/// allocated = remaining; +/// Some(remaining) +/// }) +/// .is_err() +/// { +/// return null_mut(); +/// }; +/// self.arena.get().cast::().add(allocated) +/// } +/// unsafe fn dealloc(&self, _ptr: *mut u8, _layout: Layout) {} +/// } +/// +/// fn main() { +/// let _s = format!("allocating a string!"); +/// let currently = ALLOCATOR.remaining.load(Acquire); +/// println!("allocated so far: {}", ARENA_SIZE - currently); +/// } +/// ``` +/// +/// # Safety +/// +/// The `GlobalAlloc` trait is an `unsafe` trait for a number of reasons, and +/// implementors must ensure that they adhere to these contracts: +/// +/// * It's undefined behavior if global allocators unwind. This restriction may +/// be lifted in the future, but currently a panic from any of these +/// functions may lead to memory unsafety. +/// +/// * `Layout` queries and calculations in general must be correct. Callers of +/// this trait are allowed to rely on the contracts defined on each method, +/// and implementors must ensure such contracts remain true. +/// +/// * You must not rely on allocations actually happening, even if there are explicit +/// heap allocations in the source. The optimizer may detect unused allocations that it can either +/// eliminate entirely or move to the stack and thus never invoke the allocator. The +/// optimizer may further assume that allocation is infallible, so code that used to fail due +/// to allocator failures may now suddenly work because the optimizer worked around the +/// need for an allocation. More concretely, the following code example is unsound, irrespective +/// of whether your custom allocator allows counting how many allocations have happened. +/// +/// ```rust,ignore (unsound and has placeholders) +/// drop(Box::new(42)); +/// let number_of_heap_allocs = /* call private allocator API */; +/// unsafe { std::intrinsics::assume(number_of_heap_allocs > 0); } +/// ``` +/// +/// Note that the optimizations mentioned above are not the only +/// optimization that can be applied. You may generally not rely on heap allocations +/// happening if they can be removed without changing program behavior. +/// Whether allocations happen or not is not part of the program behavior, even if it +/// could be detected via an allocator that tracks allocations by printing or otherwise +/// having side effects. +#[stable(feature = "global_alloc", since = "1.28.0")] +pub unsafe trait GlobalAlloc { + /// Allocate memory as described by the given `layout`. + /// + /// Returns a pointer to newly-allocated memory, + /// or null to indicate allocation failure. + /// + /// # Safety + /// + /// This function is unsafe because undefined behavior can result + /// if the caller does not ensure that `layout` has non-zero size. + /// + /// (Extension subtraits might provide more specific bounds on + /// behavior, e.g., guarantee a sentinel address or a null pointer + /// in response to a zero-size allocation request.) + /// + /// The allocated block of memory may or may not be initialized. + /// + /// # Errors + /// + /// Returning a null pointer indicates that either memory is exhausted + /// or `layout` does not meet this allocator's size or alignment constraints. + /// + /// Implementations are encouraged to return null on memory + /// exhaustion rather than aborting, but this is not + /// a strict requirement. (Specifically: it is *legal* to + /// implement this trait atop an underlying native allocation + /// library that aborts on memory exhaustion.) + /// + /// Clients wishing to abort computation in response to an + /// allocation error are encouraged to call the [`handle_alloc_error`] function, + /// rather than directly invoking `panic!` or similar. + /// + /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html + #[stable(feature = "global_alloc", since = "1.28.0")] + unsafe fn alloc(&self, layout: Layout) -> *mut u8; + + /// Deallocate the block of memory at the given `ptr` pointer with the given `layout`. + /// + /// # Safety + /// + /// This function is unsafe because undefined behavior can result + /// if the caller does not ensure all of the following: + /// + /// * `ptr` must denote a block of memory currently allocated via + /// this allocator, + /// + /// * `layout` must be the same layout that was used + /// to allocate that block of memory. + #[stable(feature = "global_alloc", since = "1.28.0")] + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout); + + /// Behaves like `alloc`, but also ensures that the contents + /// are set to zero before being returned. + /// + /// # Safety + /// + /// This function is unsafe for the same reasons that `alloc` is. + /// However the allocated block of memory is guaranteed to be initialized. + /// + /// # Errors + /// + /// Returning a null pointer indicates that either memory is exhausted + /// or `layout` does not meet allocator's size or alignment constraints, + /// just as in `alloc`. + /// + /// Clients wishing to abort computation in response to an + /// allocation error are encouraged to call the [`handle_alloc_error`] function, + /// rather than directly invoking `panic!` or similar. + /// + /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html + #[stable(feature = "global_alloc", since = "1.28.0")] + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let size = layout.size(); + // SAFETY: the safety contract for `alloc` must be upheld by the caller. + let ptr = unsafe { self.alloc(layout) }; + if !ptr.is_null() { + // SAFETY: as allocation succeeded, the region from `ptr` + // of size `size` is guaranteed to be valid for writes. + unsafe { ptr::write_bytes(ptr, 0, size) }; + } + ptr + } + + /// Shrink or grow a block of memory to the given `new_size`. + /// The block is described by the given `ptr` pointer and `layout`. + /// + /// If this returns a non-null pointer, then ownership of the memory block + /// referenced by `ptr` has been transferred to this allocator. + /// Any access to the old `ptr` is Undefined Behavior, even if the + /// allocation remained in-place. The newly returned pointer is the only valid pointer + /// for accessing this memory now. + /// The new memory block is allocated with `layout`, + /// but with the `size` updated to `new_size`. This new layout must be + /// used when deallocating the new memory block with `dealloc`. The range + /// `0..min(layout.size(), new_size)` of the new memory block is + /// guaranteed to have the same values as the original block. + /// + /// If this method returns null, then ownership of the memory + /// block has not been transferred to this allocator, and the + /// contents of the memory block are unaltered. + /// + /// # Safety + /// + /// This function is unsafe because undefined behavior can result + /// if the caller does not ensure all of the following: + /// + /// * `ptr` must be currently allocated via this allocator, + /// + /// * `layout` must be the same layout that was used + /// to allocate that block of memory, + /// + /// * `new_size` must be greater than zero. + /// + /// * `new_size`, when rounded up to the nearest multiple of `layout.align()`, + /// must not overflow (i.e., the rounded value must be less than `usize::MAX`). + /// + /// (Extension subtraits might provide more specific bounds on + /// behavior, e.g., guarantee a sentinel address or a null pointer + /// in response to a zero-size allocation request.) + /// + /// # Errors + /// + /// Returns null if the new layout does not meet the size + /// and alignment constraints of the allocator, or if reallocation + /// otherwise fails. + /// + /// Implementations are encouraged to return null on memory + /// exhaustion rather than panicking or aborting, but this is not + /// a strict requirement. (Specifically: it is *legal* to + /// implement this trait atop an underlying native allocation + /// library that aborts on memory exhaustion.) + /// + /// Clients wishing to abort computation in response to a + /// reallocation error are encouraged to call the [`handle_alloc_error`] function, + /// rather than directly invoking `panic!` or similar. + /// + /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html + #[stable(feature = "global_alloc", since = "1.28.0")] + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + // SAFETY: the caller must ensure that the `new_size` does not overflow. + // `layout.align()` comes from a `Layout` and is thus guaranteed to be valid. + let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; + // SAFETY: the caller must ensure that `new_layout` is greater than zero. + let new_ptr = unsafe { self.alloc(new_layout) }; + if !new_ptr.is_null() { + // SAFETY: the previously allocated block cannot overlap the newly allocated block. + // The safety contract for `dealloc` must be upheld by the caller. + unsafe { + ptr::copy_nonoverlapping(ptr, new_ptr, cmp::min(layout.size(), new_size)); + self.dealloc(ptr, layout); + } + } + new_ptr + } +} diff --git a/crux-mir/lib/core/src/alloc/layout.rs b/crux-mir/lib/core/src/alloc/layout.rs new file mode 100644 index 000000000..ac3d84718 --- /dev/null +++ b/crux-mir/lib/core/src/alloc/layout.rs @@ -0,0 +1,491 @@ +// Seemingly inconsequential code changes to this file can lead to measurable +// performance impact on compilation times, due at least in part to the fact +// that the layout code gets called from many instantiations of the various +// collections, resulting in having to optimize down excess IR multiple times. +// Your performance intuition is useless. Run perf. + +use crate::cmp; +use crate::error::Error; +use crate::fmt; +use crate::mem; +use crate::ptr::{Alignment, NonNull}; + +// While this function is used in one place and its implementation +// could be inlined, the previous attempts to do so made rustc +// slower: +// +// * https://github.com/rust-lang/rust/pull/72189 +// * https://github.com/rust-lang/rust/pull/79827 +const fn size_align() -> (usize, usize) { + (mem::size_of::(), mem::align_of::()) +} + +/// Layout of a block of memory. +/// +/// An instance of `Layout` describes a particular layout of memory. +/// You build a `Layout` up as an input to give to an allocator. +/// +/// All layouts have an associated size and a power-of-two alignment. +/// +/// (Note that layouts are *not* required to have non-zero size, +/// even though `GlobalAlloc` requires that all memory requests +/// be non-zero in size. A caller must either ensure that conditions +/// like this are met, use specific allocators with looser +/// requirements, or use the more lenient `Allocator` interface.) +#[stable(feature = "alloc_layout", since = "1.28.0")] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +#[lang = "alloc_layout"] +pub struct Layout { + // size of the requested block of memory, measured in bytes. + size: usize, + + // alignment of the requested block of memory, measured in bytes. + // we ensure that this is always a power-of-two, because API's + // like `posix_memalign` require it and it is a reasonable + // constraint to impose on Layout constructors. + // + // (However, we do not analogously require `align >= sizeof(void*)`, + // even though that is *also* a requirement of `posix_memalign`.) + align: Alignment, +} + +impl Layout { + /// Constructs a `Layout` from a given `size` and `align`, + /// or returns `LayoutError` if any of the following conditions + /// are not met: + /// + /// * `align` must not be zero, + /// + /// * `align` must be a power of two, + /// + /// * `size`, when rounded up to the nearest multiple of `align`, + /// must not overflow isize (i.e., the rounded value must be + /// less than or equal to `isize::MAX`). + #[stable(feature = "alloc_layout", since = "1.28.0")] + #[rustc_const_stable(feature = "const_alloc_layout_size_align", since = "1.50.0")] + #[inline] + #[rustc_allow_const_fn_unstable(ptr_alignment_type)] + pub const fn from_size_align(size: usize, align: usize) -> Result { + if !align.is_power_of_two() { + return Err(LayoutError); + } + + // SAFETY: just checked that align is a power of two. + Layout::from_size_alignment(size, unsafe { Alignment::new_unchecked(align) }) + } + + #[inline(always)] + const fn max_size_for_align(align: Alignment) -> usize { + // (power-of-two implies align != 0.) + + // Rounded up size is: + // size_rounded_up = (size + align - 1) & !(align - 1); + // + // We know from above that align != 0. If adding (align - 1) + // does not overflow, then rounding up will be fine. + // + // Conversely, &-masking with !(align - 1) will subtract off + // only low-order-bits. Thus if overflow occurs with the sum, + // the &-mask cannot subtract enough to undo that overflow. + // + // Above implies that checking for summation overflow is both + // necessary and sufficient. + isize::MAX as usize - (align.as_usize() - 1) + } + + /// Internal helper constructor to skip revalidating alignment validity. + #[inline] + const fn from_size_alignment(size: usize, align: Alignment) -> Result { + if size > Self::max_size_for_align(align) { + return Err(LayoutError); + } + + // SAFETY: Layout::size invariants checked above. + Ok(Layout { size, align }) + } + + /// Creates a layout, bypassing all checks. + /// + /// # Safety + /// + /// This function is unsafe as it does not verify the preconditions from + /// [`Layout::from_size_align`]. + #[stable(feature = "alloc_layout", since = "1.28.0")] + #[rustc_const_stable(feature = "const_alloc_layout_unchecked", since = "1.36.0")] + #[must_use] + #[inline] + #[rustc_allow_const_fn_unstable(ptr_alignment_type)] + pub const unsafe fn from_size_align_unchecked(size: usize, align: usize) -> Self { + // SAFETY: the caller is required to uphold the preconditions. + unsafe { Layout { size, align: Alignment::new_unchecked(align) } } + } + + /// The minimum size in bytes for a memory block of this layout. + #[stable(feature = "alloc_layout", since = "1.28.0")] + #[rustc_const_stable(feature = "const_alloc_layout_size_align", since = "1.50.0")] + #[must_use] + #[inline] + pub const fn size(&self) -> usize { + self.size + } + + /// The minimum byte alignment for a memory block of this layout. + #[stable(feature = "alloc_layout", since = "1.28.0")] + #[rustc_const_stable(feature = "const_alloc_layout_size_align", since = "1.50.0")] + #[must_use = "this returns the minimum alignment, \ + without modifying the layout"] + #[inline] + #[rustc_allow_const_fn_unstable(ptr_alignment_type)] + pub const fn align(&self) -> usize { + self.align.as_usize() + } + + /// Constructs a `Layout` suitable for holding a value of type `T`. + #[stable(feature = "alloc_layout", since = "1.28.0")] + #[rustc_const_stable(feature = "alloc_layout_const_new", since = "1.42.0")] + #[must_use] + #[inline] + pub const fn new() -> Self { + let (size, align) = size_align::(); + // SAFETY: if the type is instantiated, rustc already ensures that its + // layout is valid. Use the unchecked constructor to avoid inserting a + // panicking codepath that needs to be optimized out. + unsafe { Layout::from_size_align_unchecked(size, align) } + } + + /// Produces layout describing a record that could be used to + /// allocate backing structure for `T` (which could be a trait + /// or other unsized type like a slice). + #[stable(feature = "alloc_layout", since = "1.28.0")] + #[rustc_const_unstable(feature = "const_alloc_layout", issue = "67521")] + #[must_use] + #[inline] + pub const fn for_value(t: &T) -> Self { + let (size, align) = (mem::size_of_val(t), mem::align_of_val(t)); + // SAFETY: see rationale in `new` for why this is using the unsafe variant + unsafe { Layout::from_size_align_unchecked(size, align) } + } + + /// Produces layout describing a record that could be used to + /// allocate backing structure for `T` (which could be a trait + /// or other unsized type like a slice). + /// + /// # Safety + /// + /// This function is only safe to call if the following conditions hold: + /// + /// - If `T` is `Sized`, this function is always safe to call. + /// - If the unsized tail of `T` is: + /// - a [slice], then the length of the slice tail must be an initialized + /// integer, and the size of the *entire value* + /// (dynamic tail length + statically sized prefix) must fit in `isize`. + /// - a [trait object], then the vtable part of the pointer must point + /// to a valid vtable for the type `T` acquired by an unsizing coercion, + /// and the size of the *entire value* + /// (dynamic tail length + statically sized prefix) must fit in `isize`. + /// - an (unstable) [extern type], then this function is always safe to + /// call, but may panic or otherwise return the wrong value, as the + /// extern type's layout is not known. This is the same behavior as + /// [`Layout::for_value`] on a reference to an extern type tail. + /// - otherwise, it is conservatively not allowed to call this function. + /// + /// [trait object]: ../../book/ch17-02-trait-objects.html + /// [extern type]: ../../unstable-book/language-features/extern-types.html + #[unstable(feature = "layout_for_ptr", issue = "69835")] + #[rustc_const_unstable(feature = "const_alloc_layout", issue = "67521")] + #[must_use] + pub const unsafe fn for_value_raw(t: *const T) -> Self { + // SAFETY: we pass along the prerequisites of these functions to the caller + let (size, align) = unsafe { (mem::size_of_val_raw(t), mem::align_of_val_raw(t)) }; + // SAFETY: see rationale in `new` for why this is using the unsafe variant + unsafe { Layout::from_size_align_unchecked(size, align) } + } + + /// Creates a `NonNull` that is dangling, but well-aligned for this Layout. + /// + /// Note that the pointer value may potentially represent a valid pointer, + /// which means this must not be used as a "not yet initialized" + /// sentinel value. Types that lazily allocate must track initialization by + /// some other means. + #[unstable(feature = "alloc_layout_extra", issue = "55724")] + #[rustc_const_unstable(feature = "alloc_layout_extra", issue = "55724")] + #[must_use] + #[inline] + pub const fn dangling(&self) -> NonNull { + // SAFETY: align is guaranteed to be non-zero + unsafe { NonNull::new_unchecked(crate::ptr::invalid_mut::(self.align())) } + } + + /// Creates a layout describing the record that can hold a value + /// of the same layout as `self`, but that also is aligned to + /// alignment `align` (measured in bytes). + /// + /// If `self` already meets the prescribed alignment, then returns + /// `self`. + /// + /// Note that this method does not add any padding to the overall + /// size, regardless of whether the returned layout has a different + /// alignment. In other words, if `K` has size 16, `K.align_to(32)` + /// will *still* have size 16. + /// + /// Returns an error if the combination of `self.size()` and the given + /// `align` violates the conditions listed in [`Layout::from_size_align`]. + #[stable(feature = "alloc_layout_manipulation", since = "1.44.0")] + #[rustc_const_unstable(feature = "const_alloc_layout", issue = "67521")] + #[inline] + pub const fn align_to(&self, align: usize) -> Result { + Layout::from_size_align(self.size(), cmp::max(self.align(), align)) + } + + /// Returns the amount of padding we must insert after `self` + /// to ensure that the following address will satisfy `align` + /// (measured in bytes). + /// + /// e.g., if `self.size()` is 9, then `self.padding_needed_for(4)` + /// returns 3, because that is the minimum number of bytes of + /// padding required to get a 4-aligned address (assuming that the + /// corresponding memory block starts at a 4-aligned address). + /// + /// The return value of this function has no meaning if `align` is + /// not a power-of-two. + /// + /// Note that the utility of the returned value requires `align` + /// to be less than or equal to the alignment of the starting + /// address for the whole allocated block of memory. One way to + /// satisfy this constraint is to ensure `align <= self.align()`. + #[unstable(feature = "alloc_layout_extra", issue = "55724")] + #[rustc_const_unstable(feature = "const_alloc_layout", issue = "67521")] + #[must_use = "this returns the padding needed, \ + without modifying the `Layout`"] + #[inline] + pub const fn padding_needed_for(&self, align: usize) -> usize { + let len = self.size(); + + // Rounded up value is: + // len_rounded_up = (len + align - 1) & !(align - 1); + // and then we return the padding difference: `len_rounded_up - len`. + // + // We use modular arithmetic throughout: + // + // 1. align is guaranteed to be > 0, so align - 1 is always + // valid. + // + // 2. `len + align - 1` can overflow by at most `align - 1`, + // so the &-mask with `!(align - 1)` will ensure that in the + // case of overflow, `len_rounded_up` will itself be 0. + // Thus the returned padding, when added to `len`, yields 0, + // which trivially satisfies the alignment `align`. + // + // (Of course, attempts to allocate blocks of memory whose + // size and padding overflow in the above manner should cause + // the allocator to yield an error anyway.) + + let len_rounded_up = len.wrapping_add(align).wrapping_sub(1) & !align.wrapping_sub(1); + len_rounded_up.wrapping_sub(len) + } + + /// Creates a layout by rounding the size of this layout up to a multiple + /// of the layout's alignment. + /// + /// This is equivalent to adding the result of `padding_needed_for` + /// to the layout's current size. + #[stable(feature = "alloc_layout_manipulation", since = "1.44.0")] + #[rustc_const_unstable(feature = "const_alloc_layout", issue = "67521")] + #[must_use = "this returns a new `Layout`, \ + without modifying the original"] + #[inline] + pub const fn pad_to_align(&self) -> Layout { + let pad = self.padding_needed_for(self.align()); + // This cannot overflow. Quoting from the invariant of Layout: + // > `size`, when rounded up to the nearest multiple of `align`, + // > must not overflow isize (i.e., the rounded value must be + // > less than or equal to `isize::MAX`) + let new_size = self.size() + pad; + + // SAFETY: padded size is guaranteed to not exceed `isize::MAX`. + unsafe { Layout::from_size_align_unchecked(new_size, self.align()) } + } + + /// Creates a layout describing the record for `n` instances of + /// `self`, with a suitable amount of padding between each to + /// ensure that each instance is given its requested size and + /// alignment. On success, returns `(k, offs)` where `k` is the + /// layout of the array and `offs` is the distance between the start + /// of each element in the array. + /// + /// On arithmetic overflow, returns `LayoutError`. + #[unstable(feature = "alloc_layout_extra", issue = "55724")] + #[rustc_const_unstable(feature = "const_alloc_layout", issue = "67521")] + #[inline] + pub const fn repeat(&self, n: usize) -> Result<(Self, usize), LayoutError> { + // This cannot overflow. Quoting from the invariant of Layout: + // > `size`, when rounded up to the nearest multiple of `align`, + // > must not overflow isize (i.e., the rounded value must be + // > less than or equal to `isize::MAX`) + let padded_size = self.size() + self.padding_needed_for(self.align()); + let alloc_size = padded_size.checked_mul(n).ok_or(LayoutError)?; + + // The safe constructor is called here to enforce the isize size limit. + let layout = Layout::from_size_alignment(alloc_size, self.align)?; + Ok((layout, padded_size)) + } + + /// Creates a layout describing the record for `self` followed by + /// `next`, including any necessary padding to ensure that `next` + /// will be properly aligned, but *no trailing padding*. + /// + /// In order to match C representation layout `repr(C)`, you should + /// call `pad_to_align` after extending the layout with all fields. + /// (There is no way to match the default Rust representation + /// layout `repr(Rust)`, as it is unspecified.) + /// + /// Note that the alignment of the resulting layout will be the maximum of + /// those of `self` and `next`, in order to ensure alignment of both parts. + /// + /// Returns `Ok((k, offset))`, where `k` is layout of the concatenated + /// record and `offset` is the relative location, in bytes, of the + /// start of the `next` embedded within the concatenated record + /// (assuming that the record itself starts at offset 0). + /// + /// On arithmetic overflow, returns `LayoutError`. + /// + /// # Examples + /// + /// To calculate the layout of a `#[repr(C)]` structure and the offsets of + /// the fields from its fields' layouts: + /// + /// ```rust + /// # use std::alloc::{Layout, LayoutError}; + /// pub fn repr_c(fields: &[Layout]) -> Result<(Layout, Vec), LayoutError> { + /// let mut offsets = Vec::new(); + /// let mut layout = Layout::from_size_align(0, 1)?; + /// for &field in fields { + /// let (new_layout, offset) = layout.extend(field)?; + /// layout = new_layout; + /// offsets.push(offset); + /// } + /// // Remember to finalize with `pad_to_align`! + /// Ok((layout.pad_to_align(), offsets)) + /// } + /// # // test that it works + /// # #[repr(C)] struct S { a: u64, b: u32, c: u16, d: u32 } + /// # let s = Layout::new::(); + /// # let u16 = Layout::new::(); + /// # let u32 = Layout::new::(); + /// # let u64 = Layout::new::(); + /// # assert_eq!(repr_c(&[u64, u32, u16, u32]), Ok((s, vec![0, 8, 12, 16]))); + /// ``` + #[stable(feature = "alloc_layout_manipulation", since = "1.44.0")] + #[rustc_const_unstable(feature = "const_alloc_layout", issue = "67521")] + #[inline] + pub const fn extend(&self, next: Self) -> Result<(Self, usize), LayoutError> { + let new_align = cmp::max(self.align, next.align); + let pad = self.padding_needed_for(next.align()); + + let offset = self.size().checked_add(pad).ok_or(LayoutError)?; + let new_size = offset.checked_add(next.size()).ok_or(LayoutError)?; + + // The safe constructor is called here to enforce the isize size limit. + let layout = Layout::from_size_alignment(new_size, new_align)?; + Ok((layout, offset)) + } + + /// Creates a layout describing the record for `n` instances of + /// `self`, with no padding between each instance. + /// + /// Note that, unlike `repeat`, `repeat_packed` does not guarantee + /// that the repeated instances of `self` will be properly + /// aligned, even if a given instance of `self` is properly + /// aligned. In other words, if the layout returned by + /// `repeat_packed` is used to allocate an array, it is not + /// guaranteed that all elements in the array will be properly + /// aligned. + /// + /// On arithmetic overflow, returns `LayoutError`. + #[unstable(feature = "alloc_layout_extra", issue = "55724")] + #[rustc_const_unstable(feature = "const_alloc_layout", issue = "67521")] + #[inline] + pub const fn repeat_packed(&self, n: usize) -> Result { + let size = self.size().checked_mul(n).ok_or(LayoutError)?; + // The safe constructor is called here to enforce the isize size limit. + Layout::from_size_alignment(size, self.align) + } + + /// Creates a layout describing the record for `self` followed by + /// `next` with no additional padding between the two. Since no + /// padding is inserted, the alignment of `next` is irrelevant, + /// and is not incorporated *at all* into the resulting layout. + /// + /// On arithmetic overflow, returns `LayoutError`. + #[unstable(feature = "alloc_layout_extra", issue = "55724")] + #[rustc_const_unstable(feature = "const_alloc_layout", issue = "67521")] + #[inline] + pub const fn extend_packed(&self, next: Self) -> Result { + let new_size = self.size().checked_add(next.size()).ok_or(LayoutError)?; + // The safe constructor is called here to enforce the isize size limit. + Layout::from_size_alignment(new_size, self.align) + } + + /// Creates a layout describing the record for a `[T; n]`. + /// + /// On arithmetic overflow or when the total size would exceed + /// `isize::MAX`, returns `LayoutError`. + #[stable(feature = "alloc_layout_manipulation", since = "1.44.0")] + #[rustc_const_unstable(feature = "const_alloc_layout", issue = "67521")] + #[inline] + pub const fn array(n: usize) -> Result { + // Reduce the amount of code we need to monomorphize per `T`. + return inner(mem::size_of::(), Alignment::of::(), n); + + #[inline] + const fn inner( + element_size: usize, + align: Alignment, + n: usize, + ) -> Result { + // We need to check two things about the size: + // - That the total size won't overflow a `usize`, and + // - That the total size still fits in an `isize`. + // By using division we can check them both with a single threshold. + // That'd usually be a bad idea, but thankfully here the element size + // and alignment are constants, so the compiler will fold all of it. + if element_size != 0 && n > Layout::max_size_for_align(align) / element_size { + return Err(LayoutError); + } + + let array_size = element_size * n; + + // SAFETY: We just checked above that the `array_size` will not + // exceed `isize::MAX` even when rounded up to the alignment. + // And `Alignment` guarantees it's a power of two. + unsafe { Ok(Layout::from_size_align_unchecked(array_size, align.as_usize())) } + } + } +} + +#[stable(feature = "alloc_layout", since = "1.28.0")] +#[deprecated( + since = "1.52.0", + note = "Name does not follow std convention, use LayoutError", + suggestion = "LayoutError" +)] +pub type LayoutErr = LayoutError; + +/// The parameters given to `Layout::from_size_align` +/// or some other `Layout` constructor +/// do not satisfy its documented constraints. +#[stable(feature = "alloc_layout_error", since = "1.50.0")] +#[non_exhaustive] +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct LayoutError; + +#[stable(feature = "alloc_layout", since = "1.28.0")] +impl Error for LayoutError {} + +// (we need this for downstream impl of trait Error) +#[stable(feature = "alloc_layout", since = "1.28.0")] +impl fmt::Display for LayoutError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("invalid parameters to Layout::from_size_align") + } +} diff --git a/crux-mir/lib/core/src/alloc/mod.rs b/crux-mir/lib/core/src/alloc/mod.rs new file mode 100644 index 000000000..a6082455f --- /dev/null +++ b/crux-mir/lib/core/src/alloc/mod.rs @@ -0,0 +1,421 @@ +//! Memory allocation APIs + +#![stable(feature = "alloc_module", since = "1.28.0")] + +mod global; +mod layout; + +#[stable(feature = "global_alloc", since = "1.28.0")] +pub use self::global::GlobalAlloc; +#[stable(feature = "alloc_layout", since = "1.28.0")] +pub use self::layout::Layout; +#[stable(feature = "alloc_layout", since = "1.28.0")] +#[deprecated( + since = "1.52.0", + note = "Name does not follow std convention, use LayoutError", + suggestion = "LayoutError" +)] +#[allow(deprecated, deprecated_in_future)] +pub use self::layout::LayoutErr; + +#[stable(feature = "alloc_layout_error", since = "1.50.0")] +pub use self::layout::LayoutError; + +use crate::error::Error; +use crate::fmt; +use crate::ptr::{self, NonNull}; + +/// The `AllocError` error indicates an allocation failure +/// that may be due to resource exhaustion or to +/// something wrong when combining the given input arguments with this +/// allocator. +#[unstable(feature = "allocator_api", issue = "32838")] +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct AllocError; + +#[unstable( + feature = "allocator_api", + reason = "the precise API and guarantees it provides may be tweaked.", + issue = "32838" +)] +impl Error for AllocError {} + +// (we need this for downstream impl of trait Error) +#[unstable(feature = "allocator_api", issue = "32838")] +impl fmt::Display for AllocError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("memory allocation failed") + } +} + +/// An implementation of `Allocator` can allocate, grow, shrink, and deallocate arbitrary blocks of +/// data described via [`Layout`][]. +/// +/// `Allocator` is designed to be implemented on ZSTs, references, or smart pointers because having +/// an allocator like `MyAlloc([u8; N])` cannot be moved, without updating the pointers to the +/// allocated memory. +/// +/// Unlike [`GlobalAlloc`][], zero-sized allocations are allowed in `Allocator`. If an underlying +/// allocator does not support this (like jemalloc) or return a null pointer (such as +/// `libc::malloc`), this must be caught by the implementation. +/// +/// ### Currently allocated memory +/// +/// Some of the methods require that a memory block be *currently allocated* via an allocator. This +/// means that: +/// +/// * the starting address for that memory block was previously returned by [`allocate`], [`grow`], or +/// [`shrink`], and +/// +/// * the memory block has not been subsequently deallocated, where blocks are either deallocated +/// directly by being passed to [`deallocate`] or were changed by being passed to [`grow`] or +/// [`shrink`] that returns `Ok`. If `grow` or `shrink` have returned `Err`, the passed pointer +/// remains valid. +/// +/// [`allocate`]: Allocator::allocate +/// [`grow`]: Allocator::grow +/// [`shrink`]: Allocator::shrink +/// [`deallocate`]: Allocator::deallocate +/// +/// ### Memory fitting +/// +/// Some of the methods require that a layout *fit* a memory block. What it means for a layout to +/// "fit" a memory block means (or equivalently, for a memory block to "fit" a layout) is that the +/// following conditions must hold: +/// +/// * The block must be allocated with the same alignment as [`layout.align()`], and +/// +/// * The provided [`layout.size()`] must fall in the range `min ..= max`, where: +/// - `min` is the size of the layout most recently used to allocate the block, and +/// - `max` is the latest actual size returned from [`allocate`], [`grow`], or [`shrink`]. +/// +/// [`layout.align()`]: Layout::align +/// [`layout.size()`]: Layout::size +/// +/// # Safety +/// +/// * Memory blocks returned from an allocator must point to valid memory and retain their validity +/// until the instance and all of its clones are dropped, +/// +/// * cloning or moving the allocator must not invalidate memory blocks returned from this +/// allocator. A cloned allocator must behave like the same allocator, and +/// +/// * any pointer to a memory block which is [*currently allocated*] may be passed to any other +/// method of the allocator. +/// +/// [*currently allocated*]: #currently-allocated-memory +#[unstable(feature = "allocator_api", issue = "32838")] +#[const_trait] +pub unsafe trait Allocator { + /// Attempts to allocate a block of memory. + /// + /// On success, returns a [`NonNull<[u8]>`][NonNull] meeting the size and alignment guarantees of `layout`. + /// + /// The returned block may have a larger size than specified by `layout.size()`, and may or may + /// not have its contents initialized. + /// + /// # Errors + /// + /// Returning `Err` indicates that either memory is exhausted or `layout` does not meet + /// allocator's size or alignment constraints. + /// + /// Implementations are encouraged to return `Err` on memory exhaustion rather than panicking or + /// aborting, but this is not a strict requirement. (Specifically: it is *legal* to implement + /// this trait atop an underlying native allocation library that aborts on memory exhaustion.) + /// + /// Clients wishing to abort computation in response to an allocation error are encouraged to + /// call the [`handle_alloc_error`] function, rather than directly invoking `panic!` or similar. + /// + /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html + fn allocate(&self, layout: Layout) -> Result, AllocError>; + + /// Behaves like `allocate`, but also ensures that the returned memory is zero-initialized. + /// + /// # Errors + /// + /// Returning `Err` indicates that either memory is exhausted or `layout` does not meet + /// allocator's size or alignment constraints. + /// + /// Implementations are encouraged to return `Err` on memory exhaustion rather than panicking or + /// aborting, but this is not a strict requirement. (Specifically: it is *legal* to implement + /// this trait atop an underlying native allocation library that aborts on memory exhaustion.) + /// + /// Clients wishing to abort computation in response to an allocation error are encouraged to + /// call the [`handle_alloc_error`] function, rather than directly invoking `panic!` or similar. + /// + /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html + fn allocate_zeroed(&self, layout: Layout) -> Result, AllocError> { + let ptr = self.allocate(layout)?; + // SAFETY: `alloc` returns a valid memory block + unsafe { ptr.as_non_null_ptr().as_ptr().write_bytes(0, ptr.len()) } + Ok(ptr) + } + + /// Deallocates the memory referenced by `ptr`. + /// + /// # Safety + /// + /// * `ptr` must denote a block of memory [*currently allocated*] via this allocator, and + /// * `layout` must [*fit*] that block of memory. + /// + /// [*currently allocated*]: #currently-allocated-memory + /// [*fit*]: #memory-fitting + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout); + + /// Attempts to extend the memory block. + /// + /// Returns a new [`NonNull<[u8]>`][NonNull] containing a pointer and the actual size of the allocated + /// memory. The pointer is suitable for holding data described by `new_layout`. To accomplish + /// this, the allocator may extend the allocation referenced by `ptr` to fit the new layout. + /// + /// If this returns `Ok`, then ownership of the memory block referenced by `ptr` has been + /// transferred to this allocator. Any access to the old `ptr` is Undefined Behavior, even if the + /// allocation was grown in-place. The newly returned pointer is the only valid pointer + /// for accessing this memory now. + /// + /// If this method returns `Err`, then ownership of the memory block has not been transferred to + /// this allocator, and the contents of the memory block are unaltered. + /// + /// # Safety + /// + /// * `ptr` must denote a block of memory [*currently allocated*] via this allocator. + /// * `old_layout` must [*fit*] that block of memory (The `new_layout` argument need not fit it.). + /// * `new_layout.size()` must be greater than or equal to `old_layout.size()`. + /// + /// Note that `new_layout.align()` need not be the same as `old_layout.align()`. + /// + /// [*currently allocated*]: #currently-allocated-memory + /// [*fit*]: #memory-fitting + /// + /// # Errors + /// + /// Returns `Err` if the new layout does not meet the allocator's size and alignment + /// constraints of the allocator, or if growing otherwise fails. + /// + /// Implementations are encouraged to return `Err` on memory exhaustion rather than panicking or + /// aborting, but this is not a strict requirement. (Specifically: it is *legal* to implement + /// this trait atop an underlying native allocation library that aborts on memory exhaustion.) + /// + /// Clients wishing to abort computation in response to an allocation error are encouraged to + /// call the [`handle_alloc_error`] function, rather than directly invoking `panic!` or similar. + /// + /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html + unsafe fn grow( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + debug_assert!( + new_layout.size() >= old_layout.size(), + "`new_layout.size()` must be greater than or equal to `old_layout.size()`" + ); + + let new_ptr = self.allocate(new_layout)?; + + // SAFETY: because `new_layout.size()` must be greater than or equal to + // `old_layout.size()`, both the old and new memory allocation are valid for reads and + // writes for `old_layout.size()` bytes. Also, because the old allocation wasn't yet + // deallocated, it cannot overlap `new_ptr`. Thus, the call to `copy_nonoverlapping` is + // safe. The safety contract for `dealloc` must be upheld by the caller. + unsafe { + ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_mut_ptr(), old_layout.size()); + self.deallocate(ptr, old_layout); + } + + Ok(new_ptr) + } + + /// Behaves like `grow`, but also ensures that the new contents are set to zero before being + /// returned. + /// + /// The memory block will contain the following contents after a successful call to + /// `grow_zeroed`: + /// * Bytes `0..old_layout.size()` are preserved from the original allocation. + /// * Bytes `old_layout.size()..old_size` will either be preserved or zeroed, depending on + /// the allocator implementation. `old_size` refers to the size of the memory block prior + /// to the `grow_zeroed` call, which may be larger than the size that was originally + /// requested when it was allocated. + /// * Bytes `old_size..new_size` are zeroed. `new_size` refers to the size of the memory + /// block returned by the `grow_zeroed` call. + /// + /// # Safety + /// + /// * `ptr` must denote a block of memory [*currently allocated*] via this allocator. + /// * `old_layout` must [*fit*] that block of memory (The `new_layout` argument need not fit it.). + /// * `new_layout.size()` must be greater than or equal to `old_layout.size()`. + /// + /// Note that `new_layout.align()` need not be the same as `old_layout.align()`. + /// + /// [*currently allocated*]: #currently-allocated-memory + /// [*fit*]: #memory-fitting + /// + /// # Errors + /// + /// Returns `Err` if the new layout does not meet the allocator's size and alignment + /// constraints of the allocator, or if growing otherwise fails. + /// + /// Implementations are encouraged to return `Err` on memory exhaustion rather than panicking or + /// aborting, but this is not a strict requirement. (Specifically: it is *legal* to implement + /// this trait atop an underlying native allocation library that aborts on memory exhaustion.) + /// + /// Clients wishing to abort computation in response to an allocation error are encouraged to + /// call the [`handle_alloc_error`] function, rather than directly invoking `panic!` or similar. + /// + /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html + unsafe fn grow_zeroed( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + debug_assert!( + new_layout.size() >= old_layout.size(), + "`new_layout.size()` must be greater than or equal to `old_layout.size()`" + ); + + let new_ptr = self.allocate_zeroed(new_layout)?; + + // SAFETY: because `new_layout.size()` must be greater than or equal to + // `old_layout.size()`, both the old and new memory allocation are valid for reads and + // writes for `old_layout.size()` bytes. Also, because the old allocation wasn't yet + // deallocated, it cannot overlap `new_ptr`. Thus, the call to `copy_nonoverlapping` is + // safe. The safety contract for `dealloc` must be upheld by the caller. + unsafe { + ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_mut_ptr(), old_layout.size()); + self.deallocate(ptr, old_layout); + } + + Ok(new_ptr) + } + + /// Attempts to shrink the memory block. + /// + /// Returns a new [`NonNull<[u8]>`][NonNull] containing a pointer and the actual size of the allocated + /// memory. The pointer is suitable for holding data described by `new_layout`. To accomplish + /// this, the allocator may shrink the allocation referenced by `ptr` to fit the new layout. + /// + /// If this returns `Ok`, then ownership of the memory block referenced by `ptr` has been + /// transferred to this allocator. Any access to the old `ptr` is Undefined Behavior, even if the + /// allocation was shrunk in-place. The newly returned pointer is the only valid pointer + /// for accessing this memory now. + /// + /// If this method returns `Err`, then ownership of the memory block has not been transferred to + /// this allocator, and the contents of the memory block are unaltered. + /// + /// # Safety + /// + /// * `ptr` must denote a block of memory [*currently allocated*] via this allocator. + /// * `old_layout` must [*fit*] that block of memory (The `new_layout` argument need not fit it.). + /// * `new_layout.size()` must be smaller than or equal to `old_layout.size()`. + /// + /// Note that `new_layout.align()` need not be the same as `old_layout.align()`. + /// + /// [*currently allocated*]: #currently-allocated-memory + /// [*fit*]: #memory-fitting + /// + /// # Errors + /// + /// Returns `Err` if the new layout does not meet the allocator's size and alignment + /// constraints of the allocator, or if shrinking otherwise fails. + /// + /// Implementations are encouraged to return `Err` on memory exhaustion rather than panicking or + /// aborting, but this is not a strict requirement. (Specifically: it is *legal* to implement + /// this trait atop an underlying native allocation library that aborts on memory exhaustion.) + /// + /// Clients wishing to abort computation in response to an allocation error are encouraged to + /// call the [`handle_alloc_error`] function, rather than directly invoking `panic!` or similar. + /// + /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html + unsafe fn shrink( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + debug_assert!( + new_layout.size() <= old_layout.size(), + "`new_layout.size()` must be smaller than or equal to `old_layout.size()`" + ); + + let new_ptr = self.allocate(new_layout)?; + + // SAFETY: because `new_layout.size()` must be lower than or equal to + // `old_layout.size()`, both the old and new memory allocation are valid for reads and + // writes for `new_layout.size()` bytes. Also, because the old allocation wasn't yet + // deallocated, it cannot overlap `new_ptr`. Thus, the call to `copy_nonoverlapping` is + // safe. The safety contract for `dealloc` must be upheld by the caller. + unsafe { + ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_mut_ptr(), new_layout.size()); + self.deallocate(ptr, old_layout); + } + + Ok(new_ptr) + } + + /// Creates a "by reference" adapter for this instance of `Allocator`. + /// + /// The returned adapter also implements `Allocator` and will simply borrow this. + #[inline(always)] + fn by_ref(&self) -> &Self + where + Self: Sized, + { + self + } +} + +#[unstable(feature = "allocator_api", issue = "32838")] +unsafe impl Allocator for &A +where + A: Allocator + ?Sized, +{ + #[inline] + fn allocate(&self, layout: Layout) -> Result, AllocError> { + (**self).allocate(layout) + } + + #[inline] + fn allocate_zeroed(&self, layout: Layout) -> Result, AllocError> { + (**self).allocate_zeroed(layout) + } + + #[inline] + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + // SAFETY: the safety contract must be upheld by the caller + unsafe { (**self).deallocate(ptr, layout) } + } + + #[inline] + unsafe fn grow( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + // SAFETY: the safety contract must be upheld by the caller + unsafe { (**self).grow(ptr, old_layout, new_layout) } + } + + #[inline] + unsafe fn grow_zeroed( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + // SAFETY: the safety contract must be upheld by the caller + unsafe { (**self).grow_zeroed(ptr, old_layout, new_layout) } + } + + #[inline] + unsafe fn shrink( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + // SAFETY: the safety contract must be upheld by the caller + unsafe { (**self).shrink(ptr, old_layout, new_layout) } + } +} diff --git a/crux-mir/lib/core/src/any.rs b/crux-mir/lib/core/src/any.rs index 97ef513cb..c0fb0d993 100644 --- a/crux-mir/lib/core/src/any.rs +++ b/crux-mir/lib/core/src/any.rs @@ -1,5 +1,6 @@ -//! This module implements the `Any` trait, which enables dynamic typing -//! of any `'static` type through runtime reflection. +//! Utilities for dynamic typing or type reflection. +//! +//! # `Any` and `TypeId` //! //! `Any` itself can be used to get a `TypeId`, and has more features when used //! as a trait object. As `&dyn Any` (a borrowed trait object), it has the `is` @@ -14,7 +15,30 @@ //! //! [`Box`]: ../../std/boxed/struct.Box.html //! -//! # Examples +//! # Smart pointers and `dyn Any` +//! +//! One piece of behavior to keep in mind when using `Any` as a trait object, +//! especially with types like `Box` or `Arc`, is that simply +//! calling `.type_id()` on the value will produce the `TypeId` of the +//! *container*, not the underlying trait object. This can be avoided by +//! converting the smart pointer into a `&dyn Any` instead, which will return +//! the object's `TypeId`. For example: +//! +//! ``` +//! use std::any::{Any, TypeId}; +//! +//! let boxed: Box = Box::new(3_i32); +//! +//! // You're more likely to want this: +//! let actual_id = (&*boxed).type_id(); +//! // ... than this: +//! let boxed_id = boxed.type_id(); +//! +//! assert_eq!(actual_id, TypeId::of::()); +//! assert_eq!(boxed_id, TypeId::of::>()); +//! ``` +//! +//! ## Examples //! //! Consider a situation where we want to log out a value passed to a function. //! We know the value we're working on implements Debug, but we don't know its @@ -39,7 +63,7 @@ //! println!("String ({}): {}", as_string.len(), as_string); //! } //! None => { -//! println!("{:?}", value); +//! println!("{value:?}"); //! } //! } //! } @@ -58,6 +82,73 @@ //! do_work(&my_i8); //! } //! ``` +//! +//! # `Provider` and `Demand` +//! +//! `Provider` and the associated APIs support generic, type-driven access to data, and a mechanism +//! for implementers to provide such data. The key parts of the interface are the `Provider` +//! trait for objects which can provide data, and the [`request_value`] and [`request_ref`] +//! functions for requesting data from an object which implements `Provider`. Generally, end users +//! should not call `request_*` directly, they are helper functions for intermediate implementers +//! to use to implement a user-facing interface. This is purely for the sake of ergonomics, there is +//! no safety concern here; intermediate implementers can typically support methods rather than +//! free functions and use more specific names. +//! +//! Typically, a data provider is a trait object of a trait which extends `Provider`. A user will +//! request data from a trait object by specifying the type of the data. +//! +//! ## Data flow +//! +//! * A user requests an object of a specific type, which is delegated to `request_value` or +//! `request_ref` +//! * `request_*` creates a `Demand` object and passes it to `Provider::provide` +//! * The data provider's implementation of `Provider::provide` tries providing values of +//! different types using `Demand::provide_*`. If the type matches the type requested by +//! the user, the value will be stored in the `Demand` object. +//! * `request_*` unpacks the `Demand` object and returns any stored value to the user. +//! +//! ## Examples +//! +//! ``` +//! # #![feature(provide_any)] +//! use std::any::{Provider, Demand, request_ref}; +//! +//! // Definition of MyTrait, a data provider. +//! trait MyTrait: Provider { +//! // ... +//! } +//! +//! // Methods on `MyTrait` trait objects. +//! impl dyn MyTrait + '_ { +//! /// Get a reference to a field of the implementing struct. +//! pub fn get_context_by_ref(&self) -> Option<&T> { +//! request_ref::(self) +//! } +//! } +//! +//! // Downstream implementation of `MyTrait` and `Provider`. +//! # struct SomeConcreteType { some_string: String } +//! impl MyTrait for SomeConcreteType { +//! // ... +//! } +//! +//! impl Provider for SomeConcreteType { +//! fn provide<'a>(&'a self, demand: &mut Demand<'a>) { +//! // Provide a string reference. We could provide multiple values with +//! // different types here. +//! demand.provide_ref::(&self.some_string); +//! } +//! } +//! +//! // Downstream usage of `MyTrait`. +//! fn use_my_trait(obj: &dyn MyTrait) { +//! // Request a &String from obj. +//! let _ = obj.get_context_by_ref::().unwrap(); +//! } +//! ``` +//! +//! In this example, if the concrete type of `obj` in `use_my_trait` is `SomeConcreteType`, then +//! the `get_context_by_ref` call will return a reference to `obj.some_string` with type `&String`. #![stable(feature = "rust1", since = "1.0.0")] @@ -73,7 +164,7 @@ use crate::intrinsics; /// Most types implement `Any`. However, any type which contains a non-`'static` reference does not. /// See the [module-level documentation][mod] for more details. /// -/// [mod]: index.html +/// [mod]: crate::any // This trait is not unsafe, though we rely on the specifics of it's sole impl's // `type_id` function in unsafe code (e.g., `downcast`). Normally, that would be // a problem, but because the only impl of `Any` is a blanket implementation, no @@ -85,6 +176,7 @@ use crate::intrinsics; // unsafe traits and unsafe methods (i.e., `type_id` would still be safe to call, // but we would likely want to indicate as such in documentation). #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "Any")] pub trait Any: 'static { /// Gets the `TypeId` of `self`. /// @@ -118,7 +210,7 @@ impl Any for T { #[stable(feature = "rust1", since = "1.0.0")] impl fmt::Debug for dyn Any { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("Any") + f.debug_struct("Any").finish_non_exhaustive() } } @@ -128,19 +220,19 @@ impl fmt::Debug for dyn Any { #[stable(feature = "rust1", since = "1.0.0")] impl fmt::Debug for dyn Any + Send { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("Any") + f.debug_struct("Any").finish_non_exhaustive() } } #[stable(feature = "any_send_sync_methods", since = "1.28.0")] impl fmt::Debug for dyn Any + Send + Sync { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("Any") + f.debug_struct("Any").finish_non_exhaustive() } } impl dyn Any { - /// Returns `true` if the boxed type is the same as `T`. + /// Returns `true` if the inner type is the same as `T`. /// /// # Examples /// @@ -164,14 +256,14 @@ impl dyn Any { // Get `TypeId` of the type this function is instantiated with. let t = TypeId::of::(); - // Get `TypeId` of the type in the trait object. + // Get `TypeId` of the type in the trait object (`self`). let concrete = self.type_id(); // Compare both `TypeId`s on equality. t == concrete } - /// Returns some reference to the boxed value if it is of type `T`, or + /// Returns some reference to the inner value if it is of type `T`, or /// `None` if it isn't. /// /// # Examples @@ -197,13 +289,13 @@ impl dyn Any { // SAFETY: just checked whether we are pointing to the correct type, and we can rely on // that check for memory safety because we have implemented Any for all types; no other // impls can exist as they would conflict with our impl. - unsafe { Some(&*(self as *const dyn Any as *const T)) } + unsafe { Some(self.downcast_ref_unchecked()) } } else { None } } - /// Returns some mutable reference to the boxed value if it is of type `T`, or + /// Returns some mutable reference to the inner value if it is of type `T`, or /// `None` if it isn't. /// /// # Examples @@ -233,15 +325,73 @@ impl dyn Any { // SAFETY: just checked whether we are pointing to the correct type, and we can rely on // that check for memory safety because we have implemented Any for all types; no other // impls can exist as they would conflict with our impl. - unsafe { Some(&mut *(self as *mut dyn Any as *mut T)) } + unsafe { Some(self.downcast_mut_unchecked()) } } else { None } } + + /// Returns a reference to the inner value as type `dyn T`. + /// + /// # Examples + /// + /// ``` + /// #![feature(downcast_unchecked)] + /// + /// use std::any::Any; + /// + /// let x: Box = Box::new(1_usize); + /// + /// unsafe { + /// assert_eq!(*x.downcast_ref_unchecked::(), 1); + /// } + /// ``` + /// + /// # Safety + /// + /// The contained value must be of type `T`. Calling this method + /// with the incorrect type is *undefined behavior*. + #[unstable(feature = "downcast_unchecked", issue = "90850")] + #[inline] + pub unsafe fn downcast_ref_unchecked(&self) -> &T { + debug_assert!(self.is::()); + // SAFETY: caller guarantees that T is the correct type + unsafe { &*(self as *const dyn Any as *const T) } + } + + /// Returns a mutable reference to the inner value as type `dyn T`. + /// + /// # Examples + /// + /// ``` + /// #![feature(downcast_unchecked)] + /// + /// use std::any::Any; + /// + /// let mut x: Box = Box::new(1_usize); + /// + /// unsafe { + /// *x.downcast_mut_unchecked::() += 1; + /// } + /// + /// assert_eq!(*x.downcast_ref::().unwrap(), 2); + /// ``` + /// + /// # Safety + /// + /// The contained value must be of type `T`. Calling this method + /// with the incorrect type is *undefined behavior*. + #[unstable(feature = "downcast_unchecked", issue = "90850")] + #[inline] + pub unsafe fn downcast_mut_unchecked(&mut self) -> &mut T { + debug_assert!(self.is::()); + // SAFETY: caller guarantees that T is the correct type + unsafe { &mut *(self as *mut dyn Any as *mut T) } + } } impl dyn Any + Send { - /// Forwards to the method defined on the type `Any`. + /// Forwards to the method defined on the type `dyn Any`. /// /// # Examples /// @@ -262,10 +412,10 @@ impl dyn Any + Send { #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn is(&self) -> bool { - Any::is::(self) + ::is::(self) } - /// Forwards to the method defined on the type `Any`. + /// Forwards to the method defined on the type `dyn Any`. /// /// # Examples /// @@ -286,10 +436,10 @@ impl dyn Any + Send { #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn downcast_ref(&self) -> Option<&T> { - Any::downcast_ref::(self) + ::downcast_ref::(self) } - /// Forwards to the method defined on the type `Any`. + /// Forwards to the method defined on the type `dyn Any`. /// /// # Examples /// @@ -314,7 +464,61 @@ impl dyn Any + Send { #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn downcast_mut(&mut self) -> Option<&mut T> { - Any::downcast_mut::(self) + ::downcast_mut::(self) + } + + /// Forwards to the method defined on the type `dyn Any`. + /// + /// # Examples + /// + /// ``` + /// #![feature(downcast_unchecked)] + /// + /// use std::any::Any; + /// + /// let x: Box = Box::new(1_usize); + /// + /// unsafe { + /// assert_eq!(*x.downcast_ref_unchecked::(), 1); + /// } + /// ``` + /// + /// # Safety + /// + /// Same as the method on the type `dyn Any`. + #[unstable(feature = "downcast_unchecked", issue = "90850")] + #[inline] + pub unsafe fn downcast_ref_unchecked(&self) -> &T { + // SAFETY: guaranteed by caller + unsafe { ::downcast_ref_unchecked::(self) } + } + + /// Forwards to the method defined on the type `dyn Any`. + /// + /// # Examples + /// + /// ``` + /// #![feature(downcast_unchecked)] + /// + /// use std::any::Any; + /// + /// let mut x: Box = Box::new(1_usize); + /// + /// unsafe { + /// *x.downcast_mut_unchecked::() += 1; + /// } + /// + /// assert_eq!(*x.downcast_ref::().unwrap(), 2); + /// ``` + /// + /// # Safety + /// + /// Same as the method on the type `dyn Any`. + #[unstable(feature = "downcast_unchecked", issue = "90850")] + #[inline] + pub unsafe fn downcast_mut_unchecked(&mut self) -> &mut T { + // SAFETY: guaranteed by caller + unsafe { ::downcast_mut_unchecked::(self) } } } @@ -340,7 +544,7 @@ impl dyn Any + Send + Sync { #[stable(feature = "any_send_sync_methods", since = "1.28.0")] #[inline] pub fn is(&self) -> bool { - Any::is::(self) + ::is::(self) } /// Forwards to the method defined on the type `Any`. @@ -364,7 +568,7 @@ impl dyn Any + Send + Sync { #[stable(feature = "any_send_sync_methods", since = "1.28.0")] #[inline] pub fn downcast_ref(&self) -> Option<&T> { - Any::downcast_ref::(self) + ::downcast_ref::(self) } /// Forwards to the method defined on the type `Any`. @@ -392,7 +596,53 @@ impl dyn Any + Send + Sync { #[stable(feature = "any_send_sync_methods", since = "1.28.0")] #[inline] pub fn downcast_mut(&mut self) -> Option<&mut T> { - Any::downcast_mut::(self) + ::downcast_mut::(self) + } + + /// Forwards to the method defined on the type `Any`. + /// + /// # Examples + /// + /// ``` + /// #![feature(downcast_unchecked)] + /// + /// use std::any::Any; + /// + /// let x: Box = Box::new(1_usize); + /// + /// unsafe { + /// assert_eq!(*x.downcast_ref_unchecked::(), 1); + /// } + /// ``` + #[unstable(feature = "downcast_unchecked", issue = "90850")] + #[inline] + pub unsafe fn downcast_ref_unchecked(&self) -> &T { + // SAFETY: guaranteed by caller + unsafe { ::downcast_ref_unchecked::(self) } + } + + /// Forwards to the method defined on the type `Any`. + /// + /// # Examples + /// + /// ``` + /// #![feature(downcast_unchecked)] + /// + /// use std::any::Any; + /// + /// let mut x: Box = Box::new(1_usize); + /// + /// unsafe { + /// *x.downcast_mut_unchecked::() += 1; + /// } + /// + /// assert_eq!(*x.downcast_ref::().unwrap(), 2); + /// ``` + #[unstable(feature = "downcast_unchecked", issue = "90850")] + #[inline] + pub unsafe fn downcast_mut_unchecked(&mut self) -> &mut T { + // SAFETY: guaranteed by caller + unsafe { ::downcast_mut_unchecked::(self) } } } @@ -412,7 +662,8 @@ impl dyn Any + Send + Sync { /// While `TypeId` implements `Hash`, `PartialOrd`, and `Ord`, it is worth /// noting that the hashes and ordering will vary between Rust releases. Beware /// of relying on them inside of your code! -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] +#[derive(Clone, Copy, Debug, Hash, Eq)] +#[derive_const(PartialEq, PartialOrd, Ord)] #[stable(feature = "rust1", since = "1.0.0")] pub struct TypeId { t: u64, @@ -434,8 +685,9 @@ impl TypeId { /// assert_eq!(is_string(&0), false); /// assert_eq!(is_string(&"cookie monster".to_string()), true); /// ``` + #[must_use] #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_type_id", issue = "41875")] + #[rustc_const_unstable(feature = "const_type_id", issue = "77125")] pub const fn of() -> TypeId { TypeId { t: intrinsics::type_id::() } } @@ -446,14 +698,16 @@ impl TypeId { /// # Note /// /// This is intended for diagnostic use. The exact contents and format of the -/// string are not specified, other than being a best-effort description of the -/// type. For example, `type_name::>()` could return the -/// `"Option"` or `"std::option::Option"`, but not -/// `"foobar"`. In addition, the output may change between versions of the -/// compiler. +/// string returned are not specified, other than being a best-effort +/// description of the type. For example, amongst the strings +/// that `type_name::>()` might return are `"Option"` and +/// `"std::option::Option"`. /// -/// The type name should not be considered a unique identifier of a type; -/// multiple types may share the same type name. +/// The returned string must not be considered to be a unique identifier of a +/// type as multiple types may map to the same type name. Similarly, there is no +/// guarantee that all parts of a type will appear in the returned string: for +/// example, lifetime specifiers are currently not included. In addition, the +/// output may change between versions of the compiler. /// /// The current implementation uses the same infrastructure as compiler /// diagnostics and debuginfo, but this is not guaranteed. @@ -466,6 +720,7 @@ impl TypeId { /// "core::option::Option", /// ); /// ``` +#[must_use] #[stable(feature = "type_name", since = "1.38.0")] #[rustc_const_unstable(feature = "const_type_name", issue = "63084")] pub const fn type_name() -> &'static str { @@ -508,8 +763,530 @@ pub const fn type_name() -> &'static str { /// let y = 1.0; /// println!("{}", type_name_of_val(&y)); /// ``` +#[must_use] #[unstable(feature = "type_name_of_val", issue = "66359")] #[rustc_const_unstable(feature = "const_type_name", issue = "63084")] pub const fn type_name_of_val(_val: &T) -> &'static str { type_name::() } + +/////////////////////////////////////////////////////////////////////////////// +// Provider trait +/////////////////////////////////////////////////////////////////////////////// + +/// Trait implemented by a type which can dynamically provide values based on type. +#[unstable(feature = "provide_any", issue = "96024")] +pub trait Provider { + /// Data providers should implement this method to provide *all* values they are able to + /// provide by using `demand`. + /// + /// Note that the `provide_*` methods on `Demand` have short-circuit semantics: if an earlier + /// method has successfully provided a value, then later methods will not get an opportunity to + /// provide. + /// + /// # Examples + /// + /// Provides a reference to a field with type `String` as a `&str`, and a value of + /// type `i32`. + /// + /// ```rust + /// # #![feature(provide_any)] + /// use std::any::{Provider, Demand}; + /// # struct SomeConcreteType { field: String, num_field: i32 } + /// + /// impl Provider for SomeConcreteType { + /// fn provide<'a>(&'a self, demand: &mut Demand<'a>) { + /// demand.provide_ref::(&self.field) + /// .provide_value::(self.num_field); + /// } + /// } + /// ``` + #[unstable(feature = "provide_any", issue = "96024")] + fn provide<'a>(&'a self, demand: &mut Demand<'a>); +} + +/// Request a value from the `Provider`. +/// +/// # Examples +/// +/// Get a string value from a provider. +/// +/// ```rust +/// # #![feature(provide_any)] +/// use std::any::{Provider, request_value}; +/// +/// fn get_string(provider: &impl Provider) -> String { +/// request_value::(provider).unwrap() +/// } +/// ``` +#[unstable(feature = "provide_any", issue = "96024")] +pub fn request_value<'a, T>(provider: &'a (impl Provider + ?Sized)) -> Option +where + T: 'static, +{ + request_by_type_tag::<'a, tags::Value>(provider) +} + +/// Request a reference from the `Provider`. +/// +/// # Examples +/// +/// Get a string reference from a provider. +/// +/// ```rust +/// # #![feature(provide_any)] +/// use std::any::{Provider, request_ref}; +/// +/// fn get_str(provider: &impl Provider) -> &str { +/// request_ref::(provider).unwrap() +/// } +/// ``` +#[unstable(feature = "provide_any", issue = "96024")] +pub fn request_ref<'a, T>(provider: &'a (impl Provider + ?Sized)) -> Option<&'a T> +where + T: 'static + ?Sized, +{ + request_by_type_tag::<'a, tags::Ref>>(provider) +} + +/// Request a specific value by tag from the `Provider`. +fn request_by_type_tag<'a, I>(provider: &'a (impl Provider + ?Sized)) -> Option +where + I: tags::Type<'a>, +{ + let mut tagged = TaggedOption::<'a, I>(None); + provider.provide(tagged.as_demand()); + tagged.0 +} + +/////////////////////////////////////////////////////////////////////////////// +// Demand and its methods +/////////////////////////////////////////////////////////////////////////////// + +/// A helper object for providing data by type. +/// +/// A data provider provides values by calling this type's provide methods. +#[unstable(feature = "provide_any", issue = "96024")] +#[repr(transparent)] +pub struct Demand<'a>(dyn Erased<'a> + 'a); + +impl<'a> Demand<'a> { + /// Create a new `&mut Demand` from a `&mut dyn Erased` trait object. + fn new<'b>(erased: &'b mut (dyn Erased<'a> + 'a)) -> &'b mut Demand<'a> { + // SAFETY: transmuting `&mut (dyn Erased<'a> + 'a)` to `&mut Demand<'a>` is safe since + // `Demand` is repr(transparent). + unsafe { &mut *(erased as *mut dyn Erased<'a> as *mut Demand<'a>) } + } + + /// Provide a value or other type with only static lifetimes. + /// + /// # Examples + /// + /// Provides an `u8`. + /// + /// ```rust + /// #![feature(provide_any)] + /// + /// use std::any::{Provider, Demand}; + /// # struct SomeConcreteType { field: u8 } + /// + /// impl Provider for SomeConcreteType { + /// fn provide<'a>(&'a self, demand: &mut Demand<'a>) { + /// demand.provide_value::(self.field); + /// } + /// } + /// ``` + #[unstable(feature = "provide_any", issue = "96024")] + pub fn provide_value(&mut self, value: T) -> &mut Self + where + T: 'static, + { + self.provide::>(value) + } + + /// Provide a value or other type with only static lifetimes computed using a closure. + /// + /// # Examples + /// + /// Provides a `String` by cloning. + /// + /// ```rust + /// #![feature(provide_any)] + /// + /// use std::any::{Provider, Demand}; + /// # struct SomeConcreteType { field: String } + /// + /// impl Provider for SomeConcreteType { + /// fn provide<'a>(&'a self, demand: &mut Demand<'a>) { + /// demand.provide_value_with::(|| self.field.clone()); + /// } + /// } + /// ``` + #[unstable(feature = "provide_any", issue = "96024")] + pub fn provide_value_with(&mut self, fulfil: impl FnOnce() -> T) -> &mut Self + where + T: 'static, + { + self.provide_with::>(fulfil) + } + + /// Provide a reference. The referee type must be bounded by `'static`, + /// but may be unsized. + /// + /// # Examples + /// + /// Provides a reference to a field as a `&str`. + /// + /// ```rust + /// #![feature(provide_any)] + /// + /// use std::any::{Provider, Demand}; + /// # struct SomeConcreteType { field: String } + /// + /// impl Provider for SomeConcreteType { + /// fn provide<'a>(&'a self, demand: &mut Demand<'a>) { + /// demand.provide_ref::(&self.field); + /// } + /// } + /// ``` + #[unstable(feature = "provide_any", issue = "96024")] + pub fn provide_ref(&mut self, value: &'a T) -> &mut Self { + self.provide::>>(value) + } + + /// Provide a reference computed using a closure. The referee type + /// must be bounded by `'static`, but may be unsized. + /// + /// # Examples + /// + /// Provides a reference to a field as a `&str`. + /// + /// ```rust + /// #![feature(provide_any)] + /// + /// use std::any::{Provider, Demand}; + /// # struct SomeConcreteType { business: String, party: String } + /// # fn today_is_a_weekday() -> bool { true } + /// + /// impl Provider for SomeConcreteType { + /// fn provide<'a>(&'a self, demand: &mut Demand<'a>) { + /// demand.provide_ref_with::(|| { + /// if today_is_a_weekday() { + /// &self.business + /// } else { + /// &self.party + /// } + /// }); + /// } + /// } + /// ``` + #[unstable(feature = "provide_any", issue = "96024")] + pub fn provide_ref_with( + &mut self, + fulfil: impl FnOnce() -> &'a T, + ) -> &mut Self { + self.provide_with::>>(fulfil) + } + + /// Provide a value with the given `Type` tag. + fn provide(&mut self, value: I::Reified) -> &mut Self + where + I: tags::Type<'a>, + { + if let Some(res @ TaggedOption(None)) = self.0.downcast_mut::() { + res.0 = Some(value); + } + self + } + + /// Provide a value with the given `Type` tag, using a closure to prevent unnecessary work. + fn provide_with(&mut self, fulfil: impl FnOnce() -> I::Reified) -> &mut Self + where + I: tags::Type<'a>, + { + if let Some(res @ TaggedOption(None)) = self.0.downcast_mut::() { + res.0 = Some(fulfil()); + } + self + } + + /// Check if the `Demand` would be satisfied if provided with a + /// value of the specified type. If the type does not match or has + /// already been provided, returns false. + /// + /// # Examples + /// + /// Check if an `u8` still needs to be provided and then provides + /// it. + /// + /// ```rust + /// #![feature(provide_any)] + /// + /// use std::any::{Provider, Demand}; + /// + /// struct Parent(Option); + /// + /// impl Provider for Parent { + /// fn provide<'a>(&'a self, demand: &mut Demand<'a>) { + /// if let Some(v) = self.0 { + /// demand.provide_value::(v); + /// } + /// } + /// } + /// + /// struct Child { + /// parent: Parent, + /// } + /// + /// impl Child { + /// // Pretend that this takes a lot of resources to evaluate. + /// fn an_expensive_computation(&self) -> Option { + /// Some(99) + /// } + /// } + /// + /// impl Provider for Child { + /// fn provide<'a>(&'a self, demand: &mut Demand<'a>) { + /// // In general, we don't know if this call will provide + /// // an `u8` value or not... + /// self.parent.provide(demand); + /// + /// // ...so we check to see if the `u8` is needed before + /// // we run our expensive computation. + /// if demand.would_be_satisfied_by_value_of::() { + /// if let Some(v) = self.an_expensive_computation() { + /// demand.provide_value::(v); + /// } + /// } + /// + /// // The demand will be satisfied now, regardless of if + /// // the parent provided the value or we did. + /// assert!(!demand.would_be_satisfied_by_value_of::()); + /// } + /// } + /// + /// let parent = Parent(Some(42)); + /// let child = Child { parent }; + /// assert_eq!(Some(42), std::any::request_value::(&child)); + /// + /// let parent = Parent(None); + /// let child = Child { parent }; + /// assert_eq!(Some(99), std::any::request_value::(&child)); + /// ``` + #[unstable(feature = "provide_any", issue = "96024")] + pub fn would_be_satisfied_by_value_of(&self) -> bool + where + T: 'static, + { + self.would_be_satisfied_by::>() + } + + /// Check if the `Demand` would be satisfied if provided with a + /// reference to a value of the specified type. If the type does + /// not match or has already been provided, returns false. + /// + /// # Examples + /// + /// Check if a `&str` still needs to be provided and then provides + /// it. + /// + /// ```rust + /// #![feature(provide_any)] + /// + /// use std::any::{Provider, Demand}; + /// + /// struct Parent(Option); + /// + /// impl Provider for Parent { + /// fn provide<'a>(&'a self, demand: &mut Demand<'a>) { + /// if let Some(v) = &self.0 { + /// demand.provide_ref::(v); + /// } + /// } + /// } + /// + /// struct Child { + /// parent: Parent, + /// name: String, + /// } + /// + /// impl Child { + /// // Pretend that this takes a lot of resources to evaluate. + /// fn an_expensive_computation(&self) -> Option<&str> { + /// Some(&self.name) + /// } + /// } + /// + /// impl Provider for Child { + /// fn provide<'a>(&'a self, demand: &mut Demand<'a>) { + /// // In general, we don't know if this call will provide + /// // a `str` reference or not... + /// self.parent.provide(demand); + /// + /// // ...so we check to see if the `&str` is needed before + /// // we run our expensive computation. + /// if demand.would_be_satisfied_by_ref_of::() { + /// if let Some(v) = self.an_expensive_computation() { + /// demand.provide_ref::(v); + /// } + /// } + /// + /// // The demand will be satisfied now, regardless of if + /// // the parent provided the reference or we did. + /// assert!(!demand.would_be_satisfied_by_ref_of::()); + /// } + /// } + /// + /// let parent = Parent(Some("parent".into())); + /// let child = Child { parent, name: "child".into() }; + /// assert_eq!(Some("parent"), std::any::request_ref::(&child)); + /// + /// let parent = Parent(None); + /// let child = Child { parent, name: "child".into() }; + /// assert_eq!(Some("child"), std::any::request_ref::(&child)); + /// ``` + #[unstable(feature = "provide_any", issue = "96024")] + pub fn would_be_satisfied_by_ref_of(&self) -> bool + where + T: ?Sized + 'static, + { + self.would_be_satisfied_by::>>() + } + + fn would_be_satisfied_by(&self) -> bool + where + I: tags::Type<'a>, + { + matches!(self.0.downcast::(), Some(TaggedOption(None))) + } +} + +#[unstable(feature = "provide_any", issue = "96024")] +impl<'a> fmt::Debug for Demand<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Demand").finish_non_exhaustive() + } +} + +/////////////////////////////////////////////////////////////////////////////// +// Type tags +/////////////////////////////////////////////////////////////////////////////// + +mod tags { + //! Type tags are used to identify a type using a separate value. This module includes type tags + //! for some very common types. + //! + //! Currently type tags are not exposed to the user. But in the future, if you want to use the + //! Provider API with more complex types (typically those including lifetime parameters), you + //! will need to write your own tags. + + use crate::marker::PhantomData; + + /// This trait is implemented by specific tag types in order to allow + /// describing a type which can be requested for a given lifetime `'a`. + /// + /// A few example implementations for type-driven tags can be found in this + /// module, although crates may also implement their own tags for more + /// complex types with internal lifetimes. + pub trait Type<'a>: Sized + 'static { + /// The type of values which may be tagged by this tag for the given + /// lifetime. + type Reified: 'a; + } + + /// Similar to the [`Type`] trait, but represents a type which may be unsized (i.e., has a + /// `?Sized` bound). E.g., `str`. + pub trait MaybeSizedType<'a>: Sized + 'static { + type Reified: 'a + ?Sized; + } + + impl<'a, T: Type<'a>> MaybeSizedType<'a> for T { + type Reified = T::Reified; + } + + /// Type-based tag for types bounded by `'static`, i.e., with no borrowed elements. + #[derive(Debug)] + pub struct Value(PhantomData); + + impl<'a, T: 'static> Type<'a> for Value { + type Reified = T; + } + + /// Type-based tag similar to [`Value`] but which may be unsized (i.e., has a `?Sized` bound). + #[derive(Debug)] + pub struct MaybeSizedValue(PhantomData); + + impl<'a, T: ?Sized + 'static> MaybeSizedType<'a> for MaybeSizedValue { + type Reified = T; + } + + /// Type-based tag for reference types (`&'a T`, where T is represented by + /// `>::Reified`. + #[derive(Debug)] + pub struct Ref(PhantomData); + + impl<'a, I: MaybeSizedType<'a>> Type<'a> for Ref { + type Reified = &'a I::Reified; + } +} + +/// An `Option` with a type tag `I`. +/// +/// Since this struct implements `Erased`, the type can be erased to make a dynamically typed +/// option. The type can be checked dynamically using `Erased::tag_id` and since this is statically +/// checked for the concrete type, there is some degree of type safety. +#[repr(transparent)] +struct TaggedOption<'a, I: tags::Type<'a>>(Option); + +impl<'a, I: tags::Type<'a>> TaggedOption<'a, I> { + fn as_demand(&mut self) -> &mut Demand<'a> { + Demand::new(self as &mut (dyn Erased<'a> + 'a)) + } +} + +/// Represents a type-erased but identifiable object. +/// +/// This trait is exclusively implemented by the `TaggedOption` type. +unsafe trait Erased<'a>: 'a { + /// The `TypeId` of the erased type. + fn tag_id(&self) -> TypeId; +} + +unsafe impl<'a, I: tags::Type<'a>> Erased<'a> for TaggedOption<'a, I> { + fn tag_id(&self) -> TypeId { + TypeId::of::() + } +} + +#[unstable(feature = "provide_any", issue = "96024")] +impl<'a> dyn Erased<'a> + 'a { + /// Returns some reference to the dynamic value if it is tagged with `I`, + /// or `None` otherwise. + #[inline] + fn downcast(&self) -> Option<&TaggedOption<'a, I>> + where + I: tags::Type<'a>, + { + if self.tag_id() == TypeId::of::() { + // SAFETY: Just checked whether we're pointing to an I. + Some(unsafe { &*(self as *const Self).cast::>() }) + } else { + None + } + } + + /// Returns some mutable reference to the dynamic value if it is tagged with `I`, + /// or `None` otherwise. + #[inline] + fn downcast_mut(&mut self) -> Option<&mut TaggedOption<'a, I>> + where + I: tags::Type<'a>, + { + if self.tag_id() == TypeId::of::() { + // SAFETY: Just checked whether we're pointing to an I. + Some(unsafe { &mut *(self as *mut Self).cast::>() }) + } else { + None + } + } +} diff --git a/crux-mir/lib/core/src/arch.rs b/crux-mir/lib/core/src/arch.rs new file mode 100644 index 000000000..fc2a5b89c --- /dev/null +++ b/crux-mir/lib/core/src/arch.rs @@ -0,0 +1,30 @@ +#![doc = include_str!("../../stdarch/crates/core_arch/src/core_arch_docs.md")] + +#[stable(feature = "simd_arch", since = "1.27.0")] +pub use crate::core_arch::arch::*; + +/// Inline assembly. +/// +/// Refer to [rust by example] for a usage guide and the [reference] for +/// detailed information about the syntax and available options. +/// +/// [rust by example]: https://doc.rust-lang.org/nightly/rust-by-example/unsafe/asm.html +/// [reference]: https://doc.rust-lang.org/nightly/reference/inline-assembly.html +#[stable(feature = "asm", since = "1.59.0")] +#[rustc_builtin_macro] +pub macro asm("assembly template", $(operands,)* $(options($(option),*))?) { + /* compiler built-in */ +} + +/// Module-level inline assembly. +/// +/// Refer to [rust by example] for a usage guide and the [reference] for +/// detailed information about the syntax and available options. +/// +/// [rust by example]: https://doc.rust-lang.org/nightly/rust-by-example/unsafe/asm.html +/// [reference]: https://doc.rust-lang.org/nightly/reference/inline-assembly.html +#[stable(feature = "global_asm", since = "1.59.0")] +#[rustc_builtin_macro] +pub macro global_asm("assembly template", $(operands,)* $(options($(option),*))?) { + /* compiler built-in */ +} diff --git a/crux-mir/lib/core/src/array/equality.rs b/crux-mir/lib/core/src/array/equality.rs new file mode 100644 index 000000000..b2c895f88 --- /dev/null +++ b/crux-mir/lib/core/src/array/equality.rs @@ -0,0 +1,217 @@ +use crate::convert::TryInto; +use crate::num::{NonZeroI128, NonZeroI16, NonZeroI32, NonZeroI64, NonZeroI8, NonZeroIsize}; +use crate::num::{NonZeroU128, NonZeroU16, NonZeroU32, NonZeroU64, NonZeroU8, NonZeroUsize}; + +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialEq<[B; N]> for [A; N] +where + A: PartialEq, +{ + #[inline] + fn eq(&self, other: &[B; N]) -> bool { + SpecArrayEq::spec_eq(self, other) + } + #[inline] + fn ne(&self, other: &[B; N]) -> bool { + SpecArrayEq::spec_ne(self, other) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialEq<[B]> for [A; N] +where + A: PartialEq, +{ + #[inline] + fn eq(&self, other: &[B]) -> bool { + let b: Result<&[B; N], _> = other.try_into(); + match b { + Ok(b) => *self == *b, + Err(_) => false, + } + } + #[inline] + fn ne(&self, other: &[B]) -> bool { + let b: Result<&[B; N], _> = other.try_into(); + match b { + Ok(b) => *self != *b, + Err(_) => true, + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialEq<[A; N]> for [B] +where + B: PartialEq, +{ + #[inline] + fn eq(&self, other: &[A; N]) -> bool { + let b: Result<&[B; N], _> = self.try_into(); + match b { + Ok(b) => *b == *other, + Err(_) => false, + } + } + #[inline] + fn ne(&self, other: &[A; N]) -> bool { + let b: Result<&[B; N], _> = self.try_into(); + match b { + Ok(b) => *b != *other, + Err(_) => true, + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialEq<&[B]> for [A; N] +where + A: PartialEq, +{ + #[inline] + fn eq(&self, other: &&[B]) -> bool { + *self == **other + } + #[inline] + fn ne(&self, other: &&[B]) -> bool { + *self != **other + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialEq<[A; N]> for &[B] +where + B: PartialEq, +{ + #[inline] + fn eq(&self, other: &[A; N]) -> bool { + **self == *other + } + #[inline] + fn ne(&self, other: &[A; N]) -> bool { + **self != *other + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialEq<&mut [B]> for [A; N] +where + A: PartialEq, +{ + #[inline] + fn eq(&self, other: &&mut [B]) -> bool { + *self == **other + } + #[inline] + fn ne(&self, other: &&mut [B]) -> bool { + *self != **other + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialEq<[A; N]> for &mut [B] +where + B: PartialEq, +{ + #[inline] + fn eq(&self, other: &[A; N]) -> bool { + **self == *other + } + #[inline] + fn ne(&self, other: &[A; N]) -> bool { + **self != *other + } +} + +// NOTE: some less important impls are omitted to reduce code bloat +// __impl_slice_eq2! { [A; $N], &'b [B; $N] } +// __impl_slice_eq2! { [A; $N], &'b mut [B; $N] } + +#[stable(feature = "rust1", since = "1.0.0")] +impl Eq for [T; N] {} + +trait SpecArrayEq: Sized { + fn spec_eq(a: &[Self; N], b: &[Other; N]) -> bool; + fn spec_ne(a: &[Self; N], b: &[Other; N]) -> bool; +} + +impl, Other, const N: usize> SpecArrayEq for T { + default fn spec_eq(a: &[Self; N], b: &[Other; N]) -> bool { + a[..] == b[..] + } + default fn spec_ne(a: &[Self; N], b: &[Other; N]) -> bool { + a[..] != b[..] + } +} + +impl, U, const N: usize> SpecArrayEq for T { + fn spec_eq(a: &[T; N], b: &[U; N]) -> bool { + // SAFETY: This is why `IsRawEqComparable` is an `unsafe trait`. + unsafe { + let b = &*b.as_ptr().cast::<[T; N]>(); + crate::intrinsics::raw_eq(a, b) + } + } + fn spec_ne(a: &[T; N], b: &[U; N]) -> bool { + !Self::spec_eq(a, b) + } +} + +/// `U` exists on here mostly because `min_specialization` didn't let me +/// repeat the `T` type parameter in the above specialization, so instead +/// the `T == U` constraint comes from the impls on this. +/// # Safety +/// - Neither `Self` nor `U` has any padding. +/// - `Self` and `U` have the same layout. +/// - `Self: PartialEq` is byte-wise (this means no floats, among other things) +#[rustc_specialization_trait] +unsafe trait IsRawEqComparable: PartialEq {} + +macro_rules! is_raw_eq_comparable { + ($($t:ty),+ $(,)?) => {$( + unsafe impl IsRawEqComparable<$t> for $t {} + )+}; +} + +// SAFETY: All the ordinary integer types have no padding, and are not pointers. +is_raw_eq_comparable!(u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize); + +// SAFETY: bool and char have *niches*, but no *padding* (and these are not pointer types), so this +// is sound +is_raw_eq_comparable!(bool, char); + +// SAFETY: Similarly, the non-zero types have a niche, but no undef and no pointers, +// and they compare like their underlying numeric type. +is_raw_eq_comparable!( + NonZeroU8, + NonZeroU16, + NonZeroU32, + NonZeroU64, + NonZeroU128, + NonZeroUsize, + NonZeroI8, + NonZeroI16, + NonZeroI32, + NonZeroI64, + NonZeroI128, + NonZeroIsize, +); + +// SAFETY: The NonZero types have the "null" optimization guaranteed, and thus +// are also safe to equality-compare bitwise inside an `Option`. +// The way `PartialOrd` is defined for `Option` means that this wouldn't work +// for `<` or `>` on the signed types, but since we only do `==` it's fine. +is_raw_eq_comparable!( + Option, + Option, + Option, + Option, + Option, + Option, + Option, + Option, + Option, + Option, + Option, + Option, +); diff --git a/crux-mir/lib/core/src/array/iter.rs b/crux-mir/lib/core/src/array/iter.rs index 80eaae0d4..8259c087d 100644 --- a/crux-mir/lib/core/src/array/iter.rs +++ b/crux-mir/lib/core/src/array/iter.rs @@ -1,22 +1,17 @@ //! Defines the `IntoIter` owned iterator for arrays. -use super::LengthAtMost32; use crate::{ fmt, - iter::{ExactSizeIterator, FusedIterator, TrustedLen}, + iter::{self, ExactSizeIterator, FusedIterator, TrustedLen}, mem::{self, MaybeUninit}, - ops::Range, + ops::{IndexRange, Range}, ptr, }; /// A by-value [array] iterator. -/// -/// [array]: ../../std/primitive.array.html -#[unstable(feature = "array_value_iter", issue = "65798")] -pub struct IntoIter -where - [T; N]: LengthAtMost32, -{ +#[stable(feature = "array_value_iter", since = "1.51.0")] +#[rustc_insignificant_dtor] +pub struct IntoIter { /// This is the array we are iterating over. /// /// Elements with index `i` where `alive.start <= i < alive.end` have not @@ -34,22 +29,29 @@ where /// The elements in `data` that have not been yielded yet. /// /// Invariants: - /// - `alive.start <= alive.end` /// - `alive.end <= N` - alive: Range, + /// + /// (And the `IndexRange` type requires `alive.start <= alive.end`.) + alive: IndexRange, } -impl IntoIter -where - [T; N]: LengthAtMost32, -{ - /// Creates a new iterator over the given `array`. +// Note: the `#[rustc_skip_array_during_method_dispatch]` on `trait IntoIterator` +// hides this implementation from explicit `.into_iter()` calls on editions < 2021, +// so those calls will still resolve to the slice implementation, by reference. +#[stable(feature = "array_into_iter_impl", since = "1.53.0")] +impl IntoIterator for [T; N] { + type Item = T; + type IntoIter = IntoIter; + + /// Creates a consuming iterator, that is, one that moves each value out of + /// the array (from start to end). The array cannot be used after calling + /// this unless `T` implements `Copy`, so the whole array is copied. /// - /// *Note*: this method might never get stabilized and/or removed in the - /// future as there will likely be another, preferred way of obtaining this - /// iterator (either via `IntoIterator` for arrays or via another way). - #[unstable(feature = "array_value_iter", issue = "65798")] - pub fn new(array: [T; N]) -> Self { + /// Arrays have special behavior when calling `.into_iter()` prior to the + /// 2021 edition -- see the [array] Editions section for more information. + /// + /// [array]: prim@array + fn into_iter(self) -> Self::IntoIter { // SAFETY: The transmute here is actually safe. The docs of `MaybeUninit` // promise: // @@ -63,72 +65,196 @@ where // FIXME(LukasKalbertodt): actually use `mem::transmute` here, once it // works with const generics: - // `mem::transmute::<[T; {N}], [MaybeUninit; {N}]>(array)` + // `mem::transmute::<[T; N], [MaybeUninit; N]>(array)` // - // Until then, we do it manually here. We first create a bitwise copy - // but cast the pointer so that it is treated as a different type. Then - // we forget `array` so that it is not dropped. - let data = unsafe { - let data = ptr::read(&array as *const [T; N] as *const [MaybeUninit; N]); - mem::forget(array); - data - }; - - Self { data, alive: 0..N } + // Until then, we can use `mem::transmute_copy` to create a bitwise copy + // as a different type, then forget `array` so that it is not dropped. + unsafe { + let iter = IntoIter { data: mem::transmute_copy(&self), alive: IndexRange::zero_to(N) }; + mem::forget(self); + iter + } + } +} + +impl IntoIter { + /// Creates a new iterator over the given `array`. + #[stable(feature = "array_value_iter", since = "1.51.0")] + #[deprecated(since = "1.59.0", note = "use `IntoIterator::into_iter` instead")] + pub fn new(array: [T; N]) -> Self { + IntoIterator::into_iter(array) + } + + /// Creates an iterator over the elements in a partially-initialized buffer. + /// + /// If you have a fully-initialized array, then use [`IntoIterator`]. + /// But this is useful for returning partial results from unsafe code. + /// + /// # Safety + /// + /// - The `buffer[initialized]` elements must all be initialized. + /// - The range must be canonical, with `initialized.start <= initialized.end`. + /// - The range must be in-bounds for the buffer, with `initialized.end <= N`. + /// (Like how indexing `[0][100..100]` fails despite the range being empty.) + /// + /// It's sound to have more elements initialized than mentioned, though that + /// will most likely result in them being leaked. + /// + /// # Examples + /// + /// ``` + /// #![feature(array_into_iter_constructors)] + /// #![feature(maybe_uninit_uninit_array_transpose)] + /// #![feature(maybe_uninit_uninit_array)] + /// use std::array::IntoIter; + /// use std::mem::MaybeUninit; + /// + /// # // Hi! Thanks for reading the code. This is restricted to `Copy` because + /// # // otherwise it could leak. A fully-general version this would need a drop + /// # // guard to handle panics from the iterator, but this works for an example. + /// fn next_chunk( + /// it: &mut impl Iterator, + /// ) -> Result<[T; N], IntoIter> { + /// let mut buffer = MaybeUninit::uninit_array(); + /// let mut i = 0; + /// while i < N { + /// match it.next() { + /// Some(x) => { + /// buffer[i].write(x); + /// i += 1; + /// } + /// None => { + /// // SAFETY: We've initialized the first `i` items + /// unsafe { + /// return Err(IntoIter::new_unchecked(buffer, 0..i)); + /// } + /// } + /// } + /// } + /// + /// // SAFETY: We've initialized all N items + /// unsafe { Ok(buffer.transpose().assume_init()) } + /// } + /// + /// let r: [_; 4] = next_chunk(&mut (10..16)).unwrap(); + /// assert_eq!(r, [10, 11, 12, 13]); + /// let r: IntoIter<_, 40> = next_chunk(&mut (10..16)).unwrap_err(); + /// assert_eq!(r.collect::>(), vec![10, 11, 12, 13, 14, 15]); + /// ``` + #[unstable(feature = "array_into_iter_constructors", issue = "91583")] + #[rustc_const_unstable(feature = "const_array_into_iter_constructors", issue = "91583")] + pub const unsafe fn new_unchecked( + buffer: [MaybeUninit; N], + initialized: Range, + ) -> Self { + // SAFETY: one of our safety conditions is that the range is canonical. + let alive = unsafe { IndexRange::new_unchecked(initialized.start, initialized.end) }; + Self { data: buffer, alive } + } + + /// Creates an iterator over `T` which returns no elements. + /// + /// If you just need an empty iterator, then use + /// [`iter::empty()`](crate::iter::empty) instead. + /// And if you need an empty array, use `[]`. + /// + /// But this is useful when you need an `array::IntoIter` *specifically*. + /// + /// # Examples + /// + /// ``` + /// #![feature(array_into_iter_constructors)] + /// use std::array::IntoIter; + /// + /// let empty = IntoIter::::empty(); + /// assert_eq!(empty.len(), 0); + /// assert_eq!(empty.as_slice(), &[]); + /// + /// let empty = IntoIter::::empty(); + /// assert_eq!(empty.len(), 0); + /// ``` + /// + /// `[1, 2].into_iter()` and `[].into_iter()` have different types + /// ```should_fail,edition2021 + /// #![feature(array_into_iter_constructors)] + /// use std::array::IntoIter; + /// + /// pub fn get_bytes(b: bool) -> IntoIter { + /// if b { + /// [1, 2, 3, 4].into_iter() + /// } else { + /// [].into_iter() // error[E0308]: mismatched types + /// } + /// } + /// ``` + /// + /// But using this method you can get an empty iterator of appropriate size: + /// ```edition2021 + /// #![feature(array_into_iter_constructors)] + /// use std::array::IntoIter; + /// + /// pub fn get_bytes(b: bool) -> IntoIter { + /// if b { + /// [1, 2, 3, 4].into_iter() + /// } else { + /// IntoIter::empty() + /// } + /// } + /// + /// assert_eq!(get_bytes(true).collect::>(), vec![1, 2, 3, 4]); + /// assert_eq!(get_bytes(false).collect::>(), vec![]); + /// ``` + #[unstable(feature = "array_into_iter_constructors", issue = "91583")] + #[rustc_const_unstable(feature = "const_array_into_iter_constructors", issue = "91583")] + pub const fn empty() -> Self { + let buffer = MaybeUninit::uninit_array(); + let initialized = 0..0; + + // SAFETY: We're telling it that none of the elements are initialized, + // which is trivially true. And ∀N: usize, 0 <= N. + unsafe { Self::new_unchecked(buffer, initialized) } } /// Returns an immutable slice of all elements that have not been yielded /// yet. - fn as_slice(&self) -> &[T] { - let slice = &self.data[self.alive.clone()]; - // SAFETY: This transmute is safe. As mentioned in `new`, `MaybeUninit` retains - // the size and alignment of `T`. Furthermore, we know that all - // elements within `alive` are properly initialized. - unsafe { mem::transmute::<&[MaybeUninit], &[T]>(slice) } + #[stable(feature = "array_value_iter", since = "1.51.0")] + pub fn as_slice(&self) -> &[T] { + // SAFETY: We know that all elements within `alive` are properly initialized. + unsafe { + let slice = self.data.get_unchecked(self.alive.clone()); + MaybeUninit::slice_assume_init_ref(slice) + } } /// Returns a mutable slice of all elements that have not been yielded yet. - fn as_mut_slice(&mut self) -> &mut [T] { - // This transmute is safe, same as in `as_slice` above. - let slice = &mut self.data[self.alive.clone()]; - // SAFETY: This transmute is safe. As mentioned in `new`, `MaybeUninit` retains - // the size and alignment of `T`. Furthermore, we know that all - // elements within `alive` are properly initialized. - unsafe { mem::transmute::<&mut [MaybeUninit], &mut [T]>(slice) } + #[stable(feature = "array_value_iter", since = "1.51.0")] + pub fn as_mut_slice(&mut self) -> &mut [T] { + // SAFETY: We know that all elements within `alive` are properly initialized. + unsafe { + let slice = self.data.get_unchecked_mut(self.alive.clone()); + MaybeUninit::slice_assume_init_mut(slice) + } } } #[stable(feature = "array_value_iter_impls", since = "1.40.0")] -impl Iterator for IntoIter -where - [T; N]: LengthAtMost32, -{ +impl Iterator for IntoIter { type Item = T; fn next(&mut self) -> Option { - if self.alive.start == self.alive.end { - return None; - } - - // Bump start index. + // Get the next index from the front. // - // From the check above we know that `alive.start != alive.end`. - // Combine this with the invariant `alive.start <= alive.end`, we know - // that `alive.start < alive.end`. Increasing `alive.start` by 1 - // maintains the invariant regarding `alive`. However, due to this - // change, for a short time, the alive zone is not `data[alive]` - // anymore, but `data[idx..alive.end]`. - let idx = self.alive.start; - self.alive.start += 1; - - // Read the element from the array. - // SAFETY: This is safe: `idx` is an index - // into the "alive" region of the array. Reading this element means - // that `data[idx]` is regarded as dead now (i.e. do not touch). As - // `idx` was the start of the alive-zone, the alive zone is now - // `data[alive]` again, restoring all invariants. - let out = unsafe { self.data.get_unchecked(idx).read() }; - - Some(out) + // Increasing `alive.start` by 1 maintains the invariant regarding + // `alive`. However, due to this change, for a short time, the alive + // zone is not `data[alive]` anymore, but `data[idx..alive.end]`. + self.alive.next().map(|idx| { + // Read the element from the array. + // SAFETY: `idx` is an index into the former "alive" region of the + // array. Reading this element means that `data[idx]` is regarded as + // dead now (i.e. do not touch). As `idx` was the start of the + // alive-zone, the alive zone is now `data[alive]` again, restoring + // all invariants. + unsafe { self.data.get_unchecked(idx).assume_init_read() } + }) } fn size_hint(&self) -> (usize, Option) { @@ -136,6 +262,20 @@ where (len, Some(len)) } + #[inline] + fn fold(mut self, init: Acc, mut fold: Fold) -> Acc + where + Fold: FnMut(Acc, Self::Item) -> Acc, + { + let data = &mut self.data; + iter::ByRefSized(&mut self.alive).fold(init, |acc, idx| { + // SAFETY: idx is obtained by folding over the `alive` range, which implies the + // value is currently considered alive but as the range is being consumed each value + // we read here will only be read once and then considered dead. + fold(acc, unsafe { data.get_unchecked(idx).assume_init_read() }) + }) + } + fn count(self) -> usize { self.len() } @@ -143,49 +283,76 @@ where fn last(mut self) -> Option { self.next_back() } + + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + let original_len = self.len(); + + // This also moves the start, which marks them as conceptually "dropped", + // so if anything goes bad then our drop impl won't double-free them. + let range_to_drop = self.alive.take_prefix(n); + + // SAFETY: These elements are currently initialized, so it's fine to drop them. + unsafe { + let slice = self.data.get_unchecked_mut(range_to_drop); + ptr::drop_in_place(MaybeUninit::slice_assume_init_mut(slice)); + } + + if n > original_len { Err(original_len) } else { Ok(()) } + } } #[stable(feature = "array_value_iter_impls", since = "1.40.0")] -impl DoubleEndedIterator for IntoIter -where - [T; N]: LengthAtMost32, -{ +impl DoubleEndedIterator for IntoIter { fn next_back(&mut self) -> Option { - if self.alive.start == self.alive.end { - return None; + // Get the next index from the back. + // + // Decreasing `alive.end` by 1 maintains the invariant regarding + // `alive`. However, due to this change, for a short time, the alive + // zone is not `data[alive]` anymore, but `data[alive.start..=idx]`. + self.alive.next_back().map(|idx| { + // Read the element from the array. + // SAFETY: `idx` is an index into the former "alive" region of the + // array. Reading this element means that `data[idx]` is regarded as + // dead now (i.e. do not touch). As `idx` was the end of the + // alive-zone, the alive zone is now `data[alive]` again, restoring + // all invariants. + unsafe { self.data.get_unchecked(idx).assume_init_read() } + }) + } + + #[inline] + fn rfold(mut self, init: Acc, mut rfold: Fold) -> Acc + where + Fold: FnMut(Acc, Self::Item) -> Acc, + { + let data = &mut self.data; + iter::ByRefSized(&mut self.alive).rfold(init, |acc, idx| { + // SAFETY: idx is obtained by folding over the `alive` range, which implies the + // value is currently considered alive but as the range is being consumed each value + // we read here will only be read once and then considered dead. + rfold(acc, unsafe { data.get_unchecked(idx).assume_init_read() }) + }) + } + + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + let original_len = self.len(); + + // This also moves the end, which marks them as conceptually "dropped", + // so if anything goes bad then our drop impl won't double-free them. + let range_to_drop = self.alive.take_suffix(n); + + // SAFETY: These elements are currently initialized, so it's fine to drop them. + unsafe { + let slice = self.data.get_unchecked_mut(range_to_drop); + ptr::drop_in_place(MaybeUninit::slice_assume_init_mut(slice)); } - // Decrease end index. - // - // From the check above we know that `alive.start != alive.end`. - // Combine this with the invariant `alive.start <= alive.end`, we know - // that `alive.start < alive.end`. As `alive.start` cannot be negative, - // `alive.end` is at least 1, meaning that we can safely decrement it - // by one. This also maintains the invariant `alive.start <= - // alive.end`. However, due to this change, for a short time, the alive - // zone is not `data[alive]` anymore, but `data[alive.start..alive.end - // + 1]`. - self.alive.end -= 1; - - // Read the element from the array. - // SAFETY: This is safe: `alive.end` is an - // index into the "alive" region of the array. Compare the previous - // comment that states that the alive region is - // `data[alive.start..alive.end + 1]`. Reading this element means that - // `data[alive.end]` is regarded as dead now (i.e. do not touch). As - // `alive.end` was the end of the alive-zone, the alive zone is now - // `data[alive]` again, restoring all invariants. - let out = unsafe { self.data.get_unchecked(self.alive.end).read() }; - - Some(out) + if n > original_len { Err(original_len) } else { Ok(()) } } } #[stable(feature = "array_value_iter_impls", since = "1.40.0")] -impl Drop for IntoIter -where - [T; N]: LengthAtMost32, -{ +impl Drop for IntoIter { fn drop(&mut self) { // SAFETY: This is safe: `as_mut_slice` returns exactly the sub-slice // of elements that have not been moved out yet and that remain @@ -195,14 +362,9 @@ where } #[stable(feature = "array_value_iter_impls", since = "1.40.0")] -impl ExactSizeIterator for IntoIter -where - [T; N]: LengthAtMost32, -{ +impl ExactSizeIterator for IntoIter { fn len(&self) -> usize { - // Will never underflow due to the invariant `alive.start <= - // alive.end`. - self.alive.end - self.alive.start + self.alive.len() } fn is_empty(&self) -> bool { self.alive.is_empty() @@ -210,49 +372,37 @@ where } #[stable(feature = "array_value_iter_impls", since = "1.40.0")] -impl FusedIterator for IntoIter where [T; N]: LengthAtMost32 {} +impl FusedIterator for IntoIter {} // The iterator indeed reports the correct length. The number of "alive" // elements (that will still be yielded) is the length of the range `alive`. // This range is decremented in length in either `next` or `next_back`. It is // always decremented by 1 in those methods, but only if `Some(_)` is returned. #[stable(feature = "array_value_iter_impls", since = "1.40.0")] -unsafe impl TrustedLen for IntoIter where [T; N]: LengthAtMost32 {} +unsafe impl TrustedLen for IntoIter {} #[stable(feature = "array_value_iter_impls", since = "1.40.0")] -impl Clone for IntoIter -where - [T; N]: LengthAtMost32, -{ +impl Clone for IntoIter { fn clone(&self) -> Self { - // SAFETY: each point of unsafety is documented inside the unsafe block - unsafe { - // This creates a new uninitialized array. Note that the `assume_init` - // refers to the array, not the individual elements. And it is Ok if - // the array is in an uninitialized state as all elements may be - // uninitialized (all bit patterns are valid). Compare the - // `MaybeUninit` docs for more information. - let mut new_data: [MaybeUninit; N] = MaybeUninit::uninit().assume_init(); - - // Clone all alive elements. - for idx in self.alive.clone() { - // The element at `idx` in the old array is alive, so we can - // safely call `get_ref()`. We then clone it, and write the - // clone into the new array. - let clone = self.data.get_unchecked(idx).get_ref().clone(); - new_data.get_unchecked_mut(idx).write(clone); - } - - Self { data: new_data, alive: self.alive.clone() } + // Note, we don't really need to match the exact same alive range, so + // we can just clone into offset 0 regardless of where `self` is. + let mut new = Self { data: MaybeUninit::uninit_array(), alive: IndexRange::zero_to(0) }; + + // Clone all alive elements. + for (src, dst) in iter::zip(self.as_slice(), &mut new.data) { + // Write a clone into the new array, then update its alive range. + // If cloning panics, we'll correctly drop the previous items. + dst.write(src.clone()); + // This addition cannot overflow as we're iterating a slice + new.alive = IndexRange::zero_to(new.alive.end() + 1); } + + new } } #[stable(feature = "array_value_iter_impls", since = "1.40.0")] -impl fmt::Debug for IntoIter -where - [T; N]: LengthAtMost32, -{ +impl fmt::Debug for IntoIter { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // Only print the elements that were not yielded yet: we cannot // access the yielded elements anymore. diff --git a/crux-mir/lib/core/src/array/mod.rs b/crux-mir/lib/core/src/array/mod.rs index 811c9d71e..2825e0bbb 100644 --- a/crux-mir/lib/core/src/array/mod.rs +++ b/crux-mir/lib/core/src/array/mod.rs @@ -1,60 +1,125 @@ -//! Implementations of things like `Eq` for fixed-length arrays -//! up to a certain length. Eventually, we should be able to generalize -//! to all lengths. +//! Utilities for the array primitive type. //! -//! *[See also the array primitive type](../../std/primitive.array.html).* +//! *[See also the array primitive type](array).* #![stable(feature = "core_array", since = "1.36.0")] use crate::borrow::{Borrow, BorrowMut}; use crate::cmp::Ordering; use crate::convert::{Infallible, TryFrom}; +use crate::error::Error; use crate::fmt; use crate::hash::{self, Hash}; -use crate::marker::Unsize; +use crate::iter::TrustedLen; +use crate::mem::{self, MaybeUninit}; +use crate::ops::{ + ChangeOutputType, ControlFlow, FromResidual, Index, IndexMut, NeverShortCircuit, Residual, Try, +}; use crate::slice::{Iter, IterMut}; +mod equality; mod iter; -#[unstable(feature = "array_value_iter", issue = "65798")] +#[stable(feature = "array_value_iter", since = "1.51.0")] pub use iter::IntoIter; -/// Utility trait implemented only on arrays of fixed size +/// Creates an array of type [T; N], where each element `T` is the returned value from `cb` +/// using that element's index. /// -/// This trait can be used to implement other traits on fixed-size arrays -/// without causing much metadata bloat. +/// # Arguments /// -/// The trait is marked unsafe in order to restrict implementors to fixed-size -/// arrays. User of this trait can assume that implementors have the exact -/// layout in memory of a fixed size array (for example, for unsafe -/// initialization). +/// * `cb`: Callback where the passed argument is the current array index. /// -/// Note that the traits [`AsRef`] and [`AsMut`] provide similar methods for types that -/// may not be fixed-size arrays. Implementors should prefer those traits -/// instead. +/// # Example /// -/// [`AsRef`]: ../convert/trait.AsRef.html -/// [`AsMut`]: ../convert/trait.AsMut.html -#[unstable(feature = "fixed_size_array", issue = "27778")] -pub unsafe trait FixedSizeArray { - /// Converts the array to immutable slice - #[unstable(feature = "fixed_size_array", issue = "27778")] - fn as_slice(&self) -> &[T]; - /// Converts the array to mutable slice - #[unstable(feature = "fixed_size_array", issue = "27778")] - fn as_mut_slice(&mut self) -> &mut [T]; +/// ```rust +/// // type inference is helping us here, the way `from_fn` knows how many +/// // elements to produce is the length of array down there: only arrays of +/// // equal lengths can be compared, so the const generic parameter `N` is +/// // inferred to be 5, thus creating array of 5 elements. +/// +/// let array = core::array::from_fn(|i| i); +/// // indexes are: 0 1 2 3 4 +/// assert_eq!(array, [0, 1, 2, 3, 4]); +/// +/// let array2: [usize; 8] = core::array::from_fn(|i| i * 2); +/// // indexes are: 0 1 2 3 4 5 6 7 +/// assert_eq!(array2, [0, 2, 4, 6, 8, 10, 12, 14]); +/// +/// let bool_arr = core::array::from_fn::<_, 5, _>(|i| i % 2 == 0); +/// // indexes are: 0 1 2 3 4 +/// assert_eq!(bool_arr, [true, false, true, false, true]); +/// ``` +#[inline] +#[stable(feature = "array_from_fn", since = "1.63.0")] +pub fn from_fn(mut cb: F) -> [T; N] +where + F: FnMut(usize) -> T, +{ + let mut idx = 0; + [(); N].map(|_| { + let res = cb(idx); + idx += 1; + res + }) } -#[unstable(feature = "fixed_size_array", issue = "27778")] -unsafe impl> FixedSizeArray for A { - #[inline] - fn as_slice(&self) -> &[T] { - self - } - #[inline] - fn as_mut_slice(&mut self) -> &mut [T] { - self - } +/// Creates an array `[T; N]` where each fallible array element `T` is returned by the `cb` call. +/// Unlike [`from_fn`], where the element creation can't fail, this version will return an error +/// if any element creation was unsuccessful. +/// +/// The return type of this function depends on the return type of the closure. +/// If you return `Result` from the closure, you'll get a `Result<[T; N], E>`. +/// If you return `Option` from the closure, you'll get an `Option<[T; N]>`. +/// +/// # Arguments +/// +/// * `cb`: Callback where the passed argument is the current array index. +/// +/// # Example +/// +/// ```rust +/// #![feature(array_try_from_fn)] +/// +/// let array: Result<[u8; 5], _> = std::array::try_from_fn(|i| i.try_into()); +/// assert_eq!(array, Ok([0, 1, 2, 3, 4])); +/// +/// let array: Result<[i8; 200], _> = std::array::try_from_fn(|i| i.try_into()); +/// assert!(array.is_err()); +/// +/// let array: Option<[_; 4]> = std::array::try_from_fn(|i| i.checked_add(100)); +/// assert_eq!(array, Some([100, 101, 102, 103])); +/// +/// let array: Option<[_; 4]> = std::array::try_from_fn(|i| i.checked_sub(100)); +/// assert_eq!(array, None); +/// ``` +#[inline] +#[unstable(feature = "array_try_from_fn", issue = "89379")] +pub fn try_from_fn(cb: F) -> ChangeOutputType +where + F: FnMut(usize) -> R, + R: Try, + R::Residual: Residual<[R::Output; N]>, +{ + // SAFETY: we know for certain that this iterator will yield exactly `N` + // items. + unsafe { try_collect_into_array_unchecked(&mut (0..N).map(cb)) } +} + +/// Converts a reference to `T` into a reference to an array of length 1 (without copying). +#[stable(feature = "array_from_ref", since = "1.53.0")] +#[rustc_const_stable(feature = "const_array_from_ref_shared", since = "1.63.0")] +pub const fn from_ref(s: &T) -> &[T; 1] { + // SAFETY: Converting `&T` to `&[T; 1]` is sound. + unsafe { &*(s as *const T).cast::<[T; 1]>() } +} + +/// Converts a mutable reference to `T` into a mutable reference to an array of length 1 (without copying). +#[stable(feature = "array_from_ref", since = "1.53.0")] +#[rustc_const_unstable(feature = "const_array_from_ref", issue = "90206")] +pub const fn from_mut(s: &mut T) -> &mut [T; 1] { + // SAFETY: Converting `&mut T` to `&mut [T; 1]` is sound. + unsafe { &mut *(s as *mut T).cast::<[T; 1]>() } } /// The error type returned when a conversion from a slice to an array fails. @@ -70,6 +135,14 @@ impl fmt::Display for TryFromSliceError { } } +#[stable(feature = "try_from", since = "1.34.0")] +impl Error for TryFromSliceError { + #[allow(deprecated)] + fn description(&self) -> &str { + self.__description() + } +} + impl TryFromSliceError { #[unstable( feature = "array_error_internals", @@ -85,17 +158,15 @@ impl TryFromSliceError { } #[stable(feature = "try_from_slice_error", since = "1.36.0")] -impl From for TryFromSliceError { +#[rustc_const_unstable(feature = "const_convert", issue = "88674")] +impl const From for TryFromSliceError { fn from(x: Infallible) -> TryFromSliceError { match x {} } } #[stable(feature = "rust1", since = "1.0.0")] -impl AsRef<[T]> for [T; N] -where - [T; N]: LengthAtMost32, -{ +impl AsRef<[T]> for [T; N] { #[inline] fn as_ref(&self) -> &[T] { &self[..] @@ -103,10 +174,7 @@ where } #[stable(feature = "rust1", since = "1.0.0")] -impl AsMut<[T]> for [T; N] -where - [T; N]: LengthAtMost32, -{ +impl AsMut<[T]> for [T; N] { #[inline] fn as_mut(&mut self) -> &mut [T] { &mut self[..] @@ -114,30 +182,37 @@ where } #[stable(feature = "array_borrow", since = "1.4.0")] -impl Borrow<[T]> for [T; N] -where - [T; N]: LengthAtMost32, -{ +#[rustc_const_unstable(feature = "const_borrow", issue = "91522")] +impl const Borrow<[T]> for [T; N] { fn borrow(&self) -> &[T] { self } } #[stable(feature = "array_borrow", since = "1.4.0")] -impl BorrowMut<[T]> for [T; N] -where - [T; N]: LengthAtMost32, -{ +#[rustc_const_unstable(feature = "const_borrow", issue = "91522")] +impl const BorrowMut<[T]> for [T; N] { fn borrow_mut(&mut self) -> &mut [T] { self } } +/// Tries to create an array `[T; N]` by copying from a slice `&[T]`. Succeeds if +/// `slice.len() == N`. +/// +/// ``` +/// let bytes: [u8; 3] = [1, 0, 2]; +/// +/// let bytes_head: [u8; 2] = <[u8; 2]>::try_from(&bytes[0..2]).unwrap(); +/// assert_eq!(1, u16::from_le_bytes(bytes_head)); +/// +/// let bytes_tail: [u8; 2] = bytes[1..3].try_into().unwrap(); +/// assert_eq!(512, u16::from_le_bytes(bytes_tail)); +/// ``` #[stable(feature = "try_from", since = "1.34.0")] impl TryFrom<&[T]> for [T; N] where T: Copy, - [T; N]: LengthAtMost32, { type Error = TryFromSliceError; @@ -146,40 +221,74 @@ where } } -#[stable(feature = "try_from", since = "1.34.0")] -impl<'a, T, const N: usize> TryFrom<&'a [T]> for &'a [T; N] +/// Tries to create an array `[T; N]` by copying from a mutable slice `&mut [T]`. +/// Succeeds if `slice.len() == N`. +/// +/// ``` +/// let mut bytes: [u8; 3] = [1, 0, 2]; +/// +/// let bytes_head: [u8; 2] = <[u8; 2]>::try_from(&mut bytes[0..2]).unwrap(); +/// assert_eq!(1, u16::from_le_bytes(bytes_head)); +/// +/// let bytes_tail: [u8; 2] = (&mut bytes[1..3]).try_into().unwrap(); +/// assert_eq!(512, u16::from_le_bytes(bytes_tail)); +/// ``` +#[stable(feature = "try_from_mut_slice_to_array", since = "1.59.0")] +impl TryFrom<&mut [T]> for [T; N] where - [T; N]: LengthAtMost32, + T: Copy, { type Error = TryFromSliceError; + fn try_from(slice: &mut [T]) -> Result<[T; N], TryFromSliceError> { + ::try_from(&*slice) + } +} + +/// Tries to create an array ref `&[T; N]` from a slice ref `&[T]`. Succeeds if +/// `slice.len() == N`. +/// +/// ``` +/// let bytes: [u8; 3] = [1, 0, 2]; +/// +/// let bytes_head: &[u8; 2] = <&[u8; 2]>::try_from(&bytes[0..2]).unwrap(); +/// assert_eq!(1, u16::from_le_bytes(*bytes_head)); +/// +/// let bytes_tail: &[u8; 2] = bytes[1..3].try_into().unwrap(); +/// assert_eq!(512, u16::from_le_bytes(*bytes_tail)); +/// ``` +#[stable(feature = "try_from", since = "1.34.0")] +impl<'a, T, const N: usize> TryFrom<&'a [T]> for &'a [T; N] { + type Error = TryFromSliceError; + fn try_from(slice: &[T]) -> Result<&[T; N], TryFromSliceError> { - fn crucible_array_from_slice_hook( - slice: &[T], - len: usize, - ) -> Option<&[T; N]> { - if slice.len() == N { - let ptr = slice.as_ptr() as *const [T; N]; - // SAFETY: ok because we just checked that the length fits - unsafe { Some(&*ptr) } - } else { - None - } + if slice.len() == N { + let ptr = slice.as_ptr() as *const [T; N]; + // SAFETY: ok because we just checked that the length fits + unsafe { Ok(&*ptr) } + } else { + Err(TryFromSliceError(())) } - crucible_array_from_slice_hook(slice, N) - .ok_or(TryFromSliceError(())) } } +/// Tries to create a mutable array ref `&mut [T; N]` from a mutable slice ref +/// `&mut [T]`. Succeeds if `slice.len() == N`. +/// +/// ``` +/// let mut bytes: [u8; 3] = [1, 0, 2]; +/// +/// let bytes_head: &mut [u8; 2] = <&mut [u8; 2]>::try_from(&mut bytes[0..2]).unwrap(); +/// assert_eq!(1, u16::from_le_bytes(*bytes_head)); +/// +/// let bytes_tail: &mut [u8; 2] = (&mut bytes[1..3]).try_into().unwrap(); +/// assert_eq!(512, u16::from_le_bytes(*bytes_tail)); +/// ``` #[stable(feature = "try_from", since = "1.34.0")] -impl<'a, T, const N: usize> TryFrom<&'a mut [T]> for &'a mut [T; N] -where - [T; N]: LengthAtMost32, -{ +impl<'a, T, const N: usize> TryFrom<&'a mut [T]> for &'a mut [T; N] { type Error = TryFromSliceError; fn try_from(slice: &mut [T]) -> Result<&mut [T; N], TryFromSliceError> { - // Mut version is not supported by crucible - see comment in TransCustom.array_from_slice if slice.len() == N { let ptr = slice.as_mut_ptr() as *mut [T; N]; // SAFETY: ok because we just checked that the length fits @@ -190,31 +299,34 @@ where } } +/// The hash of an array is the same as that of the corresponding slice, +/// as required by the `Borrow` implementation. +/// +/// ``` +/// #![feature(build_hasher_simple_hash_one)] +/// use std::hash::BuildHasher; +/// +/// let b = std::collections::hash_map::RandomState::new(); +/// let a: [u8; 3] = [0xa8, 0x3c, 0x09]; +/// let s: &[u8] = &[0xa8, 0x3c, 0x09]; +/// assert_eq!(b.hash_one(a), b.hash_one(s)); +/// ``` #[stable(feature = "rust1", since = "1.0.0")] -impl Hash for [T; N] -where - [T; N]: LengthAtMost32, -{ +impl Hash for [T; N] { fn hash(&self, state: &mut H) { Hash::hash(&self[..], state) } } #[stable(feature = "rust1", since = "1.0.0")] -impl fmt::Debug for [T; N] -where - [T; N]: LengthAtMost32, -{ +impl fmt::Debug for [T; N] { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Debug::fmt(&&self[..], f) } } #[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T, const N: usize> IntoIterator for &'a [T; N] -where - [T; N]: LengthAtMost32, -{ +impl<'a, T, const N: usize> IntoIterator for &'a [T; N] { type Item = &'a T; type IntoIter = Iter<'a, T>; @@ -224,10 +336,7 @@ where } #[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T, const N: usize> IntoIterator for &'a mut [T; N] -where - [T; N]: LengthAtMost32, -{ +impl<'a, T, const N: usize> IntoIterator for &'a mut [T; N] { type Item = &'a mut T; type IntoIter = IterMut<'a, T>; @@ -236,131 +345,34 @@ where } } -#[stable(feature = "rust1", since = "1.0.0")] -impl PartialEq<[B; N]> for [A; N] +#[stable(feature = "index_trait_on_arrays", since = "1.50.0")] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +impl const Index for [T; N] where - A: PartialEq, - [A; N]: LengthAtMost32, - [B; N]: LengthAtMost32, + [T]: ~const Index, { - #[inline] - fn eq(&self, other: &[B; N]) -> bool { - self[..] == other[..] - } - #[inline] - fn ne(&self, other: &[B; N]) -> bool { - self[..] != other[..] - } -} + type Output = <[T] as Index>::Output; -#[stable(feature = "rust1", since = "1.0.0")] -impl PartialEq<[B]> for [A; N] -where - A: PartialEq, - [A; N]: LengthAtMost32, -{ - #[inline] - fn eq(&self, other: &[B]) -> bool { - self[..] == other[..] - } - #[inline] - fn ne(&self, other: &[B]) -> bool { - self[..] != other[..] - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl PartialEq<[A; N]> for [B] -where - B: PartialEq, - [A; N]: LengthAtMost32, -{ #[inline] - fn eq(&self, other: &[A; N]) -> bool { - self[..] == other[..] - } - #[inline] - fn ne(&self, other: &[A; N]) -> bool { - self[..] != other[..] + fn index(&self, index: I) -> &Self::Output { + Index::index(self as &[T], index) } } -#[stable(feature = "rust1", since = "1.0.0")] -impl<'b, A, B, const N: usize> PartialEq<&'b [B]> for [A; N] +#[stable(feature = "index_trait_on_arrays", since = "1.50.0")] +#[rustc_const_unstable(feature = "const_slice_index", issue = "none")] +impl const IndexMut for [T; N] where - A: PartialEq, - [A; N]: LengthAtMost32, + [T]: ~const IndexMut, { #[inline] - fn eq(&self, other: &&'b [B]) -> bool { - self[..] == other[..] - } - #[inline] - fn ne(&self, other: &&'b [B]) -> bool { - self[..] != other[..] + fn index_mut(&mut self, index: I) -> &mut Self::Output { + IndexMut::index_mut(self as &mut [T], index) } } #[stable(feature = "rust1", since = "1.0.0")] -impl<'b, A, B, const N: usize> PartialEq<[A; N]> for &'b [B] -where - B: PartialEq, - [A; N]: LengthAtMost32, -{ - #[inline] - fn eq(&self, other: &[A; N]) -> bool { - self[..] == other[..] - } - #[inline] - fn ne(&self, other: &[A; N]) -> bool { - self[..] != other[..] - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'b, A, B, const N: usize> PartialEq<&'b mut [B]> for [A; N] -where - A: PartialEq, - [A; N]: LengthAtMost32, -{ - #[inline] - fn eq(&self, other: &&'b mut [B]) -> bool { - self[..] == other[..] - } - #[inline] - fn ne(&self, other: &&'b mut [B]) -> bool { - self[..] != other[..] - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'b, A, B, const N: usize> PartialEq<[A; N]> for &'b mut [B] -where - B: PartialEq, - [A; N]: LengthAtMost32, -{ - #[inline] - fn eq(&self, other: &[A; N]) -> bool { - self[..] == other[..] - } - #[inline] - fn ne(&self, other: &[A; N]) -> bool { - self[..] != other[..] - } -} - -// NOTE: some less important impls are omitted to reduce code bloat -// __impl_slice_eq2! { [A; $N], &'b [B; $N] } -// __impl_slice_eq2! { [A; $N], &'b mut [B; $N] } - -#[stable(feature = "rust1", since = "1.0.0")] -impl Eq for [T; N] where [T; N]: LengthAtMost32 {} - -#[stable(feature = "rust1", since = "1.0.0")] -impl PartialOrd for [T; N] -where - [T; N]: LengthAtMost32, -{ +impl PartialOrd for [T; N] { #[inline] fn partial_cmp(&self, other: &[T; N]) -> Option { PartialOrd::partial_cmp(&&self[..], &&other[..]) @@ -383,49 +395,60 @@ where } } +/// Implements comparison of arrays [lexicographically](Ord#lexicographical-comparison). #[stable(feature = "rust1", since = "1.0.0")] -impl Ord for [T; N] -where - [T; N]: LengthAtMost32, -{ +impl Ord for [T; N] { #[inline] fn cmp(&self, other: &[T; N]) -> Ordering { Ord::cmp(&&self[..], &&other[..]) } } -/// Implemented for lengths where trait impls are allowed on arrays in core/std -#[rustc_on_unimplemented(message = "arrays only have std trait implementations for lengths 0..=32")] -#[unstable( - feature = "const_generic_impls_guard", - issue = "none", - reason = "will never be stable, just a temporary step until const generics are stable" -)] -pub trait LengthAtMost32 {} +#[stable(feature = "copy_clone_array_lib", since = "1.58.0")] +impl Copy for [T; N] {} -macro_rules! array_impls { - ($($N:literal)+) => { - $( - #[unstable(feature = "const_generic_impls_guard", issue = "none")] - impl LengthAtMost32 for [T; $N] {} - )+ +#[stable(feature = "copy_clone_array_lib", since = "1.58.0")] +impl Clone for [T; N] { + #[inline] + fn clone(&self) -> Self { + SpecArrayClone::clone(self) + } + + #[inline] + fn clone_from(&mut self, other: &Self) { + self.clone_from_slice(other); } } -array_impls! { - 0 1 2 3 4 5 6 7 8 9 - 10 11 12 13 14 15 16 17 18 19 - 20 21 22 23 24 25 26 27 28 29 - 30 31 32 +trait SpecArrayClone: Clone { + fn clone(array: &[Self; N]) -> [Self; N]; } -// The Default impls cannot be generated using the array_impls! macro because -// they require array literals. +impl SpecArrayClone for T { + #[inline] + default fn clone(array: &[T; N]) -> [T; N] { + // SAFETY: we know for certain that this iterator will yield exactly `N` + // items. + unsafe { collect_into_array_unchecked(&mut array.iter().cloned()) } + } +} + +impl SpecArrayClone for T { + #[inline] + fn clone(array: &[T; N]) -> [T; N] { + *array + } +} + +// The Default impls cannot be done with const generics because `[T; 0]` doesn't +// require Default to be implemented, and having different impl blocks for +// different numbers isn't supported yet. macro_rules! array_impl_default { {$n:expr, $t:ident $($ts:ident)*} => { #[stable(since = "1.4.0", feature = "array_default")] - impl Default for [T; $n] where T: Default { + #[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] + impl const Default for [T; $n] where T: ~const Default { fn default() -> [T; $n] { [$t::default(), $($ts::default()),*] } @@ -434,10 +457,518 @@ macro_rules! array_impl_default { }; {$n:expr,} => { #[stable(since = "1.4.0", feature = "array_default")] - impl Default for [T; $n] { + #[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] + impl const Default for [T; $n] { fn default() -> [T; $n] { [] } } }; } array_impl_default! {32, T T T T T T T T T T T T T T T T T T T T T T T T T T T T T T T T} + +impl [T; N] { + /// Returns an array of the same size as `self`, with function `f` applied to each element + /// in order. + /// + /// If you don't necessarily need a new fixed-size array, consider using + /// [`Iterator::map`] instead. + /// + /// + /// # Note on performance and stack usage + /// + /// Unfortunately, usages of this method are currently not always optimized + /// as well as they could be. This mainly concerns large arrays, as mapping + /// over small arrays seem to be optimized just fine. Also note that in + /// debug mode (i.e. without any optimizations), this method can use a lot + /// of stack space (a few times the size of the array or more). + /// + /// Therefore, in performance-critical code, try to avoid using this method + /// on large arrays or check the emitted code. Also try to avoid chained + /// maps (e.g. `arr.map(...).map(...)`). + /// + /// In many cases, you can instead use [`Iterator::map`] by calling `.iter()` + /// or `.into_iter()` on your array. `[T; N]::map` is only necessary if you + /// really need a new array of the same size as the result. Rust's lazy + /// iterators tend to get optimized very well. + /// + /// + /// # Examples + /// + /// ``` + /// let x = [1, 2, 3]; + /// let y = x.map(|v| v + 1); + /// assert_eq!(y, [2, 3, 4]); + /// + /// let x = [1, 2, 3]; + /// let mut temp = 0; + /// let y = x.map(|v| { temp += 1; v * temp }); + /// assert_eq!(y, [1, 4, 9]); + /// + /// let x = ["Ferris", "Bueller's", "Day", "Off"]; + /// let y = x.map(|v| v.len()); + /// assert_eq!(y, [6, 9, 3, 3]); + /// ``` + #[stable(feature = "array_map", since = "1.55.0")] + pub fn map(self, f: F) -> [U; N] + where + F: FnMut(T) -> U, + { + // SAFETY: we know for certain that this iterator will yield exactly `N` + // items. + unsafe { collect_into_array_unchecked(&mut IntoIterator::into_iter(self).map(f)) } + } + + /// A fallible function `f` applied to each element on array `self` in order to + /// return an array the same size as `self` or the first error encountered. + /// + /// The return type of this function depends on the return type of the closure. + /// If you return `Result` from the closure, you'll get a `Result<[T; N], E>`. + /// If you return `Option` from the closure, you'll get an `Option<[T; N]>`. + /// + /// # Examples + /// + /// ``` + /// #![feature(array_try_map)] + /// let a = ["1", "2", "3"]; + /// let b = a.try_map(|v| v.parse::()).unwrap().map(|v| v + 1); + /// assert_eq!(b, [2, 3, 4]); + /// + /// let a = ["1", "2a", "3"]; + /// let b = a.try_map(|v| v.parse::()); + /// assert!(b.is_err()); + /// + /// use std::num::NonZeroU32; + /// let z = [1, 2, 0, 3, 4]; + /// assert_eq!(z.try_map(NonZeroU32::new), None); + /// let a = [1, 2, 3]; + /// let b = a.try_map(NonZeroU32::new); + /// let c = b.map(|x| x.map(NonZeroU32::get)); + /// assert_eq!(c, Some(a)); + /// ``` + #[unstable(feature = "array_try_map", issue = "79711")] + pub fn try_map(self, f: F) -> ChangeOutputType + where + F: FnMut(T) -> R, + R: Try, + R::Residual: Residual<[R::Output; N]>, + { + // SAFETY: we know for certain that this iterator will yield exactly `N` + // items. + unsafe { try_collect_into_array_unchecked(&mut IntoIterator::into_iter(self).map(f)) } + } + + /// 'Zips up' two arrays into a single array of pairs. + /// + /// `zip()` returns a new array where every element is a tuple where the + /// first element comes from the first array, and the second element comes + /// from the second array. In other words, it zips two arrays together, + /// into a single one. + /// + /// # Examples + /// + /// ``` + /// #![feature(array_zip)] + /// let x = [1, 2, 3]; + /// let y = [4, 5, 6]; + /// let z = x.zip(y); + /// assert_eq!(z, [(1, 4), (2, 5), (3, 6)]); + /// ``` + #[unstable(feature = "array_zip", issue = "80094")] + pub fn zip(self, rhs: [U; N]) -> [(T, U); N] { + let mut iter = IntoIterator::into_iter(self).zip(rhs); + + // SAFETY: we know for certain that this iterator will yield exactly `N` + // items. + unsafe { collect_into_array_unchecked(&mut iter) } + } + + /// Returns a slice containing the entire array. Equivalent to `&s[..]`. + #[stable(feature = "array_as_slice", since = "1.57.0")] + #[rustc_const_stable(feature = "array_as_slice", since = "1.57.0")] + pub const fn as_slice(&self) -> &[T] { + self + } + + /// Returns a mutable slice containing the entire array. Equivalent to + /// `&mut s[..]`. + #[stable(feature = "array_as_slice", since = "1.57.0")] + pub fn as_mut_slice(&mut self) -> &mut [T] { + self + } + + /// Borrows each element and returns an array of references with the same + /// size as `self`. + /// + /// + /// # Example + /// + /// ``` + /// #![feature(array_methods)] + /// + /// let floats = [3.1, 2.7, -1.0]; + /// let float_refs: [&f64; 3] = floats.each_ref(); + /// assert_eq!(float_refs, [&3.1, &2.7, &-1.0]); + /// ``` + /// + /// This method is particularly useful if combined with other methods, like + /// [`map`](#method.map). This way, you can avoid moving the original + /// array if its elements are not [`Copy`]. + /// + /// ``` + /// #![feature(array_methods)] + /// + /// let strings = ["Ferris".to_string(), "♥".to_string(), "Rust".to_string()]; + /// let is_ascii = strings.each_ref().map(|s| s.is_ascii()); + /// assert_eq!(is_ascii, [true, false, true]); + /// + /// // We can still access the original array: it has not been moved. + /// assert_eq!(strings.len(), 3); + /// ``` + #[unstable(feature = "array_methods", issue = "76118")] + pub fn each_ref(&self) -> [&T; N] { + // SAFETY: we know for certain that this iterator will yield exactly `N` + // items. + unsafe { collect_into_array_unchecked(&mut self.iter()) } + } + + /// Borrows each element mutably and returns an array of mutable references + /// with the same size as `self`. + /// + /// + /// # Example + /// + /// ``` + /// #![feature(array_methods)] + /// + /// let mut floats = [3.1, 2.7, -1.0]; + /// let float_refs: [&mut f64; 3] = floats.each_mut(); + /// *float_refs[0] = 0.0; + /// assert_eq!(float_refs, [&mut 0.0, &mut 2.7, &mut -1.0]); + /// assert_eq!(floats, [0.0, 2.7, -1.0]); + /// ``` + #[unstable(feature = "array_methods", issue = "76118")] + pub fn each_mut(&mut self) -> [&mut T; N] { + // SAFETY: we know for certain that this iterator will yield exactly `N` + // items. + unsafe { collect_into_array_unchecked(&mut self.iter_mut()) } + } + + /// Divides one array reference into two at an index. + /// + /// The first will contain all indices from `[0, M)` (excluding + /// the index `M` itself) and the second will contain all + /// indices from `[M, N)` (excluding the index `N` itself). + /// + /// # Panics + /// + /// Panics if `M > N`. + /// + /// # Examples + /// + /// ``` + /// #![feature(split_array)] + /// + /// let v = [1, 2, 3, 4, 5, 6]; + /// + /// { + /// let (left, right) = v.split_array_ref::<0>(); + /// assert_eq!(left, &[]); + /// assert_eq!(right, &[1, 2, 3, 4, 5, 6]); + /// } + /// + /// { + /// let (left, right) = v.split_array_ref::<2>(); + /// assert_eq!(left, &[1, 2]); + /// assert_eq!(right, &[3, 4, 5, 6]); + /// } + /// + /// { + /// let (left, right) = v.split_array_ref::<6>(); + /// assert_eq!(left, &[1, 2, 3, 4, 5, 6]); + /// assert_eq!(right, &[]); + /// } + /// ``` + #[unstable( + feature = "split_array", + reason = "return type should have array as 2nd element", + issue = "90091" + )] + #[inline] + pub fn split_array_ref(&self) -> (&[T; M], &[T]) { + (&self[..]).split_array_ref::() + } + + /// Divides one mutable array reference into two at an index. + /// + /// The first will contain all indices from `[0, M)` (excluding + /// the index `M` itself) and the second will contain all + /// indices from `[M, N)` (excluding the index `N` itself). + /// + /// # Panics + /// + /// Panics if `M > N`. + /// + /// # Examples + /// + /// ``` + /// #![feature(split_array)] + /// + /// let mut v = [1, 0, 3, 0, 5, 6]; + /// let (left, right) = v.split_array_mut::<2>(); + /// assert_eq!(left, &mut [1, 0][..]); + /// assert_eq!(right, &mut [3, 0, 5, 6]); + /// left[1] = 2; + /// right[1] = 4; + /// assert_eq!(v, [1, 2, 3, 4, 5, 6]); + /// ``` + #[unstable( + feature = "split_array", + reason = "return type should have array as 2nd element", + issue = "90091" + )] + #[inline] + pub fn split_array_mut(&mut self) -> (&mut [T; M], &mut [T]) { + (&mut self[..]).split_array_mut::() + } + + /// Divides one array reference into two at an index from the end. + /// + /// The first will contain all indices from `[0, N - M)` (excluding + /// the index `N - M` itself) and the second will contain all + /// indices from `[N - M, N)` (excluding the index `N` itself). + /// + /// # Panics + /// + /// Panics if `M > N`. + /// + /// # Examples + /// + /// ``` + /// #![feature(split_array)] + /// + /// let v = [1, 2, 3, 4, 5, 6]; + /// + /// { + /// let (left, right) = v.rsplit_array_ref::<0>(); + /// assert_eq!(left, &[1, 2, 3, 4, 5, 6]); + /// assert_eq!(right, &[]); + /// } + /// + /// { + /// let (left, right) = v.rsplit_array_ref::<2>(); + /// assert_eq!(left, &[1, 2, 3, 4]); + /// assert_eq!(right, &[5, 6]); + /// } + /// + /// { + /// let (left, right) = v.rsplit_array_ref::<6>(); + /// assert_eq!(left, &[]); + /// assert_eq!(right, &[1, 2, 3, 4, 5, 6]); + /// } + /// ``` + #[unstable( + feature = "split_array", + reason = "return type should have array as 2nd element", + issue = "90091" + )] + #[inline] + pub fn rsplit_array_ref(&self) -> (&[T], &[T; M]) { + (&self[..]).rsplit_array_ref::() + } + + /// Divides one mutable array reference into two at an index from the end. + /// + /// The first will contain all indices from `[0, N - M)` (excluding + /// the index `N - M` itself) and the second will contain all + /// indices from `[N - M, N)` (excluding the index `N` itself). + /// + /// # Panics + /// + /// Panics if `M > N`. + /// + /// # Examples + /// + /// ``` + /// #![feature(split_array)] + /// + /// let mut v = [1, 0, 3, 0, 5, 6]; + /// let (left, right) = v.rsplit_array_mut::<4>(); + /// assert_eq!(left, &mut [1, 0]); + /// assert_eq!(right, &mut [3, 0, 5, 6][..]); + /// left[1] = 2; + /// right[1] = 4; + /// assert_eq!(v, [1, 2, 3, 4, 5, 6]); + /// ``` + #[unstable( + feature = "split_array", + reason = "return type should have array as 2nd element", + issue = "90091" + )] + #[inline] + pub fn rsplit_array_mut(&mut self) -> (&mut [T], &mut [T; M]) { + (&mut self[..]).rsplit_array_mut::() + } +} + +/// Pulls `N` items from `iter` and returns them as an array. If the iterator +/// yields fewer than `N` items, this function exhibits undefined behavior. +/// +/// See [`try_collect_into_array`] for more information. +/// +/// +/// # Safety +/// +/// It is up to the caller to guarantee that `iter` yields at least `N` items. +/// Violating this condition causes undefined behavior. +unsafe fn try_collect_into_array_unchecked(iter: &mut I) -> R::TryType +where + // Note: `TrustedLen` here is somewhat of an experiment. This is just an + // internal function, so feel free to remove if this bound turns out to be a + // bad idea. In that case, remember to also remove the lower bound + // `debug_assert!` below! + I: Iterator + TrustedLen, + I::Item: Try, + R: Residual<[T; N]>, +{ + debug_assert!(N <= iter.size_hint().1.unwrap_or(usize::MAX)); + debug_assert!(N <= iter.size_hint().0); + + // SAFETY: covered by the function contract. + unsafe { try_collect_into_array(iter).unwrap_unchecked() } +} + +// Infallible version of `try_collect_into_array_unchecked`. +unsafe fn collect_into_array_unchecked(iter: &mut I) -> [I::Item; N] +where + I: Iterator + TrustedLen, +{ + let mut map = iter.map(NeverShortCircuit); + + // SAFETY: The same safety considerations w.r.t. the iterator length + // apply for `try_collect_into_array_unchecked` as for + // `collect_into_array_unchecked` + match unsafe { try_collect_into_array_unchecked(&mut map) } { + NeverShortCircuit(array) => array, + } +} + +/// Pulls `N` items from `iter` and returns them as an array. If the iterator +/// yields fewer than `N` items, `Err` is returned containing an iterator over +/// the already yielded items. +/// +/// Since the iterator is passed as a mutable reference and this function calls +/// `next` at most `N` times, the iterator can still be used afterwards to +/// retrieve the remaining items. +/// +/// If `iter.next()` panicks, all items already yielded by the iterator are +/// dropped. +#[inline] +fn try_collect_into_array( + iter: &mut I, +) -> Result> +where + I: Iterator, + I::Item: Try, + R: Residual<[T; N]>, +{ + if N == 0 { + // SAFETY: An empty array is always inhabited and has no validity invariants. + return Ok(Try::from_output(unsafe { mem::zeroed() })); + } + + let mut array = MaybeUninit::uninit_array::(); + let mut guard = Guard { array_mut: &mut array, initialized: 0 }; + + for _ in 0..N { + match iter.next() { + Some(item_rslt) => { + let item = match item_rslt.branch() { + ControlFlow::Break(r) => { + return Ok(FromResidual::from_residual(r)); + } + ControlFlow::Continue(elem) => elem, + }; + + // SAFETY: `guard.initialized` starts at 0, which means push can be called + // at most N times, which this loop does. + unsafe { + guard.push_unchecked(item); + } + } + None => { + let alive = 0..guard.initialized; + mem::forget(guard); + // SAFETY: `array` was initialized with exactly `initialized` + // number of elements. + return Err(unsafe { IntoIter::new_unchecked(array, alive) }); + } + } + } + + mem::forget(guard); + // SAFETY: All elements of the array were populated in the loop above. + let output = unsafe { array.transpose().assume_init() }; + Ok(Try::from_output(output)) +} + +/// Panic guard for incremental initialization of arrays. +/// +/// Disarm the guard with `mem::forget` once the array has been initialized. +/// +/// # Safety +/// +/// All write accesses to this structure are unsafe and must maintain a correct +/// count of `initialized` elements. +/// +/// To minimize indirection fields are still pub but callers should at least use +/// `push_unchecked` to signal that something unsafe is going on. +pub(crate) struct Guard<'a, T, const N: usize> { + /// The array to be initialized. + pub array_mut: &'a mut [MaybeUninit; N], + /// The number of items that have been initialized so far. + pub initialized: usize, +} + +impl Guard<'_, T, N> { + /// Adds an item to the array and updates the initialized item counter. + /// + /// # Safety + /// + /// No more than N elements must be initialized. + #[inline] + pub unsafe fn push_unchecked(&mut self, item: T) { + // SAFETY: If `initialized` was correct before and the caller does not + // invoke this method more than N times then writes will be in-bounds + // and slots will not be initialized more than once. + unsafe { + self.array_mut.get_unchecked_mut(self.initialized).write(item); + self.initialized = self.initialized.unchecked_add(1); + } + } +} + +impl Drop for Guard<'_, T, N> { + fn drop(&mut self) { + debug_assert!(self.initialized <= N); + + // SAFETY: this slice will contain only initialized objects. + unsafe { + crate::ptr::drop_in_place(MaybeUninit::slice_assume_init_mut( + &mut self.array_mut.get_unchecked_mut(..self.initialized), + )); + } + } +} + +/// Returns the next chunk of `N` items from the iterator or errors with an +/// iterator over the remainder. Used for `Iterator::next_chunk`. +#[inline] +pub(crate) fn iter_next_chunk( + iter: &mut I, +) -> Result<[I::Item; N], IntoIter> +where + I: Iterator, +{ + let mut map = iter.map(NeverShortCircuit); + try_collect_into_array(&mut map).map(|NeverShortCircuit(arr)| arr) +} diff --git a/crux-mir/lib/core/src/ascii.rs b/crux-mir/lib/core/src/ascii.rs index e78dfd1ed..8a4cb78cc 100644 --- a/crux-mir/lib/core/src/ascii.rs +++ b/crux-mir/lib/core/src/ascii.rs @@ -6,8 +6,6 @@ //! //! The [`escape_default`] function provides an iterator over the bytes of an //! escaped version of the character given. -//! -//! [`escape_default`]: fn.escape_default.html #![stable(feature = "core_ascii", since = "1.26.0")] @@ -20,12 +18,11 @@ use crate::str::from_utf8_unchecked; /// /// This `struct` is created by the [`escape_default`] function. See its /// documentation for more. -/// -/// [`escape_default`]: fn.escape_default.html +#[must_use = "iterators are lazy and do nothing unless consumed"] #[stable(feature = "rust1", since = "1.0.0")] #[derive(Clone)] pub struct EscapeDefault { - range: Range, + range: Range, data: [u8; 4], } @@ -101,24 +98,22 @@ pub fn escape_default(c: u8) -> EscapeDefault { b'\'' => ([b'\\', b'\'', 0, 0], 2), b'"' => ([b'\\', b'"', 0, 0], 2), b'\x20'..=b'\x7e' => ([c, 0, 0, 0], 1), - _ => ([b'\\', b'x', hexify(c >> 4), hexify(c & 0xf)], 4), + _ => { + let hex_digits: &[u8; 16] = b"0123456789abcdef"; + ([b'\\', b'x', hex_digits[(c >> 4) as usize], hex_digits[(c & 0xf) as usize]], 4) + } }; return EscapeDefault { range: 0..len, data }; - - fn hexify(b: u8) -> u8 { - match b { - 0..=9 => b'0' + b, - _ => b'a' + b - 10, - } - } } #[stable(feature = "rust1", since = "1.0.0")] impl Iterator for EscapeDefault { type Item = u8; + + #[inline] fn next(&mut self) -> Option { - self.range.next().map(|i| self.data[i]) + self.range.next().map(|i| self.data[i as usize]) } fn size_hint(&self) -> (usize, Option) { self.range.size_hint() @@ -130,7 +125,7 @@ impl Iterator for EscapeDefault { #[stable(feature = "rust1", since = "1.0.0")] impl DoubleEndedIterator for EscapeDefault { fn next_back(&mut self) -> Option { - self.range.next_back().map(|i| self.data[i]) + self.range.next_back().map(|i| self.data[i as usize]) } } #[stable(feature = "rust1", since = "1.0.0")] @@ -142,13 +137,15 @@ impl FusedIterator for EscapeDefault {} impl fmt::Display for EscapeDefault { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // SAFETY: ok because `escape_default` created only valid utf-8 data - f.write_str(unsafe { from_utf8_unchecked(&self.data[self.range.clone()]) }) + f.write_str(unsafe { + from_utf8_unchecked(&self.data[(self.range.start as usize)..(self.range.end as usize)]) + }) } } #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for EscapeDefault { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("EscapeDefault { .. }") + f.debug_struct("EscapeDefault").finish_non_exhaustive() } } diff --git a/crux-mir/lib/core/src/asserting.rs b/crux-mir/lib/core/src/asserting.rs new file mode 100644 index 000000000..212b637d3 --- /dev/null +++ b/crux-mir/lib/core/src/asserting.rs @@ -0,0 +1,109 @@ +// Contains the machinery necessary to print useful `assert!` messages. Not intended for public +// usage, not even nightly use-cases. +// +// Based on https://github.com/dtolnay/case-studies/tree/master/autoref-specialization. When +// 'specialization' is robust enough (5 years? 10 years? Never?), `Capture` can be specialized +// to [Printable]. + +#![allow(missing_debug_implementations)] +#![doc(hidden)] +#![unstable(feature = "generic_assert_internals", issue = "44838")] + +use crate::{ + fmt::{Debug, Formatter}, + marker::PhantomData, +}; + +// ***** TryCapture - Generic ***** + +/// Marker used by [Capture] +#[unstable(feature = "generic_assert_internals", issue = "44838")] +pub struct TryCaptureWithoutDebug; + +/// Catches an arbitrary `E` and modifies `to` accordingly +#[unstable(feature = "generic_assert_internals", issue = "44838")] +pub trait TryCaptureGeneric { + /// Similar to [TryCapturePrintable] but generic to any `E`. + fn try_capture(&self, to: &mut Capture); +} + +impl TryCaptureGeneric for &Wrapper<&E> { + #[inline] + fn try_capture(&self, _: &mut Capture) {} +} + +impl Debug for Capture { + fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), core::fmt::Error> { + f.write_str("N/A") + } +} + +// ***** TryCapture - Printable ***** + +/// Marker used by [Capture] +#[unstable(feature = "generic_assert_internals", issue = "44838")] +pub struct TryCaptureWithDebug; + +/// Catches an arbitrary `E: Printable` and modifies `to` accordingly +#[unstable(feature = "generic_assert_internals", issue = "44838")] +pub trait TryCapturePrintable { + /// Similar as [TryCaptureGeneric] but specialized to any `E: Printable`. + fn try_capture(&self, to: &mut Capture); +} + +impl TryCapturePrintable for Wrapper<&E> +where + E: Printable, +{ + #[inline] + fn try_capture(&self, to: &mut Capture) { + to.elem = Some(*self.0); + } +} + +impl Debug for Capture +where + E: Printable, +{ + fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), core::fmt::Error> { + match self.elem { + None => f.write_str("N/A"), + Some(ref value) => Debug::fmt(value, f), + } + } +} + +// ***** Others ***** + +/// All possible captured `assert!` elements +/// +/// # Types +/// +/// * `E`: **E**lement that is going to be displayed. +/// * `M`: **M**arker used to differentiate [Capture]s in regards to [Debug]. +#[unstable(feature = "generic_assert_internals", issue = "44838")] +pub struct Capture { + // If None, then `E` does not implements [Printable] or `E` wasn't evaluated (`assert!( ... )` + // short-circuited). + // + // If Some, then `E` implements [Printable] and was evaluated. + pub elem: Option, + phantom: PhantomData, +} + +impl Capture { + #[inline] + pub const fn new() -> Self { + Self { elem: None, phantom: PhantomData } + } +} + +/// Necessary for the implementations of `TryCapture*` +#[unstable(feature = "generic_assert_internals", issue = "44838")] +pub struct Wrapper(pub T); + +/// Tells which elements can be copied and displayed +#[unstable(feature = "generic_assert_internals", issue = "44838")] +pub trait Printable: Copy + Debug {} + +impl Printable for T where T: Copy + Debug {} diff --git a/crux-mir/lib/core/src/async_iter/async_iter.rs b/crux-mir/lib/core/src/async_iter/async_iter.rs new file mode 100644 index 000000000..12a47f9fc --- /dev/null +++ b/crux-mir/lib/core/src/async_iter/async_iter.rs @@ -0,0 +1,111 @@ +use crate::ops::DerefMut; +use crate::pin::Pin; +use crate::task::{Context, Poll}; + +/// A trait for dealing with asynchronous iterators. +/// +/// This is the main async iterator trait. For more about the concept of async iterators +/// generally, please see the [module-level documentation]. In particular, you +/// may want to know how to [implement `AsyncIterator`][impl]. +/// +/// [module-level documentation]: index.html +/// [impl]: index.html#implementing-async-iterator +#[unstable(feature = "async_iterator", issue = "79024")] +#[must_use = "async iterators do nothing unless polled"] +#[doc(alias = "Stream")] +pub trait AsyncIterator { + /// The type of items yielded by the async iterator. + type Item; + + /// Attempt to pull out the next value of this async iterator, registering the + /// current task for wakeup if the value is not yet available, and returning + /// `None` if the async iterator is exhausted. + /// + /// # Return value + /// + /// There are several possible return values, each indicating a distinct + /// async iterator state: + /// + /// - `Poll::Pending` means that this async iterator's next value is not ready + /// yet. Implementations will ensure that the current task will be notified + /// when the next value may be ready. + /// + /// - `Poll::Ready(Some(val))` means that the async iterator has successfully + /// produced a value, `val`, and may produce further values on subsequent + /// `poll_next` calls. + /// + /// - `Poll::Ready(None)` means that the async iterator has terminated, and + /// `poll_next` should not be invoked again. + /// + /// # Panics + /// + /// Once an async iterator has finished (returned `Ready(None)` from `poll_next`), calling its + /// `poll_next` method again may panic, block forever, or cause other kinds of + /// problems; the `AsyncIterator` trait places no requirements on the effects of + /// such a call. However, as the `poll_next` method is not marked `unsafe`, + /// Rust's usual rules apply: calls must never cause undefined behavior + /// (memory corruption, incorrect use of `unsafe` functions, or the like), + /// regardless of the async iterator's state. + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll>; + + /// Returns the bounds on the remaining length of the async iterator. + /// + /// Specifically, `size_hint()` returns a tuple where the first element + /// is the lower bound, and the second element is the upper bound. + /// + /// The second half of the tuple that is returned is an [Option]<[usize]>. + /// A [`None`] here means that either there is no known upper bound, or the + /// upper bound is larger than [`usize`]. + /// + /// # Implementation notes + /// + /// It is not enforced that an async iterator implementation yields the declared + /// number of elements. A buggy async iterator may yield less than the lower bound + /// or more than the upper bound of elements. + /// + /// `size_hint()` is primarily intended to be used for optimizations such as + /// reserving space for the elements of the async iterator, but must not be + /// trusted to e.g., omit bounds checks in unsafe code. An incorrect + /// implementation of `size_hint()` should not lead to memory safety + /// violations. + /// + /// That said, the implementation should provide a correct estimation, + /// because otherwise it would be a violation of the trait's protocol. + /// + /// The default implementation returns (0, [None]) which is correct for any + /// async iterator. + #[inline] + fn size_hint(&self) -> (usize, Option) { + (0, None) + } +} + +#[unstable(feature = "async_iterator", issue = "79024")] +impl AsyncIterator for &mut S { + type Item = S::Item; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + S::poll_next(Pin::new(&mut **self), cx) + } + + fn size_hint(&self) -> (usize, Option) { + (**self).size_hint() + } +} + +#[unstable(feature = "async_iterator", issue = "79024")] +impl